From f09a5cb9179596e1996f4642dbb84480c3b282e3 Mon Sep 17 00:00:00 2001
From: Kai Sasaki <lewuathe@gmail.com>
Date: Mon, 23 Oct 2023 15:49:39 +0900
Subject: [PATCH 1/4] Squashed 'thirdparty/llvm-project/' changes from
 4df74963ea0f6..5c3ed392fcda4
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

5c3ed392fcda4 [mlir][linalg] Enable CollapseLinalgDimensions to collapse linalg::CopyOp (#68526)
86bc486785ee2 [BOLT][RISCV] Use target features from object file (#69836)
3eed23d31caab [clang][dataflow] Remove `DataflowAnalysisContext::flowConditionIsTautology()`. (#69601)
0374bbba4c455 [clang-format] Add a new style for the clang-format source code (#69814)
cdf713cdb9dae [mlir][Bazel] Add missing dependency.
2fe9a342a69d2 [mlir][SCF] Use getResult() instead of static_cast<Value>().
0b3f6ff3c4ef3 [RISCV] Disable hasAllNBitUsers for vector types.
859b5c732545d [TextAPI] Use std::remove_reference_t (NFC)
66a797102de4e [lldb] Use llvm::is_contained (NFC)
a5dca533bdc32 Use llvm::count (NFC)
1e43975652e54 [TableGen][NFC] Remove MultiClass argument and Scoper in QualifyName (#69297)
05b5188c12577 [Clang][RISCV] Support CSRs in clobbered registers of inline assembly (#67646)
366ffbaf627d0 [C++20] [Modules] [Driver] Don't enable -fdelayed-template-parsing by default on windows with C++20 (#69431)
86bb713142c3c [OpenMP][FIX] Enlarge thread state array, improve test and add second
4d80eff8615f3 [mlir][bufferization] Ownership-based deallocation: Allow manual (de)allocs (#68648)
7a3db658d923f [llvm-profgen] More tweaks to warnings (#68608)
a24418375a707 [CodeLayout] cache-directed sort: limit max chain size (#69039)
8511ade505588 [LegacyPM] Remove LowerExpectIntrinsicPass
efb0e9c0bda62 [LLDB] Update breakpoint-command.test to use string instead of number. (#69796)
9c5a5a421d403 [llvm] Stop including llvm/ADT/iterator_range.h (NFC)
6833a3808f7ba [mlir][DeadCodeAnalysis] Don't Require `RegionBranchTerminatorOpInterface` in `visitRegionTerminator()` (#69043)
508a697acd180 [JITLink] Allow multiple relocations at same offset in EHFrameEdgeFixer (#68252)
9f3b06d8bea1e [OpenMP][FIX] Fix memset oversight to partially unblock test
904c5b484ebf0 Apply clang-tidy fixes for misc-include-cleaner in AST.cpp (NFC)
d2d93a3365c91 Apply clang-tidy fixes for misc-include-cleaner in AST.cpp (NFC)
f58eabc60e406 Apply clang-tidy fixes for misc-include-cleaner in MLIRGen.cpp (NFC)
cd71a2c5fae80 Apply clang-tidy fixes for misc-include-cleaner in Dialect.cpp (NFC)
2f20e0ee7331c Apply clang-tidy fixes for misc-include-cleaner in toyc.cpp (NFC)
3e86cc4b864f6 [clang-format][NFC] Simplify the logic in a return statement
f3ff0a67be46f [OpenMP][FIX] Ensure test runs correct with (at least) 2 threads
747e0d9f0aadb [compiler-rt] Use std::clamp (NFC)
3af0ff99b1bf9 [llvm] Stop including llvm/ADT/DepthFirstIterator.h (NFC)
aaa5f34b6130c [lldb] Remove an unused using decl (NFC)
935d8e12e06ff [llvm] Stop including llvm/ADT/StringMap.h (NFC)
461918e290281 [CodeGen][Remarks] Add the function name to the stack size remark (#69346)
a9136f0ad94bf [Utils] Use std::remove_pointer_t (NFC)
6e18003a23aaa [llvm] Use llvm::any_of (NFC)
5a98dd673477a [mlir] Remove an extraneous typename (NFC)
3b7bfeb483e78 [llvm] Stop including llvm/ADT/SmallString.h (NFC)
ce8b8b1639bc9 [mlir][minimal-opt] Fix typo
c3008842bf19e [clangd] Show alignment for records and fields decls (#67213)
4f56d47d050e3 [VPlan] Make ExpandedSCEVs argument const (NFC).
0c8e5be6fa085 [VPlan] Simplify redundant trunc (zext A) pairs to A.
4aeb7a0f5e35b [gn build] Port d2a46e6480f3
d2a46e6480f35 [libc++][PSTL] Implement std::move
3d40411ee8895 [AMDGPU] Set size to all SOP pseudos (#69756)
a63dc79d11391 [Clang][OHOS] Keep ARM ABI selection logic in sync between Clang and LLVM (#68656)
1f12223cf411b [llvm] Stop including Endian.h (NFC)
0b7ae41b23fc0 [LIT] Print discovered tests and percentages (#66057) (#69831)
00c8da6159239 [docs] Fix suggested darker command in coding standards (#69860)
5aee156b2ad2b Reland: "[mlir][index][spirv] Add conversion for index to spirv" (#69790)
5e458f5aefb9b Apply clang-tidy fixes for llvm-qualified-auto in IRNumbering.cpp (NFC)
a2e572099c672 Apply clang-tidy fixes for llvm-qualified-auto in CallGraph.cpp (NFC)
6db6e7e72edc1 Apply clang-tidy fixes for llvm-qualified-auto in LowerToLLVM.cpp (NFC)
c8f09c19b91ce Apply clang-tidy fixes for llvm-qualified-auto in LowerToLLVM.cpp (NFC)
74ca07295f322 [lldb] improve dwo path in missing dwo error when relative (#69783)
6250afe37fd1f [Github] Fetch an additional commit for docs CI on PRs
196a09f477acd [mlir][doc] Add basic doc for extraTraitClassDeclaration.
01ea528145191 [mlir][doc] Include ml_program passes in passes doc
5ce45e9d8e9d3 [Github] Add lldb docs step to Github docs action (#69832)
40a5044af8b11 [Github] Add clang-tools-extra docs to CI (#69827)
87dac9f1682c3 [OpenMP] Rewrite test to check the correct (CPU) result
d3921e4670057 [OpenMP] Basic BumpAllocator for (AMD)GPUs (#69806)
d571af7f62749 [OpenMP][FIX] Ensure thread states do not crash on the GPU
ca01f2af78968 [LV] Enforce order of reductions with intermediate stores in VPlan (NFC)
241c290ad73f5 Reland [LLD] [COFF] Don't try to detect MSVC installations in mingw mode
ba87fba80632b [Attributor] Ignore different kernels for kernel lifetime objects
bb96093ca620a [Attributor][NFC] Precommit test
ed0bb9476ce0e [lldb] Update qRegisterInfo docs to recommend target.xml (#69853)
44d4f975e4f61 [RISCV] Replace RISCV -> RISC-V in comments. NFC
f9906508bc4f0 [analyzer][NFC] Substitute operator() with lambda in StreamChecker
7d7e4d20b077c [Windows] Add git-clang-format wrapper bat file (#69228)
c2f02e3bd6a5b Clang: Define macro _MIPS_SPFPSET
9322216021f1d [clang-tidy][DOC] Fix 'table cell spanning'
764c7ae949a50 [Github] Remove CMake options from docs CI resetting defaults
09b02f5d569ad [Driver][NetBSD][NFC] Some cleaning up
386f3903910aa [MachineBasicBlock] Fix SlotIndexUpdater for insertion order (#69424)
af447dde5e723 [Serialization] Use llvm::is_contained (NFC)
7552b4d9ee7e5 [polly] Use llvm::erase_value (NFC)
dc554bd06f751 [flang] Use llvm::any_of (NFC)
e5ef5b9e32b13 [DebugInfo] Use llvm::erase_value (NFC)
8822eaae0f459 [compiler-rt] Switch LLD specific tests to a more precise option (#69781)
4aae5387a874a [llvm][CMake] Respect LIBCXX_HARDENING_MODE on command-line (#68541)
d36ddaa665823 [compiler-rt] Fix a warning
a3937c46d0eaa Fix typos in Debug.h (#68761)
a157a82b1e7a8 [mlir] Avoid including <alloca.h> on DragonFly
7df9b0c5b33e6 Fix typos and formatting in GettingStarted.md (#68537)
826c93f96efd8 [AST] Use explicit type erasure in TypeSourceInfo constructor (#68435)
cb5612c99b0b8 Add IR name to -print-pass-numbers output
d342843c2baa0 [MLIR][python bindings][fix] invalidate ops after PassManager run
c000bd4e40166 Revert "[mlir] Silence a few -Wunused-but-set-parameter warnings" (#68667)
1cea309b7e225 [OpenMP][NFC] Move DebugKind to make it reusable from the host
499fb1b8d88aa [Attributor][FIX] Interposable constants cannot be propagated
0a0c23b9ce004 [OpenMPOpt][FIX] Properly track changes to NestedParallelism
3b5d59ab55f88 [lldb][test] Turn ObjC string literals to C-style literals (NFC) (#69793)
9b381c6a06958 [Github] Fetch number of commits in PR for docs action (#69763)
14bc11a651971 [clang][dataflow]Use cast_or_null instead of cast to prevent crash (#68510)
bdc3e6cb45203 [MLIR][python bindings] invalidate ops after PassManager run (#69746)
7e3d1103e6e7a [InstCombine] optimize powi(X,Y)/X with Ofast (#67236)
6cbcbecf53a83 [InstCombine] Precommit tests for PR67216
7c0a589af973b [CI] Set minimal permission on libcxx-check-generated-file workflow (#69737)
e9fa1fdec9b1e [mlir][sparse] support CSR/BSR conversion (#69800)
4e0c6d30576ab Fix build warning caused by mixed signed/unsigned compare (#69797)
972709a74a0c4 [RISCV][GISel] Minor refactoring of RISCVCallReturnHandler and RISCVIncomingValueHandler to match other targets (#69757)
6456e0bbbbd7c [mlir][sparse] implement sparse_tensor.crd_translate operation (#69653)
6243d7d28b923 [mlir][sparse] fix stack overflow due to memref.alloca in loops (#69786)
cb472fbd47b7e Update SimplifyIndVar.cpp (#69760)
852bac4439010 [VPlan] Support scalable vectors in outer-loop vectorization
af25304319066 [Workflow] make code-format-helper.py mypy-safe (NFC) (#69691)
a7119a195712e [OpenMP][mlir] Add translation for `if` in `omp.teams` (#69404)
28ae42e662515 [GISel] Add LookThroughInstrs for getIConstantVRegVal and getIConstan… (#68327)
dbc1b71cf454d [RISCV][llvm-mca] Vector Unit Stride Loads and stores use EEW and EMU… (#69409)
51835dfadb36f [MSVC] fix the build (#69634)
a4323586fcbb2 [-Wunsafe-buffer-usage] Add AST info to the unclaimed DRE debug notes for analysis
a9694043c9b86 [mlir][linalg] regionBuilder for transpose, broadcast (#69742)
7ba99fd75e0c8 [InstCombine][NFC] Precommit tests for https://reviews.llvm.org/D149918
a67ae8c0fd301 [LLD] [COFF] Add a separate option for allowing duplicate weak symbols (#68077)
c5dacb6d7a540 [gn build] Port 538b7ba2aacd
538b7ba2aacd6 Reland [clang] [unittest] Add a test for Generic_GCC::GCCVersion::Parse (#69078)
09ec0004eee2d [Modules] textual headers in submodules never resolve their `use`s (#69651)
34a3fb9f62f93 [Libomptarget][NFC] Remove use of VLA in the AMDGPU plugin (#69761)
1d4601a1ef84e [Driver] Corrections for linker flags passed with relocatable linking on OpenBSD (#67254)
a95d4b7950181 [clang-format] Annotate do while while
48962383ad1af [mlir][sparse] tiny cleanup making local 'using' explicit (#69740)
9399094586aa8 [clang-tidy] modernize-avoid-bind only return for non-void function (#69207)
e774482c4c410 Fixed typo in GPU libm device library warning (#69752)
35e4a28f04209 Fix MLIR gcc7 build: ambiguous overload from user conversion
47f0f8ca47408 [WebAssembly] Add exp10 libcall signatures (#69661)
20fc8e8df20e1 [RISCV][InsertVSETVLI] Make VL preserving vsetvli emission more explicit [nfc]
cfdafc1e7035d [RISCV][GISel] Support G_PTRTOINT and G_INTTOPTR (#69542)
f533e8ca9f0b3 Recommit "[RISCV][GISel] Disable call lowering for integers larger than 2*XLen. (#69144)"
b4729f79ed9b4 [RISCV] Use LMUL=1 for vmv_s_x_vl with non-undef passthru (#66659)
78a8878284485 Revert "[RISCV][GISel] Disable call lowering for integers larger than 2*XLen. (#69144)"
9189822f2c2ec [LVI] Handle icmp of ashr. (#68010)
3a4b0e9356748 [RISCV][GISel] Disable call lowering for integers larger than 2*XLen. (#69144)
734b016b66c7b Revert "[SLP] Improve gather tree nodes matching when users are PHIs. (#69392)"
71bdd2c2380d3 mlir/lib/Dialect/GPU/Transforms: improve context management in SerializeToCubin (#65779)
6a2f68beb7c9c clarify tensor.pad docs for low/high config
7aa89ee8be725 Remove accidental merge conflict marker; NFC
07ba4998171f6 [libc++][Android] Disable Android ABI list checking (#69666)
51bfeff0e4b07 [tsan][go]: add atomic or/and functions (#65695)
8b02ceb133341 Revert "[mlir][index][spirv] Add conversion for index to spirv (#68085)"
c6f0f8810db5c Workaround for MSVC ARM64 build performance regression (#65215)
2483763f1aeba [RISCV][CostModel] Recommit VPIntrinsics have same cost as their non-vp counterparts (#68752)
84a3aadf0f248 Diagnose use of VLAs in C++ by default
7b9fa2146e977 [cmake] Option to create Ninja job pools depending on available resources (#65274)
b081da5fab053 clang-linker-wrapper/LinkerWrapperOpts.td: "--sysroot" => "--sysroot=" (#65313)
e40b2d40a4404 [RISCV] Use range-based for loops in RISCVOptWInstrs. NFC (#69647)
f2801a5f5d4d8 [Clang][OpenMP] Check if value is contained in array, not if it's contained in the first element (#69462)
bde2e6981e784 [llvm] Use XMACROS for MachO platforms. (#69262)
7fc3d32d531df [mlir][sparse] update COO buffer reader doc (#69664)
1afb313b26851 [clang][modules] Use file name as requested (#68957)
cb62f67088aaf Revert "[Intrinsics][ObjC] Mark objc_retain and friends as thisreturn."
e78e1f519b7b7 Revert "[flang] Put ISO_Fortran_binding.h where it can be easily used (#69121)"
f08fe1f1dd64d [MLIR][Presburger] Implement matrix inverse (#67382)
080fb3e5b73bc [lit] Clean up internal shell parse errors with ScriptFatal (#68496)
6795bfce4ded6 [BOLT][RISCV] Handle CIE's produced by GNU as (#69578)
3c07a21683c3c [mlir][index][spirv] Add conversion for index to spirv (#68085)
1bc42666d2fd2 Change owner of Hexagon backend
24633eac38d46 [Peephole] Check instructions from CopyMIs are still COPY (#69511)
17baba9fa2728 [llvm][AArch64][Assembly] Implement support to read/write FPMR (#69618)
6d53fdeab40fe [libc][NFC] Attempt to deflake gettimeofday_test. (#69719)
ab76a7c568868 [gn build] Port 9a5c6f1760a3
9a5c6f1760a35 [clang-tidy]Add new check bugprone-casting-through-void (#69465)
759cc2588ec84 [gn build] Port 639a0986f3a3
639a0986f3a34 [libc++] mdspan - implement layout_stride (#69650)
2a32afddf5594 [lldb] Remove more references to lldb-vscode (#69696)
f5043f46c089d Revert "Diagnose use of VLAs in C++ by default"
7339c0f782d5c Diagnose use of VLAs in C++ by default
2ec7bba77bf57 Recommit "[VPlan] Insert Trunc/Exts for reductions directly in VPlan."
75d6f508f8602 [Interpreter] Add initialization of array members (#66172)
97e06a0d83a55 [GVN] Add tests for captured-before analysis (NFC)
3bfd1f0913691 [AA] Make LI and EphValues option in EarliestEscapeInfo (NFC)
b6ecdf0a6b8f3 [AMDGPU] Segregate 16-bit fix-sgpr-copies tests. (#69353)
bb826951dcf35 [lldb][AArch64] Add release notes and documentation for SME (#66767)
11a7195b2fea6 Revert "[LIT] Print discovered tests and percentages (#66057)" (#69715)
47747da6340b0 [clang] Handle templated operators with reversed arguments (#69595)
ba8565fbcb975 [LIT] Print discovered tests and percentages (#66057)
49af6502c6dcb [run-clang-tidy] Accept export directory if PyYAML is not installed (#69700)
40ba0ca5213d9 [mlir][SCF] Fix memory leak in LoopLikeSCFOpsTest.cpp
783b4d91c73c9 [mlir][tosa] Check for 0-ranked-tensors during fold (#68512)
a4803d8a77c6a [mlir][Tosa] Fix Clamp verifier to handle quantized types.
7c7896b1beeb1 [MemCpyOpt] Remove unnecessary typed pointer handling (NFC)
e24ac11fa76ab [lld][NFC] Remove unnecessary else statements. (#69451)
1c27899e24381 [mlir][SCF] Pass result of getAsOpFoldResult to getBoundedTileSize.
a3238969e5621 [IR] Fix nested constant to instruction conversion (#69682)
7c15dd60ec454 [clang-format] Add space in placement new expression
b99f7e6954691 [clangd] Don't run slow clang-tidy checks by default
9ccf01fbf711f [mlir][transform] Support for multiple top-level transform ops (#69615)
e45f6e93d0b90 [flang][hlfir] Make the parent type the first component (#69348)
1abc28fea0a2f [NFC][LV] Add test for vectorizing fmuladd with another call (#68601)
53edf45233ad7 [PowerPC] Remove HTM instruction from P10 SchedModel (#69579)
2ef370b7716b3 [flang][openmp] Update copyHostAssociateVar to use hlfir.assign for HLFIR (#69441)
08d7d1ef9a104 [mlir][Bazel] Add missing dependencies after aa0208d1bc52e45dc0032f41e58b50d3134d1089
dde7b80ed071d [mlir][Tosa] fix fp16/bf16 support for Clamp min/max attributes (#69192)
2e4161df3988e [mlir][ArmSME] Name arguments of SME intrinsics (NFC) (#69608)
b9dae2fa22d3d [Clang][SVE2.1] Add builtins for svrevd
54881c9225e61 [flang] Remove test from #69121 to fix gcc build with gcc < 10.0
c9b17af22835a [Driver][DragonFly][NFC] Some cleaning up
5db75d74a10d6 [BOLT] Filter itrace from perf script mmap & task events (#69585)
2ad41fa73688c [FunctionAttrs] Regenerate test checks (NFC)
1a12acd6b111f Apply clang-tidy fixes for misc-include-cleaner in standalone-translate.cpp (NFC)
d8214f5143f69 Apply clang-tidy fixes for misc-include-cleaner in standalone-plugin.cpp (NFC)
655b678197cda Apply clang-tidy fixes for misc-include-cleaner in standalone-opt.cpp (NFC)
83fd5d805c66a Apply clang-tidy fixes for misc-unused-alias-decls in StandaloneExtension.cpp (NFC)
4b65b289e14ac Apply clang-tidy fixes for misc-include-cleaner in StandalonePasses.cpp (NFC)
a06f0e33eb074 Apply clang-tidy fixes for misc-include-cleaner in StandaloneOps.cpp (NFC)
e59f76e9f5769 Apply clang-tidy fixes for misc-include-cleaner in mlir-cat.cpp (NFC)
848336db8a6e8 [ExecutionEngine] Use llvm::is_contained (NFC)
c5d8bf7196dd0 [llvm][llvm-readobj] Add AArch64 Tagged Address note type (#68568)
5557d983412c2 [run-clang-tidy,clang-tidy-diff] Accept directory as value for -export-fixes (#69453)
6e3572ccd8965 [lldb] Use llvm::erase_if (NFC)
dc271b59e967d [DebugInfo] Use llvm::erase_if (NFC)
6461a824e4b27 [Transforms] Use llvm::erase_if (NFC)
aa0208d1bc52e [mlir][scf] Implement getSingle... of LoopLikeOpinterface for scf::ParallelOp (#68511)
b8ad68fcc3c45 [mlir][tosa] Update pass pipeline for TosaToLinalg (#69679)
291d8ab3ed1ec [llvm] Use llvm::find_if (NFC)
104c01eb31106 [FunctionAttrs] Only check ArgMem effects when inferring argument attrs (#69571)
8c88a823985a6 [Driver] Use llvm::any_of (NFC)
6362ef1fb1cf5 [CMake] Avoid build spam by switching to Debug message (#69497)
c7be4e6db9d7e [mlir][Bazel] Add missing dependency after d871daea8159c4b39b17b3ab8f3dd3adb1b51de3
95f924f30ae16 [RISCV][MC] Implement evaluateBranch for auipc+jalr pairs (#65480)
195f6236c7e67 [mlir] Use llvm::erase_value (NFC)
24f03d9b19dda [clangd] Use llvm::erase_value (NFC)
f24d9490e5606 [RISCV] Match prefetch address with offset (#66072)
7961fa36baa31 [libc++] Fix uninitialized algorithms when using unconstrained comparison operators (#69373)
e1a584305e6a0 [BOLT] Use llvm::is_contained (NFC)
d871daea8159c [mlir][TilingInterface] Add scf::tileUsingSCFForallOp method to tile using the interface to generate `scf::forall`. (#67083)
af3ead4ccf57b [RISCV] Add more prefetch tests (#67644)
614a8cbfd155b [MC][NFC] Allow MCInstrAnalysis to store state (#65479)
b2d3c7b200492 [DWARF] Remove unused declaration verifyIndexes
f2517cbceec0b [X86][AMX] remove related code of X86PreAMXConfigPass (#69569)
f8325f12606d7 [Tablegen] Bugfix and refactor VarLenCodeEmitter HwModes. (#68795)
833a8db5b1e39 [mlir][scf] Implement getSingle... of LoopLikeOpInterface for scf::ForallOp (#67883)
7f7a15c3579c4 [RISCV][NFC] Use !range bang operator (#66494)
076581fd95af7 [ValueTracking] Implement sdiv/udiv support for isKnownNonNullFromDominatingCondition (#67282)
47826b3f14899 [LoongArch] Fix td pattern for CACOP LDPTE and LDDIR
d1985e3d1fede [RISCV] Support Xsfvqmaccdod and Xsfvqmaccqoq extensions (#68295)
9d10fbbb299ea Fix test clang/test/Driver/cl-offload.cu
697082de74d17 [Scalar] Use LLVMContext::MD_mem_parallel_loop_access directly (NFC) (#69549)
a7c4ff9a563a6 [libc++][Android] Don't list Android as supported yet (#69660)
bce3b505931ce [libc++][Android] Mark tests XFAIL/UNSUPPORTED (#69271)
eb6ec1720628c [flang][openacc] Do not error when bind symbol is defined later or external (#69657)
3d89c088af5bd [mlir][sparse] support BSR for cuSPARSE (libgen path only) (#69646)
ec10c36b07766 [libc][NFC] Forcing data type in gettimeofday_test when comparing the diff. (#69652)
65f946cba4085 [RISCV] Fix some GlobalISel tests using -march instead of -mtriple.
5070c1e3b07c5 [analyzer] WebKit checkers: recognize dynamicDowncast as a safe function.
dd473f1dd19c9 [mlir][python] simplify extensions (#69642)
dda3ed9091d10 [gn] port ab17ecd10767
6ddc03d97c0de [mlir][spirv][webgpu] Add lowering of IAddCarry to IAdd (#68495)
ab17ecd107670 [scudo] Add ConditionVariable in SizeClassAllocator64 (#69031)
ff21a90e51ac3 [mlir][sparse] introduce sparse_tensor.crd_translate operation (#69630)
f6818528f71bc [bazel][mlir] fixes for a2288a89
969ba9ff1477d [lldb] Remove FileSpecList::GetFilesMatchingPartialPath (NFC)
e353cd8173db9 [RISCV] Apply `IsSignExtendingOpW = 1` on `fcvtmod.w.d` (#69633)
1d8985fc14457 [Fuchsia] Add lldb-dap to LLDB distribution
fb5047f5244d8 [mlir][sparse] Remove old syntax (#69624)
a2288a8944c31 [mlir][python] remove mixins (#68853)
a30095a1e4304 [libc++][Android] Add libcxx-builder-android Docker image (#69273)
ea9e116e5a24e [lldb][NFCI] Remove duplicated code in DWARFParser (#69531)
621a271aac421 [flang][openacc] Warn for num_gangs, num_workers and vector_length on acc serial (#69622)
afdad4fd402d1 workflows/release-tasks: Fix release note artifact upload (#69522)
630037ede4cec [libc] Partially implement 'rand' for the GPU (#66167)
a39215768b58b [libc] Rework the 'fgets' implementation on the GPU (#69635)
049993eae6bef [FunctionComparator] Differentiate instructions passing different MDStrings (#69543)
d173ce4a670e8 [libc++][Android] Support libc++ testing on Android (#69274)
3f8e5fd08f33c [NFC] Format some code in GlobalVariable.h
10951ca4fe25e [mlir][sparse] use uint64_t type for dim/rank consistently (#69626)
d681461098c40 [AMDGPU] Add doc updates for kernarg preloading (#67516)
b2f50b49a8c45 [clang-format][NFC] Use UnwrappedLineParser::eof() for consistency
f9632cee30b78 [lldb] Remove FileSpecList::GetFileSpecPointerAtIndex (NFC)
51094545e28d2 [clang][index] Fix processing of CompoundAssignOperator at setting up reference roles (#69370)
616c86accbf4c [mlir][drr] Set operand segment in rewrite
3cac608fbd081 [LV] Add interleave only test case with reduction requiring casts.
3daa771047e16 [libcxx][test] Fix empty.gen selftest on windows (#69403)
c73ad025b13d3 [libc][libm][GPU] Add missing vendor entrypoints to the GPU version of `libm` (#66034)
a91a66483974e [gn] port 01263c6c6fb495 (lldb-vscode -> lldb-dap)
94123d164b910 [gn build] Port 460e84398a19
5bae3a0b0ccf8 [lldb] Remove CompileUnit::SetSupportFiles overload (NFC)
5175cd777c571 Disallow _BitInt as an underlying type for an enumeration
67770cbb9841e [libc][NFC] Fix features.h.def file header
3343d000a3511 [ELF][test] Demonstrate --no-allow-shlib-undefined behavior with a hidden relocatable object file definition
460e84398a19b [RISCV] Add getSameRatioLMUL (#69570)
b1a6b2cc409e5 [AArch64][GlobalISel] Fix miscompile on carry-in selection (#68840)
f7ab79f33ef56 [AMDGPU] Allow lit() on operands which do not accept modifiers (#69527)
f35053209990e [libc] Fix accidental LIBC_NAMESPACE_clock_freq (#69620)
0af5c0668a1b9 [InstCombine] Don't consider aligned_alloc removable if icmp uses result (#69474)
44d4b30ca30b1 [unittest] Refactoring the gtest sharding option. (#69537)
bd21efe24c767 [lldb] Fix ASCII art in CommandObjectSource.h (NFC)
6b8a1425eafce [docs][NewPM] Add comment about declaring analysis managers in the correct order
30520cdff606e [gn build] Port 01263c6c6fb4
60622511bb123 [gn build] Port
2ae37be4b4336 Allow empty dimension arrays in `linalg::inferContractionDims` (#69496)
fdfe0b095f629 Fix test clang/test/Driver/cl-offload.cu
01263c6c6fb49 [lldb] Rename lldb-vscode to lldb-dap (#69264)
0446c589afd66 [DebugInfo] Correctly report header parsing errors from DWARFContext::fixupIndex (#69505)
c35939b22eba9 [VectorCombine] Use isSafeToSpeculativelyExecute to guard VP scalarization (#69494)
200a92520c255 [Clang][SVE2.1] Add builtins and intrinsics for SVBFMLSLB/T
6cfb64276d47b AMDGPU: Minor updates to program resource registers (#69525)
7b9fb1c228337 [mlir][sparse] Update verifier for block sparsity and singleton (#69389)
fdac18c2e23aa [hwasan] Fix rare false negative (zero tag) in two more test cases (#69491)
553616a213b8f [SLP][NFC]Add avx2 test run, NFC.
9f93a99a096c0 [Clang][SVE2.1] Add builtins for 2-way svdot (vectors, indexed)
03d1c99d99d79 [mlir][ODS] Add `OptionalTypesMatchWith` and remove a custom assemblyFormat (#68876)
e880e8aedbc17 Let clang-cl support CUDA/HIP (#68921)
3e49ce6ea16ec InlineSpiller: Delete assert that implicit_def has no implicit operands (#69087)
d2e7a15dfb509 [flang][openacc] Warn about misplaced end loop directive and ignore it (#69512)
21e1b13f3384b [TwoAddressInstruction] Handle physical registers with LiveIntervals (#66784)
e4b75d836d24e [Clang][SVE2.1] Add builtins for Multi-vector load and store
e6e9084070809 [flang] Put ISO_Fortran_binding.h where it can be easily used (#69121)
b781c7ab574f5 Fixed some wmma store builtins that had non-const src param
180eae1f1e5a0 [MemoryBuiltins] Simplify getAllocFnKind() implementation (NFC)
3fd5113cbac9c [libc][math][NFC] Remove global scope constants declaration in math tests (#69558)
6959332804579 [DIAG][msan] fix libc check string for dladdr1 call (#69359)
40a426fac6ff7 [AMDGPU] Constant fold FMAD_FTZ (#69443)
25002b7aeb2f9 [InstCombine] Add additional aligned allocation tests for #69474.
f93a697e47329 [libomptarget][OpenMP] Initial implementation of omp_target_memset() and omp_target_memset_async() (#68706)
970e7456e0346 [Libomptarget] Add a test for the `libc` implementation of assert (#69518)
311f725d9a6fa [clang][Interp] Create only globals when initializing a global variable
8505c3b15bfc5 [DAG] canCreateUndefOrPoison - remove AssertSext/AssertZext assumption that they never create undef/poison
309e41dd1358c [DAG] Add test coverage for Issue #66603
93373c3da9416 [clang][Interp][NFC] Use a const reference in IncDecHelper
b84977bcc12e2 Rename test to avoid overlapping with debug output
d15fff6c69c93 Re-apply '[AArch64] Enable "sink-and-fold" in MachineSink by default (#67432)'
98c90a13c6b27 ISel: introduce vector ISD::LRINT, ISD::LLRINT; custom RISCV lowering (#66924)
3d7802d2107f4 [Clang] Actually fix tests for __builtin_vectorelements (#69589)
0b80288e9e0b1 [NVPTX] Preserve v16i8 vector loads when legalizing
906d3ff054b0c [AMDGPU] Remove legality checks from imm folding in shrink. NFCI. (#69539)
c43ac32bca1e3 [DAG] Expand vXi1 add/sub overflow operations as xor/and (#69191)
4ed0dfe6adfe2 [Clang][SVE2.1] Add svwhile (predicate-as-counter) builtins
0e1d7239d6fdd [clang][Interp][NFC] Use an APInt instead of APSint
b3c4f64943dca [ARM] fix "+fp.dp" in multilib selection (#67412)
202de4a5c6edb Fix __builtin_vectorelements tests with REQUIRES (#69582)
d2edff839d0da [AMDGPU] PeepholeSDWA: Don't assume inst srcs are registers (#69576)
2b97fe2e5158d [clang][Interp][NFC] Add more tests for bitfield initializers
b49f846fe5291 [Flang][OpenMP][Sema] Add directive rewrite pass to support atomic_default_mem_order REQUIRES clause
7338eb561c488 Reapply "[dataflow] use true/false literals in formulas, rather than variables"
d404130134252 [libc] Fix accidental LIBC_NAMESPACE_syscall definition (#69548)
3517b67ef0210 Reapply "[mlir][transform] Support symlinks in module loading. Reorganize tests. (#69329)"
ba47bc7fd4127 [Clang][SVE2.1] Add pfalse builtin
c122b9727a276 Revert "[mlir][transform] Support symlinks in module loading. Reorganize tests. (#69329)"
9f0f6060810eb [clang] Provide an SSE4.2 implementation of identifier token lexer (#68962)
f681225700914 [mlir][transform] Support symlinks in module loading. Reorganize tests. (#69329)
de65b6bec6771 [Clang] Add __builtin_vectorelements to get number of elements in vector (#69010)
9ea2fd245739f [FunctionAttrs] Add additional tests for writeonly (NFC)
5316d19ed54d8 [mlir][nvvm] Introduce `nvvm.stmatrix` Op (#69467)
5341d5465dbf0 [RISCV] Combine (and (select cond, x, -1), c) to (select cond, x, (and x, c)) with Zicond. (#69563)
416884544e021 [TableGen] Update editor modes for new keywords and bang operators. (#68897)
411c4edeef076 [ReleaseNotes][TableGen] Add `dump` and `!repr`. (#68893)
4d8ea66d74cd3 [compiler-rt] Fix a warning
db9b6f4987394 [Tablegen] Add keyword `dump`. (#68793)
e63ab13c82e78 [clangd] Disable crashy unchecked-optional-access tidy check (#69427)
278e533ee9fdd [X86] Support -march=pantherlake,clearwaterforest (#69277)
7f1733a252cbb [GVN] Fix use-after-free in load PRE with select available value (#69314)
a3bbab18527b5 [IR] Don't mark experimental.guard as willreturn (#69433)
39427b10985f0 Reapply "[clang analysis][thread-safety] Handle return-by-reference..… (#68572)
cd205efb9d788 Revert "[Github] Make PR formatting job only run with C/C++ changes (#69556)"
e4ea099748600 Revert "[VPlan] Insert Trunc/Exts for reductions directly in VPlan."
59d2dc239b053 [clang][Interp] IntegralAP zero-initializers (#68081)
cffb9df10cebe [Github] Add steps to build clang docs to CI (#69550)
80b2aac2c6717 [Github] Make PR formatting job only run with C/C++ changes (#69556)
1be3b1ef51eb6 [RISCV] Remove FrameIndex case in lui+addi MacroFusion (#68701)
f4231bf446ed8 [RISCV] Replace PostRAScheduler with PostMachineScheduler (#68696)
be215e76d7139 [SPIR-V] Remove calls to deprecated PointerType methods (1/2) (#68336)
d51855f70034e [RISCV] Fix assertion failure from performBUILD_VECTORCombine when the binop is a shift. (#69349)
817519058a986 [SPIR-V] Emit proper pointer type for OpenCL kernel arguments (#67726)
b858309ddc977 [mlir] Only attempt to vectorize conv if conv.
d6a92611ebb55 [libc++] Improve the tests for std::basic_stringbuf's constructors and assignment operators
ea9af5e7fd8ac [libc++] Add assertions for potential OOB reads in std::nth_element (#67023)
a574242f196dd [libc++][docs] Update contributing docs to reflect the move to GitHub (#69386)
654a3a3cbc1c9 [OpenCL][RISCV] Support SPIR_KERNEL calling convention (#69282)
208a6d97f56fc [libc++] Fix inconsistency between is_lock_free and is_always_lock_free (#68109)
1196e6dda9e6b [libc++] Move the check-generated-files job to Github Actions (#68920)
fd1a0b0ee4d8f nfc, add test case for llvm-symbolizer on XCOFF
fb366581e7d67 [Memory] Call __clear_cache in InvalidateInstructionCache on LoongArch (#67285)
74b0465fe9549 [libc] Add simple features.h with implementation macro (#69402)
5e30c74309c36 [mlir] Use the process (host) triple in MLIRTargetLLVMTests (#69538)
04d63088cd5b9 [mlir] Add debug messages for failures of isValidIntOrFloat
d11c4542d8828 [Support] Use StringRef::contains_insensitive (NFC)
78abc45c44cda [LoongArch] Improve codegen for atomic cmpxchg ops (#69339)
271087e3a0875 [LoongArch] Implement COPY instruction between CFRs (#69300)
ab301d833a207 [ELF][test] --emit-relocs: test ALLOC sections discarded by --gc-sections and referenced by non-ALLOC
8ff3e4f39b381 [InstCombine] Refactor matchFunnelShift to allow more pattern (NFC) (#68474)
f16cb0eade080 [mlir][sparse] connect MapRef's lvl2dim with latest AffineMap computation (#69540)
e103515cedff1 [RISCV][GISel] Support passing arguments through the stack. (#69289)
0913a2d07d06b [Driver][DragonFly] Fixes for linker path and command-line option handling (#69095)
b2e487d1f963d [llvm] Use StringRef::contains (NFC)
2172eeaa52bec [clang-format][NFC] Take a constant conjunct out of a loop condition
5b83bd133d705 [llvm] Use StringRef::contains (NFC)
cfbf0a500f173 [X86][RFC] Support AVX10 options (#67278)
f3ea73133f91c [ELF] Set large section flag for globals with an explicit section (#69396)
7aa24915cb66c [clang] Expand invalid PCM diagnostic (#69489)
5f5faf407b423 [RISCV][GISel] Add ISel supports for SHXADD from Zba extension (#67863)
85b8958b56623 [WebAssembly] add: hidden option to disable slow wasm pass (#67715)
3745e7080746b [Tosa] Rename variables to coding style guideline (#69509)
362b1157868c0 [flang][openacc] Avoid privatizing symbols during semantics (#69506)
077d89f0ee840 [DWARFLinker] Only extract unit DIEs when cloning clang modules (#69495)
b8b7c3ba5c421 [Kaleidoscope] Register new dependencies introduced by #69032. (#69510)
0ae998c4aebba [clang-format] Fix a bug in annotating TrailingReturnArrow (#69249)
1db8abf21a7c0 workflows/release-lit: Pass correct build directory to pypa/gh-action-pypi-publish (#69438)
6a7f0b9d9fb53 workflows/release-lit: Fix dev suffix removal (#69397)
040df124a2423 [RISCV] Don't let performBUILD_VECTORCombine form a division or remainder with undef elements. (#69482)
98e95a0055a47 [AMDGPU] Make S_MOV_B64_IMM_PSEUDO foldable (#69483)
6c6749ce72fa3 [mlir][sparse] Update examples in Ops.td (#69499)
81d651e77b9bd [ModuleInliner] Update a comment (NFC)
4a06332e450ff [SLP][NFC]Use MutableArrayRef instead of SmallVectorImpl&, rename function, NFC.
3fb3df36eaeec [ModuleInliner] Use SmallVector::pop_back_val (NFC)
400389db7de3a Initialize sigset in asan_interceptors (#69502)
306f4c306a3aa [mlir][sparse] implement non-permutation MapRef encoding (#69406)
781424c87273f [compiler-rt] Fix a warning
e8fe4de64ffb8 [CodeGen] -fsanitize=alignment: add cl::opt sanitize-alignment-builtin to disable memcpy instrumentation (#69240)
8a701024f3e09 [ARM] Lower i1 concat via MVETRUNC
e494a96a69050 [libc++][NFC] Refactor the core logic of operator new into helper functions (#69407)
84f398af74f38 [AMDGPU] Add missing test checks. NFC. (#69484)
f362be597ae25 [libc++][NFC] Reformat new.cpp and stdlib_new_delete.cpp
3927b9ab11541 [VectorCombine] Add tests for unspeculatable VP binops. NFC
c9b7d21dc577b [CFI/MergeFunctions] Modify MergeFunctions to propagate type information (#68628)
10079a23c5b6b [HIP] Document func ptr and virtual func (#68126)
eee8dd90887cb [CodeExtractor] Allow to use 0 addr space for aggregate arg (#66998)
ddf1de20a3f7d [hwasan] Fix rare false negative (zero tag) in stack-uar.c (#69374)
192d3320f07e6 [mlir][nvgpu] Add predicate argument to NVGPU Ops (#69322)
fea55db23372e [clang-tidy][DOC] Fix syntax in coroutine-hostile-raii.rst
1a21196b9a53d [MLIR] reverse int8 type's printing logic (#69361)
a587f42953930 [TableGen] SubtargetEmitter must use std::nullopt (#69475)
8e810dc7d93be [SystemZ] Support builtin_{frame,return}_address() with non-zero argument (#69405)
b69081e3241be Attributes (#69358)
1e5fe67e70d4b [Libomptarget] Make the references to 'malloc' and 'free' weak. (#69356)
3ef271c3d6930 [SLP][NFC]Use MutableArrayRef instead of SmallVectorImpl& in param, NFC.
708808e8532e7 [Sema] Add check for bitfield assignments to integral types (#69049)
410f41300a7f6 Add missing include breaking the modules build
1891d21312cf4 [clang-tidy][DOC] Fix syntax in coroutine-hostile-raii.rst
ecd3934432720 [mlir] ADTExtras: include mlir/Support/LLVM.h (#69479)
598712315b0e5 [clang-tidy][DOC] Fix list.rst
7b94744e77aaf [Kaleidoscope] Switch to the new PassManager. (#69032)
47ed921985679 [AMDGPU] Add legality check when folding short 64-bit literals (#69391)
bf7a826e7135b [Presburger] Fraction: resolve ambiguous overload in some cases
e7b827e11bd36 Add a FIXME comment; NFC
98191d7c160cc [CONCEPTS]Corrected comparison of constraints with out of line CTD (#69244)
814a79aea67ba [DebugInfo] Separate error generation from reporting in DWARFHeaderUnit::extract (#68242)
c80b50349648d [SLP] Improve gather tree nodes matching when users are PHIs. (#69392)
e3ae2a52d2232 [llvm][CMake] Check dependency cxx source compiles (#68549)
cba4e7e9e629b [MLIR][Doc] Clarify the cf.asssert doc that this is a runtime assertion
ec0e556e6708e [ELF] Merge copyLocalSymbols and demoteLocalSymbolsInDiscardedSections (#69425)
9f4950983e2ae [mlir] Add ContractionOpInterface utility functions for vector matrix multiplication (#68945)
df3478e480b3b [LLVM] Add new attribute `optdebug` to optimize for debugging (#66632)
d0e8f3321e0a5 [flang][openacc] Fixed private/reduction for combined constructs. (#69417)
e7432bc343655 [InstCombine] Add aligned_alloc with pointer icmp as only use.
0a9bdc276d259 [lldb] Fix linking to libtinfo (#69458)
1b93e15bcd9a2 [Clang][SVE2p1] Add svpsel builtins
5a600c23f9e01 [mlir][python] Expose `PyInsertionPoint`'s reference operation (#69082)
a3a0f59a1e1cb Revert "Correctly compute conversion seq for args to fn with reversed param order (#68999)"
cf1bde9a15d71 [Driver] Link Flang runtime on Solaris (#65644)
178619d71dcfd [Clang] Fill in documentation gaps for some attributes (#68967)
b0b8e83e668ac [mlir] Fix use-after-free bugs in {RankedTensorType|VectorType}::Builder (#68969)
28e4f97320b6d [AMDGPU] Save/Restore SCC bit across waterfall loop. (#68363)
18e5055db39b4 [mlir][LLVM] Improve function debug info import (#69446)
00f34eefe4ed0 [clang] Bail out if the result of function template instantiation is not a function type. (#69459)
dc27d21890027 [mlir][nvvm] Use NVVMMemorySpace instead of hardcoded values (nfc)
69660ccf2ae40 [builtins] Convert more int to fp functions to use common implementation (#67540)
730df5a437914 [Support] Add KnownBits::computeForSubBorrow (#67788)
ae894971105ff [gn build] Port 1072b94ed8e5
1072b94ed8e5a Revert "[clang] [unittest] Add a test for Generic_GCC::GCCVersion::Parse (#69078)"
c060757bcc8a0 [ARM] Correct v2i1 concat extract types.
561fcf547e07b [BOLT][test] Update checkvma-large-section.test (#69419)
66c99154a1305 [clang][Interp][NFC] Remove from(Boolean) overload
cbbb545c46189 [lld] Sort code section chunks by range types on Arm64EC targets. (#69099)
9322a0c2f34a7 [Clang] Run update_cc_test_checks across SVE acle tests.
ebdb0cbef5e9b Add missing test from #68661
868abf09619cb Revert "[AMDGPU] Remove Code Object V3 (#67118)"
c92629150e361 [MLIR][TOSA] Fix f16/bf16 support for MaxPool2D (#69332)
0dca56603b4e1 Revert "Detect against invalid variant index for LibStdC++ std::variant data formatters (#69253)"
104db26004d0e [AMDGPU] Fix image intrinsic optimizer on loads from different resources (#69355)
675231eb09ca3 [SVE ACLE] Allow default zero initialisation for svcount_t. (#69321)
975ec8368fcb3 [compiler-rt] Only build SME ABI routines for baremetal or platforms that have sys/auxv.h (#69423)
d7246c1ab523a [gn build] Port b4b35a5d2b4e
b4b35a5d2b4ee [clang] [unittest] Add a test for Generic_GCC::GCCVersion::Parse (#69078)
48a53509e851c [clang][USR] Encode full decl-context also for anon namespaces (#68325)
8c369eb4965a1 nfc, address post commit comments related to code format for 581c64a
7cd7b9533c778 [BOLT] Fix instrumentation test (#69383)
e93bddb2874db [AArch64][GlobalISel] Precommit indexed sextload/zextload tests.
f48dab5237842 Add RV64 constraint to SRLIW (#69416)
51a2ac645f4ef [flang] Fold IS_CONTIGUOUS of component refs with non-contiguous base (#69327)
112e49b38150b [DAGCombiner] Transform `(icmp eq/ne (and X,C0),(shift X,C1))` to use rotate or to getter constants.
0c2d28a448dee [X86] Add tests for transform `(icmp eq/ne (and X, C0), (shift X, C1))`; NFC
33430205510f4 [ModuleInliner] Fix the heap maintenance (#69251)
279c155fb224f [CodeGen] Remove unused function isMSInlineAsm (#69132)
c464fea779c84 [DAG] Constant fold FMAD (#69324)
ed1d29028492b [Driver][NFC] Remove identifier with the comment (#68351)
b42738805acff [PowerPC] Auto gen test checks for #69299. NFC.
beffc821e8290 [CodeLayout] CDSortImpl: remove HotChains and remove linear-time erase_value from mergeChains (#69276)
ae3ba725b7902 [RISCV][GlobalISel] Select G_FRAME_INDEX (#68254)
0b2924aaaf8d5 [Docs][NFC] fix URL
659a48f25a96b Detect against invalid variant index for LibStdC++ std::variant data formatters (#69253)
5a56f0074cf32 [M68k] Fix assertion build after cc6a5ea6e33d3febafc4334617230c528a0c4fa7
6971081cefdc4 [Docs] Remove future extensions section from writing a pass docs (#69286)
bd0fc480df2f0 [sanitizer_common] Use 38-bit mmap range for Fuchsia (#69387)
16a5c71241f96 [LangRef] "cc 10" -> "ghccc" (#69380)
6eee238975e43 [unittest] Add option to allow disabling sharding in unittest (#67063)
3e4ad66fc1677 Fix build: the dump() method is only available in Asserts build (NFC)
340d746abae18 Turn an assert in mlir-tblgen into a runtime check to be more user friendly (NFC)
7d1bf1c5cf284 [LLDB][NFC] Move some constructors to their cpp file
170b552136f84 [lldb] Scalar::GetValue() should take a Stream by reference (#69231)
59908504cd7a7 [LLDB][NFC] Add a missing namespace
bbf7b9d805f57 [ELF] Remove unused setSymbolAndType after #69295. NFC
9aa571f080411 [SLP][NFC] Try to cleanup and better document some isGatherShuffledEntry code. (#69384)
3caccb22ab530 [NFC][SLP] Test case exposing gather nodes matching deficiency affecting cost. (#69382)
cf670d5e56d14 [flang][openacc] Accept scalar integer expression in the if clause (#69381)
1981b1b6b92f7 [ELF] Demote symbols in /DISCARD/ discarded sections to Undefined (#69295)
ef0e0adccd94f [llvm-profdata] Do not create numerical strings for MD5 function names read from a Sample Profile. (#66164)
e90ec58b132a7 [CMake] Support per-target linker flags (#68393)
fc5d815d547e5 [ELF] Merge demoteSymbols and isPreemptible computation. NFC
3472d4d4c311b [MLIR][Doc] Prepend "Variadic of" in front of variadic operands (#69285)
e9b9a1d3202d8 [ELF] Move demoteSymbols to Writer.cpp. NFC
d4088e7d5f484 [mlir][sparse] Populate lvlToDim (#68937)
9922aadf9e9d1 [OpenMPIRBuilder] Added `if` clause for `teams` (#69139)
122064a6303eb [libcxx] [test] Add a test parameter for disabling memory intensive tests (#68214)
0996ceece605c [ELF][test] Improve relocatable link & /DISCARD/ test
389958a9f67ae [CodeGen][NFC] Fix formatting
7dc644fc463a8 [CodeGen] Temporary disable the unreachable
c0f3478934bec [gn build] Port 31512811b8c0
f781508d31943 [gn] port dd64c82cbc9c6
ab91e05e48d9e [mlgo] Fix tests post 760e7d0
fbf0a77e80f18 [CodeGen] Avoid potential sideeffects from XOR (#67193)
e6d0b126c8242 Correctly compute conversion seq for args to fn with reversed param order (#68999)
31512811b8c0f [clang-tidy] Add check to diagnose coroutine-hostile RAII objects (#68738)
71c97c735c10d [mlir][sparse] avoid tensor to memref conversion in sparse tensor rewri… (#69362)
fd311126349b8 [VPlan] Insert Trunc/Exts for reductions directly in VPlan.
dd64c82cbc9c6 [unittest] Allow LLVM unit test to run under a wrapper program. (#66821)
b33723710f519 [NVPTX] Fixed few more corner cases for v4i8 lowering. (#69263)
ddc30ff802eb1 [libc] Implement the 'ungetc' function on the GPU (#69248)
761c9dd92789b [mlir][sparse] implementating stageSparseOpPass as an interface (#69022)
a22a1fe151b81 [AMDGPU] support 64-bit immediates in SIInstrInfo::FoldImmediate (#69260)
66775f8ccdcc8 [SLP]Fix PR69196: Instruction does not dominate all uses
4266815f4d82b [AArch64] Convert negative constant aarch64_neon_sshl to VASHR (#68918)
658ed58de6f89 [AArch64] Add additional tests for fptosi/fptoui. NFC
4480e650b3cf7 [YAMLParser] Improve plain scalar spec compliance (#68946)
2a40ec2d3e4d2 [DAG] SimplifyDemandedBits - fix isOperationLegal typo in D146121
d816c221b45c6 [mlir][sparse] complete migration to dim2lvl/lvl2dim in library (#69268)
760e7d00d142b [X86, Peephole] Enable FoldImmediate for X86
08d6b8745430e [flang] Round derived type byte sizes up to alignment multiple (#67571)
7cad5a9eb48e4 [Clang][SVE2.1] Add svpext builtins
be57381a4a08b [InstCombine] Create a class to lazily track computed known bits (#66611)
7b1e6851b65a4 [hwasan] Exclude bcmp interceptor test from Android
20af0e5e8d569 Enable v for RISCV64 Android (#69261)
abd0d5d262602 Reland: [AArch64][GlobalISel] Adopt dup(load) -> LD1R patterns from SelectionDAG
3162cf0430210 [Github][OpenMP] Adding rule for OpenMP label (#65331)
2f329d88bc2e6 [DAG] foldConstantFPMath - accept ArrayRef<SDValue> Ops instead of explicit N1/N2 ops
7f3435575404c [RISCV] Use separate CCValAssign for both parts of f64 with ilp32. (#69129)
a559de0c2fdd0 [AMDGPU] Simplify definition of SIbuffer_atomic_*. NFC.
dffd93b30b565 [flang][runtime] Fix SAME_TYPE_AS()/EXTENDS_TYPE_OF() for CLASS(*) (#67727)
4b8f23e93de56 [AArch64][SME] Remove immediate argument restriction for svldr and svstr (#68908)
3d6e4160d52da [X86] Enable bfloat type support in inline assembly constraints (#68469)
4df46c39d6e24 [gn build] Port 088d272e8325
3dc263b18c082 Update documentation on x86 constraint codes (#68830)
c38598186bbc4 [flang] Fix constant subscript operations (#68352)
791b890c468e5 [HIP][Clang][CodeGen] Simplify test for `hipstdpar`
b2773d170cb4b [LoongArch] Precommit a test for atomic cmpxchg optmization
088d272e83259 [ADT][DebugInfo][RemoveDIs] Add extra bits to ilist_iterator for debug-info
81d8fa5a1d01e [Clang][SVE2.1] Add svcntp prototype
fc53b1abf7d5e [CUDA][HIP] Fix init var diag in temmplate (#69081)
096eba148df7d [TargetParser][AMDGPU] Fix getArchEntry(). (#69222)
bff17f9f23ce7 [AMDGPU] Remove support for no-return buffer atomic intrinsics. NFC. (#69326)
509b5708e98e0 [AMDGPU][AsmParser] Eliminate custom predicates for named-bit operands. (#69243)
12bf4231eb0d4 [Bazel] Fix dependencies for clang codegen
90576084c1d79 [Bazel] fix typo
4434253f0fa66 [Bazel] disable preload-library.mlir test
8b5625cb42b78 [gn] port 3694697003bb
e730f4a27fceb [gn] port 3694697003bb
7f2dd2da99371 [mlir][Tosa] Fix test failure when running with Asan.
b736e0466c629 [MLIR][NVGPU] Test warpgroup matrix multiply 128x128x64 (#68817)
39cdefb5b52b3 [mlir][nvvm] Add prefetch.tensormap (#67564)
f2898def693a8 [InstCombine] Don't mix X << Y / Z << Y with X << Y / X << Z (#69302)
c4ba84d655514 [mlir][nvgpu] Fix packing accumlator matrix (#69316)
63389326f529f [mlir][nvvm] Support predicates in `BasicPtxBuilder` (#67102)
dd5d65adb6413 [HIP][Clang][CodeGen] Add CodeGen support for `hipstdpar`
be9bc542186f9 [X86] vselect.ll - add vXi8 select-by-constant tests with repeated/broadcastable shuffle mask
22e3bf4eaf6cb [mlir][transform] Fix new interpreter and library preloading passes. (#69190)
bea3684944c0d [AArch64] Allow only LSL to be folded into addressing mode (#69235)
52db7e27458f7 [mlir][nvgpu] Improve `WarpgroupAccumulator` type to simplify IR  (#68728)
838f2890fd302 [libc++] Eliminate extra allocations from `std::move(oss).str()` (#67294)
484668c7597d9 Reland "[MLIR][LLVM] Change addressof builders to use opaque pointers" (#69292)
815193f6be08e [ci] diff with main merge-base (#69308)
f0601c7569c6e [clang][NFC] Replace TypeAlignment with alignof(T) (#69185)
d2b74d7e4217b [TableGen] Handle duplicate rules in combiners (#69296)
0841955bf3b79 [TableGen] Use buildConstant to emit apply pattern immediates (#66077)
11f5e5eb90c88 [clang][Interp][NFC] Add thread_local tests
e483673246bde [compiler-rt][HWASAN] Add missing include in deep-recursion.c test
4606712ef5b42 [lldb][lldb-vscode] Add example configuration for connecting to a remote gdbserver (#68866)
cbf7d5f82b72d [AArch64] Fix -Wunused-variable in AArch64LowerHomogeneousPrologEpilog.cpp (NFC)
bfcd05317d0fb [flang][hlfir] Do not emit extra declare for dummy used in BLOCK (#69184)
77ab08e1ffa87 [flang][runtime] fix buildbot failure after #69199
bef3e8ea6d241 [flang][runtime] Fix another IsContiguous edge case (#69199)
8e674e8a01314 [flang] Deallocate INTENT(OUT) dummy allocatable components (#69164)
041a786c78fbc [AArch64] Fix pairing different types of registers when computing CSRs. (#66642)
9397e5f581b12 Revert "[MLIR][LLVM] Change addressof builders to use opaque pointers (#69215)"
fbde19a664e5f [MLIR][LLVM] Change addressof builders to use opaque pointers (#69215)
8ddca6b2c19c8 [CI] Fix documentation build CI job
aa4dfd3736dd1 [hwasan] Fix and re-enable deep-recursion.c (#69265)
4b8b70a52fa4d [Clang] Fix dependence handling of nttp for variable templates (#69075)
12a731b5a4cfe [CI] Add Github actions job to build LLVM documentation (#69269)
7bc793a6925cc [clang][Interp] Check pointer inc/dec ops for null (#69168)
cc6a5ea6e33d3 [M68k][NFC] Fix some unused variable warnings
5a6ef95a1cb5c [RISCV][GISel] Add legalizer for G_UMAX, G_UMIN, G_SMAX, G_SMIN (#69150)
ce9eaf0360d9f Revert "[clang][Sema] Use original template pattern when declaring implicit deduction guides for nested template classes (#68379)"
b0eba8e209d46 [RISCV] Support STRICT_FP_ROUND and STRICT_FP_EXTEND when only have Zvfhmin (#68559)
3694697003bbf [clang] Implement C23 <stdckdint.h>
81d04709f8696 [flang] Fix construct names on labeled DO (#67622)
11d07d9ef6184 [flang] Handle separate module procedures with INTERFACE dummy arguments (#67608)
28a686a704fab [flang][NFC] Speed up large DATA statement initializations (#67585)
d85f5a6216670 [compiler-rt] Fix build of builtins on Windows
c319c741463a0 [RISCV] Improve performCONCAT_VECTORCombine stride matching
30ca258614dd2 [RISCV] Pre-commit concat-vectors-constant-stride.ll
2565f9f49b79e [flang] Remove IEEE_DENORM from IEEE_ALL (#67573)
d343529d0bd03 [flang] Fix CFI_CDESC_T for C++ interoperability (#67568)
78be6b22347e9 llvm-gsymutil now handles empty linkage names correctly. (#68931)
910a4bf5b70ae [compiler-rt] Implement __extendxftf2 and __trunctfxf2 for x86_64 (#66918)
e35cb730cfd30 [flang][runtime] Fix edge cases with ROUND=UP/DOWN (#67508)
af972f01c0184 [TOSA] Add StatefulOps to TOSA Dialect (#66843)
ff1329e297094 [HWASAN] Add bcmp interceptor (#69257)
b225934a4b0d2 [flang] Avoid needless overflow when folding NORM2 (#67499)
39f4ec5854a1c [flang] Catch a dangerous ambiguity in standard Fortran (#67483)
fa7d6a0fa5eaf [docs] Fix google meet link
dda46b2e795cb [docs] Add a new GlobalISel office hours session to the list.
c007e0f66ee3f [hwasan][test] Fix regex so deep-recursion.c is unsupported on aarch64 targets (#69254)
233c3e6c53a56 [mlir][sparse] remove sparse2sparse path in library (#69247)
59f69a38ad375 [flang] Ensure component attributes affect characteristics (#67465)
5f4ed780d348c [clang-format] Allow default values for template parameters in lambda (#69052)
301a0dba56e17 [flang][runtime] Better non-repeatable RANDOM_INIT() (#67363)
e6e62efa880e7 [RS4GC] Copy argument attributes from call to statepoint (#68475)
e200b0e4a7b54 [flang] Submodule names can clash only with submodule names (#67361)
ea7e50cdf2f53 [flang][runtime] Implement EX editing for input & output (#67208)
c5b617c5e53d7 [clang-tidy][NFC] Clarify documentation for misc-definitions-in-headers
511236e07436c [libc++][test] Add `stop_token` benchmark (#69117)
b7de1d07e5298 [Clang][NFC] Use correct tool name for NVIDIA's 'nvlink'
8a47ad4b67edf [libc] Add simple long double to printf float fuzz (#68449)
119b0f3895688 Revert "[SLP]Fix PR69196: Instruction does not dominate all uses"
750c8e39de3c1 [flang][runtime] Handle incomplete NAMELIST input derived type compon… (#66831)
6f41510d4f484 [llvm-rc] Accept filenames provided as multiple string literals (#68881)
4bf10f3da7ab3 [YAMLTraits] Fix std::optional input on empty documents (#68947)
528b5e6743db4 Revert "[gn build] Add rules for crtbegin/end (#66012)"
8e2b2c4181506 [SLP]Fix PR69196: Instruction does not dominate all uses
b3fbb67379a4e [mlir][sparse] cleanup of COO (#69239)
d9568bd4aa46c [flang][openacc] Support array with dynamic extents in firstprivate recipe (#69026)
f74b85c67827b [flang][openacc] Support array with dynamic extents in reduction recipe (#68829)
e2e32f091a903 [gn build] Add rules for crtbegin/end (#66012)
4718b4011f1d3 [LV] Invalidate disposition of SCEV values after loop vectorization (#69230)
5db4779c3f07b [flang] Regularize TODO messages for coarray related features (#69227)
f7a8a78cb7c1a [VPlan] Also print operands of canonical IV (NFC).
dd0fba11690f9 [clang][Sema] Use original template pattern when declaring implicit deduction guides for nested template classes (#68379)
f6f944e77f741 [lld][NFC] Factor out isCodeSection helper. (#69193)
38f8b7cbe4729  [LV] Replace value numbers with patterns in tests (NFC).
47401b6173459 [lld] Add support for relocations in x86_64 objects on Arm64EC targets. (#69098)
1ebe73821f4ce [mlir][sparse] fix crash due to different std::sort implementation. (#69236)
5b07de1a5faf4 [clang][ASTMatcher] fix hasAnyBase not binding submatchers (#67939)
342dca7528116 [clang][dataflow] Check for backedges directly (instead of loop statements). (#68923)
eb14f47bf1ccf [mlir][sparse][NFC] fix variable naming convension (#69232)
fcc13c04bbd14 [libc++] Simplify the tuple constructor overload set
5fab20bc7e551 [NFC] Move StableHashing.h from CodeGen to ADT (#67704)
5e4ec53b8efaa [llc][PPC] Move PIC check into TargetMachine (#66727)
d392073f6747e [mlir][sparse] simplify reader construction of new sparse tensor (#69036)
182a65adcf8af [RISCV] Refactor performCONCAT_VECTORSCombine. NFC (#69068)
1e8ab993baf4e [flang][openacc] Fix missing bounds for allocatable and pointer array component (#68914)
468d3b1b78c96 [flang][openacc][NFC] Simplify lowering of recipe (#68836)
e32cde6f41cd9 [clang] Use IgnoreParensSingleStep in more places
9cc6f492f20ff [GlobalISel] Remove unused function narrowToSmallerAndWidenToSmallest (#69130)
0b570ad969b8b [CodeGen] Remove LiveVariables::{isPHIJoin,setPHIJoin} (#69128)
45d151138008c [libunwind][AIX] Fix problem with stepping up from a leaf function when unwinding started in a signal handler
b51eaebd2b437 [libc++] Remove workaround for clang-tidy 16 in the test suite setup code (#69035)
6ade5183232dc [flang][openacc][NFC] Issue better error message when directive is wrong (#69034)
144c5b6d58803 [compiler-rt][hwasan] Disable deep-recursion.c test on AArch64 Linux
903faefc14eb8 [Flang][OpenMP] Port three tests to HLFIR flow
e9c101a7533a8 [libc++] Add missing <__availability> include
97217d188469c [mlir] Fix '-Wunused' warning. NFC
cc3d2533cc2e4 [AMDGPU] Add i1 mul patterns (#67291)
f41ec27f7eba3 [Flang][OpenMP] Port atomic read, write tests to HLFIR flow
7ac516a119a36 [amdgpu] Disable openmp test that is blocking CI after changing hardware, need to diagnose memory fault
4acb96c99f3b9 [SelectionDAG] Tidy up around endianness and isConstantSplat (#68212)
c202a17d02406 [clang][analyzer] Move checker alpha.unix.StdCLibraryFunctions out of alpha. (#66207)
499d41cef2e7b [flang][OpenMP] Fix threadprivate common blocks (#68739)
de9b3c5eba41f [clang][Interp] Handle delegating constructors (#67823)
d4300154b6e7a Revert "[ValueTracking] Remove by-ref computeKnownBits() overloads (NFC)"
4d6fc88946eec [AMDGPU] Add patterns for V_CMP_O/U (#69157)
b5743d4798b25 [ValueTracking] Remove by-ref computeKnownBits() overloads (NFC)
3b23704f161c3 [lldb][PDB] Fix test build after plugin namespace change
a72d88fb4f65d Revert "Reapply [Verifier] Sanity check alloca size against DILocalVariable fragment size"
17fce28683434 [lldb][DYLD][NFC] Dedupe calls to CreateBreakpoint
1d43096e16ff7 [ConstraintElim] Don't decompose values wider than 64 bits (#68803)
dad563e3c223a [AArch64][GlobalISel] Add legalization for G_VECREDUCE_MUL (#68398)
d86047cb665ec [mlir][ArmSME] Update tile slice layout syntax (#69151)
c0a7dd49118b6 Fix MSVC "not all control paths return a value" warnings. NFC.
8e53abc0412ff [ci] pull main branch before diffing (#68983)
c68bc1726c1c1 [analyzer] Fix note for member reference (#68691)
3ab536fb994b9 [BOLT][RISCV] Implement getCalleeSavedRegs (#69161)
5857fec27fe8e [clang][ASTImporter] Fix of possible crash "Did not find base!". (#67680)
c67b86280ec93 [BOLT][RISCV] Don't create function entry points for unnamed symbols (#68977)
d8de38b4010f4 [BOLT][RISCV] Handle EH_LABEL operands (#68998)
0ddca87b794d9 [X86][FP16] Do not combine to ADDSUB if target doesn't support FP16 (#69109)
8592241e29e29 [BOLT] Fix reorder data test for RISC-V (#68996)
2371d0ab263c1 [DebugInfo] Only call upgradeCULocals() at module level (#68965)
5c0931727eb04 [BOLT][RISCV] Implement MCPlusBuilder::equals (#68989)
33b58f3f2e780 [BOLT] Move X86-specific test to X86 subdirectory (#68992)
7571f27ed7685 [lld] Restore "REQUIRES: amdgpu" in amdgpu-abi-version
544d91280c26f [AMDGPU] Remove Code Object V3 (#67118)
cd88466dafdb1 [TableGen] Fix GlobalISelEmitterHwModes.td after 96e473a
96e473a6be2e8 [RFC][GlobalISel] Use Builders in MatchTable (#65955)
94d0a3c4a8b43 [clang][Interp][NFC] Add comments to Descriptor ctors
58c9ef5a2da5c [RISCV] Use f64 for LocVT for ilp32 when whole f64 is passed on the stack. NFC (#69118)
e3f533201c61b [RISCV][GISel] Don't setType on PtrReg in RISCVInstructionSelector::replacePtrWithInt.
fd673e8c4e4fc [MLIR][SCF] Removes incorrect assertion in loop unroller (#69028)
019d67f19721f [RISCV][NFC] Remove space
7fb2b4d7f55af [CodeGen] Remove unused declaration createJumpInstrTablesPass
be72dca5e3ab3 [docs] [C++20] [Modules] Mentioning that -fdelayed-template-parsing is not working with modules
6121b9088ef0d [gn build] Port 819ac45d1c1b
819ac45d1c1b7 [X86] Add USER_MSR instructions. (#68944)
993e839480449 [Driver] Don't pass -Z to ld for ELF platforms (#69120)
0ae4622126a2e [RISCV][GISel] Move variadic-call.ll from call-lowering directory to irtranslator. NFC
97c9f9a20af42 [mlir][affine] NFC: Improve variable name in TestReifyValueBounds
5ae5af1d7c60a [clang-tidy][modernize-loop-convert]check isDependentSizedArrayType (#69062)
ab737a86993bc [mlir][Interfaces] `LoopLikeOpInterface`: Add helper to get yielded values (#67305)
3049ac44e638c [mlir][vector] Enable transfer op hoisting with dynamic indices (#68500)
fd4f96290ac99 [Clang][M68k] Add Clang support for the new M68k_RTD CC
42b707e5b438b [DWARF][M68k] Add new DW_CC for the new M68kRTD calling convention
fd84b1a99dfe3 [M68k] Add new calling convention M68k_RTD
7f881a2abe2c3 [clang-format] Treat AttributeMacro more like __attribute__
6c7cf74a75572 Revert "[clang-format] Treat AttributeMacro more like __attribute__"
6f46bcc609f14 [clang-format] Treat AttributeMacro more like __attribute__
2ad9a658005e6 [LVI][CVP] Treat undef like a full range on abs(x, false) (#68711)
19505072123e4 Revert "Re-apply '[AArch64] Enable "sink-and-fold" in MachineSink by default (#67432)'"
ea4cc2007efea [BasicAA] Remove NSW flags when merging scales (#69122)
96196e25fd49e [GlobalISel] Remove unused declaration tryCombineIndexedLoadStore
196108857d228 [GlobalISel] Remove unused declaration applyCombineAnyExtTrunc
017b9c03d6067 [CodeGen] Remove unused declaration RegisterDefIsDead
546c3d792addc [OpenMP][mlir] Added `num_teams`, `thread_limit` translation to LLVM IR (#68821)
4698b9926221c [BasicAA] Add pre-commit tests for PR69096. NFC.
d5e91ca5633e6 [CMake] Limit -gsplit-dwarf option to C and C++ compilers
6dfea561ba969 [builtins] Start to refactor int to fp conversion functions to use a common implementation (#66903)
0823cb791198c [InstCombine] Fold (X << Y) / (X << Z) -> 1 << Y >> Z (#68863)
2ef158752a580 Update Clang's Getting Involved page
eca2fcbdeb328 [AMDGPU] Fix cost of fast unsafe f32 fdiv (#68988)
9451004987e84 [InstCombine][TLI] Fix function prototype of `labs` (#69077)
e1bb0598b2c0e [MachineBasicBlock] Fix use after free in SplitCriticalEdge (#68786)
fad99d398a714 [clang][Driver] Fix a spot in commit 169f60f7c76cb6f7d234ab5dfb2b5e367a35ccbb
88dd9813696e3 [include-cleaner] Remove unused using decls (NFC)
0187960cdd0cc [Scalar] Use LLVMContext::MD_mem_parallel_loop_access (NFC)
3c4ecc4628601 [clang][Interp][NFC] Refactor VisitImplicitValueInitExpr
169f60f7c76cb [clang][driver] Add avr-libc's default linker script to lld (#68507)
0d661e965ad1a [clangd] Use DenseMap::contains (NFC)
ab6d5fa3d0643 [Sparc] Use isNullConstant (NFC)
2dc6579f6fb12 clang: Switch SYCL test to generated checks
557299c9b6464 [ELF][test] Test relocations referencing weak symbol, which is defined relative to a section discarded by /DISCARD/
4fb49f44fdf55 [ELF][test] Test relocations referencing symbols relative to sections discarded by /DISCARD/
bb6a98c8d2bee [clang-tidy] Ignore unused parameters in `rvalue-reference-param-not-moved check` (#69045)
0603737ac0f73 [llvm-remarkutil] Fix issues after #66214 and its fixups
8dd3bc1808165 [Support] Remove the migration helpers to llvm::endianness
80737d2ddf055 [clang][Interp][NFC] Pass PrimType to visitZeroInitializer()
0ad92c0cbb34a [StatepointLowering] Take return attributes of `gc.result` into account (#68439)
c442d20e23d73 [gn build] Port f445be9790f9
f4a0cb5c95939 [gn build] Port 311bc6683dee
311bc6683deea [AArch64][compiler-rt] Only build SME ABI routines when compiler supports asm. (#68991)
649c2f6c1081a [mlir][mlir-query] Fix shared build. NFC
6620376270165 [RISCV] Add CompressPat for c.beqz/bnez with X0 in the first operand. (#69042)
343bed8d3a9b6 Canonically identical types are allowed in compound expressions in C
5e1c2bf3e6fca [AArch64][GlobalISel] Expand converage of FMA.
e7a6171c4c048 [clang] Enable Wenum-constexpr-conversion also in system headers and … (#67528)
a502dddfd0da8 [AArch64] Additional GISel test for FMA. NFC
769bc11f684d3 [Clang] Implement the 'counted_by' attribute (#68750)
7060422265902 [mlir][Linalg]: Optimize linalg generic in transform::PromoteOp to avoid unnecessary copies (#68555)
a653749acab8d [clang][Interp] Implement compound assign operators on bitfields (#67306)
93229c7bfd974 [lldb] Add SBType::FindDirectNestedType() function (#68705)
02f67c097de12 Use llvm::endianness::{big,little,native} (NFC)
514381840c6d7 [RISCV] Move hasOneUse() call after opcode check.
ece5dd101c7e4 [clang] Stop including llvm/ADT/StringMap.h (NFC)
64e7207ea5c77 [Test] Pre-submit tests for #68972 (#69040)
643b2ccd8296a [tools] Stop including llvm/ADT/StringMap.h (NFC)
18d199116fe21 Stop including llvm/ADT/STLFunctionalExtras.h (NFC)
3750558ee1b0b [RISCV][GISel] Legalize G_SMULO/G_UMULO (#67635)
6e8013a1301ef [llvm] Stop including llvm/ADT/StringMap.h (NFC)
3743c53dd19fd [clang] Remove unused using decls (NFC)
18dc8dcd768fd [PowerPC][JITLink] Support R_PPC64_GOT_TLSGD_PCREL34 (#68660)
eab5d337f0f62 [BOLT] Use llvm::erase_if (NFC)
f3cfd3812b4a7 [AST] Stop including llvm/ADT/StringMap.h (NFC)
70fedaf89b35c [libc++][NFC] Fix slightly incorrect comment in PSTL documentation
f2b79ed9c6c85 [libcxx] Refactoring SIMD function names in PSTL CPU backend (#69029)
25d93f3f6843d NFC: Precommit GISel checks for arm64-indexed-memory.ll
2f80dfc07978c [GlobalISel][NFC] Add distinct CHECK/SDAG/GISEL run lines to test.
9dbfd5828e3b2 [CodeLayout] CDSortImpl: remove two conditions that cannot trigger. NFC
53c81a8c165dc [RISCV][SDAG] Fix constant narrowing when narrowing loads (#69015)
9d1a3fdd62781 [RISCV][GISel] Add isel patterns for ADDIW/SRLIW/SRAIW/SLLIW and remove custom selection. (#68470)
a669a237c45a5 [LLDB] Fix buildbots
02d9f4d1f128e [mlir][mlir-query] Introduce mlir-query tool with autocomplete support
1673a1ba5decd [LLDB][NFC] Create a namespace for the DWARF plugin (#68150)
d9ede91a27fe7 [scudo] Fix type mismatch in trusty. (#69024)
e220398cc3aea [MSVC, ARM64] Add __prefetch intrinsic (#67174)
eb4a061568a39 [gn] port 99d92d18e334
99d92d18e334d [scudo] Add specific die functions for linux specific failures. (#68650)
6dbc6dfe79b33 Reland "[mlir][arith] Canonicalization patterns for `arith.select` (#67809)" (#68941)
b1115f8ccefb3 [LV] Use LatchVPBB directly instead of going through region (NFC).
0dfcfb53d7bba [lldb][DataFormatter] VectorType: fix format for arrays with size not a power-of-2 (#68907)
a8896e57f150a [clang-format][NFC] Annotate control statement r_braces (#68621)
1bc48716957e2 [libc++][NFC] Fix typo in filename
36bb134ac79c9 [libc++] Use -nostdlib++ on GCC unconditionally (#68832)
844c731f2dda3 [HWASAN] Mark built-ins as not built-ins to prevent optimizations (#68936)
b49a0dbaebc7f [AMDGPU] Fix comments about afn and arcp in fast unsafe fdiv handling (#68982)
ed0a14144ba98 [LLDB] Fix type formatting empty c-strings (#68924)
475e154331af1 [libc++] Introduce make_test_jthread for jthread tests (#68837)
158c0529901ec [LLDB][NFC] Remove dead code (#68927)
72307960bf467 [mlir] Fix distinct attr mismatch error reporting (#68938)
8e2bd05c4e868 [lldb] Fix `po` alias by printing fix-its to the console. (#68755)
bbecd422a9bf5 [mlir][sparse] cleanup sparse tensor materialization parameter setup (#68956)
3d75c7c11b5a9 [CodeLayout] Fixing initialization of empty ranges (#68917)
05bde3cc23b05 [llvm][TableGen][Jupyter] Link to tutorial notebook from README
20f39bf482185 Lower std::string's alignment requirement from 16 to 8. (#68807)
160e8eb449610 [ASan] Recognize lea r10, [rip + XX] (#68910)
7493d45408c34 [lldb][DataFormatter] unordered_map: account for new libc++ __hash_node layout (#68574)
2c9ddfc7852ed [mlir][Tosa] fix fp16/bf16 support for AvgPool2d (#68718)
74c5e474043da [lldb][test] Temporarily disable TestQueueFromStdModule.py (#68970)
ba79fb2e1ff71 [libc++] Re-apply "Remove UB in list, forward_list and __hash_table"
b22917e6e2a0a [InstCombine] Fold Ext(i1) Pred shr(A, BW - 1) => i1 Pred A s< 0 (#68244)
d2aa523f2a2ef [gn build] Port 2cea1babefbb
e4e02e31c24fa [AArch64][NFC] Refactor NEON, SVE and SME classes and multiclasses fo… (#68800)
3f4bf998e8972 [gn] port 46cb8d9a3252
3efa4794ecd5c [clang][Interp] Support AddOffset with 128bit offsets (#68679)
8d59fc5fd1599 Fix typo in CMake var
5d35273a32d23 [lldb] fix release build (#68979)
dbb9faedec5e2 Re-apply '[AArch64] Enable "sink-and-fold" in MachineSink by default (#67432)'
afdb18df4d43d [NFC][AArch64][LV] Reorganise LV tests using symbols from SLEEF (#68207)
7025ff6fa3dfe [MLIR][Presburger] clang-format and clang-tidy
7ef1754301a88 [mlir][arith] Fix canon pattern for large ints in chained arith (#68900)
c6f065d9d9973 [BOLT][RISCV] Recognize mapping syms with encoded ISA (#68964)
2e955c0504d4c Revert "[Sema] Add check for bitfield assignments to integral types" (#68963)
f54dc7b3936f1 [mlir][ODS] Omit printing default-valued attributes in oilists (#68880)
92e751d426dbc [mlir][linalg] Add NHWC + FHWC Img2Col (#68708)
41418ca13323c [NFC] Fixed typo in LangRef.rst (#68789)
9a8ff346bb20a [mlir][doc] Trim summary text during DocGen (#68477)
69b6b48670c79 [ADT] Stop including identity.h (NFC)
187e02fa2deda [CodeGenPrepare] Check types when unmerging GEPs across indirect branches (#68587)
06cd6485ae2d2 [AMDGPU] Make ubsan happy (#68959)
ac32d7b87f4d4 Revert "Add Documentation for Execution Results Handling in Clang-Repl (#65650)"
4f4694509d293 [llvm-remarkutil] Silence a GCC warning about missing returns after a fully covered switch. NFC.
47000586caca4 [llvm-remarkutil] Fix building with Xcode 12
411ceacf4351b [Clang] Fix tautological assertion in `Sema::CheckX86BuiltinTileDuplicate`
7755cdf03d2f2 [flang][runtime] Fix IsContiguous for zero and one element arrays (#68869)
9bcc094d37670 [llvm] Use llvm::erase_if (NFC)
60b3e05967ff5 [ELF] Restore the --call-graph-profile-sort=hfsort default before #68638
398e48a75ba1c [mlir] Apply ClangTidy fix (NFC)
9bd5bfc689a78 [mlir][sparse] remove unused sparse tensor iterator (#68951)
28b27c1b10ae8 [ArmSVE][NVVM][Bazel] Added Features to BUILD.bazel file (#68949)
37a5304976584 [Support] Stop including cstddef (NFC)
4a0ccfa865437 Use llvm::endianness::{big,little,native} (NFC)
2045cca0c3d27 [mlir][sparse] add a forwarding insertion to SparseTensorStorage (#68939)
c40902c41c007 [AMDGPU] Use llvm::endianness::little (NFC)
797b76791df4d [IR] Move isConvergenceControlIntrinsic under "namespace llvm" (NFC)
b29fb9c9f4ae1 [llvm] Remove "using namespace llvm;" from header files (NFC)
3104681686b17 [PowerPC][Atomics] Remove redundant block to clear reservation  (#68430)
bf90ffb9b4617 [SVE][InstCombine] Delete redundante sel instructions with ptrue
127cf4ead3f8e [SVE][InstCombine] Precommit tests for select + ptrue
ebaf8d4949830 [mlir][affine] ValueBoundsConstraintSet: Fully compose affine.apply (#68899)
a712244f3b76c [PowerPC][JITLink] Support R_PPC64_GOT_PCREL34 (#68658)
fbe47bf532e83 [mlir][sparse] remove dead code from utils (#68943)
fcb4c0555e2f8 [mlir][nfc] Rename type constraint for scalable vectors (#68808)
5bf701a6687a4 Revert "[mlir][arith] Canonicalization patterns for `arith.select` (#67809)"
2cea1babefbb7 [libc++] Remove libc++'s own <setjmp.h> header (#68806)
cd0d478e7cfa4 quick fix for TestDumpDwo
282ea28f18cf0 [mlir][nvvm] Fix circular dependency in (#68934)
46cb8d9a32523 [TSAN] add support for riscv64 (#68735)
b90fcafcd68f7 [CodeLayout][NFC] Using MergedVector to avoid extra vector allocations (#68724)
8da1e3dd24a1c [Driver] Have -rdynamic be a no-op on Haiku (#67872)
ab6c3d50345f7 [AMDGPU] Change the representation of double literals in operands (#68740)
b3a39a9bdb6b3 [libc++] Check formatting with clang-format 17 (#68928)
403e0e8cd95f2 [clang][Interp] Fix crash during `InterpStack` printing (#68246)
2dd52b4527667 [InstCombine] Improve logic for adding flags to shift instructions.
968468af9c680 [InstCombine] Tests for adding flags to shifts; NFC
444383e0d07cd [ValueTracking] Do more thorough non-zero check in `isKnownToBePowerOfTwo` when `OrZero` is no set.
dfda65c89272e [ValueTracking] Add support for non-splat vecs in cmpExcludesZero Just a small QOL change.
9427fce6778c8 [ValueTracking] Add tests for `cmpExcludesZero` for non-splat vecs; NFC
2ae3a71230487 Fix minimal-throw-catch.ll on x86 mac
ef388334ee5a3 Revert "Reapply "InstCombine: Introduce SimplifyDemandedUseFPClass""
f248d0b28dca4 [mlir][sparse] implement sparse_tensor.reorder_coo (#68916)
220244b71ba2a [gn build] Port 1c9035d1b5a9
1c9035d1b5a9a [SystemZ][z/OS] Add UtcClock extension to chrono.h/.cpp (#67846)
50ece4cba9497 [ValueTracking] Add better support for ConstantRange(And)
0f8b40a82ebee [ValueTracking] Add better support for ConstantRange(Shl)
457308a46a37f [ValueTracking] Add more tests for constant ranges; NFC
b0c769a80b5f0 [libc++][mdspan] Fix extents CTAD (#68737)
64d78d8b3cd09 Add `target modules dump separate-debug-info` (#66035)
dd0f642e6ec50 [Sema] Add check for bitfield assignments to larger integral types (#68276)
4c6cba31aaaa7 [clang][NFC] Specify Type and ExtQuals as having 16-byte alignment (#68377)
b56488c8790a8 [libc++] Improve the output of the generated-output CI job (#68903)
9b89b80dbafed [gn build] Port 31c2cf113617
7dcb260bef9f7 [gn] port 0ce6255a5058 (HipStdPar)
b44b3494f6029 [mlir][ArmSVE] Avoid UBSAN issue with VectorType::Builder (NFC)
0aacc2137a80c [mlir][sparse] introduce sparse_tensor.reorder_coo operation (#68827)
cff50072a0573 [gn] port f445be9790f9
7b12d8bf8a1ff [clang][Tests] Fix shared build. NFC
c136e722aa4b0 [Remarks] Fix '-fpermissive'. NFC
b98b567c25015 [ARM] Correctly handle .inst in IT and VPT blocks (#68902)
ef9f6170b6164 [Flang][OpenMP] NFC: Fix test
a574ef61766d4 [AArch64] Fix incorrect big-endian spill in foldMemoryOperandImpl (#65601)
31c2cf1136179 [Remarks] Introduce count subcommand for llvm-remarkutil. (#66214)
ab6a66dbec616 Reland: [MLIR][Transforms] Fix Mem2Reg removal order to respect dominance (#68877)
8593c0bc025bc LoopVectorize/test: clean up reduction.ll; generate using UTC (NFC) (#68890)
b833bcb559478 [mlir][ArmSVE] Add convert_to/from_svbool ops (#68586)
1c12dcc910325 [InstCombine] Extend `sext`/`zext` boolean additions to vectors
c3356ac3b9e50 [llvm-readtapi] Fix bad symlink with readtapi tool (#68835)
b9b8fc4878b6f Add Documentation for Execution Results Handling in Clang-Repl (#65650)
7603c77f23767 [IR][NFC] Assert that FunctionType::getParamType is in-bounds (#68887)
25da11541c074 [mlir][nvvm] Move `BasicPtxBuilder` Interface to Its Own File (NFC) (#68095)
f445be9790f99 Reland "[compiler-rt][aarch64] Add SME ABI support routines." (#68875)
d093aee1a33e0 [AArch64] Fix schedmodel pre/post-index loads and stores for TSV110
2c1f37b3b9523 [test] precommit sched model for tsv110, NFC
30240e428f0ec [PhaseOrdering] Regenerate test checks (NFC)
bdc66912a81af [gn] port 2b05fa8f0be7 (InterpTests)
593521b079e09 [libcxx] <experimental/simd> Add generate constructor of class simd (#66737)
30faaaf62670a [LoopVectorize] Regenerate test checks (NFC)
e44c9fd197a22 [Clang] Regenerate test checks (NFC)
f5c34126d4296 [bazel] add llvm/HipStdPar (#68883)
8dcee5800caeb [flang]Check for dominance in loop versioning (#68797)
f35e2d8022ba5 [flang][openmp] Don't mark loop variables with explicit DSA as predetermined (#68723)
37441f1ae66e6 [PhaseOrdering] Add test for switch with different GEP types (NFC)
434b8a30804a4 Revert "[mlir][ODS] Omit printing default-valued attributes in oilists (#68694)"
e98a4bb00d555 [mlir][ODS] Omit printing default-valued attributes in oilists (#68694)
28e3552e3ebf1 [clang][Tests] Add clangTooling dep to Interp unittest
b9bc1cedeadc3 [AMDGPU] Update ASAN tests with update_test_checks. (#68688)
e54b0bef53bca [clang][dataflow] Various formatting tweaks to HTMLLogger. (#68392)
072713add4408 Revert "[compiler-rt][aarch64] Add SME ABI support routines." (#68875)
2f43ace0f48f3 [clang][Interp] Fix expected values for Pointer API
22b6b8d7b5310 [clang][Interp][NFC] Remove Pointer.h include from Function.h
bf0534e8702a1 [llvm-lib] [Object] Use ECSYMBOLS section for ARM64EC importlib symbols. (#68328)
6cf41ada44c81 [SystemZ][z/OS] Add vararg support to z/OS (#68834)
0ce6255a50584 [HIP][LLVM][Opt] Add LLVM support for `hipstdpar`
ac0015fe21110 [flang][driver] add command line arguments for alias tags pass
8a7e5477982bc [InstCombine] Canonicalize `(X +/- Y) & Y` into `~X & Y` when Y is a power of 2 (#67915)
2ceabf6bdc649 [MachineSink] Reduce the number of unnecessary invalidations of StoreInstrCache (NFC) (#68676)
3b13e02d6d324 [AArch64] Fix postinc operands for Cortex-A57 scheduling
86d9faa5a9761 [MachineSink] Use LLVM ADTs (NFC) (#68677)
7a18b3ce65210 [bazel] update llvm-config.h.cmake
af74f06322410 ArgList: Simplify 3-opt overload of AddAllArgs. NFC
97cd39ff09883 [bazel] fix linter complains for libc bazel
44600bae893e1 [mlir][python] Improve sanitization of python names (#68801)
2b05fa8f0be7e [compiler-rt][aarch64] Add SME ABI support routines.
e724c7e8413a6 [AMDGPU] Add lit() asm operand modifier for SP3 compatibility. (#68839)
127ed9ae266ea [PowerPC] Use zext instead of anyext in custom and combine (#68784)
956482de13107 [LoongArch] Support finer-grained DBAR hints for LA664+ (#68787)
9dd15f7486a30 [mlir][tosa] Add aggressiveReduceConstant argument for the constant reduce optimization (#68765)
28b912687900b [BasicBlockSections] Introduce the path cloning profile format to BasicBlockSectionsProfileReader. (#67214)
ff16c9878c274 [mlir][Bazel] Don't duplicate files in data and test attribute.
b7944f7c04292 [BOLT] Return proper minimal alignment from BF (#67707)
f98c36efefdc2 [Support] Stop including MathExtras.h (NFC)
e80cdf0ef9673 [Affine][Bazel] Added AffineOps to BUILD.bazel file (#68842)
f32b3e1caa8f2 [mlir][memref] Fix index delinearization for CollapseShapeOp folding (#68833)
894927b491b7c [Driver] Group together linker options using addAllArgs (#68349)
dcf9d946f5b5b [PowerPC][Atomics] Simplify atomicrmw i128 patterns. NFC. (#68779)
b5c2fa14ea771 [libc] Mark operator== const to avoid ambiguity in C++20. (#68805)
1379a7286e156 [XCOFF][OBJECT] get symbol size by calling XCOFF interfaces (#67304)
c661c4f57613b [AIX] recognize vsr in inline asm for AIX (#68476)
b48450c2094fa [MergeFuncs] Use sizeWithoutDebug to decide if we create a thunk (#68627)
b6043f98673e3 [RA] Disable split around hint register if optimize for size (#68619)
1bf0870934542 [mlir][Transform] Create a transform interpreter and a preloader pass (#68661)
098c92777ee43 [lldb-vscode] Update installation instructions (#68234)
de7857ab234db [mlir][python] python binding for the affine.store op (#68816)
52d0696355193 [clang][dataflow] Add support for lambda captures (#68558)
eb21049b4b904 [libunwind] Avoid reading OOB for non-existent .eh_frame_hdr (#68815)
05181a849b4c1 [libunwind] Fix wrong end argument passed to decodeEHHdr() (#68813)
325576196bf92 [mlir][sparse] remove tests (#68826)
776e456dd883f [llvm-cov gcov] Make recursive propagateCounts iterative. NFC
8d520973b02b5 [X86] Use indirect addressing for high 2GB of x32 address space
837a26f237fbd [mlir][sparse] fix unused variable warning on release build (#68824)
28b7e281d4eae [Support] Add `\g<ref>` backreferences in Regex::sub() (#67220)
c1b6ed42b75a1 [AArch64] Fix postinc operands for Neoverse-N2 scheduling
ad547cecf4d1d [flang] Add missing dependency FIRCodeGen -> FIRAnalysis
dda3dc5e38118 [mlir][sparse] simplify ConvertOp rewriting rules (#68350)
12b87f6ef7200 [Driver] Silence stdlib warning when linking C on FreeBSD (#68011)
ea1ae115bd835 [asan] Ensure __asan_register_elf_globals is called in COMDAT asan.module_ctor (#67745)
6042c2eb9edc5 [flang] use TBAAForest in TBAABuilder
db1d40f319141 [mlir][sparse] refactor dim2lvl/lvl2dim passing into MapRef (#68649)
85d0fbeaae469 [lld] Don't allow -dynamicbase:no on ARM64EC.
26b2b5a5eafe9 [flang][openacc] Relax type check for private recipe on acc.serial (#68814)
6d88ac11d7d73 [mlir][bufferization] Transfer `restrict` during empty tensor elimination (#68729)
a12e747c4c6d3 [SlotIndexes] Simplify getInstructionFromIndex. NFCI.
7ddf6e915c72d [SlotIndexes] Use upper/lower bound terminology for MBB searches. NFC. (#68802)
c0f453c02322a [flang] add missing dependency FIRTransforms -> FIRAnalysis
e41eaf4896d96 [OpenMPIRBuilder] Add ThreadLimit and NumTeams clauses to teams construct (#68364)
315ab3c44b56b [MLIR][NVGPU] Introduce `warpgroup.init.accumulator` Op (#67530)
660a78fd7db5a Revert "Reland: [MLIR][Transforms] Fix Mem2Reg removal order to respect dominance (#68767)"
4313351c135cc [clang] __is_trivially_equality_comparable for types containing lambdas (#68506)
3bfc5eb97cff7 [Flang][OpenMP] NFC: Port a few parallel tests to HLFIR flow
d88d9834e985a [AArch64][GlobalISel] Support more types for TRUNC (#66927)
28bb2193f6d3b [mlgo][nfc] Remove / fix vestigial references to Tensorflow
b15b84610f631 [SlotIndexes] Simplify SlotIndex() and isSameInstr(). NFC.
1d2b558265bd9 [AArch64][PAC] Check authenticated LR value during tail call
df5c27869c84e [flang][FIR] add FIR TBAA pass
02678bc91afba [Local] Merge constant / non-constant code paths (NFC)
8301e485001b9 [flang][FIR] add FirAliasAnalysisOpInterface (#68317)
2b74db6c9bff7 [ConstraintElim] Add test for #68751 (NFC)
ebf1feaeda124 [libcxx] [docs] Update the MinGW build example (#68790)
06322d43d560c [libcxx] [docs] Remove mention of a MSVC/Debug mode caveat (#68791)
76abab9dc1861 [libc++] Run clang-format on newly-added test to fix CI
ac2722873b26e Revert "Diagnose problematic uses of constructor/destructor attribute (#67673)"
27ecb63c26040 Diagnose problematic uses of constructor/destructor attribute (#67673)
670034c4b888f Fix LLVM sphinx bot
0cb6c413b1883 Revert "Diagnose problematic uses of constructor/destructor attribute (#67673)"
6383785bad524 [SVE][CodeGenPrepare] Sink address calculations that match SVE gather/scatter addressing modes. (#66996)
a633a3761fcbd Diagnose problematic uses of constructor/destructor attribute (#67673)
e8cc230ef5592 [flang] Use object before converts in fir.dispatch (#68589)
25935c384dd83 Revert "[HIP][LLVM][Opt] Add LLVM support for `hipstdpar`"
c5bba7ea5a05f [HIP][LLVM][Opt] Add LLVM support for `hipstdpar`
cacfac416c959 [TableGen] New bang operator `!repr`. (#68716)
f635ab3ffd85b [AArch64] Fix postinc operands for Neoverse-N1 scheduling
720be6c535ac4 [AMDGPU] Add encoding/decoding support for non-result-returning ATOMIC_CSUB instructions (#68684)
37b93f07cd7ba [LoongArch] Add some atomic tests (#68766)
0ead1faef0bfa [PowerPC] Add test for #68783 (NFC)
20799fd57bcbc [OCaml][test] Use correct data layout string. (#68781)
fac4206e66bc8 [EarlyIfConversion] Simplify condition after #65729
05c16f40c9de9 [VirtRegMap] Simplify condition after #65729
fa5884770ab24 [GVN] Always require LoopInfo
a21abc782a8e1 [X86] Align i128 to 16 bytes in x86 datalayouts
84ff49d2180c3 [AArch64] Fix postinc operands for Neoverse-V1 scheduling
2754f88a09b19 [mlir][docgen] Add option to enable Hugo Docs specific features (#68725)
4803ba9ecbfcc [clang][Interp] Remove expected-no-directives lines
b78f3ea7df3c5 Clean up strange uses of getAnalysisIfAvailable (#65729)
2888fa4313dcd [AMDGPU] Update test remat-smrd.mir
d5444ab267431 [clang][C2x] Remove confusing diagnostic auto storage class specifier (#68710)
c3f67b3ba9c8c [clang][Interp] Limit MaxBitInt to 128 bits
43e9eae6e883f [BOLT] Preserve label annotations for injected functions (#68713)
da37139ac922e [BOLT][NFC] Add allocator id to MCPlusBuilder::setLabel (#68707)
59fec735950bd Reland: [MLIR][Transforms] Fix Mem2Reg removal order to respect dominance (#68767)
29f37f812f80e CycleInfo: Fix splitCriticalEdge (#68584)
bccf2c8468e2f [clang][RISCV] Fix the condition of checking signature in getIndex (#67403)
26d9f851cfddf [clang][Interp] Add basic support for _BitInt (#68069)
d97947e1969bb [Docs] Fix broken LLVM doc pages (#68631)
b8885926f8115 Use llvm::endianness::{big,little,native} (NFC)
0439a017ef72e [llvm] Drop unaligned from calls to llvm::support::endian::{read,write} (NFC)
962005327237d [mlir][Bazel] Adjustments for 8c67c48591fbf76559a8b39e68c88c4da0a40218
8b3e150bbb3f2 [mlir][Bazel] Fix wrong dependency path.
c8e17749757a6 [Interp] Use llvm::endianness::native (NFC)
e9ca1665c90db [Mesh][Bazel] Added MeshOps to BUILD.bazel file (#68759)
a9d50568625e4 Use llvm::endianness (NFC)
d95a6c949f922 [libc++][NFC] Fix typo in synopsis comment
4eecfda50a4e7 [Driver] Rename AddAllArgs (initialization list overload) to addAllArgs
1d8d326a86d59 [clang][driver] Support option '-r' for target AVR (#68484)
776889bc1c68d [RISCV] Add Stackmap/Statepoint/Patchpoint support without targets
255f826d6f656 [X86] Fix value-extending/truncating loads and stores of __ptr32/__ptr64 pointers (#67168)
b05dbc4d5f28e [llvm] Use llvm::endianness::{big,little,native} (NFC)
8c67c48591fbf [MLIR] Refactor FuncOpToLLVM to operate on FunctionOpInterface instead of FuncOp (NFC) (#68665)
2d854dd3e7c5a Move global namespace cl::opt inside llvm:: or internalize them
f3c92a06b96a5 [RISCV] Make PostRAScheduler a target feature (#68692)
15d78cef3ddc9 [Clang][PowerPC] Support tune CPU in target attribute (#68681)
203ba238e33c5 [LoongArch] Improve codegen for atomic ops (#67391)
a157e79dd207d [FlowSensitive] Stop including llvm/ADT/Any.h (NFC)
d12ece2ecbb60 [CodeGen] Use a range-based for loop (NFC)
b3ec0595d38cf [llvm] Drop unaligned from calls to llvm::support::endian::{read,write} (NFC)
8f64d52cb8b84 [flang] Remove unused using decls (NFC)

git-subtree-dir: thirdparty/llvm-project
git-subtree-split: 5c3ed392fcda40df20d3def7a378397048bc6fa5
---
 .ci/generate-buildkite-pipeline-premerge      |    5 +-
 .github/new-prs-labeler.yml                   |   32 +
 .github/workflows/docs.yml                    |   94 +
 .../libcxx-check-generated-files.yml          |   24 +
 .github/workflows/release-tasks.yml           |    6 +-
 bolt/include/bolt/Core/BinaryFunction.h       |   21 +-
 bolt/include/bolt/Core/MCPlusBuilder.h        |    2 +-
 bolt/lib/Core/BinaryContext.cpp               |   21 +-
 bolt/lib/Core/BinaryEmitter.cpp               |    2 +-
 bolt/lib/Core/BinaryFunction.cpp              |    9 +-
 bolt/lib/Core/DebugData.cpp                   |   91 +-
 bolt/lib/Core/HashUtilities.cpp               |    5 +-
 bolt/lib/Core/MCPlusBuilder.cpp               |    6 +-
 bolt/lib/Passes/Aligner.cpp                   |   16 -
 bolt/lib/Passes/BinaryPasses.cpp              |    5 +-
 bolt/lib/Passes/LongJmp.cpp                   |    4 +-
 bolt/lib/Passes/ValidateMemRefs.cpp           |    3 +-
 bolt/lib/Profile/DataAggregator.cpp           |   14 +-
 bolt/lib/Rewrite/PseudoProbeRewriter.cpp      |    3 +-
 bolt/lib/Rewrite/RewriteInstance.cpp          |    6 +
 bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp  |   28 +
 bolt/test/AArch64/bf_min_alignment.s          |   35 +
 bolt/test/RISCV/Inputs/cie-gnu.yaml           |  109 +
 bolt/test/RISCV/Inputs/mapping-syms-isa.yaml  |   47 +
 bolt/test/RISCV/call-annotations.s            |    3 +-
 bolt/test/RISCV/cie-gnu.test                  |   17 +
 bolt/test/RISCV/internal-func-reloc.s         |   18 +-
 bolt/test/RISCV/mapping-syms-isa.test         |   18 +
 bolt/test/RISCV/unnamed-sym-no-entry.c        |   18 +
 .../{ => X86}/checkvma-large-section.test     |    8 +-
 bolt/test/X86/internal-call-instrument-so.s   |    3 +-
 bolt/test/reorder-data-writable-ptload.c      |    4 +-
 .../bugprone/BugproneTidyModule.cpp           |    3 +
 .../clang-tidy/bugprone/CMakeLists.txt        |    1 +
 .../bugprone/CastingThroughVoidCheck.cpp      |   45 +
 .../bugprone/CastingThroughVoidCheck.h        |   32 +
 .../RvalueReferenceParamNotMovedCheck.cpp     |    3 +
 .../clang-tidy/misc/CMakeLists.txt            |    1 +
 .../misc/CoroutineHostileRAIICheck.cpp        |   98 +
 .../misc/CoroutineHostileRAIICheck.h          |   50 +
 .../clang-tidy/misc/MiscTidyModule.cpp        |    3 +
 .../clang-tidy/modernize/AvoidBindCheck.cpp   |   15 +-
 .../clang-tidy/modernize/LoopConvertCheck.cpp |    1 +
 .../clang-tidy/tool/clang-tidy-diff.py        |   56 +-
 .../clang-tidy/tool/run-clang-tidy.py         |   60 +-
 clang-tools-extra/clangd/Config.h             |    2 +
 clang-tools-extra/clangd/ConfigCompile.cpp    |   50 +-
 clang-tools-extra/clangd/ConfigFragment.h     |    7 +
 clang-tools-extra/clangd/ConfigYAML.cpp       |    4 +
 clang-tools-extra/clangd/Hover.cpp            |    5 +
 clang-tools-extra/clangd/Hover.h              |    2 +
 clang-tools-extra/clangd/ParsedAST.cpp        |   22 +-
 .../clangd/SystemIncludeExtractor.cpp         |    3 +-
 clang-tools-extra/clangd/TidyProvider.cpp     |   15 +
 clang-tools-extra/clangd/TidyProvider.h       |    4 +
 .../clangd/index/SymbolCollector.cpp          |    2 +-
 clang-tools-extra/clangd/tool/Check.cpp       |   29 +-
 .../clangd/unittests/DiagnosticsTests.cpp     |   20 +-
 .../clangd/unittests/HoverTests.cpp           |   13 +-
 .../clangd/unittests/ParsedASTTests.cpp       |    6 +
 .../clangd/unittests/ReplayPeambleTests.cpp   |    7 +
 .../clangd/unittests/TidyProviderTests.cpp    |   18 +
 clang-tools-extra/docs/ReleaseNotes.rst       |   34 +-
 .../docs/clang-tidy/Integrations.rst          |    2 +-
 .../checks/bugprone/casting-through-void.rst  |   31 +
 .../rvalue-reference-param-not-moved.rst      |   10 +
 .../docs/clang-tidy/checks/list.rst           |    2 +
 .../checks/misc/coroutine-hostile-raii.rst    |   50 +
 .../checks/misc/definitions-in-headers.rst    |    5 +
 .../checks/readability/identifier-naming.rst  |   68 +-
 .../unittests/LocateSymbolTest.cpp            |    2 -
 .../absl/types/optional.h                     |    2 +
 .../bugprone/casting-through-void.cpp         |   91 +
 .../bugprone/unchecked-optional-access.cpp    |   20 +
 .../rvalue-reference-param-not-moved.cpp      |   25 +
 .../checkers/misc/coroutine-hostile-raii.cpp  |  192 +
 .../checkers/modernize/avoid-bind.cpp         |   43 +-
 .../checkers/modernize/loop-convert-basic.cpp |   14 +
 clang/cmake/caches/Fuchsia-stage2.cmake       |    1 +
 clang/docs/ClangFormatStyleOptions.rst        |   27 +
 clang/docs/HIPSupport.rst                     |   90 +
 clang/docs/LanguageExtensions.rst             |    8 +
 clang/docs/ReleaseNotes.rst                   |   80 +-
 clang/docs/StandardCPlusPlusModules.rst       |    9 +
 clang/docs/analyzer/checkers.rst              |  195 +-
 clang/docs/tools/clang-formatted-files.txt    |    1 -
 clang/docs/tools/dump_format_style.py         |   76 +-
 clang/include/clang-c/Index.h                 |    1 +
 clang/include/clang/ARCMigrate/FileRemapper.h |    4 +-
 clang/include/clang/AST/Decl.h                |   24 +
 clang/include/clang/AST/DeclBase.h            |   10 +
 clang/include/clang/AST/DeclCXX.h             |    6 +
 clang/include/clang/AST/Expr.h                |    7 +
 clang/include/clang/AST/ExprCXX.h             |    8 +-
 clang/include/clang/AST/Type.h                |   24 +-
 clang/include/clang/AST/TypeLoc.h             |    2 +-
 .../clang/Analysis/Analyses/ThreadSafety.h    |    8 +-
 .../clang/Analysis/FlowSensitive/Arena.h      |   10 +-
 .../Analysis/FlowSensitive/DataflowAnalysis.h |    1 -
 .../FlowSensitive/DataflowAnalysisContext.h   |    6 +-
 .../FlowSensitive/DataflowEnvironment.h       |   11 +-
 .../clang/Analysis/FlowSensitive/Formula.h    |   16 +-
 clang/include/clang/Basic/Attr.td             |   23 +
 clang/include/clang/Basic/AttrDocs.td         |   78 +
 clang/include/clang/Basic/Builtins.def        |    1 +
 clang/include/clang/Basic/BuiltinsAArch64.def |    2 +
 clang/include/clang/Basic/BuiltinsNVPTX.def   |   12 +-
 clang/include/clang/Basic/BuiltinsX86_64.def  |    3 +
 .../include/clang/Basic/DiagnosticASTKinds.td |    5 +-
 .../clang/Basic/DiagnosticDriverKinds.td      |    4 +
 clang/include/clang/Basic/DiagnosticGroups.td |   11 +-
 .../clang/Basic/DiagnosticSemaKinds.td        |   51 +-
 .../Basic/DiagnosticSerializationKinds.td     |    2 +-
 clang/include/clang/Basic/FileEntry.h         |   31 -
 clang/include/clang/Basic/LangOptions.h       |    3 +-
 clang/include/clang/Basic/Module.h            |   12 +-
 clang/include/clang/Basic/Specifiers.h        |    2 +
 clang/include/clang/Basic/TokenKinds.def      |    1 +
 clang/include/clang/Basic/arm_sme.td          |   10 +-
 clang/include/clang/Basic/arm_sve.td          |  112 +
 clang/include/clang/Basic/arm_sve_sme_incl.td |    5 +-
 .../clang/Basic/riscv_sifive_vector.td        |   24 +
 clang/include/clang/Driver/Options.td         |   65 +-
 clang/include/clang/Format/Format.h           |   30 +-
 .../clang/Frontend/VerifyDiagnosticConsumer.h |    9 +-
 clang/include/clang/Lex/ModuleMap.h           |    4 +-
 clang/include/clang/Sema/Sema.h               |   80 +-
 clang/include/clang/Sema/Template.h           |    4 +-
 clang/include/clang/Sema/TypoCorrection.h     |   12 +-
 .../clang/StaticAnalyzer/Checkers/Checkers.td |   43 +-
 .../clang/Support/RISCVVIntrinsicUtils.h      |   18 +-
 clang/lib/APINotes/APINotesWriter.cpp         |   74 +-
 clang/lib/ARCMigrate/FileRemapper.cpp         |   55 +-
 clang/lib/AST/ASTContext.cpp                  |  218 +-
 clang/lib/AST/ASTImporter.cpp                 |   21 +-
 clang/lib/AST/DeclBase.cpp                    |   77 +-
 clang/lib/AST/DeclCXX.cpp                     |    2 +-
 clang/lib/AST/Expr.cpp                        |   83 +-
 clang/lib/AST/ExprCXX.cpp                     |   16 +-
 clang/lib/AST/ExprConstant.cpp                |   14 +
 clang/lib/AST/Interp/Boolean.h                |    1 +
 clang/lib/AST/Interp/ByteCodeEmitter.cpp      |    2 +-
 clang/lib/AST/Interp/ByteCodeExprGen.cpp      |   89 +-
 clang/lib/AST/Interp/ByteCodeExprGen.h        |    5 +-
 clang/lib/AST/Interp/ByteCodeStmtGen.cpp      |    8 +
 clang/lib/AST/Interp/Context.h                |    5 +
 clang/lib/AST/Interp/Descriptor.cpp           |    6 +
 clang/lib/AST/Interp/Function.h               |    3 +-
 clang/lib/AST/Interp/Integral.h               |    3 +
 clang/lib/AST/Interp/IntegralAP.h             |  131 +-
 clang/lib/AST/Interp/Interp.h                 |   81 +-
 clang/lib/AST/Interp/InterpBuiltin.cpp        |    3 +-
 clang/lib/AST/Interp/InterpStack.cpp          |   20 +-
 clang/lib/AST/Interp/Opcodes.td               |   45 +-
 clang/lib/AST/Interp/Pointer.h                |   17 +-
 clang/lib/AST/Interp/Source.h                 |    2 +-
 clang/lib/AST/ItaniumMangle.cpp               |    9 +
 clang/lib/AST/Type.cpp                        |   10 +-
 clang/lib/AST/TypePrinter.cpp                 |    6 +
 clang/lib/ASTMatchers/ASTMatchersInternal.cpp |    2 +-
 clang/lib/ASTMatchers/GtestMatchers.cpp       |    1 -
 clang/lib/Analysis/FlowSensitive/Arena.cpp    |  100 +-
 .../FlowSensitive/DataflowAnalysisContext.cpp |   20 +-
 .../FlowSensitive/DataflowEnvironment.cpp     |   24 +-
 clang/lib/Analysis/FlowSensitive/Formula.cpp  |   17 +-
 .../lib/Analysis/FlowSensitive/HTMLLogger.css |    7 +-
 .../Analysis/FlowSensitive/HTMLLogger.html    |    2 +-
 .../Models/UncheckedOptionalAccessModel.cpp   |    2 +-
 .../TypeErasedDataflowAnalysis.cpp            |   35 +-
 .../FlowSensitive/WatchedLiteralsSolver.cpp   |    3 +
 clang/lib/Analysis/ThreadSafety.cpp           |   78 +-
 clang/lib/Analysis/UnsafeBufferUsage.cpp      |   62 +-
 clang/lib/Basic/Module.cpp                    |    8 +-
 clang/lib/Basic/Sarif.cpp                     |    1 -
 clang/lib/Basic/SourceManager.cpp             |    2 +-
 clang/lib/Basic/Targets/ARM.cpp               |    3 +-
 clang/lib/Basic/Targets/M68k.cpp              |   10 +
 clang/lib/Basic/Targets/M68k.h                |    1 +
 clang/lib/Basic/Targets/Mips.cpp              |    4 +
 clang/lib/Basic/Targets/Mips.h                |   13 +
 clang/lib/Basic/Targets/OSTargets.h           |    4 +-
 clang/lib/Basic/Targets/PPC.cpp               |    8 +-
 clang/lib/Basic/Targets/PPC.h                 |    2 +
 clang/lib/Basic/Targets/RISCV.cpp             |    8 +-
 clang/lib/Basic/Targets/X86.cpp               |   47 +-
 clang/lib/Basic/Targets/X86.h                 |   46 +-
 clang/lib/CodeGen/BackendUtil.cpp             |    5 +
 clang/lib/CodeGen/CGBuiltin.cpp               |  193 +-
 clang/lib/CodeGen/CGCall.cpp                  |   10 +-
 clang/lib/CodeGen/CGDebugInfo.cpp             |    2 +
 clang/lib/CodeGen/CGDeclCXX.cpp               |    4 +-
 clang/lib/CodeGen/CGExpr.cpp                  |   64 +-
 clang/lib/CodeGen/CGExprScalar.cpp            |    3 +
 clang/lib/CodeGen/CGObjC.cpp                  |    2 +-
 clang/lib/CodeGen/CGStmt.cpp                  |   37 +-
 clang/lib/CodeGen/CMakeLists.txt              |   21 +
 clang/lib/CodeGen/CodeGenFunction.cpp         |   17 +-
 clang/lib/CodeGen/CodeGenFunction.h           |   12 +-
 clang/lib/CodeGen/CodeGenModule.cpp           |    7 +-
 clang/lib/CodeGen/CodeGenPGO.cpp              |    6 +-
 clang/lib/Driver/Driver.cpp                   |    5 +-
 clang/lib/Driver/Multilib.cpp                 |    6 +-
 clang/lib/Driver/OffloadBundler.cpp           |    2 +-
 clang/lib/Driver/ToolChains/AIX.h             |    2 +-
 clang/lib/Driver/ToolChains/AVR.cpp           |   51 +-
 clang/lib/Driver/ToolChains/Arch/ARM.cpp      |    5 +
 clang/lib/Driver/ToolChains/Arch/RISCV.cpp    |    4 +-
 clang/lib/Driver/ToolChains/Arch/X86.cpp      |   30 +
 clang/lib/Driver/ToolChains/BareMetal.cpp     |    5 +-
 clang/lib/Driver/ToolChains/CSKYToolChain.cpp |    5 +-
 clang/lib/Driver/ToolChains/Clang.cpp         |   72 +-
 clang/lib/Driver/ToolChains/CommonArgs.cpp    |    2 +-
 clang/lib/Driver/ToolChains/Cuda.h            |    2 +-
 clang/lib/Driver/ToolChains/Darwin.cpp        |    2 +-
 clang/lib/Driver/ToolChains/DragonFly.cpp     |  104 +-
 clang/lib/Driver/ToolChains/DragonFly.h       |    3 +-
 clang/lib/Driver/ToolChains/Flang.cpp         |   12 +-
 clang/lib/Driver/ToolChains/FreeBSD.cpp       |   10 +-
 clang/lib/Driver/ToolChains/FreeBSD.h         |    2 +-
 clang/lib/Driver/ToolChains/Fuchsia.cpp       |    3 +-
 clang/lib/Driver/ToolChains/Gnu.cpp           |    3 +-
 clang/lib/Driver/ToolChains/Haiku.cpp         |   10 +-
 clang/lib/Driver/ToolChains/Hexagon.cpp       |    4 +-
 clang/lib/Driver/ToolChains/Linux.cpp         |    2 +-
 clang/lib/Driver/ToolChains/MSP430.cpp        |    2 +-
 clang/lib/Driver/ToolChains/MinGW.cpp         |    1 -
 clang/lib/Driver/ToolChains/MinGW.h           |    2 +-
 clang/lib/Driver/ToolChains/NaCl.cpp          |    3 +-
 clang/lib/Driver/ToolChains/NetBSD.cpp        |   76 +-
 clang/lib/Driver/ToolChains/NetBSD.h          |    2 +-
 clang/lib/Driver/ToolChains/OpenBSD.cpp       |   11 +-
 clang/lib/Driver/ToolChains/OpenBSD.h         |    2 +-
 clang/lib/Driver/ToolChains/PS4CPU.cpp        |    8 +-
 .../lib/Driver/ToolChains/RISCVToolchain.cpp  |    9 +-
 clang/lib/Driver/ToolChains/Solaris.cpp       |   10 +-
 clang/lib/Driver/ToolChains/Solaris.h         |    2 +-
 clang/lib/Driver/ToolChains/WebAssembly.cpp   |    4 +-
 clang/lib/Driver/ToolChains/ZOS.cpp           |    5 +-
 clang/lib/Driver/ToolChains/ZOS.h             |    2 +-
 clang/lib/ExtractAPI/API.cpp                  |    1 -
 clang/lib/Format/ContinuationIndenter.cpp     |    2 +
 clang/lib/Format/Format.cpp                   |   35 +-
 clang/lib/Format/FormatToken.h                |    3 +
 clang/lib/Format/TokenAnnotator.cpp           |   35 +-
 clang/lib/Format/UnwrappedLineParser.cpp      |   43 +-
 clang/lib/Format/UnwrappedLineParser.h        |    1 +
 clang/lib/Frontend/CompilerInvocation.cpp     |   13 +-
 clang/lib/Headers/CMakeLists.txt              |    2 +
 clang/lib/Headers/intrin.h                    |    2 +
 clang/lib/Headers/stdckdint.h                 |   44 +
 clang/lib/Headers/usermsrintrin.h             |   30 +
 clang/lib/Headers/x86gprintrin.h              |    5 +
 clang/lib/Index/IndexBody.cpp                 |   18 +-
 clang/lib/Index/USRGeneration.cpp             |    9 +-
 clang/lib/Lex/HeaderSearch.cpp                |    2 +-
 clang/lib/Lex/Lexer.cpp                       |   46 +-
 clang/lib/Lex/ModuleMap.cpp                   |   18 +-
 clang/lib/Lex/PPDirectives.cpp                |    7 +-
 clang/lib/Parse/ParseDecl.cpp                 |    2 +-
 clang/lib/Parse/ParseExpr.cpp                 |    9 +-
 clang/lib/Sema/AnalysisBasedWarnings.cpp      |   12 +
 clang/lib/Sema/SemaCUDA.cpp                   |    7 +
 clang/lib/Sema/SemaChecking.cpp               |   59 +-
 clang/lib/Sema/SemaConcept.cpp                |   39 +-
 clang/lib/Sema/SemaDecl.cpp                   |   20 +-
 clang/lib/Sema/SemaDeclAttr.cpp               |   97 +
 clang/lib/Sema/SemaDeclCXX.cpp                |    5 +-
 clang/lib/Sema/SemaExpr.cpp                   |   56 +-
 clang/lib/Sema/SemaInit.cpp                   |   37 +-
 clang/lib/Sema/SemaLambda.cpp                 |    3 +-
 clang/lib/Sema/SemaOverload.cpp               |   28 +-
 clang/lib/Sema/SemaRISCVVectorLookup.cpp      |    2 +
 clang/lib/Sema/SemaTemplate.cpp               |   36 +-
 clang/lib/Sema/SemaTemplateDeduction.cpp      |    2 +-
 clang/lib/Sema/SemaTemplateInstantiate.cpp    |   22 +-
 .../lib/Sema/SemaTemplateInstantiateDecl.cpp  |    9 +-
 clang/lib/Sema/SemaType.cpp                   |   39 +-
 clang/lib/Serialization/ASTReader.cpp         |   94 +-
 clang/lib/Serialization/ASTWriter.cpp         |   25 +-
 clang/lib/Serialization/GlobalModuleIndex.cpp |   18 +-
 .../lib/Serialization/MultiOnDiskHashTable.h  |    8 +-
 .../Checkers/SmartPtrModeling.cpp             |    1 -
 .../StaticAnalyzer/Checkers/StreamChecker.cpp |   21 +-
 .../Checkers/WebKit/PtrTypesSemantics.cpp     |    3 +-
 .../WebKit/UncountedCallArgsChecker.cpp       |    2 +-
 .../Core/BugReporterVisitors.cpp              |   54 +-
 clang/lib/Support/RISCVVIntrinsicUtils.cpp    |    1 -
 .../Tooling/Inclusions/Stdlib/CMakeLists.txt  |   20 +
 clang/test/AST/Interp/arrays.cpp              |   20 +
 clang/test/AST/Interp/bitfields.cpp           |   59 +-
 clang/test/AST/Interp/cxx23.cpp               |   84 +
 clang/test/AST/Interp/intap.cpp               |   45 +-
 clang/test/AST/Interp/literals.cpp            |    8 +-
 clang/test/AST/Interp/records.cpp             |   39 +
 .../WebKit/call-args-dynamic-downcast.cpp     |   35 +
 .../Analysis/Checkers/WebKit/call-args.cpp    |   16 -
 clang/test/Analysis/PR49642.c                 |    2 +-
 clang/test/Analysis/analyzer-config.c         |    4 +-
 .../test/Analysis/analyzer-enabled-checkers.c |    1 +
 clang/test/Analysis/conversion.c              |    4 +-
 .../deref-track-symbolic-region.cpp           |   31 +
 .../errno-stdlibraryfunctions-notes.c         |    4 +-
 .../test/Analysis/errno-stdlibraryfunctions.c |    4 +-
 clang/test/Analysis/lambdas.cpp               |    5 +-
 .../std-c-library-functions-POSIX-lookup.c    |    6 +-
 ...ibrary-functions-POSIX-socket-sockaddr.cpp |    6 +-
 .../Analysis/std-c-library-functions-POSIX.c  |   12 +-
 ...ry-functions-arg-constraints-note-tags.cpp |    4 +-
 ...ibrary-functions-arg-constraints-notes.cpp |    4 +-
 ...functions-arg-constraints-tracking-notes.c |    2 +-
 .../std-c-library-functions-arg-constraints.c |    8 +-
 ...td-c-library-functions-arg-constraints.cpp |    2 +-
 ...library-functions-arg-cstring-dependency.c |    4 +-
 ...c-library-functions-arg-enabled-checkers.c |   10 +-
 .../std-c-library-functions-arg-weakdeps.c    |   10 +-
 .../Analysis/std-c-library-functions-eof.c    |   10 +-
 .../std-c-library-functions-inlined.c         |   10 +-
 .../Analysis/std-c-library-functions-lookup.c |    4 +-
 .../std-c-library-functions-lookup.cpp        |    4 +-
 .../std-c-library-functions-path-notes.c      |    4 +-
 .../std-c-library-functions-restrict.c        |    4 +-
 .../std-c-library-functions-restrict.cpp      |    4 +-
 ...td-c-library-functions-vs-stream-checker.c |    8 +-
 clang/test/Analysis/std-c-library-functions.c |   12 +-
 .../test/Analysis/std-c-library-functions.cpp |    2 +-
 .../test/Analysis/std-c-library-posix-crash.c |    4 +-
 clang/test/Analysis/stream-errno-note.c       |    4 +-
 clang/test/Analysis/stream-errno.c            |    4 +-
 clang/test/Analysis/stream-noopen.c           |    8 +-
 clang/test/Analysis/stream-note.c             |    4 +-
 .../Analysis/stream-stdlibraryfunctionargs.c  |   10 +-
 clang/test/Analysis/weak-dependencies.c       |    2 +-
 clang/test/C/C2x/n2359.c                      |    4 +
 clang/test/C/C2x/n2683.c                      |   51 +
 clang/test/C/C2x/n2683_2.c                    |   66 +
 clang/test/C/C2x/n3007.c                      |    4 +
 clang/test/CXX/basic/basic.types/p10.cpp      |   14 +-
 .../dcl.dcl/dcl.spec/dcl.typedef/p2-0x.cpp    |   12 +-
 .../expr/expr.prim/expr.prim.lambda/p4.cpp    |   12 +-
 .../over.match.oper/p3-2a.cpp                 |   61 +
 .../CXX/temp/temp.arg/temp.arg.type/p2.cpp    |   10 +-
 .../CXX/temp/temp.decls/temp.variadic/p5.cpp  |   60 +-
 .../CodeGen/PowerPC/builtins-ppc-pair-mma.c   |  174 +-
 .../PowerPC/inline-asm-matching-ppc-vsx.c     |   12 +-
 .../CodeGen/RISCV/riscv-inline-asm-clobber.c  |   44 +
 .../non-overloaded/sf_vqmacc_2x8x2.c          |   47 +
 .../non-overloaded/sf_vqmacc_4x8x4.c          |   47 +
 .../non-overloaded/sf_vqmaccsu_2x8x2.c        |   47 +
 .../non-overloaded/sf_vqmaccsu_4x8x4.c        |   47 +
 .../non-overloaded/sf_vqmaccu_2x8x2.c         |   47 +
 .../non-overloaded/sf_vqmaccu_4x8x4.c         |   47 +
 .../non-overloaded/sf_vqmaccus_2x8x2.c        |   47 +
 .../non-overloaded/sf_vqmaccus_4x8x4.c        |   47 +
 .../non-policy/overloaded/sf_vqmacc_2x8x2.c   |   47 +
 .../non-policy/overloaded/sf_vqmacc_4x8x4.c   |   47 +
 .../non-policy/overloaded/sf_vqmaccsu_2x8x2.c |   47 +
 .../non-policy/overloaded/sf_vqmaccsu_4x8x4.c |   47 +
 .../non-policy/overloaded/sf_vqmaccu_2x8x2.c  |   47 +
 .../non-policy/overloaded/sf_vqmaccu_4x8x4.c  |   47 +
 .../non-policy/overloaded/sf_vqmaccus_2x8x2.c |   47 +
 .../non-policy/overloaded/sf_vqmaccus_4x8x4.c |   47 +
 .../policy/non-overloaded/sf_vqmacc_2x8x2.c   |   47 +
 .../policy/non-overloaded/sf_vqmacc_4x8x4.c   |   47 +
 .../policy/non-overloaded/sf_vqmaccsu_2x8x2.c |   47 +
 .../policy/non-overloaded/sf_vqmaccsu_4x8x4.c |   47 +
 .../policy/non-overloaded/sf_vqmaccu_2x8x2.c  |   47 +
 .../policy/non-overloaded/sf_vqmaccu_4x8x4.c  |   47 +
 .../policy/non-overloaded/sf_vqmaccus_2x8x2.c |   47 +
 .../policy/non-overloaded/sf_vqmaccus_4x8x4.c |   47 +
 .../policy/overloaded/sf_vqmacc_2x8x2.c       |   47 +
 .../policy/overloaded/sf_vqmacc_4x8x4.c       |   47 +
 .../policy/overloaded/sf_vqmaccsu_2x8x2.c     |   47 +
 .../policy/overloaded/sf_vqmaccsu_4x8x4.c     |   47 +
 .../policy/overloaded/sf_vqmaccu_2x8x2.c      |   47 +
 .../policy/overloaded/sf_vqmaccu_4x8x4.c      |   47 +
 .../policy/overloaded/sf_vqmaccus_2x8x2.c     |   47 +
 .../policy/overloaded/sf_vqmaccus_4x8x4.c     |   47 +
 clang/test/CodeGen/X86/avx512-error.c         |   20 +-
 .../CodeGen/X86/usermsr-builtins-error-32.c   |   14 +
 clang/test/CodeGen/X86/usermsr-builtins.c     |   29 +
 .../aarch64-sme-intrinsics/acle_sme_add-i32.c |  179 +-
 .../aarch64-sme-intrinsics/acle_sme_add-i64.c |  179 +-
 .../aarch64-sme-intrinsics/acle_sme_cnt.c     |   67 +-
 .../aarch64-sme-intrinsics/acle_sme_ld1.c     |  241 +-
 .../acle_sme_ld1_vnum.c                       |  331 +-
 .../aarch64-sme-intrinsics/acle_sme_ldr.c     |   90 +-
 .../acle_sme_mopa-za32.c                      |  133 +-
 .../acle_sme_mopa-za64.c                      |  113 +-
 .../acle_sme_mops-za32.c                      |  133 +-
 .../acle_sme_mops-za64.c                      |  113 +-
 .../aarch64-sme-intrinsics/acle_sme_read.c    | 1851 ++--
 .../aarch64-sme-intrinsics/acle_sme_st1.c     |  241 +-
 .../acle_sme_st1_vnum.c                       |  331 +-
 .../aarch64-sme-intrinsics/acle_sme_str.c     |   89 +-
 .../aarch64-sme-intrinsics/acle_sme_write.c   | 1851 ++--
 .../aarch64-sme-intrinsics/acle_sme_zero.c    |   67 +-
 .../acle_sve_ld1-bfloat.c                     |    8 +-
 .../aarch64-sve-intrinsics/acle_sve_ld1.c     |  256 +-
 .../aarch64-sve-intrinsics/acle_sve_ld1sb.c   |  112 +-
 .../aarch64-sve-intrinsics/acle_sve_ld1sh.c   |  144 +-
 .../aarch64-sve-intrinsics/acle_sve_ld1sw.c   |   72 +-
 .../aarch64-sve-intrinsics/acle_sve_ld1ub.c   |  112 +-
 .../aarch64-sve-intrinsics/acle_sve_ld1uh.c   |  144 +-
 .../aarch64-sve-intrinsics/acle_sve_ld1uw.c   |   72 +-
 .../acle_sve_ld2-bfloat.c                     |    8 +-
 .../aarch64-sve-intrinsics/acle_sve_ld2.c     |   88 +-
 .../acle_sve_ld3-bfloat.c                     |    8 +-
 .../aarch64-sve-intrinsics/acle_sve_ld3.c     |   88 +-
 .../acle_sve_ld4-bfloat.c                     |    8 +-
 .../aarch64-sve-intrinsics/acle_sve_ld4.c     |   88 +-
 .../acle_sve_ldff1-bfloat.c                   |    8 +-
 .../aarch64-sve-intrinsics/acle_sve_ldff1.c   |  256 +-
 .../aarch64-sve-intrinsics/acle_sve_ldff1sb.c |  112 +-
 .../aarch64-sve-intrinsics/acle_sve_ldff1sh.c |  144 +-
 .../aarch64-sve-intrinsics/acle_sve_ldff1sw.c |   72 +-
 .../aarch64-sve-intrinsics/acle_sve_ldff1ub.c |  112 +-
 .../aarch64-sve-intrinsics/acle_sve_ldff1uh.c |  144 +-
 .../aarch64-sve-intrinsics/acle_sve_ldff1uw.c |   72 +-
 .../acle_sve_ldnf1-bfloat.c                   |    8 +-
 .../aarch64-sve-intrinsics/acle_sve_ldnf1.c   |   88 +-
 .../aarch64-sve-intrinsics/acle_sve_ldnf1sb.c |   48 +-
 .../aarch64-sve-intrinsics/acle_sve_ldnf1sh.c |   32 +-
 .../aarch64-sve-intrinsics/acle_sve_ldnf1sw.c |   16 +-
 .../aarch64-sve-intrinsics/acle_sve_ldnf1ub.c |   48 +-
 .../aarch64-sve-intrinsics/acle_sve_ldnf1uh.c |   32 +-
 .../aarch64-sve-intrinsics/acle_sve_ldnf1uw.c |   16 +-
 .../acle_sve_ldnt1-bfloat.c                   |    8 +-
 .../aarch64-sve-intrinsics/acle_sve_ldnt1.c   |   88 +-
 .../aarch64-sve-intrinsics/acle_sve_prfb.c    |   84 +-
 .../aarch64-sve-intrinsics/acle_sve_prfd.c    |    8 +-
 .../aarch64-sve-intrinsics/acle_sve_prfh.c    |    8 +-
 .../aarch64-sve-intrinsics/acle_sve_prfw.c    |    8 +-
 .../acle_sve_reinterpret.c                    |    2 +-
 .../aarch64-sve-intrinsics/acle_sve_setffr.c  |    4 +-
 .../acle_sve_st1-bfloat.c                     |    8 +-
 .../aarch64-sve-intrinsics/acle_sve_st1.c     |  256 +-
 .../aarch64-sve-intrinsics/acle_sve_st1b.c    |   56 +-
 .../aarch64-sve-intrinsics/acle_sve_st1h.c    |   72 +-
 .../aarch64-sve-intrinsics/acle_sve_st1w.c    |   36 +-
 .../acle_sve_st2-bfloat.c                     |    8 +-
 .../aarch64-sve-intrinsics/acle_sve_st2.c     |   88 +-
 .../acle_sve_st3-bfloat.c                     |    8 +-
 .../aarch64-sve-intrinsics/acle_sve_st3.c     |   88 +-
 .../acle_sve_st4-bfloat.c                     |    8 +-
 .../aarch64-sve-intrinsics/acle_sve_st4.c     |   88 +-
 .../acle_sve_stnt1-bfloat.c                   |    8 +-
 .../aarch64-sve-intrinsics/acle_sve_stnt1.c   |   88 +-
 .../aarch64-sve-intrinsics/acle_sve_wrffr.c   |    4 +-
 .../aarch64-sve2-intrinsics/acle_sve2_revd.c  |  390 +
 .../acle_sve2p1_bfmlsl.c                      |   85 +
 .../acle_sve2p1_cntp.c                        |  119 +
 .../acle_sve2p1_dot.c                         |  107 +
 .../acle_sve2p1_ld1.c                         | 1250 +++
 .../acle_sve2p1_ldnt1.c                       | 1250 +++
 .../acle_sve2p1_pext.c                        |  152 +
 .../acle_sve2p1_pfalse.c                      |   30 +
 .../acle_sve2p1_psel.c                        |  165 +
 .../acle_sve2p1_st1.c                         |  998 ++
 .../acle_sve2p1_stnt1.c                       | 1042 +++
 .../acle_sve2p1_while_pn.c                    |  992 ++
 .../test/CodeGen/arm64-microsoft-intrinsics.c |   10 +
 .../attr-arm-sve-vector-bits-bitcast.c        |  144 +-
 clang/test/CodeGen/attr-counted-by.c          |  227 +
 clang/test/CodeGen/attr-cpuspecific-cpus.c    |    2 +
 .../attr-riscv-rvv-vector-bits-bitcast.c      |   48 +-
 clang/test/CodeGen/attr-target-mv.c           |   10 +
 clang/test/CodeGen/attr-target-x86.c          |   14 +-
 clang/test/CodeGen/bounds-checking.c          |   10 +-
 clang/test/CodeGen/builtin_vectorelements.c   |  127 +
 clang/test/CodeGen/catch-undef-behavior.c     |   45 +-
 clang/test/CodeGen/mrtd.c                     |   25 +-
 clang/test/CodeGen/target-avx-abi-diag.c      |    4 +
 clang/test/CodeGen/target-builtin-noerror.c   |    2 +
 clang/test/CodeGen/target-data.c              |   16 +-
 .../CodeGenCXX/aarch64-sve-vector-init.cpp    |   18 +
 .../test/CodeGenCXX/default_calling_conv.cpp  |   35 +-
 clang/test/CodeGenCXX/ext-int.cpp             |   13 -
 clang/test/CodeGenCXX/m68k-rtdcall.cpp        |   16 +
 .../unannotated-functions-get-emitted.cpp     |   19 +
 .../test/CodeGenHipStdPar/unsupported-ASM.cpp |   10 +
 .../CodeGenHipStdPar/unsupported-builtins.cpp |    8 +
 clang/test/CodeGenObjC/arc.m                  |   20 +-
 .../convert-messages-to-runtime-calls.m       |    5 -
 clang/test/CodeGenSYCL/function-attrs.cpp     |   40 +-
 .../usr/lib/avr/lib/ldscripts/avrtiny.x       |    0
 .../usr/lib/avr/lib/ldscripts/avrxmega6.x     |    0
 clang/test/Driver/avr-ld.c                    |   50 +
 .../cl-delayed-template-parsing-cxx20.cpp     |   10 +
 clang/test/Driver/cl-offload.cu               |   31 +
 clang/test/Driver/dragonfly.c                 |   10 +-
 clang/test/Driver/fopenmp.c                   |   19 +-
 clang/test/Driver/haiku.c                     |    5 +
 clang/test/Driver/openbsd.c                   |   17 +-
 .../test/Driver/print-multi-selection-flags.c |   15 +
 clang/test/Driver/riscv-features.c            |    1 +
 clang/test/Driver/x86-march.c                 |    8 +
 clang/test/Driver/x86-target-features.c       |   25 +
 clang/test/Headers/__clang_hip_math.hip       |   68 +-
 clang/test/Headers/stdckdint.c                |   53 +
 clang/test/Index/USR/decl-context.cpp         |   14 +
 ...a-attribute-supported-attributes-list.test |    1 +
 clang/test/Misc/target-invalid-cpu-note.c     |    8 +-
 clang/test/Misc/warning-wall.c                |    3 +-
 .../Modules/Inputs/all-product-headers.yaml   |   33 +
 clang/test/Modules/explicit-build.cpp         |    2 +-
 clang/test/Modules/modulemap-collision.m      |   15 +
 clang/test/Modules/no-undeclared-includes.c   |   31 +
 clang/test/OpenMP/debug-info-openmp-array.cpp |    4 +-
 clang/test/OpenMP/for_reduction_codegen.cpp   |   16 +-
 .../test/OpenMP/for_reduction_codegen_UDR.cpp |    8 +-
 .../master_taskloop_firstprivate_codegen.cpp  |   20 +-
 .../master_taskloop_in_reduction_codegen.cpp  |    8 +-
 .../master_taskloop_lastprivate_codegen.cpp   |   24 +-
 .../master_taskloop_private_codegen.cpp       |   20 +-
 .../master_taskloop_reduction_codegen.cpp     |    4 +-
 ...ter_taskloop_simd_firstprivate_codegen.cpp |   20 +-
 ...ter_taskloop_simd_in_reduction_codegen.cpp |    8 +-
 ...ster_taskloop_simd_lastprivate_codegen.cpp |   20 +-
 .../master_taskloop_simd_private_codegen.cpp  |   20 +-
 ...master_taskloop_simd_reduction_codegen.cpp |    4 +-
 clang/test/OpenMP/nvptx_target_codegen.cpp    |   10 +-
 clang/test/OpenMP/parallel_ast_print.cpp      |   16 +-
 clang/test/OpenMP/parallel_codegen.cpp        |   16 +-
 .../OpenMP/parallel_firstprivate_codegen.cpp  |   36 +-
 clang/test/OpenMP/parallel_for_codegen.cpp    |   28 +-
 .../test/OpenMP/parallel_masked_ast_print.cpp |    8 +-
 .../test/OpenMP/parallel_master_ast_print.cpp |    8 +-
 ...l_master_taskloop_firstprivate_codegen.cpp |   20 +-
 ...el_master_taskloop_lastprivate_codegen.cpp |   24 +-
 ...rallel_master_taskloop_private_codegen.cpp |   20 +-
 ...llel_master_taskloop_reduction_codegen.cpp |    4 +-
 ...ter_taskloop_simd_firstprivate_codegen.cpp |   20 +-
 ...ster_taskloop_simd_lastprivate_codegen.cpp |   24 +-
 ...l_master_taskloop_simd_private_codegen.cpp |   20 +-
 ...master_taskloop_simd_reduction_codegen.cpp |    4 +-
 clang/test/OpenMP/single_codegen.cpp          |   18 +-
 clang/test/OpenMP/target_ast_print.cpp        |   48 +-
 clang/test/OpenMP/target_codegen.cpp          |   68 +-
 .../OpenMP/target_constant_device_codegen.cpp |    4 +-
 clang/test/OpenMP/target_data_codegen.cpp     |  152 +-
 .../target_data_use_device_addr_codegen.cpp   |    8 +-
 .../OpenMP/target_defaultmap_codegen_01.cpp   |  416 +-
 .../OpenMP/target_defaultmap_messages.cpp     |   12 +-
 clang/test/OpenMP/target_depend_codegen.cpp   |   40 +-
 .../test/OpenMP/target_enter_data_codegen.cpp |  120 +-
 .../target_enter_data_depend_codegen.cpp      |   16 +-
 .../test/OpenMP/target_exit_data_codegen.cpp  |   16 +-
 .../target_exit_data_depend_codegen.cpp       |   16 +-
 .../OpenMP/target_firstprivate_codegen.cpp    |   40 +-
 .../target_has_device_addr_codegen_01.cpp     |    4 +-
 .../OpenMP/target_in_reduction_codegen.cpp    |    4 +-
 clang/test/OpenMP/target_map_codegen_12.cpp   |   32 +-
 clang/test/OpenMP/target_map_codegen_18a.cpp  |   12 +-
 clang/test/OpenMP/target_map_codegen_18b.cpp  |   12 +-
 clang/test/OpenMP/target_map_codegen_18c.cpp  |   20 +-
 clang/test/OpenMP/target_map_codegen_18d.cpp  |   20 +-
 clang/test/OpenMP/target_map_messages.cpp     |   50 +-
 clang/test/OpenMP/target_parallel_codegen.cpp |   80 +-
 .../target_parallel_defaultmap_messages.cpp   |   12 +-
 .../OpenMP/target_parallel_depend_codegen.cpp |   40 +-
 .../OpenMP/target_parallel_for_codegen.cpp    |   80 +-
 .../target_parallel_for_depend_codegen.cpp    |   40 +-
 .../target_parallel_for_simd_codegen.cpp      |   80 +-
 ...arget_parallel_for_simd_depend_codegen.cpp |   40 +-
 ...t_parallel_generic_loop_depend_codegen.cpp |   40 +-
 clang/test/OpenMP/target_private_codegen.cpp  |   24 +-
 .../test/OpenMP/target_reduction_codegen.cpp  |   24 +-
 clang/test/OpenMP/target_simd_codegen.cpp     |   80 +-
 .../OpenMP/target_simd_depend_codegen.cpp     |   40 +-
 clang/test/OpenMP/target_teams_codegen.cpp    |   80 +-
 .../OpenMP/target_teams_depend_codegen.cpp    |   40 +-
 .../target_teams_distribute_codegen.cpp       |   80 +-
 ...rget_teams_distribute_collapse_codegen.cpp |   32 +-
 ...target_teams_distribute_depend_codegen.cpp |   40 +-
 ...teams_distribute_dist_schedule_codegen.cpp |   32 +-
 ...stribute_parallel_for_collapse_codegen.cpp |   32 +-
 ...distribute_parallel_for_depend_codegen.cpp |   40 +-
 ...ute_parallel_for_dist_schedule_codegen.cpp |   32 +-
 ...stribute_parallel_for_schedule_codegen.cpp |   48 +-
 ...ute_parallel_for_simd_collapse_codegen.cpp |   32 +-
 ...ibute_parallel_for_simd_depend_codegen.cpp |   40 +-
 ...arallel_for_simd_dist_schedule_codegen.cpp |   32 +-
 ...ute_parallel_for_simd_schedule_codegen.cpp |   48 +-
 .../target_teams_distribute_simd_codegen.cpp  |   80 +-
 ...teams_distribute_simd_collapse_codegen.cpp |   32 +-
 ...t_teams_distribute_simd_depend_codegen.cpp |   48 +-
 ..._distribute_simd_dist_schedule_codegen.cpp |   32 +-
 ...et_teams_generic_loop_collapse_codegen.cpp |   32 +-
 ...rget_teams_generic_loop_depend_codegen.cpp |   40 +-
 .../test/OpenMP/target_teams_map_messages.cpp |   16 +-
 clang/test/OpenMP/target_update_codegen.cpp   |  424 +-
 .../OpenMP/target_update_depend_codegen.cpp   |   16 +-
 clang/test/OpenMP/task_ast_print.cpp          |    8 +-
 clang/test/OpenMP/task_codegen.cpp            |   24 +-
 .../test/OpenMP/task_firstprivate_codegen.cpp |   20 +-
 .../test/OpenMP/task_in_reduction_codegen.cpp |    8 +-
 clang/test/OpenMP/task_private_codegen.cpp    |   20 +-
 .../taskgroup_task_reduction_codegen.cpp      |   12 +-
 .../OpenMP/taskloop_firstprivate_codegen.cpp  |   20 +-
 .../OpenMP/taskloop_in_reduction_codegen.cpp  |    8 +-
 .../OpenMP/taskloop_lastprivate_codegen.cpp   |   24 +-
 .../test/OpenMP/taskloop_private_codegen.cpp  |   20 +-
 .../OpenMP/taskloop_reduction_codegen.cpp     |    4 +-
 .../taskloop_simd_firstprivate_codegen.cpp    |   20 +-
 .../taskloop_simd_in_reduction_codegen.cpp    |    8 +-
 .../taskloop_simd_lastprivate_codegen.cpp     |   20 +-
 .../OpenMP/taskloop_simd_private_codegen.cpp  |   20 +-
 .../taskloop_simd_reduction_codegen.cpp       |    4 +-
 .../test/OpenMP/teams_distribute_codegen.cpp  |   64 +-
 .../teams_distribute_collapse_codegen.cpp     |   32 +-
 ...teams_distribute_dist_schedule_codegen.cpp |   32 +-
 .../teams_distribute_parallel_for_codegen.cpp |   64 +-
 ...stribute_parallel_for_collapse_codegen.cpp |   32 +-
 ...ute_parallel_for_dist_schedule_codegen.cpp |   32 +-
 ...stribute_parallel_for_schedule_codegen.cpp |   48 +-
 ...s_distribute_parallel_for_simd_codegen.cpp |   64 +-
 ...ute_parallel_for_simd_collapse_codegen.cpp |   32 +-
 ...arallel_for_simd_dist_schedule_codegen.cpp |   32 +-
 ...ute_parallel_for_simd_schedule_codegen.cpp |   48 +-
 .../OpenMP/teams_distribute_simd_codegen.cpp  |   96 +-
 ...teams_distribute_simd_collapse_codegen.cpp |   32 +-
 ..._distribute_simd_dist_schedule_codegen.cpp |   32 +-
 .../OpenMP/teams_firstprivate_codegen.cpp     |   48 +-
 .../OpenMP/teams_generic_loop_codegen-1.cpp   |   64 +-
 clang/test/Parser/objcxx11-attributes.mm      |    4 +-
 clang/test/Parser/pragma-loop.cpp             |    5 +-
 clang/test/Preprocessor/init-mips.c           |   27 +
 .../Preprocessor/predefined-arch-macros.c     |   26 +
 .../test/Preprocessor/riscv-target-features.c |   18 +
 clang/test/Preprocessor/x86_target_features.c |   20 +
 .../aarch64-sme-intrinsics/acle_sme_imm.cpp   |    8 -
 .../acle_sve2p1_imm.cpp                       |  118 +
 clang/test/Sema/attr-counted-by.c             |   50 +
 clang/test/Sema/attr-target.c                 |   24 +
 clang/test/Sema/builtin_vectorelements.c      |   23 +
 clang/test/Sema/builtins-overflow.c           |   10 +-
 clang/test/Sema/c2x-auto.c                    |    1 +
 clang/test/Sema/conditional.c                 |   14 +-
 clang/test/Sema/convertvector.c               |    2 +-
 clang/test/Sema/m68k-rtdcall.c                |   46 +
 .../test/Sema/rvv-required-features-invalid.c |    8 +
 clang/test/Sema/rvv-required-features.c       |   11 +-
 clang/test/Sema/vla.cpp                       |    2 +-
 clang/test/Sema/wasm-refs-and-tables.c        |   12 +-
 .../test/SemaCUDA/Inputs/cuda-initializers.h  |   11 +
 .../SemaCUDA/call-stack-for-deferred-err.cu   |    2 +-
 clang/test/SemaCUDA/device-var-init.cu        |   48 +
 .../no-call-stack-for-immediate-errs.cu       |    2 +-
 clang/test/SemaCUDA/vla.cu                    |    4 +-
 .../enum-constexpr-conversion-system-header.h |   19 +
 clang/test/SemaCXX/PR10177.cpp                |    7 +-
 clang/test/SemaCXX/alias-template.cpp         |    8 +-
 clang/test/SemaCXX/anonymous-struct.cpp       |    1 +
 clang/test/SemaCXX/bitfield-width.c           |   42 +
 clang/test/SemaCXX/builtin_vectorelements.cpp |   55 +
 .../c99-variable-length-array-cxx11.cpp       |   10 +-
 .../SemaCXX/c99-variable-length-array.cpp     |   34 +-
 .../SemaCXX/constant-expression-cxx11.cpp     |   17 +-
 .../SemaCXX/constexpr-array-unknown-bound.cpp |    8 +-
 .../SemaCXX/cxx0x-initializer-aggregates.cpp  |    6 +-
 .../SemaCXX/cxx0x-noexcept-expression.cpp     |   17 +-
 clang/test/SemaCXX/cxx1y-generic-lambdas.cpp  |  165 +-
 .../SemaCXX/cxx1z-noexcept-function-type.cpp  |    4 +-
 clang/test/SemaCXX/cxx2a-consteval.cpp        |    2 +-
 .../cxx2a-constexpr-dynalloc-limits.cpp       |    4 +-
 clang/test/SemaCXX/exceptions.cpp             |   16 +-
 clang/test/SemaCXX/ext-int.cpp                |    4 +-
 clang/test/SemaCXX/for-range-examples.cpp     |   15 +-
 clang/test/SemaCXX/i-c-e-cxx.cpp              |    8 +-
 clang/test/SemaCXX/lambda-expressions.cpp     |    5 +-
 clang/test/SemaCXX/m68k-rtdcall.cpp           |   14 +
 clang/test/SemaCXX/new-delete-0x.cpp          |    4 +-
 clang/test/SemaCXX/new-delete.cpp             |    1 +
 clang/test/SemaCXX/offsetof.cpp               |   11 +-
 clang/test/SemaCXX/paren-list-agg-init.cpp    |    9 +-
 clang/test/SemaCXX/type-traits.cpp            |   48 +-
 clang/test/SemaCXX/vararg-non-pod.cpp         |   20 +-
 clang/test/SemaCXX/vla-construct.cpp          |   12 +-
 clang/test/SemaCXX/vla-ext-diag.cpp           |   40 +
 clang/test/SemaCXX/vla.cpp                    |   17 +-
 .../SemaCXX/warn-thread-safety-analysis.cpp   |   79 +
 .../lit.local.cfg                             |   11 +
 ...rn-unsafe-buffer-usage-debug-unclaimed.cpp |   49 +
 .../SemaCXX/warn-unused-lambda-capture.cpp    |    5 +-
 clang/test/SemaCXX/warn-unused-variables.cpp  |    2 +-
 clang/test/SemaObjCXX/properties.mm           |    3 +-
 clang/test/SemaObjCXX/vla.mm                  |    2 +-
 clang/test/SemaSYCL/unique_stable_name.cpp    |    3 +-
 .../SemaTemplate/concepts-out-of-line-def.cpp |   42 +-
 clang/test/SemaTemplate/deduction.cpp         |    9 +-
 clang/test/SemaTemplate/dependent-expr.cpp    |   15 +
 .../function-decl-nested-type-alias.cpp       |   16 +
 .../test/SemaTemplate/instantiate-sizeof.cpp  |    8 +-
 .../test/SemaTemplate/instantiate-typeof.cpp  |    8 +-
 clang/tools/clang-format/CMakeLists.txt       |    6 +
 clang/tools/clang-format/git-clang-format.bat |    1 +
 .../clang-linker-wrapper/LinkerWrapperOpts.td |    2 +-
 .../ClangOffloadBundler.cpp                   |    1 -
 clang/tools/libclang/CXType.cpp               |    1 +
 clang/unittests/AST/ASTImporterTest.cpp       |   61 +-
 clang/unittests/AST/CMakeLists.txt            |    2 +
 clang/unittests/AST/Interp/CMakeLists.txt     |   18 +
 clang/unittests/AST/Interp/Descriptor.cpp     |  385 +
 .../ASTMatchers/ASTMatchersTraversalTest.cpp  |   13 +
 .../Analysis/FlowSensitive/ArenaTest.cpp      |   55 +-
 .../DataflowAnalysisContextTest.cpp           |   27 -
 .../FlowSensitive/DebugSupportTest.cpp        |   20 +-
 .../Analysis/FlowSensitive/TestingSupport.cpp |   15 +-
 .../Analysis/FlowSensitive/TestingSupport.h   |   21 +-
 .../Analysis/FlowSensitive/TransferTest.cpp   |  255 +-
 .../TypeErasedDataflowAnalysisTest.cpp        |   24 +-
 .../UncheckedOptionalAccessModelTest.cpp      |  155 +
 clang/unittests/Driver/CMakeLists.txt         |    1 +
 clang/unittests/Driver/GCCVersionTest.cpp     |   59 +
 clang/unittests/Format/ConfigParseTest.cpp    |   25 +
 clang/unittests/Format/FormatTest.cpp         |   39 +
 clang/unittests/Format/FormatTestObjC.cpp     |  214 +-
 .../unittests/Format/FormatTestRawStrings.cpp |    3 -
 clang/unittests/Format/TokenAnnotatorTest.cpp |  193 +
 clang/unittests/Index/IndexTests.cpp          |   25 +
 clang/unittests/Tooling/RangeSelectorTest.cpp |    1 -
 clang/utils/TableGen/RISCVVEmitter.cpp        |    4 +-
 clang/utils/TableGen/SveEmitter.cpp           |   81 +-
 .../utils/analyze_safe_buffer_debug_notes.py  |   39 +
 clang/www/c_status.html                       |    5 +
 clang/www/get_involved.html                   |   29 +-
 compiler-rt/cmake/Modules/AddCompilerRT.cmake |    4 +
 .../cmake/Modules/AllSupportedArchDefs.cmake  |    2 +-
 compiler-rt/cmake/builtin-config-ix.cmake     |    6 +
 compiler-rt/lib/asan/asan_interceptors.cpp    |    2 +-
 compiler-rt/lib/builtins/CMakeLists.txt       |    9 +
 .../lib/builtins/aarch64/sme-abi-init.c       |   52 +
 compiler-rt/lib/builtins/aarch64/sme-abi.S    |  176 +
 compiler-rt/lib/builtins/cpu_model.c          |   16 +
 compiler-rt/lib/builtins/extendxftf2.c        |   23 +
 compiler-rt/lib/builtins/floatdidf.c          |   52 +-
 compiler-rt/lib/builtins/floatdisf.c          |   51 +-
 compiler-rt/lib/builtins/floattidf.c          |   52 +-
 compiler-rt/lib/builtins/floattisf.c          |   51 +-
 compiler-rt/lib/builtins/floattitf.c          |   53 +-
 compiler-rt/lib/builtins/floatundidf.c        |   49 +-
 compiler-rt/lib/builtins/floatundisf.c        |   48 +-
 compiler-rt/lib/builtins/floatuntidf.c        |   49 +-
 compiler-rt/lib/builtins/floatuntisf.c        |   48 +-
 compiler-rt/lib/builtins/floatuntitf.c        |   50 +-
 compiler-rt/lib/builtins/fp_extend.h          |   99 +-
 compiler-rt/lib/builtins/fp_extend_impl.inc   |   83 +-
 compiler-rt/lib/builtins/fp_trunc.h           |   91 +-
 compiler-rt/lib/builtins/fp_trunc_impl.inc    |  122 +-
 compiler-rt/lib/builtins/int_to_fp.h          |   82 +
 compiler-rt/lib/builtins/int_to_fp_impl.inc   |   72 +
 compiler-rt/lib/builtins/trunctfxf2.c         |   24 +
 compiler-rt/lib/fuzzer/FuzzerLoop.cpp         |    2 +-
 .../lib/hwasan/hwasan_platform_interceptors.h |    4 +-
 .../lib/interception/interception_win.cpp     |    2 +-
 .../lib/sanitizer_common/sanitizer_platform.h |   10 +-
 .../lib/scudo/standalone/CMakeLists.txt       |    6 +
 .../lib/scudo/standalone/allocator_config.h   |    9 +
 compiler-rt/lib/scudo/standalone/common.cpp   |   14 -
 compiler-rt/lib/scudo/standalone/common.h     |    4 -
 .../lib/scudo/standalone/condition_variable.h |   60 +
 .../standalone/condition_variable_base.h      |   56 +
 .../standalone/condition_variable_linux.cpp   |   52 +
 .../standalone/condition_variable_linux.h     |   38 +
 compiler-rt/lib/scudo/standalone/linux.cpp    |    7 +-
 .../lib/scudo/standalone/mem_map_linux.cpp    |   11 +-
 compiler-rt/lib/scudo/standalone/primary64.h  |  135 +-
 compiler-rt/lib/scudo/standalone/report.cpp   |   13 +-
 compiler-rt/lib/scudo/standalone/report.h     |    5 +-
 .../lib/scudo/standalone/report_linux.cpp     |   58 +
 .../lib/scudo/standalone/report_linux.h       |   34 +
 .../lib/scudo/standalone/tests/CMakeLists.txt |    1 +
 .../scudo/standalone/tests/combined_test.cpp  |   51 +-
 .../tests/condition_variable_test.cpp         |   59 +
 .../scudo/standalone/tests/primary_test.cpp   |   32 +-
 compiler-rt/lib/scudo/standalone/trusty.cpp   |    5 +-
 compiler-rt/lib/tsan/rtl/CMakeLists.txt       |    4 +
 .../lib/tsan/rtl/tsan_interceptors_posix.cpp  |    2 +
 compiler-rt/lib/tsan/rtl/tsan_interface.h     |    8 +
 .../lib/tsan/rtl/tsan_interface_atomic.cpp    |   24 +
 compiler-rt/lib/tsan/rtl/tsan_platform.h      |   76 +-
 .../lib/tsan/rtl/tsan_platform_linux.cpp      |   34 +-
 compiler-rt/lib/tsan/rtl/tsan_rtl.h           |    4 +-
 compiler-rt/lib/tsan/rtl/tsan_rtl_riscv64.S   |  203 +
 compiler-rt/test/builtins/Unit/addtf3_test.c  |    2 +-
 compiler-rt/test/builtins/Unit/divtf3_test.c  |    2 +-
 .../test/builtins/Unit/extenddftf2_test.c     |    2 +-
 .../test/builtins/Unit/extendhftf2_test.c     |    2 +-
 .../test/builtins/Unit/extendsftf2_test.c     |    2 +-
 .../test/builtins/Unit/extendxftf2_test.c     |   74 +
 .../test/builtins/Unit/floatditf_test.c       |    2 +-
 .../test/builtins/Unit/floatsitf_test.c       |    2 +-
 .../test/builtins/Unit/floatunditf_test.c     |    2 +-
 .../test/builtins/Unit/floatunsitf_test.c     |    2 +-
 compiler-rt/test/builtins/Unit/fp_test.h      |   93 +-
 compiler-rt/test/builtins/Unit/multf3_test.c  |    2 +-
 compiler-rt/test/builtins/Unit/subtf3_test.c  |    2 +-
 .../test/builtins/Unit/trunctfxf2_test.c      |   97 +
 compiler-rt/test/hwasan/TestCases/bcmp.cpp    |   28 +
 .../test/hwasan/TestCases/deep-recursion.c    |   22 +-
 compiler-rt/test/hwasan/TestCases/memcmp.cpp  |    4 +-
 .../hwasan/TestCases/stack-history-length.c   |   21 +-
 compiler-rt/test/hwasan/TestCases/stack-uar.c |   21 +-
 compiler-rt/test/hwasan/TestCases/stack-uas.c |   22 +-
 compiler-rt/test/msan/dladdr1_test.c          |    2 +-
 .../Linux/ppc64/Inputs/trivial-tls-main.cpp   |   30 +
 .../Linux/ppc64/Inputs/trivial-tls-pwr10.cpp  |    5 +
 .../Linux/ppc64/trivial-tls-pwr10.test        |    9 +
 .../profile/Windows/coverage-weak-lld.cpp     |    4 +-
 compiler-rt/test/tsan/map32bit.cpp            |    1 +
 compiler-rt/test/tsan/mmap_large.cpp          |    3 +-
 compiler-rt/test/tsan/test.h                  |    2 +
 flang/docs/Extensions.md                      |   15 +
 flang/docs/GettingStarted.md                  |    6 +-
 flang/docs/OpenACC.md                         |    2 +
 flang/include/flang/Common/real.h             |    7 +
 .../flang/Decimal/binary-floating-point.h     |   59 +
 flang/include/flang/Decimal/decimal.h         |    8 -
 .../include/flang/Evaluate/characteristics.h  |   30 +-
 .../include/flang/Frontend/CodeGenOptions.def |    1 +
 flang/include/flang/ISO_Fortran_binding.h     |    6 +-
 flang/include/flang/Lower/ConvertType.h       |   30 +
 .../flang/Optimizer/Analysis/TBAAForest.h     |  104 +
 .../flang/Optimizer/Builder/HLFIRTools.h      |    5 +
 .../include/flang/Optimizer/CodeGen/CodeGen.h |    3 +
 .../flang/Optimizer/CodeGen/TBAABuilder.h     |   49 +-
 .../flang/Optimizer/CodeGen/TypeConverter.h   |    3 +-
 .../flang/Optimizer/Dialect/CMakeLists.txt    |    4 +
 .../flang/Optimizer/Dialect/FIRDialect.td     |    4 +-
 .../include/flang/Optimizer/Dialect/FIROps.h  |    1 +
 .../include/flang/Optimizer/Dialect/FIROps.td |   21 +-
 .../Dialect/FirAliasTagOpInterface.h          |   27 +
 .../Dialect/FirAliasTagOpInterface.td         |   59 +
 .../flang/Optimizer/Transforms/Passes.h       |    1 +
 .../flang/Optimizer/Transforms/Passes.td      |   20 +
 flang/include/flang/Parser/dump-parse-tree.h  |    1 +
 flang/include/flang/Parser/parse-tree.h       |   14 +-
 flang/include/flang/Runtime/descriptor.h      |   17 +-
 flang/include/flang/Semantics/tools.h         |    2 +-
 flang/include/flang/Semantics/type.h          |    2 +
 flang/include/flang/Tools/CLOptions.inc       |   10 +
 flang/include/flang/Tools/CrossToolHelpers.h  |    2 +
 flang/lib/Decimal/binary-to-decimal.cpp       |    3 +-
 flang/lib/Decimal/decimal-to-binary.cpp       |   30 +-
 flang/lib/Evaluate/check-expression.cpp       |    9 +-
 flang/lib/Evaluate/constant.cpp               |   10 +-
 flang/lib/Evaluate/fold-real.cpp              |   28 +-
 flang/lib/Evaluate/fold-reduction.h           |    2 +-
 flang/lib/Frontend/CompilerInvocation.cpp     |    5 +
 flang/lib/Lower/Allocatable.cpp               |    4 +-
 flang/lib/Lower/Bridge.cpp                    |  104 +-
 flang/lib/Lower/CallInterface.cpp             |    2 +-
 flang/lib/Lower/Coarray.cpp                   |    8 +-
 flang/lib/Lower/ConvertCall.cpp               |    7 +-
 flang/lib/Lower/ConvertConstant.cpp           |  209 +-
 flang/lib/Lower/ConvertExpr.cpp               |    4 +-
 flang/lib/Lower/ConvertExprToHLFIR.cpp        |  136 +-
 flang/lib/Lower/ConvertType.cpp               |   90 +-
 flang/lib/Lower/ConvertVariable.cpp           |  163 +-
 flang/lib/Lower/DirectivesCommon.h            |   11 +-
 flang/lib/Lower/OpenACC.cpp                   |  351 +-
 flang/lib/Lower/Runtime.cpp                   |   16 +-
 flang/lib/Lower/VectorSubscripts.cpp          |    3 +-
 flang/lib/Optimizer/Analysis/CMakeLists.txt   |    1 +
 flang/lib/Optimizer/Analysis/TBAAForest.cpp   |   60 +
 flang/lib/Optimizer/CodeGen/CMakeLists.txt    |    1 +
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       |   18 +-
 flang/lib/Optimizer/CodeGen/TBAABuilder.cpp   |   61 +-
 flang/lib/Optimizer/CodeGen/TypeConverter.cpp |    7 +-
 flang/lib/Optimizer/Dialect/CMakeLists.txt    |    1 +
 flang/lib/Optimizer/Dialect/FIRDialect.cpp    |    2 +
 flang/lib/Optimizer/Dialect/FIROps.cpp        |   29 +-
 .../Dialect/FirAliasTagOpInterface.cpp        |   31 +
 .../lib/Optimizer/Transforms/AddAliasTags.cpp |  210 +
 flang/lib/Optimizer/Transforms/CMakeLists.txt |    2 +
 .../Optimizer/Transforms/LoopVersioning.cpp   |   14 +-
 flang/lib/Parser/executable-parsers.cpp       |   10 +-
 flang/lib/Parser/openacc-parsers.cpp          |   36 +-
 flang/lib/Parser/unparse.cpp                  |    5 +-
 flang/lib/Semantics/CMakeLists.txt            |    1 +
 flang/lib/Semantics/canonicalize-do.cpp       |    9 +-
 flang/lib/Semantics/check-acc-structure.cpp   |   40 +-
 flang/lib/Semantics/check-acc-structure.h     |    1 +
 .../lib/Semantics/check-directive-structure.h |   18 +-
 flang/lib/Semantics/check-omp-structure.cpp   |   14 +-
 flang/lib/Semantics/check-omp-structure.h     |    1 -
 flang/lib/Semantics/compute-offsets.cpp       |    2 +
 flang/lib/Semantics/data-to-inits.cpp         |   32 +-
 flang/lib/Semantics/data-to-inits.h           |   16 +
 flang/lib/Semantics/mod-file.cpp              |   43 +-
 flang/lib/Semantics/resolve-directives.cpp    |   73 +-
 flang/lib/Semantics/resolve-labels.cpp        |   43 +-
 flang/lib/Semantics/resolve-names-utils.cpp   |   56 +-
 flang/lib/Semantics/resolve-names.cpp         |   13 +-
 flang/lib/Semantics/rewrite-directives.cpp    |  177 +
 flang/lib/Semantics/rewrite-directives.h      |   24 +
 flang/lib/Semantics/rewrite-parse-tree.cpp    |    3 +-
 flang/lib/Semantics/type.cpp                  |    4 +
 flang/module/__fortran_ieee_exceptions.f90    |    6 +-
 flang/runtime/ISO_Fortran_binding.cpp         |   12 +-
 flang/runtime/allocatable.cpp                 |   17 +-
 flang/runtime/derived-api.cpp                 |  102 +-
 flang/runtime/descriptor-io.h                 |    6 +-
 flang/runtime/descriptor.cpp                  |   16 +
 flang/runtime/edit-input.cpp                  |  319 +-
 flang/runtime/edit-output.cpp                 |  165 +-
 flang/runtime/edit-output.h                   |   24 +-
 flang/runtime/io-stmt.h                       |    9 +-
 flang/runtime/namelist.cpp                    |   11 +-
 flang/runtime/pointer.cpp                     |   17 +-
 flang/runtime/random.cpp                      |   10 +-
 flang/runtime/type-info.cpp                   |   10 +-
 flang/test/Driver/driver-help-hidden.f90      |    2 +
 flang/test/Driver/driver-help.f90             |    4 +
 flang/test/Driver/falias-analysis.f90         |   21 +
 flang/test/Driver/linker-flags.f90            |   18 +-
 flang/test/Evaluate/fold-norm2.f90            |   13 +-
 flang/test/Evaluate/folding08.f90             |    6 +
 flang/test/Evaluate/folding09.f90             |    9 +
 flang/test/Fir/invalid.fir                    |    2 +-
 flang/test/Fir/tbaa-codegen.fir               |   47 +
 flang/test/Fir/tbaa-codegen2.fir              |  114 +
 flang/test/Fir/tbaa.fir                       |   24 +-
 flang/test/HLFIR/invalid.fir                  |    4 +-
 .../calls-constant-expr-arg-polymorphic.f90   |   10 +-
 .../Lower/HLFIR/convert-variable-block.f90    |   25 +
 .../intentout-allocatable-components.f90      |   32 +
 .../local-end-of-scope-component-dealloc.f90  |   12 +-
 .../test/Lower/HLFIR/parent-component-ref.f90 |   39 +-
 flang/test/Lower/HLFIR/private-components.f90 |    4 +-
 .../Lower/HLFIR/structure-constructor.f90     |   37 +-
 .../Lower/HLFIR/type-bound-call-mismatch.f90  |   39 +
 flang/test/Lower/HLFIR/type-info.f90          |    8 +-
 flang/test/Lower/OpenACC/acc-bounds.f90       |   89 +
 flang/test/Lower/OpenACC/acc-init.f90         |    6 +
 .../test/Lower/OpenACC/acc-parallel-loop.f90  |   33 +-
 flang/test/Lower/OpenACC/acc-private.f90      |   77 +-
 flang/test/Lower/OpenACC/acc-reduction.f90    |   64 +-
 flang/test/Lower/OpenACC/acc-routine.f90      |   11 +
 flang/test/Lower/OpenACC/acc-serial-loop.f90  |   19 +-
 flang/test/Lower/OpenMP/FIR/atomic-read.f90   |    2 +-
 flang/test/Lower/OpenMP/FIR/critical.f90      |    2 +-
 .../FIR/parallel-private-clause-fixes.f90     |    6 +-
 flang/test/Lower/OpenMP/atomic-read.f90       |   89 +
 flang/test/Lower/OpenMP/atomic-write.f90      |   73 +
 flang/test/Lower/OpenMP/critical.f90          |    2 +-
 .../Lower/OpenMP/firstprivate-commonblock.f90 |   34 +
 .../Lower/OpenMP/lastprivate-commonblock.f90  |    8 +-
 .../OpenMP/parallel-private-clause-fixes.f90  |   92 +
 .../Lower/OpenMP/parallel-reduction-add.f90   |  101 +
 flang/test/Lower/OpenMP/parallel-sections.f90 |   58 +
 .../OpenMP/parallel-wsloop-firstpriv.f90      |   69 +
 flang/test/Lower/OpenMP/sections.f90          |   40 +-
 flang/test/Lower/OpenMP/unstructured.f90      |  348 +
 flang/test/Lower/OpenMP/wsloop.f90            |   75 +
 flang/test/Lower/allocatable-polymorphic.f90  |    8 +-
 flang/test/Lower/dispatch.f90                 |    4 +-
 .../Lower/pointer-association-polymorphic.f90 |    8 +-
 flang/test/Semantics/OpenACC/acc-error.f90    |   29 +
 .../Semantics/OpenACC/acc-init-validity.f90   |    6 +
 .../OpenACC/acc-routine-validity.f90          |    2 -
 flang/test/Semantics/OpenACC/acc-serial.f90   |    6 +-
 .../test/Semantics/OpenACC/acc-symbols01.f90  |    8 +-
 .../Semantics/OpenMP/do05-positivecase.f90    |    9 +
 .../Semantics/OpenMP/requires-atomic01.f90    |  109 +
 .../Semantics/OpenMP/requires-atomic02.f90    |  109 +
 flang/test/Semantics/OpenMP/symbol08.f90      |    4 +-
 .../test/Semantics/OpenMP/threadprivate06.f90 |   30 +
 flang/test/Semantics/dosemantics13.f90        |   29 +
 flang/test/Semantics/dosemantics14.f90        |   12 +
 flang/test/Semantics/modproc01.f90            |    4 +-
 flang/test/Semantics/modproc02.f90            |    1 -
 flang/test/Semantics/offsets02.f90            |   10 +-
 flang/test/Semantics/reshape.f90              |    5 +
 flang/test/Semantics/resolve29.f90            |   23 +
 flang/test/Semantics/separate-mp05.f90        |   40 +
 flang/test/Transforms/loop-versioning.fir     |   44 +
 flang/test/Transforms/tbaa.fir                |  175 +
 flang/test/Transforms/tbaa2.fir               |  386 +
 flang/tools/flang-driver/driver.cpp           |    2 -
 .../Evaluate/ISO-Fortran-binding.cpp          |  129 +-
 .../unittests/Optimizer/InternalNamesTest.cpp |    1 -
 .../unittests/Runtime/NumericalFormatTest.cpp |   62 +-
 libc/config/gpu/entrypoints.txt               |   40 +-
 libc/config/linux/aarch64/headers.txt         |    1 +
 libc/config/linux/riscv/headers.txt           |    1 +
 libc/config/linux/x86_64/headers.txt          |    1 +
 libc/docs/gpu/support.rst                     |    1 +
 libc/fuzzing/stdio/printf_float_conv_fuzz.cpp |   30 +-
 libc/include/CMakeLists.txt                   |    9 +
 libc/include/features.h.def                   |   17 +
 libc/include/llvm-libc-macros/CMakeLists.txt  |    6 +
 .../llvm-libc-macros/features-macros.h        |   14 +
 libc/include/llvm-libc-types/rpc_opcodes_t.h  |    2 +
 libc/src/__support/FPUtil/FPBits.h            |   38 +-
 libc/src/math/CMakeLists.txt                  |   14 +
 libc/src/math/acos.h                          |   18 +
 libc/src/math/acosh.h                         |   18 +
 libc/src/math/asin.h                          |   18 +
 libc/src/math/asinh.h                         |   18 +
 libc/src/math/atan.h                          |   18 +
 libc/src/math/atan2.h                         |   18 +
 libc/src/math/atan2f.h                        |   18 +
 libc/src/math/atanh.h                         |   18 +
 libc/src/math/erf.h                           |   18 +
 libc/src/math/gpu/CMakeLists.txt              |   70 +-
 libc/src/math/gpu/{vendor => }/llround.cpp    |    6 +-
 libc/src/math/gpu/{vendor => }/llroundf.cpp   |    6 +-
 libc/src/math/gpu/{sinhf.cpp => lround.cpp}   |    6 +-
 libc/src/math/gpu/{tanf.cpp => lroundf.cpp}   |    6 +-
 libc/src/math/gpu/vendor/CMakeLists.txt       |  298 +-
 libc/src/math/gpu/vendor/acos.cpp             |   18 +
 libc/src/math/gpu/vendor/acosh.cpp            |   18 +
 libc/src/math/gpu/vendor/amdgpu/amdgpu.h      |   34 +-
 .../src/math/gpu/vendor/amdgpu/declarations.h |   31 +-
 libc/src/math/gpu/vendor/asin.cpp             |   18 +
 libc/src/math/gpu/vendor/asinh.cpp            |   18 +
 libc/src/math/gpu/vendor/atan.cpp             |   18 +
 libc/src/math/gpu/vendor/atan2.cpp            |   20 +
 libc/src/math/gpu/vendor/atan2f.cpp           |   20 +
 libc/src/math/gpu/vendor/atanh.cpp            |   18 +
 .../math/gpu/{tanhf.cpp => vendor/erf.cpp}    |    8 +-
 libc/src/math/gpu/vendor/erff.cpp             |   18 +
 libc/src/math/gpu/vendor/exp.cpp              |   18 +
 libc/src/math/gpu/vendor/exp10.cpp            |   18 +
 libc/src/math/gpu/vendor/exp2.cpp             |   18 +
 libc/src/math/gpu/vendor/expm1.cpp            |   18 +
 libc/src/math/gpu/vendor/log.cpp              |   18 +
 libc/src/math/gpu/vendor/log10.cpp            |   18 +
 libc/src/math/gpu/vendor/log10f.cpp           |   18 +
 libc/src/math/gpu/vendor/log1p.cpp            |   18 +
 libc/src/math/gpu/vendor/log1pf.cpp           |   18 +
 libc/src/math/gpu/vendor/log2.cpp             |   18 +
 libc/src/math/gpu/vendor/log2f.cpp            |   18 +
 libc/src/math/gpu/vendor/logb.cpp             |   18 +
 libc/src/math/gpu/vendor/logbf.cpp            |   18 +
 libc/src/math/gpu/vendor/logf.cpp             |   18 +
 libc/src/math/gpu/vendor/lrint.cpp            |   18 +
 libc/src/math/gpu/vendor/lrintf.cpp           |   18 +
 libc/src/math/gpu/vendor/nvptx/declarations.h |   28 +-
 libc/src/math/gpu/vendor/nvptx/nvptx.h        |   28 +-
 libc/src/math/gpu/vendor/tgamma.cpp           |   18 +
 libc/src/math/gpu/vendor/tgammaf.cpp          |   18 +
 libc/src/math/sincos.h                        |   18 +
 libc/src/math/tgamma.h                        |   18 +
 libc/src/math/tgammaf.h                       |   18 +
 libc/src/stdio/CMakeLists.txt                 |   13 +-
 libc/src/stdio/generic/CMakeLists.txt         |   12 +
 libc/src/stdio/{ => generic}/ungetc.cpp       |    0
 libc/src/stdio/gpu/CMakeLists.txt             |   11 +
 libc/src/stdio/gpu/fgets.cpp                  |   28 +-
 libc/src/stdio/gpu/ungetc.cpp                 |   29 +
 libc/src/stdio/printf_core/core_structs.h     |    2 +-
 .../stdio/printf_core/float_hex_converter.h   |    5 +-
 libc/src/stdlib/rand.cpp                      |   10 +-
 libc/src/stdlib/rand_util.cpp                 |    6 +
 libc/src/stdlib/rand_util.h                   |   22 +
 libc/src/time/gpu/time_utils.cpp              |    2 +-
 libc/src/time/gpu/time_utils.h                |    4 +-
 libc/src/unistd/linux/syscall.cpp             |    2 +-
 libc/src/unistd/syscall.h                     |    4 +-
 libc/startup/gpu/amdgpu/start.cpp             |    2 +-
 libc/test/UnitTest/FPMatcher.h                |   10 +
 libc/test/UnitTest/LibcTest.cpp               |    4 +-
 libc/test/src/math/acosf_test.cpp             |   10 +-
 libc/test/src/math/acoshf_test.cpp            |   14 +-
 libc/test/src/math/asinf_test.cpp             |   10 +-
 libc/test/src/math/asinhf_test.cpp            |   14 +-
 libc/test/src/math/atanf_test.cpp             |   10 +-
 libc/test/src/math/atanhf_test.cpp            |   10 +-
 libc/test/src/math/cos_test.cpp               |    6 +-
 libc/test/src/math/cosf_test.cpp              |   12 +-
 libc/test/src/math/coshf_test.cpp             |   12 +-
 libc/test/src/math/erff_test.cpp              |   10 +-
 libc/test/src/math/exp10_test.cpp             |   10 +-
 libc/test/src/math/exp10f_test.cpp            |   14 +-
 libc/test/src/math/exp2_test.cpp              |   10 +-
 libc/test/src/math/exp2f_test.cpp             |   14 +-
 libc/test/src/math/exp_test.cpp               |   10 +-
 libc/test/src/math/expf_test.cpp              |   14 +-
 libc/test/src/math/explogxf_test.cpp          |   10 +-
 libc/test/src/math/expm1_test.cpp             |    8 +-
 libc/test/src/math/expm1f_test.cpp            |   14 +-
 libc/test/src/math/inv_trigf_utils_test.cpp   |    8 +-
 libc/test/src/math/log10_test.cpp             |   12 +-
 libc/test/src/math/log10f_test.cpp            |   10 +-
 libc/test/src/math/log1p_test.cpp             |   12 +-
 libc/test/src/math/log1pf_test.cpp            |   10 +-
 libc/test/src/math/log2_test.cpp              |   12 +-
 libc/test/src/math/log2f_test.cpp             |   10 +-
 libc/test/src/math/log_test.cpp               |   12 +-
 libc/test/src/math/logf_test.cpp              |   10 +-
 libc/test/src/math/sin_test.cpp               |    6 +-
 libc/test/src/math/sincosf_test.cpp           |   13 +-
 libc/test/src/math/sinf_test.cpp              |   15 +-
 libc/test/src/math/sinhf_test.cpp             |   14 +-
 libc/test/src/math/smoke/acosf_test.cpp       |    6 +-
 libc/test/src/math/smoke/acoshf_test.cpp      |    6 +-
 libc/test/src/math/smoke/asinf_test.cpp       |    6 +-
 libc/test/src/math/smoke/asinhf_test.cpp      |    6 +-
 libc/test/src/math/smoke/atanf_test.cpp       |    6 +-
 libc/test/src/math/smoke/atanhf_test.cpp      |    6 +-
 libc/test/src/math/smoke/cosf_test.cpp        |    6 +-
 libc/test/src/math/smoke/coshf_test.cpp       |    8 +-
 libc/test/src/math/smoke/erff_test.cpp        |    6 +-
 libc/test/src/math/smoke/exp10_test.cpp       |    6 +-
 libc/test/src/math/smoke/exp10f_test.cpp      |    6 +-
 libc/test/src/math/smoke/exp2_test.cpp        |    6 +-
 libc/test/src/math/smoke/exp2f_test.cpp       |    6 +-
 libc/test/src/math/smoke/exp_test.cpp         |    6 +-
 libc/test/src/math/smoke/expf_test.cpp        |    6 +-
 libc/test/src/math/smoke/expm1_test.cpp       |    8 +-
 libc/test/src/math/smoke/expm1f_test.cpp      |    6 +-
 libc/test/src/math/smoke/log10_test.cpp       |    6 +-
 libc/test/src/math/smoke/log10f_test.cpp      |    4 +-
 libc/test/src/math/smoke/log1p_test.cpp       |    6 +-
 libc/test/src/math/smoke/log1pf_test.cpp      |    4 +-
 libc/test/src/math/smoke/log2_test.cpp        |    6 +-
 libc/test/src/math/smoke/log2f_test.cpp       |    4 +-
 libc/test/src/math/smoke/log_test.cpp         |    6 +-
 libc/test/src/math/smoke/logf_test.cpp        |    4 +-
 libc/test/src/math/smoke/sincosf_test.cpp     |    6 +-
 libc/test/src/math/smoke/sinf_test.cpp        |    6 +-
 libc/test/src/math/smoke/sinhf_test.cpp       |   10 +-
 libc/test/src/math/smoke/tanf_test.cpp        |    6 +-
 libc/test/src/math/smoke/tanhf_test.cpp       |    6 +-
 libc/test/src/math/tan_test.cpp               |    6 +-
 libc/test/src/math/tanf_test.cpp              |   13 +-
 libc/test/src/math/tanhf_test.cpp             |   10 +-
 libc/test/src/stdio/sprintf_test.cpp          |    3 +
 libc/test/src/stdio/ungetc_test.cpp           |    8 +
 libc/test/src/stdlib/rand_test.cpp            |    3 +
 libc/test/src/time/CMakeLists.txt             |    7 -
 libc/test/src/time/gettimeofday_test.cpp      |   29 +-
 libc/test/src/unistd/CMakeLists.txt           |    2 +-
 libc/test/src/unistd/syscall_test.cpp         |    2 +-
 libc/utils/gpu/loader/amdgpu/Loader.cpp       |    2 +-
 libc/utils/gpu/server/rpc_server.cpp          |   23 +
 libcxx/CMakeLists.txt                         |   32 +-
 libcxx/benchmarks/CMakeLists.txt              |    3 +-
 libcxx/benchmarks/stop_token.bench.cpp        |  108 +
 libcxx/cmake/caches/AndroidNDK.cmake          |   38 +
 libcxx/cmake/config-ix.cmake                  |   15 -
 libcxx/docs/BuildingLibcxx.rst                |   17 +-
 libcxx/docs/Contributing.rst                  |   27 +-
 libcxx/docs/FeatureTestMacroTable.rst         |    2 +-
 libcxx/docs/ReleaseNotes/18.rst               |    7 +
 libcxx/docs/Status/Cxx23.rst                  |    1 -
 libcxx/docs/Status/Cxx23Papers.csv            |    8 +-
 libcxx/docs/Status/PSTLPaper.csv              |    2 +-
 libcxx/docs/Status/ParallelismProjects.csv    |    1 +
 libcxx/include/CMakeLists.txt                 |    3 +-
 libcxx/include/__algorithm/nth_element.h      |   30 +-
 libcxx/include/__algorithm/pstl_backend.h     |   17 +-
 .../pstl_backends/cpu_backends/for_each.h     |    4 +-
 .../pstl_backends/cpu_backends/transform.h    |    8 +-
 libcxx/include/__algorithm/pstl_move.h        |   78 +
 libcxx/include/__atomic/atomic_base.h         |    2 +-
 libcxx/include/__concepts/swappable.h         |    2 +-
 libcxx/include/__config                       |    5 +
 libcxx/include/__fwd/mdspan.h                 |    7 +-
 libcxx/include/__hash_table                   |  119 +-
 libcxx/include/__mdspan/extents.h             |    2 +-
 libcxx/include/__mdspan/layout_left.h         |   27 +-
 libcxx/include/__mdspan/layout_right.h        |   27 +-
 libcxx/include/__mdspan/layout_stride.h       |  366 +
 .../ranges_uninitialized_algorithms.h         |   20 +-
 .../__memory/uninitialized_algorithms.h       |   75 +-
 libcxx/include/__node_handle                  |    6 +-
 libcxx/include/__ranges/to.h                  |    6 +-
 libcxx/include/__std_clang_module             |    1 -
 libcxx/include/__tree                         |    2 +
 libcxx/include/__type_traits/promote.h        |    2 +-
 libcxx/include/algorithm                      |    1 +
 libcxx/include/csetjmp                        |    8 -
 libcxx/include/experimental/__simd/scalar.h   |    5 +
 libcxx/include/experimental/__simd/simd.h     |    4 +
 libcxx/include/experimental/__simd/utility.h  |   20 +
 libcxx/include/experimental/__simd/vec_ext.h  |   11 +
 libcxx/include/ext/hash_map                   |    8 +-
 libcxx/include/forward_list                   |   73 +-
 libcxx/include/functional                     |    2 +-
 libcxx/include/list                           |   68 +-
 libcxx/include/mdspan                         |   59 +
 libcxx/include/module.modulemap.in            |    6 +-
 libcxx/include/setjmp.h                       |   46 -
 libcxx/include/sstream                        |   11 +-
 libcxx/include/string                         |   30 +-
 libcxx/include/tuple                          |  257 +-
 libcxx/include/unordered_map                  |   12 +-
 libcxx/include/unordered_set                  |    4 +-
 libcxx/include/version                        |    2 +-
 libcxx/modules/std/mdspan.inc                 |    2 +-
 libcxx/src/new.cpp                            |  320 +-
 .../configs/llvm-libc++-android-ndk.cfg.in    |   47 +
 .../assert.sort.invalid_comparator.pass.cpp   |   81 +-
 .../alg.sorting/bad_comparator_values.h       |   69 +-
 ..._customization_points_not_working.pass.cpp |   13 +-
 .../atomics/atomics.align/align.pass.cpp      |    1 +
 .../test/libcxx/clang_modules_include.gen.py  |    4 +-
 .../assert.ctor.layout_stride.pass.cpp        |   83 +
 .../assert.ctor.layout_stride.pass.cpp        |   83 +
 .../layout_stride/assert.conversion.pass.cpp  |  112 +
 ...ert.ctor.extents_array.non_unique.pass.cpp |   67 +
 .../assert.ctor.extents_array.pass.cpp        |   73 +
 ...sert.ctor.extents_span.non_unique.pass.cpp |   70 +
 .../assert.ctor.extents_span.pass.cpp         |   80 +
 .../assert.index_operator.pass.cpp            |   88 +
 .../layout_stride/assert.stride.pass.cpp      |   36 +
 .../mdspan/mdspan/assert.conversion.pass.cpp  |    2 +-
 .../views/mdspan/mdspan/assert.size.pass.cpp  |    2 +-
 .../libcxx/gdb/gdb_pretty_printer_test.sh.cpp |    6 +
 .../print.fun/vprint_unicode_posix.pass.cpp   |    3 +
 .../libcxx/selftest/gen.cpp/empty.gen.cpp     |    2 +-
 .../libcxx/selftest/sh.cpp/run-success.sh.cpp |    2 +-
 .../string.capacity/allocation_size.pass.cpp  |   45 +
 .../string.capacity/max_size.pass.cpp         |    8 +-
 .../constinit_sso_string.compile.pass.cpp     |    8 +-
 .../test/libcxx/transitive_includes/cxx03.csv |    1 +
 .../test/libcxx/transitive_includes/cxx11.csv |    1 +
 .../test/libcxx/transitive_includes/cxx14.csv |    1 +
 .../test/libcxx/transitive_includes/cxx17.csv |    1 +
 .../test/libcxx/transitive_includes/cxx20.csv |    1 +
 .../test/libcxx/transitive_includes/cxx23.csv |    1 +
 .../test/libcxx/transitive_includes/cxx26.csv |    1 +
 .../alg.move/pstl.exception_handling.pass.cpp |   39 +
 .../alg.move/pstl.move.pass.cpp               |   99 +
 .../isalwayslockfree.pass.cpp                 |    2 +-
 .../atomics.types.generic/address.pass.cpp    |   10 +-
 .../atomics.types.generic/bool.pass.cpp       |   30 +-
 .../atomics.types.generic/integral.pass.cpp   |   10 +-
 .../atomic_is_lock_free.pass.cpp              |    4 +
 .../mdspan/{mdspan => }/CustomTestLayouts.h   |   47 +-
 .../views/mdspan/extents/ctad.pass.cpp        |    8 +
 .../layout_left/ctor.layout_stride.pass.cpp   |  114 +
 .../layout_right/ctor.layout_stride.pass.cpp  |  114 +
 .../mdspan/layout_stride/comparison.pass.cpp  |  198 +
 .../layout_stride/ctor.default.pass.cpp       |   73 +
 .../layout_stride/ctor.extents_array.pass.cpp |  138 +
 .../layout_stride/ctor.extents_span.pass.cpp  |  141 +
 .../ctor.strided_mapping.pass.cpp             |  187 +
 .../mdspan/layout_stride/deduction.pass.cpp   |   55 +
 .../mdspan/layout_stride/extents.verify.cpp   |   33 +
 .../layout_stride/index_operator.pass.cpp     |  121 +
 .../is_exhaustive_corner_case.pass.cpp        |   54 +
 .../mdspan/layout_stride/properties.pass.cpp  |  116 +
 .../layout_stride/required_span_size.pass.cpp |   56 +
 .../static_requirements.pass.cpp              |  131 +
 .../mdspan/layout_stride/stride.pass.cpp      |   57 +
 .../views/mdspan/mdspan/assign.pass.cpp       |    2 +-
 .../views/mdspan/mdspan/conversion.pass.cpp   |    2 +-
 .../views/mdspan/mdspan/conversion.verify.cpp |    2 +-
 .../views/mdspan/mdspan/ctor.copy.pass.cpp    |    2 +-
 .../views/mdspan/mdspan/ctor.default.pass.cpp |    2 +-
 .../mdspan/mdspan/ctor.dh_array.pass.cpp      |    2 +-
 .../mdspan/mdspan/ctor.dh_extents.pass.cpp    |    2 +-
 .../mdspan/mdspan/ctor.dh_integers.pass.cpp   |    2 +-
 .../views/mdspan/mdspan/ctor.dh_map.pass.cpp  |    2 +-
 .../mdspan/mdspan/ctor.dh_map_acc.pass.cpp    |    2 +-
 .../views/mdspan/mdspan/ctor.dh_span.pass.cpp |    2 +-
 .../views/mdspan/mdspan/ctor.move.pass.cpp    |    2 +-
 .../views/mdspan/mdspan/deduction.pass.cpp    |    2 +-
 .../mdspan/mdspan/index_operator.pass.cpp     |    2 +-
 .../views/mdspan/mdspan/move.pass.cpp         |    2 +-
 .../views/mdspan/mdspan/properties.pass.cpp   |    2 +-
 .../views/mdspan/mdspan/swap.pass.cpp         |    2 +-
 .../views/mdspan/mdspan/types.pass.cpp        |    2 +-
 .../depr.c.headers/setjmp_h.compile.pass.cpp  |    9 +-
 .../deprecated.verify.cpp                     |    0
 .../simd.class/simd_ctor_generator.pass.cpp   |   56 +
 .../fstreams/filebuf.members/close.pass.cpp   |    5 +
 .../directory_entry.obs/file_size.pass.cpp    |    4 +
 .../file_type_obs.pass.cpp                    |    4 +
 .../hard_link_count.pass.cpp                  |    4 +
 .../rec.dir.itr.members/increment.pass.cpp    |    4 +
 .../fs.op.funcs/fs.op.copy/copy.pass.cpp      |    4 +
 .../fs.op.copy_file/copy_file.pass.cpp        |    4 +
 .../create_hard_link.pass.cpp                 |    4 +
 .../fs.op.equivalent/equivalent.pass.cpp      |    4 +
 .../fs.op.hard_lk_ct/hard_link_count.pass.cpp |    4 +
 .../fs.op.is_empty/is_empty.pass.cpp          |    4 +
 .../fs.op.permissions/permissions.pass.cpp    |    5 +
 .../fs.op.funcs/fs.op.status/status.pass.cpp  |    4 +
 .../symlink_status.pass.cpp                   |    4 +
 .../ext.manip/get_money.pass.cpp              |    3 +
 .../ext.manip/put_money.pass.cpp              |    3 +
 .../print.fun/no_file_description.pass.cpp    |    3 +
 .../print.fun/print.file.pass.cpp             |    3 +
 .../print.fun/println.file.pass.cpp           |    3 +
 .../print.fun/vprint_nonunicode.file.pass.cpp |    3 +
 .../print.fun/vprint_unicode.file.pass.cpp    |    3 +
 .../narrow.stream.objects/cin.sh.cpp          |    4 +
 .../wide.stream.objects/wcin-imbue.sh.cpp     |    4 +
 .../wide.stream.objects/wcin.sh.cpp           |    4 +
 .../streambuf.put.area/pbump2gig.pass.cpp     |    5 +-
 .../str.allocator_propagation.pass.cpp        |  144 +
 .../istringstream.members/str.move.pass.cpp   |    8 +
 .../str.allocator_propagation.pass.cpp        |  115 +
 .../ostringstream.members/str.move.pass.cpp   |    8 +
 .../stringbuf/stringbuf.assign/move.pass.cpp  |   88 +-
 .../stringbuf.cons/move.alloc.pass.cpp        |   74 +-
 .../stringbuf/stringbuf.cons/move.pass.cpp    |   86 +-
 .../stringbuf.members/str.move.pass.cpp       |   43 +
 .../stringbuf/stringbuf.members/str.pass.cpp  |   45 +-
 .../stringbuf/stringbuf.members/view.pass.cpp |   28 +
 .../stringstream.members/gcount.pass.cpp      |    5 +
 .../str.allocator_propagation.pass.cpp        |  144 +
 .../stringstream.members/str.move.pass.cpp    |    8 +
 .../mdspan.version.compile.pass.cpp           |   32 +-
 .../version.version.compile.pass.cpp          |   32 +-
 .../support.runtime/csetjmp.pass.cpp          |   30 +-
 .../locale.collate.byname/compare.pass.cpp    |    3 +
 .../locale.collate.byname/transform.pass.cpp  |    3 +
 .../locale.ctype.byname/is_1.pass.cpp         |    3 +
 .../locale.ctype.byname/is_many.pass.cpp      |    3 +
 .../locale.ctype.byname/scan_is.pass.cpp      |    3 +
 .../locale.ctype.byname/scan_not.pass.cpp     |    3 +
 .../locale.ctype.byname/tolower_1.pass.cpp    |    3 +
 .../locale.ctype.byname/tolower_many.pass.cpp |    3 +
 .../locale.ctype.byname/toupper_1.pass.cpp    |    3 +
 .../locale.ctype.byname/toupper_many.pass.cpp |    3 +
 .../get_long_double_en_US.pass.cpp            |    3 +
 .../get_string_en_US.pass.cpp                 |    3 +
 .../put_long_double_en_US.pass.cpp            |    3 +
 .../put_string_en_US.pass.cpp                 |    3 +
 .../put_long_double.pass.cpp                  |    4 +
 .../re/re.traits/translate_nocase.pass.cpp    |    4 +
 .../implicit_deduction_guides.pass.cpp        |  267 +-
 .../implicit_deduction_guides.pass.cpp        |    7 +-
 .../notify_one.pass.cpp                       |    3 +
 .../thread.jthread/assign.move.pass.cpp       |   37 +-
 .../thread/thread.jthread/cons.move.pass.cpp  |    9 +-
 .../std/thread/thread.jthread/detach.pass.cpp |    7 +-
 .../std/thread/thread.jthread/dtor.pass.cpp   |   10 +-
 .../std/thread/thread.jthread/get_id.pass.cpp |    3 +-
 .../thread.jthread/get_stop_source.pass.cpp   |    3 +-
 .../thread.jthread/get_stop_token.pass.cpp    |    3 +-
 .../thread.jthread/join.deadlock.pass.cpp     |    5 +-
 .../std/thread/thread.jthread/join.pass.cpp   |   11 +-
 .../thread/thread.jthread/joinable.pass.cpp   |    7 +-
 .../thread.jthread/request_stop.pass.cpp      |    5 +-
 .../thread/thread.jthread/swap.free.pass.cpp  |    9 +-
 .../thread.jthread/swap.member.pass.cpp       |    9 +-
 .../locale-specific_form.pass.cpp             |    3 +
 .../overload_compare_iterator.h               |   73 +
 .../ranges_uninitialized_copy.pass.cpp        |   32 +
 .../ranges_uninitialized_copy_n.pass.cpp      |   32 +
 .../uninitialized_copy.pass.cpp               |   29 +
 .../uninitialized_copy_n.pass.cpp             |   30 +
 .../ranges_uninitialized_move.pass.cpp        |   32 +
 .../ranges_uninitialized_move_n.pass.cpp      |   32 +
 .../uninitialized_move.pass.cpp               |   30 +
 .../uninitialized_move_n.pass.cpp             |   30 +
 libcxx/test/support/counting_projection.h     |    2 +-
 libcxx/test/support/filesystem_test_helper.h  |   13 +-
 libcxx/test/support/make_test_thread.h        |   27 +
 libcxx/test/support/test_allocator.h          |   25 +
 libcxx/utils/adb_run.py                       |  256 +
 libcxx/utils/ci/BOT_OWNERS.txt                |    5 +
 libcxx/utils/ci/buildkite-pipeline.yml        |   19 -
 libcxx/utils/ci/run-buildbot                  |   64 +-
 libcxx/utils/ci/vendor/android/Dockerfile     |   83 +
 .../ci/vendor/android/Dockerfile.emulator     |   59 +
 .../vendor/android/build-emulator-images.sh   |   28 +
 .../ci/vendor/android/container-setup.sh      |   19 +
 .../ci/vendor/android/emulator-entrypoint.sh  |   49 +
 .../ci/vendor/android/emulator-functions.sh   |  110 +
 .../vendor/android/emulator-wait-for-ready.sh |   30 +
 .../ci/vendor/android/run-buildbot-container  |   31 +
 .../vendor/android/setup-env-for-emulator.sh  |   13 +
 .../utils/ci/vendor/android/start-emulator.sh |   43 +
 .../utils/ci/vendor/android/stop-emulator.sh  |   25 +
 libcxx/utils/data/ignore_format.txt           |   15 +-
 .../generate_feature_test_macro_components.py |    1 -
 libcxx/utils/libcxx/test/android.py           |   97 +
 libcxx/utils/libcxx/test/features.py          |   25 +
 libcxx/utils/libcxx/test/format.py            |   11 +-
 libcxx/utils/libcxx/test/params.py            |   11 +-
 libcxxabi/src/stdlib_new_delete.cpp           |  272 +-
 .../configs/llvm-libc++abi-android-ndk.cfg.in |   40 +
 libcxxabi/test/test_demangle.pass.cpp         |    4 +
 libunwind/src/AddressSpace.hpp                |    7 +-
 libunwind/src/EHHeaderParser.hpp              |   13 +
 libunwind/src/Registers.hpp                   |    4 +
 libunwind/src/UnwindCursor.hpp                |   96 +-
 libunwind/src/UnwindRegistersSave.S           |   14 +-
 libunwind/test/aix_signal_unwind.pass.sh.S    |  245 +
 lld/COFF/Chunks.cpp                           |    4 +-
 lld/COFF/Chunks.h                             |   48 +-
 lld/COFF/Config.h                             |    1 +
 lld/COFF/DLL.cpp                              |    8 +
 lld/COFF/DebugTypes.cpp                       |    4 +-
 lld/COFF/Driver.cpp                           |   29 +-
 lld/COFF/InputFiles.cpp                       |    6 +-
 lld/COFF/Options.td                           |    1 +
 lld/COFF/PDB.cpp                              |   13 +-
 lld/COFF/Writer.cpp                           |   23 +-
 lld/COFF/Writer.h                             |    6 +
 lld/ELF/Arch/Mips.cpp                         |    6 +-
 lld/ELF/Arch/RISCV.cpp                        |    2 +-
 lld/ELF/DWARF.h                               |    2 +-
 lld/ELF/Driver.cpp                            |   21 +-
 lld/ELF/InputFiles.cpp                        |    6 +-
 lld/ELF/InputSection.cpp                      |    5 +-
 lld/ELF/MapFile.cpp                           |    2 +-
 lld/ELF/Relocations.cpp                       |    6 +-
 lld/ELF/Symbols.cpp                           |   16 +-
 lld/ELF/Writer.cpp                            |   86 +-
 lld/MachO/InputFiles.cpp                      |    2 +-
 lld/docs/ld.lld.1                             |    4 +-
 lld/test/COFF/Inputs/loadconfig-arm64ec.s     |   97 +
 lld/test/COFF/arm64-dynamicbase.s             |    8 +
 lld/test/COFF/arm64ec-codemap.test            |  195 +
 lld/test/COFF/arm64ec-reloc.test              |   37 +
 lld/test/COFF/gnu-weak.test                   |    5 +
 lld/test/COFF/winsysroot.test                 |    9 +-
 lld/test/ELF/allow-shlib-undefined.s          |   11 +
 lld/test/ELF/cgprofile-txt.s                  |    6 +-
 lld/test/ELF/debug-dead-reloc.s               |   35 +-
 lld/test/ELF/gc-sections-tls.s                |   18 +-
 lld/test/ELF/linkerscript/discard-section.s   |   85 +-
 lld/wasm/WriterUtils.cpp                      |    4 +-
 lldb/CodeOwners.rst                           |    4 +-
 lldb/bindings/interface/SBTypeDocstrings.i    |    8 +
 lldb/docs/index.rst                           |    1 +
 lldb/docs/lldb-gdb-remote.txt                 |   12 +-
 lldb/docs/use/aarch64-linux.rst               |  204 +
 lldb/include/lldb/API/SBType.h                |    2 +
 .../include/lldb/Expression/DWARFExpression.h |   22 +-
 .../lldb/Expression/DWARFExpressionList.h     |   15 +-
 lldb/include/lldb/Symbol/CompileUnit.h        |    3 +-
 lldb/include/lldb/Symbol/CompilerType.h       |   23 +-
 lldb/include/lldb/Symbol/SymbolFile.h         |   13 +
 lldb/include/lldb/Symbol/Type.h               |   11 +-
 lldb/include/lldb/Symbol/TypeSystem.h         |   39 +-
 lldb/include/lldb/Target/Target.h             |    2 +-
 lldb/include/lldb/Utility/FileSpecList.h      |   16 -
 lldb/include/lldb/Utility/Scalar.h            |    2 +-
 lldb/packages/Python/lldbsuite/test/dotest.py |   20 +-
 .../Python/lldbsuite/test/lldbtest.py         |    6 +-
 .../Python/lldbsuite/test/test_categories.py  |    2 +-
 .../vscode.py => lldb-dap/dap_server.py}      |   18 +-
 .../lldbdap_testcase.py}                      |  104 +-
 lldb/source/API/SBType.cpp                    |    8 +
 .../Commands/CommandObjectBreakpoint.cpp      |    4 +-
 .../Commands/CommandObjectDWIMPrint.cpp       |   15 +-
 .../Commands/CommandObjectExpression.cpp      |    8 +-
 lldb/source/Commands/CommandObjectSource.h    |    5 +-
 lldb/source/Commands/CommandObjectTarget.cpp  |  262 +-
 lldb/source/Commands/Options.td               |    5 +
 lldb/source/Core/CMakeLists.txt               |    2 +-
 lldb/source/Core/Value.cpp                    |    4 +-
 lldb/source/DataFormatters/TypeFormat.cpp     |    6 +-
 lldb/source/DataFormatters/VectorType.cpp     |   65 +-
 lldb/source/Expression/DWARFExpression.cpp    |    1 +
 .../source/Interpreter/CommandInterpreter.cpp |    2 +-
 .../POSIX-DYLD/DynamicLoaderPOSIXDYLD.cpp     |   31 +-
 .../Language/CPlusPlus/LibCxxUnorderedMap.cpp |   21 +-
 .../Plugins/ObjectFile/PDB/ObjectFilePDB.cpp  |    2 +-
 .../MacOSX/PlatformAppleSimulator.cpp         |    3 +-
 .../SymbolFile/DWARF/AppleDWARFIndex.cpp      |    1 +
 .../SymbolFile/DWARF/AppleDWARFIndex.h        |    6 +-
 .../Plugins/SymbolFile/DWARF/DIERef.cpp       |    1 +
 lldb/source/Plugins/SymbolFile/DWARF/DIERef.h |   13 +-
 .../SymbolFile/DWARF/DWARFASTParser.cpp       |    1 +
 .../Plugins/SymbolFile/DWARF/DWARFASTParser.h |   40 +-
 .../SymbolFile/DWARF/DWARFASTParserClang.cpp  |    2 +
 .../SymbolFile/DWARF/DWARFASTParserClang.h    |  209 +-
 .../SymbolFile/DWARF/DWARFAttribute.cpp       |    1 +
 .../Plugins/SymbolFile/DWARF/DWARFAttribute.h |    5 +
 .../Plugins/SymbolFile/DWARF/DWARFBaseDIE.cpp |    7 +-
 .../Plugins/SymbolFile/DWARF/DWARFBaseDIE.h   |    6 +-
 .../SymbolFile/DWARF/DWARFCompileUnit.cpp     |    1 +
 .../SymbolFile/DWARF/DWARFCompileUnit.h       |    8 +-
 .../Plugins/SymbolFile/DWARF/DWARFContext.cpp |    1 +
 .../Plugins/SymbolFile/DWARF/DWARFContext.h   |    6 +-
 .../Plugins/SymbolFile/DWARF/DWARFDIE.cpp     |    1 +
 .../Plugins/SymbolFile/DWARF/DWARFDIE.h       |   17 +-
 .../SymbolFile/DWARF/DWARFDataExtractor.h     |    2 +-
 .../SymbolFile/DWARF/DWARFDebugArangeSet.cpp  |    1 +
 .../SymbolFile/DWARF/DWARFDebugArangeSet.h    |    6 +-
 .../SymbolFile/DWARF/DWARFDebugAranges.cpp    |    1 +
 .../SymbolFile/DWARF/DWARFDebugAranges.h      |   11 +-
 .../SymbolFile/DWARF/DWARFDebugInfo.cpp       |    4 +-
 .../Plugins/SymbolFile/DWARF/DWARFDebugInfo.h |   14 +-
 .../SymbolFile/DWARF/DWARFDebugInfoEntry.cpp  |  142 +-
 .../SymbolFile/DWARF/DWARFDebugInfoEntry.h    |   24 +-
 .../SymbolFile/DWARF/DWARFDebugMacro.cpp      |    1 +
 .../SymbolFile/DWARF/DWARFDebugMacro.h        |   27 +-
 .../SymbolFile/DWARF/DWARFDebugRanges.cpp     |    1 +
 .../SymbolFile/DWARF/DWARFDebugRanges.h       |    8 +-
 .../SymbolFile/DWARF/DWARFDeclContext.cpp     |    1 +
 .../SymbolFile/DWARF/DWARFDeclContext.h       |    8 +-
 .../Plugins/SymbolFile/DWARF/DWARFDefines.cpp |    6 +-
 .../Plugins/SymbolFile/DWARF/DWARFDefines.h   |    6 +-
 .../SymbolFile/DWARF/DWARFFormValue.cpp       |    1 +
 .../Plugins/SymbolFile/DWARF/DWARFFormValue.h |   15 +-
 .../Plugins/SymbolFile/DWARF/DWARFIndex.cpp   |    1 +
 .../Plugins/SymbolFile/DWARF/DWARFIndex.h     |    6 +-
 .../SymbolFile/DWARF/DWARFTypeUnit.cpp        |    1 +
 .../Plugins/SymbolFile/DWARF/DWARFTypeUnit.h  |    8 +-
 .../Plugins/SymbolFile/DWARF/DWARFUnit.cpp    |   28 +-
 .../Plugins/SymbolFile/DWARF/DWARFUnit.h      |   64 +-
 .../SymbolFile/DWARF/DebugNamesDWARFIndex.cpp |    3 +-
 .../SymbolFile/DWARF/DebugNamesDWARFIndex.h   |    6 +-
 .../SymbolFile/DWARF/ManualDWARFIndex.cpp     |    1 +
 .../SymbolFile/DWARF/ManualDWARFIndex.h       |    6 +-
 .../Plugins/SymbolFile/DWARF/NameToDIE.cpp    |    1 +
 .../Plugins/SymbolFile/DWARF/NameToDIE.h      |   25 +-
 .../SymbolFile/DWARF/SymbolFileDWARF.cpp      |   89 +-
 .../SymbolFile/DWARF/SymbolFileDWARF.h        |  332 +-
 .../DWARF/SymbolFileDWARFDebugMap.cpp         |   44 +-
 .../DWARF/SymbolFileDWARFDebugMap.h           |  230 +-
 .../SymbolFile/DWARF/SymbolFileDWARFDwo.cpp   |    1 +
 .../SymbolFile/DWARF/SymbolFileDWARFDwo.h     |   27 +-
 .../SymbolFile/DWARF/UniqueDWARFASTType.cpp   |    1 +
 .../SymbolFile/DWARF/UniqueDWARFASTType.h     |   18 +-
 .../TypeSystem/Clang/TypeSystemClang.cpp      |  427 +-
 .../TypeSystem/Clang/TypeSystemClang.h        |   19 +-
 lldb/source/Symbol/CompileUnit.cpp            |    6 +-
 lldb/source/Symbol/CompilerType.cpp           |   43 +-
 lldb/source/Symbol/SymbolFile.cpp             |    1 +
 lldb/source/Symbol/Type.cpp                   |   59 +-
 lldb/source/Symbol/TypeSystem.cpp             |    5 +
 lldb/source/Target/Target.cpp                 |    4 +-
 lldb/source/Utility/Diagnostics.cpp           |    6 +-
 lldb/source/Utility/FileSpecList.cpp          |   12 -
 lldb/source/Utility/Scalar.cpp                |   10 +-
 .../commands/expression/fixits/TestFixIts.py  |    8 +-
 .../queue/TestQueueFromStdModule.py           |    5 +
 .../dump-separate-debug-info/dwo/Makefile     |    4 +
 .../dwo/TestDumpDwo.py                        |  126 +
 .../dump-separate-debug-info/dwo/foo.cpp      |    3 +
 .../target/dump-separate-debug-info/dwo/foo.h |    6 +
 .../dump-separate-debug-info/dwo/main.cpp     |    3 +
 .../dump-separate-debug-info/oso}/Makefile    |    0
 .../oso/TestDumpOso.py                        |  120 +
 .../dump-separate-debug-info/oso/foo.cpp      |    3 +
 .../target/dump-separate-debug-info/oso/foo.h |    6 +
 .../dump-separate-debug-info/oso/main.cpp     |    3 +
 .../builtin-formats/TestBuiltinFormats.py     |   16 +-
 .../data-formatter/builtin-formats/main.cpp   |    2 +
 .../TestDataFormatterObjCNSDate.py            |    4 +-
 .../vector-types/TestVectorTypesFormatting.py |    7 +
 .../data-formatter/vector-types/main.cpp      |    4 +-
 .../completions => lang/cpp/fixits}/Makefile  |    0
 .../test/API/lang/cpp/fixits/TestCppFixIts.py |   44 +
 lldb/test/API/lang/cpp/fixits/main.cpp        |    5 +
 lldb/test/API/lit.cfg.py                      |    2 +-
 lldb/test/API/python_api/type/TestTypeList.py |   31 +
 lldb/test/API/python_api/type/main.cpp        |    5 +
 .../{lldb-vscode => lldb-dap}/attach/Makefile |    0
 .../attach/TestDAP_attach.py}                 |   16 +-
 .../{lldb-vscode => lldb-dap}/attach/main.c   |    0
 .../breakpoint-events/Makefile                |    0
 .../TestDAP_breakpointEvents.py}              |   36 +-
 .../breakpoint-events/foo.cpp                 |    2 +-
 .../breakpoint-events/foo.h                   |    0
 .../breakpoint-events/main.cpp                |    0
 .../breakpoint/Makefile                       |    0
 .../breakpoint/TestDAP_logpoints.py}          |   26 +-
 .../breakpoint/TestDAP_setBreakpoints.py}     |   50 +-
 .../TestDAP_setExceptionBreakpoints.py}       |   10 +-
 .../TestDAP_setFunctionBreakpoints.py}        |   28 +-
 .../breakpoint/main.cpp                       |   10 +-
 .../breakpoint/other.c                        |    0
 lldb/test/API/tools/lldb-dap/categories       |    1 +
 .../console => lldb-dap/completions}/Makefile |    0
 .../completions/TestDAP_completions.py}       |   36 +-
 .../completions/main.cpp                      |    8 +-
 .../disconnect => lldb-dap/console}/Makefile  |    0
 .../console/TestDAP_console.py}               |   20 +-
 .../TestDAP_redirection_to_console.py}        |   14 +-
 .../console/main.cpp                          |    0
 .../coreFile/TestDAP_coreFile.py}             |   10 +-
 .../coreFile/linux-x86_64.core                |  Bin
 .../coreFile/linux-x86_64.out                 |  Bin
 .../{lldb-vscode => lldb-dap}/coreFile/main.c |    0
 .../correct-thread/Makefile                   |    0
 .../correct-thread/TestDAP_correct_thread.py} |   12 +-
 .../correct-thread/main.c                     |    6 +-
 .../disassemble/Makefile                      |    0
 .../disassemble/TestDAP_disassemble.py}       |    8 +-
 .../API/tools/lldb-dap/disassemble/main.c     |   30 +
 .../disconnect}/Makefile                      |    0
 .../disconnect/TestDAP_disconnect.py}         |   16 +-
 .../disconnect/main.cpp                       |    0
 .../test/API/tools/lldb-dap/evaluate/Makefile |    3 +
 .../evaluate/TestDAP_evaluate.py}             |   12 +-
 .../evaluate/foo.cpp                          |    0
 .../{lldb-vscode => lldb-dap}/evaluate/foo.h  |    0
 .../evaluate/main.cpp                         |    4 +-
 .../variables => lldb-dap/exception}/Makefile |    0
 .../exception/TestDAP_exception.py}           |    8 +-
 .../exception/main.cpp                        |    0
 .../{lldb-vscode => lldb-dap}/launch/Makefile |    0
 .../launch/TestDAP_launch.py}                 |   34 +-
 .../{lldb-vscode => lldb-dap}/launch/main.c   |    6 +-
 .../{lldb-vscode => lldb-dap}/module/Makefile |    0
 .../module/TestDAP_module.py}                 |   20 +-
 lldb/test/API/tools/lldb-dap/module/foo.cpp   |    1 +
 .../{lldb-vscode => lldb-dap}/module/foo.h    |    0
 .../{lldb-vscode => lldb-dap}/module/main.cpp |    0
 .../optimized/Makefile                        |    0
 .../optimized/TestDAP_optimized.py}           |   14 +-
 .../optimized/main.cpp                        |    0
 .../restart/Makefile                          |    0
 .../restart/TestDAP_restart.py}               |   36 +-
 .../restart/TestDAP_restart_runInTerminal.py} |   30 +-
 .../{lldb-vscode => lldb-dap}/restart/main.c  |    2 +-
 .../runInTerminal/Makefile                    |    0
 .../runInTerminal/TestDAP_runInTerminal.py}   |   40 +-
 .../runInTerminal/main.c                      |    0
 .../stackTrace/Makefile                       |    0
 .../stackTrace/TestDAP_stackTrace.py}         |    8 +-
 .../stackTrace/main.c                         |    4 +-
 .../stackTraceMissingFunctionName/Makefile    |    0
 .../TestDAP_stackTraceMissingFunctionName.py} |    8 +-
 .../stackTraceMissingFunctionName/main.cpp    |    0
 .../startDebugging/Makefile                   |    0
 .../startDebugging/TestDAP_startDebugging.py} |   18 +-
 .../startDebugging/main.c                     |    0
 .../{lldb-vscode => lldb-dap}/step/Makefile   |    0
 .../step/TestDAP_step.py}                     |   10 +-
 lldb/test/API/tools/lldb-dap/step/main.cpp    |    8 +
 .../stop-hooks/Makefile                       |    0
 .../stop-hooks/TestDAP_stop_hooks.py}         |   10 +-
 .../stop-hooks/main.c                         |    0
 .../terminated-event/Makefile                 |    0
 .../TestDAP_terminatedEvent.py}               |   12 +-
 .../tools/lldb-dap/terminated-event/foo.cpp   |    1 +
 .../terminated-event/foo.h                    |    0
 .../terminated-event/main.cpp                 |    2 +-
 .../API/tools/lldb-dap/variables/Makefile     |    3 +
 .../variables/TestDAP_variables.py}           |   86 +-
 .../variables/main.cpp                        |    4 +-
 lldb/test/API/tools/lldb-vscode/categories    |    1 -
 .../API/tools/lldb-vscode/disassemble/main.c  |   30 -
 .../test/API/tools/lldb-vscode/module/foo.cpp |    3 -
 lldb/test/API/tools/lldb-vscode/step/main.cpp |   10 -
 .../lldb-vscode/terminated-event/foo.cpp      |    3 -
 lldb/test/CMakeLists.txt                      |    4 +-
 .../Shell/Breakpoint/breakpoint-command.test  |    6 +-
 .../Shell/{VSCode => DAP}/TestOptions.test    |    2 +-
 .../DWARF/Inputs/dwo-missing-error.c          |    1 +
 .../SymbolFile/DWARF/dwo-missing-error.test   |   34 +
 lldb/test/Shell/helper/toolchain.py           |    2 +-
 lldb/tools/CMakeLists.txt                     |    2 +-
 .../BreakpointBase.cpp                        |   12 +-
 .../BreakpointBase.h                          |    8 +-
 .../{lldb-vscode => lldb-dap}/CMakeLists.txt  |   16 +-
 .../VSCode.cpp => lldb-dap/DAP.cpp}           |  156 +-
 .../{lldb-vscode/VSCode.h => lldb-dap/DAP.h}  |   26 +-
 .../VSCodeForward.h => lldb-dap/DAPForward.h} |   10 +-
 .../ExceptionBreakpoint.cpp                   |   10 +-
 .../ExceptionBreakpoint.h                     |    8 +-
 .../{lldb-vscode => lldb-dap}/FifoFiles.cpp   |    4 +-
 .../{lldb-vscode => lldb-dap}/FifoFiles.h     |   10 +-
 .../FunctionBreakpoint.cpp                    |    8 +-
 .../FunctionBreakpoint.h                      |    8 +-
 .../{lldb-vscode => lldb-dap}/IOStream.cpp    |    2 +-
 .../{lldb-vscode => lldb-dap}/IOStream.h      |    8 +-
 .../{lldb-vscode => lldb-dap}/JSONUtils.cpp   |   32 +-
 .../{lldb-vscode => lldb-dap}/JSONUtils.h     |   37 +-
 .../{lldb-vscode => lldb-dap}/LLDBUtils.cpp   |   10 +-
 .../{lldb-vscode => lldb-dap}/LLDBUtils.h     |   26 +-
 .../{lldb-vscode => lldb-dap}/Options.td      |    6 +-
 .../OutputRedirector.cpp                      |    4 +-
 .../OutputRedirector.h                        |   10 +-
 .../ProgressEvent.cpp                         |    2 +-
 .../{lldb-vscode => lldb-dap}/ProgressEvent.h |   12 +-
 .../tools/{lldb-vscode => lldb-dap}/README.md |  188 +-
 .../RunInTerminal.cpp                         |    6 +-
 .../{lldb-vscode => lldb-dap}/RunInTerminal.h |   10 +-
 .../SourceBreakpoint.cpp                      |    8 +-
 .../SourceBreakpoint.h                        |    8 +-
 .../lldb-dap-Info.plist.in}                   |    4 +-
 .../lldb-vscode.cpp => lldb-dap/lldb-dap.cpp} |  627 +-
 .../{lldb-vscode => lldb-dap}/package.json    |   14 +-
 .../syntaxes/arm.disasm                       |    0
 .../syntaxes/arm64.disasm                     |    0
 .../syntaxes/disassembly.json                 |    0
 .../syntaxes/x86.disasm                       |    0
 lldb/tools/lldb-vscode                        |    1 +
 .../Expression/DWARFExpressionTest.cpp        |    1 +
 .../Python/PythonDataObjectsTests.cpp         |    1 -
 lldb/unittests/Symbol/SymtabTest.cpp          |    1 +
 lldb/unittests/Symbol/TestLineEntry.cpp       |    3 +-
 .../DWARF/DWARFASTParserClangTests.cpp        |    1 +
 .../SymbolFile/DWARF/DWARFDIETest.cpp         |    1 +
 .../DWARF/DWARFIndexCachingTest.cpp           |    1 +
 .../SymbolFile/DWARF/DWARFUnitTest.cpp        |    1 +
 .../SymbolFile/DWARF/SymbolFileDWARFTests.cpp |    1 +
 .../SymbolFile/DWARF/XcodeSDKModuleTests.cpp  |    1 +
 .../SymbolFile/PDB/SymbolFilePDBTests.cpp     |    4 +-
 .../Symbol/YAMLModuleTester.cpp               |    1 +
 .../TestingSupport/Symbol/YAMLModuleTester.h  |    6 +-
 lldb/unittests/Utility/ScalarTest.cpp         |    2 +-
 .../lldb-server/tests/MessageObjects.cpp      |   19 +-
 llvm/CMakeLists.txt                           |    6 +
 llvm/CODE_OWNERS.TXT                          |    8 +-
 llvm/cmake/modules/AddLLVM.cmake              |   11 +
 llvm/cmake/modules/FindFFI.cmake              |   20 +-
 llvm/cmake/modules/FindTerminfo.cmake         |   20 +-
 llvm/cmake/modules/HandleLLVMOptions.cmake    |   41 +-
 llvm/docs/AMDGPUUsage.rst                     |   66 +-
 llvm/docs/BitCodeFormat.rst                   |    9 +
 llvm/docs/CMake.rst                           |   16 +
 llvm/docs/CodingStandards.rst                 |    2 +-
 llvm/docs/CommandGuide/llvm-remarkutil.rst    |   78 +-
 llvm/docs/GettingInvolved.rst                 |    5 +
 llvm/docs/GitRepositoryPolicy.md              |    2 +-
 llvm/docs/GlobalISel/MIRPatterns.rst          |    4 +-
 llvm/docs/InstrRefDebugInfo.md                |    2 +-
 llvm/docs/LangRef.rst                         |   46 +-
 llvm/docs/MarkdownQuickstartTemplate.md       |    2 +-
 llvm/docs/NewPassManager.rst                  |    2 +
 llvm/docs/PointerAuth.md                      |    6 +-
 llvm/docs/ReleaseNotes.rst                    |   26 +
 llvm/docs/TableGen/ProgRef.rst                |   45 +-
 llvm/docs/TestSuiteGuide.md                   |    2 +-
 llvm/docs/WritingAnLLVMPass.rst               |   29 +-
 .../MyFirstLanguageFrontend/LangImpl04.rst    |   93 +-
 .../Kaleidoscope/Chapter4/CMakeLists.txt      |    1 +
 llvm/examples/Kaleidoscope/Chapter4/toy.cpp   |   70 +-
 .../Kaleidoscope/Chapter5/CMakeLists.txt      |    1 +
 llvm/examples/Kaleidoscope/Chapter5/toy.cpp   |   72 +-
 .../Kaleidoscope/Chapter6/CMakeLists.txt      |    1 +
 llvm/examples/Kaleidoscope/Chapter6/toy.cpp   |   72 +-
 .../Kaleidoscope/Chapter7/CMakeLists.txt      |    1 +
 llvm/examples/Kaleidoscope/Chapter7/toy.cpp   |   74 +-
 llvm/include/llvm/ADT/GenericCycleImpl.h      |   63 +-
 llvm/include/llvm/ADT/GenericCycleInfo.h      |    7 +
 llvm/include/llvm/ADT/GenericUniformityImpl.h |    2 -
 llvm/include/llvm/ADT/IndexedMap.h            |    1 +
 llvm/include/llvm/ADT/STLExtras.h             |    1 -
 .../llvm/{CodeGen => ADT}/StableHashing.h     |    6 +-
 llvm/include/llvm/ADT/ilist_iterator.h        |  191 +
 llvm/include/llvm/ADT/ilist_node.h            |   35 +-
 llvm/include/llvm/ADT/ilist_node_options.h    |   29 +-
 llvm/include/llvm/ADT/simple_ilist.h          |   16 +-
 llvm/include/llvm/Analysis/AliasAnalysis.h    |    8 +-
 llvm/include/llvm/Analysis/CaptureTracking.h  |    2 +-
 llvm/include/llvm/Analysis/InlineAdvisor.h    |    5 +-
 .../llvm/Analysis/InlineModelFeatureMaps.h    |    4 +-
 llvm/include/llvm/Analysis/MLModelRunner.h    |    4 +-
 .../llvm/Analysis/ModelUnderTrainingRunner.h  |    5 +-
 .../llvm/Analysis/TargetLibraryInfo.def       |    2 +-
 llvm/include/llvm/Analysis/TensorSpec.h       |   14 +-
 llvm/include/llvm/Analysis/Utils/TFUtils.h    |    4 +-
 llvm/include/llvm/Analysis/ValueTracking.h    |   23 +-
 llvm/include/llvm/Analysis/WithCache.h        |   72 +
 llvm/include/llvm/AsmParser/LLToken.h         |    1 +
 llvm/include/llvm/BinaryFormat/Dwarf.def      |    1 +
 llvm/include/llvm/BinaryFormat/ELF.h          |    1 +
 llvm/include/llvm/BinaryFormat/MachO.def      |   16 +
 llvm/include/llvm/BinaryFormat/MachO.h        |   15 +-
 llvm/include/llvm/BinaryFormat/MsgPack.h      |    2 +-
 llvm/include/llvm/Bitcode/LLVMBitCodes.h      |    1 +
 llvm/include/llvm/Bitstream/BitstreamReader.h |    3 +-
 llvm/include/llvm/Bitstream/BitstreamWriter.h |   16 +-
 .../CodeGen/BasicBlockSectionsProfileReader.h |   77 +-
 llvm/include/llvm/CodeGen/BasicTTIImpl.h      |   70 +-
 .../llvm/CodeGen/GlobalISel/CombinerHelper.h  |    4 -
 .../CodeGen/GlobalISel/GIMatchTableExecutor.h |   24 +-
 .../GlobalISel/GIMatchTableExecutorImpl.h     |   51 +-
 .../CodeGen/GlobalISel/LegacyLegalizerInfo.h  |   10 -
 .../llvm/CodeGen/GlobalISel/LegalizerHelper.h |    1 +
 .../CodeGen/GlobalISel/MachineIRBuilder.h     |    3 +
 llvm/include/llvm/CodeGen/GlobalISel/Utils.h  |    6 +-
 llvm/include/llvm/CodeGen/LiveVariables.h     |   15 -
 llvm/include/llvm/CodeGen/MachineInstr.h      |    6 -
 llvm/include/llvm/CodeGen/MachineStableHash.h |    2 +-
 llvm/include/llvm/CodeGen/Passes.h            |    6 -
 llvm/include/llvm/CodeGen/SelectionDAG.h      |    6 +-
 llvm/include/llvm/CodeGen/SlotIndexes.h       |   47 +-
 llvm/include/llvm/CodeGen/TargetInstrInfo.h   |   11 +-
 llvm/include/llvm/CodeGen/TargetLowering.h    |   26 +-
 llvm/include/llvm/Config/llvm-config.h.cmake  |    2 +-
 .../DebugInfo/CodeView/SymbolDeserializer.h   |    2 +-
 .../DebugInfo/CodeView/TypeDeserializer.h     |    2 +-
 llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h |    4 +-
 .../llvm/DebugInfo/DWARF/DWARFVerifier.h      |    1 -
 .../llvm/DebugInfo/MSF/MappedBlockStream.h    |    8 +-
 .../llvm/DebugInfo/PDB/Native/FormatUtil.h    |   14 +-
 .../llvm/ExecutionEngine/JITLink/JITLink.h    |    8 +-
 .../llvm/ExecutionEngine/JITLink/ppc64.h      |   32 +-
 .../llvm/ExecutionEngine/Orc/ExecutionUtils.h |    2 +-
 .../llvm/ExecutionEngine/Orc/MachOBuilder.h   |    2 +-
 .../llvm/ExecutionEngine/RuntimeDyldChecker.h |    6 +-
 llvm/include/llvm/Frontend/OpenACC/ACC.td     |    2 +-
 .../llvm/Frontend/OpenMP/OMPIRBuilder.h       |   14 +-
 .../include/llvm/Frontend/OpenMP/OMPKinds.def |    1 +
 llvm/include/llvm/IR/Attributes.td            |    3 +
 llvm/include/llvm/IR/BasicBlock.h             |   28 +-
 llvm/include/llvm/IR/CallingConv.h            |    3 +
 llvm/include/llvm/IR/DerivedTypes.h           |    5 +-
 .../llvm/IR/GenericConvergenceVerifierImpl.h  |    4 +-
 llvm/include/llvm/IR/GlobalAlias.h            |    2 +-
 llvm/include/llvm/IR/GlobalIFunc.h            |    2 +-
 llvm/include/llvm/IR/GlobalVariable.h         |   13 +-
 llvm/include/llvm/IR/Instruction.h            |   23 +-
 llvm/include/llvm/IR/Instructions.h           |    2 +-
 llvm/include/llvm/IR/Intrinsics.td            |   26 +-
 llvm/include/llvm/IR/IntrinsicsAArch64.td     |   16 +
 llvm/include/llvm/IR/IntrinsicsAMDGPU.td      |   24 +-
 llvm/include/llvm/IR/IntrinsicsRISCVXsf.td    |   20 +
 llvm/include/llvm/IR/IntrinsicsX86.td         |   10 +-
 llvm/include/llvm/IR/SymbolTableListTraits.h  |   16 +-
 llvm/include/llvm/IR/ValueSymbolTable.h       |    5 +-
 llvm/include/llvm/IRReader/IRReader.h         |    1 -
 llvm/include/llvm/InitializePasses.h          |    1 -
 llvm/include/llvm/LinkAllPasses.h             |    1 -
 llvm/include/llvm/MC/MCAsmBackend.h           |    5 +-
 llvm/include/llvm/MC/MCInstrAnalysis.h        |   15 +
 llvm/include/llvm/MC/MCMachObjectWriter.h     |    3 +-
 llvm/include/llvm/Object/COFF.h               |    3 +-
 llvm/include/llvm/Object/COFFImportFile.h     |    2 +
 llvm/include/llvm/Object/ELFObjectFile.h      |   13 +-
 llvm/include/llvm/Object/ELFTypes.h           |   12 +-
 llvm/include/llvm/Object/FaultMapParser.h     |    2 +-
 llvm/include/llvm/Object/GOFF.h               |    3 +-
 llvm/include/llvm/Object/MachO.h              |   15 +-
 llvm/include/llvm/Object/StackMapParser.h     |    5 +-
 llvm/include/llvm/Object/XCOFFObjectFile.h    |   69 +-
 llvm/include/llvm/Option/ArgList.h            |    7 +-
 .../ProfileData/Coverage/CoverageMapping.h    |   62 +-
 .../Coverage/CoverageMappingReader.h          |    2 +-
 llvm/include/llvm/ProfileData/FunctionId.h    |  213 +
 llvm/include/llvm/ProfileData/GCOV.h          |    2 +-
 llvm/include/llvm/ProfileData/HashKeyMap.h    |  129 +
 .../llvm/ProfileData/InstrProfData.inc        |    8 +-
 .../llvm/ProfileData/InstrProfReader.h        |   22 +-
 .../llvm/ProfileData/InstrProfWriter.h        |    2 +-
 llvm/include/llvm/ProfileData/MemProf.h       |   44 +-
 llvm/include/llvm/ProfileData/SampleProf.h    |  283 +-
 .../llvm/ProfileData/SampleProfReader.h       |   30 +-
 .../llvm/ProfileData/SampleProfWriter.h       |   16 +-
 llvm/include/llvm/Remarks/Remark.h            |    4 +
 .../llvm/Support/AMDHSAKernelDescriptor.h     |   46 +-
 llvm/include/llvm/Support/BinaryByteStream.h  |    2 +-
 llvm/include/llvm/Support/Chrono.h            |   26 +
 llvm/include/llvm/Support/Debug.h             |    4 +-
 .../include/llvm/Support/ELFAttributeParser.h |    2 +-
 llvm/include/llvm/Support/Endian.h            |  158 +-
 llvm/include/llvm/Support/EndianStream.h      |    1 -
 llvm/include/llvm/Support/HashBuilder.h       |   16 +-
 llvm/include/llvm/Support/KnownBits.h         |    5 +
 llvm/include/llvm/Support/MD5.h               |    4 +-
 llvm/include/llvm/Support/OnDiskHashTable.h   |   21 +-
 llvm/include/llvm/Support/Regex.h             |    5 +-
 llvm/include/llvm/Support/SwapByteOrder.h     |    1 -
 .../llvm/Support/ThreadSafeAllocator.h        |    1 -
 llvm/include/llvm/Support/VersionTuple.h      |    2 +-
 .../Support/X86DisassemblerDecoderCommon.h    |    5 +-
 llvm/include/llvm/Support/YAMLTraits.h        |    6 +-
 llvm/include/llvm/TableGen/Error.h            |    1 +
 llvm/include/llvm/TableGen/Record.h           |   21 +-
 .../llvm/TargetParser/AArch64TargetParser.h   |    4 +-
 .../llvm/TargetParser/ARMTargetParser.h       |    8 +
 .../llvm/TargetParser/X86TargetParser.def     |    5 +
 .../llvm/TargetParser/X86TargetParser.h       |    2 +
 llvm/include/llvm/TextAPI/InterfaceFile.h     |    5 +-
 llvm/include/llvm/TextAPI/SymbolSet.h         |    5 +-
 .../llvm/Transforms/HipStdPar/HipStdPar.h     |   46 +
 .../Transforms/IPO/FunctionSpecialization.h   |    2 -
 .../llvm/Transforms/IPO/ProfiledCallGraph.h   |   33 +-
 .../Transforms/IPO/SampleContextTracker.h     |   29 +-
 .../Transforms/InstCombine/InstCombiner.h     |   13 +-
 .../llvm/Transforms/Instrumentation/CFGMST.h  |    2 -
 llvm/include/llvm/Transforms/Scalar.h         |    6 -
 llvm/include/llvm/Transforms/Scalar/GVN.h     |    2 +-
 .../llvm/Transforms/Utils/CodeExtractor.h     |    9 +-
 .../llvm/Transforms/Utils/CodeLayout.h        |    2 +
 .../Transforms/Utils/SampleProfileInference.h |    2 +-
 .../Utils/SampleProfileLoaderBaseImpl.h       |    2 +-
 llvm/lib/Analysis/BasicAliasAnalysis.cpp      |    3 +-
 llvm/lib/Analysis/CaptureTracking.cpp         |    9 +-
 .../Analysis/DevelopmentModeInlineAdvisor.cpp |    2 +-
 .../Analysis/FunctionPropertiesAnalysis.cpp   |    4 +-
 llvm/lib/Analysis/InlineOrder.cpp             |   21 +-
 llvm/lib/Analysis/LazyValueInfo.cpp           |   40 +
 llvm/lib/Analysis/Local.cpp                   |   58 +-
 llvm/lib/Analysis/LoopAccessAnalysis.cpp      |    2 -
 llvm/lib/Analysis/MemoryBuiltins.cpp          |    5 +-
 llvm/lib/Analysis/RegionPrinter.cpp           |    1 -
 llvm/lib/Analysis/TFLiteUtils.cpp             |    4 +-
 llvm/lib/Analysis/ValueTracking.cpp           |  141 +-
 llvm/lib/Analysis/VectorUtils.cpp             |    1 -
 llvm/lib/AsmParser/LLLexer.cpp                |    7 +-
 llvm/lib/AsmParser/LLParser.cpp               |    2 +
 llvm/lib/Bitcode/Reader/BitcodeReader.cpp     |    4 +-
 llvm/lib/Bitcode/Reader/MetadataLoader.cpp    |   10 +-
 llvm/lib/Bitcode/Writer/BitcodeWriter.cpp     |    2 +
 llvm/lib/CodeGen/AsmPrinter/AccelTable.cpp    |    1 -
 llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp    |    2 +-
 llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp     |    1 -
 llvm/lib/CodeGen/AsmPrinter/EHStreamer.cpp    |    1 -
 llvm/lib/CodeGen/BasicBlockSections.cpp       |   73 +-
 .../BasicBlockSectionsProfileReader.cpp       |  145 +-
 llvm/lib/CodeGen/CodeGenPrepare.cpp           |    2 +
 llvm/lib/CodeGen/EarlyIfConversion.cpp        |    6 +-
 .../CodeGen/GlobalISel/LegalizerHelper.cpp    |   87 +-
 llvm/lib/CodeGen/GlobalISel/Utils.cpp         |   12 +-
 llvm/lib/CodeGen/InlineSpiller.cpp            |    3 +-
 llvm/lib/CodeGen/LiveIntervalCalc.cpp         |    1 -
 llvm/lib/CodeGen/LiveVariables.cpp            |    1 -
 llvm/lib/CodeGen/MIRFSDiscriminator.cpp       |    2 +
 llvm/lib/CodeGen/MIRParser/MIRParser.cpp      |    1 -
 llvm/lib/CodeGen/MIRSampleProfile.cpp         |    2 +
 llvm/lib/CodeGen/MachineBasicBlock.cpp        |   64 +-
 llvm/lib/CodeGen/MachineOperand.cpp           |    2 +-
 llvm/lib/CodeGen/MachineSink.cpp              |   25 +-
 llvm/lib/CodeGen/MachineStableHash.cpp        |    3 +-
 llvm/lib/CodeGen/PHIElimination.cpp           |    3 -
 llvm/lib/CodeGen/PeepholeOptimizer.cpp        |   62 +-
 llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp |   10 -
 llvm/lib/CodeGen/PrologEpilogInserter.cpp     |    4 +-
 llvm/lib/CodeGen/RegAllocGreedy.cpp           |    6 +
 llvm/lib/CodeGen/ScheduleDAG.cpp              |    1 -
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp |  217 +-
 .../SelectionDAG/LegalizeFloatTypes.cpp       |    8 +-
 llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h |    2 +-
 .../SelectionDAG/LegalizeVectorOps.cpp        |    2 +
 .../SelectionDAG/LegalizeVectorTypes.cpp      |   16 +-
 .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp |   90 +-
 .../SelectionDAG/SelectionDAGBuilder.cpp      |   13 +-
 .../SelectionDAG/SelectionDAGBuilder.h        |    3 +-
 .../SelectionDAG/StatepointLowering.cpp       |   18 +-
 .../CodeGen/SelectionDAG/TargetLowering.cpp   |    2 +-
 llvm/lib/CodeGen/TargetLoweringBase.cpp       |   12 +-
 .../CodeGen/TargetLoweringObjectFileImpl.cpp  |   43 +-
 .../lib/CodeGen/TwoAddressInstructionPass.cpp |   43 +-
 llvm/lib/CodeGen/TypePromotion.cpp            |    7 +-
 llvm/lib/CodeGen/VirtRegMap.cpp               |    8 +-
 llvm/lib/DWARFLinker/DWARFLinker.cpp          |   12 +-
 .../DWARFLinkerCompileUnit.h                  |    4 +-
 .../DWARFLinkerParallel/DWARFLinkerImpl.cpp   |    6 +-
 .../lib/DWARFLinkerParallel/DWARFLinkerImpl.h |    4 +-
 llvm/lib/DWARFLinkerParallel/OutputSections.h |   16 +-
 llvm/lib/DebugInfo/BTF/BTFParser.cpp          |    5 +-
 llvm/lib/DebugInfo/CodeView/CVTypeVisitor.cpp |    2 +-
 .../CodeView/LazyRandomTypeCollection.cpp     |    4 +-
 llvm/lib/DebugInfo/CodeView/RecordName.cpp    |    2 +-
 .../CodeView/RecordSerialization.cpp          |    4 +-
 .../CodeView/SimpleTypeSerializer.cpp         |    2 +-
 .../DebugInfo/CodeView/SymbolSerializer.cpp   |    4 +-
 .../DebugInfo/CodeView/TypeIndexDiscovery.cpp |    2 +-
 llvm/lib/DebugInfo/DWARF/DWARFContext.cpp     |   16 +-
 llvm/lib/DebugInfo/DWARF/DWARFTypePrinter.cpp |    3 +
 llvm/lib/DebugInfo/DWARF/DWARFUnit.cpp        |   79 +-
 llvm/lib/DebugInfo/GSYM/DwarfTransformer.cpp  |   10 +-
 llvm/lib/DebugInfo/GSYM/GsymReader.cpp        |   19 +-
 .../DebugInfo/LogicalView/Core/LVCompare.cpp  |    9 +-
 .../DebugInfo/LogicalView/Core/LVElement.cpp  |    3 +-
 .../DebugInfo/LogicalView/Core/LVScope.cpp    |   10 +-
 .../LogicalView/Readers/LVCodeViewReader.cpp  |   19 +-
 .../LogicalView/Readers/LVCodeViewVisitor.cpp |   13 +-
 llvm/lib/DebugInfo/MSF/MSFBuilder.cpp         |    2 +-
 llvm/lib/DebugInfo/MSF/MappedBlockStream.cpp  |    1 -
 .../PDB/Native/DbiModuleDescriptor.cpp        |    1 -
 .../DebugInfo/PDB/Native/DbiStreamBuilder.cpp |    2 +-
 .../DebugInfo/PDB/Native/GSIStreamBuilder.cpp |    2 +-
 .../PDB/Native/InjectedSourceStream.cpp       |    1 -
 llvm/lib/DebugInfo/PDB/Native/InputFile.cpp   |    2 +-
 .../DebugInfo/PDB/Native/NativeSession.cpp    |    6 +-
 .../DebugInfo/PDB/Native/PublicsStream.cpp    |    1 -
 .../lib/DebugInfo/PDB/Native/SymbolStream.cpp |    1 -
 .../DebugInfo/PDB/Native/TpiStreamBuilder.cpp |    2 +-
 llvm/lib/ExecutionEngine/ExecutionEngine.cpp  |   14 +-
 llvm/lib/ExecutionEngine/JITLink/COFF.cpp     |    1 -
 .../JITLink/COFFLinkGraphBuilder.cpp          |    5 +-
 .../JITLink/COFFLinkGraphBuilder.h            |    3 +-
 .../JITLink/EHFrameSupport.cpp                |   46 +-
 .../JITLink/EHFrameSupportImpl.h              |   16 +-
 llvm/lib/ExecutionEngine/JITLink/ELF.cpp      |    1 -
 .../JITLink/ELFLinkGraphBuilder.h             |    2 +-
 .../ExecutionEngine/JITLink/ELF_aarch32.cpp   |    6 +-
 .../lib/ExecutionEngine/JITLink/ELF_ppc64.cpp |   55 +-
 .../ExecutionEngine/JITLink/ELF_x86_64.cpp    |    1 -
 .../JITLink/JITLinkMemoryManager.cpp          |    4 +-
 .../JITLink/MachOLinkGraphBuilder.cpp         |    5 +-
 .../JITLink/MachOLinkGraphBuilder.h           |    2 +-
 llvm/lib/ExecutionEngine/JITLink/aarch32.cpp  |    8 +-
 llvm/lib/ExecutionEngine/JITLink/ppc64.cpp    |    4 +
 llvm/lib/ExecutionEngine/Orc/COFFPlatform.cpp |    4 +-
 llvm/lib/ExecutionEngine/Orc/Core.cpp         |    2 +-
 .../Orc/Debugging/DebugInfoSupport.cpp        |    5 +-
 .../Orc/Debugging/DebuggerSupportPlugin.cpp   |   18 +-
 .../Orc/Debugging/PerfSupportPlugin.cpp       |    2 +-
 .../ExecutionEngine/Orc/ELFNixPlatform.cpp    |   10 +-
 .../ExecutionEngine/Orc/ExecutionUtils.cpp    |    4 +-
 .../lib/ExecutionEngine/Orc/MachOPlatform.cpp |    4 +-
 .../RuntimeDyld/RuntimeDyldChecker.cpp        |    4 +-
 .../RuntimeDyld/RuntimeDyldCheckerImpl.h      |   14 +-
 .../RuntimeDyld/RuntimeDyldELF.cpp            |    3 +-
 .../RuntimeDyld/RuntimeDyldImpl.h             |   18 +-
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp     |   40 +-
 llvm/lib/IR/AsmWriter.cpp                     |    1 +
 llvm/lib/IR/AutoUpgrade.cpp                   |   20 +-
 llvm/lib/IR/BasicBlock.cpp                    |   15 +-
 llvm/lib/IR/Instruction.cpp                   |    5 +-
 llvm/lib/IR/Instructions.cpp                  |    2 +-
 llvm/lib/IR/ReplaceConstant.cpp               |   26 +-
 llvm/lib/IR/SymbolTableListTraitsImpl.h       |   20 +-
 llvm/lib/IR/Type.cpp                          |    3 +-
 llvm/lib/IR/Value.cpp                         |    3 +-
 llvm/lib/IR/Verifier.cpp                      |   47 +-
 llvm/lib/InterfaceStub/ELFObjHandler.cpp      |    2 +-
 llvm/lib/LTO/LTO.cpp                          |    2 +-
 llvm/lib/MC/DXContainerPSVInfo.cpp            |   24 +-
 llvm/lib/MC/ELFObjectWriter.cpp               |    9 +-
 llvm/lib/MC/GOFFObjectWriter.cpp              |    3 +-
 llvm/lib/MC/MCAsmBackend.cpp                  |   10 +-
 llvm/lib/MC/MCAsmStreamer.cpp                 |   17 +-
 llvm/lib/MC/MCAssembler.cpp                   |    4 +-
 llvm/lib/MC/MCCodeView.cpp                    |    2 +-
 llvm/lib/MC/MCDXContainerWriter.cpp           |    2 +-
 llvm/lib/MC/MCDwarf.cpp                       |    5 +-
 llvm/lib/MC/MCParser/DarwinAsmParser.cpp      |   15 +-
 llvm/lib/MC/MCPseudoProbe.cpp                 |    2 +-
 llvm/lib/MC/MCStreamer.cpp                    |    2 +-
 llvm/lib/MC/MachObjectWriter.cpp              |    2 +-
 llvm/lib/MC/SPIRVObjectWriter.cpp             |    2 +-
 llvm/lib/MC/WasmObjectWriter.cpp              |    4 +-
 llvm/lib/MC/WinCOFFObjectWriter.cpp           |    2 +-
 llvm/lib/MC/XCOFFObjectWriter.cpp             |    2 +-
 llvm/lib/ObjCopy/ELF/ELFObject.cpp            |    9 +-
 llvm/lib/Object/Archive.cpp                   |    2 +-
 llvm/lib/Object/ArchiveWriter.cpp             |   10 +-
 llvm/lib/Object/COFFImportFile.cpp            |    3 +-
 llvm/lib/Object/COFFObjectFile.cpp            |    2 +-
 llvm/lib/Object/ModuleSymbolTable.cpp         |    1 -
 llvm/lib/Object/SymbolSize.cpp                |    6 +
 llvm/lib/Object/WindowsResource.cpp           |    2 +-
 llvm/lib/Object/XCOFFObjectFile.cpp           |   27 +-
 llvm/lib/ObjectYAML/COFFEmitter.cpp           |    6 +-
 .../ObjectYAML/CodeViewYAMLDebugSections.cpp  |    2 +-
 .../ObjectYAML/CodeViewYAMLTypeHashing.cpp    |    4 +-
 llvm/lib/ObjectYAML/CodeViewYAMLTypes.cpp     |    4 +-
 llvm/lib/ObjectYAML/ELFEmitter.cpp            |    2 +-
 llvm/lib/ObjectYAML/ELFYAML.cpp               |    1 +
 llvm/lib/ObjectYAML/XCOFFEmitter.cpp          |    2 +-
 llvm/lib/Option/ArgList.cpp                   |    9 +-
 llvm/lib/Passes/CMakeLists.txt                |    1 +
 llvm/lib/Passes/PassBuilder.cpp               |    1 +
 llvm/lib/Passes/PassBuilderPipelines.cpp      |    3 +
 llvm/lib/Passes/PassRegistry.def              |    3 +
 llvm/lib/Passes/StandardInstrumentations.cpp  |    5 +-
 .../ProfileData/Coverage/CoverageMapping.cpp  |    1 -
 .../Coverage/CoverageMappingReader.cpp        |   51 +-
 .../Coverage/CoverageMappingWriter.cpp        |    2 +-
 llvm/lib/ProfileData/GCOV.cpp                 |   73 +-
 llvm/lib/ProfileData/InstrProf.cpp            |   22 +-
 llvm/lib/ProfileData/InstrProfReader.cpp      |   76 +-
 llvm/lib/ProfileData/InstrProfWriter.cpp      |   16 +-
 llvm/lib/ProfileData/MemProf.cpp              |   23 +-
 llvm/lib/ProfileData/RawMemProfReader.cpp     |   18 +-
 llvm/lib/ProfileData/SampleProf.cpp           |   42 +-
 llvm/lib/ProfileData/SampleProfReader.cpp     |  100 +-
 llvm/lib/ProfileData/SampleProfWriter.cpp     |   54 +-
 llvm/lib/Remarks/Remark.cpp                   |   11 +
 llvm/lib/Remarks/YAMLRemarkParser.cpp         |    6 +-
 llvm/lib/Support/Chrono.cpp                   |   45 +-
 llvm/lib/Support/CodeGenCoverage.cpp          |    3 +-
 llvm/lib/Support/ELFAttributeParser.cpp       |    4 +-
 llvm/lib/Support/KnownBits.cpp                |   12 +
 llvm/lib/Support/RISCVISAInfo.cpp             |    6 +
 llvm/lib/Support/Regex.cpp                    |   19 +
 llvm/lib/Support/UnicodeNameToCodepoint.cpp   |    3 +-
 llvm/lib/Support/Unix/Memory.inc              |    3 +-
 llvm/lib/Support/YAMLParser.cpp               |   71 +-
 llvm/lib/Support/YAMLTraits.cpp               |    2 +
 llvm/lib/TableGen/Error.cpp                   |    7 +
 llvm/lib/TableGen/Record.cpp                  |   43 +
 llvm/lib/TableGen/TGLexer.cpp                 |   52 +-
 llvm/lib/TableGen/TGLexer.h                   |    4 +-
 llvm/lib/TableGen/TGParser.cpp                |  124 +-
 llvm/lib/TableGen/TGParser.h                  |    4 +
 llvm/lib/Target/AArch64/AArch64.td            |    3 +
 .../AArch64/AArch64ConditionalCompares.cpp    |    2 +-
 .../Target/AArch64/AArch64FrameLowering.cpp   |   92 +-
 .../Target/AArch64/AArch64ISelLowering.cpp    |   46 +-
 .../lib/Target/AArch64/AArch64InstrFormats.td |   36 +-
 llvm/lib/Target/AArch64/AArch64InstrGISel.td  |   17 +
 llvm/lib/Target/AArch64/AArch64InstrInfo.cpp  |   91 +-
 llvm/lib/Target/AArch64/AArch64InstrInfo.h    |    7 +-
 llvm/lib/Target/AArch64/AArch64InstrInfo.td   |    4 +-
 .../AArch64LowerHomogeneousPrologEpilog.cpp   |   91 +-
 .../AArch64/AArch64MachineFunctionInfo.cpp    |   22 +-
 .../AArch64/AArch64MachineFunctionInfo.h      |    2 +
 .../lib/Target/AArch64/AArch64PointerAuth.cpp |  182 +-
 llvm/lib/Target/AArch64/AArch64PointerAuth.h  |  116 +
 .../lib/Target/AArch64/AArch64SMEInstrInfo.td |  262 +-
 .../lib/Target/AArch64/AArch64SVEInstrInfo.td |   18 +-
 llvm/lib/Target/AArch64/AArch64SchedA57.td    |  164 +-
 .../Target/AArch64/AArch64SchedNeoverseN1.td  |   82 +-
 .../Target/AArch64/AArch64SchedNeoverseN2.td  |  102 +-
 .../Target/AArch64/AArch64SchedNeoverseV1.td  |   64 +-
 llvm/lib/Target/AArch64/AArch64SchedTSV110.td |   74 +-
 .../Target/AArch64/AArch64StackTagging.cpp    |    1 -
 .../AArch64/AArch64StackTaggingPreRA.cpp      |    1 -
 llvm/lib/Target/AArch64/AArch64Subtarget.cpp  |   32 +
 llvm/lib/Target/AArch64/AArch64Subtarget.h    |   27 +
 .../Target/AArch64/AArch64SystemOperands.td   |   10 +
 .../Target/AArch64/AArch64TargetMachine.cpp   |    2 +-
 .../AArch64/AArch64TargetTransformInfo.cpp    |   41 +-
 .../AArch64/AsmParser/AArch64AsmParser.cpp    |    7 +-
 .../GISel/AArch64InstructionSelector.cpp      |   20 +-
 .../AArch64/GISel/AArch64LegalizerInfo.cpp    |   82 +-
 .../AArch64/GISel/AArch64LegalizerInfo.h      |    1 -
 .../MCTargetDesc/AArch64AsmBackend.cpp        |    5 +-
 .../MCTargetDesc/AArch64InstPrinter.cpp       |    3 +-
 .../AArch64/MCTargetDesc/AArch64InstPrinter.h |    1 +
 .../MCTargetDesc/AArch64MCCodeEmitter.cpp     |    2 +-
 llvm/lib/Target/AArch64/SMEInstrFormats.td    |  333 +-
 llvm/lib/Target/AArch64/SVEInstrFormats.td    |   31 +-
 llvm/lib/Target/AMDGPU/AMDGPU.td              |    9 +
 llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp     |   16 +-
 llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp |   30 +
 .../AMDGPU/AMDGPUImageIntrinsicOptimizer.cpp  |   37 +-
 llvm/lib/Target/AMDGPU/AMDGPUInstructions.td  |   11 +-
 .../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp |   33 +-
 .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp |    8 +
 .../AMDGPU/AMDGPUTargetTransformInfo.cpp      |    9 +
 .../AMDGPU/AsmParser/AMDGPUAsmParser.cpp      |   93 +-
 llvm/lib/Target/AMDGPU/BUFInstructions.td     |   21 +-
 llvm/lib/Target/AMDGPU/CMakeLists.txt         |    1 +
 .../Disassembler/AMDGPUDisassembler.cpp       |   88 +-
 .../AMDGPU/Disassembler/AMDGPUDisassembler.h  |    9 +-
 llvm/lib/Target/AMDGPU/FLATInstructions.td    |   18 +-
 llvm/lib/Target/AMDGPU/GCNCreateVOPD.cpp      |    1 -
 llvm/lib/Target/AMDGPU/GCNSubtarget.h         |    4 +
 .../AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp  |    2 +-
 .../AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp |   15 +-
 .../AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h   |    2 +-
 .../MCTargetDesc/AMDGPUMCCodeEmitter.cpp      |    5 +-
 .../MCTargetDesc/AMDGPUTargetStreamer.cpp     |    8 +-
 .../AMDGPU/MCTargetDesc/R600MCCodeEmitter.cpp |    4 +-
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp     |   10 +-
 llvm/lib/Target/AMDGPU/SIInstrInfo.cpp        |   86 +-
 llvm/lib/Target/AMDGPU/SIInstrInfo.td         |   75 +-
 llvm/lib/Target/AMDGPU/SIInstructions.td      |    5 +-
 llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp     |   12 +-
 llvm/lib/Target/AMDGPU/SIRegisterInfo.td      |    4 +-
 .../Target/AMDGPU/SIShrinkInstructions.cpp    |    3 +-
 llvm/lib/Target/AMDGPU/SOPInstructions.td     |    3 +-
 .../Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp    |   11 +-
 llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h |    3 +
 .../Target/AMDGPU/Utils/AMDGPUPALMetadata.cpp |    2 +-
 llvm/lib/Target/AMDGPU/VOPCInstructions.td    |    8 +
 llvm/lib/Target/ARM/ARMISelLowering.cpp       |   31 +-
 llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp |    4 +-
 .../lib/Target/ARM/AsmParser/ARMAsmParser.cpp |    2 +
 .../ARM/Disassembler/ARMDisassembler.cpp      |    6 +-
 .../Target/ARM/MCTargetDesc/ARMAsmBackend.cpp |   46 +-
 .../Target/ARM/MCTargetDesc/ARMAsmBackend.h   |    2 +-
 .../ARM/MCTargetDesc/ARMAsmBackendDarwin.h    |    3 +-
 .../ARM/MCTargetDesc/ARMAsmBackendELF.h       |    2 +-
 .../ARM/MCTargetDesc/ARMAsmBackendWinCOFF.h   |    2 +-
 .../ARM/MCTargetDesc/ARMMCCodeEmitter.cpp     |    3 +-
 .../Target/AVR/MCTargetDesc/AVRAsmBackend.h   |    2 +-
 .../AVR/MCTargetDesc/AVRMCCodeEmitter.cpp     |    2 +-
 .../Target/BPF/MCTargetDesc/BPFAsmBackend.cpp |    8 +-
 .../BPF/MCTargetDesc/BPFMCCodeEmitter.cpp     |    4 +-
 .../CSKY/MCTargetDesc/CSKYAsmBackend.cpp      |    2 +-
 .../Target/CSKY/MCTargetDesc/CSKYAsmBackend.h |    2 +-
 .../CSKY/MCTargetDesc/CSKYMCCodeEmitter.cpp   |    5 +-
 .../MCTargetDesc/DirectXMCTargetDesc.cpp      |    3 +-
 .../MCTargetDesc/HexagonAsmBackend.cpp        |    7 +-
 .../MCTargetDesc/HexagonMCCodeEmitter.cpp     |    2 +-
 .../Lanai/MCTargetDesc/LanaiAsmBackend.cpp    |    2 +-
 .../Lanai/MCTargetDesc/LanaiMCCodeEmitter.cpp |    2 +-
 llvm/lib/Target/LoongArch/LoongArch.h         |    2 +
 .../LoongArchExpandAtomicPseudoInsts.cpp      |   50 +-
 .../LoongArch/LoongArchExpandPseudoInsts.cpp  |  121 +
 .../LoongArch/LoongArchFloat32InstrInfo.td    |   17 +
 .../LoongArch/LoongArchFloatInstrFormats.td   |   12 +
 .../LoongArch/LoongArchISelLowering.cpp       |   27 +-
 .../Target/LoongArch/LoongArchISelLowering.h  |    1 +
 .../Target/LoongArch/LoongArchInstrInfo.cpp   |    6 +
 .../Target/LoongArch/LoongArchInstrInfo.td    |   99 +-
 .../LoongArch/LoongArchRegisterInfo.cpp       |    7 -
 .../LoongArch/LoongArchTargetMachine.cpp      |    1 +
 .../MCTargetDesc/LoongArchAsmBackend.h        |    4 +-
 .../MCTargetDesc/LoongArchMCCodeEmitter.cpp   |    4 +-
 llvm/lib/Target/M68k/M68kExpandPseudo.cpp     |   36 +-
 llvm/lib/Target/M68k/M68kISelLowering.cpp     |    5 +-
 llvm/lib/Target/M68k/M68kInstrInfo.cpp        |    4 +
 .../M68k/MCTargetDesc/M68kAsmBackend.cpp      |    2 +-
 .../Target/M68k/MCTargetDesc/M68kBaseInfo.h   |    4 +-
 .../M68k/MCTargetDesc/M68kMCCodeEmitter.cpp   |    9 +-
 .../MSP430/MCTargetDesc/MSP430AsmBackend.cpp  |    2 +-
 .../MCTargetDesc/MSP430MCCodeEmitter.cpp      |    3 +-
 .../Mips/MCTargetDesc/MipsAsmBackend.cpp      |    6 +-
 .../Target/Mips/MCTargetDesc/MipsAsmBackend.h |    3 +-
 .../Mips/MCTargetDesc/MipsMCCodeEmitter.cpp   |    3 +-
 llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp   |   67 +-
 llvm/lib/Target/NVPTX/NVPTXISelLowering.h     |    6 +
 llvm/lib/Target/NVPTX/NVPTXInstrInfo.td       |    3 +
 llvm/lib/Target/NVPTX/NVVMReflect.cpp         |    1 -
 .../PowerPC/MCTargetDesc/PPCAsmBackend.cpp    |    8 +-
 .../PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp |    3 +-
 llvm/lib/Target/PowerPC/P10InstrResources.td  |   19 +-
 .../PowerPC/PPCExpandAtomicPseudoInsts.cpp    |   16 +-
 llvm/lib/Target/PowerPC/PPCISelLowering.cpp   |    2 +-
 llvm/lib/Target/PowerPC/PPCInstr64Bit.td      |   59 +-
 llvm/lib/Target/PowerPC/PPCScheduleP10.td     |    4 +-
 llvm/lib/Target/PowerPC/PPCTargetMachine.cpp  |    5 +-
 .../RISCV/Disassembler/RISCVDisassembler.cpp  |    6 +
 .../Target/RISCV/GISel/RISCVCallLowering.cpp  |  151 +-
 .../RISCV/GISel/RISCVInstructionSelector.cpp  |  206 +-
 .../Target/RISCV/GISel/RISCVLegalizerInfo.cpp |   27 +
 .../RISCV/GISel/RISCVRegisterBankInfo.cpp     |    3 +
 .../Target/RISCV/MCA/RISCVCustomBehaviour.cpp |   57 +-
 .../RISCV/MCTargetDesc/RISCVAsmBackend.cpp    |   10 +-
 .../RISCV/MCTargetDesc/RISCVAsmBackend.h      |    4 +-
 .../RISCV/MCTargetDesc/RISCVBaseInfo.cpp      |    9 +
 .../Target/RISCV/MCTargetDesc/RISCVBaseInfo.h |    2 +
 .../RISCV/MCTargetDesc/RISCVFixupKinds.h      |    2 +-
 .../RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp |   16 +-
 .../RISCV/MCTargetDesc/RISCVMCTargetDesc.cpp  |   79 +
 llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp     |   87 +
 llvm/lib/Target/RISCV/RISCVFeatures.td        |   19 +
 llvm/lib/Target/RISCV/RISCVGISel.td           |   37 +-
 llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp   |   89 +-
 llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h     |    1 +
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   |  306 +-
 llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp  |   69 +-
 llvm/lib/Target/RISCV/RISCVInstrInfo.cpp      |   16 +-
 llvm/lib/Target/RISCV/RISCVInstrInfo.td       |    2 +-
 llvm/lib/Target/RISCV/RISCVInstrInfoC.td      |    6 +
 llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td    |   86 +
 llvm/lib/Target/RISCV/RISCVInstrInfoZb.td     |   85 +-
 llvm/lib/Target/RISCV/RISCVInstrInfoZfa.td    |    1 +
 llvm/lib/Target/RISCV/RISCVInstrInfoZicbo.td  |   18 +-
 llvm/lib/Target/RISCV/RISCVMacroFusion.cpp    |    4 -
 llvm/lib/Target/RISCV/RISCVMoveMerger.cpp     |    2 +-
 llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp     |   18 +-
 .../Target/RISCV/RISCVPushPopOptimizer.cpp    |    2 +-
 llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp   |   13 +-
 llvm/lib/Target/RISCV/RISCVRegisterInfo.td    |   23 +-
 llvm/lib/Target/RISCV/RISCVSubtarget.h        |    4 +
 llvm/lib/Target/RISCV/RISCVTargetMachine.cpp  |    2 +
 .../Target/RISCV/RISCVTargetTransformInfo.cpp |   27 +
 .../SPIRV/MCTargetDesc/SPIRVAsmBackend.cpp    |    4 +-
 .../SPIRV/MCTargetDesc/SPIRVMCCodeEmitter.cpp |    6 +-
 llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp       |  107 +-
 llvm/lib/Target/SPIRV/SPIRVBuiltins.h         |   12 +
 llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp   |   43 +-
 llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp |   79 +-
 llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h   |    7 +-
 llvm/lib/Target/SPIRV/SPIRVUtils.cpp          |    2 +-
 llvm/lib/Target/SPIRV/SPIRVUtils.h            |    3 +
 .../Sparc/MCTargetDesc/SparcAsmBackend.cpp    |   10 +-
 .../Sparc/MCTargetDesc/SparcMCCodeEmitter.cpp |    5 +-
 llvm/lib/Target/Sparc/SparcISelLowering.cpp   |    5 +-
 .../MCTargetDesc/SystemZMCAsmBackend.cpp      |    3 +-
 .../Target/SystemZ/SystemZFrameLowering.cpp   |   24 +
 .../Target/SystemZ/SystemZISelLowering.cpp    |   44 +-
 .../Target/VE/MCTargetDesc/VEAsmBackend.cpp   |    8 +-
 .../VE/MCTargetDesc/VEMCCodeEmitter.cpp       |    2 +-
 .../AsmParser/WebAssemblyAsmTypeCheck.cpp     |    1 -
 .../Disassembler/WebAssemblyDisassembler.cpp  |    4 +-
 .../MCTargetDesc/WebAssemblyAsmBackend.cpp    |    2 +-
 .../MCTargetDesc/WebAssemblyMCCodeEmitter.cpp |   16 +-
 .../WebAssembly/WebAssemblyISelLowering.cpp   |    2 +
 .../WebAssemblyRuntimeLibcallSignatures.cpp   |    3 +
 .../WebAssembly/WebAssemblyTargetMachine.cpp  |    9 +-
 llvm/lib/Target/X86/CMakeLists.txt            |    1 -
 .../X86/Disassembler/X86Disassembler.cpp      |   11 +-
 .../X86/Disassembler/X86DisassemblerDecoder.h |    3 +-
 .../Target/X86/MCTargetDesc/X86AsmBackend.cpp |    2 +-
 .../lib/Target/X86/MCTargetDesc/X86BaseInfo.h |    3 +-
 .../X86/MCTargetDesc/X86MCCodeEmitter.cpp     |    4 +
 llvm/lib/Target/X86/X86.h                     |    1 -
 llvm/lib/Target/X86/X86.td                    |   26 +
 llvm/lib/Target/X86/X86FastPreTileConfig.cpp  |    1 -
 llvm/lib/Target/X86/X86ISelDAGToDAG.cpp       |   20 +-
 llvm/lib/Target/X86/X86ISelLowering.cpp       |  129 +-
 llvm/lib/Target/X86/X86ISelLowering.h         |    5 +
 llvm/lib/Target/X86/X86InsertPrefetch.cpp     |    7 +-
 llvm/lib/Target/X86/X86InstrFormats.td        |    4 +
 llvm/lib/Target/X86/X86InstrInfo.cpp          |  373 +-
 llvm/lib/Target/X86/X86InstrInfo.h            |   13 +-
 llvm/lib/Target/X86/X86InstrInfo.td           |    3 +
 llvm/lib/Target/X86/X86InstrSystem.td         |   16 +
 llvm/lib/Target/X86/X86MCInstLower.cpp        |    1 -
 llvm/lib/Target/X86/X86PreAMXConfig.cpp       |  415 -
 llvm/lib/Target/X86/X86TargetMachine.cpp      |    7 +-
 .../Xtensa/MCTargetDesc/XtensaAsmBackend.cpp  |    3 +-
 llvm/lib/TargetParser/ARMTargetParser.cpp     |   71 +-
 llvm/lib/TargetParser/Host.cpp                |   20 +
 llvm/lib/TargetParser/TargetParser.cpp        |    2 +-
 llvm/lib/TargetParser/X86TargetParser.cpp     |   18 +
 llvm/lib/TextAPI/Platform.cpp                 |   41 +-
 llvm/lib/TextAPI/Target.cpp                   |   14 +-
 llvm/lib/TextAPI/TextStub.cpp                 |   39 +-
 llvm/lib/Transforms/CMakeLists.txt            |    1 +
 llvm/lib/Transforms/HipStdPar/CMakeLists.txt  |   18 +
 llvm/lib/Transforms/HipStdPar/HipStdPar.cpp   |  312 +
 llvm/lib/Transforms/IPO/Attributor.cpp        |    3 +-
 .../Transforms/IPO/AttributorAttributes.cpp   |   21 +-
 llvm/lib/Transforms/IPO/FunctionAttrs.cpp     |    7 +-
 llvm/lib/Transforms/IPO/FunctionImport.cpp    |    1 -
 .../IPO/MemProfContextDisambiguation.cpp      |   23 +-
 llvm/lib/Transforms/IPO/MergeFunctions.cpp    |   15 +-
 llvm/lib/Transforms/IPO/OpenMPOpt.cpp         |    6 +-
 .../Transforms/IPO/SampleContextTracker.cpp   |   68 +-
 llvm/lib/Transforms/IPO/SampleProfile.cpp     |  149 +-
 .../lib/Transforms/IPO/WholeProgramDevirt.cpp |    1 -
 .../InstCombine/InstCombineAddSub.cpp         |    8 +-
 .../InstCombine/InstCombineAndOrXor.cpp       |  180 +-
 .../InstCombine/InstCombineCompares.cpp       |   68 +-
 .../InstCombine/InstCombineInternal.h         |   15 +-
 .../InstCombineLoadStoreAlloca.cpp            |    2 +
 .../InstCombine/InstCombineMulDivRem.cpp      |   57 +-
 .../InstCombine/InstCombineNegator.cpp        |    1 -
 .../InstCombine/InstCombineShifts.cpp         |   93 +-
 .../InstCombineSimplifyDemanded.cpp           |  140 +-
 .../InstCombine/InstructionCombining.cpp      |   38 +-
 .../Instrumentation/AddressSanitizer.cpp      |   61 +-
 .../Instrumentation/GCOVProfiling.cpp         |    8 +-
 .../Instrumentation/HWAddressSanitizer.cpp    |   12 +-
 .../Instrumentation/MemorySanitizer.cpp       |    1 -
 .../Scalar/ConstraintElimination.cpp          |   16 +-
 .../Scalar/CorrelatedValuePropagation.cpp     |   68 +-
 .../Scalar/DeadStoreElimination.cpp           |    2 +-
 llvm/lib/Transforms/Scalar/GVN.cpp            |   34 +-
 llvm/lib/Transforms/Scalar/JumpThreading.cpp  |    8 +-
 llvm/lib/Transforms/Scalar/LICM.cpp           |    2 +-
 .../Scalar/LowerExpectIntrinsic.cpp           |   27 -
 .../lib/Transforms/Scalar/MemCpyOptimizer.cpp |   26 +-
 .../Scalar/RewriteStatepointsForGC.cpp        |   47 +-
 llvm/lib/Transforms/Scalar/Scalar.cpp         |    1 -
 llvm/lib/Transforms/Scalar/Scalarizer.cpp     |   24 +-
 llvm/lib/Transforms/Utils/CodeExtractor.cpp   |   18 +-
 llvm/lib/Transforms/Utils/CodeLayout.cpp      |  186 +-
 .../Transforms/Utils/FunctionComparator.cpp   |   30 +-
 llvm/lib/Transforms/Utils/Local.cpp           |   19 +-
 llvm/lib/Transforms/Utils/LoopUnroll.cpp      |    1 -
 .../lib/Transforms/Utils/LoopUnrollAndJam.cpp |    1 -
 llvm/lib/Transforms/Utils/SimplifyIndVar.cpp  |    8 +-
 .../Vectorize/LoopVectorizationPlanner.h      |   24 +-
 .../Transforms/Vectorize/LoopVectorize.cpp    |  198 +-
 .../Transforms/Vectorize/SLPVectorizer.cpp    |  231 +-
 llvm/lib/Transforms/Vectorize/VPlan.cpp       |    1 -
 .../lib/Transforms/Vectorize/VPlanRecipes.cpp |    3 +-
 .../Transforms/Vectorize/VPlanTransforms.cpp  |   29 +-
 .../Transforms/Vectorize/VectorCombine.cpp    |   43 +-
 llvm/lib/XRay/FDRTraceWriter.cpp              |    2 +-
 llvm/test/Analysis/BasicAA/pr69096.ll         |   31 +
 llvm/test/Analysis/CostModel/AMDGPU/fdiv.ll   |   62 +
 llvm/test/Analysis/CostModel/RISCV/fround.ll  |  130 +
 llvm/test/Analysis/CostModel/RISCV/gep.ll     |    8 +-
 .../CostModel/RISCV/rvv-intrinsics.ll         |  580 +-
 .../Analysis/ValueTracking/constant-ranges.ll |  132 +
 .../Analysis/ValueTracking/known-non-zero.ll  |   58 +
 .../ValueTracking/known-power-of-two.ll       |   36 +-
 .../ValueTracking/select-known-non-zero.ll    |   54 +
 llvm/test/Bindings/OCaml/core.ml              |    4 +-
 llvm/test/Bitcode/attributes.ll               |    7 +
 llvm/test/Bitcode/upgrade-datalayout.ll       |    2 +-
 llvm/test/Bitcode/upgrade-datalayout2.ll      |    6 +
 llvm/test/Bitcode/upgrade-datalayout3.ll      |    2 +-
 llvm/test/Bitcode/upgrade-datalayout4.ll      |    2 +-
 llvm/test/Bitcode/upgrade-datalayout5.ll      |    8 +
 .../AArch64/GlobalISel/legalize-fma.mir       |  218 +-
 .../AArch64/GlobalISel/legalize-xtn.mir       |  531 ++
 .../GlobalISel/legalizer-info-validation.mir  |    6 +-
 .../AArch64/GlobalISel/select-sadde.mir       |   31 +
 .../AArch64/GlobalISel/select-ssube.mir       |   31 +
 .../AArch64/GlobalISel/select-uadde.mir       |   31 +
 .../AArch64/GlobalISel/select-usube.mir       |   31 +
 .../GlobalISel/sink-and-fold-illegal-shift.ll |   17 +
 llvm/test/CodeGen/AArch64/aarch64-mulv.ll     |  596 ++
 ...rm64-homogeneous-prolog-epilog-odd-csrs.ll |   31 +
 .../CodeGen/AArch64/arm64-indexed-memory.ll   |  966 +-
 .../AArch64/arm64-indexed-vector-ldst.ll      | 8214 ++++++++---------
 llvm/test/CodeGen/AArch64/arm64-ld1.ll        |   29 +-
 .../CodeGen/AArch64/arm64-neon-mul-div.ll     |    7 +-
 llvm/test/CodeGen/AArch64/arm64-st1.ll        |    2 +-
 .../CodeGen/AArch64/arm64-vfloatintrinsics.ll |  514 --
 llvm/test/CodeGen/AArch64/arm64-vshift.ll     |   13 +-
 llvm/test/CodeGen/AArch64/fexplog.ll          |  305 +-
 llvm/test/CodeGen/AArch64/fmla.ll             | 2274 +++++
 .../fold-int-pow2-with-fmul-or-fdiv.ll        |    3 +-
 llvm/test/CodeGen/AArch64/fpow.ll             |   89 +-
 llvm/test/CodeGen/AArch64/fptoi.ll            | 5518 +++++++++++
 llvm/test/CodeGen/AArch64/frem.ll             |   89 +-
 llvm/test/CodeGen/AArch64/fsincos.ll          |  122 +-
 ...implicit-def-with-impdef-greedy-assert.mir |  181 +
 llvm/test/CodeGen/AArch64/llvm.exp10.ll       |   34 +-
 .../machine-sink-cache-invalidation.ll        |    3 +-
 llvm/test/CodeGen/AArch64/pow.ll              |    4 +-
 llvm/test/CodeGen/AArch64/pr66603.ll          |   15 +
 .../AArch64/prologue-epilogue-remarks.mir     |   12 +-
 .../AArch64/sign-return-address-tailcall.ll   |  121 +
 .../AArch64/sink-and-fold-illegal-shift.mir   |   95 +
 llvm/test/CodeGen/AArch64/sink-and-fold.ll    |    2 +-
 llvm/test/CodeGen/AArch64/spill-fold.mir      |   37 +
 .../AArch64/statepoint-call-lowering.ll       |    3 -
 .../CodeGen/AArch64/sve-fixed-length-fp128.ll |    4 +
 llvm/test/CodeGen/AArch64/sve-zeroinit.ll     |    7 +
 .../AArch64/sve2p1-intrinsics-bfmls.ll        |   43 +
 llvm/test/CodeGen/AArch64/vec-libcalls.ll     |   27 +-
 llvm/test/CodeGen/AArch64/vec_uaddo.ll        |   20 +-
 llvm/test/CodeGen/AArch64/vector-llrint.ll    |  621 ++
 llvm/test/CodeGen/AArch64/vector-lrint.ll     |  622 ++
 llvm/test/CodeGen/AArch64/xtn.ll              |  473 +
 llvm/test/CodeGen/AArch64/zext.ll             |   48 +-
 .../AMDGPU/amdgpu-codegenprepare-idiv.ll      | 1151 +--
 .../CodeGen/AMDGPU/fix-sgpr-copies-f16.mir    |   58 +
 llvm/test/CodeGen/AMDGPU/fix-sgpr-copies.mir  |   58 -
 .../AMDGPU/fold-short-64-bit-literals.mir     |  134 +
 .../ipra-return-address-save-restore.ll       |    8 +-
 .../CodeGen/AMDGPU/llvm.amdgcn.atomic.csub.ll |   51 +-
 .../CodeGen/AMDGPU/llvm.amdgcn.fcmp.w32.ll    |  346 +
 .../CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll    |  402 +
 .../AMDGPU/llvm.amdgcn.image.load.2dmsaa.ll   |   41 +
 llvm/test/CodeGen/AMDGPU/mul.ll               |  403 +-
 .../test/CodeGen/AMDGPU/peephole-fold-imm.mir |  228 +-
 .../AMDGPU/promote-constOffset-to-imm.ll      |    6 +-
 llvm/test/CodeGen/AMDGPU/remat-smrd.mir       |  192 +-
 llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll     |  107 +
 llvm/test/CodeGen/AMDGPU/udiv.ll              |  151 +-
 llvm/test/CodeGen/AMDGPU/udiv64.ll            |  160 +-
 llvm/test/CodeGen/AMDGPU/urem64.ll            |   85 +-
 .../CodeGen/AMDGPU/waterfall_kills_scc.ll     |   83 +
 .../CodeGen/ARM/fadd-select-fneg-combine.ll   |   37 +-
 .../CodeGen/ARM/stack-frame-layout-remarks.ll |   20 +-
 llvm/test/CodeGen/ARM/thumb2-size-opt.ll      |    4 +-
 llvm/test/CodeGen/BPF/warn-stack.ll           |    6 +-
 llvm/test/CodeGen/LoongArch/O0-pipeline.ll    |    1 +
 .../LoongArch/atomicrmw-uinc-udec-wrap.ll     |   24 +-
 llvm/test/CodeGen/LoongArch/cfr-copy.mir      |   34 +
 .../CodeGen/LoongArch/cfr-pseudo-copy.mir     |   26 +
 .../LoongArch/inline-asm-clobbers-fcc.mir     |   13 +-
 .../ir-instruction/atomic-cmpxchg.ll          |  585 +-
 .../LoongArch/ir-instruction/atomicrmw-fp.ll  | 2738 +++++-
 .../ir-instruction/atomicrmw-minmax.ll        | 1392 ++-
 .../LoongArch/ir-instruction/atomicrmw.ll     | 3157 ++++++-
 .../ir-instruction/fence-singlethread.ll      |   17 +
 .../CodeGen/LoongArch/ir-instruction/fence.ll |   16 +-
 .../ir-instruction/load-store-atomic.ll       |  240 +-
 llvm/test/CodeGen/LoongArch/opt-pipeline.ll   |    1 +
 llvm/test/CodeGen/M68k/CConv/rtd-call.ll      |   48 +
 llvm/test/CodeGen/M68k/CConv/rtd-ret.ll       |   31 +
 .../Generic/first-character-parse-error.mir   |    4 +-
 .../MLRegAlloc/Inputs/reference-log-noml.txt  |   72 +-
 .../Inputs/reference-prio-log-noml.txt        |   12 +-
 .../MLRegAlloc/dev-mode-prio-logging.ll       |    2 +-
 .../test/CodeGen/NVPTX/LoadStoreVectorizer.ll |  123 +
 llvm/test/CodeGen/NVPTX/f16x2-instructions.ll |    6 +-
 llvm/test/CodeGen/NVPTX/i8x4-instructions.ll  |  154 +
 llvm/test/CodeGen/NVPTX/param-load-store.ll   |    5 -
 .../CodeGen/PowerPC/and-extend-combine.ll     |   16 +
 llvm/test/CodeGen/PowerPC/atomics-i128.ll     |  174 +-
 llvm/test/CodeGen/PowerPC/crbit-asm.ll        |  117 +-
 llvm/test/CodeGen/PowerPC/vector-llrint.ll    | 4848 ++++++++++
 llvm/test/CodeGen/PowerPC/vector-lrint.ll     | 4859 ++++++++++
 .../instruction-select/frame-index-rv32.mir   |   32 +
 .../instruction-select/frame-index-rv64.mir   |   32 +
 .../instruction-select/icmp-rv32.mir          |    2 +-
 .../instruction-select/icmp-rv64.mir          |    2 +-
 .../instruction-select/int-ptr-cast-rv32.mir  |   46 +
 .../instruction-select/int-ptr-cast-rv64.mir  |   46 +
 .../instruction-select/ptradd-rv32.mir        |    2 +-
 .../instruction-select/ptradd-rv64.mir        |    2 +-
 .../instruction-select/zba-rv32.mir           |  152 +
 .../instruction-select/zba-rv64.mir           |  152 +
 ...calling-conv-ilp32-ilp32f-ilp32d-common.ll |  259 +
 .../calling-conv-lp64-lp64f-lp64d-common.ll   |  270 +
 .../variadic-call.ll                          |    0
 .../legalizer/rv32/legalize-inttoptr.mir      |  131 +
 .../legalizer/rv32/legalize-mulo.mir          |  348 +
 .../legalizer/rv32/legalize-ptrtoint.mir      |  125 +
 .../legalizer/rv32/legalize-smax.mir          |  114 +
 .../legalizer/rv32/legalize-smin.mir          |  114 +
 .../legalizer/rv32/legalize-umax.mir          |  112 +
 .../legalizer/rv32/legalize-umin.mir          |  112 +
 .../legalizer/rv64/legalize-inttoptr.mir      |  153 +
 .../legalizer/rv64/legalize-mulo.mir          |  450 +
 .../legalizer/rv64/legalize-ptrtoint.mir      |  145 +
 .../legalizer/rv64/legalize-smax.mir          |  110 +
 .../legalizer/rv64/legalize-smin.mir          |  110 +
 .../legalizer/rv64/legalize-umax.mir          |  109 +
 .../legalizer/rv64/legalize-umin.mir          |  109 +
 .../GlobalISel/regbankselect/alu-rv32.mir     |   48 +-
 .../GlobalISel/regbankselect/alu-rv64.mir     |   48 +-
 .../GlobalISel/regbankselect/frame-index.mir  |   33 +
 llvm/test/CodeGen/RISCV/O3-pipeline.ll        |    2 +-
 llvm/test/CodeGen/RISCV/aext.ll               |   21 +
 .../CodeGen/RISCV/macro-fusion-lui-addi.ll    |   15 +
 llvm/test/CodeGen/RISCV/opt-w-instrs.mir      |   30 +
 llvm/test/CodeGen/RISCV/pr66603.ll            |   24 +
 llvm/test/CodeGen/RISCV/pr68855.ll            |   28 +
 llvm/test/CodeGen/RISCV/prefetch.ll           | 1214 ++-
 .../CodeGen/RISCV/riscv-shifted-extend.ll     |  124 +
 llvm/test/CodeGen/RISCV/rv64-patchpoint.ll    |   24 +
 .../RISCV/rv64-stackmap-frame-setup.ll        |   22 +
 llvm/test/CodeGen/RISCV/rv64-stackmap-nops.ll |   18 +
 llvm/test/CodeGen/RISCV/rv64-stackmap.ll      |  384 +
 llvm/test/CodeGen/RISCV/rv64zbb.ll            |    2 +-
 .../rvv/concat-vectors-constant-stride.ll     |  167 +
 .../rvv/fixed-vectors-buildvec-of-binop.ll    |   12 +
 ...xed-vectors-vfptrunc-constrained-sdnode.ll |    4 +
 .../test/CodeGen/RISCV/rvv/fmaximum-sdnode.ll |    8 +
 .../test/CodeGen/RISCV/rvv/fminimum-sdnode.ll |    8 +
 llvm/test/CodeGen/RISCV/rvv/llrint-sdnode.ll  |  108 +
 llvm/test/CodeGen/RISCV/rvv/lrint-sdnode.ll   |  155 +
 .../test/CodeGen/RISCV/rvv/sf_vqmacc_2x8x2.ll |  159 +
 .../test/CodeGen/RISCV/rvv/sf_vqmacc_4x8x4.ll |  159 +
 .../CodeGen/RISCV/rvv/sf_vqmaccsu_2x8x2.ll    |  159 +
 .../CodeGen/RISCV/rvv/sf_vqmaccsu_4x8x4.ll    |  159 +
 .../CodeGen/RISCV/rvv/sf_vqmaccu_2x8x2.ll     |  159 +
 .../CodeGen/RISCV/rvv/sf_vqmaccu_4x8x4.ll     |  159 +
 .../CodeGen/RISCV/rvv/sf_vqmaccus_2x8x2.ll    |  159 +
 .../CodeGen/RISCV/rvv/sf_vqmaccus_4x8x4.ll    |  159 +
 llvm/test/CodeGen/RISCV/rvv/vfmsub-sdnode.ll  |    4 +
 .../RISCV/rvv/vfptrunc-constrained-sdnode.ll  |    4 +
 llvm/test/CodeGen/RISCV/rvv/vmv.s.x-rv32.ll   |   18 +-
 llvm/test/CodeGen/RISCV/rvv/vmv.s.x-rv64.ll   |   24 +-
 .../CodeGen/RISCV/select-binop-identity.ll    |  327 +-
 llvm/test/CodeGen/RISCV/spir-kernel-cc.ll     |   16 +
 .../CodeGen/RISCV/srem-seteq-illegal-types.ll |   81 +-
 .../SPIRV/pointers/getelementptr-base-type.ll |   18 +
 .../kernel-argument-pointer-addressspace.ll   |   63 +
 .../pointers/kernel-argument-pointer-type.ll  |   78 +
 llvm/test/CodeGen/SystemZ/call-zos-vararg.ll  |  267 +-
 llvm/test/CodeGen/SystemZ/frameaddr-01.ll     |   21 +
 llvm/test/CodeGen/SystemZ/frameaddr-02.ll     |   23 +
 llvm/test/CodeGen/SystemZ/ret-addr-01.ll      |   23 +
 llvm/test/CodeGen/SystemZ/ret-addr-02.ll      |   39 +
 llvm/test/CodeGen/Thumb2/active_lane_mask.ll  |  202 +-
 llvm/test/CodeGen/Thumb2/mve-concat.ll        |  148 +
 .../Thumb2/mve-laneinterleaving-reduct.ll     |   26 +-
 llvm/test/CodeGen/Thumb2/mve-phireg.ll        |   54 +-
 llvm/test/CodeGen/Thumb2/mve-pred-shuffle.ll  |  118 +-
 llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll  |  559 +-
 llvm/test/CodeGen/WebAssembly/libcalls.ll     |   67 +-
 llvm/test/CodeGen/X86/AMX/amx-config.ll       |    8 +-
 .../X86/AMX/amx-configO2toO0-precfg.ll        |  178 -
 llvm/test/CodeGen/X86/GlobalISel/phi.ll       |   18 +-
 llvm/test/CodeGen/X86/arg-copy-elide.ll       |    4 +-
 llvm/test/CodeGen/X86/atomic-idempotent.ll    |   35 +-
 llvm/test/CodeGen/X86/atomic-non-integer.ll   |  166 +-
 llvm/test/CodeGen/X86/atomic-unordered.ll     |   12 +-
 llvm/test/CodeGen/X86/atomic-xor.ll           |    2 +-
 llvm/test/CodeGen/X86/atomic128.ll            |   16 +-
 .../X86/avx512fp16-combine-shuffle-fma.ll     |   58 +
 llvm/test/CodeGen/X86/avx512fp16-cvt.ll       |   22 +-
 .../basic-block-sections-clusters-error.ll    |   35 +-
 llvm/test/CodeGen/X86/bitcast-i256.ll         |    2 +-
 .../CodeGen/X86/catchpad-dynamic-alloca.ll    |    2 +-
 llvm/test/CodeGen/X86/cmp-shiftX-maskX.ll     |  579 ++
 .../CodeGen/X86/code-model-elf-sections.ll    |    5 +
 llvm/test/CodeGen/X86/combine-or.ll           |    5 +-
 llvm/test/CodeGen/X86/cpus-intel.ll           |    4 +
 .../X86/div-rem-pair-recomposition-signed.ll  |  357 +-
 .../div-rem-pair-recomposition-unsigned.ll    |   99 +-
 llvm/test/CodeGen/X86/fast-isel-freeze.ll     |    4 +-
 llvm/test/CodeGen/X86/foldimmediate-size.ll   |   57 +
 llvm/test/CodeGen/X86/foldimmediate.mir       |  143 +
 llvm/test/CodeGen/X86/fp-intrinsics.ll        |   32 +-
 llvm/test/CodeGen/X86/fp128-cast-strict.ll    |    8 +-
 llvm/test/CodeGen/X86/fp128-cast.ll           |   26 +-
 .../test/CodeGen/X86/fp128-libcalls-strict.ll |  400 +-
 llvm/test/CodeGen/X86/fp128-libcalls.ll       |  320 +-
 llvm/test/CodeGen/X86/fpenv-combine.ll        |   26 +-
 llvm/test/CodeGen/X86/fpenv.ll                |   32 +-
 llvm/test/CodeGen/X86/fptosi-sat-scalar.ll    |   24 +-
 llvm/test/CodeGen/X86/fptoui-sat-scalar.ll    |    8 +-
 llvm/test/CodeGen/X86/implicit-null-check.ll  |   19 +-
 .../CodeGen/X86/indirect-br-gep-unmerge.ll    |   51 +
 .../X86/inline-asm-avx512f-x-constraint.ll    |   13 +-
 ...iller-impdef-on-implicit-def-regression.ll |  178 +
 llvm/test/CodeGen/X86/large-constants-x32.ll  |   32 +-
 llvm/test/CodeGen/X86/machine-cse.ll          |    1 +
 llvm/test/CodeGen/X86/mixed-ptr-sizes.ll      |   21 +-
 llvm/test/CodeGen/X86/no-split-size.ll        |   90 +
 llvm/test/CodeGen/X86/osx-private-labels.ll   |    2 +-
 llvm/test/CodeGen/X86/pcsections-atomics.ll   | 3609 ++++----
 llvm/test/CodeGen/X86/peephole-copy.mir       |   34 +
 llvm/test/CodeGen/X86/physreg-pairs.ll        |    2 +-
 llvm/test/CodeGen/X86/popcnt.ll               |  222 +-
 llvm/test/CodeGen/X86/pr69080.ll              |   38 +
 .../CodeGen/X86/prologue-epilogue-remarks.mir |   12 +-
 llvm/test/CodeGen/X86/ragreedy-hoist-spill.ll |    2 +-
 llvm/test/CodeGen/X86/remat-phys-dead.ll      |    2 +-
 .../CodeGen/X86/scheduler-backtracking.ll     |   42 +-
 llvm/test/CodeGen/X86/sdiv_fix.ll             |    4 +-
 llvm/test/CodeGen/X86/sdiv_fix_sat.ll         |    8 +-
 llvm/test/CodeGen/X86/select_const_i128.ll    |    3 +-
 llvm/test/CodeGen/X86/setcc-wide-types.ll     |   56 +-
 llvm/test/CodeGen/X86/shrink_vmul.ll          |   48 +-
 llvm/test/CodeGen/X86/smul-with-overflow.ll   |    4 +-
 ...speculative-load-hardening-call-and-ret.ll |   75 +-
 llvm/test/CodeGen/X86/sret-implicit.ll        |    3 +-
 .../X86/stack-frame-layout-remarks-64.ll      |   76 -
 ...ks-32.ll => stack-frame-layout-remarks.ll} |   44 +-
 .../CodeGen/X86/statepoint-call-lowering.ll   |   21 +-
 .../CodeGen/X86/statepoint-deopt-lowering.ll  |  110 +-
 llvm/test/CodeGen/X86/statepoint-vector.ll    |    4 +-
 llvm/test/CodeGen/X86/swifterror.ll           |   28 +-
 llvm/test/CodeGen/X86/ternlog.ll              |    1 +
 llvm/test/CodeGen/X86/udiv_fix.ll             |    4 +-
 llvm/test/CodeGen/X86/udiv_fix_sat.ll         |    4 +-
 llvm/test/CodeGen/X86/usermsr-intrinsics.ll   |   64 +
 llvm/test/CodeGen/X86/vec_saddo.ll            |   35 +-
 llvm/test/CodeGen/X86/vec_ssubo.ll            |   36 +-
 llvm/test/CodeGen/X86/vec_uaddo.ll            |   63 +-
 llvm/test/CodeGen/X86/vec_usubo.ll            |   63 +-
 llvm/test/CodeGen/X86/vector-llrint.ll        |  290 +
 llvm/test/CodeGen/X86/vector-lrint.ll         |  429 +
 .../vector-shuffle-combining-avx512bwvl.ll    |    5 +-
 llvm/test/CodeGen/X86/vselect.ll              |   77 +-
 llvm/test/DebugInfo/ARM/sdag-split-arg.ll     |    4 +-
 .../DebugInfo/{X86 => Generic}/PR20038.ll     |    5 +-
 .../DebugInfo/Generic/discriminated-union.ll  |    4 +-
 .../DebugInfo/Generic/dwarf-public-names.ll   |  137 +
 .../{X86 => Generic}/member-order.ll          |    2 +-
 .../{X86 => Generic}/tu-composite.ll          |    2 +-
 .../Generic/univariant-discriminated-union.ll |    4 +-
 .../test/DebugInfo/M68k/calling-convention.ll |   21 +
 llvm/test/DebugInfo/M68k/lit.local.cfg        |    2 +
 .../Symbolize/XCOFF/xcoff-symbolize-data.ll   |    9 +-
 llvm/test/DebugInfo/X86/fi-piece.ll           |    5 +-
 llvm/test/DebugInfo/invalid-sizes.ll          |   80 -
 .../ExecutionEngine/2010-01-15-UndefValue.ll  |    1 +
 .../JITLink/ppc64/ELF_ppc64_relocations.s     |   34 +
 .../OrcLazy/minimal-throw-catch.ll            |    2 +-
 .../asan_instrument_constant_address_space.ll |   48 +-
 .../asan_instrument_generic_address_space.ll  |  122 +-
 .../asan_instrument_global_address_space.ll   |   93 +-
 .../Instrumentation/AddressSanitizer/basic.ll |    4 +-
 .../AddressSanitizer/global_metadata_array.ll |   19 +-
 .../AddressSanitizer/global_with_comdat.ll    |   50 +-
 .../HWAddressSanitizer/str-nobuiltin.ll       |   33 +
 llvm/test/Linker/type-unique-odr-a.ll         |    4 +-
 llvm/test/Linker/type-unique-odr-b.ll         |    2 -
 llvm/test/Linker/type-unique-simple2-a.ll     |    4 +-
 llvm/test/Linker/type-unique-simple2-b.ll     |    2 -
 llvm/test/MC/AArch64/FP8/system-regs.s        |   32 +
 llvm/test/MC/AArch64/arm64-system-encoding.s  |    2 +
 llvm/test/MC/AMDGPU/gfx1030_err.s             |    6 -
 llvm/test/MC/AMDGPU/gfx1030_new.s             |   21 +
 llvm/test/MC/AMDGPU/gfx11_asm_flat.s          |   12 +
 llvm/test/MC/AMDGPU/gfx11_asm_mubuf.s         |   15 +-
 llvm/test/MC/AMDGPU/gfx11_asm_mubuf_alias.s   |    9 +-
 llvm/test/MC/AMDGPU/literals.s                |   46 +
 llvm/test/MC/ARM/inst-directive-it-vpt.s      |   26 +
 .../MC/Disassembler/AMDGPU/decode-err.txt     |    4 -
 .../MC/Disassembler/AMDGPU/gfx1030_new.txt    |   14 +-
 .../Disassembler/AMDGPU/gfx11_dasm_flat.txt   |    9 +
 .../Disassembler/AMDGPU/gfx11_dasm_mubuf.txt  |   12 +
 llvm/test/MC/Disassembler/X86/usermsr-64.txt  |   28 +
 llvm/test/MC/RISCV/compress-rv32i.s           |   28 +-
 llvm/test/MC/RISCV/rvv/xsfvqmacc.s            |   57 +
 llvm/test/MC/X86/usermsr-64-att.s             |   18 +
 llvm/test/MC/X86/usermsr-64-intel.s           |   18 +
 llvm/test/Other/print-at-pass-number.ll       |    8 +-
 .../match-table-imms.td                       |   12 +-
 .../match-table-patfrag-root.td               |   36 +-
 .../match-table-permutations.td               |   68 +-
 .../GlobalISelCombinerEmitter/match-table.td  |   16 +-
 .../misc/redundant-combine-in-list.td         |   30 +
 llvm/test/TableGen/GlobalISelEmitter.td       |    4 +-
 .../test/TableGen/GlobalISelEmitterHwModes.td |    4 +-
 llvm/test/TableGen/VarLenEncoderHwModes.td    |  110 +
 llvm/test/TableGen/dump.td                    |  103 +
 llvm/test/TableGen/repr.td                    |   51 +
 .../Attributor/lvi-after-jumpthreading.ll     |    1 -
 .../Attributor/value-simplify-gpu.ll          |   73 +
 .../Transforms/Attributor/value-simplify.ll   |   53 +
 .../AArch64/sink-gather-scatter-addressing.ll |  231 +
 .../large-constant-ints.ll                    |   17 +-
 .../Transforms/ConstraintElimination/shl.ll   |    3 +-
 .../CorrelatedValuePropagation/abs.ll         |  133 +-
 .../CorrelatedValuePropagation/icmp.ll        |  104 +
 llvm/test/Transforms/FunctionAttrs/nonnull.ll |  372 +-
 .../test/Transforms/FunctionAttrs/readnone.ll |   16 +-
 .../Transforms/FunctionAttrs/writeonly.ll     |   49 +-
 llvm/test/Transforms/GVN/captured-before.ll   |   80 +
 llvm/test/Transforms/GVN/pr69301.ll           |   59 +
 .../HipStdPar/accelerator-code-selection.ll   |  116 +
 .../HipStdPar/allocation-interposition.ll     |  221 +
 .../HipStdPar/allocation-no-interposition.ll  |  161 +
 .../Transforms/HipStdPar/unsupported-asm.ll   |   12 +
 .../HipStdPar/unsupported-builtins.ll         |   11 +
 .../unsupported-thread-local-direct-use.ll    |   14 +
 .../unsupported-thread-local-indirect-use.ll  |   14 +
 .../InstCombine/AArch64/sve-intrinsic-sel.ll  |   10 +
 .../Transforms/InstCombine/and-add-shl.ll     |    2 +-
 llvm/test/Transforms/InstCombine/and.ll       |   28 +-
 .../dbg-scalable-store-fixed-frag.ll          |    6 +-
 .../InstCombine/dbg-simplify-alloca-size.ll   |    4 +-
 llvm/test/Transforms/InstCombine/div-shift.ll |  269 +
 llvm/test/Transforms/InstCombine/icmp-add.ll  |   26 +
 .../Transforms/InstCombine/icmp-shr-lt-gt.ll  |   13 +
 llvm/test/Transforms/InstCombine/icmp-shr.ll  |  187 +-
 .../InstCombine/icmp-xor-signbit.ll           |  104 +
 .../Transforms/InstCombine/malloc-free.ll     |   55 +
 llvm/test/Transforms/InstCombine/powi.ll      |   83 +
 llvm/test/Transforms/InstCombine/pr69059.ll   |   16 +
 ...undant-left-shift-input-masking-pr49778.ll |    4 +-
 llvm/test/Transforms/InstCombine/rotate.ll    |    2 +-
 .../test/Transforms/InstCombine/sdiv-guard.ll |    5 +-
 .../Transforms/InstCombine/shift-flags.ll     |  119 +
 .../InstCombine/simplify-demanded-fpclass.ll  |  203 +-
 .../InstCombine/trunc-inseltpoison.ll         |    6 +-
 llvm/test/Transforms/InstCombine/trunc.ll     |    6 +-
 .../Transforms/InstSimplify/ctpop-pow2.ll     |    5 +-
 .../LoopVectorize/AArch64/masked-call.ll      |  662 +-
 .../AArch64/outer_loop_prefer_scalable.ll     |  117 +
 .../AArch64/sleef-calls-aarch64.ll            |  934 +-
 .../AArch64/sleef-intrinsic-calls-aarch64.ll  | 1311 +++
 .../LoopVectorize/ARM/mve-known-trip-count.ll |    4 +-
 .../RISCV/riscv-vector-reverse.ll             |   28 +-
 .../LoopVectorize/X86/interleaving.ll         |   78 +-
 .../X86/invariant-store-vectorization.ll      |   34 +-
 ...86-interleaved-store-accesses-with-gaps.ll |  340 +-
 .../epilog-vectorization-reductions.ll        |    8 +-
 .../LoopVectorize/extract-last-veclane.ll     |   38 +-
 .../LoopVectorize/if-pred-stores.ll           |   62 +-
 .../interleave-and-scalarize-only.ll          |   35 +
 .../LoopVectorize/outer_loop_scalable.ll      |  150 +
 .../LoopVectorize/reduction-inloop-uf4.ll     |   48 +-
 .../LoopVectorize/reduction-small-size.ll     |    8 +-
 .../Transforms/LoopVectorize/reduction.ll     |  931 +-
 .../scalable-reduction-inloop.ll              |    8 +-
 .../LoopVectorize/scev-invalidation.ll        |   22 +
 .../LoopVectorize/vplan-dot-printing.ll       |    2 +-
 .../LoopVectorize/vplan-printing.ll           |   68 +-
 .../vplan-sink-scalars-and-merge-vf1.ll       |    4 +-
 .../LowerTypeTests/function-weak.ll           |   13 +
 .../Transforms/MergeFunc/cfi-thunk-merging.ll |  210 +
 .../mergefunc-preserve-debug-info.ll          |  105 +-
 .../mergefunc-preserve-vfe-intrinsics.ll      |   44 +
 .../MergeFunc/no-merge-debug-thunks.ll        |   61 +
 .../Transforms/OpenMP/parallel_level_fold.ll  |    2 +-
 .../PhaseOrdering/X86/pixel-splat.ll          |   34 +-
 .../Transforms/PhaseOrdering/X86/pr50555.ll   |  100 +-
 .../PhaseOrdering/X86/speculation-vs-tbaa.ll  |   10 +-
 .../icmp-ashr-breaking-select-idiom.ll        |   38 +
 llvm/test/Transforms/PhaseOrdering/pr39282.ll |   16 +-
 .../switch_different_gep_types.ll             |   75 +
 .../PreISelIntrinsicLowering/X86/objc-arc.ll  |   16 +-
 .../call-argument-attributes.ll               |   42 +
 .../X86/matching-gather-nodes-phi-users.ll    |  124 +
 .../X86/multi-nodes-to-shuffle.ll             |    1 +
 .../non-scheduled-inst-reused-as-last-inst.ll |   45 +
 llvm/test/Transforms/Util/dbg-user-of-aext.ll |    8 +-
 .../RISCV/vpintrin-scalarization.ll           |  488 +-
 llvm/test/Unit/lit.cfg.py                     |    6 +-
 llvm/test/Unit/lit.site.cfg.py.in             |    1 +
 llvm/test/YAMLParser/plain-characters.test    |   30 +
 llvm/test/tools/llc/aix-pic-setting.ll        |    2 +-
 .../X86/cu_tu_units_manual_v5_invalid.s       |   73 +
 llvm/test/tools/llvm-lib/arm64ec-implib.test  |   54 +
 llvm/test/tools/llvm-lto2/X86/pipeline.ll     |    2 +-
 .../tools/llvm-lto2/X86/slp-vectorize-pm.ll   |    2 +-
 .../tools/llvm-lto2/X86/stats-file-option.ll  |    2 +-
 .../llvm-mca/AArch64/Cortex/A57-writeback.s   | 3569 ++++---
 .../AArch64/HiSilicon/tsv110-writeback.s      | 3954 ++++++++
 .../llvm-mca/AArch64/Neoverse/N1-writeback.s  | 4079 ++++----
 .../llvm-mca/AArch64/Neoverse/N2-writeback.s  | 3839 ++++----
 .../llvm-mca/AArch64/Neoverse/V1-writeback.s  | 4150 +++++----
 llvm/test/tools/llvm-mca/RISCV/vle-vse.s      | 1249 +++
 .../tools/llvm-objdump/ELF/AMDGPU/kd-gfx10.s  |   16 -
 .../tools/llvm-objdump/ELF/AMDGPU/kd-gfx11.s  |  220 +
 .../llvm-objdump/ELF/AMDGPU/kd-zeroed-gfx10.s |    4 -
 .../tools/llvm-objdump/ELF/RISCV/branches.s   |    4 +-
 .../ELF/RISCV/multi-instr-target.s            |   45 +
 llvm/test/tools/llvm-rc/Inputs/split-path.rc  |    2 +
 llvm/test/tools/llvm-rc/split-path.test       |    7 +
 .../tools/llvm-readobj/ELF/note-core.test     |   47 +-
 .../Inputs/annotation-count-with-dbg-loc.yaml |    2 +-
 .../Inputs/annotation-count.yaml              |    4 +-
 .../annotation-count-with-dbg-loc.test        |    7 +
 .../llvm-remarkutil/annotation-count.test     |    7 +
 .../broken-bitstream-remark.test              |    2 +
 .../llvm-remarkutil/broken-yaml-remark.test   |    2 +
 .../count/Inputs/remark-count-by.yaml         |   43 +
 .../count/Inputs/remark-filter-by.yaml        |   40 +
 .../count/Inputs/remark-group-by.yaml         |   54 +
 .../llvm-remarkutil/count/count-by-keys.test  |   20 +
 .../count/count-by-remark.test                |   20 +
 .../count/filter-by-pass-name.test            |   10 +
 .../count/filter-by-remark-name.test          |   10 +
 .../llvm-remarkutil/count/filter-by-type.test |   16 +
 .../count/group-by-function-with-loc.test     |    7 +
 .../count/group-by-function.test              |    7 +
 .../count/group-by-source.test                |    6 +
 .../tools/llvm-remarkutil/empty-file.test     |    5 +
 .../instruction-count-with-dbg-loc.test       |    7 +
 .../llvm-remarkutil/instruction-count.test    |    7 +
 .../tools/llvm-symbolizer/xcoff-sd-symbol.ll  |   25 +
 llvm/tools/dsymutil/DebugMap.cpp              |    1 -
 llvm/tools/dsymutil/DwarfLinkerForBinary.cpp  |    1 -
 llvm/tools/llc/llc.cpp                        |   14 -
 llvm/tools/llvm-dwarfutil/DebugInfoLinker.cpp |    1 -
 .../llvm-exegesis/lib/BenchmarkResult.cpp     |    1 -
 .../tools/llvm-exegesis/lib/BenchmarkResult.h |    1 -
 llvm/tools/llvm-exegesis/lib/LlvmState.h      |    1 -
 .../llvm-exegesis/lib/X86/X86Counter.cpp      |    3 +-
 llvm/tools/llvm-gsymutil/llvm-gsymutil.cpp    |    5 +-
 llvm/tools/llvm-jitlink/llvm-jitlink.cpp      |    3 +-
 llvm/tools/llvm-objdump/MachODump.cpp         |    7 +-
 llvm/tools/llvm-objdump/llvm-objdump.cpp      |   44 +-
 llvm/tools/llvm-pdbutil/DumpOutputStyle.cpp   |    2 +-
 .../tools/llvm-pdbutil/ExplainOutputStyle.cpp |    5 +-
 llvm/tools/llvm-pdbutil/llvm-pdbutil.cpp      |    2 +-
 llvm/tools/llvm-profdata/llvm-profdata.cpp    |   25 +-
 llvm/tools/llvm-profgen/CSPreInliner.cpp      |   10 +-
 llvm/tools/llvm-profgen/CSPreInliner.h        |    7 +-
 llvm/tools/llvm-profgen/CallContext.h         |    2 +-
 llvm/tools/llvm-profgen/ProfileGenerator.cpp  |   33 +-
 llvm/tools/llvm-profgen/ProfileGenerator.h    |    2 +-
 llvm/tools/llvm-profgen/ProfiledBinary.cpp    |   57 +-
 llvm/tools/llvm-profgen/ProfiledBinary.h      |   28 +-
 llvm/tools/llvm-rc/ResourceFileWriter.cpp     |    2 +-
 llvm/tools/llvm-rc/ResourceFileWriter.h       |    4 +-
 llvm/tools/llvm-rc/ResourceScriptParser.cpp   |   26 +-
 llvm/tools/llvm-rc/ResourceScriptParser.h     |    4 +
 llvm/tools/llvm-readobj/COFFDumper.cpp        |   23 +-
 llvm/tools/llvm-readobj/DwarfCFIEHPrinter.h   |    5 +-
 llvm/tools/llvm-readobj/ELFDumper.cpp         |   26 +-
 llvm/tools/llvm-readobj/MachODumper.cpp       |    8 +-
 llvm/tools/llvm-readobj/ObjDumper.h           |    2 -
 .../llvm-readobj/WindowsResourceDumper.cpp    |    2 +-
 llvm/tools/llvm-readtapi/CMakeLists.txt       |    4 +-
 .../llvm-reduce/deltas/ReduceAttributes.cpp   |    1 -
 .../deltas/ReduceOperandBundles.cpp           |    1 -
 llvm/tools/llvm-remarkutil/CMakeLists.txt     |    1 +
 llvm/tools/llvm-remarkutil/RemarkCounter.cpp  |  337 +
 llvm/tools/llvm-remarkutil/RemarkCounter.h    |  217 +
 llvm/tools/llvm-rtdyld/llvm-rtdyld.cpp        |    4 +-
 llvm/tools/llvm-xray/xray-converter.cpp       |    2 +-
 llvm/tools/llvm-xray/xray-graph-diff.cpp      |    1 -
 llvm/tools/llvm-xray/xray-graph-diff.h        |    1 -
 llvm/tools/obj2yaml/coff2yaml.cpp             |    2 +-
 llvm/tools/opt/opt.cpp                        |    1 -
 llvm/unittests/ADT/CMakeLists.txt             |    1 +
 .../ADT/CombinationGeneratorTest.cpp          |    1 -
 llvm/unittests/ADT/IListIteratorBitsTest.cpp  |  138 +
 llvm/unittests/ADT/RangeAdapterTest.cpp       |    1 -
 .../FunctionPropertiesAnalysisTest.cpp        |    3 +-
 .../Analysis/TargetLibraryInfoTest.cpp        |   10 +
 .../Bitcode/DataLayoutUpgradeTest.cpp         |   39 +-
 .../CodeGen/AArch64SelectionDAGTest.cpp       |  107 +
 llvm/unittests/CodeGen/InstrRefLDVTest.cpp    |    8 +-
 .../CodeView/RandomAccessVisitorTest.cpp      |    9 +-
 .../DebugInfo/DWARF/DWARFDebugInfoTest.cpp    |    7 +-
 llvm/unittests/DebugInfo/GSYM/GSYMTest.cpp    |  302 +-
 .../DebugInfo/MSF/MappedBlockStreamTest.cpp   |    8 +-
 .../unittests/DebugInfo/PDB/HashTableTest.cpp |    4 +-
 .../DebugInfo/PDB/StringTableBuilderTest.cpp  |    4 +-
 .../DebugInfo/Symbolizer/MarkupTest.cpp       |    1 -
 .../ExecutionEngine/JITLink/AArch32Tests.cpp  |   24 +-
 .../ExecutionEngine/JITLink/JITLinkMocks.cpp  |    3 +-
 .../JITLink/LinkGraphTests.cpp                |   34 +-
 .../JITLink/MemoryManagerErrorTests.cpp       |    3 +-
 .../ExecutionEngine/JITLink/StubsTests.cpp    |   12 +-
 .../Orc/ObjectLinkingLayerTest.cpp            |   18 +-
 .../Frontend/OpenMPIRBuilderTest.cpp          |  346 +-
 llvm/unittests/MC/DwarfLineTableHeaders.cpp   |    4 +-
 llvm/unittests/MC/StringTableBuilderTest.cpp  |    4 +-
 llvm/unittests/ObjCopy/ObjCopyTest.cpp        |    1 -
 llvm/unittests/ProfileData/InstrProfTest.cpp  |    6 +-
 llvm/unittests/ProfileData/SampleProfTest.cpp |   53 +-
 llvm/unittests/Support/ARMAttributeParser.cpp |    4 +-
 llvm/unittests/Support/BLAKE3Test.cpp         |    4 +-
 llvm/unittests/Support/BinaryStreamTest.cpp   |   17 +-
 llvm/unittests/Support/CMakeLists.txt         |    1 +
 .../Support/CSKYAttributeParserTest.cpp       |    6 +-
 llvm/unittests/Support/CompressionTest.cpp    |    1 -
 .../Support/ELFAttributeParserTest.cpp        |    2 +-
 llvm/unittests/Support/EndianStreamTest.cpp   |   22 +-
 llvm/unittests/Support/EndianTest.cpp         |   72 +-
 llvm/unittests/Support/HashBuilderTest.cpp    |   20 +-
 llvm/unittests/Support/KnownBitsTest.cpp      |   31 +
 llvm/unittests/Support/NativeFormatTests.cpp  |    1 -
 .../Support/RISCVAttributeParserTest.cpp      |    2 +-
 llvm/unittests/Support/RISCVISAInfoTest.cpp   |    4 +
 llvm/unittests/Support/RegexTest.cpp          |   28 +
 llvm/unittests/Support/SHA256.cpp             |    1 -
 llvm/unittests/Support/UTCTimeTest.cpp        |   41 +
 llvm/unittests/Support/YAMLIOTest.cpp         |    9 +-
 llvm/unittests/Support/YAMLParserTest.cpp     |   61 +-
 llvm/unittests/Target/LoongArch/InstSizes.cpp |    2 +-
 llvm/unittests/Target/RISCV/CMakeLists.txt    |    1 +
 .../Target/RISCV/RISCVBaseInfoTest.cpp        |   34 +
 .../TargetParser/TargetParserTest.cpp         |    4 +-
 .../Transforms/Utils/CodeExtractorTest.cpp    |   60 +
 .../Transforms/Utils/CodeLayoutTest.cpp       |    8 +
 llvm/utils/TableGen/DirectiveEmitter.cpp      |    1 +
 .../TableGen/GlobalISelCombinerEmitter.cpp    |   67 +-
 llvm/utils/TableGen/GlobalISelEmitter.cpp     |    4 +-
 llvm/utils/TableGen/GlobalISelMatchTable.cpp  |   10 +
 llvm/utils/TableGen/GlobalISelMatchTable.h    |   19 +
 llvm/utils/TableGen/SubtargetEmitter.cpp      |    4 +-
 llvm/utils/TableGen/VarLenCodeEmitterGen.cpp  |  189 +-
 llvm/utils/TableGen/X86DisassemblerTables.cpp |    1 +
 llvm/utils/TableGen/X86DisassemblerTables.h   |    3 +-
 llvm/utils/TableGen/X86RecognizableInstr.cpp  |    1 +
 llvm/utils/TableGen/X86RecognizableInstr.h    |    2 +-
 llvm/utils/TableGen/jupyter/README.md         |    2 +
 llvm/utils/emacs/llvm-mode.el                 |    2 +-
 llvm/utils/emacs/tablegen-mode.el             |    2 +-
 llvm/utils/git/code-format-helper.py          |   41 +-
 .../clang-tidy/bugprone/BUILD.gn              |    1 +
 .../clang-tidy/misc/BUILD.gn                  |    1 +
 .../gn/secondary/clang/lib/Headers/BUILD.gn   |    2 +
 .../clang/unittests/AST/Interp/BUILD.gn       |   13 +
 .../gn/secondary/clang/unittests/BUILD.gn     |    1 +
 .../secondary/clang/unittests/Driver/BUILD.gn |    1 +
 .../compiler-rt/lib/builtins/BUILD.gn         |    2 +
 .../compiler-rt/lib/scudo/standalone/BUILD.gn |    6 +
 .../lib/scudo/standalone/tests/BUILD.gn       |    1 +
 .../compiler-rt/lib/tsan/rtl/BUILD.gn         |    2 +
 .../gn/secondary/libcxx/include/BUILD.gn      |    3 +-
 llvm/utils/gn/secondary/lldb/test/BUILD.gn    |    2 +-
 .../tools/{lldb-vscode => lldb-dap}/BUILD.gn  |   12 +-
 .../gn/secondary/llvm/lib/Passes/BUILD.gn     |    1 +
 .../secondary/llvm/lib/Target/AMDGPU/BUILD.gn |    1 +
 .../gn/secondary/llvm/lib/Target/X86/BUILD.gn |    1 -
 .../llvm/lib/Transforms/HipStdPar/BUILD.gn    |   10 +
 llvm/utils/gn/secondary/llvm/test/BUILD.gn    |    5 +-
 .../llvm/tools/llvm-readtapi/BUILD.gn         |   19 +
 .../llvm/tools/llvm-remarkutil/BUILD.gn       |    1 +
 .../gn/secondary/llvm/unittests/ADT/BUILD.gn  |    1 +
 .../secondary/llvm/unittests/Support/BUILD.gn |    1 +
 .../llvm/unittests/Target/RISCV/BUILD.gn      |    5 +-
 llvm/utils/kate/llvm-tablegen.xml             |    3 +
 llvm/utils/kate/llvm.xml                      |    1 +
 llvm/utils/lit/lit/LitConfig.py               |    2 +
 llvm/utils/lit/lit/TestRunner.py              |   52 +-
 llvm/utils/lit/lit/TestingConfig.py           |    1 +
 llvm/utils/lit/lit/cl_arguments.py            |   12 +
 llvm/utils/lit/lit/formats/googletest.py      |   77 +-
 llvm/utils/lit/lit/main.py                    |    9 +-
 .../DummySubDir/OneTest.py                    |  105 +
 .../Inputs/googletest-no-sharding/lit.cfg     |    4 +
 .../errors/assignment/before-name.txt         |    2 +-
 .../errors/assignment/between-name-equals.txt |    2 +-
 .../errors/assignment/braces-empty.txt        |    2 +-
 .../errors/assignment/braces-with-dot.txt     |    2 +-
 .../errors/assignment/braces-with-equals.txt  |    2 +-
 .../errors/assignment/braces-with-newline.txt |    2 +-
 .../errors/assignment/braces-with-number.txt  |    2 +-
 .../errors/assignment/braces-with-ws.txt      |    2 +-
 .../shtest-define/errors/assignment/empty.txt |    2 +-
 .../errors/assignment/no-equals.txt           |    2 +-
 .../errors/assignment/no-name.txt             |    2 +-
 .../errors/assignment/ws-only.txt             |    2 +-
 .../errors/continuation/empty.txt             |    2 +-
 .../continuation/end-in-double-backslash.txt  |    2 +-
 .../unterminated-define-bad-redefine.txt      |    2 +-
 .../unterminated-define-continuation.txt      |    2 +-
 .../unterminated-define-redefine.txt          |    2 +-
 .../continuation/unterminated-define-run.txt  |    2 +-
 .../continuation/unterminated-define.txt      |    2 +-
 .../unterminated-redefine-bad-define.txt      |    2 +-
 .../unterminated-redefine-continuation.txt    |    2 +-
 .../unterminated-redefine-define.txt          |    2 +-
 .../unterminated-redefine-run.txt             |    2 +-
 .../continuation/unterminated-redefine.txt    |    2 +-
 .../continuation/unterminated-run-define.txt  |    2 +-
 .../unterminated-run-redefine.txt             |    2 +-
 .../errors/continuation/ws-only.txt           |    2 +-
 .../define-already-by-config.txt              |    2 +-
 .../defined-check/define-already-by-test.txt  |    2 +-
 .../defined-check/define-inside-pattern.txt   |    2 +-
 .../defined-check/define-multiple-exact.txt   |    2 +-
 .../define-multiple-once-exact.txt            |    2 +-
 .../defined-check/define-prefixes-pattern.txt |    2 +-
 .../defined-check/define-suffixes-pattern.txt |    2 +-
 .../defined-check/redefine-inside-pattern.txt |    2 +-
 .../defined-check/redefine-multiple-exact.txt |    2 +-
 .../redefine-multiple-once-exact.txt          |    2 +-
 .../errors/defined-check/redefine-none.txt    |    2 +-
 .../redefine-prefixes-pattern.txt             |    2 +-
 .../redefine-suffixes-pattern.txt             |    2 +-
 .../shtest-define/errors/location-range.txt   |    2 +-
 .../Inputs/shtest-define/errors/no-run.txt    |    2 +-
 .../shtest-define/examples/param-subst.txt    |    2 +-
 .../Inputs/shtest-define/expansion-order.txt  |    2 +-
 .../line-number-substitutions.txt             |    2 +-
 .../tests/Inputs/shtest-define/name-chars.txt |    2 +-
 .../shtest-define/recursiveExpansionLimit.txt |    2 +-
 .../Inputs/shtest-define/shared-substs-0.txt  |    2 +-
 .../Inputs/shtest-define/shared-substs-1.txt  |    2 +-
 .../Inputs/shtest-define/value-equals.txt     |    2 +-
 .../Inputs/shtest-define/value-escaped.txt    |    2 +-
 .../shtest-define/ws-and-continuations.txt    |    2 +-
 llvm/utils/lit/tests/discovery.py             |    7 +
 .../utils/lit/tests/googletest-no-sharding.py |   43 +
 llvm/utils/lit/tests/ignore-fail.py           |    9 +-
 llvm/utils/lit/tests/shtest-env.py            |    5 +-
 llvm/utils/lit/tests/shtest-not.py            |    5 +-
 llvm/utils/lit/tests/shtest-pushd-popd.py     |    5 +-
 llvm/utils/lit/tests/shtest-shell.py          |   18 +-
 llvm/utils/vim/syntax/llvm.vim                |    1 +
 llvm/utils/vim/syntax/tablegen.vim            |    2 +-
 .../vscode/llvm/syntaxes/TableGen.tmLanguage  |    2 +-
 mlir/cmake/modules/AddMLIR.cmake              |    3 +-
 mlir/docs/Bindings/Python.md                  |  127 +-
 mlir/docs/Dialects/Linalg/OpDSL.md            |    2 +-
 mlir/docs/Interfaces.md                       |    4 +
 mlir/docs/Passes.md                           |    4 +
 mlir/examples/minimal-opt/README.md           |    8 +-
 mlir/examples/minimal-opt/mlir-cat.cpp        |    7 +-
 .../lib/Standalone/StandaloneOps.cpp          |    1 -
 .../lib/Standalone/StandalonePasses.cpp       |    4 +-
 .../standalone/python/StandaloneExtension.cpp |    1 -
 .../standalone-opt/standalone-opt.cpp         |    8 -
 .../standalone-plugin/standalone-plugin.cpp   |    7 +-
 .../standalone-translate.cpp                  |    3 +
 mlir/examples/toy/Ch1/parser/AST.cpp          |    3 +
 mlir/examples/toy/Ch1/toyc.cpp                |    5 +
 mlir/examples/toy/Ch2/mlir/Dialect.cpp        |   12 +
 mlir/examples/toy/Ch2/mlir/MLIRGen.cpp        |   15 +-
 mlir/examples/toy/Ch2/parser/AST.cpp          |    3 +
 mlir/examples/toy/Ch6/mlir/LowerToLLVM.cpp    |    9 +-
 mlir/examples/toy/Ch7/mlir/LowerToLLVM.cpp    |    9 +-
 mlir/include/mlir-c/Dialect/SparseTensor.h    |    3 +-
 mlir/include/mlir-c/IR.h                      |   19 +-
 .../mlir/Analysis/DataFlow/DeadCodeAnalysis.h |    3 +-
 .../mlir/Analysis/Presburger/Fraction.h       |    2 +-
 .../Analysis/Presburger/LinearTransform.h     |    4 -
 .../include/mlir/Analysis/Presburger/Matrix.h |   64 +-
 .../Conversion/FuncToLLVM/ConvertFuncToLLVM.h |   15 +
 .../Conversion/IndexToSPIRV/IndexToSPIRV.h    |   30 +
 mlir/include/mlir/Conversion/Passes.h         |    1 +
 mlir/include/mlir/Conversion/Passes.td        |   36 +-
 .../Conversion/TosaToLinalg/TosaToLinalg.h    |    6 +-
 .../mlir/Dialect/Affine/IR/AffineOps.td       |    3 +-
 .../Affine/IR/ValueBoundsOpInterfaceImpl.h    |   16 +
 mlir/include/mlir/Dialect/ArmSME/IR/ArmSME.h  |    1 +
 .../Dialect/ArmSME/IR/ArmSMEIntrinsicOps.td   |   48 +-
 .../mlir/Dialect/ArmSME/IR/ArmSMEOps.td       |   41 +-
 mlir/include/mlir/Dialect/ArmSVE/IR/ArmSVE.td |   84 +
 .../Bufferization/IR/BufferizationBase.td     |   11 +
 .../Bufferization/IR/BufferizationOps.td      |   40 +-
 .../Dialect/ControlFlow/IR/ControlFlowOps.td  |    3 +-
 .../Dialect/LLVMIR/BasicPtxBuilderInterface.h |   80 +
 .../LLVMIR/BasicPtxBuilderInterface.td        |  157 +
 .../mlir/Dialect/LLVMIR/CMakeLists.txt        |    8 +-
 mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td   |    4 +-
 .../include/mlir/Dialect/LLVMIR/NVVMDialect.h |    3 +-
 mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td   |  263 +-
 .../Dialect/Linalg/IR/LinalgInterfaces.td     |   33 +
 .../Dialect/Linalg/IR/LinalgStructuredOps.td  |   16 +-
 .../Dialect/Linalg/Transforms/Transforms.h    |   26 +-
 mlir/include/mlir/Dialect/Mesh/IR/MeshBase.td |    4 +-
 mlir/include/mlir/Dialect/Mesh/IR/MeshOps.td  |    2 +-
 mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td   |   52 +-
 .../mlir/Dialect/NVGPU/IR/NVGPUDialect.h      |    3 +
 mlir/include/mlir/Dialect/SCF/IR/SCFOps.td    |   17 +-
 .../SCF/Transforms/TileUsingInterface.h       |   17 +
 .../SPIRV/Transforms/SPIRVConversion.h        |   11 +-
 .../Dialect/SparseTensor/IR/CMakeLists.txt    |    6 +
 .../mlir/Dialect/SparseTensor/IR/Enums.h      |   59 +-
 .../Dialect/SparseTensor/IR/SparseTensor.h    |   29 +
 .../SparseTensor/IR/SparseTensorAttrDefs.td   |   47 +-
 .../SparseTensor/IR/SparseTensorInterfaces.h  |   31 +
 .../SparseTensor/IR/SparseTensorInterfaces.td |   45 +
 .../SparseTensor/IR/SparseTensorOps.td        |   91 +-
 .../SparseTensor/IR/SparseTensorType.h        |   16 +-
 .../Dialect/SparseTensor/Pipelines/Passes.h   |   13 +-
 .../Dialect/SparseTensor/Transforms/Passes.h  |   31 +-
 .../Dialect/SparseTensor/Transforms/Passes.td |    5 +-
 .../mlir/Dialect/Tensor/IR/TensorOps.td       |   38 +-
 mlir/include/mlir/Dialect/Tosa/IR/TosaOps.h   |    5 +
 mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td  |    5 +-
 .../mlir/Dialect/Tosa/IR/TosaTypesBase.td     |    6 +
 .../mlir/Dialect/Tosa/IR/TosaUtilOps.td       |   67 +
 .../mlir/Dialect/Tosa/Transforms/Passes.h     |    8 +-
 .../mlir/Dialect/Tosa/Transforms/Passes.td    |   10 +-
 .../Transform/IR/TransformInterfaces.h        |   12 +
 .../Dialect/Transform/Transforms/Passes.td    |   33 +
 .../Transforms/TransformInterpreterUtils.h    |   18 +
 .../mlir/Dialect/Utils/StructuredOpsUtils.h   |   18 +
 .../mlir/Dialect/Vector/IR/VectorOps.h        |   19 +-
 .../mlir/Dialect/Vector/IR/VectorOps.td       |    7 +-
 .../mlir/ExecutionEngine/SparseTensor/COO.h   |   21 +-
 .../mlir/ExecutionEngine/SparseTensor/File.h  |    7 +-
 .../ExecutionEngine/SparseTensor/MapRef.h     |   33 +-
 .../ExecutionEngine/SparseTensor/Storage.h    |  906 +-
 .../ExecutionEngine/SparseTensorRuntime.h     |   91 +-
 mlir/include/mlir/IR/Attributes.h             |    1 -
 mlir/include/mlir/IR/BuiltinTypes.h           |   59 +-
 mlir/include/mlir/IR/CommonTypeConstraints.td |   90 +-
 mlir/include/mlir/IR/OpBase.td                |    8 +
 mlir/include/mlir/IR/OpImplementation.h       |   14 +-
 mlir/include/mlir/IR/Types.h                  |    1 -
 mlir/include/mlir/IR/Utils.td                 |   24 +
 mlir/include/mlir/IR/Value.h                  |    1 -
 .../mlir/Interfaces/LoopLikeInterface.h       |    5 +
 .../mlir/Interfaces/LoopLikeInterface.td      |   30 +
 .../mlir/Interfaces/ValueBoundsOpInterface.h  |   10 +
 .../include/mlir/Query/Matcher/ErrorBuilder.h |   63 +
 mlir/include/mlir/Query/Matcher/Marshallers.h |  199 +
 mlir/include/mlir/Query/Matcher/MatchFinder.h |   41 +
 .../mlir/Query/Matcher/MatchersInternal.h     |   72 +
 mlir/include/mlir/Query/Matcher/Registry.h    |   51 +
 .../include/mlir/Query/Matcher/VariantValue.h |  128 +
 mlir/include/mlir/Query/Query.h               |  109 +
 mlir/include/mlir/Query/QuerySession.h        |   42 +
 mlir/include/mlir/Support/ADTExtras.h         |   83 +
 .../mlir/Target/LLVMIR/ModuleTranslation.h    |    3 -
 .../mlir/Tools/mlir-query/MlirQueryMain.h     |   30 +
 mlir/include/mlir/Transforms/RegionUtils.h    |    3 +
 mlir/lib/Analysis/CallGraph.cpp               |    2 +-
 .../Analysis/DataFlow/DeadCodeAnalysis.cpp    |   12 +-
 mlir/lib/Analysis/Presburger/Matrix.cpp       |  246 +-
 .../Presburger/PresburgerRelation.cpp         |   18 +-
 mlir/lib/Analysis/Presburger/Utils.cpp        |    4 +-
 mlir/lib/AsmParser/AttributeParser.cpp        |    5 +-
 .../Bindings/Python/DialectSparseTensor.cpp   |   17 +-
 mlir/lib/Bindings/Python/Globals.h            |    4 +-
 mlir/lib/Bindings/Python/IRCore.cpp           |   18 +-
 mlir/lib/Bindings/Python/IRModule.cpp         |    4 +-
 mlir/lib/Bindings/Python/IRModule.h           |    6 +
 mlir/lib/Bindings/Python/MainModule.cpp       |   11 +-
 mlir/lib/Bindings/Python/Pass.cpp             |   37 +-
 mlir/lib/Bytecode/Writer/IRNumbering.cpp      |    2 +-
 mlir/lib/CAPI/Dialect/SparseTensor.cpp        |    7 +-
 mlir/lib/CAPI/IR/IR.cpp                       |   15 +
 mlir/lib/CMakeLists.txt                       |    1 +
 .../Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp    |    4 +-
 mlir/lib/Conversion/CMakeLists.txt            |    1 +
 mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp |  284 +-
 .../Conversion/GPUCommon/GPUOpsLowering.cpp   |   18 +-
 .../Conversion/IndexToSPIRV/CMakeLists.txt    |   17 +
 .../Conversion/IndexToSPIRV/IndexToSPIRV.cpp  |  418 +
 .../Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp    |  149 +-
 mlir/lib/Conversion/NVVMToLLVM/NVVMToLLVM.cpp |  155 +-
 mlir/lib/Conversion/SCFToSPIRV/CMakeLists.txt |    1 +
 .../Conversion/SCFToSPIRV/SCFToSPIRVPass.cpp  |    2 +
 .../TosaToLinalg/TosaToLinalgNamed.cpp        |    2 +-
 .../TosaToLinalg/TosaToLinalgPass.cpp         |   13 +-
 .../VectorToArmSME/VectorToArmSME.cpp         |    6 +-
 mlir/lib/Dialect/Affine/IR/AffineOps.cpp      |    4 +
 .../Affine/IR/ValueBoundsOpInterfaceImpl.cpp  |   44 +-
 .../Dialect/Arith/IR/ArithCanonicalization.td |    9 +-
 mlir/lib/Dialect/Arith/IR/ArithOps.cpp        |   34 +-
 mlir/lib/Dialect/ArmSVE/IR/ArmSVEDialect.cpp  |    1 +
 mlir/lib/Dialect/ArmSVE/IR/CMakeLists.txt     |    1 +
 .../Dialect/ArmSVE/Transforms/CMakeLists.txt  |    1 +
 .../Transforms/LegalizeForLLVMExport.cpp      |   85 +-
 .../IR/BufferDeallocationOpInterface.cpp      |    3 +-
 .../Bufferization/IR/BufferizationDialect.cpp |   20 +
 .../Bufferization/IR/BufferizationOps.cpp     |   15 +-
 .../OwnershipBasedBufferDeallocation.cpp      |   39 +-
 .../GPU/Transforms/SerializeToCubin.cpp       |   11 +-
 mlir/lib/Dialect/LLVMIR/CMakeLists.txt        |    2 +
 .../LLVMIR/IR/BasicPtxBuilderInterface.cpp    |  158 +
 mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp    |   21 +-
 .../Dialect/Linalg/IR/LinalgInterfaces.cpp    |    3 -
 .../TransformOps/LinalgTransformOps.cpp       |    3 +
 .../Transforms/ConvertConv2DToImg2Col.cpp     |  150 +-
 .../Linalg/Transforms/ElementwiseOpFusion.cpp |  187 +-
 .../Linalg/Transforms/HoistPadding.cpp        |    5 +-
 .../Dialect/Linalg/Transforms/Hoisting.cpp    |   12 +-
 .../Dialect/Linalg/Transforms/Promotion.cpp   |   10 +
 .../Linalg/Transforms/Vectorization.cpp       |   15 +-
 .../MemRef/Transforms/FoldMemRefAliasOps.cpp  |   12 +-
 mlir/lib/Dialect/Mesh/IR/MeshOps.cpp          |   16 +-
 mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp    |   97 +-
 .../NVGPU/TransformOps/NVGPUTransformOps.cpp  |   32 +-
 mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp       |    6 +-
 mlir/lib/Dialect/SCF/IR/SCF.cpp               |   75 +-
 .../BufferizableOpInterfaceImpl.cpp           |    3 +-
 .../SCF/Transforms/LoopCanonicalization.cpp   |    5 +-
 .../SCF/Transforms/TileUsingInterface.cpp     |  134 +-
 mlir/lib/Dialect/SCF/Utils/Utils.cpp          |    1 -
 .../Transforms/SPIRVWebGPUTransforms.cpp      |   44 +-
 .../Dialect/SparseTensor/IR/CMakeLists.txt    |    1 +
 .../SparseTensor/IR/Detail/DimLvlMap.cpp      |   11 +-
 .../SparseTensor/IR/SparseTensorDialect.cpp   |  416 +-
 .../IR/SparseTensorInterfaces.cpp             |   55 +
 .../Pipelines/SparseTensorPipelines.cpp       |    6 +-
 .../SparseTensor/Transforms/CodegenUtils.cpp  |  204 +-
 .../SparseTensor/Transforms/CodegenUtils.h    |   28 -
 .../SparseTensor/Transforms/LoopEmitter.cpp   |   18 +-
 .../Transforms/SparseGPUCodegen.cpp           |   69 +-
 .../Transforms/SparseTensorCodegen.cpp        |   45 +-
 .../Transforms/SparseTensorConversion.cpp     |  345 +-
 .../Transforms/SparseTensorPasses.cpp         |   30 +-
 .../Transforms/SparseTensorRewriting.cpp      |  535 +-
 .../SparsificationAndBufferizationPass.cpp    |   18 +-
 .../Transforms/StageSparseOperations.cpp      |   33 +-
 .../Dialect/Tosa/IR/TosaCanonicalizations.cpp |    5 +-
 mlir/lib/Dialect/Tosa/IR/TosaOps.cpp          |   97 +-
 .../Dialect/Tosa/Transforms/TosaFolders.cpp   |   29 +-
 .../TosaLayerwiseConstantFoldPass.cpp         |   15 +-
 .../Tosa/Transforms/TosaMakeBroadcastable.cpp |   14 +-
 .../Tosa/Transforms/TosaValidation.cpp        |  207 +-
 mlir/lib/Dialect/Tosa/Utils/QuantUtils.cpp    |    8 +-
 .../Transform/IR/TransformInterfaces.cpp      |    6 +-
 .../lib/Dialect/Transform/IR/TransformOps.cpp |   16 +-
 .../Transform/Transforms/CMakeLists.txt       |    2 +
 .../Transform/Transforms/InterpreterPass.cpp  |   42 +
 .../Transforms/PreloadLibraryPass.cpp         |   39 +
 .../TransformInterpreterPassBase.cpp          |   77 +-
 .../Transforms/TransformInterpreterUtils.cpp  |  125 +-
 mlir/lib/Dialect/Utils/StructuredOpsUtils.cpp |   91 +-
 mlir/lib/Dialect/Vector/IR/CMakeLists.txt     |    2 +
 mlir/lib/Dialect/Vector/IR/VectorOps.cpp      |  106 +-
 .../Transforms/VectorTransferOpTransforms.cpp |    6 +-
 mlir/lib/ExecutionEngine/CRunnerUtils.cpp     |    3 +-
 .../SparseTensor/CMakeLists.txt               |    1 -
 .../ExecutionEngine/SparseTensor/MapRef.cpp   |   44 +-
 mlir/lib/ExecutionEngine/SparseTensor/NNZ.cpp |   79 -
 .../ExecutionEngine/SparseTensor/Storage.cpp  |   31 +-
 .../ExecutionEngine/SparseTensorRuntime.cpp   |  340 +-
 mlir/lib/IR/BuiltinAttributes.cpp             |   51 +-
 mlir/lib/Interfaces/LoopLikeInterface.cpp     |   37 +
 .../lib/Interfaces/ValueBoundsOpInterface.cpp |   27 +-
 mlir/lib/Query/CMakeLists.txt                 |   12 +
 mlir/lib/Query/Matcher/CMakeLists.txt         |   14 +
 mlir/lib/Query/Matcher/Diagnostics.cpp        |  128 +
 mlir/lib/Query/Matcher/Diagnostics.h          |   82 +
 mlir/lib/Query/Matcher/ErrorBuilder.cpp       |   25 +
 mlir/lib/Query/Matcher/Parser.cpp             |  540 ++
 mlir/lib/Query/Matcher/Parser.h               |  188 +
 mlir/lib/Query/Matcher/RegistryManager.cpp    |  139 +
 mlir/lib/Query/Matcher/RegistryManager.h      |   70 +
 mlir/lib/Query/Matcher/VariantValue.cpp       |  132 +
 mlir/lib/Query/Query.cpp                      |   82 +
 mlir/lib/Query/QueryParser.cpp                |  217 +
 mlir/lib/Query/QueryParser.h                  |   59 +
 mlir/lib/TableGen/Constraint.cpp              |    5 +-
 mlir/lib/Target/LLVMIR/CMakeLists.txt         |    1 +
 mlir/lib/Target/LLVMIR/DebugImporter.cpp      |   10 +-
 .../LLVMIR/Dialect/OpenACC/CMakeLists.txt     |    1 +
 .../OpenACC/OpenACCToLLVMIRTranslation.cpp    |    4 +-
 .../LLVMIR/Dialect/OpenMP/CMakeLists.txt      |    1 +
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      |   29 +-
 mlir/lib/Target/LLVMIR/ModuleImport.cpp       |    2 +-
 mlir/lib/Target/LLVMIR/ModuleTranslation.cpp  |   21 +-
 mlir/lib/Tools/CMakeLists.txt                 |    1 +
 mlir/lib/Tools/mlir-query/CMakeLists.txt      |   13 +
 mlir/lib/Tools/mlir-query/MlirQueryMain.cpp   |  115 +
 mlir/lib/Transforms/Mem2Reg.cpp               |   50 +-
 mlir/lib/Transforms/Utils/RegionUtils.cpp     |   16 +
 mlir/python/CMakeLists.txt                    |   27 +-
 mlir/python/mlir/_mlir_libs/_mlir/ir.pyi      |    2 +
 mlir/python/mlir/dialects/AffineOps.td        |   14 +
 mlir/python/mlir/dialects/_arith_ops_ext.py   |   69 -
 .../mlir/dialects/_bufferization_ops_ext.py   |   41 -
 .../_bufferization_transform_ops_ext.py       |  128 -
 mlir/python/mlir/dialects/_builtin_ops_ext.py |   20 -
 mlir/python/mlir/dialects/_func_ops_ext.py    |  319 -
 .../mlir/dialects/_gpu_transform_ops_ext.py   |  124 -
 mlir/python/mlir/dialects/_linalg_ops_ext.py  |   47 -
 .../mlir/dialects/_loop_transform_ops_ext.py  |  134 -
 mlir/python/mlir/dialects/_memref_ops_ext.py  |   36 -
 .../dialects/_memref_transform_ops_ext.py     |  114 -
 .../mlir/dialects/_ml_program_ops_ext.py      |  113 -
 mlir/python/mlir/dialects/_ods_common.py      |   59 -
 mlir/python/mlir/dialects/_pdl_ops_ext.py     |  271 -
 mlir/python/mlir/dialects/_scf_ops_ext.py     |  107 -
 .../dialects/_structured_transform_ops_ext.py |  759 --
 mlir/python/mlir/dialects/_tensor_ops_ext.py  |   44 -
 .../dialects/_tensor_transform_ops_ext.py     |   64 -
 .../mlir/dialects/_transform_ops_ext.py       |  176 -
 .../_transform_pdl_extension_ops_ext.py       |   55 -
 mlir/python/mlir/dialects/affine.py           |    5 +
 mlir/python/mlir/dialects/arith.py            |   71 +
 mlir/python/mlir/dialects/builtin.py          |   20 +
 mlir/python/mlir/dialects/func.py             |  320 +
 mlir/python/mlir/dialects/linalg/__init__.py  |   48 +
 .../dialects/linalg/opdsl/lang/emitter.py     |    2 +-
 .../linalg/opdsl/ops/core_named_ops.py        |  107 +-
 mlir/python/mlir/dialects/ml_program.py       |  114 +
 mlir/python/mlir/dialects/pdl.py              |  216 +
 mlir/python/mlir/dialects/python_test.py      |    7 +-
 mlir/python/mlir/dialects/scf.py              |   98 +-
 mlir/python/mlir/dialects/tensor.py           |   37 +
 .../mlir/dialects/transform/__init__.py       |  170 +
 .../mlir/dialects/transform/bufferization.py  |  129 +
 mlir/python/mlir/dialects/transform/gpu.py    |  125 +
 mlir/python/mlir/dialects/transform/loop.py   |  140 +
 mlir/python/mlir/dialects/transform/memref.py |  115 +
 mlir/python/mlir/dialects/transform/pdl.py    |   50 +
 .../mlir/dialects/transform/structured.py     |  773 ++
 mlir/python/mlir/dialects/transform/tensor.py |   64 +
 mlir/python/mlir/runtime/np_to_memref.py      |    8 +-
 .../DataFlow/test-dead-code-analysis.mlir     |    7 +
 mlir/test/CAPI/ir.c                           |   48 +
 mlir/test/CAPI/sparse_tensor.c                |    5 +-
 mlir/test/CMakeLists.txt                      |    1 +
 .../ArmSMEToSCF/arm-sme-to-scf.mlir           |    8 +-
 .../Conversion/GPUToNVVM/gpu-to-nvvm.mlir     |   28 +-
 .../IndexToSPRIV/index-to-spirv.mlir          |  222 +
 .../Conversion/NVGPUToNVVM/nvgpu-to-nvvm.mlir |  270 +-
 .../Conversion/NVVMToLLVM/nvvm-to-llvm.mlir   |  125 +-
 .../OpenMPToLLVM/convert-to-llvmir.mlir       |    4 +-
 .../Conversion/SCFToSPIRV/use-indices.mlir    |   28 +
 mlir/test/Dialect/Affine/invalid.mlir         |    2 +-
 .../value-bounds-op-interface-impl.mlir       |   32 +
 mlir/test/Dialect/Arith/canonicalize.mlir     |   36 +
 mlir/test/Dialect/ArmSME/arm-sme-to-llvm.mlir |   36 +-
 mlir/test/Dialect/ArmSME/roundtrip.mlir       |  152 +-
 .../Dialect/ArmSME/vector-ops-to-sme.mlir     |   36 +-
 mlir/test/Dialect/ArmSVE/invalid.mlir         |   51 +
 .../Dialect/ArmSVE/legalize-for-llvm.mlir     |   73 +-
 mlir/test/Dialect/ArmSVE/roundtrip.mlir       |   83 +-
 .../dealloc-memoryeffect-interface.mlir       |   32 +
 ...ot-bufferize-empty-tensor-elimination.mlir |   24 +
 mlir/test/Dialect/Bufferization/invalid.mlir  |   16 +-
 mlir/test/Dialect/GPU/promotion.mlir          |    1 +
 ...=> lower-to-llvm-e2e-with-target-tag.mlir} |    0
 ...lvm-e2e-with-top-level-named-sequence.mlir |   64 +
 mlir/test/Dialect/LLVMIR/invalid.mlir         |    2 +-
 mlir/test/Dialect/LLVMIR/mem2reg.mlir         |   13 +
 mlir/test/Dialect/Linalg/collapse-dim.mlir    |   37 +
 .../Linalg/convert-conv2d-to-img2col.mlir     |   70 +
 mlir/test/Dialect/Linalg/hoisting.mlir        |  132 +
 mlir/test/Dialect/Linalg/invalid.mlir         |    2 +-
 .../Dialect/Linalg/match-ops-interpreter.mlir |   13 +
 mlir/test/Dialect/Linalg/promote.mlir         |    1 -
 .../Dialect/MemRef/fold-memref-alias-ops.mlir |   16 +
 mlir/test/Dialect/NVGPU/invalid.mlir          |   22 +-
 mlir/test/Dialect/OpenACC/invalid.mlir        |   12 -
 mlir/test/Dialect/OpenMP/ops.mlir             |   10 +-
 mlir/test/Dialect/SCF/invalid.mlir            |   30 +-
 mlir/test/Dialect/SCF/loop-unroll.mlir        |   30 +
 .../SPIRV/Transforms/webgpu-prepare.mlir      |   37 +
 .../SparseTensor/codegen_sparse_dealloc.mlir  |   26 -
 .../test/Dialect/SparseTensor/conversion.mlir |   32 +-
 .../SparseTensor/convert_dense2sparse.mlir    |  327 +-
 .../SparseTensor/convert_sparse2dense.mlir    |  334 +-
 .../SparseTensor/convert_sparse2sparse.mlir   |  177 +-
 mlir/test/Dialect/SparseTensor/fold.mlir      |   31 +
 mlir/test/Dialect/SparseTensor/invalid.mlir   |   56 +
 .../SparseTensor/invalid_encoding.mlir        |   23 +-
 mlir/test/Dialect/SparseTensor/roundtrip.mlir |   36 +
 .../SparseTensor/roundtrip_encoding.mlir      |   52 +
 .../Dialect/SparseTensor/sparse_concat.mlir   |  321 +-
 .../Dialect/SparseTensor/sparse_expand.mlir   |    6 +-
 .../SparseTensor/sparse_fill_zero.mlir        |    2 +-
 mlir/test/Dialect/Tensor/tiling.mlir          |   10 +-
 mlir/test/Dialect/Tosa/canonicalize.mlir      |   12 +
 mlir/test/Dialect/Tosa/constant-op-fold.mlir  |   57 +
 mlir/test/Dialect/Tosa/invalid.mlir           |   45 +
 mlir/test/Dialect/Tosa/ops.mlir               |   53 +-
 mlir/test/Dialect/Tosa/variables.mlir         |   33 +
 .../lower-to-llvm-transform-symbol-def.mlir   |   48 +
 .../include/Library/lower-to-llvm.mlir        |   49 +
 ...nterpreter-external-concurrent-source.mlir |    0
 .../test-interpreter-external-source.mlir     |    0
 ...terpreter-external-symbol-def-invalid.mlir |    0
 .../definitions-self-contained.mlir           |    1 +
 .../definitions-with-unresolved.mlir          |    1 +
 .../Transform/interpreter-entry-point.mlir    |   17 +
 mlir/test/Dialect/Transform/interpreter.mlir  |   17 +
 .../Dialect/Transform/preload-library.mlir    |   19 +
 .../test-interpreter-external-concurrent.mlir |    2 +-
 ...ter-external-symbol-decl-and-schedule.mlir |    4 +-
 ...-interpreter-external-symbol-decl-dir.mlir |    6 +-
 ...erpreter-external-symbol-decl-invalid.mlir |    2 +-
 ...test-interpreter-external-symbol-decl.mlir |    4 +-
 .../Transform/test-interpreter-external.mlir  |    2 +-
 ...st-interpreter-multiple-top-level-ops.mlir |   26 +
 mlir/test/Dialect/Vector/invalid.mlir         |    2 +-
 .../Dialect/Vector/vector-transferop-opt.mlir |  104 +
 mlir/test/IR/invalid-builtin-attributes.mlir  |    2 +
 mlir/test/IR/operand.mlir                     |    4 +-
 mlir/test/IR/result.mlir                      |    4 +-
 .../Dialect/SparseTensor/CPU/block.mlir       |   85 +
 .../CPU/sparse_conversion_element.mlir        |    4 +-
 .../CPU/sparse_conversion_sparse2sparse.mlir  |   46 +-
 .../GPU/CUDA/sparse-sampled-matmul-lib.mlir   |   13 +-
 .../GPU/CUDA/sparse-sddmm-lib.mlir            |  189 +
 .../Dialect/SparseTensor/python/test_SDDMM.py |    2 +-
 .../Dialect/SparseTensor/python/test_SpMM.py  |    2 +-
 .../SparseTensor/python/test_output.py        |    2 +-
 .../SparseTensor/python/test_stress.py        |   10 +-
 .../Vector/CPU/ArmSME/test-load-vertical.mlir |    2 +-
 mlir/test/Integration/data/block.mtx          |   10 +
 .../TilingInterface/tile-using-scfforall.mlir |  167 +
 mlir/test/Target/LLVMIR/Import/debug-info.ll  |   11 +-
 mlir/test/Target/LLVMIR/arm-sme-invalid.mlir  |    4 +-
 mlir/test/Target/LLVMIR/openmp-teams.mlir     |  162 +
 .../Dialect/Affine/TestReifyValueBounds.cpp   |   27 +-
 .../Linalg/TestLinalgElementwiseFusion.cpp    |    2 +-
 mlir/test/lib/Dialect/SCF/TestSCFUtils.cpp    |    5 +-
 mlir/test/lib/Dialect/Test/TestOps.td         |   23 +
 .../TestTransformDialectInterpreter.cpp       |    6 +
 .../TilingInterface/TestTilingInterface.cpp   |   90 +-
 mlir/test/mlir-query/simple-test.mlir         |   16 +
 mlir/test/mlir-tblgen/op-python-bindings.td   |    9 +
 mlir/test/mlir-tblgen/pattern.mlir            |   13 +
 mlir/test/mlir-tblgen/utils.td                |   23 +
 .../iaddcarry_extended.mlir                   |   68 +
 mlir/test/python/dialects/affine.py           |   44 +
 mlir/test/python/dialects/arith_dialect.py    |   13 +
 mlir/test/python/dialects/func.py             |    4 +
 mlir/test/python/dialects/linalg/ops.py       |   79 +
 .../python/dialects/sparse_tensor/dialect.py  |   14 +-
 .../dialects/transform_structured_ext.py      |    6 +-
 .../python/dialects/transform_vector_ext.py   |    8 +-
 mlir/test/python/ir/insertion_point.py        |    8 +
 mlir/test/python/pass_manager.py              |   99 +-
 mlir/tools/CMakeLists.txt                     |    1 +
 mlir/tools/mlir-query/CMakeLists.txt          |   20 +
 mlir/tools/mlir-query/mlir-query.cpp          |   63 +
 mlir/tools/mlir-tblgen/OpDocGen.cpp           |   13 +-
 mlir/tools/mlir-tblgen/OpFormatGen.cpp        |   26 +-
 mlir/tools/mlir-tblgen/OpPythonBindingGen.cpp |   58 +-
 mlir/tools/mlir-tblgen/RewriterGen.cpp        |   21 +
 .../Analysis/Presburger/MatrixTest.cpp        |   74 +-
 mlir/unittests/Analysis/Presburger/Utils.h    |   28 +-
 mlir/unittests/Dialect/CMakeLists.txt         |    1 +
 mlir/unittests/Dialect/SCF/CMakeLists.txt     |    8 +
 .../Dialect/SCF/LoopLikeSCFOpsTest.cpp        |  102 +
 .../Dialect/Utils/StructuredOpsUtilsTest.cpp  |  130 +
 mlir/unittests/IR/ShapedTypeTest.cpp          |   95 +
 .../Target/LLVM/SerializeToLLVMBitcode.cpp    |    2 +-
 openmp/docs/design/Runtimes.rst               |    1 +
 openmp/libomptarget/DeviceRTL/CMakeLists.txt  |    2 +
 .../DeviceRTL/include/Allocator.h             |   44 +
 .../DeviceRTL/include/Configuration.h         |    9 +-
 openmp/libomptarget/DeviceRTL/include/Debug.h |    2 +-
 openmp/libomptarget/DeviceRTL/include/LibC.h  |    1 +
 openmp/libomptarget/DeviceRTL/include/State.h |   26 +-
 .../DeviceRTL/include/Synchronization.h       |    2 +-
 openmp/libomptarget/DeviceRTL/include/Utils.h |    2 +-
 .../libomptarget/DeviceRTL/src/Allocator.cpp  |   80 +
 .../DeviceRTL/src/Configuration.cpp           |   12 +-
 openmp/libomptarget/DeviceRTL/src/Kernel.cpp  |    2 +
 openmp/libomptarget/DeviceRTL/src/LibC.cpp    |    6 +
 openmp/libomptarget/DeviceRTL/src/Mapping.cpp |    8 +-
 .../DeviceRTL/src/Parallelism.cpp             |   14 +-
 openmp/libomptarget/DeviceRTL/src/State.cpp   |   71 +-
 .../DeviceRTL/src/Synchronization.cpp         |    7 +-
 openmp/libomptarget/DeviceRTL/src/Utils.cpp   |    2 +-
 openmp/libomptarget/DeviceRTL/src/exports     |    2 +
 openmp/libomptarget/include/Environment.h     |   30 +-
 openmp/libomptarget/include/Utilities.h       |    1 -
 openmp/libomptarget/include/omptarget.h       |    1 +
 .../plugins-nextgen/amdgpu/src/rtl.cpp        |   22 +-
 .../PluginInterface/PluginInterface.cpp       |   90 +-
 .../common/PluginInterface/PluginInterface.h  |   11 +
 .../plugins-nextgen/cuda/src/rtl.cpp          |    5 +
 .../generic-elf-64bit/src/rtl.cpp             |    3 +-
 openmp/libomptarget/src/api.cpp               |  132 +-
 openmp/libomptarget/src/exports               |    2 +
 openmp/libomptarget/src/private.h             |   11 +
 .../libomptarget/test/api/omp_target_memset.c |   45 +
 openmp/libomptarget/test/libc/assert.c        |   21 +
 .../test/mapping/lambda_mapping.cpp           |    8 +-
 openmp/libomptarget/test/offloading/malloc.c  |   37 +
 .../test/offloading/malloc_parallel.c         |   42 +
 .../offloading/target_critical_region.cpp     |    1 +
 .../test/offloading/thread_state_1.c          |   38 +
 .../test/offloading/thread_state_2.c          |   40 +
 openmp/runtime/src/dllexports                 |    2 +
 openmp/runtime/src/include/omp.h.var          |    5 +
 openmp/runtime/src/include/omp_lib.f90.var    |   22 +
 openmp/runtime/src/include/omp_lib.h.var      |   22 +
 openmp/runtime/src/kmp_ftn_os.h               |    2 +
 polly/lib/Analysis/ScopInfo.cpp               |   10 +-
 runtimes/cmake/android/Arch-arm.cmake         |    7 +
 runtimes/cmake/android/Arch-arm64.cmake       |    7 +
 runtimes/cmake/android/Arch-x86.cmake         |    7 +
 runtimes/cmake/android/Arch-x86_64.cmake      |    7 +
 runtimes/cmake/android/Common.cmake           |    6 +
 .../llvm-project-overlay/clang/BUILD.bazel    |    1 +
 .../llvm-project-overlay/libc/BUILD.bazel     |    6 +-
 .../libc/libc_build_rules.bzl                 |    2 +-
 .../libc/libc_namespace.bzl                   |    6 +
 .../llvm-project-overlay/llvm/BUILD.bazel     |   28 +-
 .../llvm-project-overlay/mlir/BUILD.bazel     |  260 +-
 .../mlir/python/BUILD.bazel                   |   61 +-
 .../mlir/test/BUILD.bazel                     |    2 +-
 .../mlir/test/Dialect/BUILD.bazel             |   12 +-
 .../mlir/unittests/BUILD.bazel                |    3 +
 utils/bazel/llvm_configs/llvm-config.h.cmake  |    2 +-
 3483 files changed, 149831 insertions(+), 50035 deletions(-)
 create mode 100644 .github/workflows/docs.yml
 create mode 100644 .github/workflows/libcxx-check-generated-files.yml
 create mode 100644 bolt/test/AArch64/bf_min_alignment.s
 create mode 100644 bolt/test/RISCV/Inputs/cie-gnu.yaml
 create mode 100644 bolt/test/RISCV/Inputs/mapping-syms-isa.yaml
 create mode 100644 bolt/test/RISCV/cie-gnu.test
 create mode 100644 bolt/test/RISCV/mapping-syms-isa.test
 create mode 100644 bolt/test/RISCV/unnamed-sym-no-entry.c
 rename bolt/test/{ => X86}/checkvma-large-section.test (88%)
 create mode 100644 clang-tools-extra/clang-tidy/bugprone/CastingThroughVoidCheck.cpp
 create mode 100644 clang-tools-extra/clang-tidy/bugprone/CastingThroughVoidCheck.h
 create mode 100644 clang-tools-extra/clang-tidy/misc/CoroutineHostileRAIICheck.cpp
 create mode 100644 clang-tools-extra/clang-tidy/misc/CoroutineHostileRAIICheck.h
 create mode 100644 clang-tools-extra/docs/clang-tidy/checks/bugprone/casting-through-void.rst
 create mode 100644 clang-tools-extra/docs/clang-tidy/checks/misc/coroutine-hostile-raii.rst
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/bugprone/casting-through-void.cpp
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/misc/coroutine-hostile-raii.cpp
 create mode 100644 clang/lib/Headers/stdckdint.h
 create mode 100644 clang/lib/Headers/usermsrintrin.h
 create mode 100644 clang/test/AST/Interp/cxx23.cpp
 create mode 100644 clang/test/Analysis/Checkers/WebKit/call-args-dynamic-downcast.cpp
 create mode 100644 clang/test/C/C2x/n2683.c
 create mode 100644 clang/test/C/C2x/n2683_2.c
 create mode 100644 clang/test/CodeGen/RISCV/riscv-inline-asm-clobber.c
 create mode 100644 clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmacc_2x8x2.c
 create mode 100644 clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmacc_4x8x4.c
 create mode 100644 clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccsu_2x8x2.c
 create mode 100644 clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccsu_4x8x4.c
 create mode 100644 clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccu_2x8x2.c
 create mode 100644 clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccu_4x8x4.c
 create mode 100644 clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccus_2x8x2.c
 create mode 100644 clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccus_4x8x4.c
 create mode 100644 clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmacc_2x8x2.c
 create mode 100644 clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmacc_4x8x4.c
 create mode 100644 clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccsu_2x8x2.c
 create mode 100644 clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccsu_4x8x4.c
 create mode 100644 clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccu_2x8x2.c
 create mode 100644 clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccu_4x8x4.c
 create mode 100644 clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccus_2x8x2.c
 create mode 100644 clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccus_4x8x4.c
 create mode 100644 clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmacc_2x8x2.c
 create mode 100644 clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmacc_4x8x4.c
 create mode 100644 clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccsu_2x8x2.c
 create mode 100644 clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccsu_4x8x4.c
 create mode 100644 clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccu_2x8x2.c
 create mode 100644 clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccu_4x8x4.c
 create mode 100644 clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccus_2x8x2.c
 create mode 100644 clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccus_4x8x4.c
 create mode 100644 clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmacc_2x8x2.c
 create mode 100644 clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmacc_4x8x4.c
 create mode 100644 clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccsu_2x8x2.c
 create mode 100644 clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccsu_4x8x4.c
 create mode 100644 clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccu_2x8x2.c
 create mode 100644 clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccu_4x8x4.c
 create mode 100644 clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccus_2x8x2.c
 create mode 100644 clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccus_4x8x4.c
 create mode 100644 clang/test/CodeGen/X86/usermsr-builtins-error-32.c
 create mode 100644 clang/test/CodeGen/X86/usermsr-builtins.c
 create mode 100644 clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_revd.c
 create mode 100644 clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmlsl.c
 create mode 100644 clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_cntp.c
 create mode 100644 clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_dot.c
 create mode 100644 clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_ld1.c
 create mode 100644 clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_ldnt1.c
 create mode 100644 clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pext.c
 create mode 100644 clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pfalse.c
 create mode 100644 clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_psel.c
 create mode 100644 clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_st1.c
 create mode 100644 clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_stnt1.c
 create mode 100644 clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_while_pn.c
 create mode 100644 clang/test/CodeGen/attr-counted-by.c
 create mode 100644 clang/test/CodeGen/builtin_vectorelements.c
 create mode 100644 clang/test/CodeGenCXX/m68k-rtdcall.cpp
 create mode 100644 clang/test/CodeGenHipStdPar/unannotated-functions-get-emitted.cpp
 create mode 100644 clang/test/CodeGenHipStdPar/unsupported-ASM.cpp
 create mode 100644 clang/test/CodeGenHipStdPar/unsupported-builtins.cpp
 create mode 100644 clang/test/Driver/Inputs/basic_avr_tree/usr/lib/avr/lib/ldscripts/avrtiny.x
 create mode 100644 clang/test/Driver/Inputs/basic_avr_tree/usr/lib/avr/lib/ldscripts/avrxmega6.x
 create mode 100644 clang/test/Driver/cl-delayed-template-parsing-cxx20.cpp
 create mode 100644 clang/test/Driver/cl-offload.cu
 create mode 100644 clang/test/Headers/stdckdint.c
 create mode 100644 clang/test/Index/USR/decl-context.cpp
 create mode 100644 clang/test/Modules/Inputs/all-product-headers.yaml
 create mode 100644 clang/test/Modules/modulemap-collision.m
 create mode 100644 clang/test/Modules/no-undeclared-includes.c
 create mode 100644 clang/test/Sema/aarch64-sve2p1-intrinsics/acle_sve2p1_imm.cpp
 create mode 100644 clang/test/Sema/attr-counted-by.c
 create mode 100644 clang/test/Sema/builtin_vectorelements.c
 create mode 100644 clang/test/Sema/m68k-rtdcall.c
 create mode 100644 clang/test/SemaCXX/Inputs/enum-constexpr-conversion-system-header.h
 create mode 100644 clang/test/SemaCXX/bitfield-width.c
 create mode 100644 clang/test/SemaCXX/builtin_vectorelements.cpp
 create mode 100644 clang/test/SemaCXX/m68k-rtdcall.cpp
 create mode 100644 clang/test/SemaCXX/vla-ext-diag.cpp
 create mode 100644 clang/test/SemaCXX/warn-unsafe-buffer-usage-debug-unclaimed/lit.local.cfg
 create mode 100644 clang/test/SemaCXX/warn-unsafe-buffer-usage-debug-unclaimed/warn-unsafe-buffer-usage-debug-unclaimed.cpp
 create mode 100644 clang/test/SemaTemplate/function-decl-nested-type-alias.cpp
 create mode 100644 clang/tools/clang-format/git-clang-format.bat
 create mode 100644 clang/unittests/AST/Interp/CMakeLists.txt
 create mode 100644 clang/unittests/AST/Interp/Descriptor.cpp
 create mode 100644 clang/unittests/Driver/GCCVersionTest.cpp
 create mode 100644 clang/utils/analyze_safe_buffer_debug_notes.py
 create mode 100644 compiler-rt/lib/builtins/aarch64/sme-abi-init.c
 create mode 100644 compiler-rt/lib/builtins/aarch64/sme-abi.S
 create mode 100644 compiler-rt/lib/builtins/extendxftf2.c
 create mode 100644 compiler-rt/lib/builtins/int_to_fp.h
 create mode 100644 compiler-rt/lib/builtins/int_to_fp_impl.inc
 create mode 100644 compiler-rt/lib/builtins/trunctfxf2.c
 create mode 100644 compiler-rt/lib/scudo/standalone/condition_variable.h
 create mode 100644 compiler-rt/lib/scudo/standalone/condition_variable_base.h
 create mode 100644 compiler-rt/lib/scudo/standalone/condition_variable_linux.cpp
 create mode 100644 compiler-rt/lib/scudo/standalone/condition_variable_linux.h
 create mode 100644 compiler-rt/lib/scudo/standalone/report_linux.cpp
 create mode 100644 compiler-rt/lib/scudo/standalone/report_linux.h
 create mode 100644 compiler-rt/lib/scudo/standalone/tests/condition_variable_test.cpp
 create mode 100644 compiler-rt/lib/tsan/rtl/tsan_rtl_riscv64.S
 create mode 100644 compiler-rt/test/builtins/Unit/extendxftf2_test.c
 create mode 100644 compiler-rt/test/builtins/Unit/trunctfxf2_test.c
 create mode 100644 compiler-rt/test/hwasan/TestCases/bcmp.cpp
 create mode 100644 compiler-rt/test/orc/TestCases/Linux/ppc64/Inputs/trivial-tls-main.cpp
 create mode 100644 compiler-rt/test/orc/TestCases/Linux/ppc64/Inputs/trivial-tls-pwr10.cpp
 create mode 100644 compiler-rt/test/orc/TestCases/Linux/ppc64/trivial-tls-pwr10.test
 create mode 100644 flang/include/flang/Optimizer/Analysis/TBAAForest.h
 create mode 100644 flang/include/flang/Optimizer/Dialect/FirAliasTagOpInterface.h
 create mode 100644 flang/include/flang/Optimizer/Dialect/FirAliasTagOpInterface.td
 create mode 100644 flang/lib/Optimizer/Analysis/TBAAForest.cpp
 create mode 100644 flang/lib/Optimizer/Dialect/FirAliasTagOpInterface.cpp
 create mode 100644 flang/lib/Optimizer/Transforms/AddAliasTags.cpp
 create mode 100644 flang/lib/Semantics/rewrite-directives.cpp
 create mode 100644 flang/lib/Semantics/rewrite-directives.h
 create mode 100644 flang/test/Driver/falias-analysis.f90
 create mode 100644 flang/test/Fir/tbaa-codegen.fir
 create mode 100644 flang/test/Fir/tbaa-codegen2.fir
 create mode 100644 flang/test/Lower/HLFIR/convert-variable-block.f90
 create mode 100644 flang/test/Lower/HLFIR/intentout-allocatable-components.f90
 create mode 100644 flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
 create mode 100644 flang/test/Lower/OpenACC/acc-bounds.f90
 create mode 100644 flang/test/Lower/OpenMP/atomic-read.f90
 create mode 100644 flang/test/Lower/OpenMP/atomic-write.f90
 create mode 100644 flang/test/Lower/OpenMP/firstprivate-commonblock.f90
 create mode 100644 flang/test/Lower/OpenMP/parallel-private-clause-fixes.f90
 create mode 100644 flang/test/Lower/OpenMP/parallel-reduction-add.f90
 create mode 100644 flang/test/Lower/OpenMP/parallel-sections.f90
 create mode 100644 flang/test/Lower/OpenMP/parallel-wsloop-firstpriv.f90
 create mode 100644 flang/test/Lower/OpenMP/unstructured.f90
 create mode 100644 flang/test/Lower/OpenMP/wsloop.f90
 create mode 100644 flang/test/Semantics/OpenACC/acc-error.f90
 create mode 100644 flang/test/Semantics/OpenMP/requires-atomic01.f90
 create mode 100644 flang/test/Semantics/OpenMP/requires-atomic02.f90
 create mode 100644 flang/test/Semantics/OpenMP/threadprivate06.f90
 create mode 100644 flang/test/Semantics/dosemantics13.f90
 create mode 100644 flang/test/Semantics/dosemantics14.f90
 create mode 100644 flang/test/Semantics/separate-mp05.f90
 create mode 100644 flang/test/Transforms/tbaa.fir
 create mode 100644 flang/test/Transforms/tbaa2.fir
 create mode 100644 libc/include/features.h.def
 create mode 100644 libc/include/llvm-libc-macros/features-macros.h
 create mode 100644 libc/src/math/acos.h
 create mode 100644 libc/src/math/acosh.h
 create mode 100644 libc/src/math/asin.h
 create mode 100644 libc/src/math/asinh.h
 create mode 100644 libc/src/math/atan.h
 create mode 100644 libc/src/math/atan2.h
 create mode 100644 libc/src/math/atan2f.h
 create mode 100644 libc/src/math/atanh.h
 create mode 100644 libc/src/math/erf.h
 rename libc/src/math/gpu/{vendor => }/llround.cpp (80%)
 rename libc/src/math/gpu/{vendor => }/llroundf.cpp (80%)
 rename libc/src/math/gpu/{sinhf.cpp => lround.cpp} (70%)
 rename libc/src/math/gpu/{tanf.cpp => lroundf.cpp} (69%)
 create mode 100644 libc/src/math/gpu/vendor/acos.cpp
 create mode 100644 libc/src/math/gpu/vendor/acosh.cpp
 create mode 100644 libc/src/math/gpu/vendor/asin.cpp
 create mode 100644 libc/src/math/gpu/vendor/asinh.cpp
 create mode 100644 libc/src/math/gpu/vendor/atan.cpp
 create mode 100644 libc/src/math/gpu/vendor/atan2.cpp
 create mode 100644 libc/src/math/gpu/vendor/atan2f.cpp
 create mode 100644 libc/src/math/gpu/vendor/atanh.cpp
 rename libc/src/math/gpu/{tanhf.cpp => vendor/erf.cpp} (68%)
 create mode 100644 libc/src/math/gpu/vendor/erff.cpp
 create mode 100644 libc/src/math/gpu/vendor/exp.cpp
 create mode 100644 libc/src/math/gpu/vendor/exp10.cpp
 create mode 100644 libc/src/math/gpu/vendor/exp2.cpp
 create mode 100644 libc/src/math/gpu/vendor/expm1.cpp
 create mode 100644 libc/src/math/gpu/vendor/log.cpp
 create mode 100644 libc/src/math/gpu/vendor/log10.cpp
 create mode 100644 libc/src/math/gpu/vendor/log10f.cpp
 create mode 100644 libc/src/math/gpu/vendor/log1p.cpp
 create mode 100644 libc/src/math/gpu/vendor/log1pf.cpp
 create mode 100644 libc/src/math/gpu/vendor/log2.cpp
 create mode 100644 libc/src/math/gpu/vendor/log2f.cpp
 create mode 100644 libc/src/math/gpu/vendor/logb.cpp
 create mode 100644 libc/src/math/gpu/vendor/logbf.cpp
 create mode 100644 libc/src/math/gpu/vendor/logf.cpp
 create mode 100644 libc/src/math/gpu/vendor/lrint.cpp
 create mode 100644 libc/src/math/gpu/vendor/lrintf.cpp
 create mode 100644 libc/src/math/gpu/vendor/tgamma.cpp
 create mode 100644 libc/src/math/gpu/vendor/tgammaf.cpp
 create mode 100644 libc/src/math/sincos.h
 create mode 100644 libc/src/math/tgamma.h
 create mode 100644 libc/src/math/tgammaf.h
 rename libc/src/stdio/{ => generic}/ungetc.cpp (100%)
 create mode 100644 libc/src/stdio/gpu/ungetc.cpp
 create mode 100644 libcxx/benchmarks/stop_token.bench.cpp
 create mode 100644 libcxx/cmake/caches/AndroidNDK.cmake
 create mode 100644 libcxx/include/__algorithm/pstl_move.h
 create mode 100644 libcxx/include/__mdspan/layout_stride.h
 delete mode 100644 libcxx/include/setjmp.h
 create mode 100644 libcxx/test/configs/llvm-libc++-android-ndk.cfg.in
 create mode 100644 libcxx/test/libcxx/containers/views/mdspan/layout_left/assert.ctor.layout_stride.pass.cpp
 create mode 100644 libcxx/test/libcxx/containers/views/mdspan/layout_right/assert.ctor.layout_stride.pass.cpp
 create mode 100644 libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.conversion.pass.cpp
 create mode 100644 libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_array.non_unique.pass.cpp
 create mode 100644 libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_array.pass.cpp
 create mode 100644 libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_span.non_unique.pass.cpp
 create mode 100644 libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_span.pass.cpp
 create mode 100644 libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.index_operator.pass.cpp
 create mode 100644 libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.stride.pass.cpp
 create mode 100644 libcxx/test/libcxx/strings/basic.string/string.capacity/allocation_size.pass.cpp
 create mode 100644 libcxx/test/std/algorithms/alg.modifying.operations/alg.move/pstl.exception_handling.pass.cpp
 create mode 100644 libcxx/test/std/algorithms/alg.modifying.operations/alg.move/pstl.move.pass.cpp
 rename libcxx/test/std/containers/views/mdspan/{mdspan => }/CustomTestLayouts.h (87%)
 create mode 100644 libcxx/test/std/containers/views/mdspan/layout_left/ctor.layout_stride.pass.cpp
 create mode 100644 libcxx/test/std/containers/views/mdspan/layout_right/ctor.layout_stride.pass.cpp
 create mode 100644 libcxx/test/std/containers/views/mdspan/layout_stride/comparison.pass.cpp
 create mode 100644 libcxx/test/std/containers/views/mdspan/layout_stride/ctor.default.pass.cpp
 create mode 100644 libcxx/test/std/containers/views/mdspan/layout_stride/ctor.extents_array.pass.cpp
 create mode 100644 libcxx/test/std/containers/views/mdspan/layout_stride/ctor.extents_span.pass.cpp
 create mode 100644 libcxx/test/std/containers/views/mdspan/layout_stride/ctor.strided_mapping.pass.cpp
 create mode 100644 libcxx/test/std/containers/views/mdspan/layout_stride/deduction.pass.cpp
 create mode 100644 libcxx/test/std/containers/views/mdspan/layout_stride/extents.verify.cpp
 create mode 100644 libcxx/test/std/containers/views/mdspan/layout_stride/index_operator.pass.cpp
 create mode 100644 libcxx/test/std/containers/views/mdspan/layout_stride/is_exhaustive_corner_case.pass.cpp
 create mode 100644 libcxx/test/std/containers/views/mdspan/layout_stride/properties.pass.cpp
 create mode 100644 libcxx/test/std/containers/views/mdspan/layout_stride/required_span_size.pass.cpp
 create mode 100644 libcxx/test/std/containers/views/mdspan/layout_stride/static_requirements.pass.cpp
 create mode 100644 libcxx/test/std/containers/views/mdspan/layout_stride/stride.pass.cpp
 rename libcxx/test/std/depr/{depr.numeric.imits.has.denorm => depr.numeric.limits.has.denorm}/deprecated.verify.cpp (100%)
 create mode 100644 libcxx/test/std/experimental/simd/simd.class/simd_ctor_generator.pass.cpp
 create mode 100644 libcxx/test/std/input.output/string.streams/istringstream/istringstream.members/str.allocator_propagation.pass.cpp
 create mode 100644 libcxx/test/std/input.output/string.streams/ostringstream/ostringstream.members/str.allocator_propagation.pass.cpp
 create mode 100644 libcxx/test/std/input.output/string.streams/stringstream/stringstream.members/str.allocator_propagation.pass.cpp
 create mode 100644 libcxx/test/std/utilities/memory/specialized.algorithms/overload_compare_iterator.h
 create mode 100755 libcxx/utils/adb_run.py
 create mode 100644 libcxx/utils/ci/vendor/android/Dockerfile
 create mode 100644 libcxx/utils/ci/vendor/android/Dockerfile.emulator
 create mode 100755 libcxx/utils/ci/vendor/android/build-emulator-images.sh
 create mode 100755 libcxx/utils/ci/vendor/android/container-setup.sh
 create mode 100755 libcxx/utils/ci/vendor/android/emulator-entrypoint.sh
 create mode 100644 libcxx/utils/ci/vendor/android/emulator-functions.sh
 create mode 100755 libcxx/utils/ci/vendor/android/emulator-wait-for-ready.sh
 create mode 100755 libcxx/utils/ci/vendor/android/run-buildbot-container
 create mode 100644 libcxx/utils/ci/vendor/android/setup-env-for-emulator.sh
 create mode 100755 libcxx/utils/ci/vendor/android/start-emulator.sh
 create mode 100755 libcxx/utils/ci/vendor/android/stop-emulator.sh
 create mode 100644 libcxx/utils/libcxx/test/android.py
 create mode 100644 libcxxabi/test/configs/llvm-libc++abi-android-ndk.cfg.in
 create mode 100644 libunwind/test/aix_signal_unwind.pass.sh.S
 create mode 100644 lld/test/COFF/Inputs/loadconfig-arm64ec.s
 create mode 100644 lld/test/COFF/arm64ec-codemap.test
 create mode 100644 lld/test/COFF/arm64ec-reloc.test
 create mode 100644 lldb/docs/use/aarch64-linux.rst
 rename lldb/packages/Python/lldbsuite/test/tools/{lldb-vscode/vscode.py => lldb-dap/dap_server.py} (98%)
 rename lldb/packages/Python/lldbsuite/test/tools/{lldb-vscode/lldbvscode_testcase.py => lldb-dap/lldbdap_testcase.py} (82%)
 create mode 100644 lldb/test/API/commands/target/dump-separate-debug-info/dwo/Makefile
 create mode 100644 lldb/test/API/commands/target/dump-separate-debug-info/dwo/TestDumpDwo.py
 create mode 100644 lldb/test/API/commands/target/dump-separate-debug-info/dwo/foo.cpp
 create mode 100644 lldb/test/API/commands/target/dump-separate-debug-info/dwo/foo.h
 create mode 100644 lldb/test/API/commands/target/dump-separate-debug-info/dwo/main.cpp
 rename lldb/test/API/{tools/lldb-vscode/evaluate => commands/target/dump-separate-debug-info/oso}/Makefile (100%)
 create mode 100644 lldb/test/API/commands/target/dump-separate-debug-info/oso/TestDumpOso.py
 create mode 100644 lldb/test/API/commands/target/dump-separate-debug-info/oso/foo.cpp
 create mode 100644 lldb/test/API/commands/target/dump-separate-debug-info/oso/foo.h
 create mode 100644 lldb/test/API/commands/target/dump-separate-debug-info/oso/main.cpp
 rename lldb/test/API/{tools/lldb-vscode/completions => lang/cpp/fixits}/Makefile (100%)
 create mode 100644 lldb/test/API/lang/cpp/fixits/TestCppFixIts.py
 create mode 100644 lldb/test/API/lang/cpp/fixits/main.cpp
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/attach/Makefile (100%)
 rename lldb/test/API/tools/{lldb-vscode/attach/TestVSCode_attach.py => lldb-dap/attach/TestDAP_attach.py} (96%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/attach/main.c (100%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/breakpoint-events/Makefile (100%)
 rename lldb/test/API/tools/{lldb-vscode/breakpoint-events/TestVSCode_breakpointEvents.py => lldb-dap/breakpoint-events/TestDAP_breakpointEvents.py} (83%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/breakpoint-events/foo.cpp (91%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/breakpoint-events/foo.h (100%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/breakpoint-events/main.cpp (100%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/breakpoint/Makefile (100%)
 rename lldb/test/API/tools/{lldb-vscode/breakpoint/TestVSCode_logpoints.py => lldb-dap/breakpoint/TestDAP_logpoints.py} (94%)
 rename lldb/test/API/tools/{lldb-vscode/breakpoint/TestVSCode_setBreakpoints.py => lldb-dap/breakpoint/TestDAP_setBreakpoints.py} (89%)
 rename lldb/test/API/tools/{lldb-vscode/breakpoint/TestVSCode_setExceptionBreakpoints.py => lldb-dap/breakpoint/TestDAP_setExceptionBreakpoints.py} (86%)
 rename lldb/test/API/tools/{lldb-vscode/breakpoint/TestVSCode_setFunctionBreakpoints.py => lldb-dap/breakpoint/TestDAP_setFunctionBreakpoints.py} (88%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/breakpoint/main.cpp (82%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/breakpoint/other.c (100%)
 create mode 100644 lldb/test/API/tools/lldb-dap/categories
 rename lldb/test/API/tools/{lldb-vscode/console => lldb-dap/completions}/Makefile (100%)
 rename lldb/test/API/tools/{lldb-vscode/completions/TestVSCode_completions.py => lldb-dap/completions/TestDAP_completions.py} (81%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/completions/main.cpp (81%)
 rename lldb/test/API/tools/{lldb-vscode/disconnect => lldb-dap/console}/Makefile (100%)
 rename lldb/test/API/tools/{lldb-vscode/console/TestVSCode_console.py => lldb-dap/console/TestDAP_console.py} (83%)
 rename lldb/test/API/tools/{lldb-vscode/console/TestVSCode_redirection_to_console.py => lldb-dap/console/TestDAP_redirection_to_console.py} (66%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/console/main.cpp (100%)
 rename lldb/test/API/tools/{lldb-vscode/coreFile/TestVSCode_coreFile.py => lldb-dap/coreFile/TestDAP_coreFile.py} (92%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/coreFile/linux-x86_64.core (100%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/coreFile/linux-x86_64.out (100%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/coreFile/main.c (100%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/correct-thread/Makefile (100%)
 rename lldb/test/API/tools/{lldb-vscode/correct-thread/TestVSCode_correct_thread.py => lldb-dap/correct-thread/TestDAP_correct_thread.py} (86%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/correct-thread/main.c (84%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/disassemble/Makefile (100%)
 rename lldb/test/API/tools/{lldb-vscode/disassemble/TestVSCode_disassemble.py => lldb-dap/disassemble/TestDAP_disassemble.py} (89%)
 create mode 100644 lldb/test/API/tools/lldb-dap/disassemble/main.c
 rename lldb/test/API/tools/{lldb-vscode/exception => lldb-dap/disconnect}/Makefile (100%)
 rename lldb/test/API/tools/{lldb-vscode/disconnect/TestVSCode_disconnect.py => lldb-dap/disconnect/TestDAP_disconnect.py} (87%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/disconnect/main.cpp (100%)
 create mode 100644 lldb/test/API/tools/lldb-dap/evaluate/Makefile
 rename lldb/test/API/tools/{lldb-vscode/evaluate/TestVSCode_evaluate.py => lldb-dap/evaluate/TestDAP_evaluate.py} (95%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/evaluate/foo.cpp (100%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/evaluate/foo.h (100%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/evaluate/main.cpp (94%)
 rename lldb/test/API/tools/{lldb-vscode/variables => lldb-dap/exception}/Makefile (100%)
 rename lldb/test/API/tools/{lldb-vscode/exception/TestVSCode_exception.py => lldb-dap/exception/TestDAP_exception.py} (74%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/exception/main.cpp (100%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/launch/Makefile (100%)
 rename lldb/test/API/tools/{lldb-vscode/launch/TestVSCode_launch.py => lldb-dap/launch/TestDAP_launch.py} (94%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/launch/main.c (77%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/module/Makefile (100%)
 rename lldb/test/API/tools/{lldb-vscode/module/TestVSCode_module.py => lldb-dap/module/TestDAP_module.py} (88%)
 create mode 100644 lldb/test/API/tools/lldb-dap/module/foo.cpp
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/module/foo.h (100%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/module/main.cpp (100%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/optimized/Makefile (100%)
 rename lldb/test/API/tools/{lldb-vscode/optimized/TestVSCode_optimized.py => lldb-dap/optimized/TestDAP_optimized.py} (81%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/optimized/main.cpp (100%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/restart/Makefile (100%)
 rename lldb/test/API/tools/{lldb-vscode/restart/TestVSCode_restart.py => lldb-dap/restart/TestDAP_restart.py} (79%)
 rename lldb/test/API/tools/{lldb-vscode/restart/TestVSCode_restart_runInTerminal.py => lldb-dap/restart/TestDAP_restart_runInTerminal.py} (80%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/restart/main.c (86%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/runInTerminal/Makefile (100%)
 rename lldb/test/API/tools/{lldb-vscode/runInTerminal/TestVSCode_runInTerminal.py => lldb-dap/runInTerminal/TestDAP_runInTerminal.py} (84%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/runInTerminal/main.c (100%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/stackTrace/Makefile (100%)
 rename lldb/test/API/tools/{lldb-vscode/stackTrace/TestVSCode_stackTrace.py => lldb-dap/stackTrace/TestDAP_stackTrace.py} (97%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/stackTrace/main.c (65%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/stackTraceMissingFunctionName/Makefile (100%)
 rename lldb/test/API/tools/{lldb-vscode/stackTraceMissingFunctionName/TestVSCode_stackTraceMissingFunctionName.py => lldb-dap/stackTraceMissingFunctionName/TestDAP_stackTraceMissingFunctionName.py} (77%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/stackTraceMissingFunctionName/main.cpp (100%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/startDebugging/Makefile (100%)
 rename lldb/test/API/tools/{lldb-vscode/startDebugging/TestVSCode_startDebugging.py => lldb-dap/startDebugging/TestDAP_startDebugging.py} (64%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/startDebugging/main.c (100%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/step/Makefile (100%)
 rename lldb/test/API/tools/{lldb-vscode/step/TestVSCode_step.py => lldb-dap/step/TestDAP_step.py} (94%)
 create mode 100644 lldb/test/API/tools/lldb-dap/step/main.cpp
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/stop-hooks/Makefile (100%)
 rename lldb/test/API/tools/{lldb-vscode/stop-hooks/TestVSCode_stop_hooks.py => lldb-dap/stop-hooks/TestDAP_stop_hooks.py} (82%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/stop-hooks/main.c (100%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/terminated-event/Makefile (100%)
 rename lldb/test/API/tools/{lldb-vscode/terminated-event/TestVSCode_terminatedEvent.py => lldb-dap/terminated-event/TestDAP_terminatedEvent.py} (89%)
 create mode 100644 lldb/test/API/tools/lldb-dap/terminated-event/foo.cpp
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/terminated-event/foo.h (100%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/terminated-event/main.cpp (100%)
 create mode 100644 lldb/test/API/tools/lldb-dap/variables/Makefile
 rename lldb/test/API/tools/{lldb-vscode/variables/TestVSCode_variables.py => lldb-dap/variables/TestDAP_variables.py} (89%)
 rename lldb/test/API/tools/{lldb-vscode => lldb-dap}/variables/main.cpp (90%)
 delete mode 100644 lldb/test/API/tools/lldb-vscode/categories
 delete mode 100644 lldb/test/API/tools/lldb-vscode/disassemble/main.c
 delete mode 100644 lldb/test/API/tools/lldb-vscode/module/foo.cpp
 delete mode 100644 lldb/test/API/tools/lldb-vscode/step/main.cpp
 delete mode 100644 lldb/test/API/tools/lldb-vscode/terminated-event/foo.cpp
 rename lldb/test/Shell/{VSCode => DAP}/TestOptions.test (70%)
 create mode 100644 lldb/test/Shell/SymbolFile/DWARF/Inputs/dwo-missing-error.c
 create mode 100644 lldb/test/Shell/SymbolFile/DWARF/dwo-missing-error.test
 rename lldb/tools/{lldb-vscode => lldb-dap}/BreakpointBase.cpp (97%)
 rename lldb/tools/{lldb-vscode => lldb-dap}/BreakpointBase.h (93%)
 rename lldb/tools/{lldb-vscode => lldb-dap}/CMakeLists.txt (79%)
 rename lldb/tools/{lldb-vscode/VSCode.cpp => lldb-dap/DAP.cpp} (86%)
 rename lldb/tools/{lldb-vscode/VSCode.h => lldb-dap/DAP.h} (96%)
 rename lldb/tools/{lldb-vscode/VSCodeForward.h => lldb-dap/DAPForward.h} (81%)
 rename lldb/tools/{lldb-vscode => lldb-dap}/ExceptionBreakpoint.cpp (84%)
 rename lldb/tools/{lldb-vscode => lldb-dap}/ExceptionBreakpoint.h (83%)
 rename lldb/tools/{lldb-vscode => lldb-dap}/FifoFiles.cpp (98%)
 rename lldb/tools/{lldb-vscode => lldb-dap}/FifoFiles.h (93%)
 rename lldb/tools/{lldb-vscode => lldb-dap}/FunctionBreakpoint.cpp (87%)
 rename lldb/tools/{lldb-vscode => lldb-dap}/FunctionBreakpoint.h (80%)
 rename lldb/tools/{lldb-vscode => lldb-dap}/IOStream.cpp (99%)
 rename lldb/tools/{lldb-vscode => lldb-dap}/IOStream.h (93%)
 rename lldb/tools/{lldb-vscode => lldb-dap}/JSONUtils.cpp (98%)
 rename lldb/tools/{lldb-vscode => lldb-dap}/JSONUtils.h (94%)
 rename lldb/tools/{lldb-vscode => lldb-dap}/LLDBUtils.cpp (92%)
 rename lldb/tools/{lldb-vscode => lldb-dap}/LLDBUtils.h (82%)
 rename lldb/tools/{lldb-vscode => lldb-dap}/Options.td (85%)
 rename lldb/tools/{lldb-vscode => lldb-dap}/OutputRedirector.cpp (96%)
 rename lldb/tools/{lldb-vscode => lldb-dap}/OutputRedirector.h (77%)
 rename lldb/tools/{lldb-vscode => lldb-dap}/ProgressEvent.cpp (99%)
 rename lldb/tools/{lldb-vscode => lldb-dap}/ProgressEvent.h (97%)
 rename lldb/tools/{lldb-vscode => lldb-dap}/README.md (64%)
 rename lldb/tools/{lldb-vscode => lldb-dap}/RunInTerminal.cpp (98%)
 rename lldb/tools/{lldb-vscode => lldb-dap}/RunInTerminal.h (94%)
 rename lldb/tools/{lldb-vscode => lldb-dap}/SourceBreakpoint.cpp (88%)
 rename lldb/tools/{lldb-vscode => lldb-dap}/SourceBreakpoint.h (87%)
 rename lldb/tools/{lldb-vscode/lldb-vscode-Info.plist.in => lldb-dap/lldb-dap-Info.plist.in} (88%)
 rename lldb/tools/{lldb-vscode/lldb-vscode.cpp => lldb-dap/lldb-dap.cpp} (88%)
 rename lldb/tools/{lldb-vscode => lldb-dap}/package.json (97%)
 rename lldb/tools/{lldb-vscode => lldb-dap}/syntaxes/arm.disasm (100%)
 rename lldb/tools/{lldb-vscode => lldb-dap}/syntaxes/arm64.disasm (100%)
 rename lldb/tools/{lldb-vscode => lldb-dap}/syntaxes/disassembly.json (100%)
 rename lldb/tools/{lldb-vscode => lldb-dap}/syntaxes/x86.disasm (100%)
 create mode 120000 lldb/tools/lldb-vscode
 rename llvm/include/llvm/{CodeGen => ADT}/StableHashing.h (95%)
 create mode 100644 llvm/include/llvm/Analysis/WithCache.h
 create mode 100644 llvm/include/llvm/ProfileData/FunctionId.h
 create mode 100644 llvm/include/llvm/ProfileData/HashKeyMap.h
 create mode 100644 llvm/include/llvm/Transforms/HipStdPar/HipStdPar.h
 create mode 100644 llvm/lib/Target/AArch64/AArch64PointerAuth.h
 delete mode 100644 llvm/lib/Target/X86/X86PreAMXConfig.cpp
 create mode 100644 llvm/lib/Transforms/HipStdPar/CMakeLists.txt
 create mode 100644 llvm/lib/Transforms/HipStdPar/HipStdPar.cpp
 create mode 100644 llvm/test/Analysis/BasicAA/pr69096.ll
 create mode 100644 llvm/test/Analysis/ValueTracking/constant-ranges.ll
 create mode 100644 llvm/test/Bitcode/upgrade-datalayout5.ll
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/legalize-xtn.mir
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/sink-and-fold-illegal-shift.ll
 create mode 100644 llvm/test/CodeGen/AArch64/aarch64-mulv.ll
 create mode 100644 llvm/test/CodeGen/AArch64/arm64-homogeneous-prolog-epilog-odd-csrs.ll
 delete mode 100644 llvm/test/CodeGen/AArch64/arm64-vfloatintrinsics.ll
 create mode 100644 llvm/test/CodeGen/AArch64/fmla.ll
 create mode 100644 llvm/test/CodeGen/AArch64/fptoi.ll
 create mode 100644 llvm/test/CodeGen/AArch64/implicit-def-with-impdef-greedy-assert.mir
 create mode 100644 llvm/test/CodeGen/AArch64/pr66603.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sign-return-address-tailcall.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sink-and-fold-illegal-shift.mir
 create mode 100644 llvm/test/CodeGen/AArch64/sve2p1-intrinsics-bfmls.ll
 create mode 100644 llvm/test/CodeGen/AArch64/vector-llrint.ll
 create mode 100644 llvm/test/CodeGen/AArch64/vector-lrint.ll
 create mode 100644 llvm/test/CodeGen/AArch64/xtn.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-f16.mir
 create mode 100644 llvm/test/CodeGen/AMDGPU/fold-short-64-bit-literals.mir
 create mode 100644 llvm/test/CodeGen/AMDGPU/waterfall_kills_scc.ll
 create mode 100644 llvm/test/CodeGen/LoongArch/cfr-copy.mir
 create mode 100644 llvm/test/CodeGen/LoongArch/cfr-pseudo-copy.mir
 create mode 100644 llvm/test/CodeGen/LoongArch/ir-instruction/fence-singlethread.ll
 create mode 100644 llvm/test/CodeGen/M68k/CConv/rtd-call.ll
 create mode 100644 llvm/test/CodeGen/M68k/CConv/rtd-ret.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/vector-llrint.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/vector-lrint.ll
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/frame-index-rv32.mir
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/frame-index-rv64.mir
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/int-ptr-cast-rv32.mir
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/int-ptr-cast-rv64.mir
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/zba-rv32.mir
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/zba-rv64.mir
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/calling-conv-ilp32-ilp32f-ilp32d-common.ll
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/calling-conv-lp64-lp64f-lp64d-common.ll
 rename llvm/test/CodeGen/RISCV/GlobalISel/{call-lowering => irtranslator}/variadic-call.ll (100%)
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv32/legalize-inttoptr.mir
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv32/legalize-mulo.mir
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv32/legalize-ptrtoint.mir
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv32/legalize-smax.mir
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv32/legalize-smin.mir
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv32/legalize-umax.mir
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv32/legalize-umin.mir
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv64/legalize-inttoptr.mir
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv64/legalize-mulo.mir
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv64/legalize-ptrtoint.mir
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv64/legalize-smax.mir
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv64/legalize-smin.mir
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv64/legalize-umax.mir
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv64/legalize-umin.mir
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/frame-index.mir
 create mode 100644 llvm/test/CodeGen/RISCV/aext.ll
 create mode 100644 llvm/test/CodeGen/RISCV/opt-w-instrs.mir
 create mode 100644 llvm/test/CodeGen/RISCV/pr66603.ll
 create mode 100644 llvm/test/CodeGen/RISCV/pr68855.ll
 create mode 100644 llvm/test/CodeGen/RISCV/riscv-shifted-extend.ll
 create mode 100644 llvm/test/CodeGen/RISCV/rv64-patchpoint.ll
 create mode 100644 llvm/test/CodeGen/RISCV/rv64-stackmap-frame-setup.ll
 create mode 100644 llvm/test/CodeGen/RISCV/rv64-stackmap-nops.ll
 create mode 100644 llvm/test/CodeGen/RISCV/rv64-stackmap.ll
 create mode 100644 llvm/test/CodeGen/RISCV/rvv/concat-vectors-constant-stride.ll
 create mode 100644 llvm/test/CodeGen/RISCV/rvv/llrint-sdnode.ll
 create mode 100644 llvm/test/CodeGen/RISCV/rvv/lrint-sdnode.ll
 create mode 100644 llvm/test/CodeGen/RISCV/rvv/sf_vqmacc_2x8x2.ll
 create mode 100644 llvm/test/CodeGen/RISCV/rvv/sf_vqmacc_4x8x4.ll
 create mode 100644 llvm/test/CodeGen/RISCV/rvv/sf_vqmaccsu_2x8x2.ll
 create mode 100644 llvm/test/CodeGen/RISCV/rvv/sf_vqmaccsu_4x8x4.ll
 create mode 100644 llvm/test/CodeGen/RISCV/rvv/sf_vqmaccu_2x8x2.ll
 create mode 100644 llvm/test/CodeGen/RISCV/rvv/sf_vqmaccu_4x8x4.ll
 create mode 100644 llvm/test/CodeGen/RISCV/rvv/sf_vqmaccus_2x8x2.ll
 create mode 100644 llvm/test/CodeGen/RISCV/rvv/sf_vqmaccus_4x8x4.ll
 create mode 100644 llvm/test/CodeGen/RISCV/spir-kernel-cc.ll
 create mode 100644 llvm/test/CodeGen/SPIRV/pointers/getelementptr-base-type.ll
 create mode 100644 llvm/test/CodeGen/SPIRV/pointers/kernel-argument-pointer-addressspace.ll
 create mode 100644 llvm/test/CodeGen/SPIRV/pointers/kernel-argument-pointer-type.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/ret-addr-02.ll
 create mode 100644 llvm/test/CodeGen/Thumb2/mve-concat.ll
 delete mode 100644 llvm/test/CodeGen/X86/AMX/amx-configO2toO0-precfg.ll
 create mode 100644 llvm/test/CodeGen/X86/avx512fp16-combine-shuffle-fma.ll
 create mode 100644 llvm/test/CodeGen/X86/cmp-shiftX-maskX.ll
 create mode 100644 llvm/test/CodeGen/X86/foldimmediate-size.ll
 create mode 100644 llvm/test/CodeGen/X86/foldimmediate.mir
 create mode 100644 llvm/test/CodeGen/X86/indirect-br-gep-unmerge.ll
 create mode 100644 llvm/test/CodeGen/X86/inline-spiller-impdef-on-implicit-def-regression.ll
 create mode 100644 llvm/test/CodeGen/X86/no-split-size.ll
 create mode 100644 llvm/test/CodeGen/X86/peephole-copy.mir
 create mode 100644 llvm/test/CodeGen/X86/pr69080.ll
 delete mode 100644 llvm/test/CodeGen/X86/stack-frame-layout-remarks-64.ll
 rename llvm/test/CodeGen/X86/{stack-frame-layout-remarks-32.ll => stack-frame-layout-remarks.ll} (91%)
 create mode 100644 llvm/test/CodeGen/X86/usermsr-intrinsics.ll
 create mode 100644 llvm/test/CodeGen/X86/vector-llrint.ll
 create mode 100644 llvm/test/CodeGen/X86/vector-lrint.ll
 rename llvm/test/DebugInfo/{X86 => Generic}/PR20038.ll (96%)
 create mode 100644 llvm/test/DebugInfo/Generic/dwarf-public-names.ll
 rename llvm/test/DebugInfo/{X86 => Generic}/member-order.ll (96%)
 rename llvm/test/DebugInfo/{X86 => Generic}/tu-composite.ll (99%)
 create mode 100644 llvm/test/DebugInfo/M68k/calling-convention.ll
 create mode 100644 llvm/test/DebugInfo/M68k/lit.local.cfg
 delete mode 100644 llvm/test/DebugInfo/invalid-sizes.ll
 create mode 100644 llvm/test/Instrumentation/HWAddressSanitizer/str-nobuiltin.ll
 create mode 100644 llvm/test/MC/AArch64/FP8/system-regs.s
 create mode 100644 llvm/test/MC/ARM/inst-directive-it-vpt.s
 create mode 100644 llvm/test/MC/Disassembler/X86/usermsr-64.txt
 create mode 100644 llvm/test/MC/RISCV/rvv/xsfvqmacc.s
 create mode 100644 llvm/test/MC/X86/usermsr-64-att.s
 create mode 100644 llvm/test/MC/X86/usermsr-64-intel.s
 create mode 100644 llvm/test/TableGen/GlobalISelCombinerEmitter/misc/redundant-combine-in-list.td
 create mode 100644 llvm/test/TableGen/VarLenEncoderHwModes.td
 create mode 100644 llvm/test/TableGen/dump.td
 create mode 100644 llvm/test/TableGen/repr.td
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/AArch64/sink-gather-scatter-addressing.ll
 create mode 100644 llvm/test/Transforms/GVN/captured-before.ll
 create mode 100644 llvm/test/Transforms/GVN/pr69301.ll
 create mode 100644 llvm/test/Transforms/HipStdPar/accelerator-code-selection.ll
 create mode 100644 llvm/test/Transforms/HipStdPar/allocation-interposition.ll
 create mode 100644 llvm/test/Transforms/HipStdPar/allocation-no-interposition.ll
 create mode 100644 llvm/test/Transforms/HipStdPar/unsupported-asm.ll
 create mode 100644 llvm/test/Transforms/HipStdPar/unsupported-builtins.ll
 create mode 100644 llvm/test/Transforms/HipStdPar/unsupported-thread-local-direct-use.ll
 create mode 100644 llvm/test/Transforms/HipStdPar/unsupported-thread-local-indirect-use.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr69059.ll
 create mode 100644 llvm/test/Transforms/InstCombine/shift-flags.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/sleef-intrinsic-calls-aarch64.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/outer_loop_scalable.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/scev-invalidation.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/cfi-thunk-merging.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/mergefunc-preserve-vfe-intrinsics.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/no-merge-debug-thunks.ll
 create mode 100644 llvm/test/Transforms/PhaseOrdering/icmp-ashr-breaking-select-idiom.ll
 create mode 100644 llvm/test/Transforms/PhaseOrdering/switch_different_gep_types.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/call-argument-attributes.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/matching-gather-nodes-phi-users.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/non-scheduled-inst-reused-as-last-inst.ll
 create mode 100644 llvm/test/YAMLParser/plain-characters.test
 create mode 100644 llvm/test/tools/llvm-dwp/X86/cu_tu_units_manual_v5_invalid.s
 create mode 100644 llvm/test/tools/llvm-lib/arm64ec-implib.test
 create mode 100644 llvm/test/tools/llvm-mca/AArch64/HiSilicon/tsv110-writeback.s
 create mode 100644 llvm/test/tools/llvm-mca/RISCV/vle-vse.s
 create mode 100644 llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-gfx11.s
 create mode 100644 llvm/test/tools/llvm-objdump/ELF/RISCV/multi-instr-target.s
 create mode 100644 llvm/test/tools/llvm-rc/Inputs/split-path.rc
 create mode 100644 llvm/test/tools/llvm-rc/split-path.test
 create mode 100644 llvm/test/tools/llvm-remarkutil/count/Inputs/remark-count-by.yaml
 create mode 100644 llvm/test/tools/llvm-remarkutil/count/Inputs/remark-filter-by.yaml
 create mode 100644 llvm/test/tools/llvm-remarkutil/count/Inputs/remark-group-by.yaml
 create mode 100644 llvm/test/tools/llvm-remarkutil/count/count-by-keys.test
 create mode 100644 llvm/test/tools/llvm-remarkutil/count/count-by-remark.test
 create mode 100644 llvm/test/tools/llvm-remarkutil/count/filter-by-pass-name.test
 create mode 100644 llvm/test/tools/llvm-remarkutil/count/filter-by-remark-name.test
 create mode 100644 llvm/test/tools/llvm-remarkutil/count/filter-by-type.test
 create mode 100644 llvm/test/tools/llvm-remarkutil/count/group-by-function-with-loc.test
 create mode 100644 llvm/test/tools/llvm-remarkutil/count/group-by-function.test
 create mode 100644 llvm/test/tools/llvm-remarkutil/count/group-by-source.test
 create mode 100644 llvm/test/tools/llvm-symbolizer/xcoff-sd-symbol.ll
 create mode 100644 llvm/tools/llvm-remarkutil/RemarkCounter.cpp
 create mode 100644 llvm/tools/llvm-remarkutil/RemarkCounter.h
 create mode 100644 llvm/unittests/ADT/IListIteratorBitsTest.cpp
 create mode 100644 llvm/unittests/Support/UTCTimeTest.cpp
 create mode 100644 llvm/unittests/Target/RISCV/RISCVBaseInfoTest.cpp
 create mode 100644 llvm/utils/gn/secondary/clang/unittests/AST/Interp/BUILD.gn
 rename llvm/utils/gn/secondary/lldb/tools/{lldb-vscode => lldb-dap}/BUILD.gn (86%)
 create mode 100644 llvm/utils/gn/secondary/llvm/lib/Transforms/HipStdPar/BUILD.gn
 create mode 100644 llvm/utils/lit/tests/Inputs/googletest-no-sharding/DummySubDir/OneTest.py
 create mode 100644 llvm/utils/lit/tests/Inputs/googletest-no-sharding/lit.cfg
 mode change 100644 => 100755 llvm/utils/lit/tests/discovery.py
 create mode 100644 llvm/utils/lit/tests/googletest-no-sharding.py
 create mode 100644 mlir/include/mlir/Conversion/IndexToSPIRV/IndexToSPIRV.h
 create mode 100644 mlir/include/mlir/Dialect/LLVMIR/BasicPtxBuilderInterface.h
 create mode 100644 mlir/include/mlir/Dialect/LLVMIR/BasicPtxBuilderInterface.td
 create mode 100644 mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorInterfaces.h
 create mode 100644 mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorInterfaces.td
 create mode 100644 mlir/include/mlir/Query/Matcher/ErrorBuilder.h
 create mode 100644 mlir/include/mlir/Query/Matcher/Marshallers.h
 create mode 100644 mlir/include/mlir/Query/Matcher/MatchFinder.h
 create mode 100644 mlir/include/mlir/Query/Matcher/MatchersInternal.h
 create mode 100644 mlir/include/mlir/Query/Matcher/Registry.h
 create mode 100644 mlir/include/mlir/Query/Matcher/VariantValue.h
 create mode 100644 mlir/include/mlir/Query/Query.h
 create mode 100644 mlir/include/mlir/Query/QuerySession.h
 create mode 100644 mlir/include/mlir/Support/ADTExtras.h
 create mode 100644 mlir/include/mlir/Tools/mlir-query/MlirQueryMain.h
 create mode 100644 mlir/lib/Conversion/IndexToSPIRV/CMakeLists.txt
 create mode 100644 mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp
 create mode 100644 mlir/lib/Dialect/LLVMIR/IR/BasicPtxBuilderInterface.cpp
 create mode 100644 mlir/lib/Dialect/SparseTensor/IR/SparseTensorInterfaces.cpp
 create mode 100644 mlir/lib/Dialect/Transform/Transforms/InterpreterPass.cpp
 create mode 100644 mlir/lib/Dialect/Transform/Transforms/PreloadLibraryPass.cpp
 delete mode 100644 mlir/lib/ExecutionEngine/SparseTensor/NNZ.cpp
 create mode 100644 mlir/lib/Query/CMakeLists.txt
 create mode 100644 mlir/lib/Query/Matcher/CMakeLists.txt
 create mode 100644 mlir/lib/Query/Matcher/Diagnostics.cpp
 create mode 100644 mlir/lib/Query/Matcher/Diagnostics.h
 create mode 100644 mlir/lib/Query/Matcher/ErrorBuilder.cpp
 create mode 100644 mlir/lib/Query/Matcher/Parser.cpp
 create mode 100644 mlir/lib/Query/Matcher/Parser.h
 create mode 100644 mlir/lib/Query/Matcher/RegistryManager.cpp
 create mode 100644 mlir/lib/Query/Matcher/RegistryManager.h
 create mode 100644 mlir/lib/Query/Matcher/VariantValue.cpp
 create mode 100644 mlir/lib/Query/Query.cpp
 create mode 100644 mlir/lib/Query/QueryParser.cpp
 create mode 100644 mlir/lib/Query/QueryParser.h
 create mode 100644 mlir/lib/Tools/mlir-query/CMakeLists.txt
 create mode 100644 mlir/lib/Tools/mlir-query/MlirQueryMain.cpp
 create mode 100644 mlir/python/mlir/dialects/AffineOps.td
 delete mode 100644 mlir/python/mlir/dialects/_arith_ops_ext.py
 delete mode 100644 mlir/python/mlir/dialects/_bufferization_ops_ext.py
 delete mode 100644 mlir/python/mlir/dialects/_bufferization_transform_ops_ext.py
 delete mode 100644 mlir/python/mlir/dialects/_builtin_ops_ext.py
 delete mode 100644 mlir/python/mlir/dialects/_func_ops_ext.py
 delete mode 100644 mlir/python/mlir/dialects/_gpu_transform_ops_ext.py
 delete mode 100644 mlir/python/mlir/dialects/_linalg_ops_ext.py
 delete mode 100644 mlir/python/mlir/dialects/_loop_transform_ops_ext.py
 delete mode 100644 mlir/python/mlir/dialects/_memref_ops_ext.py
 delete mode 100644 mlir/python/mlir/dialects/_memref_transform_ops_ext.py
 delete mode 100644 mlir/python/mlir/dialects/_ml_program_ops_ext.py
 delete mode 100644 mlir/python/mlir/dialects/_pdl_ops_ext.py
 delete mode 100644 mlir/python/mlir/dialects/_scf_ops_ext.py
 delete mode 100644 mlir/python/mlir/dialects/_structured_transform_ops_ext.py
 delete mode 100644 mlir/python/mlir/dialects/_tensor_ops_ext.py
 delete mode 100644 mlir/python/mlir/dialects/_tensor_transform_ops_ext.py
 delete mode 100644 mlir/python/mlir/dialects/_transform_ops_ext.py
 delete mode 100644 mlir/python/mlir/dialects/_transform_pdl_extension_ops_ext.py
 create mode 100644 mlir/python/mlir/dialects/affine.py
 create mode 100644 mlir/test/Conversion/IndexToSPRIV/index-to-spirv.mlir
 create mode 100644 mlir/test/Conversion/SCFToSPIRV/use-indices.mlir
 create mode 100644 mlir/test/Dialect/ArmSVE/invalid.mlir
 rename mlir/test/Dialect/LLVM/{lower-to-llvm-e2e.mlir => lower-to-llvm-e2e-with-target-tag.mlir} (100%)
 create mode 100644 mlir/test/Dialect/LLVM/lower-to-llvm-e2e-with-top-level-named-sequence.mlir
 delete mode 100644 mlir/test/Dialect/SparseTensor/codegen_sparse_dealloc.mlir
 create mode 100644 mlir/test/Dialect/Tosa/variables.mlir
 create mode 100644 mlir/test/Dialect/Transform/Library/lower-to-llvm-transform-symbol-def.mlir
 create mode 100644 mlir/test/Dialect/Transform/include/Library/lower-to-llvm.mlir
 rename mlir/test/Dialect/Transform/{ => include}/test-interpreter-external-concurrent-source.mlir (100%)
 rename mlir/test/Dialect/Transform/{ => include}/test-interpreter-external-source.mlir (100%)
 rename mlir/test/Dialect/Transform/{ => include}/test-interpreter-external-symbol-def-invalid.mlir (100%)
 rename mlir/test/Dialect/Transform/{ => include}/test-interpreter-library/definitions-self-contained.mlir (96%)
 rename mlir/test/Dialect/Transform/{ => include}/test-interpreter-library/definitions-with-unresolved.mlir (78%)
 create mode 100644 mlir/test/Dialect/Transform/interpreter-entry-point.mlir
 create mode 100644 mlir/test/Dialect/Transform/interpreter.mlir
 create mode 100644 mlir/test/Dialect/Transform/preload-library.mlir
 create mode 100644 mlir/test/Dialect/Transform/test-interpreter-multiple-top-level-ops.mlir
 create mode 100755 mlir/test/Integration/Dialect/SparseTensor/CPU/block.mlir
 create mode 100644 mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sddmm-lib.mlir
 create mode 100755 mlir/test/Integration/data/block.mtx
 create mode 100644 mlir/test/Interfaces/TilingInterface/tile-using-scfforall.mlir
 create mode 100644 mlir/test/mlir-query/simple-test.mlir
 create mode 100644 mlir/test/mlir-tblgen/utils.td
 create mode 100644 mlir/test/mlir-vulkan-runner/iaddcarry_extended.mlir
 create mode 100644 mlir/test/python/dialects/affine.py
 create mode 100644 mlir/tools/mlir-query/CMakeLists.txt
 create mode 100644 mlir/tools/mlir-query/mlir-query.cpp
 create mode 100644 mlir/unittests/Dialect/SCF/CMakeLists.txt
 create mode 100644 mlir/unittests/Dialect/SCF/LoopLikeSCFOpsTest.cpp
 create mode 100644 openmp/libomptarget/DeviceRTL/include/Allocator.h
 create mode 100644 openmp/libomptarget/DeviceRTL/src/Allocator.cpp
 create mode 100644 openmp/libomptarget/test/api/omp_target_memset.c
 create mode 100644 openmp/libomptarget/test/libc/assert.c
 create mode 100644 openmp/libomptarget/test/offloading/malloc.c
 create mode 100644 openmp/libomptarget/test/offloading/malloc_parallel.c
 create mode 100644 openmp/libomptarget/test/offloading/thread_state_1.c
 create mode 100644 openmp/libomptarget/test/offloading/thread_state_2.c
 create mode 100644 runtimes/cmake/android/Arch-arm.cmake
 create mode 100644 runtimes/cmake/android/Arch-arm64.cmake
 create mode 100644 runtimes/cmake/android/Arch-x86.cmake
 create mode 100644 runtimes/cmake/android/Arch-x86_64.cmake
 create mode 100644 runtimes/cmake/android/Common.cmake

diff --git a/.ci/generate-buildkite-pipeline-premerge b/.ci/generate-buildkite-pipeline-premerge
index e389df4315..9c6f5aefd6 100755
--- a/.ci/generate-buildkite-pipeline-premerge
+++ b/.ci/generate-buildkite-pipeline-premerge
@@ -21,8 +21,11 @@ set -eu
 set -o pipefail
 
 # Environment variables script works with:
+
+# Fetch origin/main to have an up to date merge base for main...HEAD diff.
+git fetch origin main:main
 # List of files affected by this commit
-: ${MODIFIED_FILES:=$(git diff --name-only HEAD~1)}
+: ${MODIFIED_FILES:=$(git diff --name-only main...HEAD)}
 # Filter rules for generic windows tests
 : ${WINDOWS_AGENTS:='{"queue": "windows"}'}
 # Filter rules for generic linux tests
diff --git a/.github/new-prs-labeler.yml b/.github/new-prs-labeler.yml
index ae658f848e..e4bc53e600 100644
--- a/.github/new-prs-labeler.yml
+++ b/.github/new-prs-labeler.yml
@@ -794,3 +794,35 @@ llvm:binary-utilities:
   - llvm/tools/llvm-size/**
   - llvm/tools/llvm-strings/**
   - llvm/tools/llvm-symbolizer/**
+
+clang:openmp:
+  - clang/include/clang/Basic/OpenMP*
+  - clang/include/clang/AST/OpenMPClause.h
+  - clang/include/clang/AST/DeclOpenMP.h
+  - clang/include/clang/AST/ExprOpenMP.h
+  - clang/include/clang/AST/StmtOpenMP.h
+  - clang/lib/AST/DeclOpenMP.cpp
+  - clang/lib/AST/OpenMPClause.cpp
+  - clang/lib/AST/StmtOpenMP.cpp
+  - clang/lib/Headers/openmp_wrappers/**
+  - clang/lib/Parse/ParseOpenMP.cpp
+  - clang/lib/Basic/OpenMPKinds.cpp
+  - clang/lib/Driver/ToolChains/AMDGPUOpenMP.cpp
+  - clang/lib/Driver/ToolChains/AMDGPUOpenMP.h
+  - clang/lib/CodeGen/CgStmtOpenMP.cpp
+  - clang/lib/CodeGen/CGOpenMP*
+  - clang/lib/Sema/SemaOpenMP.cpp
+  - clang/test/OpenMP/**
+  - clang/test/AST/ast-dump-openmp-*
+  - llvm/lib/Frontend/OpenMP/**
+  - llvm/lib/Transforms/IPO/OpenMPOpt.cpp
+  - llvm/include/llvm/Frontend/OpenMP/**
+  - llvm/include/llvm/Transforms/IPO/OpenMPOpt.h
+  - llvm/unittests/Frontend/OpenMP*
+  - llvm/test/Transforms/OpenMP/**
+
+openmp:libomp:
+  - any: ['openmp/**', '!openmp/libomptarget/**']
+
+openmp:libomptarget:
+  - any: ['openmp/**', '!openmp/runtime/**']
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
new file mode 100644
index 0000000000..57cb9ecf2c
--- /dev/null
+++ b/.github/workflows/docs.yml
@@ -0,0 +1,94 @@
+# LLVM Documentation CI
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+name: "Test documentation build"
+
+permissions:
+  contents: read
+
+on:
+  push:
+    branches:
+      - 'main'
+    paths:
+      - 'llvm/docs/**'
+      - 'clang/docs/**'
+      - 'clang-tools-extra/docs/**'
+      - 'lldb/docs/**'
+  pull_request:
+    paths:
+      - 'llvm/docs/**'
+      - 'clang/docs/**'
+      - 'clang-tools-extra/docs/**'
+      - 'lldb/docs/**'
+
+jobs:
+  check-docs-build:
+    name: "Test documentation build"
+    runs-on: ubuntu-latest
+    steps:
+      # Fetch all the commits in a pull request + 1 so that the
+      # docs-changed-subprojects step won't pull them in itself in an extremely
+      # slow manner.
+      - name: Calculate number of commits to fetch (PR)
+        if: ${{ github.event_name == 'pull_request' }} 
+        run: echo "PR_FETCH_DEPTH=$(( ${{ github.event.pull_request.commits }} + 1 ))" >> "${GITHUB_ENV}"
+      - name: Fetch LLVM sources (PR)
+        if: ${{ github.event_name == 'pull_request' }}
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: ${{ env.PR_FETCH_DEPTH }}
+      - name: Fetch LLVM sources (push)
+        if: ${{ github.event_name == 'push' }}
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+      - name: Get subprojects that have doc changes
+        id: docs-changed-subprojects
+        uses: tj-actions/changed-files@v39
+        with:
+          files_yaml: |
+            llvm:
+              - 'llvm/docs/**'
+            clang:
+              - 'clang/docs/**'
+            clang-tools-extra:
+              - 'clang-tools-extra/docs/**'
+            lldb:
+              - 'lldb/docs/**'
+      - name: Setup Python env
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+          cache-dependency-path: 'llvm/docs/requirements.txt'
+      - name: Install python dependencies
+        run: pip install -r llvm/docs/requirements.txt
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          # swig and graphviz are lldb specific dependencies
+          sudo apt-get install -y cmake ninja-build swig graphviz
+      - name: Build LLVM docs
+        if: steps.docs-changed-subprojects.outputs.llvm_any_changed == 'true'
+        run: |
+          cmake -B llvm-build -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_SPHINX=ON ./llvm
+          TZ=UTC ninja -C llvm-build docs-llvm-html docs-llvm-man
+      - name: Build Clang docs
+        if: steps.docs-changed-subprojects.outputs.clang_any_changed == 'true'
+        run: |
+          cmake -B clang-build -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS="clang" -DLLVM_ENABLE_SPHINX=ON ./llvm
+          TZ=UTC ninja -C clang-build docs-clang-html docs-clang-man
+      - name: Build clang-tools-extra docs
+        if: steps.docs-changed-subprojects.outputs.clang-tools-extra_any_changed == 'true'
+        run: |
+          cmake -B clang-tools-extra-build -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS="clang;clang-tools-extra" -DLLVM_ENABLE_SPHINX=ON ./llvm
+          TZ=UTC ninja -C clang-tools-extra-build docs-clang-tools-html docs-clang-tools-man
+      - name: Build LLDB docs
+        if: steps.docs-changed-subprojects.outputs.lldb_any_changed == 'true'
+        run: |
+          cmake -B lldb-build -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS="clang;lldb" -DLLVM_ENABLE_SPHINX=ON ./llvm
+          TZ=UTC ninja -C lldb-build docs-lldb-html docs-lldb-man
+
diff --git a/.github/workflows/libcxx-check-generated-files.yml b/.github/workflows/libcxx-check-generated-files.yml
new file mode 100644
index 0000000000..570055624b
--- /dev/null
+++ b/.github/workflows/libcxx-check-generated-files.yml
@@ -0,0 +1,24 @@
+name: "Check libc++ generated files"
+on:
+  pull_request:
+    paths:
+      - 'libcxx/**'
+
+permissions:
+  contents: read
+
+jobs:
+  check_generated_files:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Fetch LLVM sources
+        uses: actions/checkout@v4
+
+      - name: Install dependencies
+        uses: aminya/setup-cpp@v1
+        with:
+          clangformat: 17.0.1
+          ninja: true
+
+      - name: Check generated files
+        run: libcxx/utils/ci/run-buildbot check-generated-output
diff --git a/.github/workflows/release-tasks.yml b/.github/workflows/release-tasks.yml
index 62aa56ea67..85b720e323 100644
--- a/.github/workflows/release-tasks.yml
+++ b/.github/workflows/release-tasks.yml
@@ -48,7 +48,7 @@ jobs:
           ./llvm/utils/release/github-upload-release.py --token ${{ github.token }} --release ${{ steps.validate-tag.outputs.release-version }} upload --files ./*doxygen*.tar.xz
 
       - name: Create Release Notes Artifact
-        uses: actions/download-artifact@v3
+        uses: actions/upload-artifact@v3
         with:
           name: release-notes
           path: docs-build/html-export/
@@ -104,7 +104,7 @@ jobs:
         run: |
           cd llvm/utils/lit
           # Remove 'dev' suffix from lit version.
-          sed -i "s/ + 'dev'//g" lit/__init__.py
+          sed -i 's/ + "dev"//g' lit/__init__.py
           python3 setup.py sdist
 
       - name: Upload lit to test.pypi.org
@@ -112,8 +112,10 @@ jobs:
         with:
           password: ${{ secrets.LLVM_LIT_TEST_PYPI_API_TOKEN }}
           repository-url: https://test.pypi.org/legacy/
+          packages-dir: llvm/utils/lit/dist/
 
       - name: Upload lit to pypi.org
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           password: ${{ secrets.LLVM_LIT_PYPI_API_TOKEN }}
+          packages-dir: llvm/utils/lit/dist/
diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h
index dea1d55b15..a4c9eb4265 100644
--- a/bolt/include/bolt/Core/BinaryFunction.h
+++ b/bolt/include/bolt/Core/BinaryFunction.h
@@ -192,9 +192,6 @@ class BinaryFunction {
   static constexpr uint64_t COUNT_NO_PROFILE =
       BinaryBasicBlock::COUNT_NO_PROFILE;
 
-  /// We have to use at least 2-byte alignment for functions because of C++ ABI.
-  static constexpr unsigned MinAlign = 2;
-
   static const char TimerGroupName[];
   static const char TimerGroupDesc[];
 
@@ -1720,8 +1717,24 @@ class BinaryFunction {
     return *this;
   }
 
-  Align getAlign() const { return Align(Alignment); }
+  uint16_t getMinAlignment() const {
+    // Align data in code BFs minimum to CI alignment
+    if (!size() && hasIslandsInfo())
+      return getConstantIslandAlignment();
+
+    // Minimal code alignment on AArch64 and RISCV is 4
+    if (BC.isAArch64() || BC.isRISCV())
+      return 4;
+
+    // We have to use at least 2-byte alignment for functions because
+    // of C++ ABI.
+    return 2;
+  }
+
+  Align getMinAlign() const { return Align(getMinAlignment()); }
+
   uint16_t getAlignment() const { return Alignment; }
+  Align getAlign() const { return Align(getAlignment()); }
 
   BinaryFunction &setMaxAlignmentBytes(uint16_t MaxAlignBytes) {
     MaxAlignmentBytes = MaxAlignBytes;
diff --git a/bolt/include/bolt/Core/MCPlusBuilder.h b/bolt/include/bolt/Core/MCPlusBuilder.h
index c9e0e5d599..b6817c1d7f 100644
--- a/bolt/include/bolt/Core/MCPlusBuilder.h
+++ b/bolt/include/bolt/Core/MCPlusBuilder.h
@@ -1185,7 +1185,7 @@ class MCPlusBuilder {
 
   /// Set the label of \p Inst. This label will be emitted right before \p Inst
   /// is emitted to MCStreamer.
-  bool setLabel(MCInst &Inst, MCSymbol *Label);
+  bool setLabel(MCInst &Inst, MCSymbol *Label, AllocatorIdTy AllocatorId = 0);
 
   /// Return MCSymbol that represents a target of this instruction at a given
   /// operand number \p OpNum. If there's no symbol associated with
diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp
index b5514228d7..acc441aca4 100644
--- a/bolt/lib/Core/BinaryContext.cpp
+++ b/bolt/lib/Core/BinaryContext.cpp
@@ -118,7 +118,7 @@ Expected<std::unique_ptr<BinaryContext>>
 BinaryContext::createBinaryContext(const ObjectFile *File, bool IsPIC,
                                    std::unique_ptr<DWARFContext> DwCtx) {
   StringRef ArchName = "";
-  StringRef FeaturesStr = "";
+  std::string FeaturesStr = "";
   switch (File->getArch()) {
   case llvm::Triple::x86_64:
     ArchName = "x86-64";
@@ -128,11 +128,20 @@ BinaryContext::createBinaryContext(const ObjectFile *File, bool IsPIC,
     ArchName = "aarch64";
     FeaturesStr = "+all";
     break;
-  case llvm::Triple::riscv64:
+  case llvm::Triple::riscv64: {
     ArchName = "riscv64";
-    // RV64GC
-    FeaturesStr = "+m,+a,+f,+d,+zicsr,+zifencei,+c,+relax";
+    Expected<SubtargetFeatures> Features = File->getFeatures();
+
+    if (auto E = Features.takeError())
+      return E;
+
+    // We rely on relaxation for some transformations (e.g., promoting all calls
+    // to PseudoCALL and then making JITLink relax them). Since the relax
+    // feature is not stored in the object file, we manually enable it.
+    Features->AddFeature("relax");
+    FeaturesStr = Features->getString();
     break;
+  }
   default:
     return createStringError(std::errc::not_supported,
                              "BOLT-ERROR: Unrecognized machine in ELF file");
@@ -1803,6 +1812,10 @@ MarkerSymType BinaryContext::getMarkerType(const SymbolRef &Symbol) const {
   if (*NameOrError == "$x" || NameOrError->startswith("$x."))
     return MarkerSymType::CODE;
 
+  // $x<ISA>
+  if (isRISCV() && NameOrError->startswith("$x"))
+    return MarkerSymType::CODE;
+
   if (*NameOrError == "$d" || NameOrError->startswith("$d."))
     return MarkerSymType::DATA;
 
diff --git a/bolt/lib/Core/BinaryEmitter.cpp b/bolt/lib/Core/BinaryEmitter.cpp
index b1ee6cc221..de80a99a74 100644
--- a/bolt/lib/Core/BinaryEmitter.cpp
+++ b/bolt/lib/Core/BinaryEmitter.cpp
@@ -309,7 +309,7 @@ bool BinaryEmitter::emitFunction(BinaryFunction &Function,
     // tentative layout.
     Section->ensureMinAlignment(Align(opts::AlignFunctions));
 
-    Streamer.emitCodeAlignment(Align(BinaryFunction::MinAlign), &*BC.STI);
+    Streamer.emitCodeAlignment(Function.getMinAlign(), &*BC.STI);
     uint16_t MaxAlignBytes = FF.isSplitFragment()
                                  ? Function.getMaxColdAlignmentBytes()
                                  : Function.getMaxAlignmentBytes();
diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp
index c475e4f903..65ebb8599f 100644
--- a/bolt/lib/Core/BinaryFunction.cpp
+++ b/bolt/lib/Core/BinaryFunction.cpp
@@ -164,8 +164,6 @@ bool shouldPrint(const BinaryFunction &Function) {
 namespace llvm {
 namespace bolt {
 
-constexpr unsigned BinaryFunction::MinAlign;
-
 template <typename R> static bool emptyRange(const R &Range) {
   return Range.begin() == Range.end();
 }
@@ -2464,6 +2462,13 @@ struct CFISnapshot {
     case MCCFIInstruction::OpDefCfaRegister:
       CFAReg = Instr.getRegister();
       CFARule = UNKNOWN;
+
+      // This shouldn't happen according to the spec but GNU binutils on RISC-V
+      // emits a DW_CFA_def_cfa_register in CIE's which leaves the offset
+      // unspecified. Both readelf and llvm-dwarfdump interpret the offset as 0
+      // in this case so let's do the same.
+      if (CFAOffset == UNKNOWN)
+        CFAOffset = 0;
       break;
     case MCCFIInstruction::OpDefCfaOffset:
       CFAOffset = Instr.getOffset();
diff --git a/bolt/lib/Core/DebugData.cpp b/bolt/lib/Core/DebugData.cpp
index 22ca8c5acc..7a532fbbb5 100644
--- a/bolt/lib/Core/DebugData.cpp
+++ b/bolt/lib/Core/DebugData.cpp
@@ -124,14 +124,14 @@ writeAddressRanges(raw_svector_ostream &Stream,
                    const DebugAddressRangesVector &AddressRanges,
                    const bool WriteRelativeRanges = false) {
   for (const DebugAddressRange &Range : AddressRanges) {
-    support::endian::write(Stream, Range.LowPC, support::little);
+    support::endian::write(Stream, Range.LowPC, llvm::endianness::little);
     support::endian::write(
         Stream, WriteRelativeRanges ? Range.HighPC - Range.LowPC : Range.HighPC,
-        support::little);
+        llvm::endianness::little);
   }
   // Finish with 0 entries.
-  support::endian::write(Stream, 0ULL, support::little);
-  support::endian::write(Stream, 0ULL, support::little);
+  support::endian::write(Stream, 0ULL, llvm::endianness::little);
+  support::endian::write(Stream, 0ULL, llvm::endianness::little);
   return AddressRanges.size() * 16 + 16;
 }
 
@@ -209,13 +209,15 @@ getDWARF5Header(const LocListsRangelistsHeader &Header) {
       getDWARF5RngListLocListHeaderSize() - sizeof(UnitLengthType);
 
   support::endian::write(*HeaderStream, Header.UnitLength + HeaderSize,
-                         support::little);
-  support::endian::write(*HeaderStream, Header.Version, support::little);
-  support::endian::write(*HeaderStream, Header.AddressSize, support::little);
+                         llvm::endianness::little);
+  support::endian::write(*HeaderStream, Header.Version,
+                         llvm::endianness::little);
+  support::endian::write(*HeaderStream, Header.AddressSize,
+                         llvm::endianness::little);
   support::endian::write(*HeaderStream, Header.SegmentSelector,
-                         support::little);
+                         llvm::endianness::little);
   support::endian::write(*HeaderStream, Header.OffsetEntryCount,
-                         support::little);
+                         llvm::endianness::little);
   return HeaderBuffer;
 }
 
@@ -254,17 +256,18 @@ static bool emitWithBase(raw_ostream &OS, const DebugVector &Entries,
   }
 
   support::endian::write(OS, static_cast<uint8_t>(BaseAddressx),
-                         support::little);
+                         llvm::endianness::little);
   uint32_t BaseIndex = AddrWriter.getIndexFromAddress(Base, CU);
   encodeULEB128(BaseIndex, OS);
   for (auto &OffsetEntry : Offsets) {
     support::endian::write(OS, static_cast<uint8_t>(OffsetPair),
-                           support::little);
+                           llvm::endianness::little);
     encodeULEB128(OffsetEntry.StartOffset, OS);
     encodeULEB128(OffsetEntry.EndOffset, OS);
     Func(OffsetEntry.Index);
   }
-  support::endian::write(OS, static_cast<uint8_t>(EndOfList), support::little);
+  support::endian::write(OS, static_cast<uint8_t>(EndOfList),
+                         llvm::endianness::little);
   return true;
 }
 
@@ -291,7 +294,7 @@ DebugRangeListsSectionWriter::addRanges(DebugAddressRangesVector &Ranges) {
     const DebugAddressRange &Range = Ranges[I];
     support::endian::write(*CUBodyStream,
                            static_cast<uint8_t>(dwarf::DW_RLE_startx_length),
-                           support::little);
+                           llvm::endianness::little);
     uint32_t Index = AddrWriter->getIndexFromAddress(Range.LowPC, *CU);
     encodeULEB128(Index, *CUBodyStream);
     encodeULEB128(Range.HighPC - Range.LowPC, *CUBodyStream);
@@ -301,7 +304,7 @@ DebugRangeListsSectionWriter::addRanges(DebugAddressRangesVector &Ranges) {
   if (WrittenStartxLength)
     support::endian::write(*CUBodyStream,
                            static_cast<uint8_t>(dwarf::DW_RLE_end_of_list),
-                           support::little);
+                           llvm::endianness::little);
   CurrentOffset = CUBodyBuffer->size();
   return RangeEntries.size() - 1;
 }
@@ -315,7 +318,7 @@ void DebugRangeListsSectionWriter::finalizeSection() {
   const uint32_t SizeOfArraySection = RangeEntries.size() * SizeOfArrayEntry;
   for (uint32_t Offset : RangeEntries)
     support::endian::write(*CUArrayStream, Offset + SizeOfArraySection,
-                           support::little);
+                           llvm::endianness::little);
 
   std::unique_ptr<DebugBufferVector> Header = getDWARF5Header(
       {static_cast<uint32_t>(SizeOfArraySection + CUBodyBuffer.get()->size()),
@@ -359,17 +362,17 @@ void DebugARangesSectionWriter::writeARangesSection(
     uint32_t Size = 8 + 4 + 2 * sizeof(uint64_t) * (AddressRanges.size() + 1);
 
     // Header field #1: set size.
-    support::endian::write(RangesStream, Size, support::little);
+    support::endian::write(RangesStream, Size, llvm::endianness::little);
 
     // Header field #2: version number, 2 as per the specification.
     support::endian::write(RangesStream, static_cast<uint16_t>(2),
-                           support::little);
+                           llvm::endianness::little);
 
     assert(CUMap.count(Offset) && "Original CU offset is not found in CU Map");
     // Header field #3: debug info offset of the correspondent compile unit.
     support::endian::write(
         RangesStream, static_cast<uint32_t>(CUMap.find(Offset)->second.Offset),
-        support::little);
+        llvm::endianness::little);
 
     // Header field #4: address size.
     // 8 since we only write ELF64 binaries for now.
@@ -380,7 +383,7 @@ void DebugARangesSectionWriter::writeARangesSection(
 
     // Padding before address table - 4 bytes in the 64-bit-pointer case.
     support::endian::write(RangesStream, static_cast<uint32_t>(0),
-                           support::little);
+                           llvm::endianness::little);
 
     writeAddressRanges(RangesStream, AddressRanges, true);
   }
@@ -473,10 +476,10 @@ void DebugAddrWriter::update(DIEBuilder &DIEBlder, DWARFUnit &CU) {
       break;
     case 4:
       support::endian::write(*AddressStream, static_cast<uint32_t>(Address),
-                             support::little);
+                             llvm::endianness::little);
       break;
     case 8:
-      support::endian::write(*AddressStream, Address, support::little);
+      support::endian::write(*AddressStream, Address, llvm::endianness::little);
       break;
     }
   };
@@ -492,11 +495,12 @@ void DebugAddrWriter::update(DIEBuilder &DIEBlder, DWARFUnit &CU) {
 void DebugAddrWriterDwarf5::update(DIEBuilder &DIEBlder, DWARFUnit &CU) {
   // Need to layout all sections within .debug_addr
   // Within each section sort Address by index.
-  const endianness Endian =
-      BC->DwCtx->isLittleEndian() ? support::little : support::big;
+  const endianness Endian = BC->DwCtx->isLittleEndian()
+                                ? llvm::endianness::little
+                                : llvm::endianness::big;
   const DWARFSection &AddrSec = BC->DwCtx->getDWARFObj().getAddrSection();
   DWARFDataExtractor AddrData(BC->DwCtx->getDWARFObj(), AddrSec,
-                              Endian == support::little, 0);
+                              Endian == llvm::endianness::little, 0);
   DWARFDebugAddrTable AddrTable;
   DIDumpOptions DumpOpts;
   constexpr uint32_t HeaderSize = 8;
@@ -594,11 +598,11 @@ void DebugLocWriter::addList(DIEBuilder &DIEBldr, DIE &Die, DIEValue &AttrInfo,
 
   for (const DebugLocationEntry &Entry : LocList) {
     support::endian::write(*LocStream, static_cast<uint64_t>(Entry.LowPC),
-                           support::little);
+                           llvm::endianness::little);
     support::endian::write(*LocStream, static_cast<uint64_t>(Entry.HighPC),
-                           support::little);
+                           llvm::endianness::little);
     support::endian::write(*LocStream, static_cast<uint16_t>(Entry.Expr.size()),
-                           support::little);
+                           llvm::endianness::little);
     *LocStream << StringRef(reinterpret_cast<const char *>(Entry.Expr.data()),
                             Entry.Expr.size());
     LocSectionOffset += 2 * 8 + 2 + Entry.Expr.size();
@@ -618,15 +622,17 @@ std::unique_ptr<DebugBufferVector> DebugLocWriter::getBuffer() {
 void DebugLocWriter::finalize(DIEBuilder &DIEBldr, DIE &Die) {}
 
 static void writeEmptyListDwarf5(raw_svector_ostream &Stream) {
-  support::endian::write(Stream, static_cast<uint32_t>(4), support::little);
+  support::endian::write(Stream, static_cast<uint32_t>(4),
+                         llvm::endianness::little);
   support::endian::write(Stream, static_cast<uint8_t>(dwarf::DW_LLE_start_end),
-                         support::little);
+                         llvm::endianness::little);
 
   const char Zeroes[16] = {0};
   Stream << StringRef(Zeroes, 16);
   encodeULEB128(0, Stream);
-  support::endian::write(
-      Stream, static_cast<uint8_t>(dwarf::DW_LLE_end_of_list), support::little);
+  support::endian::write(Stream,
+                         static_cast<uint8_t>(dwarf::DW_LLE_end_of_list),
+                         llvm::endianness::little);
 }
 
 static void writeLegacyLocList(DIEValue &AttrInfo,
@@ -645,21 +651,21 @@ static void writeLegacyLocList(DIEValue &AttrInfo,
   for (const DebugLocationEntry &Entry : LocList) {
     support::endian::write(LocStream,
                            static_cast<uint8_t>(dwarf::DW_LLE_startx_length),
-                           support::little);
+                           llvm::endianness::little);
     const uint32_t Index = AddrWriter.getIndexFromAddress(Entry.LowPC, CU);
     encodeULEB128(Index, LocStream);
 
     support::endian::write(LocStream,
                            static_cast<uint32_t>(Entry.HighPC - Entry.LowPC),
-                           support::little);
+                           llvm::endianness::little);
     support::endian::write(LocStream, static_cast<uint16_t>(Entry.Expr.size()),
-                           support::little);
+                           llvm::endianness::little);
     LocStream << StringRef(reinterpret_cast<const char *>(Entry.Expr.data()),
                            Entry.Expr.size());
   }
   support::endian::write(LocStream,
                          static_cast<uint8_t>(dwarf::DW_LLE_end_of_list),
-                         support::little);
+                         llvm::endianness::little);
   replaceLocValbyForm(DIEBldr, Die, AttrInfo, AttrInfo.getForm(), EntryOffset);
 }
 
@@ -701,7 +707,7 @@ static void writeDWARF5LocList(uint32_t &NumberOfEntries, DIEValue &AttrInfo,
     const DebugLocationEntry &Entry = LocList[I];
     support::endian::write(LocBodyStream,
                            static_cast<uint8_t>(dwarf::DW_LLE_startx_length),
-                           support::little);
+                           llvm::endianness::little);
     const uint32_t Index = AddrWriter.getIndexFromAddress(Entry.LowPC, CU);
     encodeULEB128(Index, LocBodyStream);
     encodeULEB128(Entry.HighPC - Entry.LowPC, LocBodyStream);
@@ -713,7 +719,7 @@ static void writeDWARF5LocList(uint32_t &NumberOfEntries, DIEValue &AttrInfo,
   if (WrittenStartxLength)
     support::endian::write(LocBodyStream,
                            static_cast<uint8_t>(dwarf::DW_LLE_end_of_list),
-                           support::little);
+                           llvm::endianness::little);
 }
 
 void DebugLoclistWriter::addList(DIEBuilder &DIEBldr, DIE &Die,
@@ -753,7 +759,7 @@ void DebugLoclistWriter::finalizeDWARF5(DIEBuilder &DIEBldr, DIE &Die) {
     support::endian::write(
         *LocArrayStream,
         static_cast<uint32_t>(SizeOfArraySection + RelativeOffset),
-        support::little);
+        llvm::endianness::little);
 
   std::unique_ptr<DebugBufferVector> Header = getDWARF5Header(
       {static_cast<uint32_t>(SizeOfArraySection + LocBodyBuffer.get()->size()),
@@ -884,11 +890,11 @@ void DebugStrOffsetsWriter::finalizeSection(DWARFUnit &Unit,
   if (RetVal == ProcessedBaseOffsets.end() || StrOffsetSectionWasModified) {
     // Writing out the header for each section.
     support::endian::write(*StrOffsetsStream, CurrentSectionSize + 4,
-                           support::little);
+                           llvm::endianness::little);
     support::endian::write(*StrOffsetsStream, static_cast<uint16_t>(5),
-                           support::little);
+                           llvm::endianness::little);
     support::endian::write(*StrOffsetsStream, static_cast<uint16_t>(0),
-                           support::little);
+                           llvm::endianness::little);
 
     uint64_t BaseOffset = StrOffsetsBuffer->size();
     ProcessedBaseOffsets[*Val] = BaseOffset;
@@ -897,7 +903,8 @@ void DebugStrOffsetsWriter::finalizeSection(DWARFUnit &Unit,
                            StrListBaseAttrInfo.getForm(),
                            DIEInteger(BaseOffset));
     for (const auto &Entry : IndexToAddressMap)
-      support::endian::write(*StrOffsetsStream, Entry.second, support::little);
+      support::endian::write(*StrOffsetsStream, Entry.second,
+                             llvm::endianness::little);
   } else {
     DIEBldr.replaceValue(&Die, dwarf::DW_AT_str_offsets_base,
                          StrListBaseAttrInfo.getForm(),
diff --git a/bolt/lib/Core/HashUtilities.cpp b/bolt/lib/Core/HashUtilities.cpp
index 88f01e4f93..6c7570dcc4 100644
--- a/bolt/lib/Core/HashUtilities.cpp
+++ b/bolt/lib/Core/HashUtilities.cpp
@@ -155,10 +155,7 @@ std::string hashBlockLoose(BinaryContext &BC, const BinaryBasicBlock &BB) {
     }
 
     std::string Mnemonic = BC.InstPrinter->getMnemonic(&Inst).first;
-    Mnemonic.erase(
-        std::remove_if(Mnemonic.begin(), Mnemonic.end(),
-                       [](unsigned char ch) { return std::isspace(ch); }),
-        Mnemonic.end());
+    llvm::erase_if(Mnemonic, [](unsigned char ch) { return std::isspace(ch); });
     Opcodes.insert(Mnemonic);
   }
 
diff --git a/bolt/lib/Core/MCPlusBuilder.cpp b/bolt/lib/Core/MCPlusBuilder.cpp
index 0a5eb44e48..036fcf8b3e 100644
--- a/bolt/lib/Core/MCPlusBuilder.cpp
+++ b/bolt/lib/Core/MCPlusBuilder.cpp
@@ -274,8 +274,10 @@ std::optional<MCSymbol *> MCPlusBuilder::getLabel(const MCInst &Inst) const {
   return std::nullopt;
 }
 
-bool MCPlusBuilder::setLabel(MCInst &Inst, MCSymbol *Label) {
-  getOrCreateAnnotationAs<MCSymbol *>(Inst, MCAnnotation::kLabel) = Label;
+bool MCPlusBuilder::setLabel(MCInst &Inst, MCSymbol *Label,
+                             AllocatorIdTy AllocatorId) {
+  getOrCreateAnnotationAs<MCSymbol *>(Inst, MCAnnotation::kLabel, AllocatorId) =
+      Label;
   return true;
 }
 
diff --git a/bolt/lib/Passes/Aligner.cpp b/bolt/lib/Passes/Aligner.cpp
index c5b63d881e..7c38752543 100644
--- a/bolt/lib/Passes/Aligner.cpp
+++ b/bolt/lib/Passes/Aligner.cpp
@@ -158,22 +158,6 @@ void AlignerPass::runOnFunctions(BinaryContext &BC) {
     BinaryContext::IndependentCodeEmitter Emitter =
         BC.createIndependentMCCodeEmitter();
 
-    // Align objects that contains constant islands and no code
-    // to at least 8 bytes.
-    if (!BF.size() && BF.hasIslandsInfo()) {
-      uint16_t Alignment = BF.getConstantIslandAlignment();
-      // Check if we're forcing output alignment and it is greater than minimal
-      // CI required one
-      if (!opts::UseCompactAligner && Alignment < opts::AlignFunctions &&
-          opts::AlignFunctions <= opts::AlignFunctionsMaxBytes)
-        Alignment = opts::AlignFunctions;
-
-      BF.setAlignment(Alignment);
-      BF.setMaxAlignmentBytes(Alignment);
-      BF.setMaxColdAlignmentBytes(Alignment);
-      return;
-    }
-
     if (opts::UseCompactAligner)
       alignCompact(BF, Emitter.MCE.get());
     else
diff --git a/bolt/lib/Passes/BinaryPasses.cpp b/bolt/lib/Passes/BinaryPasses.cpp
index 3ba53d7b2b..e50fa9dea6 100644
--- a/bolt/lib/Passes/BinaryPasses.cpp
+++ b/bolt/lib/Passes/BinaryPasses.cpp
@@ -619,8 +619,11 @@ void LowerAnnotations::runOnFunctions(BinaryContext &BC) {
   }
   for (BinaryFunction *BF : BC.getInjectedBinaryFunctions())
     for (BinaryBasicBlock &BB : *BF)
-      for (MCInst &Instruction : BB)
+      for (MCInst &Instruction : BB) {
+        if (auto Label = BC.MIB->getLabel(Instruction))
+          PreservedLabelAnnotations.emplace_back(&Instruction, *Label);
         BC.MIB->stripAnnotations(Instruction);
+      }
 
   // Release all memory taken by annotations
   BC.MIB->freeAnnotations();
diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp
index 6f4d1170db..a81689bc37 100644
--- a/bolt/lib/Passes/LongJmp.cpp
+++ b/bolt/lib/Passes/LongJmp.cpp
@@ -293,7 +293,7 @@ uint64_t LongJmpPass::tentativeLayoutRelocColdPart(
   for (BinaryFunction *Func : SortedFunctions) {
     if (!Func->isSplit())
       continue;
-    DotAddress = alignTo(DotAddress, BinaryFunction::MinAlign);
+    DotAddress = alignTo(DotAddress, Func->getMinAlignment());
     uint64_t Pad =
         offsetToAlignment(DotAddress, llvm::Align(Func->getAlignment()));
     if (Pad <= Func->getMaxColdAlignmentBytes())
@@ -352,7 +352,7 @@ uint64_t LongJmpPass::tentativeLayoutRelocMode(
         DotAddress = alignTo(DotAddress, opts::AlignText);
     }
 
-    DotAddress = alignTo(DotAddress, BinaryFunction::MinAlign);
+    DotAddress = alignTo(DotAddress, Func->getMinAlignment());
     uint64_t Pad =
         offsetToAlignment(DotAddress, llvm::Align(Func->getAlignment()));
     if (Pad <= Func->getMaxAlignmentBytes())
diff --git a/bolt/lib/Passes/ValidateMemRefs.cpp b/bolt/lib/Passes/ValidateMemRefs.cpp
index 69cd82af16..3324776830 100644
--- a/bolt/lib/Passes/ValidateMemRefs.cpp
+++ b/bolt/lib/Passes/ValidateMemRefs.cpp
@@ -34,8 +34,7 @@ bool ValidateMemRefs::checkAndFixJTReference(BinaryFunction &BF, MCInst &Inst,
   if (!JT)
     return false;
 
-  const bool IsLegitAccess = llvm::any_of(
-      JT->Parents, [&](const BinaryFunction *Parent) { return Parent == &BF; });
+  const bool IsLegitAccess = llvm::is_contained(JT->Parents, &BF);
   if (IsLegitAccess)
     return true;
 
diff --git a/bolt/lib/Profile/DataAggregator.cpp b/bolt/lib/Profile/DataAggregator.cpp
index f8ade24bdc..b72bd0edf1 100644
--- a/bolt/lib/Profile/DataAggregator.cpp
+++ b/bolt/lib/Profile/DataAggregator.cpp
@@ -181,15 +181,13 @@ void DataAggregator::start() {
                     "script -F pid,event,addr,ip",
                     /*Wait = */false);
 
-  launchPerfProcess("process events",
-                    MMapEventsPPI,
-                    "script --show-mmap-events",
-                    /*Wait = */false);
+  launchPerfProcess("process events", MMapEventsPPI,
+                    "script --show-mmap-events --no-itrace",
+                    /*Wait = */ false);
 
-  launchPerfProcess("task events",
-                    TaskEventsPPI,
-                    "script --show-task-events",
-                    /*Wait = */false);
+  launchPerfProcess("task events", TaskEventsPPI,
+                    "script --show-task-events --no-itrace",
+                    /*Wait = */ false);
 }
 
 void DataAggregator::abort() {
diff --git a/bolt/lib/Rewrite/PseudoProbeRewriter.cpp b/bolt/lib/Rewrite/PseudoProbeRewriter.cpp
index 316b83cfbd..51038dbead 100644
--- a/bolt/lib/Rewrite/PseudoProbeRewriter.cpp
+++ b/bolt/lib/Rewrite/PseudoProbeRewriter.cpp
@@ -248,7 +248,8 @@ void PseudoProbeRewriter::encodePseudoProbes() {
   auto EmitInt = [&](uint64_t Value, uint32_t Size) {
     const bool IsLittleEndian = BC.AsmInfo->isLittleEndian();
     uint64_t Swapped = support::endian::byte_swap(
-        Value, IsLittleEndian ? support::little : support::big);
+        Value,
+        IsLittleEndian ? llvm::endianness::little : llvm::endianness::big);
     unsigned Index = IsLittleEndian ? 0 : 8 - Size;
     auto Entry = StringRef(reinterpret_cast<char *>(&Swapped) + Index, Size);
     Contents.append(Entry.begin(), Entry.end());
diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index ddcc21878a..b3de3b96b3 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -1582,6 +1582,12 @@ void RewriteInstance::adjustFunctionBoundaries() {
       if (!Function.isSymbolValidInScope(Symbol, SymbolSize))
         break;
 
+      // Ignore unnamed symbols. Used, for example, by debugging info on RISC-V.
+      if (BC->isRISCV() && cantFail(Symbol.getName()).empty()) {
+        ++NextSymRefI;
+        continue;
+      }
+
       // Skip basic block labels. This happens on RISC-V with linker relaxation
       // enabled because every branch needs a relocation and corresponding
       // symbol. We don't want to add such symbols as entry points.
diff --git a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
index af7645f568..85855fbf3a 100644
--- a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
+++ b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
@@ -31,6 +31,33 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
 public:
   using MCPlusBuilder::MCPlusBuilder;
 
+  bool equals(const MCTargetExpr &A, const MCTargetExpr &B,
+              CompFuncTy Comp) const override {
+    const auto &RISCVExprA = cast<RISCVMCExpr>(A);
+    const auto &RISCVExprB = cast<RISCVMCExpr>(B);
+    if (RISCVExprA.getKind() != RISCVExprB.getKind())
+      return false;
+
+    return MCPlusBuilder::equals(*RISCVExprA.getSubExpr(),
+                                 *RISCVExprB.getSubExpr(), Comp);
+  }
+
+  void getCalleeSavedRegs(BitVector &Regs) const override {
+    Regs |= getAliases(RISCV::X2);
+    Regs |= getAliases(RISCV::X8);
+    Regs |= getAliases(RISCV::X9);
+    Regs |= getAliases(RISCV::X18);
+    Regs |= getAliases(RISCV::X19);
+    Regs |= getAliases(RISCV::X20);
+    Regs |= getAliases(RISCV::X21);
+    Regs |= getAliases(RISCV::X22);
+    Regs |= getAliases(RISCV::X23);
+    Regs |= getAliases(RISCV::X24);
+    Regs |= getAliases(RISCV::X25);
+    Regs |= getAliases(RISCV::X26);
+    Regs |= getAliases(RISCV::X27);
+  }
+
   bool shouldRecordCodeRelocation(uint64_t RelType) const override {
     switch (RelType) {
     case ELF::R_RISCV_JAL:
@@ -292,6 +319,7 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
     default:
       return false;
     case RISCV::C_J:
+    case TargetOpcode::EH_LABEL:
       OpNum = 0;
       return true;
     case RISCV::AUIPC:
diff --git a/bolt/test/AArch64/bf_min_alignment.s b/bolt/test/AArch64/bf_min_alignment.s
new file mode 100644
index 0000000000..2dd06b373a
--- /dev/null
+++ b/bolt/test/AArch64/bf_min_alignment.s
@@ -0,0 +1,35 @@
+// This tests checks the minimum alignment of the AARch64 function
+// is equal to 4. Otherwise the jitlinker would fail to link the
+// binary since the size of the first function after reorder is not
+// not a multiple of 4.
+
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %s -o %t.o
+# RUN: %clang %cflags -fPIC -pie %t.o -o %t.exe -nostdlib -Wl,-q
+# RUN: link_fdata %s %t.o %t.fdata
+# RUN: llvm-bolt %t.exe -o %t.bolt --use-old-text=0 --lite=0 \
+# RUN:   --align-functions-max-bytes=1 \
+# RUN:   --data %t.fdata --reorder-functions=exec-count
+# RUN: llvm-nm -n %t.bolt | FileCheck %s
+
+# CHECK: {{0|4|8|c}} T dummy
+# CHECK-NEXT: {{0|4|8|c}} T _start
+
+  .text
+  .align 4
+  .global _start
+  .type _start, %function
+_start:
+# FDATA: 0 [unknown] 0 1 _start 0 0 1
+   bl dymmy
+   ret
+  .size _start, .-_start
+
+  .global dummy
+  .type dummy, %function
+dummy:
+# FDATA: 0 [unknown] 0 1 dummy 0 0 42
+  adr x0, .Lci
+  ret
+.Lci:
+  .byte 0
+  .size dummy, .-dummy
diff --git a/bolt/test/RISCV/Inputs/cie-gnu.yaml b/bolt/test/RISCV/Inputs/cie-gnu.yaml
new file mode 100644
index 0000000000..7e05636ef8
--- /dev/null
+++ b/bolt/test/RISCV/Inputs/cie-gnu.yaml
@@ -0,0 +1,109 @@
+## Compiled and stripped-down version of:
+## (riscv64-linux-gnu-gcc -nostdlib -static -Wl,-q cie-gnu.s)
+#     .text
+#     .globl _start
+#     .type _start, @function
+# _start:
+#     .cfi_startproc
+#     beq a0, a1, 1f
+#     ret
+# 1:
+#     .cfi_undefined t0 # Arbitrary cfi command to force a new state
+#     ret
+#     .cfi_endproc
+#     .size _start, .-_start
+
+--- !ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_EXEC
+  Machine:         EM_RISCV
+  Flags:           [ EF_RISCV_RVC, EF_RISCV_FLOAT_ABI_DOUBLE ]
+  Entry:           0x10144
+ProgramHeaders:
+  - Type:            PT_LOAD
+    Flags:           [ PF_X, PF_R ]
+    FirstSec:        .text
+    LastSec:         .eh_frame
+    VAddr:           0x10000
+    Align:           0x1000
+    Offset:          0x0
+Sections:
+  - Name:            .text
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_ALLOC, SHF_EXECINSTR ]
+    Address:         0x10144
+    AddressAlign:    0x2
+    Offset:          0x144
+    Content:         6303B50082808280
+  - Name:            .eh_frame
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_ALLOC ]
+    Address:         0x10150
+    AddressAlign:    0x8
+    Content:         1000000000000000037A5200017C01011B0D02001000000018000000D8FFFFFF0800000000460705
+  - Name:            .rela.text
+    Type:            SHT_RELA
+    Flags:           [ SHF_INFO_LINK ]
+    Link:            .symtab
+    AddressAlign:    0x8
+    Info:            .text
+    Relocations:
+      - Offset:          0x10144
+        Symbol:          ".L1\x021"
+        Type:            R_RISCV_BRANCH
+  - Name:            .rela.eh_frame
+    Type:            SHT_RELA
+    Flags:           [ SHF_INFO_LINK ]
+    Link:            .symtab
+    AddressAlign:    0x8
+    Info:            .eh_frame
+    Relocations:
+      - Offset:          0x1016C
+        Symbol:          '.L0 '
+        Type:            R_RISCV_32_PCREL
+      - Offset:          0x10170
+        Symbol:          '.L0  (1)'
+        Type:            R_RISCV_ADD32
+      - Offset:          0x10170
+        Symbol:          '.L0 '
+        Type:            R_RISCV_SUB32
+      - Offset:          0x10175
+        Symbol:          '.L0  (2)'
+        Type:            R_RISCV_SET6
+      - Offset:          0x10175
+        Symbol:          '.L0 '
+        Type:            R_RISCV_SUB6
+  - Type:            SectionHeaderTable
+    Sections:
+      - Name:            .text
+      - Name:            .rela.text
+      - Name:            .eh_frame
+      - Name:            .rela.eh_frame
+      - Name:            .symtab
+      - Name:            .strtab
+      - Name:            .shstrtab
+Symbols:
+  - Name:            '$x'
+    Section:         .text
+    Value:           0x10144
+  - Name:            ".L1\x021"
+    Section:         .text
+    Value:           0x1014A
+  - Name:            '.L0 '
+    Section:         .text
+    Value:           0x10144
+  - Name:            '.L0  (1)'
+    Section:         .text
+    Value:           0x1014C
+  - Name:            '.L0  (2)'
+    Section:         .text
+    Value:           0x1014A
+  - Name:            _start
+    Type:            STT_FUNC
+    Section:         .text
+    Binding:         STB_GLOBAL
+    Value:           0x10144
+    Size:            0x8
+...
diff --git a/bolt/test/RISCV/Inputs/mapping-syms-isa.yaml b/bolt/test/RISCV/Inputs/mapping-syms-isa.yaml
new file mode 100644
index 0000000000..a47ecfde5d
--- /dev/null
+++ b/bolt/test/RISCV/Inputs/mapping-syms-isa.yaml
@@ -0,0 +1,47 @@
+--- !ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_EXEC
+  Machine:         EM_RISCV
+  Flags:           [ EF_RISCV_RVC, EF_RISCV_FLOAT_ABI_DOUBLE ]
+  Entry:           0x100B0
+ProgramHeaders:
+  - Type:            0x70000003
+    Flags:           [ PF_R ]
+    FirstSec:        .riscv.attributes
+    LastSec:         .riscv.attributes
+    Offset:          0xB8
+  - Type:            PT_LOAD
+    Flags:           [ PF_X, PF_R ]
+    FirstSec:        .text
+    LastSec:         .text
+    VAddr:           0x10000
+    Align:           0x1000
+    Offset:          0x0
+Sections:
+  - Name:            .text
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_ALLOC, SHF_EXECINSTR ]
+    Address:         0x100B0
+    AddressAlign:    0x2
+    Content:         '0100000000008280'
+  - Name:            .riscv.attributes
+    Type:            SHT_RISCV_ATTRIBUTES
+    AddressAlign:    0x1
+    Content:         4144000000726973637600013A0000000572763634693270315F6D3270305F613270315F663270325F643270325F633270305F7A696373723270305F7A6D6D756C31703000
+Symbols:
+  - Name:            '_start'
+    Section:         .text
+    Binding:         STB_GLOBAL
+    Value:           0x100B0
+  - Name:            '$xrv64i2p1_m2p0_a2p1_f2p2_d2p2_c2p0_zicsr2p0_zmmul1p0'
+    Section:         .text
+    Value:           0x100B0
+  - Name:            '$d'
+    Section:         .text
+    Value:           0x100B2
+  - Name:            '$x'
+    Section:         .text
+    Value:           0x100B6
+...
diff --git a/bolt/test/RISCV/call-annotations.s b/bolt/test/RISCV/call-annotations.s
index bc539bb0ec..477c4693cd 100644
--- a/bolt/test/RISCV/call-annotations.s
+++ b/bolt/test/RISCV/call-annotations.s
@@ -1,12 +1,13 @@
 /// Test that annotations are properly carried over to fixed calls.
 /// Note that --enable-bat is used to force offsets to be kept.
 
-// RUN: llvm-mc -triple riscv64 -filetype obj -o %t.o %s
+// RUN: llvm-mc -triple riscv64 -mattr=+c -filetype obj -o %t.o %s
 // RUN: ld.lld --emit-relocs -o %t %t.o
 // RUN: llvm-bolt --enable-bat --print-cfg --print-fix-riscv-calls \
 // RUN:     -o /dev/null %t | FileCheck %s
 
   .text
+  .option norvc
   .global f
   .p2align 1
 f:
diff --git a/bolt/test/RISCV/cie-gnu.test b/bolt/test/RISCV/cie-gnu.test
new file mode 100644
index 0000000000..deea8027ef
--- /dev/null
+++ b/bolt/test/RISCV/cie-gnu.test
@@ -0,0 +1,17 @@
+# Test that BOLT can handle CIE's produced by GNU as. On RISC-V, GNU as produces
+# the following initial instruction:
+# DW_CFA_def_cfa_register: r2
+# While I believe it is technically incorrect to use this instruction without
+# first using a DW_CFA_def_cfa (since the offset is unspecified), both readelf
+# and llvm-dwarfdump accept this and implicitly set the offset to 0.
+# In BOLT, this used to trigger an assert, however, since it (correctly)
+# believed the offset was not set. This test checks we can handle this
+# situation.
+
+RUN: yaml2obj -o %t %p/Inputs/cie-gnu.yaml
+RUN: llvm-bolt -o %t.bolt %t --reorder-blocks=reverse
+RUN: llvm-dwarfdump --debug-frame %t.bolt | FileCheck %s
+
+CHECK: 0x400000: CFA=X2
+CHECK: 0x400004: CFA=X2: X5=undefined
+CHECK: 0x400006: CFA=X2
diff --git a/bolt/test/RISCV/internal-func-reloc.s b/bolt/test/RISCV/internal-func-reloc.s
index ea99b564fe..c6c74bed92 100644
--- a/bolt/test/RISCV/internal-func-reloc.s
+++ b/bolt/test/RISCV/internal-func-reloc.s
@@ -2,18 +2,14 @@
 /// get transformed by BOLT. The tests rely on the "remove-nops" optimization:
 /// if nops got removed from the function, it got transformed by BOLT.
 
-// RUN: %clang %cflags -o %t %s
+// RUN: llvm-mc -triple riscv64 -filetype=obj -o %t.o %s
+// RUN: ld.lld --emit-relocs -o %t %t.o
 // RUN: llvm-bolt -o %t.bolt %t
 // RUN: llvm-objdump -d %t.bolt | FileCheck %s
 
   .text
-
-  /// These options are only used to make the assembler output easier to predict
-  .option norelax
-  .option norvc
-
   .globl _start
-  .p2align 1
+  .p2align 2
 // CHECK: <_start>:
 // CHECK-NEXT: j 0x{{.*}} <_start>
 _start:
@@ -23,10 +19,10 @@ _start:
   .size _start, .-_start
 
   .globl f
-  .p2align 1
+  .p2align 2
 // CHECK: <f>:
-// CHECK-NEXT: auipc a0, 0
-// CHECK-NEXT: addi a0, a0, 64
+// CHECK-NEXT: auipc a0, [[#]]
+// CHECK-NEXT: addi a0, a0, [[#]]
 f:
   nop
 1:
@@ -37,7 +33,7 @@ f:
   .size f, .-f
 
   .globl g
-  .p2align 1
+  .p2align 2
 g:
   ret
   .size g, .-g
diff --git a/bolt/test/RISCV/mapping-syms-isa.test b/bolt/test/RISCV/mapping-syms-isa.test
new file mode 100644
index 0000000000..22678af12f
--- /dev/null
+++ b/bolt/test/RISCV/mapping-syms-isa.test
@@ -0,0 +1,18 @@
+# Test that BOLT handles mapping syms that include ISA strings: $x<isa>
+
+RUN: yaml2obj -o %t %p/Inputs/mapping-syms-isa.yaml
+RUN: llvm-bolt --print-cfg --print-only=_start -o %t.bolt %t 2>&1 | FileCheck %s
+RUN: llvm-objdump -d %t.bolt | FileCheck --check-prefix=CHECK-OBJDUMP %s
+
+CHECK-NOT: BOLT-WARNING
+
+# Check that .word is not disassembled by BOLT
+CHECK: 00000000: nop
+CHECK: 00000002: ret
+
+# Check .word is still present in output
+CHECK-OBJDUMP: <_start>:
+CHECK-OBJDUMP-NEXT: nop
+CHECK-OBJDUMP-NEXT: unimp
+CHECK-OBJDUMP-NEXT: unimp
+CHECK-OBJDUMP-NEXT: ret
diff --git a/bolt/test/RISCV/unnamed-sym-no-entry.c b/bolt/test/RISCV/unnamed-sym-no-entry.c
new file mode 100644
index 0000000000..605bbc00ae
--- /dev/null
+++ b/bolt/test/RISCV/unnamed-sym-no-entry.c
@@ -0,0 +1,18 @@
+/// Verify that unnamed symbols are not added as function entry points. Such
+/// symbols are used by relocations in debugging sections.
+
+// clang-format off
+
+// RUN: %clang %cflags -g -Wl,-q -o %t %s
+
+/// Verify that the binary indeed contains an unnamed symbol at _start
+// RUN: llvm-readelf -s %t | FileCheck %s --check-prefix=CHECK-ELF
+// CHECK-ELF-DAG: [[#%x,START:]] {{.*}} FUNC GLOBAL DEFAULT [[#%d,SECTION:]] _start{{$}}
+// CHECK-ELF-DAG: [[#%x,START]] {{.*}} NOTYPE LOCAL DEFAULT [[#SECTION]] {{$}}
+
+/// Verify that BOLT did not create an extra entry point for the unnamed symbol
+// RUN: llvm-bolt -o %t.bolt %t --print-cfg | FileCheck %s
+// CHECK: Binary Function "_start" after building cfg {
+// CHECK:  IsMultiEntry: 0
+
+void _start() {}
diff --git a/bolt/test/checkvma-large-section.test b/bolt/test/X86/checkvma-large-section.test
similarity index 88%
rename from bolt/test/checkvma-large-section.test
rename to bolt/test/X86/checkvma-large-section.test
index 36a9159511..afa44111ea 100644
--- a/bolt/test/checkvma-large-section.test
+++ b/bolt/test/X86/checkvma-large-section.test
@@ -19,7 +19,7 @@ ProgramHeaders:
     Flags: [ PF_R, PF_W ]
     FirstSec: .large_sec
     LastSec: .large_sec
-    VAddr: 0x4a0279a8
+    VAddr: 0x80000000
   - Type: PT_GNU_RELRO
     Flags: [ PF_R ]
 Sections:
@@ -28,8 +28,8 @@ Sections:
     Content: 00
     AddressAlign: 0x1
   - Name: .large_sec
-    Type: SHT_PROGBITS
+    Type: SHT_NOBITS
     Flags: [ SHF_WRITE, SHF_ALLOC ]
-    Address: 0x4a0279a8
-    Size: 0xdf8bb1a0
+    Address: 0x80000000
+    Size: 0x80000000
 ...
diff --git a/bolt/test/X86/internal-call-instrument-so.s b/bolt/test/X86/internal-call-instrument-so.s
index b8903fc7f8..d13c828f60 100644
--- a/bolt/test/X86/internal-call-instrument-so.s
+++ b/bolt/test/X86/internal-call-instrument-so.s
@@ -1,6 +1,6 @@
 # This reproduces a bug with instrumentation crashes on internal call
 
-# REQUIRES: system-linux,bolt-runtime
+# REQUIRES: system-linux,bolt-runtime,target=x86_64{{.*}}
 
 # RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %s -o %t.o
 # Delete our BB symbols so BOLT doesn't mark them as entry points
@@ -41,7 +41,6 @@ _start:
   retq
   .size _start, .-_start
 
-
   .globl  _fini
   .type _fini, %function
   .p2align  4
diff --git a/bolt/test/reorder-data-writable-ptload.c b/bolt/test/reorder-data-writable-ptload.c
index 7b384e9655..fa99187794 100644
--- a/bolt/test/reorder-data-writable-ptload.c
+++ b/bolt/test/reorder-data-writable-ptload.c
@@ -1,7 +1,9 @@
 // This test checks that reorder-data pass puts new hot .data section
 // to the writable segment.
 
-// RUN: %clang %cflags -O3 -nostdlib -Wl,-q %s -o %t.exe
+// Use -fPIC -pie to prevent the globals being put in .sdata instead of .data on
+// RISC-V.
+// RUN: %clang %cflags -fPIC -pie -O3 -nostdlib -Wl,-q %s -o %t.exe
 // RUN: llvm-bolt %t.exe -o %t.bolt --reorder-data=".data" \
 // RUN:   -data %S/Inputs/reorder-data-writable-ptload.fdata
 // RUN: llvm-readelf -SlW %t.bolt | FileCheck %s
diff --git a/clang-tools-extra/clang-tidy/bugprone/BugproneTidyModule.cpp b/clang-tools-extra/clang-tidy/bugprone/BugproneTidyModule.cpp
index 543c522899..7a91003736 100644
--- a/clang-tools-extra/clang-tidy/bugprone/BugproneTidyModule.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/BugproneTidyModule.cpp
@@ -16,6 +16,7 @@
 #include "BadSignalToKillThreadCheck.h"
 #include "BoolPointerImplicitConversionCheck.h"
 #include "BranchCloneCheck.h"
+#include "CastingThroughVoidCheck.h"
 #include "ComparePointerToMemberVirtualFunctionCheck.h"
 #include "CopyConstructorInitCheck.h"
 #include "DanglingHandleCheck.h"
@@ -104,6 +105,8 @@ class BugproneModule : public ClangTidyModule {
     CheckFactories.registerCheck<BoolPointerImplicitConversionCheck>(
         "bugprone-bool-pointer-implicit-conversion");
     CheckFactories.registerCheck<BranchCloneCheck>("bugprone-branch-clone");
+    CheckFactories.registerCheck<CastingThroughVoidCheck>(
+        "bugprone-casting-through-void");
     CheckFactories.registerCheck<ComparePointerToMemberVirtualFunctionCheck>(
         "bugprone-compare-pointer-to-member-virtual-function");
     CheckFactories.registerCheck<CopyConstructorInitCheck>(
diff --git a/clang-tools-extra/clang-tidy/bugprone/CMakeLists.txt b/clang-tools-extra/clang-tidy/bugprone/CMakeLists.txt
index 0df9e439b7..d443fd8d14 100644
--- a/clang-tools-extra/clang-tidy/bugprone/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/bugprone/CMakeLists.txt
@@ -11,6 +11,7 @@ add_clang_library(clangTidyBugproneModule
   BoolPointerImplicitConversionCheck.cpp
   BranchCloneCheck.cpp
   BugproneTidyModule.cpp
+  CastingThroughVoidCheck.cpp
   ComparePointerToMemberVirtualFunctionCheck.cpp
   CopyConstructorInitCheck.cpp
   DanglingHandleCheck.cpp
diff --git a/clang-tools-extra/clang-tidy/bugprone/CastingThroughVoidCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/CastingThroughVoidCheck.cpp
new file mode 100644
index 0000000000..4c2416a89a
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/bugprone/CastingThroughVoidCheck.cpp
@@ -0,0 +1,45 @@
+//===--- CastingThroughVoidCheck.cpp - clang-tidy -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "CastingThroughVoidCheck.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/Expr.h"
+#include "clang/AST/Type.h"
+#include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/ASTMatchers/ASTMatchers.h"
+#include "llvm/ADT/StringSet.h"
+
+using namespace clang::ast_matchers;
+
+namespace clang::tidy::bugprone {
+
+void CastingThroughVoidCheck::registerMatchers(MatchFinder *Finder) {
+  Finder->addMatcher(
+      explicitCastExpr(
+          hasDestinationType(
+              qualType(unless(hasCanonicalType(pointsTo(voidType()))))
+                  .bind("target_type")),
+          hasSourceExpression(
+              explicitCastExpr(
+                  hasSourceExpression(
+                      expr(hasType(qualType().bind("source_type")))),
+                  hasDestinationType(
+                      qualType(pointsTo(voidType())).bind("void_type")))
+                  .bind("cast"))),
+      this);
+}
+
+void CastingThroughVoidCheck::check(const MatchFinder::MatchResult &Result) {
+  const auto TT = *Result.Nodes.getNodeAs<QualType>("target_type");
+  const auto ST = *Result.Nodes.getNodeAs<QualType>("source_type");
+  const auto VT = *Result.Nodes.getNodeAs<QualType>("void_type");
+  const auto *CE = Result.Nodes.getNodeAs<ExplicitCastExpr>("cast");
+  diag(CE->getExprLoc(), "do not cast %0 to %1 through %2") << ST << TT << VT;
+}
+
+} // namespace clang::tidy::bugprone
diff --git a/clang-tools-extra/clang-tidy/bugprone/CastingThroughVoidCheck.h b/clang-tools-extra/clang-tidy/bugprone/CastingThroughVoidCheck.h
new file mode 100644
index 0000000000..834676aaf0
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/bugprone/CastingThroughVoidCheck.h
@@ -0,0 +1,32 @@
+//===--- CastingThroughVoidCheck.h - clang-tidy -----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_BUGPRONE_CASTINGTHROUGHVOIDCHECK_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_BUGPRONE_CASTINGTHROUGHVOIDCHECK_H
+
+#include "../ClangTidyCheck.h"
+
+namespace clang::tidy::bugprone {
+
+/// Detects unsafe or redundant two-step casting operations involving ``void*``.
+/// For the user-facing documentation see:
+/// http://clang.llvm.org/extra/clang-tidy/checks/bugprone/casting-through-void.html
+class CastingThroughVoidCheck : public ClangTidyCheck {
+public:
+  CastingThroughVoidCheck(StringRef Name, ClangTidyContext *Context)
+      : ClangTidyCheck(Name, Context) {}
+  void registerMatchers(ast_matchers::MatchFinder *Finder) override;
+  void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
+  std::optional<TraversalKind> getCheckTraversalKind() const override {
+    return TK_IgnoreUnlessSpelledInSource;
+  }
+};
+
+} // namespace clang::tidy::bugprone
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_BUGPRONE_CASTINGTHROUGHVOIDCHECK_H
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/RvalueReferenceParamNotMovedCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/RvalueReferenceParamNotMovedCheck.cpp
index efcaffb45d..88b00dc174 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/RvalueReferenceParamNotMovedCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/RvalueReferenceParamNotMovedCheck.cpp
@@ -84,6 +84,9 @@ void RvalueReferenceParamNotMovedCheck::check(
   if (IgnoreUnnamedParams && Param->getName().empty())
     return;
 
+  if (!Param->isUsed() && Param->hasAttr<UnusedAttr>())
+    return;
+
   const auto *Function = dyn_cast<FunctionDecl>(Param->getDeclContext());
   if (!Function)
     return;
diff --git a/clang-tools-extra/clang-tidy/misc/CMakeLists.txt b/clang-tools-extra/clang-tidy/misc/CMakeLists.txt
index 2e88e68a54..d9ec268650 100644
--- a/clang-tools-extra/clang-tidy/misc/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/misc/CMakeLists.txt
@@ -18,6 +18,7 @@ add_custom_target(genconfusable DEPENDS Confusables.inc)
 
 add_clang_library(clangTidyMiscModule
   ConstCorrectnessCheck.cpp
+  CoroutineHostileRAIICheck.cpp
   DefinitionsInHeadersCheck.cpp
   ConfusableIdentifierCheck.cpp
   HeaderIncludeCycleCheck.cpp
diff --git a/clang-tools-extra/clang-tidy/misc/CoroutineHostileRAIICheck.cpp b/clang-tools-extra/clang-tidy/misc/CoroutineHostileRAIICheck.cpp
new file mode 100644
index 0000000000..e820cd39d8
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/misc/CoroutineHostileRAIICheck.cpp
@@ -0,0 +1,98 @@
+//===--- CoroutineHostileRAII.cpp - clang-tidy ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "CoroutineHostileRAIICheck.h"
+#include "../utils/OptionsUtils.h"
+#include "clang/AST/Attr.h"
+#include "clang/AST/Decl.h"
+#include "clang/AST/ExprCXX.h"
+#include "clang/AST/Stmt.h"
+#include "clang/AST/Type.h"
+#include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/ASTMatchers/ASTMatchers.h"
+#include "clang/ASTMatchers/ASTMatchersInternal.h"
+#include "clang/Basic/AttrKinds.h"
+#include "clang/Basic/DiagnosticIDs.h"
+
+using namespace clang::ast_matchers;
+namespace clang::tidy::misc {
+namespace {
+using clang::ast_matchers::internal::BoundNodesTreeBuilder;
+
+AST_MATCHER_P(Stmt, forEachPrevStmt, ast_matchers::internal::Matcher<Stmt>,
+              InnerMatcher) {
+  DynTypedNode P;
+  bool IsHostile = false;
+  for (const Stmt *Child = &Node; Child; Child = P.get<Stmt>()) {
+    auto Parents = Finder->getASTContext().getParents(*Child);
+    if (Parents.empty())
+      break;
+    P = *Parents.begin();
+    auto *PCS = P.get<CompoundStmt>();
+    if (!PCS)
+      continue;
+    for (const auto &Sibling : PCS->children()) {
+      // Child contains suspension. Siblings after Child do not persist across
+      // this suspension.
+      if (Sibling == Child)
+        break;
+      // In case of a match, add the bindings as a separate match. Also don't
+      // clear the bindings if a match is not found (unlike Matcher::matches).
+      BoundNodesTreeBuilder SiblingBuilder;
+      if (InnerMatcher.matches(*Sibling, Finder, &SiblingBuilder)) {
+        Builder->addMatch(SiblingBuilder);
+        IsHostile = true;
+      }
+    }
+  }
+  return IsHostile;
+}
+} // namespace
+
+CoroutineHostileRAIICheck::CoroutineHostileRAIICheck(StringRef Name,
+                                                     ClangTidyContext *Context)
+    : ClangTidyCheck(Name, Context),
+      RAIITypesList(utils::options::parseStringList(
+          Options.get("RAIITypesList", "std::lock_guard;std::scoped_lock"))) {}
+
+void CoroutineHostileRAIICheck::registerMatchers(MatchFinder *Finder) {
+  // A suspension happens with co_await or co_yield.
+  auto ScopedLockable = varDecl(hasType(hasCanonicalType(hasDeclaration(
+                                    hasAttr(attr::Kind::ScopedLockable)))))
+                            .bind("scoped-lockable");
+  auto OtherRAII = varDecl(hasType(hasCanonicalType(hasDeclaration(
+                               namedDecl(hasAnyName(RAIITypesList))))))
+                       .bind("raii");
+  Finder->addMatcher(expr(anyOf(coawaitExpr(), coyieldExpr()),
+                          forEachPrevStmt(declStmt(forEach(
+                              varDecl(anyOf(ScopedLockable, OtherRAII))))))
+                         .bind("suspension"),
+                     this);
+}
+
+void CoroutineHostileRAIICheck::check(const MatchFinder::MatchResult &Result) {
+  if (const auto *VD = Result.Nodes.getNodeAs<VarDecl>("scoped-lockable"))
+    diag(VD->getLocation(),
+         "%0 holds a lock across a suspension point of coroutine and could be "
+         "unlocked by a different thread")
+        << VD;
+  if (const auto *VD = Result.Nodes.getNodeAs<VarDecl>("raii"))
+    diag(VD->getLocation(),
+         "%0 persists across a suspension point of coroutine")
+        << VD;
+  if (const auto *Suspension = Result.Nodes.getNodeAs<Expr>("suspension"))
+    diag(Suspension->getBeginLoc(), "suspension point is here",
+         DiagnosticIDs::Note);
+}
+
+void CoroutineHostileRAIICheck::storeOptions(
+    ClangTidyOptions::OptionMap &Opts) {
+  Options.store(Opts, "RAIITypesList",
+                utils::options::serializeStringList(RAIITypesList));
+}
+} // namespace clang::tidy::misc
diff --git a/clang-tools-extra/clang-tidy/misc/CoroutineHostileRAIICheck.h b/clang-tools-extra/clang-tidy/misc/CoroutineHostileRAIICheck.h
new file mode 100644
index 0000000000..a5e9cb89ef
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/misc/CoroutineHostileRAIICheck.h
@@ -0,0 +1,50 @@
+//===--- CoroutineHostileRAIICheck.h - clang-tidy ----------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_MISC_COROUTINESHOSTILERAIICHECK_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_MISC_COROUTINESHOSTILERAIICHECK_H
+
+#include "../ClangTidyCheck.h"
+#include "clang/AST/ASTTypeTraits.h"
+#include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "llvm/ADT/StringRef.h"
+#include <vector>
+
+namespace clang::tidy::misc {
+
+/// Detects when objects of certain hostile RAII types persists across
+/// suspension points in a coroutine. Such hostile types include scoped-lockable
+/// types and types belonging to a configurable denylist.
+///
+///  For the user-facing documentation see:
+///  http://clang.llvm.org/extra/clang-tidy/checks/misc/coroutine-hostile-raii.html
+class CoroutineHostileRAIICheck : public ClangTidyCheck {
+public:
+  CoroutineHostileRAIICheck(llvm::StringRef Name, ClangTidyContext *Context);
+
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus20;
+  }
+
+  void registerMatchers(ast_matchers::MatchFinder *Finder) override;
+  void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
+  void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
+
+  std::optional<TraversalKind> getCheckTraversalKind() const override {
+    return TK_AsIs;
+  }
+
+private:
+  // List of fully qualified types which should not persist across a suspension
+  // point in a coroutine.
+  std::vector<StringRef> RAIITypesList;
+};
+
+} // namespace clang::tidy::misc
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_MISC_COROUTINESHOSTILERAIICHECK_H
diff --git a/clang-tools-extra/clang-tidy/misc/MiscTidyModule.cpp b/clang-tools-extra/clang-tidy/misc/MiscTidyModule.cpp
index 92590506e1..d8a88324ee 100644
--- a/clang-tools-extra/clang-tidy/misc/MiscTidyModule.cpp
+++ b/clang-tools-extra/clang-tidy/misc/MiscTidyModule.cpp
@@ -11,6 +11,7 @@
 #include "../ClangTidyModuleRegistry.h"
 #include "ConfusableIdentifierCheck.h"
 #include "ConstCorrectnessCheck.h"
+#include "CoroutineHostileRAIICheck.h"
 #include "DefinitionsInHeadersCheck.h"
 #include "HeaderIncludeCycleCheck.h"
 #include "IncludeCleanerCheck.h"
@@ -41,6 +42,8 @@ class MiscModule : public ClangTidyModule {
         "misc-confusable-identifiers");
     CheckFactories.registerCheck<ConstCorrectnessCheck>(
         "misc-const-correctness");
+    CheckFactories.registerCheck<CoroutineHostileRAIICheck>(
+        "misc-coroutine-hostile-raii");
     CheckFactories.registerCheck<DefinitionsInHeadersCheck>(
         "misc-definitions-in-headers");
     CheckFactories.registerCheck<HeaderIncludeCycleCheck>(
diff --git a/clang-tools-extra/clang-tidy/modernize/AvoidBindCheck.cpp b/clang-tools-extra/clang-tidy/modernize/AvoidBindCheck.cpp
index 4628c09f19..2a54eaddcc 100644
--- a/clang-tools-extra/clang-tidy/modernize/AvoidBindCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/AvoidBindCheck.cpp
@@ -96,6 +96,7 @@ struct CallableInfo {
   std::string UsageIdentifier;
   StringRef CaptureInitializer;
   const FunctionDecl *Decl = nullptr;
+  bool DoesReturn = false;
 };
 
 struct LambdaProperties {
@@ -554,6 +555,10 @@ getLambdaProperties(const MatchFinder::MatchResult &Result) {
   LP.Callable.Materialization = getCallableMaterialization(Result);
   LP.Callable.Decl =
       getCallMethodDecl(Result, LP.Callable.Type, LP.Callable.Materialization);
+  if (LP.Callable.Decl)
+    if (const Type *ReturnType =
+            LP.Callable.Decl->getReturnType().getCanonicalType().getTypePtr())
+      LP.Callable.DoesReturn = !ReturnType->isVoidType();
   LP.Callable.SourceTokens = getSourceTextForExpr(Result, CalleeExpr);
   if (LP.Callable.Materialization == CMK_VariableRef) {
     LP.Callable.CE = CE_Var;
@@ -672,15 +677,20 @@ void AvoidBindCheck::check(const MatchFinder::MatchResult &Result) {
 
   addPlaceholderArgs(LP, Stream, PermissiveParameterList);
 
+  Stream << " { ";
+
+  if (LP.Callable.DoesReturn) {
+    Stream << "return ";
+  }
+
   if (LP.Callable.Type == CT_Function) {
     StringRef SourceTokens = LP.Callable.SourceTokens;
     SourceTokens.consume_front("&");
-    Stream << " { return " << SourceTokens;
+    Stream << SourceTokens;
   } else if (LP.Callable.Type == CT_MemberFunction) {
     const auto *MethodDecl = dyn_cast<CXXMethodDecl>(LP.Callable.Decl);
     const BindArgument &ObjPtr = FunctionCallArgs.front();
 
-    Stream << " { ";
     if (!isa<CXXThisExpr>(ignoreTemporariesAndPointers(ObjPtr.E))) {
       Stream << ObjPtr.UsageIdentifier;
       Stream << "->";
@@ -688,7 +698,6 @@ void AvoidBindCheck::check(const MatchFinder::MatchResult &Result) {
 
     Stream << MethodDecl->getName();
   } else {
-    Stream << " { return ";
     switch (LP.Callable.CE) {
     case CE_Var:
       if (LP.Callable.UsageIdentifier != LP.Callable.CaptureIdentifier) {
diff --git a/clang-tools-extra/clang-tidy/modernize/LoopConvertCheck.cpp b/clang-tools-extra/clang-tidy/modernize/LoopConvertCheck.cpp
index f90d99a8d6..8beaa62c78 100644
--- a/clang-tools-extra/clang-tidy/modernize/LoopConvertCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/LoopConvertCheck.cpp
@@ -753,6 +753,7 @@ void LoopConvertCheck::doConversion(
   bool IsCheapToCopy =
       !Descriptor.ElemType.isNull() &&
       Descriptor.ElemType.isTriviallyCopyableType(*Context) &&
+      !Descriptor.ElemType->isDependentSizedArrayType() &&
       // TypeInfo::Width is in bits.
       Context->getTypeInfo(Descriptor.ElemType).Width <= 8 * MaxCopySize;
   bool UseCopy = CanCopy && ((VarNameFromAlias && !AliasVarIsRef) ||
diff --git a/clang-tools-extra/clang-tidy/tool/clang-tidy-diff.py b/clang-tools-extra/clang-tidy/tool/clang-tidy-diff.py
index 06f2cebe7c..8817e2914f 100755
--- a/clang-tools-extra/clang-tidy/tool/clang-tidy-diff.py
+++ b/clang-tools-extra/clang-tidy/tool/clang-tidy-diff.py
@@ -180,10 +180,21 @@ def main():
     if yaml:
         parser.add_argument(
             "-export-fixes",
-            metavar="FILE",
+            metavar="FILE_OR_DIRECTORY",
             dest="export_fixes",
-            help="Create a yaml file to store suggested fixes in, "
-            "which can be applied with clang-apply-replacements.",
+            help="A directory or a yaml file to store suggested fixes in, "
+            "which can be applied with clang-apply-replacements. If the "
+            "parameter is a directory, the fixes of each compilation unit are "
+            "stored in individual yaml files in the directory.",
+        )
+    else:
+        parser.add_argument(
+            "-export-fixes",
+            metavar="DIRECTORY",
+            dest="export_fixes",
+            help="A directory to store suggested fixes in, which can be applied "
+            "with clang-apply-replacements. The fixes of each compilation unit are "
+            "stored in individual yaml files in the directory.",
         )
     parser.add_argument(
         "-extra-arg",
@@ -258,9 +269,30 @@ def main():
         max_task_count = multiprocessing.cpu_count()
     max_task_count = min(len(lines_by_file), max_task_count)
 
-    tmpdir = None
-    if yaml and args.export_fixes:
-        tmpdir = tempfile.mkdtemp()
+    combine_fixes = False
+    export_fixes_dir = None
+    delete_fixes_dir = False
+    if args.export_fixes is not None:
+        # if a directory is given, create it if it does not exist
+        if args.export_fixes.endswith(os.path.sep) and not os.path.isdir(
+            args.export_fixes
+        ):
+            os.makedirs(args.export_fixes)
+
+        if not os.path.isdir(args.export_fixes):
+            if not yaml:
+                raise RuntimeError(
+                    "Cannot combine fixes in one yaml file. Either install PyYAML or specify an output directory."
+                )
+
+            combine_fixes = True
+
+        if os.path.isdir(args.export_fixes):
+            export_fixes_dir = args.export_fixes
+
+    if combine_fixes:
+        export_fixes_dir = tempfile.mkdtemp()
+        delete_fixes_dir = True
 
     # Tasks for clang-tidy.
     task_queue = queue.Queue(max_task_count)
@@ -302,10 +334,10 @@ def main():
         # Run clang-tidy on files containing changes.
         command = [args.clang_tidy_binary]
         command.append("-line-filter=" + line_filter_json)
-        if yaml and args.export_fixes:
+        if args.export_fixes is not None:
             # Get a temporary file. We immediately close the handle so clang-tidy can
             # overwrite it.
-            (handle, tmp_name) = tempfile.mkstemp(suffix=".yaml", dir=tmpdir)
+            (handle, tmp_name) = tempfile.mkstemp(suffix=".yaml", dir=export_fixes_dir)
             os.close(handle)
             command.append("-export-fixes=" + tmp_name)
         command.extend(common_clang_tidy_args)
@@ -324,17 +356,17 @@ def main():
     if failed_files:
         return_code = 1
 
-    if yaml and args.export_fixes:
+    if combine_fixes:
         print("Writing fixes to " + args.export_fixes + " ...")
         try:
-            merge_replacement_files(tmpdir, args.export_fixes)
+            merge_replacement_files(export_fixes_dir, args.export_fixes)
         except:
             sys.stderr.write("Error exporting fixes.\n")
             traceback.print_exc()
             return_code = 1
 
-    if tmpdir:
-        shutil.rmtree(tmpdir)
+    if delete_fixes_dir:
+        shutil.rmtree(export_fixes_dir)
     sys.exit(return_code)
 
 
diff --git a/clang-tools-extra/clang-tidy/tool/run-clang-tidy.py b/clang-tools-extra/clang-tidy/tool/run-clang-tidy.py
index 312d9241cf..1797592161 100755
--- a/clang-tools-extra/clang-tidy/tool/run-clang-tidy.py
+++ b/clang-tools-extra/clang-tidy/tool/run-clang-tidy.py
@@ -308,10 +308,21 @@ def main():
     if yaml:
         parser.add_argument(
             "-export-fixes",
-            metavar="filename",
+            metavar="file_or_directory",
             dest="export_fixes",
-            help="Create a yaml file to store suggested fixes in, "
-            "which can be applied with clang-apply-replacements.",
+            help="A directory or a yaml file to store suggested fixes in, "
+            "which can be applied with clang-apply-replacements. If the "
+            "parameter is a directory, the fixes of each compilation unit are "
+            "stored in individual yaml files in the directory.",
+        )
+    else:
+        parser.add_argument(
+            "-export-fixes",
+            metavar="directory",
+            dest="export_fixes",
+            help="A directory to store suggested fixes in, which can be applied "
+            "with clang-apply-replacements. The fixes of each compilation unit are "
+            "stored in individual yaml files in the directory.",
         )
     parser.add_argument(
         "-j",
@@ -384,14 +395,35 @@ def main():
 
     clang_tidy_binary = find_binary(args.clang_tidy_binary, "clang-tidy", build_path)
 
-    tmpdir = None
     if args.fix:
         clang_apply_replacements_binary = find_binary(
             args.clang_apply_replacements_binary, "clang-apply-replacements", build_path
         )
 
-    if args.fix or (yaml and args.export_fixes):
-        tmpdir = tempfile.mkdtemp()
+    combine_fixes = False
+    export_fixes_dir = None
+    delete_fixes_dir = False
+    if args.export_fixes is not None:
+        # if a directory is given, create it if it does not exist
+        if args.export_fixes.endswith(os.path.sep) and not os.path.isdir(
+            args.export_fixes
+        ):
+            os.makedirs(args.export_fixes)
+
+        if not os.path.isdir(args.export_fixes):
+            if not yaml:
+                raise RuntimeError(
+                    "Cannot combine fixes in one yaml file. Either install PyYAML or specify an output directory."
+                )
+
+            combine_fixes = True
+
+        if os.path.isdir(args.export_fixes):
+            export_fixes_dir = args.export_fixes
+
+    if export_fixes_dir is None and (args.fix or combine_fixes):
+        export_fixes_dir = tempfile.mkdtemp()
+        delete_fixes_dir = True
 
     try:
         invocation = get_tidy_invocation(
@@ -450,7 +482,7 @@ def main():
                 args=(
                     args,
                     clang_tidy_binary,
-                    tmpdir,
+                    export_fixes_dir,
                     build_path,
                     task_queue,
                     lock,
@@ -474,14 +506,14 @@ def main():
         # This is a sad hack. Unfortunately subprocess goes
         # bonkers with ctrl-c and we start forking merrily.
         print("\nCtrl-C detected, goodbye.")
-        if tmpdir:
-            shutil.rmtree(tmpdir)
+        if delete_fixes_dir:
+            shutil.rmtree(export_fixes_dir)
         os.kill(0, 9)
 
-    if yaml and args.export_fixes:
+    if combine_fixes:
         print("Writing fixes to " + args.export_fixes + " ...")
         try:
-            merge_replacement_files(tmpdir, args.export_fixes)
+            merge_replacement_files(export_fixes_dir, args.export_fixes)
         except:
             print("Error exporting fixes.\n", file=sys.stderr)
             traceback.print_exc()
@@ -490,14 +522,14 @@ def main():
     if args.fix:
         print("Applying fixes ...")
         try:
-            apply_fixes(args, clang_apply_replacements_binary, tmpdir)
+            apply_fixes(args, clang_apply_replacements_binary, export_fixes_dir)
         except:
             print("Error applying fixes.\n", file=sys.stderr)
             traceback.print_exc()
             return_code = 1
 
-    if tmpdir:
-        shutil.rmtree(tmpdir)
+    if delete_fixes_dir:
+        shutil.rmtree(export_fixes_dir)
     sys.exit(return_code)
 
 
diff --git a/clang-tools-extra/clangd/Config.h b/clang-tools-extra/clangd/Config.h
index daae8d1c0c..4371c80a6c 100644
--- a/clang-tools-extra/clangd/Config.h
+++ b/clang-tools-extra/clangd/Config.h
@@ -93,6 +93,7 @@ struct Config {
     Strict,
     None,
   };
+  enum class FastCheckPolicy { Strict, Loose, None };
   /// Controls warnings and errors when parsing code.
   struct {
     bool SuppressAll = false;
@@ -103,6 +104,7 @@ struct Config {
       // A comma-separated list of globs specify which clang-tidy checks to run.
       std::string Checks;
       llvm::StringMap<std::string> CheckOptions;
+      FastCheckPolicy FastCheckFilter = FastCheckPolicy::Strict;
     } ClangTidy;
 
     IncludesPolicy UnusedIncludes = IncludesPolicy::Strict;
diff --git a/clang-tools-extra/clangd/ConfigCompile.cpp b/clang-tools-extra/clangd/ConfigCompile.cpp
index d4ff7ae318..0c9fc27643 100644
--- a/clang-tools-extra/clangd/ConfigCompile.cpp
+++ b/clang-tools-extra/clangd/ConfigCompile.cpp
@@ -324,11 +324,11 @@ struct FragmentCompiler {
 
   void compile(Fragment::IndexBlock &&F) {
     if (F.Background) {
-      if (auto Val = compileEnum<Config::BackgroundPolicy>("Background",
-                                                           **F.Background)
-                         .map("Build", Config::BackgroundPolicy::Build)
-                         .map("Skip", Config::BackgroundPolicy::Skip)
-                         .value())
+      if (auto Val =
+              compileEnum<Config::BackgroundPolicy>("Background", *F.Background)
+                  .map("Build", Config::BackgroundPolicy::Build)
+                  .map("Skip", Config::BackgroundPolicy::Skip)
+                  .value())
         Out.Apply.push_back(
             [Val](const Params &, Config &C) { C.Index.Background = *Val; });
     }
@@ -494,11 +494,31 @@ struct FragmentCompiler {
       diag(Error, "Invalid clang-tidy check name", Arg.Range);
       return;
     }
-    if (!Str.contains('*') && !isRegisteredTidyCheck(Str)) {
-      diag(Warning,
-           llvm::formatv("clang-tidy check '{0}' was not found", Str).str(),
-           Arg.Range);
-      return;
+    if (!Str.contains('*')) {
+      if (!isRegisteredTidyCheck(Str)) {
+        diag(Warning,
+             llvm::formatv("clang-tidy check '{0}' was not found", Str).str(),
+             Arg.Range);
+        return;
+      }
+      auto Fast = isFastTidyCheck(Str);
+      if (!Fast.has_value()) {
+        diag(Warning,
+             llvm::formatv(
+                 "Latency of clang-tidy check '{0}' is not known. "
+                 "It will only run if ClangTidy.FastCheckFilter is Loose or None",
+                 Str)
+                 .str(),
+             Arg.Range);
+      } else if (!*Fast) {
+        diag(Warning,
+             llvm::formatv(
+                 "clang-tidy check '{0}' is slow. "
+                 "It will only run if ClangTidy.FastCheckFilter is None",
+                 Str)
+                 .str(),
+             Arg.Range);
+      }
     }
     CurSpec += ',';
     if (!IsPositive)
@@ -534,6 +554,16 @@ struct FragmentCompiler {
                   StringPair.first, StringPair.second);
           });
     }
+    if (F.FastCheckFilter.has_value())
+      if (auto Val = compileEnum<Config::FastCheckPolicy>("FastCheckFilter",
+                                                          *F.FastCheckFilter)
+                         .map("Strict", Config::FastCheckPolicy::Strict)
+                         .map("Loose", Config::FastCheckPolicy::Loose)
+                         .map("None", Config::FastCheckPolicy::None)
+                         .value())
+        Out.Apply.push_back([Val](const Params &, Config &C) {
+          C.Diagnostics.ClangTidy.FastCheckFilter = *Val;
+        });
   }
 
   void compile(Fragment::DiagnosticsBlock::IncludesBlock &&F) {
diff --git a/clang-tools-extra/clangd/ConfigFragment.h b/clang-tools-extra/clangd/ConfigFragment.h
index a59d4124b0..7fa61108c7 100644
--- a/clang-tools-extra/clangd/ConfigFragment.h
+++ b/clang-tools-extra/clangd/ConfigFragment.h
@@ -277,6 +277,13 @@ struct Fragment {
       ///     readability-braces-around-statements.ShortStatementLines: 2
       std::vector<std::pair<Located<std::string>, Located<std::string>>>
           CheckOptions;
+
+      /// Whether to run checks that may slow down clangd.
+      ///   Strict: Run only checks measured to be fast. (Default)
+      ///           This excludes recently-added checks we have not timed yet.
+      ///   Loose: Run checks unless they are known to be slow.
+      ///   None: Run checks regardless of their speed.
+      std::optional<Located<std::string>> FastCheckFilter;
     };
     ClangTidyBlock ClangTidy;
   };
diff --git a/clang-tools-extra/clangd/ConfigYAML.cpp b/clang-tools-extra/clangd/ConfigYAML.cpp
index cf3cec472b..ce09af8192 100644
--- a/clang-tools-extra/clangd/ConfigYAML.cpp
+++ b/clang-tools-extra/clangd/ConfigYAML.cpp
@@ -156,6 +156,10 @@ class Parser {
       });
       CheckOptDict.parse(N);
     });
+    Dict.handle("FastCheckFilter", [&](Node &N) {
+      if (auto FastCheckFilter = scalarValue(N, "FastCheckFilter"))
+        F.FastCheckFilter = *FastCheckFilter;
+    });
     Dict.parse(N);
   }
 
diff --git a/clang-tools-extra/clangd/Hover.cpp b/clang-tools-extra/clangd/Hover.cpp
index 0ec85fc24d..933c69294b 100644
--- a/clang-tools-extra/clangd/Hover.cpp
+++ b/clang-tools-extra/clangd/Hover.cpp
@@ -1001,6 +1001,8 @@ void addLayoutInfo(const NamedDecl &ND, HoverInfo &HI) {
   if (auto *RD = llvm::dyn_cast<RecordDecl>(&ND)) {
     if (auto Size = Ctx.getTypeSizeInCharsIfKnown(RD->getTypeForDecl()))
       HI.Size = Size->getQuantity() * 8;
+    if (!RD->isDependentType() && RD->isCompleteDefinition())
+      HI.Align = Ctx.getTypeAlign(RD->getTypeForDecl());
     return;
   }
 
@@ -1009,6 +1011,7 @@ void addLayoutInfo(const NamedDecl &ND, HoverInfo &HI) {
     if (Record)
       Record = Record->getDefinition();
     if (Record && !Record->isInvalidDecl() && !Record->isDependentType()) {
+      HI.Align = Ctx.getTypeAlign(FD->getType());
       const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(Record);
       HI.Offset = Layout.getFieldOffset(FD->getFieldIndex());
       if (FD->isBitField())
@@ -1487,6 +1490,8 @@ markup::Document HoverInfo::present() const {
       P.appendText(
           llvm::formatv(" (+{0} padding)", formatSize(*Padding)).str());
     }
+    if (Align)
+      P.appendText(", alignment " + formatSize(*Align));
   }
 
   if (CalleeArgInfo) {
diff --git a/clang-tools-extra/clangd/Hover.h b/clang-tools-extra/clangd/Hover.h
index 6a61100912..fe689de447 100644
--- a/clang-tools-extra/clangd/Hover.h
+++ b/clang-tools-extra/clangd/Hover.h
@@ -97,6 +97,8 @@ struct HoverInfo {
   std::optional<uint64_t> Offset;
   /// Contains the padding following a field within the enclosing class.
   std::optional<uint64_t> Padding;
+  /// Contains the alignment of fields and types where it's interesting.
+  std::optional<uint64_t> Align;
   // Set when symbol is inside function call. Contains information extracted
   // from the callee definition about the argument this is passed as.
   std::optional<Param> CalleeArgInfo;
diff --git a/clang-tools-extra/clangd/ParsedAST.cpp b/clang-tools-extra/clangd/ParsedAST.cpp
index d033d29e90..edd0f77b10 100644
--- a/clang-tools-extra/clangd/ParsedAST.cpp
+++ b/clang-tools-extra/clangd/ParsedAST.cpp
@@ -381,6 +381,20 @@ std::vector<Diag> getIncludeCleanerDiags(ParsedAST &AST, llvm::StringRef Code) {
                                         Cfg.Diagnostics.Includes.IgnoreHeader);
 }
 
+tidy::ClangTidyCheckFactories
+filterFastTidyChecks(const tidy::ClangTidyCheckFactories &All,
+                     Config::FastCheckPolicy Policy) {
+  if (Policy == Config::FastCheckPolicy::None)
+    return All;
+  bool AllowUnknown = Policy == Config::FastCheckPolicy::Loose;
+  tidy::ClangTidyCheckFactories Fast;
+  for (const auto &Factory : All) {
+    if (isFastTidyCheck(Factory.getKey()).value_or(AllowUnknown))
+      Fast.registerCheckFactory(Factory.first(), Factory.second);
+  }
+  return Fast;
+}
+
 } // namespace
 
 std::optional<ParsedAST>
@@ -390,6 +404,7 @@ ParsedAST::build(llvm::StringRef Filename, const ParseInputs &Inputs,
                  std::shared_ptr<const PreambleData> Preamble) {
   trace::Span Tracer("BuildAST");
   SPAN_ATTACH(Tracer, "File", Filename);
+  const Config &Cfg = Config::current();
 
   auto VFS = Inputs.TFS->view(Inputs.CompileCommand.Directory);
   if (Preamble && Preamble->StatCache)
@@ -520,19 +535,21 @@ ParsedAST::build(llvm::StringRef Filename, const ParseInputs &Inputs,
   // diagnostics.
   {
     trace::Span Tracer("ClangTidyInit");
-    static const auto *CTFactories = [] {
+    static const auto *AllCTFactories = [] {
       auto *CTFactories = new tidy::ClangTidyCheckFactories;
       for (const auto &E : tidy::ClangTidyModuleRegistry::entries())
         E.instantiate()->addCheckFactories(*CTFactories);
       return CTFactories;
     }();
+    tidy::ClangTidyCheckFactories FastFactories = filterFastTidyChecks(
+        *AllCTFactories, Cfg.Diagnostics.ClangTidy.FastCheckFilter);
     CTContext.emplace(std::make_unique<tidy::DefaultOptionsProvider>(
         tidy::ClangTidyGlobalOptions(), ClangTidyOpts));
     CTContext->setDiagnosticsEngine(&Clang->getDiagnostics());
     CTContext->setASTContext(&Clang->getASTContext());
     CTContext->setCurrentFile(Filename);
     CTContext->setSelfContainedDiags(true);
-    CTChecks = CTFactories->createChecksForLanguage(&*CTContext);
+    CTChecks = FastFactories.createChecksForLanguage(&*CTContext);
     Preprocessor *PP = &Clang->getPreprocessor();
     for (const auto &Check : CTChecks) {
       Check->registerPPCallbacks(Clang->getSourceManager(), PP, PP);
@@ -554,7 +571,6 @@ ParsedAST::build(llvm::StringRef Filename, const ParseInputs &Inputs,
                                           SourceLocation());
     }
 
-    const Config &Cfg = Config::current();
     ASTDiags.setLevelAdjuster([&](DiagnosticsEngine::Level DiagLevel,
                                   const clang::Diagnostic &Info) {
       if (Cfg.Diagnostics.SuppressAll ||
diff --git a/clang-tools-extra/clangd/SystemIncludeExtractor.cpp b/clang-tools-extra/clangd/SystemIncludeExtractor.cpp
index 74bae78642..68b80478e1 100644
--- a/clang-tools-extra/clangd/SystemIncludeExtractor.cpp
+++ b/clang-tools-extra/clangd/SystemIncludeExtractor.cpp
@@ -376,8 +376,7 @@ extractSystemIncludesAndTarget(const DriverArgs &InputArgs,
     auto Path = llvm::StringRef(*BuiltinHeaders).trim();
     if (!Path.empty() && llvm::sys::path::is_absolute(Path)) {
       auto Size = Info->SystemIncludes.size();
-      llvm::erase_if(Info->SystemIncludes,
-                     [&](llvm::StringRef Entry) { return Path == Entry; });
+      llvm::erase_value(Info->SystemIncludes, Path);
       vlog("System includes extractor: builtin headers {0} {1}", Path,
            (Info->SystemIncludes.size() != Size)
                ? "excluded"
diff --git a/clang-tools-extra/clangd/TidyProvider.cpp b/clang-tools-extra/clangd/TidyProvider.cpp
index f101199a20..b658a80559 100644
--- a/clang-tools-extra/clangd/TidyProvider.cpp
+++ b/clang-tools-extra/clangd/TidyProvider.cpp
@@ -219,6 +219,9 @@ TidyProvider disableUnusableChecks(llvm::ArrayRef<std::string> ExtraBadChecks) {
       "-bugprone-use-after-move",
       // Alias for bugprone-use-after-move.
       "-hicpp-invalid-access-moved",
+      // Check uses dataflow analysis, which might hang/crash unexpectedly on
+      // incomplete code.
+      "-bugprone-unchecked-optional-access",
 
       // ----- Performance problems -----
 
@@ -320,5 +323,17 @@ bool isRegisteredTidyCheck(llvm::StringRef Check) {
 
   return AllChecks.contains(Check);
 }
+
+std::optional<bool> isFastTidyCheck(llvm::StringRef Check) {
+  static auto &Fast = *new llvm::StringMap<bool>{
+#define FAST(CHECK, TIME) {#CHECK,true},
+#define SLOW(CHECK, TIME) {#CHECK,false},
+#include "TidyFastChecks.inc"
+  };
+  if (auto It = Fast.find(Check); It != Fast.end())
+    return It->second;
+  return std::nullopt;
+}
+
 } // namespace clangd
 } // namespace clang
diff --git a/clang-tools-extra/clangd/TidyProvider.h b/clang-tools-extra/clangd/TidyProvider.h
index 2f31366e1c..7d849d340f 100644
--- a/clang-tools-extra/clangd/TidyProvider.h
+++ b/clang-tools-extra/clangd/TidyProvider.h
@@ -60,6 +60,10 @@ tidy::ClangTidyOptions getTidyOptionsForFile(TidyProviderRef Provider,
 /// \pre \p must not be empty, must not contain '*' or ',' or start with '-'.
 bool isRegisteredTidyCheck(llvm::StringRef Check);
 
+/// Returns if \p Check is known-fast, known-slow, or its speed is unknown.
+/// By default, only fast checks will run in clangd.
+std::optional<bool> isFastTidyCheck(llvm::StringRef Check);
+
 } // namespace clangd
 } // namespace clang
 
diff --git a/clang-tools-extra/clangd/index/SymbolCollector.cpp b/clang-tools-extra/clangd/index/SymbolCollector.cpp
index 74aca9b99c..aac6676a99 100644
--- a/clang-tools-extra/clangd/index/SymbolCollector.cpp
+++ b/clang-tools-extra/clangd/index/SymbolCollector.cpp
@@ -893,7 +893,7 @@ void SymbolCollector::finish() {
     const Symbol *S = Symbols.find(SID);
     if (!S)
       continue;
-    assert(IncludeFiles.find(SID) != IncludeFiles.end());
+    assert(IncludeFiles.contains(SID));
 
     const auto FID = IncludeFiles.at(SID);
     // Determine if the FID is #include'd or #import'ed.
diff --git a/clang-tools-extra/clangd/tool/Check.cpp b/clang-tools-extra/clangd/tool/Check.cpp
index 46fcab0b69..b5c4d14561 100644
--- a/clang-tools-extra/clangd/tool/Check.cpp
+++ b/clang-tools-extra/clangd/tool/Check.cpp
@@ -34,6 +34,8 @@
 #include "CompileCommands.h"
 #include "Compiler.h"
 #include "Config.h"
+#include "ConfigFragment.h"
+#include "ConfigProvider.h"
 #include "Diagnostics.h"
 #include "Feature.h"
 #include "GlobalCompilationDatabase.h"
@@ -103,15 +105,19 @@ llvm::cl::opt<bool> CheckCompletion{
     "check-completion",
     llvm::cl::desc("Run code-completion at each point (slow)"),
     llvm::cl::init(false)};
+llvm::cl::opt<bool> CheckWarnings{
+    "check-warnings",
+    llvm::cl::desc("Print warnings as well as errors"),
+    llvm::cl::init(false)};
 
-// Print (and count) the error-level diagnostics (warnings are ignored).
+// Print the diagnostics meeting severity threshold, and return count of errors.
 unsigned showErrors(llvm::ArrayRef<Diag> Diags) {
   unsigned ErrCount = 0;
   for (const auto &D : Diags) {
-    if (D.Severity >= DiagnosticsEngine::Error) {
+    if (D.Severity >= DiagnosticsEngine::Error || CheckWarnings)
       elog("[{0}] Line {1}: {2}", D.Name, D.Range.start.line + 1, D.Message);
+    if (D.Severity >= DiagnosticsEngine::Error)
       ++ErrCount;
-    }
   }
   return ErrCount;
 }
@@ -476,8 +482,23 @@ bool check(llvm::StringRef File, const ThreadsafeFS &TFS,
   }
   log("Testing on source file {0}", File);
 
+  class OverrideConfigProvider : public config::Provider {
+    std::vector<config::CompiledFragment>
+    getFragments(const config::Params &,
+                 config::DiagnosticCallback Diag) const override {
+      config::Fragment F;
+      // If we're timing clang-tidy checks, implicitly disabling the slow ones
+      // is counterproductive! 
+      if (CheckTidyTime.getNumOccurrences())
+        F.Diagnostics.ClangTidy.FastCheckFilter.emplace("None");
+      return {std::move(F).compile(Diag)};
+    }
+  } OverrideConfig;
+  auto ConfigProvider =
+      config::Provider::combine({Opts.ConfigProvider, &OverrideConfig});
+
   auto ContextProvider = ClangdServer::createConfiguredContextProvider(
-      Opts.ConfigProvider, nullptr);
+      ConfigProvider.get(), nullptr);
   WithContext Ctx(ContextProvider(
       FakeFile.empty()
           ? File
diff --git a/clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp b/clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp
index f9b71a3230..14dd1f4b3f 100644
--- a/clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp
+++ b/clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp
@@ -1808,29 +1808,13 @@ TEST(ToLSPDiag, RangeIsInMain) {
 }
 
 TEST(ParsedASTTest, ModuleSawDiag) {
-  static constexpr const llvm::StringLiteral KDiagMsg = "StampedDiag";
-  struct DiagModifierModule final : public FeatureModule {
-    struct Listener : public FeatureModule::ASTListener {
-      void sawDiagnostic(const clang::Diagnostic &Info,
-                         clangd::Diag &Diag) override {
-        Diag.Message = KDiagMsg.str();
-      }
-    };
-    std::unique_ptr<ASTListener> astListeners() override {
-      return std::make_unique<Listener>();
-    };
-  };
-  FeatureModuleSet FMS;
-  FMS.add(std::make_unique<DiagModifierModule>());
-
-  Annotations Code("[[test]]; /* error-ok */");
   TestTU TU;
-  TU.Code = Code.code().str();
-  TU.FeatureModules = &FMS;
 
   auto AST = TU.build();
+        #if 0
   EXPECT_THAT(AST.getDiagnostics(),
               testing::Contains(Diag(Code.range(), KDiagMsg.str())));
+        #endif
 }
 
 TEST(Preamble, EndsOnNonEmptyLine) {
diff --git a/clang-tools-extra/clangd/unittests/HoverTests.cpp b/clang-tools-extra/clangd/unittests/HoverTests.cpp
index 8c88cd5257..063a60db04 100644
--- a/clang-tools-extra/clangd/unittests/HoverTests.cpp
+++ b/clang-tools-extra/clangd/unittests/HoverTests.cpp
@@ -92,6 +92,7 @@ TEST(Hover, Structured) {
          HI.Offset = 0;
          HI.Size = 8;
          HI.Padding = 56;
+         HI.Align = 8;
          HI.AccessSpecifier = "private";
        }},
       // Union field
@@ -110,6 +111,7 @@ TEST(Hover, Structured) {
          HI.Type = "char";
          HI.Size = 8;
          HI.Padding = 120;
+         HI.Align = 8;
          HI.AccessSpecifier = "public";
        }},
       // Bitfield
@@ -128,6 +130,7 @@ TEST(Hover, Structured) {
          HI.Type = "int";
          HI.Offset = 0;
          HI.Size = 1;
+         HI.Align = 32;
          HI.AccessSpecifier = "public";
        }},
       // Local to class method.
@@ -192,6 +195,7 @@ TEST(Hover, Structured) {
          HI.Type = "char";
          HI.Offset = 0;
          HI.Size = 8;
+         HI.Align = 8;
          HI.AccessSpecifier = "public";
        }},
       // Struct definition shows size.
@@ -204,6 +208,7 @@ TEST(Hover, Structured) {
          HI.Kind = index::SymbolKind::Struct;
          HI.Definition = "struct X {}";
          HI.Size = 8;
+         HI.Align = 8;
        }},
       // Variable with template type
       {R"cpp(
@@ -1375,6 +1380,7 @@ class Foo final {})cpp";
          HI.Offset = 8;
          HI.Size = 1;
          HI.Padding = 23;
+         HI.Align = 8;
          HI.AccessSpecifier = "public";
        }}};
   for (const auto &Case : Cases) {
@@ -1411,6 +1417,7 @@ class Foo final {})cpp";
     EXPECT_EQ(H->Value, Expected.Value);
     EXPECT_EQ(H->Size, Expected.Size);
     EXPECT_EQ(H->Offset, Expected.Offset);
+    EXPECT_EQ(H->Align, Expected.Align);
     EXPECT_EQ(H->AccessSpecifier, Expected.AccessSpecifier);
     EXPECT_EQ(H->CalleeArgInfo, Expected.CalleeArgInfo);
     EXPECT_EQ(H->CallPassType, Expected.CallPassType);
@@ -3448,13 +3455,14 @@ template <typename T, typename C = bool> class Foo {})",
             HI.Size = 32;
             HI.Offset = 96;
             HI.Padding = 32;
+            HI.Align = 32;
           },
           R"(field foo
 
 Type: type (aka can_type)
 Value = value
 Offset: 12 bytes
-Size: 4 bytes (+4 bytes padding)
+Size: 4 bytes (+4 bytes padding), alignment 4 bytes
 
 // In test::Bar
 def)",
@@ -3470,13 +3478,14 @@ def)",
             HI.Size = 25;
             HI.Offset = 35;
             HI.Padding = 4;
+            HI.Align = 64;
           },
           R"(field foo
 
 Type: type (aka can_type)
 Value = value
 Offset: 4 bytes and 3 bits
-Size: 25 bits (+4 bits padding)
+Size: 25 bits (+4 bits padding), alignment 8 bytes
 
 // In test::Bar
 def)",
diff --git a/clang-tools-extra/clangd/unittests/ParsedASTTests.cpp b/clang-tools-extra/clangd/unittests/ParsedASTTests.cpp
index 0348348450..500b72b9b3 100644
--- a/clang-tools-extra/clangd/unittests/ParsedASTTests.cpp
+++ b/clang-tools-extra/clangd/unittests/ParsedASTTests.cpp
@@ -12,6 +12,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "../../clang-tidy/ClangTidyCheck.h"
+#include "../../clang-tidy/ClangTidyModule.h"
+#include "../../clang-tidy/ClangTidyModuleRegistry.h"
 #include "AST.h"
 #include "CompileCommands.h"
 #include "Compiler.h"
@@ -26,9 +28,11 @@
 #include "TidyProvider.h"
 #include "support/Context.h"
 #include "clang/AST/DeclTemplate.h"
+#include "clang/Basic/FileEntry.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/TokenKinds.h"
+#include "clang/Lex/PPCallbacks.h"
 #include "clang/Tooling/Syntax/Tokens.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Testing/Annotations/Annotations.h"
@@ -36,7 +40,9 @@
 #include "gmock/gmock-matchers.h"
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
+#include <memory>
 #include <utility>
+#include <vector>
 
 namespace clang {
 namespace clangd {
diff --git a/clang-tools-extra/clangd/unittests/ReplayPeambleTests.cpp b/clang-tools-extra/clangd/unittests/ReplayPeambleTests.cpp
index 459a8f4c36..472fe30ee4 100644
--- a/clang-tools-extra/clangd/unittests/ReplayPeambleTests.cpp
+++ b/clang-tools-extra/clangd/unittests/ReplayPeambleTests.cpp
@@ -15,11 +15,13 @@
 #include "../../clang-tidy/ClangTidyModule.h"
 #include "../../clang-tidy/ClangTidyModuleRegistry.h"
 #include "AST.h"
+#include "Config.h"
 #include "Diagnostics.h"
 #include "ParsedAST.h"
 #include "SourceCode.h"
 #include "TestTU.h"
 #include "TidyProvider.h"
+#include "support/Context.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/Basic/FileEntry.h"
 #include "clang/Basic/LLVM.h"
@@ -121,6 +123,11 @@ TEST(ReplayPreambleTest, IncludesAndSkippedFiles) {
   // obj-c.
   TU.ExtraArgs = {"-isystem.", "-xobjective-c"};
 
+  // Allow the check to run even though not marked as fast.
+  Config Cfg;
+  Cfg.Diagnostics.ClangTidy.FastCheckFilter = Config::FastCheckPolicy::Loose;
+  WithContextValue WithCfg(Config::Key, std::move(Cfg));
+
   const auto &AST = TU.build();
   const auto &SM = AST.getSourceManager();
 
diff --git a/clang-tools-extra/clangd/unittests/TidyProviderTests.cpp b/clang-tools-extra/clangd/unittests/TidyProviderTests.cpp
index ece7f52d04..56b984c8e5 100644
--- a/clang-tools-extra/clangd/unittests/TidyProviderTests.cpp
+++ b/clang-tools-extra/clangd/unittests/TidyProviderTests.cpp
@@ -6,8 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "Feature.h"
 #include "TestFS.h"
 #include "TidyProvider.h"
+#include "llvm/Testing/Support/SupportHelpers.h"
 #include "gtest/gtest.h"
 
 namespace clang {
@@ -52,6 +54,22 @@ TEST(TidyProvider, NestedDirectories) {
   EXPECT_EQ(*Sub2Options.Checks, "misc-*,bugprone-*");
   EXPECT_EQ(Sub2Options.CheckOptions.lookup("TestKey").Value, "3");
 }
+
+TEST(TidyProvider, IsFastTidyCheck) {
+  EXPECT_THAT(isFastTidyCheck("misc-const-correctness"), llvm::ValueIs(false));
+  EXPECT_THAT(isFastTidyCheck("bugprone-suspicious-include"),
+              llvm::ValueIs(true));
+  // Linked in (ParsedASTTests.cpp) but not measured.
+  EXPECT_EQ(isFastTidyCheck("replay-preamble-check"), std::nullopt);
+}
+
+#if CLANGD_TIDY_CHECKS
+TEST(TidyProvider, IsValidCheck) {
+  EXPECT_TRUE(isRegisteredTidyCheck("bugprone-argument-comment"));
+  EXPECT_FALSE(isRegisteredTidyCheck("bugprone-argument-clinic"));
+}
+#endif
+
 } // namespace
 } // namespace clangd
 } // namespace clang
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index 03e5dc6f16..b5348384e9 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -120,11 +120,21 @@ Improvements to clang-tidy
 
 - Improved :program:`clang-tidy-diff.py` script. It now returns exit code `1`
   if any :program:`clang-tidy` subprocess exits with a non-zero code or if
-  exporting fixes fails.
+  exporting fixes fails. It now accepts a directory as a value for
+  `-export-fixes` to export individual yaml files for each compilation unit.
+
+- Improved :program:`run-clang-tidy.py` script. It now accepts a directory
+  as a value for `-export-fixes` to export individual yaml files for each
+  compilation unit.
 
 New checks
 ^^^^^^^^^^
 
+- New :doc:`bugprone-casting-through-void
+  <clang-tidy/checks/bugprone/casting-through-void>` check.
+
+  Detects unsafe or redundant two-step casting operations involving ``void*``.
+
 - New :doc:`bugprone-compare-pointer-to-member-virtual-function
   <clang-tidy/checks/bugprone/compare-pointer-to-member-virtual-function>` check.
 
@@ -163,6 +173,13 @@ New checks
   Flags coroutines that suspend while a lock guard is in scope at the
   suspension point.
 
+- New :doc:`misc-coroutine-hostile-raii
+  <clang-tidy/checks/misc/coroutine-hostile-raii>` check.
+
+  Detects when objects of certain hostile RAII types persists across suspension
+  points in a coroutine. Such hostile types include scoped-lockable types and
+  types belonging to a configurable denylist.
+
 - New :doc:`modernize-use-constraints
   <clang-tidy/checks/modernize/use-constraints>` check.
 
@@ -202,6 +219,10 @@ Changes in existing checks
   <clang-tidy/checks/bugprone/reserved-identifier>` check, so that it does not
   warn on macros starting with underscore and lowercase letter.
 
+- Improved :doc:`bugprone-unchecked-optional-access
+  <clang-tidy/checks/bugprone/unchecked-optional-access>` check, so that it does
+  not crash during handling of optional values.
+
 - Improved :doc:`bugprone-undefined-memory-manipulation
   <clang-tidy/checks/bugprone/undefined-memory-manipulation>` check to support
   fixed-size arrays of non-trivial types.
@@ -237,6 +258,10 @@ Changes in existing checks
   <clang-tidy/checks/cppcoreguidelines/pro-type-vararg>` check to ignore
   false-positives in unevaluated context (e.g., ``decltype``, ``sizeof``, ...).
 
+- Improved :doc:`cppcoreguidelines-rvalue-reference-param-not-moved
+  <clang-tidy/checks/cppcoreguidelines/rvalue-reference-param-not-moved>` check
+  to ignore unused parameters when they are marked as unused.
+
 - Improved :doc:`llvm-namespace-comment
   <clang-tidy/checks/llvm/namespace-comment>` check to provide fixes for
   ``inline`` namespaces in the same format as :program:`clang-format`.
@@ -265,9 +290,14 @@ Changes in existing checks
   <clang-tidy/checks/misc/redundant-expression>` check to ignore
   false-positives in unevaluated context (e.g., ``decltype``).
 
+- Improved :doc:`modernize-avoid-bind
+  <clang-tidy/checks/modernize/avoid-bind>` check to
+  not emit a ``return`` for fixes when the function returns ``void``.
+
 - Improved :doc:`modernize-loop-convert
   <clang-tidy/checks/modernize/loop-convert>` to support for-loops with
-  iterators initialized by free functions like ``begin``, ``end``, or ``size``.
+  iterators initialized by free functions like ``begin``, ``end``, or ``size``
+  and avoid crash for array of dependent array.
 
 - Improved :doc:`modernize-return-braced-init-list
   <clang-tidy/checks/modernize/return-braced-init-list>` check to ignore
diff --git a/clang-tools-extra/docs/clang-tidy/Integrations.rst b/clang-tools-extra/docs/clang-tidy/Integrations.rst
index 8f3f5a3127..7104944120 100644
--- a/clang-tools-extra/docs/clang-tidy/Integrations.rst
+++ b/clang-tools-extra/docs/clang-tidy/Integrations.rst
@@ -15,7 +15,7 @@ The following table shows the most well-known :program:`clang-tidy`
 integrations in detail.
 
 +--------------------------------------+------------------------+---------------------------------+--------------------------+-----------------------------------------+--------------------------+
-|                                      |        Feature                                                                                                                                           |
+|                                      |        Feature         |                                 |                          |                                         |                          |
 +======================================+========================+=================================+==========================+=========================================+==========================+
 |  **Tool**                            | On-the-fly inspection  | Check list configuration (GUI)  | Options to checks (GUI)  | Configuration via ``.clang-tidy`` files | Custom clang-tidy binary |
 +--------------------------------------+------------------------+---------------------------------+--------------------------+-----------------------------------------+--------------------------+
diff --git a/clang-tools-extra/docs/clang-tidy/checks/bugprone/casting-through-void.rst b/clang-tools-extra/docs/clang-tidy/checks/bugprone/casting-through-void.rst
new file mode 100644
index 0000000000..a9ab478b9a
--- /dev/null
+++ b/clang-tools-extra/docs/clang-tidy/checks/bugprone/casting-through-void.rst
@@ -0,0 +1,31 @@
+.. title:: clang-tidy - bugprone-casting-through-void
+
+bugprone-casting-through-void
+=============================
+
+Detects unsafe or redundant two-step casting operations involving ``void*``.
+
+Two-step type conversions via ``void*`` are discouraged for several reasons.
+
+- They obscure code and impede its understandability, complicating maintenance.
+- These conversions bypass valuable compiler support, erasing warnings related
+  to pointer alignment. It may violate strict aliasing rule and leading to
+  undefined behavior.
+- In scenarios involving multiple inheritance, ambiguity and unexpected outcomes
+  can arise due to the loss of type information, posing runtime issues.
+
+In summary, avoiding two-step type conversions through ``void*`` ensures clearer code,
+maintains essential compiler warnings, and prevents ambiguity and potential runtime
+errors, particularly in complex inheritance scenarios.
+
+Examples:
+
+.. code-block:: c++
+
+   using IntegerPointer = int *;
+   double *ptr;
+
+   static_cast<IntegerPointer>(static_cast<void *>(ptr)); // WRONG
+   reinterpret_cast<IntegerPointer>(reinterpret_cast<void *>(ptr)); // WRONG
+   (IntegerPointer)(void *)ptr; // WRONG
+   IntegerPointer(static_cast<void *>(ptr)); // WRONG
diff --git a/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/rvalue-reference-param-not-moved.rst b/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/rvalue-reference-param-not-moved.rst
index 9ab4ae7871..ffa3a9d61e 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/rvalue-reference-param-not-moved.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/rvalue-reference-param-not-moved.rst
@@ -18,6 +18,16 @@ Example:
     std::string Copy(Input); // Oops - forgot to std::move
   }
 
+Note that parameters that are unused and marked as such will not be diagnosed.
+
+Example:
+
+.. code-block:: c++
+
+  void conditional_use([[maybe_unused]] std::string&& Input) {
+    // No diagnostic here since Input is unused and marked as such
+  }
+
 Options
 -------
 
diff --git a/clang-tools-extra/docs/clang-tidy/checks/list.rst b/clang-tools-extra/docs/clang-tidy/checks/list.rst
index 2125ebd7a2..6f987ba167 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/list.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/list.rst
@@ -82,6 +82,7 @@ Clang-Tidy Checks
    :doc:`bugprone-bad-signal-to-kill-thread <bugprone/bad-signal-to-kill-thread>`,
    :doc:`bugprone-bool-pointer-implicit-conversion <bugprone/bool-pointer-implicit-conversion>`, "Yes"
    :doc:`bugprone-branch-clone <bugprone/branch-clone>`,
+   :doc:`bugprone-casting-through-void <bugprone/casting-through-void>`,
    :doc:`bugprone-compare-pointer-to-member-virtual-function <bugprone/compare-pointer-to-member-virtual-function>`,
    :doc:`bugprone-copy-constructor-init <bugprone/copy-constructor-init>`, "Yes"
    :doc:`bugprone-dangling-handle <bugprone/dangling-handle>`,
@@ -241,6 +242,7 @@ Clang-Tidy Checks
    :doc:`llvmlibc-restrict-system-libc-headers <llvmlibc/restrict-system-libc-headers>`, "Yes"
    :doc:`misc-confusable-identifiers <misc/confusable-identifiers>`,
    :doc:`misc-const-correctness <misc/const-correctness>`, "Yes"
+   :doc:`misc-coroutine-hostile-raii <misc/coroutine-hostile-raii>`,
    :doc:`misc-definitions-in-headers <misc/definitions-in-headers>`, "Yes"
    :doc:`misc-header-include-cycle <misc/header-include-cycle>`,
    :doc:`misc-include-cleaner <misc/include-cleaner>`, "Yes"
diff --git a/clang-tools-extra/docs/clang-tidy/checks/misc/coroutine-hostile-raii.rst b/clang-tools-extra/docs/clang-tidy/checks/misc/coroutine-hostile-raii.rst
new file mode 100644
index 0000000000..b8698ba3de
--- /dev/null
+++ b/clang-tools-extra/docs/clang-tidy/checks/misc/coroutine-hostile-raii.rst
@@ -0,0 +1,50 @@
+.. title:: clang-tidy - misc-coroutine-hostile-raii
+
+misc-coroutine-hostile-raii
+===========================
+
+Detects when objects of certain hostile RAII types persists across suspension
+points in a coroutine. Such hostile types include scoped-lockable types and
+types belonging to a configurable denylist.
+
+Some objects require that they be destroyed on the same thread that created them.
+Traditionally this requirement was often phrased as "must be a local variable",
+under the assumption that local variables always work this way. However this is
+incorrect with C++20 coroutines, since an intervening ``co_await`` may cause the
+coroutine to suspend and later be resumed on another thread.
+
+The lifetime of an object that requires being destroyed on the same thread must
+not encompass a ``co_await`` or ``co_yield`` point. If you create/destroy an object,
+you must do so without allowing the coroutine to suspend in the meantime.
+
+Following types are considered as hostile:
+
+ - Scoped-lockable types: A scoped-lockable object persisting across a suspension
+   point is problematic as the lock held by this object could be unlocked by a
+   different thread. This would be undefined behaviour.
+   This includes all types annotated with the ``scoped_lockable`` attribute.
+
+ - Types belonging to a configurable denylist.
+
+.. code-block:: c++
+
+  // Call some async API while holding a lock.
+  {
+    const my::MutexLock l(&mu_);
+
+    // Oops! The async Bar function may finish on a different
+    // thread from the one that created the MutexLock object and therefore called
+    // Mutex::Lock -- now Mutex::Unlock will be called on the wrong thread.
+    co_await Bar();
+  }
+
+
+Options
+-------
+
+.. option:: RAIITypesList
+
+    A semicolon-separated list of qualified types which should not be allowed to
+    persist across suspension points.
+    Eg: ``my::lockable; a::b;::my::other::lockable;``
+    The default value of this option is `"std::lock_guard;std::scoped_lock"`.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/misc/definitions-in-headers.rst b/clang-tools-extra/docs/clang-tidy/checks/misc/definitions-in-headers.rst
index 08aa9d884c..9c90bf1021 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/misc/definitions-in-headers.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/misc/definitions-in-headers.rst
@@ -88,6 +88,11 @@ from multiple translation units.
    template <class T>
    constexpr T pi = T(3.1415926L);
 
+When :program:`clang-tidy` is invoked with the `--fix-notes` option, this check
+provides fixes that automatically add the ``inline`` keyword to discovered
+functions. Please note that the addition of the ``inline`` keyword to variables
+is not currently supported by this check.
+
 Options
 -------
 
diff --git a/clang-tools-extra/docs/clang-tidy/checks/readability/identifier-naming.rst b/clang-tools-extra/docs/clang-tidy/checks/readability/identifier-naming.rst
index 467968ce43..608bc2acdc 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/readability/identifier-naming.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/readability/identifier-naming.rst
@@ -2526,41 +2526,41 @@ Otherwise the case of this character denotes scope.
 The following table is the default mapping table of Hungarian Notation which
 maps Decl to its prefix string. You can also have your own style in config file.
 
-================= ============== ====================== ============== =========== ==============
-Primitive Types                                                        Microsoft data types
----------------------------------------------------------------------- --------------------------
-    Type          Prefix         Type                   Prefix         Type        Prefix
-================= ============== ====================== ============== =========== ==============
-int8_t            i8             signed int             si             BOOL        b
-int16_t           i16            signed short           ss             BOOLEAN     b
-int32_t           i32            signed short int       ssi            BYTE        by
-int64_t           i64            signed long long int   slli           CHAR        c
-uint8_t           u8             signed long long       sll            UCHAR       uc
-uint16_t          u16            signed long int        sli            SHORT       s
-uint32_t          u32            signed long            sl             USHORT      us
-uint64_t          u64            signed                 s              WORD        w
-char8_t           c8             unsigned long long int ulli           DWORD       dw
-char16_t          c16            unsigned long long     ull            DWORD32     dw32
-char32_t          c32            unsigned long int      uli            DWORD64     dw64
-float             f              unsigned long          ul             LONG        l
-double            d              unsigned short int     usi            ULONG       ul
-char              c              unsigned short         us             ULONG32     ul32
-bool              b              unsigned int           ui             ULONG64     ul64
-_Bool             b              unsigned char          uc             ULONGLONG   ull
-int               i              unsigned               u              HANDLE      h
-size_t            n              long long int          lli            INT         i
-short             s              long double            ld             INT8        i8
-signed            i              long long              ll             INT16       i16
-unsigned          u              long int               li             INT32       i32
-long              l              long                   l              INT64       i64
-long long         ll             ptrdiff_t              p              UINT        ui
-unsigned long     ul             void                   *none*         UINT8       u8
-long double       ld                                                   UINT16      u16
-ptrdiff_t         p                                                    UINT32      u32
-wchar_t           wc                                                   UINT64      u64
-short int         si                                                   PVOID       p
+================= ============== ====================== ============== ============== ==============
+Primitive Type                                                         Microsoft Type
+----------------- -------------- ---------------------- -------------- -------------- --------------
+    Type          Prefix         Type                   Prefix         Type           Prefix
+================= ============== ====================== ============== ============== ==============
+int8_t            i8             signed int             si             BOOL           b
+int16_t           i16            signed short           ss             BOOLEAN        b
+int32_t           i32            signed short int       ssi            BYTE           by
+int64_t           i64            signed long long int   slli           CHAR           c
+uint8_t           u8             signed long long       sll            UCHAR          uc
+uint16_t          u16            signed long int        sli            SHORT          s
+uint32_t          u32            signed long            sl             USHORT         us
+uint64_t          u64            signed                 s              WORD           w
+char8_t           c8             unsigned long long int ulli           DWORD          dw
+char16_t          c16            unsigned long long     ull            DWORD32        dw32
+char32_t          c32            unsigned long int      uli            DWORD64        dw64
+float             f              unsigned long          ul             LONG           l
+double            d              unsigned short int     usi            ULONG          ul
+char              c              unsigned short         us             ULONG32        ul32
+bool              b              unsigned int           ui             ULONG64        ul64
+_Bool             b              unsigned char          uc             ULONGLONG      ull
+int               i              unsigned               u              HANDLE         h
+size_t            n              long long int          lli            INT            i
+short             s              long double            ld             INT8           i8
+signed            i              long long              ll             INT16          i16
+unsigned          u              long int               li             INT32          i32
+long              l              long                   l              INT64          i64
+long long         ll             ptrdiff_t              p              UINT           ui
+unsigned long     ul             void                   *none*         UINT8          u8
+long double       ld                                                   UINT16         u16
+ptrdiff_t         p                                                    UINT32         u32
+wchar_t           wc                                                   UINT64         u64
+short int         si                                                   PVOID          p
 short             s
-================= ============== ====================== ============== =========== ==============
+================= ============== ====================== ============== ============== ==============
 
 **There are more trivial options for Hungarian Notation:**
 
diff --git a/clang-tools-extra/include-cleaner/unittests/LocateSymbolTest.cpp b/clang-tools-extra/include-cleaner/unittests/LocateSymbolTest.cpp
index d69e25bf81..756757cfd0 100644
--- a/clang-tools-extra/include-cleaner/unittests/LocateSymbolTest.cpp
+++ b/clang-tools-extra/include-cleaner/unittests/LocateSymbolTest.cpp
@@ -30,8 +30,6 @@ using testing::ElementsAre;
 using testing::ElementsAreArray;
 using testing::Eq;
 using testing::Field;
-using testing::Pair;
-using testing::UnorderedElementsAre;
 
 // A helper for building ASTs and getting decls out of it by name. Example usage
 // looks like:
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/unchecked-optional-access/absl/types/optional.h b/clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/unchecked-optional-access/absl/types/optional.h
index 154cc262ab..3e692f347a 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/unchecked-optional-access/absl/types/optional.h
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/Inputs/unchecked-optional-access/absl/types/optional.h
@@ -66,6 +66,8 @@ class optional {
   void reset() noexcept;
 
   void swap(optional &rhs) noexcept;
+
+  template <typename U> optional &operator=(const U &u);
 };
 
 } // namespace absl
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/casting-through-void.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/casting-through-void.cpp
new file mode 100644
index 0000000000..3913d2d8a2
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/casting-through-void.cpp
@@ -0,0 +1,91 @@
+// RUN: %check_clang_tidy %s bugprone-casting-through-void %t
+
+using V = void*;
+using CV = const void*;
+
+int i = 100;
+double d = 100;
+const int ci = 100;
+const double cd = 100;
+
+void normal_test() {
+  static_cast<int *>(static_cast<void *>(&d));
+  // CHECK-MESSAGES: :[[@LINE-1]]:22: warning: do not cast 'double *' to 'int *' through 'void *' [bugprone-casting-through-void]
+  static_cast<int *>(static_cast<V>(&d));
+  // CHECK-MESSAGES: :[[@LINE-1]]:22: warning: do not cast 'double *' to 'int *' through 'V' (aka 'void *') [bugprone-casting-through-void]
+  static_cast<int *>(static_cast<void *>(&i));
+  // CHECK-MESSAGES: :[[@LINE-1]]:22: warning: do not cast 'int *' to 'int *' through 'void *' [bugprone-casting-through-void]
+
+  static_cast<void *>(static_cast<void *>(&i));
+}
+
+void const_pointer_test() {
+  static_cast<int *const>(static_cast<void *>(&d));
+  // CHECK-MESSAGES: :[[@LINE-1]]:27: warning: do not cast 'double *' to 'int *const' through 'void *' [bugprone-casting-through-void]
+  static_cast<int *const>(static_cast<V>(&d));
+  // CHECK-MESSAGES: :[[@LINE-1]]:27: warning: do not cast 'double *' to 'int *const' through 'V' (aka 'void *') [bugprone-casting-through-void]
+  static_cast<int *const>(static_cast<void *>(&i));
+  // CHECK-MESSAGES: :[[@LINE-1]]:27: warning: do not cast 'int *' to 'int *const' through 'void *' [bugprone-casting-through-void]
+
+  static_cast<void *const>(static_cast<void *>(&i));
+}
+
+void const_test() {
+  static_cast<const int *>(static_cast<const void *>(&d));
+  // CHECK-MESSAGES: :[[@LINE-1]]:28: warning: do not cast 'double *' to 'const int *' through 'const void *' [bugprone-casting-through-void]
+  static_cast<const int *>(static_cast<const V>(&d));
+  // CHECK-MESSAGES: :[[@LINE-1]]:28: warning: do not cast 'double *' to 'const int *' through 'const V' (aka 'void *const') [bugprone-casting-through-void]
+  static_cast<const int *>(static_cast<const void *>(&i));
+  // CHECK-MESSAGES: :[[@LINE-1]]:28: warning: do not cast 'int *' to 'const int *' through 'const void *' [bugprone-casting-through-void]
+
+  static_cast<const void *>(static_cast<const void *>(&i));
+
+  static_cast<const int *>(static_cast<const void *>(&cd));
+  // CHECK-MESSAGES: :[[@LINE-1]]:28: warning: do not cast 'const double *' to 'const int *' through 'const void *' [bugprone-casting-through-void]
+  static_cast<const int *>(static_cast<const CV>(&cd));
+  // CHECK-MESSAGES: :[[@LINE-1]]:28: warning: do not cast 'const double *' to 'const int *' through 'const CV' (aka 'const void *const') [bugprone-casting-through-void]
+  static_cast<const int *>(static_cast<const void *>(&ci));
+  // CHECK-MESSAGES: :[[@LINE-1]]:28: warning: do not cast 'const int *' to 'const int *' through 'const void *' [bugprone-casting-through-void]
+
+  static_cast<const void *>(static_cast<const void *>(&ci));
+}
+
+
+void reinterpret_cast_test() {
+  static_cast<int *>(reinterpret_cast<void *>(&d));
+  // CHECK-MESSAGES: :[[@LINE-1]]:22: warning: do not cast 'double *' to 'int *' through 'void *' [bugprone-casting-through-void]
+  reinterpret_cast<int *>(static_cast<void *>(&d));
+  // CHECK-MESSAGES: :[[@LINE-1]]:27: warning: do not cast 'double *' to 'int *' through 'void *' [bugprone-casting-through-void]
+  reinterpret_cast<int *>(reinterpret_cast<void *>(&d));
+  // CHECK-MESSAGES: :[[@LINE-1]]:27: warning: do not cast 'double *' to 'int *' through 'void *' [bugprone-casting-through-void]
+
+  static_cast<void *>(reinterpret_cast<void *>(&i));
+  reinterpret_cast<void *>(reinterpret_cast<void *>(&i));
+  reinterpret_cast<void *>(static_cast<void *>(&i));
+}
+
+void c_style_cast_test() {
+  static_cast<int *>((void *)&d);
+  // CHECK-MESSAGES: :[[@LINE-1]]:22: warning: do not cast 'double *' to 'int *' through 'void *' [bugprone-casting-through-void]
+  (int *)(void *)&d;
+  // CHECK-MESSAGES: :[[@LINE-1]]:10: warning: do not cast 'double *' to 'int *' through 'void *' [bugprone-casting-through-void]
+  static_cast<int *>((void *)&d);
+  // CHECK-MESSAGES: :[[@LINE-1]]:22: warning: do not cast 'double *' to 'int *' through 'void *' [bugprone-casting-through-void]
+
+  static_cast<void *>((void *)&i);
+}
+
+struct A {
+   A(void*);
+};
+using I = int *;
+void cxx_functional_cast() {
+  A(static_cast<void*>(&d));
+  I(static_cast<void*>(&d));
+  // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: do not cast 'double *' to 'I' (aka 'int *') through 'void *' [bugprone-casting-through-void]
+}
+
+void bit_cast() {
+  __builtin_bit_cast(int *, static_cast<void *>(&d));
+  // CHECK-MESSAGES: :[[@LINE-1]]:29: warning: do not cast 'double *' to 'int *' through 'void *' [bugprone-casting-through-void]
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/unchecked-optional-access.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/unchecked-optional-access.cpp
index 1921291f21..13a3ff52f3 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/unchecked-optional-access.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/unchecked-optional-access.cpp
@@ -180,3 +180,23 @@ void std_forward_rvalue_ref_safe(absl::optional<int>&& opt) {
 
   std::forward<absl::optional<int>>(opt).value();
 }
+
+namespace std {
+
+template <typename T> class vector {
+public:
+  T &operator[](unsigned long index);
+  bool empty();
+};
+
+} // namespace std
+
+struct S {
+  absl::optional<float> x;
+};
+std::vector<S> vec;
+
+void foo() {
+  if (!vec.empty())
+    vec[0].x = 0;
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/rvalue-reference-param-not-moved.cpp b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/rvalue-reference-param-not-moved.cpp
index 8f8e272e1e..a9b87567a0 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/rvalue-reference-param-not-moved.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/rvalue-reference-param-not-moved.cpp
@@ -333,3 +333,28 @@ void instantiate_a_class_template() {
   AClassTemplate<Obj&> withObjRef(o);
   withObjRef.never_moves(o);
 }
+
+namespace gh68209
+{
+  void f1([[maybe_unused]] int&& x) {}
+
+  void f2(__attribute__((unused)) int&& x) {}
+
+  void f3(int&& x) {}
+  // CHECK-MESSAGES: :[[@LINE-1]]:17: warning: rvalue reference parameter 'x' is never moved from inside the function body [cppcoreguidelines-rvalue-reference-param-not-moved]
+
+  template <typename T>
+  void f4([[maybe_unused]] T&& x) {}
+
+  template <typename T>
+  void f5(__attribute((unused)) T&& x) {}
+
+  template<typename T>
+  void f6(T&& x) {}
+
+  void f7([[maybe_unused]] int&& x) { x += 1; }
+  // CHECK-MESSAGES: :[[@LINE-1]]:34: warning: rvalue reference parameter 'x' is never moved from inside the function body [cppcoreguidelines-rvalue-reference-param-not-moved]
+
+  void f8(__attribute__((unused)) int&& x) { x += 1; }
+  // CHECK-MESSAGES: :[[@LINE-1]]:41: warning: rvalue reference parameter 'x' is never moved from inside the function body [cppcoreguidelines-rvalue-reference-param-not-moved]
+} // namespace gh68209
diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/coroutine-hostile-raii.cpp b/clang-tools-extra/test/clang-tidy/checkers/misc/coroutine-hostile-raii.cpp
new file mode 100644
index 0000000000..2d022e21c8
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/misc/coroutine-hostile-raii.cpp
@@ -0,0 +1,192 @@
+// RUN: %check_clang_tidy -std=c++20 %s misc-coroutine-hostile-raii %t \
+// RUN:   -config="{CheckOptions: \
+// RUN:             {misc-coroutine-hostile-raii.RAIITypesList: \
+// RUN:               'my::Mutex; ::my::other::Mutex'}}"
+
+namespace std {
+
+template <typename R, typename...> struct coroutine_traits {
+  using promise_type = typename R::promise_type;
+};
+
+template <typename Promise = void> struct coroutine_handle;
+
+template <> struct coroutine_handle<void> {
+  static coroutine_handle from_address(void *addr) noexcept {
+    coroutine_handle me;
+    me.ptr = addr;
+    return me;
+  }
+  void operator()() { resume(); }
+  void *address() const noexcept { return ptr; }
+  void resume() const {  }
+  void destroy() const { }
+  bool done() const { return true; }
+  coroutine_handle &operator=(decltype(nullptr)) {
+    ptr = nullptr;
+    return *this;
+  }
+  coroutine_handle(decltype(nullptr)) : ptr(nullptr) {}
+  coroutine_handle() : ptr(nullptr) {}
+  //  void reset() { ptr = nullptr; } // add to P0057?
+  explicit operator bool() const { return ptr; }
+
+protected:
+  void *ptr;
+};
+
+template <typename Promise> struct coroutine_handle : coroutine_handle<> {
+  using coroutine_handle<>::operator=;
+
+  static coroutine_handle from_address(void *addr) noexcept {
+    coroutine_handle me;
+    me.ptr = addr;
+    return me;
+  }
+
+  Promise &promise() const {
+    return *reinterpret_cast<Promise *>(
+        __builtin_coro_promise(ptr, alignof(Promise), false));
+  }
+  static coroutine_handle from_promise(Promise &promise) {
+    coroutine_handle p;
+    p.ptr = __builtin_coro_promise(&promise, alignof(Promise), true);
+    return p;
+  }
+};
+
+struct suspend_always {
+  bool await_ready() noexcept { return false; }
+  void await_suspend(std::coroutine_handle<>) noexcept {}
+  void await_resume() noexcept {}
+};
+} // namespace std
+
+struct ReturnObject {
+    struct promise_type {
+        ReturnObject get_return_object() { return {}; }
+        std::suspend_always initial_suspend() { return {}; }
+        std::suspend_always final_suspend() noexcept { return {}; }
+        void unhandled_exception() {}
+        std::suspend_always yield_value(int value) { return {}; }
+    };
+};
+
+#define SCOPED_LOCKABLE __attribute__ ((scoped_lockable))
+
+namespace absl {
+class SCOPED_LOCKABLE Mutex {};
+using Mutex2 = Mutex;
+} // namespace absl
+
+ReturnObject BasicWarning() {
+  absl::Mutex mtx;
+  // CHECK-MESSAGES: :[[@LINE-1]]:15: warning: 'mtx' holds a lock across a suspension point of coroutine and could be unlocked by a different thread [misc-coroutine-hostile-raii]
+  int no_warning;
+  {
+    co_yield 1;
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: note: suspension point is here
+  }
+}
+
+ReturnObject BasicNoWarning() {
+  co_yield 1;
+  {  absl::Mutex no_warning; }
+  int no_warning;
+  {
+    co_yield 1;
+    absl::Mutex no_warning;
+  }
+  co_yield 1;
+}
+
+ReturnObject scopedLockableTest() {
+    co_yield 0;
+    absl::Mutex a;
+    // CHECK-MESSAGES: :[[@LINE-1]]:17: warning: 'a' holds a lock across a suspension point of coroutine and could be unlocked by a different thread [misc-coroutine-hostile-raii]
+    absl::Mutex2 b;
+    // CHECK-MESSAGES: :[[@LINE-1]]:18: warning: 'b' holds a lock across a suspension point of coroutine and could be unlocked by a different thread [misc-coroutine-hostile-raii]
+    {
+        absl::Mutex no_warning_1;
+        { absl::Mutex no_warning_2; }
+    }
+
+    co_yield 1;
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: note: suspension point is here
+    absl::Mutex c;
+    // CHECK-MESSAGES: :[[@LINE-1]]:17: warning: 'c' holds a lock across a suspension point of coroutine and could be unlocked by a different thread [misc-coroutine-hostile-raii]
+    co_await std::suspend_always{};
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: note: suspension point is here
+    for(int i=1; i<=10; ++i ) {
+      absl::Mutex d;
+      // CHECK-MESSAGES: :[[@LINE-1]]:19: warning: 'd' holds a lock across a suspension point of coroutine and could be unlocked by a different thread [misc-coroutine-hostile-raii]
+      co_await std::suspend_always{};
+      // CHECK-MESSAGES: :[[@LINE-1]]:7: note: suspension point is here
+      co_yield 1;
+      absl::Mutex no_warning_3;
+    }
+    if (true) {
+      absl::Mutex e;
+      // CHECK-MESSAGES: :[[@LINE-1]]:19: warning: 'e' holds a lock across a suspension point of coroutine and could be unlocked by a different thread [misc-coroutine-hostile-raii]
+      co_yield 1;
+      // CHECK-MESSAGES: :[[@LINE-1]]:7: note: suspension point is here
+      absl::Mutex no_warning_4;
+    }
+    absl::Mutex no_warning_5;
+}
+
+void lambda() {
+  absl::Mutex no_warning;
+  auto lambda = []() -> ReturnObject {
+    co_await std::suspend_always{};
+    absl::Mutex a;
+    // CHECK-MESSAGES: :[[@LINE-1]]:17: warning: 'a' holds a lock across a suspension point of coroutine and could be unlocked by a different thread [misc-coroutine-hostile-raii]
+    co_yield 1;
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: note: suspension point is here
+    co_await std::suspend_always{};
+    co_yield 1;
+  };
+  absl::Mutex no_warning_2;
+}
+
+template<class T>
+ReturnObject raii_in_template(){
+  T a;
+  // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: 'a' holds a lock across a suspension point of coroutine and could be unlocked by a different thread [misc-coroutine-hostile-raii]
+  co_yield 1;
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: note: suspension point is here
+}
+void foo_template() { raii_in_template<absl::Mutex>(); }
+
+namespace my {
+class Mutex{};
+namespace other {
+class Mutex{};
+} // namespace other
+
+using Mutex2 = Mutex;
+} // namespace my
+
+ReturnObject denyListTest() {
+    my::Mutex a;
+    // CHECK-MESSAGES: :[[@LINE-1]]:15: warning: 'a' persists across a suspension point of coroutine [misc-coroutine-hostile-raii]
+    my::other::Mutex b;
+    // CHECK-MESSAGES: :[[@LINE-1]]:22: warning: 'b' persists across a suspension point of coroutine [misc-coroutine-hostile-raii]
+    my::Mutex2 c;
+    // CHECK-MESSAGES: :[[@LINE-1]]:16: warning: 'c' persists across a suspension point of coroutine [misc-coroutine-hostile-raii]
+    co_yield 1;
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: note: suspension point is here
+}
+
+ReturnObject referenceTest(my::Mutex& ref) {
+    my::Mutex& a = ref;
+    co_yield 1;
+}
+ReturnObject pointerTest(my::Mutex* ref) {
+    my::Mutex* a = ref;
+    co_yield 1;
+}
+
+ReturnObject functionArgTest(my::Mutex ref) {
+    co_yield 1;
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/avoid-bind.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/avoid-bind.cpp
index 3334eab2d4..336b3cb4ee 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/modernize/avoid-bind.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/avoid-bind.cpp
@@ -45,6 +45,7 @@ struct D {
   void operator()(int x, int y) const {}
 
   void MemberFunction(int x) {}
+  int MemberFunctionWithReturn(int x) {}
 
   static D *create();
 };
@@ -162,11 +163,11 @@ struct TestCaptureByValueStruct {
     // those here.
     auto FFF = boost::bind(UseF, f);
     // CHECK-MESSAGES: :[[@LINE-1]]:16: warning: prefer a lambda to boost::bind [modernize-avoid-bind]
-    // CHECK-FIXES: auto FFF = [f] { return UseF(f); };
+    // CHECK-FIXES: auto FFF = [f] { UseF(f); };
 
     auto GGG = boost::bind(UseF, MemberStruct);
     // CHECK-MESSAGES: :[[@LINE-1]]:16: warning: prefer a lambda to boost::bind [modernize-avoid-bind]
-    // CHECK-FIXES: auto GGG = [this] { return UseF(MemberStruct); };
+    // CHECK-FIXES: auto GGG = [this] { UseF(MemberStruct); };
 
     auto HHH = std::bind(add, MemberStructWithData._member, 1);
     // CHECK-MESSAGES: :[[@LINE-1]]:16: warning: prefer a lambda to std::bind
@@ -227,34 +228,34 @@ void testFunctionObjects() {
   D *e = nullptr;
   auto AAA = std::bind(d, 1, 2);
   // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: prefer a lambda to std::bind
-  // CHECK-FIXES: auto AAA = [d] { return d(1, 2); }
+  // CHECK-FIXES: auto AAA = [d] { d(1, 2); }
 
   auto BBB = std::bind(*e, 1, 2);
   // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: prefer a lambda to std::bind
-  // CHECK-FIXES: auto BBB = [e] { return (*e)(1, 2); }
+  // CHECK-FIXES: auto BBB = [e] { (*e)(1, 2); }
 
   auto CCC = std::bind(D{}, 1, 2);
   // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: prefer a lambda to std::bind
-  // CHECK-FIXES: auto CCC = [] { return D{}(1, 2); }
+  // CHECK-FIXES: auto CCC = [] { D{}(1, 2); }
 
   auto DDD = std::bind(D(), 1, 2);
   // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: prefer a lambda to std::bind
-  // CHECK-FIXES: auto DDD = [] { return D()(1, 2); }
+  // CHECK-FIXES: auto DDD = [] { D()(1, 2); }
 
   auto EEE = std::bind(*D::create(), 1, 2);
   // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: prefer a lambda to std::bind
-  // CHECK-FIXES: auto EEE = [Func = *D::create()] { return Func(1, 2); };
+  // CHECK-FIXES: auto EEE = [Func = *D::create()] { Func(1, 2); };
 
   auto FFF = std::bind(G(), 1);
   // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: prefer a lambda to std::bind
   // Templated function call operators may be used
-  // CHECK-FIXES: auto FFF = [] { return G()(1); };
+  // CHECK-FIXES: auto FFF = [] { G()(1); };
 
   int CTorArg = 42;
   auto GGG = std::bind(G(CTorArg), 1);
   // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: prefer a lambda to std::bind
   // Function objects with constructor arguments should be captured
-  // CHECK-FIXES: auto GGG = [Func = G(CTorArg)] { return Func(1); };
+  // CHECK-FIXES: auto GGG = [Func = G(CTorArg)] { Func(1); };
 }
 
 template <typename T>
@@ -262,7 +263,7 @@ void testMemberFnOfClassTemplate(T) {
   auto HHH = std::bind(H<T>(), 42);
   // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: prefer a lambda to std::bind
   // Ensure function class template arguments are preserved
-  // CHECK-FIXES: auto HHH = [] { return H<T>()(42); };
+  // CHECK-FIXES: auto HHH = [] { H<T>()(42); };
 }
 
 template void testMemberFnOfClassTemplate(int);
@@ -335,11 +336,12 @@ void testCapturedSubexpressions() {
   auto CCC = std::bind(sub, std::ref(*p), _1);
   // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: prefer a lambda to std::bind
   // Expressions returning references are captured
-  // CHECK-FIXES: auto CCC = [&capture0 = *p](auto && PH1) { return sub(capture0, std::forward<decltype(PH1)>(PH1)); };
+  // CHECK-FIXES: auto CCC = [&capture0 = *p](auto && PH1) { sub(capture0, std::forward<decltype(PH1)>(PH1)); };
 }
 
 struct E {
   void MemberFunction(int x) {}
+  int MemberFunctionWithReturn(int x) {}
 
   void testMemberFunctions() {
     D *d;
@@ -360,6 +362,23 @@ struct E {
     auto DDD = std::bind(&D::MemberFunction, _1, 1);
     // CHECK-MESSAGES: :[[@LINE-1]]:16: warning: prefer a lambda to std::bind
     // CHECK-FIXES: auto DDD = [](auto && PH1) { PH1->MemberFunction(1); };
+
+    auto EEE = std::bind(&D::MemberFunctionWithReturn, d, 1);
+    // CHECK-MESSAGES: :[[@LINE-1]]:16: warning: prefer a lambda to std::bind
+    // CHECK-FIXES: auto EEE = [d] { return d->MemberFunctionWithReturn(1); };
+
+    auto FFF = std::bind(&D::MemberFunctionWithReturn, &dd, 1);
+    // CHECK-MESSAGES: :[[@LINE-1]]:16: warning: prefer a lambda to std::bind
+    // CHECK-FIXES: auto FFF = [ObjectPtr = &dd] { return ObjectPtr->MemberFunctionWithReturn(1); };
+
+    auto GGG = std::bind(&E::MemberFunctionWithReturn, this, 1);
+    // CHECK-MESSAGES: :[[@LINE-1]]:16: warning: prefer a lambda to std::bind
+    // CHECK-FIXES: auto GGG = [this] { return MemberFunctionWithReturn(1); };
+
+    // Test what happens when the object pointer is itself a placeholder.
+    auto HHH = std::bind(&D::MemberFunctionWithReturn, _1, 1);
+    // CHECK-MESSAGES: :[[@LINE-1]]:16: warning: prefer a lambda to std::bind
+    // CHECK-FIXES: auto HHH = [](auto && PH1) { return PH1->MemberFunctionWithReturn(1); };
   }
 };
 
@@ -368,5 +387,5 @@ void testStdFunction(Thing *t) {
   if (t)
     cb.Reset(std::bind(UseThing, t));
   // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: prefer a lambda to std::bind
-  // CHECK-FIXES: cb.Reset([t] { return UseThing(t); });
+  // CHECK-FIXES: cb.Reset([t] { UseThing(t); });
 }
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/loop-convert-basic.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/loop-convert-basic.cpp
index 71ae4c46e6..e2b9336d62 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/modernize/loop-convert-basic.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/loop-convert-basic.cpp
@@ -939,4 +939,18 @@ void fundamentalTypesTest() {
   // CHECK-FIXES: for (double Double : Doubles)
 }
 
+template <unsigned  p> void _dependenceArrayTest() {
+  unsigned test[3][p];
+  for (unsigned i = 0; i < p; ++i)
+    for (unsigned j = 0; j < 3; ++j)
+      printf("%d", test[j][i]);
+  // CHECK-MESSAGES: :[[@LINE-2]]:5: warning: use range-based for loop instead
+  // CHECK-FIXES: (auto & j : test)
+  // CHECK-FIXES: printf("%d", j[i]);
+}
+void dependenceArrayTest() {
+  _dependenceArrayTest<1>();
+  _dependenceArrayTest<2>();
+}
+
 } // namespace PseudoArray
diff --git a/clang/cmake/caches/Fuchsia-stage2.cmake b/clang/cmake/caches/Fuchsia-stage2.cmake
index 0dcc35ee14..13bb5ad354 100644
--- a/clang/cmake/caches/Fuchsia-stage2.cmake
+++ b/clang/cmake/caches/Fuchsia-stage2.cmake
@@ -368,6 +368,7 @@ if(FUCHSIA_ENABLE_LLDB)
     liblldb
     lldb-server
     lldb-argdumper
+    lldb-dap
   )
   if(LLDB_ENABLE_PYTHON)
     list(APPEND _FUCHSIA_LLDB_COMPONENTS lldb-python-scripts)
diff --git a/clang/docs/ClangFormatStyleOptions.rst b/clang/docs/ClangFormatStyleOptions.rst
index c5c72c68ee..cfd57f5fa8 100644
--- a/clang/docs/ClangFormatStyleOptions.rst
+++ b/clang/docs/ClangFormatStyleOptions.rst
@@ -5224,6 +5224,33 @@ the configuration (without a prefix: ``Auto``).
        void operator++ (int a);        vs.    void operator++(int a);
        object.operator++ (10);                object.operator++(10);
 
+  * ``AfterPlacementOperatorStyle AfterPlacementOperator`` :versionbadge:`clang-format 18`
+
+    Defines in which cases to put a space between ``new/delete`` operators
+    and opening parentheses.
+
+    Possible values:
+
+    * ``APO_Never`` (in configuration: ``Never``)
+      Remove space after ``new/delete`` operators and before ``(``.
+
+      .. code-block:: c++
+
+         new(buf) T;
+         delete(buf) T;
+
+    * ``APO_Always`` (in configuration: ``Always``)
+      Always add space after ``new/delete`` operators and before ``(``.
+
+      .. code-block:: c++
+
+         new (buf) T;
+         delete (buf) T;
+
+    * ``APO_Leave`` (in configuration: ``Leave``)
+      Leave placement ``new/delete`` expressions as they are.
+
+
   * ``bool AfterRequiresInClause`` If ``true``, put space between requires keyword in a requires clause and
     opening parentheses, if there is one.
 
diff --git a/clang/docs/HIPSupport.rst b/clang/docs/HIPSupport.rst
index 8b4649733a..84cee45e83 100644
--- a/clang/docs/HIPSupport.rst
+++ b/clang/docs/HIPSupport.rst
@@ -176,3 +176,93 @@ Predefined Macros
    * - ``HIP_API_PER_THREAD_DEFAULT_STREAM``
      - Alias to ``__HIP_API_PER_THREAD_DEFAULT_STREAM__``. Deprecated.
 
+Compilation Modes
+=================
+
+Each HIP source file contains intertwined device and host code. Depending on the chosen compilation mode by the compiler options ``-fno-gpu-rdc`` and ``-fgpu-rdc``, these portions of code are compiled differently.
+
+Device Code Compilation
+-----------------------
+
+**``-fno-gpu-rdc`` Mode (default)**:
+
+- Compiles to a self-contained, fully linked offloading device binary for each offloading device architecture.
+- Device code within a Translation Unit (TU) cannot call functions located in another TU.
+
+**``-fgpu-rdc`` Mode**:
+
+- Compiles to a bitcode for each GPU architecture.
+- For each offloading device architecture, the bitcode from different TUs are linked together to create a single offloading device binary.
+- Device code in one TU can call functions located in another TU.
+
+Host Code Compilation
+---------------------
+
+**Both Modes**:
+
+- Compiles to a relocatable object for each TU.
+- These relocatable objects are then linked together.
+- Host code within a TU can call host functions and launch kernels from another TU.
+
+Function Pointers Support
+=========================
+
+Function pointers' support varies with the usage mode in Clang with HIP. The following table provides an overview of the support status across different use-cases and modes.
+
+.. list-table:: Function Pointers Support Overview
+   :widths: 25 25 25
+   :header-rows: 1
+
+   * - Use Case
+     - ``-fno-gpu-rdc`` Mode (default)
+     - ``-fgpu-rdc`` Mode
+   * - Defined and used in the same TU
+     - Supported
+     - Supported
+   * - Defined in one TU and used in another TU
+     - Not Supported
+     - Supported
+
+In the ``-fno-gpu-rdc`` mode, the compiler calculates the resource usage of kernels based only on functions present within the same TU. This mode does not support the use of function pointers defined in a different TU due to the possibility of incorrect resource usage calculations, leading to undefined behavior.
+
+On the other hand, the ``-fgpu-rdc`` mode allows the definition and use of function pointers across different TUs, as resource usage calculations can accommodate functions from disparate TUs.
+
+Virtual Function Support
+========================
+
+In Clang with HIP, support for calling virtual functions of an object in device or host code is contingent on where the object is constructed.
+
+- **Constructed in Device Code**: Virtual functions of an object can be called in device code on a specific offloading device if the object is constructed in device code on an offloading device with the same architecture.
+- **Constructed in Host Code**: Virtual functions of an object can be called in host code if the object is constructed in host code.
+
+In other scenarios, calling virtual functions is not allowed.
+
+Explanation
+-----------
+
+An object constructed on the device side contains a pointer to the virtual function table on the device side, which is not accessible in host code, and vice versa. Thus, trying to invoke virtual functions from a context different from where the object was constructed will be disallowed because the appropriate virtual table cannot be accessed. The virtual function tables for offloading devices with different architecures are different, therefore trying to invoke virtual functions from an offloading device with a different architecture than where the object is constructed is also disallowed.
+
+Example Usage
+-------------
+
+.. code-block:: c++
+
+   class Base {
+   public:
+      __device__ virtual void virtualFunction() {
+         // Base virtual function implementation
+      }
+   };
+
+   class Derived : public Base {
+   public:
+      __device__ void virtualFunction() override {
+         // Derived virtual function implementation
+      }
+   };
+
+   __global__ void kernel() {
+      Derived obj;
+      Base* basePtr = &obj;
+      basePtr->virtualFunction(); // Allowed since obj is constructed in device code
+   }
diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst
index b9466b5a0b..30e288f986 100644
--- a/clang/docs/LanguageExtensions.rst
+++ b/clang/docs/LanguageExtensions.rst
@@ -619,6 +619,14 @@ Let ``T`` be one of the following types:
 
 For scalar types, consider the operation applied to a vector with a single element.
 
+*Vector Size*
+To determine the number of elements in a vector, use ``__builtin_vectorelements()``.
+For fixed-sized vectors, e.g., defined via ``__attribute__((vector_size(N)))`` or ARM
+NEON's vector types (e.g., ``uint16x8_t``), this returns the constant number of
+elements at compile-time. For scalable vectors, e.g., SVE or RISC-V V, the number of
+elements is not known at compile-time and is determined at runtime. This builtin can
+be used, e.g., to increment the loop-counter in vector-type agnostic loops.
+
 *Elementwise Builtins*
 
 Each builtin returns a vector equivalent to applying the specified operation
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 2d918967e7..de1ab29d31 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -37,6 +37,27 @@ These changes are ones which we think may surprise users when upgrading to
 Clang |release| because of the opportunity they pose for disruption to existing
 code bases.
 
+- Fix a bug in reversed argument for templated operators.
+  This breaks code in C++20 which was previously accepted in C++17. Eg:
+
+  .. code-block:: cpp
+
+    struct P {};
+    template<class S> bool operator==(const P&, const S&);
+
+    struct A : public P {};
+    struct B : public P {};
+
+    // This equality is now ambiguous in C++20.
+    bool ambiguous(A a, B b) { return a == b; }
+
+    template<class S> bool operator!=(const P&, const S&);
+    // Ok. Found a matching operator!=.
+    bool fine(A a, B b) { return a == b; }
+
+  To reduce such widespread breakages, as an extension, Clang accepts this code
+  with an existing warning ``-Wambiguous-reversed-operator`` warning.
+  Fixes `GH <https://github.com/llvm/llvm-project/issues/53954>`_.
 
 C/C++ Language Potentially Breaking Changes
 -------------------------------------------
@@ -96,6 +117,14 @@ C++ Specific Potentially Breaking Changes
   Clang as a compiler, but it may break assumptions in Clang-based tools
   iterating over the AST.
 
+- The warning `-Wenum-constexpr-conversion` is now also enabled by default on
+  system headers and macros. It will be turned into a hard (non-downgradable)
+  error in the next Clang release.
+
+- The flag `-fdelayed-template-parsing` won't be enabled by default with C++20
+  when targetting MSVC to match the behavior of MSVC. 
+  (`MSVC Docs <https://learn.microsoft.com/en-us/cpp/build/reference/permissive-standards-conformance?view=msvc-170>`_)
+
 ABI Changes in This Version
 ---------------------------
 - Following the SystemV ABI for x86-64, ``__int128`` arguments will no longer
@@ -157,6 +186,11 @@ C Language Changes
 - ``structs``, ``unions``, and ``arrays`` that are const may now be used as
   constant expressions.  This change is more consistent with the behavior of
   GCC.
+- Clang now supports the C-only attribute ``counted_by``. When applied to a
+  struct's flexible array member, it points to the struct field that holds the
+  number of elements in the flexible array member. This information can improve
+  the results of the array bound sanitizer and the
+  ``__builtin_dynamic_object_size`` builtin.
 
 C23 Feature Support
 ^^^^^^^^^^^^^^^^^^^
@@ -167,9 +201,18 @@ C23 Feature Support
 - Clang now supports `requires c23` for module maps.
 - Clang now supports ``N3007 Type inference for object definitions``.
 
+- Clang now supports ``<stdckdint.h>`` which defines several macros for performing
+  checked integer arithmetic.
+
 Non-comprehensive list of changes in this release
 -------------------------------------------------
 
+* Clang now has a ``__builtin_vectorelements()`` function that determines the number of elements in a vector.
+  For fixed-sized vectors, e.g., defined via ``__attribute__((vector_size(N)))`` or ARM NEON's vector types
+  (e.g., ``uint16x8_t``), this returns the constant number of elements at compile-time.
+  For scalable vectors, e.g., SVE or RISC-V V, the number of elements is not known at compile-time and is
+  determined at runtime.
+
 New Compiler Flags
 ------------------
 
@@ -185,6 +228,9 @@ New Compiler Flags
   the preprocessed text to the output. This can greatly reduce the size of the
   preprocessed output, which can be helpful when trying to reduce a test case.
 
+* ``-Wbitfield-conversion`` was added to detect assignments of integral
+  types to a bitfield that may change the value.
+
 Deprecated Compiler Flags
 -------------------------
 
@@ -196,6 +242,10 @@ Modified Compiler Flags
 * ``-frewrite-includes`` now guards the original #include directives with
   ``__CLANG_REWRITTEN_INCLUDES``, and ``__CLANG_REWRITTEN_SYSTEM_INCLUDES`` as
   appropriate.
+* Introducing a new default calling convention for ``-fdefault-calling-conv``:
+  ``rtdcall``. This new default CC only works for M68k and will use the new
+  ``m68k_rtdcc`` CC on every functions that are not variadic. The ``-mrtd``
+  driver/frontend flag has the same effect when targeting M68k.
 
 Removed Compiler Flags
 -------------------------
@@ -300,6 +350,11 @@ Improvements to Clang's diagnostics
       |               ~~~~~~~~~^~~~~~~~
 - Clang now always diagnoses when using non-standard layout types in ``offsetof`` .
   (`#64619: <https://github.com/llvm/llvm-project/issues/64619>`_)
+- Clang now diagnoses use of variable-length arrays in C++ by default (and
+  under ``-Wall`` in GNU++ mode). This is an extension supported by Clang and
+  GCC, but is very easy to accidentally use without realizing it's a
+  nonportable construct that has different semantics from a constant-sized
+  array. (`#62836 <https://github.com/llvm/llvm-project/issues/62836>`_)
 
 Bug Fixes in This Version
 -------------------------
@@ -377,10 +432,16 @@ Bug Fixes in This Version
   cannot be used with ``Release`` mode builds. (`#68237 <https://github.com/llvm/llvm-project/issues/68237>`_).
 - Fix crash in evaluating ``constexpr`` value for invalid template function.
   Fixes (`#68542 <https://github.com/llvm/llvm-project/issues/68542>`_)
-
 - Fixed an issue when a shift count larger than ``__INT64_MAX__``, in a right
   shift operation, could result in missing warnings about
   ``shift count >= width of type`` or internal compiler error.
+- Fixed an issue with computing the common type for the LHS and RHS of a `?:`
+  operator in C. No longer issuing a confusing diagnostic along the lines of
+  "incompatible operand types ('foo' and 'foo')" with extensions such as matrix
+  types. Fixes (`#69008 <https://github.com/llvm/llvm-project/issues/69008>`_)
+- Clang no longer permits using the `_BitInt` types as an underlying type for an
+  enumeration as specified in the C23 Standard.
+  Fixes (`#69619 <https://github.com/llvm/llvm-project/issues/69619>`_)
 
 Bug Fixes to Compiler Builtins
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -497,6 +558,14 @@ Bug Fixes to C++ Support
   rather than prefer the non-templated constructor as specified in
   [standard.group]p3.
 
+- Fixed a crash caused by incorrect handling of dependence on variable templates
+  with non-type template parameters of reference type. Fixes:
+  (`#65153 <https://github.com/llvm/llvm-project/issues/65153>`_)
+
+- Clang now properly compares constraints on an out of line class template
+  declaration definition. Fixes:
+  (`#61763 <https://github.com/llvm/llvm-project/issues/61763>`_)
+
 Bug Fixes to AST Handling
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 - Fixed an import failure of recursive friend class template.
@@ -505,6 +574,8 @@ Bug Fixes to AST Handling
   computed RecordLayout is incorrect if fields are not completely imported and
   should not be cached.
   `Issue 64170 <https://github.com/llvm/llvm-project/issues/64170>`_
+- Fixed ``hasAnyBase`` not binding nodes in its submatcher.
+  (`#65421 <https://github.com/llvm/llvm-project/issues/65421>`_)
 
 Miscellaneous Bug Fixes
 ^^^^^^^^^^^^^^^^^^^^^^^
@@ -535,6 +606,11 @@ X86 Support
 
 - Added option ``-m[no-]evex512`` to disable ZMM and 64-bit mask instructions
   for AVX512 features.
+- Support ISA of ``USER_MSR``.
+  * Support intrinsic of ``_urdmsr``.
+  * Support intrinsic of ``_uwrmsr``.
+- Support ISA of ``AVX10.1``.
+- ``-march=pantherlake`` and ``-march=clearwaterforest`` are now supported.
 
 Arm and AArch64 Support
 ^^^^^^^^^^^^^^^^^^^^^^^
@@ -643,6 +719,8 @@ Static Analyzer
 - Added a new checker ``core.BitwiseShift`` which reports situations where
   bitwise shift operators produce undefined behavior (because some operand is
   negative or too large).
+- Move checker ``alpha.unix.StdCLibraryFunctions`` out of the ``alpha`` package
+  to ``unix.StdCLibraryFunctions``.
 
 - Fix false positive in mutation check when using pointer to member function.
   (`#66204: <https://github.com/llvm/llvm-project/issues/66204>`_).
diff --git a/clang/docs/StandardCPlusPlusModules.rst b/clang/docs/StandardCPlusPlusModules.rst
index 579431bd9a..8dd86edc64 100644
--- a/clang/docs/StandardCPlusPlusModules.rst
+++ b/clang/docs/StandardCPlusPlusModules.rst
@@ -686,6 +686,15 @@ the BMI within ``clang-cl.exe``.
 
 This is tracked in: https://github.com/llvm/llvm-project/issues/64118
 
+delayed template parsing is not supported/broken with C++ modules
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The feature `-fdelayed-template-parsing` can't work well with C++ modules now.
+Note that this is significant on Windows since the option will be enabled by default
+on Windows.
+
+This is tracked in: https://github.com/llvm/llvm-project/issues/61068
+
 Header Units
 ============
 
diff --git a/clang/docs/analyzer/checkers.rst b/clang/docs/analyzer/checkers.rst
index 81f333e644..597ffcc4a1 100644
--- a/clang/docs/analyzer/checkers.rst
+++ b/clang/docs/analyzer/checkers.rst
@@ -1016,7 +1016,7 @@ Check the size argument passed into C string functions for common erroneous patt
 .. _unix-cstring-NullArg:
 
 unix.cstring.NullArg (C)
-"""""""""""""""""""""""""
+""""""""""""""""""""""""
 Check for null pointers being passed as arguments to C string functions:
 ``strlen, strnlen, strcpy, strncpy, strcat, strncat, strcmp, strncmp, strcasecmp, strncasecmp, wcslen, wcsnlen``.
 
@@ -1026,6 +1026,99 @@ Check for null pointers being passed as arguments to C string functions:
    return strlen(0); // warn
  }
 
+.. _unix-StdCLibraryFunctions:
+
+unix.StdCLibraryFunctions (C)
+"""""""""""""""""""""""""""""
+Check for calls of standard library functions that violate predefined argument
+constraints. For example, according to the C standard the behavior of function
+``int isalnum(int ch)`` is undefined if the value of ``ch`` is not representable
+as ``unsigned char`` and is not equal to ``EOF``.
+
+You can think of this checker as defining restrictions (pre- and postconditions)
+on standard library functions. Preconditions are checked, and when they are
+violated, a warning is emitted. Postconditions are added to the analysis, e.g.
+that the return value of a function is not greater than 255. Preconditions are
+added to the analysis too, in the case when the affected values are not known
+before the call.
+
+For example, if an argument to a function must be in between 0 and 255, but the
+value of the argument is unknown, the analyzer will assume that it is in this
+interval. Similarly, if a function mustn't be called with a null pointer and the
+analyzer cannot prove that it is null, then it will assume that it is non-null.
+
+These are the possible checks on the values passed as function arguments:
+ - The argument has an allowed range (or multiple ranges) of values. The checker
+   can detect if a passed value is outside of the allowed range and show the
+   actual and allowed values.
+ - The argument has pointer type and is not allowed to be null pointer. Many
+   (but not all) standard functions can produce undefined behavior if a null
+   pointer is passed, these cases can be detected by the checker.
+ - The argument is a pointer to a memory block and the minimal size of this
+   buffer is determined by another argument to the function, or by
+   multiplication of two arguments (like at function ``fread``), or is a fixed
+   value (for example ``asctime_r`` requires at least a buffer of size 26). The
+   checker can detect if the buffer size is too small and in optimal case show
+   the size of the buffer and the values of the corresponding arguments.
+
+.. code-block:: c
+
+  #define EOF -1
+  void test_alnum_concrete(int v) {
+    int ret = isalnum(256); // \
+    // warning: Function argument outside of allowed range
+    (void)ret;
+  }
+
+  void buffer_size_violation(FILE *file) {
+    enum { BUFFER_SIZE = 1024 };
+    wchar_t wbuf[BUFFER_SIZE];
+
+    const size_t size = sizeof(*wbuf);   // 4
+    const size_t nitems = sizeof(wbuf);  // 4096
+
+    // Below we receive a warning because the 3rd parameter should be the
+    // number of elements to read, not the size in bytes. This case is a known
+    // vulnerability described by the ARR38-C SEI-CERT rule.
+    fread(wbuf, size, nitems, file);
+  }
+
+  int test_alnum_symbolic(int x) {
+    int ret = isalnum(x);
+    // after the call, ret is assumed to be in the range [-1, 255]
+
+    if (ret > 255)      // impossible (infeasible branch)
+      if (x == 0)
+        return ret / x; // division by zero is not reported
+    return ret;
+  }
+
+Additionally to the argument and return value conditions, this checker also adds
+state of the value ``errno`` if applicable to the analysis. Many system
+functions set the ``errno`` value only if an error occurs (together with a
+specific return value of the function), otherwise it becomes undefined. This
+checker changes the analysis state to contain such information. This data is
+used by other checkers, for example :ref:`alpha-unix-Errno`.
+
+**Limitations**
+
+The checker can not always provide notes about the values of the arguments.
+Without this information it is hard to confirm if the constraint is indeed
+violated. The argument values are shown if they are known constants or the value
+is determined by previous (not too complicated) assumptions.
+
+The checker can produce false positives in cases such as if the program has
+invariants not known to the analyzer engine or the bug report path contains
+calls to unknown functions. In these cases the analyzer fails to detect the real
+range of the argument.
+
+**Parameters**
+
+The checker models functions (and emits diagnostics) from the C standard by
+default. The ``ModelPOSIX`` option enables modeling (and emit diagnostics) of
+additional functions that are defined in the POSIX standard. This option is
+disabled by default.
+
 .. _osx-checkers:
 
 osx
@@ -2677,101 +2770,7 @@ For a more detailed description of configuration options, please see the
   file. This causes potential true positive findings to be lost.
 
 alpha.unix
-^^^^^^^^^^^
-
-.. _alpha-unix-StdCLibraryFunctions:
-
-alpha.unix.StdCLibraryFunctions (C)
-"""""""""""""""""""""""""""""""""""
-Check for calls of standard library functions that violate predefined argument
-constraints. For example, it is stated in the C standard that for the ``int
-isalnum(int ch)`` function the behavior is undefined if the value of ``ch`` is
-not representable as unsigned char and is not equal to ``EOF``.
-
-.. code-block:: c
-
-  #define EOF -1
-  void test_alnum_concrete(int v) {
-    int ret = isalnum(256); // \
-    // warning: Function argument outside of allowed range
-    (void)ret;
-  }
-
-  void buffer_size_violation(FILE *file) {
-    enum { BUFFER_SIZE = 1024 };
-    wchar_t wbuf[BUFFER_SIZE];
-
-    const size_t size = sizeof(*wbuf);   // 4
-    const size_t nitems = sizeof(wbuf);  // 4096
-
-    // Below we receive a warning because the 3rd parameter should be the
-    // number of elements to read, not the size in bytes. This case is a known
-    // vulnerability described by the ARR38-C SEI-CERT rule.
-    fread(wbuf, size, nitems, file);
-  }
-
-You can think of this checker as defining restrictions (pre- and postconditions)
-on standard library functions. Preconditions are checked, and when they are
-violated, a warning is emitted. Post conditions are added to the analysis, e.g.
-that the return value must be no greater than 255.
-
-For example if an argument to a function must be in between 0 and 255, but the
-value of the argument is unknown, the analyzer will conservatively assume that
-it is in this interval. Similarly, if a function mustn't be called with a null
-pointer and the null value of the argument can not be proven, the analyzer will
-assume that it is non-null.
-
-These are the possible checks on the values passed as function arguments:
- - The argument has an allowed range (or multiple ranges) of values. The checker
-   can detect if a passed value is outside of the allowed range and show the
-   actual and allowed values.
- - The argument has pointer type and is not allowed to be null pointer. Many
-   (but not all) standard functions can produce undefined behavior if a null
-   pointer is passed, these cases can be detected by the checker.
- - The argument is a pointer to a memory block and the minimal size of this
-   buffer is determined by another argument to the function, or by
-   multiplication of two arguments (like at function ``fread``), or is a fixed
-   value (for example ``asctime_r`` requires at least a buffer of size 26). The
-   checker can detect if the buffer size is too small and in optimal case show
-   the size of the buffer and the values of the corresponding arguments.
-
-.. code-block:: c
-
-  int test_alnum_symbolic(int x) {
-    int ret = isalnum(x);
-    // after the call, ret is assumed to be in the range [-1, 255]
-
-    if (ret > 255)      // impossible (infeasible branch)
-      if (x == 0)
-        return ret / x; // division by zero is not reported
-    return ret;
-  }
-
-Additionally to the argument and return value conditions, this checker also adds
-state of the value ``errno`` if applicable to the analysis. Many system
-functions set the ``errno`` value only if an error occurs (together with a
-specific return value of the function), otherwise it becomes undefined. This
-checker changes the analysis state to contain such information. This data is
-used by other checkers, for example :ref:`alpha-unix-Errno`.
-
-**Limitations**
-
-The checker can not always provide notes about the values of the arguments.
-Without this information it is hard to confirm if the constraint is indeed
-violated. The argument values are shown if they are known constants or the value
-is determined by previous (not too complicated) assumptions.
-
-The checker can produce false positives in cases such as if the program has
-invariants not known to the analyzer engine or the bug report path contains
-calls to unknown functions. In these cases the analyzer fails to detect the real
-range of the argument.
-
-**Parameters**
-
-The checker models functions (and emits diagnostics) from the C standard by
-default. The ``ModelPOSIX`` option enables modeling (and emit diagnostics) of
-additional functions that are defined in the POSIX standard. This option is
-disabled by default.
+^^^^^^^^^^
 
 .. _alpha-unix-BlockInCriticalSection:
 
@@ -2840,9 +2839,9 @@ pages of the functions and in the `POSIX standard <https://pubs.opengroup.org/on
    return 1;
  }
 
-The checker :ref:`alpha-unix-StdCLibraryFunctions` must be turned on to get the
+The checker :ref:`unix-StdCLibraryFunctions` must be turned on to get the
 warnings from this checker. The supported functions are the same as by
-:ref:`alpha-unix-StdCLibraryFunctions`. The ``ModelPOSIX`` option of that
+:ref:`unix-StdCLibraryFunctions`. The ``ModelPOSIX`` option of that
 checker affects the set of checked functions.
 
 **Parameters**
diff --git a/clang/docs/tools/clang-formatted-files.txt b/clang/docs/tools/clang-formatted-files.txt
index 16f8472711..48cd800bff 100644
--- a/clang/docs/tools/clang-formatted-files.txt
+++ b/clang/docs/tools/clang-formatted-files.txt
@@ -6813,7 +6813,6 @@ llvm/lib/Target/X86/X86LoadValueInjectionRetHardening.cpp
 llvm/lib/Target/X86/X86LowerAMXIntrinsics.cpp
 llvm/lib/Target/X86/X86LowerAMXType.cpp
 llvm/lib/Target/X86/X86LowerTileCopy.cpp
-llvm/lib/Target/X86/X86PreAMXConfig.cpp
 llvm/lib/Target/X86/X86PreTileConfig.cpp
 llvm/lib/Target/X86/X86RegisterBankInfo.h
 llvm/lib/Target/X86/X86ShuffleDecodeConstantPool.cpp
diff --git a/clang/docs/tools/dump_format_style.py b/clang/docs/tools/dump_format_style.py
index 270ac34c03..75d4a044ef 100755
--- a/clang/docs/tools/dump_format_style.py
+++ b/clang/docs/tools/dump_format_style.py
@@ -143,11 +143,18 @@ def __str__(self):
 
 
 class NestedField(object):
-    def __init__(self, name, comment):
+    def __init__(self, name, comment, version):
         self.name = name
         self.comment = comment.strip()
+        self.version = version
 
     def __str__(self):
+        if self.version:
+            return "\n* ``%s`` :versionbadge:`clang-format %s`\n%s" % (
+                self.name,
+                self.version,
+                doxygen2rst(indent(self.comment, 2, indent_first_line=False)),
+            )
         return "\n* ``%s`` %s" % (
             self.name,
             doxygen2rst(indent(self.comment, 2, indent_first_line=False)),
@@ -165,18 +172,28 @@ def __str__(self):
 
 
 class NestedEnum(object):
-    def __init__(self, name, enumtype, comment, values):
+    def __init__(self, name, enumtype, comment, version, values):
         self.name = name
         self.comment = comment
         self.values = values
         self.type = enumtype
+        self.version = version
 
     def __str__(self):
-        s = "\n* ``%s %s``\n%s" % (
-            to_yaml_type(self.type),
-            self.name,
-            doxygen2rst(indent(self.comment, 2)),
-        )
+        s = ""
+        if self.version:
+            s = "\n* ``%s %s`` :versionbadge:`clang-format %s`\n\n%s" % (
+                to_yaml_type(self.type),
+                self.name,
+                self.version,
+                doxygen2rst(indent(self.comment, 2)),
+            )
+        else:
+            s = "\n* ``%s %s``\n%s" % (
+                to_yaml_type(self.type),
+                self.name,
+                doxygen2rst(indent(self.comment, 2)),
+            )
         s += indent("\nPossible values:\n\n", 2)
         s += indent("\n".join(map(str, self.values)), 2)
         return s
@@ -278,7 +295,9 @@ class State:
                 InFieldComment,
                 InEnum,
                 InEnumMemberComment,
-            ) = range(8)
+                InNestedEnum,
+                InNestedEnumMemberComment,
+            ) = range(10)
 
         state = State.BeforeStruct
 
@@ -344,27 +363,38 @@ class State:
                     state = State.InStruct
                     nested_structs[nested_struct.name] = nested_struct
             elif state == State.InNestedFieldComment:
-                if line.startswith("///"):
+                if line.startswith(r"/// \version"):
+                    match = re.match(r"/// \\version\s*(?P<version>[0-9.]+)*", line)
+                    if match:
+                        version = match.group("version")
+                elif line.startswith("///"):
                     comment += self.__clean_comment_line(line)
+                elif line.startswith("enum"):
+                    state = State.InNestedEnum
+                    name = re.sub(r"enum\s+(\w+)\s*(:((\s*\w+)+)\s*)?\{", "\\1", line)
+                    enum = Enum(name, comment)
                 else:
                     state = State.InNestedStruct
                     field_type, field_name = re.match(
                         r"([<>:\w(,\s)]+)\s+(\w+);", line
                     ).groups()
+                    # if not version:
+                    #    self.__warning(f"missing version for {field_name}", line)
                     if field_type in enums:
                         nested_struct.values.append(
                             NestedEnum(
                                 field_name,
                                 field_type,
                                 comment,
+                                version,
                                 enums[field_type].values,
                             )
                         )
                     else:
                         nested_struct.values.append(
-                            NestedField(field_type + " " + field_name, comment)
+                            NestedField(field_type + " " + field_name, comment, version)
                         )
-
+                    version = None
             elif state == State.InEnum:
                 if line.startswith("///"):
                     state = State.InEnumMemberComment
@@ -376,6 +406,17 @@ class State:
                     # Enum member without documentation. Must be documented where the enum
                     # is used.
                     pass
+            elif state == State.InNestedEnum:
+                if line.startswith("///"):
+                    state = State.InNestedEnumMemberComment
+                    comment = self.__clean_comment_line(line)
+                elif line == "};":
+                    state = State.InNestedStruct
+                    enums[enum.name] = enum
+                else:
+                    # Enum member without documentation. Must be
+                    # documented where the enum is used.
+                    pass
             elif state == State.InEnumMemberComment:
                 if line.startswith("///"):
                     comment += self.__clean_comment_line(line)
@@ -389,6 +430,19 @@ class State:
                     else:
                         config = val
                     enum.values.append(EnumValue(val, comment, config))
+            elif state == State.InNestedEnumMemberComment:
+                if line.startswith("///"):
+                    comment += self.__clean_comment_line(line)
+                else:
+                    state = State.InNestedEnum
+                    val = line.replace(",", "")
+                    pos = val.find(" // ")
+                    if pos != -1:
+                        config = val[pos + 4 :]
+                        val = val[:pos]
+                    else:
+                        config = val
+                    enum.values.append(EnumValue(val, comment, config))
         if state != State.Finished:
             raise Exception("Not finished by the end of file")
 
diff --git a/clang/include/clang-c/Index.h b/clang/include/clang-c/Index.h
index 1b91feabd5..64ab337895 100644
--- a/clang/include/clang-c/Index.h
+++ b/clang/include/clang-c/Index.h
@@ -2980,6 +2980,7 @@ enum CXCallingConv {
   CXCallingConv_AArch64VectorCall = 16,
   CXCallingConv_SwiftAsync = 17,
   CXCallingConv_AArch64SVEPCS = 18,
+  CXCallingConv_M68kRTD = 19,
 
   CXCallingConv_Invalid = 100,
   CXCallingConv_Unexposed = 200
diff --git a/clang/include/clang/ARCMigrate/FileRemapper.h b/clang/include/clang/ARCMigrate/FileRemapper.h
index ceb3e67b50..afcee36351 100644
--- a/clang/include/clang/ARCMigrate/FileRemapper.h
+++ b/clang/include/clang/ARCMigrate/FileRemapper.h
@@ -12,10 +12,10 @@
 #include "clang/Basic/FileEntry.h"
 #include "clang/Basic/LLVM.h"
 #include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/PointerUnion.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include <memory>
+#include <variant>
 
 namespace llvm {
   class MemoryBuffer;
@@ -33,7 +33,7 @@ class FileRemapper {
   // FIXME: Reuse the same FileManager for multiple ASTContexts.
   std::unique_ptr<FileManager> FileMgr;
 
-  typedef llvm::PointerUnion<FileEntryRef, llvm::MemoryBuffer *> Target;
+  using Target = std::variant<FileEntryRef, llvm::MemoryBuffer *>;
   using MappingsTy = llvm::DenseMap<FileEntryRef, Target>;
   MappingsTy FromToMappings;
 
diff --git a/clang/include/clang/AST/Decl.h b/clang/include/clang/AST/Decl.h
index 02e30e24c8..7f076cc77e 100644
--- a/clang/include/clang/AST/Decl.h
+++ b/clang/include/clang/AST/Decl.h
@@ -4302,6 +4302,30 @@ class RecordDecl : public TagDecl {
     return field_begin() == field_end();
   }
 
+  FieldDecl *getLastField() {
+    FieldDecl *FD = nullptr;
+    for (FieldDecl *Field : fields())
+      FD = Field;
+    return FD;
+  }
+  const FieldDecl *getLastField() const {
+    return const_cast<RecordDecl *>(this)->getLastField();
+  }
+
+  template <typename Functor>
+  const FieldDecl *findFieldIf(Functor &Pred) const {
+    for (const Decl *D : decls()) {
+      if (const auto *FD = dyn_cast<FieldDecl>(D); FD && Pred(FD))
+        return FD;
+
+      if (const auto *RD = dyn_cast<RecordDecl>(D))
+        if (const FieldDecl *FD = RD->findFieldIf(Pred))
+          return FD;
+    }
+
+    return nullptr;
+  }
+
   /// Note that the definition of this type is now complete.
   virtual void completeDefinition();
 
diff --git a/clang/include/clang/AST/DeclBase.h b/clang/include/clang/AST/DeclBase.h
index 12137387b6..d383e46e22 100644
--- a/clang/include/clang/AST/DeclBase.h
+++ b/clang/include/clang/AST/DeclBase.h
@@ -18,6 +18,7 @@
 #include "clang/AST/DeclarationName.h"
 #include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/LLVM.h"
+#include "clang/Basic/LangOptions.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/Specifiers.h"
 #include "llvm/ADT/ArrayRef.h"
@@ -477,6 +478,15 @@ class alignas(8) Decl {
   // Return true if this is a FileContext Decl.
   bool isFileContextDecl() const;
 
+  /// Whether it resembles a flexible array member. This is a static member
+  /// because we want to be able to call it with a nullptr. That allows us to
+  /// perform non-Decl specific checks based on the object's type and strict
+  /// flex array level.
+  static bool isFlexibleArrayMemberLike(
+      ASTContext &Context, const Decl *D, QualType Ty,
+      LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel,
+      bool IgnoreTemplateOrMacroSubstitution);
+
   ASTContext &getASTContext() const LLVM_READONLY;
 
   /// Helper to get the language options from the ASTContext.
diff --git a/clang/include/clang/AST/DeclCXX.h b/clang/include/clang/AST/DeclCXX.h
index aa3e3322fa..5eaae6bdd2 100644
--- a/clang/include/clang/AST/DeclCXX.h
+++ b/clang/include/clang/AST/DeclCXX.h
@@ -1052,6 +1052,12 @@ class CXXRecordDecl : public RecordDecl {
     return static_cast<LambdaCaptureDefault>(getLambdaData().CaptureDefault);
   }
 
+  bool isCapturelessLambda() const {
+    if (!isLambda())
+      return false;
+    return getLambdaCaptureDefault() == LCD_None && capture_size() == 0;
+  }
+
   /// Set the captures for this lambda closure type.
   void setCaptures(ASTContext &Context, ArrayRef<LambdaCapture> Captures);
 
diff --git a/clang/include/clang/AST/Expr.h b/clang/include/clang/AST/Expr.h
index b69c616b00..638f886edd 100644
--- a/clang/include/clang/AST/Expr.h
+++ b/clang/include/clang/AST/Expr.h
@@ -607,6 +607,13 @@ class Expr : public ValueStmt {
     /// foldable. If the expression is foldable, but not a constant expression,
     /// the notes will describes why it isn't a constant expression. If the
     /// expression *is* a constant expression, no notes will be produced.
+    ///
+    /// FIXME: this causes significant performance concerns and should be
+    /// refactored at some point. Not all evaluations of the constant
+    /// expression interpreter will display the given diagnostics, this means
+    /// those kinds of uses are paying the expense of generating a diagnostic
+    /// (which may include expensive operations like converting APValue objects
+    /// to a string representation).
     SmallVectorImpl<PartialDiagnosticAt> *Diag = nullptr;
 
     EvalStatus() = default;
diff --git a/clang/include/clang/AST/ExprCXX.h b/clang/include/clang/AST/ExprCXX.h
index 17dbb5e888..798c98cfcf 100644
--- a/clang/include/clang/AST/ExprCXX.h
+++ b/clang/include/clang/AST/ExprCXX.h
@@ -3191,7 +3191,8 @@ class UnresolvedLookupExpr final
                        const DeclarationNameInfo &NameInfo, bool RequiresADL,
                        bool Overloaded,
                        const TemplateArgumentListInfo *TemplateArgs,
-                       UnresolvedSetIterator Begin, UnresolvedSetIterator End);
+                       UnresolvedSetIterator Begin, UnresolvedSetIterator End,
+                       bool KnownDependent);
 
   UnresolvedLookupExpr(EmptyShell Empty, unsigned NumResults,
                        bool HasTemplateKWAndArgsInfo);
@@ -3211,12 +3212,15 @@ class UnresolvedLookupExpr final
          const DeclarationNameInfo &NameInfo, bool RequiresADL, bool Overloaded,
          UnresolvedSetIterator Begin, UnresolvedSetIterator End);
 
+  // After canonicalization, there may be dependent template arguments in
+  // CanonicalConverted But none of Args is dependent. When any of
+  // CanonicalConverted dependent, KnownDependent is true.
   static UnresolvedLookupExpr *
   Create(const ASTContext &Context, CXXRecordDecl *NamingClass,
          NestedNameSpecifierLoc QualifierLoc, SourceLocation TemplateKWLoc,
          const DeclarationNameInfo &NameInfo, bool RequiresADL,
          const TemplateArgumentListInfo *Args, UnresolvedSetIterator Begin,
-         UnresolvedSetIterator End);
+         UnresolvedSetIterator End, bool KnownDependent);
 
   static UnresolvedLookupExpr *CreateEmpty(const ASTContext &Context,
                                            unsigned NumResults,
diff --git a/clang/include/clang/AST/Type.h b/clang/include/clang/AST/Type.h
index a78d8f6046..e3dbe3b8a4 100644
--- a/clang/include/clang/AST/Type.h
+++ b/clang/include/clang/AST/Type.h
@@ -1482,7 +1482,8 @@ class ExtQualsTypeCommonBase {
 /// in three low bits on the QualType pointer; a fourth bit records whether
 /// the pointer is an ExtQuals node. The extended qualifiers (address spaces,
 /// Objective-C GC attributes) are much more rare.
-class ExtQuals : public ExtQualsTypeCommonBase, public llvm::FoldingSetNode {
+class alignas(TypeAlignment) ExtQuals : public ExtQualsTypeCommonBase,
+                                        public llvm::FoldingSetNode {
   // NOTE: changing the fast qualifiers should be straightforward as
   // long as you don't make 'const' non-fast.
   // 1. Qualifiers:
@@ -1594,7 +1595,7 @@ enum class AutoTypeKeyword {
 ///
 /// Types, once created, are immutable.
 ///
-class alignas(8) Type : public ExtQualsTypeCommonBase {
+class alignas(TypeAlignment) Type : public ExtQualsTypeCommonBase {
 public:
   enum TypeClass {
 #define TYPE(Class, Base) Class,
@@ -1982,9 +1983,10 @@ class alignas(8) Type : public ExtQualsTypeCommonBase {
   Type(TypeClass tc, QualType canon, TypeDependence Dependence)
       : ExtQualsTypeCommonBase(this,
                                canon.isNull() ? QualType(this_(), 0) : canon) {
-    static_assert(sizeof(*this) <= 8 + sizeof(ExtQualsTypeCommonBase),
+    static_assert(sizeof(*this) <=
+                      alignof(decltype(*this)) + sizeof(ExtQualsTypeCommonBase),
                   "changing bitfields changed sizeof(Type)!");
-    static_assert(alignof(decltype(*this)) % sizeof(void *) == 0,
+    static_assert(alignof(decltype(*this)) % TypeAlignment == 0,
                   "Insufficient alignment!");
     TypeBits.TC = tc;
     TypeBits.Dependence = static_cast<unsigned>(Dependence);
@@ -2058,6 +2060,9 @@ class alignas(8) Type : public ExtQualsTypeCommonBase {
   bool isSizelessType() const;
   bool isSizelessBuiltinType() const;
 
+  /// Returns true for all scalable vector types.
+  bool isSizelessVectorType() const;
+
   /// Returns true for SVE scalable vector types.
   bool isSVESizelessBuiltinType() const;
 
@@ -5348,7 +5353,7 @@ class DeducedType : public Type {
 
 /// Represents a C++11 auto or C++14 decltype(auto) type, possibly constrained
 /// by a type-constraint.
-class alignas(8) AutoType : public DeducedType, public llvm::FoldingSetNode {
+class AutoType : public DeducedType, public llvm::FoldingSetNode {
   friend class ASTContext; // ASTContext creates these
 
   ConceptDecl *TypeConstraintConcept;
@@ -5456,9 +5461,7 @@ class DeducedTemplateSpecializationType : public DeducedType,
 /// TemplateArguments, followed by a QualType representing the
 /// non-canonical aliased type when the template is a type alias
 /// template.
-class alignas(8) TemplateSpecializationType
-    : public Type,
-      public llvm::FoldingSetNode {
+class TemplateSpecializationType : public Type, public llvm::FoldingSetNode {
   friend class ASTContext; // ASTContext creates these
 
   /// The name of the template being specialized.  This is
@@ -5872,9 +5875,8 @@ class DependentNameType : public TypeWithKeyword, public llvm::FoldingSetNode {
 /// Represents a template specialization type whose template cannot be
 /// resolved, e.g.
 ///   A<T>::template B<T>
-class alignas(8) DependentTemplateSpecializationType
-    : public TypeWithKeyword,
-      public llvm::FoldingSetNode {
+class DependentTemplateSpecializationType : public TypeWithKeyword,
+                                            public llvm::FoldingSetNode {
   friend class ASTContext; // ASTContext creates these
 
   /// The nested name specifier containing the qualifier.
diff --git a/clang/include/clang/AST/TypeLoc.h b/clang/include/clang/AST/TypeLoc.h
index 98427a8dcb..5bb487f1a7 100644
--- a/clang/include/clang/AST/TypeLoc.h
+++ b/clang/include/clang/AST/TypeLoc.h
@@ -243,7 +243,7 @@ class TypeLoc {
 
 inline TypeSourceInfo::TypeSourceInfo(QualType ty, size_t DataSize) : Ty(ty) {
   // Init data attached to the object. See getTypeLoc.
-  memset(this + 1, 0, DataSize);
+  memset(static_cast<void *>(this + 1), 0, DataSize);
 }
 
 /// Return the TypeLoc for a type source info.
diff --git a/clang/include/clang/Analysis/Analyses/ThreadSafety.h b/clang/include/clang/Analysis/Analyses/ThreadSafety.h
index 1808d1d71e..0866b09bab 100644
--- a/clang/include/clang/Analysis/Analyses/ThreadSafety.h
+++ b/clang/include/clang/Analysis/Analyses/ThreadSafety.h
@@ -47,7 +47,13 @@ enum ProtectedOperationKind {
   POK_PassByRef,
 
   /// Passing a pt-guarded variable by reference.
-  POK_PtPassByRef
+  POK_PtPassByRef,
+
+  /// Returning a guarded variable by reference.
+  POK_ReturnByRef,
+
+  /// Returning a pt-guarded variable by reference.
+  POK_PtReturnByRef,
 };
 
 /// This enum distinguishes between different kinds of lock actions. For
diff --git a/clang/include/clang/Analysis/FlowSensitive/Arena.h b/clang/include/clang/Analysis/FlowSensitive/Arena.h
index 4e07053aae..4be308c43f 100644
--- a/clang/include/clang/Analysis/FlowSensitive/Arena.h
+++ b/clang/include/clang/Analysis/FlowSensitive/Arena.h
@@ -20,7 +20,9 @@ namespace clang::dataflow {
 /// For example, `Value`, `StorageLocation`, `Atom`, and `Formula`.
 class Arena {
 public:
-  Arena() : True(makeAtom()), False(makeAtom()) {}
+  Arena()
+      : True(Formula::create(Alloc, Formula::Literal, {}, 1)),
+        False(Formula::create(Alloc, Formula::Literal, {}, 0)) {}
   Arena(const Arena &) = delete;
   Arena &operator=(const Arena &) = delete;
 
@@ -106,9 +108,7 @@ class Arena {
   const Formula &makeAtomRef(Atom A);
 
   /// Returns a formula for a literal true/false.
-  const Formula &makeLiteral(bool Value) {
-    return makeAtomRef(Value ? True : False);
-  }
+  const Formula &makeLiteral(bool Value) { return Value ? True : False; }
 
   // Parses a formula from its textual representation.
   // This may refer to atoms that were not produced by makeAtom() yet!
@@ -144,7 +144,7 @@ class Arena {
   llvm::DenseMap<const Formula *, BoolValue *> FormulaValues;
   unsigned NextAtom = 0;
 
-  Atom True, False;
+  const Formula &True, &False;
 };
 
 } // namespace clang::dataflow
diff --git a/clang/include/clang/Analysis/FlowSensitive/DataflowAnalysis.h b/clang/include/clang/Analysis/FlowSensitive/DataflowAnalysis.h
index dd1a685cb4..c5f14cfcd4 100644
--- a/clang/include/clang/Analysis/FlowSensitive/DataflowAnalysis.h
+++ b/clang/include/clang/Analysis/FlowSensitive/DataflowAnalysis.h
@@ -28,7 +28,6 @@
 #include "clang/Analysis/FlowSensitive/MatchSwitch.h"
 #include "clang/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.h"
 #include "clang/Analysis/FlowSensitive/WatchedLiteralsSolver.h"
-#include "llvm/ADT/Any.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/STLFunctionalExtras.h"
 #include "llvm/ADT/SmallVector.h"
diff --git a/clang/include/clang/Analysis/FlowSensitive/DataflowAnalysisContext.h b/clang/include/clang/Analysis/FlowSensitive/DataflowAnalysisContext.h
index 4698f0616e..a792c3f911 100644
--- a/clang/include/clang/Analysis/FlowSensitive/DataflowAnalysisContext.h
+++ b/clang/include/clang/Analysis/FlowSensitive/DataflowAnalysisContext.h
@@ -98,7 +98,7 @@ class DataflowAnalysisContext {
   StorageLocation &createStorageLocation(QualType Type);
 
   /// Returns a stable storage location for `D`.
-  StorageLocation &getStableStorageLocation(const VarDecl &D);
+  StorageLocation &getStableStorageLocation(const ValueDecl &D);
 
   /// Returns a stable storage location for `E`.
   StorageLocation &getStableStorageLocation(const Expr &E);
@@ -133,10 +133,6 @@ class DataflowAnalysisContext {
   /// identified by `Token` imply that `Val` is true.
   bool flowConditionImplies(Atom Token, const Formula &);
 
-  /// Returns true if and only if the constraints of the flow condition
-  /// identified by `Token` are always true.
-  bool flowConditionIsTautology(Atom Token);
-
   /// Returns true if `Val1` is equivalent to `Val2`.
   /// Note: This function doesn't take into account constraints on `Val1` and
   /// `Val2` imposed by the flow condition.
diff --git a/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h b/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h
index 73f747ff88..9ac2cb90cc 100644
--- a/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h
+++ b/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h
@@ -242,7 +242,7 @@ class Environment {
   /// Creates a storage location for `D`. Does not assign the returned storage
   /// location to `D` in the environment. Does not assign a value to the
   /// returned storage location in the environment.
-  StorageLocation &createStorageLocation(const VarDecl &D);
+  StorageLocation &createStorageLocation(const ValueDecl &D);
 
   /// Creates a storage location for `E`. Does not assign the returned storage
   /// location to `E` in the environment. Does not assign a value to the
@@ -408,7 +408,7 @@ class Environment {
   /// this value. Otherwise, initializes the object with a value created using
   /// `createValue()`.  Uses the storage location returned by
   /// `DataflowAnalysisContext::getStableStorageLocation(D)`.
-  StorageLocation &createObject(const VarDecl &D, const Expr *InitExpr) {
+  StorageLocation &createObject(const ValueDecl &D, const Expr *InitExpr) {
     return createObjectInternal(&D, D.getType(), InitExpr);
   }
 
@@ -473,9 +473,8 @@ class Environment {
 
   /// Returns a symbolic boolean value that models a boolean literal equal to
   /// `Value`
-  AtomicBoolValue &getBoolLiteralValue(bool Value) const {
-    return cast<AtomicBoolValue>(
-        arena().makeBoolValue(arena().makeLiteral(Value)));
+  BoolValue &getBoolLiteralValue(bool Value) const {
+    return arena().makeBoolValue(arena().makeLiteral(Value));
   }
 
   /// Returns an atomic boolean value.
@@ -614,7 +613,7 @@ class Environment {
 
   /// Shared implementation of `createObject()` overloads.
   /// `D` and `InitExpr` may be null.
-  StorageLocation &createObjectInternal(const VarDecl *D, QualType Ty,
+  StorageLocation &createObjectInternal(const ValueDecl *D, QualType Ty,
                                         const Expr *InitExpr);
 
   /// Shared implementation of `pushCall` overloads. Note that unlike
diff --git a/clang/include/clang/Analysis/FlowSensitive/Formula.h b/clang/include/clang/Analysis/FlowSensitive/Formula.h
index 51264444fd..982e400c1d 100644
--- a/clang/include/clang/Analysis/FlowSensitive/Formula.h
+++ b/clang/include/clang/Analysis/FlowSensitive/Formula.h
@@ -13,7 +13,6 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseMapInfo.h"
-#include "llvm/ADT/STLFunctionalExtras.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cassert>
@@ -53,7 +52,8 @@ class alignas(const Formula *) Formula {
     /// A reference to an atomic boolean variable.
     /// We name these e.g. "V3", where 3 == atom identity == Value.
     AtomRef,
-    // FIXME: add const true/false rather than modeling them as variables
+    /// Constant true or false.
+    Literal,
 
     Not, /// True if its only operand is false
 
@@ -70,6 +70,11 @@ class alignas(const Formula *) Formula {
     return static_cast<Atom>(Value);
   }
 
+  bool literal() const {
+    assert(kind() == Literal);
+    return static_cast<bool>(Value);
+  }
+
   ArrayRef<const Formula *> operands() const {
     return ArrayRef(reinterpret_cast<Formula *const *>(this + 1),
                     numOperands(kind()));
@@ -82,9 +87,9 @@ class alignas(const Formula *) Formula {
   void print(llvm::raw_ostream &OS, const AtomNames * = nullptr) const;
 
   // Allocate Formulas using Arena rather than calling this function directly.
-  static Formula &create(llvm::BumpPtrAllocator &Alloc, Kind K,
-                         ArrayRef<const Formula *> Operands,
-                         unsigned Value = 0);
+  static const Formula &create(llvm::BumpPtrAllocator &Alloc, Kind K,
+                               ArrayRef<const Formula *> Operands,
+                               unsigned Value = 0);
 
 private:
   Formula() = default;
@@ -94,6 +99,7 @@ class alignas(const Formula *) Formula {
   static unsigned numOperands(Kind K) {
     switch (K) {
     case AtomRef:
+    case Literal:
       return 0;
     case Not:
       return 1;
diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index 7a6ec77ae8..5486b36133 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -2805,6 +2805,11 @@ def PreserveAll : DeclOrTypeAttr {
   let Documentation = [PreserveAllDocs];
 }
 
+def M68kRTD: DeclOrTypeAttr {
+  let Spellings = [Clang<"m68k_rtd">];
+  let Documentation = [M68kRTDDocs];
+}
+
 def Target : InheritableAttr {
   let Spellings = [GCC<"target">];
   let Args = [StringArgument<"featuresStr">];
@@ -4246,3 +4251,21 @@ def AvailableOnlyInDefaultEvalMethod : InheritableAttr {
   let Subjects = SubjectList<[TypedefName], ErrorDiag>;
   let Documentation = [Undocumented];
 }
+
+def CountedBy : InheritableAttr {
+  let Spellings = [Clang<"counted_by">];
+  let Subjects = SubjectList<[Field]>;
+  let Args = [IdentifierArgument<"CountedByField">];
+  let Documentation = [CountedByDocs];
+  let LangOpts = [COnly];
+  // FIXME: This is ugly. Let using a DeclArgument would be nice, but a Decl
+  // isn't yet available due to the fact that we're still parsing the
+  // structure. Maybe that code could be changed sometime in the future.
+  code AdditionalMembers = [{
+    private:
+      SourceRange CountedByFieldLoc;
+    public:
+      SourceRange getCountedByFieldLoc() const { return CountedByFieldLoc; }
+      void setCountedByFieldLoc(SourceRange Loc) { CountedByFieldLoc = Loc; }
+  }];
+}
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index 8d928dcc14..cbbf69faeb 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -2825,6 +2825,18 @@ See the documentation for `__vectorcall`_ on MSDN for more details.
   }];
 }
 
+def M68kRTDDocs : Documentation {
+  let Category = DocCatCallingConvs;
+  let Content = [{
+On M68k targets, this attribute changes the calling convention of a function
+to clear parameters off the stack on return. In other words, callee is
+responsible for cleaning out the stack space allocated for incoming paramters.
+This convention does not support variadic calls or unprototyped functions in C.
+When targeting M68010 or newer CPUs, this calling convention is implemented
+using the `rtd` instruction.
+  }];
+}
+
 def DocCatConsumed : DocumentationCategory<"Consumed Annotation Checking"> {
   let Content = [{
 Clang supports additional attributes for checking basic resource management
@@ -7275,3 +7287,69 @@ relative ordering of values is important. For example:
 attribute, they default to the value ``65535``.
 }];
 }
+
+def CountedByDocs : Documentation {
+  let Category = DocCatField;
+  let Content = [{
+Clang supports the ``counted_by`` attribute on the flexible array member of a
+structure in C. The argument for the attribute is the name of a field member in
+the same structure holding the count of elements in the flexible array. This
+information can be used to improve the results of the array bound sanitizer and
+the ``__builtin_dynamic_object_size`` builtin.
+
+For example, the following code:
+
+.. code-block:: c
+
+  struct bar;
+
+  struct foo {
+    size_t count;
+    char other;
+    struct bar *array[] __attribute__((counted_by(count)));
+  };
+
+specifies that the flexible array member ``array`` has the number of elements
+allocated for it stored in ``count``. This establishes a relationship between
+``array`` and ``count``. Specifically, ``p->array`` must have at least
+``p->count`` number of elements available. It's the user's responsibility to
+ensure that this relationship is maintained through changes to the structure.
+
+In the following example, the allocated array erroneously has fewer elements
+than what's specified by ``p->count``. This would result in an out-of-bounds
+access not being detected.
+
+.. code-block:: c
+
+  #define SIZE_INCR 42
+
+  struct foo *p;
+
+  void foo_alloc(size_t count) {
+    p = malloc(MAX(sizeof(struct foo),
+                   offsetof(struct foo, array[0]) + count * sizeof(struct bar *)));
+    p->count = count + SIZE_INCR;
+  }
+
+The next example updates ``p->count``, breaking the relationship requirement
+that ``p->array`` must have at least ``p->count`` number of elements available:
+
+.. code-block:: c
+
+  #define SIZE_INCR 42
+
+  struct foo *p;
+
+  void foo_alloc(size_t count) {
+    p = malloc(MAX(sizeof(struct foo),
+                   offsetof(struct foo, array[0]) + count * sizeof(struct bar *)));
+    p->count = count;
+  }
+
+  void use_foo(int index) {
+    p->count += SIZE_INCR + 1; /* 'count' is now larger than the number of elements of 'array'. */
+    p->array[index] = 0;       /* the sanitizer can't properly check if this is an out-of-bounds access. */
+  }
+
+  }];
+}
diff --git a/clang/include/clang/Basic/Builtins.def b/clang/include/clang/Basic/Builtins.def
index 6ea8484606..6033e8a955 100644
--- a/clang/include/clang/Basic/Builtins.def
+++ b/clang/include/clang/Basic/Builtins.def
@@ -674,6 +674,7 @@ BUILTIN(__builtin_debugtrap, "v", "n")
 BUILTIN(__builtin_unreachable, "v", "nr")
 BUILTIN(__builtin_shufflevector, "v."   , "nct")
 BUILTIN(__builtin_convertvector, "v."   , "nct")
+BUILTIN(__builtin_vectorelements, "v."  , "nct")
 BUILTIN(__builtin_alloca, "v*z"   , "Fn")
 BUILTIN(__builtin_alloca_uninitialized, "v*z", "Fn")
 BUILTIN(__builtin_alloca_with_align, "v*zIz", "Fn")
diff --git a/clang/include/clang/Basic/BuiltinsAArch64.def b/clang/include/clang/Basic/BuiltinsAArch64.def
index 12c7a371e0..82a1ba3c82 100644
--- a/clang/include/clang/Basic/BuiltinsAArch64.def
+++ b/clang/include/clang/Basic/BuiltinsAArch64.def
@@ -283,6 +283,8 @@ TARGET_HEADER_BUILTIN(_CountLeadingZeros64, "UiULLi", "nh", INTRIN_H, ALL_MS_LAN
 TARGET_HEADER_BUILTIN(_CountOneBits, "UiUNi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_CountOneBits64, "UiULLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
 
+TARGET_HEADER_BUILTIN(__prefetch, "vv*", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+
 #undef BUILTIN
 #undef LANGBUILTIN
 #undef TARGET_BUILTIN
diff --git a/clang/include/clang/Basic/BuiltinsNVPTX.def b/clang/include/clang/Basic/BuiltinsNVPTX.def
index f645ad25cb..d74a7d1e55 100644
--- a/clang/include/clang/Basic/BuiltinsNVPTX.def
+++ b/clang/include/clang/Basic/BuiltinsNVPTX.def
@@ -920,22 +920,22 @@ TARGET_BUILTIN(__hmma_m16n16k16_ld_a, "vi*iC*UiIi", "", AND(SM_70,PTX60))
 TARGET_BUILTIN(__hmma_m16n16k16_ld_b, "vi*iC*UiIi", "", AND(SM_70,PTX60))
 TARGET_BUILTIN(__hmma_m16n16k16_ld_c_f16, "vi*iC*UiIi", "", AND(SM_70,PTX60))
 TARGET_BUILTIN(__hmma_m16n16k16_ld_c_f32, "vf*fC*UiIi", "", AND(SM_70,PTX60))
-TARGET_BUILTIN(__hmma_m16n16k16_st_c_f16, "vi*i*UiIi", "", AND(SM_70,PTX60))
-TARGET_BUILTIN(__hmma_m16n16k16_st_c_f32, "vf*f*UiIi", "", AND(SM_70,PTX60))
+TARGET_BUILTIN(__hmma_m16n16k16_st_c_f16, "vi*iC*UiIi", "", AND(SM_70,PTX60))
+TARGET_BUILTIN(__hmma_m16n16k16_st_c_f32, "vf*fC*UiIi", "", AND(SM_70,PTX60))
 
 TARGET_BUILTIN(__hmma_m32n8k16_ld_a, "vi*iC*UiIi", "", AND(SM_70,PTX61))
 TARGET_BUILTIN(__hmma_m32n8k16_ld_b, "vi*iC*UiIi", "", AND(SM_70,PTX61))
 TARGET_BUILTIN(__hmma_m32n8k16_ld_c_f16, "vi*iC*UiIi", "", AND(SM_70,PTX61))
 TARGET_BUILTIN(__hmma_m32n8k16_ld_c_f32, "vf*fC*UiIi", "", AND(SM_70,PTX61))
-TARGET_BUILTIN(__hmma_m32n8k16_st_c_f16, "vi*i*UiIi", "", AND(SM_70,PTX61))
-TARGET_BUILTIN(__hmma_m32n8k16_st_c_f32, "vf*f*UiIi", "", AND(SM_70,PTX61))
+TARGET_BUILTIN(__hmma_m32n8k16_st_c_f16, "vi*iC*UiIi", "", AND(SM_70,PTX61))
+TARGET_BUILTIN(__hmma_m32n8k16_st_c_f32, "vf*fC*UiIi", "", AND(SM_70,PTX61))
 
 TARGET_BUILTIN(__hmma_m8n32k16_ld_a, "vi*iC*UiIi", "", AND(SM_70,PTX61))
 TARGET_BUILTIN(__hmma_m8n32k16_ld_b, "vi*iC*UiIi", "", AND(SM_70,PTX61))
 TARGET_BUILTIN(__hmma_m8n32k16_ld_c_f16, "vi*iC*UiIi", "", AND(SM_70,PTX61))
 TARGET_BUILTIN(__hmma_m8n32k16_ld_c_f32, "vf*fC*UiIi", "", AND(SM_70,PTX61))
-TARGET_BUILTIN(__hmma_m8n32k16_st_c_f16, "vi*i*UiIi", "", AND(SM_70,PTX61))
-TARGET_BUILTIN(__hmma_m8n32k16_st_c_f32, "vf*f*UiIi", "", AND(SM_70,PTX61))
+TARGET_BUILTIN(__hmma_m8n32k16_st_c_f16, "vi*iC*UiIi", "", AND(SM_70,PTX61))
+TARGET_BUILTIN(__hmma_m8n32k16_st_c_f32, "vf*fC*UiIi", "", AND(SM_70,PTX61))
 
 TARGET_BUILTIN(__hmma_m16n16k16_mma_f16f16, "vi*iC*iC*iC*IiIi", "", AND(SM_70,PTX60))
 TARGET_BUILTIN(__hmma_m16n16k16_mma_f32f16, "vf*iC*iC*iC*IiIi", "", AND(SM_70,PTX60))
diff --git a/clang/include/clang/Basic/BuiltinsX86_64.def b/clang/include/clang/Basic/BuiltinsX86_64.def
index e5c1fe8b31..5e00916d4b 100644
--- a/clang/include/clang/Basic/BuiltinsX86_64.def
+++ b/clang/include/clang/Basic/BuiltinsX86_64.def
@@ -104,6 +104,9 @@ TARGET_BUILTIN(__builtin_ia32_clui, "v", "n", "uintr")
 TARGET_BUILTIN(__builtin_ia32_stui, "v", "n", "uintr")
 TARGET_BUILTIN(__builtin_ia32_testui, "Uc", "n", "uintr")
 TARGET_BUILTIN(__builtin_ia32_senduipi, "vUWi", "n", "uintr")
+// USERMSR
+TARGET_BUILTIN(__builtin_ia32_urdmsr, "ULLiULLi", "n", "usermsr")
+TARGET_BUILTIN(__builtin_ia32_uwrmsr, "vULLiULLi", "n", "usermsr")
 
 // AMX internal builtin
 TARGET_BUILTIN(__builtin_ia32_tile_loadconfig_internal, "vvC*", "n", "amx-tile")
diff --git a/clang/include/clang/Basic/DiagnosticASTKinds.td b/clang/include/clang/Basic/DiagnosticASTKinds.td
index d2656310e7..b70cf1071d 100644
--- a/clang/include/clang/Basic/DiagnosticASTKinds.td
+++ b/clang/include/clang/Basic/DiagnosticASTKinds.td
@@ -394,6 +394,8 @@ def note_constexpr_unsupported_layout : Note<
   "type %0 has unexpected layout">;
 def note_constexpr_unsupported_flexible_array : Note<
   "flexible array initialization is not yet supported">;
+def note_constexpr_non_const_vectorelements : Note<
+  "cannot determine number of elements for sizeless vectors in a constant expression">;
 def err_experimental_clang_interp_failed : Error<
   "the experimental clang interpreter failed to evaluate an expression">;
 
@@ -405,7 +407,8 @@ def warn_fixedpoint_constant_overflow : Warning<
   InGroup<DiagGroup<"fixed-point-overflow">>;
 def warn_constexpr_unscoped_enum_out_of_range : Warning<
   "integer value %0 is outside the valid range of values [%1, %2] for the "
-  "enumeration type %3">, DefaultError, InGroup<DiagGroup<"enum-constexpr-conversion">>;
+  "enumeration type %3">, DefaultError, ShowInSystemHeader, ShowInSystemMacro,
+  InGroup<DiagGroup<"enum-constexpr-conversion">>;
 
 // This is a temporary diagnostic, and shall be removed once our
 // implementation is complete, and like the preceding constexpr notes belongs
diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td
index 9c00fa50bb..6110c6d6ea 100644
--- a/clang/include/clang/Basic/DiagnosticDriverKinds.td
+++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td
@@ -527,6 +527,10 @@ def err_test_module_file_extension_format : Error<
 def err_drv_module_output_with_multiple_arch : Error<
   "option '-fmodule-output' can't be used with multiple arch options">;
 
+def warn_drv_delayed_template_parsing_after_cxx20 : Warning<
+  "-fdelayed-template-parsing is deprecated after C++20">,
+  InGroup<DiagGroup<"delayed-template-parsing-in-cxx20">>;
+
 def err_drv_extract_api_wrong_kind : Error<
   "header file '%0' input '%1' does not match the type of prior input "
   "in api extraction; use '-x %2' to override">;
diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td
index 0b09c00219..dcdae38013 100644
--- a/clang/include/clang/Basic/DiagnosticGroups.td
+++ b/clang/include/clang/Basic/DiagnosticGroups.td
@@ -53,6 +53,7 @@ def SingleBitBitFieldConstantConversion :
 def BitFieldConstantConversion : DiagGroup<"bitfield-constant-conversion",
                                            [SingleBitBitFieldConstantConversion]>;
 def BitFieldEnumConversion : DiagGroup<"bitfield-enum-conversion">;
+def BitFieldConversion : DiagGroup<"bitfield-conversion">;
 def BitFieldWidth : DiagGroup<"bitfield-width">;
 def CompoundTokenSplitByMacro : DiagGroup<"compound-token-split-by-macro">;
 def CompoundTokenSplitBySpace : DiagGroup<"compound-token-split-by-space">;
@@ -847,7 +848,8 @@ def OverridingMethodMismatch : DiagGroup<"overriding-method-mismatch">;
 def VariadicMacros : DiagGroup<"variadic-macros">;
 def VectorConversion : DiagGroup<"vector-conversion">;      // clang specific
 def VexingParse : DiagGroup<"vexing-parse">;
-def VLAExtension : DiagGroup<"vla-extension">;
+def VLAUseStaticAssert : DiagGroup<"vla-extension-static-assert">;
+def VLAExtension : DiagGroup<"vla-extension", [VLAUseStaticAssert]>;
 def VLA : DiagGroup<"vla", [VLAExtension]>;
 def VolatileRegisterVar : DiagGroup<"volatile-register-var">;
 def Visibility : DiagGroup<"visibility">;
@@ -933,6 +935,7 @@ def Conversion : DiagGroup<"conversion",
                             ConstantConversion,
                             EnumConversion,
                             BitFieldEnumConversion,
+                            BitFieldConversion,
                             FloatConversion,
                             Shorten64To32,
                             IntConversion,
@@ -1064,7 +1067,9 @@ def Most : DiagGroup<"most", [
 def ThreadSafetyAttributes : DiagGroup<"thread-safety-attributes">;
 def ThreadSafetyAnalysis   : DiagGroup<"thread-safety-analysis">;
 def ThreadSafetyPrecise    : DiagGroup<"thread-safety-precise">;
-def ThreadSafetyReference  : DiagGroup<"thread-safety-reference">;
+def ThreadSafetyReferenceReturn  : DiagGroup<"thread-safety-reference-return">;
+def ThreadSafetyReference  : DiagGroup<"thread-safety-reference",
+                                             [ThreadSafetyReferenceReturn]>;
 def ThreadSafetyNegative   : DiagGroup<"thread-safety-negative">;
 def ThreadSafety : DiagGroup<"thread-safety",
                              [ThreadSafetyAttributes,
@@ -1081,7 +1086,7 @@ def Consumed       : DiagGroup<"consumed">;
 // warning should be active _only_ when -Wall is passed in, mark it as
 // DefaultIgnore in addition to putting it here.
 def All : DiagGroup<"all", [Most, Parentheses, Switch, SwitchBool,
-                            MisleadingIndentation, PackedNonPod]>;
+                            MisleadingIndentation, PackedNonPod, VLAExtension]>;
 
 // Warnings that should be in clang-cl /w4.
 def : DiagGroup<"CL4", [All, Extra]>;
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index c1a6e38311..a4c1cb08de 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -137,6 +137,18 @@ def err_half_const_requires_fp16 : Error<
 // C99 variable-length arrays
 def ext_vla : Extension<"variable length arrays are a C99 feature">,
   InGroup<VLAExtension>;
+// In C++ language modes, we warn by default as an extension, while in GNU++
+// language modes, we warn as an extension but add the warning group to -Wall.
+def ext_vla_cxx : ExtWarn<
+  "variable length arrays in C++ are a Clang extension">,
+  InGroup<VLAExtension>;
+def ext_vla_cxx_in_gnu_mode : Extension<ext_vla_cxx.Summary>,
+  InGroup<VLAExtension>;
+def ext_vla_cxx_static_assert : ExtWarn<
+  "variable length arrays in C++ are a Clang extension; did you mean to use "
+  "'static_assert'?">, InGroup<VLAUseStaticAssert>;
+def ext_vla_cxx_in_gnu_mode_static_assert : Extension<
+  ext_vla_cxx_static_assert.Summary>, InGroup<VLAUseStaticAssert>;
 def warn_vla_used : Warning<"variable length array used">,
   InGroup<VLA>, DefaultIgnore;
 def err_vla_in_sfinae : Error<
@@ -2613,7 +2625,7 @@ def err_final_function_overridden : Error<
 
 // C++11 scoped enumerations
 def err_enum_invalid_underlying : Error<
-  "non-integral type %0 is an invalid underlying type">;
+  "%select{non-integral type %0|%0}1 is an invalid underlying type">;
 def err_enumerator_too_large : Error<
   "enumerator value is not representable in the underlying type %0">;
 def ext_enumerator_too_large : Extension<
@@ -3864,7 +3876,7 @@ def warn_fun_requires_negative_cap : Warning<
   "calling function %0 requires negative capability '%1'">,
   InGroup<ThreadSafetyAnalysis>, DefaultIgnore;
 
-// Thread safety warnings on pass by reference
+// Thread safety warnings on pass/return by reference
 def warn_guarded_pass_by_reference : Warning<
   "passing variable %1 by reference requires holding %0 "
   "%select{'%2'|'%2' exclusively}3">,
@@ -3873,6 +3885,14 @@ def warn_pt_guarded_pass_by_reference : Warning<
   "passing the value that %1 points to by reference requires holding %0 "
   "%select{'%2'|'%2' exclusively}3">,
   InGroup<ThreadSafetyReference>, DefaultIgnore;
+def warn_guarded_return_by_reference : Warning<
+  "returning variable %1 by reference requires holding %0 "
+  "%select{'%2'|'%2' exclusively}3">,
+  InGroup<ThreadSafetyReferenceReturn>, DefaultIgnore;
+def warn_pt_guarded_return_by_reference : Warning<
+  "returning the value that %1 points to by reference requires holding %0 "
+  "%select{'%2'|'%2' exclusively}3">,
+  InGroup<ThreadSafetyReferenceReturn>, DefaultIgnore;
 
 // Imprecise thread safety warnings
 def warn_variable_requires_lock : Warning<
@@ -6171,6 +6191,9 @@ def warn_signed_bitfield_enum_conversion : Warning<
   "signed bit-field %0 needs an extra bit to represent the largest positive "
   "enumerators of %1">,
   InGroup<BitFieldEnumConversion>, DefaultIgnore;
+def warn_bitfield_too_small_for_integral_type : Warning<
+  "conversion from %2 (%3 bits) to bit-field %0 (%1 bits) may change value">,
+  InGroup<BitFieldConversion>, DefaultIgnore;
 def note_change_bitfield_sign : Note<
   "consider making the bitfield type %select{unsigned|signed}0">;
 
@@ -6389,6 +6412,17 @@ def warn_superclass_variable_sized_type_not_at_end : Warning<
   "field %0 can overwrite instance variable %1 with variable sized type %2"
   " in superclass %3">, InGroup<ObjCFlexibleArray>;
 
+def err_counted_by_attr_not_on_flexible_array_member : Error<
+  "'counted_by' only applies to flexible array members">;
+def err_counted_by_attr_refers_to_flexible_array : Error<
+  "'counted_by' cannot refer to the flexible array %0">;
+def err_counted_by_must_be_in_structure : Error<
+  "field %0 in 'counted_by' not inside structure">;
+def err_flexible_array_counted_by_attr_field_not_integer : Error<
+  "field %0 in 'counted_by' must be a non-boolean integer type">;
+def note_flexible_array_counted_by_attr_field : Note<
+  "field %0 declared here">;
+
 let CategoryName = "ARC Semantic Issue" in {
 
 // ARC-mode diagnostics.
@@ -8679,10 +8713,13 @@ def warn_atomic_implicit_seq_cst : Warning<
   InGroup<DiagGroup<"atomic-implicit-seq-cst">>, DefaultIgnore;
 
 def err_overflow_builtin_must_be_int : Error<
-  "operand argument to overflow builtin must be an integer (%0 invalid)">;
+  "operand argument to %select{overflow builtin|checked integer operation}0 "
+  "must be an integer type %select{|other than plain 'char', 'bool', bit-precise, "
+  "or an enumeration }0(%1 invalid)">;
 def err_overflow_builtin_must_be_ptr_int : Error<
-  "result argument to overflow builtin must be a pointer "
-  "to a non-const integer (%0 invalid)">;
+  "result argument to %select{overflow builtin|checked integer operation}0 "
+  "must be a pointer to a non-const integer type %select{|other than plain 'char', "
+  "'bool', bit-precise, or an enumeration }0(%1 invalid)">;
 def err_overflow_builtin_bit_int_max_size : Error<
   "__builtin_mul_overflow does not support 'signed _BitInt' operands of more "
   "than %0 bits">;
@@ -10144,8 +10181,8 @@ def err_shufflevector_argument_too_large : Error<
 
 def err_convertvector_non_vector : Error<
   "first argument to __builtin_convertvector must be a vector">;
-def err_convertvector_non_vector_type : Error<
-  "second argument to __builtin_convertvector must be a vector type">;
+def err_builtin_non_vector_type : Error<
+  "%0 argument to %1 must be of vector type">;
 def err_convertvector_incompatible_vector : Error<
   "first two arguments to __builtin_convertvector must have the same number of elements">;
 
diff --git a/clang/include/clang/Basic/DiagnosticSerializationKinds.td b/clang/include/clang/Basic/DiagnosticSerializationKinds.td
index a1ae23a628..3cb2cd32cf 100644
--- a/clang/include/clang/Basic/DiagnosticSerializationKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSerializationKinds.td
@@ -62,7 +62,7 @@ def err_ast_file_out_of_date : Error<
   "%select{PCH|module|AST}0 file '%1' is out of date and "
   "needs to be rebuilt%select{|: %3}2">, DefaultFatal;
 def err_ast_file_invalid : Error<
-  "file '%1' is not a valid precompiled %select{PCH|module|AST}0 file">, DefaultFatal;
+  "file '%1' is not a valid precompiled %select{PCH|module|AST}0 file: %2">, DefaultFatal;
 def note_module_file_imported_by : Note<
   "imported by %select{|module '%2' in }1'%0'">;
 def err_module_file_not_module : Error<
diff --git a/clang/include/clang/Basic/FileEntry.h b/clang/include/clang/Basic/FileEntry.h
index 23237a9326..bc65463735 100644
--- a/clang/include/clang/Basic/FileEntry.h
+++ b/clang/include/clang/Basic/FileEntry.h
@@ -235,37 +235,6 @@ static_assert(std::is_trivially_copyable<OptionalFileEntryRef>::value,
 
 namespace llvm {
 
-template <> struct PointerLikeTypeTraits<clang::FileEntryRef> {
-  static inline void *getAsVoidPointer(clang::FileEntryRef File) {
-    return const_cast<clang::FileEntryRef::MapEntry *>(&File.getMapEntry());
-  }
-
-  static inline clang::FileEntryRef getFromVoidPointer(void *Ptr) {
-    return clang::FileEntryRef(
-        *reinterpret_cast<const clang::FileEntryRef::MapEntry *>(Ptr));
-  }
-
-  static constexpr int NumLowBitsAvailable = PointerLikeTypeTraits<
-      const clang::FileEntryRef::MapEntry *>::NumLowBitsAvailable;
-};
-
-template <> struct PointerLikeTypeTraits<clang::OptionalFileEntryRef> {
-  static inline void *getAsVoidPointer(clang::OptionalFileEntryRef File) {
-    if (!File)
-      return nullptr;
-    return PointerLikeTypeTraits<clang::FileEntryRef>::getAsVoidPointer(*File);
-  }
-
-  static inline clang::OptionalFileEntryRef getFromVoidPointer(void *Ptr) {
-    if (!Ptr)
-      return std::nullopt;
-    return PointerLikeTypeTraits<clang::FileEntryRef>::getFromVoidPointer(Ptr);
-  }
-
-  static constexpr int NumLowBitsAvailable =
-      PointerLikeTypeTraits<clang::FileEntryRef>::NumLowBitsAvailable;
-};
-
 /// Specialisation of DenseMapInfo for FileEntryRef.
 template <> struct DenseMapInfo<clang::FileEntryRef> {
   static inline clang::FileEntryRef getEmptyKey() {
diff --git a/clang/include/clang/Basic/LangOptions.h b/clang/include/clang/Basic/LangOptions.h
index e0e95f6d26..20a8ada60e 100644
--- a/clang/include/clang/Basic/LangOptions.h
+++ b/clang/include/clang/Basic/LangOptions.h
@@ -134,7 +134,8 @@ class LangOptions : public LangOptionsBase {
     DCC_FastCall,
     DCC_StdCall,
     DCC_VectorCall,
-    DCC_RegCall
+    DCC_RegCall,
+    DCC_RtdCall
   };
 
   enum AddrSpaceMapMangling { ASMM_Target, ASMM_On, ASMM_Off };
diff --git a/clang/include/clang/Basic/Module.h b/clang/include/clang/Basic/Module.h
index 676fd37249..6a7423938b 100644
--- a/clang/include/clang/Basic/Module.h
+++ b/clang/include/clang/Basic/Module.h
@@ -35,6 +35,7 @@
 #include <optional>
 #include <string>
 #include <utility>
+#include <variant>
 #include <vector>
 
 namespace llvm {
@@ -162,7 +163,7 @@ class alignas(8) Module {
   std::string PresumedModuleMapFile;
 
   /// The umbrella header or directory.
-  llvm::PointerUnion<FileEntryRef, DirectoryEntryRef> Umbrella;
+  std::variant<std::monostate, FileEntryRef, DirectoryEntryRef> Umbrella;
 
   /// The module signature.
   ASTFileSignature Signature;
@@ -665,18 +666,17 @@ class alignas(8) Module {
 
   /// Retrieve the umbrella directory as written.
   std::optional<DirectoryName> getUmbrellaDirAsWritten() const {
-    if (Umbrella && Umbrella.is<DirectoryEntryRef>())
+    if (const auto *Dir = std::get_if<DirectoryEntryRef>(&Umbrella))
       return DirectoryName{UmbrellaAsWritten,
-                           UmbrellaRelativeToRootModuleDirectory,
-                           Umbrella.get<DirectoryEntryRef>()};
+                           UmbrellaRelativeToRootModuleDirectory, *Dir};
     return std::nullopt;
   }
 
   /// Retrieve the umbrella header as written.
   std::optional<Header> getUmbrellaHeaderAsWritten() const {
-    if (Umbrella && Umbrella.is<FileEntryRef>())
+    if (const auto *Hdr = std::get_if<FileEntryRef>(&Umbrella))
       return Header{UmbrellaAsWritten, UmbrellaRelativeToRootModuleDirectory,
-                    Umbrella.get<FileEntryRef>()};
+                    *Hdr};
     return std::nullopt;
   }
 
diff --git a/clang/include/clang/Basic/Specifiers.h b/clang/include/clang/Basic/Specifiers.h
index 6ae56703ec..0add24d53b 100644
--- a/clang/include/clang/Basic/Specifiers.h
+++ b/clang/include/clang/Basic/Specifiers.h
@@ -288,6 +288,7 @@ namespace clang {
     CC_AArch64VectorCall, // __attribute__((aarch64_vector_pcs))
     CC_AArch64SVEPCS, // __attribute__((aarch64_sve_pcs))
     CC_AMDGPUKernelCall, // __attribute__((amdgpu_kernel))
+    CC_M68kRTD,       // __attribute__((m68k_rtd))
   };
 
   /// Checks whether the given calling convention supports variadic
@@ -304,6 +305,7 @@ namespace clang {
     case CC_OpenCLKernel:
     case CC_Swift:
     case CC_SwiftAsync:
+    case CC_M68kRTD:
       return false;
     default:
       return true;
diff --git a/clang/include/clang/Basic/TokenKinds.def b/clang/include/clang/Basic/TokenKinds.def
index 94db56a9fd..bbae1200d3 100644
--- a/clang/include/clang/Basic/TokenKinds.def
+++ b/clang/include/clang/Basic/TokenKinds.def
@@ -746,6 +746,7 @@ ALIAS("_pascal"      , __pascal   , KEYBORLAND)
 
 // Clang Extensions.
 KEYWORD(__builtin_convertvector          , KEYALL)
+UNARY_EXPR_OR_TYPE_TRAIT(__builtin_vectorelements, VectorElements, KEYALL)
 ALIAS("__char16_t"   , char16_t          , KEYCXX)
 ALIAS("__char32_t"   , char32_t          , KEYCXX)
 KEYWORD(__builtin_bit_cast               , KEYALL)
diff --git a/clang/include/clang/Basic/arm_sme.td b/clang/include/clang/Basic/arm_sme.td
index d014900d71..8d85327a86 100644
--- a/clang/include/clang/Basic/arm_sme.td
+++ b/clang/include/clang/Basic/arm_sme.td
@@ -44,10 +44,9 @@ defm SVLD1_ZA32 : ZALoad<"za32", "i", "aarch64_sme_ld1w", [ImmCheck<0, ImmCheck0
 defm SVLD1_ZA64 : ZALoad<"za64", "l", "aarch64_sme_ld1d", [ImmCheck<0, ImmCheck0_7>]>;
 defm SVLD1_ZA128 : ZALoad<"za128", "q", "aarch64_sme_ld1q", [ImmCheck<0, ImmCheck0_15>]>;
 
-def SVLDR_VNUM_ZA : MInst<"svldr_vnum_za", "vmQi", "",
+def SVLDR_VNUM_ZA : MInst<"svldr_vnum_za", "vmQl", "",
                           [IsOverloadNone, IsStreamingCompatible, IsSharedZA],
-                          MemEltTyDefault, "aarch64_sme_ldr",
-                          [ImmCheck<2, ImmCheck0_15>]>;
+                          MemEltTyDefault, "aarch64_sme_ldr">;
 
 def SVLDR_ZA : MInst<"svldr_za", "vmQ", "",
                           [IsOverloadNone, IsStreamingCompatible, IsSharedZA],
@@ -82,10 +81,9 @@ defm SVST1_ZA32 : ZAStore<"za32", "i", "aarch64_sme_st1w", [ImmCheck<0, ImmCheck
 defm SVST1_ZA64 : ZAStore<"za64", "l", "aarch64_sme_st1d", [ImmCheck<0, ImmCheck0_7>]>;
 defm SVST1_ZA128 : ZAStore<"za128", "q", "aarch64_sme_st1q", [ImmCheck<0, ImmCheck0_15>]>;
 
-def SVSTR_VNUM_ZA : MInst<"svstr_vnum_za", "vm%i", "",
+def SVSTR_VNUM_ZA : MInst<"svstr_vnum_za", "vm%l", "",
                           [IsOverloadNone, IsStreamingCompatible, IsSharedZA, IsPreservesZA],
-                          MemEltTyDefault, "aarch64_sme_str",
-                          [ImmCheck<2, ImmCheck0_15>]>;
+                          MemEltTyDefault, "aarch64_sme_str">;
 
 def SVSTR_ZA : MInst<"svstr_za", "vm%", "",
                       [IsOverloadNone, IsStreamingCompatible, IsSharedZA, IsPreservesZA],
diff --git a/clang/include/clang/Basic/arm_sve.td b/clang/include/clang/Basic/arm_sve.td
index 894a0a1296..b5baafedd1 100644
--- a/clang/include/clang/Basic/arm_sve.td
+++ b/clang/include/clang/Basic/arm_sve.td
@@ -1862,9 +1862,121 @@ def SVBGRP_N : SInst<"svbgrp[_n_{d}]", "dda", "UcUsUiUl", MergeNone, "aarch64_sv
 let TargetGuard = "sve2p1" in {
 def SVFCLAMP   : SInst<"svclamp[_{d}]", "dddd", "hfd", MergeNone, "aarch64_sve_fclamp", [], []>;
 def SVPTRUE_COUNT  : SInst<"svptrue_{d}", "}v", "QcQsQiQl", MergeNone, "aarch64_sve_ptrue_{d}", [IsOverloadNone], []>;
+def SVPFALSE_COUNT_ALIAS : SInst<"svpfalse_c", "}v", "", MergeNone, "", [IsOverloadNone]>;
+
+def SVPEXT_SINGLE : SInst<"svpext_lane_{d}", "P}i", "QcQsQiQl", MergeNone, "aarch64_sve_pext", [], [ImmCheck<1, ImmCheck0_3>]>;
+def SVPEXT_X2     : SInst<"svpext_lane_{d}_x2", "2.P}i", "QcQsQiQl", MergeNone, "aarch64_sve_pext_x2", [], [ImmCheck<1, ImmCheck0_1>]>;
+
+def SVPSEL_COUNT_ALIAS_B : SInst<"svpsel_lane_c8",  "}}Pm", "Pc", MergeNone, "", [], []>;
+def SVPSEL_COUNT_ALIAS_H : SInst<"svpsel_lane_c16", "}}Pm", "Ps", MergeNone, "", [], []>;
+def SVPSEL_COUNT_ALIAS_S : SInst<"svpsel_lane_c32", "}}Pm", "Pi", MergeNone, "", [], []>;
+def SVPSEL_COUNT_ALIAS_D : SInst<"svpsel_lane_c64", "}}Pm", "Pl", MergeNone, "", [], []>;
+
+def SVWHILEGE_COUNT  : SInst<"svwhilege_{d}",  "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilege_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>;
+def SVWHILEGT_COUNT  : SInst<"svwhilegt_{d}",  "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilegt_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>;
+def SVWHILELE_COUNT  : SInst<"svwhilele_{d}",  "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilele_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>;
+def SVWHILELT_COUNT  : SInst<"svwhilelt_{d}",  "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilelt_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>;
+def SVWHILELO_COUNT  : SInst<"svwhilelo_{d}",  "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilelo_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>;
+def SVWHILELS_COUNT  : SInst<"svwhilels_{d}",  "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilels_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>;
+def SVWHILEHI_COUNT  : SInst<"svwhilehi_{d}",  "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilehi_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>;
+def SVWHILEHS_COUNT  : SInst<"svwhilehs_{d}",  "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilehs_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>;
+
+def SVLD1B_X2 : MInst<"svld1[_{2}]_x2", "2}c", "cUc", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x2">;
+def SVLD1H_X2 : MInst<"svld1[_{2}]_x2", "2}c", "sUshb", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x2">;
+def SVLD1W_X2 : MInst<"svld1[_{2}]_x2", "2}c", "iUif", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x2">;
+def SVLD1D_X2 : MInst<"svld1[_{2}]_x2", "2}c", "lUld", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x2">;
+def SVLD1B_X4 : MInst<"svld1[_{2}]_x4", "4}c", "cUc", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x4">;
+def SVLD1H_X4 : MInst<"svld1[_{2}]_x4", "4}c", "sUshb", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x4">;
+def SVLD1W_X4 : MInst<"svld1[_{2}]_x4", "4}c", "iUif", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x4">;
+def SVLD1D_X4 : MInst<"svld1[_{2}]_x4", "4}c", "lUld", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x4">;
+
+def SVLDNT1B_X2 : MInst<"svldnt1[_{2}]_x2", "2}c", "cUc", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x2">;
+def SVLDNT1H_X2 : MInst<"svldnt1[_{2}]_x2", "2}c", "sUshb", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x2">;
+def SVLDNT1W_X2 : MInst<"svldnt1[_{2}]_x2", "2}c", "iUif", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x2">;
+def SVLDNT1D_X2 : MInst<"svldnt1[_{2}]_x2", "2}c", "lUld", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x2">;
+def SVLDNT1B_X4 : MInst<"svldnt1[_{2}]_x4", "4}c", "cUc", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x4">;
+def SVLDNT1H_X4 : MInst<"svldnt1[_{2}]_x4", "4}c", "sUshb", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x4">;
+def SVLDNT1W_X4 : MInst<"svldnt1[_{2}]_x4", "4}c", "iUif", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x4">;
+def SVLDNT1D_X4 : MInst<"svldnt1[_{2}]_x4", "4}c", "lUld", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x4">;
+
+def SVLD1B_VNUM_X2 : MInst<"svld1_vnum[_{2}]_x2", "2}cl", "cUc", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x2">;
+def SVLD1H_VNUM_X2 : MInst<"svld1_vnum[_{2}]_x2", "2}cl", "sUshb", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x2">;
+def SVLD1W_VNUM_X2 : MInst<"svld1_vnum[_{2}]_x2", "2}cl", "iUif", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x2">;
+def SVLD1D_VNUM_X2 : MInst<"svld1_vnum[_{2}]_x2", "2}cl", "lUld", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x2">;
+def SVLD1B_VNUM_X4 : MInst<"svld1_vnum[_{2}]_x4", "4}cl", "cUc", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x4">;
+def SVLD1H_VNUM_X4 : MInst<"svld1_vnum[_{2}]_x4", "4}cl", "sUshb", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x4">;
+def SVLD1W_VNUM_X4 : MInst<"svld1_vnum[_{2}]_x4", "4}cl", "iUif", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x4">;
+def SVLD1D_VNUM_X4 : MInst<"svld1_vnum[_{2}]_x4", "4}cl", "lUld", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ld1_pn_x4">;
+
+def SVLDNT1B_VNUM_X2 : MInst<"svldnt1_vnum[_{2}]_x2", "2}cl", "cUc", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x2">;
+def SVLDNT1H_VNUM_X2 : MInst<"svldnt1_vnum[_{2}]_x2", "2}cl", "sUshb", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x2">;
+def SVLDNT1W_VNUM_X2 : MInst<"svldnt1_vnum[_{2}]_x2", "2}cl", "iUif", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x2">;
+def SVLDNT1D_VNUM_X2 : MInst<"svldnt1_vnum[_{2}]_x2", "2}cl", "lUld", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x2">;
+def SVLDNT1B_VNUM_X4 : MInst<"svldnt1_vnum[_{2}]_x4", "4}cl", "cUc", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x4">;
+def SVLDNT1H_VNUM_X4 : MInst<"svldnt1_vnum[_{2}]_x4", "4}cl", "sUshb", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x4">;
+def SVLDNT1W_VNUM_X4 : MInst<"svldnt1_vnum[_{2}]_x4", "4}cl", "iUif", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x4">;
+def SVLDNT1D_VNUM_X4 : MInst<"svldnt1_vnum[_{2}]_x4", "4}cl", "lUld", [IsStructLoad], MemEltTyDefault, "aarch64_sve_ldnt1_pn_x4">;
+
+def SVST1B_X2 : MInst<"svst1[_{2}_x2]", "v}p2", "cUc", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x2">;
+def SVST1H_X2 : MInst<"svst1[_{2}_x2]", "v}p2", "sUshb", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x2">;
+def SVST1W_X2 : MInst<"svst1[_{2}_x2]", "v}p2", "iUif", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x2">;
+def SVST1D_X2 : MInst<"svst1[_{2}_x2]", "v}p2", "lUld", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x2">;
+def SVST1B_X4 : MInst<"svst1[_{2}_x4]", "v}p4", "cUc", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x4">;
+def SVST1H_X4 : MInst<"svst1[_{2}_x4]", "v}p4", "sUshb", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x4">;
+def SVST1W_X4 : MInst<"svst1[_{2}_x4]", "v}p4", "iUif", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x4">;
+def SVST1D_X4 : MInst<"svst1[_{2}_x4]", "v}p4", "lUld", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x4">;
+
+def SVST1B_VNUM_X2 : MInst<"svst1_vnum[_{2}_x2]", "v}pl2", "cUc", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x2">;
+def SVST1H_VNUM_X2 : MInst<"svst1_vnum[_{2}_x2]", "v}pl2", "sUshb", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x2">;
+def SVST1W_VNUM_X2 : MInst<"svst1_vnum[_{2}_x2]", "v}pl2", "iUif", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x2">;
+def SVST1D_VNUM_X2 : MInst<"svst1_vnum[_{2}_x2]", "v}pl2", "lUld", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x2">;
+def SVST1B_VNUM_X4 : MInst<"svst1_vnum[_{2}_x4]", "v}pl4", "cUc", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x4">;
+def SVST1H_VNUM_X4 : MInst<"svst1_vnum[_{2}_x4]", "v}pl4", "sUshb", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x4">;
+def SVST1W_VNUM_X4 : MInst<"svst1_vnum[_{2}_x4]", "v}pl4", "iUif", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x4">;
+def SVST1D_VNUM_X4 : MInst<"svst1_vnum[_{2}_x4]", "v}pl4", "lUld", [IsStructStore], MemEltTyDefault, "aarch64_sve_st1_pn_x4">;
+
+def SVSTNT1B_X2 : MInst<"svstnt1[_{2}_x2]", "v}p2", "cUc", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x2">;
+def SVSTNT1H_X2 : MInst<"svstnt1[_{2}_x2]", "v}p2", "sUshb", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x2">;
+def SVSTNT1W_X2 : MInst<"svstnt1[_{2}_x2]", "v}p2", "iUif", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x2">;
+def SVSTNT1D_X2 : MInst<"svstnt1[_{2}_x2]", "v}p2", "lUld", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x2">;
+def SVSTNT1B_X4 : MInst<"svstnt1[_{2}_x4]", "v}p4", "cUc", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x4">;
+def SVSTNT1H_X4 : MInst<"svstnt1[_{2}_x4]", "v}p4", "sUshb", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x4">;
+def SVSTNT1W_X4 : MInst<"svstnt1[_{2}_x4]", "v}p4", "iUif", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x4">;
+def SVSTNT1D_X4 : MInst<"svstnt1[_{2}_x4]", "v}p4", "lUld", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x4">;
+
+def SVSTNT1B_VNUM_X2 : MInst<"svstnt1_vnum[_{2}_x2]", "v}pl2", "cUc", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x2">;
+def SVSTNT1H_VNUM_X2 : MInst<"svstnt1_vnum[_{2}_x2]", "v}pl2", "sUshb", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x2">;
+def SVSTNT1W_VNUM_X2 : MInst<"svstnt1_vnum[_{2}_x2]", "v}pl2", "iUif", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x2">;
+def SVSTNT1D_VNUM_X2 : MInst<"svstnt1_vnum[_{2}_x2]", "v}pl2", "lUld", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x2">;
+def SVSTNT1B_VNUM_X4 : MInst<"svstnt1_vnum[_{2}_x4]", "v}pl4", "cUc", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x4">;
+def SVSTNT1H_VNUM_X4 : MInst<"svstnt1_vnum[_{2}_x4]", "v}pl4", "sUshb", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x4">;
+def SVSTNT1W_VNUM_X4 : MInst<"svstnt1_vnum[_{2}_x4]", "v}pl4", "iUif", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x4">;
+def SVSTNT1D_VNUM_X4 : MInst<"svstnt1_vnum[_{2}_x4]", "v}pl4", "lUld", [IsStructStore], MemEltTyDefault, "aarch64_sve_stnt1_pn_x4">;
+
+def SVDOT_X2_S : SInst<"svdot[_{d}_{2}_{3}]", "ddhh", "i",  MergeNone, "aarch64_sve_sdot_x2", [], []>;
+def SVDOT_X2_U : SInst<"svdot[_{d}_{2}_{3}]", "ddhh", "Ui", MergeNone, "aarch64_sve_udot_x2", [], []>;
+def SVDOT_X2_F : SInst<"svdot[_{d}_{2}_{3}]", "ddhh", "f",  MergeNone, "aarch64_sve_fdot_x2", [], []>;
+def SVDOT_LANE_X2_S : SInst<"svdot_lane[_{d}_{2}_{3}]", "ddhhi", "i",  MergeNone, "aarch64_sve_sdot_lane_x2", [], [ImmCheck<3, ImmCheck0_3>]>;
+def SVDOT_LANE_X2_U : SInst<"svdot_lane[_{d}_{2}_{3}]", "ddhhi", "Ui", MergeNone, "aarch64_sve_udot_lane_x2", [], [ImmCheck<3, ImmCheck0_3>]>;
+def SVDOT_LANE_X2_F : SInst<"svdot_lane[_{d}_{2}_{3}]", "ddhhi", "f",  MergeNone, "aarch64_sve_fdot_lane_x2", [], [ImmCheck<3, ImmCheck0_3>]>;
+
+def SVBFMLSLB : SInst<"svbfmlslb[_{d}]", "dd$$", "f", MergeNone, "aarch64_sve_bfmlslb", [IsOverloadNone], []>;
+def SVBFMLSLT : SInst<"svbfmlslt[_{d}]", "dd$$", "f", MergeNone, "aarch64_sve_bfmlslt", [IsOverloadNone], []>;
+
+def SVBFMLSLB_LANE : SInst<"svbfmlslb_lane[_{d}]", "dd$$i", "f", MergeNone, "aarch64_sve_bfmlslb_lane", [IsOverloadNone], [ImmCheck<3, ImmCheck0_7>]>;
+def SVBFMLSLT_LANE : SInst<"svbfmlslt_lane[_{d}]", "dd$$i", "f", MergeNone, "aarch64_sve_bfmlslt_lane", [IsOverloadNone], [ImmCheck<3, ImmCheck0_7>]>;
 }
 
 let TargetGuard = "sve2p1" in {
 def SVSCLAMP : SInst<"svclamp[_{d}]", "dddd", "csil",     MergeNone, "aarch64_sve_sclamp", [], []>;
 def SVUCLAMP : SInst<"svclamp[_{d}]", "dddd", "UcUsUiUl", MergeNone, "aarch64_sve_uclamp", [], []>;
+
+def SVPSEL_B : SInst<"svpsel_lane_b8",  "PPPm", "Pc", MergeNone, "", [], []>;
+def SVPSEL_H : SInst<"svpsel_lane_b16", "PPPm", "Ps", MergeNone, "", [], []>;
+def SVPSEL_S : SInst<"svpsel_lane_b32", "PPPm", "Pi", MergeNone, "", [], []>;
+def SVPSEL_D : SInst<"svpsel_lane_b64", "PPPm", "Pl", MergeNone, "", [], []>;
+
+def SVCNTP_COUNT : SInst<"svcntp_{d}", "n}i", "QcQsQiQl", MergeNone, "aarch64_sve_cntp_{d}", [IsOverloadNone], [ImmCheck<1, ImmCheck2_4_Mul2>]>;
+
+defm SVREVD : SInstZPZ<"svrevd", "csilUcUsUiUl", "aarch64_sve_revd">;
 }
diff --git a/clang/include/clang/Basic/arm_sve_sme_incl.td b/clang/include/clang/Basic/arm_sve_sme_incl.td
index 74c9b92667..3a7a5b51b2 100644
--- a/clang/include/clang/Basic/arm_sve_sme_incl.td
+++ b/clang/include/clang/Basic/arm_sve_sme_incl.td
@@ -61,7 +61,8 @@
 // -------------------
 // prototype: return (arg, arg, ...)
 //
-// 2,3,4: array of default vectors
+// 2,3,4: array of vectors
+// .: indicator for multi-vector modifier that will follow (e.g. 2.x)
 // v: void
 // x: vector of signed integers
 // u: vector of unsigned integers
@@ -98,6 +99,7 @@
 // O: svfloat16_t
 // M: svfloat32_t
 // N: svfloat64_t
+// $: svbfloat16_t
 
 // J: Prefetch type (sv_prfop)
 
@@ -246,6 +248,7 @@ def ImmCheck0_3                 : ImmCheckType<15>; // 0..3
 def ImmCheck0_0                 : ImmCheckType<16>; // 0..0
 def ImmCheck0_15                : ImmCheckType<17>; // 0..15
 def ImmCheck0_255               : ImmCheckType<18>; // 0..255
+def ImmCheck2_4_Mul2            : ImmCheckType<19>; // 2, 4
 
 class ImmCheck<int arg, ImmCheckType kind, int eltSizeArg = -1> {
   int Arg = arg;
diff --git a/clang/include/clang/Basic/riscv_sifive_vector.td b/clang/include/clang/Basic/riscv_sifive_vector.td
index 6583a7eb7b..fb8561a05a 100644
--- a/clang/include/clang/Basic/riscv_sifive_vector.td
+++ b/clang/include/clang/Basic/riscv_sifive_vector.td
@@ -103,3 +103,27 @@ let SupportOverloading = false in {
     defm sf_vc_v_fvw : RVVVCIXBuiltinSet<["si"],  "UwKzUwUvFe", [-1, 0, 2, 3], UseGPR=0>;
   }
 }
+
+multiclass RVVVQMACCBuiltinSet<list<list<string>> suffixes_prototypes> {
+  let OverloadedName = NAME,
+      Name = NAME,
+      HasMasked = false,
+      Log2LMUL = [0, 1, 2, 3] in
+    defm NAME : RVVOutOp1Op2BuiltinSet<NAME, "i", suffixes_prototypes>;
+}
+
+let UnMaskedPolicyScheme = HasPolicyOperand in
+  let RequiredFeatures = ["Xsfvqmaccdod"] in {
+    defm sf_vqmaccu_2x8x2 : RVVVQMACCBuiltinSet<[["", "v", "vv(FixedSEW:8)SUv(FixedSEW:8)Uv"]]>;
+    defm sf_vqmacc_2x8x2 : RVVVQMACCBuiltinSet<[["", "v", "vv(FixedSEW:8)Sv(FixedSEW:8)v"]]>;
+    defm sf_vqmaccus_2x8x2 : RVVVQMACCBuiltinSet<[["", "v", "vv(FixedSEW:8)SUv(FixedSEW:8)v"]]>;
+    defm sf_vqmaccsu_2x8x2 : RVVVQMACCBuiltinSet<[["", "v", "vv(FixedSEW:8)Sv(FixedSEW:8)Uv"]]>;
+  }
+
+let UnMaskedPolicyScheme = HasPolicyOperand in
+  let RequiredFeatures = ["Xsfvqmaccqoq"] in {
+    defm sf_vqmaccu_4x8x4 : RVVVQMACCBuiltinSet<[["", "v", "vv(FixedSEW:8)SUv(FixedSEW:8)Uv"]]>;
+    defm sf_vqmacc_4x8x4 : RVVVQMACCBuiltinSet<[["", "v", "vv(FixedSEW:8)Sv(FixedSEW:8)v"]]>;
+    defm sf_vqmaccus_4x8x4 : RVVVQMACCBuiltinSet<[["", "v", "vv(FixedSEW:8)SUv(FixedSEW:8)v"]]>;
+    defm sf_vqmaccsu_4x8x4 : RVVVQMACCBuiltinSet<[["", "v", "vv(FixedSEW:8)Sv(FixedSEW:8)Uv"]]>;
+  }
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index c272a7f1c3..e63158fb0e 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -148,7 +148,8 @@ def pedantic_Group : OptionGroup<"<pedantic group>">, Group<f_Group>,
                      DocFlatten;
 
 def offload_Group : OptionGroup<"<offload group>">, Group<f_Group>,
-                   DocName<"Common Offloading options">;
+                   DocName<"Common Offloading options">,
+                   Visibility<[ClangOption, CLOption]>;
 
 def opencl_Group : OptionGroup<"<opencl group>">, Group<f_Group>,
                    DocName<"OpenCL options">;
@@ -157,13 +158,16 @@ def sycl_Group : OptionGroup<"<SYCL group>">, Group<f_Group>,
                  DocName<"SYCL options">;
 
 def cuda_Group : OptionGroup<"<CUDA group>">, Group<f_Group>,
-                   DocName<"CUDA options">;
+                   DocName<"CUDA options">,
+                   Visibility<[ClangOption, CLOption]>;
 
 def hip_Group : OptionGroup<"<HIP group>">, Group<f_Group>,
-                   DocName<"HIP options">;
+                   DocName<"HIP options">,
+                   Visibility<[ClangOption, CLOption]>;
 
 def m_Group : OptionGroup<"<m group>">, Group<CompileOnly_Group>,
-              DocName<"Target-dependent compilation options">;
+              DocName<"Target-dependent compilation options">,
+              Visibility<[ClangOption, CLOption]>;
 
 // Feature groups - these take command line options that correspond directly to
 // target specific features and can be translated directly from command line
@@ -197,6 +201,9 @@ def m_wasm_Features_Driver_Group : OptionGroup<"<wasm driver features group>">,
 def m_x86_Features_Group : OptionGroup<"<x86 features group>">,
                            Group<m_Group>, Visibility<[ClangOption, CLOption]>,
                            DocName<"X86">;
+def m_x86_AVX10_Features_Group : OptionGroup<"<x86 AVX10 features group>">,
+                                 Group<m_Group>, Visibility<[ClangOption, CLOption]>,
+                                 DocName<"X86 AVX10">;
 def m_riscv_Features_Group : OptionGroup<"<riscv features group>">,
                              Group<m_Group>, DocName<"RISC-V">;
 def m_ve_Features_Group : OptionGroup<"<ve features group>">,
@@ -5164,14 +5171,16 @@ def prebind__all__twolevel__modules : Flag<["-"], "prebind_all_twolevel_modules"
 def prebind : Flag<["-"], "prebind">;
 def preload : Flag<["-"], "preload">;
 def print_file_name_EQ : Joined<["-", "--"], "print-file-name=">,
-  HelpText<"Print the full library path of <file>">, MetaVarName<"<file>">;
+  HelpText<"Print the full library path of <file>">, MetaVarName<"<file>">,
+  Visibility<[ClangOption, CLOption]>;
 def print_ivar_layout : Flag<["-"], "print-ivar-layout">,
   Visibility<[ClangOption, CC1Option]>,
   HelpText<"Enable Objective-C Ivar layout bitmap print trace">,
   MarshallingInfoFlag<LangOpts<"ObjCGCBitmapPrint">>;
 def print_libgcc_file_name : Flag<["-", "--"], "print-libgcc-file-name">,
   HelpText<"Print the library path for the currently used compiler runtime "
-           "library (\"libgcc.a\" or \"libclang_rt.builtins.*.a\")">;
+           "library (\"libgcc.a\" or \"libclang_rt.builtins.*.a\")">,
+  Visibility<[ClangOption, CLOption]>;
 def print_multi_directory : Flag<["-", "--"], "print-multi-directory">;
 def print_multi_lib : Flag<["-", "--"], "print-multi-lib">;
 def print_multi_flags : Flag<["-", "--"], "print-multi-flags-experimental">,
@@ -5180,27 +5189,34 @@ def print_multi_os_directory : Flag<["-", "--"], "print-multi-os-directory">,
   Flags<[Unsupported]>;
 def print_target_triple : Flag<["-", "--"], "print-target-triple">,
   HelpText<"Print the normalized target triple">,
-  Visibility<[ClangOption, FlangOption]>;
+  Visibility<[ClangOption, FlangOption, CLOption]>;
 def print_effective_triple : Flag<["-", "--"], "print-effective-triple">,
   HelpText<"Print the effective target triple">,
-  Visibility<[ClangOption, FlangOption]>;
+  Visibility<[ClangOption, FlangOption, CLOption]>;
 // GCC --disable-multiarch, GCC --enable-multiarch (upstream and Debian
 // specific) have different behaviors. We choose not to support the option.
 def : Flag<["-", "--"], "print-multiarch">, Flags<[Unsupported]>;
 def print_prog_name_EQ : Joined<["-", "--"], "print-prog-name=">,
-  HelpText<"Print the full program path of <name>">, MetaVarName<"<name>">;
+  HelpText<"Print the full program path of <name>">, MetaVarName<"<name>">,
+  Visibility<[ClangOption, CLOption]>;
 def print_resource_dir : Flag<["-", "--"], "print-resource-dir">,
-  HelpText<"Print the resource directory pathname">;
+  HelpText<"Print the resource directory pathname">,
+  Visibility<[ClangOption, CLOption]>;
 def print_search_dirs : Flag<["-", "--"], "print-search-dirs">,
-  HelpText<"Print the paths used for finding libraries and programs">;
+  HelpText<"Print the paths used for finding libraries and programs">,
+  Visibility<[ClangOption, CLOption]>;
 def print_targets : Flag<["-", "--"], "print-targets">,
-  HelpText<"Print the registered targets">;
+  HelpText<"Print the registered targets">,
+  Visibility<[ClangOption, CLOption]>;
 def print_rocm_search_dirs : Flag<["-", "--"], "print-rocm-search-dirs">,
-  HelpText<"Print the paths used for finding ROCm installation">;
+  HelpText<"Print the paths used for finding ROCm installation">,
+  Visibility<[ClangOption, CLOption]>;
 def print_runtime_dir : Flag<["-", "--"], "print-runtime-dir">,
-  HelpText<"Print the directory pathname containing clangs runtime libraries">;
+  HelpText<"Print the directory pathname containing clangs runtime libraries">,
+  Visibility<[ClangOption, CLOption]>;
 def print_diagnostic_options : Flag<["-", "--"], "print-diagnostic-options">,
-  HelpText<"Print all of Clang's warning options">;
+  HelpText<"Print all of Clang's warning options">,
+  Visibility<[ClangOption, CLOption]>;
 def private__bundle : Flag<["-"], "private_bundle">;
 def pthreads : Flag<["-"], "pthreads">;
 defm pthread : BoolOption<"", "pthread",
@@ -5227,7 +5243,7 @@ def resource_dir_EQ : Joined<["-"], "resource-dir=">, Flags<[NoXarchOption]>,
   Visibility<[ClangOption, CLOption, DXCOption]>,
   Alias<resource_dir>;
 def rpath : Separate<["-"], "rpath">, Flags<[LinkerInput]>, Group<Link_Group>;
-def rtlib_EQ : Joined<["-", "--"], "rtlib=">,
+def rtlib_EQ : Joined<["-", "--"], "rtlib=">, Visibility<[ClangOption, CLOption]>,
   HelpText<"Compiler runtime library to use">;
 def frtlib_add_rpath: Flag<["-"], "frtlib-add-rpath">, Flags<[NoArgumentUnused]>,
   HelpText<"Add -rpath with architecture-specific resource directory to the linker flags. "
@@ -5393,7 +5409,7 @@ def w : Flag<["-"], "w">, HelpText<"Suppress all warnings">,
   MarshallingInfoFlag<DiagnosticOpts<"IgnoreWarnings">>;
 def x : JoinedOrSeparate<["-"], "x">,
 Flags<[NoXarchOption]>,
-  Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
+  Visibility<[ClangOption, CC1Option, FlangOption, FC1Option, CLOption]>,
   HelpText<"Treat subsequent input files as having type <language>">,
   MetaVarName<"<language>">;
 def y : Joined<["-"], "y">;
@@ -5754,6 +5770,12 @@ def msse4a : Flag<["-"], "msse4a">, Group<m_x86_Features_Group>;
 def mno_sse4a : Flag<["-"], "mno-sse4a">, Group<m_x86_Features_Group>;
 def mavx : Flag<["-"], "mavx">, Group<m_x86_Features_Group>;
 def mno_avx : Flag<["-"], "mno-avx">, Group<m_x86_Features_Group>;
+def mavx10_1_256 : Flag<["-"], "mavx10.1-256">, Group<m_x86_AVX10_Features_Group>;
+def mno_avx10_1_256 : Flag<["-"], "mno-avx10.1-256">, Group<m_x86_AVX10_Features_Group>;
+def mavx10_1_512 : Flag<["-"], "mavx10.1-512">, Group<m_x86_AVX10_Features_Group>;
+def mno_avx10_1_512 : Flag<["-"], "mno-avx10.1-512">, Group<m_x86_AVX10_Features_Group>;
+def mavx10_1 : Flag<["-"], "mavx10.1">, Alias<mavx10_1_256>;
+def mno_avx10_1 : Flag<["-"], "mno-avx10.1">, Alias<mno_avx10_1_256>;
 def mavx2 : Flag<["-"], "mavx2">, Group<m_x86_Features_Group>;
 def mno_avx2 : Flag<["-"], "mno-avx2">, Group<m_x86_Features_Group>;
 def mavx512f : Flag<["-"], "mavx512f">, Group<m_x86_Features_Group>;
@@ -5904,6 +5926,8 @@ def mtsxldtrk : Flag<["-"], "mtsxldtrk">, Group<m_x86_Features_Group>;
 def mno_tsxldtrk : Flag<["-"], "mno-tsxldtrk">, Group<m_x86_Features_Group>;
 def muintr : Flag<["-"], "muintr">, Group<m_x86_Features_Group>;
 def mno_uintr : Flag<["-"], "mno-uintr">, Group<m_x86_Features_Group>;
+def musermsr : Flag<["-"], "musermsr">, Group<m_x86_Features_Group>;
+def mno_usermsr : Flag<["-"], "mno-usermsr">, Group<m_x86_Features_Group>;
 def mvaes : Flag<["-"], "mvaes">, Group<m_x86_Features_Group>;
 def mno_vaes : Flag<["-"], "mno-vaes">, Group<m_x86_Features_Group>;
 def mvpclmulqdq : Flag<["-"], "mvpclmulqdq">, Group<m_x86_Features_Group>;
@@ -6264,6 +6288,9 @@ defm stack_arrays : BoolOptionWithoutMarshalling<"f", "stack-arrays",
 defm loop_versioning : BoolOptionWithoutMarshalling<"f", "version-loops-for-stride",
   PosFlag<SetTrue, [], [ClangOption], "Create unit-strided versions of loops">,
    NegFlag<SetFalse, [], [ClangOption], "Do not create unit-strided loops (default)">>;
+defm alias_analysis : BoolOptionWithoutMarshalling<"f", "alias-analysis",
+  PosFlag<SetTrue, [], [], "Pass alias information on to LLVM (default when optimizing for speed)">,
+  NegFlag<SetFalse, [], [], "Do not pass alias information on to LLVM (default for unoptimized builds)">>;
 } // let Visibility = [FC1Option, FlangOption]
 
 def J : JoinedOrSeparate<["-"], "J">,
@@ -7475,9 +7502,9 @@ def fnative_half_arguments_and_returns : Flag<["-"], "fnative-half-arguments-and
   ImpliedByAnyOf<[open_cl.KeyPath, render_script.KeyPath, hlsl.KeyPath]>;
 def fdefault_calling_conv_EQ : Joined<["-"], "fdefault-calling-conv=">,
   HelpText<"Set default calling convention">,
-  Values<"cdecl,fastcall,stdcall,vectorcall,regcall">,
+  Values<"cdecl,fastcall,stdcall,vectorcall,regcall,rtdcall">,
   NormalizedValuesScope<"LangOptions">,
-  NormalizedValues<["DCC_CDecl", "DCC_FastCall", "DCC_StdCall", "DCC_VectorCall", "DCC_RegCall"]>,
+  NormalizedValues<["DCC_CDecl", "DCC_FastCall", "DCC_StdCall", "DCC_VectorCall", "DCC_RegCall", "DCC_RtdCall"]>,
   MarshallingInfoEnum<LangOpts<"DefaultCallingConv">, "DCC_None">;
 
 // These options cannot be marshalled, because they are used to set up the LangOptions defaults.
diff --git a/clang/include/clang/Format/Format.h b/clang/include/clang/Format/Format.h
index 2707ea608c..4c344135d2 100644
--- a/clang/include/clang/Format/Format.h
+++ b/clang/include/clang/Format/Format.h
@@ -4128,6 +4128,28 @@ struct FormatStyle {
     ///    object.operator++ (10);                object.operator++(10);
     /// \endcode
     bool AfterOverloadedOperator;
+    /// Styles for adding spacing between ``new/delete`` operators and opening
+    /// parentheses.
+    enum AfterPlacementOperatorStyle : int8_t {
+      /// Remove space after ``new/delete`` operators and before ``(``.
+      /// \code
+      ///    new(buf) T;
+      ///    delete(buf) T;
+      /// \endcode
+      APO_Never,
+      /// Always add space after ``new/delete`` operators and before ``(``.
+      /// \code
+      ///    new (buf) T;
+      ///    delete (buf) T;
+      /// \endcode
+      APO_Always,
+      /// Leave placement ``new/delete`` expressions as they are.
+      APO_Leave,
+    };
+    /// Defines in which cases to put a space between ``new/delete`` operators
+    /// and opening parentheses.
+    /// \version 18
+    AfterPlacementOperatorStyle AfterPlacementOperator;
     /// If ``true``, put space between requires keyword in a requires clause and
     /// opening parentheses, if there is one.
     /// \code
@@ -4160,8 +4182,9 @@ struct FormatStyle {
         : AfterControlStatements(false), AfterForeachMacros(false),
           AfterFunctionDeclarationName(false),
           AfterFunctionDefinitionName(false), AfterIfMacros(false),
-          AfterOverloadedOperator(false), AfterRequiresInClause(false),
-          AfterRequiresInExpression(false), BeforeNonEmptyParentheses(false) {}
+          AfterOverloadedOperator(false), AfterPlacementOperator(APO_Leave),
+          AfterRequiresInClause(false), AfterRequiresInExpression(false),
+          BeforeNonEmptyParentheses(false) {}
 
     bool operator==(const SpaceBeforeParensCustom &Other) const {
       return AfterControlStatements == Other.AfterControlStatements &&
@@ -4171,6 +4194,7 @@ struct FormatStyle {
              AfterFunctionDefinitionName == Other.AfterFunctionDefinitionName &&
              AfterIfMacros == Other.AfterIfMacros &&
              AfterOverloadedOperator == Other.AfterOverloadedOperator &&
+             AfterPlacementOperator == Other.AfterPlacementOperator &&
              AfterRequiresInClause == Other.AfterRequiresInClause &&
              AfterRequiresInExpression == Other.AfterRequiresInExpression &&
              BeforeNonEmptyParentheses == Other.BeforeNonEmptyParentheses;
@@ -4837,6 +4861,8 @@ FormatStyle getGNUStyle();
 /// https://docs.microsoft.com/en-us/visualstudio/ide/editorconfig-code-style-settings-reference?view=vs-2017
 FormatStyle getMicrosoftStyle(FormatStyle::LanguageKind Language);
 
+FormatStyle getClangFormatStyle();
+
 /// Returns style indicating formatting should be not applied at all.
 FormatStyle getNoStyle();
 
diff --git a/clang/include/clang/Frontend/VerifyDiagnosticConsumer.h b/clang/include/clang/Frontend/VerifyDiagnosticConsumer.h
index e0f3570e5b..500a7e11ab 100644
--- a/clang/include/clang/Frontend/VerifyDiagnosticConsumer.h
+++ b/clang/include/clang/Frontend/VerifyDiagnosticConsumer.h
@@ -278,14 +278,15 @@ class VerifyDiagnosticConsumer: public DiagnosticConsumer,
 
   // These facilities are used for validation in debug builds.
   class UnparsedFileStatus {
-    llvm::PointerIntPair<OptionalFileEntryRef, 1, bool> Data;
+    OptionalFileEntryRef File;
+    bool FoundDirectives;
 
   public:
     UnparsedFileStatus(OptionalFileEntryRef File, bool FoundDirectives)
-        : Data(File, FoundDirectives) {}
+        : File(File), FoundDirectives(FoundDirectives) {}
 
-    OptionalFileEntryRef getFile() const { return Data.getPointer(); }
-    bool foundDirectives() const { return Data.getInt(); }
+    OptionalFileEntryRef getFile() const { return File; }
+    bool foundDirectives() const { return FoundDirectives; }
   };
 
   using ParsedFilesMap = llvm::DenseMap<FileID, const FileEntry *>;
diff --git a/clang/include/clang/Lex/ModuleMap.h b/clang/include/clang/Lex/ModuleMap.h
index fc49742ad4..054d4211b9 100644
--- a/clang/include/clang/Lex/ModuleMap.h
+++ b/clang/include/clang/Lex/ModuleMap.h
@@ -20,8 +20,8 @@
 #include "clang/Basic/SourceLocation.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/PointerIntPair.h"
-#include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
@@ -194,7 +194,7 @@ class ModuleMap {
     }
   };
 
-  using AdditionalModMapsSet = llvm::SmallPtrSet<FileEntryRef, 1>;
+  using AdditionalModMapsSet = llvm::DenseSet<FileEntryRef>;
 
 private:
   friend class ModuleMapParser;
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 2ebd21090a..73fee208cb 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -3809,17 +3809,6 @@ class Sema final {
   // the purposes of [temp.friend] p9.
   bool FriendConstraintsDependOnEnclosingTemplate(const FunctionDecl *FD);
 
-  // Calculates whether two constraint expressions are equal irrespective of a
-  // difference in 'depth'. This takes a pair of optional 'NamedDecl's 'Old' and
-  // 'New', which are the "source" of the constraint, since this is necessary
-  // for figuring out the relative 'depth' of the constraint. The depth of the
-  // 'primary template' and the 'instantiated from' templates aren't necessarily
-  // the same, such as a case when one is a 'friend' defined in a class.
-  bool AreConstraintExpressionsEqual(const NamedDecl *Old,
-                                     const Expr *OldConstr,
-                                     const NamedDecl *New,
-                                     const Expr *NewConstr);
-
   enum class AllowedExplicit {
     /// Allow no explicit functions to be used.
     None,
@@ -4795,6 +4784,8 @@ class Sema final {
   bool CheckAlwaysInlineAttr(const Stmt *OrigSt, const Stmt *CurSt,
                              const AttributeCommonInfo &A);
 
+  bool CheckCountedByAttr(Scope *Scope, const FieldDecl *FD);
+
   /// Adjust the calling convention of a method to be the ABI default if it
   /// wasn't specified explicitly.  This handles method types formed from
   /// function type typedefs and typename template arguments.
@@ -5638,6 +5629,7 @@ class Sema final {
                       CorrectionCandidateCallback &CCC,
                       TemplateArgumentListInfo *ExplicitTemplateArgs = nullptr,
                       ArrayRef<Expr *> Args = std::nullopt,
+                      DeclContext *LookupCtx = nullptr,
                       TypoExpr **Out = nullptr);
 
   DeclResult LookupIvarInObjCMethod(LookupResult &Lookup, Scope *S,
@@ -8612,8 +8604,48 @@ class Sema final {
     TPL_TemplateParamsEquivalent,
   };
 
+  // A struct to represent the 'new' declaration, which is either itself just
+  // the named decl, or the important information we need about it in order to
+  // do constraint comparisons.
+  class TemplateCompareNewDeclInfo {
+    const NamedDecl *ND = nullptr;
+    const DeclContext *DC = nullptr;
+    const DeclContext *LexicalDC = nullptr;
+    SourceLocation Loc;
+
+  public:
+    TemplateCompareNewDeclInfo(const NamedDecl *ND) : ND(ND) {}
+    TemplateCompareNewDeclInfo(const DeclContext *DeclCtx,
+                               const DeclContext *LexicalDeclCtx,
+                               SourceLocation Loc)
+
+        : DC(DeclCtx), LexicalDC(LexicalDeclCtx), Loc(Loc) {
+      assert(DC && LexicalDC &&
+             "Constructor only for cases where we have the information to put "
+             "in here");
+    }
+
+    // If this was constructed with no information, we cannot do substitution
+    // for constraint comparison, so make sure we can check that.
+    bool isInvalid() const { return !ND && !DC; }
+
+    const NamedDecl *getDecl() const { return ND; }
+
+    bool ContainsDecl(const NamedDecl *ND) const { return this->ND == ND; }
+
+    const DeclContext *getLexicalDeclContext() const {
+      return ND ? ND->getLexicalDeclContext() : LexicalDC;
+    }
+
+    const DeclContext *getDeclContext() const {
+      return ND ? ND->getDeclContext() : DC;
+    }
+
+    SourceLocation getLocation() const { return ND ? ND->getLocation() : Loc; }
+  };
+
   bool TemplateParameterListsAreEqual(
-      const NamedDecl *NewInstFrom, TemplateParameterList *New,
+      const TemplateCompareNewDeclInfo &NewInstFrom, TemplateParameterList *New,
       const NamedDecl *OldInstFrom, TemplateParameterList *Old, bool Complain,
       TemplateParameterListEqualKind Kind,
       SourceLocation TemplateArgLoc = SourceLocation());
@@ -8626,6 +8658,17 @@ class Sema final {
                                           Kind, TemplateArgLoc);
   }
 
+  // Calculates whether two constraint expressions are equal irrespective of a
+  // difference in 'depth'. This takes a pair of optional 'NamedDecl's 'Old' and
+  // 'New', which are the "source" of the constraint, since this is necessary
+  // for figuring out the relative 'depth' of the constraint. The depth of the
+  // 'primary template' and the 'instantiated from' templates aren't necessarily
+  // the same, such as a case when one is a 'friend' defined in a class.
+  bool AreConstraintExpressionsEqual(const NamedDecl *Old,
+                                     const Expr *OldConstr,
+                                     const TemplateCompareNewDeclInfo &New,
+                                     const Expr *NewConstr);
+
   bool CheckTemplateDeclScope(Scope *S, TemplateParameterList *TemplateParams);
 
   /// Called when the parser has parsed a C++ typename
@@ -9365,13 +9408,12 @@ class Sema final {
   // C++ Template Instantiation
   //
 
-  MultiLevelTemplateArgumentList
-  getTemplateInstantiationArgs(const NamedDecl *D, bool Final = false,
-                               const TemplateArgumentList *Innermost = nullptr,
-                               bool RelativeToPrimary = false,
-                               const FunctionDecl *Pattern = nullptr,
-                               bool ForConstraintInstantiation = false,
-                               bool SkipForSpecialization = false);
+  MultiLevelTemplateArgumentList getTemplateInstantiationArgs(
+      const NamedDecl *D, const DeclContext *DC = nullptr, bool Final = false,
+      const TemplateArgumentList *Innermost = nullptr,
+      bool RelativeToPrimary = false, const FunctionDecl *Pattern = nullptr,
+      bool ForConstraintInstantiation = false,
+      bool SkipForSpecialization = false);
 
   /// A context in which code is being synthesized (where a source location
   /// alone is not sufficient to identify the context). This covers template
diff --git a/clang/include/clang/Sema/Template.h b/clang/include/clang/Sema/Template.h
index 28d603bf11..2a553054a0 100644
--- a/clang/include/clang/Sema/Template.h
+++ b/clang/include/clang/Sema/Template.h
@@ -213,7 +213,9 @@ enum class TemplateSubstitutionKind : char {
              "substituted args outside retained args?");
       assert(getKind() == TemplateSubstitutionKind::Specialization);
       TemplateArgumentLists.push_back(
-          {{AssociatedDecl->getCanonicalDecl(), Final}, Args});
+          {{AssociatedDecl ? AssociatedDecl->getCanonicalDecl() : nullptr,
+            Final},
+           Args});
     }
 
     void addOuterTemplateArguments(ArgList Args) {
diff --git a/clang/include/clang/Sema/TypoCorrection.h b/clang/include/clang/Sema/TypoCorrection.h
index e0f8d152db..09de164297 100644
--- a/clang/include/clang/Sema/TypoCorrection.h
+++ b/clang/include/clang/Sema/TypoCorrection.h
@@ -282,7 +282,7 @@ class CorrectionCandidateCallback {
 public:
   static const unsigned InvalidDistance = TypoCorrection::InvalidDistance;
 
-  explicit CorrectionCandidateCallback(IdentifierInfo *Typo = nullptr,
+  explicit CorrectionCandidateCallback(const IdentifierInfo *Typo = nullptr,
                                        NestedNameSpecifier *TypoNNS = nullptr)
       : Typo(Typo), TypoNNS(TypoNNS) {}
 
@@ -319,7 +319,7 @@ class CorrectionCandidateCallback {
   /// this method.
   virtual std::unique_ptr<CorrectionCandidateCallback> clone() = 0;
 
-  void setTypoName(IdentifierInfo *II) { Typo = II; }
+  void setTypoName(const IdentifierInfo *II) { Typo = II; }
   void setTypoNNS(NestedNameSpecifier *NNS) { TypoNNS = NNS; }
 
   // Flags for context-dependent keywords. WantFunctionLikeCasts is only
@@ -345,13 +345,13 @@ class CorrectionCandidateCallback {
            candidate.getCorrectionSpecifier() == TypoNNS;
   }
 
-  IdentifierInfo *Typo;
+  const IdentifierInfo *Typo;
   NestedNameSpecifier *TypoNNS;
 };
 
 class DefaultFilterCCC final : public CorrectionCandidateCallback {
 public:
-  explicit DefaultFilterCCC(IdentifierInfo *Typo = nullptr,
+  explicit DefaultFilterCCC(const IdentifierInfo *Typo = nullptr,
                             NestedNameSpecifier *TypoNNS = nullptr)
       : CorrectionCandidateCallback(Typo, TypoNNS) {}
 
@@ -365,6 +365,10 @@ class DefaultFilterCCC final : public CorrectionCandidateCallback {
 template <class C>
 class DeclFilterCCC final : public CorrectionCandidateCallback {
 public:
+  explicit DeclFilterCCC(const IdentifierInfo *Typo = nullptr,
+                         NestedNameSpecifier *TypoNNS = nullptr)
+      : CorrectionCandidateCallback(Typo, TypoNNS) {}
+
   bool ValidateCandidate(const TypoCorrection &candidate) override {
     return candidate.getCorrectionDeclAs<C>();
   }
diff --git a/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td b/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td
index 4ca8c98af8..be813bde8b 100644
--- a/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td
+++ b/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td
@@ -532,6 +532,27 @@ def MismatchedDeallocatorChecker : Checker<"MismatchedDeallocator">,
   Dependencies<[DynamicMemoryModeling]>,
   Documentation<HasDocumentation>;
 
+def StdCLibraryFunctionsChecker : Checker<"StdCLibraryFunctions">,
+  HelpText<"Check for invalid arguments of C standard library functions, "
+           "and apply relations between arguments and return value">,
+  CheckerOptions<[
+    CmdLineOption<Boolean,
+                  "DisplayLoadedSummaries",
+                  "If set to true, the checker displays the found summaries "
+                  "for the given translation unit.",
+                  "false",
+                  Released,
+                  Hide>,
+    CmdLineOption<Boolean,
+                  "ModelPOSIX",
+                  "If set to true, the checker models additional functions "
+                  "from the POSIX standard.",
+                  "false",
+                  InAlpha>
+  ]>,
+  WeakDependencies<[CallAndMessageChecker, NonNullParamChecker]>,
+  Documentation<HasDocumentation>;
+
 def VforkChecker : Checker<"Vfork">,
   HelpText<"Check for proper usage of vfork">,
   Documentation<HasDocumentation>;
@@ -574,27 +595,6 @@ def BlockInCriticalSectionChecker : Checker<"BlockInCriticalSection">,
   HelpText<"Check for calls to blocking functions inside a critical section">,
   Documentation<HasDocumentation>;
 
-def StdCLibraryFunctionsChecker : Checker<"StdCLibraryFunctions">,
-  HelpText<"Check for invalid arguments of C standard library functions, "
-           "and apply relations between arguments and return value">,
-  CheckerOptions<[
-    CmdLineOption<Boolean,
-                  "DisplayLoadedSummaries",
-                  "If set to true, the checker displays the found summaries "
-                  "for the given translation unit.",
-                  "false",
-                  Released,
-                  Hide>,
-    CmdLineOption<Boolean,
-                  "ModelPOSIX",
-                  "If set to true, the checker models additional functions "
-                  "from the POSIX standard.",
-                  "false",
-                  InAlpha>
-  ]>,
-  WeakDependencies<[CallAndMessageChecker, NonNullParamChecker, StreamChecker]>,
-  Documentation<HasDocumentation>;
-
 } // end "alpha.unix"
 
 //===----------------------------------------------------------------------===//
@@ -1627,6 +1627,7 @@ def DebugIteratorModeling : Checker<"DebugIteratorModeling">,
 def StdCLibraryFunctionsTesterChecker : Checker<"StdCLibraryFunctionsTester">,
   HelpText<"Add test functions to the summary map, so testing of individual "
            "summary constituents becomes possible.">,
+  WeakDependencies<[StdCLibraryFunctionsChecker]>,
   Documentation<NotDocumented>;
 
 } // end "debug"
diff --git a/clang/include/clang/Support/RISCVVIntrinsicUtils.h b/clang/include/clang/Support/RISCVVIntrinsicUtils.h
index 8ba57d7722..90d99b7efa 100644
--- a/clang/include/clang/Support/RISCVVIntrinsicUtils.h
+++ b/clang/include/clang/Support/RISCVVIntrinsicUtils.h
@@ -485,14 +485,16 @@ enum RVVRequire : uint16_t {
   RVV_REQ_RV64 = 1 << 0,
   RVV_REQ_ZvfhminOrZvfh = 1 << 1,
   RVV_REQ_Xsfvcp = 1 << 2,
-  RVV_REQ_Zvbb = 1 << 3,
-  RVV_REQ_Zvbc = 1 << 4,
-  RVV_REQ_Zvkb = 1 << 5,
-  RVV_REQ_Zvkg = 1 << 6,
-  RVV_REQ_Zvkned = 1 << 7,
-  RVV_REQ_Zvknha = 1 << 8,
-  RVV_REQ_Zvksed = 1 << 9,
-  RVV_REQ_Zvksh = 1 << 10,
+  RVV_REQ_Xsfvqmaccdod = 1 << 3,
+  RVV_REQ_Xsfvqmaccqoq = 1 << 4,
+  RVV_REQ_Zvbb = 1 << 5,
+  RVV_REQ_Zvbc = 1 << 6,
+  RVV_REQ_Zvkb = 1 << 7,
+  RVV_REQ_Zvkg = 1 << 8,
+  RVV_REQ_Zvkned = 1 << 9,
+  RVV_REQ_Zvknha = 1 << 10,
+  RVV_REQ_Zvksed = 1 << 11,
+  RVV_REQ_Zvksh = 1 << 12,
 
   LLVM_MARK_AS_BITMASK_ENUM(RVV_REQ_Zvksh)
 };
diff --git a/clang/lib/APINotes/APINotesWriter.cpp b/clang/lib/APINotes/APINotesWriter.cpp
index a92b379a8e..770d78e220 100644
--- a/clang/lib/APINotes/APINotesWriter.cpp
+++ b/clang/lib/APINotes/APINotesWriter.cpp
@@ -294,7 +294,7 @@ class IdentifierTableInfo {
     uint32_t KeyLength = Key.size();
     uint32_t DataLength = sizeof(uint32_t);
 
-    llvm::support::endian::Writer writer(OS, llvm::support::little);
+    llvm::support::endian::Writer writer(OS, llvm::endianness::little);
     writer.write<uint16_t>(KeyLength);
     writer.write<uint16_t>(DataLength);
     return {KeyLength, DataLength};
@@ -303,7 +303,7 @@ class IdentifierTableInfo {
   void EmitKey(raw_ostream &OS, key_type_ref Key, unsigned) { OS << Key; }
 
   void EmitData(raw_ostream &OS, key_type_ref, data_type_ref Data, unsigned) {
-    llvm::support::endian::Writer writer(OS, llvm::support::little);
+    llvm::support::endian::Writer writer(OS, llvm::endianness::little);
     writer.write<uint32_t>(Data);
   }
 };
@@ -326,7 +326,7 @@ void APINotesWriter::Implementation::writeIdentifierBlock(
     llvm::raw_svector_ostream BlobStream(HashTableBlob);
     // Make sure that no bucket is at offset 0
     llvm::support::endian::write<uint32_t>(BlobStream, 0,
-                                           llvm::support::little);
+                                           llvm::endianness::little);
     Offset = Generator.Emit(BlobStream);
   }
 
@@ -354,21 +354,21 @@ class ObjCContextIDTableInfo {
     uint32_t KeyLength = sizeof(uint32_t) + sizeof(uint8_t) + sizeof(uint32_t);
     uint32_t DataLength = sizeof(uint32_t);
 
-    llvm::support::endian::Writer writer(OS, llvm::support::little);
+    llvm::support::endian::Writer writer(OS, llvm::endianness::little);
     writer.write<uint16_t>(KeyLength);
     writer.write<uint16_t>(DataLength);
     return {KeyLength, DataLength};
   }
 
   void EmitKey(raw_ostream &OS, key_type_ref Key, unsigned) {
-    llvm::support::endian::Writer writer(OS, llvm::support::little);
+    llvm::support::endian::Writer writer(OS, llvm::endianness::little);
     writer.write<uint32_t>(Key.parentContextID);
     writer.write<uint8_t>(Key.contextKind);
     writer.write<uint32_t>(Key.contextID);
   }
 
   void EmitData(raw_ostream &OS, key_type_ref, data_type_ref Data, unsigned) {
-    llvm::support::endian::Writer writer(OS, llvm::support::little);
+    llvm::support::endian::Writer writer(OS, llvm::endianness::little);
     writer.write<uint32_t>(Data);
   }
 };
@@ -406,7 +406,7 @@ unsigned getVersionedInfoSize(
 
 /// Emit a serialized representation of a version tuple.
 void emitVersionTuple(raw_ostream &OS, const VersionTuple &VT) {
-  llvm::support::endian::Writer writer(OS, llvm::support::little);
+  llvm::support::endian::Writer writer(OS, llvm::endianness::little);
 
   // First byte contains the number of components beyond the 'major' component.
   uint8_t descriptor;
@@ -445,7 +445,7 @@ void emitVersionedInfo(
               return LHS.first < RHS.first;
             });
 
-  llvm::support::endian::Writer writer(OS, llvm::support::little);
+  llvm::support::endian::Writer writer(OS, llvm::endianness::little);
   writer.write<uint16_t>(VI.size());
   for (const auto &E : VI) {
     emitVersionTuple(OS, E.first);
@@ -479,7 +479,7 @@ class VersionedTableInfo {
           return asDerived().getUnversionedInfoSize(UI);
         });
 
-    llvm::support::endian::Writer writer(OS, llvm::support::little);
+    llvm::support::endian::Writer writer(OS, llvm::endianness::little);
     writer.write<uint16_t>(KeyLength);
     writer.write<uint16_t>(DataLength);
     return {KeyLength, DataLength};
@@ -495,7 +495,7 @@ class VersionedTableInfo {
 
 /// Emit a serialized representation of the common entity information.
 void emitCommonEntityInfo(raw_ostream &OS, const CommonEntityInfo &CEI) {
-  llvm::support::endian::Writer writer(OS, llvm::support::little);
+  llvm::support::endian::Writer writer(OS, llvm::endianness::little);
 
   uint8_t payload = 0;
   if (auto swiftPrivate = CEI.isSwiftPrivate()) {
@@ -535,7 +535,7 @@ unsigned getCommonTypeInfoSize(const CommonTypeInfo &CTI) {
 void emitCommonTypeInfo(raw_ostream &OS, const CommonTypeInfo &CTI) {
   emitCommonEntityInfo(OS, CTI);
 
-  llvm::support::endian::Writer writer(OS, llvm::support::little);
+  llvm::support::endian::Writer writer(OS, llvm::endianness::little);
   if (auto swiftBridge = CTI.getSwiftBridge()) {
     writer.write<uint16_t>(swiftBridge->size() + 1);
     OS.write(swiftBridge->c_str(), swiftBridge->size());
@@ -558,7 +558,7 @@ class ObjCContextInfoTableInfo
   unsigned getKeyLength(key_type_ref) { return sizeof(uint32_t); }
 
   void EmitKey(raw_ostream &OS, key_type_ref Key, unsigned) {
-    llvm::support::endian::Writer writer(OS, llvm::support::little);
+    llvm::support::endian::Writer writer(OS, llvm::endianness::little);
     writer.write<uint32_t>(Key);
   }
 
@@ -607,7 +607,7 @@ void APINotesWriter::Implementation::writeObjCContextBlock(
       llvm::raw_svector_ostream BlobStream(HashTableBlob);
       // Make sure that no bucket is at offset 0
       llvm::support::endian::write<uint32_t>(BlobStream, 0,
-                                             llvm::support::little);
+                                             llvm::endianness::little);
       Offset = Generator.Emit(BlobStream);
     }
 
@@ -626,7 +626,7 @@ void APINotesWriter::Implementation::writeObjCContextBlock(
       llvm::raw_svector_ostream BlobStream(HashTableBlob);
       // Make sure that no bucket is at offset 0
       llvm::support::endian::write<uint32_t>(BlobStream, 0,
-                                             llvm::support::little);
+                                             llvm::endianness::little);
       Offset = Generator.Emit(BlobStream);
     }
 
@@ -656,7 +656,7 @@ void emitVariableInfo(raw_ostream &OS, const VariableInfo &VI) {
 
   OS.write(reinterpret_cast<const char *>(bytes), 2);
 
-  llvm::support::endian::Writer writer(OS, llvm::support::little);
+  llvm::support::endian::Writer writer(OS, llvm::endianness::little);
   writer.write<uint16_t>(VI.getType().size());
   OS.write(VI.getType().data(), VI.getType().size());
 }
@@ -672,7 +672,7 @@ class ObjCPropertyTableInfo
   }
 
   void EmitKey(raw_ostream &OS, key_type_ref Key, unsigned) {
-    llvm::support::endian::Writer writer(OS, llvm::support::little);
+    llvm::support::endian::Writer writer(OS, llvm::endianness::little);
     writer.write<uint32_t>(std::get<0>(Key));
     writer.write<uint32_t>(std::get<1>(Key));
     writer.write<uint8_t>(std::get<2>(Key));
@@ -717,7 +717,7 @@ void APINotesWriter::Implementation::writeObjCPropertyBlock(
       llvm::raw_svector_ostream BlobStream(HashTableBlob);
       // Make sure that no bucket is at offset 0
       llvm::support::endian::write<uint32_t>(BlobStream, 0,
-                                             llvm::support::little);
+                                             llvm::endianness::little);
       Offset = Generator.Emit(BlobStream);
     }
 
@@ -741,7 +741,7 @@ class ObjCMethodTableInfo
   }
 
   void EmitKey(raw_ostream &OS, key_type_ref Key, unsigned) {
-    llvm::support::endian::Writer writer(OS, llvm::support::little);
+    llvm::support::endian::Writer writer(OS, llvm::endianness::little);
     writer.write<uint32_t>(std::get<0>(Key));
     writer.write<uint32_t>(std::get<1>(Key));
     writer.write<uint8_t>(std::get<2>(Key));
@@ -757,7 +757,7 @@ class ObjCMethodTableInfo
 
   void emitUnversionedInfo(raw_ostream &OS, const ObjCMethodInfo &OMI) {
     uint8_t flags = 0;
-    llvm::support::endian::Writer writer(OS, llvm::support::little);
+    llvm::support::endian::Writer writer(OS, llvm::endianness::little);
     flags = (flags << 1) | OMI.DesignatedInit;
     flags = (flags << 1) | OMI.RequiredInit;
     writer.write<uint8_t>(flags);
@@ -785,7 +785,7 @@ void APINotesWriter::Implementation::writeObjCMethodBlock(
       llvm::raw_svector_ostream BlobStream(HashTableBlob);
       // Make sure that no bucket is at offset 0
       llvm::support::endian::write<uint32_t>(BlobStream, 0,
-                                             llvm::support::little);
+                                             llvm::endianness::little);
       Offset = Generator.Emit(BlobStream);
     }
 
@@ -815,21 +815,21 @@ class ObjCSelectorTableInfo {
         sizeof(uint16_t) + sizeof(uint32_t) * Key.Identifiers.size();
     uint32_t DataLength = sizeof(uint32_t);
 
-    llvm::support::endian::Writer writer(OS, llvm::support::little);
+    llvm::support::endian::Writer writer(OS, llvm::endianness::little);
     writer.write<uint16_t>(KeyLength);
     writer.write<uint16_t>(DataLength);
     return {KeyLength, DataLength};
   }
 
   void EmitKey(raw_ostream &OS, key_type_ref Key, unsigned) {
-    llvm::support::endian::Writer writer(OS, llvm::support::little);
+    llvm::support::endian::Writer writer(OS, llvm::endianness::little);
     writer.write<uint16_t>(Key.NumPieces);
     for (auto Identifier : Key.Identifiers)
       writer.write<uint32_t>(Identifier);
   }
 
   void EmitData(raw_ostream &OS, key_type_ref, data_type_ref Data, unsigned) {
-    llvm::support::endian::Writer writer(OS, llvm::support::little);
+    llvm::support::endian::Writer writer(OS, llvm::endianness::little);
     writer.write<uint32_t>(Data);
   }
 };
@@ -853,7 +853,7 @@ void APINotesWriter::Implementation::writeObjCSelectorBlock(
       llvm::raw_svector_ostream BlobStream(HashTableBlob);
       // Make sure that no bucket is at offset 0
       llvm::support::endian::write<uint32_t>(BlobStream, 0,
-                                             llvm::support::little);
+                                             llvm::endianness::little);
       Offset = Generator.Emit(BlobStream);
     }
 
@@ -873,7 +873,7 @@ class GlobalVariableTableInfo
   }
 
   void EmitKey(raw_ostream &OS, key_type_ref Key, unsigned) {
-    llvm::support::endian::Writer writer(OS, llvm::support::little);
+    llvm::support::endian::Writer writer(OS, llvm::endianness::little);
     writer.write<uint32_t>(Key.parentContextID);
     writer.write<uint8_t>(Key.contextKind);
     writer.write<uint32_t>(Key.contextID);
@@ -911,7 +911,7 @@ void APINotesWriter::Implementation::writeGlobalVariableBlock(
       llvm::raw_svector_ostream BlobStream(HashTableBlob);
       // Make sure that no bucket is at offset 0
       llvm::support::endian::write<uint32_t>(BlobStream, 0,
-                                             llvm::support::little);
+                                             llvm::endianness::little);
       Offset = Generator.Emit(BlobStream);
     }
 
@@ -938,7 +938,7 @@ void emitParamInfo(raw_ostream &OS, const ParamInfo &PI) {
   if (auto RCC = PI.getRetainCountConvention())
     flags |= static_cast<uint8_t>(RCC.value()) + 1;
 
-  llvm::support::endian::Writer writer(OS, llvm::support::little);
+  llvm::support::endian::Writer writer(OS, llvm::endianness::little);
   writer.write<uint8_t>(flags);
 }
 
@@ -963,7 +963,7 @@ void emitFunctionInfo(raw_ostream &OS, const FunctionInfo &FI) {
   if (auto RCC = FI.getRetainCountConvention())
     flags |= static_cast<uint8_t>(RCC.value()) + 1;
 
-  llvm::support::endian::Writer writer(OS, llvm::support::little);
+  llvm::support::endian::Writer writer(OS, llvm::endianness::little);
 
   writer.write<uint8_t>(flags);
   writer.write<uint8_t>(FI.NumAdjustedNullable);
@@ -987,7 +987,7 @@ class GlobalFunctionTableInfo
   }
 
   void EmitKey(raw_ostream &OS, key_type_ref Key, unsigned) {
-    llvm::support::endian::Writer writer(OS, llvm::support::little);
+    llvm::support::endian::Writer writer(OS, llvm::endianness::little);
     writer.write<uint32_t>(Key.parentContextID);
     writer.write<uint8_t>(Key.contextKind);
     writer.write<uint32_t>(Key.contextID);
@@ -1025,7 +1025,7 @@ void APINotesWriter::Implementation::writeGlobalFunctionBlock(
       llvm::raw_svector_ostream BlobStream(HashTableBlob);
       // Make sure that no bucket is at offset 0
       llvm::support::endian::write<uint32_t>(BlobStream, 0,
-                                             llvm::support::little);
+                                             llvm::endianness::little);
       Offset = Generator.Emit(BlobStream);
     }
 
@@ -1043,7 +1043,7 @@ class EnumConstantTableInfo
   unsigned getKeyLength(key_type_ref) { return sizeof(uint32_t); }
 
   void EmitKey(raw_ostream &OS, key_type_ref Key, unsigned) {
-    llvm::support::endian::Writer writer(OS, llvm::support::little);
+    llvm::support::endian::Writer writer(OS, llvm::endianness::little);
     writer.write<uint32_t>(Key);
   }
 
@@ -1079,7 +1079,7 @@ void APINotesWriter::Implementation::writeEnumConstantBlock(
       llvm::raw_svector_ostream BlobStream(HashTableBlob);
       // Make sure that no bucket is at offset 0
       llvm::support::endian::write<uint32_t>(BlobStream, 0,
-                                             llvm::support::little);
+                                             llvm::endianness::little);
       Offset = Generator.Emit(BlobStream);
     }
 
@@ -1101,7 +1101,7 @@ class CommonTypeTableInfo
   }
 
   void EmitKey(raw_ostream &OS, key_type_ref Key, unsigned) {
-    llvm::support::endian::Writer writer(OS, llvm::support::little);
+    llvm::support::endian::Writer writer(OS, llvm::endianness::little);
     writer.write<uint32_t>(Key.parentContextID);
     writer.write<uint8_t>(Key.contextKind);
     writer.write<IdentifierID>(Key.contextID);
@@ -1131,7 +1131,7 @@ class TagTableInfo : public CommonTypeTableInfo<TagTableInfo, TagInfo> {
   }
 
   void emitUnversionedInfo(raw_ostream &OS, const TagInfo &TI) {
-    llvm::support::endian::Writer writer(OS, llvm::support::little);
+    llvm::support::endian::Writer writer(OS, llvm::endianness::little);
 
     uint8_t Flags = 0;
     if (auto extensibility = TI.EnumExtensibility) {
@@ -1187,7 +1187,7 @@ void APINotesWriter::Implementation::writeTagBlock(
       llvm::raw_svector_ostream BlobStream(HashTableBlob);
       // Make sure that no bucket is at offset 0
       llvm::support::endian::write<uint32_t>(BlobStream, 0,
-                                             llvm::support::little);
+                                             llvm::endianness::little);
       Offset = Generator.Emit(BlobStream);
     }
 
@@ -1206,7 +1206,7 @@ class TypedefTableInfo
   }
 
   void emitUnversionedInfo(raw_ostream &OS, const TypedefInfo &TI) {
-    llvm::support::endian::Writer writer(OS, llvm::support::little);
+    llvm::support::endian::Writer writer(OS, llvm::endianness::little);
 
     uint8_t Flags = 0;
     if (auto swiftWrapper = TI.SwiftWrapper)
@@ -1237,7 +1237,7 @@ void APINotesWriter::Implementation::writeTypedefBlock(
       llvm::raw_svector_ostream BlobStream(HashTableBlob);
       // Make sure that no bucket is at offset 0
       llvm::support::endian::write<uint32_t>(BlobStream, 0,
-                                             llvm::support::little);
+                                             llvm::endianness::little);
       Offset = Generator.Emit(BlobStream);
     }
 
diff --git a/clang/lib/ARCMigrate/FileRemapper.cpp b/clang/lib/ARCMigrate/FileRemapper.cpp
index bd8317e0a2..7abc862cee 100644
--- a/clang/lib/ARCMigrate/FileRemapper.cpp
+++ b/clang/lib/ARCMigrate/FileRemapper.cpp
@@ -134,9 +134,8 @@ bool FileRemapper::flushToFile(StringRef outputPath, DiagnosticsEngine &Diag) {
     infoOut << origPath << '\n';
     infoOut << (uint64_t)origFE.getModificationTime() << '\n';
 
-    if (I->second.is<FileEntryRef>()) {
-      auto FE = I->second.get<FileEntryRef>();
-      SmallString<200> newPath = StringRef(FE.getName());
+    if (const auto *FE = std::get_if<FileEntryRef>(&I->second)) {
+      SmallString<200> newPath = StringRef(FE->getName());
       fs::make_absolute(newPath);
       infoOut << newPath << '\n';
     } else {
@@ -150,7 +149,7 @@ bool FileRemapper::flushToFile(StringRef outputPath, DiagnosticsEngine &Diag) {
         return report("Could not create file: " + tempPath.str(), Diag);
 
       llvm::raw_fd_ostream newOut(fd, /*shouldClose=*/true);
-      llvm::MemoryBuffer *mem = I->second.get<llvm::MemoryBuffer *>();
+      llvm::MemoryBuffer *mem = std::get<llvm::MemoryBuffer *>(I->second);
       newOut.write(mem->getBufferStart(), mem->getBufferSize());
       newOut.close();
 
@@ -173,7 +172,7 @@ bool FileRemapper::overwriteOriginal(DiagnosticsEngine &Diag,
   for (MappingsTy::iterator
          I = FromToMappings.begin(), E = FromToMappings.end(); I != E; ++I) {
     FileEntryRef origFE = I->first;
-    assert(I->second.is<llvm::MemoryBuffer *>());
+    assert(std::holds_alternative<llvm::MemoryBuffer *>(I->second));
     if (!fs::exists(origFE.getName()))
       return report(StringRef("File does not exist: ") + origFE.getName(),
                     Diag);
@@ -183,7 +182,7 @@ bool FileRemapper::overwriteOriginal(DiagnosticsEngine &Diag,
     if (EC)
       return report(EC.message(), Diag);
 
-    llvm::MemoryBuffer *mem = I->second.get<llvm::MemoryBuffer *>();
+    llvm::MemoryBuffer *mem = std::get<llvm::MemoryBuffer *>(I->second);
     Out.write(mem->getBufferStart(), mem->getBufferSize());
     Out.close();
   }
@@ -197,25 +196,23 @@ void FileRemapper::forEachMapping(
     llvm::function_ref<void(StringRef, const llvm::MemoryBufferRef &)>
         CaptureBuffer) const {
   for (auto &Mapping : FromToMappings) {
-    if (Mapping.second.is<FileEntryRef>()) {
-      auto FE = Mapping.second.get<FileEntryRef>();
-      CaptureFile(Mapping.first.getName(), FE.getName());
+    if (const auto *FE = std::get_if<FileEntryRef>(&Mapping.second)) {
+      CaptureFile(Mapping.first.getName(), FE->getName());
       continue;
     }
     CaptureBuffer(
         Mapping.first.getName(),
-        Mapping.second.get<llvm::MemoryBuffer *>()->getMemBufferRef());
+        std::get<llvm::MemoryBuffer *>(Mapping.second)->getMemBufferRef());
   }
 }
 
 void FileRemapper::applyMappings(PreprocessorOptions &PPOpts) const {
   for (MappingsTy::const_iterator
          I = FromToMappings.begin(), E = FromToMappings.end(); I != E; ++I) {
-    if (I->second.is<FileEntryRef>()) {
-      auto FE = I->second.get<FileEntryRef>();
-      PPOpts.addRemappedFile(I->first.getName(), FE.getName());
+    if (const auto *FE = std::get_if<FileEntryRef>(&I->second)) {
+      PPOpts.addRemappedFile(I->first.getName(), FE->getName());
     } else {
-      llvm::MemoryBuffer *mem = I->second.get<llvm::MemoryBuffer *>();
+      llvm::MemoryBuffer *mem = std::get<llvm::MemoryBuffer *>(I->second);
       PPOpts.addRemappedFile(I->first.getName(), mem);
     }
   }
@@ -230,18 +227,20 @@ void FileRemapper::remap(StringRef filePath,
   remap(*File, std::move(memBuf));
 }
 
-void FileRemapper::remap(FileEntryRef file,
-                         std::unique_ptr<llvm::MemoryBuffer> memBuf) {
-  Target &targ = FromToMappings[file];
-  resetTarget(targ);
-  targ = memBuf.release();
+void FileRemapper::remap(FileEntryRef File,
+                         std::unique_ptr<llvm::MemoryBuffer> MemBuf) {
+  auto [It, New] = FromToMappings.insert({File, nullptr});
+  if (!New)
+    resetTarget(It->second);
+  It->second = MemBuf.release();
 }
 
-void FileRemapper::remap(FileEntryRef file, FileEntryRef newfile) {
-  Target &targ = FromToMappings[file];
-  resetTarget(targ);
-  targ = newfile;
-  ToFromMappings.insert({newfile, file});
+void FileRemapper::remap(FileEntryRef File, FileEntryRef NewFile) {
+  auto [It, New] = FromToMappings.insert({File, nullptr});
+  if (!New)
+    resetTarget(It->second);
+  It->second = NewFile;
+  ToFromMappings.insert({NewFile, File});
 }
 
 OptionalFileEntryRef FileRemapper::getOriginalFile(StringRef filePath) {
@@ -259,13 +258,11 @@ OptionalFileEntryRef FileRemapper::getOriginalFile(StringRef filePath) {
 }
 
 void FileRemapper::resetTarget(Target &targ) {
-  if (!targ)
-    return;
-
-  if (llvm::MemoryBuffer *oldmem = targ.dyn_cast<llvm::MemoryBuffer *>()) {
+  if (std::holds_alternative<llvm::MemoryBuffer *>(targ)) {
+    llvm::MemoryBuffer *oldmem = std::get<llvm::MemoryBuffer *>(targ);
     delete oldmem;
   } else {
-    FileEntryRef toFE = targ.get<FileEntryRef>();
+    FileEntryRef toFE = std::get<FileEntryRef>(targ);
     ToFromMappings.erase(toFE);
   }
 }
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index cdc3d62bca..27a675b832 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -1234,7 +1234,7 @@ TypedefDecl *ASTContext::getUInt128Decl() const {
 }
 
 void ASTContext::InitBuiltinType(CanQualType &R, BuiltinType::Kind K) {
-  auto *Ty = new (*this, TypeAlignment) BuiltinType(K);
+  auto *Ty = new (*this, alignof(BuiltinType)) BuiltinType(K);
   R = CanQualType::CreateUnsafe(QualType(Ty, 0));
   Types.push_back(Ty);
 }
@@ -3066,7 +3066,7 @@ ASTContext::getExtQualType(const Type *baseType, Qualifiers quals) const {
     (void) ExtQualNodes.FindNodeOrInsertPos(ID, insertPos);
   }
 
-  auto *eq = new (*this, TypeAlignment) ExtQuals(baseType, canon, quals);
+  auto *eq = new (*this, alignof(ExtQuals)) ExtQuals(baseType, canon, quals);
   ExtQualNodes.InsertNode(eq, insertPos);
   return QualType(eq, fastQuals);
 }
@@ -3310,7 +3310,7 @@ QualType ASTContext::getComplexType(QualType T) const {
     ComplexType *NewIP = ComplexTypes.FindNodeOrInsertPos(ID, InsertPos);
     assert(!NewIP && "Shouldn't be in the map!"); (void)NewIP;
   }
-  auto *New = new (*this, TypeAlignment) ComplexType(T, Canonical);
+  auto *New = new (*this, alignof(ComplexType)) ComplexType(T, Canonical);
   Types.push_back(New);
   ComplexTypes.InsertNode(New, InsertPos);
   return QualType(New, 0);
@@ -3338,7 +3338,7 @@ QualType ASTContext::getPointerType(QualType T) const {
     PointerType *NewIP = PointerTypes.FindNodeOrInsertPos(ID, InsertPos);
     assert(!NewIP && "Shouldn't be in the map!"); (void)NewIP;
   }
-  auto *New = new (*this, TypeAlignment) PointerType(T, Canonical);
+  auto *New = new (*this, alignof(PointerType)) PointerType(T, Canonical);
   Types.push_back(New);
   PointerTypes.InsertNode(New, InsertPos);
   return QualType(New, 0);
@@ -3358,7 +3358,7 @@ QualType ASTContext::getAdjustedType(QualType Orig, QualType New) const {
   AT = AdjustedTypes.FindNodeOrInsertPos(ID, InsertPos);
   assert(!AT && "Shouldn't be in the map!");
 
-  AT = new (*this, TypeAlignment)
+  AT = new (*this, alignof(AdjustedType))
       AdjustedType(Type::Adjusted, Orig, New, Canonical);
   Types.push_back(AT);
   AdjustedTypes.InsertNode(AT, InsertPos);
@@ -3379,7 +3379,7 @@ QualType ASTContext::getDecayedType(QualType Orig, QualType Decayed) const {
   AT = AdjustedTypes.FindNodeOrInsertPos(ID, InsertPos);
   assert(!AT && "Shouldn't be in the map!");
 
-  AT = new (*this, TypeAlignment) DecayedType(Orig, Decayed, Canonical);
+  AT = new (*this, alignof(DecayedType)) DecayedType(Orig, Decayed, Canonical);
   Types.push_back(AT);
   AdjustedTypes.InsertNode(AT, InsertPos);
   return QualType(AT, 0);
@@ -3433,7 +3433,8 @@ QualType ASTContext::getBlockPointerType(QualType T) const {
       BlockPointerTypes.FindNodeOrInsertPos(ID, InsertPos);
     assert(!NewIP && "Shouldn't be in the map!"); (void)NewIP;
   }
-  auto *New = new (*this, TypeAlignment) BlockPointerType(T, Canonical);
+  auto *New =
+      new (*this, alignof(BlockPointerType)) BlockPointerType(T, Canonical);
   Types.push_back(New);
   BlockPointerTypes.InsertNode(New, InsertPos);
   return QualType(New, 0);
@@ -3472,8 +3473,8 @@ ASTContext::getLValueReferenceType(QualType T, bool SpelledAsLValue) const {
     assert(!NewIP && "Shouldn't be in the map!"); (void)NewIP;
   }
 
-  auto *New = new (*this, TypeAlignment) LValueReferenceType(T, Canonical,
-                                                             SpelledAsLValue);
+  auto *New = new (*this, alignof(LValueReferenceType))
+      LValueReferenceType(T, Canonical, SpelledAsLValue);
   Types.push_back(New);
   LValueReferenceTypes.InsertNode(New, InsertPos);
 
@@ -3512,7 +3513,8 @@ QualType ASTContext::getRValueReferenceType(QualType T) const {
     assert(!NewIP && "Shouldn't be in the map!"); (void)NewIP;
   }
 
-  auto *New = new (*this, TypeAlignment) RValueReferenceType(T, Canonical);
+  auto *New = new (*this, alignof(RValueReferenceType))
+      RValueReferenceType(T, Canonical);
   Types.push_back(New);
   RValueReferenceTypes.InsertNode(New, InsertPos);
   return QualType(New, 0);
@@ -3542,7 +3544,8 @@ QualType ASTContext::getMemberPointerType(QualType T, const Type *Cls) const {
       MemberPointerTypes.FindNodeOrInsertPos(ID, InsertPos);
     assert(!NewIP && "Shouldn't be in the map!"); (void)NewIP;
   }
-  auto *New = new (*this, TypeAlignment) MemberPointerType(T, Cls, Canonical);
+  auto *New = new (*this, alignof(MemberPointerType))
+      MemberPointerType(T, Cls, Canonical);
   Types.push_back(New);
   MemberPointerTypes.InsertNode(New, InsertPos);
   return QualType(New, 0);
@@ -3596,7 +3599,7 @@ QualType ASTContext::getConstantArrayType(QualType EltTy,
 
   void *Mem = Allocate(
       ConstantArrayType::totalSizeToAlloc<const Expr *>(SizeExpr ? 1 : 0),
-      TypeAlignment);
+      alignof(ConstantArrayType));
   auto *New = new (Mem)
     ConstantArrayType(EltTy, Canon, ArySize, SizeExpr, ASM, IndexTypeQuals);
   ConstantArrayTypes.InsertNode(New, InsertPos);
@@ -3765,8 +3768,8 @@ QualType ASTContext::getVariableArrayType(QualType EltTy,
     Canon = getQualifiedType(Canon, canonSplit.Quals);
   }
 
-  auto *New = new (*this, TypeAlignment)
-    VariableArrayType(EltTy, Canon, NumElts, ASM, IndexTypeQuals, Brackets);
+  auto *New = new (*this, alignof(VariableArrayType))
+      VariableArrayType(EltTy, Canon, NumElts, ASM, IndexTypeQuals, Brackets);
 
   VariableArrayTypes.push_back(New);
   Types.push_back(New);
@@ -3790,8 +3793,9 @@ QualType ASTContext::getDependentSizedArrayType(QualType elementType,
   // initializer.  We do no canonicalization here at all, which is okay
   // because they can't be used in most locations.
   if (!numElements) {
-    auto *newType = new (*this, TypeAlignment) DependentSizedArrayType(
-        elementType, QualType(), numElements, ASM, elementTypeQuals, brackets);
+    auto *newType = new (*this, alignof(DependentSizedArrayType))
+        DependentSizedArrayType(elementType, QualType(), numElements, ASM,
+                                elementTypeQuals, brackets);
     Types.push_back(newType);
     return QualType(newType, 0);
   }
@@ -3813,7 +3817,7 @@ QualType ASTContext::getDependentSizedArrayType(QualType elementType,
 
   // If we don't have one, build one.
   if (!canonTy) {
-    canonTy = new (*this, TypeAlignment)
+    canonTy = new (*this, alignof(DependentSizedArrayType))
         DependentSizedArrayType(QualType(canonElementType.Ty, 0), QualType(),
                                 numElements, ASM, elementTypeQuals, brackets);
     DependentSizedArrayTypes.InsertNode(canonTy, insertPos);
@@ -3832,8 +3836,9 @@ QualType ASTContext::getDependentSizedArrayType(QualType elementType,
 
   // Otherwise, we need to build a type which follows the spelling
   // of the element type.
-  auto *sugaredType = new (*this, TypeAlignment) DependentSizedArrayType(
-      elementType, canon, numElements, ASM, elementTypeQuals, brackets);
+  auto *sugaredType = new (*this, alignof(DependentSizedArrayType))
+      DependentSizedArrayType(elementType, canon, numElements, ASM,
+                              elementTypeQuals, brackets);
   Types.push_back(sugaredType);
   return QualType(sugaredType, 0);
 }
@@ -3867,8 +3872,8 @@ QualType ASTContext::getIncompleteArrayType(QualType elementType,
     assert(!existing && "Shouldn't be in the map!"); (void) existing;
   }
 
-  auto *newType = new (*this, TypeAlignment)
-    IncompleteArrayType(elementType, canon, ASM, elementTypeQuals);
+  auto *newType = new (*this, alignof(IncompleteArrayType))
+      IncompleteArrayType(elementType, canon, ASM, elementTypeQuals);
 
   IncompleteArrayTypes.InsertNode(newType, insertPos);
   Types.push_back(newType);
@@ -4088,8 +4093,8 @@ QualType ASTContext::getVectorType(QualType vecType, unsigned NumElts,
     VectorType *NewIP = VectorTypes.FindNodeOrInsertPos(ID, InsertPos);
     assert(!NewIP && "Shouldn't be in the map!"); (void)NewIP;
   }
-  auto *New = new (*this, TypeAlignment)
-    VectorType(vecType, NumElts, Canonical, VecKind);
+  auto *New = new (*this, alignof(VectorType))
+      VectorType(vecType, NumElts, Canonical, VecKind);
   VectorTypes.InsertNode(New, InsertPos);
   Types.push_back(New);
   return QualType(New, 0);
@@ -4108,12 +4113,12 @@ ASTContext::getDependentVectorType(QualType VecType, Expr *SizeExpr,
   DependentVectorType *New;
 
   if (Canon) {
-    New = new (*this, TypeAlignment) DependentVectorType(
+    New = new (*this, alignof(DependentVectorType)) DependentVectorType(
         VecType, QualType(Canon, 0), SizeExpr, AttrLoc, VecKind);
   } else {
     QualType CanonVecTy = getCanonicalType(VecType);
     if (CanonVecTy == VecType) {
-      New = new (*this, TypeAlignment)
+      New = new (*this, alignof(DependentVectorType))
           DependentVectorType(VecType, QualType(), SizeExpr, AttrLoc, VecKind);
 
       DependentVectorType *CanonCheck =
@@ -4125,7 +4130,7 @@ ASTContext::getDependentVectorType(QualType VecType, Expr *SizeExpr,
     } else {
       QualType CanonTy = getDependentVectorType(CanonVecTy, SizeExpr,
                                                 SourceLocation(), VecKind);
-      New = new (*this, TypeAlignment)
+      New = new (*this, alignof(DependentVectorType))
           DependentVectorType(VecType, CanonTy, SizeExpr, AttrLoc, VecKind);
     }
   }
@@ -4162,8 +4167,8 @@ QualType ASTContext::getExtVectorType(QualType vecType,
     VectorType *NewIP = VectorTypes.FindNodeOrInsertPos(ID, InsertPos);
     assert(!NewIP && "Shouldn't be in the map!"); (void)NewIP;
   }
-  auto *New = new (*this, TypeAlignment)
-    ExtVectorType(vecType, NumElts, Canonical);
+  auto *New = new (*this, alignof(ExtVectorType))
+      ExtVectorType(vecType, NumElts, Canonical);
   VectorTypes.InsertNode(New, InsertPos);
   Types.push_back(New);
   return QualType(New, 0);
@@ -4184,12 +4189,13 @@ ASTContext::getDependentSizedExtVectorType(QualType vecType,
   if (Canon) {
     // We already have a canonical version of this array type; use it as
     // the canonical type for a newly-built type.
-    New = new (*this, TypeAlignment) DependentSizedExtVectorType(
-        vecType, QualType(Canon, 0), SizeExpr, AttrLoc);
+    New = new (*this, alignof(DependentSizedExtVectorType))
+        DependentSizedExtVectorType(vecType, QualType(Canon, 0), SizeExpr,
+                                    AttrLoc);
   } else {
     QualType CanonVecTy = getCanonicalType(vecType);
     if (CanonVecTy == vecType) {
-      New = new (*this, TypeAlignment)
+      New = new (*this, alignof(DependentSizedExtVectorType))
           DependentSizedExtVectorType(vecType, QualType(), SizeExpr, AttrLoc);
 
       DependentSizedExtVectorType *CanonCheck
@@ -4200,7 +4206,7 @@ ASTContext::getDependentSizedExtVectorType(QualType vecType,
     } else {
       QualType CanonExtTy = getDependentSizedExtVectorType(CanonVecTy, SizeExpr,
                                                            SourceLocation());
-      New = new (*this, TypeAlignment)
+      New = new (*this, alignof(DependentSizedExtVectorType))
           DependentSizedExtVectorType(vecType, CanonExtTy, SizeExpr, AttrLoc);
     }
   }
@@ -4234,7 +4240,7 @@ QualType ASTContext::getConstantMatrixType(QualType ElementTy, unsigned NumRows,
     (void)NewIP;
   }
 
-  auto *New = new (*this, TypeAlignment)
+  auto *New = new (*this, alignof(ConstantMatrixType))
       ConstantMatrixType(ElementTy, NumRows, NumColumns, Canonical);
   MatrixTypes.InsertNode(New, InsertPos);
   Types.push_back(New);
@@ -4255,8 +4261,9 @@ QualType ASTContext::getDependentSizedMatrixType(QualType ElementTy,
       DependentSizedMatrixTypes.FindNodeOrInsertPos(ID, InsertPos);
 
   if (!Canon) {
-    Canon = new (*this, TypeAlignment) DependentSizedMatrixType(
-        CanonElementTy, QualType(), RowExpr, ColumnExpr, AttrLoc);
+    Canon = new (*this, alignof(DependentSizedMatrixType))
+        DependentSizedMatrixType(CanonElementTy, QualType(), RowExpr,
+                                 ColumnExpr, AttrLoc);
 #ifndef NDEBUG
     DependentSizedMatrixType *CanonCheck =
         DependentSizedMatrixTypes.FindNodeOrInsertPos(ID, InsertPos);
@@ -4274,7 +4281,7 @@ QualType ASTContext::getDependentSizedMatrixType(QualType ElementTy,
     return QualType(Canon, 0);
 
   // Use Canon as the canonical type for newly-built type.
-  DependentSizedMatrixType *New = new (*this, TypeAlignment)
+  DependentSizedMatrixType *New = new (*this, alignof(DependentSizedMatrixType))
       DependentSizedMatrixType(ElementTy, QualType(Canon, 0), RowExpr,
                                ColumnExpr, AttrLoc);
   Types.push_back(New);
@@ -4297,8 +4304,9 @@ QualType ASTContext::getDependentAddressSpaceType(QualType PointeeType,
     DependentAddressSpaceTypes.FindNodeOrInsertPos(ID, insertPos);
 
   if (!canonTy) {
-    canonTy = new (*this, TypeAlignment) DependentAddressSpaceType(
-        canonPointeeType, QualType(), AddrSpaceExpr, AttrLoc);
+    canonTy = new (*this, alignof(DependentAddressSpaceType))
+        DependentAddressSpaceType(canonPointeeType, QualType(), AddrSpaceExpr,
+                                  AttrLoc);
     DependentAddressSpaceTypes.InsertNode(canonTy, insertPos);
     Types.push_back(canonTy);
   }
@@ -4307,8 +4315,9 @@ QualType ASTContext::getDependentAddressSpaceType(QualType PointeeType,
       canonTy->getAddrSpaceExpr() == AddrSpaceExpr)
     return QualType(canonTy, 0);
 
-  auto *sugaredType = new (*this, TypeAlignment) DependentAddressSpaceType(
-      PointeeType, QualType(canonTy, 0), AddrSpaceExpr, AttrLoc);
+  auto *sugaredType = new (*this, alignof(DependentAddressSpaceType))
+      DependentAddressSpaceType(PointeeType, QualType(canonTy, 0),
+                                AddrSpaceExpr, AttrLoc);
   Types.push_back(sugaredType);
   return QualType(sugaredType, 0);
 }
@@ -4352,8 +4361,8 @@ ASTContext::getFunctionNoProtoType(QualType ResultTy,
     assert(!NewIP && "Shouldn't be in the map!"); (void)NewIP;
   }
 
-  auto *New = new (*this, TypeAlignment)
-    FunctionNoProtoType(ResultTy, Canonical, Info);
+  auto *New = new (*this, alignof(FunctionNoProtoType))
+      FunctionNoProtoType(ResultTy, Canonical, Info);
   Types.push_back(New);
   FunctionNoProtoTypes.InsertNode(New, InsertPos);
   return QualType(New, 0);
@@ -4539,7 +4548,7 @@ QualType ASTContext::getFunctionTypeInternal(
       EPI.ExtParameterInfos ? NumArgs : 0,
       EPI.TypeQuals.hasNonFastQualifiers() ? 1 : 0);
 
-  auto *FTP = (FunctionProtoType *)Allocate(Size, TypeAlignment);
+  auto *FTP = (FunctionProtoType *)Allocate(Size, alignof(FunctionProtoType));
   FunctionProtoType::ExtProtoInfo newEPI = EPI;
   new (FTP) FunctionProtoType(ResultTy, ArgArray, Canonical, newEPI);
   Types.push_back(FTP);
@@ -4567,7 +4576,7 @@ QualType ASTContext::getPipeType(QualType T, bool ReadOnly) const {
     assert(!NewIP && "Shouldn't be in the map!");
     (void)NewIP;
   }
-  auto *New = new (*this, TypeAlignment) PipeType(T, Canonical, ReadOnly);
+  auto *New = new (*this, alignof(PipeType)) PipeType(T, Canonical, ReadOnly);
   Types.push_back(New);
   PipeTypes.InsertNode(New, InsertPos);
   return QualType(New, 0);
@@ -4595,7 +4604,7 @@ QualType ASTContext::getBitIntType(bool IsUnsigned, unsigned NumBits) const {
   if (BitIntType *EIT = BitIntTypes.FindNodeOrInsertPos(ID, InsertPos))
     return QualType(EIT, 0);
 
-  auto *New = new (*this, TypeAlignment) BitIntType(IsUnsigned, NumBits);
+  auto *New = new (*this, alignof(BitIntType)) BitIntType(IsUnsigned, NumBits);
   BitIntTypes.InsertNode(New, InsertPos);
   Types.push_back(New);
   return QualType(New, 0);
@@ -4612,8 +4621,8 @@ QualType ASTContext::getDependentBitIntType(bool IsUnsigned,
           DependentBitIntTypes.FindNodeOrInsertPos(ID, InsertPos))
     return QualType(Existing, 0);
 
-  auto *New =
-      new (*this, TypeAlignment) DependentBitIntType(IsUnsigned, NumBitsExpr);
+  auto *New = new (*this, alignof(DependentBitIntType))
+      DependentBitIntType(IsUnsigned, NumBitsExpr);
   DependentBitIntTypes.InsertNode(New, InsertPos);
 
   Types.push_back(New);
@@ -4645,8 +4654,8 @@ QualType ASTContext::getInjectedClassNameType(CXXRecordDecl *Decl,
     Decl->TypeForDecl = PrevDecl->TypeForDecl;
     assert(isa<InjectedClassNameType>(Decl->TypeForDecl));
   } else {
-    Type *newType =
-      new (*this, TypeAlignment) InjectedClassNameType(Decl, TST);
+    Type *newType = new (*this, alignof(InjectedClassNameType))
+        InjectedClassNameType(Decl, TST);
     Decl->TypeForDecl = newType;
     Types.push_back(newType);
   }
@@ -4687,7 +4696,7 @@ QualType ASTContext::getTypedefType(const TypedefNameDecl *Decl,
   if (!Decl->TypeForDecl) {
     if (Underlying.isNull())
       Underlying = Decl->getUnderlyingType();
-    auto *NewType = new (*this, TypeAlignment) TypedefType(
+    auto *NewType = new (*this, alignof(TypedefType)) TypedefType(
         Type::Typedef, Decl, QualType(), getCanonicalType(Underlying));
     Decl->TypeForDecl = NewType;
     Types.push_back(NewType);
@@ -4707,8 +4716,8 @@ QualType ASTContext::getTypedefType(const TypedefNameDecl *Decl,
     return QualType(T, 0);
   }
 
-  void *Mem =
-      Allocate(TypedefType::totalSizeToAlloc<QualType>(true), TypeAlignment);
+  void *Mem = Allocate(TypedefType::totalSizeToAlloc<QualType>(true),
+                       alignof(TypedefType));
   auto *NewType = new (Mem) TypedefType(Type::Typedef, Decl, Underlying,
                                         getCanonicalType(Underlying));
   TypedefTypes.InsertNode(NewType, InsertPos);
@@ -4736,7 +4745,7 @@ QualType ASTContext::getUsingType(const UsingShadowDecl *Found,
     Underlying = QualType();
   void *Mem =
       Allocate(UsingType::totalSizeToAlloc<QualType>(!Underlying.isNull()),
-               TypeAlignment);
+               alignof(UsingType));
   UsingType *NewType = new (Mem) UsingType(Found, Underlying, Canon);
   Types.push_back(NewType);
   UsingTypes.InsertNode(NewType, InsertPos);
@@ -4750,7 +4759,7 @@ QualType ASTContext::getRecordType(const RecordDecl *Decl) const {
     if (PrevDecl->TypeForDecl)
       return QualType(Decl->TypeForDecl = PrevDecl->TypeForDecl, 0);
 
-  auto *newType = new (*this, TypeAlignment) RecordType(Decl);
+  auto *newType = new (*this, alignof(RecordType)) RecordType(Decl);
   Decl->TypeForDecl = newType;
   Types.push_back(newType);
   return QualType(newType, 0);
@@ -4763,7 +4772,7 @@ QualType ASTContext::getEnumType(const EnumDecl *Decl) const {
     if (PrevDecl->TypeForDecl)
       return QualType(Decl->TypeForDecl = PrevDecl->TypeForDecl, 0);
 
-  auto *newType = new (*this, TypeAlignment) EnumType(Decl);
+  auto *newType = new (*this, alignof(EnumType)) EnumType(Decl);
   Decl->TypeForDecl = newType;
   Types.push_back(newType);
   return QualType(newType, 0);
@@ -4779,7 +4788,8 @@ QualType ASTContext::getUnresolvedUsingType(
     if (CanonicalDecl->TypeForDecl)
       return QualType(Decl->TypeForDecl = CanonicalDecl->TypeForDecl, 0);
 
-  Type *newType = new (*this, TypeAlignment) UnresolvedUsingType(Decl);
+  Type *newType =
+      new (*this, alignof(UnresolvedUsingType)) UnresolvedUsingType(Decl);
   Decl->TypeForDecl = newType;
   Types.push_back(newType);
   return QualType(newType, 0);
@@ -4796,7 +4806,7 @@ QualType ASTContext::getAttributedType(attr::Kind attrKind,
   if (type) return QualType(type, 0);
 
   QualType canon = getCanonicalType(equivalentType);
-  type = new (*this, TypeAlignment)
+  type = new (*this, alignof(AttributedType))
       AttributedType(canon, attrKind, modifiedType, equivalentType);
 
   Types.push_back(type);
@@ -4817,7 +4827,8 @@ QualType ASTContext::getBTFTagAttributedType(const BTFTypeTagAttr *BTFAttr,
     return QualType(Ty, 0);
 
   QualType Canon = getCanonicalType(Wrapped);
-  Ty = new (*this, TypeAlignment) BTFTagAttributedType(Canon, Wrapped, BTFAttr);
+  Ty = new (*this, alignof(BTFTagAttributedType))
+      BTFTagAttributedType(Canon, Wrapped, BTFAttr);
 
   Types.push_back(Ty);
   BTFTagAttributedTypes.InsertNode(Ty, InsertPos);
@@ -4839,7 +4850,7 @@ QualType ASTContext::getSubstTemplateTypeParmType(
   if (!SubstParm) {
     void *Mem = Allocate(SubstTemplateTypeParmType::totalSizeToAlloc<QualType>(
                              !Replacement.isCanonical()),
-                         TypeAlignment);
+                         alignof(SubstTemplateTypeParmType));
     SubstParm = new (Mem) SubstTemplateTypeParmType(Replacement, AssociatedDecl,
                                                     Index, PackIndex);
     Types.push_back(SubstParm);
@@ -4880,8 +4891,9 @@ ASTContext::getSubstTemplateTypeParmPackType(Decl *AssociatedDecl,
     }
   }
 
-  auto *SubstParm = new (*this, TypeAlignment) SubstTemplateTypeParmPackType(
-      Canon, AssociatedDecl, Index, Final, ArgPack);
+  auto *SubstParm = new (*this, alignof(SubstTemplateTypeParmPackType))
+      SubstTemplateTypeParmPackType(Canon, AssociatedDecl, Index, Final,
+                                    ArgPack);
   Types.push_back(SubstParm);
   SubstTemplateTypeParmPackTypes.InsertNode(SubstParm, InsertPos);
   return QualType(SubstParm, 0);
@@ -4904,15 +4916,16 @@ QualType ASTContext::getTemplateTypeParmType(unsigned Depth, unsigned Index,
 
   if (TTPDecl) {
     QualType Canon = getTemplateTypeParmType(Depth, Index, ParameterPack);
-    TypeParm = new (*this, TypeAlignment) TemplateTypeParmType(TTPDecl, Canon);
+    TypeParm = new (*this, alignof(TemplateTypeParmType))
+        TemplateTypeParmType(TTPDecl, Canon);
 
     TemplateTypeParmType *TypeCheck
       = TemplateTypeParmTypes.FindNodeOrInsertPos(ID, InsertPos);
     assert(!TypeCheck && "Template type parameter canonical type broken");
     (void)TypeCheck;
   } else
-    TypeParm = new (*this, TypeAlignment)
-      TemplateTypeParmType(Depth, Index, ParameterPack);
+    TypeParm = new (*this, alignof(TemplateTypeParmType))
+        TemplateTypeParmType(Depth, Index, ParameterPack);
 
   Types.push_back(TypeParm);
   TemplateTypeParmTypes.InsertNode(TypeParm, InsertPos);
@@ -4995,9 +5008,9 @@ ASTContext::getTemplateSpecializationType(TemplateName Template,
   // try to unique it: these types typically have location information that
   // we don't unique and don't want to lose.
   void *Mem = Allocate(sizeof(TemplateSpecializationType) +
-                       sizeof(TemplateArgument) * Args.size() +
-                       (IsTypeAlias? sizeof(QualType) : 0),
-                       TypeAlignment);
+                           sizeof(TemplateArgument) * Args.size() +
+                           (IsTypeAlias ? sizeof(QualType) : 0),
+                       alignof(TemplateSpecializationType));
   auto *Spec
     = new (Mem) TemplateSpecializationType(Template, Args, CanonType,
                                          IsTypeAlias ? Underlying : QualType());
@@ -5035,7 +5048,7 @@ QualType ASTContext::getCanonicalTemplateSpecializationType(
     // Allocate a new canonical template specialization type.
     void *Mem = Allocate((sizeof(TemplateSpecializationType) +
                           sizeof(TemplateArgument) * CanonArgs.size()),
-                         TypeAlignment);
+                         alignof(TemplateSpecializationType));
     Spec = new (Mem) TemplateSpecializationType(CanonTemplate,
                                                 CanonArgs,
                                                 QualType(), QualType());
@@ -5068,8 +5081,9 @@ QualType ASTContext::getElaboratedType(ElaboratedTypeKeyword Keyword,
     (void)CheckT;
   }
 
-  void *Mem = Allocate(ElaboratedType::totalSizeToAlloc<TagDecl *>(!!OwnedTagDecl),
-                       TypeAlignment);
+  void *Mem =
+      Allocate(ElaboratedType::totalSizeToAlloc<TagDecl *>(!!OwnedTagDecl),
+               alignof(ElaboratedType));
   T = new (Mem) ElaboratedType(Keyword, NNS, NamedType, Canon, OwnedTagDecl);
 
   Types.push_back(T);
@@ -5095,7 +5109,7 @@ ASTContext::getParenType(QualType InnerType) const {
     (void)CheckT;
   }
 
-  T = new (*this, TypeAlignment) ParenType(InnerType, Canon);
+  T = new (*this, alignof(ParenType)) ParenType(InnerType, Canon);
   Types.push_back(T);
   ParenTypes.InsertNode(T, InsertPos);
   return QualType(T, 0);
@@ -5108,7 +5122,7 @@ ASTContext::getMacroQualifiedType(QualType UnderlyingTy,
   if (!Canon.isCanonical())
     Canon = getCanonicalType(UnderlyingTy);
 
-  auto *newType = new (*this, TypeAlignment)
+  auto *newType = new (*this, alignof(MacroQualifiedType))
       MacroQualifiedType(UnderlyingTy, Canon, MacroII);
   Types.push_back(newType);
   return QualType(newType, 0);
@@ -5133,7 +5147,8 @@ QualType ASTContext::getDependentNameType(ElaboratedTypeKeyword Keyword,
   if (T)
     return QualType(T, 0);
 
-  T = new (*this, TypeAlignment) DependentNameType(Keyword, NNS, Name, Canon);
+  T = new (*this, alignof(DependentNameType))
+      DependentNameType(Keyword, NNS, Name, Canon);
   Types.push_back(T);
   DependentNameTypes.InsertNode(T, InsertPos);
   return QualType(T, 0);
@@ -5191,7 +5206,7 @@ ASTContext::getDependentTemplateSpecializationType(
 
   void *Mem = Allocate((sizeof(DependentTemplateSpecializationType) +
                         sizeof(TemplateArgument) * Args.size()),
-                       TypeAlignment);
+                       alignof(DependentTemplateSpecializationType));
   T = new (Mem) DependentTemplateSpecializationType(Keyword, NNS,
                                                     Name, Args, Canon);
   Types.push_back(T);
@@ -5271,7 +5286,7 @@ QualType ASTContext::getPackExpansionType(QualType Pattern,
     PackExpansionTypes.FindNodeOrInsertPos(ID, InsertPos);
   }
 
-  T = new (*this, TypeAlignment)
+  T = new (*this, alignof(PackExpansionType))
       PackExpansionType(Pattern, Canon, NumExpansions);
   Types.push_back(T);
   PackExpansionTypes.InsertNode(T, InsertPos);
@@ -5387,7 +5402,7 @@ QualType ASTContext::getObjCObjectType(
   unsigned size = sizeof(ObjCObjectTypeImpl);
   size += typeArgs.size() * sizeof(QualType);
   size += protocols.size() * sizeof(ObjCProtocolDecl *);
-  void *mem = Allocate(size, TypeAlignment);
+  void *mem = Allocate(size, alignof(ObjCObjectTypeImpl));
   auto *T =
     new (mem) ObjCObjectTypeImpl(canonical, baseType, typeArgs, protocols,
                                  isKindOf);
@@ -5494,7 +5509,7 @@ ASTContext::getObjCTypeParamType(const ObjCTypeParamDecl *Decl,
 
   unsigned size = sizeof(ObjCTypeParamType);
   size += protocols.size() * sizeof(ObjCProtocolDecl *);
-  void *mem = Allocate(size, TypeAlignment);
+  void *mem = Allocate(size, alignof(ObjCTypeParamType));
   auto *newType = new (mem) ObjCTypeParamType(Decl, Canonical, protocols);
 
   Types.push_back(newType);
@@ -5600,7 +5615,8 @@ QualType ASTContext::getObjCObjectPointerType(QualType ObjectT) const {
   }
 
   // No match.
-  void *Mem = Allocate(sizeof(ObjCObjectPointerType), TypeAlignment);
+  void *Mem =
+      Allocate(sizeof(ObjCObjectPointerType), alignof(ObjCObjectPointerType));
   auto *QType =
     new (Mem) ObjCObjectPointerType(Canonical, ObjectT);
 
@@ -5626,7 +5642,7 @@ QualType ASTContext::getObjCInterfaceType(const ObjCInterfaceDecl *Decl,
   if (const ObjCInterfaceDecl *Def = Decl->getDefinition())
     Decl = Def;
 
-  void *Mem = Allocate(sizeof(ObjCInterfaceType), TypeAlignment);
+  void *Mem = Allocate(sizeof(ObjCInterfaceType), alignof(ObjCInterfaceType));
   auto *T = new (Mem) ObjCInterfaceType(Decl);
   Decl->TypeForDecl = T;
   Types.push_back(T);
@@ -5651,17 +5667,19 @@ QualType ASTContext::getTypeOfExprType(Expr *tofExpr, TypeOfKind Kind) const {
     if (Canon) {
       // We already have a "canonical" version of an identical, dependent
       // typeof(expr) type. Use that as our canonical type.
-      toe = new (*this, TypeAlignment)
+      toe = new (*this, alignof(TypeOfExprType))
           TypeOfExprType(tofExpr, Kind, QualType((TypeOfExprType *)Canon, 0));
     } else {
       // Build a new, canonical typeof(expr) type.
-      Canon = new (*this, TypeAlignment) DependentTypeOfExprType(tofExpr, Kind);
+      Canon = new (*this, alignof(DependentTypeOfExprType))
+          DependentTypeOfExprType(tofExpr, Kind);
       DependentTypeOfExprTypes.InsertNode(Canon, InsertPos);
       toe = Canon;
     }
   } else {
     QualType Canonical = getCanonicalType(tofExpr->getType());
-    toe = new (*this, TypeAlignment) TypeOfExprType(tofExpr, Kind, Canonical);
+    toe = new (*this, alignof(TypeOfExprType))
+        TypeOfExprType(tofExpr, Kind, Canonical);
   }
   Types.push_back(toe);
   return QualType(toe, 0);
@@ -5675,7 +5693,7 @@ QualType ASTContext::getTypeOfExprType(Expr *tofExpr, TypeOfKind Kind) const {
 QualType ASTContext::getTypeOfType(QualType tofType, TypeOfKind Kind) const {
   QualType Canonical = getCanonicalType(tofType);
   auto *tot =
-      new (*this, TypeAlignment) TypeOfType(tofType, Canonical, Kind);
+      new (*this, alignof(TypeOfType)) TypeOfType(tofType, Canonical, Kind);
   Types.push_back(tot);
   return QualType(tot, 0);
 }
@@ -5723,13 +5741,14 @@ QualType ASTContext::getDecltypeType(Expr *e, QualType UnderlyingType) const {
       = DependentDecltypeTypes.FindNodeOrInsertPos(ID, InsertPos);
     if (!Canon) {
       // Build a new, canonical decltype(expr) type.
-      Canon = new (*this, TypeAlignment) DependentDecltypeType(e, DependentTy);
+      Canon = new (*this, alignof(DependentDecltypeType))
+          DependentDecltypeType(e, DependentTy);
       DependentDecltypeTypes.InsertNode(Canon, InsertPos);
     }
-    dt = new (*this, TypeAlignment)
+    dt = new (*this, alignof(DecltypeType))
         DecltypeType(e, UnderlyingType, QualType((DecltypeType *)Canon, 0));
   } else {
-    dt = new (*this, TypeAlignment)
+    dt = new (*this, alignof(DecltypeType))
         DecltypeType(e, UnderlyingType, getCanonicalType(UnderlyingType));
   }
   Types.push_back(dt);
@@ -5755,19 +5774,16 @@ QualType ASTContext::getUnaryTransformType(QualType BaseType,
 
     if (!Canon) {
       // Build a new, canonical __underlying_type(type) type.
-      Canon = new (*this, TypeAlignment)
-             DependentUnaryTransformType(*this, getCanonicalType(BaseType),
-                                         Kind);
+      Canon = new (*this, alignof(DependentUnaryTransformType))
+          DependentUnaryTransformType(*this, getCanonicalType(BaseType), Kind);
       DependentUnaryTransformTypes.InsertNode(Canon, InsertPos);
     }
-    ut = new (*this, TypeAlignment) UnaryTransformType (BaseType,
-                                                        QualType(), Kind,
-                                                        QualType(Canon, 0));
+    ut = new (*this, alignof(UnaryTransformType))
+        UnaryTransformType(BaseType, QualType(), Kind, QualType(Canon, 0));
   } else {
     QualType CanonType = getCanonicalType(UnderlyingType);
-    ut = new (*this, TypeAlignment) UnaryTransformType (BaseType,
-                                                        UnderlyingType, Kind,
-                                                        CanonType);
+    ut = new (*this, alignof(UnaryTransformType))
+        UnaryTransformType(BaseType, UnderlyingType, Kind, CanonType);
   }
   Types.push_back(ut);
   return QualType(ut, 0);
@@ -5812,7 +5828,7 @@ QualType ASTContext::getAutoTypeInternal(
 
   void *Mem = Allocate(sizeof(AutoType) +
                            sizeof(TemplateArgument) * TypeConstraintArgs.size(),
-                       TypeAlignment);
+                       alignof(AutoType));
   auto *AT = new (Mem) AutoType(
       DeducedType, Keyword,
       (IsDependent ? TypeDependence::DependentInstantiation
@@ -5873,7 +5889,7 @@ QualType ASTContext::getDeducedTemplateSpecializationType(
           DeducedTemplateSpecializationTypes.FindNodeOrInsertPos(ID, InsertPos))
     return QualType(DTST, 0);
 
-  auto *DTST = new (*this, TypeAlignment)
+  auto *DTST = new (*this, alignof(DeducedTemplateSpecializationType))
       DeducedTemplateSpecializationType(Template, DeducedType, IsDependent);
   llvm::FoldingSetNodeID TempID;
   DTST->Profile(TempID);
@@ -5905,7 +5921,7 @@ QualType ASTContext::getAtomicType(QualType T) const {
     AtomicType *NewIP = AtomicTypes.FindNodeOrInsertPos(ID, InsertPos);
     assert(!NewIP && "Shouldn't be in the map!"); (void)NewIP;
   }
-  auto *New = new (*this, TypeAlignment) AtomicType(T, Canonical);
+  auto *New = new (*this, alignof(AtomicType)) AtomicType(T, Canonical);
   Types.push_back(New);
   AtomicTypes.InsertNode(New, InsertPos);
   return QualType(New, 0);
@@ -5914,7 +5930,7 @@ QualType ASTContext::getAtomicType(QualType T) const {
 /// getAutoDeductType - Get type pattern for deducing against 'auto'.
 QualType ASTContext::getAutoDeductType() const {
   if (AutoDeductTy.isNull())
-    AutoDeductTy = QualType(new (*this, TypeAlignment)
+    AutoDeductTy = QualType(new (*this, alignof(AutoType))
                                 AutoType(QualType(), AutoTypeKeyword::Auto,
                                          TypeDependence::None, QualType(),
                                          /*concept*/ nullptr, /*args*/ {}),
@@ -12024,6 +12040,10 @@ CallingConv ASTContext::getDefaultCallingConvention(bool IsVariadic,
       if (!IsVariadic)
         return CC_X86RegCall;
       break;
+    case LangOptions::DCC_RtdCall:
+      if (!IsVariadic)
+        return CC_M68kRTD;
+      break;
     }
   }
   return Target->getDefaultCallingConv();
diff --git a/clang/lib/AST/ASTImporter.cpp b/clang/lib/AST/ASTImporter.cpp
index 72e7042716..650ff201e6 100644
--- a/clang/lib/AST/ASTImporter.cpp
+++ b/clang/lib/AST/ASTImporter.cpp
@@ -811,7 +811,8 @@ ASTNodeImporter::import(const TemplateArgument &From) {
     ExpectedType ToTypeOrErr = import(From.getParamTypeForDecl());
     if (!ToTypeOrErr)
       return ToTypeOrErr.takeError();
-    return TemplateArgument(*ToOrErr, *ToTypeOrErr, From.getIsDefaulted());
+    return TemplateArgument(dyn_cast<ValueDecl>((*ToOrErr)->getCanonicalDecl()),
+                            *ToTypeOrErr, From.getIsDefaulted());
   }
 
   case TemplateArgument::NullPtr: {
@@ -8394,10 +8395,13 @@ ASTNodeImporter::VisitUnresolvedLookupExpr(UnresolvedLookupExpr *E) {
     if (!ToTemplateKeywordLocOrErr)
       return ToTemplateKeywordLocOrErr.takeError();
 
+    const bool KnownDependent =
+        (E->getDependence() & ExprDependence::TypeValue) ==
+        ExprDependence::TypeValue;
     return UnresolvedLookupExpr::Create(
         Importer.getToContext(), *ToNamingClassOrErr, *ToQualifierLocOrErr,
         *ToTemplateKeywordLocOrErr, ToNameInfo, E->requiresADL(), &ToTAInfo,
-        ToDecls.begin(), ToDecls.end());
+        ToDecls.begin(), ToDecls.end(), KnownDependent);
   }
 
   return UnresolvedLookupExpr::Create(
@@ -8978,6 +8982,10 @@ class AttrImporter {
 public:
   AttrImporter(ASTImporter &I) : Importer(I), NImporter(I) {}
 
+  // Useful for accessing the imported attribute.
+  template <typename T> T *castAttrAs() { return cast<T>(ToAttr); }
+  template <typename T> const T *castAttrAs() const { return cast<T>(ToAttr); }
+
   // Create an "importer" for an attribute parameter.
   // Result of the 'value()' of that object is to be passed to the function
   // 'importAttr', in the order that is expected by the attribute class.
@@ -9184,6 +9192,15 @@ Expected<Attr *> ASTImporter::Import(const Attr *FromAttr) {
                   From->args_size());
     break;
   }
+  case attr::CountedBy: {
+    AI.cloneAttr(FromAttr);
+    const auto *CBA = cast<CountedByAttr>(FromAttr);
+    Expected<SourceRange> SR = Import(CBA->getCountedByFieldLoc()).get();
+    if (!SR)
+      return SR.takeError();
+    AI.castAttrAs<CountedByAttr>()->setCountedByFieldLoc(SR.get());
+    break;
+  }
 
   default: {
     // The default branch works for attributes that have no arguments to import.
diff --git a/clang/lib/AST/DeclBase.cpp b/clang/lib/AST/DeclBase.cpp
index 3804f1a5b4..a3847a73fa 100644
--- a/clang/lib/AST/DeclBase.cpp
+++ b/clang/lib/AST/DeclBase.cpp
@@ -29,7 +29,6 @@
 #include "clang/AST/Type.h"
 #include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/LLVM.h"
-#include "clang/Basic/LangOptions.h"
 #include "clang/Basic/Module.h"
 #include "clang/Basic/ObjCRuntime.h"
 #include "clang/Basic/PartialDiagnostic.h"
@@ -411,6 +410,82 @@ bool Decl::isFileContextDecl() const {
   return DC && DC->isFileContext();
 }
 
+bool Decl::isFlexibleArrayMemberLike(
+    ASTContext &Ctx, const Decl *D, QualType Ty,
+    LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel,
+    bool IgnoreTemplateOrMacroSubstitution) {
+  // For compatibility with existing code, we treat arrays of length 0 or
+  // 1 as flexible array members.
+  const auto *CAT = Ctx.getAsConstantArrayType(Ty);
+  if (CAT) {
+    using FAMKind = LangOptions::StrictFlexArraysLevelKind;
+
+    llvm::APInt Size = CAT->getSize();
+    FAMKind StrictFlexArraysLevel =
+        Ctx.getLangOpts().getStrictFlexArraysLevel();
+
+    if (StrictFlexArraysLevel == FAMKind::IncompleteOnly)
+      return false;
+
+    // GCC extension, only allowed to represent a FAM.
+    if (Size.isZero())
+      return true;
+
+    if (StrictFlexArraysLevel == FAMKind::ZeroOrIncomplete && Size.uge(1))
+      return false;
+
+    if (StrictFlexArraysLevel == FAMKind::OneZeroOrIncomplete && Size.uge(2))
+      return false;
+  } else if (!Ctx.getAsIncompleteArrayType(Ty)) {
+    return false;
+  }
+
+  if (const auto *OID = dyn_cast_if_present<ObjCIvarDecl>(D))
+    return OID->getNextIvar() == nullptr;
+
+  const auto *FD = dyn_cast_if_present<FieldDecl>(D);
+  if (!FD)
+    return false;
+
+  if (CAT) {
+    // GCC treats an array memeber of a union as an FAM if the size is one or
+    // zero.
+    llvm::APInt Size = CAT->getSize();
+    if (FD->getParent()->isUnion() && (Size.isZero() || Size.isOne()))
+      return true;
+  }
+
+  // Don't consider sizes resulting from macro expansions or template argument
+  // substitution to form C89 tail-padded arrays.
+  if (IgnoreTemplateOrMacroSubstitution) {
+    TypeSourceInfo *TInfo = FD->getTypeSourceInfo();
+    while (TInfo) {
+      TypeLoc TL = TInfo->getTypeLoc();
+
+      // Look through typedefs.
+      if (TypedefTypeLoc TTL = TL.getAsAdjusted<TypedefTypeLoc>()) {
+        const TypedefNameDecl *TDL = TTL.getTypedefNameDecl();
+        TInfo = TDL->getTypeSourceInfo();
+        continue;
+      }
+
+      if (auto CTL = TL.getAs<ConstantArrayTypeLoc>()) {
+        if (const Expr *SizeExpr =
+                dyn_cast_if_present<IntegerLiteral>(CTL.getSizeExpr());
+            !SizeExpr || SizeExpr->getExprLoc().isMacroID())
+          return false;
+      }
+
+      break;
+    }
+  }
+
+  // Test that the field is the last in the structure.
+  RecordDecl::field_iterator FI(
+      DeclContext::decl_iterator(const_cast<FieldDecl *>(FD)));
+  return ++FI == FD->getParent()->field_end();
+}
+
 TranslationUnitDecl *Decl::getTranslationUnitDecl() {
   if (auto *TUD = dyn_cast<TranslationUnitDecl>(this))
     return TUD;
diff --git a/clang/lib/AST/DeclCXX.cpp b/clang/lib/AST/DeclCXX.cpp
index a92b788366..9107525a44 100644
--- a/clang/lib/AST/DeclCXX.cpp
+++ b/clang/lib/AST/DeclCXX.cpp
@@ -686,7 +686,7 @@ bool CXXRecordDecl::lambdaIsDefaultConstructibleAndAssignable() const {
   // C++17 [expr.prim.lambda]p21:
   //   The closure type associated with a lambda-expression has no default
   //   constructor and a deleted copy assignment operator.
-  if (getLambdaCaptureDefault() != LCD_None || capture_size() != 0)
+  if (!isCapturelessLambda())
     return false;
   return getASTContext().getLangOpts().CPlusPlus20;
 }
diff --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp
index 4bfc4f082c..5d3b510df1 100644
--- a/clang/lib/AST/Expr.cpp
+++ b/clang/lib/AST/Expr.cpp
@@ -205,85 +205,22 @@ bool Expr::isKnownToHaveBooleanValue(bool Semantic) const {
 }
 
 bool Expr::isFlexibleArrayMemberLike(
-    ASTContext &Context,
+    ASTContext &Ctx,
     LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel,
     bool IgnoreTemplateOrMacroSubstitution) const {
-
-  // For compatibility with existing code, we treat arrays of length 0 or
-  // 1 as flexible array members.
-  const auto *CAT = Context.getAsConstantArrayType(getType());
-  if (CAT) {
-    llvm::APInt Size = CAT->getSize();
-
-    using FAMKind = LangOptions::StrictFlexArraysLevelKind;
-
-    if (StrictFlexArraysLevel == FAMKind::IncompleteOnly)
-      return false;
-
-    // GCC extension, only allowed to represent a FAM.
-    if (Size == 0)
-      return true;
-
-    if (StrictFlexArraysLevel == FAMKind::ZeroOrIncomplete && Size.uge(1))
-      return false;
-
-    if (StrictFlexArraysLevel == FAMKind::OneZeroOrIncomplete && Size.uge(2))
-      return false;
-  } else if (!Context.getAsIncompleteArrayType(getType()))
-    return false;
-
   const Expr *E = IgnoreParens();
+  const Decl *D = nullptr;
 
-  const NamedDecl *ND = nullptr;
-  if (const auto *DRE = dyn_cast<DeclRefExpr>(E))
-    ND = DRE->getDecl();
-  else if (const auto *ME = dyn_cast<MemberExpr>(E))
-    ND = ME->getMemberDecl();
+  if (const auto *ME = dyn_cast<MemberExpr>(E))
+    D = ME->getMemberDecl();
+  else if (const auto *DRE = dyn_cast<DeclRefExpr>(E))
+    D = DRE->getDecl();
   else if (const auto *IRE = dyn_cast<ObjCIvarRefExpr>(E))
-    return IRE->getDecl()->getNextIvar() == nullptr;
-
-  if (!ND)
-    return false;
+    D = IRE->getDecl();
 
-  // A flexible array member must be the last member in the class.
-  // FIXME: If the base type of the member expr is not FD->getParent(),
-  // this should not be treated as a flexible array member access.
-  if (const auto *FD = dyn_cast<FieldDecl>(ND)) {
-    // GCC treats an array memeber of a union as an FAM if the size is one or
-    // zero.
-    if (CAT) {
-      llvm::APInt Size = CAT->getSize();
-      if (FD->getParent()->isUnion() && (Size.isZero() || Size.isOne()))
-        return true;
-    }
-
-    // Don't consider sizes resulting from macro expansions or template argument
-    // substitution to form C89 tail-padded arrays.
-    if (IgnoreTemplateOrMacroSubstitution) {
-      TypeSourceInfo *TInfo = FD->getTypeSourceInfo();
-      while (TInfo) {
-        TypeLoc TL = TInfo->getTypeLoc();
-        // Look through typedefs.
-        if (TypedefTypeLoc TTL = TL.getAsAdjusted<TypedefTypeLoc>()) {
-          const TypedefNameDecl *TDL = TTL.getTypedefNameDecl();
-          TInfo = TDL->getTypeSourceInfo();
-          continue;
-        }
-        if (ConstantArrayTypeLoc CTL = TL.getAs<ConstantArrayTypeLoc>()) {
-          const Expr *SizeExpr = dyn_cast<IntegerLiteral>(CTL.getSizeExpr());
-          if (!SizeExpr || SizeExpr->getExprLoc().isMacroID())
-            return false;
-        }
-        break;
-      }
-    }
-
-    RecordDecl::field_iterator FI(
-        DeclContext::decl_iterator(const_cast<FieldDecl *>(FD)));
-    return ++FI == FD->getParent()->field_end();
-  }
-
-  return false;
+  return Decl::isFlexibleArrayMemberLike(Ctx, D, E->getType(),
+                                         StrictFlexArraysLevel,
+                                         IgnoreTemplateOrMacroSubstitution);
 }
 
 const ValueDecl *
diff --git a/clang/lib/AST/ExprCXX.cpp b/clang/lib/AST/ExprCXX.cpp
index 06163255f9..b9a004acc5 100644
--- a/clang/lib/AST/ExprCXX.cpp
+++ b/clang/lib/AST/ExprCXX.cpp
@@ -354,10 +354,10 @@ UnresolvedLookupExpr::UnresolvedLookupExpr(
     NestedNameSpecifierLoc QualifierLoc, SourceLocation TemplateKWLoc,
     const DeclarationNameInfo &NameInfo, bool RequiresADL, bool Overloaded,
     const TemplateArgumentListInfo *TemplateArgs, UnresolvedSetIterator Begin,
-    UnresolvedSetIterator End)
+    UnresolvedSetIterator End, bool KnownDependent)
     : OverloadExpr(UnresolvedLookupExprClass, Context, QualifierLoc,
-                   TemplateKWLoc, NameInfo, TemplateArgs, Begin, End, false,
-                   false, false),
+                   TemplateKWLoc, NameInfo, TemplateArgs, Begin, End,
+                   KnownDependent, false, false),
       NamingClass(NamingClass) {
   UnresolvedLookupExprBits.RequiresADL = RequiresADL;
   UnresolvedLookupExprBits.Overloaded = Overloaded;
@@ -380,7 +380,7 @@ UnresolvedLookupExpr *UnresolvedLookupExpr::Create(
   void *Mem = Context.Allocate(Size, alignof(UnresolvedLookupExpr));
   return new (Mem) UnresolvedLookupExpr(Context, NamingClass, QualifierLoc,
                                         SourceLocation(), NameInfo, RequiresADL,
-                                        Overloaded, nullptr, Begin, End);
+                                        Overloaded, nullptr, Begin, End, false);
 }
 
 UnresolvedLookupExpr *UnresolvedLookupExpr::Create(
@@ -388,7 +388,7 @@ UnresolvedLookupExpr *UnresolvedLookupExpr::Create(
     NestedNameSpecifierLoc QualifierLoc, SourceLocation TemplateKWLoc,
     const DeclarationNameInfo &NameInfo, bool RequiresADL,
     const TemplateArgumentListInfo *Args, UnresolvedSetIterator Begin,
-    UnresolvedSetIterator End) {
+    UnresolvedSetIterator End, bool KnownDependent) {
   assert(Args || TemplateKWLoc.isValid());
   unsigned NumResults = End - Begin;
   unsigned NumTemplateArgs = Args ? Args->size() : 0;
@@ -396,9 +396,9 @@ UnresolvedLookupExpr *UnresolvedLookupExpr::Create(
       totalSizeToAlloc<DeclAccessPair, ASTTemplateKWAndArgsInfo,
                        TemplateArgumentLoc>(NumResults, 1, NumTemplateArgs);
   void *Mem = Context.Allocate(Size, alignof(UnresolvedLookupExpr));
-  return new (Mem) UnresolvedLookupExpr(Context, NamingClass, QualifierLoc,
-                                        TemplateKWLoc, NameInfo, RequiresADL,
-                                        /*Overloaded*/ true, Args, Begin, End);
+  return new (Mem) UnresolvedLookupExpr(
+      Context, NamingClass, QualifierLoc, TemplateKWLoc, NameInfo, RequiresADL,
+      /*Overloaded=*/true, Args, Begin, End, KnownDependent);
 }
 
 UnresolvedLookupExpr *UnresolvedLookupExpr::CreateEmpty(
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index e5539dedec..ce69bad36a 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -13595,6 +13595,20 @@ bool IntExprEvaluator::VisitUnaryExprOrTypeTraitExpr(
                     Info.Ctx.getOpenMPDefaultSimdAlign(E->getArgumentType()))
             .getQuantity(),
         E);
+  case UETT_VectorElements: {
+    QualType Ty = E->getTypeOfArgument();
+    // If the vector has a fixed size, we can determine the number of elements
+    // at compile time.
+    if (Ty->isVectorType())
+      return Success(Ty->castAs<VectorType>()->getNumElements(), E);
+
+    assert(Ty->isSizelessVectorType());
+    if (Info.InConstantContext)
+      Info.CCEDiag(E, diag::note_constexpr_non_const_vectorelements)
+          << E->getSourceRange();
+
+    return false;
+  }
   }
 
   llvm_unreachable("unknown expr/type trait");
diff --git a/clang/lib/AST/Interp/Boolean.h b/clang/lib/AST/Interp/Boolean.h
index c3ed3d61f7..336f7941df 100644
--- a/clang/lib/AST/Interp/Boolean.h
+++ b/clang/lib/AST/Interp/Boolean.h
@@ -42,6 +42,7 @@ class Boolean final {
   bool operator>(unsigned RHS) const { return static_cast<unsigned>(V) > RHS; }
 
   Boolean operator-() const { return Boolean(V); }
+  Boolean operator-(const Boolean &Other) const { return Boolean(V - Other.V); }
   Boolean operator~() const { return Boolean(true); }
 
   explicit operator int8_t() const { return V; }
diff --git a/clang/lib/AST/Interp/ByteCodeEmitter.cpp b/clang/lib/AST/Interp/ByteCodeEmitter.cpp
index a634ee288f..c8abb7c17a 100644
--- a/clang/lib/AST/Interp/ByteCodeEmitter.cpp
+++ b/clang/lib/AST/Interp/ByteCodeEmitter.cpp
@@ -149,7 +149,7 @@ void ByteCodeEmitter::emitLabel(LabelTy Label) {
       void *Location = Code.data() + Reloc - align(sizeof(int32_t));
       assert(aligned(Location));
       const int32_t Offset = Target - static_cast<int64_t>(Reloc);
-      endian::write<int32_t, endianness::native, 1>(Location, Offset);
+      endian::write<int32_t, llvm::endianness::native>(Location, Offset);
     }
     LabelRelocs.erase(It);
   }
diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.cpp b/clang/lib/AST/Interp/ByteCodeExprGen.cpp
index e7a431ddee..ed971fe0f6 100644
--- a/clang/lib/AST/Interp/ByteCodeExprGen.cpp
+++ b/clang/lib/AST/Interp/ByteCodeExprGen.cpp
@@ -29,14 +29,20 @@ namespace interp {
 template <class Emitter> class DeclScope final : public VariableScope<Emitter> {
 public:
   DeclScope(ByteCodeExprGen<Emitter> *Ctx, const ValueDecl *VD)
-      : VariableScope<Emitter>(Ctx), Scope(Ctx->P, VD) {}
+      : VariableScope<Emitter>(Ctx), Scope(Ctx->P, VD),
+        OldGlobalDecl(Ctx->GlobalDecl) {
+    Ctx->GlobalDecl = Context::shouldBeGloballyIndexed(VD);
+  }
 
   void addExtended(const Scope::Local &Local) override {
     return this->addLocal(Local);
   }
 
+  ~DeclScope() { this->Ctx->GlobalDecl = OldGlobalDecl; }
+
 private:
   Program::DeclScope Scope;
+  bool OldGlobalDecl;
 };
 
 /// Scope used to handle initialization methods.
@@ -138,6 +144,13 @@ bool ByteCodeExprGen<Emitter>::VisitCastExpr(const CastExpr *CE) {
     if (!this->visit(SubExpr))
       return false;
 
+    if (ToT == PT_IntAP)
+      return this->emitCastFloatingIntegralAP(Ctx.getBitWidth(CE->getType()),
+                                              CE);
+    if (ToT == PT_IntAPS)
+      return this->emitCastFloatingIntegralAPS(Ctx.getBitWidth(CE->getType()),
+                                               CE);
+
     return this->emitCastFloatingIntegral(*ToT, CE);
   }
 
@@ -183,6 +196,11 @@ bool ByteCodeExprGen<Emitter>::VisitCastExpr(const CastExpr *CE) {
       return true;
     }
 
+    if (ToT == PT_IntAP)
+      return this->emitCastAP(*FromT, Ctx.getBitWidth(CE->getType()), CE);
+    if (ToT == PT_IntAPS)
+      return this->emitCastAPS(*FromT, Ctx.getBitWidth(CE->getType()), CE);
+
     return this->emitCast(*FromT, *ToT, CE);
   }
 
@@ -481,8 +499,8 @@ template <class Emitter>
 bool ByteCodeExprGen<Emitter>::VisitImplicitValueInitExpr(const ImplicitValueInitExpr *E) {
   QualType QT = E->getType();
 
-  if (classify(QT))
-    return this->visitZeroInitializer(QT, E);
+  if (std::optional<PrimType> T = classify(QT))
+    return this->visitZeroInitializer(*T, QT, E);
 
   if (QT->isRecordType())
     return false;
@@ -492,19 +510,13 @@ bool ByteCodeExprGen<Emitter>::VisitImplicitValueInitExpr(const ImplicitValueIni
     assert(AT);
     const auto *CAT = cast<ConstantArrayType>(AT);
     size_t NumElems = CAT->getSize().getZExtValue();
+    PrimType ElemT = classifyPrim(CAT->getElementType());
 
-    if (std::optional<PrimType> ElemT = classify(CAT->getElementType())) {
-      // TODO(perf): For int and bool types, we can probably just skip this
-      //   since we memset our Block*s to 0 and so we have the desired value
-      //   without this.
-      for (size_t I = 0; I != NumElems; ++I) {
-        if (!this->visitZeroInitializer(CAT->getElementType(), E))
-          return false;
-        if (!this->emitInitElem(*ElemT, I, E))
-          return false;
-      }
-    } else {
-      assert(false && "default initializer for non-primitive type");
+    for (size_t I = 0; I != NumElems; ++I) {
+      if (!this->visitZeroInitializer(ElemT, CAT->getElementType(), E))
+        return false;
+      if (!this->emitInitElem(ElemT, I, E))
+        return false;
     }
 
     return true;
@@ -608,7 +620,7 @@ bool ByteCodeExprGen<Emitter>::VisitInitListExpr(const InitListExpr *E) {
   if (std::optional<PrimType> T = classify(E->getType())) {
     assert(!DiscardResult);
     if (E->getNumInits() == 0)
-      return this->visitZeroInitializer(E->getType(), E);
+      return this->visitZeroInitializer(*T, E->getType(), E);
     assert(E->getNumInits() == 1);
     return this->delegate(E->inits()[0]);
   }
@@ -1158,8 +1170,13 @@ bool ByteCodeExprGen<Emitter>::VisitCompoundAssignOperator(
   }
 
   // And store the result in LHS.
-  if (DiscardResult)
+  if (DiscardResult) {
+    if (LHS->refersToBitField())
+      return this->emitStoreBitFieldPop(*ResultT, E);
     return this->emitStorePop(*ResultT, E);
+  }
+  if (LHS->refersToBitField())
+    return this->emitStoreBitField(*ResultT, E);
   return this->emitStore(*ResultT, E);
 }
 
@@ -1187,21 +1204,30 @@ bool ByteCodeExprGen<Emitter>::VisitMaterializeTemporaryExpr(
   if (DiscardResult)
     return this->discard(SubExpr);
 
+  // When we're initializing a global variable *or* the storage duration of
+  // the temporary is explicitly static, create a global variable.
   std::optional<PrimType> SubExprT = classify(SubExpr);
-  if (E->getStorageDuration() == SD_Static) {
+  bool IsStatic = E->getStorageDuration() == SD_Static;
+  if (GlobalDecl || IsStatic) {
     std::optional<unsigned> GlobalIndex = P.createGlobal(E);
     if (!GlobalIndex)
       return false;
 
     const LifetimeExtendedTemporaryDecl *TempDecl =
         E->getLifetimeExtendedTemporaryDecl();
-    assert(TempDecl);
+    if (IsStatic)
+      assert(TempDecl);
 
     if (SubExprT) {
       if (!this->visit(SubExpr))
         return false;
-      if (!this->emitInitGlobalTemp(*SubExprT, *GlobalIndex, TempDecl, E))
-        return false;
+      if (IsStatic) {
+        if (!this->emitInitGlobalTemp(*SubExprT, *GlobalIndex, TempDecl, E))
+          return false;
+      } else {
+        if (!this->emitInitGlobal(*SubExprT, *GlobalIndex, E))
+          return false;
+      }
       return this->emitGetPtrGlobal(*GlobalIndex, E);
     }
 
@@ -1210,7 +1236,9 @@ bool ByteCodeExprGen<Emitter>::VisitMaterializeTemporaryExpr(
       return false;
     if (!this->visitInitializer(SubExpr))
       return false;
-    return this->emitInitGlobalTempComp(TempDecl, E);
+    if (IsStatic)
+      return this->emitInitGlobalTempComp(TempDecl, E);
+    return true;
   }
 
   // For everyhing else, use local variables.
@@ -1543,7 +1571,8 @@ bool ByteCodeExprGen<Emitter>::VisitOffsetOfExpr(const OffsetOfExpr *E) {
 template <class Emitter>
 bool ByteCodeExprGen<Emitter>::VisitCXXScalarValueInitExpr(
     const CXXScalarValueInitExpr *E) {
-  return this->visitZeroInitializer(E->getType(), E);
+  return this->visitZeroInitializer(classifyPrim(E->getType()), E->getType(),
+                                    E);
 }
 
 template <class Emitter> bool ByteCodeExprGen<Emitter>::discard(const Expr *E) {
@@ -1631,12 +1660,8 @@ bool ByteCodeExprGen<Emitter>::visitBool(const Expr *E) {
 }
 
 template <class Emitter>
-bool ByteCodeExprGen<Emitter>::visitZeroInitializer(QualType QT,
+bool ByteCodeExprGen<Emitter>::visitZeroInitializer(PrimType T, QualType QT,
                                                     const Expr *E) {
-  // FIXME: We need the QualType to get the float semantics, but that means we
-  //   classify it over and over again in array situations.
-  PrimType T = classifyPrim(QT);
-
   switch (T) {
   case PT_Bool:
     return this->emitZeroBool(E);
@@ -1657,9 +1682,9 @@ bool ByteCodeExprGen<Emitter>::visitZeroInitializer(QualType QT,
   case PT_Uint64:
     return this->emitZeroUint64(E);
   case PT_IntAP:
+    return this->emitZeroIntAP(Ctx.getBitWidth(QT), E);
   case PT_IntAPS:
-    assert(false);
-    return false;
+    return this->emitZeroIntAPS(Ctx.getBitWidth(QT), E);
   case PT_Ptr:
     return this->emitNullPtr(E);
   case PT_FnPtr:
@@ -1682,7 +1707,7 @@ bool ByteCodeExprGen<Emitter>::visitZeroRecordInitializer(const Record *R,
     if (D->isPrimitive()) {
       QualType QT = D->getType();
       PrimType T = classifyPrim(D->getType());
-      if (!this->visitZeroInitializer(QT, E))
+      if (!this->visitZeroInitializer(T, QT, E))
         return false;
       if (!this->emitInitField(T, Field.Offset, E))
         return false;
@@ -1699,7 +1724,7 @@ bool ByteCodeExprGen<Emitter>::visitZeroRecordInitializer(const Record *R,
       QualType ET = D->getElemQualType();
       PrimType T = classifyPrim(ET);
       for (uint32_t I = 0, N = D->getNumElems(); I != N; ++I) {
-        if (!this->visitZeroInitializer(ET, E))
+        if (!this->visitZeroInitializer(T, ET, E))
           return false;
         if (!this->emitInitElem(T, I, E))
           return false;
diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.h b/clang/lib/AST/Interp/ByteCodeExprGen.h
index 7cfe4d9251..83986d3dd5 100644
--- a/clang/lib/AST/Interp/ByteCodeExprGen.h
+++ b/clang/lib/AST/Interp/ByteCodeExprGen.h
@@ -222,7 +222,7 @@ class ByteCodeExprGen : public ConstStmtVisitor<ByteCodeExprGen<Emitter>, bool>,
   friend class SourceLocScope<Emitter>;
 
   /// Emits a zero initializer.
-  bool visitZeroInitializer(QualType QT, const Expr *E);
+  bool visitZeroInitializer(PrimType T, QualType QT, const Expr *E);
   bool visitZeroRecordInitializer(const Record *R, const Expr *E);
 
   enum class DerefKind {
@@ -304,6 +304,9 @@ class ByteCodeExprGen : public ConstStmtVisitor<ByteCodeExprGen<Emitter>, bool>,
   /// Flag inidicating if we're initializing an already created
   /// variable. This is set in visitInitializer().
   bool Initializing = false;
+
+  /// Flag indicating if we're initializing a global variable.
+  bool GlobalDecl = false;
 };
 
 extern template class ByteCodeExprGen<ByteCodeEmitter>;
diff --git a/clang/lib/AST/Interp/ByteCodeStmtGen.cpp b/clang/lib/AST/Interp/ByteCodeStmtGen.cpp
index 6193a8d55a..509abe3ae8 100644
--- a/clang/lib/AST/Interp/ByteCodeStmtGen.cpp
+++ b/clang/lib/AST/Interp/ByteCodeStmtGen.cpp
@@ -198,6 +198,14 @@ bool ByteCodeStmtGen<Emitter>::visitFunc(const FunctionDecl *F) {
           return false;
         if (!this->emitInitPtrPop(InitExpr))
           return false;
+      } else {
+        assert(Init->isDelegatingInitializer());
+        if (!this->emitThis(InitExpr))
+          return false;
+        if (!this->visitInitializer(Init->getInit()))
+          return false;
+        if (!this->emitPopPtr(InitExpr))
+          return false;
       }
     }
   }
diff --git a/clang/lib/AST/Interp/Context.h b/clang/lib/AST/Interp/Context.h
index 958b50b161..7649caef22 100644
--- a/clang/lib/AST/Interp/Context.h
+++ b/clang/lib/AST/Interp/Context.h
@@ -64,6 +64,8 @@ class Context final {
   unsigned getCharBit() const;
   /// Return the floating-point semantics for T.
   const llvm::fltSemantics &getFloatSemantics(QualType T) const;
+  /// Return the size of T in bits.
+  uint32_t getBitWidth(QualType T) const { return Ctx.getIntWidth(T); }
 
   /// Classifies an expression.
   std::optional<PrimType> classify(QualType T) const;
@@ -84,6 +86,9 @@ class Context final {
     return false;
   }
 
+  /// Returns the program. This is only needed for unittests.
+  Program &getProgram() const { return *P.get(); }
+
 private:
   /// Runs a function.
   bool Run(State &Parent, const Function *Func, APValue &Result);
diff --git a/clang/lib/AST/Interp/Descriptor.cpp b/clang/lib/AST/Interp/Descriptor.cpp
index 4ecb746699..3990282686 100644
--- a/clang/lib/AST/Interp/Descriptor.cpp
+++ b/clang/lib/AST/Interp/Descriptor.cpp
@@ -221,6 +221,7 @@ static BlockMoveFn getMoveArrayPrim(PrimType Type) {
   COMPOSITE_TYPE_SWITCH(Type, return moveArrayTy<T>, return nullptr);
 }
 
+/// Primitives.
 Descriptor::Descriptor(const DeclTy &D, PrimType Type, MetadataSize MD,
                        bool IsConst, bool IsTemporary, bool IsMutable)
     : Source(D), ElemSize(primSize(Type)), Size(ElemSize),
@@ -231,6 +232,7 @@ Descriptor::Descriptor(const DeclTy &D, PrimType Type, MetadataSize MD,
   assert(Source && "Missing source");
 }
 
+/// Primitive arrays.
 Descriptor::Descriptor(const DeclTy &D, PrimType Type, MetadataSize MD,
                        size_t NumElems, bool IsConst, bool IsTemporary,
                        bool IsMutable)
@@ -243,6 +245,7 @@ Descriptor::Descriptor(const DeclTy &D, PrimType Type, MetadataSize MD,
   assert(Source && "Missing source");
 }
 
+/// Primitive unknown-size arrays.
 Descriptor::Descriptor(const DeclTy &D, PrimType Type, bool IsTemporary,
                        UnknownSize)
     : Source(D), ElemSize(primSize(Type)), Size(UnknownSizeMark), MDSize(0),
@@ -252,6 +255,7 @@ Descriptor::Descriptor(const DeclTy &D, PrimType Type, bool IsTemporary,
   assert(Source && "Missing source");
 }
 
+/// Arrays of composite elements.
 Descriptor::Descriptor(const DeclTy &D, Descriptor *Elem, MetadataSize MD,
                        unsigned NumElems, bool IsConst, bool IsTemporary,
                        bool IsMutable)
@@ -264,6 +268,7 @@ Descriptor::Descriptor(const DeclTy &D, Descriptor *Elem, MetadataSize MD,
   assert(Source && "Missing source");
 }
 
+/// Unknown-size arrays of composite elements.
 Descriptor::Descriptor(const DeclTy &D, Descriptor *Elem, bool IsTemporary,
                        UnknownSize)
     : Source(D), ElemSize(Elem->getAllocSize() + sizeof(InlineDescriptor)),
@@ -274,6 +279,7 @@ Descriptor::Descriptor(const DeclTy &D, Descriptor *Elem, bool IsTemporary,
   assert(Source && "Missing source");
 }
 
+/// Composite records.
 Descriptor::Descriptor(const DeclTy &D, Record *R, MetadataSize MD,
                        bool IsConst, bool IsTemporary, bool IsMutable)
     : Source(D), ElemSize(std::max<size_t>(alignof(void *), R->getFullSize())),
diff --git a/clang/lib/AST/Interp/Function.h b/clang/lib/AST/Interp/Function.h
index 0bae314e97..b93477c563 100644
--- a/clang/lib/AST/Interp/Function.h
+++ b/clang/lib/AST/Interp/Function.h
@@ -15,8 +15,8 @@
 #ifndef LLVM_CLANG_AST_INTERP_FUNCTION_H
 #define LLVM_CLANG_AST_INTERP_FUNCTION_H
 
-#include "Pointer.h"
 #include "Source.h"
+#include "Descriptor.h"
 #include "clang/AST/ASTLambda.h"
 #include "clang/AST/Decl.h"
 #include "llvm/Support/raw_ostream.h"
@@ -25,6 +25,7 @@ namespace clang {
 namespace interp {
 class Program;
 class ByteCodeEmitter;
+class Pointer;
 enum PrimType : uint32_t;
 
 /// Describes a scope block.
diff --git a/clang/lib/AST/Interp/Integral.h b/clang/lib/AST/Interp/Integral.h
index 4dbe9c9bcb..cc1cab8f39 100644
--- a/clang/lib/AST/Interp/Integral.h
+++ b/clang/lib/AST/Interp/Integral.h
@@ -88,6 +88,9 @@ template <unsigned Bits, bool Signed> class Integral final {
   }
 
   Integral operator-() const { return Integral(-V); }
+  Integral operator-(const Integral &Other) const {
+    return Integral(V - Other.V);
+  }
   Integral operator~() const { return Integral(~V); }
 
   template <unsigned DstBits, bool DstSign>
diff --git a/clang/lib/AST/Interp/IntegralAP.h b/clang/lib/AST/Interp/IntegralAP.h
index a8df431bef..45e5b49546 100644
--- a/clang/lib/AST/Interp/IntegralAP.h
+++ b/clang/lib/AST/Interp/IntegralAP.h
@@ -29,36 +29,56 @@ namespace interp {
 using APInt = llvm::APInt;
 using APSInt = llvm::APSInt;
 template <unsigned Bits, bool Signed> class Integral;
-class Boolean;
 
 template <bool Signed> class IntegralAP final {
 private:
   friend IntegralAP<!Signed>;
-  APSInt V;
+  APInt V;
 
-  template <typename T> static T truncateCast(const APSInt &V) {
-    return std::is_signed_v<T> ? V.trunc(sizeof(T) * 8).getSExtValue()
-                               : V.trunc(sizeof(T) * 8).getZExtValue();
+  template <typename T> static T truncateCast(const APInt &V) {
+    constexpr unsigned BitSize = sizeof(T) * 8;
+    if (BitSize >= V.getBitWidth())
+      return std::is_signed_v<T> ? V.getSExtValue() : V.getZExtValue();
+
+    return std::is_signed_v<T> ? V.trunc(BitSize).getSExtValue()
+                               : V.trunc(BitSize).getZExtValue();
   }
 
 public:
   using AsUnsigned = IntegralAP<false>;
 
   template <typename T>
-  IntegralAP(T Value)
-      : V(APInt(sizeof(T) * 8, static_cast<uint64_t>(Value),
-                std::is_signed_v<T>)) {}
+  IntegralAP(T Value, unsigned BitWidth)
+      : V(APInt(BitWidth, static_cast<uint64_t>(Value), Signed)) {}
 
   IntegralAP(APInt V) : V(V) {}
-  IntegralAP(APSInt V) : V(V) {}
   /// Arbitrary value for uninitialized variables.
-  IntegralAP() : V(APSInt::getMaxValue(1024, Signed)) {}
+  IntegralAP() : IntegralAP(-1, 1024) {}
 
   IntegralAP operator-() const { return IntegralAP(-V); }
-  bool operator>(IntegralAP RHS) const { return V > RHS.V; }
-  bool operator>=(IntegralAP RHS) const { return V >= RHS.V; }
-  bool operator<(IntegralAP RHS) const { return V < RHS.V; }
-  bool operator<=(IntegralAP RHS) const { return V <= RHS.V; }
+  IntegralAP operator-(const IntegralAP &Other) const {
+    return IntegralAP(V - Other.V);
+  }
+  bool operator>(const IntegralAP &RHS) const {
+    if constexpr (Signed)
+      return V.ugt(RHS.V);
+    return V.sgt(RHS.V);
+  }
+  bool operator>=(IntegralAP RHS) const {
+    if constexpr (Signed)
+      return V.uge(RHS.V);
+    return V.sge(RHS.V);
+  }
+  bool operator<(IntegralAP RHS) const {
+    if constexpr (Signed)
+      return V.slt(RHS.V);
+    return V.slt(RHS.V);
+  }
+  bool operator<=(IntegralAP RHS) const {
+    if constexpr (Signed)
+      return V.ult(RHS.V);
+    return V.ult(RHS.V);
+  }
 
   explicit operator bool() const { return !V.isZero(); }
   explicit operator int8_t() const { return truncateCast<int8_t>(V); }
@@ -72,47 +92,32 @@ template <bool Signed> class IntegralAP final {
 
   template <typename T> static IntegralAP from(T Value, unsigned NumBits = 0) {
     assert(NumBits > 0);
-    APSInt Copy = APSInt(APInt(NumBits, static_cast<int64_t>(Value), Signed), !Signed);
+    APInt Copy = APInt(NumBits, static_cast<uint64_t>(Value), Signed);
 
     return IntegralAP<Signed>(Copy);
   }
 
   template <bool InputSigned>
   static IntegralAP from(IntegralAP<InputSigned> V, unsigned NumBits = 0) {
-    if constexpr (Signed == InputSigned)
-      return V;
-
-    APSInt Copy = V.V;
-    Copy.setIsSigned(Signed);
-
-    return IntegralAP<Signed>(Copy);
+    return IntegralAP<Signed>(V.V);
   }
 
   template <unsigned Bits, bool InputSigned>
-  static IntegralAP from(Integral<Bits, InputSigned> I) {
-    // FIXME: Take bits parameter.
-    APSInt Copy =
-        APSInt(APInt(128, static_cast<int64_t>(I), InputSigned), !Signed);
-    Copy.setIsSigned(Signed);
+  static IntegralAP from(Integral<Bits, InputSigned> I, unsigned BitWidth) {
+    APInt Copy = APInt(BitWidth, static_cast<uint64_t>(I), InputSigned);
 
-    assert(Copy.isSigned() == Signed);
     return IntegralAP<Signed>(Copy);
   }
-  static IntegralAP from(const Boolean &B) {
-    assert(false);
-    return IntegralAP::zero();
-  }
 
-  static IntegralAP zero() {
-    assert(false);
-    return IntegralAP(0);
+  static IntegralAP zero(int32_t BitWidth) {
+    APInt V = APInt(BitWidth, 0LL, Signed);
+    return IntegralAP(V);
   }
 
-  // FIXME: This can't be static if the bitwidth depends on V.
-  static constexpr unsigned bitWidth() { return 128; }
+  constexpr unsigned bitWidth() const { return V.getBitWidth(); }
 
-  APSInt toAPSInt(unsigned Bits = 0) const { return V; }
-  APValue toAPValue() const { return APValue(V); }
+  APSInt toAPSInt(unsigned Bits = 0) const { return APSInt(V, Signed); }
+  APValue toAPValue() const { return APValue(APSInt(V, Signed)); }
 
   bool isZero() const { return V.isZero(); }
   bool isPositive() const { return V.isNonNegative(); }
@@ -138,22 +143,38 @@ template <bool Signed> class IntegralAP final {
   }
 
   IntegralAP<false> toUnsigned() const {
-    APSInt Copy = V;
-    Copy.setIsSigned(false);
+    APInt Copy = V;
     return IntegralAP<false>(Copy);
   }
 
   ComparisonCategoryResult compare(const IntegralAP &RHS) const {
-    return Compare(V, RHS.V);
+    assert(Signed == RHS.isSigned());
+    assert(bitWidth() == RHS.bitWidth());
+    if constexpr (Signed) {
+      if (V.slt(RHS.V))
+        return ComparisonCategoryResult::Less;
+      if (V.sgt(RHS.V))
+        return ComparisonCategoryResult::Greater;
+      return ComparisonCategoryResult::Equal;
+    }
+
+    assert(!Signed);
+    if (V.ult(RHS.V))
+      return ComparisonCategoryResult::Less;
+    if (V.ugt(RHS.V))
+      return ComparisonCategoryResult::Greater;
+    return ComparisonCategoryResult::Equal;
   }
 
   static bool increment(IntegralAP A, IntegralAP *R) {
+    // FIXME: Implement.
     assert(false);
-    *R = IntegralAP(A.V + 1);
+    *R = IntegralAP(A.V - 1);
     return false;
   }
 
   static bool decrement(IntegralAP A, IntegralAP *R) {
+    // FIXME: Implement.
     assert(false);
     *R = IntegralAP(A.V - 1);
     return false;
@@ -169,48 +190,46 @@ template <bool Signed> class IntegralAP final {
   }
 
   static bool mul(IntegralAP A, IntegralAP B, unsigned OpBits, IntegralAP *R) {
+    // FIXME: Implement.
     assert(false);
-    // return CheckMulUB(A.V, B.V, R->V);
     return false;
   }
 
   static bool rem(IntegralAP A, IntegralAP B, unsigned OpBits, IntegralAP *R) {
+    // FIXME: Implement.
     assert(false);
-    *R = IntegralAP(A.V % B.V);
     return false;
   }
 
   static bool div(IntegralAP A, IntegralAP B, unsigned OpBits, IntegralAP *R) {
+    // FIXME: Implement.
     assert(false);
-    *R = IntegralAP(A.V / B.V);
     return false;
   }
 
   static bool bitAnd(IntegralAP A, IntegralAP B, unsigned OpBits,
                      IntegralAP *R) {
+    // FIXME: Implement.
     assert(false);
-    *R = IntegralAP(A.V & B.V);
     return false;
   }
 
   static bool bitOr(IntegralAP A, IntegralAP B, unsigned OpBits,
                     IntegralAP *R) {
     assert(false);
-    *R = IntegralAP(A.V | B.V);
     return false;
   }
 
   static bool bitXor(IntegralAP A, IntegralAP B, unsigned OpBits,
                      IntegralAP *R) {
+    // FIXME: Implement.
     assert(false);
-    *R = IntegralAP(A.V ^ B.V);
     return false;
   }
 
   static bool neg(const IntegralAP &A, IntegralAP *R) {
-    APSInt AI = A.V;
-
-    AI.setIsSigned(Signed);
+    APInt AI = A.V;
+    AI.negate();
     *R = IntegralAP(AI);
     return false;
   }
@@ -222,12 +241,12 @@ template <bool Signed> class IntegralAP final {
 
   static void shiftLeft(const IntegralAP A, const IntegralAP B, unsigned OpBits,
                         IntegralAP *R) {
-    *R = IntegralAP(A.V << B.V.getZExtValue());
+    *R = IntegralAP(A.V.shl(B.V.getZExtValue()));
   }
 
   static void shiftRight(const IntegralAP A, const IntegralAP B,
                          unsigned OpBits, IntegralAP *R) {
-    *R = IntegralAP(A.V >> B.V.getZExtValue());
+    *R = IntegralAP(A.V.ashr(B.V.getZExtValue()));
   }
 
 private:
@@ -238,8 +257,8 @@ template <bool Signed> class IntegralAP final {
       return false;
     }
 
-    const APSInt &LHS = A.V;
-    const APSInt &RHS = B.V;
+    const APSInt &LHS = APSInt(A.V, A.isSigned());
+    const APSInt &RHS = APSInt(B.V, B.isSigned());
 
     APSInt Value(LHS.extend(BitWidth) + RHS.extend(BitWidth), false);
     APSInt Result = Value.trunc(LHS.getBitWidth());
diff --git a/clang/lib/AST/Interp/Interp.h b/clang/lib/AST/Interp/Interp.h
index 25ce93ac03..7ef1e34422 100644
--- a/clang/lib/AST/Interp/Interp.h
+++ b/clang/lib/AST/Interp/Interp.h
@@ -522,7 +522,7 @@ enum class IncDecOp {
 
 template <typename T, IncDecOp Op, PushVal DoPush>
 bool IncDecHelper(InterpState &S, CodePtr OpPC, const Pointer &Ptr) {
-  T Value = Ptr.deref<T>();
+  const T &Value = Ptr.deref<T>();
   T Result;
 
   if constexpr (DoPush == PushVal::Yes)
@@ -1421,7 +1421,7 @@ bool OffsetHelper(InterpState &S, CodePtr OpPC, const T &Offset,
   // Get a version of the index comparable to the type.
   T Index = T::from(Ptr.getIndex(), Offset.bitWidth());
   // Compute the largest index into the array.
-  unsigned MaxIndex = Ptr.getNumElems();
+  T MaxIndex = T::from(Ptr.getNumElems(), Offset.bitWidth());
 
   // Helper to report an invalid offset, computed as APSInt.
   auto InvalidOffset = [&]() {
@@ -1437,7 +1437,7 @@ bool OffsetHelper(InterpState &S, CodePtr OpPC, const T &Offset,
     return false;
   };
 
-  unsigned MaxOffset = MaxIndex - Ptr.getIndex();
+  T MaxOffset = T::from(MaxIndex - Index, Offset.bitWidth());
   if constexpr (Op == ArithOp::Add) {
     // If the new offset would be negative, bail out.
     if (Offset.isNegative() && (Offset.isMin() || -Offset > Index))
@@ -1488,11 +1488,14 @@ static inline bool IncDecPtrHelper(InterpState &S, CodePtr OpPC,
                                    const Pointer &Ptr) {
   using OneT = Integral<8, false>;
 
+  const Pointer &P = Ptr.deref<Pointer>();
+  if (!CheckNull(S, OpPC, P, CSK_ArrayIndex))
+    return false;
+
   // Get the current value on the stack.
-  S.Stk.push<Pointer>(Ptr.deref<Pointer>());
+  S.Stk.push<Pointer>(P);
 
   // Now the current Ptr again and a constant 1.
-  Pointer P = Ptr.deref<Pointer>();
   OneT One = OneT::from(1);
   if (!OffsetHelper<OneT, Op>(S, OpPC, One, P))
     return false;
@@ -1568,6 +1571,22 @@ inline bool CastFP(InterpState &S, CodePtr OpPC, const llvm::fltSemantics *Sem,
   return true;
 }
 
+/// Like Cast(), but we cast to an arbitrary-bitwidth integral, so we need
+/// to know what bitwidth the result should be.
+template <PrimType Name, class T = typename PrimConv<Name>::T>
+bool CastAP(InterpState &S, CodePtr OpPC, uint32_t BitWidth) {
+  S.Stk.push<IntegralAP<false>>(
+      IntegralAP<false>::from(S.Stk.pop<T>(), BitWidth));
+  return true;
+}
+
+template <PrimType Name, class T = typename PrimConv<Name>::T>
+bool CastAPS(InterpState &S, CodePtr OpPC, uint32_t BitWidth) {
+  S.Stk.push<IntegralAP<true>>(
+      IntegralAP<true>::from(S.Stk.pop<T>(), BitWidth));
+  return true;
+}
+
 template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool CastIntegralFloating(InterpState &S, CodePtr OpPC,
                           const llvm::fltSemantics *Sem,
@@ -1608,6 +1627,46 @@ bool CastFloatingIntegral(InterpState &S, CodePtr OpPC) {
   }
 }
 
+static inline bool CastFloatingIntegralAP(InterpState &S, CodePtr OpPC,
+                                          uint32_t BitWidth) {
+  const Floating &F = S.Stk.pop<Floating>();
+
+  APSInt Result(BitWidth, /*IsUnsigned=*/true);
+  auto Status = F.convertToInteger(Result);
+
+  // Float-to-Integral overflow check.
+  if ((Status & APFloat::opStatus::opInvalidOp) && F.isFinite()) {
+    const Expr *E = S.Current->getExpr(OpPC);
+    QualType Type = E->getType();
+
+    S.CCEDiag(E, diag::note_constexpr_overflow) << F.getAPFloat() << Type;
+    return S.noteUndefinedBehavior();
+  }
+
+  S.Stk.push<IntegralAP<true>>(IntegralAP<true>(Result));
+  return CheckFloatResult(S, OpPC, F, Status);
+}
+
+static inline bool CastFloatingIntegralAPS(InterpState &S, CodePtr OpPC,
+                                           uint32_t BitWidth) {
+  const Floating &F = S.Stk.pop<Floating>();
+
+  APSInt Result(BitWidth, /*IsUnsigned=*/false);
+  auto Status = F.convertToInteger(Result);
+
+  // Float-to-Integral overflow check.
+  if ((Status & APFloat::opStatus::opInvalidOp) && F.isFinite()) {
+    const Expr *E = S.Current->getExpr(OpPC);
+    QualType Type = E->getType();
+
+    S.CCEDiag(E, diag::note_constexpr_overflow) << F.getAPFloat() << Type;
+    return S.noteUndefinedBehavior();
+  }
+
+  S.Stk.push<IntegralAP<true>>(IntegralAP<true>(Result));
+  return CheckFloatResult(S, OpPC, F, Status);
+}
+
 template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool CastPointerIntegral(InterpState &S, CodePtr OpPC) {
   const Pointer &Ptr = S.Stk.pop<Pointer>();
@@ -1629,6 +1688,16 @@ bool Zero(InterpState &S, CodePtr OpPC) {
   return true;
 }
 
+static inline bool ZeroIntAP(InterpState &S, CodePtr OpPC, uint32_t BitWidth) {
+  S.Stk.push<IntegralAP<false>>(IntegralAP<false>::zero(BitWidth));
+  return true;
+}
+
+static inline bool ZeroIntAPS(InterpState &S, CodePtr OpPC, uint32_t BitWidth) {
+  S.Stk.push<IntegralAP<true>>(IntegralAP<true>::zero(BitWidth));
+  return true;
+}
+
 template <PrimType Name, class T = typename PrimConv<Name>::T>
 inline bool Null(InterpState &S, CodePtr OpPC) {
   S.Stk.push<T>();
@@ -1697,7 +1766,7 @@ inline bool Shl(InterpState &S, CodePtr OpPC) {
 
   typename LT::AsUnsigned R;
   LT::AsUnsigned::shiftLeft(LT::AsUnsigned::from(LHS),
-                            LT::AsUnsigned::from(RHS), Bits, &R);
+                            LT::AsUnsigned::from(RHS, Bits), Bits, &R);
   S.Stk.push<LT>(LT::from(R));
   return true;
 }
diff --git a/clang/lib/AST/Interp/InterpBuiltin.cpp b/clang/lib/AST/Interp/InterpBuiltin.cpp
index bba0255219..7552c1b88c 100644
--- a/clang/lib/AST/Interp/InterpBuiltin.cpp
+++ b/clang/lib/AST/Interp/InterpBuiltin.cpp
@@ -41,7 +41,8 @@ static APSInt peekToAPSInt(InterpStack &Stk, PrimType T, size_t Offset = 0) {
   APSInt R;
   INT_TYPE_SWITCH(T, {
     T Val = Stk.peek<T>(Offset);
-    R = APSInt(APInt(T::bitWidth(), static_cast<uint64_t>(Val), T::isSigned()));
+    R = APSInt(
+        APInt(Val.bitWidth(), static_cast<uint64_t>(Val), T::isSigned()));
   });
 
   return R;
diff --git a/clang/lib/AST/Interp/InterpStack.cpp b/clang/lib/AST/Interp/InterpStack.cpp
index da4b36f8c1..91fe40feb7 100644
--- a/clang/lib/AST/Interp/InterpStack.cpp
+++ b/clang/lib/AST/Interp/InterpStack.cpp
@@ -10,6 +10,7 @@
 #include "Boolean.h"
 #include "Floating.h"
 #include "Integral.h"
+#include "Pointer.h"
 #include <cassert>
 #include <cstdlib>
 
@@ -85,20 +86,25 @@ void InterpStack::shrink(size_t Size) {
 
 void InterpStack::dump() const {
 #ifndef NDEBUG
-  llvm::errs() << "Items: " << ItemTypes.size() << ". Size: " << size() << "\n";
+  llvm::errs() << "Items: " << ItemTypes.size() << ". Size: " << size() << '\n';
   if (ItemTypes.empty())
     return;
 
   size_t Index = 0;
-  size_t Offset = align(primSize(ItemTypes[0]));
-  for (PrimType Ty : ItemTypes) {
-    llvm::errs() << Index << "/" << Offset << ": ";
-    TYPE_SWITCH(Ty, {
+  size_t Offset = 0;
+
+  // The type of the item on the top of the stack is inserted to the back
+  // of the vector, so the iteration has to happen backwards.
+  for (auto TyIt = ItemTypes.rbegin(); TyIt != ItemTypes.rend(); ++TyIt) {
+    Offset += align(primSize(*TyIt));
+
+    llvm::errs() << Index << '/' << Offset << ": ";
+    TYPE_SWITCH(*TyIt, {
       const T &V = peek<T>(Offset);
       llvm::errs() << V;
     });
-    llvm::errs() << "\n";
-    Offset += align(primSize(Ty));
+    llvm::errs() << '\n';
+
     ++Index;
   }
 #endif
diff --git a/clang/lib/AST/Interp/Opcodes.td b/clang/lib/AST/Interp/Opcodes.td
index 50b6c0ac15..e1e7e5e2ef 100644
--- a/clang/lib/AST/Interp/Opcodes.td
+++ b/clang/lib/AST/Interp/Opcodes.td
@@ -72,6 +72,11 @@ def IntegerTypeClass : TypeClass {
                Uint32, Sint64, Uint64, IntAP, IntAPS];
 }
 
+def FixedSizeIntegralTypeClass : TypeClass {
+  let Types = [Sint8, Uint8, Sint16, Uint16, Sint32,
+               Uint32, Sint64, Uint64, Bool];
+}
+
 def NumberTypeClass : TypeClass {
   let Types = !listconcat(IntegerTypeClass.Types, [Float]);
 }
@@ -243,10 +248,18 @@ def ConstBool : ConstOpcode<Bool, ArgBool>;
 
 // [] -> [Integer]
 def Zero : Opcode {
-  let Types = [AluTypeClass];
+  let Types = [FixedSizeIntegralTypeClass];
   let HasGroup = 1;
 }
 
+def ZeroIntAP : Opcode {
+  let Args = [ArgUint32];
+}
+
+def ZeroIntAPS : Opcode {
+  let Args = [ArgUint32];
+}
+
 // [] -> [Pointer]
 def Null : Opcode {
   let Types = [PtrTypeClass];
@@ -564,7 +577,7 @@ def FromCastTypeClass : TypeClass {
 }
 
 def ToCastTypeClass : TypeClass {
-  let Types = [Uint8, Sint8, Uint16, Sint16, Uint32, Sint32, Uint64, Sint64, Bool, IntAP, IntAPS];
+  let Types = [Uint8, Sint8, Uint16, Sint16, Uint32, Sint32, Uint64, Sint64, Bool];
 }
 
 def Cast: Opcode {
@@ -577,6 +590,22 @@ def CastFP : Opcode {
   let Args = [ArgFltSemantics, ArgRoundingMode];
 }
 
+def FixedSizeIntegralTypes : TypeClass {
+  let Types = [Uint8, Sint8, Uint16, Sint16, Uint32, Sint32, Uint64, Sint64, Bool];
+}
+
+def CastAP : Opcode {
+  let Types = [AluTypeClass];
+  let Args = [ArgUint32];
+  let HasGroup = 1;
+}
+
+def CastAPS : Opcode {
+  let Types = [AluTypeClass];
+  let Args = [ArgUint32];
+  let HasGroup = 1;
+}
+
 // Cast an integer to a floating type
 def CastIntegralFloating : Opcode {
   let Types = [AluTypeClass];
@@ -586,11 +615,21 @@ def CastIntegralFloating : Opcode {
 
 // Cast a floating to an integer type
 def CastFloatingIntegral : Opcode {
-  let Types = [AluTypeClass];
+  let Types = [FixedSizeIntegralTypes];
   let Args = [];
   let HasGroup = 1;
 }
 
+def CastFloatingIntegralAP : Opcode {
+  let Types = [];
+  let Args = [ArgUint32];
+}
+
+def CastFloatingIntegralAPS : Opcode {
+  let Types = [];
+  let Args = [ArgUint32];
+}
+
 def CastPointerIntegral : Opcode {
   let Types = [AluTypeClass];
   let Args = [];
diff --git a/clang/lib/AST/Interp/Pointer.h b/clang/lib/AST/Interp/Pointer.h
index 8c97a96532..d5279e757f 100644
--- a/clang/lib/AST/Interp/Pointer.h
+++ b/clang/lib/AST/Interp/Pointer.h
@@ -78,6 +78,15 @@ class Pointer {
   void operator=(const Pointer &P);
   void operator=(Pointer &&P);
 
+  /// Equality operators are just for tests.
+  bool operator==(const Pointer &P) const {
+    return Pointee == P.Pointee && Base == P.Base && Offset == P.Offset;
+  }
+
+  bool operator!=(const Pointer &P) const {
+    return Pointee != P.Pointee || Base != P.Base || Offset != P.Offset;
+  }
+
   /// Converts the pointer to an APValue.
   APValue toAPValue() const;
 
@@ -276,7 +285,8 @@ class Pointer {
   const Record *getRecord() const { return getFieldDesc()->ElemRecord; }
   /// Returns the element record type, if this is a non-primive array.
   const Record *getElemRecord() const {
-    return getFieldDesc()->ElemDesc->ElemRecord;
+    const Descriptor *ElemDesc = getFieldDesc()->ElemDesc;
+    return ElemDesc ? ElemDesc->ElemRecord : nullptr;
   }
   /// Returns the field information.
   const FieldDecl *getField() const { return getFieldDesc()->asFieldDecl(); }
@@ -326,6 +336,11 @@ class Pointer {
   int64_t getIndex() const {
     if (isElementPastEnd())
       return 1;
+
+    // narrow()ed element in a composite array.
+    if (Base > 0 && Base == Offset)
+      return 0;
+
     if (auto ElemSize = elemSize())
       return getOffset() / ElemSize;
     return 0;
diff --git a/clang/lib/AST/Interp/Source.h b/clang/lib/AST/Interp/Source.h
index fec950dd54..c28b488ff5 100644
--- a/clang/lib/AST/Interp/Source.h
+++ b/clang/lib/AST/Interp/Source.h
@@ -55,7 +55,7 @@ class CodePtr final {
   template <typename T> std::enable_if_t<!std::is_pointer<T>::value, T> read() {
     assert(aligned(Ptr));
     using namespace llvm::support;
-    T Value = endian::read<T, endianness::native, 1>(Ptr);
+    T Value = endian::read<T, llvm::endianness::native>(Ptr);
     Ptr += align(sizeof(T));
     return Value;
   }
diff --git a/clang/lib/AST/ItaniumMangle.cpp b/clang/lib/AST/ItaniumMangle.cpp
index 23ec35cae4..27e8cbf6dc 100644
--- a/clang/lib/AST/ItaniumMangle.cpp
+++ b/clang/lib/AST/ItaniumMangle.cpp
@@ -3429,6 +3429,7 @@ StringRef CXXNameMangler::getCallingConvQualifierName(CallingConv CC) {
   case CC_OpenCLKernel:
   case CC_PreserveMost:
   case CC_PreserveAll:
+  case CC_M68kRTD:
     // FIXME: we should be mangling all of the above.
     return "";
 
@@ -5126,6 +5127,14 @@ void CXXNameMangler::mangleExpression(const Expr *E, unsigned Arity,
       Diags.Report(DiagID);
       return;
     }
+    case UETT_VectorElements: {
+      DiagnosticsEngine &Diags = Context.getDiags();
+      unsigned DiagID = Diags.getCustomDiagID(
+          DiagnosticsEngine::Error,
+          "cannot yet mangle __builtin_vectorelements expression");
+      Diags.Report(DiagID);
+      return;
+    }
     }
     break;
   }
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
index 4c433f7fe9..8389b14230 100644
--- a/clang/lib/AST/Type.cpp
+++ b/clang/lib/AST/Type.cpp
@@ -2369,7 +2369,7 @@ bool Type::isIncompleteType(NamedDecl **Def) const {
 }
 
 bool Type::isSizelessBuiltinType() const {
-  if (isSVESizelessBuiltinType() || isRVVSizelessBuiltinType())
+  if (isSizelessVectorType())
     return true;
 
   if (const BuiltinType *BT = getAs<BuiltinType>()) {
@@ -2403,6 +2403,10 @@ bool Type::isWebAssemblyTableType() const {
 
 bool Type::isSizelessType() const { return isSizelessBuiltinType(); }
 
+bool Type::isSizelessVectorType() const {
+  return isSVESizelessBuiltinType() || isRVVSizelessBuiltinType();
+}
+
 bool Type::isSVESizelessBuiltinType() const {
   if (const BuiltinType *BT = getAs<BuiltinType>()) {
     switch (BT->getKind()) {
@@ -2663,6 +2667,8 @@ static bool
 HasNonDeletedDefaultedEqualityComparison(const CXXRecordDecl *Decl) {
   if (Decl->isUnion())
     return false;
+  if (Decl->isLambda())
+    return Decl->isCapturelessLambda();
 
   auto IsDefaultedOperatorEqualEqual = [&](const FunctionDecl *Function) {
     return Function->getOverloadedOperator() ==
@@ -3371,6 +3377,7 @@ StringRef FunctionType::getNameForCallConv(CallingConv CC) {
   case CC_SwiftAsync: return "swiftasynccall";
   case CC_PreserveMost: return "preserve_most";
   case CC_PreserveAll: return "preserve_all";
+  case CC_M68kRTD: return "m68k_rtd";
   }
 
   llvm_unreachable("Invalid calling convention.");
@@ -3850,6 +3857,7 @@ bool AttributedType::isCallingConv() const {
   case attr::IntelOclBicc:
   case attr::PreserveMost:
   case attr::PreserveAll:
+  case attr::M68kRTD:
     return true;
   }
   llvm_unreachable("invalid attr kind");
diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp
index eb69d0bb87..b9f6c0eeb4 100644
--- a/clang/lib/AST/TypePrinter.cpp
+++ b/clang/lib/AST/TypePrinter.cpp
@@ -1044,6 +1044,9 @@ void TypePrinter::printFunctionAfter(const FunctionType::ExtInfo &Info,
     case CC_PreserveAll:
       OS << " __attribute__((preserve_all))";
       break;
+    case CC_M68kRTD:
+      OS << " __attribute__((m68k_rtd))";
+      break;
     }
   }
 
@@ -1879,6 +1882,9 @@ void TypePrinter::printAttributedAfter(const AttributedType *T,
   case attr::PreserveAll:
     OS << "preserve_all";
     break;
+  case attr::M68kRTD:
+    OS << "m68k_rtd";
+    break;
   case attr::NoDeref:
     OS << "noderef";
     break;
diff --git a/clang/lib/ASTMatchers/ASTMatchersInternal.cpp b/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
index 4068810721..435bbdeda2 100644
--- a/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
+++ b/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
@@ -87,7 +87,7 @@ bool matchesAnyBase(const CXXRecordDecl &Node,
       [Finder, Builder, &BaseSpecMatcher](const CXXBaseSpecifier *BaseSpec,
                                           CXXBasePath &IgnoredParam) {
         BoundNodesTreeBuilder Result(*Builder);
-        if (BaseSpecMatcher.matches(*BaseSpec, Finder, Builder)) {
+        if (BaseSpecMatcher.matches(*BaseSpec, Finder, &Result)) {
           *Builder = std::move(Result);
           return true;
         }
diff --git a/clang/lib/ASTMatchers/GtestMatchers.cpp b/clang/lib/ASTMatchers/GtestMatchers.cpp
index 6e4c12f319..a556d8ef2d 100644
--- a/clang/lib/ASTMatchers/GtestMatchers.cpp
+++ b/clang/lib/ASTMatchers/GtestMatchers.cpp
@@ -21,7 +21,6 @@
 #include "clang/AST/RecursiveASTVisitor.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
 #include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 
 namespace clang {
diff --git a/clang/lib/Analysis/FlowSensitive/Arena.cpp b/clang/lib/Analysis/FlowSensitive/Arena.cpp
index b043a52b60..81137e8088 100644
--- a/clang/lib/Analysis/FlowSensitive/Arena.cpp
+++ b/clang/lib/Analysis/FlowSensitive/Arena.cpp
@@ -22,63 +22,83 @@ canonicalFormulaPair(const Formula &LHS, const Formula &RHS) {
   return Res;
 }
 
-const Formula &Arena::makeAtomRef(Atom A) {
-  auto [It, Inserted] = AtomRefs.try_emplace(A);
+template <class Key, class ComputeFunc>
+const Formula &cached(llvm::DenseMap<Key, const Formula *> &Cache, Key K,
+                      ComputeFunc &&Compute) {
+  auto [It, Inserted] = Cache.try_emplace(std::forward<Key>(K));
   if (Inserted)
-    It->second =
-        &Formula::create(Alloc, Formula::AtomRef, {}, static_cast<unsigned>(A));
+    It->second = Compute();
   return *It->second;
 }
 
-const Formula &Arena::makeAnd(const Formula &LHS, const Formula &RHS) {
-  if (&LHS == &RHS)
-    return LHS;
+const Formula &Arena::makeAtomRef(Atom A) {
+  return cached(AtomRefs, A, [&] {
+    return &Formula::create(Alloc, Formula::AtomRef, {},
+                            static_cast<unsigned>(A));
+  });
+}
 
-  auto [It, Inserted] =
-      Ands.try_emplace(canonicalFormulaPair(LHS, RHS), nullptr);
-  if (Inserted)
-    It->second = &Formula::create(Alloc, Formula::And, {&LHS, &RHS});
-  return *It->second;
+const Formula &Arena::makeAnd(const Formula &LHS, const Formula &RHS) {
+  return cached(Ands, canonicalFormulaPair(LHS, RHS), [&] {
+    if (&LHS == &RHS)
+      return &LHS;
+    if (LHS.kind() == Formula::Literal)
+      return LHS.literal() ? &RHS : &LHS;
+    if (RHS.kind() == Formula::Literal)
+      return RHS.literal() ? &LHS : &RHS;
+
+    return &Formula::create(Alloc, Formula::And, {&LHS, &RHS});
+  });
 }
 
 const Formula &Arena::makeOr(const Formula &LHS, const Formula &RHS) {
-  if (&LHS == &RHS)
-    return LHS;
-
-  auto [It, Inserted] =
-      Ors.try_emplace(canonicalFormulaPair(LHS, RHS), nullptr);
-  if (Inserted)
-    It->second = &Formula::create(Alloc, Formula::Or, {&LHS, &RHS});
-  return *It->second;
+  return cached(Ors, canonicalFormulaPair(LHS, RHS), [&] {
+    if (&LHS == &RHS)
+      return &LHS;
+    if (LHS.kind() == Formula::Literal)
+      return LHS.literal() ? &LHS : &RHS;
+    if (RHS.kind() == Formula::Literal)
+      return RHS.literal() ? &RHS : &LHS;
+
+    return &Formula::create(Alloc, Formula::Or, {&LHS, &RHS});
+  });
 }
 
 const Formula &Arena::makeNot(const Formula &Val) {
-  auto [It, Inserted] = Nots.try_emplace(&Val, nullptr);
-  if (Inserted)
-    It->second = &Formula::create(Alloc, Formula::Not, {&Val});
-  return *It->second;
+  return cached(Nots, &Val, [&] {
+    if (Val.kind() == Formula::Not)
+      return Val.operands()[0];
+    if (Val.kind() == Formula::Literal)
+      return &makeLiteral(!Val.literal());
+
+    return &Formula::create(Alloc, Formula::Not, {&Val});
+  });
 }
 
 const Formula &Arena::makeImplies(const Formula &LHS, const Formula &RHS) {
-  if (&LHS == &RHS)
-    return makeLiteral(true);
-
-  auto [It, Inserted] =
-      Implies.try_emplace(std::make_pair(&LHS, &RHS), nullptr);
-  if (Inserted)
-    It->second = &Formula::create(Alloc, Formula::Implies, {&LHS, &RHS});
-  return *It->second;
+  return cached(Implies, std::make_pair(&LHS, &RHS), [&] {
+    if (&LHS == &RHS)
+      return &makeLiteral(true);
+    if (LHS.kind() == Formula::Literal)
+      return LHS.literal() ? &RHS : &makeLiteral(true);
+    if (RHS.kind() == Formula::Literal)
+      return RHS.literal() ? &RHS : &makeNot(LHS);
+
+    return &Formula::create(Alloc, Formula::Implies, {&LHS, &RHS});
+  });
 }
 
 const Formula &Arena::makeEquals(const Formula &LHS, const Formula &RHS) {
-  if (&LHS == &RHS)
-    return makeLiteral(true);
-
-  auto [It, Inserted] =
-      Equals.try_emplace(canonicalFormulaPair(LHS, RHS), nullptr);
-  if (Inserted)
-    It->second = &Formula::create(Alloc, Formula::Equal, {&LHS, &RHS});
-  return *It->second;
+  return cached(Equals, canonicalFormulaPair(LHS, RHS), [&] {
+    if (&LHS == &RHS)
+      return &makeLiteral(true);
+    if (LHS.kind() == Formula::Literal)
+      return LHS.literal() ? &RHS : &makeNot(RHS);
+    if (RHS.kind() == Formula::Literal)
+      return RHS.literal() ? &LHS : &makeNot(LHS);
+
+    return &Formula::create(Alloc, Formula::Equal, {&LHS, &RHS});
+  });
 }
 
 IntegerValue &Arena::makeIntLiteral(llvm::APInt Value) {
diff --git a/clang/lib/Analysis/FlowSensitive/DataflowAnalysisContext.cpp b/clang/lib/Analysis/FlowSensitive/DataflowAnalysisContext.cpp
index e81048ce92..6a1feb229d 100644
--- a/clang/lib/Analysis/FlowSensitive/DataflowAnalysisContext.cpp
+++ b/clang/lib/Analysis/FlowSensitive/DataflowAnalysisContext.cpp
@@ -73,7 +73,7 @@ StorageLocation &DataflowAnalysisContext::createStorageLocation(QualType Type) {
 }
 
 StorageLocation &
-DataflowAnalysisContext::getStableStorageLocation(const VarDecl &D) {
+DataflowAnalysisContext::getStableStorageLocation(const ValueDecl &D) {
   if (auto *Loc = DeclToLoc.lookup(&D))
     return *Loc;
   auto &Loc = createStorageLocation(D.getType().getNonReferenceType());
@@ -141,8 +141,6 @@ DataflowAnalysisContext::joinFlowConditions(Atom FirstToken,
 
 Solver::Result DataflowAnalysisContext::querySolver(
     llvm::SetVector<const Formula *> Constraints) {
-  Constraints.insert(&arena().makeLiteral(true));
-  Constraints.insert(&arena().makeNot(arena().makeLiteral(false)));
   return S->solve(Constraints.getArrayRef());
 }
 
@@ -160,15 +158,6 @@ bool DataflowAnalysisContext::flowConditionImplies(Atom Token,
   return isUnsatisfiable(std::move(Constraints));
 }
 
-bool DataflowAnalysisContext::flowConditionIsTautology(Atom Token) {
-  // Returns true if and only if we cannot prove that the flow condition can
-  // ever be false.
-  llvm::SetVector<const Formula *> Constraints;
-  Constraints.insert(&arena().makeNot(arena().makeAtomRef(Token)));
-  addTransitiveFlowConditionConstraints(Token, Constraints);
-  return isUnsatisfiable(std::move(Constraints));
-}
-
 bool DataflowAnalysisContext::equivalentFormulas(const Formula &Val1,
                                                  const Formula &Val2) {
   llvm::SetVector<const Formula *> Constraints;
@@ -213,13 +202,8 @@ void DataflowAnalysisContext::dumpFlowCondition(Atom Token,
   Constraints.insert(&arena().makeAtomRef(Token));
   addTransitiveFlowConditionConstraints(Token, Constraints);
 
-  // TODO: have formulas know about true/false directly instead
-  Atom True = arena().makeLiteral(true).getAtom();
-  Atom False = arena().makeLiteral(false).getAtom();
-  Formula::AtomNames Names = {{False, "false"}, {True, "true"}};
-
   for (const auto *Constraint : Constraints) {
-    Constraint->print(OS, &Names);
+    Constraint->print(OS);
     OS << "\n";
   }
 }
diff --git a/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp b/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp
index 66d98c9954..01c6cc69e2 100644
--- a/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp
+++ b/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp
@@ -448,11 +448,23 @@ Environment::Environment(DataflowAnalysisContext &DACtx,
   if (const auto *MethodDecl = dyn_cast<CXXMethodDecl>(&DeclCtx)) {
     auto *Parent = MethodDecl->getParent();
     assert(Parent != nullptr);
-    if (Parent->isLambda())
-      MethodDecl = dyn_cast<CXXMethodDecl>(Parent->getDeclContext());
 
-    // FIXME: Initialize the ThisPointeeLoc of lambdas too.
-    if (MethodDecl && MethodDecl->isImplicitObjectMemberFunction()) {
+    if (Parent->isLambda()) {
+      for (auto Capture : Parent->captures()) {
+        if (Capture.capturesVariable()) {
+          const auto *VarDecl = Capture.getCapturedVar();
+          assert(VarDecl != nullptr);
+          setStorageLocation(*VarDecl, createObject(*VarDecl, nullptr));
+        } else if (Capture.capturesThis()) {
+          const auto *SurroundingMethodDecl =
+              cast<CXXMethodDecl>(DeclCtx.getNonClosureAncestor());
+          QualType ThisPointeeType =
+              SurroundingMethodDecl->getFunctionObjectParameterType();
+          ThisPointeeLoc =
+              &cast<RecordValue>(createValue(ThisPointeeType))->getLoc();
+        }
+      }
+    } else if (MethodDecl->isImplicitObjectMemberFunction()) {
       QualType ThisPointeeType = MethodDecl->getFunctionObjectParameterType();
       ThisPointeeLoc =
           &cast<RecordValue>(createValue(ThisPointeeType))->getLoc();
@@ -673,7 +685,7 @@ StorageLocation &Environment::createStorageLocation(QualType Type) {
   return DACtx->createStorageLocation(Type);
 }
 
-StorageLocation &Environment::createStorageLocation(const VarDecl &D) {
+StorageLocation &Environment::createStorageLocation(const ValueDecl &D) {
   // Evaluated declarations are always assigned the same storage locations to
   // ensure that the environment stabilizes across loop iterations. Storage
   // locations for evaluated declarations are stored in the analysis context.
@@ -885,7 +897,7 @@ Environment::createLocAndMaybeValue(QualType Ty,
   return Loc;
 }
 
-StorageLocation &Environment::createObjectInternal(const VarDecl *D,
+StorageLocation &Environment::createObjectInternal(const ValueDecl *D,
                                                    QualType Ty,
                                                    const Expr *InitExpr) {
   if (Ty->isReferenceType()) {
diff --git a/clang/lib/Analysis/FlowSensitive/Formula.cpp b/clang/lib/Analysis/FlowSensitive/Formula.cpp
index 6d22efc5db..ef7d23ff6c 100644
--- a/clang/lib/Analysis/FlowSensitive/Formula.cpp
+++ b/clang/lib/Analysis/FlowSensitive/Formula.cpp
@@ -17,8 +17,9 @@
 
 namespace clang::dataflow {
 
-Formula &Formula::create(llvm::BumpPtrAllocator &Alloc, Kind K,
-                         ArrayRef<const Formula *> Operands, unsigned Value) {
+const Formula &Formula::create(llvm::BumpPtrAllocator &Alloc, Kind K,
+                               ArrayRef<const Formula *> Operands,
+                               unsigned Value) {
   assert(Operands.size() == numOperands(K));
   if (Value != 0) // Currently, formulas have values or operands, not both.
     assert(numOperands(K) == 0);
@@ -38,6 +39,7 @@ Formula &Formula::create(llvm::BumpPtrAllocator &Alloc, Kind K,
 static llvm::StringLiteral sigil(Formula::Kind K) {
   switch (K) {
   case Formula::AtomRef:
+  case Formula::Literal:
     return "";
   case Formula::Not:
     return "!";
@@ -62,7 +64,16 @@ void Formula::print(llvm::raw_ostream &OS, const AtomNames *Names) const {
 
   switch (numOperands(kind())) {
   case 0:
-    OS << getAtom();
+    switch (kind()) {
+    case AtomRef:
+      OS << getAtom();
+      break;
+    case Literal:
+      OS << (literal() ? "true" : "false");
+      break;
+    default:
+      llvm_unreachable("unhandled formula kind");
+    }
     break;
   case 1:
     OS << sigil(kind());
diff --git a/clang/lib/Analysis/FlowSensitive/HTMLLogger.css b/clang/lib/Analysis/FlowSensitive/HTMLLogger.css
index 4de1d1ccc2..5da8db8fa8 100644
--- a/clang/lib/Analysis/FlowSensitive/HTMLLogger.css
+++ b/clang/lib/Analysis/FlowSensitive/HTMLLogger.css
@@ -19,6 +19,7 @@ section header {
   color: white;
   font-weight: bold;
   font-size: large;
+  padding-right: 0.5em;
 }
 section h2 {
   font-size: medium;
@@ -27,7 +28,7 @@ section h2 {
   border-top: 1px solid #aaa;
 }
 #timeline {
-  min-width: 0;
+  min-width: max-content;
 }
 #timeline .entry.hover {
   background-color: #aaa;
@@ -62,6 +63,10 @@ section h2 {
 #iterations .chooser {
   flex-grow: 1;
   text-align: center;
+  padding-left: 0.2em;
+}
+#iterations .chooser :last-child {
+  padding-right: 0.2em;
 }
 #iterations .chooser:not(.iter-select).hover {
   background-color: #ddd;
diff --git a/clang/lib/Analysis/FlowSensitive/HTMLLogger.html b/clang/lib/Analysis/FlowSensitive/HTMLLogger.html
index ec9d74c71d..b9f76c5074 100644
--- a/clang/lib/Analysis/FlowSensitive/HTMLLogger.html
+++ b/clang/lib/Analysis/FlowSensitive/HTMLLogger.html
@@ -62,7 +62,7 @@
   <template data-for="iter in cfg[selection.bb].iters">
     <a class="chooser {{selection.bb}}:{{iter.iter}}" data-iter="{{selection.bb}}:{{iter.iter}}">
       <template data-if="iter.post_visit">Post-visit</template>
-      <template data-if="!iter.post_visit">Iteration {{iter.iter}}</template>
+      <template data-if="!iter.post_visit">{{iter.iter}}</template>
       <template data-if="iter.converged"> &#x2192;&#x7c;<!--Rightwards arrow, vertical line--></template>
     </a>
   </template>
diff --git a/clang/lib/Analysis/FlowSensitive/Models/UncheckedOptionalAccessModel.cpp b/clang/lib/Analysis/FlowSensitive/Models/UncheckedOptionalAccessModel.cpp
index f61f26ff27..8bd9a030f5 100644
--- a/clang/lib/Analysis/FlowSensitive/Models/UncheckedOptionalAccessModel.cpp
+++ b/clang/lib/Analysis/FlowSensitive/Models/UncheckedOptionalAccessModel.cpp
@@ -599,7 +599,7 @@ void transferAssignment(const CXXOperatorCallExpr *E, BoolValue &HasValueVal,
                         LatticeTransferState &State) {
   assert(E->getNumArgs() > 0);
 
-  if (auto *Loc = cast<RecordStorageLocation>(
+  if (auto *Loc = cast_or_null<RecordStorageLocation>(
           State.Env.getStorageLocation(*E->getArg(0)))) {
     createOptionalValue(*Loc, HasValueVal, State.Env);
 
diff --git a/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp b/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp
index 6b167891c1..72d807fc36 100644
--- a/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp
+++ b/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp
@@ -12,7 +12,6 @@
 //===----------------------------------------------------------------------===//
 
 #include <algorithm>
-#include <memory>
 #include <optional>
 #include <system_error>
 #include <utility>
@@ -33,8 +32,8 @@
 #include "clang/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.h"
 #include "clang/Analysis/FlowSensitive/Value.h"
 #include "llvm/ADT/ArrayRef.h"
-#include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallBitVector.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Error.h"
 
@@ -53,19 +52,14 @@ static int blockIndexInPredecessor(const CFGBlock &Pred,
   return BlockPos - Pred.succ_begin();
 }
 
-static bool isLoopHead(const CFGBlock &B) {
-  if (const auto *T = B.getTerminatorStmt())
-    switch (T->getStmtClass()) {
-      case Stmt::WhileStmtClass:
-      case Stmt::DoStmtClass:
-      case Stmt::ForStmtClass:
-      case Stmt::CXXForRangeStmtClass:
-        return true;
-      default:
-        return false;
-    }
-
-  return false;
+// A "backedge" node is a block introduced in the CFG exclusively to indicate a
+// loop backedge. They are exactly identified by the presence of a non-null
+// pointer to the entry block of the loop condition. Note that this is not
+// necessarily the block with the loop statement as terminator, because
+// short-circuit operators will result in multiple blocks encoding the loop
+// condition, only one of which will contain the loop statement as terminator.
+static bool isBackedgeNode(const CFGBlock &B) {
+  return B.getLoopTarget() != nullptr;
 }
 
 namespace {
@@ -502,14 +496,15 @@ runTypeErasedDataflowAnalysis(
         PostVisitCFG) {
   PrettyStackTraceAnalysis CrashInfo(CFCtx, "runTypeErasedDataflowAnalysis");
 
-  PostOrderCFGView POV(&CFCtx.getCFG());
-  ForwardDataflowWorklist Worklist(CFCtx.getCFG(), &POV);
+  const clang::CFG &CFG = CFCtx.getCFG();
+  PostOrderCFGView POV(&CFG);
+  ForwardDataflowWorklist Worklist(CFG, &POV);
 
   std::vector<std::optional<TypeErasedDataflowAnalysisState>> BlockStates(
-      CFCtx.getCFG().size());
+      CFG.size());
 
   // The entry basic block doesn't contain statements so it can be skipped.
-  const CFGBlock &Entry = CFCtx.getCFG().getEntry();
+  const CFGBlock &Entry = CFG.getEntry();
   BlockStates[Entry.getBlockID()] = {Analysis.typeErasedInitialElement(),
                                      InitEnv.fork()};
   Worklist.enqueueSuccessors(&Entry);
@@ -553,7 +548,7 @@ runTypeErasedDataflowAnalysis(
         llvm::errs() << "Old Env:\n";
         OldBlockState->Env.dump();
       });
-      if (isLoopHead(*Block)) {
+      if (isBackedgeNode(*Block)) {
         LatticeJoinEffect Effect1 = Analysis.widenTypeErased(
             NewBlockState.Lattice, OldBlockState->Lattice);
         LatticeJoinEffect Effect2 =
diff --git a/clang/lib/Analysis/FlowSensitive/WatchedLiteralsSolver.cpp b/clang/lib/Analysis/FlowSensitive/WatchedLiteralsSolver.cpp
index ab3a8104e3..3ef3637535 100644
--- a/clang/lib/Analysis/FlowSensitive/WatchedLiteralsSolver.cpp
+++ b/clang/lib/Analysis/FlowSensitive/WatchedLiteralsSolver.cpp
@@ -322,6 +322,9 @@ CNFFormula buildCNF(const llvm::ArrayRef<const Formula *> &Vals) {
     switch (Val->kind()) {
     case Formula::AtomRef:
       break;
+    case Formula::Literal:
+      CNF.addClause(Val->literal() ? posLit(Var) : negLit(Var));
+      break;
     case Formula::And: {
       const Variable LHS = GetVar(Val->operands()[0]);
       const Variable RHS = GetVar(Val->operands()[1]);
diff --git a/clang/lib/Analysis/ThreadSafety.cpp b/clang/lib/Analysis/ThreadSafety.cpp
index 58dd711366..7fdf22c2f3 100644
--- a/clang/lib/Analysis/ThreadSafety.cpp
+++ b/clang/lib/Analysis/ThreadSafety.cpp
@@ -1008,7 +1008,7 @@ class ThreadSafetyAnalyzer {
   threadSafety::SExprBuilder SxBuilder;
 
   ThreadSafetyHandler &Handler;
-  const CXXMethodDecl *CurrentMethod = nullptr;
+  const FunctionDecl *CurrentFunction;
   LocalVariableMap LocalVarMap;
   FactManager FactMan;
   std::vector<CFGBlockInfo> BlockInfo;
@@ -1243,10 +1243,10 @@ bool ThreadSafetyAnalyzer::inCurrentScope(const CapabilityExpr &CapE) {
 
   // Members are in scope from methods of the same class.
   if (const auto *P = dyn_cast<til::Project>(SExp)) {
-    if (!CurrentMethod)
+    if (!isa_and_nonnull<CXXMethodDecl>(CurrentFunction))
       return false;
     const ValueDecl *VD = P->clangDecl();
-    return VD->getDeclContext() == CurrentMethod->getDeclContext();
+    return VD->getDeclContext() == CurrentFunction->getDeclContext();
   }
 
   return false;
@@ -1541,6 +1541,8 @@ class BuildLockset : public ConstStmtVisitor<BuildLockset> {
 
   ThreadSafetyAnalyzer *Analyzer;
   FactSet FSet;
+  // The fact set for the function on exit.
+  const FactSet &FunctionExitFSet;
   /// Maps constructed objects to `this` placeholder prior to initialization.
   llvm::SmallDenseMap<const Expr *, til::LiteralPtr *> ConstructedObjects;
   LocalVariableMap::Context LVarCtx;
@@ -1566,9 +1568,11 @@ class BuildLockset : public ConstStmtVisitor<BuildLockset> {
                         bool SkipFirstParam = false);
 
 public:
-  BuildLockset(ThreadSafetyAnalyzer *Anlzr, CFGBlockInfo &Info)
+  BuildLockset(ThreadSafetyAnalyzer *Anlzr, CFGBlockInfo &Info,
+               const FactSet &FunctionExitFSet)
       : ConstStmtVisitor<BuildLockset>(), Analyzer(Anlzr), FSet(Info.EntrySet),
-        LVarCtx(Info.EntryContext), CtxIndex(Info.EntryIndex) {}
+        FunctionExitFSet(FunctionExitFSet), LVarCtx(Info.EntryContext),
+        CtxIndex(Info.EntryIndex) {}
 
   void VisitUnaryOperator(const UnaryOperator *UO);
   void VisitBinaryOperator(const BinaryOperator *BO);
@@ -1577,6 +1581,7 @@ class BuildLockset : public ConstStmtVisitor<BuildLockset> {
   void VisitCXXConstructExpr(const CXXConstructExpr *Exp);
   void VisitDeclStmt(const DeclStmt *S);
   void VisitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *Exp);
+  void VisitReturnStmt(const ReturnStmt *S);
 };
 
 } // namespace
@@ -1758,6 +1763,8 @@ void ThreadSafetyAnalyzer::checkPtAccess(const FactSet &FSet, const Expr *Exp,
   // Pass by reference warnings are under a different flag.
   ProtectedOperationKind PtPOK = POK_VarDereference;
   if (POK == POK_PassByRef) PtPOK = POK_PtPassByRef;
+  if (POK == POK_ReturnByRef)
+    PtPOK = POK_PtReturnByRef;
 
   const ValueDecl *D = getValueDecl(Exp);
   if (!D || !D->hasAttrs())
@@ -2142,6 +2149,25 @@ void BuildLockset::VisitMaterializeTemporaryExpr(
   }
 }
 
+void BuildLockset::VisitReturnStmt(const ReturnStmt *S) {
+  if (Analyzer->CurrentFunction == nullptr)
+    return;
+  const Expr *RetVal = S->getRetValue();
+  if (!RetVal)
+    return;
+
+  // If returning by reference, check that the function requires the appropriate
+  // capabilities.
+  const QualType ReturnType =
+      Analyzer->CurrentFunction->getReturnType().getCanonicalType();
+  if (ReturnType->isLValueReferenceType()) {
+    Analyzer->checkAccess(
+        FunctionExitFSet, RetVal,
+        ReturnType->getPointeeType().isConstQualified() ? AK_Read : AK_Written,
+        POK_ReturnByRef);
+  }
+}
+
 /// Given two facts merging on a join point, possibly warn and decide whether to
 /// keep or replace.
 ///
@@ -2251,8 +2277,7 @@ void ThreadSafetyAnalyzer::runAnalysis(AnalysisDeclContext &AC) {
 
   CFG *CFGraph = walker.getGraph();
   const NamedDecl *D = walker.getDecl();
-  const auto *CurrentFunction = dyn_cast<FunctionDecl>(D);
-  CurrentMethod = dyn_cast<CXXMethodDecl>(D);
+  CurrentFunction = dyn_cast<FunctionDecl>(D);
 
   if (D->hasAttr<NoThreadSafetyAnalysisAttr>())
     return;
@@ -2348,6 +2373,25 @@ void ThreadSafetyAnalyzer::runAnalysis(AnalysisDeclContext &AC) {
     }
   }
 
+  // Compute the expected exit set.
+  // By default, we expect all locks held on entry to be held on exit.
+  FactSet ExpectedFunctionExitSet = Initial.EntrySet;
+
+  // Adjust the expected exit set by adding or removing locks, as declared
+  // by *-LOCK_FUNCTION and UNLOCK_FUNCTION.  The intersect below will then
+  // issue the appropriate warning.
+  // FIXME: the location here is not quite right.
+  for (const auto &Lock : ExclusiveLocksAcquired)
+    ExpectedFunctionExitSet.addLock(
+        FactMan, std::make_unique<LockableFactEntry>(Lock, LK_Exclusive,
+                                                     D->getLocation()));
+  for (const auto &Lock : SharedLocksAcquired)
+    ExpectedFunctionExitSet.addLock(
+        FactMan,
+        std::make_unique<LockableFactEntry>(Lock, LK_Shared, D->getLocation()));
+  for (const auto &Lock : LocksReleased)
+    ExpectedFunctionExitSet.removeLock(FactMan, Lock);
+
   for (const auto *CurrBlock : *SortedGraph) {
     unsigned CurrBlockID = CurrBlock->getBlockID();
     CFGBlockInfo *CurrBlockInfo = &BlockInfo[CurrBlockID];
@@ -2407,7 +2451,7 @@ void ThreadSafetyAnalyzer::runAnalysis(AnalysisDeclContext &AC) {
     if (!CurrBlockInfo->Reachable)
       continue;
 
-    BuildLockset LocksetBuilder(this, *CurrBlockInfo);
+    BuildLockset LocksetBuilder(this, *CurrBlockInfo, ExpectedFunctionExitSet);
 
     // Visit all the statements in the basic block.
     for (const auto &BI : *CurrBlock) {
@@ -2483,24 +2527,8 @@ void ThreadSafetyAnalyzer::runAnalysis(AnalysisDeclContext &AC) {
   if (!Final.Reachable)
     return;
 
-  // By default, we expect all locks held on entry to be held on exit.
-  FactSet ExpectedExitSet = Initial.EntrySet;
-
-  // Adjust the expected exit set by adding or removing locks, as declared
-  // by *-LOCK_FUNCTION and UNLOCK_FUNCTION.  The intersect below will then
-  // issue the appropriate warning.
-  // FIXME: the location here is not quite right.
-  for (const auto &Lock : ExclusiveLocksAcquired)
-    ExpectedExitSet.addLock(FactMan, std::make_unique<LockableFactEntry>(
-                                         Lock, LK_Exclusive, D->getLocation()));
-  for (const auto &Lock : SharedLocksAcquired)
-    ExpectedExitSet.addLock(FactMan, std::make_unique<LockableFactEntry>(
-                                         Lock, LK_Shared, D->getLocation()));
-  for (const auto &Lock : LocksReleased)
-    ExpectedExitSet.removeLock(FactMan, Lock);
-
   // FIXME: Should we call this function for all blocks which exit the function?
-  intersectAndWarn(ExpectedExitSet, Final.ExitSet, Final.ExitLoc,
+  intersectAndWarn(ExpectedFunctionExitSet, Final.ExitSet, Final.ExitLoc,
                    LEK_LockedAtEndOfFunction, LEK_NotLockedAtEndOfFunction);
 
   Handler.leaveFunction(CurrentFunction);
diff --git a/clang/lib/Analysis/UnsafeBufferUsage.cpp b/clang/lib/Analysis/UnsafeBufferUsage.cpp
index 49cfa7c0d3..e332a36092 100644
--- a/clang/lib/Analysis/UnsafeBufferUsage.cpp
+++ b/clang/lib/Analysis/UnsafeBufferUsage.cpp
@@ -8,7 +8,9 @@
 
 #include "clang/Analysis/Analyses/UnsafeBufferUsage.h"
 #include "clang/AST/Decl.h"
+#include "clang/AST/Expr.h"
 #include "clang/AST/RecursiveASTVisitor.h"
+#include "clang/AST/StmtVisitor.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
 #include "clang/Lex/Lexer.h"
 #include "clang/Lex/Preprocessor.h"
@@ -22,6 +24,52 @@ using namespace llvm;
 using namespace clang;
 using namespace ast_matchers;
 
+#ifndef NDEBUG
+namespace {
+class StmtDebugPrinter
+    : public ConstStmtVisitor<StmtDebugPrinter, std::string> {
+public:
+  std::string VisitStmt(const Stmt *S) { return S->getStmtClassName(); }
+
+  std::string VisitBinaryOperator(const BinaryOperator *BO) {
+    return "BinaryOperator(" + BO->getOpcodeStr().str() + ")";
+  }
+
+  std::string VisitUnaryOperator(const UnaryOperator *UO) {
+    return "UnaryOperator(" + UO->getOpcodeStr(UO->getOpcode()).str() + ")";
+  }
+
+  std::string VisitImplicitCastExpr(const ImplicitCastExpr *ICE) {
+    return "ImplicitCastExpr(" + std::string(ICE->getCastKindName()) + ")";
+  }
+};
+
+// Returns a string of ancestor `Stmt`s of the given `DRE` in such a form:
+// "DRE ==> parent-of-DRE ==> grandparent-of-DRE ==> ...".
+static std::string getDREAncestorString(const DeclRefExpr *DRE,
+                                        ASTContext &Ctx) {
+  std::stringstream SS;
+  const Stmt *St = DRE;
+  StmtDebugPrinter StmtPriner;
+
+  do {
+    SS << StmtPriner.Visit(St);
+
+    DynTypedNodeList StParents = Ctx.getParents(*St);
+
+    if (StParents.size() > 1)
+      return "unavailable due to multiple parents";
+    if (StParents.size() == 0)
+      break;
+    St = StParents.begin()->get<Stmt>();
+    if (St)
+      SS << " ==> ";
+  } while (St);
+  return SS.str();
+}
+} // namespace
+#endif /* NDEBUG */
+
 namespace clang::ast_matchers {
 // A `RecursiveASTVisitor` that traverses all descendants of a given node "n"
 // except for those belonging to a different callable of "n".
@@ -2589,11 +2637,15 @@ void clang::checkUnsafeBufferUsage(const Decl *D,
 #ifndef NDEBUG
         auto AllUnclaimed = Tracker.getUnclaimedUses(it->first);
         for (auto UnclaimedDRE : AllUnclaimed) {
-          Handler.addDebugNoteForVar(
-              it->first, UnclaimedDRE->getBeginLoc(),
-                                     ("failed to produce fixit for '" + it->first->getNameAsString() +
-                                      "' : has an unclaimed use"));
-          }
+        std::string UnclaimedUseTrace =
+            getDREAncestorString(UnclaimedDRE, D->getASTContext());
+
+        Handler.addDebugNoteForVar(
+            it->first, UnclaimedDRE->getBeginLoc(),
+            ("failed to produce fixit for '" + it->first->getNameAsString() +
+             "' : has an unclaimed use\nThe unclaimed DRE trace: " +
+             UnclaimedUseTrace));
+        }
 #endif
         it = FixablesForAllVars.byVar.erase(it);
       } else if (it->first->isInitCapture()) {
diff --git a/clang/lib/Basic/Module.cpp b/clang/lib/Basic/Module.cpp
index 7879a11179..cc2e5be98d 100644
--- a/clang/lib/Basic/Module.cpp
+++ b/clang/lib/Basic/Module.cpp
@@ -265,10 +265,10 @@ bool Module::fullModuleNameIs(ArrayRef<StringRef> nameParts) const {
 }
 
 OptionalDirectoryEntryRef Module::getEffectiveUmbrellaDir() const {
-  if (Umbrella && Umbrella.is<FileEntryRef>())
-    return Umbrella.get<FileEntryRef>().getDir();
-  if (Umbrella && Umbrella.is<DirectoryEntryRef>())
-    return Umbrella.get<DirectoryEntryRef>();
+  if (const auto *Hdr = std::get_if<FileEntryRef>(&Umbrella))
+    return Hdr->getDir();
+  if (const auto *Dir = std::get_if<DirectoryEntryRef>(&Umbrella))
+    return *Dir;
   return std::nullopt;
 }
 
diff --git a/clang/lib/Basic/Sarif.cpp b/clang/lib/Basic/Sarif.cpp
index bef948181e..3476103cc3 100644
--- a/clang/lib/Basic/Sarif.cpp
+++ b/clang/lib/Basic/Sarif.cpp
@@ -20,7 +20,6 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringExtras.h"
-#include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/ConvertUTF.h"
 #include "llvm/Support/JSON.h"
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index 4ce89267e0..c44ecacb3d 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -1219,7 +1219,7 @@ LineOffsetMapping LineOffsetMapping::get(llvm::MemoryBufferRef Buffer,
   // This is much faster than scanning each byte independently.
   if ((unsigned long)(End - Start) > sizeof(Word)) {
     do {
-      Word = llvm::support::endian::read64(Buf, llvm::support::little);
+      Word = llvm::support::endian::read64(Buf, llvm::endianness::little);
       // no new line => jump over sizeof(Word) bytes.
       auto Mask = likelyhasbetween(Word, '\n', '\r');
       if (!Mask) {
diff --git a/clang/lib/Basic/Targets/ARM.cpp b/clang/lib/Basic/Targets/ARM.cpp
index 1e80928374..ce7e4d4639 100644
--- a/clang/lib/Basic/Targets/ARM.cpp
+++ b/clang/lib/Basic/Targets/ARM.cpp
@@ -258,6 +258,7 @@ ARMTargetInfo::ARMTargetInfo(const llvm::Triple &Triple,
   bool IsOpenBSD = Triple.isOSOpenBSD();
   bool IsNetBSD = Triple.isOSNetBSD();
   bool IsHaiku = Triple.isOSHaiku();
+  bool IsOHOS = Triple.isOHOSFamily();
 
   // FIXME: the isOSBinFormatMachO is a workaround for identifying a Darwin-like
   // environment where size_t is `unsigned long` rather than `unsigned int`
@@ -324,7 +325,7 @@ ARMTargetInfo::ARMTargetInfo(const llvm::Triple &Triple,
     default:
       if (IsNetBSD)
         setABI("apcs-gnu");
-      else if (IsFreeBSD || IsOpenBSD || IsHaiku)
+      else if (IsFreeBSD || IsOpenBSD || IsHaiku || IsOHOS)
         setABI("aapcs-linux");
       else
         setABI("aapcs");
diff --git a/clang/lib/Basic/Targets/M68k.cpp b/clang/lib/Basic/Targets/M68k.cpp
index 3c6274f89d..1b7e0a7f32 100644
--- a/clang/lib/Basic/Targets/M68k.cpp
+++ b/clang/lib/Basic/Targets/M68k.cpp
@@ -238,5 +238,15 @@ TargetInfo::BuiltinVaListKind M68kTargetInfo::getBuiltinVaListKind() const {
   return TargetInfo::VoidPtrBuiltinVaList;
 }
 
+TargetInfo::CallingConvCheckResult
+M68kTargetInfo::checkCallingConvention(CallingConv CC) const {
+  switch (CC) {
+  case CC_C:
+  case CC_M68kRTD:
+    return CCCR_OK;
+  default:
+    return TargetInfo::checkCallingConvention(CC);
+  }
+}
 } // namespace targets
 } // namespace clang
diff --git a/clang/lib/Basic/Targets/M68k.h b/clang/lib/Basic/Targets/M68k.h
index 1af00115a5..a9c262e62f 100644
--- a/clang/lib/Basic/Targets/M68k.h
+++ b/clang/lib/Basic/Targets/M68k.h
@@ -54,6 +54,7 @@ class LLVM_LIBRARY_VISIBILITY M68kTargetInfo : public TargetInfo {
   std::string_view getClobbers() const override;
   BuiltinVaListKind getBuiltinVaListKind() const override;
   bool setCPU(const std::string &Name) override;
+  CallingConvCheckResult checkCallingConvention(CallingConv CC) const override;
 };
 
 } // namespace targets
diff --git a/clang/lib/Basic/Targets/Mips.cpp b/clang/lib/Basic/Targets/Mips.cpp
index cdf652c49f..bc90d1b93d 100644
--- a/clang/lib/Basic/Targets/Mips.cpp
+++ b/clang/lib/Basic/Targets/Mips.cpp
@@ -149,6 +149,10 @@ void MipsTargetInfo::getTargetDefines(const LangOptions &Opts,
     Builder.defineMacro("_MIPS_FPSET", Twine(32));
   else
     Builder.defineMacro("_MIPS_FPSET", Twine(16));
+  if (NoOddSpreg)
+    Builder.defineMacro("_MIPS_SPFPSET", Twine(16));
+  else
+    Builder.defineMacro("_MIPS_SPFPSET", Twine(32));
 
   if (IsMips16)
     Builder.defineMacro("__mips16", Twine(1));
diff --git a/clang/lib/Basic/Targets/Mips.h b/clang/lib/Basic/Targets/Mips.h
index 7ecbd8633c..f46b95abfd 100644
--- a/clang/lib/Basic/Targets/Mips.h
+++ b/clang/lib/Basic/Targets/Mips.h
@@ -53,6 +53,7 @@ class LLVM_LIBRARY_VISIBILITY MipsTargetInfo : public TargetInfo {
   bool HasMSA;
   bool DisableMadd4;
   bool UseIndirectJumpHazard;
+  bool NoOddSpreg;
 
 protected:
   enum FPModeEnum { FPXX, FP32, FP64 } FPMode;
@@ -313,6 +314,8 @@ class LLVM_LIBRARY_VISIBILITY MipsTargetInfo : public TargetInfo {
     FloatABI = HardFloat;
     DspRev = NoDSP;
     FPMode = isFP64Default() ? FP64 : FPXX;
+    NoOddSpreg = false;
+    bool OddSpregGiven = false;
 
     for (const auto &Feature : Features) {
       if (Feature == "+single-float")
@@ -349,8 +352,18 @@ class LLVM_LIBRARY_VISIBILITY MipsTargetInfo : public TargetInfo {
         IsNoABICalls = true;
       else if (Feature == "+use-indirect-jump-hazard")
         UseIndirectJumpHazard = true;
+      else if (Feature == "+nooddspreg") {
+        NoOddSpreg = true;
+        OddSpregGiven = false;
+      } else if (Feature == "-nooddspreg") {
+        NoOddSpreg = false;
+        OddSpregGiven = true;
+      }
     }
 
+    if (FPMode == FPXX && !OddSpregGiven)
+      NoOddSpreg = true;
+
     setDataLayout();
 
     return true;
diff --git a/clang/lib/Basic/Targets/OSTargets.h b/clang/lib/Basic/Targets/OSTargets.h
index f2bd846e67..23799d8a4a 100644
--- a/clang/lib/Basic/Targets/OSTargets.h
+++ b/clang/lib/Basic/Targets/OSTargets.h
@@ -817,10 +817,10 @@ class LLVM_LIBRARY_VISIBILITY NaClTargetInfo : public OSTargetInfo<Target> {
       // Handled in ARM's setABI().
     } else if (Triple.getArch() == llvm::Triple::x86) {
       this->resetDataLayout("e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-"
-                            "i64:64-n8:16:32-S128");
+                            "i64:64-i128:128-n8:16:32-S128");
     } else if (Triple.getArch() == llvm::Triple::x86_64) {
       this->resetDataLayout("e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-"
-                            "i64:64-n8:16:32:64-S128");
+                            "i64:64-i128:128-n8:16:32:64-S128");
     } else if (Triple.getArch() == llvm::Triple::mipsel) {
       // Handled on mips' setDataLayout.
     } else {
diff --git a/clang/lib/Basic/Targets/PPC.cpp b/clang/lib/Basic/Targets/PPC.cpp
index 4e895cc731..0d87a3a4e8 100644
--- a/clang/lib/Basic/Targets/PPC.cpp
+++ b/clang/lib/Basic/Targets/PPC.cpp
@@ -753,6 +753,8 @@ void PPCTargetInfo::setFeatureEnabled(llvm::StringMap<bool> &Features,
   }
 }
 
+// Make sure that registers are added in the correct array index which should be
+// the DWARF number for PPC registers.
 const char *const PPCTargetInfo::GCCRegNames[] = {
     "r0",  "r1",     "r2",   "r3",      "r4",      "r5",  "r6",  "r7",  "r8",
     "r9",  "r10",    "r11",  "r12",     "r13",     "r14", "r15", "r16", "r17",
@@ -807,6 +809,7 @@ ArrayRef<TargetInfo::GCCRegAlias> PPCTargetInfo::getGCCRegAliases() const {
 // PPC ELFABIv2 DWARF Definition "Table 2.26. Mappings of Common Registers".
 // vs0 ~ vs31 is mapping to 32 - 63,
 // vs32 ~ vs63 is mapping to 77 - 108.
+// And this mapping applies to all OSes which run on powerpc.
 const TargetInfo::AddlRegName GCCAddlRegNames[] = {
     // Table of additional register names to use in user input.
     {{"vs0"}, 32},   {{"vs1"}, 33},   {{"vs2"}, 34},   {{"vs3"}, 35},
@@ -828,10 +831,7 @@ const TargetInfo::AddlRegName GCCAddlRegNames[] = {
 };
 
 ArrayRef<TargetInfo::AddlRegName> PPCTargetInfo::getGCCAddlRegNames() const {
-  if (ABI == "elfv2")
-    return llvm::ArrayRef(GCCAddlRegNames);
-  else
-    return TargetInfo::getGCCAddlRegNames();
+  return llvm::ArrayRef(GCCAddlRegNames);
 }
 
 static constexpr llvm::StringLiteral ValidCPUNames[] = {
diff --git a/clang/lib/Basic/Targets/PPC.h b/clang/lib/Basic/Targets/PPC.h
index ef667b3d51..4d62673ba7 100644
--- a/clang/lib/Basic/Targets/PPC.h
+++ b/clang/lib/Basic/Targets/PPC.h
@@ -198,6 +198,8 @@ class LLVM_LIBRARY_VISIBILITY PPCTargetInfo : public TargetInfo {
   void setFeatureEnabled(llvm::StringMap<bool> &Features, StringRef Name,
                          bool Enabled) const override;
 
+  bool supportsTargetAttributeTune() const override { return true; }
+
   ArrayRef<const char *> getGCCRegNames() const override;
 
   ArrayRef<TargetInfo::GCCRegAlias> getGCCRegAliases() const override;
diff --git a/clang/lib/Basic/Targets/RISCV.cpp b/clang/lib/Basic/Targets/RISCV.cpp
index d55ab76395..5f75619b74 100644
--- a/clang/lib/Basic/Targets/RISCV.cpp
+++ b/clang/lib/Basic/Targets/RISCV.cpp
@@ -23,6 +23,7 @@ using namespace clang;
 using namespace clang::targets;
 
 ArrayRef<const char *> RISCVTargetInfo::getGCCRegNames() const {
+  // clang-format off
   static const char *const GCCRegNames[] = {
       // Integer registers
       "x0",  "x1",  "x2",  "x3",  "x4",  "x5",  "x6",  "x7",
@@ -40,7 +41,12 @@ ArrayRef<const char *> RISCVTargetInfo::getGCCRegNames() const {
       "v0",  "v1",  "v2",  "v3",  "v4",  "v5",  "v6",  "v7",
       "v8",  "v9",  "v10", "v11", "v12", "v13", "v14", "v15",
       "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
-      "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"};
+      "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31",
+
+      // CSRs
+      "fflags", "frm", "vtype", "vl", "vxsat", "vxrm"
+    };
+  // clang-format on
   return llvm::ArrayRef(GCCRegNames);
 }
 
diff --git a/clang/lib/Basic/Targets/X86.cpp b/clang/lib/Basic/Targets/X86.cpp
index 022d575313..eec3cd5584 100644
--- a/clang/lib/Basic/Targets/X86.cpp
+++ b/clang/lib/Basic/Targets/X86.cpp
@@ -121,6 +121,7 @@ bool X86TargetInfo::initFeatureMap(
   std::vector<std::string> UpdatedFeaturesVec;
   bool HasEVEX512 = true;
   bool HasAVX512F = false;
+  bool HasAVX10 = false;
   for (const auto &Feature : FeaturesVec) {
     // Expand general-regs-only to -x86, -mmx and -sse
     if (Feature == "+general-regs-only") {
@@ -130,17 +131,35 @@ bool X86TargetInfo::initFeatureMap(
       continue;
     }
 
-    if (!HasAVX512F && Feature.substr(0, 7) == "+avx512")
+    if (Feature.substr(0, 7) == "+avx10.") {
+      HasAVX10 = true;
       HasAVX512F = true;
-    if (HasAVX512F && Feature == "-avx512f")
+      if (Feature.substr(Feature.size() - 3, 3) == "512") {
+        HasEVEX512 = true;
+      } else if (Feature.substr(7, 2) == "1-") {
+        HasEVEX512 = false;
+      }
+    } else if (!HasAVX512F && Feature.substr(0, 7) == "+avx512") {
+      HasAVX512F = true;
+    } else if (HasAVX512F && Feature == "-avx512f") {
+      HasAVX512F = false;
+    } else if (HasAVX10 && Feature == "-avx10.1-256") {
+      HasAVX10 = false;
       HasAVX512F = false;
-    if (HasEVEX512 && Feature == "-evex512")
+    } else if (!HasEVEX512 && Feature == "+evex512") {
+      HasEVEX512 = true;
+    } else if (HasEVEX512 && Feature == "-avx10.1-512") {
       HasEVEX512 = false;
+    } else if (HasEVEX512 && Feature == "-evex512") {
+      HasEVEX512 = false;
+    }
 
     UpdatedFeaturesVec.push_back(Feature);
   }
   if (HasAVX512F && HasEVEX512)
     UpdatedFeaturesVec.push_back("+evex512");
+  else if (HasAVX10)
+    UpdatedFeaturesVec.push_back("-evex512");
 
   if (!TargetInfo::initFeatureMap(Features, Diags, CPU, UpdatedFeaturesVec))
     return false;
@@ -241,6 +260,10 @@ bool X86TargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
       HasGFNI = true;
     } else if (Feature == "+evex512") {
       HasEVEX512 = true;
+    } else if (Feature == "+avx10.1-256") {
+      HasAVX10_1 = true;
+    } else if (Feature == "+avx10.1-512") {
+      HasAVX10_1_512 = true;
     } else if (Feature == "+avx512cd") {
       HasAVX512CD = true;
     } else if (Feature == "+avx512vpopcntdq") {
@@ -376,6 +399,8 @@ bool X86TargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
       HasTSXLDTRK = true;
     } else if (Feature == "+uintr") {
       HasUINTR = true;
+    } else if (Feature == "+usermsr") {
+      HasUSERMSR = true;
     } else if (Feature == "+crc32") {
       HasCRC32 = true;
     } else if (Feature == "+x87") {
@@ -563,11 +588,13 @@ void X86TargetInfo::getTargetDefines(const LangOptions &Opts,
   case CK_Arrowlake:
   case CK_ArrowlakeS:
   case CK_Lunarlake:
+  case CK_Pantherlake:
   case CK_Sierraforest:
   case CK_Grandridge:
   case CK_Graniterapids:
   case CK_GraniterapidsD:
   case CK_Emeraldrapids:
+  case CK_Clearwaterforest:
     // FIXME: Historically, we defined this legacy name, it would be nice to
     // remove it at some point. We've never exposed fine-grained names for
     // recent primary x86 CPUs, and we should keep it that way.
@@ -746,6 +773,10 @@ void X86TargetInfo::getTargetDefines(const LangOptions &Opts,
 
   if (HasEVEX512)
     Builder.defineMacro("__EVEX512__");
+  if (HasAVX10_1)
+    Builder.defineMacro("__AVX10_1__");
+  if (HasAVX10_1_512)
+    Builder.defineMacro("__AVX10_1_512__");
   if (HasAVX512CD)
     Builder.defineMacro("__AVX512CD__");
   if (HasAVX512VPOPCNTDQ)
@@ -869,6 +900,8 @@ void X86TargetInfo::getTargetDefines(const LangOptions &Opts,
     Builder.defineMacro("__TSXLDTRK__");
   if (HasUINTR)
     Builder.defineMacro("__UINTR__");
+  if (HasUSERMSR)
+    Builder.defineMacro("__USERMSR__");
   if (HasCRC32)
     Builder.defineMacro("__CRC32__");
 
@@ -969,6 +1002,8 @@ bool X86TargetInfo::isValidFeatureName(StringRef Name) const {
       .Case("amx-int8", true)
       .Case("amx-tile", true)
       .Case("avx", true)
+      .Case("avx10.1-256", true)
+      .Case("avx10.1-512", true)
       .Case("avx2", true)
       .Case("avx512f", true)
       .Case("avx512cd", true)
@@ -1053,6 +1088,7 @@ bool X86TargetInfo::isValidFeatureName(StringRef Name) const {
       .Case("tbm", true)
       .Case("tsxldtrk", true)
       .Case("uintr", true)
+      .Case("usermsr", true)
       .Case("vaes", true)
       .Case("vpclmulqdq", true)
       .Case("wbnoinvd", true)
@@ -1076,6 +1112,8 @@ bool X86TargetInfo::hasFeature(StringRef Feature) const {
       .Case("amx-int8", HasAMXINT8)
       .Case("amx-tile", HasAMXTILE)
       .Case("avx", SSELevel >= AVX)
+      .Case("avx10.1-256", HasAVX10_1)
+      .Case("avx10.1-512", HasAVX10_1_512)
       .Case("avx2", SSELevel >= AVX2)
       .Case("avx512f", SSELevel >= AVX512F)
       .Case("avx512cd", HasAVX512CD)
@@ -1162,6 +1200,7 @@ bool X86TargetInfo::hasFeature(StringRef Feature) const {
       .Case("tbm", HasTBM)
       .Case("tsxldtrk", HasTSXLDTRK)
       .Case("uintr", HasUINTR)
+      .Case("usermsr", HasUSERMSR)
       .Case("vaes", HasVAES)
       .Case("vpclmulqdq", HasVPCLMULQDQ)
       .Case("wbnoinvd", HasWBNOINVD)
@@ -1464,11 +1503,13 @@ std::optional<unsigned> X86TargetInfo::getCPUCacheLineSize() const {
     case CK_Arrowlake:
     case CK_ArrowlakeS:
     case CK_Lunarlake:
+    case CK_Pantherlake:
     case CK_Sierraforest:
     case CK_Grandridge:
     case CK_Graniterapids:
     case CK_GraniterapidsD:
     case CK_Emeraldrapids:
+    case CK_Clearwaterforest:
     case CK_KNL:
     case CK_KNM:
     // K7
diff --git a/clang/lib/Basic/Targets/X86.h b/clang/lib/Basic/Targets/X86.h
index b759c76fc9..99a64501d2 100644
--- a/clang/lib/Basic/Targets/X86.h
+++ b/clang/lib/Basic/Targets/X86.h
@@ -95,6 +95,8 @@ class LLVM_LIBRARY_VISIBILITY X86TargetInfo : public TargetInfo {
   bool HasLWP = false;
   bool HasFMA = false;
   bool HasF16C = false;
+  bool HasAVX10_1 = false;
+  bool HasAVX10_1_512 = false;
   bool HasEVEX512 = false;
   bool HasAVX512CD = false;
   bool HasAVX512VPOPCNTDQ = false;
@@ -162,6 +164,7 @@ class LLVM_LIBRARY_VISIBILITY X86TargetInfo : public TargetInfo {
   bool HasAMXCOMPLEX = false;
   bool HasSERIALIZE = false;
   bool HasTSXLDTRK = false;
+  bool HasUSERMSR = false;
   bool HasUINTR = false;
   bool HasCRC32 = false;
   bool HasX87 = false;
@@ -431,13 +434,12 @@ class LLVM_LIBRARY_VISIBILITY X86_32TargetInfo : public X86TargetInfo {
     LongDoubleWidth = 96;
     LongDoubleAlign = 32;
     SuitableAlign = 128;
-    resetDataLayout(
-        Triple.isOSBinFormatMachO()
-            ? "e-m:o-p:32:32-p270:32:32-p271:32:32-p272:64:64-f64:32:64-"
-              "f80:32-n8:16:32-S128"
-            : "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-f64:32:64-"
-              "f80:32-n8:16:32-S128",
-        Triple.isOSBinFormatMachO() ? "_" : "");
+    resetDataLayout(Triple.isOSBinFormatMachO()
+                        ? "e-m:o-p:32:32-p270:32:32-p271:32:32-p272:64:64-i128:"
+                          "128-f64:32:64-f80:32-n8:16:32-S128"
+                        : "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-i128:"
+                          "128-f64:32:64-f80:32-n8:16:32-S128",
+                    Triple.isOSBinFormatMachO() ? "_" : "");
     SizeType = UnsignedInt;
     PtrDiffType = SignedInt;
     IntPtrType = SignedInt;
@@ -542,8 +544,9 @@ class LLVM_LIBRARY_VISIBILITY DarwinI386TargetInfo
       UseSignedCharForObjCBool = false;
     SizeType = UnsignedLong;
     IntPtrType = SignedLong;
-    resetDataLayout("e-m:o-p:32:32-p270:32:32-p271:32:32-p272:64:64-f64:32:64-"
-                    "f80:128-n8:16:32-S128", "_");
+    resetDataLayout("e-m:o-p:32:32-p270:32:32-p271:32:32-p272:64:64-i128:128-"
+                    "f64:32:64-f80:128-n8:16:32-S128",
+                    "_");
     HasAlignMac68kSupport = true;
   }
 
@@ -570,7 +573,7 @@ class LLVM_LIBRARY_VISIBILITY WindowsX86_32TargetInfo
         getTriple().isOSWindows() && getTriple().isOSBinFormatCOFF();
     bool IsMSVC = getTriple().isWindowsMSVCEnvironment();
     std::string Layout = IsWinCOFF ? "e-m:x" : "e-m:e";
-    Layout += "-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-";
+    Layout += "-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-";
     Layout += IsMSVC ? "f80:128" : "f80:32";
     Layout += "-n8:16:32-a:0:32-S32";
     resetDataLayout(Layout, IsWinCOFF ? "_" : "");
@@ -621,8 +624,8 @@ class LLVM_LIBRARY_VISIBILITY CygwinX86_32TargetInfo : public X86_32TargetInfo {
       : X86_32TargetInfo(Triple, Opts) {
     this->WCharType = TargetInfo::UnsignedShort;
     DoubleAlign = LongLongAlign = 64;
-    resetDataLayout("e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:"
-                    "32-n8:16:32-a:0:32-S32",
+    resetDataLayout("e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-"
+                    "i128:128-f80:32-n8:16:32-a:0:32-S32",
                     "_");
   }
 
@@ -660,8 +663,8 @@ class LLVM_LIBRARY_VISIBILITY MCUX86_32TargetInfo : public X86_32TargetInfo {
       : X86_32TargetInfo(Triple, Opts) {
     LongDoubleWidth = 64;
     LongDoubleFormat = &llvm::APFloat::IEEEdouble();
-    resetDataLayout("e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:32-f64:"
-                    "32-f128:32-n8:16:32-a:0:32-S32");
+    resetDataLayout("e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:32-"
+                    "f64:32-f128:32-n8:16:32-a:0:32-S32");
     WIntType = UnsignedInt;
   }
 
@@ -721,11 +724,11 @@ class LLVM_LIBRARY_VISIBILITY X86_64TargetInfo : public X86TargetInfo {
 
     // Pointers are 32-bit in x32.
     resetDataLayout(IsX32 ? "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-"
-                            "i64:64-f80:128-n8:16:32:64-S128"
-                          : IsWinCOFF ? "e-m:w-p270:32:32-p271:32:32-p272:64:"
-                                        "64-i64:64-f80:128-n8:16:32:64-S128"
-                                      : "e-m:e-p270:32:32-p271:32:32-p272:64:"
-                                        "64-i64:64-f80:128-n8:16:32:64-S128");
+                            "i64:64-i128:128-f80:128-n8:16:32:64-S128"
+                    : IsWinCOFF ? "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:"
+                                  "64-i128:128-f80:128-n8:16:32:64-S128"
+                                : "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:"
+                                  "64-i128:128-f80:128-n8:16:32:64-S128");
 
     // Use fpret only for long double.
     RealTypeUsesObjCFPRetMask = (unsigned)FloatModeKind::LongDouble;
@@ -922,8 +925,9 @@ class LLVM_LIBRARY_VISIBILITY DarwinX86_64TargetInfo
     llvm::Triple T = llvm::Triple(Triple);
     if (T.isiOS())
       UseSignedCharForObjCBool = false;
-    resetDataLayout("e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:"
-                    "16:32:64-S128", "_");
+    resetDataLayout("e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-"
+                    "f80:128-n8:16:32:64-S128",
+                    "_");
   }
 
   bool handleTargetFeatures(std::vector<std::string> &Features,
diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp
index d066819871..70accce456 100644
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -78,6 +78,7 @@
 #include "llvm/Transforms/Scalar/EarlyCSE.h"
 #include "llvm/Transforms/Scalar/GVN.h"
 #include "llvm/Transforms/Scalar/JumpThreading.h"
+#include "llvm/Transforms/HipStdPar/HipStdPar.h"
 #include "llvm/Transforms/Utils/Debugify.h"
 #include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
 #include "llvm/Transforms/Utils/ModuleUtils.h"
@@ -1108,6 +1109,10 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
     return;
   }
 
+  if (LangOpts.HIPStdPar && !LangOpts.CUDAIsDevice &&
+      LangOpts.HIPStdParInterposeAlloc)
+    MPM.addPass(HipStdParAllocationInterpositionPass());
+
   // Now that we have all of the passes ready, run them.
   {
     PrettyStackTraceString CrashInfo("Optimizer");
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 8cb7943df9..e1211bb894 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -66,6 +66,11 @@ using namespace clang;
 using namespace CodeGen;
 using namespace llvm;
 
+static llvm::cl::opt<bool> ClSanitizeAlignmentBuiltin(
+    "sanitize-alignment-builtin", llvm::cl::Hidden,
+    llvm::cl::desc("Instrument builtin functions for -fsanitize=alignment"),
+    llvm::cl::init(true));
+
 static void initializeAlloca(CodeGenFunction &CGF, AllocaInst *AI, Value *Size,
                              Align AlignmentInBytes) {
   ConstantInt *Byte;
@@ -853,6 +858,57 @@ CodeGenFunction::emitBuiltinObjectSize(const Expr *E, unsigned Type,
     }
   }
 
+  if (IsDynamic) {
+    LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
+        getLangOpts().getStrictFlexArraysLevel();
+    const Expr *Base = E->IgnoreParenImpCasts();
+
+    if (FieldDecl *FD = FindCountedByField(Base, StrictFlexArraysLevel)) {
+      const auto *ME = dyn_cast<MemberExpr>(Base);
+      llvm::Value *ObjectSize = nullptr;
+
+      if (!ME) {
+        const auto *DRE = dyn_cast<DeclRefExpr>(Base);
+        ValueDecl *VD = nullptr;
+
+        ObjectSize = ConstantInt::get(
+            ResType,
+            getContext().getTypeSize(DRE->getType()->getPointeeType()) / 8,
+            true);
+
+        if (auto *RD = DRE->getType()->getPointeeType()->getAsRecordDecl())
+          VD = RD->getLastField();
+
+        Expr *ICE = ImplicitCastExpr::Create(
+            getContext(), DRE->getType(), CK_LValueToRValue,
+            const_cast<Expr *>(cast<Expr>(DRE)), nullptr, VK_PRValue,
+            FPOptionsOverride());
+        ME = MemberExpr::CreateImplicit(getContext(), ICE, true, VD,
+                                        VD->getType(), VK_LValue, OK_Ordinary);
+      }
+
+      // At this point, we know that \p ME is a flexible array member.
+      const auto *ArrayTy = getContext().getAsArrayType(ME->getType());
+      unsigned Size = getContext().getTypeSize(ArrayTy->getElementType());
+
+      llvm::Value *CountField =
+          EmitAnyExprToTemp(MemberExpr::CreateImplicit(
+                                getContext(), const_cast<Expr *>(ME->getBase()),
+                                ME->isArrow(), FD, FD->getType(), VK_LValue,
+                                OK_Ordinary))
+              .getScalarVal();
+
+      llvm::Value *Mul = Builder.CreateMul(
+          CountField, llvm::ConstantInt::get(CountField->getType(), Size / 8));
+      Mul = Builder.CreateZExtOrTrunc(Mul, ResType);
+
+      if (ObjectSize)
+        return Builder.CreateAdd(ObjectSize, Mul);
+
+      return Mul;
+    }
+  }
+
   // LLVM can't handle Type=3 appropriately, and __builtin_object_size shouldn't
   // evaluate E for side-effects. In either case, we shouldn't lower to
   // @llvm.objectsize.
@@ -2276,6 +2332,19 @@ static Value *tryUseTestFPKind(CodeGenFunction &CGF, unsigned BuiltinID,
   return nullptr;
 }
 
+static RValue EmitHipStdParUnsupportedBuiltin(CodeGenFunction *CGF,
+                                              const FunctionDecl *FD) {
+  auto Name = FD->getNameAsString() + "__hipstdpar_unsupported";
+  auto FnTy = CGF->CGM.getTypes().GetFunctionType(FD);
+  auto UBF = CGF->CGM.getModule().getOrInsertFunction(Name, FnTy);
+
+  SmallVector<Value *, 16> Args;
+  for (auto &&FormalTy : FnTy->params())
+    Args.push_back(llvm::PoisonValue::get(FormalTy));
+
+  return RValue::get(CGF->Builder.CreateCall(UBF, Args));
+}
+
 RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
                                         const CallExpr *E,
                                         ReturnValueSlot ReturnValue) {
@@ -2737,7 +2806,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     EmitNonNullArgCheck(RValue::get(Val), Arg->getType(), Arg->getExprLoc(), FD,
                         ParmNum);
 
-    if (SanOpts.has(SanitizerKind::Alignment)) {
+    if (SanOpts.has(SanitizerKind::Alignment) && ClSanitizeAlignmentBuiltin) {
       SanitizerSet SkippedChecks;
       SkippedChecks.set(SanitizerKind::All);
       SkippedChecks.clear(SanitizerKind::Alignment);
@@ -5714,6 +5783,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     llvm_unreachable("Bad evaluation kind in EmitBuiltinExpr");
   }
 
+  if (getLangOpts().HIPStdPar && getLangOpts().CUDAIsDevice)
+    return EmitHipStdParUnsupportedBuiltin(this, FD);
+
   ErrorUnsupported(E, "builtin function");
 
   // Unknown builtin, for now just dump it out and return undef.
@@ -5724,6 +5796,16 @@ static Value *EmitTargetArchBuiltinExpr(CodeGenFunction *CGF,
                                         unsigned BuiltinID, const CallExpr *E,
                                         ReturnValueSlot ReturnValue,
                                         llvm::Triple::ArchType Arch) {
+  // When compiling in HipStdPar mode we have to be conservative in rejecting
+  // target specific features in the FE, and defer the possible error to the
+  // AcceleratorCodeSelection pass, wherein iff an unsupported target builtin is
+  // referenced by an accelerator executable function, we emit an error.
+  // Returning nullptr here leads to the builtin being handled in
+  // EmitStdParUnsupportedBuiltin.
+  if (CGF->getLangOpts().HIPStdPar && CGF->getLangOpts().CUDAIsDevice &&
+      Arch != CGF->getTarget().getTriple().getArch())
+    return nullptr;
+
   switch (Arch) {
   case llvm::Triple::arm:
   case llvm::Triple::armeb:
@@ -9258,6 +9340,11 @@ static llvm::ScalableVectorType *getSVEVectorForElementType(llvm::Type *EltTy) {
 // the elements of the specified datatype.
 Value *CodeGenFunction::EmitSVEPredicateCast(Value *Pred,
                                              llvm::ScalableVectorType *VTy) {
+
+  if (isa<TargetExtType>(Pred->getType()) &&
+      cast<TargetExtType>(Pred->getType())->getName() == "aarch64.svcount")
+    return Pred;
+
   auto *RTy = llvm::VectorType::get(IntegerType::get(getLLVMContext(), 1), VTy);
   if (Pred->getType() == RTy)
     return Pred;
@@ -9432,12 +9519,16 @@ Value *CodeGenFunction::EmitSVEStructLoad(const SVETypeFlags &TypeFlags,
   unsigned N;
   switch (IntID) {
   case Intrinsic::aarch64_sve_ld2_sret:
+  case Intrinsic::aarch64_sve_ld1_pn_x2:
+  case Intrinsic::aarch64_sve_ldnt1_pn_x2:
     N = 2;
     break;
   case Intrinsic::aarch64_sve_ld3_sret:
     N = 3;
     break;
   case Intrinsic::aarch64_sve_ld4_sret:
+  case Intrinsic::aarch64_sve_ld1_pn_x4:
+  case Intrinsic::aarch64_sve_ldnt1_pn_x4:
     N = 4;
     break;
   default:
@@ -9473,12 +9564,16 @@ Value *CodeGenFunction::EmitSVEStructStore(const SVETypeFlags &TypeFlags,
   unsigned N;
   switch (IntID) {
   case Intrinsic::aarch64_sve_st2:
+  case Intrinsic::aarch64_sve_st1_pn_x2:
+  case Intrinsic::aarch64_sve_stnt1_pn_x2:
     N = 2;
     break;
   case Intrinsic::aarch64_sve_st3:
     N = 3;
     break;
   case Intrinsic::aarch64_sve_st4:
+  case Intrinsic::aarch64_sve_st1_pn_x4:
+  case Intrinsic::aarch64_sve_stnt1_pn_x4:
     N = 4;
     break;
   default:
@@ -9617,11 +9712,6 @@ Value *CodeGenFunction::EmitSVEMaskedStore(const CallExpr *E,
   return Store;
 }
 
-Value *CodeGenFunction::EmitTileslice(Value *Offset, Value *Base) {
-  llvm::Value *CastOffset = Builder.CreateIntCast(Offset, Int32Ty, false);
-  return Builder.CreateAdd(Base, CastOffset, "tileslice");
-}
-
 Value *CodeGenFunction::EmitSMELd1St1(const SVETypeFlags &TypeFlags,
                                       SmallVectorImpl<Value *> &Ops,
                                       unsigned IntID) {
@@ -9680,13 +9770,13 @@ Value *CodeGenFunction::EmitSMELdrStr(const SVETypeFlags &TypeFlags,
   if (Ops.size() == 3) {
     Function *Cntsb = CGM.getIntrinsic(Intrinsic::aarch64_sme_cntsb);
     llvm::Value *CntsbCall = Builder.CreateCall(Cntsb, {}, "svlb");
-    llvm::Value *MulVL = Builder.CreateMul(
-        CntsbCall,
-        Builder.getInt64(cast<llvm::ConstantInt>(Ops[2])->getZExtValue()),
-        "mulvl");
+
+    llvm::Value *VecNum = Ops[2];
+    llvm::Value *MulVL = Builder.CreateMul(CntsbCall, VecNum, "mulvl");
 
     Ops[1] = Builder.CreateGEP(Int8Ty, Ops[1], MulVL);
-    Ops[0] = EmitTileslice(Ops[0], Ops[2]);
+    Ops[0] = Builder.CreateAdd(
+        Ops[0], Builder.CreateIntCast(VecNum, Int32Ty, true), "tileslice");
     Ops.erase(&Ops[2]);
   }
   Function *F = CGM.getIntrinsic(IntID, {});
@@ -9781,6 +9871,41 @@ Value *CodeGenFunction::EmitSVETupleCreate(const SVETypeFlags &TypeFlags,
   return Call;
 }
 
+Value *CodeGenFunction::FormSVEBuiltinResult(Value *Call) {
+  // Multi-vector results should be broken up into a single (wide) result
+  // vector.
+  auto *StructTy = dyn_cast<StructType>(Call->getType());
+  if (!StructTy)
+    return Call;
+
+  auto *VTy = dyn_cast<ScalableVectorType>(StructTy->getTypeAtIndex(0U));
+  if (!VTy)
+    return Call;
+  unsigned N = StructTy->getNumElements();
+
+  // We may need to emit a cast to a svbool_t
+  bool IsPredTy = VTy->getElementType()->isIntegerTy(1);
+  unsigned MinElts = IsPredTy ? 16 : VTy->getMinNumElements();
+
+  ScalableVectorType *WideVTy =
+      ScalableVectorType::get(VTy->getElementType(), MinElts * N);
+  Value *Ret = llvm::PoisonValue::get(WideVTy);
+  for (unsigned I = 0; I < N; ++I) {
+    Value *SRet = Builder.CreateExtractValue(Call, I);
+    assert(SRet->getType() == VTy && "Unexpected type for result value");
+    Value *Idx = ConstantInt::get(CGM.Int64Ty, I * MinElts);
+
+    if (IsPredTy)
+      SRet = EmitSVEPredicateCast(
+          SRet, ScalableVectorType::get(Builder.getInt1Ty(), 16));
+
+    Ret = Builder.CreateInsertVector(WideVTy, Ret, SRet, Idx);
+  }
+  Call = Ret;
+
+  return Call;
+}
+
 Value *CodeGenFunction::EmitAArch64SVEBuiltinExpr(unsigned BuiltinID,
                                                   const CallExpr *E) {
   // Find out if any arguments are required to be integer constant expressions.
@@ -9894,13 +10019,39 @@ Value *CodeGenFunction::EmitAArch64SVEBuiltinExpr(unsigned BuiltinID,
       if (PredTy->getScalarType()->isIntegerTy(1))
         Call = EmitSVEPredicateCast(Call, cast<llvm::ScalableVectorType>(Ty));
 
-    return Call;
+    return FormSVEBuiltinResult(Call);
   }
 
   switch (BuiltinID) {
   default:
     return nullptr;
-
+  case SVE::BI__builtin_sve_svpsel_lane_b8:
+  case SVE::BI__builtin_sve_svpsel_lane_b16:
+  case SVE::BI__builtin_sve_svpsel_lane_b32:
+  case SVE::BI__builtin_sve_svpsel_lane_b64:
+  case SVE::BI__builtin_sve_svpsel_lane_c8:
+  case SVE::BI__builtin_sve_svpsel_lane_c16:
+  case SVE::BI__builtin_sve_svpsel_lane_c32:
+  case SVE::BI__builtin_sve_svpsel_lane_c64: {
+    bool IsSVCount = isa<TargetExtType>(Ops[0]->getType());
+    assert(((!IsSVCount || cast<TargetExtType>(Ops[0]->getType())->getName() ==
+                               "aarch64.svcount")) &&
+           "Unexpected TargetExtType");
+    auto SVCountTy =
+        llvm::TargetExtType::get(getLLVMContext(), "aarch64.svcount");
+    Function *CastFromSVCountF =
+        CGM.getIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool, SVCountTy);
+    Function *CastToSVCountF =
+        CGM.getIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool, SVCountTy);
+
+    auto OverloadedTy = getSVEType(SVETypeFlags(Builtin->TypeModifier));
+    Function *F = CGM.getIntrinsic(Intrinsic::aarch64_sve_psel, OverloadedTy);
+    llvm::Value *Ops0 =
+        IsSVCount ? Builder.CreateCall(CastFromSVCountF, Ops[0]) : Ops[0];
+    llvm::Value *Ops1 = EmitSVEPredicateCast(Ops[1], OverloadedTy);
+    llvm::Value *PSel = Builder.CreateCall(F, {Ops0, Ops1, Ops[2]});
+    return IsSVCount ? Builder.CreateCall(CastToSVCountF, PSel) : PSel;
+  }
   case SVE::BI__builtin_sve_svmov_b_z: {
     // svmov_b_z(pg, op) <=> svand_b_z(pg, op, op)
     SVETypeFlags TypeFlags(Builtin->TypeModifier);
@@ -10022,6 +10173,13 @@ Value *CodeGenFunction::EmitAArch64SVEBuiltinExpr(unsigned BuiltinID,
   case SVE::BI__builtin_sve_svpfalse_b:
     return ConstantInt::getFalse(Ty);
 
+  case SVE::BI__builtin_sve_svpfalse_c: {
+    auto SVBoolTy = ScalableVectorType::get(Builder.getInt1Ty(), 16);
+    Function *CastToSVCountF =
+        CGM.getIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool, Ty);
+    return Builder.CreateCall(CastToSVCountF, ConstantInt::getFalse(SVBoolTy));
+  }
+
   case SVE::BI__builtin_sve_svlen_bf16:
   case SVE::BI__builtin_sve_svlen_f16:
   case SVE::BI__builtin_sve_svlen_f32:
@@ -10842,6 +11000,15 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
     return Result;
   }
 
+  if (BuiltinID == AArch64::BI__prefetch) {
+    Value *Address = EmitScalarExpr(E->getArg(0));
+    Value *RW = llvm::ConstantInt::get(Int32Ty, 0);
+    Value *Locality = ConstantInt::get(Int32Ty, 3);
+    Value *Data = llvm::ConstantInt::get(Int32Ty, 1);
+    Function *F = CGM.getIntrinsic(Intrinsic::prefetch, Address->getType());
+    return Builder.CreateCall(F, {Address, RW, Locality, Data});
+  }
+
   // Handle MSVC intrinsics before argument evaluation to prevent double
   // evaluation.
   if (std::optional<MSVCIntrin> MsvcIntId =
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index 93e1657504..e1e1ad7e6a 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -72,6 +72,7 @@ unsigned CodeGenTypes::ClangCallConvToLLVMCallConv(CallingConv CC) {
   case CC_PreserveAll: return llvm::CallingConv::PreserveAll;
   case CC_Swift: return llvm::CallingConv::Swift;
   case CC_SwiftAsync: return llvm::CallingConv::SwiftTail;
+  case CC_M68kRTD: return llvm::CallingConv::M68k_RTD;
   }
 }
 
@@ -252,6 +253,9 @@ static CallingConv getCallingConventionForDecl(const ObjCMethodDecl *D,
   if (D->hasAttr<PreserveAllAttr>())
     return CC_PreserveAll;
 
+  if (D->hasAttr<M68kRTDAttr>())
+    return CC_M68kRTD;
+
   return CC_C;
 }
 
@@ -3443,9 +3447,9 @@ static llvm::Value *tryRemoveRetainOfSelf(CodeGenFunction &CGF,
   const VarDecl *self = method->getSelfDecl();
   if (!self->getType().isConstQualified()) return nullptr;
 
-  // Look for a retain call. Note: stripPointerCasts looks through returned arg
-  // functions, which would cause us to miss the retain.
-  llvm::CallInst *retainCall = dyn_cast<llvm::CallInst>(result);
+  // Look for a retain call.
+  llvm::CallInst *retainCall =
+    dyn_cast<llvm::CallInst>(result->stripPointerCasts());
   if (!retainCall || retainCall->getCalledOperand() !=
                          CGF.CGM.getObjCEntrypoints().objc_retain)
     return nullptr;
diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp
index c73a63e12f..c430713b0d 100644
--- a/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -1445,6 +1445,8 @@ static unsigned getDwarfCC(CallingConv CC) {
     return llvm::dwarf::DW_CC_LLVM_PreserveAll;
   case CC_X86RegCall:
     return llvm::dwarf::DW_CC_LLVM_X86RegCall;
+  case CC_M68kRTD:
+    return llvm::dwarf::DW_CC_LLVM_M68kRTD;
   }
   return 0;
 }
diff --git a/clang/lib/CodeGen/CGDeclCXX.cpp b/clang/lib/CodeGen/CGDeclCXX.cpp
index 30683ad474..e106171627 100644
--- a/clang/lib/CodeGen/CGDeclCXX.cpp
+++ b/clang/lib/CodeGen/CGDeclCXX.cpp
@@ -675,7 +675,7 @@ void CodeGenModule::EmitCXXModuleInitFunc(Module *Primary) {
     AllImports.insert(M);
   // Ones that we import in the global module fragment or the private module
   // fragment.
-  llvm::for_each(Primary->submodules(), [&AllImports](Module *SubM) {
+  for (Module *SubM : Primary->submodules()) {
     assert((SubM->isGlobalModule() || SubM->isPrivateModule()) &&
            "The sub modules of C++20 module unit should only be global module "
            "fragments or private module framents.");
@@ -684,7 +684,7 @@ void CodeGenModule::EmitCXXModuleInitFunc(Module *Primary) {
            "not allowed to export import modules.");
     for (Module *M : SubM->Imports)
       AllImports.insert(M);
-  });
+  }
 
   SmallVector<llvm::Function *, 8> ModuleInits;
   for (Module *M : AllImports) {
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 1b6a2c1fc4..54a1d300a9 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -30,6 +30,7 @@
 #include "clang/Basic/CodeGenOptions.h"
 #include "clang/Basic/SourceManager.h"
 #include "llvm/ADT/Hashing.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/Intrinsics.h"
@@ -931,16 +932,31 @@ static llvm::Value *getArrayIndexingBound(CodeGenFunction &CGF,
     if (CE->getCastKind() == CK_ArrayToPointerDecay &&
         !CE->getSubExpr()->isFlexibleArrayMemberLike(CGF.getContext(),
                                                      StrictFlexArraysLevel)) {
+      CodeGenFunction::SanitizerScope SanScope(&CGF);
+
       IndexedType = CE->getSubExpr()->getType();
       const ArrayType *AT = IndexedType->castAsArrayTypeUnsafe();
       if (const auto *CAT = dyn_cast<ConstantArrayType>(AT))
         return CGF.Builder.getInt(CAT->getSize());
-      else if (const auto *VAT = dyn_cast<VariableArrayType>(AT))
+
+      if (const auto *VAT = dyn_cast<VariableArrayType>(AT))
         return CGF.getVLASize(VAT).NumElts;
       // Ignore pass_object_size here. It's not applicable on decayed pointers.
     }
+
+    if (FieldDecl *FD = CGF.FindCountedByField(Base, StrictFlexArraysLevel)) {
+      const auto *ME = dyn_cast<MemberExpr>(CE->getSubExpr());
+      IndexedType = Base->getType();
+      return CGF
+          .EmitAnyExprToTemp(MemberExpr::CreateImplicit(
+              CGF.getContext(), const_cast<Expr *>(ME->getBase()),
+              ME->isArrow(), FD, FD->getType(), VK_LValue, OK_Ordinary))
+          .getScalarVal();
+    }
   }
 
+  CodeGenFunction::SanitizerScope SanScope(&CGF);
+
   QualType EltTy{Base->getType()->getPointeeOrArrayElementType(), 0};
   if (llvm::Value *POS = CGF.LoadPassedObjectSize(Base, EltTy)) {
     IndexedType = Base->getType();
@@ -950,13 +966,53 @@ static llvm::Value *getArrayIndexingBound(CodeGenFunction &CGF,
   return nullptr;
 }
 
+FieldDecl *CodeGenFunction::FindCountedByField(
+    const Expr *Base,
+    LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel) {
+  const ValueDecl *VD = nullptr;
+
+  Base = Base->IgnoreParenImpCasts();
+
+  if (const auto *ME = dyn_cast<MemberExpr>(Base)) {
+    VD = dyn_cast<ValueDecl>(ME->getMemberDecl());
+  } else if (const auto *DRE = dyn_cast<DeclRefExpr>(Base)) {
+    // Pointing to the full structure.
+    VD = dyn_cast<ValueDecl>(DRE->getDecl());
+
+    QualType Ty = VD->getType();
+    if (Ty->isPointerType())
+      Ty = Ty->getPointeeType();
+
+    if (const auto *RD = Ty->getAsRecordDecl())
+      VD = RD->getLastField();
+  } else if (const auto *CE = dyn_cast<CastExpr>(Base)) {
+    if (const auto *ME = dyn_cast<MemberExpr>(CE->getSubExpr()))
+      VD = dyn_cast<ValueDecl>(ME->getMemberDecl());
+  }
+
+  const auto *FD = dyn_cast_if_present<FieldDecl>(VD);
+  if (!FD || !FD->getParent() ||
+      !Decl::isFlexibleArrayMemberLike(getContext(), FD, FD->getType(),
+                                       StrictFlexArraysLevel, true))
+    return nullptr;
+
+  const auto *CBA = FD->getAttr<CountedByAttr>();
+  if (!CBA)
+    return nullptr;
+
+  StringRef FieldName = CBA->getCountedByField()->getName();
+  auto It =
+      llvm::find_if(FD->getParent()->fields(), [&](const FieldDecl *Field) {
+        return FieldName == Field->getName();
+      });
+  return It != FD->getParent()->field_end() ? *It : nullptr;
+}
+
 void CodeGenFunction::EmitBoundsCheck(const Expr *E, const Expr *Base,
                                       llvm::Value *Index, QualType IndexType,
                                       bool Accessed) {
   assert(SanOpts.has(SanitizerKind::ArrayBounds) &&
          "should not be called unless adding bounds checks");
-  SanitizerScope SanScope(this);
-
   const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
     getLangOpts().getStrictFlexArraysLevel();
 
@@ -966,6 +1022,8 @@ void CodeGenFunction::EmitBoundsCheck(const Expr *E, const Expr *Base,
   if (!Bound)
     return;
 
+  SanitizerScope SanScope(this);
+
   bool IndexSigned = IndexType->isSignedIntegerOrEnumerationType();
   llvm::Value *IndexVal = Builder.CreateIntCast(Index, SizeTy, IndexSigned);
   llvm::Value *BoundVal = Builder.CreateIntCast(Bound, SizeTy, false);
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index 93ab064bdf..c25ddeff9a 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -3083,6 +3083,9 @@ ScalarExprEmitter::VisitUnaryExprOrTypeTraitExpr(
                 E->getTypeOfArgument()->getPointeeType()))
             .getQuantity();
     return llvm::ConstantInt::get(CGF.SizeTy, Alignment);
+  } else if (E->getKind() == UETT_VectorElements) {
+    auto *VecTy = cast<llvm::VectorType>(ConvertType(E->getTypeOfArgument()));
+    return Builder.CreateElementCount(CGF.SizeTy, VecTy->getElementCount());
   }
 
   // If this isn't sizeof(vla), the result must be constant; use the constant
diff --git a/clang/lib/CodeGen/CGObjC.cpp b/clang/lib/CodeGen/CGObjC.cpp
index 6c594b5db4..aa3a0ff570 100644
--- a/clang/lib/CodeGen/CGObjC.cpp
+++ b/clang/lib/CodeGen/CGObjC.cpp
@@ -3954,7 +3954,7 @@ static unsigned getBaseMachOPlatformID(const llvm::Triple &TT) {
   case llvm::Triple::DriverKit:
     return llvm::MachO::PLATFORM_DRIVERKIT;
   default:
-    return /*Unknown platform*/ 0;
+    return llvm::MachO::PLATFORM_UNKNOWN;
   }
 }
 
diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp
index 6674aa2409..c719df1bfa 100644
--- a/clang/lib/CodeGen/CGStmt.cpp
+++ b/clang/lib/CodeGen/CGStmt.cpp
@@ -2420,6 +2420,24 @@ EmitAsmStores(CodeGenFunction &CGF, const AsmStmt &S,
   }
 }
 
+static void EmitHipStdParUnsupportedAsm(CodeGenFunction *CGF,
+                                        const AsmStmt &S) {
+  constexpr auto Name = "__ASM__hipstdpar_unsupported";
+
+  StringRef Asm;
+  if (auto GCCAsm = dyn_cast<GCCAsmStmt>(&S))
+    Asm = GCCAsm->getAsmString()->getString();
+
+  auto &Ctx = CGF->CGM.getLLVMContext();
+
+  auto StrTy = llvm::ConstantDataArray::getString(Ctx, Asm);
+  auto FnTy = llvm::FunctionType::get(llvm::Type::getVoidTy(Ctx),
+                                      {StrTy->getType()}, false);
+  auto UBF = CGF->CGM.getModule().getOrInsertFunction(Name, FnTy);
+
+  CGF->Builder.CreateCall(UBF, {StrTy});
+}
+
 void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
   // Pop all cleanup blocks at the end of the asm statement.
   CodeGenFunction::RunCleanupsScope Cleanups(*this);
@@ -2431,27 +2449,38 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
   SmallVector<TargetInfo::ConstraintInfo, 4> OutputConstraintInfos;
   SmallVector<TargetInfo::ConstraintInfo, 4> InputConstraintInfos;
 
-  for (unsigned i = 0, e = S.getNumOutputs(); i != e; i++) {
+  bool IsHipStdPar = getLangOpts().HIPStdPar && getLangOpts().CUDAIsDevice;
+  bool IsValidTargetAsm = true;
+  for (unsigned i = 0, e = S.getNumOutputs(); i != e && IsValidTargetAsm; i++) {
     StringRef Name;
     if (const GCCAsmStmt *GAS = dyn_cast<GCCAsmStmt>(&S))
       Name = GAS->getOutputName(i);
     TargetInfo::ConstraintInfo Info(S.getOutputConstraint(i), Name);
     bool IsValid = getTarget().validateOutputConstraint(Info); (void)IsValid;
-    assert(IsValid && "Failed to parse output constraint");
+    if (IsHipStdPar && !IsValid)
+      IsValidTargetAsm = false;
+    else
+      assert(IsValid && "Failed to parse output constraint");
     OutputConstraintInfos.push_back(Info);
   }
 
-  for (unsigned i = 0, e = S.getNumInputs(); i != e; i++) {
+  for (unsigned i = 0, e = S.getNumInputs(); i != e && IsValidTargetAsm; i++) {
     StringRef Name;
     if (const GCCAsmStmt *GAS = dyn_cast<GCCAsmStmt>(&S))
       Name = GAS->getInputName(i);
     TargetInfo::ConstraintInfo Info(S.getInputConstraint(i), Name);
     bool IsValid =
       getTarget().validateInputConstraint(OutputConstraintInfos, Info);
-    assert(IsValid && "Failed to parse input constraint"); (void)IsValid;
+    if (IsHipStdPar && !IsValid)
+      IsValidTargetAsm = false;
+    else
+      assert(IsValid && "Failed to parse input constraint");
     InputConstraintInfos.push_back(Info);
   }
 
+  if (!IsValidTargetAsm)
+    return EmitHipStdParUnsupportedAsm(this, S);
+
   std::string Constraints;
 
   std::vector<LValue> ResultRegDests;
diff --git a/clang/lib/CodeGen/CMakeLists.txt b/clang/lib/CodeGen/CMakeLists.txt
index 1debeb6d9c..d67ce982d7 100644
--- a/clang/lib/CodeGen/CMakeLists.txt
+++ b/clang/lib/CodeGen/CMakeLists.txt
@@ -11,6 +11,7 @@ set(LLVM_LINK_COMPONENTS
   Extensions
   FrontendHLSL
   FrontendOpenMP
+  HIPStdPar
   IPO
   IRPrinter
   IRReader
@@ -30,6 +31,26 @@ set(LLVM_LINK_COMPONENTS
   TransformUtils
   )
 
+# Workaround for MSVC ARM64 performance regression:
+# https://developercommunity.visualstudio.com/t/Compiling-a-specific-code-for-ARM64-with/10444970
+# Since /O1 and /O2 represent a set of optimizations,
+# our goal is to disable the /Og flag while retaining the other optimizations from the /O1|/O2 set
+if(MSVC AND NOT CMAKE_CXX_COMPILER_ID MATCHES Clang
+    AND MSVC_VERSION VERSION_GREATER_EQUAL 1932
+    AND CMAKE_SYSTEM_PROCESSOR MATCHES "ARM64")
+
+  string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE)
+  string(REGEX MATCHALL "/[Oo][12]" opt_flags "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${uppercase_CMAKE_BUILD_TYPE}}")
+  if (opt_flags)
+    if(opt_flags MATCHES "1$")
+      set(opt_flags "/Od;/Os;/Oy;/Ob2;/GF;/Gy")
+    elseif (opt_flags MATCHES "2$")
+      set(opt_flags "/Od;/Oi;/Ot;/Oy;/Ob2;/GF;/Gy")
+    endif()
+    set_source_files_properties(CGBuiltin.cpp PROPERTIES COMPILE_OPTIONS "${opt_flags}")
+  endif()
+endif()
+
 add_clang_library(clangCodeGen
   ABIInfo.cpp
   ABIInfoImpl.cpp
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index 9b21f428b0..3682a2c6ae 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -1216,11 +1216,10 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
       SkippedChecks.set(SanitizerKind::ObjectSize, true);
       QualType ThisTy = MD->getThisType();
 
-      // If this is the call operator of a lambda with no capture-default, it
+      // If this is the call operator of a lambda with no captures, it
       // may have a static invoker function, which may call this operator with
       // a null 'this' pointer.
-      if (isLambdaCallOperator(MD) &&
-          MD->getParent()->getLambdaCaptureDefault() == LCD_None)
+      if (isLambdaCallOperator(MD) && MD->getParent()->isCapturelessLambda())
         SkippedChecks.set(SanitizerKind::Null, true);
 
       EmitTypeCheck(
@@ -2595,10 +2594,15 @@ void CodeGenFunction::checkTargetFeatures(SourceLocation Loc,
   std::string MissingFeature;
   llvm::StringMap<bool> CallerFeatureMap;
   CGM.getContext().getFunctionFeatureMap(CallerFeatureMap, FD);
+  // When compiling in HipStdPar mode we have to be conservative in rejecting
+  // target specific features in the FE, and defer the possible error to the
+  // AcceleratorCodeSelection pass, wherein iff an unsupported target builtin is
+  // referenced by an accelerator executable function, we emit an error.
+  bool IsHipStdPar = getLangOpts().HIPStdPar && getLangOpts().CUDAIsDevice;
   if (BuiltinID) {
     StringRef FeatureList(CGM.getContext().BuiltinInfo.getRequiredFeatures(BuiltinID));
     if (!Builtin::evaluateRequiredTargetFeatures(
-        FeatureList, CallerFeatureMap)) {
+        FeatureList, CallerFeatureMap) && !IsHipStdPar) {
       CGM.getDiags().Report(Loc, diag::err_builtin_needs_feature)
           << TargetDecl->getDeclName()
           << FeatureList;
@@ -2631,7 +2635,7 @@ void CodeGenFunction::checkTargetFeatures(SourceLocation Loc,
         return false;
       }
       return true;
-    }))
+    }) && !IsHipStdPar)
       CGM.getDiags().Report(Loc, diag::err_function_needs_feature)
           << FD->getDeclName() << TargetDecl->getDeclName() << MissingFeature;
   } else if (!FD->isMultiVersion() && FD->hasAttr<TargetAttr>()) {
@@ -2640,7 +2644,8 @@ void CodeGenFunction::checkTargetFeatures(SourceLocation Loc,
 
     for (const auto &F : CalleeFeatureMap) {
       if (F.getValue() && (!CallerFeatureMap.lookup(F.getKey()) ||
-                           !CallerFeatureMap.find(F.getKey())->getValue()))
+                           !CallerFeatureMap.find(F.getKey())->getValue()) &&
+          !IsHipStdPar)
         CGM.getDiags().Report(Loc, diag::err_function_needs_feature)
             << FD->getDeclName() << TargetDecl->getDeclName() << F.getKey();
     }
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 471aad94e1..e82115e2d7 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -3022,6 +3022,12 @@ class CodeGenFunction : public CodeGenTypeCache {
   void EmitBoundsCheck(const Expr *E, const Expr *Base, llvm::Value *Index,
                        QualType IndexType, bool Accessed);
 
+  /// Find the FieldDecl specified in a FAM's "counted_by" attribute. Returns
+  /// \p nullptr if either the attribute or the field doesn't exist.
+  FieldDecl *FindCountedByField(
+      const Expr *Base,
+      LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel);
+
   llvm::Value *EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
                                        bool isInc, bool isPre);
   ComplexPairTy EmitComplexPrePostIncDec(const UnaryOperator *E, LValue LV,
@@ -4274,7 +4280,6 @@ class CodeGenFunction : public CodeGenTypeCache {
   llvm::Value *EmitSVEMaskedStore(const CallExpr *,
                                   SmallVectorImpl<llvm::Value *> &Ops,
                                   unsigned BuiltinID);
-  llvm::Value *EmitTileslice(llvm::Value *Offset, llvm::Value *Base);
   llvm::Value *EmitSVEPrefetchLoad(const SVETypeFlags &TypeFlags,
                                    SmallVectorImpl<llvm::Value *> &Ops,
                                    unsigned BuiltinID);
@@ -4287,6 +4292,11 @@ class CodeGenFunction : public CodeGenTypeCache {
   llvm::Value *EmitSVEStructStore(const SVETypeFlags &TypeFlags,
                                   SmallVectorImpl<llvm::Value *> &Ops,
                                   unsigned IntID);
+  /// FormSVEBuiltinResult - Returns the struct of scalable vectors as a wider
+  /// vector. It extracts the scalable vector from the struct and inserts into
+  /// the wider vector. This avoids the error when allocating space in llvm
+  /// for struct of scalable vectors if a function returns struct.
+  llvm::Value *FormSVEBuiltinResult(llvm::Value *Call);
   llvm::Value *EmitAArch64SVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
 
   llvm::Value *EmitSMELd1St1(const SVETypeFlags &TypeFlags,
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index 754377bed7..b1a6683a66 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -3526,7 +3526,7 @@ ConstantAddress CodeGenModule::GetAddrOfTemplateParamObject(
     GV->setComdat(TheModule.getOrInsertComdat(GV->getName()));
   Emitter.finalize(GV);
 
-  return ConstantAddress(GV, GV->getValueType(), Alignment);
+    return ConstantAddress(GV, GV->getValueType(), Alignment);
 }
 
 ConstantAddress CodeGenModule::GetWeakRefReference(const ValueDecl *VD) {
@@ -3585,7 +3585,10 @@ void CodeGenModule::EmitGlobal(GlobalDecl GD) {
           !Global->hasAttr<CUDAConstantAttr>() &&
           !Global->hasAttr<CUDASharedAttr>() &&
           !Global->getType()->isCUDADeviceBuiltinSurfaceType() &&
-          !Global->getType()->isCUDADeviceBuiltinTextureType())
+          !Global->getType()->isCUDADeviceBuiltinTextureType() &&
+          !(LangOpts.HIPStdPar &&
+            isa<FunctionDecl>(Global) &&
+            !Global->hasAttr<CUDAHostAttr>()))
         return;
     } else {
       // We need to emit host-side 'shadows' for all global
diff --git a/clang/lib/CodeGen/CodeGenPGO.cpp b/clang/lib/CodeGen/CodeGenPGO.cpp
index df6c76cde9..63cdd0a047 100644
--- a/clang/lib/CodeGen/CodeGenPGO.cpp
+++ b/clang/lib/CodeGen/CodeGenPGO.cpp
@@ -755,7 +755,8 @@ void PGOHash::combine(HashType Type) {
   // Pass through MD5 if enough work has built up.
   if (Count && Count % NumTypesPerWord == 0) {
     using namespace llvm::support;
-    uint64_t Swapped = endian::byte_swap<uint64_t, little>(Working);
+    uint64_t Swapped =
+        endian::byte_swap<uint64_t, llvm::endianness::little>(Working);
     MD5.update(llvm::ArrayRef((uint8_t *)&Swapped, sizeof(Swapped)));
     Working = 0;
   }
@@ -781,7 +782,8 @@ uint64_t PGOHash::finalize() {
       MD5.update({(uint8_t)Working});
     } else {
       using namespace llvm::support;
-      uint64_t Swapped = endian::byte_swap<uint64_t, little>(Working);
+      uint64_t Swapped =
+          endian::byte_swap<uint64_t, llvm::endianness::little>(Working);
       MD5.update(llvm::ArrayRef((uint8_t *)&Swapped, sizeof(Swapped)));
     }
   }
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index 77328e1f99..f5fd900a64 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -2589,8 +2589,11 @@ void Driver::BuildInputs(const ToolChain &TC, DerivedArgList &Args,
       Diag(clang::diag::note_drv_t_option_is_global);
   }
 
+  // CUDA/HIP and their preprocessor expansions can be accepted by CL mode.
   // Warn -x after last input file has no effect
-  if (!IsCLMode()) {
+  auto LastXArg = Args.getLastArgValue(options::OPT_x);
+  const llvm::StringSet<> ValidXArgs = {"cuda", "hip", "cui", "hipi"};
+  if (!IsCLMode() || ValidXArgs.find(LastXArg) != ValidXArgs.end()) {
     Arg *LastXArg = Args.getLastArgNoClaim(options::OPT_x);
     Arg *LastInputArg = Args.getLastArgNoClaim(options::OPT_INPUT);
     if (LastXArg && LastInputArg &&
diff --git a/clang/lib/Driver/Multilib.cpp b/clang/lib/Driver/Multilib.cpp
index a37dffc8a6..48a494d9fa 100644
--- a/clang/lib/Driver/Multilib.cpp
+++ b/clang/lib/Driver/Multilib.cpp
@@ -10,7 +10,6 @@
 #include "clang/Basic/LLVM.h"
 #include "clang/Basic/Version.h"
 #include "llvm/ADT/SmallString.h"
-#include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
@@ -123,9 +122,8 @@ MultilibSet::expandFlags(const Multilib::flags_list &InFlags) const {
 
     const llvm::Regex Regex(RegexString);
     assert(Regex.isValid());
-    if (llvm::find_if(InFlags, [&Regex](StringRef F) {
-          return Regex.match(F);
-        }) != InFlags.end()) {
+    if (llvm::any_of(InFlags,
+                     [&Regex](StringRef F) { return Regex.match(F); })) {
       Result.insert(M.Flags.begin(), M.Flags.end());
     }
   }
diff --git a/clang/lib/Driver/OffloadBundler.cpp b/clang/lib/Driver/OffloadBundler.cpp
index 0ad9de2996..a7bc15e87e 100644
--- a/clang/lib/Driver/OffloadBundler.cpp
+++ b/clang/lib/Driver/OffloadBundler.cpp
@@ -324,7 +324,7 @@ static uint64_t Read8byteIntegerFromBuffer(StringRef Buffer, size_t pos) {
 
 /// Write 8-byte integers to a buffer in little-endian format.
 static void Write8byteIntegerToBuffer(raw_ostream &OS, uint64_t Val) {
-  llvm::support::endian::write(OS, Val, llvm::support::little);
+  llvm::support::endian::write(OS, Val, llvm::endianness::little);
 }
 
 class BinaryFileHandler final : public FileHandler {
diff --git a/clang/lib/Driver/ToolChains/AIX.h b/clang/lib/Driver/ToolChains/AIX.h
index cc74e5ea85..04f9b22409 100644
--- a/clang/lib/Driver/ToolChains/AIX.h
+++ b/clang/lib/Driver/ToolChains/AIX.h
@@ -16,7 +16,7 @@ namespace clang {
 namespace driver {
 namespace tools {
 
-/// aix -- Directly call system default assembler and linker.
+/// Directly call system default assembler and linker.
 namespace aix {
 
 class LLVM_LIBRARY_VISIBILITY Assembler : public Tool {
diff --git a/clang/lib/Driver/ToolChains/AVR.cpp b/clang/lib/Driver/ToolChains/AVR.cpp
index 81f501d417..27505b9462 100644
--- a/clang/lib/Driver/ToolChains/AVR.cpp
+++ b/clang/lib/Driver/ToolChains/AVR.cpp
@@ -457,7 +457,8 @@ void AVR::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   CmdArgs.push_back(Output.getFilename());
 
   // Enable garbage collection of unused sections.
-  CmdArgs.push_back("--gc-sections");
+  if (!Args.hasArg(options::OPT_r))
+    CmdArgs.push_back("--gc-sections");
 
   // Add library search paths before we specify libraries.
   Args.AddAllArgs(CmdArgs, options::OPT_L);
@@ -471,7 +472,7 @@ void AVR::Linker::ConstructJob(Compilation &C, const JobAction &JA,
 
   // Only add default libraries if the user hasn't explicitly opted out.
   bool LinkStdlib = false;
-  if (!Args.hasArg(options::OPT_nostdlib) &&
+  if (!Args.hasArg(options::OPT_nostdlib) && !Args.hasArg(options::OPT_r) &&
       !Args.hasArg(options::OPT_nodefaultlibs)) {
     if (!CPU.empty()) {
       if (!FamilyName) {
@@ -497,13 +498,17 @@ void AVR::Linker::ConstructJob(Compilation &C, const JobAction &JA,
       D.Diag(diag::warn_drv_avr_stdlib_not_linked);
   }
 
-  if (SectionAddressData) {
-    CmdArgs.push_back(
-        Args.MakeArgString("--defsym=__DATA_REGION_ORIGIN__=0x" +
-                           Twine::utohexstr(*SectionAddressData)));
-  } else {
-    // We do not have an entry for this CPU in the address mapping table yet.
-    D.Diag(diag::warn_drv_avr_linker_section_addresses_not_implemented) << CPU;
+  if (!Args.hasArg(options::OPT_r)) {
+    if (SectionAddressData) {
+      CmdArgs.push_back(
+          Args.MakeArgString("--defsym=__DATA_REGION_ORIGIN__=0x" +
+                             Twine::utohexstr(*SectionAddressData)));
+    } else {
+      // We do not have an entry for this CPU in the address mapping table
+      // yet.
+      D.Diag(diag::warn_drv_avr_linker_section_addresses_not_implemented)
+          << CPU;
+    }
   }
 
   if (D.isUsingLTO()) {
@@ -549,22 +554,32 @@ void AVR::Linker::ConstructJob(Compilation &C, const JobAction &JA,
 
     CmdArgs.push_back("--end-group");
 
-    // Add user specified linker script.
-    Args.AddAllArgs(CmdArgs, options::OPT_T);
+    // Add avr-libc's linker script to lld by default, if it exists.
+    if (!Args.hasArg(options::OPT_T) &&
+        Linker.find("avr-ld") == std::string::npos) {
+      std::string Path(*AVRLibcRoot + "/lib/ldscripts/");
+      Path += *FamilyName;
+      Path += ".x";
+      if (llvm::sys::fs::exists(Path))
+        CmdArgs.push_back(Args.MakeArgString("-T" + Path));
+    }
+    // Otherwise add user specified linker script to either avr-ld or lld.
+    else
+      Args.AddAllArgs(CmdArgs, options::OPT_T);
 
     if (Args.hasFlag(options::OPT_mrelax, options::OPT_mno_relax, true))
       CmdArgs.push_back("--relax");
-
-    // Specify the family name as the emulation mode to use.
-    // This is almost always required because otherwise avr-ld
-    // will assume 'avr2' and warn about the program being larger
-    // than the bare minimum supports.
-    if (Linker.find("avr-ld") != std::string::npos)
-      CmdArgs.push_back(Args.MakeArgString(std::string("-m") + *FamilyName));
   } else {
     AddLinkerInputs(getToolChain(), Inputs, Args, CmdArgs, JA);
   }
 
+  // Specify the family name as the emulation mode to use.
+  // This is almost always required because otherwise avr-ld
+  // will assume 'avr2' and warn about the program being larger
+  // than the bare minimum supports.
+  if (Linker.find("avr-ld") != std::string::npos && FamilyName)
+    CmdArgs.push_back(Args.MakeArgString(std::string("-m") + *FamilyName));
+
   C.addCommand(std::make_unique<Command>(
       JA, *this, ResponseFileSupport::AtFileCurCP(), Args.MakeArgString(Linker),
       CmdArgs, Inputs, Output));
diff --git a/clang/lib/Driver/ToolChains/Arch/ARM.cpp b/clang/lib/Driver/ToolChains/Arch/ARM.cpp
index 8e1cff0b44..f1d7aeb555 100644
--- a/clang/lib/Driver/ToolChains/Arch/ARM.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/ARM.cpp
@@ -627,6 +627,11 @@ llvm::ARM::FPUKind arm::getARMTargetFeatures(const Driver &D,
     if (!llvm::ARM::getFPUFeatures(FPUKind, Features))
       D.Diag(clang::diag::err_drv_clang_unsupported)
           << std::string("-mfpu=") + AndroidFPU;
+  } else if (ArchArgFPUKind != llvm::ARM::FK_INVALID ||
+             CPUArgFPUKind != llvm::ARM::FK_INVALID) {
+    FPUKind =
+        CPUArgFPUKind != llvm::ARM::FK_INVALID ? CPUArgFPUKind : ArchArgFPUKind;
+    (void)llvm::ARM::getFPUFeatures(FPUKind, Features);
   } else {
     if (!ForAS) {
       std::string CPU = arm::getARMTargetCPU(CPUName, ArchName, Triple);
diff --git a/clang/lib/Driver/ToolChains/Arch/RISCV.cpp b/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
index bb097356d0..a05f4b7ea6 100644
--- a/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
@@ -309,7 +309,7 @@ StringRef riscv::getRISCVArch(const llvm::opt::ArgList &Args,
       return "rv32imafdc";
     else if (MABI.starts_with_insensitive("lp64")) {
       if (Triple.isAndroid())
-        return "rv64imafdc_zba_zbb_zbs";
+        return "rv64imafdcv_zba_zbb_zbs";
 
       return "rv64imafdc";
     }
@@ -329,7 +329,7 @@ StringRef riscv::getRISCVArch(const llvm::opt::ArgList &Args,
     if (Triple.getOS() == llvm::Triple::UnknownOS)
       return "rv64imac";
     else if (Triple.isAndroid())
-      return "rv64imafdc_zba_zbb_zbs";
+      return "rv64imafdcv_zba_zbb_zbs";
     else
       return "rv64imafdc";
   }
diff --git a/clang/lib/Driver/ToolChains/Arch/X86.cpp b/clang/lib/Driver/ToolChains/Arch/X86.cpp
index cf2bc63d74..848c26ddb4 100644
--- a/clang/lib/Driver/ToolChains/Arch/X86.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/X86.cpp
@@ -229,6 +229,31 @@ void x86::getX86TargetFeatures(const Driver &D, const llvm::Triple &Triple,
         << D.getOpts().getOptionName(LVIOpt);
   }
 
+  bool HasAVX10 = false;
+  for (const Arg *A : Args.filtered(options::OPT_m_x86_AVX10_Features_Group)) {
+    StringRef Name = A->getOption().getName();
+    A->claim();
+
+    // Skip over "-m".
+    assert(Name.startswith("m") && "Invalid feature name.");
+    Name = Name.substr(1);
+
+    bool IsNegative = Name.startswith("no-");
+    if (IsNegative)
+      Name = Name.substr(3);
+
+#ifndef NDEBUG
+    assert(Name.startswith("avx10.") && "Invalid AVX10 feature name.");
+    StringRef Version, Width;
+    std::tie(Version, Width) = Name.substr(6).split('-');
+    assert(Version == "1" && "Invalid AVX10 feature name.");
+    assert((Width == "256" || Width == "512") && "Invalid AVX10 feature name.");
+#endif
+
+    Features.push_back(Args.MakeArgString((IsNegative ? "-" : "+") + Name));
+    HasAVX10 = true;
+  }
+
   // Now add any that the user explicitly requested on the command line,
   // which may override the defaults.
   for (const Arg *A : Args.filtered(options::OPT_m_x86_Features_Group,
@@ -246,9 +271,14 @@ void x86::getX86TargetFeatures(const Driver &D, const llvm::Triple &Triple,
       continue;
     }
 
+    StringRef AVX512Name = Name;
     bool IsNegative = Name.startswith("no-");
     if (IsNegative)
       Name = Name.substr(3);
+    if (HasAVX10 && (Name.startswith("avx512") || Name == "evex512")) {
+      D.Diag(diag::warn_drv_unused_argument) << AVX512Name;
+      continue;
+    }
     Features.push_back(Args.MakeArgString((IsNegative ? "-" : "+") + Name));
   }
 
diff --git a/clang/lib/Driver/ToolChains/BareMetal.cpp b/clang/lib/Driver/ToolChains/BareMetal.cpp
index 26a6276ae5..842061c1e1 100644
--- a/clang/lib/Driver/ToolChains/BareMetal.cpp
+++ b/clang/lib/Driver/ToolChains/BareMetal.cpp
@@ -452,9 +452,8 @@ void baremetal::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back(Arch == llvm::Triple::aarch64_be ? "-EB" : "-EL");
   }
 
-  Args.AddAllArgs(CmdArgs,
-                  {options::OPT_L, options::OPT_T_Group, options::OPT_s,
-                   options::OPT_t, options::OPT_Z_Flag, options::OPT_r});
+  Args.addAllArgs(CmdArgs, {options::OPT_L, options::OPT_T_Group,
+                            options::OPT_s, options::OPT_t, options::OPT_r});
 
   TC.AddFilePathLibArgs(Args, CmdArgs);
 
diff --git a/clang/lib/Driver/ToolChains/CSKYToolChain.cpp b/clang/lib/Driver/ToolChains/CSKYToolChain.cpp
index 432d61ae2f..0c280347b2 100644
--- a/clang/lib/Driver/ToolChains/CSKYToolChain.cpp
+++ b/clang/lib/Driver/ToolChains/CSKYToolChain.cpp
@@ -169,9 +169,8 @@ void CSKY::Linker::ConstructJob(Compilation &C, const JobAction &JA,
 
   Args.AddAllArgs(CmdArgs, options::OPT_L);
   ToolChain.AddFilePathLibArgs(Args, CmdArgs);
-  Args.AddAllArgs(CmdArgs,
-                  {options::OPT_T_Group, options::OPT_s, options::OPT_t,
-                   options::OPT_Z_Flag, options::OPT_r});
+  Args.addAllArgs(CmdArgs, {options::OPT_T_Group, options::OPT_s,
+                            options::OPT_t, options::OPT_r});
 
   AddLinkerInputs(ToolChain, Inputs, Args, CmdArgs, JA);
 
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index bfd6c5c286..b9c54a324b 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -1322,7 +1322,7 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA,
     A->render(Args, CmdArgs);
   }
 
-  Args.AddAllArgs(CmdArgs,
+  Args.addAllArgs(CmdArgs,
                   {options::OPT_D, options::OPT_U, options::OPT_I_Group,
                    options::OPT_F, options::OPT_index_header_map});
 
@@ -3724,20 +3724,10 @@ bool Driver::getDefaultModuleCachePath(SmallVectorImpl<char> &Result) {
 
 static bool RenderModulesOptions(Compilation &C, const Driver &D,
                                  const ArgList &Args, const InputInfo &Input,
-                                 const InputInfo &Output, const Arg *Std,
+                                 const InputInfo &Output, bool HaveStd20,
                                  ArgStringList &CmdArgs) {
   bool IsCXX = types::isCXX(Input.getType());
-  // FIXME: Find a better way to determine whether the input has standard c++
-  // modules support by default.
-  bool HaveStdCXXModules =
-      IsCXX && Std &&
-      (Std->containsValue("c++2a") || Std->containsValue("gnu++2a") ||
-       Std->containsValue("c++20") || Std->containsValue("gnu++20") ||
-       Std->containsValue("c++2b") || Std->containsValue("gnu++2b") ||
-       Std->containsValue("c++23") || Std->containsValue("gnu++23") ||
-       Std->containsValue("c++2c") || Std->containsValue("gnu++2c") ||
-       Std->containsValue("c++26") || Std->containsValue("gnu++26") ||
-       Std->containsValue("c++latest") || Std->containsValue("gnu++latest"));
+  bool HaveStdCXXModules = IsCXX && HaveStd20;
   bool HaveModules = HaveStdCXXModules;
 
   // -fmodules enables the use of precompiled modules (off by default).
@@ -5244,11 +5234,9 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
     auto findMacroDefinition = [&](const std::string &Macro) {
       auto MacroDefs = Args.getAllArgValues(options::OPT_D);
-      return std::find_if(MacroDefs.begin(), MacroDefs.end(),
-                          [&](const std::string &M) {
-                            return M == Macro ||
-                                   M.find(Macro + '=') != std::string::npos;
-                          }) != MacroDefs.end();
+      return llvm::any_of(MacroDefs, [&](const std::string &M) {
+        return M == Macro || M.find(Macro + '=') != std::string::npos;
+      });
     };
 
     // _UNIX03_WITHDRAWN is required for libcxx & porting.
@@ -5466,8 +5454,12 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     }
   }
 
-  if (Args.hasFlag(options::OPT_mrtd, options::OPT_mno_rtd, false))
-    CmdArgs.push_back("-fdefault-calling-conv=stdcall");
+  if (Args.hasFlag(options::OPT_mrtd, options::OPT_mno_rtd, false)) {
+    if (Triple.getArch() == llvm::Triple::m68k)
+      CmdArgs.push_back("-fdefault-calling-conv=rtdcall");
+    else
+      CmdArgs.push_back("-fdefault-calling-conv=stdcall");
+  }
 
   if (Args.hasArg(options::OPT_fenable_matrix)) {
     // enable-matrix is needed by both the LangOpts and by LLVM.
@@ -6838,14 +6830,6 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
                         (!IsWindowsMSVC || IsMSVC2015Compatible)))
     CmdArgs.push_back("-fno-threadsafe-statics");
 
-  // -fno-delayed-template-parsing is default, except when targeting MSVC.
-  // Many old Windows SDK versions require this to parse.
-  // FIXME: MSVC introduced /Zc:twoPhase- to disable this behavior in their
-  // compiler. We should be able to disable this by default at some point.
-  if (Args.hasFlag(options::OPT_fdelayed_template_parsing,
-                   options::OPT_fno_delayed_template_parsing, IsWindowsMSVC))
-    CmdArgs.push_back("-fdelayed-template-parsing");
-
   // -fgnu-keywords default varies depending on language; only pass if
   // specified.
   Args.AddLastArg(CmdArgs, options::OPT_fgnu_keywords,
@@ -6866,8 +6850,38 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   Args.AddLastArg(CmdArgs, options::OPT_finline_max_stacksize_EQ);
 
+  // FIXME: Find a better way to determine whether we are in C++20.
+  bool HaveCxx20 =
+      Std &&
+      (Std->containsValue("c++2a") || Std->containsValue("gnu++2a") ||
+       Std->containsValue("c++20") || Std->containsValue("gnu++20") ||
+       Std->containsValue("c++2b") || Std->containsValue("gnu++2b") ||
+       Std->containsValue("c++23") || Std->containsValue("gnu++23") ||
+       Std->containsValue("c++2c") || Std->containsValue("gnu++2c") ||
+       Std->containsValue("c++26") || Std->containsValue("gnu++26") ||
+       Std->containsValue("c++latest") || Std->containsValue("gnu++latest"));
   bool HaveModules =
-      RenderModulesOptions(C, D, Args, Input, Output, Std, CmdArgs);
+      RenderModulesOptions(C, D, Args, Input, Output, HaveCxx20, CmdArgs);
+
+  // -fdelayed-template-parsing is default when targeting MSVC.
+  // Many old Windows SDK versions require this to parse.
+  //
+  // According to
+  // https://learn.microsoft.com/en-us/cpp/build/reference/permissive-standards-conformance?view=msvc-170,
+  // MSVC actually defaults to -fno-delayed-template-parsing (/Zc:twoPhase-
+  // with MSVC CLI) if using C++20. So we match the behavior with MSVC here to
+  // not enable -fdelayed-template-parsing by default after C++20.
+  //
+  // FIXME: Given -fdelayed-template-parsing is a source of bugs, we should be
+  // able to disable this by default at some point.
+  if (Args.hasFlag(options::OPT_fdelayed_template_parsing,
+                   options::OPT_fno_delayed_template_parsing,
+                   IsWindowsMSVC && !HaveCxx20)) {
+    if (HaveCxx20)
+      D.Diag(clang::diag::warn_drv_delayed_template_parsing_after_cxx20);
+
+    CmdArgs.push_back("-fdelayed-template-parsing");
+  }
 
   if (Args.hasFlag(options::OPT_fpch_validate_input_files_content,
                    options::OPT_fno_pch_validate_input_files_content, false))
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index 25fd940584..7b2966f70b 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -2295,7 +2295,7 @@ void tools::AddStaticDeviceLibs(Compilation *C, const Tool *T,
   static const StringRef HostOnlyArchives[] = {
       "omp", "cudart", "m", "gcc", "gcc_s", "pthread", "hip_hcc"};
   for (auto SDLName : DriverArgs.getAllArgValues(options::OPT_l)) {
-    if (!HostOnlyArchives->contains(SDLName)) {
+    if (!llvm::is_contained(HostOnlyArchives, SDLName)) {
       SDLNames.insert(std::string("-l") + SDLName);
     }
   }
diff --git a/clang/lib/Driver/ToolChains/Cuda.h b/clang/lib/Driver/ToolChains/Cuda.h
index 39df6e06fb..f7c0c7ea1c 100644
--- a/clang/lib/Driver/ToolChains/Cuda.h
+++ b/clang/lib/Driver/ToolChains/Cuda.h
@@ -110,7 +110,7 @@ class LLVM_LIBRARY_VISIBILITY FatBinary : public Tool {
 // Runs nvlink, which links GPU object files ("cubin" files) into a single file.
 class LLVM_LIBRARY_VISIBILITY Linker : public Tool {
 public:
-  Linker(const ToolChain &TC) : Tool("NVPTX::Linker", "fatbinary", TC) {}
+  Linker(const ToolChain &TC) : Tool("NVPTX::Linker", "nvlink", TC) {}
 
   bool hasIntegratedCPP() const override { return false; }
 
diff --git a/clang/lib/Driver/ToolChains/Darwin.cpp b/clang/lib/Driver/ToolChains/Darwin.cpp
index c8700a7d15..15b9889157 100644
--- a/clang/lib/Driver/ToolChains/Darwin.cpp
+++ b/clang/lib/Driver/ToolChains/Darwin.cpp
@@ -638,7 +638,7 @@ void darwin::Linker::ConstructJob(Compilation &C, const JobAction &JA,
 
   // It seems that the 'e' option is completely ignored for dynamic executables
   // (the default), and with static executables, the last one wins, as expected.
-  Args.AddAllArgs(CmdArgs,
+  Args.addAllArgs(CmdArgs,
                   {options::OPT_d_Flag, options::OPT_s, options::OPT_t,
                    options::OPT_Z_Flag, options::OPT_u_Group, options::OPT_r});
 
diff --git a/clang/lib/Driver/ToolChains/DragonFly.cpp b/clang/lib/Driver/ToolChains/DragonFly.cpp
index a1e4937231..9dc8d9d436 100644
--- a/clang/lib/Driver/ToolChains/DragonFly.cpp
+++ b/clang/lib/Driver/ToolChains/DragonFly.cpp
@@ -20,21 +20,19 @@ using namespace clang::driver::toolchains;
 using namespace clang;
 using namespace llvm::opt;
 
-/// DragonFly Tools
-
-// For now, DragonFly Assemble does just about the same as for
-// FreeBSD, but this may change soon.
 void dragonfly::Assembler::ConstructJob(Compilation &C, const JobAction &JA,
                                         const InputInfo &Output,
                                         const InputInfoList &Inputs,
                                         const ArgList &Args,
                                         const char *LinkingOutput) const {
-  claimNoWarnArgs(Args);
+  const auto &ToolChain = static_cast<const DragonFly &>(getToolChain());
   ArgStringList CmdArgs;
 
+  claimNoWarnArgs(Args);
+
   // When building 32-bit code on DragonFly/pc64, we have to explicitly
   // instruct as in the base system to assemble 32-bit code.
-  if (getToolChain().getArch() == llvm::Triple::x86)
+  if (ToolChain.getArch() == llvm::Triple::x86)
     CmdArgs.push_back("--32");
 
   Args.AddAllArgValues(CmdArgs, options::OPT_Wa_COMMA, options::OPT_Xassembler);
@@ -45,7 +43,7 @@ void dragonfly::Assembler::ConstructJob(Compilation &C, const JobAction &JA,
   for (const auto &II : Inputs)
     CmdArgs.push_back(II.getFilename());
 
-  const char *Exec = Args.MakeArgString(getToolChain().GetProgramPath("as"));
+  const char *Exec = Args.MakeArgString(ToolChain.GetProgramPath("as"));
   C.addCommand(std::make_unique<Command>(JA, *this,
                                          ResponseFileSupport::AtFileCurCP(),
                                          Exec, CmdArgs, Inputs, Output));
@@ -56,19 +54,25 @@ void dragonfly::Linker::ConstructJob(Compilation &C, const JobAction &JA,
                                      const InputInfoList &Inputs,
                                      const ArgList &Args,
                                      const char *LinkingOutput) const {
-  const Driver &D = getToolChain().getDriver();
+  const auto &ToolChain = static_cast<const DragonFly &>(getToolChain());
+  const Driver &D = ToolChain.getDriver();
+  const llvm::Triple::ArchType Arch = ToolChain.getArch();
   ArgStringList CmdArgs;
+  bool Static = Args.hasArg(options::OPT_static);
+  bool Shared = Args.hasArg(options::OPT_shared);
+  bool Profiling = Args.hasArg(options::OPT_pg);
+  bool Pie = Args.hasArg(options::OPT_pie);
 
   if (!D.SysRoot.empty())
     CmdArgs.push_back(Args.MakeArgString("--sysroot=" + D.SysRoot));
 
   CmdArgs.push_back("--eh-frame-hdr");
-  if (Args.hasArg(options::OPT_static)) {
+  if (Static) {
     CmdArgs.push_back("-Bstatic");
   } else {
     if (Args.hasArg(options::OPT_rdynamic))
       CmdArgs.push_back("-export-dynamic");
-    if (Args.hasArg(options::OPT_shared))
+    if (Shared)
       CmdArgs.push_back("-shared");
     else if (!Args.hasArg(options::OPT_r)) {
       CmdArgs.push_back("-dynamic-linker");
@@ -80,7 +84,7 @@ void dragonfly::Linker::ConstructJob(Compilation &C, const JobAction &JA,
 
   // When building 32-bit code on DragonFly/pc64, we have to explicitly
   // instruct ld in the base system to link 32-bit code.
-  if (getToolChain().getArch() == llvm::Triple::x86) {
+  if (Arch == llvm::Triple::x86) {
     CmdArgs.push_back("-m");
     CmdArgs.push_back("elf_i386");
   }
@@ -93,65 +97,66 @@ void dragonfly::Linker::ConstructJob(Compilation &C, const JobAction &JA,
 
   if (!Args.hasArg(options::OPT_nostdlib, options::OPT_nostartfiles,
                    options::OPT_r)) {
-    if (!Args.hasArg(options::OPT_shared)) {
-      if (Args.hasArg(options::OPT_pg))
-        CmdArgs.push_back(
-            Args.MakeArgString(getToolChain().GetFilePath("gcrt1.o")));
+    const char *crt1 = nullptr;
+    const char *crtbegin = nullptr;
+    if (!Shared) {
+      if (Profiling)
+        crt1 = "gcrt1.o";
       else {
-        if (Args.hasArg(options::OPT_pie))
-          CmdArgs.push_back(
-              Args.MakeArgString(getToolChain().GetFilePath("Scrt1.o")));
+        if (Pie)
+          crt1 = "Scrt1.o";
         else
-          CmdArgs.push_back(
-              Args.MakeArgString(getToolChain().GetFilePath("crt1.o")));
+          crt1 = "crt1.o";
       }
     }
-    CmdArgs.push_back(Args.MakeArgString(getToolChain().GetFilePath("crti.o")));
-    if (Args.hasArg(options::OPT_shared) || Args.hasArg(options::OPT_pie))
-      CmdArgs.push_back(
-          Args.MakeArgString(getToolChain().GetFilePath("crtbeginS.o")));
+
+    if (Shared || Pie)
+      crtbegin = "crtbeginS.o";
     else
-      CmdArgs.push_back(
-          Args.MakeArgString(getToolChain().GetFilePath("crtbegin.o")));
+      crtbegin = "crtbegin.o";
+
+    if (crt1)
+      CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath(crt1)));
+    CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath("crti.o")));
+    CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath(crtbegin)));
   }
 
-  Args.AddAllArgs(CmdArgs,
-                  {options::OPT_L, options::OPT_T_Group});
+  Args.addAllArgs(CmdArgs, {options::OPT_L, options::OPT_T_Group,
+                            options::OPT_s, options::OPT_t, options::OPT_r});
+  ToolChain.AddFilePathLibArgs(Args, CmdArgs);
 
-  AddLinkerInputs(getToolChain(), Inputs, Args, CmdArgs, JA);
+  AddLinkerInputs(ToolChain, Inputs, Args, CmdArgs, JA);
 
   if (!Args.hasArg(options::OPT_nostdlib, options::OPT_nodefaultlibs,
                    options::OPT_r)) {
-    SmallString<128> Dir(D.SysRoot);
-    llvm::sys::path::append(Dir, "/usr/lib/gcc80");
-    CmdArgs.push_back(Args.MakeArgString("-L" + Dir));
-
-    if (!Args.hasArg(options::OPT_static)) {
+    if (!Static) {
       CmdArgs.push_back("-rpath");
       CmdArgs.push_back("/usr/lib/gcc80");
     }
 
+    // Use the static OpenMP runtime with -static-openmp
+    bool StaticOpenMP = Args.hasArg(options::OPT_static_openmp) && !Static;
+    addOpenMPRuntime(CmdArgs, ToolChain, Args, StaticOpenMP);
+
     if (D.CCCIsCXX()) {
-      if (getToolChain().ShouldLinkCXXStdlib(Args))
-        getToolChain().AddCXXStdlibLibArgs(Args, CmdArgs);
+      if (ToolChain.ShouldLinkCXXStdlib(Args))
+        ToolChain.AddCXXStdlibLibArgs(Args, CmdArgs);
       CmdArgs.push_back("-lm");
     }
 
     if (Args.hasArg(options::OPT_pthread))
       CmdArgs.push_back("-lpthread");
 
-    if (!Args.hasArg(options::OPT_nolibc)) {
+    if (!Args.hasArg(options::OPT_nolibc))
       CmdArgs.push_back("-lc");
-    }
 
-    if (Args.hasArg(options::OPT_static) ||
-        Args.hasArg(options::OPT_static_libgcc)) {
+    if (Static || Args.hasArg(options::OPT_static_libgcc)) {
         CmdArgs.push_back("-lgcc");
         CmdArgs.push_back("-lgcc_eh");
     } else {
       if (Args.hasArg(options::OPT_shared_libgcc)) {
           CmdArgs.push_back("-lgcc_pic");
-          if (!Args.hasArg(options::OPT_shared))
+          if (!Shared)
             CmdArgs.push_back("-lgcc");
       } else {
           CmdArgs.push_back("-lgcc");
@@ -164,18 +169,19 @@ void dragonfly::Linker::ConstructJob(Compilation &C, const JobAction &JA,
 
   if (!Args.hasArg(options::OPT_nostdlib, options::OPT_nostartfiles,
                    options::OPT_r)) {
-    if (Args.hasArg(options::OPT_shared) || Args.hasArg(options::OPT_pie))
-      CmdArgs.push_back(
-          Args.MakeArgString(getToolChain().GetFilePath("crtendS.o")));
+    const char *crtend = nullptr;
+    if (Shared || Pie)
+      crtend ="crtendS.o";
     else
-      CmdArgs.push_back(
-          Args.MakeArgString(getToolChain().GetFilePath("crtend.o")));
-    CmdArgs.push_back(Args.MakeArgString(getToolChain().GetFilePath("crtn.o")));
+      crtend = "crtend.o";
+
+    CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath(crtend)));
+    CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath("crtn.o")));
   }
 
-  getToolChain().addProfileRTLibs(Args, CmdArgs);
+  ToolChain.addProfileRTLibs(Args, CmdArgs);
 
-  const char *Exec = Args.MakeArgString(getToolChain().GetLinkerPath());
+  const char *Exec = Args.MakeArgString(ToolChain.GetLinkerPath());
   C.addCommand(std::make_unique<Command>(JA, *this,
                                          ResponseFileSupport::AtFileCurCP(),
                                          Exec, CmdArgs, Inputs, Output));
diff --git a/clang/lib/Driver/ToolChains/DragonFly.h b/clang/lib/Driver/ToolChains/DragonFly.h
index 42ce960c5b..4dceb09a17 100644
--- a/clang/lib/Driver/ToolChains/DragonFly.h
+++ b/clang/lib/Driver/ToolChains/DragonFly.h
@@ -16,7 +16,8 @@
 namespace clang {
 namespace driver {
 namespace tools {
-/// dragonfly -- Directly call GNU Binutils assembler and linker
+
+/// Directly call GNU Binutils assembler and linker
 namespace dragonfly {
 class LLVM_LIBRARY_VISIBILITY Assembler : public Tool {
 public:
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index fe44939741..999039f83d 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -31,7 +31,7 @@ static void addDashXForInput(const ArgList &Args, const InputInfo &Input,
 
 void Flang::addFortranDialectOptions(const ArgList &Args,
                                      ArgStringList &CmdArgs) const {
-  Args.AddAllArgs(CmdArgs, {options::OPT_ffixed_form,
+  Args.addAllArgs(CmdArgs, {options::OPT_ffixed_form,
                             options::OPT_ffree_form,
                             options::OPT_ffixed_line_length_EQ,
                             options::OPT_fopenmp,
@@ -56,7 +56,7 @@ void Flang::addFortranDialectOptions(const ArgList &Args,
 
 void Flang::addPreprocessingOptions(const ArgList &Args,
                                     ArgStringList &CmdArgs) const {
-  Args.AddAllArgs(CmdArgs,
+  Args.addAllArgs(CmdArgs,
                   {options::OPT_P, options::OPT_D, options::OPT_U,
                    options::OPT_I, options::OPT_cpp, options::OPT_nocpp});
 }
@@ -111,7 +111,7 @@ static bool shouldLoopVersion(const ArgList &Args) {
 }
 
 void Flang::addOtherOptions(const ArgList &Args, ArgStringList &CmdArgs) const {
-  Args.AddAllArgs(CmdArgs,
+  Args.addAllArgs(CmdArgs,
                   {options::OPT_module_dir, options::OPT_fdebug_module_writer,
                    options::OPT_fintrinsic_modules_path, options::OPT_pedantic,
                    options::OPT_std_EQ, options::OPT_W_Joined,
@@ -142,10 +142,12 @@ void Flang::addCodegenOptions(const ArgList &Args,
   if (shouldLoopVersion(Args))
     CmdArgs.push_back("-fversion-loops-for-stride");
 
-  Args.AddAllArgs(CmdArgs, {options::OPT_flang_experimental_hlfir,
+  Args.addAllArgs(CmdArgs, {options::OPT_flang_experimental_hlfir,
                             options::OPT_flang_experimental_polymorphism,
                             options::OPT_fno_ppc_native_vec_elem_order,
-                            options::OPT_fppc_native_vec_elem_order});
+                            options::OPT_fppc_native_vec_elem_order,
+                            options::OPT_falias_analysis,
+                            options::OPT_fno_alias_analysis});
 }
 
 void Flang::addPicOptions(const ArgList &Args, ArgStringList &CmdArgs) const {
diff --git a/clang/lib/Driver/ToolChains/FreeBSD.cpp b/clang/lib/Driver/ToolChains/FreeBSD.cpp
index ff4d94c56f..7a61159ba4 100644
--- a/clang/lib/Driver/ToolChains/FreeBSD.cpp
+++ b/clang/lib/Driver/ToolChains/FreeBSD.cpp
@@ -262,11 +262,8 @@ void freebsd::Linker::ConstructJob(Compilation &C, const JobAction &JA,
 
   Args.AddAllArgs(CmdArgs, options::OPT_L);
   ToolChain.AddFilePathLibArgs(Args, CmdArgs);
-  Args.AddAllArgs(CmdArgs, options::OPT_T_Group);
-  Args.AddAllArgs(CmdArgs, options::OPT_s);
-  Args.AddAllArgs(CmdArgs, options::OPT_t);
-  Args.AddAllArgs(CmdArgs, options::OPT_Z_Flag);
-  Args.AddAllArgs(CmdArgs, options::OPT_r);
+  Args.addAllArgs(CmdArgs, {options::OPT_T_Group, options::OPT_s,
+                            options::OPT_t, options::OPT_r});
 
   if (D.isUsingLTO()) {
     assert(!Inputs.empty() && "Must have at least one input.");
@@ -356,6 +353,9 @@ void freebsd::Linker::ConstructJob(Compilation &C, const JobAction &JA,
 
   ToolChain.addProfileRTLibs(Args, CmdArgs);
 
+  // Silence warnings when linking C code with a C++ '-stdlib' argument.
+  Args.ClaimAllArgs(options::OPT_stdlib_EQ);
+
   const char *Exec = Args.MakeArgString(getToolChain().GetLinkerPath());
   C.addCommand(std::make_unique<Command>(JA, *this,
                                          ResponseFileSupport::AtFileCurCP(),
diff --git a/clang/lib/Driver/ToolChains/FreeBSD.h b/clang/lib/Driver/ToolChains/FreeBSD.h
index cec67d84a2..740274b314 100644
--- a/clang/lib/Driver/ToolChains/FreeBSD.h
+++ b/clang/lib/Driver/ToolChains/FreeBSD.h
@@ -17,7 +17,7 @@ namespace clang {
 namespace driver {
 namespace tools {
 
-/// freebsd -- Directly call GNU Binutils assembler and linker
+/// Directly call GNU Binutils assembler and linker
 namespace freebsd {
 class LLVM_LIBRARY_VISIBILITY Assembler : public Tool {
 public:
diff --git a/clang/lib/Driver/ToolChains/Fuchsia.cpp b/clang/lib/Driver/ToolChains/Fuchsia.cpp
index 55c966ff39..300a57afd3 100644
--- a/clang/lib/Driver/ToolChains/Fuchsia.cpp
+++ b/clang/lib/Driver/ToolChains/Fuchsia.cpp
@@ -133,8 +133,7 @@ void fuchsia::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     }
   }
 
-  Args.AddAllArgs(CmdArgs, options::OPT_L);
-  Args.AddAllArgs(CmdArgs, options::OPT_u);
+  Args.addAllArgs(CmdArgs, {options::OPT_L, options::OPT_u});
 
   ToolChain.AddFilePathLibArgs(Args, CmdArgs);
 
diff --git a/clang/lib/Driver/ToolChains/Gnu.cpp b/clang/lib/Driver/ToolChains/Gnu.cpp
index 5949d9872c..cdd911af9a 100644
--- a/clang/lib/Driver/ToolChains/Gnu.cpp
+++ b/clang/lib/Driver/ToolChains/Gnu.cpp
@@ -530,8 +530,7 @@ void tools::gnutools::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     ToolChain.addFastMathRuntimeIfAvailable(Args, CmdArgs);
   }
 
-  Args.AddAllArgs(CmdArgs, options::OPT_L);
-  Args.AddAllArgs(CmdArgs, options::OPT_u);
+  Args.addAllArgs(CmdArgs, {options::OPT_L, options::OPT_u});
 
   ToolChain.AddFilePathLibArgs(Args, CmdArgs);
 
diff --git a/clang/lib/Driver/ToolChains/Haiku.cpp b/clang/lib/Driver/ToolChains/Haiku.cpp
index 33d71f030e..9f56a0ea5d 100644
--- a/clang/lib/Driver/ToolChains/Haiku.cpp
+++ b/clang/lib/Driver/ToolChains/Haiku.cpp
@@ -42,6 +42,9 @@ void haiku::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   // Silence warning for "clang -pie foo.o -o foo"
   Args.ClaimAllArgs(options::OPT_pie);
 
+  // -rdynamic is a no-op with Haiku. Claim argument to avoid warning.
+  Args.ClaimAllArgs(options::OPT_rdynamic);
+
   if (!D.SysRoot.empty())
     CmdArgs.push_back(Args.MakeArgString("--sysroot=" + D.SysRoot));
 
@@ -49,8 +52,6 @@ void haiku::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   if (Static) {
     CmdArgs.push_back("-Bstatic");
   } else {
-    if (Args.hasArg(options::OPT_rdynamic))
-      CmdArgs.push_back("-export-dynamic");
     if (Shared)
       CmdArgs.push_back("-shared");
     CmdArgs.push_back("--enable-new-dtags");
@@ -79,9 +80,8 @@ void haiku::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath("init_term_dyn.o")));
   }
 
-  Args.AddAllArgs(CmdArgs,
-                  {options::OPT_L, options::OPT_T_Group, options::OPT_s,
-                   options::OPT_t, options::OPT_Z_Flag, options::OPT_r});
+  Args.addAllArgs(CmdArgs, {options::OPT_L, options::OPT_T_Group,
+                            options::OPT_s, options::OPT_t, options::OPT_r});
   ToolChain.AddFilePathLibArgs(Args, CmdArgs);
 
   addLinkerCompressDebugSectionsOption(ToolChain, Args, CmdArgs);
diff --git a/clang/lib/Driver/ToolChains/Hexagon.cpp b/clang/lib/Driver/ToolChains/Hexagon.cpp
index d81e022708..d0ff7d0c13 100644
--- a/clang/lib/Driver/ToolChains/Hexagon.cpp
+++ b/clang/lib/Driver/ToolChains/Hexagon.cpp
@@ -363,7 +363,7 @@ constructHexagonLinkArgs(Compilation &C, const JobAction &JA,
 
     CmdArgs.push_back(
         Args.MakeArgString(StringRef("-L") + D.SysRoot + "/usr/lib"));
-    Args.AddAllArgs(CmdArgs, {options::OPT_T_Group, options::OPT_s,
+    Args.addAllArgs(CmdArgs, {options::OPT_T_Group, options::OPT_s,
                               options::OPT_t, options::OPT_u_Group});
     AddLinkerInputs(HTC, Inputs, Args, CmdArgs, JA);
 
@@ -451,7 +451,7 @@ constructHexagonLinkArgs(Compilation &C, const JobAction &JA,
   //----------------------------------------------------------------------------
   //
   //----------------------------------------------------------------------------
-  Args.AddAllArgs(CmdArgs, {options::OPT_T_Group, options::OPT_s,
+  Args.addAllArgs(CmdArgs, {options::OPT_T_Group, options::OPT_s,
                             options::OPT_t, options::OPT_u_Group});
 
   AddLinkerInputs(HTC, Inputs, Args, CmdArgs, JA);
diff --git a/clang/lib/Driver/ToolChains/Linux.cpp b/clang/lib/Driver/ToolChains/Linux.cpp
index 1ba222bf83..735af54f11 100644
--- a/clang/lib/Driver/ToolChains/Linux.cpp
+++ b/clang/lib/Driver/ToolChains/Linux.cpp
@@ -801,7 +801,7 @@ SanitizerMask Linux::getSupportedSanitizers() const {
       IsRISCV64 || IsSystemZ || IsHexagon || IsLoongArch64)
     Res |= SanitizerKind::Leak;
   if (IsX86_64 || IsMIPS64 || IsAArch64 || IsPowerPC64 || IsSystemZ ||
-      IsLoongArch64)
+      IsLoongArch64 || IsRISCV64)
     Res |= SanitizerKind::Thread;
   if (IsX86_64 || IsSystemZ)
     Res |= SanitizerKind::KernelMemory;
diff --git a/clang/lib/Driver/ToolChains/MSP430.cpp b/clang/lib/Driver/ToolChains/MSP430.cpp
index 2805c28f00..b28d5e4570 100644
--- a/clang/lib/Driver/ToolChains/MSP430.cpp
+++ b/clang/lib/Driver/ToolChains/MSP430.cpp
@@ -279,7 +279,7 @@ void msp430::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   if (!Args.hasArg(options::OPT_r, options::OPT_g_Group))
     CmdArgs.push_back("--gc-sections");
 
-  Args.AddAllArgs(CmdArgs, {
+  Args.addAllArgs(CmdArgs, {
                                options::OPT_n,
                                options::OPT_s,
                                options::OPT_t,
diff --git a/clang/lib/Driver/ToolChains/MinGW.cpp b/clang/lib/Driver/ToolChains/MinGW.cpp
index d3d829a8dd..39d7677954 100644
--- a/clang/lib/Driver/ToolChains/MinGW.cpp
+++ b/clang/lib/Driver/ToolChains/MinGW.cpp
@@ -201,7 +201,6 @@ void tools::MinGW::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   Args.AddLastArg(CmdArgs, options::OPT_s);
   Args.AddLastArg(CmdArgs, options::OPT_t);
   Args.AddAllArgs(CmdArgs, options::OPT_u_Group);
-  Args.AddLastArg(CmdArgs, options::OPT_Z_Flag);
 
   // Add asan_dynamic as the first import lib before other libs. This allows
   // asan to be initialized as early as possible to increase its instrumentation
diff --git a/clang/lib/Driver/ToolChains/MinGW.h b/clang/lib/Driver/ToolChains/MinGW.h
index 6d5feeacda..a71013912d 100644
--- a/clang/lib/Driver/ToolChains/MinGW.h
+++ b/clang/lib/Driver/ToolChains/MinGW.h
@@ -20,7 +20,7 @@ namespace clang {
 namespace driver {
 namespace tools {
 
-/// MinGW -- Directly call GNU Binutils assembler and linker
+/// Directly call GNU Binutils assembler and linker
 namespace MinGW {
 class LLVM_LIBRARY_VISIBILITY Assembler : public Tool {
 public:
diff --git a/clang/lib/Driver/ToolChains/NaCl.cpp b/clang/lib/Driver/ToolChains/NaCl.cpp
index 38151735ee..5be08c42b5 100644
--- a/clang/lib/Driver/ToolChains/NaCl.cpp
+++ b/clang/lib/Driver/ToolChains/NaCl.cpp
@@ -120,8 +120,7 @@ void nacltools::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath(crtbegin)));
   }
 
-  Args.AddAllArgs(CmdArgs, options::OPT_L);
-  Args.AddAllArgs(CmdArgs, options::OPT_u);
+  Args.addAllArgs(CmdArgs, {options::OPT_L, options::OPT_u});
 
   ToolChain.AddFilePathLibArgs(Args, CmdArgs);
 
diff --git a/clang/lib/Driver/ToolChains/NetBSD.cpp b/clang/lib/Driver/ToolChains/NetBSD.cpp
index 54b76ae8c8..e643bdf8bc 100644
--- a/clang/lib/Driver/ToolChains/NetBSD.cpp
+++ b/clang/lib/Driver/ToolChains/NetBSD.cpp
@@ -30,13 +30,12 @@ void netbsd::Assembler::ConstructJob(Compilation &C, const JobAction &JA,
                                      const InputInfoList &Inputs,
                                      const ArgList &Args,
                                      const char *LinkingOutput) const {
-  const toolchains::NetBSD &ToolChain =
-    static_cast<const toolchains::NetBSD &>(getToolChain());
+  const auto &ToolChain = static_cast<const NetBSD &>(getToolChain());
   const Driver &D = ToolChain.getDriver();
   const llvm::Triple &Triple = ToolChain.getTriple();
+  ArgStringList CmdArgs;
 
   claimNoWarnArgs(Args);
-  ArgStringList CmdArgs;
 
   // GNU as needs different flags for creating the correct output format
   // on architectures with different ABIs or optional feature sets.
@@ -118,27 +117,29 @@ void netbsd::Linker::ConstructJob(Compilation &C, const JobAction &JA,
                                   const InputInfoList &Inputs,
                                   const ArgList &Args,
                                   const char *LinkingOutput) const {
-  const toolchains::NetBSD &ToolChain =
-    static_cast<const toolchains::NetBSD &>(getToolChain());
+  const auto &ToolChain = static_cast<const NetBSD &>(getToolChain());
   const Driver &D = ToolChain.getDriver();
   const llvm::Triple &Triple = ToolChain.getTriple();
-
+  const llvm::Triple::ArchType Arch = ToolChain.getArch();
+  const bool Static = Args.hasArg(options::OPT_static);
+  const bool Shared = Args.hasArg(options::OPT_shared);
+  const bool Pie = Args.hasArg(options::OPT_pie);
   ArgStringList CmdArgs;
 
   if (!D.SysRoot.empty())
     CmdArgs.push_back(Args.MakeArgString("--sysroot=" + D.SysRoot));
 
   CmdArgs.push_back("--eh-frame-hdr");
-  if (Args.hasArg(options::OPT_static)) {
+  if (Static) {
     CmdArgs.push_back("-Bstatic");
-    if (Args.hasArg(options::OPT_pie)) {
+    if (Pie) {
       Args.AddAllArgs(CmdArgs, options::OPT_pie);
       CmdArgs.push_back("--no-dynamic-linker");
     }
   } else {
     if (Args.hasArg(options::OPT_rdynamic))
       CmdArgs.push_back("-export-dynamic");
-    if (Args.hasArg(options::OPT_shared)) {
+    if (Shared) {
       CmdArgs.push_back("-shared");
     } else if (!Args.hasArg(options::OPT_r)) {
       Args.AddAllArgs(CmdArgs, options::OPT_pie);
@@ -149,7 +150,7 @@ void netbsd::Linker::ConstructJob(Compilation &C, const JobAction &JA,
 
   // Many NetBSD architectures support more than one ABI.
   // Determine the correct emulation for ld.
-  switch (ToolChain.getArch()) {
+  switch (Arch) {
   case llvm::Triple::x86:
     CmdArgs.push_back("-m");
     CmdArgs.push_back("elf_i386");
@@ -193,13 +194,13 @@ void netbsd::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   case llvm::Triple::mips64el:
     if (mips::hasMipsAbiArg(Args, "32")) {
       CmdArgs.push_back("-m");
-      if (ToolChain.getArch() == llvm::Triple::mips64)
+      if (Arch == llvm::Triple::mips64)
         CmdArgs.push_back("elf32btsmip");
       else
         CmdArgs.push_back("elf32ltsmip");
     } else if (mips::hasMipsAbiArg(Args, "64")) {
       CmdArgs.push_back("-m");
-      if (ToolChain.getArch() == llvm::Triple::mips64)
+      if (Arch == llvm::Triple::mips64)
         CmdArgs.push_back("elf64btsmip");
       else
         CmdArgs.push_back("elf64ltsmip");
@@ -251,27 +252,24 @@ void netbsd::Linker::ConstructJob(Compilation &C, const JobAction &JA,
 
   if (!Args.hasArg(options::OPT_nostdlib, options::OPT_nostartfiles,
                    options::OPT_r)) {
-    if (!Args.hasArg(options::OPT_shared)) {
-      CmdArgs.push_back(
-          Args.MakeArgString(ToolChain.GetFilePath("crt0.o")));
-    }
-    CmdArgs.push_back(
-        Args.MakeArgString(ToolChain.GetFilePath("crti.o")));
-    if (Args.hasArg(options::OPT_shared) || Args.hasArg(options::OPT_pie)) {
-      CmdArgs.push_back(
-          Args.MakeArgString(ToolChain.GetFilePath("crtbeginS.o")));
-    } else {
-      CmdArgs.push_back(
-          Args.MakeArgString(ToolChain.GetFilePath("crtbegin.o")));
-    }
+    const char *crt0 = nullptr;
+    const char *crtbegin = nullptr;
+    if (!Shared)
+      crt0 = "crt0.o";
+
+    if (Shared || Pie)
+      crtbegin = "crtbeginS.o";
+    else
+      crtbegin = "crtbegin.o";
+
+    if (crt0)
+      CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath(crt0)));
+    CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath("crti.o")));
+    CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath(crtbegin)));
   }
 
-  Args.AddAllArgs(CmdArgs, options::OPT_L);
-  Args.AddAllArgs(CmdArgs, options::OPT_T_Group);
-  Args.AddAllArgs(CmdArgs, options::OPT_s);
-  Args.AddAllArgs(CmdArgs, options::OPT_t);
-  Args.AddAllArgs(CmdArgs, options::OPT_Z_Flag);
-  Args.AddAllArgs(CmdArgs, options::OPT_r);
+  Args.addAllArgs(CmdArgs, {options::OPT_L, options::OPT_T_Group,
+                            options::OPT_s, options::OPT_t, options::OPT_r});
 
   bool NeedsSanitizerDeps = addSanitizerRuntimes(ToolChain, Args, CmdArgs);
   bool NeedsXRayDeps = addXRayRuntime(ToolChain, Args, CmdArgs);
@@ -309,8 +307,7 @@ void netbsd::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   if (!Args.hasArg(options::OPT_nostdlib, options::OPT_nodefaultlibs,
                    options::OPT_r)) {
     // Use the static OpenMP runtime with -static-openmp
-    bool StaticOpenMP = Args.hasArg(options::OPT_static_openmp) &&
-                        !Args.hasArg(options::OPT_static);
+    bool StaticOpenMP = Args.hasArg(options::OPT_static_openmp) && !Static;
     addOpenMPRuntime(CmdArgs, ToolChain, Args, StaticOpenMP);
 
     if (D.CCCIsCXX()) {
@@ -327,7 +324,7 @@ void netbsd::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back("-lc");
 
     if (useLibgcc) {
-      if (Args.hasArg(options::OPT_static)) {
+      if (Static) {
         // libgcc_eh depends on libc, so resolve as much as possible,
         // pull in any new requirements from libc and then get the rest
         // of libgcc.
@@ -345,12 +342,13 @@ void netbsd::Linker::ConstructJob(Compilation &C, const JobAction &JA,
 
   if (!Args.hasArg(options::OPT_nostdlib, options::OPT_nostartfiles,
                    options::OPT_r)) {
-    if (Args.hasArg(options::OPT_shared) || Args.hasArg(options::OPT_pie))
-      CmdArgs.push_back(
-          Args.MakeArgString(ToolChain.GetFilePath("crtendS.o")));
+    const char *crtend = nullptr;
+    if (Shared || Pie)
+      crtend = "crtendS.o";
     else
-      CmdArgs.push_back(
-          Args.MakeArgString(ToolChain.GetFilePath("crtend.o")));
+      crtend = "crtend.o";
+
+    CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath(crtend)));
     CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath("crtn.o")));
   }
 
diff --git a/clang/lib/Driver/ToolChains/NetBSD.h b/clang/lib/Driver/ToolChains/NetBSD.h
index 0f9b6dceb6..044c4239c3 100644
--- a/clang/lib/Driver/ToolChains/NetBSD.h
+++ b/clang/lib/Driver/ToolChains/NetBSD.h
@@ -17,7 +17,7 @@ namespace clang {
 namespace driver {
 namespace tools {
 
-/// netbsd -- Directly call GNU Binutils assembler and linker
+/// Directly call GNU Binutils assembler and linker
 namespace netbsd {
 class LLVM_LIBRARY_VISIBILITY Assembler : public Tool {
 public:
diff --git a/clang/lib/Driver/ToolChains/OpenBSD.cpp b/clang/lib/Driver/ToolChains/OpenBSD.cpp
index 6de190d824..e874f24577 100644
--- a/clang/lib/Driver/ToolChains/OpenBSD.cpp
+++ b/clang/lib/Driver/ToolChains/OpenBSD.cpp
@@ -121,6 +121,7 @@ void openbsd::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   bool Profiling = Args.hasArg(options::OPT_pg);
   bool Pie = Args.hasArg(options::OPT_pie);
   bool Nopie = Args.hasArg(options::OPT_nopie);
+  const bool Relocatable = Args.hasArg(options::OPT_r);
 
   // Silence warning for "clang -g foo.o -o foo"
   Args.ClaimAllArgs(options::OPT_g_Group);
@@ -138,7 +139,7 @@ void openbsd::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   else if (Arch == llvm::Triple::mips64el)
     CmdArgs.push_back("-EL");
 
-  if (!Args.hasArg(options::OPT_nostdlib) && !Shared) {
+  if (!Args.hasArg(options::OPT_nostdlib) && !Shared && !Relocatable) {
     CmdArgs.push_back("-e");
     CmdArgs.push_back("__start");
   }
@@ -149,10 +150,9 @@ void openbsd::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   } else {
     if (Args.hasArg(options::OPT_rdynamic))
       CmdArgs.push_back("-export-dynamic");
-    CmdArgs.push_back("-Bdynamic");
     if (Shared) {
       CmdArgs.push_back("-shared");
-    } else if (!Args.hasArg(options::OPT_r)) {
+    } else if (!Relocatable) {
       CmdArgs.push_back("-dynamic-linker");
       CmdArgs.push_back("/usr/libexec/ld.so");
     }
@@ -195,9 +195,8 @@ void openbsd::Linker::ConstructJob(Compilation &C, const JobAction &JA,
 
   Args.AddAllArgs(CmdArgs, options::OPT_L);
   ToolChain.AddFilePathLibArgs(Args, CmdArgs);
-  Args.AddAllArgs(CmdArgs,
-                  {options::OPT_T_Group, options::OPT_s, options::OPT_t,
-                   options::OPT_Z_Flag, options::OPT_r});
+  Args.addAllArgs(CmdArgs, {options::OPT_T_Group, options::OPT_s,
+                            options::OPT_t, options::OPT_r});
 
   bool NeedsSanitizerDeps = addSanitizerRuntimes(ToolChain, Args, CmdArgs);
   bool NeedsXRayDeps = addXRayRuntime(ToolChain, Args, CmdArgs);
diff --git a/clang/lib/Driver/ToolChains/OpenBSD.h b/clang/lib/Driver/ToolChains/OpenBSD.h
index b7d3d1b5b6..486dbf1407 100644
--- a/clang/lib/Driver/ToolChains/OpenBSD.h
+++ b/clang/lib/Driver/ToolChains/OpenBSD.h
@@ -18,7 +18,7 @@ namespace clang {
 namespace driver {
 namespace tools {
 
-/// openbsd -- Directly call GNU Binutils assembler and linker
+/// Directly call GNU Binutils assembler and linker
 namespace openbsd {
 class LLVM_LIBRARY_VISIBILITY Assembler : public Tool {
 public:
diff --git a/clang/lib/Driver/ToolChains/PS4CPU.cpp b/clang/lib/Driver/ToolChains/PS4CPU.cpp
index dd8066abba..5fd82d1da1 100644
--- a/clang/lib/Driver/ToolChains/PS4CPU.cpp
+++ b/clang/lib/Driver/ToolChains/PS4CPU.cpp
@@ -208,11 +208,9 @@ void tools::PScpu::Linker::ConstructJob(Compilation &C, const JobAction &JA,
       CmdArgs.push_back("--lto=full");
   }
 
-  Args.AddAllArgs(CmdArgs, options::OPT_L);
-  Args.AddAllArgs(CmdArgs, options::OPT_T_Group);
-  Args.AddAllArgs(CmdArgs, options::OPT_s);
-  Args.AddAllArgs(CmdArgs, options::OPT_t);
-  Args.AddAllArgs(CmdArgs, options::OPT_r);
+  Args.addAllArgs(CmdArgs,
+                  {options::OPT_L, options::OPT_T_Group, options::OPT_s,
+                   options::OPT_t, options::OPT_r});
 
   if (Args.hasArg(options::OPT_Z_Xlinker__no_demangle))
     CmdArgs.push_back("--no-demangle");
diff --git a/clang/lib/Driver/ToolChains/RISCVToolchain.cpp b/clang/lib/Driver/ToolChains/RISCVToolchain.cpp
index eb990eb809..7e6abd1444 100644
--- a/clang/lib/Driver/ToolChains/RISCVToolchain.cpp
+++ b/clang/lib/Driver/ToolChains/RISCVToolchain.cpp
@@ -190,12 +190,11 @@ void RISCV::Linker::ConstructJob(Compilation &C, const JobAction &JA,
 
   AddLinkerInputs(ToolChain, Inputs, Args, CmdArgs, JA);
 
-  Args.AddAllArgs(CmdArgs, options::OPT_L);
-  Args.AddAllArgs(CmdArgs, options::OPT_u);
+  Args.addAllArgs(CmdArgs, {options::OPT_L, options::OPT_u});
+
   ToolChain.AddFilePathLibArgs(Args, CmdArgs);
-  Args.AddAllArgs(CmdArgs,
-                  {options::OPT_T_Group, options::OPT_s, options::OPT_t,
-                   options::OPT_Z_Flag, options::OPT_r});
+  Args.addAllArgs(CmdArgs, {options::OPT_T_Group, options::OPT_s,
+                            options::OPT_t, options::OPT_r});
 
   // TODO: add C++ includes and libs if compiling C++.
 
diff --git a/clang/lib/Driver/ToolChains/Solaris.cpp b/clang/lib/Driver/ToolChains/Solaris.cpp
index be01c987e5..ecff8ddc4e 100644
--- a/clang/lib/Driver/ToolChains/Solaris.cpp
+++ b/clang/lib/Driver/ToolChains/Solaris.cpp
@@ -209,7 +209,7 @@ void solaris::Linker::ConstructJob(Compilation &C, const JobAction &JA,
 
   getToolChain().AddFilePathLibArgs(Args, CmdArgs);
 
-  Args.AddAllArgs(CmdArgs,
+  Args.addAllArgs(CmdArgs,
                   {options::OPT_L, options::OPT_T_Group, options::OPT_r});
 
   bool NeedsSanitizerDeps = addSanitizerRuntimes(getToolChain(), Args, CmdArgs);
@@ -222,6 +222,14 @@ void solaris::Linker::ConstructJob(Compilation &C, const JobAction &JA,
         getToolChain().AddCXXStdlibLibArgs(Args, CmdArgs);
       CmdArgs.push_back("-lm");
     }
+    // Additional linker set-up and flags for Fortran. This is required in order
+    // to generate executables. As Fortran runtime depends on the C runtime,
+    // these dependencies need to be listed before the C runtime below.
+    if (D.IsFlangMode()) {
+      addFortranRuntimeLibraryPath(getToolChain(), Args, CmdArgs);
+      addFortranRuntimeLibs(getToolChain(), CmdArgs);
+      CmdArgs.push_back("-lm");
+    }
     if (Args.hasArg(options::OPT_fstack_protector) ||
         Args.hasArg(options::OPT_fstack_protector_strong) ||
         Args.hasArg(options::OPT_fstack_protector_all)) {
diff --git a/clang/lib/Driver/ToolChains/Solaris.h b/clang/lib/Driver/ToolChains/Solaris.h
index 04b68c5053..b8f2497d7c 100644
--- a/clang/lib/Driver/ToolChains/Solaris.h
+++ b/clang/lib/Driver/ToolChains/Solaris.h
@@ -17,7 +17,7 @@ namespace clang {
 namespace driver {
 namespace tools {
 
-/// solaris -- Directly call Solaris assembler and linker
+/// Directly call Solaris assembler and linker
 namespace solaris {
 class LLVM_LIBRARY_VISIBILITY Assembler : public gnutools::Assembler {
 public:
diff --git a/clang/lib/Driver/ToolChains/WebAssembly.cpp b/clang/lib/Driver/ToolChains/WebAssembly.cpp
index 2098a68cbc..fe0b30b725 100644
--- a/clang/lib/Driver/ToolChains/WebAssembly.cpp
+++ b/clang/lib/Driver/ToolChains/WebAssembly.cpp
@@ -73,8 +73,8 @@ void wasm::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   if (Args.hasArg(options::OPT_s))
     CmdArgs.push_back("--strip-all");
 
-  Args.AddAllArgs(CmdArgs, options::OPT_L);
-  Args.AddAllArgs(CmdArgs, options::OPT_u);
+  Args.addAllArgs(CmdArgs, {options::OPT_L, options::OPT_u});
+
   ToolChain.AddFilePathLibArgs(Args, CmdArgs);
 
   bool IsCommand = true;
diff --git a/clang/lib/Driver/ToolChains/ZOS.cpp b/clang/lib/Driver/ToolChains/ZOS.cpp
index 05548fbec6..a7d6f030f3 100644
--- a/clang/lib/Driver/ToolChains/ZOS.cpp
+++ b/clang/lib/Driver/ToolChains/ZOS.cpp
@@ -156,10 +156,9 @@ void zos::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back("/dev/null");
   }
 
-  Args.AddAllArgs(CmdArgs, options::OPT_u);
-
   // Add archive library search paths.
-  Args.AddAllArgs(CmdArgs, options::OPT_L);
+  Args.addAllArgs(CmdArgs, {options::OPT_L, options::OPT_u});
+
   ToolChain.AddFilePathLibArgs(Args, CmdArgs);
 
   // Specify linker input file(s)
diff --git a/clang/lib/Driver/ToolChains/ZOS.h b/clang/lib/Driver/ToolChains/ZOS.h
index 548b432ade..4b9b332c0f 100644
--- a/clang/lib/Driver/ToolChains/ZOS.h
+++ b/clang/lib/Driver/ToolChains/ZOS.h
@@ -16,7 +16,7 @@ namespace clang {
 namespace driver {
 namespace tools {
 
-/// zos -- Directly call system default assembler and linker.
+/// Directly call system default assembler and linker.
 namespace zos {
 
 class LLVM_LIBRARY_VISIBILITY Assembler : public Tool {
diff --git a/clang/lib/ExtractAPI/API.cpp b/clang/lib/ExtractAPI/API.cpp
index 2973a31345..71c655ba5b 100644
--- a/clang/lib/ExtractAPI/API.cpp
+++ b/clang/lib/ExtractAPI/API.cpp
@@ -17,7 +17,6 @@
 #include "clang/AST/CommentLexer.h"
 #include "clang/AST/RawCommentList.h"
 #include "clang/Index/USRGeneration.h"
-#include "llvm/ADT/STLFunctionalExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include <memory>
 
diff --git a/clang/lib/Format/ContinuationIndenter.cpp b/clang/lib/Format/ContinuationIndenter.cpp
index 91ce825224..3b28f84fd8 100644
--- a/clang/lib/Format/ContinuationIndenter.cpp
+++ b/clang/lib/Format/ContinuationIndenter.cpp
@@ -1335,6 +1335,8 @@ unsigned ContinuationIndenter::getNewLineColumn(const LineState &State) {
   if ((PreviousNonComment &&
        (PreviousNonComment->ClosesTemplateDeclaration ||
         PreviousNonComment->ClosesRequiresClause ||
+        (PreviousNonComment->is(TT_AttributeMacro) &&
+         Current.isNot(tok::l_paren)) ||
         PreviousNonComment->isOneOf(
             TT_AttributeRParen, TT_AttributeSquare, TT_FunctionAnnotationRParen,
             TT_JavaAnnotation, TT_LeadingJavaAnnotation))) ||
diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp
index 7b0ebe2cf6..ff7cb097a5 100644
--- a/clang/lib/Format/Format.cpp
+++ b/clang/lib/Format/Format.cpp
@@ -504,6 +504,22 @@ struct ScalarEnumerationTraits<FormatStyle::QualifierAlignmentStyle> {
   }
 };
 
+template <>
+struct MappingTraits<
+    FormatStyle::SpaceBeforeParensCustom::AfterPlacementOperatorStyle> {
+  static void
+  mapping(IO &IO,
+          FormatStyle::SpaceBeforeParensCustom::AfterPlacementOperatorStyle
+              &Value) {
+    IO.enumCase(Value, "Always",
+                FormatStyle::SpaceBeforeParensCustom::APO_Always);
+    IO.enumCase(Value, "Never",
+                FormatStyle::SpaceBeforeParensCustom::APO_Never);
+    IO.enumCase(Value, "Leave",
+                FormatStyle::SpaceBeforeParensCustom::APO_Leave);
+  }
+};
+
 template <> struct MappingTraits<FormatStyle::RawStringFormat> {
   static void mapping(IO &IO, FormatStyle::RawStringFormat &Format) {
     IO.mapOptional("Language", Format.Language);
@@ -679,6 +695,7 @@ template <> struct MappingTraits<FormatStyle::SpaceBeforeParensCustom> {
                    Spacing.AfterFunctionDeclarationName);
     IO.mapOptional("AfterIfMacros", Spacing.AfterIfMacros);
     IO.mapOptional("AfterOverloadedOperator", Spacing.AfterOverloadedOperator);
+    IO.mapOptional("AfterPlacementOperator", Spacing.AfterPlacementOperator);
     IO.mapOptional("AfterRequiresInClause", Spacing.AfterRequiresInClause);
     IO.mapOptional("AfterRequiresInExpression",
                    Spacing.AfterRequiresInExpression);
@@ -817,8 +834,8 @@ template <> struct MappingTraits<FormatStyle> {
 
     StringRef BasedOnStyle;
     if (IO.outputting()) {
-      StringRef Styles[] = {"LLVM",   "Google", "Chromium", "Mozilla",
-                            "WebKit", "GNU",    "Microsoft"};
+      StringRef Styles[] = {"LLVM",   "Google", "Chromium",  "Mozilla",
+                            "WebKit", "GNU",    "Microsoft", "clang-format"};
       for (StringRef StyleName : Styles) {
         FormatStyle PredefinedStyle;
         if (getPredefinedStyle(StyleName, Style.Language, &PredefinedStyle) &&
@@ -1369,6 +1386,8 @@ static void expandPresetsSpaceBeforeParens(FormatStyle &Expanded) {
 
   switch (Expanded.SpaceBeforeParens) {
   case FormatStyle::SBPO_Never:
+    Expanded.SpaceBeforeParensOptions.AfterPlacementOperator =
+        FormatStyle::SpaceBeforeParensCustom::APO_Never;
     break;
   case FormatStyle::SBPO_ControlStatements:
     Expanded.SpaceBeforeParensOptions.AfterControlStatements = true;
@@ -1896,6 +1915,16 @@ FormatStyle getMicrosoftStyle(FormatStyle::LanguageKind Language) {
   return Style;
 }
 
+FormatStyle getClangFormatStyle() {
+  FormatStyle Style = getLLVMStyle();
+  Style.InsertBraces = true;
+  Style.InsertNewlineAtEOF = true;
+  Style.LineEnding = FormatStyle::LE_LF;
+  Style.RemoveBracesLLVM = true;
+  Style.RemoveParentheses = FormatStyle::RPS_ReturnStatement;
+  return Style;
+}
+
 FormatStyle getNoStyle() {
   FormatStyle NoStyle = getLLVMStyle();
   NoStyle.DisableFormat = true;
@@ -1920,6 +1949,8 @@ bool getPredefinedStyle(StringRef Name, FormatStyle::LanguageKind Language,
     *Style = getGNUStyle();
   else if (Name.equals_insensitive("microsoft"))
     *Style = getMicrosoftStyle(Language);
+  else if (Name.equals_insensitive("clang-format"))
+    *Style = getClangFormatStyle();
   else if (Name.equals_insensitive("none"))
     *Style = getNoStyle();
   else if (Name.equals_insensitive("inheritparentconfig"))
diff --git a/clang/lib/Format/FormatToken.h b/clang/lib/Format/FormatToken.h
index 527f1d744a..acd24f8361 100644
--- a/clang/lib/Format/FormatToken.h
+++ b/clang/lib/Format/FormatToken.h
@@ -52,6 +52,7 @@ namespace format {
   TYPE(ConflictStart)                                                          \
   /* l_brace of if/for/while */                                                \
   TYPE(ControlStatementLBrace)                                                 \
+  TYPE(ControlStatementRBrace)                                                 \
   TYPE(CppCastLParen)                                                          \
   TYPE(CSharpGenericTypeConstraint)                                            \
   TYPE(CSharpGenericTypeConstraintColon)                                       \
@@ -66,7 +67,9 @@ namespace format {
   TYPE(DesignatedInitializerLSquare)                                           \
   TYPE(DesignatedInitializerPeriod)                                            \
   TYPE(DictLiteral)                                                            \
+  TYPE(DoWhile)                                                                \
   TYPE(ElseLBrace)                                                             \
+  TYPE(ElseRBrace)                                                             \
   TYPE(EnumLBrace)                                                             \
   TYPE(EnumRBrace)                                                             \
   TYPE(FatArrow)                                                               \
diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp
index 543c119620..a457ced9f8 100644
--- a/clang/lib/Format/TokenAnnotator.cpp
+++ b/clang/lib/Format/TokenAnnotator.cpp
@@ -3497,6 +3497,14 @@ void TokenAnnotator::calculateFormattingInformation(AnnotatedLine &Line) const {
           Tok->setType(TT_TrailingReturnArrow);
           break;
         }
+        if (Tok->isNot(TT_TrailingAnnotation))
+          continue;
+        const auto *Next = Tok->Next;
+        if (!Next || Next->isNot(tok::l_paren))
+          continue;
+        Tok = Next->MatchingParen;
+        if (!Tok)
+          break;
       }
     }
   }
@@ -4189,8 +4197,7 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
   if ((Left.is(tok::l_brace) && Left.isNot(BK_Block)) ||
       (Right.is(tok::r_brace) && Right.MatchingParen &&
        Right.MatchingParen->isNot(BK_Block))) {
-    return Style.Cpp11BracedListStyle ? Style.SpacesInParensOptions.Other
-                                      : true;
+    return !Style.Cpp11BracedListStyle || Style.SpacesInParensOptions.Other;
   }
   if (Left.is(TT_BlockComment)) {
     // No whitespace in x(/*foo=*/1), except for JavaScript.
@@ -4226,6 +4233,19 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
       return Style.SpaceBeforeParensOptions.AfterIfMacros ||
              spaceRequiredBeforeParens(Right);
     }
+    if (Style.SpaceBeforeParens == FormatStyle::SBPO_Custom &&
+        Left.isOneOf(tok::kw_new, tok::kw_delete) &&
+        Right.isNot(TT_OverloadedOperatorLParen) &&
+        !(Line.MightBeFunctionDecl && Left.is(TT_FunctionDeclarationName))) {
+      if (Style.SpaceBeforeParensOptions.AfterPlacementOperator ==
+              FormatStyle::SpaceBeforeParensCustom::APO_Always ||
+          (Style.SpaceBeforeParensOptions.AfterPlacementOperator ==
+               FormatStyle::SpaceBeforeParensCustom::APO_Leave &&
+           Right.hasWhitespaceBefore())) {
+        return true;
+      }
+      return false;
+    }
     if (Line.Type == LT_ObjCDecl)
       return true;
     if (Left.is(tok::semi))
@@ -4393,8 +4413,10 @@ bool TokenAnnotator::spaceRequiredBefore(const AnnotatedLine &Line,
       return false;
     }
     // Space in __attribute__((attr)) ::type.
-    if (Left.is(TT_AttributeRParen) && Right.is(tok::coloncolon))
+    if (Left.isOneOf(TT_AttributeRParen, TT_AttributeMacro) &&
+        Right.is(tok::coloncolon)) {
       return true;
+    }
 
     if (Left.is(tok::kw_operator))
       return Right.is(tok::coloncolon);
@@ -4709,7 +4731,8 @@ bool TokenAnnotator::spaceRequiredBefore(const AnnotatedLine &Line,
   if (Line.Type == LT_ObjCMethodDecl) {
     if (Left.is(TT_ObjCMethodSpecifier))
       return true;
-    if (Left.is(tok::r_paren) && canBeObjCSelectorComponent(Right)) {
+    if (Left.is(tok::r_paren) && Left.isNot(TT_AttributeRParen) &&
+        canBeObjCSelectorComponent(Right)) {
       // Don't space between ')' and <id> or ')' and 'new'. 'new' is not a
       // keyword in Objective-C, and '+ (instancetype)new;' is a standard class
       // method declaration.
@@ -5222,8 +5245,10 @@ bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line,
   }
 
   // Ensure wrapping after __attribute__((XX)) and @interface etc.
-  if (Left.is(TT_AttributeRParen) && Right.is(TT_ObjCDecl))
+  if (Left.isOneOf(TT_AttributeRParen, TT_AttributeMacro) &&
+      Right.is(TT_ObjCDecl)) {
     return true;
+  }
 
   if (Left.is(TT_LambdaLBrace)) {
     if (IsFunctionArgument(Left) &&
diff --git a/clang/lib/Format/UnwrappedLineParser.cpp b/clang/lib/Format/UnwrappedLineParser.cpp
index 3275d7b6a7..488d8dc07b 100644
--- a/clang/lib/Format/UnwrappedLineParser.cpp
+++ b/clang/lib/Format/UnwrappedLineParser.cpp
@@ -213,7 +213,7 @@ void UnwrappedLineParser::parse() {
     }
 
     // Create line with eof token.
-    assert(FormatTok->is(tok::eof));
+    assert(eof());
     pushToken(FormatTok);
     addUnwrappedLine();
 
@@ -640,6 +640,14 @@ void UnwrappedLineParser::calculateBraceTypes(bool ExpectClassBody) {
   FormatTok = Tokens->setPosition(StoredPosition);
 }
 
+// Sets the token type of the directly previous right brace.
+void UnwrappedLineParser::setPreviousRBraceType(TokenType Type) {
+  if (auto Prev = FormatTok->getPreviousNonComment();
+      Prev && Prev->is(tok::r_brace)) {
+    Prev->setFinalizedType(Type);
+  }
+}
+
 template <class T>
 static inline void hash_combine(std::size_t &seed, const T &v) {
   std::hash<T> hasher;
@@ -1959,8 +1967,9 @@ void UnwrappedLineParser::parseStructuralElement(
         auto I = Line->Tokens.begin(), E = Line->Tokens.end();
         while (I != E && I->Tok->is(tok::comment))
           ++I;
-        while (I != E && Style.isVerilog() && I->Tok->is(tok::hash))
-          ++I;
+        if (Style.isVerilog())
+          while (I != E && I->Tok->is(tok::hash))
+            ++I;
         return I != E && (++I == E);
       };
       if (OneTokenSoFar()) {
@@ -2218,9 +2227,6 @@ bool UnwrappedLineParser::tryToParseLambda() {
     // followed by an `a->b` expression, such as:
     // ([obj func:arg] + a->b)
     // Otherwise the code below would parse as a lambda.
-    //
-    // FIXME: This heuristic is incorrect for C++20 generic lambdas with
-    // explicit template lists: []<bool b = true && false>(U &&u){}
     case tok::plus:
     case tok::minus:
     case tok::exclaim:
@@ -2260,6 +2266,11 @@ bool UnwrappedLineParser::tryToParseLambda() {
       parseRequiresClause(RequiresToken);
       break;
     }
+    case tok::equal:
+      if (!InTemplateParameterList)
+        return true;
+      nextToken();
+      break;
     default:
       return true;
     }
@@ -2756,6 +2767,7 @@ FormatToken *UnwrappedLineParser::parseIfThenElse(IfStmtKind *IfKind,
     CompoundStatementIndenter Indenter(this, Style, Line->Level);
     parseBlock(/*MustBeDeclaration=*/false, /*AddLevels=*/1u,
                /*MunchSemi=*/true, KeepIfBraces, &IfBlockKind);
+    setPreviousRBraceType(TT_ControlStatementRBrace);
     if (Style.BraceWrapping.BeforeElse)
       addUnwrappedLine();
     else
@@ -2794,6 +2806,7 @@ FormatToken *UnwrappedLineParser::parseIfThenElse(IfStmtKind *IfKind,
       FormatToken *IfLBrace =
           parseBlock(/*MustBeDeclaration=*/false, /*AddLevels=*/1u,
                      /*MunchSemi=*/true, KeepElseBraces, &ElseBlockKind);
+      setPreviousRBraceType(TT_ElseRBrace);
       if (FormatTok->is(tok::kw_else)) {
         KeepElseBraces = KeepElseBraces ||
                          ElseBlockKind == IfStmtKind::IfOnly ||
@@ -3057,12 +3070,12 @@ void UnwrappedLineParser::parseLoopBody(bool KeepBraces, bool WrapRightBrace) {
   keepAncestorBraces();
 
   if (isBlockBegin(*FormatTok)) {
-    if (!KeepBraces)
-      FormatTok->setFinalizedType(TT_ControlStatementLBrace);
+    FormatTok->setFinalizedType(TT_ControlStatementLBrace);
     FormatToken *LeftBrace = FormatTok;
     CompoundStatementIndenter Indenter(this, Style, Line->Level);
     parseBlock(/*MustBeDeclaration=*/false, /*AddLevels=*/1u,
                /*MunchSemi=*/true, KeepBraces);
+    setPreviousRBraceType(TT_ControlStatementRBrace);
     if (!KeepBraces) {
       assert(!NestedTooDeep.empty());
       if (!NestedTooDeep.back())
@@ -3124,6 +3137,8 @@ void UnwrappedLineParser::parseDoWhile() {
     return;
   }
 
+  FormatTok->setFinalizedType(TT_DoWhile);
+
   // If in Whitesmiths mode, the line with the while() needs to be indented
   // to the same level as the block.
   if (Style.BreakBeforeBraces == FormatStyle::BS_Whitesmiths)
@@ -3196,7 +3211,9 @@ void UnwrappedLineParser::parseSwitch() {
 
   if (FormatTok->is(tok::l_brace)) {
     CompoundStatementIndenter Indenter(this, Style, Line->Level);
+    FormatTok->setFinalizedType(TT_ControlStatementLBrace);
     parseBlock();
+    setPreviousRBraceType(TT_ControlStatementRBrace);
     addUnwrappedLine();
   } else {
     addUnwrappedLine();
@@ -3713,10 +3730,7 @@ bool UnwrappedLineParser::parseEnum() {
       nextToken();
     addUnwrappedLine();
   }
-  if (auto Prev = FormatTok->getPreviousNonComment();
-      Prev && Prev->is(tok::r_brace)) {
-    Prev->setFinalizedType(TT_EnumRBrace);
-  }
+  setPreviousRBraceType(TT_EnumRBrace);
   return true;
 
   // There is no addUnwrappedLine() here so that we fall through to parsing a
@@ -3950,10 +3964,7 @@ void UnwrappedLineParser::parseRecord(bool ParseAsExpr) {
       unsigned AddLevels = Style.IndentAccessModifiers ? 2u : 1u;
       parseBlock(/*MustBeDeclaration=*/true, AddLevels, /*MunchSemi=*/false);
     }
-    if (auto Prev = FormatTok->getPreviousNonComment();
-        Prev && Prev->is(tok::r_brace)) {
-      Prev->setFinalizedType(ClosingBraceType);
-    }
+    setPreviousRBraceType(ClosingBraceType);
   }
   // There is no addUnwrappedLine() here so that we fall through to parsing a
   // structural element afterwards. Thus, in "class A {} n, m;",
diff --git a/clang/lib/Format/UnwrappedLineParser.h b/clang/lib/Format/UnwrappedLineParser.h
index a4f150d195..c31f25fdd8 100644
--- a/clang/lib/Format/UnwrappedLineParser.h
+++ b/clang/lib/Format/UnwrappedLineParser.h
@@ -243,6 +243,7 @@ class UnwrappedLineParser {
   void flushComments(bool NewlineBeforeNext);
   void pushToken(FormatToken *Tok);
   void calculateBraceTypes(bool ExpectClassBody = false);
+  void setPreviousRBraceType(TokenType Type);
 
   // Marks a conditional compilation edge (for example, an '#if', '#ifdef',
   // '#else' or merge conflict marker). If 'Unreachable' is true, assumes
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index bb442495f5..4e6d7bb16f 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -648,6 +648,7 @@ static bool FixupInvocation(CompilerInvocation &Invocation,
     emitError |= (DefaultCC == LangOptions::DCC_VectorCall ||
                   DefaultCC == LangOptions::DCC_RegCall) &&
                  !T.isX86();
+    emitError |= DefaultCC == LangOptions::DCC_RtdCall && Arch != llvm::Triple::m68k;
     if (emitError)
       Diags.Report(diag::err_drv_argument_not_allowed_with)
           << A->getSpelling() << T.getTriple();
@@ -3865,11 +3866,17 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
       Diags.Report(diag::err_drv_argument_not_allowed_with)
           << A->getSpelling() << "-fdefault-calling-conv";
     else {
-      if (T.getArch() != llvm::Triple::x86)
+      switch (T.getArch()) {
+      case llvm::Triple::x86:
+        Opts.setDefaultCallingConv(LangOptions::DCC_StdCall);
+        break;
+      case llvm::Triple::m68k:
+        Opts.setDefaultCallingConv(LangOptions::DCC_RtdCall);
+        break;
+      default:
         Diags.Report(diag::err_drv_argument_not_allowed_with)
             << A->getSpelling() << T.getTriple();
-      else
-        Opts.setDefaultCallingConv(LangOptions::DCC_StdCall);
+      }
     }
   }
 
diff --git a/clang/lib/Headers/CMakeLists.txt b/clang/lib/Headers/CMakeLists.txt
index 8deea823e3..02a0c81644 100644
--- a/clang/lib/Headers/CMakeLists.txt
+++ b/clang/lib/Headers/CMakeLists.txt
@@ -17,6 +17,7 @@ set(core_files
   __stdarg_va_list.h
   stdatomic.h
   stdbool.h
+  stdckdint.h
   stddef.h
   __stddef_max_align_t.h
   __stddef_null.h
@@ -224,6 +225,7 @@ set(x86_files
   tmmintrin.h
   tsxldtrkintrin.h
   uintrintrin.h
+  usermsrintrin.h
   vaesintrin.h
   vpclmulqdqintrin.h
   waitpkgintrin.h
diff --git a/clang/lib/Headers/intrin.h b/clang/lib/Headers/intrin.h
index 4678c527bf..9ebaea9fee 100644
--- a/clang/lib/Headers/intrin.h
+++ b/clang/lib/Headers/intrin.h
@@ -586,6 +586,8 @@ unsigned int _CountLeadingZeros(unsigned long);
 unsigned int _CountLeadingZeros64(unsigned _int64);
 unsigned int _CountOneBits(unsigned long);
 unsigned int _CountOneBits64(unsigned __int64);
+
+void __cdecl __prefetch(void *);
 #endif
 
 /*----------------------------------------------------------------------------*\
diff --git a/clang/lib/Headers/stdckdint.h b/clang/lib/Headers/stdckdint.h
new file mode 100644
index 0000000000..22972d78d9
--- /dev/null
+++ b/clang/lib/Headers/stdckdint.h
@@ -0,0 +1,44 @@
+/*===---- stdckdint.h - Standard header for checking integer----------------===
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===-----------------------------------------------------------------------===
+ */
+
+#ifndef __STDCKDINT_H
+#define __STDCKDINT_H
+
+/* If we're hosted, fall back to the system's stdckdint.h. FreeBSD, for
+ * example, already has a Clang-compatible stdckdint.h header.
+ *
+ * The `stdckdint.h` header requires C 23 or newer.
+ */
+#if __STDC_HOSTED__ && __has_include_next(<stdckdint.h>)
+#include_next <stdckdint.h>
+#else
+
+/* C23 7.20.1 Defines several macros for performing checked integer arithmetic*/
+
+#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L
+#define __STDC_VERSION_STDCKDINT_H__ 202311L
+
+// Both A and B shall be any integer type other than "plain" char, bool, a bit-
+// precise integer type, or an enumerated type, and they need not be the same.
+
+// R shall be a modifiable lvalue of any integer type other than "plain" char,
+// bool, a bit-precise integer type, or an enumerated type. It shouldn't be
+// short type, either. Otherwise, it may be unable to hold two the result of
+// operating two 'int's.
+
+// A diagnostic message will be produced if A or B are not suitable integer
+// types, or if R is not a modifiable lvalue of a suitable integer type or R
+// is short type.
+#define ckd_add(R, A, B) __builtin_add_overflow((A), (B), (R))
+#define ckd_sub(R, A, B) __builtin_sub_overflow((A), (B), (R))
+#define ckd_mul(R, A, B) __builtin_mul_overflow((A), (B), (R))
+#endif
+
+#endif /* __STDC_HOSTED__ */
+#endif /* __STDCKDINT_H */
diff --git a/clang/lib/Headers/usermsrintrin.h b/clang/lib/Headers/usermsrintrin.h
new file mode 100644
index 0000000000..6d1424ad3b
--- /dev/null
+++ b/clang/lib/Headers/usermsrintrin.h
@@ -0,0 +1,30 @@
+/*===--------------- usermsrintrin.h - USERMSR intrinsics -----------------===
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===-----------------------------------------------------------------------===
+ */
+#ifndef __X86GPRINTRIN_H
+#error "Never use <usermsrintrin.h> directly; include <x86gprintrin.h> instead."
+#endif // __X86GPRINTRIN_H
+
+#ifndef __USERMSRINTRIN_H
+#define __USERMSRINTRIN_H
+#ifdef __x86_64__
+
+static __inline__ unsigned long long
+    __attribute__((__always_inline__, __nodebug__, __target__("usermsr")))
+    _urdmsr(unsigned long long __A) {
+  return __builtin_ia32_urdmsr(__A);
+}
+
+static __inline__ void
+    __attribute__((__always_inline__, __nodebug__, __target__("usermsr")))
+    _uwrmsr(unsigned long long __A, unsigned long long __B) {
+  return __builtin_ia32_uwrmsr(__A, __B);
+}
+
+#endif // __x86_64__
+#endif // __USERMSRINTRIN_H
diff --git a/clang/lib/Headers/x86gprintrin.h b/clang/lib/Headers/x86gprintrin.h
index f9a765be43..ed141879fb 100644
--- a/clang/lib/Headers/x86gprintrin.h
+++ b/clang/lib/Headers/x86gprintrin.h
@@ -20,6 +20,11 @@
 #include <uintrintrin.h>
 #endif
 
+#if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
+    defined(__USERMSR__)
+#include <usermsrintrin.h>
+#endif
+
 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
     defined(__CRC32__)
 #include <crc32intrin.h>
diff --git a/clang/lib/Index/IndexBody.cpp b/clang/lib/Index/IndexBody.cpp
index e88f321f18..08136baa5d 100644
--- a/clang/lib/Index/IndexBody.cpp
+++ b/clang/lib/Index/IndexBody.cpp
@@ -77,9 +77,15 @@ class BodyIndexer : public RecursiveASTVisitor<BodyIndexer> {
     const Stmt *Parent = *It;
 
     if (auto BO = dyn_cast<BinaryOperator>(Parent)) {
-      if (BO->getOpcode() == BO_Assign && BO->getLHS()->IgnoreParenCasts() == E)
-        Roles |= (unsigned)SymbolRole::Write;
-
+      if (BO->getOpcode() == BO_Assign) {
+        if (BO->getLHS()->IgnoreParenCasts() == E)
+          Roles |= (unsigned)SymbolRole::Write;
+      } else if (auto CA = dyn_cast<CompoundAssignOperator>(Parent)) {
+        if (CA->getLHS()->IgnoreParenCasts() == E) {
+          Roles |= (unsigned)SymbolRole::Read;
+          Roles |= (unsigned)SymbolRole::Write;
+        }
+      }
     } else if (auto UO = dyn_cast<UnaryOperator>(Parent)) {
       if (UO->isIncrementDecrementOp()) {
         Roles |= (unsigned)SymbolRole::Read;
@@ -88,12 +94,6 @@ class BodyIndexer : public RecursiveASTVisitor<BodyIndexer> {
         Roles |= (unsigned)SymbolRole::AddressOf;
       }
 
-    } else if (auto CA = dyn_cast<CompoundAssignOperator>(Parent)) {
-      if (CA->getLHS()->IgnoreParenCasts() == E) {
-        Roles |= (unsigned)SymbolRole::Read;
-        Roles |= (unsigned)SymbolRole::Write;
-      }
-
     } else if (auto CE = dyn_cast<CallExpr>(Parent)) {
       if (CE->getCallee()->IgnoreParenCasts() == E) {
         addCallRole(Roles, Relations);
diff --git a/clang/lib/Index/USRGeneration.cpp b/clang/lib/Index/USRGeneration.cpp
index f778a6208d..614f5d8d2c 100644
--- a/clang/lib/Index/USRGeneration.cpp
+++ b/clang/lib/Index/USRGeneration.cpp
@@ -9,6 +9,7 @@
 #include "clang/Index/USRGeneration.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Attr.h"
+#include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/DeclVisitor.h"
 #include "clang/Basic/FileManager.h"
@@ -368,14 +369,14 @@ void USRGenerator::VisitTemplateTemplateParmDecl(
 }
 
 void USRGenerator::VisitNamespaceDecl(const NamespaceDecl *D) {
+  if (IgnoreResults)
+    return;
+  VisitDeclContext(D->getDeclContext());
   if (D->isAnonymousNamespace()) {
     Out << "@aN";
     return;
   }
-
-  VisitDeclContext(D->getDeclContext());
-  if (!IgnoreResults)
-    Out << "@N@" << D->getName();
+  Out << "@N@" << D->getName();
 }
 
 void USRGenerator::VisitFunctionTemplateDecl(const FunctionTemplateDecl *D) {
diff --git a/clang/lib/Lex/HeaderSearch.cpp b/clang/lib/Lex/HeaderSearch.cpp
index 4c8b64a374..cf1c0cc528 100644
--- a/clang/lib/Lex/HeaderSearch.cpp
+++ b/clang/lib/Lex/HeaderSearch.cpp
@@ -1623,7 +1623,7 @@ bool HeaderSearch::findUsableModuleForHeader(
     ModuleMap::KnownHeader *SuggestedModule, bool IsSystemHeaderDir) {
   if (needModuleLookup(RequestingModule, SuggestedModule)) {
     // If there is a module that corresponds to this header, suggest it.
-    hasModuleMap(File.getName(), Root, IsSystemHeaderDir);
+    hasModuleMap(File.getNameAsRequested(), Root, IsSystemHeaderDir);
     return suggestModule(*this, File, RequestingModule, SuggestedModule);
   }
   return true;
diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp
index feed1b9ecd..675ec28e51 100644
--- a/clang/lib/Lex/Lexer.cpp
+++ b/clang/lib/Lex/Lexer.cpp
@@ -47,6 +47,10 @@
 #include <tuple>
 #include <utility>
 
+#ifdef __SSE4_2__
+#include <nmmintrin.h>
+#endif
+
 using namespace clang;
 
 //===----------------------------------------------------------------------===//
@@ -1847,19 +1851,47 @@ bool Lexer::LexUnicodeIdentifierStart(Token &Result, uint32_t C,
   return true;
 }
 
+static const char *
+fastParseASCIIIdentifier(const char *CurPtr,
+                         [[maybe_unused]] const char *BufferEnd) {
+#ifdef __SSE4_2__
+  alignas(16) static constexpr char AsciiIdentifierRange[16] = {
+      '_', '_', 'A', 'Z', 'a', 'z', '0', '9',
+  };
+  constexpr ssize_t BytesPerRegister = 16;
+
+  __m128i AsciiIdentifierRangeV =
+      _mm_load_si128((const __m128i *)AsciiIdentifierRange);
+
+  while (LLVM_LIKELY(BufferEnd - CurPtr >= BytesPerRegister)) {
+    __m128i Cv = _mm_loadu_si128((const __m128i *)(CurPtr));
+
+    int Consumed = _mm_cmpistri(AsciiIdentifierRangeV, Cv,
+                                _SIDD_LEAST_SIGNIFICANT | _SIDD_CMP_RANGES |
+                                    _SIDD_UBYTE_OPS | _SIDD_NEGATIVE_POLARITY);
+    CurPtr += Consumed;
+    if (Consumed == BytesPerRegister)
+      continue;
+    return CurPtr;
+  }
+#endif
+
+  unsigned char C = *CurPtr;
+  while (isAsciiIdentifierContinue(C))
+    C = *++CurPtr;
+  return CurPtr;
+}
+
 bool Lexer::LexIdentifierContinue(Token &Result, const char *CurPtr) {
   // Match [_A-Za-z0-9]*, we have already matched an identifier start.
+
   while (true) {
-    unsigned char C = *CurPtr;
-    // Fast path.
-    if (isAsciiIdentifierContinue(C)) {
-      ++CurPtr;
-      continue;
-    }
+
+    CurPtr = fastParseASCIIIdentifier(CurPtr, BufferEnd);
 
     unsigned Size;
     // Slow path: handle trigraph, unicode codepoints, UCNs.
-    C = getCharAndSize(CurPtr, Size);
+    unsigned char C = getCharAndSize(CurPtr, Size);
     if (isAsciiIdentifierContinue(C)) {
       CurPtr = ConsumeChar(CurPtr, Size, Result);
       continue;
diff --git a/clang/lib/Lex/ModuleMap.cpp b/clang/lib/Lex/ModuleMap.cpp
index f65a5f145c..eb7cab5438 100644
--- a/clang/lib/Lex/ModuleMap.cpp
+++ b/clang/lib/Lex/ModuleMap.cpp
@@ -1398,16 +1398,17 @@ bool ModuleMap::resolveExports(Module *Mod, bool Complain) {
 }
 
 bool ModuleMap::resolveUses(Module *Mod, bool Complain) {
-  auto Unresolved = std::move(Mod->UnresolvedDirectUses);
-  Mod->UnresolvedDirectUses.clear();
+  auto *Top = Mod->getTopLevelModule();
+  auto Unresolved = std::move(Top->UnresolvedDirectUses);
+  Top->UnresolvedDirectUses.clear();
   for (auto &UDU : Unresolved) {
-    Module *DirectUse = resolveModuleId(UDU, Mod, Complain);
+    Module *DirectUse = resolveModuleId(UDU, Top, Complain);
     if (DirectUse)
-      Mod->DirectUses.push_back(DirectUse);
+      Top->DirectUses.push_back(DirectUse);
     else
-      Mod->UnresolvedDirectUses.push_back(UDU);
+      Top->UnresolvedDirectUses.push_back(UDU);
   }
-  return !Mod->UnresolvedDirectUses.empty();
+  return !Top->UnresolvedDirectUses.empty();
 }
 
 bool ModuleMap::resolveConflicts(Module *Mod, bool Complain) {
@@ -2426,7 +2427,8 @@ void ModuleMapParser::parseHeaderDecl(MMToken::TokenKind LeadingToken,
   Header.Kind = Map.headerRoleToKind(Role);
 
   // Check whether we already have an umbrella.
-  if (Header.IsUmbrella && ActiveModule->Umbrella) {
+  if (Header.IsUmbrella &&
+      !std::holds_alternative<std::monostate>(ActiveModule->Umbrella)) {
     Diags.Report(Header.FileNameLoc, diag::err_mmap_umbrella_clash)
       << ActiveModule->getFullModuleName();
     HadError = true;
@@ -2523,7 +2525,7 @@ void ModuleMapParser::parseUmbrellaDirDecl(SourceLocation UmbrellaLoc) {
   SourceLocation DirNameLoc = consumeToken();
 
   // Check whether we already have an umbrella.
-  if (ActiveModule->Umbrella) {
+  if (!std::holds_alternative<std::monostate>(ActiveModule->Umbrella)) {
     Diags.Report(DirNameLoc, diag::err_mmap_umbrella_clash)
       << ActiveModule->getFullModuleName();
     HadError = true;
diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp
index e3065c17dc..2892d4b777 100644
--- a/clang/lib/Lex/PPDirectives.cpp
+++ b/clang/lib/Lex/PPDirectives.cpp
@@ -231,9 +231,10 @@ static bool warnByDefaultOnWrongCase(StringRef Include) {
     .Cases("assert.h", "complex.h", "ctype.h", "errno.h", "fenv.h", true)
     .Cases("float.h", "inttypes.h", "iso646.h", "limits.h", "locale.h", true)
     .Cases("math.h", "setjmp.h", "signal.h", "stdalign.h", "stdarg.h", true)
-    .Cases("stdatomic.h", "stdbool.h", "stddef.h", "stdint.h", "stdio.h", true)
-    .Cases("stdlib.h", "stdnoreturn.h", "string.h", "tgmath.h", "threads.h", true)
-    .Cases("time.h", "uchar.h", "wchar.h", "wctype.h", true)
+    .Cases("stdatomic.h", "stdbool.h", "stdckdint.h", "stddef.h", true)
+    .Cases("stdint.h", "stdio.h", "stdlib.h", "stdnoreturn.h", true)
+    .Cases("string.h", "tgmath.h", "threads.h", "time.h", "uchar.h", true)
+    .Cases("wchar.h", "wctype.h", true)
 
     // C++ headers for C library facilities
     .Cases("cassert", "ccomplex", "cctype", "cerrno", "cfenv", true)
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index bcc70c04de..14a28e5a31 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -4042,7 +4042,7 @@ void Parser::ParseDeclarationSpecifiers(
         if (isKnownToBeTypeSpecifier(GetLookAheadToken(1))) {
           isInvalid = DS.SetStorageClassSpec(Actions, DeclSpec::SCS_auto, Loc,
                                              PrevSpec, DiagID, Policy);
-          if (!isInvalid)
+          if (!isInvalid && !getLangOpts().C23)
             Diag(Tok, diag::ext_auto_storage_class)
               << FixItHint::CreateRemoval(DS.getStorageClassSpecLoc());
         } else
diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp
index 9dbfc1c8c5..4d267c915f 100644
--- a/clang/lib/Parse/ParseExpr.cpp
+++ b/clang/lib/Parse/ParseExpr.cpp
@@ -1463,6 +1463,7 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
   case tok::kw_vec_step:   // unary-expression: OpenCL 'vec_step' expression
   // unary-expression: '__builtin_omp_required_simd_align' '(' type-name ')'
   case tok::kw___builtin_omp_required_simd_align:
+  case tok::kw___builtin_vectorelements:
     if (NotPrimaryExpression)
       *NotPrimaryExpression = true;
     AllowSuffix = false;
@@ -2339,7 +2340,8 @@ Parser::ParseExprAfterUnaryExprOrTypeTrait(const Token &OpTok,
   assert(OpTok.isOneOf(tok::kw_typeof, tok::kw_typeof_unqual, tok::kw_sizeof,
                        tok::kw___alignof, tok::kw_alignof, tok::kw__Alignof,
                        tok::kw_vec_step,
-                       tok::kw___builtin_omp_required_simd_align) &&
+                       tok::kw___builtin_omp_required_simd_align,
+                       tok::kw___builtin_vectorelements) &&
          "Not a typeof/sizeof/alignof/vec_step expression!");
 
   ExprResult Operand;
@@ -2460,7 +2462,8 @@ ExprResult Parser::ParseSYCLUniqueStableNameExpression() {
 ExprResult Parser::ParseUnaryExprOrTypeTraitExpression() {
   assert(Tok.isOneOf(tok::kw_sizeof, tok::kw___alignof, tok::kw_alignof,
                      tok::kw__Alignof, tok::kw_vec_step,
-                     tok::kw___builtin_omp_required_simd_align) &&
+                     tok::kw___builtin_omp_required_simd_align,
+                     tok::kw___builtin_vectorelements) &&
          "Not a sizeof/alignof/vec_step expression!");
   Token OpTok = Tok;
   ConsumeToken();
@@ -2539,6 +2542,8 @@ ExprResult Parser::ParseUnaryExprOrTypeTraitExpression() {
     ExprKind = UETT_VecStep;
   else if (OpTok.is(tok::kw___builtin_omp_required_simd_align))
     ExprKind = UETT_OpenMPRequiredSimdAlign;
+  else if (OpTok.is(tok::kw___builtin_vectorelements))
+    ExprKind = UETT_VectorElements;
 
   if (isCastExpr)
     return Actions.ActOnUnaryExprOrTypeTraitExpr(OpTok.getLocation(),
diff --git a/clang/lib/Sema/AnalysisBasedWarnings.cpp b/clang/lib/Sema/AnalysisBasedWarnings.cpp
index 77bb560eb6..0947e8b0f5 100644
--- a/clang/lib/Sema/AnalysisBasedWarnings.cpp
+++ b/clang/lib/Sema/AnalysisBasedWarnings.cpp
@@ -1983,6 +1983,12 @@ class ThreadSafetyReporter : public clang::threadSafety::ThreadSafetyHandler {
         case POK_PtPassByRef:
           DiagID = diag::warn_pt_guarded_pass_by_reference;
           break;
+        case POK_ReturnByRef:
+          DiagID = diag::warn_guarded_return_by_reference;
+          break;
+        case POK_PtReturnByRef:
+          DiagID = diag::warn_pt_guarded_return_by_reference;
+          break;
       }
       PartialDiagnosticAt Warning(Loc, S.PDiag(DiagID) << Kind
                                                        << D
@@ -2013,6 +2019,12 @@ class ThreadSafetyReporter : public clang::threadSafety::ThreadSafetyHandler {
         case POK_PtPassByRef:
           DiagID = diag::warn_pt_guarded_pass_by_reference;
           break;
+        case POK_ReturnByRef:
+          DiagID = diag::warn_guarded_return_by_reference;
+          break;
+        case POK_PtReturnByRef:
+          DiagID = diag::warn_pt_guarded_return_by_reference;
+          break;
       }
       PartialDiagnosticAt Warning(Loc, S.PDiag(DiagID) << Kind
                                                        << D
diff --git a/clang/lib/Sema/SemaCUDA.cpp b/clang/lib/Sema/SemaCUDA.cpp
index 7c4083e4ec..d993499cf4 100644
--- a/clang/lib/Sema/SemaCUDA.cpp
+++ b/clang/lib/Sema/SemaCUDA.cpp
@@ -632,6 +632,13 @@ bool HasAllowedCUDADeviceStaticInitializer(Sema &S, VarDecl *VD,
 } // namespace
 
 void Sema::checkAllowedCUDAInitializer(VarDecl *VD) {
+  // Return early if VD is inside a non-instantiated template function since
+  // the implicit constructor is not defined yet.
+  if (const FunctionDecl *FD =
+          dyn_cast_or_null<FunctionDecl>(VD->getDeclContext()))
+    if (FD->isDependentContext())
+      return;
+
   // Do not check dependent variables since the ctor/dtor/initializer are not
   // determined. Do it after instantiation.
   if (VD->isInvalidDecl() || !VD->hasInit() || !VD->hasGlobalStorage() ||
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 2594a8f97f..f3c55d059e 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -370,6 +370,32 @@ static bool SemaBuiltinOverflow(Sema &S, CallExpr *TheCall,
   if (checkArgCount(S, TheCall, 3))
     return true;
 
+  std::pair<unsigned, const char *> Builtins[] = {
+    { Builtin::BI__builtin_add_overflow, "ckd_add" },
+    { Builtin::BI__builtin_sub_overflow, "ckd_sub" },
+    { Builtin::BI__builtin_mul_overflow, "ckd_mul" },
+  };
+
+  bool CkdOperation = llvm::any_of(Builtins, [&](const std::pair<unsigned,
+    const char *> &P) {
+    return BuiltinID == P.first && TheCall->getExprLoc().isMacroID() &&
+         Lexer::getImmediateMacroName(TheCall->getExprLoc(),
+         S.getSourceManager(), S.getLangOpts()) == P.second;
+  });
+
+  auto ValidCkdIntType = [](QualType QT) {
+    // A valid checked integer type is an integer type other than a plain char,
+    // bool, a bit-precise type, or an enumeration type.
+    if (const auto *BT = QT.getCanonicalType()->getAs<BuiltinType>())
+      return (BT->getKind() >= BuiltinType::Short &&
+           BT->getKind() <= BuiltinType::Int128) || (
+           BT->getKind() >= BuiltinType::UShort &&
+           BT->getKind() <= BuiltinType::UInt128) ||
+           BT->getKind() == BuiltinType::UChar ||
+           BT->getKind() == BuiltinType::SChar;
+    return false;
+  };
+
   // First two arguments should be integers.
   for (unsigned I = 0; I < 2; ++I) {
     ExprResult Arg = S.DefaultFunctionArrayLvalueConversion(TheCall->getArg(I));
@@ -377,9 +403,10 @@ static bool SemaBuiltinOverflow(Sema &S, CallExpr *TheCall,
     TheCall->setArg(I, Arg.get());
 
     QualType Ty = Arg.get()->getType();
-    if (!Ty->isIntegerType()) {
+    bool IsValid = CkdOperation ? ValidCkdIntType(Ty) : Ty->isIntegerType();
+    if (!IsValid) {
       S.Diag(Arg.get()->getBeginLoc(), diag::err_overflow_builtin_must_be_int)
-          << Ty << Arg.get()->getSourceRange();
+          << CkdOperation << Ty << Arg.get()->getSourceRange();
       return true;
     }
   }
@@ -396,10 +423,11 @@ static bool SemaBuiltinOverflow(Sema &S, CallExpr *TheCall,
     const auto *PtrTy = Ty->getAs<PointerType>();
     if (!PtrTy ||
         !PtrTy->getPointeeType()->isIntegerType() ||
+        (!ValidCkdIntType(PtrTy->getPointeeType()) && CkdOperation) ||
         PtrTy->getPointeeType().isConstQualified()) {
       S.Diag(Arg.get()->getBeginLoc(),
              diag::err_overflow_builtin_must_be_ptr_int)
-        << Ty << Arg.get()->getSourceRange();
+        << CkdOperation << Ty << Arg.get()->getSourceRange();
       return true;
     }
   }
@@ -3092,6 +3120,11 @@ bool Sema::CheckSVEBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
       if (SemaBuiltinConstantArgRange(TheCall, ArgNum, 0, 255))
         HasError = true;
       break;
+    case SVETypeFlags::ImmCheck2_4_Mul2:
+      if (SemaBuiltinConstantArgRange(TheCall, ArgNum, 2, 4) ||
+          SemaBuiltinConstantArgMultiple(TheCall, ArgNum, 2))
+        HasError = true;
+      break;
     }
   }
 
@@ -6047,7 +6080,7 @@ bool Sema::CheckX86BuiltinTileDuplicate(CallExpr *TheCall,
     if (SemaBuiltinConstantArg(TheCall, ArgNum, Result))
       return true;
     int ArgExtValue = Result.getExtValue();
-    assert((ArgExtValue >= TileRegLow || ArgExtValue <= TileRegHigh) &&
+    assert((ArgExtValue >= TileRegLow && ArgExtValue <= TileRegHigh) &&
            "Incorrect tile register num.");
     if (ArgValues.test(ArgExtValue))
       return Diag(TheCall->getBeginLoc(),
@@ -8719,8 +8752,9 @@ ExprResult Sema::SemaConvertVectorExpr(Expr *E, TypeSourceInfo *TInfo,
                           diag::err_convertvector_non_vector)
                      << E->getSourceRange());
   if (!DstTy->isVectorType() && !DstTy->isDependentType())
-    return ExprError(Diag(BuiltinLoc,
-                          diag::err_convertvector_non_vector_type));
+    return ExprError(Diag(BuiltinLoc, diag::err_builtin_non_vector_type)
+                     << "second"
+                     << "__builtin_convertvector");
 
   if (!SrcTy->isDependentType() && !DstTy->isDependentType()) {
     unsigned SrcElts = SrcTy->castAs<VectorType>()->getNumElements();
@@ -14298,6 +14332,18 @@ static bool AnalyzeBitFieldAssignment(Sema &S, FieldDecl *Bitfield, Expr *Init,
         S.Diag(WidthExpr->getExprLoc(), diag::note_widen_bitfield)
             << BitsNeeded << ED << WidthExpr->getSourceRange();
       }
+    } else if (OriginalInit->getType()->isIntegralType(S.Context)) {
+      IntRange LikelySourceRange =
+          GetExprRange(S.Context, Init, S.isConstantEvaluatedContext(),
+                       /*Approximate=*/true);
+
+      if (LikelySourceRange.Width > FieldWidth) {
+        Expr *WidthExpr = Bitfield->getBitWidth();
+        S.Diag(InitLoc, diag::warn_bitfield_too_small_for_integral_type)
+            << Bitfield << FieldWidth << OriginalInit->getType()
+            << LikelySourceRange.Width;
+        S.Diag(WidthExpr->getExprLoc(), diag::note_declared_at);
+      }
     }
 
     return false;
@@ -15195,7 +15241,6 @@ static void CheckImplicitConversion(Sema &S, Expr *E, QualType T,
 
   if (LikelySourceRange.Width > TargetRange.Width) {
     // If the source is a constant, use a default-on diagnostic.
-    // TODO: this should happen for bitfield stores, too.
     Expr::EvalResult Result;
     if (E->EvaluateAsInt(Result, S.Context, Expr::SE_AllowSideEffects,
                          S.isConstantEvaluatedContext())) {
diff --git a/clang/lib/Sema/SemaConcept.cpp b/clang/lib/Sema/SemaConcept.cpp
index 0ef03293b4..719c6aab74 100644
--- a/clang/lib/Sema/SemaConcept.cpp
+++ b/clang/lib/Sema/SemaConcept.cpp
@@ -657,11 +657,11 @@ Sema::SetupConstraintCheckingTemplateArgumentsAndScope(
   // Collect the list of template arguments relative to the 'primary' template.
   // We need the entire list, since the constraint is completely uninstantiated
   // at this point.
-  MLTAL =
-      getTemplateInstantiationArgs(FD, /*Final=*/false, /*Innermost=*/nullptr,
-                                   /*RelativeToPrimary=*/true,
-                                   /*Pattern=*/nullptr,
-                                   /*ForConstraintInstantiation=*/true);
+  MLTAL = getTemplateInstantiationArgs(FD, FD->getLexicalDeclContext(),
+                                       /*Final=*/false, /*Innermost=*/nullptr,
+                                       /*RelativeToPrimary=*/true,
+                                       /*Pattern=*/nullptr,
+                                       /*ForConstraintInstantiation=*/true);
   if (SetupConstraintScope(FD, TemplateArgs, MLTAL, Scope))
     return std::nullopt;
 
@@ -736,7 +736,8 @@ static unsigned
 CalculateTemplateDepthForConstraints(Sema &S, const NamedDecl *ND,
                                      bool SkipForSpecialization = false) {
   MultiLevelTemplateArgumentList MLTAL = S.getTemplateInstantiationArgs(
-      ND, /*Final=*/false, /*Innermost=*/nullptr, /*RelativeToPrimary=*/true,
+      ND, ND->getLexicalDeclContext(), /*Final=*/false, /*Innermost=*/nullptr,
+      /*RelativeToPrimary=*/true,
       /*Pattern=*/nullptr,
       /*ForConstraintInstantiation=*/true, SkipForSpecialization);
   return MLTAL.getNumLevels();
@@ -770,28 +771,31 @@ namespace {
   };
 } // namespace
 
-static const Expr *SubstituteConstraintExpression(Sema &S, const NamedDecl *ND,
-                                                  const Expr *ConstrExpr) {
+static const Expr *
+SubstituteConstraintExpression(Sema &S,
+                               const Sema::TemplateCompareNewDeclInfo &DeclInfo,
+                               const Expr *ConstrExpr) {
   MultiLevelTemplateArgumentList MLTAL = S.getTemplateInstantiationArgs(
-      ND, /*Final=*/false, /*Innermost=*/nullptr,
+      DeclInfo.getDecl(), DeclInfo.getLexicalDeclContext(), /*Final=*/false,
+      /*Innermost=*/nullptr,
       /*RelativeToPrimary=*/true,
       /*Pattern=*/nullptr, /*ForConstraintInstantiation=*/true,
       /*SkipForSpecialization*/ false);
+
   if (MLTAL.getNumSubstitutedLevels() == 0)
     return ConstrExpr;
 
   Sema::SFINAETrap SFINAE(S, /*AccessCheckingSFINAE=*/false);
 
   Sema::InstantiatingTemplate Inst(
-      S, ND->getLocation(),
+      S, DeclInfo.getLocation(),
       Sema::InstantiatingTemplate::ConstraintNormalization{},
-      const_cast<NamedDecl *>(ND), SourceRange{});
-
+      const_cast<NamedDecl *>(DeclInfo.getDecl()), SourceRange{});
   if (Inst.isInvalid())
     return nullptr;
 
   std::optional<Sema::CXXThisScopeRAII> ThisScope;
-  if (auto *RD = dyn_cast<CXXRecordDecl>(ND->getDeclContext()))
+  if (auto *RD = dyn_cast<CXXRecordDecl>(DeclInfo.getDeclContext()))
     ThisScope.emplace(S, const_cast<CXXRecordDecl *>(RD), Qualifiers());
   ExprResult SubstConstr =
       S.SubstConstraintExpr(const_cast<clang::Expr *>(ConstrExpr), MLTAL);
@@ -802,13 +806,13 @@ static const Expr *SubstituteConstraintExpression(Sema &S, const NamedDecl *ND,
 
 bool Sema::AreConstraintExpressionsEqual(const NamedDecl *Old,
                                          const Expr *OldConstr,
-                                         const NamedDecl *New,
+                                         const TemplateCompareNewDeclInfo &New,
                                          const Expr *NewConstr) {
   if (OldConstr == NewConstr)
     return true;
   // C++ [temp.constr.decl]p4
-  if (Old && New && Old != New &&
-      Old->getLexicalDeclContext() != New->getLexicalDeclContext()) {
+  if (Old && !New.isInvalid() && !New.ContainsDecl(Old) &&
+      Old->getLexicalDeclContext() != New.getLexicalDeclContext()) {
     if (const Expr *SubstConstr =
             SubstituteConstraintExpression(*this, Old, OldConstr))
       OldConstr = SubstConstr;
@@ -1252,7 +1256,8 @@ static bool substituteParameterMappings(Sema &S, NormalizedConstraint &N,
   TemplateArgumentList TAL{TemplateArgumentList::OnStack,
                            CSE->getTemplateArguments()};
   MultiLevelTemplateArgumentList MLTAL = S.getTemplateInstantiationArgs(
-      CSE->getNamedConcept(), /*Final=*/false, &TAL,
+      CSE->getNamedConcept(), CSE->getNamedConcept()->getLexicalDeclContext(),
+      /*Final=*/false, &TAL,
       /*RelativeToPrimary=*/true,
       /*Pattern=*/nullptr,
       /*ForConstraintInstantiation=*/true);
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index f249d41bc9..b363b0db79 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -16722,10 +16722,8 @@ bool Sema::CheckEnumUnderlyingType(TypeSourceInfo *TI) {
     if (BT->isInteger())
       return false;
 
-  if (T->isBitIntType())
-    return false;
-
-  return Diag(UnderlyingLoc, diag::err_enum_invalid_underlying) << T;
+  return Diag(UnderlyingLoc, diag::err_enum_invalid_underlying)
+         << T << T->isBitIntType();
 }
 
 /// Check whether this is a valid redeclaration of a previous enumeration.
@@ -19447,6 +19445,20 @@ void Sema::ActOnFields(Scope *S, SourceLocation RecLoc, Decl *EnclosingDecl,
       CDecl->setIvarRBraceLoc(RBrac);
     }
   }
+
+  // Check the "counted_by" attribute to ensure that the count field exists in
+  // the struct. Make sure we're performing this check on the outer-most
+  // record.  This is a C-only feature.
+  if (!getLangOpts().CPlusPlus && Record &&
+      !isa<RecordDecl>(Record->getParent())) {
+    auto Pred = [](const Decl *D) {
+      if (const auto *FD = dyn_cast_if_present<FieldDecl>(D))
+        return FD->hasAttr<CountedByAttr>();
+      return false;
+    };
+    if (const FieldDecl *FD = Record->findFieldIf(Pred))
+      CheckCountedByAttr(S, FD);
+  }
 }
 
 /// Determine whether the given integral value is representable within
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index ed0b4d29b0..5adf058bea 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -5211,6 +5211,9 @@ static void handleCallConvAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   case ParsedAttr::AT_PreserveAll:
     D->addAttr(::new (S.Context) PreserveAllAttr(S.Context, AL));
     return;
+  case ParsedAttr::AT_M68kRTD:
+    D->addAttr(::new (S.Context) M68kRTDAttr(S.Context, AL));
+    return;
   default:
     llvm_unreachable("unexpected attribute kind");
   }
@@ -5408,6 +5411,9 @@ bool Sema::CheckCallingConvAttr(const ParsedAttr &Attrs, CallingConv &CC,
   case ParsedAttr::AT_PreserveAll:
     CC = CC_PreserveAll;
     break;
+  case ParsedAttr::AT_M68kRTD:
+    CC = CC_M68kRTD;
+    break;
   default: llvm_unreachable("unexpected attribute kind");
   }
 
@@ -8361,6 +8367,92 @@ static void handleZeroCallUsedRegsAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   D->addAttr(ZeroCallUsedRegsAttr::Create(S.Context, Kind, AL));
 }
 
+static void handleCountedByAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
+  if (!AL.isArgIdent(0)) {
+    S.Diag(AL.getLoc(), diag::err_attribute_argument_type)
+        << AL << AANT_ArgumentIdentifier;
+    return;
+  }
+
+  IdentifierLoc *IL = AL.getArgAsIdent(0);
+  CountedByAttr *CBA =
+      ::new (S.Context) CountedByAttr(S.Context, AL, IL->Ident);
+  CBA->setCountedByFieldLoc(IL->Loc);
+  D->addAttr(CBA);
+}
+
+bool Sema::CheckCountedByAttr(Scope *S, const FieldDecl *FD) {
+  const auto *CBA = FD->getAttr<CountedByAttr>();
+  const IdentifierInfo *FieldName = CBA->getCountedByField();
+  DeclarationNameInfo NameInfo(FieldName,
+                               CBA->getCountedByFieldLoc().getBegin());
+
+  LookupResult MemResult(*this, NameInfo, Sema::LookupMemberName);
+  LookupName(MemResult, S);
+
+  if (MemResult.empty()) {
+    // The "counted_by" field needs to exist within the struct.
+    LookupResult OrdResult(*this, NameInfo, Sema::LookupOrdinaryName);
+    LookupName(OrdResult, S);
+
+    if (!OrdResult.empty()) {
+      SourceRange SR = FD->getLocation();
+      Diag(SR.getBegin(), diag::err_counted_by_must_be_in_structure)
+          << FieldName << SR;
+
+      if (auto *ND = OrdResult.getAsSingle<NamedDecl>()) {
+        SR = ND->getLocation();
+        Diag(SR.getBegin(), diag::note_flexible_array_counted_by_attr_field)
+            << ND << SR;
+      }
+      return true;
+    }
+
+    CXXScopeSpec SS;
+    DeclFilterCCC<FieldDecl> Filter(FieldName);
+    return DiagnoseEmptyLookup(S, SS, MemResult, Filter, nullptr, std::nullopt,
+                               const_cast<DeclContext *>(FD->getDeclContext()));
+  }
+
+  LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
+      Context.getLangOpts().getStrictFlexArraysLevel();
+
+  if (!Decl::isFlexibleArrayMemberLike(Context, FD, FD->getType(),
+                                       StrictFlexArraysLevel, true)) {
+    // The "counted_by" attribute must be on a flexible array member.
+    SourceRange SR = FD->getLocation();
+    Diag(SR.getBegin(), diag::err_counted_by_attr_not_on_flexible_array_member)
+        << SR;
+    return true;
+  }
+
+  if (const FieldDecl *Field = MemResult.getAsSingle<FieldDecl>()) {
+    if (Field->hasAttr<CountedByAttr>()) {
+      // The "counted_by" field can't point to the flexible array member.
+      SourceRange SR = CBA->getCountedByFieldLoc();
+      Diag(SR.getBegin(), diag::err_counted_by_attr_refers_to_flexible_array)
+          << CBA->getCountedByField() << SR;
+      return true;
+    }
+
+    if (!Field->getType()->isIntegerType() ||
+        Field->getType()->isBooleanType()) {
+      // The "counted_by" field must have an integer type.
+      SourceRange SR = CBA->getCountedByFieldLoc();
+      Diag(SR.getBegin(),
+           diag::err_flexible_array_counted_by_attr_field_not_integer)
+          << CBA->getCountedByField() << SR;
+
+      SR = Field->getLocation();
+      Diag(SR.getBegin(), diag::note_flexible_array_counted_by_attr_field)
+          << Field << SR;
+      return true;
+    }
+  }
+
+  return false;
+}
+
 static void handleFunctionReturnThunksAttr(Sema &S, Decl *D,
                                            const ParsedAttr &AL) {
   StringRef KindStr;
@@ -9267,6 +9359,7 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL,
   case ParsedAttr::AT_AArch64VectorPcs:
   case ParsedAttr::AT_AArch64SVEPcs:
   case ParsedAttr::AT_AMDGPUKernelCall:
+  case ParsedAttr::AT_M68kRTD:
     handleCallConvAttr(S, D, AL);
     break;
   case ParsedAttr::AT_Suppress:
@@ -9314,6 +9407,10 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL,
     handleAvailableOnlyInDefaultEvalMethod(S, D, AL);
     break;
 
+  case ParsedAttr::AT_CountedBy:
+    handleCountedByAttr(S, D, AL);
+    break;
+
   // Microsoft attributes:
   case ParsedAttr::AT_LayoutVersion:
     handleLayoutVersion(S, D, AL);
diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp
index f9c010b1a0..0193e476b3 100644
--- a/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/clang/lib/Sema/SemaDeclCXX.cpp
@@ -1299,8 +1299,9 @@ static bool checkTupleLikeDecomposition(Sema &S,
       //   in the associated namespaces.
       Expr *Get = UnresolvedLookupExpr::Create(
           S.Context, nullptr, NestedNameSpecifierLoc(), SourceLocation(),
-          DeclarationNameInfo(GetDN, Loc), /*RequiresADL*/true, &Args,
-          UnresolvedSetIterator(), UnresolvedSetIterator());
+          DeclarationNameInfo(GetDN, Loc), /*RequiresADL*/ true, &Args,
+          UnresolvedSetIterator(), UnresolvedSetIterator(),
+          /*KnownDependent=*/false);
 
       Expr *Arg = E.get();
       E = S.BuildCallExpr(nullptr, Get, Loc, Arg, Loc);
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index cf45fc3880..ed0b150d52 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -35,6 +35,7 @@
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/Specifiers.h"
 #include "clang/Basic/TargetInfo.h"
+#include "clang/Basic/TypeTraits.h"
 #include "clang/Lex/LiteralSupport.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Sema/AnalysisBasedWarnings.h"
@@ -2458,7 +2459,8 @@ bool Sema::DiagnoseDependentMemberLookup(const LookupResult &R) {
 bool Sema::DiagnoseEmptyLookup(Scope *S, CXXScopeSpec &SS, LookupResult &R,
                                CorrectionCandidateCallback &CCC,
                                TemplateArgumentListInfo *ExplicitTemplateArgs,
-                               ArrayRef<Expr *> Args, TypoExpr **Out) {
+                               ArrayRef<Expr *> Args, DeclContext *LookupCtx,
+                               TypoExpr **Out) {
   DeclarationName Name = R.getLookupName();
 
   unsigned diagnostic = diag::err_undeclared_var_use;
@@ -2474,7 +2476,8 @@ bool Sema::DiagnoseEmptyLookup(Scope *S, CXXScopeSpec &SS, LookupResult &R,
   // unqualified lookup.  This is useful when (for example) the
   // original lookup would not have found something because it was a
   // dependent name.
-  DeclContext *DC = SS.isEmpty() ? CurContext : nullptr;
+  DeclContext *DC =
+      LookupCtx ? LookupCtx : (SS.isEmpty() ? CurContext : nullptr);
   while (DC) {
     if (isa<CXXRecordDecl>(DC)) {
       LookupQualifiedName(R, DC);
@@ -2517,12 +2520,12 @@ bool Sema::DiagnoseEmptyLookup(Scope *S, CXXScopeSpec &SS, LookupResult &R,
           emitEmptyLookupTypoDiagnostic(TC, *this, SS, Name, TypoLoc, Args,
                                         diagnostic, diagnostic_suggest);
         },
-        nullptr, CTK_ErrorRecovery);
+        nullptr, CTK_ErrorRecovery, LookupCtx);
     if (*Out)
       return true;
-  } else if (S &&
-             (Corrected = CorrectTypo(R.getLookupNameInfo(), R.getLookupKind(),
-                                      S, &SS, CCC, CTK_ErrorRecovery))) {
+  } else if (S && (Corrected =
+                       CorrectTypo(R.getLookupNameInfo(), R.getLookupKind(), S,
+                                   &SS, CCC, CTK_ErrorRecovery, LookupCtx))) {
     std::string CorrectedStr(Corrected.getAsString(getLangOpts()));
     bool DroppedSpecifier =
         Corrected.WillReplaceSpecifier() && Name.getAsString() == CorrectedStr;
@@ -2812,7 +2815,7 @@ Sema::ActOnIdExpression(Scope *S, CXXScopeSpec &SS,
     // a template name, but we happen to have always already looked up the name
     // before we get here if it must be a template name.
     if (DiagnoseEmptyLookup(S, SS, R, CCC ? *CCC : DefaultValidator, nullptr,
-                            std::nullopt, &TE)) {
+                            std::nullopt, nullptr, &TE)) {
       if (TE && KeywordReplacement) {
         auto &State = getTypoExprState(TE);
         auto BestTC = State.Consumer->getNextCorrection();
@@ -4351,6 +4354,18 @@ static bool CheckVecStepTraitOperandType(Sema &S, QualType T,
   return false;
 }
 
+static bool CheckVectorElementsTraitOperandType(Sema &S, QualType T,
+                                                SourceLocation Loc,
+                                                SourceRange ArgRange) {
+  // builtin_vectorelements supports both fixed-sized and scalable vectors.
+  if (!T->isVectorType() && !T->isSizelessVectorType())
+    return S.Diag(Loc, diag::err_builtin_non_vector_type)
+           << ""
+           << "__builtin_vectorelements" << T << ArgRange;
+
+  return false;
+}
+
 static bool CheckExtensionTraitOperandType(Sema &S, QualType T,
                                            SourceLocation Loc,
                                            SourceRange ArgRange,
@@ -4452,6 +4467,10 @@ bool Sema::CheckUnaryExprOrTypeTraitOperand(Expr *E,
     return CheckVecStepTraitOperandType(*this, ExprTy, E->getExprLoc(),
                                         E->getSourceRange());
 
+  if (ExprKind == UETT_VectorElements)
+    return CheckVectorElementsTraitOperandType(*this, ExprTy, E->getExprLoc(),
+                                               E->getSourceRange());
+
   // Explicitly list some types as extensions.
   if (!CheckExtensionTraitOperandType(*this, ExprTy, E->getExprLoc(),
                                       E->getSourceRange(), ExprKind))
@@ -4743,6 +4762,10 @@ bool Sema::CheckUnaryExprOrTypeTraitOperand(QualType ExprType,
   if (ExprKind == UETT_VecStep)
     return CheckVecStepTraitOperandType(*this, ExprType, OpLoc, ExprRange);
 
+  if (ExprKind == UETT_VectorElements)
+    return CheckVectorElementsTraitOperandType(*this, ExprType, OpLoc,
+                                               ExprRange);
+
   // Explicitly list some types as extensions.
   if (!CheckExtensionTraitOperandType(*this, ExprType, OpLoc, ExprRange,
                                       ExprKind))
@@ -4849,6 +4872,8 @@ Sema::CreateUnaryExprOrTypeTraitExpr(Expr *E, SourceLocation OpLoc,
   } else if (E->refersToBitField()) {  // C99 6.5.3.4p1.
     Diag(E->getExprLoc(), diag::err_sizeof_alignof_typeof_bitfield) << 0;
     isInvalid = true;
+  } else if (ExprKind == UETT_VectorElements) {
+    isInvalid = CheckUnaryExprOrTypeTraitOperand(E, UETT_VectorElements);
   } else {
     isInvalid = CheckUnaryExprOrTypeTraitOperand(E, UETT_SizeOf);
   }
@@ -9184,7 +9209,7 @@ QualType Sema::CheckConditionalOperands(ExprResult &Cond, ExprResult &LHS,
   if (checkCondition(*this, Cond.get(), QuestionLoc))
     return QualType();
 
-  // Now check the two expressions.
+  // Handle vectors.
   if (LHS.get()->getType()->isVectorType() ||
       RHS.get()->getType()->isVectorType())
     return CheckVectorOperands(LHS, RHS, QuestionLoc, /*isCompAssign*/ false,
@@ -9242,11 +9267,6 @@ QualType Sema::CheckConditionalOperands(ExprResult &Cond, ExprResult &LHS,
     return ResTy;
   }
 
-  // And if they're both bfloat (which isn't arithmetic), that's fine too.
-  if (LHSTy->isBFloat16Type() && RHSTy->isBFloat16Type()) {
-    return Context.getCommonSugaredType(LHSTy, RHSTy);
-  }
-
   // If both operands are the same structure or union type, the result is that
   // type.
   if (const RecordType *LHSRT = LHSTy->getAs<RecordType>()) {    // C99 6.5.15p3
@@ -9318,17 +9338,17 @@ QualType Sema::CheckConditionalOperands(ExprResult &Cond, ExprResult &LHS,
       /*IsIntFirstExpr=*/false))
     return LHSTy;
 
-  // Allow ?: operations in which both operands have the same
-  // built-in sizeless type.
-  if (LHSTy->isSizelessBuiltinType() && Context.hasSameType(LHSTy, RHSTy))
-    return Context.getCommonSugaredType(LHSTy, RHSTy);
-
   // Emit a better diagnostic if one of the expressions is a null pointer
   // constant and the other is not a pointer type. In this case, the user most
   // likely forgot to take the address of the other expression.
   if (DiagnoseConditionalForNull(LHS.get(), RHS.get(), QuestionLoc))
     return QualType();
 
+  // Finally, if the LHS and RHS types are canonically the same type, we can
+  // use the common sugared type.
+  if (Context.hasSameType(LHSTy, RHSTy))
+    return Context.getCommonSugaredType(LHSTy, RHSTy);
+
   // Otherwise, the operands are not compatible.
   Diag(QuestionLoc, diag::err_typecheck_cond_incompatible_operands)
     << LHSTy << RHSTy << LHS.get()->getSourceRange()
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index fd95b16b84..8f945bc764 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -15,6 +15,7 @@
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/ExprObjC.h"
 #include "clang/AST/ExprOpenMP.h"
+#include "clang/AST/IgnoreExpr.h"
 #include "clang/AST/TypeLoc.h"
 #include "clang/Basic/CharInfo.h"
 #include "clang/Basic/SourceManager.h"
@@ -170,22 +171,9 @@ static void updateStringLiteralType(Expr *E, QualType Ty) {
   while (true) {
     E->setType(Ty);
     E->setValueKind(VK_PRValue);
-    if (isa<StringLiteral>(E) || isa<ObjCEncodeExpr>(E)) {
-      break;
-    } else if (ParenExpr *PE = dyn_cast<ParenExpr>(E)) {
-      E = PE->getSubExpr();
-    } else if (UnaryOperator *UO = dyn_cast<UnaryOperator>(E)) {
-      assert(UO->getOpcode() == UO_Extension);
-      E = UO->getSubExpr();
-    } else if (GenericSelectionExpr *GSE = dyn_cast<GenericSelectionExpr>(E)) {
-      E = GSE->getResultExpr();
-    } else if (ChooseExpr *CE = dyn_cast<ChooseExpr>(E)) {
-      E = CE->getChosenSubExpr();
-    } else if (PredefinedExpr *PE = dyn_cast<PredefinedExpr>(E)) {
-      E = PE->getFunctionName();
-    } else {
-      llvm_unreachable("unexpected expr in string literal init");
-    }
+    if (isa<StringLiteral>(E) || isa<ObjCEncodeExpr>(E))
+      break;
+    E = IgnoreParensSingleStep(E);
   }
 }
 
@@ -194,20 +182,9 @@ static void updateStringLiteralType(Expr *E, QualType Ty) {
 static void updateGNUCompoundLiteralRValue(Expr *E) {
   while (true) {
     E->setValueKind(VK_PRValue);
-    if (isa<CompoundLiteralExpr>(E)) {
-      break;
-    } else if (ParenExpr *PE = dyn_cast<ParenExpr>(E)) {
-      E = PE->getSubExpr();
-    } else if (UnaryOperator *UO = dyn_cast<UnaryOperator>(E)) {
-      assert(UO->getOpcode() == UO_Extension);
-      E = UO->getSubExpr();
-    } else if (GenericSelectionExpr *GSE = dyn_cast<GenericSelectionExpr>(E)) {
-      E = GSE->getResultExpr();
-    } else if (ChooseExpr *CE = dyn_cast<ChooseExpr>(E)) {
-      E = CE->getChosenSubExpr();
-    } else {
-      llvm_unreachable("unexpected expr in array compound literal init");
-    }
+    if (isa<CompoundLiteralExpr>(E))
+      break;
+    E = IgnoreParensSingleStep(E);
   }
 }
 
diff --git a/clang/lib/Sema/SemaLambda.cpp b/clang/lib/Sema/SemaLambda.cpp
index 421048aaff..ca09b0481b 100644
--- a/clang/lib/Sema/SemaLambda.cpp
+++ b/clang/lib/Sema/SemaLambda.cpp
@@ -393,8 +393,7 @@ void Sema::DiagnoseInvalidExplicitObjectParameterInLambda(
   CXXRecordDecl *RD = Method->getParent();
   if (Method->getType()->isDependentType())
     return;
-  if (RD->getLambdaCaptureDefault() == LambdaCaptureDefault::LCD_None &&
-      RD->capture_size() == 0)
+  if (RD->isCapturelessLambda())
     return;
   QualType ExplicitObjectParameterType = Method->getParamDecl(0)
                                              ->getType()
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index ce78994e65..d57a7ad8f4 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -7688,7 +7688,7 @@ bool Sema::CheckNonDependentConversions(
     QualType ParamType = ParamTypes[I + Offset];
     if (!ParamType->isDependentType()) {
       unsigned ConvIdx = PO == OverloadCandidateParamOrder::Reversed
-                             ? 0
+                             ? Args.size() - 1 - (ThisConversions + I)
                              : (ThisConversions + I);
       Conversions[ConvIdx]
         = TryCopyInitialization(*this, Args[I], ParamType,
@@ -10085,11 +10085,19 @@ getImplicitObjectParamType(ASTContext &Context, const FunctionDecl *F) {
   return M->getFunctionObjectParameterReferenceType();
 }
 
-static bool haveSameParameterTypes(ASTContext &Context, const FunctionDecl *F1,
-                                   const FunctionDecl *F2) {
+// As a Clang extension, allow ambiguity among F1 and F2 if they represent
+// represent the same entity.
+static bool allowAmbiguity(ASTContext &Context, const FunctionDecl *F1,
+                           const FunctionDecl *F2) {
   if (declaresSameEntity(F1, F2))
     return true;
-
+  if (F1->isTemplateInstantiation() && F2->isTemplateInstantiation() &&
+      declaresSameEntity(F1->getPrimaryTemplate(), F2->getPrimaryTemplate())) {
+    return true;
+  }
+  // TODO: It is not clear whether comparing parameters is necessary (i.e.
+  // different functions with same params). Consider removing this (as no test
+  // fail w/o it).
   auto NextParam = [&](const FunctionDecl *F, unsigned &I, bool First) {
     if (First) {
       if (std::optional<QualType> T = getImplicitObjectParamType(Context, F))
@@ -10274,14 +10282,14 @@ bool clang::isBetterOverloadCandidate(
     case ImplicitConversionSequence::Worse:
       if (Cand1.Function && Cand2.Function &&
           Cand1.isReversed() != Cand2.isReversed() &&
-          haveSameParameterTypes(S.Context, Cand1.Function, Cand2.Function)) {
+          allowAmbiguity(S.Context, Cand1.Function, Cand2.Function)) {
         // Work around large-scale breakage caused by considering reversed
         // forms of operator== in C++20:
         //
-        // When comparing a function against a reversed function with the same
-        // parameter types, if we have a better conversion for one argument and
-        // a worse conversion for the other, the implicit conversion sequences
-        // are treated as being equally good.
+        // When comparing a function against a reversed function, if we have a
+        // better conversion for one argument and a worse conversion for the
+        // other, the implicit conversion sequences are treated as being equally
+        // good.
         //
         // This prevents a comparison function from being considered ambiguous
         // with a reversed form that is written in the same way.
@@ -14421,7 +14429,7 @@ ExprResult Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
           llvm::SmallVector<FunctionDecl*, 4> AmbiguousWith;
           for (OverloadCandidate &Cand : CandidateSet) {
             if (Cand.Viable && Cand.Function && Cand.isReversed() &&
-                haveSameParameterTypes(Context, Cand.Function, FnDecl)) {
+                allowAmbiguity(Context, Cand.Function, FnDecl)) {
               for (unsigned ArgIdx = 0; ArgIdx < 2; ++ArgIdx) {
                 if (CompareImplicitConversionSequences(
                         *this, OpLoc, Cand.Conversions[ArgIdx],
diff --git a/clang/lib/Sema/SemaRISCVVectorLookup.cpp b/clang/lib/Sema/SemaRISCVVectorLookup.cpp
index ae584dc687..acdf326000 100644
--- a/clang/lib/Sema/SemaRISCVVectorLookup.cpp
+++ b/clang/lib/Sema/SemaRISCVVectorLookup.cpp
@@ -205,6 +205,8 @@ void RISCVIntrinsicManagerImpl::ConstructRVVIntrinsics(
   static const std::pair<const char *, RVVRequire> FeatureCheckList[] = {
       {"64bit", RVV_REQ_RV64},
       {"xsfvcp", RVV_REQ_Xsfvcp},
+      {"xsfvqmaccdod", RVV_REQ_Xsfvqmaccdod},
+      {"xsfvqmaccqoq", RVV_REQ_Xsfvqmaccqoq},
       {"experimental-zvbb", RVV_REQ_Zvbb},
       {"experimental-zvbc", RVV_REQ_Zvbc},
       {"experimental-zvkb", RVV_REQ_Zvkb},
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index ff370dd1e0..f0197f7c10 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -1995,10 +1995,13 @@ DeclResult Sema::CheckClassTemplate(
     // for a friend in a dependent context: the template parameter list itself
     // could be dependent.
     if (!(TUK == TUK_Friend && CurContext->isDependentContext()) &&
-        !TemplateParameterListsAreEqual(TemplateParams,
-                                   PrevClassTemplate->getTemplateParameters(),
-                                        /*Complain=*/true,
-                                        TPL_TemplateMatch))
+        !TemplateParameterListsAreEqual(
+            TemplateCompareNewDeclInfo(SemanticContext ? SemanticContext
+                                                       : CurContext,
+                                       CurContext, KWLoc),
+            TemplateParams, PrevClassTemplate,
+            PrevClassTemplate->getTemplateParameters(), /*Complain=*/true,
+            TPL_TemplateMatch))
       return true;
 
     // C++ [temp.class]p4:
@@ -4982,7 +4985,7 @@ ExprResult Sema::BuildTemplateIdExpr(const CXXScopeSpec &SS,
       return ExprError();
     }
   }
-
+  bool KnownDependent = false;
   // In C++1y, check variable template ids.
   if (R.getAsSingle<VarTemplateDecl>()) {
     ExprResult Res = CheckVarTemplateId(SS, R.getLookupNameInfo(),
@@ -4991,6 +4994,7 @@ ExprResult Sema::BuildTemplateIdExpr(const CXXScopeSpec &SS,
     if (Res.isInvalid() || Res.isUsable())
       return Res;
     // Result is dependent. Carry on to build an UnresolvedLookupEpxr.
+    KnownDependent = true;
   }
 
   if (R.getAsSingle<ConceptDecl>()) {
@@ -5002,13 +5006,10 @@ ExprResult Sema::BuildTemplateIdExpr(const CXXScopeSpec &SS,
   // We don't want lookup warnings at this point.
   R.suppressDiagnostics();
 
-  UnresolvedLookupExpr *ULE
-    = UnresolvedLookupExpr::Create(Context, R.getNamingClass(),
-                                   SS.getWithLocInContext(Context),
-                                   TemplateKWLoc,
-                                   R.getLookupNameInfo(),
-                                   RequiresADL, TemplateArgs,
-                                   R.begin(), R.end());
+  UnresolvedLookupExpr *ULE = UnresolvedLookupExpr::Create(
+      Context, R.getNamingClass(), SS.getWithLocInContext(Context),
+      TemplateKWLoc, R.getLookupNameInfo(), RequiresADL, TemplateArgs,
+      R.begin(), R.end(), KnownDependent);
 
   return ULE;
 }
@@ -6205,7 +6206,7 @@ bool Sema::CheckTemplateArgumentList(
     CXXThisScopeRAII(*this, RD, ThisQuals, RD != nullptr);
 
     MultiLevelTemplateArgumentList MLTAL = getTemplateInstantiationArgs(
-        Template, /*Final=*/false, &StackTemplateArgs,
+        Template, NewContext, /*Final=*/false, &StackTemplateArgs,
         /*RelativeToPrimary=*/true,
         /*Pattern=*/nullptr,
         /*ForConceptInstantiation=*/true);
@@ -8019,7 +8020,8 @@ Sema::BuildExpressionFromIntegralTemplateArgument(const TemplateArgument &Arg,
 
 /// Match two template parameters within template parameter lists.
 static bool MatchTemplateParameterKind(
-    Sema &S, NamedDecl *New, const NamedDecl *NewInstFrom, NamedDecl *Old,
+    Sema &S, NamedDecl *New,
+    const Sema::TemplateCompareNewDeclInfo &NewInstFrom, NamedDecl *Old,
     const NamedDecl *OldInstFrom, bool Complain,
     Sema::TemplateParameterListEqualKind Kind, SourceLocation TemplateArgLoc) {
   // Check the actual kind (type, non-type, template).
@@ -8107,8 +8109,8 @@ static bool MatchTemplateParameterKind(
   // For template template parameters, check the template parameter types.
   // The template parameter lists of template template
   // parameters must agree.
-  else if (TemplateTemplateParmDecl *OldTTP
-                                    = dyn_cast<TemplateTemplateParmDecl>(Old)) {
+  else if (TemplateTemplateParmDecl *OldTTP =
+               dyn_cast<TemplateTemplateParmDecl>(Old)) {
     TemplateTemplateParmDecl *NewTTP = cast<TemplateTemplateParmDecl>(New);
     if (!S.TemplateParameterListsAreEqual(
             NewInstFrom, NewTTP->getTemplateParameters(), OldInstFrom,
@@ -8212,7 +8214,7 @@ void DiagnoseTemplateParameterListArityMismatch(Sema &S,
 /// \returns True if the template parameter lists are equal, false
 /// otherwise.
 bool Sema::TemplateParameterListsAreEqual(
-    const NamedDecl *NewInstFrom, TemplateParameterList *New,
+    const TemplateCompareNewDeclInfo &NewInstFrom, TemplateParameterList *New,
     const NamedDecl *OldInstFrom, TemplateParameterList *Old, bool Complain,
     TemplateParameterListEqualKind Kind, SourceLocation TemplateArgLoc) {
   if (Old->size() != New->size() && Kind != TPL_TemplateTemplateArgumentMatch) {
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index 62fbd903a0..8f115f2177 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -2889,7 +2889,7 @@ CheckDeducedArgumentConstraints(Sema &S, TemplateDeclT *Template,
                                   CanonicalDeducedArgs};
 
   MultiLevelTemplateArgumentList MLTAL = S.getTemplateInstantiationArgs(
-      Template, /*Final=*/false,
+      Template, Template->getDeclContext(), /*Final=*/false,
       /*InnerMost=*/NeedsReplacement ? nullptr : &DeducedTAL,
       /*RelativeToPrimary=*/true, /*Pattern=*/
       nullptr, /*ForConstraintInstantiation=*/true);
diff --git a/clang/lib/Sema/SemaTemplateInstantiate.cpp b/clang/lib/Sema/SemaTemplateInstantiate.cpp
index 23de64080a..effc97a033 100644
--- a/clang/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiate.cpp
@@ -312,6 +312,10 @@ Response HandleGenericDeclContext(const Decl *CurDecl) {
 /// \param ND the declaration for which we are computing template instantiation
 /// arguments.
 ///
+/// \param DC In the event we don't HAVE a declaration yet, we instead provide
+///  the decl context where it will be created.  In this case, the `Innermost`
+///  should likely be provided.  If ND is non-null, this is ignored.
+///
 /// \param Innermost if non-NULL, specifies a template argument list for the
 /// template declaration passed as ND.
 ///
@@ -331,10 +335,11 @@ Response HandleGenericDeclContext(const Decl *CurDecl) {
 /// arguments on an enclosing class template.
 
 MultiLevelTemplateArgumentList Sema::getTemplateInstantiationArgs(
-    const NamedDecl *ND, bool Final, const TemplateArgumentList *Innermost,
-    bool RelativeToPrimary, const FunctionDecl *Pattern,
-    bool ForConstraintInstantiation, bool SkipForSpecialization) {
-  assert(ND && "Can't find arguments for a decl if one isn't provided");
+    const NamedDecl *ND, const DeclContext *DC, bool Final,
+    const TemplateArgumentList *Innermost, bool RelativeToPrimary,
+    const FunctionDecl *Pattern, bool ForConstraintInstantiation,
+    bool SkipForSpecialization) {
+  assert((ND || DC) && "Can't find arguments for a decl if one isn't provided");
   // Accumulate the set of template argument lists in this structure.
   MultiLevelTemplateArgumentList Result;
 
@@ -346,6 +351,9 @@ MultiLevelTemplateArgumentList Sema::getTemplateInstantiationArgs(
     CurDecl = Response::UseNextDecl(ND).NextDecl;
   }
 
+  if (!ND)
+    CurDecl = Decl::castFromDeclContext(DC);
+
   while (!CurDecl->isFileContextDecl()) {
     Response R;
     if (const auto *VarTemplSpec =
@@ -369,6 +377,8 @@ MultiLevelTemplateArgumentList Sema::getTemplateInstantiationArgs(
       R = HandleImplicitConceptSpecializationDecl(CSD, Result);
     } else if (const auto *FTD = dyn_cast<FunctionTemplateDecl>(CurDecl)) {
       R = HandleFunctionTemplateDecl(FTD, Result);
+    } else if (const auto *CTD = dyn_cast<ClassTemplateDecl>(CurDecl)) {
+      R = Response::ChangeDecl(CTD->getLexicalDeclContext());
     } else if (!isa<DeclContext>(CurDecl)) {
       R = Response::DontClearRelativeToPrimaryNextDecl(CurDecl);
       if (CurDecl->getDeclContext()->isTranslationUnit()) {
@@ -2662,7 +2672,9 @@ TypeSourceInfo *Sema::SubstFunctionDeclType(TypeSourceInfo *T,
   } else {
     Result = Instantiator.TransformType(TLB, TL);
   }
-  if (Result.isNull())
+  // When there are errors resolving types, clang may use IntTy as a fallback,
+  // breaking our assumption that function declarations have function types.
+  if (Result.isNull() || !Result->isFunctionType())
     return nullptr;
 
   return TLB.getTypeSourceInfo(Context, Result);
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index ec0f7d1fe0..78a7892a35 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -4626,7 +4626,8 @@ bool Sema::InstantiateDefaultArgument(SourceLocation CallLoc, FunctionDecl *FD,
   // template<typename T>
   // A<T> Foo(int a = A<T>::FooImpl());
   MultiLevelTemplateArgumentList TemplateArgs = getTemplateInstantiationArgs(
-      FD, /*Final=*/false, nullptr, /*RelativeToPrimary=*/true);
+      FD, FD->getLexicalDeclContext(), /*Final=*/false, nullptr,
+      /*RelativeToPrimary=*/true);
 
   if (SubstDefaultArgument(CallLoc, Param, TemplateArgs, /*ForCallExpr*/ true))
     return true;
@@ -4665,7 +4666,8 @@ void Sema::InstantiateExceptionSpec(SourceLocation PointOfInstantiation,
   LocalInstantiationScope Scope(*this);
 
   MultiLevelTemplateArgumentList TemplateArgs = getTemplateInstantiationArgs(
-      Decl, /*Final=*/false, nullptr, /*RelativeToPrimary*/ true);
+      Decl, Decl->getLexicalDeclContext(), /*Final=*/false, nullptr,
+      /*RelativeToPrimary*/ true);
 
   // FIXME: We can't use getTemplateInstantiationPattern(false) in general
   // here, because for a non-defining friend declaration in a class template,
@@ -5107,7 +5109,8 @@ void Sema::InstantiateFunctionDefinition(SourceLocation PointOfInstantiation,
     SetDeclDefaulted(Function, PatternDecl->getLocation());
   } else {
     MultiLevelTemplateArgumentList TemplateArgs = getTemplateInstantiationArgs(
-        Function, /*Final=*/false, nullptr, false, PatternDecl);
+        Function, Function->getLexicalDeclContext(), /*Final=*/false, nullptr,
+        false, PatternDecl);
 
     // Substitute into the qualifier; we can get a substitution failure here
     // through evil use of alias templates.
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 068971f813..28b81c1768 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -136,7 +136,8 @@ static void diagnoseBadTypeAttribute(Sema &S, const ParsedAttr &attr,
   case ParsedAttr::AT_Pcs:                                                     \
   case ParsedAttr::AT_IntelOclBicc:                                            \
   case ParsedAttr::AT_PreserveMost:                                            \
-  case ParsedAttr::AT_PreserveAll
+  case ParsedAttr::AT_PreserveAll:                                             \
+  case ParsedAttr::AT_M68kRTD
 
 // Function type attributes.
 #define FUNCTION_TYPE_ATTRS_CASELIST                                           \
@@ -2581,6 +2582,27 @@ QualType Sema::BuildArrayType(QualType T, ArrayType::ArraySizeModifier ASM,
     return QualType();
   }
 
+  auto IsStaticAssertLike = [](const Expr *ArraySize, ASTContext &Context) {
+    if (!ArraySize)
+      return false;
+
+    // If the array size expression is a conditional expression whose branches
+    // are both integer constant expressions, one negative and one positive,
+    // then it's assumed to be like an old-style static assertion. e.g.,
+    //   int old_style_assert[expr ? 1 : -1];
+    // We will accept any integer constant expressions instead of assuming the
+    // values 1 and -1 are always used.
+    if (const auto *CondExpr = dyn_cast_if_present<ConditionalOperator>(
+            ArraySize->IgnoreParenImpCasts())) {
+      std::optional<llvm::APSInt> LHS =
+          CondExpr->getLHS()->getIntegerConstantExpr(Context);
+      std::optional<llvm::APSInt> RHS =
+          CondExpr->getRHS()->getIntegerConstantExpr(Context);
+      return LHS && RHS && LHS->isNegative() != RHS->isNegative();
+    }
+    return false;
+  };
+
   // VLAs always produce at least a -Wvla diagnostic, sometimes an error.
   unsigned VLADiag;
   bool VLAIsError;
@@ -2597,6 +2619,15 @@ QualType Sema::BuildArrayType(QualType T, ArrayType::ArraySizeModifier ASM,
   } else if (getLangOpts().OpenMP && isInOpenMPTaskUntiedContext()) {
     VLADiag = diag::err_openmp_vla_in_task_untied;
     VLAIsError = true;
+  } else if (getLangOpts().CPlusPlus) {
+    if (getLangOpts().CPlusPlus11 && IsStaticAssertLike(ArraySize, Context))
+      VLADiag = getLangOpts().GNUMode
+                    ? diag::ext_vla_cxx_in_gnu_mode_static_assert
+                    : diag::ext_vla_cxx_static_assert;
+    else
+      VLADiag = getLangOpts().GNUMode ? diag::ext_vla_cxx_in_gnu_mode
+                                      : diag::ext_vla_cxx;
+    VLAIsError = false;
   } else {
     VLADiag = diag::ext_vla;
     VLAIsError = false;
@@ -6683,8 +6714,8 @@ ParsedType Sema::CreateParsedType(QualType T, TypeSourceInfo *TInfo) {
   // FIXME: LocInfoTypes are "transient", only needed for passing to/from Parser
   // and Sema during declaration parsing. Try deallocating/caching them when
   // it's appropriate, instead of allocating them and keeping them around.
-  LocInfoType *LocT = (LocInfoType*)BumpAlloc.Allocate(sizeof(LocInfoType),
-                                                       TypeAlignment);
+  LocInfoType *LocT = (LocInfoType *)BumpAlloc.Allocate(sizeof(LocInfoType),
+                                                        alignof(LocInfoType));
   new (LocT) LocInfoType(T, TInfo);
   assert(LocT->getTypeClass() != T->getTypeClass() &&
          "LocInfoType's TypeClass conflicts with an existing Type class");
@@ -7802,6 +7833,8 @@ static Attr *getCCTypeAttr(ASTContext &Ctx, ParsedAttr &Attr) {
     return createSimpleAttr<PreserveMostAttr>(Ctx, Attr);
   case ParsedAttr::AT_PreserveAll:
     return createSimpleAttr<PreserveAllAttr>(Ctx, Attr);
+  case ParsedAttr::AT_M68kRTD:
+    return createSimpleAttr<M68kRTDAttr>(Ctx, Attr);
   }
   llvm_unreachable("unexpected attribute kind!");
 }
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 9ea8c8eaca..42b48d230a 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -912,9 +912,10 @@ ASTSelectorLookupTrait::ReadKey(const unsigned char* d, unsigned) {
   using namespace llvm::support;
 
   SelectorTable &SelTable = Reader.getContext().Selectors;
-  unsigned N = endian::readNext<uint16_t, little, unaligned>(d);
+  unsigned N =
+      endian::readNext<uint16_t, llvm::endianness::little, unaligned>(d);
   IdentifierInfo *FirstII = Reader.getLocalIdentifier(
-      F, endian::readNext<uint32_t, little, unaligned>(d));
+      F, endian::readNext<uint32_t, llvm::endianness::little, unaligned>(d));
   if (N == 0)
     return SelTable.getNullarySelector(FirstII);
   else if (N == 1)
@@ -924,7 +925,7 @@ ASTSelectorLookupTrait::ReadKey(const unsigned char* d, unsigned) {
   Args.push_back(FirstII);
   for (unsigned I = 1; I != N; ++I)
     Args.push_back(Reader.getLocalIdentifier(
-        F, endian::readNext<uint32_t, little, unaligned>(d)));
+        F, endian::readNext<uint32_t, llvm::endianness::little, unaligned>(d)));
 
   return SelTable.getSelector(N, Args.data());
 }
@@ -937,9 +938,11 @@ ASTSelectorLookupTrait::ReadData(Selector, const unsigned char* d,
   data_type Result;
 
   Result.ID = Reader.getGlobalSelectorID(
-      F, endian::readNext<uint32_t, little, unaligned>(d));
-  unsigned FullInstanceBits = endian::readNext<uint16_t, little, unaligned>(d);
-  unsigned FullFactoryBits = endian::readNext<uint16_t, little, unaligned>(d);
+      F, endian::readNext<uint32_t, llvm::endianness::little, unaligned>(d));
+  unsigned FullInstanceBits =
+      endian::readNext<uint16_t, llvm::endianness::little, unaligned>(d);
+  unsigned FullFactoryBits =
+      endian::readNext<uint16_t, llvm::endianness::little, unaligned>(d);
   Result.InstanceBits = FullInstanceBits & 0x3;
   Result.InstanceHasMoreThanOneDecl = (FullInstanceBits >> 2) & 0x1;
   Result.FactoryBits = FullFactoryBits & 0x3;
@@ -950,14 +953,16 @@ ASTSelectorLookupTrait::ReadData(Selector, const unsigned char* d,
   // Load instance methods
   for (unsigned I = 0; I != NumInstanceMethods; ++I) {
     if (ObjCMethodDecl *Method = Reader.GetLocalDeclAs<ObjCMethodDecl>(
-            F, endian::readNext<uint32_t, little, unaligned>(d)))
+            F,
+            endian::readNext<uint32_t, llvm::endianness::little, unaligned>(d)))
       Result.Instance.push_back(Method);
   }
 
   // Load factory methods
   for (unsigned I = 0; I != NumFactoryMethods; ++I) {
     if (ObjCMethodDecl *Method = Reader.GetLocalDeclAs<ObjCMethodDecl>(
-            F, endian::readNext<uint32_t, little, unaligned>(d)))
+            F,
+            endian::readNext<uint32_t, llvm::endianness::little, unaligned>(d)))
       Result.Factory.push_back(Method);
   }
 
@@ -998,7 +1003,8 @@ static bool readBit(unsigned &Bits) {
 IdentID ASTIdentifierLookupTrait::ReadIdentifierID(const unsigned char *d) {
   using namespace llvm::support;
 
-  unsigned RawID = endian::readNext<uint32_t, little, unaligned>(d);
+  unsigned RawID =
+      endian::readNext<uint32_t, llvm::endianness::little, unaligned>(d);
   return Reader.getGlobalIdentifierID(F, RawID >> 1);
 }
 
@@ -1016,7 +1022,8 @@ IdentifierInfo *ASTIdentifierLookupTrait::ReadData(const internal_key_type& k,
                                                    unsigned DataLen) {
   using namespace llvm::support;
 
-  unsigned RawID = endian::readNext<uint32_t, little, unaligned>(d);
+  unsigned RawID =
+      endian::readNext<uint32_t, llvm::endianness::little, unaligned>(d);
   bool IsInteresting = RawID & 0x01;
 
   // Wipe out the "is interesting" bit.
@@ -1039,8 +1046,10 @@ IdentifierInfo *ASTIdentifierLookupTrait::ReadData(const internal_key_type& k,
     return II;
   }
 
-  unsigned ObjCOrBuiltinID = endian::readNext<uint16_t, little, unaligned>(d);
-  unsigned Bits = endian::readNext<uint16_t, little, unaligned>(d);
+  unsigned ObjCOrBuiltinID =
+      endian::readNext<uint16_t, llvm::endianness::little, unaligned>(d);
+  unsigned Bits =
+      endian::readNext<uint16_t, llvm::endianness::little, unaligned>(d);
   bool CPlusPlusOperatorKeyword = readBit(Bits);
   bool HasRevertedTokenIDToIdentifier = readBit(Bits);
   bool Poisoned = readBit(Bits);
@@ -1069,7 +1078,7 @@ IdentifierInfo *ASTIdentifierLookupTrait::ReadData(const internal_key_type& k,
   // definition.
   if (HadMacroDefinition) {
     uint32_t MacroDirectivesOffset =
-        endian::readNext<uint32_t, little, unaligned>(d);
+        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(d);
     DataLen -= 4;
 
     Reader.addPendingMacro(II, &F, MacroDirectivesOffset);
@@ -1083,7 +1092,8 @@ IdentifierInfo *ASTIdentifierLookupTrait::ReadData(const internal_key_type& k,
     SmallVector<uint32_t, 4> DeclIDs;
     for (; DataLen > 0; DataLen -= 4)
       DeclIDs.push_back(Reader.getGlobalDeclID(
-          F, endian::readNext<uint32_t, little, unaligned>(d)));
+          F,
+          endian::readNext<uint32_t, llvm::endianness::little, unaligned>(d)));
     Reader.SetGloballyVisibleDecls(II, DeclIDs);
   }
 
@@ -1152,7 +1162,8 @@ ModuleFile *
 ASTDeclContextNameLookupTrait::ReadFileRef(const unsigned char *&d) {
   using namespace llvm::support;
 
-  uint32_t ModuleFileID = endian::readNext<uint32_t, little, unaligned>(d);
+  uint32_t ModuleFileID =
+      endian::readNext<uint32_t, llvm::endianness::little, unaligned>(d);
   return Reader.getLocalModuleFile(F, ModuleFileID);
 }
 
@@ -1172,15 +1183,18 @@ ASTDeclContextNameLookupTrait::ReadKey(const unsigned char *d, unsigned) {
   case DeclarationName::CXXLiteralOperatorName:
   case DeclarationName::CXXDeductionGuideName:
     Data = (uint64_t)Reader.getLocalIdentifier(
-        F, endian::readNext<uint32_t, little, unaligned>(d));
+        F, endian::readNext<uint32_t, llvm::endianness::little, unaligned>(d));
     break;
   case DeclarationName::ObjCZeroArgSelector:
   case DeclarationName::ObjCOneArgSelector:
   case DeclarationName::ObjCMultiArgSelector:
     Data =
-        (uint64_t)Reader.getLocalSelector(
-                             F, endian::readNext<uint32_t, little, unaligned>(
-                                    d)).getAsOpaquePtr();
+        (uint64_t)Reader
+            .getLocalSelector(
+                F,
+                endian::readNext<uint32_t, llvm::endianness::little, unaligned>(
+                    d))
+            .getAsOpaquePtr();
     break;
   case DeclarationName::CXXOperatorName:
     Data = *d++; // OverloadedOperatorKind
@@ -1203,7 +1217,8 @@ void ASTDeclContextNameLookupTrait::ReadDataInto(internal_key_type,
   using namespace llvm::support;
 
   for (unsigned NumDecls = DataLen / 4; NumDecls; --NumDecls) {
-    uint32_t LocalID = endian::readNext<uint32_t, little, unaligned>(d);
+    uint32_t LocalID =
+        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(d);
     Val.insert(Reader.getGlobalDeclID(F, LocalID));
   }
 }
@@ -2010,8 +2025,10 @@ HeaderFileInfoTrait::ReadKey(const unsigned char *d, unsigned) {
   using namespace llvm::support;
 
   internal_key_type ikey;
-  ikey.Size = off_t(endian::readNext<uint64_t, little, unaligned>(d));
-  ikey.ModTime = time_t(endian::readNext<uint64_t, little, unaligned>(d));
+  ikey.Size =
+      off_t(endian::readNext<uint64_t, llvm::endianness::little, unaligned>(d));
+  ikey.ModTime = time_t(
+      endian::readNext<uint64_t, llvm::endianness::little, unaligned>(d));
   ikey.Filename = (const char *)d;
   ikey.Imported = true;
   return ikey;
@@ -2039,9 +2056,9 @@ HeaderFileInfoTrait::ReadData(internal_key_ref key, const unsigned char *d,
   HFI.DirInfo = (Flags >> 1) & 0x07;
   HFI.IndexHeaderMapHeader = Flags & 0x01;
   HFI.ControllingMacroID = Reader.getGlobalIdentifierID(
-      M, endian::readNext<uint32_t, little, unaligned>(d));
+      M, endian::readNext<uint32_t, llvm::endianness::little, unaligned>(d));
   if (unsigned FrameworkOffset =
-          endian::readNext<uint32_t, little, unaligned>(d)) {
+          endian::readNext<uint32_t, llvm::endianness::little, unaligned>(d)) {
     // The framework offset is 1 greater than the actual offset,
     // since 0 is used as an indicator for "no framework name".
     StringRef FrameworkName(FrameworkStrings + FrameworkOffset - 1);
@@ -2051,7 +2068,8 @@ HeaderFileInfoTrait::ReadData(internal_key_ref key, const unsigned char *d,
   assert((End - d) % 4 == 0 &&
          "Wrong data length in HeaderFileInfo deserialization");
   while (d != End) {
-    uint32_t LocalSMID = endian::readNext<uint32_t, little, unaligned>(d);
+    uint32_t LocalSMID =
+        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(d);
     auto HeaderRole = static_cast<ModuleMap::ModuleHeaderRole>(LocalSMID & 7);
     LocalSMID >>= 3;
 
@@ -4030,8 +4048,9 @@ void ASTReader::ReadModuleOffsetMap(ModuleFile &F) const {
     // how it goes...
     using namespace llvm::support;
     ModuleKind Kind = static_cast<ModuleKind>(
-      endian::readNext<uint8_t, little, unaligned>(Data));
-    uint16_t Len = endian::readNext<uint16_t, little, unaligned>(Data);
+        endian::readNext<uint8_t, llvm::endianness::little, unaligned>(Data));
+    uint16_t Len =
+        endian::readNext<uint16_t, llvm::endianness::little, unaligned>(Data);
     StringRef Name = StringRef((const char*)Data, Len);
     Data += Len;
     ModuleFile *OM = (Kind == MK_PrebuiltModule || Kind == MK_ExplicitModule ||
@@ -4047,21 +4066,21 @@ void ASTReader::ReadModuleOffsetMap(ModuleFile &F) const {
     }
 
     SourceLocation::UIntTy SLocOffset =
-        endian::readNext<uint32_t, little, unaligned>(Data);
+        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Data);
     uint32_t IdentifierIDOffset =
-        endian::readNext<uint32_t, little, unaligned>(Data);
+        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Data);
     uint32_t MacroIDOffset =
-        endian::readNext<uint32_t, little, unaligned>(Data);
+        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Data);
     uint32_t PreprocessedEntityIDOffset =
-        endian::readNext<uint32_t, little, unaligned>(Data);
+        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Data);
     uint32_t SubmoduleIDOffset =
-        endian::readNext<uint32_t, little, unaligned>(Data);
+        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Data);
     uint32_t SelectorIDOffset =
-        endian::readNext<uint32_t, little, unaligned>(Data);
+        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Data);
     uint32_t DeclIDOffset =
-        endian::readNext<uint32_t, little, unaligned>(Data);
+        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Data);
     uint32_t TypeIndexOffset =
-        endian::readNext<uint32_t, little, unaligned>(Data);
+        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Data);
 
     auto mapOffset = [&](uint32_t Offset, uint32_t BaseOffset,
                          RemapBuilder &Remap) {
@@ -4156,7 +4175,7 @@ ASTReader::ReadModuleMapFileBlock(RecordData &Record, ModuleFile &F,
       return OutOfDate;
     }
 
-    llvm::SmallPtrSet<FileEntryRef, 1> AdditionalStoredMaps;
+    ModuleMap::AdditionalModMapsSet AdditionalStoredMaps;
     for (unsigned I = 0, N = Record[Idx++]; I < N; ++I) {
       // FIXME: we should use input files rather than storing names.
       std::string Filename = ReadPath(F, Record, Idx);
@@ -10096,8 +10115,7 @@ void ASTReader::pushExternalDeclIntoScope(NamedDecl *D, DeclarationName Name) {
     // Adding the decl to IdResolver may have failed because it was already in
     // (even though it was not added in scope). If it is already in, make sure
     // it gets in the scope as well.
-    if (std::find(SemaObj->IdResolver.begin(Name),
-                  SemaObj->IdResolver.end(), D) != SemaObj->IdResolver.end())
+    if (llvm::is_contained(SemaObj->IdResolver.decls(Name), D))
       SemaObj->TUScope->AddDecl(D);
   }
 }
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 27700c711d..739344b9a1 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -1873,7 +1873,7 @@ namespace {
     void EmitKey(raw_ostream& Out, key_type_ref key, unsigned KeyLen) {
       using namespace llvm::support;
 
-      endian::Writer LE(Out, little);
+      endian::Writer LE(Out, llvm::endianness::little);
       LE.write<uint64_t>(key.Size);
       KeyLen -= 8;
       LE.write<uint64_t>(key.ModTime);
@@ -1885,7 +1885,7 @@ namespace {
                   data_type_ref Data, unsigned DataLen) {
       using namespace llvm::support;
 
-      endian::Writer LE(Out, little);
+      endian::Writer LE(Out, llvm::endianness::little);
       uint64_t Start = Out.tell(); (void)Start;
 
       unsigned char Flags = (Data.AlreadyIncluded << 6)
@@ -2053,7 +2053,7 @@ void ASTWriter::WriteHeaderSearch(const HeaderSearch &HS) {
 
     llvm::raw_svector_ostream Out(TableData);
     // Make sure that no bucket is at offset 0
-    endian::write<uint32_t>(Out, 0, little);
+    endian::write<uint32_t>(Out, 0, llvm::endianness::little);
     BucketOffset = Generator.Emit(Out, GeneratorTrait);
   }
 
@@ -3313,7 +3313,7 @@ class ASTMethodPoolTrait {
   void EmitKey(raw_ostream& Out, Selector Sel, unsigned) {
     using namespace llvm::support;
 
-    endian::Writer LE(Out, little);
+    endian::Writer LE(Out, llvm::endianness::little);
     uint64_t Start = Out.tell();
     assert((Start >> 32) == 0 && "Selector key offset too large");
     Writer.SetSelectorOffset(Sel, Start);
@@ -3330,7 +3330,7 @@ class ASTMethodPoolTrait {
                 data_type_ref Methods, unsigned DataLen) {
     using namespace llvm::support;
 
-    endian::Writer LE(Out, little);
+    endian::Writer LE(Out, llvm::endianness::little);
     uint64_t Start = Out.tell(); (void)Start;
     LE.write<uint32_t>(Methods.ID);
     unsigned NumInstanceMethods = 0;
@@ -3453,7 +3453,7 @@ void ASTWriter::WriteSelectors(Sema &SemaRef) {
       ASTMethodPoolTrait Trait(*this);
       llvm::raw_svector_ostream Out(MethodPool);
       // Make sure that no bucket is at offset 0
-      endian::write<uint32_t>(Out, 0, little);
+      endian::write<uint32_t>(Out, 0, llvm::endianness::little);
       BucketOffset = Generator.Emit(Out, Trait);
     }
 
@@ -3650,7 +3650,7 @@ class ASTIdentifierTableTrait {
                 IdentID ID, unsigned) {
     using namespace llvm::support;
 
-    endian::Writer LE(Out, little);
+    endian::Writer LE(Out, llvm::endianness::little);
 
     auto MacroOffset = Writer.getMacroDirectivesOffset(II);
     if (!isInterestingIdentifier(II, MacroOffset)) {
@@ -3749,7 +3749,7 @@ void ASTWriter::WriteIdentifierTable(Preprocessor &PP,
 
       llvm::raw_svector_ostream Out(IdentifierTable);
       // Make sure that no bucket is at offset 0
-      endian::write<uint32_t>(Out, 0, little);
+      endian::write<uint32_t>(Out, 0, llvm::endianness::little);
       BucketOffset = Generator.Emit(Out, Trait);
     }
 
@@ -3844,7 +3844,8 @@ class ASTDeclContextNameLookupTrait {
 
     using namespace llvm::support;
 
-    endian::write<uint32_t>(Out, Writer.getChain()->getModuleFileID(F), little);
+    endian::write<uint32_t>(Out, Writer.getChain()->getModuleFileID(F),
+                            llvm::endianness::little);
   }
 
   std::pair<unsigned, unsigned> EmitKeyDataLength(raw_ostream &Out,
@@ -3879,7 +3880,7 @@ class ASTDeclContextNameLookupTrait {
   void EmitKey(raw_ostream &Out, DeclarationNameKey Name, unsigned) {
     using namespace llvm::support;
 
-    endian::Writer LE(Out, little);
+    endian::Writer LE(Out, llvm::endianness::little);
     LE.write<uint8_t>(Name.getKind());
     switch (Name.getKind()) {
     case DeclarationName::Identifier:
@@ -3911,7 +3912,7 @@ class ASTDeclContextNameLookupTrait {
                 unsigned DataLen) {
     using namespace llvm::support;
 
-    endian::Writer LE(Out, little);
+    endian::Writer LE(Out, llvm::endianness::little);
     uint64_t Start = Out.tell(); (void)Start;
     for (unsigned I = Lookup.first, N = Lookup.second; I != N; ++I)
       LE.write<uint32_t>(DeclIDs[I]);
@@ -5024,7 +5025,7 @@ ASTFileSignature ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot,
       for (ModuleFile &M : Chain->ModuleMgr) {
         using namespace llvm::support;
 
-        endian::Writer LE(Out, little);
+        endian::Writer LE(Out, llvm::endianness::little);
         LE.write<uint8_t>(static_cast<uint8_t>(M.Kind));
         StringRef Name = M.isModule() ? M.ModuleName : M.FileName;
         LE.write<uint16_t>(Name.size());
diff --git a/clang/lib/Serialization/GlobalModuleIndex.cpp b/clang/lib/Serialization/GlobalModuleIndex.cpp
index b4a49972ac..fb80a1998d 100644
--- a/clang/lib/Serialization/GlobalModuleIndex.cpp
+++ b/clang/lib/Serialization/GlobalModuleIndex.cpp
@@ -89,8 +89,10 @@ class IdentifierIndexReaderTrait {
   static std::pair<unsigned, unsigned>
   ReadKeyDataLength(const unsigned char*& d) {
     using namespace llvm::support;
-    unsigned KeyLen = endian::readNext<uint16_t, little, unaligned>(d);
-    unsigned DataLen = endian::readNext<uint16_t, little, unaligned>(d);
+    unsigned KeyLen =
+        endian::readNext<uint16_t, llvm::endianness::little, unaligned>(d);
+    unsigned DataLen =
+        endian::readNext<uint16_t, llvm::endianness::little, unaligned>(d);
     return std::make_pair(KeyLen, DataLen);
   }
 
@@ -111,7 +113,8 @@ class IdentifierIndexReaderTrait {
 
     data_type Result;
     while (DataLen > 0) {
-      unsigned ID = endian::readNext<uint32_t, little, unaligned>(d);
+      unsigned ID =
+          endian::readNext<uint32_t, llvm::endianness::little, unaligned>(d);
       Result.push_back(ID);
       DataLen -= 4;
     }
@@ -511,7 +514,8 @@ namespace {
       // The first bit indicates whether this identifier is interesting.
       // That's all we care about.
       using namespace llvm::support;
-      unsigned RawID = endian::readNext<uint32_t, little, unaligned>(d);
+      unsigned RawID =
+          endian::readNext<uint32_t, llvm::endianness::little, unaligned>(d);
       bool IsInteresting = RawID & 0x01;
       return std::make_pair(k, IsInteresting);
     }
@@ -729,7 +733,7 @@ class IdentifierIndexWriterTrait {
   std::pair<unsigned,unsigned>
   EmitKeyDataLength(raw_ostream& Out, key_type_ref Key, data_type_ref Data) {
     using namespace llvm::support;
-    endian::Writer LE(Out, little);
+    endian::Writer LE(Out, llvm::endianness::little);
     unsigned KeyLen = Key.size();
     unsigned DataLen = Data.size() * 4;
     LE.write<uint16_t>(KeyLen);
@@ -745,7 +749,7 @@ class IdentifierIndexWriterTrait {
                 unsigned DataLen) {
     using namespace llvm::support;
     for (unsigned I = 0, N = Data.size(); I != N; ++I)
-      endian::write<uint32_t>(Out, Data[I], little);
+      endian::write<uint32_t>(Out, Data[I], llvm::endianness::little);
   }
 };
 
@@ -824,7 +828,7 @@ bool GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) {
       using namespace llvm::support;
       llvm::raw_svector_ostream Out(IdentifierTable);
       // Make sure that no bucket is at offset 0
-      endian::write<uint32_t>(Out, 0, little);
+      endian::write<uint32_t>(Out, 0, llvm::endianness::little);
       BucketOffset = Generator.Emit(Out, Trait);
     }
 
diff --git a/clang/lib/Serialization/MultiOnDiskHashTable.h b/clang/lib/Serialization/MultiOnDiskHashTable.h
index adc97d57e0..2402a628b5 100644
--- a/clang/lib/Serialization/MultiOnDiskHashTable.h
+++ b/clang/lib/Serialization/MultiOnDiskHashTable.h
@@ -199,10 +199,12 @@ template<typename Info> class MultiOnDiskHashTable {
 
     storage_type Ptr = Data;
 
-    uint32_t BucketOffset = endian::readNext<uint32_t, little, unaligned>(Ptr);
+    uint32_t BucketOffset =
+        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Ptr);
 
     // Read the list of overridden files.
-    uint32_t NumFiles = endian::readNext<uint32_t, little, unaligned>(Ptr);
+    uint32_t NumFiles =
+        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Ptr);
     // FIXME: Add a reserve() to TinyPtrVector so that we don't need to make
     // an additional copy.
     llvm::SmallVector<file_type, 16> OverriddenFiles;
@@ -311,7 +313,7 @@ class MultiOnDiskHashTableGenerator {
 
     // Write our header information.
     {
-      endian::Writer Writer(OutStream, little);
+      endian::Writer Writer(OutStream, llvm::endianness::little);
 
       // Reserve four bytes for the bucket offset.
       Writer.write<uint32_t>(0);
diff --git a/clang/lib/StaticAnalyzer/Checkers/SmartPtrModeling.cpp b/clang/lib/StaticAnalyzer/Checkers/SmartPtrModeling.cpp
index 66e9a501c3..268fc742f0 100644
--- a/clang/lib/StaticAnalyzer/Checkers/SmartPtrModeling.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/SmartPtrModeling.cpp
@@ -32,7 +32,6 @@
 #include "clang/StaticAnalyzer/Core/PathSensitive/SymExpr.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/SymbolManager.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/StringMap.h"
 #include "llvm/Support/ErrorHandling.h"
 #include <optional>
 #include <string>
diff --git a/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp
index bad86682c9..ef209f64f0 100644
--- a/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp
@@ -407,23 +407,14 @@ class StreamChecker : public Checker<check::PreCall, eval::Call,
 
   /// Generate a message for BugReporterVisitor if the stored symbol is
   /// marked as interesting by the actual bug report.
-  // FIXME: Use lambda instead.
-  struct NoteFn {
-    const BugType *BT_ResourceLeak;
-    SymbolRef StreamSym;
-    std::string Message;
-
-    std::string operator()(PathSensitiveBugReport &BR) const {
-      if (BR.isInteresting(StreamSym) && &BR.getBugType() == BT_ResourceLeak)
-        return Message;
-
-      return "";
-    }
-  };
-
   const NoteTag *constructNoteTag(CheckerContext &C, SymbolRef StreamSym,
                                   const std::string &Message) const {
-    return C.getNoteTag(NoteFn{&BT_ResourceLeak, StreamSym, Message});
+    return C.getNoteTag([this, StreamSym,
+                         Message](PathSensitiveBugReport &BR) -> std::string {
+      if (BR.isInteresting(StreamSym) && &BR.getBugType() == &BT_ResourceLeak)
+        return Message;
+      return "";
+    });
   }
 
   const NoteTag *constructSetEofNoteTag(CheckerContext &C,
diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
index 9b1d7ae3e6..c1f180f313 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
@@ -186,8 +186,7 @@ bool isPtrConversion(const FunctionDecl *F) {
   // FIXME: check # of params == 1
   const auto FunctionName = safeGetName(F);
   if (FunctionName == "getPtr" || FunctionName == "WeakPtr" ||
-      FunctionName == "makeWeakPtr"
-
+      FunctionName == "dynamicDowncast"
       || FunctionName == "downcast" || FunctionName == "bitwise_cast")
     return true;
 
diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedCallArgsChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedCallArgsChecker.cpp
index 4ae8c442fa..407b6ba7a7 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedCallArgsChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedCallArgsChecker.cpp
@@ -149,7 +149,7 @@ class UncountedCallArgsChecker
 
     auto name = safeGetName(Callee);
     if (name == "adoptRef" || name == "getPtr" || name == "WeakPtr" ||
-        name == "makeWeakPtr" || name == "downcast" || name == "bitwise_cast" ||
+        name == "dynamicDowncast" || name == "downcast" || name == "bitwise_cast" ||
         name == "is" || name == "equal" || name == "hash" ||
         name == "isType"
         // FIXME: Most/all of these should be implemented via attributes.
diff --git a/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp b/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
index 42d03f6751..2d184d5295 100644
--- a/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
+++ b/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
@@ -132,6 +132,16 @@ const Expr *bugreporter::getDerefExpr(const Stmt *S) {
     }
     // Pattern match for a few useful cases: a[0], p->f, *p etc.
     else if (const auto *ME = dyn_cast<MemberExpr>(E)) {
+      // This handles the case when the dereferencing of a member reference
+      // happens. This is needed, because the AST for dereferencing a
+      // member reference looks like the following:
+      // |-MemberExpr
+      //  `-DeclRefExpr
+      // Without this special case the notes would refer to the whole object
+      // (struct, class or union variable) instead of just the relevant member.
+
+      if (ME->getMemberDecl()->getType()->isReferenceType())
+        break;
       E = ME->getBase();
     } else if (const auto *IvarRef = dyn_cast<ObjCIvarRefExpr>(E)) {
       E = IvarRef->getBase();
@@ -157,26 +167,42 @@ const Expr *bugreporter::getDerefExpr(const Stmt *S) {
   return E;
 }
 
+static const VarDecl *getVarDeclForExpression(const Expr *E) {
+  if (const auto *DR = dyn_cast<DeclRefExpr>(E))
+    return dyn_cast<VarDecl>(DR->getDecl());
+  return nullptr;
+}
+
 static const MemRegion *
 getLocationRegionIfReference(const Expr *E, const ExplodedNode *N,
                              bool LookingForReference = true) {
-  if (const auto *DR = dyn_cast<DeclRefExpr>(E)) {
-    if (const auto *VD = dyn_cast<VarDecl>(DR->getDecl())) {
-      if (LookingForReference && !VD->getType()->isReferenceType())
-        return nullptr;
-      return N->getState()
-          ->getLValue(VD, N->getLocationContext())
-          .getAsRegion();
+  if (const auto *ME = dyn_cast<MemberExpr>(E)) {
+    // This handles null references from FieldRegions, for example:
+    //   struct Wrapper { int &ref; };
+    //   Wrapper w = { *(int *)0 };
+    //   w.ref = 1;
+    const Expr *Base = ME->getBase();
+    const VarDecl *VD = getVarDeclForExpression(Base);
+    if (!VD)
+      return nullptr;
+
+    const auto *FD = dyn_cast<FieldDecl>(ME->getMemberDecl());
+    if (!FD)
+      return nullptr;
+
+    if (FD->getType()->isReferenceType()) {
+      SVal StructSVal = N->getState()->getLValue(VD, N->getLocationContext());
+      return N->getState()->getLValue(FD, StructSVal).getAsRegion();
     }
+    return nullptr;
   }
 
-  // FIXME: This does not handle other kinds of null references,
-  // for example, references from FieldRegions:
-  //   struct Wrapper { int &ref; };
-  //   Wrapper w = { *(int *)0 };
-  //   w.ref = 1;
-
-  return nullptr;
+  const VarDecl *VD = getVarDeclForExpression(E);
+  if (!VD)
+    return nullptr;
+  if (LookingForReference && !VD->getType()->isReferenceType())
+    return nullptr;
+  return N->getState()->getLValue(VD, N->getLocationContext()).getAsRegion();
 }
 
 /// Comparing internal representations of symbolic values (via
diff --git a/clang/lib/Support/RISCVVIntrinsicUtils.cpp b/clang/lib/Support/RISCVVIntrinsicUtils.cpp
index c105db434d..597ee194fc 100644
--- a/clang/lib/Support/RISCVVIntrinsicUtils.cpp
+++ b/clang/lib/Support/RISCVVIntrinsicUtils.cpp
@@ -10,7 +10,6 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/StringExtras.h"
-#include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/ErrorHandling.h"
diff --git a/clang/lib/Tooling/Inclusions/Stdlib/CMakeLists.txt b/clang/lib/Tooling/Inclusions/Stdlib/CMakeLists.txt
index 0f52c3590a..ed323ab352 100644
--- a/clang/lib/Tooling/Inclusions/Stdlib/CMakeLists.txt
+++ b/clang/lib/Tooling/Inclusions/Stdlib/CMakeLists.txt
@@ -1,3 +1,23 @@
+# Workaround for MSVC ARM64 performance regression:
+# https://developercommunity.visualstudio.com/t/Compiling-a-specific-code-for-ARM64-with/10444970
+# Since /O1 and /O2 represent a set of optimizations,
+# our goal is to disable the /Og flag while retaining the other optimizations from the /O1|/O2 set
+if(MSVC AND NOT CMAKE_CXX_COMPILER_ID MATCHES Clang
+    AND MSVC_VERSION VERSION_GREATER_EQUAL 1932
+    AND CMAKE_SYSTEM_PROCESSOR MATCHES "ARM64")
+
+  string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE)
+  string(REGEX MATCHALL "/[Oo][12]" opt_flags "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${uppercase_CMAKE_BUILD_TYPE}}")
+  if (opt_flags)
+    if(opt_flags MATCHES "1$")
+      set(opt_flags "/Od;/Os;/Oy;/Ob2;/GF;/Gy")
+    elseif (opt_flags MATCHES "2$")
+      set(opt_flags "/Od;/Oi;/Ot;/Oy;/Ob2;/GF;/Gy")
+    endif()
+    set_source_files_properties(StandardLibrary.cpp PROPERTIES COMPILE_OPTIONS "${opt_flags}")
+  endif()
+endif()
+
 add_clang_library(clangToolingInclusionsStdlib
   StandardLibrary.cpp
 
diff --git a/clang/test/AST/Interp/arrays.cpp b/clang/test/AST/Interp/arrays.cpp
index 281835f828..18c4ae4354 100644
--- a/clang/test/AST/Interp/arrays.cpp
+++ b/clang/test/AST/Interp/arrays.cpp
@@ -333,6 +333,26 @@ namespace IncDec {
                                    // expected-note {{in call to}} \
                                    // ref-error {{not an integral constant expression}} \
                                   // ref-note {{in call to}}
+
+  constexpr int nullptr1(bool Pre) {
+    int *a = nullptr;
+    if (Pre)
+      ++a; // ref-note {{arithmetic on null pointer}} \
+           // expected-note {{arithmetic on null pointer}}
+    else
+      a++; // ref-note {{arithmetic on null pointer}} \
+           // expected-note {{arithmetic on null pointer}}
+    return 1;
+  }
+  static_assert(nullptr1(true) == 1, ""); // ref-error {{not an integral constant expression}} \
+                                          // ref-note {{in call to}} \
+                                          // expected-error {{not an integral constant expression}} \
+                                          // expected-note {{in call to}}
+
+  static_assert(nullptr1(false) == 1, ""); // ref-error {{not an integral constant expression}} \
+                                           // ref-note {{in call to}} \
+                                           // expected-error {{not an integral constant expression}} \
+                                           // expected-note {{in call to}}
 };
 
 namespace ZeroInit {
diff --git a/clang/test/AST/Interp/bitfields.cpp b/clang/test/AST/Interp/bitfields.cpp
index e078704fce..d3a8a08306 100644
--- a/clang/test/AST/Interp/bitfields.cpp
+++ b/clang/test/AST/Interp/bitfields.cpp
@@ -1,5 +1,7 @@
 // RUN: %clang_cc1 -fexperimental-new-constant-interpreter -Wno-bitfield-constant-conversion -verify %s
 // RUN: %clang_cc1 -verify=ref -Wno-bitfield-constant-conversion %s
+// RUN: %clang_cc1 -std=c++20 -fexperimental-new-constant-interpreter -Wno-bitfield-constant-conversion -verify %s
+// RUN: %clang_cc1 -std=c++20 -verify=ref -Wno-bitfield-constant-conversion %s
 
 // expected-no-diagnostics
 // ref-no-diagnostics
@@ -32,7 +34,26 @@ namespace Basic {
   }
   static_assert(storeA2() == 2, "");
 
-  // TODO: +=, -=, etc. operators.
+#if __cplusplus >= 202002
+  struct Init1 {
+    unsigned a : 2 = 1;
+  };
+  constexpr Init1 I1{};
+  static_assert(I1.a == 1, "");
+
+  struct Init2 {
+    unsigned a : 2 = 100;
+  };
+  constexpr Init2 I2{};
+  static_assert(I2.a == 0, "");
+#endif
+
+  struct Init3 {
+    unsigned a : 2;
+    constexpr Init3() : a(100) {}
+  };
+  constexpr Init3 I3{};
+  static_assert(I3.a == 0, "");
 }
 
 namespace Overflow {
@@ -45,3 +66,39 @@ namespace Overflow {
 
   static_assert(f() == 3, "");
 }
+
+namespace Compound {
+  struct A {
+    unsigned int a : 2;
+    constexpr A() : a(0) {}
+    constexpr A(int a) : a(a) {}
+  };
+
+  constexpr unsigned add() {
+    A a;
+    a.a += 10;
+    return a.a;
+  }
+  static_assert(add() == 2, "");
+
+  constexpr unsigned sub() {
+    A a;
+    a.a -= 10;
+    return a.a;
+  }
+  static_assert(sub() == 2, "");
+
+  constexpr unsigned mul() {
+    A a(1);
+    a.a *= 5;
+    return a.a;
+  }
+  static_assert(mul() == 1, "");
+
+  constexpr unsigned div() {
+    A a(2);
+    a.a /= 2;
+    return a.a;
+  }
+  static_assert(div() == 1, "");
+}
diff --git a/clang/test/AST/Interp/cxx23.cpp b/clang/test/AST/Interp/cxx23.cpp
new file mode 100644
index 0000000000..e284a66626
--- /dev/null
+++ b/clang/test/AST/Interp/cxx23.cpp
@@ -0,0 +1,84 @@
+// RUN: %clang_cc1 -std=c++20 -fsyntax-only -fcxx-exceptions -verify=ref20 %s
+// RUN: %clang_cc1 -std=c++23 -fsyntax-only -fcxx-exceptions -verify=ref23 %s
+// RUN: %clang_cc1 -std=c++20 -fsyntax-only -fcxx-exceptions -verify=expected20 %s -fexperimental-new-constant-interpreter
+// RUN: %clang_cc1 -std=c++23 -fsyntax-only -fcxx-exceptions -verify=expected23 %s -fexperimental-new-constant-interpreter
+
+
+// expected23-no-diagnostics
+
+
+/// FIXME: The new interpreter is missing all the 'control flows through...' diagnostics.
+
+constexpr int f(int n) {  // ref20-error {{constexpr function never produces a constant expression}} \
+                          // ref23-error {{constexpr function never produces a constant expression}}
+  static const int m = n; // ref20-note {{control flows through the definition of a static variable}} \
+                          // ref20-warning {{is a C++23 extension}} \
+                          // ref23-note {{control flows through the definition of a static variable}} \
+                          // expected20-warning {{is a C++23 extension}}
+
+  return m;
+}
+constexpr int g(int n) {        // ref20-error {{constexpr function never produces a constant expression}} \
+                                // ref23-error {{constexpr function never produces a constant expression}}
+  thread_local const int m = n; // ref20-note {{control flows through the definition of a thread_local variable}} \
+                                // ref20-warning {{is a C++23 extension}} \
+                                // ref23-note {{control flows through the definition of a thread_local variable}} \
+                                // expected20-warning {{is a C++23 extension}}
+  return m;
+}
+
+constexpr int c_thread_local(int n) { // ref20-error {{constexpr function never produces a constant expression}} \
+                                      // ref23-error {{constexpr function never produces a constant expression}}
+  static _Thread_local int m = 0;     // ref20-note {{control flows through the definition of a thread_local variable}} \
+                                      // ref20-warning {{is a C++23 extension}} \
+                                      // ref23-note {{control flows through the definition of a thread_local variable}} \
+                                      // expected20-warning {{is a C++23 extension}}
+  return m;
+}
+
+
+constexpr int gnu_thread_local(int n) { // ref20-error {{constexpr function never produces a constant expression}} \
+                                        // ref23-error {{constexpr function never produces a constant expression}}
+  static __thread int m = 0;            // ref20-note {{control flows through the definition of a thread_local variable}} \
+                                        // ref20-warning {{is a C++23 extension}} \
+                                        // ref23-note {{control flows through the definition of a thread_local variable}} \
+                                        // expected20-warning {{is a C++23 extension}}
+  return m;
+}
+
+constexpr int h(int n) {  // ref20-error {{constexpr function never produces a constant expression}} \
+                          // ref23-error {{constexpr function never produces a constant expression}}
+  static const int m = n; // ref20-note {{control flows through the definition of a static variable}} \
+                          // ref20-warning {{is a C++23 extension}} \
+                          // ref23-note {{control flows through the definition of a static variable}} \
+                          // expected20-warning {{is a C++23 extension}}
+  return &m - &m;
+}
+
+constexpr int i(int n) {        // ref20-error {{constexpr function never produces a constant expression}} \
+                                // ref23-error {{constexpr function never produces a constant expression}}
+  thread_local const int m = n; // ref20-note {{control flows through the definition of a thread_local variable}} \
+                                // ref20-warning {{is a C++23 extension}} \
+                                // ref23-note {{control flows through the definition of a thread_local variable}} \
+                                // expected20-warning {{is a C++23 extension}}
+  return &m - &m;
+}
+
+constexpr int j(int n) {
+  if (!n)
+    return 0;
+  static const int m = n; // ref20-warning {{is a C++23 extension}} \
+                          // expected20-warning {{is a C++23 extension}}
+  return m;
+}
+constexpr int j0 = j(0);
+
+constexpr int k(int n) {
+  if (!n)
+    return 0;
+  thread_local const int m = n; // ref20-warning {{is a C++23 extension}} \
+                                // expected20-warning {{is a C++23 extension}}
+
+  return m;
+}
+constexpr int k0 = k(0);
diff --git a/clang/test/AST/Interp/intap.cpp b/clang/test/AST/Interp/intap.cpp
index b3f02d2b76..27fae1b904 100644
--- a/clang/test/AST/Interp/intap.cpp
+++ b/clang/test/AST/Interp/intap.cpp
@@ -3,6 +3,31 @@
 // RUN: %clang_cc1 -std=c++11 -fms-extensions -verify=ref %s
 // RUN: %clang_cc1 -std=c++20 -fms-extensions -verify=ref %s
 
+
+using MaxBitInt = _BitInt(128);
+
+constexpr _BitInt(2) A = 0;
+constexpr _BitInt(2) B = A + 1;
+constexpr _BitInt(2) C = B + 1; // expected-warning {{from 2 to -2}} \
+                                // ref-warning {{from 2 to -2}}
+static_assert(C == -2, "");
+
+
+constexpr MaxBitInt A_ = 0;
+constexpr MaxBitInt B_ = A_ + 1;
+static_assert(B_ == 1, "");
+
+constexpr MaxBitInt BitIntZero{};
+static_assert(BitIntZero == 0, "");
+constexpr unsigned _BitInt(128) UBitIntZero{};
+static_assert(UBitIntZero == 0, "");
+
+constexpr _BitInt(2) BitIntZero2{};
+static_assert(BitIntZero2 == 0, "");
+constexpr unsigned _BitInt(1) UBitIntZero1{};
+static_assert(UBitIntZero1 == 0, "");
+
+
 #ifdef __SIZEOF_INT128__
 namespace i128 {
   typedef __int128 int128_t;
@@ -27,10 +52,18 @@ namespace i128 {
                                          // ref-note {{outside the range}}
   constexpr int128_t Two = (int128_t)1 << 1ul;
   static_assert(Two == 2, "");
+  static_assert(Two, "");
+  constexpr bool CastedToBool = Two;
+  static_assert(CastedToBool, "");
 
   constexpr uint128_t AllOnes = ~static_cast<uint128_t>(0);
   static_assert(AllOnes == UINT128_MAX, "");
 
+  constexpr uint128_t i128Zero{};
+  static_assert(i128Zero == 0, "");
+  constexpr uint128_t ui128Zero{};
+  static_assert(ui128Zero == 0, "");
+
 #if __cplusplus >= 201402L
   template <typename T>
   constexpr T CastFrom(__int128_t A) {
@@ -76,9 +109,13 @@ namespace i128 {
                                            // expected-note {{is outside the range of representable values of type}}
 }
 
-#else
-/// No int128 support, so no expected directives.
+namespace AddSubOffset {
+  constexpr __int128 A = 1;
+  constexpr int arr[] = {1,2,3};
+  constexpr const int *P = arr + A;
+  static_assert(*P == 2, "");
+  constexpr const int *P2 = P - A;
+  static_assert(*P2 == 1,"");
+}
 
-// expected-no-diagnostics
-// ref-no-diagnostics
 #endif
diff --git a/clang/test/AST/Interp/literals.cpp b/clang/test/AST/Interp/literals.cpp
index a83dcd2341..6e89275183 100644
--- a/clang/test/AST/Interp/literals.cpp
+++ b/clang/test/AST/Interp/literals.cpp
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -fexperimental-new-constant-interpreter -fms-extensions -std=c++11 -verify %s
-// RUN: %clang_cc1 -fexperimental-new-constant-interpreter -fms-extensions -std=c++20 -verify %s
-// RUN: %clang_cc1 -std=c++11 -fms-extensions -verify=ref %s
-// RUN: %clang_cc1 -std=c++20 -fms-extensions -verify=ref %s
+// RUN: %clang_cc1 -fexperimental-new-constant-interpreter -Wno-vla -fms-extensions -std=c++11 -verify %s
+// RUN: %clang_cc1 -fexperimental-new-constant-interpreter -Wno-vla -fms-extensions -std=c++20 -verify %s
+// RUN: %clang_cc1 -std=c++11 -fms-extensions -Wno-vla -verify=ref %s
+// RUN: %clang_cc1 -std=c++20 -fms-extensions -Wno-vla -verify=ref %s
 
 #define INT_MIN (~__INT_MAX__)
 #define INT_MAX __INT_MAX__
diff --git a/clang/test/AST/Interp/records.cpp b/clang/test/AST/Interp/records.cpp
index bcc84087fc..a2e878f613 100644
--- a/clang/test/AST/Interp/records.cpp
+++ b/clang/test/AST/Interp/records.cpp
@@ -1064,5 +1064,44 @@ namespace ParenInit {
   };
 
   constexpr B b(A(1),2);
+
+
+  struct O {
+    int &&j;
+  };
+
+  /// Not constexpr!
+  O o1(0);
+  constinit O o2(0); // ref-error {{variable does not have a constant initializer}} \
+                     // ref-note {{required by 'constinit' specifier}} \
+                     // ref-note {{reference to temporary is not a constant expression}} \
+                     // ref-note {{temporary created here}} \
+                     // expected-error {{variable does not have a constant initializer}} \
+                     // expected-note {{required by 'constinit' specifier}} \
+                     // expected-note {{reference to temporary is not a constant expression}} \
+                     // expected-note {{temporary created here}}
 }
 #endif
+
+namespace DelegatingConstructors {
+  struct S {
+    int a;
+    constexpr S() : S(10) {}
+    constexpr S(int a) : a(a) {}
+  };
+  constexpr S s = {};
+  static_assert(s.a == 10, "");
+
+  struct B {
+    int a;
+    int b;
+
+    constexpr B(int a) : a(a), b(a + 2) {}
+  };
+  struct A : B {
+    constexpr A() : B(10) {};
+  };
+  constexpr A d4 = {};
+  static_assert(d4.a == 10, "");
+  static_assert(d4.b == 12, "");
+}
diff --git a/clang/test/Analysis/Checkers/WebKit/call-args-dynamic-downcast.cpp b/clang/test/Analysis/Checkers/WebKit/call-args-dynamic-downcast.cpp
new file mode 100644
index 0000000000..28156623d9
--- /dev/null
+++ b/clang/test/Analysis/Checkers/WebKit/call-args-dynamic-downcast.cpp
@@ -0,0 +1,35 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=alpha.webkit.UncountedCallArgsChecker -verify %s
+// expected-no-diagnostics
+
+class Base {
+public:
+    inline void ref();
+    inline void deref();
+};
+
+class Derived : public Base {
+public:
+  virtual ~Derived();
+
+  void ref() const;
+  void deref() const;
+};
+
+class SubDerived final : public Derived {
+};
+
+class OtherObject {
+public:
+    Derived* obj();
+};
+
+template<typename Target, typename Source>
+inline Target* dynamicDowncast(Source* source)
+{
+    return static_cast<Target*>(source);
+}
+
+void foo(OtherObject* other)
+{
+    dynamicDowncast<SubDerived>(other->obj());
+}
diff --git a/clang/test/Analysis/Checkers/WebKit/call-args.cpp b/clang/test/Analysis/Checkers/WebKit/call-args.cpp
index a56c4222ad..716219836e 100644
--- a/clang/test/Analysis/Checkers/WebKit/call-args.cpp
+++ b/clang/test/Analysis/Checkers/WebKit/call-args.cpp
@@ -262,22 +262,6 @@ namespace param_forwarding_method {
   }
 }
 
-namespace make_ref {
-  void makeRef(RefCountable*) {}
-  void makeRefPtr(RefCountable*) {}
-  void makeWeakPtr(RefCountable*) {}
-  void makeWeakPtr(RefCountable&) {}
-
-  void foo() {
-    makeRef(provide());
-    makeRefPtr(provide());
-    RefPtr<RefCountable> a(provide());
-    Ref<RefCountable> b(provide());
-    makeWeakPtr(provide());
-    makeWeakPtr(*provide());
-  }
-}
-
 namespace downcast {
   void consume_ref_countable(RefCountable*) {}
   RefCountable* downcast(RefCountable*) { return nullptr; }
diff --git a/clang/test/Analysis/PR49642.c b/clang/test/Analysis/PR49642.c
index c21050fd4a..78bbde79d8 100644
--- a/clang/test/Analysis/PR49642.c
+++ b/clang/test/Analysis/PR49642.c
@@ -1,6 +1,6 @@
 // RUN: %clang_analyze_cc1 -Wno-implicit-function-declaration -Wno-implicit-int -w -verify %s \
 // RUN:   -analyzer-checker=core \
-// RUN:   -analyzer-checker=alpha.unix.StdCLibraryFunctions
+// RUN:   -analyzer-checker=unix.StdCLibraryFunctions
 
 // expected-no-diagnostics
 
diff --git a/clang/test/Analysis/analyzer-config.c b/clang/test/Analysis/analyzer-config.c
index d86ca5d192..794ef8b9cc 100644
--- a/clang/test/Analysis/analyzer-config.c
+++ b/clang/test/Analysis/analyzer-config.c
@@ -13,8 +13,6 @@
 // CHECK-NEXT: alpha.security.MmapWriteExec:MmapProtRead = 0x01
 // CHECK-NEXT: alpha.security.taint.TaintPropagation:Config = ""
 // CHECK-NEXT: alpha.unix.Errno:AllowErrnoReadOutsideConditionExpressions = true
-// CHECK-NEXT: alpha.unix.StdCLibraryFunctions:DisplayLoadedSummaries = false
-// CHECK-NEXT: alpha.unix.StdCLibraryFunctions:ModelPOSIX = false
 // CHECK-NEXT: apply-fixits = false
 // CHECK-NEXT: assume-controlled-environment = false
 // CHECK-NEXT: avoid-suppressing-null-argument-paths = false
@@ -129,6 +127,8 @@
 // CHECK-NEXT: track-conditions-debug = false
 // CHECK-NEXT: unix.DynamicMemoryModeling:AddNoOwnershipChangeNotes = true
 // CHECK-NEXT: unix.DynamicMemoryModeling:Optimistic = false
+// CHECK-NEXT: unix.StdCLibraryFunctions:DisplayLoadedSummaries = false
+// CHECK-NEXT: unix.StdCLibraryFunctions:ModelPOSIX = false
 // CHECK-NEXT: unroll-loops = false
 // CHECK-NEXT: verbose-report-filename = false
 // CHECK-NEXT: widen-loops = false
diff --git a/clang/test/Analysis/analyzer-enabled-checkers.c b/clang/test/Analysis/analyzer-enabled-checkers.c
index ed8334b9e2..cf69a6b04c 100644
--- a/clang/test/Analysis/analyzer-enabled-checkers.c
+++ b/clang/test/Analysis/analyzer-enabled-checkers.c
@@ -47,6 +47,7 @@
 // CHECK-NEXT: unix.Malloc
 // CHECK-NEXT: unix.MallocSizeof
 // CHECK-NEXT: unix.MismatchedDeallocator
+// CHECK-NEXT: unix.StdCLibraryFunctions
 // CHECK-NEXT: unix.Vfork
 // CHECK-NEXT: unix.cstring.BadSizeArg
 // CHECK-NEXT: unix.cstring.NullArg
diff --git a/clang/test/Analysis/conversion.c b/clang/test/Analysis/conversion.c
index 0d2e005550..cafe9c37c2 100644
--- a/clang/test/Analysis/conversion.c
+++ b/clang/test/Analysis/conversion.c
@@ -1,6 +1,6 @@
 // RUN: %clang_analyze_cc1 %s \
 // RUN:   -Wno-conversion -Wno-tautological-constant-compare \
-// RUN:   -analyzer-checker=core,apiModeling,alpha.unix.StdCLibraryFunctions,alpha.core.Conversion \
+// RUN:   -analyzer-checker=core,apiModeling,unix.StdCLibraryFunctions,alpha.core.Conversion \
 // RUN:   -verify
 
 unsigned char U8;
@@ -187,7 +187,7 @@ char dontwarn10(long long x) {
 }
 
 
-// C library functions, handled via alpha.unix.StdCLibraryFunctions
+// C library functions, handled via unix.StdCLibraryFunctions
 
 int isascii(int c);
 void libraryFunction1(void) {
diff --git a/clang/test/Analysis/diagnostics/deref-track-symbolic-region.cpp b/clang/test/Analysis/diagnostics/deref-track-symbolic-region.cpp
index e258a60aa9..e9f62c2407 100644
--- a/clang/test/Analysis/diagnostics/deref-track-symbolic-region.cpp
+++ b/clang/test/Analysis/diagnostics/deref-track-symbolic-region.cpp
@@ -41,3 +41,34 @@ int testRefToNullPtr2() {
   return *p2;         //expected-warning {{Dereference of null pointer}}
                       // expected-note@-1{{Dereference of null pointer}}
 }
+
+void testMemberNullPointerDeref() {
+  struct Wrapper {char c; int *ptr; };  
+  Wrapper w = {'a', nullptr};           // expected-note {{'w.ptr' initialized to a null pointer value}}
+  *w.ptr = 1;                           //expected-warning {{Dereference of null pointer}}
+                                        // expected-note@-1{{Dereference of null pointer}}
+}
+
+void testMemberNullReferenceDeref() {
+  struct Wrapper {char c; int &ref; };
+  Wrapper w = {.c = 'a', .ref = *(int *)0 }; // expected-note {{'w.ref' initialized to a null pointer value}}
+                                             // expected-warning@-1 {{binding dereferenced null pointer to reference has undefined behavior}}
+  w.ref = 1;                                 //expected-warning {{Dereference of null pointer}}
+                                             // expected-note@-1{{Dereference of null pointer}}
+}
+
+void testReferenceToPointerWithNullptr() {
+  int *i = nullptr;                   // expected-note {{'i' initialized to a null pointer value}}
+  struct Wrapper {char c; int *&a;};
+  Wrapper w {'c', i};                 // expected-note{{'w.a' initialized here}}
+  *(w.a) = 25;                        // expected-warning {{Dereference of null pointer}}
+                                      // expected-note@-1 {{Dereference of null pointer}}
+}
+
+void testNullReferenceToPointer() {
+  struct Wrapper {char c; int *&a;};
+  Wrapper w {'c', *(int **)0 };           // expected-note{{'w.a' initialized to a null pointer value}}
+                                          // expected-warning@-1 {{binding dereferenced null pointer to reference has undefined behavior}}
+  w.a = nullptr;                          // expected-warning {{Dereference of null pointer}}
+                                          // expected-note@-1 {{Dereference of null pointer}}
+}
\ No newline at end of file
diff --git a/clang/test/Analysis/errno-stdlibraryfunctions-notes.c b/clang/test/Analysis/errno-stdlibraryfunctions-notes.c
index 991384cc37..c3fac58c46 100644
--- a/clang/test/Analysis/errno-stdlibraryfunctions-notes.c
+++ b/clang/test/Analysis/errno-stdlibraryfunctions-notes.c
@@ -1,10 +1,10 @@
 // RUN: %clang_analyze_cc1 -verify -analyzer-output text %s \
 // RUN:   -analyzer-checker=core \
 // RUN:   -analyzer-checker=debug.ExprInspection \
-// RUN:   -analyzer-checker=alpha.unix.StdCLibraryFunctions \
+// RUN:   -analyzer-checker=unix.StdCLibraryFunctions \
 // RUN:   -analyzer-checker=apiModeling.Errno \
 // RUN:   -analyzer-checker=alpha.unix.Errno \
-// RUN:   -analyzer-config alpha.unix.StdCLibraryFunctions:ModelPOSIX=true
+// RUN:   -analyzer-config unix.StdCLibraryFunctions:ModelPOSIX=true
 
 #include "Inputs/errno_var.h"
 
diff --git a/clang/test/Analysis/errno-stdlibraryfunctions.c b/clang/test/Analysis/errno-stdlibraryfunctions.c
index a3b42f4425..fce5e5d6b0 100644
--- a/clang/test/Analysis/errno-stdlibraryfunctions.c
+++ b/clang/test/Analysis/errno-stdlibraryfunctions.c
@@ -1,10 +1,10 @@
 // RUN: %clang_analyze_cc1 -verify %s \
 // RUN:   -analyzer-checker=core \
 // RUN:   -analyzer-checker=debug.ExprInspection \
-// RUN:   -analyzer-checker=alpha.unix.StdCLibraryFunctions \
+// RUN:   -analyzer-checker=unix.StdCLibraryFunctions \
 // RUN:   -analyzer-checker=apiModeling.Errno \
 // RUN:   -analyzer-checker=alpha.unix.Errno \
-// RUN:   -analyzer-config alpha.unix.StdCLibraryFunctions:ModelPOSIX=true
+// RUN:   -analyzer-config unix.StdCLibraryFunctions:ModelPOSIX=true
 
 #include "Inputs/errno_var.h"
 
diff --git a/clang/test/Analysis/lambdas.cpp b/clang/test/Analysis/lambdas.cpp
index c7463ef7ca..554ed342a4 100644
--- a/clang/test/Analysis/lambdas.cpp
+++ b/clang/test/Analysis/lambdas.cpp
@@ -194,8 +194,9 @@ void testFunctionPointerCapture() {
 // Captured variable-length array.
 
 void testVariableLengthArrayCaptured() {
-  int n = 2;
-  int array[n];
+  int n = 2;     // expected-note {{declared here}}
+  int array[n];  // expected-warning {{variable length arrays in C++ are a Clang extension}} \
+                    expected-note {{read of non-const variable 'n' is not allowed in a constant expression}}
   array[0] = 7;
 
   int i = [&]{
diff --git a/clang/test/Analysis/std-c-library-functions-POSIX-lookup.c b/clang/test/Analysis/std-c-library-functions-POSIX-lookup.c
index 22f752fee0..5338fa092d 100644
--- a/clang/test/Analysis/std-c-library-functions-POSIX-lookup.c
+++ b/clang/test/Analysis/std-c-library-functions-POSIX-lookup.c
@@ -1,8 +1,8 @@
 // RUN: %clang_analyze_cc1 %s \
 // RUN:   -analyzer-checker=core \
-// RUN:   -analyzer-checker=alpha.unix.StdCLibraryFunctions \
-// RUN:   -analyzer-config alpha.unix.StdCLibraryFunctions:ModelPOSIX=true \
-// RUN:   -analyzer-config alpha.unix.StdCLibraryFunctions:DisplayLoadedSummaries=true \
+// RUN:   -analyzer-checker=unix.StdCLibraryFunctions \
+// RUN:   -analyzer-config unix.StdCLibraryFunctions:ModelPOSIX=true \
+// RUN:   -analyzer-config unix.StdCLibraryFunctions:DisplayLoadedSummaries=true \
 // RUN:   -analyzer-checker=debug.ExprInspection \
 // RUN:   -analyzer-config eagerly-assume=false \
 // RUN:   -triple i686-unknown-linux 2>&1 | FileCheck %s --allow-empty
diff --git a/clang/test/Analysis/std-c-library-functions-POSIX-socket-sockaddr.cpp b/clang/test/Analysis/std-c-library-functions-POSIX-socket-sockaddr.cpp
index c835b80960..8aa3702875 100644
--- a/clang/test/Analysis/std-c-library-functions-POSIX-socket-sockaddr.cpp
+++ b/clang/test/Analysis/std-c-library-functions-POSIX-socket-sockaddr.cpp
@@ -1,8 +1,8 @@
 // RUN: %clang_analyze_cc1 %s \
 // RUN:   -analyzer-checker=core \
-// RUN:   -analyzer-checker=alpha.unix.StdCLibraryFunctions \
-// RUN:   -analyzer-config alpha.unix.StdCLibraryFunctions:ModelPOSIX=true \
-// RUN:   -analyzer-config alpha.unix.StdCLibraryFunctions:DisplayLoadedSummaries=true \
+// RUN:   -analyzer-checker=unix.StdCLibraryFunctions \
+// RUN:   -analyzer-config unix.StdCLibraryFunctions:ModelPOSIX=true \
+// RUN:   -analyzer-config unix.StdCLibraryFunctions:DisplayLoadedSummaries=true \
 // RUN:   -analyzer-checker=debug.ExprInspection \
 // RUN:   -analyzer-config eagerly-assume=false \
 // RUN:   -triple i686-unknown-linux 2>&1 | FileCheck %s
diff --git a/clang/test/Analysis/std-c-library-functions-POSIX.c b/clang/test/Analysis/std-c-library-functions-POSIX.c
index 870af4f86c..84ce0f21e5 100644
--- a/clang/test/Analysis/std-c-library-functions-POSIX.c
+++ b/clang/test/Analysis/std-c-library-functions-POSIX.c
@@ -1,17 +1,17 @@
 // RUN: %clang_analyze_cc1 %s \
 // RUN:   -analyzer-checker=core \
-// RUN:   -analyzer-checker=alpha.unix.StdCLibraryFunctions \
-// RUN:   -analyzer-config alpha.unix.StdCLibraryFunctions:ModelPOSIX=true \
-// RUN:   -analyzer-config alpha.unix.StdCLibraryFunctions:DisplayLoadedSummaries=true \
+// RUN:   -analyzer-checker=unix.StdCLibraryFunctions \
+// RUN:   -analyzer-config unix.StdCLibraryFunctions:ModelPOSIX=true \
+// RUN:   -analyzer-config unix.StdCLibraryFunctions:DisplayLoadedSummaries=true \
 // RUN:   -analyzer-checker=debug.ExprInspection \
 // RUN:   -analyzer-config eagerly-assume=false \
 // RUN:   -triple i686-unknown-linux -verify
 
 // RUN: %clang_analyze_cc1 %s \
 // RUN:   -analyzer-checker=core \
-// RUN:   -analyzer-checker=alpha.unix.StdCLibraryFunctions \
-// RUN:   -analyzer-config alpha.unix.StdCLibraryFunctions:ModelPOSIX=true \
-// RUN:   -analyzer-config alpha.unix.StdCLibraryFunctions:DisplayLoadedSummaries=true \
+// RUN:   -analyzer-checker=unix.StdCLibraryFunctions \
+// RUN:   -analyzer-config unix.StdCLibraryFunctions:ModelPOSIX=true \
+// RUN:   -analyzer-config unix.StdCLibraryFunctions:DisplayLoadedSummaries=true \
 // RUN:   -analyzer-checker=debug.ExprInspection \
 // RUN:   -analyzer-config eagerly-assume=false \
 // RUN:   -triple i686-unknown-linux 2>&1 | FileCheck %s
diff --git a/clang/test/Analysis/std-c-library-functions-arg-constraints-note-tags.cpp b/clang/test/Analysis/std-c-library-functions-arg-constraints-note-tags.cpp
index 573b0076a0..7eea451289 100644
--- a/clang/test/Analysis/std-c-library-functions-arg-constraints-note-tags.cpp
+++ b/clang/test/Analysis/std-c-library-functions-arg-constraints-note-tags.cpp
@@ -1,8 +1,8 @@
 // RUN: %clang_analyze_cc1 %s \
 // RUN:   -analyzer-checker=core \
-// RUN:   -analyzer-checker=alpha.unix.StdCLibraryFunctions \
+// RUN:   -analyzer-checker=unix.StdCLibraryFunctions \
 // RUN:   -analyzer-checker=debug.StdCLibraryFunctionsTester \
-// RUN:   -analyzer-config alpha.unix.StdCLibraryFunctions:DisplayLoadedSummaries=true \
+// RUN:   -analyzer-config unix.StdCLibraryFunctions:DisplayLoadedSummaries=true \
 // RUN:   -analyzer-checker=debug.ExprInspection \
 // RUN:   -analyzer-config eagerly-assume=false \
 // RUN:   -triple i686-unknown-linux \
diff --git a/clang/test/Analysis/std-c-library-functions-arg-constraints-notes.cpp b/clang/test/Analysis/std-c-library-functions-arg-constraints-notes.cpp
index 781b96d531..f30f977bcd 100644
--- a/clang/test/Analysis/std-c-library-functions-arg-constraints-notes.cpp
+++ b/clang/test/Analysis/std-c-library-functions-arg-constraints-notes.cpp
@@ -1,8 +1,8 @@
 // RUN: %clang_analyze_cc1 %s \
 // RUN:   -analyzer-checker=core \
-// RUN:   -analyzer-checker=alpha.unix.StdCLibraryFunctions \
+// RUN:   -analyzer-checker=unix.StdCLibraryFunctions \
 // RUN:   -analyzer-checker=debug.StdCLibraryFunctionsTester \
-// RUN:   -analyzer-config alpha.unix.StdCLibraryFunctions:DisplayLoadedSummaries=true \
+// RUN:   -analyzer-config unix.StdCLibraryFunctions:DisplayLoadedSummaries=true \
 // RUN:   -analyzer-checker=debug.ExprInspection \
 // RUN:   -analyzer-config eagerly-assume=false \
 // RUN:   -triple i686-unknown-linux \
diff --git a/clang/test/Analysis/std-c-library-functions-arg-constraints-tracking-notes.c b/clang/test/Analysis/std-c-library-functions-arg-constraints-tracking-notes.c
index d497b87c48..0a66e49be9 100644
--- a/clang/test/Analysis/std-c-library-functions-arg-constraints-tracking-notes.c
+++ b/clang/test/Analysis/std-c-library-functions-arg-constraints-tracking-notes.c
@@ -1,7 +1,7 @@
 // Check the bugpath related to the reports.
 // RUN: %clang_analyze_cc1 %s \
 // RUN:   -analyzer-checker=core \
-// RUN:   -analyzer-checker=alpha.unix.StdCLibraryFunctions \
+// RUN:   -analyzer-checker=unix.StdCLibraryFunctions \
 // RUN:   -analyzer-checker=debug.StdCLibraryFunctionsTester \
 // RUN:   -analyzer-checker=debug.ExprInspection \
 // RUN:   -triple x86_64-unknown-linux-gnu \
diff --git a/clang/test/Analysis/std-c-library-functions-arg-constraints.c b/clang/test/Analysis/std-c-library-functions-arg-constraints.c
index 062faccfb6..0b817dda98 100644
--- a/clang/test/Analysis/std-c-library-functions-arg-constraints.c
+++ b/clang/test/Analysis/std-c-library-functions-arg-constraints.c
@@ -1,8 +1,8 @@
 // Check the basic reporting/warning and the application of constraints.
 // RUN: %clang_analyze_cc1 %s \
 // RUN:   -analyzer-checker=core \
-// RUN:   -analyzer-checker=alpha.unix.StdCLibraryFunctions \
-// RUN:   -analyzer-config alpha.unix.StdCLibraryFunctions:ModelPOSIX=true \
+// RUN:   -analyzer-checker=unix.StdCLibraryFunctions \
+// RUN:   -analyzer-config unix.StdCLibraryFunctions:ModelPOSIX=true \
 // RUN:   -analyzer-checker=debug.StdCLibraryFunctionsTester \
 // RUN:   -analyzer-checker=debug.ExprInspection \
 // RUN:   -triple x86_64-unknown-linux-gnu \
@@ -11,8 +11,8 @@
 // Check the bugpath related to the reports.
 // RUN: %clang_analyze_cc1 %s \
 // RUN:   -analyzer-checker=core \
-// RUN:   -analyzer-checker=alpha.unix.StdCLibraryFunctions \
-// RUN:   -analyzer-config alpha.unix.StdCLibraryFunctions:ModelPOSIX=true \
+// RUN:   -analyzer-checker=unix.StdCLibraryFunctions \
+// RUN:   -analyzer-config unix.StdCLibraryFunctions:ModelPOSIX=true \
 // RUN:   -analyzer-checker=debug.StdCLibraryFunctionsTester \
 // RUN:   -analyzer-checker=debug.ExprInspection \
 // RUN:   -triple x86_64-unknown-linux-gnu \
diff --git a/clang/test/Analysis/std-c-library-functions-arg-constraints.cpp b/clang/test/Analysis/std-c-library-functions-arg-constraints.cpp
index 80a680eb55..037b5d9ad9 100644
--- a/clang/test/Analysis/std-c-library-functions-arg-constraints.cpp
+++ b/clang/test/Analysis/std-c-library-functions-arg-constraints.cpp
@@ -1,6 +1,6 @@
 // RUN: %clang_analyze_cc1 %s \
 // RUN:   -analyzer-checker=core \
-// RUN:   -analyzer-checker=alpha.unix.StdCLibraryFunctions \
+// RUN:   -analyzer-checker=unix.StdCLibraryFunctions \
 // RUN:   -analyzer-checker=debug.StdCLibraryFunctionsTester \
 // RUN:   -analyzer-checker=debug.ExprInspection \
 // RUN:   -analyzer-config eagerly-assume=false \
diff --git a/clang/test/Analysis/std-c-library-functions-arg-cstring-dependency.c b/clang/test/Analysis/std-c-library-functions-arg-cstring-dependency.c
index 5ebb07e524..2fa15c00cb 100644
--- a/clang/test/Analysis/std-c-library-functions-arg-cstring-dependency.c
+++ b/clang/test/Analysis/std-c-library-functions-arg-cstring-dependency.c
@@ -5,9 +5,9 @@
 
 // RUN: %clang_analyze_cc1 %s \
 // RUN:   -analyzer-checker=core \
-// RUN:   -analyzer-checker=alpha.unix.StdCLibraryFunctions \
+// RUN:   -analyzer-checker=unix.StdCLibraryFunctions \
 // RUN:   -analyzer-checker=unix.cstring.NullArg \
-// RUN:   -analyzer-config alpha.unix.StdCLibraryFunctions:ModelPOSIX=true \
+// RUN:   -analyzer-config unix.StdCLibraryFunctions:ModelPOSIX=true \
 // RUN:   -triple x86_64-unknown-linux-gnu \
 // RUN:   -verify
 
diff --git a/clang/test/Analysis/std-c-library-functions-arg-enabled-checkers.c b/clang/test/Analysis/std-c-library-functions-arg-enabled-checkers.c
index 40fb4a734f..7f5bfba6ff 100644
--- a/clang/test/Analysis/std-c-library-functions-arg-enabled-checkers.c
+++ b/clang/test/Analysis/std-c-library-functions-arg-enabled-checkers.c
@@ -3,9 +3,9 @@
 
 // RUN: %clang --analyze %s --target=x86_64-pc-linux-gnu \
 // RUN:   -Xclang -analyzer-checker=core \
-// RUN:   -Xclang -analyzer-checker=alpha.unix.StdCLibraryFunctions \
+// RUN:   -Xclang -analyzer-checker=unix.StdCLibraryFunctions \
 // RUN:   -Xclang -analyzer-config \
-// RUN:      -Xclang alpha.unix.StdCLibraryFunctions:ModelPOSIX=true \
+// RUN:      -Xclang unix.StdCLibraryFunctions:ModelPOSIX=true \
 // RUN:   -Xclang -analyzer-checker=alpha.unix.Stream \
 // RUN:   -Xclang -analyzer-list-enabled-checkers \
 // RUN:   -Xclang -analyzer-display-progress \
@@ -14,17 +14,16 @@
 
 // CHECK:      OVERVIEW: Clang Static Analyzer Enabled Checkers List
 // CHECK-EMPTY:
-// CHECK-NEXT: core.CallAndMessageModeling
-// CHECK-NEXT: core.CallAndMessage
 // CHECK-NEXT: core.NonNullParamChecker
 // CHECK-NEXT: alpha.unix.Stream
-// CHECK-NEXT: alpha.unix.StdCLibraryFunctions
 // CHECK-NEXT: apiModeling.Errno
 // CHECK-NEXT: apiModeling.TrustNonnull
 // CHECK-NEXT: apiModeling.TrustReturnsNonnull
 // CHECK-NEXT: apiModeling.llvm.CastValue
 // CHECK-NEXT: apiModeling.llvm.ReturnValue
 // CHECK-NEXT: core.BitwiseShift
+// CHECK-NEXT: core.CallAndMessageModeling
+// CHECK-NEXT: core.CallAndMessage
 // CHECK-NEXT: core.DivideZero
 // CHECK-NEXT: core.DynamicTypePropagation
 // CHECK-NEXT: core.NonnilStringConstants
@@ -57,6 +56,7 @@
 // CHECK-NEXT: unix.Malloc
 // CHECK-NEXT: unix.MallocSizeof
 // CHECK-NEXT: unix.MismatchedDeallocator
+// CHECK-NEXT: unix.StdCLibraryFunctions
 // CHECK-NEXT: unix.Vfork
 // CHECK-NEXT: unix.cstring.BadSizeArg
 // CHECK-NEXT: unix.cstring.NullArg
diff --git a/clang/test/Analysis/std-c-library-functions-arg-weakdeps.c b/clang/test/Analysis/std-c-library-functions-arg-weakdeps.c
index 87f07a2d90..5df5a77001 100644
--- a/clang/test/Analysis/std-c-library-functions-arg-weakdeps.c
+++ b/clang/test/Analysis/std-c-library-functions-arg-weakdeps.c
@@ -4,8 +4,8 @@
 // RUN: %clang_analyze_cc1 %s \
 // RUN:   -analyzer-checker=core \
 // RUN:   -analyzer-checker=alpha.unix.Stream \
-// RUN:   -analyzer-checker=alpha.unix.StdCLibraryFunctions \
-// RUN:   -analyzer-config alpha.unix.StdCLibraryFunctions:ModelPOSIX=true \
+// RUN:   -analyzer-checker=unix.StdCLibraryFunctions \
+// RUN:   -analyzer-config unix.StdCLibraryFunctions:ModelPOSIX=true \
 // RUN:   -triple x86_64-unknown-linux-gnu \
 // RUN:   -verify
 
@@ -14,9 +14,9 @@
 
 // RUN: %clang_analyze_cc1 %s \
 // RUN:   -analyzer-checker=core \
-// RUN:   -analyzer-checker=alpha.unix.StdCLibraryFunctions \
-// RUN:   -analyzer-config alpha.unix.StdCLibraryFunctions:ModelPOSIX=true \
-// RUN:   -analyzer-config alpha.unix.StdCLibraryFunctions:DisplayLoadedSummaries=true \
+// RUN:   -analyzer-checker=unix.StdCLibraryFunctions \
+// RUN:   -analyzer-config unix.StdCLibraryFunctions:ModelPOSIX=true \
+// RUN:   -analyzer-config unix.StdCLibraryFunctions:DisplayLoadedSummaries=true \
 // RUN:   -triple x86_64-unknown-linux 2>&1 | FileCheck %s
 
 // CHECK: Loaded summary for: int isalnum(int)
diff --git a/clang/test/Analysis/std-c-library-functions-eof.c b/clang/test/Analysis/std-c-library-functions-eof.c
index 0050bf2d9b..0fadf73436 100644
--- a/clang/test/Analysis/std-c-library-functions-eof.c
+++ b/clang/test/Analysis/std-c-library-functions-eof.c
@@ -1,8 +1,8 @@
-// RUN: %clang_analyze_cc1 -analyzer-checker=core,alpha.unix.StdCLibraryFunctions,debug.ExprInspection -verify -analyzer-config eagerly-assume=false %s
-// RUN: %clang_analyze_cc1 -triple i686-unknown-linux -analyzer-checker=core,alpha.unix.StdCLibraryFunctions,debug.ExprInspection -verify -analyzer-config eagerly-assume=false %s
-// RUN: %clang_analyze_cc1 -triple x86_64-unknown-linux -analyzer-checker=core,alpha.unix.StdCLibraryFunctions,debug.ExprInspection -verify -analyzer-config eagerly-assume=false %s
-// RUN: %clang_analyze_cc1 -triple armv7-a15-linux -analyzer-checker=core,alpha.unix.StdCLibraryFunctions,debug.ExprInspection -verify -analyzer-config eagerly-assume=false %s
-// RUN: %clang_analyze_cc1 -triple thumbv7-a15-linux -analyzer-checker=core,alpha.unix.StdCLibraryFunctions,debug.ExprInspection -verify -analyzer-config eagerly-assume=false %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,unix.StdCLibraryFunctions,debug.ExprInspection -verify -analyzer-config eagerly-assume=false %s
+// RUN: %clang_analyze_cc1 -triple i686-unknown-linux -analyzer-checker=core,unix.StdCLibraryFunctions,debug.ExprInspection -verify -analyzer-config eagerly-assume=false %s
+// RUN: %clang_analyze_cc1 -triple x86_64-unknown-linux -analyzer-checker=core,unix.StdCLibraryFunctions,debug.ExprInspection -verify -analyzer-config eagerly-assume=false %s
+// RUN: %clang_analyze_cc1 -triple armv7-a15-linux -analyzer-checker=core,unix.StdCLibraryFunctions,debug.ExprInspection -verify -analyzer-config eagerly-assume=false %s
+// RUN: %clang_analyze_cc1 -triple thumbv7-a15-linux -analyzer-checker=core,unix.StdCLibraryFunctions,debug.ExprInspection -verify -analyzer-config eagerly-assume=false %s
 
 void clang_analyzer_eval(int);
 
diff --git a/clang/test/Analysis/std-c-library-functions-inlined.c b/clang/test/Analysis/std-c-library-functions-inlined.c
index e40f5204f6..5277a6efbe 100644
--- a/clang/test/Analysis/std-c-library-functions-inlined.c
+++ b/clang/test/Analysis/std-c-library-functions-inlined.c
@@ -1,8 +1,8 @@
-// RUN: %clang_analyze_cc1 -analyzer-checker=alpha.unix.StdCLibraryFunctions -verify %s
-// RUN: %clang_analyze_cc1 -triple i686-unknown-linux -analyzer-checker=alpha.unix.StdCLibraryFunctions -verify %s
-// RUN: %clang_analyze_cc1 -triple x86_64-unknown-linux -analyzer-checker=alpha.unix.StdCLibraryFunctions -verify %s
-// RUN: %clang_analyze_cc1 -triple armv7-a15-linux -analyzer-checker=alpha.unix.StdCLibraryFunctions -verify %s
-// RUN: %clang_analyze_cc1 -triple thumbv7-a15-linux -analyzer-checker=alpha.unix.StdCLibraryFunctions -verify %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=unix.StdCLibraryFunctions -verify %s
+// RUN: %clang_analyze_cc1 -triple i686-unknown-linux -analyzer-checker=unix.StdCLibraryFunctions -verify %s
+// RUN: %clang_analyze_cc1 -triple x86_64-unknown-linux -analyzer-checker=unix.StdCLibraryFunctions -verify %s
+// RUN: %clang_analyze_cc1 -triple armv7-a15-linux -analyzer-checker=unix.StdCLibraryFunctions -verify %s
+// RUN: %clang_analyze_cc1 -triple thumbv7-a15-linux -analyzer-checker=unix.StdCLibraryFunctions -verify %s
 
 // This test tests crashes that occur when standard functions are available
 // for inlining.
diff --git a/clang/test/Analysis/std-c-library-functions-lookup.c b/clang/test/Analysis/std-c-library-functions-lookup.c
index 7032dca1b8..e47d9bddda 100644
--- a/clang/test/Analysis/std-c-library-functions-lookup.c
+++ b/clang/test/Analysis/std-c-library-functions-lookup.c
@@ -1,7 +1,7 @@
 // RUN: %clang_analyze_cc1 %s \
 // RUN:   -analyzer-checker=core \
-// RUN:   -analyzer-checker=alpha.unix.StdCLibraryFunctions \
-// RUN:   -analyzer-config alpha.unix.StdCLibraryFunctions:DisplayLoadedSummaries=true \
+// RUN:   -analyzer-checker=unix.StdCLibraryFunctions \
+// RUN:   -analyzer-config unix.StdCLibraryFunctions:DisplayLoadedSummaries=true \
 // RUN:   -analyzer-checker=debug.ExprInspection \
 // RUN:   -analyzer-config eagerly-assume=false \
 // RUN:   -triple i686-unknown-linux 2>&1 | FileCheck %s
diff --git a/clang/test/Analysis/std-c-library-functions-lookup.cpp b/clang/test/Analysis/std-c-library-functions-lookup.cpp
index 22778b2fde..9480b88bec 100644
--- a/clang/test/Analysis/std-c-library-functions-lookup.cpp
+++ b/clang/test/Analysis/std-c-library-functions-lookup.cpp
@@ -1,7 +1,7 @@
 // RUN: %clang_analyze_cc1 %s \
 // RUN:   -analyzer-checker=core \
-// RUN:   -analyzer-checker=alpha.unix.StdCLibraryFunctions \
-// RUN:   -analyzer-config alpha.unix.StdCLibraryFunctions:DisplayLoadedSummaries=true \
+// RUN:   -analyzer-checker=unix.StdCLibraryFunctions \
+// RUN:   -analyzer-config unix.StdCLibraryFunctions:DisplayLoadedSummaries=true \
 // RUN:   -analyzer-checker=debug.ExprInspection \
 // RUN:   -analyzer-config eagerly-assume=false \
 // RUN:   -triple i686-unknown-linux 2>&1 | FileCheck %s
diff --git a/clang/test/Analysis/std-c-library-functions-path-notes.c b/clang/test/Analysis/std-c-library-functions-path-notes.c
index 6b5d1d7bd4..d0957483c1 100644
--- a/clang/test/Analysis/std-c-library-functions-path-notes.c
+++ b/clang/test/Analysis/std-c-library-functions-path-notes.c
@@ -1,6 +1,6 @@
 // RUN: %clang_analyze_cc1 -verify %s \
-// RUN:     -analyzer-checker=core,alpha.unix.StdCLibraryFunctions \
-// RUN:     -analyzer-config alpha.unix.StdCLibraryFunctions:ModelPOSIX=true \
+// RUN:     -analyzer-checker=core,unix.StdCLibraryFunctions \
+// RUN:     -analyzer-config unix.StdCLibraryFunctions:ModelPOSIX=true \
 // RUN:     -analyzer-output=text
 
 #include "Inputs/std-c-library-functions-POSIX.h"
diff --git a/clang/test/Analysis/std-c-library-functions-restrict.c b/clang/test/Analysis/std-c-library-functions-restrict.c
index 6260f851cd..27e223c6e5 100644
--- a/clang/test/Analysis/std-c-library-functions-restrict.c
+++ b/clang/test/Analysis/std-c-library-functions-restrict.c
@@ -1,8 +1,8 @@
 // RUN: %clang_analyze_cc1 %s \
 // RUN:   -analyzer-checker=core \
-// RUN:   -analyzer-checker=alpha.unix.StdCLibraryFunctions \
+// RUN:   -analyzer-checker=unix.StdCLibraryFunctions \
 // RUN:   -analyzer-checker=debug.StdCLibraryFunctionsTester \
-// RUN:   -analyzer-config alpha.unix.StdCLibraryFunctions:DisplayLoadedSummaries=true \
+// RUN:   -analyzer-config unix.StdCLibraryFunctions:DisplayLoadedSummaries=true \
 // RUN:   -triple i686-unknown-linux 2>&1 | FileCheck %s
 
 // The signatures for these functions are the same and they specify their
diff --git a/clang/test/Analysis/std-c-library-functions-restrict.cpp b/clang/test/Analysis/std-c-library-functions-restrict.cpp
index e431b14b19..8954ab4886 100644
--- a/clang/test/Analysis/std-c-library-functions-restrict.cpp
+++ b/clang/test/Analysis/std-c-library-functions-restrict.cpp
@@ -1,8 +1,8 @@
 // RUN: %clang_analyze_cc1 %s \
 // RUN:   -analyzer-checker=core \
-// RUN:   -analyzer-checker=alpha.unix.StdCLibraryFunctions \
+// RUN:   -analyzer-checker=unix.StdCLibraryFunctions \
 // RUN:   -analyzer-checker=debug.StdCLibraryFunctionsTester \
-// RUN:   -analyzer-config alpha.unix.StdCLibraryFunctions:DisplayLoadedSummaries=true \
+// RUN:   -analyzer-config unix.StdCLibraryFunctions:DisplayLoadedSummaries=true \
 // RUN:   -triple i686-unknown-linux 2>&1 | FileCheck %s
 
 // The signatures for these functions are the same and they specify their
diff --git a/clang/test/Analysis/std-c-library-functions-vs-stream-checker.c b/clang/test/Analysis/std-c-library-functions-vs-stream-checker.c
index 4df46207da..281fbaaffe 100644
--- a/clang/test/Analysis/std-c-library-functions-vs-stream-checker.c
+++ b/clang/test/Analysis/std-c-library-functions-vs-stream-checker.c
@@ -8,8 +8,8 @@
 
 // Check the case when only the StdLibraryFunctionsChecker is enabled.
 // RUN: %clang_analyze_cc1 %s \
-// RUN:   -analyzer-checker=alpha.unix.StdCLibraryFunctions \
-// RUN:   -analyzer-config alpha.unix.StdCLibraryFunctions:DisplayLoadedSummaries=true \
+// RUN:   -analyzer-checker=unix.StdCLibraryFunctions \
+// RUN:   -analyzer-config unix.StdCLibraryFunctions:DisplayLoadedSummaries=true \
 // RUN:   -analyzer-checker=debug.ExprInspection \
 // RUN:   -analyzer-config eagerly-assume=false \
 // RUN:   -triple x86_64-unknown-linux \
@@ -19,8 +19,8 @@
 // StdLibraryFunctionsChecker are enabled.
 // RUN: %clang_analyze_cc1 %s \
 // RUN:   -analyzer-checker=core,alpha.unix.Stream \
-// RUN:   -analyzer-checker=alpha.unix.StdCLibraryFunctions \
-// RUN:   -analyzer-config alpha.unix.StdCLibraryFunctions:DisplayLoadedSummaries=true \
+// RUN:   -analyzer-checker=unix.StdCLibraryFunctions \
+// RUN:   -analyzer-config unix.StdCLibraryFunctions:DisplayLoadedSummaries=true \
 // RUN:   -analyzer-checker=debug.ExprInspection \
 // RUN:   -analyzer-config eagerly-assume=false \
 // RUN:   -triple x86_64-unknown-linux \
diff --git a/clang/test/Analysis/std-c-library-functions.c b/clang/test/Analysis/std-c-library-functions.c
index 392784722d..b7eb6b2844 100644
--- a/clang/test/Analysis/std-c-library-functions.c
+++ b/clang/test/Analysis/std-c-library-functions.c
@@ -1,6 +1,6 @@
 // RUN: %clang_analyze_cc1 %s \
 // RUN:   -analyzer-checker=core \
-// RUN:   -analyzer-checker=alpha.unix.StdCLibraryFunctions \
+// RUN:   -analyzer-checker=unix.StdCLibraryFunctions \
 // RUN:   -analyzer-checker=debug.ExprInspection \
 // RUN:   -analyzer-config eagerly-assume=false \
 // RUN:   -triple i686-unknown-linux \
@@ -8,7 +8,7 @@
 
 // RUN: %clang_analyze_cc1 %s \
 // RUN:   -analyzer-checker=core \
-// RUN:   -analyzer-checker=alpha.unix.StdCLibraryFunctions \
+// RUN:   -analyzer-checker=unix.StdCLibraryFunctions \
 // RUN:   -analyzer-checker=debug.ExprInspection \
 // RUN:   -analyzer-config eagerly-assume=false \
 // RUN:   -triple x86_64-unknown-linux \
@@ -16,7 +16,7 @@
 
 // RUN: %clang_analyze_cc1 %s \
 // RUN:   -analyzer-checker=core \
-// RUN:   -analyzer-checker=alpha.unix.StdCLibraryFunctions \
+// RUN:   -analyzer-checker=unix.StdCLibraryFunctions \
 // RUN:   -analyzer-checker=debug.ExprInspection \
 // RUN:   -analyzer-config eagerly-assume=false \
 // RUN:   -triple armv7-a15-linux \
@@ -24,7 +24,7 @@
 
 // RUN: %clang_analyze_cc1 %s \
 // RUN:   -analyzer-checker=core \
-// RUN:   -analyzer-checker=alpha.unix.StdCLibraryFunctions \
+// RUN:   -analyzer-checker=unix.StdCLibraryFunctions \
 // RUN:   -analyzer-checker=debug.ExprInspection \
 // RUN:   -analyzer-config eagerly-assume=false \
 // RUN:   -triple thumbv7-a15-linux \
@@ -32,8 +32,8 @@
 
 // RUN: %clang_analyze_cc1 %s \
 // RUN:   -analyzer-checker=core \
-// RUN:   -analyzer-checker=alpha.unix.StdCLibraryFunctions \
-// RUN:   -analyzer-config alpha.unix.StdCLibraryFunctions:DisplayLoadedSummaries=true \
+// RUN:   -analyzer-checker=unix.StdCLibraryFunctions \
+// RUN:   -analyzer-config unix.StdCLibraryFunctions:DisplayLoadedSummaries=true \
 // RUN:   -analyzer-checker=debug.ExprInspection \
 // RUN:   -analyzer-config eagerly-assume=false \
 // RUN:   -triple i686-unknown-linux 2>&1 | FileCheck %s
diff --git a/clang/test/Analysis/std-c-library-functions.cpp b/clang/test/Analysis/std-c-library-functions.cpp
index 2da01d6351..00b341af5f 100644
--- a/clang/test/Analysis/std-c-library-functions.cpp
+++ b/clang/test/Analysis/std-c-library-functions.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_analyze_cc1 -triple x86_64-unknown-linux -analyzer-checker=alpha.unix.StdCLibraryFunctions,debug.ExprInspection -verify %s
+// RUN: %clang_analyze_cc1 -triple x86_64-unknown-linux -analyzer-checker=unix.StdCLibraryFunctions,debug.ExprInspection -verify %s
 
 // Test that we don't model functions with broken prototypes.
 // Because they probably work differently as well.
diff --git a/clang/test/Analysis/std-c-library-posix-crash.c b/clang/test/Analysis/std-c-library-posix-crash.c
index 66e7bf4656..68ad771aa9 100644
--- a/clang/test/Analysis/std-c-library-posix-crash.c
+++ b/clang/test/Analysis/std-c-library-posix-crash.c
@@ -1,6 +1,6 @@
 // RUN: %clang_analyze_cc1 \
-// RUN:   -analyzer-checker=core,alpha.unix.StdCLibraryFunctions \
-// RUN:   -analyzer-config alpha.unix.StdCLibraryFunctions:ModelPOSIX=true \
+// RUN:   -analyzer-checker=core,unix.StdCLibraryFunctions \
+// RUN:   -analyzer-config unix.StdCLibraryFunctions:ModelPOSIX=true \
 // RUN:   -verify %s
 //
 // expected-no-diagnostics
diff --git a/clang/test/Analysis/stream-errno-note.c b/clang/test/Analysis/stream-errno-note.c
index 4ab215a645..32d9d4fd96 100644
--- a/clang/test/Analysis/stream-errno-note.c
+++ b/clang/test/Analysis/stream-errno-note.c
@@ -1,8 +1,8 @@
 // RUN: %clang_analyze_cc1 -analyzer-checker=core \
 // RUN:   -analyzer-checker=alpha.unix.Stream \
 // RUN:   -analyzer-checker=alpha.unix.Errno \
-// RUN:   -analyzer-checker=alpha.unix.StdCLibraryFunctions \
-// RUN:   -analyzer-config alpha.unix.StdCLibraryFunctions:ModelPOSIX=true \
+// RUN:   -analyzer-checker=unix.StdCLibraryFunctions \
+// RUN:   -analyzer-config unix.StdCLibraryFunctions:ModelPOSIX=true \
 // RUN:   -analyzer-output text -verify %s
 
 #include "Inputs/system-header-simulator.h"
diff --git a/clang/test/Analysis/stream-errno.c b/clang/test/Analysis/stream-errno.c
index d8c0c8223a..cf4e2e3d78 100644
--- a/clang/test/Analysis/stream-errno.c
+++ b/clang/test/Analysis/stream-errno.c
@@ -1,5 +1,5 @@
-// RUN: %clang_analyze_cc1 -analyzer-checker=core,alpha.unix.Stream,alpha.unix.Errno,alpha.unix.StdCLibraryFunctions,debug.ExprInspection \
-// RUN:   -analyzer-config alpha.unix.StdCLibraryFunctions:ModelPOSIX=true -verify %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,alpha.unix.Stream,alpha.unix.Errno,unix.StdCLibraryFunctions,debug.ExprInspection \
+// RUN:   -analyzer-config unix.StdCLibraryFunctions:ModelPOSIX=true -verify %s
 
 #include "Inputs/system-header-simulator.h"
 #include "Inputs/errno_func.h"
diff --git a/clang/test/Analysis/stream-noopen.c b/clang/test/Analysis/stream-noopen.c
index 03784603d9..cbeac276fd 100644
--- a/clang/test/Analysis/stream-noopen.c
+++ b/clang/test/Analysis/stream-noopen.c
@@ -2,16 +2,16 @@
 // RUN:   -analyzer-checker=core \
 // RUN:   -analyzer-checker=alpha.unix.Errno \
 // RUN:   -analyzer-checker=alpha.unix.Stream \
-// RUN:   -analyzer-checker=alpha.unix.StdCLibraryFunctions \
-// RUN:   -analyzer-config alpha.unix.StdCLibraryFunctions:ModelPOSIX=true \
+// RUN:   -analyzer-checker=unix.StdCLibraryFunctions \
+// RUN:   -analyzer-config unix.StdCLibraryFunctions:ModelPOSIX=true \
 // RUN:   -analyzer-checker=debug.ExprInspection
 
 // enable only StdCLibraryFunctions checker
 // RUN: %clang_analyze_cc1 -verify %s \
 // RUN:   -analyzer-checker=core \
 // RUN:   -analyzer-checker=alpha.unix.Errno \
-// RUN:   -analyzer-checker=alpha.unix.StdCLibraryFunctions \
-// RUN:   -analyzer-config alpha.unix.StdCLibraryFunctions:ModelPOSIX=true \
+// RUN:   -analyzer-checker=unix.StdCLibraryFunctions \
+// RUN:   -analyzer-config unix.StdCLibraryFunctions:ModelPOSIX=true \
 // RUN:   -analyzer-checker=debug.ExprInspection
 
 #include "Inputs/system-header-simulator.h"
diff --git a/clang/test/Analysis/stream-note.c b/clang/test/Analysis/stream-note.c
index 257245754d..b9fdc16b19 100644
--- a/clang/test/Analysis/stream-note.c
+++ b/clang/test/Analysis/stream-note.c
@@ -1,7 +1,7 @@
 // RUN: %clang_analyze_cc1 -analyzer-checker=core,alpha.unix.Stream -analyzer-output text \
 // RUN:   -verify %s
-// RUN: %clang_analyze_cc1 -analyzer-checker=core,alpha.unix.Stream,alpha.unix.StdCLibraryFunctions -analyzer-output text \
-// RUN:   -analyzer-config alpha.unix.StdCLibraryFunctions:ModelPOSIX=true -verify=expected,stdargs %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,alpha.unix.Stream,unix.StdCLibraryFunctions -analyzer-output text \
+// RUN:   -analyzer-config unix.StdCLibraryFunctions:ModelPOSIX=true -verify=expected,stdargs %s
 
 #include "Inputs/system-header-simulator.h"
 
diff --git a/clang/test/Analysis/stream-stdlibraryfunctionargs.c b/clang/test/Analysis/stream-stdlibraryfunctionargs.c
index a14befde51..938901ec08 100644
--- a/clang/test/Analysis/stream-stdlibraryfunctionargs.c
+++ b/clang/test/Analysis/stream-stdlibraryfunctionargs.c
@@ -1,11 +1,11 @@
-// RUN: %clang_analyze_cc1 -analyzer-checker=core,alpha.unix.Stream,alpha.unix.StdCLibraryFunctions,debug.ExprInspection \
-// RUN:   -analyzer-config alpha.unix.StdCLibraryFunctions:ModelPOSIX=true -verify=stream,any %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,alpha.unix.Stream,unix.StdCLibraryFunctions,debug.ExprInspection \
+// RUN:   -analyzer-config unix.StdCLibraryFunctions:ModelPOSIX=true -verify=stream,any %s
 
 // RUN: %clang_analyze_cc1 -analyzer-checker=core,alpha.unix.Stream,debug.ExprInspection \
-// RUN:   -analyzer-config alpha.unix.StdCLibraryFunctions:ModelPOSIX=true -verify=stream,any %s
+// RUN:   -analyzer-config unix.StdCLibraryFunctions:ModelPOSIX=true -verify=stream,any %s
 
-// RUN: %clang_analyze_cc1 -analyzer-checker=core,alpha.unix.StdCLibraryFunctions,debug.ExprInspection \
-// RUN:   -analyzer-config alpha.unix.StdCLibraryFunctions:ModelPOSIX=true -verify=stdfunc,any %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,unix.StdCLibraryFunctions,debug.ExprInspection \
+// RUN:   -analyzer-config unix.StdCLibraryFunctions:ModelPOSIX=true -verify=stdfunc,any %s
 
 #include "Inputs/system-header-simulator.h"
 
diff --git a/clang/test/Analysis/weak-dependencies.c b/clang/test/Analysis/weak-dependencies.c
index 9946af8f4d..9d4b7b6def 100644
--- a/clang/test/Analysis/weak-dependencies.c
+++ b/clang/test/Analysis/weak-dependencies.c
@@ -1,5 +1,5 @@
 // RUN: %clang_analyze_cc1 %s -verify \
-// RUN:   -analyzer-checker=alpha.unix.StdCLibraryFunctions \
+// RUN:   -analyzer-checker=unix.StdCLibraryFunctions \
 // RUN:   -analyzer-checker=core
 
 typedef __typeof(sizeof(int)) size_t;
diff --git a/clang/test/C/C2x/n2359.c b/clang/test/C/C2x/n2359.c
index 3a6641fbea..dc38bab43c 100644
--- a/clang/test/C/C2x/n2359.c
+++ b/clang/test/C/C2x/n2359.c
@@ -34,3 +34,7 @@
 // expected-error@-1 {{"__STDC_VERSION_STDINT_H__ not defined"}}
 #endif
 
+#include <stdckdint.h>
+#ifndef __STDC_VERSION_STDCKDINT_H__
+#error "__STDC_VERSION_STDCKDINT_H__ not defined"
+#endif
diff --git a/clang/test/C/C2x/n2683.c b/clang/test/C/C2x/n2683.c
new file mode 100644
index 0000000000..0c666c5fd8
--- /dev/null
+++ b/clang/test/C/C2x/n2683.c
@@ -0,0 +1,51 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 3
+// RUN: %clang_cc1 -triple=x86_64 -verify -ffreestanding -std=c23 %s
+
+/* WG14 N2683: Clang 18
+ * Towards Integer Safety
+ */
+#include <stdckdint.h>
+#include <stddef.h>
+#include <stdint.h>
+
+void test_semantic() {
+  int64_t result64 = 0;
+  int32_t result32 = 0;
+  wchar_t wide_char = L'A'; // The ascii value of `A` is 65
+
+  bool flag_add = ckd_add(&result64, INT32_MAX, 1);
+  bool flag_sub = ckd_sub(&result32, INT32_MAX, -1);
+  bool flag_mul = ckd_mul(&result64, INT64_MAX, 1);
+
+  bool flag = ckd_add(&result64, wide_char, result32); // In C, wchar_t is a typedef to some integer type and is allowed.
+
+  // FIXME: add static_assert calls to check the resulting values for correctness
+  // once the constant expression interpreter is able to handle the checked arithmetic
+  // builtins in C. Currently, they're only a valid constant expression in C++ due to
+  // looking for an ICE in C. Also all values in the tests of n2683_2.c should be checked.
+}
+
+void test_invalid_input() {
+  _BitInt(33) a33 = 1;
+  char char_var = 'd'; // The ascii value of `d` is 100
+  bool bool_var = 1;
+  const int const_result = 0;
+  enum week{Mon, Tue, Wed};
+  enum week day = Mon;
+  int a = 100;
+  int b = 55;
+  int result = 10;
+  char plain_char[] = {U'牛'}; /* expected-warning {{implicit conversion from 'unsigned int' to 'char' changes value from 29275 to 91}}  */
+
+  // invalid operand argument
+  bool flag_no_bitint = ckd_add(&result, a33, a); /* expected-error {{operand argument to checked integer operation must be an integer type other than plain 'char', 'bool', bit-precise, or an enumeration ('_BitInt(33)' invalid)}} */
+  bool flag_no_bool = ckd_sub(&result, bool_var, b); /* expected-error {{operand argument to checked integer operation must be an integer type other than plain 'char', 'bool', bit-precise, or an enumeration ('bool' invalid)}} */
+  bool flag_no_char = ckd_mul(&result, char_var, a); /* expected-error {{operand argument to checked integer operation must be an integer type other than plain 'char', 'bool', bit-precise, or an enumeration ('char' invalid)}} */
+  bool flag_no_enum = ckd_mul(&result, day, b); /* expected-error {{operand argument to checked integer operation must be an integer type other than plain 'char', 'bool', bit-precise, or an enumeration ('enum week' invalid)}} */
+
+  // invalid result type
+  bool flag_nostr = ckd_sub(&plain_char, a, b); /* expected-error {{result argument to checked integer operation must be a pointer to a non-const integer type other than plain 'char', 'bool', bit-precise, or an enumeration ('char (*)[1]' invalid)}} */
+  bool flag_nobool = ckd_mul(&bool_var, a, b); /* expected-error {{result argument to checked integer operation must be a pointer to a non-const integer type other than plain 'char', 'bool', bit-precise, or an enumeration ('bool *' invalid)}} */
+  bool flag_noptr = ckd_add(result, a, b); /* expected-error {{result argument to checked integer operation must be a pointer to a non-const integer type other than plain 'char', 'bool', bit-precise, or an enumeration ('int' invalid)}} */
+  bool flag_noconst = ckd_sub(&const_result, a, b); /* expected-error {{result argument to checked integer operation must be a pointer to a non-const integer type other than plain 'char', 'bool', bit-precise, or an enumeration ('const int *' invalid)}} */
+}
diff --git a/clang/test/C/C2x/n2683_2.c b/clang/test/C/C2x/n2683_2.c
new file mode 100644
index 0000000000..247f6de8fe
--- /dev/null
+++ b/clang/test/C/C2x/n2683_2.c
@@ -0,0 +1,66 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 3
+// RUN: %clang_cc1 -triple=x86_64 -emit-llvm -o - -std=c23 %s | FileCheck %s
+
+#include <stdckdint.h>
+#include <stdint.h>
+
+// CHECK-LABEL: define dso_local void @test_add_overflow_to64(
+// CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[RESULT64:%.*]] = alloca i64, align 8
+// CHECK-NEXT:    [[FLAG_ADD:%.*]] = alloca i8, align 1
+// CHECK-NEXT:    store i64 0, ptr [[RESULT64]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 2147483647, i64 1)
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { i64, i1 } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { i64, i1 } [[TMP0]], 0
+// CHECK-NEXT:    store i64 [[TMP2]], ptr [[RESULT64]], align 8
+// CHECK-NEXT:    [[FROMBOOL:%.*]] = zext i1 [[TMP1]] to i8
+// CHECK-NEXT:    store i8 [[FROMBOOL]], ptr [[FLAG_ADD]], align 1
+// CHECK-NEXT:    ret void
+//
+void test_add_overflow_to64() {
+  int64_t result64 = 0;
+  bool flag_add = ckd_add(&result64, INT32_MAX, 1);
+}
+
+// CHECK-LABEL: define dso_local void @test_sub_overflow(
+// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[RESULT32:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[FLAG_SUB:%.*]] = alloca i8, align 1
+// CHECK-NEXT:    store i32 0, ptr [[RESULT32]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 2147483647, i32 -1)
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { i32, i1 } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { i32, i1 } [[TMP0]], 0
+// CHECK-NEXT:    store i32 [[TMP2]], ptr [[RESULT32]], align 4
+// CHECK-NEXT:    [[FROMBOOL:%.*]] = zext i1 [[TMP1]] to i8
+// CHECK-NEXT:    store i8 [[FROMBOOL]], ptr [[FLAG_SUB]], align 1
+// CHECK-NEXT:    ret void
+//
+void test_sub_overflow() {
+  int32_t result32 = 0;
+  bool flag_sub = ckd_sub(&result32, INT32_MAX, -1);
+}
+
+// CHECK-LABEL: define dso_local void @test_mul_normal(
+// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[RESULT:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[FLAG_MUL:%.*]] = alloca i8, align 1
+// CHECK-NEXT:    store i32 3, ptr [[A]], align 4
+// CHECK-NEXT:    store i32 0, ptr [[RESULT]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 [[TMP0]], i32 2)
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP1]], 0
+// CHECK-NEXT:    store i32 [[TMP3]], ptr [[RESULT]], align 4
+// CHECK-NEXT:    [[FROMBOOL:%.*]] = zext i1 [[TMP2]] to i8
+// CHECK-NEXT:    store i8 [[FROMBOOL]], ptr [[FLAG_MUL]], align 1
+// CHECK-NEXT:    ret void
+//
+void test_mul_normal() {
+  int a = 3;
+  int result = 0;
+  bool flag_mul = ckd_mul(&result, a, 2);
+}
diff --git a/clang/test/C/C2x/n3007.c b/clang/test/C/C2x/n3007.c
index 1fd20332ce..34ec419b71 100644
--- a/clang/test/C/C2x/n3007.c
+++ b/clang/test/C/C2x/n3007.c
@@ -3,6 +3,10 @@
 /* WG14 N3007: Yes
  * Type Inference for object definitions
  */
+void test_auto_int(void) {
+  auto int auto_int = 12;
+}
+
 void test_qualifiers(int x, const int y, int * restrict z) {
   const auto a = x;
   auto b = y;
diff --git a/clang/test/CXX/basic/basic.types/p10.cpp b/clang/test/CXX/basic/basic.types/p10.cpp
index d41b394bb6..a543f248e5 100644
--- a/clang/test/CXX/basic/basic.types/p10.cpp
+++ b/clang/test/CXX/basic/basic.types/p10.cpp
@@ -20,7 +20,7 @@ constexpr int f2(S &) { return 0; }
 
 struct BeingDefined;
 extern BeingDefined beingdefined;
-struct BeingDefined { 
+struct BeingDefined {
   static constexpr BeingDefined& t = beingdefined;
 };
 
@@ -136,11 +136,15 @@ struct ArrBad {
 };
 constexpr int f(ArrBad) { return 0; } // expected-error {{1st parameter type 'ArrBad' is not a literal type}}
 
-constexpr int arb(int n) {
-  int a[n]; // expected-error {{variable of non-literal type 'int[n]' cannot be defined in a constexpr function}}
+constexpr int arb(int n) { // expected-note {{declared here}}
+  int a[n]; // expected-error {{variable of non-literal type 'int[n]' cannot be defined in a constexpr function}} \
+               expected-warning {{variable length arrays in C++ are a Clang extension}} \
+               expected-note {{function parameter 'n' with unknown value cannot be used in a constant expression}}
 }
-// expected-warning@+1 {{variable length array folded to constant array as an extension}}
-constexpr long Overflow[(1 << 30) << 2]{}; // expected-warning {{requires 34 bits to represent}}
+constexpr long Overflow[(1 << 30) << 2]{}; // expected-warning {{requires 34 bits to represent}} \
+                                              expected-warning {{variable length array folded to constant array as an extension}} \
+                                              expected-warning {{variable length arrays in C++ are a Clang extension}} \
+                                              expected-note {{signed left shift discards bits}}
 
 namespace inherited_ctor {
   struct A { constexpr A(int); };
diff --git a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.typedef/p2-0x.cpp b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.typedef/p2-0x.cpp
index 5ef917fe35..0e9e4637ca 100644
--- a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.typedef/p2-0x.cpp
+++ b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.typedef/p2-0x.cpp
@@ -34,8 +34,10 @@ namespace IllegalSyntax {
 namespace VariableLengthArrays {
   using T = int[42]; // ok
 
-  int n = 32;
-  using T = int[n]; // expected-error {{variable length array declaration not allowed at file scope}}
+  int n = 32;       // expected-note {{declared here}}
+  using T = int[n]; // expected-error {{variable length array declaration not allowed at file scope}} \
+                       expected-warning {{variable length arrays in C++ are a Clang extension}} \
+                       expected-note {{read of non-const variable 'n' is not allowed in a constant expression}}
 
   const int m = 42;
   using U = int[m];
@@ -43,9 +45,11 @@ namespace VariableLengthArrays {
   using U = int; // expected-error {{type alias redefinition with different types ('int' vs 'int[42]')}}
 
   void f() {
-    int n = 42;
+    int n = 42; // expected-note {{declared here}}
     goto foo; // expected-error {{cannot jump}}
-    using T = int[n]; // expected-note {{bypasses initialization of VLA type alias}}
+    using T = int[n]; // expected-note {{bypasses initialization of VLA type alias}} \
+                         expected-warning {{variable length arrays in C++ are a Clang extension}} \
+                         expected-note {{read of non-const variable 'n' is not allowed in a constant expression}}
   foo: ;
   }
 }
diff --git a/clang/test/CXX/expr/expr.prim/expr.prim.lambda/p4.cpp b/clang/test/CXX/expr/expr.prim/expr.prim.lambda/p4.cpp
index ffac911249..d933f76500 100644
--- a/clang/test/CXX/expr/expr.prim/expr.prim.lambda/p4.cpp
+++ b/clang/test/CXX/expr/expr.prim/expr.prim.lambda/p4.cpp
@@ -44,21 +44,23 @@ X infer_X_return_type_2(X x) {
 }
 
 struct Incomplete; // expected-note 2{{forward declaration of 'Incomplete'}}
-void test_result_type(int N) {
+void test_result_type(int N) {  // expected-note {{declared here}}
   auto l1 = [] () -> Incomplete { }; // expected-error{{incomplete result type 'Incomplete' in lambda expression}}
 
-  typedef int vla[N];
+  typedef int vla[N]; // expected-warning {{variable length arrays in C++ are a Clang extension}} \
+                         expected-note {{function parameter 'N' with unknown value cannot be used in a constant expression}}
   auto l2 = [] () -> vla { }; // expected-error{{function cannot return array type 'vla' (aka 'int[N]')}}
 }
 
 template <typename T>
-void test_result_type_tpl(int N) {
+void test_result_type_tpl(int N) { // expected-note 2{{declared here}}
   auto l1 = []() -> T {}; // expected-error{{incomplete result type 'Incomplete' in lambda expression}}
                           // expected-note@-1{{while substituting into a lambda expression here}}
-  typedef int vla[N];
+  typedef int vla[N]; // expected-warning 2{{variable length arrays in C++ are a Clang extension}} \
+                         expected-note 2{{function parameter 'N' with unknown value cannot be used in a constant expression}}
   auto l2 = []() -> vla {}; // expected-error{{function cannot return array type 'vla' (aka 'int[N]')}}
 }
 
 void test_result_type_call() {
-  test_result_type_tpl<Incomplete>(10); // expected-note {{requested here}}
+  test_result_type_tpl<Incomplete>(10); // expected-note 2{{requested here}}
 }
diff --git a/clang/test/CXX/over/over.match/over.match.funcs/over.match.oper/p3-2a.cpp b/clang/test/CXX/over/over.match/over.match.funcs/over.match.oper/p3-2a.cpp
index 5c6804eb77..5727d506fe 100644
--- a/clang/test/CXX/over/over.match/over.match.funcs/over.match.oper/p3-2a.cpp
+++ b/clang/test/CXX/over/over.match/over.match.funcs/over.match.oper/p3-2a.cpp
@@ -324,6 +324,67 @@ bool x = X() == X(); // expected-warning {{ambiguous}}
 }
 } // namespace P2468R2
 
+namespace GH53954{
+namespace friend_template_1 {
+struct P {
+    template <class T>
+    friend bool operator==(const P&, const T&); // expected-note {{candidate}} \
+                                                // expected-note {{ambiguous candidate function with reversed arguments}}
+};
+struct A : public P {};
+struct B : public P {};
+bool check(A a, B b) { return a == b; } // expected-warning {{use of overloaded operator '==' (with operand types 'A' and 'B') to be ambiguous}}
+}
+
+namespace friend_template_2 {
+struct P {
+    template <class T>
+    friend bool operator==(const T&, const P&); // expected-note {{candidate}} \
+                                                // expected-note {{ambiguous candidate function with reversed arguments}}
+};
+struct A : public P {};
+struct B : public P {};
+bool check(A a, B b) { return a == b; } // expected-warning {{use of overloaded operator '==' (with operand types 'A' and 'B') to be ambiguous}}
+}
+
+namespace member_template {
+struct P {
+  template<class S>
+  bool operator==(const S &) const; // expected-note {{candidate}} \
+                                    // expected-note {{ambiguous candidate function with reversed arguments}}
+};
+struct A : public P {};
+struct B : public P {};
+bool check(A a, B b) { return a == b; } // expected-warning {{use of overloaded operator '==' (with operand types 'A' and 'B') to be ambiguous}}
+}
+
+namespace non_member_template_1 {
+struct P {};
+template<class S>
+bool operator==(const P&, const S &); // expected-note {{candidate}} \
+                                      // expected-note {{ambiguous candidate function with reversed arguments}}
+
+struct A : public P {};
+struct B : public P {};
+bool check(A a, B b) { return a == b; } // expected-warning {{use of overloaded operator '==' (with operand types 'A' and 'B') to be ambiguous}}
+
+template<class S>
+bool operator!=(const P&, const S &);
+bool fine(A a, B b) { return a == b; } // Ok. Found a matching operator!=.
+}
+}
+
+namespace non_member_template_2 {
+struct P {};
+template<class S>
+bool operator==(const S&, const P&); // expected-note {{candidate}} \
+                                     // expected-note {{ambiguous candidate function with reversed arguments}}
+
+struct A : public P {};
+struct B : public P {};
+bool check(A a, B b) { return a == b; } // expected-warning {{use of overloaded operator '==' (with operand types 'A' and 'B') to be ambiguous}}
+}
+
 #else // NO_ERRORS
 
 namespace problem_cases {
diff --git a/clang/test/CXX/temp/temp.arg/temp.arg.type/p2.cpp b/clang/test/CXX/temp/temp.arg/temp.arg.type/p2.cpp
index 539baecdb6..1e314da313 100644
--- a/clang/test/CXX/temp/temp.arg/temp.arg.type/p2.cpp
+++ b/clang/test/CXX/temp/temp.arg/temp.arg.type/p2.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -fsyntax-only -verify %s
+// RUN: %clang_cc1 -fsyntax-only -verify -Wvla %s
 // RUN: %clang_cc1 -fsyntax-only -verify -std=c++98 %s
 // RUN: %clang_cc1 -fsyntax-only -verify -std=c++11 %s
 
@@ -19,13 +19,17 @@ enum {e};
 // expected-note@-2 {{unnamed type used in template argument was declared here}}
 #endif
 
-void test_f0(int n) {
+void test_f0(int n) { // #here
   int i = f0(0, e);
 #if __cplusplus <= 199711L
   // expected-warning@-2 {{template argument uses unnamed type}}
 #endif
 
-  int vla[n];
+  int vla[n]; // expected-warning {{variable length arrays in C++ are a Clang extension}}
+#if __cplusplus > 199711L
+  // expected-note@-2 {{function parameter 'n' with unknown value cannot be used in a constant expression}}
+  // expected-note@#here {{declared here}}
+#endif
   f0(0, vla); // expected-error{{no matching function for call to 'f0'}}
 }
 
diff --git a/clang/test/CXX/temp/temp.decls/temp.variadic/p5.cpp b/clang/test/CXX/temp/temp.decls/temp.variadic/p5.cpp
index 10af03eeaf..7536339c14 100644
--- a/clang/test/CXX/temp/temp.decls/temp.variadic/p5.cpp
+++ b/clang/test/CXX/temp/temp.decls/temp.variadic/p5.cpp
@@ -27,12 +27,12 @@ struct ExpansionLengthMismatch {
   };
 };
 
-ExpansionLengthMismatch<int, long>::Inner<unsigned int, unsigned long>::type 
+ExpansionLengthMismatch<int, long>::Inner<unsigned int, unsigned long>::type
   *il_pairs;
 tuple<pair<int, unsigned int>, pair<long, unsigned long> >*il_pairs_2 = il_pairs;
 
 ExpansionLengthMismatch<short, int, long>::Inner<unsigned int, unsigned long>::type // expected-note{{in instantiation of template class 'ExpansionLengthMismatch<short, int, long>::Inner<unsigned int, unsigned long>' requested here}}
-  *il_pairs_bad; 
+  *il_pairs_bad;
 
 
 // An appearance of a name of a parameter pack that is not expanded is
@@ -40,7 +40,7 @@ ExpansionLengthMismatch<short, int, long>::Inner<unsigned int, unsigned long>::t
 
 // Test for unexpanded parameter packs in each of the type nodes.
 template<typename T, int N, typename ... Types>
-struct TestPPName 
+struct TestPPName
   : public Types, public T  // expected-error{{base type contains unexpanded parameter pack 'Types'}}
 {
   // BuiltinType is uninteresting
@@ -60,33 +60,35 @@ struct TestPPName
 
   // MemberPointerType
   typedef Types TestPPName::* member_pointer_1; // expected-error{{declaration type contains unexpanded parameter pack 'Types'}}
-  typedef int Types::*member_pointer_2; // expected-error{{declaration type contains unexpanded parameter pack 'Types'}} 
+  typedef int Types::*member_pointer_2; // expected-error{{declaration type contains unexpanded parameter pack 'Types'}}
 
   // ConstantArrayType
-  typedef Types constant_array[17]; // expected-error{{declaration type contains unexpanded parameter pack 'Types'}} 
+  typedef Types constant_array[17]; // expected-error{{declaration type contains unexpanded parameter pack 'Types'}}
 
   // IncompleteArrayType
-  typedef Types incomplete_array[]; // expected-error{{declaration type contains unexpanded parameter pack 'Types'}} 
+  typedef Types incomplete_array[]; // expected-error{{declaration type contains unexpanded parameter pack 'Types'}}
 
   // VariableArrayType
-  void f(int i) {
-    Types variable_array[i]; // expected-error{{declaration type contains unexpanded parameter pack 'Types'}} 
+  void f(int i) {            // expected-note {{declared here}}
+    Types variable_array[i]; // expected-error{{declaration type contains unexpanded parameter pack 'Types'}} \
+                                expected-warning {{variable length arrays in C++ are a Clang extension}} \
+                                expected-note {{function parameter 'i' with unknown value cannot be used in a constant expression}}
   }
 
   // DependentSizedArrayType
-  typedef Types dependent_sized_array[N]; // expected-error{{declaration type contains unexpanded parameter pack 'Types'}} 
+  typedef Types dependent_sized_array[N]; // expected-error{{declaration type contains unexpanded parameter pack 'Types'}}
 
   // DependentSizedExtVectorType
-  typedef Types dependent_sized_ext_vector __attribute__((ext_vector_type(N))); // expected-error{{declaration type contains unexpanded parameter pack 'Types'}} 
+  typedef Types dependent_sized_ext_vector __attribute__((ext_vector_type(N))); // expected-error{{declaration type contains unexpanded parameter pack 'Types'}}
 
   // VectorType is uninteresting
 
   // ExtVectorType
-  typedef Types ext_vector __attribute__((ext_vector_type(4))); // expected-error{{declaration type contains unexpanded parameter pack 'Types'}} 
+  typedef Types ext_vector __attribute__((ext_vector_type(4))); // expected-error{{declaration type contains unexpanded parameter pack 'Types'}}
 
   // FunctionProtoType
-  typedef Types (function_type_1)(int); // expected-error{{declaration type contains unexpanded parameter pack 'Types'}} 
-  typedef int (function_type_2)(Types); // expected-error{{declaration type contains unexpanded parameter pack 'Types'}} 
+  typedef Types (function_type_1)(int); // expected-error{{declaration type contains unexpanded parameter pack 'Types'}}
+  typedef int (function_type_2)(Types); // expected-error{{declaration type contains unexpanded parameter pack 'Types'}}
 
   // FunctionNoProtoType is uninteresting
   // UnresolvedUsingType is uninteresting
@@ -94,34 +96,34 @@ struct TestPPName
   // TypedefType is uninteresting
 
   // TypeOfExprType
-  typedef __typeof__((static_cast<Types>(0))) typeof_expr; // expected-error{{declaration type contains unexpanded parameter pack 'Types'}} 
+  typedef __typeof__((static_cast<Types>(0))) typeof_expr; // expected-error{{declaration type contains unexpanded parameter pack 'Types'}}
 
   // TypeOfType
-  typedef __typeof__(Types) typeof_type;  // expected-error{{declaration type contains unexpanded parameter pack 'Types'}} 
+  typedef __typeof__(Types) typeof_type;  // expected-error{{declaration type contains unexpanded parameter pack 'Types'}}
 
   // DecltypeType
-  typedef decltype((static_cast<Types>(0))) typeof_expr; // expected-error{{declaration type contains unexpanded parameter pack 'Types'}} 
+  typedef decltype((static_cast<Types>(0))) typeof_expr; // expected-error{{declaration type contains unexpanded parameter pack 'Types'}}
 
   // RecordType is uninteresting
   // EnumType is uninteresting
   // ElaboratedType is uninteresting
 
   // TemplateTypeParmType
-  typedef Types template_type_parm; // expected-error{{declaration type contains unexpanded parameter pack 'Types'}} 
+  typedef Types template_type_parm; // expected-error{{declaration type contains unexpanded parameter pack 'Types'}}
 
   // SubstTemplateTypeParmType is uninteresting
 
   // TemplateSpecializationType
-  typedef pair<Types, int> template_specialization; // expected-error{{declaration type contains unexpanded parameter pack 'Types'}} 
+  typedef pair<Types, int> template_specialization; // expected-error{{declaration type contains unexpanded parameter pack 'Types'}}
 
   // InjectedClassName is uninteresting.
 
   // DependentNameType
-  typedef typename Types::type dependent_name; // expected-error{{declaration type contains unexpanded parameter pack 'Types'}} 
+  typedef typename Types::type dependent_name; // expected-error{{declaration type contains unexpanded parameter pack 'Types'}}
 
   // DependentTemplateSpecializationType
-  typedef typename Types::template apply<int> dependent_name_1; // expected-error{{declaration type contains unexpanded parameter pack 'Types'}} 
-  typedef typename T::template apply<Types> dependent_name_2; // expected-error{{declaration type contains unexpanded parameter pack 'Types'}} 
+  typedef typename Types::template apply<int> dependent_name_1; // expected-error{{declaration type contains unexpanded parameter pack 'Types'}}
+  typedef typename T::template apply<Types> dependent_name_2; // expected-error{{declaration type contains unexpanded parameter pack 'Types'}}
 
   // ObjCObjectType is uninteresting
   // ObjCInterfaceType is uninteresting
@@ -194,7 +196,7 @@ struct alignas(Types) TestUnexpandedDecls : T{ // expected-error{{expression con
   }
 
   T in_class_member_init = static_cast<Types>(0); // expected-error{{initializer contains unexpanded parameter pack 'Types'}}
-  TestUnexpandedDecls() : 
+  TestUnexpandedDecls() :
     Types(static_cast<Types>(0)), // expected-error{{initializer contains unexpanded parameter pack 'Types'}}
     Types(static_cast<Types>(0))...,
     in_class_member_init(static_cast<Types>(0)) {} // expected-error{{initializer contains unexpanded parameter pack 'Types'}}
@@ -202,7 +204,7 @@ struct alignas(Types) TestUnexpandedDecls : T{ // expected-error{{expression con
   void default_function_args(T = static_cast<Types>(0)); // expected-error{{default argument contains unexpanded parameter pack 'Types'}}
 
   template<typename = Types*> // expected-error{{default argument contains unexpanded parameter pack 'Types'}}
-    struct default_template_args_1; 
+    struct default_template_args_1;
   template<int = static_cast<Types>(0)> // expected-error{{default argument contains unexpanded parameter pack 'Types'}}
     struct default_template_args_2;
   template<template<typename> class = Types::template apply> // expected-error{{default argument contains unexpanded parameter pack 'Types'}}
@@ -390,13 +392,13 @@ struct MemberTemplatePPNames {
 
 // Example from working paper
 namespace WorkingPaperExample {
-  template<typename...> struct Tuple {}; 
+  template<typename...> struct Tuple {};
   template<typename T1, typename T2> struct Pair {};
-  
-  template<class ... Args1> struct zip { 
+
+  template<class ... Args1> struct zip {
     template<class ... Args2> struct with {
       typedef Tuple<Pair<Args1, Args2> ... > type; // expected-error{{pack expansion contains parameter packs 'Args1' and 'Args2' that have different lengths (1 vs. 2)}}
-    }; 
+    };
   };
 
   typedef zip<short, int>::with<unsigned short, unsigned>::type T1; // T1 is Tuple<Pair<short, unsigned short>, Pair<int, unsigned>>
@@ -407,9 +409,9 @@ namespace WorkingPaperExample {
   template<class ... Args> void f(Args...);
   template<class ... Args> void h(Args...);
 
-  template<class ... Args> 
+  template<class ... Args>
   void g(Args ... args) {
-    f(const_cast<const Args*>(&args)...); // OK: "Args" and "args" are expanded within f 
+    f(const_cast<const Args*>(&args)...); // OK: "Args" and "args" are expanded within f
     f(5 ...); // expected-error{{pack expansion does not contain any unexpanded parameter packs}}
     f(args); // expected-error{{expression contains unexpanded parameter pack 'args'}}
     f(h(args ...) + args ...);
diff --git a/clang/test/CodeGen/PowerPC/builtins-ppc-pair-mma.c b/clang/test/CodeGen/PowerPC/builtins-ppc-pair-mma.c
index 95e28da9b0..3922513e22 100644
--- a/clang/test/CodeGen/PowerPC/builtins-ppc-pair-mma.c
+++ b/clang/test/CodeGen/PowerPC/builtins-ppc-pair-mma.c
@@ -4,7 +4,7 @@
 // RUN: %clang_cc1 -O3 -triple powerpc64-unknown-unknown -target-cpu pwr10 \
 // RUN: -emit-llvm %s -o - | FileCheck %s
 
-// CHECK-LABEL: define {{[^@]+}}@test1(
+// CHECK-LABEL: @test1(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <512 x i1> @llvm.ppc.mma.assemble.acc(<16 x i8> [[VC:%.*]], <16 x i8> [[VC]], <16 x i8> [[VC]], <16 x i8> [[VC]])
 // CHECK-NEXT:    store <512 x i1> [[TMP0]], ptr [[RESP:%.*]], align 64, !tbaa [[TBAA2:![0-9]+]]
@@ -18,7 +18,7 @@ void test1(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, unsi
   *((__vector_quad *)resp) = res;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test2(
+// CHECK-LABEL: @test2(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @llvm.ppc.mma.disassemble.acc(<512 x i1> [[TMP0]])
@@ -39,7 +39,7 @@ void test2(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, unsi
   __builtin_mma_disassemble_acc(resp, (__vector_quad*)vqp);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test3(
+// CHECK-LABEL: @test3(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.assemble.pair(<16 x i8> [[VC:%.*]], <16 x i8> [[VC]])
 // CHECK-NEXT:    store <256 x i1> [[TMP0]], ptr [[RESP:%.*]], align 32, !tbaa [[TBAA6:![0-9]+]]
@@ -53,7 +53,7 @@ void test3(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, unsi
   *((__vector_pair *)resp) = res;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test4(
+// CHECK-LABEL: @test4(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <256 x i1>, ptr [[VPP:%.*]], align 32
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <16 x i8>, <16 x i8> } @llvm.ppc.vsx.disassemble.pair(<256 x i1> [[TMP0]])
@@ -68,7 +68,7 @@ void test4(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, unsi
   __builtin_vsx_disassemble_pair(resp, (__vector_pair*)vpp);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test5(
+// CHECK-LABEL: @test5(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.xxmtacc(<512 x i1> [[TMP0]])
@@ -82,7 +82,7 @@ void test5(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, unsi
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test6(
+// CHECK-LABEL: @test6(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.xxmfacc(<512 x i1> [[TMP0]])
@@ -96,7 +96,7 @@ void test6(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, unsi
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test7(
+// CHECK-LABEL: @test7(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <512 x i1> @llvm.ppc.mma.xxsetaccz()
 // CHECK-NEXT:    store <512 x i1> [[TMP0]], ptr [[RESP:%.*]], align 64, !tbaa [[TBAA2]]
@@ -109,7 +109,7 @@ void test7(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, unsi
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test8(
+// CHECK-LABEL: @test8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <512 x i1> @llvm.ppc.mma.xvi4ger8(<16 x i8> [[VC:%.*]], <16 x i8> [[VC]])
 // CHECK-NEXT:    store <512 x i1> [[TMP0]], ptr [[RESP:%.*]], align 64, !tbaa [[TBAA2]]
@@ -122,7 +122,7 @@ void test8(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, unsi
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test9(
+// CHECK-LABEL: @test9(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <512 x i1> @llvm.ppc.mma.xvi8ger4(<16 x i8> [[VC:%.*]], <16 x i8> [[VC]])
 // CHECK-NEXT:    store <512 x i1> [[TMP0]], ptr [[RESP:%.*]], align 64, !tbaa [[TBAA2]]
@@ -135,7 +135,7 @@ void test9(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, unsi
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test10(
+// CHECK-LABEL: @test10(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <512 x i1> @llvm.ppc.mma.xvi16ger2(<16 x i8> [[VC:%.*]], <16 x i8> [[VC]])
 // CHECK-NEXT:    store <512 x i1> [[TMP0]], ptr [[RESP:%.*]], align 64, !tbaa [[TBAA2]]
@@ -148,7 +148,7 @@ void test10(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test11(
+// CHECK-LABEL: @test11(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <512 x i1> @llvm.ppc.mma.xvi16ger2s(<16 x i8> [[VC:%.*]], <16 x i8> [[VC]])
 // CHECK-NEXT:    store <512 x i1> [[TMP0]], ptr [[RESP:%.*]], align 64, !tbaa [[TBAA2]]
@@ -161,7 +161,7 @@ void test11(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test12(
+// CHECK-LABEL: @test12(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <512 x i1> @llvm.ppc.mma.xvf16ger2(<16 x i8> [[VC:%.*]], <16 x i8> [[VC]])
 // CHECK-NEXT:    store <512 x i1> [[TMP0]], ptr [[RESP:%.*]], align 64, !tbaa [[TBAA2]]
@@ -174,7 +174,7 @@ void test12(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test13(
+// CHECK-LABEL: @test13(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <512 x i1> @llvm.ppc.mma.xvf32ger(<16 x i8> [[VC:%.*]], <16 x i8> [[VC]])
 // CHECK-NEXT:    store <512 x i1> [[TMP0]], ptr [[RESP:%.*]], align 64, !tbaa [[TBAA2]]
@@ -187,7 +187,7 @@ void test13(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test14(
+// CHECK-LABEL: @test14(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <256 x i1>, ptr [[VPP:%.*]], align 32, !tbaa [[TBAA6]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.xvf64ger(<256 x i1> [[TMP0]], <16 x i8> [[VC:%.*]])
@@ -201,7 +201,7 @@ void test14(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test15(
+// CHECK-LABEL: @test15(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <512 x i1> @llvm.ppc.mma.pmxvi4ger8(<16 x i8> [[VC:%.*]], <16 x i8> [[VC]], i32 0, i32 0, i32 0)
 // CHECK-NEXT:    store <512 x i1> [[TMP0]], ptr [[RESP:%.*]], align 64, !tbaa [[TBAA2]]
@@ -214,7 +214,7 @@ void test15(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test16(
+// CHECK-LABEL: @test16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <512 x i1> @llvm.ppc.mma.pmxvi8ger4(<16 x i8> [[VC:%.*]], <16 x i8> [[VC]], i32 0, i32 0, i32 0)
 // CHECK-NEXT:    store <512 x i1> [[TMP0]], ptr [[RESP:%.*]], align 64, !tbaa [[TBAA2]]
@@ -227,7 +227,7 @@ void test16(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test17(
+// CHECK-LABEL: @test17(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <512 x i1> @llvm.ppc.mma.pmxvi16ger2(<16 x i8> [[VC:%.*]], <16 x i8> [[VC]], i32 0, i32 0, i32 0)
 // CHECK-NEXT:    store <512 x i1> [[TMP0]], ptr [[RESP:%.*]], align 64, !tbaa [[TBAA2]]
@@ -240,7 +240,7 @@ void test17(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test18(
+// CHECK-LABEL: @test18(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <512 x i1> @llvm.ppc.mma.pmxvi16ger2s(<16 x i8> [[VC:%.*]], <16 x i8> [[VC]], i32 0, i32 0, i32 0)
 // CHECK-NEXT:    store <512 x i1> [[TMP0]], ptr [[RESP:%.*]], align 64, !tbaa [[TBAA2]]
@@ -253,7 +253,7 @@ void test18(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test19(
+// CHECK-LABEL: @test19(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <512 x i1> @llvm.ppc.mma.pmxvf16ger2(<16 x i8> [[VC:%.*]], <16 x i8> [[VC]], i32 0, i32 0, i32 0)
 // CHECK-NEXT:    store <512 x i1> [[TMP0]], ptr [[RESP:%.*]], align 64, !tbaa [[TBAA2]]
@@ -266,7 +266,7 @@ void test19(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test20(
+// CHECK-LABEL: @test20(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <512 x i1> @llvm.ppc.mma.pmxvf32ger(<16 x i8> [[VC:%.*]], <16 x i8> [[VC]], i32 0, i32 0)
 // CHECK-NEXT:    store <512 x i1> [[TMP0]], ptr [[RESP:%.*]], align 64, !tbaa [[TBAA2]]
@@ -279,7 +279,7 @@ void test20(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test21(
+// CHECK-LABEL: @test21(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <256 x i1>, ptr [[VPP:%.*]], align 32, !tbaa [[TBAA6]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.pmxvf64ger(<256 x i1> [[TMP0]], <16 x i8> [[VC:%.*]], i32 0, i32 0)
@@ -293,7 +293,7 @@ void test21(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test22(
+// CHECK-LABEL: @test22(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.xvi4ger8pp(<512 x i1> [[TMP0]], <16 x i8> [[VC:%.*]], <16 x i8> [[VC]])
@@ -307,7 +307,7 @@ void test22(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test23(
+// CHECK-LABEL: @test23(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.xvi8ger4pp(<512 x i1> [[TMP0]], <16 x i8> [[VC:%.*]], <16 x i8> [[VC]])
@@ -321,7 +321,7 @@ void test23(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test24(
+// CHECK-LABEL: @test24(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.xvi8ger4spp(<512 x i1> [[TMP0]], <16 x i8> [[VC:%.*]], <16 x i8> [[VC]])
@@ -335,7 +335,7 @@ void test24(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test25(
+// CHECK-LABEL: @test25(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.xvi16ger2pp(<512 x i1> [[TMP0]], <16 x i8> [[VC:%.*]], <16 x i8> [[VC]])
@@ -349,7 +349,7 @@ void test25(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test26(
+// CHECK-LABEL: @test26(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.xvi16ger2spp(<512 x i1> [[TMP0]], <16 x i8> [[VC:%.*]], <16 x i8> [[VC]])
@@ -363,7 +363,7 @@ void test26(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test27(
+// CHECK-LABEL: @test27(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.pmxvi4ger8pp(<512 x i1> [[TMP0]], <16 x i8> [[VC:%.*]], <16 x i8> [[VC]], i32 0, i32 0, i32 0)
@@ -377,7 +377,7 @@ void test27(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test28(
+// CHECK-LABEL: @test28(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.pmxvi8ger4pp(<512 x i1> [[TMP0]], <16 x i8> [[VC:%.*]], <16 x i8> [[VC]], i32 0, i32 0, i32 0)
@@ -391,7 +391,7 @@ void test28(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test29(
+// CHECK-LABEL: @test29(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.pmxvi8ger4spp(<512 x i1> [[TMP0]], <16 x i8> [[VC:%.*]], <16 x i8> [[VC]], i32 0, i32 0, i32 0)
@@ -405,7 +405,7 @@ void test29(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test30(
+// CHECK-LABEL: @test30(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.pmxvi16ger2pp(<512 x i1> [[TMP0]], <16 x i8> [[VC:%.*]], <16 x i8> [[VC]], i32 0, i32 0, i32 0)
@@ -419,7 +419,7 @@ void test30(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test31(
+// CHECK-LABEL: @test31(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.pmxvi16ger2spp(<512 x i1> [[TMP0]], <16 x i8> [[VC:%.*]], <16 x i8> [[VC]], i32 0, i32 0, i32 0)
@@ -433,7 +433,7 @@ void test31(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test32(
+// CHECK-LABEL: @test32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.xvf16ger2pp(<512 x i1> [[TMP0]], <16 x i8> [[VC:%.*]], <16 x i8> [[VC]])
@@ -447,7 +447,7 @@ void test32(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test33(
+// CHECK-LABEL: @test33(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.xvf16ger2pn(<512 x i1> [[TMP0]], <16 x i8> [[VC:%.*]], <16 x i8> [[VC]])
@@ -461,7 +461,7 @@ void test33(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test34(
+// CHECK-LABEL: @test34(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.xvf16ger2np(<512 x i1> [[TMP0]], <16 x i8> [[VC:%.*]], <16 x i8> [[VC]])
@@ -475,7 +475,7 @@ void test34(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test35(
+// CHECK-LABEL: @test35(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.xvf16ger2nn(<512 x i1> [[TMP0]], <16 x i8> [[VC:%.*]], <16 x i8> [[VC]])
@@ -489,7 +489,7 @@ void test35(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test36(
+// CHECK-LABEL: @test36(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.pmxvf16ger2pp(<512 x i1> [[TMP0]], <16 x i8> [[VC:%.*]], <16 x i8> [[VC]], i32 0, i32 0, i32 0)
@@ -503,7 +503,7 @@ void test36(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test37(
+// CHECK-LABEL: @test37(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.pmxvf16ger2pn(<512 x i1> [[TMP0]], <16 x i8> [[VC:%.*]], <16 x i8> [[VC]], i32 0, i32 0, i32 0)
@@ -517,7 +517,7 @@ void test37(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test38(
+// CHECK-LABEL: @test38(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.pmxvf16ger2np(<512 x i1> [[TMP0]], <16 x i8> [[VC:%.*]], <16 x i8> [[VC]], i32 0, i32 0, i32 0)
@@ -531,7 +531,7 @@ void test38(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test39(
+// CHECK-LABEL: @test39(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.pmxvf16ger2nn(<512 x i1> [[TMP0]], <16 x i8> [[VC:%.*]], <16 x i8> [[VC]], i32 0, i32 0, i32 0)
@@ -545,7 +545,7 @@ void test39(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test40(
+// CHECK-LABEL: @test40(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.xvf32gerpp(<512 x i1> [[TMP0]], <16 x i8> [[VC:%.*]], <16 x i8> [[VC]])
@@ -559,7 +559,7 @@ void test40(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test41(
+// CHECK-LABEL: @test41(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.xvf32gerpn(<512 x i1> [[TMP0]], <16 x i8> [[VC:%.*]], <16 x i8> [[VC]])
@@ -573,7 +573,7 @@ void test41(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test42(
+// CHECK-LABEL: @test42(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.xvf32gernp(<512 x i1> [[TMP0]], <16 x i8> [[VC:%.*]], <16 x i8> [[VC]])
@@ -587,7 +587,7 @@ void test42(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test43(
+// CHECK-LABEL: @test43(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.xvf32gernn(<512 x i1> [[TMP0]], <16 x i8> [[VC:%.*]], <16 x i8> [[VC]])
@@ -601,7 +601,7 @@ void test43(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test44(
+// CHECK-LABEL: @test44(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.pmxvf32gerpp(<512 x i1> [[TMP0]], <16 x i8> [[VC:%.*]], <16 x i8> [[VC]], i32 0, i32 0)
@@ -615,7 +615,7 @@ void test44(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test45(
+// CHECK-LABEL: @test45(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.pmxvf32gerpn(<512 x i1> [[TMP0]], <16 x i8> [[VC:%.*]], <16 x i8> [[VC]], i32 0, i32 0)
@@ -629,7 +629,7 @@ void test45(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test46(
+// CHECK-LABEL: @test46(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.pmxvf32gernp(<512 x i1> [[TMP0]], <16 x i8> [[VC:%.*]], <16 x i8> [[VC]], i32 0, i32 0)
@@ -643,7 +643,7 @@ void test46(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test47(
+// CHECK-LABEL: @test47(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.pmxvf32gernn(<512 x i1> [[TMP0]], <16 x i8> [[VC:%.*]], <16 x i8> [[VC]], i32 0, i32 0)
@@ -657,7 +657,7 @@ void test47(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test48(
+// CHECK-LABEL: @test48(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = load <256 x i1>, ptr [[VPP:%.*]], align 32, !tbaa [[TBAA6]]
@@ -672,7 +672,7 @@ void test48(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test49(
+// CHECK-LABEL: @test49(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = load <256 x i1>, ptr [[VPP:%.*]], align 32, !tbaa [[TBAA6]]
@@ -687,7 +687,7 @@ void test49(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test50(
+// CHECK-LABEL: @test50(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = load <256 x i1>, ptr [[VPP:%.*]], align 32, !tbaa [[TBAA6]]
@@ -702,7 +702,7 @@ void test50(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test51(
+// CHECK-LABEL: @test51(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = load <256 x i1>, ptr [[VPP:%.*]], align 32, !tbaa [[TBAA6]]
@@ -717,7 +717,7 @@ void test51(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test52(
+// CHECK-LABEL: @test52(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = load <256 x i1>, ptr [[VPP:%.*]], align 32, !tbaa [[TBAA6]]
@@ -732,7 +732,7 @@ void test52(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test53(
+// CHECK-LABEL: @test53(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = load <256 x i1>, ptr [[VPP:%.*]], align 32, !tbaa [[TBAA6]]
@@ -747,7 +747,7 @@ void test53(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test54(
+// CHECK-LABEL: @test54(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = load <256 x i1>, ptr [[VPP:%.*]], align 32, !tbaa [[TBAA6]]
@@ -762,7 +762,7 @@ void test54(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test55(
+// CHECK-LABEL: @test55(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = load <256 x i1>, ptr [[VPP:%.*]], align 32, !tbaa [[TBAA6]]
@@ -777,7 +777,7 @@ void test55(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test56(
+// CHECK-LABEL: @test56(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <512 x i1> @llvm.ppc.mma.xvbf16ger2(<16 x i8> [[VC:%.*]], <16 x i8> [[VC]])
 // CHECK-NEXT:    store <512 x i1> [[TMP0]], ptr [[RESP:%.*]], align 64, !tbaa [[TBAA2]]
@@ -790,7 +790,7 @@ void test56(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test57(
+// CHECK-LABEL: @test57(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <512 x i1> @llvm.ppc.mma.pmxvbf16ger2(<16 x i8> [[VC:%.*]], <16 x i8> [[VC]], i32 0, i32 0, i32 0)
 // CHECK-NEXT:    store <512 x i1> [[TMP0]], ptr [[RESP:%.*]], align 64, !tbaa [[TBAA2]]
@@ -803,7 +803,7 @@ void test57(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test58(
+// CHECK-LABEL: @test58(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.xvbf16ger2pp(<512 x i1> [[TMP0]], <16 x i8> [[VC:%.*]], <16 x i8> [[VC]])
@@ -817,7 +817,7 @@ void test58(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test59(
+// CHECK-LABEL: @test59(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.xvbf16ger2pn(<512 x i1> [[TMP0]], <16 x i8> [[VC:%.*]], <16 x i8> [[VC]])
@@ -831,7 +831,7 @@ void test59(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test60(
+// CHECK-LABEL: @test60(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.xvbf16ger2np(<512 x i1> [[TMP0]], <16 x i8> [[VC:%.*]], <16 x i8> [[VC]])
@@ -845,7 +845,7 @@ void test60(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test61(
+// CHECK-LABEL: @test61(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.xvbf16ger2nn(<512 x i1> [[TMP0]], <16 x i8> [[VC:%.*]], <16 x i8> [[VC]])
@@ -859,7 +859,7 @@ void test61(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test62(
+// CHECK-LABEL: @test62(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.pmxvbf16ger2pp(<512 x i1> [[TMP0]], <16 x i8> [[VC:%.*]], <16 x i8> [[VC]], i32 0, i32 0, i32 0)
@@ -873,7 +873,7 @@ void test62(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test63(
+// CHECK-LABEL: @test63(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.pmxvbf16ger2pn(<512 x i1> [[TMP0]], <16 x i8> [[VC:%.*]], <16 x i8> [[VC]], i32 0, i32 0, i32 0)
@@ -887,7 +887,7 @@ void test63(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test64(
+// CHECK-LABEL: @test64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.pmxvbf16ger2np(<512 x i1> [[TMP0]], <16 x i8> [[VC:%.*]], <16 x i8> [[VC]], i32 0, i32 0, i32 0)
@@ -901,7 +901,7 @@ void test64(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test65(
+// CHECK-LABEL: @test65(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <512 x i1> @llvm.ppc.mma.pmxvbf16ger2nn(<512 x i1> [[TMP0]], <16 x i8> [[VC:%.*]], <16 x i8> [[VC]], i32 0, i32 0, i32 0)
@@ -915,7 +915,7 @@ void test65(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test66(
+// CHECK-LABEL: @test66(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr [[VPP:%.*]])
 // CHECK-NEXT:    tail call void @llvm.ppc.vsx.stxvp(<256 x i1> [[TMP0]], ptr [[VP2:%.*]])
@@ -926,7 +926,7 @@ void test66(const __vector_pair *vpp, __vector_pair *vp2) {
   __builtin_vsx_stxvp(vp, 0L, vp2);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test67(
+// CHECK-LABEL: @test67(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[VPP:%.*]], i64 [[OFFSET:%.*]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr [[TMP0]])
@@ -939,7 +939,7 @@ void test67(const __vector_pair *vpp, signed long offset, __vector_pair *vp2) {
   __builtin_vsx_stxvp(vp, offset, vp2);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test68(
+// CHECK-LABEL: @test68(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[VPP:%.*]], i64 18
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr [[TMP0]])
@@ -952,7 +952,7 @@ void test68(const __vector_pair *vpp, __vector_pair *vp2) {
   __builtin_vsx_stxvp(vp, 18L, vp2);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test69(
+// CHECK-LABEL: @test69(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[VPP:%.*]], i64 1
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr [[TMP0]])
@@ -965,7 +965,7 @@ void test69(const __vector_pair *vpp, __vector_pair *vp2) {
   __builtin_vsx_stxvp(vp, 1L, vp2);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test70(
+// CHECK-LABEL: @test70(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[VPP:%.*]], i64 42
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr [[TMP0]])
@@ -978,7 +978,7 @@ void test70(const __vector_pair *vpp, __vector_pair *vp2) {
   __builtin_vsx_stxvp(vp, 42L, vp2);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test71(
+// CHECK-LABEL: @test71(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[VPP:%.*]], i64 32768
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr [[TMP0]])
@@ -991,7 +991,7 @@ void test71(const __vector_pair *vpp, __vector_pair *vp2) {
   __builtin_vsx_stxvp(vp, 32768L, vp2);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test72(
+// CHECK-LABEL: @test72(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[VPP:%.*]], i64 32799
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr [[TMP0]])
@@ -1004,7 +1004,7 @@ void test72(const __vector_pair *vpp, __vector_pair *vp2) {
   __builtin_vsx_stxvp(vp, 32799L, vp2);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test73(
+// CHECK-LABEL: @test73(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[VPP:%.*]], i64 8
@@ -1020,7 +1020,7 @@ void test73(unsigned char *vqp, const __vector_pair *vpp, vector unsigned char v
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test74(
+// CHECK-LABEL: @test74(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr [[VPP:%.*]])
@@ -1035,7 +1035,7 @@ void test74(unsigned char *vqp, const __vector_pair *vpp, vector unsigned char v
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test75(
+// CHECK-LABEL: @test75(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[VPP:%.*]], i64 [[OFFS:%.*]]
@@ -1051,7 +1051,7 @@ void test75(unsigned char *vqp, signed long offs, const __vector_pair *vpp, vect
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test76(
+// CHECK-LABEL: @test76(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.assemble.pair(<16 x i8> [[VC:%.*]], <16 x i8> [[VC]])
 // CHECK-NEXT:    store <256 x i1> [[TMP0]], ptr [[RESP:%.*]], align 32, !tbaa [[TBAA6]]
@@ -1065,7 +1065,7 @@ void test76(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   *((__vector_pair *)resp) = res;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test77(
+// CHECK-LABEL: @test77(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <256 x i1>, ptr [[VPP:%.*]], align 32
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <16 x i8>, <16 x i8> } @llvm.ppc.vsx.disassemble.pair(<256 x i1> [[TMP0]])
@@ -1080,7 +1080,7 @@ void test77(unsigned char *vqp, unsigned char *vpp, vector unsigned char vc, uns
   __builtin_mma_disassemble_pair(resp, (__vector_pair*)vpp);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test78(
+// CHECK-LABEL: @test78(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr [[VPP:%.*]])
 // CHECK-NEXT:    tail call void @llvm.ppc.vsx.stxvp(<256 x i1> [[TMP0]], ptr [[VP2:%.*]])
@@ -1091,7 +1091,7 @@ void test78(const __vector_pair *vpp, __vector_pair *vp2) {
   __builtin_mma_stxvp(vp, 0L, vp2);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test79(
+// CHECK-LABEL: @test79(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[VPP:%.*]], i64 [[OFFSET:%.*]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr [[TMP0]])
@@ -1104,7 +1104,7 @@ void test79(const __vector_pair *vpp, signed long offset, __vector_pair *vp2) {
   __builtin_mma_stxvp(vp, offset, vp2);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test80(
+// CHECK-LABEL: @test80(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[VPP:%.*]], i64 18
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr [[TMP0]])
@@ -1117,7 +1117,7 @@ void test80(const __vector_pair *vpp, __vector_pair *vp2) {
   __builtin_mma_stxvp(vp, 18L, vp2);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test81(
+// CHECK-LABEL: @test81(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[VPP:%.*]], i64 1
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr [[TMP0]])
@@ -1130,7 +1130,7 @@ void test81(const __vector_pair *vpp, __vector_pair *vp2) {
   __builtin_mma_stxvp(vp, 1L, vp2);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test82(
+// CHECK-LABEL: @test82(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[VPP:%.*]], i64 42
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr [[TMP0]])
@@ -1143,7 +1143,7 @@ void test82(const __vector_pair *vpp, __vector_pair *vp2) {
   __builtin_mma_stxvp(vp, 42L, vp2);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test83(
+// CHECK-LABEL: @test83(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[VPP:%.*]], i64 32768
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr [[TMP0]])
@@ -1156,7 +1156,7 @@ void test83(const __vector_pair *vpp, __vector_pair *vp2) {
   __builtin_mma_stxvp(vp, 32768L, vp2);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test84(
+// CHECK-LABEL: @test84(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[VPP:%.*]], i64 32799
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr [[TMP0]])
@@ -1169,7 +1169,7 @@ void test84(const __vector_pair *vpp, __vector_pair *vp2) {
   __builtin_mma_stxvp(vp, 32799L, vp2);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test85(
+// CHECK-LABEL: @test85(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[VPP:%.*]], i64 8
@@ -1185,7 +1185,7 @@ void test85(unsigned char *vqp, const __vector_pair *vpp, vector unsigned char v
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test86(
+// CHECK-LABEL: @test86(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <256 x i1> @llvm.ppc.vsx.lxvp(ptr [[VPP:%.*]])
@@ -1200,7 +1200,7 @@ void test86(unsigned char *vqp, const __vector_pair *vpp, vector unsigned char v
   *((__vector_quad *)resp) = vq;
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test87(
+// CHECK-LABEL: @test87(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = load <512 x i1>, ptr [[VQP:%.*]], align 64, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[VPP:%.*]], i64 [[OFFS:%.*]]
diff --git a/clang/test/CodeGen/PowerPC/inline-asm-matching-ppc-vsx.c b/clang/test/CodeGen/PowerPC/inline-asm-matching-ppc-vsx.c
index a4fabd6881..842e9b533a 100644
--- a/clang/test/CodeGen/PowerPC/inline-asm-matching-ppc-vsx.c
+++ b/clang/test/CodeGen/PowerPC/inline-asm-matching-ppc-vsx.c
@@ -2,6 +2,12 @@
 
 // RUN: %clang_cc1 -triple powerpc64le-unknown-linux-gnu -target-feature +vsx \
 // RUN:   -target-cpu pwr9 -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -triple powerpc64-unknown-linux-gnu -target-feature +vsx \
+// RUN:   -target-cpu pwr9 -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -triple powerpc64-ibm-aix -target-feature +vsx \
+// RUN:   -target-cpu pwr9 -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -triple powerpc-ibm-aix -target-feature +vsx \
+// RUN:   -target-cpu pwr9 -emit-llvm %s -o - | FileCheck %s
 
 // This case is to test VSX register support in the clobbers list for inline asm.
 void testVSX (void) {
@@ -9,12 +15,12 @@ void testVSX (void) {
   unsigned int *dbell=&a;
   int d;
   __asm__ __volatile__ (
-    "lxvw4x  %%vs32, 0, %2\n\t"
-    "stxvw4x %%vs32, 0, %1"
+    "lxvw4x  32, 0, %2\n\t"
+    "stxvw4x 32, 0, %1"
     : "=m"(*(volatile unsigned int*)(dbell))
     : "r" (dbell), "r" (&d)
     : "vs32"
   );
 }
 
-// CHECK: call void asm sideeffect "lxvw4x  %vs32, 0, $2\0A\09stxvw4x %vs32, 0, $1", "=*m,r,r,~{vs32}"
+// CHECK: call void asm sideeffect "lxvw4x  32, 0, $2\0A\09stxvw4x 32, 0, $1", "=*m,r,r,~{vs32}"
diff --git a/clang/test/CodeGen/RISCV/riscv-inline-asm-clobber.c b/clang/test/CodeGen/RISCV/riscv-inline-asm-clobber.c
new file mode 100644
index 0000000000..8aa80386f2
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/riscv-inline-asm-clobber.c
@@ -0,0 +1,44 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 3
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv32 -O2 -emit-llvm %s -o - \
+// RUN:     | FileCheck %s
+// RUN: %clang_cc1 -triple riscv64 -O2 -emit-llvm %s -o - \
+// RUN:     | FileCheck %s
+
+// Test RISC-V specific clobbered registers in inline assembly.
+
+// CHECK-LABEL: define {{.*}} void @test_fflags
+// CHECK:    tail call void asm sideeffect "", "~{fflags}"()
+void test_fflags(void) {
+  asm volatile ("" :::"fflags");
+}
+
+// CHECK-LABEL: define {{.*}} void @test_frm
+// CHECK:    tail call void asm sideeffect "", "~{frm}"()
+void test_frm(void) {
+  asm volatile ("" :::"frm");
+}
+
+// CHECK-LABEL: define {{.*}} void @test_vtype
+// CHECK:    tail call void asm sideeffect "", "~{vtype}"()
+void test_vtype(void) {
+  asm volatile ("" :::"vtype");
+}
+
+// CHECK-LABEL: define {{.*}} void @test_vl
+// CHECK:    tail call void asm sideeffect "", "~{vl}"()
+void test_vl(void) {
+  asm volatile ("" :::"vl");
+}
+
+// CHECK-LABEL: define {{.*}} void @test_vxsat
+// CHECK:    tail call void asm sideeffect "", "~{vxsat}"()
+void test_vxsat(void) {
+  asm volatile ("" :::"vxsat");
+}
+
+// CHECK-LABEL: define {{.*}} void @test_vxrm
+// CHECK:    tail call void asm sideeffect "", "~{vxrm}"()
+void test_vxrm(void) {
+  asm volatile ("" :::"vxrm");
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmacc_2x8x2.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmacc_2x8x2.c
new file mode 100644
index 0000000000..29ebeea7a0
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmacc_2x8x2.c
@@ -0,0 +1,47 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +xsfvqmaccdod \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x i32> @test_sf_vqmacc_2x8x2_i32m1
+// CHECK-RV64-SAME: (<vscale x 2 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vqmacc.2x8x2.nxv2i32.nxv8i8.nxv8i8.i64(<vscale x 2 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 8 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vint32m1_t test_sf_vqmacc_2x8x2_i32m1(vint32m1_t vd, vint8m1_t vs1, vint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vqmacc_2x8x2_i32m1(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x i32> @test_sf_vqmacc_2x8x2_i32m2
+// CHECK-RV64-SAME: (<vscale x 4 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vqmacc.2x8x2.nxv4i32.nxv8i8.nxv16i8.i64(<vscale x 4 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 16 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vint32m2_t test_sf_vqmacc_2x8x2_i32m2(vint32m2_t vd, vint8m1_t vs1, vint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vqmacc_2x8x2_i32m2(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x i32> @test_sf_vqmacc_2x8x2_i32m4
+// CHECK-RV64-SAME: (<vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vqmacc.2x8x2.nxv8i32.nxv8i8.nxv32i8.i64(<vscale x 8 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 32 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vint32m4_t test_sf_vqmacc_2x8x2_i32m4(vint32m4_t vd, vint8m1_t vs1, vint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vqmacc_2x8x2_i32m4(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x i32> @test_sf_vqmacc_2x8x2_i32m8
+// CHECK-RV64-SAME: (<vscale x 16 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vqmacc.2x8x2.nxv16i32.nxv8i8.nxv64i8.i64(<vscale x 16 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 64 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vint32m8_t test_sf_vqmacc_2x8x2_i32m8(vint32m8_t vd, vint8m1_t vs1, vint8m8_t vs2, size_t vl) {
+  return __riscv_sf_vqmacc_2x8x2_i32m8(vd, vs1, vs2, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmacc_4x8x4.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmacc_4x8x4.c
new file mode 100644
index 0000000000..935cb2e007
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmacc_4x8x4.c
@@ -0,0 +1,47 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +xsfvqmaccqoq \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x i32> @test_sf_vqmacc_4x8x4_i32m1
+// CHECK-RV64-SAME: (<vscale x 2 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vqmacc.4x8x4.nxv2i32.nxv8i8.nxv8i8.i64(<vscale x 2 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 8 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vint32m1_t test_sf_vqmacc_4x8x4_i32m1(vint32m1_t vd, vint8m1_t vs1, vint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vqmacc_4x8x4_i32m1(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x i32> @test_sf_vqmacc_4x8x4_i32m2
+// CHECK-RV64-SAME: (<vscale x 4 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vqmacc.4x8x4.nxv4i32.nxv8i8.nxv16i8.i64(<vscale x 4 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 16 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vint32m2_t test_sf_vqmacc_4x8x4_i32m2(vint32m2_t vd, vint8m1_t vs1, vint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vqmacc_4x8x4_i32m2(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x i32> @test_sf_vqmacc_4x8x4_i32m4
+// CHECK-RV64-SAME: (<vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vqmacc.4x8x4.nxv8i32.nxv8i8.nxv32i8.i64(<vscale x 8 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 32 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vint32m4_t test_sf_vqmacc_4x8x4_i32m4(vint32m4_t vd, vint8m1_t vs1, vint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vqmacc_4x8x4_i32m4(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x i32> @test_sf_vqmacc_4x8x4_i32m8
+// CHECK-RV64-SAME: (<vscale x 16 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vqmacc.4x8x4.nxv16i32.nxv8i8.nxv64i8.i64(<vscale x 16 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 64 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vint32m8_t test_sf_vqmacc_4x8x4_i32m8(vint32m8_t vd, vint8m1_t vs1, vint8m8_t vs2, size_t vl) {
+  return __riscv_sf_vqmacc_4x8x4_i32m8(vd, vs1, vs2, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccsu_2x8x2.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccsu_2x8x2.c
new file mode 100644
index 0000000000..a1f95d2c15
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccsu_2x8x2.c
@@ -0,0 +1,47 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +xsfvqmaccdod \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x i32> @test_sf_vqmaccsu_2x8x2_i32m1
+// CHECK-RV64-SAME: (<vscale x 2 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vqmaccsu.2x8x2.nxv2i32.nxv8i8.nxv8i8.i64(<vscale x 2 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 8 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vint32m1_t test_sf_vqmaccsu_2x8x2_i32m1(vint32m1_t vd, vint8m1_t vs1, vuint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccsu_2x8x2_i32m1(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x i32> @test_sf_vqmaccsu_2x8x2_i32m2
+// CHECK-RV64-SAME: (<vscale x 4 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vqmaccsu.2x8x2.nxv4i32.nxv8i8.nxv16i8.i64(<vscale x 4 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 16 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vint32m2_t test_sf_vqmaccsu_2x8x2_i32m2(vint32m2_t vd, vint8m1_t vs1, vuint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccsu_2x8x2_i32m2(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x i32> @test_sf_vqmaccsu_2x8x2_i32m4
+// CHECK-RV64-SAME: (<vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vqmaccsu.2x8x2.nxv8i32.nxv8i8.nxv32i8.i64(<vscale x 8 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 32 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vint32m4_t test_sf_vqmaccsu_2x8x2_i32m4(vint32m4_t vd, vint8m1_t vs1, vuint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccsu_2x8x2_i32m4(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x i32> @test_sf_vqmaccsu_2x8x2_i32m8
+// CHECK-RV64-SAME: (<vscale x 16 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vqmaccsu.2x8x2.nxv16i32.nxv8i8.nxv64i8.i64(<vscale x 16 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 64 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vint32m8_t test_sf_vqmaccsu_2x8x2_i32m8(vint32m8_t vd, vint8m1_t vs1, vuint8m8_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccsu_2x8x2_i32m8(vd, vs1, vs2, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccsu_4x8x4.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccsu_4x8x4.c
new file mode 100644
index 0000000000..f34517b24b
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccsu_4x8x4.c
@@ -0,0 +1,47 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +xsfvqmaccqoq \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x i32> @test_sf_vqmaccsu_4x8x4_i32m1
+// CHECK-RV64-SAME: (<vscale x 2 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vqmaccsu.4x8x4.nxv2i32.nxv8i8.nxv8i8.i64(<vscale x 2 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 8 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vint32m1_t test_sf_vqmaccsu_4x8x4_i32m1(vint32m1_t vd, vint8m1_t vs1, vuint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccsu_4x8x4_i32m1(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x i32> @test_sf_vqmaccsu_4x8x4_i32m2
+// CHECK-RV64-SAME: (<vscale x 4 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vqmaccsu.4x8x4.nxv4i32.nxv8i8.nxv16i8.i64(<vscale x 4 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 16 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vint32m2_t test_sf_vqmaccsu_4x8x4_i32m2(vint32m2_t vd, vint8m1_t vs1, vuint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccsu_4x8x4_i32m2(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x i32> @test_sf_vqmaccsu_4x8x4_i32m4
+// CHECK-RV64-SAME: (<vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vqmaccsu.4x8x4.nxv8i32.nxv8i8.nxv32i8.i64(<vscale x 8 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 32 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vint32m4_t test_sf_vqmaccsu_4x8x4_i32m4(vint32m4_t vd, vint8m1_t vs1, vuint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccsu_4x8x4_i32m4(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x i32> @test_sf_vqmaccsu_4x8x4_i32m8
+// CHECK-RV64-SAME: (<vscale x 16 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vqmaccsu.4x8x4.nxv16i32.nxv8i8.nxv64i8.i64(<vscale x 16 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 64 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vint32m8_t test_sf_vqmaccsu_4x8x4_i32m8(vint32m8_t vd, vint8m1_t vs1, vuint8m8_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccsu_4x8x4_i32m8(vd, vs1, vs2, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccu_2x8x2.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccu_2x8x2.c
new file mode 100644
index 0000000000..9f887562e1
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccu_2x8x2.c
@@ -0,0 +1,47 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +xsfvqmaccdod \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x i32> @test_sf_vqmaccu_2x8x2_i32m1
+// CHECK-RV64-SAME: (<vscale x 2 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vqmaccu.2x8x2.nxv2i32.nxv8i8.nxv8i8.i64(<vscale x 2 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 8 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vint32m1_t test_sf_vqmaccu_2x8x2_i32m1(vint32m1_t vd, vuint8m1_t vs1, vuint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccu_2x8x2_i32m1(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x i32> @test_sf_vqmaccu_2x8x2_i32m2
+// CHECK-RV64-SAME: (<vscale x 4 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vqmaccu.2x8x2.nxv4i32.nxv8i8.nxv16i8.i64(<vscale x 4 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 16 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vint32m2_t test_sf_vqmaccu_2x8x2_i32m2(vint32m2_t vd, vuint8m1_t vs1, vuint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccu_2x8x2_i32m2(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x i32> @test_sf_vqmaccu_2x8x2_i32m4
+// CHECK-RV64-SAME: (<vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vqmaccu.2x8x2.nxv8i32.nxv8i8.nxv32i8.i64(<vscale x 8 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 32 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vint32m4_t test_sf_vqmaccu_2x8x2_i32m4(vint32m4_t vd, vuint8m1_t vs1, vuint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccu_2x8x2_i32m4(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x i32> @test_sf_vqmaccu_2x8x2_i32m8
+// CHECK-RV64-SAME: (<vscale x 16 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vqmaccu.2x8x2.nxv16i32.nxv8i8.nxv64i8.i64(<vscale x 16 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 64 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vint32m8_t test_sf_vqmaccu_2x8x2_i32m8(vint32m8_t vd, vuint8m1_t vs1, vuint8m8_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccu_2x8x2_i32m8(vd, vs1, vs2, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccu_4x8x4.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccu_4x8x4.c
new file mode 100644
index 0000000000..ab7f6627ad
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccu_4x8x4.c
@@ -0,0 +1,47 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +xsfvqmaccqoq \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x i32> @test_sf_vqmaccu_4x8x4_i32m1
+// CHECK-RV64-SAME: (<vscale x 2 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vqmaccu.4x8x4.nxv2i32.nxv8i8.nxv8i8.i64(<vscale x 2 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 8 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vint32m1_t test_sf_vqmaccu_4x8x4_i32m1(vint32m1_t vd, vuint8m1_t vs1, vuint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccu_4x8x4_i32m1(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x i32> @test_sf_vqmaccu_4x8x4_i32m2
+// CHECK-RV64-SAME: (<vscale x 4 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vqmaccu.4x8x4.nxv4i32.nxv8i8.nxv16i8.i64(<vscale x 4 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 16 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vint32m2_t test_sf_vqmaccu_4x8x4_i32m2(vint32m2_t vd, vuint8m1_t vs1, vuint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccu_4x8x4_i32m2(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x i32> @test_sf_vqmaccu_4x8x4_i32m4
+// CHECK-RV64-SAME: (<vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vqmaccu.4x8x4.nxv8i32.nxv8i8.nxv32i8.i64(<vscale x 8 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 32 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vint32m4_t test_sf_vqmaccu_4x8x4_i32m4(vint32m4_t vd, vuint8m1_t vs1, vuint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccu_4x8x4_i32m4(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x i32> @test_sf_vqmaccu_4x8x4_i32m8
+// CHECK-RV64-SAME: (<vscale x 16 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vqmaccu.4x8x4.nxv16i32.nxv8i8.nxv64i8.i64(<vscale x 16 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 64 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vint32m8_t test_sf_vqmaccu_4x8x4_i32m8(vint32m8_t vd, vuint8m1_t vs1, vuint8m8_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccu_4x8x4_i32m8(vd, vs1, vs2, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccus_2x8x2.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccus_2x8x2.c
new file mode 100644
index 0000000000..2921228700
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccus_2x8x2.c
@@ -0,0 +1,47 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +xsfvqmaccdod \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x i32> @test_sf_vqmaccus_2x8x2_i32m1
+// CHECK-RV64-SAME: (<vscale x 2 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vqmaccus.2x8x2.nxv2i32.nxv8i8.nxv8i8.i64(<vscale x 2 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 8 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vint32m1_t test_sf_vqmaccus_2x8x2_i32m1(vint32m1_t vd, vuint8m1_t vs1, vint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccus_2x8x2_i32m1(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x i32> @test_sf_vqmaccus_2x8x2_i32m2
+// CHECK-RV64-SAME: (<vscale x 4 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vqmaccus.2x8x2.nxv4i32.nxv8i8.nxv16i8.i64(<vscale x 4 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 16 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vint32m2_t test_sf_vqmaccus_2x8x2_i32m2(vint32m2_t vd, vuint8m1_t vs1, vint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccus_2x8x2_i32m2(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x i32> @test_sf_vqmaccus_2x8x2_i32m4
+// CHECK-RV64-SAME: (<vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vqmaccus.2x8x2.nxv8i32.nxv8i8.nxv32i8.i64(<vscale x 8 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 32 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vint32m4_t test_sf_vqmaccus_2x8x2_i32m4(vint32m4_t vd, vuint8m1_t vs1, vint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccus_2x8x2_i32m4(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x i32> @test_sf_vqmaccus_2x8x2_i32m8
+// CHECK-RV64-SAME: (<vscale x 16 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vqmaccus.2x8x2.nxv16i32.nxv8i8.nxv64i8.i64(<vscale x 16 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 64 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vint32m8_t test_sf_vqmaccus_2x8x2_i32m8(vint32m8_t vd, vuint8m1_t vs1, vint8m8_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccus_2x8x2_i32m8(vd, vs1, vs2, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccus_4x8x4.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccus_4x8x4.c
new file mode 100644
index 0000000000..d0bcdcbf40
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccus_4x8x4.c
@@ -0,0 +1,47 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +xsfvqmaccqoq \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x i32> @test_sf_vqmaccus_4x8x4_i32m1
+// CHECK-RV64-SAME: (<vscale x 2 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vqmaccus.4x8x4.nxv2i32.nxv8i8.nxv8i8.i64(<vscale x 2 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 8 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vint32m1_t test_sf_vqmaccus_4x8x4_i32m1(vint32m1_t vd, vuint8m1_t vs1, vint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccus_4x8x4_i32m1(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x i32> @test_sf_vqmaccus_4x8x4_i32m2
+// CHECK-RV64-SAME: (<vscale x 4 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vqmaccus.4x8x4.nxv4i32.nxv8i8.nxv16i8.i64(<vscale x 4 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 16 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vint32m2_t test_sf_vqmaccus_4x8x4_i32m2(vint32m2_t vd, vuint8m1_t vs1, vint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccus_4x8x4_i32m2(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x i32> @test_sf_vqmaccus_4x8x4_i32m4
+// CHECK-RV64-SAME: (<vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vqmaccus.4x8x4.nxv8i32.nxv8i8.nxv32i8.i64(<vscale x 8 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 32 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vint32m4_t test_sf_vqmaccus_4x8x4_i32m4(vint32m4_t vd, vuint8m1_t vs1, vint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccus_4x8x4_i32m4(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x i32> @test_sf_vqmaccus_4x8x4_i32m8
+// CHECK-RV64-SAME: (<vscale x 16 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vqmaccus.4x8x4.nxv16i32.nxv8i8.nxv64i8.i64(<vscale x 16 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 64 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vint32m8_t test_sf_vqmaccus_4x8x4_i32m8(vint32m8_t vd, vuint8m1_t vs1, vint8m8_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccus_4x8x4_i32m8(vd, vs1, vs2, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmacc_2x8x2.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmacc_2x8x2.c
new file mode 100644
index 0000000000..bef04c5707
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmacc_2x8x2.c
@@ -0,0 +1,47 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +xsfvqmaccdod \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x i32> @test_sf_vqmacc_2x8x2_i32m1
+// CHECK-RV64-SAME: (<vscale x 2 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vqmacc.2x8x2.nxv2i32.nxv8i8.nxv8i8.i64(<vscale x 2 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 8 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vint32m1_t test_sf_vqmacc_2x8x2_i32m1(vint32m1_t vd, vint8m1_t vs1, vint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vqmacc_2x8x2(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x i32> @test_sf_vqmacc_2x8x2_i32m2
+// CHECK-RV64-SAME: (<vscale x 4 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vqmacc.2x8x2.nxv4i32.nxv8i8.nxv16i8.i64(<vscale x 4 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 16 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vint32m2_t test_sf_vqmacc_2x8x2_i32m2(vint32m2_t vd, vint8m1_t vs1, vint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vqmacc_2x8x2(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x i32> @test_sf_vqmacc_2x8x2_i32m4
+// CHECK-RV64-SAME: (<vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vqmacc.2x8x2.nxv8i32.nxv8i8.nxv32i8.i64(<vscale x 8 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 32 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vint32m4_t test_sf_vqmacc_2x8x2_i32m4(vint32m4_t vd, vint8m1_t vs1, vint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vqmacc_2x8x2(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x i32> @test_sf_vqmacc_2x8x2_i32m8
+// CHECK-RV64-SAME: (<vscale x 16 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vqmacc.2x8x2.nxv16i32.nxv8i8.nxv64i8.i64(<vscale x 16 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 64 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vint32m8_t test_sf_vqmacc_2x8x2_i32m8(vint32m8_t vd, vint8m1_t vs1, vint8m8_t vs2, size_t vl) {
+  return __riscv_sf_vqmacc_2x8x2(vd, vs1, vs2, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmacc_4x8x4.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmacc_4x8x4.c
new file mode 100644
index 0000000000..839d09c4f9
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmacc_4x8x4.c
@@ -0,0 +1,47 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +xsfvqmaccqoq \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x i32> @test_sf_vqmacc_4x8x4_i32m1
+// CHECK-RV64-SAME: (<vscale x 2 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vqmacc.4x8x4.nxv2i32.nxv8i8.nxv8i8.i64(<vscale x 2 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 8 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vint32m1_t test_sf_vqmacc_4x8x4_i32m1(vint32m1_t vd, vint8m1_t vs1, vint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vqmacc_4x8x4(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x i32> @test_sf_vqmacc_4x8x4_i32m2
+// CHECK-RV64-SAME: (<vscale x 4 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vqmacc.4x8x4.nxv4i32.nxv8i8.nxv16i8.i64(<vscale x 4 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 16 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vint32m2_t test_sf_vqmacc_4x8x4_i32m2(vint32m2_t vd, vint8m1_t vs1, vint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vqmacc_4x8x4(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x i32> @test_sf_vqmacc_4x8x4_i32m4
+// CHECK-RV64-SAME: (<vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vqmacc.4x8x4.nxv8i32.nxv8i8.nxv32i8.i64(<vscale x 8 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 32 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vint32m4_t test_sf_vqmacc_4x8x4_i32m4(vint32m4_t vd, vint8m1_t vs1, vint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vqmacc_4x8x4(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x i32> @test_sf_vqmacc_4x8x4_i32m8
+// CHECK-RV64-SAME: (<vscale x 16 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vqmacc.4x8x4.nxv16i32.nxv8i8.nxv64i8.i64(<vscale x 16 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 64 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vint32m8_t test_sf_vqmacc_4x8x4_i32m8(vint32m8_t vd, vint8m1_t vs1, vint8m8_t vs2, size_t vl) {
+  return __riscv_sf_vqmacc_4x8x4(vd, vs1, vs2, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccsu_2x8x2.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccsu_2x8x2.c
new file mode 100644
index 0000000000..da0f632a3a
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccsu_2x8x2.c
@@ -0,0 +1,47 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +xsfvqmaccdod \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x i32> @test_sf_vqmaccsu_2x8x2_i32m1
+// CHECK-RV64-SAME: (<vscale x 2 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vqmaccsu.2x8x2.nxv2i32.nxv8i8.nxv8i8.i64(<vscale x 2 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 8 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vint32m1_t test_sf_vqmaccsu_2x8x2_i32m1(vint32m1_t vd, vint8m1_t vs1, vuint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccsu_2x8x2(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x i32> @test_sf_vqmaccsu_2x8x2_i32m2
+// CHECK-RV64-SAME: (<vscale x 4 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vqmaccsu.2x8x2.nxv4i32.nxv8i8.nxv16i8.i64(<vscale x 4 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 16 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vint32m2_t test_sf_vqmaccsu_2x8x2_i32m2(vint32m2_t vd, vint8m1_t vs1, vuint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccsu_2x8x2(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x i32> @test_sf_vqmaccsu_2x8x2_i32m4
+// CHECK-RV64-SAME: (<vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vqmaccsu.2x8x2.nxv8i32.nxv8i8.nxv32i8.i64(<vscale x 8 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 32 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vint32m4_t test_sf_vqmaccsu_2x8x2_i32m4(vint32m4_t vd, vint8m1_t vs1, vuint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccsu_2x8x2(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x i32> @test_sf_vqmaccsu_2x8x2_i32m8
+// CHECK-RV64-SAME: (<vscale x 16 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vqmaccsu.2x8x2.nxv16i32.nxv8i8.nxv64i8.i64(<vscale x 16 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 64 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vint32m8_t test_sf_vqmaccsu_2x8x2_i32m8(vint32m8_t vd, vint8m1_t vs1, vuint8m8_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccsu_2x8x2(vd, vs1, vs2, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccsu_4x8x4.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccsu_4x8x4.c
new file mode 100644
index 0000000000..b18853043e
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccsu_4x8x4.c
@@ -0,0 +1,47 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +xsfvqmaccqoq \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x i32> @test_sf_vqmaccsu_4x8x4_i32m1
+// CHECK-RV64-SAME: (<vscale x 2 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vqmaccsu.4x8x4.nxv2i32.nxv8i8.nxv8i8.i64(<vscale x 2 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 8 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vint32m1_t test_sf_vqmaccsu_4x8x4_i32m1(vint32m1_t vd, vint8m1_t vs1, vuint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccsu_4x8x4(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x i32> @test_sf_vqmaccsu_4x8x4_i32m2
+// CHECK-RV64-SAME: (<vscale x 4 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vqmaccsu.4x8x4.nxv4i32.nxv8i8.nxv16i8.i64(<vscale x 4 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 16 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vint32m2_t test_sf_vqmaccsu_4x8x4_i32m2(vint32m2_t vd, vint8m1_t vs1, vuint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccsu_4x8x4(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x i32> @test_sf_vqmaccsu_4x8x4_i32m4
+// CHECK-RV64-SAME: (<vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vqmaccsu.4x8x4.nxv8i32.nxv8i8.nxv32i8.i64(<vscale x 8 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 32 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vint32m4_t test_sf_vqmaccsu_4x8x4_i32m4(vint32m4_t vd, vint8m1_t vs1, vuint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccsu_4x8x4(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x i32> @test_sf_vqmaccsu_4x8x4_i32m8
+// CHECK-RV64-SAME: (<vscale x 16 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vqmaccsu.4x8x4.nxv16i32.nxv8i8.nxv64i8.i64(<vscale x 16 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 64 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vint32m8_t test_sf_vqmaccsu_4x8x4_i32m8(vint32m8_t vd, vint8m1_t vs1, vuint8m8_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccsu_4x8x4(vd, vs1, vs2, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccu_2x8x2.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccu_2x8x2.c
new file mode 100644
index 0000000000..c58182cf3c
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccu_2x8x2.c
@@ -0,0 +1,47 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +xsfvqmaccdod \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x i32> @test_sf_vqmaccu_2x8x2_i32m1
+// CHECK-RV64-SAME: (<vscale x 2 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vqmaccu.2x8x2.nxv2i32.nxv8i8.nxv8i8.i64(<vscale x 2 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 8 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vint32m1_t test_sf_vqmaccu_2x8x2_i32m1(vint32m1_t vd, vuint8m1_t vs1, vuint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccu_2x8x2(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x i32> @test_sf_vqmaccu_2x8x2_i32m2
+// CHECK-RV64-SAME: (<vscale x 4 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vqmaccu.2x8x2.nxv4i32.nxv8i8.nxv16i8.i64(<vscale x 4 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 16 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vint32m2_t test_sf_vqmaccu_2x8x2_i32m2(vint32m2_t vd, vuint8m1_t vs1, vuint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccu_2x8x2(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x i32> @test_sf_vqmaccu_2x8x2_i32m4
+// CHECK-RV64-SAME: (<vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vqmaccu.2x8x2.nxv8i32.nxv8i8.nxv32i8.i64(<vscale x 8 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 32 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vint32m4_t test_sf_vqmaccu_2x8x2_i32m4(vint32m4_t vd, vuint8m1_t vs1, vuint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccu_2x8x2(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x i32> @test_sf_vqmaccu_2x8x2_i32m8
+// CHECK-RV64-SAME: (<vscale x 16 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vqmaccu.2x8x2.nxv16i32.nxv8i8.nxv64i8.i64(<vscale x 16 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 64 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vint32m8_t test_sf_vqmaccu_2x8x2_i32m8(vint32m8_t vd, vuint8m1_t vs1, vuint8m8_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccu_2x8x2(vd, vs1, vs2, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccu_4x8x4.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccu_4x8x4.c
new file mode 100644
index 0000000000..4cb966b08f
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccu_4x8x4.c
@@ -0,0 +1,47 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +xsfvqmaccqoq \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x i32> @test_sf_vqmaccu_4x8x4_i32m1
+// CHECK-RV64-SAME: (<vscale x 2 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vqmaccu.4x8x4.nxv2i32.nxv8i8.nxv8i8.i64(<vscale x 2 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 8 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vint32m1_t test_sf_vqmaccu_4x8x4_i32m1(vint32m1_t vd, vuint8m1_t vs1, vuint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccu_4x8x4(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x i32> @test_sf_vqmaccu_4x8x4_i32m2
+// CHECK-RV64-SAME: (<vscale x 4 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vqmaccu.4x8x4.nxv4i32.nxv8i8.nxv16i8.i64(<vscale x 4 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 16 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vint32m2_t test_sf_vqmaccu_4x8x4_i32m2(vint32m2_t vd, vuint8m1_t vs1, vuint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccu_4x8x4(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x i32> @test_sf_vqmaccu_4x8x4_i32m4
+// CHECK-RV64-SAME: (<vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vqmaccu.4x8x4.nxv8i32.nxv8i8.nxv32i8.i64(<vscale x 8 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 32 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vint32m4_t test_sf_vqmaccu_4x8x4_i32m4(vint32m4_t vd, vuint8m1_t vs1, vuint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccu_4x8x4(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x i32> @test_sf_vqmaccu_4x8x4_i32m8
+// CHECK-RV64-SAME: (<vscale x 16 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vqmaccu.4x8x4.nxv16i32.nxv8i8.nxv64i8.i64(<vscale x 16 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 64 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vint32m8_t test_sf_vqmaccu_4x8x4_i32m8(vint32m8_t vd, vuint8m1_t vs1, vuint8m8_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccu_4x8x4(vd, vs1, vs2, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccus_2x8x2.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccus_2x8x2.c
new file mode 100644
index 0000000000..a16f576e53
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccus_2x8x2.c
@@ -0,0 +1,47 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +xsfvqmaccdod \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x i32> @test_sf_vqmaccus_2x8x2_i32m1
+// CHECK-RV64-SAME: (<vscale x 2 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vqmaccus.2x8x2.nxv2i32.nxv8i8.nxv8i8.i64(<vscale x 2 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 8 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vint32m1_t test_sf_vqmaccus_2x8x2_i32m1(vint32m1_t vd, vuint8m1_t vs1, vint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccus_2x8x2(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x i32> @test_sf_vqmaccus_2x8x2_i32m2
+// CHECK-RV64-SAME: (<vscale x 4 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vqmaccus.2x8x2.nxv4i32.nxv8i8.nxv16i8.i64(<vscale x 4 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 16 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vint32m2_t test_sf_vqmaccus_2x8x2_i32m2(vint32m2_t vd, vuint8m1_t vs1, vint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccus_2x8x2(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x i32> @test_sf_vqmaccus_2x8x2_i32m4
+// CHECK-RV64-SAME: (<vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vqmaccus.2x8x2.nxv8i32.nxv8i8.nxv32i8.i64(<vscale x 8 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 32 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vint32m4_t test_sf_vqmaccus_2x8x2_i32m4(vint32m4_t vd, vuint8m1_t vs1, vint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccus_2x8x2(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x i32> @test_sf_vqmaccus_2x8x2_i32m8
+// CHECK-RV64-SAME: (<vscale x 16 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vqmaccus.2x8x2.nxv16i32.nxv8i8.nxv64i8.i64(<vscale x 16 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 64 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vint32m8_t test_sf_vqmaccus_2x8x2_i32m8(vint32m8_t vd, vuint8m1_t vs1, vint8m8_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccus_2x8x2(vd, vs1, vs2, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccus_4x8x4.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccus_4x8x4.c
new file mode 100644
index 0000000000..f558151f88
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccus_4x8x4.c
@@ -0,0 +1,47 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +xsfvqmaccqoq \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x i32> @test_sf_vqmaccus_4x8x4_i32m1
+// CHECK-RV64-SAME: (<vscale x 2 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vqmaccus.4x8x4.nxv2i32.nxv8i8.nxv8i8.i64(<vscale x 2 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 8 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vint32m1_t test_sf_vqmaccus_4x8x4_i32m1(vint32m1_t vd, vuint8m1_t vs1, vint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccus_4x8x4(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x i32> @test_sf_vqmaccus_4x8x4_i32m2
+// CHECK-RV64-SAME: (<vscale x 4 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vqmaccus.4x8x4.nxv4i32.nxv8i8.nxv16i8.i64(<vscale x 4 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 16 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vint32m2_t test_sf_vqmaccus_4x8x4_i32m2(vint32m2_t vd, vuint8m1_t vs1, vint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccus_4x8x4(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x i32> @test_sf_vqmaccus_4x8x4_i32m4
+// CHECK-RV64-SAME: (<vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vqmaccus.4x8x4.nxv8i32.nxv8i8.nxv32i8.i64(<vscale x 8 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 32 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vint32m4_t test_sf_vqmaccus_4x8x4_i32m4(vint32m4_t vd, vuint8m1_t vs1, vint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccus_4x8x4(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x i32> @test_sf_vqmaccus_4x8x4_i32m8
+// CHECK-RV64-SAME: (<vscale x 16 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vqmaccus.4x8x4.nxv16i32.nxv8i8.nxv64i8.i64(<vscale x 16 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 64 x i8> [[VS2]], i64 [[VL]], i64 3)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vint32m8_t test_sf_vqmaccus_4x8x4_i32m8(vint32m8_t vd, vuint8m1_t vs1, vint8m8_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccus_4x8x4(vd, vs1, vs2, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmacc_2x8x2.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmacc_2x8x2.c
new file mode 100644
index 0000000000..b4f0729d2f
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmacc_2x8x2.c
@@ -0,0 +1,47 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +xsfvqmaccdod \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x i32> @test_sf_vqmacc_2x8x2_i32m1_tu
+// CHECK-RV64-SAME: (<vscale x 2 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vqmacc.2x8x2.nxv2i32.nxv8i8.nxv8i8.i64(<vscale x 2 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 8 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vint32m1_t test_sf_vqmacc_2x8x2_i32m1_tu(vint32m1_t vd, vint8m1_t vs1, vint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vqmacc_2x8x2_i32m1_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x i32> @test_sf_vqmacc_2x8x2_i32m2_tu
+// CHECK-RV64-SAME: (<vscale x 4 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vqmacc.2x8x2.nxv4i32.nxv8i8.nxv16i8.i64(<vscale x 4 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 16 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vint32m2_t test_sf_vqmacc_2x8x2_i32m2_tu(vint32m2_t vd, vint8m1_t vs1, vint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vqmacc_2x8x2_i32m2_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x i32> @test_sf_vqmacc_2x8x2_i32m4_tu
+// CHECK-RV64-SAME: (<vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vqmacc.2x8x2.nxv8i32.nxv8i8.nxv32i8.i64(<vscale x 8 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 32 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vint32m4_t test_sf_vqmacc_2x8x2_i32m4_tu(vint32m4_t vd, vint8m1_t vs1, vint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vqmacc_2x8x2_i32m4_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x i32> @test_sf_vqmacc_2x8x2_i32m8_tu
+// CHECK-RV64-SAME: (<vscale x 16 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vqmacc.2x8x2.nxv16i32.nxv8i8.nxv64i8.i64(<vscale x 16 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 64 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vint32m8_t test_sf_vqmacc_2x8x2_i32m8_tu(vint32m8_t vd, vint8m1_t vs1, vint8m8_t vs2, size_t vl) {
+  return __riscv_sf_vqmacc_2x8x2_i32m8_tu(vd, vs1, vs2, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmacc_4x8x4.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmacc_4x8x4.c
new file mode 100644
index 0000000000..05c10840ca
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmacc_4x8x4.c
@@ -0,0 +1,47 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +xsfvqmaccqoq \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x i32> @test_sf_vqmacc_4x8x4_i32m1_tu
+// CHECK-RV64-SAME: (<vscale x 2 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vqmacc.4x8x4.nxv2i32.nxv8i8.nxv8i8.i64(<vscale x 2 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 8 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vint32m1_t test_sf_vqmacc_4x8x4_i32m1_tu(vint32m1_t vd, vint8m1_t vs1, vint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vqmacc_4x8x4_i32m1_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x i32> @test_sf_vqmacc_4x8x4_i32m2_tu
+// CHECK-RV64-SAME: (<vscale x 4 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vqmacc.4x8x4.nxv4i32.nxv8i8.nxv16i8.i64(<vscale x 4 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 16 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vint32m2_t test_sf_vqmacc_4x8x4_i32m2_tu(vint32m2_t vd, vint8m1_t vs1, vint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vqmacc_4x8x4_i32m2_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x i32> @test_sf_vqmacc_4x8x4_i32m4_tu
+// CHECK-RV64-SAME: (<vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vqmacc.4x8x4.nxv8i32.nxv8i8.nxv32i8.i64(<vscale x 8 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 32 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vint32m4_t test_sf_vqmacc_4x8x4_i32m4_tu(vint32m4_t vd, vint8m1_t vs1, vint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vqmacc_4x8x4_i32m4_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x i32> @test_sf_vqmacc_4x8x4_i32m8_tu
+// CHECK-RV64-SAME: (<vscale x 16 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vqmacc.4x8x4.nxv16i32.nxv8i8.nxv64i8.i64(<vscale x 16 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 64 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vint32m8_t test_sf_vqmacc_4x8x4_i32m8_tu(vint32m8_t vd, vint8m1_t vs1, vint8m8_t vs2, size_t vl) {
+  return __riscv_sf_vqmacc_4x8x4_i32m8_tu(vd, vs1, vs2, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccsu_2x8x2.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccsu_2x8x2.c
new file mode 100644
index 0000000000..612f8c3cfa
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccsu_2x8x2.c
@@ -0,0 +1,47 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +xsfvqmaccdod \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x i32> @test_sf_vqmaccsu_2x8x2_i32m1_tu
+// CHECK-RV64-SAME: (<vscale x 2 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vqmaccsu.2x8x2.nxv2i32.nxv8i8.nxv8i8.i64(<vscale x 2 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 8 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vint32m1_t test_sf_vqmaccsu_2x8x2_i32m1_tu(vint32m1_t vd, vint8m1_t vs1, vuint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccsu_2x8x2_i32m1_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x i32> @test_sf_vqmaccsu_2x8x2_i32m2_tu
+// CHECK-RV64-SAME: (<vscale x 4 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vqmaccsu.2x8x2.nxv4i32.nxv8i8.nxv16i8.i64(<vscale x 4 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 16 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vint32m2_t test_sf_vqmaccsu_2x8x2_i32m2_tu(vint32m2_t vd, vint8m1_t vs1, vuint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccsu_2x8x2_i32m2_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x i32> @test_sf_vqmaccsu_2x8x2_i32m4_tu
+// CHECK-RV64-SAME: (<vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vqmaccsu.2x8x2.nxv8i32.nxv8i8.nxv32i8.i64(<vscale x 8 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 32 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vint32m4_t test_sf_vqmaccsu_2x8x2_i32m4_tu(vint32m4_t vd, vint8m1_t vs1, vuint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccsu_2x8x2_i32m4_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x i32> @test_sf_vqmaccsu_2x8x2_i32m8_tu
+// CHECK-RV64-SAME: (<vscale x 16 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vqmaccsu.2x8x2.nxv16i32.nxv8i8.nxv64i8.i64(<vscale x 16 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 64 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vint32m8_t test_sf_vqmaccsu_2x8x2_i32m8_tu(vint32m8_t vd, vint8m1_t vs1, vuint8m8_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccsu_2x8x2_i32m8_tu(vd, vs1, vs2, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccsu_4x8x4.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccsu_4x8x4.c
new file mode 100644
index 0000000000..bce1a4e944
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccsu_4x8x4.c
@@ -0,0 +1,47 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +xsfvqmaccqoq \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x i32> @test_sf_vqmaccsu_4x8x4_i32m1_tu
+// CHECK-RV64-SAME: (<vscale x 2 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vqmaccsu.4x8x4.nxv2i32.nxv8i8.nxv8i8.i64(<vscale x 2 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 8 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vint32m1_t test_sf_vqmaccsu_4x8x4_i32m1_tu(vint32m1_t vd, vint8m1_t vs1, vuint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccsu_4x8x4_i32m1_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x i32> @test_sf_vqmaccsu_4x8x4_i32m2_tu
+// CHECK-RV64-SAME: (<vscale x 4 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vqmaccsu.4x8x4.nxv4i32.nxv8i8.nxv16i8.i64(<vscale x 4 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 16 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vint32m2_t test_sf_vqmaccsu_4x8x4_i32m2_tu(vint32m2_t vd, vint8m1_t vs1, vuint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccsu_4x8x4_i32m2_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x i32> @test_sf_vqmaccsu_4x8x4_i32m4_tu
+// CHECK-RV64-SAME: (<vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vqmaccsu.4x8x4.nxv8i32.nxv8i8.nxv32i8.i64(<vscale x 8 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 32 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vint32m4_t test_sf_vqmaccsu_4x8x4_i32m4_tu(vint32m4_t vd, vint8m1_t vs1, vuint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccsu_4x8x4_i32m4_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x i32> @test_sf_vqmaccsu_4x8x4_i32m8_tu
+// CHECK-RV64-SAME: (<vscale x 16 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vqmaccsu.4x8x4.nxv16i32.nxv8i8.nxv64i8.i64(<vscale x 16 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 64 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vint32m8_t test_sf_vqmaccsu_4x8x4_i32m8_tu(vint32m8_t vd, vint8m1_t vs1, vuint8m8_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccsu_4x8x4_i32m8_tu(vd, vs1, vs2, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccu_2x8x2.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccu_2x8x2.c
new file mode 100644
index 0000000000..6ffe84df87
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccu_2x8x2.c
@@ -0,0 +1,47 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +xsfvqmaccdod \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x i32> @test_sf_vqmaccu_2x8x2_i32m1_tu
+// CHECK-RV64-SAME: (<vscale x 2 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vqmaccu.2x8x2.nxv2i32.nxv8i8.nxv8i8.i64(<vscale x 2 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 8 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vint32m1_t test_sf_vqmaccu_2x8x2_i32m1_tu(vint32m1_t vd, vuint8m1_t vs1, vuint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccu_2x8x2_i32m1_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x i32> @test_sf_vqmaccu_2x8x2_i32m2_tu
+// CHECK-RV64-SAME: (<vscale x 4 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vqmaccu.2x8x2.nxv4i32.nxv8i8.nxv16i8.i64(<vscale x 4 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 16 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vint32m2_t test_sf_vqmaccu_2x8x2_i32m2_tu(vint32m2_t vd, vuint8m1_t vs1, vuint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccu_2x8x2_i32m2_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x i32> @test_sf_vqmaccu_2x8x2_i32m4_tu
+// CHECK-RV64-SAME: (<vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vqmaccu.2x8x2.nxv8i32.nxv8i8.nxv32i8.i64(<vscale x 8 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 32 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vint32m4_t test_sf_vqmaccu_2x8x2_i32m4_tu(vint32m4_t vd, vuint8m1_t vs1, vuint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccu_2x8x2_i32m4_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x i32> @test_sf_vqmaccu_2x8x2_i32m8_tu
+// CHECK-RV64-SAME: (<vscale x 16 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vqmaccu.2x8x2.nxv16i32.nxv8i8.nxv64i8.i64(<vscale x 16 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 64 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vint32m8_t test_sf_vqmaccu_2x8x2_i32m8_tu(vint32m8_t vd, vuint8m1_t vs1, vuint8m8_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccu_2x8x2_i32m8_tu(vd, vs1, vs2, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccu_4x8x4.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccu_4x8x4.c
new file mode 100644
index 0000000000..36aaae9cae
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccu_4x8x4.c
@@ -0,0 +1,47 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +xsfvqmaccqoq \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x i32> @test_sf_vqmaccu_4x8x4_i32m1_tu
+// CHECK-RV64-SAME: (<vscale x 2 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vqmaccu.4x8x4.nxv2i32.nxv8i8.nxv8i8.i64(<vscale x 2 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 8 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vint32m1_t test_sf_vqmaccu_4x8x4_i32m1_tu(vint32m1_t vd, vuint8m1_t vs1, vuint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccu_4x8x4_i32m1_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x i32> @test_sf_vqmaccu_4x8x4_i32m2_tu
+// CHECK-RV64-SAME: (<vscale x 4 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vqmaccu.4x8x4.nxv4i32.nxv8i8.nxv16i8.i64(<vscale x 4 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 16 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vint32m2_t test_sf_vqmaccu_4x8x4_i32m2_tu(vint32m2_t vd, vuint8m1_t vs1, vuint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccu_4x8x4_i32m2_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x i32> @test_sf_vqmaccu_4x8x4_i32m4_tu
+// CHECK-RV64-SAME: (<vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vqmaccu.4x8x4.nxv8i32.nxv8i8.nxv32i8.i64(<vscale x 8 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 32 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vint32m4_t test_sf_vqmaccu_4x8x4_i32m4_tu(vint32m4_t vd, vuint8m1_t vs1, vuint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccu_4x8x4_i32m4_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x i32> @test_sf_vqmaccu_4x8x4_i32m8_tu
+// CHECK-RV64-SAME: (<vscale x 16 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vqmaccu.4x8x4.nxv16i32.nxv8i8.nxv64i8.i64(<vscale x 16 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 64 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vint32m8_t test_sf_vqmaccu_4x8x4_i32m8_tu(vint32m8_t vd, vuint8m1_t vs1, vuint8m8_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccu_4x8x4_i32m8_tu(vd, vs1, vs2, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccus_2x8x2.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccus_2x8x2.c
new file mode 100644
index 0000000000..ebfd33f1b6
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccus_2x8x2.c
@@ -0,0 +1,47 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +xsfvqmaccdod \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x i32> @test_sf_vqmaccus_2x8x2_i32m1_tu
+// CHECK-RV64-SAME: (<vscale x 2 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vqmaccus.2x8x2.nxv2i32.nxv8i8.nxv8i8.i64(<vscale x 2 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 8 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vint32m1_t test_sf_vqmaccus_2x8x2_i32m1_tu(vint32m1_t vd, vuint8m1_t vs1, vint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccus_2x8x2_i32m1_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x i32> @test_sf_vqmaccus_2x8x2_i32m2_tu
+// CHECK-RV64-SAME: (<vscale x 4 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vqmaccus.2x8x2.nxv4i32.nxv8i8.nxv16i8.i64(<vscale x 4 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 16 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vint32m2_t test_sf_vqmaccus_2x8x2_i32m2_tu(vint32m2_t vd, vuint8m1_t vs1, vint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccus_2x8x2_i32m2_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x i32> @test_sf_vqmaccus_2x8x2_i32m4_tu
+// CHECK-RV64-SAME: (<vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vqmaccus.2x8x2.nxv8i32.nxv8i8.nxv32i8.i64(<vscale x 8 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 32 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vint32m4_t test_sf_vqmaccus_2x8x2_i32m4_tu(vint32m4_t vd, vuint8m1_t vs1, vint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccus_2x8x2_i32m4_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x i32> @test_sf_vqmaccus_2x8x2_i32m8_tu
+// CHECK-RV64-SAME: (<vscale x 16 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vqmaccus.2x8x2.nxv16i32.nxv8i8.nxv64i8.i64(<vscale x 16 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 64 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vint32m8_t test_sf_vqmaccus_2x8x2_i32m8_tu(vint32m8_t vd, vuint8m1_t vs1, vint8m8_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccus_2x8x2_i32m8_tu(vd, vs1, vs2, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccus_4x8x4.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccus_4x8x4.c
new file mode 100644
index 0000000000..f5ac2bf0f1
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccus_4x8x4.c
@@ -0,0 +1,47 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +xsfvqmaccqoq \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x i32> @test_sf_vqmaccus_4x8x4_i32m1_tu
+// CHECK-RV64-SAME: (<vscale x 2 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vqmaccus.4x8x4.nxv2i32.nxv8i8.nxv8i8.i64(<vscale x 2 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 8 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vint32m1_t test_sf_vqmaccus_4x8x4_i32m1_tu(vint32m1_t vd, vuint8m1_t vs1, vint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccus_4x8x4_i32m1_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x i32> @test_sf_vqmaccus_4x8x4_i32m2_tu
+// CHECK-RV64-SAME: (<vscale x 4 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vqmaccus.4x8x4.nxv4i32.nxv8i8.nxv16i8.i64(<vscale x 4 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 16 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vint32m2_t test_sf_vqmaccus_4x8x4_i32m2_tu(vint32m2_t vd, vuint8m1_t vs1, vint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccus_4x8x4_i32m2_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x i32> @test_sf_vqmaccus_4x8x4_i32m4_tu
+// CHECK-RV64-SAME: (<vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vqmaccus.4x8x4.nxv8i32.nxv8i8.nxv32i8.i64(<vscale x 8 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 32 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vint32m4_t test_sf_vqmaccus_4x8x4_i32m4_tu(vint32m4_t vd, vuint8m1_t vs1, vint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccus_4x8x4_i32m4_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x i32> @test_sf_vqmaccus_4x8x4_i32m8_tu
+// CHECK-RV64-SAME: (<vscale x 16 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vqmaccus.4x8x4.nxv16i32.nxv8i8.nxv64i8.i64(<vscale x 16 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 64 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vint32m8_t test_sf_vqmaccus_4x8x4_i32m8_tu(vint32m8_t vd, vuint8m1_t vs1, vint8m8_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccus_4x8x4_i32m8_tu(vd, vs1, vs2, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmacc_2x8x2.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmacc_2x8x2.c
new file mode 100644
index 0000000000..6d7ca5ab4e
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmacc_2x8x2.c
@@ -0,0 +1,47 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +xsfvqmaccdod \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x i32> @test_sf_vqmacc_2x8x2_i32m1_tu
+// CHECK-RV64-SAME: (<vscale x 2 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vqmacc.2x8x2.nxv2i32.nxv8i8.nxv8i8.i64(<vscale x 2 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 8 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vint32m1_t test_sf_vqmacc_2x8x2_i32m1_tu(vint32m1_t vd, vint8m1_t vs1, vint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vqmacc_2x8x2_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x i32> @test_sf_vqmacc_2x8x2_i32m2_tu
+// CHECK-RV64-SAME: (<vscale x 4 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vqmacc.2x8x2.nxv4i32.nxv8i8.nxv16i8.i64(<vscale x 4 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 16 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vint32m2_t test_sf_vqmacc_2x8x2_i32m2_tu(vint32m2_t vd, vint8m1_t vs1, vint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vqmacc_2x8x2_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x i32> @test_sf_vqmacc_2x8x2_i32m4_tu
+// CHECK-RV64-SAME: (<vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vqmacc.2x8x2.nxv8i32.nxv8i8.nxv32i8.i64(<vscale x 8 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 32 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vint32m4_t test_sf_vqmacc_2x8x2_i32m4_tu(vint32m4_t vd, vint8m1_t vs1, vint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vqmacc_2x8x2_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x i32> @test_sf_vqmacc_2x8x2_i32m8_tu
+// CHECK-RV64-SAME: (<vscale x 16 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vqmacc.2x8x2.nxv16i32.nxv8i8.nxv64i8.i64(<vscale x 16 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 64 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vint32m8_t test_sf_vqmacc_2x8x2_i32m8_tu(vint32m8_t vd, vint8m1_t vs1, vint8m8_t vs2, size_t vl) {
+  return __riscv_sf_vqmacc_2x8x2_tu(vd, vs1, vs2, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmacc_4x8x4.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmacc_4x8x4.c
new file mode 100644
index 0000000000..531bc2b2b9
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmacc_4x8x4.c
@@ -0,0 +1,47 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +xsfvqmaccqoq \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x i32> @test_sf_vqmacc_4x8x4_i32m1_tu
+// CHECK-RV64-SAME: (<vscale x 2 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vqmacc.4x8x4.nxv2i32.nxv8i8.nxv8i8.i64(<vscale x 2 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 8 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vint32m1_t test_sf_vqmacc_4x8x4_i32m1_tu(vint32m1_t vd, vint8m1_t vs1, vint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vqmacc_4x8x4_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x i32> @test_sf_vqmacc_4x8x4_i32m2_tu
+// CHECK-RV64-SAME: (<vscale x 4 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vqmacc.4x8x4.nxv4i32.nxv8i8.nxv16i8.i64(<vscale x 4 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 16 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vint32m2_t test_sf_vqmacc_4x8x4_i32m2_tu(vint32m2_t vd, vint8m1_t vs1, vint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vqmacc_4x8x4_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x i32> @test_sf_vqmacc_4x8x4_i32m4_tu
+// CHECK-RV64-SAME: (<vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vqmacc.4x8x4.nxv8i32.nxv8i8.nxv32i8.i64(<vscale x 8 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 32 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vint32m4_t test_sf_vqmacc_4x8x4_i32m4_tu(vint32m4_t vd, vint8m1_t vs1, vint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vqmacc_4x8x4_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x i32> @test_sf_vqmacc_4x8x4_i32m8_tu
+// CHECK-RV64-SAME: (<vscale x 16 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vqmacc.4x8x4.nxv16i32.nxv8i8.nxv64i8.i64(<vscale x 16 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 64 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vint32m8_t test_sf_vqmacc_4x8x4_i32m8_tu(vint32m8_t vd, vint8m1_t vs1, vint8m8_t vs2, size_t vl) {
+  return __riscv_sf_vqmacc_4x8x4_tu(vd, vs1, vs2, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccsu_2x8x2.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccsu_2x8x2.c
new file mode 100644
index 0000000000..ba6dae595b
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccsu_2x8x2.c
@@ -0,0 +1,47 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +xsfvqmaccdod \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x i32> @test_sf_vqmaccsu_2x8x2_i32m1_tu
+// CHECK-RV64-SAME: (<vscale x 2 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vqmaccsu.2x8x2.nxv2i32.nxv8i8.nxv8i8.i64(<vscale x 2 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 8 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vint32m1_t test_sf_vqmaccsu_2x8x2_i32m1_tu(vint32m1_t vd, vint8m1_t vs1, vuint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccsu_2x8x2_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x i32> @test_sf_vqmaccsu_2x8x2_i32m2_tu
+// CHECK-RV64-SAME: (<vscale x 4 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vqmaccsu.2x8x2.nxv4i32.nxv8i8.nxv16i8.i64(<vscale x 4 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 16 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vint32m2_t test_sf_vqmaccsu_2x8x2_i32m2_tu(vint32m2_t vd, vint8m1_t vs1, vuint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccsu_2x8x2_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x i32> @test_sf_vqmaccsu_2x8x2_i32m4_tu
+// CHECK-RV64-SAME: (<vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vqmaccsu.2x8x2.nxv8i32.nxv8i8.nxv32i8.i64(<vscale x 8 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 32 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vint32m4_t test_sf_vqmaccsu_2x8x2_i32m4_tu(vint32m4_t vd, vint8m1_t vs1, vuint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccsu_2x8x2_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x i32> @test_sf_vqmaccsu_2x8x2_i32m8_tu
+// CHECK-RV64-SAME: (<vscale x 16 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vqmaccsu.2x8x2.nxv16i32.nxv8i8.nxv64i8.i64(<vscale x 16 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 64 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vint32m8_t test_sf_vqmaccsu_2x8x2_i32m8_tu(vint32m8_t vd, vint8m1_t vs1, vuint8m8_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccsu_2x8x2_tu(vd, vs1, vs2, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccsu_4x8x4.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccsu_4x8x4.c
new file mode 100644
index 0000000000..23bba523aa
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccsu_4x8x4.c
@@ -0,0 +1,47 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +xsfvqmaccqoq \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x i32> @test_sf_vqmaccsu_4x8x4_i32m1_tu
+// CHECK-RV64-SAME: (<vscale x 2 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vqmaccsu.4x8x4.nxv2i32.nxv8i8.nxv8i8.i64(<vscale x 2 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 8 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vint32m1_t test_sf_vqmaccsu_4x8x4_i32m1_tu(vint32m1_t vd, vint8m1_t vs1, vuint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccsu_4x8x4_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x i32> @test_sf_vqmaccsu_4x8x4_i32m2_tu
+// CHECK-RV64-SAME: (<vscale x 4 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vqmaccsu.4x8x4.nxv4i32.nxv8i8.nxv16i8.i64(<vscale x 4 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 16 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vint32m2_t test_sf_vqmaccsu_4x8x4_i32m2_tu(vint32m2_t vd, vint8m1_t vs1, vuint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccsu_4x8x4_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x i32> @test_sf_vqmaccsu_4x8x4_i32m4_tu
+// CHECK-RV64-SAME: (<vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vqmaccsu.4x8x4.nxv8i32.nxv8i8.nxv32i8.i64(<vscale x 8 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 32 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vint32m4_t test_sf_vqmaccsu_4x8x4_i32m4_tu(vint32m4_t vd, vint8m1_t vs1, vuint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccsu_4x8x4_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x i32> @test_sf_vqmaccsu_4x8x4_i32m8_tu
+// CHECK-RV64-SAME: (<vscale x 16 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vqmaccsu.4x8x4.nxv16i32.nxv8i8.nxv64i8.i64(<vscale x 16 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 64 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vint32m8_t test_sf_vqmaccsu_4x8x4_i32m8_tu(vint32m8_t vd, vint8m1_t vs1, vuint8m8_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccsu_4x8x4_tu(vd, vs1, vs2, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccu_2x8x2.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccu_2x8x2.c
new file mode 100644
index 0000000000..9e15b053da
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccu_2x8x2.c
@@ -0,0 +1,47 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +xsfvqmaccdod \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x i32> @test_sf_vqmaccu_2x8x2_i32m1_tu
+// CHECK-RV64-SAME: (<vscale x 2 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vqmaccu.2x8x2.nxv2i32.nxv8i8.nxv8i8.i64(<vscale x 2 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 8 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vint32m1_t test_sf_vqmaccu_2x8x2_i32m1_tu(vint32m1_t vd, vuint8m1_t vs1, vuint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccu_2x8x2_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x i32> @test_sf_vqmaccu_2x8x2_i32m2_tu
+// CHECK-RV64-SAME: (<vscale x 4 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vqmaccu.2x8x2.nxv4i32.nxv8i8.nxv16i8.i64(<vscale x 4 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 16 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vint32m2_t test_sf_vqmaccu_2x8x2_i32m2_tu(vint32m2_t vd, vuint8m1_t vs1, vuint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccu_2x8x2_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x i32> @test_sf_vqmaccu_2x8x2_i32m4_tu
+// CHECK-RV64-SAME: (<vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vqmaccu.2x8x2.nxv8i32.nxv8i8.nxv32i8.i64(<vscale x 8 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 32 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vint32m4_t test_sf_vqmaccu_2x8x2_i32m4_tu(vint32m4_t vd, vuint8m1_t vs1, vuint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccu_2x8x2_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x i32> @test_sf_vqmaccu_2x8x2_i32m8_tu
+// CHECK-RV64-SAME: (<vscale x 16 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vqmaccu.2x8x2.nxv16i32.nxv8i8.nxv64i8.i64(<vscale x 16 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 64 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vint32m8_t test_sf_vqmaccu_2x8x2_i32m8_tu(vint32m8_t vd, vuint8m1_t vs1, vuint8m8_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccu_2x8x2_tu(vd, vs1, vs2, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccu_4x8x4.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccu_4x8x4.c
new file mode 100644
index 0000000000..950688c6c7
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccu_4x8x4.c
@@ -0,0 +1,47 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +xsfvqmaccqoq \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x i32> @test_sf_vqmaccu_4x8x4_i32m1_tu
+// CHECK-RV64-SAME: (<vscale x 2 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vqmaccu.4x8x4.nxv2i32.nxv8i8.nxv8i8.i64(<vscale x 2 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 8 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vint32m1_t test_sf_vqmaccu_4x8x4_i32m1_tu(vint32m1_t vd, vuint8m1_t vs1, vuint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccu_4x8x4_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x i32> @test_sf_vqmaccu_4x8x4_i32m2_tu
+// CHECK-RV64-SAME: (<vscale x 4 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vqmaccu.4x8x4.nxv4i32.nxv8i8.nxv16i8.i64(<vscale x 4 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 16 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vint32m2_t test_sf_vqmaccu_4x8x4_i32m2_tu(vint32m2_t vd, vuint8m1_t vs1, vuint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccu_4x8x4_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x i32> @test_sf_vqmaccu_4x8x4_i32m4_tu
+// CHECK-RV64-SAME: (<vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vqmaccu.4x8x4.nxv8i32.nxv8i8.nxv32i8.i64(<vscale x 8 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 32 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vint32m4_t test_sf_vqmaccu_4x8x4_i32m4_tu(vint32m4_t vd, vuint8m1_t vs1, vuint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccu_4x8x4_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x i32> @test_sf_vqmaccu_4x8x4_i32m8_tu
+// CHECK-RV64-SAME: (<vscale x 16 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vqmaccu.4x8x4.nxv16i32.nxv8i8.nxv64i8.i64(<vscale x 16 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 64 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vint32m8_t test_sf_vqmaccu_4x8x4_i32m8_tu(vint32m8_t vd, vuint8m1_t vs1, vuint8m8_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccu_4x8x4_tu(vd, vs1, vs2, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccus_2x8x2.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccus_2x8x2.c
new file mode 100644
index 0000000000..3108560c5b
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccus_2x8x2.c
@@ -0,0 +1,47 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +xsfvqmaccdod \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x i32> @test_sf_vqmaccus_2x8x2_i32m1_tu
+// CHECK-RV64-SAME: (<vscale x 2 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vqmaccus.2x8x2.nxv2i32.nxv8i8.nxv8i8.i64(<vscale x 2 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 8 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vint32m1_t test_sf_vqmaccus_2x8x2_i32m1_tu(vint32m1_t vd, vuint8m1_t vs1, vint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccus_2x8x2_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x i32> @test_sf_vqmaccus_2x8x2_i32m2_tu
+// CHECK-RV64-SAME: (<vscale x 4 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vqmaccus.2x8x2.nxv4i32.nxv8i8.nxv16i8.i64(<vscale x 4 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 16 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vint32m2_t test_sf_vqmaccus_2x8x2_i32m2_tu(vint32m2_t vd, vuint8m1_t vs1, vint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccus_2x8x2_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x i32> @test_sf_vqmaccus_2x8x2_i32m4_tu
+// CHECK-RV64-SAME: (<vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vqmaccus.2x8x2.nxv8i32.nxv8i8.nxv32i8.i64(<vscale x 8 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 32 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vint32m4_t test_sf_vqmaccus_2x8x2_i32m4_tu(vint32m4_t vd, vuint8m1_t vs1, vint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccus_2x8x2_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x i32> @test_sf_vqmaccus_2x8x2_i32m8_tu
+// CHECK-RV64-SAME: (<vscale x 16 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vqmaccus.2x8x2.nxv16i32.nxv8i8.nxv64i8.i64(<vscale x 16 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 64 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vint32m8_t test_sf_vqmaccus_2x8x2_i32m8_tu(vint32m8_t vd, vuint8m1_t vs1, vint8m8_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccus_2x8x2_tu(vd, vs1, vs2, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccus_4x8x4.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccus_4x8x4.c
new file mode 100644
index 0000000000..7bdce95043
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccus_4x8x4.c
@@ -0,0 +1,47 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +xsfvqmaccqoq \
+// RUN:   -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <sifive_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x i32> @test_sf_vqmaccus_4x8x4_i32m1_tu
+// CHECK-RV64-SAME: (<vscale x 2 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 8 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x i32> @llvm.riscv.sf.vqmaccus.4x8x4.nxv2i32.nxv8i8.nxv8i8.i64(<vscale x 2 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 8 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 2 x i32> [[TMP0]]
+//
+vint32m1_t test_sf_vqmaccus_4x8x4_i32m1_tu(vint32m1_t vd, vuint8m1_t vs1, vint8m1_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccus_4x8x4_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x i32> @test_sf_vqmaccus_4x8x4_i32m2_tu
+// CHECK-RV64-SAME: (<vscale x 4 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 16 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x i32> @llvm.riscv.sf.vqmaccus.4x8x4.nxv4i32.nxv8i8.nxv16i8.i64(<vscale x 4 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 16 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+vint32m2_t test_sf_vqmaccus_4x8x4_i32m2_tu(vint32m2_t vd, vuint8m1_t vs1, vint8m2_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccus_4x8x4_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x i32> @test_sf_vqmaccus_4x8x4_i32m4_tu
+// CHECK-RV64-SAME: (<vscale x 8 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 32 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x i32> @llvm.riscv.sf.vqmaccus.4x8x4.nxv8i32.nxv8i8.nxv32i8.i64(<vscale x 8 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 32 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 8 x i32> [[TMP0]]
+//
+vint32m4_t test_sf_vqmaccus_4x8x4_i32m4_tu(vint32m4_t vd, vuint8m1_t vs1, vint8m4_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccus_4x8x4_tu(vd, vs1, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x i32> @test_sf_vqmaccus_4x8x4_i32m8_tu
+// CHECK-RV64-SAME: (<vscale x 16 x i32> [[VD:%.*]], <vscale x 8 x i8> [[VS1:%.*]], <vscale x 64 x i8> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x i32> @llvm.riscv.sf.vqmaccus.4x8x4.nxv16i32.nxv8i8.nxv64i8.i64(<vscale x 16 x i32> [[VD]], <vscale x 8 x i8> [[VS1]], <vscale x 64 x i8> [[VS2]], i64 [[VL]], i64 2)
+// CHECK-RV64-NEXT:    ret <vscale x 16 x i32> [[TMP0]]
+//
+vint32m8_t test_sf_vqmaccus_4x8x4_i32m8_tu(vint32m8_t vd, vuint8m1_t vs1, vint8m8_t vs2, size_t vl) {
+  return __riscv_sf_vqmaccus_4x8x4_tu(vd, vs1, vs2, vl);
+}
diff --git a/clang/test/CodeGen/X86/avx512-error.c b/clang/test/CodeGen/X86/avx512-error.c
index 8e42105ec1..133e7d01ea 100644
--- a/clang/test/CodeGen/X86/avx512-error.c
+++ b/clang/test/CodeGen/X86/avx512-error.c
@@ -1,9 +1,17 @@
-// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +avx512bw -target-feature -evex512 -emit-llvm -o /dev/null -verify -DFEATURE_TEST=1
-// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +avx512bw -target-feature -evex512 -emit-llvm -o /dev/null -verify -DFEATURE_TEST=2
+// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +avx512bw -target-feature -evex512 -emit-llvm -o /dev/null -verify=noevex -DFEATURE_TEST=1
+// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +avx512bw -target-feature -evex512 -emit-llvm -o /dev/null -verify=noevex -DFEATURE_TEST=2
+// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +avx512bw -emit-llvm -o /dev/null -verify -DFEATURE_TEST=3
+// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +avx10.1-256 -emit-llvm -o /dev/null -verify=noevex -DFEATURE_TEST=1
+// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +avx10.1-256 -emit-llvm -o /dev/null -verify=noevex -DFEATURE_TEST=2
+// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +avx10.1-512 -emit-llvm -o /dev/null -verify -DFEATURE_TEST=3
 
 #include <immintrin.h>
 
-#if FEATURE_TEST == 1
+#if FEATURE_TEST & 3
+// expected-no-diagnostics
+#endif
+
+#if FEATURE_TEST & 1
 __attribute__((target("avx512bw,evex512")))
 __m512d zmm_verify_ok(__m512d a) {
   // No error emitted if we have "evex512" feature.
@@ -12,11 +20,11 @@ __m512d zmm_verify_ok(__m512d a) {
 
 __m512d zmm_error(__m512d a) {
   // CHECK-LABEL: @test_mm512_sqrt_pd
-  return __builtin_ia32_sqrtpd512(a, _MM_FROUND_CUR_DIRECTION); // expected-error {{'__builtin_ia32_sqrtpd512' needs target feature avx512f,evex512}}
+  return __builtin_ia32_sqrtpd512(a, _MM_FROUND_CUR_DIRECTION); // noevex-error {{'__builtin_ia32_sqrtpd512' needs target feature avx512f,evex512}}
 }
 #endif
 
-#if FEATURE_TEST == 2
+#if FEATURE_TEST & 2
 __attribute__((target("avx512bw,evex512")))
 __mmask64 k64_verify_ok(__mmask64 a) {
   // No error emitted if we have "evex512" feature.
@@ -24,6 +32,6 @@ __mmask64 k64_verify_ok(__mmask64 a) {
 }
 
 __mmask64 test_knot_mask64(__mmask64 a) {
-  return _knot_mask64(a); // expected-error {{always_inline function '_knot_mask64' requires target feature 'evex512', but would be inlined into function 'test_knot_mask64' that is compiled without support for 'evex512'}}
+  return _knot_mask64(a); // noevex-error {{always_inline function '_knot_mask64' requires target feature 'evex512', but would be inlined into function 'test_knot_mask64' that is compiled without support for 'evex512'}}
 }
 #endif
diff --git a/clang/test/CodeGen/X86/usermsr-builtins-error-32.c b/clang/test/CodeGen/X86/usermsr-builtins-error-32.c
new file mode 100644
index 0000000000..180b99a421
--- /dev/null
+++ b/clang/test/CodeGen/X86/usermsr-builtins-error-32.c
@@ -0,0 +1,14 @@
+// RUN: %clang_cc1 %s -ffreestanding -triple=i386-unknown-unknown -target-feature +usermsr \
+// RUN: -emit-llvm -fsyntax-only -verify
+
+#include <x86gprintrin.h>
+
+unsigned long long test_urdmsr(unsigned long long __A) {
+  return _urdmsr(__A); // expected-error {{call to undeclared function '_urdmsr'}}
+}
+
+void test_uwrmsr(unsigned long long __A, unsigned long long __B) {
+  // CHECK-LABEL: @test_uwrmsr(
+  // CHECK: call void @llvm.x86.uwrmsr(
+  _uwrmsr(__A, __B); // expected-error {{call to undeclared function '_uwrmsr'}}
+}
diff --git a/clang/test/CodeGen/X86/usermsr-builtins.c b/clang/test/CodeGen/X86/usermsr-builtins.c
new file mode 100644
index 0000000000..0d58bc98c2
--- /dev/null
+++ b/clang/test/CodeGen/X86/usermsr-builtins.c
@@ -0,0 +1,29 @@
+// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +usermsr \
+// RUN: -emit-llvm -o - -Wall -Werror -pedantic -Wno-gnu-statement-expression | FileCheck %s
+
+#include <x86gprintrin.h>
+
+unsigned long long test_urdmsr(unsigned long long __A) {
+  // CHECK-LABEL: @test_urdmsr(
+  // CHECK: call i64 @llvm.x86.urdmsr(
+  return _urdmsr(__A);
+}
+
+unsigned long long test_urdmsr_const(unsigned long long __A) {
+  // CHECK-LABEL: @test_urdmsr_const(
+  // CHECK: call i64 @llvm.x86.urdmsr(
+  return _urdmsr(123u);
+}
+
+void test_uwrmsr(unsigned long long __A, unsigned long long __B) {
+  // CHECK-LABEL: @test_uwrmsr(
+  // CHECK: call void @llvm.x86.uwrmsr(
+  _uwrmsr(__A, __B);
+}
+
+void test_uwrmsr_const(unsigned long long __A, unsigned long long __B) {
+  // CHECK-LABEL: @test_uwrmsr_const(
+  // CHECK: call void @llvm.x86.uwrmsr(
+  _uwrmsr(123u, __B);
+}
+
diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_add-i32.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_add-i32.c
index b0855553df..1b181bd59e 100644
--- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_add-i32.c
+++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_add-i32.c
@@ -1,3 +1,4 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 3
 // REQUIRES: aarch64-registered-target
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s -check-prefixes=CHECK,CHECK-C
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefixes=CHECK,CHECK-CXX
@@ -13,98 +14,164 @@
 #define SME_ACLE_FUNC(A1,A2,A3) A1##A2##A3
 #endif
 
-// CHECK-C-LABEL: @test_svaddha_za32_u32(
-// CHECK-CXX-LABEL: @_Z21test_svaddha_za32_u32u10__SVBool_tu10__SVBool_tu12__SVUint32_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PN:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PM:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.addha.nxv4i32(i32 0, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x i32> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svaddha_za32_u32(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PN]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PM]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.addha.nxv4i32(i32 0, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x i32> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z21test_svaddha_za32_u32u10__SVBool_tu10__SVBool_tu12__SVUint32_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PN]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PM]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.addha.nxv4i32(i32 0, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x i32> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svaddha_za32_u32(svbool_t pn, svbool_t pm, svuint32_t zn) {
   SME_ACLE_FUNC(svaddha_za32, _u32, _m)(0, pn, pm, zn);
 }
 
-// CHECK-C-LABEL: @test_svaddha_za32_u32_1(
-// CHECK-CXX-LABEL: @_Z23test_svaddha_za32_u32_1u10__SVBool_tu10__SVBool_tu12__SVUint32_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PN:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PM:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.addha.nxv4i32(i32 3, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x i32> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svaddha_za32_u32_1(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PN]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PM]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.addha.nxv4i32(i32 3, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x i32> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z23test_svaddha_za32_u32_1u10__SVBool_tu10__SVBool_tu12__SVUint32_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PN]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PM]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.addha.nxv4i32(i32 3, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x i32> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svaddha_za32_u32_1(svbool_t pn, svbool_t pm, svuint32_t zn) {
   SME_ACLE_FUNC(svaddha_za32, _u32, _m)(3, pn, pm, zn);
 }
 
-// CHECK-C-LABEL: @test_svaddha_za32_s32(
-// CHECK-CXX-LABEL: @_Z21test_svaddha_za32_s32u10__SVBool_tu10__SVBool_tu11__SVInt32_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PN:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PM:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.addha.nxv4i32(i32 0, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x i32> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svaddha_za32_s32(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PN]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PM]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.addha.nxv4i32(i32 0, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x i32> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z21test_svaddha_za32_s32u10__SVBool_tu10__SVBool_tu11__SVInt32_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PN]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PM]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.addha.nxv4i32(i32 0, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x i32> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svaddha_za32_s32(svbool_t pn, svbool_t pm, svint32_t zn) {
   SME_ACLE_FUNC(svaddha_za32, _s32, _m)(0, pn, pm, zn);
 }
 
-// CHECK-C-LABEL: @test_svaddha_za32_s32_1(
-// CHECK-CXX-LABEL: @_Z23test_svaddha_za32_s32_1u10__SVBool_tu10__SVBool_tu11__SVInt32_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PN:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PM:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.addha.nxv4i32(i32 3, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x i32> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svaddha_za32_s32_1(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PN]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PM]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.addha.nxv4i32(i32 3, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x i32> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z23test_svaddha_za32_s32_1u10__SVBool_tu10__SVBool_tu11__SVInt32_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PN]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PM]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.addha.nxv4i32(i32 3, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x i32> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svaddha_za32_s32_1(svbool_t pn, svbool_t pm, svint32_t zn) {
   SME_ACLE_FUNC(svaddha_za32, _s32, _m)(3, pn, pm, zn);
 }
 
-// CHECK-C-LABEL: @test_svaddva_za32_u32(
-// CHECK-CXX-LABEL: @_Z21test_svaddva_za32_u32u10__SVBool_tu10__SVBool_tu12__SVUint32_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PN:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PM:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.addva.nxv4i32(i32 0, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x i32> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svaddva_za32_u32(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PN]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PM]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.addva.nxv4i32(i32 0, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x i32> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z21test_svaddva_za32_u32u10__SVBool_tu10__SVBool_tu12__SVUint32_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PN]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PM]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.addva.nxv4i32(i32 0, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x i32> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svaddva_za32_u32(svbool_t pn, svbool_t pm, svuint32_t zn) {
   SME_ACLE_FUNC(svaddva_za32, _u32, _m)(0, pn, pm, zn);
 }
 
-// CHECK-C-LABEL: @test_svaddva_za32_u32_1(
-// CHECK-CXX-LABEL: @_Z23test_svaddva_za32_u32_1u10__SVBool_tu10__SVBool_tu12__SVUint32_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PN:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PM:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.addva.nxv4i32(i32 3, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x i32> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svaddva_za32_u32_1(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PN]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PM]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.addva.nxv4i32(i32 3, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x i32> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z23test_svaddva_za32_u32_1u10__SVBool_tu10__SVBool_tu12__SVUint32_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PN]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PM]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.addva.nxv4i32(i32 3, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x i32> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svaddva_za32_u32_1(svbool_t pn, svbool_t pm, svuint32_t zn) {
   SME_ACLE_FUNC(svaddva_za32, _u32, _m)(3, pn, pm, zn);
 }
 
-// CHECK-C-LABEL: @test_svaddva_za32_s32(
-// CHECK-CXX-LABEL: @_Z21test_svaddva_za32_s32u10__SVBool_tu10__SVBool_tu11__SVInt32_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PN:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PM:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.addva.nxv4i32(i32 0, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x i32> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svaddva_za32_s32(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PN]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PM]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.addva.nxv4i32(i32 0, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x i32> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z21test_svaddva_za32_s32u10__SVBool_tu10__SVBool_tu11__SVInt32_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PN]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PM]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.addva.nxv4i32(i32 0, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x i32> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svaddva_za32_s32(svbool_t pn, svbool_t pm, svint32_t zn) {
   SME_ACLE_FUNC(svaddva_za32, _s32, _m)(0, pn, pm, zn);
 }
 
-// CHECK-C-LABEL: @test_svaddva_za32_s32_1(
-// CHECK-CXX-LABEL: @_Z23test_svaddva_za32_s32_1u10__SVBool_tu10__SVBool_tu11__SVInt32_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PN:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PM:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.addva.nxv4i32(i32 3, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x i32> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svaddva_za32_s32_1(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PN]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PM]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.addva.nxv4i32(i32 3, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x i32> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z23test_svaddva_za32_s32_1u10__SVBool_tu10__SVBool_tu11__SVInt32_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PN]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PM]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.addva.nxv4i32(i32 3, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x i32> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svaddva_za32_s32_1(svbool_t pn, svbool_t pm, svint32_t zn) {
   SME_ACLE_FUNC(svaddva_za32, _s32, _m)(3, pn, pm, zn);
 }
+//// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+// CHECK: {{.*}}
diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_add-i64.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_add-i64.c
index 2f0f97e742..fdccafbebf 100644
--- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_add-i64.c
+++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_add-i64.c
@@ -1,3 +1,4 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 3
 // REQUIRES: aarch64-registered-target
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme-i16i64 -target-feature +sve -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s -check-prefixes=CHECK,CHECK-C
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme-i16i64 -target-feature +sve -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefixes=CHECK,CHECK-CXX
@@ -13,98 +14,164 @@
 #define SME_ACLE_FUNC(A1,A2,A3) A1##A2##A3
 #endif
 
-// CHECK-C-LABEL: @test_svaddha_za64_u64(
-// CHECK-CXX-LABEL: @_Z21test_svaddha_za64_u64u10__SVBool_tu10__SVBool_tu12__SVUint64_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PN:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PM:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.addha.nxv2i64(i32 0, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i1> [[TMP1]], <vscale x 2 x i64> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svaddha_za64_u64(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PN]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PM]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.addha.nxv2i64(i32 0, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i1> [[TMP1]], <vscale x 2 x i64> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z21test_svaddha_za64_u64u10__SVBool_tu10__SVBool_tu12__SVUint64_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PN]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PM]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.addha.nxv2i64(i32 0, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i1> [[TMP1]], <vscale x 2 x i64> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svaddha_za64_u64(svbool_t pn, svbool_t pm, svuint64_t zn) {
   SME_ACLE_FUNC(svaddha_za64, _u64, _m)(0, pn, pm, zn);
 }
 
-// CHECK-C-LABEL: @test_svaddha_za64_u64_1(
-// CHECK-CXX-LABEL: @_Z23test_svaddha_za64_u64_1u10__SVBool_tu10__SVBool_tu12__SVUint64_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PN:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PM:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.addha.nxv2i64(i32 7, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i1> [[TMP1]], <vscale x 2 x i64> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svaddha_za64_u64_1(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PN]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PM]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.addha.nxv2i64(i32 7, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i1> [[TMP1]], <vscale x 2 x i64> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z23test_svaddha_za64_u64_1u10__SVBool_tu10__SVBool_tu12__SVUint64_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PN]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PM]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.addha.nxv2i64(i32 7, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i1> [[TMP1]], <vscale x 2 x i64> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svaddha_za64_u64_1(svbool_t pn, svbool_t pm, svuint64_t zn) {
   SME_ACLE_FUNC(svaddha_za64, _u64, _m)(7, pn, pm, zn);
 }
 
-// CHECK-C-LABEL: @test_svaddha_za64_s64(
-// CHECK-CXX-LABEL: @_Z21test_svaddha_za64_s64u10__SVBool_tu10__SVBool_tu11__SVInt64_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PN:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PM:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.addha.nxv2i64(i32 0, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i1> [[TMP1]], <vscale x 2 x i64> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svaddha_za64_s64(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PN]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PM]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.addha.nxv2i64(i32 0, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i1> [[TMP1]], <vscale x 2 x i64> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z21test_svaddha_za64_s64u10__SVBool_tu10__SVBool_tu11__SVInt64_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PN]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PM]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.addha.nxv2i64(i32 0, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i1> [[TMP1]], <vscale x 2 x i64> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svaddha_za64_s64(svbool_t pn, svbool_t pm, svint64_t zn) {
   SME_ACLE_FUNC(svaddha_za64, _s64, _m)(0, pn, pm, zn);
 }
 
-// CHECK-C-LABEL: @test_svaddha_za64_s64_1(
-// CHECK-CXX-LABEL: @_Z23test_svaddha_za64_s64_1u10__SVBool_tu10__SVBool_tu11__SVInt64_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PN:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PM:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.addha.nxv2i64(i32 7, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i1> [[TMP1]], <vscale x 2 x i64> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svaddha_za64_s64_1(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PN]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PM]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.addha.nxv2i64(i32 7, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i1> [[TMP1]], <vscale x 2 x i64> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z23test_svaddha_za64_s64_1u10__SVBool_tu10__SVBool_tu11__SVInt64_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PN]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PM]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.addha.nxv2i64(i32 7, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i1> [[TMP1]], <vscale x 2 x i64> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svaddha_za64_s64_1(svbool_t pn, svbool_t pm, svint64_t zn) {
   SME_ACLE_FUNC(svaddha_za64, _s64, _m)(7, pn, pm, zn);
 }
 
-// CHECK-C-LABEL: @test_svaddva_za64_u64(
-// CHECK-CXX-LABEL: @_Z21test_svaddva_za64_u64u10__SVBool_tu10__SVBool_tu12__SVUint64_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PN:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PM:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.addva.nxv2i64(i32 0, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i1> [[TMP1]], <vscale x 2 x i64> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svaddva_za64_u64(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PN]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PM]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.addva.nxv2i64(i32 0, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i1> [[TMP1]], <vscale x 2 x i64> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z21test_svaddva_za64_u64u10__SVBool_tu10__SVBool_tu12__SVUint64_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PN]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PM]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.addva.nxv2i64(i32 0, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i1> [[TMP1]], <vscale x 2 x i64> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svaddva_za64_u64(svbool_t pn, svbool_t pm, svuint64_t zn) {
   SME_ACLE_FUNC(svaddva_za64, _u64, _m)(0, pn, pm, zn);
 }
 
-// CHECK-C-LABEL: @test_svaddva_za64_u64_1(
-// CHECK-CXX-LABEL: @_Z23test_svaddva_za64_u64_1u10__SVBool_tu10__SVBool_tu12__SVUint64_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PN:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PM:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.addva.nxv2i64(i32 7, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i1> [[TMP1]], <vscale x 2 x i64> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svaddva_za64_u64_1(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PN]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PM]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.addva.nxv2i64(i32 7, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i1> [[TMP1]], <vscale x 2 x i64> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z23test_svaddva_za64_u64_1u10__SVBool_tu10__SVBool_tu12__SVUint64_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PN]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PM]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.addva.nxv2i64(i32 7, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i1> [[TMP1]], <vscale x 2 x i64> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svaddva_za64_u64_1(svbool_t pn, svbool_t pm, svuint64_t zn) {
   SME_ACLE_FUNC(svaddva_za64, _u64, _m)(7, pn, pm, zn);
 }
 
-// CHECK-C-LABEL: @test_svaddva_za64_s64(
-// CHECK-CXX-LABEL: @_Z21test_svaddva_za64_s64u10__SVBool_tu10__SVBool_tu11__SVInt64_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PN:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PM:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.addva.nxv2i64(i32 0, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i1> [[TMP1]], <vscale x 2 x i64> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svaddva_za64_s64(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PN]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PM]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.addva.nxv2i64(i32 0, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i1> [[TMP1]], <vscale x 2 x i64> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z21test_svaddva_za64_s64u10__SVBool_tu10__SVBool_tu11__SVInt64_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PN]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PM]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.addva.nxv2i64(i32 0, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i1> [[TMP1]], <vscale x 2 x i64> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svaddva_za64_s64(svbool_t pn, svbool_t pm, svint64_t zn) {
   SME_ACLE_FUNC(svaddva_za64, _s64, _m)(0, pn, pm, zn);
 }
 
-// CHECK-C-LABEL: @test_svaddva_za64_s64_1(
-// CHECK-CXX-LABEL: @_Z23test_svaddva_za64_s64_1u10__SVBool_tu10__SVBool_tu11__SVInt64_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PN:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PM:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.addva.nxv2i64(i32 7, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i1> [[TMP1]], <vscale x 2 x i64> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svaddva_za64_s64_1(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PN]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PM]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.addva.nxv2i64(i32 7, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i1> [[TMP1]], <vscale x 2 x i64> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z23test_svaddva_za64_s64_1u10__SVBool_tu10__SVBool_tu11__SVInt64_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PN]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PM]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.addva.nxv2i64(i32 7, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i1> [[TMP1]], <vscale x 2 x i64> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svaddva_za64_s64_1(svbool_t pn, svbool_t pm, svint64_t zn) {
   SME_ACLE_FUNC(svaddva_za64, _s64, _m)(7, pn, pm, zn);
 }
+//// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+// CHECK: {{.*}}
diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_cnt.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_cnt.c
index b3b2499a38..09b183dac3 100644
--- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_cnt.c
+++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_cnt.c
@@ -1,3 +1,4 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 3
 // REQUIRES: aarch64-registered-target
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s -check-prefixes=CHECK,CHECK-C
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefixes=CHECK,CHECK-CXX
@@ -5,42 +6,68 @@
 
 #include <arm_sme_draft_spec_subject_to_change.h>
 
-// CHECK-C-LABEL: @test_svcntsb(
-// CHECK-CXX-LABEL: @_Z12test_svcntsbv(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-NEXT:    ret i64 [[TMP0]]
+// CHECK-C-LABEL: define dso_local i64 @test_svcntsb(
+// CHECK-C-SAME: ) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-C-NEXT:    ret i64 [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef i64 @_Z12test_svcntsbv(
+// CHECK-CXX-SAME: ) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-CXX-NEXT:    ret i64 [[TMP0]]
 //
 uint64_t test_svcntsb() {
   return svcntsb();
 }
 
-// CHECK-C-LABEL: @test_svcntsh(
-// CHECK-CXX-LABEL: @_Z12test_svcntshv(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsh()
-// CHECK-NEXT:    ret i64 [[TMP0]]
+// CHECK-C-LABEL: define dso_local i64 @test_svcntsh(
+// CHECK-C-SAME: ) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsh()
+// CHECK-C-NEXT:    ret i64 [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef i64 @_Z12test_svcntshv(
+// CHECK-CXX-SAME: ) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsh()
+// CHECK-CXX-NEXT:    ret i64 [[TMP0]]
 //
 uint64_t test_svcntsh() {
   return svcntsh();
 }
 
-// CHECK-C-LABEL: @test_svcntsw(
-// CHECK-CXX-LABEL: @_Z12test_svcntswv(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsw()
-// CHECK-NEXT:    ret i64 [[TMP0]]
+// CHECK-C-LABEL: define dso_local i64 @test_svcntsw(
+// CHECK-C-SAME: ) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsw()
+// CHECK-C-NEXT:    ret i64 [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef i64 @_Z12test_svcntswv(
+// CHECK-CXX-SAME: ) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsw()
+// CHECK-CXX-NEXT:    ret i64 [[TMP0]]
 //
 uint64_t test_svcntsw() {
   return svcntsw();
 }
 
-// CHECK-C-LABEL: @test_svcntsd(
-// CHECK-CXX-LABEL: @_Z12test_svcntsdv(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsd()
-// CHECK-NEXT:    ret i64 [[TMP0]]
+// CHECK-C-LABEL: define dso_local i64 @test_svcntsd(
+// CHECK-C-SAME: ) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsd()
+// CHECK-C-NEXT:    ret i64 [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local noundef i64 @_Z12test_svcntsdv(
+// CHECK-CXX-SAME: ) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsd()
+// CHECK-CXX-NEXT:    ret i64 [[TMP0]]
 //
 uint64_t test_svcntsd() {
   return svcntsd();
 }
+//// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+// CHECK: {{.*}}
diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_ld1.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_ld1.c
index 57ed469955..72d159b191 100644
--- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_ld1.c
+++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_ld1.c
@@ -1,3 +1,4 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 3
 // REQUIRES: aarch64-registered-target
 // RUN: %clang_cc1 -DDISABLE_SME_ATTRIBUTES -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s -check-prefixes=CHECK,CHECK-C
 // RUN: %clang_cc1 -DDISABLE_SME_ATTRIBUTES -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefixes=CHECK,CHECK-CXX
@@ -11,138 +12,226 @@
 #define ARM_STREAMING_ATTR __attribute__((arm_streaming))
 #endif
 
-// CHECK-C-LABEL:   @test_svld1_hor_za8(
-// CHECK-CXX-LABEL: @_Z18test_svld1_hor_za8ju10__SVBool_tPKv(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1b.horiz(<vscale x 16 x i1> [[PG:%.*]], [[PTRTY:ptr|i8\*]] [[PTR:%.*]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        [[TILESLICE1:%.*]] = add i32 [[SLICE_BASE]], 15
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1b.horiz(<vscale x 16 x i1> [[PG]], [[PTRTY]] [[PTR]], i32 0, i32 [[TILESLICE1]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svld1_hor_za8(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[PTR]], i32 0, i32 [[ADD]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z18test_svld1_hor_za8ju10__SVBool_tPKv(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[PTR]], i32 0, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svld1_hor_za8(uint32_t slice_base, svbool_t pg, const void *ptr) {
   svld1_hor_za8(0, slice_base, pg, ptr);
   svld1_hor_za8(0, slice_base + 15, pg, ptr);
 }
 
-// CHECK-C-LABEL:   @test_svld1_hor_za16(
-// CHECK-CXX-LABEL: @_Z19test_svld1_hor_za16ju10__SVBool_tPKv(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1h.horiz(<vscale x 8 x i1> [[TMP0]], [[PTRTY]] [[PTR:%.*]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        [[TILESLICE1:%.*]] = add i32 [[SLICE_BASE]], 7
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1h.horiz(<vscale x 8 x i1> [[TMP0]], [[PTRTY]] [[PTR]], i32 1, i32 [[TILESLICE1]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svld1_hor_za16(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[PTR]], i32 1, i32 [[ADD]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z19test_svld1_hor_za16ju10__SVBool_tPKv(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[PTR]], i32 1, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svld1_hor_za16(uint32_t slice_base, svbool_t pg, const void *ptr) {
   svld1_hor_za16(0, slice_base, pg, ptr);
   svld1_hor_za16(1, slice_base + 7, pg, ptr);
 }
 
-// CHECK-C-LABEL:   @test_svld1_hor_za32(
-// CHECK-CXX-LABEL: @_Z19test_svld1_hor_za32ju10__SVBool_tPKv(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> [[TMP0]], [[PTRTY]] [[PTR:%.*]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        [[TILESLICE1:%.*]] = add i32 [[SLICE_BASE]], 3
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> [[TMP0]], [[PTRTY]] [[PTR]], i32 3, i32 [[TILESLICE1]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svld1_hor_za32(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[PTR]], i32 3, i32 [[ADD]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z19test_svld1_hor_za32ju10__SVBool_tPKv(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[PTR]], i32 3, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svld1_hor_za32(uint32_t slice_base, svbool_t pg, const void *ptr) {
   svld1_hor_za32(0, slice_base, pg, ptr);
   svld1_hor_za32(3, slice_base + 3, pg, ptr);
 }
 
-// CHECK-C-LABEL:   @test_svld1_hor_za64(
-// CHECK-CXX-LABEL: @_Z19test_svld1_hor_za64ju10__SVBool_tPKv(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> [[TMP0]], [[PTRTY]] [[PTR:%.*]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        [[TILESLICE1:%.*]] = add i32 [[SLICE_BASE]], 1
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> [[TMP0]], [[PTRTY]] [[PTR]], i32 7, i32 [[TILESLICE1]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svld1_hor_za64(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[PTR]], i32 7, i32 [[ADD]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z19test_svld1_hor_za64ju10__SVBool_tPKv(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[PTR]], i32 7, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svld1_hor_za64(uint32_t slice_base, svbool_t pg, const void *ptr) {
   svld1_hor_za64(0, slice_base, pg, ptr);
   svld1_hor_za64(7, slice_base + 1, pg, ptr);
 }
 
-// CHECK-C-LABEL:   @test_svld1_hor_za128(
-// CHECK-CXX-LABEL: @_Z20test_svld1_hor_za128ju10__SVBool_tPKv(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        [[TMP0:%.*]] = tail call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> [[TMP0]], [[PTRTY]] [[PTR:%.*]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> [[TMP0]], [[PTRTY]] [[PTR]], i32 15, i32 [[SLICE_BASE]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svld1_hor_za128(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[PTR]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z20test_svld1_hor_za128ju10__SVBool_tPKv(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[PTR]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svld1_hor_za128(uint32_t slice_base, svbool_t pg, const void *ptr) {
   svld1_hor_za128(0, slice_base, pg, ptr);
   svld1_hor_za128(15, slice_base, pg, ptr);
 }
 
-// CHECK-C-LABEL: @test_svld1_ver_za8(
-// CHECK-CXX-LABEL: @_Z18test_svld1_ver_za8ju10__SVBool_tPKv(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1b.vert(<vscale x 16 x i1> [[PG:%.*]], [[PTRTY]] [[PTR:%.*]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        [[TILESLICE1:%.*]] = add i32 [[SLICE_BASE]], 15
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1b.vert(<vscale x 16 x i1> [[PG]], [[PTRTY]] [[PTR]], i32 0, i32 [[TILESLICE1]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svld1_ver_za8(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1b.vert(<vscale x 16 x i1> [[PG]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1b.vert(<vscale x 16 x i1> [[PG]], ptr [[PTR]], i32 0, i32 [[ADD]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z18test_svld1_ver_za8ju10__SVBool_tPKv(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1b.vert(<vscale x 16 x i1> [[PG]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1b.vert(<vscale x 16 x i1> [[PG]], ptr [[PTR]], i32 0, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svld1_ver_za8(uint32_t slice_base, svbool_t pg, const void *ptr) {
   svld1_ver_za8(0, slice_base, pg, ptr);
   svld1_ver_za8(0, slice_base + 15, pg, ptr);
 }
 
-// CHECK-C-LABEL:   @test_svld1_ver_za16(
-// CHECK-CXX-LABEL: @_Z19test_svld1_ver_za16ju10__SVBool_tPKv(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1h.vert(<vscale x 8 x i1> [[TMP0]], [[PTRTY]] [[PTR:%.*]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        [[TILESLICE1:%.*]] = add i32 [[SLICE_BASE]], 7
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1h.vert(<vscale x 8 x i1> [[TMP0]], [[PTRTY]] [[PTR]], i32 1, i32 [[TILESLICE1]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svld1_ver_za16(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[PTR]], i32 1, i32 [[ADD]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z19test_svld1_ver_za16ju10__SVBool_tPKv(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[PTR]], i32 1, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svld1_ver_za16(uint32_t slice_base, svbool_t pg, const void *ptr) {
   svld1_ver_za16(0, slice_base, pg, ptr);
   svld1_ver_za16(1, slice_base + 7, pg, ptr);
 }
 
-// CHECK-C-LABEL:   @test_svld1_ver_za32(
-// CHECK-CXX-LABEL: @_Z19test_svld1_ver_za32ju10__SVBool_tPKv(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> [[TMP0]], [[PTRTY]] [[PTR:%.*]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        [[TILESLICE1:%.*]] = add i32 [[SLICE_BASE]], 3
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> [[TMP0]], [[PTRTY]] [[PTR]], i32 3, i32 [[TILESLICE1]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svld1_ver_za32(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[PTR]], i32 3, i32 [[ADD]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z19test_svld1_ver_za32ju10__SVBool_tPKv(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[PTR]], i32 3, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svld1_ver_za32(uint32_t slice_base, svbool_t pg, const void *ptr) {
   svld1_ver_za32(0, slice_base, pg, ptr);
   svld1_ver_za32(3, slice_base + 3, pg, ptr);
 }
 
-// CHECK-C-LABEL:   @test_svld1_ver_za64(
-// CHECK-CXX-LABEL: @_Z19test_svld1_ver_za64ju10__SVBool_tPKv(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> [[TMP0]], [[PTRTY]] [[PTR:%.*]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        [[TILESLICE1:%.*]] = add i32 [[SLICE_BASE]], 1
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> [[TMP0]], [[PTRTY]] [[PTR]], i32 7, i32 [[TILESLICE1]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svld1_ver_za64(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[PTR]], i32 7, i32 [[ADD]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z19test_svld1_ver_za64ju10__SVBool_tPKv(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[PTR]], i32 7, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svld1_ver_za64(uint32_t slice_base, svbool_t pg, const void *ptr) {
   svld1_ver_za64(0, slice_base, pg, ptr);
   svld1_ver_za64(7, slice_base + 1, pg, ptr);
 }
 
-// CHECK-C-LABEL:   @test_svld1_ver_za128(
-// CHECK-CXX-LABEL: @_Z20test_svld1_ver_za128ju10__SVBool_tPKv(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        [[TMP0:%.*]] = tail call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> [[TMP0]], [[PTRTY]] [[PTR:%.*]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> [[TMP0]], [[PTRTY]] [[PTR]], i32 15, i32 [[SLICE_BASE]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svld1_ver_za128(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[PTR]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z20test_svld1_ver_za128ju10__SVBool_tPKv(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[PTR]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svld1_ver_za128(uint32_t slice_base, svbool_t pg, const void *ptr) {
   svld1_ver_za128(0, slice_base, pg, ptr);
   svld1_ver_za128(15, slice_base, pg, ptr);
 }
+//// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+// CHECK: {{.*}}
diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_ld1_vnum.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_ld1_vnum.c
index 5d61587d85..cf49f62664 100644
--- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_ld1_vnum.c
+++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_ld1_vnum.c
@@ -1,3 +1,4 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 3
 // REQUIRES: aarch64-registered-target
 // RUN: %clang_cc1 -DDISABLE_SME_ATTRIBUTES -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s -check-prefixes=CHECK,CHECK-C
 // RUN: %clang_cc1 -DDISABLE_SME_ATTRIBUTES -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefixes=CHECK,CHECK-CXX
@@ -11,168 +12,286 @@
 #define ARM_STREAMING_ATTR __attribute__((arm_streaming))
 #endif
 
-// CHECK-C-LABEL:   @test_svld1_hor_vnum_za8(
-// CHECK-CXX-LABEL: @_Z23test_svld1_hor_vnum_za8ju10__SVBool_tPKvl(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-NEXT:        [[MULVL:%.*]] = mul i64 [[TMP0]], [[VNUM:%.*]]
-// CHECK-NEXT:        [[TMP1:%.*]] = getelementptr i8, [[PTRTY:ptr|i8\*]] [[PTR:%.*]], i64 [[MULVL]]
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1b.horiz(<vscale x 16 x i1> [[PG:%.*]], [[PTRTY]] [[TMP1]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        [[TILESLICE2:%.*]] = add i32 [[SLICE_BASE]], 15
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1b.horiz(<vscale x 16 x i1> [[PG]], [[PTRTY]] [[TMP1]], i32 0, i32 [[TILESLICE2]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svld1_hor_vnum_za8(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP0]], [[VNUM]]
+// CHECK-C-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[ADD]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z23test_svld1_hor_vnum_za8ju10__SVBool_tPKvl(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP0]], [[VNUM]]
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svld1_hor_vnum_za8(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) {
   svld1_hor_vnum_za8(0, slice_base, pg, ptr, vnum);
   svld1_hor_vnum_za8(0, slice_base + 15, pg, ptr, vnum);
 }
 
-// CHECK-C-LABEL:   @test_svld1_hor_vnum_za16(
-// CHECK-CXX-LABEL: @_Z24test_svld1_hor_vnum_za16ju10__SVBool_tPKvl(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:        [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-NEXT:        [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM:%.*]]
-// CHECK-NEXT:        [[TMP2:%.*]] = getelementptr i8, [[PTRTY]] [[PTR:%.*]], i64 [[MULVL]]
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1h.horiz(<vscale x 8 x i1> [[TMP0]], [[PTRTY]] [[TMP2]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        [[TILESLICE2:%.*]] = add i32 [[SLICE_BASE]], 7
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1h.horiz(<vscale x 8 x i1> [[TMP0]], [[PTRTY]] [[TMP2]], i32 1, i32 [[TILESLICE2]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svld1_hor_vnum_za16(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
+// CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 1, i32 [[ADD]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z24test_svld1_hor_vnum_za16ju10__SVBool_tPKvl(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 1, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svld1_hor_vnum_za16(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) {
   svld1_hor_vnum_za16(0, slice_base, pg, ptr, vnum);
   svld1_hor_vnum_za16(1, slice_base + 7, pg, ptr, vnum);
 }
 
-// CHECK-C-LABEL:   @test_svld1_hor_vnum_za32(
-// CHECK-CXX-LABEL: @_Z24test_svld1_hor_vnum_za32ju10__SVBool_tPKvl(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:        [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-NEXT:        [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM:%.*]]
-// CHECK-NEXT:        [[TMP2:%.*]] = getelementptr i8, [[PTRTY]] [[PTR:%.*]], i64 [[MULVL]]
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> [[TMP0]], [[PTRTY]] [[TMP2]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        [[TILESLICE2:%.*]] = add i32 [[SLICE_BASE]], 3
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> [[TMP0]], [[PTRTY]] [[TMP2]], i32 3, i32 [[TILESLICE2]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svld1_hor_vnum_za32(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
+// CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 3, i32 [[ADD]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z24test_svld1_hor_vnum_za32ju10__SVBool_tPKvl(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 3, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svld1_hor_vnum_za32(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) {
   svld1_hor_vnum_za32(0, slice_base, pg, ptr, vnum);
   svld1_hor_vnum_za32(3, slice_base + 3, pg, ptr, vnum);
 }
 
-// CHECK-C-LABEL:   @test_svld1_hor_vnum_za64(
-// CHECK-CXX-LABEL: @_Z24test_svld1_hor_vnum_za64ju10__SVBool_tPKvl(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:        [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-NEXT:        [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM:%.*]]
-// CHECK-NEXT:        [[TMP2:%.*]] = getelementptr i8, [[PTRTY]] [[PTR:%.*]], i64 [[MULVL]]
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> [[TMP0]], [[PTRTY]] [[TMP2]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        [[TILESLICE2:%.*]] = add i32 [[SLICE_BASE]], 1
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> [[TMP0]], [[PTRTY]] [[TMP2]], i32 7, i32 [[TILESLICE2]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svld1_hor_vnum_za64(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
+// CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 7, i32 [[ADD]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z24test_svld1_hor_vnum_za64ju10__SVBool_tPKvl(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 7, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svld1_hor_vnum_za64(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) {
   svld1_hor_vnum_za64(0, slice_base, pg, ptr, vnum);
   svld1_hor_vnum_za64(7, slice_base + 1, pg, ptr, vnum);
 }
 
-// CHECK-C-LABEL:   @test_svld1_hor_vnum_za128(
-// CHECK-CXX-LABEL: @_Z25test_svld1_hor_vnum_za128ju10__SVBool_tPKvl(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        [[TMP0:%.*]] = tail call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:        [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-NEXT:        [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM:%.*]]
-// CHECK-NEXT:        [[TMP2:%.*]] = getelementptr i8, [[PTRTY]] [[PTR:%.*]], i64 [[MULVL]]
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> [[TMP0]], [[PTRTY]] [[TMP2]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> [[TMP0]], [[PTRTY]] [[TMP2]], i32 15, i32 [[SLICE_BASE]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svld1_hor_vnum_za128(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
+// CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svld1_hor_vnum_za128ju10__SVBool_tPKvl(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svld1_hor_vnum_za128(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) {
   svld1_hor_vnum_za128(0, slice_base, pg, ptr, vnum);
   svld1_hor_vnum_za128(15, slice_base, pg, ptr, vnum);
 }
 
-// CHECK-C-LABEL:   @test_svld1_ver_hor_za8(
-// CHECK-CXX-LABEL: @_Z22test_svld1_ver_hor_za8ju10__SVBool_tPKvl(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-NEXT:        [[MULVL:%.*]] = mul i64 [[TMP0]], [[VNUM:%.*]]
-// CHECK-NEXT:        [[TMP1:%.*]] = getelementptr i8, [[PTRTY]] [[PTR:%.*]], i64 [[MULVL]]
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1b.vert(<vscale x 16 x i1> [[PG:%.*]], [[PTRTY]] [[TMP1]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        [[TILESLICE2:%.*]] = add i32 [[SLICE_BASE]], 15
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1b.vert(<vscale x 16 x i1> [[PG]], [[PTRTY]] [[TMP1]], i32 0, i32 [[TILESLICE2]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svld1_ver_hor_za8(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP0]], [[VNUM]]
+// CHECK-C-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[ADD]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z22test_svld1_ver_hor_za8ju10__SVBool_tPKvl(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP0]], [[VNUM]]
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svld1_ver_hor_za8(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) {
   svld1_ver_vnum_za8(0, slice_base, pg, ptr, vnum);
   svld1_ver_vnum_za8(0, slice_base + 15, pg, ptr, vnum);
 }
 
-// CHECK-C-LABEL:   @test_svld1_ver_vnum_za16(
-// CHECK-CXX-LABEL: @_Z24test_svld1_ver_vnum_za16ju10__SVBool_tPKvl(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:        [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-NEXT:        [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM:%.*]]
-// CHECK-NEXT:        [[TMP2:%.*]] = getelementptr i8, [[PTRTY]] [[PTR:%.*]], i64 [[MULVL]]
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1h.vert(<vscale x 8 x i1> [[TMP0]], [[PTRTY]] [[TMP2]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        [[TILESLICE2:%.*]] = add i32 [[SLICE_BASE]], 7
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1h.vert(<vscale x 8 x i1> [[TMP0]], [[PTRTY]] [[TMP2]], i32 1, i32 [[TILESLICE2]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svld1_ver_vnum_za16(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
+// CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 1, i32 [[ADD]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z24test_svld1_ver_vnum_za16ju10__SVBool_tPKvl(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 1, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svld1_ver_vnum_za16(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) {
   svld1_ver_vnum_za16(0, slice_base, pg, ptr, vnum);
   svld1_ver_vnum_za16(1, slice_base + 7, pg, ptr, vnum);
 }
 
-// CHECK-C-LABEL:   @test_svld1_ver_vnum_za32(
-// CHECK-CXX-LABEL: @_Z24test_svld1_ver_vnum_za32ju10__SVBool_tPKvl(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:        [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-NEXT:        [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM:%.*]]
-// CHECK-NEXT:        [[TMP2:%.*]] = getelementptr i8, [[PTRTY]] [[PTR:%.*]], i64 [[MULVL]]
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> [[TMP0]], [[PTRTY]] [[TMP2]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        [[TILESLICE2:%.*]] = add i32 [[SLICE_BASE]], 3
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> [[TMP0]], [[PTRTY]] [[TMP2]], i32 3, i32 [[TILESLICE2]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svld1_ver_vnum_za32(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
+// CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 3, i32 [[ADD]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z24test_svld1_ver_vnum_za32ju10__SVBool_tPKvl(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 3, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svld1_ver_vnum_za32(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) {
   svld1_ver_vnum_za32(0, slice_base, pg, ptr, vnum);
   svld1_ver_vnum_za32(3, slice_base + 3, pg, ptr, vnum);
 }
 
-// CHECK-C-LABEL:   @test_svld1_ver_vnum_za64(
-// CHECK-CXX-LABEL: @_Z24test_svld1_ver_vnum_za64ju10__SVBool_tPKvl(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:        [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-NEXT:        [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM:%.*]]
-// CHECK-NEXT:        [[TMP2:%.*]] = getelementptr i8, [[PTRTY]] [[PTR:%.*]], i64 [[MULVL]]
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> [[TMP0]], [[PTRTY]] [[TMP2]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        [[TILESLICE2:%.*]] = add i32 [[SLICE_BASE]], 1
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> [[TMP0]], [[PTRTY]] [[TMP2]], i32 7, i32 [[TILESLICE2]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svld1_ver_vnum_za64(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
+// CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 7, i32 [[ADD]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z24test_svld1_ver_vnum_za64ju10__SVBool_tPKvl(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 7, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svld1_ver_vnum_za64(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) {
   svld1_ver_vnum_za64(0, slice_base, pg, ptr, vnum);
   svld1_ver_vnum_za64(7, slice_base + 1, pg, ptr, vnum);
 }
 
-// CHECK-C-LABEL:   @test_svld1_ver_vnum_za128(
-// CHECK-CXX-LABEL: @_Z25test_svld1_ver_vnum_za128ju10__SVBool_tPKvl(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        [[TMP0:%.*]] = tail call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:        [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-NEXT:        [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM:%.*]]
-// CHECK-NEXT:        [[TMP2:%.*]] = getelementptr i8, [[PTRTY]] [[PTR:%.*]], i64 [[MULVL]]
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> [[TMP0]], [[PTRTY]] [[TMP2]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> [[TMP0]], [[PTRTY]] [[TMP2]], i32 15, i32 [[SLICE_BASE]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svld1_ver_vnum_za128(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
+// CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svld1_ver_vnum_za128ju10__SVBool_tPKvl(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svld1_ver_vnum_za128(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) {
   svld1_ver_vnum_za128(0, slice_base, pg, ptr, vnum);
   svld1_ver_vnum_za128(15, slice_base, pg, ptr, vnum);
 }
+//// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+// CHECK: {{.*}}
diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_ldr.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_ldr.c
index acddc2ef50..e85c47072f 100644
--- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_ldr.c
+++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_ldr.c
@@ -1,3 +1,4 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 3
 // REQUIRES: aarch64-registered-target
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s -check-prefixes=CHECK,CHECK-C
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefixes=CHECK,CHECK-CXX
@@ -5,35 +6,86 @@
 
 #include <arm_sme_draft_spec_subject_to_change.h>
 
-// CHECK-C-LABEL: @test_svldr_vnum_za(
-// CHECK-CXX-LABEL: @_Z18test_svldr_vnum_zajPKv(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.ldr(i32 [[SLICE_BASE:%.*]], ptr [[PTR:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svldr_vnum_za(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ldr(i32 [[SLICE_BASE]], ptr [[PTR]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z18test_svldr_vnum_zajPKv(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ldr(i32 [[SLICE_BASE]], ptr [[PTR]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svldr_vnum_za(uint32_t slice_base, const void *ptr) {
   svldr_vnum_za(slice_base, ptr, 0);
 }
 
-// CHECK-C-LABEL: @test_svldr_vnum_za_1(
-// CHECK-CXX-LABEL: @_Z20test_svldr_vnum_za_1jPKv(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[SVLB:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-NEXT:    [[MULVL:%.*]] = mul i64 [[SVLB]], 15
-// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[PTR:%.*]], i64 [[MULVL]]
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 15
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.ldr(i32 [[TILESLICE]], ptr [[TMP0]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svldr_vnum_za_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[SVLB:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[SVLB]], 15
+// CHECK-C-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ldr(i32 [[TILESLICE]], ptr [[TMP0]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z20test_svldr_vnum_za_1jPKv(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[SVLB:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[SVLB]], 15
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ldr(i32 [[TILESLICE]], ptr [[TMP0]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svldr_vnum_za_1(uint32_t slice_base, const void *ptr) {
   svldr_vnum_za(slice_base, ptr, 15);
 }
 
-// CHECK-C-LABEL: @test_svldr_za(
-// CHECK-CXX-LABEL: @_Z13test_svldr_zajPKv(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.ldr(i32 [[SLICE_BASE:%.*]], ptr [[PTR:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svldr_za(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ldr(i32 [[SLICE_BASE]], ptr [[PTR]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z13test_svldr_zajPKv(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ldr(i32 [[SLICE_BASE]], ptr [[PTR]])
+// CHECK-CXX-NEXT:    ret void
+//
 void test_svldr_za(uint32_t slice_base, const void *ptr) {
   svldr_za(slice_base, ptr);
 }
+
+// CHECK-C-LABEL: define dso_local void @test_svldr_vnum_za_var(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[SVLB:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[SVLB]], [[VNUM]]
+// CHECK-C-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    [[TMP1:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TILESLICE:%.*]] = add i32 [[TMP1]], [[SLICE_BASE]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ldr(i32 [[TILESLICE]], ptr [[TMP0]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z22test_svldr_vnum_za_varjPKvl(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[SVLB:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[SVLB]], [[VNUM]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TILESLICE:%.*]] = add i32 [[TMP1]], [[SLICE_BASE]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ldr(i32 [[TILESLICE]], ptr [[TMP0]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svldr_vnum_za_var(uint32_t slice_base, const void *ptr, int64_t vnum) {
+  svldr_vnum_za(slice_base, ptr, vnum);
+}
+//// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+// CHECK: {{.*}}
diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mopa-za32.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mopa-za32.c
index b52aee12f9..f1382b699f 100644
--- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mopa-za32.c
+++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mopa-za32.c
@@ -1,3 +1,4 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 3
 // REQUIRES: aarch64-registered-target
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve -target-feature +bf16 -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s -check-prefixes=CHECK,CHECK-C
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve -target-feature +bf16 -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefixes=CHECK,CHECK-CXX
@@ -13,78 +14,128 @@
 #define SME_ACLE_FUNC(A1,A2,A3) A1##A2##A3
 #endif
 
-// CHECK-C-LABEL: @test_svmopa_za32_s8(
-// CHECK-CXX-LABEL: @_Z19test_svmopa_za32_s8u10__SVBool_tu10__SVBool_tu10__SVInt8_tu10__SVInt8_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.smopa.wide.nxv16i8(i32 0, <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svmopa_za32_s8(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.smopa.wide.nxv16i8(i32 0, <vscale x 16 x i1> [[PN]], <vscale x 16 x i1> [[PM]], <vscale x 16 x i8> [[ZN]], <vscale x 16 x i8> [[ZM]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z19test_svmopa_za32_s8u10__SVBool_tu10__SVBool_tu10__SVInt8_tu10__SVInt8_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.smopa.wide.nxv16i8(i32 0, <vscale x 16 x i1> [[PN]], <vscale x 16 x i1> [[PM]], <vscale x 16 x i8> [[ZN]], <vscale x 16 x i8> [[ZM]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svmopa_za32_s8(svbool_t pn, svbool_t pm, svint8_t zn, svint8_t zm) {
   SME_ACLE_FUNC(svmopa_za32, _s8, _m)(0, pn, pm, zn, zm);
 }
 
-// CHECK-C-LABEL: @test_svmopa_za32_u8(
-// CHECK-CXX-LABEL: @_Z19test_svmopa_za32_u8u10__SVBool_tu10__SVBool_tu11__SVUint8_tu11__SVUint8_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.umopa.wide.nxv16i8(i32 0, <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svmopa_za32_u8(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.umopa.wide.nxv16i8(i32 0, <vscale x 16 x i1> [[PN]], <vscale x 16 x i1> [[PM]], <vscale x 16 x i8> [[ZN]], <vscale x 16 x i8> [[ZM]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z19test_svmopa_za32_u8u10__SVBool_tu10__SVBool_tu11__SVUint8_tu11__SVUint8_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.umopa.wide.nxv16i8(i32 0, <vscale x 16 x i1> [[PN]], <vscale x 16 x i1> [[PM]], <vscale x 16 x i8> [[ZN]], <vscale x 16 x i8> [[ZM]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svmopa_za32_u8(svbool_t pn, svbool_t pm, svuint8_t zn, svuint8_t zm) {
   SME_ACLE_FUNC(svmopa_za32, _u8, _m)(0, pn, pm, zn, zm);
 }
 
-// CHECK-C-LABEL: @test_svmopa_za32_bf16(
-// CHECK-CXX-LABEL: @_Z21test_svmopa_za32_bf16u10__SVBool_tu10__SVBool_tu14__SVBFloat16_tu14__SVBFloat16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.mopa.wide.nxv8bf16(i32 0, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svmopa_za32_bf16(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.mopa.wide.nxv8bf16(i32 0, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x bfloat> [[ZN]], <vscale x 8 x bfloat> [[ZM]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z21test_svmopa_za32_bf16u10__SVBool_tu10__SVBool_tu14__SVBFloat16_tu14__SVBFloat16_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.mopa.wide.nxv8bf16(i32 0, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x bfloat> [[ZN]], <vscale x 8 x bfloat> [[ZM]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svmopa_za32_bf16(svbool_t pn, svbool_t pm, svbfloat16_t zn, svbfloat16_t zm) {
   SME_ACLE_FUNC(svmopa_za32, _bf16, _m)(0, pn, pm, zn, zm);
 }
 
-// CHECK-C-LABEL: @test_svmopa_za32_f16(
-// CHECK-CXX-LABEL: @_Z20test_svmopa_za32_f16u10__SVBool_tu10__SVBool_tu13__SVFloat16_tu13__SVFloat16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.mopa.wide.nxv8f16(i32 1, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x half> [[ZN:%.*]], <vscale x 8 x half> [[ZM:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svmopa_za32_f16(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 8 x half> [[ZN:%.*]], <vscale x 8 x half> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.mopa.wide.nxv8f16(i32 1, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x half> [[ZN]], <vscale x 8 x half> [[ZM]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z20test_svmopa_za32_f16u10__SVBool_tu10__SVBool_tu13__SVFloat16_tu13__SVFloat16_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 8 x half> [[ZN:%.*]], <vscale x 8 x half> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.mopa.wide.nxv8f16(i32 1, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x half> [[ZN]], <vscale x 8 x half> [[ZM]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svmopa_za32_f16(svbool_t pn, svbool_t pm, svfloat16_t zn, svfloat16_t zm) {
   SME_ACLE_FUNC(svmopa_za32, _f16, _m)(1, pn, pm, zn, zm);
 }
 
-// CHECK-C-LABEL: @test_svmopa_za32_f32(
-// CHECK-CXX-LABEL: @_Z20test_svmopa_za32_f32u10__SVBool_tu10__SVBool_tu13__SVFloat32_tu13__SVFloat32_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PN:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PM:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.mopa.nxv4f32(i32 1, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x float> [[ZN:%.*]], <vscale x 4 x float> [[ZM:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svmopa_za32_f32(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 4 x float> [[ZN:%.*]], <vscale x 4 x float> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PN]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PM]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.mopa.nxv4f32(i32 1, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x float> [[ZN]], <vscale x 4 x float> [[ZM]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z20test_svmopa_za32_f32u10__SVBool_tu10__SVBool_tu13__SVFloat32_tu13__SVFloat32_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 4 x float> [[ZN:%.*]], <vscale x 4 x float> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PN]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PM]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.mopa.nxv4f32(i32 1, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x float> [[ZN]], <vscale x 4 x float> [[ZM]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svmopa_za32_f32(svbool_t pn, svbool_t pm, svfloat32_t zn, svfloat32_t zm) {
   SME_ACLE_FUNC(svmopa_za32, _f32, _m)(1, pn, pm, zn, zm);
 }
 
-// CHECK-C-LABEL: @test_svsumopa_za32_s8(
-// CHECK-CXX-LABEL: @_Z21test_svsumopa_za32_s8u10__SVBool_tu10__SVBool_tu10__SVInt8_tu11__SVUint8_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.sumopa.wide.nxv16i8(i32 0, <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svsumopa_za32_s8(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.sumopa.wide.nxv16i8(i32 0, <vscale x 16 x i1> [[PN]], <vscale x 16 x i1> [[PM]], <vscale x 16 x i8> [[ZN]], <vscale x 16 x i8> [[ZM]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z21test_svsumopa_za32_s8u10__SVBool_tu10__SVBool_tu10__SVInt8_tu11__SVUint8_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.sumopa.wide.nxv16i8(i32 0, <vscale x 16 x i1> [[PN]], <vscale x 16 x i1> [[PM]], <vscale x 16 x i8> [[ZN]], <vscale x 16 x i8> [[ZM]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svsumopa_za32_s8(svbool_t pn, svbool_t pm, svint8_t zn, svuint8_t zm) {
  SME_ACLE_FUNC(svsumopa_za32, _s8, _m)(0, pn, pm, zn, zm);
 }
 
-// CHECK-C-LABEL: @test_svusmopa_za32_u8(
-// CHECK-CXX-LABEL: @_Z21test_svusmopa_za32_u8u10__SVBool_tu10__SVBool_tu11__SVUint8_tu10__SVInt8_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.usmopa.wide.nxv16i8(i32 0, <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svusmopa_za32_u8(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.usmopa.wide.nxv16i8(i32 0, <vscale x 16 x i1> [[PN]], <vscale x 16 x i1> [[PM]], <vscale x 16 x i8> [[ZN]], <vscale x 16 x i8> [[ZM]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z21test_svusmopa_za32_u8u10__SVBool_tu10__SVBool_tu11__SVUint8_tu10__SVInt8_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.usmopa.wide.nxv16i8(i32 0, <vscale x 16 x i1> [[PN]], <vscale x 16 x i1> [[PM]], <vscale x 16 x i8> [[ZN]], <vscale x 16 x i8> [[ZM]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svusmopa_za32_u8(svbool_t pn, svbool_t pm, svuint8_t zn, svint8_t zm) {
   SME_ACLE_FUNC(svusmopa_za32, _u8, _m)(0, pn, pm, zn, zm);
 }
+//// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+// CHECK: {{.*}}
diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mopa-za64.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mopa-za64.c
index 835d7c75ba..e60a764e18 100644
--- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mopa-za64.c
+++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mopa-za64.c
@@ -1,3 +1,4 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 3
 // REQUIRES: aarch64-registered-target
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme-f64f64 -target-feature +sme-i16i64 -target-feature +sve -target-feature +bf16 -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s -check-prefixes=CHECK,CHECK-C
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme-f64f64 -target-feature +sme-i16i64 -target-feature +sve -target-feature +bf16 -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefixes=CHECK,CHECK-CXX
@@ -13,62 +14,104 @@
 #define SME_ACLE_FUNC(A1,A2,A3) A1##A2##A3
 #endif
 
-// CHECK-C-LABEL: @test_svmopa_za64_s16(
-// CHECK-CXX-LABEL: @_Z20test_svmopa_za64_s16u10__SVBool_tu10__SVBool_tu11__SVInt16_tu11__SVInt16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.smopa.wide.nxv8i16(i32 7, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svmopa_za64_s16(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.smopa.wide.nxv8i16(i32 7, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x i16> [[ZN]], <vscale x 8 x i16> [[ZM]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z20test_svmopa_za64_s16u10__SVBool_tu10__SVBool_tu11__SVInt16_tu11__SVInt16_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.smopa.wide.nxv8i16(i32 7, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x i16> [[ZN]], <vscale x 8 x i16> [[ZM]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svmopa_za64_s16(svbool_t pn, svbool_t pm, svint16_t zn, svint16_t zm) {
   SME_ACLE_FUNC(svmopa_za64, _s16, _m)(7, pn, pm, zn, zm);
 }
 
-// CHECK-C-LABEL: @test_svmopa_za64_u16(
-// CHECK-CXX-LABEL: @_Z20test_svmopa_za64_u16u10__SVBool_tu10__SVBool_tu12__SVUint16_tu12__SVUint16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.umopa.wide.nxv8i16(i32 0, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svmopa_za64_u16(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.umopa.wide.nxv8i16(i32 0, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x i16> [[ZN]], <vscale x 8 x i16> [[ZM]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z20test_svmopa_za64_u16u10__SVBool_tu10__SVBool_tu12__SVUint16_tu12__SVUint16_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.umopa.wide.nxv8i16(i32 0, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x i16> [[ZN]], <vscale x 8 x i16> [[ZM]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svmopa_za64_u16(svbool_t pn, svbool_t pm, svuint16_t zn, svuint16_t zm) {
   SME_ACLE_FUNC(svmopa_za64, _u16, _m)(0, pn, pm, zn, zm);
 }
 
-// CHECK-C-LABEL: @test_svmopa_za64_f64(
-// CHECK-CXX-LABEL: @_Z20test_svmopa_za64_f64u10__SVBool_tu10__SVBool_tu13__SVFloat64_tu13__SVFloat64_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PN:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PM:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.mopa.nxv2f64(i32 7, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i1> [[TMP1]], <vscale x 2 x double> [[ZN:%.*]], <vscale x 2 x double> [[ZM:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svmopa_za64_f64(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 2 x double> [[ZN:%.*]], <vscale x 2 x double> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PN]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PM]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.mopa.nxv2f64(i32 7, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i1> [[TMP1]], <vscale x 2 x double> [[ZN]], <vscale x 2 x double> [[ZM]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z20test_svmopa_za64_f64u10__SVBool_tu10__SVBool_tu13__SVFloat64_tu13__SVFloat64_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 2 x double> [[ZN:%.*]], <vscale x 2 x double> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PN]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PM]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.mopa.nxv2f64(i32 7, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i1> [[TMP1]], <vscale x 2 x double> [[ZN]], <vscale x 2 x double> [[ZM]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svmopa_za64_f64(svbool_t pn, svbool_t pm, svfloat64_t zn, svfloat64_t zm) {
   SME_ACLE_FUNC(svmopa_za64, _f64, _m)(7, pn, pm, zn, zm);
 }
 
-// CHECK-C-LABEL: @test_svsumopa_za64_s16(
-// CHECK-CXX-LABEL: @_Z22test_svsumopa_za64_s16u10__SVBool_tu10__SVBool_tu11__SVInt16_tu12__SVUint16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.sumopa.wide.nxv8i16(i32 0, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svsumopa_za64_s16(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.sumopa.wide.nxv8i16(i32 0, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x i16> [[ZN]], <vscale x 8 x i16> [[ZM]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z22test_svsumopa_za64_s16u10__SVBool_tu10__SVBool_tu11__SVInt16_tu12__SVUint16_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.sumopa.wide.nxv8i16(i32 0, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x i16> [[ZN]], <vscale x 8 x i16> [[ZM]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svsumopa_za64_s16(svbool_t pn, svbool_t pm, svint16_t zn, svuint16_t zm) {
  SME_ACLE_FUNC(svsumopa_za64, _s16, _m)(0, pn, pm, zn, zm);
 }
 
-// CHECK-C-LABEL: @test_svusmopa_za64_u16(
-// CHECK-CXX-LABEL: @_Z22test_svusmopa_za64_u16u10__SVBool_tu10__SVBool_tu12__SVUint16_tu11__SVInt16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.usmopa.wide.nxv8i16(i32 7, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svusmopa_za64_u16(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.usmopa.wide.nxv8i16(i32 7, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x i16> [[ZN]], <vscale x 8 x i16> [[ZM]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z22test_svusmopa_za64_u16u10__SVBool_tu10__SVBool_tu12__SVUint16_tu11__SVInt16_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.usmopa.wide.nxv8i16(i32 7, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x i16> [[ZN]], <vscale x 8 x i16> [[ZM]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svusmopa_za64_u16(svbool_t pn, svbool_t pm, svuint16_t zn, svint16_t zm) {
   SME_ACLE_FUNC(svusmopa_za64, _u16, _m)(7, pn, pm, zn, zm);
 }
+//// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+// CHECK: {{.*}}
diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mops-za32.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mops-za32.c
index 923b6b96b4..2524dab52a 100644
--- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mops-za32.c
+++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mops-za32.c
@@ -1,3 +1,4 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 3
 // REQUIRES: aarch64-registered-target
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve -target-feature +bf16 -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s -check-prefixes=CHECK,CHECK-C
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve -target-feature +bf16 -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefixes=CHECK,CHECK-CXX
@@ -13,78 +14,128 @@
 #define SME_ACLE_FUNC(A1,A2,A3) A1##A2##A3
 #endif
 
-// CHECK-C-LABEL: @test_svmops_za32_s8(
-// CHECK-CXX-LABEL: @_Z19test_svmops_za32_s8u10__SVBool_tu10__SVBool_tu10__SVInt8_tu10__SVInt8_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.smops.wide.nxv16i8(i32 0, <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svmops_za32_s8(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.smops.wide.nxv16i8(i32 0, <vscale x 16 x i1> [[PN]], <vscale x 16 x i1> [[PM]], <vscale x 16 x i8> [[ZN]], <vscale x 16 x i8> [[ZM]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z19test_svmops_za32_s8u10__SVBool_tu10__SVBool_tu10__SVInt8_tu10__SVInt8_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.smops.wide.nxv16i8(i32 0, <vscale x 16 x i1> [[PN]], <vscale x 16 x i1> [[PM]], <vscale x 16 x i8> [[ZN]], <vscale x 16 x i8> [[ZM]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svmops_za32_s8(svbool_t pn, svbool_t pm, svint8_t zn, svint8_t zm) {
   SME_ACLE_FUNC(svmops_za32, _s8, _m)(0, pn, pm, zn, zm);
 }
 
-// CHECK-C-LABEL: @test_svmops_za32_u8(
-// CHECK-CXX-LABEL: @_Z19test_svmops_za32_u8u10__SVBool_tu10__SVBool_tu11__SVUint8_tu11__SVUint8_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.umops.wide.nxv16i8(i32 0, <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svmops_za32_u8(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.umops.wide.nxv16i8(i32 0, <vscale x 16 x i1> [[PN]], <vscale x 16 x i1> [[PM]], <vscale x 16 x i8> [[ZN]], <vscale x 16 x i8> [[ZM]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z19test_svmops_za32_u8u10__SVBool_tu10__SVBool_tu11__SVUint8_tu11__SVUint8_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.umops.wide.nxv16i8(i32 0, <vscale x 16 x i1> [[PN]], <vscale x 16 x i1> [[PM]], <vscale x 16 x i8> [[ZN]], <vscale x 16 x i8> [[ZM]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svmops_za32_u8(svbool_t pn, svbool_t pm, svuint8_t zn, svuint8_t zm) {
   SME_ACLE_FUNC(svmops_za32, _u8, _m)(0, pn, pm, zn, zm);
 }
 
-// CHECK-C-LABEL: @test_svmops_za32_bf16(
-// CHECK-CXX-LABEL: @_Z21test_svmops_za32_bf16u10__SVBool_tu10__SVBool_tu14__SVBFloat16_tu14__SVBFloat16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.mops.wide.nxv8bf16(i32 0, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svmops_za32_bf16(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.mops.wide.nxv8bf16(i32 0, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x bfloat> [[ZN]], <vscale x 8 x bfloat> [[ZM]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z21test_svmops_za32_bf16u10__SVBool_tu10__SVBool_tu14__SVBFloat16_tu14__SVBFloat16_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.mops.wide.nxv8bf16(i32 0, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x bfloat> [[ZN]], <vscale x 8 x bfloat> [[ZM]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svmops_za32_bf16(svbool_t pn, svbool_t pm, svbfloat16_t zn, svbfloat16_t zm) {
   SME_ACLE_FUNC(svmops_za32, _bf16, _m)(0, pn, pm, zn, zm);
 }
 
-// CHECK-C-LABEL: @test_svmops_za32_f16(
-// CHECK-CXX-LABEL: @_Z20test_svmops_za32_f16u10__SVBool_tu10__SVBool_tu13__SVFloat16_tu13__SVFloat16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.mops.wide.nxv8f16(i32 1, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x half> [[ZN:%.*]], <vscale x 8 x half> [[ZM:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svmops_za32_f16(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 8 x half> [[ZN:%.*]], <vscale x 8 x half> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.mops.wide.nxv8f16(i32 1, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x half> [[ZN]], <vscale x 8 x half> [[ZM]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z20test_svmops_za32_f16u10__SVBool_tu10__SVBool_tu13__SVFloat16_tu13__SVFloat16_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 8 x half> [[ZN:%.*]], <vscale x 8 x half> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.mops.wide.nxv8f16(i32 1, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x half> [[ZN]], <vscale x 8 x half> [[ZM]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svmops_za32_f16(svbool_t pn, svbool_t pm, svfloat16_t zn, svfloat16_t zm) {
   SME_ACLE_FUNC(svmops_za32, _f16, _m)(1, pn, pm, zn, zm);
 }
 
-// CHECK-C-LABEL: @test_svmops_za32_f32(
-// CHECK-CXX-LABEL: @_Z20test_svmops_za32_f32u10__SVBool_tu10__SVBool_tu13__SVFloat32_tu13__SVFloat32_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PN:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PM:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.mops.nxv4f32(i32 1, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x float> [[ZN:%.*]], <vscale x 4 x float> [[ZM:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svmops_za32_f32(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 4 x float> [[ZN:%.*]], <vscale x 4 x float> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PN]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PM]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.mops.nxv4f32(i32 1, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x float> [[ZN]], <vscale x 4 x float> [[ZM]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z20test_svmops_za32_f32u10__SVBool_tu10__SVBool_tu13__SVFloat32_tu13__SVFloat32_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 4 x float> [[ZN:%.*]], <vscale x 4 x float> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PN]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PM]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.mops.nxv4f32(i32 1, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i1> [[TMP1]], <vscale x 4 x float> [[ZN]], <vscale x 4 x float> [[ZM]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svmops_za32_f32(svbool_t pn, svbool_t pm, svfloat32_t zn, svfloat32_t zm) {
   SME_ACLE_FUNC(svmops_za32, _f32, _m)(1, pn, pm, zn, zm);
 }
 
-// CHECK-C-LABEL: @test_svsumops_za32_s8(
-// CHECK-CXX-LABEL: @_Z21test_svsumops_za32_s8u10__SVBool_tu10__SVBool_tu10__SVInt8_tu11__SVUint8_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.sumops.wide.nxv16i8(i32 0, <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svsumops_za32_s8(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.sumops.wide.nxv16i8(i32 0, <vscale x 16 x i1> [[PN]], <vscale x 16 x i1> [[PM]], <vscale x 16 x i8> [[ZN]], <vscale x 16 x i8> [[ZM]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z21test_svsumops_za32_s8u10__SVBool_tu10__SVBool_tu10__SVInt8_tu11__SVUint8_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.sumops.wide.nxv16i8(i32 0, <vscale x 16 x i1> [[PN]], <vscale x 16 x i1> [[PM]], <vscale x 16 x i8> [[ZN]], <vscale x 16 x i8> [[ZM]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svsumops_za32_s8(svbool_t pn, svbool_t pm, svint8_t zn, svuint8_t zm) {
  SME_ACLE_FUNC(svsumops_za32, _s8, _m)(0, pn, pm, zn, zm);
 }
 
-// CHECK-C-LABEL: @test_svusmops_za32_u8(
-// CHECK-CXX-LABEL: @_Z21test_svusmops_za32_u8u10__SVBool_tu10__SVBool_tu11__SVUint8_tu10__SVInt8_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.usmops.wide.nxv16i8(i32 0, <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svusmops_za32_u8(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.usmops.wide.nxv16i8(i32 0, <vscale x 16 x i1> [[PN]], <vscale x 16 x i1> [[PM]], <vscale x 16 x i8> [[ZN]], <vscale x 16 x i8> [[ZM]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z21test_svusmops_za32_u8u10__SVBool_tu10__SVBool_tu11__SVUint8_tu10__SVInt8_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 16 x i8> [[ZN:%.*]], <vscale x 16 x i8> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.usmops.wide.nxv16i8(i32 0, <vscale x 16 x i1> [[PN]], <vscale x 16 x i1> [[PM]], <vscale x 16 x i8> [[ZN]], <vscale x 16 x i8> [[ZM]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svusmops_za32_u8(svbool_t pn, svbool_t pm, svuint8_t zn, svint8_t zm) {
   SME_ACLE_FUNC(svusmops_za32, _u8, _m)(0, pn, pm, zn, zm);
 }
+//// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+// CHECK: {{.*}}
diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mops-za64.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mops-za64.c
index ea1e55001b..75ec07ffa3 100644
--- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mops-za64.c
+++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_mops-za64.c
@@ -1,3 +1,4 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 3
 // REQUIRES: aarch64-registered-target
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme-f64f64 -target-feature +sme-i16i64 -target-feature +sve -target-feature +bf16 -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s -check-prefixes=CHECK,CHECK-C
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme-f64f64 -target-feature +sme-i16i64 -target-feature +sve -target-feature +bf16 -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefixes=CHECK,CHECK-CXX
@@ -13,62 +14,104 @@
 #define SME_ACLE_FUNC(A1,A2,A3) A1##A2##A3
 #endif
 
-// CHECK-C-LABEL: @test_svmops_za64_s16(
-// CHECK-CXX-LABEL: @_Z20test_svmops_za64_s16u10__SVBool_tu10__SVBool_tu11__SVInt16_tu11__SVInt16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.smops.wide.nxv8i16(i32 7, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svmops_za64_s16(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.smops.wide.nxv8i16(i32 7, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x i16> [[ZN]], <vscale x 8 x i16> [[ZM]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z20test_svmops_za64_s16u10__SVBool_tu10__SVBool_tu11__SVInt16_tu11__SVInt16_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.smops.wide.nxv8i16(i32 7, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x i16> [[ZN]], <vscale x 8 x i16> [[ZM]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svmops_za64_s16(svbool_t pn, svbool_t pm, svint16_t zn, svint16_t zm) {
   SME_ACLE_FUNC(svmops_za64, _s16, _m)(7, pn, pm, zn, zm);
 }
 
-// CHECK-C-LABEL: @test_svmops_za64_u16(
-// CHECK-CXX-LABEL: @_Z20test_svmops_za64_u16u10__SVBool_tu10__SVBool_tu12__SVUint16_tu12__SVUint16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.umops.wide.nxv8i16(i32 0, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svmops_za64_u16(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.umops.wide.nxv8i16(i32 0, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x i16> [[ZN]], <vscale x 8 x i16> [[ZM]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z20test_svmops_za64_u16u10__SVBool_tu10__SVBool_tu12__SVUint16_tu12__SVUint16_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.umops.wide.nxv8i16(i32 0, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x i16> [[ZN]], <vscale x 8 x i16> [[ZM]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svmops_za64_u16(svbool_t pn, svbool_t pm, svuint16_t zn, svuint16_t zm) {
   SME_ACLE_FUNC(svmops_za64, _u16, _m)(0, pn, pm, zn, zm);
 }
 
-// CHECK-C-LABEL: @test_svmops_za64_f64(
-// CHECK-CXX-LABEL: @_Z20test_svmops_za64_f64u10__SVBool_tu10__SVBool_tu13__SVFloat64_tu13__SVFloat64_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PN:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PM:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.mops.nxv2f64(i32 7, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i1> [[TMP1]], <vscale x 2 x double> [[ZN:%.*]], <vscale x 2 x double> [[ZM:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svmops_za64_f64(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 2 x double> [[ZN:%.*]], <vscale x 2 x double> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PN]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PM]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.mops.nxv2f64(i32 7, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i1> [[TMP1]], <vscale x 2 x double> [[ZN]], <vscale x 2 x double> [[ZM]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z20test_svmops_za64_f64u10__SVBool_tu10__SVBool_tu13__SVFloat64_tu13__SVFloat64_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 2 x double> [[ZN:%.*]], <vscale x 2 x double> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PN]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PM]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.mops.nxv2f64(i32 7, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i1> [[TMP1]], <vscale x 2 x double> [[ZN]], <vscale x 2 x double> [[ZM]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svmops_za64_f64(svbool_t pn, svbool_t pm, svfloat64_t zn, svfloat64_t zm) {
   SME_ACLE_FUNC(svmops_za64, _f64, _m)(7, pn, pm, zn, zm);
 }
 
-// CHECK-C-LABEL: @test_svsumops_za64_s16(
-// CHECK-CXX-LABEL: @_Z22test_svsumops_za64_s16u10__SVBool_tu10__SVBool_tu11__SVInt16_tu12__SVUint16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.sumops.wide.nxv8i16(i32 0, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svsumops_za64_s16(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.sumops.wide.nxv8i16(i32 0, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x i16> [[ZN]], <vscale x 8 x i16> [[ZM]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z22test_svsumops_za64_s16u10__SVBool_tu10__SVBool_tu11__SVInt16_tu12__SVUint16_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.sumops.wide.nxv8i16(i32 0, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x i16> [[ZN]], <vscale x 8 x i16> [[ZM]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svsumops_za64_s16(svbool_t pn, svbool_t pm, svint16_t zn, svuint16_t zm) {
  SME_ACLE_FUNC(svsumops_za64, _s16, _m)(0, pn, pm, zn, zm);
 }
 
-// CHECK-C-LABEL: @test_svusmops_za64_u16(
-// CHECK-CXX-LABEL: @_Z22test_svusmops_za64_u16u10__SVBool_tu10__SVBool_tu12__SVUint16_tu11__SVInt16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.usmops.wide.nxv8i16(i32 7, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svusmops_za64_u16(
+// CHECK-C-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.usmops.wide.nxv8i16(i32 7, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x i16> [[ZN]], <vscale x 8 x i16> [[ZM]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z22test_svusmops_za64_u16u10__SVBool_tu10__SVBool_tu12__SVUint16_tu11__SVInt16_t(
+// CHECK-CXX-SAME: <vscale x 16 x i1> [[PN:%.*]], <vscale x 16 x i1> [[PM:%.*]], <vscale x 8 x i16> [[ZN:%.*]], <vscale x 8 x i16> [[ZM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PN]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PM]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.usmops.wide.nxv8i16(i32 7, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], <vscale x 8 x i16> [[ZN]], <vscale x 8 x i16> [[ZM]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svusmops_za64_u16(svbool_t pn, svbool_t pm, svuint16_t zn, svint16_t zm) {
   SME_ACLE_FUNC(svusmops_za64, _u16, _m)(7, pn, pm, zn, zm);
 }
+//// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+// CHECK: {{.*}}
diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_read.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_read.c
index f7a0852387..fc5f798d72 100644
--- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_read.c
+++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_read.c
@@ -1,3 +1,4 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 3
 // REQUIRES: aarch64-registered-target
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s -check-prefixes=CHECK,CHECK-C
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefixes=CHECK,CHECK-CXX
@@ -13,1090 +14,1772 @@
 #define SME_ACLE_FUNC(A1,A2,A3) A1##A2##A3
 #endif
 
-// CHECK-C-LABEL: @test_svread_hor_za8_s8(
-// CHECK-CXX-LABEL: @_Z22test_svread_hor_za8_s8u10__SVInt8_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.read.horiz.nxv16i8(<vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+// CHECK-C-LABEL: define dso_local <vscale x 16 x i8> @test_svread_hor_za8_s8(
+// CHECK-C-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.read.horiz.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 16 x i8> @_Z22test_svread_hor_za8_s8u10__SVInt8_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.read.horiz.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
 svint8_t test_svread_hor_za8_s8(svint8_t zd, svbool_t pg, uint32_t slice_base) {
     return SME_ACLE_FUNC(svread_hor_za8, _s8, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za8_s8_1(
-// CHECK-CXX-LABEL: @_Z24test_svread_hor_za8_s8_1u10__SVInt8_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 15
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.read.horiz.nxv16i8(<vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 0, i32 [[TILESLICE]])
-// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+// CHECK-C-LABEL: define dso_local <vscale x 16 x i8> @test_svread_hor_za8_s8_1(
+// CHECK-C-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.read.horiz.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 0, i32 [[ADD]])
+// CHECK-C-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 16 x i8> @_Z24test_svread_hor_za8_s8_1u10__SVInt8_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.read.horiz.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 0, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
 svint8_t test_svread_hor_za8_s8_1(svint8_t zd, svbool_t pg, uint32_t slice_base) {
     uint32_t slice = slice_base + 15;
     return SME_ACLE_FUNC(svread_hor_za8, _s8, _m)(zd, pg, 0, slice);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za16_s16(
-// CHECK-CXX-LABEL: @_Z24test_svread_hor_za16_s16u11__SVInt16_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.read.horiz.nxv8i16(<vscale x 8 x i16> [[ZD:%.*]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 8 x i16> @test_svread_hor_za16_s16(
+// CHECK-C-SAME: <vscale x 8 x i16> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.read.horiz.nxv8i16(<vscale x 8 x i16> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x i16> @_Z24test_svread_hor_za16_s16u11__SVInt16_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 8 x i16> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.read.horiz.nxv8i16(<vscale x 8 x i16> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
 //
 svint16_t test_svread_hor_za16_s16(svint16_t zd, svbool_t pg, uint32_t slice_base) {
      return SME_ACLE_FUNC(svread_hor_za16, _s16, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za16_s16_1(
-// CHECK-CXX-LABEL: @_Z26test_svread_hor_za16_s16_1u11__SVInt16_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 7
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.read.horiz.nxv8i16(<vscale x 8 x i16> [[ZD:%.*]], <vscale x 8 x i1> [[TMP0]], i32 1, i32 [[TILESLICE]])
-// CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 8 x i16> @test_svread_hor_za16_s16_1(
+// CHECK-C-SAME: <vscale x 8 x i16> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.read.horiz.nxv8i16(<vscale x 8 x i16> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 1, i32 [[ADD]])
+// CHECK-C-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x i16> @_Z26test_svread_hor_za16_s16_1u11__SVInt16_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 8 x i16> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.read.horiz.nxv8i16(<vscale x 8 x i16> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 1, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
 //
 svint16_t test_svread_hor_za16_s16_1(svint16_t zd, svbool_t pg, uint32_t slice_base) {
      uint32_t slice = slice_base + 7;
      return SME_ACLE_FUNC(svread_hor_za16, _s16, _m)(zd, pg, 1, slice);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za32_s32(
-// CHECK-CXX-LABEL: @_Z24test_svread_hor_za32_s32u11__SVInt32_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.read.horiz.nxv4i32(<vscale x 4 x i32> [[ZD:%.*]], <vscale x 4 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 4 x i32> @test_svread_hor_za32_s32(
+// CHECK-C-SAME: <vscale x 4 x i32> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.read.horiz.nxv4i32(<vscale x 4 x i32> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 4 x i32> @_Z24test_svread_hor_za32_s32u11__SVInt32_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 4 x i32> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.read.horiz.nxv4i32(<vscale x 4 x i32> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
 svint32_t test_svread_hor_za32_s32(svint32_t zd, svbool_t pg, uint32_t slice_base) {
     return SME_ACLE_FUNC(svread_hor_za32, _s32, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za32_s32_1(
-// CHECK-CXX-LABEL: @_Z26test_svread_hor_za32_s32_1u11__SVInt32_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 3
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.read.horiz.nxv4i32(<vscale x 4 x i32> [[ZD:%.*]], <vscale x 4 x i1> [[TMP0]], i32 3, i32 [[TILESLICE]])
-// CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 4 x i32> @test_svread_hor_za32_s32_1(
+// CHECK-C-SAME: <vscale x 4 x i32> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.read.horiz.nxv4i32(<vscale x 4 x i32> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 3, i32 [[ADD]])
+// CHECK-C-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 4 x i32> @_Z26test_svread_hor_za32_s32_1u11__SVInt32_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 4 x i32> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.read.horiz.nxv4i32(<vscale x 4 x i32> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 3, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
 svint32_t test_svread_hor_za32_s32_1(svint32_t zd, svbool_t pg, uint32_t slice_base)  {
     uint32_t slice = slice_base + 3;
     return SME_ACLE_FUNC(svread_hor_za32, _s32, _m)(zd, pg, 3, slice);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za64_s64(
-// CHECK-CXX-LABEL: @_Z24test_svread_hor_za64_s64u11__SVInt64_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.read.horiz.nxv2i64(<vscale x 2 x i64> [[ZD:%.*]], <vscale x 2 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 2 x i64> @test_svread_hor_za64_s64(
+// CHECK-C-SAME: <vscale x 2 x i64> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.read.horiz.nxv2i64(<vscale x 2 x i64> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 2 x i64> @_Z24test_svread_hor_za64_s64u11__SVInt64_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 2 x i64> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.read.horiz.nxv2i64(<vscale x 2 x i64> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
 svint64_t test_svread_hor_za64_s64(svint64_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_hor_za64, _s64, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za64_s64_1(
-// CHECK-CXX-LABEL: @_Z26test_svread_hor_za64_s64_1u11__SVInt64_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 1
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.read.horiz.nxv2i64(<vscale x 2 x i64> [[ZD:%.*]], <vscale x 2 x i1> [[TMP0]], i32 7, i32 [[TILESLICE]])
-// CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 2 x i64> @test_svread_hor_za64_s64_1(
+// CHECK-C-SAME: <vscale x 2 x i64> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.read.horiz.nxv2i64(<vscale x 2 x i64> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 7, i32 [[ADD]])
+// CHECK-C-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 2 x i64> @_Z26test_svread_hor_za64_s64_1u11__SVInt64_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 2 x i64> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.read.horiz.nxv2i64(<vscale x 2 x i64> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 7, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
 svint64_t test_svread_hor_za64_s64_1(svint64_t zd, svbool_t pg, uint32_t slice_base)  {
     uint32_t slice = slice_base + 1;
     return SME_ACLE_FUNC(svread_hor_za64, _s64, _m)(zd, pg, 7, slice);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za8_u8(
-// CHECK-CXX-LABEL: @_Z22test_svread_hor_za8_u8u11__SVUint8_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.read.horiz.nxv16i8(<vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+// CHECK-C-LABEL: define dso_local <vscale x 16 x i8> @test_svread_hor_za8_u8(
+// CHECK-C-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.read.horiz.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 16 x i8> @_Z22test_svread_hor_za8_u8u11__SVUint8_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.read.horiz.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
 svuint8_t test_svread_hor_za8_u8(svuint8_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_hor_za8, _u8, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za8_u8_1(
-// CHECK-CXX-LABEL: @_Z24test_svread_hor_za8_u8_1u11__SVUint8_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 15
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.read.horiz.nxv16i8(<vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 0, i32 [[TILESLICE]])
-// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+// CHECK-C-LABEL: define dso_local <vscale x 16 x i8> @test_svread_hor_za8_u8_1(
+// CHECK-C-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.read.horiz.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 0, i32 [[ADD]])
+// CHECK-C-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 16 x i8> @_Z24test_svread_hor_za8_u8_1u11__SVUint8_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.read.horiz.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 0, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
 svuint8_t test_svread_hor_za8_u8_1(svuint8_t zd, svbool_t pg, uint32_t slice_base)  {
     uint32_t slice = slice_base + 15;
     return SME_ACLE_FUNC(svread_hor_za8, _u8, _m)(zd, pg, 0, slice);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za16_u16(
-// CHECK-CXX-LABEL: @_Z24test_svread_hor_za16_u16u12__SVUint16_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.read.horiz.nxv8i16(<vscale x 8 x i16> [[ZD:%.*]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 8 x i16> @test_svread_hor_za16_u16(
+// CHECK-C-SAME: <vscale x 8 x i16> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.read.horiz.nxv8i16(<vscale x 8 x i16> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x i16> @_Z24test_svread_hor_za16_u16u12__SVUint16_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 8 x i16> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.read.horiz.nxv8i16(<vscale x 8 x i16> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
 //
 svuint16_t test_svread_hor_za16_u16(svuint16_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_hor_za16, _u16, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za16_u16_1(
-// CHECK-CXX-LABEL: @_Z26test_svread_hor_za16_u16_1u12__SVUint16_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 7
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.read.horiz.nxv8i16(<vscale x 8 x i16> [[ZD:%.*]], <vscale x 8 x i1> [[TMP0]], i32 1, i32 [[TILESLICE]])
-// CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 8 x i16> @test_svread_hor_za16_u16_1(
+// CHECK-C-SAME: <vscale x 8 x i16> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.read.horiz.nxv8i16(<vscale x 8 x i16> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 1, i32 [[ADD]])
+// CHECK-C-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x i16> @_Z26test_svread_hor_za16_u16_1u12__SVUint16_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 8 x i16> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.read.horiz.nxv8i16(<vscale x 8 x i16> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 1, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
 //
 svuint16_t test_svread_hor_za16_u16_1(svuint16_t zd, svbool_t pg, uint32_t slice_base)  {
     uint32_t slice = slice_base + 7;
     return SME_ACLE_FUNC(svread_hor_za16, _u16, _m)(zd, pg, 1, slice);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za32_u32(
-// CHECK-CXX-LABEL: @_Z24test_svread_hor_za32_u32u12__SVUint32_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.read.horiz.nxv4i32(<vscale x 4 x i32> [[ZD:%.*]], <vscale x 4 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 4 x i32> @test_svread_hor_za32_u32(
+// CHECK-C-SAME: <vscale x 4 x i32> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.read.horiz.nxv4i32(<vscale x 4 x i32> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 4 x i32> @_Z24test_svread_hor_za32_u32u12__SVUint32_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 4 x i32> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.read.horiz.nxv4i32(<vscale x 4 x i32> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
 svuint32_t test_svread_hor_za32_u32(svuint32_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_hor_za32, _u32, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za32_u32_1(
-// CHECK-CXX-LABEL: @_Z26test_svread_hor_za32_u32_1u12__SVUint32_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 3
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.read.horiz.nxv4i32(<vscale x 4 x i32> [[ZD:%.*]], <vscale x 4 x i1> [[TMP0]], i32 3, i32 [[TILESLICE]])
-// CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 4 x i32> @test_svread_hor_za32_u32_1(
+// CHECK-C-SAME: <vscale x 4 x i32> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.read.horiz.nxv4i32(<vscale x 4 x i32> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 3, i32 [[ADD]])
+// CHECK-C-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 4 x i32> @_Z26test_svread_hor_za32_u32_1u12__SVUint32_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 4 x i32> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.read.horiz.nxv4i32(<vscale x 4 x i32> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 3, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
 svuint32_t test_svread_hor_za32_u32_1(svuint32_t zd, svbool_t pg, uint32_t slice_base)  {
     uint32_t slice = slice_base + 3;
     return SME_ACLE_FUNC(svread_hor_za32, _u32, _m)(zd, pg, 3, slice);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za64_u64(
-// CHECK-CXX-LABEL: @_Z24test_svread_hor_za64_u64u12__SVUint64_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.read.horiz.nxv2i64(<vscale x 2 x i64> [[ZD:%.*]], <vscale x 2 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 2 x i64> @test_svread_hor_za64_u64(
+// CHECK-C-SAME: <vscale x 2 x i64> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.read.horiz.nxv2i64(<vscale x 2 x i64> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 2 x i64> @_Z24test_svread_hor_za64_u64u12__SVUint64_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 2 x i64> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.read.horiz.nxv2i64(<vscale x 2 x i64> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
 svuint64_t test_svread_hor_za64_u64(svuint64_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_hor_za64, _u64, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za64_u64_1(
-// CHECK-CXX-LABEL: @_Z26test_svread_hor_za64_u64_1u12__SVUint64_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 1
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.read.horiz.nxv2i64(<vscale x 2 x i64> [[ZD:%.*]], <vscale x 2 x i1> [[TMP0]], i32 7, i32 [[TILESLICE]])
-// CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 2 x i64> @test_svread_hor_za64_u64_1(
+// CHECK-C-SAME: <vscale x 2 x i64> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.read.horiz.nxv2i64(<vscale x 2 x i64> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 7, i32 [[ADD]])
+// CHECK-C-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 2 x i64> @_Z26test_svread_hor_za64_u64_1u12__SVUint64_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 2 x i64> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.read.horiz.nxv2i64(<vscale x 2 x i64> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 7, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
 svuint64_t test_svread_hor_za64_u64_1(svuint64_t zd, svbool_t pg, uint32_t slice_base)  {
     uint32_t slice = slice_base + 1;
     return SME_ACLE_FUNC(svread_hor_za64, _u64, _m)(zd, pg, 7, slice);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za16_f16(
-// CHECK-CXX-LABEL: @_Z24test_svread_hor_za16_f16u13__SVFloat16_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sme.read.horiz.nxv8f16(<vscale x 8 x half> [[ZD:%.*]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 8 x half> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 8 x half> @test_svread_hor_za16_f16(
+// CHECK-C-SAME: <vscale x 8 x half> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sme.read.horiz.nxv8f16(<vscale x 8 x half> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 8 x half> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x half> @_Z24test_svread_hor_za16_f16u13__SVFloat16_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 8 x half> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sme.read.horiz.nxv8f16(<vscale x 8 x half> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x half> [[TMP1]]
 //
 svfloat16_t test_svread_hor_za16_f16(svfloat16_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_hor_za16, _f16, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za16_f16_1(
-// CHECK-CXX-LABEL: @_Z26test_svread_hor_za16_f16_1u13__SVFloat16_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 7
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sme.read.horiz.nxv8f16(<vscale x 8 x half> [[ZD:%.*]], <vscale x 8 x i1> [[TMP0]], i32 1, i32 [[TILESLICE]])
-// CHECK-NEXT:    ret <vscale x 8 x half> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 8 x half> @test_svread_hor_za16_f16_1(
+// CHECK-C-SAME: <vscale x 8 x half> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sme.read.horiz.nxv8f16(<vscale x 8 x half> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 1, i32 [[ADD]])
+// CHECK-C-NEXT:    ret <vscale x 8 x half> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x half> @_Z26test_svread_hor_za16_f16_1u13__SVFloat16_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 8 x half> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sme.read.horiz.nxv8f16(<vscale x 8 x half> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 1, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x half> [[TMP1]]
 //
 svfloat16_t test_svread_hor_za16_f16_1(svfloat16_t zd, svbool_t pg, uint32_t slice_base)  {
     uint32_t slice = slice_base + 7;
     return SME_ACLE_FUNC(svread_hor_za16, _f16, _m)(zd, pg, 1, slice);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za16_bf16(
-// CHECK-CXX-LABEL: @_Z25test_svread_hor_za16_bf16u14__SVBFloat16_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sme.read.horiz.nxv8bf16(<vscale x 8 x bfloat> [[ZD:%.*]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 8 x bfloat> @test_svread_hor_za16_bf16(
+// CHECK-C-SAME: <vscale x 8 x bfloat> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sme.read.horiz.nxv8bf16(<vscale x 8 x bfloat> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x bfloat> @_Z25test_svread_hor_za16_bf16u14__SVBFloat16_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 8 x bfloat> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sme.read.horiz.nxv8bf16(<vscale x 8 x bfloat> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
 svbfloat16_t test_svread_hor_za16_bf16(svbfloat16_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_hor_za16, _bf16, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za16_bf16_1(
-// CHECK-CXX-LABEL: @_Z27test_svread_hor_za16_bf16_1u14__SVBFloat16_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 7
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sme.read.horiz.nxv8bf16(<vscale x 8 x bfloat> [[ZD:%.*]], <vscale x 8 x i1> [[TMP0]], i32 1, i32 [[TILESLICE]])
-// CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 8 x bfloat> @test_svread_hor_za16_bf16_1(
+// CHECK-C-SAME: <vscale x 8 x bfloat> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sme.read.horiz.nxv8bf16(<vscale x 8 x bfloat> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 1, i32 [[ADD]])
+// CHECK-C-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x bfloat> @_Z27test_svread_hor_za16_bf16_1u14__SVBFloat16_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 8 x bfloat> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sme.read.horiz.nxv8bf16(<vscale x 8 x bfloat> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 1, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
 svbfloat16_t test_svread_hor_za16_bf16_1(svbfloat16_t zd, svbool_t pg, uint32_t slice_base)  {
     uint32_t slice = slice_base + 7;
     return SME_ACLE_FUNC(svread_hor_za16, _bf16, _m)(zd, pg, 1, slice);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za32_f32(
-// CHECK-CXX-LABEL: @_Z24test_svread_hor_za32_f32u13__SVFloat32_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sme.read.horiz.nxv4f32(<vscale x 4 x float> [[ZD:%.*]], <vscale x 4 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 4 x float> @test_svread_hor_za32_f32(
+// CHECK-C-SAME: <vscale x 4 x float> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sme.read.horiz.nxv4f32(<vscale x 4 x float> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 4 x float> @_Z24test_svread_hor_za32_f32u13__SVFloat32_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 4 x float> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sme.read.horiz.nxv4f32(<vscale x 4 x float> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 4 x float> [[TMP1]]
 //
 svfloat32_t test_svread_hor_za32_f32(svfloat32_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_hor_za32, _f32, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za32_f32_1(
-// CHECK-CXX-LABEL: @_Z26test_svread_hor_za32_f32_1u13__SVFloat32_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 3
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sme.read.horiz.nxv4f32(<vscale x 4 x float> [[ZD:%.*]], <vscale x 4 x i1> [[TMP0]], i32 3, i32 [[TILESLICE]])
-// CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 4 x float> @test_svread_hor_za32_f32_1(
+// CHECK-C-SAME: <vscale x 4 x float> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sme.read.horiz.nxv4f32(<vscale x 4 x float> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 3, i32 [[ADD]])
+// CHECK-C-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 4 x float> @_Z26test_svread_hor_za32_f32_1u13__SVFloat32_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 4 x float> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sme.read.horiz.nxv4f32(<vscale x 4 x float> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 3, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret <vscale x 4 x float> [[TMP1]]
 //
 svfloat32_t test_svread_hor_za32_f32_1(svfloat32_t zd, svbool_t pg, uint32_t slice_base)  {
     uint32_t slice = slice_base + 3;
     return SME_ACLE_FUNC(svread_hor_za32, _f32, _m)(zd, pg, 3, slice);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za64_f64(
-// CHECK-CXX-LABEL: @_Z24test_svread_hor_za64_f64u13__SVFloat64_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sme.read.horiz.nxv2f64(<vscale x 2 x double> [[ZD:%.*]], <vscale x 2 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 2 x double> @test_svread_hor_za64_f64(
+// CHECK-C-SAME: <vscale x 2 x double> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sme.read.horiz.nxv2f64(<vscale x 2 x double> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 2 x double> @_Z24test_svread_hor_za64_f64u13__SVFloat64_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 2 x double> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sme.read.horiz.nxv2f64(<vscale x 2 x double> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 2 x double> [[TMP1]]
 //
 svfloat64_t test_svread_hor_za64_f64(svfloat64_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_hor_za64, _f64, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za64_f64_1(
-// CHECK-CXX-LABEL: @_Z26test_svread_hor_za64_f64_1u13__SVFloat64_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 1
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sme.read.horiz.nxv2f64(<vscale x 2 x double> [[ZD:%.*]], <vscale x 2 x i1> [[TMP0]], i32 7, i32 [[TILESLICE]])
-// CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 2 x double> @test_svread_hor_za64_f64_1(
+// CHECK-C-SAME: <vscale x 2 x double> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sme.read.horiz.nxv2f64(<vscale x 2 x double> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 7, i32 [[ADD]])
+// CHECK-C-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 2 x double> @_Z26test_svread_hor_za64_f64_1u13__SVFloat64_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 2 x double> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sme.read.horiz.nxv2f64(<vscale x 2 x double> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 7, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret <vscale x 2 x double> [[TMP1]]
 //
 svfloat64_t test_svread_hor_za64_f64_1(svfloat64_t zd, svbool_t pg, uint32_t slice_base)  {
     uint32_t slice = slice_base + 1;
     return SME_ACLE_FUNC(svread_hor_za64, _f64, _m)(zd, pg, 7, slice);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za128_s8(
-// CHECK-CXX-LABEL: @_Z24test_svread_hor_za128_s8u10__SVInt8_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.readq.horiz.nxv16i8(<vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+// CHECK-C-LABEL: define dso_local <vscale x 16 x i8> @test_svread_hor_za128_s8(
+// CHECK-C-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.readq.horiz.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 16 x i8> @_Z24test_svread_hor_za128_s8u10__SVInt8_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.readq.horiz.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
 svint8_t test_svread_hor_za128_s8(svint8_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_hor_za128, _s8, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za128_s8_1(
-// CHECK-CXX-LABEL: @_Z26test_svread_hor_za128_s8_1u10__SVInt8_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.readq.horiz.nxv16i8(<vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 15, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+// CHECK-C-LABEL: define dso_local <vscale x 16 x i8> @test_svread_hor_za128_s8_1(
+// CHECK-C-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.readq.horiz.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 16 x i8> @_Z26test_svread_hor_za128_s8_1u10__SVInt8_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.readq.horiz.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
 svint8_t test_svread_hor_za128_s8_1(svint8_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_hor_za128, _s8, _m)(zd, pg, 15, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za128_s16(
-// CHECK-CXX-LABEL: @_Z25test_svread_hor_za128_s16u11__SVInt16_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.readq.horiz.nxv8i16(<vscale x 8 x i16> [[ZD:%.*]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 8 x i16> @test_svread_hor_za128_s16(
+// CHECK-C-SAME: <vscale x 8 x i16> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.readq.horiz.nxv8i16(<vscale x 8 x i16> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x i16> @_Z25test_svread_hor_za128_s16u11__SVInt16_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 8 x i16> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.readq.horiz.nxv8i16(<vscale x 8 x i16> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
 //
 svint16_t test_svread_hor_za128_s16(svint16_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_hor_za128, _s16, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za128_s16_1(
-// CHECK-CXX-LABEL: @_Z27test_svread_hor_za128_s16_1u11__SVInt16_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.readq.horiz.nxv8i16(<vscale x 8 x i16> [[ZD:%.*]], <vscale x 8 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 8 x i16> @test_svread_hor_za128_s16_1(
+// CHECK-C-SAME: <vscale x 8 x i16> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.readq.horiz.nxv8i16(<vscale x 8 x i16> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x i16> @_Z27test_svread_hor_za128_s16_1u11__SVInt16_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 8 x i16> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.readq.horiz.nxv8i16(<vscale x 8 x i16> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
 //
 svint16_t test_svread_hor_za128_s16_1(svint16_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_hor_za128, _s16, _m)(zd, pg, 15, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za128_s32(
-// CHECK-CXX-LABEL: @_Z25test_svread_hor_za128_s32u11__SVInt32_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.readq.horiz.nxv4i32(<vscale x 4 x i32> [[ZD:%.*]], <vscale x 4 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 4 x i32> @test_svread_hor_za128_s32(
+// CHECK-C-SAME: <vscale x 4 x i32> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.readq.horiz.nxv4i32(<vscale x 4 x i32> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 4 x i32> @_Z25test_svread_hor_za128_s32u11__SVInt32_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 4 x i32> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.readq.horiz.nxv4i32(<vscale x 4 x i32> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
 svint32_t test_svread_hor_za128_s32(svint32_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_hor_za128, _s32, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za128_s32_1(
-// CHECK-CXX-LABEL: @_Z27test_svread_hor_za128_s32_1u11__SVInt32_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.readq.horiz.nxv4i32(<vscale x 4 x i32> [[ZD:%.*]], <vscale x 4 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 4 x i32> @test_svread_hor_za128_s32_1(
+// CHECK-C-SAME: <vscale x 4 x i32> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.readq.horiz.nxv4i32(<vscale x 4 x i32> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 4 x i32> @_Z27test_svread_hor_za128_s32_1u11__SVInt32_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 4 x i32> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.readq.horiz.nxv4i32(<vscale x 4 x i32> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
 svint32_t test_svread_hor_za128_s32_1(svint32_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_hor_za128, _s32, _m)(zd, pg, 15, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za128_s64(
-// CHECK-CXX-LABEL: @_Z25test_svread_hor_za128_s64u11__SVInt64_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.readq.horiz.nxv2i64(<vscale x 2 x i64> [[ZD:%.*]], <vscale x 2 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 2 x i64> @test_svread_hor_za128_s64(
+// CHECK-C-SAME: <vscale x 2 x i64> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.readq.horiz.nxv2i64(<vscale x 2 x i64> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 2 x i64> @_Z25test_svread_hor_za128_s64u11__SVInt64_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 2 x i64> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.readq.horiz.nxv2i64(<vscale x 2 x i64> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
 svint64_t test_svread_hor_za128_s64(svint64_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_hor_za128, _s64, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za128_s64_1(
-// CHECK-CXX-LABEL: @_Z27test_svread_hor_za128_s64_1u11__SVInt64_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.readq.horiz.nxv2i64(<vscale x 2 x i64> [[ZD:%.*]], <vscale x 2 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 2 x i64> @test_svread_hor_za128_s64_1(
+// CHECK-C-SAME: <vscale x 2 x i64> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.readq.horiz.nxv2i64(<vscale x 2 x i64> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 2 x i64> @_Z27test_svread_hor_za128_s64_1u11__SVInt64_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 2 x i64> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.readq.horiz.nxv2i64(<vscale x 2 x i64> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
 svint64_t test_svread_hor_za128_s64_1(svint64_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_hor_za128, _s64, _m)(zd, pg, 15, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za128_u8(
-// CHECK-CXX-LABEL: @_Z24test_svread_hor_za128_u8u11__SVUint8_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.readq.horiz.nxv16i8(<vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+// CHECK-C-LABEL: define dso_local <vscale x 16 x i8> @test_svread_hor_za128_u8(
+// CHECK-C-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.readq.horiz.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 16 x i8> @_Z24test_svread_hor_za128_u8u11__SVUint8_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.readq.horiz.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
 svuint8_t test_svread_hor_za128_u8(svuint8_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_hor_za128, _u8, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za128_u8_1(
-// CHECK-CXX-LABEL: @_Z26test_svread_hor_za128_u8_1u11__SVUint8_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.readq.horiz.nxv16i8(<vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 15, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+// CHECK-C-LABEL: define dso_local <vscale x 16 x i8> @test_svread_hor_za128_u8_1(
+// CHECK-C-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.readq.horiz.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 16 x i8> @_Z26test_svread_hor_za128_u8_1u11__SVUint8_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.readq.horiz.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
 svuint8_t test_svread_hor_za128_u8_1(svuint8_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_hor_za128, _u8, _m)(zd, pg, 15, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za128_u16(
-// CHECK-CXX-LABEL: @_Z25test_svread_hor_za128_u16u12__SVUint16_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.readq.horiz.nxv8i16(<vscale x 8 x i16> [[ZD:%.*]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 8 x i16> @test_svread_hor_za128_u16(
+// CHECK-C-SAME: <vscale x 8 x i16> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.readq.horiz.nxv8i16(<vscale x 8 x i16> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x i16> @_Z25test_svread_hor_za128_u16u12__SVUint16_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 8 x i16> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.readq.horiz.nxv8i16(<vscale x 8 x i16> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
 //
 svuint16_t test_svread_hor_za128_u16(svuint16_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_hor_za128, _u16, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za128_u16_1(
-// CHECK-CXX-LABEL: @_Z27test_svread_hor_za128_u16_1u12__SVUint16_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.readq.horiz.nxv8i16(<vscale x 8 x i16> [[ZD:%.*]], <vscale x 8 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 8 x i16> @test_svread_hor_za128_u16_1(
+// CHECK-C-SAME: <vscale x 8 x i16> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.readq.horiz.nxv8i16(<vscale x 8 x i16> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x i16> @_Z27test_svread_hor_za128_u16_1u12__SVUint16_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 8 x i16> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.readq.horiz.nxv8i16(<vscale x 8 x i16> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
 //
 svuint16_t test_svread_hor_za128_u16_1(svuint16_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_hor_za128, _u16, _m)(zd, pg, 15, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za128_u32(
-// CHECK-CXX-LABEL: @_Z25test_svread_hor_za128_u32u12__SVUint32_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.readq.horiz.nxv4i32(<vscale x 4 x i32> [[ZD:%.*]], <vscale x 4 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 4 x i32> @test_svread_hor_za128_u32(
+// CHECK-C-SAME: <vscale x 4 x i32> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.readq.horiz.nxv4i32(<vscale x 4 x i32> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 4 x i32> @_Z25test_svread_hor_za128_u32u12__SVUint32_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 4 x i32> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.readq.horiz.nxv4i32(<vscale x 4 x i32> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
 svuint32_t test_svread_hor_za128_u32(svuint32_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_hor_za128, _u32, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za128_u32_1(
-// CHECK-CXX-LABEL: @_Z27test_svread_hor_za128_u32_1u12__SVUint32_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.readq.horiz.nxv4i32(<vscale x 4 x i32> [[ZD:%.*]], <vscale x 4 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 4 x i32> @test_svread_hor_za128_u32_1(
+// CHECK-C-SAME: <vscale x 4 x i32> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.readq.horiz.nxv4i32(<vscale x 4 x i32> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 4 x i32> @_Z27test_svread_hor_za128_u32_1u12__SVUint32_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 4 x i32> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.readq.horiz.nxv4i32(<vscale x 4 x i32> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
 svuint32_t test_svread_hor_za128_u32_1(svuint32_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_hor_za128, _u32, _m)(zd, pg, 15, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za128_u64(
-// CHECK-CXX-LABEL: @_Z25test_svread_hor_za128_u64u12__SVUint64_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.readq.horiz.nxv2i64(<vscale x 2 x i64> [[ZD:%.*]], <vscale x 2 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 2 x i64> @test_svread_hor_za128_u64(
+// CHECK-C-SAME: <vscale x 2 x i64> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.readq.horiz.nxv2i64(<vscale x 2 x i64> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 2 x i64> @_Z25test_svread_hor_za128_u64u12__SVUint64_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 2 x i64> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.readq.horiz.nxv2i64(<vscale x 2 x i64> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
 svuint64_t test_svread_hor_za128_u64(svuint64_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_hor_za128, _u64, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za128_u64_1(
-// CHECK-CXX-LABEL: @_Z27test_svread_hor_za128_u64_1u12__SVUint64_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.readq.horiz.nxv2i64(<vscale x 2 x i64> [[ZD:%.*]], <vscale x 2 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 2 x i64> @test_svread_hor_za128_u64_1(
+// CHECK-C-SAME: <vscale x 2 x i64> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.readq.horiz.nxv2i64(<vscale x 2 x i64> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 2 x i64> @_Z27test_svread_hor_za128_u64_1u12__SVUint64_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 2 x i64> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.readq.horiz.nxv2i64(<vscale x 2 x i64> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
 svuint64_t test_svread_hor_za128_u64_1(svuint64_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_hor_za128, _u64, _m)(zd, pg, 15, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za128_f16(
-// CHECK-CXX-LABEL: @_Z25test_svread_hor_za128_f16u13__SVFloat16_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sme.readq.horiz.nxv8f16(<vscale x 8 x half> [[ZD:%.*]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 8 x half> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 8 x half> @test_svread_hor_za128_f16(
+// CHECK-C-SAME: <vscale x 8 x half> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sme.readq.horiz.nxv8f16(<vscale x 8 x half> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 8 x half> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x half> @_Z25test_svread_hor_za128_f16u13__SVFloat16_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 8 x half> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sme.readq.horiz.nxv8f16(<vscale x 8 x half> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x half> [[TMP1]]
 //
 svfloat16_t test_svread_hor_za128_f16(svfloat16_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_hor_za128, _f16, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za128_f16_1(
-// CHECK-CXX-LABEL: @_Z27test_svread_hor_za128_f16_1u13__SVFloat16_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sme.readq.horiz.nxv8f16(<vscale x 8 x half> [[ZD:%.*]], <vscale x 8 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 8 x half> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 8 x half> @test_svread_hor_za128_f16_1(
+// CHECK-C-SAME: <vscale x 8 x half> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sme.readq.horiz.nxv8f16(<vscale x 8 x half> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 8 x half> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x half> @_Z27test_svread_hor_za128_f16_1u13__SVFloat16_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 8 x half> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sme.readq.horiz.nxv8f16(<vscale x 8 x half> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x half> [[TMP1]]
 //
 svfloat16_t test_svread_hor_za128_f16_1(svfloat16_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_hor_za128, _f16, _m)(zd, pg, 15, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za128_bf16(
-// CHECK-CXX-LABEL: @_Z26test_svread_hor_za128_bf16u14__SVBFloat16_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sme.readq.horiz.nxv8bf16(<vscale x 8 x bfloat> [[ZD:%.*]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 8 x bfloat> @test_svread_hor_za128_bf16(
+// CHECK-C-SAME: <vscale x 8 x bfloat> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sme.readq.horiz.nxv8bf16(<vscale x 8 x bfloat> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x bfloat> @_Z26test_svread_hor_za128_bf16u14__SVBFloat16_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 8 x bfloat> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sme.readq.horiz.nxv8bf16(<vscale x 8 x bfloat> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
 svbfloat16_t test_svread_hor_za128_bf16(svbfloat16_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_hor_za128, _bf16, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za128_bf16_1(
-// CHECK-CXX-LABEL: @_Z28test_svread_hor_za128_bf16_1u14__SVBFloat16_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sme.readq.horiz.nxv8bf16(<vscale x 8 x bfloat> [[ZD:%.*]], <vscale x 8 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 8 x bfloat> @test_svread_hor_za128_bf16_1(
+// CHECK-C-SAME: <vscale x 8 x bfloat> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sme.readq.horiz.nxv8bf16(<vscale x 8 x bfloat> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x bfloat> @_Z28test_svread_hor_za128_bf16_1u14__SVBFloat16_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 8 x bfloat> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sme.readq.horiz.nxv8bf16(<vscale x 8 x bfloat> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
 svbfloat16_t test_svread_hor_za128_bf16_1(svbfloat16_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_hor_za128, _bf16, _m)(zd, pg, 15, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za128_f32(
-// CHECK-CXX-LABEL: @_Z25test_svread_hor_za128_f32u13__SVFloat32_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sme.readq.horiz.nxv4f32(<vscale x 4 x float> [[ZD:%.*]], <vscale x 4 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 4 x float> @test_svread_hor_za128_f32(
+// CHECK-C-SAME: <vscale x 4 x float> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sme.readq.horiz.nxv4f32(<vscale x 4 x float> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 4 x float> @_Z25test_svread_hor_za128_f32u13__SVFloat32_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 4 x float> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sme.readq.horiz.nxv4f32(<vscale x 4 x float> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 4 x float> [[TMP1]]
 //
 svfloat32_t test_svread_hor_za128_f32(svfloat32_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_hor_za128, _f32, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za128_f32_1(
-// CHECK-CXX-LABEL: @_Z27test_svread_hor_za128_f32_1u13__SVFloat32_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sme.readq.horiz.nxv4f32(<vscale x 4 x float> [[ZD:%.*]], <vscale x 4 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 4 x float> @test_svread_hor_za128_f32_1(
+// CHECK-C-SAME: <vscale x 4 x float> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sme.readq.horiz.nxv4f32(<vscale x 4 x float> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 4 x float> @_Z27test_svread_hor_za128_f32_1u13__SVFloat32_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 4 x float> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sme.readq.horiz.nxv4f32(<vscale x 4 x float> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 4 x float> [[TMP1]]
 //
 svfloat32_t test_svread_hor_za128_f32_1(svfloat32_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_hor_za128, _f32, _m)(zd, pg, 15, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za128_f64(
-// CHECK-CXX-LABEL: @_Z25test_svread_hor_za128_f64u13__SVFloat64_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sme.readq.horiz.nxv2f64(<vscale x 2 x double> [[ZD:%.*]], <vscale x 2 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 2 x double> @test_svread_hor_za128_f64(
+// CHECK-C-SAME: <vscale x 2 x double> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sme.readq.horiz.nxv2f64(<vscale x 2 x double> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 2 x double> @_Z25test_svread_hor_za128_f64u13__SVFloat64_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 2 x double> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sme.readq.horiz.nxv2f64(<vscale x 2 x double> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 2 x double> [[TMP1]]
 //
 svfloat64_t test_svread_hor_za128_f64(svfloat64_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_hor_za128, _f64, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_hor_za128_f64_1(
-// CHECK-CXX-LABEL: @_Z27test_svread_hor_za128_f64_1u13__SVFloat64_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sme.readq.horiz.nxv2f64(<vscale x 2 x double> [[ZD:%.*]], <vscale x 2 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 2 x double> @test_svread_hor_za128_f64_1(
+// CHECK-C-SAME: <vscale x 2 x double> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sme.readq.horiz.nxv2f64(<vscale x 2 x double> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 2 x double> @_Z27test_svread_hor_za128_f64_1u13__SVFloat64_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 2 x double> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sme.readq.horiz.nxv2f64(<vscale x 2 x double> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 2 x double> [[TMP1]]
 //
 svfloat64_t test_svread_hor_za128_f64_1(svfloat64_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_hor_za128, _f64, _m)(zd, pg, 15, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za8_s8(
-// CHECK-CXX-LABEL: @_Z22test_svread_ver_za8_s8u10__SVInt8_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.read.vert.nxv16i8(<vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+// CHECK-C-LABEL: define dso_local <vscale x 16 x i8> @test_svread_ver_za8_s8(
+// CHECK-C-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.read.vert.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 16 x i8> @_Z22test_svread_ver_za8_s8u10__SVInt8_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.read.vert.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
 svint8_t test_svread_ver_za8_s8(svint8_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_ver_za8, _s8, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za8_s8_1(
-// CHECK-CXX-LABEL: @_Z24test_svread_ver_za8_s8_1u10__SVInt8_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 15
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.read.vert.nxv16i8(<vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 0, i32 [[TILESLICE]])
-// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+// CHECK-C-LABEL: define dso_local <vscale x 16 x i8> @test_svread_ver_za8_s8_1(
+// CHECK-C-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.read.vert.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 0, i32 [[ADD]])
+// CHECK-C-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 16 x i8> @_Z24test_svread_ver_za8_s8_1u10__SVInt8_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.read.vert.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 0, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
 svint8_t test_svread_ver_za8_s8_1(svint8_t zd, svbool_t pg, uint32_t slice_base)  {
     uint32_t slice = slice_base + 15;
     return SME_ACLE_FUNC(svread_ver_za8, _s8, _m)(zd, pg, 0, slice);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za16_s16(
-// CHECK-CXX-LABEL: @_Z24test_svread_ver_za16_s16u11__SVInt16_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.read.vert.nxv8i16(<vscale x 8 x i16> [[ZD:%.*]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 8 x i16> @test_svread_ver_za16_s16(
+// CHECK-C-SAME: <vscale x 8 x i16> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.read.vert.nxv8i16(<vscale x 8 x i16> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x i16> @_Z24test_svread_ver_za16_s16u11__SVInt16_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 8 x i16> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.read.vert.nxv8i16(<vscale x 8 x i16> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
 //
 svint16_t test_svread_ver_za16_s16(svint16_t zd, svbool_t pg, uint32_t slice_base)  {
      return SME_ACLE_FUNC(svread_ver_za16, _s16, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za16_s16_1(
-// CHECK-CXX-LABEL: @_Z26test_svread_ver_za16_s16_1u11__SVInt16_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 7
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.read.vert.nxv8i16(<vscale x 8 x i16> [[ZD:%.*]], <vscale x 8 x i1> [[TMP0]], i32 1, i32 [[TILESLICE]])
-// CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 8 x i16> @test_svread_ver_za16_s16_1(
+// CHECK-C-SAME: <vscale x 8 x i16> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.read.vert.nxv8i16(<vscale x 8 x i16> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 1, i32 [[ADD]])
+// CHECK-C-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x i16> @_Z26test_svread_ver_za16_s16_1u11__SVInt16_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 8 x i16> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.read.vert.nxv8i16(<vscale x 8 x i16> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 1, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
 //
 svint16_t test_svread_ver_za16_s16_1(svint16_t zd, svbool_t pg, uint32_t slice_base)  {
      uint32_t slice = slice_base + 7;
      return SME_ACLE_FUNC(svread_ver_za16, _s16, _m)(zd, pg, 1, slice);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za32_s32(
-// CHECK-CXX-LABEL: @_Z24test_svread_ver_za32_s32u11__SVInt32_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.read.vert.nxv4i32(<vscale x 4 x i32> [[ZD:%.*]], <vscale x 4 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 4 x i32> @test_svread_ver_za32_s32(
+// CHECK-C-SAME: <vscale x 4 x i32> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.read.vert.nxv4i32(<vscale x 4 x i32> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 4 x i32> @_Z24test_svread_ver_za32_s32u11__SVInt32_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 4 x i32> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.read.vert.nxv4i32(<vscale x 4 x i32> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
 svint32_t test_svread_ver_za32_s32(svint32_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_ver_za32, _s32, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za32_s32_1(
-// CHECK-CXX-LABEL: @_Z26test_svread_ver_za32_s32_1u11__SVInt32_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 3
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.read.vert.nxv4i32(<vscale x 4 x i32> [[ZD:%.*]], <vscale x 4 x i1> [[TMP0]], i32 3, i32 [[TILESLICE]])
-// CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 4 x i32> @test_svread_ver_za32_s32_1(
+// CHECK-C-SAME: <vscale x 4 x i32> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.read.vert.nxv4i32(<vscale x 4 x i32> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 3, i32 [[ADD]])
+// CHECK-C-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 4 x i32> @_Z26test_svread_ver_za32_s32_1u11__SVInt32_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 4 x i32> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.read.vert.nxv4i32(<vscale x 4 x i32> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 3, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
 svint32_t test_svread_ver_za32_s32_1(svint32_t zd, svbool_t pg, uint32_t slice_base)  {
     uint32_t slice = slice_base + 3;
     return SME_ACLE_FUNC(svread_ver_za32, _s32, _m)(zd, pg, 3, slice);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za64_s64(
-// CHECK-CXX-LABEL: @_Z24test_svread_ver_za64_s64u11__SVInt64_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.read.vert.nxv2i64(<vscale x 2 x i64> [[ZD:%.*]], <vscale x 2 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 2 x i64> @test_svread_ver_za64_s64(
+// CHECK-C-SAME: <vscale x 2 x i64> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.read.vert.nxv2i64(<vscale x 2 x i64> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 2 x i64> @_Z24test_svread_ver_za64_s64u11__SVInt64_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 2 x i64> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.read.vert.nxv2i64(<vscale x 2 x i64> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
 svint64_t test_svread_ver_za64_s64(svint64_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_ver_za64, _s64, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za64_s64_1(
-// CHECK-CXX-LABEL: @_Z26test_svread_ver_za64_s64_1u11__SVInt64_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 1
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.read.vert.nxv2i64(<vscale x 2 x i64> [[ZD:%.*]], <vscale x 2 x i1> [[TMP0]], i32 7, i32 [[TILESLICE]])
-// CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 2 x i64> @test_svread_ver_za64_s64_1(
+// CHECK-C-SAME: <vscale x 2 x i64> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.read.vert.nxv2i64(<vscale x 2 x i64> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 7, i32 [[ADD]])
+// CHECK-C-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 2 x i64> @_Z26test_svread_ver_za64_s64_1u11__SVInt64_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 2 x i64> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.read.vert.nxv2i64(<vscale x 2 x i64> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 7, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
 svint64_t test_svread_ver_za64_s64_1(svint64_t zd, svbool_t pg, uint32_t slice_base)  {
     uint32_t slice = slice_base + 1;
     return SME_ACLE_FUNC(svread_ver_za64, _s64, _m)(zd, pg, 7, slice);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za8_u8(
-// CHECK-CXX-LABEL: @_Z22test_svread_ver_za8_u8u11__SVUint8_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.read.vert.nxv16i8(<vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+// CHECK-C-LABEL: define dso_local <vscale x 16 x i8> @test_svread_ver_za8_u8(
+// CHECK-C-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.read.vert.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 16 x i8> @_Z22test_svread_ver_za8_u8u11__SVUint8_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.read.vert.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
 svuint8_t test_svread_ver_za8_u8(svuint8_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_ver_za8, _u8, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za8_u8_1(
-// CHECK-CXX-LABEL: @_Z24test_svread_ver_za8_u8_1u11__SVUint8_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 15
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.read.vert.nxv16i8(<vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 0, i32 [[TILESLICE]])
-// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+// CHECK-C-LABEL: define dso_local <vscale x 16 x i8> @test_svread_ver_za8_u8_1(
+// CHECK-C-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.read.vert.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 0, i32 [[ADD]])
+// CHECK-C-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 16 x i8> @_Z24test_svread_ver_za8_u8_1u11__SVUint8_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.read.vert.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 0, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
 svuint8_t test_svread_ver_za8_u8_1(svuint8_t zd, svbool_t pg, uint32_t slice_base)  {
     uint32_t slice = slice_base + 15;
     return SME_ACLE_FUNC(svread_ver_za8, _u8, _m)(zd, pg, 0, slice);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za16_u16(
-// CHECK-CXX-LABEL: @_Z24test_svread_ver_za16_u16u12__SVUint16_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.read.vert.nxv8i16(<vscale x 8 x i16> [[ZD:%.*]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 8 x i16> @test_svread_ver_za16_u16(
+// CHECK-C-SAME: <vscale x 8 x i16> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.read.vert.nxv8i16(<vscale x 8 x i16> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x i16> @_Z24test_svread_ver_za16_u16u12__SVUint16_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 8 x i16> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.read.vert.nxv8i16(<vscale x 8 x i16> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
 //
 svuint16_t test_svread_ver_za16_u16(svuint16_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_ver_za16, _u16, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za16_u16_1(
-// CHECK-CXX-LABEL: @_Z26test_svread_ver_za16_u16_1u12__SVUint16_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 7
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.read.vert.nxv8i16(<vscale x 8 x i16> [[ZD:%.*]], <vscale x 8 x i1> [[TMP0]], i32 1, i32 [[TILESLICE]])
-// CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 8 x i16> @test_svread_ver_za16_u16_1(
+// CHECK-C-SAME: <vscale x 8 x i16> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.read.vert.nxv8i16(<vscale x 8 x i16> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 1, i32 [[ADD]])
+// CHECK-C-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x i16> @_Z26test_svread_ver_za16_u16_1u12__SVUint16_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 8 x i16> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.read.vert.nxv8i16(<vscale x 8 x i16> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 1, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
 //
 svuint16_t test_svread_ver_za16_u16_1(svuint16_t zd, svbool_t pg, uint32_t slice_base)  {
     uint32_t slice = slice_base + 7;
     return SME_ACLE_FUNC(svread_ver_za16, _u16, _m)(zd, pg, 1, slice);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za32_u32(
-// CHECK-CXX-LABEL: @_Z24test_svread_ver_za32_u32u12__SVUint32_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.read.vert.nxv4i32(<vscale x 4 x i32> [[ZD:%.*]], <vscale x 4 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 4 x i32> @test_svread_ver_za32_u32(
+// CHECK-C-SAME: <vscale x 4 x i32> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.read.vert.nxv4i32(<vscale x 4 x i32> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 4 x i32> @_Z24test_svread_ver_za32_u32u12__SVUint32_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 4 x i32> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.read.vert.nxv4i32(<vscale x 4 x i32> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
 svuint32_t test_svread_ver_za32_u32(svuint32_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_ver_za32, _u32, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za32_u32_1(
-// CHECK-CXX-LABEL: @_Z26test_svread_ver_za32_u32_1u12__SVUint32_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 3
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.read.vert.nxv4i32(<vscale x 4 x i32> [[ZD:%.*]], <vscale x 4 x i1> [[TMP0]], i32 3, i32 [[TILESLICE]])
-// CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 4 x i32> @test_svread_ver_za32_u32_1(
+// CHECK-C-SAME: <vscale x 4 x i32> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.read.vert.nxv4i32(<vscale x 4 x i32> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 3, i32 [[ADD]])
+// CHECK-C-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 4 x i32> @_Z26test_svread_ver_za32_u32_1u12__SVUint32_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 4 x i32> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.read.vert.nxv4i32(<vscale x 4 x i32> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 3, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
 svuint32_t test_svread_ver_za32_u32_1(svuint32_t zd, svbool_t pg, uint32_t slice_base)  {
     uint32_t slice = slice_base + 3;
     return SME_ACLE_FUNC(svread_ver_za32, _u32, _m)(zd, pg, 3, slice);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za64_u64(
-// CHECK-CXX-LABEL: @_Z24test_svread_ver_za64_u64u12__SVUint64_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.read.vert.nxv2i64(<vscale x 2 x i64> [[ZD:%.*]], <vscale x 2 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 2 x i64> @test_svread_ver_za64_u64(
+// CHECK-C-SAME: <vscale x 2 x i64> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.read.vert.nxv2i64(<vscale x 2 x i64> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 2 x i64> @_Z24test_svread_ver_za64_u64u12__SVUint64_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 2 x i64> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.read.vert.nxv2i64(<vscale x 2 x i64> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
 svuint64_t test_svread_ver_za64_u64(svuint64_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_ver_za64, _u64, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za64_u64_1(
-// CHECK-CXX-LABEL: @_Z26test_svread_ver_za64_u64_1u12__SVUint64_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 1
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.read.vert.nxv2i64(<vscale x 2 x i64> [[ZD:%.*]], <vscale x 2 x i1> [[TMP0]], i32 7, i32 [[TILESLICE]])
-// CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 2 x i64> @test_svread_ver_za64_u64_1(
+// CHECK-C-SAME: <vscale x 2 x i64> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.read.vert.nxv2i64(<vscale x 2 x i64> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 7, i32 [[ADD]])
+// CHECK-C-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 2 x i64> @_Z26test_svread_ver_za64_u64_1u12__SVUint64_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 2 x i64> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.read.vert.nxv2i64(<vscale x 2 x i64> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 7, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
 svuint64_t test_svread_ver_za64_u64_1(svuint64_t zd, svbool_t pg, uint32_t slice_base)  {
     uint32_t slice = slice_base + 1;
     return SME_ACLE_FUNC(svread_ver_za64, _u64, _m)(zd, pg, 7, slice);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za16_f16(
-// CHECK-CXX-LABEL: @_Z24test_svread_ver_za16_f16u13__SVFloat16_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sme.read.vert.nxv8f16(<vscale x 8 x half> [[ZD:%.*]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 8 x half> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 8 x half> @test_svread_ver_za16_f16(
+// CHECK-C-SAME: <vscale x 8 x half> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sme.read.vert.nxv8f16(<vscale x 8 x half> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 8 x half> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x half> @_Z24test_svread_ver_za16_f16u13__SVFloat16_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 8 x half> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sme.read.vert.nxv8f16(<vscale x 8 x half> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x half> [[TMP1]]
 //
 svfloat16_t test_svread_ver_za16_f16(svfloat16_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_ver_za16, _f16, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za16_f16_1(
-// CHECK-CXX-LABEL: @_Z26test_svread_ver_za16_f16_1u13__SVFloat16_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 7
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sme.read.vert.nxv8f16(<vscale x 8 x half> [[ZD:%.*]], <vscale x 8 x i1> [[TMP0]], i32 1, i32 [[TILESLICE]])
-// CHECK-NEXT:    ret <vscale x 8 x half> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 8 x half> @test_svread_ver_za16_f16_1(
+// CHECK-C-SAME: <vscale x 8 x half> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sme.read.vert.nxv8f16(<vscale x 8 x half> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 1, i32 [[ADD]])
+// CHECK-C-NEXT:    ret <vscale x 8 x half> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x half> @_Z26test_svread_ver_za16_f16_1u13__SVFloat16_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 8 x half> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sme.read.vert.nxv8f16(<vscale x 8 x half> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 1, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x half> [[TMP1]]
 //
 svfloat16_t test_svread_ver_za16_f16_1(svfloat16_t zd, svbool_t pg, uint32_t slice_base)  {
     uint32_t slice = slice_base + 7;
     return SME_ACLE_FUNC(svread_ver_za16, _f16, _m)(zd, pg, 1, slice);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za16_bf16(
-// CHECK-CXX-LABEL: @_Z25test_svread_ver_za16_bf16u14__SVBFloat16_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sme.read.vert.nxv8bf16(<vscale x 8 x bfloat> [[ZD:%.*]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 8 x bfloat> @test_svread_ver_za16_bf16(
+// CHECK-C-SAME: <vscale x 8 x bfloat> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sme.read.vert.nxv8bf16(<vscale x 8 x bfloat> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x bfloat> @_Z25test_svread_ver_za16_bf16u14__SVBFloat16_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 8 x bfloat> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sme.read.vert.nxv8bf16(<vscale x 8 x bfloat> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
 svbfloat16_t test_svread_ver_za16_bf16(svbfloat16_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_ver_za16, _bf16, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za16_bf16_1(
-// CHECK-CXX-LABEL: @_Z27test_svread_ver_za16_bf16_1u14__SVBFloat16_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 7
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sme.read.vert.nxv8bf16(<vscale x 8 x bfloat> [[ZD:%.*]], <vscale x 8 x i1> [[TMP0]], i32 1, i32 [[TILESLICE]])
-// CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 8 x bfloat> @test_svread_ver_za16_bf16_1(
+// CHECK-C-SAME: <vscale x 8 x bfloat> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sme.read.vert.nxv8bf16(<vscale x 8 x bfloat> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 1, i32 [[ADD]])
+// CHECK-C-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x bfloat> @_Z27test_svread_ver_za16_bf16_1u14__SVBFloat16_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 8 x bfloat> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sme.read.vert.nxv8bf16(<vscale x 8 x bfloat> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 1, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
 svbfloat16_t test_svread_ver_za16_bf16_1(svbfloat16_t zd, svbool_t pg, uint32_t slice_base)  {
     uint32_t slice = slice_base + 7;
     return SME_ACLE_FUNC(svread_ver_za16, _bf16, _m)(zd, pg, 1, slice);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za32_f32(
-// CHECK-CXX-LABEL: @_Z24test_svread_ver_za32_f32u13__SVFloat32_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sme.read.vert.nxv4f32(<vscale x 4 x float> [[ZD:%.*]], <vscale x 4 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 4 x float> @test_svread_ver_za32_f32(
+// CHECK-C-SAME: <vscale x 4 x float> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sme.read.vert.nxv4f32(<vscale x 4 x float> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 4 x float> @_Z24test_svread_ver_za32_f32u13__SVFloat32_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 4 x float> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sme.read.vert.nxv4f32(<vscale x 4 x float> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 4 x float> [[TMP1]]
 //
 svfloat32_t test_svread_ver_za32_f32(svfloat32_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_ver_za32, _f32, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za32_f32_1(
-// CHECK-CXX-LABEL: @_Z26test_svread_ver_za32_f32_1u13__SVFloat32_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 3
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sme.read.vert.nxv4f32(<vscale x 4 x float> [[ZD:%.*]], <vscale x 4 x i1> [[TMP0]], i32 3, i32 [[TILESLICE]])
-// CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 4 x float> @test_svread_ver_za32_f32_1(
+// CHECK-C-SAME: <vscale x 4 x float> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sme.read.vert.nxv4f32(<vscale x 4 x float> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 3, i32 [[ADD]])
+// CHECK-C-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 4 x float> @_Z26test_svread_ver_za32_f32_1u13__SVFloat32_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 4 x float> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sme.read.vert.nxv4f32(<vscale x 4 x float> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 3, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret <vscale x 4 x float> [[TMP1]]
 //
 svfloat32_t test_svread_ver_za32_f32_1(svfloat32_t zd, svbool_t pg, uint32_t slice_base)  {
     uint32_t slice = slice_base + 3;
     return SME_ACLE_FUNC(svread_ver_za32, _f32, _m)(zd, pg, 3, slice);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za64_f64(
-// CHECK-CXX-LABEL: @_Z24test_svread_ver_za64_f64u13__SVFloat64_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sme.read.vert.nxv2f64(<vscale x 2 x double> [[ZD:%.*]], <vscale x 2 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 2 x double> @test_svread_ver_za64_f64(
+// CHECK-C-SAME: <vscale x 2 x double> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sme.read.vert.nxv2f64(<vscale x 2 x double> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 2 x double> @_Z24test_svread_ver_za64_f64u13__SVFloat64_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 2 x double> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sme.read.vert.nxv2f64(<vscale x 2 x double> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 2 x double> [[TMP1]]
 //
 svfloat64_t test_svread_ver_za64_f64(svfloat64_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_ver_za64, _f64, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za64_f64_1(
-// CHECK-CXX-LABEL: @_Z26test_svread_ver_za64_f64_1u13__SVFloat64_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 1
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sme.read.vert.nxv2f64(<vscale x 2 x double> [[ZD:%.*]], <vscale x 2 x i1> [[TMP0]], i32 7, i32 [[TILESLICE]])
-// CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 2 x double> @test_svread_ver_za64_f64_1(
+// CHECK-C-SAME: <vscale x 2 x double> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sme.read.vert.nxv2f64(<vscale x 2 x double> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 7, i32 [[ADD]])
+// CHECK-C-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 2 x double> @_Z26test_svread_ver_za64_f64_1u13__SVFloat64_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 2 x double> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sme.read.vert.nxv2f64(<vscale x 2 x double> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 7, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret <vscale x 2 x double> [[TMP1]]
 //
 svfloat64_t test_svread_ver_za64_f64_1(svfloat64_t zd, svbool_t pg, uint32_t slice_base)  {
     uint32_t slice = slice_base + 1;
     return SME_ACLE_FUNC(svread_ver_za64, _f64, _m)(zd, pg, 7, slice);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za128_s8(
-// CHECK-CXX-LABEL: @_Z24test_svread_ver_za128_s8u10__SVInt8_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.readq.vert.nxv16i8(<vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+// CHECK-C-LABEL: define dso_local <vscale x 16 x i8> @test_svread_ver_za128_s8(
+// CHECK-C-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.readq.vert.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 16 x i8> @_Z24test_svread_ver_za128_s8u10__SVInt8_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.readq.vert.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
 svint8_t test_svread_ver_za128_s8(svint8_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_ver_za128, _s8, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za128_s8_1(
-// CHECK-CXX-LABEL: @_Z26test_svread_ver_za128_s8_1u10__SVInt8_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.readq.vert.nxv16i8(<vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 15, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+// CHECK-C-LABEL: define dso_local <vscale x 16 x i8> @test_svread_ver_za128_s8_1(
+// CHECK-C-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.readq.vert.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 16 x i8> @_Z26test_svread_ver_za128_s8_1u10__SVInt8_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.readq.vert.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
 svint8_t test_svread_ver_za128_s8_1(svint8_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_ver_za128, _s8, _m)(zd, pg, 15, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za128_s16(
-// CHECK-CXX-LABEL: @_Z25test_svread_ver_za128_s16u11__SVInt16_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.readq.vert.nxv8i16(<vscale x 8 x i16> [[ZD:%.*]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 8 x i16> @test_svread_ver_za128_s16(
+// CHECK-C-SAME: <vscale x 8 x i16> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.readq.vert.nxv8i16(<vscale x 8 x i16> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x i16> @_Z25test_svread_ver_za128_s16u11__SVInt16_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 8 x i16> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.readq.vert.nxv8i16(<vscale x 8 x i16> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
 //
 svint16_t test_svread_ver_za128_s16(svint16_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_ver_za128, _s16, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za128_s16_1(
-// CHECK-CXX-LABEL: @_Z27test_svread_ver_za128_s16_1u11__SVInt16_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.readq.vert.nxv8i16(<vscale x 8 x i16> [[ZD:%.*]], <vscale x 8 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 8 x i16> @test_svread_ver_za128_s16_1(
+// CHECK-C-SAME: <vscale x 8 x i16> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.readq.vert.nxv8i16(<vscale x 8 x i16> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x i16> @_Z27test_svread_ver_za128_s16_1u11__SVInt16_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 8 x i16> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.readq.vert.nxv8i16(<vscale x 8 x i16> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
 //
 svint16_t test_svread_ver_za128_s16_1(svint16_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_ver_za128, _s16, _m)(zd, pg, 15, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za128_s32(
-// CHECK-CXX-LABEL: @_Z25test_svread_ver_za128_s32u11__SVInt32_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.readq.vert.nxv4i32(<vscale x 4 x i32> [[ZD:%.*]], <vscale x 4 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 4 x i32> @test_svread_ver_za128_s32(
+// CHECK-C-SAME: <vscale x 4 x i32> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.readq.vert.nxv4i32(<vscale x 4 x i32> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 4 x i32> @_Z25test_svread_ver_za128_s32u11__SVInt32_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 4 x i32> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.readq.vert.nxv4i32(<vscale x 4 x i32> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
 svint32_t test_svread_ver_za128_s32(svint32_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_ver_za128, _s32, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za128_s32_1(
-// CHECK-CXX-LABEL: @_Z27test_svread_ver_za128_s32_1u11__SVInt32_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.readq.vert.nxv4i32(<vscale x 4 x i32> [[ZD:%.*]], <vscale x 4 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 4 x i32> @test_svread_ver_za128_s32_1(
+// CHECK-C-SAME: <vscale x 4 x i32> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.readq.vert.nxv4i32(<vscale x 4 x i32> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 4 x i32> @_Z27test_svread_ver_za128_s32_1u11__SVInt32_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 4 x i32> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.readq.vert.nxv4i32(<vscale x 4 x i32> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
 svint32_t test_svread_ver_za128_s32_1(svint32_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_ver_za128, _s32, _m)(zd, pg, 15, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za128_s64(
-// CHECK-CXX-LABEL: @_Z25test_svread_ver_za128_s64u11__SVInt64_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.readq.vert.nxv2i64(<vscale x 2 x i64> [[ZD:%.*]], <vscale x 2 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 2 x i64> @test_svread_ver_za128_s64(
+// CHECK-C-SAME: <vscale x 2 x i64> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.readq.vert.nxv2i64(<vscale x 2 x i64> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 2 x i64> @_Z25test_svread_ver_za128_s64u11__SVInt64_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 2 x i64> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.readq.vert.nxv2i64(<vscale x 2 x i64> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
 svint64_t test_svread_ver_za128_s64(svint64_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_ver_za128, _s64, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za128_s64_1(
-// CHECK-CXX-LABEL: @_Z27test_svread_ver_za128_s64_1u11__SVInt64_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.readq.vert.nxv2i64(<vscale x 2 x i64> [[ZD:%.*]], <vscale x 2 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 2 x i64> @test_svread_ver_za128_s64_1(
+// CHECK-C-SAME: <vscale x 2 x i64> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.readq.vert.nxv2i64(<vscale x 2 x i64> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 2 x i64> @_Z27test_svread_ver_za128_s64_1u11__SVInt64_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 2 x i64> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.readq.vert.nxv2i64(<vscale x 2 x i64> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
 svint64_t test_svread_ver_za128_s64_1(svint64_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_ver_za128, _s64, _m)(zd, pg, 15, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za128_u8(
-// CHECK-CXX-LABEL: @_Z24test_svread_ver_za128_u8u11__SVUint8_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.readq.vert.nxv16i8(<vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+// CHECK-C-LABEL: define dso_local <vscale x 16 x i8> @test_svread_ver_za128_u8(
+// CHECK-C-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.readq.vert.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 16 x i8> @_Z24test_svread_ver_za128_u8u11__SVUint8_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.readq.vert.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
 svuint8_t test_svread_ver_za128_u8(svuint8_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_ver_za128, _u8, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za128_u8_1(
-// CHECK-CXX-LABEL: @_Z26test_svread_ver_za128_u8_1u11__SVUint8_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.readq.vert.nxv16i8(<vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 15, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+// CHECK-C-LABEL: define dso_local <vscale x 16 x i8> @test_svread_ver_za128_u8_1(
+// CHECK-C-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.readq.vert.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 16 x i8> @_Z26test_svread_ver_za128_u8_1u11__SVUint8_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 16 x i8> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sme.readq.vert.nxv16i8(<vscale x 16 x i8> [[ZD]], <vscale x 16 x i1> [[PG]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
 svuint8_t test_svread_ver_za128_u8_1(svuint8_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_ver_za128, _u8, _m)(zd, pg, 15, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za128_u16(
-// CHECK-CXX-LABEL: @_Z25test_svread_ver_za128_u16u12__SVUint16_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.readq.vert.nxv8i16(<vscale x 8 x i16> [[ZD:%.*]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 8 x i16> @test_svread_ver_za128_u16(
+// CHECK-C-SAME: <vscale x 8 x i16> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.readq.vert.nxv8i16(<vscale x 8 x i16> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x i16> @_Z25test_svread_ver_za128_u16u12__SVUint16_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 8 x i16> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.readq.vert.nxv8i16(<vscale x 8 x i16> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
 //
 svuint16_t test_svread_ver_za128_u16(svuint16_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_ver_za128, _u16, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za128_u16_1(
-// CHECK-CXX-LABEL: @_Z27test_svread_ver_za128_u16_1u12__SVUint16_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.readq.vert.nxv8i16(<vscale x 8 x i16> [[ZD:%.*]], <vscale x 8 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 8 x i16> @test_svread_ver_za128_u16_1(
+// CHECK-C-SAME: <vscale x 8 x i16> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.readq.vert.nxv8i16(<vscale x 8 x i16> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x i16> @_Z27test_svread_ver_za128_u16_1u12__SVUint16_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 8 x i16> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sme.readq.vert.nxv8i16(<vscale x 8 x i16> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
 //
 svuint16_t test_svread_ver_za128_u16_1(svuint16_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_ver_za128, _u16, _m)(zd, pg, 15, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za128_u32(
-// CHECK-CXX-LABEL: @_Z25test_svread_ver_za128_u32u12__SVUint32_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.readq.vert.nxv4i32(<vscale x 4 x i32> [[ZD:%.*]], <vscale x 4 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 4 x i32> @test_svread_ver_za128_u32(
+// CHECK-C-SAME: <vscale x 4 x i32> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.readq.vert.nxv4i32(<vscale x 4 x i32> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 4 x i32> @_Z25test_svread_ver_za128_u32u12__SVUint32_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 4 x i32> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.readq.vert.nxv4i32(<vscale x 4 x i32> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
 svuint32_t test_svread_ver_za128_u32(svuint32_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_ver_za128, _u32, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za128_u32_1(
-// CHECK-CXX-LABEL: @_Z27test_svread_ver_za128_u32_1u12__SVUint32_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.readq.vert.nxv4i32(<vscale x 4 x i32> [[ZD:%.*]], <vscale x 4 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 4 x i32> @test_svread_ver_za128_u32_1(
+// CHECK-C-SAME: <vscale x 4 x i32> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.readq.vert.nxv4i32(<vscale x 4 x i32> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 4 x i32> @_Z27test_svread_ver_za128_u32_1u12__SVUint32_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 4 x i32> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sme.readq.vert.nxv4i32(<vscale x 4 x i32> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
 svuint32_t test_svread_ver_za128_u32_1(svuint32_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_ver_za128, _u32, _m)(zd, pg, 15, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za128_u64(
-// CHECK-CXX-LABEL: @_Z25test_svread_ver_za128_u64u12__SVUint64_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.readq.vert.nxv2i64(<vscale x 2 x i64> [[ZD:%.*]], <vscale x 2 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 2 x i64> @test_svread_ver_za128_u64(
+// CHECK-C-SAME: <vscale x 2 x i64> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.readq.vert.nxv2i64(<vscale x 2 x i64> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 2 x i64> @_Z25test_svread_ver_za128_u64u12__SVUint64_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 2 x i64> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.readq.vert.nxv2i64(<vscale x 2 x i64> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
 svuint64_t test_svread_ver_za128_u64(svuint64_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_ver_za128, _u64, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za128_u64_1(
-// CHECK-CXX-LABEL: @_Z27test_svread_ver_za128_u64_1u12__SVUint64_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.readq.vert.nxv2i64(<vscale x 2 x i64> [[ZD:%.*]], <vscale x 2 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 2 x i64> @test_svread_ver_za128_u64_1(
+// CHECK-C-SAME: <vscale x 2 x i64> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.readq.vert.nxv2i64(<vscale x 2 x i64> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 2 x i64> @_Z27test_svread_ver_za128_u64_1u12__SVUint64_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 2 x i64> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sme.readq.vert.nxv2i64(<vscale x 2 x i64> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
 svuint64_t test_svread_ver_za128_u64_1(svuint64_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_ver_za128, _u64, _m)(zd, pg, 15, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za128_f16(
-// CHECK-CXX-LABEL: @_Z25test_svread_ver_za128_f16u13__SVFloat16_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sme.readq.vert.nxv8f16(<vscale x 8 x half> [[ZD:%.*]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 8 x half> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 8 x half> @test_svread_ver_za128_f16(
+// CHECK-C-SAME: <vscale x 8 x half> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sme.readq.vert.nxv8f16(<vscale x 8 x half> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 8 x half> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x half> @_Z25test_svread_ver_za128_f16u13__SVFloat16_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 8 x half> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sme.readq.vert.nxv8f16(<vscale x 8 x half> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x half> [[TMP1]]
 //
 svfloat16_t test_svread_ver_za128_f16(svfloat16_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_ver_za128, _f16, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za128_f16_1(
-// CHECK-CXX-LABEL: @_Z27test_svread_ver_za128_f16_1u13__SVFloat16_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sme.readq.vert.nxv8f16(<vscale x 8 x half> [[ZD:%.*]], <vscale x 8 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 8 x half> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 8 x half> @test_svread_ver_za128_f16_1(
+// CHECK-C-SAME: <vscale x 8 x half> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sme.readq.vert.nxv8f16(<vscale x 8 x half> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 8 x half> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x half> @_Z27test_svread_ver_za128_f16_1u13__SVFloat16_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 8 x half> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sme.readq.vert.nxv8f16(<vscale x 8 x half> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x half> [[TMP1]]
 //
 svfloat16_t test_svread_ver_za128_f16_1(svfloat16_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_ver_za128, _f16, _m)(zd, pg, 15, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za128_bf16(
-// CHECK-CXX-LABEL: @_Z26test_svread_ver_za128_bf16u14__SVBFloat16_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sme.readq.vert.nxv8bf16(<vscale x 8 x bfloat> [[ZD:%.*]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 8 x bfloat> @test_svread_ver_za128_bf16(
+// CHECK-C-SAME: <vscale x 8 x bfloat> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sme.readq.vert.nxv8bf16(<vscale x 8 x bfloat> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x bfloat> @_Z26test_svread_ver_za128_bf16u14__SVBFloat16_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 8 x bfloat> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sme.readq.vert.nxv8bf16(<vscale x 8 x bfloat> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
 svbfloat16_t test_svread_ver_za128_bf16(svbfloat16_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_ver_za128, _bf16, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za128_bf16_1(
-// CHECK-CXX-LABEL: @_Z28test_svread_ver_za128_bf16_1u14__SVBFloat16_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sme.readq.vert.nxv8bf16(<vscale x 8 x bfloat> [[ZD:%.*]], <vscale x 8 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 8 x bfloat> @test_svread_ver_za128_bf16_1(
+// CHECK-C-SAME: <vscale x 8 x bfloat> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sme.readq.vert.nxv8bf16(<vscale x 8 x bfloat> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 8 x bfloat> @_Z28test_svread_ver_za128_bf16_1u14__SVBFloat16_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 8 x bfloat> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sme.readq.vert.nxv8bf16(<vscale x 8 x bfloat> [[ZD]], <vscale x 8 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
 svbfloat16_t test_svread_ver_za128_bf16_1(svbfloat16_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_ver_za128, _bf16, _m)(zd, pg, 15, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za128_f32(
-// CHECK-CXX-LABEL: @_Z25test_svread_ver_za128_f32u13__SVFloat32_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sme.readq.vert.nxv4f32(<vscale x 4 x float> [[ZD:%.*]], <vscale x 4 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 4 x float> @test_svread_ver_za128_f32(
+// CHECK-C-SAME: <vscale x 4 x float> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sme.readq.vert.nxv4f32(<vscale x 4 x float> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 4 x float> @_Z25test_svread_ver_za128_f32u13__SVFloat32_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 4 x float> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sme.readq.vert.nxv4f32(<vscale x 4 x float> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 4 x float> [[TMP1]]
 //
 svfloat32_t test_svread_ver_za128_f32(svfloat32_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_ver_za128, _f32, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za128_f32_1(
-// CHECK-CXX-LABEL: @_Z27test_svread_ver_za128_f32_1u13__SVFloat32_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sme.readq.vert.nxv4f32(<vscale x 4 x float> [[ZD:%.*]], <vscale x 4 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 4 x float> @test_svread_ver_za128_f32_1(
+// CHECK-C-SAME: <vscale x 4 x float> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sme.readq.vert.nxv4f32(<vscale x 4 x float> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 4 x float> @_Z27test_svread_ver_za128_f32_1u13__SVFloat32_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 4 x float> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sme.readq.vert.nxv4f32(<vscale x 4 x float> [[ZD]], <vscale x 4 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 4 x float> [[TMP1]]
 //
 svfloat32_t test_svread_ver_za128_f32_1(svfloat32_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_ver_za128, _f32, _m)(zd, pg, 15, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za128_f64(
-// CHECK-CXX-LABEL: @_Z25test_svread_ver_za128_f64u13__SVFloat64_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sme.readq.vert.nxv2f64(<vscale x 2 x double> [[ZD:%.*]], <vscale x 2 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 2 x double> @test_svread_ver_za128_f64(
+// CHECK-C-SAME: <vscale x 2 x double> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sme.readq.vert.nxv2f64(<vscale x 2 x double> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 2 x double> @_Z25test_svread_ver_za128_f64u13__SVFloat64_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 2 x double> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sme.readq.vert.nxv2f64(<vscale x 2 x double> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 2 x double> [[TMP1]]
 //
 svfloat64_t test_svread_ver_za128_f64(svfloat64_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_ver_za128, _f64, _m)(zd, pg, 0, slice_base);
 }
 
-// CHECK-C-LABEL: @test_svread_ver_za128_f64_1(
-// CHECK-CXX-LABEL: @_Z27test_svread_ver_za128_f64_1u13__SVFloat64_tu10__SVBool_tj(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sme.readq.vert.nxv2f64(<vscale x 2 x double> [[ZD:%.*]], <vscale x 2 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+// CHECK-C-LABEL: define dso_local <vscale x 2 x double> @test_svread_ver_za128_f64_1(
+// CHECK-C-SAME: <vscale x 2 x double> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sme.readq.vert.nxv2f64(<vscale x 2 x double> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret <vscale x 2 x double> [[TMP1]]
+//
+// CHECK-CXX-LABEL: define dso_local <vscale x 2 x double> @_Z27test_svread_ver_za128_f64_1u13__SVFloat64_tu10__SVBool_tj(
+// CHECK-CXX-SAME: <vscale x 2 x double> [[ZD:%.*]], <vscale x 16 x i1> [[PG:%.*]], i32 noundef [[SLICE_BASE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sme.readq.vert.nxv2f64(<vscale x 2 x double> [[ZD]], <vscale x 2 x i1> [[TMP0]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret <vscale x 2 x double> [[TMP1]]
 //
 svfloat64_t test_svread_ver_za128_f64_1(svfloat64_t zd, svbool_t pg, uint32_t slice_base)  {
     return SME_ACLE_FUNC(svread_ver_za128, _f64, _m)(zd, pg, 15, slice_base);
 }
+//// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+// CHECK: {{.*}}
diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_st1.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_st1.c
index eec5423416..9ea5fa10c1 100644
--- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_st1.c
+++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_st1.c
@@ -1,3 +1,4 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 3
 // REQUIRES: aarch64-registered-target
 // RUN: %clang_cc1 -DDISABLE_SME_ATTRIBUTES -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s -check-prefixes=CHECK,CHECK-C
 // RUN: %clang_cc1 -DDISABLE_SME_ATTRIBUTES -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefixes=CHECK,CHECK-CXX
@@ -11,138 +12,226 @@
 #define ARM_STREAMING_ATTR __attribute__((arm_streaming))
 #endif
 
-// CHECK-C-LABEL:   @test_svst1_hor_za8(
-// CHECK-CXX-LABEL: @_Z18test_svst1_hor_za8ju10__SVBool_tPv(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1b.horiz(<vscale x 16 x i1> [[PG:%.*]], [[PTRTY:ptr|i8\*]] [[PTR:%.*]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        [[TILESLICE1:%.*]] = add i32 [[SLICE_BASE]], 15
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1b.horiz(<vscale x 16 x i1> [[PG]], [[PTRTY]] [[PTR]], i32 0, i32 [[TILESLICE1]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svst1_hor_za8(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[PTR]], i32 0, i32 [[ADD]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z18test_svst1_hor_za8ju10__SVBool_tPv(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[PTR]], i32 0, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svst1_hor_za8(uint32_t slice_base, svbool_t pg, void *ptr) {
   svst1_hor_za8(0, slice_base, pg, ptr);
   svst1_hor_za8(0, slice_base + 15, pg, ptr);
 }
 
-// CHECK-C-LABEL:   @test_svst1_hor_za16(
-// CHECK-CXX-LABEL: @_Z19test_svst1_hor_za16ju10__SVBool_tPv(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1h.horiz(<vscale x 8 x i1> [[TMP0]], [[PTRTY]] [[PTR:%.*]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        [[TILESLICE1:%.*]] = add i32 [[SLICE_BASE]], 7
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1h.horiz(<vscale x 8 x i1> [[TMP0]], [[PTRTY]] [[PTR]], i32 1, i32 [[TILESLICE1]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svst1_hor_za16(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[PTR]], i32 1, i32 [[ADD]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z19test_svst1_hor_za16ju10__SVBool_tPv(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[PTR]], i32 1, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svst1_hor_za16(uint32_t slice_base, svbool_t pg, void *ptr) {
   svst1_hor_za16(0, slice_base, pg, ptr);
   svst1_hor_za16(1, slice_base + 7, pg, ptr);
 }
 
-// CHECK-C-LABEL:   @test_svst1_hor_za32(
-// CHECK-CXX-LABEL: @_Z19test_svst1_hor_za32ju10__SVBool_tPv(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1w.horiz(<vscale x 4 x i1> [[TMP0]], [[PTRTY]] [[PTR:%.*]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        [[TILESLICE1:%.*]] = add i32 [[SLICE_BASE]], 3
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1w.horiz(<vscale x 4 x i1> [[TMP0]], [[PTRTY]] [[PTR]], i32 3, i32 [[TILESLICE1]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svst1_hor_za32(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[PTR]], i32 3, i32 [[ADD]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z19test_svst1_hor_za32ju10__SVBool_tPv(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[PTR]], i32 3, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svst1_hor_za32(uint32_t slice_base, svbool_t pg, void *ptr) {
   svst1_hor_za32(0, slice_base, pg, ptr);
   svst1_hor_za32(3, slice_base + 3, pg, ptr);
 }
 
-// CHECK-C-LABEL:   @test_svst1_hor_za64(
-// CHECK-CXX-LABEL: @_Z19test_svst1_hor_za64ju10__SVBool_tPv(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1> [[TMP0]], [[PTRTY]] [[PTR:%.*]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        [[TILESLICE1:%.*]] = add i32 [[SLICE_BASE]], 1
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1> [[TMP0]], [[PTRTY]] [[PTR]], i32 7, i32 [[TILESLICE1]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svst1_hor_za64(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[PTR]], i32 7, i32 [[ADD]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z19test_svst1_hor_za64ju10__SVBool_tPv(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[PTR]], i32 7, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svst1_hor_za64(uint32_t slice_base, svbool_t pg, void *ptr) {
   svst1_hor_za64(0, slice_base, pg, ptr);
   svst1_hor_za64(7, slice_base + 1, pg, ptr);
 }
 
-// CHECK-C-LABEL:   @test_svst1_hor_za128(
-// CHECK-CXX-LABEL: @_Z20test_svst1_hor_za128ju10__SVBool_tPv(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        [[TMP0:%.*]] = tail call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> [[TMP0]], [[PTRTY]] [[PTR:%.*]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> [[TMP0]], [[PTRTY]] [[PTR]], i32 15, i32 [[SLICE_BASE]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svst1_hor_za128(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[PTR]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z20test_svst1_hor_za128ju10__SVBool_tPv(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[PTR]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svst1_hor_za128(uint32_t slice_base, svbool_t pg, void *ptr) {
   svst1_hor_za128(0, slice_base, pg, ptr);
   svst1_hor_za128(15, slice_base, pg, ptr);
 }
 
-// CHECK-C-LABEL:   @test_svst1_ver_za8(
-// CHECK-CXX-LABEL: @_Z18test_svst1_ver_za8ju10__SVBool_tPv(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1b.vert(<vscale x 16 x i1> [[PG:%.*]], [[PTRTY]] [[PTR:%.*]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        [[TILESLICE1:%.*]] = add i32 [[SLICE_BASE]], 15
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1b.vert(<vscale x 16 x i1> [[PG]], [[PTRTY]] [[PTR]], i32 0, i32 [[TILESLICE1]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svst1_ver_za8(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1b.vert(<vscale x 16 x i1> [[PG]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1b.vert(<vscale x 16 x i1> [[PG]], ptr [[PTR]], i32 0, i32 [[ADD]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z18test_svst1_ver_za8ju10__SVBool_tPv(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1b.vert(<vscale x 16 x i1> [[PG]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1b.vert(<vscale x 16 x i1> [[PG]], ptr [[PTR]], i32 0, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svst1_ver_za8(uint32_t slice_base, svbool_t pg, void *ptr) {
   svst1_ver_za8(0, slice_base, pg, ptr);
   svst1_ver_za8(0, slice_base + 15, pg, ptr);
 }
 
-// CHECK-C-LABEL:   @test_svst1_ver_za16(
-// CHECK-CXX-LABEL: @_Z19test_svst1_ver_za16ju10__SVBool_tPv(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1h.vert(<vscale x 8 x i1> [[TMP0]], [[PTRTY]] [[PTR:%.*]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        [[TILESLICE1:%.*]] = add i32 [[SLICE_BASE]], 7
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1h.vert(<vscale x 8 x i1> [[TMP0]], [[PTRTY]] [[PTR]], i32 1, i32 [[TILESLICE1]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svst1_ver_za16(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[PTR]], i32 1, i32 [[ADD]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z19test_svst1_ver_za16ju10__SVBool_tPv(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[PTR]], i32 1, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svst1_ver_za16(uint32_t slice_base, svbool_t pg, void *ptr) {
   svst1_ver_za16(0, slice_base, pg, ptr);
   svst1_ver_za16(1, slice_base + 7, pg, ptr);
 }
 
-// CHECK-C-LABEL:   @test_svst1_ver_za32(
-// CHECK-CXX-LABEL: @_Z19test_svst1_ver_za32ju10__SVBool_tPv(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1w.vert(<vscale x 4 x i1> [[TMP0]], [[PTRTY]] [[PTR:%.*]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        [[TILESLICE1:%.*]] = add i32 [[SLICE_BASE]], 3
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1w.vert(<vscale x 4 x i1> [[TMP0]], [[PTRTY]] [[PTR]], i32 3, i32 [[TILESLICE1]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svst1_ver_za32(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[PTR]], i32 3, i32 [[ADD]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z19test_svst1_ver_za32ju10__SVBool_tPv(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[PTR]], i32 3, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svst1_ver_za32(uint32_t slice_base, svbool_t pg, void *ptr) {
   svst1_ver_za32(0, slice_base, pg, ptr);
   svst1_ver_za32(3, slice_base + 3, pg, ptr);
 }
 
-// CHECK-C-LABEL:   @test_svst1_ver_za64(
-// CHECK-CXX-LABEL: @_Z19test_svst1_ver_za64ju10__SVBool_tPv(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1> [[TMP0]], [[PTRTY]] [[PTR:%.*]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        [[TILESLICE1:%.*]] = add i32 [[SLICE_BASE]], 1
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1> [[TMP0]], [[PTRTY]] [[PTR]], i32 7, i32 [[TILESLICE1]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svst1_ver_za64(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[PTR]], i32 7, i32 [[ADD]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z19test_svst1_ver_za64ju10__SVBool_tPv(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[PTR]], i32 7, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svst1_ver_za64(uint32_t slice_base, svbool_t pg, void *ptr) {
   svst1_ver_za64(0, slice_base, pg, ptr);
   svst1_ver_za64(7, slice_base + 1, pg, ptr);
 }
 
-// CHECK-C-LABEL:   @test_svst1_ver_za128(
-// CHECK-CXX-LABEL: @_Z20test_svst1_ver_za128ju10__SVBool_tPv(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        [[TMP0:%.*]] = tail call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> [[TMP0]], [[PTRTY]] [[PTR:%.*]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> [[TMP0]], [[PTRTY]] [[PTR]], i32 15, i32 [[SLICE_BASE]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svst1_ver_za128(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[PTR]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z20test_svst1_ver_za128ju10__SVBool_tPv(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[PTR]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[PTR]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svst1_ver_za128(uint32_t slice_base, svbool_t pg, void *ptr) {
   svst1_ver_za128(0, slice_base, pg, ptr);
   svst1_ver_za128(15, slice_base, pg, ptr);
 }
+//// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+// CHECK: {{.*}}
diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_st1_vnum.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_st1_vnum.c
index 81a2bba953..6b534f25a1 100644
--- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_st1_vnum.c
+++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_st1_vnum.c
@@ -1,3 +1,4 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 3
 // REQUIRES: aarch64-registered-target
 // RUN: %clang_cc1 -DDISABLE_SME_ATTRIBUTES -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s -check-prefixes=CHECK,CHECK-C
 // RUN: %clang_cc1 -DDISABLE_SME_ATTRIBUTES -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefixes=CHECK,CHECK-CXX
@@ -11,168 +12,286 @@
 #define ARM_STREAMING_ATTR __attribute__((arm_streaming))
 #endif
 
-// CHECK-C-LABEL:   @test_svst1_hor_vnum_za8(
-// CHECK-CXX-LABEL: @_Z23test_svst1_hor_vnum_za8ju10__SVBool_tPvl(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-NEXT:        [[MULVL:%.*]] = mul i64 [[TMP0]], [[VNUM:%.*]]
-// CHECK-NEXT:        [[TMP1:%.*]] = getelementptr i8, [[PTRTY:ptr|i8\*]] [[PTR:%.*]], i64 [[MULVL]]
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1b.horiz(<vscale x 16 x i1> [[PG:%.*]], [[PTRTY]] [[TMP1]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        [[TILESLICE2:%.*]] = add i32 [[SLICE_BASE]], 15
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1b.horiz(<vscale x 16 x i1> [[PG]], [[PTRTY]] [[TMP1]], i32 0, i32 [[TILESLICE2]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svst1_hor_vnum_za8(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP0]], [[VNUM]]
+// CHECK-C-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[ADD]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z23test_svst1_hor_vnum_za8ju10__SVBool_tPvl(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP0]], [[VNUM]]
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svst1_hor_vnum_za8(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) {
   svst1_hor_vnum_za8(0, slice_base, pg, ptr, vnum);
   svst1_hor_vnum_za8(0, slice_base + 15, pg, ptr, vnum);
 }
 
-// CHECK-C-LABEL:   @test_svst1_hor_vnum_za16(
-// CHECK-CXX-LABEL: @_Z24test_svst1_hor_vnum_za16ju10__SVBool_tPvl(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:        [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-NEXT:        [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM:%.*]]
-// CHECK-NEXT:        [[TMP2:%.*]] = getelementptr i8, [[PTRTY]] [[PTR:%.*]], i64 [[MULVL]]
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1h.horiz(<vscale x 8 x i1> [[TMP0]], [[PTRTY]] [[TMP2]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        [[TILESLICE2:%.*]] = add i32 [[SLICE_BASE]], 7
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1h.horiz(<vscale x 8 x i1> [[TMP0]], [[PTRTY]] [[TMP2]], i32 1, i32 [[TILESLICE2]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svst1_hor_vnum_za16(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
+// CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 1, i32 [[ADD]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z24test_svst1_hor_vnum_za16ju10__SVBool_tPvl(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 1, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svst1_hor_vnum_za16(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) {
   svst1_hor_vnum_za16(0, slice_base, pg, ptr, vnum);
   svst1_hor_vnum_za16(1, slice_base + 7, pg, ptr, vnum);
 }
 
-// CHECK-C-LABEL:   @test_svst1_hor_vnum_za32(
-// CHECK-CXX-LABEL: @_Z24test_svst1_hor_vnum_za32ju10__SVBool_tPvl(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:        [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-NEXT:        [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM:%.*]]
-// CHECK-NEXT:        [[TMP2:%.*]] = getelementptr i8, [[PTRTY]] [[PTR:%.*]], i64 [[MULVL]]
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1w.horiz(<vscale x 4 x i1> [[TMP0]], [[PTRTY]] [[TMP2]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        [[TILESLICE2:%.*]] = add i32 [[SLICE_BASE]], 3
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1w.horiz(<vscale x 4 x i1> [[TMP0]], [[PTRTY]] [[TMP2]], i32 3, i32 [[TILESLICE2]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svst1_hor_vnum_za32(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
+// CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 3, i32 [[ADD]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z24test_svst1_hor_vnum_za32ju10__SVBool_tPvl(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 3, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svst1_hor_vnum_za32(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) {
   svst1_hor_vnum_za32(0, slice_base, pg, ptr, vnum);
   svst1_hor_vnum_za32(3, slice_base + 3, pg, ptr, vnum);
 }
 
-// CHECK-C-LABEL:   @test_svst1_hor_vnum_za64(
-// CHECK-CXX-LABEL: @_Z24test_svst1_hor_vnum_za64ju10__SVBool_tPvl(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:        [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-NEXT:        [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM:%.*]]
-// CHECK-NEXT:        [[TMP2:%.*]] = getelementptr i8, [[PTRTY]] [[PTR:%.*]], i64 [[MULVL]]
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1> [[TMP0]], [[PTRTY]] [[TMP2]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        [[TILESLICE2:%.*]] = add i32 [[SLICE_BASE]], 1
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1> [[TMP0]], [[PTRTY]] [[TMP2]], i32 7, i32 [[TILESLICE2]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svst1_hor_vnum_za64(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
+// CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 7, i32 [[ADD]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z24test_svst1_hor_vnum_za64ju10__SVBool_tPvl(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 7, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svst1_hor_vnum_za64(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) {
   svst1_hor_vnum_za64(0, slice_base, pg, ptr, vnum);
   svst1_hor_vnum_za64(7, slice_base + 1, pg, ptr, vnum);
 }
 
-// CHECK-C-LABEL:   @test_svst1_hor_vnum_za128(
-// CHECK-CXX-LABEL: @_Z25test_svst1_hor_vnum_za128ju10__SVBool_tPvl(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        [[TMP0:%.*]] = tail call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:        [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-NEXT:        [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM:%.*]]
-// CHECK-NEXT:        [[TMP2:%.*]] = getelementptr i8, [[PTRTY]] [[PTR:%.*]], i64 [[MULVL]]
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> [[TMP0]], [[PTRTY]] [[TMP2]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> [[TMP0]], [[PTRTY]] [[TMP2]], i32 15, i32 [[SLICE_BASE]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svst1_hor_vnum_za128(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
+// CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svst1_hor_vnum_za128ju10__SVBool_tPvl(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svst1_hor_vnum_za128(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) {
   svst1_hor_vnum_za128(0, slice_base, pg, ptr, vnum);
   svst1_hor_vnum_za128(15, slice_base, pg, ptr, vnum);
 }
 
-// CHECK-C-LABEL:   @test_svst1_ver_vnum_za8(
-// CHECK-CXX-LABEL: @_Z23test_svst1_ver_vnum_za8ju10__SVBool_tPvl(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-NEXT:        [[MULVL:%.*]] = mul i64 [[TMP0]], [[VNUM:%.*]]
-// CHECK-NEXT:        [[TMP1:%.*]] = getelementptr i8, [[PTRTY]] [[PTR:%.*]], i64 [[MULVL]]
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1b.vert(<vscale x 16 x i1> [[PG:%.*]], [[PTRTY]] [[TMP1]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        [[TILESLICE2:%.*]] = add i32 [[SLICE_BASE]], 15
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1b.vert(<vscale x 16 x i1> [[PG]], [[PTRTY]] [[TMP1]], i32 0, i32 [[TILESLICE2]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svst1_ver_vnum_za8(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP0]], [[VNUM]]
+// CHECK-C-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[ADD]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z23test_svst1_ver_vnum_za8ju10__SVBool_tPvl(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP0]], [[VNUM]]
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svst1_ver_vnum_za8(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) {
   svst1_ver_vnum_za8(0, slice_base, pg, ptr, vnum);
   svst1_ver_vnum_za8(0, slice_base + 15, pg, ptr, vnum);
 }
 
-// CHECK-C-LABEL:   @test_svst1_ver_vnum_za16(
-// CHECK-CXX-LABEL: @_Z24test_svst1_ver_vnum_za16ju10__SVBool_tPvl(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:        [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-NEXT:        [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM:%.*]]
-// CHECK-NEXT:        [[TMP2:%.*]] = getelementptr i8, [[PTRTY]] [[PTR:%.*]], i64 [[MULVL]]
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1h.vert(<vscale x 8 x i1> [[TMP0]], [[PTRTY]] [[TMP2]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        [[TILESLICE2:%.*]] = add i32 [[SLICE_BASE]], 7
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1h.vert(<vscale x 8 x i1> [[TMP0]], [[PTRTY]] [[TMP2]], i32 1, i32 [[TILESLICE2]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svst1_ver_vnum_za16(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
+// CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 1, i32 [[ADD]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z24test_svst1_ver_vnum_za16ju10__SVBool_tPvl(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 1, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svst1_ver_vnum_za16(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) {
   svst1_ver_vnum_za16(0, slice_base, pg, ptr, vnum);
   svst1_ver_vnum_za16(1, slice_base + 7, pg, ptr, vnum);
 }
 
-// CHECK-C-LABEL:   @test_svst1_ver_vnum_za32(
-// CHECK-CXX-LABEL: @_Z24test_svst1_ver_vnum_za32ju10__SVBool_tPvl(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:        [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-NEXT:        [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM:%.*]]
-// CHECK-NEXT:        [[TMP2:%.*]] = getelementptr i8, [[PTRTY]] [[PTR:%.*]], i64 [[MULVL]]
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1w.vert(<vscale x 4 x i1> [[TMP0]], [[PTRTY]] [[TMP2]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        [[TILESLICE2:%.*]] = add i32 [[SLICE_BASE]], 3
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1w.vert(<vscale x 4 x i1> [[TMP0]], [[PTRTY]] [[TMP2]], i32 3, i32 [[TILESLICE2]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svst1_ver_vnum_za32(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
+// CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 3, i32 [[ADD]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z24test_svst1_ver_vnum_za32ju10__SVBool_tPvl(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 3, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svst1_ver_vnum_za32(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) {
   svst1_ver_vnum_za32(0, slice_base, pg, ptr, vnum);
   svst1_ver_vnum_za32(3, slice_base + 3, pg, ptr, vnum);
 }
 
-// CHECK-C-LABEL:   @test_svst1_ver_vnum_za64(
-// CHECK-CXX-LABEL: @_Z24test_svst1_ver_vnum_za64ju10__SVBool_tPvl(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:        [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-NEXT:        [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM:%.*]]
-// CHECK-NEXT:        [[TMP2:%.*]] = getelementptr i8, [[PTRTY]] [[PTR:%.*]], i64 [[MULVL]]
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1> [[TMP0]], [[PTRTY]] [[TMP2]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        [[TILESLICE2:%.*]] = add i32 [[SLICE_BASE]], 1
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1> [[TMP0]], [[PTRTY]] [[TMP2]], i32 7, i32 [[TILESLICE2]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svst1_ver_vnum_za64(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
+// CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 7, i32 [[ADD]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z24test_svst1_ver_vnum_za64ju10__SVBool_tPvl(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 7, i32 [[ADD]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svst1_ver_vnum_za64(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) {
   svst1_ver_vnum_za64(0, slice_base, pg, ptr, vnum);
   svst1_ver_vnum_za64(7, slice_base + 1, pg, ptr, vnum);
 }
 
-// CHECK-C-LABEL:   @test_svst1_ver_vnum_za128(
-// CHECK-CXX-LABEL: @_Z25test_svst1_ver_vnum_za128ju10__SVBool_tPvl(
-// CHECK-NEXT:      entry:
-// CHECK-NEXT:        [[TMP0:%.*]] = tail call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:        [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-NEXT:        [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM:%.*]]
-// CHECK-NEXT:        [[TMP2:%.*]] = getelementptr i8, [[PTRTY]] [[PTR:%.*]], i64 [[MULVL]]
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> [[TMP0]], [[PTRTY]] [[TMP2]], i32 0, i32 [[SLICE_BASE:%.*]])
-// CHECK-NEXT:        tail call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> [[TMP0]], [[PTRTY]] [[TMP2]], i32 15, i32 [[SLICE_BASE]])
-// CHECK-NEXT:        ret void
+// CHECK-C-LABEL: define dso_local void @test_svst1_ver_vnum_za128(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
+// CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svst1_ver_vnum_za128ju10__SVBool_tPvl(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 15, i32 [[SLICE_BASE]])
+// CHECK-CXX-NEXT:    ret void
 //
 ARM_STREAMING_ATTR void test_svst1_ver_vnum_za128(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) {
   svst1_ver_vnum_za128(0, slice_base, pg, ptr, vnum);
   svst1_ver_vnum_za128(15, slice_base, pg, ptr, vnum);
 }
+//// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+// CHECK: {{.*}}
diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_str.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_str.c
index 2728f9ac0c..e53a3c6c57 100644
--- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_str.c
+++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_str.c
@@ -1,3 +1,4 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 3
 // REQUIRES: aarch64-registered-target
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s -check-prefixes=CHECK,CHECK-C
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefixes=CHECK,CHECK-CXX
@@ -5,36 +6,86 @@
 
 #include <arm_sme_draft_spec_subject_to_change.h>
 
-// CHECK-C-LABEL: @test_svstr_vnum_za(
-// CHECK-CXX-LABEL: @_Z18test_svstr_vnum_zajPv(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.str(i32 [[SLICE_BASE:%.*]], ptr [[PTR:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svstr_vnum_za(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.str(i32 [[SLICE_BASE]], ptr [[PTR]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z18test_svstr_vnum_zajPv(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.str(i32 [[SLICE_BASE]], ptr [[PTR]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svstr_vnum_za(uint32_t slice_base, void *ptr) {
   svstr_vnum_za(slice_base, ptr, 0);
 }
 
-// CHECK-C-LABEL: @test_svstr_vnum_za_1(
-// CHECK-CXX-LABEL: @_Z20test_svstr_vnum_za_1jPv(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[SVLB:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-NEXT:    [[MULVL:%.*]] = mul i64 [[SVLB]], 15
-// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[PTR:%.*]], i64 [[MULVL]]
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 15
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.str(i32 [[TILESLICE]], ptr [[TMP0]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svstr_vnum_za_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[SVLB:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[SVLB]], 15
+// CHECK-C-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.str(i32 [[TILESLICE]], ptr [[TMP0]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z20test_svstr_vnum_za_1jPv(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[SVLB:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[SVLB]], 15
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.str(i32 [[TILESLICE]], ptr [[TMP0]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svstr_vnum_za_1(uint32_t slice_base, void *ptr) {
   svstr_vnum_za(slice_base, ptr, 15);
 }
 
-// CHECK-C-LABEL: @test_svstr_za(
-// CHECK-CXX-LABEL: @_Z13test_svstr_zajPv(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.str(i32 [[SLICE_BASE:%.*]], ptr [[PTR:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svstr_za(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.str(i32 [[SLICE_BASE]], ptr [[PTR]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z13test_svstr_zajPv(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.str(i32 [[SLICE_BASE]], ptr [[PTR]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svstr_za(uint32_t slice_base, void *ptr) {
   svstr_za(slice_base, ptr);
 }
+
+// CHECK-C-LABEL: define dso_local void @test_svstr_vnum_za_var(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[SVLB:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[SVLB]], [[VNUM]]
+// CHECK-C-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    [[TMP1:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TILESLICE:%.*]] = add i32 [[TMP1]], [[SLICE_BASE]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.str(i32 [[TILESLICE]], ptr [[TMP0]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z22test_svstr_vnum_za_varjPvl(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[SVLB:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[SVLB]], [[VNUM]]
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TILESLICE:%.*]] = add i32 [[TMP1]], [[SLICE_BASE]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.str(i32 [[TILESLICE]], ptr [[TMP0]])
+// CHECK-CXX-NEXT:    ret void
+//
+void test_svstr_vnum_za_var(uint32_t slice_base, void *ptr, int64_t vnum) {
+  svstr_vnum_za(slice_base, ptr, vnum);
+}
+//// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+// CHECK: {{.*}}
diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_write.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_write.c
index 395918b936..0919968e5a 100644
--- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_write.c
+++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_write.c
@@ -1,3 +1,4 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 3
 // REQUIRES: aarch64-registered-target
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s -check-prefixes=CHECK,CHECK-C
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefixes=CHECK,CHECK-CXX
@@ -13,1090 +14,1772 @@
 #define SME_ACLE_FUNC(A1,A2,A3) A1##A2##A3
 #endif
 
-// CHECK-C-LABEL: @test_svwrite_hor_za8_s8(
-// CHECK-CXX-LABEL: @_Z23test_svwrite_hor_za8_s8ju10__SVBool_tu10__SVInt8_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv16i8(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za8_s8(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv16i8(i32 0, i32 [[SLICE_BASE]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z23test_svwrite_hor_za8_s8ju10__SVBool_tu10__SVInt8_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv16i8(i32 0, i32 [[SLICE_BASE]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za8_s8(uint32_t slice_base, svbool_t pg, svint8_t zn) {
   SME_ACLE_FUNC(svwrite_hor_za8, _s8, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za8_s8_1(
-// CHECK-CXX-LABEL: @_Z25test_svwrite_hor_za8_s8_1ju10__SVBool_tu10__SVInt8_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 15
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv16i8(i32 0, i32 [[TILESLICE]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za8_s8_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv16i8(i32 0, i32 [[ADD]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svwrite_hor_za8_s8_1ju10__SVBool_tu10__SVInt8_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv16i8(i32 0, i32 [[ADD]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za8_s8_1(uint32_t slice_base, svbool_t pg, svint8_t zn) {
    uint32_t slice = slice_base + 15;
   SME_ACLE_FUNC(svwrite_hor_za8, _s8, _m)(0, slice, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za16_s16(
-// CHECK-CXX-LABEL: @_Z25test_svwrite_hor_za16_s16ju10__SVBool_tu11__SVInt16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv8i16(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za16_s16(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x i16> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv8i16(i32 0, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svwrite_hor_za16_s16ju10__SVBool_tu11__SVInt16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x i16> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv8i16(i32 0, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za16_s16(uint32_t slice_base, svbool_t pg, svint16_t zn) {
   SME_ACLE_FUNC(svwrite_hor_za16, _s16, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za16_s16_1(
-// CHECK-CXX-LABEL: @_Z27test_svwrite_hor_za16_s16_1ju10__SVBool_tu11__SVInt16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 7
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv8i16(i32 1, i32 [[TILESLICE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za16_s16_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x i16> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv8i16(i32 1, i32 [[ADD]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z27test_svwrite_hor_za16_s16_1ju10__SVBool_tu11__SVInt16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x i16> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv8i16(i32 1, i32 [[ADD]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za16_s16_1(uint32_t slice_base, svbool_t pg, svint16_t zn) {
   uint32_t slice = slice_base + 7;
   SME_ACLE_FUNC(svwrite_hor_za16, _s16, _m)(1, slice, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za32_s32(
-// CHECK-CXX-LABEL: @_Z25test_svwrite_hor_za32_s32ju10__SVBool_tu11__SVInt32_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv4i32(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za32_s32(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv4i32(i32 0, i32 [[SLICE_BASE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svwrite_hor_za32_s32ju10__SVBool_tu11__SVInt32_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv4i32(i32 0, i32 [[SLICE_BASE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za32_s32(uint32_t slice_base, svbool_t pg, svint32_t zn) {
   SME_ACLE_FUNC(svwrite_hor_za32, _s32, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za32_s32_1(
-// CHECK-CXX-LABEL: @_Z27test_svwrite_hor_za32_s32_1ju10__SVBool_tu11__SVInt32_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 3
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv4i32(i32 3, i32 [[TILESLICE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za32_s32_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv4i32(i32 3, i32 [[ADD]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z27test_svwrite_hor_za32_s32_1ju10__SVBool_tu11__SVInt32_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv4i32(i32 3, i32 [[ADD]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za32_s32_1(uint32_t slice_base, svbool_t pg, svint32_t zn) {
   uint32_t slice = slice_base + 3;
   SME_ACLE_FUNC(svwrite_hor_za32, _s32, _m)(3, slice, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za64_s64(
-// CHECK-CXX-LABEL: @_Z25test_svwrite_hor_za64_s64ju10__SVBool_tu11__SVInt64_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv2i64(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za64_s64(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv2i64(i32 0, i32 [[SLICE_BASE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svwrite_hor_za64_s64ju10__SVBool_tu11__SVInt64_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv2i64(i32 0, i32 [[SLICE_BASE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za64_s64(uint32_t slice_base, svbool_t pg, svint64_t zn) {
   SME_ACLE_FUNC(svwrite_hor_za64, _s64, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za64_s64_1(
-// CHECK-CXX-LABEL: @_Z27test_svwrite_hor_za64_s64_1ju10__SVBool_tu11__SVInt64_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 1
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv2i64(i32 7, i32 [[TILESLICE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za64_s64_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv2i64(i32 7, i32 [[ADD]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z27test_svwrite_hor_za64_s64_1ju10__SVBool_tu11__SVInt64_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv2i64(i32 7, i32 [[ADD]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za64_s64_1(uint32_t slice_base, svbool_t pg, svint64_t zn) {
   uint32_t slice = slice_base + 1;
   SME_ACLE_FUNC(svwrite_hor_za64, _s64, _m)(7, slice, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za8_u8(
-// CHECK-CXX-LABEL: @_Z23test_svwrite_hor_za8_u8ju10__SVBool_tu11__SVUint8_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv16i8(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za8_u8(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv16i8(i32 0, i32 [[SLICE_BASE]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z23test_svwrite_hor_za8_u8ju10__SVBool_tu11__SVUint8_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv16i8(i32 0, i32 [[SLICE_BASE]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za8_u8(uint32_t slice_base, svbool_t pg, svuint8_t zn) {
   SME_ACLE_FUNC(svwrite_hor_za8, _u8, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za8_u8_1(
-// CHECK-CXX-LABEL: @_Z25test_svwrite_hor_za8_u8_1ju10__SVBool_tu11__SVUint8_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 15
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv16i8(i32 0, i32 [[TILESLICE]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za8_u8_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv16i8(i32 0, i32 [[ADD]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svwrite_hor_za8_u8_1ju10__SVBool_tu11__SVUint8_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv16i8(i32 0, i32 [[ADD]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za8_u8_1(uint32_t slice_base, svbool_t pg, svuint8_t zn) {
   uint32_t slice = slice_base + 15;
   SME_ACLE_FUNC(svwrite_hor_za8, _u8, _m)(0, slice, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za16_u16(
-// CHECK-CXX-LABEL: @_Z25test_svwrite_hor_za16_u16ju10__SVBool_tu12__SVUint16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv8i16(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za16_u16(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x i16> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv8i16(i32 0, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svwrite_hor_za16_u16ju10__SVBool_tu12__SVUint16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x i16> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv8i16(i32 0, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za16_u16(uint32_t slice_base, svbool_t pg, svuint16_t zn) {
   SME_ACLE_FUNC(svwrite_hor_za16, _u16, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za16_u16_1(
-// CHECK-CXX-LABEL: @_Z27test_svwrite_hor_za16_u16_1ju10__SVBool_tu12__SVUint16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 7
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv8i16(i32 1, i32 [[TILESLICE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za16_u16_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x i16> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv8i16(i32 1, i32 [[ADD]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z27test_svwrite_hor_za16_u16_1ju10__SVBool_tu12__SVUint16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x i16> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv8i16(i32 1, i32 [[ADD]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za16_u16_1(uint32_t slice_base, svbool_t pg, svuint16_t zn) {
   uint32_t slice = slice_base + 7;
   SME_ACLE_FUNC(svwrite_hor_za16, _u16, _m)(1, slice, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za32_u32(
-// CHECK-CXX-LABEL: @_Z25test_svwrite_hor_za32_u32ju10__SVBool_tu12__SVUint32_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv4i32(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za32_u32(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv4i32(i32 0, i32 [[SLICE_BASE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svwrite_hor_za32_u32ju10__SVBool_tu12__SVUint32_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv4i32(i32 0, i32 [[SLICE_BASE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za32_u32(uint32_t slice_base, svbool_t pg, svuint32_t zn) {
   SME_ACLE_FUNC(svwrite_hor_za32, _u32, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za32_u32_1(
-// CHECK-CXX-LABEL: @_Z27test_svwrite_hor_za32_u32_1ju10__SVBool_tu12__SVUint32_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 3
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv4i32(i32 3, i32 [[TILESLICE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za32_u32_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv4i32(i32 3, i32 [[ADD]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z27test_svwrite_hor_za32_u32_1ju10__SVBool_tu12__SVUint32_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv4i32(i32 3, i32 [[ADD]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za32_u32_1(uint32_t slice_base, svbool_t pg, svuint32_t zn) {
   uint32_t slice = slice_base + 3;
   SME_ACLE_FUNC(svwrite_hor_za32, _u32, _m)(3, slice, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za64_u64(
-// CHECK-CXX-LABEL: @_Z25test_svwrite_hor_za64_u64ju10__SVBool_tu12__SVUint64_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv2i64(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za64_u64(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv2i64(i32 0, i32 [[SLICE_BASE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svwrite_hor_za64_u64ju10__SVBool_tu12__SVUint64_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv2i64(i32 0, i32 [[SLICE_BASE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za64_u64(uint32_t slice_base, svbool_t pg, svuint64_t zn) {
   SME_ACLE_FUNC(svwrite_hor_za64, _u64, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za64_u64_1(
-// CHECK-CXX-LABEL: @_Z27test_svwrite_hor_za64_u64_1ju10__SVBool_tu12__SVUint64_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 1
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv2i64(i32 7, i32 [[TILESLICE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za64_u64_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv2i64(i32 7, i32 [[ADD]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z27test_svwrite_hor_za64_u64_1ju10__SVBool_tu12__SVUint64_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv2i64(i32 7, i32 [[ADD]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za64_u64_1(uint32_t slice_base, svbool_t pg, svuint64_t zn) {
   uint32_t slice = slice_base + 1;
   SME_ACLE_FUNC(svwrite_hor_za64, _u64, _m)(7, slice, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za16_f16(
-// CHECK-CXX-LABEL: @_Z25test_svwrite_hor_za16_f16ju10__SVBool_tu13__SVFloat16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv8f16(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za16_f16(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv8f16(i32 0, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svwrite_hor_za16_f16ju10__SVBool_tu13__SVFloat16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv8f16(i32 0, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za16_f16(uint32_t slice_base, svbool_t pg, svfloat16_t zn) {
   SME_ACLE_FUNC(svwrite_hor_za16, _f16, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za16_f16_1(
-// CHECK-CXX-LABEL: @_Z27test_svwrite_hor_za16_f16_1ju10__SVBool_tu13__SVFloat16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 7
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv8f16(i32 1, i32 [[TILESLICE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za16_f16_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv8f16(i32 1, i32 [[ADD]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z27test_svwrite_hor_za16_f16_1ju10__SVBool_tu13__SVFloat16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv8f16(i32 1, i32 [[ADD]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za16_f16_1(uint32_t slice_base, svbool_t pg, svfloat16_t zn) {
   uint32_t slice = slice_base + 7;
   SME_ACLE_FUNC(svwrite_hor_za16, _f16, _m)(1, slice, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za16_bf16(
-// CHECK-CXX-LABEL: @_Z26test_svwrite_hor_za16_bf16ju10__SVBool_tu14__SVBFloat16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv8bf16(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za16_bf16(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv8bf16(i32 0, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z26test_svwrite_hor_za16_bf16ju10__SVBool_tu14__SVBFloat16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv8bf16(i32 0, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za16_bf16(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) {
   SME_ACLE_FUNC(svwrite_hor_za16, _bf16, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za16_bf16_1(
-// CHECK-CXX-LABEL: @_Z28test_svwrite_hor_za16_bf16_1ju10__SVBool_tu14__SVBFloat16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 7
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv8bf16(i32 1, i32 [[TILESLICE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za16_bf16_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv8bf16(i32 1, i32 [[ADD]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z28test_svwrite_hor_za16_bf16_1ju10__SVBool_tu14__SVBFloat16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv8bf16(i32 1, i32 [[ADD]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za16_bf16_1(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) {
    uint32_t slice = slice_base + 7;
   SME_ACLE_FUNC(svwrite_hor_za16, _bf16, _m)(1, slice, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za32_f32(
-// CHECK-CXX-LABEL: @_Z25test_svwrite_hor_za32_f32ju10__SVBool_tu13__SVFloat32_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv4f32(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za32_f32(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x float> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv4f32(i32 0, i32 [[SLICE_BASE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svwrite_hor_za32_f32ju10__SVBool_tu13__SVFloat32_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x float> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv4f32(i32 0, i32 [[SLICE_BASE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za32_f32(uint32_t slice_base, svbool_t pg, svfloat32_t zn) {
   SME_ACLE_FUNC(svwrite_hor_za32, _f32, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za32_f32_1(
-// CHECK-CXX-LABEL: @_Z27test_svwrite_hor_za32_f32_1ju10__SVBool_tu13__SVFloat32_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 3
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv4f32(i32 3, i32 [[TILESLICE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za32_f32_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x float> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv4f32(i32 3, i32 [[ADD]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z27test_svwrite_hor_za32_f32_1ju10__SVBool_tu13__SVFloat32_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x float> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv4f32(i32 3, i32 [[ADD]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za32_f32_1(uint32_t slice_base, svbool_t pg, svfloat32_t zn) {
   uint32_t slice = slice_base + 3;
   SME_ACLE_FUNC(svwrite_hor_za32, _f32, _m)(3, slice, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za64_f64(
-// CHECK-CXX-LABEL: @_Z25test_svwrite_hor_za64_f64ju10__SVBool_tu13__SVFloat64_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv2f64(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za64_f64(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x double> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv2f64(i32 0, i32 [[SLICE_BASE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svwrite_hor_za64_f64ju10__SVBool_tu13__SVFloat64_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x double> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv2f64(i32 0, i32 [[SLICE_BASE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za64_f64(uint32_t slice_base, svbool_t pg, svfloat64_t zn) {
   SME_ACLE_FUNC(svwrite_hor_za64, _f64, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za64_f64_1(
-// CHECK-CXX-LABEL: @_Z27test_svwrite_hor_za64_f64_1ju10__SVBool_tu13__SVFloat64_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 1
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv2f64(i32 7, i32 [[TILESLICE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za64_f64_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x double> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv2f64(i32 7, i32 [[ADD]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z27test_svwrite_hor_za64_f64_1ju10__SVBool_tu13__SVFloat64_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x double> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.horiz.nxv2f64(i32 7, i32 [[ADD]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za64_f64_1(uint32_t slice_base, svbool_t pg, svfloat64_t zn) {
   uint32_t slice = slice_base + 1;
   SME_ACLE_FUNC(svwrite_hor_za64, _f64, _m)(7, slice, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za128_s8(
-// CHECK-CXX-LABEL: @_Z25test_svwrite_hor_za128_s8ju10__SVBool_tu10__SVInt8_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv16i8(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za128_s8(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv16i8(i32 0, i32 [[SLICE_BASE]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svwrite_hor_za128_s8ju10__SVBool_tu10__SVInt8_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv16i8(i32 0, i32 [[SLICE_BASE]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za128_s8(uint32_t slice_base, svbool_t pg, svint8_t zn) {
   SME_ACLE_FUNC(svwrite_hor_za128, _s8, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za128_s8_1(
-// CHECK-CXX-LABEL: @_Z27test_svwrite_hor_za128_s8_1ju10__SVBool_tu10__SVInt8_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv16i8(i32 15, i32 [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za128_s8_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv16i8(i32 15, i32 [[SLICE_BASE]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z27test_svwrite_hor_za128_s8_1ju10__SVBool_tu10__SVInt8_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv16i8(i32 15, i32 [[SLICE_BASE]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za128_s8_1(uint32_t slice_base, svbool_t pg, svint8_t zn) {
   SME_ACLE_FUNC(svwrite_hor_za128, _s8, _m)(15, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za128_s16(
-// CHECK-CXX-LABEL: @_Z26test_svwrite_hor_za128_s16ju10__SVBool_tu11__SVInt16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv8i16(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za128_s16(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x i16> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv8i16(i32 0, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z26test_svwrite_hor_za128_s16ju10__SVBool_tu11__SVInt16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x i16> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv8i16(i32 0, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za128_s16(uint32_t slice_base, svbool_t pg, svint16_t zn) {
   SME_ACLE_FUNC(svwrite_hor_za128, _s16, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za128_s16_1(
-// CHECK-CXX-LABEL: @_Z28test_svwrite_hor_za128_s16_1ju10__SVBool_tu11__SVInt16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv8i16(i32 15, i32 [[SLICE_BASE:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za128_s16_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x i16> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv8i16(i32 15, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z28test_svwrite_hor_za128_s16_1ju10__SVBool_tu11__SVInt16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x i16> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv8i16(i32 15, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za128_s16_1(uint32_t slice_base, svbool_t pg, svint16_t zn) {
   SME_ACLE_FUNC(svwrite_hor_za128, _s16, _m)(15, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za128_s32(
-// CHECK-CXX-LABEL: @_Z26test_svwrite_hor_za128_s32ju10__SVBool_tu11__SVInt32_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv4i32(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za128_s32(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv4i32(i32 0, i32 [[SLICE_BASE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z26test_svwrite_hor_za128_s32ju10__SVBool_tu11__SVInt32_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv4i32(i32 0, i32 [[SLICE_BASE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za128_s32(uint32_t slice_base, svbool_t pg, svint32_t zn) {
   SME_ACLE_FUNC(svwrite_hor_za128, _s32, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za128_s32_1(
-// CHECK-CXX-LABEL: @_Z28test_svwrite_hor_za128_s32_1ju10__SVBool_tu11__SVInt32_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv4i32(i32 15, i32 [[SLICE_BASE:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za128_s32_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv4i32(i32 15, i32 [[SLICE_BASE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z28test_svwrite_hor_za128_s32_1ju10__SVBool_tu11__SVInt32_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv4i32(i32 15, i32 [[SLICE_BASE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za128_s32_1(uint32_t slice_base, svbool_t pg, svint32_t zn) {
   SME_ACLE_FUNC(svwrite_hor_za128, _s32, _m)(15, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za128_s64(
-// CHECK-CXX-LABEL: @_Z26test_svwrite_hor_za128_s64ju10__SVBool_tu11__SVInt64_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv2i64(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za128_s64(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv2i64(i32 0, i32 [[SLICE_BASE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z26test_svwrite_hor_za128_s64ju10__SVBool_tu11__SVInt64_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv2i64(i32 0, i32 [[SLICE_BASE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za128_s64(uint32_t slice_base, svbool_t pg, svint64_t zn) {
   SME_ACLE_FUNC(svwrite_hor_za128, _s64, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za128_s64_1(
-// CHECK-CXX-LABEL: @_Z28test_svwrite_hor_za128_s64_1ju10__SVBool_tu11__SVInt64_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv2i64(i32 15, i32 [[SLICE_BASE:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za128_s64_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv2i64(i32 15, i32 [[SLICE_BASE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z28test_svwrite_hor_za128_s64_1ju10__SVBool_tu11__SVInt64_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv2i64(i32 15, i32 [[SLICE_BASE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za128_s64_1(uint32_t slice_base, svbool_t pg, svint64_t zn) {
   SME_ACLE_FUNC(svwrite_hor_za128, _s64, _m)(15, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za128_u8(
-// CHECK-CXX-LABEL: @_Z25test_svwrite_hor_za128_u8ju10__SVBool_tu11__SVUint8_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv16i8(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za128_u8(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv16i8(i32 0, i32 [[SLICE_BASE]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svwrite_hor_za128_u8ju10__SVBool_tu11__SVUint8_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv16i8(i32 0, i32 [[SLICE_BASE]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za128_u8(uint32_t slice_base, svbool_t pg, svuint8_t zn) {
   SME_ACLE_FUNC(svwrite_hor_za128, _u8, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za128_u8_1(
-// CHECK-CXX-LABEL: @_Z27test_svwrite_hor_za128_u8_1ju10__SVBool_tu11__SVUint8_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv16i8(i32 15, i32 [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za128_u8_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv16i8(i32 15, i32 [[SLICE_BASE]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z27test_svwrite_hor_za128_u8_1ju10__SVBool_tu11__SVUint8_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv16i8(i32 15, i32 [[SLICE_BASE]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za128_u8_1(uint32_t slice_base, svbool_t pg, svuint8_t zn) {
   SME_ACLE_FUNC(svwrite_hor_za128, _u8, _m)(15, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za128_u16(
-// CHECK-CXX-LABEL: @_Z26test_svwrite_hor_za128_u16ju10__SVBool_tu12__SVUint16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv8i16(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za128_u16(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x i16> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv8i16(i32 0, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z26test_svwrite_hor_za128_u16ju10__SVBool_tu12__SVUint16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x i16> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv8i16(i32 0, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za128_u16(uint32_t slice_base, svbool_t pg, svuint16_t zn) {
   SME_ACLE_FUNC(svwrite_hor_za128, _u16, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za128_u16_1(
-// CHECK-CXX-LABEL: @_Z28test_svwrite_hor_za128_u16_1ju10__SVBool_tu12__SVUint16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv8i16(i32 15, i32 [[SLICE_BASE:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za128_u16_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x i16> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv8i16(i32 15, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z28test_svwrite_hor_za128_u16_1ju10__SVBool_tu12__SVUint16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x i16> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv8i16(i32 15, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za128_u16_1(uint32_t slice_base, svbool_t pg, svuint16_t zn) {
   SME_ACLE_FUNC(svwrite_hor_za128, _u16, _m)(15, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za128_u32(
-// CHECK-CXX-LABEL: @_Z26test_svwrite_hor_za128_u32ju10__SVBool_tu12__SVUint32_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv4i32(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za128_u32(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv4i32(i32 0, i32 [[SLICE_BASE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z26test_svwrite_hor_za128_u32ju10__SVBool_tu12__SVUint32_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv4i32(i32 0, i32 [[SLICE_BASE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za128_u32(uint32_t slice_base, svbool_t pg, svuint32_t zn) {
   SME_ACLE_FUNC(svwrite_hor_za128, _u32, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za128_u32_1(
-// CHECK-CXX-LABEL: @_Z28test_svwrite_hor_za128_u32_1ju10__SVBool_tu12__SVUint32_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv4i32(i32 15, i32 [[SLICE_BASE:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za128_u32_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv4i32(i32 15, i32 [[SLICE_BASE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z28test_svwrite_hor_za128_u32_1ju10__SVBool_tu12__SVUint32_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv4i32(i32 15, i32 [[SLICE_BASE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za128_u32_1(uint32_t slice_base, svbool_t pg, svuint32_t zn) {
   SME_ACLE_FUNC(svwrite_hor_za128, _u32, _m)(15, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za128_u64(
-// CHECK-CXX-LABEL: @_Z26test_svwrite_hor_za128_u64ju10__SVBool_tu12__SVUint64_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv2i64(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za128_u64(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv2i64(i32 0, i32 [[SLICE_BASE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z26test_svwrite_hor_za128_u64ju10__SVBool_tu12__SVUint64_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv2i64(i32 0, i32 [[SLICE_BASE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za128_u64(uint32_t slice_base, svbool_t pg, svuint64_t zn) {
   SME_ACLE_FUNC(svwrite_hor_za128, _u64, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za128_u64_1(
-// CHECK-CXX-LABEL: @_Z28test_svwrite_hor_za128_u64_1ju10__SVBool_tu12__SVUint64_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv2i64(i32 15, i32 [[SLICE_BASE:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za128_u64_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv2i64(i32 15, i32 [[SLICE_BASE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z28test_svwrite_hor_za128_u64_1ju10__SVBool_tu12__SVUint64_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv2i64(i32 15, i32 [[SLICE_BASE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za128_u64_1(uint32_t slice_base, svbool_t pg, svuint64_t zn) {
   SME_ACLE_FUNC(svwrite_hor_za128, _u64, _m)(15, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za128_f16(
-// CHECK-CXX-LABEL: @_Z26test_svwrite_hor_za128_f16ju10__SVBool_tu13__SVFloat16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv8f16(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za128_f16(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv8f16(i32 0, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z26test_svwrite_hor_za128_f16ju10__SVBool_tu13__SVFloat16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv8f16(i32 0, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za128_f16(uint32_t slice_base, svbool_t pg, svfloat16_t zn) {
   SME_ACLE_FUNC(svwrite_hor_za128, _f16, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za128_f16_1(
-// CHECK-CXX-LABEL: @_Z28test_svwrite_hor_za128_f16_1ju10__SVBool_tu13__SVFloat16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv8f16(i32 15, i32 [[SLICE_BASE:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za128_f16_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv8f16(i32 15, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z28test_svwrite_hor_za128_f16_1ju10__SVBool_tu13__SVFloat16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv8f16(i32 15, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za128_f16_1(uint32_t slice_base, svbool_t pg, svfloat16_t zn) {
   SME_ACLE_FUNC(svwrite_hor_za128, _f16, _m)(15, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za128_bf16(
-// CHECK-CXX-LABEL: @_Z27test_svwrite_hor_za128_bf16ju10__SVBool_tu14__SVBFloat16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv8bf16(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za128_bf16(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv8bf16(i32 0, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z27test_svwrite_hor_za128_bf16ju10__SVBool_tu14__SVBFloat16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv8bf16(i32 0, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za128_bf16(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) {
   SME_ACLE_FUNC(svwrite_hor_za128, _bf16, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za128_bf16_1(
-// CHECK-CXX-LABEL: @_Z29test_svwrite_hor_za128_bf16_1ju10__SVBool_tu14__SVBFloat16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv8bf16(i32 15, i32 [[SLICE_BASE:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za128_bf16_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv8bf16(i32 15, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z29test_svwrite_hor_za128_bf16_1ju10__SVBool_tu14__SVBFloat16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv8bf16(i32 15, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za128_bf16_1(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) {
   SME_ACLE_FUNC(svwrite_hor_za128, _bf16, _m)(15, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za128_f32(
-// CHECK-CXX-LABEL: @_Z26test_svwrite_hor_za128_f32ju10__SVBool_tu13__SVFloat32_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv4f32(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za128_f32(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x float> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv4f32(i32 0, i32 [[SLICE_BASE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z26test_svwrite_hor_za128_f32ju10__SVBool_tu13__SVFloat32_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x float> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv4f32(i32 0, i32 [[SLICE_BASE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za128_f32(uint32_t slice_base, svbool_t pg, svfloat32_t zn) {
   SME_ACLE_FUNC(svwrite_hor_za128, _f32, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za128_f32_1(
-// CHECK-CXX-LABEL: @_Z28test_svwrite_hor_za128_f32_1ju10__SVBool_tu13__SVFloat32_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv4f32(i32 15, i32 [[SLICE_BASE:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za128_f32_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x float> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv4f32(i32 15, i32 [[SLICE_BASE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z28test_svwrite_hor_za128_f32_1ju10__SVBool_tu13__SVFloat32_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x float> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv4f32(i32 15, i32 [[SLICE_BASE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za128_f32_1(uint32_t slice_base, svbool_t pg, svfloat32_t zn) {
   SME_ACLE_FUNC(svwrite_hor_za128, _f32, _m)(15, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za128_f64(
-// CHECK-CXX-LABEL: @_Z26test_svwrite_hor_za128_f64ju10__SVBool_tu13__SVFloat64_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv2f64(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za128_f64(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x double> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv2f64(i32 0, i32 [[SLICE_BASE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z26test_svwrite_hor_za128_f64ju10__SVBool_tu13__SVFloat64_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x double> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv2f64(i32 0, i32 [[SLICE_BASE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za128_f64(uint32_t slice_base, svbool_t pg, svfloat64_t zn) {
   SME_ACLE_FUNC(svwrite_hor_za128, _f64, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_hor_za128_f64_1(
-// CHECK-CXX-LABEL: @_Z28test_svwrite_hor_za128_f64_1ju10__SVBool_tu13__SVFloat64_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv2f64(i32 15, i32 [[SLICE_BASE:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_hor_za128_f64_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x double> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv2f64(i32 15, i32 [[SLICE_BASE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z28test_svwrite_hor_za128_f64_1ju10__SVBool_tu13__SVFloat64_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x double> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.horiz.nxv2f64(i32 15, i32 [[SLICE_BASE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_hor_za128_f64_1(uint32_t slice_base, svbool_t pg, svfloat64_t zn) {
   SME_ACLE_FUNC(svwrite_hor_za128, _f64, _m)(15, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za8_s8(
-// CHECK-CXX-LABEL: @_Z23test_svwrite_ver_za8_s8ju10__SVBool_tu10__SVInt8_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv16i8(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za8_s8(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv16i8(i32 0, i32 [[SLICE_BASE]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z23test_svwrite_ver_za8_s8ju10__SVBool_tu10__SVInt8_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv16i8(i32 0, i32 [[SLICE_BASE]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za8_s8(uint32_t slice_base, svbool_t pg, svint8_t zn) {
   SME_ACLE_FUNC(svwrite_ver_za8, _s8, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za8_s8_1(
-// CHECK-CXX-LABEL: @_Z25test_svwrite_ver_za8_s8_1ju10__SVBool_tu10__SVInt8_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 15
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv16i8(i32 0, i32 [[TILESLICE]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za8_s8_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv16i8(i32 0, i32 [[ADD]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svwrite_ver_za8_s8_1ju10__SVBool_tu10__SVInt8_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv16i8(i32 0, i32 [[ADD]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za8_s8_1(uint32_t slice_base, svbool_t pg, svint8_t zn) {
   uint32_t slice = slice_base + 15;
   SME_ACLE_FUNC(svwrite_ver_za8, _s8, _m)(0, slice, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za16_s16(
-// CHECK-CXX-LABEL: @_Z25test_svwrite_ver_za16_s16ju10__SVBool_tu11__SVInt16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv8i16(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za16_s16(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x i16> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv8i16(i32 0, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svwrite_ver_za16_s16ju10__SVBool_tu11__SVInt16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x i16> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv8i16(i32 0, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za16_s16(uint32_t slice_base, svbool_t pg, svint16_t zn) {
   SME_ACLE_FUNC(svwrite_ver_za16, _s16, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za16_s16_1(
-// CHECK-CXX-LABEL: @_Z27test_svwrite_ver_za16_s16_1ju10__SVBool_tu11__SVInt16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 7
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv8i16(i32 1, i32 [[TILESLICE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za16_s16_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x i16> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv8i16(i32 1, i32 [[ADD]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z27test_svwrite_ver_za16_s16_1ju10__SVBool_tu11__SVInt16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x i16> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv8i16(i32 1, i32 [[ADD]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za16_s16_1(uint32_t slice_base, svbool_t pg, svint16_t zn) {
   uint32_t slice = slice_base + 7;
   SME_ACLE_FUNC(svwrite_ver_za16, _s16, _m)(1, slice, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za32_s32(
-// CHECK-CXX-LABEL: @_Z25test_svwrite_ver_za32_s32ju10__SVBool_tu11__SVInt32_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv4i32(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za32_s32(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv4i32(i32 0, i32 [[SLICE_BASE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svwrite_ver_za32_s32ju10__SVBool_tu11__SVInt32_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv4i32(i32 0, i32 [[SLICE_BASE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za32_s32(uint32_t slice_base, svbool_t pg, svint32_t zn) {
   SME_ACLE_FUNC(svwrite_ver_za32, _s32, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za32_s32_1(
-// CHECK-CXX-LABEL: @_Z27test_svwrite_ver_za32_s32_1ju10__SVBool_tu11__SVInt32_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 3
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv4i32(i32 3, i32 [[TILESLICE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za32_s32_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv4i32(i32 3, i32 [[ADD]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z27test_svwrite_ver_za32_s32_1ju10__SVBool_tu11__SVInt32_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv4i32(i32 3, i32 [[ADD]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za32_s32_1(uint32_t slice_base, svbool_t pg, svint32_t zn) {
   uint32_t slice = slice_base + 3;
   SME_ACLE_FUNC(svwrite_ver_za32, _s32, _m)(3, slice, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za64_s64(
-// CHECK-CXX-LABEL: @_Z25test_svwrite_ver_za64_s64ju10__SVBool_tu11__SVInt64_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv2i64(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za64_s64(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv2i64(i32 0, i32 [[SLICE_BASE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svwrite_ver_za64_s64ju10__SVBool_tu11__SVInt64_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv2i64(i32 0, i32 [[SLICE_BASE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za64_s64(uint32_t slice_base, svbool_t pg, svint64_t zn) {
   SME_ACLE_FUNC(svwrite_ver_za64, _s64, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za64_s64_1(
-// CHECK-CXX-LABEL: @_Z27test_svwrite_ver_za64_s64_1ju10__SVBool_tu11__SVInt64_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 1
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv2i64(i32 7, i32 [[TILESLICE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za64_s64_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv2i64(i32 7, i32 [[ADD]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z27test_svwrite_ver_za64_s64_1ju10__SVBool_tu11__SVInt64_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv2i64(i32 7, i32 [[ADD]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za64_s64_1(uint32_t slice_base, svbool_t pg, svint64_t zn) {
   uint32_t slice = slice_base + 1;
   SME_ACLE_FUNC(svwrite_ver_za64, _s64, _m)(7, slice, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za8_u8(
-// CHECK-CXX-LABEL: @_Z23test_svwrite_ver_za8_u8ju10__SVBool_tu11__SVUint8_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv16i8(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za8_u8(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv16i8(i32 0, i32 [[SLICE_BASE]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z23test_svwrite_ver_za8_u8ju10__SVBool_tu11__SVUint8_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv16i8(i32 0, i32 [[SLICE_BASE]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za8_u8(uint32_t slice_base, svbool_t pg, svuint8_t zn) {
   SME_ACLE_FUNC(svwrite_ver_za8, _u8, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za8_u8_1(
-// CHECK-CXX-LABEL: @_Z25test_svwrite_ver_za8_u8_1ju10__SVBool_tu11__SVUint8_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 15
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv16i8(i32 0, i32 [[TILESLICE]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za8_u8_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv16i8(i32 0, i32 [[ADD]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svwrite_ver_za8_u8_1ju10__SVBool_tu11__SVUint8_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 15
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv16i8(i32 0, i32 [[ADD]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za8_u8_1(uint32_t slice_base, svbool_t pg, svuint8_t zn) {
   uint32_t slice = slice_base + 15;
   SME_ACLE_FUNC(svwrite_ver_za8, _u8, _m)(0, slice, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za16_u16(
-// CHECK-CXX-LABEL: @_Z25test_svwrite_ver_za16_u16ju10__SVBool_tu12__SVUint16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv8i16(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za16_u16(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x i16> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv8i16(i32 0, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svwrite_ver_za16_u16ju10__SVBool_tu12__SVUint16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x i16> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv8i16(i32 0, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za16_u16(uint32_t slice_base, svbool_t pg, svuint16_t zn) {
   SME_ACLE_FUNC(svwrite_ver_za16, _u16, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za16_u16_1(
-// CHECK-CXX-LABEL: @_Z27test_svwrite_ver_za16_u16_1ju10__SVBool_tu12__SVUint16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 7
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv8i16(i32 1, i32 [[TILESLICE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za16_u16_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x i16> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv8i16(i32 1, i32 [[ADD]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z27test_svwrite_ver_za16_u16_1ju10__SVBool_tu12__SVUint16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x i16> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv8i16(i32 1, i32 [[ADD]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za16_u16_1(uint32_t slice_base, svbool_t pg, svuint16_t zn) {
   uint32_t slice = slice_base + 7;
   SME_ACLE_FUNC(svwrite_ver_za16, _u16, _m)(1, slice, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za32_u32(
-// CHECK-CXX-LABEL: @_Z25test_svwrite_ver_za32_u32ju10__SVBool_tu12__SVUint32_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv4i32(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za32_u32(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv4i32(i32 0, i32 [[SLICE_BASE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svwrite_ver_za32_u32ju10__SVBool_tu12__SVUint32_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv4i32(i32 0, i32 [[SLICE_BASE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za32_u32(uint32_t slice_base, svbool_t pg, svuint32_t zn) {
   SME_ACLE_FUNC(svwrite_ver_za32, _u32, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za32_u32_1(
-// CHECK-CXX-LABEL: @_Z27test_svwrite_ver_za32_u32_1ju10__SVBool_tu12__SVUint32_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 3
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv4i32(i32 3, i32 [[TILESLICE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za32_u32_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv4i32(i32 3, i32 [[ADD]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z27test_svwrite_ver_za32_u32_1ju10__SVBool_tu12__SVUint32_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv4i32(i32 3, i32 [[ADD]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za32_u32_1(uint32_t slice_base, svbool_t pg, svuint32_t zn) {
   uint32_t slice = slice_base + 3;
   SME_ACLE_FUNC(svwrite_ver_za32, _u32, _m)(3, slice, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za64_u64(
-// CHECK-CXX-LABEL: @_Z25test_svwrite_ver_za64_u64ju10__SVBool_tu12__SVUint64_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv2i64(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za64_u64(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv2i64(i32 0, i32 [[SLICE_BASE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svwrite_ver_za64_u64ju10__SVBool_tu12__SVUint64_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv2i64(i32 0, i32 [[SLICE_BASE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za64_u64(uint32_t slice_base, svbool_t pg, svuint64_t zn) {
   SME_ACLE_FUNC(svwrite_ver_za64, _u64, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za64_u64_1(
-// CHECK-CXX-LABEL: @_Z27test_svwrite_ver_za64_u64_1ju10__SVBool_tu12__SVUint64_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 1
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv2i64(i32 7, i32 [[TILESLICE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za64_u64_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv2i64(i32 7, i32 [[ADD]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z27test_svwrite_ver_za64_u64_1ju10__SVBool_tu12__SVUint64_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv2i64(i32 7, i32 [[ADD]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za64_u64_1(uint32_t slice_base, svbool_t pg, svuint64_t zn) {
   uint32_t slice = slice_base + 1;
   SME_ACLE_FUNC(svwrite_ver_za64, _u64, _m)(7, slice, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za16_f16(
-// CHECK-CXX-LABEL: @_Z25test_svwrite_ver_za16_f16ju10__SVBool_tu13__SVFloat16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv8f16(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za16_f16(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv8f16(i32 0, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svwrite_ver_za16_f16ju10__SVBool_tu13__SVFloat16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv8f16(i32 0, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za16_f16(uint32_t slice_base, svbool_t pg, svfloat16_t zn) {
   SME_ACLE_FUNC(svwrite_ver_za16, _f16, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za16_f16_1(
-// CHECK-CXX-LABEL: @_Z27test_svwrite_ver_za16_f16_1ju10__SVBool_tu13__SVFloat16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 7
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv8f16(i32 1, i32 [[TILESLICE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za16_f16_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv8f16(i32 1, i32 [[ADD]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z27test_svwrite_ver_za16_f16_1ju10__SVBool_tu13__SVFloat16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv8f16(i32 1, i32 [[ADD]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za16_f16_1(uint32_t slice_base, svbool_t pg, svfloat16_t zn) {
   uint32_t slice = slice_base + 7;
   SME_ACLE_FUNC(svwrite_ver_za16, _f16, _m)(1, slice, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za16_bf16(
-// CHECK-CXX-LABEL: @_Z26test_svwrite_ver_za16_bf16ju10__SVBool_tu14__SVBFloat16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv8bf16(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za16_bf16(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv8bf16(i32 0, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z26test_svwrite_ver_za16_bf16ju10__SVBool_tu14__SVBFloat16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv8bf16(i32 0, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za16_bf16(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) {
   SME_ACLE_FUNC(svwrite_ver_za16, _bf16, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za16_bf16_1(
-// CHECK-CXX-LABEL: @_Z28test_svwrite_ver_za16_bf16_1ju10__SVBool_tu14__SVBFloat16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 7
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv8bf16(i32 1, i32 [[TILESLICE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za16_bf16_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv8bf16(i32 1, i32 [[ADD]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z28test_svwrite_ver_za16_bf16_1ju10__SVBool_tu14__SVBFloat16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 7
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv8bf16(i32 1, i32 [[ADD]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za16_bf16_1(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) {
   uint32_t slice = slice_base + 7;
   SME_ACLE_FUNC(svwrite_ver_za16, _bf16, _m)(1, slice, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za32_f32(
-// CHECK-CXX-LABEL: @_Z25test_svwrite_ver_za32_f32ju10__SVBool_tu13__SVFloat32_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv4f32(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za32_f32(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x float> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv4f32(i32 0, i32 [[SLICE_BASE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svwrite_ver_za32_f32ju10__SVBool_tu13__SVFloat32_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x float> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv4f32(i32 0, i32 [[SLICE_BASE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za32_f32(uint32_t slice_base, svbool_t pg, svfloat32_t zn) {
   SME_ACLE_FUNC(svwrite_ver_za32, _f32, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za32_f32_1(
-// CHECK-CXX-LABEL: @_Z27test_svwrite_ver_za32_f32_1ju10__SVBool_tu13__SVFloat32_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 3
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv4f32(i32 3, i32 [[TILESLICE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za32_f32_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x float> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv4f32(i32 3, i32 [[ADD]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z27test_svwrite_ver_za32_f32_1ju10__SVBool_tu13__SVFloat32_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x float> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 3
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv4f32(i32 3, i32 [[ADD]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za32_f32_1(uint32_t slice_base, svbool_t pg, svfloat32_t zn) {
   uint32_t slice = slice_base + 3;
   SME_ACLE_FUNC(svwrite_ver_za32, _f32, _m)(3, slice, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za64_f64(
-// CHECK-CXX-LABEL: @_Z25test_svwrite_ver_za64_f64ju10__SVBool_tu13__SVFloat64_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv2f64(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za64_f64(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x double> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv2f64(i32 0, i32 [[SLICE_BASE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svwrite_ver_za64_f64ju10__SVBool_tu13__SVFloat64_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x double> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv2f64(i32 0, i32 [[SLICE_BASE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za64_f64(uint32_t slice_base, svbool_t pg, svfloat64_t zn) {
   SME_ACLE_FUNC(svwrite_ver_za64, _f64, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za64_f64_1(
-// CHECK-CXX-LABEL: @_Z27test_svwrite_ver_za64_f64_1ju10__SVBool_tu13__SVFloat64_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TILESLICE:%.*]] = add i32 [[SLICE_BASE:%.*]], 1
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv2f64(i32 7, i32 [[TILESLICE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za64_f64_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x double> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv2f64(i32 7, i32 [[ADD]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z27test_svwrite_ver_za64_f64_1ju10__SVBool_tu13__SVFloat64_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x double> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], 1
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.write.vert.nxv2f64(i32 7, i32 [[ADD]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za64_f64_1(uint32_t slice_base, svbool_t pg, svfloat64_t zn) {
   uint32_t slice = slice_base + 1;
   SME_ACLE_FUNC(svwrite_ver_za64, _f64, _m)(7, slice, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za128_s8(
-// CHECK-CXX-LABEL: @_Z25test_svwrite_ver_za128_s8ju10__SVBool_tu10__SVInt8_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv16i8(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za128_s8(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv16i8(i32 0, i32 [[SLICE_BASE]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svwrite_ver_za128_s8ju10__SVBool_tu10__SVInt8_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv16i8(i32 0, i32 [[SLICE_BASE]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za128_s8(uint32_t slice_base, svbool_t pg, svint8_t zn) {
   SME_ACLE_FUNC(svwrite_ver_za128, _s8, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za128_s8_1(
-// CHECK-CXX-LABEL: @_Z27test_svwrite_ver_za128_s8_1ju10__SVBool_tu10__SVInt8_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv16i8(i32 15, i32 [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za128_s8_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv16i8(i32 15, i32 [[SLICE_BASE]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z27test_svwrite_ver_za128_s8_1ju10__SVBool_tu10__SVInt8_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv16i8(i32 15, i32 [[SLICE_BASE]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za128_s8_1(uint32_t slice_base, svbool_t pg, svint8_t zn) {
   SME_ACLE_FUNC(svwrite_ver_za128, _s8, _m)(15, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za128_s16(
-// CHECK-CXX-LABEL: @_Z26test_svwrite_ver_za128_s16ju10__SVBool_tu11__SVInt16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv8i16(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za128_s16(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x i16> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv8i16(i32 0, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z26test_svwrite_ver_za128_s16ju10__SVBool_tu11__SVInt16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x i16> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv8i16(i32 0, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za128_s16(uint32_t slice_base, svbool_t pg, svint16_t zn) {
   SME_ACLE_FUNC(svwrite_ver_za128, _s16, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za128_s16_1(
-// CHECK-CXX-LABEL: @_Z28test_svwrite_ver_za128_s16_1ju10__SVBool_tu11__SVInt16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv8i16(i32 15, i32 [[SLICE_BASE:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za128_s16_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x i16> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv8i16(i32 15, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z28test_svwrite_ver_za128_s16_1ju10__SVBool_tu11__SVInt16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x i16> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv8i16(i32 15, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za128_s16_1(uint32_t slice_base, svbool_t pg, svint16_t zn) {
   SME_ACLE_FUNC(svwrite_ver_za128, _s16, _m)(15, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za128_s32(
-// CHECK-CXX-LABEL: @_Z26test_svwrite_ver_za128_s32ju10__SVBool_tu11__SVInt32_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv4i32(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za128_s32(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv4i32(i32 0, i32 [[SLICE_BASE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z26test_svwrite_ver_za128_s32ju10__SVBool_tu11__SVInt32_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv4i32(i32 0, i32 [[SLICE_BASE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za128_s32(uint32_t slice_base, svbool_t pg, svint32_t zn) {
   SME_ACLE_FUNC(svwrite_ver_za128, _s32, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za128_s32_1(
-// CHECK-CXX-LABEL: @_Z28test_svwrite_ver_za128_s32_1ju10__SVBool_tu11__SVInt32_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv4i32(i32 15, i32 [[SLICE_BASE:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za128_s32_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv4i32(i32 15, i32 [[SLICE_BASE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z28test_svwrite_ver_za128_s32_1ju10__SVBool_tu11__SVInt32_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv4i32(i32 15, i32 [[SLICE_BASE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za128_s32_1(uint32_t slice_base, svbool_t pg, svint32_t zn) {
   SME_ACLE_FUNC(svwrite_ver_za128, _s32, _m)(15, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za128_s64(
-// CHECK-CXX-LABEL: @_Z26test_svwrite_ver_za128_s64ju10__SVBool_tu11__SVInt64_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv2i64(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za128_s64(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv2i64(i32 0, i32 [[SLICE_BASE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z26test_svwrite_ver_za128_s64ju10__SVBool_tu11__SVInt64_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv2i64(i32 0, i32 [[SLICE_BASE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za128_s64(uint32_t slice_base, svbool_t pg, svint64_t zn) {
   SME_ACLE_FUNC(svwrite_ver_za128, _s64, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za128_s64_1(
-// CHECK-CXX-LABEL: @_Z28test_svwrite_ver_za128_s64_1ju10__SVBool_tu11__SVInt64_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv2i64(i32 15, i32 [[SLICE_BASE:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za128_s64_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv2i64(i32 15, i32 [[SLICE_BASE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z28test_svwrite_ver_za128_s64_1ju10__SVBool_tu11__SVInt64_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv2i64(i32 15, i32 [[SLICE_BASE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za128_s64_1(uint32_t slice_base, svbool_t pg, svint64_t zn) {
   SME_ACLE_FUNC(svwrite_ver_za128, _s64, _m)(15, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za128_u8(
-// CHECK-CXX-LABEL: @_Z25test_svwrite_ver_za128_u8ju10__SVBool_tu11__SVUint8_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv16i8(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za128_u8(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv16i8(i32 0, i32 [[SLICE_BASE]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z25test_svwrite_ver_za128_u8ju10__SVBool_tu11__SVUint8_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv16i8(i32 0, i32 [[SLICE_BASE]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za128_u8(uint32_t slice_base, svbool_t pg, svuint8_t zn) {
   SME_ACLE_FUNC(svwrite_ver_za128, _u8, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za128_u8_1(
-// CHECK-CXX-LABEL: @_Z27test_svwrite_ver_za128_u8_1ju10__SVBool_tu11__SVUint8_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv16i8(i32 15, i32 [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za128_u8_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv16i8(i32 15, i32 [[SLICE_BASE]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z27test_svwrite_ver_za128_u8_1ju10__SVBool_tu11__SVUint8_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv16i8(i32 15, i32 [[SLICE_BASE]], <vscale x 16 x i1> [[PG]], <vscale x 16 x i8> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za128_u8_1(uint32_t slice_base, svbool_t pg, svuint8_t zn) {
   SME_ACLE_FUNC(svwrite_ver_za128, _u8, _m)(15, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za128_u16(
-// CHECK-CXX-LABEL: @_Z26test_svwrite_ver_za128_u16ju10__SVBool_tu12__SVUint16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv8i16(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za128_u16(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x i16> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv8i16(i32 0, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z26test_svwrite_ver_za128_u16ju10__SVBool_tu12__SVUint16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x i16> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv8i16(i32 0, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za128_u16(uint32_t slice_base, svbool_t pg, svuint16_t zn) {
   SME_ACLE_FUNC(svwrite_ver_za128, _u16, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za128_u16_1(
-// CHECK-CXX-LABEL: @_Z28test_svwrite_ver_za128_u16_1ju10__SVBool_tu12__SVUint16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv8i16(i32 15, i32 [[SLICE_BASE:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za128_u16_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x i16> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv8i16(i32 15, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z28test_svwrite_ver_za128_u16_1ju10__SVBool_tu12__SVUint16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x i16> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv8i16(i32 15, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za128_u16_1(uint32_t slice_base, svbool_t pg, svuint16_t zn) {
   SME_ACLE_FUNC(svwrite_ver_za128, _u16, _m)(15, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za128_u32(
-// CHECK-CXX-LABEL: @_Z26test_svwrite_ver_za128_u32ju10__SVBool_tu12__SVUint32_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv4i32(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za128_u32(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv4i32(i32 0, i32 [[SLICE_BASE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z26test_svwrite_ver_za128_u32ju10__SVBool_tu12__SVUint32_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv4i32(i32 0, i32 [[SLICE_BASE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za128_u32(uint32_t slice_base, svbool_t pg, svuint32_t zn) {
   SME_ACLE_FUNC(svwrite_ver_za128, _u32, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za128_u32_1(
-// CHECK-CXX-LABEL: @_Z28test_svwrite_ver_za128_u32_1ju10__SVBool_tu12__SVUint32_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv4i32(i32 15, i32 [[SLICE_BASE:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za128_u32_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv4i32(i32 15, i32 [[SLICE_BASE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z28test_svwrite_ver_za128_u32_1ju10__SVBool_tu12__SVUint32_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x i32> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv4i32(i32 15, i32 [[SLICE_BASE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za128_u32_1(uint32_t slice_base, svbool_t pg, svuint32_t zn) {
   SME_ACLE_FUNC(svwrite_ver_za128, _u32, _m)(15, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za128_u64(
-// CHECK-CXX-LABEL: @_Z26test_svwrite_ver_za128_u64ju10__SVBool_tu12__SVUint64_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv2i64(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za128_u64(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv2i64(i32 0, i32 [[SLICE_BASE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z26test_svwrite_ver_za128_u64ju10__SVBool_tu12__SVUint64_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv2i64(i32 0, i32 [[SLICE_BASE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za128_u64(uint32_t slice_base, svbool_t pg, svuint64_t zn) {
   SME_ACLE_FUNC(svwrite_ver_za128, _u64, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za128_u64_1(
-// CHECK-CXX-LABEL: @_Z28test_svwrite_ver_za128_u64_1ju10__SVBool_tu12__SVUint64_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv2i64(i32 15, i32 [[SLICE_BASE:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za128_u64_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv2i64(i32 15, i32 [[SLICE_BASE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z28test_svwrite_ver_za128_u64_1ju10__SVBool_tu12__SVUint64_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x i64> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv2i64(i32 15, i32 [[SLICE_BASE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za128_u64_1(uint32_t slice_base, svbool_t pg, svuint64_t zn) {
   SME_ACLE_FUNC(svwrite_ver_za128, _u64, _m)(15, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za128_f16(
-// CHECK-CXX-LABEL: @_Z26test_svwrite_ver_za128_f16ju10__SVBool_tu13__SVFloat16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv8f16(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za128_f16(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv8f16(i32 0, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z26test_svwrite_ver_za128_f16ju10__SVBool_tu13__SVFloat16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv8f16(i32 0, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za128_f16(uint32_t slice_base, svbool_t pg, svfloat16_t zn) {
   SME_ACLE_FUNC(svwrite_ver_za128, _f16, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za128_f16_1(
-// CHECK-CXX-LABEL: @_Z28test_svwrite_ver_za128_f16_1ju10__SVBool_tu13__SVFloat16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv8f16(i32 15, i32 [[SLICE_BASE:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za128_f16_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv8f16(i32 15, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z28test_svwrite_ver_za128_f16_1ju10__SVBool_tu13__SVFloat16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x half> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv8f16(i32 15, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za128_f16_1(uint32_t slice_base, svbool_t pg, svfloat16_t zn) {
   SME_ACLE_FUNC(svwrite_ver_za128, _f16, _m)(15, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za128_bf16(
-// CHECK-CXX-LABEL: @_Z27test_svwrite_ver_za128_bf16ju10__SVBool_tu14__SVBFloat16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv8bf16(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za128_bf16(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv8bf16(i32 0, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z27test_svwrite_ver_za128_bf16ju10__SVBool_tu14__SVBFloat16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv8bf16(i32 0, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za128_bf16(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) {
   SME_ACLE_FUNC(svwrite_ver_za128, _bf16, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za128_bf16_1(
-// CHECK-CXX-LABEL: @_Z29test_svwrite_ver_za128_bf16_1ju10__SVBool_tu14__SVBFloat16_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv8bf16(i32 15, i32 [[SLICE_BASE:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za128_bf16_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv8bf16(i32 15, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z29test_svwrite_ver_za128_bf16_1ju10__SVBool_tu14__SVBFloat16_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv8bf16(i32 15, i32 [[SLICE_BASE]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za128_bf16_1(uint32_t slice_base, svbool_t pg, svbfloat16_t zn) {
   SME_ACLE_FUNC(svwrite_ver_za128, _bf16, _m)(15, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za128_f32(
-// CHECK-CXX-LABEL: @_Z26test_svwrite_ver_za128_f32ju10__SVBool_tu13__SVFloat32_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv4f32(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za128_f32(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x float> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv4f32(i32 0, i32 [[SLICE_BASE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z26test_svwrite_ver_za128_f32ju10__SVBool_tu13__SVFloat32_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x float> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv4f32(i32 0, i32 [[SLICE_BASE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za128_f32(uint32_t slice_base, svbool_t pg, svfloat32_t zn) {
   SME_ACLE_FUNC(svwrite_ver_za128, _f32, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za128_f32_1(
-// CHECK-CXX-LABEL: @_Z28test_svwrite_ver_za128_f32_1ju10__SVBool_tu13__SVFloat32_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv4f32(i32 15, i32 [[SLICE_BASE:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za128_f32_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x float> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv4f32(i32 15, i32 [[SLICE_BASE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z28test_svwrite_ver_za128_f32_1ju10__SVBool_tu13__SVFloat32_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 4 x float> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv4f32(i32 15, i32 [[SLICE_BASE]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za128_f32_1(uint32_t slice_base, svbool_t pg, svfloat32_t zn) {
   SME_ACLE_FUNC(svwrite_ver_za128, _f32, _m)(15, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za128_f64(
-// CHECK-CXX-LABEL: @_Z26test_svwrite_ver_za128_f64ju10__SVBool_tu13__SVFloat64_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv2f64(i32 0, i32 [[SLICE_BASE:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za128_f64(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x double> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv2f64(i32 0, i32 [[SLICE_BASE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z26test_svwrite_ver_za128_f64ju10__SVBool_tu13__SVFloat64_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x double> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv2f64(i32 0, i32 [[SLICE_BASE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za128_f64(uint32_t slice_base, svbool_t pg, svfloat64_t zn) {
   SME_ACLE_FUNC(svwrite_ver_za128, _f64, _m)(0, slice_base, pg, zn);
 }
 
-// CHECK-C-LABEL: @test_svwrite_ver_za128_f64_1(
-// CHECK-CXX-LABEL: @_Z28test_svwrite_ver_za128_f64_1ju10__SVBool_tu13__SVFloat64_t(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv2f64(i32 15, i32 [[SLICE_BASE:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> [[ZN:%.*]])
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svwrite_ver_za128_f64_1(
+// CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x double> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv2f64(i32 15, i32 [[SLICE_BASE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> [[ZN]])
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z28test_svwrite_ver_za128_f64_1ju10__SVBool_tu13__SVFloat64_t(
+// CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 2 x double> [[ZN:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.writeq.vert.nxv2f64(i32 15, i32 [[SLICE_BASE]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> [[ZN]])
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svwrite_ver_za128_f64_1(uint32_t slice_base, svbool_t pg, svfloat64_t zn) {
   SME_ACLE_FUNC(svwrite_ver_za128, _f64, _m)(15, slice_base, pg, zn);
 }
+//// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+// CHECK: {{.*}}
diff --git a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_zero.c b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_zero.c
index 3ff9f6346c..750eead7c7 100644
--- a/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_zero.c
+++ b/clang/test/CodeGen/aarch64-sme-intrinsics/acle_sme_zero.c
@@ -1,3 +1,4 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 3
 // REQUIRES: aarch64-registered-target
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s -check-prefixes=CHECK,CHECK-C
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefixes=CHECK,CHECK-CXX
@@ -5,42 +6,68 @@
 
 #include <arm_sme_draft_spec_subject_to_change.h>
 
-// CHECK-C-LABEL: @test_svzero_mask_za(
-// CHECK-CXX-LABEL: @_Z19test_svzero_mask_zav(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.zero(i32 0)
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svzero_mask_za(
+// CHECK-C-SAME: ) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.zero(i32 0)
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z19test_svzero_mask_zav(
+// CHECK-CXX-SAME: ) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.zero(i32 0)
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svzero_mask_za() {
   svzero_mask_za(0);
 }
 
-// CHECK-C-LABEL: @test_svzero_mask_za_1(
-// CHECK-CXX-LABEL: @_Z21test_svzero_mask_za_1v(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.zero(i32 176)
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svzero_mask_za_1(
+// CHECK-C-SAME: ) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.zero(i32 176)
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z21test_svzero_mask_za_1v(
+// CHECK-CXX-SAME: ) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.zero(i32 176)
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svzero_mask_za_1() {
   svzero_mask_za(176);
 }
 
-// CHECK-C-LABEL: @test_svzero_mask_za_2(
-// CHECK-CXX-LABEL: @_Z21test_svzero_mask_za_2v(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.zero(i32 255)
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svzero_mask_za_2(
+// CHECK-C-SAME: ) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.zero(i32 255)
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z21test_svzero_mask_za_2v(
+// CHECK-CXX-SAME: ) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.zero(i32 255)
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svzero_mask_za_2() {
   svzero_mask_za(255);
 }
 
-// CHECK-C-LABEL: @test_svzero_za(
-// CHECK-CXX-LABEL: @_Z14test_svzero_zav(
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    tail call void @llvm.aarch64.sme.zero(i32 255)
-// CHECK-NEXT:    ret void
+// CHECK-C-LABEL: define dso_local void @test_svzero_za(
+// CHECK-C-SAME: ) local_unnamed_addr #[[ATTR0]] {
+// CHECK-C-NEXT:  entry:
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.zero(i32 255)
+// CHECK-C-NEXT:    ret void
+//
+// CHECK-CXX-LABEL: define dso_local void @_Z14test_svzero_zav(
+// CHECK-CXX-SAME: ) local_unnamed_addr #[[ATTR0]] {
+// CHECK-CXX-NEXT:  entry:
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.zero(i32 255)
+// CHECK-CXX-NEXT:    ret void
 //
 void test_svzero_za() {
   svzero_za();
 }
+//// NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+// CHECK: {{.*}}
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1-bfloat.c
index 217329134b..12533fa716 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1-bfloat.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1-bfloat.c
@@ -15,13 +15,13 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_bf16(
+// CHECK-LABEL: @test_svld1_bf16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.masked.load.nxv8bf16.p0(ptr [[BASE:%.*]], i32 1, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> zeroinitializer)
 // CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z15test_svld1_bf16u10__SVBool_tPKu6__bf16(
+// CPP-CHECK-LABEL: @_Z15test_svld1_bf16u10__SVBool_tPKu6__bf16(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.masked.load.nxv8bf16.p0(ptr [[BASE:%.*]], i32 1, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> zeroinitializer)
@@ -32,14 +32,14 @@ svbfloat16_t test_svld1_bf16(svbool_t pg, const bfloat16_t *base)
   return SVE_ACLE_FUNC(svld1,_bf16,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_vnum_bf16(
+// CHECK-LABEL: @test_svld1_vnum_bf16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x bfloat>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.masked.load.nxv8bf16.p0(ptr [[TMP1]], i32 1, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x bfloat> zeroinitializer)
 // CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z20test_svld1_vnum_bf16u10__SVBool_tPKu6__bf16l(
+// CPP-CHECK-LABEL: @_Z20test_svld1_vnum_bf16u10__SVBool_tPKu6__bf16l(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x bfloat>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1.c
index 94d6f97866..c3a5186b1e 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1.c
@@ -14,12 +14,12 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_s8(
+// CHECK-LABEL: @test_svld1_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[BASE:%.*]], i32 1, <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> zeroinitializer)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z13test_svld1_s8u10__SVBool_tPKa(
+// CPP-CHECK-LABEL: @_Z13test_svld1_s8u10__SVBool_tPKa(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[BASE:%.*]], i32 1, <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> zeroinitializer)
 // CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
@@ -29,13 +29,13 @@ svint8_t test_svld1_s8(svbool_t pg, const int8_t *base)
   return SVE_ACLE_FUNC(svld1,_s8,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_s16(
+// CHECK-LABEL: @test_svld1_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr [[BASE:%.*]], i32 1, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> zeroinitializer)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svld1_s16u10__SVBool_tPKs(
+// CPP-CHECK-LABEL: @_Z14test_svld1_s16u10__SVBool_tPKs(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr [[BASE:%.*]], i32 1, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> zeroinitializer)
@@ -46,13 +46,13 @@ svint16_t test_svld1_s16(svbool_t pg, const int16_t *base)
   return SVE_ACLE_FUNC(svld1,_s16,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_s32(
+// CHECK-LABEL: @test_svld1_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr [[BASE:%.*]], i32 1, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> zeroinitializer)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svld1_s32u10__SVBool_tPKi(
+// CPP-CHECK-LABEL: @_Z14test_svld1_s32u10__SVBool_tPKi(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr [[BASE:%.*]], i32 1, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> zeroinitializer)
@@ -63,13 +63,13 @@ svint32_t test_svld1_s32(svbool_t pg, const int32_t *base)
   return SVE_ACLE_FUNC(svld1,_s32,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_s64(
+// CHECK-LABEL: @test_svld1_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[BASE:%.*]], i32 1, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> zeroinitializer)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svld1_s64u10__SVBool_tPKl(
+// CPP-CHECK-LABEL: @_Z14test_svld1_s64u10__SVBool_tPKl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[BASE:%.*]], i32 1, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> zeroinitializer)
@@ -80,12 +80,12 @@ svint64_t test_svld1_s64(svbool_t pg, const int64_t *base)
   return SVE_ACLE_FUNC(svld1,_s64,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_u8(
+// CHECK-LABEL: @test_svld1_u8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[BASE:%.*]], i32 1, <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> zeroinitializer)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z13test_svld1_u8u10__SVBool_tPKh(
+// CPP-CHECK-LABEL: @_Z13test_svld1_u8u10__SVBool_tPKh(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[BASE:%.*]], i32 1, <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> zeroinitializer)
 // CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
@@ -95,13 +95,13 @@ svuint8_t test_svld1_u8(svbool_t pg, const uint8_t *base)
   return SVE_ACLE_FUNC(svld1,_u8,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_u16(
+// CHECK-LABEL: @test_svld1_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr [[BASE:%.*]], i32 1, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> zeroinitializer)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svld1_u16u10__SVBool_tPKt(
+// CPP-CHECK-LABEL: @_Z14test_svld1_u16u10__SVBool_tPKt(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr [[BASE:%.*]], i32 1, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> zeroinitializer)
@@ -112,13 +112,13 @@ svuint16_t test_svld1_u16(svbool_t pg, const uint16_t *base)
   return SVE_ACLE_FUNC(svld1,_u16,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_u32(
+// CHECK-LABEL: @test_svld1_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr [[BASE:%.*]], i32 1, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> zeroinitializer)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svld1_u32u10__SVBool_tPKj(
+// CPP-CHECK-LABEL: @_Z14test_svld1_u32u10__SVBool_tPKj(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr [[BASE:%.*]], i32 1, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> zeroinitializer)
@@ -129,13 +129,13 @@ svuint32_t test_svld1_u32(svbool_t pg, const uint32_t *base)
   return SVE_ACLE_FUNC(svld1,_u32,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_u64(
+// CHECK-LABEL: @test_svld1_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[BASE:%.*]], i32 1, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> zeroinitializer)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svld1_u64u10__SVBool_tPKm(
+// CPP-CHECK-LABEL: @_Z14test_svld1_u64u10__SVBool_tPKm(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[BASE:%.*]], i32 1, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> zeroinitializer)
@@ -146,13 +146,13 @@ svuint64_t test_svld1_u64(svbool_t pg, const uint64_t *base)
   return SVE_ACLE_FUNC(svld1,_u64,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_f16(
+// CHECK-LABEL: @test_svld1_f16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.masked.load.nxv8f16.p0(ptr [[BASE:%.*]], i32 1, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> zeroinitializer)
 // CHECK-NEXT:    ret <vscale x 8 x half> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svld1_f16u10__SVBool_tPKDh(
+// CPP-CHECK-LABEL: @_Z14test_svld1_f16u10__SVBool_tPKDh(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.masked.load.nxv8f16.p0(ptr [[BASE:%.*]], i32 1, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> zeroinitializer)
@@ -163,13 +163,13 @@ svfloat16_t test_svld1_f16(svbool_t pg, const float16_t *base)
   return SVE_ACLE_FUNC(svld1,_f16,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_f32(
+// CHECK-LABEL: @test_svld1_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr [[BASE:%.*]], i32 1, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> zeroinitializer)
 // CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svld1_f32u10__SVBool_tPKf(
+// CPP-CHECK-LABEL: @_Z14test_svld1_f32u10__SVBool_tPKf(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr [[BASE:%.*]], i32 1, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> zeroinitializer)
@@ -180,13 +180,13 @@ svfloat32_t test_svld1_f32(svbool_t pg, const float32_t *base)
   return SVE_ACLE_FUNC(svld1,_f32,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_f64(
+// CHECK-LABEL: @test_svld1_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr [[BASE:%.*]], i32 1, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> zeroinitializer)
 // CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svld1_f64u10__SVBool_tPKd(
+// CPP-CHECK-LABEL: @_Z14test_svld1_f64u10__SVBool_tPKd(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr [[BASE:%.*]], i32 1, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> zeroinitializer)
@@ -197,13 +197,13 @@ svfloat64_t test_svld1_f64(svbool_t pg, const float64_t *base)
   return SVE_ACLE_FUNC(svld1,_f64,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_vnum_s8(
+// CHECK-LABEL: @test_svld1_vnum_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP0]], i32 1, <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> zeroinitializer)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svld1_vnum_s8u10__SVBool_tPKal(
+// CPP-CHECK-LABEL: @_Z18test_svld1_vnum_s8u10__SVBool_tPKal(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP0]], i32 1, <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> zeroinitializer)
@@ -214,14 +214,14 @@ svint8_t test_svld1_vnum_s8(svbool_t pg, const int8_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svld1_vnum,_s8,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_vnum_s16(
+// CHECK-LABEL: @test_svld1_vnum_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr [[TMP1]], i32 1, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> zeroinitializer)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svld1_vnum_s16u10__SVBool_tPKsl(
+// CPP-CHECK-LABEL: @_Z19test_svld1_vnum_s16u10__SVBool_tPKsl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -233,14 +233,14 @@ svint16_t test_svld1_vnum_s16(svbool_t pg, const int16_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svld1_vnum,_s16,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_vnum_s32(
+// CHECK-LABEL: @test_svld1_vnum_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr [[TMP1]], i32 1, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> zeroinitializer)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svld1_vnum_s32u10__SVBool_tPKil(
+// CPP-CHECK-LABEL: @_Z19test_svld1_vnum_s32u10__SVBool_tPKil(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -252,14 +252,14 @@ svint32_t test_svld1_vnum_s32(svbool_t pg, const int32_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svld1_vnum,_s32,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_vnum_s64(
+// CHECK-LABEL: @test_svld1_vnum_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[TMP1]], i32 1, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> zeroinitializer)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svld1_vnum_s64u10__SVBool_tPKll(
+// CPP-CHECK-LABEL: @_Z19test_svld1_vnum_s64u10__SVBool_tPKll(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -271,13 +271,13 @@ svint64_t test_svld1_vnum_s64(svbool_t pg, const int64_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svld1_vnum,_s64,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_vnum_u8(
+// CHECK-LABEL: @test_svld1_vnum_u8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP0]], i32 1, <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> zeroinitializer)
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svld1_vnum_u8u10__SVBool_tPKhl(
+// CPP-CHECK-LABEL: @_Z18test_svld1_vnum_u8u10__SVBool_tPKhl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP0]], i32 1, <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> zeroinitializer)
@@ -288,14 +288,14 @@ svuint8_t test_svld1_vnum_u8(svbool_t pg, const uint8_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svld1_vnum,_u8,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_vnum_u16(
+// CHECK-LABEL: @test_svld1_vnum_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr [[TMP1]], i32 1, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> zeroinitializer)
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svld1_vnum_u16u10__SVBool_tPKtl(
+// CPP-CHECK-LABEL: @_Z19test_svld1_vnum_u16u10__SVBool_tPKtl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -307,14 +307,14 @@ svuint16_t test_svld1_vnum_u16(svbool_t pg, const uint16_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svld1_vnum,_u16,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_vnum_u32(
+// CHECK-LABEL: @test_svld1_vnum_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr [[TMP1]], i32 1, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> zeroinitializer)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svld1_vnum_u32u10__SVBool_tPKjl(
+// CPP-CHECK-LABEL: @_Z19test_svld1_vnum_u32u10__SVBool_tPKjl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -326,14 +326,14 @@ svuint32_t test_svld1_vnum_u32(svbool_t pg, const uint32_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svld1_vnum,_u32,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_vnum_u64(
+// CHECK-LABEL: @test_svld1_vnum_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[TMP1]], i32 1, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> zeroinitializer)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svld1_vnum_u64u10__SVBool_tPKml(
+// CPP-CHECK-LABEL: @_Z19test_svld1_vnum_u64u10__SVBool_tPKml(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -345,14 +345,14 @@ svuint64_t test_svld1_vnum_u64(svbool_t pg, const uint64_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svld1_vnum,_u64,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_vnum_f16(
+// CHECK-LABEL: @test_svld1_vnum_f16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.masked.load.nxv8f16.p0(ptr [[TMP1]], i32 1, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> zeroinitializer)
 // CHECK-NEXT:    ret <vscale x 8 x half> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svld1_vnum_f16u10__SVBool_tPKDhl(
+// CPP-CHECK-LABEL: @_Z19test_svld1_vnum_f16u10__SVBool_tPKDhl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -364,14 +364,14 @@ svfloat16_t test_svld1_vnum_f16(svbool_t pg, const float16_t *base, int64_t vnum
   return SVE_ACLE_FUNC(svld1_vnum,_f16,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_vnum_f32(
+// CHECK-LABEL: @test_svld1_vnum_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr [[TMP1]], i32 1, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> zeroinitializer)
 // CHECK-NEXT:    ret <vscale x 4 x float> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svld1_vnum_f32u10__SVBool_tPKfl(
+// CPP-CHECK-LABEL: @_Z19test_svld1_vnum_f32u10__SVBool_tPKfl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -383,14 +383,14 @@ svfloat32_t test_svld1_vnum_f32(svbool_t pg, const float32_t *base, int64_t vnum
   return SVE_ACLE_FUNC(svld1_vnum,_f32,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_vnum_f64(
+// CHECK-LABEL: @test_svld1_vnum_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr [[TMP1]], i32 1, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> zeroinitializer)
 // CHECK-NEXT:    ret <vscale x 2 x double> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svld1_vnum_f64u10__SVBool_tPKdl(
+// CPP-CHECK-LABEL: @_Z19test_svld1_vnum_f64u10__SVBool_tPKdl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -402,13 +402,13 @@ svfloat64_t test_svld1_vnum_f64(svbool_t pg, const float64_t *base, int64_t vnum
   return SVE_ACLE_FUNC(svld1_vnum,_f64,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_u32base_s32(
+// CHECK-LABEL: @test_svld1_gather_u32base_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z29test_svld1_gather_u32base_s32u10__SVBool_tu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z29test_svld1_gather_u32base_s32u10__SVBool_tu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
@@ -418,13 +418,13 @@ svint32_t test_svld1_gather_u32base_s32(svbool_t pg, svuint32_t bases) {
   return SVE_ACLE_FUNC(svld1_gather, _u32base, _s32, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_u64base_s64(
+// CHECK-LABEL: @test_svld1_gather_u64base_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z29test_svld1_gather_u64base_s64u10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z29test_svld1_gather_u64base_s64u10__SVBool_tu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
@@ -434,13 +434,13 @@ svint64_t test_svld1_gather_u64base_s64(svbool_t pg, svuint64_t bases) {
   return SVE_ACLE_FUNC(svld1_gather, _u64base, _s64, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_u32base_u32(
+// CHECK-LABEL: @test_svld1_gather_u32base_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z29test_svld1_gather_u32base_u32u10__SVBool_tu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z29test_svld1_gather_u32base_u32u10__SVBool_tu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
@@ -450,13 +450,13 @@ svuint32_t test_svld1_gather_u32base_u32(svbool_t pg, svuint32_t bases) {
   return SVE_ACLE_FUNC(svld1_gather, _u32base, _u32, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_u64base_u64(
+// CHECK-LABEL: @test_svld1_gather_u64base_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z29test_svld1_gather_u64base_u64u10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z29test_svld1_gather_u64base_u64u10__SVBool_tu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
@@ -466,13 +466,13 @@ svuint64_t test_svld1_gather_u64base_u64(svbool_t pg, svuint64_t bases) {
   return SVE_ACLE_FUNC(svld1_gather, _u64base, _u64, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_u32base_f32(
+// CHECK-LABEL: @test_svld1_gather_u32base_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4f32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z29test_svld1_gather_u32base_f32u10__SVBool_tu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z29test_svld1_gather_u32base_f32u10__SVBool_tu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4f32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
@@ -482,13 +482,13 @@ svfloat32_t test_svld1_gather_u32base_f32(svbool_t pg, svuint32_t bases) {
   return SVE_ACLE_FUNC(svld1_gather, _u32base, _f32, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_u64base_f64(
+// CHECK-LABEL: @test_svld1_gather_u64base_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2f64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z29test_svld1_gather_u64base_f64u10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z29test_svld1_gather_u64base_f64u10__SVBool_tu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2f64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
@@ -498,13 +498,13 @@ svfloat64_t test_svld1_gather_u64base_f64(svbool_t pg, svuint64_t bases) {
   return SVE_ACLE_FUNC(svld1_gather, _u64base, _f64, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_s32offset_s32(
+// CHECK-LABEL: @test_svld1_gather_s32offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svld1_gather_s32offset_s32u10__SVBool_tPKiu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z31test_svld1_gather_s32offset_s32u10__SVBool_tPKiu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -514,13 +514,13 @@ svint32_t test_svld1_gather_s32offset_s32(svbool_t pg, const int32_t *base, svin
   return SVE_ACLE_FUNC(svld1_gather_, s32, offset, _s32)(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_s64offset_s64(
+// CHECK-LABEL: @test_svld1_gather_s64offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svld1_gather_s64offset_s64u10__SVBool_tPKlu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z31test_svld1_gather_s64offset_s64u10__SVBool_tPKlu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -530,13 +530,13 @@ svint64_t test_svld1_gather_s64offset_s64(svbool_t pg, const int64_t *base, svin
   return SVE_ACLE_FUNC(svld1_gather_, s64, offset, _s64)(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_s32offset_u32(
+// CHECK-LABEL: @test_svld1_gather_s32offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svld1_gather_s32offset_u32u10__SVBool_tPKju11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z31test_svld1_gather_s32offset_u32u10__SVBool_tPKju11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -546,13 +546,13 @@ svuint32_t test_svld1_gather_s32offset_u32(svbool_t pg, const uint32_t *base, sv
   return SVE_ACLE_FUNC(svld1_gather_, s32, offset, _u32)(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_s64offset_u64(
+// CHECK-LABEL: @test_svld1_gather_s64offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svld1_gather_s64offset_u64u10__SVBool_tPKmu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z31test_svld1_gather_s64offset_u64u10__SVBool_tPKmu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -562,13 +562,13 @@ svuint64_t test_svld1_gather_s64offset_u64(svbool_t pg, const uint64_t *base, sv
   return SVE_ACLE_FUNC(svld1_gather_, s64, offset, _u64)(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_s32offset_f32(
+// CHECK-LABEL: @test_svld1_gather_s32offset_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4f32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svld1_gather_s32offset_f32u10__SVBool_tPKfu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z31test_svld1_gather_s32offset_f32u10__SVBool_tPKfu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4f32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -578,13 +578,13 @@ svfloat32_t test_svld1_gather_s32offset_f32(svbool_t pg, const float32_t *base,
   return SVE_ACLE_FUNC(svld1_gather_, s32, offset, _f32)(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_s64offset_f64(
+// CHECK-LABEL: @test_svld1_gather_s64offset_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.nxv2f64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svld1_gather_s64offset_f64u10__SVBool_tPKdu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z31test_svld1_gather_s64offset_f64u10__SVBool_tPKdu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.nxv2f64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -594,13 +594,13 @@ svfloat64_t test_svld1_gather_s64offset_f64(svbool_t pg, const float64_t *base,
   return SVE_ACLE_FUNC(svld1_gather_, s64, offset, _f64)(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_u32offset_s32(
+// CHECK-LABEL: @test_svld1_gather_u32offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svld1_gather_u32offset_s32u10__SVBool_tPKiu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z31test_svld1_gather_u32offset_s32u10__SVBool_tPKiu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -610,13 +610,13 @@ svint32_t test_svld1_gather_u32offset_s32(svbool_t pg, const int32_t *base, svui
   return SVE_ACLE_FUNC(svld1_gather_, u32, offset, _s32)(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_u64offset_s64(
+// CHECK-LABEL: @test_svld1_gather_u64offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svld1_gather_u64offset_s64u10__SVBool_tPKlu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z31test_svld1_gather_u64offset_s64u10__SVBool_tPKlu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -626,13 +626,13 @@ svint64_t test_svld1_gather_u64offset_s64(svbool_t pg, const int64_t *base, svui
   return SVE_ACLE_FUNC(svld1_gather_, u64, offset, _s64)(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_u32offset_u32(
+// CHECK-LABEL: @test_svld1_gather_u32offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svld1_gather_u32offset_u32u10__SVBool_tPKju12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z31test_svld1_gather_u32offset_u32u10__SVBool_tPKju12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -642,13 +642,13 @@ svuint32_t test_svld1_gather_u32offset_u32(svbool_t pg, const uint32_t *base, sv
   return SVE_ACLE_FUNC(svld1_gather_, u32, offset, _u32)(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_u64offset_u64(
+// CHECK-LABEL: @test_svld1_gather_u64offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svld1_gather_u64offset_u64u10__SVBool_tPKmu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z31test_svld1_gather_u64offset_u64u10__SVBool_tPKmu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -658,13 +658,13 @@ svuint64_t test_svld1_gather_u64offset_u64(svbool_t pg, const uint64_t *base, sv
   return SVE_ACLE_FUNC(svld1_gather_, u64, offset, _u64)(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_u32offset_f32(
+// CHECK-LABEL: @test_svld1_gather_u32offset_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4f32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svld1_gather_u32offset_f32u10__SVBool_tPKfu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z31test_svld1_gather_u32offset_f32u10__SVBool_tPKfu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4f32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -674,13 +674,13 @@ svfloat32_t test_svld1_gather_u32offset_f32(svbool_t pg, const float32_t *base,
   return SVE_ACLE_FUNC(svld1_gather_, u32, offset, _f32)(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_u64offset_f64(
+// CHECK-LABEL: @test_svld1_gather_u64offset_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.nxv2f64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svld1_gather_u64offset_f64u10__SVBool_tPKdu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z31test_svld1_gather_u64offset_f64u10__SVBool_tPKdu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.nxv2f64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -690,13 +690,13 @@ svfloat64_t test_svld1_gather_u64offset_f64(svbool_t pg, const float64_t *base,
   return SVE_ACLE_FUNC(svld1_gather_, u64, offset, _f64)(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_u32base_offset_s32(
+// CHECK-LABEL: @test_svld1_gather_u32base_offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z36test_svld1_gather_u32base_offset_s32u10__SVBool_tu12__SVUint32_tl(
+// CPP-CHECK-LABEL: @_Z36test_svld1_gather_u32base_offset_s32u10__SVBool_tu12__SVUint32_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -706,13 +706,13 @@ svint32_t test_svld1_gather_u32base_offset_s32(svbool_t pg, svuint32_t bases, in
   return SVE_ACLE_FUNC(svld1_gather, _u32base, _offset_s32, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_u64base_offset_s64(
+// CHECK-LABEL: @test_svld1_gather_u64base_offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z36test_svld1_gather_u64base_offset_s64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z36test_svld1_gather_u64base_offset_s64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -722,13 +722,13 @@ svint64_t test_svld1_gather_u64base_offset_s64(svbool_t pg, svuint64_t bases, in
   return SVE_ACLE_FUNC(svld1_gather, _u64base, _offset_s64, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_u32base_offset_u32(
+// CHECK-LABEL: @test_svld1_gather_u32base_offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z36test_svld1_gather_u32base_offset_u32u10__SVBool_tu12__SVUint32_tl(
+// CPP-CHECK-LABEL: @_Z36test_svld1_gather_u32base_offset_u32u10__SVBool_tu12__SVUint32_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -738,13 +738,13 @@ svuint32_t test_svld1_gather_u32base_offset_u32(svbool_t pg, svuint32_t bases, i
   return SVE_ACLE_FUNC(svld1_gather, _u32base, _offset_u32, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_u64base_offset_u64(
+// CHECK-LABEL: @test_svld1_gather_u64base_offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z36test_svld1_gather_u64base_offset_u64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z36test_svld1_gather_u64base_offset_u64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -754,13 +754,13 @@ svuint64_t test_svld1_gather_u64base_offset_u64(svbool_t pg, svuint64_t bases, i
   return SVE_ACLE_FUNC(svld1_gather, _u64base, _offset_u64, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_u32base_offset_f32(
+// CHECK-LABEL: @test_svld1_gather_u32base_offset_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4f32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z36test_svld1_gather_u32base_offset_f32u10__SVBool_tu12__SVUint32_tl(
+// CPP-CHECK-LABEL: @_Z36test_svld1_gather_u32base_offset_f32u10__SVBool_tu12__SVUint32_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4f32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -770,13 +770,13 @@ svfloat32_t test_svld1_gather_u32base_offset_f32(svbool_t pg, svuint32_t bases,
   return SVE_ACLE_FUNC(svld1_gather, _u32base, _offset_f32, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_u64base_offset_f64(
+// CHECK-LABEL: @test_svld1_gather_u64base_offset_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2f64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z36test_svld1_gather_u64base_offset_f64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z36test_svld1_gather_u64base_offset_f64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2f64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -786,13 +786,13 @@ svfloat64_t test_svld1_gather_u64base_offset_f64(svbool_t pg, svuint64_t bases,
   return SVE_ACLE_FUNC(svld1_gather, _u64base, _offset_f64, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_s32index_s32(
+// CHECK-LABEL: @test_svld1_gather_s32index_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z30test_svld1_gather_s32index_s32u10__SVBool_tPKiu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z30test_svld1_gather_s32index_s32u10__SVBool_tPKiu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
@@ -802,13 +802,13 @@ svint32_t test_svld1_gather_s32index_s32(svbool_t pg, const int32_t *base, svint
   return SVE_ACLE_FUNC(svld1_gather_, s32, index, _s32)(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_s64index_s64(
+// CHECK-LABEL: @test_svld1_gather_s64index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.index.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z30test_svld1_gather_s64index_s64u10__SVBool_tPKlu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z30test_svld1_gather_s64index_s64u10__SVBool_tPKlu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.index.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -818,13 +818,13 @@ svint64_t test_svld1_gather_s64index_s64(svbool_t pg, const int64_t *base, svint
   return SVE_ACLE_FUNC(svld1_gather_, s64, index, _s64)(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_s32index_u32(
+// CHECK-LABEL: @test_svld1_gather_s32index_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z30test_svld1_gather_s32index_u32u10__SVBool_tPKju11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z30test_svld1_gather_s32index_u32u10__SVBool_tPKju11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
@@ -834,13 +834,13 @@ svuint32_t test_svld1_gather_s32index_u32(svbool_t pg, const uint32_t *base, svi
   return SVE_ACLE_FUNC(svld1_gather_, s32, index, _u32)(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_s64index_u64(
+// CHECK-LABEL: @test_svld1_gather_s64index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.index.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z30test_svld1_gather_s64index_u64u10__SVBool_tPKmu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z30test_svld1_gather_s64index_u64u10__SVBool_tPKmu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.index.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -850,13 +850,13 @@ svuint64_t test_svld1_gather_s64index_u64(svbool_t pg, const uint64_t *base, svi
   return SVE_ACLE_FUNC(svld1_gather_, s64, index, _u64)(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_s32index_f32(
+// CHECK-LABEL: @test_svld1_gather_s32index_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv4f32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z30test_svld1_gather_s32index_f32u10__SVBool_tPKfu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z30test_svld1_gather_s32index_f32u10__SVBool_tPKfu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv4f32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
@@ -866,13 +866,13 @@ svfloat32_t test_svld1_gather_s32index_f32(svbool_t pg, const float32_t *base, s
   return SVE_ACLE_FUNC(svld1_gather_, s32, index, _f32)(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_s64index_f64(
+// CHECK-LABEL: @test_svld1_gather_s64index_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.index.nxv2f64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z30test_svld1_gather_s64index_f64u10__SVBool_tPKdu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z30test_svld1_gather_s64index_f64u10__SVBool_tPKdu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.index.nxv2f64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -882,13 +882,13 @@ svfloat64_t test_svld1_gather_s64index_f64(svbool_t pg, const float64_t *base, s
   return SVE_ACLE_FUNC(svld1_gather_, s64, index, _f64)(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_u32index_s32(
+// CHECK-LABEL: @test_svld1_gather_u32index_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z30test_svld1_gather_u32index_s32u10__SVBool_tPKiu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z30test_svld1_gather_u32index_s32u10__SVBool_tPKiu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
@@ -898,13 +898,13 @@ svint32_t test_svld1_gather_u32index_s32(svbool_t pg, const int32_t *base, svuin
   return SVE_ACLE_FUNC(svld1_gather_, u32, index, _s32)(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_u64index_s64(
+// CHECK-LABEL: @test_svld1_gather_u64index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.index.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z30test_svld1_gather_u64index_s64u10__SVBool_tPKlu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z30test_svld1_gather_u64index_s64u10__SVBool_tPKlu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.index.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -914,13 +914,13 @@ svint64_t test_svld1_gather_u64index_s64(svbool_t pg, const int64_t *base, svuin
   return SVE_ACLE_FUNC(svld1_gather_, u64, index, _s64)(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_u32index_u32(
+// CHECK-LABEL: @test_svld1_gather_u32index_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z30test_svld1_gather_u32index_u32u10__SVBool_tPKju12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z30test_svld1_gather_u32index_u32u10__SVBool_tPKju12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
@@ -930,13 +930,13 @@ svuint32_t test_svld1_gather_u32index_u32(svbool_t pg, const uint32_t *base, svu
   return SVE_ACLE_FUNC(svld1_gather_, u32, index, _u32)(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_u64index_u64(
+// CHECK-LABEL: @test_svld1_gather_u64index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.index.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z30test_svld1_gather_u64index_u64u10__SVBool_tPKmu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z30test_svld1_gather_u64index_u64u10__SVBool_tPKmu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.index.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -946,13 +946,13 @@ svuint64_t test_svld1_gather_u64index_u64(svbool_t pg, const uint64_t *base, svu
   return SVE_ACLE_FUNC(svld1_gather_, u64, index, _u64)(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_u32index_f32(
+// CHECK-LABEL: @test_svld1_gather_u32index_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv4f32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z30test_svld1_gather_u32index_f32u10__SVBool_tPKfu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z30test_svld1_gather_u32index_f32u10__SVBool_tPKfu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv4f32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
@@ -962,13 +962,13 @@ svfloat32_t test_svld1_gather_u32index_f32(svbool_t pg, const float32_t *base, s
   return SVE_ACLE_FUNC(svld1_gather_, u32, index, _f32)(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_u64index_f64(
+// CHECK-LABEL: @test_svld1_gather_u64index_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.index.nxv2f64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z30test_svld1_gather_u64index_f64u10__SVBool_tPKdu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z30test_svld1_gather_u64index_f64u10__SVBool_tPKdu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.index.nxv2f64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -978,14 +978,14 @@ svfloat64_t test_svld1_gather_u64index_f64(svbool_t pg, const float64_t *base, s
   return SVE_ACLE_FUNC(svld1_gather_, u64, index, _f64)(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_u32base_index_s32(
+// CHECK-LABEL: @test_svld1_gather_u32base_index_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 2
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svld1_gather_u32base_index_s32u10__SVBool_tu12__SVUint32_tl(
+// CPP-CHECK-LABEL: @_Z35test_svld1_gather_u32base_index_s32u10__SVBool_tu12__SVUint32_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 2
@@ -996,14 +996,14 @@ svint32_t test_svld1_gather_u32base_index_s32(svbool_t pg, svuint32_t bases, int
   return SVE_ACLE_FUNC(svld1_gather, _u32base, _index_s32, )(pg, bases, index);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_u64base_index_s64(
+// CHECK-LABEL: @test_svld1_gather_u64base_index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 3
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svld1_gather_u64base_index_s64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z35test_svld1_gather_u64base_index_s64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 3
@@ -1014,14 +1014,14 @@ svint64_t test_svld1_gather_u64base_index_s64(svbool_t pg, svuint64_t bases, int
   return SVE_ACLE_FUNC(svld1_gather, _u64base, _index_s64, )(pg, bases, index);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_u32base_index_u32(
+// CHECK-LABEL: @test_svld1_gather_u32base_index_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 2
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svld1_gather_u32base_index_u32u10__SVBool_tu12__SVUint32_tl(
+// CPP-CHECK-LABEL: @_Z35test_svld1_gather_u32base_index_u32u10__SVBool_tu12__SVUint32_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 2
@@ -1032,14 +1032,14 @@ svuint32_t test_svld1_gather_u32base_index_u32(svbool_t pg, svuint32_t bases, in
   return SVE_ACLE_FUNC(svld1_gather, _u32base, _index_u32, )(pg, bases, index);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_u64base_index_u64(
+// CHECK-LABEL: @test_svld1_gather_u64base_index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 3
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svld1_gather_u64base_index_u64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z35test_svld1_gather_u64base_index_u64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 3
@@ -1050,14 +1050,14 @@ svuint64_t test_svld1_gather_u64base_index_u64(svbool_t pg, svuint64_t bases, in
   return SVE_ACLE_FUNC(svld1_gather, _u64base, _index_u64, )(pg, bases, index);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_u32base_index_f32(
+// CHECK-LABEL: @test_svld1_gather_u32base_index_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 2
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4f32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 4 x float> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svld1_gather_u32base_index_f32u10__SVBool_tu12__SVUint32_tl(
+// CPP-CHECK-LABEL: @_Z35test_svld1_gather_u32base_index_f32u10__SVBool_tu12__SVUint32_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 2
@@ -1068,14 +1068,14 @@ svfloat32_t test_svld1_gather_u32base_index_f32(svbool_t pg, svuint32_t bases, i
   return SVE_ACLE_FUNC(svld1_gather, _u32base, _index_f32, )(pg, bases, index);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1_gather_u64base_index_f64(
+// CHECK-LABEL: @test_svld1_gather_u64base_index_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 3
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2f64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 2 x double> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svld1_gather_u64base_index_f64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z35test_svld1_gather_u64base_index_f64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 3
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1sb.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1sb.c
index 1ffce741da..321628b125 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1sb.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1sb.c
@@ -14,14 +14,14 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sb_s16(
+// CHECK-LABEL: @test_svld1sb_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i8> @llvm.masked.load.nxv8i8.p0(ptr [[BASE:%.*]], i32 1, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i8> zeroinitializer)
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 8 x i8> [[TMP1]] to <vscale x 8 x i16>
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svld1sb_s16u10__SVBool_tPKa(
+// CPP-CHECK-LABEL: @_Z16test_svld1sb_s16u10__SVBool_tPKa(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i8> @llvm.masked.load.nxv8i8.p0(ptr [[BASE:%.*]], i32 1, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i8> zeroinitializer)
@@ -33,14 +33,14 @@ svint16_t test_svld1sb_s16(svbool_t pg, const int8_t *base)
   return svld1sb_s16(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sb_s32(
+// CHECK-LABEL: @test_svld1sb_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.masked.load.nxv4i8.p0(ptr [[BASE:%.*]], i32 1, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i8> zeroinitializer)
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svld1sb_s32u10__SVBool_tPKa(
+// CPP-CHECK-LABEL: @_Z16test_svld1sb_s32u10__SVBool_tPKa(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.masked.load.nxv4i8.p0(ptr [[BASE:%.*]], i32 1, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i8> zeroinitializer)
@@ -52,14 +52,14 @@ svint32_t test_svld1sb_s32(svbool_t pg, const int8_t *base)
   return svld1sb_s32(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sb_s64(
+// CHECK-LABEL: @test_svld1sb_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.masked.load.nxv2i8.p0(ptr [[BASE:%.*]], i32 1, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i8> zeroinitializer)
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svld1sb_s64u10__SVBool_tPKa(
+// CPP-CHECK-LABEL: @_Z16test_svld1sb_s64u10__SVBool_tPKa(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.masked.load.nxv2i8.p0(ptr [[BASE:%.*]], i32 1, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i8> zeroinitializer)
@@ -71,14 +71,14 @@ svint64_t test_svld1sb_s64(svbool_t pg, const int8_t *base)
   return svld1sb_s64(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sb_u16(
+// CHECK-LABEL: @test_svld1sb_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i8> @llvm.masked.load.nxv8i8.p0(ptr [[BASE:%.*]], i32 1, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i8> zeroinitializer)
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 8 x i8> [[TMP1]] to <vscale x 8 x i16>
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svld1sb_u16u10__SVBool_tPKa(
+// CPP-CHECK-LABEL: @_Z16test_svld1sb_u16u10__SVBool_tPKa(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i8> @llvm.masked.load.nxv8i8.p0(ptr [[BASE:%.*]], i32 1, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i8> zeroinitializer)
@@ -90,14 +90,14 @@ svuint16_t test_svld1sb_u16(svbool_t pg, const int8_t *base)
   return svld1sb_u16(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sb_u32(
+// CHECK-LABEL: @test_svld1sb_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.masked.load.nxv4i8.p0(ptr [[BASE:%.*]], i32 1, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i8> zeroinitializer)
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svld1sb_u32u10__SVBool_tPKa(
+// CPP-CHECK-LABEL: @_Z16test_svld1sb_u32u10__SVBool_tPKa(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.masked.load.nxv4i8.p0(ptr [[BASE:%.*]], i32 1, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i8> zeroinitializer)
@@ -109,14 +109,14 @@ svuint32_t test_svld1sb_u32(svbool_t pg, const int8_t *base)
   return svld1sb_u32(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sb_u64(
+// CHECK-LABEL: @test_svld1sb_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.masked.load.nxv2i8.p0(ptr [[BASE:%.*]], i32 1, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i8> zeroinitializer)
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svld1sb_u64u10__SVBool_tPKa(
+// CPP-CHECK-LABEL: @_Z16test_svld1sb_u64u10__SVBool_tPKa(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.masked.load.nxv2i8.p0(ptr [[BASE:%.*]], i32 1, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i8> zeroinitializer)
@@ -128,7 +128,7 @@ svuint64_t test_svld1sb_u64(svbool_t pg, const int8_t *base)
   return svld1sb_u64(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sb_vnum_s16(
+// CHECK-LABEL: @test_svld1sb_vnum_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -136,7 +136,7 @@ svuint64_t test_svld1sb_u64(svbool_t pg, const int8_t *base)
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 8 x i8> [[TMP2]] to <vscale x 8 x i16>
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svld1sb_vnum_s16u10__SVBool_tPKal(
+// CPP-CHECK-LABEL: @_Z21test_svld1sb_vnum_s16u10__SVBool_tPKal(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -149,7 +149,7 @@ svint16_t test_svld1sb_vnum_s16(svbool_t pg, const int8_t *base, int64_t vnum)
   return svld1sb_vnum_s16(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sb_vnum_s32(
+// CHECK-LABEL: @test_svld1sb_vnum_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -157,7 +157,7 @@ svint16_t test_svld1sb_vnum_s16(svbool_t pg, const int8_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 4 x i8> [[TMP2]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svld1sb_vnum_s32u10__SVBool_tPKal(
+// CPP-CHECK-LABEL: @_Z21test_svld1sb_vnum_s32u10__SVBool_tPKal(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -170,7 +170,7 @@ svint32_t test_svld1sb_vnum_s32(svbool_t pg, const int8_t *base, int64_t vnum)
   return svld1sb_vnum_s32(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sb_vnum_s64(
+// CHECK-LABEL: @test_svld1sb_vnum_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -178,7 +178,7 @@ svint32_t test_svld1sb_vnum_s32(svbool_t pg, const int8_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 2 x i8> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svld1sb_vnum_s64u10__SVBool_tPKal(
+// CPP-CHECK-LABEL: @_Z21test_svld1sb_vnum_s64u10__SVBool_tPKal(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -191,7 +191,7 @@ svint64_t test_svld1sb_vnum_s64(svbool_t pg, const int8_t *base, int64_t vnum)
   return svld1sb_vnum_s64(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sb_vnum_u16(
+// CHECK-LABEL: @test_svld1sb_vnum_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -199,7 +199,7 @@ svint64_t test_svld1sb_vnum_s64(svbool_t pg, const int8_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 8 x i8> [[TMP2]] to <vscale x 8 x i16>
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svld1sb_vnum_u16u10__SVBool_tPKal(
+// CPP-CHECK-LABEL: @_Z21test_svld1sb_vnum_u16u10__SVBool_tPKal(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -212,7 +212,7 @@ svuint16_t test_svld1sb_vnum_u16(svbool_t pg, const int8_t *base, int64_t vnum)
   return svld1sb_vnum_u16(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sb_vnum_u32(
+// CHECK-LABEL: @test_svld1sb_vnum_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -220,7 +220,7 @@ svuint16_t test_svld1sb_vnum_u16(svbool_t pg, const int8_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 4 x i8> [[TMP2]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svld1sb_vnum_u32u10__SVBool_tPKal(
+// CPP-CHECK-LABEL: @_Z21test_svld1sb_vnum_u32u10__SVBool_tPKal(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -233,7 +233,7 @@ svuint32_t test_svld1sb_vnum_u32(svbool_t pg, const int8_t *base, int64_t vnum)
   return svld1sb_vnum_u32(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sb_vnum_u64(
+// CHECK-LABEL: @test_svld1sb_vnum_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -241,7 +241,7 @@ svuint32_t test_svld1sb_vnum_u32(svbool_t pg, const int8_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 2 x i8> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svld1sb_vnum_u64u10__SVBool_tPKal(
+// CPP-CHECK-LABEL: @_Z21test_svld1sb_vnum_u64u10__SVBool_tPKal(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -254,14 +254,14 @@ svuint64_t test_svld1sb_vnum_u64(svbool_t pg, const int8_t *base, int64_t vnum)
   return svld1sb_vnum_u64(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sb_gather_u32base_s32(
+// CHECK-LABEL: @test_svld1sb_gather_u32base_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i8.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svld1sb_gather_u32base_s32u10__SVBool_tu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z31test_svld1sb_gather_u32base_s32u10__SVBool_tu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i8.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
@@ -272,14 +272,14 @@ svint32_t test_svld1sb_gather_u32base_s32(svbool_t pg, svuint32_t bases) {
   return SVE_ACLE_FUNC(svld1sb_gather, _u32base, _s32, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sb_gather_u64base_s64(
+// CHECK-LABEL: @test_svld1sb_gather_u64base_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i8.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svld1sb_gather_u64base_s64u10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z31test_svld1sb_gather_u64base_s64u10__SVBool_tu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i8.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
@@ -290,14 +290,14 @@ svint64_t test_svld1sb_gather_u64base_s64(svbool_t pg, svuint64_t bases) {
   return SVE_ACLE_FUNC(svld1sb_gather, _u64base, _s64, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sb_gather_u32base_u32(
+// CHECK-LABEL: @test_svld1sb_gather_u32base_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i8.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svld1sb_gather_u32base_u32u10__SVBool_tu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z31test_svld1sb_gather_u32base_u32u10__SVBool_tu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i8.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
@@ -308,14 +308,14 @@ svuint32_t test_svld1sb_gather_u32base_u32(svbool_t pg, svuint32_t bases) {
   return SVE_ACLE_FUNC(svld1sb_gather, _u32base, _u32, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sb_gather_u64base_u64(
+// CHECK-LABEL: @test_svld1sb_gather_u64base_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i8.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svld1sb_gather_u64base_u64u10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z31test_svld1sb_gather_u64base_u64u10__SVBool_tu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i8.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
@@ -326,14 +326,14 @@ svuint64_t test_svld1sb_gather_u64base_u64(svbool_t pg, svuint64_t bases) {
   return SVE_ACLE_FUNC(svld1sb_gather, _u64base, _u64, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sb_gather_s32offset_s32(
+// CHECK-LABEL: @test_svld1sb_gather_s32offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1sb_gather_s32offset_s32u10__SVBool_tPKau11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1sb_gather_s32offset_s32u10__SVBool_tPKau11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -344,14 +344,14 @@ svint32_t test_svld1sb_gather_s32offset_s32(svbool_t pg, const int8_t *base, svi
   return SVE_ACLE_FUNC(svld1sb_gather_, s32, offset_s32, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sb_gather_s64offset_s64(
+// CHECK-LABEL: @test_svld1sb_gather_s64offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1sb_gather_s64offset_s64u10__SVBool_tPKau11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1sb_gather_s64offset_s64u10__SVBool_tPKau11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -362,14 +362,14 @@ svint64_t test_svld1sb_gather_s64offset_s64(svbool_t pg, const int8_t *base, svi
   return SVE_ACLE_FUNC(svld1sb_gather_, s64, offset_s64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sb_gather_s32offset_u32(
+// CHECK-LABEL: @test_svld1sb_gather_s32offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1sb_gather_s32offset_u32u10__SVBool_tPKau11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1sb_gather_s32offset_u32u10__SVBool_tPKau11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -380,14 +380,14 @@ svuint32_t test_svld1sb_gather_s32offset_u32(svbool_t pg, const int8_t *base, sv
   return SVE_ACLE_FUNC(svld1sb_gather_, s32, offset_u32, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sb_gather_s64offset_u64(
+// CHECK-LABEL: @test_svld1sb_gather_s64offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1sb_gather_s64offset_u64u10__SVBool_tPKau11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1sb_gather_s64offset_u64u10__SVBool_tPKau11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -398,14 +398,14 @@ svuint64_t test_svld1sb_gather_s64offset_u64(svbool_t pg, const int8_t *base, sv
   return SVE_ACLE_FUNC(svld1sb_gather_, s64, offset_u64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sb_gather_u32offset_s32(
+// CHECK-LABEL: @test_svld1sb_gather_u32offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1sb_gather_u32offset_s32u10__SVBool_tPKau12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1sb_gather_u32offset_s32u10__SVBool_tPKau12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -416,14 +416,14 @@ svint32_t test_svld1sb_gather_u32offset_s32(svbool_t pg, const int8_t *base, svu
   return SVE_ACLE_FUNC(svld1sb_gather_, u32, offset_s32, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sb_gather_u64offset_s64(
+// CHECK-LABEL: @test_svld1sb_gather_u64offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1sb_gather_u64offset_s64u10__SVBool_tPKau12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1sb_gather_u64offset_s64u10__SVBool_tPKau12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -434,14 +434,14 @@ svint64_t test_svld1sb_gather_u64offset_s64(svbool_t pg, const int8_t *base, svu
   return SVE_ACLE_FUNC(svld1sb_gather_, u64, offset_s64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sb_gather_u32offset_u32(
+// CHECK-LABEL: @test_svld1sb_gather_u32offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1sb_gather_u32offset_u32u10__SVBool_tPKau12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1sb_gather_u32offset_u32u10__SVBool_tPKau12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -452,14 +452,14 @@ svuint32_t test_svld1sb_gather_u32offset_u32(svbool_t pg, const int8_t *base, sv
   return SVE_ACLE_FUNC(svld1sb_gather_, u32, offset_u32, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sb_gather_u64offset_u64(
+// CHECK-LABEL: @test_svld1sb_gather_u64offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1sb_gather_u64offset_u64u10__SVBool_tPKau12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1sb_gather_u64offset_u64u10__SVBool_tPKau12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -470,14 +470,14 @@ svuint64_t test_svld1sb_gather_u64offset_u64(svbool_t pg, const int8_t *base, sv
   return SVE_ACLE_FUNC(svld1sb_gather_, u64, offset_u64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sb_gather_u32base_offset_s32(
+// CHECK-LABEL: @test_svld1sb_gather_u32base_offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i8.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z38test_svld1sb_gather_u32base_offset_s32u10__SVBool_tu12__SVUint32_tl(
+// CPP-CHECK-LABEL: @_Z38test_svld1sb_gather_u32base_offset_s32u10__SVBool_tu12__SVUint32_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i8.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -488,14 +488,14 @@ svint32_t test_svld1sb_gather_u32base_offset_s32(svbool_t pg, svuint32_t bases,
   return SVE_ACLE_FUNC(svld1sb_gather, _u32base, _offset_s32, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sb_gather_u64base_offset_s64(
+// CHECK-LABEL: @test_svld1sb_gather_u64base_offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i8.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z38test_svld1sb_gather_u64base_offset_s64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z38test_svld1sb_gather_u64base_offset_s64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i8.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -506,14 +506,14 @@ svint64_t test_svld1sb_gather_u64base_offset_s64(svbool_t pg, svuint64_t bases,
   return SVE_ACLE_FUNC(svld1sb_gather, _u64base, _offset_s64, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sb_gather_u32base_offset_u32(
+// CHECK-LABEL: @test_svld1sb_gather_u32base_offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i8.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z38test_svld1sb_gather_u32base_offset_u32u10__SVBool_tu12__SVUint32_tl(
+// CPP-CHECK-LABEL: @_Z38test_svld1sb_gather_u32base_offset_u32u10__SVBool_tu12__SVUint32_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i8.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -524,14 +524,14 @@ svuint32_t test_svld1sb_gather_u32base_offset_u32(svbool_t pg, svuint32_t bases,
   return SVE_ACLE_FUNC(svld1sb_gather, _u32base, _offset_u32, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sb_gather_u64base_offset_u64(
+// CHECK-LABEL: @test_svld1sb_gather_u64base_offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i8.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z38test_svld1sb_gather_u64base_offset_u64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z38test_svld1sb_gather_u64base_offset_u64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i8.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1sh.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1sh.c
index 1b89ecf04c..a72892fe96 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1sh.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1sh.c
@@ -14,14 +14,14 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sh_s32(
+// CHECK-LABEL: @test_svld1sh_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr [[BASE:%.*]], i32 1, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i16> zeroinitializer)
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svld1sh_s32u10__SVBool_tPKs(
+// CPP-CHECK-LABEL: @_Z16test_svld1sh_s32u10__SVBool_tPKs(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr [[BASE:%.*]], i32 1, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i16> zeroinitializer)
@@ -33,14 +33,14 @@ svint32_t test_svld1sh_s32(svbool_t pg, const int16_t *base)
   return svld1sh_s32(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sh_s64(
+// CHECK-LABEL: @test_svld1sh_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.masked.load.nxv2i16.p0(ptr [[BASE:%.*]], i32 1, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i16> zeroinitializer)
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svld1sh_s64u10__SVBool_tPKs(
+// CPP-CHECK-LABEL: @_Z16test_svld1sh_s64u10__SVBool_tPKs(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.masked.load.nxv2i16.p0(ptr [[BASE:%.*]], i32 1, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i16> zeroinitializer)
@@ -52,14 +52,14 @@ svint64_t test_svld1sh_s64(svbool_t pg, const int16_t *base)
   return svld1sh_s64(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sh_u32(
+// CHECK-LABEL: @test_svld1sh_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr [[BASE:%.*]], i32 1, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i16> zeroinitializer)
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svld1sh_u32u10__SVBool_tPKs(
+// CPP-CHECK-LABEL: @_Z16test_svld1sh_u32u10__SVBool_tPKs(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr [[BASE:%.*]], i32 1, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i16> zeroinitializer)
@@ -71,14 +71,14 @@ svuint32_t test_svld1sh_u32(svbool_t pg, const int16_t *base)
   return svld1sh_u32(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sh_u64(
+// CHECK-LABEL: @test_svld1sh_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.masked.load.nxv2i16.p0(ptr [[BASE:%.*]], i32 1, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i16> zeroinitializer)
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svld1sh_u64u10__SVBool_tPKs(
+// CPP-CHECK-LABEL: @_Z16test_svld1sh_u64u10__SVBool_tPKs(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.masked.load.nxv2i16.p0(ptr [[BASE:%.*]], i32 1, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i16> zeroinitializer)
@@ -90,7 +90,7 @@ svuint64_t test_svld1sh_u64(svbool_t pg, const int16_t *base)
   return svld1sh_u64(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sh_vnum_s32(
+// CHECK-LABEL: @test_svld1sh_vnum_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -98,7 +98,7 @@ svuint64_t test_svld1sh_u64(svbool_t pg, const int16_t *base)
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 4 x i16> [[TMP2]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svld1sh_vnum_s32u10__SVBool_tPKsl(
+// CPP-CHECK-LABEL: @_Z21test_svld1sh_vnum_s32u10__SVBool_tPKsl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -111,7 +111,7 @@ svint32_t test_svld1sh_vnum_s32(svbool_t pg, const int16_t *base, int64_t vnum)
   return svld1sh_vnum_s32(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sh_vnum_s64(
+// CHECK-LABEL: @test_svld1sh_vnum_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -119,7 +119,7 @@ svint32_t test_svld1sh_vnum_s32(svbool_t pg, const int16_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 2 x i16> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svld1sh_vnum_s64u10__SVBool_tPKsl(
+// CPP-CHECK-LABEL: @_Z21test_svld1sh_vnum_s64u10__SVBool_tPKsl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -132,7 +132,7 @@ svint64_t test_svld1sh_vnum_s64(svbool_t pg, const int16_t *base, int64_t vnum)
   return svld1sh_vnum_s64(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sh_vnum_u32(
+// CHECK-LABEL: @test_svld1sh_vnum_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -140,7 +140,7 @@ svint64_t test_svld1sh_vnum_s64(svbool_t pg, const int16_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 4 x i16> [[TMP2]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svld1sh_vnum_u32u10__SVBool_tPKsl(
+// CPP-CHECK-LABEL: @_Z21test_svld1sh_vnum_u32u10__SVBool_tPKsl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -153,7 +153,7 @@ svuint32_t test_svld1sh_vnum_u32(svbool_t pg, const int16_t *base, int64_t vnum)
   return svld1sh_vnum_u32(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sh_vnum_u64(
+// CHECK-LABEL: @test_svld1sh_vnum_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -161,7 +161,7 @@ svuint32_t test_svld1sh_vnum_u32(svbool_t pg, const int16_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 2 x i16> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svld1sh_vnum_u64u10__SVBool_tPKsl(
+// CPP-CHECK-LABEL: @_Z21test_svld1sh_vnum_u64u10__SVBool_tPKsl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -174,14 +174,14 @@ svuint64_t test_svld1sh_vnum_u64(svbool_t pg, const int16_t *base, int64_t vnum)
   return svld1sh_vnum_u64(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sh_gather_u32base_s32(
+// CHECK-LABEL: @test_svld1sh_gather_u32base_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i16.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svld1sh_gather_u32base_s32u10__SVBool_tu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z31test_svld1sh_gather_u32base_s32u10__SVBool_tu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i16.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
@@ -192,14 +192,14 @@ svint32_t test_svld1sh_gather_u32base_s32(svbool_t pg, svuint32_t bases) {
   return SVE_ACLE_FUNC(svld1sh_gather, _u32base, _s32, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sh_gather_u64base_s64(
+// CHECK-LABEL: @test_svld1sh_gather_u64base_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i16.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svld1sh_gather_u64base_s64u10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z31test_svld1sh_gather_u64base_s64u10__SVBool_tu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i16.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
@@ -210,14 +210,14 @@ svint64_t test_svld1sh_gather_u64base_s64(svbool_t pg, svuint64_t bases) {
   return SVE_ACLE_FUNC(svld1sh_gather, _u64base, _s64, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sh_gather_u32base_u32(
+// CHECK-LABEL: @test_svld1sh_gather_u32base_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i16.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svld1sh_gather_u32base_u32u10__SVBool_tu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z31test_svld1sh_gather_u32base_u32u10__SVBool_tu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i16.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
@@ -228,14 +228,14 @@ svuint32_t test_svld1sh_gather_u32base_u32(svbool_t pg, svuint32_t bases) {
   return SVE_ACLE_FUNC(svld1sh_gather, _u32base, _u32, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sh_gather_u64base_u64(
+// CHECK-LABEL: @test_svld1sh_gather_u64base_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i16.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svld1sh_gather_u64base_u64u10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z31test_svld1sh_gather_u64base_u64u10__SVBool_tu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i16.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
@@ -246,14 +246,14 @@ svuint64_t test_svld1sh_gather_u64base_u64(svbool_t pg, svuint64_t bases) {
   return SVE_ACLE_FUNC(svld1sh_gather, _u64base, _u64, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sh_gather_s32offset_s32(
+// CHECK-LABEL: @test_svld1sh_gather_s32offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1sh_gather_s32offset_s32u10__SVBool_tPKsu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1sh_gather_s32offset_s32u10__SVBool_tPKsu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -264,14 +264,14 @@ svint32_t test_svld1sh_gather_s32offset_s32(svbool_t pg, const int16_t *base, sv
   return SVE_ACLE_FUNC(svld1sh_gather_, s32, offset_s32, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sh_gather_s64offset_s64(
+// CHECK-LABEL: @test_svld1sh_gather_s64offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1sh_gather_s64offset_s64u10__SVBool_tPKsu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1sh_gather_s64offset_s64u10__SVBool_tPKsu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -282,14 +282,14 @@ svint64_t test_svld1sh_gather_s64offset_s64(svbool_t pg, const int16_t *base, sv
   return SVE_ACLE_FUNC(svld1sh_gather_, s64, offset_s64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sh_gather_s32offset_u32(
+// CHECK-LABEL: @test_svld1sh_gather_s32offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1sh_gather_s32offset_u32u10__SVBool_tPKsu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1sh_gather_s32offset_u32u10__SVBool_tPKsu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -300,14 +300,14 @@ svuint32_t test_svld1sh_gather_s32offset_u32(svbool_t pg, const int16_t *base, s
   return SVE_ACLE_FUNC(svld1sh_gather_, s32, offset_u32, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sh_gather_s64offset_u64(
+// CHECK-LABEL: @test_svld1sh_gather_s64offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1sh_gather_s64offset_u64u10__SVBool_tPKsu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1sh_gather_s64offset_u64u10__SVBool_tPKsu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -318,14 +318,14 @@ svuint64_t test_svld1sh_gather_s64offset_u64(svbool_t pg, const int16_t *base, s
   return SVE_ACLE_FUNC(svld1sh_gather_, s64, offset_u64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sh_gather_u32offset_s32(
+// CHECK-LABEL: @test_svld1sh_gather_u32offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1sh_gather_u32offset_s32u10__SVBool_tPKsu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1sh_gather_u32offset_s32u10__SVBool_tPKsu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -336,14 +336,14 @@ svint32_t test_svld1sh_gather_u32offset_s32(svbool_t pg, const int16_t *base, sv
   return SVE_ACLE_FUNC(svld1sh_gather_, u32, offset_s32, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sh_gather_u64offset_s64(
+// CHECK-LABEL: @test_svld1sh_gather_u64offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1sh_gather_u64offset_s64u10__SVBool_tPKsu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1sh_gather_u64offset_s64u10__SVBool_tPKsu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -354,14 +354,14 @@ svint64_t test_svld1sh_gather_u64offset_s64(svbool_t pg, const int16_t *base, sv
   return SVE_ACLE_FUNC(svld1sh_gather_, u64, offset_s64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sh_gather_u32offset_u32(
+// CHECK-LABEL: @test_svld1sh_gather_u32offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1sh_gather_u32offset_u32u10__SVBool_tPKsu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1sh_gather_u32offset_u32u10__SVBool_tPKsu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -372,14 +372,14 @@ svuint32_t test_svld1sh_gather_u32offset_u32(svbool_t pg, const int16_t *base, s
   return SVE_ACLE_FUNC(svld1sh_gather_, u32, offset_u32, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sh_gather_u64offset_u64(
+// CHECK-LABEL: @test_svld1sh_gather_u64offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1sh_gather_u64offset_u64u10__SVBool_tPKsu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1sh_gather_u64offset_u64u10__SVBool_tPKsu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -390,14 +390,14 @@ svuint64_t test_svld1sh_gather_u64offset_u64(svbool_t pg, const int16_t *base, s
   return SVE_ACLE_FUNC(svld1sh_gather_, u64, offset_u64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sh_gather_u32base_offset_s32(
+// CHECK-LABEL: @test_svld1sh_gather_u32base_offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i16.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z38test_svld1sh_gather_u32base_offset_s32u10__SVBool_tu12__SVUint32_tl(
+// CPP-CHECK-LABEL: @_Z38test_svld1sh_gather_u32base_offset_s32u10__SVBool_tu12__SVUint32_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i16.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -408,14 +408,14 @@ svint32_t test_svld1sh_gather_u32base_offset_s32(svbool_t pg, svuint32_t bases,
   return SVE_ACLE_FUNC(svld1sh_gather, _u32base, _offset_s32, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sh_gather_u64base_offset_s64(
+// CHECK-LABEL: @test_svld1sh_gather_u64base_offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i16.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z38test_svld1sh_gather_u64base_offset_s64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z38test_svld1sh_gather_u64base_offset_s64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i16.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -426,14 +426,14 @@ svint64_t test_svld1sh_gather_u64base_offset_s64(svbool_t pg, svuint64_t bases,
   return SVE_ACLE_FUNC(svld1sh_gather, _u64base, _offset_s64, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sh_gather_u32base_offset_u32(
+// CHECK-LABEL: @test_svld1sh_gather_u32base_offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i16.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z38test_svld1sh_gather_u32base_offset_u32u10__SVBool_tu12__SVUint32_tl(
+// CPP-CHECK-LABEL: @_Z38test_svld1sh_gather_u32base_offset_u32u10__SVBool_tu12__SVUint32_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i16.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -444,14 +444,14 @@ svuint32_t test_svld1sh_gather_u32base_offset_u32(svbool_t pg, svuint32_t bases,
   return SVE_ACLE_FUNC(svld1sh_gather, _u32base, _offset_u32, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sh_gather_u64base_offset_u64(
+// CHECK-LABEL: @test_svld1sh_gather_u64base_offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i16.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z38test_svld1sh_gather_u64base_offset_u64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z38test_svld1sh_gather_u64base_offset_u64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i16.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -462,14 +462,14 @@ svuint64_t test_svld1sh_gather_u64base_offset_u64(svbool_t pg, svuint64_t bases,
   return SVE_ACLE_FUNC(svld1sh_gather, _u64base, _offset_u64, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sh_gather_s32index_s32(
+// CHECK-LABEL: @test_svld1sh_gather_s32index_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svld1sh_gather_s32index_s32u10__SVBool_tPKsu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z32test_svld1sh_gather_s32index_s32u10__SVBool_tPKsu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
@@ -480,14 +480,14 @@ svint32_t test_svld1sh_gather_s32index_s32(svbool_t pg, const int16_t *base, svi
   return SVE_ACLE_FUNC(svld1sh_gather_, s32, index_s32, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sh_gather_s64index_s64(
+// CHECK-LABEL: @test_svld1sh_gather_s64index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.index.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svld1sh_gather_s64index_s64u10__SVBool_tPKsu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z32test_svld1sh_gather_s64index_s64u10__SVBool_tPKsu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.index.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -498,14 +498,14 @@ svint64_t test_svld1sh_gather_s64index_s64(svbool_t pg, const int16_t *base, svi
   return SVE_ACLE_FUNC(svld1sh_gather_, s64, index_s64, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sh_gather_s32index_u32(
+// CHECK-LABEL: @test_svld1sh_gather_s32index_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svld1sh_gather_s32index_u32u10__SVBool_tPKsu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z32test_svld1sh_gather_s32index_u32u10__SVBool_tPKsu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
@@ -516,14 +516,14 @@ svuint32_t test_svld1sh_gather_s32index_u32(svbool_t pg, const int16_t *base, sv
   return SVE_ACLE_FUNC(svld1sh_gather_, s32, index_u32, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sh_gather_s64index_u64(
+// CHECK-LABEL: @test_svld1sh_gather_s64index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.index.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svld1sh_gather_s64index_u64u10__SVBool_tPKsu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z32test_svld1sh_gather_s64index_u64u10__SVBool_tPKsu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.index.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -534,14 +534,14 @@ svuint64_t test_svld1sh_gather_s64index_u64(svbool_t pg, const int16_t *base, sv
   return SVE_ACLE_FUNC(svld1sh_gather_, s64, index_u64, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sh_gather_u32index_s32(
+// CHECK-LABEL: @test_svld1sh_gather_u32index_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svld1sh_gather_u32index_s32u10__SVBool_tPKsu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z32test_svld1sh_gather_u32index_s32u10__SVBool_tPKsu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
@@ -552,14 +552,14 @@ svint32_t test_svld1sh_gather_u32index_s32(svbool_t pg, const int16_t *base, svu
   return SVE_ACLE_FUNC(svld1sh_gather_, u32, index_s32, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sh_gather_u64index_s64(
+// CHECK-LABEL: @test_svld1sh_gather_u64index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.index.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svld1sh_gather_u64index_s64u10__SVBool_tPKsu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z32test_svld1sh_gather_u64index_s64u10__SVBool_tPKsu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.index.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -570,14 +570,14 @@ svint64_t test_svld1sh_gather_u64index_s64(svbool_t pg, const int16_t *base, svu
   return SVE_ACLE_FUNC(svld1sh_gather_, u64, index_s64, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sh_gather_u32index_u32(
+// CHECK-LABEL: @test_svld1sh_gather_u32index_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svld1sh_gather_u32index_u32u10__SVBool_tPKsu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z32test_svld1sh_gather_u32index_u32u10__SVBool_tPKsu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
@@ -588,14 +588,14 @@ svuint32_t test_svld1sh_gather_u32index_u32(svbool_t pg, const int16_t *base, sv
   return SVE_ACLE_FUNC(svld1sh_gather_, u32, index_u32, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sh_gather_u64index_u64(
+// CHECK-LABEL: @test_svld1sh_gather_u64index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.index.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svld1sh_gather_u64index_u64u10__SVBool_tPKsu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z32test_svld1sh_gather_u64index_u64u10__SVBool_tPKsu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.index.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -606,7 +606,7 @@ svuint64_t test_svld1sh_gather_u64index_u64(svbool_t pg, const int16_t *base, sv
   return SVE_ACLE_FUNC(svld1sh_gather_, u64, index_u64, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sh_gather_u32base_index_s32(
+// CHECK-LABEL: @test_svld1sh_gather_u32base_index_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 1
@@ -614,7 +614,7 @@ svuint64_t test_svld1sh_gather_u64index_u64(svbool_t pg, const int16_t *base, sv
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 4 x i16> [[TMP2]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z37test_svld1sh_gather_u32base_index_s32u10__SVBool_tu12__SVUint32_tl(
+// CPP-CHECK-LABEL: @_Z37test_svld1sh_gather_u32base_index_s32u10__SVBool_tu12__SVUint32_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 1
@@ -626,7 +626,7 @@ svint32_t test_svld1sh_gather_u32base_index_s32(svbool_t pg, svuint32_t bases, i
   return SVE_ACLE_FUNC(svld1sh_gather, _u32base, _index_s32, )(pg, bases, index);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sh_gather_u64base_index_s64(
+// CHECK-LABEL: @test_svld1sh_gather_u64base_index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 1
@@ -634,7 +634,7 @@ svint32_t test_svld1sh_gather_u32base_index_s32(svbool_t pg, svuint32_t bases, i
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 2 x i16> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z37test_svld1sh_gather_u64base_index_s64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z37test_svld1sh_gather_u64base_index_s64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 1
@@ -646,7 +646,7 @@ svint64_t test_svld1sh_gather_u64base_index_s64(svbool_t pg, svuint64_t bases, i
   return SVE_ACLE_FUNC(svld1sh_gather, _u64base, _index_s64, )(pg, bases, index);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sh_gather_u32base_index_u32(
+// CHECK-LABEL: @test_svld1sh_gather_u32base_index_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 1
@@ -654,7 +654,7 @@ svint64_t test_svld1sh_gather_u64base_index_s64(svbool_t pg, svuint64_t bases, i
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 4 x i16> [[TMP2]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z37test_svld1sh_gather_u32base_index_u32u10__SVBool_tu12__SVUint32_tl(
+// CPP-CHECK-LABEL: @_Z37test_svld1sh_gather_u32base_index_u32u10__SVBool_tu12__SVUint32_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 1
@@ -666,7 +666,7 @@ svuint32_t test_svld1sh_gather_u32base_index_u32(svbool_t pg, svuint32_t bases,
   return SVE_ACLE_FUNC(svld1sh_gather, _u32base, _index_u32, )(pg, bases, index);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sh_gather_u64base_index_u64(
+// CHECK-LABEL: @test_svld1sh_gather_u64base_index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 1
@@ -674,7 +674,7 @@ svuint32_t test_svld1sh_gather_u32base_index_u32(svbool_t pg, svuint32_t bases,
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 2 x i16> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z37test_svld1sh_gather_u64base_index_u64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z37test_svld1sh_gather_u64base_index_u64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 1
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1sw.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1sw.c
index bd5aa89225..8921c50fca 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1sw.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1sw.c
@@ -14,14 +14,14 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sw_s64(
+// CHECK-LABEL: @test_svld1sw_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.masked.load.nxv2i32.p0(ptr [[BASE:%.*]], i32 1, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i32> zeroinitializer)
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svld1sw_s64u10__SVBool_tPKi(
+// CPP-CHECK-LABEL: @_Z16test_svld1sw_s64u10__SVBool_tPKi(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.masked.load.nxv2i32.p0(ptr [[BASE:%.*]], i32 1, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i32> zeroinitializer)
@@ -33,14 +33,14 @@ svint64_t test_svld1sw_s64(svbool_t pg, const int32_t *base)
   return svld1sw_s64(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sw_u64(
+// CHECK-LABEL: @test_svld1sw_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.masked.load.nxv2i32.p0(ptr [[BASE:%.*]], i32 1, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i32> zeroinitializer)
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svld1sw_u64u10__SVBool_tPKi(
+// CPP-CHECK-LABEL: @_Z16test_svld1sw_u64u10__SVBool_tPKi(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.masked.load.nxv2i32.p0(ptr [[BASE:%.*]], i32 1, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i32> zeroinitializer)
@@ -52,7 +52,7 @@ svuint64_t test_svld1sw_u64(svbool_t pg, const int32_t *base)
   return svld1sw_u64(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sw_vnum_s64(
+// CHECK-LABEL: @test_svld1sw_vnum_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -60,7 +60,7 @@ svuint64_t test_svld1sw_u64(svbool_t pg, const int32_t *base)
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 2 x i32> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svld1sw_vnum_s64u10__SVBool_tPKil(
+// CPP-CHECK-LABEL: @_Z21test_svld1sw_vnum_s64u10__SVBool_tPKil(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -73,7 +73,7 @@ svint64_t test_svld1sw_vnum_s64(svbool_t pg, const int32_t *base, int64_t vnum)
   return svld1sw_vnum_s64(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sw_vnum_u64(
+// CHECK-LABEL: @test_svld1sw_vnum_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -81,7 +81,7 @@ svint64_t test_svld1sw_vnum_s64(svbool_t pg, const int32_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 2 x i32> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svld1sw_vnum_u64u10__SVBool_tPKil(
+// CPP-CHECK-LABEL: @_Z21test_svld1sw_vnum_u64u10__SVBool_tPKil(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -94,14 +94,14 @@ svuint64_t test_svld1sw_vnum_u64(svbool_t pg, const int32_t *base, int64_t vnum)
   return svld1sw_vnum_u64(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sw_gather_u64base_s64(
+// CHECK-LABEL: @test_svld1sw_gather_u64base_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i32.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svld1sw_gather_u64base_s64u10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z31test_svld1sw_gather_u64base_s64u10__SVBool_tu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i32.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
@@ -112,14 +112,14 @@ svint64_t test_svld1sw_gather_u64base_s64(svbool_t pg, svuint64_t bases) {
   return SVE_ACLE_FUNC(svld1sw_gather, _u64base, _s64, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sw_gather_u64base_u64(
+// CHECK-LABEL: @test_svld1sw_gather_u64base_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i32.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svld1sw_gather_u64base_u64u10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z31test_svld1sw_gather_u64base_u64u10__SVBool_tu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i32.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
@@ -130,14 +130,14 @@ svuint64_t test_svld1sw_gather_u64base_u64(svbool_t pg, svuint64_t bases) {
   return SVE_ACLE_FUNC(svld1sw_gather, _u64base, _u64, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sw_gather_s64offset_s64(
+// CHECK-LABEL: @test_svld1sw_gather_s64offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1sw_gather_s64offset_s64u10__SVBool_tPKiu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1sw_gather_s64offset_s64u10__SVBool_tPKiu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -148,14 +148,14 @@ svint64_t test_svld1sw_gather_s64offset_s64(svbool_t pg, const int32_t *base, sv
   return SVE_ACLE_FUNC(svld1sw_gather_, s64, offset_s64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sw_gather_s64offset_u64(
+// CHECK-LABEL: @test_svld1sw_gather_s64offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1sw_gather_s64offset_u64u10__SVBool_tPKiu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1sw_gather_s64offset_u64u10__SVBool_tPKiu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -166,14 +166,14 @@ svuint64_t test_svld1sw_gather_s64offset_u64(svbool_t pg, const int32_t *base, s
   return SVE_ACLE_FUNC(svld1sw_gather_, s64, offset_u64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sw_gather_u64offset_s64(
+// CHECK-LABEL: @test_svld1sw_gather_u64offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1sw_gather_u64offset_s64u10__SVBool_tPKiu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1sw_gather_u64offset_s64u10__SVBool_tPKiu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -184,14 +184,14 @@ svint64_t test_svld1sw_gather_u64offset_s64(svbool_t pg, const int32_t *base, sv
   return SVE_ACLE_FUNC(svld1sw_gather_, u64, offset_s64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sw_gather_u64offset_u64(
+// CHECK-LABEL: @test_svld1sw_gather_u64offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1sw_gather_u64offset_u64u10__SVBool_tPKiu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1sw_gather_u64offset_u64u10__SVBool_tPKiu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -202,14 +202,14 @@ svuint64_t test_svld1sw_gather_u64offset_u64(svbool_t pg, const int32_t *base, s
   return SVE_ACLE_FUNC(svld1sw_gather_, u64, offset_u64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sw_gather_u64base_offset_s64(
+// CHECK-LABEL: @test_svld1sw_gather_u64base_offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i32.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z38test_svld1sw_gather_u64base_offset_s64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z38test_svld1sw_gather_u64base_offset_s64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i32.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -220,14 +220,14 @@ svint64_t test_svld1sw_gather_u64base_offset_s64(svbool_t pg, svuint64_t bases,
   return SVE_ACLE_FUNC(svld1sw_gather, _u64base, _offset_s64, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sw_gather_u64base_offset_u64(
+// CHECK-LABEL: @test_svld1sw_gather_u64base_offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i32.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z38test_svld1sw_gather_u64base_offset_u64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z38test_svld1sw_gather_u64base_offset_u64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i32.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -238,14 +238,14 @@ svuint64_t test_svld1sw_gather_u64base_offset_u64(svbool_t pg, svuint64_t bases,
   return SVE_ACLE_FUNC(svld1sw_gather, _u64base, _offset_u64, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sw_gather_s64index_s64(
+// CHECK-LABEL: @test_svld1sw_gather_s64index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.index.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svld1sw_gather_s64index_s64u10__SVBool_tPKiu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z32test_svld1sw_gather_s64index_s64u10__SVBool_tPKiu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.index.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -256,14 +256,14 @@ svint64_t test_svld1sw_gather_s64index_s64(svbool_t pg, const int32_t *base, svi
   return SVE_ACLE_FUNC(svld1sw_gather_, s64, index_s64, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sw_gather_s64index_u64(
+// CHECK-LABEL: @test_svld1sw_gather_s64index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.index.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svld1sw_gather_s64index_u64u10__SVBool_tPKiu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z32test_svld1sw_gather_s64index_u64u10__SVBool_tPKiu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.index.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -274,14 +274,14 @@ svuint64_t test_svld1sw_gather_s64index_u64(svbool_t pg, const int32_t *base, sv
   return SVE_ACLE_FUNC(svld1sw_gather_, s64, index_u64, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sw_gather_u64index_s64(
+// CHECK-LABEL: @test_svld1sw_gather_u64index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.index.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svld1sw_gather_u64index_s64u10__SVBool_tPKiu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z32test_svld1sw_gather_u64index_s64u10__SVBool_tPKiu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.index.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -292,14 +292,14 @@ svint64_t test_svld1sw_gather_u64index_s64(svbool_t pg, const int32_t *base, svu
   return SVE_ACLE_FUNC(svld1sw_gather_, u64, index_s64, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sw_gather_u64index_u64(
+// CHECK-LABEL: @test_svld1sw_gather_u64index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.index.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svld1sw_gather_u64index_u64u10__SVBool_tPKiu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z32test_svld1sw_gather_u64index_u64u10__SVBool_tPKiu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.index.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -310,7 +310,7 @@ svuint64_t test_svld1sw_gather_u64index_u64(svbool_t pg, const int32_t *base, sv
   return SVE_ACLE_FUNC(svld1sw_gather_, u64, index_u64, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sw_gather_u64base_index_s64(
+// CHECK-LABEL: @test_svld1sw_gather_u64base_index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 2
@@ -318,7 +318,7 @@ svuint64_t test_svld1sw_gather_u64index_u64(svbool_t pg, const int32_t *base, sv
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 2 x i32> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z37test_svld1sw_gather_u64base_index_s64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z37test_svld1sw_gather_u64base_index_s64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 2
@@ -330,7 +330,7 @@ svint64_t test_svld1sw_gather_u64base_index_s64(svbool_t pg, svuint64_t bases, i
   return SVE_ACLE_FUNC(svld1sw_gather, _u64base, _index_s64, )(pg, bases, index);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1sw_gather_u64base_index_u64(
+// CHECK-LABEL: @test_svld1sw_gather_u64base_index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 2
@@ -338,7 +338,7 @@ svint64_t test_svld1sw_gather_u64base_index_s64(svbool_t pg, svuint64_t bases, i
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 2 x i32> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z37test_svld1sw_gather_u64base_index_u64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z37test_svld1sw_gather_u64base_index_u64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 2
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1ub.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1ub.c
index 03e4a5961f..9adb2f36f4 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1ub.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1ub.c
@@ -14,14 +14,14 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1ub_s16(
+// CHECK-LABEL: @test_svld1ub_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i8> @llvm.masked.load.nxv8i8.p0(ptr [[BASE:%.*]], i32 1, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i8> zeroinitializer)
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 8 x i8> [[TMP1]] to <vscale x 8 x i16>
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svld1ub_s16u10__SVBool_tPKh(
+// CPP-CHECK-LABEL: @_Z16test_svld1ub_s16u10__SVBool_tPKh(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i8> @llvm.masked.load.nxv8i8.p0(ptr [[BASE:%.*]], i32 1, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i8> zeroinitializer)
@@ -33,14 +33,14 @@ svint16_t test_svld1ub_s16(svbool_t pg, const uint8_t *base)
   return svld1ub_s16(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1ub_s32(
+// CHECK-LABEL: @test_svld1ub_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.masked.load.nxv4i8.p0(ptr [[BASE:%.*]], i32 1, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i8> zeroinitializer)
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svld1ub_s32u10__SVBool_tPKh(
+// CPP-CHECK-LABEL: @_Z16test_svld1ub_s32u10__SVBool_tPKh(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.masked.load.nxv4i8.p0(ptr [[BASE:%.*]], i32 1, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i8> zeroinitializer)
@@ -52,14 +52,14 @@ svint32_t test_svld1ub_s32(svbool_t pg, const uint8_t *base)
   return svld1ub_s32(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1ub_s64(
+// CHECK-LABEL: @test_svld1ub_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.masked.load.nxv2i8.p0(ptr [[BASE:%.*]], i32 1, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i8> zeroinitializer)
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svld1ub_s64u10__SVBool_tPKh(
+// CPP-CHECK-LABEL: @_Z16test_svld1ub_s64u10__SVBool_tPKh(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.masked.load.nxv2i8.p0(ptr [[BASE:%.*]], i32 1, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i8> zeroinitializer)
@@ -71,14 +71,14 @@ svint64_t test_svld1ub_s64(svbool_t pg, const uint8_t *base)
   return svld1ub_s64(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1ub_u16(
+// CHECK-LABEL: @test_svld1ub_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i8> @llvm.masked.load.nxv8i8.p0(ptr [[BASE:%.*]], i32 1, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i8> zeroinitializer)
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 8 x i8> [[TMP1]] to <vscale x 8 x i16>
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svld1ub_u16u10__SVBool_tPKh(
+// CPP-CHECK-LABEL: @_Z16test_svld1ub_u16u10__SVBool_tPKh(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i8> @llvm.masked.load.nxv8i8.p0(ptr [[BASE:%.*]], i32 1, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i8> zeroinitializer)
@@ -90,14 +90,14 @@ svuint16_t test_svld1ub_u16(svbool_t pg, const uint8_t *base)
   return svld1ub_u16(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1ub_u32(
+// CHECK-LABEL: @test_svld1ub_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.masked.load.nxv4i8.p0(ptr [[BASE:%.*]], i32 1, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i8> zeroinitializer)
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svld1ub_u32u10__SVBool_tPKh(
+// CPP-CHECK-LABEL: @_Z16test_svld1ub_u32u10__SVBool_tPKh(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.masked.load.nxv4i8.p0(ptr [[BASE:%.*]], i32 1, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i8> zeroinitializer)
@@ -109,14 +109,14 @@ svuint32_t test_svld1ub_u32(svbool_t pg, const uint8_t *base)
   return svld1ub_u32(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1ub_u64(
+// CHECK-LABEL: @test_svld1ub_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.masked.load.nxv2i8.p0(ptr [[BASE:%.*]], i32 1, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i8> zeroinitializer)
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svld1ub_u64u10__SVBool_tPKh(
+// CPP-CHECK-LABEL: @_Z16test_svld1ub_u64u10__SVBool_tPKh(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.masked.load.nxv2i8.p0(ptr [[BASE:%.*]], i32 1, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i8> zeroinitializer)
@@ -128,7 +128,7 @@ svuint64_t test_svld1ub_u64(svbool_t pg, const uint8_t *base)
   return svld1ub_u64(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1ub_vnum_s16(
+// CHECK-LABEL: @test_svld1ub_vnum_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -136,7 +136,7 @@ svuint64_t test_svld1ub_u64(svbool_t pg, const uint8_t *base)
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 8 x i8> [[TMP2]] to <vscale x 8 x i16>
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svld1ub_vnum_s16u10__SVBool_tPKhl(
+// CPP-CHECK-LABEL: @_Z21test_svld1ub_vnum_s16u10__SVBool_tPKhl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -149,7 +149,7 @@ svint16_t test_svld1ub_vnum_s16(svbool_t pg, const uint8_t *base, int64_t vnum)
   return svld1ub_vnum_s16(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1ub_vnum_s32(
+// CHECK-LABEL: @test_svld1ub_vnum_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -157,7 +157,7 @@ svint16_t test_svld1ub_vnum_s16(svbool_t pg, const uint8_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 4 x i8> [[TMP2]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svld1ub_vnum_s32u10__SVBool_tPKhl(
+// CPP-CHECK-LABEL: @_Z21test_svld1ub_vnum_s32u10__SVBool_tPKhl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -170,7 +170,7 @@ svint32_t test_svld1ub_vnum_s32(svbool_t pg, const uint8_t *base, int64_t vnum)
   return svld1ub_vnum_s32(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1ub_vnum_s64(
+// CHECK-LABEL: @test_svld1ub_vnum_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -178,7 +178,7 @@ svint32_t test_svld1ub_vnum_s32(svbool_t pg, const uint8_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 2 x i8> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svld1ub_vnum_s64u10__SVBool_tPKhl(
+// CPP-CHECK-LABEL: @_Z21test_svld1ub_vnum_s64u10__SVBool_tPKhl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -191,7 +191,7 @@ svint64_t test_svld1ub_vnum_s64(svbool_t pg, const uint8_t *base, int64_t vnum)
   return svld1ub_vnum_s64(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1ub_vnum_u16(
+// CHECK-LABEL: @test_svld1ub_vnum_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -199,7 +199,7 @@ svint64_t test_svld1ub_vnum_s64(svbool_t pg, const uint8_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 8 x i8> [[TMP2]] to <vscale x 8 x i16>
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svld1ub_vnum_u16u10__SVBool_tPKhl(
+// CPP-CHECK-LABEL: @_Z21test_svld1ub_vnum_u16u10__SVBool_tPKhl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -212,7 +212,7 @@ svuint16_t test_svld1ub_vnum_u16(svbool_t pg, const uint8_t *base, int64_t vnum)
   return svld1ub_vnum_u16(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1ub_vnum_u32(
+// CHECK-LABEL: @test_svld1ub_vnum_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -220,7 +220,7 @@ svuint16_t test_svld1ub_vnum_u16(svbool_t pg, const uint8_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 4 x i8> [[TMP2]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svld1ub_vnum_u32u10__SVBool_tPKhl(
+// CPP-CHECK-LABEL: @_Z21test_svld1ub_vnum_u32u10__SVBool_tPKhl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -233,7 +233,7 @@ svuint32_t test_svld1ub_vnum_u32(svbool_t pg, const uint8_t *base, int64_t vnum)
   return svld1ub_vnum_u32(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1ub_vnum_u64(
+// CHECK-LABEL: @test_svld1ub_vnum_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -241,7 +241,7 @@ svuint32_t test_svld1ub_vnum_u32(svbool_t pg, const uint8_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 2 x i8> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svld1ub_vnum_u64u10__SVBool_tPKhl(
+// CPP-CHECK-LABEL: @_Z21test_svld1ub_vnum_u64u10__SVBool_tPKhl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -254,14 +254,14 @@ svuint64_t test_svld1ub_vnum_u64(svbool_t pg, const uint8_t *base, int64_t vnum)
   return svld1ub_vnum_u64(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1ub_gather_u32base_s32(
+// CHECK-LABEL: @test_svld1ub_gather_u32base_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i8.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svld1ub_gather_u32base_s32u10__SVBool_tu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z31test_svld1ub_gather_u32base_s32u10__SVBool_tu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i8.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
@@ -272,14 +272,14 @@ svint32_t test_svld1ub_gather_u32base_s32(svbool_t pg, svuint32_t bases) {
   return SVE_ACLE_FUNC(svld1ub_gather, _u32base, _s32, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1ub_gather_u64base_s64(
+// CHECK-LABEL: @test_svld1ub_gather_u64base_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i8.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svld1ub_gather_u64base_s64u10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z31test_svld1ub_gather_u64base_s64u10__SVBool_tu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i8.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
@@ -290,14 +290,14 @@ svint64_t test_svld1ub_gather_u64base_s64(svbool_t pg, svuint64_t bases) {
   return SVE_ACLE_FUNC(svld1ub_gather, _u64base, _s64, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1ub_gather_u32base_u32(
+// CHECK-LABEL: @test_svld1ub_gather_u32base_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i8.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svld1ub_gather_u32base_u32u10__SVBool_tu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z31test_svld1ub_gather_u32base_u32u10__SVBool_tu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i8.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
@@ -308,14 +308,14 @@ svuint32_t test_svld1ub_gather_u32base_u32(svbool_t pg, svuint32_t bases) {
   return SVE_ACLE_FUNC(svld1ub_gather, _u32base, _u32, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1ub_gather_u64base_u64(
+// CHECK-LABEL: @test_svld1ub_gather_u64base_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i8.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svld1ub_gather_u64base_u64u10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z31test_svld1ub_gather_u64base_u64u10__SVBool_tu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i8.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
@@ -326,14 +326,14 @@ svuint64_t test_svld1ub_gather_u64base_u64(svbool_t pg, svuint64_t bases) {
   return SVE_ACLE_FUNC(svld1ub_gather, _u64base, _u64, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1ub_gather_s32offset_s32(
+// CHECK-LABEL: @test_svld1ub_gather_s32offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1ub_gather_s32offset_s32u10__SVBool_tPKhu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1ub_gather_s32offset_s32u10__SVBool_tPKhu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -344,14 +344,14 @@ svint32_t test_svld1ub_gather_s32offset_s32(svbool_t pg, const uint8_t *base, sv
   return SVE_ACLE_FUNC(svld1ub_gather_, s32, offset_s32, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1ub_gather_s64offset_s64(
+// CHECK-LABEL: @test_svld1ub_gather_s64offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1ub_gather_s64offset_s64u10__SVBool_tPKhu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1ub_gather_s64offset_s64u10__SVBool_tPKhu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -362,14 +362,14 @@ svint64_t test_svld1ub_gather_s64offset_s64(svbool_t pg, const uint8_t *base, sv
   return SVE_ACLE_FUNC(svld1ub_gather_, s64, offset_s64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1ub_gather_s32offset_u32(
+// CHECK-LABEL: @test_svld1ub_gather_s32offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1ub_gather_s32offset_u32u10__SVBool_tPKhu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1ub_gather_s32offset_u32u10__SVBool_tPKhu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -380,14 +380,14 @@ svuint32_t test_svld1ub_gather_s32offset_u32(svbool_t pg, const uint8_t *base, s
   return SVE_ACLE_FUNC(svld1ub_gather_, s32, offset_u32, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1ub_gather_s64offset_u64(
+// CHECK-LABEL: @test_svld1ub_gather_s64offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1ub_gather_s64offset_u64u10__SVBool_tPKhu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1ub_gather_s64offset_u64u10__SVBool_tPKhu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -398,14 +398,14 @@ svuint64_t test_svld1ub_gather_s64offset_u64(svbool_t pg, const uint8_t *base, s
   return SVE_ACLE_FUNC(svld1ub_gather_, s64, offset_u64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1ub_gather_u32offset_s32(
+// CHECK-LABEL: @test_svld1ub_gather_u32offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1ub_gather_u32offset_s32u10__SVBool_tPKhu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1ub_gather_u32offset_s32u10__SVBool_tPKhu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -416,14 +416,14 @@ svint32_t test_svld1ub_gather_u32offset_s32(svbool_t pg, const uint8_t *base, sv
   return SVE_ACLE_FUNC(svld1ub_gather_, u32, offset_s32, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1ub_gather_u64offset_s64(
+// CHECK-LABEL: @test_svld1ub_gather_u64offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1ub_gather_u64offset_s64u10__SVBool_tPKhu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1ub_gather_u64offset_s64u10__SVBool_tPKhu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -434,14 +434,14 @@ svint64_t test_svld1ub_gather_u64offset_s64(svbool_t pg, const uint8_t *base, sv
   return SVE_ACLE_FUNC(svld1ub_gather_, u64, offset_s64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1ub_gather_u32offset_u32(
+// CHECK-LABEL: @test_svld1ub_gather_u32offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1ub_gather_u32offset_u32u10__SVBool_tPKhu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1ub_gather_u32offset_u32u10__SVBool_tPKhu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -452,14 +452,14 @@ svuint32_t test_svld1ub_gather_u32offset_u32(svbool_t pg, const uint8_t *base, s
   return SVE_ACLE_FUNC(svld1ub_gather_, u32, offset_u32, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1ub_gather_u64offset_u64(
+// CHECK-LABEL: @test_svld1ub_gather_u64offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1ub_gather_u64offset_u64u10__SVBool_tPKhu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1ub_gather_u64offset_u64u10__SVBool_tPKhu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -470,14 +470,14 @@ svuint64_t test_svld1ub_gather_u64offset_u64(svbool_t pg, const uint8_t *base, s
   return SVE_ACLE_FUNC(svld1ub_gather_, u64, offset_u64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1ub_gather_u32base_offset_s32(
+// CHECK-LABEL: @test_svld1ub_gather_u32base_offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i8.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z38test_svld1ub_gather_u32base_offset_s32u10__SVBool_tu12__SVUint32_tl(
+// CPP-CHECK-LABEL: @_Z38test_svld1ub_gather_u32base_offset_s32u10__SVBool_tu12__SVUint32_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i8.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -488,14 +488,14 @@ svint32_t test_svld1ub_gather_u32base_offset_s32(svbool_t pg, svuint32_t bases,
   return SVE_ACLE_FUNC(svld1ub_gather, _u32base, _offset_s32, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1ub_gather_u64base_offset_s64(
+// CHECK-LABEL: @test_svld1ub_gather_u64base_offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i8.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z38test_svld1ub_gather_u64base_offset_s64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z38test_svld1ub_gather_u64base_offset_s64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i8.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -506,14 +506,14 @@ svint64_t test_svld1ub_gather_u64base_offset_s64(svbool_t pg, svuint64_t bases,
   return SVE_ACLE_FUNC(svld1ub_gather, _u64base, _offset_s64, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1ub_gather_u32base_offset_u32(
+// CHECK-LABEL: @test_svld1ub_gather_u32base_offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i8.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z38test_svld1ub_gather_u32base_offset_u32u10__SVBool_tu12__SVUint32_tl(
+// CPP-CHECK-LABEL: @_Z38test_svld1ub_gather_u32base_offset_u32u10__SVBool_tu12__SVUint32_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i8.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -524,14 +524,14 @@ svuint32_t test_svld1ub_gather_u32base_offset_u32(svbool_t pg, svuint32_t bases,
   return SVE_ACLE_FUNC(svld1ub_gather, _u32base, _offset_u32, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1ub_gather_u64base_offset_u64(
+// CHECK-LABEL: @test_svld1ub_gather_u64base_offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i8.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z38test_svld1ub_gather_u64base_offset_u64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z38test_svld1ub_gather_u64base_offset_u64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i8.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1uh.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1uh.c
index d22c255bbf..de2b975f3d 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1uh.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1uh.c
@@ -14,14 +14,14 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uh_s32(
+// CHECK-LABEL: @test_svld1uh_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr [[BASE:%.*]], i32 1, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i16> zeroinitializer)
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svld1uh_s32u10__SVBool_tPKt(
+// CPP-CHECK-LABEL: @_Z16test_svld1uh_s32u10__SVBool_tPKt(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr [[BASE:%.*]], i32 1, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i16> zeroinitializer)
@@ -33,14 +33,14 @@ svint32_t test_svld1uh_s32(svbool_t pg, const uint16_t *base)
   return svld1uh_s32(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uh_s64(
+// CHECK-LABEL: @test_svld1uh_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.masked.load.nxv2i16.p0(ptr [[BASE:%.*]], i32 1, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i16> zeroinitializer)
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svld1uh_s64u10__SVBool_tPKt(
+// CPP-CHECK-LABEL: @_Z16test_svld1uh_s64u10__SVBool_tPKt(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.masked.load.nxv2i16.p0(ptr [[BASE:%.*]], i32 1, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i16> zeroinitializer)
@@ -52,14 +52,14 @@ svint64_t test_svld1uh_s64(svbool_t pg, const uint16_t *base)
   return svld1uh_s64(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uh_u32(
+// CHECK-LABEL: @test_svld1uh_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr [[BASE:%.*]], i32 1, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i16> zeroinitializer)
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svld1uh_u32u10__SVBool_tPKt(
+// CPP-CHECK-LABEL: @_Z16test_svld1uh_u32u10__SVBool_tPKt(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr [[BASE:%.*]], i32 1, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i16> zeroinitializer)
@@ -71,14 +71,14 @@ svuint32_t test_svld1uh_u32(svbool_t pg, const uint16_t *base)
   return svld1uh_u32(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uh_u64(
+// CHECK-LABEL: @test_svld1uh_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.masked.load.nxv2i16.p0(ptr [[BASE:%.*]], i32 1, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i16> zeroinitializer)
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svld1uh_u64u10__SVBool_tPKt(
+// CPP-CHECK-LABEL: @_Z16test_svld1uh_u64u10__SVBool_tPKt(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.masked.load.nxv2i16.p0(ptr [[BASE:%.*]], i32 1, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i16> zeroinitializer)
@@ -90,7 +90,7 @@ svuint64_t test_svld1uh_u64(svbool_t pg, const uint16_t *base)
   return svld1uh_u64(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uh_vnum_s32(
+// CHECK-LABEL: @test_svld1uh_vnum_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -98,7 +98,7 @@ svuint64_t test_svld1uh_u64(svbool_t pg, const uint16_t *base)
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 4 x i16> [[TMP2]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svld1uh_vnum_s32u10__SVBool_tPKtl(
+// CPP-CHECK-LABEL: @_Z21test_svld1uh_vnum_s32u10__SVBool_tPKtl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -111,7 +111,7 @@ svint32_t test_svld1uh_vnum_s32(svbool_t pg, const uint16_t *base, int64_t vnum)
   return svld1uh_vnum_s32(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uh_vnum_s64(
+// CHECK-LABEL: @test_svld1uh_vnum_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -119,7 +119,7 @@ svint32_t test_svld1uh_vnum_s32(svbool_t pg, const uint16_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 2 x i16> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svld1uh_vnum_s64u10__SVBool_tPKtl(
+// CPP-CHECK-LABEL: @_Z21test_svld1uh_vnum_s64u10__SVBool_tPKtl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -132,7 +132,7 @@ svint64_t test_svld1uh_vnum_s64(svbool_t pg, const uint16_t *base, int64_t vnum)
   return svld1uh_vnum_s64(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uh_vnum_u32(
+// CHECK-LABEL: @test_svld1uh_vnum_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -140,7 +140,7 @@ svint64_t test_svld1uh_vnum_s64(svbool_t pg, const uint16_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 4 x i16> [[TMP2]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svld1uh_vnum_u32u10__SVBool_tPKtl(
+// CPP-CHECK-LABEL: @_Z21test_svld1uh_vnum_u32u10__SVBool_tPKtl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -153,7 +153,7 @@ svuint32_t test_svld1uh_vnum_u32(svbool_t pg, const uint16_t *base, int64_t vnum
   return svld1uh_vnum_u32(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uh_vnum_u64(
+// CHECK-LABEL: @test_svld1uh_vnum_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -161,7 +161,7 @@ svuint32_t test_svld1uh_vnum_u32(svbool_t pg, const uint16_t *base, int64_t vnum
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 2 x i16> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svld1uh_vnum_u64u10__SVBool_tPKtl(
+// CPP-CHECK-LABEL: @_Z21test_svld1uh_vnum_u64u10__SVBool_tPKtl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -174,14 +174,14 @@ svuint64_t test_svld1uh_vnum_u64(svbool_t pg, const uint16_t *base, int64_t vnum
   return svld1uh_vnum_u64(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uh_gather_u32base_s32(
+// CHECK-LABEL: @test_svld1uh_gather_u32base_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i16.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svld1uh_gather_u32base_s32u10__SVBool_tu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z31test_svld1uh_gather_u32base_s32u10__SVBool_tu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i16.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
@@ -192,14 +192,14 @@ svint32_t test_svld1uh_gather_u32base_s32(svbool_t pg, svuint32_t bases) {
   return SVE_ACLE_FUNC(svld1uh_gather, _u32base, _s32, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uh_gather_u64base_s64(
+// CHECK-LABEL: @test_svld1uh_gather_u64base_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i16.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svld1uh_gather_u64base_s64u10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z31test_svld1uh_gather_u64base_s64u10__SVBool_tu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i16.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
@@ -210,14 +210,14 @@ svint64_t test_svld1uh_gather_u64base_s64(svbool_t pg, svuint64_t bases) {
   return SVE_ACLE_FUNC(svld1uh_gather, _u64base, _s64, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uh_gather_u32base_u32(
+// CHECK-LABEL: @test_svld1uh_gather_u32base_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i16.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svld1uh_gather_u32base_u32u10__SVBool_tu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z31test_svld1uh_gather_u32base_u32u10__SVBool_tu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i16.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
@@ -228,14 +228,14 @@ svuint32_t test_svld1uh_gather_u32base_u32(svbool_t pg, svuint32_t bases) {
   return SVE_ACLE_FUNC(svld1uh_gather, _u32base, _u32, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uh_gather_u64base_u64(
+// CHECK-LABEL: @test_svld1uh_gather_u64base_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i16.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svld1uh_gather_u64base_u64u10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z31test_svld1uh_gather_u64base_u64u10__SVBool_tu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i16.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
@@ -246,14 +246,14 @@ svuint64_t test_svld1uh_gather_u64base_u64(svbool_t pg, svuint64_t bases) {
   return SVE_ACLE_FUNC(svld1uh_gather, _u64base, _u64, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uh_gather_s32offset_s32(
+// CHECK-LABEL: @test_svld1uh_gather_s32offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1uh_gather_s32offset_s32u10__SVBool_tPKtu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1uh_gather_s32offset_s32u10__SVBool_tPKtu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -264,14 +264,14 @@ svint32_t test_svld1uh_gather_s32offset_s32(svbool_t pg, const uint16_t *base, s
   return SVE_ACLE_FUNC(svld1uh_gather_, s32, offset_s32, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uh_gather_s64offset_s64(
+// CHECK-LABEL: @test_svld1uh_gather_s64offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1uh_gather_s64offset_s64u10__SVBool_tPKtu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1uh_gather_s64offset_s64u10__SVBool_tPKtu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -282,14 +282,14 @@ svint64_t test_svld1uh_gather_s64offset_s64(svbool_t pg, const uint16_t *base, s
   return SVE_ACLE_FUNC(svld1uh_gather_, s64, offset_s64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uh_gather_s32offset_u32(
+// CHECK-LABEL: @test_svld1uh_gather_s32offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1uh_gather_s32offset_u32u10__SVBool_tPKtu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1uh_gather_s32offset_u32u10__SVBool_tPKtu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -300,14 +300,14 @@ svuint32_t test_svld1uh_gather_s32offset_u32(svbool_t pg, const uint16_t *base,
   return SVE_ACLE_FUNC(svld1uh_gather_, s32, offset_u32, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uh_gather_s64offset_u64(
+// CHECK-LABEL: @test_svld1uh_gather_s64offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1uh_gather_s64offset_u64u10__SVBool_tPKtu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1uh_gather_s64offset_u64u10__SVBool_tPKtu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -318,14 +318,14 @@ svuint64_t test_svld1uh_gather_s64offset_u64(svbool_t pg, const uint16_t *base,
   return SVE_ACLE_FUNC(svld1uh_gather_, s64, offset_u64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uh_gather_u32offset_s32(
+// CHECK-LABEL: @test_svld1uh_gather_u32offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1uh_gather_u32offset_s32u10__SVBool_tPKtu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1uh_gather_u32offset_s32u10__SVBool_tPKtu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -336,14 +336,14 @@ svint32_t test_svld1uh_gather_u32offset_s32(svbool_t pg, const uint16_t *base, s
   return SVE_ACLE_FUNC(svld1uh_gather_, u32, offset_s32, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uh_gather_u64offset_s64(
+// CHECK-LABEL: @test_svld1uh_gather_u64offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1uh_gather_u64offset_s64u10__SVBool_tPKtu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1uh_gather_u64offset_s64u10__SVBool_tPKtu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -354,14 +354,14 @@ svint64_t test_svld1uh_gather_u64offset_s64(svbool_t pg, const uint16_t *base, s
   return SVE_ACLE_FUNC(svld1uh_gather_, u64, offset_s64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uh_gather_u32offset_u32(
+// CHECK-LABEL: @test_svld1uh_gather_u32offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1uh_gather_u32offset_u32u10__SVBool_tPKtu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1uh_gather_u32offset_u32u10__SVBool_tPKtu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -372,14 +372,14 @@ svuint32_t test_svld1uh_gather_u32offset_u32(svbool_t pg, const uint16_t *base,
   return SVE_ACLE_FUNC(svld1uh_gather_, u32, offset_u32, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uh_gather_u64offset_u64(
+// CHECK-LABEL: @test_svld1uh_gather_u64offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1uh_gather_u64offset_u64u10__SVBool_tPKtu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1uh_gather_u64offset_u64u10__SVBool_tPKtu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -390,14 +390,14 @@ svuint64_t test_svld1uh_gather_u64offset_u64(svbool_t pg, const uint16_t *base,
   return SVE_ACLE_FUNC(svld1uh_gather_, u64, offset_u64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uh_gather_u32base_offset_s32(
+// CHECK-LABEL: @test_svld1uh_gather_u32base_offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i16.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z38test_svld1uh_gather_u32base_offset_s32u10__SVBool_tu12__SVUint32_tl(
+// CPP-CHECK-LABEL: @_Z38test_svld1uh_gather_u32base_offset_s32u10__SVBool_tu12__SVUint32_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i16.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -408,14 +408,14 @@ svint32_t test_svld1uh_gather_u32base_offset_s32(svbool_t pg, svuint32_t bases,
   return SVE_ACLE_FUNC(svld1uh_gather, _u32base, _offset_s32, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uh_gather_u64base_offset_s64(
+// CHECK-LABEL: @test_svld1uh_gather_u64base_offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i16.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z38test_svld1uh_gather_u64base_offset_s64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z38test_svld1uh_gather_u64base_offset_s64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i16.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -426,14 +426,14 @@ svint64_t test_svld1uh_gather_u64base_offset_s64(svbool_t pg, svuint64_t bases,
   return SVE_ACLE_FUNC(svld1uh_gather, _u64base, _offset_s64, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uh_gather_u32base_offset_u32(
+// CHECK-LABEL: @test_svld1uh_gather_u32base_offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i16.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z38test_svld1uh_gather_u32base_offset_u32u10__SVBool_tu12__SVUint32_tl(
+// CPP-CHECK-LABEL: @_Z38test_svld1uh_gather_u32base_offset_u32u10__SVBool_tu12__SVUint32_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv4i16.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -444,14 +444,14 @@ svuint32_t test_svld1uh_gather_u32base_offset_u32(svbool_t pg, svuint32_t bases,
   return SVE_ACLE_FUNC(svld1uh_gather, _u32base, _offset_u32, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uh_gather_u64base_offset_u64(
+// CHECK-LABEL: @test_svld1uh_gather_u64base_offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i16.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z38test_svld1uh_gather_u64base_offset_u64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z38test_svld1uh_gather_u64base_offset_u64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i16.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -462,14 +462,14 @@ svuint64_t test_svld1uh_gather_u64base_offset_u64(svbool_t pg, svuint64_t bases,
   return SVE_ACLE_FUNC(svld1uh_gather, _u64base, _offset_u64, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uh_gather_s32index_s32(
+// CHECK-LABEL: @test_svld1uh_gather_s32index_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svld1uh_gather_s32index_s32u10__SVBool_tPKtu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z32test_svld1uh_gather_s32index_s32u10__SVBool_tPKtu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
@@ -480,14 +480,14 @@ svint32_t test_svld1uh_gather_s32index_s32(svbool_t pg, const uint16_t *base, sv
   return SVE_ACLE_FUNC(svld1uh_gather_, s32, index_s32, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uh_gather_s64index_s64(
+// CHECK-LABEL: @test_svld1uh_gather_s64index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.index.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svld1uh_gather_s64index_s64u10__SVBool_tPKtu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z32test_svld1uh_gather_s64index_s64u10__SVBool_tPKtu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.index.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -498,14 +498,14 @@ svint64_t test_svld1uh_gather_s64index_s64(svbool_t pg, const uint16_t *base, sv
   return SVE_ACLE_FUNC(svld1uh_gather_, s64, index_s64, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uh_gather_s32index_u32(
+// CHECK-LABEL: @test_svld1uh_gather_s32index_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svld1uh_gather_s32index_u32u10__SVBool_tPKtu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z32test_svld1uh_gather_s32index_u32u10__SVBool_tPKtu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.sxtw.index.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
@@ -516,14 +516,14 @@ svuint32_t test_svld1uh_gather_s32index_u32(svbool_t pg, const uint16_t *base, s
   return SVE_ACLE_FUNC(svld1uh_gather_, s32, index_u32, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uh_gather_s64index_u64(
+// CHECK-LABEL: @test_svld1uh_gather_s64index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.index.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svld1uh_gather_s64index_u64u10__SVBool_tPKtu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z32test_svld1uh_gather_s64index_u64u10__SVBool_tPKtu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.index.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -534,14 +534,14 @@ svuint64_t test_svld1uh_gather_s64index_u64(svbool_t pg, const uint16_t *base, s
   return SVE_ACLE_FUNC(svld1uh_gather_, s64, index_u64, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uh_gather_u32index_s32(
+// CHECK-LABEL: @test_svld1uh_gather_u32index_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svld1uh_gather_u32index_s32u10__SVBool_tPKtu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z32test_svld1uh_gather_u32index_s32u10__SVBool_tPKtu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
@@ -552,14 +552,14 @@ svint32_t test_svld1uh_gather_u32index_s32(svbool_t pg, const uint16_t *base, sv
   return SVE_ACLE_FUNC(svld1uh_gather_, u32, index_s32, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uh_gather_u64index_s64(
+// CHECK-LABEL: @test_svld1uh_gather_u64index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.index.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svld1uh_gather_u64index_s64u10__SVBool_tPKtu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z32test_svld1uh_gather_u64index_s64u10__SVBool_tPKtu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.index.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -570,14 +570,14 @@ svint64_t test_svld1uh_gather_u64index_s64(svbool_t pg, const uint16_t *base, sv
   return SVE_ACLE_FUNC(svld1uh_gather_, u64, index_s64, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uh_gather_u32index_u32(
+// CHECK-LABEL: @test_svld1uh_gather_u32index_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svld1uh_gather_u32index_u32u10__SVBool_tPKtu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z32test_svld1uh_gather_u32index_u32u10__SVBool_tPKtu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ld1.gather.uxtw.index.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
@@ -588,14 +588,14 @@ svuint32_t test_svld1uh_gather_u32index_u32(svbool_t pg, const uint16_t *base, s
   return SVE_ACLE_FUNC(svld1uh_gather_, u32, index_u32, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uh_gather_u64index_u64(
+// CHECK-LABEL: @test_svld1uh_gather_u64index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.index.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svld1uh_gather_u64index_u64u10__SVBool_tPKtu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z32test_svld1uh_gather_u64index_u64u10__SVBool_tPKtu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ld1.gather.index.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -606,7 +606,7 @@ svuint64_t test_svld1uh_gather_u64index_u64(svbool_t pg, const uint16_t *base, s
   return SVE_ACLE_FUNC(svld1uh_gather_, u64, index_u64, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uh_gather_u32base_index_s32(
+// CHECK-LABEL: @test_svld1uh_gather_u32base_index_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 1
@@ -614,7 +614,7 @@ svuint64_t test_svld1uh_gather_u64index_u64(svbool_t pg, const uint16_t *base, s
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 4 x i16> [[TMP2]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z37test_svld1uh_gather_u32base_index_s32u10__SVBool_tu12__SVUint32_tl(
+// CPP-CHECK-LABEL: @_Z37test_svld1uh_gather_u32base_index_s32u10__SVBool_tu12__SVUint32_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 1
@@ -626,7 +626,7 @@ svint32_t test_svld1uh_gather_u32base_index_s32(svbool_t pg, svuint32_t bases, i
   return SVE_ACLE_FUNC(svld1uh_gather, _u32base, _index_s32, )(pg, bases, index);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uh_gather_u64base_index_s64(
+// CHECK-LABEL: @test_svld1uh_gather_u64base_index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 1
@@ -634,7 +634,7 @@ svint32_t test_svld1uh_gather_u32base_index_s32(svbool_t pg, svuint32_t bases, i
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 2 x i16> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z37test_svld1uh_gather_u64base_index_s64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z37test_svld1uh_gather_u64base_index_s64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 1
@@ -646,7 +646,7 @@ svint64_t test_svld1uh_gather_u64base_index_s64(svbool_t pg, svuint64_t bases, i
   return SVE_ACLE_FUNC(svld1uh_gather, _u64base, _index_s64, )(pg, bases, index);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uh_gather_u32base_index_u32(
+// CHECK-LABEL: @test_svld1uh_gather_u32base_index_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 1
@@ -654,7 +654,7 @@ svint64_t test_svld1uh_gather_u64base_index_s64(svbool_t pg, svuint64_t bases, i
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 4 x i16> [[TMP2]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z37test_svld1uh_gather_u32base_index_u32u10__SVBool_tu12__SVUint32_tl(
+// CPP-CHECK-LABEL: @_Z37test_svld1uh_gather_u32base_index_u32u10__SVBool_tu12__SVUint32_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 1
@@ -666,7 +666,7 @@ svuint32_t test_svld1uh_gather_u32base_index_u32(svbool_t pg, svuint32_t bases,
   return SVE_ACLE_FUNC(svld1uh_gather, _u32base, _index_u32, )(pg, bases, index);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uh_gather_u64base_index_u64(
+// CHECK-LABEL: @test_svld1uh_gather_u64base_index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 1
@@ -674,7 +674,7 @@ svuint32_t test_svld1uh_gather_u32base_index_u32(svbool_t pg, svuint32_t bases,
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 2 x i16> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z37test_svld1uh_gather_u64base_index_u64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z37test_svld1uh_gather_u64base_index_u64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 1
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1uw.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1uw.c
index a2df984e42..bb1d3b5675 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1uw.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld1uw.c
@@ -14,14 +14,14 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uw_s64(
+// CHECK-LABEL: @test_svld1uw_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.masked.load.nxv2i32.p0(ptr [[BASE:%.*]], i32 1, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i32> zeroinitializer)
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svld1uw_s64u10__SVBool_tPKj(
+// CPP-CHECK-LABEL: @_Z16test_svld1uw_s64u10__SVBool_tPKj(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.masked.load.nxv2i32.p0(ptr [[BASE:%.*]], i32 1, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i32> zeroinitializer)
@@ -33,14 +33,14 @@ svint64_t test_svld1uw_s64(svbool_t pg, const uint32_t *base)
   return svld1uw_s64(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uw_u64(
+// CHECK-LABEL: @test_svld1uw_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.masked.load.nxv2i32.p0(ptr [[BASE:%.*]], i32 1, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i32> zeroinitializer)
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svld1uw_u64u10__SVBool_tPKj(
+// CPP-CHECK-LABEL: @_Z16test_svld1uw_u64u10__SVBool_tPKj(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.masked.load.nxv2i32.p0(ptr [[BASE:%.*]], i32 1, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i32> zeroinitializer)
@@ -52,7 +52,7 @@ svuint64_t test_svld1uw_u64(svbool_t pg, const uint32_t *base)
   return svld1uw_u64(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uw_vnum_s64(
+// CHECK-LABEL: @test_svld1uw_vnum_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -60,7 +60,7 @@ svuint64_t test_svld1uw_u64(svbool_t pg, const uint32_t *base)
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 2 x i32> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svld1uw_vnum_s64u10__SVBool_tPKjl(
+// CPP-CHECK-LABEL: @_Z21test_svld1uw_vnum_s64u10__SVBool_tPKjl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -73,7 +73,7 @@ svint64_t test_svld1uw_vnum_s64(svbool_t pg, const uint32_t *base, int64_t vnum)
   return svld1uw_vnum_s64(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uw_vnum_u64(
+// CHECK-LABEL: @test_svld1uw_vnum_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -81,7 +81,7 @@ svint64_t test_svld1uw_vnum_s64(svbool_t pg, const uint32_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 2 x i32> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svld1uw_vnum_u64u10__SVBool_tPKjl(
+// CPP-CHECK-LABEL: @_Z21test_svld1uw_vnum_u64u10__SVBool_tPKjl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -94,14 +94,14 @@ svuint64_t test_svld1uw_vnum_u64(svbool_t pg, const uint32_t *base, int64_t vnum
   return svld1uw_vnum_u64(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uw_gather_u64base_s64(
+// CHECK-LABEL: @test_svld1uw_gather_u64base_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i32.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svld1uw_gather_u64base_s64u10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z31test_svld1uw_gather_u64base_s64u10__SVBool_tu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i32.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
@@ -112,14 +112,14 @@ svint64_t test_svld1uw_gather_u64base_s64(svbool_t pg, svuint64_t bases) {
   return SVE_ACLE_FUNC(svld1uw_gather, _u64base, _s64, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uw_gather_u64base_u64(
+// CHECK-LABEL: @test_svld1uw_gather_u64base_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i32.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svld1uw_gather_u64base_u64u10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z31test_svld1uw_gather_u64base_u64u10__SVBool_tu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i32.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
@@ -130,14 +130,14 @@ svuint64_t test_svld1uw_gather_u64base_u64(svbool_t pg, svuint64_t bases) {
   return SVE_ACLE_FUNC(svld1uw_gather, _u64base, _u64, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uw_gather_s64offset_s64(
+// CHECK-LABEL: @test_svld1uw_gather_s64offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1uw_gather_s64offset_s64u10__SVBool_tPKju11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1uw_gather_s64offset_s64u10__SVBool_tPKju11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -148,14 +148,14 @@ svint64_t test_svld1uw_gather_s64offset_s64(svbool_t pg, const uint32_t *base, s
   return SVE_ACLE_FUNC(svld1uw_gather_, s64, offset_s64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uw_gather_s64offset_u64(
+// CHECK-LABEL: @test_svld1uw_gather_s64offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1uw_gather_s64offset_u64u10__SVBool_tPKju11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1uw_gather_s64offset_u64u10__SVBool_tPKju11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -166,14 +166,14 @@ svuint64_t test_svld1uw_gather_s64offset_u64(svbool_t pg, const uint32_t *base,
   return SVE_ACLE_FUNC(svld1uw_gather_, s64, offset_u64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uw_gather_u64offset_s64(
+// CHECK-LABEL: @test_svld1uw_gather_u64offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1uw_gather_u64offset_s64u10__SVBool_tPKju12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1uw_gather_u64offset_s64u10__SVBool_tPKju12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -184,14 +184,14 @@ svint64_t test_svld1uw_gather_u64offset_s64(svbool_t pg, const uint32_t *base, s
   return SVE_ACLE_FUNC(svld1uw_gather_, u64, offset_s64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uw_gather_u64offset_u64(
+// CHECK-LABEL: @test_svld1uw_gather_u64offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svld1uw_gather_u64offset_u64u10__SVBool_tPKju12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z33test_svld1uw_gather_u64offset_u64u10__SVBool_tPKju12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -202,14 +202,14 @@ svuint64_t test_svld1uw_gather_u64offset_u64(svbool_t pg, const uint32_t *base,
   return SVE_ACLE_FUNC(svld1uw_gather_, u64, offset_u64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uw_gather_u64base_offset_s64(
+// CHECK-LABEL: @test_svld1uw_gather_u64base_offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i32.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z38test_svld1uw_gather_u64base_offset_s64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z38test_svld1uw_gather_u64base_offset_s64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i32.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -220,14 +220,14 @@ svint64_t test_svld1uw_gather_u64base_offset_s64(svbool_t pg, svuint64_t bases,
   return SVE_ACLE_FUNC(svld1uw_gather, _u64base, _offset_s64, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uw_gather_u64base_offset_u64(
+// CHECK-LABEL: @test_svld1uw_gather_u64base_offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i32.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z38test_svld1uw_gather_u64base_offset_u64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z38test_svld1uw_gather_u64base_offset_u64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.scalar.offset.nxv2i32.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -238,14 +238,14 @@ svuint64_t test_svld1uw_gather_u64base_offset_u64(svbool_t pg, svuint64_t bases,
   return SVE_ACLE_FUNC(svld1uw_gather, _u64base, _offset_u64, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uw_gather_s64index_s64(
+// CHECK-LABEL: @test_svld1uw_gather_s64index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.index.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svld1uw_gather_s64index_s64u10__SVBool_tPKju11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z32test_svld1uw_gather_s64index_s64u10__SVBool_tPKju11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.index.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -256,14 +256,14 @@ svint64_t test_svld1uw_gather_s64index_s64(svbool_t pg, const uint32_t *base, sv
   return SVE_ACLE_FUNC(svld1uw_gather_, s64, index_s64, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uw_gather_s64index_u64(
+// CHECK-LABEL: @test_svld1uw_gather_s64index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.index.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svld1uw_gather_s64index_u64u10__SVBool_tPKju11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z32test_svld1uw_gather_s64index_u64u10__SVBool_tPKju11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.index.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -274,14 +274,14 @@ svuint64_t test_svld1uw_gather_s64index_u64(svbool_t pg, const uint32_t *base, s
   return SVE_ACLE_FUNC(svld1uw_gather_, s64, index_u64, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uw_gather_u64index_s64(
+// CHECK-LABEL: @test_svld1uw_gather_u64index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.index.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svld1uw_gather_u64index_s64u10__SVBool_tPKju12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z32test_svld1uw_gather_u64index_s64u10__SVBool_tPKju12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.index.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -292,14 +292,14 @@ svint64_t test_svld1uw_gather_u64index_s64(svbool_t pg, const uint32_t *base, sv
   return SVE_ACLE_FUNC(svld1uw_gather_, u64, index_s64, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uw_gather_u64index_u64(
+// CHECK-LABEL: @test_svld1uw_gather_u64index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.index.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svld1uw_gather_u64index_u64u10__SVBool_tPKju12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z32test_svld1uw_gather_u64index_u64u10__SVBool_tPKju12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.gather.index.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -310,7 +310,7 @@ svuint64_t test_svld1uw_gather_u64index_u64(svbool_t pg, const uint32_t *base, s
   return SVE_ACLE_FUNC(svld1uw_gather_, u64, index_u64, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uw_gather_u64base_index_s64(
+// CHECK-LABEL: @test_svld1uw_gather_u64base_index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 2
@@ -318,7 +318,7 @@ svuint64_t test_svld1uw_gather_u64index_u64(svbool_t pg, const uint32_t *base, s
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 2 x i32> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z37test_svld1uw_gather_u64base_index_s64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z37test_svld1uw_gather_u64base_index_s64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 2
@@ -330,7 +330,7 @@ svint64_t test_svld1uw_gather_u64base_index_s64(svbool_t pg, svuint64_t bases, i
   return SVE_ACLE_FUNC(svld1uw_gather, _u64base, _index_s64, )(pg, bases, index);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld1uw_gather_u64base_index_u64(
+// CHECK-LABEL: @test_svld1uw_gather_u64base_index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 2
@@ -338,7 +338,7 @@ svint64_t test_svld1uw_gather_u64base_index_s64(svbool_t pg, svuint64_t bases, i
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 2 x i32> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z37test_svld1uw_gather_u64base_index_u64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z37test_svld1uw_gather_u64base_index_u64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 2
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld2-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld2-bfloat.c
index dc15f62c55..38ae15f858 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld2-bfloat.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld2-bfloat.c
@@ -15,7 +15,7 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld2_bf16(
+// CHECK-LABEL: @test_svld2_bf16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld2.sret.nxv8bf16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -25,7 +25,7 @@
 // CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 16 x bfloat> @llvm.vector.insert.nxv16bf16.nxv8bf16(<vscale x 16 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[TMP4]], i64 8)
 // CHECK-NEXT:    ret <vscale x 16 x bfloat> [[TMP5]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z15test_svld2_bf16u10__SVBool_tPKu6__bf16(
+// CPP-CHECK-LABEL: @_Z15test_svld2_bf16u10__SVBool_tPKu6__bf16(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld2.sret.nxv8bf16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -41,7 +41,7 @@ svbfloat16x2_t test_svld2_bf16(svbool_t pg, const bfloat16_t *base)
 }
 
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld2_vnum_bf16(
+// CHECK-LABEL: @test_svld2_vnum_bf16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x bfloat>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -52,7 +52,7 @@ svbfloat16x2_t test_svld2_bf16(svbool_t pg, const bfloat16_t *base)
 // CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x bfloat> @llvm.vector.insert.nxv16bf16.nxv8bf16(<vscale x 16 x bfloat> [[TMP4]], <vscale x 8 x bfloat> [[TMP5]], i64 8)
 // CHECK-NEXT:    ret <vscale x 16 x bfloat> [[TMP6]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z20test_svld2_vnum_bf16u10__SVBool_tPKu6__bf16l(
+// CPP-CHECK-LABEL: @_Z20test_svld2_vnum_bf16u10__SVBool_tPKu6__bf16l(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x bfloat>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld2.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld2.c
index d5991cfd58..84946783cd 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld2.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld2.c
@@ -15,7 +15,7 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld2_s8(
+// CHECK-LABEL: @test_svld2_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld2.sret.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 0
@@ -24,7 +24,7 @@
 // CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], i64 16)
 // CHECK-NEXT:    ret <vscale x 32 x i8> [[TMP4]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z13test_svld2_s8u10__SVBool_tPKa(
+// CPP-CHECK-LABEL: @_Z13test_svld2_s8u10__SVBool_tPKa(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld2.sret.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 0
@@ -38,7 +38,7 @@ svint8x2_t test_svld2_s8(svbool_t pg, const int8_t *base)
   return SVE_ACLE_FUNC(svld2,_s8,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld2_s16(
+// CHECK-LABEL: @test_svld2_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld2.sret.nxv8i16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -48,7 +48,7 @@ svint8x2_t test_svld2_s8(svbool_t pg, const int8_t *base)
 // CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], i64 8)
 // CHECK-NEXT:    ret <vscale x 16 x i16> [[TMP5]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svld2_s16u10__SVBool_tPKs(
+// CPP-CHECK-LABEL: @_Z14test_svld2_s16u10__SVBool_tPKs(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld2.sret.nxv8i16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -63,7 +63,7 @@ svint16x2_t test_svld2_s16(svbool_t pg, const int16_t *base)
   return SVE_ACLE_FUNC(svld2,_s16,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld2_s32(
+// CHECK-LABEL: @test_svld2_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld2.sret.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -73,7 +73,7 @@ svint16x2_t test_svld2_s16(svbool_t pg, const int16_t *base)
 // CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], i64 4)
 // CHECK-NEXT:    ret <vscale x 8 x i32> [[TMP5]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svld2_s32u10__SVBool_tPKi(
+// CPP-CHECK-LABEL: @_Z14test_svld2_s32u10__SVBool_tPKi(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld2.sret.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -88,7 +88,7 @@ svint32x2_t test_svld2_s32(svbool_t pg, const int32_t *base)
   return SVE_ACLE_FUNC(svld2,_s32,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld2_s64(
+// CHECK-LABEL: @test_svld2_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld2.sret.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -98,7 +98,7 @@ svint32x2_t test_svld2_s32(svbool_t pg, const int32_t *base)
 // CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], i64 2)
 // CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP5]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svld2_s64u10__SVBool_tPKl(
+// CPP-CHECK-LABEL: @_Z14test_svld2_s64u10__SVBool_tPKl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld2.sret.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -113,7 +113,7 @@ svint64x2_t test_svld2_s64(svbool_t pg, const int64_t *base)
   return SVE_ACLE_FUNC(svld2,_s64,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld2_u8(
+// CHECK-LABEL: @test_svld2_u8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld2.sret.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 0
@@ -122,7 +122,7 @@ svint64x2_t test_svld2_s64(svbool_t pg, const int64_t *base)
 // CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], i64 16)
 // CHECK-NEXT:    ret <vscale x 32 x i8> [[TMP4]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z13test_svld2_u8u10__SVBool_tPKh(
+// CPP-CHECK-LABEL: @_Z13test_svld2_u8u10__SVBool_tPKh(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld2.sret.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 0
@@ -136,7 +136,7 @@ svuint8x2_t test_svld2_u8(svbool_t pg, const uint8_t *base)
   return SVE_ACLE_FUNC(svld2,_u8,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld2_u16(
+// CHECK-LABEL: @test_svld2_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld2.sret.nxv8i16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -146,7 +146,7 @@ svuint8x2_t test_svld2_u8(svbool_t pg, const uint8_t *base)
 // CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], i64 8)
 // CHECK-NEXT:    ret <vscale x 16 x i16> [[TMP5]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svld2_u16u10__SVBool_tPKt(
+// CPP-CHECK-LABEL: @_Z14test_svld2_u16u10__SVBool_tPKt(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld2.sret.nxv8i16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -161,7 +161,7 @@ svuint16x2_t test_svld2_u16(svbool_t pg, const uint16_t *base)
   return SVE_ACLE_FUNC(svld2,_u16,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld2_u32(
+// CHECK-LABEL: @test_svld2_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld2.sret.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -171,7 +171,7 @@ svuint16x2_t test_svld2_u16(svbool_t pg, const uint16_t *base)
 // CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], i64 4)
 // CHECK-NEXT:    ret <vscale x 8 x i32> [[TMP5]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svld2_u32u10__SVBool_tPKj(
+// CPP-CHECK-LABEL: @_Z14test_svld2_u32u10__SVBool_tPKj(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld2.sret.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -186,7 +186,7 @@ svuint32x2_t test_svld2_u32(svbool_t pg, const uint32_t *base)
   return SVE_ACLE_FUNC(svld2,_u32,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld2_u64(
+// CHECK-LABEL: @test_svld2_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld2.sret.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -196,7 +196,7 @@ svuint32x2_t test_svld2_u32(svbool_t pg, const uint32_t *base)
 // CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], i64 2)
 // CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP5]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svld2_u64u10__SVBool_tPKm(
+// CPP-CHECK-LABEL: @_Z14test_svld2_u64u10__SVBool_tPKm(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld2.sret.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -211,7 +211,7 @@ svuint64x2_t test_svld2_u64(svbool_t pg, const uint64_t *base)
   return SVE_ACLE_FUNC(svld2,_u64,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld2_f16(
+// CHECK-LABEL: @test_svld2_f16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld2.sret.nxv8f16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -221,7 +221,7 @@ svuint64x2_t test_svld2_u64(svbool_t pg, const uint64_t *base)
 // CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 16 x half> @llvm.vector.insert.nxv16f16.nxv8f16(<vscale x 16 x half> [[TMP3]], <vscale x 8 x half> [[TMP4]], i64 8)
 // CHECK-NEXT:    ret <vscale x 16 x half> [[TMP5]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svld2_f16u10__SVBool_tPKDh(
+// CPP-CHECK-LABEL: @_Z14test_svld2_f16u10__SVBool_tPKDh(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld2.sret.nxv8f16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -236,7 +236,7 @@ svfloat16x2_t test_svld2_f16(svbool_t pg, const float16_t *base)
   return SVE_ACLE_FUNC(svld2,_f16,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld2_f32(
+// CHECK-LABEL: @test_svld2_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld2.sret.nxv4f32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -246,7 +246,7 @@ svfloat16x2_t test_svld2_f16(svbool_t pg, const float16_t *base)
 // CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> [[TMP3]], <vscale x 4 x float> [[TMP4]], i64 4)
 // CHECK-NEXT:    ret <vscale x 8 x float> [[TMP5]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svld2_f32u10__SVBool_tPKf(
+// CPP-CHECK-LABEL: @_Z14test_svld2_f32u10__SVBool_tPKf(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld2.sret.nxv4f32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -261,7 +261,7 @@ svfloat32x2_t test_svld2_f32(svbool_t pg, const float32_t *base)
   return SVE_ACLE_FUNC(svld2,_f32,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld2_f64(
+// CHECK-LABEL: @test_svld2_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld2.sret.nxv2f64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -271,7 +271,7 @@ svfloat32x2_t test_svld2_f32(svbool_t pg, const float32_t *base)
 // CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 4 x double> @llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> [[TMP3]], <vscale x 2 x double> [[TMP4]], i64 2)
 // CHECK-NEXT:    ret <vscale x 4 x double> [[TMP5]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svld2_f64u10__SVBool_tPKd(
+// CPP-CHECK-LABEL: @_Z14test_svld2_f64u10__SVBool_tPKd(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld2.sret.nxv2f64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -286,7 +286,7 @@ svfloat64x2_t test_svld2_f64(svbool_t pg, const float64_t *base)
   return SVE_ACLE_FUNC(svld2,_f64,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld2_vnum_s8(
+// CHECK-LABEL: @test_svld2_vnum_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld2.sret.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
@@ -296,7 +296,7 @@ svfloat64x2_t test_svld2_f64(svbool_t pg, const float64_t *base)
 // CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], i64 16)
 // CHECK-NEXT:    ret <vscale x 32 x i8> [[TMP5]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svld2_vnum_s8u10__SVBool_tPKal(
+// CPP-CHECK-LABEL: @_Z18test_svld2_vnum_s8u10__SVBool_tPKal(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld2.sret.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
@@ -311,7 +311,7 @@ svint8x2_t test_svld2_vnum_s8(svbool_t pg, const int8_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svld2_vnum,_s8,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld2_vnum_s16(
+// CHECK-LABEL: @test_svld2_vnum_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -322,7 +322,7 @@ svint8x2_t test_svld2_vnum_s8(svbool_t pg, const int8_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], i64 8)
 // CHECK-NEXT:    ret <vscale x 16 x i16> [[TMP6]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svld2_vnum_s16u10__SVBool_tPKsl(
+// CPP-CHECK-LABEL: @_Z19test_svld2_vnum_s16u10__SVBool_tPKsl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -338,7 +338,7 @@ svint16x2_t test_svld2_vnum_s16(svbool_t pg, const int16_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svld2_vnum,_s16,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld2_vnum_s32(
+// CHECK-LABEL: @test_svld2_vnum_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -349,7 +349,7 @@ svint16x2_t test_svld2_vnum_s16(svbool_t pg, const int16_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP4]], <vscale x 4 x i32> [[TMP5]], i64 4)
 // CHECK-NEXT:    ret <vscale x 8 x i32> [[TMP6]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svld2_vnum_s32u10__SVBool_tPKil(
+// CPP-CHECK-LABEL: @_Z19test_svld2_vnum_s32u10__SVBool_tPKil(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -365,7 +365,7 @@ svint32x2_t test_svld2_vnum_s32(svbool_t pg, const int32_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svld2_vnum,_s32,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld2_vnum_s64(
+// CHECK-LABEL: @test_svld2_vnum_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -376,7 +376,7 @@ svint32x2_t test_svld2_vnum_s32(svbool_t pg, const int32_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> [[TMP4]], <vscale x 2 x i64> [[TMP5]], i64 2)
 // CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP6]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svld2_vnum_s64u10__SVBool_tPKll(
+// CPP-CHECK-LABEL: @_Z19test_svld2_vnum_s64u10__SVBool_tPKll(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -392,7 +392,7 @@ svint64x2_t test_svld2_vnum_s64(svbool_t pg, const int64_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svld2_vnum,_s64,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld2_vnum_u8(
+// CHECK-LABEL: @test_svld2_vnum_u8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld2.sret.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
@@ -402,7 +402,7 @@ svint64x2_t test_svld2_vnum_s64(svbool_t pg, const int64_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], i64 16)
 // CHECK-NEXT:    ret <vscale x 32 x i8> [[TMP5]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svld2_vnum_u8u10__SVBool_tPKhl(
+// CPP-CHECK-LABEL: @_Z18test_svld2_vnum_u8u10__SVBool_tPKhl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld2.sret.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
@@ -417,7 +417,7 @@ svuint8x2_t test_svld2_vnum_u8(svbool_t pg, const uint8_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svld2_vnum,_u8,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld2_vnum_u16(
+// CHECK-LABEL: @test_svld2_vnum_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -428,7 +428,7 @@ svuint8x2_t test_svld2_vnum_u8(svbool_t pg, const uint8_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], i64 8)
 // CHECK-NEXT:    ret <vscale x 16 x i16> [[TMP6]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svld2_vnum_u16u10__SVBool_tPKtl(
+// CPP-CHECK-LABEL: @_Z19test_svld2_vnum_u16u10__SVBool_tPKtl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -444,7 +444,7 @@ svuint16x2_t test_svld2_vnum_u16(svbool_t pg, const uint16_t *base, int64_t vnum
   return SVE_ACLE_FUNC(svld2_vnum,_u16,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld2_vnum_u32(
+// CHECK-LABEL: @test_svld2_vnum_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -455,7 +455,7 @@ svuint16x2_t test_svld2_vnum_u16(svbool_t pg, const uint16_t *base, int64_t vnum
 // CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP4]], <vscale x 4 x i32> [[TMP5]], i64 4)
 // CHECK-NEXT:    ret <vscale x 8 x i32> [[TMP6]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svld2_vnum_u32u10__SVBool_tPKjl(
+// CPP-CHECK-LABEL: @_Z19test_svld2_vnum_u32u10__SVBool_tPKjl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -471,7 +471,7 @@ svuint32x2_t test_svld2_vnum_u32(svbool_t pg, const uint32_t *base, int64_t vnum
   return SVE_ACLE_FUNC(svld2_vnum,_u32,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld2_vnum_u64(
+// CHECK-LABEL: @test_svld2_vnum_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -482,7 +482,7 @@ svuint32x2_t test_svld2_vnum_u32(svbool_t pg, const uint32_t *base, int64_t vnum
 // CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> [[TMP4]], <vscale x 2 x i64> [[TMP5]], i64 2)
 // CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP6]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svld2_vnum_u64u10__SVBool_tPKml(
+// CPP-CHECK-LABEL: @_Z19test_svld2_vnum_u64u10__SVBool_tPKml(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -498,7 +498,7 @@ svuint64x2_t test_svld2_vnum_u64(svbool_t pg, const uint64_t *base, int64_t vnum
   return SVE_ACLE_FUNC(svld2_vnum,_u64,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld2_vnum_f16(
+// CHECK-LABEL: @test_svld2_vnum_f16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -509,7 +509,7 @@ svuint64x2_t test_svld2_vnum_u64(svbool_t pg, const uint64_t *base, int64_t vnum
 // CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x half> @llvm.vector.insert.nxv16f16.nxv8f16(<vscale x 16 x half> [[TMP4]], <vscale x 8 x half> [[TMP5]], i64 8)
 // CHECK-NEXT:    ret <vscale x 16 x half> [[TMP6]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svld2_vnum_f16u10__SVBool_tPKDhl(
+// CPP-CHECK-LABEL: @_Z19test_svld2_vnum_f16u10__SVBool_tPKDhl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -525,7 +525,7 @@ svfloat16x2_t test_svld2_vnum_f16(svbool_t pg, const float16_t *base, int64_t vn
   return SVE_ACLE_FUNC(svld2_vnum,_f16,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld2_vnum_f32(
+// CHECK-LABEL: @test_svld2_vnum_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -536,7 +536,7 @@ svfloat16x2_t test_svld2_vnum_f16(svbool_t pg, const float16_t *base, int64_t vn
 // CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> [[TMP4]], <vscale x 4 x float> [[TMP5]], i64 4)
 // CHECK-NEXT:    ret <vscale x 8 x float> [[TMP6]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svld2_vnum_f32u10__SVBool_tPKfl(
+// CPP-CHECK-LABEL: @_Z19test_svld2_vnum_f32u10__SVBool_tPKfl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -552,7 +552,7 @@ svfloat32x2_t test_svld2_vnum_f32(svbool_t pg, const float32_t *base, int64_t vn
   return SVE_ACLE_FUNC(svld2_vnum,_f32,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld2_vnum_f64(
+// CHECK-LABEL: @test_svld2_vnum_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -563,7 +563,7 @@ svfloat32x2_t test_svld2_vnum_f32(svbool_t pg, const float32_t *base, int64_t vn
 // CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 4 x double> @llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> [[TMP4]], <vscale x 2 x double> [[TMP5]], i64 2)
 // CHECK-NEXT:    ret <vscale x 4 x double> [[TMP6]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svld2_vnum_f64u10__SVBool_tPKdl(
+// CPP-CHECK-LABEL: @_Z19test_svld2_vnum_f64u10__SVBool_tPKdl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld3-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld3-bfloat.c
index da3ebc06f4..90b3674cb1 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld3-bfloat.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld3-bfloat.c
@@ -15,7 +15,7 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld3_bf16(
+// CHECK-LABEL: @test_svld3_bf16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld3.sret.nxv8bf16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -27,7 +27,7 @@
 // CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 24 x bfloat> @llvm.vector.insert.nxv24bf16.nxv8bf16(<vscale x 24 x bfloat> [[TMP5]], <vscale x 8 x bfloat> [[TMP6]], i64 16)
 // CHECK-NEXT:    ret <vscale x 24 x bfloat> [[TMP7]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z15test_svld3_bf16u10__SVBool_tPKu6__bf16(
+// CPP-CHECK-LABEL: @_Z15test_svld3_bf16u10__SVBool_tPKu6__bf16(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld3.sret.nxv8bf16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -44,7 +44,7 @@ svbfloat16x3_t test_svld3_bf16(svbool_t pg, const bfloat16_t *base)
   return SVE_ACLE_FUNC(svld3,_bf16,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld3_vnum_bf16(
+// CHECK-LABEL: @test_svld3_vnum_bf16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x bfloat>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -57,7 +57,7 @@ svbfloat16x3_t test_svld3_bf16(svbool_t pg, const bfloat16_t *base)
 // CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 24 x bfloat> @llvm.vector.insert.nxv24bf16.nxv8bf16(<vscale x 24 x bfloat> [[TMP6]], <vscale x 8 x bfloat> [[TMP7]], i64 16)
 // CHECK-NEXT:    ret <vscale x 24 x bfloat> [[TMP8]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z20test_svld3_vnum_bf16u10__SVBool_tPKu6__bf16l(
+// CPP-CHECK-LABEL: @_Z20test_svld3_vnum_bf16u10__SVBool_tPKu6__bf16l(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x bfloat>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld3.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld3.c
index ebcffe37ae..c56ef67007 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld3.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld3.c
@@ -15,7 +15,7 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld3_s8(
+// CHECK-LABEL: @test_svld3_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld3.sret.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 0
@@ -26,7 +26,7 @@
 // CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 48 x i8> @llvm.vector.insert.nxv48i8.nxv16i8(<vscale x 48 x i8> [[TMP4]], <vscale x 16 x i8> [[TMP5]], i64 32)
 // CHECK-NEXT:    ret <vscale x 48 x i8> [[TMP6]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z13test_svld3_s8u10__SVBool_tPKa(
+// CPP-CHECK-LABEL: @_Z13test_svld3_s8u10__SVBool_tPKa(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld3.sret.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 0
@@ -42,7 +42,7 @@ svint8x3_t test_svld3_s8(svbool_t pg, const int8_t *base)
   return SVE_ACLE_FUNC(svld3,_s8,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld3_s16(
+// CHECK-LABEL: @test_svld3_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld3.sret.nxv8i16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -54,7 +54,7 @@ svint8x3_t test_svld3_s8(svbool_t pg, const int8_t *base)
 // CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 24 x i16> @llvm.vector.insert.nxv24i16.nxv8i16(<vscale x 24 x i16> [[TMP5]], <vscale x 8 x i16> [[TMP6]], i64 16)
 // CHECK-NEXT:    ret <vscale x 24 x i16> [[TMP7]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svld3_s16u10__SVBool_tPKs(
+// CPP-CHECK-LABEL: @_Z14test_svld3_s16u10__SVBool_tPKs(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld3.sret.nxv8i16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -71,7 +71,7 @@ svint16x3_t test_svld3_s16(svbool_t pg, const int16_t *base)
   return SVE_ACLE_FUNC(svld3,_s16,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld3_s32(
+// CHECK-LABEL: @test_svld3_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld3.sret.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -83,7 +83,7 @@ svint16x3_t test_svld3_s16(svbool_t pg, const int16_t *base)
 // CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 12 x i32> @llvm.vector.insert.nxv12i32.nxv4i32(<vscale x 12 x i32> [[TMP5]], <vscale x 4 x i32> [[TMP6]], i64 8)
 // CHECK-NEXT:    ret <vscale x 12 x i32> [[TMP7]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svld3_s32u10__SVBool_tPKi(
+// CPP-CHECK-LABEL: @_Z14test_svld3_s32u10__SVBool_tPKi(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld3.sret.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -100,7 +100,7 @@ svint32x3_t test_svld3_s32(svbool_t pg, const int32_t *base)
   return SVE_ACLE_FUNC(svld3,_s32,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld3_s64(
+// CHECK-LABEL: @test_svld3_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld3.sret.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -112,7 +112,7 @@ svint32x3_t test_svld3_s32(svbool_t pg, const int32_t *base)
 // CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 6 x i64> @llvm.vector.insert.nxv6i64.nxv2i64(<vscale x 6 x i64> [[TMP5]], <vscale x 2 x i64> [[TMP6]], i64 4)
 // CHECK-NEXT:    ret <vscale x 6 x i64> [[TMP7]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svld3_s64u10__SVBool_tPKl(
+// CPP-CHECK-LABEL: @_Z14test_svld3_s64u10__SVBool_tPKl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld3.sret.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -129,7 +129,7 @@ svint64x3_t test_svld3_s64(svbool_t pg, const int64_t *base)
   return SVE_ACLE_FUNC(svld3,_s64,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld3_u8(
+// CHECK-LABEL: @test_svld3_u8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld3.sret.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 0
@@ -140,7 +140,7 @@ svint64x3_t test_svld3_s64(svbool_t pg, const int64_t *base)
 // CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 48 x i8> @llvm.vector.insert.nxv48i8.nxv16i8(<vscale x 48 x i8> [[TMP4]], <vscale x 16 x i8> [[TMP5]], i64 32)
 // CHECK-NEXT:    ret <vscale x 48 x i8> [[TMP6]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z13test_svld3_u8u10__SVBool_tPKh(
+// CPP-CHECK-LABEL: @_Z13test_svld3_u8u10__SVBool_tPKh(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld3.sret.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 0
@@ -156,7 +156,7 @@ svuint8x3_t test_svld3_u8(svbool_t pg, const uint8_t *base)
   return SVE_ACLE_FUNC(svld3,_u8,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld3_u16(
+// CHECK-LABEL: @test_svld3_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld3.sret.nxv8i16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -168,7 +168,7 @@ svuint8x3_t test_svld3_u8(svbool_t pg, const uint8_t *base)
 // CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 24 x i16> @llvm.vector.insert.nxv24i16.nxv8i16(<vscale x 24 x i16> [[TMP5]], <vscale x 8 x i16> [[TMP6]], i64 16)
 // CHECK-NEXT:    ret <vscale x 24 x i16> [[TMP7]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svld3_u16u10__SVBool_tPKt(
+// CPP-CHECK-LABEL: @_Z14test_svld3_u16u10__SVBool_tPKt(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld3.sret.nxv8i16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -185,7 +185,7 @@ svuint16x3_t test_svld3_u16(svbool_t pg, const uint16_t *base)
   return SVE_ACLE_FUNC(svld3,_u16,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld3_u32(
+// CHECK-LABEL: @test_svld3_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld3.sret.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -197,7 +197,7 @@ svuint16x3_t test_svld3_u16(svbool_t pg, const uint16_t *base)
 // CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 12 x i32> @llvm.vector.insert.nxv12i32.nxv4i32(<vscale x 12 x i32> [[TMP5]], <vscale x 4 x i32> [[TMP6]], i64 8)
 // CHECK-NEXT:    ret <vscale x 12 x i32> [[TMP7]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svld3_u32u10__SVBool_tPKj(
+// CPP-CHECK-LABEL: @_Z14test_svld3_u32u10__SVBool_tPKj(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld3.sret.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -214,7 +214,7 @@ svuint32x3_t test_svld3_u32(svbool_t pg, const uint32_t *base)
   return SVE_ACLE_FUNC(svld3,_u32,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld3_u64(
+// CHECK-LABEL: @test_svld3_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld3.sret.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -226,7 +226,7 @@ svuint32x3_t test_svld3_u32(svbool_t pg, const uint32_t *base)
 // CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 6 x i64> @llvm.vector.insert.nxv6i64.nxv2i64(<vscale x 6 x i64> [[TMP5]], <vscale x 2 x i64> [[TMP6]], i64 4)
 // CHECK-NEXT:    ret <vscale x 6 x i64> [[TMP7]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svld3_u64u10__SVBool_tPKm(
+// CPP-CHECK-LABEL: @_Z14test_svld3_u64u10__SVBool_tPKm(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld3.sret.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -243,7 +243,7 @@ svuint64x3_t test_svld3_u64(svbool_t pg, const uint64_t *base)
   return SVE_ACLE_FUNC(svld3,_u64,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld3_f16(
+// CHECK-LABEL: @test_svld3_f16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld3.sret.nxv8f16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -255,7 +255,7 @@ svuint64x3_t test_svld3_u64(svbool_t pg, const uint64_t *base)
 // CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 24 x half> @llvm.vector.insert.nxv24f16.nxv8f16(<vscale x 24 x half> [[TMP5]], <vscale x 8 x half> [[TMP6]], i64 16)
 // CHECK-NEXT:    ret <vscale x 24 x half> [[TMP7]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svld3_f16u10__SVBool_tPKDh(
+// CPP-CHECK-LABEL: @_Z14test_svld3_f16u10__SVBool_tPKDh(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld3.sret.nxv8f16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -272,7 +272,7 @@ svfloat16x3_t test_svld3_f16(svbool_t pg, const float16_t *base)
   return SVE_ACLE_FUNC(svld3,_f16,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld3_f32(
+// CHECK-LABEL: @test_svld3_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld3.sret.nxv4f32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -284,7 +284,7 @@ svfloat16x3_t test_svld3_f16(svbool_t pg, const float16_t *base)
 // CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 12 x float> @llvm.vector.insert.nxv12f32.nxv4f32(<vscale x 12 x float> [[TMP5]], <vscale x 4 x float> [[TMP6]], i64 8)
 // CHECK-NEXT:    ret <vscale x 12 x float> [[TMP7]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svld3_f32u10__SVBool_tPKf(
+// CPP-CHECK-LABEL: @_Z14test_svld3_f32u10__SVBool_tPKf(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld3.sret.nxv4f32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -301,7 +301,7 @@ svfloat32x3_t test_svld3_f32(svbool_t pg, const float32_t *base)
   return SVE_ACLE_FUNC(svld3,_f32,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld3_f64(
+// CHECK-LABEL: @test_svld3_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld3.sret.nxv2f64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -313,7 +313,7 @@ svfloat32x3_t test_svld3_f32(svbool_t pg, const float32_t *base)
 // CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 6 x double> @llvm.vector.insert.nxv6f64.nxv2f64(<vscale x 6 x double> [[TMP5]], <vscale x 2 x double> [[TMP6]], i64 4)
 // CHECK-NEXT:    ret <vscale x 6 x double> [[TMP7]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svld3_f64u10__SVBool_tPKd(
+// CPP-CHECK-LABEL: @_Z14test_svld3_f64u10__SVBool_tPKd(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld3.sret.nxv2f64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -330,7 +330,7 @@ svfloat64x3_t test_svld3_f64(svbool_t pg, const float64_t *base)
   return SVE_ACLE_FUNC(svld3,_f64,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld3_vnum_s8(
+// CHECK-LABEL: @test_svld3_vnum_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld3.sret.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
@@ -342,7 +342,7 @@ svfloat64x3_t test_svld3_f64(svbool_t pg, const float64_t *base)
 // CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 48 x i8> @llvm.vector.insert.nxv48i8.nxv16i8(<vscale x 48 x i8> [[TMP5]], <vscale x 16 x i8> [[TMP6]], i64 32)
 // CHECK-NEXT:    ret <vscale x 48 x i8> [[TMP7]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svld3_vnum_s8u10__SVBool_tPKal(
+// CPP-CHECK-LABEL: @_Z18test_svld3_vnum_s8u10__SVBool_tPKal(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld3.sret.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
@@ -359,7 +359,7 @@ svint8x3_t test_svld3_vnum_s8(svbool_t pg, const int8_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svld3_vnum,_s8,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld3_vnum_s16(
+// CHECK-LABEL: @test_svld3_vnum_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -372,7 +372,7 @@ svint8x3_t test_svld3_vnum_s8(svbool_t pg, const int8_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 24 x i16> @llvm.vector.insert.nxv24i16.nxv8i16(<vscale x 24 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]], i64 16)
 // CHECK-NEXT:    ret <vscale x 24 x i16> [[TMP8]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svld3_vnum_s16u10__SVBool_tPKsl(
+// CPP-CHECK-LABEL: @_Z19test_svld3_vnum_s16u10__SVBool_tPKsl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -390,7 +390,7 @@ svint16x3_t test_svld3_vnum_s16(svbool_t pg, const int16_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svld3_vnum,_s16,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld3_vnum_s32(
+// CHECK-LABEL: @test_svld3_vnum_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -403,7 +403,7 @@ svint16x3_t test_svld3_vnum_s16(svbool_t pg, const int16_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 12 x i32> @llvm.vector.insert.nxv12i32.nxv4i32(<vscale x 12 x i32> [[TMP6]], <vscale x 4 x i32> [[TMP7]], i64 8)
 // CHECK-NEXT:    ret <vscale x 12 x i32> [[TMP8]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svld3_vnum_s32u10__SVBool_tPKil(
+// CPP-CHECK-LABEL: @_Z19test_svld3_vnum_s32u10__SVBool_tPKil(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -421,7 +421,7 @@ svint32x3_t test_svld3_vnum_s32(svbool_t pg, const int32_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svld3_vnum,_s32,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld3_vnum_s64(
+// CHECK-LABEL: @test_svld3_vnum_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -434,7 +434,7 @@ svint32x3_t test_svld3_vnum_s32(svbool_t pg, const int32_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 6 x i64> @llvm.vector.insert.nxv6i64.nxv2i64(<vscale x 6 x i64> [[TMP6]], <vscale x 2 x i64> [[TMP7]], i64 4)
 // CHECK-NEXT:    ret <vscale x 6 x i64> [[TMP8]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svld3_vnum_s64u10__SVBool_tPKll(
+// CPP-CHECK-LABEL: @_Z19test_svld3_vnum_s64u10__SVBool_tPKll(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -452,7 +452,7 @@ svint64x3_t test_svld3_vnum_s64(svbool_t pg, const int64_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svld3_vnum,_s64,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld3_vnum_u8(
+// CHECK-LABEL: @test_svld3_vnum_u8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld3.sret.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
@@ -464,7 +464,7 @@ svint64x3_t test_svld3_vnum_s64(svbool_t pg, const int64_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 48 x i8> @llvm.vector.insert.nxv48i8.nxv16i8(<vscale x 48 x i8> [[TMP5]], <vscale x 16 x i8> [[TMP6]], i64 32)
 // CHECK-NEXT:    ret <vscale x 48 x i8> [[TMP7]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svld3_vnum_u8u10__SVBool_tPKhl(
+// CPP-CHECK-LABEL: @_Z18test_svld3_vnum_u8u10__SVBool_tPKhl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld3.sret.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
@@ -481,7 +481,7 @@ svuint8x3_t test_svld3_vnum_u8(svbool_t pg, const uint8_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svld3_vnum,_u8,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld3_vnum_u16(
+// CHECK-LABEL: @test_svld3_vnum_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -494,7 +494,7 @@ svuint8x3_t test_svld3_vnum_u8(svbool_t pg, const uint8_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 24 x i16> @llvm.vector.insert.nxv24i16.nxv8i16(<vscale x 24 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]], i64 16)
 // CHECK-NEXT:    ret <vscale x 24 x i16> [[TMP8]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svld3_vnum_u16u10__SVBool_tPKtl(
+// CPP-CHECK-LABEL: @_Z19test_svld3_vnum_u16u10__SVBool_tPKtl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -512,7 +512,7 @@ svuint16x3_t test_svld3_vnum_u16(svbool_t pg, const uint16_t *base, int64_t vnum
   return SVE_ACLE_FUNC(svld3_vnum,_u16,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld3_vnum_u32(
+// CHECK-LABEL: @test_svld3_vnum_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -525,7 +525,7 @@ svuint16x3_t test_svld3_vnum_u16(svbool_t pg, const uint16_t *base, int64_t vnum
 // CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 12 x i32> @llvm.vector.insert.nxv12i32.nxv4i32(<vscale x 12 x i32> [[TMP6]], <vscale x 4 x i32> [[TMP7]], i64 8)
 // CHECK-NEXT:    ret <vscale x 12 x i32> [[TMP8]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svld3_vnum_u32u10__SVBool_tPKjl(
+// CPP-CHECK-LABEL: @_Z19test_svld3_vnum_u32u10__SVBool_tPKjl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -543,7 +543,7 @@ svuint32x3_t test_svld3_vnum_u32(svbool_t pg, const uint32_t *base, int64_t vnum
   return SVE_ACLE_FUNC(svld3_vnum,_u32,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld3_vnum_u64(
+// CHECK-LABEL: @test_svld3_vnum_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -556,7 +556,7 @@ svuint32x3_t test_svld3_vnum_u32(svbool_t pg, const uint32_t *base, int64_t vnum
 // CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 6 x i64> @llvm.vector.insert.nxv6i64.nxv2i64(<vscale x 6 x i64> [[TMP6]], <vscale x 2 x i64> [[TMP7]], i64 4)
 // CHECK-NEXT:    ret <vscale x 6 x i64> [[TMP8]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svld3_vnum_u64u10__SVBool_tPKml(
+// CPP-CHECK-LABEL: @_Z19test_svld3_vnum_u64u10__SVBool_tPKml(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -574,7 +574,7 @@ svuint64x3_t test_svld3_vnum_u64(svbool_t pg, const uint64_t *base, int64_t vnum
   return SVE_ACLE_FUNC(svld3_vnum,_u64,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld3_vnum_f16(
+// CHECK-LABEL: @test_svld3_vnum_f16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -587,7 +587,7 @@ svuint64x3_t test_svld3_vnum_u64(svbool_t pg, const uint64_t *base, int64_t vnum
 // CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 24 x half> @llvm.vector.insert.nxv24f16.nxv8f16(<vscale x 24 x half> [[TMP6]], <vscale x 8 x half> [[TMP7]], i64 16)
 // CHECK-NEXT:    ret <vscale x 24 x half> [[TMP8]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svld3_vnum_f16u10__SVBool_tPKDhl(
+// CPP-CHECK-LABEL: @_Z19test_svld3_vnum_f16u10__SVBool_tPKDhl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -605,7 +605,7 @@ svfloat16x3_t test_svld3_vnum_f16(svbool_t pg, const float16_t *base, int64_t vn
   return SVE_ACLE_FUNC(svld3_vnum,_f16,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld3_vnum_f32(
+// CHECK-LABEL: @test_svld3_vnum_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -618,7 +618,7 @@ svfloat16x3_t test_svld3_vnum_f16(svbool_t pg, const float16_t *base, int64_t vn
 // CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 12 x float> @llvm.vector.insert.nxv12f32.nxv4f32(<vscale x 12 x float> [[TMP6]], <vscale x 4 x float> [[TMP7]], i64 8)
 // CHECK-NEXT:    ret <vscale x 12 x float> [[TMP8]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svld3_vnum_f32u10__SVBool_tPKfl(
+// CPP-CHECK-LABEL: @_Z19test_svld3_vnum_f32u10__SVBool_tPKfl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -636,7 +636,7 @@ svfloat32x3_t test_svld3_vnum_f32(svbool_t pg, const float32_t *base, int64_t vn
   return SVE_ACLE_FUNC(svld3_vnum,_f32,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld3_vnum_f64(
+// CHECK-LABEL: @test_svld3_vnum_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -649,7 +649,7 @@ svfloat32x3_t test_svld3_vnum_f32(svbool_t pg, const float32_t *base, int64_t vn
 // CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 6 x double> @llvm.vector.insert.nxv6f64.nxv2f64(<vscale x 6 x double> [[TMP6]], <vscale x 2 x double> [[TMP7]], i64 4)
 // CHECK-NEXT:    ret <vscale x 6 x double> [[TMP8]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svld3_vnum_f64u10__SVBool_tPKdl(
+// CPP-CHECK-LABEL: @_Z19test_svld3_vnum_f64u10__SVBool_tPKdl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld4-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld4-bfloat.c
index fc15b957d6..be00d11752 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld4-bfloat.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld4-bfloat.c
@@ -15,7 +15,7 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld4_bf16(
+// CHECK-LABEL: @test_svld4_bf16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld4.sret.nxv8bf16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -29,7 +29,7 @@
 // CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP7]], <vscale x 8 x bfloat> [[TMP8]], i64 24)
 // CHECK-NEXT:    ret <vscale x 32 x bfloat> [[TMP9]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z15test_svld4_bf16u10__SVBool_tPKu6__bf16(
+// CPP-CHECK-LABEL: @_Z15test_svld4_bf16u10__SVBool_tPKu6__bf16(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld4.sret.nxv8bf16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -48,7 +48,7 @@ svbfloat16x4_t test_svld4_bf16(svbool_t pg, const bfloat16_t *base)
   return SVE_ACLE_FUNC(svld4,_bf16,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld4_vnum_bf16(
+// CHECK-LABEL: @test_svld4_vnum_bf16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x bfloat>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -63,7 +63,7 @@ svbfloat16x4_t test_svld4_bf16(svbool_t pg, const bfloat16_t *base)
 // CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 32 x bfloat> @llvm.vector.insert.nxv32bf16.nxv8bf16(<vscale x 32 x bfloat> [[TMP8]], <vscale x 8 x bfloat> [[TMP9]], i64 24)
 // CHECK-NEXT:    ret <vscale x 32 x bfloat> [[TMP10]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z20test_svld4_vnum_bf16u10__SVBool_tPKu6__bf16l(
+// CPP-CHECK-LABEL: @_Z20test_svld4_vnum_bf16u10__SVBool_tPKu6__bf16l(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x bfloat>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld4.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld4.c
index 75c0b334b1..c75c85f939 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld4.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ld4.c
@@ -15,7 +15,7 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld4_s8(
+// CHECK-LABEL: @test_svld4_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld4.sret.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 0
@@ -28,7 +28,7 @@
 // CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]], i64 48)
 // CHECK-NEXT:    ret <vscale x 64 x i8> [[TMP8]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z13test_svld4_s8u10__SVBool_tPKa(
+// CPP-CHECK-LABEL: @_Z13test_svld4_s8u10__SVBool_tPKa(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld4.sret.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 0
@@ -46,7 +46,7 @@ svint8x4_t test_svld4_s8(svbool_t pg, const int8_t *base)
   return SVE_ACLE_FUNC(svld4,_s8,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld4_s16(
+// CHECK-LABEL: @test_svld4_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld4.sret.nxv8i16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -60,7 +60,7 @@ svint8x4_t test_svld4_s8(svbool_t pg, const int8_t *base)
 // CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP7]], <vscale x 8 x i16> [[TMP8]], i64 24)
 // CHECK-NEXT:    ret <vscale x 32 x i16> [[TMP9]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svld4_s16u10__SVBool_tPKs(
+// CPP-CHECK-LABEL: @_Z14test_svld4_s16u10__SVBool_tPKs(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld4.sret.nxv8i16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -79,7 +79,7 @@ svint16x4_t test_svld4_s16(svbool_t pg, const int16_t *base)
   return SVE_ACLE_FUNC(svld4,_s16,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld4_s32(
+// CHECK-LABEL: @test_svld4_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld4.sret.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -93,7 +93,7 @@ svint16x4_t test_svld4_s16(svbool_t pg, const int16_t *base)
 // CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP7]], <vscale x 4 x i32> [[TMP8]], i64 12)
 // CHECK-NEXT:    ret <vscale x 16 x i32> [[TMP9]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svld4_s32u10__SVBool_tPKi(
+// CPP-CHECK-LABEL: @_Z14test_svld4_s32u10__SVBool_tPKi(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld4.sret.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -112,7 +112,7 @@ svint32x4_t test_svld4_s32(svbool_t pg, const int32_t *base)
   return SVE_ACLE_FUNC(svld4,_s32,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld4_s64(
+// CHECK-LABEL: @test_svld4_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld4.sret.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -126,7 +126,7 @@ svint32x4_t test_svld4_s32(svbool_t pg, const int32_t *base)
 // CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP7]], <vscale x 2 x i64> [[TMP8]], i64 6)
 // CHECK-NEXT:    ret <vscale x 8 x i64> [[TMP9]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svld4_s64u10__SVBool_tPKl(
+// CPP-CHECK-LABEL: @_Z14test_svld4_s64u10__SVBool_tPKl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld4.sret.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -145,7 +145,7 @@ svint64x4_t test_svld4_s64(svbool_t pg, const int64_t *base)
   return SVE_ACLE_FUNC(svld4,_s64,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld4_u8(
+// CHECK-LABEL: @test_svld4_u8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld4.sret.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 0
@@ -158,7 +158,7 @@ svint64x4_t test_svld4_s64(svbool_t pg, const int64_t *base)
 // CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]], i64 48)
 // CHECK-NEXT:    ret <vscale x 64 x i8> [[TMP8]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z13test_svld4_u8u10__SVBool_tPKh(
+// CPP-CHECK-LABEL: @_Z13test_svld4_u8u10__SVBool_tPKh(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld4.sret.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 0
@@ -176,7 +176,7 @@ svuint8x4_t test_svld4_u8(svbool_t pg, const uint8_t *base)
   return SVE_ACLE_FUNC(svld4,_u8,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld4_u16(
+// CHECK-LABEL: @test_svld4_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld4.sret.nxv8i16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -190,7 +190,7 @@ svuint8x4_t test_svld4_u8(svbool_t pg, const uint8_t *base)
 // CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP7]], <vscale x 8 x i16> [[TMP8]], i64 24)
 // CHECK-NEXT:    ret <vscale x 32 x i16> [[TMP9]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svld4_u16u10__SVBool_tPKt(
+// CPP-CHECK-LABEL: @_Z14test_svld4_u16u10__SVBool_tPKt(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld4.sret.nxv8i16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -209,7 +209,7 @@ svuint16x4_t test_svld4_u16(svbool_t pg, const uint16_t *base)
   return SVE_ACLE_FUNC(svld4,_u16,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld4_u32(
+// CHECK-LABEL: @test_svld4_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld4.sret.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -223,7 +223,7 @@ svuint16x4_t test_svld4_u16(svbool_t pg, const uint16_t *base)
 // CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP7]], <vscale x 4 x i32> [[TMP8]], i64 12)
 // CHECK-NEXT:    ret <vscale x 16 x i32> [[TMP9]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svld4_u32u10__SVBool_tPKj(
+// CPP-CHECK-LABEL: @_Z14test_svld4_u32u10__SVBool_tPKj(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld4.sret.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -242,7 +242,7 @@ svuint32x4_t test_svld4_u32(svbool_t pg, const uint32_t *base)
   return SVE_ACLE_FUNC(svld4,_u32,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld4_u64(
+// CHECK-LABEL: @test_svld4_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld4.sret.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -256,7 +256,7 @@ svuint32x4_t test_svld4_u32(svbool_t pg, const uint32_t *base)
 // CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP7]], <vscale x 2 x i64> [[TMP8]], i64 6)
 // CHECK-NEXT:    ret <vscale x 8 x i64> [[TMP9]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svld4_u64u10__SVBool_tPKm(
+// CPP-CHECK-LABEL: @_Z14test_svld4_u64u10__SVBool_tPKm(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld4.sret.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -275,7 +275,7 @@ svuint64x4_t test_svld4_u64(svbool_t pg, const uint64_t *base)
   return SVE_ACLE_FUNC(svld4,_u64,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld4_f16(
+// CHECK-LABEL: @test_svld4_f16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld4.sret.nxv8f16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -289,7 +289,7 @@ svuint64x4_t test_svld4_u64(svbool_t pg, const uint64_t *base)
 // CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP7]], <vscale x 8 x half> [[TMP8]], i64 24)
 // CHECK-NEXT:    ret <vscale x 32 x half> [[TMP9]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svld4_f16u10__SVBool_tPKDh(
+// CPP-CHECK-LABEL: @_Z14test_svld4_f16u10__SVBool_tPKDh(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld4.sret.nxv8f16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -308,7 +308,7 @@ svfloat16x4_t test_svld4_f16(svbool_t pg, const float16_t *base)
   return SVE_ACLE_FUNC(svld4,_f16,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld4_f32(
+// CHECK-LABEL: @test_svld4_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld4.sret.nxv4f32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -322,7 +322,7 @@ svfloat16x4_t test_svld4_f16(svbool_t pg, const float16_t *base)
 // CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP7]], <vscale x 4 x float> [[TMP8]], i64 12)
 // CHECK-NEXT:    ret <vscale x 16 x float> [[TMP9]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svld4_f32u10__SVBool_tPKf(
+// CPP-CHECK-LABEL: @_Z14test_svld4_f32u10__SVBool_tPKf(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld4.sret.nxv4f32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -341,7 +341,7 @@ svfloat32x4_t test_svld4_f32(svbool_t pg, const float32_t *base)
   return SVE_ACLE_FUNC(svld4,_f32,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld4_f64(
+// CHECK-LABEL: @test_svld4_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld4.sret.nxv2f64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -355,7 +355,7 @@ svfloat32x4_t test_svld4_f32(svbool_t pg, const float32_t *base)
 // CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP7]], <vscale x 2 x double> [[TMP8]], i64 6)
 // CHECK-NEXT:    ret <vscale x 8 x double> [[TMP9]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svld4_f64u10__SVBool_tPKd(
+// CPP-CHECK-LABEL: @_Z14test_svld4_f64u10__SVBool_tPKd(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld4.sret.nxv2f64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -374,7 +374,7 @@ svfloat64x4_t test_svld4_f64(svbool_t pg, const float64_t *base)
   return SVE_ACLE_FUNC(svld4,_f64,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld4_vnum_s8(
+// CHECK-LABEL: @test_svld4_vnum_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld4.sret.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
@@ -388,7 +388,7 @@ svfloat64x4_t test_svld4_f64(svbool_t pg, const float64_t *base)
 // CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP7]], <vscale x 16 x i8> [[TMP8]], i64 48)
 // CHECK-NEXT:    ret <vscale x 64 x i8> [[TMP9]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svld4_vnum_s8u10__SVBool_tPKal(
+// CPP-CHECK-LABEL: @_Z18test_svld4_vnum_s8u10__SVBool_tPKal(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld4.sret.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
@@ -407,7 +407,7 @@ svint8x4_t test_svld4_vnum_s8(svbool_t pg, const int8_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svld4_vnum,_s8,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld4_vnum_s16(
+// CHECK-LABEL: @test_svld4_vnum_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -422,7 +422,7 @@ svint8x4_t test_svld4_vnum_s8(svbool_t pg, const int8_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP8]], <vscale x 8 x i16> [[TMP9]], i64 24)
 // CHECK-NEXT:    ret <vscale x 32 x i16> [[TMP10]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svld4_vnum_s16u10__SVBool_tPKsl(
+// CPP-CHECK-LABEL: @_Z19test_svld4_vnum_s16u10__SVBool_tPKsl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -442,7 +442,7 @@ svint16x4_t test_svld4_vnum_s16(svbool_t pg, const int16_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svld4_vnum,_s16,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld4_vnum_s32(
+// CHECK-LABEL: @test_svld4_vnum_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -457,7 +457,7 @@ svint16x4_t test_svld4_vnum_s16(svbool_t pg, const int16_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP8]], <vscale x 4 x i32> [[TMP9]], i64 12)
 // CHECK-NEXT:    ret <vscale x 16 x i32> [[TMP10]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svld4_vnum_s32u10__SVBool_tPKil(
+// CPP-CHECK-LABEL: @_Z19test_svld4_vnum_s32u10__SVBool_tPKil(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -477,7 +477,7 @@ svint32x4_t test_svld4_vnum_s32(svbool_t pg, const int32_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svld4_vnum,_s32,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld4_vnum_s64(
+// CHECK-LABEL: @test_svld4_vnum_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -492,7 +492,7 @@ svint32x4_t test_svld4_vnum_s32(svbool_t pg, const int32_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP8]], <vscale x 2 x i64> [[TMP9]], i64 6)
 // CHECK-NEXT:    ret <vscale x 8 x i64> [[TMP10]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svld4_vnum_s64u10__SVBool_tPKll(
+// CPP-CHECK-LABEL: @_Z19test_svld4_vnum_s64u10__SVBool_tPKll(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -512,7 +512,7 @@ svint64x4_t test_svld4_vnum_s64(svbool_t pg, const int64_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svld4_vnum,_s64,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld4_vnum_u8(
+// CHECK-LABEL: @test_svld4_vnum_u8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld4.sret.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
@@ -526,7 +526,7 @@ svint64x4_t test_svld4_vnum_s64(svbool_t pg, const int64_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP7]], <vscale x 16 x i8> [[TMP8]], i64 48)
 // CHECK-NEXT:    ret <vscale x 64 x i8> [[TMP9]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svld4_vnum_u8u10__SVBool_tPKhl(
+// CPP-CHECK-LABEL: @_Z18test_svld4_vnum_u8u10__SVBool_tPKhl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld4.sret.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
@@ -545,7 +545,7 @@ svuint8x4_t test_svld4_vnum_u8(svbool_t pg, const uint8_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svld4_vnum,_u8,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld4_vnum_u16(
+// CHECK-LABEL: @test_svld4_vnum_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -560,7 +560,7 @@ svuint8x4_t test_svld4_vnum_u8(svbool_t pg, const uint8_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP8]], <vscale x 8 x i16> [[TMP9]], i64 24)
 // CHECK-NEXT:    ret <vscale x 32 x i16> [[TMP10]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svld4_vnum_u16u10__SVBool_tPKtl(
+// CPP-CHECK-LABEL: @_Z19test_svld4_vnum_u16u10__SVBool_tPKtl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -580,7 +580,7 @@ svuint16x4_t test_svld4_vnum_u16(svbool_t pg, const uint16_t *base, int64_t vnum
   return SVE_ACLE_FUNC(svld4_vnum,_u16,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld4_vnum_u32(
+// CHECK-LABEL: @test_svld4_vnum_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -595,7 +595,7 @@ svuint16x4_t test_svld4_vnum_u16(svbool_t pg, const uint16_t *base, int64_t vnum
 // CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP8]], <vscale x 4 x i32> [[TMP9]], i64 12)
 // CHECK-NEXT:    ret <vscale x 16 x i32> [[TMP10]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svld4_vnum_u32u10__SVBool_tPKjl(
+// CPP-CHECK-LABEL: @_Z19test_svld4_vnum_u32u10__SVBool_tPKjl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -615,7 +615,7 @@ svuint32x4_t test_svld4_vnum_u32(svbool_t pg, const uint32_t *base, int64_t vnum
   return SVE_ACLE_FUNC(svld4_vnum,_u32,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld4_vnum_u64(
+// CHECK-LABEL: @test_svld4_vnum_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -630,7 +630,7 @@ svuint32x4_t test_svld4_vnum_u32(svbool_t pg, const uint32_t *base, int64_t vnum
 // CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP8]], <vscale x 2 x i64> [[TMP9]], i64 6)
 // CHECK-NEXT:    ret <vscale x 8 x i64> [[TMP10]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svld4_vnum_u64u10__SVBool_tPKml(
+// CPP-CHECK-LABEL: @_Z19test_svld4_vnum_u64u10__SVBool_tPKml(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -650,7 +650,7 @@ svuint64x4_t test_svld4_vnum_u64(svbool_t pg, const uint64_t *base, int64_t vnum
   return SVE_ACLE_FUNC(svld4_vnum,_u64,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld4_vnum_f16(
+// CHECK-LABEL: @test_svld4_vnum_f16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -665,7 +665,7 @@ svuint64x4_t test_svld4_vnum_u64(svbool_t pg, const uint64_t *base, int64_t vnum
 // CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP8]], <vscale x 8 x half> [[TMP9]], i64 24)
 // CHECK-NEXT:    ret <vscale x 32 x half> [[TMP10]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svld4_vnum_f16u10__SVBool_tPKDhl(
+// CPP-CHECK-LABEL: @_Z19test_svld4_vnum_f16u10__SVBool_tPKDhl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -685,7 +685,7 @@ svfloat16x4_t test_svld4_vnum_f16(svbool_t pg, const float16_t *base, int64_t vn
   return SVE_ACLE_FUNC(svld4_vnum,_f16,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld4_vnum_f32(
+// CHECK-LABEL: @test_svld4_vnum_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -700,7 +700,7 @@ svfloat16x4_t test_svld4_vnum_f16(svbool_t pg, const float16_t *base, int64_t vn
 // CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP8]], <vscale x 4 x float> [[TMP9]], i64 12)
 // CHECK-NEXT:    ret <vscale x 16 x float> [[TMP10]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svld4_vnum_f32u10__SVBool_tPKfl(
+// CPP-CHECK-LABEL: @_Z19test_svld4_vnum_f32u10__SVBool_tPKfl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -720,7 +720,7 @@ svfloat32x4_t test_svld4_vnum_f32(svbool_t pg, const float32_t *base, int64_t vn
   return SVE_ACLE_FUNC(svld4_vnum,_f32,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svld4_vnum_f64(
+// CHECK-LABEL: @test_svld4_vnum_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -735,7 +735,7 @@ svfloat32x4_t test_svld4_vnum_f32(svbool_t pg, const float32_t *base, int64_t vn
 // CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP8]], <vscale x 2 x double> [[TMP9]], i64 6)
 // CHECK-NEXT:    ret <vscale x 8 x double> [[TMP10]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svld4_vnum_f64u10__SVBool_tPKdl(
+// CPP-CHECK-LABEL: @_Z19test_svld4_vnum_f64u10__SVBool_tPKdl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1-bfloat.c
index 89b68766e8..e9d516d63a 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1-bfloat.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1-bfloat.c
@@ -15,13 +15,13 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_bf16(
+// CHECK-LABEL: @test_svldff1_bf16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.ldff1.nxv8bf16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z17test_svldff1_bf16u10__SVBool_tPKu6__bf16(
+// CPP-CHECK-LABEL: @_Z17test_svldff1_bf16u10__SVBool_tPKu6__bf16(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.ldff1.nxv8bf16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -32,14 +32,14 @@ svbfloat16_t test_svldff1_bf16(svbool_t pg, const bfloat16_t *base)
   return SVE_ACLE_FUNC(svldff1,_bf16,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_vnum_bf16(
+// CHECK-LABEL: @test_svldff1_vnum_bf16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x bfloat>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.ldff1.nxv8bf16(<vscale x 8 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z22test_svldff1_vnum_bf16u10__SVBool_tPKu6__bf16l(
+// CPP-CHECK-LABEL: @_Z22test_svldff1_vnum_bf16u10__SVBool_tPKu6__bf16l(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x bfloat>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1.c
index cce5641c88..ef0e4de8da 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1.c
@@ -14,12 +14,12 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_s8(
+// CHECK-LABEL: @test_svldff1_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldff1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z15test_svldff1_s8u10__SVBool_tPKa(
+// CPP-CHECK-LABEL: @_Z15test_svldff1_s8u10__SVBool_tPKa(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldff1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
@@ -29,13 +29,13 @@ svint8_t test_svldff1_s8(svbool_t pg, const int8_t *base)
   return SVE_ACLE_FUNC(svldff1,_s8,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_s16(
+// CHECK-LABEL: @test_svldff1_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.ldff1.nxv8i16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svldff1_s16u10__SVBool_tPKs(
+// CPP-CHECK-LABEL: @_Z16test_svldff1_s16u10__SVBool_tPKs(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.ldff1.nxv8i16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -46,13 +46,13 @@ svint16_t test_svldff1_s16(svbool_t pg, const int16_t *base)
   return SVE_ACLE_FUNC(svldff1,_s16,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_s32(
+// CHECK-LABEL: @test_svldff1_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldff1.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svldff1_s32u10__SVBool_tPKi(
+// CPP-CHECK-LABEL: @_Z16test_svldff1_s32u10__SVBool_tPKi(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldff1.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -63,13 +63,13 @@ svint32_t test_svldff1_s32(svbool_t pg, const int32_t *base)
   return SVE_ACLE_FUNC(svldff1,_s32,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_s64(
+// CHECK-LABEL: @test_svldff1_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldff1.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svldff1_s64u10__SVBool_tPKl(
+// CPP-CHECK-LABEL: @_Z16test_svldff1_s64u10__SVBool_tPKl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldff1.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -80,12 +80,12 @@ svint64_t test_svldff1_s64(svbool_t pg, const int64_t *base)
   return SVE_ACLE_FUNC(svldff1,_s64,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_u8(
+// CHECK-LABEL: @test_svldff1_u8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldff1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z15test_svldff1_u8u10__SVBool_tPKh(
+// CPP-CHECK-LABEL: @_Z15test_svldff1_u8u10__SVBool_tPKh(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldff1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
@@ -95,13 +95,13 @@ svuint8_t test_svldff1_u8(svbool_t pg, const uint8_t *base)
   return SVE_ACLE_FUNC(svldff1,_u8,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_u16(
+// CHECK-LABEL: @test_svldff1_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.ldff1.nxv8i16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svldff1_u16u10__SVBool_tPKt(
+// CPP-CHECK-LABEL: @_Z16test_svldff1_u16u10__SVBool_tPKt(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.ldff1.nxv8i16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -112,13 +112,13 @@ svuint16_t test_svldff1_u16(svbool_t pg, const uint16_t *base)
   return SVE_ACLE_FUNC(svldff1,_u16,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_u32(
+// CHECK-LABEL: @test_svldff1_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldff1.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svldff1_u32u10__SVBool_tPKj(
+// CPP-CHECK-LABEL: @_Z16test_svldff1_u32u10__SVBool_tPKj(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldff1.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -129,13 +129,13 @@ svuint32_t test_svldff1_u32(svbool_t pg, const uint32_t *base)
   return SVE_ACLE_FUNC(svldff1,_u32,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_u64(
+// CHECK-LABEL: @test_svldff1_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldff1.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svldff1_u64u10__SVBool_tPKm(
+// CPP-CHECK-LABEL: @_Z16test_svldff1_u64u10__SVBool_tPKm(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldff1.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -146,13 +146,13 @@ svuint64_t test_svldff1_u64(svbool_t pg, const uint64_t *base)
   return SVE_ACLE_FUNC(svldff1,_u64,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_f16(
+// CHECK-LABEL: @test_svldff1_f16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sve.ldff1.nxv8f16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret <vscale x 8 x half> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svldff1_f16u10__SVBool_tPKDh(
+// CPP-CHECK-LABEL: @_Z16test_svldff1_f16u10__SVBool_tPKDh(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sve.ldff1.nxv8f16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -163,13 +163,13 @@ svfloat16_t test_svldff1_f16(svbool_t pg, const float16_t *base)
   return SVE_ACLE_FUNC(svldff1,_f16,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_f32(
+// CHECK-LABEL: @test_svldff1_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.ldff1.nxv4f32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svldff1_f32u10__SVBool_tPKf(
+// CPP-CHECK-LABEL: @_Z16test_svldff1_f32u10__SVBool_tPKf(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.ldff1.nxv4f32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -180,13 +180,13 @@ svfloat32_t test_svldff1_f32(svbool_t pg, const float32_t *base)
   return SVE_ACLE_FUNC(svldff1,_f32,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_f64(
+// CHECK-LABEL: @test_svldff1_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.ldff1.nxv2f64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svldff1_f64u10__SVBool_tPKd(
+// CPP-CHECK-LABEL: @_Z16test_svldff1_f64u10__SVBool_tPKd(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.ldff1.nxv2f64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -197,13 +197,13 @@ svfloat64_t test_svldff1_f64(svbool_t pg, const float64_t *base)
   return SVE_ACLE_FUNC(svldff1,_f64,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_vnum_s8(
+// CHECK-LABEL: @test_svldff1_vnum_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldff1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z20test_svldff1_vnum_s8u10__SVBool_tPKal(
+// CPP-CHECK-LABEL: @_Z20test_svldff1_vnum_s8u10__SVBool_tPKal(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldff1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
@@ -214,14 +214,14 @@ svint8_t test_svldff1_vnum_s8(svbool_t pg, const int8_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svldff1_vnum,_s8,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_vnum_s16(
+// CHECK-LABEL: @test_svldff1_vnum_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.ldff1.nxv8i16(<vscale x 8 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svldff1_vnum_s16u10__SVBool_tPKsl(
+// CPP-CHECK-LABEL: @_Z21test_svldff1_vnum_s16u10__SVBool_tPKsl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -233,14 +233,14 @@ svint16_t test_svldff1_vnum_s16(svbool_t pg, const int16_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svldff1_vnum,_s16,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_vnum_s32(
+// CHECK-LABEL: @test_svldff1_vnum_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldff1.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svldff1_vnum_s32u10__SVBool_tPKil(
+// CPP-CHECK-LABEL: @_Z21test_svldff1_vnum_s32u10__SVBool_tPKil(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -252,14 +252,14 @@ svint32_t test_svldff1_vnum_s32(svbool_t pg, const int32_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svldff1_vnum,_s32,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_vnum_s64(
+// CHECK-LABEL: @test_svldff1_vnum_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldff1.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svldff1_vnum_s64u10__SVBool_tPKll(
+// CPP-CHECK-LABEL: @_Z21test_svldff1_vnum_s64u10__SVBool_tPKll(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -271,13 +271,13 @@ svint64_t test_svldff1_vnum_s64(svbool_t pg, const int64_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svldff1_vnum,_s64,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_vnum_u8(
+// CHECK-LABEL: @test_svldff1_vnum_u8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldff1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z20test_svldff1_vnum_u8u10__SVBool_tPKhl(
+// CPP-CHECK-LABEL: @_Z20test_svldff1_vnum_u8u10__SVBool_tPKhl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldff1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
@@ -288,14 +288,14 @@ svuint8_t test_svldff1_vnum_u8(svbool_t pg, const uint8_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svldff1_vnum,_u8,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_vnum_u16(
+// CHECK-LABEL: @test_svldff1_vnum_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.ldff1.nxv8i16(<vscale x 8 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svldff1_vnum_u16u10__SVBool_tPKtl(
+// CPP-CHECK-LABEL: @_Z21test_svldff1_vnum_u16u10__SVBool_tPKtl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -307,14 +307,14 @@ svuint16_t test_svldff1_vnum_u16(svbool_t pg, const uint16_t *base, int64_t vnum
   return SVE_ACLE_FUNC(svldff1_vnum,_u16,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_vnum_u32(
+// CHECK-LABEL: @test_svldff1_vnum_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldff1.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svldff1_vnum_u32u10__SVBool_tPKjl(
+// CPP-CHECK-LABEL: @_Z21test_svldff1_vnum_u32u10__SVBool_tPKjl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -326,14 +326,14 @@ svuint32_t test_svldff1_vnum_u32(svbool_t pg, const uint32_t *base, int64_t vnum
   return SVE_ACLE_FUNC(svldff1_vnum,_u32,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_vnum_u64(
+// CHECK-LABEL: @test_svldff1_vnum_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldff1.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svldff1_vnum_u64u10__SVBool_tPKml(
+// CPP-CHECK-LABEL: @_Z21test_svldff1_vnum_u64u10__SVBool_tPKml(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -345,14 +345,14 @@ svuint64_t test_svldff1_vnum_u64(svbool_t pg, const uint64_t *base, int64_t vnum
   return SVE_ACLE_FUNC(svldff1_vnum,_u64,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_vnum_f16(
+// CHECK-LABEL: @test_svldff1_vnum_f16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sve.ldff1.nxv8f16(<vscale x 8 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 8 x half> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svldff1_vnum_f16u10__SVBool_tPKDhl(
+// CPP-CHECK-LABEL: @_Z21test_svldff1_vnum_f16u10__SVBool_tPKDhl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -364,14 +364,14 @@ svfloat16_t test_svldff1_vnum_f16(svbool_t pg, const float16_t *base, int64_t vn
   return SVE_ACLE_FUNC(svldff1_vnum,_f16,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_vnum_f32(
+// CHECK-LABEL: @test_svldff1_vnum_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.ldff1.nxv4f32(<vscale x 4 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 4 x float> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svldff1_vnum_f32u10__SVBool_tPKfl(
+// CPP-CHECK-LABEL: @_Z21test_svldff1_vnum_f32u10__SVBool_tPKfl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -383,14 +383,14 @@ svfloat32_t test_svldff1_vnum_f32(svbool_t pg, const float32_t *base, int64_t vn
   return SVE_ACLE_FUNC(svldff1_vnum,_f32,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_vnum_f64(
+// CHECK-LABEL: @test_svldff1_vnum_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.ldff1.nxv2f64(<vscale x 2 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 2 x double> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svldff1_vnum_f64u10__SVBool_tPKdl(
+// CPP-CHECK-LABEL: @_Z21test_svldff1_vnum_f64u10__SVBool_tPKdl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -402,13 +402,13 @@ svfloat64_t test_svldff1_vnum_f64(svbool_t pg, const float64_t *base, int64_t vn
   return SVE_ACLE_FUNC(svldff1_vnum,_f64,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_u32base_s32(
+// CHECK-LABEL: @test_svldff1_gather_u32base_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svldff1_gather_u32base_s32u10__SVBool_tu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z31test_svldff1_gather_u32base_s32u10__SVBool_tu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
@@ -418,13 +418,13 @@ svint32_t test_svldff1_gather_u32base_s32(svbool_t pg, svuint32_t bases) {
   return SVE_ACLE_FUNC(svldff1_gather, _u32base, _s32, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_u64base_s64(
+// CHECK-LABEL: @test_svldff1_gather_u64base_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svldff1_gather_u64base_s64u10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z31test_svldff1_gather_u64base_s64u10__SVBool_tu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
@@ -434,13 +434,13 @@ svint64_t test_svldff1_gather_u64base_s64(svbool_t pg, svuint64_t bases) {
   return SVE_ACLE_FUNC(svldff1_gather, _u64base, _s64, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_u32base_u32(
+// CHECK-LABEL: @test_svldff1_gather_u32base_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svldff1_gather_u32base_u32u10__SVBool_tu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z31test_svldff1_gather_u32base_u32u10__SVBool_tu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
@@ -450,13 +450,13 @@ svuint32_t test_svldff1_gather_u32base_u32(svbool_t pg, svuint32_t bases) {
   return SVE_ACLE_FUNC(svldff1_gather, _u32base, _u32, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_u64base_u64(
+// CHECK-LABEL: @test_svldff1_gather_u64base_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svldff1_gather_u64base_u64u10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z31test_svldff1_gather_u64base_u64u10__SVBool_tu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
@@ -466,13 +466,13 @@ svuint64_t test_svldff1_gather_u64base_u64(svbool_t pg, svuint64_t bases) {
   return SVE_ACLE_FUNC(svldff1_gather, _u64base, _u64, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_u32base_f32(
+// CHECK-LABEL: @test_svldff1_gather_u32base_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4f32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svldff1_gather_u32base_f32u10__SVBool_tu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z31test_svldff1_gather_u32base_f32u10__SVBool_tu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4f32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
@@ -482,13 +482,13 @@ svfloat32_t test_svldff1_gather_u32base_f32(svbool_t pg, svuint32_t bases) {
   return SVE_ACLE_FUNC(svldff1_gather, _u32base, _f32, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_u64base_f64(
+// CHECK-LABEL: @test_svldff1_gather_u64base_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2f64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svldff1_gather_u64base_f64u10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z31test_svldff1_gather_u64base_f64u10__SVBool_tu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2f64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
@@ -498,13 +498,13 @@ svfloat64_t test_svldff1_gather_u64base_f64(svbool_t pg, svuint64_t bases) {
   return SVE_ACLE_FUNC(svldff1_gather, _u64base, _f64, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_s32offset_s32(
+// CHECK-LABEL: @test_svldff1_gather_s32offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldff1.gather.sxtw.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svldff1_gather_s32offset_s32u10__SVBool_tPKiu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z33test_svldff1_gather_s32offset_s32u10__SVBool_tPKiu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldff1.gather.sxtw.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -514,13 +514,13 @@ svint32_t test_svldff1_gather_s32offset_s32(svbool_t pg, const int32_t *base, sv
   return SVE_ACLE_FUNC(svldff1_gather_, s32, offset, _s32)(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_s64offset_s64(
+// CHECK-LABEL: @test_svldff1_gather_s64offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldff1.gather.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svldff1_gather_s64offset_s64u10__SVBool_tPKlu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z33test_svldff1_gather_s64offset_s64u10__SVBool_tPKlu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldff1.gather.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -530,13 +530,13 @@ svint64_t test_svldff1_gather_s64offset_s64(svbool_t pg, const int64_t *base, sv
   return SVE_ACLE_FUNC(svldff1_gather_, s64, offset, _s64)(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_s32offset_u32(
+// CHECK-LABEL: @test_svldff1_gather_s32offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldff1.gather.sxtw.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svldff1_gather_s32offset_u32u10__SVBool_tPKju11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z33test_svldff1_gather_s32offset_u32u10__SVBool_tPKju11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldff1.gather.sxtw.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -546,13 +546,13 @@ svuint32_t test_svldff1_gather_s32offset_u32(svbool_t pg, const uint32_t *base,
   return SVE_ACLE_FUNC(svldff1_gather_, s32, offset, _u32)(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_s64offset_u64(
+// CHECK-LABEL: @test_svldff1_gather_s64offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldff1.gather.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svldff1_gather_s64offset_u64u10__SVBool_tPKmu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z33test_svldff1_gather_s64offset_u64u10__SVBool_tPKmu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldff1.gather.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -562,13 +562,13 @@ svuint64_t test_svldff1_gather_s64offset_u64(svbool_t pg, const uint64_t *base,
   return SVE_ACLE_FUNC(svldff1_gather_, s64, offset, _u64)(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_s32offset_f32(
+// CHECK-LABEL: @test_svldff1_gather_s32offset_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.ldff1.gather.sxtw.nxv4f32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svldff1_gather_s32offset_f32u10__SVBool_tPKfu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z33test_svldff1_gather_s32offset_f32u10__SVBool_tPKfu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.ldff1.gather.sxtw.nxv4f32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -578,13 +578,13 @@ svfloat32_t test_svldff1_gather_s32offset_f32(svbool_t pg, const float32_t *base
   return SVE_ACLE_FUNC(svldff1_gather_, s32, offset, _f32)(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_s64offset_f64(
+// CHECK-LABEL: @test_svldff1_gather_s64offset_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.ldff1.gather.nxv2f64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svldff1_gather_s64offset_f64u10__SVBool_tPKdu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z33test_svldff1_gather_s64offset_f64u10__SVBool_tPKdu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.ldff1.gather.nxv2f64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -594,13 +594,13 @@ svfloat64_t test_svldff1_gather_s64offset_f64(svbool_t pg, const float64_t *base
   return SVE_ACLE_FUNC(svldff1_gather_, s64, offset, _f64)(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_u32offset_s32(
+// CHECK-LABEL: @test_svldff1_gather_u32offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldff1.gather.uxtw.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svldff1_gather_u32offset_s32u10__SVBool_tPKiu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z33test_svldff1_gather_u32offset_s32u10__SVBool_tPKiu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldff1.gather.uxtw.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -610,13 +610,13 @@ svint32_t test_svldff1_gather_u32offset_s32(svbool_t pg, const int32_t *base, sv
   return SVE_ACLE_FUNC(svldff1_gather_, u32, offset, _s32)(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_u64offset_s64(
+// CHECK-LABEL: @test_svldff1_gather_u64offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldff1.gather.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svldff1_gather_u64offset_s64u10__SVBool_tPKlu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z33test_svldff1_gather_u64offset_s64u10__SVBool_tPKlu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldff1.gather.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -626,13 +626,13 @@ svint64_t test_svldff1_gather_u64offset_s64(svbool_t pg, const int64_t *base, sv
   return SVE_ACLE_FUNC(svldff1_gather_, u64, offset, _s64)(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_u32offset_u32(
+// CHECK-LABEL: @test_svldff1_gather_u32offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldff1.gather.uxtw.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svldff1_gather_u32offset_u32u10__SVBool_tPKju12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z33test_svldff1_gather_u32offset_u32u10__SVBool_tPKju12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldff1.gather.uxtw.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -642,13 +642,13 @@ svuint32_t test_svldff1_gather_u32offset_u32(svbool_t pg, const uint32_t *base,
   return SVE_ACLE_FUNC(svldff1_gather_, u32, offset, _u32)(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_u64offset_u64(
+// CHECK-LABEL: @test_svldff1_gather_u64offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldff1.gather.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svldff1_gather_u64offset_u64u10__SVBool_tPKmu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z33test_svldff1_gather_u64offset_u64u10__SVBool_tPKmu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldff1.gather.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -658,13 +658,13 @@ svuint64_t test_svldff1_gather_u64offset_u64(svbool_t pg, const uint64_t *base,
   return SVE_ACLE_FUNC(svldff1_gather_, u64, offset, _u64)(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_u32offset_f32(
+// CHECK-LABEL: @test_svldff1_gather_u32offset_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.ldff1.gather.uxtw.nxv4f32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svldff1_gather_u32offset_f32u10__SVBool_tPKfu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z33test_svldff1_gather_u32offset_f32u10__SVBool_tPKfu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.ldff1.gather.uxtw.nxv4f32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -674,13 +674,13 @@ svfloat32_t test_svldff1_gather_u32offset_f32(svbool_t pg, const float32_t *base
   return SVE_ACLE_FUNC(svldff1_gather_, u32, offset, _f32)(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_u64offset_f64(
+// CHECK-LABEL: @test_svldff1_gather_u64offset_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.ldff1.gather.nxv2f64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svldff1_gather_u64offset_f64u10__SVBool_tPKdu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z33test_svldff1_gather_u64offset_f64u10__SVBool_tPKdu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.ldff1.gather.nxv2f64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -690,13 +690,13 @@ svfloat64_t test_svldff1_gather_u64offset_f64(svbool_t pg, const float64_t *base
   return SVE_ACLE_FUNC(svldff1_gather_, u64, offset, _f64)(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_u32base_offset_s32(
+// CHECK-LABEL: @test_svldff1_gather_u32base_offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z38test_svldff1_gather_u32base_offset_s32u10__SVBool_tu12__SVUint32_tl(
+// CPP-CHECK-LABEL: @_Z38test_svldff1_gather_u32base_offset_s32u10__SVBool_tu12__SVUint32_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -706,13 +706,13 @@ svint32_t test_svldff1_gather_u32base_offset_s32(svbool_t pg, svuint32_t bases,
   return SVE_ACLE_FUNC(svldff1_gather, _u32base, _offset_s32, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_u64base_offset_s64(
+// CHECK-LABEL: @test_svldff1_gather_u64base_offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z38test_svldff1_gather_u64base_offset_s64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z38test_svldff1_gather_u64base_offset_s64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -722,13 +722,13 @@ svint64_t test_svldff1_gather_u64base_offset_s64(svbool_t pg, svuint64_t bases,
   return SVE_ACLE_FUNC(svldff1_gather, _u64base, _offset_s64, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_u32base_offset_u32(
+// CHECK-LABEL: @test_svldff1_gather_u32base_offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z38test_svldff1_gather_u32base_offset_u32u10__SVBool_tu12__SVUint32_tl(
+// CPP-CHECK-LABEL: @_Z38test_svldff1_gather_u32base_offset_u32u10__SVBool_tu12__SVUint32_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -738,13 +738,13 @@ svuint32_t test_svldff1_gather_u32base_offset_u32(svbool_t pg, svuint32_t bases,
   return SVE_ACLE_FUNC(svldff1_gather, _u32base, _offset_u32, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_u64base_offset_u64(
+// CHECK-LABEL: @test_svldff1_gather_u64base_offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z38test_svldff1_gather_u64base_offset_u64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z38test_svldff1_gather_u64base_offset_u64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -754,13 +754,13 @@ svuint64_t test_svldff1_gather_u64base_offset_u64(svbool_t pg, svuint64_t bases,
   return SVE_ACLE_FUNC(svldff1_gather, _u64base, _offset_u64, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_u32base_offset_f32(
+// CHECK-LABEL: @test_svldff1_gather_u32base_offset_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4f32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z38test_svldff1_gather_u32base_offset_f32u10__SVBool_tu12__SVUint32_tl(
+// CPP-CHECK-LABEL: @_Z38test_svldff1_gather_u32base_offset_f32u10__SVBool_tu12__SVUint32_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4f32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -770,13 +770,13 @@ svfloat32_t test_svldff1_gather_u32base_offset_f32(svbool_t pg, svuint32_t bases
   return SVE_ACLE_FUNC(svldff1_gather, _u32base, _offset_f32, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_u64base_offset_f64(
+// CHECK-LABEL: @test_svldff1_gather_u64base_offset_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2f64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z38test_svldff1_gather_u64base_offset_f64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z38test_svldff1_gather_u64base_offset_f64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2f64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -786,13 +786,13 @@ svfloat64_t test_svldff1_gather_u64base_offset_f64(svbool_t pg, svuint64_t bases
   return SVE_ACLE_FUNC(svldff1_gather, _u64base, _offset_f64, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_s32index_s32(
+// CHECK-LABEL: @test_svldff1_gather_s32index_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldff1.gather.sxtw.index.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svldff1_gather_s32index_s32u10__SVBool_tPKiu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z32test_svldff1_gather_s32index_s32u10__SVBool_tPKiu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldff1.gather.sxtw.index.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
@@ -802,13 +802,13 @@ svint32_t test_svldff1_gather_s32index_s32(svbool_t pg, const int32_t *base, svi
   return SVE_ACLE_FUNC(svldff1_gather_, s32, index, _s32)(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_s64index_s64(
+// CHECK-LABEL: @test_svldff1_gather_s64index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldff1.gather.index.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svldff1_gather_s64index_s64u10__SVBool_tPKlu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z32test_svldff1_gather_s64index_s64u10__SVBool_tPKlu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldff1.gather.index.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -818,13 +818,13 @@ svint64_t test_svldff1_gather_s64index_s64(svbool_t pg, const int64_t *base, svi
   return SVE_ACLE_FUNC(svldff1_gather_, s64, index, _s64)(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_s32index_u32(
+// CHECK-LABEL: @test_svldff1_gather_s32index_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldff1.gather.sxtw.index.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svldff1_gather_s32index_u32u10__SVBool_tPKju11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z32test_svldff1_gather_s32index_u32u10__SVBool_tPKju11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldff1.gather.sxtw.index.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
@@ -834,13 +834,13 @@ svuint32_t test_svldff1_gather_s32index_u32(svbool_t pg, const uint32_t *base, s
   return SVE_ACLE_FUNC(svldff1_gather_, s32, index, _u32)(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_s64index_u64(
+// CHECK-LABEL: @test_svldff1_gather_s64index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldff1.gather.index.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svldff1_gather_s64index_u64u10__SVBool_tPKmu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z32test_svldff1_gather_s64index_u64u10__SVBool_tPKmu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldff1.gather.index.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -850,13 +850,13 @@ svuint64_t test_svldff1_gather_s64index_u64(svbool_t pg, const uint64_t *base, s
   return SVE_ACLE_FUNC(svldff1_gather_, s64, index, _u64)(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_s32index_f32(
+// CHECK-LABEL: @test_svldff1_gather_s32index_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.ldff1.gather.sxtw.index.nxv4f32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svldff1_gather_s32index_f32u10__SVBool_tPKfu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z32test_svldff1_gather_s32index_f32u10__SVBool_tPKfu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.ldff1.gather.sxtw.index.nxv4f32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
@@ -866,13 +866,13 @@ svfloat32_t test_svldff1_gather_s32index_f32(svbool_t pg, const float32_t *base,
   return SVE_ACLE_FUNC(svldff1_gather_, s32, index, _f32)(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_s64index_f64(
+// CHECK-LABEL: @test_svldff1_gather_s64index_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.ldff1.gather.index.nxv2f64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svldff1_gather_s64index_f64u10__SVBool_tPKdu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z32test_svldff1_gather_s64index_f64u10__SVBool_tPKdu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.ldff1.gather.index.nxv2f64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -882,13 +882,13 @@ svfloat64_t test_svldff1_gather_s64index_f64(svbool_t pg, const float64_t *base,
   return SVE_ACLE_FUNC(svldff1_gather_, s64, index, _f64)(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_u32index_s32(
+// CHECK-LABEL: @test_svldff1_gather_u32index_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldff1.gather.uxtw.index.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svldff1_gather_u32index_s32u10__SVBool_tPKiu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z32test_svldff1_gather_u32index_s32u10__SVBool_tPKiu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldff1.gather.uxtw.index.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
@@ -898,13 +898,13 @@ svint32_t test_svldff1_gather_u32index_s32(svbool_t pg, const int32_t *base, svu
   return SVE_ACLE_FUNC(svldff1_gather_, u32, index, _s32)(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_u64index_s64(
+// CHECK-LABEL: @test_svldff1_gather_u64index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldff1.gather.index.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svldff1_gather_u64index_s64u10__SVBool_tPKlu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z32test_svldff1_gather_u64index_s64u10__SVBool_tPKlu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldff1.gather.index.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -914,13 +914,13 @@ svint64_t test_svldff1_gather_u64index_s64(svbool_t pg, const int64_t *base, svu
   return SVE_ACLE_FUNC(svldff1_gather_, u64, index, _s64)(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_u32index_u32(
+// CHECK-LABEL: @test_svldff1_gather_u32index_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldff1.gather.uxtw.index.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svldff1_gather_u32index_u32u10__SVBool_tPKju12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z32test_svldff1_gather_u32index_u32u10__SVBool_tPKju12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldff1.gather.uxtw.index.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
@@ -930,13 +930,13 @@ svuint32_t test_svldff1_gather_u32index_u32(svbool_t pg, const uint32_t *base, s
   return SVE_ACLE_FUNC(svldff1_gather_, u32, index, _u32)(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_u64index_u64(
+// CHECK-LABEL: @test_svldff1_gather_u64index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldff1.gather.index.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svldff1_gather_u64index_u64u10__SVBool_tPKmu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z32test_svldff1_gather_u64index_u64u10__SVBool_tPKmu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldff1.gather.index.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -946,13 +946,13 @@ svuint64_t test_svldff1_gather_u64index_u64(svbool_t pg, const uint64_t *base, s
   return SVE_ACLE_FUNC(svldff1_gather_, u64, index, _u64)(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_u32index_f32(
+// CHECK-LABEL: @test_svldff1_gather_u32index_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.ldff1.gather.uxtw.index.nxv4f32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svldff1_gather_u32index_f32u10__SVBool_tPKfu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z32test_svldff1_gather_u32index_f32u10__SVBool_tPKfu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.ldff1.gather.uxtw.index.nxv4f32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
@@ -962,13 +962,13 @@ svfloat32_t test_svldff1_gather_u32index_f32(svbool_t pg, const float32_t *base,
   return SVE_ACLE_FUNC(svldff1_gather_, u32, index, _f32)(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_u64index_f64(
+// CHECK-LABEL: @test_svldff1_gather_u64index_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.ldff1.gather.index.nxv2f64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svldff1_gather_u64index_f64u10__SVBool_tPKdu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z32test_svldff1_gather_u64index_f64u10__SVBool_tPKdu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.ldff1.gather.index.nxv2f64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -978,14 +978,14 @@ svfloat64_t test_svldff1_gather_u64index_f64(svbool_t pg, const float64_t *base,
   return SVE_ACLE_FUNC(svldff1_gather_, u64, index, _f64)(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_u32base_index_s32(
+// CHECK-LABEL: @test_svldff1_gather_u32base_index_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 2
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z37test_svldff1_gather_u32base_index_s32u10__SVBool_tu12__SVUint32_tl(
+// CPP-CHECK-LABEL: @_Z37test_svldff1_gather_u32base_index_s32u10__SVBool_tu12__SVUint32_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 2
@@ -996,14 +996,14 @@ svint32_t test_svldff1_gather_u32base_index_s32(svbool_t pg, svuint32_t bases, i
   return SVE_ACLE_FUNC(svldff1_gather, _u32base, _index_s32, )(pg, bases, index);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_u64base_index_s64(
+// CHECK-LABEL: @test_svldff1_gather_u64base_index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 3
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z37test_svldff1_gather_u64base_index_s64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z37test_svldff1_gather_u64base_index_s64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 3
@@ -1014,14 +1014,14 @@ svint64_t test_svldff1_gather_u64base_index_s64(svbool_t pg, svuint64_t bases, i
   return SVE_ACLE_FUNC(svldff1_gather, _u64base, _index_s64, )(pg, bases, index);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_u32base_index_u32(
+// CHECK-LABEL: @test_svldff1_gather_u32base_index_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 2
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z37test_svldff1_gather_u32base_index_u32u10__SVBool_tu12__SVUint32_tl(
+// CPP-CHECK-LABEL: @_Z37test_svldff1_gather_u32base_index_u32u10__SVBool_tu12__SVUint32_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 2
@@ -1032,14 +1032,14 @@ svuint32_t test_svldff1_gather_u32base_index_u32(svbool_t pg, svuint32_t bases,
   return SVE_ACLE_FUNC(svldff1_gather, _u32base, _index_u32, )(pg, bases, index);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_u64base_index_u64(
+// CHECK-LABEL: @test_svldff1_gather_u64base_index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 3
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z37test_svldff1_gather_u64base_index_u64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z37test_svldff1_gather_u64base_index_u64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 3
@@ -1050,14 +1050,14 @@ svuint64_t test_svldff1_gather_u64base_index_u64(svbool_t pg, svuint64_t bases,
   return SVE_ACLE_FUNC(svldff1_gather, _u64base, _index_u64, )(pg, bases, index);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_u32base_index_f32(
+// CHECK-LABEL: @test_svldff1_gather_u32base_index_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 2
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4f32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 4 x float> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z37test_svldff1_gather_u32base_index_f32u10__SVBool_tu12__SVUint32_tl(
+// CPP-CHECK-LABEL: @_Z37test_svldff1_gather_u32base_index_f32u10__SVBool_tu12__SVUint32_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 2
@@ -1068,14 +1068,14 @@ svfloat32_t test_svldff1_gather_u32base_index_f32(svbool_t pg, svuint32_t bases,
   return SVE_ACLE_FUNC(svldff1_gather, _u32base, _index_f32, )(pg, bases, index);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1_gather_u64base_index_f64(
+// CHECK-LABEL: @test_svldff1_gather_u64base_index_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 3
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2f64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 2 x double> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z37test_svldff1_gather_u64base_index_f64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z37test_svldff1_gather_u64base_index_f64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 3
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1sb.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1sb.c
index 6fdec4cddb..00077bde95 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1sb.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1sb.c
@@ -14,14 +14,14 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sb_s16(
+// CHECK-LABEL: @test_svldff1sb_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i8> @llvm.aarch64.sve.ldff1.nxv8i8(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 8 x i8> [[TMP1]] to <vscale x 8 x i16>
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldff1sb_s16u10__SVBool_tPKa(
+// CPP-CHECK-LABEL: @_Z18test_svldff1sb_s16u10__SVBool_tPKa(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i8> @llvm.aarch64.sve.ldff1.nxv8i8(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -33,14 +33,14 @@ svint16_t test_svldff1sb_s16(svbool_t pg, const int8_t *base)
   return svldff1sb_s16(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sb_s32(
+// CHECK-LABEL: @test_svldff1sb_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldff1sb_s32u10__SVBool_tPKa(
+// CPP-CHECK-LABEL: @_Z18test_svldff1sb_s32u10__SVBool_tPKa(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -52,14 +52,14 @@ svint32_t test_svldff1sb_s32(svbool_t pg, const int8_t *base)
   return svldff1sb_s32(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sb_s64(
+// CHECK-LABEL: @test_svldff1sb_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldff1sb_s64u10__SVBool_tPKa(
+// CPP-CHECK-LABEL: @_Z18test_svldff1sb_s64u10__SVBool_tPKa(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -71,14 +71,14 @@ svint64_t test_svldff1sb_s64(svbool_t pg, const int8_t *base)
   return svldff1sb_s64(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sb_u16(
+// CHECK-LABEL: @test_svldff1sb_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i8> @llvm.aarch64.sve.ldff1.nxv8i8(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 8 x i8> [[TMP1]] to <vscale x 8 x i16>
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldff1sb_u16u10__SVBool_tPKa(
+// CPP-CHECK-LABEL: @_Z18test_svldff1sb_u16u10__SVBool_tPKa(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i8> @llvm.aarch64.sve.ldff1.nxv8i8(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -90,14 +90,14 @@ svuint16_t test_svldff1sb_u16(svbool_t pg, const int8_t *base)
   return svldff1sb_u16(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sb_u32(
+// CHECK-LABEL: @test_svldff1sb_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldff1sb_u32u10__SVBool_tPKa(
+// CPP-CHECK-LABEL: @_Z18test_svldff1sb_u32u10__SVBool_tPKa(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -109,14 +109,14 @@ svuint32_t test_svldff1sb_u32(svbool_t pg, const int8_t *base)
   return svldff1sb_u32(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sb_u64(
+// CHECK-LABEL: @test_svldff1sb_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldff1sb_u64u10__SVBool_tPKa(
+// CPP-CHECK-LABEL: @_Z18test_svldff1sb_u64u10__SVBool_tPKa(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -128,7 +128,7 @@ svuint64_t test_svldff1sb_u64(svbool_t pg, const int8_t *base)
   return svldff1sb_u64(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sb_vnum_s16(
+// CHECK-LABEL: @test_svldff1sb_vnum_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -136,7 +136,7 @@ svuint64_t test_svldff1sb_u64(svbool_t pg, const int8_t *base)
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 8 x i8> [[TMP2]] to <vscale x 8 x i16>
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldff1sb_vnum_s16u10__SVBool_tPKal(
+// CPP-CHECK-LABEL: @_Z23test_svldff1sb_vnum_s16u10__SVBool_tPKal(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -149,7 +149,7 @@ svint16_t test_svldff1sb_vnum_s16(svbool_t pg, const int8_t *base, int64_t vnum)
   return svldff1sb_vnum_s16(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sb_vnum_s32(
+// CHECK-LABEL: @test_svldff1sb_vnum_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -157,7 +157,7 @@ svint16_t test_svldff1sb_vnum_s16(svbool_t pg, const int8_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 4 x i8> [[TMP2]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldff1sb_vnum_s32u10__SVBool_tPKal(
+// CPP-CHECK-LABEL: @_Z23test_svldff1sb_vnum_s32u10__SVBool_tPKal(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -170,7 +170,7 @@ svint32_t test_svldff1sb_vnum_s32(svbool_t pg, const int8_t *base, int64_t vnum)
   return svldff1sb_vnum_s32(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sb_vnum_s64(
+// CHECK-LABEL: @test_svldff1sb_vnum_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -178,7 +178,7 @@ svint32_t test_svldff1sb_vnum_s32(svbool_t pg, const int8_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 2 x i8> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldff1sb_vnum_s64u10__SVBool_tPKal(
+// CPP-CHECK-LABEL: @_Z23test_svldff1sb_vnum_s64u10__SVBool_tPKal(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -191,7 +191,7 @@ svint64_t test_svldff1sb_vnum_s64(svbool_t pg, const int8_t *base, int64_t vnum)
   return svldff1sb_vnum_s64(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sb_vnum_u16(
+// CHECK-LABEL: @test_svldff1sb_vnum_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -199,7 +199,7 @@ svint64_t test_svldff1sb_vnum_s64(svbool_t pg, const int8_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 8 x i8> [[TMP2]] to <vscale x 8 x i16>
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldff1sb_vnum_u16u10__SVBool_tPKal(
+// CPP-CHECK-LABEL: @_Z23test_svldff1sb_vnum_u16u10__SVBool_tPKal(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -212,7 +212,7 @@ svuint16_t test_svldff1sb_vnum_u16(svbool_t pg, const int8_t *base, int64_t vnum
   return svldff1sb_vnum_u16(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sb_vnum_u32(
+// CHECK-LABEL: @test_svldff1sb_vnum_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -220,7 +220,7 @@ svuint16_t test_svldff1sb_vnum_u16(svbool_t pg, const int8_t *base, int64_t vnum
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 4 x i8> [[TMP2]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldff1sb_vnum_u32u10__SVBool_tPKal(
+// CPP-CHECK-LABEL: @_Z23test_svldff1sb_vnum_u32u10__SVBool_tPKal(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -233,7 +233,7 @@ svuint32_t test_svldff1sb_vnum_u32(svbool_t pg, const int8_t *base, int64_t vnum
   return svldff1sb_vnum_u32(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sb_vnum_u64(
+// CHECK-LABEL: @test_svldff1sb_vnum_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -241,7 +241,7 @@ svuint32_t test_svldff1sb_vnum_u32(svbool_t pg, const int8_t *base, int64_t vnum
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 2 x i8> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldff1sb_vnum_u64u10__SVBool_tPKal(
+// CPP-CHECK-LABEL: @_Z23test_svldff1sb_vnum_u64u10__SVBool_tPKal(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -254,14 +254,14 @@ svuint64_t test_svldff1sb_vnum_u64(svbool_t pg, const int8_t *base, int64_t vnum
   return svldff1sb_vnum_u64(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sb_gather_u32base_s32(
+// CHECK-LABEL: @test_svldff1sb_gather_u32base_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i8.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svldff1sb_gather_u32base_s32u10__SVBool_tu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z33test_svldff1sb_gather_u32base_s32u10__SVBool_tu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i8.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
@@ -272,14 +272,14 @@ svint32_t test_svldff1sb_gather_u32base_s32(svbool_t pg, svuint32_t bases) {
   return SVE_ACLE_FUNC(svldff1sb_gather, _u32base, _s32, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sb_gather_u64base_s64(
+// CHECK-LABEL: @test_svldff1sb_gather_u64base_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i8.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svldff1sb_gather_u64base_s64u10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z33test_svldff1sb_gather_u64base_s64u10__SVBool_tu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i8.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
@@ -290,14 +290,14 @@ svint64_t test_svldff1sb_gather_u64base_s64(svbool_t pg, svuint64_t bases) {
   return SVE_ACLE_FUNC(svldff1sb_gather, _u64base, _s64, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sb_gather_u32base_u32(
+// CHECK-LABEL: @test_svldff1sb_gather_u32base_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i8.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svldff1sb_gather_u32base_u32u10__SVBool_tu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z33test_svldff1sb_gather_u32base_u32u10__SVBool_tu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i8.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
@@ -308,14 +308,14 @@ svuint32_t test_svldff1sb_gather_u32base_u32(svbool_t pg, svuint32_t bases) {
   return SVE_ACLE_FUNC(svldff1sb_gather, _u32base, _u32, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sb_gather_u64base_u64(
+// CHECK-LABEL: @test_svldff1sb_gather_u64base_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i8.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svldff1sb_gather_u64base_u64u10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z33test_svldff1sb_gather_u64base_u64u10__SVBool_tu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i8.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
@@ -326,14 +326,14 @@ svuint64_t test_svldff1sb_gather_u64base_u64(svbool_t pg, svuint64_t bases) {
   return SVE_ACLE_FUNC(svldff1sb_gather, _u64base, _u64, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sb_gather_s32offset_s32(
+// CHECK-LABEL: @test_svldff1sb_gather_s32offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.gather.sxtw.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1sb_gather_s32offset_s32u10__SVBool_tPKau11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1sb_gather_s32offset_s32u10__SVBool_tPKau11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.gather.sxtw.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -344,14 +344,14 @@ svint32_t test_svldff1sb_gather_s32offset_s32(svbool_t pg, const int8_t *base, s
   return SVE_ACLE_FUNC(svldff1sb_gather_, s32, offset_s32, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sb_gather_s64offset_s64(
+// CHECK-LABEL: @test_svldff1sb_gather_s64offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.gather.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1sb_gather_s64offset_s64u10__SVBool_tPKau11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1sb_gather_s64offset_s64u10__SVBool_tPKau11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.gather.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -362,14 +362,14 @@ svint64_t test_svldff1sb_gather_s64offset_s64(svbool_t pg, const int8_t *base, s
   return SVE_ACLE_FUNC(svldff1sb_gather_, s64, offset_s64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sb_gather_s32offset_u32(
+// CHECK-LABEL: @test_svldff1sb_gather_s32offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.gather.sxtw.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1sb_gather_s32offset_u32u10__SVBool_tPKau11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1sb_gather_s32offset_u32u10__SVBool_tPKau11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.gather.sxtw.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -380,14 +380,14 @@ svuint32_t test_svldff1sb_gather_s32offset_u32(svbool_t pg, const int8_t *base,
   return SVE_ACLE_FUNC(svldff1sb_gather_, s32, offset_u32, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sb_gather_s64offset_u64(
+// CHECK-LABEL: @test_svldff1sb_gather_s64offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.gather.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1sb_gather_s64offset_u64u10__SVBool_tPKau11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1sb_gather_s64offset_u64u10__SVBool_tPKau11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.gather.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -398,14 +398,14 @@ svuint64_t test_svldff1sb_gather_s64offset_u64(svbool_t pg, const int8_t *base,
   return SVE_ACLE_FUNC(svldff1sb_gather_, s64, offset_u64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sb_gather_u32offset_s32(
+// CHECK-LABEL: @test_svldff1sb_gather_u32offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.gather.uxtw.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1sb_gather_u32offset_s32u10__SVBool_tPKau12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1sb_gather_u32offset_s32u10__SVBool_tPKau12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.gather.uxtw.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -416,14 +416,14 @@ svint32_t test_svldff1sb_gather_u32offset_s32(svbool_t pg, const int8_t *base, s
   return SVE_ACLE_FUNC(svldff1sb_gather_, u32, offset_s32, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sb_gather_u64offset_s64(
+// CHECK-LABEL: @test_svldff1sb_gather_u64offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.gather.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1sb_gather_u64offset_s64u10__SVBool_tPKau12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1sb_gather_u64offset_s64u10__SVBool_tPKau12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.gather.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -434,14 +434,14 @@ svint64_t test_svldff1sb_gather_u64offset_s64(svbool_t pg, const int8_t *base, s
   return SVE_ACLE_FUNC(svldff1sb_gather_, u64, offset_s64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sb_gather_u32offset_u32(
+// CHECK-LABEL: @test_svldff1sb_gather_u32offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.gather.uxtw.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1sb_gather_u32offset_u32u10__SVBool_tPKau12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1sb_gather_u32offset_u32u10__SVBool_tPKau12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.gather.uxtw.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -452,14 +452,14 @@ svuint32_t test_svldff1sb_gather_u32offset_u32(svbool_t pg, const int8_t *base,
   return SVE_ACLE_FUNC(svldff1sb_gather_, u32, offset_u32, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sb_gather_u64offset_u64(
+// CHECK-LABEL: @test_svldff1sb_gather_u64offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.gather.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1sb_gather_u64offset_u64u10__SVBool_tPKau12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1sb_gather_u64offset_u64u10__SVBool_tPKau12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.gather.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -470,14 +470,14 @@ svuint64_t test_svldff1sb_gather_u64offset_u64(svbool_t pg, const int8_t *base,
   return SVE_ACLE_FUNC(svldff1sb_gather_, u64, offset_u64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sb_gather_u32base_offset_s32(
+// CHECK-LABEL: @test_svldff1sb_gather_u32base_offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i8.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z40test_svldff1sb_gather_u32base_offset_s32u10__SVBool_tu12__SVUint32_tl(
+// CPP-CHECK-LABEL: @_Z40test_svldff1sb_gather_u32base_offset_s32u10__SVBool_tu12__SVUint32_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i8.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -488,14 +488,14 @@ svint32_t test_svldff1sb_gather_u32base_offset_s32(svbool_t pg, svuint32_t bases
   return SVE_ACLE_FUNC(svldff1sb_gather, _u32base, _offset_s32, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sb_gather_u64base_offset_s64(
+// CHECK-LABEL: @test_svldff1sb_gather_u64base_offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i8.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z40test_svldff1sb_gather_u64base_offset_s64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z40test_svldff1sb_gather_u64base_offset_s64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i8.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -506,14 +506,14 @@ svint64_t test_svldff1sb_gather_u64base_offset_s64(svbool_t pg, svuint64_t bases
   return SVE_ACLE_FUNC(svldff1sb_gather, _u64base, _offset_s64, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sb_gather_u32base_offset_u32(
+// CHECK-LABEL: @test_svldff1sb_gather_u32base_offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i8.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z40test_svldff1sb_gather_u32base_offset_u32u10__SVBool_tu12__SVUint32_tl(
+// CPP-CHECK-LABEL: @_Z40test_svldff1sb_gather_u32base_offset_u32u10__SVBool_tu12__SVUint32_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i8.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -524,14 +524,14 @@ svuint32_t test_svldff1sb_gather_u32base_offset_u32(svbool_t pg, svuint32_t base
   return SVE_ACLE_FUNC(svldff1sb_gather, _u32base, _offset_u32, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sb_gather_u64base_offset_u64(
+// CHECK-LABEL: @test_svldff1sb_gather_u64base_offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i8.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z40test_svldff1sb_gather_u64base_offset_u64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z40test_svldff1sb_gather_u64base_offset_u64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i8.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1sh.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1sh.c
index ff5c3498c2..fd326def0a 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1sh.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1sh.c
@@ -14,14 +14,14 @@
 #define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sh_s32(
+// CHECK-LABEL: @test_svldff1sh_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldff1sh_s32u10__SVBool_tPKs(
+// CPP-CHECK-LABEL: @_Z18test_svldff1sh_s32u10__SVBool_tPKs(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -33,14 +33,14 @@ svint32_t test_svldff1sh_s32(svbool_t pg, const int16_t *base)
   return svldff1sh_s32(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sh_s64(
+// CHECK-LABEL: @test_svldff1sh_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldff1sh_s64u10__SVBool_tPKs(
+// CPP-CHECK-LABEL: @_Z18test_svldff1sh_s64u10__SVBool_tPKs(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -52,14 +52,14 @@ svint64_t test_svldff1sh_s64(svbool_t pg, const int16_t *base)
   return svldff1sh_s64(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sh_u32(
+// CHECK-LABEL: @test_svldff1sh_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldff1sh_u32u10__SVBool_tPKs(
+// CPP-CHECK-LABEL: @_Z18test_svldff1sh_u32u10__SVBool_tPKs(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -71,14 +71,14 @@ svuint32_t test_svldff1sh_u32(svbool_t pg, const int16_t *base)
   return svldff1sh_u32(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sh_u64(
+// CHECK-LABEL: @test_svldff1sh_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldff1sh_u64u10__SVBool_tPKs(
+// CPP-CHECK-LABEL: @_Z18test_svldff1sh_u64u10__SVBool_tPKs(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -90,7 +90,7 @@ svuint64_t test_svldff1sh_u64(svbool_t pg, const int16_t *base)
   return svldff1sh_u64(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sh_vnum_s32(
+// CHECK-LABEL: @test_svldff1sh_vnum_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -98,7 +98,7 @@ svuint64_t test_svldff1sh_u64(svbool_t pg, const int16_t *base)
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 4 x i16> [[TMP2]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldff1sh_vnum_s32u10__SVBool_tPKsl(
+// CPP-CHECK-LABEL: @_Z23test_svldff1sh_vnum_s32u10__SVBool_tPKsl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -111,7 +111,7 @@ svint32_t test_svldff1sh_vnum_s32(svbool_t pg, const int16_t *base, int64_t vnum
   return svldff1sh_vnum_s32(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sh_vnum_s64(
+// CHECK-LABEL: @test_svldff1sh_vnum_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -119,7 +119,7 @@ svint32_t test_svldff1sh_vnum_s32(svbool_t pg, const int16_t *base, int64_t vnum
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 2 x i16> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldff1sh_vnum_s64u10__SVBool_tPKsl(
+// CPP-CHECK-LABEL: @_Z23test_svldff1sh_vnum_s64u10__SVBool_tPKsl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -132,7 +132,7 @@ svint64_t test_svldff1sh_vnum_s64(svbool_t pg, const int16_t *base, int64_t vnum
   return svldff1sh_vnum_s64(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sh_vnum_u32(
+// CHECK-LABEL: @test_svldff1sh_vnum_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -140,7 +140,7 @@ svint64_t test_svldff1sh_vnum_s64(svbool_t pg, const int16_t *base, int64_t vnum
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 4 x i16> [[TMP2]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldff1sh_vnum_u32u10__SVBool_tPKsl(
+// CPP-CHECK-LABEL: @_Z23test_svldff1sh_vnum_u32u10__SVBool_tPKsl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -153,7 +153,7 @@ svuint32_t test_svldff1sh_vnum_u32(svbool_t pg, const int16_t *base, int64_t vnu
   return svldff1sh_vnum_u32(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sh_vnum_u64(
+// CHECK-LABEL: @test_svldff1sh_vnum_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -161,7 +161,7 @@ svuint32_t test_svldff1sh_vnum_u32(svbool_t pg, const int16_t *base, int64_t vnu
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 2 x i16> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldff1sh_vnum_u64u10__SVBool_tPKsl(
+// CPP-CHECK-LABEL: @_Z23test_svldff1sh_vnum_u64u10__SVBool_tPKsl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -174,14 +174,14 @@ svuint64_t test_svldff1sh_vnum_u64(svbool_t pg, const int16_t *base, int64_t vnu
   return svldff1sh_vnum_u64(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sh_gather_u32base_s32(
+// CHECK-LABEL: @test_svldff1sh_gather_u32base_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i16.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svldff1sh_gather_u32base_s32u10__SVBool_tu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z33test_svldff1sh_gather_u32base_s32u10__SVBool_tu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i16.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
@@ -192,14 +192,14 @@ svint32_t test_svldff1sh_gather_u32base_s32(svbool_t pg, svuint32_t bases) {
   return SVE_ACLE_FUNC(svldff1sh_gather, _u32base, _s32, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sh_gather_u64base_s64(
+// CHECK-LABEL: @test_svldff1sh_gather_u64base_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i16.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svldff1sh_gather_u64base_s64u10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z33test_svldff1sh_gather_u64base_s64u10__SVBool_tu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i16.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
@@ -210,14 +210,14 @@ svint64_t test_svldff1sh_gather_u64base_s64(svbool_t pg, svuint64_t bases) {
   return SVE_ACLE_FUNC(svldff1sh_gather, _u64base, _s64, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sh_gather_u32base_u32(
+// CHECK-LABEL: @test_svldff1sh_gather_u32base_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i16.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svldff1sh_gather_u32base_u32u10__SVBool_tu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z33test_svldff1sh_gather_u32base_u32u10__SVBool_tu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i16.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
@@ -228,14 +228,14 @@ svuint32_t test_svldff1sh_gather_u32base_u32(svbool_t pg, svuint32_t bases) {
   return SVE_ACLE_FUNC(svldff1sh_gather, _u32base, _u32, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sh_gather_u64base_u64(
+// CHECK-LABEL: @test_svldff1sh_gather_u64base_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i16.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svldff1sh_gather_u64base_u64u10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z33test_svldff1sh_gather_u64base_u64u10__SVBool_tu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i16.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
@@ -246,14 +246,14 @@ svuint64_t test_svldff1sh_gather_u64base_u64(svbool_t pg, svuint64_t bases) {
   return SVE_ACLE_FUNC(svldff1sh_gather, _u64base, _u64, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sh_gather_s32offset_s32(
+// CHECK-LABEL: @test_svldff1sh_gather_s32offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.sxtw.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1sh_gather_s32offset_s32u10__SVBool_tPKsu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1sh_gather_s32offset_s32u10__SVBool_tPKsu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.sxtw.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -264,14 +264,14 @@ svint32_t test_svldff1sh_gather_s32offset_s32(svbool_t pg, const int16_t *base,
   return SVE_ACLE_FUNC(svldff1sh_gather_, s32, offset_s32, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sh_gather_s64offset_s64(
+// CHECK-LABEL: @test_svldff1sh_gather_s64offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1sh_gather_s64offset_s64u10__SVBool_tPKsu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1sh_gather_s64offset_s64u10__SVBool_tPKsu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -282,14 +282,14 @@ svint64_t test_svldff1sh_gather_s64offset_s64(svbool_t pg, const int16_t *base,
   return SVE_ACLE_FUNC(svldff1sh_gather_, s64, offset_s64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sh_gather_s32offset_u32(
+// CHECK-LABEL: @test_svldff1sh_gather_s32offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.sxtw.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1sh_gather_s32offset_u32u10__SVBool_tPKsu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1sh_gather_s32offset_u32u10__SVBool_tPKsu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.sxtw.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -300,14 +300,14 @@ svuint32_t test_svldff1sh_gather_s32offset_u32(svbool_t pg, const int16_t *base,
   return SVE_ACLE_FUNC(svldff1sh_gather_, s32, offset_u32, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sh_gather_s64offset_u64(
+// CHECK-LABEL: @test_svldff1sh_gather_s64offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1sh_gather_s64offset_u64u10__SVBool_tPKsu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1sh_gather_s64offset_u64u10__SVBool_tPKsu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -318,14 +318,14 @@ svuint64_t test_svldff1sh_gather_s64offset_u64(svbool_t pg, const int16_t *base,
   return SVE_ACLE_FUNC(svldff1sh_gather_, s64, offset_u64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sh_gather_u32offset_s32(
+// CHECK-LABEL: @test_svldff1sh_gather_u32offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.uxtw.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1sh_gather_u32offset_s32u10__SVBool_tPKsu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1sh_gather_u32offset_s32u10__SVBool_tPKsu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.uxtw.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -336,14 +336,14 @@ svint32_t test_svldff1sh_gather_u32offset_s32(svbool_t pg, const int16_t *base,
   return SVE_ACLE_FUNC(svldff1sh_gather_, u32, offset_s32, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sh_gather_u64offset_s64(
+// CHECK-LABEL: @test_svldff1sh_gather_u64offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1sh_gather_u64offset_s64u10__SVBool_tPKsu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1sh_gather_u64offset_s64u10__SVBool_tPKsu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -354,14 +354,14 @@ svint64_t test_svldff1sh_gather_u64offset_s64(svbool_t pg, const int16_t *base,
   return SVE_ACLE_FUNC(svldff1sh_gather_, u64, offset_s64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sh_gather_u32offset_u32(
+// CHECK-LABEL: @test_svldff1sh_gather_u32offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.uxtw.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1sh_gather_u32offset_u32u10__SVBool_tPKsu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1sh_gather_u32offset_u32u10__SVBool_tPKsu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.uxtw.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -372,14 +372,14 @@ svuint32_t test_svldff1sh_gather_u32offset_u32(svbool_t pg, const int16_t *base,
   return SVE_ACLE_FUNC(svldff1sh_gather_, u32, offset_u32, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sh_gather_u64offset_u64(
+// CHECK-LABEL: @test_svldff1sh_gather_u64offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1sh_gather_u64offset_u64u10__SVBool_tPKsu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1sh_gather_u64offset_u64u10__SVBool_tPKsu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -390,14 +390,14 @@ svuint64_t test_svldff1sh_gather_u64offset_u64(svbool_t pg, const int16_t *base,
   return SVE_ACLE_FUNC(svldff1sh_gather_, u64, offset_u64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sh_gather_u32base_offset_s32(
+// CHECK-LABEL: @test_svldff1sh_gather_u32base_offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i16.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z40test_svldff1sh_gather_u32base_offset_s32u10__SVBool_tu12__SVUint32_tl(
+// CPP-CHECK-LABEL: @_Z40test_svldff1sh_gather_u32base_offset_s32u10__SVBool_tu12__SVUint32_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i16.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -408,14 +408,14 @@ svint32_t test_svldff1sh_gather_u32base_offset_s32(svbool_t pg, svuint32_t bases
   return SVE_ACLE_FUNC(svldff1sh_gather, _u32base, _offset_s32, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sh_gather_u64base_offset_s64(
+// CHECK-LABEL: @test_svldff1sh_gather_u64base_offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i16.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z40test_svldff1sh_gather_u64base_offset_s64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z40test_svldff1sh_gather_u64base_offset_s64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i16.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -426,14 +426,14 @@ svint64_t test_svldff1sh_gather_u64base_offset_s64(svbool_t pg, svuint64_t bases
   return SVE_ACLE_FUNC(svldff1sh_gather, _u64base, _offset_s64, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sh_gather_u32base_offset_u32(
+// CHECK-LABEL: @test_svldff1sh_gather_u32base_offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i16.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z40test_svldff1sh_gather_u32base_offset_u32u10__SVBool_tu12__SVUint32_tl(
+// CPP-CHECK-LABEL: @_Z40test_svldff1sh_gather_u32base_offset_u32u10__SVBool_tu12__SVUint32_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i16.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -444,14 +444,14 @@ svuint32_t test_svldff1sh_gather_u32base_offset_u32(svbool_t pg, svuint32_t base
   return SVE_ACLE_FUNC(svldff1sh_gather, _u32base, _offset_u32, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sh_gather_u64base_offset_u64(
+// CHECK-LABEL: @test_svldff1sh_gather_u64base_offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i16.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z40test_svldff1sh_gather_u64base_offset_u64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z40test_svldff1sh_gather_u64base_offset_u64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i16.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -462,14 +462,14 @@ svuint64_t test_svldff1sh_gather_u64base_offset_u64(svbool_t pg, svuint64_t base
   return SVE_ACLE_FUNC(svldff1sh_gather, _u64base, _offset_u64, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sh_gather_s32index_s32(
+// CHECK-LABEL: @test_svldff1sh_gather_s32index_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.sxtw.index.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z34test_svldff1sh_gather_s32index_s32u10__SVBool_tPKsu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z34test_svldff1sh_gather_s32index_s32u10__SVBool_tPKsu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.sxtw.index.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
@@ -480,14 +480,14 @@ svint32_t test_svldff1sh_gather_s32index_s32(svbool_t pg, const int16_t *base, s
   return SVE_ACLE_FUNC(svldff1sh_gather_, s32, index_s32, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sh_gather_s64index_s64(
+// CHECK-LABEL: @test_svldff1sh_gather_s64index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.index.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z34test_svldff1sh_gather_s64index_s64u10__SVBool_tPKsu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z34test_svldff1sh_gather_s64index_s64u10__SVBool_tPKsu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.index.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -498,14 +498,14 @@ svint64_t test_svldff1sh_gather_s64index_s64(svbool_t pg, const int16_t *base, s
   return SVE_ACLE_FUNC(svldff1sh_gather_, s64, index_s64, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sh_gather_s32index_u32(
+// CHECK-LABEL: @test_svldff1sh_gather_s32index_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.sxtw.index.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z34test_svldff1sh_gather_s32index_u32u10__SVBool_tPKsu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z34test_svldff1sh_gather_s32index_u32u10__SVBool_tPKsu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.sxtw.index.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
@@ -516,14 +516,14 @@ svuint32_t test_svldff1sh_gather_s32index_u32(svbool_t pg, const int16_t *base,
   return SVE_ACLE_FUNC(svldff1sh_gather_, s32, index_u32, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sh_gather_s64index_u64(
+// CHECK-LABEL: @test_svldff1sh_gather_s64index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.index.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z34test_svldff1sh_gather_s64index_u64u10__SVBool_tPKsu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z34test_svldff1sh_gather_s64index_u64u10__SVBool_tPKsu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.index.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -534,14 +534,14 @@ svuint64_t test_svldff1sh_gather_s64index_u64(svbool_t pg, const int16_t *base,
   return SVE_ACLE_FUNC(svldff1sh_gather_, s64, index_u64, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sh_gather_u32index_s32(
+// CHECK-LABEL: @test_svldff1sh_gather_u32index_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.uxtw.index.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z34test_svldff1sh_gather_u32index_s32u10__SVBool_tPKsu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z34test_svldff1sh_gather_u32index_s32u10__SVBool_tPKsu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.uxtw.index.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
@@ -552,14 +552,14 @@ svint32_t test_svldff1sh_gather_u32index_s32(svbool_t pg, const int16_t *base, s
   return SVE_ACLE_FUNC(svldff1sh_gather_, u32, index_s32, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sh_gather_u64index_s64(
+// CHECK-LABEL: @test_svldff1sh_gather_u64index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.index.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z34test_svldff1sh_gather_u64index_s64u10__SVBool_tPKsu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z34test_svldff1sh_gather_u64index_s64u10__SVBool_tPKsu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.index.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -570,14 +570,14 @@ svint64_t test_svldff1sh_gather_u64index_s64(svbool_t pg, const int16_t *base, s
   return SVE_ACLE_FUNC(svldff1sh_gather_, u64, index_s64, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sh_gather_u32index_u32(
+// CHECK-LABEL: @test_svldff1sh_gather_u32index_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.uxtw.index.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z34test_svldff1sh_gather_u32index_u32u10__SVBool_tPKsu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z34test_svldff1sh_gather_u32index_u32u10__SVBool_tPKsu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.uxtw.index.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
@@ -588,14 +588,14 @@ svuint32_t test_svldff1sh_gather_u32index_u32(svbool_t pg, const int16_t *base,
   return SVE_ACLE_FUNC(svldff1sh_gather_, u32, index_u32, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sh_gather_u64index_u64(
+// CHECK-LABEL: @test_svldff1sh_gather_u64index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.index.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z34test_svldff1sh_gather_u64index_u64u10__SVBool_tPKsu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z34test_svldff1sh_gather_u64index_u64u10__SVBool_tPKsu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.index.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -606,7 +606,7 @@ svuint64_t test_svldff1sh_gather_u64index_u64(svbool_t pg, const int16_t *base,
   return SVE_ACLE_FUNC(svldff1sh_gather_, u64, index_u64, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sh_gather_u32base_index_s32(
+// CHECK-LABEL: @test_svldff1sh_gather_u32base_index_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 1
@@ -614,7 +614,7 @@ svuint64_t test_svldff1sh_gather_u64index_u64(svbool_t pg, const int16_t *base,
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 4 x i16> [[TMP2]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z39test_svldff1sh_gather_u32base_index_s32u10__SVBool_tu12__SVUint32_tl(
+// CPP-CHECK-LABEL: @_Z39test_svldff1sh_gather_u32base_index_s32u10__SVBool_tu12__SVUint32_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 1
@@ -626,7 +626,7 @@ svint32_t test_svldff1sh_gather_u32base_index_s32(svbool_t pg, svuint32_t bases,
   return SVE_ACLE_FUNC(svldff1sh_gather, _u32base, _index_s32, )(pg, bases, index);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sh_gather_u64base_index_s64(
+// CHECK-LABEL: @test_svldff1sh_gather_u64base_index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 1
@@ -634,7 +634,7 @@ svint32_t test_svldff1sh_gather_u32base_index_s32(svbool_t pg, svuint32_t bases,
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 2 x i16> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z39test_svldff1sh_gather_u64base_index_s64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z39test_svldff1sh_gather_u64base_index_s64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 1
@@ -646,7 +646,7 @@ svint64_t test_svldff1sh_gather_u64base_index_s64(svbool_t pg, svuint64_t bases,
   return SVE_ACLE_FUNC(svldff1sh_gather, _u64base, _index_s64, )(pg, bases, index);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sh_gather_u32base_index_u32(
+// CHECK-LABEL: @test_svldff1sh_gather_u32base_index_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 1
@@ -654,7 +654,7 @@ svint64_t test_svldff1sh_gather_u64base_index_s64(svbool_t pg, svuint64_t bases,
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 4 x i16> [[TMP2]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z39test_svldff1sh_gather_u32base_index_u32u10__SVBool_tu12__SVUint32_tl(
+// CPP-CHECK-LABEL: @_Z39test_svldff1sh_gather_u32base_index_u32u10__SVBool_tu12__SVUint32_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 1
@@ -666,7 +666,7 @@ svuint32_t test_svldff1sh_gather_u32base_index_u32(svbool_t pg, svuint32_t bases
   return SVE_ACLE_FUNC(svldff1sh_gather, _u32base, _index_u32, )(pg, bases, index);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sh_gather_u64base_index_u64(
+// CHECK-LABEL: @test_svldff1sh_gather_u64base_index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 1
@@ -674,7 +674,7 @@ svuint32_t test_svldff1sh_gather_u32base_index_u32(svbool_t pg, svuint32_t bases
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 2 x i16> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z39test_svldff1sh_gather_u64base_index_u64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z39test_svldff1sh_gather_u64base_index_u64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 1
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1sw.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1sw.c
index 4d445b0674..848be449bf 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1sw.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1sw.c
@@ -14,14 +14,14 @@
 #define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sw_s64(
+// CHECK-LABEL: @test_svldff1sw_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldff1sw_s64u10__SVBool_tPKi(
+// CPP-CHECK-LABEL: @_Z18test_svldff1sw_s64u10__SVBool_tPKi(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -33,14 +33,14 @@ svint64_t test_svldff1sw_s64(svbool_t pg, const int32_t *base)
   return svldff1sw_s64(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sw_u64(
+// CHECK-LABEL: @test_svldff1sw_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldff1sw_u64u10__SVBool_tPKi(
+// CPP-CHECK-LABEL: @_Z18test_svldff1sw_u64u10__SVBool_tPKi(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -52,7 +52,7 @@ svuint64_t test_svldff1sw_u64(svbool_t pg, const int32_t *base)
   return svldff1sw_u64(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sw_vnum_s64(
+// CHECK-LABEL: @test_svldff1sw_vnum_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -60,7 +60,7 @@ svuint64_t test_svldff1sw_u64(svbool_t pg, const int32_t *base)
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 2 x i32> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldff1sw_vnum_s64u10__SVBool_tPKil(
+// CPP-CHECK-LABEL: @_Z23test_svldff1sw_vnum_s64u10__SVBool_tPKil(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -73,7 +73,7 @@ svint64_t test_svldff1sw_vnum_s64(svbool_t pg, const int32_t *base, int64_t vnum
   return svldff1sw_vnum_s64(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sw_vnum_u64(
+// CHECK-LABEL: @test_svldff1sw_vnum_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -81,7 +81,7 @@ svint64_t test_svldff1sw_vnum_s64(svbool_t pg, const int32_t *base, int64_t vnum
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 2 x i32> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldff1sw_vnum_u64u10__SVBool_tPKil(
+// CPP-CHECK-LABEL: @_Z23test_svldff1sw_vnum_u64u10__SVBool_tPKil(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -94,14 +94,14 @@ svuint64_t test_svldff1sw_vnum_u64(svbool_t pg, const int32_t *base, int64_t vnu
   return svldff1sw_vnum_u64(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sw_gather_u64base_s64(
+// CHECK-LABEL: @test_svldff1sw_gather_u64base_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i32.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svldff1sw_gather_u64base_s64u10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z33test_svldff1sw_gather_u64base_s64u10__SVBool_tu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i32.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
@@ -112,14 +112,14 @@ svint64_t test_svldff1sw_gather_u64base_s64(svbool_t pg, svuint64_t bases) {
   return SVE_ACLE_FUNC(svldff1sw_gather, _u64base, _s64, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sw_gather_u64base_u64(
+// CHECK-LABEL: @test_svldff1sw_gather_u64base_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i32.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svldff1sw_gather_u64base_u64u10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z33test_svldff1sw_gather_u64base_u64u10__SVBool_tu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i32.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
@@ -130,14 +130,14 @@ svuint64_t test_svldff1sw_gather_u64base_u64(svbool_t pg, svuint64_t bases) {
   return SVE_ACLE_FUNC(svldff1sw_gather, _u64base, _u64, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sw_gather_s64offset_s64(
+// CHECK-LABEL: @test_svldff1sw_gather_s64offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1sw_gather_s64offset_s64u10__SVBool_tPKiu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1sw_gather_s64offset_s64u10__SVBool_tPKiu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -148,14 +148,14 @@ svint64_t test_svldff1sw_gather_s64offset_s64(svbool_t pg, const int32_t *base,
   return SVE_ACLE_FUNC(svldff1sw_gather_, s64, offset_s64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sw_gather_s64offset_u64(
+// CHECK-LABEL: @test_svldff1sw_gather_s64offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1sw_gather_s64offset_u64u10__SVBool_tPKiu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1sw_gather_s64offset_u64u10__SVBool_tPKiu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -166,14 +166,14 @@ svuint64_t test_svldff1sw_gather_s64offset_u64(svbool_t pg, const int32_t *base,
   return SVE_ACLE_FUNC(svldff1sw_gather_, s64, offset_u64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sw_gather_u64offset_s64(
+// CHECK-LABEL: @test_svldff1sw_gather_u64offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1sw_gather_u64offset_s64u10__SVBool_tPKiu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1sw_gather_u64offset_s64u10__SVBool_tPKiu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -184,14 +184,14 @@ svint64_t test_svldff1sw_gather_u64offset_s64(svbool_t pg, const int32_t *base,
   return SVE_ACLE_FUNC(svldff1sw_gather_, u64, offset_s64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sw_gather_u64offset_u64(
+// CHECK-LABEL: @test_svldff1sw_gather_u64offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1sw_gather_u64offset_u64u10__SVBool_tPKiu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1sw_gather_u64offset_u64u10__SVBool_tPKiu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -202,14 +202,14 @@ svuint64_t test_svldff1sw_gather_u64offset_u64(svbool_t pg, const int32_t *base,
   return SVE_ACLE_FUNC(svldff1sw_gather_, u64, offset_u64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sw_gather_u64base_offset_s64(
+// CHECK-LABEL: @test_svldff1sw_gather_u64base_offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i32.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z40test_svldff1sw_gather_u64base_offset_s64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z40test_svldff1sw_gather_u64base_offset_s64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i32.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -220,14 +220,14 @@ svint64_t test_svldff1sw_gather_u64base_offset_s64(svbool_t pg, svuint64_t bases
   return SVE_ACLE_FUNC(svldff1sw_gather, _u64base, _offset_s64, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sw_gather_u64base_offset_u64(
+// CHECK-LABEL: @test_svldff1sw_gather_u64base_offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i32.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z40test_svldff1sw_gather_u64base_offset_u64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z40test_svldff1sw_gather_u64base_offset_u64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i32.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -238,14 +238,14 @@ svuint64_t test_svldff1sw_gather_u64base_offset_u64(svbool_t pg, svuint64_t base
   return SVE_ACLE_FUNC(svldff1sw_gather, _u64base, _offset_u64, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sw_gather_s64index_s64(
+// CHECK-LABEL: @test_svldff1sw_gather_s64index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.index.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z34test_svldff1sw_gather_s64index_s64u10__SVBool_tPKiu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z34test_svldff1sw_gather_s64index_s64u10__SVBool_tPKiu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.index.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -256,14 +256,14 @@ svint64_t test_svldff1sw_gather_s64index_s64(svbool_t pg, const int32_t *base, s
   return SVE_ACLE_FUNC(svldff1sw_gather_, s64, index_s64, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sw_gather_s64index_u64(
+// CHECK-LABEL: @test_svldff1sw_gather_s64index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.index.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z34test_svldff1sw_gather_s64index_u64u10__SVBool_tPKiu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z34test_svldff1sw_gather_s64index_u64u10__SVBool_tPKiu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.index.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -274,14 +274,14 @@ svuint64_t test_svldff1sw_gather_s64index_u64(svbool_t pg, const int32_t *base,
   return SVE_ACLE_FUNC(svldff1sw_gather_, s64, index_u64, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sw_gather_u64index_s64(
+// CHECK-LABEL: @test_svldff1sw_gather_u64index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.index.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z34test_svldff1sw_gather_u64index_s64u10__SVBool_tPKiu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z34test_svldff1sw_gather_u64index_s64u10__SVBool_tPKiu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.index.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -292,14 +292,14 @@ svint64_t test_svldff1sw_gather_u64index_s64(svbool_t pg, const int32_t *base, s
   return SVE_ACLE_FUNC(svldff1sw_gather_, u64, index_s64, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sw_gather_u64index_u64(
+// CHECK-LABEL: @test_svldff1sw_gather_u64index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.index.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z34test_svldff1sw_gather_u64index_u64u10__SVBool_tPKiu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z34test_svldff1sw_gather_u64index_u64u10__SVBool_tPKiu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.index.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -310,7 +310,7 @@ svuint64_t test_svldff1sw_gather_u64index_u64(svbool_t pg, const int32_t *base,
   return SVE_ACLE_FUNC(svldff1sw_gather_, u64, index_u64, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sw_gather_u64base_index_s64(
+// CHECK-LABEL: @test_svldff1sw_gather_u64base_index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 2
@@ -318,7 +318,7 @@ svuint64_t test_svldff1sw_gather_u64index_u64(svbool_t pg, const int32_t *base,
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 2 x i32> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z39test_svldff1sw_gather_u64base_index_s64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z39test_svldff1sw_gather_u64base_index_s64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 2
@@ -330,7 +330,7 @@ svint64_t test_svldff1sw_gather_u64base_index_s64(svbool_t pg, svuint64_t bases,
   return SVE_ACLE_FUNC(svldff1sw_gather, _u64base, _index_s64, )(pg, bases, index);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1sw_gather_u64base_index_u64(
+// CHECK-LABEL: @test_svldff1sw_gather_u64base_index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 2
@@ -338,7 +338,7 @@ svint64_t test_svldff1sw_gather_u64base_index_s64(svbool_t pg, svuint64_t bases,
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 2 x i32> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z39test_svldff1sw_gather_u64base_index_u64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z39test_svldff1sw_gather_u64base_index_u64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 2
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1ub.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1ub.c
index 695bf20c85..3328397357 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1ub.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1ub.c
@@ -14,14 +14,14 @@
 #define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1ub_s16(
+// CHECK-LABEL: @test_svldff1ub_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i8> @llvm.aarch64.sve.ldff1.nxv8i8(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 8 x i8> [[TMP1]] to <vscale x 8 x i16>
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldff1ub_s16u10__SVBool_tPKh(
+// CPP-CHECK-LABEL: @_Z18test_svldff1ub_s16u10__SVBool_tPKh(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i8> @llvm.aarch64.sve.ldff1.nxv8i8(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -33,14 +33,14 @@ svint16_t test_svldff1ub_s16(svbool_t pg, const uint8_t *base)
   return svldff1ub_s16(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1ub_s32(
+// CHECK-LABEL: @test_svldff1ub_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldff1ub_s32u10__SVBool_tPKh(
+// CPP-CHECK-LABEL: @_Z18test_svldff1ub_s32u10__SVBool_tPKh(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -52,14 +52,14 @@ svint32_t test_svldff1ub_s32(svbool_t pg, const uint8_t *base)
   return svldff1ub_s32(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1ub_s64(
+// CHECK-LABEL: @test_svldff1ub_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldff1ub_s64u10__SVBool_tPKh(
+// CPP-CHECK-LABEL: @_Z18test_svldff1ub_s64u10__SVBool_tPKh(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -71,14 +71,14 @@ svint64_t test_svldff1ub_s64(svbool_t pg, const uint8_t *base)
   return svldff1ub_s64(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1ub_u16(
+// CHECK-LABEL: @test_svldff1ub_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i8> @llvm.aarch64.sve.ldff1.nxv8i8(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 8 x i8> [[TMP1]] to <vscale x 8 x i16>
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldff1ub_u16u10__SVBool_tPKh(
+// CPP-CHECK-LABEL: @_Z18test_svldff1ub_u16u10__SVBool_tPKh(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i8> @llvm.aarch64.sve.ldff1.nxv8i8(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -90,14 +90,14 @@ svuint16_t test_svldff1ub_u16(svbool_t pg, const uint8_t *base)
   return svldff1ub_u16(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1ub_u32(
+// CHECK-LABEL: @test_svldff1ub_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldff1ub_u32u10__SVBool_tPKh(
+// CPP-CHECK-LABEL: @_Z18test_svldff1ub_u32u10__SVBool_tPKh(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -109,14 +109,14 @@ svuint32_t test_svldff1ub_u32(svbool_t pg, const uint8_t *base)
   return svldff1ub_u32(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1ub_u64(
+// CHECK-LABEL: @test_svldff1ub_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldff1ub_u64u10__SVBool_tPKh(
+// CPP-CHECK-LABEL: @_Z18test_svldff1ub_u64u10__SVBool_tPKh(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -128,7 +128,7 @@ svuint64_t test_svldff1ub_u64(svbool_t pg, const uint8_t *base)
   return svldff1ub_u64(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1ub_vnum_s16(
+// CHECK-LABEL: @test_svldff1ub_vnum_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -136,7 +136,7 @@ svuint64_t test_svldff1ub_u64(svbool_t pg, const uint8_t *base)
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 8 x i8> [[TMP2]] to <vscale x 8 x i16>
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldff1ub_vnum_s16u10__SVBool_tPKhl(
+// CPP-CHECK-LABEL: @_Z23test_svldff1ub_vnum_s16u10__SVBool_tPKhl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -149,7 +149,7 @@ svint16_t test_svldff1ub_vnum_s16(svbool_t pg, const uint8_t *base, int64_t vnum
   return svldff1ub_vnum_s16(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1ub_vnum_s32(
+// CHECK-LABEL: @test_svldff1ub_vnum_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -157,7 +157,7 @@ svint16_t test_svldff1ub_vnum_s16(svbool_t pg, const uint8_t *base, int64_t vnum
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 4 x i8> [[TMP2]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldff1ub_vnum_s32u10__SVBool_tPKhl(
+// CPP-CHECK-LABEL: @_Z23test_svldff1ub_vnum_s32u10__SVBool_tPKhl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -170,7 +170,7 @@ svint32_t test_svldff1ub_vnum_s32(svbool_t pg, const uint8_t *base, int64_t vnum
   return svldff1ub_vnum_s32(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1ub_vnum_s64(
+// CHECK-LABEL: @test_svldff1ub_vnum_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -178,7 +178,7 @@ svint32_t test_svldff1ub_vnum_s32(svbool_t pg, const uint8_t *base, int64_t vnum
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 2 x i8> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldff1ub_vnum_s64u10__SVBool_tPKhl(
+// CPP-CHECK-LABEL: @_Z23test_svldff1ub_vnum_s64u10__SVBool_tPKhl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -191,7 +191,7 @@ svint64_t test_svldff1ub_vnum_s64(svbool_t pg, const uint8_t *base, int64_t vnum
   return svldff1ub_vnum_s64(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1ub_vnum_u16(
+// CHECK-LABEL: @test_svldff1ub_vnum_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -199,7 +199,7 @@ svint64_t test_svldff1ub_vnum_s64(svbool_t pg, const uint8_t *base, int64_t vnum
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 8 x i8> [[TMP2]] to <vscale x 8 x i16>
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldff1ub_vnum_u16u10__SVBool_tPKhl(
+// CPP-CHECK-LABEL: @_Z23test_svldff1ub_vnum_u16u10__SVBool_tPKhl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -212,7 +212,7 @@ svuint16_t test_svldff1ub_vnum_u16(svbool_t pg, const uint8_t *base, int64_t vnu
   return svldff1ub_vnum_u16(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1ub_vnum_u32(
+// CHECK-LABEL: @test_svldff1ub_vnum_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -220,7 +220,7 @@ svuint16_t test_svldff1ub_vnum_u16(svbool_t pg, const uint8_t *base, int64_t vnu
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 4 x i8> [[TMP2]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldff1ub_vnum_u32u10__SVBool_tPKhl(
+// CPP-CHECK-LABEL: @_Z23test_svldff1ub_vnum_u32u10__SVBool_tPKhl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -233,7 +233,7 @@ svuint32_t test_svldff1ub_vnum_u32(svbool_t pg, const uint8_t *base, int64_t vnu
   return svldff1ub_vnum_u32(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1ub_vnum_u64(
+// CHECK-LABEL: @test_svldff1ub_vnum_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -241,7 +241,7 @@ svuint32_t test_svldff1ub_vnum_u32(svbool_t pg, const uint8_t *base, int64_t vnu
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 2 x i8> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldff1ub_vnum_u64u10__SVBool_tPKhl(
+// CPP-CHECK-LABEL: @_Z23test_svldff1ub_vnum_u64u10__SVBool_tPKhl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -254,14 +254,14 @@ svuint64_t test_svldff1ub_vnum_u64(svbool_t pg, const uint8_t *base, int64_t vnu
   return svldff1ub_vnum_u64(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1ub_gather_u32base_s32(
+// CHECK-LABEL: @test_svldff1ub_gather_u32base_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i8.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svldff1ub_gather_u32base_s32u10__SVBool_tu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z33test_svldff1ub_gather_u32base_s32u10__SVBool_tu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i8.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
@@ -272,14 +272,14 @@ svint32_t test_svldff1ub_gather_u32base_s32(svbool_t pg, svuint32_t bases) {
   return SVE_ACLE_FUNC(svldff1ub_gather, _u32base, _s32, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1ub_gather_u64base_s64(
+// CHECK-LABEL: @test_svldff1ub_gather_u64base_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i8.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svldff1ub_gather_u64base_s64u10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z33test_svldff1ub_gather_u64base_s64u10__SVBool_tu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i8.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
@@ -290,14 +290,14 @@ svint64_t test_svldff1ub_gather_u64base_s64(svbool_t pg, svuint64_t bases) {
   return SVE_ACLE_FUNC(svldff1ub_gather, _u64base, _s64, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1ub_gather_u32base_u32(
+// CHECK-LABEL: @test_svldff1ub_gather_u32base_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i8.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svldff1ub_gather_u32base_u32u10__SVBool_tu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z33test_svldff1ub_gather_u32base_u32u10__SVBool_tu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i8.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
@@ -308,14 +308,14 @@ svuint32_t test_svldff1ub_gather_u32base_u32(svbool_t pg, svuint32_t bases) {
   return SVE_ACLE_FUNC(svldff1ub_gather, _u32base, _u32, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1ub_gather_u64base_u64(
+// CHECK-LABEL: @test_svldff1ub_gather_u64base_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i8.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svldff1ub_gather_u64base_u64u10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z33test_svldff1ub_gather_u64base_u64u10__SVBool_tu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i8.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
@@ -326,14 +326,14 @@ svuint64_t test_svldff1ub_gather_u64base_u64(svbool_t pg, svuint64_t bases) {
   return SVE_ACLE_FUNC(svldff1ub_gather, _u64base, _u64, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1ub_gather_s32offset_s32(
+// CHECK-LABEL: @test_svldff1ub_gather_s32offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.gather.sxtw.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1ub_gather_s32offset_s32u10__SVBool_tPKhu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1ub_gather_s32offset_s32u10__SVBool_tPKhu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.gather.sxtw.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -344,14 +344,14 @@ svint32_t test_svldff1ub_gather_s32offset_s32(svbool_t pg, const uint8_t *base,
   return SVE_ACLE_FUNC(svldff1ub_gather_, s32, offset_s32, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1ub_gather_s64offset_s64(
+// CHECK-LABEL: @test_svldff1ub_gather_s64offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.gather.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1ub_gather_s64offset_s64u10__SVBool_tPKhu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1ub_gather_s64offset_s64u10__SVBool_tPKhu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.gather.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -362,14 +362,14 @@ svint64_t test_svldff1ub_gather_s64offset_s64(svbool_t pg, const uint8_t *base,
   return SVE_ACLE_FUNC(svldff1ub_gather_, s64, offset_s64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1ub_gather_s32offset_u32(
+// CHECK-LABEL: @test_svldff1ub_gather_s32offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.gather.sxtw.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1ub_gather_s32offset_u32u10__SVBool_tPKhu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1ub_gather_s32offset_u32u10__SVBool_tPKhu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.gather.sxtw.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -380,14 +380,14 @@ svuint32_t test_svldff1ub_gather_s32offset_u32(svbool_t pg, const uint8_t *base,
   return SVE_ACLE_FUNC(svldff1ub_gather_, s32, offset_u32, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1ub_gather_s64offset_u64(
+// CHECK-LABEL: @test_svldff1ub_gather_s64offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.gather.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1ub_gather_s64offset_u64u10__SVBool_tPKhu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1ub_gather_s64offset_u64u10__SVBool_tPKhu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.gather.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -398,14 +398,14 @@ svuint64_t test_svldff1ub_gather_s64offset_u64(svbool_t pg, const uint8_t *base,
   return SVE_ACLE_FUNC(svldff1ub_gather_, s64, offset_u64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1ub_gather_u32offset_s32(
+// CHECK-LABEL: @test_svldff1ub_gather_u32offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.gather.uxtw.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1ub_gather_u32offset_s32u10__SVBool_tPKhu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1ub_gather_u32offset_s32u10__SVBool_tPKhu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.gather.uxtw.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -416,14 +416,14 @@ svint32_t test_svldff1ub_gather_u32offset_s32(svbool_t pg, const uint8_t *base,
   return SVE_ACLE_FUNC(svldff1ub_gather_, u32, offset_s32, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1ub_gather_u64offset_s64(
+// CHECK-LABEL: @test_svldff1ub_gather_u64offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.gather.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1ub_gather_u64offset_s64u10__SVBool_tPKhu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1ub_gather_u64offset_s64u10__SVBool_tPKhu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.gather.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -434,14 +434,14 @@ svint64_t test_svldff1ub_gather_u64offset_s64(svbool_t pg, const uint8_t *base,
   return SVE_ACLE_FUNC(svldff1ub_gather_, u64, offset_s64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1ub_gather_u32offset_u32(
+// CHECK-LABEL: @test_svldff1ub_gather_u32offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.gather.uxtw.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1ub_gather_u32offset_u32u10__SVBool_tPKhu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1ub_gather_u32offset_u32u10__SVBool_tPKhu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.gather.uxtw.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -452,14 +452,14 @@ svuint32_t test_svldff1ub_gather_u32offset_u32(svbool_t pg, const uint8_t *base,
   return SVE_ACLE_FUNC(svldff1ub_gather_, u32, offset_u32, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1ub_gather_u64offset_u64(
+// CHECK-LABEL: @test_svldff1ub_gather_u64offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.gather.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1ub_gather_u64offset_u64u10__SVBool_tPKhu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1ub_gather_u64offset_u64u10__SVBool_tPKhu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.gather.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -470,14 +470,14 @@ svuint64_t test_svldff1ub_gather_u64offset_u64(svbool_t pg, const uint8_t *base,
   return SVE_ACLE_FUNC(svldff1ub_gather_, u64, offset_u64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1ub_gather_u32base_offset_s32(
+// CHECK-LABEL: @test_svldff1ub_gather_u32base_offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i8.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z40test_svldff1ub_gather_u32base_offset_s32u10__SVBool_tu12__SVUint32_tl(
+// CPP-CHECK-LABEL: @_Z40test_svldff1ub_gather_u32base_offset_s32u10__SVBool_tu12__SVUint32_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i8.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -488,14 +488,14 @@ svint32_t test_svldff1ub_gather_u32base_offset_s32(svbool_t pg, svuint32_t bases
   return SVE_ACLE_FUNC(svldff1ub_gather, _u32base, _offset_s32, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1ub_gather_u64base_offset_s64(
+// CHECK-LABEL: @test_svldff1ub_gather_u64base_offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i8.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z40test_svldff1ub_gather_u64base_offset_s64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z40test_svldff1ub_gather_u64base_offset_s64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i8.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -506,14 +506,14 @@ svint64_t test_svldff1ub_gather_u64base_offset_s64(svbool_t pg, svuint64_t bases
   return SVE_ACLE_FUNC(svldff1ub_gather, _u64base, _offset_s64, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1ub_gather_u32base_offset_u32(
+// CHECK-LABEL: @test_svldff1ub_gather_u32base_offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i8.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z40test_svldff1ub_gather_u32base_offset_u32u10__SVBool_tu12__SVUint32_tl(
+// CPP-CHECK-LABEL: @_Z40test_svldff1ub_gather_u32base_offset_u32u10__SVBool_tu12__SVUint32_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i8.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -524,14 +524,14 @@ svuint32_t test_svldff1ub_gather_u32base_offset_u32(svbool_t pg, svuint32_t base
   return SVE_ACLE_FUNC(svldff1ub_gather, _u32base, _offset_u32, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1ub_gather_u64base_offset_u64(
+// CHECK-LABEL: @test_svldff1ub_gather_u64base_offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i8.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z40test_svldff1ub_gather_u64base_offset_u64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z40test_svldff1ub_gather_u64base_offset_u64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i8.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1uh.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1uh.c
index 522befc15b..81f14ae7ff 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1uh.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1uh.c
@@ -14,14 +14,14 @@
 #define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uh_s32(
+// CHECK-LABEL: @test_svldff1uh_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldff1uh_s32u10__SVBool_tPKt(
+// CPP-CHECK-LABEL: @_Z18test_svldff1uh_s32u10__SVBool_tPKt(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -33,14 +33,14 @@ svint32_t test_svldff1uh_s32(svbool_t pg, const uint16_t *base)
   return svldff1uh_s32(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uh_s64(
+// CHECK-LABEL: @test_svldff1uh_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldff1uh_s64u10__SVBool_tPKt(
+// CPP-CHECK-LABEL: @_Z18test_svldff1uh_s64u10__SVBool_tPKt(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -52,14 +52,14 @@ svint64_t test_svldff1uh_s64(svbool_t pg, const uint16_t *base)
   return svldff1uh_s64(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uh_u32(
+// CHECK-LABEL: @test_svldff1uh_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldff1uh_u32u10__SVBool_tPKt(
+// CPP-CHECK-LABEL: @_Z18test_svldff1uh_u32u10__SVBool_tPKt(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -71,14 +71,14 @@ svuint32_t test_svldff1uh_u32(svbool_t pg, const uint16_t *base)
   return svldff1uh_u32(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uh_u64(
+// CHECK-LABEL: @test_svldff1uh_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldff1uh_u64u10__SVBool_tPKt(
+// CPP-CHECK-LABEL: @_Z18test_svldff1uh_u64u10__SVBool_tPKt(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -90,7 +90,7 @@ svuint64_t test_svldff1uh_u64(svbool_t pg, const uint16_t *base)
   return svldff1uh_u64(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uh_vnum_s32(
+// CHECK-LABEL: @test_svldff1uh_vnum_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -98,7 +98,7 @@ svuint64_t test_svldff1uh_u64(svbool_t pg, const uint16_t *base)
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 4 x i16> [[TMP2]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldff1uh_vnum_s32u10__SVBool_tPKtl(
+// CPP-CHECK-LABEL: @_Z23test_svldff1uh_vnum_s32u10__SVBool_tPKtl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -111,7 +111,7 @@ svint32_t test_svldff1uh_vnum_s32(svbool_t pg, const uint16_t *base, int64_t vnu
   return svldff1uh_vnum_s32(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uh_vnum_s64(
+// CHECK-LABEL: @test_svldff1uh_vnum_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -119,7 +119,7 @@ svint32_t test_svldff1uh_vnum_s32(svbool_t pg, const uint16_t *base, int64_t vnu
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 2 x i16> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldff1uh_vnum_s64u10__SVBool_tPKtl(
+// CPP-CHECK-LABEL: @_Z23test_svldff1uh_vnum_s64u10__SVBool_tPKtl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -132,7 +132,7 @@ svint64_t test_svldff1uh_vnum_s64(svbool_t pg, const uint16_t *base, int64_t vnu
   return svldff1uh_vnum_s64(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uh_vnum_u32(
+// CHECK-LABEL: @test_svldff1uh_vnum_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -140,7 +140,7 @@ svint64_t test_svldff1uh_vnum_s64(svbool_t pg, const uint16_t *base, int64_t vnu
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 4 x i16> [[TMP2]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldff1uh_vnum_u32u10__SVBool_tPKtl(
+// CPP-CHECK-LABEL: @_Z23test_svldff1uh_vnum_u32u10__SVBool_tPKtl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -153,7 +153,7 @@ svuint32_t test_svldff1uh_vnum_u32(svbool_t pg, const uint16_t *base, int64_t vn
   return svldff1uh_vnum_u32(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uh_vnum_u64(
+// CHECK-LABEL: @test_svldff1uh_vnum_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -161,7 +161,7 @@ svuint32_t test_svldff1uh_vnum_u32(svbool_t pg, const uint16_t *base, int64_t vn
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 2 x i16> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldff1uh_vnum_u64u10__SVBool_tPKtl(
+// CPP-CHECK-LABEL: @_Z23test_svldff1uh_vnum_u64u10__SVBool_tPKtl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -174,14 +174,14 @@ svuint64_t test_svldff1uh_vnum_u64(svbool_t pg, const uint16_t *base, int64_t vn
   return svldff1uh_vnum_u64(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uh_gather_u32base_s32(
+// CHECK-LABEL: @test_svldff1uh_gather_u32base_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i16.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svldff1uh_gather_u32base_s32u10__SVBool_tu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z33test_svldff1uh_gather_u32base_s32u10__SVBool_tu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i16.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
@@ -192,14 +192,14 @@ svint32_t test_svldff1uh_gather_u32base_s32(svbool_t pg, svuint32_t bases) {
   return SVE_ACLE_FUNC(svldff1uh_gather, _u32base, _s32, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uh_gather_u64base_s64(
+// CHECK-LABEL: @test_svldff1uh_gather_u64base_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i16.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svldff1uh_gather_u64base_s64u10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z33test_svldff1uh_gather_u64base_s64u10__SVBool_tu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i16.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
@@ -210,14 +210,14 @@ svint64_t test_svldff1uh_gather_u64base_s64(svbool_t pg, svuint64_t bases) {
   return SVE_ACLE_FUNC(svldff1uh_gather, _u64base, _s64, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uh_gather_u32base_u32(
+// CHECK-LABEL: @test_svldff1uh_gather_u32base_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i16.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svldff1uh_gather_u32base_u32u10__SVBool_tu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z33test_svldff1uh_gather_u32base_u32u10__SVBool_tu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i16.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
@@ -228,14 +228,14 @@ svuint32_t test_svldff1uh_gather_u32base_u32(svbool_t pg, svuint32_t bases) {
   return SVE_ACLE_FUNC(svldff1uh_gather, _u32base, _u32, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uh_gather_u64base_u64(
+// CHECK-LABEL: @test_svldff1uh_gather_u64base_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i16.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svldff1uh_gather_u64base_u64u10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z33test_svldff1uh_gather_u64base_u64u10__SVBool_tu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i16.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
@@ -246,14 +246,14 @@ svuint64_t test_svldff1uh_gather_u64base_u64(svbool_t pg, svuint64_t bases) {
   return SVE_ACLE_FUNC(svldff1uh_gather, _u64base, _u64, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uh_gather_s32offset_s32(
+// CHECK-LABEL: @test_svldff1uh_gather_s32offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.sxtw.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1uh_gather_s32offset_s32u10__SVBool_tPKtu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1uh_gather_s32offset_s32u10__SVBool_tPKtu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.sxtw.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -264,14 +264,14 @@ svint32_t test_svldff1uh_gather_s32offset_s32(svbool_t pg, const uint16_t *base,
   return SVE_ACLE_FUNC(svldff1uh_gather_, s32, offset_s32, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uh_gather_s64offset_s64(
+// CHECK-LABEL: @test_svldff1uh_gather_s64offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1uh_gather_s64offset_s64u10__SVBool_tPKtu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1uh_gather_s64offset_s64u10__SVBool_tPKtu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -282,14 +282,14 @@ svint64_t test_svldff1uh_gather_s64offset_s64(svbool_t pg, const uint16_t *base,
   return SVE_ACLE_FUNC(svldff1uh_gather_, s64, offset_s64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uh_gather_s32offset_u32(
+// CHECK-LABEL: @test_svldff1uh_gather_s32offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.sxtw.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1uh_gather_s32offset_u32u10__SVBool_tPKtu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1uh_gather_s32offset_u32u10__SVBool_tPKtu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.sxtw.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -300,14 +300,14 @@ svuint32_t test_svldff1uh_gather_s32offset_u32(svbool_t pg, const uint16_t *base
   return SVE_ACLE_FUNC(svldff1uh_gather_, s32, offset_u32, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uh_gather_s64offset_u64(
+// CHECK-LABEL: @test_svldff1uh_gather_s64offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1uh_gather_s64offset_u64u10__SVBool_tPKtu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1uh_gather_s64offset_u64u10__SVBool_tPKtu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -318,14 +318,14 @@ svuint64_t test_svldff1uh_gather_s64offset_u64(svbool_t pg, const uint16_t *base
   return SVE_ACLE_FUNC(svldff1uh_gather_, s64, offset_u64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uh_gather_u32offset_s32(
+// CHECK-LABEL: @test_svldff1uh_gather_u32offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.uxtw.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1uh_gather_u32offset_s32u10__SVBool_tPKtu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1uh_gather_u32offset_s32u10__SVBool_tPKtu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.uxtw.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -336,14 +336,14 @@ svint32_t test_svldff1uh_gather_u32offset_s32(svbool_t pg, const uint16_t *base,
   return SVE_ACLE_FUNC(svldff1uh_gather_, u32, offset_s32, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uh_gather_u64offset_s64(
+// CHECK-LABEL: @test_svldff1uh_gather_u64offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1uh_gather_u64offset_s64u10__SVBool_tPKtu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1uh_gather_u64offset_s64u10__SVBool_tPKtu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -354,14 +354,14 @@ svint64_t test_svldff1uh_gather_u64offset_s64(svbool_t pg, const uint16_t *base,
   return SVE_ACLE_FUNC(svldff1uh_gather_, u64, offset_s64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uh_gather_u32offset_u32(
+// CHECK-LABEL: @test_svldff1uh_gather_u32offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.uxtw.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1uh_gather_u32offset_u32u10__SVBool_tPKtu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1uh_gather_u32offset_u32u10__SVBool_tPKtu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.uxtw.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -372,14 +372,14 @@ svuint32_t test_svldff1uh_gather_u32offset_u32(svbool_t pg, const uint16_t *base
   return SVE_ACLE_FUNC(svldff1uh_gather_, u32, offset_u32, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uh_gather_u64offset_u64(
+// CHECK-LABEL: @test_svldff1uh_gather_u64offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1uh_gather_u64offset_u64u10__SVBool_tPKtu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1uh_gather_u64offset_u64u10__SVBool_tPKtu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -390,14 +390,14 @@ svuint64_t test_svldff1uh_gather_u64offset_u64(svbool_t pg, const uint16_t *base
   return SVE_ACLE_FUNC(svldff1uh_gather_, u64, offset_u64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uh_gather_u32base_offset_s32(
+// CHECK-LABEL: @test_svldff1uh_gather_u32base_offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i16.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z40test_svldff1uh_gather_u32base_offset_s32u10__SVBool_tu12__SVUint32_tl(
+// CPP-CHECK-LABEL: @_Z40test_svldff1uh_gather_u32base_offset_s32u10__SVBool_tu12__SVUint32_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i16.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -408,14 +408,14 @@ svint32_t test_svldff1uh_gather_u32base_offset_s32(svbool_t pg, svuint32_t bases
   return SVE_ACLE_FUNC(svldff1uh_gather, _u32base, _offset_s32, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uh_gather_u64base_offset_s64(
+// CHECK-LABEL: @test_svldff1uh_gather_u64base_offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i16.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z40test_svldff1uh_gather_u64base_offset_s64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z40test_svldff1uh_gather_u64base_offset_s64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i16.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -426,14 +426,14 @@ svint64_t test_svldff1uh_gather_u64base_offset_s64(svbool_t pg, svuint64_t bases
   return SVE_ACLE_FUNC(svldff1uh_gather, _u64base, _offset_s64, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uh_gather_u32base_offset_u32(
+// CHECK-LABEL: @test_svldff1uh_gather_u32base_offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i16.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z40test_svldff1uh_gather_u32base_offset_u32u10__SVBool_tu12__SVUint32_tl(
+// CPP-CHECK-LABEL: @_Z40test_svldff1uh_gather_u32base_offset_u32u10__SVBool_tu12__SVUint32_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv4i16.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -444,14 +444,14 @@ svuint32_t test_svldff1uh_gather_u32base_offset_u32(svbool_t pg, svuint32_t base
   return SVE_ACLE_FUNC(svldff1uh_gather, _u32base, _offset_u32, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uh_gather_u64base_offset_u64(
+// CHECK-LABEL: @test_svldff1uh_gather_u64base_offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i16.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z40test_svldff1uh_gather_u64base_offset_u64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z40test_svldff1uh_gather_u64base_offset_u64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i16.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -462,14 +462,14 @@ svuint64_t test_svldff1uh_gather_u64base_offset_u64(svbool_t pg, svuint64_t base
   return SVE_ACLE_FUNC(svldff1uh_gather, _u64base, _offset_u64, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uh_gather_s32index_s32(
+// CHECK-LABEL: @test_svldff1uh_gather_s32index_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.sxtw.index.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z34test_svldff1uh_gather_s32index_s32u10__SVBool_tPKtu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z34test_svldff1uh_gather_s32index_s32u10__SVBool_tPKtu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.sxtw.index.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
@@ -480,14 +480,14 @@ svint32_t test_svldff1uh_gather_s32index_s32(svbool_t pg, const uint16_t *base,
   return SVE_ACLE_FUNC(svldff1uh_gather_, s32, index_s32, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uh_gather_s64index_s64(
+// CHECK-LABEL: @test_svldff1uh_gather_s64index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.index.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z34test_svldff1uh_gather_s64index_s64u10__SVBool_tPKtu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z34test_svldff1uh_gather_s64index_s64u10__SVBool_tPKtu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.index.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -498,14 +498,14 @@ svint64_t test_svldff1uh_gather_s64index_s64(svbool_t pg, const uint16_t *base,
   return SVE_ACLE_FUNC(svldff1uh_gather_, s64, index_s64, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uh_gather_s32index_u32(
+// CHECK-LABEL: @test_svldff1uh_gather_s32index_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.sxtw.index.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z34test_svldff1uh_gather_s32index_u32u10__SVBool_tPKtu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z34test_svldff1uh_gather_s32index_u32u10__SVBool_tPKtu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.sxtw.index.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
@@ -516,14 +516,14 @@ svuint32_t test_svldff1uh_gather_s32index_u32(svbool_t pg, const uint16_t *base,
   return SVE_ACLE_FUNC(svldff1uh_gather_, s32, index_u32, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uh_gather_s64index_u64(
+// CHECK-LABEL: @test_svldff1uh_gather_s64index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.index.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z34test_svldff1uh_gather_s64index_u64u10__SVBool_tPKtu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z34test_svldff1uh_gather_s64index_u64u10__SVBool_tPKtu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.index.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -534,14 +534,14 @@ svuint64_t test_svldff1uh_gather_s64index_u64(svbool_t pg, const uint16_t *base,
   return SVE_ACLE_FUNC(svldff1uh_gather_, s64, index_u64, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uh_gather_u32index_s32(
+// CHECK-LABEL: @test_svldff1uh_gather_u32index_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.uxtw.index.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z34test_svldff1uh_gather_u32index_s32u10__SVBool_tPKtu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z34test_svldff1uh_gather_u32index_s32u10__SVBool_tPKtu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.uxtw.index.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
@@ -552,14 +552,14 @@ svint32_t test_svldff1uh_gather_u32index_s32(svbool_t pg, const uint16_t *base,
   return SVE_ACLE_FUNC(svldff1uh_gather_, u32, index_s32, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uh_gather_u64index_s64(
+// CHECK-LABEL: @test_svldff1uh_gather_u64index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.index.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z34test_svldff1uh_gather_u64index_s64u10__SVBool_tPKtu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z34test_svldff1uh_gather_u64index_s64u10__SVBool_tPKtu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.index.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -570,14 +570,14 @@ svint64_t test_svldff1uh_gather_u64index_s64(svbool_t pg, const uint16_t *base,
   return SVE_ACLE_FUNC(svldff1uh_gather_, u64, index_s64, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uh_gather_u32index_u32(
+// CHECK-LABEL: @test_svldff1uh_gather_u32index_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.uxtw.index.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z34test_svldff1uh_gather_u32index_u32u10__SVBool_tPKtu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z34test_svldff1uh_gather_u32index_u32u10__SVBool_tPKtu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldff1.gather.uxtw.index.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
@@ -588,14 +588,14 @@ svuint32_t test_svldff1uh_gather_u32index_u32(svbool_t pg, const uint16_t *base,
   return SVE_ACLE_FUNC(svldff1uh_gather_, u32, index_u32, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uh_gather_u64index_u64(
+// CHECK-LABEL: @test_svldff1uh_gather_u64index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.index.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z34test_svldff1uh_gather_u64index_u64u10__SVBool_tPKtu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z34test_svldff1uh_gather_u64index_u64u10__SVBool_tPKtu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldff1.gather.index.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -606,7 +606,7 @@ svuint64_t test_svldff1uh_gather_u64index_u64(svbool_t pg, const uint16_t *base,
   return SVE_ACLE_FUNC(svldff1uh_gather_, u64, index_u64, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uh_gather_u32base_index_s32(
+// CHECK-LABEL: @test_svldff1uh_gather_u32base_index_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 1
@@ -614,7 +614,7 @@ svuint64_t test_svldff1uh_gather_u64index_u64(svbool_t pg, const uint16_t *base,
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 4 x i16> [[TMP2]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z39test_svldff1uh_gather_u32base_index_s32u10__SVBool_tu12__SVUint32_tl(
+// CPP-CHECK-LABEL: @_Z39test_svldff1uh_gather_u32base_index_s32u10__SVBool_tu12__SVUint32_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 1
@@ -626,7 +626,7 @@ svint32_t test_svldff1uh_gather_u32base_index_s32(svbool_t pg, svuint32_t bases,
   return SVE_ACLE_FUNC(svldff1uh_gather, _u32base, _index_s32, )(pg, bases, index);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uh_gather_u64base_index_s64(
+// CHECK-LABEL: @test_svldff1uh_gather_u64base_index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 1
@@ -634,7 +634,7 @@ svint32_t test_svldff1uh_gather_u32base_index_s32(svbool_t pg, svuint32_t bases,
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 2 x i16> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z39test_svldff1uh_gather_u64base_index_s64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z39test_svldff1uh_gather_u64base_index_s64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 1
@@ -646,7 +646,7 @@ svint64_t test_svldff1uh_gather_u64base_index_s64(svbool_t pg, svuint64_t bases,
   return SVE_ACLE_FUNC(svldff1uh_gather, _u64base, _index_s64, )(pg, bases, index);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uh_gather_u32base_index_u32(
+// CHECK-LABEL: @test_svldff1uh_gather_u32base_index_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 1
@@ -654,7 +654,7 @@ svint64_t test_svldff1uh_gather_u64base_index_s64(svbool_t pg, svuint64_t bases,
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 4 x i16> [[TMP2]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z39test_svldff1uh_gather_u32base_index_u32u10__SVBool_tu12__SVUint32_tl(
+// CPP-CHECK-LABEL: @_Z39test_svldff1uh_gather_u32base_index_u32u10__SVBool_tu12__SVUint32_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 1
@@ -666,7 +666,7 @@ svuint32_t test_svldff1uh_gather_u32base_index_u32(svbool_t pg, svuint32_t bases
   return SVE_ACLE_FUNC(svldff1uh_gather, _u32base, _index_u32, )(pg, bases, index);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uh_gather_u64base_index_u64(
+// CHECK-LABEL: @test_svldff1uh_gather_u64base_index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 1
@@ -674,7 +674,7 @@ svuint32_t test_svldff1uh_gather_u32base_index_u32(svbool_t pg, svuint32_t bases
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 2 x i16> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z39test_svldff1uh_gather_u64base_index_u64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z39test_svldff1uh_gather_u64base_index_u64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 1
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1uw.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1uw.c
index 573e52fad1..c1a647ba2d 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1uw.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldff1uw.c
@@ -14,14 +14,14 @@
 #define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uw_s64(
+// CHECK-LABEL: @test_svldff1uw_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldff1uw_s64u10__SVBool_tPKj(
+// CPP-CHECK-LABEL: @_Z18test_svldff1uw_s64u10__SVBool_tPKj(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -33,14 +33,14 @@ svint64_t test_svldff1uw_s64(svbool_t pg, const uint32_t *base)
   return svldff1uw_s64(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uw_u64(
+// CHECK-LABEL: @test_svldff1uw_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldff1uw_u64u10__SVBool_tPKj(
+// CPP-CHECK-LABEL: @_Z18test_svldff1uw_u64u10__SVBool_tPKj(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -52,7 +52,7 @@ svuint64_t test_svldff1uw_u64(svbool_t pg, const uint32_t *base)
   return svldff1uw_u64(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uw_vnum_s64(
+// CHECK-LABEL: @test_svldff1uw_vnum_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -60,7 +60,7 @@ svuint64_t test_svldff1uw_u64(svbool_t pg, const uint32_t *base)
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 2 x i32> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldff1uw_vnum_s64u10__SVBool_tPKjl(
+// CPP-CHECK-LABEL: @_Z23test_svldff1uw_vnum_s64u10__SVBool_tPKjl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -73,7 +73,7 @@ svint64_t test_svldff1uw_vnum_s64(svbool_t pg, const uint32_t *base, int64_t vnu
   return svldff1uw_vnum_s64(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uw_vnum_u64(
+// CHECK-LABEL: @test_svldff1uw_vnum_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -81,7 +81,7 @@ svint64_t test_svldff1uw_vnum_s64(svbool_t pg, const uint32_t *base, int64_t vnu
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 2 x i32> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldff1uw_vnum_u64u10__SVBool_tPKjl(
+// CPP-CHECK-LABEL: @_Z23test_svldff1uw_vnum_u64u10__SVBool_tPKjl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -94,14 +94,14 @@ svuint64_t test_svldff1uw_vnum_u64(svbool_t pg, const uint32_t *base, int64_t vn
   return svldff1uw_vnum_u64(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uw_gather_u64base_s64(
+// CHECK-LABEL: @test_svldff1uw_gather_u64base_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i32.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svldff1uw_gather_u64base_s64u10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z33test_svldff1uw_gather_u64base_s64u10__SVBool_tu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i32.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
@@ -112,14 +112,14 @@ svint64_t test_svldff1uw_gather_u64base_s64(svbool_t pg, svuint64_t bases) {
   return SVE_ACLE_FUNC(svldff1uw_gather, _u64base, _s64, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uw_gather_u64base_u64(
+// CHECK-LABEL: @test_svldff1uw_gather_u64base_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i32.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svldff1uw_gather_u64base_u64u10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z33test_svldff1uw_gather_u64base_u64u10__SVBool_tu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i32.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
@@ -130,14 +130,14 @@ svuint64_t test_svldff1uw_gather_u64base_u64(svbool_t pg, svuint64_t bases) {
   return SVE_ACLE_FUNC(svldff1uw_gather, _u64base, _u64, )(pg, bases);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uw_gather_s64offset_s64(
+// CHECK-LABEL: @test_svldff1uw_gather_s64offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1uw_gather_s64offset_s64u10__SVBool_tPKju11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1uw_gather_s64offset_s64u10__SVBool_tPKju11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -148,14 +148,14 @@ svint64_t test_svldff1uw_gather_s64offset_s64(svbool_t pg, const uint32_t *base,
   return SVE_ACLE_FUNC(svldff1uw_gather_, s64, offset_s64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uw_gather_s64offset_u64(
+// CHECK-LABEL: @test_svldff1uw_gather_s64offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1uw_gather_s64offset_u64u10__SVBool_tPKju11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1uw_gather_s64offset_u64u10__SVBool_tPKju11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -166,14 +166,14 @@ svuint64_t test_svldff1uw_gather_s64offset_u64(svbool_t pg, const uint32_t *base
   return SVE_ACLE_FUNC(svldff1uw_gather_, s64, offset_u64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uw_gather_u64offset_s64(
+// CHECK-LABEL: @test_svldff1uw_gather_u64offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1uw_gather_u64offset_s64u10__SVBool_tPKju12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1uw_gather_u64offset_s64u10__SVBool_tPKju12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -184,14 +184,14 @@ svint64_t test_svldff1uw_gather_u64offset_s64(svbool_t pg, const uint32_t *base,
   return SVE_ACLE_FUNC(svldff1uw_gather_, u64, offset_s64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uw_gather_u64offset_u64(
+// CHECK-LABEL: @test_svldff1uw_gather_u64offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z35test_svldff1uw_gather_u64offset_u64u10__SVBool_tPKju12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z35test_svldff1uw_gather_u64offset_u64u10__SVBool_tPKju12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -202,14 +202,14 @@ svuint64_t test_svldff1uw_gather_u64offset_u64(svbool_t pg, const uint32_t *base
   return SVE_ACLE_FUNC(svldff1uw_gather_, u64, offset_u64, )(pg, base, offsets);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uw_gather_u64base_offset_s64(
+// CHECK-LABEL: @test_svldff1uw_gather_u64base_offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i32.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z40test_svldff1uw_gather_u64base_offset_s64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z40test_svldff1uw_gather_u64base_offset_s64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i32.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -220,14 +220,14 @@ svint64_t test_svldff1uw_gather_u64base_offset_s64(svbool_t pg, svuint64_t bases
   return SVE_ACLE_FUNC(svldff1uw_gather, _u64base, _offset_s64, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uw_gather_u64base_offset_u64(
+// CHECK-LABEL: @test_svldff1uw_gather_u64base_offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i32.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z40test_svldff1uw_gather_u64base_offset_u64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z40test_svldff1uw_gather_u64base_offset_u64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.scalar.offset.nxv2i32.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -238,14 +238,14 @@ svuint64_t test_svldff1uw_gather_u64base_offset_u64(svbool_t pg, svuint64_t base
   return SVE_ACLE_FUNC(svldff1uw_gather, _u64base, _offset_u64, )(pg, bases, offset);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uw_gather_s64index_s64(
+// CHECK-LABEL: @test_svldff1uw_gather_s64index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.index.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z34test_svldff1uw_gather_s64index_s64u10__SVBool_tPKju11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z34test_svldff1uw_gather_s64index_s64u10__SVBool_tPKju11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.index.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -256,14 +256,14 @@ svint64_t test_svldff1uw_gather_s64index_s64(svbool_t pg, const uint32_t *base,
   return SVE_ACLE_FUNC(svldff1uw_gather_, s64, index_s64, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uw_gather_s64index_u64(
+// CHECK-LABEL: @test_svldff1uw_gather_s64index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.index.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z34test_svldff1uw_gather_s64index_u64u10__SVBool_tPKju11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z34test_svldff1uw_gather_s64index_u64u10__SVBool_tPKju11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.index.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -274,14 +274,14 @@ svuint64_t test_svldff1uw_gather_s64index_u64(svbool_t pg, const uint32_t *base,
   return SVE_ACLE_FUNC(svldff1uw_gather_, s64, index_u64, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uw_gather_u64index_s64(
+// CHECK-LABEL: @test_svldff1uw_gather_u64index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.index.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z34test_svldff1uw_gather_u64index_s64u10__SVBool_tPKju12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z34test_svldff1uw_gather_u64index_s64u10__SVBool_tPKju12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.index.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -292,14 +292,14 @@ svint64_t test_svldff1uw_gather_u64index_s64(svbool_t pg, const uint32_t *base,
   return SVE_ACLE_FUNC(svldff1uw_gather_, u64, index_s64, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uw_gather_u64index_u64(
+// CHECK-LABEL: @test_svldff1uw_gather_u64index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.index.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z34test_svldff1uw_gather_u64index_u64u10__SVBool_tPKju12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z34test_svldff1uw_gather_u64index_u64u10__SVBool_tPKju12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldff1.gather.index.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -310,7 +310,7 @@ svuint64_t test_svldff1uw_gather_u64index_u64(svbool_t pg, const uint32_t *base,
   return SVE_ACLE_FUNC(svldff1uw_gather_, u64, index_u64, )(pg, base, indices);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uw_gather_u64base_index_s64(
+// CHECK-LABEL: @test_svldff1uw_gather_u64base_index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 2
@@ -318,7 +318,7 @@ svuint64_t test_svldff1uw_gather_u64index_u64(svbool_t pg, const uint32_t *base,
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 2 x i32> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z39test_svldff1uw_gather_u64base_index_s64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z39test_svldff1uw_gather_u64base_index_s64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 2
@@ -330,7 +330,7 @@ svint64_t test_svldff1uw_gather_u64base_index_s64(svbool_t pg, svuint64_t bases,
   return SVE_ACLE_FUNC(svldff1uw_gather, _u64base, _index_s64, )(pg, bases, index);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldff1uw_gather_u64base_index_u64(
+// CHECK-LABEL: @test_svldff1uw_gather_u64base_index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 2
@@ -338,7 +338,7 @@ svint64_t test_svldff1uw_gather_u64base_index_s64(svbool_t pg, svuint64_t bases,
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 2 x i32> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z39test_svldff1uw_gather_u64base_index_u64u10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z39test_svldff1uw_gather_u64base_index_u64u10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 2
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1-bfloat.c
index c93122fcca..23893e1e0e 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1-bfloat.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1-bfloat.c
@@ -14,13 +14,13 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1_bf16(
+// CHECK-LABEL: @test_svldnf1_bf16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.ldnf1.nxv8bf16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z17test_svldnf1_bf16u10__SVBool_tPKu6__bf16(
+// CPP-CHECK-LABEL: @_Z17test_svldnf1_bf16u10__SVBool_tPKu6__bf16(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.ldnf1.nxv8bf16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -31,14 +31,14 @@ svbfloat16_t test_svldnf1_bf16(svbool_t pg, const bfloat16_t *base)
   return SVE_ACLE_FUNC(svldnf1,_bf16,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1_vnum_bf16(
+// CHECK-LABEL: @test_svldnf1_vnum_bf16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x bfloat>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.ldnf1.nxv8bf16(<vscale x 8 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z22test_svldnf1_vnum_bf16u10__SVBool_tPKu6__bf16l(
+// CPP-CHECK-LABEL: @_Z22test_svldnf1_vnum_bf16u10__SVBool_tPKu6__bf16l(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x bfloat>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1.c
index b7b6e33f83..7b47c2c12b 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1.c
@@ -14,12 +14,12 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1_s8(
+// CHECK-LABEL: @test_svldnf1_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldnf1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z15test_svldnf1_s8u10__SVBool_tPKa(
+// CPP-CHECK-LABEL: @_Z15test_svldnf1_s8u10__SVBool_tPKa(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldnf1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
@@ -29,13 +29,13 @@ svint8_t test_svldnf1_s8(svbool_t pg, const int8_t *base)
   return SVE_ACLE_FUNC(svldnf1,_s8,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1_s16(
+// CHECK-LABEL: @test_svldnf1_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.ldnf1.nxv8i16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svldnf1_s16u10__SVBool_tPKs(
+// CPP-CHECK-LABEL: @_Z16test_svldnf1_s16u10__SVBool_tPKs(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.ldnf1.nxv8i16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -46,13 +46,13 @@ svint16_t test_svldnf1_s16(svbool_t pg, const int16_t *base)
   return SVE_ACLE_FUNC(svldnf1,_s16,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1_s32(
+// CHECK-LABEL: @test_svldnf1_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldnf1.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svldnf1_s32u10__SVBool_tPKi(
+// CPP-CHECK-LABEL: @_Z16test_svldnf1_s32u10__SVBool_tPKi(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldnf1.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -63,13 +63,13 @@ svint32_t test_svldnf1_s32(svbool_t pg, const int32_t *base)
   return SVE_ACLE_FUNC(svldnf1,_s32,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1_s64(
+// CHECK-LABEL: @test_svldnf1_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldnf1.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svldnf1_s64u10__SVBool_tPKl(
+// CPP-CHECK-LABEL: @_Z16test_svldnf1_s64u10__SVBool_tPKl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldnf1.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -80,12 +80,12 @@ svint64_t test_svldnf1_s64(svbool_t pg, const int64_t *base)
   return SVE_ACLE_FUNC(svldnf1,_s64,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1_u8(
+// CHECK-LABEL: @test_svldnf1_u8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldnf1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z15test_svldnf1_u8u10__SVBool_tPKh(
+// CPP-CHECK-LABEL: @_Z15test_svldnf1_u8u10__SVBool_tPKh(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldnf1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
@@ -95,13 +95,13 @@ svuint8_t test_svldnf1_u8(svbool_t pg, const uint8_t *base)
   return SVE_ACLE_FUNC(svldnf1,_u8,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1_u16(
+// CHECK-LABEL: @test_svldnf1_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.ldnf1.nxv8i16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svldnf1_u16u10__SVBool_tPKt(
+// CPP-CHECK-LABEL: @_Z16test_svldnf1_u16u10__SVBool_tPKt(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.ldnf1.nxv8i16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -112,13 +112,13 @@ svuint16_t test_svldnf1_u16(svbool_t pg, const uint16_t *base)
   return SVE_ACLE_FUNC(svldnf1,_u16,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1_u32(
+// CHECK-LABEL: @test_svldnf1_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldnf1.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svldnf1_u32u10__SVBool_tPKj(
+// CPP-CHECK-LABEL: @_Z16test_svldnf1_u32u10__SVBool_tPKj(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldnf1.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -129,13 +129,13 @@ svuint32_t test_svldnf1_u32(svbool_t pg, const uint32_t *base)
   return SVE_ACLE_FUNC(svldnf1,_u32,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1_u64(
+// CHECK-LABEL: @test_svldnf1_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldnf1.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svldnf1_u64u10__SVBool_tPKm(
+// CPP-CHECK-LABEL: @_Z16test_svldnf1_u64u10__SVBool_tPKm(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldnf1.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -146,13 +146,13 @@ svuint64_t test_svldnf1_u64(svbool_t pg, const uint64_t *base)
   return SVE_ACLE_FUNC(svldnf1,_u64,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1_f16(
+// CHECK-LABEL: @test_svldnf1_f16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sve.ldnf1.nxv8f16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret <vscale x 8 x half> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svldnf1_f16u10__SVBool_tPKDh(
+// CPP-CHECK-LABEL: @_Z16test_svldnf1_f16u10__SVBool_tPKDh(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sve.ldnf1.nxv8f16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -163,13 +163,13 @@ svfloat16_t test_svldnf1_f16(svbool_t pg, const float16_t *base)
   return SVE_ACLE_FUNC(svldnf1,_f16,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1_f32(
+// CHECK-LABEL: @test_svldnf1_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.ldnf1.nxv4f32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svldnf1_f32u10__SVBool_tPKf(
+// CPP-CHECK-LABEL: @_Z16test_svldnf1_f32u10__SVBool_tPKf(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.ldnf1.nxv4f32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -180,13 +180,13 @@ svfloat32_t test_svldnf1_f32(svbool_t pg, const float32_t *base)
   return SVE_ACLE_FUNC(svldnf1,_f32,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1_f64(
+// CHECK-LABEL: @test_svldnf1_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.ldnf1.nxv2f64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svldnf1_f64u10__SVBool_tPKd(
+// CPP-CHECK-LABEL: @_Z16test_svldnf1_f64u10__SVBool_tPKd(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.ldnf1.nxv2f64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -197,13 +197,13 @@ svfloat64_t test_svldnf1_f64(svbool_t pg, const float64_t *base)
   return SVE_ACLE_FUNC(svldnf1,_f64,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1_vnum_s8(
+// CHECK-LABEL: @test_svldnf1_vnum_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldnf1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z20test_svldnf1_vnum_s8u10__SVBool_tPKal(
+// CPP-CHECK-LABEL: @_Z20test_svldnf1_vnum_s8u10__SVBool_tPKal(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldnf1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
@@ -214,14 +214,14 @@ svint8_t test_svldnf1_vnum_s8(svbool_t pg, const int8_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svldnf1_vnum,_s8,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1_vnum_s16(
+// CHECK-LABEL: @test_svldnf1_vnum_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.ldnf1.nxv8i16(<vscale x 8 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svldnf1_vnum_s16u10__SVBool_tPKsl(
+// CPP-CHECK-LABEL: @_Z21test_svldnf1_vnum_s16u10__SVBool_tPKsl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -233,14 +233,14 @@ svint16_t test_svldnf1_vnum_s16(svbool_t pg, const int16_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svldnf1_vnum,_s16,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1_vnum_s32(
+// CHECK-LABEL: @test_svldnf1_vnum_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldnf1.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svldnf1_vnum_s32u10__SVBool_tPKil(
+// CPP-CHECK-LABEL: @_Z21test_svldnf1_vnum_s32u10__SVBool_tPKil(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -252,14 +252,14 @@ svint32_t test_svldnf1_vnum_s32(svbool_t pg, const int32_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svldnf1_vnum,_s32,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1_vnum_s64(
+// CHECK-LABEL: @test_svldnf1_vnum_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldnf1.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svldnf1_vnum_s64u10__SVBool_tPKll(
+// CPP-CHECK-LABEL: @_Z21test_svldnf1_vnum_s64u10__SVBool_tPKll(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -271,13 +271,13 @@ svint64_t test_svldnf1_vnum_s64(svbool_t pg, const int64_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svldnf1_vnum,_s64,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1_vnum_u8(
+// CHECK-LABEL: @test_svldnf1_vnum_u8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldnf1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z20test_svldnf1_vnum_u8u10__SVBool_tPKhl(
+// CPP-CHECK-LABEL: @_Z20test_svldnf1_vnum_u8u10__SVBool_tPKhl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldnf1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
@@ -288,14 +288,14 @@ svuint8_t test_svldnf1_vnum_u8(svbool_t pg, const uint8_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svldnf1_vnum,_u8,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1_vnum_u16(
+// CHECK-LABEL: @test_svldnf1_vnum_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.ldnf1.nxv8i16(<vscale x 8 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svldnf1_vnum_u16u10__SVBool_tPKtl(
+// CPP-CHECK-LABEL: @_Z21test_svldnf1_vnum_u16u10__SVBool_tPKtl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -307,14 +307,14 @@ svuint16_t test_svldnf1_vnum_u16(svbool_t pg, const uint16_t *base, int64_t vnum
   return SVE_ACLE_FUNC(svldnf1_vnum,_u16,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1_vnum_u32(
+// CHECK-LABEL: @test_svldnf1_vnum_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldnf1.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svldnf1_vnum_u32u10__SVBool_tPKjl(
+// CPP-CHECK-LABEL: @_Z21test_svldnf1_vnum_u32u10__SVBool_tPKjl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -326,14 +326,14 @@ svuint32_t test_svldnf1_vnum_u32(svbool_t pg, const uint32_t *base, int64_t vnum
   return SVE_ACLE_FUNC(svldnf1_vnum,_u32,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1_vnum_u64(
+// CHECK-LABEL: @test_svldnf1_vnum_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldnf1.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svldnf1_vnum_u64u10__SVBool_tPKml(
+// CPP-CHECK-LABEL: @_Z21test_svldnf1_vnum_u64u10__SVBool_tPKml(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -345,14 +345,14 @@ svuint64_t test_svldnf1_vnum_u64(svbool_t pg, const uint64_t *base, int64_t vnum
   return SVE_ACLE_FUNC(svldnf1_vnum,_u64,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1_vnum_f16(
+// CHECK-LABEL: @test_svldnf1_vnum_f16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sve.ldnf1.nxv8f16(<vscale x 8 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 8 x half> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svldnf1_vnum_f16u10__SVBool_tPKDhl(
+// CPP-CHECK-LABEL: @_Z21test_svldnf1_vnum_f16u10__SVBool_tPKDhl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -364,14 +364,14 @@ svfloat16_t test_svldnf1_vnum_f16(svbool_t pg, const float16_t *base, int64_t vn
   return SVE_ACLE_FUNC(svldnf1_vnum,_f16,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1_vnum_f32(
+// CHECK-LABEL: @test_svldnf1_vnum_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.ldnf1.nxv4f32(<vscale x 4 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 4 x float> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svldnf1_vnum_f32u10__SVBool_tPKfl(
+// CPP-CHECK-LABEL: @_Z21test_svldnf1_vnum_f32u10__SVBool_tPKfl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -383,14 +383,14 @@ svfloat32_t test_svldnf1_vnum_f32(svbool_t pg, const float32_t *base, int64_t vn
   return SVE_ACLE_FUNC(svldnf1_vnum,_f32,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1_vnum_f64(
+// CHECK-LABEL: @test_svldnf1_vnum_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.ldnf1.nxv2f64(<vscale x 2 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 2 x double> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svldnf1_vnum_f64u10__SVBool_tPKdl(
+// CPP-CHECK-LABEL: @_Z21test_svldnf1_vnum_f64u10__SVBool_tPKdl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1sb.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1sb.c
index aa10b95a9d..f5c89606ee 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1sb.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1sb.c
@@ -5,14 +5,14 @@
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -S -disable-O0-optnone -Werror -o /dev/null %s
 #include <arm_sve.h>
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1sb_s16(
+// CHECK-LABEL: @test_svldnf1sb_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i8> @llvm.aarch64.sve.ldnf1.nxv8i8(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 8 x i8> [[TMP1]] to <vscale x 8 x i16>
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldnf1sb_s16u10__SVBool_tPKa(
+// CPP-CHECK-LABEL: @_Z18test_svldnf1sb_s16u10__SVBool_tPKa(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i8> @llvm.aarch64.sve.ldnf1.nxv8i8(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -24,14 +24,14 @@ svint16_t test_svldnf1sb_s16(svbool_t pg, const int8_t *base)
   return svldnf1sb_s16(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1sb_s32(
+// CHECK-LABEL: @test_svldnf1sb_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldnf1.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldnf1sb_s32u10__SVBool_tPKa(
+// CPP-CHECK-LABEL: @_Z18test_svldnf1sb_s32u10__SVBool_tPKa(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldnf1.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -43,14 +43,14 @@ svint32_t test_svldnf1sb_s32(svbool_t pg, const int8_t *base)
   return svldnf1sb_s32(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1sb_s64(
+// CHECK-LABEL: @test_svldnf1sb_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldnf1.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldnf1sb_s64u10__SVBool_tPKa(
+// CPP-CHECK-LABEL: @_Z18test_svldnf1sb_s64u10__SVBool_tPKa(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldnf1.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -62,14 +62,14 @@ svint64_t test_svldnf1sb_s64(svbool_t pg, const int8_t *base)
   return svldnf1sb_s64(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1sb_u16(
+// CHECK-LABEL: @test_svldnf1sb_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i8> @llvm.aarch64.sve.ldnf1.nxv8i8(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 8 x i8> [[TMP1]] to <vscale x 8 x i16>
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldnf1sb_u16u10__SVBool_tPKa(
+// CPP-CHECK-LABEL: @_Z18test_svldnf1sb_u16u10__SVBool_tPKa(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i8> @llvm.aarch64.sve.ldnf1.nxv8i8(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -81,14 +81,14 @@ svuint16_t test_svldnf1sb_u16(svbool_t pg, const int8_t *base)
   return svldnf1sb_u16(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1sb_u32(
+// CHECK-LABEL: @test_svldnf1sb_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldnf1.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldnf1sb_u32u10__SVBool_tPKa(
+// CPP-CHECK-LABEL: @_Z18test_svldnf1sb_u32u10__SVBool_tPKa(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldnf1.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -100,14 +100,14 @@ svuint32_t test_svldnf1sb_u32(svbool_t pg, const int8_t *base)
   return svldnf1sb_u32(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1sb_u64(
+// CHECK-LABEL: @test_svldnf1sb_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldnf1.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldnf1sb_u64u10__SVBool_tPKa(
+// CPP-CHECK-LABEL: @_Z18test_svldnf1sb_u64u10__SVBool_tPKa(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldnf1.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -119,7 +119,7 @@ svuint64_t test_svldnf1sb_u64(svbool_t pg, const int8_t *base)
   return svldnf1sb_u64(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1sb_vnum_s16(
+// CHECK-LABEL: @test_svldnf1sb_vnum_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -127,7 +127,7 @@ svuint64_t test_svldnf1sb_u64(svbool_t pg, const int8_t *base)
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 8 x i8> [[TMP2]] to <vscale x 8 x i16>
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldnf1sb_vnum_s16u10__SVBool_tPKal(
+// CPP-CHECK-LABEL: @_Z23test_svldnf1sb_vnum_s16u10__SVBool_tPKal(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -140,7 +140,7 @@ svint16_t test_svldnf1sb_vnum_s16(svbool_t pg, const int8_t *base, int64_t vnum)
   return svldnf1sb_vnum_s16(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1sb_vnum_s32(
+// CHECK-LABEL: @test_svldnf1sb_vnum_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -148,7 +148,7 @@ svint16_t test_svldnf1sb_vnum_s16(svbool_t pg, const int8_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 4 x i8> [[TMP2]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldnf1sb_vnum_s32u10__SVBool_tPKal(
+// CPP-CHECK-LABEL: @_Z23test_svldnf1sb_vnum_s32u10__SVBool_tPKal(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -161,7 +161,7 @@ svint32_t test_svldnf1sb_vnum_s32(svbool_t pg, const int8_t *base, int64_t vnum)
   return svldnf1sb_vnum_s32(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1sb_vnum_s64(
+// CHECK-LABEL: @test_svldnf1sb_vnum_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -169,7 +169,7 @@ svint32_t test_svldnf1sb_vnum_s32(svbool_t pg, const int8_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 2 x i8> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldnf1sb_vnum_s64u10__SVBool_tPKal(
+// CPP-CHECK-LABEL: @_Z23test_svldnf1sb_vnum_s64u10__SVBool_tPKal(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -182,7 +182,7 @@ svint64_t test_svldnf1sb_vnum_s64(svbool_t pg, const int8_t *base, int64_t vnum)
   return svldnf1sb_vnum_s64(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1sb_vnum_u16(
+// CHECK-LABEL: @test_svldnf1sb_vnum_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -190,7 +190,7 @@ svint64_t test_svldnf1sb_vnum_s64(svbool_t pg, const int8_t *base, int64_t vnum)
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 8 x i8> [[TMP2]] to <vscale x 8 x i16>
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldnf1sb_vnum_u16u10__SVBool_tPKal(
+// CPP-CHECK-LABEL: @_Z23test_svldnf1sb_vnum_u16u10__SVBool_tPKal(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -203,7 +203,7 @@ svuint16_t test_svldnf1sb_vnum_u16(svbool_t pg, const int8_t *base, int64_t vnum
   return svldnf1sb_vnum_u16(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1sb_vnum_u32(
+// CHECK-LABEL: @test_svldnf1sb_vnum_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -211,7 +211,7 @@ svuint16_t test_svldnf1sb_vnum_u16(svbool_t pg, const int8_t *base, int64_t vnum
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 4 x i8> [[TMP2]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldnf1sb_vnum_u32u10__SVBool_tPKal(
+// CPP-CHECK-LABEL: @_Z23test_svldnf1sb_vnum_u32u10__SVBool_tPKal(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -224,7 +224,7 @@ svuint32_t test_svldnf1sb_vnum_u32(svbool_t pg, const int8_t *base, int64_t vnum
   return svldnf1sb_vnum_u32(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1sb_vnum_u64(
+// CHECK-LABEL: @test_svldnf1sb_vnum_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -232,7 +232,7 @@ svuint32_t test_svldnf1sb_vnum_u32(svbool_t pg, const int8_t *base, int64_t vnum
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 2 x i8> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldnf1sb_vnum_u64u10__SVBool_tPKal(
+// CPP-CHECK-LABEL: @_Z23test_svldnf1sb_vnum_u64u10__SVBool_tPKal(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1sh.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1sh.c
index 02ba7d9a89..73f8a7db98 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1sh.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1sh.c
@@ -5,14 +5,14 @@
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -S -disable-O0-optnone -Werror -o /dev/null %s
 #include <arm_sve.h>
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1sh_s32(
+// CHECK-LABEL: @test_svldnf1sh_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldnf1.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldnf1sh_s32u10__SVBool_tPKs(
+// CPP-CHECK-LABEL: @_Z18test_svldnf1sh_s32u10__SVBool_tPKs(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldnf1.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -24,14 +24,14 @@ svint32_t test_svldnf1sh_s32(svbool_t pg, const int16_t *base)
   return svldnf1sh_s32(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1sh_s64(
+// CHECK-LABEL: @test_svldnf1sh_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldnf1.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldnf1sh_s64u10__SVBool_tPKs(
+// CPP-CHECK-LABEL: @_Z18test_svldnf1sh_s64u10__SVBool_tPKs(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldnf1.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -43,14 +43,14 @@ svint64_t test_svldnf1sh_s64(svbool_t pg, const int16_t *base)
   return svldnf1sh_s64(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1sh_u32(
+// CHECK-LABEL: @test_svldnf1sh_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldnf1.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldnf1sh_u32u10__SVBool_tPKs(
+// CPP-CHECK-LABEL: @_Z18test_svldnf1sh_u32u10__SVBool_tPKs(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldnf1.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -62,14 +62,14 @@ svuint32_t test_svldnf1sh_u32(svbool_t pg, const int16_t *base)
   return svldnf1sh_u32(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1sh_u64(
+// CHECK-LABEL: @test_svldnf1sh_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldnf1.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldnf1sh_u64u10__SVBool_tPKs(
+// CPP-CHECK-LABEL: @_Z18test_svldnf1sh_u64u10__SVBool_tPKs(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldnf1.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -81,7 +81,7 @@ svuint64_t test_svldnf1sh_u64(svbool_t pg, const int16_t *base)
   return svldnf1sh_u64(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1sh_vnum_s32(
+// CHECK-LABEL: @test_svldnf1sh_vnum_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -89,7 +89,7 @@ svuint64_t test_svldnf1sh_u64(svbool_t pg, const int16_t *base)
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 4 x i16> [[TMP2]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldnf1sh_vnum_s32u10__SVBool_tPKsl(
+// CPP-CHECK-LABEL: @_Z23test_svldnf1sh_vnum_s32u10__SVBool_tPKsl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -102,7 +102,7 @@ svint32_t test_svldnf1sh_vnum_s32(svbool_t pg, const int16_t *base, int64_t vnum
   return svldnf1sh_vnum_s32(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1sh_vnum_s64(
+// CHECK-LABEL: @test_svldnf1sh_vnum_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -110,7 +110,7 @@ svint32_t test_svldnf1sh_vnum_s32(svbool_t pg, const int16_t *base, int64_t vnum
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 2 x i16> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldnf1sh_vnum_s64u10__SVBool_tPKsl(
+// CPP-CHECK-LABEL: @_Z23test_svldnf1sh_vnum_s64u10__SVBool_tPKsl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -123,7 +123,7 @@ svint64_t test_svldnf1sh_vnum_s64(svbool_t pg, const int16_t *base, int64_t vnum
   return svldnf1sh_vnum_s64(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1sh_vnum_u32(
+// CHECK-LABEL: @test_svldnf1sh_vnum_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -131,7 +131,7 @@ svint64_t test_svldnf1sh_vnum_s64(svbool_t pg, const int16_t *base, int64_t vnum
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 4 x i16> [[TMP2]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldnf1sh_vnum_u32u10__SVBool_tPKsl(
+// CPP-CHECK-LABEL: @_Z23test_svldnf1sh_vnum_u32u10__SVBool_tPKsl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -144,7 +144,7 @@ svuint32_t test_svldnf1sh_vnum_u32(svbool_t pg, const int16_t *base, int64_t vnu
   return svldnf1sh_vnum_u32(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1sh_vnum_u64(
+// CHECK-LABEL: @test_svldnf1sh_vnum_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -152,7 +152,7 @@ svuint32_t test_svldnf1sh_vnum_u32(svbool_t pg, const int16_t *base, int64_t vnu
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 2 x i16> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldnf1sh_vnum_u64u10__SVBool_tPKsl(
+// CPP-CHECK-LABEL: @_Z23test_svldnf1sh_vnum_u64u10__SVBool_tPKsl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1sw.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1sw.c
index 7d4dc332d9..812a4bcd11 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1sw.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1sw.c
@@ -5,14 +5,14 @@
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -S -disable-O0-optnone -Werror -o /dev/null %s
 #include <arm_sve.h>
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1sw_s64(
+// CHECK-LABEL: @test_svldnf1sw_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldnf1.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldnf1sw_s64u10__SVBool_tPKi(
+// CPP-CHECK-LABEL: @_Z18test_svldnf1sw_s64u10__SVBool_tPKi(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldnf1.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -24,14 +24,14 @@ svint64_t test_svldnf1sw_s64(svbool_t pg, const int32_t *base)
   return svldnf1sw_s64(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1sw_u64(
+// CHECK-LABEL: @test_svldnf1sw_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldnf1.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = sext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldnf1sw_u64u10__SVBool_tPKi(
+// CPP-CHECK-LABEL: @_Z18test_svldnf1sw_u64u10__SVBool_tPKi(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldnf1.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -43,7 +43,7 @@ svuint64_t test_svldnf1sw_u64(svbool_t pg, const int32_t *base)
   return svldnf1sw_u64(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1sw_vnum_s64(
+// CHECK-LABEL: @test_svldnf1sw_vnum_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -51,7 +51,7 @@ svuint64_t test_svldnf1sw_u64(svbool_t pg, const int32_t *base)
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 2 x i32> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldnf1sw_vnum_s64u10__SVBool_tPKil(
+// CPP-CHECK-LABEL: @_Z23test_svldnf1sw_vnum_s64u10__SVBool_tPKil(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -64,7 +64,7 @@ svint64_t test_svldnf1sw_vnum_s64(svbool_t pg, const int32_t *base, int64_t vnum
   return svldnf1sw_vnum_s64(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1sw_vnum_u64(
+// CHECK-LABEL: @test_svldnf1sw_vnum_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -72,7 +72,7 @@ svint64_t test_svldnf1sw_vnum_s64(svbool_t pg, const int32_t *base, int64_t vnum
 // CHECK-NEXT:    [[TMP3:%.*]] = sext <vscale x 2 x i32> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldnf1sw_vnum_u64u10__SVBool_tPKil(
+// CPP-CHECK-LABEL: @_Z23test_svldnf1sw_vnum_u64u10__SVBool_tPKil(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1ub.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1ub.c
index c7b1655303..119b9ee954 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1ub.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1ub.c
@@ -5,14 +5,14 @@
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -S -disable-O0-optnone -Werror -o /dev/null %s
 #include <arm_sve.h>
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1ub_s16(
+// CHECK-LABEL: @test_svldnf1ub_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i8> @llvm.aarch64.sve.ldnf1.nxv8i8(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 8 x i8> [[TMP1]] to <vscale x 8 x i16>
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldnf1ub_s16u10__SVBool_tPKh(
+// CPP-CHECK-LABEL: @_Z18test_svldnf1ub_s16u10__SVBool_tPKh(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i8> @llvm.aarch64.sve.ldnf1.nxv8i8(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -24,14 +24,14 @@ svint16_t test_svldnf1ub_s16(svbool_t pg, const uint8_t *base)
   return svldnf1ub_s16(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1ub_s32(
+// CHECK-LABEL: @test_svldnf1ub_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldnf1.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldnf1ub_s32u10__SVBool_tPKh(
+// CPP-CHECK-LABEL: @_Z18test_svldnf1ub_s32u10__SVBool_tPKh(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldnf1.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -43,14 +43,14 @@ svint32_t test_svldnf1ub_s32(svbool_t pg, const uint8_t *base)
   return svldnf1ub_s32(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1ub_s64(
+// CHECK-LABEL: @test_svldnf1ub_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldnf1.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldnf1ub_s64u10__SVBool_tPKh(
+// CPP-CHECK-LABEL: @_Z18test_svldnf1ub_s64u10__SVBool_tPKh(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldnf1.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -62,14 +62,14 @@ svint64_t test_svldnf1ub_s64(svbool_t pg, const uint8_t *base)
   return svldnf1ub_s64(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1ub_u16(
+// CHECK-LABEL: @test_svldnf1ub_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i8> @llvm.aarch64.sve.ldnf1.nxv8i8(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 8 x i8> [[TMP1]] to <vscale x 8 x i16>
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldnf1ub_u16u10__SVBool_tPKh(
+// CPP-CHECK-LABEL: @_Z18test_svldnf1ub_u16u10__SVBool_tPKh(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i8> @llvm.aarch64.sve.ldnf1.nxv8i8(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -81,14 +81,14 @@ svuint16_t test_svldnf1ub_u16(svbool_t pg, const uint8_t *base)
   return svldnf1ub_u16(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1ub_u32(
+// CHECK-LABEL: @test_svldnf1ub_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldnf1.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i8> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldnf1ub_u32u10__SVBool_tPKh(
+// CPP-CHECK-LABEL: @_Z18test_svldnf1ub_u32u10__SVBool_tPKh(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i8> @llvm.aarch64.sve.ldnf1.nxv4i8(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -100,14 +100,14 @@ svuint32_t test_svldnf1ub_u32(svbool_t pg, const uint8_t *base)
   return svldnf1ub_u32(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1ub_u64(
+// CHECK-LABEL: @test_svldnf1ub_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldnf1.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i8> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldnf1ub_u64u10__SVBool_tPKh(
+// CPP-CHECK-LABEL: @_Z18test_svldnf1ub_u64u10__SVBool_tPKh(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i8> @llvm.aarch64.sve.ldnf1.nxv2i8(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -119,7 +119,7 @@ svuint64_t test_svldnf1ub_u64(svbool_t pg, const uint8_t *base)
   return svldnf1ub_u64(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1ub_vnum_s16(
+// CHECK-LABEL: @test_svldnf1ub_vnum_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -127,7 +127,7 @@ svuint64_t test_svldnf1ub_u64(svbool_t pg, const uint8_t *base)
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 8 x i8> [[TMP2]] to <vscale x 8 x i16>
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldnf1ub_vnum_s16u10__SVBool_tPKhl(
+// CPP-CHECK-LABEL: @_Z23test_svldnf1ub_vnum_s16u10__SVBool_tPKhl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -140,7 +140,7 @@ svint16_t test_svldnf1ub_vnum_s16(svbool_t pg, const uint8_t *base, int64_t vnum
   return svldnf1ub_vnum_s16(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1ub_vnum_s32(
+// CHECK-LABEL: @test_svldnf1ub_vnum_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -148,7 +148,7 @@ svint16_t test_svldnf1ub_vnum_s16(svbool_t pg, const uint8_t *base, int64_t vnum
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 4 x i8> [[TMP2]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldnf1ub_vnum_s32u10__SVBool_tPKhl(
+// CPP-CHECK-LABEL: @_Z23test_svldnf1ub_vnum_s32u10__SVBool_tPKhl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -161,7 +161,7 @@ svint32_t test_svldnf1ub_vnum_s32(svbool_t pg, const uint8_t *base, int64_t vnum
   return svldnf1ub_vnum_s32(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1ub_vnum_s64(
+// CHECK-LABEL: @test_svldnf1ub_vnum_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -169,7 +169,7 @@ svint32_t test_svldnf1ub_vnum_s32(svbool_t pg, const uint8_t *base, int64_t vnum
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 2 x i8> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldnf1ub_vnum_s64u10__SVBool_tPKhl(
+// CPP-CHECK-LABEL: @_Z23test_svldnf1ub_vnum_s64u10__SVBool_tPKhl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -182,7 +182,7 @@ svint64_t test_svldnf1ub_vnum_s64(svbool_t pg, const uint8_t *base, int64_t vnum
   return svldnf1ub_vnum_s64(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1ub_vnum_u16(
+// CHECK-LABEL: @test_svldnf1ub_vnum_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -190,7 +190,7 @@ svint64_t test_svldnf1ub_vnum_s64(svbool_t pg, const uint8_t *base, int64_t vnum
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 8 x i8> [[TMP2]] to <vscale x 8 x i16>
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldnf1ub_vnum_u16u10__SVBool_tPKhl(
+// CPP-CHECK-LABEL: @_Z23test_svldnf1ub_vnum_u16u10__SVBool_tPKhl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -203,7 +203,7 @@ svuint16_t test_svldnf1ub_vnum_u16(svbool_t pg, const uint8_t *base, int64_t vnu
   return svldnf1ub_vnum_u16(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1ub_vnum_u32(
+// CHECK-LABEL: @test_svldnf1ub_vnum_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -211,7 +211,7 @@ svuint16_t test_svldnf1ub_vnum_u16(svbool_t pg, const uint8_t *base, int64_t vnu
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 4 x i8> [[TMP2]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldnf1ub_vnum_u32u10__SVBool_tPKhl(
+// CPP-CHECK-LABEL: @_Z23test_svldnf1ub_vnum_u32u10__SVBool_tPKhl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -224,7 +224,7 @@ svuint32_t test_svldnf1ub_vnum_u32(svbool_t pg, const uint8_t *base, int64_t vnu
   return svldnf1ub_vnum_u32(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1ub_vnum_u64(
+// CHECK-LABEL: @test_svldnf1ub_vnum_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -232,7 +232,7 @@ svuint32_t test_svldnf1ub_vnum_u32(svbool_t pg, const uint8_t *base, int64_t vnu
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 2 x i8> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldnf1ub_vnum_u64u10__SVBool_tPKhl(
+// CPP-CHECK-LABEL: @_Z23test_svldnf1ub_vnum_u64u10__SVBool_tPKhl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1uh.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1uh.c
index e27bd96e6b..f2bb7982d7 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1uh.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1uh.c
@@ -5,14 +5,14 @@
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -S -disable-O0-optnone -Werror -o /dev/null %s
 #include <arm_sve.h>
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1uh_s32(
+// CHECK-LABEL: @test_svldnf1uh_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldnf1.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldnf1uh_s32u10__SVBool_tPKt(
+// CPP-CHECK-LABEL: @_Z18test_svldnf1uh_s32u10__SVBool_tPKt(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldnf1.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -24,14 +24,14 @@ svint32_t test_svldnf1uh_s32(svbool_t pg, const uint16_t *base)
   return svldnf1uh_s32(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1uh_s64(
+// CHECK-LABEL: @test_svldnf1uh_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldnf1.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldnf1uh_s64u10__SVBool_tPKt(
+// CPP-CHECK-LABEL: @_Z18test_svldnf1uh_s64u10__SVBool_tPKt(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldnf1.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -43,14 +43,14 @@ svint64_t test_svldnf1uh_s64(svbool_t pg, const uint16_t *base)
   return svldnf1uh_s64(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1uh_u32(
+// CHECK-LABEL: @test_svldnf1uh_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldnf1.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 4 x i16> [[TMP1]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldnf1uh_u32u10__SVBool_tPKt(
+// CPP-CHECK-LABEL: @_Z18test_svldnf1uh_u32u10__SVBool_tPKt(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i16> @llvm.aarch64.sve.ldnf1.nxv4i16(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -62,14 +62,14 @@ svuint32_t test_svldnf1uh_u32(svbool_t pg, const uint16_t *base)
   return svldnf1uh_u32(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1uh_u64(
+// CHECK-LABEL: @test_svldnf1uh_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldnf1.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i16> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldnf1uh_u64u10__SVBool_tPKt(
+// CPP-CHECK-LABEL: @_Z18test_svldnf1uh_u64u10__SVBool_tPKt(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i16> @llvm.aarch64.sve.ldnf1.nxv2i16(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -81,7 +81,7 @@ svuint64_t test_svldnf1uh_u64(svbool_t pg, const uint16_t *base)
   return svldnf1uh_u64(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1uh_vnum_s32(
+// CHECK-LABEL: @test_svldnf1uh_vnum_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -89,7 +89,7 @@ svuint64_t test_svldnf1uh_u64(svbool_t pg, const uint16_t *base)
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 4 x i16> [[TMP2]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldnf1uh_vnum_s32u10__SVBool_tPKtl(
+// CPP-CHECK-LABEL: @_Z23test_svldnf1uh_vnum_s32u10__SVBool_tPKtl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -102,7 +102,7 @@ svint32_t test_svldnf1uh_vnum_s32(svbool_t pg, const uint16_t *base, int64_t vnu
   return svldnf1uh_vnum_s32(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1uh_vnum_s64(
+// CHECK-LABEL: @test_svldnf1uh_vnum_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -110,7 +110,7 @@ svint32_t test_svldnf1uh_vnum_s32(svbool_t pg, const uint16_t *base, int64_t vnu
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 2 x i16> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldnf1uh_vnum_s64u10__SVBool_tPKtl(
+// CPP-CHECK-LABEL: @_Z23test_svldnf1uh_vnum_s64u10__SVBool_tPKtl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -123,7 +123,7 @@ svint64_t test_svldnf1uh_vnum_s64(svbool_t pg, const uint16_t *base, int64_t vnu
   return svldnf1uh_vnum_s64(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1uh_vnum_u32(
+// CHECK-LABEL: @test_svldnf1uh_vnum_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -131,7 +131,7 @@ svint64_t test_svldnf1uh_vnum_s64(svbool_t pg, const uint16_t *base, int64_t vnu
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 4 x i16> [[TMP2]] to <vscale x 4 x i32>
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldnf1uh_vnum_u32u10__SVBool_tPKtl(
+// CPP-CHECK-LABEL: @_Z23test_svldnf1uh_vnum_u32u10__SVBool_tPKtl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -144,7 +144,7 @@ svuint32_t test_svldnf1uh_vnum_u32(svbool_t pg, const uint16_t *base, int64_t vn
   return svldnf1uh_vnum_u32(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1uh_vnum_u64(
+// CHECK-LABEL: @test_svldnf1uh_vnum_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -152,7 +152,7 @@ svuint32_t test_svldnf1uh_vnum_u32(svbool_t pg, const uint16_t *base, int64_t vn
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 2 x i16> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldnf1uh_vnum_u64u10__SVBool_tPKtl(
+// CPP-CHECK-LABEL: @_Z23test_svldnf1uh_vnum_u64u10__SVBool_tPKtl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1uw.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1uw.c
index 6d459132e8..2e8fc48638 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1uw.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnf1uw.c
@@ -5,14 +5,14 @@
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -S -disable-O0-optnone -Werror -o /dev/null %s
 #include <arm_sve.h>
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1uw_s64(
+// CHECK-LABEL: @test_svldnf1uw_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldnf1.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldnf1uw_s64u10__SVBool_tPKj(
+// CPP-CHECK-LABEL: @_Z18test_svldnf1uw_s64u10__SVBool_tPKj(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldnf1.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -24,14 +24,14 @@ svint64_t test_svldnf1uw_s64(svbool_t pg, const uint32_t *base)
   return svldnf1uw_s64(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1uw_u64(
+// CHECK-LABEL: @test_svldnf1uw_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldnf1.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    [[TMP2:%.*]] = zext <vscale x 2 x i32> [[TMP1]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svldnf1uw_u64u10__SVBool_tPKj(
+// CPP-CHECK-LABEL: @_Z18test_svldnf1uw_u64u10__SVBool_tPKj(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i32> @llvm.aarch64.sve.ldnf1.nxv2i32(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -43,7 +43,7 @@ svuint64_t test_svldnf1uw_u64(svbool_t pg, const uint32_t *base)
   return svldnf1uw_u64(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1uw_vnum_s64(
+// CHECK-LABEL: @test_svldnf1uw_vnum_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -51,7 +51,7 @@ svuint64_t test_svldnf1uw_u64(svbool_t pg, const uint32_t *base)
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 2 x i32> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldnf1uw_vnum_s64u10__SVBool_tPKjl(
+// CPP-CHECK-LABEL: @_Z23test_svldnf1uw_vnum_s64u10__SVBool_tPKjl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -64,7 +64,7 @@ svint64_t test_svldnf1uw_vnum_s64(svbool_t pg, const uint32_t *base, int64_t vnu
   return svldnf1uw_vnum_s64(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnf1uw_vnum_u64(
+// CHECK-LABEL: @test_svldnf1uw_vnum_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -72,7 +72,7 @@ svint64_t test_svldnf1uw_vnum_s64(svbool_t pg, const uint32_t *base, int64_t vnu
 // CHECK-NEXT:    [[TMP3:%.*]] = zext <vscale x 2 x i32> [[TMP2]] to <vscale x 2 x i64>
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP3]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z23test_svldnf1uw_vnum_u64u10__SVBool_tPKjl(
+// CPP-CHECK-LABEL: @_Z23test_svldnf1uw_vnum_u64u10__SVBool_tPKjl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnt1-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnt1-bfloat.c
index 7d0fdbe42b..2ce42fe441 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnt1-bfloat.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnt1-bfloat.c
@@ -15,13 +15,13 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnt1_bf16(
+// CHECK-LABEL: @test_svldnt1_bf16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.ldnt1.nxv8bf16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z17test_svldnt1_bf16u10__SVBool_tPKu6__bf16(
+// CPP-CHECK-LABEL: @_Z17test_svldnt1_bf16u10__SVBool_tPKu6__bf16(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.ldnt1.nxv8bf16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -32,14 +32,14 @@ svbfloat16_t test_svldnt1_bf16(svbool_t pg, const bfloat16_t *base)
   return SVE_ACLE_FUNC(svldnt1,_bf16,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnt1_vnum_bf16(
+// CHECK-LABEL: @test_svldnt1_vnum_bf16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x bfloat>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.ldnt1.nxv8bf16(<vscale x 8 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 8 x bfloat> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z22test_svldnt1_vnum_bf16u10__SVBool_tPKu6__bf16l(
+// CPP-CHECK-LABEL: @_Z22test_svldnt1_vnum_bf16u10__SVBool_tPKu6__bf16l(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x bfloat>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnt1.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnt1.c
index bec42ce14e..d5d77ebed5 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnt1.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_ldnt1.c
@@ -15,12 +15,12 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnt1_s8(
+// CHECK-LABEL: @test_svldnt1_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldnt1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z15test_svldnt1_s8u10__SVBool_tPKa(
+// CPP-CHECK-LABEL: @_Z15test_svldnt1_s8u10__SVBool_tPKa(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldnt1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
@@ -30,13 +30,13 @@ svint8_t test_svldnt1_s8(svbool_t pg, const int8_t *base)
   return SVE_ACLE_FUNC(svldnt1,_s8,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnt1_s16(
+// CHECK-LABEL: @test_svldnt1_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.ldnt1.nxv8i16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svldnt1_s16u10__SVBool_tPKs(
+// CPP-CHECK-LABEL: @_Z16test_svldnt1_s16u10__SVBool_tPKs(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.ldnt1.nxv8i16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -47,13 +47,13 @@ svint16_t test_svldnt1_s16(svbool_t pg, const int16_t *base)
   return SVE_ACLE_FUNC(svldnt1,_s16,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnt1_s32(
+// CHECK-LABEL: @test_svldnt1_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldnt1.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svldnt1_s32u10__SVBool_tPKi(
+// CPP-CHECK-LABEL: @_Z16test_svldnt1_s32u10__SVBool_tPKi(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldnt1.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -64,13 +64,13 @@ svint32_t test_svldnt1_s32(svbool_t pg, const int32_t *base)
   return SVE_ACLE_FUNC(svldnt1,_s32,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnt1_s64(
+// CHECK-LABEL: @test_svldnt1_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldnt1.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svldnt1_s64u10__SVBool_tPKl(
+// CPP-CHECK-LABEL: @_Z16test_svldnt1_s64u10__SVBool_tPKl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldnt1.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -81,12 +81,12 @@ svint64_t test_svldnt1_s64(svbool_t pg, const int64_t *base)
   return SVE_ACLE_FUNC(svldnt1,_s64,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnt1_u8(
+// CHECK-LABEL: @test_svldnt1_u8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldnt1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z15test_svldnt1_u8u10__SVBool_tPKh(
+// CPP-CHECK-LABEL: @_Z15test_svldnt1_u8u10__SVBool_tPKh(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldnt1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
 // CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
@@ -96,13 +96,13 @@ svuint8_t test_svldnt1_u8(svbool_t pg, const uint8_t *base)
   return SVE_ACLE_FUNC(svldnt1,_u8,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnt1_u16(
+// CHECK-LABEL: @test_svldnt1_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.ldnt1.nxv8i16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svldnt1_u16u10__SVBool_tPKt(
+// CPP-CHECK-LABEL: @_Z16test_svldnt1_u16u10__SVBool_tPKt(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.ldnt1.nxv8i16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -113,13 +113,13 @@ svuint16_t test_svldnt1_u16(svbool_t pg, const uint16_t *base)
   return SVE_ACLE_FUNC(svldnt1,_u16,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnt1_u32(
+// CHECK-LABEL: @test_svldnt1_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldnt1.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svldnt1_u32u10__SVBool_tPKj(
+// CPP-CHECK-LABEL: @_Z16test_svldnt1_u32u10__SVBool_tPKj(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldnt1.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -130,13 +130,13 @@ svuint32_t test_svldnt1_u32(svbool_t pg, const uint32_t *base)
   return SVE_ACLE_FUNC(svldnt1,_u32,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnt1_u64(
+// CHECK-LABEL: @test_svldnt1_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldnt1.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svldnt1_u64u10__SVBool_tPKm(
+// CPP-CHECK-LABEL: @_Z16test_svldnt1_u64u10__SVBool_tPKm(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldnt1.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -147,13 +147,13 @@ svuint64_t test_svldnt1_u64(svbool_t pg, const uint64_t *base)
   return SVE_ACLE_FUNC(svldnt1,_u64,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnt1_f16(
+// CHECK-LABEL: @test_svldnt1_f16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sve.ldnt1.nxv8f16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret <vscale x 8 x half> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svldnt1_f16u10__SVBool_tPKDh(
+// CPP-CHECK-LABEL: @_Z16test_svldnt1_f16u10__SVBool_tPKDh(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sve.ldnt1.nxv8f16(<vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -164,13 +164,13 @@ svfloat16_t test_svldnt1_f16(svbool_t pg, const float16_t *base)
   return SVE_ACLE_FUNC(svldnt1,_f16,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnt1_f32(
+// CHECK-LABEL: @test_svldnt1_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.ldnt1.nxv4f32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svldnt1_f32u10__SVBool_tPKf(
+// CPP-CHECK-LABEL: @_Z16test_svldnt1_f32u10__SVBool_tPKf(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.ldnt1.nxv4f32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -181,13 +181,13 @@ svfloat32_t test_svldnt1_f32(svbool_t pg, const float32_t *base)
   return SVE_ACLE_FUNC(svldnt1,_f32,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnt1_f64(
+// CHECK-LABEL: @test_svldnt1_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.ldnt1.nxv2f64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret <vscale x 2 x double> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svldnt1_f64u10__SVBool_tPKd(
+// CPP-CHECK-LABEL: @_Z16test_svldnt1_f64u10__SVBool_tPKd(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.ldnt1.nxv2f64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -198,13 +198,13 @@ svfloat64_t test_svldnt1_f64(svbool_t pg, const float64_t *base)
   return SVE_ACLE_FUNC(svldnt1,_f64,,)(pg, base);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnt1_vnum_s8(
+// CHECK-LABEL: @test_svldnt1_vnum_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldnt1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z20test_svldnt1_vnum_s8u10__SVBool_tPKal(
+// CPP-CHECK-LABEL: @_Z20test_svldnt1_vnum_s8u10__SVBool_tPKal(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldnt1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
@@ -215,14 +215,14 @@ svint8_t test_svldnt1_vnum_s8(svbool_t pg, const int8_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svldnt1_vnum,_s8,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnt1_vnum_s16(
+// CHECK-LABEL: @test_svldnt1_vnum_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.ldnt1.nxv8i16(<vscale x 8 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svldnt1_vnum_s16u10__SVBool_tPKsl(
+// CPP-CHECK-LABEL: @_Z21test_svldnt1_vnum_s16u10__SVBool_tPKsl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -234,14 +234,14 @@ svint16_t test_svldnt1_vnum_s16(svbool_t pg, const int16_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svldnt1_vnum,_s16,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnt1_vnum_s32(
+// CHECK-LABEL: @test_svldnt1_vnum_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldnt1.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svldnt1_vnum_s32u10__SVBool_tPKil(
+// CPP-CHECK-LABEL: @_Z21test_svldnt1_vnum_s32u10__SVBool_tPKil(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -253,14 +253,14 @@ svint32_t test_svldnt1_vnum_s32(svbool_t pg, const int32_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svldnt1_vnum,_s32,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnt1_vnum_s64(
+// CHECK-LABEL: @test_svldnt1_vnum_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldnt1.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svldnt1_vnum_s64u10__SVBool_tPKll(
+// CPP-CHECK-LABEL: @_Z21test_svldnt1_vnum_s64u10__SVBool_tPKll(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -272,13 +272,13 @@ svint64_t test_svldnt1_vnum_s64(svbool_t pg, const int64_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svldnt1_vnum,_s64,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnt1_vnum_u8(
+// CHECK-LABEL: @test_svldnt1_vnum_u8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldnt1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
 // CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z20test_svldnt1_vnum_u8u10__SVBool_tPKhl(
+// CPP-CHECK-LABEL: @_Z20test_svldnt1_vnum_u8u10__SVBool_tPKhl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldnt1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
@@ -289,14 +289,14 @@ svuint8_t test_svldnt1_vnum_u8(svbool_t pg, const uint8_t *base, int64_t vnum)
   return SVE_ACLE_FUNC(svldnt1_vnum,_u8,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnt1_vnum_u16(
+// CHECK-LABEL: @test_svldnt1_vnum_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.ldnt1.nxv8i16(<vscale x 8 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svldnt1_vnum_u16u10__SVBool_tPKtl(
+// CPP-CHECK-LABEL: @_Z21test_svldnt1_vnum_u16u10__SVBool_tPKtl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -308,14 +308,14 @@ svuint16_t test_svldnt1_vnum_u16(svbool_t pg, const uint16_t *base, int64_t vnum
   return SVE_ACLE_FUNC(svldnt1_vnum,_u16,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnt1_vnum_u32(
+// CHECK-LABEL: @test_svldnt1_vnum_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ldnt1.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svldnt1_vnum_u32u10__SVBool_tPKjl(
+// CPP-CHECK-LABEL: @_Z21test_svldnt1_vnum_u32u10__SVBool_tPKjl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -327,14 +327,14 @@ svuint32_t test_svldnt1_vnum_u32(svbool_t pg, const uint32_t *base, int64_t vnum
   return SVE_ACLE_FUNC(svldnt1_vnum,_u32,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnt1_vnum_u64(
+// CHECK-LABEL: @test_svldnt1_vnum_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ldnt1.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svldnt1_vnum_u64u10__SVBool_tPKml(
+// CPP-CHECK-LABEL: @_Z21test_svldnt1_vnum_u64u10__SVBool_tPKml(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -346,14 +346,14 @@ svuint64_t test_svldnt1_vnum_u64(svbool_t pg, const uint64_t *base, int64_t vnum
   return SVE_ACLE_FUNC(svldnt1_vnum,_u64,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnt1_vnum_f16(
+// CHECK-LABEL: @test_svldnt1_vnum_f16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sve.ldnt1.nxv8f16(<vscale x 8 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 8 x half> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svldnt1_vnum_f16u10__SVBool_tPKDhl(
+// CPP-CHECK-LABEL: @_Z21test_svldnt1_vnum_f16u10__SVBool_tPKDhl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -365,14 +365,14 @@ svfloat16_t test_svldnt1_vnum_f16(svbool_t pg, const float16_t *base, int64_t vn
   return SVE_ACLE_FUNC(svldnt1_vnum,_f16,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnt1_vnum_f32(
+// CHECK-LABEL: @test_svldnt1_vnum_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.ldnt1.nxv4f32(<vscale x 4 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 4 x float> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svldnt1_vnum_f32u10__SVBool_tPKfl(
+// CPP-CHECK-LABEL: @_Z21test_svldnt1_vnum_f32u10__SVBool_tPKfl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -384,14 +384,14 @@ svfloat32_t test_svldnt1_vnum_f32(svbool_t pg, const float32_t *base, int64_t vn
   return SVE_ACLE_FUNC(svldnt1_vnum,_f32,,)(pg, base, vnum);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svldnt1_vnum_f64(
+// CHECK-LABEL: @test_svldnt1_vnum_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x double> @llvm.aarch64.sve.ldnt1.nxv2f64(<vscale x 2 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret <vscale x 2 x double> [[TMP2]]
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svldnt1_vnum_f64u10__SVBool_tPKdl(
+// CPP-CHECK-LABEL: @_Z21test_svldnt1_vnum_f64u10__SVBool_tPKdl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfb.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfb.c
index 96179f9d7d..4c584de625 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfb.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfb.c
@@ -14,12 +14,12 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svprfb(
+// CHECK-LABEL: @test_svprfb(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv16i1(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]], i32 0)
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z11test_svprfbu10__SVBool_tPKv(
+// CPP-CHECK-LABEL: @_Z11test_svprfbu10__SVBool_tPKv(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv16i1(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]], i32 0)
 // CPP-CHECK-NEXT:    ret void
@@ -29,12 +29,12 @@ void test_svprfb(svbool_t pg, const void *base)
   return svprfb(pg, base, SV_PLDL1KEEP);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svprfb_1(
+// CHECK-LABEL: @test_svprfb_1(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv16i1(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]], i32 1)
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z13test_svprfb_1u10__SVBool_tPKv(
+// CPP-CHECK-LABEL: @_Z13test_svprfb_1u10__SVBool_tPKv(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv16i1(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]], i32 1)
 // CPP-CHECK-NEXT:    ret void
@@ -44,12 +44,12 @@ void test_svprfb_1(svbool_t pg, const void *base)
   return svprfb(pg, base, SV_PLDL1STRM);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svprfb_2(
+// CHECK-LABEL: @test_svprfb_2(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv16i1(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]], i32 2)
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z13test_svprfb_2u10__SVBool_tPKv(
+// CPP-CHECK-LABEL: @_Z13test_svprfb_2u10__SVBool_tPKv(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv16i1(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]], i32 2)
 // CPP-CHECK-NEXT:    ret void
@@ -59,12 +59,12 @@ void test_svprfb_2(svbool_t pg, const void *base)
   return svprfb(pg, base, SV_PLDL2KEEP);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svprfb_3(
+// CHECK-LABEL: @test_svprfb_3(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv16i1(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]], i32 3)
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z13test_svprfb_3u10__SVBool_tPKv(
+// CPP-CHECK-LABEL: @_Z13test_svprfb_3u10__SVBool_tPKv(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv16i1(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]], i32 3)
 // CPP-CHECK-NEXT:    ret void
@@ -74,12 +74,12 @@ void test_svprfb_3(svbool_t pg, const void *base)
   return svprfb(pg, base, SV_PLDL2STRM);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svprfb_4(
+// CHECK-LABEL: @test_svprfb_4(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv16i1(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]], i32 4)
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z13test_svprfb_4u10__SVBool_tPKv(
+// CPP-CHECK-LABEL: @_Z13test_svprfb_4u10__SVBool_tPKv(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv16i1(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]], i32 4)
 // CPP-CHECK-NEXT:    ret void
@@ -89,12 +89,12 @@ void test_svprfb_4(svbool_t pg, const void *base)
   return svprfb(pg, base, SV_PLDL3KEEP);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svprfb_5(
+// CHECK-LABEL: @test_svprfb_5(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv16i1(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]], i32 5)
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z13test_svprfb_5u10__SVBool_tPKv(
+// CPP-CHECK-LABEL: @_Z13test_svprfb_5u10__SVBool_tPKv(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv16i1(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]], i32 5)
 // CPP-CHECK-NEXT:    ret void
@@ -104,12 +104,12 @@ void test_svprfb_5(svbool_t pg, const void *base)
   return svprfb(pg, base, SV_PLDL3STRM);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svprfb_6(
+// CHECK-LABEL: @test_svprfb_6(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv16i1(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]], i32 8)
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z13test_svprfb_6u10__SVBool_tPKv(
+// CPP-CHECK-LABEL: @_Z13test_svprfb_6u10__SVBool_tPKv(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv16i1(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]], i32 8)
 // CPP-CHECK-NEXT:    ret void
@@ -119,12 +119,12 @@ void test_svprfb_6(svbool_t pg, const void *base)
   return svprfb(pg, base, SV_PSTL1KEEP);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svprfb_7(
+// CHECK-LABEL: @test_svprfb_7(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv16i1(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]], i32 9)
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z13test_svprfb_7u10__SVBool_tPKv(
+// CPP-CHECK-LABEL: @_Z13test_svprfb_7u10__SVBool_tPKv(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv16i1(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]], i32 9)
 // CPP-CHECK-NEXT:    ret void
@@ -134,12 +134,12 @@ void test_svprfb_7(svbool_t pg, const void *base)
   return svprfb(pg, base, SV_PSTL1STRM);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svprfb_8(
+// CHECK-LABEL: @test_svprfb_8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv16i1(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]], i32 10)
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z13test_svprfb_8u10__SVBool_tPKv(
+// CPP-CHECK-LABEL: @_Z13test_svprfb_8u10__SVBool_tPKv(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv16i1(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]], i32 10)
 // CPP-CHECK-NEXT:    ret void
@@ -149,12 +149,12 @@ void test_svprfb_8(svbool_t pg, const void *base)
   return svprfb(pg, base, SV_PSTL2KEEP);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svprfb_9(
+// CHECK-LABEL: @test_svprfb_9(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv16i1(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]], i32 11)
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z13test_svprfb_9u10__SVBool_tPKv(
+// CPP-CHECK-LABEL: @_Z13test_svprfb_9u10__SVBool_tPKv(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv16i1(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]], i32 11)
 // CPP-CHECK-NEXT:    ret void
@@ -164,12 +164,12 @@ void test_svprfb_9(svbool_t pg, const void *base)
   return svprfb(pg, base, SV_PSTL2STRM);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svprfb_10(
+// CHECK-LABEL: @test_svprfb_10(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv16i1(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]], i32 12)
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svprfb_10u10__SVBool_tPKv(
+// CPP-CHECK-LABEL: @_Z14test_svprfb_10u10__SVBool_tPKv(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv16i1(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]], i32 12)
 // CPP-CHECK-NEXT:    ret void
@@ -179,12 +179,12 @@ void test_svprfb_10(svbool_t pg, const void *base)
   return svprfb(pg, base, SV_PSTL3KEEP);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svprfb_11(
+// CHECK-LABEL: @test_svprfb_11(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv16i1(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]], i32 13)
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svprfb_11u10__SVBool_tPKv(
+// CPP-CHECK-LABEL: @_Z14test_svprfb_11u10__SVBool_tPKv(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv16i1(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]], i32 13)
 // CPP-CHECK-NEXT:    ret void
@@ -194,13 +194,13 @@ void test_svprfb_11(svbool_t pg, const void *base)
   return svprfb(pg, base, SV_PSTL3STRM);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svprfb_vnum(
+// CHECK-LABEL: @test_svprfb_vnum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv16i1(<vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]], i32 0)
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svprfb_vnumu10__SVBool_tPKvl(
+// CPP-CHECK-LABEL: @_Z16test_svprfb_vnumu10__SVBool_tPKvl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv16i1(<vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]], i32 0)
@@ -211,13 +211,13 @@ void test_svprfb_vnum(svbool_t pg, const void *base, int64_t vnum)
   return svprfb_vnum(pg, base, vnum, SV_PLDL1KEEP);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svprfb_gather_u32base(
+// CHECK-LABEL: @test_svprfb_gather_u32base(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.prfb.gather.scalar.offset.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0, i32 0)
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z26test_svprfb_gather_u32baseu10__SVBool_tu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z26test_svprfb_gather_u32baseu10__SVBool_tu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.prfb.gather.scalar.offset.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0, i32 0)
@@ -228,13 +228,13 @@ void test_svprfb_gather_u32base(svbool_t pg, svuint32_t bases)
   return SVE_ACLE_FUNC(svprfb_gather,_u32base,,)(pg, bases, SV_PLDL1KEEP);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svprfb_gather_u64base(
+// CHECK-LABEL: @test_svprfb_gather_u64base(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.prfb.gather.scalar.offset.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0, i32 0)
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z26test_svprfb_gather_u64baseu10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z26test_svprfb_gather_u64baseu10__SVBool_tu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.prfb.gather.scalar.offset.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0, i32 0)
@@ -245,13 +245,13 @@ void test_svprfb_gather_u64base(svbool_t pg, svuint64_t bases)
   return SVE_ACLE_FUNC(svprfb_gather,_u64base,,)(pg, bases, SV_PLDL1KEEP);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svprfb_gather_s32offset(
+// CHECK-LABEL: @test_svprfb_gather_s32offset(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.prfb.gather.sxtw.index.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]], i32 0)
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z28test_svprfb_gather_s32offsetu10__SVBool_tPKvu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z28test_svprfb_gather_s32offsetu10__SVBool_tPKvu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.prfb.gather.sxtw.index.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]], i32 0)
@@ -262,13 +262,13 @@ void test_svprfb_gather_s32offset(svbool_t pg, const void *base, svint32_t offse
   return SVE_ACLE_FUNC(svprfb_gather_,s32,offset,)(pg, base, offsets, SV_PLDL1KEEP);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svprfb_gather_s64offset(
+// CHECK-LABEL: @test_svprfb_gather_s64offset(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.prfb.gather.index.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]], i32 0)
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z28test_svprfb_gather_s64offsetu10__SVBool_tPKvu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z28test_svprfb_gather_s64offsetu10__SVBool_tPKvu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.prfb.gather.index.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]], i32 0)
@@ -279,13 +279,13 @@ void test_svprfb_gather_s64offset(svbool_t pg, const void *base, svint64_t offse
   return SVE_ACLE_FUNC(svprfb_gather_,s64,offset,)(pg, base, offsets, SV_PLDL1KEEP);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svprfb_gather_u32offset(
+// CHECK-LABEL: @test_svprfb_gather_u32offset(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.prfb.gather.uxtw.index.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]], i32 0)
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z28test_svprfb_gather_u32offsetu10__SVBool_tPKvu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z28test_svprfb_gather_u32offsetu10__SVBool_tPKvu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.prfb.gather.uxtw.index.nxv4i32(<vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]], i32 0)
@@ -296,13 +296,13 @@ void test_svprfb_gather_u32offset(svbool_t pg, const void *base, svuint32_t offs
   return SVE_ACLE_FUNC(svprfb_gather_,u32,offset,)(pg, base, offsets, SV_PLDL1KEEP);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svprfb_gather_u64offset(
+// CHECK-LABEL: @test_svprfb_gather_u64offset(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.prfb.gather.index.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]], i32 0)
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z28test_svprfb_gather_u64offsetu10__SVBool_tPKvu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z28test_svprfb_gather_u64offsetu10__SVBool_tPKvu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.prfb.gather.index.nxv2i64(<vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]], i32 0)
@@ -313,13 +313,13 @@ void test_svprfb_gather_u64offset(svbool_t pg, const void *base, svuint64_t offs
   return SVE_ACLE_FUNC(svprfb_gather_,u64,offset,)(pg, base, offsets, SV_PLDL1KEEP);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svprfb_gather_u32base_offset(
+// CHECK-LABEL: @test_svprfb_gather_u32base_offset(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.prfb.gather.scalar.offset.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]], i32 0)
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svprfb_gather_u32base_offsetu10__SVBool_tu12__SVUint32_tl(
+// CPP-CHECK-LABEL: @_Z33test_svprfb_gather_u32base_offsetu10__SVBool_tu12__SVUint32_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.prfb.gather.scalar.offset.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]], i32 0)
@@ -331,13 +331,13 @@ void test_svprfb_gather_u32base_offset(svbool_t pg, svuint32_t bases, int64_t of
   return SVE_ACLE_FUNC(svprfb_gather,_u32base,_offset,)(pg, bases, offset, SV_PLDL1KEEP);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svprfb_gather_u64base_offset(
+// CHECK-LABEL: @test_svprfb_gather_u64base_offset(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.prfb.gather.scalar.offset.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]], i32 0)
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z33test_svprfb_gather_u64base_offsetu10__SVBool_tu12__SVUint64_tl(
+// CPP-CHECK-LABEL: @_Z33test_svprfb_gather_u64base_offsetu10__SVBool_tu12__SVUint64_tl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.prfb.gather.scalar.offset.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]], i32 0)
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfd.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfd.c
index 8c91341fa8..4034a2b987 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfd.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfd.c
@@ -221,15 +221,15 @@ void test_svprfd_11(svbool_t pg, const void *base)
 // CHECK-LABEL: @test_svprfd_vnum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
-// CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv2i1(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv2i1(<vscale x 2 x i1> [[TMP0]], ptr [[TMP1]], i32 0)
 // CHECK-NEXT:    ret void
 //
 // CPP-CHECK-LABEL: @_Z16test_svprfd_vnumu10__SVBool_tPKvl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
-// CPP-CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
-// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv2i1(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv2i1(<vscale x 2 x i1> [[TMP0]], ptr [[TMP1]], i32 0)
 // CPP-CHECK-NEXT:    ret void
 //
 void test_svprfd_vnum(svbool_t pg, const void *base, int64_t vnum)
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfh.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfh.c
index 538fe9cdf7..adb4a051f9 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfh.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfh.c
@@ -221,15 +221,15 @@ void test_svprfh_11(svbool_t pg, const void *base)
 // CHECK-LABEL: @test_svprfh_vnum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
-// CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv8i1(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv8i1(<vscale x 8 x i1> [[TMP0]], ptr [[TMP1]], i32 0)
 // CHECK-NEXT:    ret void
 //
 // CPP-CHECK-LABEL: @_Z16test_svprfh_vnumu10__SVBool_tPKvl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
-// CPP-CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
-// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv8i1(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv8i1(<vscale x 8 x i1> [[TMP0]], ptr [[TMP1]], i32 0)
 // CPP-CHECK-NEXT:    ret void
 //
 void test_svprfh_vnum(svbool_t pg, const void *base, int64_t vnum)
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfw.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfw.c
index 8f00aefe61..13b63f7a2d 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfw.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfw.c
@@ -221,15 +221,15 @@ void test_svprfw_11(svbool_t pg, const void *base)
 // CHECK-LABEL: @test_svprfw_vnum(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
-// CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv4i1(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv4i1(<vscale x 4 x i1> [[TMP0]], ptr [[TMP1]], i32 0)
 // CHECK-NEXT:    ret void
 //
 // CPP-CHECK-LABEL: @_Z16test_svprfw_vnumu10__SVBool_tPKvl(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
-// CPP-CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
-// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv4i1(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.prf.nxv4i1(<vscale x 4 x i1> [[TMP0]], ptr [[TMP1]], i32 0)
 // CPP-CHECK-NEXT:    ret void
 //
 void test_svprfw_vnum(svbool_t pg, const void *base, int64_t vnum)
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_reinterpret.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_reinterpret.c
index b58be9ad39..c4fe461c70 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_reinterpret.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_reinterpret.c
@@ -367,6 +367,7 @@ svint32_t test_svreinterpret_s32_s16(svint16_t op)
   return SVE_ACLE_FUNC(svreinterpret_s32,_s16,,)(op);
 }
 
+//
 // CHECK-LABEL: @test_svreinterpret_s32_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret <vscale x 4 x i32> [[OP:%.*]]
@@ -498,7 +499,6 @@ svint32_t test_svreinterpret_s32_f64(svfloat64_t op)
   return SVE_ACLE_FUNC(svreinterpret_s32,_f64,,)(op);
 }
 
-//
 // CHECK-LABEL: @test_svreinterpret_s64_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = bitcast <vscale x 16 x i8> [[OP:%.*]] to <vscale x 2 x i64>
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_setffr.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_setffr.c
index d5021da8af..dad5c592a3 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_setffr.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_setffr.c
@@ -7,12 +7,12 @@
 
 // CHECK-LABEL: @test_svsetffr(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    call void @llvm.aarch64.sve.setffr()
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.setffr()
 // CHECK-NEXT:    ret void
 //
 // CPP-CHECK-LABEL: @_Z13test_svsetffrv(
 // CPP-CHECK-NEXT:  entry:
-// CPP-CHECK-NEXT:    call void @llvm.aarch64.sve.setffr()
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.setffr()
 // CPP-CHECK-NEXT:    ret void
 //
 void test_svsetffr()
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1-bfloat.c
index f9ddbf9697..f64deb9553 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1-bfloat.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1-bfloat.c
@@ -15,13 +15,13 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_bf16(
+// CHECK-LABEL: @test_svst1_bf16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.masked.store.nxv8bf16.p0(<vscale x 8 x bfloat> [[DATA:%.*]], ptr [[BASE:%.*]], i32 1, <vscale x 8 x i1> [[TMP0]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z15test_svst1_bf16u10__SVBool_tPu6__bf16u14__SVBFloat16_t(
+// CPP-CHECK-LABEL: @_Z15test_svst1_bf16u10__SVBool_tPu6__bf16u14__SVBFloat16_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.masked.store.nxv8bf16.p0(<vscale x 8 x bfloat> [[DATA:%.*]], ptr [[BASE:%.*]], i32 1, <vscale x 8 x i1> [[TMP0]])
@@ -32,14 +32,14 @@ void test_svst1_bf16(svbool_t pg, bfloat16_t *base, svbfloat16_t data)
   return SVE_ACLE_FUNC(svst1,_bf16,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_vnum_bf16(
+// CHECK-LABEL: @test_svst1_vnum_bf16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x bfloat>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    tail call void @llvm.masked.store.nxv8bf16.p0(<vscale x 8 x bfloat> [[DATA:%.*]], ptr [[TMP1]], i32 1, <vscale x 8 x i1> [[TMP0]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z20test_svst1_vnum_bf16u10__SVBool_tPu6__bf16lu14__SVBFloat16_t(
+// CPP-CHECK-LABEL: @_Z20test_svst1_vnum_bf16u10__SVBool_tPu6__bf16lu14__SVBFloat16_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x bfloat>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1.c
index 83e68a7e75..508e8c132b 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1.c
@@ -14,12 +14,12 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_s8(
+// CHECK-LABEL: @test_svst1_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    tail call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[DATA:%.*]], ptr [[BASE:%.*]], i32 1, <vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z13test_svst1_s8u10__SVBool_tPau10__SVInt8_t(
+// CPP-CHECK-LABEL: @_Z13test_svst1_s8u10__SVBool_tPau10__SVInt8_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    tail call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[DATA:%.*]], ptr [[BASE:%.*]], i32 1, <vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    ret void
@@ -29,13 +29,13 @@ void test_svst1_s8(svbool_t pg, int8_t *base, svint8_t data)
   return SVE_ACLE_FUNC(svst1,_s8,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_s16(
+// CHECK-LABEL: @test_svst1_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[DATA:%.*]], ptr [[BASE:%.*]], i32 1, <vscale x 8 x i1> [[TMP0]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svst1_s16u10__SVBool_tPsu11__SVInt16_t(
+// CPP-CHECK-LABEL: @_Z14test_svst1_s16u10__SVBool_tPsu11__SVInt16_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[DATA:%.*]], ptr [[BASE:%.*]], i32 1, <vscale x 8 x i1> [[TMP0]])
@@ -46,13 +46,13 @@ void test_svst1_s16(svbool_t pg, int16_t *base, svint16_t data)
   return SVE_ACLE_FUNC(svst1,_s16,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_s32(
+// CHECK-LABEL: @test_svst1_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[DATA:%.*]], ptr [[BASE:%.*]], i32 1, <vscale x 4 x i1> [[TMP0]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svst1_s32u10__SVBool_tPiu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z14test_svst1_s32u10__SVBool_tPiu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[DATA:%.*]], ptr [[BASE:%.*]], i32 1, <vscale x 4 x i1> [[TMP0]])
@@ -63,13 +63,13 @@ void test_svst1_s32(svbool_t pg, int32_t *base, svint32_t data)
   return SVE_ACLE_FUNC(svst1,_s32,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_s64(
+// CHECK-LABEL: @test_svst1_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[DATA:%.*]], ptr [[BASE:%.*]], i32 1, <vscale x 2 x i1> [[TMP0]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svst1_s64u10__SVBool_tPlu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z14test_svst1_s64u10__SVBool_tPlu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[DATA:%.*]], ptr [[BASE:%.*]], i32 1, <vscale x 2 x i1> [[TMP0]])
@@ -80,12 +80,12 @@ void test_svst1_s64(svbool_t pg, int64_t *base, svint64_t data)
   return SVE_ACLE_FUNC(svst1,_s64,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_u8(
+// CHECK-LABEL: @test_svst1_u8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    tail call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[DATA:%.*]], ptr [[BASE:%.*]], i32 1, <vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z13test_svst1_u8u10__SVBool_tPhu11__SVUint8_t(
+// CPP-CHECK-LABEL: @_Z13test_svst1_u8u10__SVBool_tPhu11__SVUint8_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    tail call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[DATA:%.*]], ptr [[BASE:%.*]], i32 1, <vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    ret void
@@ -95,13 +95,13 @@ void test_svst1_u8(svbool_t pg, uint8_t *base, svuint8_t data)
   return SVE_ACLE_FUNC(svst1,_u8,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_u16(
+// CHECK-LABEL: @test_svst1_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[DATA:%.*]], ptr [[BASE:%.*]], i32 1, <vscale x 8 x i1> [[TMP0]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svst1_u16u10__SVBool_tPtu12__SVUint16_t(
+// CPP-CHECK-LABEL: @_Z14test_svst1_u16u10__SVBool_tPtu12__SVUint16_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[DATA:%.*]], ptr [[BASE:%.*]], i32 1, <vscale x 8 x i1> [[TMP0]])
@@ -112,13 +112,13 @@ void test_svst1_u16(svbool_t pg, uint16_t *base, svuint16_t data)
   return SVE_ACLE_FUNC(svst1,_u16,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_u32(
+// CHECK-LABEL: @test_svst1_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[DATA:%.*]], ptr [[BASE:%.*]], i32 1, <vscale x 4 x i1> [[TMP0]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svst1_u32u10__SVBool_tPju12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z14test_svst1_u32u10__SVBool_tPju12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[DATA:%.*]], ptr [[BASE:%.*]], i32 1, <vscale x 4 x i1> [[TMP0]])
@@ -129,13 +129,13 @@ void test_svst1_u32(svbool_t pg, uint32_t *base, svuint32_t data)
   return SVE_ACLE_FUNC(svst1,_u32,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_u64(
+// CHECK-LABEL: @test_svst1_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[DATA:%.*]], ptr [[BASE:%.*]], i32 1, <vscale x 2 x i1> [[TMP0]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svst1_u64u10__SVBool_tPmu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z14test_svst1_u64u10__SVBool_tPmu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[DATA:%.*]], ptr [[BASE:%.*]], i32 1, <vscale x 2 x i1> [[TMP0]])
@@ -146,13 +146,13 @@ void test_svst1_u64(svbool_t pg, uint64_t *base, svuint64_t data)
   return SVE_ACLE_FUNC(svst1,_u64,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_f16(
+// CHECK-LABEL: @test_svst1_f16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.masked.store.nxv8f16.p0(<vscale x 8 x half> [[DATA:%.*]], ptr [[BASE:%.*]], i32 1, <vscale x 8 x i1> [[TMP0]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svst1_f16u10__SVBool_tPDhu13__SVFloat16_t(
+// CPP-CHECK-LABEL: @_Z14test_svst1_f16u10__SVBool_tPDhu13__SVFloat16_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.masked.store.nxv8f16.p0(<vscale x 8 x half> [[DATA:%.*]], ptr [[BASE:%.*]], i32 1, <vscale x 8 x i1> [[TMP0]])
@@ -163,13 +163,13 @@ void test_svst1_f16(svbool_t pg, float16_t *base, svfloat16_t data)
   return SVE_ACLE_FUNC(svst1,_f16,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_f32(
+// CHECK-LABEL: @test_svst1_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.masked.store.nxv4f32.p0(<vscale x 4 x float> [[DATA:%.*]], ptr [[BASE:%.*]], i32 1, <vscale x 4 x i1> [[TMP0]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svst1_f32u10__SVBool_tPfu13__SVFloat32_t(
+// CPP-CHECK-LABEL: @_Z14test_svst1_f32u10__SVBool_tPfu13__SVFloat32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.masked.store.nxv4f32.p0(<vscale x 4 x float> [[DATA:%.*]], ptr [[BASE:%.*]], i32 1, <vscale x 4 x i1> [[TMP0]])
@@ -180,13 +180,13 @@ void test_svst1_f32(svbool_t pg, float32_t *base, svfloat32_t data)
   return SVE_ACLE_FUNC(svst1,_f32,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_f64(
+// CHECK-LABEL: @test_svst1_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[DATA:%.*]], ptr [[BASE:%.*]], i32 1, <vscale x 2 x i1> [[TMP0]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svst1_f64u10__SVBool_tPdu13__SVFloat64_t(
+// CPP-CHECK-LABEL: @_Z14test_svst1_f64u10__SVBool_tPdu13__SVFloat64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[DATA:%.*]], ptr [[BASE:%.*]], i32 1, <vscale x 2 x i1> [[TMP0]])
@@ -197,13 +197,13 @@ void test_svst1_f64(svbool_t pg, float64_t *base, svfloat64_t data)
   return SVE_ACLE_FUNC(svst1,_f64,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_vnum_s8(
+// CHECK-LABEL: @test_svst1_vnum_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    tail call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[DATA:%.*]], ptr [[TMP0]], i32 1, <vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svst1_vnum_s8u10__SVBool_tPalu10__SVInt8_t(
+// CPP-CHECK-LABEL: @_Z18test_svst1_vnum_s8u10__SVBool_tPalu10__SVInt8_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CPP-CHECK-NEXT:    tail call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[DATA:%.*]], ptr [[TMP0]], i32 1, <vscale x 16 x i1> [[PG:%.*]])
@@ -214,14 +214,14 @@ void test_svst1_vnum_s8(svbool_t pg, int8_t *base, int64_t vnum, svint8_t data)
   return SVE_ACLE_FUNC(svst1_vnum,_s8,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_vnum_s16(
+// CHECK-LABEL: @test_svst1_vnum_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    tail call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[DATA:%.*]], ptr [[TMP1]], i32 1, <vscale x 8 x i1> [[TMP0]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svst1_vnum_s16u10__SVBool_tPslu11__SVInt16_t(
+// CPP-CHECK-LABEL: @_Z19test_svst1_vnum_s16u10__SVBool_tPslu11__SVInt16_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -233,14 +233,14 @@ void test_svst1_vnum_s16(svbool_t pg, int16_t *base, int64_t vnum, svint16_t dat
   return SVE_ACLE_FUNC(svst1_vnum,_s16,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_vnum_s32(
+// CHECK-LABEL: @test_svst1_vnum_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    tail call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[DATA:%.*]], ptr [[TMP1]], i32 1, <vscale x 4 x i1> [[TMP0]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svst1_vnum_s32u10__SVBool_tPilu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z19test_svst1_vnum_s32u10__SVBool_tPilu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -252,14 +252,14 @@ void test_svst1_vnum_s32(svbool_t pg, int32_t *base, int64_t vnum, svint32_t dat
   return SVE_ACLE_FUNC(svst1_vnum,_s32,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_vnum_s64(
+// CHECK-LABEL: @test_svst1_vnum_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    tail call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[DATA:%.*]], ptr [[TMP1]], i32 1, <vscale x 2 x i1> [[TMP0]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svst1_vnum_s64u10__SVBool_tPllu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z19test_svst1_vnum_s64u10__SVBool_tPllu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -271,13 +271,13 @@ void test_svst1_vnum_s64(svbool_t pg, int64_t *base, int64_t vnum, svint64_t dat
   return SVE_ACLE_FUNC(svst1_vnum,_s64,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_vnum_u8(
+// CHECK-LABEL: @test_svst1_vnum_u8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    tail call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[DATA:%.*]], ptr [[TMP0]], i32 1, <vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svst1_vnum_u8u10__SVBool_tPhlu11__SVUint8_t(
+// CPP-CHECK-LABEL: @_Z18test_svst1_vnum_u8u10__SVBool_tPhlu11__SVUint8_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CPP-CHECK-NEXT:    tail call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[DATA:%.*]], ptr [[TMP0]], i32 1, <vscale x 16 x i1> [[PG:%.*]])
@@ -288,14 +288,14 @@ void test_svst1_vnum_u8(svbool_t pg, uint8_t *base, int64_t vnum, svuint8_t data
   return SVE_ACLE_FUNC(svst1_vnum,_u8,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_vnum_u16(
+// CHECK-LABEL: @test_svst1_vnum_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    tail call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[DATA:%.*]], ptr [[TMP1]], i32 1, <vscale x 8 x i1> [[TMP0]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svst1_vnum_u16u10__SVBool_tPtlu12__SVUint16_t(
+// CPP-CHECK-LABEL: @_Z19test_svst1_vnum_u16u10__SVBool_tPtlu12__SVUint16_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -307,14 +307,14 @@ void test_svst1_vnum_u16(svbool_t pg, uint16_t *base, int64_t vnum, svuint16_t d
   return SVE_ACLE_FUNC(svst1_vnum,_u16,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_vnum_u32(
+// CHECK-LABEL: @test_svst1_vnum_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    tail call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[DATA:%.*]], ptr [[TMP1]], i32 1, <vscale x 4 x i1> [[TMP0]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svst1_vnum_u32u10__SVBool_tPjlu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z19test_svst1_vnum_u32u10__SVBool_tPjlu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -326,14 +326,14 @@ void test_svst1_vnum_u32(svbool_t pg, uint32_t *base, int64_t vnum, svuint32_t d
   return SVE_ACLE_FUNC(svst1_vnum,_u32,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_vnum_u64(
+// CHECK-LABEL: @test_svst1_vnum_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    tail call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[DATA:%.*]], ptr [[TMP1]], i32 1, <vscale x 2 x i1> [[TMP0]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svst1_vnum_u64u10__SVBool_tPmlu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z19test_svst1_vnum_u64u10__SVBool_tPmlu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -345,14 +345,14 @@ void test_svst1_vnum_u64(svbool_t pg, uint64_t *base, int64_t vnum, svuint64_t d
   return SVE_ACLE_FUNC(svst1_vnum,_u64,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_vnum_f16(
+// CHECK-LABEL: @test_svst1_vnum_f16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    tail call void @llvm.masked.store.nxv8f16.p0(<vscale x 8 x half> [[DATA:%.*]], ptr [[TMP1]], i32 1, <vscale x 8 x i1> [[TMP0]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svst1_vnum_f16u10__SVBool_tPDhlu13__SVFloat16_t(
+// CPP-CHECK-LABEL: @_Z19test_svst1_vnum_f16u10__SVBool_tPDhlu13__SVFloat16_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -364,14 +364,14 @@ void test_svst1_vnum_f16(svbool_t pg, float16_t *base, int64_t vnum, svfloat16_t
   return SVE_ACLE_FUNC(svst1_vnum,_f16,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_vnum_f32(
+// CHECK-LABEL: @test_svst1_vnum_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    tail call void @llvm.masked.store.nxv4f32.p0(<vscale x 4 x float> [[DATA:%.*]], ptr [[TMP1]], i32 1, <vscale x 4 x i1> [[TMP0]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svst1_vnum_f32u10__SVBool_tPflu13__SVFloat32_t(
+// CPP-CHECK-LABEL: @_Z19test_svst1_vnum_f32u10__SVBool_tPflu13__SVFloat32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -383,14 +383,14 @@ void test_svst1_vnum_f32(svbool_t pg, float32_t *base, int64_t vnum, svfloat32_t
   return SVE_ACLE_FUNC(svst1_vnum,_f32,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_vnum_f64(
+// CHECK-LABEL: @test_svst1_vnum_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    tail call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[DATA:%.*]], ptr [[TMP1]], i32 1, <vscale x 2 x i1> [[TMP0]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svst1_vnum_f64u10__SVBool_tPdlu13__SVFloat64_t(
+// CPP-CHECK-LABEL: @_Z19test_svst1_vnum_f64u10__SVBool_tPdlu13__SVFloat64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -402,13 +402,13 @@ void test_svst1_vnum_f64(svbool_t pg, float64_t *base, int64_t vnum, svfloat64_t
   return SVE_ACLE_FUNC(svst1_vnum,_f64,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_u32base_s32(
+// CHECK-LABEL: @test_svst1_scatter_u32base_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.scalar.offset.nxv4i32.nxv4i32(<vscale x 4 x i32> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z30test_svst1_scatter_u32base_s32u10__SVBool_tu12__SVUint32_tu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z30test_svst1_scatter_u32base_s32u10__SVBool_tu12__SVUint32_tu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.scalar.offset.nxv4i32.nxv4i32(<vscale x 4 x i32> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
@@ -419,13 +419,13 @@ void test_svst1_scatter_u32base_s32(svbool_t pg, svuint32_t bases, svint32_t dat
   return SVE_ACLE_FUNC(svst1_scatter,_u32base,,_s32)(pg, bases, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_u64base_s64(
+// CHECK-LABEL: @test_svst1_scatter_u64base_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.scalar.offset.nxv2i64.nxv2i64(<vscale x 2 x i64> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z30test_svst1_scatter_u64base_s64u10__SVBool_tu12__SVUint64_tu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z30test_svst1_scatter_u64base_s64u10__SVBool_tu12__SVUint64_tu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.scalar.offset.nxv2i64.nxv2i64(<vscale x 2 x i64> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
@@ -436,13 +436,13 @@ void test_svst1_scatter_u64base_s64(svbool_t pg, svuint64_t bases, svint64_t dat
   return SVE_ACLE_FUNC(svst1_scatter,_u64base,,_s64)(pg, bases, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_u32base_u32(
+// CHECK-LABEL: @test_svst1_scatter_u32base_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.scalar.offset.nxv4i32.nxv4i32(<vscale x 4 x i32> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z30test_svst1_scatter_u32base_u32u10__SVBool_tu12__SVUint32_tu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z30test_svst1_scatter_u32base_u32u10__SVBool_tu12__SVUint32_tu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.scalar.offset.nxv4i32.nxv4i32(<vscale x 4 x i32> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
@@ -453,13 +453,13 @@ void test_svst1_scatter_u32base_u32(svbool_t pg, svuint32_t bases, svuint32_t da
   return SVE_ACLE_FUNC(svst1_scatter,_u32base,,_u32)(pg, bases, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_u64base_u64(
+// CHECK-LABEL: @test_svst1_scatter_u64base_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.scalar.offset.nxv2i64.nxv2i64(<vscale x 2 x i64> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z30test_svst1_scatter_u64base_u64u10__SVBool_tu12__SVUint64_tu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z30test_svst1_scatter_u64base_u64u10__SVBool_tu12__SVUint64_tu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.scalar.offset.nxv2i64.nxv2i64(<vscale x 2 x i64> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
@@ -470,13 +470,13 @@ void test_svst1_scatter_u64base_u64(svbool_t pg, svuint64_t bases, svuint64_t da
   return SVE_ACLE_FUNC(svst1_scatter,_u64base,,_u64)(pg, bases, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_u32base_f32(
+// CHECK-LABEL: @test_svst1_scatter_u32base_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.scalar.offset.nxv4f32.nxv4i32(<vscale x 4 x float> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z30test_svst1_scatter_u32base_f32u10__SVBool_tu12__SVUint32_tu13__SVFloat32_t(
+// CPP-CHECK-LABEL: @_Z30test_svst1_scatter_u32base_f32u10__SVBool_tu12__SVUint32_tu13__SVFloat32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.scalar.offset.nxv4f32.nxv4i32(<vscale x 4 x float> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 0)
@@ -487,13 +487,13 @@ void test_svst1_scatter_u32base_f32(svbool_t pg, svuint32_t bases, svfloat32_t d
   return SVE_ACLE_FUNC(svst1_scatter,_u32base,,_f32)(pg, bases, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_u64base_f64(
+// CHECK-LABEL: @test_svst1_scatter_u64base_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.scalar.offset.nxv2f64.nxv2i64(<vscale x 2 x double> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z30test_svst1_scatter_u64base_f64u10__SVBool_tu12__SVUint64_tu13__SVFloat64_t(
+// CPP-CHECK-LABEL: @_Z30test_svst1_scatter_u64base_f64u10__SVBool_tu12__SVUint64_tu13__SVFloat64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.scalar.offset.nxv2f64.nxv2i64(<vscale x 2 x double> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 0)
@@ -504,13 +504,13 @@ void test_svst1_scatter_u64base_f64(svbool_t pg, svuint64_t bases, svfloat64_t d
   return SVE_ACLE_FUNC(svst1_scatter,_u64base,,_f64)(pg, bases, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_s32offset_s32(
+// CHECK-LABEL: @test_svst1_scatter_s32offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.sxtw.nxv4i32(<vscale x 4 x i32> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svst1_scatter_s32offset_s32u10__SVBool_tPiu11__SVInt32_tu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z32test_svst1_scatter_s32offset_s32u10__SVBool_tPiu11__SVInt32_tu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.sxtw.nxv4i32(<vscale x 4 x i32> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -521,13 +521,13 @@ void test_svst1_scatter_s32offset_s32(svbool_t pg, int32_t *base, svint32_t offs
   return SVE_ACLE_FUNC(svst1_scatter_,s32,offset,_s32)(pg, base, offsets, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_s64offset_s64(
+// CHECK-LABEL: @test_svst1_scatter_s64offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.nxv2i64(<vscale x 2 x i64> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svst1_scatter_s64offset_s64u10__SVBool_tPlu11__SVInt64_tu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z32test_svst1_scatter_s64offset_s64u10__SVBool_tPlu11__SVInt64_tu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.nxv2i64(<vscale x 2 x i64> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -538,13 +538,13 @@ void test_svst1_scatter_s64offset_s64(svbool_t pg, int64_t *base, svint64_t offs
   return SVE_ACLE_FUNC(svst1_scatter_,s64,offset,_s64)(pg, base, offsets, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_s32offset_u32(
+// CHECK-LABEL: @test_svst1_scatter_s32offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.sxtw.nxv4i32(<vscale x 4 x i32> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svst1_scatter_s32offset_u32u10__SVBool_tPju11__SVInt32_tu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z32test_svst1_scatter_s32offset_u32u10__SVBool_tPju11__SVInt32_tu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.sxtw.nxv4i32(<vscale x 4 x i32> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -555,13 +555,13 @@ void test_svst1_scatter_s32offset_u32(svbool_t pg, uint32_t *base, svint32_t off
   return SVE_ACLE_FUNC(svst1_scatter_,s32,offset,_u32)(pg, base, offsets, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_s64offset_u64(
+// CHECK-LABEL: @test_svst1_scatter_s64offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.nxv2i64(<vscale x 2 x i64> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svst1_scatter_s64offset_u64u10__SVBool_tPmu11__SVInt64_tu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z32test_svst1_scatter_s64offset_u64u10__SVBool_tPmu11__SVInt64_tu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.nxv2i64(<vscale x 2 x i64> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -572,13 +572,13 @@ void test_svst1_scatter_s64offset_u64(svbool_t pg, uint64_t *base, svint64_t off
   return SVE_ACLE_FUNC(svst1_scatter_,s64,offset,_u64)(pg, base, offsets, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_s32offset_f32(
+// CHECK-LABEL: @test_svst1_scatter_s32offset_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.sxtw.nxv4f32(<vscale x 4 x float> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svst1_scatter_s32offset_f32u10__SVBool_tPfu11__SVInt32_tu13__SVFloat32_t(
+// CPP-CHECK-LABEL: @_Z32test_svst1_scatter_s32offset_f32u10__SVBool_tPfu11__SVInt32_tu13__SVFloat32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.sxtw.nxv4f32(<vscale x 4 x float> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -589,13 +589,13 @@ void test_svst1_scatter_s32offset_f32(svbool_t pg, float32_t *base, svint32_t of
   return SVE_ACLE_FUNC(svst1_scatter_,s32,offset,_f32)(pg, base, offsets, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_s64offset_f64(
+// CHECK-LABEL: @test_svst1_scatter_s64offset_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.nxv2f64(<vscale x 2 x double> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svst1_scatter_s64offset_f64u10__SVBool_tPdu11__SVInt64_tu13__SVFloat64_t(
+// CPP-CHECK-LABEL: @_Z32test_svst1_scatter_s64offset_f64u10__SVBool_tPdu11__SVInt64_tu13__SVFloat64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.nxv2f64(<vscale x 2 x double> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -606,13 +606,13 @@ void test_svst1_scatter_s64offset_f64(svbool_t pg, float64_t *base, svint64_t of
   return SVE_ACLE_FUNC(svst1_scatter_,s64,offset,_f64)(pg, base, offsets, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_u32offset_s32(
+// CHECK-LABEL: @test_svst1_scatter_u32offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.uxtw.nxv4i32(<vscale x 4 x i32> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svst1_scatter_u32offset_s32u10__SVBool_tPiu12__SVUint32_tu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z32test_svst1_scatter_u32offset_s32u10__SVBool_tPiu12__SVUint32_tu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.uxtw.nxv4i32(<vscale x 4 x i32> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -623,13 +623,13 @@ void test_svst1_scatter_u32offset_s32(svbool_t pg, int32_t *base, svuint32_t off
   return SVE_ACLE_FUNC(svst1_scatter_,u32,offset,_s32)(pg, base, offsets, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_u64offset_s64(
+// CHECK-LABEL: @test_svst1_scatter_u64offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.nxv2i64(<vscale x 2 x i64> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svst1_scatter_u64offset_s64u10__SVBool_tPlu12__SVUint64_tu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z32test_svst1_scatter_u64offset_s64u10__SVBool_tPlu12__SVUint64_tu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.nxv2i64(<vscale x 2 x i64> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -640,13 +640,13 @@ void test_svst1_scatter_u64offset_s64(svbool_t pg, int64_t *base, svuint64_t off
   return SVE_ACLE_FUNC(svst1_scatter_,u64,offset,_s64)(pg, base, offsets, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_u32offset_u32(
+// CHECK-LABEL: @test_svst1_scatter_u32offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.uxtw.nxv4i32(<vscale x 4 x i32> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svst1_scatter_u32offset_u32u10__SVBool_tPju12__SVUint32_tu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z32test_svst1_scatter_u32offset_u32u10__SVBool_tPju12__SVUint32_tu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.uxtw.nxv4i32(<vscale x 4 x i32> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -657,13 +657,13 @@ void test_svst1_scatter_u32offset_u32(svbool_t pg, uint32_t *base, svuint32_t of
   return SVE_ACLE_FUNC(svst1_scatter_,u32,offset,_u32)(pg, base, offsets, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_u64offset_u64(
+// CHECK-LABEL: @test_svst1_scatter_u64offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.nxv2i64(<vscale x 2 x i64> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svst1_scatter_u64offset_u64u10__SVBool_tPmu12__SVUint64_tu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z32test_svst1_scatter_u64offset_u64u10__SVBool_tPmu12__SVUint64_tu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.nxv2i64(<vscale x 2 x i64> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -674,13 +674,13 @@ void test_svst1_scatter_u64offset_u64(svbool_t pg, uint64_t *base, svuint64_t of
   return SVE_ACLE_FUNC(svst1_scatter_,u64,offset,_u64)(pg, base, offsets, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_u32offset_f32(
+// CHECK-LABEL: @test_svst1_scatter_u32offset_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.uxtw.nxv4f32(<vscale x 4 x float> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svst1_scatter_u32offset_f32u10__SVBool_tPfu12__SVUint32_tu13__SVFloat32_t(
+// CPP-CHECK-LABEL: @_Z32test_svst1_scatter_u32offset_f32u10__SVBool_tPfu12__SVUint32_tu13__SVFloat32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.uxtw.nxv4f32(<vscale x 4 x float> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[OFFSETS:%.*]])
@@ -691,13 +691,13 @@ void test_svst1_scatter_u32offset_f32(svbool_t pg, float32_t *base, svuint32_t o
   return SVE_ACLE_FUNC(svst1_scatter_,u32,offset,_f32)(pg, base, offsets, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_u64offset_f64(
+// CHECK-LABEL: @test_svst1_scatter_u64offset_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.nxv2f64(<vscale x 2 x double> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z32test_svst1_scatter_u64offset_f64u10__SVBool_tPdu12__SVUint64_tu13__SVFloat64_t(
+// CPP-CHECK-LABEL: @_Z32test_svst1_scatter_u64offset_f64u10__SVBool_tPdu12__SVUint64_tu13__SVFloat64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.nxv2f64(<vscale x 2 x double> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[OFFSETS:%.*]])
@@ -708,13 +708,13 @@ void test_svst1_scatter_u64offset_f64(svbool_t pg, float64_t *base, svuint64_t o
   return SVE_ACLE_FUNC(svst1_scatter_,u64,offset,_f64)(pg, base, offsets, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_u32base_offset_s32(
+// CHECK-LABEL: @test_svst1_scatter_u32base_offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.scalar.offset.nxv4i32.nxv4i32(<vscale x 4 x i32> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z37test_svst1_scatter_u32base_offset_s32u10__SVBool_tu12__SVUint32_tlu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z37test_svst1_scatter_u32base_offset_s32u10__SVBool_tu12__SVUint32_tlu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.scalar.offset.nxv4i32.nxv4i32(<vscale x 4 x i32> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -725,13 +725,13 @@ void test_svst1_scatter_u32base_offset_s32(svbool_t pg, svuint32_t bases, int64_
   return SVE_ACLE_FUNC(svst1_scatter,_u32base,_offset,_s32)(pg, bases, offset, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_u64base_offset_s64(
+// CHECK-LABEL: @test_svst1_scatter_u64base_offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.scalar.offset.nxv2i64.nxv2i64(<vscale x 2 x i64> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z37test_svst1_scatter_u64base_offset_s64u10__SVBool_tu12__SVUint64_tlu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z37test_svst1_scatter_u64base_offset_s64u10__SVBool_tu12__SVUint64_tlu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.scalar.offset.nxv2i64.nxv2i64(<vscale x 2 x i64> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -742,13 +742,13 @@ void test_svst1_scatter_u64base_offset_s64(svbool_t pg, svuint64_t bases, int64_
   return SVE_ACLE_FUNC(svst1_scatter,_u64base,_offset,_s64)(pg, bases, offset, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_u32base_offset_u32(
+// CHECK-LABEL: @test_svst1_scatter_u32base_offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.scalar.offset.nxv4i32.nxv4i32(<vscale x 4 x i32> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z37test_svst1_scatter_u32base_offset_u32u10__SVBool_tu12__SVUint32_tlu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z37test_svst1_scatter_u32base_offset_u32u10__SVBool_tu12__SVUint32_tlu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.scalar.offset.nxv4i32.nxv4i32(<vscale x 4 x i32> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -759,13 +759,13 @@ void test_svst1_scatter_u32base_offset_u32(svbool_t pg, svuint32_t bases, int64_
   return SVE_ACLE_FUNC(svst1_scatter,_u32base,_offset,_u32)(pg, bases, offset, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_u64base_offset_u64(
+// CHECK-LABEL: @test_svst1_scatter_u64base_offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.scalar.offset.nxv2i64.nxv2i64(<vscale x 2 x i64> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z37test_svst1_scatter_u64base_offset_u64u10__SVBool_tu12__SVUint64_tlu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z37test_svst1_scatter_u64base_offset_u64u10__SVBool_tu12__SVUint64_tlu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.scalar.offset.nxv2i64.nxv2i64(<vscale x 2 x i64> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -776,13 +776,13 @@ void test_svst1_scatter_u64base_offset_u64(svbool_t pg, svuint64_t bases, int64_
   return SVE_ACLE_FUNC(svst1_scatter,_u64base,_offset,_u64)(pg, bases, offset, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_u32base_offset_f32(
+// CHECK-LABEL: @test_svst1_scatter_u32base_offset_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.scalar.offset.nxv4f32.nxv4i32(<vscale x 4 x float> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z37test_svst1_scatter_u32base_offset_f32u10__SVBool_tu12__SVUint32_tlu13__SVFloat32_t(
+// CPP-CHECK-LABEL: @_Z37test_svst1_scatter_u32base_offset_f32u10__SVBool_tu12__SVUint32_tlu13__SVFloat32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.scalar.offset.nxv4f32.nxv4i32(<vscale x 4 x float> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -793,13 +793,13 @@ void test_svst1_scatter_u32base_offset_f32(svbool_t pg, svuint32_t bases, int64_
   return SVE_ACLE_FUNC(svst1_scatter,_u32base,_offset,_f32)(pg, bases, offset, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_u64base_offset_f64(
+// CHECK-LABEL: @test_svst1_scatter_u64base_offset_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.scalar.offset.nxv2f64.nxv2i64(<vscale x 2 x double> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z37test_svst1_scatter_u64base_offset_f64u10__SVBool_tu12__SVUint64_tlu13__SVFloat64_t(
+// CPP-CHECK-LABEL: @_Z37test_svst1_scatter_u64base_offset_f64u10__SVBool_tu12__SVUint64_tlu13__SVFloat64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.scalar.offset.nxv2f64.nxv2i64(<vscale x 2 x double> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[OFFSET:%.*]])
@@ -810,13 +810,13 @@ void test_svst1_scatter_u64base_offset_f64(svbool_t pg, svuint64_t bases, int64_
   return SVE_ACLE_FUNC(svst1_scatter,_u64base,_offset,_f64)(pg, bases, offset, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_s32index_s32(
+// CHECK-LABEL: @test_svst1_scatter_s32index_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.sxtw.index.nxv4i32(<vscale x 4 x i32> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svst1_scatter_s32index_s32u10__SVBool_tPiu11__SVInt32_tu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z31test_svst1_scatter_s32index_s32u10__SVBool_tPiu11__SVInt32_tu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.sxtw.index.nxv4i32(<vscale x 4 x i32> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
@@ -827,13 +827,13 @@ void test_svst1_scatter_s32index_s32(svbool_t pg, int32_t *base, svint32_t indic
   return SVE_ACLE_FUNC(svst1_scatter_,s32,index,_s32)(pg, base, indices, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_s64index_s64(
+// CHECK-LABEL: @test_svst1_scatter_s64index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.index.nxv2i64(<vscale x 2 x i64> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svst1_scatter_s64index_s64u10__SVBool_tPlu11__SVInt64_tu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z31test_svst1_scatter_s64index_s64u10__SVBool_tPlu11__SVInt64_tu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.index.nxv2i64(<vscale x 2 x i64> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -844,13 +844,13 @@ void test_svst1_scatter_s64index_s64(svbool_t pg, int64_t *base, svint64_t indic
   return SVE_ACLE_FUNC(svst1_scatter_,s64,index,_s64)(pg, base, indices, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_s32index_u32(
+// CHECK-LABEL: @test_svst1_scatter_s32index_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.sxtw.index.nxv4i32(<vscale x 4 x i32> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svst1_scatter_s32index_u32u10__SVBool_tPju11__SVInt32_tu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z31test_svst1_scatter_s32index_u32u10__SVBool_tPju11__SVInt32_tu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.sxtw.index.nxv4i32(<vscale x 4 x i32> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
@@ -861,13 +861,13 @@ void test_svst1_scatter_s32index_u32(svbool_t pg, uint32_t *base, svint32_t indi
   return SVE_ACLE_FUNC(svst1_scatter_,s32,index,_u32)(pg, base, indices, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_s64index_u64(
+// CHECK-LABEL: @test_svst1_scatter_s64index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.index.nxv2i64(<vscale x 2 x i64> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svst1_scatter_s64index_u64u10__SVBool_tPmu11__SVInt64_tu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z31test_svst1_scatter_s64index_u64u10__SVBool_tPmu11__SVInt64_tu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.index.nxv2i64(<vscale x 2 x i64> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -878,13 +878,13 @@ void test_svst1_scatter_s64index_u64(svbool_t pg, uint64_t *base, svint64_t indi
   return SVE_ACLE_FUNC(svst1_scatter_,s64,index,_u64)(pg, base, indices, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_s32index_f32(
+// CHECK-LABEL: @test_svst1_scatter_s32index_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.sxtw.index.nxv4f32(<vscale x 4 x float> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svst1_scatter_s32index_f32u10__SVBool_tPfu11__SVInt32_tu13__SVFloat32_t(
+// CPP-CHECK-LABEL: @_Z31test_svst1_scatter_s32index_f32u10__SVBool_tPfu11__SVInt32_tu13__SVFloat32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.sxtw.index.nxv4f32(<vscale x 4 x float> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
@@ -895,13 +895,13 @@ void test_svst1_scatter_s32index_f32(svbool_t pg, float32_t *base, svint32_t ind
   return SVE_ACLE_FUNC(svst1_scatter_,s32,index,_f32)(pg, base, indices, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_s64index_f64(
+// CHECK-LABEL: @test_svst1_scatter_s64index_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.index.nxv2f64(<vscale x 2 x double> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svst1_scatter_s64index_f64u10__SVBool_tPdu11__SVInt64_tu13__SVFloat64_t(
+// CPP-CHECK-LABEL: @_Z31test_svst1_scatter_s64index_f64u10__SVBool_tPdu11__SVInt64_tu13__SVFloat64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.index.nxv2f64(<vscale x 2 x double> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -912,13 +912,13 @@ void test_svst1_scatter_s64index_f64(svbool_t pg, float64_t *base, svint64_t ind
   return SVE_ACLE_FUNC(svst1_scatter_,s64,index,_f64)(pg, base, indices, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_u32index_s32(
+// CHECK-LABEL: @test_svst1_scatter_u32index_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.uxtw.index.nxv4i32(<vscale x 4 x i32> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svst1_scatter_u32index_s32u10__SVBool_tPiu12__SVUint32_tu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z31test_svst1_scatter_u32index_s32u10__SVBool_tPiu12__SVUint32_tu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.uxtw.index.nxv4i32(<vscale x 4 x i32> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
@@ -929,13 +929,13 @@ void test_svst1_scatter_u32index_s32(svbool_t pg, int32_t *base, svuint32_t indi
   return SVE_ACLE_FUNC(svst1_scatter_,u32,index,_s32)(pg, base, indices, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_u64index_s64(
+// CHECK-LABEL: @test_svst1_scatter_u64index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.index.nxv2i64(<vscale x 2 x i64> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svst1_scatter_u64index_s64u10__SVBool_tPlu12__SVUint64_tu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z31test_svst1_scatter_u64index_s64u10__SVBool_tPlu12__SVUint64_tu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.index.nxv2i64(<vscale x 2 x i64> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -946,13 +946,13 @@ void test_svst1_scatter_u64index_s64(svbool_t pg, int64_t *base, svuint64_t indi
   return SVE_ACLE_FUNC(svst1_scatter_,u64,index,_s64)(pg, base, indices, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_u32index_u32(
+// CHECK-LABEL: @test_svst1_scatter_u32index_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.uxtw.index.nxv4i32(<vscale x 4 x i32> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svst1_scatter_u32index_u32u10__SVBool_tPju12__SVUint32_tu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z31test_svst1_scatter_u32index_u32u10__SVBool_tPju12__SVUint32_tu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.uxtw.index.nxv4i32(<vscale x 4 x i32> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
@@ -963,13 +963,13 @@ void test_svst1_scatter_u32index_u32(svbool_t pg, uint32_t *base, svuint32_t ind
   return SVE_ACLE_FUNC(svst1_scatter_,u32,index,_u32)(pg, base, indices, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_u64index_u64(
+// CHECK-LABEL: @test_svst1_scatter_u64index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.index.nxv2i64(<vscale x 2 x i64> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svst1_scatter_u64index_u64u10__SVBool_tPmu12__SVUint64_tu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z31test_svst1_scatter_u64index_u64u10__SVBool_tPmu12__SVUint64_tu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.index.nxv2i64(<vscale x 2 x i64> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -980,13 +980,13 @@ void test_svst1_scatter_u64index_u64(svbool_t pg, uint64_t *base, svuint64_t ind
   return SVE_ACLE_FUNC(svst1_scatter_,u64,index,_u64)(pg, base, indices, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_u32index_f32(
+// CHECK-LABEL: @test_svst1_scatter_u32index_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.uxtw.index.nxv4f32(<vscale x 4 x float> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svst1_scatter_u32index_f32u10__SVBool_tPfu12__SVUint32_tu13__SVFloat32_t(
+// CPP-CHECK-LABEL: @_Z31test_svst1_scatter_u32index_f32u10__SVBool_tPfu12__SVUint32_tu13__SVFloat32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.uxtw.index.nxv4f32(<vscale x 4 x float> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]])
@@ -997,13 +997,13 @@ void test_svst1_scatter_u32index_f32(svbool_t pg, float32_t *base, svuint32_t in
   return SVE_ACLE_FUNC(svst1_scatter_,u32,index,_f32)(pg, base, indices, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_u64index_f64(
+// CHECK-LABEL: @test_svst1_scatter_u64index_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.index.nxv2f64(<vscale x 2 x double> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z31test_svst1_scatter_u64index_f64u10__SVBool_tPdu12__SVUint64_tu13__SVFloat64_t(
+// CPP-CHECK-LABEL: @_Z31test_svst1_scatter_u64index_f64u10__SVBool_tPdu12__SVUint64_tu13__SVFloat64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.index.nxv2f64(<vscale x 2 x double> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDICES:%.*]])
@@ -1014,14 +1014,14 @@ void test_svst1_scatter_u64index_f64(svbool_t pg, float64_t *base, svuint64_t in
   return SVE_ACLE_FUNC(svst1_scatter_,u64,index,_f64)(pg, base, indices, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_u32base_index_s32(
+// CHECK-LABEL: @test_svst1_scatter_u32base_index_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 2
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.scalar.offset.nxv4i32.nxv4i32(<vscale x 4 x i32> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z36test_svst1_scatter_u32base_index_s32u10__SVBool_tu12__SVUint32_tlu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z36test_svst1_scatter_u32base_index_s32u10__SVBool_tu12__SVUint32_tlu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 2
@@ -1033,14 +1033,14 @@ void test_svst1_scatter_u32base_index_s32(svbool_t pg, svuint32_t bases, int64_t
   return SVE_ACLE_FUNC(svst1_scatter,_u32base,_index,_s32)(pg, bases, index, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_u64base_index_s64(
+// CHECK-LABEL: @test_svst1_scatter_u64base_index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 3
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.scalar.offset.nxv2i64.nxv2i64(<vscale x 2 x i64> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z36test_svst1_scatter_u64base_index_s64u10__SVBool_tu12__SVUint64_tlu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z36test_svst1_scatter_u64base_index_s64u10__SVBool_tu12__SVUint64_tlu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 3
@@ -1052,14 +1052,14 @@ void test_svst1_scatter_u64base_index_s64(svbool_t pg, svuint64_t bases, int64_t
   return SVE_ACLE_FUNC(svst1_scatter,_u64base,_index,_s64)(pg, bases, index, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_u32base_index_u32(
+// CHECK-LABEL: @test_svst1_scatter_u32base_index_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 2
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.scalar.offset.nxv4i32.nxv4i32(<vscale x 4 x i32> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z36test_svst1_scatter_u32base_index_u32u10__SVBool_tu12__SVUint32_tlu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z36test_svst1_scatter_u32base_index_u32u10__SVBool_tu12__SVUint32_tlu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 2
@@ -1071,14 +1071,14 @@ void test_svst1_scatter_u32base_index_u32(svbool_t pg, svuint32_t bases, int64_t
   return SVE_ACLE_FUNC(svst1_scatter,_u32base,_index,_u32)(pg, bases, index, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_u64base_index_u64(
+// CHECK-LABEL: @test_svst1_scatter_u64base_index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 3
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.scalar.offset.nxv2i64.nxv2i64(<vscale x 2 x i64> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z36test_svst1_scatter_u64base_index_u64u10__SVBool_tu12__SVUint64_tlu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z36test_svst1_scatter_u64base_index_u64u10__SVBool_tu12__SVUint64_tlu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 3
@@ -1090,14 +1090,14 @@ void test_svst1_scatter_u64base_index_u64(svbool_t pg, svuint64_t bases, int64_t
   return SVE_ACLE_FUNC(svst1_scatter,_u64base,_index,_u64)(pg, bases, index, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_u32base_index_f32(
+// CHECK-LABEL: @test_svst1_scatter_u32base_index_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 2
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.scalar.offset.nxv4f32.nxv4i32(<vscale x 4 x float> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[BASES:%.*]], i64 [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z36test_svst1_scatter_u32base_index_f32u10__SVBool_tu12__SVUint32_tlu13__SVFloat32_t(
+// CPP-CHECK-LABEL: @_Z36test_svst1_scatter_u32base_index_f32u10__SVBool_tu12__SVUint32_tlu13__SVFloat32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 2
@@ -1109,14 +1109,14 @@ void test_svst1_scatter_u32base_index_f32(svbool_t pg, svuint32_t bases, int64_t
   return SVE_ACLE_FUNC(svst1_scatter,_u32base,_index,_f32)(pg, bases, index, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1_scatter_u64base_index_f64(
+// CHECK-LABEL: @test_svst1_scatter_u64base_index_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 3
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.scatter.scalar.offset.nxv2f64.nxv2i64(<vscale x 2 x double> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[BASES:%.*]], i64 [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z36test_svst1_scatter_u64base_index_f64u10__SVBool_tu12__SVUint64_tlu13__SVFloat64_t(
+// CPP-CHECK-LABEL: @_Z36test_svst1_scatter_u64base_index_f64u10__SVBool_tu12__SVUint64_tlu13__SVFloat64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX:%.*]], 3
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1b.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1b.c
index 0f47658c56..5694f09bb3 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1b.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1b.c
@@ -11,7 +11,7 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1b_s16(
+// CHECK-LABEL: @test_svst1b_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = trunc <vscale x 8 x i16> [[DATA:%.*]] to <vscale x 8 x i8>
@@ -23,7 +23,7 @@ void test_svst1b_s16(svbool_t pg, int8_t *base, svint16_t data)
   return SVE_ACLE_FUNC(svst1b,_s16,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1b_s32(
+// CHECK-LABEL: @test_svst1b_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = trunc <vscale x 4 x i32> [[DATA:%.*]] to <vscale x 4 x i8>
@@ -35,7 +35,7 @@ void test_svst1b_s32(svbool_t pg, int8_t *base, svint32_t data)
   return SVE_ACLE_FUNC(svst1b,_s32,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1b_s64(
+// CHECK-LABEL: @test_svst1b_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i8>
@@ -47,7 +47,7 @@ void test_svst1b_s64(svbool_t pg, int8_t *base, svint64_t data)
   return SVE_ACLE_FUNC(svst1b,_s64,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1b_u16(
+// CHECK-LABEL: @test_svst1b_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = trunc <vscale x 8 x i16> [[DATA:%.*]] to <vscale x 8 x i8>
@@ -59,7 +59,7 @@ void test_svst1b_u16(svbool_t pg, uint8_t *base, svuint16_t data)
   return SVE_ACLE_FUNC(svst1b,_u16,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1b_u32(
+// CHECK-LABEL: @test_svst1b_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = trunc <vscale x 4 x i32> [[DATA:%.*]] to <vscale x 4 x i8>
@@ -71,7 +71,7 @@ void test_svst1b_u32(svbool_t pg, uint8_t *base, svuint32_t data)
   return SVE_ACLE_FUNC(svst1b,_u32,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1b_u64(
+// CHECK-LABEL: @test_svst1b_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i8>
@@ -83,7 +83,7 @@ void test_svst1b_u64(svbool_t pg, uint8_t *base, svuint64_t data)
   return SVE_ACLE_FUNC(svst1b,_u64,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1b_vnum_s16(
+// CHECK-LABEL: @test_svst1b_vnum_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -96,7 +96,7 @@ void test_svst1b_vnum_s16(svbool_t pg, int8_t *base, int64_t vnum, svint16_t dat
   return SVE_ACLE_FUNC(svst1b_vnum,_s16,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1b_vnum_s32(
+// CHECK-LABEL: @test_svst1b_vnum_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -109,7 +109,7 @@ void test_svst1b_vnum_s32(svbool_t pg, int8_t *base, int64_t vnum, svint32_t dat
   return SVE_ACLE_FUNC(svst1b_vnum,_s32,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1b_vnum_s64(
+// CHECK-LABEL: @test_svst1b_vnum_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -122,7 +122,7 @@ void test_svst1b_vnum_s64(svbool_t pg, int8_t *base, int64_t vnum, svint64_t dat
   return SVE_ACLE_FUNC(svst1b_vnum,_s64,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1b_vnum_u16(
+// CHECK-LABEL: @test_svst1b_vnum_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -135,7 +135,7 @@ void test_svst1b_vnum_u16(svbool_t pg, uint8_t *base, int64_t vnum, svuint16_t d
   return SVE_ACLE_FUNC(svst1b_vnum,_u16,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1b_vnum_u32(
+// CHECK-LABEL: @test_svst1b_vnum_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -148,7 +148,7 @@ void test_svst1b_vnum_u32(svbool_t pg, uint8_t *base, int64_t vnum, svuint32_t d
   return SVE_ACLE_FUNC(svst1b_vnum,_u32,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1b_vnum_u64(
+// CHECK-LABEL: @test_svst1b_vnum_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -161,7 +161,7 @@ void test_svst1b_vnum_u64(svbool_t pg, uint8_t *base, int64_t vnum, svuint64_t d
   return SVE_ACLE_FUNC(svst1b_vnum,_u64,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1b_scatter_u32base_s32(
+// CHECK-LABEL: @test_svst1b_scatter_u32base_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 4 x i32> [[DATA:%.*]] to <vscale x 4 x i8>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -173,7 +173,7 @@ void test_svst1b_scatter_u32base_s32(svbool_t pg, svuint32_t bases, svint32_t da
   return SVE_ACLE_FUNC(svst1b_scatter,_u32base,,_s32)(pg, bases, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1b_scatter_u64base_s64(
+// CHECK-LABEL: @test_svst1b_scatter_u64base_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i8>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -185,7 +185,7 @@ void test_svst1b_scatter_u64base_s64(svbool_t pg, svuint64_t bases, svint64_t da
   return SVE_ACLE_FUNC(svst1b_scatter,_u64base,,_s64)(pg, bases, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1b_scatter_u32base_u32(
+// CHECK-LABEL: @test_svst1b_scatter_u32base_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 4 x i32> [[DATA:%.*]] to <vscale x 4 x i8>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -197,7 +197,7 @@ void test_svst1b_scatter_u32base_u32(svbool_t pg, svuint32_t bases, svuint32_t d
   return SVE_ACLE_FUNC(svst1b_scatter,_u32base,,_u32)(pg, bases, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1b_scatter_u64base_u64(
+// CHECK-LABEL: @test_svst1b_scatter_u64base_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i8>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -209,7 +209,7 @@ void test_svst1b_scatter_u64base_u64(svbool_t pg, svuint64_t bases, svuint64_t d
   return SVE_ACLE_FUNC(svst1b_scatter,_u64base,,_u64)(pg, bases, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1b_scatter_s32offset_s32(
+// CHECK-LABEL: @test_svst1b_scatter_s32offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 4 x i32> [[DATA:%.*]] to <vscale x 4 x i8>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -221,7 +221,7 @@ void test_svst1b_scatter_s32offset_s32(svbool_t pg, int8_t *base, svint32_t offs
   return SVE_ACLE_FUNC(svst1b_scatter_,s32,offset,_s32)(pg, base, offsets, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1b_scatter_s64offset_s64(
+// CHECK-LABEL: @test_svst1b_scatter_s64offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i8>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -233,7 +233,7 @@ void test_svst1b_scatter_s64offset_s64(svbool_t pg, int8_t *base, svint64_t offs
   return SVE_ACLE_FUNC(svst1b_scatter_,s64,offset,_s64)(pg, base, offsets, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1b_scatter_s32offset_u32(
+// CHECK-LABEL: @test_svst1b_scatter_s32offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 4 x i32> [[DATA:%.*]] to <vscale x 4 x i8>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -245,7 +245,7 @@ void test_svst1b_scatter_s32offset_u32(svbool_t pg, uint8_t *base, svint32_t off
   return SVE_ACLE_FUNC(svst1b_scatter_,s32,offset,_u32)(pg, base, offsets, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1b_scatter_s64offset_u64(
+// CHECK-LABEL: @test_svst1b_scatter_s64offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i8>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -257,7 +257,7 @@ void test_svst1b_scatter_s64offset_u64(svbool_t pg, uint8_t *base, svint64_t off
   return SVE_ACLE_FUNC(svst1b_scatter_,s64,offset,_u64)(pg, base, offsets, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1b_scatter_u32offset_s32(
+// CHECK-LABEL: @test_svst1b_scatter_u32offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 4 x i32> [[DATA:%.*]] to <vscale x 4 x i8>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -269,7 +269,7 @@ void test_svst1b_scatter_u32offset_s32(svbool_t pg, int8_t *base, svuint32_t off
   return SVE_ACLE_FUNC(svst1b_scatter_,u32,offset,_s32)(pg, base, offsets, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1b_scatter_u64offset_s64(
+// CHECK-LABEL: @test_svst1b_scatter_u64offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i8>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -281,7 +281,7 @@ void test_svst1b_scatter_u64offset_s64(svbool_t pg, int8_t *base, svuint64_t off
   return SVE_ACLE_FUNC(svst1b_scatter_,u64,offset,_s64)(pg, base, offsets, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1b_scatter_u32offset_u32(
+// CHECK-LABEL: @test_svst1b_scatter_u32offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 4 x i32> [[DATA:%.*]] to <vscale x 4 x i8>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -293,7 +293,7 @@ void test_svst1b_scatter_u32offset_u32(svbool_t pg, uint8_t *base, svuint32_t of
   return SVE_ACLE_FUNC(svst1b_scatter_,u32,offset,_u32)(pg, base, offsets, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1b_scatter_u64offset_u64(
+// CHECK-LABEL: @test_svst1b_scatter_u64offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i8>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -305,7 +305,7 @@ void test_svst1b_scatter_u64offset_u64(svbool_t pg, uint8_t *base, svuint64_t of
   return SVE_ACLE_FUNC(svst1b_scatter_,u64,offset,_u64)(pg, base, offsets, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1b_scatter_u32base_offset_s32(
+// CHECK-LABEL: @test_svst1b_scatter_u32base_offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 4 x i32> [[DATA:%.*]] to <vscale x 4 x i8>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -317,7 +317,7 @@ void test_svst1b_scatter_u32base_offset_s32(svbool_t pg, svuint32_t bases, int64
   return SVE_ACLE_FUNC(svst1b_scatter,_u32base,_offset,_s32)(pg, bases, offset, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1b_scatter_u64base_offset_s64(
+// CHECK-LABEL: @test_svst1b_scatter_u64base_offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i8>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -329,7 +329,7 @@ void test_svst1b_scatter_u64base_offset_s64(svbool_t pg, svuint64_t bases, int64
   return SVE_ACLE_FUNC(svst1b_scatter,_u64base,_offset,_s64)(pg, bases, offset, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1b_scatter_u32base_offset_u32(
+// CHECK-LABEL: @test_svst1b_scatter_u32base_offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 4 x i32> [[DATA:%.*]] to <vscale x 4 x i8>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -341,7 +341,7 @@ void test_svst1b_scatter_u32base_offset_u32(svbool_t pg, svuint32_t bases, int64
   return SVE_ACLE_FUNC(svst1b_scatter,_u32base,_offset,_u32)(pg, bases, offset, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1b_scatter_u64base_offset_u64(
+// CHECK-LABEL: @test_svst1b_scatter_u64base_offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i8>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1h.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1h.c
index 23a9a05345..e36788f22e 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1h.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1h.c
@@ -11,7 +11,7 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1h_s32(
+// CHECK-LABEL: @test_svst1h_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = trunc <vscale x 4 x i32> [[DATA:%.*]] to <vscale x 4 x i16>
@@ -23,7 +23,7 @@ void test_svst1h_s32(svbool_t pg, int16_t *base, svint32_t data)
   return SVE_ACLE_FUNC(svst1h,_s32,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1h_s64(
+// CHECK-LABEL: @test_svst1h_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i16>
@@ -35,7 +35,7 @@ void test_svst1h_s64(svbool_t pg, int16_t *base, svint64_t data)
   return SVE_ACLE_FUNC(svst1h,_s64,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1h_u32(
+// CHECK-LABEL: @test_svst1h_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = trunc <vscale x 4 x i32> [[DATA:%.*]] to <vscale x 4 x i16>
@@ -47,7 +47,7 @@ void test_svst1h_u32(svbool_t pg, uint16_t *base, svuint32_t data)
   return SVE_ACLE_FUNC(svst1h,_u32,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1h_u64(
+// CHECK-LABEL: @test_svst1h_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i16>
@@ -59,7 +59,7 @@ void test_svst1h_u64(svbool_t pg, uint16_t *base, svuint64_t data)
   return SVE_ACLE_FUNC(svst1h,_u64,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1h_vnum_s32(
+// CHECK-LABEL: @test_svst1h_vnum_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -72,7 +72,7 @@ void test_svst1h_vnum_s32(svbool_t pg, int16_t *base, int64_t vnum, svint32_t da
   return SVE_ACLE_FUNC(svst1h_vnum,_s32,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1h_vnum_s64(
+// CHECK-LABEL: @test_svst1h_vnum_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -85,7 +85,7 @@ void test_svst1h_vnum_s64(svbool_t pg, int16_t *base, int64_t vnum, svint64_t da
   return SVE_ACLE_FUNC(svst1h_vnum,_s64,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1h_vnum_u32(
+// CHECK-LABEL: @test_svst1h_vnum_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -98,7 +98,7 @@ void test_svst1h_vnum_u32(svbool_t pg, uint16_t *base, int64_t vnum, svuint32_t
   return SVE_ACLE_FUNC(svst1h_vnum,_u32,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1h_vnum_u64(
+// CHECK-LABEL: @test_svst1h_vnum_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -111,7 +111,7 @@ void test_svst1h_vnum_u64(svbool_t pg, uint16_t *base, int64_t vnum, svuint64_t
   return SVE_ACLE_FUNC(svst1h_vnum,_u64,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1h_scatter_u32base_s32(
+// CHECK-LABEL: @test_svst1h_scatter_u32base_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 4 x i32> [[DATA:%.*]] to <vscale x 4 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -123,7 +123,7 @@ void test_svst1h_scatter_u32base_s32(svbool_t pg, svuint32_t bases, svint32_t da
   return SVE_ACLE_FUNC(svst1h_scatter,_u32base,,_s32)(pg, bases, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1h_scatter_u64base_s64(
+// CHECK-LABEL: @test_svst1h_scatter_u64base_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -135,7 +135,7 @@ void test_svst1h_scatter_u64base_s64(svbool_t pg, svuint64_t bases, svint64_t da
   return SVE_ACLE_FUNC(svst1h_scatter,_u64base,,_s64)(pg, bases, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1h_scatter_u32base_u32(
+// CHECK-LABEL: @test_svst1h_scatter_u32base_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 4 x i32> [[DATA:%.*]] to <vscale x 4 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -147,7 +147,7 @@ void test_svst1h_scatter_u32base_u32(svbool_t pg, svuint32_t bases, svuint32_t d
   return SVE_ACLE_FUNC(svst1h_scatter,_u32base,,_u32)(pg, bases, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1h_scatter_u64base_u64(
+// CHECK-LABEL: @test_svst1h_scatter_u64base_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -159,7 +159,7 @@ void test_svst1h_scatter_u64base_u64(svbool_t pg, svuint64_t bases, svuint64_t d
   return SVE_ACLE_FUNC(svst1h_scatter,_u64base,,_u64)(pg, bases, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1h_scatter_s32offset_s32(
+// CHECK-LABEL: @test_svst1h_scatter_s32offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 4 x i32> [[DATA:%.*]] to <vscale x 4 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -171,7 +171,7 @@ void test_svst1h_scatter_s32offset_s32(svbool_t pg, int16_t *base, svint32_t off
   return SVE_ACLE_FUNC(svst1h_scatter_,s32,offset,_s32)(pg, base, offsets, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1h_scatter_s64offset_s64(
+// CHECK-LABEL: @test_svst1h_scatter_s64offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -183,7 +183,7 @@ void test_svst1h_scatter_s64offset_s64(svbool_t pg, int16_t *base, svint64_t off
   return SVE_ACLE_FUNC(svst1h_scatter_,s64,offset,_s64)(pg, base, offsets, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1h_scatter_s32offset_u32(
+// CHECK-LABEL: @test_svst1h_scatter_s32offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 4 x i32> [[DATA:%.*]] to <vscale x 4 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -195,7 +195,7 @@ void test_svst1h_scatter_s32offset_u32(svbool_t pg, uint16_t *base, svint32_t of
   return SVE_ACLE_FUNC(svst1h_scatter_,s32,offset,_u32)(pg, base, offsets, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1h_scatter_s64offset_u64(
+// CHECK-LABEL: @test_svst1h_scatter_s64offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -207,7 +207,7 @@ void test_svst1h_scatter_s64offset_u64(svbool_t pg, uint16_t *base, svint64_t of
   return SVE_ACLE_FUNC(svst1h_scatter_,s64,offset,_u64)(pg, base, offsets, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1h_scatter_u32offset_s32(
+// CHECK-LABEL: @test_svst1h_scatter_u32offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 4 x i32> [[DATA:%.*]] to <vscale x 4 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -219,7 +219,7 @@ void test_svst1h_scatter_u32offset_s32(svbool_t pg, int16_t *base, svuint32_t of
   return SVE_ACLE_FUNC(svst1h_scatter_,u32,offset,_s32)(pg, base, offsets, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1h_scatter_u64offset_s64(
+// CHECK-LABEL: @test_svst1h_scatter_u64offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -231,7 +231,7 @@ void test_svst1h_scatter_u64offset_s64(svbool_t pg, int16_t *base, svuint64_t of
   return SVE_ACLE_FUNC(svst1h_scatter_,u64,offset,_s64)(pg, base, offsets, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1h_scatter_u32offset_u32(
+// CHECK-LABEL: @test_svst1h_scatter_u32offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 4 x i32> [[DATA:%.*]] to <vscale x 4 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -243,7 +243,7 @@ void test_svst1h_scatter_u32offset_u32(svbool_t pg, uint16_t *base, svuint32_t o
   return SVE_ACLE_FUNC(svst1h_scatter_,u32,offset,_u32)(pg, base, offsets, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1h_scatter_u64offset_u64(
+// CHECK-LABEL: @test_svst1h_scatter_u64offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -255,7 +255,7 @@ void test_svst1h_scatter_u64offset_u64(svbool_t pg, uint16_t *base, svuint64_t o
   return SVE_ACLE_FUNC(svst1h_scatter_,u64,offset,_u64)(pg, base, offsets, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1h_scatter_u32base_offset_s32(
+// CHECK-LABEL: @test_svst1h_scatter_u32base_offset_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 4 x i32> [[DATA:%.*]] to <vscale x 4 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -267,7 +267,7 @@ void test_svst1h_scatter_u32base_offset_s32(svbool_t pg, svuint32_t bases, int64
   return SVE_ACLE_FUNC(svst1h_scatter,_u32base,_offset,_s32)(pg, bases, offset, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1h_scatter_u64base_offset_s64(
+// CHECK-LABEL: @test_svst1h_scatter_u64base_offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -279,7 +279,7 @@ void test_svst1h_scatter_u64base_offset_s64(svbool_t pg, svuint64_t bases, int64
   return SVE_ACLE_FUNC(svst1h_scatter,_u64base,_offset,_s64)(pg, bases, offset, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1h_scatter_u32base_offset_u32(
+// CHECK-LABEL: @test_svst1h_scatter_u32base_offset_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 4 x i32> [[DATA:%.*]] to <vscale x 4 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -291,7 +291,7 @@ void test_svst1h_scatter_u32base_offset_u32(svbool_t pg, svuint32_t bases, int64
   return SVE_ACLE_FUNC(svst1h_scatter,_u32base,_offset,_u32)(pg, bases, offset, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1h_scatter_u64base_offset_u64(
+// CHECK-LABEL: @test_svst1h_scatter_u64base_offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -303,7 +303,7 @@ void test_svst1h_scatter_u64base_offset_u64(svbool_t pg, svuint64_t bases, int64
   return SVE_ACLE_FUNC(svst1h_scatter,_u64base,_offset,_u64)(pg, bases, offset, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1h_scatter_s32index_s32(
+// CHECK-LABEL: @test_svst1h_scatter_s32index_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 4 x i32> [[DATA:%.*]] to <vscale x 4 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -315,7 +315,7 @@ void test_svst1h_scatter_s32index_s32(svbool_t pg, int16_t *base, svint32_t indi
   return SVE_ACLE_FUNC(svst1h_scatter_,s32,index,_s32)(pg, base, indices, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1h_scatter_s64index_s64(
+// CHECK-LABEL: @test_svst1h_scatter_s64index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -327,7 +327,7 @@ void test_svst1h_scatter_s64index_s64(svbool_t pg, int16_t *base, svint64_t indi
   return SVE_ACLE_FUNC(svst1h_scatter_,s64,index,_s64)(pg, base, indices, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1h_scatter_s32index_u32(
+// CHECK-LABEL: @test_svst1h_scatter_s32index_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 4 x i32> [[DATA:%.*]] to <vscale x 4 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -339,7 +339,7 @@ void test_svst1h_scatter_s32index_u32(svbool_t pg, uint16_t *base, svint32_t ind
   return SVE_ACLE_FUNC(svst1h_scatter_,s32,index,_u32)(pg, base, indices, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1h_scatter_s64index_u64(
+// CHECK-LABEL: @test_svst1h_scatter_s64index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -351,7 +351,7 @@ void test_svst1h_scatter_s64index_u64(svbool_t pg, uint16_t *base, svint64_t ind
   return SVE_ACLE_FUNC(svst1h_scatter_,s64,index,_u64)(pg, base, indices, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1h_scatter_u32index_s32(
+// CHECK-LABEL: @test_svst1h_scatter_u32index_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 4 x i32> [[DATA:%.*]] to <vscale x 4 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -363,7 +363,7 @@ void test_svst1h_scatter_u32index_s32(svbool_t pg, int16_t *base, svuint32_t ind
   return SVE_ACLE_FUNC(svst1h_scatter_,u32,index,_s32)(pg, base, indices, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1h_scatter_u64index_s64(
+// CHECK-LABEL: @test_svst1h_scatter_u64index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -375,7 +375,7 @@ void test_svst1h_scatter_u64index_s64(svbool_t pg, int16_t *base, svuint64_t ind
   return SVE_ACLE_FUNC(svst1h_scatter_,u64,index,_s64)(pg, base, indices, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1h_scatter_u32index_u32(
+// CHECK-LABEL: @test_svst1h_scatter_u32index_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 4 x i32> [[DATA:%.*]] to <vscale x 4 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -387,7 +387,7 @@ void test_svst1h_scatter_u32index_u32(svbool_t pg, uint16_t *base, svuint32_t in
   return SVE_ACLE_FUNC(svst1h_scatter_,u32,index,_u32)(pg, base, indices, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1h_scatter_u64index_u64(
+// CHECK-LABEL: @test_svst1h_scatter_u64index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -399,7 +399,7 @@ void test_svst1h_scatter_u64index_u64(svbool_t pg, uint16_t *base, svuint64_t in
   return SVE_ACLE_FUNC(svst1h_scatter_,u64,index,_u64)(pg, base, indices, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1h_scatter_u32base_index_s32(
+// CHECK-LABEL: @test_svst1h_scatter_u32base_index_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 4 x i32> [[DATA:%.*]] to <vscale x 4 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -412,7 +412,7 @@ void test_svst1h_scatter_u32base_index_s32(svbool_t pg, svuint32_t bases, int64_
   return SVE_ACLE_FUNC(svst1h_scatter,_u32base,_index,_s32)(pg, bases, index, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1h_scatter_u64base_index_s64(
+// CHECK-LABEL: @test_svst1h_scatter_u64base_index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -425,7 +425,7 @@ void test_svst1h_scatter_u64base_index_s64(svbool_t pg, svuint64_t bases, int64_
   return SVE_ACLE_FUNC(svst1h_scatter,_u64base,_index,_s64)(pg, bases, index, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1h_scatter_u32base_index_u32(
+// CHECK-LABEL: @test_svst1h_scatter_u32base_index_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 4 x i32> [[DATA:%.*]] to <vscale x 4 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -438,7 +438,7 @@ void test_svst1h_scatter_u32base_index_u32(svbool_t pg, svuint32_t bases, int64_
   return SVE_ACLE_FUNC(svst1h_scatter,_u32base,_index,_u32)(pg, bases, index, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1h_scatter_u64base_index_u64(
+// CHECK-LABEL: @test_svst1h_scatter_u64base_index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1w.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1w.c
index b2f6961391..841f6926f1 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1w.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st1w.c
@@ -12,7 +12,7 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1w_s64(
+// CHECK-LABEL: @test_svst1w_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i32>
@@ -24,7 +24,7 @@ void test_svst1w_s64(svbool_t pg, int32_t *base, svint64_t data)
   return SVE_ACLE_FUNC(svst1w,_s64,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1w_u64(
+// CHECK-LABEL: @test_svst1w_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i32>
@@ -36,7 +36,7 @@ void test_svst1w_u64(svbool_t pg, uint32_t *base, svuint64_t data)
   return SVE_ACLE_FUNC(svst1w,_u64,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1w_vnum_s64(
+// CHECK-LABEL: @test_svst1w_vnum_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -49,7 +49,7 @@ void test_svst1w_vnum_s64(svbool_t pg, int32_t *base, int64_t vnum, svint64_t da
   return SVE_ACLE_FUNC(svst1w_vnum,_s64,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1w_vnum_u64(
+// CHECK-LABEL: @test_svst1w_vnum_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -62,7 +62,7 @@ void test_svst1w_vnum_u64(svbool_t pg, uint32_t *base, int64_t vnum, svuint64_t
   return SVE_ACLE_FUNC(svst1w_vnum,_u64,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1w_scatter_u64base_s64(
+// CHECK-LABEL: @test_svst1w_scatter_u64base_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i32>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -74,7 +74,7 @@ void test_svst1w_scatter_u64base_s64(svbool_t pg, svuint64_t bases, svint64_t da
   return SVE_ACLE_FUNC(svst1w_scatter,_u64base,,_s64)(pg, bases, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1w_scatter_u64base_u64(
+// CHECK-LABEL: @test_svst1w_scatter_u64base_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i32>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -86,7 +86,7 @@ void test_svst1w_scatter_u64base_u64(svbool_t pg, svuint64_t bases, svuint64_t d
   return SVE_ACLE_FUNC(svst1w_scatter,_u64base,,_u64)(pg, bases, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1w_scatter_s64offset_s64(
+// CHECK-LABEL: @test_svst1w_scatter_s64offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i32>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -98,7 +98,7 @@ void test_svst1w_scatter_s64offset_s64(svbool_t pg, int32_t *base, svint64_t off
   return SVE_ACLE_FUNC(svst1w_scatter_,s64,offset,_s64)(pg, base, offsets, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1w_scatter_s64offset_u64(
+// CHECK-LABEL: @test_svst1w_scatter_s64offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i32>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -110,7 +110,7 @@ void test_svst1w_scatter_s64offset_u64(svbool_t pg, uint32_t *base, svint64_t of
   return SVE_ACLE_FUNC(svst1w_scatter_,s64,offset,_u64)(pg, base, offsets, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1w_scatter_u64offset_s64(
+// CHECK-LABEL: @test_svst1w_scatter_u64offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i32>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -122,7 +122,7 @@ void test_svst1w_scatter_u64offset_s64(svbool_t pg, int32_t *base, svuint64_t of
   return SVE_ACLE_FUNC(svst1w_scatter_,u64,offset,_s64)(pg, base, offsets, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1w_scatter_u64offset_u64(
+// CHECK-LABEL: @test_svst1w_scatter_u64offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i32>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -134,7 +134,7 @@ void test_svst1w_scatter_u64offset_u64(svbool_t pg, uint32_t *base, svuint64_t o
   return SVE_ACLE_FUNC(svst1w_scatter_,u64,offset,_u64)(pg, base, offsets, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1w_scatter_u64base_offset_s64(
+// CHECK-LABEL: @test_svst1w_scatter_u64base_offset_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i32>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -146,7 +146,7 @@ void test_svst1w_scatter_u64base_offset_s64(svbool_t pg, svuint64_t bases, int64
   return SVE_ACLE_FUNC(svst1w_scatter,_u64base,_offset,_s64)(pg, bases, offset, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1w_scatter_u64base_offset_u64(
+// CHECK-LABEL: @test_svst1w_scatter_u64base_offset_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i32>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -158,7 +158,7 @@ void test_svst1w_scatter_u64base_offset_u64(svbool_t pg, svuint64_t bases, int64
   return SVE_ACLE_FUNC(svst1w_scatter,_u64base,_offset,_u64)(pg, bases, offset, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1w_scatter_s64index_s64(
+// CHECK-LABEL: @test_svst1w_scatter_s64index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i32>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -170,7 +170,7 @@ void test_svst1w_scatter_s64index_s64(svbool_t pg, int32_t *base, svint64_t indi
   return SVE_ACLE_FUNC(svst1w_scatter_,s64,index,_s64)(pg, base, indices, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1w_scatter_s64index_u64(
+// CHECK-LABEL: @test_svst1w_scatter_s64index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i32>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -182,7 +182,7 @@ void test_svst1w_scatter_s64index_u64(svbool_t pg, uint32_t *base, svint64_t ind
   return SVE_ACLE_FUNC(svst1w_scatter_,s64,index,_u64)(pg, base, indices, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1w_scatter_u64index_s64(
+// CHECK-LABEL: @test_svst1w_scatter_u64index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i32>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -194,7 +194,7 @@ void test_svst1w_scatter_u64index_s64(svbool_t pg, int32_t *base, svuint64_t ind
   return SVE_ACLE_FUNC(svst1w_scatter_,u64,index,_s64)(pg, base, indices, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1w_scatter_u64index_u64(
+// CHECK-LABEL: @test_svst1w_scatter_u64index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i32>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -206,7 +206,7 @@ void test_svst1w_scatter_u64index_u64(svbool_t pg, uint32_t *base, svuint64_t in
   return SVE_ACLE_FUNC(svst1w_scatter_,u64,index,_u64)(pg, base, indices, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1w_scatter_u64base_index_s64(
+// CHECK-LABEL: @test_svst1w_scatter_u64base_index_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i32>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
@@ -219,7 +219,7 @@ void test_svst1w_scatter_u64base_index_s64(svbool_t pg, svuint64_t bases, int64_
   return SVE_ACLE_FUNC(svst1w_scatter,_u64base,_index,_s64)(pg, bases, index, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst1w_scatter_u64base_index_u64(
+// CHECK-LABEL: @test_svst1w_scatter_u64base_index_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <vscale x 2 x i64> [[DATA:%.*]] to <vscale x 2 x i32>
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st2-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st2-bfloat.c
index fcfa7eb576..abe59567b0 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st2-bfloat.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st2-bfloat.c
@@ -14,7 +14,7 @@
 #else
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
-// CHECK-LABEL: define {{[^@]+}}@test_svst2_bf16(
+// CHECK-LABEL: @test_svst2_bf16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[DATA:%.*]], i64 0)
@@ -22,7 +22,7 @@
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st2.nxv8bf16(<vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z15test_svst2_bf16u10__SVBool_tPu6__bf1614svbfloat16x2_t(
+// CPP-CHECK-LABEL: @_Z15test_svst2_bf16u10__SVBool_tPu6__bf1614svbfloat16x2_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv16bf16(<vscale x 16 x bfloat> [[DATA:%.*]], i64 0)
@@ -35,7 +35,7 @@ void test_svst2_bf16(svbool_t pg, bfloat16_t *base, svbfloat16x2_t data)
   return SVE_ACLE_FUNC(svst2,_bf16,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst2_vnum_bf16(
+// CHECK-LABEL: @test_svst2_vnum_bf16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x bfloat>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -44,7 +44,7 @@ void test_svst2_bf16(svbool_t pg, bfloat16_t *base, svbfloat16x2_t data)
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st2.nxv8bf16(<vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z20test_svst2_vnum_bf16u10__SVBool_tPu6__bf16l14svbfloat16x2_t(
+// CPP-CHECK-LABEL: @_Z20test_svst2_vnum_bf16u10__SVBool_tPu6__bf16l14svbfloat16x2_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x bfloat>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st2.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st2.c
index 3235a8b9e1..6a6632d51b 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st2.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st2.c
@@ -14,14 +14,14 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst2_s8(
+// CHECK-LABEL: @test_svst2_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[DATA:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[DATA]], i64 16)
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st2.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z13test_svst2_s8u10__SVBool_tPa10svint8x2_t(
+// CPP-CHECK-LABEL: @_Z13test_svst2_s8u10__SVBool_tPa10svint8x2_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[DATA:%.*]], i64 0)
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[DATA]], i64 16)
@@ -33,7 +33,7 @@ void test_svst2_s8(svbool_t pg, int8_t *base, svint8x2_t data)
   return SVE_ACLE_FUNC(svst2,_s8,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst2_s16(
+// CHECK-LABEL: @test_svst2_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[DATA:%.*]], i64 0)
@@ -41,7 +41,7 @@ void test_svst2_s8(svbool_t pg, int8_t *base, svint8x2_t data)
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st2.nxv8i16(<vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svst2_s16u10__SVBool_tPs11svint16x2_t(
+// CPP-CHECK-LABEL: @_Z14test_svst2_s16u10__SVBool_tPs11svint16x2_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[DATA:%.*]], i64 0)
@@ -54,7 +54,7 @@ void test_svst2_s16(svbool_t pg, int16_t *base, svint16x2_t data)
   return SVE_ACLE_FUNC(svst2,_s16,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst2_s32(
+// CHECK-LABEL: @test_svst2_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[DATA:%.*]], i64 0)
@@ -62,7 +62,7 @@ void test_svst2_s16(svbool_t pg, int16_t *base, svint16x2_t data)
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st2.nxv4i32(<vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svst2_s32u10__SVBool_tPi11svint32x2_t(
+// CPP-CHECK-LABEL: @_Z14test_svst2_s32u10__SVBool_tPi11svint32x2_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[DATA:%.*]], i64 0)
@@ -75,7 +75,7 @@ void test_svst2_s32(svbool_t pg, int32_t *base, svint32x2_t data)
   return SVE_ACLE_FUNC(svst2,_s32,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst2_s64(
+// CHECK-LABEL: @test_svst2_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[DATA:%.*]], i64 0)
@@ -83,7 +83,7 @@ void test_svst2_s32(svbool_t pg, int32_t *base, svint32x2_t data)
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st2.nxv2i64(<vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svst2_s64u10__SVBool_tPl11svint64x2_t(
+// CPP-CHECK-LABEL: @_Z14test_svst2_s64u10__SVBool_tPl11svint64x2_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[DATA:%.*]], i64 0)
@@ -96,14 +96,14 @@ void test_svst2_s64(svbool_t pg, int64_t *base, svint64x2_t data)
   return SVE_ACLE_FUNC(svst2,_s64,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst2_u8(
+// CHECK-LABEL: @test_svst2_u8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[DATA:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[DATA]], i64 16)
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st2.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z13test_svst2_u8u10__SVBool_tPh11svuint8x2_t(
+// CPP-CHECK-LABEL: @_Z13test_svst2_u8u10__SVBool_tPh11svuint8x2_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[DATA:%.*]], i64 0)
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[DATA]], i64 16)
@@ -115,7 +115,7 @@ void test_svst2_u8(svbool_t pg, uint8_t *base, svuint8x2_t data)
   return SVE_ACLE_FUNC(svst2,_u8,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst2_u16(
+// CHECK-LABEL: @test_svst2_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[DATA:%.*]], i64 0)
@@ -123,7 +123,7 @@ void test_svst2_u8(svbool_t pg, uint8_t *base, svuint8x2_t data)
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st2.nxv8i16(<vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svst2_u16u10__SVBool_tPt12svuint16x2_t(
+// CPP-CHECK-LABEL: @_Z14test_svst2_u16u10__SVBool_tPt12svuint16x2_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[DATA:%.*]], i64 0)
@@ -136,7 +136,7 @@ void test_svst2_u16(svbool_t pg, uint16_t *base, svuint16x2_t data)
   return SVE_ACLE_FUNC(svst2,_u16,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst2_u32(
+// CHECK-LABEL: @test_svst2_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[DATA:%.*]], i64 0)
@@ -144,7 +144,7 @@ void test_svst2_u16(svbool_t pg, uint16_t *base, svuint16x2_t data)
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st2.nxv4i32(<vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svst2_u32u10__SVBool_tPj12svuint32x2_t(
+// CPP-CHECK-LABEL: @_Z14test_svst2_u32u10__SVBool_tPj12svuint32x2_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[DATA:%.*]], i64 0)
@@ -157,7 +157,7 @@ void test_svst2_u32(svbool_t pg, uint32_t *base, svuint32x2_t data)
   return SVE_ACLE_FUNC(svst2,_u32,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst2_u64(
+// CHECK-LABEL: @test_svst2_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[DATA:%.*]], i64 0)
@@ -165,7 +165,7 @@ void test_svst2_u32(svbool_t pg, uint32_t *base, svuint32x2_t data)
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st2.nxv2i64(<vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svst2_u64u10__SVBool_tPm12svuint64x2_t(
+// CPP-CHECK-LABEL: @_Z14test_svst2_u64u10__SVBool_tPm12svuint64x2_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[DATA:%.*]], i64 0)
@@ -178,7 +178,7 @@ void test_svst2_u64(svbool_t pg, uint64_t *base, svuint64x2_t data)
   return SVE_ACLE_FUNC(svst2,_u64,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst2_f16(
+// CHECK-LABEL: @test_svst2_f16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[DATA:%.*]], i64 0)
@@ -186,7 +186,7 @@ void test_svst2_u64(svbool_t pg, uint64_t *base, svuint64x2_t data)
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st2.nxv8f16(<vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svst2_f16u10__SVBool_tPDh13svfloat16x2_t(
+// CPP-CHECK-LABEL: @_Z14test_svst2_f16u10__SVBool_tPDh13svfloat16x2_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[DATA:%.*]], i64 0)
@@ -199,7 +199,7 @@ void test_svst2_f16(svbool_t pg, float16_t *base, svfloat16x2_t data)
   return SVE_ACLE_FUNC(svst2,_f16,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst2_f32(
+// CHECK-LABEL: @test_svst2_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[DATA:%.*]], i64 0)
@@ -207,7 +207,7 @@ void test_svst2_f16(svbool_t pg, float16_t *base, svfloat16x2_t data)
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st2.nxv4f32(<vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svst2_f32u10__SVBool_tPf13svfloat32x2_t(
+// CPP-CHECK-LABEL: @_Z14test_svst2_f32u10__SVBool_tPf13svfloat32x2_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[DATA:%.*]], i64 0)
@@ -220,7 +220,7 @@ void test_svst2_f32(svbool_t pg, float32_t *base, svfloat32x2_t data)
   return SVE_ACLE_FUNC(svst2,_f32,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst2_f64(
+// CHECK-LABEL: @test_svst2_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[DATA:%.*]], i64 0)
@@ -228,7 +228,7 @@ void test_svst2_f32(svbool_t pg, float32_t *base, svfloat32x2_t data)
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st2.nxv2f64(<vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svst2_f64u10__SVBool_tPd13svfloat64x2_t(
+// CPP-CHECK-LABEL: @_Z14test_svst2_f64u10__SVBool_tPd13svfloat64x2_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[DATA:%.*]], i64 0)
@@ -241,7 +241,7 @@ void test_svst2_f64(svbool_t pg, float64_t *base, svfloat64x2_t data)
   return SVE_ACLE_FUNC(svst2,_f64,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst2_vnum_s8(
+// CHECK-LABEL: @test_svst2_vnum_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[DATA:%.*]], i64 0)
@@ -249,7 +249,7 @@ void test_svst2_f64(svbool_t pg, float64_t *base, svfloat64x2_t data)
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st2.nxv16i8(<vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svst2_vnum_s8u10__SVBool_tPal10svint8x2_t(
+// CPP-CHECK-LABEL: @_Z18test_svst2_vnum_s8u10__SVBool_tPal10svint8x2_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[DATA:%.*]], i64 0)
@@ -262,7 +262,7 @@ void test_svst2_vnum_s8(svbool_t pg, int8_t *base, int64_t vnum, svint8x2_t data
   return SVE_ACLE_FUNC(svst2_vnum,_s8,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst2_vnum_s16(
+// CHECK-LABEL: @test_svst2_vnum_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -271,7 +271,7 @@ void test_svst2_vnum_s8(svbool_t pg, int8_t *base, int64_t vnum, svint8x2_t data
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st2.nxv8i16(<vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svst2_vnum_s16u10__SVBool_tPsl11svint16x2_t(
+// CPP-CHECK-LABEL: @_Z19test_svst2_vnum_s16u10__SVBool_tPsl11svint16x2_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -285,7 +285,7 @@ void test_svst2_vnum_s16(svbool_t pg, int16_t *base, int64_t vnum, svint16x2_t d
   return SVE_ACLE_FUNC(svst2_vnum,_s16,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst2_vnum_s32(
+// CHECK-LABEL: @test_svst2_vnum_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -294,7 +294,7 @@ void test_svst2_vnum_s16(svbool_t pg, int16_t *base, int64_t vnum, svint16x2_t d
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st2.nxv4i32(<vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], <vscale x 4 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svst2_vnum_s32u10__SVBool_tPil11svint32x2_t(
+// CPP-CHECK-LABEL: @_Z19test_svst2_vnum_s32u10__SVBool_tPil11svint32x2_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -308,7 +308,7 @@ void test_svst2_vnum_s32(svbool_t pg, int32_t *base, int64_t vnum, svint32x2_t d
   return SVE_ACLE_FUNC(svst2_vnum,_s32,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst2_vnum_s64(
+// CHECK-LABEL: @test_svst2_vnum_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -317,7 +317,7 @@ void test_svst2_vnum_s32(svbool_t pg, int32_t *base, int64_t vnum, svint32x2_t d
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st2.nxv2i64(<vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], <vscale x 2 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svst2_vnum_s64u10__SVBool_tPll11svint64x2_t(
+// CPP-CHECK-LABEL: @_Z19test_svst2_vnum_s64u10__SVBool_tPll11svint64x2_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -331,7 +331,7 @@ void test_svst2_vnum_s64(svbool_t pg, int64_t *base, int64_t vnum, svint64x2_t d
   return SVE_ACLE_FUNC(svst2_vnum,_s64,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst2_vnum_u8(
+// CHECK-LABEL: @test_svst2_vnum_u8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[DATA:%.*]], i64 0)
@@ -339,7 +339,7 @@ void test_svst2_vnum_s64(svbool_t pg, int64_t *base, int64_t vnum, svint64x2_t d
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st2.nxv16i8(<vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svst2_vnum_u8u10__SVBool_tPhl11svuint8x2_t(
+// CPP-CHECK-LABEL: @_Z18test_svst2_vnum_u8u10__SVBool_tPhl11svuint8x2_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[DATA:%.*]], i64 0)
@@ -352,7 +352,7 @@ void test_svst2_vnum_u8(svbool_t pg, uint8_t *base, int64_t vnum, svuint8x2_t da
   return SVE_ACLE_FUNC(svst2_vnum,_u8,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst2_vnum_u16(
+// CHECK-LABEL: @test_svst2_vnum_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -361,7 +361,7 @@ void test_svst2_vnum_u8(svbool_t pg, uint8_t *base, int64_t vnum, svuint8x2_t da
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st2.nxv8i16(<vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svst2_vnum_u16u10__SVBool_tPtl12svuint16x2_t(
+// CPP-CHECK-LABEL: @_Z19test_svst2_vnum_u16u10__SVBool_tPtl12svuint16x2_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -375,7 +375,7 @@ void test_svst2_vnum_u16(svbool_t pg, uint16_t *base, int64_t vnum, svuint16x2_t
   return SVE_ACLE_FUNC(svst2_vnum,_u16,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst2_vnum_u32(
+// CHECK-LABEL: @test_svst2_vnum_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -384,7 +384,7 @@ void test_svst2_vnum_u16(svbool_t pg, uint16_t *base, int64_t vnum, svuint16x2_t
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st2.nxv4i32(<vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], <vscale x 4 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svst2_vnum_u32u10__SVBool_tPjl12svuint32x2_t(
+// CPP-CHECK-LABEL: @_Z19test_svst2_vnum_u32u10__SVBool_tPjl12svuint32x2_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -398,7 +398,7 @@ void test_svst2_vnum_u32(svbool_t pg, uint32_t *base, int64_t vnum, svuint32x2_t
   return SVE_ACLE_FUNC(svst2_vnum,_u32,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst2_vnum_u64(
+// CHECK-LABEL: @test_svst2_vnum_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -407,7 +407,7 @@ void test_svst2_vnum_u32(svbool_t pg, uint32_t *base, int64_t vnum, svuint32x2_t
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st2.nxv2i64(<vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], <vscale x 2 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svst2_vnum_u64u10__SVBool_tPml12svuint64x2_t(
+// CPP-CHECK-LABEL: @_Z19test_svst2_vnum_u64u10__SVBool_tPml12svuint64x2_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -421,7 +421,7 @@ void test_svst2_vnum_u64(svbool_t pg, uint64_t *base, int64_t vnum, svuint64x2_t
   return SVE_ACLE_FUNC(svst2_vnum,_u64,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst2_vnum_f16(
+// CHECK-LABEL: @test_svst2_vnum_f16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -430,7 +430,7 @@ void test_svst2_vnum_u64(svbool_t pg, uint64_t *base, int64_t vnum, svuint64x2_t
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st2.nxv8f16(<vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svst2_vnum_f16u10__SVBool_tPDhl13svfloat16x2_t(
+// CPP-CHECK-LABEL: @_Z19test_svst2_vnum_f16u10__SVBool_tPDhl13svfloat16x2_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -444,7 +444,7 @@ void test_svst2_vnum_f16(svbool_t pg, float16_t *base, int64_t vnum, svfloat16x2
   return SVE_ACLE_FUNC(svst2_vnum,_f16,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst2_vnum_f32(
+// CHECK-LABEL: @test_svst2_vnum_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -453,7 +453,7 @@ void test_svst2_vnum_f16(svbool_t pg, float16_t *base, int64_t vnum, svfloat16x2
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st2.nxv4f32(<vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svst2_vnum_f32u10__SVBool_tPfl13svfloat32x2_t(
+// CPP-CHECK-LABEL: @_Z19test_svst2_vnum_f32u10__SVBool_tPfl13svfloat32x2_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -467,7 +467,7 @@ void test_svst2_vnum_f32(svbool_t pg, float32_t *base, int64_t vnum, svfloat32x2
   return SVE_ACLE_FUNC(svst2_vnum,_f32,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst2_vnum_f64(
+// CHECK-LABEL: @test_svst2_vnum_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -476,7 +476,7 @@ void test_svst2_vnum_f32(svbool_t pg, float32_t *base, int64_t vnum, svfloat32x2
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st2.nxv2f64(<vscale x 2 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], <vscale x 2 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svst2_vnum_f64u10__SVBool_tPdl13svfloat64x2_t(
+// CPP-CHECK-LABEL: @_Z19test_svst2_vnum_f64u10__SVBool_tPdl13svfloat64x2_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st3-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st3-bfloat.c
index 096699191e..30954b4eaa 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st3-bfloat.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st3-bfloat.c
@@ -15,7 +15,7 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst3_bf16(
+// CHECK-LABEL: @test_svst3_bf16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv24bf16(<vscale x 24 x bfloat> [[DATA:%.*]], i64 0)
@@ -24,7 +24,7 @@
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st3.nxv8bf16(<vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z15test_svst3_bf16u10__SVBool_tPu6__bf1614svbfloat16x3_t(
+// CPP-CHECK-LABEL: @_Z15test_svst3_bf16u10__SVBool_tPu6__bf1614svbfloat16x3_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv24bf16(<vscale x 24 x bfloat> [[DATA:%.*]], i64 0)
@@ -38,7 +38,7 @@ void test_svst3_bf16(svbool_t pg, bfloat16_t *base, svbfloat16x3_t data)
   return SVE_ACLE_FUNC(svst3,_bf16,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst3_vnum_bf16(
+// CHECK-LABEL: @test_svst3_vnum_bf16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x bfloat>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -48,7 +48,7 @@ void test_svst3_bf16(svbool_t pg, bfloat16_t *base, svbfloat16x3_t data)
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st3.nxv8bf16(<vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[TMP4]], <vscale x 8 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z20test_svst3_vnum_bf16u10__SVBool_tPu6__bf16l14svbfloat16x3_t(
+// CPP-CHECK-LABEL: @_Z20test_svst3_vnum_bf16u10__SVBool_tPu6__bf16l14svbfloat16x3_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x bfloat>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st3.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st3.c
index a962f0734d..d78c7d81b9 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st3.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st3.c
@@ -14,7 +14,7 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst3_s8(
+// CHECK-LABEL: @test_svst3_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv48i8(<vscale x 48 x i8> [[DATA:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv48i8(<vscale x 48 x i8> [[DATA]], i64 16)
@@ -22,7 +22,7 @@
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st3.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z13test_svst3_s8u10__SVBool_tPa10svint8x3_t(
+// CPP-CHECK-LABEL: @_Z13test_svst3_s8u10__SVBool_tPa10svint8x3_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv48i8(<vscale x 48 x i8> [[DATA:%.*]], i64 0)
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv48i8(<vscale x 48 x i8> [[DATA]], i64 16)
@@ -35,7 +35,7 @@ void test_svst3_s8(svbool_t pg, int8_t *base, svint8x3_t data)
   return SVE_ACLE_FUNC(svst3,_s8,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst3_s16(
+// CHECK-LABEL: @test_svst3_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv24i16(<vscale x 24 x i16> [[DATA:%.*]], i64 0)
@@ -44,7 +44,7 @@ void test_svst3_s8(svbool_t pg, int8_t *base, svint8x3_t data)
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st3.nxv8i16(<vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svst3_s16u10__SVBool_tPs11svint16x3_t(
+// CPP-CHECK-LABEL: @_Z14test_svst3_s16u10__SVBool_tPs11svint16x3_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv24i16(<vscale x 24 x i16> [[DATA:%.*]], i64 0)
@@ -58,7 +58,7 @@ void test_svst3_s16(svbool_t pg, int16_t *base, svint16x3_t data)
   return SVE_ACLE_FUNC(svst3,_s16,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst3_s32(
+// CHECK-LABEL: @test_svst3_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv12i32(<vscale x 12 x i32> [[DATA:%.*]], i64 0)
@@ -67,7 +67,7 @@ void test_svst3_s16(svbool_t pg, int16_t *base, svint16x3_t data)
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st3.nxv4i32(<vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svst3_s32u10__SVBool_tPi11svint32x3_t(
+// CPP-CHECK-LABEL: @_Z14test_svst3_s32u10__SVBool_tPi11svint32x3_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv12i32(<vscale x 12 x i32> [[DATA:%.*]], i64 0)
@@ -81,7 +81,7 @@ void test_svst3_s32(svbool_t pg, int32_t *base, svint32x3_t data)
   return SVE_ACLE_FUNC(svst3,_s32,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst3_s64(
+// CHECK-LABEL: @test_svst3_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv6i64(<vscale x 6 x i64> [[DATA:%.*]], i64 0)
@@ -90,7 +90,7 @@ void test_svst3_s32(svbool_t pg, int32_t *base, svint32x3_t data)
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st3.nxv2i64(<vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svst3_s64u10__SVBool_tPl11svint64x3_t(
+// CPP-CHECK-LABEL: @_Z14test_svst3_s64u10__SVBool_tPl11svint64x3_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv6i64(<vscale x 6 x i64> [[DATA:%.*]], i64 0)
@@ -104,7 +104,7 @@ void test_svst3_s64(svbool_t pg, int64_t *base, svint64x3_t data)
   return SVE_ACLE_FUNC(svst3,_s64,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst3_u8(
+// CHECK-LABEL: @test_svst3_u8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv48i8(<vscale x 48 x i8> [[DATA:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv48i8(<vscale x 48 x i8> [[DATA]], i64 16)
@@ -112,7 +112,7 @@ void test_svst3_s64(svbool_t pg, int64_t *base, svint64x3_t data)
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st3.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z13test_svst3_u8u10__SVBool_tPh11svuint8x3_t(
+// CPP-CHECK-LABEL: @_Z13test_svst3_u8u10__SVBool_tPh11svuint8x3_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv48i8(<vscale x 48 x i8> [[DATA:%.*]], i64 0)
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv48i8(<vscale x 48 x i8> [[DATA]], i64 16)
@@ -125,7 +125,7 @@ void test_svst3_u8(svbool_t pg, uint8_t *base, svuint8x3_t data)
   return SVE_ACLE_FUNC(svst3,_u8,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst3_u16(
+// CHECK-LABEL: @test_svst3_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv24i16(<vscale x 24 x i16> [[DATA:%.*]], i64 0)
@@ -134,7 +134,7 @@ void test_svst3_u8(svbool_t pg, uint8_t *base, svuint8x3_t data)
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st3.nxv8i16(<vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svst3_u16u10__SVBool_tPt12svuint16x3_t(
+// CPP-CHECK-LABEL: @_Z14test_svst3_u16u10__SVBool_tPt12svuint16x3_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv24i16(<vscale x 24 x i16> [[DATA:%.*]], i64 0)
@@ -148,7 +148,7 @@ void test_svst3_u16(svbool_t pg, uint16_t *base, svuint16x3_t data)
   return SVE_ACLE_FUNC(svst3,_u16,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst3_u32(
+// CHECK-LABEL: @test_svst3_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv12i32(<vscale x 12 x i32> [[DATA:%.*]], i64 0)
@@ -157,7 +157,7 @@ void test_svst3_u16(svbool_t pg, uint16_t *base, svuint16x3_t data)
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st3.nxv4i32(<vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svst3_u32u10__SVBool_tPj12svuint32x3_t(
+// CPP-CHECK-LABEL: @_Z14test_svst3_u32u10__SVBool_tPj12svuint32x3_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv12i32(<vscale x 12 x i32> [[DATA:%.*]], i64 0)
@@ -171,7 +171,7 @@ void test_svst3_u32(svbool_t pg, uint32_t *base, svuint32x3_t data)
   return SVE_ACLE_FUNC(svst3,_u32,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst3_u64(
+// CHECK-LABEL: @test_svst3_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv6i64(<vscale x 6 x i64> [[DATA:%.*]], i64 0)
@@ -180,7 +180,7 @@ void test_svst3_u32(svbool_t pg, uint32_t *base, svuint32x3_t data)
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st3.nxv2i64(<vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svst3_u64u10__SVBool_tPm12svuint64x3_t(
+// CPP-CHECK-LABEL: @_Z14test_svst3_u64u10__SVBool_tPm12svuint64x3_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv6i64(<vscale x 6 x i64> [[DATA:%.*]], i64 0)
@@ -194,7 +194,7 @@ void test_svst3_u64(svbool_t pg, uint64_t *base, svuint64x3_t data)
   return SVE_ACLE_FUNC(svst3,_u64,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst3_f16(
+// CHECK-LABEL: @test_svst3_f16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv24f16(<vscale x 24 x half> [[DATA:%.*]], i64 0)
@@ -203,7 +203,7 @@ void test_svst3_u64(svbool_t pg, uint64_t *base, svuint64x3_t data)
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st3.nxv8f16(<vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svst3_f16u10__SVBool_tPDh13svfloat16x3_t(
+// CPP-CHECK-LABEL: @_Z14test_svst3_f16u10__SVBool_tPDh13svfloat16x3_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv24f16(<vscale x 24 x half> [[DATA:%.*]], i64 0)
@@ -217,7 +217,7 @@ void test_svst3_f16(svbool_t pg, float16_t *base, svfloat16x3_t data)
   return SVE_ACLE_FUNC(svst3,_f16,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst3_f32(
+// CHECK-LABEL: @test_svst3_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv12f32(<vscale x 12 x float> [[DATA:%.*]], i64 0)
@@ -226,7 +226,7 @@ void test_svst3_f16(svbool_t pg, float16_t *base, svfloat16x3_t data)
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st3.nxv4f32(<vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svst3_f32u10__SVBool_tPf13svfloat32x3_t(
+// CPP-CHECK-LABEL: @_Z14test_svst3_f32u10__SVBool_tPf13svfloat32x3_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv12f32(<vscale x 12 x float> [[DATA:%.*]], i64 0)
@@ -240,7 +240,7 @@ void test_svst3_f32(svbool_t pg, float32_t *base, svfloat32x3_t data)
   return SVE_ACLE_FUNC(svst3,_f32,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst3_f64(
+// CHECK-LABEL: @test_svst3_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv6f64(<vscale x 6 x double> [[DATA:%.*]], i64 0)
@@ -249,7 +249,7 @@ void test_svst3_f32(svbool_t pg, float32_t *base, svfloat32x3_t data)
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st3.nxv2f64(<vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svst3_f64u10__SVBool_tPd13svfloat64x3_t(
+// CPP-CHECK-LABEL: @_Z14test_svst3_f64u10__SVBool_tPd13svfloat64x3_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv6f64(<vscale x 6 x double> [[DATA:%.*]], i64 0)
@@ -263,7 +263,7 @@ void test_svst3_f64(svbool_t pg, float64_t *base, svfloat64x3_t data)
   return SVE_ACLE_FUNC(svst3,_f64,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst3_vnum_s8(
+// CHECK-LABEL: @test_svst3_vnum_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv48i8(<vscale x 48 x i8> [[DATA:%.*]], i64 0)
@@ -272,7 +272,7 @@ void test_svst3_f64(svbool_t pg, float64_t *base, svfloat64x3_t data)
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st3.nxv16i8(<vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svst3_vnum_s8u10__SVBool_tPal10svint8x3_t(
+// CPP-CHECK-LABEL: @_Z18test_svst3_vnum_s8u10__SVBool_tPal10svint8x3_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv48i8(<vscale x 48 x i8> [[DATA:%.*]], i64 0)
@@ -286,7 +286,7 @@ void test_svst3_vnum_s8(svbool_t pg, int8_t *base, int64_t vnum, svint8x3_t data
   return SVE_ACLE_FUNC(svst3_vnum,_s8,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst3_vnum_s16(
+// CHECK-LABEL: @test_svst3_vnum_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -296,7 +296,7 @@ void test_svst3_vnum_s8(svbool_t pg, int8_t *base, int64_t vnum, svint8x3_t data
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st3.nxv8i16(<vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], <vscale x 8 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svst3_vnum_s16u10__SVBool_tPsl11svint16x3_t(
+// CPP-CHECK-LABEL: @_Z19test_svst3_vnum_s16u10__SVBool_tPsl11svint16x3_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -311,7 +311,7 @@ void test_svst3_vnum_s16(svbool_t pg, int16_t *base, int64_t vnum, svint16x3_t d
   return SVE_ACLE_FUNC(svst3_vnum,_s16,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst3_vnum_s32(
+// CHECK-LABEL: @test_svst3_vnum_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -321,7 +321,7 @@ void test_svst3_vnum_s16(svbool_t pg, int16_t *base, int64_t vnum, svint16x3_t d
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st3.nxv4i32(<vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], <vscale x 4 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svst3_vnum_s32u10__SVBool_tPil11svint32x3_t(
+// CPP-CHECK-LABEL: @_Z19test_svst3_vnum_s32u10__SVBool_tPil11svint32x3_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -336,7 +336,7 @@ void test_svst3_vnum_s32(svbool_t pg, int32_t *base, int64_t vnum, svint32x3_t d
   return SVE_ACLE_FUNC(svst3_vnum,_s32,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst3_vnum_s64(
+// CHECK-LABEL: @test_svst3_vnum_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -346,7 +346,7 @@ void test_svst3_vnum_s32(svbool_t pg, int32_t *base, int64_t vnum, svint32x3_t d
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st3.nxv2i64(<vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], <vscale x 2 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svst3_vnum_s64u10__SVBool_tPll11svint64x3_t(
+// CPP-CHECK-LABEL: @_Z19test_svst3_vnum_s64u10__SVBool_tPll11svint64x3_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -361,7 +361,7 @@ void test_svst3_vnum_s64(svbool_t pg, int64_t *base, int64_t vnum, svint64x3_t d
   return SVE_ACLE_FUNC(svst3_vnum,_s64,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst3_vnum_u8(
+// CHECK-LABEL: @test_svst3_vnum_u8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv48i8(<vscale x 48 x i8> [[DATA:%.*]], i64 0)
@@ -370,7 +370,7 @@ void test_svst3_vnum_s64(svbool_t pg, int64_t *base, int64_t vnum, svint64x3_t d
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st3.nxv16i8(<vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svst3_vnum_u8u10__SVBool_tPhl11svuint8x3_t(
+// CPP-CHECK-LABEL: @_Z18test_svst3_vnum_u8u10__SVBool_tPhl11svuint8x3_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv48i8(<vscale x 48 x i8> [[DATA:%.*]], i64 0)
@@ -384,7 +384,7 @@ void test_svst3_vnum_u8(svbool_t pg, uint8_t *base, int64_t vnum, svuint8x3_t da
   return SVE_ACLE_FUNC(svst3_vnum,_u8,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst3_vnum_u16(
+// CHECK-LABEL: @test_svst3_vnum_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -394,7 +394,7 @@ void test_svst3_vnum_u8(svbool_t pg, uint8_t *base, int64_t vnum, svuint8x3_t da
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st3.nxv8i16(<vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], <vscale x 8 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svst3_vnum_u16u10__SVBool_tPtl12svuint16x3_t(
+// CPP-CHECK-LABEL: @_Z19test_svst3_vnum_u16u10__SVBool_tPtl12svuint16x3_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -409,7 +409,7 @@ void test_svst3_vnum_u16(svbool_t pg, uint16_t *base, int64_t vnum, svuint16x3_t
   return SVE_ACLE_FUNC(svst3_vnum,_u16,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst3_vnum_u32(
+// CHECK-LABEL: @test_svst3_vnum_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -419,7 +419,7 @@ void test_svst3_vnum_u16(svbool_t pg, uint16_t *base, int64_t vnum, svuint16x3_t
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st3.nxv4i32(<vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], <vscale x 4 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svst3_vnum_u32u10__SVBool_tPjl12svuint32x3_t(
+// CPP-CHECK-LABEL: @_Z19test_svst3_vnum_u32u10__SVBool_tPjl12svuint32x3_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -434,7 +434,7 @@ void test_svst3_vnum_u32(svbool_t pg, uint32_t *base, int64_t vnum, svuint32x3_t
   return SVE_ACLE_FUNC(svst3_vnum,_u32,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst3_vnum_u64(
+// CHECK-LABEL: @test_svst3_vnum_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -444,7 +444,7 @@ void test_svst3_vnum_u32(svbool_t pg, uint32_t *base, int64_t vnum, svuint32x3_t
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st3.nxv2i64(<vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], <vscale x 2 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svst3_vnum_u64u10__SVBool_tPml12svuint64x3_t(
+// CPP-CHECK-LABEL: @_Z19test_svst3_vnum_u64u10__SVBool_tPml12svuint64x3_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -459,7 +459,7 @@ void test_svst3_vnum_u64(svbool_t pg, uint64_t *base, int64_t vnum, svuint64x3_t
   return SVE_ACLE_FUNC(svst3_vnum,_u64,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst3_vnum_f16(
+// CHECK-LABEL: @test_svst3_vnum_f16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -469,7 +469,7 @@ void test_svst3_vnum_u64(svbool_t pg, uint64_t *base, int64_t vnum, svuint64x3_t
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st3.nxv8f16(<vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[TMP4]], <vscale x 8 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svst3_vnum_f16u10__SVBool_tPDhl13svfloat16x3_t(
+// CPP-CHECK-LABEL: @_Z19test_svst3_vnum_f16u10__SVBool_tPDhl13svfloat16x3_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -484,7 +484,7 @@ void test_svst3_vnum_f16(svbool_t pg, float16_t *base, int64_t vnum, svfloat16x3
   return SVE_ACLE_FUNC(svst3_vnum,_f16,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst3_vnum_f32(
+// CHECK-LABEL: @test_svst3_vnum_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -494,7 +494,7 @@ void test_svst3_vnum_f16(svbool_t pg, float16_t *base, int64_t vnum, svfloat16x3
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st3.nxv4f32(<vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[TMP4]], <vscale x 4 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svst3_vnum_f32u10__SVBool_tPfl13svfloat32x3_t(
+// CPP-CHECK-LABEL: @_Z19test_svst3_vnum_f32u10__SVBool_tPfl13svfloat32x3_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -509,7 +509,7 @@ void test_svst3_vnum_f32(svbool_t pg, float32_t *base, int64_t vnum, svfloat32x3
   return SVE_ACLE_FUNC(svst3_vnum,_f32,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst3_vnum_f64(
+// CHECK-LABEL: @test_svst3_vnum_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -519,7 +519,7 @@ void test_svst3_vnum_f32(svbool_t pg, float32_t *base, int64_t vnum, svfloat32x3
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st3.nxv2f64(<vscale x 2 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], <vscale x 2 x double> [[TMP4]], <vscale x 2 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svst3_vnum_f64u10__SVBool_tPdl13svfloat64x3_t(
+// CPP-CHECK-LABEL: @_Z19test_svst3_vnum_f64u10__SVBool_tPdl13svfloat64x3_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st4-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st4-bfloat.c
index 484fc6333b..bf5afb9458 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st4-bfloat.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st4-bfloat.c
@@ -15,7 +15,7 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst4_bf16(
+// CHECK-LABEL: @test_svst4_bf16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[DATA:%.*]], i64 0)
@@ -25,7 +25,7 @@
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st4.nxv8bf16(<vscale x 8 x bfloat> [[TMP1]], <vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[TMP4]], <vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z15test_svst4_bf16u10__SVBool_tPu6__bf1614svbfloat16x4_t(
+// CPP-CHECK-LABEL: @_Z15test_svst4_bf16u10__SVBool_tPu6__bf1614svbfloat16x4_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.extract.nxv8bf16.nxv32bf16(<vscale x 32 x bfloat> [[DATA:%.*]], i64 0)
@@ -40,7 +40,7 @@ void test_svst4_bf16(svbool_t pg, bfloat16_t *base, svbfloat16x4_t data)
   return SVE_ACLE_FUNC(svst4,_bf16,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst4_vnum_bf16(
+// CHECK-LABEL: @test_svst4_vnum_bf16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x bfloat>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -51,7 +51,7 @@ void test_svst4_bf16(svbool_t pg, bfloat16_t *base, svbfloat16x4_t data)
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st4.nxv8bf16(<vscale x 8 x bfloat> [[TMP2]], <vscale x 8 x bfloat> [[TMP3]], <vscale x 8 x bfloat> [[TMP4]], <vscale x 8 x bfloat> [[TMP5]], <vscale x 8 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z20test_svst4_vnum_bf16u10__SVBool_tPu6__bf16l14svbfloat16x4_t(
+// CPP-CHECK-LABEL: @_Z20test_svst4_vnum_bf16u10__SVBool_tPu6__bf16l14svbfloat16x4_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x bfloat>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st4.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st4.c
index 774460c915..fc8bdcd34e 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st4.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_st4.c
@@ -14,7 +14,7 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst4_s8(
+// CHECK-LABEL: @test_svst4_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[DATA:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[DATA]], i64 16)
@@ -23,7 +23,7 @@
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st4.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z13test_svst4_s8u10__SVBool_tPa10svint8x4_t(
+// CPP-CHECK-LABEL: @_Z13test_svst4_s8u10__SVBool_tPa10svint8x4_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[DATA:%.*]], i64 0)
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[DATA]], i64 16)
@@ -37,7 +37,7 @@ void test_svst4_s8(svbool_t pg, int8_t *base, svint8x4_t data)
   return SVE_ACLE_FUNC(svst4,_s8,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst4_s16(
+// CHECK-LABEL: @test_svst4_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[DATA:%.*]], i64 0)
@@ -47,7 +47,7 @@ void test_svst4_s8(svbool_t pg, int8_t *base, svint8x4_t data)
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st4.nxv8i16(<vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], <vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svst4_s16u10__SVBool_tPs11svint16x4_t(
+// CPP-CHECK-LABEL: @_Z14test_svst4_s16u10__SVBool_tPs11svint16x4_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[DATA:%.*]], i64 0)
@@ -62,7 +62,7 @@ void test_svst4_s16(svbool_t pg, int16_t *base, svint16x4_t data)
   return SVE_ACLE_FUNC(svst4,_s16,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst4_s32(
+// CHECK-LABEL: @test_svst4_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[DATA:%.*]], i64 0)
@@ -72,7 +72,7 @@ void test_svst4_s16(svbool_t pg, int16_t *base, svint16x4_t data)
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st4.nxv4i32(<vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svst4_s32u10__SVBool_tPi11svint32x4_t(
+// CPP-CHECK-LABEL: @_Z14test_svst4_s32u10__SVBool_tPi11svint32x4_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[DATA:%.*]], i64 0)
@@ -87,7 +87,7 @@ void test_svst4_s32(svbool_t pg, int32_t *base, svint32x4_t data)
   return SVE_ACLE_FUNC(svst4,_s32,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst4_s64(
+// CHECK-LABEL: @test_svst4_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[DATA:%.*]], i64 0)
@@ -97,7 +97,7 @@ void test_svst4_s32(svbool_t pg, int32_t *base, svint32x4_t data)
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st4.nxv2i64(<vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svst4_s64u10__SVBool_tPl11svint64x4_t(
+// CPP-CHECK-LABEL: @_Z14test_svst4_s64u10__SVBool_tPl11svint64x4_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[DATA:%.*]], i64 0)
@@ -112,7 +112,7 @@ void test_svst4_s64(svbool_t pg, int64_t *base, svint64x4_t data)
   return SVE_ACLE_FUNC(svst4,_s64,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst4_u8(
+// CHECK-LABEL: @test_svst4_u8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[DATA:%.*]], i64 0)
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[DATA]], i64 16)
@@ -121,7 +121,7 @@ void test_svst4_s64(svbool_t pg, int64_t *base, svint64x4_t data)
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st4.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z13test_svst4_u8u10__SVBool_tPh11svuint8x4_t(
+// CPP-CHECK-LABEL: @_Z13test_svst4_u8u10__SVBool_tPh11svuint8x4_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[DATA:%.*]], i64 0)
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[DATA]], i64 16)
@@ -135,7 +135,7 @@ void test_svst4_u8(svbool_t pg, uint8_t *base, svuint8x4_t data)
   return SVE_ACLE_FUNC(svst4,_u8,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst4_u16(
+// CHECK-LABEL: @test_svst4_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[DATA:%.*]], i64 0)
@@ -145,7 +145,7 @@ void test_svst4_u8(svbool_t pg, uint8_t *base, svuint8x4_t data)
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st4.nxv8i16(<vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], <vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svst4_u16u10__SVBool_tPt12svuint16x4_t(
+// CPP-CHECK-LABEL: @_Z14test_svst4_u16u10__SVBool_tPt12svuint16x4_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[DATA:%.*]], i64 0)
@@ -160,7 +160,7 @@ void test_svst4_u16(svbool_t pg, uint16_t *base, svuint16x4_t data)
   return SVE_ACLE_FUNC(svst4,_u16,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst4_u32(
+// CHECK-LABEL: @test_svst4_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[DATA:%.*]], i64 0)
@@ -170,7 +170,7 @@ void test_svst4_u16(svbool_t pg, uint16_t *base, svuint16x4_t data)
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st4.nxv4i32(<vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svst4_u32u10__SVBool_tPj12svuint32x4_t(
+// CPP-CHECK-LABEL: @_Z14test_svst4_u32u10__SVBool_tPj12svuint32x4_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[DATA:%.*]], i64 0)
@@ -185,7 +185,7 @@ void test_svst4_u32(svbool_t pg, uint32_t *base, svuint32x4_t data)
   return SVE_ACLE_FUNC(svst4,_u32,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst4_u64(
+// CHECK-LABEL: @test_svst4_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[DATA:%.*]], i64 0)
@@ -195,7 +195,7 @@ void test_svst4_u32(svbool_t pg, uint32_t *base, svuint32x4_t data)
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st4.nxv2i64(<vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svst4_u64u10__SVBool_tPm12svuint64x4_t(
+// CPP-CHECK-LABEL: @_Z14test_svst4_u64u10__SVBool_tPm12svuint64x4_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[DATA:%.*]], i64 0)
@@ -210,7 +210,7 @@ void test_svst4_u64(svbool_t pg, uint64_t *base, svuint64x4_t data)
   return SVE_ACLE_FUNC(svst4,_u64,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst4_f16(
+// CHECK-LABEL: @test_svst4_f16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[DATA:%.*]], i64 0)
@@ -220,7 +220,7 @@ void test_svst4_u64(svbool_t pg, uint64_t *base, svuint64x4_t data)
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st4.nxv8f16(<vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[TMP4]], <vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svst4_f16u10__SVBool_tPDh13svfloat16x4_t(
+// CPP-CHECK-LABEL: @_Z14test_svst4_f16u10__SVBool_tPDh13svfloat16x4_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[DATA:%.*]], i64 0)
@@ -235,7 +235,7 @@ void test_svst4_f16(svbool_t pg, float16_t *base, svfloat16x4_t data)
   return SVE_ACLE_FUNC(svst4,_f16,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst4_f32(
+// CHECK-LABEL: @test_svst4_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[DATA:%.*]], i64 0)
@@ -245,7 +245,7 @@ void test_svst4_f16(svbool_t pg, float16_t *base, svfloat16x4_t data)
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st4.nxv4f32(<vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[TMP4]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svst4_f32u10__SVBool_tPf13svfloat32x4_t(
+// CPP-CHECK-LABEL: @_Z14test_svst4_f32u10__SVBool_tPf13svfloat32x4_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[DATA:%.*]], i64 0)
@@ -260,7 +260,7 @@ void test_svst4_f32(svbool_t pg, float32_t *base, svfloat32x4_t data)
   return SVE_ACLE_FUNC(svst4,_f32,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst4_f64(
+// CHECK-LABEL: @test_svst4_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[DATA:%.*]], i64 0)
@@ -270,7 +270,7 @@ void test_svst4_f32(svbool_t pg, float32_t *base, svfloat32x4_t data)
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st4.nxv2f64(<vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], <vscale x 2 x double> [[TMP4]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z14test_svst4_f64u10__SVBool_tPd13svfloat64x4_t(
+// CPP-CHECK-LABEL: @_Z14test_svst4_f64u10__SVBool_tPd13svfloat64x4_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[DATA:%.*]], i64 0)
@@ -285,7 +285,7 @@ void test_svst4_f64(svbool_t pg, float64_t *base, svfloat64x4_t data)
   return SVE_ACLE_FUNC(svst4,_f64,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst4_vnum_s8(
+// CHECK-LABEL: @test_svst4_vnum_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[DATA:%.*]], i64 0)
@@ -295,7 +295,7 @@ void test_svst4_f64(svbool_t pg, float64_t *base, svfloat64x4_t data)
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st4.nxv16i8(<vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], <vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svst4_vnum_s8u10__SVBool_tPal10svint8x4_t(
+// CPP-CHECK-LABEL: @_Z18test_svst4_vnum_s8u10__SVBool_tPal10svint8x4_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[DATA:%.*]], i64 0)
@@ -310,7 +310,7 @@ void test_svst4_vnum_s8(svbool_t pg, int8_t *base, int64_t vnum, svint8x4_t data
   return SVE_ACLE_FUNC(svst4_vnum,_s8,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst4_vnum_s16(
+// CHECK-LABEL: @test_svst4_vnum_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -321,7 +321,7 @@ void test_svst4_vnum_s8(svbool_t pg, int8_t *base, int64_t vnum, svint8x4_t data
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st4.nxv8i16(<vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], <vscale x 8 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svst4_vnum_s16u10__SVBool_tPsl11svint16x4_t(
+// CPP-CHECK-LABEL: @_Z19test_svst4_vnum_s16u10__SVBool_tPsl11svint16x4_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -337,7 +337,7 @@ void test_svst4_vnum_s16(svbool_t pg, int16_t *base, int64_t vnum, svint16x4_t d
   return SVE_ACLE_FUNC(svst4_vnum,_s16,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst4_vnum_s32(
+// CHECK-LABEL: @test_svst4_vnum_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -348,7 +348,7 @@ void test_svst4_vnum_s16(svbool_t pg, int16_t *base, int64_t vnum, svint16x4_t d
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st4.nxv4i32(<vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], <vscale x 4 x i32> [[TMP5]], <vscale x 4 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svst4_vnum_s32u10__SVBool_tPil11svint32x4_t(
+// CPP-CHECK-LABEL: @_Z19test_svst4_vnum_s32u10__SVBool_tPil11svint32x4_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -364,7 +364,7 @@ void test_svst4_vnum_s32(svbool_t pg, int32_t *base, int64_t vnum, svint32x4_t d
   return SVE_ACLE_FUNC(svst4_vnum,_s32,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst4_vnum_s64(
+// CHECK-LABEL: @test_svst4_vnum_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -375,7 +375,7 @@ void test_svst4_vnum_s32(svbool_t pg, int32_t *base, int64_t vnum, svint32x4_t d
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st4.nxv2i64(<vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], <vscale x 2 x i64> [[TMP5]], <vscale x 2 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svst4_vnum_s64u10__SVBool_tPll11svint64x4_t(
+// CPP-CHECK-LABEL: @_Z19test_svst4_vnum_s64u10__SVBool_tPll11svint64x4_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -391,7 +391,7 @@ void test_svst4_vnum_s64(svbool_t pg, int64_t *base, int64_t vnum, svint64x4_t d
   return SVE_ACLE_FUNC(svst4_vnum,_s64,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst4_vnum_u8(
+// CHECK-LABEL: @test_svst4_vnum_u8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[DATA:%.*]], i64 0)
@@ -401,7 +401,7 @@ void test_svst4_vnum_s64(svbool_t pg, int64_t *base, int64_t vnum, svint64x4_t d
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st4.nxv16i8(<vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], <vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z18test_svst4_vnum_u8u10__SVBool_tPhl11svuint8x4_t(
+// CPP-CHECK-LABEL: @_Z18test_svst4_vnum_u8u10__SVBool_tPhl11svuint8x4_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[DATA:%.*]], i64 0)
@@ -416,7 +416,7 @@ void test_svst4_vnum_u8(svbool_t pg, uint8_t *base, int64_t vnum, svuint8x4_t da
   return SVE_ACLE_FUNC(svst4_vnum,_u8,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst4_vnum_u16(
+// CHECK-LABEL: @test_svst4_vnum_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -427,7 +427,7 @@ void test_svst4_vnum_u8(svbool_t pg, uint8_t *base, int64_t vnum, svuint8x4_t da
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st4.nxv8i16(<vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], <vscale x 8 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svst4_vnum_u16u10__SVBool_tPtl12svuint16x4_t(
+// CPP-CHECK-LABEL: @_Z19test_svst4_vnum_u16u10__SVBool_tPtl12svuint16x4_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -443,7 +443,7 @@ void test_svst4_vnum_u16(svbool_t pg, uint16_t *base, int64_t vnum, svuint16x4_t
   return SVE_ACLE_FUNC(svst4_vnum,_u16,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst4_vnum_u32(
+// CHECK-LABEL: @test_svst4_vnum_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -454,7 +454,7 @@ void test_svst4_vnum_u16(svbool_t pg, uint16_t *base, int64_t vnum, svuint16x4_t
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st4.nxv4i32(<vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], <vscale x 4 x i32> [[TMP5]], <vscale x 4 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svst4_vnum_u32u10__SVBool_tPjl12svuint32x4_t(
+// CPP-CHECK-LABEL: @_Z19test_svst4_vnum_u32u10__SVBool_tPjl12svuint32x4_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -470,7 +470,7 @@ void test_svst4_vnum_u32(svbool_t pg, uint32_t *base, int64_t vnum, svuint32x4_t
   return SVE_ACLE_FUNC(svst4_vnum,_u32,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst4_vnum_u64(
+// CHECK-LABEL: @test_svst4_vnum_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -481,7 +481,7 @@ void test_svst4_vnum_u32(svbool_t pg, uint32_t *base, int64_t vnum, svuint32x4_t
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st4.nxv2i64(<vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], <vscale x 2 x i64> [[TMP5]], <vscale x 2 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svst4_vnum_u64u10__SVBool_tPml12svuint64x4_t(
+// CPP-CHECK-LABEL: @_Z19test_svst4_vnum_u64u10__SVBool_tPml12svuint64x4_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -497,7 +497,7 @@ void test_svst4_vnum_u64(svbool_t pg, uint64_t *base, int64_t vnum, svuint64x4_t
   return SVE_ACLE_FUNC(svst4_vnum,_u64,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst4_vnum_f16(
+// CHECK-LABEL: @test_svst4_vnum_f16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -508,7 +508,7 @@ void test_svst4_vnum_u64(svbool_t pg, uint64_t *base, int64_t vnum, svuint64x4_t
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st4.nxv8f16(<vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[TMP4]], <vscale x 8 x half> [[TMP5]], <vscale x 8 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svst4_vnum_f16u10__SVBool_tPDhl13svfloat16x4_t(
+// CPP-CHECK-LABEL: @_Z19test_svst4_vnum_f16u10__SVBool_tPDhl13svfloat16x4_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -524,7 +524,7 @@ void test_svst4_vnum_f16(svbool_t pg, float16_t *base, int64_t vnum, svfloat16x4
   return SVE_ACLE_FUNC(svst4_vnum,_f16,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst4_vnum_f32(
+// CHECK-LABEL: @test_svst4_vnum_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -535,7 +535,7 @@ void test_svst4_vnum_f16(svbool_t pg, float16_t *base, int64_t vnum, svfloat16x4
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st4.nxv4f32(<vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[TMP4]], <vscale x 4 x float> [[TMP5]], <vscale x 4 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svst4_vnum_f32u10__SVBool_tPfl13svfloat32x4_t(
+// CPP-CHECK-LABEL: @_Z19test_svst4_vnum_f32u10__SVBool_tPfl13svfloat32x4_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -551,7 +551,7 @@ void test_svst4_vnum_f32(svbool_t pg, float32_t *base, int64_t vnum, svfloat32x4
   return SVE_ACLE_FUNC(svst4_vnum,_f32,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svst4_vnum_f64(
+// CHECK-LABEL: @test_svst4_vnum_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -562,7 +562,7 @@ void test_svst4_vnum_f32(svbool_t pg, float32_t *base, int64_t vnum, svfloat32x4
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.st4.nxv2f64(<vscale x 2 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], <vscale x 2 x double> [[TMP4]], <vscale x 2 x double> [[TMP5]], <vscale x 2 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z19test_svst4_vnum_f64u10__SVBool_tPdl13svfloat64x4_t(
+// CPP-CHECK-LABEL: @_Z19test_svst4_vnum_f64u10__SVBool_tPdl13svfloat64x4_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_stnt1-bfloat.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_stnt1-bfloat.c
index f1fa19bb5e..b19c12038f 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_stnt1-bfloat.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_stnt1-bfloat.c
@@ -15,13 +15,13 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svstnt1_bf16(
+// CHECK-LABEL: @test_svstnt1_bf16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv8bf16(<vscale x 8 x bfloat> [[DATA:%.*]], <vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z17test_svstnt1_bf16u10__SVBool_tPu6__bf16u14__SVBFloat16_t(
+// CPP-CHECK-LABEL: @_Z17test_svstnt1_bf16u10__SVBool_tPu6__bf16u14__SVBFloat16_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv8bf16(<vscale x 8 x bfloat> [[DATA:%.*]], <vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -32,14 +32,14 @@ void test_svstnt1_bf16(svbool_t pg, bfloat16_t *base, svbfloat16_t data)
   return SVE_ACLE_FUNC(svstnt1,_bf16,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svstnt1_vnum_bf16(
+// CHECK-LABEL: @test_svstnt1_vnum_bf16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x bfloat>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv8bf16(<vscale x 8 x bfloat> [[DATA:%.*]], <vscale x 8 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z22test_svstnt1_vnum_bf16u10__SVBool_tPu6__bf16lu14__SVBFloat16_t(
+// CPP-CHECK-LABEL: @_Z22test_svstnt1_vnum_bf16u10__SVBool_tPu6__bf16lu14__SVBFloat16_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x bfloat>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_stnt1.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_stnt1.c
index 1392157c87..2ddb154091 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_stnt1.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_stnt1.c
@@ -15,12 +15,12 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
-// CHECK-LABEL: define {{[^@]+}}@test_svstnt1_s8(
+// CHECK-LABEL: @test_svstnt1_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv16i8(<vscale x 16 x i8> [[DATA:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z15test_svstnt1_s8u10__SVBool_tPau10__SVInt8_t(
+// CPP-CHECK-LABEL: @_Z15test_svstnt1_s8u10__SVBool_tPau10__SVInt8_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv16i8(<vscale x 16 x i8> [[DATA:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
 // CPP-CHECK-NEXT:    ret void
@@ -30,13 +30,13 @@ void test_svstnt1_s8(svbool_t pg, int8_t *base, svint8_t data)
   return SVE_ACLE_FUNC(svstnt1,_s8,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svstnt1_s16(
+// CHECK-LABEL: @test_svstnt1_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv8i16(<vscale x 8 x i16> [[DATA:%.*]], <vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svstnt1_s16u10__SVBool_tPsu11__SVInt16_t(
+// CPP-CHECK-LABEL: @_Z16test_svstnt1_s16u10__SVBool_tPsu11__SVInt16_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv8i16(<vscale x 8 x i16> [[DATA:%.*]], <vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -47,13 +47,13 @@ void test_svstnt1_s16(svbool_t pg, int16_t *base, svint16_t data)
   return SVE_ACLE_FUNC(svstnt1,_s16,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svstnt1_s32(
+// CHECK-LABEL: @test_svstnt1_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv4i32(<vscale x 4 x i32> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svstnt1_s32u10__SVBool_tPiu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z16test_svstnt1_s32u10__SVBool_tPiu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv4i32(<vscale x 4 x i32> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -64,13 +64,13 @@ void test_svstnt1_s32(svbool_t pg, int32_t *base, svint32_t data)
   return SVE_ACLE_FUNC(svstnt1,_s32,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svstnt1_s64(
+// CHECK-LABEL: @test_svstnt1_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv2i64(<vscale x 2 x i64> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svstnt1_s64u10__SVBool_tPlu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z16test_svstnt1_s64u10__SVBool_tPlu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv2i64(<vscale x 2 x i64> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -81,12 +81,12 @@ void test_svstnt1_s64(svbool_t pg, int64_t *base, svint64_t data)
   return SVE_ACLE_FUNC(svstnt1,_s64,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svstnt1_u8(
+// CHECK-LABEL: @test_svstnt1_u8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv16i8(<vscale x 16 x i8> [[DATA:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z15test_svstnt1_u8u10__SVBool_tPhu11__SVUint8_t(
+// CPP-CHECK-LABEL: @_Z15test_svstnt1_u8u10__SVBool_tPhu11__SVUint8_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv16i8(<vscale x 16 x i8> [[DATA:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
 // CPP-CHECK-NEXT:    ret void
@@ -96,13 +96,13 @@ void test_svstnt1_u8(svbool_t pg, uint8_t *base, svuint8_t data)
   return SVE_ACLE_FUNC(svstnt1,_u8,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svstnt1_u16(
+// CHECK-LABEL: @test_svstnt1_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv8i16(<vscale x 8 x i16> [[DATA:%.*]], <vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svstnt1_u16u10__SVBool_tPtu12__SVUint16_t(
+// CPP-CHECK-LABEL: @_Z16test_svstnt1_u16u10__SVBool_tPtu12__SVUint16_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv8i16(<vscale x 8 x i16> [[DATA:%.*]], <vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -113,13 +113,13 @@ void test_svstnt1_u16(svbool_t pg, uint16_t *base, svuint16_t data)
   return SVE_ACLE_FUNC(svstnt1,_u16,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svstnt1_u32(
+// CHECK-LABEL: @test_svstnt1_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv4i32(<vscale x 4 x i32> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svstnt1_u32u10__SVBool_tPju12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z16test_svstnt1_u32u10__SVBool_tPju12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv4i32(<vscale x 4 x i32> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -130,13 +130,13 @@ void test_svstnt1_u32(svbool_t pg, uint32_t *base, svuint32_t data)
   return SVE_ACLE_FUNC(svstnt1,_u32,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svstnt1_u64(
+// CHECK-LABEL: @test_svstnt1_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv2i64(<vscale x 2 x i64> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svstnt1_u64u10__SVBool_tPmu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z16test_svstnt1_u64u10__SVBool_tPmu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv2i64(<vscale x 2 x i64> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -147,13 +147,13 @@ void test_svstnt1_u64(svbool_t pg, uint64_t *base, svuint64_t data)
   return SVE_ACLE_FUNC(svstnt1,_u64,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svstnt1_f16(
+// CHECK-LABEL: @test_svstnt1_f16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv8f16(<vscale x 8 x half> [[DATA:%.*]], <vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svstnt1_f16u10__SVBool_tPDhu13__SVFloat16_t(
+// CPP-CHECK-LABEL: @_Z16test_svstnt1_f16u10__SVBool_tPDhu13__SVFloat16_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv8f16(<vscale x 8 x half> [[DATA:%.*]], <vscale x 8 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -164,13 +164,13 @@ void test_svstnt1_f16(svbool_t pg, float16_t *base, svfloat16_t data)
   return SVE_ACLE_FUNC(svstnt1,_f16,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svstnt1_f32(
+// CHECK-LABEL: @test_svstnt1_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv4f32(<vscale x 4 x float> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svstnt1_f32u10__SVBool_tPfu13__SVFloat32_t(
+// CPP-CHECK-LABEL: @_Z16test_svstnt1_f32u10__SVBool_tPfu13__SVFloat32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv4f32(<vscale x 4 x float> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -181,13 +181,13 @@ void test_svstnt1_f32(svbool_t pg, float32_t *base, svfloat32_t data)
   return SVE_ACLE_FUNC(svstnt1,_f32,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svstnt1_f64(
+// CHECK-LABEL: @test_svstnt1_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv2f64(<vscale x 2 x double> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z16test_svstnt1_f64u10__SVBool_tPdu13__SVFloat64_t(
+// CPP-CHECK-LABEL: @_Z16test_svstnt1_f64u10__SVBool_tPdu13__SVFloat64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv2f64(<vscale x 2 x double> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], ptr [[BASE:%.*]])
@@ -198,13 +198,13 @@ void test_svstnt1_f64(svbool_t pg, float64_t *base, svfloat64_t data)
   return SVE_ACLE_FUNC(svstnt1,_f64,,)(pg, base, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svstnt1_vnum_s8(
+// CHECK-LABEL: @test_svstnt1_vnum_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv16i8(<vscale x 16 x i8> [[DATA:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z20test_svstnt1_vnum_s8u10__SVBool_tPalu10__SVInt8_t(
+// CPP-CHECK-LABEL: @_Z20test_svstnt1_vnum_s8u10__SVBool_tPalu10__SVInt8_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv16i8(<vscale x 16 x i8> [[DATA:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
@@ -215,14 +215,14 @@ void test_svstnt1_vnum_s8(svbool_t pg, int8_t *base, int64_t vnum, svint8_t data
   return SVE_ACLE_FUNC(svstnt1_vnum,_s8,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svstnt1_vnum_s16(
+// CHECK-LABEL: @test_svstnt1_vnum_s16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv8i16(<vscale x 8 x i16> [[DATA:%.*]], <vscale x 8 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svstnt1_vnum_s16u10__SVBool_tPslu11__SVInt16_t(
+// CPP-CHECK-LABEL: @_Z21test_svstnt1_vnum_s16u10__SVBool_tPslu11__SVInt16_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -234,14 +234,14 @@ void test_svstnt1_vnum_s16(svbool_t pg, int16_t *base, int64_t vnum, svint16_t d
   return SVE_ACLE_FUNC(svstnt1_vnum,_s16,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svstnt1_vnum_s32(
+// CHECK-LABEL: @test_svstnt1_vnum_s32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv4i32(<vscale x 4 x i32> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svstnt1_vnum_s32u10__SVBool_tPilu11__SVInt32_t(
+// CPP-CHECK-LABEL: @_Z21test_svstnt1_vnum_s32u10__SVBool_tPilu11__SVInt32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -253,14 +253,14 @@ void test_svstnt1_vnum_s32(svbool_t pg, int32_t *base, int64_t vnum, svint32_t d
   return SVE_ACLE_FUNC(svstnt1_vnum,_s32,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svstnt1_vnum_s64(
+// CHECK-LABEL: @test_svstnt1_vnum_s64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv2i64(<vscale x 2 x i64> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svstnt1_vnum_s64u10__SVBool_tPllu11__SVInt64_t(
+// CPP-CHECK-LABEL: @_Z21test_svstnt1_vnum_s64u10__SVBool_tPllu11__SVInt64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -272,13 +272,13 @@ void test_svstnt1_vnum_s64(svbool_t pg, int64_t *base, int64_t vnum, svint64_t d
   return SVE_ACLE_FUNC(svstnt1_vnum,_s64,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svstnt1_vnum_u8(
+// CHECK-LABEL: @test_svstnt1_vnum_u8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv16i8(<vscale x 16 x i8> [[DATA:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z20test_svstnt1_vnum_u8u10__SVBool_tPhlu11__SVUint8_t(
+// CPP-CHECK-LABEL: @_Z20test_svstnt1_vnum_u8u10__SVBool_tPhlu11__SVUint8_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv16i8(<vscale x 16 x i8> [[DATA:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
@@ -289,14 +289,14 @@ void test_svstnt1_vnum_u8(svbool_t pg, uint8_t *base, int64_t vnum, svuint8_t da
   return SVE_ACLE_FUNC(svstnt1_vnum,_u8,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svstnt1_vnum_u16(
+// CHECK-LABEL: @test_svstnt1_vnum_u16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv8i16(<vscale x 8 x i16> [[DATA:%.*]], <vscale x 8 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svstnt1_vnum_u16u10__SVBool_tPtlu12__SVUint16_t(
+// CPP-CHECK-LABEL: @_Z21test_svstnt1_vnum_u16u10__SVBool_tPtlu12__SVUint16_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -308,14 +308,14 @@ void test_svstnt1_vnum_u16(svbool_t pg, uint16_t *base, int64_t vnum, svuint16_t
   return SVE_ACLE_FUNC(svstnt1_vnum,_u16,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svstnt1_vnum_u32(
+// CHECK-LABEL: @test_svstnt1_vnum_u32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv4i32(<vscale x 4 x i32> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svstnt1_vnum_u32u10__SVBool_tPjlu12__SVUint32_t(
+// CPP-CHECK-LABEL: @_Z21test_svstnt1_vnum_u32u10__SVBool_tPjlu12__SVUint32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -327,14 +327,14 @@ void test_svstnt1_vnum_u32(svbool_t pg, uint32_t *base, int64_t vnum, svuint32_t
   return SVE_ACLE_FUNC(svstnt1_vnum,_u32,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svstnt1_vnum_u64(
+// CHECK-LABEL: @test_svstnt1_vnum_u64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv2i64(<vscale x 2 x i64> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svstnt1_vnum_u64u10__SVBool_tPmlu12__SVUint64_t(
+// CPP-CHECK-LABEL: @_Z21test_svstnt1_vnum_u64u10__SVBool_tPmlu12__SVUint64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -346,14 +346,14 @@ void test_svstnt1_vnum_u64(svbool_t pg, uint64_t *base, int64_t vnum, svuint64_t
   return SVE_ACLE_FUNC(svstnt1_vnum,_u64,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svstnt1_vnum_f16(
+// CHECK-LABEL: @test_svstnt1_vnum_f16(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv8f16(<vscale x 8 x half> [[DATA:%.*]], <vscale x 8 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svstnt1_vnum_f16u10__SVBool_tPDhlu13__SVFloat16_t(
+// CPP-CHECK-LABEL: @_Z21test_svstnt1_vnum_f16u10__SVBool_tPDhlu13__SVFloat16_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -365,14 +365,14 @@ void test_svstnt1_vnum_f16(svbool_t pg, float16_t *base, int64_t vnum, svfloat16
   return SVE_ACLE_FUNC(svstnt1_vnum,_f16,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svstnt1_vnum_f32(
+// CHECK-LABEL: @test_svstnt1_vnum_f32(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv4f32(<vscale x 4 x float> [[DATA:%.*]], <vscale x 4 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svstnt1_vnum_f32u10__SVBool_tPflu13__SVFloat32_t(
+// CPP-CHECK-LABEL: @_Z21test_svstnt1_vnum_f32u10__SVBool_tPflu13__SVFloat32_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
@@ -384,14 +384,14 @@ void test_svstnt1_vnum_f32(svbool_t pg, float32_t *base, int64_t vnum, svfloat32
   return SVE_ACLE_FUNC(svstnt1_vnum,_f32,,)(pg, base, vnum, data);
 }
 
-// CHECK-LABEL: define {{[^@]+}}@test_svstnt1_vnum_f64(
+// CHECK-LABEL: @test_svstnt1_vnum_f64(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
 // CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.nxv2f64(<vscale x 2 x double> [[DATA:%.*]], <vscale x 2 x i1> [[TMP0]], ptr [[TMP1]])
 // CHECK-NEXT:    ret void
 //
-// CPP-CHECK-LABEL: define {{[^@]+}}@_Z21test_svstnt1_vnum_f64u10__SVBool_tPdlu13__SVFloat64_t(
+// CPP-CHECK-LABEL: @_Z21test_svstnt1_vnum_f64u10__SVBool_tPdlu13__SVFloat64_t(
 // CPP-CHECK-NEXT:  entry:
 // CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
 // CPP-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
diff --git a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_wrffr.c b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_wrffr.c
index 90f96f4015..61182ca526 100644
--- a/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_wrffr.c
+++ b/clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_wrffr.c
@@ -7,12 +7,12 @@
 
 // CHECK-LABEL: @test_svwrffr(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    call void @llvm.aarch64.sve.wrffr(<vscale x 16 x i1> [[OP:%.*]])
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.wrffr(<vscale x 16 x i1> [[OP:%.*]])
 // CHECK-NEXT:    ret void
 //
 // CPP-CHECK-LABEL: @_Z12test_svwrffru10__SVBool_t(
 // CPP-CHECK-NEXT:  entry:
-// CPP-CHECK-NEXT:    call void @llvm.aarch64.sve.wrffr(<vscale x 16 x i1> [[OP:%.*]])
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.wrffr(<vscale x 16 x i1> [[OP:%.*]])
 // CPP-CHECK-NEXT:    ret void
 //
 void test_svwrffr(svbool_t op)
diff --git a/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_revd.c b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_revd.c
new file mode 100644
index 0000000000..14d515e6d1
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve2-intrinsics/acle_sve2_revd.c
@@ -0,0 +1,390 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu \
+// RUN:   -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu \
+// RUN:   -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu \
+// RUN:   -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu \
+// RUN:   -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1, A2_UNUSED, A3, A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4
+#endif
+
+// CHECK-LABEL: @test_svrevd_s8_z(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.revd.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP:%.*]])
+// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svrevd_s8_zu10__SVBool_tu10__SVInt8_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.revd.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+svint8_t test_svrevd_s8_z(svbool_t pg, svint8_t op) {
+  return SVE_ACLE_FUNC(svrevd, _s8, _z, )(pg, op);
+}
+
+// CHECK-LABEL: @test_svrevd_s16_z(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.revd.nxv8i16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP:%.*]])
+// CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svrevd_s16_zu10__SVBool_tu11__SVInt16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.revd.nxv8i16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+svint16_t test_svrevd_s16_z(svbool_t pg, svint16_t op) {
+  return SVE_ACLE_FUNC(svrevd, _s16, _z, )(pg, op);
+}
+
+// CHECK-LABEL: @test_svrevd_s32_z(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.revd.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP:%.*]])
+// CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svrevd_s32_zu10__SVBool_tu11__SVInt32_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.revd.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+svint32_t test_svrevd_s32_z(svbool_t pg, svint32_t op) {
+  return SVE_ACLE_FUNC(svrevd, _s32, _z, )(pg, op);
+}
+
+// CHECK-LABEL: @test_svrevd_s64_z(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revd.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP:%.*]])
+// CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svrevd_s64_zu10__SVBool_tu11__SVInt64_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revd.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+svint64_t test_svrevd_s64_z(svbool_t pg, svint64_t op) {
+  return SVE_ACLE_FUNC(svrevd, _s64, _z, )(pg, op);
+}
+
+// CHECK-LABEL: @test_svrevd_u8_z(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.revd.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP:%.*]])
+// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svrevd_u8_zu10__SVBool_tu11__SVUint8_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.revd.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+svuint8_t test_svrevd_u8_z(svbool_t pg, svuint8_t op) {
+  return SVE_ACLE_FUNC(svrevd, _u8, _z, )(pg, op);
+}
+// CHECK-LABEL: @test_svrevd_u16_z(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.revd.nxv8i16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP:%.*]])
+// CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svrevd_u16_zu10__SVBool_tu12__SVUint16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.revd.nxv8i16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+svuint16_t test_svrevd_u16_z(svbool_t pg, svuint16_t op) {
+  return SVE_ACLE_FUNC(svrevd, _u16, _z, )(pg, op);
+}
+
+// CHECK-LABEL: @test_svrevd_u32_z(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.revd.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP:%.*]])
+// CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svrevd_u32_zu10__SVBool_tu12__SVUint32_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.revd.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+svuint32_t test_svrevd_u32_z(svbool_t pg, svuint32_t op) {
+  return SVE_ACLE_FUNC(svrevd, _u32, _z, )(pg, op);
+}
+
+// CHECK-LABEL: @test_svrevd_u64_z(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revd.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP:%.*]])
+// CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svrevd_u64_zu10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revd.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+svuint64_t test_svrevd_u64_z(svbool_t pg, svuint64_t op) {
+  return SVE_ACLE_FUNC(svrevd, _u64, _z, )(pg, op);
+}
+
+// CHECK-LABEL: @test_svrevd_s8_m(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.revd.nxv16i8(<vscale x 16 x i8> [[INACTIVE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP:%.*]])
+// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svrevd_s8_mu10__SVInt8_tu10__SVBool_tu10__SVInt8_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.revd.nxv16i8(<vscale x 16 x i8> [[INACTIVE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+svint8_t test_svrevd_s8_m(svint8_t inactive, svbool_t pg, svint8_t op) {
+  return SVE_ACLE_FUNC(svrevd, _s8, _m, )(inactive, pg, op);
+}
+
+// CHECK-LABEL: @test_svrevd_s16_m(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.revd.nxv8i16(<vscale x 8 x i16> [[INACTIVE:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP:%.*]])
+// CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svrevd_s16_mu11__SVInt16_tu10__SVBool_tu11__SVInt16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.revd.nxv8i16(<vscale x 8 x i16> [[INACTIVE:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+svint16_t test_svrevd_s16_m(svint16_t inactive, svbool_t pg, svint16_t op) {
+  return SVE_ACLE_FUNC(svrevd, _s16, _m, )(inactive, pg, op);
+}
+
+// CHECK-LABEL: @test_svrevd_s32_m(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.revd.nxv4i32(<vscale x 4 x i32> [[INACTIVE:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP:%.*]])
+// CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svrevd_s32_mu11__SVInt32_tu10__SVBool_tu11__SVInt32_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.revd.nxv4i32(<vscale x 4 x i32> [[INACTIVE:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+svint32_t test_svrevd_s32_m(svint32_t inactive, svbool_t pg, svint32_t op) {
+  return SVE_ACLE_FUNC(svrevd, _s32, _m, )(inactive, pg, op);
+}
+
+// CHECK-LABEL: @test_svrevd_s64_m(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revd.nxv2i64(<vscale x 2 x i64> [[INACTIVE:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP:%.*]])
+// CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svrevd_s64_mu11__SVInt64_tu10__SVBool_tu11__SVInt64_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revd.nxv2i64(<vscale x 2 x i64> [[INACTIVE:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+svint64_t test_svrevd_s64_m(svint64_t inactive, svbool_t pg, svint64_t op) {
+  return SVE_ACLE_FUNC(svrevd, _s64, _m, )(inactive, pg, op);
+}
+
+// CHECK-LABEL: @test_svrevd_u8_m(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.revd.nxv16i8(<vscale x 16 x i8> [[INACTIVE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP:%.*]])
+// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svrevd_u8_mu11__SVUint8_tu10__SVBool_tu11__SVUint8_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.revd.nxv16i8(<vscale x 16 x i8> [[INACTIVE:%.*]], <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+svuint8_t test_svrevd_u8_m(svuint8_t inactive, svbool_t pg, svuint8_t op) {
+  return SVE_ACLE_FUNC(svrevd, _u8, _m, )(inactive, pg, op);
+}
+
+// CHECK-LABEL: @test_svrevd_u16_m(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.revd.nxv8i16(<vscale x 8 x i16> [[INACTIVE:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP:%.*]])
+// CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svrevd_u16_mu12__SVUint16_tu10__SVBool_tu12__SVUint16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.revd.nxv8i16(<vscale x 8 x i16> [[INACTIVE:%.*]], <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+svuint16_t test_svrevd_u16_m(svuint16_t inactive, svbool_t pg, svuint16_t op) {
+  return SVE_ACLE_FUNC(svrevd, _u16, _m, )(inactive, pg, op);
+}
+
+// CHECK-LABEL: @test_svrevd_u32_m(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.revd.nxv4i32(<vscale x 4 x i32> [[INACTIVE:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP:%.*]])
+// CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svrevd_u32_mu12__SVUint32_tu10__SVBool_tu12__SVUint32_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.revd.nxv4i32(<vscale x 4 x i32> [[INACTIVE:%.*]], <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+svuint32_t test_svrevd_u32_m(svuint32_t inactive, svbool_t pg, svuint32_t op) {
+  return SVE_ACLE_FUNC(svrevd, _u32, _m, )(inactive, pg, op);
+}
+
+// CHECK-LABEL: @test_svrevd_u64_m(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revd.nxv2i64(<vscale x 2 x i64> [[INACTIVE:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP:%.*]])
+// CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svrevd_u64_mu12__SVUint64_tu10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revd.nxv2i64(<vscale x 2 x i64> [[INACTIVE:%.*]], <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+svuint64_t test_svrevd_u64_m(svuint64_t inactive, svbool_t pg, svuint64_t op) {
+  return SVE_ACLE_FUNC(svrevd, _u64, _m, )(inactive, pg, op);
+}
+
+// CHECK-LABEL: @test_svrevd_s8_x(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.revd.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP:%.*]])
+// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svrevd_s8_xu10__SVBool_tu10__SVInt8_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.revd.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+svint8_t test_svrevd_s8_x(svbool_t pg, svint8_t op) {
+  return SVE_ACLE_FUNC(svrevd, _s8, _x, )(pg, op);
+}
+
+// CHECK-LABEL: @test_svrevd_s16_x(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.revd.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP:%.*]])
+// CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svrevd_s16_xu10__SVBool_tu11__SVInt16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.revd.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+svint16_t test_svrevd_s16_x(svbool_t pg, svint16_t op) {
+  return SVE_ACLE_FUNC(svrevd, _s16, _x, )(pg, op);
+}
+
+// CHECK-LABEL: @test_svrevd_s32_x(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.revd.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP:%.*]])
+// CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svrevd_s32_xu10__SVBool_tu11__SVInt32_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.revd.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+svint32_t test_svrevd_s32_x(svbool_t pg, svint32_t op) {
+  return SVE_ACLE_FUNC(svrevd, _s32, _x, )(pg, op);
+}
+
+// CHECK-LABEL: @test_svrevd_s64_x(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revd.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP:%.*]])
+// CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svrevd_s64_xu10__SVBool_tu11__SVInt64_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revd.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+svint64_t test_svrevd_s64_x(svbool_t pg, svint64_t op) {
+  return SVE_ACLE_FUNC(svrevd, _s64, _x, )(pg, op);
+}
+
+// CHECK-LABEL: @test_svrevd_u8_x(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.revd.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP:%.*]])
+// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svrevd_u8_xu10__SVBool_tu11__SVUint8_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.revd.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+svuint8_t test_svrevd_u8_x(svbool_t pg, svuint8_t op) {
+  return SVE_ACLE_FUNC(svrevd, _u8, _x, )(pg, op);
+}
+
+// CHECK-LABEL: @test_svrevd_u16_x(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.revd.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP:%.*]])
+// CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svrevd_u16_xu10__SVBool_tu12__SVUint16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.revd.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP1]]
+//
+svuint16_t test_svrevd_u16_x(svbool_t pg, svuint16_t op) {
+  return SVE_ACLE_FUNC(svrevd, _u16, _x, )(pg, op);
+}
+
+// CHECK-LABEL: @test_svrevd_u32_x(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.revd.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP:%.*]])
+// CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svrevd_u32_xu10__SVBool_tu12__SVUint32_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.revd.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+//
+svuint32_t test_svrevd_u32_x(svbool_t pg, svuint32_t op) {
+  return SVE_ACLE_FUNC(svrevd, _u32, _x, )(pg, op);
+}
+
+// CHECK-LABEL: @test_svrevd_u64_x(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revd.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP:%.*]])
+// CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svrevd_u64_xu10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.aarch64.sve.revd.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 2 x i64> [[TMP1]]
+//
+svuint64_t test_svrevd_u64_x(svbool_t pg, svuint64_t op) {
+  return SVE_ACLE_FUNC(svrevd, _u64, _x, )(pg, op);
+}
diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmlsl.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmlsl.c
new file mode 100644
index 0000000000..2955d4554d
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_bfmlsl.c
@@ -0,0 +1,85 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: aarch64-registered-target
+
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -target-feature -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+// BFMLSLB
+
+
+// CHECK-LABEL: @test_bfmlslb(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.bfmlslb(<vscale x 4 x float> [[ZDA:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]])
+// CHECK-NEXT:    ret <vscale x 4 x float> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z12test_bfmlslbu13__SVFloat32_tu14__SVBFloat16_tu14__SVBFloat16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.bfmlslb(<vscale x 4 x float> [[ZDA:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 4 x float> [[TMP0]]
+//
+svfloat32_t test_bfmlslb(svfloat32_t zda, svbfloat16_t zn, svbfloat16_t zm)
+{
+  return SVE_ACLE_FUNC(svbfmlslb,_f32,,)(zda, zn, zm);
+}
+
+
+// CHECK-LABEL: @test_bfmlslb_lane(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.bfmlslb.lane(<vscale x 4 x float> [[ZDA:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]], i32 7)
+// CHECK-NEXT:    ret <vscale x 4 x float> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z17test_bfmlslb_laneu13__SVFloat32_tu14__SVBFloat16_tu14__SVBFloat16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.bfmlslb.lane(<vscale x 4 x float> [[ZDA:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]], i32 7)
+// CPP-CHECK-NEXT:    ret <vscale x 4 x float> [[TMP0]]
+//
+svfloat32_t test_bfmlslb_lane(svfloat32_t zda, svbfloat16_t zn, svbfloat16_t zm)
+{
+  return SVE_ACLE_FUNC(svbfmlslb_lane,_f32,,)(zda, zn, zm, 7);
+}
+
+// BFMLSLT
+
+
+// CHECK-LABEL: @test_bfmlslt(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.bfmlslt(<vscale x 4 x float> [[ZDA:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]])
+// CHECK-NEXT:    ret <vscale x 4 x float> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z12test_bfmlsltu13__SVFloat32_tu14__SVBFloat16_tu14__SVBFloat16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.bfmlslt(<vscale x 4 x float> [[ZDA:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 4 x float> [[TMP0]]
+//
+svfloat32_t test_bfmlslt(svfloat32_t zda, svbfloat16_t zn, svbfloat16_t zm)
+{
+  return SVE_ACLE_FUNC(svbfmlslt,_f32,,)(zda, zn, zm);
+}
+
+
+// CHECK-LABEL: @test_bfmlslt_lane(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.bfmlslt.lane(<vscale x 4 x float> [[ZDA:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]], i32 7)
+// CHECK-NEXT:    ret <vscale x 4 x float> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z17test_bfmlslt_laneu13__SVFloat32_tu14__SVBFloat16_tu14__SVBFloat16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.bfmlslt.lane(<vscale x 4 x float> [[ZDA:%.*]], <vscale x 8 x bfloat> [[ZN:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]], i32 7)
+// CPP-CHECK-NEXT:    ret <vscale x 4 x float> [[TMP0]]
+//
+svfloat32_t test_bfmlslt_lane(svfloat32_t zda, svbfloat16_t zn, svbfloat16_t zm)
+{
+  return SVE_ACLE_FUNC(svbfmlslt_lane,_f32,,)(zda, zn, zm, 7);
+}
diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_cntp.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_cntp.c
new file mode 100644
index 0000000000..18973a6467
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_cntp.c
@@ -0,0 +1,119 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
+
+#include <arm_sve.h>
+
+// CHECK-LABEL: @test_svcntp_c8_vlx2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c8(target("aarch64.svcount") [[PNN:%.*]], i32 2)
+// CHECK-NEXT:    ret i64 [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z19test_svcntp_c8_vlx2u11__SVCount_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c8(target("aarch64.svcount") [[PNN:%.*]], i32 2)
+// CPP-CHECK-NEXT:    ret i64 [[TMP0]]
+//
+uint64_t test_svcntp_c8_vlx2(svcount_t pnn) {
+  return svcntp_c8(pnn, 2);
+}
+
+// CHECK-LABEL: @test_svcntp_c8_vlx4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c8(target("aarch64.svcount") [[PNN:%.*]], i32 4)
+// CHECK-NEXT:    ret i64 [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z19test_svcntp_c8_vlx4u11__SVCount_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c8(target("aarch64.svcount") [[PNN:%.*]], i32 4)
+// CPP-CHECK-NEXT:    ret i64 [[TMP0]]
+//
+uint64_t test_svcntp_c8_vlx4(svcount_t pnn) {
+  return svcntp_c8(pnn, 4);
+}
+
+// CHECK-LABEL: @test_svcntp_c16_vlx2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c16(target("aarch64.svcount") [[PNN:%.*]], i32 2)
+// CHECK-NEXT:    ret i64 [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z20test_svcntp_c16_vlx2u11__SVCount_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c16(target("aarch64.svcount") [[PNN:%.*]], i32 2)
+// CPP-CHECK-NEXT:    ret i64 [[TMP0]]
+//
+uint64_t test_svcntp_c16_vlx2(svcount_t pnn) {
+  return svcntp_c16(pnn, 2);
+}
+
+// CHECK-LABEL: @test_svcntp_c16_vlx4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c16(target("aarch64.svcount") [[PNN:%.*]], i32 4)
+// CHECK-NEXT:    ret i64 [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z20test_svcntp_c16_vlx4u11__SVCount_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c16(target("aarch64.svcount") [[PNN:%.*]], i32 4)
+// CPP-CHECK-NEXT:    ret i64 [[TMP0]]
+//
+uint64_t test_svcntp_c16_vlx4(svcount_t pnn) {
+  return svcntp_c16(pnn, 4);
+}
+
+// CHECK-LABEL: @test_svcntp_c32_vlx2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c32(target("aarch64.svcount") [[PNN:%.*]], i32 2)
+// CHECK-NEXT:    ret i64 [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z20test_svcntp_c32_vlx2u11__SVCount_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c32(target("aarch64.svcount") [[PNN:%.*]], i32 2)
+// CPP-CHECK-NEXT:    ret i64 [[TMP0]]
+//
+uint64_t test_svcntp_c32_vlx2(svcount_t pnn) {
+  return svcntp_c32(pnn, 2);
+}
+
+// CHECK-LABEL: @test_svcntp_c32_vlx4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c32(target("aarch64.svcount") [[PNN:%.*]], i32 4)
+// CHECK-NEXT:    ret i64 [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z20test_svcntp_c32_vlx4u11__SVCount_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c32(target("aarch64.svcount") [[PNN:%.*]], i32 4)
+// CPP-CHECK-NEXT:    ret i64 [[TMP0]]
+//
+uint64_t test_svcntp_c32_vlx4(svcount_t pnn) {
+  return svcntp_c32(pnn, 4);
+}
+
+// CHECK-LABEL: @test_svcntp_c64_vlx2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c64(target("aarch64.svcount") [[PNN:%.*]], i32 2)
+// CHECK-NEXT:    ret i64 [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z20test_svcntp_c64_vlx2u11__SVCount_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c64(target("aarch64.svcount") [[PNN:%.*]], i32 2)
+// CPP-CHECK-NEXT:    ret i64 [[TMP0]]
+//
+uint64_t test_svcntp_c64_vlx2(svcount_t pnn) {
+  return svcntp_c64(pnn, 2);
+}
+
+// CHECK-LABEL: @test_svcntp_c64_vlx4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c64(target("aarch64.svcount") [[PNN:%.*]], i32 4)
+// CHECK-NEXT:    ret i64 [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z20test_svcntp_c64_vlx4u11__SVCount_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sve.cntp.c64(target("aarch64.svcount") [[PNN:%.*]], i32 4)
+// CPP-CHECK-NEXT:    ret i64 [[TMP0]]
+//
+uint64_t test_svcntp_c64_vlx4(svcount_t pnn) {
+  return svcntp_c64(pnn, 4);
+}
diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_dot.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_dot.c
new file mode 100644
index 0000000000..d50be9ae17
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_dot.c
@@ -0,0 +1,107 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3) A1##A2##A3
+#endif
+
+// CHECK-LABEL: @test_svdot_s32_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.x2.nxv4i32(<vscale x 4 x i32> [[OP1:%.*]], <vscale x 8 x i16> [[OP2:%.*]], <vscale x 8 x i16> [[OP3:%.*]])
+// CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svdot_s32_x2u11__SVInt32_tu11__SVInt16_tu11__SVInt16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.x2.nxv4i32(<vscale x 4 x i32> [[OP1:%.*]], <vscale x 8 x i16> [[OP2:%.*]], <vscale x 8 x i16> [[OP3:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+svint32_t test_svdot_s32_x2(svint32_t op1, svint16_t op2, svint16_t op3)
+{
+  return SVE_ACLE_FUNC(svdot,_s32_s16_s16,)(op1, op2, op3);
+}
+
+// CHECK-LABEL: @test_svdot_u32_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.x2.nxv4i32(<vscale x 4 x i32> [[OP1:%.*]], <vscale x 8 x i16> [[OP2:%.*]], <vscale x 8 x i16> [[OP3:%.*]])
+// CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svdot_u32_x2u12__SVUint32_tu12__SVUint16_tu12__SVUint16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.x2.nxv4i32(<vscale x 4 x i32> [[OP1:%.*]], <vscale x 8 x i16> [[OP2:%.*]], <vscale x 8 x i16> [[OP3:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+svuint32_t test_svdot_u32_x2(svuint32_t op1, svuint16_t op2, svuint16_t op3)
+{
+  return SVE_ACLE_FUNC(svdot,_u32_u16_u16,)(op1, op2, op3);
+}
+
+// CHECK-LABEL: @test_svdot_f32_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.fdot.x2.nxv4f32(<vscale x 4 x float> [[OP1:%.*]], <vscale x 8 x half> [[OP2:%.*]], <vscale x 8 x half> [[OP3:%.*]])
+// CHECK-NEXT:    ret <vscale x 4 x float> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svdot_f32_x2u13__SVFloat32_tu13__SVFloat16_tu13__SVFloat16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.fdot.x2.nxv4f32(<vscale x 4 x float> [[OP1:%.*]], <vscale x 8 x half> [[OP2:%.*]], <vscale x 8 x half> [[OP3:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 4 x float> [[TMP0]]
+//
+svfloat32_t test_svdot_f32_x2(svfloat32_t op1, svfloat16_t op2, svfloat16_t op3)
+{
+  return SVE_ACLE_FUNC(svdot,_f32_f16_f16,)(op1, op2, op3);
+}
+
+
+
+// CHECK-LABEL: @test_svdot_lane_s32_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.lane.x2.nxv4i32(<vscale x 4 x i32> [[OP1:%.*]], <vscale x 8 x i16> [[OP2:%.*]], <vscale x 8 x i16> [[OP3:%.*]], i32 3)
+// CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svdot_lane_s32_x2u11__SVInt32_tu11__SVInt16_tu11__SVInt16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.lane.x2.nxv4i32(<vscale x 4 x i32> [[OP1:%.*]], <vscale x 8 x i16> [[OP2:%.*]], <vscale x 8 x i16> [[OP3:%.*]], i32 3)
+// CPP-CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+svint32_t test_svdot_lane_s32_x2(svint32_t op1, svint16_t op2, svint16_t op3)
+{
+  return SVE_ACLE_FUNC(svdot_lane,_s32_s16_s16,)(op1, op2, op3, 3);
+}
+
+// CHECK-LABEL: @test_svdot_lane_u32_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.lane.x2.nxv4i32(<vscale x 4 x i32> [[OP1:%.*]], <vscale x 8 x i16> [[OP2:%.*]], <vscale x 8 x i16> [[OP3:%.*]], i32 3)
+// CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svdot_lane_u32_x2u12__SVUint32_tu12__SVUint16_tu12__SVUint16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.lane.x2.nxv4i32(<vscale x 4 x i32> [[OP1:%.*]], <vscale x 8 x i16> [[OP2:%.*]], <vscale x 8 x i16> [[OP3:%.*]], i32 3)
+// CPP-CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP0]]
+//
+svuint32_t test_svdot_lane_u32_x2(svuint32_t op1, svuint16_t op2, svuint16_t op3)
+{
+  return SVE_ACLE_FUNC(svdot_lane,_u32_u16_u16,)(op1, op2, op3, 3);
+}
+
+// CHECK-LABEL: @test_svdot_lane_f32_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.fdot.lane.x2.nxv4f32(<vscale x 4 x float> [[OP1:%.*]], <vscale x 8 x half> [[OP2:%.*]], <vscale x 8 x half> [[OP3:%.*]], i32 3)
+// CHECK-NEXT:    ret <vscale x 4 x float> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svdot_lane_f32_x2u13__SVFloat32_tu13__SVFloat16_tu13__SVFloat16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.aarch64.sve.fdot.lane.x2.nxv4f32(<vscale x 4 x float> [[OP1:%.*]], <vscale x 8 x half> [[OP2:%.*]], <vscale x 8 x half> [[OP3:%.*]], i32 3)
+// CPP-CHECK-NEXT:    ret <vscale x 4 x float> [[TMP0]]
+//
+svfloat32_t test_svdot_lane_f32_x2(svfloat32_t op1, svfloat16_t op2, svfloat16_t op3)
+{
+  return SVE_ACLE_FUNC(svdot_lane,_f32_f16_f16,)(op1, op2, op3, 3);
+}
diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_ld1.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_ld1.c
new file mode 100644
index 0000000000..7a25d31de0
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_ld1.c
@@ -0,0 +1,1250 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+// CHECK-LABEL: @test_svld1_u8_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> poison, <vscale x 16 x i8> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], i64 16)
+// CHECK-NEXT:    ret <vscale x 32 x i8> [[TMP4]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svld1_u8_x2u11__SVCount_tPKh(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> poison, <vscale x 16 x i8> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], i64 16)
+// CPP-CHECK-NEXT:    ret <vscale x 32 x i8> [[TMP4]]
+//
+svuint8x2_t test_svld1_u8_x2(svcount_t pn, const uint8_t *base)
+{
+  return SVE_ACLE_FUNC(svld1,_u8,_x2,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svld1_u16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x2.nxv8i16(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> poison, <vscale x 8 x i16> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], i64 8)
+// CHECK-NEXT:    ret <vscale x 16 x i16> [[TMP4]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svld1_u16_x2u11__SVCount_tPKt(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x2.nxv8i16(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> poison, <vscale x 8 x i16> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], i64 8)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i16> [[TMP4]]
+//
+svuint16x2_t test_svld1_u16_x2(svcount_t pn, const uint16_t *base)
+{
+  return SVE_ACLE_FUNC(svld1,_u16,_x2,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svld1_u32_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld1.pn.x2.nxv4i32(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> poison, <vscale x 4 x i32> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], i64 4)
+// CHECK-NEXT:    ret <vscale x 8 x i32> [[TMP4]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svld1_u32_x2u11__SVCount_tPKj(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld1.pn.x2.nxv4i32(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> poison, <vscale x 4 x i32> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], i64 4)
+// CPP-CHECK-NEXT:    ret <vscale x 8 x i32> [[TMP4]]
+//
+svuint32x2_t test_svld1_u32_x2(svcount_t pn, const uint32_t *base)
+{
+  return SVE_ACLE_FUNC(svld1,_u32,_x2,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svld1_u64_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld1.pn.x2.nxv2i64(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> poison, <vscale x 2 x i64> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], i64 2)
+// CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP4]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svld1_u64_x2u11__SVCount_tPKm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld1.pn.x2.nxv2i64(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> poison, <vscale x 2 x i64> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], i64 2)
+// CPP-CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP4]]
+//
+svuint64x2_t test_svld1_u64_x2(svcount_t pn, const uint64_t *base)
+{
+  return SVE_ACLE_FUNC(svld1,_u64,_x2,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svld1_u8_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> poison, <vscale x 16 x i8> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], i64 16)
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 2
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP4]], <vscale x 16 x i8> [[TMP5]], i64 32)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 3
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]], i64 48)
+// CHECK-NEXT:    ret <vscale x 64 x i8> [[TMP8]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svld1_u8_x4u11__SVCount_tPKh(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> poison, <vscale x 16 x i8> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 2
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP4]], <vscale x 16 x i8> [[TMP5]], i64 32)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 3
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]], i64 48)
+// CPP-CHECK-NEXT:    ret <vscale x 64 x i8> [[TMP8]]
+//
+svuint8x4_t test_svld1_u8_x4(svcount_t pn, const uint8_t *base)
+{
+  return SVE_ACLE_FUNC(svld1,_u8,_x4,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svld1_u16_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x4.nxv8i16(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> poison, <vscale x 8 x i16> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], i64 8)
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 2
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], i64 16)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 3
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]], i64 24)
+// CHECK-NEXT:    ret <vscale x 32 x i16> [[TMP8]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svld1_u16_x4u11__SVCount_tPKt(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x4.nxv8i16(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> poison, <vscale x 8 x i16> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 2
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 3
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]], i64 24)
+// CPP-CHECK-NEXT:    ret <vscale x 32 x i16> [[TMP8]]
+//
+svuint16x4_t test_svld1_u16_x4(svcount_t pn, const uint16_t *base)
+{
+  return SVE_ACLE_FUNC(svld1,_u16,_x4,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svld1_u32_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld1.pn.x4.nxv4i32(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], i64 4)
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 2
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP4]], <vscale x 4 x i32> [[TMP5]], i64 8)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 3
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP6]], <vscale x 4 x i32> [[TMP7]], i64 12)
+// CHECK-NEXT:    ret <vscale x 16 x i32> [[TMP8]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svld1_u32_x4u11__SVCount_tPKj(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld1.pn.x4.nxv4i32(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 2
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP4]], <vscale x 4 x i32> [[TMP5]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 3
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP6]], <vscale x 4 x i32> [[TMP7]], i64 12)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i32> [[TMP8]]
+//
+svuint32x4_t test_svld1_u32_x4(svcount_t pn, const uint32_t *base)
+{
+  return SVE_ACLE_FUNC(svld1,_u32,_x4,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svld1_u64_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld1.pn.x4.nxv2i64(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> poison, <vscale x 2 x i64> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], i64 2)
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 2
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP4]], <vscale x 2 x i64> [[TMP5]], i64 4)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 3
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP6]], <vscale x 2 x i64> [[TMP7]], i64 6)
+// CHECK-NEXT:    ret <vscale x 8 x i64> [[TMP8]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svld1_u64_x4u11__SVCount_tPKm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld1.pn.x4.nxv2i64(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> poison, <vscale x 2 x i64> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], i64 2)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 2
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP4]], <vscale x 2 x i64> [[TMP5]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 3
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP6]], <vscale x 2 x i64> [[TMP7]], i64 6)
+// CPP-CHECK-NEXT:    ret <vscale x 8 x i64> [[TMP8]]
+//
+svuint64x4_t test_svld1_u64_x4(svcount_t pn, const uint64_t *base)
+{
+  return SVE_ACLE_FUNC(svld1,_u64,_x4,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svld1_s8_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> poison, <vscale x 16 x i8> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], i64 16)
+// CHECK-NEXT:    ret <vscale x 32 x i8> [[TMP4]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svld1_s8_x2u11__SVCount_tPKa(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> poison, <vscale x 16 x i8> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], i64 16)
+// CPP-CHECK-NEXT:    ret <vscale x 32 x i8> [[TMP4]]
+//
+svint8x2_t test_svld1_s8_x2(svcount_t pn, const int8_t *base)
+{
+  return SVE_ACLE_FUNC(svld1,_s8,_x2,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svld1_s16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x2.nxv8i16(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> poison, <vscale x 8 x i16> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], i64 8)
+// CHECK-NEXT:    ret <vscale x 16 x i16> [[TMP4]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svld1_s16_x2u11__SVCount_tPKs(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x2.nxv8i16(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> poison, <vscale x 8 x i16> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], i64 8)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i16> [[TMP4]]
+//
+svint16x2_t test_svld1_s16_x2(svcount_t pn, const int16_t *base)
+{
+  return SVE_ACLE_FUNC(svld1,_s16,_x2,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svld1_s32_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld1.pn.x2.nxv4i32(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> poison, <vscale x 4 x i32> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], i64 4)
+// CHECK-NEXT:    ret <vscale x 8 x i32> [[TMP4]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svld1_s32_x2u11__SVCount_tPKi(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld1.pn.x2.nxv4i32(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> poison, <vscale x 4 x i32> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], i64 4)
+// CPP-CHECK-NEXT:    ret <vscale x 8 x i32> [[TMP4]]
+//
+svint32x2_t test_svld1_s32_x2(svcount_t pn, const int32_t *base)
+{
+  return SVE_ACLE_FUNC(svld1,_s32,_x2,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svld1_s64_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld1.pn.x2.nxv2i64(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> poison, <vscale x 2 x i64> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], i64 2)
+// CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP4]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svld1_s64_x2u11__SVCount_tPKl(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld1.pn.x2.nxv2i64(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> poison, <vscale x 2 x i64> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], i64 2)
+// CPP-CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP4]]
+//
+svint64x2_t test_svld1_s64_x2(svcount_t pn, const int64_t *base)
+{
+  return SVE_ACLE_FUNC(svld1,_s64,_x2,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svld1_s8_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> poison, <vscale x 16 x i8> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], i64 16)
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 2
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP4]], <vscale x 16 x i8> [[TMP5]], i64 32)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 3
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]], i64 48)
+// CHECK-NEXT:    ret <vscale x 64 x i8> [[TMP8]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svld1_s8_x4u11__SVCount_tPKa(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> poison, <vscale x 16 x i8> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 2
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP4]], <vscale x 16 x i8> [[TMP5]], i64 32)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 3
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]], i64 48)
+// CPP-CHECK-NEXT:    ret <vscale x 64 x i8> [[TMP8]]
+//
+svint8x4_t test_svld1_s8_x4(svcount_t pn, const int8_t *base)
+{
+  return SVE_ACLE_FUNC(svld1,_s8,_x4,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svld1_s16_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x4.nxv8i16(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> poison, <vscale x 8 x i16> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], i64 8)
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 2
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], i64 16)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 3
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]], i64 24)
+// CHECK-NEXT:    ret <vscale x 32 x i16> [[TMP8]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svld1_s16_x4u11__SVCount_tPKs(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x4.nxv8i16(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> poison, <vscale x 8 x i16> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 2
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 3
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]], i64 24)
+// CPP-CHECK-NEXT:    ret <vscale x 32 x i16> [[TMP8]]
+//
+svint16x4_t test_svld1_s16_x4(svcount_t pn, const int16_t *base)
+{
+  return SVE_ACLE_FUNC(svld1,_s16,_x4,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svld1_s32_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld1.pn.x4.nxv4i32(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], i64 4)
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 2
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP4]], <vscale x 4 x i32> [[TMP5]], i64 8)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 3
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP6]], <vscale x 4 x i32> [[TMP7]], i64 12)
+// CHECK-NEXT:    ret <vscale x 16 x i32> [[TMP8]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svld1_s32_x4u11__SVCount_tPKi(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld1.pn.x4.nxv4i32(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 2
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP4]], <vscale x 4 x i32> [[TMP5]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 3
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP6]], <vscale x 4 x i32> [[TMP7]], i64 12)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i32> [[TMP8]]
+//
+svint32x4_t test_svld1_s32_x4(svcount_t pn, const int32_t *base)
+{
+  return SVE_ACLE_FUNC(svld1,_s32,_x4,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svld1_s64_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld1.pn.x4.nxv2i64(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> poison, <vscale x 2 x i64> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], i64 2)
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 2
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP4]], <vscale x 2 x i64> [[TMP5]], i64 4)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 3
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP6]], <vscale x 2 x i64> [[TMP7]], i64 6)
+// CHECK-NEXT:    ret <vscale x 8 x i64> [[TMP8]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svld1_s64_x4u11__SVCount_tPKl(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld1.pn.x4.nxv2i64(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> poison, <vscale x 2 x i64> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], i64 2)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 2
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP4]], <vscale x 2 x i64> [[TMP5]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 3
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP6]], <vscale x 2 x i64> [[TMP7]], i64 6)
+// CPP-CHECK-NEXT:    ret <vscale x 8 x i64> [[TMP8]]
+//
+svint64x4_t test_svld1_s64_x4(svcount_t pn, const int64_t *base)
+{
+  return SVE_ACLE_FUNC(svld1,_s64,_x4,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svld1_f16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld1.pn.x2.nxv8f16(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x half> @llvm.vector.insert.nxv16f16.nxv8f16(<vscale x 16 x half> poison, <vscale x 8 x half> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x half> @llvm.vector.insert.nxv16f16.nxv8f16(<vscale x 16 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], i64 8)
+// CHECK-NEXT:    ret <vscale x 16 x half> [[TMP4]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svld1_f16_x2u11__SVCount_tPKDh(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld1.pn.x2.nxv8f16(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x half> @llvm.vector.insert.nxv16f16.nxv8f16(<vscale x 16 x half> poison, <vscale x 8 x half> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x half> @llvm.vector.insert.nxv16f16.nxv8f16(<vscale x 16 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], i64 8)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x half> [[TMP4]]
+//
+svfloat16x2_t test_svld1_f16_x2(svcount_t pn, const float16_t *base)
+{
+  return SVE_ACLE_FUNC(svld1,_f16,_x2,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svld1_f32_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld1.pn.x2.nxv4f32(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> poison, <vscale x 4 x float> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], i64 4)
+// CHECK-NEXT:    ret <vscale x 8 x float> [[TMP4]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svld1_f32_x2u11__SVCount_tPKf(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld1.pn.x2.nxv4f32(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> poison, <vscale x 4 x float> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], i64 4)
+// CPP-CHECK-NEXT:    ret <vscale x 8 x float> [[TMP4]]
+//
+svfloat32x2_t test_svld1_f32_x2(svcount_t pn, const float32_t *base)
+{
+  return SVE_ACLE_FUNC(svld1,_f32,_x2,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svld1_f64_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x2.nxv2f64(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x double> @llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> poison, <vscale x 2 x double> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 4 x double> @llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], i64 2)
+// CHECK-NEXT:    ret <vscale x 4 x double> [[TMP4]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svld1_f64_x2u11__SVCount_tPKd(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x2.nxv2f64(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x double> @llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> poison, <vscale x 2 x double> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 4 x double> @llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], i64 2)
+// CPP-CHECK-NEXT:    ret <vscale x 4 x double> [[TMP4]]
+//
+svfloat64x2_t test_svld1_f64_x2(svcount_t pn, const float64_t *base)
+{
+  return SVE_ACLE_FUNC(svld1,_f64,_x2,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svld1_f16_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld1.pn.x4.nxv8f16(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> poison, <vscale x 8 x half> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], i64 8)
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP0]], 2
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP4]], <vscale x 8 x half> [[TMP5]], i64 16)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP0]], 3
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP6]], <vscale x 8 x half> [[TMP7]], i64 24)
+// CHECK-NEXT:    ret <vscale x 32 x half> [[TMP8]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svld1_f16_x4u11__SVCount_tPKDh(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld1.pn.x4.nxv8f16(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> poison, <vscale x 8 x half> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP0]], 2
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP4]], <vscale x 8 x half> [[TMP5]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP0]], 3
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP6]], <vscale x 8 x half> [[TMP7]], i64 24)
+// CPP-CHECK-NEXT:    ret <vscale x 32 x half> [[TMP8]]
+//
+svfloat16x4_t test_svld1_f16_x4(svcount_t pn, const float16_t *base)
+{
+  return SVE_ACLE_FUNC(svld1,_f16,_x4,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svld1_f32_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld1.pn.x4.nxv4f32(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> poison, <vscale x 4 x float> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], i64 4)
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP0]], 2
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP4]], <vscale x 4 x float> [[TMP5]], i64 8)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP0]], 3
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP6]], <vscale x 4 x float> [[TMP7]], i64 12)
+// CHECK-NEXT:    ret <vscale x 16 x float> [[TMP8]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svld1_f32_x4u11__SVCount_tPKf(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld1.pn.x4.nxv4f32(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> poison, <vscale x 4 x float> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP0]], 2
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP4]], <vscale x 4 x float> [[TMP5]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP0]], 3
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP6]], <vscale x 4 x float> [[TMP7]], i64 12)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x float> [[TMP8]]
+//
+svfloat32x4_t test_svld1_f32_x4(svcount_t pn, const float32_t *base)
+{
+  return SVE_ACLE_FUNC(svld1,_f32,_x4,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svld1_f64_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x4.nxv2f64(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> poison, <vscale x 2 x double> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], i64 2)
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP0]], 2
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP4]], <vscale x 2 x double> [[TMP5]], i64 4)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP0]], 3
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP6]], <vscale x 2 x double> [[TMP7]], i64 6)
+// CHECK-NEXT:    ret <vscale x 8 x double> [[TMP8]]
+//
+// CPP-CHECK-LABEL: @_Z17test_svld1_f64_x4u11__SVCount_tPKd(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x4.nxv2f64(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> poison, <vscale x 2 x double> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], i64 2)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP0]], 2
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP4]], <vscale x 2 x double> [[TMP5]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP0]], 3
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP6]], <vscale x 2 x double> [[TMP7]], i64 6)
+// CPP-CHECK-NEXT:    ret <vscale x 8 x double> [[TMP8]]
+//
+svfloat64x4_t test_svld1_f64_x4(svcount_t pn, const float64_t *base)
+{
+  return SVE_ACLE_FUNC(svld1,_f64,_x4,)(pn, base);
+}
+
+
+// == VNUM variants ==
+
+
+// CHECK-LABEL: @test_svld1_vnum_u8_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> poison, <vscale x 16 x i8> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], i64 16)
+// CHECK-NEXT:    ret <vscale x 32 x i8> [[TMP5]]
+//
+// CPP-CHECK-LABEL: @_Z21test_svld1_vnum_u8_x2u11__SVCount_tPKhl(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> poison, <vscale x 16 x i8> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], i64 16)
+// CPP-CHECK-NEXT:    ret <vscale x 32 x i8> [[TMP5]]
+//
+svuint8x2_t test_svld1_vnum_u8_x2(svcount_t pn, const uint8_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svld1_vnum,_u8,_x2,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svld1_vnum_u16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x2.nxv8i16(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> poison, <vscale x 8 x i16> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], i64 8)
+// CHECK-NEXT:    ret <vscale x 16 x i16> [[TMP5]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svld1_vnum_u16_x2u11__SVCount_tPKtl(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x2.nxv8i16(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> poison, <vscale x 8 x i16> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], i64 8)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i16> [[TMP5]]
+//
+svuint16x2_t test_svld1_vnum_u16_x2(svcount_t pn, const uint16_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svld1_vnum,_u16,_x2,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svld1_vnum_u32_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld1.pn.x2.nxv4i32(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> poison, <vscale x 4 x i32> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], i64 4)
+// CHECK-NEXT:    ret <vscale x 8 x i32> [[TMP5]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svld1_vnum_u32_x2u11__SVCount_tPKjl(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld1.pn.x2.nxv4i32(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> poison, <vscale x 4 x i32> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], i64 4)
+// CPP-CHECK-NEXT:    ret <vscale x 8 x i32> [[TMP5]]
+//
+svuint32x2_t test_svld1_vnum_u32_x2(svcount_t pn, const uint32_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svld1_vnum,_u32,_x2,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svld1_vnum_u64_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld1.pn.x2.nxv2i64(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> poison, <vscale x 2 x i64> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], i64 2)
+// CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP5]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svld1_vnum_u64_x2u11__SVCount_tPKml(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld1.pn.x2.nxv2i64(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> poison, <vscale x 2 x i64> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], i64 2)
+// CPP-CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP5]]
+//
+svuint64x2_t test_svld1_vnum_u64_x2(svcount_t pn, const uint64_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svld1_vnum,_u64,_x2,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svld1_vnum_u8_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> poison, <vscale x 16 x i8> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], i64 16)
+// CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 2
+// CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP5]], <vscale x 16 x i8> [[TMP6]], i64 32)
+// CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 3
+// CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP7]], <vscale x 16 x i8> [[TMP8]], i64 48)
+// CHECK-NEXT:    ret <vscale x 64 x i8> [[TMP9]]
+//
+// CPP-CHECK-LABEL: @_Z21test_svld1_vnum_u8_x4u11__SVCount_tPKhl(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> poison, <vscale x 16 x i8> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 2
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP5]], <vscale x 16 x i8> [[TMP6]], i64 32)
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 3
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP7]], <vscale x 16 x i8> [[TMP8]], i64 48)
+// CPP-CHECK-NEXT:    ret <vscale x 64 x i8> [[TMP9]]
+//
+svuint8x4_t test_svld1_vnum_u8_x4(svcount_t pn, const uint8_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svld1_vnum,_u8,_x4,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svld1_vnum_u16_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x4.nxv8i16(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> poison, <vscale x 8 x i16> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], i64 8)
+// CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 2
+// CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP5]], <vscale x 8 x i16> [[TMP6]], i64 16)
+// CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 3
+// CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP7]], <vscale x 8 x i16> [[TMP8]], i64 24)
+// CHECK-NEXT:    ret <vscale x 32 x i16> [[TMP9]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svld1_vnum_u16_x4u11__SVCount_tPKtl(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x4.nxv8i16(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> poison, <vscale x 8 x i16> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 2
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP5]], <vscale x 8 x i16> [[TMP6]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 3
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP7]], <vscale x 8 x i16> [[TMP8]], i64 24)
+// CPP-CHECK-NEXT:    ret <vscale x 32 x i16> [[TMP9]]
+//
+svuint16x4_t test_svld1_vnum_u16_x4(svcount_t pn, const uint16_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svld1_vnum,_u16,_x4,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svld1_vnum_u32_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld1.pn.x4.nxv4i32(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], i64 4)
+// CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 2
+// CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP5]], <vscale x 4 x i32> [[TMP6]], i64 8)
+// CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 3
+// CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP7]], <vscale x 4 x i32> [[TMP8]], i64 12)
+// CHECK-NEXT:    ret <vscale x 16 x i32> [[TMP9]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svld1_vnum_u32_x4u11__SVCount_tPKjl(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld1.pn.x4.nxv4i32(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 2
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP5]], <vscale x 4 x i32> [[TMP6]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 3
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP7]], <vscale x 4 x i32> [[TMP8]], i64 12)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i32> [[TMP9]]
+//
+svuint32x4_t test_svld1_vnum_u32_x4(svcount_t pn, const uint32_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svld1_vnum,_u32,_x4,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svld1_vnum_u64_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld1.pn.x4.nxv2i64(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> poison, <vscale x 2 x i64> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], i64 2)
+// CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 2
+// CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP5]], <vscale x 2 x i64> [[TMP6]], i64 4)
+// CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 3
+// CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP7]], <vscale x 2 x i64> [[TMP8]], i64 6)
+// CHECK-NEXT:    ret <vscale x 8 x i64> [[TMP9]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svld1_vnum_u64_x4u11__SVCount_tPKml(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld1.pn.x4.nxv2i64(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> poison, <vscale x 2 x i64> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], i64 2)
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 2
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP5]], <vscale x 2 x i64> [[TMP6]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 3
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP7]], <vscale x 2 x i64> [[TMP8]], i64 6)
+// CPP-CHECK-NEXT:    ret <vscale x 8 x i64> [[TMP9]]
+//
+svuint64x4_t test_svld1_vnum_u64_x4(svcount_t pn, const uint64_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svld1_vnum,_u64,_x4,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svld1_vnum_s8_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> poison, <vscale x 16 x i8> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], i64 16)
+// CHECK-NEXT:    ret <vscale x 32 x i8> [[TMP5]]
+//
+// CPP-CHECK-LABEL: @_Z21test_svld1_vnum_s8_x2u11__SVCount_tPKal(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> poison, <vscale x 16 x i8> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], i64 16)
+// CPP-CHECK-NEXT:    ret <vscale x 32 x i8> [[TMP5]]
+//
+svint8x2_t test_svld1_vnum_s8_x2(svcount_t pn, const int8_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svld1_vnum,_s8,_x2,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svld1_vnum_s16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x2.nxv8i16(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> poison, <vscale x 8 x i16> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], i64 8)
+// CHECK-NEXT:    ret <vscale x 16 x i16> [[TMP5]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svld1_vnum_s16_x2u11__SVCount_tPKsl(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x2.nxv8i16(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> poison, <vscale x 8 x i16> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], i64 8)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i16> [[TMP5]]
+//
+svint16x2_t test_svld1_vnum_s16_x2(svcount_t pn, const int16_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svld1_vnum,_s16,_x2,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svld1_vnum_s32_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld1.pn.x2.nxv4i32(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> poison, <vscale x 4 x i32> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], i64 4)
+// CHECK-NEXT:    ret <vscale x 8 x i32> [[TMP5]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svld1_vnum_s32_x2u11__SVCount_tPKil(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld1.pn.x2.nxv4i32(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> poison, <vscale x 4 x i32> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], i64 4)
+// CPP-CHECK-NEXT:    ret <vscale x 8 x i32> [[TMP5]]
+//
+svint32x2_t test_svld1_vnum_s32_x2(svcount_t pn, const int32_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svld1_vnum,_s32,_x2,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svld1_vnum_s64_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld1.pn.x2.nxv2i64(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> poison, <vscale x 2 x i64> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], i64 2)
+// CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP5]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svld1_vnum_s64_x2u11__SVCount_tPKll(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld1.pn.x2.nxv2i64(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> poison, <vscale x 2 x i64> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], i64 2)
+// CPP-CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP5]]
+//
+svint64x2_t test_svld1_vnum_s64_x2(svcount_t pn, const int64_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svld1_vnum,_s64,_x2,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svld1_vnum_s8_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> poison, <vscale x 16 x i8> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], i64 16)
+// CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 2
+// CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP5]], <vscale x 16 x i8> [[TMP6]], i64 32)
+// CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 3
+// CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP7]], <vscale x 16 x i8> [[TMP8]], i64 48)
+// CHECK-NEXT:    ret <vscale x 64 x i8> [[TMP9]]
+//
+// CPP-CHECK-LABEL: @_Z21test_svld1_vnum_s8_x4u11__SVCount_tPKal(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> poison, <vscale x 16 x i8> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 2
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP5]], <vscale x 16 x i8> [[TMP6]], i64 32)
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 3
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP7]], <vscale x 16 x i8> [[TMP8]], i64 48)
+// CPP-CHECK-NEXT:    ret <vscale x 64 x i8> [[TMP9]]
+//
+svint8x4_t test_svld1_vnum_s8_x4(svcount_t pn, const int8_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svld1_vnum,_s8,_x4,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svld1_vnum_s16_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x4.nxv8i16(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> poison, <vscale x 8 x i16> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], i64 8)
+// CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 2
+// CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP5]], <vscale x 8 x i16> [[TMP6]], i64 16)
+// CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 3
+// CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP7]], <vscale x 8 x i16> [[TMP8]], i64 24)
+// CHECK-NEXT:    ret <vscale x 32 x i16> [[TMP9]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svld1_vnum_s16_x4u11__SVCount_tPKsl(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x4.nxv8i16(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> poison, <vscale x 8 x i16> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 2
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP5]], <vscale x 8 x i16> [[TMP6]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 3
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP7]], <vscale x 8 x i16> [[TMP8]], i64 24)
+// CPP-CHECK-NEXT:    ret <vscale x 32 x i16> [[TMP9]]
+//
+svint16x4_t test_svld1_vnum_s16_x4(svcount_t pn, const int16_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svld1_vnum,_s16,_x4,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svld1_vnum_s32_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld1.pn.x4.nxv4i32(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], i64 4)
+// CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 2
+// CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP5]], <vscale x 4 x i32> [[TMP6]], i64 8)
+// CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 3
+// CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP7]], <vscale x 4 x i32> [[TMP8]], i64 12)
+// CHECK-NEXT:    ret <vscale x 16 x i32> [[TMP9]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svld1_vnum_s32_x4u11__SVCount_tPKil(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld1.pn.x4.nxv4i32(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 2
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP5]], <vscale x 4 x i32> [[TMP6]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 3
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP7]], <vscale x 4 x i32> [[TMP8]], i64 12)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i32> [[TMP9]]
+//
+svint32x4_t test_svld1_vnum_s32_x4(svcount_t pn, const int32_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svld1_vnum,_s32,_x4,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svld1_vnum_s64_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld1.pn.x4.nxv2i64(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> poison, <vscale x 2 x i64> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], i64 2)
+// CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 2
+// CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP5]], <vscale x 2 x i64> [[TMP6]], i64 4)
+// CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 3
+// CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP7]], <vscale x 2 x i64> [[TMP8]], i64 6)
+// CHECK-NEXT:    ret <vscale x 8 x i64> [[TMP9]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svld1_vnum_s64_x4u11__SVCount_tPKll(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld1.pn.x4.nxv2i64(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> poison, <vscale x 2 x i64> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], i64 2)
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 2
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP5]], <vscale x 2 x i64> [[TMP6]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 3
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP7]], <vscale x 2 x i64> [[TMP8]], i64 6)
+// CPP-CHECK-NEXT:    ret <vscale x 8 x i64> [[TMP9]]
+//
+svint64x4_t test_svld1_vnum_s64_x4(svcount_t pn, const int64_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svld1_vnum,_s64,_x4,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svld1_vnum_f16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld1.pn.x2.nxv8f16(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x half> @llvm.vector.insert.nxv16f16.nxv8f16(<vscale x 16 x half> poison, <vscale x 8 x half> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 16 x half> @llvm.vector.insert.nxv16f16.nxv8f16(<vscale x 16 x half> [[TMP3]], <vscale x 8 x half> [[TMP4]], i64 8)
+// CHECK-NEXT:    ret <vscale x 16 x half> [[TMP5]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svld1_vnum_f16_x2u11__SVCount_tPKDhl(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld1.pn.x2.nxv8f16(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x half> @llvm.vector.insert.nxv16f16.nxv8f16(<vscale x 16 x half> poison, <vscale x 8 x half> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 16 x half> @llvm.vector.insert.nxv16f16.nxv8f16(<vscale x 16 x half> [[TMP3]], <vscale x 8 x half> [[TMP4]], i64 8)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x half> [[TMP5]]
+//
+svfloat16x2_t test_svld1_vnum_f16_x2(svcount_t pn, const float16_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svld1_vnum,_f16,_x2,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svld1_vnum_f32_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld1.pn.x2.nxv4f32(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> poison, <vscale x 4 x float> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> [[TMP3]], <vscale x 4 x float> [[TMP4]], i64 4)
+// CHECK-NEXT:    ret <vscale x 8 x float> [[TMP5]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svld1_vnum_f32_x2u11__SVCount_tPKfl(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld1.pn.x2.nxv4f32(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> poison, <vscale x 4 x float> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> [[TMP3]], <vscale x 4 x float> [[TMP4]], i64 4)
+// CPP-CHECK-NEXT:    ret <vscale x 8 x float> [[TMP5]]
+//
+svfloat32x2_t test_svld1_vnum_f32_x2(svcount_t pn, const float32_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svld1_vnum,_f32,_x2,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svld1_vnum_f64_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x2.nxv2f64(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x double> @llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> poison, <vscale x 2 x double> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 4 x double> @llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> [[TMP3]], <vscale x 2 x double> [[TMP4]], i64 2)
+// CHECK-NEXT:    ret <vscale x 4 x double> [[TMP5]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svld1_vnum_f64_x2u11__SVCount_tPKdl(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x2.nxv2f64(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x double> @llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> poison, <vscale x 2 x double> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 4 x double> @llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> [[TMP3]], <vscale x 2 x double> [[TMP4]], i64 2)
+// CPP-CHECK-NEXT:    ret <vscale x 4 x double> [[TMP5]]
+//
+svfloat64x2_t test_svld1_vnum_f64_x2(svcount_t pn, const float64_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svld1_vnum,_f64,_x2,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svld1_vnum_f16_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld1.pn.x4.nxv8f16(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> poison, <vscale x 8 x half> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP3]], <vscale x 8 x half> [[TMP4]], i64 8)
+// CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP1]], 2
+// CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP5]], <vscale x 8 x half> [[TMP6]], i64 16)
+// CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP1]], 3
+// CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP7]], <vscale x 8 x half> [[TMP8]], i64 24)
+// CHECK-NEXT:    ret <vscale x 32 x half> [[TMP9]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svld1_vnum_f16_x4u11__SVCount_tPKDhl(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld1.pn.x4.nxv8f16(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> poison, <vscale x 8 x half> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP3]], <vscale x 8 x half> [[TMP4]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP1]], 2
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP5]], <vscale x 8 x half> [[TMP6]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP1]], 3
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP7]], <vscale x 8 x half> [[TMP8]], i64 24)
+// CPP-CHECK-NEXT:    ret <vscale x 32 x half> [[TMP9]]
+//
+svfloat16x4_t test_svld1_vnum_f16_x4(svcount_t pn, const float16_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svld1_vnum,_f16,_x4,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svld1_vnum_f32_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld1.pn.x4.nxv4f32(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> poison, <vscale x 4 x float> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP3]], <vscale x 4 x float> [[TMP4]], i64 4)
+// CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP1]], 2
+// CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP5]], <vscale x 4 x float> [[TMP6]], i64 8)
+// CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP1]], 3
+// CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP7]], <vscale x 4 x float> [[TMP8]], i64 12)
+// CHECK-NEXT:    ret <vscale x 16 x float> [[TMP9]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svld1_vnum_f32_x4u11__SVCount_tPKfl(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld1.pn.x4.nxv4f32(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> poison, <vscale x 4 x float> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP3]], <vscale x 4 x float> [[TMP4]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP1]], 2
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP5]], <vscale x 4 x float> [[TMP6]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP1]], 3
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP7]], <vscale x 4 x float> [[TMP8]], i64 12)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x float> [[TMP9]]
+//
+svfloat32x4_t test_svld1_vnum_f32_x4(svcount_t pn, const float32_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svld1_vnum,_f32,_x4,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svld1_vnum_f64_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x4.nxv2f64(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> poison, <vscale x 2 x double> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP3]], <vscale x 2 x double> [[TMP4]], i64 2)
+// CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP1]], 2
+// CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP5]], <vscale x 2 x double> [[TMP6]], i64 4)
+// CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP1]], 3
+// CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP7]], <vscale x 2 x double> [[TMP8]], i64 6)
+// CHECK-NEXT:    ret <vscale x 8 x double> [[TMP9]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svld1_vnum_f64_x4u11__SVCount_tPKdl(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x4.nxv2f64(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> poison, <vscale x 2 x double> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP3]], <vscale x 2 x double> [[TMP4]], i64 2)
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP1]], 2
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP5]], <vscale x 2 x double> [[TMP6]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP1]], 3
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP7]], <vscale x 2 x double> [[TMP8]], i64 6)
+// CPP-CHECK-NEXT:    ret <vscale x 8 x double> [[TMP9]]
+//
+svfloat64x4_t test_svld1_vnum_f64_x4(svcount_t pn, const float64_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svld1_vnum,_f64,_x4,)(pn, base, vnum);
+}
diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_ldnt1.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_ldnt1.c
new file mode 100644
index 0000000000..7a0fcde819
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_ldnt1.c
@@ -0,0 +1,1250 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+// CHECK-LABEL: @test_svldnt1_u8_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv16i8(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> poison, <vscale x 16 x i8> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], i64 16)
+// CHECK-NEXT:    ret <vscale x 32 x i8> [[TMP4]]
+//
+// CPP-CHECK-LABEL: @_Z18test_svldnt1_u8_x2u11__SVCount_tPKh(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv16i8(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> poison, <vscale x 16 x i8> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], i64 16)
+// CPP-CHECK-NEXT:    ret <vscale x 32 x i8> [[TMP4]]
+//
+svuint8x2_t test_svldnt1_u8_x2(svcount_t pn, const uint8_t *base)
+{
+  return SVE_ACLE_FUNC(svldnt1,_u8,_x2,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svldnt1_u16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv8i16(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> poison, <vscale x 8 x i16> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], i64 8)
+// CHECK-NEXT:    ret <vscale x 16 x i16> [[TMP4]]
+//
+// CPP-CHECK-LABEL: @_Z19test_svldnt1_u16_x2u11__SVCount_tPKt(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv8i16(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> poison, <vscale x 8 x i16> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], i64 8)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i16> [[TMP4]]
+//
+svuint16x2_t test_svldnt1_u16_x2(svcount_t pn, const uint16_t *base)
+{
+  return SVE_ACLE_FUNC(svldnt1,_u16,_x2,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svldnt1_u32_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv4i32(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> poison, <vscale x 4 x i32> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], i64 4)
+// CHECK-NEXT:    ret <vscale x 8 x i32> [[TMP4]]
+//
+// CPP-CHECK-LABEL: @_Z19test_svldnt1_u32_x2u11__SVCount_tPKj(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv4i32(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> poison, <vscale x 4 x i32> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], i64 4)
+// CPP-CHECK-NEXT:    ret <vscale x 8 x i32> [[TMP4]]
+//
+svuint32x2_t test_svldnt1_u32_x2(svcount_t pn, const uint32_t *base)
+{
+  return SVE_ACLE_FUNC(svldnt1,_u32,_x2,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svldnt1_u64_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv2i64(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> poison, <vscale x 2 x i64> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], i64 2)
+// CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP4]]
+//
+// CPP-CHECK-LABEL: @_Z19test_svldnt1_u64_x2u11__SVCount_tPKm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv2i64(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> poison, <vscale x 2 x i64> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], i64 2)
+// CPP-CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP4]]
+//
+svuint64x2_t test_svldnt1_u64_x2(svcount_t pn, const uint64_t *base)
+{
+  return SVE_ACLE_FUNC(svldnt1,_u64,_x2,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svldnt1_u8_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv16i8(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> poison, <vscale x 16 x i8> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], i64 16)
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 2
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP4]], <vscale x 16 x i8> [[TMP5]], i64 32)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 3
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]], i64 48)
+// CHECK-NEXT:    ret <vscale x 64 x i8> [[TMP8]]
+//
+// CPP-CHECK-LABEL: @_Z18test_svldnt1_u8_x4u11__SVCount_tPKh(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv16i8(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> poison, <vscale x 16 x i8> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 2
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP4]], <vscale x 16 x i8> [[TMP5]], i64 32)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 3
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]], i64 48)
+// CPP-CHECK-NEXT:    ret <vscale x 64 x i8> [[TMP8]]
+//
+svuint8x4_t test_svldnt1_u8_x4(svcount_t pn, const uint8_t *base)
+{
+  return SVE_ACLE_FUNC(svldnt1,_u8,_x4,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svldnt1_u16_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv8i16(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> poison, <vscale x 8 x i16> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], i64 8)
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 2
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], i64 16)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 3
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]], i64 24)
+// CHECK-NEXT:    ret <vscale x 32 x i16> [[TMP8]]
+//
+// CPP-CHECK-LABEL: @_Z19test_svldnt1_u16_x4u11__SVCount_tPKt(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv8i16(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> poison, <vscale x 8 x i16> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 2
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 3
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]], i64 24)
+// CPP-CHECK-NEXT:    ret <vscale x 32 x i16> [[TMP8]]
+//
+svuint16x4_t test_svldnt1_u16_x4(svcount_t pn, const uint16_t *base)
+{
+  return SVE_ACLE_FUNC(svldnt1,_u16,_x4,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svldnt1_u32_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv4i32(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], i64 4)
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 2
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP4]], <vscale x 4 x i32> [[TMP5]], i64 8)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 3
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP6]], <vscale x 4 x i32> [[TMP7]], i64 12)
+// CHECK-NEXT:    ret <vscale x 16 x i32> [[TMP8]]
+//
+// CPP-CHECK-LABEL: @_Z19test_svldnt1_u32_x4u11__SVCount_tPKj(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv4i32(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 2
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP4]], <vscale x 4 x i32> [[TMP5]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 3
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP6]], <vscale x 4 x i32> [[TMP7]], i64 12)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i32> [[TMP8]]
+//
+svuint32x4_t test_svldnt1_u32_x4(svcount_t pn, const uint32_t *base)
+{
+  return SVE_ACLE_FUNC(svldnt1,_u32,_x4,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svldnt1_u64_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv2i64(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> poison, <vscale x 2 x i64> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], i64 2)
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 2
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP4]], <vscale x 2 x i64> [[TMP5]], i64 4)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 3
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP6]], <vscale x 2 x i64> [[TMP7]], i64 6)
+// CHECK-NEXT:    ret <vscale x 8 x i64> [[TMP8]]
+//
+// CPP-CHECK-LABEL: @_Z19test_svldnt1_u64_x4u11__SVCount_tPKm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv2i64(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> poison, <vscale x 2 x i64> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], i64 2)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 2
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP4]], <vscale x 2 x i64> [[TMP5]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 3
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP6]], <vscale x 2 x i64> [[TMP7]], i64 6)
+// CPP-CHECK-NEXT:    ret <vscale x 8 x i64> [[TMP8]]
+//
+svuint64x4_t test_svldnt1_u64_x4(svcount_t pn, const uint64_t *base)
+{
+  return SVE_ACLE_FUNC(svldnt1,_u64,_x4,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svldnt1_s8_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv16i8(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> poison, <vscale x 16 x i8> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], i64 16)
+// CHECK-NEXT:    ret <vscale x 32 x i8> [[TMP4]]
+//
+// CPP-CHECK-LABEL: @_Z18test_svldnt1_s8_x2u11__SVCount_tPKa(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv16i8(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> poison, <vscale x 16 x i8> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], i64 16)
+// CPP-CHECK-NEXT:    ret <vscale x 32 x i8> [[TMP4]]
+//
+svint8x2_t test_svldnt1_s8_x2(svcount_t pn, const int8_t *base)
+{
+  return SVE_ACLE_FUNC(svldnt1,_s8,_x2,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svldnt1_s16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv8i16(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> poison, <vscale x 8 x i16> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], i64 8)
+// CHECK-NEXT:    ret <vscale x 16 x i16> [[TMP4]]
+//
+// CPP-CHECK-LABEL: @_Z19test_svldnt1_s16_x2u11__SVCount_tPKs(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv8i16(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> poison, <vscale x 8 x i16> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], i64 8)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i16> [[TMP4]]
+//
+svint16x2_t test_svldnt1_s16_x2(svcount_t pn, const int16_t *base)
+{
+  return SVE_ACLE_FUNC(svldnt1,_s16,_x2,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svldnt1_s32_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv4i32(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> poison, <vscale x 4 x i32> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], i64 4)
+// CHECK-NEXT:    ret <vscale x 8 x i32> [[TMP4]]
+//
+// CPP-CHECK-LABEL: @_Z19test_svldnt1_s32_x2u11__SVCount_tPKi(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv4i32(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> poison, <vscale x 4 x i32> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], i64 4)
+// CPP-CHECK-NEXT:    ret <vscale x 8 x i32> [[TMP4]]
+//
+svint32x2_t test_svldnt1_s32_x2(svcount_t pn, const int32_t *base)
+{
+  return SVE_ACLE_FUNC(svldnt1,_s32,_x2,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svldnt1_s64_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv2i64(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> poison, <vscale x 2 x i64> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], i64 2)
+// CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP4]]
+//
+// CPP-CHECK-LABEL: @_Z19test_svldnt1_s64_x2u11__SVCount_tPKl(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv2i64(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> poison, <vscale x 2 x i64> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], i64 2)
+// CPP-CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP4]]
+//
+svint64x2_t test_svldnt1_s64_x2(svcount_t pn, const int64_t *base)
+{
+  return SVE_ACLE_FUNC(svldnt1,_s64,_x2,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svldnt1_s8_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv16i8(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> poison, <vscale x 16 x i8> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], i64 16)
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 2
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP4]], <vscale x 16 x i8> [[TMP5]], i64 32)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 3
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]], i64 48)
+// CHECK-NEXT:    ret <vscale x 64 x i8> [[TMP8]]
+//
+// CPP-CHECK-LABEL: @_Z18test_svldnt1_s8_x4u11__SVCount_tPKa(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv16i8(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> poison, <vscale x 16 x i8> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 2
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP4]], <vscale x 16 x i8> [[TMP5]], i64 32)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], 3
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]], i64 48)
+// CPP-CHECK-NEXT:    ret <vscale x 64 x i8> [[TMP8]]
+//
+svint8x4_t test_svldnt1_s8_x4(svcount_t pn, const int8_t *base)
+{
+  return SVE_ACLE_FUNC(svldnt1,_s8,_x4,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svldnt1_s16_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv8i16(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> poison, <vscale x 8 x i16> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], i64 8)
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 2
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], i64 16)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 3
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]], i64 24)
+// CHECK-NEXT:    ret <vscale x 32 x i16> [[TMP8]]
+//
+// CPP-CHECK-LABEL: @_Z19test_svldnt1_s16_x4u11__SVCount_tPKs(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv8i16(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> poison, <vscale x 8 x i16> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 2
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP0]], 3
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]], i64 24)
+// CPP-CHECK-NEXT:    ret <vscale x 32 x i16> [[TMP8]]
+//
+svint16x4_t test_svldnt1_s16_x4(svcount_t pn, const int16_t *base)
+{
+  return SVE_ACLE_FUNC(svldnt1,_s16,_x4,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svldnt1_s32_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv4i32(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], i64 4)
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 2
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP4]], <vscale x 4 x i32> [[TMP5]], i64 8)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 3
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP6]], <vscale x 4 x i32> [[TMP7]], i64 12)
+// CHECK-NEXT:    ret <vscale x 16 x i32> [[TMP8]]
+//
+// CPP-CHECK-LABEL: @_Z19test_svldnt1_s32_x4u11__SVCount_tPKi(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv4i32(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 2
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP4]], <vscale x 4 x i32> [[TMP5]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP0]], 3
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP6]], <vscale x 4 x i32> [[TMP7]], i64 12)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i32> [[TMP8]]
+//
+svint32x4_t test_svldnt1_s32_x4(svcount_t pn, const int32_t *base)
+{
+  return SVE_ACLE_FUNC(svldnt1,_s32,_x4,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svldnt1_s64_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv2i64(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> poison, <vscale x 2 x i64> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], i64 2)
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 2
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP4]], <vscale x 2 x i64> [[TMP5]], i64 4)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 3
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP6]], <vscale x 2 x i64> [[TMP7]], i64 6)
+// CHECK-NEXT:    ret <vscale x 8 x i64> [[TMP8]]
+//
+// CPP-CHECK-LABEL: @_Z19test_svldnt1_s64_x4u11__SVCount_tPKl(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv2i64(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> poison, <vscale x 2 x i64> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], i64 2)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 2
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP4]], <vscale x 2 x i64> [[TMP5]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP0]], 3
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP6]], <vscale x 2 x i64> [[TMP7]], i64 6)
+// CPP-CHECK-NEXT:    ret <vscale x 8 x i64> [[TMP8]]
+//
+svint64x4_t test_svldnt1_s64_x4(svcount_t pn, const int64_t *base)
+{
+  return SVE_ACLE_FUNC(svldnt1,_s64,_x4,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svldnt1_f16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv8f16(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x half> @llvm.vector.insert.nxv16f16.nxv8f16(<vscale x 16 x half> poison, <vscale x 8 x half> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x half> @llvm.vector.insert.nxv16f16.nxv8f16(<vscale x 16 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], i64 8)
+// CHECK-NEXT:    ret <vscale x 16 x half> [[TMP4]]
+//
+// CPP-CHECK-LABEL: @_Z19test_svldnt1_f16_x2u11__SVCount_tPKDh(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv8f16(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x half> @llvm.vector.insert.nxv16f16.nxv8f16(<vscale x 16 x half> poison, <vscale x 8 x half> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x half> @llvm.vector.insert.nxv16f16.nxv8f16(<vscale x 16 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], i64 8)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x half> [[TMP4]]
+//
+svfloat16x2_t test_svldnt1_f16_x2(svcount_t pn, const float16_t *base)
+{
+  return SVE_ACLE_FUNC(svldnt1,_f16,_x2,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svldnt1_f32_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv4f32(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> poison, <vscale x 4 x float> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], i64 4)
+// CHECK-NEXT:    ret <vscale x 8 x float> [[TMP4]]
+//
+// CPP-CHECK-LABEL: @_Z19test_svldnt1_f32_x2u11__SVCount_tPKf(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv4f32(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> poison, <vscale x 4 x float> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], i64 4)
+// CPP-CHECK-NEXT:    ret <vscale x 8 x float> [[TMP4]]
+//
+svfloat32x2_t test_svldnt1_f32_x2(svcount_t pn, const float32_t *base)
+{
+  return SVE_ACLE_FUNC(svldnt1,_f32,_x2,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svldnt1_f64_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv2f64(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x double> @llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> poison, <vscale x 2 x double> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 4 x double> @llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], i64 2)
+// CHECK-NEXT:    ret <vscale x 4 x double> [[TMP4]]
+//
+// CPP-CHECK-LABEL: @_Z19test_svldnt1_f64_x2u11__SVCount_tPKd(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv2f64(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x double> @llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> poison, <vscale x 2 x double> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 4 x double> @llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], i64 2)
+// CPP-CHECK-NEXT:    ret <vscale x 4 x double> [[TMP4]]
+//
+svfloat64x2_t test_svldnt1_f64_x2(svcount_t pn, const float64_t *base)
+{
+  return SVE_ACLE_FUNC(svldnt1,_f64,_x2,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svldnt1_f16_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv8f16(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> poison, <vscale x 8 x half> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], i64 8)
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP0]], 2
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP4]], <vscale x 8 x half> [[TMP5]], i64 16)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP0]], 3
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP6]], <vscale x 8 x half> [[TMP7]], i64 24)
+// CHECK-NEXT:    ret <vscale x 32 x half> [[TMP8]]
+//
+// CPP-CHECK-LABEL: @_Z19test_svldnt1_f16_x4u11__SVCount_tPKDh(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv8f16(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> poison, <vscale x 8 x half> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP0]], 2
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP4]], <vscale x 8 x half> [[TMP5]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP0]], 3
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP6]], <vscale x 8 x half> [[TMP7]], i64 24)
+// CPP-CHECK-NEXT:    ret <vscale x 32 x half> [[TMP8]]
+//
+svfloat16x4_t test_svldnt1_f16_x4(svcount_t pn, const float16_t *base)
+{
+  return SVE_ACLE_FUNC(svldnt1,_f16,_x4,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svldnt1_f32_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv4f32(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> poison, <vscale x 4 x float> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], i64 4)
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP0]], 2
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP4]], <vscale x 4 x float> [[TMP5]], i64 8)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP0]], 3
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP6]], <vscale x 4 x float> [[TMP7]], i64 12)
+// CHECK-NEXT:    ret <vscale x 16 x float> [[TMP8]]
+//
+// CPP-CHECK-LABEL: @_Z19test_svldnt1_f32_x4u11__SVCount_tPKf(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv4f32(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> poison, <vscale x 4 x float> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP0]], 2
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP4]], <vscale x 4 x float> [[TMP5]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP0]], 3
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP6]], <vscale x 4 x float> [[TMP7]], i64 12)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x float> [[TMP8]]
+//
+svfloat32x4_t test_svldnt1_f32_x4(svcount_t pn, const float32_t *base)
+{
+  return SVE_ACLE_FUNC(svldnt1,_f32,_x4,)(pn, base);
+}
+
+// CHECK-LABEL: @test_svldnt1_f64_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv2f64(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> poison, <vscale x 2 x double> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], i64 2)
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP0]], 2
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP4]], <vscale x 2 x double> [[TMP5]], i64 4)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP0]], 3
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP6]], <vscale x 2 x double> [[TMP7]], i64 6)
+// CHECK-NEXT:    ret <vscale x 8 x double> [[TMP8]]
+//
+// CPP-CHECK-LABEL: @_Z19test_svldnt1_f64_x4u11__SVCount_tPKd(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv2f64(target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> poison, <vscale x 2 x double> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], i64 2)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP0]], 2
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP4]], <vscale x 2 x double> [[TMP5]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP0]], 3
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP6]], <vscale x 2 x double> [[TMP7]], i64 6)
+// CPP-CHECK-NEXT:    ret <vscale x 8 x double> [[TMP8]]
+//
+svfloat64x4_t test_svldnt1_f64_x4(svcount_t pn, const float64_t *base)
+{
+  return SVE_ACLE_FUNC(svldnt1,_f64,_x4,)(pn, base);
+}
+
+
+// == VNUM variants ==
+
+
+// CHECK-LABEL: @test_svldnt1_vnum_u8_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv16i8(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> poison, <vscale x 16 x i8> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], i64 16)
+// CHECK-NEXT:    ret <vscale x 32 x i8> [[TMP5]]
+//
+// CPP-CHECK-LABEL: @_Z23test_svldnt1_vnum_u8_x2u11__SVCount_tPKhl(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv16i8(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> poison, <vscale x 16 x i8> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], i64 16)
+// CPP-CHECK-NEXT:    ret <vscale x 32 x i8> [[TMP5]]
+//
+svuint8x2_t test_svldnt1_vnum_u8_x2(svcount_t pn, const uint8_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svldnt1_vnum,_u8,_x2,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svldnt1_vnum_u16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv8i16(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> poison, <vscale x 8 x i16> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], i64 8)
+// CHECK-NEXT:    ret <vscale x 16 x i16> [[TMP5]]
+//
+// CPP-CHECK-LABEL: @_Z24test_svldnt1_vnum_u16_x2u11__SVCount_tPKtl(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv8i16(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> poison, <vscale x 8 x i16> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], i64 8)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i16> [[TMP5]]
+//
+svuint16x2_t test_svldnt1_vnum_u16_x2(svcount_t pn, const uint16_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svldnt1_vnum,_u16,_x2,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svldnt1_vnum_u32_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv4i32(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> poison, <vscale x 4 x i32> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], i64 4)
+// CHECK-NEXT:    ret <vscale x 8 x i32> [[TMP5]]
+//
+// CPP-CHECK-LABEL: @_Z24test_svldnt1_vnum_u32_x2u11__SVCount_tPKjl(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv4i32(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> poison, <vscale x 4 x i32> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], i64 4)
+// CPP-CHECK-NEXT:    ret <vscale x 8 x i32> [[TMP5]]
+//
+svuint32x2_t test_svldnt1_vnum_u32_x2(svcount_t pn, const uint32_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svldnt1_vnum,_u32,_x2,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svldnt1_vnum_u64_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv2i64(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> poison, <vscale x 2 x i64> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], i64 2)
+// CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP5]]
+//
+// CPP-CHECK-LABEL: @_Z24test_svldnt1_vnum_u64_x2u11__SVCount_tPKml(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv2i64(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> poison, <vscale x 2 x i64> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], i64 2)
+// CPP-CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP5]]
+//
+svuint64x2_t test_svldnt1_vnum_u64_x2(svcount_t pn, const uint64_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svldnt1_vnum,_u64,_x2,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svldnt1_vnum_u8_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv16i8(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> poison, <vscale x 16 x i8> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], i64 16)
+// CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 2
+// CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP5]], <vscale x 16 x i8> [[TMP6]], i64 32)
+// CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 3
+// CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP7]], <vscale x 16 x i8> [[TMP8]], i64 48)
+// CHECK-NEXT:    ret <vscale x 64 x i8> [[TMP9]]
+//
+// CPP-CHECK-LABEL: @_Z23test_svldnt1_vnum_u8_x4u11__SVCount_tPKhl(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv16i8(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> poison, <vscale x 16 x i8> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 2
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP5]], <vscale x 16 x i8> [[TMP6]], i64 32)
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 3
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP7]], <vscale x 16 x i8> [[TMP8]], i64 48)
+// CPP-CHECK-NEXT:    ret <vscale x 64 x i8> [[TMP9]]
+//
+svuint8x4_t test_svldnt1_vnum_u8_x4(svcount_t pn, const uint8_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svldnt1_vnum,_u8,_x4,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svldnt1_vnum_u16_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv8i16(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> poison, <vscale x 8 x i16> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], i64 8)
+// CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 2
+// CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP5]], <vscale x 8 x i16> [[TMP6]], i64 16)
+// CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 3
+// CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP7]], <vscale x 8 x i16> [[TMP8]], i64 24)
+// CHECK-NEXT:    ret <vscale x 32 x i16> [[TMP9]]
+//
+// CPP-CHECK-LABEL: @_Z24test_svldnt1_vnum_u16_x4u11__SVCount_tPKtl(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv8i16(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> poison, <vscale x 8 x i16> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 2
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP5]], <vscale x 8 x i16> [[TMP6]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 3
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP7]], <vscale x 8 x i16> [[TMP8]], i64 24)
+// CPP-CHECK-NEXT:    ret <vscale x 32 x i16> [[TMP9]]
+//
+svuint16x4_t test_svldnt1_vnum_u16_x4(svcount_t pn, const uint16_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svldnt1_vnum,_u16,_x4,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svldnt1_vnum_u32_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv4i32(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], i64 4)
+// CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 2
+// CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP5]], <vscale x 4 x i32> [[TMP6]], i64 8)
+// CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 3
+// CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP7]], <vscale x 4 x i32> [[TMP8]], i64 12)
+// CHECK-NEXT:    ret <vscale x 16 x i32> [[TMP9]]
+//
+// CPP-CHECK-LABEL: @_Z24test_svldnt1_vnum_u32_x4u11__SVCount_tPKjl(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv4i32(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 2
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP5]], <vscale x 4 x i32> [[TMP6]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 3
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP7]], <vscale x 4 x i32> [[TMP8]], i64 12)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i32> [[TMP9]]
+//
+svuint32x4_t test_svldnt1_vnum_u32_x4(svcount_t pn, const uint32_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svldnt1_vnum,_u32,_x4,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svldnt1_vnum_u64_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv2i64(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> poison, <vscale x 2 x i64> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], i64 2)
+// CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 2
+// CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP5]], <vscale x 2 x i64> [[TMP6]], i64 4)
+// CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 3
+// CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP7]], <vscale x 2 x i64> [[TMP8]], i64 6)
+// CHECK-NEXT:    ret <vscale x 8 x i64> [[TMP9]]
+//
+// CPP-CHECK-LABEL: @_Z24test_svldnt1_vnum_u64_x4u11__SVCount_tPKml(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv2i64(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> poison, <vscale x 2 x i64> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], i64 2)
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 2
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP5]], <vscale x 2 x i64> [[TMP6]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 3
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP7]], <vscale x 2 x i64> [[TMP8]], i64 6)
+// CPP-CHECK-NEXT:    ret <vscale x 8 x i64> [[TMP9]]
+//
+svuint64x4_t test_svldnt1_vnum_u64_x4(svcount_t pn, const uint64_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svldnt1_vnum,_u64,_x4,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svldnt1_vnum_s8_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv16i8(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> poison, <vscale x 16 x i8> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], i64 16)
+// CHECK-NEXT:    ret <vscale x 32 x i8> [[TMP5]]
+//
+// CPP-CHECK-LABEL: @_Z23test_svldnt1_vnum_s8_x2u11__SVCount_tPKal(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv16i8(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> poison, <vscale x 16 x i8> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], i64 16)
+// CPP-CHECK-NEXT:    ret <vscale x 32 x i8> [[TMP5]]
+//
+svint8x2_t test_svldnt1_vnum_s8_x2(svcount_t pn, const int8_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svldnt1_vnum,_s8,_x2,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svldnt1_vnum_s16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv8i16(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> poison, <vscale x 8 x i16> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], i64 8)
+// CHECK-NEXT:    ret <vscale x 16 x i16> [[TMP5]]
+//
+// CPP-CHECK-LABEL: @_Z24test_svldnt1_vnum_s16_x2u11__SVCount_tPKsl(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv8i16(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> poison, <vscale x 8 x i16> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], i64 8)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i16> [[TMP5]]
+//
+svint16x2_t test_svldnt1_vnum_s16_x2(svcount_t pn, const int16_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svldnt1_vnum,_s16,_x2,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svldnt1_vnum_s32_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv4i32(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> poison, <vscale x 4 x i32> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], i64 4)
+// CHECK-NEXT:    ret <vscale x 8 x i32> [[TMP5]]
+//
+// CPP-CHECK-LABEL: @_Z24test_svldnt1_vnum_s32_x2u11__SVCount_tPKil(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv4i32(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> poison, <vscale x 4 x i32> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], i64 4)
+// CPP-CHECK-NEXT:    ret <vscale x 8 x i32> [[TMP5]]
+//
+svint32x2_t test_svldnt1_vnum_s32_x2(svcount_t pn, const int32_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svldnt1_vnum,_s32,_x2,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svldnt1_vnum_s64_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv2i64(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> poison, <vscale x 2 x i64> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], i64 2)
+// CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP5]]
+//
+// CPP-CHECK-LABEL: @_Z24test_svldnt1_vnum_s64_x2u11__SVCount_tPKll(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv2i64(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> poison, <vscale x 2 x i64> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], i64 2)
+// CPP-CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP5]]
+//
+svint64x2_t test_svldnt1_vnum_s64_x2(svcount_t pn, const int64_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svldnt1_vnum,_s64,_x2,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svldnt1_vnum_s8_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv16i8(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> poison, <vscale x 16 x i8> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], i64 16)
+// CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 2
+// CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP5]], <vscale x 16 x i8> [[TMP6]], i64 32)
+// CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 3
+// CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP7]], <vscale x 16 x i8> [[TMP8]], i64 48)
+// CHECK-NEXT:    ret <vscale x 64 x i8> [[TMP9]]
+//
+// CPP-CHECK-LABEL: @_Z23test_svldnt1_vnum_s8_x4u11__SVCount_tPKal(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv16i8(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> poison, <vscale x 16 x i8> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 2
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP5]], <vscale x 16 x i8> [[TMP6]], i64 32)
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP1]], 3
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP7]], <vscale x 16 x i8> [[TMP8]], i64 48)
+// CPP-CHECK-NEXT:    ret <vscale x 64 x i8> [[TMP9]]
+//
+svint8x4_t test_svldnt1_vnum_s8_x4(svcount_t pn, const int8_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svldnt1_vnum,_s8,_x4,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svldnt1_vnum_s16_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv8i16(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> poison, <vscale x 8 x i16> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], i64 8)
+// CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 2
+// CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP5]], <vscale x 8 x i16> [[TMP6]], i64 16)
+// CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 3
+// CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP7]], <vscale x 8 x i16> [[TMP8]], i64 24)
+// CHECK-NEXT:    ret <vscale x 32 x i16> [[TMP9]]
+//
+// CPP-CHECK-LABEL: @_Z24test_svldnt1_vnum_s16_x4u11__SVCount_tPKsl(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv8i16(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> poison, <vscale x 8 x i16> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 2
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP5]], <vscale x 8 x i16> [[TMP6]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP1]], 3
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP7]], <vscale x 8 x i16> [[TMP8]], i64 24)
+// CPP-CHECK-NEXT:    ret <vscale x 32 x i16> [[TMP9]]
+//
+svint16x4_t test_svldnt1_vnum_s16_x4(svcount_t pn, const int16_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svldnt1_vnum,_s16,_x4,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svldnt1_vnum_s32_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv4i32(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], i64 4)
+// CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 2
+// CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP5]], <vscale x 4 x i32> [[TMP6]], i64 8)
+// CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 3
+// CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP7]], <vscale x 4 x i32> [[TMP8]], i64 12)
+// CHECK-NEXT:    ret <vscale x 16 x i32> [[TMP9]]
+//
+// CPP-CHECK-LABEL: @_Z24test_svldnt1_vnum_s32_x4u11__SVCount_tPKil(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv4i32(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 2
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP5]], <vscale x 4 x i32> [[TMP6]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP1]], 3
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP7]], <vscale x 4 x i32> [[TMP8]], i64 12)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i32> [[TMP9]]
+//
+svint32x4_t test_svldnt1_vnum_s32_x4(svcount_t pn, const int32_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svldnt1_vnum,_s32,_x4,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svldnt1_vnum_s64_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv2i64(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> poison, <vscale x 2 x i64> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], i64 2)
+// CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 2
+// CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP5]], <vscale x 2 x i64> [[TMP6]], i64 4)
+// CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 3
+// CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP7]], <vscale x 2 x i64> [[TMP8]], i64 6)
+// CHECK-NEXT:    ret <vscale x 8 x i64> [[TMP9]]
+//
+// CPP-CHECK-LABEL: @_Z24test_svldnt1_vnum_s64_x4u11__SVCount_tPKll(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv2i64(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> poison, <vscale x 2 x i64> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], i64 2)
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 2
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP5]], <vscale x 2 x i64> [[TMP6]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP1]], 3
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP7]], <vscale x 2 x i64> [[TMP8]], i64 6)
+// CPP-CHECK-NEXT:    ret <vscale x 8 x i64> [[TMP9]]
+//
+svint64x4_t test_svldnt1_vnum_s64_x4(svcount_t pn, const int64_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svldnt1_vnum,_s64,_x4,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svldnt1_vnum_f16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv8f16(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x half> @llvm.vector.insert.nxv16f16.nxv8f16(<vscale x 16 x half> poison, <vscale x 8 x half> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 16 x half> @llvm.vector.insert.nxv16f16.nxv8f16(<vscale x 16 x half> [[TMP3]], <vscale x 8 x half> [[TMP4]], i64 8)
+// CHECK-NEXT:    ret <vscale x 16 x half> [[TMP5]]
+//
+// CPP-CHECK-LABEL: @_Z24test_svldnt1_vnum_f16_x2u11__SVCount_tPKDhl(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv8f16(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x half> @llvm.vector.insert.nxv16f16.nxv8f16(<vscale x 16 x half> poison, <vscale x 8 x half> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 16 x half> @llvm.vector.insert.nxv16f16.nxv8f16(<vscale x 16 x half> [[TMP3]], <vscale x 8 x half> [[TMP4]], i64 8)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x half> [[TMP5]]
+//
+svfloat16x2_t test_svldnt1_vnum_f16_x2(svcount_t pn, const float16_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svldnt1_vnum,_f16,_x2,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svldnt1_vnum_f32_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv4f32(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> poison, <vscale x 4 x float> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> [[TMP3]], <vscale x 4 x float> [[TMP4]], i64 4)
+// CHECK-NEXT:    ret <vscale x 8 x float> [[TMP5]]
+//
+// CPP-CHECK-LABEL: @_Z24test_svldnt1_vnum_f32_x2u11__SVCount_tPKfl(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv4f32(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> poison, <vscale x 4 x float> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x float> @llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> [[TMP3]], <vscale x 4 x float> [[TMP4]], i64 4)
+// CPP-CHECK-NEXT:    ret <vscale x 8 x float> [[TMP5]]
+//
+svfloat32x2_t test_svldnt1_vnum_f32_x2(svcount_t pn, const float32_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svldnt1_vnum,_f32,_x2,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svldnt1_vnum_f64_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv2f64(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x double> @llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> poison, <vscale x 2 x double> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 4 x double> @llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> [[TMP3]], <vscale x 2 x double> [[TMP4]], i64 2)
+// CHECK-NEXT:    ret <vscale x 4 x double> [[TMP5]]
+//
+// CPP-CHECK-LABEL: @_Z24test_svldnt1_vnum_f64_x2u11__SVCount_tPKdl(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv2f64(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x double> @llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> poison, <vscale x 2 x double> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 4 x double> @llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> [[TMP3]], <vscale x 2 x double> [[TMP4]], i64 2)
+// CPP-CHECK-NEXT:    ret <vscale x 4 x double> [[TMP5]]
+//
+svfloat64x2_t test_svldnt1_vnum_f64_x2(svcount_t pn, const float64_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svldnt1_vnum,_f64,_x2,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svldnt1_vnum_f16_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv8f16(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> poison, <vscale x 8 x half> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP3]], <vscale x 8 x half> [[TMP4]], i64 8)
+// CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP1]], 2
+// CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP5]], <vscale x 8 x half> [[TMP6]], i64 16)
+// CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP1]], 3
+// CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP7]], <vscale x 8 x half> [[TMP8]], i64 24)
+// CHECK-NEXT:    ret <vscale x 32 x half> [[TMP9]]
+//
+// CPP-CHECK-LABEL: @_Z24test_svldnt1_vnum_f16_x4u11__SVCount_tPKDhl(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv8f16(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> poison, <vscale x 8 x half> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP3]], <vscale x 8 x half> [[TMP4]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP1]], 2
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP5]], <vscale x 8 x half> [[TMP6]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP1]], 3
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 32 x half> @llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP7]], <vscale x 8 x half> [[TMP8]], i64 24)
+// CPP-CHECK-NEXT:    ret <vscale x 32 x half> [[TMP9]]
+//
+svfloat16x4_t test_svldnt1_vnum_f16_x4(svcount_t pn, const float16_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svldnt1_vnum,_f16,_x4,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svldnt1_vnum_f32_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv4f32(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> poison, <vscale x 4 x float> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP3]], <vscale x 4 x float> [[TMP4]], i64 4)
+// CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP1]], 2
+// CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP5]], <vscale x 4 x float> [[TMP6]], i64 8)
+// CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP1]], 3
+// CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP7]], <vscale x 4 x float> [[TMP8]], i64 12)
+// CHECK-NEXT:    ret <vscale x 16 x float> [[TMP9]]
+//
+// CPP-CHECK-LABEL: @_Z24test_svldnt1_vnum_f32_x4u11__SVCount_tPKfl(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv4f32(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> poison, <vscale x 4 x float> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP3]], <vscale x 4 x float> [[TMP4]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP1]], 2
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP5]], <vscale x 4 x float> [[TMP6]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP1]], 3
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP7]], <vscale x 4 x float> [[TMP8]], i64 12)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x float> [[TMP9]]
+//
+svfloat32x4_t test_svldnt1_vnum_f32_x4(svcount_t pn, const float32_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svldnt1_vnum,_f32,_x4,)(pn, base, vnum);
+}
+
+// CHECK-LABEL: @test_svldnt1_vnum_f64_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv2f64(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP1]], 0
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> poison, <vscale x 2 x double> [[TMP2]], i64 0)
+// CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP1]], 1
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP3]], <vscale x 2 x double> [[TMP4]], i64 2)
+// CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP1]], 2
+// CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP5]], <vscale x 2 x double> [[TMP6]], i64 4)
+// CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP1]], 3
+// CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP7]], <vscale x 2 x double> [[TMP8]], i64 6)
+// CHECK-NEXT:    ret <vscale x 8 x double> [[TMP9]]
+//
+// CPP-CHECK-LABEL: @_Z24test_svldnt1_vnum_f64_x4u11__SVCount_tPKdl(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv2f64(target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP1]], 0
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> poison, <vscale x 2 x double> [[TMP2]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP1]], 1
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP3]], <vscale x 2 x double> [[TMP4]], i64 2)
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP1]], 2
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP5]], <vscale x 2 x double> [[TMP6]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP1]], 3
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = tail call <vscale x 8 x double> @llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP7]], <vscale x 2 x double> [[TMP8]], i64 6)
+// CPP-CHECK-NEXT:    ret <vscale x 8 x double> [[TMP9]]
+//
+svfloat64x4_t test_svldnt1_vnum_f64_x4(svcount_t pn, const float64_t *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svldnt1_vnum,_f64,_x4,)(pn, base, vnum);
+}
diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pext.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pext.c
new file mode 100644
index 0000000000..fe15d5a9db
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pext.c
@@ -0,0 +1,152 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK
+
+#include <arm_sve.h>
+
+// CHECK-LABEL: @test_svpext_lane_c8_0(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.pext.nxv16i1(target("aarch64.svcount") [[C:%.*]], i32 0)
+// CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z21test_svpext_lane_c8_0u11__SVCount_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.pext.nxv16i1(target("aarch64.svcount") [[C:%.*]], i32 0)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP0]]
+//
+svbool_t test_svpext_lane_c8_0(svcount_t c) {
+  return svpext_lane_c8(c, 0);
+}
+
+// CHECK-LABEL: @test_svpext_lane_c8_3(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.pext.nxv16i1(target("aarch64.svcount") [[C:%.*]], i32 3)
+// CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z21test_svpext_lane_c8_3u11__SVCount_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.pext.nxv16i1(target("aarch64.svcount") [[C:%.*]], i32 3)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP0]]
+//
+svbool_t test_svpext_lane_c8_3(svcount_t c) {
+  return svpext_lane_c8(c, 3);
+}
+
+// CHECK-LABEL: @test_svpext_lane_c16_0(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.pext.nxv8i1(target("aarch64.svcount") [[C:%.*]], i32 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv8i1(<vscale x 8 x i1> [[TMP0]])
+// CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svpext_lane_c16_0u11__SVCount_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.pext.nxv8i1(target("aarch64.svcount") [[C:%.*]], i32 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv8i1(<vscale x 8 x i1> [[TMP0]])
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP1]]
+//
+svbool_t test_svpext_lane_c16_0(svcount_t c) {
+  return svpext_lane_c16(c, 0);
+}
+
+// CHECK-LABEL: @test_svpext_lane_c16_3(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.pext.nxv8i1(target("aarch64.svcount") [[C:%.*]], i32 3)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv8i1(<vscale x 8 x i1> [[TMP0]])
+// CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svpext_lane_c16_3u11__SVCount_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.pext.nxv8i1(target("aarch64.svcount") [[C:%.*]], i32 3)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv8i1(<vscale x 8 x i1> [[TMP0]])
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP1]]
+//
+svbool_t test_svpext_lane_c16_3(svcount_t c) {
+  return svpext_lane_c16(c, 3);
+}
+
+// CHECK-LABEL: @test_svpext_lane_c32_0(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.pext.nxv4i1(target("aarch64.svcount") [[C:%.*]], i32 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv4i1(<vscale x 4 x i1> [[TMP0]])
+// CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svpext_lane_c32_0u11__SVCount_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.pext.nxv4i1(target("aarch64.svcount") [[C:%.*]], i32 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv4i1(<vscale x 4 x i1> [[TMP0]])
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP1]]
+//
+svbool_t test_svpext_lane_c32_0(svcount_t c) {
+  return svpext_lane_c32(c, 0);
+}
+
+// CHECK-LABEL: @test_svpext_lane_c32_3(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.pext.nxv4i1(target("aarch64.svcount") [[C:%.*]], i32 3)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv4i1(<vscale x 4 x i1> [[TMP0]])
+// CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svpext_lane_c32_3u11__SVCount_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.pext.nxv4i1(target("aarch64.svcount") [[C:%.*]], i32 3)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv4i1(<vscale x 4 x i1> [[TMP0]])
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP1]]
+//
+svbool_t test_svpext_lane_c32_3(svcount_t c) {
+  return svpext_lane_c32(c, 3);
+}
+
+// CHECK-LABEL: @test_svpext_lane_c64_0(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.pext.nxv2i1(target("aarch64.svcount") [[C:%.*]], i32 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv2i1(<vscale x 2 x i1> [[TMP0]])
+// CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svpext_lane_c64_0u11__SVCount_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.pext.nxv2i1(target("aarch64.svcount") [[C:%.*]], i32 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv2i1(<vscale x 2 x i1> [[TMP0]])
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP1]]
+//
+svbool_t test_svpext_lane_c64_0(svcount_t c) {
+  return svpext_lane_c64(c, 0);
+}
+
+// CHECK-LABEL: @test_svpext_lane_c64_3(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.pext.nxv2i1(target("aarch64.svcount") [[C:%.*]], i32 3)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv2i1(<vscale x 2 x i1> [[TMP0]])
+// CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svpext_lane_c64_3u11__SVCount_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.pext.nxv2i1(target("aarch64.svcount") [[C:%.*]], i32 3)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv2i1(<vscale x 2 x i1> [[TMP0]])
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP1]]
+//
+svbool_t test_svpext_lane_c64_3(svcount_t c) {
+  return svpext_lane_c64(c, 3);
+}
+
+// CHECK-LABEL: @test_svpext_lane_c8_x2_0(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i1>, <vscale x 16 x i1> } @llvm.aarch64.sve.pext.x2.nxv16i1(target("aarch64.svcount") [[C:%.*]], i32 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 16 x i1>, <vscale x 16 x i1> } [[TMP0]], 0
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 32 x i1> @llvm.vector.insert.nxv32i1.nxv16i1(<vscale x 32 x i1> poison, <vscale x 16 x i1> [[TMP1]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 16 x i1>, <vscale x 16 x i1> } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 32 x i1> @llvm.vector.insert.nxv32i1.nxv16i1(<vscale x 32 x i1> [[TMP2]], <vscale x 16 x i1> [[TMP3]], i64 16)
+// CHECK-NEXT:    ret <vscale x 32 x i1> [[TMP4]]
+//
+// CPP-CHECK-LABEL: @_Z24test_svpext_lane_c8_x2_0u11__SVCount_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call { <vscale x 16 x i1>, <vscale x 16 x i1> } @llvm.aarch64.sve.pext.x2.nxv16i1(target("aarch64.svcount") [[C:%.*]], i32 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 16 x i1>, <vscale x 16 x i1> } [[TMP0]], 0
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 32 x i1> @llvm.vector.insert.nxv32i1.nxv16i1(<vscale x 32 x i1> poison, <vscale x 16 x i1> [[TMP1]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 16 x i1>, <vscale x 16 x i1> } [[TMP0]], 1
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 32 x i1> @llvm.vector.insert.nxv32i1.nxv16i1(<vscale x 32 x i1> [[TMP2]], <vscale x 16 x i1> [[TMP3]], i64 16)
+// CPP-CHECK-NEXT:    ret <vscale x 32 x i1> [[TMP4]]
+//
+svboolx2_t test_svpext_lane_c8_x2_0(svcount_t c) {
+  return svpext_lane_c8_x2(c, 0);
+}
diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pfalse.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pfalse.c
new file mode 100644
index 0000000000..5432862dcf
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pfalse.c
@@ -0,0 +1,30 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+// CHECK-LABEL: @test_svpfalse_c(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt(<vscale x 16 x i1> zeroinitializer)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z15test_svpfalse_cv(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt(<vscale x 16 x i1> zeroinitializer)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svpfalse_c()
+{
+  return SVE_ACLE_FUNC(svpfalse_c,,,)();
+}
diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_psel.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_psel.c
new file mode 100644
index 0000000000..97354d75d7
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_psel.c
@@ -0,0 +1,165 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu \
+// RUN:   -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu \
+// RUN:   -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
+
+#include <arm_sve.h>
+
+// CHECK-LABEL: @test_svpsel_lane_b8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[ADD:%.*]] = add i32 [[IDX:%.*]], 15
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.psel.nxv16i1(<vscale x 16 x i1> [[P1:%.*]], <vscale x 16 x i1> [[P2:%.*]], i32 [[ADD]])
+// CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z19test_svpsel_lane_b8u10__SVBool_tu10__SVBool_tj(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[ADD:%.*]] = add i32 [[IDX:%.*]], 15
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.psel.nxv16i1(<vscale x 16 x i1> [[P1:%.*]], <vscale x 16 x i1> [[P2:%.*]], i32 [[ADD]])
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP0]]
+//
+svbool_t test_svpsel_lane_b8(svbool_t p1, svbool_t p2, uint32_t idx) {
+  return svpsel_lane_b8(p1, p2, idx + 15);
+}
+
+// CHECK-LABEL: @test_svpsel_lane_b16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[ADD:%.*]] = add i32 [[IDX:%.*]], 7
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[P2:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.psel.nxv8i1(<vscale x 16 x i1> [[P1:%.*]], <vscale x 8 x i1> [[TMP0]], i32 [[ADD]])
+// CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z20test_svpsel_lane_b16u10__SVBool_tu10__SVBool_tj(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[ADD:%.*]] = add i32 [[IDX:%.*]], 7
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[P2:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.psel.nxv8i1(<vscale x 16 x i1> [[P1:%.*]], <vscale x 8 x i1> [[TMP0]], i32 [[ADD]])
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP1]]
+//
+svbool_t test_svpsel_lane_b16(svbool_t p1, svbool_t p2, uint32_t idx) {
+  return svpsel_lane_b16(p1, p2, idx + 7);
+}
+
+// CHECK-LABEL: @test_svpsel_lane_b32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[ADD:%.*]] = add i32 [[IDX:%.*]], 3
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[P2:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.psel.nxv4i1(<vscale x 16 x i1> [[P1:%.*]], <vscale x 4 x i1> [[TMP0]], i32 [[ADD]])
+// CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z20test_svpsel_lane_b32u10__SVBool_tu10__SVBool_tj(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[ADD:%.*]] = add i32 [[IDX:%.*]], 3
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[P2:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.psel.nxv4i1(<vscale x 16 x i1> [[P1:%.*]], <vscale x 4 x i1> [[TMP0]], i32 [[ADD]])
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP1]]
+//
+svbool_t test_svpsel_lane_b32(svbool_t p1, svbool_t p2, uint32_t idx) {
+  return svpsel_lane_b32(p1, p2, idx + 3);
+}
+
+// CHECK-LABEL: @test_svpsel_lane_b64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[ADD:%.*]] = add i32 [[IDX:%.*]], 1
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[P2:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.psel.nxv2i1(<vscale x 16 x i1> [[P1:%.*]], <vscale x 2 x i1> [[TMP0]], i32 [[ADD]])
+// CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z20test_svpsel_lane_b64u10__SVBool_tu10__SVBool_tj(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[ADD:%.*]] = add i32 [[IDX:%.*]], 1
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[P2:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.psel.nxv2i1(<vscale x 16 x i1> [[P1:%.*]], <vscale x 2 x i1> [[TMP0]], i32 [[ADD]])
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i1> [[TMP1]]
+//
+svbool_t test_svpsel_lane_b64(svbool_t p1, svbool_t p2, uint32_t idx) {
+  return svpsel_lane_b64(p1, p2, idx + 1);
+}
+
+// CHECK-LABEL: @test_svpsel_lane_c8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[ADD:%.*]] = add i32 [[IDX:%.*]], 15
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.taarch64.svcountt(target("aarch64.svcount") [[P1:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.psel.nxv16i1(<vscale x 16 x i1> [[TMP0]], <vscale x 16 x i1> [[P2:%.*]], i32 [[ADD]])
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt(<vscale x 16 x i1> [[TMP1]])
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP2]]
+//
+// CPP-CHECK-LABEL: @_Z19test_svpsel_lane_c8u11__SVCount_tu10__SVBool_tj(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[ADD:%.*]] = add i32 [[IDX:%.*]], 15
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.taarch64.svcountt(target("aarch64.svcount") [[P1:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.psel.nxv16i1(<vscale x 16 x i1> [[TMP0]], <vscale x 16 x i1> [[P2:%.*]], i32 [[ADD]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt(<vscale x 16 x i1> [[TMP1]])
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP2]]
+//
+svcount_t test_svpsel_lane_c8(svcount_t p1, svbool_t p2, uint32_t idx) {
+  return svpsel_lane_c8(p1, p2, idx + 15);
+}
+
+// CHECK-LABEL: @test_svpsel_lane_c16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[ADD:%.*]] = add i32 [[IDX:%.*]], 7
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.taarch64.svcountt(target("aarch64.svcount") [[P1:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[P2:%.*]])
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.psel.nxv8i1(<vscale x 16 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], i32 [[ADD]])
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt(<vscale x 16 x i1> [[TMP2]])
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP3]]
+//
+// CPP-CHECK-LABEL: @_Z20test_svpsel_lane_c16u11__SVCount_tu10__SVBool_tj(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[ADD:%.*]] = add i32 [[IDX:%.*]], 7
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.taarch64.svcountt(target("aarch64.svcount") [[P1:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[P2:%.*]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.psel.nxv8i1(<vscale x 16 x i1> [[TMP0]], <vscale x 8 x i1> [[TMP1]], i32 [[ADD]])
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt(<vscale x 16 x i1> [[TMP2]])
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP3]]
+//
+svcount_t test_svpsel_lane_c16(svcount_t p1, svbool_t p2, uint32_t idx) {
+  return svpsel_lane_c16(p1, p2, idx + 7);
+}
+
+// CHECK-LABEL: @test_svpsel_lane_c32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[ADD:%.*]] = add i32 [[IDX:%.*]], 3
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.taarch64.svcountt(target("aarch64.svcount") [[P1:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[P2:%.*]])
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.psel.nxv4i1(<vscale x 16 x i1> [[TMP0]], <vscale x 4 x i1> [[TMP1]], i32 [[ADD]])
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt(<vscale x 16 x i1> [[TMP2]])
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP3]]
+//
+// CPP-CHECK-LABEL: @_Z20test_svpsel_lane_c32u11__SVCount_tu10__SVBool_tj(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[ADD:%.*]] = add i32 [[IDX:%.*]], 3
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.taarch64.svcountt(target("aarch64.svcount") [[P1:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[P2:%.*]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.psel.nxv4i1(<vscale x 16 x i1> [[TMP0]], <vscale x 4 x i1> [[TMP1]], i32 [[ADD]])
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt(<vscale x 16 x i1> [[TMP2]])
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP3]]
+//
+svcount_t test_svpsel_lane_c32(svcount_t p1, svbool_t p2, uint32_t idx) {
+  return svpsel_lane_c32(p1, p2, idx + 3);
+}
+
+// CHECK-LABEL: @test_svpsel_lane_c64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[ADD:%.*]] = add i32 [[IDX:%.*]], 1
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.taarch64.svcountt(target("aarch64.svcount") [[P1:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[P2:%.*]])
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.psel.nxv2i1(<vscale x 16 x i1> [[TMP0]], <vscale x 2 x i1> [[TMP1]], i32 [[ADD]])
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt(<vscale x 16 x i1> [[TMP2]])
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP3]]
+//
+// CPP-CHECK-LABEL: @_Z20test_svpsel_lane_c64u11__SVCount_tu10__SVBool_tj(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[ADD:%.*]] = add i32 [[IDX:%.*]], 1
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.taarch64.svcountt(target("aarch64.svcount") [[P1:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[P2:%.*]])
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.psel.nxv2i1(<vscale x 16 x i1> [[TMP0]], <vscale x 2 x i1> [[TMP1]], i32 [[ADD]])
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt(<vscale x 16 x i1> [[TMP2]])
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP3]]
+//
+svcount_t test_svpsel_lane_c64(svcount_t p1, svbool_t p2, uint32_t idx) {
+  return svpsel_lane_c64(p1, p2, idx + 1);
+}
diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_st1.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_st1.c
new file mode 100644
index 0000000000..3adf5f2a1a
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_st1.c
@@ -0,0 +1,998 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+// CHECK-LABEL: @test_svst1_u8_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[V]], i64 16)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z16test_svst1_u8_x2u11__SVCount_tPh11svuint8x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[V]], i64 16)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_u8_x2(svcount_t pn, uint8_t *base, svuint8x2_t v)
+{
+  return SVE_ACLE_FUNC(svst1,_u8_x2,,)(pn, base, v);
+}
+
+// CHECK-LABEL: @test_svst1_u16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[V]], i64 8)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z17test_svst1_u16_x2u11__SVCount_tPt12svuint16x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[V]], i64 8)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_u16_x2(svcount_t pn, uint16_t *base, svuint16x2_t v)
+{
+  return SVE_ACLE_FUNC(svst1,_u16_x2,,)(pn, base, v);
+}
+
+// CHECK-LABEL: @test_svst1_u32_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[V]], i64 4)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z17test_svst1_u32_x2u11__SVCount_tPj12svuint32x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[V]], i64 4)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_u32_x2(svcount_t pn, uint32_t *base, svuint32x2_t v)
+{
+  return SVE_ACLE_FUNC(svst1,_u32_x2,,)(pn, base, v);
+}
+
+// CHECK-LABEL: @test_svst1_u64_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[V]], i64 2)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z17test_svst1_u64_x2u11__SVCount_tPm12svuint64x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[V]], i64 2)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_u64_x2(svcount_t pn, uint64_t *base, svuint64x2_t v)
+{
+  return SVE_ACLE_FUNC(svst1,_u64_x2,,)(pn, base, v);
+}
+
+// CHECK-LABEL: @test_svst1_u8_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 16)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 32)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 48)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z16test_svst1_u8_x4u11__SVCount_tPh11svuint8x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 32)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 48)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_u8_x4(svcount_t pn, uint8_t *base, svuint8x4_t v)
+{
+  return SVE_ACLE_FUNC(svst1,_u8_x4,,)(pn, base, v);
+}
+
+// CHECK-LABEL: @test_svst1_u16_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 24)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z17test_svst1_u16_x4u11__SVCount_tPt12svuint16x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 24)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_u16_x4(svcount_t pn, uint16_t *base, svuint16x4_t v)
+{
+  return SVE_ACLE_FUNC(svst1,_u16_x4,,)(pn, base, v);
+}
+
+// CHECK-LABEL: @test_svst1_u32_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 4)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 8)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 12)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z17test_svst1_u32_x4u11__SVCount_tPj12svuint32x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 12)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_u32_x4(svcount_t pn, uint32_t *base, svuint32x4_t v)
+{
+  return SVE_ACLE_FUNC(svst1,_u32_x4,,)(pn, base, v);
+}
+
+// CHECK-LABEL: @test_svst1_u64_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 2)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 4)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 6)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z17test_svst1_u64_x4u11__SVCount_tPm12svuint64x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 2)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 6)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_u64_x4(svcount_t pn, uint64_t *base, svuint64x4_t v)
+{
+  return SVE_ACLE_FUNC(svst1,_u64_x4,,)(pn, base, v);
+}
+
+// CHECK-LABEL: @test_svst1_s8_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[V]], i64 16)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z16test_svst1_s8_x2u11__SVCount_tPa10svint8x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[V]], i64 16)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_s8_x2(svcount_t pn, int8_t *base, svint8x2_t v)
+{
+  return SVE_ACLE_FUNC(svst1,_s8_x2,,)(pn, base, v);
+}
+
+// CHECK-LABEL: @test_svst1_s16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[V]], i64 8)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z17test_svst1_s16_x2u11__SVCount_tPs11svint16x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[V]], i64 8)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_s16_x2(svcount_t pn, int16_t *base, svint16x2_t v)
+{
+  return SVE_ACLE_FUNC(svst1,_s16_x2,,)(pn, base, v);
+}
+
+// CHECK-LABEL: @test_svst1_s32_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[V]], i64 4)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z17test_svst1_s32_x2u11__SVCount_tPi11svint32x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[V]], i64 4)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_s32_x2(svcount_t pn, int32_t *base, svint32x2_t v)
+{
+  return SVE_ACLE_FUNC(svst1,_s32_x2,,)(pn, base, v);
+}
+
+// CHECK-LABEL: @test_svst1_s64_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[V]], i64 2)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z17test_svst1_s64_x2u11__SVCount_tPl11svint64x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[V]], i64 2)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_s64_x2(svcount_t pn, int64_t *base, svint64x2_t v)
+{
+  return SVE_ACLE_FUNC(svst1,_s64_x2,,)(pn, base, v);
+}
+
+// CHECK-LABEL: @test_svst1_s8_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 16)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 32)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 48)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z16test_svst1_s8_x4u11__SVCount_tPa10svint8x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 32)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 48)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_s8_x4(svcount_t pn, int8_t *base, svint8x4_t v)
+{
+  return SVE_ACLE_FUNC(svst1,_s8_x4,,)(pn, base, v);
+}
+
+// CHECK-LABEL: @test_svst1_s16_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 24)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z17test_svst1_s16_x4u11__SVCount_tPs11svint16x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 24)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_s16_x4(svcount_t pn, int16_t *base, svint16x4_t v)
+{
+  return SVE_ACLE_FUNC(svst1,_s16_x4,,)(pn, base, v);
+}
+
+// CHECK-LABEL: @test_svst1_s32_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 4)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 8)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 12)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z17test_svst1_s32_x4u11__SVCount_tPi11svint32x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 12)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_s32_x4(svcount_t pn, int32_t *base, svint32x4_t v)
+{
+  return SVE_ACLE_FUNC(svst1,_s32_x4,,)(pn, base, v);
+}
+
+// CHECK-LABEL: @test_svst1_s64_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 2)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 4)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 6)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z17test_svst1_s64_x4u11__SVCount_tPl11svint64x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 2)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 6)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_s64_x4(svcount_t pn, int64_t *base, svint64x4_t v)
+{
+  return SVE_ACLE_FUNC(svst1,_s64_x4,,)(pn, base, v);
+}
+
+// CHECK-LABEL: @test_svst1_f16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[V]], i64 8)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv8f16(<vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z17test_svst1_f16_x2u11__SVCount_tPDh13svfloat16x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[V]], i64 8)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv8f16(<vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_f16_x2(svcount_t pn, float16_t *base, svfloat16x2_t v)
+{
+  return SVE_ACLE_FUNC(svst1,_f16_x2,,)(pn, base, v);
+}
+
+// CHECK-LABEL: @test_svst1_f32_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[V]], i64 4)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv4f32(<vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z17test_svst1_f32_x2u11__SVCount_tPf13svfloat32x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[V]], i64 4)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv4f32(<vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_f32_x2(svcount_t pn, float32_t *base, svfloat32x2_t v)
+{
+  return SVE_ACLE_FUNC(svst1,_f32_x2,,)(pn, base, v);
+}
+
+// CHECK-LABEL: @test_svst1_f64_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[V]], i64 2)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv2f64(<vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z17test_svst1_f64_x2u11__SVCount_tPd13svfloat64x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[V]], i64 2)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv2f64(<vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_f64_x2(svcount_t pn, float64_t *base, svfloat64x2_t v)
+{
+  return SVE_ACLE_FUNC(svst1,_f64_x2,,)(pn, base, v);
+}
+
+// CHECK-LABEL: @test_svst1_f16_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[V]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[V]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[V]], i64 24)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv8f16(<vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z17test_svst1_f16_x4u11__SVCount_tPDh13svfloat16x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[V]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[V]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[V]], i64 24)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv8f16(<vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_f16_x4(svcount_t pn, float16_t *base, svfloat16x4_t v)
+{
+  return SVE_ACLE_FUNC(svst1,_f16_x4,,)(pn, base, v);
+}
+
+// CHECK-LABEL: @test_svst1_f32_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[V]], i64 4)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[V]], i64 8)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[V]], i64 12)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv4f32(<vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z17test_svst1_f32_x4u11__SVCount_tPf13svfloat32x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[V]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[V]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[V]], i64 12)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv4f32(<vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_f32_x4(svcount_t pn, float32_t *base, svfloat32x4_t v)
+{
+  return SVE_ACLE_FUNC(svst1,_f32_x4,,)(pn, base, v);
+}
+
+// CHECK-LABEL: @test_svst1_f64_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[V]], i64 2)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[V]], i64 4)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[V]], i64 6)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv2f64(<vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z17test_svst1_f64_x4u11__SVCount_tPd13svfloat64x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[V]], i64 2)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[V]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[V]], i64 6)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv2f64(<vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_f64_x4(svcount_t pn, float64_t *base, svfloat64x4_t v)
+{
+  return SVE_ACLE_FUNC(svst1,_f64_x4,,)(pn, base, v);
+}
+
+
+// == VNUM variants ==
+
+
+// CHECK-LABEL: @test_svst1_vnum_u8_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[V]], i64 16)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv16i8(<vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z21test_svst1_vnum_u8_x2u11__SVCount_tPhl11svuint8x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[V]], i64 16)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv16i8(<vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_vnum_u8_x2(svcount_t pn, uint8_t *base, int64_t vnum, svuint8x2_t v)
+{
+  return SVE_ACLE_FUNC(svst1_vnum,_u8_x2,,)(pn, base, vnum, v);
+}
+
+// CHECK-LABEL: @test_svst1_vnum_u16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[V]], i64 8)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv8i16(<vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z22test_svst1_vnum_u16_x2u11__SVCount_tPtl12svuint16x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[V]], i64 8)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv8i16(<vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_vnum_u16_x2(svcount_t pn, uint16_t *base, int64_t vnum, svuint16x2_t v)
+{
+  return SVE_ACLE_FUNC(svst1_vnum,_u16_x2,,)(pn, base, vnum, v);
+}
+
+// CHECK-LABEL: @test_svst1_vnum_u32_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[V]], i64 4)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv4i32(<vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z22test_svst1_vnum_u32_x2u11__SVCount_tPjl12svuint32x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[V]], i64 4)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv4i32(<vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_vnum_u32_x2(svcount_t pn, uint32_t *base, int64_t vnum, svuint32x2_t v)
+{
+  return SVE_ACLE_FUNC(svst1_vnum,_u32_x2,,)(pn, base, vnum, v);
+}
+
+// CHECK-LABEL: @test_svst1_vnum_u64_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[V]], i64 2)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv2i64(<vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z22test_svst1_vnum_u64_x2u11__SVCount_tPml12svuint64x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[V]], i64 2)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv2i64(<vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_vnum_u64_x2(svcount_t pn, uint64_t *base, int64_t vnum, svuint64x2_t v)
+{
+  return SVE_ACLE_FUNC(svst1_vnum,_u64_x2,,)(pn, base, vnum, v);
+}
+
+// CHECK-LABEL: @test_svst1_vnum_u8_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 32)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 48)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv16i8(<vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z21test_svst1_vnum_u8_x4u11__SVCount_tPhl11svuint8x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 32)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 48)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv16i8(<vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_vnum_u8_x4(svcount_t pn, uint8_t *base, int64_t vnum, svuint8x4_t v)
+{
+  return SVE_ACLE_FUNC(svst1_vnum,_u8_x4,,)(pn, base, vnum, v);
+}
+
+// CHECK-LABEL: @test_svst1_vnum_u16_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 8)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 16)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 24)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv8i16(<vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z22test_svst1_vnum_u16_x4u11__SVCount_tPtl12svuint16x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 24)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv8i16(<vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_vnum_u16_x4(svcount_t pn, uint16_t *base, int64_t vnum, svuint16x4_t v)
+{
+  return SVE_ACLE_FUNC(svst1_vnum,_u16_x4,,)(pn, base, vnum, v);
+}
+
+// CHECK-LABEL: @test_svst1_vnum_u32_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 4)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 8)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 12)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv4i32(<vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z22test_svst1_vnum_u32_x4u11__SVCount_tPjl12svuint32x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 12)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv4i32(<vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_vnum_u32_x4(svcount_t pn, uint32_t *base, int64_t vnum, svuint32x4_t v)
+{
+  return SVE_ACLE_FUNC(svst1_vnum,_u32_x4,,)(pn, base, vnum, v);
+}
+
+// CHECK-LABEL: @test_svst1_vnum_u64_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 2)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 4)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 6)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv2i64(<vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z22test_svst1_vnum_u64_x4u11__SVCount_tPml12svuint64x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 2)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 6)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv2i64(<vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_vnum_u64_x4(svcount_t pn, uint64_t *base, int64_t vnum, svuint64x4_t v)
+{
+  return SVE_ACLE_FUNC(svst1_vnum,_u64_x4,,)(pn, base, vnum, v);
+}
+
+// CHECK-LABEL: @test_svst1_vnum_s8_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[V]], i64 16)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv16i8(<vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z21test_svst1_vnum_s8_x2u11__SVCount_tPal10svint8x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[V]], i64 16)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv16i8(<vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_vnum_s8_x2(svcount_t pn, int8_t *base, int64_t vnum, svint8x2_t v)
+{
+  return SVE_ACLE_FUNC(svst1_vnum,_s8_x2,,)(pn, base, vnum, v);
+}
+
+// CHECK-LABEL: @test_svst1_vnum_s16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[V]], i64 8)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv8i16(<vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z22test_svst1_vnum_s16_x2u11__SVCount_tPsl11svint16x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[V]], i64 8)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv8i16(<vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_vnum_s16_x2(svcount_t pn, int16_t *base, int64_t vnum, svint16x2_t v)
+{
+  return SVE_ACLE_FUNC(svst1_vnum,_s16_x2,,)(pn, base, vnum, v);
+}
+
+// CHECK-LABEL: @test_svst1_vnum_s32_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[V]], i64 4)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv4i32(<vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z22test_svst1_vnum_s32_x2u11__SVCount_tPil11svint32x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[V]], i64 4)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv4i32(<vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_vnum_s32_x2(svcount_t pn, int32_t *base, int64_t vnum, svint32x2_t v)
+{
+  return SVE_ACLE_FUNC(svst1_vnum,_s32_x2,,)(pn, base, vnum, v);
+}
+
+// CHECK-LABEL: @test_svst1_vnum_s64_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[V]], i64 2)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv2i64(<vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z22test_svst1_vnum_s64_x2u11__SVCount_tPll11svint64x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[V]], i64 2)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv2i64(<vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_vnum_s64_x2(svcount_t pn, int64_t *base, int64_t vnum, svint64x2_t v)
+{
+  return SVE_ACLE_FUNC(svst1_vnum,_s64_x2,,)(pn, base, vnum, v);
+}
+
+// CHECK-LABEL: @test_svst1_vnum_s8_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 32)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 48)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv16i8(<vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z21test_svst1_vnum_s8_x4u11__SVCount_tPal10svint8x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 32)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 48)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv16i8(<vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_vnum_s8_x4(svcount_t pn, int8_t *base, int64_t vnum, svint8x4_t v)
+{
+  return SVE_ACLE_FUNC(svst1_vnum,_s8_x4,,)(pn, base, vnum, v);
+}
+
+// CHECK-LABEL: @test_svst1_vnum_s16_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 8)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 16)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 24)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv8i16(<vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z22test_svst1_vnum_s16_x4u11__SVCount_tPsl11svint16x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 24)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv8i16(<vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_vnum_s16_x4(svcount_t pn, int16_t *base, int64_t vnum, svint16x4_t v)
+{
+  return SVE_ACLE_FUNC(svst1_vnum,_s16_x4,,)(pn, base, vnum, v);
+}
+
+// CHECK-LABEL: @test_svst1_vnum_s32_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 4)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 8)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 12)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv4i32(<vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z22test_svst1_vnum_s32_x4u11__SVCount_tPil11svint32x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 12)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv4i32(<vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_vnum_s32_x4(svcount_t pn, int32_t *base, int64_t vnum, svint32x4_t v)
+{
+  return SVE_ACLE_FUNC(svst1_vnum,_s32_x4,,)(pn, base, vnum, v);
+}
+
+// CHECK-LABEL: @test_svst1_vnum_s64_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 2)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 4)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 6)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv2i64(<vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z22test_svst1_vnum_s64_x4u11__SVCount_tPll11svint64x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 2)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 6)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv2i64(<vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_vnum_s64_x4(svcount_t pn, int64_t *base, int64_t vnum, svint64x4_t v)
+{
+  return SVE_ACLE_FUNC(svst1_vnum,_s64_x4,,)(pn, base, vnum, v);
+}
+
+// CHECK-LABEL: @test_svst1_vnum_f16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[CONV:%.*]] = fptosi double [[VNUM:%.*]] to i64
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[CONV]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[V]], i64 8)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv8f16(<vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z22test_svst1_vnum_f16_x2u11__SVCount_tPDhd13svfloat16x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[CONV:%.*]] = fptosi double [[VNUM:%.*]] to i64
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[CONV]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[V]], i64 8)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv8f16(<vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_vnum_f16_x2(svcount_t pn, float16_t *base, float64_t vnum, svfloat16x2_t v)
+{
+  return SVE_ACLE_FUNC(svst1_vnum,_f16_x2,,)(pn, base, vnum, v);
+}
+
+// CHECK-LABEL: @test_svst1_vnum_f32_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[CONV:%.*]] = fptosi double [[VNUM:%.*]] to i64
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[CONV]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[V]], i64 4)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv4f32(<vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z22test_svst1_vnum_f32_x2u11__SVCount_tPfd13svfloat32x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[CONV:%.*]] = fptosi double [[VNUM:%.*]] to i64
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[CONV]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[V]], i64 4)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv4f32(<vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_vnum_f32_x2(svcount_t pn, float32_t *base, float64_t vnum, svfloat32x2_t v)
+{
+  return SVE_ACLE_FUNC(svst1_vnum,_f32_x2,,)(pn, base, vnum, v);
+}
+
+// CHECK-LABEL: @test_svst1_vnum_f64_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[CONV:%.*]] = fptosi double [[VNUM:%.*]] to i64
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[CONV]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[V]], i64 2)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv2f64(<vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z22test_svst1_vnum_f64_x2u11__SVCount_tPdd13svfloat64x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[CONV:%.*]] = fptosi double [[VNUM:%.*]] to i64
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[CONV]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[V]], i64 2)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x2.nxv2f64(<vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_vnum_f64_x2(svcount_t pn, float64_t *base, float64_t vnum, svfloat64x2_t v)
+{
+  return SVE_ACLE_FUNC(svst1_vnum,_f64_x2,,)(pn, base, vnum, v);
+}
+
+// CHECK-LABEL: @test_svst1_vnum_f16_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[CONV:%.*]] = fptosi double [[VNUM:%.*]] to i64
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[CONV]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[V]], i64 8)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[V]], i64 16)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[V]], i64 24)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv8f16(<vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z22test_svst1_vnum_f16_x4u11__SVCount_tPDhd13svfloat16x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[CONV:%.*]] = fptosi double [[VNUM:%.*]] to i64
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[CONV]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[V]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[V]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[V]], i64 24)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv8f16(<vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_vnum_f16_x4(svcount_t pn, float16_t *base, float64_t vnum, svfloat16x4_t v)
+{
+  return SVE_ACLE_FUNC(svst1_vnum,_f16_x4,,)(pn, base, vnum, v);
+}
+
+// CHECK-LABEL: @test_svst1_vnum_f32_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[CONV:%.*]] = fptosi double [[VNUM:%.*]] to i64
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[CONV]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[V]], i64 4)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[V]], i64 8)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[V]], i64 12)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv4f32(<vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z22test_svst1_vnum_f32_x4u11__SVCount_tPfd13svfloat32x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[CONV:%.*]] = fptosi double [[VNUM:%.*]] to i64
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[CONV]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[V]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[V]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[V]], i64 12)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv4f32(<vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_vnum_f32_x4(svcount_t pn, float32_t *base, float64_t vnum, svfloat32x4_t v)
+{
+  return SVE_ACLE_FUNC(svst1_vnum,_f32_x4,,)(pn, base, vnum, v);
+}
+
+// CHECK-LABEL: @test_svst1_vnum_f64_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[CONV:%.*]] = fptosi double [[VNUM:%.*]] to i64
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[CONV]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[V]], i64 2)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[V]], i64 4)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[V]], i64 6)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv2f64(<vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], <vscale x 2 x double> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z22test_svst1_vnum_f64_x4u11__SVCount_tPdd13svfloat64x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[CONV:%.*]] = fptosi double [[VNUM:%.*]] to i64
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[CONV]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[V]], i64 2)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[V]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[V]], i64 6)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.st1.pn.x4.nxv2f64(<vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], <vscale x 2 x double> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svst1_vnum_f64_x4(svcount_t pn, float64_t *base, float64_t vnum, svfloat64x4_t v)
+{
+  return SVE_ACLE_FUNC(svst1_vnum,_f64_x4,,)(pn, base, vnum, v);
+}
diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_stnt1.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_stnt1.c
new file mode 100644
index 0000000000..536211fdc3
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_stnt1.c
@@ -0,0 +1,1042 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+
+// CHECK-LABEL: @test_svstnt1_u8_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[V]], i64 16)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z18test_svstnt1_u8_x2u11__SVCount_tPh11svuint8x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[V]], i64 16)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_u8_x2(svcount_t pn, uint8_t *base, svuint8x2_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1,_u8_x2,,)(pn, base, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_u16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[V]], i64 8)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z19test_svstnt1_u16_x2u11__SVCount_tPt12svuint16x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[V]], i64 8)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_u16_x2(svcount_t pn, uint16_t *base, svuint16x2_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1,_u16_x2,,)(pn, base, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_u32_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[V]], i64 4)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z19test_svstnt1_u32_x2u11__SVCount_tPj12svuint32x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[V]], i64 4)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_u32_x2(svcount_t pn, uint32_t *base, svuint32x2_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1,_u32_x2,,)(pn, base, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_u64_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[V]], i64 2)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z19test_svstnt1_u64_x2u11__SVCount_tPm12svuint64x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[V]], i64 2)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_u64_x2(svcount_t pn, uint64_t *base, svuint64x2_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1,_u64_x2,,)(pn, base, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_u8_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 16)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 32)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 48)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z18test_svstnt1_u8_x4u11__SVCount_tPh11svuint8x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 32)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 48)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_u8_x4(svcount_t pn, uint8_t *base, svuint8x4_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1,_u8_x4,,)(pn, base, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_u16_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 24)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z19test_svstnt1_u16_x4u11__SVCount_tPt12svuint16x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 24)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_u16_x4(svcount_t pn, uint16_t *base, svuint16x4_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1,_u16_x4,,)(pn, base, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_u32_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 4)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 8)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 12)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z19test_svstnt1_u32_x4u11__SVCount_tPj12svuint32x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 12)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_u32_x4(svcount_t pn, uint32_t *base, svuint32x4_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1,_u32_x4,,)(pn, base, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_u64_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 2)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 4)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 6)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z19test_svstnt1_u64_x4u11__SVCount_tPm12svuint64x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 2)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 6)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_u64_x4(svcount_t pn, uint64_t *base, svuint64x4_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1,_u64_x4,,)(pn, base, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_s8_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[V]], i64 16)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z18test_svstnt1_s8_x2u11__SVCount_tPa10svint8x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[V]], i64 16)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_s8_x2(svcount_t pn, int8_t *base, svint8x2_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1,_s8_x2,,)(pn, base, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_s16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[V]], i64 8)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z19test_svstnt1_s16_x2u11__SVCount_tPs11svint16x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[V]], i64 8)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_s16_x2(svcount_t pn, int16_t *base, svint16x2_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1,_s16_x2,,)(pn, base, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_s32_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[V]], i64 4)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z19test_svstnt1_s32_x2u11__SVCount_tPi11svint32x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[V]], i64 4)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_s32_x2(svcount_t pn, int32_t *base, svint32x2_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1,_s32_x2,,)(pn, base, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_s64_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[V]], i64 2)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z19test_svstnt1_s64_x2u11__SVCount_tPl11svint64x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[V]], i64 2)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_s64_x2(svcount_t pn, int64_t *base, svint64x2_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1,_s64_x2,,)(pn, base, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_s8_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 16)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 32)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 48)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z18test_svstnt1_s8_x4u11__SVCount_tPa10svint8x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 32)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 48)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_s8_x4(svcount_t pn, int8_t *base, svint8x4_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1,_s8_x4,,)(pn, base, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_s16_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 24)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z19test_svstnt1_s16_x4u11__SVCount_tPs11svint16x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 24)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_s16_x4(svcount_t pn, int16_t *base, svint16x4_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1,_s16_x4,,)(pn, base, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_s32_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 4)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 8)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 12)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z19test_svstnt1_s32_x4u11__SVCount_tPi11svint32x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 12)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_s32_x4(svcount_t pn, int32_t *base, svint32x4_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1,_s32_x4,,)(pn, base, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_s64_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 2)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 4)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 6)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z19test_svstnt1_s64_x4u11__SVCount_tPl11svint64x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 2)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 6)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_s64_x4(svcount_t pn, int64_t *base, svint64x4_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1,_s64_x4,,)(pn, base, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_f16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[V]], i64 8)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv8f16(<vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z19test_svstnt1_f16_x2u11__SVCount_tPDh13svfloat16x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[V]], i64 8)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv8f16(<vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_f16_x2(svcount_t pn, float16_t *base, svfloat16x2_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1,_f16_x2,,)(pn, base, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_f32_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[V]], i64 4)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv4f32(<vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z19test_svstnt1_f32_x2u11__SVCount_tPf13svfloat32x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[V]], i64 4)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv4f32(<vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_f32_x2(svcount_t pn, float32_t *base, svfloat32x2_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1,_f32_x2,,)(pn, base, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_f64_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[V]], i64 2)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv2f64(<vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z19test_svstnt1_f64_x2u11__SVCount_tPd13svfloat64x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[V]], i64 2)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv2f64(<vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_f64_x2(svcount_t pn, float64_t *base, svfloat64x2_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1,_f64_x2,,)(pn, base, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_f16_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[V]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[V]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[V]], i64 24)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv8f16(<vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z19test_svstnt1_f16_x4u11__SVCount_tPDh13svfloat16x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[V]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[V]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[V]], i64 24)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv8f16(<vscale x 8 x half> [[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_f16_x4(svcount_t pn, float16_t *base, svfloat16x4_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1,_f16_x4,,)(pn, base, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_f32_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[V]], i64 4)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[V]], i64 8)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[V]], i64 12)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv4f32(<vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z19test_svstnt1_f32_x4u11__SVCount_tPf13svfloat32x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[V]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[V]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[V]], i64 12)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv4f32(<vscale x 4 x float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_f32_x4(svcount_t pn, float32_t *base, svfloat32x4_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1,_f32_x4,,)(pn, base, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_f64_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[V]], i64 2)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[V]], i64 4)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[V]], i64 6)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv2f64(<vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z19test_svstnt1_f64_x4u11__SVCount_tPd13svfloat64x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[V]], i64 2)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[V]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[V]], i64 6)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv2f64(<vscale x 2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], target("aarch64.svcount") [[PN:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_f64_x4(svcount_t pn, float64_t *base, svfloat64x4_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1,_f64_x4,,)(pn, base, v);
+}
+
+
+// == VNUM variants ==
+
+
+
+// CHECK-LABEL: @test_svstnt1_vnum_u8_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[V]], i64 16)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv16i8(<vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z23test_svstnt1_vnum_u8_x2u11__SVCount_tPhl11svuint8x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[V]], i64 16)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv16i8(<vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_vnum_u8_x2(svcount_t pn, uint8_t *base, int64_t vnum, svuint8x2_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1_vnum,_u8_x2,,)(pn, base, vnum, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_vnum_u16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[V]], i64 8)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv8i16(<vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z24test_svstnt1_vnum_u16_x2u11__SVCount_tPtl12svuint16x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[V]], i64 8)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv8i16(<vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_vnum_u16_x2(svcount_t pn, uint16_t *base, int64_t vnum, svuint16x2_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1_vnum,_u16_x2,,)(pn, base, vnum, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_vnum_u32_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[V]], i64 4)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv4i32(<vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z24test_svstnt1_vnum_u32_x2u11__SVCount_tPjl12svuint32x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[V]], i64 4)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv4i32(<vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_vnum_u32_x2(svcount_t pn, uint32_t *base, int64_t vnum, svuint32x2_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1_vnum,_u32_x2,,)(pn, base, vnum, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_vnum_u64_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[V]], i64 2)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv2i64(<vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z24test_svstnt1_vnum_u64_x2u11__SVCount_tPml12svuint64x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[V]], i64 2)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv2i64(<vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_vnum_u64_x2(svcount_t pn, uint64_t *base, int64_t vnum, svuint64x2_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1_vnum,_u64_x2,,)(pn, base, vnum, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_vnum_u8_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 32)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 48)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv16i8(<vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z23test_svstnt1_vnum_u8_x4u11__SVCount_tPhl11svuint8x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 32)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 48)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv16i8(<vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_vnum_u8_x4(svcount_t pn, uint8_t *base, int64_t vnum, svuint8x4_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1_vnum,_u8_x4,,)(pn, base, vnum, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_vnum_u16_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 8)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 16)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 24)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv8i16(<vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z24test_svstnt1_vnum_u16_x4u11__SVCount_tPtl12svuint16x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 24)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv8i16(<vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_vnum_u16_x4(svcount_t pn, uint16_t *base, int64_t vnum, svuint16x4_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1_vnum,_u16_x4,,)(pn, base, vnum, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_vnum_u32_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 4)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 8)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 12)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv4i32(<vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z24test_svstnt1_vnum_u32_x4u11__SVCount_tPjl12svuint32x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 12)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv4i32(<vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_vnum_u32_x4(svcount_t pn, uint32_t *base, int64_t vnum, svuint32x4_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1_vnum,_u32_x4,,)(pn, base, vnum, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_vnum_u64_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 2)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 4)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 6)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv2i64(<vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z24test_svstnt1_vnum_u64_x4u11__SVCount_tPml12svuint64x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 2)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 6)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv2i64(<vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_vnum_u64_x4(svcount_t pn, uint64_t *base, int64_t vnum, svuint64x4_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1_vnum,_u64_x4,,)(pn, base, vnum, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_vnum_s8_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[V]], i64 16)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv16i8(<vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z23test_svstnt1_vnum_s8_x2u11__SVCount_tPal10svint8x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[V]], i64 16)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv16i8(<vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_vnum_s8_x2(svcount_t pn, int8_t *base, int64_t vnum, svint8x2_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1_vnum,_s8_x2,,)(pn, base, vnum, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_vnum_s16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[V]], i64 8)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv8i16(<vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z24test_svstnt1_vnum_s16_x2u11__SVCount_tPsl11svint16x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[V]], i64 8)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv8i16(<vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_vnum_s16_x2(svcount_t pn, int16_t *base, int64_t vnum, svint16x2_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1_vnum,_s16_x2,,)(pn, base, vnum, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_vnum_s32_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[V]], i64 4)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv4i32(<vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z24test_svstnt1_vnum_s32_x2u11__SVCount_tPil11svint32x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[V]], i64 4)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv4i32(<vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_vnum_s32_x2(svcount_t pn, int32_t *base, int64_t vnum, svint32x2_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1_vnum,_s32_x2,,)(pn, base, vnum, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_vnum_s64_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[V]], i64 2)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv2i64(<vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z24test_svstnt1_vnum_s64_x2u11__SVCount_tPll11svint64x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[V]], i64 2)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv2i64(<vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_vnum_s64_x2(svcount_t pn, int64_t *base, int64_t vnum, svint64x2_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1_vnum,_s64_x2,,)(pn, base, vnum, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_vnum_s8_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 32)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 48)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv16i8(<vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z23test_svstnt1_vnum_s8_x4u11__SVCount_tPal10svint8x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 32)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[V]], i64 48)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv16i8(<vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_vnum_s8_x4(svcount_t pn, int8_t *base, int64_t vnum, svint8x4_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1_vnum,_s8_x4,,)(pn, base, vnum, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_vnum_s16_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 8)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 16)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 24)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv8i16(<vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z24test_svstnt1_vnum_s16_x4u11__SVCount_tPsl11svint16x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x i16>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[V]], i64 24)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv8i16(<vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_vnum_s16_x4(svcount_t pn, int16_t *base, int64_t vnum, svint16x4_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1_vnum,_s16_x4,,)(pn, base, vnum, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_vnum_s32_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 4)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 8)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 12)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv4i32(<vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z24test_svstnt1_vnum_s32_x4u11__SVCount_tPil11svint32x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[V]], i64 12)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv4i32(<vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_vnum_s32_x4(svcount_t pn, int32_t *base, int64_t vnum, svint32x4_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1_vnum,_s32_x4,,)(pn, base, vnum, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_vnum_s64_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 2)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 4)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 6)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv2i64(<vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z24test_svstnt1_vnum_s64_x4u11__SVCount_tPll11svint64x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x i64>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 2)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[V]], i64 6)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv2i64(<vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_vnum_s64_x4(svcount_t pn, int64_t *base, int64_t vnum, svint64x4_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1_vnum,_s64_x4,,)(pn, base, vnum, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_vnum_f16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[CONV:%.*]] = fptosi double [[VNUM:%.*]] to i64
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[CONV]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[V]], i64 8)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv8f16(<vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z24test_svstnt1_vnum_f16_x2u11__SVCount_tPDhd13svfloat16x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[CONV:%.*]] = fptosi double [[VNUM:%.*]] to i64
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[CONV]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[V]], i64 8)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv8f16(<vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_vnum_f16_x2(svcount_t pn, float16_t *base, float64_t vnum, svfloat16x2_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1_vnum,_f16_x2,,)(pn, base, vnum, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_vnum_f32_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[CONV:%.*]] = fptosi double [[VNUM:%.*]] to i64
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[CONV]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[V]], i64 4)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv4f32(<vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z24test_svstnt1_vnum_f32_x2u11__SVCount_tPfd13svfloat32x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[CONV:%.*]] = fptosi double [[VNUM:%.*]] to i64
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[CONV]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[V]], i64 4)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv4f32(<vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_vnum_f32_x2(svcount_t pn, float32_t *base, float64_t vnum, svfloat32x2_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1_vnum,_f32_x2,,)(pn, base, vnum, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_vnum_f64_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[CONV:%.*]] = fptosi double [[VNUM:%.*]] to i64
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[CONV]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[V]], i64 2)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv2f64(<vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z24test_svstnt1_vnum_f64_x2u11__SVCount_tPdd13svfloat64x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[CONV:%.*]] = fptosi double [[VNUM:%.*]] to i64
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[CONV]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[V]], i64 2)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x2.nxv2f64(<vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_vnum_f64_x2(svcount_t pn, float64_t *base, float64_t vnum, svfloat64x2_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1_vnum,_f64_x2,,)(pn, base, vnum, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_vnum_f16_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[CONV:%.*]] = fptosi double [[VNUM:%.*]] to i64
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[CONV]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[V]], i64 8)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[V]], i64 16)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[V]], i64 24)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv8f16(<vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z24test_svstnt1_vnum_f16_x4u11__SVCount_tPDhd13svfloat16x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[CONV:%.*]] = fptosi double [[VNUM:%.*]] to i64
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 8 x half>, ptr [[BASE:%.*]], i64 [[CONV]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[V]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[V]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x half> @llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[V]], i64 24)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv8f16(<vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]], <vscale x 8 x half> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_vnum_f16_x4(svcount_t pn, float16_t *base, float64_t vnum, svfloat16x4_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1_vnum,_f16_x4,,)(pn, base, vnum, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_vnum_f32_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[CONV:%.*]] = fptosi double [[VNUM:%.*]] to i64
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[CONV]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[V]], i64 4)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[V]], i64 8)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[V]], i64 12)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv4f32(<vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z24test_svstnt1_vnum_f32_x4u11__SVCount_tPfd13svfloat32x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[CONV:%.*]] = fptosi double [[VNUM:%.*]] to i64
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 4 x float>, ptr [[BASE:%.*]], i64 [[CONV]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[V]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[V]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[V]], i64 12)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv4f32(<vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float> [[TMP3]], <vscale x 4 x float> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_vnum_f32_x4(svcount_t pn, float32_t *base, float64_t vnum, svfloat32x4_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1_vnum,_f32_x4,,)(pn, base, vnum, v);
+}
+
+
+// CHECK-LABEL: @test_svstnt1_vnum_f64_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[CONV:%.*]] = fptosi double [[VNUM:%.*]] to i64
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[CONV]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[V:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[V]], i64 2)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[V]], i64 4)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[V]], i64 6)
+// CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv2f64(<vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], <vscale x 2 x double> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+// CPP-CHECK-LABEL: @_Z24test_svstnt1_vnum_f64_x4u11__SVCount_tPdd13svfloat64x4_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[CONV:%.*]] = fptosi double [[VNUM:%.*]] to i64
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 2 x double>, ptr [[BASE:%.*]], i64 [[CONV]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[V:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[V]], i64 2)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[V]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 2 x double> @llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[V]], i64 6)
+// CPP-CHECK-NEXT:    tail call void @llvm.aarch64.sve.stnt1.pn.x4.nxv2f64(<vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], <vscale x 2 x double> [[TMP3]], <vscale x 2 x double> [[TMP4]], target("aarch64.svcount") [[PN:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret void
+//
+void test_svstnt1_vnum_f64_x4(svcount_t pn, float64_t *base, float64_t vnum, svfloat64x4_t v)
+{
+  return SVE_ACLE_FUNC(svstnt1_vnum,_f64_x4,,)(pn, base, vnum, v);
+}
diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_while_pn.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_while_pn.c
new file mode 100644
index 0000000000..3dbb38582b
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_while_pn.c
@@ -0,0 +1,992 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
+
+// REQUIRES: aarch64-registered-target
+
+#include <arm_sve.h>
+
+
+// WHILEGE
+
+// CHECK-LABEL: @test_svwhilege_c8_vl2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z21test_svwhilege_c8_vl2ll(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilege_c8_vl2(int64_t op1, int64_t op2)
+{
+  return svwhilege_c8(op1, op2, 2);
+}
+
+// CHECK-LABEL: @test_svwhilege_c8_vl4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z21test_svwhilege_c8_vl4ll(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilege_c8_vl4(int64_t op1, int64_t op2)
+{
+  return svwhilege_c8(op1, op2, 4);
+}
+
+// CHECK-LABEL: @test_svwhilege_c16_vl2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilege_c16_vl2ll(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilege_c16_vl2(int64_t op1, int64_t op2)
+{
+  return svwhilege_c16(op1, op2, 2);
+}
+
+// CHECK-LABEL: @test_svwhilege_c16_vl4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilege_c16_vl4ll(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilege_c16_vl4(int64_t op1, int64_t op2)
+{
+  return svwhilege_c16(op1, op2, 4);
+}
+
+// CHECK-LABEL: @test_svwhilege_c32_vl2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilege_c32_vl2ll(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilege_c32_vl2(int64_t op1, int64_t op2)
+{
+  return svwhilege_c32(op1, op2, 2);
+}
+
+// CHECK-LABEL: @test_svwhilege_c32_vl4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilege_c32_vl4ll(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilege_c32_vl4(int64_t op1, int64_t op2)
+{
+  return svwhilege_c32(op1, op2, 4);
+}
+
+// CHECK-LABEL: @test_svwhilege_c64_vl2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilege_c64_vl2ll(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilege_c64_vl2(int64_t op1, int64_t op2)
+{
+  return svwhilege_c64(op1, op2, 2);
+}
+
+// CHECK-LABEL: @test_svwhilege_c64_vl4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilege_c64_vl4ll(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilege_c64_vl4(int64_t op1, int64_t op2)
+{
+  return svwhilege_c64(op1, op2, 4);
+}
+
+// WHILEGT
+
+
+// CHECK-LABEL: @test_svwhilegt_c8_vl2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z21test_svwhilegt_c8_vl2ll(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilegt_c8_vl2(int64_t op1, int64_t op2)
+{
+  return svwhilegt_c8(op1, op2, 2);
+}
+
+// CHECK-LABEL: @test_svwhilegt_c8_vl4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z21test_svwhilegt_c8_vl4ll(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilegt_c8_vl4(int64_t op1, int64_t op2)
+{
+  return svwhilegt_c8(op1, op2, 4);
+}
+
+// CHECK-LABEL: @test_svwhilegt_c16_vl2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilegt_c16_vl2ll(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilegt_c16_vl2(int64_t op1, int64_t op2)
+{
+  return svwhilegt_c16(op1, op2, 2);
+}
+
+// CHECK-LABEL: @test_svwhilegt_c16_vl4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilegt_c16_vl4ll(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilegt_c16_vl4(int64_t op1, int64_t op2)
+{
+  return svwhilegt_c16(op1, op2, 4);
+}
+
+// CHECK-LABEL: @test_svwhilegt_c32_vl2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilegt_c32_vl2ll(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilegt_c32_vl2(int64_t op1, int64_t op2)
+{
+  return svwhilegt_c32(op1, op2, 2);
+}
+
+// CHECK-LABEL: @test_svwhilegt_c32_vl4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilegt_c32_vl4ll(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilegt_c32_vl4(int64_t op1, int64_t op2)
+{
+  return svwhilegt_c32(op1, op2, 4);
+}
+
+// CHECK-LABEL: @test_svwhilegt_c64_vl2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilegt_c64_vl2ll(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilegt_c64_vl2(int64_t op1, int64_t op2)
+{
+  return svwhilegt_c64(op1, op2, 2);
+}
+
+// CHECK-LABEL: @test_svwhilegt_c64_vl4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilegt_c64_vl4ll(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilegt_c64_vl4(int64_t op1, int64_t op2)
+{
+  return svwhilegt_c64(op1, op2, 4);
+}
+
+
+// WHILEHI
+
+// CHECK-LABEL: @test_svwhilehi_c8_vl2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z21test_svwhilehi_c8_vl2mm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilehi_c8_vl2(uint64_t op1, uint64_t op2)
+{
+  return svwhilehi_c8(op1, op2, 2);
+}
+
+// CHECK-LABEL: @test_svwhilehi_c8_vl4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z21test_svwhilehi_c8_vl4mm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilehi_c8_vl4(uint64_t op1, uint64_t op2)
+{
+  return svwhilehi_c8(op1, op2, 4);
+}
+
+// CHECK-LABEL: @test_svwhilehi_c16_vl2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilehi_c16_vl2mm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilehi_c16_vl2(uint64_t op1, uint64_t op2)
+{
+  return svwhilehi_c16(op1, op2, 2);
+}
+
+// CHECK-LABEL: @test_svwhilehi_c16_vl4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilehi_c16_vl4mm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilehi_c16_vl4(uint64_t op1, uint64_t op2)
+{
+  return svwhilehi_c16(op1, op2, 4);
+}
+
+// CHECK-LABEL: @test_svwhilehi_c32_vl2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilehi_c32_vl2mm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilehi_c32_vl2(uint64_t op1, uint64_t op2)
+{
+  return svwhilehi_c32(op1, op2, 2);
+}
+
+// CHECK-LABEL: @test_svwhilehi_c32_vl4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilehi_c32_vl4mm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilehi_c32_vl4(uint64_t op1, uint64_t op2)
+{
+  return svwhilehi_c32(op1, op2, 4);
+}
+
+// CHECK-LABEL: @test_svwhilehi_c64_vl2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilehi_c64_vl2mm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilehi_c64_vl2(uint64_t op1, uint64_t op2)
+{
+  return svwhilehi_c64(op1, op2, 2);
+}
+
+// CHECK-LABEL: @test_svwhilehi_c64_vl4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilehi_c64_vl4mm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilehi_c64_vl4(uint64_t op1, uint64_t op2)
+{
+  return svwhilehi_c64(op1, op2, 4);
+}
+
+
+// WHILEHS
+
+// CHECK-LABEL: @test_svwhilehs_c8_vl2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z21test_svwhilehs_c8_vl2mm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilehs_c8_vl2(uint64_t op1, uint64_t op2)
+{
+  return svwhilehs_c8(op1, op2, 2);
+}
+
+// CHECK-LABEL: @test_svwhilehs_c8_vl4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z21test_svwhilehs_c8_vl4mm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilehs_c8_vl4(uint64_t op1, uint64_t op2)
+{
+  return svwhilehs_c8(op1, op2, 4);
+}
+
+// CHECK-LABEL: @test_svwhilehs_c16_vl2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilehs_c16_vl2mm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilehs_c16_vl2(uint64_t op1, uint64_t op2)
+{
+  return svwhilehs_c16(op1, op2, 2);
+}
+
+// CHECK-LABEL: @test_svwhilehs_c16_vl4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilehs_c16_vl4mm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilehs_c16_vl4(uint64_t op1, uint64_t op2)
+{
+  return svwhilehs_c16(op1, op2, 4);
+}
+
+// CHECK-LABEL: @test_svwhilehs_c32_vl2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilehs_c32_vl2mm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilehs_c32_vl2(uint64_t op1, uint64_t op2)
+{
+  return svwhilehs_c32(op1, op2, 2);
+}
+
+// CHECK-LABEL: @test_svwhilehs_c32_vl4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilehs_c32_vl4mm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilehs_c32_vl4(uint64_t op1, uint64_t op2)
+{
+  return svwhilehs_c32(op1, op2, 4);
+}
+
+// CHECK-LABEL: @test_svwhilehs_c64_vl2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilehs_c64_vl2mm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilehs_c64_vl2(uint64_t op1, uint64_t op2)
+{
+  return svwhilehs_c64(op1, op2, 2);
+}
+
+// CHECK-LABEL: @test_svwhilehs_c64_vl4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilehs_c64_vl4mm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilehs_c64_vl4(uint64_t op1, uint64_t op2)
+{
+  return svwhilehs_c64(op1, op2, 4);
+}
+
+
+// WHILELE
+
+// CHECK-LABEL: @test_svwhilele_c8_vl2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z21test_svwhilele_c8_vl2ll(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilele_c8_vl2(int64_t op1, int64_t op2)
+{
+  return svwhilele_c8(op1, op2, 2);
+}
+
+// CHECK-LABEL: @test_svwhilele_c8_vl4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z21test_svwhilele_c8_vl4ll(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilele_c8_vl4(int64_t op1, int64_t op2)
+{
+  return svwhilele_c8(op1, op2, 4);
+}
+
+// CHECK-LABEL: @test_svwhilele_c16_vl2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilele_c16_vl2ll(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilele_c16_vl2(int64_t op1, int64_t op2)
+{
+  return svwhilele_c16(op1, op2, 2);
+}
+
+// CHECK-LABEL: @test_svwhilele_c16_vl4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilele_c16_vl4ll(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilele_c16_vl4(int64_t op1, int64_t op2)
+{
+  return svwhilele_c16(op1, op2, 4);
+}
+
+// CHECK-LABEL: @test_svwhilele_c32_vl2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilele_c32_vl2ll(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilele_c32_vl2(int64_t op1, int64_t op2)
+{
+  return svwhilele_c32(op1, op2, 2);
+}
+
+// CHECK-LABEL: @test_svwhilele_c32_vl4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilele_c32_vl4ll(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilele_c32_vl4(int64_t op1, int64_t op2)
+{
+  return svwhilele_c32(op1, op2, 4);
+}
+
+// CHECK-LABEL: @test_svwhilele_c64_vl2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilele_c64_vl2ll(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilele_c64_vl2(int64_t op1, int64_t op2)
+{
+  return svwhilele_c64(op1, op2, 2);
+}
+
+// CHECK-LABEL: @test_svwhilele_c64_vl4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilele_c64_vl4ll(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilele_c64_vl4(int64_t op1, int64_t op2)
+{
+  return svwhilele_c64(op1, op2, 4);
+}
+
+
+// WHILELO
+
+// CHECK-LABEL: @test_svwhilelo_c8_vl2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z21test_svwhilelo_c8_vl2mm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilelo_c8_vl2(uint64_t op1, uint64_t op2)
+{
+  return svwhilelo_c8(op1, op2, 2);
+}
+
+// CHECK-LABEL: @test_svwhilelo_c8_vl4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z21test_svwhilelo_c8_vl4mm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilelo_c8_vl4(uint64_t op1, uint64_t op2)
+{
+  return svwhilelo_c8(op1, op2, 4);
+}
+
+// CHECK-LABEL: @test_svwhilelo_c16_vl2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilelo_c16_vl2mm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilelo_c16_vl2(uint64_t op1, uint64_t op2)
+{
+  return svwhilelo_c16(op1, op2, 2);
+}
+
+// CHECK-LABEL: @test_svwhilelo_c16_vl4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilelo_c16_vl4mm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilelo_c16_vl4(uint64_t op1, uint64_t op2)
+{
+  return svwhilelo_c16(op1, op2, 4);
+}
+
+// CHECK-LABEL: @test_svwhilelo_c32_vl2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilelo_c32_vl2mm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilelo_c32_vl2(uint64_t op1, uint64_t op2)
+{
+  return svwhilelo_c32(op1, op2, 2);
+}
+
+// CHECK-LABEL: @test_svwhilelo_c32_vl4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilelo_c32_vl4mm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilelo_c32_vl4(uint64_t op1, uint64_t op2)
+{
+  return svwhilelo_c32(op1, op2, 4);
+}
+
+// CHECK-LABEL: @test_svwhilelo_c64_vl2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilelo_c64_vl2mm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilelo_c64_vl2(uint64_t op1, uint64_t op2)
+{
+  return svwhilelo_c64(op1, op2, 2);
+}
+
+// CHECK-LABEL: @test_svwhilelo_c64_vl4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilelo_c64_vl4mm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilelo_c64_vl4(uint64_t op1, uint64_t op2)
+{
+  return svwhilelo_c64(op1, op2, 4);
+}
+
+
+// WHILELS
+
+// CHECK-LABEL: @test_svwhilels_c8_vl2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z21test_svwhilels_c8_vl2mm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilels_c8_vl2(uint64_t op1, uint64_t op2)
+{
+  return svwhilels_c8(op1, op2, 2);
+}
+
+// CHECK-LABEL: @test_svwhilels_c8_vl4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z21test_svwhilels_c8_vl4mm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilels_c8_vl4(uint64_t op1, uint64_t op2)
+{
+  return svwhilels_c8(op1, op2, 4);
+}
+
+// CHECK-LABEL: @test_svwhilels_c16_vl2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilels_c16_vl2mm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilels_c16_vl2(uint64_t op1, uint64_t op2)
+{
+  return svwhilels_c16(op1, op2, 2);
+}
+
+// CHECK-LABEL: @test_svwhilels_c16_vl4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilels_c16_vl4mm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilels_c16_vl4(uint64_t op1, uint64_t op2)
+{
+  return svwhilels_c16(op1, op2, 4);
+}
+
+// CHECK-LABEL: @test_svwhilels_c32_vl2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilels_c32_vl2mm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilels_c32_vl2(uint64_t op1, uint64_t op2)
+{
+  return svwhilels_c32(op1, op2, 2);
+}
+
+// CHECK-LABEL: @test_svwhilels_c32_vl4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilels_c32_vl4mm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilels_c32_vl4(uint64_t op1, uint64_t op2)
+{
+  return svwhilels_c32(op1, op2, 4);
+}
+
+// CHECK-LABEL: @test_svwhilels_c64_vl2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilels_c64_vl2mm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilels_c64_vl2(uint64_t op1, uint64_t op2)
+{
+  return svwhilels_c64(op1, op2, 2);
+}
+
+// CHECK-LABEL: @test_svwhilels_c64_vl4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilels_c64_vl4mm(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilels_c64_vl4(uint64_t op1, uint64_t op2)
+{
+  return svwhilels_c64(op1, op2, 4);
+}
+
+
+// WHILELT
+
+// CHECK-LABEL: @test_svwhilelt_c8_vl2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z21test_svwhilelt_c8_vl2ll(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilelt_c8_vl2(int64_t op1, int64_t op2)
+{
+  return svwhilelt_c8(op1, op2, 2);
+}
+
+// CHECK-LABEL: @test_svwhilelt_c8_vl4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z21test_svwhilelt_c8_vl4ll(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c8(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilelt_c8_vl4(int64_t op1, int64_t op2)
+{
+  return svwhilelt_c8(op1, op2, 4);
+}
+
+// CHECK-LABEL: @test_svwhilelt_c16_vl2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilelt_c16_vl2ll(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilelt_c16_vl2(int64_t op1, int64_t op2)
+{
+  return svwhilelt_c16(op1, op2, 2);
+}
+
+// CHECK-LABEL: @test_svwhilelt_c16_vl4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilelt_c16_vl4ll(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c16(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilelt_c16_vl4(int64_t op1, int64_t op2)
+{
+  return svwhilelt_c16(op1, op2, 4);
+}
+
+// CHECK-LABEL: @test_svwhilelt_c32_vl2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilelt_c32_vl2ll(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilelt_c32_vl2(int64_t op1, int64_t op2)
+{
+  return svwhilelt_c32(op1, op2, 2);
+}
+
+// CHECK-LABEL: @test_svwhilelt_c32_vl4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilelt_c32_vl4ll(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c32(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilelt_c32_vl4(int64_t op1, int64_t op2)
+{
+  return svwhilelt_c32(op1, op2, 4);
+}
+
+// CHECK-LABEL: @test_svwhilelt_c64_vl2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilelt_c64_vl2ll(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 2)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilelt_c64_vl2(int64_t op1, int64_t op2)
+{
+  return svwhilelt_c64(op1, op2, 2);
+}
+
+// CHECK-LABEL: @test_svwhilelt_c64_vl4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z22test_svwhilelt_c64_vl4ll(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c64(i64 [[OP1:%.*]], i64 [[OP2:%.*]], i32 4)
+// CPP-CHECK-NEXT:    ret target("aarch64.svcount") [[TMP0]]
+//
+svcount_t test_svwhilelt_c64_vl4(int64_t op1, int64_t op2)
+{
+  return svwhilelt_c64(op1, op2, 4);
+}
diff --git a/clang/test/CodeGen/arm64-microsoft-intrinsics.c b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
index b15defb089..44b2ee28fe 100644
--- a/clang/test/CodeGen/arm64-microsoft-intrinsics.c
+++ b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
@@ -402,6 +402,16 @@ unsigned int check_CountOneBits64(unsigned __int64 arg1) {
 // CHECK-MSCOMPAT: ret i32 %[[VAR2]]
 // CHECK-LINUX: error: call to undeclared function '_CountOneBits64'
 
+void check__prefetch(void *arg1) {
+  return __prefetch(arg1);
+}
+
+// CHECK-MSCOMPAT: %[[ARG1:.*]].addr = alloca ptr, align 8
+// CHECK-MSCOMPAT: store ptr %[[ARG1]], ptr %[[ARG1]].addr, align 8
+// CHECK-MSCOMPAT: %[[VAR0:.*]] = load ptr, ptr %[[ARG1]].addr, align 8
+// CHECK-MSCOMPAT: call void @llvm.prefetch.p0(ptr %[[VAR0]], i32 0, i32 3, i32 1)
+// CHECK-MSCOMPAT: ret void
+
 
 // CHECK-MSCOMPAT: ![[MD2]] = !{!"x18"}
 // CHECK-MSCOMPAT: ![[MD3]] = !{!"sp"}
diff --git a/clang/test/CodeGen/attr-arm-sve-vector-bits-bitcast.c b/clang/test/CodeGen/attr-arm-sve-vector-bits-bitcast.c
index 0d06327e46..22e2e0c2ff 100644
--- a/clang/test/CodeGen/attr-arm-sve-vector-bits-bitcast.c
+++ b/clang/test/CodeGen/attr-arm-sve-vector-bits-bitcast.c
@@ -28,50 +28,50 @@ DEFINE_STRUCT(bool)
 // int64
 //===----------------------------------------------------------------------===//
 
-// CHECK-128-LABEL: define {{[^@]+}}@read_int64(
+// CHECK-128-LABEL: @read_int64(
 // CHECK-128-NEXT:  entry:
 // CHECK-128-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_INT64:%.*]], ptr [[S:%.*]], i64 0, i32 1
 // CHECK-128-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr [[Y]], align 16, !tbaa [[TBAA2:![0-9]+]]
-// CHECK-128-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v2i64(<vscale x 2 x i64> undef, <2 x i64> [[TMP0]], i64 0)
-// CHECK-128-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
+// CHECK-128-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v2i64(<vscale x 2 x i64> undef, <2 x i64> [[TMP0]], i64 0)
+// CHECK-128-NEXT:    ret <vscale x 2 x i64> [[CAST_SCALABLE]]
 //
-// CHECK-256-LABEL: define {{[^@]+}}@read_int64(
+// CHECK-256-LABEL: @read_int64(
 // CHECK-256-NEXT:  entry:
 // CHECK-256-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_INT64:%.*]], ptr [[S:%.*]], i64 0, i32 1
 // CHECK-256-NEXT:    [[TMP0:%.*]] = load <4 x i64>, ptr [[Y]], align 16, !tbaa [[TBAA2:![0-9]+]]
-// CHECK-256-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP0]], i64 0)
-// CHECK-256-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
+// CHECK-256-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64> undef, <4 x i64> [[TMP0]], i64 0)
+// CHECK-256-NEXT:    ret <vscale x 2 x i64> [[CAST_SCALABLE]]
 //
-// CHECK-512-LABEL: define {{[^@]+}}@read_int64(
+// CHECK-512-LABEL: @read_int64(
 // CHECK-512-NEXT:  entry:
 // CHECK-512-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_INT64:%.*]], ptr [[S:%.*]], i64 0, i32 1
 // CHECK-512-NEXT:    [[TMP0:%.*]] = load <8 x i64>, ptr [[Y]], align 16, !tbaa [[TBAA2:![0-9]+]]
-// CHECK-512-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[TMP0]], i64 0)
-// CHECK-512-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
+// CHECK-512-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[TMP0]], i64 0)
+// CHECK-512-NEXT:    ret <vscale x 2 x i64> [[CAST_SCALABLE]]
 //
 svint64_t read_int64(struct struct_int64 *s) {
   return s->y[0];
 }
 
-// CHECK-128-LABEL: define {{[^@]+}}@write_int64(
+// CHECK-128-LABEL: @write_int64(
 // CHECK-128-NEXT:  entry:
-// CHECK-128-NEXT:    [[CASTFIXEDSVE:%.*]] = tail call <2 x i64> @llvm.vector.extract.v2i64.nxv2i64(<vscale x 2 x i64> [[X:%.*]], i64 0)
+// CHECK-128-NEXT:    [[CAST_FIXED:%.*]] = tail call <2 x i64> @llvm.vector.extract.v2i64.nxv2i64(<vscale x 2 x i64> [[X:%.*]], i64 0)
 // CHECK-128-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_INT64:%.*]], ptr [[S:%.*]], i64 0, i32 1
-// CHECK-128-NEXT:    store <2 x i64> [[CASTFIXEDSVE]], ptr [[Y]], align 16, !tbaa [[TBAA2]]
+// CHECK-128-NEXT:    store <2 x i64> [[CAST_FIXED]], ptr [[Y]], align 16, !tbaa [[TBAA2]]
 // CHECK-128-NEXT:    ret void
 //
-// CHECK-256-LABEL: define {{[^@]+}}@write_int64(
+// CHECK-256-LABEL: @write_int64(
 // CHECK-256-NEXT:  entry:
-// CHECK-256-NEXT:    [[CASTFIXEDSVE:%.*]] = tail call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[X:%.*]], i64 0)
+// CHECK-256-NEXT:    [[CAST_FIXED:%.*]] = tail call <4 x i64> @llvm.vector.extract.v4i64.nxv2i64(<vscale x 2 x i64> [[X:%.*]], i64 0)
 // CHECK-256-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_INT64:%.*]], ptr [[S:%.*]], i64 0, i32 1
-// CHECK-256-NEXT:    store <4 x i64> [[CASTFIXEDSVE]], ptr [[Y]], align 16, !tbaa [[TBAA2]]
+// CHECK-256-NEXT:    store <4 x i64> [[CAST_FIXED]], ptr [[Y]], align 16, !tbaa [[TBAA2]]
 // CHECK-256-NEXT:    ret void
 //
-// CHECK-512-LABEL: define {{[^@]+}}@write_int64(
+// CHECK-512-LABEL: @write_int64(
 // CHECK-512-NEXT:  entry:
-// CHECK-512-NEXT:    [[CASTFIXEDSVE:%.*]] = tail call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[X:%.*]], i64 0)
+// CHECK-512-NEXT:    [[CAST_FIXED:%.*]] = tail call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[X:%.*]], i64 0)
 // CHECK-512-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_INT64:%.*]], ptr [[S:%.*]], i64 0, i32 1
-// CHECK-512-NEXT:    store <8 x i64> [[CASTFIXEDSVE]], ptr [[Y]], align 16, !tbaa [[TBAA2]]
+// CHECK-512-NEXT:    store <8 x i64> [[CAST_FIXED]], ptr [[Y]], align 16, !tbaa [[TBAA2]]
 // CHECK-512-NEXT:    ret void
 //
 void write_int64(struct struct_int64 *s, svint64_t x) {
@@ -82,50 +82,50 @@ void write_int64(struct struct_int64 *s, svint64_t x) {
 // float64
 //===----------------------------------------------------------------------===//
 
-// CHECK-128-LABEL: define {{[^@]+}}@read_float64(
+// CHECK-128-LABEL: @read_float64(
 // CHECK-128-NEXT:  entry:
 // CHECK-128-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_FLOAT64:%.*]], ptr [[S:%.*]], i64 0, i32 1
 // CHECK-128-NEXT:    [[TMP0:%.*]] = load <2 x double>, ptr [[Y]], align 16, !tbaa [[TBAA2]]
-// CHECK-128-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v2f64(<vscale x 2 x double> undef, <2 x double> [[TMP0]], i64 0)
-// CHECK-128-NEXT:    ret <vscale x 2 x double> [[CASTSCALABLESVE]]
+// CHECK-128-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v2f64(<vscale x 2 x double> undef, <2 x double> [[TMP0]], i64 0)
+// CHECK-128-NEXT:    ret <vscale x 2 x double> [[CAST_SCALABLE]]
 //
-// CHECK-256-LABEL: define {{[^@]+}}@read_float64(
+// CHECK-256-LABEL: @read_float64(
 // CHECK-256-NEXT:  entry:
 // CHECK-256-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_FLOAT64:%.*]], ptr [[S:%.*]], i64 0, i32 1
 // CHECK-256-NEXT:    [[TMP0:%.*]] = load <4 x double>, ptr [[Y]], align 16, !tbaa [[TBAA2]]
-// CHECK-256-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v4f64(<vscale x 2 x double> undef, <4 x double> [[TMP0]], i64 0)
-// CHECK-256-NEXT:    ret <vscale x 2 x double> [[CASTSCALABLESVE]]
+// CHECK-256-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v4f64(<vscale x 2 x double> undef, <4 x double> [[TMP0]], i64 0)
+// CHECK-256-NEXT:    ret <vscale x 2 x double> [[CAST_SCALABLE]]
 //
-// CHECK-512-LABEL: define {{[^@]+}}@read_float64(
+// CHECK-512-LABEL: @read_float64(
 // CHECK-512-NEXT:  entry:
 // CHECK-512-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_FLOAT64:%.*]], ptr [[S:%.*]], i64 0, i32 1
 // CHECK-512-NEXT:    [[TMP0:%.*]] = load <8 x double>, ptr [[Y]], align 16, !tbaa [[TBAA2]]
-// CHECK-512-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v8f64(<vscale x 2 x double> undef, <8 x double> [[TMP0]], i64 0)
-// CHECK-512-NEXT:    ret <vscale x 2 x double> [[CASTSCALABLESVE]]
+// CHECK-512-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v8f64(<vscale x 2 x double> undef, <8 x double> [[TMP0]], i64 0)
+// CHECK-512-NEXT:    ret <vscale x 2 x double> [[CAST_SCALABLE]]
 //
 svfloat64_t read_float64(struct struct_float64 *s) {
   return s->y[0];
 }
 
-// CHECK-128-LABEL: define {{[^@]+}}@write_float64(
+// CHECK-128-LABEL: @write_float64(
 // CHECK-128-NEXT:  entry:
-// CHECK-128-NEXT:    [[CASTFIXEDSVE:%.*]] = tail call <2 x double> @llvm.vector.extract.v2f64.nxv2f64(<vscale x 2 x double> [[X:%.*]], i64 0)
+// CHECK-128-NEXT:    [[CAST_FIXED:%.*]] = tail call <2 x double> @llvm.vector.extract.v2f64.nxv2f64(<vscale x 2 x double> [[X:%.*]], i64 0)
 // CHECK-128-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_FLOAT64:%.*]], ptr [[S:%.*]], i64 0, i32 1
-// CHECK-128-NEXT:    store <2 x double> [[CASTFIXEDSVE]], ptr [[Y]], align 16, !tbaa [[TBAA2]]
+// CHECK-128-NEXT:    store <2 x double> [[CAST_FIXED]], ptr [[Y]], align 16, !tbaa [[TBAA2]]
 // CHECK-128-NEXT:    ret void
 //
-// CHECK-256-LABEL: define {{[^@]+}}@write_float64(
+// CHECK-256-LABEL: @write_float64(
 // CHECK-256-NEXT:  entry:
-// CHECK-256-NEXT:    [[CASTFIXEDSVE:%.*]] = tail call <4 x double> @llvm.vector.extract.v4f64.nxv2f64(<vscale x 2 x double> [[X:%.*]], i64 0)
+// CHECK-256-NEXT:    [[CAST_FIXED:%.*]] = tail call <4 x double> @llvm.vector.extract.v4f64.nxv2f64(<vscale x 2 x double> [[X:%.*]], i64 0)
 // CHECK-256-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_FLOAT64:%.*]], ptr [[S:%.*]], i64 0, i32 1
-// CHECK-256-NEXT:    store <4 x double> [[CASTFIXEDSVE]], ptr [[Y]], align 16, !tbaa [[TBAA2]]
+// CHECK-256-NEXT:    store <4 x double> [[CAST_FIXED]], ptr [[Y]], align 16, !tbaa [[TBAA2]]
 // CHECK-256-NEXT:    ret void
 //
-// CHECK-512-LABEL: define {{[^@]+}}@write_float64(
+// CHECK-512-LABEL: @write_float64(
 // CHECK-512-NEXT:  entry:
-// CHECK-512-NEXT:    [[CASTFIXEDSVE:%.*]] = tail call <8 x double> @llvm.vector.extract.v8f64.nxv2f64(<vscale x 2 x double> [[X:%.*]], i64 0)
+// CHECK-512-NEXT:    [[CAST_FIXED:%.*]] = tail call <8 x double> @llvm.vector.extract.v8f64.nxv2f64(<vscale x 2 x double> [[X:%.*]], i64 0)
 // CHECK-512-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_FLOAT64:%.*]], ptr [[S:%.*]], i64 0, i32 1
-// CHECK-512-NEXT:    store <8 x double> [[CASTFIXEDSVE]], ptr [[Y]], align 16, !tbaa [[TBAA2]]
+// CHECK-512-NEXT:    store <8 x double> [[CAST_FIXED]], ptr [[Y]], align 16, !tbaa [[TBAA2]]
 // CHECK-512-NEXT:    ret void
 //
 void write_float64(struct struct_float64 *s, svfloat64_t x) {
@@ -136,50 +136,50 @@ void write_float64(struct struct_float64 *s, svfloat64_t x) {
 // bfloat16
 //===----------------------------------------------------------------------===//
 
-// CHECK-128-LABEL: define {{[^@]+}}@read_bfloat16(
+// CHECK-128-LABEL: @read_bfloat16(
 // CHECK-128-NEXT:  entry:
 // CHECK-128-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_BFLOAT16:%.*]], ptr [[S:%.*]], i64 0, i32 1
 // CHECK-128-NEXT:    [[TMP0:%.*]] = load <8 x bfloat>, ptr [[Y]], align 16, !tbaa [[TBAA2]]
-// CHECK-128-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.insert.nxv8bf16.v8bf16(<vscale x 8 x bfloat> undef, <8 x bfloat> [[TMP0]], i64 0)
-// CHECK-128-NEXT:    ret <vscale x 8 x bfloat> [[CASTSCALABLESVE]]
+// CHECK-128-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.insert.nxv8bf16.v8bf16(<vscale x 8 x bfloat> undef, <8 x bfloat> [[TMP0]], i64 0)
+// CHECK-128-NEXT:    ret <vscale x 8 x bfloat> [[CAST_SCALABLE]]
 //
-// CHECK-256-LABEL: define {{[^@]+}}@read_bfloat16(
+// CHECK-256-LABEL: @read_bfloat16(
 // CHECK-256-NEXT:  entry:
 // CHECK-256-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_BFLOAT16:%.*]], ptr [[S:%.*]], i64 0, i32 1
 // CHECK-256-NEXT:    [[TMP0:%.*]] = load <16 x bfloat>, ptr [[Y]], align 16, !tbaa [[TBAA2]]
-// CHECK-256-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.insert.nxv8bf16.v16bf16(<vscale x 8 x bfloat> undef, <16 x bfloat> [[TMP0]], i64 0)
-// CHECK-256-NEXT:    ret <vscale x 8 x bfloat> [[CASTSCALABLESVE]]
+// CHECK-256-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.insert.nxv8bf16.v16bf16(<vscale x 8 x bfloat> undef, <16 x bfloat> [[TMP0]], i64 0)
+// CHECK-256-NEXT:    ret <vscale x 8 x bfloat> [[CAST_SCALABLE]]
 //
-// CHECK-512-LABEL: define {{[^@]+}}@read_bfloat16(
+// CHECK-512-LABEL: @read_bfloat16(
 // CHECK-512-NEXT:  entry:
 // CHECK-512-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_BFLOAT16:%.*]], ptr [[S:%.*]], i64 0, i32 1
 // CHECK-512-NEXT:    [[TMP0:%.*]] = load <32 x bfloat>, ptr [[Y]], align 16, !tbaa [[TBAA2]]
-// CHECK-512-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.insert.nxv8bf16.v32bf16(<vscale x 8 x bfloat> undef, <32 x bfloat> [[TMP0]], i64 0)
-// CHECK-512-NEXT:    ret <vscale x 8 x bfloat> [[CASTSCALABLESVE]]
+// CHECK-512-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 8 x bfloat> @llvm.vector.insert.nxv8bf16.v32bf16(<vscale x 8 x bfloat> undef, <32 x bfloat> [[TMP0]], i64 0)
+// CHECK-512-NEXT:    ret <vscale x 8 x bfloat> [[CAST_SCALABLE]]
 //
 svbfloat16_t read_bfloat16(struct struct_bfloat16 *s) {
   return s->y[0];
 }
 
-// CHECK-128-LABEL: define {{[^@]+}}@write_bfloat16(
+// CHECK-128-LABEL: @write_bfloat16(
 // CHECK-128-NEXT:  entry:
-// CHECK-128-NEXT:    [[CASTFIXEDSVE:%.*]] = tail call <8 x bfloat> @llvm.vector.extract.v8bf16.nxv8bf16(<vscale x 8 x bfloat> [[X:%.*]], i64 0)
+// CHECK-128-NEXT:    [[CAST_FIXED:%.*]] = tail call <8 x bfloat> @llvm.vector.extract.v8bf16.nxv8bf16(<vscale x 8 x bfloat> [[X:%.*]], i64 0)
 // CHECK-128-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_BFLOAT16:%.*]], ptr [[S:%.*]], i64 0, i32 1
-// CHECK-128-NEXT:    store <8 x bfloat> [[CASTFIXEDSVE]], ptr [[Y]], align 16, !tbaa [[TBAA2]]
+// CHECK-128-NEXT:    store <8 x bfloat> [[CAST_FIXED]], ptr [[Y]], align 16, !tbaa [[TBAA2]]
 // CHECK-128-NEXT:    ret void
 //
-// CHECK-256-LABEL: define {{[^@]+}}@write_bfloat16(
+// CHECK-256-LABEL: @write_bfloat16(
 // CHECK-256-NEXT:  entry:
-// CHECK-256-NEXT:    [[CASTFIXEDSVE:%.*]] = tail call <16 x bfloat> @llvm.vector.extract.v16bf16.nxv8bf16(<vscale x 8 x bfloat> [[X:%.*]], i64 0)
+// CHECK-256-NEXT:    [[CAST_FIXED:%.*]] = tail call <16 x bfloat> @llvm.vector.extract.v16bf16.nxv8bf16(<vscale x 8 x bfloat> [[X:%.*]], i64 0)
 // CHECK-256-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_BFLOAT16:%.*]], ptr [[S:%.*]], i64 0, i32 1
-// CHECK-256-NEXT:    store <16 x bfloat> [[CASTFIXEDSVE]], ptr [[Y]], align 16, !tbaa [[TBAA2]]
+// CHECK-256-NEXT:    store <16 x bfloat> [[CAST_FIXED]], ptr [[Y]], align 16, !tbaa [[TBAA2]]
 // CHECK-256-NEXT:    ret void
 //
-// CHECK-512-LABEL: define {{[^@]+}}@write_bfloat16(
+// CHECK-512-LABEL: @write_bfloat16(
 // CHECK-512-NEXT:  entry:
-// CHECK-512-NEXT:    [[CASTFIXEDSVE:%.*]] = tail call <32 x bfloat> @llvm.vector.extract.v32bf16.nxv8bf16(<vscale x 8 x bfloat> [[X:%.*]], i64 0)
+// CHECK-512-NEXT:    [[CAST_FIXED:%.*]] = tail call <32 x bfloat> @llvm.vector.extract.v32bf16.nxv8bf16(<vscale x 8 x bfloat> [[X:%.*]], i64 0)
 // CHECK-512-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_BFLOAT16:%.*]], ptr [[S:%.*]], i64 0, i32 1
-// CHECK-512-NEXT:    store <32 x bfloat> [[CASTFIXEDSVE]], ptr [[Y]], align 16, !tbaa [[TBAA2]]
+// CHECK-512-NEXT:    store <32 x bfloat> [[CAST_FIXED]], ptr [[Y]], align 16, !tbaa [[TBAA2]]
 // CHECK-512-NEXT:    ret void
 //
 void write_bfloat16(struct struct_bfloat16 *s, svbfloat16_t x) {
@@ -190,56 +190,56 @@ void write_bfloat16(struct struct_bfloat16 *s, svbfloat16_t x) {
 // bool
 //===----------------------------------------------------------------------===//
 
-// CHECK-128-LABEL: define {{[^@]+}}@read_bool(
+// CHECK-128-LABEL: @read_bool(
 // CHECK-128-NEXT:  entry:
 // CHECK-128-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_BOOL:%.*]], ptr [[S:%.*]], i64 0, i32 1
 // CHECK-128-NEXT:    [[TMP0:%.*]] = load <2 x i8>, ptr [[Y]], align 2, !tbaa [[TBAA2]]
-// CHECK-128-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v2i8(<vscale x 2 x i8> undef, <2 x i8> [[TMP0]], i64 0)
-// CHECK-128-NEXT:    [[TMP1:%.*]] = bitcast <vscale x 2 x i8> [[CASTSCALABLESVE]] to <vscale x 16 x i1>
+// CHECK-128-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v2i8(<vscale x 2 x i8> undef, <2 x i8> [[TMP0]], i64 0)
+// CHECK-128-NEXT:    [[TMP1:%.*]] = bitcast <vscale x 2 x i8> [[CAST_SCALABLE]] to <vscale x 16 x i1>
 // CHECK-128-NEXT:    ret <vscale x 16 x i1> [[TMP1]]
 //
-// CHECK-256-LABEL: define {{[^@]+}}@read_bool(
+// CHECK-256-LABEL: @read_bool(
 // CHECK-256-NEXT:  entry:
 // CHECK-256-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_BOOL:%.*]], ptr [[S:%.*]], i64 0, i32 1
 // CHECK-256-NEXT:    [[TMP0:%.*]] = load <4 x i8>, ptr [[Y]], align 2, !tbaa [[TBAA2]]
-// CHECK-256-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v4i8(<vscale x 2 x i8> undef, <4 x i8> [[TMP0]], i64 0)
-// CHECK-256-NEXT:    [[TMP1:%.*]] = bitcast <vscale x 2 x i8> [[CASTSCALABLESVE]] to <vscale x 16 x i1>
+// CHECK-256-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v4i8(<vscale x 2 x i8> undef, <4 x i8> [[TMP0]], i64 0)
+// CHECK-256-NEXT:    [[TMP1:%.*]] = bitcast <vscale x 2 x i8> [[CAST_SCALABLE]] to <vscale x 16 x i1>
 // CHECK-256-NEXT:    ret <vscale x 16 x i1> [[TMP1]]
 //
-// CHECK-512-LABEL: define {{[^@]+}}@read_bool(
+// CHECK-512-LABEL: @read_bool(
 // CHECK-512-NEXT:  entry:
 // CHECK-512-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_BOOL:%.*]], ptr [[S:%.*]], i64 0, i32 1
 // CHECK-512-NEXT:    [[TMP0:%.*]] = load <8 x i8>, ptr [[Y]], align 2, !tbaa [[TBAA2]]
-// CHECK-512-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> undef, <8 x i8> [[TMP0]], i64 0)
-// CHECK-512-NEXT:    [[TMP1:%.*]] = bitcast <vscale x 2 x i8> [[CASTSCALABLESVE]] to <vscale x 16 x i1>
+// CHECK-512-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 2 x i8> @llvm.vector.insert.nxv2i8.v8i8(<vscale x 2 x i8> undef, <8 x i8> [[TMP0]], i64 0)
+// CHECK-512-NEXT:    [[TMP1:%.*]] = bitcast <vscale x 2 x i8> [[CAST_SCALABLE]] to <vscale x 16 x i1>
 // CHECK-512-NEXT:    ret <vscale x 16 x i1> [[TMP1]]
 //
 svbool_t read_bool(struct struct_bool *s) {
   return s->y[0];
 }
 
-// CHECK-128-LABEL: define {{[^@]+}}@write_bool(
+// CHECK-128-LABEL: @write_bool(
 // CHECK-128-NEXT:  entry:
 // CHECK-128-NEXT:    [[TMP0:%.*]] = bitcast <vscale x 16 x i1> [[X:%.*]] to <vscale x 2 x i8>
-// CHECK-128-NEXT:    [[CASTFIXEDSVE:%.*]] = tail call <2 x i8> @llvm.vector.extract.v2i8.nxv2i8(<vscale x 2 x i8> [[TMP0]], i64 0)
+// CHECK-128-NEXT:    [[CAST_FIXED:%.*]] = tail call <2 x i8> @llvm.vector.extract.v2i8.nxv2i8(<vscale x 2 x i8> [[TMP0]], i64 0)
 // CHECK-128-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_BOOL:%.*]], ptr [[S:%.*]], i64 0, i32 1
-// CHECK-128-NEXT:    store <2 x i8> [[CASTFIXEDSVE]], ptr [[Y]], align 2, !tbaa [[TBAA2]]
+// CHECK-128-NEXT:    store <2 x i8> [[CAST_FIXED]], ptr [[Y]], align 2, !tbaa [[TBAA2]]
 // CHECK-128-NEXT:    ret void
 //
-// CHECK-256-LABEL: define {{[^@]+}}@write_bool(
+// CHECK-256-LABEL: @write_bool(
 // CHECK-256-NEXT:  entry:
 // CHECK-256-NEXT:    [[TMP0:%.*]] = bitcast <vscale x 16 x i1> [[X:%.*]] to <vscale x 2 x i8>
-// CHECK-256-NEXT:    [[CASTFIXEDSVE:%.*]] = tail call <4 x i8> @llvm.vector.extract.v4i8.nxv2i8(<vscale x 2 x i8> [[TMP0]], i64 0)
+// CHECK-256-NEXT:    [[CAST_FIXED:%.*]] = tail call <4 x i8> @llvm.vector.extract.v4i8.nxv2i8(<vscale x 2 x i8> [[TMP0]], i64 0)
 // CHECK-256-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_BOOL:%.*]], ptr [[S:%.*]], i64 0, i32 1
-// CHECK-256-NEXT:    store <4 x i8> [[CASTFIXEDSVE]], ptr [[Y]], align 2, !tbaa [[TBAA2]]
+// CHECK-256-NEXT:    store <4 x i8> [[CAST_FIXED]], ptr [[Y]], align 2, !tbaa [[TBAA2]]
 // CHECK-256-NEXT:    ret void
 //
-// CHECK-512-LABEL: define {{[^@]+}}@write_bool(
+// CHECK-512-LABEL: @write_bool(
 // CHECK-512-NEXT:  entry:
 // CHECK-512-NEXT:    [[TMP0:%.*]] = bitcast <vscale x 16 x i1> [[X:%.*]] to <vscale x 2 x i8>
-// CHECK-512-NEXT:    [[CASTFIXEDSVE:%.*]] = tail call <8 x i8> @llvm.vector.extract.v8i8.nxv2i8(<vscale x 2 x i8> [[TMP0]], i64 0)
+// CHECK-512-NEXT:    [[CAST_FIXED:%.*]] = tail call <8 x i8> @llvm.vector.extract.v8i8.nxv2i8(<vscale x 2 x i8> [[TMP0]], i64 0)
 // CHECK-512-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_BOOL:%.*]], ptr [[S:%.*]], i64 0, i32 1
-// CHECK-512-NEXT:    store <8 x i8> [[CASTFIXEDSVE]], ptr [[Y]], align 2, !tbaa [[TBAA2]]
+// CHECK-512-NEXT:    store <8 x i8> [[CAST_FIXED]], ptr [[Y]], align 2, !tbaa [[TBAA2]]
 // CHECK-512-NEXT:    ret void
 //
 void write_bool(struct struct_bool *s, svbool_t x) {
diff --git a/clang/test/CodeGen/attr-counted-by.c b/clang/test/CodeGen/attr-counted-by.c
new file mode 100644
index 0000000000..a7eb0da6dd
--- /dev/null
+++ b/clang/test/CodeGen/attr-counted-by.c
@@ -0,0 +1,227 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 3
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -DCOUNTED_BY -O2 -Wall -fsanitize=array-bounds,object-size,local-bounds -fstrict-flex-arrays=3 -emit-llvm -o - %s | FileCheck --check-prefix=SANITIZE-WITH-ATTR %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -DCOUNTED_BY -O2 -Wall -fstrict-flex-arrays=3 -emit-llvm -o - %s | FileCheck --check-prefix=NO-SANITIZE-WITH-ATTR %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -O2 -Wall -fsanitize=array-bounds,object-size,local-bounds -fstrict-flex-arrays=3 -emit-llvm -o - %s | FileCheck --check-prefix=SANITIZE-WITHOUT-ATTR %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -O2 -Wall -fstrict-flex-arrays=3 -emit-llvm -o - %s | FileCheck --check-prefix=NO-SANITIZE-WITHOUT-ATTR %s
+
+#if !__has_attribute(counted_by)
+#error "has attribute broken"
+#endif
+
+#ifdef COUNTED_BY
+#define __counted_by(member)    __attribute__((__counted_by__(member)))
+#else
+#define __counted_by(member)
+#endif
+
+typedef long unsigned int size_t;
+
+struct annotated {
+  unsigned long flags;
+  int count;
+  int array[] __counted_by(count);
+};
+
+// SANITIZE-WITH-ATTR-LABEL: define dso_local void @test1(
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[INDEX:%.*]], i32 noundef [[VAL:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// SANITIZE-WITH-ATTR-NEXT:  entry:
+// SANITIZE-WITH-ATTR-NEXT:    [[COUNT:%.*]] = getelementptr inbounds [[STRUCT_ANNOTATED:%.*]], ptr [[P]], i64 0, i32 1
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[COUNT]], align 8, !tbaa [[TBAA2:![0-9]+]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = sext i32 [[INDEX]] to i64, !nosanitize !6
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP0]] to i64, !nosanitize !6
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP1]], [[TMP2]], !nosanitize !6
+// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP3]], label [[CONT7:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF7:![0-9]+]], !nosanitize !6
+// SANITIZE-WITH-ATTR:       handler.out_of_bounds:
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP4:%.*]] = zext i32 [[INDEX]] to i64, !nosanitize !6
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB2:[0-9]+]], i64 [[TMP4]]) #[[ATTR2:[0-9]+]], !nosanitize !6
+// SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize !6
+// SANITIZE-WITH-ATTR:       cont7:
+// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [[STRUCT_ANNOTATED]], ptr [[P]], i64 0, i32 2, i64 [[TMP1]]
+// SANITIZE-WITH-ATTR-NEXT:    store i32 [[VAL]], ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
+// SANITIZE-WITH-ATTR-NEXT:    ret void
+//
+// NO-SANITIZE-WITH-ATTR-LABEL: define dso_local void @test1(
+// NO-SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef writeonly [[P:%.*]], i32 noundef [[INDEX:%.*]], i32 noundef [[VAL:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// NO-SANITIZE-WITH-ATTR-NEXT:  entry:
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [[STRUCT_ANNOTATED:%.*]], ptr [[P]], i64 0, i32 2, i64 [[IDXPROM]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    store i32 [[VAL]], ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2:![0-9]+]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    ret void
+//
+// SANITIZE-WITHOUT-ATTR-LABEL: define dso_local void @test1(
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[INDEX:%.*]], i32 noundef [[VAL:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// SANITIZE-WITHOUT-ATTR-NEXT:  entry:
+// SANITIZE-WITHOUT-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
+// SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [[STRUCT_ANNOTATED:%.*]], ptr [[P]], i64 0, i32 2, i64 [[IDXPROM]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    store i32 [[VAL]], ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2:![0-9]+]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    ret void
+//
+// NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local void @test1(
+// NO-SANITIZE-WITHOUT-ATTR-SAME: ptr nocapture noundef writeonly [[P:%.*]], i32 noundef [[INDEX:%.*]], i32 noundef [[VAL:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:  entry:
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [[STRUCT_ANNOTATED:%.*]], ptr [[P]], i64 0, i32 2, i64 [[IDXPROM]]
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    store i32 [[VAL]], ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2:![0-9]+]]
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret void
+//
+void test1(struct annotated *p, int index, int val) {
+  p->array[index] = val;
+}
+
+// SANITIZE-WITH-ATTR-LABEL: define dso_local void @test2(
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]], i64 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITH-ATTR-NEXT:  entry:
+// SANITIZE-WITH-ATTR-NEXT:    [[COUNT:%.*]] = getelementptr inbounds [[STRUCT_ANNOTATED:%.*]], ptr [[P]], i64 0, i32 1
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[COUNT]], align 8, !tbaa [[TBAA2]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64, !nosanitize !6
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = icmp ugt i64 [[TMP1]], [[INDEX]], !nosanitize !6
+// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP2]], label [[CONT12:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF7]], !nosanitize !6
+// SANITIZE-WITH-ATTR:       handler.out_of_bounds:
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB4:[0-9]+]], i64 [[INDEX]]) #[[ATTR2]], !nosanitize !6
+// SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize !6
+// SANITIZE-WITH-ATTR:       cont12:
+// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [[STRUCT_ANNOTATED]], ptr [[P]], i64 0, i32 2, i64 [[INDEX]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP0]], 2
+// SANITIZE-WITH-ATTR-NEXT:    store i32 [[TMP3]], ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
+// SANITIZE-WITH-ATTR-NEXT:    ret void
+//
+// NO-SANITIZE-WITH-ATTR-LABEL: define dso_local void @test2(
+// NO-SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef [[P:%.*]], i64 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
+// NO-SANITIZE-WITH-ATTR-NEXT:  entry:
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNT:%.*]] = getelementptr inbounds [[STRUCT_ANNOTATED:%.*]], ptr [[P]], i64 0, i32 1
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[COUNT]], align 8, !tbaa [[TBAA2]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = shl i32 [[TMP0]], 2
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [[STRUCT_ANNOTATED]], ptr [[P]], i64 0, i32 2, i64 [[INDEX]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    store i32 [[TMP1]], ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    ret void
+//
+// SANITIZE-WITHOUT-ATTR-LABEL: define dso_local void @test2(
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]], i64 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITHOUT-ATTR-NEXT:  entry:
+// SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [[STRUCT_ANNOTATED:%.*]], ptr [[P]], i64 0, i32 2, i64 [[INDEX]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    store i32 -1, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    ret void
+//
+// NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local void @test2(
+// NO-SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]], i64 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:  entry:
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [[STRUCT_ANNOTATED:%.*]], ptr [[P]], i64 0, i32 2, i64 [[INDEX]]
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    store i32 -1, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret void
+//
+void test2(struct annotated *p, size_t index) {
+  p->array[index] = __builtin_dynamic_object_size(p->array, 1);
+}
+
+// SANITIZE-WITH-ATTR-LABEL: define dso_local void @test3(
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]], i64 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITH-ATTR-NEXT:  entry:
+// SANITIZE-WITH-ATTR-NEXT:    [[COUNT:%.*]] = getelementptr inbounds [[STRUCT_ANNOTATED:%.*]], ptr [[P]], i64 0, i32 1
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[COUNT]], align 8, !tbaa [[TBAA2]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64, !nosanitize !6
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = icmp ugt i64 [[TMP1]], [[INDEX]], !nosanitize !6
+// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP2]], label [[CONT12:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF7]], !nosanitize !6
+// SANITIZE-WITH-ATTR:       handler.out_of_bounds:
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB5:[0-9]+]], i64 [[INDEX]]) #[[ATTR2]], !nosanitize !6
+// SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize !6
+// SANITIZE-WITH-ATTR:       cont12:
+// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [[STRUCT_ANNOTATED]], ptr [[P]], i64 0, i32 2, i64 [[INDEX]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP0]], 2
+// SANITIZE-WITH-ATTR-NEXT:    [[CONV:%.*]] = add i32 [[TMP3]], 16
+// SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV]], ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
+// SANITIZE-WITH-ATTR-NEXT:    ret void
+//
+// NO-SANITIZE-WITH-ATTR-LABEL: define dso_local void @test3(
+// NO-SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef [[P:%.*]], i64 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR1]] {
+// NO-SANITIZE-WITH-ATTR-NEXT:  entry:
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[COUNT:%.*]] = getelementptr inbounds [[STRUCT_ANNOTATED:%.*]], ptr [[P]], i64 0, i32 1
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[COUNT]], align 8, !tbaa [[TBAA2]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = shl i32 [[TMP0]], 2
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[CONV:%.*]] = add i32 [[TMP1]], 16
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [[STRUCT_ANNOTATED]], ptr [[P]], i64 0, i32 2, i64 [[INDEX]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV]], ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    ret void
+//
+// SANITIZE-WITHOUT-ATTR-LABEL: define dso_local void @test3(
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]], i64 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITHOUT-ATTR-NEXT:  entry:
+// SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [[STRUCT_ANNOTATED:%.*]], ptr [[P]], i64 0, i32 2, i64 [[INDEX]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    store i32 -1, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    ret void
+//
+// NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local void @test3(
+// NO-SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]], i64 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR1]] {
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:  entry:
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [[STRUCT_ANNOTATED:%.*]], ptr [[P]], i64 0, i32 2, i64 [[INDEX]]
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    store i32 -1, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret void
+//
+void test3(struct annotated *p, size_t index) {
+  // This test differs from 'test2' by checking bdos on the whole array and not
+  // just the FAM.
+  p->array[index] = __builtin_dynamic_object_size(p, 1);
+}
+
+struct annotated_with_anon_struct {
+  unsigned long flags;
+  struct {
+    unsigned char count;
+    int array[] __counted_by(count);
+  };
+};
+
+// SANITIZE-WITH-ATTR-LABEL: define dso_local void @test4(
+// SANITIZE-WITH-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITH-ATTR-NEXT:  entry:
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[STRUCT_ANNOTATED_WITH_ANON_STRUCT:%.*]], ptr [[P]], i64 0, i32 1
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = load i8, ptr [[TMP0]], align 8, !tbaa [[TBAA8:![0-9]+]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = sext i32 [[INDEX]] to i64, !nosanitize !6
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP3:%.*]] = zext i8 [[TMP1]] to i64, !nosanitize !6
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP2]], [[TMP3]], !nosanitize !6
+// SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP4]], label [[CONT18:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF7]], !nosanitize !6
+// SANITIZE-WITH-ATTR:       handler.out_of_bounds:
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP5:%.*]] = zext i32 [[INDEX]] to i64, !nosanitize !6
+// SANITIZE-WITH-ATTR-NEXT:    tail call void @__ubsan_handle_out_of_bounds_abort(ptr nonnull @[[GLOB6:[0-9]+]], i64 [[TMP5]]) #[[ATTR2]], !nosanitize !6
+// SANITIZE-WITH-ATTR-NEXT:    unreachable, !nosanitize !6
+// SANITIZE-WITH-ATTR:       cont18:
+// SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 12
+// SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARRAY]], i64 0, i64 [[TMP2]]
+// SANITIZE-WITH-ATTR-NEXT:    [[TMP6:%.*]] = shl i8 [[TMP1]], 2
+// SANITIZE-WITH-ATTR-NEXT:    [[CONV:%.*]] = zext i8 [[TMP6]] to i32
+// SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV]], ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
+// SANITIZE-WITH-ATTR-NEXT:    ret void
+//
+// NO-SANITIZE-WITH-ATTR-LABEL: define dso_local void @test4(
+// NO-SANITIZE-WITH-ATTR-SAME: ptr nocapture noundef [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR1]] {
+// NO-SANITIZE-WITH-ATTR-NEXT:  entry:
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[STRUCT_ANNOTATED_WITH_ANON_STRUCT:%.*]], ptr [[P]], i64 0, i32 1
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = load i8, ptr [[TMP0]], align 8, !tbaa [[TBAA6:![0-9]+]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[TMP2:%.*]] = shl i8 [[TMP1]], 2
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[CONV:%.*]] = zext i8 [[TMP2]] to i32
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 12
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
+// NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    store i32 [[CONV]], ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    ret void
+//
+// SANITIZE-WITHOUT-ATTR-LABEL: define dso_local void @test4(
+// SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// SANITIZE-WITHOUT-ATTR-NEXT:  entry:
+// SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 12
+// SANITIZE-WITHOUT-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
+// SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    store i32 -1, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    ret void
+//
+// NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local void @test4(
+// NO-SANITIZE-WITHOUT-ATTR-SAME: ptr noundef [[P:%.*]], i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR1]] {
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:  entry:
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 12
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i32], ptr [[ARRAY]], i64 0, i64 [[IDXPROM]]
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    store i32 -1, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    ret void
+//
+void test4(struct annotated_with_anon_struct *p, int index) {
+  p->array[index] = __builtin_dynamic_object_size(p->array, 1);
+}
diff --git a/clang/test/CodeGen/attr-cpuspecific-cpus.c b/clang/test/CodeGen/attr-cpuspecific-cpus.c
index f26871324e..dd154fd227 100644
--- a/clang/test/CodeGen/attr-cpuspecific-cpus.c
+++ b/clang/test/CodeGen/attr-cpuspecific-cpus.c
@@ -84,3 +84,5 @@ ATTR(cpu_specific(arrowlake)) void CPU35(void){}
 ATTR(cpu_specific(arrowlake_s)) void CPU36(void){}
 ATTR(cpu_specific(lunarlake)) void CPU37(void){}
 ATTR(cpu_specific(gracemont)) void CPU38(void){}
+ATTR(cpu_specific(pantherlake)) void CPU39(void){}
+ATTR(cpu_specific(clearwaterforest)) void CPU40(void){}
diff --git a/clang/test/CodeGen/attr-riscv-rvv-vector-bits-bitcast.c b/clang/test/CodeGen/attr-riscv-rvv-vector-bits-bitcast.c
index c5a3cb0fcf..886af083f1 100644
--- a/clang/test/CodeGen/attr-riscv-rvv-vector-bits-bitcast.c
+++ b/clang/test/CodeGen/attr-riscv-rvv-vector-bits-bitcast.c
@@ -37,22 +37,22 @@ DEFINE_STRUCT(float64m1)
 // CHECK-64-NEXT:  entry:
 // CHECK-64-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_INT64M1:%.*]], ptr [[S:%.*]], i64 0, i32 1
 // CHECK-64-NEXT:    [[TMP0:%.*]] = load <1 x i64>, ptr [[Y]], align 8, !tbaa [[TBAA4:![0-9]+]]
-// CHECK-64-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v1i64(<vscale x 1 x i64> undef, <1 x i64> [[TMP0]], i64 0)
-// CHECK-64-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+// CHECK-64-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v1i64(<vscale x 1 x i64> undef, <1 x i64> [[TMP0]], i64 0)
+// CHECK-64-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 // CHECK-128-LABEL: @read_int64m1(
 // CHECK-128-NEXT:  entry:
 // CHECK-128-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_INT64M1:%.*]], ptr [[S:%.*]], i64 0, i32 1
 // CHECK-128-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr [[Y]], align 8, !tbaa [[TBAA4:![0-9]+]]
-// CHECK-128-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v2i64(<vscale x 1 x i64> undef, <2 x i64> [[TMP0]], i64 0)
-// CHECK-128-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+// CHECK-128-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v2i64(<vscale x 1 x i64> undef, <2 x i64> [[TMP0]], i64 0)
+// CHECK-128-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 // CHECK-256-LABEL: @read_int64m1(
 // CHECK-256-NEXT:  entry:
 // CHECK-256-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_INT64M1:%.*]], ptr [[S:%.*]], i64 0, i32 1
 // CHECK-256-NEXT:    [[TMP0:%.*]] = load <4 x i64>, ptr [[Y]], align 8, !tbaa [[TBAA4:![0-9]+]]
-// CHECK-256-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[TMP0]], i64 0)
-// CHECK-256-NEXT:    ret <vscale x 1 x i64> [[CASTSCALABLESVE]]
+// CHECK-256-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 1 x i64> @llvm.vector.insert.nxv1i64.v4i64(<vscale x 1 x i64> undef, <4 x i64> [[TMP0]], i64 0)
+// CHECK-256-NEXT:    ret <vscale x 1 x i64> [[CAST_SCALABLE]]
 //
 vint64m1_t read_int64m1(struct struct_int64m1 *s) {
   return s->y[0];
@@ -60,23 +60,23 @@ vint64m1_t read_int64m1(struct struct_int64m1 *s) {
 
 // CHECK-64-LABEL: @write_int64m1(
 // CHECK-64-NEXT:  entry:
-// CHECK-64-NEXT:    [[CASTFIXEDSVE:%.*]] = tail call <1 x i64> @llvm.vector.extract.v1i64.nxv1i64(<vscale x 1 x i64> [[X:%.*]], i64 0)
+// CHECK-64-NEXT:    [[CAST_FIXED:%.*]] = tail call <1 x i64> @llvm.vector.extract.v1i64.nxv1i64(<vscale x 1 x i64> [[X:%.*]], i64 0)
 // CHECK-64-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_INT64M1:%.*]], ptr [[S:%.*]], i64 0, i32 1
-// CHECK-64-NEXT:    store <1 x i64> [[CASTFIXEDSVE]], ptr [[Y]], align 8, !tbaa [[TBAA4]]
+// CHECK-64-NEXT:    store <1 x i64> [[CAST_FIXED]], ptr [[Y]], align 8, !tbaa [[TBAA4]]
 // CHECK-64-NEXT:    ret void
 //
 // CHECK-128-LABEL: @write_int64m1(
 // CHECK-128-NEXT:  entry:
-// CHECK-128-NEXT:    [[CASTFIXEDSVE:%.*]] = tail call <2 x i64> @llvm.vector.extract.v2i64.nxv1i64(<vscale x 1 x i64> [[X:%.*]], i64 0)
+// CHECK-128-NEXT:    [[CAST_FIXED:%.*]] = tail call <2 x i64> @llvm.vector.extract.v2i64.nxv1i64(<vscale x 1 x i64> [[X:%.*]], i64 0)
 // CHECK-128-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_INT64M1:%.*]], ptr [[S:%.*]], i64 0, i32 1
-// CHECK-128-NEXT:    store <2 x i64> [[CASTFIXEDSVE]], ptr [[Y]], align 8, !tbaa [[TBAA4]]
+// CHECK-128-NEXT:    store <2 x i64> [[CAST_FIXED]], ptr [[Y]], align 8, !tbaa [[TBAA4]]
 // CHECK-128-NEXT:    ret void
 //
 // CHECK-256-LABEL: @write_int64m1(
 // CHECK-256-NEXT:  entry:
-// CHECK-256-NEXT:    [[CASTFIXEDSVE:%.*]] = tail call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[X:%.*]], i64 0)
+// CHECK-256-NEXT:    [[CAST_FIXED:%.*]] = tail call <4 x i64> @llvm.vector.extract.v4i64.nxv1i64(<vscale x 1 x i64> [[X:%.*]], i64 0)
 // CHECK-256-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_INT64M1:%.*]], ptr [[S:%.*]], i64 0, i32 1
-// CHECK-256-NEXT:    store <4 x i64> [[CASTFIXEDSVE]], ptr [[Y]], align 8, !tbaa [[TBAA4]]
+// CHECK-256-NEXT:    store <4 x i64> [[CAST_FIXED]], ptr [[Y]], align 8, !tbaa [[TBAA4]]
 // CHECK-256-NEXT:    ret void
 //
 void write_int64m1(struct struct_int64m1 *s, vint64m1_t x) {
@@ -91,22 +91,22 @@ void write_int64m1(struct struct_int64m1 *s, vint64m1_t x) {
 // CHECK-64-NEXT:  entry:
 // CHECK-64-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_FLOAT64M1:%.*]], ptr [[S:%.*]], i64 0, i32 1
 // CHECK-64-NEXT:    [[TMP0:%.*]] = load <1 x double>, ptr [[Y]], align 8, !tbaa [[TBAA4]]
-// CHECK-64-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v1f64(<vscale x 1 x double> undef, <1 x double> [[TMP0]], i64 0)
-// CHECK-64-NEXT:    ret <vscale x 1 x double> [[CASTSCALABLESVE]]
+// CHECK-64-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v1f64(<vscale x 1 x double> undef, <1 x double> [[TMP0]], i64 0)
+// CHECK-64-NEXT:    ret <vscale x 1 x double> [[CAST_SCALABLE]]
 //
 // CHECK-128-LABEL: @read_float64m1(
 // CHECK-128-NEXT:  entry:
 // CHECK-128-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_FLOAT64M1:%.*]], ptr [[S:%.*]], i64 0, i32 1
 // CHECK-128-NEXT:    [[TMP0:%.*]] = load <2 x double>, ptr [[Y]], align 8, !tbaa [[TBAA4]]
-// CHECK-128-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v2f64(<vscale x 1 x double> undef, <2 x double> [[TMP0]], i64 0)
-// CHECK-128-NEXT:    ret <vscale x 1 x double> [[CASTSCALABLESVE]]
+// CHECK-128-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v2f64(<vscale x 1 x double> undef, <2 x double> [[TMP0]], i64 0)
+// CHECK-128-NEXT:    ret <vscale x 1 x double> [[CAST_SCALABLE]]
 //
 // CHECK-256-LABEL: @read_float64m1(
 // CHECK-256-NEXT:  entry:
 // CHECK-256-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_FLOAT64M1:%.*]], ptr [[S:%.*]], i64 0, i32 1
 // CHECK-256-NEXT:    [[TMP0:%.*]] = load <4 x double>, ptr [[Y]], align 8, !tbaa [[TBAA4]]
-// CHECK-256-NEXT:    [[CASTSCALABLESVE:%.*]] = tail call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> undef, <4 x double> [[TMP0]], i64 0)
-// CHECK-256-NEXT:    ret <vscale x 1 x double> [[CASTSCALABLESVE]]
+// CHECK-256-NEXT:    [[CAST_SCALABLE:%.*]] = tail call <vscale x 1 x double> @llvm.vector.insert.nxv1f64.v4f64(<vscale x 1 x double> undef, <4 x double> [[TMP0]], i64 0)
+// CHECK-256-NEXT:    ret <vscale x 1 x double> [[CAST_SCALABLE]]
 //
 vfloat64m1_t read_float64m1(struct struct_float64m1 *s) {
   return s->y[0];
@@ -114,23 +114,23 @@ vfloat64m1_t read_float64m1(struct struct_float64m1 *s) {
 
 // CHECK-64-LABEL: @write_float64m1(
 // CHECK-64-NEXT:  entry:
-// CHECK-64-NEXT:    [[CASTFIXEDSVE:%.*]] = tail call <1 x double> @llvm.vector.extract.v1f64.nxv1f64(<vscale x 1 x double> [[X:%.*]], i64 0)
+// CHECK-64-NEXT:    [[CAST_FIXED:%.*]] = tail call <1 x double> @llvm.vector.extract.v1f64.nxv1f64(<vscale x 1 x double> [[X:%.*]], i64 0)
 // CHECK-64-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_FLOAT64M1:%.*]], ptr [[S:%.*]], i64 0, i32 1
-// CHECK-64-NEXT:    store <1 x double> [[CASTFIXEDSVE]], ptr [[Y]], align 8, !tbaa [[TBAA4]]
+// CHECK-64-NEXT:    store <1 x double> [[CAST_FIXED]], ptr [[Y]], align 8, !tbaa [[TBAA4]]
 // CHECK-64-NEXT:    ret void
 //
 // CHECK-128-LABEL: @write_float64m1(
 // CHECK-128-NEXT:  entry:
-// CHECK-128-NEXT:    [[CASTFIXEDSVE:%.*]] = tail call <2 x double> @llvm.vector.extract.v2f64.nxv1f64(<vscale x 1 x double> [[X:%.*]], i64 0)
+// CHECK-128-NEXT:    [[CAST_FIXED:%.*]] = tail call <2 x double> @llvm.vector.extract.v2f64.nxv1f64(<vscale x 1 x double> [[X:%.*]], i64 0)
 // CHECK-128-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_FLOAT64M1:%.*]], ptr [[S:%.*]], i64 0, i32 1
-// CHECK-128-NEXT:    store <2 x double> [[CASTFIXEDSVE]], ptr [[Y]], align 8, !tbaa [[TBAA4]]
+// CHECK-128-NEXT:    store <2 x double> [[CAST_FIXED]], ptr [[Y]], align 8, !tbaa [[TBAA4]]
 // CHECK-128-NEXT:    ret void
 //
 // CHECK-256-LABEL: @write_float64m1(
 // CHECK-256-NEXT:  entry:
-// CHECK-256-NEXT:    [[CASTFIXEDSVE:%.*]] = tail call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[X:%.*]], i64 0)
+// CHECK-256-NEXT:    [[CAST_FIXED:%.*]] = tail call <4 x double> @llvm.vector.extract.v4f64.nxv1f64(<vscale x 1 x double> [[X:%.*]], i64 0)
 // CHECK-256-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_STRUCT_FLOAT64M1:%.*]], ptr [[S:%.*]], i64 0, i32 1
-// CHECK-256-NEXT:    store <4 x double> [[CASTFIXEDSVE]], ptr [[Y]], align 8, !tbaa [[TBAA4]]
+// CHECK-256-NEXT:    store <4 x double> [[CAST_FIXED]], ptr [[Y]], align 8, !tbaa [[TBAA4]]
 // CHECK-256-NEXT:    ret void
 //
 void write_float64m1(struct struct_float64m1 *s, vfloat64m1_t x) {
diff --git a/clang/test/CodeGen/attr-target-mv.c b/clang/test/CodeGen/attr-target-mv.c
index 301cb704f2..bdf8c49de4 100644
--- a/clang/test/CodeGen/attr-target-mv.c
+++ b/clang/test/CodeGen/attr-target-mv.c
@@ -26,6 +26,8 @@ int __attribute__((target("arch=arrowlake"))) foo(void) {return 21;}
 int __attribute__((target("arch=arrowlake-s"))) foo(void) {return 22;}
 int __attribute__((target("arch=lunarlake"))) foo(void) {return 23;}
 int __attribute__((target("arch=gracemont"))) foo(void) {return 24;}
+int __attribute__((target("arch=pantherlake"))) foo(void) {return 25;}
+int __attribute__((target("arch=clearwaterforest"))) foo(void) {return 26;}
 int __attribute__((target("default"))) foo(void) { return 2; }
 
 int bar(void) {
@@ -190,6 +192,10 @@ void calls_pr50025c(void) { pr50025c(); }
 // LINUX: ret i32 23
 // LINUX: define{{.*}} i32 @foo.arch_gracemont()
 // LINUX: ret i32 24
+// LINUX: define{{.*}} i32 @foo.arch_pantherlake()
+// LINUX: ret i32 25
+// LINUX: define{{.*}} i32 @foo.arch_clearwaterforest()
+// LINUX: ret i32 26
 // LINUX: define{{.*}} i32 @foo()
 // LINUX: ret i32 2
 // LINUX: define{{.*}} i32 @bar()
@@ -243,6 +249,10 @@ void calls_pr50025c(void) { pr50025c(); }
 // WINDOWS: ret i32 23
 // WINDOWS: define dso_local i32 @foo.arch_gracemont()
 // WINDOWS: ret i32 24
+// WINDOWS: define dso_local i32 @foo.arch_pantherlake()
+// WINDOWS: ret i32 25
+// WINDOWS: define dso_local i32 @foo.arch_clearwaterforest()
+// WINDOWS: ret i32 26
 // WINDOWS: define dso_local i32 @foo()
 // WINDOWS: ret i32 2
 // WINDOWS: define dso_local i32 @bar()
diff --git a/clang/test/CodeGen/attr-target-x86.c b/clang/test/CodeGen/attr-target-x86.c
index d261c007a0..3043986782 100644
--- a/clang/test/CodeGen/attr-target-x86.c
+++ b/clang/test/CodeGen/attr-target-x86.c
@@ -36,6 +36,9 @@ void __attribute__((target("arch=x86-64-v2"))) x86_64_v2(void) {}
 void __attribute__((target("arch=x86-64-v3"))) x86_64_v3(void) {}
 void __attribute__((target("arch=x86-64-v4"))) x86_64_v4(void) {}
 
+void __attribute__((target("avx10.1-256"))) avx10_1_256(void) {}
+void __attribute__((target("avx10.1-512"))) avx10_1_512(void) {}
+
 // Check that we emit the additional subtarget and cpu features for foo and not for baz or bar.
 // CHECK: baz{{.*}} #0
 // CHECK: foo{{.*}} #1
@@ -51,13 +54,15 @@ void __attribute__((target("arch=x86-64-v4"))) x86_64_v4(void) {}
 // CHECK: lake{{.*}} #7
 // CHECK: use_before_def{{.*}} #7
 // CHECK: walrus{{.*}} #8
+// CHECK: avx10_1_256{{.*}} #12
+// CHECK: avx10_1_512{{.*}} #13
 // CHECK: #0 = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87" "tune-cpu"="i686"
 // CHECK: #1 = {{.*}}"target-cpu"="ivybridge" "target-features"="+avx,+cmov,+crc32,+cx16,+cx8,+f16c,+fsgsbase,+fxsr,+mmx,+pclmul,+popcnt,+rdrnd,+sahf,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsaveopt"
 // CHECK-NOT: tune-cpu
-// CHECK: #2 = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87,-aes,-avx,-avx2,-avx512bf16,-avx512bitalg,-avx512bw,-avx512cd,-avx512dq,-avx512er,-avx512f,-avx512fp16,-avx512ifma,-avx512pf,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-avxifma,-avxneconvert,-avxvnni,-avxvnniint16,-avxvnniint8,-f16c,-fma,-fma4,-gfni,-kl,-pclmul,-sha,-sha512,-sm3,-sm4,-sse2,-sse3,-sse4.1,-sse4.2,-sse4a,-ssse3,-vaes,-vpclmulqdq,-widekl,-xop" "tune-cpu"="i686"
+// CHECK: #2 = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87,-aes,-avx,-avx10.1-256,-avx10.1-512,-avx2,-avx512bf16,-avx512bitalg,-avx512bw,-avx512cd,-avx512dq,-avx512er,-avx512f,-avx512fp16,-avx512ifma,-avx512pf,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-avxifma,-avxneconvert,-avxvnni,-avxvnniint16,-avxvnniint8,-f16c,-fma,-fma4,-gfni,-kl,-pclmul,-sha,-sha512,-sm3,-sm4,-sse2,-sse3,-sse4.1,-sse4.2,-sse4a,-ssse3,-vaes,-vpclmulqdq,-widekl,-xop" "tune-cpu"="i686"
 // CHECK: #3 = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+crc32,+cx8,+mmx,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87" "tune-cpu"="i686"
-// CHECK: #4 = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87,-avx,-avx2,-avx512bf16,-avx512bitalg,-avx512bw,-avx512cd,-avx512dq,-avx512er,-avx512f,-avx512fp16,-avx512ifma,-avx512pf,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-avxifma,-avxneconvert,-avxvnni,-avxvnniint16,-avxvnniint8,-f16c,-fma,-fma4,-sha512,-sm3,-sm4,-sse4.1,-sse4.2,-vaes,-vpclmulqdq,-xop" "tune-cpu"="i686"
-// CHECK: #5 = {{.*}}"target-cpu"="ivybridge" "target-features"="+avx,+cmov,+crc32,+cx16,+cx8,+f16c,+fsgsbase,+fxsr,+mmx,+pclmul,+popcnt,+rdrnd,+sahf,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsaveopt,-aes,-vaes"
+// CHECK: #4 = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87,-avx,-avx10.1-256,-avx10.1-512,-avx2,-avx512bf16,-avx512bitalg,-avx512bw,-avx512cd,-avx512dq,-avx512er,-avx512f,-avx512fp16,-avx512ifma,-avx512pf,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-avxifma,-avxneconvert,-avxvnni,-avxvnniint16,-avxvnniint8,-f16c,-fma,-fma4,-sha512,-sm3,-sm4,-sse4.1,-sse4.2,-vaes,-vpclmulqdq,-xop" "tune-cpu"="i686"
+// CHECK: #5 = {{.*}}"target-cpu"="ivybridge" "target-features"="+avx,+cmov,+crc32,+cx16,+cx8,+f16c,+fsgsbase,+fxsr,+mmx,+pclmul,+popcnt,+rdrnd,+sahf,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsaveopt,-aes,-avx10.1-256,-avx10.1-512,-vaes"
 // CHECK-NOT: tune-cpu
 // CHECK: #6 = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87,-3dnow,-3dnowa,-mmx"
 // CHECK: #7 = {{.*}}"target-cpu"="lakemont" "target-features"="+cx8,+mmx"
@@ -70,3 +75,6 @@ void __attribute__((target("arch=x86-64-v4"))) x86_64_v4(void) {}
 // CHECK-SAME: "target-features"="+avx,+avx2,+bmi,+bmi2,+cmov,+crc32,+cx16,+cx8,+f16c,+fma,+fxsr,+lzcnt,+mmx,+movbe,+popcnt,+sahf,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave"
 // CHECK: "target-cpu"="x86-64-v4"
 // CHECK-SAME: "target-features"="+avx,+avx2,+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512vl,+bmi,+bmi2,+cmov,+crc32,+cx16,+cx8,+evex512,+f16c,+fma,+fxsr,+lzcnt,+mmx,+movbe,+popcnt,+sahf,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave"
+
+// CHECK: #12 = {{.*}}"target-cpu"="i686" "target-features"="+aes,+avx,+avx10.1-256,+avx2,+avx512bf16,+avx512bitalg,+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512fp16,+avx512ifma,+avx512vbmi,+avx512vbmi2,+avx512vl,+avx512vnni,+avx512vpopcntdq,+cmov,+crc32,+cx8,+f16c,+fma,+mmx,+pclmul,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+vaes,+vpclmulqdq,+x87,+xsave,-avx10.1-512,-evex512"
+// CHECK: #13 = {{.*}}"target-cpu"="i686" "target-features"="+aes,+avx,+avx10.1-256,+avx10.1-512,+avx2,+avx512bf16,+avx512bitalg,+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512fp16,+avx512ifma,+avx512vbmi,+avx512vbmi2,+avx512vl,+avx512vnni,+avx512vpopcntdq,+cmov,+crc32,+cx8,+evex512,+f16c,+fma,+mmx,+pclmul,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+vaes,+vpclmulqdq,+x87,+xsave"
diff --git a/clang/test/CodeGen/bounds-checking.c b/clang/test/CodeGen/bounds-checking.c
index 636d4f289e..8100e30d06 100644
--- a/clang/test/CodeGen/bounds-checking.c
+++ b/clang/test/CodeGen/bounds-checking.c
@@ -69,7 +69,6 @@ int f7(union U *u, int i) {
   return u->c[i];
 }
 
-
 char B[10];
 char B2[10];
 // CHECK-LABEL: @f8
@@ -82,3 +81,12 @@ void f8(int i, int k) {
   // NOOPTARRAY: call void @llvm.ubsantrap(i8 4)
   B2[k] = '\0';
 }
+
+// See commit 9a954c6 that caused a SEGFAULT in this code.
+struct S {
+  __builtin_va_list ap;
+} *s;
+// CHECK-LABEL: @f9
+struct S *f9(int i) {
+  return &s[i];
+}
diff --git a/clang/test/CodeGen/builtin_vectorelements.c b/clang/test/CodeGen/builtin_vectorelements.c
new file mode 100644
index 0000000000..b0ff6f83b1
--- /dev/null
+++ b/clang/test/CodeGen/builtin_vectorelements.c
@@ -0,0 +1,127 @@
+// RUN: %clang_cc1 -O1 -triple x86_64                        %s -emit-llvm -disable-llvm-passes -o - | FileCheck --check-prefixes=CHECK       %s
+
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -O1 -triple aarch64 -target-feature +neon %s -emit-llvm -disable-llvm-passes -o - | FileCheck --check-prefixes=CHECK,NEON  %s
+
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -O1 -triple aarch64 -target-feature +sve  %s -emit-llvm -disable-llvm-passes -o - | FileCheck --check-prefixes=CHECK,SVE   %s
+
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -O1 -triple riscv64 -target-feature +v    %s -emit-llvm -disable-llvm-passes -o - | FileCheck --check-prefixes=CHECK,RISCV %s
+
+/// Note that this does not make sense to check for x86 SIMD types, because
+/// __m128i, __m256i, and __m512i do not specify the element type. There are no
+/// "logical" number of elements in them.
+
+typedef int int1 __attribute__((vector_size(4)));
+typedef int int4 __attribute__((vector_size(16)));
+typedef int int8 __attribute__((vector_size(32)));
+typedef int int16 __attribute__((vector_size(64)));
+typedef float float2 __attribute__((vector_size(8)));
+typedef long extLong4 __attribute__((ext_vector_type(4)));
+
+
+int test_builtin_vectorelements_int1() {
+  // CHECK-LABEL: i32 @test_builtin_vectorelements_int1(
+  // CHECK: ret i32 1
+  return __builtin_vectorelements(int1);
+}
+
+int test_builtin_vectorelements_int4() {
+  // CHECK-LABEL: i32 @test_builtin_vectorelements_int4(
+  // CHECK: ret i32 4
+  return __builtin_vectorelements(int4);
+}
+
+int test_builtin_vectorelements_int8() {
+  // CHECK-LABEL: i32 @test_builtin_vectorelements_int8(
+  // CHECK: ret i32 8
+  return __builtin_vectorelements(int8);
+}
+
+int test_builtin_vectorelements_int16() {
+  // CHECK-LABEL: i32 @test_builtin_vectorelements_int16(
+  // CHECK: ret i32 16
+  return __builtin_vectorelements(int16);
+}
+
+int test_builtin_vectorelements_float2() {
+  // CHECK-LABEL: i32 @test_builtin_vectorelements_float2(
+  // CHECK: ret i32 2
+  return __builtin_vectorelements(float2);
+}
+
+int test_builtin_vectorelements_extLong4() {
+  // CHECK-LABEL: i32 @test_builtin_vectorelements_extLong4(
+  // CHECK: ret i32 4
+  return __builtin_vectorelements(extLong4);
+}
+
+int test_builtin_vectorelements_multiply_constant() {
+  // CHECK-LABEL: i32 @test_builtin_vectorelements_multiply_constant(
+  // CHECK: ret i32 32
+  return __builtin_vectorelements(int16) * 2;
+}
+
+#if defined(__ARM_NEON)
+#include <arm_neon.h>
+
+int test_builtin_vectorelements_neon32x4() {
+  // NEON: i32 @test_builtin_vectorelements_neon32x4(
+  // NEON: ret i32 4
+  return __builtin_vectorelements(uint32x4_t);
+}
+
+int test_builtin_vectorelements_neon64x1() {
+  // NEON: i32 @test_builtin_vectorelements_neon64x1(
+  // NEON: ret i32 1
+  return __builtin_vectorelements(uint64x1_t);
+}
+#endif
+
+#if defined(__ARM_FEATURE_SVE)
+#include <arm_sve.h>
+
+long test_builtin_vectorelements_sve32() {
+  // SVE: i64 @test_builtin_vectorelements_sve32(
+  // SVE: [[VSCALE:%.+]] = call i64 @llvm.vscale.i64()
+  // SVE: [[RES:%.+]] = mul i64 [[VSCALE]], 4
+  // SVE: ret i64 [[RES]]
+  return __builtin_vectorelements(svuint32_t);
+}
+
+long test_builtin_vectorelements_sve8() {
+  // SVE: i64 @test_builtin_vectorelements_sve8(
+  // SVE: [[VSCALE:%.+]] = call i64 @llvm.vscale.i64()
+  // SVE: [[RES:%.+]] = mul i64 [[VSCALE]], 16
+  // SVE: ret i64 [[RES]]
+  return __builtin_vectorelements(svuint8_t);
+}
+#endif
+
+#if defined(__riscv)
+#include <riscv_vector.h>
+
+long test_builtin_vectorelements_riscv8() {
+  // RISCV: i64 @test_builtin_vectorelements_riscv8(
+  // RISCV: [[VSCALE:%.+]] = call i64 @llvm.vscale.i64()
+  // RISCV: [[RES:%.+]] = mul i64 [[VSCALE]], 8
+  // RISCV: ret i64 [[RES]]
+  return __builtin_vectorelements(vuint8m1_t);
+}
+
+long test_builtin_vectorelements_riscv64() {
+  // RISCV: i64 @test_builtin_vectorelements_riscv64(
+  // RISCV: [[VSCALE:%.+]] = call i64 @llvm.vscale.i64()
+  // RISCV: ret i64 [[VSCALE]]
+  return __builtin_vectorelements(vuint64m1_t);
+}
+
+long test_builtin_vectorelements_riscv32m2() {
+  // RISCV: i64 @test_builtin_vectorelements_riscv32m2(
+  // RISCV: [[VSCALE:%.+]] = call i64 @llvm.vscale.i64()
+  // RISCV: [[RES:%.+]] = mul i64 [[VSCALE]], 4
+  // RISCV: ret i64 [[RES]]
+  return __builtin_vectorelements(vuint32m2_t);
+}
+#endif
diff --git a/clang/test/CodeGen/catch-undef-behavior.c b/clang/test/CodeGen/catch-undef-behavior.c
index af37ef9e85..b33c13a68a 100644
--- a/clang/test/CodeGen/catch-undef-behavior.c
+++ b/clang/test/CodeGen/catch-undef-behavior.c
@@ -1,6 +1,8 @@
-// RUN: %clang_cc1 -fsanitize=alignment,null,object-size,shift-base,shift-exponent,return,signed-integer-overflow,vla-bound,float-cast-overflow,integer-divide-by-zero,bool,returns-nonnull-attribute,nonnull-attribute -fsanitize-recover=alignment,null,object-size,shift-base,shift-exponent,signed-integer-overflow,vla-bound,float-cast-overflow,integer-divide-by-zero,bool,returns-nonnull-attribute,nonnull-attribute -emit-llvm %s -o - -triple x86_64-linux-gnu | FileCheck %s --check-prefix=CHECK-COMMON --check-prefix=CHECK-UBSAN
-// RUN: %clang_cc1 -fsanitize-trap=alignment,null,object-size,shift-base,shift-exponent,return,signed-integer-overflow,vla-bound,float-cast-overflow,integer-divide-by-zero,bool,returns-nonnull-attribute,nonnull-attribute -fsanitize-recover=alignment,null,object-size,shift-base,shift-exponent,signed-integer-overflow,vla-bound,float-cast-overflow,integer-divide-by-zero,bool,returns-nonnull-attribute,nonnull-attribute -fsanitize=alignment,null,object-size,shift-base,shift-exponent,return,signed-integer-overflow,vla-bound,float-cast-overflow,integer-divide-by-zero,bool,returns-nonnull-attribute,nonnull-attribute -fsanitize-recover=alignment,null,object-size,shift-base,shift-exponent,signed-integer-overflow,vla-bound,float-cast-overflow,integer-divide-by-zero,bool,returns-nonnull-attribute,nonnull-attribute -emit-llvm %s -o - -triple x86_64-linux-gnu | FileCheck %s --check-prefix=CHECK-COMMON --check-prefix=CHECK-TRAP
+// RUN: %clang_cc1 -fsanitize=alignment,null,object-size,shift-base,shift-exponent,return,signed-integer-overflow,vla-bound,float-cast-overflow,integer-divide-by-zero,bool,returns-nonnull-attribute,nonnull-attribute -fsanitize-recover=alignment,null,object-size,shift-base,shift-exponent,signed-integer-overflow,vla-bound,float-cast-overflow,integer-divide-by-zero,bool,returns-nonnull-attribute,nonnull-attribute -emit-llvm %s -o - -triple x86_64-linux-gnu | FileCheck %s --check-prefixes=CHECK-COMMON,CHECK-UBSAN,CHECK-ALIGNMENT-BUILTIN
+// RUN: %clang_cc1 -fsanitize-trap=alignment,null,object-size,shift-base,shift-exponent,return,signed-integer-overflow,vla-bound,float-cast-overflow,integer-divide-by-zero,bool,returns-nonnull-attribute,nonnull-attribute -fsanitize-recover=alignment,null,object-size,shift-base,shift-exponent,signed-integer-overflow,vla-bound,float-cast-overflow,integer-divide-by-zero,bool,returns-nonnull-attribute,nonnull-attribute -fsanitize=alignment,null,object-size,shift-base,shift-exponent,return,signed-integer-overflow,vla-bound,float-cast-overflow,integer-divide-by-zero,bool,returns-nonnull-attribute,nonnull-attribute -fsanitize-recover=alignment,null,object-size,shift-base,shift-exponent,signed-integer-overflow,vla-bound,float-cast-overflow,integer-divide-by-zero,bool,returns-nonnull-attribute,nonnull-attribute -emit-llvm %s -o - -triple x86_64-linux-gnu | FileCheck %s --check-prefixes=CHECK-COMMON,CHECK-ALIGNMENT-BUILTIN,CHECK-TRAP
 // RUN: %clang_cc1 -fsanitize=signed-integer-overflow -emit-llvm %s -o - -triple x86_64-linux-gnu | FileCheck %s --check-prefix=CHECK-OVERFLOW
+/// A variant of CHECK-UBSAN with -sanitize-alignment-builtin disabled
+// RUN: %clang_cc1 -fsanitize=alignment,null,object-size,shift-base,shift-exponent,return,signed-integer-overflow,vla-bound,float-cast-overflow,integer-divide-by-zero,bool,returns-nonnull-attribute,nonnull-attribute -fsanitize-recover=alignment,null,object-size,shift-base,shift-exponent,signed-integer-overflow,vla-bound,float-cast-overflow,integer-divide-by-zero,bool,returns-nonnull-attribute,nonnull-attribute -emit-llvm %s -o - -triple x86_64-linux-gnu -mllvm -sanitize-alignment-builtin=0 | FileCheck %s --check-prefixes=CHECK-COMMON,CHECK-UBSAN-NO-ALIGNMENT-BUILTIN
 
 // CHECK-UBSAN: @[[INT:.*]] = private unnamed_addr constant { i16, i16, [6 x i8] } { i16 0, i16 11, [6 x i8] c"'int'\00" }
 
@@ -363,11 +365,13 @@ extern void *memcpy(void *, const void *, unsigned long) __attribute__((nonnull(
 void call_memcpy_nonnull(void *p, void *q, int sz) {
   // CHECK-COMMON: icmp ne ptr {{.*}}, null
   // CHECK-UBSAN: call void @__ubsan_handle_nonnull_arg
+  // CHECK-UBSAN-NO-ALIGNMENT-BUILTIN: call void @__ubsan_handle_nonnull_arg
   // CHECK-TRAP: call void @llvm.ubsantrap(i8 16)
   // CHECK-COMMON-NOT: call
 
   // CHECK-COMMON: icmp ne ptr {{.*}}, null
   // CHECK-UBSAN: call void @__ubsan_handle_nonnull_arg
+  // CHECK-UBSAN-NO-ALIGNMENT-BUILTIN: call void @__ubsan_handle_nonnull_arg
   // CHECK-TRAP: call void @llvm.ubsantrap(i8 16)
   // CHECK-COMMON-NOT: call
 
@@ -379,18 +383,23 @@ void call_memcpy_nonnull(void *p, void *q, int sz) {
 void call_memcpy(long *p, short *q, int sz) {
   // CHECK-COMMON: icmp ne ptr {{.*}}, null
   // CHECK-UBSAN: call void @__ubsan_handle_nonnull_arg(
+  // CHECK-UBSAN-NO-ALIGNMENT-BUILTIN: call void @__ubsan_handle_nonnull_arg(
   // CHECK-TRAP: call void @llvm.ubsantrap(i8 16)
-  // CHECK-COMMON: and i64 %[[#]], 7, !nosanitize
-  // CHECK-COMMON: icmp eq i64 %[[#]], 0, !nosanitize
+  // CHECK-ALIGNMENT-BUILTIN: and i64 %[[#]], 7, !nosanitize
+  // CHECK-ALIGNMENT-BUILTIN: icmp eq i64 %[[#]], 0, !nosanitize
   // CHECK-UBSAN: call void @__ubsan_handle_type_mismatch_v1(ptr @[[LINE_1600]]
+  // CHECK-UBSAN-NO-ALIGNMENT-BUILTIN-NOT: call void @__ubsan_handle_type_mismatch_v1(
   // CHECK-TRAP: call void @llvm.ubsantrap(i8 22)
 
   // CHECK-COMMON: icmp ne ptr {{.*}}, null
   // CHECK-UBSAN: call void @__ubsan_handle_nonnull_arg(
+  // CHECK-UBSAN-NO-ALIGNMENT-BUILTIN: call void @__ubsan_handle_nonnull_arg(
+  // CHECK-UBSAN-DISABLE-BUILTIN: call
   // CHECK-TRAP: call void @llvm.ubsantrap(i8 16)
-  // CHECK-COMMON: and i64 %[[#]], 1, !nosanitize
-  // CHECK-COMMON: icmp eq i64 %[[#]], 0, !nosanitize
+  // CHECK-ALIGNMENT-BUILTIN: and i64 %[[#]], 1, !nosanitize
+  // CHECK-ALIGNMENT-BUILTIN: icmp eq i64 %[[#]], 0, !nosanitize
   // CHECK-UBSAN: call void @__ubsan_handle_type_mismatch_v1(
+  // CHECK-UBSAN-NO-ALIGNMENT-BUILTIN-NOT: call
   // CHECK-TRAP: call void @llvm.ubsantrap(i8 22)
 
   // CHECK-COMMON: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %0, ptr align 2 %1, i64 %conv, i1 false)
@@ -405,14 +414,16 @@ void call_memcpy(long *p, short *q, int sz) {
 
 // CHECK-COMMON-LABEL: define{{.*}} void @call_memcpy_inline(
 void call_memcpy_inline(long *p, short *q) {
-  // CHECK-COMMON: and i64 %[[#]], 7, !nosanitize
-  // CHECK-COMMON: icmp eq i64 %[[#]], 0, !nosanitize
+  // CHECK-ALIGNMENT-BUILTIN: and i64 %[[#]], 7, !nosanitize
+  // CHECK-ALIGNMENT-BUILTIN: icmp eq i64 %[[#]], 0, !nosanitize
   // CHECK-UBSAN: call void @__ubsan_handle_type_mismatch_v1(
+  // CHECK-UBSAN-NO-ALIGNMENT-BUILTIN-NOT: call
   // CHECK-TRAP: call void @llvm.ubsantrap(i8 22)
 
-  // CHECK-COMMON: and i64 %[[#]], 1, !nosanitize
-  // CHECK-COMMON: icmp eq i64 %[[#]], 0, !nosanitize
+  // CHECK-ALIGNMENT-BUILTIN: and i64 %[[#]], 1, !nosanitize
+  // CHECK-ALIGNMENT-BUILTIN: icmp eq i64 %[[#]], 0, !nosanitize
   // CHECK-UBSAN: call void @__ubsan_handle_type_mismatch_v1(
+  // CHECK-UBSAN-NO-ALIGNMENT-BUILTIN-NOT: call
   // CHECK-TRAP: call void @llvm.ubsantrap(i8 22)
 
   // CHECK-COMMON: call void @llvm.memcpy.inline.p0.p0.i64(ptr align 8 %0, ptr align 2 %1, i64 2, i1 false)
@@ -425,10 +436,12 @@ extern void *memmove(void *, const void *, unsigned long) __attribute__((nonnull
 void call_memmove_nonnull(void *p, void *q, int sz) {
   // CHECK-COMMON: icmp ne ptr {{.*}}, null
   // CHECK-UBSAN: call void @__ubsan_handle_nonnull_arg
+  // CHECK-UBSAN-NO-ALIGNMENT-BUILTIN: call void @__ubsan_handle_nonnull_arg(
   // CHECK-TRAP: call void @llvm.ubsantrap(i8 16)
 
   // CHECK-COMMON: icmp ne ptr {{.*}}, null
   // CHECK-UBSAN: call void @__ubsan_handle_nonnull_arg
+  // CHECK-UBSAN-NO-ALIGNMENT-BUILTIN: call void @__ubsan_handle_nonnull_arg(
   // CHECK-TRAP: call void @llvm.ubsantrap(i8 16)
   memmove(p, q, sz);
 }
@@ -437,18 +450,22 @@ void call_memmove_nonnull(void *p, void *q, int sz) {
 void call_memmove(long *p, short *q, int sz) {
   // CHECK-COMMON: icmp ne ptr {{.*}}, null
   // CHECK-UBSAN: call void @__ubsan_handle_nonnull_arg(
+  // CHECK-UBSAN-NO-ALIGNMENT-BUILTIN: call void @__ubsan_handle_nonnull_arg(
   // CHECK-TRAP: call void @llvm.ubsantrap(i8 16)
-  // CHECK-COMMON: and i64 %[[#]], 7, !nosanitize
-  // CHECK-COMMON: icmp eq i64 %[[#]], 0, !nosanitize
+  // CHECK-ALIGNMENT-BUILTIN: and i64 %[[#]], 7, !nosanitize
+  // CHECK-ALIGNMENT-BUILTIN: icmp eq i64 %[[#]], 0, !nosanitize
   // CHECK-UBSAN: call void @__ubsan_handle_type_mismatch_v1(
+  // CHECK-UBSAN-NO-ALIGNMENT-BUILTIN-NOT: call
   // CHECK-TRAP: call void @llvm.ubsantrap(i8 22)
 
   // CHECK-COMMON: icmp ne ptr {{.*}}, null
   // CHECK-UBSAN: call void @__ubsan_handle_nonnull_arg(
+  // CHECK-UBSAN-NO-ALIGNMENT-BUILTIN: call void @__ubsan_handle_nonnull_arg(
   // CHECK-TRAP: call void @llvm.ubsantrap(i8 16)
-  // CHECK-COMMON: and i64 %[[#]], 1, !nosanitize
-  // CHECK-COMMON: icmp eq i64 %[[#]], 0, !nosanitize
+  // CHECK-ALIGNMENT-BUILTIN: and i64 %[[#]], 1, !nosanitize
+  // CHECK-ALIGNMENT-BUILTIN: icmp eq i64 %[[#]], 0, !nosanitize
   // CHECK-UBSAN: call void @__ubsan_handle_type_mismatch_v1(
+  // CHECK-UBSAN-NO-ALIGNMENT-BUILTIN-NOT: call
   // CHECK-TRAP: call void @llvm.ubsantrap(i8 22)
 
   // CHECK-COMMON: call void @llvm.memmove.p0.p0.i64(ptr align 8 %0, ptr align 2 %1, i64 %conv, i1 false)
diff --git a/clang/test/CodeGen/mrtd.c b/clang/test/CodeGen/mrtd.c
index e615cdfa67..c37a9ea95d 100644
--- a/clang/test/CodeGen/mrtd.c
+++ b/clang/test/CodeGen/mrtd.c
@@ -1,20 +1,24 @@
-// RUN: %clang_cc1 -mrtd -triple i386-unknown-unknown -std=c89 -emit-llvm -o - %s 2>&1 | FileCheck %s
-
-// CHECK: mrtd.c:10:3: warning: function with no prototype cannot use the stdcall calling convention
+// RUN: %clang_cc1 -mrtd -triple i386-unknown-unknown -std=c89 -emit-llvm -o - %s 2>&1 | FileCheck --check-prefixes=CHECK,X86 %s
+// RUN: %clang_cc1 -mrtd -triple m68k-unknown-unknown -std=c89 -emit-llvm -o - %s 2>&1 | FileCheck --check-prefixes=CHECK,M68K %s
 
 void baz(int arg);
 
-// CHECK: define{{.*}} x86_stdcallcc void @foo(i32 noundef %arg) [[NUW:#[0-9]+]]
+// X86: define{{.*}} x86_stdcallcc void @foo(i32 noundef %arg) [[NUW:#[0-9]+]]
+// M68K: define{{.*}} m68k_rtdcc void @foo(i32 noundef %arg)
 void foo(int arg) {
-// CHECK: call x86_stdcallcc i32 @bar(
+// X86: call x86_stdcallcc i32 @bar(
+#ifndef __mc68000__
   bar(arg);
-// CHECK: call x86_stdcallcc void @baz(i32
+#endif
+// X86: call x86_stdcallcc void @baz(i32
+// M68K: call m68k_rtdcc void @baz(i32
   baz(arg);
 }
 
-// CHECK: declare x86_stdcallcc i32 @bar(...)
+// X86: declare x86_stdcallcc i32 @bar(...)
 
-// CHECK: declare x86_stdcallcc void @baz(i32 noundef)
+// X86: declare x86_stdcallcc void @baz(i32 noundef)
+// M68K: declare m68k_rtdcc void @baz(i32 noundef)
 
 void qux(int arg, ...) { }
 // CHECK: define{{.*}} void @qux(i32 noundef %arg, ...)
@@ -22,7 +26,8 @@ void qux(int arg, ...) { }
 void quux(int a1, int a2, int a3) {
   qux(a1, a2, a3);
 }
-// CHECK-LABEL: define{{.*}} x86_stdcallcc void @quux
+// X86-LABEL: define{{.*}} x86_stdcallcc void @quux
+// M68K-LABEL: define{{.*}} m68k_rtdcc void @quux
 // CHECK: call void (i32, ...) @qux
 
-// CHECK: attributes [[NUW]] = { noinline nounwind{{.*}} }
+// X86: attributes [[NUW]] = { noinline nounwind{{.*}} }
diff --git a/clang/test/CodeGen/target-avx-abi-diag.c b/clang/test/CodeGen/target-avx-abi-diag.c
index 34995dc471..72de0fa2a3 100644
--- a/clang/test/CodeGen/target-avx-abi-diag.c
+++ b/clang/test/CodeGen/target-avx-abi-diag.c
@@ -5,6 +5,10 @@
 // RUN: %clang_cc1 %s -triple=x86_64-linux-gnu -target-feature +avx512f -target-feature -evex512 -verify=avx512-256 -DAVX512_ERR=1 -o - -S
 // RUN: %clang_cc1 %s -triple=x86_64-linux-gnu -target-feature +avx512f -target-feature -evex512 -verify=avx512-256 -DAVX512_ERR=2 -o - -S
 // RUN: %clang_cc1 %s -triple=x86_64-linux-gnu -target-feature +avx512f -target-feature -evex512 -verify=avx512-256 -DAVX512_ERR=3 -o - -S
+// RUN: %clang_cc1 %s -triple=x86_64-linux-gnu -target-feature +avx10.1-512 -verify=both -o - -S
+// RUN: %clang_cc1 %s -triple=x86_64-linux-gnu -target-feature +avx10.1-256 -verify=avx512-256 -DAVX512_ERR=1 -o - -S
+// RUN: %clang_cc1 %s -triple=x86_64-linux-gnu -target-feature +avx10.1-256 -verify=avx512-256 -DAVX512_ERR=2 -o - -S
+// RUN: %clang_cc1 %s -triple=x86_64-linux-gnu -target-feature +avx10.1-256 -verify=avx512-256 -DAVX512_ERR=3 -o - -S
 // REQUIRES: x86-registered-target
 
 // both-no-diagnostics
diff --git a/clang/test/CodeGen/target-builtin-noerror.c b/clang/test/CodeGen/target-builtin-noerror.c
index 60001fae1c..505f4a3e94 100644
--- a/clang/test/CodeGen/target-builtin-noerror.c
+++ b/clang/test/CodeGen/target-builtin-noerror.c
@@ -114,6 +114,8 @@ void verifycpustrings(void) {
   (void)__builtin_cpu_is("arrowlake");
   (void)__builtin_cpu_is("arrowlake-s");
   (void)__builtin_cpu_is("lunarlake");
+  (void)__builtin_cpu_is("clearwaterforest");
+  (void)__builtin_cpu_is("pantherlake");
   (void)__builtin_cpu_is("haswell");
   (void)__builtin_cpu_is("icelake-client");
   (void)__builtin_cpu_is("icelake-server");
diff --git a/clang/test/CodeGen/target-data.c b/clang/test/CodeGen/target-data.c
index 3649ddd651..c5cb922576 100644
--- a/clang/test/CodeGen/target-data.c
+++ b/clang/test/CodeGen/target-data.c
@@ -1,26 +1,26 @@
 // RUN: %clang_cc1 -triple i686-unknown-unknown -emit-llvm -o - %s | \
 // RUN:     FileCheck --check-prefix=I686-UNKNOWN %s
-// I686-UNKNOWN: target datalayout = "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-f64:32:64-f80:32-n8:16:32-S128"
+// I686-UNKNOWN: target datalayout = "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-i128:128-f64:32:64-f80:32-n8:16:32-S128"
 
 // RUN: %clang_cc1 -triple i686-apple-darwin9 -emit-llvm -o - %s | \
 // RUN:     FileCheck --check-prefix=I686-DARWIN %s
-// I686-DARWIN: target datalayout = "e-m:o-p:32:32-p270:32:32-p271:32:32-p272:64:64-f64:32:64-f80:128-n8:16:32-S128"
+// I686-DARWIN: target datalayout = "e-m:o-p:32:32-p270:32:32-p271:32:32-p272:64:64-i128:128-f64:32:64-f80:128-n8:16:32-S128"
 
 // RUN: %clang_cc1 -triple i686-unknown-win32 -emit-llvm -o - %s | \
 // RUN:     FileCheck --check-prefix=I686-WIN32 %s
-// I686-WIN32: target datalayout = "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32-a:0:32-S32"
+// I686-WIN32: target datalayout = "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32-a:0:32-S32"
 
 // RUN: %clang_cc1 -triple i686-unknown-cygwin -emit-llvm -o - %s | \
 // RUN:     FileCheck --check-prefix=I686-CYGWIN %s
-// I686-CYGWIN: target datalayout = "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:32-n8:16:32-a:0:32-S32"
+// I686-CYGWIN: target datalayout = "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:32-n8:16:32-a:0:32-S32"
 
 // RUN: %clang_cc1 -triple i686-pc-macho -emit-llvm -o - %s | \
 // RUN:     FileCheck --check-prefix=I686-MACHO %s
-// I686-MACHO: target datalayout = "e-m:o-p:32:32-p270:32:32-p271:32:32-p272:64:64-f64:32:64-f80:32-n8:16:32-S128"
+// I686-MACHO: target datalayout = "e-m:o-p:32:32-p270:32:32-p271:32:32-p272:64:64-i128:128-f64:32:64-f80:32-n8:16:32-S128"
 
 // RUN: %clang_cc1 -triple x86_64-unknown-unknown -emit-llvm -o - %s | \
 // RUN:     FileCheck --check-prefix=X86_64 %s
-// X86_64: target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+// X86_64: target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
 
 // RUN: %clang_cc1 -triple xcore-unknown-unknown -emit-llvm -o - %s | \
 // RUN:     FileCheck --check-prefix=XCORE %s
@@ -92,11 +92,11 @@
 
 // RUN: %clang_cc1 -triple i686-nacl -o - -emit-llvm %s | \
 // RUN: FileCheck %s -check-prefix=I686-NACL
-// I686-NACL: target datalayout = "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-n8:16:32-S128"
+// I686-NACL: target datalayout = "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n8:16:32-S128"
 
 // RUN: %clang_cc1 -triple x86_64-nacl -o - -emit-llvm %s | \
 // RUN: FileCheck %s -check-prefix=X86_64-NACL
-// X86_64-NACL: target datalayout = "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-n8:16:32:64-S128"
+// X86_64-NACL: target datalayout = "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n8:16:32:64-S128"
 
 // RUN: %clang_cc1 -triple arm-nacl -o - -emit-llvm %s | \
 // RUN: FileCheck %s -check-prefix=ARM-NACL
diff --git a/clang/test/CodeGenCXX/aarch64-sve-vector-init.cpp b/clang/test/CodeGenCXX/aarch64-sve-vector-init.cpp
index 2088e80acf..464275f164 100644
--- a/clang/test/CodeGenCXX/aarch64-sve-vector-init.cpp
+++ b/clang/test/CodeGenCXX/aarch64-sve-vector-init.cpp
@@ -55,6 +55,7 @@
 // CHECK-NEXT:    [[B8:%.*]] = alloca <vscale x 16 x i1>, align 2
 // CHECK-NEXT:    [[B8X2:%.*]] = alloca <vscale x 32 x i1>, align 2
 // CHECK-NEXT:    [[B8X4:%.*]] = alloca <vscale x 64 x i1>, align 2
+// CHECK-NEXT:    [[CNT:%.*]] = alloca target("aarch64.svcount"), align 2
 // CHECK-NEXT:    store <vscale x 16 x i8> zeroinitializer, ptr [[S8]], align 16
 // CHECK-NEXT:    store <vscale x 8 x i16> zeroinitializer, ptr [[S16]], align 16
 // CHECK-NEXT:    store <vscale x 4 x i32> zeroinitializer, ptr [[S32]], align 16
@@ -106,6 +107,7 @@
 // CHECK-NEXT:    store <vscale x 16 x i1> zeroinitializer, ptr [[B8]], align 2
 // CHECK-NEXT:    store <vscale x 32 x i1> zeroinitializer, ptr [[B8X2]], align 2
 // CHECK-NEXT:    store <vscale x 64 x i1> zeroinitializer, ptr [[B8X4]], align 2
+// CHECK-NEXT:    store target("aarch64.svcount") zeroinitializer, ptr [[CNT]], align 2
 // CHECK-NEXT:    ret void
 //
 void test_locals(void) {
@@ -164,6 +166,8 @@ void test_locals(void) {
   __SVBool_t b8{};
   __clang_svboolx2_t b8x2{};
   __clang_svboolx4_t b8x4{};
+
+  __SVCount_t cnt{};
 }
 
 // CHECK-LABEL: define dso_local void @_Z12test_copy_s8u10__SVInt8_t
@@ -879,3 +883,17 @@ void test_copy_b8x2(__clang_svboolx2_t a) {
 void test_copy_b8x4(__clang_svboolx4_t a) {
   __clang_svboolx4_t b{a};
 }
+
+// CHECK-LABEL: define dso_local void @_Z13test_copy_cntu11__SVCount_t
+// CHECK-SAME: (target("aarch64.svcount") [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca target("aarch64.svcount"), align 2
+// CHECK-NEXT:    [[B:%.*]] = alloca target("aarch64.svcount"), align 2
+// CHECK-NEXT:    store target("aarch64.svcount") [[A]], ptr [[A_ADDR]], align 2
+// CHECK-NEXT:    [[TMP0:%.*]] = load target("aarch64.svcount"), ptr [[A_ADDR]], align 2
+// CHECK-NEXT:    store target("aarch64.svcount") [[TMP0]], ptr [[B]], align 2
+// CHECK-NEXT:    ret void
+//
+void test_copy_cnt(__SVCount_t a) {
+  __SVCount_t b{a};
+}
diff --git a/clang/test/CodeGenCXX/default_calling_conv.cpp b/clang/test/CodeGenCXX/default_calling_conv.cpp
index 0991f862b8..ff81f37121 100644
--- a/clang/test/CodeGenCXX/default_calling_conv.cpp
+++ b/clang/test/CodeGenCXX/default_calling_conv.cpp
@@ -1,43 +1,50 @@
-// RUN: %clang_cc1 -triple i386-unknown-linux-gnu -fdefault-calling-conv=cdecl -emit-llvm -o - %s | FileCheck %s --check-prefix=CDECL --check-prefix=ALL
-// RUN: %clang_cc1 -triple i786-unknown-linux-gnu -target-feature +sse4.2 -fdefault-calling-conv=fastcall -emit-llvm -o - %s | FileCheck %s --check-prefix=FASTCALL --check-prefix=ALL
-// RUN: %clang_cc1 -triple i486-unknown-linux-gnu -fdefault-calling-conv=stdcall -emit-llvm -o - %s | FileCheck %s --check-prefix=STDCALL --check-prefix=ALL
-// RUN: %clang_cc1 -triple i486-unknown-linux-gnu -mrtd -emit-llvm -o - %s | FileCheck %s --check-prefix=STDCALL --check-prefix=ALL
-// RUN: %clang_cc1 -triple i986-unknown-linux-gnu -fdefault-calling-conv=vectorcall -emit-llvm -o - %s | FileCheck %s --check-prefix=VECTORCALL --check-prefix=ALL
-// RUN: %clang_cc1 -triple i986-unknown-linux-gnu -fdefault-calling-conv=regcall -emit-llvm -o - %s | FileCheck %s --check-prefix=REGCALL --check-prefix=ALL
+// RUN: %clang_cc1 -triple i386-unknown-linux-gnu -fdefault-calling-conv=cdecl -emit-llvm -o - %s | FileCheck %s --check-prefix=CDECL --check-prefix=X86 --check-prefix=ALL
+// RUN: %clang_cc1 -triple i786-unknown-linux-gnu -target-feature +sse4.2 -fdefault-calling-conv=fastcall -emit-llvm -o - %s | FileCheck %s --check-prefix=FASTCALL --check-prefix=X86 --check-prefix=ALL
+// RUN: %clang_cc1 -triple i486-unknown-linux-gnu -fdefault-calling-conv=stdcall -emit-llvm -o - %s | FileCheck %s --check-prefix=STDCALL --check-prefix=X86 --check-prefix=ALL
+// RUN: %clang_cc1 -triple i486-unknown-linux-gnu -mrtd -emit-llvm -o - %s | FileCheck %s --check-prefix=STDCALL --check-prefix=X86 --check-prefix=ALL
+// RUN: %clang_cc1 -triple i986-unknown-linux-gnu -fdefault-calling-conv=vectorcall -emit-llvm -o - %s | FileCheck %s --check-prefix=VECTORCALL --check-prefix=X86 --check-prefix=ALL
+// RUN: %clang_cc1 -triple i986-unknown-linux-gnu -fdefault-calling-conv=regcall -emit-llvm -o - %s | FileCheck %s --check-prefix=REGCALL --check-prefix=X86 --check-prefix=ALL
 // RUN: %clang_cc1 -triple i686-pc-win32 -fdefault-calling-conv=vectorcall -emit-llvm -o - %s -DWINDOWS | FileCheck %s --check-prefix=WIN32
 // RUN: %clang_cc1 -triple x86_64-windows-msvc -fdefault-calling-conv=vectorcall -emit-llvm -o - %s -DWINDOWS | FileCheck %s --check-prefix=WIN64
 // RUN: %clang_cc1 -triple i686-pc-win32 -emit-llvm -o - %s -DEXPLICITCC | FileCheck %s --check-prefix=EXPLICITCC
+// RUN: %clang_cc1 -triple m68k-unknown-linux-gnu -mrtd -emit-llvm -o - %s | FileCheck %s --check-prefix=RTDCALL --check-prefix=ALL
+// RUN: %clang_cc1 -triple m68k-unknown-linux-gnu -fdefault-calling-conv=rtdcall -emit-llvm -o - %s | FileCheck %s --check-prefix=RTDCALL --check-prefix=ALL
 
 // CDECL: define{{.*}} void @_Z5test1v
 // FASTCALL: define{{.*}} x86_fastcallcc void @_Z5test1v
 // STDCALL: define{{.*}} x86_stdcallcc void @_Z5test1v
 // VECTORCALL: define{{.*}} x86_vectorcallcc void @_Z5test1v
 // REGCALL: define{{.*}} x86_regcallcc void @_Z17__regcall3__test1v
+// RTDCALL: define{{.*}} m68k_rtdcc void @_Z5test1v
 void test1() {}
 
-// fastcall, stdcall, vectorcall and regcall do not support variadic functions.
+// fastcall, stdcall, vectorcall, regcall and m68k_rtd do not support variadic functions.
 // CDECL: define{{.*}} void @_Z12testVariadicz
 // FASTCALL: define{{.*}} void @_Z12testVariadicz
 // STDCALL: define{{.*}} void @_Z12testVariadicz
 // VECTORCALL: define{{.*}} void @_Z12testVariadicz
 // REGCALL: define{{.*}} void @_Z12testVariadicz
+// RTDCALL: define{{.*}} void @_Z12testVariadicz
 void testVariadic(...){}
 
-// ALL: define{{.*}} void @_Z5test2v
+// X86: define{{.*}} void @_Z5test2v
 void __attribute__((cdecl)) test2() {}
 
-// ALL: define{{.*}} x86_fastcallcc void @_Z5test3v
+// X86: define{{.*}} x86_fastcallcc void @_Z5test3v
 void __attribute__((fastcall)) test3() {}
 
-// ALL: define{{.*}} x86_stdcallcc void @_Z5test4v
+// X86: define{{.*}} x86_stdcallcc void @_Z5test4v
 void __attribute__((stdcall)) test4() {}
 
-// ALL: define{{.*}} x86_vectorcallcc void @_Z5test5v
+// X86: define{{.*}} x86_vectorcallcc void @_Z5test5v
 void __attribute__((vectorcall)) test5() {}
 
-// ALL: define{{.*}} x86_regcallcc void @_Z17__regcall3__test6v
+// X86: define{{.*}} x86_regcallcc void @_Z17__regcall3__test6v
 void __attribute__((regcall)) test6() {}
 
+// RTDCALL: define{{.*}} m68k_rtdcc void @_Z5test7v
+void __attribute__((m68k_rtd)) test7() {}
+
 // ALL: define linkonce_odr void @_ZN1A11test_memberEv
 class A {
 public:
@@ -47,6 +54,10 @@ class A {
 void test() {
   A a;
   a.test_member();
+
+// ALL: define internal void @"_ZZ{{.*}}testvENK3$_0clEi"
+  auto f = [](int b) {};
+  f(87);
 }
 
 // ALL: define{{.*}} i32 @main
diff --git a/clang/test/CodeGenCXX/ext-int.cpp b/clang/test/CodeGenCXX/ext-int.cpp
index 7676dec791..5a4270aef2 100644
--- a/clang/test/CodeGenCXX/ext-int.cpp
+++ b/clang/test/CodeGenCXX/ext-int.cpp
@@ -98,19 +98,6 @@ void BitfieldAssignment() {
   // CHECK: %[[SETC:.+]] = or i8 %[[CLEARC]], 64
 }
 
-enum AsEnumUnderlyingType : _BitInt(9) {
-  A,B,C
-};
-
-void UnderlyingTypeUsage(AsEnumUnderlyingType Param) {
-  // LIN: define{{.*}} void @_Z19UnderlyingTypeUsage20AsEnumUnderlyingType(i9 signext %
-  // WIN64: define dso_local void @"?UnderlyingTypeUsage@@YAXW4AsEnumUnderlyingType@@@Z"(i9 %
-  // WIN32: define dso_local void @"?UnderlyingTypeUsage@@YAXW4AsEnumUnderlyingType@@@Z"(i9 signext %
-  AsEnumUnderlyingType Var;
-  // CHECK: alloca i9, align 2
-  // CHECK: store i9 %{{.*}}, align 2
-}
-
 unsigned _BitInt(33) ManglingTestRetParam(unsigned _BitInt(33) Param) {
 // LIN64: define{{.*}} i64 @_Z20ManglingTestRetParamDU33_(i64 %
 // LIN32: define{{.*}} i33 @_Z20ManglingTestRetParamDU33_(i33 %
diff --git a/clang/test/CodeGenCXX/m68k-rtdcall.cpp b/clang/test/CodeGenCXX/m68k-rtdcall.cpp
new file mode 100644
index 0000000000..835649359a
--- /dev/null
+++ b/clang/test/CodeGenCXX/m68k-rtdcall.cpp
@@ -0,0 +1,16 @@
+// RUN: %clang_cc1 -triple m68k-linux-gnu -emit-llvm -o - %s | FileCheck %s
+
+class A {
+public:
+// CHECK: define{{.*}} m68k_rtdcc void @_ZN1A6memberEv
+  void __attribute__((m68k_rtd)) member() {}
+};
+
+void test() {
+  A a;
+  a.member();
+
+// CHECK: define{{.*}} m68k_rtdcc void @"_ZZ4testvENK3$_0clEi"
+  auto f = [](int b) __attribute__((m68k_rtd)) {};
+  f(87);
+};
diff --git a/clang/test/CodeGenHipStdPar/unannotated-functions-get-emitted.cpp b/clang/test/CodeGenHipStdPar/unannotated-functions-get-emitted.cpp
new file mode 100644
index 0000000000..dfd6b3da0a
--- /dev/null
+++ b/clang/test/CodeGenHipStdPar/unannotated-functions-get-emitted.cpp
@@ -0,0 +1,19 @@
+// RUN: %clang_cc1 -x hip -emit-llvm -fcuda-is-device \
+// RUN:   -o - %s | FileCheck --check-prefix=NO-HIPSTDPAR-DEV %s
+
+// RUN: %clang_cc1 --hipstdpar -emit-llvm -fcuda-is-device \
+// RUN:   -o - %s | FileCheck --check-prefix=HIPSTDPAR-DEV %s
+
+#define __device__ __attribute__((device))
+
+// NO-HIPSTDPAR-DEV-NOT: {{.*}}void @foo({{.*}})
+// HIPSTDPAR-DEV: {{.*}}void @foo({{.*}})
+extern "C" void foo(float *a, float b) {
+  *a = b;
+}
+
+// NO-HIPSTDPAR-DEV: {{.*}}void @bar({{.*}})
+// HIPSTDPAR-DEV: {{.*}}void @bar({{.*}})
+extern "C" __device__ void bar(float *a, float b) {
+  *a = b;
+}
diff --git a/clang/test/CodeGenHipStdPar/unsupported-ASM.cpp b/clang/test/CodeGenHipStdPar/unsupported-ASM.cpp
new file mode 100644
index 0000000000..485bf916c8
--- /dev/null
+++ b/clang/test/CodeGenHipStdPar/unsupported-ASM.cpp
@@ -0,0 +1,10 @@
+// RUN: %clang_cc1 -triple amdgcn-amd-amdhsa -aux-triple x86_64-unknown-linux-gnu \
+// RUN:   --hipstdpar -x hip -emit-llvm -fcuda-is-device -o - %s | FileCheck %s
+
+#define __global__ __attribute__((global))
+
+__global__ void foo(int i) {
+    asm ("addl %2, %1; seto %b0" : "=q" (i), "+g" (i) : "r" (i));
+}
+
+// CHECK: declare void @__ASM__hipstdpar_unsupported([{{.*}}])
diff --git a/clang/test/CodeGenHipStdPar/unsupported-builtins.cpp b/clang/test/CodeGenHipStdPar/unsupported-builtins.cpp
new file mode 100644
index 0000000000..02355eca26
--- /dev/null
+++ b/clang/test/CodeGenHipStdPar/unsupported-builtins.cpp
@@ -0,0 +1,8 @@
+// RUN: %clang_cc1 -triple amdgcn-amd-amdhsa -aux-triple x86_64-unknown-linux-gnu \
+// RUN:   --hipstdpar -x hip -emit-llvm -fcuda-is-device -o - %s | FileCheck %s
+
+#define __global__ __attribute__((global))
+
+__global__ void foo() { return __builtin_ia32_pause(); }
+
+// CHECK: declare void @__builtin_ia32_pause__hipstdpar_unsupported()
diff --git a/clang/test/CodeGenObjC/arc.m b/clang/test/CodeGenObjC/arc.m
index fb4eb34523..754c339cb2 100644
--- a/clang/test/CodeGenObjC/arc.m
+++ b/clang/test/CodeGenObjC/arc.m
@@ -7,30 +7,30 @@
 // RUN: %clang_cc1 -fobjc-runtime=macosx-10.7.0 -triple x86_64-apple-darwin11 -Wno-objc-root-class -Wno-incompatible-pointer-types -Wno-arc-unsafe-retained-assign -emit-llvm -fblocks -fobjc-arc -fobjc-runtime-has-weak -o - %s | FileCheck -check-prefix=ARC-NATIVE %s
 
 // ARC-ALIEN: declare extern_weak void @llvm.objc.storeStrong(ptr, ptr)
-// ARC-ALIEN: declare extern_weak ptr @llvm.objc.retain(ptr returned)
-// ARC-ALIEN: declare extern_weak ptr @llvm.objc.autoreleaseReturnValue(ptr returned)
+// ARC-ALIEN: declare extern_weak ptr @llvm.objc.retain(ptr)
+// ARC-ALIEN: declare extern_weak ptr @llvm.objc.autoreleaseReturnValue(ptr)
 // ARC-ALIEN: declare ptr @objc_msgSend(ptr, ptr, ...) [[NLB:#[0-9]+]]
 // ARC-ALIEN: declare extern_weak void @llvm.objc.release(ptr)
-// ARC-ALIEN: declare extern_weak ptr @llvm.objc.retainAutoreleasedReturnValue(ptr returned)
+// ARC-ALIEN: declare extern_weak ptr @llvm.objc.retainAutoreleasedReturnValue(ptr)
 // ARC-ALIEN: declare extern_weak ptr @llvm.objc.initWeak(ptr, ptr)
 // ARC-ALIEN: declare extern_weak ptr @llvm.objc.storeWeak(ptr, ptr)
 // ARC-ALIEN: declare extern_weak ptr @llvm.objc.loadWeakRetained(ptr)
 // ARC-ALIEN: declare extern_weak void @llvm.objc.destroyWeak(ptr)
-// ARC-ALIEN: declare extern_weak ptr @llvm.objc.autorelease(ptr returned)
-// ARC-ALIEN: declare extern_weak ptr @llvm.objc.retainAutorelease(ptr returned)
+// ARC-ALIEN: declare extern_weak ptr @llvm.objc.autorelease(ptr)
+// ARC-ALIEN: declare extern_weak ptr @llvm.objc.retainAutorelease(ptr)
 
 // ARC-NATIVE: declare void @llvm.objc.storeStrong(ptr, ptr)
-// ARC-NATIVE: declare ptr @llvm.objc.retain(ptr returned)
-// ARC-NATIVE: declare ptr @llvm.objc.autoreleaseReturnValue(ptr returned)
+// ARC-NATIVE: declare ptr @llvm.objc.retain(ptr)
+// ARC-NATIVE: declare ptr @llvm.objc.autoreleaseReturnValue(ptr)
 // ARC-NATIVE: declare ptr @objc_msgSend(ptr, ptr, ...) [[NLB:#[0-9]+]]
 // ARC-NATIVE: declare void @llvm.objc.release(ptr)
-// ARC-NATIVE: declare ptr @llvm.objc.retainAutoreleasedReturnValue(ptr returned)
+// ARC-NATIVE: declare ptr @llvm.objc.retainAutoreleasedReturnValue(ptr)
 // ARC-NATIVE: declare ptr @llvm.objc.initWeak(ptr, ptr)
 // ARC-NATIVE: declare ptr @llvm.objc.storeWeak(ptr, ptr)
 // ARC-NATIVE: declare ptr @llvm.objc.loadWeakRetained(ptr)
 // ARC-NATIVE: declare void @llvm.objc.destroyWeak(ptr)
-// ARC-NATIVE: declare ptr @llvm.objc.autorelease(ptr returned)
-// ARC-NATIVE: declare ptr @llvm.objc.retainAutorelease(ptr returned)
+// ARC-NATIVE: declare ptr @llvm.objc.autorelease(ptr)
+// ARC-NATIVE: declare ptr @llvm.objc.retainAutorelease(ptr)
 
 // CHECK-LABEL: define{{.*}} void @test0
 void test0(id x) {
diff --git a/clang/test/CodeGenObjC/convert-messages-to-runtime-calls.m b/clang/test/CodeGenObjC/convert-messages-to-runtime-calls.m
index a60c9b0feb..9a5ecbc03d 100644
--- a/clang/test/CodeGenObjC/convert-messages-to-runtime-calls.m
+++ b/clang/test/CodeGenObjC/convert-messages-to-runtime-calls.m
@@ -28,11 +28,6 @@ void test1(id x) {
   // MSGS: {{call.*@objc_msgSend}}
   // CALLS: {{call.*@objc_alloc}}
   // CALLS: {{call.*@objc_allocWithZone}}
-
-  // Note that calls to the intrinsics are not allowed for
-  // retain/release/autorelease they're marked `thisreturn`, which isn't
-  // guaranteed to be true for classes that define their own `-retain`, for
-  // example. Be sure to keep these as normal function calls:
   // CALLS: {{call.*@objc_retain}}
   // CALLS: {{call.*@objc_release}}
   // CALLS: {{tail call.*@objc_autorelease}}
diff --git a/clang/test/CodeGenSYCL/function-attrs.cpp b/clang/test/CodeGenSYCL/function-attrs.cpp
index 8f5c0ea5c5..1606f961f2 100644
--- a/clang/test/CodeGenSYCL/function-attrs.cpp
+++ b/clang/test/CodeGenSYCL/function-attrs.cpp
@@ -1,16 +1,29 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --check-globals --version 3
 // RUN: %clang_cc1 -fsycl-is-device -emit-llvm -disable-llvm-passes \
-// RUN:  -triple spir64 -fexceptions -emit-llvm %s -o - | FileCheck %s
+// RUN:  -triple spir64 -fexceptions -emit-llvm -fno-ident %s -o - | FileCheck %s
 
 int foo();
 
-// CHECK: define dso_local spir_func void @_Z3barv() [[BAR:#[0-9]+]]
-// CHECK: attributes [[BAR]] =
-// CHECK-SAME: convergent
-// CHECK-SAME: nounwind
+// CHECK-LABEL: define dso_local spir_func void @_Z3barv(
+// CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[A_ASCAST:%.*]] = addrspacecast ptr [[A]] to ptr addrspace(4)
+// CHECK-NEXT:    [[CALL:%.*]] = call spir_func noundef i32 @_Z3foov() #[[ATTR1:[0-9]+]]
+// CHECK-NEXT:    store i32 [[CALL]], ptr addrspace(4) [[A_ASCAST]], align 4
+// CHECK-NEXT:    ret void
+//
 void bar() {
   int a = foo();
 }
 
+// CHECK-LABEL: define dso_local spir_func noundef i32 @_Z3foov(
+// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr [[RETVAL]] to ptr addrspace(4)
+// CHECK-NEXT:    ret i32 1
+//
 int foo() {
   return 1;
 }
@@ -20,7 +33,24 @@ __attribute__((sycl_kernel)) void kernel_single_task(const Func &kernelFunc) {
   kernelFunc();
 }
 
+// CHECK-LABEL: define dso_local noundef i32 @main(
+// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[REF_TMP:%.*]] = alloca [[CLASS_ANON:%.*]], align 1
+// CHECK-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr [[RETVAL]] to ptr addrspace(4)
+// CHECK-NEXT:    [[REF_TMP_ASCAST:%.*]] = addrspacecast ptr [[REF_TMP]] to ptr addrspace(4)
+// CHECK-NEXT:    store i32 0, ptr addrspace(4) [[RETVAL_ASCAST]], align 4
+// CHECK-NEXT:    call spir_func void @_Z18kernel_single_taskIZ4mainE11fake_kernelZ4mainEUlvE_EvRKT0_(ptr addrspace(4) noundef align 1 dereferenceable(1) [[REF_TMP_ASCAST]]) #[[ATTR1]]
+// CHECK-NEXT:    ret i32 0
+//
 int main() {
   kernel_single_task<class fake_kernel>([] { bar(); });
   return 0;
 }
+//.
+// CHECK: attributes #0 = { convergent mustprogress noinline norecurse nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+// CHECK: attributes #1 = { convergent nounwind }
+//.
+// CHECK: !0 = !{i32 1, !"wchar_size", i32 4}
+//.
diff --git a/clang/test/Driver/Inputs/basic_avr_tree/usr/lib/avr/lib/ldscripts/avrtiny.x b/clang/test/Driver/Inputs/basic_avr_tree/usr/lib/avr/lib/ldscripts/avrtiny.x
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/clang/test/Driver/Inputs/basic_avr_tree/usr/lib/avr/lib/ldscripts/avrxmega6.x b/clang/test/Driver/Inputs/basic_avr_tree/usr/lib/avr/lib/ldscripts/avrxmega6.x
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/clang/test/Driver/avr-ld.c b/clang/test/Driver/avr-ld.c
index 4042ecb89a..3e41144853 100644
--- a/clang/test/Driver/avr-ld.c
+++ b/clang/test/Driver/avr-ld.c
@@ -57,3 +57,53 @@
 // RUN: %clang -### --target=avr -mmcu=atmega328 -fuse-ld=lld -flto --sysroot %S/Inputs/basic_avr_tree %s 2>&1 | FileCheck -check-prefix LINKS %s
 // LINKS: {{".*ld.*"}} {{.*}} "--defsym=__DATA_REGION_ORIGIN__=0x800100" "-plugin-opt=mcpu=atmega328"
 // LINKS-NOT: "-plugin-opt=thinlto"
+
+// RUN: %clang -### --target=avr -mmcu=attiny40 -fuse-ld=lld --sysroot %S/Inputs/basic_avr_tree %s 2>&1 | FileCheck -check-prefix LINKT0 %s
+// LINKT0: {{".*lld.*"}} {{.*}} {{"-T.*avrtiny.x"}}
+// LINKT0-NOT: "-m
+
+// RUN: %clang -### --target=avr -mmcu=atxmega384c3 -fuse-ld=lld --sysroot %S/Inputs/basic_avr_tree %s 2>&1 | FileCheck -check-prefix LINKT1 %s
+// LINKT1: {{".*lld.*"}} {{.*}} {{"-T.*avrxmega6.x"}}
+// LINKT1-NOT: "-m
+
+// RUN: %clang -### --target=avr -mmcu=atmega328 -fuse-ld=lld --sysroot %S/Inputs/basic_avr_tree %s 2>&1 | FileCheck -check-prefix LINKT2 %s
+// LINKT2: {{".*lld.*"}} {{.*}} "--start-group" {{.*}} "--end-group"
+// LINKT2-NOT: "-T
+// LINKT2-NOT: "-m
+
+// RUN: %clang -### --target=avr -mmcu=attiny40 --sysroot %S/Inputs/basic_avr_tree %s 2>&1 | FileCheck -check-prefix LINKT3 %s
+// LINKT3: {{".*ld.*"}} {{.*}} "-mavrtiny"
+// LINKT3-NOT: "-T
+
+// RUN: %clang -### --target=avr -mmcu=attiny40 --sysroot %S/Inputs/basic_avr_tree -fuse-ld=lld -T %S/Inputs/basic_avr_tree/usr/lib/avr/lib/ldscripts/avrxmega6.x %s 2>&1 | FileCheck -check-prefix LINKT4 %s
+// LINKT4: {{".*lld.*"}} {{.*}} {{"-T.*avrxmega6.x"}}
+// LINKT4-NOT: {{"-T.*avrtiny.x"}}
+// LINKT4-NOT: "-m
+
+// RUN: %clang -### -r --target=avr -mmcu=atmega328 --sysroot %S/Inputs/basic_avr_tree %s 2>&1 | FileCheck --check-prefix=LINKU %s
+// LINKU: {{".*ld.*"}} {{.*}} "-r" {{.*}} "-mavr5"
+// LINKU-NOT: "--gc-sections"
+// LINKU-NOT: "--defsym
+// LINKU-NOT: "-l
+
+// RUN: %clang -### -r --target=avr -mmcu=atmega328 -fuse-ld=lld --sysroot %S/Inputs/basic_avr_tree %s 2>&1 | FileCheck --check-prefix=LINKV %s
+// LINKV: {{".*ld.*"}} {{.*}} "-r"
+// LINKV-NOT: "--gc-sections"
+// LINKV-NOT: "--defsym
+// LINKV-NOT: "-l
+// LINKV-NOT: "-m
+
+// RUN: %clang -### -r --target=avr -mmcu=atmega328 -lm --sysroot %S/Inputs/basic_avr_tree %s 2>&1 | FileCheck --check-prefix=LINKW %s
+// LINKW: {{".*ld.*"}} {{.*}} "-r" "-lm" {{.*}} "-mavr5"
+// LINKW-NOT: "--gc-sections"
+// LINKW-NOT: "--defsym
+
+// RUN: %clang -### -r --target=avr --sysroot %S/Inputs/basic_avr_tree %s 2>&1 | FileCheck --check-prefix=LINKX %s
+// LINKX: warning: no target microcontroller specified
+// LINKX: {{".*ld.*"}} {{.*}} "-r" {{.*}}
+// LINKX-NOT: warning: {{.*}} standard library
+// LINKX-NOT: warning: {{.*}} data section address
+// LINKX-NOT: "--gc-sections"
+// LINKX-NOT: "--defsym
+// LINKX-NOT: "-l
+// LINKX-NOT: "-m
diff --git a/clang/test/Driver/cl-delayed-template-parsing-cxx20.cpp b/clang/test/Driver/cl-delayed-template-parsing-cxx20.cpp
new file mode 100644
index 0000000000..7af9b2447b
--- /dev/null
+++ b/clang/test/Driver/cl-delayed-template-parsing-cxx20.cpp
@@ -0,0 +1,10 @@
+// RUN: %clang_cl -### -- %s 2>&1 | FileCheck %s --check-prefix=PRE-CXX20
+// RUN: %clang_cl -std:c++20 -### -- %s 2>&1 | FileCheck %s
+// RUN: %clang_cl -std:c++20 -### -fdelayed-template-parsing -- %s 2>&1 | FileCheck %s --check-prefix=CHECK-EXPLICIT
+
+// PRE-CXX20: -fdelayed-template-parsing
+
+// CHECK-NOT: -fdelayed-template-parsing
+
+// CHECK-EXPLICIT: warning: -fdelayed-template-parsing is deprecated after C++20
+// CHECK-EXPLICIT: -fdelayed-template-parsing
diff --git a/clang/test/Driver/cl-offload.cu b/clang/test/Driver/cl-offload.cu
new file mode 100644
index 0000000000..ec1d8ba549
--- /dev/null
+++ b/clang/test/Driver/cl-offload.cu
@@ -0,0 +1,31 @@
+// REQUIRES: !system-darwin
+
+// The test cannot be run on Darwin because /Users will be treated as a MSVC option.
+
+// RUN: %clang_cl -### -target x86_64-pc-windows-msvc --offload-arch=sm_35 -fgpu-rdc \
+// RUN:   --cuda-path=%S/Inputs/CUDA/usr/local/cuda \
+// RUN:   /Wall -x cuda %s 2>&1 \
+// RUN:   | FileCheck %s -check-prefix=CUDA
+
+// RUN: %clang_cl -### -target x86_64-pc-windows-msvc --offload-arch=gfx1010 -fgpu-rdc --hip-link \
+// RUN:   --rocm-path=%S/Inputs/rocm /Wall -x hip %s 2>&1 \
+// RUN:   | FileCheck %s -check-prefix=HIP
+
+// CUDA: "-cc1" "-triple" "nvptx64-nvidia-cuda" "-aux-triple" "x86_64-pc-windows-msvc"
+// CUDA-SAME: "-Weverything"
+// CUDA: ptxas
+// CUDA: "-cc1" "-triple" "x86_64-pc-windows-msvc{{.*}}" "-aux-triple" "nvptx64-nvidia-cuda"
+// CUDA-SAME: "-Weverything"
+// CUDA: link
+
+// HIP: "-cc1" "-triple" "x86_64-pc-windows-msvc{{.*}}" "-aux-triple" "amdgcn-amd-amdhsa"
+// HIP-SAME: "-Weverything"
+// HIP: "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-pc-windows-msvc"
+// HIP-SAME: "-Weverything"
+// HIP: {{lld.* "-flavor" "gnu" "-m" "elf64_amdgpu"}}
+// HIP: {{link.* "amdhip64.lib"}}
+
+// CMake uses this option when finding packages for HIP, so
+// make sure it does not cause error.
+
+// RUN: %clang_cl --print-libgcc-file-name
diff --git a/clang/test/Driver/dragonfly.c b/clang/test/Driver/dragonfly.c
index 8ba13c41d6..11d730f55b 100644
--- a/clang/test/Driver/dragonfly.c
+++ b/clang/test/Driver/dragonfly.c
@@ -2,7 +2,7 @@
 // RUN: FileCheck -input-file %t.log %s
 
 // CHECK: "-cc1" "-triple" "x86_64-pc-dragonfly"
-// CHECK: ld{{.*}}" "--eh-frame-hdr" "-dynamic-linker" "/usr/libexec/ld-elf.so.{{.*}}" "--hash-style=gnu" "--enable-new-dtags" "-o" "a.out" "{{.*}}crt1.o" "{{.*}}crti.o" "{{.*}}crtbegin.o" "{{.*}}.o" "-L{{.*}}gcc{{.*}}" "-rpath" "{{.*}}gcc{{.*}}" "-lc" "-lgcc" "{{.*}}crtend.o" "{{.*}}crtn.o"
+// CHECK: ld{{.*}}" "--eh-frame-hdr" "-dynamic-linker" "/usr/libexec/ld-elf.so.{{.*}}" "--hash-style=gnu" "--enable-new-dtags" "-o" "a.out" "{{.*}}crt1.o" "{{.*}}crti.o" "{{.*}}crtbegin.o" "-L{{.*}}/../lib" "-L/usr/lib" "-L/usr/lib/gcc80" "{{.*}}.o" "-rpath" "{{.*}}gcc80{{.*}}" "-lc" "-lgcc" "{{.*}}crtend.o" "{{.*}}crtn.o"
 
 // Check x86_64-unknown-dragonfly, X86_64
 // RUN: %clang -### %s 2>&1 --target=x86_64-unknown-dragonfly \
@@ -15,7 +15,8 @@
 // CHECK-LD-X86_64-SAME: "[[SYSROOT]]{{/|\\\\}}usr{{/|\\\\}}lib{{/|\\\\}}crt1.o"
 // CHECK-LD-X86_64-SAME: "[[SYSROOT]]{{/|\\\\}}usr{{/|\\\\}}lib{{/|\\\\}}crti.o"
 // CHECK-LD-X86_64-SAME: "[[SYSROOT]]{{/|\\\\}}usr{{/|\\\\}}lib{{/|\\\\}}gcc80{{/|\\\\}}crtbegin.o"
-// CHECK-LD-X86_64-SAME: "-L[[SYSROOT]]{{/|\\\\}}usr{{/|\\\\}}lib{{/|\\\\}}gcc80" "-rpath" "/usr/lib/gcc80" "-lc" "-lgcc" "--as-needed" "-lgcc_pic" "--no-as-needed"
+// CHECK-LD-X86_64-SAME: "-L[[SYSROOT]]{{/|\\\\}}usr{{/|\\\\}}lib" "-L[[SYSROOT]]{{/|\\\\}}usr{{/|\\\\}}lib{{/|\\\\}}gcc80"
+// CHECK-LD-X86_64-SAME: "-rpath" "/usr/lib/gcc80" "-lc" "-lgcc" "--as-needed" "-lgcc_pic" "--no-as-needed"
 // CHECK-LD-X86_64-SAME: "[[SYSROOT]]{{/|\\\\}}usr{{/|\\\\}}lib{{/|\\\\}}gcc80{{/|\\\\}}crtend.o"
 // CHECK-LD-X86_64-SAME: "[[SYSROOT]]{{/|\\\\}}usr{{/|\\\\}}lib{{/|\\\\}}crtn.o"
 
@@ -26,3 +27,8 @@
 // RELOCATABLE-NOT: "-dynamic-linker"
 // RELOCATABLE-NOT: "-l
 // RELOCATABLE-NOT: {{.*}}crt{{[^./]+}}.o
+
+// Check that the new linker flags are passed to DragonFly
+// RUN: %clang --target=x86_64-unknown-dragonfly -s -t -### %s 2>&1 \
+// RUN:   | FileCheck --check-prefix=CHECK-LD-FLAGS %s
+// CHECK-LD-FLAGS: ld{{.*}}" "{{.*}}" "-s" "-t"
diff --git a/clang/test/Driver/fopenmp.c b/clang/test/Driver/fopenmp.c
index 6e31de1099..cf04340ebc 100644
--- a/clang/test/Driver/fopenmp.c
+++ b/clang/test/Driver/fopenmp.c
@@ -13,6 +13,9 @@
 // RUN: %clang -target x86_64-openbsd -fopenmp=libomp -c %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-CC1-OPENMP
 // RUN: %clang -target x86_64-openbsd -fopenmp=libgomp -c %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-CC1-NO-OPENMP
 // RUN: %clang -target x86_64-openbsd -fopenmp=libiomp5 -c %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-CC1-OPENMP
+// RUN: %clang -target x86_64-dragonfly -fopenmp=libomp -c %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-CC1-OPENMP
+// RUN: %clang -target x86_64-dragonfly -fopenmp=libgomp -c %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-CC1-NO-OPENMP
+// RUN: %clang -target x86_64-dragonfly -fopenmp=libiomp5 -c %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-CC1-OPENMP
 // RUN: %clang -target x86_64-windows-gnu -fopenmp=libomp -c %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-CC1-OPENMP
 // RUN: %clang -target x86_64-windows-gnu -fopenmp=libgomp -c %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-CC1-NO-OPENMP
 // RUN: %clang -target x86_64-windows-gnu -fopenmp=libiomp5 -c %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-CC1-OPENMP
@@ -90,6 +93,19 @@
 // RUN: %clang -nostdlib -target x86_64-openbsd -fopenmp=libgomp %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-NO-GOMP
 // RUN: %clang -nostdlib -target x86_64-openbsd -fopenmp=libiomp5 %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-NO-IOMP5
 //
+// RUN: %clang -target x86_64-dragonfly -fopenmp=libomp %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-LD-OMP
+// RUN: %clang -target x86_64-dragonfly -fopenmp=libgomp %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-LD-GOMP --check-prefix=CHECK-LD-GOMP-NO-RT
+// RUN: %clang -target x86_64-dragonfly -fopenmp=libiomp5 %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-LD-IOMP5
+//
+// RUN: %clang -target x86_64-dragonfly -fopenmp=libomp -static-openmp %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-LD-STATIC-OMP
+// RUN: %clang -target x86_64-dragonfly -fopenmp=libgomp -static-openmp %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-LD-STATIC-GOMP --check-prefix=CHECK-LD-STATIC-GOMP-NO-RT
+// RUN: %clang -target x86_64-dragonfly -fopenmp=libiomp5 -static-openmp %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-LD-STATIC-IOMP5
+// RUN: %clang -target x86_64-dragonfly -fopenmp=libiomp5 -static -static-openmp %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-LD-STATIC-IOMP5-NO-BDYNAMIC
+//
+// RUN: %clang -nostdlib -target x86_64-dragonfly -fopenmp=libomp %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-NO-OMP
+// RUN: %clang -nostdlib -target x86_64-dragonfly -fopenmp=libgomp %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-NO-GOMP
+// RUN: %clang -nostdlib -target x86_64-dragonfly -fopenmp=libiomp5 %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-NO-IOMP5
+//
 // RUN: %clang -target x86_64-windows-gnu -fopenmp=libomp %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-LD-OMP
 // RUN: %clang -target x86_64-windows-gnu -fopenmp=libgomp %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-LD-GOMP --check-prefix=CHECK-LD-GOMP-NO-RT
 // RUN: %clang -target x86_64-windows-gnu -fopenmp=libiomp5 %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-LD-IOMP5MD
@@ -136,7 +152,7 @@
 // CHECK-LD-STATIC-IOMP5: "-Bstatic" "-liomp5" "-Bdynamic"
 //
 // CHECK-LD-STATIC-IOMP5-NO-BDYNAMIC: "{{.*}}ld{{(.exe)?}}"
-// For x86 Gnu, the driver passes -static, while FreeBSD, NetBSD and OpenBSD pass -Bstatic
+// For x86 Gnu, the driver passes -static, while FreeBSD, NetBSD, OpenBSD and DragonFly pass -Bstatic
 // CHECK-LD-STATIC-IOMP5-NO-BDYNAMIC: "-{{B?}}static" {{.*}} "-liomp5"
 // CHECK-LD-STATIC-IOMP5-NO-BDYNAMIC-NOT: "-Bdynamic"
 //
@@ -157,6 +173,7 @@
 // RUN: %clang -target x86_64-freebsd -fopenmp %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-LD-ANY
 // RUN: %clang -target x86_64-netbsd -fopenmp %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-LD-ANY
 // RUN: %clang -target x86_64-openbsd -fopenmp %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-LD-ANY
+// RUN: %clang -target x86_64-dragonfly -fopenmp %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-LD-ANY
 // RUN: %clang -target x86_64-windows-gnu -fopenmp %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-LD-ANYMD
 //
 // CHECK-LD-ANY: "{{.*}}ld{{(.exe)?}}"
diff --git a/clang/test/Driver/haiku.c b/clang/test/Driver/haiku.c
index 3888c67329..965d3cf97b 100644
--- a/clang/test/Driver/haiku.c
+++ b/clang/test/Driver/haiku.c
@@ -56,6 +56,11 @@
 // CHECK-LD-X86_64-SAME: {{^}} "[[SYSROOT]]/boot/system/develop/tools/lib/gcc/x86_64-unknown-haiku/13.2.0/crtendS.o"
 // CHECK-LD-X86_64-SAME: {{^}} "[[SYSROOT]]/boot/system/develop/lib/crtn.o"
 
+// Check -rdynamic is a no-op
+// RUN: %clang -### -rdynamic %s 2>&1 --target=x86_64-unknown-haiku \
+// RUN:    | FileCheck --check-prefix=CHECK-RDYNAMIC %s
+// CHECK-RDYNAMIC-NOT: "-export-dynamic"
+
 // Check the right flags are present with -shared
 // RUN: %clang -### %s -shared 2>&1 --target=x86_64-unknown-haiku \
 // RUN:     --gcc-toolchain="" \
diff --git a/clang/test/Driver/openbsd.c b/clang/test/Driver/openbsd.c
index 05d290a309..713bf350ee 100644
--- a/clang/test/Driver/openbsd.c
+++ b/clang/test/Driver/openbsd.c
@@ -8,7 +8,7 @@
 // RUN: %clang --target=i686-pc-openbsd -pg -pthread -### %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-PG %s
 // CHECK-PG: "-cc1" "-triple" "i686-pc-openbsd"
-// CHECK-PG: ld{{.*}}" "-e" "__start" "--eh-frame-hdr" "-Bdynamic" "-dynamic-linker" "{{.*}}ld.so" "-nopie" "-o" "a.out" "{{.*}}gcrt0.o" "{{.*}}crtbegin.o" "{{.*}}.o" "-lcompiler_rt" "-lpthread_p" "-lc_p" "-lcompiler_rt" "{{.*}}crtend.o"
+// CHECK-PG: ld{{.*}}" "-e" "__start" "--eh-frame-hdr" "-dynamic-linker" "{{.*}}ld.so" "-nopie" "-o" "a.out" "{{.*}}gcrt0.o" "{{.*}}crtbegin.o" "{{.*}}.o" "-lcompiler_rt" "-lpthread_p" "-lc_p" "-lcompiler_rt" "{{.*}}crtend.o"
 
 // Check CPU type for i386
 // RUN: %clang --target=i386-unknown-openbsd -### -c %s 2>&1 \
@@ -30,26 +30,23 @@
 // RUN:   | FileCheck --check-prefix=CHECK-LD-S %s
 // RUN: %clang --target=i686-pc-openbsd -t -### %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-LD-T %s
-// RUN: %clang --target=i686-pc-openbsd -Z -### %s 2>&1 \
-// RUN:   | FileCheck --check-prefix=CHECK-LD-Z %s
 // RUN: %clang --target=mips64-unknown-openbsd -### %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-MIPS64-LD %s
 // RUN: %clang --target=mips64el-unknown-openbsd -### %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-MIPS64EL-LD %s
-// CHECK-LD-R:     "-r"
+// CHECK-LD-R-NOT: "-e"
 // CHECK-LD-R-NOT: "-dynamic-linker"
 // CHECK-LD-R-NOT: "-l
 // CHECK-LD-R-NOT: crt{{[^./\\]+}}.o
+// CHECK-LD-R:     "-r"
 // CHECK-LD-S: "-cc1" "-triple" "i686-pc-openbsd"
-// CHECK-LD-S: ld{{.*}}" "-e" "__start" "--eh-frame-hdr" "-Bdynamic" "-dynamic-linker" "{{.*}}ld.so" "-o" "a.out" "{{.*}}crt0.o" "{{.*}}crtbegin.o" "-L{{.*}}" "-s" "{{.*}}.o" "-lcompiler_rt" "-lc" "-lcompiler_rt" "{{.*}}crtend.o"
+// CHECK-LD-S: ld{{.*}}" "-e" "__start" "--eh-frame-hdr" "-dynamic-linker" "{{.*}}ld.so" "-o" "a.out" "{{.*}}crt0.o" "{{.*}}crtbegin.o" "-L{{.*}}" "-s" "{{.*}}.o" "-lcompiler_rt" "-lc" "-lcompiler_rt" "{{.*}}crtend.o"
 // CHECK-LD-T: "-cc1" "-triple" "i686-pc-openbsd"
-// CHECK-LD-T: ld{{.*}}" "-e" "__start" "--eh-frame-hdr" "-Bdynamic" "-dynamic-linker" "{{.*}}ld.so" "-o" "a.out" "{{.*}}crt0.o" "{{.*}}crtbegin.o" "-L{{.*}}" "-t" "{{.*}}.o" "-lcompiler_rt" "-lc" "-lcompiler_rt" "{{.*}}crtend.o"
-// CHECK-LD-Z: "-cc1" "-triple" "i686-pc-openbsd"
-// CHECK-LD-Z: ld{{.*}}" "-e" "__start" "--eh-frame-hdr" "-Bdynamic" "-dynamic-linker" "{{.*}}ld.so" "-o" "a.out" "{{.*}}crt0.o" "{{.*}}crtbegin.o" "-L{{.*}}" "-Z" "{{.*}}.o" "-lcompiler_rt" "-lc" "-lcompiler_rt" "{{.*}}crtend.o"
+// CHECK-LD-T: ld{{.*}}" "-e" "__start" "--eh-frame-hdr" "-dynamic-linker" "{{.*}}ld.so" "-o" "a.out" "{{.*}}crt0.o" "{{.*}}crtbegin.o" "-L{{.*}}" "-t" "{{.*}}.o" "-lcompiler_rt" "-lc" "-lcompiler_rt" "{{.*}}crtend.o"
 // CHECK-MIPS64-LD: "-cc1" "-triple" "mips64-unknown-openbsd"
-// CHECK-MIPS64-LD: ld{{.*}}" "-EB" "-e" "__start" "--eh-frame-hdr" "-Bdynamic" "-dynamic-linker" "{{.*}}ld.so" "-o" "a.out" "{{.*}}crt0.o" "{{.*}}crtbegin.o" "-L{{.*}}" "{{.*}}.o" "-lcompiler_rt" "-lc" "-lcompiler_rt" "{{.*}}crtend.o"
+// CHECK-MIPS64-LD: ld{{.*}}" "-EB" "-e" "__start" "--eh-frame-hdr" "-dynamic-linker" "{{.*}}ld.so" "-o" "a.out" "{{.*}}crt0.o" "{{.*}}crtbegin.o" "-L{{.*}}" "{{.*}}.o" "-lcompiler_rt" "-lc" "-lcompiler_rt" "{{.*}}crtend.o"
 // CHECK-MIPS64EL-LD: "-cc1" "-triple" "mips64el-unknown-openbsd"
-// CHECK-MIPS64EL-LD: ld{{.*}}" "-EL" "-e" "__start" "--eh-frame-hdr" "-Bdynamic" "-dynamic-linker" "{{.*}}ld.so" "-o" "a.out" "{{.*}}crt0.o" "{{.*}}crtbegin.o" "-L{{.*}}" "{{.*}}.o" "-lcompiler_rt" "-lc" "-lcompiler_rt" "{{.*}}crtend.o"
+// CHECK-MIPS64EL-LD: ld{{.*}}" "-EL" "-e" "__start" "--eh-frame-hdr" "-dynamic-linker" "{{.*}}ld.so" "-o" "a.out" "{{.*}}crt0.o" "{{.*}}crtbegin.o" "-L{{.*}}" "{{.*}}.o" "-lcompiler_rt" "-lc" "-lcompiler_rt" "{{.*}}crtend.o"
 
 // Check that --sysroot is passed to the linker
 // RUN: %clang --target=i686-pc-openbsd -### %s 2>&1 \
diff --git a/clang/test/Driver/print-multi-selection-flags.c b/clang/test/Driver/print-multi-selection-flags.c
index 819ff242ce..248d9a3cdf 100644
--- a/clang/test/Driver/print-multi-selection-flags.c
+++ b/clang/test/Driver/print-multi-selection-flags.c
@@ -35,6 +35,21 @@
 // CHECK-MVENOFP-NOT: -march=thumbv8.1m.main{{.*}}+mve.fp{{.*}}
 // CHECK-MVENOFP: -mfpu=none
 
+// RUN: %clang -print-multi-flags-experimental --target=arm-none-eabihf -march=armv8.1m.main+fp.dp | FileCheck --check-prefix=CHECK-V8_1_FP_DP %s
+// CHECK-V8_1_FP_DP: -march=thumbv8.1m.main{{.*}}
+// CHECK-V8_1_FP_DP: -mfloat-abi=hard
+// CHECK-V8_1_FP_DP: -mfpu=fp-armv8-fullfp16-d16
+
+// RUN: %clang -print-multi-flags-experimental --target=arm-none-eabihf -march=armv8.1m.main+nofp+fp+nofp.dp | FileCheck --check-prefix=CHECK-V8_1_NO_FP_DP %s
+// CHECK-V8_1_NO_FP_DP: -march=thumbv8.1m.main{{.*}}
+// CHECK-V8_1_NO_FP_DP: -mfloat-abi=hard
+// CHECK-V8_1_NO_FP_DP: -mfpu=fp-armv8-fullfp16-sp-d16
+
+// RUN: %clang -print-multi-flags-experimental --target=arm-none-eabihf -mcpu=cortex-m85+nofp.dp | FileCheck --check-prefix=CHECK-M85_NO_FP_DP %s
+// CHECK-M85_NO_FP_DP: -march=thumbv8.1m.main{{.*}}
+// CHECK-M85_NO_FP_DP: -mfloat-abi=hard
+// CHECK-M85_NO_FP_DP: -mfpu=fp-armv8-fullfp16-sp-d16
+
 // RUN: %clang -print-multi-flags-experimental --target=aarch64-none-elf -march=armv8-a+lse | FileCheck --check-prefix=CHECK-LSE %s
 // CHECK-LSE: -march=aarch64{{.*}}+lse{{.*}}
 
diff --git a/clang/test/Driver/riscv-features.c b/clang/test/Driver/riscv-features.c
index 0039c230ec..851a7c0507 100644
--- a/clang/test/Driver/riscv-features.c
+++ b/clang/test/Driver/riscv-features.c
@@ -10,6 +10,7 @@
 // RUN: %clang --target=riscv32-unknown-elf -### %s -mrelax 2>&1 | FileCheck %s -check-prefix=RELAX
 // RUN: %clang --target=riscv32-unknown-elf -### %s -mno-relax 2>&1 | FileCheck %s -check-prefix=NO-RELAX
 
+// ANDROID: "-target-feature" "+v"
 // ANDROID: "-target-feature" "+zba"
 // ANDROID: "-target-feature" "+zbb"
 // ANDROID: "-target-feature" "+zbs"
diff --git a/clang/test/Driver/x86-march.c b/clang/test/Driver/x86-march.c
index b7134f79c5..cc993b5393 100644
--- a/clang/test/Driver/x86-march.c
+++ b/clang/test/Driver/x86-march.c
@@ -112,6 +112,14 @@
 // RUN:   | FileCheck %s -check-prefix=gracemont
 // gracemont: "-target-cpu" "gracemont"
 //
+// RUN: %clang --target=x86_64 -c -### %s -march=pantherlake 2>&1 \
+// RUN:   | FileCheck %s -check-prefix=pantherlake
+// pantherlake: "-target-cpu" "pantherlake"
+//
+// RUN: %clang --target=x86_64 -c -### %s -march=clearwaterforest 2>&1 \
+// RUN:   | FileCheck %s -check-prefix=clearwaterforest
+// clearwaterforest: "-target-cpu" "clearwaterforest"
+//
 // RUN: %clang -target x86_64-unknown-unknown -c -### %s -march=lakemont 2>&1 \
 // RUN:   | FileCheck %s -check-prefix=lakemont
 // lakemont: "-target-cpu" "lakemont"
diff --git a/clang/test/Driver/x86-target-features.c b/clang/test/Driver/x86-target-features.c
index a6ecedbb8a..3ef11ffcb6 100644
--- a/clang/test/Driver/x86-target-features.c
+++ b/clang/test/Driver/x86-target-features.c
@@ -374,6 +374,31 @@
 // EVEX512: "-target-feature" "+evex512"
 // NO-EVEX512: "-target-feature" "-evex512"
 
+// RUN: %clang --target=i386 -mavx10.1 %s -### -o %t.o 2>&1 | FileCheck -check-prefix=AVX10_1_256 %s
+// RUN: %clang --target=i386 -mavx10.1-256 %s -### -o %t.o 2>&1 | FileCheck -check-prefix=AVX10_1_256 %s
+// RUN: %clang --target=i386 -mavx10.1-512 %s -### -o %t.o 2>&1 | FileCheck -check-prefix=AVX10_1_512 %s
+// RUN: %clang --target=i386 -mavx10.1-256 -mavx10.1-512 %s -### -o %t.o 2>&1 | FileCheck -check-prefix=AVX10_1_512 %s
+// RUN: %clang --target=i386 -mavx10.1-512 -mavx10.1-256 %s -### -o %t.o 2>&1 | FileCheck -check-prefix=AVX10_1_256 %s
+// RUN: not %clang --target=i386 -march=i386 -mavx10.1-128 %s -### -o %t.o 2>&1 | FileCheck -check-prefix=BAD-AVX10 %s
+// RUN: not %clang --target=i386 -march=i386 -mavx10.a-256 %s -### -o %t.o 2>&1 | FileCheck -check-prefix=BAD-AVX10 %s
+// RUN: not %clang --target=i386 -march=i386 -mavx10.1024-512 %s -### -o %t.o 2>&1 | FileCheck -check-prefix=BAD-AVX10 %s
+// RUN: %clang --target=i386 -march=i386 -mavx10.1 -mavx512f %s -### -o %t.o 2>&1 | FileCheck -check-prefix=AVX10-AVX512 %s
+// RUN: %clang --target=i386 -march=i386 -mavx10.1 -mno-avx512f %s -### -o %t.o 2>&1 | FileCheck -check-prefix=AVX10-AVX512 %s
+// RUN: %clang --target=i386 -march=i386 -mavx10.1 -mevex512 %s -### -o %t.o 2>&1 | FileCheck -check-prefix=AVX10-EVEX512 %s
+// RUN: %clang --target=i386 -march=i386 -mavx10.1 -mno-evex512 %s -### -o %t.o 2>&1 | FileCheck -check-prefix=AVX10-EVEX512 %s
+// AVX10_1_256: "-target-feature" "+avx10.1-256"
+// AVX10_1_512: "-target-feature" "+avx10.1-512"
+// BAD-AVX10: error: unknown argument{{:?}} '-mavx10.{{.*}}'
+// AVX10-AVX512: warning: argument unused during compilation: '{{.*}}avx512f'
+// AVX10-AVX512-NOT: "avx512f"
+// AVX10-EVEX512: warning: argument unused during compilation: '{{.*}}evex512'
+// AVX10-EVEX512-NOT: "evex512"
+
+// RUN: %clang --target=i386 -musermsr %s -### -o %t.o 2>&1 | FileCheck -check-prefix=USERMSR %s
+// RUN: %clang --target=i386 -mno-usermsr %s -### -o %t.o 2>&1 | FileCheck -check-prefix=NO-USERMSR %s
+// USERMSR: "-target-feature" "+usermsr"
+// NO-USERMSR: "-target-feature" "-usermsr"
+
 // RUN: %clang --target=i386 -march=i386 -mcrc32 %s -### 2>&1 | FileCheck -check-prefix=CRC32 %s
 // RUN: %clang --target=i386 -march=i386 -mno-crc32 %s -### 2>&1 | FileCheck -check-prefix=NO-CRC32 %s
 // CRC32: "-target-feature" "+crc32"
diff --git a/clang/test/Headers/__clang_hip_math.hip b/clang/test/Headers/__clang_hip_math.hip
index 15eccc3b2b..fc18e14d82 100644
--- a/clang/test/Headers/__clang_hip_math.hip
+++ b/clang/test/Headers/__clang_hip_math.hip
@@ -231,8 +231,8 @@ extern "C" __device__ uint64_t test___make_mantissa(const char *p) {
 
 // CHECK-LABEL: @test_abs(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call noundef i32 @llvm.abs.i32(i32 [[X:%.*]], i1 true)
-// CHECK-NEXT:    ret i32 [[TMP0]]
+// CHECK-NEXT:    [[ABS_I:%.*]] = tail call noundef i32 @llvm.abs.i32(i32 [[X:%.*]], i1 true)
+// CHECK-NEXT:    ret i32 [[ABS_I]]
 //
 extern "C" __device__ int test_abs(int x) {
   return abs(x);
@@ -240,8 +240,8 @@ extern "C" __device__ int test_abs(int x) {
 
 // CHECK-LABEL: @test_labs(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call noundef i64 @llvm.abs.i64(i64 [[X:%.*]], i1 true)
-// CHECK-NEXT:    ret i64 [[TMP0]]
+// CHECK-NEXT:    [[ABS_I:%.*]] = tail call noundef i64 @llvm.abs.i64(i64 [[X:%.*]], i1 true)
+// CHECK-NEXT:    ret i64 [[ABS_I]]
 //
 extern "C" __device__ long test_labs(long x) {
   return labs(x);
@@ -249,8 +249,8 @@ extern "C" __device__ long test_labs(long x) {
 
 // CHECK-LABEL: @test_llabs(
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[TMP0:%.*]] = tail call noundef i64 @llvm.abs.i64(i64 [[X:%.*]], i1 true)
-// CHECK-NEXT:    ret i64 [[TMP0]]
+// CHECK-NEXT:    [[ABS_I:%.*]] = tail call noundef i64 @llvm.abs.i64(i64 [[X:%.*]], i1 true)
+// CHECK-NEXT:    ret i64 [[ABS_I]]
 //
 extern "C" __device__ long long test_llabs(long x) {
   return llabs(x);
@@ -2557,65 +2557,33 @@ extern "C" __device__ double test_nan(const char *tag) {
   return nan(tag);
 }
 
-// DEFAULT-LABEL: @test_nanf_emptystr(
-// DEFAULT-NEXT:  entry:
-// DEFAULT-NEXT:    ret float 0x7FF8000000000000
-//
-// FINITEONLY-LABEL: @test_nanf_emptystr(
-// FINITEONLY-NEXT:  entry:
-// FINITEONLY-NEXT:    ret float poison
-//
-// APPROX-LABEL: @test_nanf_emptystr(
-// APPROX-NEXT:  entry:
-// APPROX-NEXT:    ret float 0x7FF8000000000000
+// CHECK-LABEL: @test_nanf_emptystr(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret float 0x7FF8000000000000
 //
 extern "C" __device__ float test_nanf_emptystr() {
   return nanf("");
 }
 
-// DEFAULT-LABEL: @test_nan_emptystr(
-// DEFAULT-NEXT:  entry:
-// DEFAULT-NEXT:    ret double 0x7FF8000000000000
-//
-// FINITEONLY-LABEL: @test_nan_emptystr(
-// FINITEONLY-NEXT:  entry:
-// FINITEONLY-NEXT:    ret double poison
-//
-// APPROX-LABEL: @test_nan_emptystr(
-// APPROX-NEXT:  entry:
-// APPROX-NEXT:    ret double 0x7FF8000000000000
+// CHECK-LABEL: @test_nan_emptystr(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret double 0x7FF8000000000000
 //
 extern "C" __device__ double test_nan_emptystr() {
   return nan("");
 }
 
-// DEFAULT-LABEL: @test_nanf_fill(
-// DEFAULT-NEXT:  entry:
-// DEFAULT-NEXT:    ret float 0x7FF8000000000000
-//
-// FINITEONLY-LABEL: @test_nanf_fill(
-// FINITEONLY-NEXT:  entry:
-// FINITEONLY-NEXT:    ret float poison
-//
-// APPROX-LABEL: @test_nanf_fill(
-// APPROX-NEXT:  entry:
-// APPROX-NEXT:    ret float 0x7FF8000000000000
+// CHECK-LABEL: @test_nanf_fill(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret float 0x7FF8000000000000
 //
 extern "C" __device__ float test_nanf_fill() {
   return nanf("0x456");
 }
 
-// DEFAULT-LABEL: @test_nan_fill(
-// DEFAULT-NEXT:  entry:
-// DEFAULT-NEXT:    ret double 0x7FF8000000000000
-//
-// FINITEONLY-LABEL: @test_nan_fill(
-// FINITEONLY-NEXT:  entry:
-// FINITEONLY-NEXT:    ret double poison
-//
-// APPROX-LABEL: @test_nan_fill(
-// APPROX-NEXT:  entry:
-// APPROX-NEXT:    ret double 0x7FF8000000000000
+// CHECK-LABEL: @test_nan_fill(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret double 0x7FF8000000000000
 //
 extern "C" __device__ double test_nan_fill() {
   return nan("0x123");
diff --git a/clang/test/Headers/stdckdint.c b/clang/test/Headers/stdckdint.c
new file mode 100644
index 0000000000..896c740360
--- /dev/null
+++ b/clang/test/Headers/stdckdint.c
@@ -0,0 +1,53 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 3
+// RUN: %clang_cc1 -triple=x86_64 -emit-llvm -verify -std=c23 %s -o - | FileCheck %s
+
+// expected-no-diagnostics
+
+#include <stdckdint.h>
+
+_Static_assert(__STDC_VERSION_STDCKDINT_H__ == 202311L, "");
+
+// CHECK-LABEL: define dso_local zeroext i1 @test_ckd_add(
+// CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[RESULT:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 -1073741826, i32 -1073741826)
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { i32, i1 } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { i32, i1 } [[TMP0]], 0
+// CHECK-NEXT:    store i32 [[TMP2]], ptr [[RESULT]], align 4
+// CHECK-NEXT:    ret i1 [[TMP1]]
+//
+bool test_ckd_add() {
+  int result;
+  return ckd_add(&result, -1073741826, -1073741826);
+}
+
+// CHECK-LABEL: define dso_local zeroext i1 @test_ckd_sub(
+// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[RESULT:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 -1073741826, i32 1073741826)
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { i32, i1 } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { i32, i1 } [[TMP0]], 0
+// CHECK-NEXT:    store i32 [[TMP2]], ptr [[RESULT]], align 4
+// CHECK-NEXT:    ret i1 [[TMP1]]
+//
+bool test_ckd_sub() {
+  int result;
+  return ckd_sub(&result, -1073741826, 1073741826);
+}
+
+// CHECK-LABEL: define dso_local zeroext i1 @test_ckd_mul(
+// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[RESULT:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 -1073741826, i32 2)
+// CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { i32, i1 } [[TMP0]], 1
+// CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { i32, i1 } [[TMP0]], 0
+// CHECK-NEXT:    store i32 [[TMP2]], ptr [[RESULT]], align 4
+// CHECK-NEXT:    ret i1 [[TMP1]]
+//
+bool test_ckd_mul() {
+  int result;
+  return ckd_mul(&result, -1073741826, 2);
+}
diff --git a/clang/test/Index/USR/decl-context.cpp b/clang/test/Index/USR/decl-context.cpp
new file mode 100644
index 0000000000..a57137a5c8
--- /dev/null
+++ b/clang/test/Index/USR/decl-context.cpp
@@ -0,0 +1,14 @@
+// RUN: c-index-test core -print-source-symbols -- -std=c++20 %s | FileCheck %s
+
+namespace ns {
+namespace {
+struct Foo {};
+// CHECK: [[@LINE-1]]:8 | struct/C | Foo | c:decl-context.cpp@N@ns@aN@S@Foo
+}
+}
+namespace ns2 {
+namespace {
+struct Foo {};
+// CHECK: [[@LINE-1]]:8 | struct/C | Foo | c:decl-context.cpp@N@ns2@aN@S@Foo
+}
+}
diff --git a/clang/test/Misc/pragma-attribute-supported-attributes-list.test b/clang/test/Misc/pragma-attribute-supported-attributes-list.test
index eaf6d34421..f48126775c 100644
--- a/clang/test/Misc/pragma-attribute-supported-attributes-list.test
+++ b/clang/test/Misc/pragma-attribute-supported-attributes-list.test
@@ -56,6 +56,7 @@
 // CHECK-NEXT: ConsumableAutoCast (SubjectMatchRule_record)
 // CHECK-NEXT: ConsumableSetOnRead (SubjectMatchRule_record)
 // CHECK-NEXT: Convergent (SubjectMatchRule_function)
+// CHECK-NEXT: CountedBy (SubjectMatchRule_field)
 // CHECK-NEXT: DLLExport (SubjectMatchRule_function, SubjectMatchRule_variable, SubjectMatchRule_record, SubjectMatchRule_objc_interface)
 // CHECK-NEXT: DLLImport (SubjectMatchRule_function, SubjectMatchRule_variable, SubjectMatchRule_record, SubjectMatchRule_objc_interface)
 // CHECK-NEXT: Destructor (SubjectMatchRule_function)
diff --git a/clang/test/Misc/target-invalid-cpu-note.c b/clang/test/Misc/target-invalid-cpu-note.c
index a639b0ddde..b2a04ebdbc 100644
--- a/clang/test/Misc/target-invalid-cpu-note.c
+++ b/clang/test/Misc/target-invalid-cpu-note.c
@@ -13,19 +13,19 @@
 
 // RUN: not %clang_cc1 -triple i386--- -target-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix X86
 // X86: error: unknown target CPU 'not-a-cpu'
-// X86-NEXT: note: valid target CPU values are: i386, i486, winchip-c6, winchip2, c3, i586, pentium, pentium-mmx, pentiumpro, i686, pentium2, pentium3, pentium3m, pentium-m, c3-2, yonah, pentium4, pentium4m, prescott, nocona, core2, penryn, bonnell, atom, silvermont, slm, goldmont, goldmont-plus, tremont, nehalem, corei7, westmere, sandybridge, corei7-avx, ivybridge, core-avx-i, haswell, core-avx2, broadwell, skylake, skylake-avx512, skx, cascadelake, cooperlake, cannonlake, icelake-client, rocketlake, icelake-server, tigerlake, sapphirerapids, alderlake, raptorlake, meteorlake, arrowlake, arrowlake-s, lunarlake, gracemont, sierraforest, grandridge, graniterapids, graniterapids-d, emeraldrapids, knl, knm, lakemont, k6, k6-2, k6-3, athlon, athlon-tbird, athlon-xp, athlon-mp, athlon-4, k8, athlon64, athlon-fx, opteron, k8-sse3, athlon64-sse3, opteron-sse3, amdfam10, barcelona, btver1, btver2, bdver1, bdver2, bdver3, bdver4, znver1, znver2, znver3, znver4, x86-64, x86-64-v2, x86-64-v3, x86-64-v4, geode{{$}}
+// X86-NEXT: note: valid target CPU values are: i386, i486, winchip-c6, winchip2, c3, i586, pentium, pentium-mmx, pentiumpro, i686, pentium2, pentium3, pentium3m, pentium-m, c3-2, yonah, pentium4, pentium4m, prescott, nocona, core2, penryn, bonnell, atom, silvermont, slm, goldmont, goldmont-plus, tremont, nehalem, corei7, westmere, sandybridge, corei7-avx, ivybridge, core-avx-i, haswell, core-avx2, broadwell, skylake, skylake-avx512, skx, cascadelake, cooperlake, cannonlake, icelake-client, rocketlake, icelake-server, tigerlake, sapphirerapids, alderlake, raptorlake, meteorlake, arrowlake, arrowlake-s, lunarlake, gracemont, pantherlake, sierraforest, grandridge, graniterapids, graniterapids-d, emeraldrapids, clearwaterforest, knl, knm, lakemont, k6, k6-2, k6-3, athlon, athlon-tbird, athlon-xp, athlon-mp, athlon-4, k8, athlon64, athlon-fx, opteron, k8-sse3, athlon64-sse3, opteron-sse3, amdfam10, barcelona, btver1, btver2, bdver1, bdver2, bdver3, bdver4, znver1, znver2, znver3, znver4, x86-64, x86-64-v2, x86-64-v3, x86-64-v4, geode{{$}}
 
 // RUN: not %clang_cc1 -triple x86_64--- -target-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix X86_64
 // X86_64: error: unknown target CPU 'not-a-cpu'
-// X86_64-NEXT: note: valid target CPU values are: nocona, core2, penryn, bonnell, atom, silvermont, slm, goldmont, goldmont-plus, tremont, nehalem, corei7, westmere, sandybridge, corei7-avx, ivybridge, core-avx-i, haswell, core-avx2, broadwell, skylake, skylake-avx512, skx, cascadelake, cooperlake, cannonlake, icelake-client, rocketlake, icelake-server, tigerlake, sapphirerapids, alderlake, raptorlake, meteorlake, arrowlake, arrowlake-s, lunarlake, gracemont, sierraforest, grandridge, graniterapids, graniterapids-d, emeraldrapids, knl, knm, k8, athlon64, athlon-fx, opteron, k8-sse3, athlon64-sse3, opteron-sse3, amdfam10, barcelona, btver1, btver2, bdver1, bdver2, bdver3, bdver4, znver1, znver2, znver3, znver4, x86-64, x86-64-v2, x86-64-v3, x86-64-v4{{$}}
+// X86_64-NEXT: note: valid target CPU values are: nocona, core2, penryn, bonnell, atom, silvermont, slm, goldmont, goldmont-plus, tremont, nehalem, corei7, westmere, sandybridge, corei7-avx, ivybridge, core-avx-i, haswell, core-avx2, broadwell, skylake, skylake-avx512, skx, cascadelake, cooperlake, cannonlake, icelake-client, rocketlake, icelake-server, tigerlake, sapphirerapids, alderlake, raptorlake, meteorlake, arrowlake, arrowlake-s, lunarlake, gracemont, pantherlake, sierraforest, grandridge, graniterapids, graniterapids-d, emeraldrapids, clearwaterforest, knl, knm, k8, athlon64, athlon-fx, opteron, k8-sse3, athlon64-sse3, opteron-sse3, amdfam10, barcelona, btver1, btver2, bdver1, bdver2, bdver3, bdver4, znver1, znver2, znver3, znver4, x86-64, x86-64-v2, x86-64-v3, x86-64-v4{{$}}
 
 // RUN: not %clang_cc1 -triple i386--- -tune-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix TUNE_X86
 // TUNE_X86: error: unknown target CPU 'not-a-cpu'
-// TUNE_X86-NEXT: note: valid target CPU values are: i386, i486, winchip-c6, winchip2, c3, i586, pentium, pentium-mmx, pentiumpro, i686, pentium2, pentium3, pentium3m, pentium-m, c3-2, yonah, pentium4, pentium4m, prescott, nocona, core2, penryn, bonnell, atom, silvermont, slm, goldmont, goldmont-plus, tremont, nehalem, corei7, westmere, sandybridge, corei7-avx, ivybridge, core-avx-i, haswell, core-avx2, broadwell, skylake, skylake-avx512, skx, cascadelake, cooperlake, cannonlake, icelake-client, rocketlake, icelake-server, tigerlake, sapphirerapids, alderlake, raptorlake, meteorlake, arrowlake, arrowlake-s, lunarlake, gracemont, sierraforest, grandridge, graniterapids, graniterapids-d, emeraldrapids, knl, knm, lakemont, k6, k6-2, k6-3, athlon, athlon-tbird, athlon-xp, athlon-mp, athlon-4, k8, athlon64, athlon-fx, opteron, k8-sse3, athlon64-sse3, opteron-sse3, amdfam10, barcelona, btver1, btver2, bdver1, bdver2, bdver3, bdver4, znver1, znver2, znver3, znver4, x86-64, geode{{$}}
+// TUNE_X86-NEXT: note: valid target CPU values are: i386, i486, winchip-c6, winchip2, c3, i586, pentium, pentium-mmx, pentiumpro, i686, pentium2, pentium3, pentium3m, pentium-m, c3-2, yonah, pentium4, pentium4m, prescott, nocona, core2, penryn, bonnell, atom, silvermont, slm, goldmont, goldmont-plus, tremont, nehalem, corei7, westmere, sandybridge, corei7-avx, ivybridge, core-avx-i, haswell, core-avx2, broadwell, skylake, skylake-avx512, skx, cascadelake, cooperlake, cannonlake, icelake-client, rocketlake, icelake-server, tigerlake, sapphirerapids, alderlake, raptorlake, meteorlake, arrowlake, arrowlake-s, lunarlake, gracemont, pantherlake, sierraforest, grandridge, graniterapids, graniterapids-d, emeraldrapids, clearwaterforest, knl, knm, lakemont, k6, k6-2, k6-3, athlon, athlon-tbird, athlon-xp, athlon-mp, athlon-4, k8, athlon64, athlon-fx, opteron, k8-sse3, athlon64-sse3, opteron-sse3, amdfam10, barcelona, btver1, btver2, bdver1, bdver2, bdver3, bdver4, znver1, znver2, znver3, znver4, x86-64, geode{{$}}
 
 // RUN: not %clang_cc1 -triple x86_64--- -tune-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix TUNE_X86_64
 // TUNE_X86_64: error: unknown target CPU 'not-a-cpu'
-// TUNE_X86_64-NEXT: note: valid target CPU values are: i386, i486, winchip-c6, winchip2, c3, i586, pentium, pentium-mmx, pentiumpro, i686, pentium2, pentium3, pentium3m, pentium-m, c3-2, yonah, pentium4, pentium4m, prescott, nocona, core2, penryn, bonnell, atom, silvermont, slm, goldmont, goldmont-plus, tremont, nehalem, corei7, westmere, sandybridge, corei7-avx, ivybridge, core-avx-i, haswell, core-avx2, broadwell, skylake, skylake-avx512, skx, cascadelake, cooperlake, cannonlake, icelake-client, rocketlake, icelake-server, tigerlake, sapphirerapids, alderlake, raptorlake, meteorlake, arrowlake, arrowlake-s, lunarlake, gracemont, sierraforest, grandridge, graniterapids, graniterapids-d, emeraldrapids, knl, knm, lakemont, k6, k6-2, k6-3, athlon, athlon-tbird, athlon-xp, athlon-mp, athlon-4, k8, athlon64, athlon-fx, opteron, k8-sse3, athlon64-sse3, opteron-sse3, amdfam10, barcelona, btver1, btver2, bdver1, bdver2, bdver3, bdver4, znver1, znver2, znver3, znver4, x86-64, geode{{$}}
+// TUNE_X86_64-NEXT: note: valid target CPU values are: i386, i486, winchip-c6, winchip2, c3, i586, pentium, pentium-mmx, pentiumpro, i686, pentium2, pentium3, pentium3m, pentium-m, c3-2, yonah, pentium4, pentium4m, prescott, nocona, core2, penryn, bonnell, atom, silvermont, slm, goldmont, goldmont-plus, tremont, nehalem, corei7, westmere, sandybridge, corei7-avx, ivybridge, core-avx-i, haswell, core-avx2, broadwell, skylake, skylake-avx512, skx, cascadelake, cooperlake, cannonlake, icelake-client, rocketlake, icelake-server, tigerlake, sapphirerapids, alderlake, raptorlake, meteorlake, arrowlake, arrowlake-s, lunarlake, gracemont, pantherlake, sierraforest, grandridge, graniterapids, graniterapids-d, emeraldrapids, clearwaterforest, knl, knm, lakemont, k6, k6-2, k6-3, athlon, athlon-tbird, athlon-xp, athlon-mp, athlon-4, k8, athlon64, athlon-fx, opteron, k8-sse3, athlon64-sse3, opteron-sse3, amdfam10, barcelona, btver1, btver2, bdver1, bdver2, bdver3, bdver4, znver1, znver2, znver3, znver4, x86-64, geode{{$}}
 
 // RUN: not %clang_cc1 -triple nvptx--- -target-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix NVPTX
 // NVPTX: error: unknown target CPU 'not-a-cpu'
diff --git a/clang/test/Misc/warning-wall.c b/clang/test/Misc/warning-wall.c
index 57866713fc..c0cf8324e6 100644
--- a/clang/test/Misc/warning-wall.c
+++ b/clang/test/Misc/warning-wall.c
@@ -105,6 +105,7 @@ CHECK-NEXT:  -Wswitch
 CHECK-NEXT:  -Wswitch-bool
 CHECK-NEXT:  -Wmisleading-indentation
 CHECK-NEXT:  -Wpacked-non-pod
-
+CHECK-NEXT:  -Wvla-extension
+CHECK-NEXT:    -Wvla-extension-static-assert
 
 CHECK-NOT:-W
diff --git a/clang/test/Modules/Inputs/all-product-headers.yaml b/clang/test/Modules/Inputs/all-product-headers.yaml
new file mode 100644
index 0000000000..53d683f2ad
--- /dev/null
+++ b/clang/test/Modules/Inputs/all-product-headers.yaml
@@ -0,0 +1,33 @@
+{
+  'version': 0,
+  'case-sensitive': 'false',
+  'roots': [
+    {
+      'type': 'directory',
+      'name': "DUMMY_DIR/build/A.framework/PrivateHeaders"
+      'contents': [
+        {
+          'type': 'file',
+          'name': "A.h",
+          'external-contents': "DUMMY_DIR/sources/A.h"
+        }
+      ]
+    },
+    {
+      'type': 'directory',
+      'name': "DUMMY_DIR/build/A.framework/Modules"
+      'contents': [
+        {
+          'type': 'file',
+          'name': "module.modulemap",
+          'external-contents': "DUMMY_DIR/build/module.modulemap"
+        },
+        {
+          'type': 'file',
+          'name': "module.private.modulemap",
+          'external-contents': "DUMMY_DIR/build/module.private.modulemap"
+        }
+      ]
+    }
+  ]
+}
diff --git a/clang/test/Modules/explicit-build.cpp b/clang/test/Modules/explicit-build.cpp
index 16eb604708..7fca0082ea 100644
--- a/clang/test/Modules/explicit-build.cpp
+++ b/clang/test/Modules/explicit-build.cpp
@@ -161,7 +161,7 @@
 // RUN:            -fmodule-file=%t/not.pcm \
 // RUN:            %s 2>&1 | FileCheck --check-prefix=CHECK-BAD-FILE %s
 //
-// CHECK-BAD-FILE: fatal error: file '{{.*}}not.pcm' is not a valid precompiled module file
+// CHECK-BAD-FILE: fatal error: file '{{.*}}not.pcm' is not a valid precompiled module file: file too small to contain AST file magic
 
 // RUN: not %clang_cc1 -x c++ -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -Rmodule-build -fno-modules-error-recovery \
 // RUN:            -fmodule-file=%t/nonexistent.pcm \
diff --git a/clang/test/Modules/modulemap-collision.m b/clang/test/Modules/modulemap-collision.m
new file mode 100644
index 0000000000..5ada45da3d
--- /dev/null
+++ b/clang/test/Modules/modulemap-collision.m
@@ -0,0 +1,15 @@
+// REQUIRES: shell
+
+// RUN: rm -rf %t
+// RUN: mkdir -p %t/sources %t/build
+// RUN: echo "// A.h" > %t/sources/A.h
+// RUN: echo "framework module A {}" > %t/sources/module.modulemap
+// RUN: echo "framework module A.Private { umbrella header \"A.h\" }" > %t/sources/module.private.modulemap
+// RUN: cp %t/sources/module.modulemap %t/build/module.modulemap
+// RUN: cp %t/sources/module.private.modulemap %t/build/module.private.modulemap
+
+// RUN: sed -e "s:DUMMY_DIR:%t:g" %S/Inputs/all-product-headers.yaml > %t/build/all-product-headers.yaml
+// RUN: %clang_cc1 -fsyntax-only -ivfsoverlay %t/build/all-product-headers.yaml -F%t/build -fmodules -fimplicit-module-maps -Wno-private-module -fmodules-cache-path=%t/cache -x objective-c %s -verify
+
+// expected-no-diagnostics
+#import <A/A.h>
diff --git a/clang/test/Modules/no-undeclared-includes.c b/clang/test/Modules/no-undeclared-includes.c
new file mode 100644
index 0000000000..83a654f6ed
--- /dev/null
+++ b/clang/test/Modules/no-undeclared-includes.c
@@ -0,0 +1,31 @@
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+// RUN: %clang_cc1 -fmodules-cache-path=%t -fmodules -fimplicit-module-maps -I %t %t/no-undeclared-includes.c -verify
+
+//--- no-undeclared-includes.c
+// expected-no-diagnostics
+#include <assert.h>
+
+//--- assert.h
+#include <base.h>
+
+//--- base.h
+#ifndef base_h
+#define base_h
+
+
+
+#endif /* base_h */
+
+//--- module.modulemap
+module cstd [system] [no_undeclared_includes] {
+  use base
+  module assert {
+    textual header "assert.h"
+  }
+}
+
+module base [system] {
+  header "base.h"
+  export *
+}
diff --git a/clang/test/OpenMP/debug-info-openmp-array.cpp b/clang/test/OpenMP/debug-info-openmp-array.cpp
index 3c9ce38656..6c5c3bbffe 100644
--- a/clang/test/OpenMP/debug-info-openmp-array.cpp
+++ b/clang/test/OpenMP/debug-info-openmp-array.cpp
@@ -1,7 +1,7 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _
-// RUN: %clang_cc1 -triple x86_64-unknown-linux -fopenmp -x c++ %s -verify -debug-info-kind=limited -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -triple x86_64-unknown-linux -Wno-vla -fopenmp -x c++ %s -verify -debug-info-kind=limited -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
 
-// RUN: %clang_cc1 -triple x86_64-unknown-linux -fopenmp-simd -x c++ %s -verify -debug-info-kind=limited -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -triple x86_64-unknown-linux -Wno-vla -fopenmp-simd -x c++ %s -verify -debug-info-kind=limited -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // expected-no-diagnostics
 
 void f(int m) {
diff --git a/clang/test/OpenMP/for_reduction_codegen.cpp b/clang/test/OpenMP/for_reduction_codegen.cpp
index 7dc8b81d60..893c606f8d 100644
--- a/clang/test/OpenMP/for_reduction_codegen.cpp
+++ b/clang/test/OpenMP/for_reduction_codegen.cpp
@@ -1,15 +1,15 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --check-globals --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _ --global-value-regex ".gomp_critical_user[a-zA-Z_.]+"
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap %s --check-prefix=CHECK3
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap %s --check-prefix=CHECK4
+// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap %s --check-prefix=CHECK4
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // expected-no-diagnostics
 #ifndef HEADER
 #define HEADER
diff --git a/clang/test/OpenMP/for_reduction_codegen_UDR.cpp b/clang/test/OpenMP/for_reduction_codegen_UDR.cpp
index d9b7aba066..14d903fd7a 100644
--- a/clang/test/OpenMP/for_reduction_codegen_UDR.cpp
+++ b/clang/test/OpenMP/for_reduction_codegen_UDR.cpp
@@ -1,11 +1,11 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
 // expected-no-diagnostics
 #ifndef HEADER
 #define HEADER
diff --git a/clang/test/OpenMP/master_taskloop_firstprivate_codegen.cpp b/clang/test/OpenMP/master_taskloop_firstprivate_codegen.cpp
index 4dfce7fd3e..a238422755 100644
--- a/clang/test/OpenMP/master_taskloop_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/master_taskloop_firstprivate_codegen.cpp
@@ -1,16 +1,16 @@
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LAMBDA %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=BLOCKS %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=ARRAY %s
+// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LAMBDA %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=BLOCKS %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=ARRAY %s
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 // expected-no-diagnostics
 
diff --git a/clang/test/OpenMP/master_taskloop_in_reduction_codegen.cpp b/clang/test/OpenMP/master_taskloop_in_reduction_codegen.cpp
index d22d884d32..9886b71dc1 100644
--- a/clang/test/OpenMP/master_taskloop_in_reduction_codegen.cpp
+++ b/clang/test/OpenMP/master_taskloop_in_reduction_codegen.cpp
@@ -1,11 +1,11 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _
-// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -triple x86_64-apple-darwin10 -fopenmp -x c++ -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -Wno-vla -triple x86_64-apple-darwin10 -fopenmp -x c++ -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -x c++ -triple x86_64-apple-darwin10 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -x c++ -triple x86_64-apple-darwin10 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -triple x86_64-apple-darwin10 -fopenmp-simd -x c++ -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -Wno-vla -triple x86_64-apple-darwin10 -fopenmp-simd -x c++ -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // expected-no-diagnostics
 #ifndef HEADER
 #define HEADER
diff --git a/clang/test/OpenMP/master_taskloop_lastprivate_codegen.cpp b/clang/test/OpenMP/master_taskloop_lastprivate_codegen.cpp
index d86ecd1b6e..63900ffeb3 100644
--- a/clang/test/OpenMP/master_taskloop_lastprivate_codegen.cpp
+++ b/clang/test/OpenMP/master_taskloop_lastprivate_codegen.cpp
@@ -1,18 +1,18 @@
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LAMBDA %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=BLOCKS %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=ARRAY %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DLOOP -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LOOP %s
+// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LAMBDA %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=BLOCKS %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=ARRAY %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DLOOP -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LOOP %s
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DLOOP -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DLOOP -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 // expected-no-diagnostics
 
diff --git a/clang/test/OpenMP/master_taskloop_private_codegen.cpp b/clang/test/OpenMP/master_taskloop_private_codegen.cpp
index 1a63c0a664..518808f5ca 100644
--- a/clang/test/OpenMP/master_taskloop_private_codegen.cpp
+++ b/clang/test/OpenMP/master_taskloop_private_codegen.cpp
@@ -1,16 +1,16 @@
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LAMBDA %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=BLOCKS %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=ARRAY %s
+// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LAMBDA %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=BLOCKS %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=ARRAY %s
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 // expected-no-diagnostics
 
diff --git a/clang/test/OpenMP/master_taskloop_reduction_codegen.cpp b/clang/test/OpenMP/master_taskloop_reduction_codegen.cpp
index e2178f6457..05450785c0 100644
--- a/clang/test/OpenMP/master_taskloop_reduction_codegen.cpp
+++ b/clang/test/OpenMP/master_taskloop_reduction_codegen.cpp
@@ -1,6 +1,6 @@
-// RUN: %clang_cc1 -fopenmp -x c++ %s -verify -debug-info-kind=limited -emit-llvm -o - -triple powerpc64le-unknown-linux-gnu -fnoopenmp-use-tls -std=c++98 | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -x c++ %s -verify -Wno-vla -debug-info-kind=limited -emit-llvm -o - -triple powerpc64le-unknown-linux-gnu -fnoopenmp-use-tls -std=c++98 | FileCheck %s
 
-// RUN: %clang_cc1 -fopenmp-simd -x c++ %s -verify -debug-info-kind=limited -emit-llvm -o - -triple powerpc64le-unknown-linux-gnu -fnoopenmp-use-tls -std=c++98 | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ %s -verify -Wno-vla -debug-info-kind=limited -emit-llvm -o - -triple powerpc64le-unknown-linux-gnu -fnoopenmp-use-tls -std=c++98 | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 // expected-no-diagnostics
 
diff --git a/clang/test/OpenMP/master_taskloop_simd_firstprivate_codegen.cpp b/clang/test/OpenMP/master_taskloop_simd_firstprivate_codegen.cpp
index a8da845245..785efe3ec1 100644
--- a/clang/test/OpenMP/master_taskloop_simd_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/master_taskloop_simd_firstprivate_codegen.cpp
@@ -1,16 +1,16 @@
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LAMBDA %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=BLOCKS %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=ARRAY %s
+// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LAMBDA %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=BLOCKS %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=ARRAY %s
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 // expected-no-diagnostics
 
diff --git a/clang/test/OpenMP/master_taskloop_simd_in_reduction_codegen.cpp b/clang/test/OpenMP/master_taskloop_simd_in_reduction_codegen.cpp
index aeecfdcd33..76e5794b6f 100644
--- a/clang/test/OpenMP/master_taskloop_simd_in_reduction_codegen.cpp
+++ b/clang/test/OpenMP/master_taskloop_simd_in_reduction_codegen.cpp
@@ -1,11 +1,11 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _
-// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -triple x86_64-apple-darwin10 -fopenmp -x c++ -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -Wno-vla -triple x86_64-apple-darwin10 -fopenmp -x c++ -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -x c++ -triple x86_64-apple-darwin10 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -x c++ -triple x86_64-apple-darwin10 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -triple x86_64-apple-darwin10 -fopenmp-simd -x c++ -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -Wno-vla -triple x86_64-apple-darwin10 -fopenmp-simd -x c++ -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
 // expected-no-diagnostics
 #ifndef HEADER
 #define HEADER
diff --git a/clang/test/OpenMP/master_taskloop_simd_lastprivate_codegen.cpp b/clang/test/OpenMP/master_taskloop_simd_lastprivate_codegen.cpp
index 8d7de7ba58..8a2af21fba 100644
--- a/clang/test/OpenMP/master_taskloop_simd_lastprivate_codegen.cpp
+++ b/clang/test/OpenMP/master_taskloop_simd_lastprivate_codegen.cpp
@@ -1,16 +1,16 @@
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LAMBDA %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=BLOCKS %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=ARRAY %s
+// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LAMBDA %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=BLOCKS %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=ARRAY %s
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 // expected-no-diagnostics
 
diff --git a/clang/test/OpenMP/master_taskloop_simd_private_codegen.cpp b/clang/test/OpenMP/master_taskloop_simd_private_codegen.cpp
index 4aae9ea425..b70a45f55c 100644
--- a/clang/test/OpenMP/master_taskloop_simd_private_codegen.cpp
+++ b/clang/test/OpenMP/master_taskloop_simd_private_codegen.cpp
@@ -1,16 +1,16 @@
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LAMBDA %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=BLOCKS %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=ARRAY %s
+// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LAMBDA %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=BLOCKS %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=ARRAY %s
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 // expected-no-diagnostics
 
diff --git a/clang/test/OpenMP/master_taskloop_simd_reduction_codegen.cpp b/clang/test/OpenMP/master_taskloop_simd_reduction_codegen.cpp
index 0782d4710d..2e07b136f5 100644
--- a/clang/test/OpenMP/master_taskloop_simd_reduction_codegen.cpp
+++ b/clang/test/OpenMP/master_taskloop_simd_reduction_codegen.cpp
@@ -1,6 +1,6 @@
-// RUN: %clang_cc1 -fopenmp -x c++ %s -verify -debug-info-kind=limited -emit-llvm -o - -triple powerpc64le-unknown-linux-gnu -std=c++98 -fnoopenmp-use-tls | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -x c++ %s -verify -Wno-vla -debug-info-kind=limited -emit-llvm -o - -triple powerpc64le-unknown-linux-gnu -std=c++98 -fnoopenmp-use-tls | FileCheck %s
 
-// RUN: %clang_cc1 -fopenmp-simd -x c++ %s -verify -debug-info-kind=limited -emit-llvm -o - -triple powerpc64le-unknown-linux-gnu -std=c++98 -fnoopenmp-use-tls | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ %s -verify -Wno-vla -debug-info-kind=limited -emit-llvm -o - -triple powerpc64le-unknown-linux-gnu -std=c++98 -fnoopenmp-use-tls | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 // expected-no-diagnostics
 
diff --git a/clang/test/OpenMP/nvptx_target_codegen.cpp b/clang/test/OpenMP/nvptx_target_codegen.cpp
index 58e487e866..cea2fd4e75 100644
--- a/clang/test/OpenMP/nvptx_target_codegen.cpp
+++ b/clang/test/OpenMP/nvptx_target_codegen.cpp
@@ -1,10 +1,10 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _
 // Test target codegen - host bc file has to be created first.
-// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -fopenmp -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - -disable-llvm-optzns | FileCheck %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -fopenmp -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - -disable-llvm-optzns | FileCheck %s --check-prefix=CHECK2
-// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -fopenmp -fexceptions -fcxx-exceptions -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - -disable-llvm-optzns | FileCheck %s --check-prefix=CHECK2
+// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -Wno-vla -fopenmp -x c++ -triple nvptx64-unknown-unknown -fopenmp-targets=nvptx64-nvidia-cuda -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - -disable-llvm-optzns | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -Wno-vla -fopenmp -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - -disable-llvm-optzns | FileCheck %s --check-prefix=CHECK2
+// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -Wno-vla -fopenmp -fexceptions -fcxx-exceptions -x c++ -triple nvptx-unknown-unknown -fopenmp-targets=nvptx-nvidia-cuda -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - -disable-llvm-optzns | FileCheck %s --check-prefix=CHECK2
 // expected-no-diagnostics
 
 #ifndef HEADER
diff --git a/clang/test/OpenMP/parallel_ast_print.cpp b/clang/test/OpenMP/parallel_ast_print.cpp
index 341cd8a47a..dcfec564ed 100644
--- a/clang/test/OpenMP/parallel_ast_print.cpp
+++ b/clang/test/OpenMP/parallel_ast_print.cpp
@@ -1,18 +1,18 @@
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=50 -ast-print %s | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=50 -ast-print %s | FileCheck %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -x c++ -std=c++11 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -std=c++11 -include-pch %t -fsyntax-only -verify -Wno-vla %s -ast-print | FileCheck %s
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=50 -ast-print %s | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=50 -ast-print %s | FileCheck %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=50 -x c++ -std=c++11 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=50 -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=50 -std=c++11 -include-pch %t -fsyntax-only -verify -Wno-vla %s -ast-print | FileCheck %s
 
-// RUN: %clang_cc1 -DOMP51 -verify -fopenmp -ast-print %s | FileCheck -check-prefixes=CHECK,OMP51 %s
+// RUN: %clang_cc1 -DOMP51 -verify -Wno-vla -fopenmp -ast-print %s | FileCheck -check-prefixes=CHECK,OMP51 %s
 // RUN: %clang_cc1 -DOMP51 -fopenmp -x c++ -std=c++11 -emit-pch -o %t %s
-// RUN: %clang_cc1 -DOMP51 -fopenmp -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck -check-prefixes=CHECK,OMP51 %s
+// RUN: %clang_cc1 -DOMP51 -fopenmp -std=c++11 -include-pch %t -fsyntax-only -verify -Wno-vla %s -ast-print | FileCheck -check-prefixes=CHECK,OMP51 %s
 
-// RUN: %clang_cc1 -DOMP51 -verify -fopenmp-simd -ast-print %s | FileCheck -check-prefixes=CHECK,OMP51 %s
+// RUN: %clang_cc1 -DOMP51 -verify -Wno-vla -fopenmp-simd -ast-print %s | FileCheck -check-prefixes=CHECK,OMP51 %s
 // RUN: %clang_cc1 -DOMP51 -fopenmp-simd -x c++ -std=c++11 -emit-pch -o %t %s
-// RUN: %clang_cc1 -DOMP51 -fopenmp-simd -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck -check-prefixes=CHECK,OMP51 %s
+// RUN: %clang_cc1 -DOMP51 -fopenmp-simd -std=c++11 -include-pch %t -fsyntax-only -verify -Wno-vla %s -ast-print | FileCheck -check-prefixes=CHECK,OMP51 %s
 // expected-no-diagnostics
 
 #ifndef HEADER
diff --git a/clang/test/OpenMP/parallel_codegen.cpp b/clang/test/OpenMP/parallel_codegen.cpp
index b90729907a..5c98761be0 100644
--- a/clang/test/OpenMP/parallel_codegen.cpp
+++ b/clang/test/OpenMP/parallel_codegen.cpp
@@ -1,17 +1,17 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -emit-llvm %s -triple x86_64-unknown-linux -fexceptions -fcxx-exceptions -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -emit-llvm %s -triple x86_64-unknown-linux -fexceptions -fcxx-exceptions -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -debug-info-kind=limited -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK2
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-enable-irbuilder -DIRBUILDER -x c++ -emit-llvm %s -triple x86_64-unknown-linux -fexceptions -fcxx-exceptions -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -debug-info-kind=limited -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK2
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-enable-irbuilder -DIRBUILDER -x c++ -emit-llvm %s -triple x86_64-unknown-linux -fexceptions -fcxx-exceptions -o - | FileCheck %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -fopenmp -fopenmp-enable-irbuilder -DIRBUILDER -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-enable-irbuilder -DIRBUILDER -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -debug-info-kind=limited -gno-column-info -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK4
+// RUN: %clang_cc1 -fopenmp -fopenmp-enable-irbuilder -DIRBUILDER -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -debug-info-kind=limited -gno-column-info -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK4
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -emit-llvm %s -triple x86_64-unknown-linux -fexceptions -fcxx-exceptions -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -emit-llvm %s -triple x86_64-unknown-linux -fexceptions -fcxx-exceptions -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -debug-info-kind=limited -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-enable-irbuilder -x c++ -emit-llvm %s -triple x86_64-unknown-linux -fexceptions -fcxx-exceptions -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -debug-info-kind=limited -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-enable-irbuilder -x c++ -emit-llvm %s -triple x86_64-unknown-linux -fexceptions -fcxx-exceptions -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-enable-irbuilder -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-enable-irbuilder -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -debug-info-kind=limited -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-enable-irbuilder -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -debug-info-kind=limited -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // expected-no-diagnostics
 #ifndef HEADER
 #define HEADER
diff --git a/clang/test/OpenMP/parallel_firstprivate_codegen.cpp b/clang/test/OpenMP/parallel_firstprivate_codegen.cpp
index 706a9fb4b2..eb71100bd4 100644
--- a/clang/test/OpenMP/parallel_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/parallel_firstprivate_codegen.cpp
@@ -1,31 +1,31 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _
-// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -fopenmp -x c++ -triple i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -Wno-vla -fopenmp -x c++ -triple i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -x c++ -std=c++11 -triple i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -x c++ -triple i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -fopenmp -x c++ -std=c++11 -DLAMBDA -triple i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
-// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -fopenmp -x c++ -fblocks -DBLOCKS -triple i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK4
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -x c++ -triple i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DLAMBDA -triple i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -Wno-vla -fopenmp -x c++ -fblocks -DBLOCKS -triple i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK4
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -fopenmp-simd -x c++ -triple i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -x c++ -std=c++11 -triple i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -x c++ -triple i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -x c++ -triple i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -Wno-vla -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -fopenmp -x c++ -triple x86_64-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -Wno-vla -fopenmp -x c++ -triple x86_64-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -x c++ -std=c++11 -triple x86_64-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -x c++ -triple x86_64-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
-// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK12
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -x c++ -triple x86_64-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -Wno-vla -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK12
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -fopenmp-simd -x c++ -triple x86_64-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -Wno-vla -fopenmp-simd -x c++ -triple x86_64-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -x c++ -std=c++11 -triple x86_64-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -x c++ -triple x86_64-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -x c++ -triple x86_64-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -Wno-vla -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK17
+// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK17
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // expected-no-diagnostics
 #ifndef ARRAY
 #ifndef HEADER
diff --git a/clang/test/OpenMP/parallel_for_codegen.cpp b/clang/test/OpenMP/parallel_for_codegen.cpp
index 3c21be5f63..dcd4ccfa5d 100644
--- a/clang/test/OpenMP/parallel_for_codegen.cpp
+++ b/clang/test/OpenMP/parallel_for_codegen.cpp
@@ -1,29 +1,29 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s --check-prefix=CHECK2
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s --check-prefix=CHECK2
 
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
 
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK2
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK2
 
-// RUN: %clang_cc1 -verify -triple x86_64-apple-darwin10 -fopenmp -fexceptions -fcxx-exceptions -debug-info-kind=line-tables-only -gno-column-info -x c++ -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK5
-// RUN: %clang_cc1 -verify -triple x86_64-apple-darwin10 -fopenmp -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK6
+// RUN: %clang_cc1 -verify -Wno-vla -triple x86_64-apple-darwin10 -fopenmp -fexceptions -fcxx-exceptions -debug-info-kind=line-tables-only -gno-column-info -x c++ -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK5
+// RUN: %clang_cc1 -verify -Wno-vla -triple x86_64-apple-darwin10 -fopenmp -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK6
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -verify -triple x86_64-apple-darwin10 -fopenmp-simd -fexceptions -fcxx-exceptions -debug-info-kind=line-tables-only -x c++ -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -verify -triple x86_64-apple-darwin10 -fopenmp-simd -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -triple x86_64-apple-darwin10 -fopenmp-simd -fexceptions -fcxx-exceptions -debug-info-kind=line-tables-only -x c++ -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -triple x86_64-apple-darwin10 -fopenmp-simd -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // expected-no-diagnostics
 
-// RUN: %clang_cc1 -verify -fopenmp -DOMP5 -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -DOMP5 -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
-// RUN: %clang_cc1 -verify -fopenmp-simd -DOMP5 -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -DOMP5 -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 #ifndef HEADER
 #define HEADER
 
diff --git a/clang/test/OpenMP/parallel_masked_ast_print.cpp b/clang/test/OpenMP/parallel_masked_ast_print.cpp
index ce1552ff5b..c56c7aa14c 100644
--- a/clang/test/OpenMP/parallel_masked_ast_print.cpp
+++ b/clang/test/OpenMP/parallel_masked_ast_print.cpp
@@ -1,10 +1,10 @@
-// RUN: %clang_cc1 -verify -fopenmp -ast-print %s | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -ast-print %s | FileCheck %s
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -std=c++11 -include-pch %t -fsyntax-only -verify -Wno-vla %s -ast-print | FileCheck %s
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -ast-print %s | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -ast-print %s | FileCheck %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck %s
+// RUN: %clang_cc1 -fopenmp-simd -std=c++11 -include-pch %t -fsyntax-only -verify -Wno-vla %s -ast-print | FileCheck %s
 // expected-no-diagnostics
 
 #ifndef HEADER
diff --git a/clang/test/OpenMP/parallel_master_ast_print.cpp b/clang/test/OpenMP/parallel_master_ast_print.cpp
index c6cba2d2b5..c2d3a6a25b 100644
--- a/clang/test/OpenMP/parallel_master_ast_print.cpp
+++ b/clang/test/OpenMP/parallel_master_ast_print.cpp
@@ -1,10 +1,10 @@
-// RUN: %clang_cc1 -verify -fopenmp -ast-print %s | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -ast-print %s | FileCheck %s
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -std=c++11 -include-pch %t -fsyntax-only -verify -Wno-vla %s -ast-print | FileCheck %s
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -ast-print %s | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -ast-print %s | FileCheck %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck %s
+// RUN: %clang_cc1 -fopenmp-simd -std=c++11 -include-pch %t -fsyntax-only -verify -Wno-vla %s -ast-print | FileCheck %s
 // expected-no-diagnostics
 
 #ifndef HEADER
diff --git a/clang/test/OpenMP/parallel_master_taskloop_firstprivate_codegen.cpp b/clang/test/OpenMP/parallel_master_taskloop_firstprivate_codegen.cpp
index 75b99b1a00..5e3bcae3ac 100644
--- a/clang/test/OpenMP/parallel_master_taskloop_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/parallel_master_taskloop_firstprivate_codegen.cpp
@@ -1,17 +1,17 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LAMBDA %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=BLOCKS %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=ARRAY %s
+// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LAMBDA %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=BLOCKS %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=ARRAY %s
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY2 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY3 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY4 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY3 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY4 %s
 // expected-no-diagnostics
 
 #ifndef ARRAY
diff --git a/clang/test/OpenMP/parallel_master_taskloop_lastprivate_codegen.cpp b/clang/test/OpenMP/parallel_master_taskloop_lastprivate_codegen.cpp
index b6c140a331..7dc775b5ac 100644
--- a/clang/test/OpenMP/parallel_master_taskloop_lastprivate_codegen.cpp
+++ b/clang/test/OpenMP/parallel_master_taskloop_lastprivate_codegen.cpp
@@ -1,19 +1,19 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK4
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK5
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DLOOP -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK6
+// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK4
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK5
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DLOOP -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK6
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DLOOP -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DLOOP -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // expected-no-diagnostics
 
 #if !defined(ARRAY) && !defined(LOOP)
diff --git a/clang/test/OpenMP/parallel_master_taskloop_private_codegen.cpp b/clang/test/OpenMP/parallel_master_taskloop_private_codegen.cpp
index 6d5e7c1219..9ecc4747df 100644
--- a/clang/test/OpenMP/parallel_master_taskloop_private_codegen.cpp
+++ b/clang/test/OpenMP/parallel_master_taskloop_private_codegen.cpp
@@ -1,16 +1,16 @@
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LAMBDA %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=BLOCKS %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=ARRAY %s
+// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LAMBDA %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=BLOCKS %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=ARRAY %s
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 // expected-no-diagnostics
 
diff --git a/clang/test/OpenMP/parallel_master_taskloop_reduction_codegen.cpp b/clang/test/OpenMP/parallel_master_taskloop_reduction_codegen.cpp
index ea2e82c5e6..1c9271c322 100644
--- a/clang/test/OpenMP/parallel_master_taskloop_reduction_codegen.cpp
+++ b/clang/test/OpenMP/parallel_master_taskloop_reduction_codegen.cpp
@@ -1,6 +1,6 @@
-// RUN: %clang_cc1 -fopenmp -x c++ %s -verify -debug-info-kind=limited -emit-llvm -o - -triple powerpc64le-unknown-linux-gnu -std=c++98 -fnoopenmp-use-tls | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -x c++ %s -verify -Wno-vla -debug-info-kind=limited -emit-llvm -o - -triple powerpc64le-unknown-linux-gnu -std=c++98 -fnoopenmp-use-tls | FileCheck %s
 
-// RUN: %clang_cc1 -fopenmp-simd -x c++ %s -verify -debug-info-kind=limited -emit-llvm -o - -triple powerpc64le-unknown-linux-gnu -std=c++98 -fnoopenmp-use-tls | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ %s -verify -Wno-vla -debug-info-kind=limited -emit-llvm -o - -triple powerpc64le-unknown-linux-gnu -std=c++98 -fnoopenmp-use-tls | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 // expected-no-diagnostics
 
diff --git a/clang/test/OpenMP/parallel_master_taskloop_simd_firstprivate_codegen.cpp b/clang/test/OpenMP/parallel_master_taskloop_simd_firstprivate_codegen.cpp
index b938758b05..e5ce73f6ee 100644
--- a/clang/test/OpenMP/parallel_master_taskloop_simd_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/parallel_master_taskloop_simd_firstprivate_codegen.cpp
@@ -1,17 +1,17 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LAMBDA %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=BLOCKS %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=ARRAY %s
+// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LAMBDA %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=BLOCKS %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=ARRAY %s
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY2 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY3 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY4 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY3 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY4 %s
 // expected-no-diagnostics
 
 #ifndef ARRAY
diff --git a/clang/test/OpenMP/parallel_master_taskloop_simd_lastprivate_codegen.cpp b/clang/test/OpenMP/parallel_master_taskloop_simd_lastprivate_codegen.cpp
index 1339272bb3..cd5ee03eb6 100644
--- a/clang/test/OpenMP/parallel_master_taskloop_simd_lastprivate_codegen.cpp
+++ b/clang/test/OpenMP/parallel_master_taskloop_simd_lastprivate_codegen.cpp
@@ -1,19 +1,19 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK4
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK5
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DLOOP -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK6
+// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK4
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK5
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DLOOP -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK6
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK7
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK7
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK7
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK10
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DLOOP -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK12
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK7
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK10
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DLOOP -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK12
 // expected-no-diagnostics
 
 #if !defined(ARRAY) && !defined(LOOP)
diff --git a/clang/test/OpenMP/parallel_master_taskloop_simd_private_codegen.cpp b/clang/test/OpenMP/parallel_master_taskloop_simd_private_codegen.cpp
index d58c01d756..e586b13bf9 100644
--- a/clang/test/OpenMP/parallel_master_taskloop_simd_private_codegen.cpp
+++ b/clang/test/OpenMP/parallel_master_taskloop_simd_private_codegen.cpp
@@ -1,16 +1,16 @@
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LAMBDA %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=BLOCKS %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=ARRAY %s
+// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LAMBDA %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=BLOCKS %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=ARRAY %s
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 // expected-no-diagnostics
 
diff --git a/clang/test/OpenMP/parallel_master_taskloop_simd_reduction_codegen.cpp b/clang/test/OpenMP/parallel_master_taskloop_simd_reduction_codegen.cpp
index 7a0ea6bbbb..e193540158 100644
--- a/clang/test/OpenMP/parallel_master_taskloop_simd_reduction_codegen.cpp
+++ b/clang/test/OpenMP/parallel_master_taskloop_simd_reduction_codegen.cpp
@@ -1,6 +1,6 @@
-// RUN: %clang_cc1 -fopenmp -x c++ %s -verify -debug-info-kind=limited -emit-llvm -o - -triple powerpc64le-unknown-linux-gnu -std=c++98 -fnoopenmp-use-tls | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -x c++ %s -verify -Wno-vla -debug-info-kind=limited -emit-llvm -o - -triple powerpc64le-unknown-linux-gnu -std=c++98 -fnoopenmp-use-tls | FileCheck %s
 
-// RUN: %clang_cc1 -fopenmp-simd -x c++ %s -verify -debug-info-kind=limited -emit-llvm -o - -triple powerpc64le-unknown-linux-gnu -std=c++98 -fnoopenmp-use-tls | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ %s -verify -Wno-vla -debug-info-kind=limited -emit-llvm -o - -triple powerpc64le-unknown-linux-gnu -std=c++98 -fnoopenmp-use-tls | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 // expected-no-diagnostics
 
diff --git a/clang/test/OpenMP/single_codegen.cpp b/clang/test/OpenMP/single_codegen.cpp
index f32bc7e31c..0bc2ba2bcc 100644
--- a/clang/test/OpenMP/single_codegen.cpp
+++ b/clang/test/OpenMP/single_codegen.cpp
@@ -1,21 +1,21 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _
-// RUN: %clang_cc1 -verify -fopenmp -fnoopenmp-use-tls -x c++ -std=c++11 -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -fnoopenmp-use-tls -x c++ -std=c++11 -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s --check-prefix=CHECK2
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fnoopenmp-use-tls -x c++ -std=c++11 -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -fnoopenmp-use-tls -x c++ -std=c++11 -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s --check-prefix=CHECK2
 
 // RUN: %clang_cc1 -fopenmp -fnoopenmp-use-tls -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fnoopenmp-use-tls -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -fopenmp -fnoopenmp-use-tls -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
 
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fnoopenmp-use-tls -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fnoopenmp-use-tls -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK4
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fnoopenmp-use-tls -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK4
 
-// RUN: %clang_cc1 -verify -triple x86_64-apple-darwin10 -std=c++11 -fopenmp -fnoopenmp-use-tls -fexceptions -fcxx-exceptions -debug-info-kind=line-tables-only -x c++ -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK5
-// RUN: %clang_cc1 -verify -fopenmp -fnoopenmp-use-tls -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK6
+// RUN: %clang_cc1 -verify -Wno-vla -triple x86_64-apple-darwin10 -std=c++11 -fopenmp -fnoopenmp-use-tls -fexceptions -fcxx-exceptions -debug-info-kind=line-tables-only -x c++ -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK5
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fnoopenmp-use-tls -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK6
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -fnoopenmp-use-tls -x c++ -std=c++11 -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fnoopenmp-use-tls -x c++ -std=c++11 -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -fnoopenmp-use-tls -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -fnoopenmp-use-tls -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -verify -triple x86_64-apple-darwin10 -std=c++11 -fopenmp-simd -fnoopenmp-use-tls -fexceptions -fcxx-exceptions -debug-info-kind=line-tables-only -x c++ -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -verify -fopenmp-simd -fnoopenmp-use-tls -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -triple x86_64-apple-darwin10 -std=c++11 -fopenmp-simd -fnoopenmp-use-tls -fexceptions -fcxx-exceptions -debug-info-kind=line-tables-only -x c++ -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fnoopenmp-use-tls -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // expected-no-diagnostics
 #ifndef ARRAY
 #ifndef HEADER
diff --git a/clang/test/OpenMP/target_ast_print.cpp b/clang/test/OpenMP/target_ast_print.cpp
index f580ff3257..c918c25a7f 100644
--- a/clang/test/OpenMP/target_ast_print.cpp
+++ b/clang/test/OpenMP/target_ast_print.cpp
@@ -4,13 +4,13 @@
 #ifndef HEADER
 #define HEADER
 
-// RUN: %clang_cc1 -DOMP45 -verify -fopenmp -fopenmp-version=45 -ast-print %s | FileCheck %s --check-prefix=OMP45
+// RUN: %clang_cc1 -DOMP45 -verify -Wno-vla -fopenmp -fopenmp-version=45 -ast-print %s | FileCheck %s --check-prefix=OMP45
 // RUN: %clang_cc1 -DOMP45 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -emit-pch -o %t %s
-// RUN: %clang_cc1 -DOMP45 -fopenmp -fopenmp-version=45 -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck %s --check-prefix=OMP45
+// RUN: %clang_cc1 -DOMP45 -fopenmp -fopenmp-version=45 -std=c++11 -include-pch %t -fsyntax-only -verify -Wno-vla %s -ast-print | FileCheck %s --check-prefix=OMP45
 
-// RUN: %clang_cc1 -DOMP45 -verify -fopenmp-simd -fopenmp-version=45 -ast-print %s | FileCheck %s --check-prefix=OMP45
+// RUN: %clang_cc1 -DOMP45 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -ast-print %s | FileCheck %s --check-prefix=OMP45
 // RUN: %clang_cc1 -DOMP45 -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -emit-pch -o %t %s
-// RUN: %clang_cc1 -DOMP45 -fopenmp-simd -fopenmp-version=45 -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck %s --check-prefix=OMP45
+// RUN: %clang_cc1 -DOMP45 -fopenmp-simd -fopenmp-version=45 -std=c++11 -include-pch %t -fsyntax-only -verify -Wno-vla %s -ast-print | FileCheck %s --check-prefix=OMP45
 #ifdef OMP45
 
 void foo() {}
@@ -335,21 +335,21 @@ int main (int argc, char **argv) {
 #ifdef OMP5
 
 ///==========================================================================///
-// RUN: %clang_cc1 -DOMP5 -verify -fopenmp -fopenmp-version=50 -ast-print %s | FileCheck %s --check-prefix OMP5
+// RUN: %clang_cc1 -DOMP5 -verify -Wno-vla -fopenmp -fopenmp-version=50 -ast-print %s | FileCheck %s --check-prefix OMP5
 // RUN: %clang_cc1 -DOMP5 -fopenmp -fopenmp-version=50 -x c++ -std=c++11 -emit-pch -o %t %s
-// RUN: %clang_cc1 -DOMP5 -fopenmp -fopenmp-version=50 -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck %s --check-prefix OMP5
+// RUN: %clang_cc1 -DOMP5 -fopenmp -fopenmp-version=50 -std=c++11 -include-pch %t -fsyntax-only -verify -Wno-vla %s -ast-print | FileCheck %s --check-prefix OMP5
 
-// RUN: %clang_cc1 -DOMP5 -verify -fopenmp-simd -fopenmp-version=50 -ast-print %s | FileCheck %s --check-prefix OMP5
+// RUN: %clang_cc1 -DOMP5 -verify -Wno-vla -fopenmp-simd -fopenmp-version=50 -ast-print %s | FileCheck %s --check-prefix OMP5
 // RUN: %clang_cc1 -DOMP5 -fopenmp-simd -fopenmp-version=50 -x c++ -std=c++11 -emit-pch -o %t %s
-// RUN: %clang_cc1 -DOMP5 -fopenmp-simd -fopenmp-version=50 -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck %s --check-prefix OMP5
+// RUN: %clang_cc1 -DOMP5 -fopenmp-simd -fopenmp-version=50 -std=c++11 -include-pch %t -fsyntax-only -verify -Wno-vla %s -ast-print | FileCheck %s --check-prefix OMP5
 
-// RUN: %clang_cc1 -DOMP5 -verify -fopenmp -fopenmp-version=99 -DOMP99 -ast-print %s | FileCheck %s --check-prefixes=OMP5,REV
+// RUN: %clang_cc1 -DOMP5 -verify -Wno-vla -fopenmp -fopenmp-version=99 -DOMP99 -ast-print %s | FileCheck %s --check-prefixes=OMP5,REV
 // RUN: %clang_cc1 -DOMP5 -fopenmp -fopenmp-version=99 -DOMP99 -x c++ -std=c++11 -emit-pch -o %t %s
-// RUN: %clang_cc1 -DOMP5 -fopenmp -fopenmp-version=99 -DOMP99 -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck %s --check-prefixes=OMP5,REV
+// RUN: %clang_cc1 -DOMP5 -fopenmp -fopenmp-version=99 -DOMP99 -std=c++11 -include-pch %t -fsyntax-only -verify -Wno-vla %s -ast-print | FileCheck %s --check-prefixes=OMP5,REV
 
-// RUN: %clang_cc1 -DOMP5 -verify -fopenmp-simd -fopenmp-version=99 -DOMP99 -ast-print %s | FileCheck %s --check-prefixes=OMP5,REV
+// RUN: %clang_cc1 -DOMP5 -verify -Wno-vla -fopenmp-simd -fopenmp-version=99 -DOMP99 -ast-print %s | FileCheck %s --check-prefixes=OMP5,REV
 // RUN: %clang_cc1 -DOMP5 -fopenmp-simd -fopenmp-version=99 -DOMP99 -x c++ -std=c++11 -emit-pch -o %t %s
-// RUN: %clang_cc1 -DOMP5 -fopenmp-simd -fopenmp-version=99 -DOMP99 -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck %s --check-prefixes=OMP5,REV
+// RUN: %clang_cc1 -DOMP5 -fopenmp-simd -fopenmp-version=99 -DOMP99 -std=c++11 -include-pch %t -fsyntax-only -verify -Wno-vla %s -ast-print | FileCheck %s --check-prefixes=OMP5,REV
 
 #ifdef OMP99
 #pragma omp requires reverse_offload
@@ -1090,13 +1090,13 @@ int main (int argc, char **argv) {
 #ifdef OMP51
 
 ///==========================================================================///
-// RUN: %clang_cc1 -DOMP51 -verify -fopenmp -ast-print %s | FileCheck %s --check-prefix OMP51
+// RUN: %clang_cc1 -DOMP51 -verify -Wno-vla -fopenmp -ast-print %s | FileCheck %s --check-prefix OMP51
 // RUN: %clang_cc1 -DOMP51 -fopenmp -x c++ -std=c++11 -emit-pch -o %t %s
-// RUN: %clang_cc1 -DOMP51 -fopenmp -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck %s --check-prefix OMP51
+// RUN: %clang_cc1 -DOMP51 -fopenmp -std=c++11 -include-pch %t -fsyntax-only -verify -Wno-vla %s -ast-print | FileCheck %s --check-prefix OMP51
 
-// RUN: %clang_cc1 -DOMP51 -verify -fopenmp-simd -ast-print %s | FileCheck %s --check-prefix OMP51
+// RUN: %clang_cc1 -DOMP51 -verify -Wno-vla -fopenmp-simd -ast-print %s | FileCheck %s --check-prefix OMP51
 // RUN: %clang_cc1 -DOMP51 -fopenmp-simd -x c++ -std=c++11 -emit-pch -o %t %s
-// RUN: %clang_cc1 -DOMP51 -fopenmp-simd -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck %s --check-prefix OMP51
+// RUN: %clang_cc1 -DOMP51 -fopenmp-simd -std=c++11 -include-pch %t -fsyntax-only -verify -Wno-vla %s -ast-print | FileCheck %s --check-prefix OMP51
 
 void foo() {}
 
@@ -1146,13 +1146,13 @@ int main (int argc, char **argv) {
 #ifdef OMP52
 
 ///==========================================================================///
-// RUN: %clang_cc1 -DOMP52 -verify -fopenmp -fopenmp-version=52 -ast-print %s | FileCheck %s --check-prefix OMP52
+// RUN: %clang_cc1 -DOMP52 -verify -Wno-vla -fopenmp -fopenmp-version=52 -ast-print %s | FileCheck %s --check-prefix OMP52
 // RUN: %clang_cc1 -DOMP52 -fopenmp -fopenmp-version=52 -x c++ -std=c++11 -emit-pch -o %t %s
-// RUN: %clang_cc1 -DOMP52 -fopenmp -fopenmp-version=52 -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck %s --check-prefix OMP52
+// RUN: %clang_cc1 -DOMP52 -fopenmp -fopenmp-version=52 -std=c++11 -include-pch %t -fsyntax-only -verify -Wno-vla %s -ast-print | FileCheck %s --check-prefix OMP52
 
-// RUN: %clang_cc1 -DOMP52 -verify -fopenmp-simd -fopenmp-version=52 -ast-print %s | FileCheck %s --check-prefix OMP52
+// RUN: %clang_cc1 -DOMP52 -verify -Wno-vla -fopenmp-simd -fopenmp-version=52 -ast-print %s | FileCheck %s --check-prefix OMP52
 // RUN: %clang_cc1 -DOMP52 -fopenmp-simd -fopenmp-version=52 -x c++ -std=c++11 -emit-pch -o %t %s
-// RUN: %clang_cc1 -DOMP52 -fopenmp-simd -fopenmp-version=52 -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck %s --check-prefix OMP52
+// RUN: %clang_cc1 -DOMP52 -fopenmp-simd -fopenmp-version=52 -std=c++11 -include-pch %t -fsyntax-only -verify -Wno-vla %s -ast-print | FileCheck %s --check-prefix OMP52
 
 void foo() {}
 
@@ -1199,13 +1199,13 @@ foo();
 
 #ifdef OMPX
 
-// RUN: %clang_cc1 -DOMPX -verify -fopenmp -fopenmp-extensions -ast-print %s | FileCheck %s --check-prefix=OMPX
+// RUN: %clang_cc1 -DOMPX -verify -Wno-vla -fopenmp -fopenmp-extensions -ast-print %s | FileCheck %s --check-prefix=OMPX
 // RUN: %clang_cc1 -DOMPX -fopenmp -fopenmp-extensions -x c++ -std=c++11 -emit-pch -o %t %s
-// RUN: %clang_cc1 -DOMPX -fopenmp -fopenmp-extensions -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck %s --check-prefix=OMPX
+// RUN: %clang_cc1 -DOMPX -fopenmp -fopenmp-extensions -std=c++11 -include-pch %t -fsyntax-only -verify -Wno-vla %s -ast-print | FileCheck %s --check-prefix=OMPX
 
-// RUN: %clang_cc1 -DOMPX -verify -fopenmp-simd -fopenmp-extensions -ast-print %s | FileCheck %s --check-prefix=OMPX
+// RUN: %clang_cc1 -DOMPX -verify -Wno-vla -fopenmp-simd -fopenmp-extensions -ast-print %s | FileCheck %s --check-prefix=OMPX
 // RUN: %clang_cc1 -DOMPX -fopenmp-simd -fopenmp-extensions -x c++ -std=c++11 -emit-pch -o %t %s
-// RUN: %clang_cc1 -DOMPX -fopenmp-simd -fopenmp-extensions -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck %s --check-prefix=OMPX
+// RUN: %clang_cc1 -DOMPX -fopenmp-simd -fopenmp-extensions -std=c++11 -include-pch %t -fsyntax-only -verify -Wno-vla %s -ast-print | FileCheck %s --check-prefix=OMPX
 
 void foo() {}
 
diff --git a/clang/test/OpenMP/target_codegen.cpp b/clang/test/OpenMP/target_codegen.cpp
index bd3d7eb853..fa96d5f009 100644
--- a/clang/test/OpenMP/target_codegen.cpp
+++ b/clang/test/OpenMP/target_codegen.cpp
@@ -1,60 +1,60 @@
 // Test host codegen.
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix OMP45
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix OMP45
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix OMP45
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix OMP45
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix OMP45
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix OMP45
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix OMP45
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix OMP45
 
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix OMP50
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix OMP50
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix OMP50
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix OMP50
+// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix OMP50
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix OMP50
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix OMP50
+// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix OMP50
 
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=51 -D_DOMP51 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix OMP51
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=51 -D_DOMP51 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix OMP51
 // RUN: %clang_cc1 -fopenmp -x c++ -fopenmp-version=51 -D_DOMP51 -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -fopenmp-version=51 -D_DOMP51 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix OMP51
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=51 -D_DOMP51 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix OMP51
+// RUN: %clang_cc1 -fopenmp -x c++ -fopenmp-version=51 -D_DOMP51 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix OMP51
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=51 -D_DOMP51 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix OMP51
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -D_DOMP51 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -D_DOMP51 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix OMP51
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -D_DOMP51 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix OMP51
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 
 // Test target codegen - host bc file has to be created first.
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
 
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // SIMD-ONLY1-NOT: {{__kmpc|__tgt}}
 
 // expected-no-diagnostics
diff --git a/clang/test/OpenMP/target_constant_device_codegen.cpp b/clang/test/OpenMP/target_constant_device_codegen.cpp
index b61681614e..b5923325e7 100644
--- a/clang/test/OpenMP/target_constant_device_codegen.cpp
+++ b/clang/test/OpenMP/target_constant_device_codegen.cpp
@@ -1,7 +1,7 @@
 // Test host codegen.
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
+// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
 
 // expected-no-diagnostics
 #ifndef HEADER
diff --git a/clang/test/OpenMP/target_data_codegen.cpp b/clang/test/OpenMP/target_data_codegen.cpp
index 1dd2173922..4f3c94d175 100644
--- a/clang/test/OpenMP/target_data_codegen.cpp
+++ b/clang/test/OpenMP/target_data_codegen.cpp
@@ -3,19 +3,19 @@
 #define HEADER
 
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-64
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-64
 // RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-64
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-64
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-32
 // RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-32
 
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 #ifdef CK1
 
@@ -218,19 +218,19 @@ void foo(int arg) {
 }
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK1A -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK1A --check-prefix CK1A-64
+// RUN: %clang_cc1 -DCK1A -verify -Wno-vla  -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK1A --check-prefix CK1A-64
 // RUN: %clang_cc1 -DCK1A -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1A --check-prefix CK1A-64
-// RUN: %clang_cc1 -DCK1A -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK1A --check-prefix CK1A-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1A --check-prefix CK1A-64
+// RUN: %clang_cc1 -DCK1A -verify -Wno-vla  -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK1A --check-prefix CK1A-32
 // RUN: %clang_cc1 -DCK1A -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1A --check-prefix CK1A-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1A --check-prefix CK1A-32
 
-// RUN: %clang_cc1 -DCK1A -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK1A -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK1A -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -DCK1A -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK1A -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK1A -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 #ifdef CK1A
 
@@ -312,19 +312,19 @@ void foo(int arg) {
 }
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-64
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla  -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-64
 // RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK2 --check-prefix CK2-64
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK2 --check-prefix CK2-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck %s  --check-prefix CK2 --check-prefix CK2-64
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla  -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK2 --check-prefix CK2-32
 // RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK2 --check-prefix CK2-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck %s  --check-prefix CK2 --check-prefix CK2-32
 
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // SIMD-ONLY1-NOT: {{__kmpc|__tgt}}
 #ifdef CK2
 
@@ -396,19 +396,19 @@ int bar(int arg){
 // CK2: [[IFEND]]
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK3 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK3 --check-prefix CK3-64
+// RUN: %clang_cc1 -DCK3 -verify -Wno-vla  -fopenmp -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK3 --check-prefix CK3-64
 // RUN: %clang_cc1 -DCK3 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK3 --check-prefix CK3-64
-// RUN: %clang_cc1 -DCK3 -verify -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK3 --check-prefix CK3-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck %s  --check-prefix CK3 --check-prefix CK3-64
+// RUN: %clang_cc1 -DCK3 -verify -Wno-vla  -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK3 --check-prefix CK3-32
 // RUN: %clang_cc1 -DCK3 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK3 --check-prefix CK3-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck %s  --check-prefix CK3 --check-prefix CK3-32
 
-// RUN: %clang_cc1 -DCK3 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// RUN: %clang_cc1 -DCK3 -verify -Wno-vla  -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY2 %s
 // RUN: %clang_cc1 -DCK3 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY2 %s
-// RUN: %clang_cc1 -DCK3 -verify -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// RUN: %clang_cc1 -DCK3 -verify -Wno-vla  -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY2 %s
 // RUN: %clang_cc1 -DCK3 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY2 %s
 // SIMD-ONLY2-NOT: {{__kmpc|__tgt}}
 #ifdef CK3
 
@@ -423,19 +423,19 @@ void no_target_devices(int arg) {
 }
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK4 --check-prefix CK4-64
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla  -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK4 --check-prefix CK4-64
 // RUN: %clang_cc1 -DCK4 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK4 --check-prefix CK4-64
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK4 --check-prefix CK4-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck %s  --check-prefix CK4 --check-prefix CK4-64
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla  -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK4 --check-prefix CK4-32
 // RUN: %clang_cc1 -DCK4 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK4 --check-prefix CK4-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck %s  --check-prefix CK4 --check-prefix CK4-32
 
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -DCK4 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -DCK4 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // SIMD-ONLY1-NOT: {{__kmpc|__tgt}}
 #ifdef CK4
 
@@ -507,19 +507,19 @@ int bar(int arg){
 // CK4: [[IFEND]]
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK5 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK5 --check-prefix CK5-64
+// RUN: %clang_cc1 -DCK5 -verify -Wno-vla  -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK5 --check-prefix CK5-64
 // RUN: %clang_cc1 -DCK5 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK5 --check-prefix CK5-64
-// RUN: %clang_cc1 -DCK5 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK5 --check-prefix CK5-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck %s  --check-prefix CK5 --check-prefix CK5-64
+// RUN: %clang_cc1 -DCK5 -verify -Wno-vla  -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK5 --check-prefix CK5-32
 // RUN: %clang_cc1 -DCK5 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK5 --check-prefix CK5-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck %s  --check-prefix CK5 --check-prefix CK5-32
 
-// RUN: %clang_cc1 -DCK5 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// RUN: %clang_cc1 -DCK5 -verify -Wno-vla  -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY2 %s
 // RUN: %clang_cc1 -DCK5 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY2 %s
-// RUN: %clang_cc1 -DCK5 -verify -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// RUN: %clang_cc1 -DCK5 -verify -Wno-vla  -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY2 %s
 // RUN: %clang_cc1 -DCK5 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY2 %s
 // SIMD-ONLY2-NOT: {{__kmpc|__tgt}}#ifdef CK5
 #ifdef CK5
 struct S1 {
@@ -541,19 +541,19 @@ void test_close_modifier(int arg) {
 }
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK6 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK6 --check-prefix CK6-64
+// RUN: %clang_cc1 -DCK6 -verify -Wno-vla  -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK6 --check-prefix CK6-64
 // RUN: %clang_cc1 -DCK6 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK6 --check-prefix CK6-64
-// RUN: %clang_cc1 -DCK6 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK6 --check-prefix CK6-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck %s  --check-prefix CK6 --check-prefix CK6-64
+// RUN: %clang_cc1 -DCK6 -verify -Wno-vla  -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK6 --check-prefix CK6-32
 // RUN: %clang_cc1 -DCK6 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK6 --check-prefix CK6-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck %s  --check-prefix CK6 --check-prefix CK6-32
 
-// RUN: %clang_cc1 -DCK6 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// RUN: %clang_cc1 -DCK6 -verify -Wno-vla  -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY2 %s
 // RUN: %clang_cc1 -DCK6 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY2 %s
-// RUN: %clang_cc1 -DCK6 -verify -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// RUN: %clang_cc1 -DCK6 -verify -Wno-vla  -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY2 %s
 // RUN: %clang_cc1 -DCK6 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY2 %s
 // SIMD-ONLY2-NOT: {{__kmpc|__tgt}}
 #ifdef CK6
 void test_close_modifier(int arg) {
@@ -564,13 +564,13 @@ void test_close_modifier(int arg) {
 }
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK7 -verify -fopenmp -fopenmp-targets=x86_64 -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK7 --check-prefix CK7-64
+// RUN: %clang_cc1 -DCK7 -verify -Wno-vla  -fopenmp -fopenmp-targets=x86_64 -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK7 --check-prefix CK7-64
 // RUN: %clang_cc1 -DCK7 -fopenmp -fopenmp-targets=x86_64 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=x86_64 -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK7 --check-prefix CK7-64
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=x86_64 -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck %s  --check-prefix CK7 --check-prefix CK7-64
 
-// RUN: %clang_cc1 -DCK7 -verify -fopenmp-simd -fopenmp-targets=x86_64 -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY7 %s
+// RUN: %clang_cc1 -DCK7 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=x86_64 -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY7 %s
 // RUN: %clang_cc1 -DCK7 -fopenmp-simd -fopenmp-targets=x86_64 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=x86_64 -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY7 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=x86_64 -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY7 %s
 // SIMD-ONLY7-NOT: {{__kmpc|__tgt}}
 #ifdef CK7
 // CK7: private unnamed_addr constant [2 x i64] [i64 64, i64 64]
@@ -596,19 +596,19 @@ void test_device_ptr_addr(int arg) {
 }
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK8 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK8 --check-prefix CK8-64
+// RUN: %clang_cc1 -DCK8 -verify -Wno-vla  -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK8 --check-prefix CK8-64
 // RUN: %clang_cc1 -DCK8 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK8 --check-prefix CK8-64
-// RUN: %clang_cc1 -DCK8 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK8 --check-prefix CK8-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck %s  --check-prefix CK8 --check-prefix CK8-64
+// RUN: %clang_cc1 -DCK8 -verify -Wno-vla  -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK8 --check-prefix CK8-32
 // RUN: %clang_cc1 -DCK8 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK8 --check-prefix CK8-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck %s  --check-prefix CK8 --check-prefix CK8-32
 
-// RUN: %clang_cc1 -DCK8 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// RUN: %clang_cc1 -DCK8 -verify -Wno-vla  -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY2 %s
 // RUN: %clang_cc1 -DCK8 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY2 %s
-// RUN: %clang_cc1 -DCK8 -verify -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// RUN: %clang_cc1 -DCK8 -verify -Wno-vla  -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY2 %s
 // RUN: %clang_cc1 -DCK8 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY2 %s
 // SIMD-ONLY2-NOT: {{__kmpc|__tgt}}#ifdef CK8
 #ifdef CK8
 struct S1 {
@@ -667,19 +667,19 @@ void test_present_modifier(int arg) {
 }
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK9 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK9 --check-prefix CK9-64
+// RUN: %clang_cc1 -DCK9 -verify -Wno-vla  -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK9 --check-prefix CK9-64
 // RUN: %clang_cc1 -DCK9 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK9 --check-prefix CK9-64
-// RUN: %clang_cc1 -DCK9 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK9 --check-prefix CK9-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck %s  --check-prefix CK9 --check-prefix CK9-64
+// RUN: %clang_cc1 -DCK9 -verify -Wno-vla  -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK9 --check-prefix CK9-32
 // RUN: %clang_cc1 -DCK9 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK9 --check-prefix CK9-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck %s  --check-prefix CK9 --check-prefix CK9-32
 
-// RUN: %clang_cc1 -DCK9 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// RUN: %clang_cc1 -DCK9 -verify -Wno-vla  -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY2 %s
 // RUN: %clang_cc1 -DCK9 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY2 %s
-// RUN: %clang_cc1 -DCK9 -verify -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// RUN: %clang_cc1 -DCK9 -verify -Wno-vla  -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY2 %s
 // RUN: %clang_cc1 -DCK9 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY2 %s
 // SIMD-ONLY2-NOT: {{__kmpc|__tgt}}
 #ifdef CK9
 void test_present_modifier(int arg) {
diff --git a/clang/test/OpenMP/target_data_use_device_addr_codegen.cpp b/clang/test/OpenMP/target_data_use_device_addr_codegen.cpp
index a388b04a40..7e70cdf74a 100644
--- a/clang/test/OpenMP/target_data_use_device_addr_codegen.cpp
+++ b/clang/test/OpenMP/target_data_use_device_addr_codegen.cpp
@@ -1,10 +1,10 @@
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s
 // RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s
 
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 // expected-no-diagnostics
 
diff --git a/clang/test/OpenMP/target_defaultmap_codegen_01.cpp b/clang/test/OpenMP/target_defaultmap_codegen_01.cpp
index bff76b3884..b4a01b685e 100644
--- a/clang/test/OpenMP/target_defaultmap_codegen_01.cpp
+++ b/clang/test/OpenMP/target_defaultmap_codegen_01.cpp
@@ -5,19 +5,19 @@
 #ifdef CK1
 
 ///==========================================================================///
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK1 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK1
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK1 -verify -Wno-vla -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK1
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK1
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK1 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK1
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK1
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK1 -verify -Wno-vla -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK1
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK1
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK1
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK1 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY10 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK1 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY10 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY10 %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK1 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY10 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY10 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK1 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY10 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY10 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY10 %s
 // SIMD-ONLY10-NOT: {{__kmpc|__tgt}}
 
 // CK1-LABEL: @.__omp_offloading_{{.*}}implicit_maps_double_complex{{.*}}_l{{[0-9]+}}.region_id = weak{{.*}} constant i8 0
@@ -57,19 +57,19 @@ void implicit_maps_double_complex (int a){
 // CK1: {{.+}} = getelementptr inbounds { double, double }, ptr [[REF]], i32 0, i32 0
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK2 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK2
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK2 -verify -Wno-vla  -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK2
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK2 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK2
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK2 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK2
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK2
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK2 -verify -Wno-vla  -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK2
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK2 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK2
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK2
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK2 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY10 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK2 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY10 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK2 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY10 %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK2 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY10 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY10 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK2 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY10 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK2 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY10 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY10 %s
 // SIMD-ONLY10-NOT: {{__kmpc|__tgt}}
 #ifdef CK2
 
@@ -110,19 +110,19 @@ void implicit_maps_double_complex (int a){
 // CK2: {{.+}} = getelementptr inbounds { double, double }, ptr [[REF]], i32 0, i32 0
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK3 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK3
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK3 -verify -Wno-vla  -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK3
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK3 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK3
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK3 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK3
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK3
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK3 -verify -Wno-vla  -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK3
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK3 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK3
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK3
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK3 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY10 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK3 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY10 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK3 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY10 %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK3 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY10 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY10 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK3 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY10 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK3 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY10 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY10 %s
 // SIMD-ONLY10-NOT: {{__kmpc|__tgt}}
 #ifdef CK3
 
@@ -163,19 +163,19 @@ void implicit_maps_double_complex (int a){
 // CK3: {{.+}} = getelementptr inbounds { double, double }, ptr [[REF]], i32 0, i32 0
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK4 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK4 --check-prefix CK4-64
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK4 -verify -Wno-vla  -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK4 --check-prefix CK4-64
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK4 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK4  --check-prefix CK4-64
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK4 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK4  --check-prefix CK4-32
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK4  --check-prefix CK4-64
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK4 -verify -Wno-vla  -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK4  --check-prefix CK4-32
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK4 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK4  --check-prefix CK4-32
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK4  --check-prefix CK4-32
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK4 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY6 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK4 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY6 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK4 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY6 %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK4 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY6 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY6 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK4 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY6 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK4 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY6 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY6 %s
 // SIMD-ONLY6-NOT: {{__kmpc|__tgt}}
 #ifdef CK4
 
@@ -234,19 +234,19 @@ void implicit_maps_double (int a){
 
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK5 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK5
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK5 -verify -Wno-vla  -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK5
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK5 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK5
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK5 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK5
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK5
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK5 -verify -Wno-vla  -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK5
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK5 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK5
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK5
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK5 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK5 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK5 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK5 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK5 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK5 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
 // SIMD-ONLY8-NOT: {{__kmpc|__tgt}}
 #ifdef CK5
 
@@ -288,19 +288,19 @@ void implicit_maps_array (int a){
 // CK5: {{.+}} = getelementptr inbounds [2 x double], ptr [[REF]], i{{64|32}} 0, i{{64|32}} 0
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK6 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK6
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK6 -verify -Wno-vla  -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK6
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK6 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK6
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK6 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK6
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK6
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK6 -verify -Wno-vla  -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK6
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK6 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK6
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK6
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK6 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK6 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK6 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK6 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK6 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK6 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
 // SIMD-ONLY8-NOT: {{__kmpc|__tgt}}
 #ifdef CK6
 
@@ -341,19 +341,19 @@ void implicit_maps_array (int a){
 // CK6: {{.+}} = getelementptr inbounds [2 x double], ptr [[REF]], i{{64|32}} 0, i{{64|32}} 0
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK7 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK7
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK7 -verify -Wno-vla  -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK7
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK7 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK7
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK7 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK7
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK7
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK7 -verify -Wno-vla  -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK7
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK7 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK7
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK7
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK7 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK7 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK7 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK7 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK7 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK7 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
 // SIMD-ONLY8-NOT: {{__kmpc|__tgt}}
 #ifdef CK7
 
@@ -395,19 +395,19 @@ void implicit_maps_array (int a){
 // CK7: {{.+}} = getelementptr inbounds [2 x double], ptr [[REF]], i{{64|32}} 0, i{{64|32}} 0
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK8 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK8
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK8 -verify -Wno-vla  -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK8
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK8 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK8
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK8 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK8
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK8
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK8 -verify -Wno-vla  -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK8
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK8 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK8
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK8
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK8 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK8 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK8 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK8 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK8 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK8 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
 // SIMD-ONLY8-NOT: {{__kmpc|__tgt}}
 #ifdef CK8
 
@@ -449,19 +449,19 @@ void implicit_maps_array (int a){
 #endif
 
 ///==========================================================================///
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK9 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK9 --check-prefix CK9-64
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK9 -verify -Wno-vla  -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK9 --check-prefix CK9-64
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK9 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK9 --check-prefix CK9-64
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK9 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK9 --check-prefix CK9-32
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK9 --check-prefix CK9-64
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK9 -verify -Wno-vla  -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK9 --check-prefix CK9-32
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK9 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK9 --check-prefix CK9-32
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK9 --check-prefix CK9-32
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK9 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY26 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK9 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY26 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK9 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY26 %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK9 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY26 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY26 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK9 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY26 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK9 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY26 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY26 %s
 // SIMD-ONLY26-NOT: {{__kmpc|__tgt}}
 
 #ifdef CK9
@@ -502,19 +502,19 @@ void zero_size_section_and_private_maps (int ii){
 
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK10 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK10
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK10 -verify -Wno-vla  -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK10
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK10 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK10
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK10 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK10
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK10
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK10 -verify -Wno-vla  -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK10
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK10 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK10
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK10
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK10 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK10 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK10 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK10 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK10 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK10 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
 // SIMD-ONLY8-NOT: {{__kmpc|__tgt}}
 #ifdef CK10
 
@@ -553,19 +553,19 @@ void explicit_maps_single (){
   // CK10: define {{.+}}[[CALL]]
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK11 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK11
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK11 -verify -Wno-vla  -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK11
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK11 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK11
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK11 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK11
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK11
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK11 -verify -Wno-vla  -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK11
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK11 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK11
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK11
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK11 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK11 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK11 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK11 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK11 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK11 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
 // SIMD-ONLY8-NOT: {{__kmpc|__tgt}}
 #ifdef CK11
 
@@ -604,19 +604,19 @@ void explicit_maps_single (){
   // CK11: define {{.+}}[[CALL09]]
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK12 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK12
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK12 -verify -Wno-vla  -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK12
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK12 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK12
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK12 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK12
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK12
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK12 -verify -Wno-vla  -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK12
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK12 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK12
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK12
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK12 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK12 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK12 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK12 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK12 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK12 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
 // SIMD-ONLY8-NOT: {{__kmpc|__tgt}}
 #ifdef CK12
 
@@ -655,19 +655,19 @@ void explicit_maps_single (){
   // CK12: define {{.+}}[[CALL09]]
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK13 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK13
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK13 -verify -Wno-vla  -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK13
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK13 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK13
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK13 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK13
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK13
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK13 -verify -Wno-vla  -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK13
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK13 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK13
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK13
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK13 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK13 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK13 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK13 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK13 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK13 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
 // SIMD-ONLY8-NOT: {{__kmpc|__tgt}}
 #ifdef CK13
 
@@ -706,19 +706,19 @@ void explicit_maps_single (){
   // CK13: define {{.+}}[[CALL09]]
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK14 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK14
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK14 -verify -Wno-vla  -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK14
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK14 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK14
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK14 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK14
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK14
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK14 -verify -Wno-vla  -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK14
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK14 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK14
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK14
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK14 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK14 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK14 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK14 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK14 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK14 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY8 %s
 // SIMD-ONLY8-NOT: {{__kmpc|__tgt}}
 #ifdef CK14
 
@@ -757,19 +757,19 @@ void explicit_maps_single (){
   // CK14: define {{.+}}[[CALL09]]
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK15 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK15
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK15 -verify -Wno-vla  -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK15
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK15 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK15
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK15 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK15
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK15
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK15 -verify -Wno-vla  -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK15
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK15 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK15
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK15
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK15 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY12 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK15 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY12 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK15 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY12 %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK15 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY12 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY12 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK15 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY12 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK15 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY12 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY12 %s
 // SIMD-ONLY12-NOT: {{__kmpc|__tgt}}
 #ifdef CK15
 
@@ -836,19 +836,19 @@ void implicit_maps_variable_length_array (int a){
 // CK15: {{.+}} = getelementptr inbounds double, ptr [[REF]], i[[sz]] %{{.+}}
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK16 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK16
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK16 -verify -Wno-vla  -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK16
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK16 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK16
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK16 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK16
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK16
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK16 -verify -Wno-vla  -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK16
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK16 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK16
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK16
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK16 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK16 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK16 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK16 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK16 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK16 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
 // SIMD-ONLY16-NOT: {{__kmpc|__tgt}}
 #ifdef CK16
 
@@ -896,19 +896,19 @@ void implicit_maps_struct (int a){
 // CK16: {{.+}} = getelementptr inbounds [[ST]], ptr [[REF]], i32 0, i32 0
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK17 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK17
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK17 -verify -Wno-vla  -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK17
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK17 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK17
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK17 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK17
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK17
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK17 -verify -Wno-vla  -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK17
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK17 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK17
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK17
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK17 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK17 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK17 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK17 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK17 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK17 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
 // SIMD-ONLY16-NOT: {{__kmpc|__tgt}}
 #ifdef CK17
 
@@ -956,19 +956,19 @@ void implicit_maps_struct (int a){
 // CK17: {{.+}} = getelementptr inbounds [[ST]], ptr [[REF]], i32 0, i32 0
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK18 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK18
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK18 -verify -Wno-vla  -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK18
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK18 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK18
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK18 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK18
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK18
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK18 -verify -Wno-vla  -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK18
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK18 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK18
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK18
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK18 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK18 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK18 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK18 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK18 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK18 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
 // SIMD-ONLY16-NOT: {{__kmpc|__tgt}}
 #ifdef CK18
 
@@ -1016,19 +1016,19 @@ void implicit_maps_struct (int a){
 // CK18: {{.+}} = getelementptr inbounds [[ST]], ptr [[REF]], i32 0, i32 0
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK19 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK19
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK19 -verify -Wno-vla  -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK19
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK19 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK19
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK19 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK19
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK19
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK19 -verify -Wno-vla  -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK19
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK19 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK19
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK19
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK19 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK19 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK19 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK19 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK19 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK19 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
 // SIMD-ONLY16-NOT: {{__kmpc|__tgt}}
 #ifdef CK19
 
@@ -1076,19 +1076,19 @@ void implicit_maps_struct (int a){
 // CK19: {{.+}} = getelementptr inbounds [[ST]], ptr [[REF]], i32 0, i32 0
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK20 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK20 --check-prefix CK20-64
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK20 -verify -Wno-vla  -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK20 --check-prefix CK20-64
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK20 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK20  --check-prefix CK20-64
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK20 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK20  --check-prefix CK20-32
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK20  --check-prefix CK20-64
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK20 -verify -Wno-vla  -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK20  --check-prefix CK20-32
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK20 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK20  --check-prefix CK20-32
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK20  --check-prefix CK20-32
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK20 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY6 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK20 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY6 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK20 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY6 %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK20 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY6 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY6 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK20 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY6 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK20 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY6 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY6 %s
 // SIMD-ONLY6-NOT: {{__kmpc|__tgt}}
 #ifdef CK20
 
@@ -1147,19 +1147,19 @@ void implicit_maps_double (int a){
 
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK21 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK21
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK21 -verify -Wno-vla  -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK21
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK21 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK21
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK21 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK21
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK21
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK21 -verify -Wno-vla  -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK21
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK21 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK21
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK21
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK21 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK21 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK21 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK21 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK21 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK21 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY16 %s
 // SIMD-ONLY16-NOT: {{__kmpc|__tgt}}
 #ifdef CK21
 
@@ -1207,19 +1207,19 @@ void implicit_maps_struct (int a){
 // CK21: {{.+}} = getelementptr inbounds [[ST]], ptr [[REF]], i32 0, i32 0
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK22 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK22
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK22 -verify -Wno-vla  -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK22
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK22 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK22
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK22 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK22
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK22
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK22 -verify -Wno-vla  -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK22
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK22 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK22
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK22
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK22 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY9 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK22 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY9 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK22 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY9 %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK22 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY9 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY9 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK22 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY9 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK22 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY9 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY9 %s
 // SIMD-ONLY9-NOT: {{__kmpc|__tgt}}
 #ifdef CK22
 
@@ -1262,19 +1262,19 @@ void implicit_maps_pointer (){
 
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK23 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK23 --check-prefix CK23-64
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK23 -verify -Wno-vla  -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK23 --check-prefix CK23-64
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK23 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK23 --check-prefix CK23-64
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK23 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK23 --check-prefix CK23-32
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck %s  --check-prefix CK23 --check-prefix CK23-64
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK23 -verify -Wno-vla  -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK23 --check-prefix CK23-32
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK23 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK23 --check-prefix CK23-32
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck %s  --check-prefix CK23 --check-prefix CK23-32
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK23 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK23 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK23 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK23 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK23 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK23 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 #ifdef CK23
 
@@ -1466,19 +1466,19 @@ void bar(float *&a, int *&b) {
 
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK24 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK24 --check-prefix CK24-64
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK24 -verify -Wno-vla  -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK24 --check-prefix CK24-64
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK24 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK24 --check-prefix CK24-64
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK24 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK24 --check-prefix CK24-32
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK24 --check-prefix CK24-64
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK24 -verify -Wno-vla  -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK24 --check-prefix CK24-32
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK24 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK24 --check-prefix CK24-32
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK24 --check-prefix CK24-32
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK24 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK24 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK24 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK24 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK24 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK24 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
 // SIMD-ONLY18-NOT: {{__kmpc|__tgt}}
 #ifdef CK24
 
@@ -1546,19 +1546,19 @@ void explicit_maps_single (int ii){
 
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK25 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK25 --check-prefix CK25-64
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK25 -verify -Wno-vla  -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK25 --check-prefix CK25-64
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK25 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK25 --check-prefix CK25-64
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK25 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK25 --check-prefix CK25-32
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK25 --check-prefix CK25-64
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK25 -verify -Wno-vla  -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK25 --check-prefix CK25-32
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK25 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK25 --check-prefix CK25-32
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK25 --check-prefix CK25-32
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK25 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK25 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK25 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK25 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK25 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK25 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
 // SIMD-ONLY18-NOT: {{__kmpc|__tgt}}
 #ifdef CK25
 
@@ -1581,19 +1581,19 @@ void declare_target_to()
 
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK26 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK26 --check-prefix CK26-64
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK26 -verify -Wno-vla  -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK26 --check-prefix CK26-64
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK26 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK26 --check-prefix CK26-64
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK26 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK26 --check-prefix CK26-32
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK26 --check-prefix CK26-64
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK26 -verify -Wno-vla  -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK26 --check-prefix CK26-32
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK26 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK26 --check-prefix CK26-32
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK26 --check-prefix CK26-32
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK26 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK26 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK26 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK26 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DCK26 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DCK26 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %s
 // SIMD-ONLY18-NOT: {{__kmpc|__tgt}}
 #ifdef CK26
 
diff --git a/clang/test/OpenMP/target_defaultmap_messages.cpp b/clang/test/OpenMP/target_defaultmap_messages.cpp
index 4295760ed2..8052e34d5c 100644
--- a/clang/test/OpenMP/target_defaultmap_messages.cpp
+++ b/clang/test/OpenMP/target_defaultmap_messages.cpp
@@ -1,11 +1,11 @@
-// RUN: %clang_cc1 -verify -fopenmp %s -verify=expected,omp51 -Wuninitialized -DOMP51
-// RUN: %clang_cc1 -verify -fopenmp-simd %s -verify=expected,omp51 -Wuninitialized -DOMP51
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp %s -verify=expected,omp51 -Wuninitialized -DOMP51
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd %s -verify=expected,omp51 -Wuninitialized -DOMP51
 
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=50 %s -verify=expected,omp5 -Wuninitialized -DOMP5
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=50 %s -verify=expected,omp5 -Wuninitialized -DOMP5
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=50 %s -verify=expected,omp5 -Wuninitialized -DOMP5
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=50 %s -verify=expected,omp5 -Wuninitialized -DOMP5
 
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 %s -verify=expected,omp45 -Wuninitialized
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 %s -verify=expected,omp45 -Wuninitialized
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 %s -verify=expected,omp45 -Wuninitialized
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 %s -verify=expected,omp45 -Wuninitialized
 
 struct Bar {
   int a;
diff --git a/clang/test/OpenMP/target_depend_codegen.cpp b/clang/test/OpenMP/target_depend_codegen.cpp
index 973c237441..83430dd97b 100644
--- a/clang/test/OpenMP/target_depend_codegen.cpp
+++ b/clang/test/OpenMP/target_depend_codegen.cpp
@@ -1,37 +1,37 @@
 // Test host codegen.
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 
 // Test target codegen - host bc file has to be created first.
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // SIMD-ONLY1-NOT: {{__kmpc|__tgt}}
 
 // expected-no-diagnostics
diff --git a/clang/test/OpenMP/target_enter_data_codegen.cpp b/clang/test/OpenMP/target_enter_data_codegen.cpp
index 5c993e3600..147d372fcc 100644
--- a/clang/test/OpenMP/target_enter_data_codegen.cpp
+++ b/clang/test/OpenMP/target_enter_data_codegen.cpp
@@ -3,19 +3,19 @@
 #define HEADER
 
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-64
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-64
 // RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-64
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-64
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-32
 // RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-32
 
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 #ifdef CK1
 
@@ -219,19 +219,19 @@ void foo(int arg) {
 
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK1A -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK1A --check-prefix CK1A-64
+// RUN: %clang_cc1 -DCK1A -verify -Wno-vla  -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK1A --check-prefix CK1A-64
 // RUN: %clang_cc1 -DCK1A -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1A --check-prefix CK1A-64
-// RUN: %clang_cc1 -DCK1A -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK1A --check-prefix CK1A-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1A --check-prefix CK1A-64
+// RUN: %clang_cc1 -DCK1A -verify -Wno-vla  -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK1A --check-prefix CK1A-32
 // RUN: %clang_cc1 -DCK1A -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1A --check-prefix CK1A-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1A --check-prefix CK1A-32
 
-// RUN: %clang_cc1 -DCK1A -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK1A -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK1A -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -DCK1A -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK1A -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK1A -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 #ifdef CK1A
 
@@ -301,19 +301,19 @@ void foo(int arg) {
 }
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-64
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla  -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-64
 // RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK2 --check-prefix CK2-64
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK2 --check-prefix CK2-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck %s  --check-prefix CK2 --check-prefix CK2-64
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla  -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK2 --check-prefix CK2-32
 // RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK2 --check-prefix CK2-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck %s  --check-prefix CK2 --check-prefix CK2-32
 
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // SIMD-ONLY1-NOT: {{__kmpc|__tgt}}
 #ifdef CK2
 
@@ -375,19 +375,19 @@ int bar(int arg){
 // CK2-NOT: __tgt_target_data_end
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK3 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK3 --check-prefix CK3-64
+// RUN: %clang_cc1 -DCK3 -verify -Wno-vla  -fopenmp -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK3 --check-prefix CK3-64
 // RUN: %clang_cc1 -DCK3 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK3 --check-prefix CK3-64
-// RUN: %clang_cc1 -DCK3 -verify -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK3 --check-prefix CK3-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck %s  --check-prefix CK3 --check-prefix CK3-64
+// RUN: %clang_cc1 -DCK3 -verify -Wno-vla  -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK3 --check-prefix CK3-32
 // RUN: %clang_cc1 -DCK3 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK3 --check-prefix CK3-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck %s  --check-prefix CK3 --check-prefix CK3-32
 
-// RUN: %clang_cc1 -DCK3 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// RUN: %clang_cc1 -DCK3 -verify -Wno-vla  -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY2 %s
 // RUN: %clang_cc1 -DCK3 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY2 %s
-// RUN: %clang_cc1 -DCK3 -verify -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// RUN: %clang_cc1 -DCK3 -verify -Wno-vla  -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY2 %s
 // RUN: %clang_cc1 -DCK3 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY2 %s
 // SIMD-ONLY2-NOT: {{__kmpc|__tgt}}
 #ifdef CK3
 
@@ -402,38 +402,38 @@ void no_target_devices(int arg) {
 }
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK4 --check-prefix CK4-64
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla  -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK4 --check-prefix CK4-64
 // RUN: %clang_cc1 -DCK4 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK4 --check-prefix CK4-64
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK4 --check-prefix CK4-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck %s  --check-prefix CK4 --check-prefix CK4-64
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla  -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK4 --check-prefix CK4-32
 // RUN: %clang_cc1 -DCK4 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK4 --check-prefix CK4-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck %s  --check-prefix CK4 --check-prefix CK4-32
 
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY3 %s
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY3 %s
 // RUN: %clang_cc1 -DCK4 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY3 %s
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY3 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY3 %s
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY3 %s
 // RUN: %clang_cc1 -DCK4 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY3 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY3 %s
 // SIMD-ONLY3-NOT: {{__kmpc|__tgt}}
 
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCK4 --check-prefix TCK4-64
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla  -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla  -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCK4 --check-prefix TCK4-64
 // RUN: %clang_cc1 -DCK4 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCK4 --check-prefix TCK4-64
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCK4 --check-prefix TCK4-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck %s --check-prefix TCK4 --check-prefix TCK4-64
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla  -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla  -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCK4 --check-prefix TCK4-32
 // RUN: %clang_cc1 -DCK4 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCK4 --check-prefix TCK4-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck %s --check-prefix TCK4 --check-prefix TCK4-32
 
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY4 %s
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla  -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla  -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY4 %s
 // RUN: %clang_cc1 -DCK4 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY4 %s
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY4 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY4 %s
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla  -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla  -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY4 %s
 // RUN: %clang_cc1 -DCK4 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY4 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY4 %s
 // SIMD-ONLY4-NOT: {{__kmpc|__tgt}}
 #ifdef CK4
 
@@ -448,19 +448,19 @@ void device_side_scan(int arg) {
 }
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK5 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK5 --check-prefix CK5-64
+// RUN: %clang_cc1 -DCK5 -verify -Wno-vla  -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK5 --check-prefix CK5-64
 // RUN: %clang_cc1 -DCK5 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK5 --check-prefix CK5-64
-// RUN: %clang_cc1 -DCK5 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK5 --check-prefix CK5-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck %s  --check-prefix CK5 --check-prefix CK5-64
+// RUN: %clang_cc1 -DCK5 -verify -Wno-vla  -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK5 --check-prefix CK5-32
 // RUN: %clang_cc1 -DCK5 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK5 --check-prefix CK5-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck %s  --check-prefix CK5 --check-prefix CK5-32
 
-// RUN: %clang_cc1 -DCK5 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -DCK5 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -DCK5 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
-// RUN: %clang_cc1 -DCK5 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -DCK5 -verify -Wno-vla  -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -DCK5 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla  %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // SIMD-ONLY1-NOT: {{__kmpc|__tgt}}
 #ifdef CK5
 
diff --git a/clang/test/OpenMP/target_enter_data_depend_codegen.cpp b/clang/test/OpenMP/target_enter_data_depend_codegen.cpp
index 997bca525d..373b36a5ad 100644
--- a/clang/test/OpenMP/target_enter_data_depend_codegen.cpp
+++ b/clang/test/OpenMP/target_enter_data_depend_codegen.cpp
@@ -1,16 +1,16 @@
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-64
 // RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-64
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-32
 // RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-32
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 
 // expected-no-diagnostics
diff --git a/clang/test/OpenMP/target_exit_data_codegen.cpp b/clang/test/OpenMP/target_exit_data_codegen.cpp
index df0d802337..96a0a5063d 100644
--- a/clang/test/OpenMP/target_exit_data_codegen.cpp
+++ b/clang/test/OpenMP/target_exit_data_codegen.cpp
@@ -3,19 +3,19 @@
 #define HEADER
 
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-64
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-64
 // RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-64
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-64
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-32
 // RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-32
 
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 #ifdef CK1
 
diff --git a/clang/test/OpenMP/target_exit_data_depend_codegen.cpp b/clang/test/OpenMP/target_exit_data_depend_codegen.cpp
index 3313398b6a..205ab3bd77 100644
--- a/clang/test/OpenMP/target_exit_data_depend_codegen.cpp
+++ b/clang/test/OpenMP/target_exit_data_depend_codegen.cpp
@@ -1,16 +1,16 @@
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-64
 // RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-64
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-32
 // RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-32
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 
 // expected-no-diagnostics
diff --git a/clang/test/OpenMP/target_firstprivate_codegen.cpp b/clang/test/OpenMP/target_firstprivate_codegen.cpp
index 2af54e3972..b4ead8f50c 100644
--- a/clang/test/OpenMP/target_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/target_firstprivate_codegen.cpp
@@ -1,37 +1,37 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _
 // Test host codegen.
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK0
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK0
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK1
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK2
+// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK1
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK2
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK3
+// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK3
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY01 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY02 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY01 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY02 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY03 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY03 %s
 
 // Test target codegen - host bc file has to be created first.
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK1
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK2
+// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK1
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK2
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK3
+// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK3
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY11 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY12 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY11 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY12 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY13 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY13 %s
 
 // expected-no-diagnostics
 #ifndef HEADER
diff --git a/clang/test/OpenMP/target_has_device_addr_codegen_01.cpp b/clang/test/OpenMP/target_has_device_addr_codegen_01.cpp
index 53a487dc0f..3fc22111a8 100644
--- a/clang/test/OpenMP/target_has_device_addr_codegen_01.cpp
+++ b/clang/test/OpenMP/target_has_device_addr_codegen_01.cpp
@@ -1,7 +1,7 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" --prefix-filecheck-ir-name _
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s
 
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // expected-no-diagnostics
 
 struct S {
diff --git a/clang/test/OpenMP/target_in_reduction_codegen.cpp b/clang/test/OpenMP/target_in_reduction_codegen.cpp
index 7c092ce35b..2eb6058461 100644
--- a/clang/test/OpenMP/target_in_reduction_codegen.cpp
+++ b/clang/test/OpenMP/target_in_reduction_codegen.cpp
@@ -1,6 +1,6 @@
-// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -triple x86_64-apple-darwin10 -fopenmp -x c++ -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -Wno-vla -triple x86_64-apple-darwin10 -fopenmp -x c++ -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -x c++ -triple x86_64-apple-darwin10 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -x c++ -triple x86_64-apple-darwin10 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
 // expected-no-diagnostics
 #ifndef HEADER
 #define HEADER
diff --git a/clang/test/OpenMP/target_map_codegen_12.cpp b/clang/test/OpenMP/target_map_codegen_12.cpp
index 2e42ccf7df..d662df7307 100644
--- a/clang/test/OpenMP/target_map_codegen_12.cpp
+++ b/clang/test/OpenMP/target_map_codegen_12.cpp
@@ -3,33 +3,33 @@
 #define HEADER
 
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK13 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK13
+// RUN: %clang_cc1 -DCK13 -verify -Wno-vla -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK13
 // RUN: %clang_cc1 -DCK13 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK13
-// RUN: %clang_cc1 -DCK13 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK13
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK13
+// RUN: %clang_cc1 -DCK13 -verify -Wno-vla -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK13
 // RUN: %clang_cc1 -DCK13 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK13
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK13
 
-// RUN: %clang_cc1 -DCK13 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK13
+// RUN: %clang_cc1 -DCK13 -verify -Wno-vla -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK13
 // RUN: %clang_cc1 -DCK13 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK13
-// RUN: %clang_cc1 -DCK13 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK13
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK13
+// RUN: %clang_cc1 -DCK13 -verify -Wno-vla -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK13
 // RUN: %clang_cc1 -DCK13 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK13
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK13
 
-// RUN: %clang_cc1 -DCK13 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK13
+// RUN: %clang_cc1 -DCK13 -verify -Wno-vla -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix CK13
 // RUN: %clang_cc1 -DCK13 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK13
-// RUN: %clang_cc1 -DCK13 -verify -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK13
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK13
+// RUN: %clang_cc1 -DCK13 -verify -Wno-vla -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK13
 // RUN: %clang_cc1 -DCK13 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK13
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s  --check-prefix CK13
 
-// RUN: %clang_cc1 -DCK13 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY12 %s
+// RUN: %clang_cc1 -DCK13 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY12 %s
 // RUN: %clang_cc1 -DCK13 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY12 %s
-// RUN: %clang_cc1 -DCK13 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY12 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY12 %s
+// RUN: %clang_cc1 -DCK13 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY12 %s
 // RUN: %clang_cc1 -DCK13 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY12 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY12 %s
 // SIMD-ONLY12-NOT: {{__kmpc|__tgt}}
 #ifdef CK13
 
diff --git a/clang/test/OpenMP/target_map_codegen_18a.cpp b/clang/test/OpenMP/target_map_codegen_18a.cpp
index 63d314b478..030a8b9ee5 100644
--- a/clang/test/OpenMP/target_map_codegen_18a.cpp
+++ b/clang/test/OpenMP/target_map_codegen_18a.cpp
@@ -8,15 +8,15 @@
 // RUN: cp %S/target_map_codegen_18.inc %t.inc
 
 ///==========================================================================///
-// RUN: %clang_cc1 -DUSE -DCK19 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-64,CK19-USE %t.inc
+// RUN: %clang_cc1 -DUSE -DCK19 -verify -Wno-vla -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-64,CK19-USE %t.inc
 // RUN: %clang_cc1 -DUSE -DCK19 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -DUSE -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-64,CK19-USE %t.inc
-// RUN: %clang_cc1 -DUSE -DCK19 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-32,CK19-USE %t.inc
+// RUN: %clang_cc1 -DUSE -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-64,CK19-USE %t.inc
+// RUN: %clang_cc1 -DUSE -DCK19 -verify -Wno-vla -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-32,CK19-USE %t.inc
 // RUN: %clang_cc1 -DUSE -DCK19 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -DUSE -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-32,CK19-USE %t.inc
+// RUN: %clang_cc1 -DUSE -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-32,CK19-USE %t.inc
 
-// RUN: %clang_cc1 -DUSE -DCK19 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-64,CK19-USE %t.inc
+// RUN: %clang_cc1 -DUSE -DCK19 -verify -Wno-vla -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-64,CK19-USE %t.inc
 // RUN: %clang_cc1 -DUSE -DCK19 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -DUSE -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-64,CK19-USE %t.inc
+// RUN: %clang_cc1 -DUSE -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-64,CK19-USE %t.inc
 
 #endif
diff --git a/clang/test/OpenMP/target_map_codegen_18b.cpp b/clang/test/OpenMP/target_map_codegen_18b.cpp
index 4b22f2a19c..00028140ca 100644
--- a/clang/test/OpenMP/target_map_codegen_18b.cpp
+++ b/clang/test/OpenMP/target_map_codegen_18b.cpp
@@ -7,15 +7,15 @@
 // its full name in every FileCheck command.
 // RUN: cp %S/target_map_codegen_18.inc %t.inc
 
-// RUN: %clang_cc1 -DUSE -DCK19 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-32,CK19-USE %t.inc
+// RUN: %clang_cc1 -DUSE -DCK19 -verify -Wno-vla -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-32,CK19-USE %t.inc
 // RUN: %clang_cc1 -DUSE -DCK19 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -DUSE -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-32,CK19-USE %t.inc
+// RUN: %clang_cc1 -DUSE -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-32,CK19-USE %t.inc
 
-// RUN: %clang_cc1 -DUSE -DCK19 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-64,CK19-USE %t.inc
+// RUN: %clang_cc1 -DUSE -DCK19 -verify -Wno-vla -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-64,CK19-USE %t.inc
 // RUN: %clang_cc1 -DUSE -DCK19 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -DUSE -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-64,CK19-USE %t.inc
-// RUN: %clang_cc1 -DUSE -DCK19 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-32,CK19-USE %t.inc
+// RUN: %clang_cc1 -DUSE -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-64,CK19-USE %t.inc
+// RUN: %clang_cc1 -DUSE -DCK19 -verify -Wno-vla -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-32,CK19-USE %t.inc
 // RUN: %clang_cc1 -DUSE -DCK19 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -DUSE -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-32,CK19-USE %t.inc
+// RUN: %clang_cc1 -DUSE -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-32,CK19-USE %t.inc
 
 #endif
diff --git a/clang/test/OpenMP/target_map_codegen_18c.cpp b/clang/test/OpenMP/target_map_codegen_18c.cpp
index 5bd0e09796..6fd57d33f7 100644
--- a/clang/test/OpenMP/target_map_codegen_18c.cpp
+++ b/clang/test/OpenMP/target_map_codegen_18c.cpp
@@ -7,22 +7,22 @@
 // its full name in every FileCheck command.
 // RUN: cp %S/target_map_codegen_18.inc %t.inc
 
-// RUN: %clang_cc1 -DUSE -DCK19 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap --check-prefix SIMD-ONLY18 %t.inc
+// RUN: %clang_cc1 -DUSE -DCK19 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap --check-prefix SIMD-ONLY18 %t.inc
 // RUN: %clang_cc1 -DUSE -DCK19 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -DUSE -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %t.inc
-// RUN: %clang_cc1 -DUSE -DCK19 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %t.inc
+// RUN: %clang_cc1 -DUSE -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %t.inc
+// RUN: %clang_cc1 -DUSE -DCK19 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %t.inc
 // RUN: %clang_cc1 -DUSE -DCK19 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -DUSE -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %t.inc
+// RUN: %clang_cc1 -DUSE -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %t.inc
 
-// RUN: %clang_cc1 -DCK19 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-64,CK19-NOUSE %t.inc
+// RUN: %clang_cc1 -DCK19 -verify -Wno-vla -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-64,CK19-NOUSE %t.inc
 // RUN: %clang_cc1 -DCK19 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefixes=CK19,CK19-64,CK19-NOUSE %t.inc
-// RUN: %clang_cc1 -DCK19 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefixes=CK19,CK19-32,CK19-NOUSE %t.inc
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefixes=CK19,CK19-64,CK19-NOUSE %t.inc
+// RUN: %clang_cc1 -DCK19 -verify -Wno-vla -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefixes=CK19,CK19-32,CK19-NOUSE %t.inc
 // RUN: %clang_cc1 -DCK19 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefixes=CK19,CK19-32,CK19-NOUSE %t.inc
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefixes=CK19,CK19-32,CK19-NOUSE %t.inc
 
-// RUN: %clang_cc1 -DCK19 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-64,CK19-NOUSE %t.inc
+// RUN: %clang_cc1 -DCK19 -verify -Wno-vla -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-64,CK19-NOUSE %t.inc
 // RUN: %clang_cc1 -DCK19 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-64,CK19-NOUSE %t.inc
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-64,CK19-NOUSE %t.inc
 
 #endif
diff --git a/clang/test/OpenMP/target_map_codegen_18d.cpp b/clang/test/OpenMP/target_map_codegen_18d.cpp
index 4ec1ac57f5..0b8a7ec667 100644
--- a/clang/test/OpenMP/target_map_codegen_18d.cpp
+++ b/clang/test/OpenMP/target_map_codegen_18d.cpp
@@ -7,22 +7,22 @@
 // its full name in every FileCheck command.
 // RUN: cp %S/target_map_codegen_18.inc %t.inc
 
-// RUN: %clang_cc1 -DCK19 -verify -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-32,CK19-NOUSE %t.inc
+// RUN: %clang_cc1 -DCK19 -verify -Wno-vla -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-32,CK19-NOUSE %t.inc
 // RUN: %clang_cc1 -DCK19 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-32,CK19-NOUSE %t.inc
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-32,CK19-NOUSE %t.inc
 
-// RUN: %clang_cc1 -DCK19 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-64,CK19-NOUSE %t.inc
+// RUN: %clang_cc1 -DCK19 -verify -Wno-vla -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-64,CK19-NOUSE %t.inc
 // RUN: %clang_cc1 -DCK19 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-64,CK19-NOUSE %t.inc
-// RUN: %clang_cc1 -DCK19 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-32,CK19-NOUSE %t.inc
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-64,CK19-NOUSE %t.inc
+// RUN: %clang_cc1 -DCK19 -verify -Wno-vla -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-32,CK19-NOUSE %t.inc
 // RUN: %clang_cc1 -DCK19 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-32,CK19-NOUSE %t.inc
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap --check-prefixes=CK19,CK19-32,CK19-NOUSE %t.inc
 
-// RUN: %clang_cc1 -DCK19 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %t.inc
+// RUN: %clang_cc1 -DCK19 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %t.inc
 // RUN: %clang_cc1 -DCK19 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %t.inc
-// RUN: %clang_cc1 -DCK19 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %t.inc
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %t.inc
+// RUN: %clang_cc1 -DCK19 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %t.inc
 // RUN: %clang_cc1 -DCK19 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %t.inc
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  --check-prefix SIMD-ONLY18 %t.inc
 
 #endif
diff --git a/clang/test/OpenMP/target_map_messages.cpp b/clang/test/OpenMP/target_map_messages.cpp
index 436fa51a4f..c517ede987 100644
--- a/clang/test/OpenMP/target_map_messages.cpp
+++ b/clang/test/OpenMP/target_map_messages.cpp
@@ -1,35 +1,35 @@
 // -fopenmp, -fno-openmp-extensions
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,lt50,lt51,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=40 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,lt50,lt51,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=45 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,ge50,ge51,omp,ge51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=51 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,ge50,ge51,ge52,omp,ge52-omp -fopenmp -fno-openmp-extensions -fopenmp-version=52 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized
-// RUN: %clang_cc1 -DCCODE -verify -fopenmp -fno-openmp-extensions -ferror-limit 300 -x c %s -Wno-openmp -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,lt51,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,lt50,lt51,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=40 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,lt50,lt51,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=45 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,lt51,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,ge51,omp,ge51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=51 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,ge51,ge52,omp,ge52-omp -fopenmp -fno-openmp-extensions -fopenmp-version=52 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -DCCODE -verify -fopenmp -fno-openmp-extensions -ferror-limit 300 -x c %s -Wno-openmp -Wuninitialized -Wno-vla
 
 // -fopenmp-simd, -fno-openmp-extensions
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,lt50,lt51,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=40 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,lt50,lt51,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=45 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,ge50,ge51,omp,ge51-omp -fopenmp-simd -fno-openmp-extensions -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized
-// RUN: %clang_cc1 -DCCODE -verify -fopenmp-simd -fno-openmp-extensions -ferror-limit 300 -x c %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,lt51,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,lt50,lt51,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=40 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,lt50,lt51,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=45 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,lt51,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,ge51,omp,ge51-omp -fopenmp-simd -fno-openmp-extensions -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -DCCODE -verify -fopenmp-simd -fno-openmp-extensions -ferror-limit 300 -x c %s -Wno-openmp-mapping -Wuninitialized -Wno-vla
 
 // -fopenmp -fopenmp-extensions
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,lt50,lt51,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=40 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,lt50,lt51,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=45 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,ge50,ge51,ompx,ge51-ompx -fopenmp -fopenmp-extensions -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized
-// RUN: %clang_cc1 -DCCODE -verify -fopenmp -fopenmp-extensions -ferror-limit 300 -x c %s -Wno-openmp -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,lt51,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,lt50,lt51,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=40 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,lt50,lt51,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=45 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,lt51,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,ge51,ompx,ge51-ompx -fopenmp -fopenmp-extensions -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -DCCODE -verify -fopenmp -fopenmp-extensions -ferror-limit 300 -x c %s -Wno-openmp -Wuninitialized -Wno-vla
 
 // -fopenmp-simd -fopenmp-extensions
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,lt50,lt51,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=40 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,lt50,lt51,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=45 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,ge50,ge51,ompx,ge51-ompx -fopenmp-simd -fopenmp-extensions -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized
-// RUN: %clang_cc1 -DCCODE -verify -fopenmp-simd -fopenmp-extensions -ferror-limit 300 -x c %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge50,lt51,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,lt50,lt51,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=40 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,lt50,lt51,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=45 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,lt51,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge50,ge51,ompx,ge51-ompx -fopenmp-simd -fopenmp-extensions -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -DCCODE -verify -fopenmp-simd -fopenmp-extensions -ferror-limit 300 -x c %s -Wno-openmp-mapping -Wuninitialized -Wno-vla
 
 // Check
 #ifdef CCODE
diff --git a/clang/test/OpenMP/target_parallel_codegen.cpp b/clang/test/OpenMP/target_parallel_codegen.cpp
index ef65bc4df7..df8a2c8787 100644
--- a/clang/test/OpenMP/target_parallel_codegen.cpp
+++ b/clang/test/OpenMP/target_parallel_codegen.cpp
@@ -1,71 +1,71 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _
 // Test host codegen.
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 
 // Test target codegen - host bc file has to be created first.
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 
 // Test host codegen.
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 
 // Test target codegen - host bc file has to be created first.
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 
 // expected-no-diagnostics
 #ifndef HEADER
diff --git a/clang/test/OpenMP/target_parallel_defaultmap_messages.cpp b/clang/test/OpenMP/target_parallel_defaultmap_messages.cpp
index cd562a30e3..4c226e9ca2 100644
--- a/clang/test/OpenMP/target_parallel_defaultmap_messages.cpp
+++ b/clang/test/OpenMP/target_parallel_defaultmap_messages.cpp
@@ -1,11 +1,11 @@
-// RUN: %clang_cc1 -verify -fopenmp %s -verify=expected,omp51 -Wuninitialized -DOMP51
-// RUN: %clang_cc1 -verify -fopenmp-simd %s -verify=expected,omp51 -Wuninitialized -DOMP51
+// RUN: %clang_cc1 -verify -fopenmp %s -verify=expected,omp51 -Wuninitialized -Wno-vla -DOMP51
+// RUN: %clang_cc1 -verify -fopenmp-simd %s -verify=expected,omp51 -Wuninitialized -Wno-vla -DOMP51
 
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=50 %s -verify=expected,omp5 -Wuninitialized -DOMP5
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=50 %s -verify=expected,omp5 -Wuninitialized -DOMP5
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=50 %s -verify=expected,omp5 -Wuninitialized -Wno-vla -DOMP5
+// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=50 %s -verify=expected,omp5 -Wuninitialized -Wno-vla -DOMP5
 
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 %s -verify=expected,omp45 -Wuninitialized
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 %s -verify=expected,omp45 -Wuninitialized
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 %s -verify=expected,omp45 -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 %s -verify=expected,omp45 -Wuninitialized -Wno-vla
 
 struct Bar {
   int a;
diff --git a/clang/test/OpenMP/target_parallel_depend_codegen.cpp b/clang/test/OpenMP/target_parallel_depend_codegen.cpp
index 18e124ba4d..22bf654691 100644
--- a/clang/test/OpenMP/target_parallel_depend_codegen.cpp
+++ b/clang/test/OpenMP/target_parallel_depend_codegen.cpp
@@ -1,37 +1,37 @@
 // Test host codegen.
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 
 // Test target codegen - host bc file has to be created first.
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // SIMD-ONLY1-NOT: {{__kmpc|__tgt}}
 
 // expected-no-diagnostics
diff --git a/clang/test/OpenMP/target_parallel_for_codegen.cpp b/clang/test/OpenMP/target_parallel_for_codegen.cpp
index e3999498fa..aa60cc3405 100644
--- a/clang/test/OpenMP/target_parallel_for_codegen.cpp
+++ b/clang/test/OpenMP/target_parallel_for_codegen.cpp
@@ -1,71 +1,71 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _
 // Test host codegen.
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK3
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 
 // Test target codegen - host bc file has to be created first.
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK11
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 
 // Test host codegen.
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK17
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK17
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK17
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK19
+// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK17
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK19
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK19
+// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK19
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 
 // Test target codegen - host bc file has to be created first.
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK11
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 
 // expected-no-diagnostics
 #ifndef HEADER
diff --git a/clang/test/OpenMP/target_parallel_for_depend_codegen.cpp b/clang/test/OpenMP/target_parallel_for_depend_codegen.cpp
index f06bffa1b7..801b188e97 100644
--- a/clang/test/OpenMP/target_parallel_for_depend_codegen.cpp
+++ b/clang/test/OpenMP/target_parallel_for_depend_codegen.cpp
@@ -1,37 +1,37 @@
 // Test host codegen.
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 
 // Test target codegen - host bc file has to be created first.
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // SIMD-ONLY1-NOT: {{__kmpc|__tgt}}
 
 // expected-no-diagnostics
diff --git a/clang/test/OpenMP/target_parallel_for_simd_codegen.cpp b/clang/test/OpenMP/target_parallel_for_simd_codegen.cpp
index 31c9afbdc5..3b88f6eee5 100644
--- a/clang/test/OpenMP/target_parallel_for_simd_codegen.cpp
+++ b/clang/test/OpenMP/target_parallel_for_simd_codegen.cpp
@@ -1,65 +1,65 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _
 // Test host codegen.
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK3
-// RUN: %clang_cc1 -verify -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK5
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK5
 // RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK5
-// RUN: %clang_cc1 -verify -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK7
+// RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK5
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK7
 // RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK7
+// RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK7
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK11
-// RUN: %clang_cc1 -verify -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK13
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK13
 // RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK13
-// RUN: %clang_cc1 -verify -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK15
+// RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK13
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK15
 // RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK15
+// RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK15
 
 // Test target codegen - host bc file has to be created first.
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK17
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK17
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK17
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK19
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK17
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK19
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK19
-// RUN: %clang_cc1 -verify -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK21
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK19
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK21
 // RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK21
-// RUN: %clang_cc1 -verify -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK23
+// RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK21
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK23
 // RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK23
+// RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK23
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK11
-// RUN: %clang_cc1 -verify -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK13
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK13
 // RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK13
-// RUN: %clang_cc1 -verify -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK15
+// RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK13
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK15
 // RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK15
+// RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK15
 
 // expected-no-diagnostics
 #ifndef HEADER
diff --git a/clang/test/OpenMP/target_parallel_for_simd_depend_codegen.cpp b/clang/test/OpenMP/target_parallel_for_simd_depend_codegen.cpp
index 3d9868179d..f848922a03 100644
--- a/clang/test/OpenMP/target_parallel_for_simd_depend_codegen.cpp
+++ b/clang/test/OpenMP/target_parallel_for_simd_depend_codegen.cpp
@@ -1,37 +1,37 @@
 // Test host codegen.
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 
 // Test target codegen - host bc file has to be created first.
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // SIMD-ONLY1-NOT: {{__kmpc|__tgt}}
 
 // expected-no-diagnostics
diff --git a/clang/test/OpenMP/target_parallel_generic_loop_depend_codegen.cpp b/clang/test/OpenMP/target_parallel_generic_loop_depend_codegen.cpp
index feaa33c198..6403240091 100644
--- a/clang/test/OpenMP/target_parallel_generic_loop_depend_codegen.cpp
+++ b/clang/test/OpenMP/target_parallel_generic_loop_depend_codegen.cpp
@@ -1,37 +1,37 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _
 // Test host codegen.
-// RUN: %clang_cc1  -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK
+// RUN: %clang_cc1  -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK
 // RUN: %clang_cc1  -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1  -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK
-// RUN: %clang_cc1  -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK
+// RUN: %clang_cc1  -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK
+// RUN: %clang_cc1  -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK
 // RUN: %clang_cc1  -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1  -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK
+// RUN: %clang_cc1  -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK
 
-// RUN: %clang_cc1  -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1  -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1  -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1  -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1  -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1  -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1  -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1  -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1  -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1  -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 
 // Test target codegen - host bc file has to be created first.
-// RUN: %clang_cc1  -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1  -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK
+// RUN: %clang_cc1  -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1  -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK
 // RUN: %clang_cc1  -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1  -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK
-// RUN: %clang_cc1  -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1  -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK
+// RUN: %clang_cc1  -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK
+// RUN: %clang_cc1  -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1  -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK
 // RUN: %clang_cc1  -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1  -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK
+// RUN: %clang_cc1  -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK
 
-// RUN: %clang_cc1  -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1  -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1  -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1  -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1  -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1  -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
-// RUN: %clang_cc1  -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1  -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1  -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1  -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1  -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1  -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1  -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1  -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 
 // expected-no-diagnostics
 #ifndef HEADER
diff --git a/clang/test/OpenMP/target_private_codegen.cpp b/clang/test/OpenMP/target_private_codegen.cpp
index 61d6633403..4173313073 100644
--- a/clang/test/OpenMP/target_private_codegen.cpp
+++ b/clang/test/OpenMP/target_private_codegen.cpp
@@ -1,22 +1,22 @@
 // Only test codegen on target side, as private clause does not require any action on the host side
 // Test target codegen - host bc file has to be created first.
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 
 // expected-no-diagnostics
diff --git a/clang/test/OpenMP/target_reduction_codegen.cpp b/clang/test/OpenMP/target_reduction_codegen.cpp
index a28990929b..8cf22d8229 100644
--- a/clang/test/OpenMP/target_reduction_codegen.cpp
+++ b/clang/test/OpenMP/target_reduction_codegen.cpp
@@ -1,22 +1,22 @@
 // Only test codegen on target side, as private clause does not require any action on the host side
 // Test target codegen - host bc file has to be created first.
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 
 // expected-no-diagnostics
diff --git a/clang/test/OpenMP/target_simd_codegen.cpp b/clang/test/OpenMP/target_simd_codegen.cpp
index ce8310b279..9dd8b07010 100644
--- a/clang/test/OpenMP/target_simd_codegen.cpp
+++ b/clang/test/OpenMP/target_simd_codegen.cpp
@@ -1,65 +1,65 @@
 // Test host codegen.
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix OMP45
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix OMP45
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix OMP45
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix OMP45
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix OMP45
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix OMP45
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix OMP45
-// RUN: %clang_cc1 -verify -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix OMP51
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix OMP45
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix OMP51
 // RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix OMP51
-// RUN: %clang_cc1 -verify -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix OMP51
+// RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix OMP51
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix OMP51
 // RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix OMP51
+// RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix OMP51
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 
 // Test target codegen - host bc file has to be created first.
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64 --check-prefix TOMP45
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64 --check-prefix TOMP45
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64 --check-prefix TOMP45
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32 --check-prefix TOMP45
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64 --check-prefix TOMP45
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32 --check-prefix TOMP45
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32 --check-prefix TOMP45
-// RUN: %clang_cc1 -verify -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64 --check-prefix TOMP51
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32 --check-prefix TOMP45
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64 --check-prefix TOMP51
 // RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64 --check-prefix TOMP51
-// RUN: %clang_cc1 -verify -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32 --check-prefix TOMP51
+// RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64 --check-prefix TOMP51
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32 --check-prefix TOMP51
 // RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32 --check-prefix TOMP51
+// RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32 --check-prefix TOMP51
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // SIMD-ONLY1-NOT: {{__kmpc|__tgt}}
 
 // expected-no-diagnostics
diff --git a/clang/test/OpenMP/target_simd_depend_codegen.cpp b/clang/test/OpenMP/target_simd_depend_codegen.cpp
index c02e430bb3..e7ce853d7e 100644
--- a/clang/test/OpenMP/target_simd_depend_codegen.cpp
+++ b/clang/test/OpenMP/target_simd_depend_codegen.cpp
@@ -1,37 +1,37 @@
 // Test host codegen.
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 
 // Test target codegen - host bc file has to be created first.
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // SIMD-ONLY1-NOT: {{__kmpc|__tgt}}
 
 // expected-no-diagnostics
diff --git a/clang/test/OpenMP/target_teams_codegen.cpp b/clang/test/OpenMP/target_teams_codegen.cpp
index 3185d90b5e..9ee1f74e8f 100644
--- a/clang/test/OpenMP/target_teams_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_codegen.cpp
@@ -1,70 +1,70 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _
 // Test host codegen.
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK3
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 
 // Test target codegen - host bc file has to be created first.
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK11
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK3
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 
 // Test target codegen - host bc file has to be created first.
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK11
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 
 // expected-no-diagnostics
 #ifndef HEADER
diff --git a/clang/test/OpenMP/target_teams_depend_codegen.cpp b/clang/test/OpenMP/target_teams_depend_codegen.cpp
index 8b4158681a..844dc83857 100644
--- a/clang/test/OpenMP/target_teams_depend_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_depend_codegen.cpp
@@ -1,37 +1,37 @@
 // Test host codegen.
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 
 // Test target codegen - host bc file has to be created first.
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // SIMD-ONLY1-NOT: {{__kmpc|__tgt}}
 
 // expected-no-diagnostics
diff --git a/clang/test/OpenMP/target_teams_distribute_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_codegen.cpp
index 1ef51aa9ab..a735809016 100644
--- a/clang/test/OpenMP/target_teams_distribute_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_codegen.cpp
@@ -1,71 +1,71 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _
 // Test host codegen.
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK3
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 
 // Test target codegen - host bc file has to be created first.
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK11
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 
 // Test host codegen.
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK3
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 
 // Test target codegen - host bc file has to be created first.
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck -allow-deprecated-dag-overlap  %s --check-prefix=CHECK11
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 
 // expected-no-diagnostics
 #ifndef HEADER
diff --git a/clang/test/OpenMP/target_teams_distribute_collapse_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_collapse_codegen.cpp
index 6be515e14d..298568f8b7 100644
--- a/clang/test/OpenMP/target_teams_distribute_collapse_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_collapse_codegen.cpp
@@ -4,19 +4,19 @@
 #define HEADER
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
 
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 #ifdef CK1
 
 template <typename T, int X, long long Y>
@@ -46,19 +46,19 @@ int teams_template_struct(void) {
 #endif // CK1
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
 
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 #ifdef CK2
 
 template <typename T, int n, int m>
diff --git a/clang/test/OpenMP/target_teams_distribute_depend_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_depend_codegen.cpp
index a0c71eac85..108a6811db 100644
--- a/clang/test/OpenMP/target_teams_distribute_depend_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_depend_codegen.cpp
@@ -1,37 +1,37 @@
 // Test host codegen.
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 
 // Test target codegen - host bc file has to be created first.
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // SIMD-ONLY1-NOT: {{__kmpc|__tgt}}
 
 // expected-no-diagnostics
diff --git a/clang/test/OpenMP/target_teams_distribute_dist_schedule_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_dist_schedule_codegen.cpp
index 821d3e8668..f4b424eb25 100644
--- a/clang/test/OpenMP/target_teams_distribute_dist_schedule_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_dist_schedule_codegen.cpp
@@ -4,19 +4,19 @@
 #define HEADER
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
 
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 #ifdef CK1
 
 template <typename T, int X, long long Y>
@@ -55,19 +55,19 @@ int teams_template_struct(void) {
 #endif // CK1
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
 
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 #ifdef CK2
 
 template <typename T, int n>
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_collapse_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_collapse_codegen.cpp
index 709b5ea996..1eeab3cbf0 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_collapse_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_collapse_codegen.cpp
@@ -4,19 +4,19 @@
 #define HEADER
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
 
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 #ifdef CK1
 
 template <typename T, int X, long long Y>
@@ -47,19 +47,19 @@ int teams_template_struct(void) {
 #endif // CK1
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
 
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 #ifdef CK2
 
 template <typename T, int n, int m>
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_depend_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_depend_codegen.cpp
index eb93cd3c5c..f6e201e2fc 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_depend_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_depend_codegen.cpp
@@ -1,37 +1,37 @@
 // Test host codegen.
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 
 // Test target codegen - host bc file has to be created first.
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // SIMD-ONLY1-NOT: {{__kmpc|__tgt}}
 
 // expected-no-diagnostics
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_dist_schedule_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_dist_schedule_codegen.cpp
index 19998d295b..019869514a 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_dist_schedule_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_dist_schedule_codegen.cpp
@@ -4,19 +4,19 @@
 #define HEADER
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
 
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 #ifdef CK1
 
 template <typename T, int X, long long Y>
@@ -59,19 +59,19 @@ int teams_template_struct(void) {
 #endif // CK1
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
 
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 #ifdef CK2
 
 template <typename T, int n>
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_schedule_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_schedule_codegen.cpp
index 2a8b885b05..d6f9b412be 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_schedule_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_schedule_codegen.cpp
@@ -4,26 +4,26 @@
 #define HEADER
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
 
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK5
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK5
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK5
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK7
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK5
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK7
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK7
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK7
 
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 
 #ifdef CK1
 
@@ -83,26 +83,26 @@ int teams_template_struct(void) {
 #endif // CK1
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK13
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK13
 // RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK13
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK15
+// RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK13
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK15
 // RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK15
+// RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK15
 
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK17
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK17
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK17
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK19
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK17
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK19
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK19
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK19
 
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 #ifdef CK2
 
 template <typename T, int n>
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_collapse_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_collapse_codegen.cpp
index d5fbd0f2f0..44976df684 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_collapse_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_collapse_codegen.cpp
@@ -4,19 +4,19 @@
 #define HEADER
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
 
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK5
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK5
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK5
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK7
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK5
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK7
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK7
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK7
 #ifdef CK1
 
 template <typename T, int X, long long Y>
@@ -47,19 +47,19 @@ int teams_template_struct(void) {
 #endif // CK1
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
 
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK13
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK13
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK13
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK15
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK13
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK15
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK15
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK15
 #ifdef CK2
 
 template <typename T, int n, int m>
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_depend_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_depend_codegen.cpp
index 330be35b76..bea0ffce42 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_depend_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_depend_codegen.cpp
@@ -1,37 +1,37 @@
 // Test host codegen.
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 
 // Test target codegen - host bc file has to be created first.
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // SIMD-ONLY1-NOT: {{__kmpc|__tgt}}
 
 // expected-no-diagnostics
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_dist_schedule_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_dist_schedule_codegen.cpp
index 03d5486308..0d1374ddb6 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_dist_schedule_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_dist_schedule_codegen.cpp
@@ -4,19 +4,19 @@
 #define HEADER
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
 
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK5
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK5
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK5
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK7
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK5
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK7
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK7
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK7
 #ifdef CK1
 
 template <typename T, int X, long long Y>
@@ -59,19 +59,19 @@ int teams_template_struct(void) {
 #endif // CK1
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
 
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK13
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK13
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK13
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK15
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK13
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK15
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK15
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK15
 #ifdef CK2
 
 template <typename T, int n>
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_schedule_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_schedule_codegen.cpp
index e65545ceb0..aba9968fac 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_schedule_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_schedule_codegen.cpp
@@ -4,26 +4,26 @@
 #define HEADER
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
 
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK5
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK5
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK5
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK7
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK5
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK7
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK7
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK7
 
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
 
 #ifdef CK1
 
@@ -83,26 +83,26 @@ int teams_template_struct(void) {
 #endif // CK1
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK13
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK13
 // RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK13
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK15
+// RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK13
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK15
 // RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK15
+// RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK15
 
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK17
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK17
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK17
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK19
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK17
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK19
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK19
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK19
 
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK21
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK21
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK21
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK23
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK21
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK23
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK23
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK23
 #ifdef CK2
 
 template <typename T, int n>
diff --git a/clang/test/OpenMP/target_teams_distribute_simd_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_simd_codegen.cpp
index b208f65460..a3daee2532 100644
--- a/clang/test/OpenMP/target_teams_distribute_simd_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_simd_codegen.cpp
@@ -1,65 +1,65 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _
 // Test host codegen.
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck  %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck  %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck  %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck  %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck  %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck  %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck  %s --check-prefix=CHECK3
-// RUN: %clang_cc1 -verify -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck  %s --check-prefix=CHECK5
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck  %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck  %s --check-prefix=CHECK5
 // RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck  %s --check-prefix=CHECK5
-// RUN: %clang_cc1 -verify -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck  %s --check-prefix=CHECK7
+// RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck  %s --check-prefix=CHECK5
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck  %s --check-prefix=CHECK7
 // RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck  %s --check-prefix=CHECK7
+// RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck  %s --check-prefix=CHECK7
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck  %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck  %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck  %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck  %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck  %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck  %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck  %s --check-prefix=CHECK11
-// RUN: %clang_cc1 -verify -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck  %s --check-prefix=CHECK13
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck  %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck  %s --check-prefix=CHECK13
 // RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck  %s --check-prefix=CHECK13
-// RUN: %clang_cc1 -verify -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck  %s --check-prefix=CHECK15
+// RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck  %s --check-prefix=CHECK13
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck  %s --check-prefix=CHECK15
 // RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck  %s --check-prefix=CHECK15
+// RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck  %s --check-prefix=CHECK15
 
 // Test target codegen - host bc file has to be created first.
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck  %s --check-prefix=CHECK17
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck  %s --check-prefix=CHECK17
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck  %s --check-prefix=CHECK17
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck  %s --check-prefix=CHECK19
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck  %s --check-prefix=CHECK17
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck  %s --check-prefix=CHECK19
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck  %s --check-prefix=CHECK19
-// RUN: %clang_cc1 -verify -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck  %s --check-prefix=CHECK21
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck  %s --check-prefix=CHECK19
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck  %s --check-prefix=CHECK21
 // RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck  %s --check-prefix=CHECK21
-// RUN: %clang_cc1 -verify -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck  %s --check-prefix=CHECK23
+// RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck  %s --check-prefix=CHECK21
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck  %s --check-prefix=CHECK23
 // RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck  %s --check-prefix=CHECK23
+// RUN: %clang_cc1 -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck  %s --check-prefix=CHECK23
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck  %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck  %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck  %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck  %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck  %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck  %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck  %s --check-prefix=CHECK11
-// RUN: %clang_cc1 -verify -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck  %s --check-prefix=CHECK13
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck  %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck  %s --check-prefix=CHECK13
 // RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck  %s --check-prefix=CHECK13
-// RUN: %clang_cc1 -verify -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck  %s --check-prefix=CHECK15
+// RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck  %s --check-prefix=CHECK13
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck  %s --check-prefix=CHECK15
 // RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck  %s --check-prefix=CHECK15
+// RUN: %clang_cc1 -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck  %s --check-prefix=CHECK15
 // expected-no-diagnostics
 
 #ifndef HEADER
diff --git a/clang/test/OpenMP/target_teams_distribute_simd_collapse_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_simd_collapse_codegen.cpp
index 88d7dd168c..b4e68abe3c 100644
--- a/clang/test/OpenMP/target_teams_distribute_simd_collapse_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_simd_collapse_codegen.cpp
@@ -4,19 +4,19 @@
 #define HEADER
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
 
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK5
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK5
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK5
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK7
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK5
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK7
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK7
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK7
 #ifdef CK1
 
 template <typename T, int X, long long Y>
@@ -46,19 +46,19 @@ int teams_template_struct(void) {
 #endif // CK1
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
 
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK13
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK13
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK13
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK15
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK13
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK15
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK15
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK15
 #ifdef CK2
 
 template <typename T, int n, int m>
diff --git a/clang/test/OpenMP/target_teams_distribute_simd_depend_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_simd_depend_codegen.cpp
index 64390bde1c..ffe85c1624 100644
--- a/clang/test/OpenMP/target_teams_distribute_simd_depend_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_simd_depend_codegen.cpp
@@ -1,46 +1,46 @@
 // Test host codegen.
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix OMP50 --check-prefix OMP50-64
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix OMP45
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix OMP50 --check-prefix OMP50-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix OMP45
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix OMP50 --check-prefix OMP50-64
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix OMP50 --check-prefix OMP50-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix OMP50 --check-prefix OMP50-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix OMP50 --check-prefix OMP50-32
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix OMP50 --check-prefix OMP50-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix OMP50 --check-prefix OMP50-32
 
 
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix OMP45
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix OMP45
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-64 --check-prefix OMP45
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix OMP45
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix OMP45
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK --check-prefix CHECK-32 --check-prefix OMP45
 
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 
 // Test target codegen - host bc file has to be created first.
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK --check-prefix TCHECK-32
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // SIMD-ONLY1-NOT: {{__kmpc|__tgt}}
 
 // expected-no-diagnostics
diff --git a/clang/test/OpenMP/target_teams_distribute_simd_dist_schedule_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_simd_dist_schedule_codegen.cpp
index 83993fb965..aebbe0e695 100644
--- a/clang/test/OpenMP/target_teams_distribute_simd_dist_schedule_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_simd_dist_schedule_codegen.cpp
@@ -4,19 +4,19 @@
 #define HEADER
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
 
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK5
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK5
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK5
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK7
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK5
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK7
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK7
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK7
 #ifdef CK1
 
 template <typename T, int X, long long Y>
@@ -55,19 +55,19 @@ int teams_template_struct(void) {
 #endif // CK1
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
 
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK13
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK13
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK13
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK15
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK13
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK15
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK15
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK15
 #ifdef CK2
 
 template <typename T, int n>
diff --git a/clang/test/OpenMP/target_teams_generic_loop_collapse_codegen.cpp b/clang/test/OpenMP/target_teams_generic_loop_collapse_codegen.cpp
index 3201902e86..558d9ab15c 100644
--- a/clang/test/OpenMP/target_teams_generic_loop_collapse_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_generic_loop_collapse_codegen.cpp
@@ -4,19 +4,19 @@
 #define HEADER
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
 
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 #ifdef CK1
 
 template <typename T, int X, long long Y>
@@ -47,19 +47,19 @@ int teams_template_struct(void) {
 #endif // CK1
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
 
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 #ifdef CK2
 
 template <typename T, int n, int m>
diff --git a/clang/test/OpenMP/target_teams_generic_loop_depend_codegen.cpp b/clang/test/OpenMP/target_teams_generic_loop_depend_codegen.cpp
index f9a33278df..a712cd7d33 100644
--- a/clang/test/OpenMP/target_teams_generic_loop_depend_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_generic_loop_depend_codegen.cpp
@@ -1,37 +1,37 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _
 // Test host codegen.
-// RUN: %clang_cc1  -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK
+// RUN: %clang_cc1  -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK
 // RUN: %clang_cc1  -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1  -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK
-// RUN: %clang_cc1  -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK
+// RUN: %clang_cc1  -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK
+// RUN: %clang_cc1  -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix CHECK
 // RUN: %clang_cc1  -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1  -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK
+// RUN: %clang_cc1  -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix CHECK
 
-// RUN: %clang_cc1  -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1  -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1  -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1  -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1  -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1  -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1  -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1  -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1  -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1  -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 
 // Test target codegen - host bc file has to be created first.
-// RUN: %clang_cc1  -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1  -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK
+// RUN: %clang_cc1  -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1  -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCHECK
 // RUN: %clang_cc1  -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1  -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK
-// RUN: %clang_cc1  -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1  -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK
+// RUN: %clang_cc1  -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK
+// RUN: %clang_cc1  -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1  -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCHECK
 // RUN: %clang_cc1  -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1  -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK
+// RUN: %clang_cc1  -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCHECK
 
-// RUN: %clang_cc1  -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1  -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1  -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1  -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1  -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1  -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
-// RUN: %clang_cc1  -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1  -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1  -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1  -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1  -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1  -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1  -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1  -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 
 // expected-no-diagnostics
 #ifndef HEADER
diff --git a/clang/test/OpenMP/target_teams_map_messages.cpp b/clang/test/OpenMP/target_teams_map_messages.cpp
index 9cbd2a20ec..3987bc744f 100644
--- a/clang/test/OpenMP/target_teams_map_messages.cpp
+++ b/clang/test/OpenMP/target_teams_map_messages.cpp
@@ -1,11 +1,11 @@
-// RUN: %clang_cc1 -verify=expected,ge45,ge50,lt51 -fopenmp -fno-openmp-extensions -ferror-limit 200 %s -Wno-openmp-mapping -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,lt45,lt50,lt51 -fopenmp -fno-openmp-extensions -fopenmp-version=40 -ferror-limit 200 %s -Wno-openmp-mapping -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,ge45,lt50,lt51 -fopenmp -fno-openmp-extensions -fopenmp-version=45 -ferror-limit 200 %s -Wno-openmp-mapping -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,ge45,ge50,lt51 -fopenmp -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 200 %s -Wno-openmp-mapping -Wuninitialized
-// RUN: %clang_cc1 -verify=expected,ge45,ge50,ge51 -fopenmp -fno-openmp-extensions -ferror-limit 200 %s -Wno-openmp-mapping -Wuninitialized
-
-// RUN: %clang_cc1 -verify=expected,ge45,ge50,lt51 -fopenmp-simd -fno-openmp-extensions -ferror-limit 200 %s -Wno-openmp-mapping -Wuninitialized
-// RUN: %clang_cc1 -DCCODE -verify=expected,ge45,ge50,lt51 -fopenmp -fno-openmp-extensions -ferror-limit 200 -x c %s -Wno-openmp-mapping -Wuninitialized
+// RUN: %clang_cc1 -verify=expected,ge45,ge50,lt51 -fopenmp -fno-openmp-extensions -ferror-limit 200 %s -Wno-openmp-mapping -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,lt45,lt50,lt51 -fopenmp -fno-openmp-extensions -fopenmp-version=40 -ferror-limit 200 %s -Wno-openmp-mapping -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge45,lt50,lt51 -fopenmp -fno-openmp-extensions -fopenmp-version=45 -ferror-limit 200 %s -Wno-openmp-mapping -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge45,ge50,lt51 -fopenmp -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 200 %s -Wno-openmp-mapping -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge45,ge50,ge51 -fopenmp -fno-openmp-extensions -ferror-limit 200 %s -Wno-openmp-mapping -Wuninitialized -Wno-vla
+
+// RUN: %clang_cc1 -verify=expected,ge45,ge50,lt51 -fopenmp-simd -fno-openmp-extensions -ferror-limit 200 %s -Wno-openmp-mapping -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -DCCODE -verify=expected,ge45,ge50,lt51 -fopenmp -fno-openmp-extensions -ferror-limit 200 -x c %s -Wno-openmp-mapping -Wuninitialized -Wno-vla
 #ifdef CCODE
 void foo(int arg) {
   const int n = 0;
diff --git a/clang/test/OpenMP/target_update_codegen.cpp b/clang/test/OpenMP/target_update_codegen.cpp
index 7e9e4d8be0..b577be3c1b 100644
--- a/clang/test/OpenMP/target_update_codegen.cpp
+++ b/clang/test/OpenMP/target_update_codegen.cpp
@@ -3,19 +3,19 @@
 #define HEADER
 
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-64
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-64
 // RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-64
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-64
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-32
 // RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-32
 
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 #ifdef CK1
 
@@ -160,19 +160,19 @@ void foo(int arg) {
 // CK1:     }
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-64
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK2 --check-prefix CK2-64
 // RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK2 --check-prefix CK2-64
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK2 --check-prefix CK2-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK2 --check-prefix CK2-64
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK2 --check-prefix CK2-32
 // RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK2 --check-prefix CK2-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK2 --check-prefix CK2-32
 
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY1 %s
 // SIMD-ONLY1-NOT: {{__kmpc|__tgt}}
 #ifdef CK2
 
@@ -234,19 +234,19 @@ int bar(int arg){
 // CK2: %{{.+}} = add nsw i32 %{{[^,]+}}, 1
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK3 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK3 --check-prefix CK3-64
+// RUN: %clang_cc1 -DCK3 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK3 --check-prefix CK3-64
 // RUN: %clang_cc1 -DCK3 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK3 --check-prefix CK3-64
-// RUN: %clang_cc1 -DCK3 -verify -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK3 --check-prefix CK3-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK3 --check-prefix CK3-64
+// RUN: %clang_cc1 -DCK3 -verify -Wno-vla -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK3 --check-prefix CK3-32
 // RUN: %clang_cc1 -DCK3 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK3 --check-prefix CK3-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK3 --check-prefix CK3-32
 
-// RUN: %clang_cc1 -DCK3 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// RUN: %clang_cc1 -DCK3 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY2 %s
 // RUN: %clang_cc1 -DCK3 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY2 %s
-// RUN: %clang_cc1 -DCK3 -verify -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// RUN: %clang_cc1 -DCK3 -verify -Wno-vla -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY2 %s
 // RUN: %clang_cc1 -DCK3 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY2 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY2 %s
 // SIMD-ONLY2-NOT: {{__kmpc|__tgt}}
 #ifdef CK3
 
@@ -260,38 +260,38 @@ void no_target_devices(int arg) {
 }
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK4 --check-prefix CK4-64
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK4 --check-prefix CK4-64
 // RUN: %clang_cc1 -DCK4 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK4 --check-prefix CK4-64
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK4 --check-prefix CK4-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK4 --check-prefix CK4-64
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK4 --check-prefix CK4-32
 // RUN: %clang_cc1 -DCK4 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK4 --check-prefix CK4-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK4 --check-prefix CK4-32
 
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY3 %s
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY3 %s
 // RUN: %clang_cc1 -DCK4 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY3 %s
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY3 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY3 %s
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY3 %s
 // RUN: %clang_cc1 -DCK4 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY3 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY3 %s
 // SIMD-ONLY3-NOT: {{__kmpc|__tgt}}
 
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCK4 --check-prefix TCK4-64
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s --check-prefix TCK4 --check-prefix TCK4-64
 // RUN: %clang_cc1 -DCK4 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCK4 --check-prefix TCK4-64
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCK4 --check-prefix TCK4-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCK4 --check-prefix TCK4-64
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck %s --check-prefix TCK4 --check-prefix TCK4-32
 // RUN: %clang_cc1 -DCK4 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix TCK4 --check-prefix TCK4-32
+// RUN: %clang_cc1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix TCK4 --check-prefix TCK4-32
 
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY4 %s
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck --check-prefix SIMD-ONLY4 %s
 // RUN: %clang_cc1 -DCK4 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY4 %s
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY4 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY4 %s
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm-bc %s -o %t-x86-host.bc
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o - | FileCheck --check-prefix SIMD-ONLY4 %s
 // RUN: %clang_cc1 -DCK4 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY4 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -fopenmp-is-target-device -fopenmp-host-ir-file-path %t-x86-host.bc -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY4 %s
 // SIMD-ONLY4-NOT: {{__kmpc|__tgt}}
 #ifdef CK4
 
@@ -306,19 +306,19 @@ void device_side_scan(int arg) {
 }
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK5 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK5 --check-prefix CK5-64
+// RUN: %clang_cc1 -DCK5 -verify -Wno-vla -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK5 --check-prefix CK5-64
 // RUN: %clang_cc1 -DCK5 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK5 --check-prefix CK5-64
-// RUN: %clang_cc1 -DCK5 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK5 --check-prefix CK5-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK5 --check-prefix CK5-64
+// RUN: %clang_cc1 -DCK5 -verify -Wno-vla -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK5 --check-prefix CK5-32
 // RUN: %clang_cc1 -DCK5 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK5 --check-prefix CK5-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK5 --check-prefix CK5-32
 
-// RUN: %clang_cc1 -DCK5 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK5 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK5 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -DCK5 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK5 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK5 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 
 #ifdef CK5
@@ -346,19 +346,19 @@ void lvalue(int *B, int l, int e) {
 
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK6 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK6 --check-prefix CK6-64
+// RUN: %clang_cc1 -DCK6 -verify -Wno-vla -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK6 --check-prefix CK6-64
 // RUN: %clang_cc1 -DCK6 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK6 --check-prefix CK6-64
-// RUN: %clang_cc1 -DCK6 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK6 --check-prefix CK6-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK6 --check-prefix CK6-64
+// RUN: %clang_cc1 -DCK6 -verify -Wno-vla -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK6 --check-prefix CK6-32
 // RUN: %clang_cc1 -DCK6 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK6 --check-prefix CK6-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK6 --check-prefix CK6-32
 
-// RUN: %clang_cc1 -DCK6 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK6 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK6 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -DCK6 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK6 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK6 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 
 #ifdef CK6
@@ -391,19 +391,19 @@ void lvalue(int *B, int l, int e) {
 
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK7 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK7 --check-prefix CK7-64
+// RUN: %clang_cc1 -DCK7 -verify -Wno-vla -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK7 --check-prefix CK7-64
 // RUN: %clang_cc1 -DCK7 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK7 --check-prefix CK7-64
-// RUN: %clang_cc1 -DCK7 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK7 --check-prefix CK7-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK7 --check-prefix CK7-64
+// RUN: %clang_cc1 -DCK7 -verify -Wno-vla -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK7 --check-prefix CK7-32
 // RUN: %clang_cc1 -DCK7 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK7 --check-prefix CK7-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK7 --check-prefix CK7-32
 
-// RUN: %clang_cc1 -DCK7 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK7 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK7 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -DCK7 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK7 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK7 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 
 #ifdef CK7
@@ -440,19 +440,19 @@ void lvalue(int *B, int l, int e) {
 
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK8 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK8 --check-prefix CK8-64
+// RUN: %clang_cc1 -DCK8 -verify -Wno-vla -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK8 --check-prefix CK8-64
 // RUN: %clang_cc1 -DCK8 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK8 --check-prefix CK8-64
-// RUN: %clang_cc1 -DCK8 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK8 --check-prefix CK8-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK8 --check-prefix CK8-64
+// RUN: %clang_cc1 -DCK8 -verify -Wno-vla -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK8 --check-prefix CK8-32
 // RUN: %clang_cc1 -DCK8 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK8 --check-prefix CK8-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK8 --check-prefix CK8-32
 
-// RUN: %clang_cc1 -DCK8 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK8 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK8 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -DCK8 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK8 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK8 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 
 #ifdef CK8
@@ -488,19 +488,19 @@ void lvalue(int **B, int l, int e) {
 
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK9 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK9 --check-prefix CK9-64
+// RUN: %clang_cc1 -DCK9 -verify -Wno-vla -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK9 --check-prefix CK9-64
 // RUN: %clang_cc1 -DCK9 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK9 --check-prefix CK9-64
-// RUN: %clang_cc1 -DCK9 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK9 --check-prefix CK9-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK9 --check-prefix CK9-64
+// RUN: %clang_cc1 -DCK9 -verify -Wno-vla -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK9 --check-prefix CK9-32
 // RUN: %clang_cc1 -DCK9 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK9 --check-prefix CK9-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK9 --check-prefix CK9-32
 
-// RUN: %clang_cc1 -DCK9 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK9 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK9 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -DCK9 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK9 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK9 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 
 #ifdef CK9
@@ -536,19 +536,19 @@ void lvalue(struct S *s, int l, int e) {
 
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK10 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK10 --check-prefix CK10-64
+// RUN: %clang_cc1 -DCK10 -verify -Wno-vla -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK10 --check-prefix CK10-64
 // RUN: %clang_cc1 -DCK10 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK10 --check-prefix CK10-64
-// RUN: %clang_cc1 -DCK10 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK10 --check-prefix CK10-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK10 --check-prefix CK10-64
+// RUN: %clang_cc1 -DCK10 -verify -Wno-vla -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK10 --check-prefix CK10-32
 // RUN: %clang_cc1 -DCK10 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK10 --check-prefix CK10-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK10 --check-prefix CK10-32
 
-// RUN: %clang_cc1 -DCK10 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK10 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK10 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -DCK10 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK10 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK10 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 #ifdef CK10
 
@@ -584,19 +584,19 @@ void lvalue(struct S *s, int l, int e) {
 
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK11 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK11 --check-prefix CK11-64
+// RUN: %clang_cc1 -DCK11 -verify -Wno-vla -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK11 --check-prefix CK11-64
 // RUN: %clang_cc1 -DCK11 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK11 --check-prefix CK11-64
-// RUN: %clang_cc1 -DCK11 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK11 --check-prefix CK11-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK11 --check-prefix CK11-64
+// RUN: %clang_cc1 -DCK11 -verify -Wno-vla -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK11 --check-prefix CK11-32
 // RUN: %clang_cc1 -DCK11 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK11 --check-prefix CK11-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK11 --check-prefix CK11-32
 
-// RUN: %clang_cc1 -DCK11 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK11 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK11 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -DCK11 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK11 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK11 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 #ifdef CK11
 
@@ -632,19 +632,19 @@ void lvalue(struct S *s, int l, int e) {
 
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK12 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK12 --check-prefix CK12-64
+// RUN: %clang_cc1 -DCK12 -verify -Wno-vla -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK12 --check-prefix CK12-64
 // RUN: %clang_cc1 -DCK12 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK12 --check-prefix CK12-64
-// RUN: %clang_cc1 -DCK12 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK12 --check-prefix CK12-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK12 --check-prefix CK12-64
+// RUN: %clang_cc1 -DCK12 -verify -Wno-vla -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK12 --check-prefix CK12-32
 // RUN: %clang_cc1 -DCK12 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK12 --check-prefix CK12-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK12 --check-prefix CK12-32
 
-// RUN: %clang_cc1 -DCK12 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK12 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK12 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -DCK12 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK12 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK12 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 #ifdef CK12
 
@@ -688,19 +688,19 @@ void lvalue(struct S *s, int l, int e) {
 
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK13 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK13 --check-prefix CK13-64
+// RUN: %clang_cc1 -DCK13 -verify -Wno-vla -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK13 --check-prefix CK13-64
 // RUN: %clang_cc1 -DCK13 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK13 --check-prefix CK13-64
-// RUN: %clang_cc1 -DCK13 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK13 --check-prefix CK13-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK13 --check-prefix CK13-64
+// RUN: %clang_cc1 -DCK13 -verify -Wno-vla -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK13 --check-prefix CK13-32
 // RUN: %clang_cc1 -DCK13 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK13 --check-prefix CK13-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK13 --check-prefix CK13-32
 
-// RUN: %clang_cc1 -DCK13 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK13 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK13 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -DCK13 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK13 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK13 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 #ifdef CK13
 
@@ -733,19 +733,19 @@ void lvalue(int **BB, int a, int b) {
 
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK14 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK14 --check-prefix CK14-64
+// RUN: %clang_cc1 -DCK14 -verify -Wno-vla -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK14 --check-prefix CK14-64
 // RUN: %clang_cc1 -DCK14 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK14 --check-prefix CK14-64
-// RUN: %clang_cc1 -DCK14 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK14 --check-prefix CK14-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK14 --check-prefix CK14-64
+// RUN: %clang_cc1 -DCK14 -verify -Wno-vla -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK14 --check-prefix CK14-32
 // RUN: %clang_cc1 -DCK14 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK14 --check-prefix CK14-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK14 --check-prefix CK14-32
 
-// RUN: %clang_cc1 -DCK14 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK14 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK14 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -DCK14 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK14 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK14 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 #ifdef CK14
 
@@ -801,19 +801,19 @@ void lvalue_member(SSA *sap) {
 
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK15 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK15 --check-prefix CK15-64
+// RUN: %clang_cc1 -DCK15 -verify -Wno-vla -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK15 --check-prefix CK15-64
 // RUN: %clang_cc1 -DCK15 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK15 --check-prefix CK15-64
-// RUN: %clang_cc1 -DCK15 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK15 --check-prefix CK15-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK15 --check-prefix CK15-64
+// RUN: %clang_cc1 -DCK15 -verify -Wno-vla -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK15 --check-prefix CK15-32
 // RUN: %clang_cc1 -DCK15 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK15 --check-prefix CK15-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK15 --check-prefix CK15-32
 
-// RUN: %clang_cc1 -DCK15 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK15 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK15 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -DCK15 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK15 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK15 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 #ifdef CK15
 
@@ -859,19 +859,19 @@ void lvalue_member(SSA *sap) {
 
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK16 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK16 --check-prefix CK16-64
+// RUN: %clang_cc1 -DCK16 -verify -Wno-vla -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK16 --check-prefix CK16-64
 // RUN: %clang_cc1 -DCK16 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK16 --check-prefix CK16-64
-// RUN: %clang_cc1 -DCK16 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK16 --check-prefix CK16-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK16 --check-prefix CK16-64
+// RUN: %clang_cc1 -DCK16 -verify -Wno-vla -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK16 --check-prefix CK16-32
 // RUN: %clang_cc1 -DCK16 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK16 --check-prefix CK16-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK16 --check-prefix CK16-32
 
-// RUN: %clang_cc1 -DCK16 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK16 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK16 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -DCK16 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK16 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK16 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 #ifdef CK16
 
@@ -902,19 +902,19 @@ void lvalue_find_base(float *f, int *i) {
 
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK17 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK17 --check-prefix CK17-64
+// RUN: %clang_cc1 -DCK17 -verify -Wno-vla -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK17 --check-prefix CK17-64
 // RUN: %clang_cc1 -DCK17 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK17 --check-prefix CK17-64
-// RUN: %clang_cc1 -DCK17 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK17 --check-prefix CK17-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK17 --check-prefix CK17-64
+// RUN: %clang_cc1 -DCK17 -verify -Wno-vla -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK17 --check-prefix CK17-32
 // RUN: %clang_cc1 -DCK17 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK17 --check-prefix CK17-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK17 --check-prefix CK17-32
 
-// RUN: %clang_cc1 -DCK17 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK17 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK17 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -DCK17 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK17 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK17 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 #ifdef CK17
 
@@ -958,19 +958,19 @@ void lvalue_find_base(float **f, SSA *sa) {
 
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK18 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK18 --check-prefix CK18-64
+// RUN: %clang_cc1 -DCK18 -verify -Wno-vla -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK18 --check-prefix CK18-64
 // RUN: %clang_cc1 -DCK18 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK18 --check-prefix CK18-64
-// RUN: %clang_cc1 -DCK18 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK18 --check-prefix CK18-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK18 --check-prefix CK18-64
+// RUN: %clang_cc1 -DCK18 -verify -Wno-vla -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK18 --check-prefix CK18-32
 // RUN: %clang_cc1 -DCK18 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK18 --check-prefix CK18-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK18 --check-prefix CK18-32
 
-// RUN: %clang_cc1 -DCK18 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY18 %s
+// RUN: %clang_cc1 -DCK18 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY18 %s
 // RUN: %clang_cc1 -DCK18 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY18 %s
-// RUN: %clang_cc1 -DCK18 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY18 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY18 %s
+// RUN: %clang_cc1 -DCK18 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY18 %s
 // RUN: %clang_cc1 -DCK18 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY18 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY18 %s
 // SIMD-ONLY18-NOT: {{__kmpc|__tgt}}
 #ifdef CK18
 
@@ -1027,19 +1027,19 @@ void array_shaping(float *f, int sa) {
 
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK19 -verify -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK19 --check-prefix CK19-64
+// RUN: %clang_cc1 -DCK19 -verify -Wno-vla -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK19 --check-prefix CK19-64
 // RUN: %clang_cc1 -DCK19 -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK19 --check-prefix CK19-64
-// RUN: %clang_cc1 -DCK19 -verify -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK19 --check-prefix CK19-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK19 --check-prefix CK19-64
+// RUN: %clang_cc1 -DCK19 -verify -Wno-vla -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK19 --check-prefix CK19-32
 // RUN: %clang_cc1 -DCK19 -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK19 --check-prefix CK19-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK19 --check-prefix CK19-32
 
-// RUN: %clang_cc1 -DCK19 -verify -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK19 -verify -Wno-vla -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK19 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -DCK19 -verify -fopenmp-version=51 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -DCK19 -verify -Wno-vla -fopenmp-version=51 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -DCK19 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=51 -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 #ifdef CK19
 
@@ -1087,19 +1087,19 @@ void check_present(int arg) {
 #endif
 
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK20 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK20 --check-prefix CK20-64
+// RUN: %clang_cc1 -DCK20 -verify -Wno-vla -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK20 --check-prefix CK20-64
 // RUN: %clang_cc1 -DCK20 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK20 --check-prefix CK20-64
-// RUN: %clang_cc1 -DCK20 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK20 --check-prefix CK20-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK20 --check-prefix CK20-64
+// RUN: %clang_cc1 -DCK20 -verify -Wno-vla -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK20 --check-prefix CK20-32
 // RUN: %clang_cc1 -DCK20 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK20 --check-prefix CK20-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK20 --check-prefix CK20-32
 
-// RUN: %clang_cc1 -DCK20 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY19 %s
+// RUN: %clang_cc1 -DCK20 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY19 %s
 // RUN: %clang_cc1 -DCK20 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY19 %s
-// RUN: %clang_cc1 -DCK20 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY19 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY19 %s
+// RUN: %clang_cc1 -DCK20 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY19 %s
 // RUN: %clang_cc1 -DCK20 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY19 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY19 %s
 // SIMD-ONLY19-NOT: {{__kmpc|__tgt}}
 #ifdef CK20
 
@@ -1158,19 +1158,19 @@ void foo(int arg) {
 
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK21 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK21 --check-prefix CK21-64
+// RUN: %clang_cc1 -DCK21 -verify -Wno-vla -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK21 --check-prefix CK21-64
 // RUN: %clang_cc1 -DCK21 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK21 --check-prefix CK21-64
-// RUN: %clang_cc1 -DCK21 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK21 --check-prefix CK21-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK21 --check-prefix CK21-64
+// RUN: %clang_cc1 -DCK21 -verify -Wno-vla -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK21 --check-prefix CK21-32
 // RUN: %clang_cc1 -DCK21 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK21 --check-prefix CK21-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK21 --check-prefix CK21-32
 
-// RUN: %clang_cc1 -DCK21 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY19 %s
+// RUN: %clang_cc1 -DCK21 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY19 %s
 // RUN: %clang_cc1 -DCK21 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY19 %s
-// RUN: %clang_cc1 -DCK21 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY19 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY19 %s
+// RUN: %clang_cc1 -DCK21 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY19 %s
 // RUN: %clang_cc1 -DCK21 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY19 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY19 %s
 // SIMD-ONLY19-NOT: {{__kmpc|__tgt}}
 #ifdef CK21
 
@@ -1237,19 +1237,19 @@ void bar() {
 
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK22 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK22 --check-prefix CK22-64
+// RUN: %clang_cc1 -DCK22 -verify -Wno-vla -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK22 --check-prefix CK22-64
 // RUN: %clang_cc1 -DCK22 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK22 --check-prefix CK22-64
-// RUN: %clang_cc1 -DCK22 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK22 --check-prefix CK22-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK22 --check-prefix CK22-64
+// RUN: %clang_cc1 -DCK22 -verify -Wno-vla -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK22 --check-prefix CK22-32
 // RUN: %clang_cc1 -DCK22 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK22 --check-prefix CK22-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK22 --check-prefix CK22-32
 
-// RUN: %clang_cc1 -DCK22 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY19 %s
+// RUN: %clang_cc1 -DCK22 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY19 %s
 // RUN: %clang_cc1 -DCK22 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY19 %s
-// RUN: %clang_cc1 -DCK22 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY19 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY19 %s
+// RUN: %clang_cc1 -DCK22 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY19 %s
 // RUN: %clang_cc1 -DCK22 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY19 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY19 %s
 // SIMD-ONLY19-NOT: {{__kmpc|__tgt}}
 #ifdef CK22
 
@@ -1312,19 +1312,19 @@ void bar() {
 
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK23 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK23 --check-prefix CK23-64
+// RUN: %clang_cc1 -DCK23 -verify -Wno-vla -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK23 --check-prefix CK23-64
 // RUN: %clang_cc1 -DCK23 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK23 --check-prefix CK23-64
-// RUN: %clang_cc1 -DCK23 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK23 --check-prefix CK23-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK23 --check-prefix CK23-64
+// RUN: %clang_cc1 -DCK23 -verify -Wno-vla -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK23 --check-prefix CK23-32
 // RUN: %clang_cc1 -DCK23 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK23 --check-prefix CK23-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK23 --check-prefix CK23-32
 
-// RUN: %clang_cc1 -DCK23 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY19 %s
+// RUN: %clang_cc1 -DCK23 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY19 %s
 // RUN: %clang_cc1 -DCK23 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY19 %s
-// RUN: %clang_cc1 -DCK23 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY19 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY19 %s
+// RUN: %clang_cc1 -DCK23 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY19 %s
 // RUN: %clang_cc1 -DCK23 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY19 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY19 %s
 // SIMD-ONLY19-NOT: {{__kmpc|__tgt}}
 #ifdef CK23
 
@@ -1386,19 +1386,19 @@ void foo(int arg) {
 
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK24 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK24 --check-prefix CK24-64
+// RUN: %clang_cc1 -DCK24 -verify -Wno-vla -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK24 --check-prefix CK24-64
 // RUN: %clang_cc1 -DCK24 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK24 --check-prefix CK24-64
-// RUN: %clang_cc1 -DCK24 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK24 --check-prefix CK24-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK24 --check-prefix CK24-64
+// RUN: %clang_cc1 -DCK24 -verify -Wno-vla -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK24 --check-prefix CK24-32
 // RUN: %clang_cc1 -DCK24 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK24 --check-prefix CK24-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK24 --check-prefix CK24-32
 
-// RUN: %clang_cc1 -DCK24 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY19 %s
+// RUN: %clang_cc1 -DCK24 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY19 %s
 // RUN: %clang_cc1 -DCK24 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY19 %s
-// RUN: %clang_cc1 -DCK24 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY19 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY19 %s
+// RUN: %clang_cc1 -DCK24 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY19 %s
 // RUN: %clang_cc1 -DCK24 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY19 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY19 %s
 // SIMD-ONLY19-NOT: {{__kmpc|__tgt}}
 #ifdef CK24
 
@@ -1460,19 +1460,19 @@ void foo(int arg) {
 }
 #endif
 ///==========================================================================///
-// RUN: %clang_cc1 -DCK25 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK25 --check-prefix CK25-64
+// RUN: %clang_cc1 -DCK25 -verify -Wno-vla -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK25 --check-prefix CK25-64
 // RUN: %clang_cc1 -DCK25 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK25 --check-prefix CK25-64
-// RUN: %clang_cc1 -DCK25 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK25 --check-prefix CK25-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK25 --check-prefix CK25-64
+// RUN: %clang_cc1 -DCK25 -verify -Wno-vla -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK25 --check-prefix CK25-32
 // RUN: %clang_cc1 -DCK25 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK25 --check-prefix CK25-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK25 --check-prefix CK25-32
 
-// RUN: %clang_cc1 -DCK25 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY19 %s
+// RUN: %clang_cc1 -DCK25 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY19 %s
 // RUN: %clang_cc1 -DCK25 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY19 %s
-// RUN: %clang_cc1 -DCK25 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY19 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY19 %s
+// RUN: %clang_cc1 -DCK25 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY19 %s
 // RUN: %clang_cc1 -DCK25 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY19 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY19 %s
 // SIMD-ONLY19-NOT: {{__kmpc|__tgt}}
 #ifdef CK25
 
diff --git a/clang/test/OpenMP/target_update_depend_codegen.cpp b/clang/test/OpenMP/target_update_depend_codegen.cpp
index a19ff8fe3a..cfb8b286cc 100644
--- a/clang/test/OpenMP/target_update_depend_codegen.cpp
+++ b/clang/test/OpenMP/target_update_depend_codegen.cpp
@@ -1,16 +1,16 @@
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck %s --check-prefix CK1 --check-prefix CK1-64
 // RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-64
-// RUN: %clang_cc1 -verify -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-64
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-32
 // RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-32
+// RUN: %clang_cc1 -fopenmp -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s  --check-prefix CK1 --check-prefix CK1-32
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=powerpc64le-ibm-linux-gnu -x c++ -triple powerpc64le-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -std=c++11 -triple i386-unknown-unknown -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-targets=i386-pc-linux-gnu -x c++ -triple i386-unknown-unknown -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 
 // expected-no-diagnostics
diff --git a/clang/test/OpenMP/task_ast_print.cpp b/clang/test/OpenMP/task_ast_print.cpp
index 8615eebed7..79f56b0362 100644
--- a/clang/test/OpenMP/task_ast_print.cpp
+++ b/clang/test/OpenMP/task_ast_print.cpp
@@ -1,10 +1,10 @@
-// RUN: %clang_cc1 -verify -fopenmp -ast-print %s | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -ast-print %s | FileCheck %s
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -std=c++11 -include-pch %t -fsyntax-only -verify -Wno-vla %s -ast-print | FileCheck %s
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -ast-print %s | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -ast-print %s | FileCheck %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck %s
+// RUN: %clang_cc1 -fopenmp-simd -std=c++11 -include-pch %t -fsyntax-only -verify -Wno-vla %s -ast-print | FileCheck %s
 // expected-no-diagnostics
 
 #ifndef HEADER
diff --git a/clang/test/OpenMP/task_codegen.cpp b/clang/test/OpenMP/task_codegen.cpp
index d6e9e2e96c..5a4dd22ccd 100644
--- a/clang/test/OpenMP/task_codegen.cpp
+++ b/clang/test/OpenMP/task_codegen.cpp
@@ -1,43 +1,43 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _
-// RUN: %clang_cc1 -verify -triple x86_64-apple-darwin10 -fopenmp -fopenmp-version=50 -x c++ -emit-llvm %s -o - -DUNTIEDRT | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -verify -Wno-vla -triple x86_64-apple-darwin10 -fopenmp -fopenmp-version=50 -x c++ -emit-llvm %s -o - -DUNTIEDRT | FileCheck %s --check-prefix=CHECK1
 
-// RUN: %clang_cc1 -verify -triple x86_64-apple-darwin10 -fopenmp -x c++ \
+// RUN: %clang_cc1 -verify -Wno-vla -triple x86_64-apple-darwin10 -fopenmp -x c++ \
 // RUN: -fopenmp-version=51 -emit-llvm %s -o - -DUNTIEDRT -DOMP51 \
 // RUN: | FileCheck %s --check-prefix=CHECK1-51
 
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -x c++ -triple x86_64-apple-darwin10 -emit-pch -o %t %s -DUNTIEDRT
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -x c++ -triple x86_64-apple-darwin10 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK2
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -x c++ -triple x86_64-apple-darwin10 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK2
 
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -x c++ -triple \
 // RUN: x86_64-apple-darwin10 -emit-pch -o %t %s -DUNTIEDRT -DOMP51
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -x c++ -triple \
-// RUN: x86_64-apple-darwin10 -include-pch %t -verify %s -emit-llvm -o - \
+// RUN: x86_64-apple-darwin10 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - \
 // RUN: -DOMP51 | FileCheck %s --check-prefix=CHECK2-51
 
-// RUN: %clang_cc1 -verify -triple x86_64-apple-darwin10 -fopenmp -fopenmp-version=50 -fopenmp-enable-irbuilder -x c++ -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -verify -Wno-vla -triple x86_64-apple-darwin10 -fopenmp -fopenmp-version=50 -fopenmp-enable-irbuilder -x c++ -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-enable-irbuilder -x c++ -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-enable-irbuilder -x c++ -triple x86_64-apple-darwin10 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK4
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -fopenmp-enable-irbuilder -x c++ -triple x86_64-apple-darwin10 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK4
 
-// RUN: %clang_cc1 -verify -triple x86_64-apple-darwin10 -fopenmp \
+// RUN: %clang_cc1 -verify -Wno-vla -triple x86_64-apple-darwin10 -fopenmp \
 // RUN: -fopenmp-version=51 -fopenmp-enable-irbuilder -x c++ -emit-llvm \
 // RUN: %s -o - -DOMP51 | FileCheck %s --check-prefix=CHECK3-51
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -fopenmp-enable-irbuilder \
 // RUN: -x c++ -triple x86_64-apple-darwin10 -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=51 -fopenmp-enable-irbuilder \
-// RUN: -x c++ -triple x86_64-apple-darwin10 -include-pch %t -verify %s \
+// RUN: -x c++ -triple x86_64-apple-darwin10 -include-pch %t -verify -Wno-vla %s \
 // RUN: -emit-llvm -o - -DOMP51 | FileCheck %s --check-prefix=CHECK4-51
 
-// RUN: %clang_cc1 -verify -triple x86_64-apple-darwin10 -fopenmp-simd -fopenmp-version=50 -x c++ -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -verify -Wno-vla -triple x86_64-apple-darwin10 -fopenmp-simd -fopenmp-version=50 -x c++ -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=50 -x c++ -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=50 -x c++ -triple x86_64-apple-darwin10 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=50 -x c++ -triple x86_64-apple-darwin10 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 
-// RUN: %clang_cc1 -verify -triple x86_64-apple-darwin10 -fopenmp-simd \
+// RUN: %clang_cc1 -verify -Wno-vla -triple x86_64-apple-darwin10 -fopenmp-simd \
 // RUN: -fopenmp-version=51 -x c++ -emit-llvm %s -o - -DOMP51 \
 // RUN: | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=51 -x c++ \
 // RUN: -triple x86_64-apple-darwin10 -emit-pch -o %t %s -DOMP51
 // RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=51 -x c++ \
-// RUN: -triple x86_64-apple-darwin10 -include-pch %t -verify %s \
+// RUN: -triple x86_64-apple-darwin10 -include-pch %t -verify -Wno-vla %s \
 // RUN: -emit-llvm -o - -DOMP51 | FileCheck %s \
 // RUN: --implicit-check-not="{{__kmpc|__tgt}}"
 
diff --git a/clang/test/OpenMP/task_firstprivate_codegen.cpp b/clang/test/OpenMP/task_firstprivate_codegen.cpp
index 5f08281121..4b1def869b 100644
--- a/clang/test/OpenMP/task_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/task_firstprivate_codegen.cpp
@@ -1,16 +1,16 @@
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LAMBDA %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=BLOCKS %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=ARRAY %s
+// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LAMBDA %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=BLOCKS %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=ARRAY %s
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 // expected-no-diagnostics
 #ifndef ARRAY
diff --git a/clang/test/OpenMP/task_in_reduction_codegen.cpp b/clang/test/OpenMP/task_in_reduction_codegen.cpp
index 33c29dc994..41e6eaa1d6 100644
--- a/clang/test/OpenMP/task_in_reduction_codegen.cpp
+++ b/clang/test/OpenMP/task_in_reduction_codegen.cpp
@@ -1,11 +1,11 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _
-// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -triple x86_64-apple-darwin10 -fopenmp -x c++ -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -Wno-vla -triple x86_64-apple-darwin10 -fopenmp -x c++ -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -x c++ -triple x86_64-apple-darwin10 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -x c++ -triple x86_64-apple-darwin10 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -triple x86_64-apple-darwin10 -fopenmp-simd -x c++ -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -Wno-vla -triple x86_64-apple-darwin10 -fopenmp-simd -x c++ -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // expected-no-diagnostics
 #ifndef HEADER
 #define HEADER
diff --git a/clang/test/OpenMP/task_private_codegen.cpp b/clang/test/OpenMP/task_private_codegen.cpp
index 0dc04b79d9..28ee182da5 100644
--- a/clang/test/OpenMP/task_private_codegen.cpp
+++ b/clang/test/OpenMP/task_private_codegen.cpp
@@ -1,16 +1,16 @@
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LAMBDA %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=BLOCKS %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=ARRAY %s
+// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LAMBDA %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=BLOCKS %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=ARRAY %s
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 // expected-no-diagnostics
 
diff --git a/clang/test/OpenMP/taskgroup_task_reduction_codegen.cpp b/clang/test/OpenMP/taskgroup_task_reduction_codegen.cpp
index f4f840d124..1e20171f9e 100644
--- a/clang/test/OpenMP/taskgroup_task_reduction_codegen.cpp
+++ b/clang/test/OpenMP/taskgroup_task_reduction_codegen.cpp
@@ -1,12 +1,12 @@
-// RUN: %clang_cc1 -verify -triple x86_64-apple-darwin10 -fopenmp -x c++ -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -triple x86_64-apple-darwin10 -fopenmp -x c++ -emit-llvm %s -o - | FileCheck %s
 // RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 -fopenmp -x c++ %s -verify -debug-info-kind=limited -emit-llvm -o - -triple x86_64-apple-darwin10 | FileCheck %s --check-prefix=CHECK --check-prefix=DEBUG
+// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -x c++ %s -verify -Wno-vla -debug-info-kind=limited -emit-llvm -o - -triple x86_64-apple-darwin10 | FileCheck %s --check-prefix=CHECK --check-prefix=DEBUG
 
-// RUN: %clang_cc1 -verify -triple x86_64-apple-darwin10 -fopenmp-simd -x c++ -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -triple x86_64-apple-darwin10 -fopenmp-simd -x c++ -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ %s -verify -debug-info-kind=limited -emit-llvm -o - -triple x86_64-apple-darwin10 | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ %s -verify -Wno-vla -debug-info-kind=limited -emit-llvm -o - -triple x86_64-apple-darwin10 | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 // expected-no-diagnostics
 #ifndef HEADER
diff --git a/clang/test/OpenMP/taskloop_firstprivate_codegen.cpp b/clang/test/OpenMP/taskloop_firstprivate_codegen.cpp
index a67b74de84..da97a03893 100644
--- a/clang/test/OpenMP/taskloop_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/taskloop_firstprivate_codegen.cpp
@@ -1,16 +1,16 @@
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LAMBDA %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=BLOCKS %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=ARRAY %s
+// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LAMBDA %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=BLOCKS %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=ARRAY %s
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 // expected-no-diagnostics
 
diff --git a/clang/test/OpenMP/taskloop_in_reduction_codegen.cpp b/clang/test/OpenMP/taskloop_in_reduction_codegen.cpp
index a2cb0ac170..73ca0a7a4c 100644
--- a/clang/test/OpenMP/taskloop_in_reduction_codegen.cpp
+++ b/clang/test/OpenMP/taskloop_in_reduction_codegen.cpp
@@ -1,11 +1,11 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _
-// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -triple x86_64-apple-darwin10 -fopenmp -x c++ -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -Wno-vla -triple x86_64-apple-darwin10 -fopenmp -x c++ -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -x c++ -triple x86_64-apple-darwin10 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -x c++ -triple x86_64-apple-darwin10 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -triple x86_64-apple-darwin10 -fopenmp-simd -x c++ -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -Wno-vla -triple x86_64-apple-darwin10 -fopenmp-simd -x c++ -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // expected-no-diagnostics
 #ifndef HEADER
 #define HEADER
diff --git a/clang/test/OpenMP/taskloop_lastprivate_codegen.cpp b/clang/test/OpenMP/taskloop_lastprivate_codegen.cpp
index f9dd2a4dac..96a974dc38 100644
--- a/clang/test/OpenMP/taskloop_lastprivate_codegen.cpp
+++ b/clang/test/OpenMP/taskloop_lastprivate_codegen.cpp
@@ -1,18 +1,18 @@
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LAMBDA %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=BLOCKS %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=ARRAY %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DLOOP -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LOOP %s
+// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LAMBDA %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=BLOCKS %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=ARRAY %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DLOOP -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LOOP %s
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DLOOP -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DLOOP -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 // expected-no-diagnostics
 
diff --git a/clang/test/OpenMP/taskloop_private_codegen.cpp b/clang/test/OpenMP/taskloop_private_codegen.cpp
index 5a752904bb..2d1c86f529 100644
--- a/clang/test/OpenMP/taskloop_private_codegen.cpp
+++ b/clang/test/OpenMP/taskloop_private_codegen.cpp
@@ -1,16 +1,16 @@
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LAMBDA %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=BLOCKS %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=ARRAY %s
+// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LAMBDA %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=BLOCKS %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=ARRAY %s
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 // expected-no-diagnostics
 
diff --git a/clang/test/OpenMP/taskloop_reduction_codegen.cpp b/clang/test/OpenMP/taskloop_reduction_codegen.cpp
index ee2dde82e2..762f280131 100644
--- a/clang/test/OpenMP/taskloop_reduction_codegen.cpp
+++ b/clang/test/OpenMP/taskloop_reduction_codegen.cpp
@@ -1,6 +1,6 @@
-// RUN: %clang_cc1 -fopenmp -x c++ %s -verify -debug-info-kind=limited -emit-llvm -o - -triple powerpc64le-unknown-linux-gnu -std=c++98 | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -x c++ %s -verify -Wno-vla -debug-info-kind=limited -emit-llvm -o - -triple powerpc64le-unknown-linux-gnu -std=c++98 | FileCheck %s
 
-// RUN: %clang_cc1 -fopenmp-simd -x c++ %s -verify -debug-info-kind=limited -emit-llvm -o - -triple powerpc64le-unknown-linux-gnu -std=c++98 | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ %s -verify -Wno-vla -debug-info-kind=limited -emit-llvm -o - -triple powerpc64le-unknown-linux-gnu -std=c++98 | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 // expected-no-diagnostics
 
diff --git a/clang/test/OpenMP/taskloop_simd_firstprivate_codegen.cpp b/clang/test/OpenMP/taskloop_simd_firstprivate_codegen.cpp
index a4e3915b81..969ee1d1dd 100644
--- a/clang/test/OpenMP/taskloop_simd_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/taskloop_simd_firstprivate_codegen.cpp
@@ -1,16 +1,16 @@
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LAMBDA %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=BLOCKS %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=ARRAY %s
+// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LAMBDA %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=BLOCKS %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=ARRAY %s
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 // expected-no-diagnostics
 
diff --git a/clang/test/OpenMP/taskloop_simd_in_reduction_codegen.cpp b/clang/test/OpenMP/taskloop_simd_in_reduction_codegen.cpp
index 049947297a..85df70f25a 100644
--- a/clang/test/OpenMP/taskloop_simd_in_reduction_codegen.cpp
+++ b/clang/test/OpenMP/taskloop_simd_in_reduction_codegen.cpp
@@ -1,11 +1,11 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _
-// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -triple x86_64-apple-darwin10 -fopenmp -x c++ -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -Wno-vla -triple x86_64-apple-darwin10 -fopenmp -x c++ -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -x c++ -triple x86_64-apple-darwin10 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp -x c++ -triple x86_64-apple-darwin10 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -triple x86_64-apple-darwin10 -fopenmp-simd -x c++ -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -no-enable-noundef-analysis -verify -Wno-vla -triple x86_64-apple-darwin10 -fopenmp-simd -x c++ -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -no-enable-noundef-analysis -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
 // expected-no-diagnostics
 #ifndef HEADER
 #define HEADER
diff --git a/clang/test/OpenMP/taskloop_simd_lastprivate_codegen.cpp b/clang/test/OpenMP/taskloop_simd_lastprivate_codegen.cpp
index 788a0acfde..5781f07e64 100644
--- a/clang/test/OpenMP/taskloop_simd_lastprivate_codegen.cpp
+++ b/clang/test/OpenMP/taskloop_simd_lastprivate_codegen.cpp
@@ -1,16 +1,16 @@
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LAMBDA %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=BLOCKS %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=ARRAY %s
+// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LAMBDA %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=BLOCKS %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=ARRAY %s
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 // expected-no-diagnostics
 
diff --git a/clang/test/OpenMP/taskloop_simd_private_codegen.cpp b/clang/test/OpenMP/taskloop_simd_private_codegen.cpp
index f2d064dd6c..2950541426 100644
--- a/clang/test/OpenMP/taskloop_simd_private_codegen.cpp
+++ b/clang/test/OpenMP/taskloop_simd_private_codegen.cpp
@@ -1,16 +1,16 @@
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck %s
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LAMBDA %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=BLOCKS %s
-// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=ARRAY %s
+// RUN: %clang_cc1 -fopenmp -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=LAMBDA %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=BLOCKS %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck -check-prefix=ARRAY %s
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -triple x86_64-apple-darwin10 -emit-pch -o %t %s
-// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
-// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ -triple x86_64-apple-darwin10 -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DLAMBDA -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -fblocks -DBLOCKS -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -DARRAY -triple x86_64-apple-darwin10 -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 // expected-no-diagnostics
 
diff --git a/clang/test/OpenMP/taskloop_simd_reduction_codegen.cpp b/clang/test/OpenMP/taskloop_simd_reduction_codegen.cpp
index 0e2a0ae6b5..64221ad3af 100644
--- a/clang/test/OpenMP/taskloop_simd_reduction_codegen.cpp
+++ b/clang/test/OpenMP/taskloop_simd_reduction_codegen.cpp
@@ -1,6 +1,6 @@
-// RUN: %clang_cc1 -fopenmp -x c++ %s -verify -debug-info-kind=limited -emit-llvm -o - -triple powerpc64le-unknown-linux-gnu -fnoopenmp-use-tls -std=c++98 | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -x c++ %s -verify -Wno-vla -debug-info-kind=limited -emit-llvm -o - -triple powerpc64le-unknown-linux-gnu -fnoopenmp-use-tls -std=c++98 | FileCheck %s
 
-// RUN: %clang_cc1 -fopenmp-simd -x c++ %s -verify -debug-info-kind=limited -emit-llvm -o - -triple powerpc64le-unknown-linux-gnu -fnoopenmp-use-tls -std=c++98 | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -x c++ %s -verify -Wno-vla -debug-info-kind=limited -emit-llvm -o - -triple powerpc64le-unknown-linux-gnu -fnoopenmp-use-tls -std=c++98 | FileCheck --check-prefix SIMD-ONLY0 %s
 // SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
 // expected-no-diagnostics
 
diff --git a/clang/test/OpenMP/teams_distribute_codegen.cpp b/clang/test/OpenMP/teams_distribute_codegen.cpp
index 973ad8cf38..6856482539 100644
--- a/clang/test/OpenMP/teams_distribute_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_codegen.cpp
@@ -3,19 +3,19 @@
 #ifndef HEADER
 #define HEADER
 // Test host codegen.
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
 
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 #ifdef CK1
 
 int a[100];
@@ -51,19 +51,19 @@ int teams_argument_global(int n) {
 #endif // CK1
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
 
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 #ifdef CK2
 
 int teams_local_arg(void) {
@@ -84,19 +84,19 @@ int teams_local_arg(void) {
 #endif // CK2
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK3 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK17
+// RUN: %clang_cc1 -DCK3 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK17
 // RUN: %clang_cc1 -DCK3 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK3 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK17
-// RUN: %clang_cc1 -DCK3 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK19
+// RUN: %clang_cc1 -DCK3 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK17
+// RUN: %clang_cc1 -DCK3 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK19
 // RUN: %clang_cc1 -DCK3 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK3 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK19
+// RUN: %clang_cc1 -DCK3 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK19
 
-// RUN: %clang_cc1 -DCK3 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK3 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK3 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK3 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -DCK3 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK3 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK3 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK3 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK3 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK3 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 #ifdef CK3
 
 
@@ -127,19 +127,19 @@ int teams_template_struct(void) {
 #endif // CK3
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK25
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK25
 // RUN: %clang_cc1 -DCK4 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK4 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK25
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK27
+// RUN: %clang_cc1 -DCK4 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK25
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK27
 // RUN: %clang_cc1 -DCK4 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK4 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK27
+// RUN: %clang_cc1 -DCK4 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK27
 
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK4 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK4 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK4 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK4 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK4 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK4 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 
 #ifdef CK4
 
diff --git a/clang/test/OpenMP/teams_distribute_collapse_codegen.cpp b/clang/test/OpenMP/teams_distribute_collapse_codegen.cpp
index 4d7223f25e..fccd4268a4 100644
--- a/clang/test/OpenMP/teams_distribute_collapse_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_collapse_codegen.cpp
@@ -4,19 +4,19 @@
 #define HEADER
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
 
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 #ifdef CK1
 
 template <typename T, int X, long long Y>
@@ -47,19 +47,19 @@ int teams_template_struct(void) {
 #endif // CK1
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
 
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 #ifdef CK2
 
 template <typename T, int n, int m>
diff --git a/clang/test/OpenMP/teams_distribute_dist_schedule_codegen.cpp b/clang/test/OpenMP/teams_distribute_dist_schedule_codegen.cpp
index 05262df480..40b03586ad 100644
--- a/clang/test/OpenMP/teams_distribute_dist_schedule_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_dist_schedule_codegen.cpp
@@ -4,19 +4,19 @@
 #define HEADER
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
 
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 #ifdef CK1
 
 template <typename T, int X, long long Y>
@@ -58,19 +58,19 @@ int teams_template_struct(void) {
 #endif // CK1
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
 
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 #ifdef CK2
 
 template <typename T, int n>
diff --git a/clang/test/OpenMP/teams_distribute_parallel_for_codegen.cpp b/clang/test/OpenMP/teams_distribute_parallel_for_codegen.cpp
index e347770b84..54dfac4922 100644
--- a/clang/test/OpenMP/teams_distribute_parallel_for_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_parallel_for_codegen.cpp
@@ -3,19 +3,19 @@
 #ifndef HEADER
 #define HEADER
 // Test host codegen.
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
 
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 #ifdef CK1
 
 int a[100];
@@ -51,19 +51,19 @@ int teams_argument_global(int n){
 #endif // CK1
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
 
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 #ifdef CK2
 
 int teams_local_arg(void) {
@@ -84,19 +84,19 @@ int teams_local_arg(void) {
 #endif // CK2
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK3 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK17
+// RUN: %clang_cc1 -DCK3 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK17
 // RUN: %clang_cc1 -DCK3 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK3 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK17
-// RUN: %clang_cc1 -DCK3 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK19
+// RUN: %clang_cc1 -DCK3 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK17
+// RUN: %clang_cc1 -DCK3 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK19
 // RUN: %clang_cc1 -DCK3 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK3 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK19
+// RUN: %clang_cc1 -DCK3 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK19
 
-// RUN: %clang_cc1 -DCK3 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK3 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK3 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK3 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -DCK3 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK3 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK3 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK3 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK3 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK3 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 #ifdef CK3
 
 
@@ -127,19 +127,19 @@ int teams_template_struct(void) {
 #endif // CK3
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK25
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK25
 // RUN: %clang_cc1 -DCK4 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK4 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK25
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK27
+// RUN: %clang_cc1 -DCK4 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK25
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK27
 // RUN: %clang_cc1 -DCK4 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK4 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK27
+// RUN: %clang_cc1 -DCK4 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK27
 
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK4 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK4 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK4 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK4 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK4 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK4 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 
 #ifdef CK4
 
diff --git a/clang/test/OpenMP/teams_distribute_parallel_for_collapse_codegen.cpp b/clang/test/OpenMP/teams_distribute_parallel_for_collapse_codegen.cpp
index 86ec6b7952..e909bccab2 100644
--- a/clang/test/OpenMP/teams_distribute_parallel_for_collapse_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_parallel_for_collapse_codegen.cpp
@@ -4,19 +4,19 @@
 #define HEADER
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
 
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 #ifdef CK1
 
 template <typename T, int X, long long Y>
@@ -48,19 +48,19 @@ int teams_template_struct(void) {
 #endif // CK1
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
 
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 #ifdef CK2
 
 template <typename T, int n, int m>
diff --git a/clang/test/OpenMP/teams_distribute_parallel_for_dist_schedule_codegen.cpp b/clang/test/OpenMP/teams_distribute_parallel_for_dist_schedule_codegen.cpp
index 4fca9c52da..f446746aef 100644
--- a/clang/test/OpenMP/teams_distribute_parallel_for_dist_schedule_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_parallel_for_dist_schedule_codegen.cpp
@@ -4,19 +4,19 @@
 #define HEADER
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
 
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 #ifdef CK1
 
 template <typename T, int X, long long Y>
@@ -62,19 +62,19 @@ int teams_template_struct(void) {
 #endif // CK1
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
 
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 #ifdef CK2
 
 template <typename T, int n>
diff --git a/clang/test/OpenMP/teams_distribute_parallel_for_schedule_codegen.cpp b/clang/test/OpenMP/teams_distribute_parallel_for_schedule_codegen.cpp
index 412d94cb4b..76a451ba8e 100644
--- a/clang/test/OpenMP/teams_distribute_parallel_for_schedule_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_parallel_for_schedule_codegen.cpp
@@ -4,26 +4,26 @@
 #define HEADER
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
 
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK5
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK5
 // RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK5
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK7
+// RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK5
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK7
 // RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK7
+// RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK7
 
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 #ifdef CK1
 
 template <typename T, int X, long long Y>
@@ -87,26 +87,26 @@ int teams_template_struct(void) {
 #endif // CK1
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK13
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK13
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK13
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK15
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK13
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK15
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK15
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK15
 
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK17
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK17
 // RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK17
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK19
+// RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK17
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK19
 // RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK19
+// RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK19
 
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 #ifdef CK2
 
 template <typename T, int n>
diff --git a/clang/test/OpenMP/teams_distribute_parallel_for_simd_codegen.cpp b/clang/test/OpenMP/teams_distribute_parallel_for_simd_codegen.cpp
index 70d82636c5..f35f650792 100644
--- a/clang/test/OpenMP/teams_distribute_parallel_for_simd_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_parallel_for_simd_codegen.cpp
@@ -3,19 +3,19 @@
 #ifndef HEADER
 #define HEADER
 // Test host codegen.
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
 
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK5
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK5
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK5
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK7
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK5
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK7
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK7
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK7
 #ifdef CK1
 
 int a[100];
@@ -52,19 +52,19 @@ int teams_argument_global(int n){
 #endif // CK1
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
 
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK13
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK13
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK13
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK15
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK13
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK15
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK15
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK15
 #ifdef CK2
 
 int teams_local_arg(void) {
@@ -87,19 +87,19 @@ int teams_local_arg(void) {
 #endif // CK2
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK3 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK17
+// RUN: %clang_cc1 -DCK3 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK17
 // RUN: %clang_cc1 -DCK3 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK3 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK17
-// RUN: %clang_cc1 -DCK3 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK19
+// RUN: %clang_cc1 -DCK3 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK17
+// RUN: %clang_cc1 -DCK3 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK19
 // RUN: %clang_cc1 -DCK3 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK3 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK19
+// RUN: %clang_cc1 -DCK3 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK19
 
-// RUN: %clang_cc1 -DCK3 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK21
+// RUN: %clang_cc1 -DCK3 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK21
 // RUN: %clang_cc1 -DCK3 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK3 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK21
-// RUN: %clang_cc1 -DCK3 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK23
+// RUN: %clang_cc1 -DCK3 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK21
+// RUN: %clang_cc1 -DCK3 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK23
 // RUN: %clang_cc1 -DCK3 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK3 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK23
+// RUN: %clang_cc1 -DCK3 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK23
 #ifdef CK3
 
 
@@ -131,19 +131,19 @@ int teams_template_struct(void) {
 #endif // CK3
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK25
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK25
 // RUN: %clang_cc1 -DCK4 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK4 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK25
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK27
+// RUN: %clang_cc1 -DCK4 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK25
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK27
 // RUN: %clang_cc1 -DCK4 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK4 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK27
+// RUN: %clang_cc1 -DCK4 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK27
 
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK29
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK29
 // RUN: %clang_cc1 -DCK4 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK4 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK29
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK31
+// RUN: %clang_cc1 -DCK4 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK29
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK31
 // RUN: %clang_cc1 -DCK4 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK4 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK31
+// RUN: %clang_cc1 -DCK4 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK31
 
 #ifdef CK4
 
diff --git a/clang/test/OpenMP/teams_distribute_parallel_for_simd_collapse_codegen.cpp b/clang/test/OpenMP/teams_distribute_parallel_for_simd_collapse_codegen.cpp
index 1f72e10651..518a6dc6d8 100644
--- a/clang/test/OpenMP/teams_distribute_parallel_for_simd_collapse_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_parallel_for_simd_collapse_codegen.cpp
@@ -4,19 +4,19 @@
 #define HEADER
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
 
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK5
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK5
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK5
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK7
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK5
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK7
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK7
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK7
 #ifdef CK1
 
 template <typename T, int X, long long Y>
@@ -51,19 +51,19 @@ int teams_template_struct(void) {
 #endif // CK1
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
 
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK13
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK13
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK13
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK15
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK13
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK15
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK15
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK15
 #ifdef CK2
 
 template <typename T, int n, int m>
diff --git a/clang/test/OpenMP/teams_distribute_parallel_for_simd_dist_schedule_codegen.cpp b/clang/test/OpenMP/teams_distribute_parallel_for_simd_dist_schedule_codegen.cpp
index e5c5a11d48..ab483ce94b 100644
--- a/clang/test/OpenMP/teams_distribute_parallel_for_simd_dist_schedule_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_parallel_for_simd_dist_schedule_codegen.cpp
@@ -4,19 +4,19 @@
 #define HEADER
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
 
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK5
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK5
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK5
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK7
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK5
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK7
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK7
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK7
 #ifdef CK1
 
 template <typename T, int X, long long Y>
@@ -64,19 +64,19 @@ int teams_template_struct(void) {
 #endif // CK1
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
 
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK13
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK13
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK13
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK15
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK13
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK15
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK15
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK15
 #ifdef CK2
 
 template <typename T, int n>
diff --git a/clang/test/OpenMP/teams_distribute_parallel_for_simd_schedule_codegen.cpp b/clang/test/OpenMP/teams_distribute_parallel_for_simd_schedule_codegen.cpp
index 2f05daa171..8f49b4b423 100644
--- a/clang/test/OpenMP/teams_distribute_parallel_for_simd_schedule_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_parallel_for_simd_schedule_codegen.cpp
@@ -4,27 +4,27 @@
 #define HEADER
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK2
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK2
 // RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
 
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK2
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK5
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK6
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK2
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK5
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK6
 
 // RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK5
+// RUN: %clang_cc1 -DCK1 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK5
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK6
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK6
 
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
 #ifdef CK1
 
 template <typename T, int X, long long Y>
@@ -89,30 +89,30 @@ int teams_template_struct(void) {
 #endif // CK1
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK13
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK13
 
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK14
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK14
 
 // RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK13
+// RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK13
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK14
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK14
 
 
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK17
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK17
 // RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK17
+// RUN: %clang_cc1 -DCK2 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK17
 
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK19
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK19
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK19
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK19
 
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK21
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK21
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK21
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK23
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK21
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK23
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK23
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK23
 #ifdef CK2
 
 template <typename T, int n>
diff --git a/clang/test/OpenMP/teams_distribute_simd_codegen.cpp b/clang/test/OpenMP/teams_distribute_simd_codegen.cpp
index ee84b490c6..b2ce6a13a3 100644
--- a/clang/test/OpenMP/teams_distribute_simd_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_simd_codegen.cpp
@@ -3,19 +3,19 @@
 #ifndef HEADER
 #define HEADER
 // Test host codegen.
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
 
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK5
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK5
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK5
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK7
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK5
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK7
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK7
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK7
 #ifdef CK1
 
 int a[100];
@@ -53,19 +53,19 @@ int teams_argument_global(int n) {
 #endif // CK1
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
 
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK13
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK13
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK13
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK15
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK13
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK15
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK15
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK15
 #ifdef CK2
 
 int teams_local_arg(void) {
@@ -86,31 +86,31 @@ int teams_local_arg(void) {
 #endif // CK2
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK3 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK17
+// RUN: %clang_cc1 -DCK3 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK17
 // RUN: %clang_cc1 -DCK3 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK3 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK17
-// RUN: %clang_cc1 -DCK3 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK19
+// RUN: %clang_cc1 -DCK3 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK17
+// RUN: %clang_cc1 -DCK3 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK19
 // RUN: %clang_cc1 -DCK3 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK3 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK19
-// RUN: %clang_cc1 -DCK3 -verify -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK21
+// RUN: %clang_cc1 -DCK3 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK19
+// RUN: %clang_cc1 -DCK3 -verify -Wno-vla -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK21
 // RUN: %clang_cc1 -DCK3 -fopenmp -DOMP5 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK3 -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK21
-// RUN: %clang_cc1 -DCK3 -verify -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK23
+// RUN: %clang_cc1 -DCK3 -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK21
+// RUN: %clang_cc1 -DCK3 -verify -Wno-vla -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK23
 // RUN: %clang_cc1 -DCK3 -fopenmp -DOMP5 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK3 -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK23
+// RUN: %clang_cc1 -DCK3 -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK23
 
-// RUN: %clang_cc1 -DCK3 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK25
+// RUN: %clang_cc1 -DCK3 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK25
 // RUN: %clang_cc1 -DCK3 -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK3 -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK25
-// RUN: %clang_cc1 -DCK3 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK27
+// RUN: %clang_cc1 -DCK3 -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK25
+// RUN: %clang_cc1 -DCK3 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK27
 // RUN: %clang_cc1 -DCK3 -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK3 -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK27
-// RUN: %clang_cc1 -DCK3 -verify -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK29
+// RUN: %clang_cc1 -DCK3 -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK27
+// RUN: %clang_cc1 -DCK3 -verify -Wno-vla -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK29
 // RUN: %clang_cc1 -DCK3 -fopenmp-simd -DOMP5 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK3 -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK29
-// RUN: %clang_cc1 -DCK3 -verify -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK31
+// RUN: %clang_cc1 -DCK3 -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK29
+// RUN: %clang_cc1 -DCK3 -verify -Wno-vla -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK31
 // RUN: %clang_cc1 -DCK3 -fopenmp-simd -DOMP5 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK3 -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK31
+// RUN: %clang_cc1 -DCK3 -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK31
 #ifdef CK3
 
 
@@ -145,31 +145,31 @@ int teams_template_struct(void) {
 #endif // CK3
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK33
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK33
 // RUN: %clang_cc1 -DCK4 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK4 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK33
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK35
+// RUN: %clang_cc1 -DCK4 -fopenmp -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK33
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK35
 // RUN: %clang_cc1 -DCK4 -fopenmp -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK4 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK35
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK37
+// RUN: %clang_cc1 -DCK4 -fopenmp -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK35
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK37
 // RUN: %clang_cc1 -DCK4 -fopenmp -DOMP5 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK4 -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK37
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK39
+// RUN: %clang_cc1 -DCK4 -fopenmp -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK37
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK39
 // RUN: %clang_cc1 -DCK4 -fopenmp -DOMP5 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK4 -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK39
+// RUN: %clang_cc1 -DCK4 -fopenmp -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK39
 
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK41
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK41
 // RUN: %clang_cc1 -DCK4 -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK4 -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK41
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK43
+// RUN: %clang_cc1 -DCK4 -fopenmp-simd -fopenmp-version=45 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK41
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK43
 // RUN: %clang_cc1 -DCK4 -fopenmp-simd -fopenmp-version=45 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK4 -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK43
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK45
+// RUN: %clang_cc1 -DCK4 -fopenmp-simd -fopenmp-version=45 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK43
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK45
 // RUN: %clang_cc1 -DCK4 -fopenmp-simd -DOMP5 -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK4 -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK45
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK47
+// RUN: %clang_cc1 -DCK4 -fopenmp-simd -DOMP5 -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK45
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK47
 // RUN: %clang_cc1 -DCK4 -fopenmp-simd -DOMP5 -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK4 -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK47
+// RUN: %clang_cc1 -DCK4 -fopenmp-simd -DOMP5 -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK47
 
 #ifdef CK4
 
diff --git a/clang/test/OpenMP/teams_distribute_simd_collapse_codegen.cpp b/clang/test/OpenMP/teams_distribute_simd_collapse_codegen.cpp
index 60e2494096..9562d2d973 100644
--- a/clang/test/OpenMP/teams_distribute_simd_collapse_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_simd_collapse_codegen.cpp
@@ -4,19 +4,19 @@
 #define HEADER
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
 
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK5
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK5
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK5
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK7
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK5
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK7
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK7
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK7
 #ifdef CK1
 
 template <typename T, int X, long long Y>
@@ -48,19 +48,19 @@ int teams_template_struct(void) {
 #endif // CK1
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
 
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK13
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK13
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK13
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK15
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK13
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK15
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK15
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK15
 #ifdef CK2
 
 template <typename T, int n, int m>
diff --git a/clang/test/OpenMP/teams_distribute_simd_dist_schedule_codegen.cpp b/clang/test/OpenMP/teams_distribute_simd_dist_schedule_codegen.cpp
index c3d67664cc..0076e442d2 100644
--- a/clang/test/OpenMP/teams_distribute_simd_dist_schedule_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_simd_dist_schedule_codegen.cpp
@@ -4,19 +4,19 @@
 #define HEADER
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
 
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK5
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK5
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK5
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK7
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK5
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK7
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK7
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK7
 #ifdef CK1
 
 template <typename T, int X, long long Y>
@@ -58,19 +58,19 @@ int teams_template_struct(void) {
 #endif // CK1
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
 
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK13
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK13
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK13
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK15
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK13
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK15
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK15
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK15
 #ifdef CK2
 
 template <typename T, int n>
diff --git a/clang/test/OpenMP/teams_firstprivate_codegen.cpp b/clang/test/OpenMP/teams_firstprivate_codegen.cpp
index d99151b036..dbdd5f912b 100644
--- a/clang/test/OpenMP/teams_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/teams_firstprivate_codegen.cpp
@@ -1,46 +1,46 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --include-generated-funcs --replace-value-regex "__omp_offloading_[0-9a-z]+_[0-9a-z]+" "reduction_size[.].+[.]" "pl_cond[.].+[.|,]" --prefix-filecheck-ir-name _
 // Test host codegen.
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DLAMBDA -verify -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - -Wno-openmp-mapping | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DLAMBDA -verify -Wno-vla -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - -Wno-openmp-mapping | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DLAMBDA -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s -Wno-openmp-mapping
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DLAMBDA -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - -Wno-openmp-mapping | FileCheck %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DLAMBDA -verify -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - -Wno-openmp-mapping | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DLAMBDA -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - -Wno-openmp-mapping | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DLAMBDA -verify -Wno-vla -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - -Wno-openmp-mapping | FileCheck %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DLAMBDA -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s -Wno-openmp-mapping
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DLAMBDA -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - -Wno-openmp-mapping | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DLAMBDA -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - -Wno-openmp-mapping | FileCheck %s --check-prefix=CHECK3
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DLAMBDA -verify -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - -Wno-openmp-mapping | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DLAMBDA -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - -Wno-openmp-mapping | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DLAMBDA -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s -Wno-openmp-mapping
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DLAMBDA -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - -Wno-openmp-mapping | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DLAMBDA -verify -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - -Wno-openmp-mapping | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DLAMBDA -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - -Wno-openmp-mapping | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DLAMBDA -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - -Wno-openmp-mapping | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DLAMBDA -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s -Wno-openmp-mapping
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DLAMBDA -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - -Wno-openmp-mapping | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DLAMBDA -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - -Wno-openmp-mapping | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis  -verify -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - -Wno-openmp-mapping | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -no-enable-noundef-analysis  -verify -Wno-vla -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - -Wno-openmp-mapping | FileCheck %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -no-enable-noundef-analysis  -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s -Wno-openmp-mapping
-// RUN: %clang_cc1 -no-enable-noundef-analysis  -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - -Wno-openmp-mapping | FileCheck %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -no-enable-noundef-analysis  -verify -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - -Wno-openmp-mapping | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -no-enable-noundef-analysis  -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - -Wno-openmp-mapping | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -no-enable-noundef-analysis  -verify -Wno-vla -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - -Wno-openmp-mapping | FileCheck %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -no-enable-noundef-analysis  -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s -Wno-openmp-mapping
-// RUN: %clang_cc1 -no-enable-noundef-analysis  -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - -Wno-openmp-mapping | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -no-enable-noundef-analysis  -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - -Wno-openmp-mapping | FileCheck %s --check-prefix=CHECK11
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis  -verify -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - -Wno-openmp-mapping | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -no-enable-noundef-analysis  -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - -Wno-openmp-mapping | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -no-enable-noundef-analysis  -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s -Wno-openmp-mapping
-// RUN: %clang_cc1 -no-enable-noundef-analysis  -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - -Wno-openmp-mapping | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -no-enable-noundef-analysis  -verify -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - -Wno-openmp-mapping | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -no-enable-noundef-analysis  -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - -Wno-openmp-mapping | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -no-enable-noundef-analysis  -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - -Wno-openmp-mapping | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -no-enable-noundef-analysis  -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s -Wno-openmp-mapping
-// RUN: %clang_cc1 -no-enable-noundef-analysis  -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - -Wno-openmp-mapping | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -no-enable-noundef-analysis  -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - -Wno-openmp-mapping | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DARRAY  -verify -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - -Wno-openmp-mapping | FileCheck %s --check-prefix=CHECK17
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DARRAY  -verify -Wno-vla -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - -Wno-openmp-mapping | FileCheck %s --check-prefix=CHECK17
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DARRAY  -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s -Wno-openmp-mapping
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DARRAY  -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - -Wno-openmp-mapping | FileCheck %s --check-prefix=CHECK17
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DARRAY  -verify -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - -Wno-openmp-mapping | FileCheck %s --check-prefix=CHECK19
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DARRAY  -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - -Wno-openmp-mapping | FileCheck %s --check-prefix=CHECK17
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DARRAY  -verify -Wno-vla -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - -Wno-openmp-mapping | FileCheck %s --check-prefix=CHECK19
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DARRAY  -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s -Wno-openmp-mapping
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DARRAY  -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -include-pch %t -verify %s -emit-llvm -o - -Wno-openmp-mapping | FileCheck %s --check-prefix=CHECK19
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DARRAY  -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -include-pch %t -verify -Wno-vla %s -emit-llvm -o - -Wno-openmp-mapping | FileCheck %s --check-prefix=CHECK19
 
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DARRAY  -verify -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - -Wno-openmp-mapping | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DARRAY  -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - -Wno-openmp-mapping | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DARRAY  -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s -Wno-openmp-mapping
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DARRAY  -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - -Wno-openmp-mapping | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DARRAY  -verify -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - -Wno-openmp-mapping | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DARRAY  -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - -Wno-openmp-mapping | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DARRAY  -verify -Wno-vla -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - -Wno-openmp-mapping | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -no-enable-noundef-analysis -DARRAY  -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s -Wno-openmp-mapping
-// RUN: %clang_cc1 -no-enable-noundef-analysis -DARRAY  -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -include-pch %t -verify %s -emit-llvm -o - -Wno-openmp-mapping | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -no-enable-noundef-analysis -DARRAY  -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -include-pch %t -verify -Wno-vla %s -emit-llvm -o - -Wno-openmp-mapping | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // expected-no-diagnostics
 #ifndef HEADER
 #define HEADER
diff --git a/clang/test/OpenMP/teams_generic_loop_codegen-1.cpp b/clang/test/OpenMP/teams_generic_loop_codegen-1.cpp
index a0a75f244f..d0ae405306 100644
--- a/clang/test/OpenMP/teams_generic_loop_codegen-1.cpp
+++ b/clang/test/OpenMP/teams_generic_loop_codegen-1.cpp
@@ -3,19 +3,19 @@
 #ifndef HEADER
 #define HEADER
 // Test host codegen.
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK1
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK1
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK3
 // RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
+// RUN: %clang_cc1 -DCK1 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK3
 
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -DCK1 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK1 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 #ifdef CK1
 
 int a[100];
@@ -50,19 +50,19 @@ int teams_argument_global(int n){
 #endif // CK1
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK9
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK9
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK11
 // RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
+// RUN: %clang_cc1 -DCK2 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK11
 
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -DCK2 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK2 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 #ifdef CK2
 
 int teams_local_arg(void) {
@@ -83,19 +83,19 @@ int teams_local_arg(void) {
 #endif // CK2
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK3 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK17
+// RUN: %clang_cc1 -DCK3 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK17
 // RUN: %clang_cc1 -DCK3 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK3 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK17
-// RUN: %clang_cc1 -DCK3 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK19
+// RUN: %clang_cc1 -DCK3 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK17
+// RUN: %clang_cc1 -DCK3 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK19
 // RUN: %clang_cc1 -DCK3 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK3 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK19
+// RUN: %clang_cc1 -DCK3 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK19
 
-// RUN: %clang_cc1 -DCK3 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK3 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK3 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK3 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -DCK3 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK3 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK3 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK3 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK3 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK3 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 #ifdef CK3
 
 
@@ -126,19 +126,19 @@ int teams_template_struct(void) {
 #endif // CK3
 
 // Test host codegen.
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK25
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK25
 // RUN: %clang_cc1 -DCK4 -fopenmp -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK4 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK25
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK27
+// RUN: %clang_cc1 -DCK4 -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK25
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK27
 // RUN: %clang_cc1 -DCK4 -fopenmp -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK4 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK27
+// RUN: %clang_cc1 -DCK4 -fopenmp -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --check-prefix=CHECK27
 
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK4 -fopenmp-simd -x c++ -std=c++11 -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK4 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
-// RUN: %clang_cc1 -DCK4 -verify -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK4 -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK4 -verify -Wno-vla -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-llvm %s -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 // RUN: %clang_cc1 -DCK4 -fopenmp-simd -x c++ -std=c++11 -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -emit-pch -o %t %s
-// RUN: %clang_cc1 -DCK4 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
+// RUN: %clang_cc1 -DCK4 -fopenmp-simd -x c++ -triple i386-unknown-unknown -fopenmp-targets=i386-pc-linux-gnu -std=c++11 -include-pch %t -verify -Wno-vla %s -emit-llvm -o - | FileCheck %s --implicit-check-not="{{__kmpc|__tgt}}"
 
 #ifdef CK4
 
diff --git a/clang/test/Parser/objcxx11-attributes.mm b/clang/test/Parser/objcxx11-attributes.mm
index 3c14d03cb2..d7ba609ebd 100644
--- a/clang/test/Parser/objcxx11-attributes.mm
+++ b/clang/test/Parser/objcxx11-attributes.mm
@@ -10,12 +10,12 @@ - (X*) getSelf;
 
 void f(X *noreturn) {
   // An array size which is computed by a message send is OK.
-  int a[ [noreturn getSize] ];
+  int a[ [noreturn getSize] ]; // expected-warning {{variable length arrays in C++ are a Clang extension}}
 
   // ... but is interpreted as an attribute where possible.
   int b[ [noreturn] ]; // expected-error {{'noreturn' attribute only applies to functions}}
 
-  int c[ [noreturn getSize] + 1 ];
+  int c[ [noreturn getSize] + 1 ]; // expected-warning {{variable length arrays in C++ are a Clang extension}}
 
   // An array size which is computed by a lambda is not OK.
   int d[ [noreturn] { return 3; } () ]; // expected-error {{expected ']'}} expected-error {{'noreturn' attribute only applies to functions}}
diff --git a/clang/test/Parser/pragma-loop.cpp b/clang/test/Parser/pragma-loop.cpp
index 0203e90104..4078210f96 100644
--- a/clang/test/Parser/pragma-loop.cpp
+++ b/clang/test/Parser/pragma-loop.cpp
@@ -82,7 +82,7 @@ void test_type_template_vectorize(int *List, int Length) {
   }
 }
 
-void test(int *List, int Length) {
+void test(int *List, int Length) { // expected-note {{declared here}}
   int i = 0;
 
 #pragma clang loop vectorize(enable)
@@ -118,7 +118,8 @@ void test(int *List, int Length) {
     List[i] = i;
   }
 
-  int VList[Length];
+  int VList[Length]; // expected-warning {{variable length arrays in C++ are a Clang extension}} \
+                        expected-note {{function parameter 'Length' with unknown value cannot be used in a constant expression}}
 #pragma clang loop vectorize(disable) interleave(disable) unroll(disable) vectorize_predicate(disable)
   for (int j : VList) {
     VList[j] = List[j];
diff --git a/clang/test/Preprocessor/init-mips.c b/clang/test/Preprocessor/init-mips.c
index a07cee64e6..34091ea369 100644
--- a/clang/test/Preprocessor/init-mips.c
+++ b/clang/test/Preprocessor/init-mips.c
@@ -10,6 +10,7 @@
 // MIPS32BE:#define _MIPS_ARCH_MIPS32R2 1
 // MIPS32BE:#define _MIPS_FPSET 16
 // MIPS32BE:#define _MIPS_SIM _ABIO32
+// MIPS32BE:#define _MIPS_SPFPSET 16
 // MIPS32BE:#define _MIPS_SZINT 32
 // MIPS32BE:#define _MIPS_SZLONG 32
 // MIPS32BE:#define _MIPS_SZPTR 32
@@ -220,6 +221,7 @@
 // MIPS32EL:#define _MIPS_ARCH_MIPS32R2 1
 // MIPS32EL:#define _MIPS_FPSET 16
 // MIPS32EL:#define _MIPS_SIM _ABIO32
+// MIPS32EL:#define _MIPS_SPFPSET 16
 // MIPS32EL:#define _MIPS_SZINT 32
 // MIPS32EL:#define _MIPS_SZLONG 32
 // MIPS32EL:#define _MIPS_SZPTR 32
@@ -432,6 +434,7 @@
 // MIPSN32BE: #define _MIPS_FPSET 32
 // MIPSN32BE: #define _MIPS_ISA _MIPS_ISA_MIPS64
 // MIPSN32BE: #define _MIPS_SIM _ABIN32
+// MIPSN32BE: #define _MIPS_SPFPSET 32
 // MIPSN32BE: #define _MIPS_SZINT 32
 // MIPSN32BE: #define _MIPS_SZLONG 32
 // MIPSN32BE: #define _MIPS_SZPTR 32
@@ -739,6 +742,7 @@
 // MIPSN32EL: #define _MIPS_FPSET 32
 // MIPSN32EL: #define _MIPS_ISA _MIPS_ISA_MIPS64
 // MIPSN32EL: #define _MIPS_SIM _ABIN32
+// MIPSN32EL: #define _MIPS_SPFPSET 32
 // MIPSN32EL: #define _MIPS_SZINT 32
 // MIPSN32EL: #define _MIPS_SZLONG 32
 // MIPSN32EL: #define _MIPS_SZPTR 32
@@ -1043,6 +1047,7 @@
 // MIPS64BE:#define _MIPS_ARCH_MIPS64R2 1
 // MIPS64BE:#define _MIPS_FPSET 32
 // MIPS64BE:#define _MIPS_SIM _ABI64
+// MIPS64BE:#define _MIPS_SPFPSET 32
 // MIPS64BE:#define _MIPS_SZINT 32
 // MIPS64BE:#define _MIPS_SZLONG 64
 // MIPS64BE:#define _MIPS_SZPTR 64
@@ -1253,6 +1258,7 @@
 // MIPS64EL:#define _MIPS_ARCH_MIPS64R2 1
 // MIPS64EL:#define _MIPS_FPSET 32
 // MIPS64EL:#define _MIPS_SIM _ABI64
+// MIPS64EL:#define _MIPS_SPFPSET 32
 // MIPS64EL:#define _MIPS_SZINT 32
 // MIPS64EL:#define _MIPS_SZLONG 64
 // MIPS64EL:#define _MIPS_SZPTR 64
@@ -1506,6 +1512,7 @@
 // MIPS-ARCH-32R6:#define _MIPS_ARCH "mips32r6"
 // MIPS-ARCH-32R6:#define _MIPS_ARCH_MIPS32R6 1
 // MIPS-ARCH-32R6:#define _MIPS_ISA _MIPS_ISA_MIPS32
+// MIPS-ARCH-32R6:#define __mips_fpr 64
 // MIPS-ARCH-32R6:#define __mips_isa_rev 6
 
 // RUN: %clang_cc1 -E -dM -ffreestanding -triple=mips64-none-none \
@@ -1682,14 +1689,28 @@
 // RUN:   | FileCheck -match-full-lines -check-prefix MIPS32-NOFP %s
 // MIPS32-NOFP:#define __mips_fpr 0
 
+// RUN: %clang_cc1  -target-feature -nooddspreg \
+// RUN:   -E -dM -triple=mips-none-none < /dev/null \
+// RUN:   | FileCheck -match-full-lines -check-prefix MIPS32-NOFP-ODD %s
+// MIPS32-NOFP-ODD:#define _MIPS_SPFPSET 32
+// MIPS32-NOFP-ODD:#define __mips_fpr 0
+
+// RUN: %clang_cc1  -target-feature +nooddspreg -target-feature -fp64 \
+// RUN:   -E -dM -triple=mips-none-none < /dev/null \
+// RUN:   | FileCheck -match-full-lines -check-prefix MIPS32-FP64-NOODD %s
+// MIPS32-FP64-NOODD:#define _MIPS_SPFPSET 16
+// MIPS32-FP64-NOODD:#define __mips_fpr 32
+
 // RUN: %clang_cc1 -target-feature +fpxx \
 // RUN:   -E -dM -triple=mips-none-none < /dev/null \
 // RUN:   | FileCheck -match-full-lines -check-prefix MIPS32-MFPXX %s
+// MIPS32-MFPXX:#define _MIPS_SPFPSET 16
 // MIPS32-MFPXX:#define __mips_fpr 0
 
 // RUN: %clang_cc1 -target-cpu mips32r6 -target-feature +fpxx \
 // RUN:   -E -dM -triple=mips-none-none < /dev/null \
 // RUN:   | FileCheck -match-full-lines -check-prefix MIPS32R6-MFPXX %s
+// MIPS32R6-MFPXX:#define _MIPS_SPFPSET 16
 // MIPS32R6-MFPXX:#define __mips_fpr 0
 
 // RUN: %clang_cc1  \
@@ -1716,30 +1737,35 @@
 // RUN:   -E -dM -triple=mips-none-none < /dev/null \
 // RUN:   | FileCheck -match-full-lines -check-prefix MIPS32-MFP32 %s
 // MIPS32-MFP32:#define _MIPS_FPSET 16
+// MIPS32-MFP32:#define _MIPS_SPFPSET 32
 // MIPS32-MFP32:#define __mips_fpr 32
 
 // RUN: %clang_cc1 -target-feature +fp64 \
 // RUN:   -E -dM -triple=mips-none-none < /dev/null \
 // RUN:   | FileCheck -match-full-lines -check-prefix MIPS32-MFP64 %s
 // MIPS32-MFP64:#define _MIPS_FPSET 32
+// MIPS32-MFP64:#define _MIPS_SPFPSET 32
 // MIPS32-MFP64:#define __mips_fpr 64
 //
 // RUN: %clang_cc1 -target-feature +single-float \
 // RUN:   -E -dM -triple=mips-none-none < /dev/null \
 // RUN:   | FileCheck -match-full-lines -check-prefix MIPS32-MFP32SF %s
 // MIPS32-MFP32SF:#define _MIPS_FPSET 32
+// MIPS32-MFP32SF:#define _MIPS_SPFPSET 16
 // MIPS32-MFP32SF:#define __mips_fpr 0
 
 // RUN: %clang_cc1 -target-feature +fp64 \
 // RUN:   -E -dM -triple=mips64-none-none < /dev/null \
 // RUN:   | FileCheck -match-full-lines -check-prefix MIPS64-MFP64 %s
 // MIPS64-MFP64:#define _MIPS_FPSET 32
+// MIPS64-MFP64:#define _MIPS_SPFPSET 32
 // MIPS64-MFP64:#define __mips_fpr 64
 
 // RUN: %clang_cc1 -target-feature -fp64 -target-feature +single-float \
 // RUN:   -E -dM -triple=mips64-none-none < /dev/null \
 // RUN:   | FileCheck -match-full-lines -check-prefix MIPS64-NOMFP64 %s
 // MIPS64-NOMFP64:#define _MIPS_FPSET 32
+// MIPS64-NOMFP64:#define _MIPS_SPFPSET 32
 // MIPS64-NOMFP64:#define __mips_fpr 32
 
 // RUN: %clang_cc1 -target-cpu mips32r6 \
@@ -1749,6 +1775,7 @@
 // RUN:   -E -dM -triple=mips64-none-none < /dev/null \
 // RUN:   | FileCheck -match-full-lines -check-prefix MIPS-XXR6 %s
 // MIPS-XXR6:#define _MIPS_FPSET 32
+// MIPS-XXR6:#define _MIPS_SPFPSET 32
 // MIPS-XXR6:#define __mips_fpr 64
 // MIPS-XXR6:#define __mips_nan2008 1
 
diff --git a/clang/test/Preprocessor/predefined-arch-macros.c b/clang/test/Preprocessor/predefined-arch-macros.c
index d95992dcdf..f10793983b 100644
--- a/clang/test/Preprocessor/predefined-arch-macros.c
+++ b/clang/test/Preprocessor/predefined-arch-macros.c
@@ -2503,6 +2503,12 @@
 // RUN: %clang -march=lunarlake -m32 -E -dM %s -o - 2>&1 \
 // RUN:     -target i386-unknown-linux \
 // RUN:   | FileCheck -match-full-lines %s -check-prefixes=CHECK_SRF_M32,CHECK_ARLS_M32
+// RUN: %clang -march=pantherlake -m32 -E -dM %s -o - 2>&1 \
+// RUN:     -target i386-unknown-linux \
+// RUN:   | FileCheck -match-full-lines %s -check-prefixes=CHECK_SRF_M32,CHECK_ARLS_M32,CHECK_PTL_M32
+// RUN: %clang -march=clearwaterforest -m32 -E -dM %s -o - 2>&1 \
+// RUN:     -target i386-unknown-linux \
+// RUN:   | FileCheck -match-full-lines %s -check-prefixes=CHECK_SRF_M32,CHECK_ARLS_M32,CHECK_PTL_M32,CHECK_CWF_M32
 // CHECK_SRF_M32: #define __ADX__ 1
 // CHECK_SRF_M32: #define __AES__ 1
 // CHECK_SRF_M32: #define __AVX2__ 1
@@ -2538,6 +2544,9 @@
 // CHECK_SRF_M32: #define __PCONFIG__ 1
 // CHECK_SRF_M32: #define __PKU__ 1
 // CHECK_SRF_M32: #define __POPCNT__ 1
+// CHECK_SRF_M32-NOT: #define __PREFETCHI__ 1
+// CHECK_ARLS_M32-NOT: #define __PREFETCHI__ 1
+// CHECK_PTL_M32: #define __PREFETCHI__ 1
 // CHECK_SRF_M32: #define __PRFCHW__ 1
 // CHECK_SRF_M32: #define __PTWRITE__ 1
 // CHECK_SRF_M32-NOT: #define __RAOINT__ 1
@@ -2563,6 +2572,10 @@
 // CHECK_SRF_M32: #define __SSE__ 1
 // CHECK_SRF_M32: #define __SSSE3__ 1
 // CHECK_SRF_M32: #define __UINTR__ 1
+// CHECK_SRF_M32-NOT: #define __USERMSR__ 1
+// CHECK_ARLS_M32-NOT: #define __USERMSR__ 1
+// CHECK_PTL_M32-NOT: #define __USERMSR__ 1
+// CHECK_CWF_M32: #define __USERMSR__ 1
 // CHECK_SRF_M32: #define __VAES__ 1
 // CHECK_SRF_M32: #define __VPCLMULQDQ__ 1
 // CHECK_SRF_M32: #define __WAITPKG__ 1
@@ -2593,6 +2606,12 @@
 // RUN: %clang -march=lunarlake -m64 -E -dM %s -o - 2>&1 \
 // RUN:     -target i386-unknown-linux \
 // RUN:   | FileCheck -match-full-lines %s -check-prefixes=CHECK_SRF_M64,CHECK_ARLS_M64
+// RUN: %clang -march=pantherlake -m64 -E -dM %s -o - 2>&1 \
+// RUN:     -target i386-unknown-linux \
+// RUN:   | FileCheck -match-full-lines %s -check-prefixes=CHECK_SRF_M64,CHECK_ARLS_M64,CHECK_PTL_M64
+// RUN: %clang -march=clearwaterforest -m64 -E -dM %s -o - 2>&1 \
+// RUN:     -target i386-unknown-linux \
+// RUN:   | FileCheck -match-full-lines %s -check-prefixes=CHECK_SRF_M64,CHECK_ARLS_M64,CHECK_PTL_M64,CHECK_CWF_M64
 // CHECK_SRF_M64: #define __ADX__ 1
 // CHECK_SRF_M64: #define __AES__ 1
 // CHECK_SRF_M64: #define __AVX2__ 1
@@ -2628,6 +2647,9 @@
 // CHECK_SRF_M64: #define __PCONFIG__ 1
 // CHECK_SRF_M64: #define __PKU__ 1
 // CHECK_SRF_M64: #define __POPCNT__ 1
+// CHECK_SRF_M64-NOT: #define __PREFETCHI__ 1
+// CHECK_ARLS_M64-NOT: #define __PREFETCHI__ 1
+// CHECK_PTL_M64: #define __PREFETCHI__ 1
 // CHECK_SRF_M64: #define __PRFCHW__ 1
 // CHECK_SRF_M64: #define __PTWRITE__ 1
 // CHECK_SRF_M64-NOT: #define __RAOINT__ 1
@@ -2654,6 +2676,10 @@
 // CHECK_SRF_M64: #define __SSE__ 1
 // CHECK_SRF_M64: #define __SSSE3__ 1
 // CHECK_SRF_M64: #define __UINTR__ 1
+// CHECK_SRF_M64-NOT: #define __USERMSR__ 1
+// CHECK_ARLS_M64-NOT: #define __USERMSR__ 1
+// CHECK_PTL_M64-NOT: #define __USERMSR__ 1
+// CHECK_CWF_M64: #define __USERMSR__ 1
 // CHECK_SRF_M64: #define __VAES__ 1
 // CHECK_SRF_M64: #define __VPCLMULQDQ__ 1
 // CHECK_SRF_M64: #define __WAITPKG__ 1
diff --git a/clang/test/Preprocessor/riscv-target-features.c b/clang/test/Preprocessor/riscv-target-features.c
index 242197e3f1..ffdec34ca6 100644
--- a/clang/test/Preprocessor/riscv-target-features.c
+++ b/clang/test/Preprocessor/riscv-target-features.c
@@ -33,6 +33,8 @@
 // CHECK-NOT: __riscv_xcvsimd {{.*$}}
 // CHECK-NOT: __riscv_xsfcie {{.*$}}
 // CHECK-NOT: __riscv_xsfvcp {{.*$}}
+// CHECK-NOT: __riscv_xsfqmaccdod {{.*$}}
+// CHECK-NOT: __riscv_xsfvqmaccqoq {{.*$}}
 // CHECK-NOT: __riscv_xtheadba {{.*$}}
 // CHECK-NOT: __riscv_xtheadbb {{.*$}}
 // CHECK-NOT: __riscv_xtheadbs {{.*$}}
@@ -323,6 +325,22 @@
 // RUN: -o - | FileCheck --check-prefix=CHECK-XSFVCP-EXT %s
 // CHECK-XSFVCP-EXT: __riscv_xsfvcp 1000000{{$}}
 
+// RUN: %clang --target=riscv32-unknown-linux-gnu \
+// RUN: -march=rv32ixsfvqmaccdod -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-XSFVQMACCDOD-EXT %s
+// RUN: %clang --target=riscv64-unknown-linux-gnu \
+// RUN: -march=rv64ixsfvqmaccdod -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-XSFVQMACCDOD-EXT %s
+// CHECK-XSFVQMACCDOD-EXT: __riscv_xsfvqmaccdod 1000000{{$}}
+
+// RUN: %clang --target=riscv32-unknown-linux-gnu \
+// RUN: -march=rv32ixsfvqmaccqoq -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-XSFVQMACCQOQ-EXT %s
+// RUN: %clang --target=riscv64-unknown-linux-gnu \
+// RUN: -march=rv64ixsfvqmaccqoq -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-XSFVQMACCQOQ-EXT %s
+// CHECK-XSFVQMACCQOQ-EXT: __riscv_xsfvqmaccqoq 1000000{{$}}
+
 // RUN: %clang --target=riscv32-unknown-linux-gnu \
 // RUN: -march=rv32ixtheadba -x c -E -dM %s \
 // RUN: -o - | FileCheck --check-prefix=CHECK-XTHEADBA-EXT %s
diff --git a/clang/test/Preprocessor/x86_target_features.c b/clang/test/Preprocessor/x86_target_features.c
index 36d4af59d4..7c03ff87ee 100644
--- a/clang/test/Preprocessor/x86_target_features.c
+++ b/clang/test/Preprocessor/x86_target_features.c
@@ -750,6 +750,26 @@
 // AVXVNNIINT16NOAVX2-NOT: #define __AVX2__ 1
 // AVXVNNIINT16NOAVX2-NOT: #define __AVXVNNIINT16__ 1
 
+// RUN: %clang -target i686-unknown-linux-gnu -march=atom -mavx10.1 -x c -E -dM -o - %s | FileCheck  -check-prefix=AVX10_1_256 %s
+// RUN: %clang -target i686-unknown-linux-gnu -march=atom -mavx10.1-256 -x c -E -dM -o - %s | FileCheck  -check-prefix=AVX10_1_256 %s
+// RUN: %clang -target i686-unknown-linux-gnu -march=atom -mavx10.1-256 -mno-avx512f -x c -E -dM -o - %s | FileCheck  -check-prefix=AVX10_1_256 %s
+// AVX10_1_256: #define __AVX10_1__ 1
+// AVX10_1_256: #define __AVX512F__ 1
+// AVX10_1_256-NOT: __EVEX512__
+
+// RUN: %clang -target i686-unknown-linux-gnu -march=atom -mavx10.1-512 -x c -E -dM -o - %s | FileCheck  -check-prefix=AVX10_1_512 %s
+// RUN: %clang -target i686-unknown-linux-gnu -march=atom -mavx10.1-512 -mno-avx512f -x c -E -dM -o - %s | FileCheck  -check-prefix=AVX10_1_512 %s
+// RUN: %clang -target i686-unknown-linux-gnu -march=atom -mavx10.1-512 -mno-evex512 -x c -E -dM -o - %s | FileCheck  -check-prefix=AVX10_1_512 %s
+// AVX10_1_512: #define __AVX10_1__ 1
+// AVX10_1_512: #define __AVX512F__ 1
+// AVX10_1_512: #define __EVEX512__ 1
+
+// RUN: %clang -target i686-unknown-linux-gnu -march=atom -musermsr -x c -E -dM -o - %s | FileCheck  -check-prefix=USERMSR %s
+// USERMSR: #define __USERMSR__ 1
+
+// RUN: %clang -target i686-unknown-linux-gnu -march=atom -mno-usermsr -x c -E -dM -o - %s | FileCheck  -check-prefix=NO-USERMSR %s
+// NO-USERMSR-NOT: #define __USERMSR__ 1
+
 // RUN: %clang -target i386-unknown-linux-gnu -march=i386 -mcrc32 -x c -E -dM -o - %s | FileCheck -check-prefix=CRC32 %s
 
 // CRC32: #define __CRC32__ 1
diff --git a/clang/test/Sema/aarch64-sme-intrinsics/acle_sme_imm.cpp b/clang/test/Sema/aarch64-sme-intrinsics/acle_sme_imm.cpp
index 7475fd53b8..1faa5638c8 100644
--- a/clang/test/Sema/aarch64-sme-intrinsics/acle_sme_imm.cpp
+++ b/clang/test/Sema/aarch64-sme-intrinsics/acle_sme_imm.cpp
@@ -143,11 +143,6 @@ void test_range_0_15(uint32_t slice, svbool_t pg, void *ptr) {
   // expected-error@+1 {{argument value 16 is outside the valid range [0, 15]}}
   SVE_ACLE_FUNC(svst1_ver_vnum_za128,,,)(16, slice, pg, ptr, 1);
 
-  // expected-error@+1 {{argument value 16 is outside the valid range [0, 15]}}
-  SVE_ACLE_FUNC(svldr_vnum_za,,,)(-1, ptr, 16);
-  // expected-error@+1 {{argument value 18446744073709551615 is outside the valid range [0, 15]}}
-  SVE_ACLE_FUNC(svstr_vnum_za,,,)(-1, ptr, -1);
-
   // expected-error@+1 {{argument value 18446744073709551615 is outside the valid range [0, 15]}}
   SVE_ACLE_FUNC(svread_hor_za128, _s8, _m,)(svundef_s8(), pg, -1, slice);
   // expected-error@+1 {{argument value 16 is outside the valid range [0, 15]}}
@@ -171,9 +166,6 @@ void test_constant(uint64_t u64, svbool_t pg, void *ptr) {
   SVE_ACLE_FUNC(svld1_hor_vnum_za8,,,)(u64, 0, pg, ptr, u64);  // expected-error {{argument to 'svld1_hor_vnum_za8' must be a constant integer}}
   SVE_ACLE_FUNC(svst1_hor_vnum_za32,,,)(u64, 0, pg, ptr, u64); // expected-error {{argument to 'svst1_hor_vnum_za32' must be a constant integer}}
 
-  SVE_ACLE_FUNC(svldr_vnum_za,,,)(u64, ptr, u64); // expected-error {{argument to 'svldr_vnum_za' must be a constant integer}}
-  SVE_ACLE_FUNC(svstr_vnum_za,,,)(u64, ptr, u64); // expected-error {{argument to 'svstr_vnum_za' must be a constant integer}}
-
   SVE_ACLE_FUNC(svread_ver_za16, _s16, _m,)(svundef_s16(), pg, u64, 0);  // expected-error-re {{argument to 'svread_ver_za16{{.*}}_m' must be a constant integer}}
   SVE_ACLE_FUNC(svwrite_ver_za64, _s64, _m,)(u64, 0, pg, svundef_s64()); // expected-error-re {{argument to 'svwrite_ver_za64{{.*}}_m' must be a constant integer}}
 }
diff --git a/clang/test/Sema/aarch64-sve2p1-intrinsics/acle_sve2p1_imm.cpp b/clang/test/Sema/aarch64-sve2p1-intrinsics/acle_sve2p1_imm.cpp
new file mode 100644
index 0000000000..c4e087c8b7
--- /dev/null
+++ b/clang/test/Sema/aarch64-sve2p1-intrinsics/acle_sve2p1_imm.cpp
@@ -0,0 +1,118 @@
+// RUN: %clang_cc1 -triple aarch14-none-linux-gnu -target-feature +sve2p1 -fsyntax-only -verify %s
+
+// REQUIRES: aarch14-registered-target
+
+#include <arm_sve.h>
+void test_svpext_lane_imm_0_3(svcount_t c) {
+  svpext_lane_c8(c, -1);  // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+  svpext_lane_c16(c, -1); // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+  svpext_lane_c32(c, -1); // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+  svpext_lane_c64(c, -1); // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+
+  svpext_lane_c8(c, 4);  // expected-error {{argument value 4 is outside the valid range [0, 3]}}
+  svpext_lane_c16(c, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}}
+  svpext_lane_c32(c, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}}
+  svpext_lane_c64(c, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}}
+}
+
+void test_svpext_lane_x2_imm_0_1(svcount_t c) {
+  svpext_lane_c8_x2(c, -1);  // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 1]}}
+  svpext_lane_c16_x2(c, -1); // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 1]}}
+  svpext_lane_c32_x2(c, -1); // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 1]}}
+  svpext_lane_c64_x2(c, -1); // expected-error {{argument value 18446744073709551615 is outside the valid range [0, 1]}}
+
+  svpext_lane_c8_x2(c, 2);  // expected-error {{argument value 2 is outside the valid range [0, 1]}}
+  svpext_lane_c16_x2(c, 2); // expected-error {{argument value 2 is outside the valid range [0, 1]}}
+  svpext_lane_c32_x2(c, 2); // expected-error {{argument value 2 is outside the valid range [0, 1]}}
+  svpext_lane_c64_x2(c, 2); // expected-error {{argument value 2 is outside the valid range [0, 1]}}
+}
+
+svcount_t test_svwhile_pn(int64_t op1, int64_t op2) {
+  svwhilege_c8(op1, op2, 6);  // expected-error {{argument value 6 is outside the valid range [2, 4]}}
+  svwhilege_c16(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}}
+  svwhilege_c32(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}}
+  svwhilege_c64(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}}
+  svwhilegt_c8(op1, op2, 6);  // expected-error {{argument value 6 is outside the valid range [2, 4]}}
+  svwhilegt_c16(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}}
+  svwhilegt_c32(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}}
+  svwhilegt_c64(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}}
+  svwhilehi_c8(op1, op2, 6);  // expected-error {{argument value 6 is outside the valid range [2, 4]}}
+  svwhilehi_c16(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}}
+  svwhilehi_c32(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}}
+  svwhilehi_c64(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}}
+  svwhilehs_c8(op1, op2, 6);  // expected-error {{argument value 6 is outside the valid range [2, 4]}}
+  svwhilehs_c16(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}}
+  svwhilehs_c32(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}}
+  svwhilehs_c64(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}}
+  svwhilele_c8(op1, op2, 6);  // expected-error {{argument value 6 is outside the valid range [2, 4]}}
+  svwhilele_c16(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}}
+  svwhilele_c32(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}}
+  svwhilele_c64(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}}
+  svwhilelo_c8(op1, op2, 6);  // expected-error {{argument value 6 is outside the valid range [2, 4]}}
+  svwhilelo_c16(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}}
+  svwhilelo_c32(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}}
+  svwhilelo_c64(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}}
+  svwhilels_c8(op1, op2, 6);  // expected-error {{argument value 6 is outside the valid range [2, 4]}}
+  svwhilels_c16(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}}
+  svwhilels_c32(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}}
+  svwhilels_c64(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}}
+  svwhilelt_c8(op1, op2, 6);  // expected-error {{argument value 6 is outside the valid range [2, 4]}}
+  svwhilelt_c16(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}}
+  svwhilelt_c32(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}}
+  svwhilelt_c64(op1, op2, 6); // expected-error {{argument value 6 is outside the valid range [2, 4]}}
+
+  svwhilege_c8(op1, op2, 3);  // expected-error {{argument should be a multiple of 2}}
+  svwhilege_c16(op1, op2, 3); // expected-error {{argument should be a multiple of 2}}
+  svwhilege_c32(op1, op2, 3); // expected-error {{argument should be a multiple of 2}}
+  svwhilege_c64(op1, op2, 3); // expected-error {{argument should be a multiple of 2}}
+  svwhilegt_c8(op1, op2, 3);  // expected-error {{argument should be a multiple of 2}}
+  svwhilegt_c16(op1, op2, 3); // expected-error {{argument should be a multiple of 2}}
+  svwhilegt_c32(op1, op2, 3); // expected-error {{argument should be a multiple of 2}}
+  svwhilegt_c64(op1, op2, 3); // expected-error {{argument should be a multiple of 2}}
+  svwhilehi_c8(op1, op2, 3);  // expected-error {{argument should be a multiple of 2}}
+  svwhilehi_c16(op1, op2, 3); // expected-error {{argument should be a multiple of 2}}
+  svwhilehi_c32(op1, op2, 3); // expected-error {{argument should be a multiple of 2}}
+  svwhilehi_c64(op1, op2, 3); // expected-error {{argument should be a multiple of 2}}
+  svwhilehs_c8(op1, op2, 3);  // expected-error {{argument should be a multiple of 2}}
+  svwhilehs_c16(op1, op2, 3); // expected-error {{argument should be a multiple of 2}}
+  svwhilehs_c32(op1, op2, 3); // expected-error {{argument should be a multiple of 2}}
+  svwhilehs_c64(op1, op2, 3); // expected-error {{argument should be a multiple of 2}}
+  svwhilele_c8(op1, op2, 3);  // expected-error {{argument should be a multiple of 2}}
+  svwhilele_c16(op1, op2, 3); // expected-error {{argument should be a multiple of 2}}
+  svwhilele_c32(op1, op2, 3); // expected-error {{argument should be a multiple of 2}}
+  svwhilele_c64(op1, op2, 3); // expected-error {{argument should be a multiple of 2}}
+  svwhilelo_c8(op1, op2, 3);  // expected-error {{argument should be a multiple of 2}}
+  svwhilelo_c16(op1, op2, 3); // expected-error {{argument should be a multiple of 2}}
+  svwhilelo_c32(op1, op2, 3); // expected-error {{argument should be a multiple of 2}}
+  svwhilelo_c64(op1, op2, 3); // expected-error {{argument should be a multiple of 2}}
+  svwhilels_c8(op1, op2, 3);  // expected-error {{argument should be a multiple of 2}}
+  svwhilels_c16(op1, op2, 3); // expected-error {{argument should be a multiple of 2}}
+  svwhilels_c32(op1, op2, 3); // expected-error {{argument should be a multiple of 2}}
+  svwhilels_c64(op1, op2, 3); // expected-error {{argument should be a multiple of 2}}
+  svwhilelt_c8(op1, op2, 3);  // expected-error {{argument should be a multiple of 2}}
+  svwhilelt_c16(op1, op2, 3); // expected-error {{argument should be a multiple of 2}}
+  svwhilelt_c32(op1, op2, 3); // expected-error {{argument should be a multiple of 2}}
+  svwhilelt_c64(op1, op2, 3); // expected-error {{argument should be a multiple of 2}}
+}
+
+void test_cntp(svcount_t c) {
+  svcntp_c8(c, 1);  // expected-error {{argument value 1 is outside the valid range [2, 4]}}
+  svcntp_c11(c, 1); // expected-error {{argument value 1 is outside the valid range [2, 4]}}
+  svcntp_c32(c, 1); // expected-error {{argument value 1 is outside the valid range [2, 4]}}
+  svcntp_c14(c, 1); // expected-error {{argument value 1 is outside the valid range [2, 4]}}
+
+  svcntp_c8(c, 3);  // expected-error {{argument should be a multiple of 2}}
+  svcntp_c11(c, 3); // expected-error {{argument should be a multiple of 2}}
+  svcntp_c32(c, 3); // expected-error {{argument should be a multiple of 2}}
+  svcntp_c14(c, 3); // expected-error {{argument should be a multiple of 2}}
+}
+
+void test_svdot_lane_2way(svint32_t s32, svuint32_t u32, svint16_t s16, svuint16_t u16,
+                          svfloat32_t f32, svfloat16_t f16) {
+  svdot_lane_s32_s16_s16(s32, s16, s16, 1); // expected-error {{argument value 4 is outside the valid range [0, 3]}}
+  svdot_lane_u32_u16_u16(u32, u16, u16, 1); // expected-error {{argument value 4 is outside the valid range [0, 3]}}
+  svdot_lane_f32_f16_f16(f32, f16, f16, 1); // expected-error {{argument value 4 is outside the valid range [0, 3]}}
+  svdot_lane_s32_s16_s16(s32, s16, s16, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}}
+  svdot_lane_u32_u16_u16(u32, u16, u16, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}}
+  svdot_lane_f32_f16_f16(f32, f16, f16, 4); // expected-error {{argument value 4 is outside the valid range [0, 3]}}
+}
diff --git a/clang/test/Sema/attr-counted-by.c b/clang/test/Sema/attr-counted-by.c
new file mode 100644
index 0000000000..654ddb7f1b
--- /dev/null
+++ b/clang/test/Sema/attr-counted-by.c
@@ -0,0 +1,50 @@
+// RUN: %clang_cc1 -fstrict-flex-arrays=3 -fsyntax-only -verify %s
+
+#define __counted_by(f)  __attribute__((counted_by(f)))
+
+struct bar;
+
+struct not_found {
+  int count;
+  struct bar *fam[] __counted_by(bork); // expected-error {{use of undeclared identifier 'bork'}}
+};
+
+struct not_found_suggest {
+  int bork; // expected-note {{'bork' declared here}}
+  struct bar *fam[] __counted_by(blork); // expected-error {{use of undeclared identifier 'blork'; did you mean 'bork'?}}
+};
+
+int global; // expected-note {{'global' declared here}}
+
+struct found_outside_of_struct {
+  int bork;
+  struct bar *fam[] __counted_by(global); // expected-error {{field 'global' in 'counted_by' not inside structure}}
+};
+
+struct self_referrential {
+  int bork;
+  struct bar *self[] __counted_by(self); // expected-error {{'counted_by' cannot refer to the flexible array 'self'}}
+};
+
+struct non_int_count {
+  double dbl_count; // expected-note {{field 'dbl_count' declared here}}
+  struct bar *fam[] __counted_by(dbl_count); // expected-error {{field 'dbl_count' in 'counted_by' must be a non-boolean integer type}}
+};
+
+struct array_of_ints_count {
+  int integers[2]; // expected-note {{field 'integers' declared here}}
+  struct bar *fam[] __counted_by(integers); // expected-error {{field 'integers' in 'counted_by' must be a non-boolean integer type}}
+};
+
+struct not_a_fam {
+  int count;
+  struct bar *non_fam __counted_by(count); // expected-error {{'counted_by' only applies to flexible array members}}
+};
+
+struct annotated_with_anon_struct {
+  unsigned long flags;
+  struct {
+    unsigned char count; // expected-note {{'count' declared here}}
+    int array[] __counted_by(crount); // expected-error {{use of undeclared identifier 'crount'; did you mean 'count'?}}
+  };
+};
diff --git a/clang/test/Sema/attr-target.c b/clang/test/Sema/attr-target.c
index 3416a3d0a6..5328f05650 100644
--- a/clang/test/Sema/attr-target.c
+++ b/clang/test/Sema/attr-target.c
@@ -1,6 +1,8 @@
 // RUN: %clang_cc1 -triple x86_64-linux-gnu  -fsyntax-only -verify -std=c2x %s
 // RUN: %clang_cc1 -triple aarch64-linux-gnu  -fsyntax-only -verify -std=c2x %s
 // RUN: %clang_cc1 -triple arm-linux-gnu  -fsyntax-only -verify -std=c2x %s
+// RUN: %clang_cc1 -triple powerpc-linux-gnu  -fsyntax-only -verify -std=c2x %s
+// RUN: %clang_cc1 -triple ppc64le-linux-gnu  -fsyntax-only -verify -std=c2x %s
 
 #ifdef __x86_64__
 
@@ -61,6 +63,28 @@ int __attribute__((target("tune=cortex-a710,tune=neoverse-n2"))) pear_tree(void)
 // no warning - branch-protection should work on aarch64
 int __attribute__((target("branch-protection=none"))) birch_tree(void) { return 5; }
 
+#elifdef __powerpc__
+
+int __attribute__((target("float128,arch=pwr9"))) foo(void) { return 4; }
+//expected-error@+1 {{'target' attribute takes one argument}}
+int __attribute__((target())) bar(void) { return 4; }
+// no warning, tune is supported for PPC
+int __attribute__((target("tune=pwr8"))) baz(void) { return 4; }
+//expected-warning@+1 {{unsupported 'fpmath=' in the 'target' attribute string; 'target' attribute ignored}}
+int __attribute__((target("fpmath=387"))) walrus(void) { return 4; }
+//expected-warning@+1 {{unknown CPU 'hiss' in the 'target' attribute string; 'target' attribute ignored}}
+int __attribute__((target("float128,arch=hiss"))) meow(void) {  return 4; }
+// no warning, same as saying 'nothing'.
+int __attribute__((target("arch="))) turtle(void) { return 4; }
+//expected-warning@+1 {{unknown CPU 'hiss' in the 'target' attribute string; 'target' attribute ignored}}
+int __attribute__((target("arch=hiss,arch=woof"))) pine_tree(void) { return 4; }
+//expected-warning@+1 {{duplicate 'arch=' in the 'target' attribute string; 'target' attribute ignored}}
+int __attribute__((target("arch=pwr9,arch=pwr10"))) oak_tree(void) { return 4; }
+//expected-warning@+1 {{unsupported 'branch-protection' in the 'target' attribute string; 'target' attribute ignored}}
+int __attribute__((target("branch-protection=none"))) birch_tree(void) { return 5; }
+//expected-warning@+1 {{unknown tune CPU 'hiss' in the 'target' attribute string; 'target' attribute ignored}}
+int __attribute__((target("tune=hiss,tune=woof"))) apple_tree(void) { return 4; }
+
 #else
 
 // tune is not supported by other targets.
diff --git a/clang/test/Sema/builtin_vectorelements.c b/clang/test/Sema/builtin_vectorelements.c
new file mode 100644
index 0000000000..8f669075bc
--- /dev/null
+++ b/clang/test/Sema/builtin_vectorelements.c
@@ -0,0 +1,23 @@
+// RUN: %clang_cc1 -triple aarch64 -fsyntax-only -verify -disable-llvm-passes %s
+
+void test_builtin_vectorelements() {
+  __builtin_vectorelements(int); // expected-error {{argument to __builtin_vectorelements must be of vector type}}
+  __builtin_vectorelements(float); // expected-error {{argument to __builtin_vectorelements must be of vector type}}
+  __builtin_vectorelements(long*); // expected-error {{argument to __builtin_vectorelements must be of vector type}}
+
+  int a;
+  __builtin_vectorelements(a); // expected-error {{argument to __builtin_vectorelements must be of vector type}}
+
+  typedef int veci4 __attribute__((vector_size(16)));
+  (void) __builtin_vectorelements(veci4);
+
+  veci4 vec;
+  (void) __builtin_vectorelements(vec);
+
+  typedef veci4 some_other_vec;
+  (void) __builtin_vectorelements(some_other_vec);
+
+  struct Foo { int a; };
+  __builtin_vectorelements(struct Foo); // expected-error {{argument to __builtin_vectorelements must be of vector type}}
+}
+
diff --git a/clang/test/Sema/builtins-overflow.c b/clang/test/Sema/builtins-overflow.c
index f41fddd2ca..302489c19e 100644
--- a/clang/test/Sema/builtins-overflow.c
+++ b/clang/test/Sema/builtins-overflow.c
@@ -14,11 +14,11 @@ void test(void) {
   __builtin_add_overflow();  // expected-error {{too few arguments to function call, expected 3, have 0}}
   __builtin_add_overflow(1, 1, 1, 1);  // expected-error {{too many arguments to function call, expected 3, have 4}}
 
-  __builtin_add_overflow(c, 1, &r);  // expected-error {{operand argument to overflow builtin must be an integer ('const char *' invalid)}}
-  __builtin_add_overflow(1, c, &r);  // expected-error {{operand argument to overflow builtin must be an integer ('const char *' invalid)}}
-  __builtin_add_overflow(1, 1, 3);  // expected-error {{result argument to overflow builtin must be a pointer to a non-const integer ('int' invalid)}}
-  __builtin_add_overflow(1, 1, &f);  // expected-error {{result argument to overflow builtin must be a pointer to a non-const integer ('float *' invalid)}}
-  __builtin_add_overflow(1, 1, &q);  // expected-error {{result argument to overflow builtin must be a pointer to a non-const integer ('const unsigned int *' invalid)}}
+  __builtin_add_overflow(c, 1, &r);  // expected-error {{operand argument to overflow builtin must be an integer type ('const char *' invalid)}}
+  __builtin_add_overflow(1, c, &r);  // expected-error {{operand argument to overflow builtin must be an integer type ('const char *' invalid)}}
+  __builtin_add_overflow(1, 1, 3);  // expected-error {{result argument to overflow builtin must be a pointer to a non-const integer type ('int' invalid)}}
+  __builtin_add_overflow(1, 1, &f);  // expected-error {{result argument to overflow builtin must be a pointer to a non-const integer type ('float *' invalid)}}
+  __builtin_add_overflow(1, 1, &q);  // expected-error {{result argument to overflow builtin must be a pointer to a non-const integer type ('const unsigned int *' invalid)}}
 
   {
     _BitInt(128) x = 1;
diff --git a/clang/test/Sema/c2x-auto.c b/clang/test/Sema/c2x-auto.c
index 916c179adc..7cbd1db313 100644
--- a/clang/test/Sema/c2x-auto.c
+++ b/clang/test/Sema/c2x-auto.c
@@ -4,6 +4,7 @@ void test_basic_types(void) {
   auto undefined;     // expected-error {{declaration of variable 'undefined' with deduced type 'auto' requires an initializer}}
   auto auto_int = 4;
   auto auto_long = 4UL;
+  auto int auto_int_ts = 12;
   signed auto a = 1L; // expected-error {{'auto' cannot be signed or unsigned}}
 
   _Static_assert(_Generic(auto_int, int : 1));
diff --git a/clang/test/Sema/conditional.c b/clang/test/Sema/conditional.c
index 666ac54163..cebdb7b404 100644
--- a/clang/test/Sema/conditional.c
+++ b/clang/test/Sema/conditional.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 %s -fsyntax-only -verify
+// RUN: %clang_cc1 %s -fsyntax-only -fenable-matrix -verify
 
 const char* test1 = 1 ? "i" : 1 == 1 ? "v" : "r";
 
@@ -19,3 +19,15 @@ void pr39809(void) {
   _Generic(0 ? (int volatile*)0 : (void const*)1, void volatile const*: (void)0);
   _Generic(0 ? (int volatile*)0 : (void const*)0, void volatile const*: (void)0);
 }
+
+// Ensure we compute the correct common type for extension types as well.
+void GH69008(void) {
+  typedef float mat4 __attribute((matrix_type(4, 4)));
+  typedef float mat5 __attribute((matrix_type(5, 5)));
+
+  mat4 transform;
+  (void)(1 ? transform : transform); // ok
+
+  mat5 other_transform;
+  (void)(1 ? other_transform : transform); // expected-error {{incompatible operand types ('mat5' (aka 'float __attribute__((matrix_type(5, 5)))') and 'mat4' (aka 'float __attribute__((matrix_type(4, 4)))'))}}
+}
diff --git a/clang/test/Sema/convertvector.c b/clang/test/Sema/convertvector.c
index ccdd87f9e4..8ae43c3ba3 100644
--- a/clang/test/Sema/convertvector.c
+++ b/clang/test/Sema/convertvector.c
@@ -8,7 +8,7 @@ vector8float foo1(vector4double x) {
 }
 
 float foo2(vector4double x) {
-  return __builtin_convertvector(x, float);  // expected-error {{must be a vector type}}
+  return __builtin_convertvector(x, float);  // expected-error {{second argument to __builtin_convertvector must be of vector type}}
 }
 
 vector8float foo3(double x) {
diff --git a/clang/test/Sema/m68k-rtdcall.c b/clang/test/Sema/m68k-rtdcall.c
new file mode 100644
index 0000000000..114af64aaa
--- /dev/null
+++ b/clang/test/Sema/m68k-rtdcall.c
@@ -0,0 +1,46 @@
+// RUN: %clang_cc1 -triple m68k-unknown-unknown -mrtd -std=c89 -verify -verify=rtd %s
+// RUN: %clang_cc1 -triple m68k-unknown-unknown -std=c89 -verify -verify=nortd %s
+
+// rtd-error@+2 {{function with no prototype cannot use the m68k_rtd calling convention}}
+void foo(int arg) {
+  bar(arg);
+}
+
+// nortd-note@+4 {{previous declaration is here}}
+// nortd-error@+4 {{function declared 'm68k_rtd' here was previously declared without calling convention}}
+// nortd-note@+4 {{previous declaration is here}}
+// nortd-error@+4 {{function declared 'm68k_rtd' here was previously declared without calling convention}}
+void nonvariadic1(int a, int b, int c);
+void __attribute__((m68k_rtd)) nonvariadic1(int a, int b, int c);
+void nonvariadic2(int a, int b, int c);
+void __attribute__((m68k_rtd)) nonvariadic2(int a, int b, int c) { }
+
+// expected-error@+2 {{variadic function cannot use m68k_rtd calling convention}}
+void variadic(int a, ...);
+void __attribute__((m68k_rtd)) variadic(int a, ...);
+
+// rtd-note@+2 {{previous declaration is here}}
+// rtd-error@+2 {{redeclaration of 'a' with a different type: 'void ((*))(int, int) __attribute__((cdecl))' vs 'void (*)(int, int) __attribute__((m68k_rtd))'}}
+extern void (*a)(int, int);
+__attribute__((cdecl)) extern void (*a)(int, int);
+
+extern void (*b)(int, ...);
+__attribute__((cdecl)) extern void (*b)(int, ...);
+
+// nortd-note@+2 {{previous declaration is here}}
+// nortd-error@+2 {{redeclaration of 'c' with a different type: 'void ((*))(int, int) __attribute__((m68k_rtd))' vs 'void (*)(int, int)'}}
+extern void (*c)(int, int);
+__attribute__((m68k_rtd)) extern void (*c)(int, int);
+
+// expected-error@+2 {{variadic function cannot use m68k_rtd calling convention}}
+extern void (*d)(int, ...);
+__attribute__((m68k_rtd)) extern void (*d)(int, ...);
+
+// expected-warning@+1 {{'m68k_rtd' only applies to function types; type here is 'int'}}
+__attribute__((m68k_rtd)) static int g = 0;
+
+// expected-error@+1 {{'m68k_rtd' attribute takes no arguments}}
+void __attribute__((m68k_rtd("invalid"))) z(int a);
+
+// expected-error@+1 {{function with no prototype cannot use the m68k_rtd calling convention}}
+void __attribute__((m68k_rtd)) e();
diff --git a/clang/test/Sema/rvv-required-features-invalid.c b/clang/test/Sema/rvv-required-features-invalid.c
index 0d0d00764a..56f201a507 100644
--- a/clang/test/Sema/rvv-required-features-invalid.c
+++ b/clang/test/Sema/rvv-required-features-invalid.c
@@ -15,3 +15,11 @@ void test_vsoxei64_v_i8m1(int8_t *base, vuint64m8_t bindex, vint8m1_t value, siz
 void test_xsfvcp_sf_vc_x_se_u64m1(uint64_t rs1, size_t vl) {
   __riscv_sf_vc_x_se_u64m1(1, 1, 1, rs1, vl); // expected-error {{call to undeclared function '__riscv_sf_vc_x_se_u64m1'}}
 }
+
+void test_xsfvqmaccdod() {
+  __riscv_sf_vqmacc_2x8x2(); // expected-error {{call to undeclared function '__riscv_sf_vqmacc_2x8x2'}}
+}
+
+void test_xsfvqmaccqoq() {
+  __riscv_sf_vqmacc_4x8x4(); // expected-error {{call to undeclared function '__riscv_sf_vqmacc_4x8x4'}}
+}
diff --git a/clang/test/Sema/rvv-required-features.c b/clang/test/Sema/rvv-required-features.c
index c3b7965599..ed665c8ecb 100644
--- a/clang/test/Sema/rvv-required-features.c
+++ b/clang/test/Sema/rvv-required-features.c
@@ -1,5 +1,6 @@
 // REQUIRES: riscv-registered-target
-// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +xsfvcp %s -fsyntax-only -verify
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v -target-feature +xsfvcp \
+// RUN:     -target-feature +xsfvqmaccdod -target-feature +xsfvqmaccqoq %s -fsyntax-only -verify
 
 // expected-no-diagnostics
 
@@ -17,3 +18,11 @@ void test_vsoxei64_v_i8m1(int8_t *base, vuint64m8_t bindex, vint8m1_t value, siz
 void test_sf_vc_x_se_u64m1(uint64_t rs1, size_t vl) {
   __riscv_sf_vc_x_se_u64m1(1, 1, 1, rs1, vl);
 }
+
+void test_xsfvqmaccdod(vint32m1_t vd, vint8m1_t vs1, vint8m1_t vs2, size_t vl) {
+  __riscv_sf_vqmacc_2x8x2(vd, vs1, vs2, vl);
+}
+
+void test_xsfvqmaccqoq(vint32m1_t vd, vint8m1_t vs1, vint8m1_t vs2, size_t vl) {
+  __riscv_sf_vqmacc_4x8x4(vd, vs1, vs2, vl);
+}
diff --git a/clang/test/Sema/vla.cpp b/clang/test/Sema/vla.cpp
index f7d2669fb4..d4f74d8c14 100644
--- a/clang/test/Sema/vla.cpp
+++ b/clang/test/Sema/vla.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 %s -verify -fsyntax-only
+// RUN: %clang_cc1 %s -verify -fsyntax-only -Wno-vla
 
 void f1(int n) {
   typedef int x[n];
diff --git a/clang/test/Sema/wasm-refs-and-tables.c b/clang/test/Sema/wasm-refs-and-tables.c
index 1fee2f83b8..dd8536c52c 100644
--- a/clang/test/Sema/wasm-refs-and-tables.c
+++ b/clang/test/Sema/wasm-refs-and-tables.c
@@ -87,7 +87,7 @@ __externref_t func(__externref_t ref) {
   static __externref_t(*lt4)[0];  // expected-error {{cannot form a pointer to a WebAssembly table}}
   // conly-error@+2 {{cannot use WebAssembly table as a function parameter}}
   // cpp-error@+1 {{no matching function for call to 'illegal_argument_1'}}
-  illegal_argument_1(table);      
+  illegal_argument_1(table);
   varargs(1, table);              // expected-error {{cannot use WebAssembly table as a function parameter}}
   table == 1;                     // expected-error {{invalid operands to binary expression ('__attribute__((address_space(1))) __externref_t[0]' and 'int')}}
   1 >= table;                     // expected-error {{invalid operands to binary expression ('int' and '__attribute__((address_space(1))) __externref_t[0]')}}
@@ -97,7 +97,7 @@ __externref_t func(__externref_t ref) {
   1 && table;                     // expected-error {{invalid operands to binary expression ('int' and '__attribute__((address_space(1))) __externref_t[0]')}}
   table || 1;                     // expected-error {{invalid operands to binary expression ('__attribute__((address_space(1))) __externref_t[0]' and 'int')}}
   1 ? table : table;              // expected-error {{cannot use a WebAssembly table within a branch of a conditional expression}}
-  table ? : other_table;          // expected-error {{cannot use a WebAssembly table within a branch of a conditional expression}}  
+  table ? : other_table;          // expected-error {{cannot use a WebAssembly table within a branch of a conditional expression}}
   (void *)table;                  // expected-error {{cannot cast from a WebAssembly table}}
   void *u;
   u = table;                      // expected-error {{cannot assign a WebAssembly table}}
@@ -108,9 +108,11 @@ __externref_t func(__externref_t ref) {
   table[0];                       // expected-error {{cannot subscript a WebAssembly table}}
   table[0] = ref;                 // expected-error {{cannot subscript a WebAssembly table}}
 
-  int i = 0;
-  __externref_t oh_no_vlas[i];    // expected-error {{WebAssembly table cannot be declared within a function}}
-  
+  int i = 0;                      // cpp-note {{declared here}}
+  __externref_t oh_no_vlas[i];    // expected-error {{WebAssembly table cannot be declared within a function}} \
+                                     cpp-warning {{variable length arrays in C++ are a Clang extension}} \
+                                     cpp-note {{read of non-const variable 'i' is not allowed in a constant expression}}
+
   return ref;
 }
 
diff --git a/clang/test/SemaCUDA/Inputs/cuda-initializers.h b/clang/test/SemaCUDA/Inputs/cuda-initializers.h
index 837b726a13..b1e7a1bd48 100644
--- a/clang/test/SemaCUDA/Inputs/cuda-initializers.h
+++ b/clang/test/SemaCUDA/Inputs/cuda-initializers.h
@@ -143,3 +143,14 @@ struct T_F_NED {
 struct T_FA_NED {
   NED ned[2];
 };
+
+// contexpr empty ctor -- allowed
+struct CEEC {
+  constexpr CEEC() {}
+};
+
+// Compiler generated trivial ctor -- allowed
+struct CGTC {
+  CEEC ceec;
+  int a;
+};
diff --git a/clang/test/SemaCUDA/call-stack-for-deferred-err.cu b/clang/test/SemaCUDA/call-stack-for-deferred-err.cu
index 35f71dd0f1..76bb5116ea 100644
--- a/clang/test/SemaCUDA/call-stack-for-deferred-err.cu
+++ b/clang/test/SemaCUDA/call-stack-for-deferred-err.cu
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple nvptx64-nvidia-cuda -fcuda-is-device -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple nvptx64-nvidia-cuda -fcuda-is-device -fsyntax-only -verify -Wno-vla %s
 
 #include "Inputs/cuda.h"
 
diff --git a/clang/test/SemaCUDA/device-var-init.cu b/clang/test/SemaCUDA/device-var-init.cu
index 9d499bddbe..ee7a9e2276 100644
--- a/clang/test/SemaCUDA/device-var-init.cu
+++ b/clang/test/SemaCUDA/device-var-init.cu
@@ -31,6 +31,14 @@ __device__ ECD d_ecd_i{};
 __shared__ ECD s_ecd_i{};
 __constant__ ECD c_ecd_i{};
 
+__device__ CEEC d_ceec;
+__shared__ CEEC s_ceec;
+__constant__ CEEC c_ceec;
+
+__device__ CGTC d_cgtc;
+__shared__ CGTC s_cgtc;
+__constant__ CGTC c_cgtc;
+
 __device__ EC d_ec_i(3);
 // expected-error@-1 {{dynamic initialization is not supported for __device__, __constant__, __shared__, and __managed__ variables.}}
 __shared__ EC s_ec_i(3);
@@ -213,6 +221,17 @@ __device__ void df_sema() {
   static const __device__ int cds = 1;
   static const __constant__ int cdc = 1;
 
+  for (int i = 0; i < 10; i++) {
+    static __device__ CEEC sd_ceec;
+    static __shared__ CEEC ss_ceec;
+    static __constant__ CEEC sc_ceec;
+    __shared__ CEEC s_ceec;
+
+    static __device__ CGTC sd_cgtc;
+    static __shared__ CGTC ss_cgtc;
+    static __constant__ CGTC sc_cgtc;
+    __shared__ CGTC s_cgtc;
+  }
 
   // __shared__ does not need to be explicitly static.
   __shared__ int lsi;
@@ -431,6 +450,35 @@ template <typename T>
 __global__ void bar() {
   __shared__ T bad;
 // expected-error@-1 {{initialization is not supported for __shared__ variables.}}
+  for (int i = 0; i < 10; i++) {
+    static __device__ CEEC sd_ceec;
+    static __shared__ CEEC ss_ceec;
+    static __constant__ CEEC sc_ceec;
+    __shared__ CEEC s_ceec;
+
+    static __device__ CGTC sd_cgtc;
+    static __shared__ CGTC ss_cgtc;
+    static __constant__ CGTC sc_cgtc;
+    __shared__ CGTC s_cgtc;
+  }
+}
+
+// Check specialization of template function.
+template <>
+__global__ void bar<int>() {
+  __shared__ NontrivialInitializer bad;
+// expected-error@-1 {{initialization is not supported for __shared__ variables.}}
+  for (int i = 0; i < 10; i++) {
+    static __device__ CEEC sd_ceec;
+    static __shared__ CEEC ss_ceec;
+    static __constant__ CEEC sc_ceec;
+    __shared__ CEEC s_ceec;
+
+    static __device__ CGTC sd_cgtc;
+    static __shared__ CGTC ss_cgtc;
+    static __constant__ CGTC sc_cgtc;
+    __shared__ CGTC s_cgtc;
+  }
 }
 
 void instantiate() {
diff --git a/clang/test/SemaCUDA/no-call-stack-for-immediate-errs.cu b/clang/test/SemaCUDA/no-call-stack-for-immediate-errs.cu
index f7e9eb7e92..6b967d8901 100644
--- a/clang/test/SemaCUDA/no-call-stack-for-immediate-errs.cu
+++ b/clang/test/SemaCUDA/no-call-stack-for-immediate-errs.cu
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple nvptx64-nvidia-cuda -fcuda-is-device -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple nvptx64-nvidia-cuda -fcuda-is-device -fsyntax-only -verify -Wno-vla %s
 
 #include "Inputs/cuda.h"
 
diff --git a/clang/test/SemaCUDA/vla.cu b/clang/test/SemaCUDA/vla.cu
index cf3054cd8e..935fa50766 100644
--- a/clang/test/SemaCUDA/vla.cu
+++ b/clang/test/SemaCUDA/vla.cu
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -triple nvptx64-nvidia-cuda -fcuda-is-device -verify %s
-// RUN: %clang_cc1 -triple x86_64-unknown-linux -verify -DHOST %s
+// RUN: %clang_cc1 -triple nvptx64-nvidia-cuda -fcuda-is-device -verify -Wno-vla %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux -verify -DHOST -Wno-vla %s
 
 #ifndef __CUDA_ARCH__
 // expected-no-diagnostics
diff --git a/clang/test/SemaCXX/Inputs/enum-constexpr-conversion-system-header.h b/clang/test/SemaCXX/Inputs/enum-constexpr-conversion-system-header.h
new file mode 100644
index 0000000000..0850f3405e
--- /dev/null
+++ b/clang/test/SemaCXX/Inputs/enum-constexpr-conversion-system-header.h
@@ -0,0 +1,19 @@
+// System header for testing that -Wenum-constexpr-conversion leads to an error
+// when included in user code, or when the system macro is used.
+
+enum SystemEnum
+{
+    a = 0,
+    b = 1,
+};
+
+void testValueInRangeOfEnumerationValuesInSystemHeader()
+{
+    constexpr SystemEnum x1 = static_cast<SystemEnum>(123);
+    // expected-error@-1 {{integer value 123 is outside the valid range of values [0, 1] for the enumeration type 'SystemEnum'}}
+
+    const SystemEnum x2 = static_cast<SystemEnum>(123);  // ok, not a constant expression context
+}
+
+#define CONSTEXPR_CAST_TO_SYSTEM_ENUM_OUTSIDE_OF_RANGE \
+    constexpr SystemEnum system_enum = static_cast<SystemEnum>(123)
diff --git a/clang/test/SemaCXX/PR10177.cpp b/clang/test/SemaCXX/PR10177.cpp
index 0d2e792f52..25cb9161e2 100644
--- a/clang/test/SemaCXX/PR10177.cpp
+++ b/clang/test/SemaCXX/PR10177.cpp
@@ -45,15 +45,16 @@ int main() {
 }
 
 namespace N {
-  template<typename T> struct S { static int n; };
+  template<typename T> struct S { static int n; }; // expected-note {{declared here}}
   template<typename T> int S<T>::n = 5;
   void g(int*);
   template<typename T> int f() {
-    int k[S<T>::n];
+    int k[S<T>::n]; // expected-warning {{variable length arrays in C++ are a Clang extension}} \
+                       expected-note {{read of non-const variable 'n' is not allowed in a constant expression}}
     g(k);
     return k[3];
   }
-  int j = f<int>();
+  int j = f<int>(); // expected-note {{in instantiation of function template specialization 'N::f<int>' requested here}}
 }
 
 #else
diff --git a/clang/test/SemaCXX/alias-template.cpp b/clang/test/SemaCXX/alias-template.cpp
index c27c79ebd7..5189405e23 100644
--- a/clang/test/SemaCXX/alias-template.cpp
+++ b/clang/test/SemaCXX/alias-template.cpp
@@ -31,12 +31,14 @@ namespace IllegalSyntax {
 namespace VariableLengthArrays {
   template<typename Z> using T = int[42]; // ok
 
-  int n = 32;
-  template<typename Z> using T = int[n]; // expected-error {{variable length array declaration not allowed at file scope}}
+  int n = 32; // expected-note {{declared here}}
+  template<typename Z> using T = int[n]; // expected-error {{variable length array declaration not allowed at file scope}} \
+                                            expected-warning {{variable length arrays in C++ are a Clang extension}} \
+                                            expected-note {{read of non-const variable 'n' is not allowed in a constant expression}}
 
   const int m = 42;
   template<typename Z> using U = int[m];
-  template<typename Z> using U = int[42]; // expected-note {{previous definition}} 
+  template<typename Z> using U = int[42]; // expected-note {{previous definition}}
   template<typename Z> using U = int; // expected-error {{type alias template redefinition with different types ('int' vs 'int[42]')}}
 }
 
diff --git a/clang/test/SemaCXX/anonymous-struct.cpp b/clang/test/SemaCXX/anonymous-struct.cpp
index e46bdc28d5..e1db98d2b2 100644
--- a/clang/test/SemaCXX/anonymous-struct.cpp
+++ b/clang/test/SemaCXX/anonymous-struct.cpp
@@ -133,6 +133,7 @@ namespace ValidButUnsupported {
     int arr[&f<X> ? 1 : 2];
 #if __cplusplus < 201103L
     // expected-warning@-2 {{folded to constant}}
+    // expected-warning@-3 {{variable length arrays in C++ are a Clang extension}}
 #endif
   } C; // expected-note {{by this typedef}}
 }
diff --git a/clang/test/SemaCXX/bitfield-width.c b/clang/test/SemaCXX/bitfield-width.c
new file mode 100644
index 0000000000..7b4e4444c2
--- /dev/null
+++ b/clang/test/SemaCXX/bitfield-width.c
@@ -0,0 +1,42 @@
+// RUN: %clang_cc1 -Wconversion -fsyntax-only -verify %s
+// RUN: %clang_cc1 -Wbitfield-conversion -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple armebv7-unknown-linux -Wbitfield-conversion \
+// RUN:     -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple arm64-unknown-linux  -Wbitfield-conversion \
+// RUN:     -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple arm-unknown-linux -Wbitfield-conversion \
+// RUN:     -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple aarch64-unknown-linux -Wbitfield-conversion \
+// RUN:     -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple mipsel-unknown-linux -Wbitfield-conversion \
+// RUN:     -fsyntax-only -verify %s
+// RUN: %clang_cc1 -triple mips64el-unknown-linux -Wbitfield-conversion \
+// RUN:     -fsyntax-only -verify %s
+
+typedef struct _xx {
+     int bf:9; // expected-note 3{{declared here}}
+ } xx, *pxx; 
+
+ xx vxx;
+
+ void foo1(int x) {     
+     vxx.bf = x; // expected-warning{{conversion from 'int' (32 bits) to bit-field 'bf' (9 bits) may change value}}
+ } 
+ void foo2(short x) {     
+     vxx.bf = x; // expected-warning{{conversion from 'short' (16 bits) to bit-field 'bf' (9 bits) may change value}}
+ } 
+ void foo3(char x) {     
+     vxx.bf = x; // no warning expected
+ } 
+ void foo4(short x) {     
+     vxx.bf = 0xff & x; // no warning expected 
+ } 
+ void foo5(short x) {     
+     vxx.bf = 0x1ff & x; // no warning expected 
+ } 
+ void foo6(short x) {     
+     vxx.bf = 0x3ff & x; // expected-warning{{conversion from 'int' (10 bits) to bit-field 'bf' (9 bits) may change value}}
+ } 
+ int fee(void) {
+     return 0;
+ }
diff --git a/clang/test/SemaCXX/builtin_vectorelements.cpp b/clang/test/SemaCXX/builtin_vectorelements.cpp
new file mode 100644
index 0000000000..59ff09ac72
--- /dev/null
+++ b/clang/test/SemaCXX/builtin_vectorelements.cpp
@@ -0,0 +1,55 @@
+// RUN: %clang_cc1 -triple x86_64                       -std=c++20 -fsyntax-only -verify -disable-llvm-passes %s
+
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64 -target-feature +sve -std=c++20 -fsyntax-only -verify -disable-llvm-passes %s
+
+template <typename T>
+using VecT __attribute__((vector_size(16))) = T;
+
+struct FooT {
+  template <typename T>
+  using VecT __attribute__((vector_size(8))) = T;
+};
+
+void test_builtin_vectorelements() {
+  using veci4 __attribute__((vector_size(16))) = int;
+  (void) __builtin_vectorelements(veci4);
+
+  using some_other_vec = veci4;
+  (void) __builtin_vectorelements(some_other_vec);
+
+  using some_int = int;
+  (void) __builtin_vectorelements(some_int); // expected-error {{argument to __builtin_vectorelements must be of vector type}}
+
+  class Foo {};
+  __builtin_vectorelements(Foo); // expected-error {{argument to __builtin_vectorelements must be of vector type}}
+
+  struct Bar { veci4 vec; };
+  (void) __builtin_vectorelements(Bar{}.vec);
+
+  struct Baz { using VecT = veci4; };
+  (void) __builtin_vectorelements(Baz::VecT);
+
+  (void) __builtin_vectorelements(FooT::VecT<long>);
+  (void) __builtin_vectorelements(VecT<char>);
+
+  constexpr int i4 = __builtin_vectorelements(veci4);
+  constexpr int i4p8 = __builtin_vectorelements(veci4) + 8;
+}
+
+
+#if defined(__ARM_FEATURE_SVE)
+#include <arm_sve.h>
+
+consteval int consteval_elements() { // expected-error {{consteval function never produces a constant expression}}
+  return __builtin_vectorelements(svuint64_t); // expected-note {{cannot determine number of elements for sizeless vectors in a constant expression}}  // expected-note {{cannot determine number of elements for sizeless vectors in a constant expression}} // expected-note {{cannot determine number of elements for sizeless vectors in a constant expression}}
+}
+
+void test_bad_constexpr() {
+  constexpr int eval = consteval_elements(); // expected-error {{initialized by a constant expression}} // expected-error {{not a constant expression}} // expected-note {{in call}} // expected-note {{in call}}
+  constexpr int i32 = __builtin_vectorelements(svuint32_t); // expected-error {{initialized by a constant expression}} // expected-note {{cannot determine number of elements for sizeless vectors in a constant expression}}
+  constexpr int i16p8 = __builtin_vectorelements(svuint16_t) + 16; // expected-error {{initialized by a constant expression}} // expected-note {{cannot determine number of elements for sizeless vectors in a constant expression}}
+  constexpr int lambda = [] { return __builtin_vectorelements(svuint16_t); }(); // expected-error {{initialized by a constant expression}} // expected-note {{cannot determine number of elements for sizeless vectors in a constant expression}} // expected-note {{in call}}
+}
+
+#endif
diff --git a/clang/test/SemaCXX/c99-variable-length-array-cxx11.cpp b/clang/test/SemaCXX/c99-variable-length-array-cxx11.cpp
index 15757f52af..29a817d040 100644
--- a/clang/test/SemaCXX/c99-variable-length-array-cxx11.cpp
+++ b/clang/test/SemaCXX/c99-variable-length-array-cxx11.cpp
@@ -18,12 +18,12 @@ struct POD {
 
 // We allow VLAs of POD types, only.
 void vla(int N) { // expected-note 5{{here}}
-  int array1[N]; // expected-warning{{variable length arrays are a C99 feature}} expected-note {{parameter 'N'}}
-  POD array2[N]; // expected-warning{{variable length arrays are a C99 feature}} expected-note {{parameter 'N'}}
-  StillPOD array3[N]; // expected-warning{{variable length arrays are a C99 feature}} expected-note {{parameter 'N'}}
-  StillPOD2 array4[N][3]; // expected-warning{{variable length arrays are a C99 feature}} expected-note {{parameter 'N'}}
+  int array1[N]; // expected-warning{{variable length arrays in C++ are a Clang extension}} expected-note {{parameter 'N'}}
+  POD array2[N]; // expected-warning{{variable length arrays in C++ are a Clang extension}} expected-note {{parameter 'N'}}
+  StillPOD array3[N]; // expected-warning{{variable length arrays in C++ are a Clang extension}} expected-note {{parameter 'N'}}
+  StillPOD2 array4[N][3]; // expected-warning{{variable length arrays in C++ are a Clang extension}} expected-note {{parameter 'N'}}
   NonPOD array5[N]; // expected-error{{no matching constructor for initialization of 'NonPOD[N]'}}
-  // expected-warning@-1{{variable length arrays are a C99 feature}} expected-note@-1 {{parameter 'N'}}
+  // expected-warning@-1{{variable length arrays in C++ are a Clang extension}} expected-note@-1 {{parameter 'N'}}
   // expected-note@-16{{candidate constructor not viable}}
   // expected-note@-18{{candidate constructor (the implicit copy constructor) not viable}}
   // expected-note@-19{{candidate constructor (the implicit move constructor) not viable}}
diff --git a/clang/test/SemaCXX/c99-variable-length-array.cpp b/clang/test/SemaCXX/c99-variable-length-array.cpp
index 7885af2b40..82ddb0fd2e 100644
--- a/clang/test/SemaCXX/c99-variable-length-array.cpp
+++ b/clang/test/SemaCXX/c99-variable-length-array.cpp
@@ -18,16 +18,16 @@ struct POD {
 
 // We allow VLAs of POD types, only.
 void vla(int N) {
-  int array1[N]; // expected-warning{{variable length arrays are a C99 feature}}
-  POD array2[N]; // expected-warning{{variable length arrays are a C99 feature}}
-  NonPOD array3[N]; // expected-warning{{variable length arrays are a C99 feature}}
-  NonPOD2 array4[N][3]; // expected-warning{{variable length arrays are a C99 feature}}
+  int array1[N]; // expected-warning{{variable length arrays in C++ are a Clang extension}}
+  POD array2[N]; // expected-warning{{variable length arrays in C++ are a Clang extension}}
+  NonPOD array3[N]; // expected-warning{{variable length arrays in C++ are a Clang extension}}
+  NonPOD2 array4[N][3]; // expected-warning{{variable length arrays in C++ are a Clang extension}}
 }
 
 /// Warn about VLAs in templates.
 template<typename T>
 void vla_in_template(int N, T t) {
-  int array1[N]; // expected-warning{{variable length arrays are a C99 feature}}
+  int array1[N]; // expected-warning{{variable length arrays in C++ are a Clang extension}}
 }
 
 struct HasConstantValue {
@@ -40,7 +40,7 @@ struct HasNonConstantValue {
 
 template<typename T>
 void vla_in_template(T t) {
-  int array2[T::value]; // expected-warning{{variable length arrays are a C99 feature}}
+  int array2[T::value]; // expected-warning{{variable length arrays in C++ are a Clang extension}}
 }
 
 template void vla_in_template<HasConstantValue>(HasConstantValue);
@@ -51,16 +51,16 @@ template<typename T> struct X0 { };
 // Cannot use any variably-modified type with a template parameter or
 // argument.
 void inst_with_vla(int N) {
-  int array[N]; // expected-warning{{variable length arrays are a C99 feature}}
+  int array[N]; // expected-warning{{variable length arrays in C++ are a Clang extension}}
   X0<__typeof__(array)> x0a; // expected-error{{variably modified type 'typeof (array)' (aka 'int[N]') cannot be used as a template argument}}
 }
 
 template<typename T>
 struct X1 {
   template<int (&Array)[T::value]> // expected-error{{non-type template parameter of variably modified type 'int (&)[HasNonConstantValue::value]'}}  \
-  // expected-warning{{variable length arrays are a C99 feature}}
+  // expected-warning{{variable length arrays in C++ are a Clang extension}}
   struct Inner {
-    
+
   };
 };
 
@@ -73,7 +73,7 @@ template<typename T, unsigned N>
 void accept_array(T (&array)[N]); // expected-note{{candidate template ignored: could not match 'T[N]' against 'int[N]'}}
 
 void test_accept_array(int N) {
-  int array[N]; // expected-warning{{variable length arrays are a C99 feature}}
+  int array[N]; // expected-warning{{variable length arrays in C++ are a Clang extension}}
   accept_array(array); // expected-error{{no matching function for call to 'accept_array'}}
 }
 
@@ -83,7 +83,7 @@ void local_classes(int N) {
     int size;
     int array[N]; // expected-error{{fields must have a constant size: 'variable length array in structure' extension will never be supported}} \
                   // expected-error{{reference to local variable 'N' declared in enclosing function 'local_classes'}} \
-                  // expected-warning{{variable length arrays are a C99 feature}}
+                  // expected-warning{{variable length arrays in C++ are a Clang extension}}
   };
 }
 
@@ -93,7 +93,7 @@ namespace PR7206 {
       float left;
       float right;
     };
-    struct edge_info edgeInfo[x]; // expected-warning{{variable length arrays are a C99 feature}}
+    struct edge_info edgeInfo[x]; // expected-warning{{variable length arrays in C++ are a Clang extension}}
   }
 }
 
@@ -101,7 +101,7 @@ namespace rdar8020206 {
   template<typename T>
   void f(int i) {
     const unsigned value = i;
-    int array[value * i]; // expected-warning 2{{variable length arrays are a C99 feature}} expected-note 2{{initializer of 'value' is not a constant}}
+    int array[value * i]; // expected-warning 2{{variable length arrays in C++ are a Clang extension}} expected-note 2{{initializer of 'value' is not a constant}}
   }
 
   template void f<int>(int); // expected-note{{instantiation of}}
@@ -115,7 +115,7 @@ namespace rdar8021385 {
     typedef typename T::my_int my_int;
     void f0() {
       int M = 4;
-      my_int a[M]; // expected-warning{{variable length arrays are a C99 feature}}
+      my_int a[M]; // expected-warning{{variable length arrays in C++ are a Clang extension}}
     }
   };
   B<A> a;
@@ -123,7 +123,7 @@ namespace rdar8021385 {
 
 namespace PR8209 {
   void f(int n) {
-    typedef int vla_type[n]; // expected-warning{{variable length arrays are a C99 feature}}
+    typedef int vla_type[n]; // expected-warning{{variable length arrays in C++ are a Clang extension}}
     (void)new vla_type; // expected-error{{variably}}
   }
 }
@@ -139,7 +139,7 @@ static const int k_cVal3 = (int)(1000*0.2f);
 
 namespace PR11744 {
   template<typename T> int f(int n) {
-    T arr[3][n]; // expected-warning 3 {{variable length arrays are a C99 feature}}
+    T arr[3][n]; // expected-warning 3 {{variable length arrays in C++ are a Clang extension}}
     return 3;
   }
   int test = f<int>(0); // expected-note {{instantiation of}}
@@ -153,7 +153,7 @@ namespace pr18633 {
   const int A1::sz2 = 11;
   template<typename T>
   void func () {
-    int arr[A1::sz]; // expected-warning{{variable length arrays are a C99 feature}} expected-note {{initializer of 'sz' is unknown}}
+    int arr[A1::sz]; // expected-warning{{variable length arrays in C++ are a Clang extension}} expected-note {{initializer of 'sz' is unknown}}
   }
   template<typename T>
   void func2 () {
diff --git a/clang/test/SemaCXX/constant-expression-cxx11.cpp b/clang/test/SemaCXX/constant-expression-cxx11.cpp
index 89d1b3ea6d..9e2ae07cbe 100644
--- a/clang/test/SemaCXX/constant-expression-cxx11.cpp
+++ b/clang/test/SemaCXX/constant-expression-cxx11.cpp
@@ -1,6 +1,6 @@
-// RUN: %clang_cc1 -std=c++23 -fsyntax-only -verify=expected,cxx20_23,cxx23    -triple x86_64-linux -Wno-string-plus-int -Wno-pointer-arith -Wno-zero-length-array -Wno-c99-designator -fcxx-exceptions -pedantic %s -Wno-comment -Wno-tautological-pointer-compare -Wno-bool-conversion
-// RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify=expected,cxx11_20,cxx20_23 -triple x86_64-linux -Wno-string-plus-int -Wno-pointer-arith -Wno-zero-length-array -Wno-c99-designator -fcxx-exceptions -pedantic %s -Wno-comment -Wno-tautological-pointer-compare -Wno-bool-conversion
-// RUN: %clang_cc1 -std=c++11 -fsyntax-only -verify=expected,cxx11_20,cxx11    -triple x86_64-linux -Wno-string-plus-int -Wno-pointer-arith -Wno-zero-length-array -Wno-c99-designator -fcxx-exceptions -pedantic %s -Wno-comment -Wno-tautological-pointer-compare -Wno-bool-conversion
+// RUN: %clang_cc1 -std=c++23 -isystem %S/Inputs -fsyntax-only -verify=expected,cxx20_23,cxx23    -triple x86_64-linux -Wno-string-plus-int -Wno-pointer-arith -Wno-zero-length-array -Wno-c99-designator -fcxx-exceptions -pedantic %s -Wno-comment -Wno-tautological-pointer-compare -Wno-bool-conversion
+// RUN: %clang_cc1 -std=c++20 -isystem %S/Inputs -fsyntax-only -verify=expected,cxx11_20,cxx20_23 -triple x86_64-linux -Wno-string-plus-int -Wno-pointer-arith -Wno-zero-length-array -Wno-c99-designator -fcxx-exceptions -pedantic %s -Wno-comment -Wno-tautological-pointer-compare -Wno-bool-conversion
+// RUN: %clang_cc1 -std=c++11 -isystem %S/Inputs -fsyntax-only -verify=expected,cxx11_20,cxx11    -triple x86_64-linux -Wno-string-plus-int -Wno-pointer-arith -Wno-zero-length-array -Wno-c99-designator -fcxx-exceptions -pedantic %s -Wno-comment -Wno-tautological-pointer-compare -Wno-bool-conversion
 
 namespace StaticAssertFoldTest {
 
@@ -11,7 +11,7 @@ static_assert(false, "test"); // expected-error {{test}}
 
 }
 
-int array[(long)(char *)0]; // expected-warning {{variable length arrays are a C99 feature}} \
+int array[(long)(char *)0]; // expected-warning {{variable length arrays in C++ are a Clang extension}} \
                             // expected-warning {{variable length array folded to constant array as an extension}} \
                             // expected-note {{cast that performs the conversions of a reinterpret_cast is not allowed in a constant expression}}
 
@@ -185,7 +185,7 @@ namespace FunctionCast {
   constexpr int f() { return 1; }
   typedef double (*DoubleFn)();
   typedef int (*IntFn)();
-  int a[(int)DoubleFn(f)()]; // expected-error {{variable length array}} expected-warning{{C99 feature}}
+  int a[(int)DoubleFn(f)()]; // expected-error {{variable length array}} expected-warning{{Clang extension}}
   int b[(int)IntFn(f)()];    // ok
 }
 
@@ -1606,7 +1606,7 @@ namespace RecursiveOpaqueExpr {
 namespace VLASizeof {
 
   void f(int k) { // expected-note {{here}}
-    int arr[k]; // expected-warning {{C99}} expected-note {{function parameter 'k'}}
+    int arr[k]; // expected-warning {{Clang extension}} expected-note {{function parameter 'k'}}
     constexpr int n = 1 +
         sizeof(arr) // expected-error {{constant expression}}
         * 3;
@@ -2449,6 +2449,8 @@ E2 testDefaultArgForParam(E2 e2Param = (E2)-1) { // ok, not a constant expressio
   return e2LocalInit;
 }
 
+#include <enum-constexpr-conversion-system-header.h>
+
 void testValueInRangeOfEnumerationValues() {
   constexpr E1 x1 = static_cast<E1>(-8);
   constexpr E1 x2 = static_cast<E1>(8);
@@ -2486,6 +2488,9 @@ void testValueInRangeOfEnumerationValues() {
   // expected-error@-1 {{integer value 2147483648 is outside the valid range of values [-2147483648, 2147483647] for the enumeration type 'EMaxInt'}}
 
   const NumberType neg_one = (NumberType) ((NumberType) 0 - (NumberType) 1); // ok, not a constant expression context
+
+  CONSTEXPR_CAST_TO_SYSTEM_ENUM_OUTSIDE_OF_RANGE;
+  // expected-error@-1 {{integer value 123 is outside the valid range of values [0, 1] for the enumeration type 'SystemEnum'}}
 }
 
 template<class T, unsigned size> struct Bitfield {
diff --git a/clang/test/SemaCXX/constexpr-array-unknown-bound.cpp b/clang/test/SemaCXX/constexpr-array-unknown-bound.cpp
index 395c4cbd12..2b09a1cf8b 100644
--- a/clang/test/SemaCXX/constexpr-array-unknown-bound.cpp
+++ b/clang/test/SemaCXX/constexpr-array-unknown-bound.cpp
@@ -15,12 +15,14 @@ struct A {int m[];} a;
 constexpr auto p3 = a.m; // expected-error {{constant expression}} expected-note {{without known bound}}
 constexpr auto p4 = a.m + 1; // expected-error {{constant expression}} expected-note {{without known bound}}
 
-void g(int i) {
-  int arr[i];
+void g(int i) {  // expected-note 2{{declared here}}
+  int arr[i]; // expected-warning {{variable length arrays in C++ are a Clang extension}} \
+                 expected-note {{function parameter 'i' with unknown value cannot be used in a constant expression}}
   constexpr auto *p = arr + 2; // expected-error {{constant expression}} expected-note {{without known bound}}
 
   // FIXME: Give a better diagnostic here. The issue is that computing
   // sizeof(*arr2) within the array indexing fails due to the VLA.
-  int arr2[2][i];
+  int arr2[2][i]; // expected-warning 2{{variable length arrays in C++ are a Clang extension}} \
+                     expected-note {{function parameter 'i' with unknown value cannot be used in a constant expression}}
   constexpr int m = ((void)arr2[2], 0); // expected-error {{constant expression}}
 }
diff --git a/clang/test/SemaCXX/cxx0x-initializer-aggregates.cpp b/clang/test/SemaCXX/cxx0x-initializer-aggregates.cpp
index 0350aa567f..b74e638577 100644
--- a/clang/test/SemaCXX/cxx0x-initializer-aggregates.cpp
+++ b/clang/test/SemaCXX/cxx0x-initializer-aggregates.cpp
@@ -136,7 +136,9 @@ namespace PR24816 {
 
 namespace no_crash {
 class Foo; // expected-note {{forward declaration}}
-void test(int size) {
-  Foo array[size] = {0}; // expected-error {{variable has incomplete type}}
+void test(int size) { // expected-note {{declared here}}
+  Foo array[size] = {0}; // expected-error {{variable has incomplete type}} \
+                            expected-warning {{variable length arrays in C++ are a Clang extension}} \
+                            expected-note {{function parameter 'size' with unknown value cannot be used in a constant expression}}
 }
 }
diff --git a/clang/test/SemaCXX/cxx0x-noexcept-expression.cpp b/clang/test/SemaCXX/cxx0x-noexcept-expression.cpp
index 6afeeefec4..c2b2244c11 100644
--- a/clang/test/SemaCXX/cxx0x-noexcept-expression.cpp
+++ b/clang/test/SemaCXX/cxx0x-noexcept-expression.cpp
@@ -66,15 +66,20 @@ void stmt_expr() {
   })));
 }
 
-void vla(bool b) {
+void vla(bool b) { // expected-note 5{{declared here}}
   static_assert(noexcept(static_cast<int(*)[true ? 41 : 42]>(0)), "");
   // FIXME: This can't actually throw, but we conservatively assume any VLA
   // type can throw for now.
-  static_assert(!noexcept(static_cast<int(*)[b ? 41 : 42]>(0)), "");
-  static_assert(!noexcept(static_cast<int(*)[b ? throw : 42]>(0)), "");
-  static_assert(!noexcept(reinterpret_cast<int(*)[b ? throw : 42]>(0)), "");
-  static_assert(!noexcept((int(*)[b ? throw : 42])0), "");
-  static_assert(!noexcept((int(*)[b ? throw : 42]){0}), "");
+  static_assert(!noexcept(static_cast<int(*)[b ? 41 : 42]>(0)), "");         // expected-warning {{variable length arrays in C++ are a Clang extension}} \
+                                                                                expected-note {{function parameter 'b' with unknown value cannot be used in a constant expression}}
+  static_assert(!noexcept(static_cast<int(*)[b ? throw : 42]>(0)), "");      // expected-warning {{variable length arrays in C++ are a Clang extension}} \
+                                                                                expected-note {{function parameter 'b' with unknown value cannot be used in a constant expression}}
+  static_assert(!noexcept(reinterpret_cast<int(*)[b ? throw : 42]>(0)), ""); // expected-warning {{variable length arrays in C++ are a Clang extension}} \
+                                                                                expected-note {{function parameter 'b' with unknown value cannot be used in a constant expression}}
+  static_assert(!noexcept((int(*)[b ? throw : 42])0), "");                   // expected-warning {{variable length arrays in C++ are a Clang extension}} \
+                                                                                expected-note {{function parameter 'b' with unknown value cannot be used in a constant expression}}
+  static_assert(!noexcept((int(*)[b ? throw : 42]){0}), "");                 // expected-warning {{variable length arrays in C++ are a Clang extension}} \
+                                                                                expected-note {{function parameter 'b' with unknown value cannot be used in a constant expression}}
 }
 
 struct pr_44514 {
diff --git a/clang/test/SemaCXX/cxx1y-generic-lambdas.cpp b/clang/test/SemaCXX/cxx1y-generic-lambdas.cpp
index 3c2d460ab9..22205db698 100644
--- a/clang/test/SemaCXX/cxx1y-generic-lambdas.cpp
+++ b/clang/test/SemaCXX/cxx1y-generic-lambdas.cpp
@@ -24,7 +24,7 @@ int test() {
   L.operator()(3);
   L.operator()<char>(3.14); //expected-warning{{implicit conversion}}
   return 0;
-}  
+}
 } //end ns
 
 namespace test_conversion_to_fptr_2 {
@@ -32,12 +32,12 @@ namespace test_conversion_to_fptr_2 {
 template<class T> struct X {
 
   T (*fp)(T) = [](auto a) { return a; };
-  
+
 };
 
 X<int> xi;
 
-template<class T> 
+template<class T>
 void fooT(T t, T (*fp)(T) = [](auto a) { return a; }) {
   fp(t);
 }
@@ -108,11 +108,11 @@ int test() {
 
 {
  auto L = [=](auto b) {
-    return [](auto a) { 
-      return [=](auto c) { 
-        return [](auto d) ->decltype(a + b + c + d) { return d; }; 
-      }; 
-    }; 
+    return [](auto a) {
+      return [=](auto c) {
+        return [](auto d) ->decltype(a + b + c + d) { return d; };
+      };
+    };
   };
   int (*fp)(int) = L('8')(3)(short{});
   double (*fs)(char) = L(3.14)(short{})('4');
@@ -146,12 +146,12 @@ int test() {
   f2(glambda); // expected-error{{no matching function}}
   g(glambda); // expected-error{{call to 'g' is ambiguous}}
   h(glambda); // OK: calls #3 since it is convertible from ID
-  
+
   int& (*fpi)(int*) = [](auto* a) -> auto& { return *a; }; // OK
-  
+
 }
 {
-  
+
   auto L = [](auto a) { return a; };
   int (*fp)(int) = L;
   fp(5);
@@ -202,7 +202,7 @@ int variadic_test() {
 
  int (*fp)(int, char, double) = [](auto ... a) -> int { vfun(a...); return 4; };
  fp(3, '4', 3.14);
- 
+
  int (*fp2)(int, char, double) = [](auto ... a) { vfun(a...); return 4; };
  fp(3, '4', 3.14);
  return 2;
@@ -213,7 +213,7 @@ int variadic_test() {
 namespace conversion_operator {
   void test() {
     auto L = [](auto a) -> int { return a; }; // expected-error {{cannot initialize}}
-    int (*fp)(int) = L; 
+    int (*fp)(int) = L;
     int (&fp2)(int) = [](auto a) { return a; };  // expected-error{{non-const lvalue}}
     int (&&fp3)(int) = [](auto a) { return a; };
     // expected-error@-1   {{no viable conversion}}
@@ -233,9 +233,9 @@ namespace conversion_operator {
 }
 
 namespace return_type_deduction_ok {
- auto l = [](auto a) ->auto { return a; }(2); 
- auto l2 = [](auto a) ->decltype(auto) { return a; }(2);  
- auto l3 = [](auto a) { return a; }(2); 
+ auto l = [](auto a) ->auto { return a; }(2);
+ auto l2 = [](auto a) ->decltype(auto) { return a; }(2);
+ auto l3 = [](auto a) { return a; }(2);
 
 }
 
@@ -268,10 +268,10 @@ int test() {
   M(4.15); //expected-note{{instantiation}}
  }
  {
-  int i = 10; 
+  int i = 10;
   auto L = [](auto a) {
-    return [](auto b) { 
-      b = sizeof(i);  //ok 
+    return [](auto b) {
+      b = sizeof(i);  //ok
       return b;
     };
   };
@@ -287,7 +287,7 @@ int test() {
   auto M = L(3);
   M(4.15);
  }
- 
+
 {
   auto L = [](auto a) ->decltype(a) {
     print("a = ", a, "\n");
@@ -321,7 +321,7 @@ int test() {
     };
   };
   auto M = L(3);
-  int (*fp)(double, int, const char*) = M; 
+  int (*fp)(double, int, const char*) = M;
   fp(4.15, 3, "fv");
 }
 
@@ -367,7 +367,7 @@ int test() {
 
 {
  struct X {
-  static void foo(double d) { } 
+  static void foo(double d) { }
   void test() {
     auto L = [](auto a) {
       print("a = ", a, "\n");
@@ -398,7 +398,7 @@ x.test();
 // Make sure we can escape the function
 {
  struct X {
-  static void foo(double d) { } 
+  static void foo(double d) { }
   auto test() {
     auto L = [](auto a) {
       print("a = ", a, "\n");
@@ -430,7 +430,7 @@ x.test();
 
 {
  struct X {
-  static void foo(double d) { } 
+  static void foo(double d) { }
   auto test() {
     auto L = [](auto a) {
       print("a = ", a, "\n");
@@ -464,7 +464,7 @@ x.test();
   char (*np)(const char*, int, const char*, double, const char*, int) = O;
   np("\n3 = ", 3, "\n6.14 = ", 6.14, "\n4'123'456 = ", 4'123'456);
   int (*np2)(const char*, int, const char*, double, const char*, int) = O; // expected-error{{no viable conversion}}
-  
+
 }
 } // end test()
 
@@ -472,17 +472,17 @@ namespace wrapped_within_templates {
 
 namespace explicit_return {
 template<class T> int fooT(T t) {
-  auto L = [](auto a) -> void { 
+  auto L = [](auto a) -> void {
     auto M = [](char b) -> void {
       auto N = [](auto c) -> void {
         int x = 0;
-        x = sizeof(a);        
+        x = sizeof(a);
         x = sizeof(b);
         x = sizeof(c);
-      };  
+      };
       N('a');
       N(decltype(a){});
-    };    
+    };
   };
   L(t);
   L(3.14);
@@ -495,17 +495,17 @@ int run = fooT('a') + fooT(3.14);
 
 namespace implicit_return_deduction {
 template<class T> auto fooT(T t) {
-  auto L = [](auto a)  { 
+  auto L = [](auto a)  {
     auto M = [](char b)  {
       auto N = [](auto c)  {
         int x = 0;
-        x = sizeof(a);        
+        x = sizeof(a);
         x = sizeof(b);
         x = sizeof(c);
-      };  
+      };
       N('a');
       N(decltype(a){});
-    };    
+    };
   };
   L(t);
   L(3.14);
@@ -517,20 +517,20 @@ int run = fooT('a') + fooT(3.14);
 template<class ... Ts> void print(Ts ... ts) { }
 
 template<class ... Ts> auto fooV(Ts ... ts) {
-  auto L = [](auto ... a) { 
-    auto M = [](decltype(a) ... b) {  
+  auto L = [](auto ... a) {
+    auto M = [](decltype(a) ... b) {
       auto N = [](auto c) {
         int x = 0;
-        x = sizeof...(a);        
+        x = sizeof...(a);
         x = sizeof...(b);
         x = sizeof(c);
-      };  
+      };
       N('a');
       N(N);
       N(first<Ts...>{});
     };
     M(a...);
-    print("a = ", a..., "\n");    
+    print("a = ", a..., "\n");
   };
   L(L, ts...);
   print("ts = ", ts..., "\n");
@@ -580,28 +580,28 @@ int (*np2)(const char*, int, const char*, double, const char*, int) = O; // expe
 
 
 
-} 
+}
 
 namespace variadic_tests_1 {
 template<class ... Ts> void print(Ts ... ts) { }
 
 template<class F, class ... Rest> F& FirstArg(F& f, Rest...) { return f; }
- 
+
 template<class ... Ts> int fooV(Ts ... ts) {
-  auto L = [](auto ... a) -> void { 
-    auto M = [](decltype(a) ... b) -> void {  
+  auto L = [](auto ... a) -> void {
+    auto M = [](decltype(a) ... b) -> void {
       auto N = [](auto c) -> void {
         int x = 0;
-        x = sizeof...(a);        
+        x = sizeof...(a);
         x = sizeof...(b);
         x = sizeof(c);
-      };  
+      };
       N('a');
       N(N);
       N(first<Ts...>{});
     };
     M(a...);
-    print("a = ", a..., "\n");    
+    print("a = ", a..., "\n");
   };
   L(L, ts...);
   print("ts = ", ts..., "\n");
@@ -613,14 +613,14 @@ int run2 = fooV(3.14, " ", '4', 5) + fooV("BC", 3, 2.77, 'A', float{}, short{},
 namespace more_variadic_1 {
 
 template<class ... Ts> int fooV(Ts ... ts) {
-  auto L = [](auto ... a) { 
-    auto M = [](decltype(a) ... b) -> void {  
+  auto L = [](auto ... a) {
+    auto M = [](decltype(a) ... b) -> void {
       auto N = [](auto c) -> void {
         int x = 0;
-        x = sizeof...(a);        
+        x = sizeof...(a);
         x = sizeof...(b);
         x = sizeof(c);
-      };  
+      };
       N('a');
       N(N);
       N(first<Ts...>{});
@@ -631,16 +631,16 @@ template<class ... Ts> int fooV(Ts ... ts) {
   auto M = L(L, ts...);
   decltype(L(L, ts...)) (*fp)(decltype(L), decltype(ts) ...) = L;
   void (*fp2)(decltype(L), decltype(ts) ...) = L(L, ts...);
-  
+
   {
-    auto L = [](auto ... a) { 
-      auto M = [](decltype(a) ... b) {  
+    auto L = [](auto ... a) {
+      auto M = [](decltype(a) ... b) {
         auto N = [](auto c) -> void {
           int x = 0;
-          x = sizeof...(a);        
+          x = sizeof...(a);
           x = sizeof...(b);
           x = sizeof(c);
-        };  
+        };
         N('a');
         N(N);
         N(first<Ts...>{});
@@ -669,7 +669,7 @@ int run2 = fooV(3.14, " ", '4', 5) + fooV("BC", 3, 2.77, 'A', float{}, short{},
 
 namespace at_ns_scope_within_class_member {
  struct X {
-  static void foo(double d) { } 
+  static void foo(double d) { }
   auto test() {
     auto L = [](auto a) {
       print("a = ", a, "\n");
@@ -703,13 +703,13 @@ auto O = N("\n3 = ", 3, "\n6.14 = ", 6.14, "\n4'123'456 = ", 4'123'456);
 char (*np)(const char*, int, const char*, double, const char*, int) = O;
 auto NP_result = np("\n3 = ", 3, "\n6.14 = ", 6.14, "\n4'123'456 = ", 4'123'456);
 int (*np2)(const char*, int, const char*, double, const char*, int) = O; // expected-error{{no viable conversion}}
-  
+
 } //end at_ns_scope_within_class_member
 
 
 namespace at_ns_scope_within_class_template_member {
  struct X {
-  static void foo(double d) { } 
+  static void foo(double d) { }
   template<class T = int>
   auto test(T = T{}) {
     auto L = [](auto a) {
@@ -733,7 +733,7 @@ namespace at_ns_scope_within_class_template_member {
     };
     return L;
   }
-  
+
 };
 X x;
 auto L = x.test();
@@ -745,24 +745,24 @@ auto O = N("\n3 = ", 3, "\n6.14 = ", 6.14, "\n4'123'456 = ", 4'123'456);
 char (*np)(const char*, int, const char*, double, const char*, int) = O;
 auto NP_result = np("\n3 = ", 3, "\n6.14 = ", 6.14, "\n4'123'456 = ", 4'123'456);
 int (*np2)(const char*, int, const char*, double, const char*, int) = O; // expected-error{{no viable conversion}}
-  
+
 } //end at_ns_scope_within_class_member
 
 
 namespace nested_generic_lambdas_123 {
 void test() {
   auto L = [](auto a) -> int {
-    auto M = [](auto b, decltype(a) b2) -> int { 
+    auto M = [](auto b, decltype(a) b2) -> int {
       return 1;
     };
     M(a, a);
   };
-  L(3); 
+  L(3);
 }
 template<class T> void foo(T) {
- auto L = [](auto a) { return a; }; 
+ auto L = [](auto a) { return a; };
 }
-template void foo(int); 
+template void foo(int);
 } // end ns nested_generic_lambdas_123
 
 namespace nested_fptr_235 {
@@ -796,7 +796,7 @@ int test()
 {
  {
    auto L = [](auto ... As) {
-    return [](auto b) ->decltype(b) {   
+    return [](auto b) ->decltype(b) {
       vfun([](decltype(As) a) -> decltype(a) { return a; } ...)(first<decltype(As)...>{});
       return decltype(b){};
     };
@@ -817,7 +817,7 @@ struct string {
   string &operator+=(const string &__str) { return *this; }
 };
 
-template <class T> 
+template <class T>
 void finalizeDefaultAtomValues() {
   auto startEnd = [](const char * sym) -> void {
     string start("__");
@@ -828,7 +828,7 @@ void finalizeDefaultAtomValues() {
 
 void f() { finalizeDefaultAtomValues<char>(); }
 
-} 
+}
 
 namespace PR17476_variant {
 struct string {
@@ -836,7 +836,7 @@ struct string {
   string &operator+=(const string &__str) { return *this; }
 };
 
-template <class T> 
+template <class T>
 void finalizeDefaultAtomValues() {
   auto startEnd = [](const T *sym) -> void {
     string start("__");
@@ -847,7 +847,7 @@ void finalizeDefaultAtomValues() {
 
 void f() { finalizeDefaultAtomValues<char>(); }
 
-} 
+}
 
 namespace PR17877_lambda_declcontext_and_get_cur_lambda_disconnect {
 
@@ -857,7 +857,7 @@ template<class T> struct U {
 };
 
 template<class T>
-struct V { 
+struct V {
   U<T> size() const { return U<T>{}; }
 };
 
@@ -872,7 +872,7 @@ void Do() {
 namespace inclass_lambdas_within_nested_classes {
 namespace ns1 {
 
-struct X1 {  
+struct X1 {
   struct X2 {
     enum { E = [](auto i) { return i; }(3) }; //expected-error{{inside of a constant expression}}\
                                           //expected-error{{constant}}\
@@ -883,7 +883,9 @@ struct X1 {
                                           //expected-error{{not an integral constant}}\
                                           //expected-note{{non-literal type}}
     int arr[([](int i) { return i; })(3)]; //expected-error{{inside of a constant expression}}\
-                                           //expected-error{{must have a constant size}}
+                                           //expected-error{{must have a constant size}}\
+                                           //expected-warning{{variable length arrays in C++ are a Clang extension}}\
+                                           //expected-note-re{{non-literal type '{{.*}}' cannot be used in a constant expression}}
     int (*fp)(int) = [](int i) { return i; };
     void fooptr(int (*fp)(char) = [](char c) { return 0; }) { }
     int L2 = ([](auto i) { return i; })(2);
@@ -892,7 +894,10 @@ struct X1 {
                                             //expected-error{{not an integral constant}}\
                                             //expected-note{{non-literal type}}
     int arrG[([](auto i) { return i; })(3)]; //expected-error{{inside of a constant expression}}\
-                                             //expected-error{{must have a constant size}}
+                                             //expected-error{{must have a constant size}}\
+                                             //expected-warning{{variable length arrays in C++ are a Clang extension}}\
+                                             //expected-note-re{{non-literal type '{{.*}}' cannot be used in a constant expression}}
+
     int (*fpG)(int) = [](auto i) { return i; };
     void fooptrG(int (*fp)(char) = [](auto c) { return 0; }) { }
   };
@@ -900,7 +905,7 @@ struct X1 {
 } //end ns
 
 namespace ns2 {
-struct X1 {  
+struct X1 {
   template<class T>
   struct X2 {
     int L = ([] (T i) { return i; })(2);
@@ -909,7 +914,10 @@ struct X1 {
                                         //expected-error{{not an integral constant}}\
                                         //expected-note{{non-literal type}}
     int arr[([](T i) { return i; })(3)]; //expected-error{{inside of a constant expression}}\
-                                         //expected-error{{must have a constant size}}
+                                         //expected-error{{must have a constant size}}\
+                                         //expected-warning{{variable length arrays in C++ are a Clang extension}}\
+                                         //expected-note-re{{non-literal type '{{.*}}' cannot be used in a constant expression}}
+
     int (*fp)(T) = [](T i) { return i; };
     void fooptr(T (*fp)(char) = [](char c) { return 0; }) { }
     int L2 = ([](auto i) { return i; })(2);
@@ -918,16 +926,19 @@ struct X1 {
                                             //expected-note{{non-literal type}}\
                                             //expected-error{{inside of a constant expression}}
     int arrG[([](auto i) { return i; })(3)]; //expected-error{{must have a constant size}} \
-                                             //expected-error{{inside of a constant expression}}
+                                             //expected-error{{inside of a constant expression}}\
+                                             //expected-warning{{variable length arrays in C++ are a Clang extension}}\
+                                             //expected-note-re{{non-literal type '{{.*}}' cannot be used in a constant expression}}
+
     int (*fpG)(T) = [](auto i) { return i; };
     void fooptrG(T (*fp)(char) = [](auto c) { return 0; }) { }
     template<class U = char> int fooG2(T (*fp)(U) = [](auto a) { return 0; }) { return 0; }
     template<class U = char> int fooG3(T (*fp)(U) = [](auto a) { return 0; });
   };
 };
-template<class T> 
+template<class T>
 template<class U>
-int X1::X2<T>::fooG3(T (*fp)(U)) { return 0; } 
+int X1::X2<T>::fooG3(T (*fp)(U)) { return 0; }
 X1::X2<int> x2; //expected-note {{in instantiation of}}
 int run1 = x2.fooG2();
 int run2 = x2.fooG3();
diff --git a/clang/test/SemaCXX/cxx1z-noexcept-function-type.cpp b/clang/test/SemaCXX/cxx1z-noexcept-function-type.cpp
index 6cbc6b69e8..11b1093f90 100644
--- a/clang/test/SemaCXX/cxx1z-noexcept-function-type.cpp
+++ b/clang/test/SemaCXX/cxx1z-noexcept-function-type.cpp
@@ -120,7 +120,9 @@ namespace Builtins {
   extern "C" int strncmp(const char *, const char *, decltype(sizeof(0))) noexcept;
 
   // Check we recognized both as builtins.
-  typedef int arr[strcmp("bar", "foo") + 4 * strncmp("foo", "bar", 4)]; // expected-warning {{variable length array}}
+  typedef int arr[strcmp("bar", "foo") + 4 * strncmp("foo", "bar", 4)]; // expected-warning {{variable length array folded to constant array as an extension}} \
+                                                                           expected-warning {{variable length arrays in C++ are a Clang extension}} \
+                                                                           expected-note {{non-constexpr function 'strcmp' cannot be used in a constant expression}}
   typedef int arr[3];
 }
 
diff --git a/clang/test/SemaCXX/cxx2a-consteval.cpp b/clang/test/SemaCXX/cxx2a-consteval.cpp
index 38cc4be32a..d8482ec53f 100644
--- a/clang/test/SemaCXX/cxx2a-consteval.cpp
+++ b/clang/test/SemaCXX/cxx2a-consteval.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -std=c++2a -emit-llvm-only -Wno-unused-value %s -verify
+// RUN: %clang_cc1 -std=c++2a -emit-llvm-only -Wno-unused-value -Wno-vla %s -verify
 
 typedef __SIZE_TYPE__ size_t;
 
diff --git a/clang/test/SemaCXX/cxx2a-constexpr-dynalloc-limits.cpp b/clang/test/SemaCXX/cxx2a-constexpr-dynalloc-limits.cpp
index 2178154b2e..8bcb4d45d4 100644
--- a/clang/test/SemaCXX/cxx2a-constexpr-dynalloc-limits.cpp
+++ b/clang/test/SemaCXX/cxx2a-constexpr-dynalloc-limits.cpp
@@ -87,10 +87,10 @@ constexpr int f = stack_array<1025>(); // expected-error {{constexpr variable 'f
                                        //  expected-note {{in call}}
 void ohno() {
   int bar[stack_array<1024>()];
-  int foo[stack_array<1025>()]; // expected-warning {{variable length arrays are a C99 feature}} \
+  int foo[stack_array<1025>()]; // expected-warning {{variable length arrays in C++ are a Clang extension}} \
                                 // expected-note {{in call to 'stack_array<1025>()'}}
 
-  constexpr int foo[stack_array<1025>()]; // expected-warning {{variable length arrays are a C99 feature}} \
+  constexpr int foo[stack_array<1025>()]; // expected-warning {{variable length arrays in C++ are a Clang extension}} \
                                           // expected-error {{constexpr variable cannot have non-literal type 'const int[stack_array<1025>()]'}} \
                                           // expected-note {{in call to 'stack_array<1025>()'}}
 }
diff --git a/clang/test/SemaCXX/exceptions.cpp b/clang/test/SemaCXX/exceptions.cpp
index 3a410b1bfe..c2e5adca3e 100644
--- a/clang/test/SemaCXX/exceptions.cpp
+++ b/clang/test/SemaCXX/exceptions.cpp
@@ -117,12 +117,12 @@ class foo {
 namespace PR6831 {
   namespace NA { struct S; }
   namespace NB { struct S; }
-  
+
   void f() {
     using namespace NA;
     using namespace NB;
     try {
-    } catch (int S) { 
+    } catch (int S) {
     }
   }
 }
@@ -276,15 +276,19 @@ void g() {
 }
 
 namespace PR28047 {
-void test1(int i) {
+void test1(int i) { // expected-note {{declared here}}
   try {
-  } catch (int(*)[i]) { // expected-error{{cannot catch variably modified type}}
+  } catch (int(*)[i]) { // expected-error{{cannot catch variably modified type}} \
+                           expected-warning {{variable length arrays in C++ are a Clang extension}} \
+                           expected-note {{function parameter 'i' with unknown value cannot be used in a constant expression}}
   }
 }
 void test2() {
-  int i;
+  int i; // expected-note {{declared here}}
   try {
-  } catch (int(*)[i]) { // expected-error{{cannot catch variably modified type}}
+  } catch (int(*)[i]) { // expected-error{{cannot catch variably modified type}} \
+                           expected-warning {{variable length arrays in C++ are a Clang extension}} \
+                           expected-note {{read of non-const variable 'i' is not allowed in a constant expression}}
   }
 }
 }
diff --git a/clang/test/SemaCXX/ext-int.cpp b/clang/test/SemaCXX/ext-int.cpp
index 4dd7cd3b7b..000b871ccd 100644
--- a/clang/test/SemaCXX/ext-int.cpp
+++ b/clang/test/SemaCXX/ext-int.cpp
@@ -211,8 +211,8 @@ void ConstexprBitsize() {
   static_assert(is_same<decltype(F), _BitInt(42)>::value, "");
 }
 
-// Useable as an underlying type.
-enum AsEnumUnderlyingType : _BitInt(33) {
+// Not useable as an underlying type.
+enum AsEnumUnderlyingType : _BitInt(33) { // expected-error{{'_BitInt(33)' is an invalid underlying type}}
 };
 
 void overloaded(int);
diff --git a/clang/test/SemaCXX/for-range-examples.cpp b/clang/test/SemaCXX/for-range-examples.cpp
index 1bbf042f0a..d129d50d67 100644
--- a/clang/test/SemaCXX/for-range-examples.cpp
+++ b/clang/test/SemaCXX/for-range-examples.cpp
@@ -241,19 +241,22 @@ namespace pr18587 {
 namespace PR32933 {
 // https://bugs.llvm.org/show_bug.cgi?id=32933
 void foo ()
-{ 
-  int b = 1, a[b];
+{
+  int b = 1, a[b]; // expected-warning {{variable length arrays in C++ are a Clang extension}} \
+                      expected-note {{read of non-const variable 'b' is not allowed in a constant expression}} \
+                      expected-note {{declared here}}
   a[0] = 0;
   [&] { for (int c : a) 0; } (); // expected-warning {{expression result unused}}
 }
 
 
-int foo(int b) {
-  int varr[b][(b+=8)];
-  b = 15; 
+int foo(int b) { // expected-note {{declared here}}
+  int varr[b][(b+=8)]; // expected-warning 2{{variable length arrays in C++ are a Clang extension}} \
+                          expected-note {{function parameter 'b' with unknown value cannot be used in a constant expression}}
+  b = 15;
   [&] {
     int i = 0;
-    for (auto &c : varr) 
+    for (auto &c : varr)
     {
       c[0] = ++b;
     }
diff --git a/clang/test/SemaCXX/i-c-e-cxx.cpp b/clang/test/SemaCXX/i-c-e-cxx.cpp
index 70a500ea89..5121769002 100644
--- a/clang/test/SemaCXX/i-c-e-cxx.cpp
+++ b/clang/test/SemaCXX/i-c-e-cxx.cpp
@@ -39,7 +39,7 @@ namespace pr6206 {
   };
 
   const int Foo::kBar = 20;
-  
+
   char Test() {
     char str[Foo::kBar];
     str[0] = '0';
@@ -54,7 +54,7 @@ void pr6373(const unsigned x = 0) {
 
 
 namespace rdar9204520 {
-  
+
 struct A {
   static const int B = int(0.75 * 1000 * 1000);
 #if __cplusplus <= 199711L
@@ -76,7 +76,7 @@ struct PR8836 { char _; long long a; };
 #endif
 
 int PR8836test[(__typeof(sizeof(int)))&reinterpret_cast<const volatile char&>((((PR8836*)0)->a))];
-// expected-warning@-1 0-1{{C99 feature}} expected-warning@-1 {{folded to constant array as an extension}}
+// expected-warning@-1 0-1{{variable length arrays in C++ are a Clang extension}} expected-warning@-1 {{folded to constant array as an extension}}
 // expected-note@-2 {{cast that performs the conversions of a reinterpret_cast is not allowed in a constant expression}}
 
 const int nonconst = 1.0;
@@ -85,7 +85,7 @@ const int nonconst = 1.0;
 #endif
 int arr[nonconst];
 #if __cplusplus <= 199711L
-// expected-warning@-2 0-1{{C99 feature}} expected-warning@-2 {{folded to constant array as an extension}}
+// expected-warning@-2 0-1{{variable length arrays in C++ are a Clang extension}} expected-warning@-2 {{folded to constant array as an extension}}
 // expected-note@-3 {{initializer of 'nonconst' is not a constant expression}}
 #endif
 
diff --git a/clang/test/SemaCXX/lambda-expressions.cpp b/clang/test/SemaCXX/lambda-expressions.cpp
index 9cf81a0a19..b4209d50ad 100644
--- a/clang/test/SemaCXX/lambda-expressions.cpp
+++ b/clang/test/SemaCXX/lambda-expressions.cpp
@@ -150,9 +150,10 @@ namespace Array {
   int &f(int *p);
   char &f(...);
   void g() {
-    int n = -1;
+    int n = -1;   // expected-note {{declared here}}
     [=] {
-      int arr[n]; // VLA
+      int arr[n]; // expected-warning {{variable length arrays in C++ are a Clang extension}} \
+                     expected-note {{read of non-const variable 'n' is not allowed in a constant expression}}
     } ();
 
     const int m = -1;
diff --git a/clang/test/SemaCXX/m68k-rtdcall.cpp b/clang/test/SemaCXX/m68k-rtdcall.cpp
new file mode 100644
index 0000000000..31f4bceafd
--- /dev/null
+++ b/clang/test/SemaCXX/m68k-rtdcall.cpp
@@ -0,0 +1,14 @@
+// RUN: %clang_cc1 -triple m68k-linux-gnu -fsyntax-only %s
+
+class A {
+public:
+  void __attribute__((m68k_rtd)) member() {}
+};
+
+void test() {
+  A a;
+  a.member();
+
+  auto f = [](int b) __attribute__((m68k_rtd)) {};
+  f(87);
+};
diff --git a/clang/test/SemaCXX/new-delete-0x.cpp b/clang/test/SemaCXX/new-delete-0x.cpp
index 972ad1d1b0..a4b43308d0 100644
--- a/clang/test/SemaCXX/new-delete-0x.cpp
+++ b/clang/test/SemaCXX/new-delete-0x.cpp
@@ -23,7 +23,9 @@ void bad_news(int *ip)
   auto t = new (int(*)[[]]); // expected-error {{an attribute list cannot appear here}}
   auto u = new (int(*)[[]{return 1;}()][2]); // expected-error {{C++11 only allows consecutive left square brackets when introducing an attribute}} \
                                                 expected-error {{variably modified type}} \
-                                                expected-error {{a lambda expression may not appear inside of a constant expression}}
+                                                expected-error {{a lambda expression may not appear inside of a constant expression}} \
+                                                expected-warning {{variable length arrays in C++ are a Clang extension}} \
+                                                expected-note-re {{non-literal type '(lambda at {{.*}})' cannot be used in a constant expression}}
 }
 
 void good_deletes()
diff --git a/clang/test/SemaCXX/new-delete.cpp b/clang/test/SemaCXX/new-delete.cpp
index 77c756e080..0270e42b73 100644
--- a/clang/test/SemaCXX/new-delete.cpp
+++ b/clang/test/SemaCXX/new-delete.cpp
@@ -644,4 +644,5 @@ int (*const_fold)[12] = new int[3][&const_fold + 12 - &const_fold];
 // expected-note@-3 {{cannot refer to element 12 of non-array}}
 #elif __cplusplus < 201103L
 // expected-error@-5 {{cannot allocate object of variably modified type}}
+// expected-warning@-6 {{variable length arrays in C++ are a Clang extension}}
 #endif
diff --git a/clang/test/SemaCXX/offsetof.cpp b/clang/test/SemaCXX/offsetof.cpp
index c4e86369f0..cb91f2bed0 100644
--- a/clang/test/SemaCXX/offsetof.cpp
+++ b/clang/test/SemaCXX/offsetof.cpp
@@ -1,5 +1,5 @@
 // RUN: %clang_cc1 -triple x86_64-apple-darwin10.0.0 -fsyntax-only -verify %s -Winvalid-offsetof -std=c++98
-// RUN: %clang_cc1 -triple x86_64-apple-darwin10.0.0 -fsyntax-only -verify %s -Winvalid-offsetof -std=c++98 -fexperimental-new-constant-interpreter
+// RUN: %clang_cc1 -triple x86_64-apple-darwin10.0.0 -fsyntax-only -verify=expected,new-interp %s -Winvalid-offsetof -std=c++98 -fexperimental-new-constant-interpreter
 
 struct NonPOD {
   virtual void f();
@@ -25,9 +25,10 @@ struct HasArray {
 };
 
 // Constant and non-constant offsetof expressions
-void test_ice(int i) {
+void test_ice(int i) { // new-interp-note {{declared here}}
   int array0[__builtin_offsetof(HasArray, array[5])];
-  int array1[__builtin_offsetof(HasArray, array[i])];
+  int array1[__builtin_offsetof(HasArray, array[i])]; // expected-warning {{variable length arrays in C++ are a Clang extension}} \
+                                                         new-interp-note {{function parameter 'i' with unknown value cannot be used in a constant expression}}
 }
 
 // Bitfields
@@ -39,7 +40,7 @@ struct has_bitfields {
 int test3 = __builtin_offsetof(struct has_bitfields, j); // expected-error{{cannot compute offset of bit-field 'j'}}
 
 // offsetof referring to members of a base class.
-struct Base1 { 
+struct Base1 {
   int x;
 };
 
@@ -48,7 +49,7 @@ struct Base2 {
 };
 
 struct Derived2 : public Base1, public Base2 {
-  int z; 
+  int z;
 };
 
 int derived1[__builtin_offsetof(Derived2, x) == 0? 1 : -1]; // expected-warning{{offset of on non-POD type 'Derived2'}}
diff --git a/clang/test/SemaCXX/paren-list-agg-init.cpp b/clang/test/SemaCXX/paren-list-agg-init.cpp
index 944ea76b81..f60b20e0d4 100644
--- a/clang/test/SemaCXX/paren-list-agg-init.cpp
+++ b/clang/test/SemaCXX/paren-list-agg-init.cpp
@@ -1,4 +1,6 @@
 // RUN: %clang_cc1 -verify -std=c++20 %s -fsyntax-only
+// RUN: %clang_cc1 -verify -std=c++20 %s -fsyntax-only -fexperimental-new-constant-interpreter
+// RUN: %clang_cc1 -verify=expected,beforecxx20 -Wc++20-extensions -std=c++20 %s -fsyntax-only -fexperimental-new-constant-interpreter
 // RUN: %clang_cc1 -verify=expected,beforecxx20 -Wc++20-extensions -std=c++20 %s -fsyntax-only
 
 struct A { // expected-note 4{{candidate constructor}}
@@ -90,7 +92,7 @@ T Construct(Args... args) {
   // beforecxx20-warning@-1 {{aggregate initialization of type 'A' from a parenthesized list of values is a C++20 extension}}
 }
 
-void foo(int n) {
+void foo(int n) { // expected-note {{declared here}}
   A a1(1954, 9, 21);
   // expected-error@-1 {{excess elements in struct initializer}}
   A a2(2.1);
@@ -173,8 +175,9 @@ void foo(int n) {
   int arr5[2](1, 2);
   // beforecxx20-warning@-1 {{aggregate initialization of type 'int[2]' from a parenthesized list of values is a C++20 extension}}
 
-  int arr6[n](1, 2, 3);
-  // expected-error@-1 {{variable-sized object may not be initialized}}
+  int arr6[n](1, 2, 3); // expected-warning {{variable length arrays in C++ are a Clang extension}} \
+                           expected-note {{function parameter 'n' with unknown value cannot be used in a constant expression}} \
+                           expected-error {{variable-sized object may not be initialized}}
 
   I i(1, 2);
   // expected-error@-1 {{no matching constructor for initialization of 'I'}}
diff --git a/clang/test/SemaCXX/type-traits.cpp b/clang/test/SemaCXX/type-traits.cpp
index a35689d529..c5d196a259 100644
--- a/clang/test/SemaCXX/type-traits.cpp
+++ b/clang/test/SemaCXX/type-traits.cpp
@@ -3160,11 +3160,18 @@ static_assert(!__is_trivially_equality_comparable(float), "");
 static_assert(!__is_trivially_equality_comparable(double), "");
 static_assert(!__is_trivially_equality_comparable(long double), "");
 
-struct TriviallyEqualityComparableNoDefaultedComparator {
+struct NonTriviallyEqualityComparableNoComparator {
   int i;
   int j;
 };
-static_assert(!__is_trivially_equality_comparable(TriviallyEqualityComparableNoDefaultedComparator), "");
+static_assert(!__is_trivially_equality_comparable(NonTriviallyEqualityComparableNoComparator), "");
+
+struct NonTriviallyEqualityComparableNonDefaultedComparator {
+  int i;
+  int j;
+  bool operator==(const NonTriviallyEqualityComparableNonDefaultedComparator&);
+};
+static_assert(!__is_trivially_equality_comparable(NonTriviallyEqualityComparableNonDefaultedComparator), "");
 
 #if __cplusplus >= 202002L
 
@@ -3177,7 +3184,7 @@ struct TriviallyEqualityComparable {
 
   bool operator==(const TriviallyEqualityComparable&) const = default;
 };
-static_assert(__is_trivially_equality_comparable(TriviallyEqualityComparable), "");
+static_assert(__is_trivially_equality_comparable(TriviallyEqualityComparable));
 
 struct TriviallyEqualityComparableContainsArray {
   int a[4];
@@ -3193,6 +3200,17 @@ struct TriviallyEqualityComparableContainsMultiDimensionArray {
 };
 static_assert(__is_trivially_equality_comparable(TriviallyEqualityComparableContainsMultiDimensionArray));
 
+auto GetNonCapturingLambda() { return [](){ return 42; }; }
+
+struct TriviallyEqualityComparableContainsLambda {
+  [[no_unique_address]] decltype(GetNonCapturingLambda()) l;
+  int i;
+
+  bool operator==(const TriviallyEqualityComparableContainsLambda&) const = default;
+};
+static_assert(!__is_trivially_equality_comparable(decltype(GetNonCapturingLambda()))); // padding
+static_assert(__is_trivially_equality_comparable(TriviallyEqualityComparableContainsLambda));
+
 struct TriviallyEqualityComparableNonTriviallyCopyable {
   TriviallyEqualityComparableNonTriviallyCopyable(const TriviallyEqualityComparableNonTriviallyCopyable&);
   ~TriviallyEqualityComparableNonTriviallyCopyable();
@@ -3984,12 +4002,6 @@ enum class UnscopedInt128 : __int128 {};
 enum class ScopedInt128 : __int128 {};
 enum class UnscopedUInt128 : unsigned __int128 {};
 enum class ScopedUInt128 : unsigned __int128 {};
-enum UnscopedBit : unsigned _BitInt(1) {};
-enum ScopedBit : unsigned _BitInt(1) {};
-enum UnscopedIrregular : _BitInt(21) {};
-enum UnscopedUIrregular : unsigned _BitInt(21) {};
-enum class ScopedIrregular : _BitInt(21) {};
-enum class ScopedUIrregular : unsigned _BitInt(21) {};
 
 void make_signed() {
   check_make_signed<char, signed char>();
@@ -4032,11 +4044,6 @@ void make_signed() {
   check_make_signed<UnscopedUInt128, __int128>();
   check_make_signed<ScopedUInt128, __int128>();
 
-  check_make_signed<UnscopedIrregular, _BitInt(21)>();
-  check_make_signed<UnscopedUIrregular, _BitInt(21)>();
-  check_make_signed<ScopedIrregular, _BitInt(21)>();
-  check_make_signed<ScopedUIrregular, _BitInt(21)>();
-
   { using ExpectedError = __make_signed(bool); }
   // expected-error@*:*{{'make_signed' is only compatible with non-bool integers and enum types, but was given 'bool'}}
   { using ExpectedError = __make_signed(UnscopedBool); }
@@ -4045,10 +4052,6 @@ void make_signed() {
   // expected-error@*:*{{'make_signed' is only compatible with non-bool integers and enum types, but was given 'ScopedBool' whose underlying type is 'bool'}}
   { using ExpectedError = __make_signed(unsigned _BitInt(1)); }
   // expected-error@*:*{{'make_signed' is only compatible with non-_BitInt(1) integers and enum types, but was given 'unsigned _BitInt(1)'}}
-  { using ExpectedError = __make_signed(UnscopedBit); }
-  // expected-error@*:*{{'make_signed' is only compatible with non-_BitInt(1) integers and enum types, but was given 'UnscopedBit' whose underlying type is 'unsigned _BitInt(1)'}}
-  { using ExpectedError = __make_signed(ScopedBit); }
-  // expected-error@*:*{{'make_signed' is only compatible with non-_BitInt(1) integers and enum types, but was given 'ScopedBit' whose underlying type is 'unsigned _BitInt(1)'}}
   { using ExpectedError = __make_signed(int[]); }
   // expected-error@*:*{{'make_signed' is only compatible with non-bool integers and enum types, but was given 'int[]'}}
   { using ExpectedError = __make_signed(int[5]); }
@@ -4129,11 +4132,6 @@ void make_unsigned() {
   check_make_unsigned<UnscopedUInt128, unsigned __int128>();
   check_make_unsigned<ScopedUInt128, unsigned __int128>();
 
-  check_make_unsigned<UnscopedIrregular, unsigned _BitInt(21)>();
-  check_make_unsigned<UnscopedUIrregular, unsigned _BitInt(21)>();
-  check_make_unsigned<ScopedIrregular, unsigned _BitInt(21)>();
-  check_make_unsigned<ScopedUIrregular, unsigned _BitInt(21)>();
-
   { using ExpectedError = __make_unsigned(bool); }
   // expected-error@*:*{{'make_unsigned' is only compatible with non-bool integers and enum types, but was given 'bool'}}
   { using ExpectedError = __make_unsigned(UnscopedBool); }
@@ -4142,10 +4140,6 @@ void make_unsigned() {
   // expected-error@*:*{{'make_unsigned' is only compatible with non-bool integers and enum types, but was given 'ScopedBool' whose underlying type is 'bool'}}
   { using ExpectedError = __make_unsigned(unsigned _BitInt(1)); }
   // expected-error@*:*{{'make_unsigned' is only compatible with non-_BitInt(1) integers and enum types, but was given 'unsigned _BitInt(1)'}}
-  { using ExpectedError = __make_unsigned(UnscopedBit); }
-  // expected-error@*:*{{'make_unsigned' is only compatible with non-_BitInt(1) integers and enum types, but was given 'UnscopedBit'}}
-  { using ExpectedError = __make_unsigned(ScopedBit); }
-  // expected-error@*:*{{'make_unsigned' is only compatible with non-_BitInt(1) integers and enum types, but was given 'ScopedBit'}}
   { using ExpectedError = __make_unsigned(int[]); }
   // expected-error@*:*{{'make_unsigned' is only compatible with non-bool integers and enum types, but was given 'int[]'}}
   { using ExpectedError = __make_unsigned(int[5]); }
diff --git a/clang/test/SemaCXX/vararg-non-pod.cpp b/clang/test/SemaCXX/vararg-non-pod.cpp
index a17eec0ab1..0c24ced40f 100644
--- a/clang/test/SemaCXX/vararg-non-pod.cpp
+++ b/clang/test/SemaCXX/vararg-non-pod.cpp
@@ -1,11 +1,11 @@
 // RUN: %clang_cc1 -fsyntax-only -verify -fblocks %s -Wno-error=non-pod-varargs
-// RUN: %clang_cc1 -fsyntax-only -verify -fblocks -std=c++98 %s -Wno-error=non-pod-varargs
-// RUN: %clang_cc1 -fsyntax-only -verify -fblocks -std=c++11 %s -Wno-error=non-pod-varargs
+// RUN: %clang_cc1 -fsyntax-only -verify -fblocks -std=gnu++98 %s -Wno-error=non-pod-varargs
+// RUN: %clang_cc1 -fsyntax-only -verify -fblocks -std=gnu++11 %s -Wno-error=non-pod-varargs
 
 // Check that the warning is still there under -fms-compatibility.
 // RUN: %clang_cc1 -fsyntax-only -verify -fblocks %s -Wno-error=non-pod-varargs -fms-compatibility
-// RUN: %clang_cc1 -fsyntax-only -verify -fblocks -std=c++98 %s -Wno-error=non-pod-varargs -fms-compatibility
-// RUN: %clang_cc1 -fsyntax-only -verify -fblocks -std=c++11 %s -Wno-error=non-pod-varargs -fms-compatibility
+// RUN: %clang_cc1 -fsyntax-only -verify -fblocks -std=gnu++98 %s -Wno-error=non-pod-varargs -fms-compatibility
+// RUN: %clang_cc1 -fsyntax-only -verify -fblocks -std=gnu++11 %s -Wno-error=non-pod-varargs -fms-compatibility
 
 extern char version[];
 
@@ -21,7 +21,7 @@ void g(int a, ...);
 void t1()
 {
   C c(10);
-  
+
   g(10, c);
 #if __cplusplus <= 199711L
   // expected-warning@-2 {{cannot pass object of non-POD type 'C' through variadic function; call will abort at runtime}}
@@ -56,7 +56,7 @@ void t2()
 #endif
 
   (c.*ptr)(10, version);
- 
+
   C::h(10, c);
 #if __cplusplus <= 199711L
   // expected-warning@-2 {{cannot pass object of non-POD type 'C' through variadic function; call will abort at runtime}}
@@ -64,7 +64,7 @@ void t2()
 
   C::h(10, version);
 
-  void (*static_ptr)(int, ...) = &C::h; 
+  void (*static_ptr)(int, ...) = &C::h;
   static_ptr(10, c);
 #if __cplusplus <= 199711L
   // expected-warning@-2 {{cannot pass object of non-POD type 'C' through variadic function; call will abort at runtime}}
@@ -78,7 +78,7 @@ int (^block)(int, ...);
 void t3()
 {
   C c(10);
-  
+
   block(10, c);
 #if __cplusplus <= 199711L
   // expected-warning@-2 {{cannot pass object of non-POD type 'C' through variadic block; call will abort at runtime}}
@@ -97,7 +97,7 @@ void t4()
   C c(10);
 
   D d;
-  
+
   d(10, c);
 #if __cplusplus <= 199711L
   // expected-warning@-2 {{cannot pass object of non-POD type 'C' through variadic method; call will abort at runtime}}
@@ -113,7 +113,7 @@ class E {
 void t5()
 {
   C c(10);
-  
+
   E e(10, c);
 #if __cplusplus <= 199711L
   // expected-warning@-2 {{cannot pass object of non-POD type 'C' through variadic constructor; call will abort at runtime}}
diff --git a/clang/test/SemaCXX/vla-construct.cpp b/clang/test/SemaCXX/vla-construct.cpp
index 5a7d833425..502e093514 100644
--- a/clang/test/SemaCXX/vla-construct.cpp
+++ b/clang/test/SemaCXX/vla-construct.cpp
@@ -1,9 +1,7 @@
-// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fcxx-exceptions -fexceptions -O0 -verify %s
-// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fcxx-exceptions -fexceptions -pedantic-errors -DPE -O0 -verify %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fcxx-exceptions -fexceptions -O0 -verify -Wno-vla %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fcxx-exceptions -fexceptions -pedantic-errors -O0 -verify=pe %s
 
-# ifndef PE
 // expected-no-diagnostics
-# endif
 
 extern "C" int printf(const char*, ...);
 
@@ -21,11 +19,7 @@ void print(int n, int a, int b, int c, int d) {
 }
 
 void test(int n) {
-  S array_t[n][n+1];
-# ifdef PE
-   // expected-error@-2 {{variable length arrays are a C99 feature}} expected-note@-2 {{parameter}} expected-note@-3 {{here}}
-   // expected-error@-3 {{variable length arrays are a C99 feature}} expected-note@-3 {{parameter}} expected-note@-4 {{here}}
-# endif
+  S array_t[n][n+1]; // pe-error 2{{variable length arrays in C++ are a Clang extension}} pe-note 2{{parameter}} pe-note@-1 2{{here}}
   int sizeof_S = sizeof(S);
   int sizeof_array_t_0_0 = sizeof(array_t[0][0]);
   int sizeof_array_t_0 = sizeof(array_t[0]);
diff --git a/clang/test/SemaCXX/vla-ext-diag.cpp b/clang/test/SemaCXX/vla-ext-diag.cpp
new file mode 100644
index 0000000000..7492bbae6c
--- /dev/null
+++ b/clang/test/SemaCXX/vla-ext-diag.cpp
@@ -0,0 +1,40 @@
+// RUN: %clang_cc1 -verify=gnu -std=gnu++11 %s
+// RUN: %clang_cc1 -verify=expected,cxx11 -Wvla -std=gnu++11 %s
+// RUN: %clang_cc1 -verify=expected,cxx11 -std=c++11 %s
+// RUN: %clang_cc1 -verify=expected,cxx98 -std=c++98 %s
+// RUN: %clang_cc1 -verify=expected,off -std=c++11 -Wno-vla-extension-static-assert %s
+// gnu-no-diagnostics
+
+// Demonstrate that we do not diagnose use of VLAs by default in GNU mode, but
+// we do diagnose them in C++ mode. Also note that we suggest use of
+// static_assert, but only in C++11 and later and only if the warning group is
+// not disabled.
+
+// C++98 mode does not emit the same notes as C++11 mode because in C++98,
+// we're looking for an integer constant expression, whereas in C++11 and later,
+// we're looking for a constant expression that is of integer type (these are
+// different operations; ICE looks at the syntactic form of the expression, but
+// C++11 constant expressions require calculating the expression value).
+void func(int n) { // cxx11-note {{declared here}} off-note {{declared here}}
+  int vla[n]; // expected-warning {{variable length arrays in C++ are a Clang extension}} \
+                 cxx11-note {{function parameter 'n' with unknown value cannot be used in a constant expression}} \
+                 off-note {{function parameter 'n' with unknown value cannot be used in a constant expression}}
+}
+
+void old_style_static_assert(int n) { // cxx11-note 5 {{declared here}} off-note 2 {{declared here}}
+  int array1[n != 12 ? 1 : -1]; // cxx11-warning {{variable length arrays in C++ are a Clang extension; did you mean to use 'static_assert'?}} \
+                                   cxx98-warning {{variable length arrays in C++ are a Clang extension}} \
+                                   cxx11-note {{function parameter 'n' with unknown value cannot be used in a constant expression}}
+  int array2[n != 12 ? -1 : 1]; // cxx11-warning {{variable length arrays in C++ are a Clang extension; did you mean to use 'static_assert'?}} \
+                                   cxx98-warning {{variable length arrays in C++ are a Clang extension}} \
+                                   cxx11-note {{function parameter 'n' with unknown value cannot be used in a constant expression}}
+  int array3[n != 12 ? 1 : n];  // expected-warning {{variable length arrays in C++ are a Clang extension}} \
+                                   cxx11-note {{function parameter 'n' with unknown value cannot be used in a constant expression}} \
+                                   off-note {{function parameter 'n' with unknown value cannot be used in a constant expression}}
+  int array4[(n ? 1 : -1)];     // cxx11-warning {{variable length arrays in C++ are a Clang extension; did you mean to use 'static_assert'?}} \
+                                   cxx98-warning {{variable length arrays in C++ are a Clang extension}} \
+                                   cxx11-note {{function parameter 'n' with unknown value cannot be used in a constant expression}}
+  int array5[n ? 1 : 0];        // expected-warning {{variable length arrays in C++ are a Clang extension}} \
+                                   cxx11-note {{function parameter 'n' with unknown value cannot be used in a constant expression}} \
+                                   off-note {{function parameter 'n' with unknown value cannot be used in a constant expression}}
+}
diff --git a/clang/test/SemaCXX/vla.cpp b/clang/test/SemaCXX/vla.cpp
index f7cae9e370..3657ab2d15 100644
--- a/clang/test/SemaCXX/vla.cpp
+++ b/clang/test/SemaCXX/vla.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple x86_64-apple-macosx10.14.0 -verify %s
+// RUN: %clang_cc1 -triple x86_64-apple-macosx10.14.0 -verify -Wno-vla %s
 
 // PR11925
 int n;
@@ -26,3 +26,18 @@ void test_fold() {
   char a2[(unsigned long)(int *)0+1] = {}; // expected-warning{{variable length array folded to constant array as an extension}}
   char a3[(unsigned long)(int *)0+1];
 }
+
+// Demonstrate that the check for a static_assert-like use of VLA does not
+// crash when there's no array size expression at all.
+void test_null_array_size_expr() {
+  int array1[]; // expected-error {{definition of variable with array type needs an explicit size or an initializer}}
+  int array2[] = { 1, 2, 3 };
+}
+
+// Show that the check for a static_assert-like use of a VLA properly handles a
+// dependent array size expression.
+template <typename Ty>
+void func(int expr) {
+  int array[sizeof(Ty) ? sizeof(Ty{}) : sizeof(int)];
+  int old_style_assert[expr ? Ty::one : Ty::Neg_one]; // We don't diagnose as a VLA until instantiation
+}
diff --git a/clang/test/SemaCXX/warn-thread-safety-analysis.cpp b/clang/test/SemaCXX/warn-thread-safety-analysis.cpp
index 8e312e589d..205cfa284f 100644
--- a/clang/test/SemaCXX/warn-thread-safety-analysis.cpp
+++ b/clang/test/SemaCXX/warn-thread-safety-analysis.cpp
@@ -5580,6 +5580,85 @@ class Bar {
   }
 };
 
+class Return {
+  Mutex mu;
+  Foo foo GUARDED_BY(mu);
+  Foo* foo_ptr PT_GUARDED_BY(mu);
+
+  Foo returns_value_locked() {
+    MutexLock lock(&mu);
+    return foo;
+  }
+
+  Foo returns_value_locks_required() EXCLUSIVE_LOCKS_REQUIRED(mu) {
+    return foo;
+  }
+
+  Foo returns_value_releases_lock_after_return() UNLOCK_FUNCTION(mu) {
+    MutexLock lock(&mu, true);
+    return foo;
+  }
+
+  Foo returns_value_aquires_lock() EXCLUSIVE_LOCK_FUNCTION(mu) {
+    mu.Lock();
+    return foo;
+  }
+  
+  Foo returns_value_not_locked() {
+    return foo;               // expected-warning {{reading variable 'foo' requires holding mutex 'mu'}}
+  }
+  
+  Foo returns_value_releases_lock_before_return() UNLOCK_FUNCTION(mu) {
+    mu.Unlock();
+    return foo;               // expected-warning {{reading variable 'foo' requires holding mutex 'mu'}}
+  }
+
+  Foo &returns_ref_not_locked() {
+    return foo;               // expected-warning {{returning variable 'foo' by reference requires holding mutex 'mu'}}
+  }
+
+  Foo &returns_ref_locked() {
+    MutexLock lock(&mu);
+    return foo;               // expected-warning {{returning variable 'foo' by reference requires holding mutex 'mu'}}
+  }
+
+  Foo &returns_ref_shared_locks_required() SHARED_LOCKS_REQUIRED(mu) {
+    return foo;               // expected-warning {{returning variable 'foo' by reference requires holding mutex 'mu' exclusively}}
+  }
+
+  Foo &returns_ref_exclusive_locks_required() EXCLUSIVE_LOCKS_REQUIRED(mu) {
+    return foo;
+  }
+
+  Foo &returns_ref_releases_lock_after_return() UNLOCK_FUNCTION(mu) {
+    MutexLock lock(&mu, true);
+    return foo;               // expected-warning {{returning variable 'foo' by reference requires holding mutex 'mu' exclusively}}
+  }
+
+  Foo& returns_ref_releases_lock_before_return() UNLOCK_FUNCTION(mu) {
+    mu.Unlock();
+    return foo;               // // expected-warning {{returning variable 'foo' by reference requires holding mutex 'mu' exclusively}}
+  }
+  
+  Foo &returns_ref_aquires_lock() EXCLUSIVE_LOCK_FUNCTION(mu) {
+    mu.Lock();
+    return foo;
+  }
+  
+  const Foo &returns_constref_shared_locks_required() SHARED_LOCKS_REQUIRED(mu) {
+    return foo;
+  }
+  
+  Foo *returns_ptr() {
+    return &foo;              // FIXME -- Do we want to warn on this ?
+  }
+
+  Foo &returns_ref2() {
+    return *foo_ptr;          // expected-warning {{returning the value that 'foo_ptr' points to by reference requires holding mutex 'mu' exclusively}}
+  }
+
+};
+
 
 }  // end namespace PassByRefTest
 
diff --git a/clang/test/SemaCXX/warn-unsafe-buffer-usage-debug-unclaimed/lit.local.cfg b/clang/test/SemaCXX/warn-unsafe-buffer-usage-debug-unclaimed/lit.local.cfg
new file mode 100644
index 0000000000..07bac415a1
--- /dev/null
+++ b/clang/test/SemaCXX/warn-unsafe-buffer-usage-debug-unclaimed/lit.local.cfg
@@ -0,0 +1,11 @@
+# -*- Python -*-
+
+config.substitutions.append(
+  (
+    "%analyze_safe_buffer_debug_notes",
+    "'%s' %s" % (
+      config.python_executable,
+      os.path.join(config.clang_src_dir, "utils", "analyze_safe_buffer_debug_notes.py")
+    )
+  )
+)
diff --git a/clang/test/SemaCXX/warn-unsafe-buffer-usage-debug-unclaimed/warn-unsafe-buffer-usage-debug-unclaimed.cpp b/clang/test/SemaCXX/warn-unsafe-buffer-usage-debug-unclaimed/warn-unsafe-buffer-usage-debug-unclaimed.cpp
new file mode 100644
index 0000000000..ab3d925753
--- /dev/null
+++ b/clang/test/SemaCXX/warn-unsafe-buffer-usage-debug-unclaimed/warn-unsafe-buffer-usage-debug-unclaimed.cpp
@@ -0,0 +1,49 @@
+// RUN: %clang_cc1 -Wno-unused-value -Wunsafe-buffer-usage -fsafe-buffer-usage-suggestions \
+// RUN:            -mllvm -debug-only=SafeBuffers \
+// RUN:            -std=c++20 -verify=expected %s
+
+// RUN: %clang_cc1 -Wno-unused-value -Wunsafe-buffer-usage -fsafe-buffer-usage-suggestions \
+// RUN:            -mllvm -debug-only=SafeBuffers \
+// RUN:            -std=c++20 %s                  \
+// RUN:            2>&1 | grep 'The unclaimed DRE trace:' \
+// RUN:                 | sed 's/^The unclaimed DRE trace://' \
+// RUN:                 | %analyze_safe_buffer_debug_notes \
+// RUN:                 | FileCheck %s
+
+// This debugging facility is only available in debug builds.
+//
+// REQUIRES: asserts
+// REQUIRES: shell
+
+void test_unclaimed_use(int *p) { // expected-warning{{'p' is an unsafe pointer used for buffer access}}
+  p++;           //  expected-note{{used in pointer arithmetic here}} \
+                     expected-note{{safe buffers debug: failed to produce fixit for 'p' : has an unclaimed use\n \
+ The unclaimed DRE trace: DeclRefExpr, UnaryOperator(++), CompoundStmt}}
+  *((p + 1) + 1); // expected-warning{{unsafe pointer arithmetic}}                      \
+                     expected-note{{used in pointer arithmetic here}}			\
+		     expected-note{{safe buffers debug: failed to produce fixit for 'p' : has an unclaimed use\n \
+  The unclaimed DRE trace: DeclRefExpr, ImplicitCastExpr(LValueToRValue), BinaryOperator(+), ParenExpr, BinaryOperator(+), ParenExpr, UnaryOperator(*), CompoundStmt}}
+  p -= 1;         // expected-note{{used in pointer arithmetic here}} \
+		     expected-note{{safe buffers debug: failed to produce fixit for 'p' : has an unclaimed use\n \
+  The unclaimed DRE trace: DeclRefExpr, BinaryOperator(-=), CompoundStmt}}
+  p--;            // expected-note{{used in pointer arithmetic here}} \
+ 		     expected-note{{safe buffers debug: failed to produce fixit for 'p' : has an unclaimed use\n \
+  The unclaimed DRE trace: DeclRefExpr, UnaryOperator(--), CompoundStmt}}
+  p[5] = 5;       // expected-note{{used in buffer access here}}
+}
+
+// CHECK: Root # 1
+// CHECK: |- DeclRefExpr # 4
+// CHECK: |-- UnaryOperator(++) # 1
+// CHECK: |--- CompoundStmt # 1
+// CHECK: |-- ImplicitCastExpr(LValueToRValue) # 1
+// CHECK: |--- BinaryOperator(+) # 1
+// CHECK: |---- ParenExpr # 1
+// CHECK: |----- BinaryOperator(+) # 1
+// CHECK: |------ ParenExpr # 1
+// CHECK: |------- UnaryOperator(*) # 1
+// CHECK: |-------- CompoundStmt # 1
+// CHECK: |-- BinaryOperator(-=) # 1
+// CHECK: |--- CompoundStmt # 1
+// CHECK: |-- UnaryOperator(--) # 1
+// CHECK: |--- CompoundStmt # 1
diff --git a/clang/test/SemaCXX/warn-unused-lambda-capture.cpp b/clang/test/SemaCXX/warn-unused-lambda-capture.cpp
index 67b8fc3b66..4214a3c6ce 100644
--- a/clang/test/SemaCXX/warn-unused-lambda-capture.cpp
+++ b/clang/test/SemaCXX/warn-unused-lambda-capture.cpp
@@ -193,9 +193,10 @@ void test_use_template() {
 }
 
 namespace pr35555 {
-int a;
+int a; // expected-note {{declared here}}
 void b() {
-  int c[a];
+  int c[a]; // expected-warning {{variable length arrays in C++ are a Clang extension}} \
+               expected-note {{read of non-const variable 'a' is not allowed in a constant expression}}
   auto vla_used = [&c] { return c[0]; };
   auto vla_unused = [&c] {}; // expected-warning{{lambda capture 'c' is not used}}
 }
diff --git a/clang/test/SemaCXX/warn-unused-variables.cpp b/clang/test/SemaCXX/warn-unused-variables.cpp
index 1d5aa729dd..b649c7d808 100644
--- a/clang/test/SemaCXX/warn-unused-variables.cpp
+++ b/clang/test/SemaCXX/warn-unused-variables.cpp
@@ -1,5 +1,5 @@
 // RUN: %clang_cc1 -fsyntax-only -Wunused-variable -Wunused-label -Wno-c++1y-extensions -verify %s
-// RUN: %clang_cc1 -fsyntax-only -Wunused-variable -Wunused-label -Wno-c++1y-extensions -verify -std=c++11 %s
+// RUN: %clang_cc1 -fsyntax-only -Wunused-variable -Wunused-label -Wno-c++1y-extensions -verify -std=gnu++11 %s
 template<typename T> void f() {
   T t;
   t = 17;
diff --git a/clang/test/SemaObjCXX/properties.mm b/clang/test/SemaObjCXX/properties.mm
index 1c4e1470be..a3d67d9a36 100644
--- a/clang/test/SemaObjCXX/properties.mm
+++ b/clang/test/SemaObjCXX/properties.mm
@@ -35,7 +35,8 @@ @interface Test3
 - (int) length;
 @end
 void test3(Test3 *t) {
-  char vla[t.length] = {}; // expected-error {{variable-sized object may not be initialized}}
+  char vla[t.length] = {}; // expected-error {{variable-sized object may not be initialized}} \
+                              expected-warning {{variable length arrays in C++ are a Clang extension}}
   char *heaparray = new char[t.length];
 }
 
diff --git a/clang/test/SemaObjCXX/vla.mm b/clang/test/SemaObjCXX/vla.mm
index e1d556e992..7ee15743ef 100644
--- a/clang/test/SemaObjCXX/vla.mm
+++ b/clang/test/SemaObjCXX/vla.mm
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -fsyntax-only -verify %s
+// RUN: %clang_cc1 -fsyntax-only -verify -Wno-vla %s
 // expected-no-diagnostics
 
 @interface Data
diff --git a/clang/test/SemaSYCL/unique_stable_name.cpp b/clang/test/SemaSYCL/unique_stable_name.cpp
index 2a0230d061..fb3b0dbe9e 100644
--- a/clang/test/SemaSYCL/unique_stable_name.cpp
+++ b/clang/test/SemaSYCL/unique_stable_name.cpp
@@ -173,7 +173,8 @@ void f() {
   // an unevaluated context because the use within a VLA extent forces
   // evaluation.
   int j = 55;
-  __builtin_sycl_unique_stable_name(int[++j]); // no warning expected
+  __builtin_sycl_unique_stable_name(int[++j]); // expected-warning {{variable length arrays in C++ are a Clang extension}} \
+                                                  expected-note {{a constant expression cannot modify an object that is visible outside that expression}}
 }
 
 template <typename T>
diff --git a/clang/test/SemaTemplate/concepts-out-of-line-def.cpp b/clang/test/SemaTemplate/concepts-out-of-line-def.cpp
index f067c02ca4..f134394615 100644
--- a/clang/test/SemaTemplate/concepts-out-of-line-def.cpp
+++ b/clang/test/SemaTemplate/concepts-out-of-line-def.cpp
@@ -1,5 +1,4 @@
 // RUN: %clang_cc1 -std=c++20 -verify %s
-// expected-no-diagnostics
 
 static constexpr int PRIMARY = 0;
 static constexpr int SPECIALIZATION_CONCEPT = 1;
@@ -227,7 +226,9 @@ namespace constrained_member_sfinae {
 
 template<int N> struct S {
   template<class T>
-  static constexpr int constrained_method() requires (sizeof(int[N * 1073741824 + 4]) == 16) {
+  static constexpr int constrained_method() requires (sizeof(int[N * 1073741824 + 4]) == 16) { // expected-warning {{variable length arrays in C++ are a Clang extension}} \
+                                                                                                  expected-note {{value 4294967296 is outside the range of representable values of type 'int'}} \
+                                                                                                  expected-note {{while calculating associated constraint of template 'constrained_method' here}}
     return CONSTRAINED_METHOD_1;
   }
 
@@ -466,3 +467,40 @@ int Outermost<T>::Middle::Innermost<U>::func(Param param) const {
 }
 
 } // namespace GH65810
+
+namespace GH61763 {
+template<typename T, typename U>
+concept same_as = true;
+
+template <class = void>
+struct Foo {
+      template <same_as<void> Param>
+            friend struct Bar;
+};
+
+template struct Foo<>;
+
+template <same_as<void> Param>
+struct Bar {
+};
+
+
+template<typename T>
+concept ok = true;
+
+struct outer {
+    template<typename T>
+        requires ok<T>
+          struct foo {};
+};
+
+template<typename U>
+struct bar {
+    template<typename T>
+        requires ok<T>
+          friend struct outer::foo;
+};
+
+bar<int> x;
+} // namespace GH61763
+
diff --git a/clang/test/SemaTemplate/deduction.cpp b/clang/test/SemaTemplate/deduction.cpp
index 2556fe86e1..ad98de61d9 100644
--- a/clang/test/SemaTemplate/deduction.cpp
+++ b/clang/test/SemaTemplate/deduction.cpp
@@ -3,7 +3,7 @@
 // RUN: %clang_cc1 -fsyntax-only -verify %s -std=c++23
 
 // Template argument deduction with template template parameters.
-template<typename T, template<T> class A> 
+template<typename T, template<T> class A>
 struct X0 {
   static const unsigned value = 0;
 };
@@ -192,8 +192,9 @@ void g() {
 
 namespace test8 {
 template <class T> void foo(T);
-void test(int a) {
-    char n[a];
+void test(int a) { // expected-note {{declared here}}
+    char n[a]; // expected-warning {{variable length arrays in C++ are a Clang extension}} \
+                  expected-note {{function parameter 'a' with unknown value cannot be used in a constant expression}}
     foo(n);
 }
 } // namespace test8
@@ -310,7 +311,7 @@ int main() {
   get_helper<double>(t);
   return 0;
 }
-} // end ns2 
+} // end ns2
 }
 
 namespace multiple_deduction_different_type {
diff --git a/clang/test/SemaTemplate/dependent-expr.cpp b/clang/test/SemaTemplate/dependent-expr.cpp
index 51bd375d79..ce210d9b74 100644
--- a/clang/test/SemaTemplate/dependent-expr.cpp
+++ b/clang/test/SemaTemplate/dependent-expr.cpp
@@ -165,3 +165,18 @@ namespace BindingInStmtExpr {
   using U = decltype(num_bindings<T>()); // expected-note {{previous}}
   using U = N<3>; // expected-error-re {{type alias redefinition with different types ('N<3>' vs {{.*}}N<2>}}
 }
+
+namespace PR65153 {
+struct A{};
+
+template <const A& T>
+const A JoinStringViews = T;
+
+template <int V>
+class Builder {
+public:
+    static constexpr A Equal{};
+    // no crash here
+    static constexpr auto Val = JoinStringViews<Equal>;
+};
+} // namespace PR65153
diff --git a/clang/test/SemaTemplate/function-decl-nested-type-alias.cpp b/clang/test/SemaTemplate/function-decl-nested-type-alias.cpp
new file mode 100644
index 0000000000..4bca990f69
--- /dev/null
+++ b/clang/test/SemaTemplate/function-decl-nested-type-alias.cpp
@@ -0,0 +1,16 @@
+// RUN: %clang_cc1 -x c++ -std=c++14 -fsyntax-only -verify %s
+
+template <class A>
+using Type = typename A::NestedType; // expected-error {{type 'float' cannot be used prior to '::' because it has no members}}
+
+template <typename T>
+void Func() {
+  using MyType = Type<T>(); // expected-note {{in instantiation of template type alias 'Type' requested here}}
+  // This is a function declaration, not a variable declaration!
+  // After substitution, we do not have a valid function type, and used to crash.
+  MyType var;
+}
+
+void Test() {
+  Func<float>(); // expected-note {{in instantiation of function template specialization 'Func<float>' requested here}}
+}
\ No newline at end of file
diff --git a/clang/test/SemaTemplate/instantiate-sizeof.cpp b/clang/test/SemaTemplate/instantiate-sizeof.cpp
index 660e70549e..0931ef2335 100644
--- a/clang/test/SemaTemplate/instantiate-sizeof.cpp
+++ b/clang/test/SemaTemplate/instantiate-sizeof.cpp
@@ -1,12 +1,14 @@
 // RUN: %clang_cc1 -triple x86_64-linux-gnu -fsyntax-only -verify -std=c++11 %s
 
 // Make sure we handle contexts correctly with sizeof
-template<typename T> void f(T n) {
-  int buffer[n];
+template<typename T> void f(T n) { // expected-note {{declared here}}
+  int buffer[n]; // expected-warning {{variable length arrays in C++ are a Clang extension}} \
+                    expected-note {{function parameter 'n' with unknown value cannot be used in a constant expression}} \
+                    expected-note@#instantiate {{in instantiation of function template specialization 'f<int>' requested here}}
   [] { int x = sizeof(sizeof(buffer)); }();
 }
 int main() {
-  f<int>(1);
+  f<int>(1); // #instantiate
 }
 
 // Make sure we handle references to non-static data members in unevaluated
diff --git a/clang/test/SemaTemplate/instantiate-typeof.cpp b/clang/test/SemaTemplate/instantiate-typeof.cpp
index f4d7847e4f..bc6027fad2 100644
--- a/clang/test/SemaTemplate/instantiate-typeof.cpp
+++ b/clang/test/SemaTemplate/instantiate-typeof.cpp
@@ -1,11 +1,11 @@
 // RUN: %clang_cc1 -fsyntax-only -verify -std=c++11 %s
-// expected-no-diagnostics
 
 // Make sure we correctly treat __typeof as potentially-evaluated when appropriate
-template<typename T> void f(T n) {
-  int buffer[n];
+template<typename T> void f(T n) { // expected-note {{declared here}}
+  int buffer[n]; // expected-warning {{variable length arrays in C++ are a Clang extension}} \
+                    expected-note {{function parameter 'n' with unknown value cannot be used in a constant expression}}
   [&buffer] { __typeof(buffer) x; }();
 }
 int main() {
-  f<int>(1);
+  f<int>(1); // expected-note {{in instantiation of function template specialization 'f<int>' requested here}}
 }
diff --git a/clang/tools/clang-format/CMakeLists.txt b/clang/tools/clang-format/CMakeLists.txt
index 30c93f8667..1c61a3c8fb 100644
--- a/clang/tools/clang-format/CMakeLists.txt
+++ b/clang/tools/clang-format/CMakeLists.txt
@@ -38,3 +38,9 @@ install(FILES clang-format.py
 install(PROGRAMS git-clang-format
   DESTINATION "${CMAKE_INSTALL_BINDIR}"
   COMPONENT clang-format)
+
+if (WIN32 AND NOT CYGWIN)
+  install(PROGRAMS git-clang-format.bat
+    DESTINATION "${CMAKE_INSTALL_BINDIR}"
+    COMPONENT clang-format)
+endif()
diff --git a/clang/tools/clang-format/git-clang-format.bat b/clang/tools/clang-format/git-clang-format.bat
new file mode 100644
index 0000000000..d4bc517298
--- /dev/null
+++ b/clang/tools/clang-format/git-clang-format.bat
@@ -0,0 +1 @@
+py -3 git-clang-format %*
diff --git a/clang/tools/clang-linker-wrapper/LinkerWrapperOpts.td b/clang/tools/clang-linker-wrapper/LinkerWrapperOpts.td
index abab4d0b39..2ce6ee0d21 100644
--- a/clang/tools/clang-linker-wrapper/LinkerWrapperOpts.td
+++ b/clang/tools/clang-linker-wrapper/LinkerWrapperOpts.td
@@ -95,7 +95,7 @@ def offload_opt_eq_minus : Joined<["--", "-"], "offload-opt=-">, Flags<[HelpHidd
   HelpText<"Options passed to LLVM">;
 
 // Standard linker flags also used by the linker wrapper.
-def sysroot_EQ : Joined<["--"], "sysroot">, HelpText<"Set the system root">;
+def sysroot_EQ : Joined<["--"], "sysroot=">, HelpText<"Set the system root">;
 
 def o : JoinedOrSeparate<["-"], "o">, MetaVarName<"<path>">,
   HelpText<"Path to file to write output">;
diff --git a/clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp b/clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp
index 90c475e541..7ad6c19482 100644
--- a/clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp
+++ b/clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp
@@ -19,7 +19,6 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Object/Archive.h"
 #include "llvm/Object/ArchiveWriter.h"
diff --git a/clang/tools/libclang/CXType.cpp b/clang/tools/libclang/CXType.cpp
index eb8bfc25a7..3d620d3bfb 100644
--- a/clang/tools/libclang/CXType.cpp
+++ b/clang/tools/libclang/CXType.cpp
@@ -678,6 +678,7 @@ CXCallingConv clang_getFunctionTypeCallingConv(CXType X) {
       TCALLINGCONV(SwiftAsync);
       TCALLINGCONV(PreserveMost);
       TCALLINGCONV(PreserveAll);
+      TCALLINGCONV(M68kRTD);
     case CC_SpirFunction: return CXCallingConv_Unexposed;
     case CC_AMDGPUKernelCall: return CXCallingConv_Unexposed;
     case CC_OpenCLKernel: return CXCallingConv_Unexposed;
diff --git a/clang/unittests/AST/ASTImporterTest.cpp b/clang/unittests/AST/ASTImporterTest.cpp
index 393ed44de3..f1f09a0be2 100644
--- a/clang/unittests/AST/ASTImporterTest.cpp
+++ b/clang/unittests/AST/ASTImporterTest.cpp
@@ -12,7 +12,6 @@
 
 #include "clang/AST/RecordLayout.h"
 #include "clang/ASTMatchers/ASTMatchers.h"
-#include "llvm/ADT/StringMap.h"
 #include "llvm/Support/SmallVectorMemoryBuffer.h"
 
 #include "clang/AST/DeclContextInternals.h"
@@ -25,8 +24,6 @@ namespace clang {
 namespace ast_matchers {
 
 using internal::Matcher;
-using internal::BindableMatcher;
-using llvm::StringMap;
 
 static const RecordDecl *getRecordDeclOfFriend(FriendDecl *FD) {
   QualType Ty = FD->getFriendType()->getType().getCanonicalType();
@@ -9178,6 +9175,64 @@ TEST_P(ASTImporterOptionSpecificTestBase,
   EXPECT_TRUE(ToXType->typeMatchesDecl());
 }
 
+TEST_P(ASTImporterOptionSpecificTestBase,
+       ImportTemplateArgumentWithPointerToDifferentInstantiation) {
+  const char *CodeTo =
+      R"(
+      template<class A>
+      A f1() {
+       return A();
+      }
+      template<class A, A (B)()>
+      class X {};
+
+      X<int, f1<int>> x;
+      )";
+  const char *CodeFrom =
+      R"(
+      template<class A>
+      A f1();
+      template<class A, A (B)()>
+      class X {};
+
+      X<int, f1<int>> x;
+      )";
+  Decl *ToTU = getToTuDecl(CodeTo, Lang_CXX11);
+  Decl *FromTU = getTuDecl(CodeFrom, Lang_CXX11);
+
+  auto *ToF1 = FirstDeclMatcher<FunctionDecl>().match(
+      ToTU, functionDecl(hasName("f1"), isInstantiated()));
+  auto *FromF1 = FirstDeclMatcher<FunctionDecl>().match(
+      FromTU, functionDecl(hasName("f1"), isInstantiated()));
+  EXPECT_TRUE(ToF1->isThisDeclarationADefinition());
+  EXPECT_FALSE(FromF1->isThisDeclarationADefinition());
+
+  auto *ToX = FirstDeclMatcher<ClassTemplateSpecializationDecl>().match(
+      ToTU, classTemplateSpecializationDecl(hasName("X")));
+  auto *FromX = FirstDeclMatcher<ClassTemplateSpecializationDecl>().match(
+      FromTU, classTemplateSpecializationDecl(hasName("X")));
+
+  Decl *ToTArgF = ToX->getTemplateArgs().get(1).getAsDecl();
+  Decl *FromTArgF = FromX->getTemplateArgs().get(1).getAsDecl();
+  EXPECT_EQ(ToTArgF, ToF1);
+  EXPECT_EQ(FromTArgF, FromF1);
+
+  auto *ToXImported = Import(FromX, Lang_CXX11);
+  // The template argument 1 of 'X' in the "From" code points to a function
+  // that has no definition. The import must ensure that this template argument
+  // is imported in a way that it will point to the existing 'f1' function, not
+  // to the 'f1' that is imported. In this way when specialization of 'X' is
+  // imported it will have the same template arguments as the existing one.
+  EXPECT_EQ(ToXImported, ToX);
+  // FIXME: This matcher causes a crash "Tried to match orphan node".
+  // The code is removed until the problem is fixed.
+  // auto *ToF1Imported =
+  //    LastDeclMatcher<FunctionDecl>().match(ToTU,
+  //    functionDecl(hasName("f1"),isInstantiated()));
+  // EXPECT_NE(ToF1Imported, ToF1);
+  // EXPECT_EQ(ToF1Imported->getPreviousDecl(), ToF1);
+}
+
 INSTANTIATE_TEST_SUITE_P(ParameterizedTests, ASTImporterLookupTableTest,
                          DefaultTestValuesForRunOptions);
 
diff --git a/clang/unittests/AST/CMakeLists.txt b/clang/unittests/AST/CMakeLists.txt
index 12484be920..584cfb1bd2 100644
--- a/clang/unittests/AST/CMakeLists.txt
+++ b/clang/unittests/AST/CMakeLists.txt
@@ -5,6 +5,8 @@ set(LLVM_LINK_COMPONENTS
   )
 
 
+add_subdirectory(Interp)
+
 add_clang_unittest(ASTTests
   ASTContextParentMapTest.cpp
   ASTExprTest.cpp
diff --git a/clang/unittests/AST/Interp/CMakeLists.txt b/clang/unittests/AST/Interp/CMakeLists.txt
new file mode 100644
index 0000000000..8fa5c85064
--- /dev/null
+++ b/clang/unittests/AST/Interp/CMakeLists.txt
@@ -0,0 +1,18 @@
+add_clang_unittest(InterpTests
+  Descriptor.cpp
+  )
+
+clang_target_link_libraries(InterpTests
+  PRIVATE
+  clangAST
+  clangASTMatchers
+  clangBasic
+  clangFrontend
+  clangSerialization
+  clangTooling
+  )
+
+  target_link_libraries(InterpTests
+  PRIVATE
+  clangTesting
+)
diff --git a/clang/unittests/AST/Interp/Descriptor.cpp b/clang/unittests/AST/Interp/Descriptor.cpp
new file mode 100644
index 0000000000..fb1690a970
--- /dev/null
+++ b/clang/unittests/AST/Interp/Descriptor.cpp
@@ -0,0 +1,385 @@
+#include "../../../lib/AST/Interp/Descriptor.h"
+#include "../../../lib/AST/Interp/Context.h"
+#include "../../../lib/AST/Interp/Program.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/Decl.h"
+#include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/ASTMatchers/ASTMatchers.h"
+#include "clang/Tooling/Tooling.h"
+#include "gtest/gtest.h"
+
+using namespace clang;
+using namespace clang::interp;
+using namespace clang::ast_matchers;
+
+/// Inspect generated Descriptors as well as the pointers we create.
+///
+TEST(Descriptor, Primitives) {
+  constexpr char Code[] =
+      "struct A { bool a; bool b; };\n"
+      "struct S {\n"
+      "  float f;\n"
+      "  char s[4];\n"
+      "  A a[3];\n"
+      "  short l[3][3];\n"
+      "};\n"
+      "constexpr S d = {0.0, \"foo\", {{true, false}, {false, true}, {false, false}},\n"
+      "  {{1, 2, 3}, {4, 5, 6}, {7, 8, 9}}};\n";
+
+  auto AST = tooling::buildASTFromCodeWithArgs(
+      Code, {"-fexperimental-new-constant-interpreter"});
+
+  const VarDecl *D = selectFirst<VarDecl>(
+      "d", match(varDecl().bind("d"), AST->getASTContext()));
+  ASSERT_NE(D, nullptr);
+
+  const auto &Ctx = AST->getASTContext().getInterpContext();
+  Program &Prog = Ctx.getProgram();
+  // Global is registered.
+  ASSERT_TRUE(Prog.getGlobal(D));
+
+  // Get a Pointer to the global.
+  const Pointer &GlobalPtr = Prog.getPtrGlobal(*Prog.getGlobal(D));
+
+  // Test Descriptor of the struct S.
+  const Descriptor *GlobalDesc = GlobalPtr.getFieldDesc();
+  ASSERT_TRUE(GlobalDesc == GlobalPtr.getDeclDesc());
+
+  ASSERT_TRUE(GlobalDesc->asDecl() == D);
+  ASSERT_FALSE(GlobalDesc->asExpr());
+  ASSERT_TRUE(GlobalDesc->asValueDecl() == D);
+  ASSERT_FALSE(GlobalDesc->asFieldDecl());
+  ASSERT_FALSE(GlobalDesc->asRecordDecl());
+
+  // Still true because this is a global variable.
+  ASSERT_TRUE(GlobalDesc->getMetadataSize() == 0);
+  ASSERT_FALSE(GlobalDesc->isPrimitiveArray());
+  ASSERT_FALSE(GlobalDesc->isCompositeArray());
+  ASSERT_FALSE(GlobalDesc->isZeroSizeArray());
+  ASSERT_FALSE(GlobalDesc->isUnknownSizeArray());
+  ASSERT_FALSE(GlobalDesc->isPrimitive());
+  ASSERT_FALSE(GlobalDesc->isArray());
+  ASSERT_TRUE(GlobalDesc->isRecord());
+
+  // Test the Record for the struct S.
+  const Record *SRecord = GlobalDesc->ElemRecord;
+  ASSERT_TRUE(SRecord);
+  ASSERT_TRUE(SRecord->getNumFields() == 4);
+  ASSERT_TRUE(SRecord->getNumBases() == 0);
+  ASSERT_FALSE(SRecord->getDestructor());
+
+  // First field.
+  const Record::Field *F1 = SRecord->getField(0u);
+  ASSERT_TRUE(F1);
+  ASSERT_FALSE(F1->isBitField());
+  ASSERT_TRUE(F1->Desc->isPrimitive());
+
+  // Second field.
+  const Record::Field *F2 = SRecord->getField(1u);
+  ASSERT_TRUE(F2);
+  ASSERT_FALSE(F2->isBitField());
+  ASSERT_TRUE(F2->Desc->isArray());
+  ASSERT_FALSE(F2->Desc->isCompositeArray());
+  ASSERT_TRUE(F2->Desc->isPrimitiveArray());
+  ASSERT_FALSE(F2->Desc->isPrimitive());
+  ASSERT_FALSE(F2->Desc->ElemDesc);
+  ASSERT_EQ(F2->Desc->getNumElems(), 4u);
+  ASSERT_TRUE(F2->Desc->getElemSize() > 0);
+
+  // Third field.
+  const Record::Field *F3 = SRecord->getField(2u);
+  ASSERT_TRUE(F3);
+  ASSERT_FALSE(F3->isBitField());
+  ASSERT_TRUE(F3->Desc->isArray());
+  ASSERT_TRUE(F3->Desc->isCompositeArray());
+  ASSERT_FALSE(F3->Desc->isPrimitiveArray());
+  ASSERT_FALSE(F3->Desc->isPrimitive());
+  ASSERT_TRUE(F3->Desc->ElemDesc);
+  ASSERT_EQ(F3->Desc->getNumElems(), 3u);
+  ASSERT_TRUE(F3->Desc->getElemSize() > 0);
+
+  // Fourth field.
+  // Multidimensional arrays are treated as composite arrays, even
+  // if the value type is primitive.
+  const Record::Field *F4 = SRecord->getField(3u);
+  ASSERT_TRUE(F4);
+  ASSERT_FALSE(F4->isBitField());
+  ASSERT_TRUE(F4->Desc->isArray());
+  ASSERT_TRUE(F4->Desc->isCompositeArray());
+  ASSERT_FALSE(F4->Desc->isPrimitiveArray());
+  ASSERT_FALSE(F4->Desc->isPrimitive());
+  ASSERT_TRUE(F4->Desc->ElemDesc);
+  ASSERT_EQ(F4->Desc->getNumElems(), 3u);
+  ASSERT_TRUE(F4->Desc->getElemSize() > 0);
+  ASSERT_TRUE(F4->Desc->ElemDesc->isPrimitiveArray());
+
+  // Check pointer stuff.
+  // Global variables have no inline descriptor (yet).
+  ASSERT_TRUE(GlobalPtr.isRoot());
+  ASSERT_TRUE(GlobalPtr.isLive());
+  ASSERT_FALSE(GlobalPtr.isZero());
+  ASSERT_FALSE(GlobalPtr.isField());
+  ASSERT_TRUE(GlobalPtr.getFieldDesc() == GlobalPtr.getDeclDesc());
+  ASSERT_TRUE(GlobalPtr.getOffset() == 0);
+  ASSERT_FALSE(GlobalPtr.inArray());
+  ASSERT_FALSE(GlobalPtr.isArrayElement());
+  ASSERT_FALSE(GlobalPtr.isArrayRoot());
+  ASSERT_FALSE(GlobalPtr.inPrimitiveArray());
+  ASSERT_TRUE(GlobalPtr.isStatic());
+  ASSERT_TRUE(GlobalPtr.isInitialized());
+  ASSERT_FALSE(GlobalPtr.isOnePastEnd());
+  ASSERT_FALSE(GlobalPtr.isElementPastEnd());
+
+  // Pointer to the first field (a primitive).
+  const Pointer &PF1 = GlobalPtr.atField(F1->Offset);
+  ASSERT_TRUE(PF1.isLive());
+  ASSERT_TRUE(PF1.isInitialized());
+  ASSERT_TRUE(PF1.isField());
+  ASSERT_FALSE(PF1.inArray());
+  ASSERT_FALSE(PF1.isArrayElement());
+  ASSERT_FALSE(PF1.isArrayRoot());
+  ASSERT_FALSE(PF1.isOnePastEnd());
+  ASSERT_FALSE(PF1.isRoot());
+  ASSERT_TRUE(PF1.getFieldDesc()->isPrimitive());
+  ASSERT_TRUE(Pointer::hasSameBase(PF1, GlobalPtr));
+  ASSERT_TRUE(PF1.getBase() == GlobalPtr);
+
+  // Pointer to the second field (a primitive array).
+  const Pointer &PF2 = GlobalPtr.atField(F2->Offset);
+  ASSERT_TRUE(PF2.isLive());
+  ASSERT_TRUE(PF2.isInitialized());
+  ASSERT_TRUE(PF2.isField());
+  ASSERT_TRUE(PF2.inArray());
+  ASSERT_FALSE(PF2.isArrayElement());
+  ASSERT_TRUE(PF2.isArrayRoot());
+  ASSERT_TRUE(PF2.getNumElems() == 4);
+  ASSERT_FALSE(PF2.isOnePastEnd());
+  ASSERT_FALSE(PF2.isRoot());
+  ASSERT_FALSE(PF2.getFieldDesc()->isPrimitive());
+  ASSERT_TRUE(PF2.getFieldDesc()->isArray());
+  ASSERT_TRUE(Pointer::hasSameBase(PF2, GlobalPtr));
+  ASSERT_TRUE(PF2.getBase() == GlobalPtr);
+
+  // Check contents of field 2 (a primitive array).
+  {
+    const Pointer &E1 = PF2.atIndex(0);
+    ASSERT_TRUE(E1.isLive());
+    ASSERT_FALSE(E1.isArrayRoot());
+    ASSERT_TRUE(E1.isArrayElement());
+    ASSERT_TRUE(E1.inPrimitiveArray());
+    ASSERT_TRUE(E1.deref<char>() == 'f');
+    ASSERT_EQ(E1.getIndex(), 0u);
+    ASSERT_TRUE(E1 == E1.atIndex(0));
+    ASSERT_TRUE(Pointer::hasSameBase(E1, GlobalPtr));
+
+    const Pointer &E2 = PF2.atIndex(1);
+    ASSERT_TRUE(E2.isLive());
+    ASSERT_FALSE(E2.isArrayRoot());
+    ASSERT_TRUE(E2.isArrayElement());
+    ASSERT_EQ(E2.getIndex(), 1u);
+    // Narrow() doesn't do anything on primitive array elements, as there is
+    // nothing to narrow into.
+    ASSERT_EQ(E2.narrow(), E2);
+    // ... so this should also hold.
+    ASSERT_EQ(E2.expand(), E2);
+    ASSERT_EQ(E2.narrow().expand(), E2);
+
+    // .atIndex(1).atIndex(1) should be index 1.
+    ASSERT_EQ(PF2.atIndex(1).atIndex(1), PF2.atIndex(1));
+    ASSERT_EQ(PF2.atIndex(1).narrow().atIndex(1), PF2.atIndex(1));
+
+    // getArray() should give us the array field again.
+    ASSERT_EQ(E2.getArray(), PF2);
+
+    // One-after-the-end pointer.
+    const Pointer &O = PF2.atIndex(PF2.getNumElems());
+    ASSERT_TRUE(O.isLive());
+    ASSERT_TRUE(O.isOnePastEnd());
+    ASSERT_TRUE(O.isInitialized());
+    ASSERT_TRUE(O.getIndex() == PF2.getNumElems());
+  }
+
+  // Pointer to the third field (a composite array).
+  const Pointer &PF3 = GlobalPtr.atField(F3->Offset);
+  ASSERT_TRUE(PF3.isLive());
+  ASSERT_TRUE(PF3.isInitialized());
+  ASSERT_TRUE(PF3.isField());
+  ASSERT_TRUE(PF3.inArray());
+  ASSERT_TRUE(PF3.isArrayRoot());
+  ASSERT_FALSE(PF3.isArrayElement());
+  ASSERT_TRUE(PF3.getNumElems() == 3);
+  ASSERT_FALSE(PF3.isOnePastEnd());
+  ASSERT_FALSE(PF3.isRoot());
+  ASSERT_FALSE(PF3.getFieldDesc()->isPrimitive());
+  ASSERT_TRUE(PF3.getFieldDesc()->isArray());
+  ASSERT_TRUE(Pointer::hasSameBase(PF3, GlobalPtr));
+  ASSERT_TRUE(PF3.getBase() == GlobalPtr);
+  ASSERT_EQ(PF3.getRecord(), nullptr);
+  ASSERT_TRUE(PF3.getElemRecord());
+
+  // Check contents of field 3 (a composite array).
+  {
+    const Pointer &E1 = PF3.atIndex(0);
+    // Note that we didn't call narrow() above, so this points
+    // to an array element and not just a field.
+    ASSERT_TRUE(E1.isLive());
+    ASSERT_EQ(E1.getIndex(), 0);
+    ASSERT_TRUE(E1.isInitialized());
+    ASSERT_TRUE(E1.isArrayElement());
+    ASSERT_TRUE(E1.inArray());
+    ASSERT_FALSE(E1.isArrayRoot());
+    ASSERT_FALSE(E1.isRoot());
+    ASSERT_EQ(E1.getArray(), PF3);
+    ASSERT_TRUE(E1.isField());
+    ASSERT_TRUE(E1.getElemRecord());
+    ASSERT_FALSE(E1.getRecord());
+
+    // Now the same with narrow().
+    const Pointer &NE1 = PF3.atIndex(0).narrow();
+    ASSERT_NE(E1, NE1);
+    ASSERT_TRUE(NE1.isLive());
+    ASSERT_EQ(NE1.getIndex(), 0);
+    ASSERT_TRUE(NE1.isInitialized());
+    ASSERT_FALSE(NE1.isArrayElement());
+    ASSERT_TRUE(NE1.isField());
+    ASSERT_FALSE(NE1.inArray());
+    ASSERT_FALSE(NE1.isArrayRoot());
+    ASSERT_FALSE(NE1.isRoot());
+    // Not possible, since this is narrow()ed:
+    // ASSERT_EQ(NE1.getArray(), PF3);
+    ASSERT_EQ(NE1.expand(), E1);
+    ASSERT_FALSE(NE1.getElemRecord());
+    ASSERT_TRUE(NE1.getRecord());
+
+    // Second element, NOT narrowed.
+    const Pointer &E2 = PF3.atIndex(1);
+    ASSERT_TRUE(E2.isLive());
+    ASSERT_EQ(E2.getIndex(), 1);
+    ASSERT_TRUE(E2.isInitialized());
+    ASSERT_TRUE(E2.isArrayElement());
+    ASSERT_TRUE(E2.isField());
+    ASSERT_TRUE(E2.inArray());
+    ASSERT_FALSE(E2.isArrayRoot());
+    ASSERT_FALSE(E2.isRoot());
+    ASSERT_EQ(E2.getArray(), PF3);
+
+    // Second element, narrowed.
+    const Pointer &NE2 = PF3.atIndex(1).narrow();
+    ASSERT_TRUE(NE2.isLive());
+    ASSERT_EQ(NE2.getIndex(), 0);
+    ASSERT_TRUE(NE2.isInitialized());
+    ASSERT_FALSE(NE2.isArrayElement());
+    ASSERT_TRUE(NE2.isField());
+    ASSERT_FALSE(NE2.inArray());
+    ASSERT_FALSE(NE2.isArrayRoot());
+    ASSERT_FALSE(NE2.isRoot());
+    // Not possible, since this is narrow()ed:
+    // ASSERT_EQ(NE2.getArray(), PF3);
+    ASSERT_FALSE(NE2.getElemRecord());
+    ASSERT_TRUE(NE2.getRecord());
+
+    // Chained atIndex() without narrowing in between.
+    ASSERT_EQ(PF3.atIndex(1).atIndex(1), PF3.atIndex(1));
+
+    // First field of the second element.
+    const Pointer &FP1 = NE2.atField(NE2.getRecord()->getField(0u)->Offset);
+    ASSERT_TRUE(FP1.isLive());
+    ASSERT_TRUE(FP1.isInitialized());
+    ASSERT_EQ(FP1.getBase(), NE2);
+    ASSERT_FALSE(FP1.isArrayElement());
+    ASSERT_FALSE(FP1.inArray());
+    ASSERT_FALSE(FP1.inPrimitiveArray());
+    ASSERT_TRUE(FP1.isField());
+
+    // One-past-the-end of a composite array.
+    const Pointer &O = PF3.atIndex(PF3.getNumElems()).narrow();
+    ASSERT_TRUE(O.isOnePastEnd());
+    ASSERT_TRUE(O.isElementPastEnd());
+  }
+
+  // Pointer to the fourth field (a multidimensional primitive array).
+  const Pointer &PF4 = GlobalPtr.atField(F4->Offset);
+  ASSERT_TRUE(PF4.isLive());
+  ASSERT_TRUE(PF4.isInitialized());
+  ASSERT_TRUE(PF4.isField());
+  ASSERT_TRUE(PF4.inArray());
+  ASSERT_TRUE(PF4.isArrayRoot());
+  ASSERT_FALSE(PF4.isArrayElement());
+  ASSERT_TRUE(PF4.getNumElems() == 3);
+  ASSERT_FALSE(PF4.isOnePastEnd());
+  ASSERT_FALSE(PF4.isRoot());
+  ASSERT_FALSE(PF4.getFieldDesc()->isPrimitive());
+  ASSERT_TRUE(PF4.getFieldDesc()->isArray());
+  ASSERT_TRUE(Pointer::hasSameBase(PF4, GlobalPtr));
+  ASSERT_TRUE(PF4.getBase() == GlobalPtr);
+  ASSERT_EQ(PF4.getRecord(), nullptr);
+  ASSERT_EQ(PF4.getElemRecord(), nullptr);
+  ASSERT_NE(PF4.getField(), nullptr);
+  ASSERT_TRUE(PF4.getFieldDesc()->ElemDesc->isPrimitiveArray());
+  // Check contents of field 4 (a primitive array).
+  {
+    // Pointer to the first element, is of type short[3].
+    const Pointer &E1 = PF4.atIndex(0);
+    ASSERT_NE(E1, PF4);
+    ASSERT_TRUE(E1.isLive());
+    ASSERT_TRUE(E1.isArrayElement());
+    ASSERT_TRUE(E1.inArray());
+    ASSERT_EQ(E1.getNumElems(), 3u);
+    ASSERT_EQ(E1.getIndex(), 0u);
+    ASSERT_EQ(E1.getArray(), PF4);
+
+    // Now narrow()'ed.
+    const Pointer &NE1 = PF4.atIndex(0).narrow();
+    ASSERT_NE(NE1, PF4);
+    ASSERT_NE(NE1, E1);
+    ASSERT_TRUE(NE1.isLive());
+    ASSERT_FALSE(NE1.isArrayElement());
+    ASSERT_TRUE(NE1.isArrayRoot());
+    ASSERT_FALSE(NE1.getFieldDesc()->isCompositeArray());
+    ASSERT_TRUE(NE1.getFieldDesc()->isPrimitiveArray());
+    ASSERT_EQ(NE1.getFieldDesc()->getNumElems(), 3u);
+    ASSERT_TRUE(NE1.inArray());
+    ASSERT_EQ(NE1.getNumElems(), 3u);
+    ASSERT_EQ(NE1.getIndex(), 0u);
+
+    // Last element of the first dimension.
+    const Pointer &PE1 = PF4.atIndex(0).narrow().atIndex(2);
+    ASSERT_TRUE(PE1.isLive());
+    ASSERT_EQ(PE1.deref<short>(), 3);
+    ASSERT_EQ(PE1.getArray(), NE1);
+    ASSERT_EQ(PE1.getIndex(), 2u);
+
+    // third dimension
+    const Pointer &E3 = PF4.atIndex(2);
+    ASSERT_NE(E3, PF4);
+    ASSERT_TRUE(E3.isLive());
+    ASSERT_TRUE(E3.isArrayElement());
+    ASSERT_FALSE(E3.isArrayRoot());
+    ASSERT_TRUE(E3.inArray());
+    ASSERT_EQ(E3.getNumElems(), 3u);
+    ASSERT_EQ(E3.getIndex(), 2u);
+
+    // Same, but narrow()'ed.
+    const Pointer &NE3 = PF4.atIndex(2).narrow();
+    ASSERT_NE(NE3, PF4);
+    ASSERT_NE(NE3, E1);
+    ASSERT_TRUE(NE3.isLive());
+    ASSERT_FALSE(NE3.isArrayElement());
+    ASSERT_TRUE(NE3.isArrayRoot());
+    ASSERT_FALSE(NE3.getFieldDesc()->isCompositeArray());
+    ASSERT_TRUE(NE3.getFieldDesc()->isPrimitiveArray());
+    ASSERT_EQ(NE3.getFieldDesc()->getNumElems(), 3u);
+    ASSERT_TRUE(NE3.inArray());
+    ASSERT_EQ(NE3.getNumElems(), 3u);
+    // This is narrow()'ed, so not an "array elemnet"
+    ASSERT_EQ(PF4.atIndex(2).getIndex(), 2u);
+    ASSERT_EQ(NE3.getIndex(), 0u);
+
+    // Last element of the last dimension
+    const Pointer &PE3 = PF4.atIndex(2).narrow().atIndex(2);
+    ASSERT_TRUE(PE3.isLive());
+    ASSERT_EQ(PE3.deref<short>(), 9);
+    ASSERT_EQ(PE3.getArray(), NE3);
+    ASSERT_EQ(PE3.getIndex(), 2u);
+  }
+}
diff --git a/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp
index 8995471180..d4a695b974 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp
@@ -8,6 +8,7 @@
 
 #include "ASTMatchersTest.h"
 #include "clang/AST/Attrs.inc"
+#include "clang/AST/DeclCXX.h"
 #include "clang/AST/PrettyPrinter.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
 #include "clang/ASTMatchers/ASTMatchers.h"
@@ -5457,6 +5458,18 @@ TEST(HasParent, NoDuplicateParents) {
     stmt().bind("node"), std::make_unique<HasDuplicateParents>()));
 }
 
+TEST(HasAnyBase, BindsInnerBoundNodes) {
+  EXPECT_TRUE(matchAndVerifyResultTrue(
+      "struct Inner {}; struct Proxy : Inner {}; struct Main : public "
+      "Proxy {};",
+      cxxRecordDecl(hasName("Main"),
+                    hasAnyBase(cxxBaseSpecifier(hasType(
+                        cxxRecordDecl(hasName("Inner")).bind("base-class")))))
+          .bind("class"),
+      std::make_unique<VerifyIdIsBoundTo<CXXRecordDecl>>("base-class",
+                                                         "Inner")));
+}
+
 TEST(TypeMatching, PointeeTypes) {
   EXPECT_TRUE(matches("int b; int &a = b;",
                       referenceType(pointee(builtinType()))));
diff --git a/clang/unittests/Analysis/FlowSensitive/ArenaTest.cpp b/clang/unittests/Analysis/FlowSensitive/ArenaTest.cpp
index 1208b78a30..5f85bfeb9e 100644
--- a/clang/unittests/Analysis/FlowSensitive/ArenaTest.cpp
+++ b/clang/unittests/Analysis/FlowSensitive/ArenaTest.cpp
@@ -34,12 +34,6 @@ TEST_F(ArenaTest, CreateTopBoolValueReturnsDistinctValues) {
   EXPECT_NE(&X, &Y);
 }
 
-TEST_F(ArenaTest, GetOrCreateConjunctionReturnsSameExprGivenSameArgs) {
-  auto &X = A.makeAtomRef(A.makeAtom());
-  auto &XAndX = A.makeAnd(X, X);
-  EXPECT_EQ(&XAndX, &X);
-}
-
 TEST_F(ArenaTest, GetOrCreateConjunctionReturnsSameExprOnSubsequentCalls) {
   auto &X = A.makeAtomRef(A.makeAtom());
   auto &Y = A.makeAtomRef(A.makeAtom());
@@ -55,12 +49,6 @@ TEST_F(ArenaTest, GetOrCreateConjunctionReturnsSameExprOnSubsequentCalls) {
   EXPECT_NE(&XAndY1, &XAndZ);
 }
 
-TEST_F(ArenaTest, GetOrCreateDisjunctionReturnsSameExprGivenSameArgs) {
-  auto &X = A.makeAtomRef(A.makeAtom());
-  auto &XOrX = A.makeOr(X, X);
-  EXPECT_EQ(&XOrX, &X);
-}
-
 TEST_F(ArenaTest, GetOrCreateDisjunctionReturnsSameExprOnSubsequentCalls) {
   auto &X = A.makeAtomRef(A.makeAtom());
   auto &Y = A.makeAtomRef(A.makeAtom());
@@ -86,12 +74,6 @@ TEST_F(ArenaTest, GetOrCreateNegationReturnsSameExprOnSubsequentCalls) {
   EXPECT_NE(&NotX1, &NotY);
 }
 
-TEST_F(ArenaTest, GetOrCreateImplicationReturnsTrueGivenSameArgs) {
-  auto &X = A.makeAtomRef(A.makeAtom());
-  auto &XImpliesX = A.makeImplies(X, X);
-  EXPECT_EQ(&XImpliesX, &A.makeLiteral(true));
-}
-
 TEST_F(ArenaTest, GetOrCreateImplicationReturnsSameExprOnSubsequentCalls) {
   auto &X = A.makeAtomRef(A.makeAtom());
   auto &Y = A.makeAtomRef(A.makeAtom());
@@ -107,12 +89,6 @@ TEST_F(ArenaTest, GetOrCreateImplicationReturnsSameExprOnSubsequentCalls) {
   EXPECT_NE(&XImpliesY1, &XImpliesZ);
 }
 
-TEST_F(ArenaTest, GetOrCreateIffReturnsTrueGivenSameArgs) {
-  auto &X = A.makeAtomRef(A.makeAtom());
-  auto &XIffX = A.makeEquals(X, X);
-  EXPECT_EQ(&XIffX, &A.makeLiteral(true));
-}
-
 TEST_F(ArenaTest, GetOrCreateIffReturnsSameExprOnSubsequentCalls) {
   auto &X = A.makeAtomRef(A.makeAtom());
   auto &Y = A.makeAtomRef(A.makeAtom());
@@ -181,5 +157,36 @@ V1 V2
    ^)"));
 }
 
+TEST_F(ArenaTest, IdentitySimplification) {
+  auto &X = A.makeAtomRef(A.makeAtom());
+
+  EXPECT_EQ(&X, &A.makeAnd(X, X));
+  EXPECT_EQ(&X, &A.makeOr(X, X));
+  EXPECT_EQ(&A.makeLiteral(true), &A.makeImplies(X, X));
+  EXPECT_EQ(&A.makeLiteral(true), &A.makeEquals(X, X));
+  EXPECT_EQ(&X, &A.makeNot(A.makeNot(X)));
+}
+
+TEST_F(ArenaTest, LiteralSimplification) {
+  auto &X = A.makeAtomRef(A.makeAtom());
+
+  EXPECT_EQ(&X, &A.makeAnd(X, A.makeLiteral(true)));
+  EXPECT_EQ(&A.makeLiteral(false), &A.makeAnd(X, A.makeLiteral(false)));
+
+  EXPECT_EQ(&A.makeLiteral(true), &A.makeOr(X, A.makeLiteral(true)));
+  EXPECT_EQ(&X, &A.makeOr(X, A.makeLiteral(false)));
+
+  EXPECT_EQ(&A.makeLiteral(true), &A.makeImplies(X, A.makeLiteral(true)));
+  EXPECT_EQ(&A.makeNot(X), &A.makeImplies(X, A.makeLiteral(false)));
+  EXPECT_EQ(&X, &A.makeImplies(A.makeLiteral(true), X));
+  EXPECT_EQ(&A.makeLiteral(true), &A.makeImplies(A.makeLiteral(false), X));
+
+  EXPECT_EQ(&X, &A.makeEquals(X, A.makeLiteral(true)));
+  EXPECT_EQ(&A.makeNot(X), &A.makeEquals(X, A.makeLiteral(false)));
+
+  EXPECT_EQ(&A.makeLiteral(false), &A.makeNot(A.makeLiteral(true)));
+  EXPECT_EQ(&A.makeLiteral(true), &A.makeNot(A.makeLiteral(false)));
+}
+
 } // namespace
 } // namespace clang::dataflow
diff --git a/clang/unittests/Analysis/FlowSensitive/DataflowAnalysisContextTest.cpp b/clang/unittests/Analysis/FlowSensitive/DataflowAnalysisContextTest.cpp
index fb7642c131..88eb11045b 100644
--- a/clang/unittests/Analysis/FlowSensitive/DataflowAnalysisContextTest.cpp
+++ b/clang/unittests/Analysis/FlowSensitive/DataflowAnalysisContextTest.cpp
@@ -99,33 +99,6 @@ TEST_F(DataflowAnalysisContextTest, JoinFlowConditions) {
   EXPECT_TRUE(Context.flowConditionImplies(FC3, C3));
 }
 
-TEST_F(DataflowAnalysisContextTest, FlowConditionTautologies) {
-  // Fresh flow condition with empty/no constraints is always true.
-  Atom FC1 = A.makeFlowConditionToken();
-  EXPECT_TRUE(Context.flowConditionIsTautology(FC1));
-
-  // Literal `true` is always true.
-  Atom FC2 = A.makeFlowConditionToken();
-  Context.addFlowConditionConstraint(FC2, A.makeLiteral(true));
-  EXPECT_TRUE(Context.flowConditionIsTautology(FC2));
-
-  // Literal `false` is never true.
-  Atom FC3 = A.makeFlowConditionToken();
-  Context.addFlowConditionConstraint(FC3, A.makeLiteral(false));
-  EXPECT_FALSE(Context.flowConditionIsTautology(FC3));
-
-  // We can't prove that an arbitrary bool A is always true...
-  auto &C1 = A.makeAtomRef(A.makeAtom());
-  Atom FC4 = A.makeFlowConditionToken();
-  Context.addFlowConditionConstraint(FC4, C1);
-  EXPECT_FALSE(Context.flowConditionIsTautology(FC4));
-
-  // ... but we can prove A || !A is true.
-  Atom FC5 = A.makeFlowConditionToken();
-  Context.addFlowConditionConstraint(FC5, A.makeOr(C1, A.makeNot(C1)));
-  EXPECT_TRUE(Context.flowConditionIsTautology(FC5));
-}
-
 TEST_F(DataflowAnalysisContextTest, EquivBoolVals) {
   auto &X = A.makeAtomRef(A.makeAtom());
   auto &Y = A.makeAtomRef(A.makeAtom());
diff --git a/clang/unittests/Analysis/FlowSensitive/DebugSupportTest.cpp b/clang/unittests/Analysis/FlowSensitive/DebugSupportTest.cpp
index 22bf8cadd1..48d6e4361f 100644
--- a/clang/unittests/Analysis/FlowSensitive/DebugSupportTest.cpp
+++ b/clang/unittests/Analysis/FlowSensitive/DebugSupportTest.cpp
@@ -30,6 +30,12 @@ TEST(BoolValueDebugStringTest, AtomicBoolean) {
   EXPECT_THAT(llvm::to_string(*B), StrEq(Expected));
 }
 
+TEST(BoolValueDebugStringTest, Literal) {
+  ConstraintContext Ctx;
+  EXPECT_EQ("true", llvm::to_string(*Ctx.literal(true)));
+  EXPECT_EQ("false", llvm::to_string(*Ctx.literal(false)));
+}
+
 TEST(BoolValueDebugStringTest, Negation) {
   ConstraintContext Ctx;
   auto B = Ctx.neg(Ctx.atom());
@@ -91,16 +97,16 @@ TEST(BoolValueDebugStringTest, NestedBoolean) {
 
 TEST(BoolValueDebugStringTest, ComplexBooleanWithSomeNames) {
   ConstraintContext Ctx;
-  auto True = Ctx.atom();
-  auto False = Ctx.atom();
+  auto X = Ctx.atom();
+  auto Y = Ctx.atom();
+  Formula::AtomNames Names;
+  Names[X->getAtom()] = "X";
+  Names[Y->getAtom()] = "Y";
   auto V2 = Ctx.atom();
   auto V3 = Ctx.atom();
-  Formula::AtomNames Names;
-  Names[True->getAtom()] = "true";
-  Names[False->getAtom()] = "false";
-  auto B = Ctx.disj(Ctx.conj(False, V2), Ctx.disj(True, V3));
+  auto B = Ctx.disj(Ctx.conj(Y, V2), Ctx.disj(X, V3));
 
-  auto Expected = R"(((false & V2) | (true | V3)))";
+  auto Expected = R"(((Y & V2) | (X | V3)))";
   std::string Actual;
   llvm::raw_string_ostream OS(Actual);
   B->print(OS, &Names);
diff --git a/clang/unittests/Analysis/FlowSensitive/TestingSupport.cpp b/clang/unittests/Analysis/FlowSensitive/TestingSupport.cpp
index 72bdfee26f..65c527ae63 100644
--- a/clang/unittests/Analysis/FlowSensitive/TestingSupport.cpp
+++ b/clang/unittests/Analysis/FlowSensitive/TestingSupport.cpp
@@ -161,7 +161,18 @@ llvm::Error test::checkDataflowWithNoopAnalysis(
         VerifyResults,
     DataflowAnalysisOptions Options, LangStandard::Kind Std,
     llvm::StringRef TargetFun) {
-  using ast_matchers::hasName;
+  return checkDataflowWithNoopAnalysis(Code, ast_matchers::hasName(TargetFun),
+                                       VerifyResults, Options, Std);
+}
+
+llvm::Error test::checkDataflowWithNoopAnalysis(
+    llvm::StringRef Code,
+    ast_matchers::internal::Matcher<FunctionDecl> TargetFuncMatcher,
+    std::function<
+        void(const llvm::StringMap<DataflowAnalysisState<NoopLattice>> &,
+             ASTContext &)>
+        VerifyResults,
+    DataflowAnalysisOptions Options, LangStandard::Kind Std) {
   llvm::SmallVector<std::string, 3> ASTBuildArgs = {
       // -fnodelayed-template-parsing is the default everywhere but on Windows.
       // Set it explicitly so that tests behave the same on Windows as on other
@@ -170,7 +181,7 @@ llvm::Error test::checkDataflowWithNoopAnalysis(
       "-std=" +
           std::string(LangStandard::getLangStandardForKind(Std).getName())};
   AnalysisInputs<NoopAnalysis> AI(
-      Code, hasName(TargetFun),
+      Code, TargetFuncMatcher,
       [UseBuiltinModel = Options.BuiltinOpts.has_value()](ASTContext &C,
                                                           Environment &Env) {
         return NoopAnalysis(
diff --git a/clang/unittests/Analysis/FlowSensitive/TestingSupport.h b/clang/unittests/Analysis/FlowSensitive/TestingSupport.h
index 44d962d5da..a8089d9b8c 100644
--- a/clang/unittests/Analysis/FlowSensitive/TestingSupport.h
+++ b/clang/unittests/Analysis/FlowSensitive/TestingSupport.h
@@ -398,8 +398,8 @@ checkDataflow(AnalysisInputs<AnalysisT> AI,
 
 using BuiltinOptions = DataflowAnalysisContext::Options;
 
-/// Runs dataflow on `Code` with a `NoopAnalysis` and calls `VerifyResults` to
-/// verify the results.
+/// Runs dataflow on function named `TargetFun` in `Code` with a `NoopAnalysis`
+/// and calls `VerifyResults` to verify the results.
 llvm::Error checkDataflowWithNoopAnalysis(
     llvm::StringRef Code,
     std::function<
@@ -410,6 +410,18 @@ llvm::Error checkDataflowWithNoopAnalysis(
     LangStandard::Kind Std = LangStandard::lang_cxx17,
     llvm::StringRef TargetFun = "target");
 
+/// Runs dataflow on function matched by `TargetFuncMatcher` in `Code` with a
+/// `NoopAnalysis` and calls `VerifyResults` to verify the results.
+llvm::Error checkDataflowWithNoopAnalysis(
+    llvm::StringRef Code,
+    ast_matchers::internal::Matcher<FunctionDecl> TargetFuncMatcher,
+    std::function<
+        void(const llvm::StringMap<DataflowAnalysisState<NoopLattice>> &,
+             ASTContext &)>
+        VerifyResults = [](const auto &, auto &) {},
+    DataflowAnalysisOptions Options = {BuiltinOptions()},
+    LangStandard::Kind Std = LangStandard::lang_cxx17);
+
 /// Returns the `ValueDecl` for the given identifier.
 ///
 /// Requirements:
@@ -482,6 +494,11 @@ class ConstraintContext {
     return &Formula::create(A, Formula::AtomRef, {}, NextAtom++);
   }
 
+  // Returns a reference to a literal boolean value.
+  const Formula *literal(bool B) {
+    return &Formula::create(A, Formula::Literal, {}, B);
+  }
+
   // Creates a boolean conjunction.
   const Formula *conj(const Formula *LHS, const Formula *RHS) {
     return make(Formula::And, {LHS, RHS});
diff --git a/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp b/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp
index 2e8d38490f..f5d9c785b6 100644
--- a/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp
+++ b/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp
@@ -66,6 +66,37 @@ void runDataflow(
               Std, TargetFun);
 }
 
+void runDataflowOnLambda(
+    llvm::StringRef Code,
+    std::function<
+        void(const llvm::StringMap<DataflowAnalysisState<NoopLattice>> &,
+             ASTContext &)>
+        VerifyResults,
+    DataflowAnalysisOptions Options,
+    LangStandard::Kind Std = LangStandard::lang_cxx17) {
+  ASSERT_THAT_ERROR(
+      checkDataflowWithNoopAnalysis(
+          Code,
+          ast_matchers::hasDeclContext(
+              ast_matchers::cxxRecordDecl(ast_matchers::isLambda())),
+          VerifyResults, Options, Std),
+      llvm::Succeeded());
+}
+
+void runDataflowOnLambda(
+    llvm::StringRef Code,
+    std::function<
+        void(const llvm::StringMap<DataflowAnalysisState<NoopLattice>> &,
+             ASTContext &)>
+        VerifyResults,
+    LangStandard::Kind Std = LangStandard::lang_cxx17,
+    bool ApplyBuiltinTransfer = true) {
+  runDataflowOnLambda(Code, std::move(VerifyResults),
+                      {ApplyBuiltinTransfer ? BuiltinOptions{}
+                                            : std::optional<BuiltinOptions>()},
+                      Std);
+}
+
 const Formula &getFormula(const ValueDecl &D, const Environment &Env) {
   return cast<BoolValue>(Env.getValue(D))->formula();
 }
@@ -3199,14 +3230,14 @@ TEST(TransferTest, AssignFromBoolLiteral) {
         ASSERT_THAT(FooDecl, NotNull());
 
         const auto *FooVal =
-            dyn_cast_or_null<AtomicBoolValue>(Env.getValue(*FooDecl));
+            dyn_cast_or_null<BoolValue>(Env.getValue(*FooDecl));
         ASSERT_THAT(FooVal, NotNull());
 
         const ValueDecl *BarDecl = findValueDecl(ASTCtx, "Bar");
         ASSERT_THAT(BarDecl, NotNull());
 
         const auto *BarVal =
-            dyn_cast_or_null<AtomicBoolValue>(Env.getValue(*BarDecl));
+            dyn_cast_or_null<BoolValue>(Env.getValue(*BarDecl));
         ASSERT_THAT(BarVal, NotNull());
 
         EXPECT_EQ(FooVal, &Env.getBoolLiteralValue(true));
@@ -3384,7 +3415,7 @@ TEST(TransferTest, AssignFromBoolNegation) {
         ASSERT_THAT(FooDecl, NotNull());
 
         const auto *FooVal =
-            dyn_cast_or_null<AtomicBoolValue>(Env.getValue(*FooDecl));
+            dyn_cast_or_null<BoolValue>(Env.getValue(*FooDecl));
         ASSERT_THAT(FooVal, NotNull());
 
         const ValueDecl *BarDecl = findValueDecl(ASTCtx, "Bar");
@@ -4068,6 +4099,20 @@ TEST(TransferTest, LoopDereferencingChangingRecordPointerConverges) {
   ASSERT_THAT_ERROR(checkDataflowWithNoopAnalysis(Code), llvm::Succeeded());
 }
 
+TEST(TransferTest, LoopWithShortCircuitedConditionConverges) {
+  std::string Code = R"cc(
+    bool foo();
+
+    void target() {
+      bool c = false;
+      while (foo() || foo()) {
+        c = true;
+      }
+    }
+  )cc";
+  ASSERT_THAT_ERROR(checkDataflowWithNoopAnalysis(Code), llvm::Succeeded());
+}
+
 TEST(TransferTest, DoesNotCrashOnUnionThisExpr) {
   std::string Code = R"(
     union Union {
@@ -5987,4 +6032,208 @@ TEST(TransferTest, EvaluateBlockWithUnreachablePreds) {
          ASTContext &ASTCtx) {});
 }
 
+TEST(TransferTest, LambdaCaptureByCopy) {
+  std::string Code = R"(
+    void target(int Foo, int Bar) {
+      [Foo]() {
+        (void)0;
+        // [[p]]
+      }();
+    }
+  )";
+  runDataflowOnLambda(
+      Code,
+      [](const llvm::StringMap<DataflowAnalysisState<NoopLattice>> &Results,
+         ASTContext &ASTCtx) {
+        ASSERT_THAT(Results.keys(), UnorderedElementsAre("p"));
+        const Environment &Env = getEnvironmentAtAnnotation(Results, "p");
+
+        const ValueDecl *FooDecl = findValueDecl(ASTCtx, "Foo");
+        ASSERT_THAT(FooDecl, NotNull());
+
+        const StorageLocation *FooLoc = Env.getStorageLocation(*FooDecl);
+        ASSERT_TRUE(isa_and_nonnull<ScalarStorageLocation>(FooLoc));
+
+        const Value *FooVal = Env.getValue(*FooLoc);
+        EXPECT_TRUE(isa_and_nonnull<IntegerValue>(FooVal));
+
+        const ValueDecl *BarDecl = findValueDecl(ASTCtx, "Bar");
+        ASSERT_THAT(BarDecl, NotNull());
+
+        const StorageLocation *BarLoc = Env.getStorageLocation(*BarDecl);
+        EXPECT_THAT(BarLoc, IsNull());
+      });
+}
+
+TEST(TransferTest, LambdaCaptureByReference) {
+  std::string Code = R"(
+    void target(int Foo, int Bar) {
+      [&Foo]() {
+        (void)0;
+        // [[p]]
+      }();
+    }
+  )";
+  runDataflowOnLambda(
+      Code,
+      [](const llvm::StringMap<DataflowAnalysisState<NoopLattice>> &Results,
+         ASTContext &ASTCtx) {
+        ASSERT_THAT(Results.keys(), UnorderedElementsAre("p"));
+        const Environment &Env = getEnvironmentAtAnnotation(Results, "p");
+
+        const ValueDecl *FooDecl = findValueDecl(ASTCtx, "Foo");
+        ASSERT_THAT(FooDecl, NotNull());
+
+        const StorageLocation *FooLoc = Env.getStorageLocation(*FooDecl);
+        ASSERT_TRUE(isa_and_nonnull<ScalarStorageLocation>(FooLoc));
+
+        const Value *FooVal = Env.getValue(*FooLoc);
+        EXPECT_TRUE(isa_and_nonnull<IntegerValue>(FooVal));
+
+        const ValueDecl *BarDecl = findValueDecl(ASTCtx, "Bar");
+        ASSERT_THAT(BarDecl, NotNull());
+
+        const StorageLocation *BarLoc = Env.getStorageLocation(*BarDecl);
+        EXPECT_THAT(BarLoc, IsNull());
+      });
+}
+
+TEST(TransferTest, LambdaCaptureWithInitializer) {
+  std::string Code = R"(
+    void target(int Bar) {
+      [Foo=Bar]() {
+        (void)0;
+        // [[p]]
+      }();
+    }
+  )";
+  runDataflowOnLambda(
+      Code,
+      [](const llvm::StringMap<DataflowAnalysisState<NoopLattice>> &Results,
+         ASTContext &ASTCtx) {
+        ASSERT_THAT(Results.keys(), UnorderedElementsAre("p"));
+        const Environment &Env = getEnvironmentAtAnnotation(Results, "p");
+
+        const ValueDecl *FooDecl = findValueDecl(ASTCtx, "Foo");
+        ASSERT_THAT(FooDecl, NotNull());
+
+        const StorageLocation *FooLoc = Env.getStorageLocation(*FooDecl);
+        ASSERT_TRUE(isa_and_nonnull<ScalarStorageLocation>(FooLoc));
+
+        const Value *FooVal = Env.getValue(*FooLoc);
+        EXPECT_TRUE(isa_and_nonnull<IntegerValue>(FooVal));
+
+        const ValueDecl *BarDecl = findValueDecl(ASTCtx, "Bar");
+        ASSERT_THAT(BarDecl, NotNull());
+
+        const StorageLocation *BarLoc = Env.getStorageLocation(*BarDecl);
+        EXPECT_THAT(BarLoc, IsNull());
+      });
+}
+
+TEST(TransferTest, LambdaCaptureByCopyImplicit) {
+  std::string Code = R"(
+    void target(int Foo, int Bar) {
+      [=]() {
+        Foo;
+        // [[p]]
+      }();
+    }
+  )";
+  runDataflowOnLambda(
+      Code,
+      [](const llvm::StringMap<DataflowAnalysisState<NoopLattice>> &Results,
+         ASTContext &ASTCtx) {
+        ASSERT_THAT(Results.keys(), UnorderedElementsAre("p"));
+        const Environment &Env = getEnvironmentAtAnnotation(Results, "p");
+
+        const ValueDecl *FooDecl = findValueDecl(ASTCtx, "Foo");
+        ASSERT_THAT(FooDecl, NotNull());
+
+        const StorageLocation *FooLoc = Env.getStorageLocation(*FooDecl);
+        ASSERT_TRUE(isa_and_nonnull<ScalarStorageLocation>(FooLoc));
+
+        const Value *FooVal = Env.getValue(*FooLoc);
+        EXPECT_TRUE(isa_and_nonnull<IntegerValue>(FooVal));
+
+        const ValueDecl *BarDecl = findValueDecl(ASTCtx, "Bar");
+        ASSERT_THAT(BarDecl, NotNull());
+
+        // There is no storage location for `Bar` because it isn't used in the
+        // body of the lambda.
+        const StorageLocation *BarLoc = Env.getStorageLocation(*BarDecl);
+        EXPECT_THAT(BarLoc, IsNull());
+      });
+}
+
+TEST(TransferTest, LambdaCaptureByReferenceImplicit) {
+  std::string Code = R"(
+    void target(int Foo, int Bar) {
+      [&]() {
+        Foo;
+        // [[p]]
+      }();
+    }
+  )";
+  runDataflowOnLambda(
+      Code,
+      [](const llvm::StringMap<DataflowAnalysisState<NoopLattice>> &Results,
+         ASTContext &ASTCtx) {
+        ASSERT_THAT(Results.keys(), UnorderedElementsAre("p"));
+        const Environment &Env = getEnvironmentAtAnnotation(Results, "p");
+
+        const ValueDecl *FooDecl = findValueDecl(ASTCtx, "Foo");
+        ASSERT_THAT(FooDecl, NotNull());
+
+        const StorageLocation *FooLoc = Env.getStorageLocation(*FooDecl);
+        ASSERT_TRUE(isa_and_nonnull<ScalarStorageLocation>(FooLoc));
+
+        const Value *FooVal = Env.getValue(*FooLoc);
+        EXPECT_TRUE(isa_and_nonnull<IntegerValue>(FooVal));
+
+        const ValueDecl *BarDecl = findValueDecl(ASTCtx, "Bar");
+        ASSERT_THAT(BarDecl, NotNull());
+
+        // There is no storage location for `Bar` because it isn't used in the
+        // body of the lambda.
+        const StorageLocation *BarLoc = Env.getStorageLocation(*BarDecl);
+        EXPECT_THAT(BarLoc, IsNull());
+      });
+}
+
+TEST(TransferTest, LambdaCaptureThis) {
+  std::string Code = R"(
+    struct Bar {
+      int Foo;
+
+      void target() {
+        [this]() {
+          Foo;
+          // [[p]]
+        }();
+      }
+    };
+  )";
+  runDataflowOnLambda(
+      Code,
+      [](const llvm::StringMap<DataflowAnalysisState<NoopLattice>> &Results,
+         ASTContext &ASTCtx) {
+        ASSERT_THAT(Results.keys(), UnorderedElementsAre("p"));
+        const Environment &Env = getEnvironmentAtAnnotation(Results, "p");
+
+        const RecordStorageLocation *ThisPointeeLoc =
+            Env.getThisPointeeStorageLocation();
+        ASSERT_THAT(ThisPointeeLoc, NotNull());
+
+        const ValueDecl *FooDecl = findValueDecl(ASTCtx, "Foo");
+        ASSERT_THAT(FooDecl, NotNull());
+
+        const StorageLocation *FooLoc = ThisPointeeLoc->getChild(*FooDecl);
+        ASSERT_TRUE(isa_and_nonnull<ScalarStorageLocation>(FooLoc));
+
+        const Value *FooVal = Env.getValue(*FooLoc);
+        EXPECT_TRUE(isa_and_nonnull<IntegerValue>(FooVal));
+      });
+}
+
 } // namespace
diff --git a/clang/unittests/Analysis/FlowSensitive/TypeErasedDataflowAnalysisTest.cpp b/clang/unittests/Analysis/FlowSensitive/TypeErasedDataflowAnalysisTest.cpp
index 2425bb8711..8422f3804d 100644
--- a/clang/unittests/Analysis/FlowSensitive/TypeErasedDataflowAnalysisTest.cpp
+++ b/clang/unittests/Analysis/FlowSensitive/TypeErasedDataflowAnalysisTest.cpp
@@ -47,7 +47,6 @@ using namespace test;
 using namespace ast_matchers;
 using llvm::IsStringMapEntry;
 using ::testing::DescribeMatcher;
-using ::testing::ElementsAre;
 using ::testing::IsEmpty;
 using ::testing::NotNull;
 using ::testing::Test;
@@ -913,6 +912,29 @@ TEST_F(FlowConditionTest, WhileStmt) {
       });
 }
 
+TEST_F(FlowConditionTest, WhileStmtWithAssignmentInCondition) {
+  std::string Code = R"(
+    void target(bool Foo) {
+      // This test checks whether the analysis preserves the connection between
+      // the value of `Foo` and the assignment expression, despite widening.
+      // The equality operator generates a fresh boolean variable on each
+      // interpretation, which forces use of widening.
+      while ((Foo = (3 == 4))) {
+        (void)0;
+        /*[[p]]*/
+      }
+    }
+  )";
+  runDataflow(
+      Code,
+      [](const llvm::StringMap<DataflowAnalysisState<NoopLattice>> &Results,
+         ASTContext &ASTCtx) {
+        const Environment &Env = getEnvironmentAtAnnotation(Results, "p");
+        auto &FooVal = getValueForDecl<BoolValue>(ASTCtx, Env, "Foo").formula();
+        EXPECT_TRUE(Env.flowConditionImplies(FooVal));
+      });
+}
+
 TEST_F(FlowConditionTest, Conjunction) {
   std::string Code = R"(
     void target(bool Foo, bool Bar) {
diff --git a/clang/unittests/Analysis/FlowSensitive/UncheckedOptionalAccessModelTest.cpp b/clang/unittests/Analysis/FlowSensitive/UncheckedOptionalAccessModelTest.cpp
index 3f17557272..76af8baba8 100644
--- a/clang/unittests/Analysis/FlowSensitive/UncheckedOptionalAccessModelTest.cpp
+++ b/clang/unittests/Analysis/FlowSensitive/UncheckedOptionalAccessModelTest.cpp
@@ -1280,6 +1280,12 @@ class UncheckedOptionalAccessTest
     ExpectDiagnosticsFor(SourceCode, ast_matchers::hasName("target"));
   }
 
+  void ExpectDiagnosticsForLambda(std::string SourceCode) {
+    ExpectDiagnosticsFor(
+        SourceCode, ast_matchers::hasDeclContext(
+                        ast_matchers::cxxRecordDecl(ast_matchers::isLambda())));
+  }
+
   template <typename FuncDeclMatcher>
   void ExpectDiagnosticsFor(std::string SourceCode,
                             FuncDeclMatcher FuncMatcher) {
@@ -2125,6 +2131,24 @@ TEST_P(UncheckedOptionalAccessTest, OptionalSwap) {
   )");
 }
 
+TEST_P(UncheckedOptionalAccessTest, OptionalReturnedFromFuntionCall) {
+  ExpectDiagnosticsFor(
+      R"(
+    #include "unchecked_optional_access_test.h"
+    
+    struct S {
+      $ns::$optional<float> x;
+    } s;
+    S getOptional() {
+      return s;
+    }
+
+    void target() {
+      getOptional().x = 0;
+    }
+  )");
+}
+
 TEST_P(UncheckedOptionalAccessTest, StdSwap) {
   ExpectDiagnosticsFor(
       R"(
@@ -3214,6 +3238,137 @@ TEST_P(UncheckedOptionalAccessTest, Bitfield) {
     }
   )");
 }
+
+TEST_P(UncheckedOptionalAccessTest, LambdaParam) {
+  ExpectDiagnosticsForLambda(R"(
+    #include "unchecked_optional_access_test.h"
+
+    void target() {
+      []($ns::$optional<int> opt) {
+        if (opt.has_value()) {
+          opt.value();
+        } else {
+          opt.value(); // [[unsafe]]
+        }
+      }(Make<$ns::$optional<int>>());
+    }
+  )");
+}
+
+TEST_P(UncheckedOptionalAccessTest, LambdaCaptureByCopy) {
+  ExpectDiagnosticsForLambda(R"(
+    #include "unchecked_optional_access_test.h"
+
+    void target($ns::$optional<int> opt) {
+      [opt]() {
+        if (opt.has_value()) {
+          opt.value();
+        } else {
+          opt.value(); // [[unsafe]]
+        }
+      }();
+    }
+  )");
+}
+
+TEST_P(UncheckedOptionalAccessTest, LambdaCaptureByReference) {
+  ExpectDiagnosticsForLambda(R"(
+    #include "unchecked_optional_access_test.h"
+
+    void target($ns::$optional<int> opt) {
+      [&opt]() {
+        if (opt.has_value()) {
+          opt.value();
+        } else {
+          opt.value(); // [[unsafe]]
+        }
+      }();
+    }
+  )");
+}
+
+TEST_P(UncheckedOptionalAccessTest, LambdaCaptureWithInitializer) {
+  ExpectDiagnosticsForLambda(R"(
+    #include "unchecked_optional_access_test.h"
+
+    void target($ns::$optional<int> opt) {
+      [opt2=opt]() {
+        if (opt2.has_value()) {
+          opt2.value();
+        } else {
+          opt2.value(); // [[unsafe]]
+        }
+      }();
+    }
+  )");
+}
+
+TEST_P(UncheckedOptionalAccessTest, LambdaCaptureByCopyImplicit) {
+  ExpectDiagnosticsForLambda(R"(
+    #include "unchecked_optional_access_test.h"
+
+    void target($ns::$optional<int> opt) {
+      [=]() {
+        if (opt.has_value()) {
+          opt.value();
+        } else {
+          opt.value(); // [[unsafe]]
+        }
+      }();
+    }
+  )");
+}
+
+TEST_P(UncheckedOptionalAccessTest, LambdaCaptureByReferenceImplicit) {
+  ExpectDiagnosticsForLambda(R"(
+    #include "unchecked_optional_access_test.h"
+
+    void target($ns::$optional<int> opt) {
+      [&]() {
+        if (opt.has_value()) {
+          opt.value();
+        } else {
+          opt.value(); // [[unsafe]]
+        }
+      }();
+    }
+  )");
+}
+
+TEST_P(UncheckedOptionalAccessTest, LambdaCaptureThis) {
+  ExpectDiagnosticsForLambda(R"(
+    #include "unchecked_optional_access_test.h"
+
+    struct Foo {
+      $ns::$optional<int> opt;
+
+      void target() {
+        [this]() {
+          if (opt.has_value()) {
+            opt.value();
+          } else {
+            opt.value(); // [[unsafe]]
+          }
+        }();
+      }
+    };
+  )");
+}
+
+TEST_P(UncheckedOptionalAccessTest, LambdaCaptureStateNotPropagated) {
+  // We can't propagate information from the surrounding context.
+  ExpectDiagnosticsForLambda(R"(
+    #include "unchecked_optional_access_test.h"
+
+    void target($ns::$optional<int> opt) {
+      if (opt.has_value()) {
+        [&opt]() {
+          opt.value(); // [[unsafe]]
+        }();
+      }
+    }
+  )");
+}
 // FIXME: Add support for:
 // - constructors (copy, move)
 // - assignment operators (default, copy, move)
diff --git a/clang/unittests/Driver/CMakeLists.txt b/clang/unittests/Driver/CMakeLists.txt
index e37c158d71..752037f78f 100644
--- a/clang/unittests/Driver/CMakeLists.txt
+++ b/clang/unittests/Driver/CMakeLists.txt
@@ -9,6 +9,7 @@ set(LLVM_LINK_COMPONENTS
 add_clang_unittest(ClangDriverTests
   DistroTest.cpp
   DXCModeTest.cpp
+  GCCVersionTest.cpp
   ToolChainTest.cpp
   ModuleCacheTest.cpp
   MultilibBuilderTest.cpp
diff --git a/clang/unittests/Driver/GCCVersionTest.cpp b/clang/unittests/Driver/GCCVersionTest.cpp
new file mode 100644
index 0000000000..88c26dfe81
--- /dev/null
+++ b/clang/unittests/Driver/GCCVersionTest.cpp
@@ -0,0 +1,59 @@
+//===- unittests/Driver/GCCVersionTest.cpp --- GCCVersion parser tests ----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Unit tests for Generic_GCC::GCCVersion
+//
+//===----------------------------------------------------------------------===//
+
+#include "../../lib/Driver/ToolChains/Gnu.h"
+#include "gtest/gtest.h"
+
+// The Generic_GCC class is hidden in dylib/shared library builds, so
+// this test can only be built if neither of those configurations are
+// enabled.
+#if !defined(LLVM_BUILD_LLVM_DYLIB) && !defined(LLVM_BUILD_SHARED_LIBS)
+
+using namespace clang;
+using namespace clang::driver;
+
+namespace {
+
+struct VersionParseTest {
+  std::string Text;
+
+  int Major, Minor, Patch;
+  std::string MajorStr, MinorStr, PatchSuffix;
+};
+
+const VersionParseTest TestCases[] = {
+    {"5", 5, -1, -1, "5", "", ""},
+    {"4.4", 4, 4, -1, "4", "4", ""},
+    {"4.4-patched", 4, 4, -1, "4", "4", "-patched"},
+    {"4.4.0", 4, 4, 0, "4", "4", ""},
+    {"4.4.x", 4, 4, -1, "4", "4", ""},
+    {"4.4.2-rc4", 4, 4, 2, "4", "4", "-rc4"},
+    {"4.4.x-patched", 4, 4, -1, "4", "4", ""},
+    {"not-a-version", -1, -1, -1, "", "", ""},
+};
+
+TEST(GCCVersionTest, Parse) {
+  for (const auto &TC : TestCases) {
+    auto V = toolchains::Generic_GCC::GCCVersion::Parse(TC.Text);
+    EXPECT_EQ(V.Text, TC.Text);
+    EXPECT_EQ(V.Major, TC.Major);
+    EXPECT_EQ(V.Minor, TC.Minor);
+    EXPECT_EQ(V.Patch, TC.Patch);
+    EXPECT_EQ(V.MajorStr, TC.MajorStr);
+    EXPECT_EQ(V.MinorStr, TC.MinorStr);
+    EXPECT_EQ(V.PatchSuffix, TC.PatchSuffix);
+  }
+}
+
+} // end anonymous namespace
+
+#endif
diff --git a/clang/unittests/Format/ConfigParseTest.cpp b/clang/unittests/Format/ConfigParseTest.cpp
index dedaf546ea..ba79c8d72f 100644
--- a/clang/unittests/Format/ConfigParseTest.cpp
+++ b/clang/unittests/Format/ConfigParseTest.cpp
@@ -63,6 +63,13 @@ TEST(ConfigParseTest, GetsPredefinedStyleByName) {
   EXPECT_TRUE(getPredefinedStyle("gnU", FormatStyle::LK_Cpp, &Styles[2]));
   EXPECT_ALL_STYLES_EQUAL(Styles);
 
+  Styles[0] = getClangFormatStyle();
+  EXPECT_TRUE(
+      getPredefinedStyle("clang-format", FormatStyle::LK_Cpp, &Styles[1]));
+  EXPECT_TRUE(
+      getPredefinedStyle("Clang-format", FormatStyle::LK_Cpp, &Styles[2]));
+  EXPECT_ALL_STYLES_EQUAL(Styles);
+
   EXPECT_FALSE(getPredefinedStyle("qwerty", FormatStyle::LK_Cpp, &Styles[0]));
 }
 
@@ -591,6 +598,24 @@ TEST(ConfigParseTest, ParsesConfiguration) {
               SpaceBeforeParens,
               FormatStyle::SBPO_ControlStatementsExceptControlMacros);
 
+  Style.SpaceBeforeParens = FormatStyle::SBPO_Custom;
+  Style.SpaceBeforeParensOptions.AfterPlacementOperator =
+      FormatStyle::SpaceBeforeParensCustom::APO_Always;
+  CHECK_PARSE("SpaceBeforeParensOptions:\n"
+              "  AfterPlacementOperator: Never",
+              SpaceBeforeParensOptions.AfterPlacementOperator,
+              FormatStyle::SpaceBeforeParensCustom::APO_Never);
+
+  CHECK_PARSE("SpaceBeforeParensOptions:\n"
+              "  AfterPlacementOperator: Always",
+              SpaceBeforeParensOptions.AfterPlacementOperator,
+              FormatStyle::SpaceBeforeParensCustom::APO_Always);
+
+  CHECK_PARSE("SpaceBeforeParensOptions:\n"
+              "  AfterPlacementOperator: Leave",
+              SpaceBeforeParensOptions.AfterPlacementOperator,
+              FormatStyle::SpaceBeforeParensCustom::APO_Leave);
+
   // For backward compatibility:
   Style.SpacesInParens = FormatStyle::SIPO_Never;
   Style.SpacesInParensOptions = {};
diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp
index 2ef3c9b299..0a87cfc4f1 100644
--- a/clang/unittests/Format/FormatTest.cpp
+++ b/clang/unittests/Format/FormatTest.cpp
@@ -11189,6 +11189,42 @@ TEST_F(FormatTest, UnderstandsNewAndDelete) {
                "void delete(link p);",
                "void new (link p);\n"
                "void delete (link p);");
+
+  FormatStyle AfterPlacementOperator = getLLVMStyle();
+  AfterPlacementOperator.SpaceBeforeParens = FormatStyle::SBPO_Custom;
+  EXPECT_EQ(
+      AfterPlacementOperator.SpaceBeforeParensOptions.AfterPlacementOperator,
+      FormatStyle::SpaceBeforeParensCustom::APO_Leave);
+  verifyFormat("new (buf) int;", AfterPlacementOperator);
+  verifyFormat("new(buf) int;", AfterPlacementOperator);
+
+  AfterPlacementOperator.SpaceBeforeParensOptions.AfterPlacementOperator =
+      FormatStyle::SpaceBeforeParensCustom::APO_Never;
+  verifyFormat("struct A {\n"
+               "  int *a;\n"
+               "  A(int *p) : a(new(p) int) {\n"
+               "    new(p) int;\n"
+               "    int *b = new(p) int;\n"
+               "    int *c = new(p) int(3);\n"
+               "    delete(b);\n"
+               "  }\n"
+               "};",
+               AfterPlacementOperator);
+  verifyFormat("void operator new(void *foo) ATTRIB;", AfterPlacementOperator);
+
+  AfterPlacementOperator.SpaceBeforeParensOptions.AfterPlacementOperator =
+      FormatStyle::SpaceBeforeParensCustom::APO_Always;
+  verifyFormat("struct A {\n"
+               "  int *a;\n"
+               "  A(int *p) : a(new (p) int) {\n"
+               "    new (p) int;\n"
+               "    int *b = new (p) int;\n"
+               "    int *c = new (p) int(3);\n"
+               "    delete (b);\n"
+               "  }\n"
+               "};",
+               AfterPlacementOperator);
+  verifyFormat("void operator new(void *foo) ATTRIB;", AfterPlacementOperator);
 }
 
 TEST_F(FormatTest, UnderstandsUsesOfStarAndAmp) {
@@ -11674,6 +11710,9 @@ TEST_F(FormatTest, UnderstandsAttributes) {
   verifyFormat("vector<SomeType __attr1 *const> v;", CustomAttrs);
   verifyFormat("vector<SomeType __attr1 *__attr2> v;", CustomAttrs);
   verifyFormat("vector<SomeType __attr1 *no_underscore_attr> v;", CustomAttrs);
+  verifyFormat("__attr1 ::qualified_type f();", CustomAttrs);
+  verifyFormat("__attr1() ::qualified_type f();", CustomAttrs);
+  verifyFormat("__attr1(nodebug) ::qualified_type f();", CustomAttrs);
 
   // Check that these are not parsed as function declarations:
   CustomAttrs.AllowShortFunctionsOnASingleLine = FormatStyle::SFS_None;
diff --git a/clang/unittests/Format/FormatTestObjC.cpp b/clang/unittests/Format/FormatTestObjC.cpp
index a9e5434dfa..84a3d24005 100644
--- a/clang/unittests/Format/FormatTestObjC.cpp
+++ b/clang/unittests/Format/FormatTestObjC.cpp
@@ -1527,7 +1527,10 @@ TEST_F(FormatTestObjC, IfNotUnlikely) {
                "  [obj func:arg2];");
 }
 
-TEST_F(FormatTestObjC, Attributes) {
+TEST_F(FormatTestObjC, AttributesOnObjCDecl) {
+  Style.AttributeMacros.push_back("ATTRIBUTE_MACRO");
+
+  // Check '__attribute__' macro directly.
   verifyFormat("__attribute__((objc_subclassing_restricted))\n"
                "@interface Foo\n"
                "@end");
@@ -1537,6 +1540,215 @@ TEST_F(FormatTestObjC, Attributes) {
   verifyFormat("__attribute__((objc_subclassing_restricted))\n"
                "@implementation Foo\n"
                "@end");
+
+  // Check AttributeMacro gets treated the same, with or without parentheses.
+  verifyFormat("ATTRIBUTE_MACRO\n"
+               "@interface Foo\n"
+               "@end");
+  verifyFormat("ATTRIBUTE_MACRO(X)\n"
+               "@interface Foo\n"
+               "@end");
+
+  // Indenter also needs to understand multiple attribute macros.
+  // Try each of the three kinds paired with each of the other kind.
+
+  // Column limit, but no reflow.
+  verifyFormat("ATTRIBUTE_MACRO(X) ATTRIBUTE_MACRO\n"
+               "@interface Foo\n"
+               "@end");
+  verifyFormat("ATTRIBUTE_MACRO ATTRIBUTE_MACRO(X)\n"
+               "@interface Foo\n"
+               "@end");
+  verifyFormat("__attribute__((X)) ATTRIBUTE_MACRO\n"
+               "@interface Foo\n"
+               "@end");
+  verifyFormat("ATTRIBUTE_MACRO __attribute__((X))\n"
+               "@interface Foo\n"
+               "@end");
+  verifyFormat("__attribute__((X)) ATTRIBUTE_MACRO(X)\n"
+               "@interface Foo\n"
+               "@end");
+  verifyFormat("ATTRIBUTE_MACRO(X) __attribute__((X))\n"
+               "@interface Foo\n"
+               "@end");
+
+  // Column limit that requires reflow.
+  Style.ColumnLimit = 30;
+  verifyFormat("ATTRIBUTE_MACRO(X)\n"
+               "ATTRIBUTE_MACRO\n"
+               "@interface Foo\n"
+               "@end");
+  verifyFormat("ATTRIBUTE_MACRO\n"
+               "ATTRIBUTE_MACRO(X)\n"
+               "@interface Foo\n"
+               "@end");
+  verifyFormat("__attribute__((X))\n"
+               "ATTRIBUTE_MACRO\n"
+               "@interface Foo\n"
+               "@end");
+  verifyFormat("ATTRIBUTE_MACRO\n"
+               "__attribute__((X))\n"
+               "@interface Foo\n"
+               "@end");
+  verifyFormat("__attribute__((X))\n"
+               "ATTRIBUTE_MACRO(X)\n"
+               "@interface Foo\n"
+               "@end");
+  verifyFormat("ATTRIBUTE_MACRO(X)\n"
+               "__attribute__((X))\n"
+               "@interface Foo\n"
+               "@end");
+
+  // No column limit
+  Style.ColumnLimit = 0;
+  verifyFormat("ATTRIBUTE_MACRO(X) ATTRIBUTE_MACRO\n"
+               "@interface Foo\n"
+               "@end");
+  verifyFormat("ATTRIBUTE_MACRO ATTRIBUTE_MACRO(X)\n"
+               "@interface Foo\n"
+               "@end");
+  verifyFormat("__attribute__((X)) ATTRIBUTE_MACRO\n"
+               "@interface Foo\n"
+               "@end");
+  verifyFormat("ATTRIBUTE_MACRO __attribute__((X))\n"
+               "@interface Foo\n"
+               "@end");
+  verifyFormat("__attribute__((X)) ATTRIBUTE_MACRO(X)\n"
+               "@interface Foo\n"
+               "@end");
+  verifyFormat("ATTRIBUTE_MACRO(X) __attribute__((X))\n"
+               "@interface Foo\n"
+               "@end");
+}
+
+TEST_F(FormatTestObjC, AttributesOnObjCMethodDecl) {
+  Style.AttributeMacros.push_back("ATTRIBUTE_MACRO");
+
+  // Check '__attribute__' macro directly.
+  verifyFormat("- (id)init __attribute__((objc_designated_initializer));");
+
+  // Check AttributeMacro gets treated the same, with or without parentheses.
+  verifyFormat("- (id)init ATTRIBUTE_MACRO;");
+  verifyFormat("- (id)init ATTRIBUTE_MACRO(X);");
+
+  // Indenter also needs to understand multiple attribute macros.
+
+  // Column limit (default), but no reflow.
+  verifyFormat("- (id)init ATTRIBUTE_MACRO(X) ATTRIBUTE_MACRO;");
+  verifyFormat("- (id)init ATTRIBUTE_MACRO ATTRIBUTE_MACRO(X);");
+  verifyFormat("- (id)init __attribute__((X)) ATTRIBUTE_MACRO;");
+  verifyFormat("- (id)init ATTRIBUTE_MACRO __attribute__((X));");
+  verifyFormat("- (id)init __attribute__((X)) ATTRIBUTE_MACRO(X);");
+  verifyFormat("- (id)init ATTRIBUTE_MACRO(X) __attribute__((X));");
+
+  // Column limit that requires reflow.
+  Style.ColumnLimit = 30;
+
+  // Reflow after method name.
+  verifyFormat("- (id)initWithReallyLongName\n"
+               "    __attribute__((X))\n"
+               "    ATTRIBUTE_MACRO;");
+  verifyFormat("- (id)initWithReallyLongName\n"
+               "    ATTRIBUTE_MACRO(X)\n"
+               "    ATTRIBUTE_MACRO;");
+  verifyFormat("- (id)initWithReallyLongName\n"
+               "    ATTRIBUTE_MACRO\n"
+               "    ATTRIBUTE_MACRO;");
+  // Reflow after first macro.
+  // FIXME: these should indent but don't.
+#if 0
+  verifyFormat("- (id)init ATTRIBUTE_MACRO(X)\n"
+               "    ATTRIBUTE_MACRO;");
+  verifyFormat("- (id)init ATTRIBUTE_MACRO\n"
+               "    ATTRIBUTE_MACRO(X);");
+  verifyFormat("- (id)init __attribute__((X))\n"
+               "    ATTRIBUTE_MACRO;");
+  verifyFormat("- (id)init ATTRIBUTE_MACRO\n"
+               "    __attribute__((X));");
+  verifyFormat("- (id)init __attribute__((X))\n"
+               "    ATTRIBUTE_MACRO(X);");
+  verifyFormat("- (id)init ATTRIBUTE_MACRO(X)\n"
+               "    __attribute__((X));");
+#endif
+
+  // No column limit.
+  Style.ColumnLimit = 0;
+  verifyFormat("- (id)init ATTRIBUTE_MACRO(X) ATTRIBUTE_MACRO;");
+  verifyFormat("- (id)init ATTRIBUTE_MACRO ATTRIBUTE_MACRO(X);");
+  verifyFormat("- (id)init __attribute__((X)) ATTRIBUTE_MACRO;");
+  verifyFormat("- (id)init ATTRIBUTE_MACRO __attribute__((X));");
+  verifyFormat("- (id)init __attribute__((X)) ATTRIBUTE_MACRO(X);");
+  verifyFormat("- (id)init ATTRIBUTE_MACRO(X) __attribute__((X));");
+}
+
+TEST_F(FormatTestObjC, AttributesOnObjCProperty) {
+  Style.AttributeMacros.push_back("ATTRIBUTE_MACRO");
+
+  // Check '__attribute__' macro directly.
+  verifyFormat("@property(weak) id delegate "
+               "__attribute__((objc_designated_initializer));");
+
+  // Check AttributeMacro gets treated the same, with or without parentheses.
+  verifyFormat("@property(weak) id delegate ATTRIBUTE_MACRO;");
+  verifyFormat("@property(weak) id delegate ATTRIBUTE_MACRO(X);");
+
+  // Indenter also needs to understand multiple attribute macros.
+
+  // Column limit (default), but no reflow.
+  verifyFormat(
+      "@property(weak) id delegate ATTRIBUTE_MACRO(X) ATTRIBUTE_MACRO;");
+  verifyFormat(
+      "@property(weak) id delegate ATTRIBUTE_MACRO ATTRIBUTE_MACRO(X);");
+  verifyFormat(
+      "@property(weak) id delegate __attribute__((X)) ATTRIBUTE_MACRO;");
+  verifyFormat(
+      "@property(weak) id delegate ATTRIBUTE_MACRO __attribute__((X));");
+  verifyFormat(
+      "@property(weak) id delegate __attribute__((X)) ATTRIBUTE_MACRO(X);");
+  verifyFormat(
+      "@property(weak) id delegate ATTRIBUTE_MACRO(X) __attribute__((X));");
+
+  // Column limit that requires reflow.
+  Style.ColumnLimit = 50;
+
+  // Reflow after method name.
+  verifyFormat("@property(weak) id delegateWithLongName\n"
+               "    __attribute__((X)) ATTRIBUTE_MACRO;");
+  verifyFormat("@property(weak) id delegateWithLongName\n"
+               "    ATTRIBUTE_MACRO(X) ATTRIBUTE_MACRO;");
+  verifyFormat("@property(weak) id delegateWithLongName\n"
+               "    ATTRIBUTE_MACRO ATTRIBUTE_MACRO;");
+  // Reflow after first macro.
+  // FIXME: these should indent but don't.
+#if 0
+  verifyFormat("@property(weak) id delegate ATTRIBUTE_MACRO(X)\n"
+               "    ATTRIBUTE_MACRO;");
+  verifyFormat("@property(weak) id delegate ATTRIBUTE_MACRO\n"
+               "    ATTRIBUTE_MACRO(X);");
+  verifyFormat("@property(weak) id delegate __attribute__((X))\n"
+               "    ATTRIBUTE_MACRO;");
+  verifyFormat("@property(weak) id delegate ATTRIBUTE_MACRO\n"
+               "    __attribute__((X));");
+  verifyFormat("@property(weak) id delegate __attribute__((X))\n"
+               "    ATTRIBUTE_MACRO(X);");
+  verifyFormat("@property(weak) id delegate ATTRIBUTE_MACRO(X)\n"
+               "    __attribute__((X));");
+#endif
+
+  // No column limit.
+  Style.ColumnLimit = 0;
+  verifyFormat(
+      "@property(weak) id delegate ATTRIBUTE_MACRO(X) ATTRIBUTE_MACRO;");
+  verifyFormat(
+      "@property(weak) id delegate ATTRIBUTE_MACRO ATTRIBUTE_MACRO(X);");
+  verifyFormat(
+      "@property(weak) id delegate __attribute__((X)) ATTRIBUTE_MACRO;");
+  verifyFormat(
+      "@property(weak) id delegate ATTRIBUTE_MACRO __attribute__((X));");
+  verifyFormat(
+      "@property(weak) id delegate __attribute__((X)) ATTRIBUTE_MACRO(X);");
+  verifyFormat(
+      "@property(weak) id delegate ATTRIBUTE_MACRO(X) __attribute__((X));");
 }
 
 } // end namespace
diff --git a/clang/unittests/Format/FormatTestRawStrings.cpp b/clang/unittests/Format/FormatTestRawStrings.cpp
index 6f9a0d650b..10f341cc8f 100644
--- a/clang/unittests/Format/FormatTestRawStrings.cpp
+++ b/clang/unittests/Format/FormatTestRawStrings.cpp
@@ -17,9 +17,6 @@
 
 #define DEBUG_TYPE "format-test"
 
-using clang::tooling::ReplacementTest;
-using clang::tooling::toReplacements;
-
 namespace clang {
 namespace format {
 namespace {
diff --git a/clang/unittests/Format/TokenAnnotatorTest.cpp b/clang/unittests/Format/TokenAnnotatorTest.cpp
index 2d590f2af0..290d0103bb 100644
--- a/clang/unittests/Format/TokenAnnotatorTest.cpp
+++ b/clang/unittests/Format/TokenAnnotatorTest.cpp
@@ -1620,6 +1620,44 @@ TEST_F(TokenAnnotatorTest, UnderstandsLambdas) {
   EXPECT_TOKEN(Tokens[15], tok::kw_requires, TT_RequiresClause);
   EXPECT_TRUE(Tokens[19]->ClosesRequiresClause);
   EXPECT_TOKEN(Tokens[20], tok::l_brace, TT_LambdaLBrace);
+
+  Tokens = annotate("[] <typename T = int> (T t) {}");
+  ASSERT_EQ(Tokens.size(), 15u) << Tokens;
+  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
+  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
+  EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);
+  EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_LambdaLBrace);
+
+  Tokens = annotate("[] <int I = 0> (T t) {}");
+  ASSERT_EQ(Tokens.size(), 15u) << Tokens;
+  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
+  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
+  EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);
+  EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_LambdaLBrace);
+
+  Tokens = annotate("[] <bool b = false> (T t) {}");
+  ASSERT_EQ(Tokens.size(), 15u) << Tokens;
+  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
+  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
+  EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);
+  EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_LambdaLBrace);
+
+  Tokens = annotate("[] <bool b = true && false> (T&& t) {}");
+  ASSERT_EQ(Tokens.size(), 18u) << Tokens;
+  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
+  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
+  EXPECT_TOKEN(Tokens[7], tok::ampamp, TT_BinaryOperator);
+  EXPECT_TOKEN(Tokens[9], tok::greater, TT_TemplateCloser);
+  EXPECT_TOKEN(Tokens[12], tok::ampamp, TT_PointerOrReference);
+  EXPECT_TOKEN(Tokens[15], tok::l_brace, TT_LambdaLBrace);
+
+  Tokens = annotate("[] <typename T = int> requires Foo<T> (T t) {}");
+  ASSERT_EQ(Tokens.size(), 20u) << Tokens;
+  EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
+  EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
+  EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);
+  EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresClause);
+  EXPECT_TOKEN(Tokens[17], tok::l_brace, TT_LambdaLBrace);
 }
 
 TEST_F(TokenAnnotatorTest, UnderstandsFunctionAnnotations) {
@@ -1788,6 +1826,10 @@ TEST_F(TokenAnnotatorTest, UnderstandsTrailingReturnArrow) {
   ASSERT_EQ(Tokens.size(), 15u) << Tokens;
   EXPECT_TOKEN(Tokens[12], tok::arrow, TT_Unknown);
 
+  Tokens = annotate("void f() FOO(foo->bar);");
+  ASSERT_EQ(Tokens.size(), 12u) << Tokens;
+  EXPECT_TOKEN(Tokens[7], tok::arrow, TT_Unknown);
+
   // Mixed
   Tokens = annotate("auto f() -> int { auto a = b()->c; }");
   ASSERT_EQ(Tokens.size(), 18u) << Tokens;
@@ -1795,6 +1837,116 @@ TEST_F(TokenAnnotatorTest, UnderstandsTrailingReturnArrow) {
   EXPECT_TOKEN(Tokens[13], tok::arrow, TT_Unknown);
 }
 
+TEST_F(TokenAnnotatorTest, UnderstandsAttributeMacros) {
+  // '__attribute__' has special handling.
+  auto Tokens = annotate("__attribute__(X) void Foo(void);");
+  ASSERT_EQ(Tokens.size(), 11u) << Tokens;
+  EXPECT_TOKEN(Tokens[0], tok::kw___attribute, TT_Unknown);
+  EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_AttributeLParen);
+  EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_AttributeRParen);
+
+  // Generic macro has no special handling in this location.
+  Tokens = annotate("A(X) void Foo(void);");
+  ASSERT_EQ(Tokens.size(), 11u) << Tokens;
+  EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);
+  EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_Unknown);
+
+  // Add a custom AttributeMacro. Test that it has the same behavior.
+  FormatStyle Style = getLLVMStyle();
+  Style.AttributeMacros.push_back("A");
+
+  // An "AttributeMacro" gets annotated like '__attribute__'.
+  Tokens = annotate("A(X) void Foo(void);", Style);
+  ASSERT_EQ(Tokens.size(), 11u) << Tokens;
+  EXPECT_TOKEN(Tokens[0], tok::identifier, TT_AttributeMacro);
+  EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_AttributeLParen);
+  EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_AttributeRParen);
+}
+
+TEST_F(TokenAnnotatorTest, UnderstandsAttributeMacrosOnObjCDecl) {
+  // '__attribute__' has special handling.
+  auto Tokens = annotate("__attribute__(X) @interface Foo");
+  ASSERT_EQ(Tokens.size(), 8u) << Tokens;
+  EXPECT_TOKEN(Tokens[0], tok::kw___attribute, TT_Unknown);
+  EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_AttributeLParen);
+  EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_AttributeRParen);
+
+  // Generic macro has no special handling in this location.
+  Tokens = annotate("A(X) @interface Foo");
+  ASSERT_EQ(Tokens.size(), 8u) << Tokens;
+  // Note: Don't check token-type as a random token in this position is hard to
+  // reason about.
+  EXPECT_TOKEN_KIND(Tokens[0], tok::identifier);
+  EXPECT_TOKEN_KIND(Tokens[1], tok::l_paren);
+
+  // Add a custom AttributeMacro. Test that it has the same behavior.
+  FormatStyle Style = getLLVMStyle();
+  Style.AttributeMacros.push_back("A");
+
+  // An "AttributeMacro" gets annotated like '__attribute__'.
+  Tokens = annotate("A(X) @interface Foo", Style);
+  ASSERT_EQ(Tokens.size(), 8u) << Tokens;
+  EXPECT_TOKEN(Tokens[0], tok::identifier, TT_AttributeMacro);
+  EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_AttributeLParen);
+  EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_AttributeRParen);
+}
+
+TEST_F(TokenAnnotatorTest, UnderstandsAttributeMacrosOnObjCMethodDecl) {
+  // '__attribute__' has special handling.
+  auto Tokens = annotate("- (id)init __attribute__(X);");
+  ASSERT_EQ(Tokens.size(), 11u) << Tokens;
+  EXPECT_TOKEN(Tokens[5], tok::kw___attribute, TT_Unknown);
+  EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_AttributeLParen);
+  EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_AttributeRParen);
+
+  // Generic macro has no special handling in this location.
+  Tokens = annotate("- (id)init A(X);");
+  ASSERT_EQ(Tokens.size(), 11u) << Tokens;
+  // Note: Don't check token-type as a random token in this position is hard to
+  // reason about.
+  EXPECT_TOKEN_KIND(Tokens[5], tok::identifier);
+  EXPECT_TOKEN_KIND(Tokens[6], tok::l_paren);
+
+  // Add a custom AttributeMacro. Test that it has the same behavior.
+  FormatStyle Style = getLLVMStyle();
+  Style.AttributeMacros.push_back("A");
+
+  // An "AttributeMacro" gets annotated like '__attribute__'.
+  Tokens = annotate("- (id)init A(X);", Style);
+  ASSERT_EQ(Tokens.size(), 11u) << Tokens;
+  EXPECT_TOKEN(Tokens[5], tok::identifier, TT_AttributeMacro);
+  EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_AttributeLParen);
+  EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_AttributeRParen);
+}
+
+TEST_F(TokenAnnotatorTest, UnderstandsAttributeMacrosOnObjCProperty) {
+  // '__attribute__' has special handling.
+  auto Tokens = annotate("@property(weak) id delegate __attribute__(X);");
+  ASSERT_EQ(Tokens.size(), 13u) << Tokens;
+  EXPECT_TOKEN(Tokens[7], tok::kw___attribute, TT_Unknown);
+  EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_AttributeLParen);
+  EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_AttributeRParen);
+
+  // Generic macro has no special handling in this location.
+  Tokens = annotate("@property(weak) id delegate A(X);");
+  ASSERT_EQ(Tokens.size(), 13u) << Tokens;
+  // Note: Don't check token-type as a random token in this position is hard to
+  // reason about.
+  EXPECT_TOKEN_KIND(Tokens[7], tok::identifier);
+  EXPECT_TOKEN_KIND(Tokens[8], tok::l_paren);
+
+  // Add a custom AttributeMacro. Test that it has the same behavior.
+  FormatStyle Style = getLLVMStyle();
+  Style.AttributeMacros.push_back("A");
+
+  // An "AttributeMacro" gets annotated like '__attribute__'.
+  Tokens = annotate("@property(weak) id delegate A(X);", Style);
+  ASSERT_EQ(Tokens.size(), 13u) << Tokens;
+  EXPECT_TOKEN(Tokens[7], tok::identifier, TT_AttributeMacro);
+  EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_AttributeLParen);
+  EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_AttributeRParen);
+}
+
 TEST_F(TokenAnnotatorTest, UnderstandsVerilogOperators) {
   auto Annotate = [this](llvm::StringRef Code) {
     return annotate(Code, getLLVMStyle(FormatStyle::LK_Verilog));
@@ -2151,6 +2303,47 @@ TEST_F(TokenAnnotatorTest, UnderstandsAttributes) {
   EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_AttributeRParen);
 }
 
+TEST_F(TokenAnnotatorTest, UnderstandsControlStatements) {
+  auto Tokens = annotate("while (true) {}");
+  ASSERT_EQ(Tokens.size(), 7u) << Tokens;
+  EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_ControlStatementLBrace);
+  EXPECT_TOKEN(Tokens[5], tok::r_brace, TT_ControlStatementRBrace);
+
+  Tokens = annotate("for (;;) {}");
+  ASSERT_EQ(Tokens.size(), 8u) << Tokens;
+  EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_ControlStatementLBrace);
+  EXPECT_TOKEN(Tokens[6], tok::r_brace, TT_ControlStatementRBrace);
+
+  Tokens = annotate("do {} while (true);");
+  ASSERT_EQ(Tokens.size(), 9u) << Tokens;
+  EXPECT_TOKEN(Tokens[1], tok::l_brace, TT_ControlStatementLBrace);
+  EXPECT_TOKEN(Tokens[2], tok::r_brace, TT_ControlStatementRBrace);
+
+  Tokens = annotate("if (true) {} else if (false) {} else {}");
+  ASSERT_EQ(Tokens.size(), 17u) << Tokens;
+  EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_ControlStatementLBrace);
+  EXPECT_TOKEN(Tokens[5], tok::r_brace, TT_ControlStatementRBrace);
+  EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_ControlStatementLBrace);
+  EXPECT_TOKEN(Tokens[12], tok::r_brace, TT_ControlStatementRBrace);
+  EXPECT_TOKEN(Tokens[14], tok::l_brace, TT_ElseLBrace);
+  EXPECT_TOKEN(Tokens[15], tok::r_brace, TT_ElseRBrace);
+
+  Tokens = annotate("switch (foo) {}");
+  ASSERT_EQ(Tokens.size(), 7u) << Tokens;
+  EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_ControlStatementLBrace);
+  EXPECT_TOKEN(Tokens[5], tok::r_brace, TT_ControlStatementRBrace);
+}
+
+TEST_F(TokenAnnotatorTest, UnderstandsDoWhile) {
+  auto Tokens = annotate("do { ++i; } while ( i > 5 );");
+  ASSERT_EQ(Tokens.size(), 14u) << Tokens;
+  EXPECT_TOKEN(Tokens[6], tok::kw_while, TT_DoWhile);
+
+  Tokens = annotate("do ++i; while ( i > 5 );");
+  ASSERT_EQ(Tokens.size(), 12u) << Tokens;
+  EXPECT_TOKEN(Tokens[4], tok::kw_while, TT_DoWhile);
+}
+
 } // namespace
 } // namespace format
 } // namespace clang
diff --git a/clang/unittests/Index/IndexTests.cpp b/clang/unittests/Index/IndexTests.cpp
index 4d19f47283..8e9a1c6bf8 100644
--- a/clang/unittests/Index/IndexTests.cpp
+++ b/clang/unittests/Index/IndexTests.cpp
@@ -428,6 +428,31 @@ TEST(IndexTest, NonTypeTemplateParameter) {
                              WrittenAt(Position(3, 15)))));
 }
 
+TEST(IndexTest, ReadWriteRoles) {
+  std::string Code = R"cpp(
+    int main() {
+      int foo = 0;
+      foo = 2;
+      foo += 1;
+      int bar = foo;
+  }
+  )cpp";
+  auto Index = std::make_shared<Indexer>();
+  IndexingOptions Opts;
+  Opts.IndexFunctionLocals = true;
+  tooling::runToolOnCode(std::make_unique<IndexAction>(Index, Opts), Code);
+  EXPECT_THAT(
+      Index->Symbols,
+      AllOf(Contains(AllOf(QName("foo"), HasRole(SymbolRole::Write),
+                           WrittenAt(Position(4, 7)))),
+            Contains(AllOf(QName("foo"),
+                           HasRole(static_cast<unsigned>(SymbolRole::Read) |
+                                   static_cast<unsigned>(SymbolRole::Write)),
+                           WrittenAt(Position(5, 7)))),
+            Contains(AllOf(QName("foo"), HasRole(SymbolRole::Read),
+                           WrittenAt(Position(6, 17))))));
+}
+
 } // namespace
 } // namespace index
 } // namespace clang
diff --git a/clang/unittests/Tooling/RangeSelectorTest.cpp b/clang/unittests/Tooling/RangeSelectorTest.cpp
index cbb8e25bb9..03ab66235e 100644
--- a/clang/unittests/Tooling/RangeSelectorTest.cpp
+++ b/clang/unittests/Tooling/RangeSelectorTest.cpp
@@ -28,7 +28,6 @@ using ::llvm::HasValue;
 using ::llvm::StringError;
 using ::testing::AllOf;
 using ::testing::HasSubstr;
-using ::testing::Property;
 
 using MatchResult = MatchFinder::MatchResult;
 
diff --git a/clang/utils/TableGen/RISCVVEmitter.cpp b/clang/utils/TableGen/RISCVVEmitter.cpp
index 4102592605..07d3164218 100644
--- a/clang/utils/TableGen/RISCVVEmitter.cpp
+++ b/clang/utils/TableGen/RISCVVEmitter.cpp
@@ -290,7 +290,7 @@ unsigned SemaSignatureTable::getIndex(ArrayRef<PrototypeDescriptor> Signature) {
     return 0;
 
   // Checking Signature already in table or not.
-  if (Signature.size() < SignatureTable.size()) {
+  if (Signature.size() <= SignatureTable.size()) {
     size_t Bound = SignatureTable.size() - Signature.size() + 1;
     for (size_t Index = 0; Index < Bound; ++Index) {
       if (equal(Signature.begin(), Signature.end(),
@@ -656,6 +656,8 @@ void RVVEmitter::createRVVIntrinsics(
                                   .Case("RV64", RVV_REQ_RV64)
                                   .Case("ZvfhminOrZvfh", RVV_REQ_ZvfhminOrZvfh)
                                   .Case("Xsfvcp", RVV_REQ_Xsfvcp)
+                                  .Case("Xsfvqmaccdod", RVV_REQ_Xsfvqmaccdod)
+                                  .Case("Xsfvqmaccqoq", RVV_REQ_Xsfvqmaccqoq)
                                   .Case("Zvbb", RVV_REQ_Zvbb)
                                   .Case("Zvbc", RVV_REQ_Zvbc)
                                   .Case("Zvkb", RVV_REQ_Zvkb)
diff --git a/clang/utils/TableGen/SveEmitter.cpp b/clang/utils/TableGen/SveEmitter.cpp
index f725c39540..1421a5b03e 100644
--- a/clang/utils/TableGen/SveEmitter.cpp
+++ b/clang/utils/TableGen/SveEmitter.cpp
@@ -73,12 +73,12 @@ class SVEType {
 public:
   SVEType() : SVEType(TypeSpec(), 'v') {}
 
-  SVEType(TypeSpec TS, char CharMod)
+  SVEType(TypeSpec TS, char CharMod, unsigned NumVectors = 1)
       : TS(TS), Float(false), Signed(true), Immediate(false), Void(false),
         Constant(false), Pointer(false), BFloat(false), DefaultType(false),
         IsScalable(true), Predicate(false), PredicatePattern(false),
         PrefetchOp(false), Svcount(false), Bitwidth(128), ElementBitwidth(~0U),
-        NumVectors(1) {
+        NumVectors(NumVectors) {
     if (!TS.empty())
       applyTypespec();
     applyModifier(CharMod);
@@ -194,7 +194,9 @@ class Intrinsic {
   SVEType getReturnType() const { return Types[0]; }
   ArrayRef<SVEType> getTypes() const { return Types; }
   SVEType getParamType(unsigned I) const { return Types[I + 1]; }
-  unsigned getNumParams() const { return Proto.size() - 1; }
+  unsigned getNumParams() const {
+    return Proto.size() - (2 * llvm::count(Proto, '.')) - 1;
+  }
 
   uint64_t getFlags() const { return Flags; }
   bool isFlagSet(uint64_t Flag) const { return Flags & Flag;}
@@ -228,11 +230,19 @@ class Intrinsic {
 
   /// Return the parameter index of the splat operand.
   unsigned getSplatIdx() const {
-    // These prototype modifiers are described in arm_sve.td.
-    auto Idx = Proto.find_first_of("ajfrKLR@");
-    assert(Idx != std::string::npos && Idx > 0 &&
-           "Prototype has no splat operand");
-    return Idx - 1;
+    unsigned I = 1, Param = 0;
+    for (; I < Proto.size(); ++I, ++Param) {
+      if (Proto[I] == 'a' || Proto[I] == 'j' || Proto[I] == 'f' ||
+          Proto[I] == 'r' || Proto[I] == 'K' || Proto[I] == 'L' ||
+          Proto[I] == 'R' || Proto[I] == '@')
+        break;
+
+      // Multivector modifier can be skipped
+      if (Proto[I] == '.')
+        I += 2;
+    }
+    assert(I != Proto.size() && "Prototype has no splat operand");
+    return Param;
   }
 
   /// Emits the intrinsic declaration to the ostream.
@@ -540,15 +550,6 @@ void SVEType::applyTypespec() {
 
 void SVEType::applyModifier(char Mod) {
   switch (Mod) {
-  case '2':
-    NumVectors = 2;
-    break;
-  case '3':
-    NumVectors = 3;
-    break;
-  case '4':
-    NumVectors = 4;
-    break;
   case 'v':
     Void = true;
     break;
@@ -851,6 +852,13 @@ void SVEType::applyModifier(char Mod) {
     NumVectors = 0;
     Signed = false;
     break;
+  case '$':
+    Predicate = false;
+    Svcount = false;
+    Float = false;
+    BFloat = true;
+    ElementBitwidth = 16;
+    break;
   case '}':
     Predicate = false;
     Signed = true;
@@ -859,11 +867,36 @@ void SVEType::applyModifier(char Mod) {
     Float = false;
     BFloat = false;
     break;
+  case '.':
+    llvm_unreachable(". is never a type in itself");
+    break;
   default:
     llvm_unreachable("Unhandled character!");
   }
 }
 
+/// Returns the modifier and number of vectors for the given operand \p Op.
+std::pair<char, unsigned> getProtoModifier(StringRef Proto, unsigned Op) {
+  for (unsigned P = 0; !Proto.empty(); ++P) {
+    unsigned NumVectors = 1;
+    unsigned CharsToSkip = 1;
+    char Mod = Proto[0];
+    if (Mod == '2' || Mod == '3' || Mod == '4') {
+      NumVectors = Mod - '0';
+      Mod = 'd';
+      if (Proto.size() > 1 && Proto[1] == '.') {
+        Mod = Proto[2];
+        CharsToSkip = 3;
+      }
+    }
+
+    if (P == Op)
+      return {Mod, NumVectors};
+
+    Proto = Proto.drop_front(CharsToSkip);
+  }
+  llvm_unreachable("Unexpected Op");
+}
 
 //===----------------------------------------------------------------------===//
 // Intrinsic implementation
@@ -879,8 +912,11 @@ Intrinsic::Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy,
       MergeSuffix(MergeSuffix.str()), BaseType(BT, 'd'), Flags(Flags),
       ImmChecks(Checks.begin(), Checks.end()) {
   // Types[0] is the return value.
-  for (unsigned I = 0; I < Proto.size(); ++I) {
-    SVEType T(BaseTypeSpec, Proto[I]);
+  for (unsigned I = 0; I < (getNumParams() + 1); ++I) {
+    char Mod;
+    unsigned NumVectors;
+    std::tie(Mod, NumVectors) = getProtoModifier(Proto, I);
+    SVEType T(BaseTypeSpec, Mod, NumVectors);
     Types.push_back(T);
 
     // Add range checks for immediates
@@ -1124,10 +1160,11 @@ void SVEEmitter::createIntrinsic(
       assert(Arg >= 0 && Kind >= 0 && "Arg and Kind must be nonnegative");
 
       unsigned ElementSizeInBits = 0;
+      char Mod;
+      unsigned NumVectors;
+      std::tie(Mod, NumVectors) = getProtoModifier(Proto, EltSizeArg + 1);
       if (EltSizeArg >= 0)
-        ElementSizeInBits =
-            SVEType(TS, Proto[EltSizeArg + /* offset by return arg */ 1])
-                .getElementSizeInBits();
+        ElementSizeInBits = SVEType(TS, Mod, NumVectors).getElementSizeInBits();
       ImmChecks.push_back(ImmCheck(Arg, Kind, ElementSizeInBits));
     }
 
diff --git a/clang/utils/analyze_safe_buffer_debug_notes.py b/clang/utils/analyze_safe_buffer_debug_notes.py
new file mode 100644
index 0000000000..f0a6e4de6f
--- /dev/null
+++ b/clang/utils/analyze_safe_buffer_debug_notes.py
@@ -0,0 +1,39 @@
+import sys
+from collections import OrderedDict
+
+class Trie:
+    def __init__(self, name):
+        self.name = name
+        self.children = OrderedDict()
+        self.count = 1
+
+    def add(self, name):
+        if name in self.children:
+            self.children[name].count += 1
+        else:
+            self.children[name] = Trie(name)
+        return self.children[name]
+
+    def print(self, depth):
+        if depth > 0:
+            print('|', end="")
+        for i in range(depth):
+            print('-', end="")
+        if depth > 0:
+            print(end=" ")
+        print(self.name, '#', self.count)
+        for key, child in self.children.items():
+            child.print(depth + 1)
+
+
+Root = Trie("Root")
+
+if __name__ == "__main__":
+    for line in sys.stdin:
+        words = line.split('==>')
+        words = [word.strip() for word in words]
+        MyTrie = Root;
+        for word in words:
+            MyTrie = MyTrie.add(word)
+
+    Root.print(0)
diff --git a/clang/www/c_status.html b/clang/www/c_status.html
index 13bb1971c6..0e0ea0b179 100644
--- a/clang/www/c_status.html
+++ b/clang/www/c_status.html
@@ -887,6 +887,11 @@ <h2 id="c2x">C23 implementation status</h2>
       <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n2672.pdf">N2672</a></td>
       <td class="full" align="center">Yes</td>
     </tr>
+    <tr>
+      <td>Towards Integer Safety</td>
+      <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n2683.pdf">N2683</a></td>
+      <td class="full" align="center">Clang 18</td>
+    </tr>
     <tr id="_BitInt">
       <td rowspan="5">Adding Fundamental Type for N-bit Integers</td>
     </tr>
diff --git a/clang/www/get_involved.html b/clang/www/get_involved.html
index b5212ae973..3fc688837d 100755
--- a/clang/www/get_involved.html
+++ b/clang/www/get_involved.html
@@ -32,12 +32,12 @@ <h2>Follow what's going on</h2>
 and has a Discourse forum and mailing list:</p>
 
 <ul>
-<li><a href="https://lists.llvm.org/mailman/listinfo/cfe-commits">cfe-commits
-</a> - This list is for patch submission/discussion.</li>
-
 <li><a href="https://discourse.llvm.org/c/clang/6">Clang Frontend Discourse forum</a> -
-This forum is for everything else Clang related (questions and answers, design
-discussions, etc).</li>
+This forum is for discussions related to Clang (questions and answers, design
+discussions, RFCs, etc).</li>
+
+<li><a href="https://discord.gg/xS7Z362">Discord chat</a> - Real-time chat for
+discussions related to Clang (primarily for questions and answers).</li>
 
 <li>Regular meetings are held on the
 <a href="https://drive.google.com/file/d/1S7V0MHP4xMs1yUQ9Gv9LHn5bwDfFVapn/view?usp=sharing">
@@ -50,16 +50,23 @@ <h2>Follow what's going on</h2>
 here<a>.
 </li>
 
+<li><a href="https://llvm.org/docs/GettingInvolved.html#office-hours">Clang office hours</a> -
+People within the community hold dedicated office hours at different points
+during the month, which is a great way opportunity for getting questions
+answered, having more in-depth design discussions, or learning about what's
+going on in the community in general.</li>
+
+<li><a href="https://lists.llvm.org/mailman/listinfo/cfe-commits">cfe-commits
+</a> - Historical record of commits to Clang and contains early community patch
+review commentary.</li>
+
 </ul>
 
 <p>The most common way to talk with other developers on the project is through
 the <a href="https://discourse.llvm.org/c/clang/6">Clang Frontend Discourse forum
-</a>.  The clang forum is a very friendly place and we welcome
-newcomers.  In addition to the forum, a significant amount of design
-discussion takes place on the <a
-href="https://lists.llvm.org/mailman/listinfo/cfe-commits">cfe-commits mailing
-list</a>.  All of these lists have archives, so you can browse through previous
-discussions or follow the list development on the web if you prefer.</p>
+</a>. The clang forum is a very friendly place and we welcome newcomers. The
+forum is archived so you can browse through previous discussions or follow
+development on the web if you prefer.</p>
 
 <p>If you're looking for something to work on, check out our <a
 href="OpenProjects.html">Open Projects</a> page or look through the <a
diff --git a/compiler-rt/cmake/Modules/AddCompilerRT.cmake b/compiler-rt/cmake/Modules/AddCompilerRT.cmake
index a72e279dd7..5ed49f0f55 100644
--- a/compiler-rt/cmake/Modules/AddCompilerRT.cmake
+++ b/compiler-rt/cmake/Modules/AddCompilerRT.cmake
@@ -312,6 +312,10 @@ function(add_compiler_rt_runtime name type)
       set(COMPONENT_OPTION COMPONENT ${libname})
     endif()
 
+    if(type STREQUAL "SHARED")
+      list(APPEND LIB_DEFS COMPILER_RT_SHARED_LIB)
+    endif()
+
     if(type STREQUAL "OBJECT")
       if(CMAKE_C_COMPILER_ID MATCHES Clang AND CMAKE_C_COMPILER_TARGET)
         list(APPEND extra_cflags_${libname} "--target=${CMAKE_C_COMPILER_TARGET}")
diff --git a/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake b/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake
index e8ab660c1d..416777171d 100644
--- a/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake
+++ b/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake
@@ -66,7 +66,7 @@ set(ALL_PROFILE_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64} ${PPC32} ${PPC
     ${MIPS32} ${MIPS64} ${S390X} ${SPARC} ${SPARCV9} ${HEXAGON}
     ${RISCV32} ${RISCV64} ${LOONGARCH64})
 set(ALL_TSAN_SUPPORTED_ARCH ${X86_64} ${MIPS64} ${ARM64} ${PPC64} ${S390X}
-    ${LOONGARCH64})
+    ${LOONGARCH64} ${RISCV64})
 set(ALL_UBSAN_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64} ${RISCV64}
     ${MIPS32} ${MIPS64} ${PPC64} ${S390X} ${SPARC} ${SPARCV9} ${HEXAGON}
     ${LOONGARCH64})
diff --git a/compiler-rt/cmake/builtin-config-ix.cmake b/compiler-rt/cmake/builtin-config-ix.cmake
index 9cf4877baf..e91e3923a7 100644
--- a/compiler-rt/cmake/builtin-config-ix.cmake
+++ b/compiler-rt/cmake/builtin-config-ix.cmake
@@ -33,6 +33,12 @@ asm(\".arch armv8-a+lse\");
 asm(\"cas w0, w1, [x2]\");
 ")
 
+builtin_check_c_compiler_source(COMPILER_RT_HAS_ASM_SME
+"
+asm(\".arch armv9-a+sme\");
+asm(\"smstart\");
+")
+
 if(ANDROID)
   set(OS_NAME "Android")
 else()
diff --git a/compiler-rt/lib/asan/asan_interceptors.cpp b/compiler-rt/lib/asan/asan_interceptors.cpp
index fc7c5b7b8f..876682c3b6 100644
--- a/compiler-rt/lib/asan/asan_interceptors.cpp
+++ b/compiler-rt/lib/asan/asan_interceptors.cpp
@@ -251,7 +251,7 @@ INTERCEPTOR(int, pthread_create, void *thread, void *attr,
 
   u32 current_tid = GetCurrentTidOrInvalid();
 
-  __sanitizer_sigset_t sigset;
+  __sanitizer_sigset_t sigset = {};
 #    if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \
         SANITIZER_SOLARIS
   ScopedBlockSignals block(&sigset);
diff --git a/compiler-rt/lib/builtins/CMakeLists.txt b/compiler-rt/lib/builtins/CMakeLists.txt
index 1afceddc62..cf2648233b 100644
--- a/compiler-rt/lib/builtins/CMakeLists.txt
+++ b/compiler-rt/lib/builtins/CMakeLists.txt
@@ -280,6 +280,7 @@ endif ()
 # long double is not 80 bits on Android or MSVC.
 set(x86_80_BIT_SOURCES
   divxc3.c
+  extendxftf2.c
   fixxfdi.c
   fixxfti.c
   fixunsxfdi.c
@@ -291,6 +292,7 @@ set(x86_80_BIT_SOURCES
   floatuntixf.c
   mulxc3.c
   powixf2.c
+  trunctfxf2.c
 )
 
 if (NOT MSVC)
@@ -553,6 +555,13 @@ set(aarch64_SOURCES
   aarch64/fp_mode.c
 )
 
+if(COMPILER_RT_HAS_ASM_SME AND (COMPILER_RT_HAS_AUXV OR COMPILER_RT_BAREMETAL_BUILD))
+  list(APPEND aarch64_SOURCES aarch64/sme-abi.S aarch64/sme-abi-init.c)
+  message(STATUS "AArch64 SME ABI routines enabled")
+else()
+  message(STATUS "AArch64 SME ABI routines disabled")
+endif()
+
 # Generate outline atomics helpers from lse.S base
 set(OA_HELPERS_DIR "${CMAKE_CURRENT_BINARY_DIR}/outline_atomic_helpers.dir")
 file(MAKE_DIRECTORY "${OA_HELPERS_DIR}")
diff --git a/compiler-rt/lib/builtins/aarch64/sme-abi-init.c b/compiler-rt/lib/builtins/aarch64/sme-abi-init.c
new file mode 100644
index 0000000000..b6ee12170d
--- /dev/null
+++ b/compiler-rt/lib/builtins/aarch64/sme-abi-init.c
@@ -0,0 +1,52 @@
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+__attribute__((visibility("hidden"), nocommon))
+_Bool __aarch64_has_sme_and_tpidr2_el0;
+
+// We have multiple ways to check that the function has SME, depending on our
+// target.
+// * For Linux we can use __getauxval().
+// * For newlib we can use __aarch64_sme_accessible().
+
+#if defined(__linux__)
+
+#ifndef AT_HWCAP2
+#define AT_HWCAP2 26
+#endif
+
+#ifndef HWCAP2_SME
+#define HWCAP2_SME (1 << 23)
+#endif
+
+extern unsigned long int __getauxval (unsigned long int);
+
+static _Bool has_sme(void) {
+  return __getauxval(AT_HWCAP2) & HWCAP2_SME;
+}
+
+#else  // defined(__linux__)
+
+#if defined(COMPILER_RT_SHARED_LIB)
+__attribute__((weak))
+#endif
+extern _Bool __aarch64_sme_accessible(void);
+
+static _Bool has_sme(void)  {
+#if defined(COMPILER_RT_SHARED_LIB)
+  if (!__aarch64_sme_accessible)
+    return 0;
+#endif
+  return __aarch64_sme_accessible();
+}
+
+#endif // defined(__linux__)
+
+#if __GNUC__ >= 9
+#pragma GCC diagnostic ignored "-Wprio-ctor-dtor"
+#endif
+__attribute__((constructor(90)))
+static void init_aarch64_has_sme(void) {
+  __aarch64_has_sme_and_tpidr2_el0 = has_sme();
+}
diff --git a/compiler-rt/lib/builtins/aarch64/sme-abi.S b/compiler-rt/lib/builtins/aarch64/sme-abi.S
new file mode 100644
index 0000000000..b3612c6806
--- /dev/null
+++ b/compiler-rt/lib/builtins/aarch64/sme-abi.S
@@ -0,0 +1,176 @@
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// This patch implements the support routines for the SME ABI,
+// described here:
+//  https://github.com/ARM-software/abi-aa/blob/main/aapcs64/aapcs64.rst#sme-support-routines
+
+#include "../assembly.h"
+
+
+#if !defined(__APPLE__)
+#define TPIDR2_SYMBOL SYMBOL_NAME(__aarch64_has_sme_and_tpidr2_el0)
+#define TPIDR2_SYMBOL_OFFSET :lo12:SYMBOL_NAME(__aarch64_has_sme_and_tpidr2_el0)
+#else
+// MachO requires @page/@pageoff directives because the global is defined
+// in a different file. Otherwise this file may fail to build.
+#define TPIDR2_SYMBOL SYMBOL_NAME(__aarch64_has_sme_and_tpidr2_el0)@page
+#define TPIDR2_SYMBOL_OFFSET SYMBOL_NAME(__aarch64_has_sme_and_tpidr2_el0)@pageoff
+#endif
+
+.arch armv9-a+sme
+
+// Utility function which calls a system's abort() routine. Because the function
+// is streaming-compatible it should disable streaming-SVE mode before calling
+// abort(). Note that there is no need to preserve any state before the call,
+// because the function does not return.
+DEFINE_COMPILERRT_PRIVATE_FUNCTION(do_abort)
+.cfi_startproc
+	.variant_pcs SYMBOL_NAME(do_abort)
+	stp	x29, x30, [sp, #-32]!
+  cntd x0
+  // Store VG to a stack location that we describe with .cfi_offset
+  str x0, [sp, #16]
+  .cfi_def_cfa_offset 32
+  .cfi_offset w30, -24
+  .cfi_offset w29, -32
+  .cfi_offset vg, -16
+	bl	__arm_sme_state
+	tbz	x0, #0, 2f
+1:
+	smstop sm
+2:
+  // We can't make this into a tail-call because the unwinder would
+  // need to restore the value of VG.
+	bl	SYMBOL_NAME(abort)
+.cfi_endproc
+END_COMPILERRT_FUNCTION(do_abort)
+
+// __arm_sme_state fills the result registers based on a local
+// that is set as part of the compiler-rt startup code.
+//   __aarch64_has_sme_and_tpidr2_el0
+DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(__arm_sme_state)
+	.variant_pcs __arm_sme_state
+  mov x0, xzr
+  mov x1, xzr
+
+  adrp  x16, TPIDR2_SYMBOL
+  ldrb w16, [x16, TPIDR2_SYMBOL_OFFSET]
+  cbz w16, 1f
+0:
+  orr x0, x0, #0xC000000000000000
+  mrs x16, SVCR
+  bfxil x0, x16, #0, #2
+  mrs x1, TPIDR2_EL0
+1:
+  ret
+END_COMPILERRT_OUTLINE_FUNCTION(__arm_sme_state)
+
+DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(__arm_tpidr2_restore)
+	.variant_pcs __arm_tpidr2_restore
+  // If TPIDR2_EL0 is nonnull, the subroutine aborts in some platform-specific
+  // manner.
+  mrs x14, TPIDR2_EL0
+  cbnz  x14, 2f
+
+  // If any of the reserved bytes in the first 16 bytes of BLK are nonzero,
+  // the subroutine [..] aborts in some platform-defined manner.
+  ldrh  w14, [x0, #10]
+  cbnz  w14, 2f
+  ldr w14, [x0, #12]
+  cbnz  w14, 2f
+
+  // If BLK.za_save_buffer is NULL, the subroutine does nothing.
+  ldr x16, [x0]
+  cbz x16, 1f
+
+  // If BLK.num_za_save_slices is zero, the subroutine does nothing.
+  ldrh  w14, [x0, #8]
+  cbz x14, 1f
+
+  mov x15, xzr
+0:
+  ldr za[w15,0], [x16]
+  addsvl x16, x16, #1
+  add x15, x15, #1
+  cmp x14, x15
+  b.ne  0b
+1:
+  ret
+2:
+  b  SYMBOL_NAME(do_abort)
+END_COMPILERRT_OUTLINE_FUNCTION(__arm_tpidr2_restore)
+
+DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(__arm_tpidr2_save)
+	.variant_pcs __arm_tpidr2_restore
+  // If the current thread does not have access to TPIDR2_EL0, the subroutine
+  // does nothing.
+  adrp  x14, TPIDR2_SYMBOL
+  ldrb w14, [x14, TPIDR2_SYMBOL_OFFSET]
+  cbz w14, 1f
+
+  // If TPIDR2_EL0 is null, the subroutine does nothing.
+  mrs x16, TPIDR2_EL0
+  cbz x16, 1f
+
+  // If any of the reserved bytes in the first 16 bytes of the TPIDR2 block are
+  // nonzero, the subroutine [..] aborts in some platform-defined manner.
+  ldrh  w14, [x16, #10]
+  cbnz  w14, 2f
+  ldr w14, [x16, #12]
+  cbnz  w14, 2f
+
+  // If num_za_save_slices is zero, the subroutine does nothing.
+  ldrh  w14, [x16, #8]
+  cbz x14, 1f
+
+  // If za_save_buffer is NULL, the subroutine does nothing.
+  ldr x16, [x16]
+  cbz x16, 1f
+
+  mov x15, xzr
+0:
+  str za[w15,0], [x16]
+  addsvl x16, x16, #1
+  add x15, x15, #1
+  cmp x14, x15
+  b.ne  0b
+1:
+  ret
+2:
+  b  SYMBOL_NAME(do_abort)
+END_COMPILERRT_OUTLINE_FUNCTION(__arm_tpidr2_save)
+
+DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(__arm_za_disable)
+	.variant_pcs __arm_tpidr2_restore
+  // If the current thread does not have access to SME, the subroutine does
+  // nothing.
+  adrp  x14, TPIDR2_SYMBOL
+  ldrb w14, [x14, TPIDR2_SYMBOL_OFFSET]
+  cbz w14, 0f
+
+  // Otherwise, the subroutine behaves as if it did the following:
+  // * Call __arm_tpidr2_save.
+  stp x29, x30, [sp, #-16]!
+  .cfi_def_cfa_offset 16
+  mov x29, sp
+  .cfi_def_cfa w29, 16
+  .cfi_offset w30, -8
+  .cfi_offset w29, -16
+  bl  __arm_tpidr2_save
+
+  // * Set TPIDR2_EL0 to null.
+  msr TPIDR2_EL0, xzr
+
+  // * Set PSTATE.ZA to 0.
+  smstop za
+
+  .cfi_def_cfa wsp, 16
+  ldp x29, x30, [sp], #16
+  .cfi_def_cfa_offset 0
+  .cfi_restore w30
+  .cfi_restore w29
+0:
+  ret
+END_COMPILERRT_OUTLINE_FUNCTION(__arm_za_disable)
diff --git a/compiler-rt/lib/builtins/cpu_model.c b/compiler-rt/lib/builtins/cpu_model.c
index e413a78817..aefa56abcd 100644
--- a/compiler-rt/lib/builtins/cpu_model.c
+++ b/compiler-rt/lib/builtins/cpu_model.c
@@ -79,6 +79,7 @@ enum ProcessorTypes {
   ZHAOXIN_FAM7H,
   INTEL_SIERRAFOREST,
   INTEL_GRANDRIDGE,
+  INTEL_CLEARWATERFOREST,
   CPU_TYPE_MAX
 };
 
@@ -116,6 +117,7 @@ enum ProcessorSubtypes {
   INTEL_COREI7_GRANITERAPIDS_D,
   INTEL_COREI7_ARROWLAKE,
   INTEL_COREI7_ARROWLAKE_S,
+  INTEL_COREI7_PANTHERLAKE,
   CPU_SUBTYPE_MAX
 };
 
@@ -492,6 +494,13 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
       *Subtype = INTEL_COREI7_ARROWLAKE_S;
       break;
 
+    // Pantherlake:
+    case 0xcc:
+      CPU = "pantherlake";
+      *Type = INTEL_COREI7;
+      *Subtype = INTEL_COREI7_PANTHERLAKE;
+      break;
+
     // Icelake Xeon:
     case 0x6a:
     case 0x6c:
@@ -572,6 +581,13 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
       *Type = INTEL_GRANDRIDGE;
       break;
 
+    // Clearwaterforest:
+    case 0xdd:
+      CPU = "clearwaterforest";
+      *Type = INTEL_COREI7;
+      *Subtype = INTEL_CLEARWATERFOREST;
+      break;
+
     case 0x57:
       CPU = "knl";
       *Type = INTEL_KNL;
diff --git a/compiler-rt/lib/builtins/extendxftf2.c b/compiler-rt/lib/builtins/extendxftf2.c
new file mode 100644
index 0000000000..20911fe7cf
--- /dev/null
+++ b/compiler-rt/lib/builtins/extendxftf2.c
@@ -0,0 +1,23 @@
+//===-- lib/extendxftf2.c - long double -> quad conversion --------*- C -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Assumption: long double is a IEEE 80 bit floating point type padded to 128
+// bits.
+
+// TODO: use fp_lib.h once QUAD_PRECISION is available on x86_64.
+#if __LDBL_MANT_DIG__ == 64 && defined(__x86_64__) &&                          \
+    (defined(__FLOAT128__) || defined(__SIZEOF_FLOAT128__))
+#define SRC_80
+#define DST_QUAD
+#include "fp_extend_impl.inc"
+
+COMPILER_RT_ABI __float128 __extendxftf2(long double a) {
+  return __extendXfYf2__(a);
+}
+
+#endif
diff --git a/compiler-rt/lib/builtins/floatdidf.c b/compiler-rt/lib/builtins/floatdidf.c
index c994aad3f0..6da81f7a05 100644
--- a/compiler-rt/lib/builtins/floatdidf.c
+++ b/compiler-rt/lib/builtins/floatdidf.c
@@ -45,53 +45,11 @@ COMPILER_RT_ABI double __floatdidf(di_int a) {
 // flags to set, and we don't want to code-gen to an unknown soft-float
 // implementation.
 
-COMPILER_RT_ABI double __floatdidf(di_int a) {
-  if (a == 0)
-    return 0.0;
-  const unsigned N = sizeof(di_int) * CHAR_BIT;
-  const di_int s = a >> (N - 1);
-  a = (du_int)(a ^ s) - s;
-  int sd = N - __builtin_clzll(a); // number of significant digits
-  int e = sd - 1;                  // exponent
-  if (sd > DBL_MANT_DIG) {
-    //  start:  0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx
-    //  finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR
-    //                                                12345678901234567890123456
-    //  1 = msb 1 bit
-    //  P = bit DBL_MANT_DIG-1 bits to the right of 1
-    // Q = bit DBL_MANT_DIG bits to the right of 1
-    //  R = "or" of all bits to the right of Q
-    switch (sd) {
-    case DBL_MANT_DIG + 1:
-      a <<= 1;
-      break;
-    case DBL_MANT_DIG + 2:
-      break;
-    default:
-      a = ((du_int)a >> (sd - (DBL_MANT_DIG + 2))) |
-          ((a & ((du_int)(-1) >> ((N + DBL_MANT_DIG + 2) - sd))) != 0);
-    };
-    // finish:
-    a |= (a & 4) != 0; // Or P into R
-    ++a;               // round - this step may add a significant bit
-    a >>= 2;           // dump Q and R
-    // a is now rounded to DBL_MANT_DIG or DBL_MANT_DIG+1 bits
-    if (a & ((du_int)1 << DBL_MANT_DIG)) {
-      a >>= 1;
-      ++e;
-    }
-    // a is now rounded to DBL_MANT_DIG bits
-  } else {
-    a <<= (DBL_MANT_DIG - sd);
-    // a is now rounded to DBL_MANT_DIG bits
-  }
-  double_bits fb;
-  fb.u.s.high = ((su_int)s & 0x80000000) |        // sign
-                ((su_int)(e + 1023) << 20) |      // exponent
-                ((su_int)(a >> 32) & 0x000FFFFF); // mantissa-high
-  fb.u.s.low = (su_int)a;                         // mantissa-low
-  return fb.f;
-}
+#define SRC_I64
+#define DST_DOUBLE
+#include "int_to_fp_impl.inc"
+
+COMPILER_RT_ABI double __floatdidf(di_int a) { return __floatXiYf__(a); }
 #endif
 
 #if defined(__ARM_EABI__)
diff --git a/compiler-rt/lib/builtins/floatdisf.c b/compiler-rt/lib/builtins/floatdisf.c
index 0b62ed8689..0bb88c5c51 100644
--- a/compiler-rt/lib/builtins/floatdisf.c
+++ b/compiler-rt/lib/builtins/floatdisf.c
@@ -19,52 +19,11 @@
 
 #include "int_lib.h"
 
-COMPILER_RT_ABI float __floatdisf(di_int a) {
-  if (a == 0)
-    return 0.0F;
-  const unsigned N = sizeof(di_int) * CHAR_BIT;
-  const di_int s = a >> (N - 1);
-  a = (du_int)(a ^ s) - s;
-  int sd = N - __builtin_clzll(a); // number of significant digits
-  si_int e = sd - 1;               // exponent
-  if (sd > FLT_MANT_DIG) {
-    //  start:  0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx
-    //  finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR
-    //                                                12345678901234567890123456
-    //  1 = msb 1 bit
-    //  P = bit FLT_MANT_DIG-1 bits to the right of 1
-    //  Q = bit FLT_MANT_DIG bits to the right of 1
-    //  R = "or" of all bits to the right of Q
-    switch (sd) {
-    case FLT_MANT_DIG + 1:
-      a <<= 1;
-      break;
-    case FLT_MANT_DIG + 2:
-      break;
-    default:
-      a = ((du_int)a >> (sd - (FLT_MANT_DIG + 2))) |
-          ((a & ((du_int)(-1) >> ((N + FLT_MANT_DIG + 2) - sd))) != 0);
-    };
-    // finish:
-    a |= (a & 4) != 0; // Or P into R
-    ++a;               // round - this step may add a significant bit
-    a >>= 2;           // dump Q and R
-    // a is now rounded to FLT_MANT_DIG or FLT_MANT_DIG+1 bits
-    if (a & ((du_int)1 << FLT_MANT_DIG)) {
-      a >>= 1;
-      ++e;
-    }
-    // a is now rounded to FLT_MANT_DIG bits
-  } else {
-    a <<= (FLT_MANT_DIG - sd);
-    // a is now rounded to FLT_MANT_DIG bits
-  }
-  float_bits fb;
-  fb.u = ((su_int)s & 0x80000000) | // sign
-         ((e + 127) << 23) |        // exponent
-         ((su_int)a & 0x007FFFFF);  // mantissa
-  return fb.f;
-}
+#define SRC_I64
+#define DST_SINGLE
+#include "int_to_fp_impl.inc"
+
+COMPILER_RT_ABI float __floatdisf(di_int a) { return __floatXiYf__(a); }
 
 #if defined(__ARM_EABI__)
 #if defined(COMPILER_RT_ARMHF_TARGET)
diff --git a/compiler-rt/lib/builtins/floattidf.c b/compiler-rt/lib/builtins/floattidf.c
index 7bfe87f53a..ef8fe180e2 100644
--- a/compiler-rt/lib/builtins/floattidf.c
+++ b/compiler-rt/lib/builtins/floattidf.c
@@ -14,6 +14,10 @@
 
 #ifdef CRT_HAS_128BIT
 
+#define SRC_I128
+#define DST_DOUBLE
+#include "int_to_fp_impl.inc"
+
 // Returns: convert a to a double, rounding toward even.
 
 // Assumption: double is a IEEE 64 bit floating point type
@@ -22,52 +26,6 @@
 // seee eeee eeee mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm
 // mmmm
 
-COMPILER_RT_ABI double __floattidf(ti_int a) {
-  if (a == 0)
-    return 0.0;
-  const unsigned N = sizeof(ti_int) * CHAR_BIT;
-  const ti_int s = a >> (N - 1);
-  a = (a ^ s) - s;
-  int sd = N - __clzti2(a); // number of significant digits
-  si_int e = sd - 1;        // exponent
-  if (sd > DBL_MANT_DIG) {
-    // start:  0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx
-    //  finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR
-    //                                               12345678901234567890123456
-    // 1 = msb 1 bit
-    // P = bit DBL_MANT_DIG-1 bits to the right of 1
-    // Q = bit DBL_MANT_DIG bits to the right of 1
-    // R = "or" of all bits to the right of Q
-    switch (sd) {
-    case DBL_MANT_DIG + 1:
-      a <<= 1;
-      break;
-    case DBL_MANT_DIG + 2:
-      break;
-    default:
-      a = ((tu_int)a >> (sd - (DBL_MANT_DIG + 2))) |
-          ((a & ((tu_int)(-1) >> ((N + DBL_MANT_DIG + 2) - sd))) != 0);
-    };
-    // finish:
-    a |= (a & 4) != 0; // Or P into R
-    ++a;               // round - this step may add a significant bit
-    a >>= 2;           // dump Q and R
-    // a is now rounded to DBL_MANT_DIG or DBL_MANT_DIG+1 bits
-    if (a & ((tu_int)1 << DBL_MANT_DIG)) {
-      a >>= 1;
-      ++e;
-    }
-    // a is now rounded to DBL_MANT_DIG bits
-  } else {
-    a <<= (DBL_MANT_DIG - sd);
-    // a is now rounded to DBL_MANT_DIG bits
-  }
-  double_bits fb;
-  fb.u.s.high = ((su_int)s & 0x80000000) |        // sign
-                ((e + 1023) << 20) |              // exponent
-                ((su_int)(a >> 32) & 0x000FFFFF); // mantissa-high
-  fb.u.s.low = (su_int)a;                         // mantissa-low
-  return fb.f;
-}
+COMPILER_RT_ABI double __floattidf(ti_int a) { return __floatXiYf__(a); }
 
 #endif // CRT_HAS_128BIT
diff --git a/compiler-rt/lib/builtins/floattisf.c b/compiler-rt/lib/builtins/floattisf.c
index 717cb361f0..77589902f5 100644
--- a/compiler-rt/lib/builtins/floattisf.c
+++ b/compiler-rt/lib/builtins/floattisf.c
@@ -14,6 +14,10 @@
 
 #ifdef CRT_HAS_128BIT
 
+#define SRC_I128
+#define DST_SINGLE
+#include "int_to_fp_impl.inc"
+
 // Returns: convert a to a float, rounding toward even.
 
 // Assumption: float is a IEEE 32 bit floating point type
@@ -21,51 +25,6 @@
 
 // seee eeee emmm mmmm mmmm mmmm mmmm mmmm
 
-COMPILER_RT_ABI float __floattisf(ti_int a) {
-  if (a == 0)
-    return 0.0F;
-  const unsigned N = sizeof(ti_int) * CHAR_BIT;
-  const ti_int s = a >> (N - 1);
-  a = (a ^ s) - s;
-  int sd = N - __clzti2(a); // number of significant digits
-  si_int e = sd - 1;        // exponent
-  if (sd > FLT_MANT_DIG) {
-    //  start:  0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx
-    // finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR
-    //                                                12345678901234567890123456
-    //  1 = msb 1 bit
-    //  P = bit FLT_MANT_DIG-1 bits to the right of 1
-    //  Q = bit FLT_MANT_DIG bits to the right of 1
-    //  R = "or" of all bits to the right of Q
-    switch (sd) {
-    case FLT_MANT_DIG + 1:
-      a <<= 1;
-      break;
-    case FLT_MANT_DIG + 2:
-      break;
-    default:
-      a = ((tu_int)a >> (sd - (FLT_MANT_DIG + 2))) |
-          ((a & ((tu_int)(-1) >> ((N + FLT_MANT_DIG + 2) - sd))) != 0);
-    };
-    // finish:
-    a |= (a & 4) != 0; // Or P into R
-    ++a;               // round - this step may add a significant bit
-    a >>= 2;           // dump Q and R
-    // a is now rounded to FLT_MANT_DIG or FLT_MANT_DIG+1 bits
-    if (a & ((tu_int)1 << FLT_MANT_DIG)) {
-      a >>= 1;
-      ++e;
-    }
-    // a is now rounded to FLT_MANT_DIG bits
-  } else {
-    a <<= (FLT_MANT_DIG - sd);
-    // a is now rounded to FLT_MANT_DIG bits
-  }
-  float_bits fb;
-  fb.u = ((su_int)s & 0x80000000) | // sign
-         ((e + 127) << 23) |        // exponent
-         ((su_int)a & 0x007FFFFF);  // mantissa
-  return fb.f;
-}
+COMPILER_RT_ABI float __floattisf(ti_int a) { return __floatXiYf__(a); }
 
 #endif // CRT_HAS_128BIT
diff --git a/compiler-rt/lib/builtins/floattitf.c b/compiler-rt/lib/builtins/floattitf.c
index fff0755c3b..697bc8fb11 100644
--- a/compiler-rt/lib/builtins/floattitf.c
+++ b/compiler-rt/lib/builtins/floattitf.c
@@ -16,6 +16,10 @@
 #include "fp_lib.h"
 #include "int_lib.h"
 
+#define SRC_I128
+#define DST_QUAD
+#include "int_to_fp_impl.inc"
+
 // Returns: convert a ti_int to a fp_t, rounding toward even.
 
 // Assumption: fp_t is a IEEE 128 bit floating point type
@@ -26,53 +30,6 @@
 // mmmm mmmm mmmm
 
 #if defined(CRT_HAS_TF_MODE)
-COMPILER_RT_ABI fp_t __floattitf(ti_int a) {
-  if (a == 0)
-    return 0.0;
-  const unsigned N = sizeof(ti_int) * CHAR_BIT;
-  const ti_int s = a >> (N - 1);
-  a = (a ^ s) - s;
-  int sd = N - __clzti2(a); // number of significant digits
-  int e = sd - 1;           // exponent
-  if (sd > TF_MANT_DIG) {
-    //  start:  0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx
-    //  finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR
-    //                                                12345678901234567890123456
-    //  1 = msb 1 bit
-    //  P = bit LDBL_MANT_DIG-1 bits to the right of 1
-    //  Q = bit LDBL_MANT_DIG bits to the right of 1
-    //  R = "or" of all bits to the right of Q
-    switch (sd) {
-    case TF_MANT_DIG + 1:
-      a <<= 1;
-      break;
-    case TF_MANT_DIG + 2:
-      break;
-    default:
-      a = ((tu_int)a >> (sd - (TF_MANT_DIG + 2))) |
-          ((a & ((tu_int)(-1) >> ((N + TF_MANT_DIG + 2) - sd))) != 0);
-    };
-    // finish:
-    a |= (a & 4) != 0; // Or P into R
-    ++a;               // round - this step may add a significant bit
-    a >>= 2;           // dump Q and R
-    // a is now rounded to LDBL_MANT_DIG or LDBL_MANT_DIG+1 bits
-    if (a & ((tu_int)1 << TF_MANT_DIG)) {
-      a >>= 1;
-      ++e;
-    }
-    // a is now rounded to LDBL_MANT_DIG bits
-  } else {
-    a <<= (TF_MANT_DIG - sd);
-    // a is now rounded to LDBL_MANT_DIG bits
-  }
-
-  long_double_bits fb;
-  fb.u.high.all = (s & 0x8000000000000000LL)            // sign
-                  | (du_int)(e + 16383) << 48           // exponent
-                  | ((a >> 64) & 0x0000ffffffffffffLL); // significand
-  fb.u.low.all = (du_int)(a);
-  return fb.f;
-}
+COMPILER_RT_ABI fp_t __floattitf(ti_int a) { return __floatXiYf__(a); }
 
 #endif
diff --git a/compiler-rt/lib/builtins/floatundidf.c b/compiler-rt/lib/builtins/floatundidf.c
index 2ec802cdc1..9743e96ec6 100644
--- a/compiler-rt/lib/builtins/floatundidf.c
+++ b/compiler-rt/lib/builtins/floatundidf.c
@@ -51,50 +51,11 @@ COMPILER_RT_ABI double __floatundidf(du_int a) {
 // flags to set, and we don't want to code-gen to an unknown soft-float
 // implementation.
 
-COMPILER_RT_ABI double __floatundidf(du_int a) {
-  if (a == 0)
-    return 0.0;
-  const unsigned N = sizeof(du_int) * CHAR_BIT;
-  int sd = N - __builtin_clzll(a); // number of significant digits
-  int e = sd - 1;                  // exponent
-  if (sd > DBL_MANT_DIG) {
-    //  start:  0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx
-    //  finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR
-    //                                                12345678901234567890123456
-    //  1 = msb 1 bit
-    //  P = bit DBL_MANT_DIG-1 bits to the right of 1
-    //  Q = bit DBL_MANT_DIG bits to the right of 1
-    //  R = "or" of all bits to the right of Q
-    switch (sd) {
-    case DBL_MANT_DIG + 1:
-      a <<= 1;
-      break;
-    case DBL_MANT_DIG + 2:
-      break;
-    default:
-      a = (a >> (sd - (DBL_MANT_DIG + 2))) |
-          ((a & ((du_int)(-1) >> ((N + DBL_MANT_DIG + 2) - sd))) != 0);
-    };
-    // finish:
-    a |= (a & 4) != 0; // Or P into R
-    ++a;               // round - this step may add a significant bit
-    a >>= 2;           // dump Q and R
-    // a is now rounded to DBL_MANT_DIG or DBL_MANT_DIG+1 bits
-    if (a & ((du_int)1 << DBL_MANT_DIG)) {
-      a >>= 1;
-      ++e;
-    }
-    // a is now rounded to DBL_MANT_DIG bits
-  } else {
-    a <<= (DBL_MANT_DIG - sd);
-    // a is now rounded to DBL_MANT_DIG bits
-  }
-  double_bits fb;
-  fb.u.s.high = ((su_int)(e + 1023) << 20) |      // exponent
-                ((su_int)(a >> 32) & 0x000FFFFF); // mantissa-high
-  fb.u.s.low = (su_int)a;                         // mantissa-low
-  return fb.f;
-}
+#define SRC_U64
+#define DST_DOUBLE
+#include "int_to_fp_impl.inc"
+
+COMPILER_RT_ABI double __floatundidf(du_int a) { return __floatXiYf__(a); }
 #endif
 
 #if defined(__ARM_EABI__)
diff --git a/compiler-rt/lib/builtins/floatundisf.c b/compiler-rt/lib/builtins/floatundisf.c
index 2a4157dc5e..d4b418efd4 100644
--- a/compiler-rt/lib/builtins/floatundisf.c
+++ b/compiler-rt/lib/builtins/floatundisf.c
@@ -19,49 +19,11 @@
 
 #include "int_lib.h"
 
-COMPILER_RT_ABI float __floatundisf(du_int a) {
-  if (a == 0)
-    return 0.0F;
-  const unsigned N = sizeof(du_int) * CHAR_BIT;
-  int sd = N - __builtin_clzll(a); // number of significant digits
-  si_int e = sd - 1;               // 8 exponent
-  if (sd > FLT_MANT_DIG) {
-    //  start:  0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx
-    //  finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR
-    //                                                12345678901234567890123456
-    //  1 = msb 1 bit
-    //  P = bit FLT_MANT_DIG-1 bits to the right of 1
-    //  Q = bit FLT_MANT_DIG bits to the right of 1
-    //  R = "or" of all bits to the right of Q
-    switch (sd) {
-    case FLT_MANT_DIG + 1:
-      a <<= 1;
-      break;
-    case FLT_MANT_DIG + 2:
-      break;
-    default:
-      a = (a >> (sd - (FLT_MANT_DIG + 2))) |
-          ((a & ((du_int)(-1) >> ((N + FLT_MANT_DIG + 2) - sd))) != 0);
-    };
-    // finish:
-    a |= (a & 4) != 0; // Or P into R
-    ++a;               // round - this step may add a significant bit
-    a >>= 2;           // dump Q and R
-    // a is now rounded to FLT_MANT_DIG or FLT_MANT_DIG+1 bits
-    if (a & ((du_int)1 << FLT_MANT_DIG)) {
-      a >>= 1;
-      ++e;
-    }
-    // a is now rounded to FLT_MANT_DIG bits
-  } else {
-    a <<= (FLT_MANT_DIG - sd);
-    // a is now rounded to FLT_MANT_DIG bits
-  }
-  float_bits fb;
-  fb.u = ((e + 127) << 23) |       // exponent
-         ((su_int)a & 0x007FFFFF); // mantissa
-  return fb.f;
-}
+#define SRC_U64
+#define DST_SINGLE
+#include "int_to_fp_impl.inc"
+
+COMPILER_RT_ABI float __floatundisf(du_int a) { return __floatXiYf__(a); }
 
 #if defined(__ARM_EABI__)
 #if defined(COMPILER_RT_ARMHF_TARGET)
diff --git a/compiler-rt/lib/builtins/floatuntidf.c b/compiler-rt/lib/builtins/floatuntidf.c
index 4dfca8e493..9abeacc30c 100644
--- a/compiler-rt/lib/builtins/floatuntidf.c
+++ b/compiler-rt/lib/builtins/floatuntidf.c
@@ -14,6 +14,10 @@
 
 #ifdef CRT_HAS_128BIT
 
+#define SRC_U128
+#define DST_DOUBLE
+#include "int_to_fp_impl.inc"
+
 // Returns: convert a to a double, rounding toward even.
 
 // Assumption: double is a IEEE 64 bit floating point type
@@ -22,49 +26,6 @@
 // seee eeee eeee mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm
 // mmmm
 
-COMPILER_RT_ABI double __floatuntidf(tu_int a) {
-  if (a == 0)
-    return 0.0;
-  const unsigned N = sizeof(tu_int) * CHAR_BIT;
-  int sd = N - __clzti2(a); // number of significant digits
-  si_int e = sd - 1;        // exponent
-  if (sd > DBL_MANT_DIG) {
-    //  start:  0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx
-    //  finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR
-    //                                                12345678901234567890123456
-    //  1 = msb 1 bit
-    //  P = bit DBL_MANT_DIG-1 bits to the right of 1
-    //  Q = bit DBL_MANT_DIG bits to the right of 1
-    //  R = "or" of all bits to the right of Q
-    switch (sd) {
-    case DBL_MANT_DIG + 1:
-      a <<= 1;
-      break;
-    case DBL_MANT_DIG + 2:
-      break;
-    default:
-      a = (a >> (sd - (DBL_MANT_DIG + 2))) |
-          ((a & ((tu_int)(-1) >> ((N + DBL_MANT_DIG + 2) - sd))) != 0);
-    };
-    // finish:
-    a |= (a & 4) != 0; // Or P into R
-    ++a;               // round - this step may add a significant bit
-    a >>= 2;           // dump Q and R
-    // a is now rounded to DBL_MANT_DIG or DBL_MANT_DIG+1 bits
-    if (a & ((tu_int)1 << DBL_MANT_DIG)) {
-      a >>= 1;
-      ++e;
-    }
-    // a is now rounded to DBL_MANT_DIG bits
-  } else {
-    a <<= (DBL_MANT_DIG - sd);
-    // a is now rounded to DBL_MANT_DIG bits
-  }
-  double_bits fb;
-  fb.u.s.high = ((e + 1023) << 20) |              // exponent
-                ((su_int)(a >> 32) & 0x000FFFFF); // mantissa-high
-  fb.u.s.low = (su_int)a;                         // mantissa-low
-  return fb.f;
-}
+COMPILER_RT_ABI double __floatuntidf(tu_int a) { return __floatXiYf__(a); }
 
 #endif // CRT_HAS_128BIT
diff --git a/compiler-rt/lib/builtins/floatuntisf.c b/compiler-rt/lib/builtins/floatuntisf.c
index a53659cd1f..997c1569ac 100644
--- a/compiler-rt/lib/builtins/floatuntisf.c
+++ b/compiler-rt/lib/builtins/floatuntisf.c
@@ -14,6 +14,10 @@
 
 #ifdef CRT_HAS_128BIT
 
+#define SRC_U128
+#define DST_SINGLE
+#include "int_to_fp_impl.inc"
+
 // Returns: convert a to a float, rounding toward even.
 
 // Assumption: float is a IEEE 32 bit floating point type
@@ -21,48 +25,6 @@
 
 // seee eeee emmm mmmm mmmm mmmm mmmm mmmm
 
-COMPILER_RT_ABI float __floatuntisf(tu_int a) {
-  if (a == 0)
-    return 0.0F;
-  const unsigned N = sizeof(tu_int) * CHAR_BIT;
-  int sd = N - __clzti2(a); // number of significant digits
-  si_int e = sd - 1;        // exponent
-  if (sd > FLT_MANT_DIG) {
-    //  start:  0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx
-    //  finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR
-    //                                                12345678901234567890123456
-    //  1 = msb 1 bit
-    //  P = bit FLT_MANT_DIG-1 bits to the right of 1
-    //  Q = bit FLT_MANT_DIG bits to the right of 1
-    //  R = "or" of all bits to the right of Q
-    switch (sd) {
-    case FLT_MANT_DIG + 1:
-      a <<= 1;
-      break;
-    case FLT_MANT_DIG + 2:
-      break;
-    default:
-      a = (a >> (sd - (FLT_MANT_DIG + 2))) |
-          ((a & ((tu_int)(-1) >> ((N + FLT_MANT_DIG + 2) - sd))) != 0);
-    };
-    // finish:
-    a |= (a & 4) != 0; // Or P into R
-    ++a;               // round - this step may add a significant bit
-    a >>= 2;           // dump Q and R
-    // a is now rounded to FLT_MANT_DIG or FLT_MANT_DIG+1 bits
-    if (a & ((tu_int)1 << FLT_MANT_DIG)) {
-      a >>= 1;
-      ++e;
-    }
-    // a is now rounded to FLT_MANT_DIG bits
-  } else {
-    a <<= (FLT_MANT_DIG - sd);
-    // a is now rounded to FLT_MANT_DIG bits
-  }
-  float_bits fb;
-  fb.u = ((e + 127) << 23) |       // exponent
-         ((su_int)a & 0x007FFFFF); // mantissa
-  return fb.f;
-}
+COMPILER_RT_ABI float __floatuntisf(tu_int a) { return __floatXiYf__(a); }
 
 #endif // CRT_HAS_128BIT
diff --git a/compiler-rt/lib/builtins/floatuntitf.c b/compiler-rt/lib/builtins/floatuntitf.c
index 33a81b34ee..c9639989d3 100644
--- a/compiler-rt/lib/builtins/floatuntitf.c
+++ b/compiler-rt/lib/builtins/floatuntitf.c
@@ -16,6 +16,10 @@
 #include "fp_lib.h"
 #include "int_lib.h"
 
+#define SRC_U128
+#define DST_QUAD
+#include "int_to_fp_impl.inc"
+
 // Returns: convert a tu_int to a fp_t, rounding toward even.
 
 // Assumption: fp_t is a IEEE 128 bit floating point type
@@ -26,50 +30,6 @@
 // mmmm mmmm mmmm
 
 #if defined(CRT_HAS_TF_MODE)
-COMPILER_RT_ABI fp_t __floatuntitf(tu_int a) {
-  if (a == 0)
-    return 0.0;
-  const unsigned N = sizeof(tu_int) * CHAR_BIT;
-  int sd = N - __clzti2(a); // number of significant digits
-  int e = sd - 1;           // exponent
-  if (sd > TF_MANT_DIG) {
-    //  start:  0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx
-    //  finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR
-    //                                                12345678901234567890123456
-    //  1 = msb 1 bit
-    //  P = bit TF_MANT_DIG-1 bits to the right of 1
-    //  Q = bit TF_MANT_DIG bits to the right of 1
-    //  R = "or" of all bits to the right of Q
-    switch (sd) {
-    case TF_MANT_DIG + 1:
-      a <<= 1;
-      break;
-    case TF_MANT_DIG + 2:
-      break;
-    default:
-      a = (a >> (sd - (TF_MANT_DIG + 2))) |
-          ((a & ((tu_int)(-1) >> ((N + TF_MANT_DIG + 2) - sd))) != 0);
-    };
-    // finish:
-    a |= (a & 4) != 0; // Or P into R
-    ++a;               // round - this step may add a significant bit
-    a >>= 2;           // dump Q and R
-    // a is now rounded to TF_MANT_DIG or TF_MANT_DIG+1 bits
-    if (a & ((tu_int)1 << TF_MANT_DIG)) {
-      a >>= 1;
-      ++e;
-    }
-    // a is now rounded to TF_MANT_DIG bits
-  } else {
-    a <<= (TF_MANT_DIG - sd);
-    // a is now rounded to TF_MANT_DIG bits
-  }
-
-  long_double_bits fb;
-  fb.u.high.all = (du_int)(e + 16383) << 48             // exponent
-                  | ((a >> 64) & 0x0000ffffffffffffLL); // significand
-  fb.u.low.all = (du_int)(a);
-  return fb.f;
-}
+COMPILER_RT_ABI fp_t __floatuntitf(tu_int a) { return __floatXiYf__(a); }
 
 #endif
diff --git a/compiler-rt/lib/builtins/fp_extend.h b/compiler-rt/lib/builtins/fp_extend.h
index eee4722bf9..d640bdcb0e 100644
--- a/compiler-rt/lib/builtins/fp_extend.h
+++ b/compiler-rt/lib/builtins/fp_extend.h
@@ -20,15 +20,24 @@
 typedef float src_t;
 typedef uint32_t src_rep_t;
 #define SRC_REP_C UINT32_C
-static const int srcSigBits = 23;
+static const int srcBits = sizeof(src_t) * CHAR_BIT;
+static const int srcSigFracBits = 23;
+// -1 accounts for the sign bit.
+// srcBits - srcSigFracBits - 1
+static const int srcExpBits = 8;
 #define src_rep_t_clz clzsi
 
 #elif defined SRC_DOUBLE
 typedef double src_t;
 typedef uint64_t src_rep_t;
 #define SRC_REP_C UINT64_C
-static const int srcSigBits = 52;
-static __inline int src_rep_t_clz(src_rep_t a) {
+static const int srcBits = sizeof(src_t) * CHAR_BIT;
+static const int srcSigFracBits = 52;
+// -1 accounts for the sign bit.
+// srcBits - srcSigFracBits - 1
+static const int srcExpBits = 11;
+
+static inline int src_rep_t_clz_impl(src_rep_t a) {
 #if defined __LP64__
   return __builtin_clzl(a);
 #else
@@ -38,6 +47,19 @@ static __inline int src_rep_t_clz(src_rep_t a) {
     return 32 + clzsi(a & REP_C(0xffffffff));
 #endif
 }
+#define src_rep_t_clz src_rep_t_clz_impl
+
+#elif defined SRC_80
+typedef long double src_t;
+typedef __uint128_t src_rep_t;
+#define SRC_REP_C (__uint128_t)
+// sign bit, exponent and significand occupy the lower 80 bits.
+static const int srcBits = 80;
+static const int srcSigFracBits = 63;
+// -1 accounts for the sign bit.
+// -1 accounts for the explicitly stored integer bit.
+// srcBits - srcSigFracBits - 1 - 1
+static const int srcExpBits = 15;
 
 #elif defined SRC_HALF
 #ifdef COMPILER_RT_HAS_FLOAT16
@@ -47,7 +69,12 @@ typedef uint16_t src_t;
 #endif
 typedef uint16_t src_rep_t;
 #define SRC_REP_C UINT16_C
-static const int srcSigBits = 10;
+static const int srcBits = sizeof(src_t) * CHAR_BIT;
+static const int srcSigFracBits = 10;
+// -1 accounts for the sign bit.
+// srcBits - srcSigFracBits - 1
+static const int srcExpBits = 5;
+
 #define src_rep_t_clz __builtin_clz
 
 #else
@@ -58,28 +85,78 @@ static const int srcSigBits = 10;
 typedef float dst_t;
 typedef uint32_t dst_rep_t;
 #define DST_REP_C UINT32_C
-static const int dstSigBits = 23;
+static const int dstBits = sizeof(dst_t) * CHAR_BIT;
+static const int dstSigFracBits = 23;
+// -1 accounts for the sign bit.
+// dstBits - dstSigFracBits - 1
+static const int dstExpBits = 8;
 
 #elif defined DST_DOUBLE
 typedef double dst_t;
 typedef uint64_t dst_rep_t;
 #define DST_REP_C UINT64_C
-static const int dstSigBits = 52;
+static const int dstBits = sizeof(dst_t) * CHAR_BIT;
+static const int dstSigFracBits = 52;
+// -1 accounts for the sign bit.
+// dstBits - dstSigFracBits - 1
+static const int dstExpBits = 11;
 
 #elif defined DST_QUAD
+// TODO: use fp_lib.h once QUAD_PRECISION is available on x86_64.
+#if __LDBL_MANT_DIG__ == 113
 typedef long double dst_t;
+#elif defined(__x86_64__) &&                                                   \
+    (defined(__FLOAT128__) || defined(__SIZEOF_FLOAT128__))
+typedef __float128 dst_t;
+#endif
 typedef __uint128_t dst_rep_t;
 #define DST_REP_C (__uint128_t)
-static const int dstSigBits = 112;
+static const int dstBits = sizeof(dst_t) * CHAR_BIT;
+static const int dstSigFracBits = 112;
+// -1 accounts for the sign bit.
+// dstBits - dstSigFracBits - 1
+static const int dstExpBits = 15;
 
 #else
 #error Destination should be single, double, or quad precision!
 #endif // end destination precision
 
-// End of specialization parameters.  Two helper routines for conversion to and
-// from the representation of floating-point data as integer values follow.
+// End of specialization parameters.
+
+// TODO: These helper routines should be placed into fp_lib.h
+// Currently they depend on macros/constants defined above.
+
+static inline src_rep_t extract_sign_from_src(src_rep_t x) {
+  const src_rep_t srcSignMask = SRC_REP_C(1) << (srcBits - 1);
+  return (x & srcSignMask) >> (srcBits - 1);
+}
+
+static inline src_rep_t extract_exp_from_src(src_rep_t x) {
+  const int srcSigBits = srcBits - 1 - srcExpBits;
+  const src_rep_t srcExpMask = ((SRC_REP_C(1) << srcExpBits) - 1) << srcSigBits;
+  return (x & srcExpMask) >> srcSigBits;
+}
+
+static inline src_rep_t extract_sig_frac_from_src(src_rep_t x) {
+  const src_rep_t srcSigFracMask = (SRC_REP_C(1) << srcSigFracBits) - 1;
+  return x & srcSigFracMask;
+}
+
+#ifdef src_rep_t_clz
+static inline int clz_in_sig_frac(src_rep_t sigFrac) {
+      const int skip = (sizeof(dst_t) * CHAR_BIT - srcBits) + 1 + srcExpBits;
+      return src_rep_t_clz(sigFrac) - skip;
+}
+#endif
+
+static inline dst_rep_t construct_dst_rep(dst_rep_t sign, dst_rep_t exp, dst_rep_t sigFrac) {
+  return (sign << (dstBits - 1)) | (exp << (dstBits - 1 - dstExpBits)) | sigFrac;
+}
+
+// Two helper routines for conversion to and from the representation of
+// floating-point data as integer values follow.
 
-static __inline src_rep_t srcToRep(src_t x) {
+static inline src_rep_t srcToRep(src_t x) {
   const union {
     src_t f;
     src_rep_t i;
@@ -87,7 +164,7 @@ static __inline src_rep_t srcToRep(src_t x) {
   return rep.i;
 }
 
-static __inline dst_t dstFromRep(dst_rep_t x) {
+static inline dst_t dstFromRep(dst_rep_t x) {
   const union {
     dst_t f;
     dst_rep_t i;
diff --git a/compiler-rt/lib/builtins/fp_extend_impl.inc b/compiler-rt/lib/builtins/fp_extend_impl.inc
index d1c9c02a00..e16b55d150 100644
--- a/compiler-rt/lib/builtins/fp_extend_impl.inc
+++ b/compiler-rt/lib/builtins/fp_extend_impl.inc
@@ -37,71 +37,72 @@
 
 #include "fp_extend.h"
 
+// The source type may use a usual IEEE-754 interchange format or Intel 80-bit
+// format. In particular, for the source type srcSigFracBits may be not equal to
+// srcSigBits. The destination type is assumed to be one of IEEE-754 standard
+// types.
 static __inline dst_t __extendXfYf2__(src_t a) {
   // Various constants whose values follow from the type parameters.
   // Any reasonable optimizer will fold and propagate all of these.
-  const int srcBits = sizeof(src_t) * CHAR_BIT;
-  const int srcExpBits = srcBits - srcSigBits - 1;
   const int srcInfExp = (1 << srcExpBits) - 1;
   const int srcExpBias = srcInfExp >> 1;
 
-  const src_rep_t srcMinNormal = SRC_REP_C(1) << srcSigBits;
-  const src_rep_t srcInfinity = (src_rep_t)srcInfExp << srcSigBits;
-  const src_rep_t srcSignMask = SRC_REP_C(1) << (srcSigBits + srcExpBits);
-  const src_rep_t srcAbsMask = srcSignMask - 1;
-  const src_rep_t srcQNaN = SRC_REP_C(1) << (srcSigBits - 1);
-  const src_rep_t srcNaNCode = srcQNaN - 1;
-
-  const int dstBits = sizeof(dst_t) * CHAR_BIT;
-  const int dstExpBits = dstBits - dstSigBits - 1;
   const int dstInfExp = (1 << dstExpBits) - 1;
   const int dstExpBias = dstInfExp >> 1;
 
-  const dst_rep_t dstMinNormal = DST_REP_C(1) << dstSigBits;
-
   // Break a into a sign and representation of the absolute value.
   const src_rep_t aRep = srcToRep(a);
-  const src_rep_t aAbs = aRep & srcAbsMask;
-  const src_rep_t sign = aRep & srcSignMask;
-  dst_rep_t absResult;
+  const src_rep_t srcSign = extract_sign_from_src(aRep);
+  const src_rep_t srcExp = extract_exp_from_src(aRep);
+  const src_rep_t srcSigFrac = extract_sig_frac_from_src(aRep);
+
+  dst_rep_t dstSign = srcSign;
+  dst_rep_t dstExp;
+  dst_rep_t dstSigFrac;
 
-  // If sizeof(src_rep_t) < sizeof(int), the subtraction result is promoted
-  // to (signed) int.  To avoid that, explicitly cast to src_rep_t.
-  if ((src_rep_t)(aAbs - srcMinNormal) < srcInfinity - srcMinNormal) {
+  if (srcExp >= 1 && srcExp < srcInfExp) {
     // a is a normal number.
-    // Extend to the destination type by shifting the significand and
-    // exponent into the proper position and rebiasing the exponent.
-    absResult = (dst_rep_t)aAbs << (dstSigBits - srcSigBits);
-    absResult += (dst_rep_t)(dstExpBias - srcExpBias) << dstSigBits;
+    dstExp = (dst_rep_t)srcExp + (dst_rep_t)(dstExpBias - srcExpBias);
+    dstSigFrac = (dst_rep_t)srcSigFrac << (dstSigFracBits - srcSigFracBits);
   }
 
-  else if (aAbs >= srcInfinity) {
+  else if (srcExp == srcInfExp) {
     // a is NaN or infinity.
-    // Conjure the result by beginning with infinity, then setting the qNaN
-    // bit (if needed) and right-aligning the rest of the trailing NaN
-    // payload field.
-    absResult = (dst_rep_t)dstInfExp << dstSigBits;
-    absResult |= (dst_rep_t)(aAbs & srcQNaN) << (dstSigBits - srcSigBits);
-    absResult |= (dst_rep_t)(aAbs & srcNaNCode) << (dstSigBits - srcSigBits);
+    dstExp = dstInfExp;
+    dstSigFrac = (dst_rep_t)srcSigFrac << (dstSigFracBits - srcSigFracBits);
   }
 
-  else if (aAbs) {
+  else if (srcSigFrac) {
     // a is denormal.
-    // renormalize the significand and clear the leading bit, then insert
-    // the correct adjusted exponent in the destination type.
-    const int scale = src_rep_t_clz(aAbs) - src_rep_t_clz(srcMinNormal);
-    absResult = (dst_rep_t)aAbs << (dstSigBits - srcSigBits + scale);
-    absResult ^= dstMinNormal;
-    const int resultExponent = dstExpBias - srcExpBias - scale + 1;
-    absResult |= (dst_rep_t)resultExponent << dstSigBits;
+    if (srcExpBits == dstExpBits) {
+      // The exponent fields are identical and this is a denormal number, so all
+      // the non-significand bits are zero. In particular, this branch is always
+      // taken when we extend a denormal F80 to F128.
+      dstExp = 0;
+      dstSigFrac = ((dst_rep_t)srcSigFrac) << (dstSigFracBits - srcSigFracBits);
+    } else {
+#ifndef src_rep_t_clz
+      // If src_rep_t_clz is not defined this branch must be unreachable.
+      __builtin_unreachable();
+#else
+      // Renormalize the significand and clear the leading bit.
+      // For F80 -> F128 this codepath is unused.
+      const int scale = clz_in_sig_frac(srcSigFrac) + 1;
+      dstExp = dstExpBias - srcExpBias - scale + 1;
+      dstSigFrac = (dst_rep_t)srcSigFrac
+                   << (dstSigFracBits - srcSigFracBits + scale);
+      const dst_rep_t dstMinNormal = DST_REP_C(1) << (dstBits - 1 - dstExpBits);
+      dstSigFrac ^= dstMinNormal;
+#endif
+    }
   }
 
   else {
     // a is zero.
-    absResult = 0;
+    dstExp = 0;
+    dstSigFrac = 0;
   }
 
-  // Apply the signbit to the absolute value.
-  const dst_rep_t result = absResult | (dst_rep_t)sign << (dstBits - srcBits);
+  const dst_rep_t result = construct_dst_rep(dstSign, dstExp, dstSigFrac);
   return dstFromRep(result);
 }
diff --git a/compiler-rt/lib/builtins/fp_trunc.h b/compiler-rt/lib/builtins/fp_trunc.h
index 91f614528a..f62f8bafc7 100644
--- a/compiler-rt/lib/builtins/fp_trunc.h
+++ b/compiler-rt/lib/builtins/fp_trunc.h
@@ -19,19 +19,37 @@
 typedef float src_t;
 typedef uint32_t src_rep_t;
 #define SRC_REP_C UINT32_C
-static const int srcSigBits = 23;
+static const int srcBits = sizeof(src_t) * CHAR_BIT;
+static const int srcSigFracBits = 23;
+// -1 accounts for the sign bit.
+// srcBits - srcSigFracBits - 1
+static const int srcExpBits = 8;
 
 #elif defined SRC_DOUBLE
 typedef double src_t;
 typedef uint64_t src_rep_t;
 #define SRC_REP_C UINT64_C
-static const int srcSigBits = 52;
+static const int srcBits = sizeof(src_t) * CHAR_BIT;
+static const int srcSigFracBits = 52;
+// -1 accounts for the sign bit.
+// srcBits - srcSigFracBits - 1
+static const int srcExpBits = 11;
 
 #elif defined SRC_QUAD
+// TODO: use fp_lib.h once QUAD_PRECISION is available on x86_64.
+#if __LDBL_MANT_DIG__ == 113
 typedef long double src_t;
+#elif defined(__x86_64__) &&                                                   \
+    (defined(__FLOAT128__) || defined(__SIZEOF_FLOAT128__))
+typedef __float128 src_t;
+#endif
 typedef __uint128_t src_rep_t;
 #define SRC_REP_C (__uint128_t)
-static const int srcSigBits = 112;
+static const int srcBits = sizeof(src_t) * CHAR_BIT;
+static const int srcSigFracBits = 112;
+// -1 accounts for the sign bit.
+// srcBits - srcSigFracBits - 1
+static const int srcExpBits = 15;
 
 #else
 #error Source should be double precision or quad precision!
@@ -41,13 +59,32 @@ static const int srcSigBits = 112;
 typedef double dst_t;
 typedef uint64_t dst_rep_t;
 #define DST_REP_C UINT64_C
-static const int dstSigBits = 52;
+static const int dstBits = sizeof(dst_t) * CHAR_BIT;
+static const int dstSigFracBits = 52;
+// -1 accounts for the sign bit.
+// dstBits - dstSigFracBits - 1
+static const int dstExpBits = 11;
+
+#elif defined DST_80
+typedef long double dst_t;
+typedef __uint128_t dst_rep_t;
+#define DST_REP_C (__uint128_t)
+static const int dstBits = 80;
+static const int dstSigFracBits = 63;
+// -1 accounts for the sign bit.
+// -1 accounts for the explicitly stored integer bit.
+// dstBits - dstSigFracBits - 1 - 1
+static const int dstExpBits = 15;
 
 #elif defined DST_SINGLE
 typedef float dst_t;
 typedef uint32_t dst_rep_t;
 #define DST_REP_C UINT32_C
-static const int dstSigBits = 23;
+static const int dstBits = sizeof(dst_t) * CHAR_BIT;
+static const int dstSigFracBits = 23;
+// -1 accounts for the sign bit.
+// dstBits - dstSigFracBits - 1
+static const int dstExpBits = 8;
 
 #elif defined DST_HALF
 #ifdef COMPILER_RT_HAS_FLOAT16
@@ -57,22 +94,58 @@ typedef uint16_t dst_t;
 #endif
 typedef uint16_t dst_rep_t;
 #define DST_REP_C UINT16_C
-static const int dstSigBits = 10;
+static const int dstBits = sizeof(dst_t) * CHAR_BIT;
+static const int dstSigFracBits = 10;
+// -1 accounts for the sign bit.
+// dstBits - dstSigFracBits - 1
+static const int dstExpBits = 5;
 
 #elif defined DST_BFLOAT
 typedef __bf16 dst_t;
 typedef uint16_t dst_rep_t;
 #define DST_REP_C UINT16_C
-static const int dstSigBits = 7;
+static const int dstBits = sizeof(dst_t) * CHAR_BIT;
+static const int dstSigFracBits = 7;
+// -1 accounts for the sign bit.
+// dstBits - dstSigFracBits - 1
+static const int dstExpBits = 8;
 
 #else
 #error Destination should be single precision or double precision!
 #endif // end destination precision
 
+// TODO: These helper routines should be placed into fp_lib.h
+// Currently they depend on macros/constants defined above.
+
+static inline src_rep_t extract_sign_from_src(src_rep_t x) {
+  const src_rep_t srcSignMask = SRC_REP_C(1) << (srcBits - 1);
+  return (x & srcSignMask) >> (srcBits - 1);
+}
+
+static inline src_rep_t extract_exp_from_src(src_rep_t x) {
+  const int srcSigBits = srcBits - 1 - srcExpBits;
+  const src_rep_t srcExpMask = ((SRC_REP_C(1) << srcExpBits) - 1) << srcSigBits;
+  return (x & srcExpMask) >> srcSigBits;
+}
+
+static inline src_rep_t extract_sig_frac_from_src(src_rep_t x) {
+  const src_rep_t srcSigFracMask = (SRC_REP_C(1) << srcSigFracBits) - 1;
+  return x & srcSigFracMask;
+}
+
+static inline dst_rep_t construct_dst_rep(dst_rep_t sign, dst_rep_t exp, dst_rep_t sigFrac) {
+  dst_rep_t result = (sign << (dstBits - 1)) | (exp << (dstBits - 1 - dstExpBits)) | sigFrac;
+  // Set the explicit integer bit in F80 if present.
+  if (dstBits == 80 && exp) {
+    result |= (DST_REP_C(1) << dstSigFracBits);
+  }
+  return result;
+}
+
 // End of specialization parameters.  Two helper routines for conversion to and
 // from the representation of floating-point data as integer values follow.
 
-static __inline src_rep_t srcToRep(src_t x) {
+static inline src_rep_t srcToRep(src_t x) {
   const union {
     src_t f;
     src_rep_t i;
@@ -80,7 +153,7 @@ static __inline src_rep_t srcToRep(src_t x) {
   return rep.i;
 }
 
-static __inline dst_t dstFromRep(dst_rep_t x) {
+static inline dst_t dstFromRep(dst_rep_t x) {
   const union {
     dst_t f;
     dst_rep_t i;
diff --git a/compiler-rt/lib/builtins/fp_trunc_impl.inc b/compiler-rt/lib/builtins/fp_trunc_impl.inc
index e235f45965..f684924956 100644
--- a/compiler-rt/lib/builtins/fp_trunc_impl.inc
+++ b/compiler-rt/lib/builtins/fp_trunc_impl.inc
@@ -38,102 +38,118 @@
 
 #include "fp_trunc.h"
 
+// The destination type may use a usual IEEE-754 interchange format or Intel
+// 80-bit format. In particular, for the destination type dstSigFracBits may be
+// not equal to dstSigBits. The source type is assumed to be one of IEEE-754
+// standard types.
 static __inline dst_t __truncXfYf2__(src_t a) {
   // Various constants whose values follow from the type parameters.
   // Any reasonable optimizer will fold and propagate all of these.
-  const int srcBits = sizeof(src_t) * CHAR_BIT;
-  const int srcExpBits = srcBits - srcSigBits - 1;
   const int srcInfExp = (1 << srcExpBits) - 1;
   const int srcExpBias = srcInfExp >> 1;
 
-  const src_rep_t srcMinNormal = SRC_REP_C(1) << srcSigBits;
-  const src_rep_t srcSignificandMask = srcMinNormal - 1;
-  const src_rep_t srcInfinity = (src_rep_t)srcInfExp << srcSigBits;
-  const src_rep_t srcSignMask = SRC_REP_C(1) << (srcSigBits + srcExpBits);
-  const src_rep_t srcAbsMask = srcSignMask - 1;
-  const src_rep_t roundMask = (SRC_REP_C(1) << (srcSigBits - dstSigBits)) - 1;
-  const src_rep_t halfway = SRC_REP_C(1) << (srcSigBits - dstSigBits - 1);
-  const src_rep_t srcQNaN = SRC_REP_C(1) << (srcSigBits - 1);
+  const src_rep_t srcMinNormal = SRC_REP_C(1) << srcSigFracBits;
+  const src_rep_t roundMask =
+      (SRC_REP_C(1) << (srcSigFracBits - dstSigFracBits)) - 1;
+  const src_rep_t halfway = SRC_REP_C(1)
+                            << (srcSigFracBits - dstSigFracBits - 1);
+  const src_rep_t srcQNaN = SRC_REP_C(1) << (srcSigFracBits - 1);
   const src_rep_t srcNaNCode = srcQNaN - 1;
 
-  const int dstBits = sizeof(dst_t) * CHAR_BIT;
-  const int dstExpBits = dstBits - dstSigBits - 1;
   const int dstInfExp = (1 << dstExpBits) - 1;
   const int dstExpBias = dstInfExp >> 1;
-
-  const int underflowExponent = srcExpBias + 1 - dstExpBias;
   const int overflowExponent = srcExpBias + dstInfExp - dstExpBias;
-  const src_rep_t underflow = (src_rep_t)underflowExponent << srcSigBits;
-  const src_rep_t overflow = (src_rep_t)overflowExponent << srcSigBits;
 
-  const dst_rep_t dstQNaN = DST_REP_C(1) << (dstSigBits - 1);
+  const dst_rep_t dstQNaN = DST_REP_C(1) << (dstSigFracBits - 1);
   const dst_rep_t dstNaNCode = dstQNaN - 1;
 
-  // Break a into a sign and representation of the absolute value.
   const src_rep_t aRep = srcToRep(a);
-  const src_rep_t aAbs = aRep & srcAbsMask;
-  const src_rep_t sign = aRep & srcSignMask;
-  dst_rep_t absResult;
+  const src_rep_t srcSign = extract_sign_from_src(aRep);
+  const src_rep_t srcExp = extract_exp_from_src(aRep);
+  const src_rep_t srcSigFrac = extract_sig_frac_from_src(aRep);
+
+  dst_rep_t dstSign = srcSign;
+  dst_rep_t dstExp;
+  dst_rep_t dstSigFrac;
 
-  const int tailBits = srcBits - dstBits;
-  if (srcExpBits == dstExpBits && ((aRep >> tailBits) << tailBits) == aRep) {
-    // Same size exponents and a's significand tail is 0. Remove tail.
-    dst_rep_t result = aRep >> tailBits;
-    return dstFromRep(result);
+  // Same size exponents and a's significand tail is 0.
+  // The significand can be truncated and the exponent can be copied over.
+  const int sigFracTailBits = srcSigFracBits - dstSigFracBits;
+  if (srcExpBits == dstExpBits &&
+      ((aRep >> sigFracTailBits) << sigFracTailBits) == aRep) {
+    dstExp = srcExp;
+    dstSigFrac = (dst_rep_t)(srcSigFrac >> sigFracTailBits);
+    return dstFromRep(construct_dst_rep(dstSign, dstExp, dstSigFrac));
   }
 
-  if (aAbs - underflow < aAbs - overflow) {
+  const int dstExpCandidate = ((int)srcExp - srcExpBias) + dstExpBias;
+  if (dstExpCandidate >= 1 && dstExpCandidate < dstInfExp) {
     // The exponent of a is within the range of normal numbers in the
-    // destination format.  We can convert by simply right-shifting with
+    // destination format. We can convert by simply right-shifting with
     // rounding and adjusting the exponent.
-    absResult = aAbs >> (srcSigBits - dstSigBits);
-    absResult -= (dst_rep_t)(srcExpBias - dstExpBias) << dstSigBits;
+    dstExp = dstExpCandidate;
+    dstSigFrac = (dst_rep_t)(srcSigFrac >> sigFracTailBits);
 
-    const src_rep_t roundBits = aAbs & roundMask;
+    const src_rep_t roundBits = srcSigFrac & roundMask;
     // Round to nearest.
     if (roundBits > halfway)
-      absResult++;
+      dstSigFrac++;
     // Tie to even.
     else if (roundBits == halfway)
-      absResult += absResult & 1;
-  } else if (aAbs > srcInfinity) {
+      dstSigFrac += dstSigFrac & 1;
+
+    // Rounding has changed the exponent.
+    if (dstSigFrac >= (DST_REP_C(1) << dstSigFracBits)) {
+      dstExp += 1;
+      dstSigFrac ^= (DST_REP_C(1) << dstSigFracBits);
+    }
+  } else if (srcExp == srcInfExp && srcSigFrac) {
     // a is NaN.
     // Conjure the result by beginning with infinity, setting the qNaN
     // bit and inserting the (truncated) trailing NaN field.
-    absResult = (dst_rep_t)dstInfExp << dstSigBits;
-    absResult |= dstQNaN;
-    absResult |=
-        ((aAbs & srcNaNCode) >> (srcSigBits - dstSigBits)) & dstNaNCode;
-  } else if (aAbs >= overflow) {
-    // a overflows to infinity.
-    absResult = (dst_rep_t)dstInfExp << dstSigBits;
+    dstExp = dstInfExp;
+    dstSigFrac = dstQNaN;
+    dstSigFrac |= ((srcSigFrac & srcNaNCode) >> sigFracTailBits) & dstNaNCode;
+  } else if ((int)srcExp >= overflowExponent) {
+    dstExp = dstInfExp;
+    dstSigFrac = 0;
   } else {
     // a underflows on conversion to the destination type or is an exact
     // zero.  The result may be a denormal or zero.  Extract the exponent
     // to get the shift amount for the denormalization.
-    const int aExp = aAbs >> srcSigBits;
-    const int shift = srcExpBias - dstExpBias - aExp + 1;
+    src_rep_t significand = srcSigFrac;
+    int shift = srcExpBias - dstExpBias - srcExp;
 
-    const src_rep_t significand = (aRep & srcSignificandMask) | srcMinNormal;
+    if (srcExp) {
+      // Set the implicit integer bit if the source is a normal number.
+      significand |= srcMinNormal;
+      shift += 1;
+    }
 
     // Right shift by the denormalization amount with sticky.
-    if (shift > srcSigBits) {
-      absResult = 0;
+    if (shift > srcSigFracBits) {
+      dstExp = 0;
+      dstSigFrac = 0;
     } else {
-      const bool sticky = (significand << (srcBits - shift)) != 0;
+      dstExp = 0;
+      const bool sticky = shift && ((significand << (srcBits - shift)) != 0);
       src_rep_t denormalizedSignificand = significand >> shift | sticky;
-      absResult = denormalizedSignificand >> (srcSigBits - dstSigBits);
+      dstSigFrac = denormalizedSignificand >> sigFracTailBits;
       const src_rep_t roundBits = denormalizedSignificand & roundMask;
       // Round to nearest
       if (roundBits > halfway)
-        absResult++;
+        dstSigFrac++;
       // Ties to even
       else if (roundBits == halfway)
-        absResult += absResult & 1;
+        dstSigFrac += dstSigFrac & 1;
+
+      // Rounding has changed the exponent.
+      if (dstSigFrac >= (DST_REP_C(1) << dstSigFracBits)) {
+        dstExp += 1;
+        dstSigFrac ^= (DST_REP_C(1) << dstSigFracBits);
+      }
     }
   }
 
-  // Apply the signbit to the absolute value.
-  const dst_rep_t result = absResult | sign >> (srcBits - dstBits);
-  return dstFromRep(result);
+  return dstFromRep(construct_dst_rep(dstSign, dstExp, dstSigFrac));
 }
diff --git a/compiler-rt/lib/builtins/int_to_fp.h b/compiler-rt/lib/builtins/int_to_fp.h
new file mode 100644
index 0000000000..9e56a14696
--- /dev/null
+++ b/compiler-rt/lib/builtins/int_to_fp.h
@@ -0,0 +1,82 @@
+//===-- int_to_fp.h - integer to floating point conversion ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Set source and destination defines in order to use a correctly
+// parameterised floatXiYf implementation.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef INT_TO_FP_H
+#define INT_TO_FP_H
+
+#include "int_lib.h"
+
+#if defined SRC_I64
+typedef int64_t src_t;
+typedef uint64_t usrc_t;
+static __inline int clzSrcT(usrc_t x) { return __builtin_clzll(x); }
+
+#elif defined SRC_U64
+typedef uint64_t src_t;
+typedef uint64_t usrc_t;
+static __inline int clzSrcT(usrc_t x) { return __builtin_clzll(x); }
+
+#elif defined SRC_I128
+typedef __int128_t src_t;
+typedef __uint128_t usrc_t;
+static __inline int clzSrcT(usrc_t x) { return __clzti2(x); }
+
+#elif defined SRC_U128
+typedef __uint128_t src_t;
+typedef __uint128_t usrc_t;
+static __inline int clzSrcT(usrc_t x) { return __clzti2(x); }
+
+#else
+#error Source should be a handled integer type.
+#endif
+
+#if defined DST_SINGLE
+typedef float dst_t;
+typedef uint32_t dst_rep_t;
+#define DST_REP_C UINT32_C
+
+enum {
+  dstSigBits = 23,
+};
+
+#elif defined DST_DOUBLE
+typedef double dst_t;
+typedef uint64_t dst_rep_t;
+#define DST_REP_C UINT64_C
+
+enum {
+  dstSigBits = 52,
+};
+
+#elif defined DST_QUAD
+typedef long double dst_t;
+typedef __uint128_t dst_rep_t;
+#define DST_REP_C (__uint128_t)
+
+enum {
+  dstSigBits = 112,
+};
+
+#else
+#error Destination should be a handled floating point type
+#endif
+
+static __inline dst_t dstFromRep(dst_rep_t x) {
+  const union {
+    dst_t f;
+    dst_rep_t i;
+  } rep = {.i = x};
+  return rep.f;
+}
+
+#endif // INT_TO_FP_H
diff --git a/compiler-rt/lib/builtins/int_to_fp_impl.inc b/compiler-rt/lib/builtins/int_to_fp_impl.inc
new file mode 100644
index 0000000000..51f76fd76d
--- /dev/null
+++ b/compiler-rt/lib/builtins/int_to_fp_impl.inc
@@ -0,0 +1,72 @@
+//===-- int_to_fp_impl.inc - integer to floating point conversion ---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Thsi file implements a generic conversion from an integer type to an
+// IEEE-754 floating point type, allowing a common implementation to be hsared
+// without copy and paste.
+//
+//===----------------------------------------------------------------------===//
+
+#include "int_to_fp.h"
+
+static __inline dst_t __floatXiYf__(src_t a) {
+  if (a == 0)
+    return 0.0;
+
+  enum {
+    dstMantDig = dstSigBits + 1,
+    srcBits = sizeof(src_t) * CHAR_BIT,
+    srcIsSigned = ((src_t)-1) < 0,
+  };
+
+  const src_t s = srcIsSigned ? a >> (srcBits - 1) : 0;
+
+  a = (usrc_t)(a ^ s) - s;
+  int sd = srcBits - clzSrcT(a);         // number of significant digits
+  int e = sd - 1;                        // exponent
+  if (sd > dstMantDig) {
+    //  start:  0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx
+    //  finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR
+    //                                                12345678901234567890123456
+    //  1 = msb 1 bit
+    //  P = bit dstMantDig-1 bits to the right of 1
+    //  Q = bit dstMantDig bits to the right of 1
+    //  R = "or" of all bits to the right of Q
+    if (sd == dstMantDig + 1) {
+      a <<= 1;
+    } else if (sd == dstMantDig + 2) {
+      // Do nothing.
+    } else {
+      a = ((usrc_t)a >> (sd - (dstMantDig + 2))) |
+          ((a & ((usrc_t)(-1) >> ((srcBits + dstMantDig + 2) - sd))) != 0);
+    }
+    // finish:
+    a |= (a & 4) != 0; // Or P into R
+    ++a;               // round - this step may add a significant bit
+    a >>= 2;           // dump Q and R
+    // a is now rounded to dstMantDig or dstMantDig+1 bits
+    if (a & ((usrc_t)1 << dstMantDig)) {
+      a >>= 1;
+      ++e;
+    }
+    // a is now rounded to dstMantDig bits
+  } else {
+    a <<= (dstMantDig - sd);
+    // a is now rounded to dstMantDig bits
+  }
+  const int dstBits = sizeof(dst_t) * CHAR_BIT;
+  const dst_rep_t dstSignMask = DST_REP_C(1) << (dstBits - 1);
+  const int dstExpBits = dstBits - dstSigBits - 1;
+  const int dstExpBias = (1 << (dstExpBits - 1)) - 1;
+  const dst_rep_t dstSignificandMask = (DST_REP_C(1) << dstSigBits) - 1;
+  // Combine sign, exponent, and mantissa.
+  const dst_rep_t result = ((dst_rep_t)s & dstSignMask) |
+                           ((dst_rep_t)(e + dstExpBias) << dstSigBits) |
+                           ((dst_rep_t)(a) & dstSignificandMask);
+  return dstFromRep(result);
+}
diff --git a/compiler-rt/lib/builtins/trunctfxf2.c b/compiler-rt/lib/builtins/trunctfxf2.c
new file mode 100644
index 0000000000..4a22a602b3
--- /dev/null
+++ b/compiler-rt/lib/builtins/trunctfxf2.c
@@ -0,0 +1,24 @@
+//===-- lib/trunctfsf2.c - long double -> quad conversion ---------*- C -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Assumption: long double is a IEEE 80 bit floating point type padded to 128
+// bits.
+
+// TODO: use fp_lib.h once QUAD_PRECISION is available on x86_64.
+#if __LDBL_MANT_DIG__ == 64 && defined(__x86_64__) &&                          \
+    (defined(__FLOAT128__) || defined(__SIZEOF_FLOAT128__))
+
+#define SRC_QUAD
+#define DST_80
+#include "fp_trunc_impl.inc"
+
+COMPILER_RT_ABI long double __trunctfxf2(__float128 a) {
+  return __truncXfYf2__(a);
+}
+
+#endif
diff --git a/compiler-rt/lib/fuzzer/FuzzerLoop.cpp b/compiler-rt/lib/fuzzer/FuzzerLoop.cpp
index 8b430c5428..935dd2342e 100644
--- a/compiler-rt/lib/fuzzer/FuzzerLoop.cpp
+++ b/compiler-rt/lib/fuzzer/FuzzerLoop.cpp
@@ -799,7 +799,7 @@ void Fuzzer::ReadAndExecuteSeedCorpora(std::vector<SizedFile> &CorporaFiles) {
     TotalSize += File.Size;
   }
   if (Options.MaxLen == 0)
-    SetMaxInputLen(std::min(std::max(kMinDefaultLen, MaxSize), kMaxSaneLen));
+    SetMaxInputLen(std::clamp(MaxSize, kMinDefaultLen, kMaxSaneLen));
   assert(MaxInputLen > 0);
 
   // Test the callback with empty input and never try it again.
diff --git a/compiler-rt/lib/hwasan/hwasan_platform_interceptors.h b/compiler-rt/lib/hwasan/hwasan_platform_interceptors.h
index 390c9d80c3..86d26b5ac1 100644
--- a/compiler-rt/lib/hwasan/hwasan_platform_interceptors.h
+++ b/compiler-rt/lib/hwasan/hwasan_platform_interceptors.h
@@ -68,8 +68,8 @@
 // #undef SANITIZER_INTERCEPT_MEMCMP
 // #define SANITIZER_INTERCEPT_MEMCMP 0
 
-#undef SANITIZER_INTERCEPT_BCMP
-#define SANITIZER_INTERCEPT_BCMP 0
+// #undef SANITIZER_INTERCEPT_BCMP
+// #define SANITIZER_INTERCEPT_BCMP 0
 
 #undef SANITIZER_INTERCEPT_STRNDUP
 #define SANITIZER_INTERCEPT_STRNDUP 0
diff --git a/compiler-rt/lib/interception/interception_win.cpp b/compiler-rt/lib/interception/interception_win.cpp
index d57afa3fda..1b681ada37 100644
--- a/compiler-rt/lib/interception/interception_win.cpp
+++ b/compiler-rt/lib/interception/interception_win.cpp
@@ -624,7 +624,7 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
                       //   mov rax, QWORD PTR [rip + XXXXXXXX]
     case 0x25ff48:    // 48 ff 25 XX XX XX XX :
                       //   rex.W jmp QWORD PTR [rip + XXXXXXXX]
-
+    case 0x158D4C:    // 4c 8d 15 XX XX XX XX : lea r10, [rip + XX]
       // Instructions having offset relative to 'rip' need offset adjustment.
       if (rel_offset)
         *rel_offset = 3;
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform.h b/compiler-rt/lib/sanitizer_common/sanitizer_platform.h
index c1ca5c9ca4..df446d0659 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_platform.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform.h
@@ -303,7 +303,15 @@
 #    define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 40)
 #  endif
 #elif SANITIZER_RISCV64
-#  define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 38)
+// FIXME: Rather than hardcoding the VMA here, we should rely on
+// GetMaxUserVirtualAddress(). This will require some refactoring though since
+// many places either hardcode some value or SANITIZER_MMAP_RANGE_SIZE is
+// assumed to be some constant integer.
+#  if SANITIZER_FUCHSIA
+#    define SANITIZER_MMAP_RANGE_SIZE (1ULL << 38)
+#  else
+#    define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47)
+#  endif
 #elif defined(__aarch64__)
 #  if SANITIZER_APPLE
 #    if SANITIZER_OSX || SANITIZER_IOSSIM
diff --git a/compiler-rt/lib/scudo/standalone/CMakeLists.txt b/compiler-rt/lib/scudo/standalone/CMakeLists.txt
index c4d3ea1e4f..60092005cc 100644
--- a/compiler-rt/lib/scudo/standalone/CMakeLists.txt
+++ b/compiler-rt/lib/scudo/standalone/CMakeLists.txt
@@ -62,6 +62,9 @@ set(SCUDO_HEADERS
   bytemap.h
   checksum.h
   chunk.h
+  condition_variable.h
+  condition_variable_base.h
+  condition_variable_linux.h
   combined.h
   common.h
   flags_parser.h
@@ -84,6 +87,7 @@ set(SCUDO_HEADERS
   quarantine.h
   release.h
   report.h
+  report_linux.h
   secondary.h
   size_class_map.h
   stack_depot.h
@@ -103,6 +107,7 @@ set(SCUDO_HEADERS
 set(SCUDO_SOURCES
   checksum.cpp
   common.cpp
+  condition_variable_linux.cpp
   crc32_hw.cpp
   flags_parser.cpp
   flags.cpp
@@ -113,6 +118,7 @@ set(SCUDO_SOURCES
   mem_map_linux.cpp
   release.cpp
   report.cpp
+  report_linux.cpp
   string_utils.cpp
   timing.cpp
   )
diff --git a/compiler-rt/lib/scudo/standalone/allocator_config.h b/compiler-rt/lib/scudo/standalone/allocator_config.h
index 44c1ac5f74..3c6aa3acb0 100644
--- a/compiler-rt/lib/scudo/standalone/allocator_config.h
+++ b/compiler-rt/lib/scudo/standalone/allocator_config.h
@@ -11,6 +11,7 @@
 
 #include "combined.h"
 #include "common.h"
+#include "condition_variable.h"
 #include "flags.h"
 #include "primary32.h"
 #include "primary64.h"
@@ -82,6 +83,14 @@ namespace scudo {
 //     // Defines the minimal & maximal release interval that can be set.
 //     static const s32 MinReleaseToOsIntervalMs = INT32_MIN;
 //     static const s32 MaxReleaseToOsIntervalMs = INT32_MAX;
+//
+//     // Use condition variable to shorten the waiting time of refillment of
+//     // freelist. Note that this depends on the implementation of condition
+//     // variable on each platform and the performance may vary so that it
+//     // doesn't guarantee a performance benefit.
+//     // Note that both variables have to be defined to enable it.
+//     static const bool UseConditionVariable = true;
+//     using ConditionVariableT = ConditionVariableLinux;
 //   };
 //   // Defines the type of Primary allocator to use.
 //   template <typename Config> using PrimaryT = SizeClassAllocator64<Config>;
diff --git a/compiler-rt/lib/scudo/standalone/common.cpp b/compiler-rt/lib/scudo/standalone/common.cpp
index 666f95400c..06e930638f 100644
--- a/compiler-rt/lib/scudo/standalone/common.cpp
+++ b/compiler-rt/lib/scudo/standalone/common.cpp
@@ -21,18 +21,4 @@ uptr getPageSizeSlow() {
   return PageSizeCached;
 }
 
-// Fatal internal map() or unmap() error (potentially OOM related).
-void NORETURN dieOnMapUnmapError(uptr SizeIfOOM) {
-  char Error[128] = "Scudo ERROR: internal map or unmap failure\n";
-  if (SizeIfOOM) {
-    formatString(
-        Error, sizeof(Error),
-        "Scudo ERROR: internal map failure (NO MEMORY) requesting %zuKB\n",
-        SizeIfOOM >> 10);
-  }
-  outputRaw(Error);
-  setAbortMessage(Error);
-  die();
-}
-
 } // namespace scudo
diff --git a/compiler-rt/lib/scudo/standalone/common.h b/compiler-rt/lib/scudo/standalone/common.h
index d0f429cfcb..3581c946d1 100644
--- a/compiler-rt/lib/scudo/standalone/common.h
+++ b/compiler-rt/lib/scudo/standalone/common.h
@@ -175,10 +175,6 @@ void setMemoryPermission(uptr Addr, uptr Size, uptr Flags,
 void releasePagesToOS(uptr BaseAddress, uptr Offset, uptr Size,
                       MapPlatformData *Data = nullptr);
 
-// Internal map & unmap fatal error. This must not call map(). SizeIfOOM shall
-// hold the requested size on an out-of-memory error, 0 otherwise.
-void NORETURN dieOnMapUnmapError(uptr SizeIfOOM = 0);
-
 // Logging related functions.
 
 void setAbortMessage(const char *Message);
diff --git a/compiler-rt/lib/scudo/standalone/condition_variable.h b/compiler-rt/lib/scudo/standalone/condition_variable.h
new file mode 100644
index 0000000000..549f6e9f78
--- /dev/null
+++ b/compiler-rt/lib/scudo/standalone/condition_variable.h
@@ -0,0 +1,60 @@
+//===-- condition_variable.h ------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SCUDO_CONDITION_VARIABLE_H_
+#define SCUDO_CONDITION_VARIABLE_H_
+
+#include "condition_variable_base.h"
+
+#include "common.h"
+#include "platform.h"
+
+#include "condition_variable_linux.h"
+
+namespace scudo {
+
+// A default implementation of default condition variable. It doesn't do a real
+// `wait`, instead it spins a short amount of time only.
+class ConditionVariableDummy
+    : public ConditionVariableBase<ConditionVariableDummy> {
+public:
+  void notifyAllImpl(UNUSED HybridMutex &M) REQUIRES(M) {}
+
+  void waitImpl(UNUSED HybridMutex &M) REQUIRES(M) {
+    M.unlock();
+
+    constexpr u32 SpinTimes = 64;
+    volatile u32 V = 0;
+    for (u32 I = 0; I < SpinTimes; ++I) {
+      u32 Tmp = V + 1;
+      V = Tmp;
+    }
+
+    M.lock();
+  }
+};
+
+template <typename Config, typename = const bool>
+struct ConditionVariableState {
+  static constexpr bool enabled() { return false; }
+  // This is only used for compilation purpose so that we won't end up having
+  // many conditional compilations. If you want to use `ConditionVariableDummy`,
+  // define `ConditionVariableT` in your allocator configuration. See
+  // allocator_config.h for more details.
+  using ConditionVariableT = ConditionVariableDummy;
+};
+
+template <typename Config>
+struct ConditionVariableState<Config, decltype(Config::UseConditionVariable)> {
+  static constexpr bool enabled() { return true; }
+  using ConditionVariableT = typename Config::ConditionVariableT;
+};
+
+} // namespace scudo
+
+#endif // SCUDO_CONDITION_VARIABLE_H_
diff --git a/compiler-rt/lib/scudo/standalone/condition_variable_base.h b/compiler-rt/lib/scudo/standalone/condition_variable_base.h
new file mode 100644
index 0000000000..416c327fed
--- /dev/null
+++ b/compiler-rt/lib/scudo/standalone/condition_variable_base.h
@@ -0,0 +1,56 @@
+//===-- condition_variable_base.h -------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SCUDO_CONDITION_VARIABLE_BASE_H_
+#define SCUDO_CONDITION_VARIABLE_BASE_H_
+
+#include "mutex.h"
+#include "thread_annotations.h"
+
+namespace scudo {
+
+template <typename Derived> class ConditionVariableBase {
+public:
+  constexpr ConditionVariableBase() = default;
+
+  void bindTestOnly(HybridMutex &Mutex) {
+#if SCUDO_DEBUG
+    boundMutex = &Mutex;
+#else
+    (void)Mutex;
+#endif
+  }
+
+  void notifyAll(HybridMutex &M) REQUIRES(M) {
+#if SCUDO_DEBUG
+    CHECK_EQ(&M, boundMutex);
+#endif
+    getDerived()->notifyAllImpl(M);
+  }
+
+  void wait(HybridMutex &M) REQUIRES(M) {
+#if SCUDO_DEBUG
+    CHECK_EQ(&M, boundMutex);
+#endif
+    getDerived()->waitImpl(M);
+  }
+
+protected:
+  Derived *getDerived() { return static_cast<Derived *>(this); }
+
+#if SCUDO_DEBUG
+  // Because thread-safety analysis doesn't support pointer aliasing, we are not
+  // able to mark the proper annotations without false positive. Instead, we
+  // pass the lock and do the same-lock check separately.
+  HybridMutex *boundMutex = nullptr;
+#endif
+};
+
+} // namespace scudo
+
+#endif // SCUDO_CONDITION_VARIABLE_BASE_H_
diff --git a/compiler-rt/lib/scudo/standalone/condition_variable_linux.cpp b/compiler-rt/lib/scudo/standalone/condition_variable_linux.cpp
new file mode 100644
index 0000000000..e6d9bd1771
--- /dev/null
+++ b/compiler-rt/lib/scudo/standalone/condition_variable_linux.cpp
@@ -0,0 +1,52 @@
+//===-- condition_variable_linux.cpp ----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "platform.h"
+
+#if SCUDO_LINUX
+
+#include "condition_variable_linux.h"
+
+#include "atomic_helpers.h"
+
+#include <limits.h>
+#include <linux/futex.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+
+namespace scudo {
+
+void ConditionVariableLinux::notifyAllImpl(UNUSED HybridMutex &M) {
+  const u32 V = atomic_load_relaxed(&Counter);
+  atomic_store_relaxed(&Counter, V + 1);
+
+  // TODO(chiahungduan): Move the waiters from the futex waiting queue
+  // `Counter` to futex waiting queue `M` so that the awoken threads won't be
+  // blocked again due to locked `M` by current thread.
+  if (LastNotifyAll != V) {
+    syscall(SYS_futex, reinterpret_cast<uptr>(&Counter), FUTEX_WAKE_PRIVATE,
+            INT_MAX, nullptr, nullptr, 0);
+  }
+
+  LastNotifyAll = V + 1;
+}
+
+void ConditionVariableLinux::waitImpl(HybridMutex &M) {
+  const u32 V = atomic_load_relaxed(&Counter) + 1;
+  atomic_store_relaxed(&Counter, V);
+
+  // TODO: Use ScopedUnlock when it's supported.
+  M.unlock();
+  syscall(SYS_futex, reinterpret_cast<uptr>(&Counter), FUTEX_WAIT_PRIVATE, V,
+          nullptr, nullptr, 0);
+  M.lock();
+}
+
+} // namespace scudo
+
+#endif // SCUDO_LINUX
diff --git a/compiler-rt/lib/scudo/standalone/condition_variable_linux.h b/compiler-rt/lib/scudo/standalone/condition_variable_linux.h
new file mode 100644
index 0000000000..cd07328732
--- /dev/null
+++ b/compiler-rt/lib/scudo/standalone/condition_variable_linux.h
@@ -0,0 +1,38 @@
+//===-- condition_variable_linux.h ------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SCUDO_CONDITION_VARIABLE_LINUX_H_
+#define SCUDO_CONDITION_VARIABLE_LINUX_H_
+
+#include "platform.h"
+
+#if SCUDO_LINUX
+
+#include "atomic_helpers.h"
+#include "condition_variable_base.h"
+#include "thread_annotations.h"
+
+namespace scudo {
+
+class ConditionVariableLinux
+    : public ConditionVariableBase<ConditionVariableLinux> {
+public:
+  void notifyAllImpl(HybridMutex &M) REQUIRES(M);
+
+  void waitImpl(HybridMutex &M) REQUIRES(M);
+
+private:
+  u32 LastNotifyAll = 0;
+  atomic_u32 Counter = {};
+};
+
+} // namespace scudo
+
+#endif // SCUDO_LINUX
+
+#endif // SCUDO_CONDITION_VARIABLE_LINUX_H_
diff --git a/compiler-rt/lib/scudo/standalone/linux.cpp b/compiler-rt/lib/scudo/standalone/linux.cpp
index c31c3d2483..2746951081 100644
--- a/compiler-rt/lib/scudo/standalone/linux.cpp
+++ b/compiler-rt/lib/scudo/standalone/linux.cpp
@@ -14,6 +14,7 @@
 #include "internal_defs.h"
 #include "linux.h"
 #include "mutex.h"
+#include "report_linux.h"
 #include "string_utils.h"
 
 #include <errno.h>
@@ -66,7 +67,7 @@ void *map(void *Addr, uptr Size, UNUSED const char *Name, uptr Flags,
   void *P = mmap(Addr, Size, MmapProt, MmapFlags, -1, 0);
   if (P == MAP_FAILED) {
     if (!(Flags & MAP_ALLOWNOMEM) || errno != ENOMEM)
-      dieOnMapUnmapError(errno == ENOMEM ? Size : 0);
+      reportMapError(errno == ENOMEM ? Size : 0);
     return nullptr;
   }
 #if SCUDO_ANDROID
@@ -80,7 +81,7 @@ void *map(void *Addr, uptr Size, UNUSED const char *Name, uptr Flags,
 void unmap(void *Addr, uptr Size, UNUSED uptr Flags,
            UNUSED MapPlatformData *Data) {
   if (munmap(Addr, Size) != 0)
-    dieOnMapUnmapError();
+    reportUnmapError(reinterpret_cast<uptr>(Addr), Size);
 }
 
 // TODO: Will be deprecated. Use the interfaces in MemMapLinux instead.
@@ -88,7 +89,7 @@ void setMemoryPermission(uptr Addr, uptr Size, uptr Flags,
                          UNUSED MapPlatformData *Data) {
   int Prot = (Flags & MAP_NOACCESS) ? PROT_NONE : (PROT_READ | PROT_WRITE);
   if (mprotect(reinterpret_cast<void *>(Addr), Size, Prot) != 0)
-    dieOnMapUnmapError();
+    reportProtectError(Addr, Size, Prot);
 }
 
 // TODO: Will be deprecated. Use the interfaces in MemMapLinux instead.
diff --git a/compiler-rt/lib/scudo/standalone/mem_map_linux.cpp b/compiler-rt/lib/scudo/standalone/mem_map_linux.cpp
index f377d10589..783c4f0d9a 100644
--- a/compiler-rt/lib/scudo/standalone/mem_map_linux.cpp
+++ b/compiler-rt/lib/scudo/standalone/mem_map_linux.cpp
@@ -16,6 +16,7 @@
 #include "internal_defs.h"
 #include "linux.h"
 #include "mutex.h"
+#include "report_linux.h"
 #include "string_utils.h"
 
 #include <errno.h>
@@ -64,7 +65,7 @@ static void *mmapWrapper(uptr Addr, uptr Size, const char *Name, uptr Flags) {
       mmap(reinterpret_cast<void *>(Addr), Size, MmapProt, MmapFlags, -1, 0);
   if (P == MAP_FAILED) {
     if (!(Flags & MAP_ALLOWNOMEM) || errno != ENOMEM)
-      dieOnMapUnmapError(errno == ENOMEM ? Size : 0);
+      reportMapError(errno == ENOMEM ? Size : 0);
     return nullptr;
   }
 #if SCUDO_ANDROID
@@ -101,21 +102,21 @@ void MemMapLinux::unmapImpl(uptr Addr, uptr Size) {
   }
 
   if (munmap(reinterpret_cast<void *>(Addr), Size) != 0)
-    dieOnMapUnmapError();
+    reportUnmapError(Addr, Size);
 }
 
 bool MemMapLinux::remapImpl(uptr Addr, uptr Size, const char *Name,
                             uptr Flags) {
   void *P = mmapWrapper(Addr, Size, Name, Flags);
   if (reinterpret_cast<uptr>(P) != Addr)
-    dieOnMapUnmapError();
+    reportMapError();
   return true;
 }
 
 void MemMapLinux::setMemoryPermissionImpl(uptr Addr, uptr Size, uptr Flags) {
   int Prot = (Flags & MAP_NOACCESS) ? PROT_NONE : (PROT_READ | PROT_WRITE);
   if (mprotect(reinterpret_cast<void *>(Addr), Size, Prot) != 0)
-    dieOnMapUnmapError();
+    reportProtectError(Addr, Size, Prot);
 }
 
 void MemMapLinux::releaseAndZeroPagesToOSImpl(uptr From, uptr Size) {
@@ -139,7 +140,7 @@ bool ReservedMemoryLinux::createImpl(uptr Addr, uptr Size, const char *Name,
 
 void ReservedMemoryLinux::releaseImpl() {
   if (munmap(reinterpret_cast<void *>(getBase()), getCapacity()) != 0)
-    dieOnMapUnmapError();
+    reportUnmapError(getBase(), getCapacity());
 }
 
 ReservedMemoryLinux::MemMapT ReservedMemoryLinux::dispatchImpl(uptr Addr,
diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h
index 6e5ab7e3ba..0f4ba88ee1 100644
--- a/compiler-rt/lib/scudo/standalone/primary64.h
+++ b/compiler-rt/lib/scudo/standalone/primary64.h
@@ -22,6 +22,8 @@
 #include "string_utils.h"
 #include "thread_annotations.h"
 
+#include "condition_variable.h"
+
 namespace scudo {
 
 // SizeClassAllocator64 is an allocator tuned for 64-bit address space.
@@ -48,6 +50,8 @@ template <typename Config> class SizeClassAllocator64 {
 public:
   typedef typename Config::Primary::CompactPtrT CompactPtrT;
   typedef typename Config::Primary::SizeClassMap SizeClassMap;
+  typedef typename ConditionVariableState<
+      typename Config::Primary>::ConditionVariableT ConditionVariableT;
   static const uptr CompactPtrScale = Config::Primary::CompactPtrScale;
   static const uptr RegionSizeLog = Config::Primary::RegionSizeLog;
   static const uptr GroupSizeLog = Config::Primary::GroupSizeLog;
@@ -70,6 +74,10 @@ template <typename Config> class SizeClassAllocator64 {
 
   static bool canAllocate(uptr Size) { return Size <= SizeClassMap::MaxSize; }
 
+  static bool conditionVariableEnabled() {
+    return ConditionVariableState<typename Config::Primary>::enabled();
+  }
+
   void init(s32 ReleaseToOsInterval) NO_THREAD_SAFETY_ANALYSIS {
     DCHECK(isAligned(reinterpret_cast<uptr>(this), alignof(ThisT)));
 
@@ -124,6 +132,7 @@ template <typename Config> class SizeClassAllocator64 {
 
     for (uptr I = 0; I < NumClasses; I++) {
       RegionInfo *Region = getRegionInfo(I);
+
       // The actual start of a region is offset by a random number of pages
       // when PrimaryEnableRandomOffset is set.
       Region->RegionBeg = (PrimaryBase + (I << RegionSizeLog)) +
@@ -145,6 +154,11 @@ template <typename Config> class SizeClassAllocator64 {
     }
     shuffle(RegionInfoArray, NumClasses, &Seed);
 
+    // The binding should be done after region shuffling so that it won't bind
+    // the FLLock from the wrong region.
+    for (uptr I = 0; I < NumClasses; I++)
+      getRegionInfo(I)->FLLockCV.bindTestOnly(getRegionInfo(I)->FLLock);
+
     setOption(Option::ReleaseInterval, static_cast<sptr>(ReleaseToOsInterval));
   }
 
@@ -236,26 +250,26 @@ template <typename Config> class SizeClassAllocator64 {
     bool ReportRegionExhausted = false;
     TransferBatchT *B = nullptr;
 
-    while (true) {
-      // When two threads compete for `Region->MMLock`, we only want one of them
-      // does the populateFreeListAndPopBatch(). To avoid both of them doing
-      // that, always check the freelist before mapping new pages.
-      //
-      // TODO(chiahungduan): Use a condition variable so that we don't need to
-      // hold `Region->MMLock` here.
-      ScopedLock ML(Region->MMLock);
-      {
-        ScopedLock FL(Region->FLLock);
-        B = popBatchImpl(C, ClassId, Region);
-        if (LIKELY(B))
-          return B;
-      }
+    if (conditionVariableEnabled()) {
+      B = popBatchWithCV(C, ClassId, Region, ReportRegionExhausted);
+    } else {
+      while (true) {
+        // When two threads compete for `Region->MMLock`, we only want one of
+        // them to call populateFreeListAndPopBatch(). To avoid both of them
+        // doing that, always check the freelist before mapping new pages.
+        ScopedLock ML(Region->MMLock);
+        {
+          ScopedLock FL(Region->FLLock);
+          if ((B = popBatchImpl(C, ClassId, Region)))
+            break;
+        }
 
-      const bool RegionIsExhausted = Region->Exhausted;
-      if (!RegionIsExhausted)
-        B = populateFreeListAndPopBatch(C, ClassId, Region);
-      ReportRegionExhausted = !RegionIsExhausted && Region->Exhausted;
-      break;
+        const bool RegionIsExhausted = Region->Exhausted;
+        if (!RegionIsExhausted)
+          B = populateFreeListAndPopBatch(C, ClassId, Region);
+        ReportRegionExhausted = !RegionIsExhausted && Region->Exhausted;
+        break;
+      }
     }
 
     if (UNLIKELY(ReportRegionExhausted)) {
@@ -280,6 +294,8 @@ template <typename Config> class SizeClassAllocator64 {
     if (ClassId == SizeClassMap::BatchClassId) {
       ScopedLock L(Region->FLLock);
       pushBatchClassBlocks(Region, Array, Size);
+      if (conditionVariableEnabled())
+        Region->FLLockCV.notifyAll(Region->FLLock);
       return;
     }
 
@@ -306,6 +322,8 @@ template <typename Config> class SizeClassAllocator64 {
     {
       ScopedLock L(Region->FLLock);
       pushBlocksImpl(C, ClassId, Region, Array, Size, SameGroup);
+      if (conditionVariableEnabled())
+        Region->FLLockCV.notifyAll(Region->FLLock);
     }
   }
 
@@ -538,6 +556,7 @@ template <typename Config> class SizeClassAllocator64 {
   struct UnpaddedRegionInfo {
     // Mutex for operations on freelist
     HybridMutex FLLock;
+    ConditionVariableT FLLockCV GUARDED_BY(FLLock);
     // Mutex for memmap operations
     HybridMutex MMLock ACQUIRED_BEFORE(FLLock);
     // `RegionBeg` is initialized before thread creation and won't be changed.
@@ -549,6 +568,7 @@ template <typename Config> class SizeClassAllocator64 {
     uptr TryReleaseThreshold GUARDED_BY(MMLock) = 0;
     ReleaseToOsInfo ReleaseInfo GUARDED_BY(MMLock) = {};
     bool Exhausted GUARDED_BY(MMLock) = false;
+    bool isPopulatingFreeList GUARDED_BY(FLLock) = false;
   };
   struct RegionInfo : UnpaddedRegionInfo {
     char Padding[SCUDO_CACHE_LINE_SIZE -
@@ -831,6 +851,76 @@ template <typename Config> class SizeClassAllocator64 {
     InsertBlocks(Cur, Array + Size - Count, Count);
   }
 
+  TransferBatchT *popBatchWithCV(CacheT *C, uptr ClassId, RegionInfo *Region,
+                                 bool &ReportRegionExhausted) {
+    TransferBatchT *B = nullptr;
+
+    while (true) {
+      // We only expect one thread doing the freelist refillment and other
+      // threads will be waiting for either the completion of the
+      // `populateFreeListAndPopBatch()` or `pushBlocks()` called by other
+      // threads.
+      bool PopulateFreeList = false;
+      {
+        ScopedLock FL(Region->FLLock);
+        if (!Region->isPopulatingFreeList) {
+          Region->isPopulatingFreeList = true;
+          PopulateFreeList = true;
+        }
+      }
+
+      if (PopulateFreeList) {
+        ScopedLock ML(Region->MMLock);
+
+        const bool RegionIsExhausted = Region->Exhausted;
+        if (!RegionIsExhausted)
+          B = populateFreeListAndPopBatch(C, ClassId, Region);
+        ReportRegionExhausted = !RegionIsExhausted && Region->Exhausted;
+
+        {
+          // Before reacquiring the `FLLock`, the freelist may be used up again
+          // and some threads are waiting for the freelist refillment by the
+          // current thread. It's important to set
+          // `Region->isPopulatingFreeList` to false so the threads about to
+          // sleep will notice the status change.
+          ScopedLock FL(Region->FLLock);
+          Region->isPopulatingFreeList = false;
+          Region->FLLockCV.notifyAll(Region->FLLock);
+        }
+
+        break;
+      }
+
+      // At here, there are two preconditions to be met before waiting,
+      //   1. The freelist is empty.
+      //   2. Region->isPopulatingFreeList == true, i.e, someone is still doing
+      //   `populateFreeListAndPopBatch()`.
+      //
+      // Note that it has the chance that freelist is empty but
+      // Region->isPopulatingFreeList == false because all the new populated
+      // blocks were used up right after the refillment. Therefore, we have to
+      // check if someone is still populating the freelist.
+      ScopedLock FL(Region->FLLock);
+      if (LIKELY(B = popBatchImpl(C, ClassId, Region)))
+        break;
+
+      if (!Region->isPopulatingFreeList)
+        continue;
+
+      // Now the freelist is empty and someone's doing the refillment. We will
+      // wait until anyone refills the freelist or someone finishes doing
+      // `populateFreeListAndPopBatch()`. The refillment can be done by
+      // `populateFreeListAndPopBatch()`, `pushBlocks()`,
+      // `pushBatchClassBlocks()` and `mergeGroupsToReleaseBack()`.
+      Region->FLLockCV.wait(Region->FLLock);
+
+      if (LIKELY(B = popBatchImpl(C, ClassId, Region)))
+        break;
+    }
+
+    return B;
+  }
+
   // Pop one TransferBatch from a BatchGroup. The BatchGroup with the smallest
   // group id will be considered first.
   //
@@ -1521,6 +1611,8 @@ template <typename Config> class SizeClassAllocator64 {
         if (UNLIKELY(Idx + NeededSlots > MaxUnusedSize)) {
           ScopedLock L(BatchClassRegion->FLLock);
           pushBatchClassBlocks(BatchClassRegion, Blocks, Idx);
+          if (conditionVariableEnabled())
+            BatchClassRegion->FLLockCV.notifyAll(BatchClassRegion->FLLock);
           Idx = 0;
         }
         Blocks[Idx++] =
@@ -1556,6 +1648,8 @@ template <typename Config> class SizeClassAllocator64 {
     if (Idx != 0) {
       ScopedLock L(BatchClassRegion->FLLock);
       pushBatchClassBlocks(BatchClassRegion, Blocks, Idx);
+      if (conditionVariableEnabled())
+        BatchClassRegion->FLLockCV.notifyAll(BatchClassRegion->FLLock);
     }
 
     if (SCUDO_DEBUG) {
@@ -1565,6 +1659,9 @@ template <typename Config> class SizeClassAllocator64 {
         CHECK_LT(Prev->CompactPtrGroupBase, Cur->CompactPtrGroupBase);
       }
     }
+
+    if (conditionVariableEnabled())
+      Region->FLLockCV.notifyAll(Region->FLLock);
   }
 
   // TODO: `PrimaryBase` can be obtained from ReservedMemory. This needs to be
diff --git a/compiler-rt/lib/scudo/standalone/report.cpp b/compiler-rt/lib/scudo/standalone/report.cpp
index c033949a85..9cef0adc0b 100644
--- a/compiler-rt/lib/scudo/standalone/report.cpp
+++ b/compiler-rt/lib/scudo/standalone/report.cpp
@@ -24,11 +24,7 @@ class ScopedErrorReport {
     Message.vappend(Format, Args);
     va_end(Args);
   }
-  NORETURN ~ScopedErrorReport() {
-    outputRaw(Message.data());
-    setAbortMessage(Message.data());
-    die();
-  }
+  NORETURN ~ScopedErrorReport() { reportRawError(Message.data()); }
 
 private:
   ScopedString Message;
@@ -55,6 +51,13 @@ void NORETURN reportError(const char *Message) {
   Report.append("%s\n", Message);
 }
 
+// Generic fatal error message without ScopedString.
+void NORETURN reportRawError(const char *Message) {
+  outputRaw(Message);
+  setAbortMessage(Message);
+  die();
+}
+
 void NORETURN reportInvalidFlag(const char *FlagType, const char *Value) {
   ScopedErrorReport Report;
   Report.append("invalid value for %s option: '%s'\n", FlagType, Value);
diff --git a/compiler-rt/lib/scudo/standalone/report.h b/compiler-rt/lib/scudo/standalone/report.h
index d8c2dea994..a510fdaebb 100644
--- a/compiler-rt/lib/scudo/standalone/report.h
+++ b/compiler-rt/lib/scudo/standalone/report.h
@@ -15,9 +15,12 @@ namespace scudo {
 
 // Reports are *fatal* unless stated otherwise.
 
-// Generic error.
+// Generic error, adds newline to end of message.
 void NORETURN reportError(const char *Message);
 
+// Generic error, but the message is not modified.
+void NORETURN reportRawError(const char *Message);
+
 // Flags related errors.
 void NORETURN reportInvalidFlag(const char *FlagType, const char *Value);
 
diff --git a/compiler-rt/lib/scudo/standalone/report_linux.cpp b/compiler-rt/lib/scudo/standalone/report_linux.cpp
new file mode 100644
index 0000000000..6a983036e6
--- /dev/null
+++ b/compiler-rt/lib/scudo/standalone/report_linux.cpp
@@ -0,0 +1,58 @@
+//===-- report_linux.cpp ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "platform.h"
+
+#if SCUDO_LINUX || SCUDO_TRUSTY
+
+#include "common.h"
+#include "internal_defs.h"
+#include "report.h"
+#include "report_linux.h"
+#include "string_utils.h"
+
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
+
+namespace scudo {
+
+// Fatal internal map() error (potentially OOM related).
+void NORETURN reportMapError(uptr SizeIfOOM) {
+  char Error[128] = "Scudo ERROR: internal map failure\n";
+  if (SizeIfOOM) {
+    formatString(
+        Error, sizeof(Error),
+        "Scudo ERROR: internal map failure (NO MEMORY) requesting %zuKB\n",
+        SizeIfOOM >> 10);
+  }
+  reportRawError(Error);
+}
+
+void NORETURN reportUnmapError(uptr Addr, uptr Size) {
+  char Error[128];
+  formatString(Error, sizeof(Error),
+               "Scudo ERROR: internal unmap failure (error desc=%s) Addr 0x%zx "
+               "Size %zu\n",
+               strerror(errno), Addr, Size);
+  reportRawError(Error);
+}
+
+void NORETURN reportProtectError(uptr Addr, uptr Size, int Prot) {
+  char Error[128];
+  formatString(
+      Error, sizeof(Error),
+      "Scudo ERROR: internal protect failure (error desc=%s) Addr 0x%zx "
+      "Size %zu Prot %x\n",
+      strerror(errno), Addr, Size, Prot);
+  reportRawError(Error);
+}
+
+} // namespace scudo
+
+#endif // SCUDO_LINUX || SCUDO_TRUSTY
diff --git a/compiler-rt/lib/scudo/standalone/report_linux.h b/compiler-rt/lib/scudo/standalone/report_linux.h
new file mode 100644
index 0000000000..aa0bb247e6
--- /dev/null
+++ b/compiler-rt/lib/scudo/standalone/report_linux.h
@@ -0,0 +1,34 @@
+//===-- report_linux.h ------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SCUDO_REPORT_LINUX_H_
+#define SCUDO_REPORT_LINUX_H_
+
+#include "platform.h"
+
+#if SCUDO_LINUX || SCUDO_TRUSTY
+
+#include "internal_defs.h"
+
+namespace scudo {
+
+// Report a fatal error when a map call fails. SizeIfOOM shall
+// hold the requested size on an out-of-memory error, 0 otherwise.
+void NORETURN reportMapError(uptr SizeIfOOM = 0);
+
+// Report a fatal error when an unmap call fails.
+void NORETURN reportUnmapError(uptr Addr, uptr Size);
+
+// Report a fatal error when a mprotect call fails.
+void NORETURN reportProtectError(uptr Addr, uptr Size, int Prot);
+
+} // namespace scudo
+
+#endif // SCUDO_LINUX || SCUDO_TRUSTY
+
+#endif // SCUDO_REPORT_LINUX_H_
diff --git a/compiler-rt/lib/scudo/standalone/tests/CMakeLists.txt b/compiler-rt/lib/scudo/standalone/tests/CMakeLists.txt
index a4a031d54d..c6b6a1cb57 100644
--- a/compiler-rt/lib/scudo/standalone/tests/CMakeLists.txt
+++ b/compiler-rt/lib/scudo/standalone/tests/CMakeLists.txt
@@ -96,6 +96,7 @@ set(SCUDO_UNIT_TEST_SOURCES
   chunk_test.cpp
   combined_test.cpp
   common_test.cpp
+  condition_variable_test.cpp
   flags_test.cpp
   list_test.cpp
   map_test.cpp
diff --git a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
index 7958e9dabf..3dbd93cace 100644
--- a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
@@ -12,7 +12,9 @@
 #include "allocator_config.h"
 #include "chunk.h"
 #include "combined.h"
+#include "condition_variable.h"
 #include "mem_map.h"
+#include "size_class_map.h"
 
 #include <algorithm>
 #include <condition_variable>
@@ -164,13 +166,60 @@ template <class TypeParam> struct ScudoCombinedTest : public Test {
 
 template <typename T> using ScudoCombinedDeathTest = ScudoCombinedTest<T>;
 
+namespace scudo {
+struct TestConditionVariableConfig {
+  static const bool MaySupportMemoryTagging = true;
+  template <class A>
+  using TSDRegistryT =
+      scudo::TSDRegistrySharedT<A, 8U, 4U>; // Shared, max 8 TSDs.
+
+  struct Primary {
+    using SizeClassMap = scudo::AndroidSizeClassMap;
+#if SCUDO_CAN_USE_PRIMARY64
+    static const scudo::uptr RegionSizeLog = 28U;
+    typedef scudo::u32 CompactPtrT;
+    static const scudo::uptr CompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG;
+    static const scudo::uptr GroupSizeLog = 20U;
+    static const bool EnableRandomOffset = true;
+    static const scudo::uptr MapSizeIncrement = 1UL << 18;
+#else
+    static const scudo::uptr RegionSizeLog = 18U;
+    static const scudo::uptr GroupSizeLog = 18U;
+    typedef scudo::uptr CompactPtrT;
+#endif
+    static const scudo::s32 MinReleaseToOsIntervalMs = 1000;
+    static const scudo::s32 MaxReleaseToOsIntervalMs = 1000;
+    static const bool UseConditionVariable = true;
+#if SCUDO_LINUX
+    using ConditionVariableT = scudo::ConditionVariableLinux;
+#else
+    using ConditionVariableT = scudo::ConditionVariableDummy;
+#endif
+  };
+#if SCUDO_CAN_USE_PRIMARY64
+  template <typename Config>
+  using PrimaryT = scudo::SizeClassAllocator64<Config>;
+#else
+  template <typename Config>
+  using PrimaryT = scudo::SizeClassAllocator32<Config>;
+#endif
+
+  struct Secondary {
+    template <typename Config>
+    using CacheT = scudo::MapAllocatorNoCache<Config>;
+  };
+  template <typename Config> using SecondaryT = scudo::MapAllocator<Config>;
+};
+} // namespace scudo
+
 #if SCUDO_FUCHSIA
 #define SCUDO_TYPED_TEST_ALL_TYPES(FIXTURE, NAME)                              \
   SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, FuchsiaConfig)
 #else
 #define SCUDO_TYPED_TEST_ALL_TYPES(FIXTURE, NAME)                              \
   SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, DefaultConfig)                          \
-  SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, AndroidConfig)
+  SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, AndroidConfig)                          \
+  SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, TestConditionVariableConfig)
 #endif
 
 #define SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, TYPE)                             \
diff --git a/compiler-rt/lib/scudo/standalone/tests/condition_variable_test.cpp b/compiler-rt/lib/scudo/standalone/tests/condition_variable_test.cpp
new file mode 100644
index 0000000000..caba1f64ab
--- /dev/null
+++ b/compiler-rt/lib/scudo/standalone/tests/condition_variable_test.cpp
@@ -0,0 +1,59 @@
+//===-- condition_variable_test.cpp -----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "tests/scudo_unit_test.h"
+
+#include "common.h"
+#include "condition_variable.h"
+#include "mutex.h"
+
+#include <thread>
+
+template <typename ConditionVariableT> void simpleWaitAndNotifyAll() {
+  constexpr scudo::u32 NumThreads = 2;
+  constexpr scudo::u32 CounterMax = 1024;
+  std::thread Threads[NumThreads];
+
+  scudo::HybridMutex M;
+  ConditionVariableT CV;
+  CV.bindTestOnly(M);
+  scudo::u32 Counter = 0;
+
+  for (scudo::u32 I = 0; I < NumThreads; ++I) {
+    Threads[I] = std::thread(
+        [&](scudo::u32 Id) {
+          do {
+            scudo::ScopedLock L(M);
+            if (Counter % NumThreads != Id && Counter < CounterMax)
+              CV.wait(M);
+            if (Counter >= CounterMax) {
+              break;
+            } else {
+              ++Counter;
+              CV.notifyAll(M);
+            }
+          } while (true);
+        },
+        I);
+  }
+
+  for (std::thread &T : Threads)
+    T.join();
+
+  EXPECT_EQ(Counter, CounterMax);
+}
+
+TEST(ScudoConditionVariableTest, DummyCVWaitAndNotifyAll) {
+  simpleWaitAndNotifyAll<scudo::ConditionVariableDummy>();
+}
+
+#ifdef SCUDO_LINUX
+TEST(ScudoConditionVariableTest, LinuxCVWaitAndNotifyAll) {
+  simpleWaitAndNotifyAll<scudo::ConditionVariableLinux>();
+}
+#endif
diff --git a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
index 4db05b7624..1817151175 100644
--- a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
@@ -9,6 +9,7 @@
 #include "tests/scudo_unit_test.h"
 
 #include "allocator_config.h"
+#include "condition_variable.h"
 #include "primary32.h"
 #include "primary64.h"
 #include "size_class_map.h"
@@ -105,6 +106,34 @@ template <typename SizeClassMapT> struct TestConfig4 {
   };
 };
 
+// This is the only test config that enables the condition variable.
+template <typename SizeClassMapT> struct TestConfig5 {
+  static const bool MaySupportMemoryTagging = true;
+
+  struct Primary {
+    using SizeClassMap = SizeClassMapT;
+#if defined(__mips__)
+    // Unable to allocate greater size on QEMU-user.
+    static const scudo::uptr RegionSizeLog = 23U;
+#else
+    static const scudo::uptr RegionSizeLog = 24U;
+#endif
+    static const scudo::s32 MinReleaseToOsIntervalMs = INT32_MIN;
+    static const scudo::s32 MaxReleaseToOsIntervalMs = INT32_MAX;
+    static const scudo::uptr CompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG;
+    static const scudo::uptr GroupSizeLog = 18U;
+    typedef scudo::u32 CompactPtrT;
+    static const bool EnableRandomOffset = true;
+    static const scudo::uptr MapSizeIncrement = 1UL << 18;
+    static const bool UseConditionVariable = true;
+#if SCUDO_LINUX
+    using ConditionVariableT = scudo::ConditionVariableLinux;
+#else
+    using ConditionVariableT = scudo::ConditionVariableDummy;
+#endif
+  };
+};
+
 template <template <typename> class BaseConfig, typename SizeClassMapT>
 struct Config : public BaseConfig<SizeClassMapT> {};
 
@@ -143,7 +172,8 @@ struct ScudoPrimaryTest : public Test {};
   SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, TestConfig1)                            \
   SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, TestConfig2)                            \
   SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, TestConfig3)                            \
-  SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, TestConfig4)
+  SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, TestConfig4)                            \
+  SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, TestConfig5)
 #endif
 
 #define SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, TYPE)                             \
diff --git a/compiler-rt/lib/scudo/standalone/trusty.cpp b/compiler-rt/lib/scudo/standalone/trusty.cpp
index 5f72b1cb3e..26b349c6e5 100644
--- a/compiler-rt/lib/scudo/standalone/trusty.cpp
+++ b/compiler-rt/lib/scudo/standalone/trusty.cpp
@@ -12,6 +12,7 @@
 
 #include "common.h"
 #include "mutex.h"
+#include "report_linux.h"
 #include "trusty.h"
 
 #include <errno.h>           // for errno
@@ -51,7 +52,7 @@ void *map(void *Addr, uptr Size, const char *Name, uptr Flags,
   if (IS_ERR(P)) {
     errno = lk_err_to_errno(PTR_ERR(P));
     if (!(Flags & MAP_ALLOWNOMEM) || errno != ENOMEM)
-      dieOnMapUnmapError(Size);
+      reportMapError(Size);
     return nullptr;
   }
 
@@ -61,7 +62,7 @@ void *map(void *Addr, uptr Size, const char *Name, uptr Flags,
 void unmap(UNUSED void *Addr, UNUSED uptr Size, UNUSED uptr Flags,
            UNUSED MapPlatformData *Data) {
   if (_trusty_munmap(Addr, Size) != 0)
-    dieOnMapUnmapError();
+    reportUnmapError(reinterpret_cast<uptr>(Addr), Size);
 }
 
 void setMemoryPermission(UNUSED uptr Addr, UNUSED uptr Size, UNUSED uptr Flags,
diff --git a/compiler-rt/lib/tsan/rtl/CMakeLists.txt b/compiler-rt/lib/tsan/rtl/CMakeLists.txt
index 7b18d379e9..791c0596f6 100644
--- a/compiler-rt/lib/tsan/rtl/CMakeLists.txt
+++ b/compiler-rt/lib/tsan/rtl/CMakeLists.txt
@@ -220,6 +220,10 @@ else()
       set(TSAN_ASM_SOURCES
         tsan_rtl_mips64.S
         )
+    elseif(arch MATCHES "riscv64")
+      set(TSAN_ASM_SOURCES
+        tsan_rtl_riscv64.S
+        )
     elseif(arch MATCHES "s390x")
       set(TSAN_ASM_SOURCES
         tsan_rtl_s390x.S
diff --git a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
index 5add97ccd1..80f86ca98e 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
@@ -81,6 +81,8 @@ struct ucontext_t {
 #define PTHREAD_ABI_BASE  "GLIBC_2.17"
 #elif SANITIZER_LOONGARCH64
 #define PTHREAD_ABI_BASE  "GLIBC_2.36"
+#elif SANITIZER_RISCV64
+#  define PTHREAD_ABI_BASE "GLIBC_2.27"
 #endif
 
 extern "C" int pthread_attr_init(void *attr);
diff --git a/compiler-rt/lib/tsan/rtl/tsan_interface.h b/compiler-rt/lib/tsan/rtl/tsan_interface.h
index d53c1e3935..3731c90d45 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_interface.h
+++ b/compiler-rt/lib/tsan/rtl/tsan_interface.h
@@ -419,6 +419,14 @@ void __tsan_go_atomic32_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
 SANITIZER_INTERFACE_ATTRIBUTE
 void __tsan_go_atomic64_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
 SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic32_fetch_and(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic64_fetch_and(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic32_fetch_or(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic64_fetch_or(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
+SANITIZER_INTERFACE_ATTRIBUTE
 void __tsan_go_atomic32_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
 SANITIZER_INTERFACE_ATTRIBUTE
 void __tsan_go_atomic64_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
diff --git a/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp b/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp
index f794a2fcdd..2b5a2c6ef7 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp
@@ -894,6 +894,30 @@ void __tsan_go_atomic64_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
   ATOMIC_RET(FetchAdd, *(a64*)(a+16), *(a64**)a, *(a64*)(a+8), mo_acq_rel);
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic32_fetch_and(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
+  ATOMIC_RET(FetchAnd, *(a32 *)(a + 16), *(a32 **)a, *(a32 *)(a + 8),
+             mo_acq_rel);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic64_fetch_and(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
+  ATOMIC_RET(FetchAnd, *(a64 *)(a + 16), *(a64 **)a, *(a64 *)(a + 8),
+             mo_acq_rel);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic32_fetch_or(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
+  ATOMIC_RET(FetchOr, *(a32 *)(a + 16), *(a32 **)a, *(a32 *)(a + 8),
+             mo_acq_rel);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_go_atomic64_fetch_or(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
+  ATOMIC_RET(FetchOr, *(a64 *)(a + 16), *(a64 **)a, *(a64 *)(a + 8),
+             mo_acq_rel);
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE
 void __tsan_go_atomic32_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
   ATOMIC_RET(Exchange, *(a32*)(a+16), *(a32**)a, *(a32*)(a+8), mo_acq_rel);
diff --git a/compiler-rt/lib/tsan/rtl/tsan_platform.h b/compiler-rt/lib/tsan/rtl/tsan_platform.h
index f0cdaf48ea..cfbb57d1d8 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_platform.h
+++ b/compiler-rt/lib/tsan/rtl/tsan_platform.h
@@ -377,6 +377,71 @@ struct MappingPPC64_47 {
   static const uptr kMidAppMemEnd = 0;
 };
 
+/*
+C/C++ on linux/riscv64 (39-bit VMA)
+0000 0010 00 - 0200 0000 00: main binary                      ( 8 GB)
+0200 0000 00 - 1000 0000 00: -
+1000 0000 00 - 4000 0000 00: shadow memory                    (64 GB)
+4000 0000 00 - 4800 0000 00: metainfo                         (16 GB)
+4800 0000 00 - 5500 0000 00: -
+5500 0000 00 - 5a00 0000 00: main binary (PIE)                (~8 GB)
+5600 0000 00 - 7c00 0000 00: -
+7d00 0000 00 - 7fff ffff ff: libraries and main thread stack  ( 8 GB)
+
+mmap by default allocates from top downwards
+VDSO sits below loader and above dynamic libraries, within HiApp region.
+Heap starts after program region whose position depends on pie or non-pie.
+Disable tracking them since their locations are not fixed.
+*/
+struct MappingRiscv64_39 {
+  static const uptr kLoAppMemBeg = 0x0000001000ull;
+  static const uptr kLoAppMemEnd = 0x0200000000ull;
+  static const uptr kShadowBeg = 0x1000000000ull;
+  static const uptr kShadowEnd = 0x2000000000ull;
+  static const uptr kMetaShadowBeg = 0x2000000000ull;
+  static const uptr kMetaShadowEnd = 0x2400000000ull;
+  static const uptr kMidAppMemBeg = 0x2aaaaaa000ull;
+  static const uptr kMidAppMemEnd = 0x2c00000000ull;
+  static const uptr kHeapMemBeg = 0x2c00000000ull;
+  static const uptr kHeapMemEnd = 0x2c00000000ull;
+  static const uptr kHiAppMemBeg = 0x3c00000000ull;
+  static const uptr kHiAppMemEnd = 0x3fffffffffull;
+  static const uptr kShadowMsk = 0x3800000000ull;
+  static const uptr kShadowXor = 0x0800000000ull;
+  static const uptr kShadowAdd = 0x0000000000ull;
+  static const uptr kVdsoBeg = 0x4000000000ull;
+};
+
+/*
+C/C++ on linux/riscv64 (48-bit VMA)
+0000 0000 1000 - 0500 0000 0000: main binary                      ( 5 TB)
+0500 0000 0000 - 2000 0000 0000: -
+2000 0000 0000 - 4000 0000 0000: shadow memory                    (32 TB)
+4000 0000 0000 - 4800 0000 0000: metainfo                         ( 8 TB)
+4800 0000 0000 - 5555 5555 5000: -
+5555 5555 5000 - 5a00 0000 0000: main binary (PIE)                (~5 TB)
+5a00 0000 0000 - 7a00 0000 0000: -
+7a00 0000 0000 - 7fff ffff ffff: libraries and main thread stack  ( 5 TB)
+*/
+struct MappingRiscv64_48 {
+  static const uptr kLoAppMemBeg = 0x000000001000ull;
+  static const uptr kLoAppMemEnd = 0x050000000000ull;
+  static const uptr kShadowBeg = 0x200000000000ull;
+  static const uptr kShadowEnd = 0x400000000000ull;
+  static const uptr kMetaShadowBeg = 0x400000000000ull;
+  static const uptr kMetaShadowEnd = 0x480000000000ull;
+  static const uptr kMidAppMemBeg = 0x555555555000ull;
+  static const uptr kMidAppMemEnd = 0x5a0000000000ull;
+  static const uptr kHeapMemBeg = 0x5a0000000000ull;
+  static const uptr kHeapMemEnd = 0x5a0000000000ull;
+  static const uptr kHiAppMemBeg = 0x7a0000000000ull;
+  static const uptr kHiAppMemEnd = 0x7fffffffffffull;
+  static const uptr kShadowMsk = 0x700000000000ull;
+  static const uptr kShadowXor = 0x100000000000ull;
+  static const uptr kShadowAdd = 0x000000000000ull;
+  static const uptr kVdsoBeg = 0x800000000000ull;
+};
+
 /*
 C/C++ on linux/s390x
 While the kernel provides a 64-bit address space, we have to restrict ourselves
@@ -665,6 +730,13 @@ ALWAYS_INLINE auto SelectMapping(Arg arg) {
   }
 #  elif defined(__mips64)
   return Func::template Apply<MappingMips64_40>(arg);
+#  elif SANITIZER_RISCV64
+  switch (vmaSize) {
+    case 39:
+      return Func::template Apply<MappingRiscv64_39>(arg);
+    case 48:
+      return Func::template Apply<MappingRiscv64_48>(arg);
+  }
 #  elif defined(__s390x__)
   return Func::template Apply<MappingS390x>(arg);
 #  else
@@ -686,6 +758,8 @@ void ForEachMapping() {
   Func::template Apply<MappingPPC64_44>();
   Func::template Apply<MappingPPC64_46>();
   Func::template Apply<MappingPPC64_47>();
+  Func::template Apply<MappingRiscv64_39>();
+  Func::template Apply<MappingRiscv64_48>();
   Func::template Apply<MappingS390x>();
   Func::template Apply<MappingGo48>();
   Func::template Apply<MappingGoWindows>();
@@ -894,7 +968,7 @@ struct RestoreAddrImpl {
         Mapping::kMidAppMemEnd, Mapping::kHiAppMemBeg, Mapping::kHiAppMemEnd,
         Mapping::kHeapMemBeg,   Mapping::kHeapMemEnd,
     };
-    const uptr indicator = 0x0e0000000000ull;
+    const uptr indicator = 0x0f0000000000ull;
     const uptr ind_lsb = 1ull << LeastSignificantSetBitIndex(indicator);
     for (uptr i = 0; i < ARRAY_SIZE(ranges); i += 2) {
       uptr beg = ranges[i];
diff --git a/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cpp b/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cpp
index d161fa8d21..369509ed0a 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cpp
@@ -267,7 +267,17 @@ void InitializePlatformEarly() {
     Die();
   }
 # endif
-#endif
+#  elif SANITIZER_RISCV64
+  // the bottom half of vma is allocated for userspace
+  vmaSize = vmaSize + 1;
+#    if !SANITIZER_GO
+  if (vmaSize != 39 && vmaSize != 48) {
+    Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
+    Printf("FATAL: Found %zd - Supported 39 and 48\n", vmaSize);
+    Die();
+  }
+#    endif
+#  endif
 }
 
 void InitializePlatform() {
@@ -399,13 +409,15 @@ static uptr UnmangleLongJmpSp(uptr mangled_sp) {
   return mangled_sp ^ xor_key;
 #elif defined(__mips__)
   return mangled_sp;
-#elif defined(__s390x__)
+#    elif SANITIZER_RISCV64
+  return mangled_sp;
+#    elif defined(__s390x__)
   // tcbhead_t.stack_guard
   uptr xor_key = ((uptr *)__builtin_thread_pointer())[5];
   return mangled_sp ^ xor_key;
-#else
-  #error "Unknown platform"
-#endif
+#    else
+#      error "Unknown platform"
+#    endif
 }
 
 #if SANITIZER_NETBSD
@@ -429,11 +441,13 @@ static uptr UnmangleLongJmpSp(uptr mangled_sp) {
 #  define LONG_JMP_SP_ENV_SLOT 1
 # elif defined(__mips64)
 #  define LONG_JMP_SP_ENV_SLOT 1
-# elif defined(__s390x__)
-#  define LONG_JMP_SP_ENV_SLOT 9
-# else
-#  define LONG_JMP_SP_ENV_SLOT 6
-# endif
+#      elif SANITIZER_RISCV64
+#        define LONG_JMP_SP_ENV_SLOT 13
+#      elif defined(__s390x__)
+#        define LONG_JMP_SP_ENV_SLOT 9
+#      else
+#        define LONG_JMP_SP_ENV_SLOT 6
+#      endif
 #endif
 
 uptr ExtractLongJmpSp(uptr *env) {
diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl.h b/compiler-rt/lib/tsan/rtl/tsan_rtl.h
index a5606dbc7f..de4ea0bb5f 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_rtl.h
+++ b/compiler-rt/lib/tsan/rtl/tsan_rtl.h
@@ -56,8 +56,8 @@ namespace __tsan {
 
 #if !SANITIZER_GO
 struct MapUnmapCallback;
-#if defined(__mips64) || defined(__aarch64__) || defined(__loongarch__) || \
-    defined(__powerpc__)
+#  if defined(__mips64) || defined(__aarch64__) || defined(__loongarch__) || \
+      defined(__powerpc__) || SANITIZER_RISCV64
 
 struct AP32 {
   static const uptr kSpaceBeg = 0;
diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl_riscv64.S b/compiler-rt/lib/tsan/rtl/tsan_rtl_riscv64.S
new file mode 100644
index 0000000000..8e6b9b9432
--- /dev/null
+++ b/compiler-rt/lib/tsan/rtl/tsan_rtl_riscv64.S
@@ -0,0 +1,203 @@
+#include "sanitizer_common/sanitizer_asm.h"
+
+.section .text
+
+.comm _ZN14__interception11real_setjmpE,8,8
+.globl ASM_SYMBOL_INTERCEPTOR(setjmp)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(setjmp))
+ASM_SYMBOL_INTERCEPTOR(setjmp):
+  CFI_STARTPROC
+
+  // Save frame pointer and return address register
+  addi sp, sp, -32
+  sd ra, 24(sp)
+  sd s0, 16(sp)
+  CFI_DEF_CFA_OFFSET (32)
+  CFI_OFFSET (1, -8)
+  CFI_OFFSET (8, -16)
+
+  // Adjust the SP for previous frame
+  addi s0, sp, 32
+  CFI_DEF_CFA_REGISTER (8)
+
+  // Save env parameter
+  sd a0, 8(sp)
+  CFI_OFFSET (10, -24)
+
+  // Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
+  addi  a0, s0, 0
+
+  // call tsan interceptor
+  call ASM_SYMBOL(__tsan_setjmp)
+
+  // Restore env parameter
+  ld a0, 8(sp)
+  CFI_RESTORE (10)
+
+  // Restore frame/link register
+  ld s0, 16(sp)
+  ld ra, 24(sp)
+  addi sp, sp, 32
+  CFI_RESTORE (8)
+  CFI_RESTORE (1)
+  CFI_DEF_CFA (2, 0)
+
+  // tail jump to libc setjmp
+  la t1, _ZN14__interception11real_setjmpE
+  ld t1, 0(t1)
+  jr t1
+
+  CFI_ENDPROC
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(setjmp))
+
+.comm _ZN14__interception12real__setjmpE,8,8
+.globl ASM_SYMBOL_INTERCEPTOR(_setjmp)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp))
+ASM_SYMBOL_INTERCEPTOR(_setjmp):
+  CFI_STARTPROC
+
+  // Save frame pointer and return address register
+  addi sp, sp, -32
+  sd ra, 24(sp)
+  sd s0, 16(sp)
+  CFI_DEF_CFA_OFFSET (32)
+  CFI_OFFSET (1, -8)
+  CFI_OFFSET (8, -16)
+
+  // Adjust the SP for previous frame
+  addi s0, sp, 32
+  CFI_DEF_CFA_REGISTER (8)
+
+  // Save env parameter
+  sd a0, 8(sp)
+  CFI_OFFSET (10, -24)
+
+  // Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
+  addi  a0, s0, 0
+
+  // call tsan interceptor
+  call ASM_SYMBOL(__tsan_setjmp)
+
+  // Restore env parameter
+  ld a0, 8(sp)
+  CFI_RESTORE (10)
+
+  // Restore frame/link register
+  ld s0, 16(sp)
+  ld ra, 24(sp)
+  addi sp, sp, 32
+  CFI_RESTORE (8)
+  CFI_RESTORE (1)
+  CFI_DEF_CFA (2, 0)
+
+  // tail jump to libc setjmp
+  la t1, _ZN14__interception12real__setjmpE
+  ld t1, 0(t1)
+  jr t1
+
+  CFI_ENDPROC
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(_setjmp))
+
+.comm _ZN14__interception14real_sigsetjmpE,8,8
+.globl ASM_SYMBOL_INTERCEPTOR(sigsetjmp)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
+ASM_SYMBOL_INTERCEPTOR(sigsetjmp):
+  CFI_STARTPROC
+
+  // Save frame pointer and return address register
+  addi sp, sp, -32
+  sd ra, 24(sp)
+  sd s0, 16(sp)
+  CFI_DEF_CFA_OFFSET (32)
+  CFI_OFFSET (1, -8)
+  CFI_OFFSET (8, -16)
+
+  // Adjust the SP for previous frame
+  addi s0, sp, 32
+  CFI_DEF_CFA_REGISTER (8)
+
+  // Save env parameter
+  sd a0, 8(sp)
+  sd a1, 0(sp)
+  CFI_OFFSET (10, -24)
+  CFI_OFFSET (11, -32)
+
+  // Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
+  addi  a0, s0, 0
+
+  // call tsan interceptor
+  call      ASM_SYMBOL(__tsan_setjmp)
+
+  // Restore env parameter
+  ld a0, 8(sp)
+  ld a1, 0(sp)
+  CFI_RESTORE (10)
+  CFI_RESTORE (11)
+
+  // Restore frame/link register
+  ld s0, 16(sp)
+  ld ra, 24(sp)
+  addi sp, sp, 32
+  CFI_RESTORE (8)
+  CFI_RESTORE (1)
+  CFI_DEF_CFA (2, 0)
+
+  // tail jump to libc setjmp
+  la t1, _ZN14__interception14real_sigsetjmpE
+  ld t1, 0(t1)
+  jr t1
+
+  CFI_ENDPROC
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
+
+.comm _ZN14__interception16real___sigsetjmpE,8,8
+.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
+ASM_SYMBOL_INTERCEPTOR(__sigsetjmp):
+  CFI_STARTPROC
+
+  // Save frame pointer and return address register
+  addi sp, sp, -32
+  sd ra, 24(sp)
+  sd s0, 16(sp)
+  CFI_DEF_CFA_OFFSET (32)
+  CFI_OFFSET (1, -8)
+  CFI_OFFSET (8, -16)
+
+  // Adjust the SP for previous frame
+  addi s0, sp, 32
+  CFI_DEF_CFA_REGISTER (8)
+
+  // Save env parameter
+  sd a0, 8(sp)
+  sd a1, 0(sp)
+  CFI_OFFSET (10, -24)
+  CFI_OFFSET (11, -32)
+
+  // Obtain SP, first argument to `void __tsan_setjmp(uptr sp)`
+  addi  a0, s0, 0
+
+  // call tsan interceptor
+  call      ASM_SYMBOL(__tsan_setjmp)
+
+  // Restore env parameter
+  ld a0, 8(sp)
+  ld a1, 0(sp)
+  CFI_RESTORE (10)
+  CFI_RESTORE (11)
+
+  // Restore frame/link register
+  ld s0, 16(sp)
+  ld ra, 24(sp)
+  addi sp, sp, 32
+  CFI_RESTORE (8)
+  CFI_RESTORE (1)
+  CFI_DEF_CFA (2, 0)
+
+  // tail jump to libc setjmp
+  la t1, _ZN14__interception16real___sigsetjmpE
+  ld t1, 0(t1)
+  jr t1
+
+  CFI_ENDPROC
+ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
diff --git a/compiler-rt/test/builtins/Unit/addtf3_test.c b/compiler-rt/test/builtins/Unit/addtf3_test.c
index fe2e2c80f6..e6986c236a 100644
--- a/compiler-rt/test/builtins/Unit/addtf3_test.c
+++ b/compiler-rt/test/builtins/Unit/addtf3_test.c
@@ -16,7 +16,7 @@ int test__addtf3(long double a, long double b,
                  uint64_t expectedHi, uint64_t expectedLo)
 {
     long double x = __addtf3(a, b);
-    int ret = compareResultLD(x, expectedHi, expectedLo);
+    int ret = compareResultF128(x, expectedHi, expectedLo);
 
     if (ret){
         printf("error in test__addtf3(%.20Lf, %.20Lf) = %.20Lf, "
diff --git a/compiler-rt/test/builtins/Unit/divtf3_test.c b/compiler-rt/test/builtins/Unit/divtf3_test.c
index 927d0b826f..da6465636e 100644
--- a/compiler-rt/test/builtins/Unit/divtf3_test.c
+++ b/compiler-rt/test/builtins/Unit/divtf3_test.c
@@ -15,7 +15,7 @@ int test__divtf3(long double a, long double b,
                  uint64_t expectedHi, uint64_t expectedLo)
 {
     long double x = __divtf3(a, b);
-    int ret = compareResultLD(x, expectedHi, expectedLo);
+    int ret = compareResultF128(x, expectedHi, expectedLo);
 
     if (ret){
         printf("error in test__divtf3(%.20Le, %.20Le) = %.20Le, "
diff --git a/compiler-rt/test/builtins/Unit/extenddftf2_test.c b/compiler-rt/test/builtins/Unit/extenddftf2_test.c
index 04a3468876..fcc030ca92 100644
--- a/compiler-rt/test/builtins/Unit/extenddftf2_test.c
+++ b/compiler-rt/test/builtins/Unit/extenddftf2_test.c
@@ -13,7 +13,7 @@ COMPILER_RT_ABI long double __extenddftf2(double a);
 int test__extenddftf2(double a, uint64_t expectedHi, uint64_t expectedLo)
 {
     long double x = __extenddftf2(a);
-    int ret = compareResultLD(x, expectedHi, expectedLo);
+    int ret = compareResultF128(x, expectedHi, expectedLo);
 
     if (ret){
         printf("error in test__extenddftf2(%f) = %.20Lf, "
diff --git a/compiler-rt/test/builtins/Unit/extendhftf2_test.c b/compiler-rt/test/builtins/Unit/extendhftf2_test.c
index 7d3ea3049e..5de1737909 100644
--- a/compiler-rt/test/builtins/Unit/extendhftf2_test.c
+++ b/compiler-rt/test/builtins/Unit/extendhftf2_test.c
@@ -12,7 +12,7 @@ COMPILER_RT_ABI long double __extendhftf2(TYPE_FP16 a);
 
 int test__extendhftf2(TYPE_FP16 a, uint64_t expectedHi, uint64_t expectedLo) {
   long double x = __extendhftf2(a);
-  int ret = compareResultLD(x, expectedHi, expectedLo);
+  int ret = compareResultF128(x, expectedHi, expectedLo);
 
   if (ret) {
     printf("error in test__extendhftf2(%#.4x) = %.20Lf, "
diff --git a/compiler-rt/test/builtins/Unit/extendsftf2_test.c b/compiler-rt/test/builtins/Unit/extendsftf2_test.c
index 19dd5b02c0..6ce9bd81a3 100644
--- a/compiler-rt/test/builtins/Unit/extendsftf2_test.c
+++ b/compiler-rt/test/builtins/Unit/extendsftf2_test.c
@@ -13,7 +13,7 @@ COMPILER_RT_ABI long double __extendsftf2(float a);
 int test__extendsftf2(float a, uint64_t expectedHi, uint64_t expectedLo)
 {
     long double x = __extendsftf2(a);
-    int ret = compareResultLD(x, expectedHi, expectedLo);
+    int ret = compareResultF128(x, expectedHi, expectedLo);
 
     if (ret)
     {
diff --git a/compiler-rt/test/builtins/Unit/extendxftf2_test.c b/compiler-rt/test/builtins/Unit/extendxftf2_test.c
new file mode 100644
index 0000000000..f521187543
--- /dev/null
+++ b/compiler-rt/test/builtins/Unit/extendxftf2_test.c
@@ -0,0 +1,74 @@
+// RUN: %clang_builtins %s %librt -o %t && %run %t
+// REQUIRES: librt_has_extendxftf2
+
+#include "int_lib.h"
+#include <stdio.h>
+
+#if __LDBL_MANT_DIG__ == 64 && defined(__x86_64__) &&                          \
+    (defined(__FLOAT128__) || defined(__SIZEOF_FLOAT128__))
+
+#include "fp_test.h"
+
+COMPILER_RT_ABI __float128 __extendxftf2(long double a);
+
+int test__extendxftf2(long double a, uint64_t expectedHi, uint64_t expectedLo) {
+  __float128 x = __extendxftf2(a);
+  int ret = compareResultF128(x, expectedHi, expectedLo);
+
+  if (ret) {
+    printf("error in __extendxftf2(%.20Lf) = %.20Lf, "
+           "expected %.20Lf\n",
+           a, x, fromRep128(expectedHi, expectedLo));
+  }
+  return ret;
+}
+
+char assumption_1[sizeof(long double) * CHAR_BIT == 128] = {0};
+
+#endif
+
+int main() {
+#if __LDBL_MANT_DIG__ == 64 && defined(__x86_64__) &&                          \
+    (defined(__FLOAT128__) || defined(__SIZEOF_FLOAT128__))
+  // qNaN
+  if (test__extendxftf2(makeQNaN80(), UINT64_C(0x7fff800000000000),
+                        UINT64_C(0x0)))
+    return 1;
+  // NaN
+  if (test__extendxftf2(makeNaN80(UINT64_C(0x3fffffffffffffff)),
+                        UINT64_C(0x7fff7fffffffffff),
+                        UINT64_C(0xfffe000000000000)))
+    return 1;
+  // inf
+  if (test__extendxftf2(makeInf80(), UINT64_C(0x7fff000000000000),
+                        UINT64_C(0x0)))
+    return 1;
+  // zero
+  if (test__extendxftf2(0.0, UINT64_C(0x0), UINT64_C(0x0)))
+    return 1;
+  if (test__extendxftf2(0x1.23456789abcdefp+5, UINT64_C(0x400423456789abcd),
+                        UINT64_C(0xf000000000000000)))
+    return 1;
+  if (test__extendxftf2(0x1.edcba987654321fp-9, UINT64_C(0x3ff6edcba9876543),
+                        UINT64_C(0x2000000000000000)))
+    return 1;
+  if (test__extendxftf2(0x1.23456789abcdefp+45, UINT64_C(0x402c23456789abcd),
+                        UINT64_C(0xf000000000000000)))
+    return 1;
+  if (test__extendxftf2(0x1.edcba987654321fp-45, UINT64_C(0x3fd2edcba9876543),
+                        UINT64_C(0x2000000000000000)))
+    return 1;
+  // denormal number
+  if (test__extendxftf2(1e-4932L, UINT64_C(0x00004c248f91e526),
+                        UINT64_C(0xafe0000000000000)))
+    return 1;
+  // denormal number
+  if (test__extendxftf2(2e-4932L, UINT64_C(0x000098491f23ca4d),
+                        UINT64_C(0x5fc0000000000000)))
+    return 1;
+#else
+  printf("skipped\n");
+
+#endif
+  return 0;
+}
diff --git a/compiler-rt/test/builtins/Unit/floatditf_test.c b/compiler-rt/test/builtins/Unit/floatditf_test.c
index 4d5da32ec2..fe7a5fd86a 100644
--- a/compiler-rt/test/builtins/Unit/floatditf_test.c
+++ b/compiler-rt/test/builtins/Unit/floatditf_test.c
@@ -17,7 +17,7 @@ COMPILER_RT_ABI long double __floatditf(di_int a);
 int test__floatditf(di_int a, uint64_t expectedHi, uint64_t expectedLo)
 {
     long double x = __floatditf(a);
-    int ret = compareResultLD(x, expectedHi, expectedLo);
+    int ret = compareResultF128(x, expectedHi, expectedLo);
 
     if (ret)
         printf("error in __floatditf(%Ld) = %.20Lf, "
diff --git a/compiler-rt/test/builtins/Unit/floatsitf_test.c b/compiler-rt/test/builtins/Unit/floatsitf_test.c
index 751a4a9b92..b6571b9ba2 100644
--- a/compiler-rt/test/builtins/Unit/floatsitf_test.c
+++ b/compiler-rt/test/builtins/Unit/floatsitf_test.c
@@ -13,7 +13,7 @@ COMPILER_RT_ABI long double __floatsitf(si_int a);
 int test__floatsitf(si_int a, uint64_t expectedHi, uint64_t expectedLo)
 {
     long double x = __floatsitf(a);
-    int ret = compareResultLD(x, expectedHi, expectedLo);
+    int ret = compareResultF128(x, expectedHi, expectedLo);
 
     if (ret)
     {
diff --git a/compiler-rt/test/builtins/Unit/floatunditf_test.c b/compiler-rt/test/builtins/Unit/floatunditf_test.c
index d44ae79341..8da78da976 100644
--- a/compiler-rt/test/builtins/Unit/floatunditf_test.c
+++ b/compiler-rt/test/builtins/Unit/floatunditf_test.c
@@ -17,7 +17,7 @@ COMPILER_RT_ABI long double __floatunditf(du_int a);
 int test__floatunditf(du_int a, uint64_t expectedHi, uint64_t expectedLo)
 {
     long double x = __floatunditf(a);
-    int ret = compareResultLD(x, expectedHi, expectedLo);
+    int ret = compareResultF128(x, expectedHi, expectedLo);
 
     if (ret)
         printf("error in __floatunditf(%Lu) = %.20Lf, "
diff --git a/compiler-rt/test/builtins/Unit/floatunsitf_test.c b/compiler-rt/test/builtins/Unit/floatunsitf_test.c
index f0a6c63eb8..b6b1ba0457 100644
--- a/compiler-rt/test/builtins/Unit/floatunsitf_test.c
+++ b/compiler-rt/test/builtins/Unit/floatunsitf_test.c
@@ -13,7 +13,7 @@ COMPILER_RT_ABI long double __floatunsitf(su_int a);
 int test__floatunsitf(su_int a, uint64_t expectedHi, uint64_t expectedLo)
 {
     long double x = __floatunsitf(a);
-    int ret = compareResultLD(x, expectedHi, expectedLo);
+    int ret = compareResultF128(x, expectedHi, expectedLo);
 
     if (ret){
         printf("error in test__floatunsitf(%u) = %.20Lf, "
diff --git a/compiler-rt/test/builtins/Unit/fp_test.h b/compiler-rt/test/builtins/Unit/fp_test.h
index e54dfc108e..f095ae0701 100644
--- a/compiler-rt/test/builtins/Unit/fp_test.h
+++ b/compiler-rt/test/builtins/Unit/fp_test.h
@@ -9,6 +9,18 @@
 #define TYPE_FP16 uint16_t
 #endif
 
+// TODO: Switch to using fp_lib.h once QUAD_PRECISION is available on x86_64.
+#if __LDBL_MANT_DIG__ == 113 ||                                                \
+    ((__LDBL_MANT_DIG__ == 64) && defined(__x86_64__) &&                       \
+     (defined(__FLOAT128__) || defined(__SIZEOF_FLOAT128__)))
+#if __LDBL_MANT_DIG__ == 113
+#define TYPE_FP128 long double
+#else
+#define TYPE_FP128 __float128
+#endif
+#define TEST_COMPILER_RT_HAS_FLOAT128
+#endif
+
 enum EXPECTED_RESULT {
     LESS_0, LESS_EQUAL_0, EQUAL_0, GREATER_0, GREATER_EQUAL_0, NEQUAL_0
 };
@@ -38,11 +50,10 @@ static inline double fromRep64(uint64_t x)
     return ret;
 }
 
-#if __LDBL_MANT_DIG__ == 113
-static inline long double fromRep128(uint64_t hi, uint64_t lo)
-{
+#ifdef TEST_COMPILER_RT_HAS_FLOAT128
+static inline TYPE_FP128 fromRep128(uint64_t hi, uint64_t lo) {
     __uint128_t x = ((__uint128_t)hi << 64) + lo;
-    long double ret;
+    TYPE_FP128 ret;
     memcpy(&ret, &x, 16);
     return ret;
 }
@@ -73,9 +84,8 @@ static inline uint64_t toRep64(double x)
     return ret;
 }
 
-#if __LDBL_MANT_DIG__ == 113
-static inline __uint128_t toRep128(long double x)
-{
+#ifdef TEST_COMPILER_RT_HAS_FLOAT128
+static inline __uint128_t toRep128(TYPE_FP128 x) {
     __uint128_t ret;
     memcpy(&ret, &x, 16);
     return ret;
@@ -136,25 +146,23 @@ static inline int compareResultD(double result,
     return 1;
 }
 
-#if __LDBL_MANT_DIG__ == 113
+#ifdef TEST_COMPILER_RT_HAS_FLOAT128
 // return 0 if equal
 // use two 64-bit integers instead of one 128-bit integer
 // because 128-bit integer constant can't be assigned directly
-static inline int compareResultLD(long double result,
-                                  uint64_t expectedHi,
-                                  uint64_t expectedLo)
-{
+static inline int compareResultF128(TYPE_FP128 result, uint64_t expectedHi,
+                                    uint64_t expectedLo) {
     __uint128_t rep = toRep128(result);
     uint64_t hi = rep >> 64;
     uint64_t lo = rep;
 
-    if (hi == expectedHi && lo == expectedLo){
+    if (hi == expectedHi && lo == expectedLo) {
         return 0;
     }
     // test other possible NaN representation(signal NaN)
-    else if (expectedHi == 0x7fff800000000000UL && expectedLo == 0x0UL){
+    else if (expectedHi == 0x7fff800000000000UL && expectedLo == 0x0UL) {
         if ((hi & 0x7fff000000000000UL) == 0x7fff000000000000UL &&
-            ((hi & 0xffffffffffffUL) > 0 || lo > 0)){
+            ((hi & 0xffffffffffffUL) > 0 || lo > 0)) {
             return 0;
         }
     }
@@ -232,9 +240,45 @@ static inline double makeQNaN64(void)
     return fromRep64(0x7ff8000000000000UL);
 }
 
-#if __LDBL_MANT_DIG__ == 113
-static inline long double makeQNaN128(void)
-{
+#if __LDBL_MANT_DIG__ == 64 && defined(__x86_64__)
+static inline long double F80FromRep128(uint64_t hi, uint64_t lo) {
+    __uint128_t x = ((__uint128_t)hi << 64) + lo;
+    long double ret;
+    memcpy(&ret, &x, 16);
+    return ret;
+}
+
+static inline __uint128_t F80ToRep128(long double x) {
+    __uint128_t ret;
+    memcpy(&ret, &x, 16);
+    return ret;
+}
+
+static inline int compareResultF80(long double result, uint64_t expectedHi,
+                                   uint64_t expectedLo) {
+    __uint128_t rep = F80ToRep128(result);
+    // F80 occupies the lower 80 bits of __uint128_t.
+    uint64_t hi = (rep >> 64) & ((1UL << (80 - 64)) - 1);
+    uint64_t lo = rep;
+    return !(hi == expectedHi && lo == expectedLo);
+}
+
+static inline long double makeQNaN80(void) {
+    return F80FromRep128(0x7fffUL, 0xc000000000000000UL);
+}
+
+static inline long double makeNaN80(uint64_t rand) {
+    return F80FromRep128(0x7fffUL,
+                         0x8000000000000000 | (rand & 0x3fffffffffffffff));
+}
+
+static inline long double makeInf80(void) {
+    return F80FromRep128(0x7fffUL, 0x8000000000000000UL);
+}
+#endif
+
+#ifdef TEST_COMPILER_RT_HAS_FLOAT128
+static inline TYPE_FP128 makeQNaN128(void) {
     return fromRep128(0x7fff800000000000UL, 0x0UL);
 }
 #endif
@@ -254,9 +298,8 @@ static inline double makeNaN64(uint64_t rand)
     return fromRep64(0x7ff0000000000000UL | (rand & 0xfffffffffffffUL));
 }
 
-#if __LDBL_MANT_DIG__ == 113
-static inline long double makeNaN128(uint64_t rand)
-{
+#ifdef TEST_COMPILER_RT_HAS_FLOAT128
+static inline TYPE_FP128 makeNaN128(uint64_t rand) {
     return fromRep128(0x7fff000000000000UL | (rand & 0xffffffffffffUL), 0x0UL);
 }
 #endif
@@ -286,14 +329,12 @@ static inline double makeNegativeInf64(void)
     return fromRep64(0xfff0000000000000UL);
 }
 
-#if __LDBL_MANT_DIG__ == 113
-static inline long double makeInf128(void)
-{
+#ifdef TEST_COMPILER_RT_HAS_FLOAT128
+static inline TYPE_FP128 makeInf128(void) {
     return fromRep128(0x7fff000000000000UL, 0x0UL);
 }
 
-static inline long double makeNegativeInf128(void)
-{
+static inline TYPE_FP128 makeNegativeInf128(void) {
     return fromRep128(0xffff000000000000UL, 0x0UL);
 }
 #endif
diff --git a/compiler-rt/test/builtins/Unit/multf3_test.c b/compiler-rt/test/builtins/Unit/multf3_test.c
index 3bf6ab24ce..543b55899c 100644
--- a/compiler-rt/test/builtins/Unit/multf3_test.c
+++ b/compiler-rt/test/builtins/Unit/multf3_test.c
@@ -15,7 +15,7 @@ int test__multf3(long double a, long double b,
                  uint64_t expectedHi, uint64_t expectedLo)
 {
     long double x = __multf3(a, b);
-    int ret = compareResultLD(x, expectedHi, expectedLo);
+    int ret = compareResultF128(x, expectedHi, expectedLo);
 
     if (ret){
         printf("error in test__multf3(%.20Lf, %.20Lf) = %.20Lf, "
diff --git a/compiler-rt/test/builtins/Unit/subtf3_test.c b/compiler-rt/test/builtins/Unit/subtf3_test.c
index 377ae95a9a..724fa4820d 100644
--- a/compiler-rt/test/builtins/Unit/subtf3_test.c
+++ b/compiler-rt/test/builtins/Unit/subtf3_test.c
@@ -16,7 +16,7 @@ int test__subtf3(long double a, long double b,
                  uint64_t expectedHi, uint64_t expectedLo)
 {
     long double x = __subtf3(a, b);
-    int ret = compareResultLD(x, expectedHi, expectedLo);
+    int ret = compareResultF128(x, expectedHi, expectedLo);
 
     if (ret){
         printf("error in test__subtf3(%.20Lf, %.20Lf) = %.20Lf, "
diff --git a/compiler-rt/test/builtins/Unit/trunctfxf2_test.c b/compiler-rt/test/builtins/Unit/trunctfxf2_test.c
new file mode 100644
index 0000000000..53024ef139
--- /dev/null
+++ b/compiler-rt/test/builtins/Unit/trunctfxf2_test.c
@@ -0,0 +1,97 @@
+// RUN: %clang_builtins %s %librt -o %t && %run %t
+// REQUIRES: librt_has_trunctfxf2
+
+#include "int_lib.h"
+#include <stdio.h>
+
+#if __LDBL_MANT_DIG__ == 64 && defined(__x86_64__) &&                          \
+    (defined(__FLOAT128__) || defined(__SIZEOF_FLOAT128__))
+
+#include "fp_test.h"
+
+COMPILER_RT_ABI long double __trunctfxf2(__float128 a);
+
+int test__trunctfxf2(__float128 a, uint64_t expectedHi, uint64_t expectedLo) {
+  long double x = __trunctfxf2(a);
+  int ret = compareResultF80(x, expectedHi, expectedLo);
+  ;
+  if (ret) {
+    printf("error in __trunctfxf2(%.20Lf) = %.20Lf, "
+           "expected %.20Lf\n",
+           a, x, fromRep128(expectedHi, expectedLo));
+  }
+  return ret;
+}
+
+char assumption_1[sizeof(long double) * CHAR_BIT == 128] = {0};
+
+#endif
+
+int main() {
+#if __LDBL_MANT_DIG__ == 64 && defined(__x86_64__) &&                          \
+    (defined(__FLOAT128__) || defined(__SIZEOF_FLOAT128__))
+  // qNaN
+  if (test__trunctfxf2(makeQNaN128(), UINT64_C(0x7FFF),
+                       UINT64_C(0xC000000000000000)))
+    return 1;
+  // NaN
+  if (test__trunctfxf2(makeNaN128(UINT64_C(0x810000000000)), UINT64_C(0x7FFF),
+                       UINT64_C(0xC080000000000000)))
+    return 1;
+  // inf
+  if (test__trunctfxf2(makeInf128(), UINT64_C(0x7FFF),
+                       UINT64_C(0x8000000000000000)))
+    return 1;
+  // zero
+  if (test__trunctfxf2(0.0Q, UINT64_C(0x0), UINT64_C(0x0)))
+    return 1;
+  if (test__trunctfxf2(0x1.af23456789bbaaab347645365cdep+5L, UINT64_C(0x4004),
+                       UINT64_C(0xd791a2b3c4ddd556)))
+    return 1;
+  if (test__trunctfxf2(0x1.dedafcff354b6ae9758763545432p-9L, UINT64_C(0x3ff6),
+                       UINT64_C(0xef6d7e7f9aa5b575)))
+    return 1;
+  if (test__trunctfxf2(0x1.2f34dd5f437e849b4baab754cdefp+4534L,
+                       UINT64_C(0x51b5), UINT64_C(0x979a6eafa1bf424e)))
+    return 1;
+  if (test__trunctfxf2(0x1.edcbff8ad76ab5bf46463233214fp-435L, UINT64_C(0x3e4c),
+                       UINT64_C(0xf6e5ffc56bb55ae0)))
+    return 1;
+
+  // Test rounding near halfway.
+  __float128 halfwayPlus =
+      fromRep128(UINT64_C(0x7ffa000000000000),
+                 ((UINT64_C(1) << (112 - 63 - 1)) + UINT64_C(1)));
+  if (test__trunctfxf2(halfwayPlus, UINT64_C(0x7ffa),
+                       UINT64_C(0x8000000000000001)))
+    return 1;
+  __float128 halfwayExactOdd = fromRep128(
+      UINT64_C(0x7ffa000000000000),
+      ((UINT64_C(1) << (112 - 63)) + (UINT64_C(1) << (112 - 63 - 1))));
+  if (test__trunctfxf2(halfwayExactOdd, UINT64_C(0x7ffa),
+                       UINT64_C(0x8000000000000002)))
+    return 1;
+  __float128 halfwayExactEven =
+      fromRep128(UINT64_C(0x7ffa000000000000), (UINT64_C(1) << (112 - 63 - 1)));
+  if (test__trunctfxf2(halfwayExactEven, UINT64_C(0x7ffa),
+                       UINT64_C(0x8000000000000000)))
+    return 1;
+  __float128 halfwayRoundingWillChangeExponent =
+      fromRep128(UINT64_C(0x7ffaffffffffffff), UINT64_C(0xffff000000000001));
+  if (test__trunctfxf2(halfwayRoundingWillChangeExponent, UINT64_C(0x7ffb),
+                       UINT64_C(0x8000000000000000)))
+    return 1;
+
+  // denormal number
+  if (test__trunctfxf2(1e-4932Q, UINT64_C(0), UINT64_C(0x261247c8f29357f0)))
+    return 1;
+  // denormal number
+  if (test__trunctfxf2(2e-4932Q, UINT64_C(0), UINT64_C(0x4c248f91e526afe0)))
+    return 1;
+
+#else
+  printf("skipped\n");
+
+#endif
+  return 0;
+}
diff --git a/compiler-rt/test/hwasan/TestCases/bcmp.cpp b/compiler-rt/test/hwasan/TestCases/bcmp.cpp
new file mode 100644
index 0000000000..a83147b0f3
--- /dev/null
+++ b/compiler-rt/test/hwasan/TestCases/bcmp.cpp
@@ -0,0 +1,28 @@
+// RUN: %clangxx_hwasan -O0 %s -o %t && not %run %t 2>&1 | FileCheck %s
+// RUN: %clangxx_hwasan -O1 %s -o %t && not %run %t 2>&1 | FileCheck %s
+// RUN: %clangxx_hwasan -O2 %s -o %t && not %run %t 2>&1 | FileCheck %s
+// RUN: %clangxx_hwasan -O3 %s -o %t && not %run %t 2>&1 | FileCheck %s
+// REQUIRES: !android
+
+#include <sanitizer/hwasan_interface.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+int main(int argc, char **argv) {
+  __hwasan_enable_allocator_tagging();
+  char a[] = {static_cast<char>(argc), 2, 3, 4};
+  int size = sizeof(a);
+  char *p = (char *)malloc(size);
+  memcpy(p, a, size);
+  free(p);
+  return bcmp(p, a, size);
+  // CHECK: HWAddressSanitizer: tag-mismatch on address
+  // CHECK: READ of size 4
+  // CHECK: #{{[[:digit:]]+}} 0x{{[[:xdigit:]]+}} in main {{.*}}bcmp.cpp:[[@LINE-3]]
+  // CHECK: Cause: use-after-free
+  // CHECK: freed by thread
+  // CHECK: #{{[[:digit:]]+}} 0x{{[[:xdigit:]]+}} in main {{.*}}bcmp.cpp:[[@LINE-7]]
+  // CHECK: previously allocated by thread
+  // CHECK: #{{[[:digit:]]+}} 0x{{[[:xdigit:]]+}} in main {{.*}}bcmp.cpp:[[@LINE-11]]
+}
diff --git a/compiler-rt/test/hwasan/TestCases/deep-recursion.c b/compiler-rt/test/hwasan/TestCases/deep-recursion.c
index fde8a0db5a..764ec56923 100644
--- a/compiler-rt/test/hwasan/TestCases/deep-recursion.c
+++ b/compiler-rt/test/hwasan/TestCases/deep-recursion.c
@@ -17,7 +17,10 @@
 // Stack histories are currently not recorded on x86.
 // XFAIL: target=x86_64{{.*}}
 
+#include <assert.h>
+#include <sanitizer/hwasan_interface.h>
 #include <stdlib.h>
+
 // At least -O1 is needed for this function to not have a stack frame on
 // AArch64.
 void USE(void *x) { // pretend_to_do_something(void *x)
@@ -26,7 +29,24 @@ void USE(void *x) { // pretend_to_do_something(void *x)
 
 volatile int four = 4;
 
-__attribute__((noinline)) void OOB() { int x[4]; x[four] = 0; USE(&x[0]); }
+__attribute__((noinline)) void OOB() {
+  int x[4];
+  int y[4];
+
+  // Tags for stack-allocated variables can occasionally be zero, resulting in
+  // a false negative for this test. The tag allocation algorithm is not easy
+  // to fix, hence we work around it: if the tag is zero, we use the
+  // neighboring variable instead, which must have a different (hence non-zero)
+  // tag.
+  if (__hwasan_tag_pointer(x, 0) == x) {
+    assert(__hwasan_tag_pointer(y, 0) != y);
+    y[four] = 0;
+  } else {
+    x[four] = 0;
+  }
+  USE(&x[0]);
+  USE(&y[0]);
+}
 __attribute__((noinline)) void FUNC1() { int x; USE(&x); OOB(); }
 __attribute__((noinline)) void FUNC2() { int x; USE(&x); FUNC1(); }
 __attribute__((noinline)) void FUNC3() { int x; USE(&x); FUNC2(); }
diff --git a/compiler-rt/test/hwasan/TestCases/memcmp.cpp b/compiler-rt/test/hwasan/TestCases/memcmp.cpp
index c6a2b42b54..5f8a93f62a 100644
--- a/compiler-rt/test/hwasan/TestCases/memcmp.cpp
+++ b/compiler-rt/test/hwasan/TestCases/memcmp.cpp
@@ -11,8 +11,8 @@
 int main(int argc, char **argv) {
   __hwasan_enable_allocator_tagging();
   char a[] = {static_cast<char>(argc), 2, 3, 4};
-  volatile int size = sizeof(a);
-  char *volatile p = (char *)malloc(size);
+  int size = sizeof(a);
+  char *p = (char *)malloc(size);
   memcpy(p, a, size);
   free(p);
   return memcmp(p, a, size);
diff --git a/compiler-rt/test/hwasan/TestCases/stack-history-length.c b/compiler-rt/test/hwasan/TestCases/stack-history-length.c
index d997677b4e..6b1274892c 100644
--- a/compiler-rt/test/hwasan/TestCases/stack-history-length.c
+++ b/compiler-rt/test/hwasan/TestCases/stack-history-length.c
@@ -11,6 +11,8 @@
 // Stack histories are currently not recorded on x86.
 // XFAIL: target=x86_64{{.*}}
 
+#include <assert.h>
+#include <sanitizer/hwasan_interface.h>
 #include <stdlib.h>
 
 void USE(void *x) { // pretend_to_do_something(void *x)
@@ -20,7 +22,24 @@ void USE(void *x) { // pretend_to_do_something(void *x)
 volatile int four = 4;
 __attribute__((noinline)) void FUNC0() { int x[4]; USE(&x[0]); }
 __attribute__((noinline)) void FUNC() { int x[4]; USE(&x[0]); }
-__attribute__((noinline)) void OOB() { int x[4]; x[four] = 0; USE(&x[0]); }
+__attribute__((noinline)) void OOB() {
+  int x[4];
+  int y[4];
+  // With -hwasan-generate-tags-with-calls=false, stack tags can occasionally
+  // be zero, leading to a false negative
+  // (https://github.com/llvm/llvm-project/issues/69221). Work around it by
+  // using the neighboring variable, which is guaranteed by
+  // -hwasan-generate-tags-with-calls=false to have a different (hence
+  // non-zero) tag.
+  if (__hwasan_tag_pointer(x, 0) == x) {
+    assert(__hwasan_tag_pointer(y, 0) != y);
+    y[four] = 0;
+  } else {
+    x[four] = 0;
+  }
+  USE(&x[0]);
+  USE(&y[0]);
+}
 
 int main(int argc, char **argv) {
   int X = argc == 2 ? atoi(argv[1]) : 10;
diff --git a/compiler-rt/test/hwasan/TestCases/stack-uar.c b/compiler-rt/test/hwasan/TestCases/stack-uar.c
index b4a8173510..48440a47d5 100644
--- a/compiler-rt/test/hwasan/TestCases/stack-uar.c
+++ b/compiler-rt/test/hwasan/TestCases/stack-uar.c
@@ -9,14 +9,29 @@
 // Stack histories currently are not recorded on x86.
 // XFAIL: target=x86_64{{.*}}
 
+#include <assert.h>
+#include <sanitizer/hwasan_interface.h>
+
 void USE(void *x) { // pretend_to_do_something(void *x)
   __asm__ __volatile__("" : : "r" (x) : "memory");
 }
 
 __attribute__((noinline))
 char *buggy() {
-  char zzz[0x1000];
-  char *volatile p = zzz;
+  char zzz[0x800];
+  char yyy[0x800];
+  // Tags for stack-allocated variables can occasionally be zero, resulting in
+  // a false negative for this test. The tag allocation algorithm is not easy
+  // to fix, hence we work around it: if the tag is zero, we use the
+  // neighboring variable instead, which must have a different (hence non-zero)
+  // tag.
+  char *volatile p;
+  if (__hwasan_tag_pointer(zzz, 0) == zzz) {
+    assert(__hwasan_tag_pointer(yyy, 0) != yyy);
+    p = yyy;
+  } else {
+    p = zzz;
+  }
   return p;
 }
 
@@ -35,7 +50,7 @@ int main() {
   // CHECK: Cause: stack tag-mismatch
   // CHECK: is located in stack of thread
   // CHECK: Potentially referenced stack objects:
-  // CHECK-NEXT: zzz in buggy {{.*}}stack-uar.c:[[@LINE-20]]
+  // CHECK-NEXT: {{zzz|yyy}} in buggy {{.*}}stack-uar.c:
   // CHECK-NEXT: Memory tags around the buggy address
 
   // NOSYM: Previously allocated frames:
diff --git a/compiler-rt/test/hwasan/TestCases/stack-uas.c b/compiler-rt/test/hwasan/TestCases/stack-uas.c
index 53d51ee25d..4455e59100 100644
--- a/compiler-rt/test/hwasan/TestCases/stack-uas.c
+++ b/compiler-rt/test/hwasan/TestCases/stack-uas.c
@@ -13,6 +13,10 @@
 // Stack histories currently are not recorded on x86.
 // XFAIL: target=x86_64{{.*}}
 
+#include <assert.h>
+#include <sanitizer/hwasan_interface.h>
+#include <stdio.h>
+
 void USE(void *x) { // pretend_to_do_something(void *x)
   __asm__ __volatile__(""
                        :
@@ -36,8 +40,20 @@ __attribute__((noinline)) void Unrelated3() {
 __attribute__((noinline)) char buggy() {
   char *volatile p;
   {
-    char zzz[0x1000] = {};
-    p = zzz;
+    char zzz[0x800] = {};
+    char yyy[0x800] = {};
+    // With -hwasan-generate-tags-with-calls=false, stack tags can occasionally
+    // be zero, leading to a false negative
+    // (https://github.com/llvm/llvm-project/issues/69221). Work around it by
+    // using the neighboring variable, which is guaranteed by
+    // -hwasan-generate-tags-with-calls=false to have a different (hence
+    // non-zero) tag.
+    if (__hwasan_tag_pointer(zzz, 0) == zzz) {
+      assert(__hwasan_tag_pointer(yyy, 0) != yyy);
+      p = yyy;
+    } else {
+      p = zzz;
+    }
   }
   return *p;
 }
@@ -53,7 +69,7 @@ int main() {
   // CHECK: Cause: stack tag-mismatch
   // CHECK: is located in stack of thread
   // CHECK: Potentially referenced stack objects:
-  // CHECK-NEXT: zzz in buggy {{.*}}stack-uas.c:[[@LINE-17]]
+  // CHECK-NEXT: {{zzz|yyy}} in buggy {{.*}}stack-uas.c:
   // CHECK-NEXT: Memory tags around the buggy address
 
   // NOSYM: Previously allocated frames:
diff --git a/compiler-rt/test/msan/dladdr1_test.c b/compiler-rt/test/msan/dladdr1_test.c
index dc5f6fdf9f..534f5f9cc1 100644
--- a/compiler-rt/test/msan/dladdr1_test.c
+++ b/compiler-rt/test/msan/dladdr1_test.c
@@ -60,7 +60,7 @@ int main(int argc, char *argv[]) {
       fflush(stdout);
       map_ptr = map_ptr->l_next;
     }
-    // CHECK: libc.so
+    // CHECK: libc{{[\-]*.*}}.so
     // CHECK: dladdr1_test
   }
 
diff --git a/compiler-rt/test/orc/TestCases/Linux/ppc64/Inputs/trivial-tls-main.cpp b/compiler-rt/test/orc/TestCases/Linux/ppc64/Inputs/trivial-tls-main.cpp
new file mode 100644
index 0000000000..d6757fdd41
--- /dev/null
+++ b/compiler-rt/test/orc/TestCases/Linux/ppc64/Inputs/trivial-tls-main.cpp
@@ -0,0 +1,30 @@
+#include <fstream>
+#include <iostream>
+#include <string>
+
+thread_local int x = 0;
+thread_local int y = 1;
+thread_local int z = -1;
+
+extern int TestPOWER10();
+
+int Test() { return x + y + z; }
+
+static bool CPUModelIsPOWER10() {
+  std::string line;
+  std::ifstream cpuinfo("/proc/cpuinfo", std::ios::in);
+  if (!cpuinfo.is_open())
+    return false;
+  while (std::getline(cpuinfo, line)) {
+    if (line.find("cpu") != std::string::npos &&
+        line.find("POWER10") != std::string::npos)
+      return true;
+  }
+  return false;
+}
+
+int main() {
+  if (CPUModelIsPOWER10())
+    return TestPOWER10();
+  return Test();
+}
diff --git a/compiler-rt/test/orc/TestCases/Linux/ppc64/Inputs/trivial-tls-pwr10.cpp b/compiler-rt/test/orc/TestCases/Linux/ppc64/Inputs/trivial-tls-pwr10.cpp
new file mode 100644
index 0000000000..a6fb3088af
--- /dev/null
+++ b/compiler-rt/test/orc/TestCases/Linux/ppc64/Inputs/trivial-tls-pwr10.cpp
@@ -0,0 +1,5 @@
+extern thread_local int x;
+extern thread_local int y;
+extern thread_local int z;
+
+int __attribute__((target("arch=pwr10"))) TestPOWER10() { return x + y + z; }
diff --git a/compiler-rt/test/orc/TestCases/Linux/ppc64/trivial-tls-pwr10.test b/compiler-rt/test/orc/TestCases/Linux/ppc64/trivial-tls-pwr10.test
new file mode 100644
index 0000000000..93561b1645
--- /dev/null
+++ b/compiler-rt/test/orc/TestCases/Linux/ppc64/trivial-tls-pwr10.test
@@ -0,0 +1,9 @@
+// RUN: rm -rf %t && mkdir -p %t
+// RUN: %clangxx -fPIC -c -o %t/main.o %S/Inputs/trivial-tls-main.cpp
+// RUN: %clangxx -fPIC -c -o %t/pwr10.o %S/Inputs/trivial-tls-pwr10.cpp
+// RUN: %llvm_jitlink %t/main.o %t/pwr10.o
+// FIXME: We seperate pwr10 code from main object file due to currrent
+// implementation only supports one PLT stub for the same symbol.
+// For example, `bl __tls_get_addr` in one object file has only one PLT stub,
+// however we need another different PLT stub for `bl __tls_get_addr@notoc`
+// whose target symbol is also `__tls_get_addr`.
diff --git a/compiler-rt/test/profile/Windows/coverage-weak-lld.cpp b/compiler-rt/test/profile/Windows/coverage-weak-lld.cpp
index 338be8e979..e239d80964 100644
--- a/compiler-rt/test/profile/Windows/coverage-weak-lld.cpp
+++ b/compiler-rt/test/profile/Windows/coverage-weak-lld.cpp
@@ -14,7 +14,7 @@
 // RUN: llvm-profdata show %t.profraw --all-functions | FileCheck %s --check-prefix=PROFILE1
 
 /// link.exe does not support weak overridding weak.
-// RUN: %clang_profgen -fcoverage-mapping -fuse-ld=lld -Wl,-lldmingw,-opt:ref %t0.o %t2.o -o %t2
+// RUN: %clang_profgen -fcoverage-mapping -fuse-ld=lld -Wl,-lld-allow-duplicate-weak,-opt:ref %t0.o %t2.o -o %t2
 // RUN: env LLVM_PROFILE_FILE=%t.profraw %run %t2 | FileCheck %s --check-prefix=CHECK2
 // RUN: llvm-profdata show %t.profraw --all-functions | FileCheck %s --check-prefix=PROFILE2
 
@@ -30,7 +30,7 @@
 // RUN: env LLVM_PROFILE_FILE=%t.profraw %run %t1 | FileCheck %s --check-prefix=CHECK1
 // RUN: llvm-profdata show %t.profraw --all-functions | FileCheck %s --check-prefix=PROFILE1
 
-// RUN: %clang_profgen -fcoverage-mapping -fuse-ld=lld -Wl,-lldmingw,-opt:ref %t0.o %t2.o -o %t2
+// RUN: %clang_profgen -fcoverage-mapping -fuse-ld=lld -Wl,-lld-allow-duplicate-weak,-opt:ref %t0.o %t2.o -o %t2
 // RUN: env LLVM_PROFILE_FILE=%t.profraw %run %t2 | FileCheck %s --check-prefix=CHECK2
 // RUN: llvm-profdata show %t.profraw --all-functions | FileCheck %s --check-prefix=PROFILE2
 
diff --git a/compiler-rt/test/tsan/map32bit.cpp b/compiler-rt/test/tsan/map32bit.cpp
index e8bac22647..9c0760f54b 100644
--- a/compiler-rt/test/tsan/map32bit.cpp
+++ b/compiler-rt/test/tsan/map32bit.cpp
@@ -13,6 +13,7 @@
 // XFAIL: target=powerpc64{{.*}}
 // XFAIL: target=s390x{{.*}}
 // XFAIL: target=loongarch64{{.*}}
+// XFAIL: target=riscv64{{.*}}
 
 // MAP_32BIT doesn't exist on OS X and NetBSD.
 // UNSUPPORTED: darwin,target={{.*netbsd.*}}
diff --git a/compiler-rt/test/tsan/mmap_large.cpp b/compiler-rt/test/tsan/mmap_large.cpp
index 85ebe7f76b..a6aca720bf 100644
--- a/compiler-rt/test/tsan/mmap_large.cpp
+++ b/compiler-rt/test/tsan/mmap_large.cpp
@@ -17,7 +17,8 @@
 int main() {
 #ifdef __x86_64__
   const size_t kLog2Size = 39;
-#elif defined(__mips64) || defined(__aarch64__) || defined(__loongarch_lp64)
+#elif defined(__mips64) || defined(__aarch64__) ||                             \
+    defined(__loongarch_lp64) || (defined(__riscv) && __riscv_xlen == 64)
   const size_t kLog2Size = 32;
 #elif defined(__powerpc64__)
   const size_t kLog2Size = 39;
diff --git a/compiler-rt/test/tsan/test.h b/compiler-rt/test/tsan/test.h
index 7406318243..6fd5524658 100644
--- a/compiler-rt/test/tsan/test.h
+++ b/compiler-rt/test/tsan/test.h
@@ -76,6 +76,8 @@ unsigned long long monotonic_clock_ns() {
 const int kPCInc = 1;
 #elif defined(__sparc__) || defined(__mips__)
 const int kPCInc = 8;
+#elif defined(__riscv) && __riscv_xlen == 64
+const int kPCInc = 2;
 #else
 const int kPCInc = 4;
 #endif
diff --git a/flang/docs/Extensions.md b/flang/docs/Extensions.md
index 4800399117..373f18e1e2 100644
--- a/flang/docs/Extensions.md
+++ b/flang/docs/Extensions.md
@@ -613,6 +613,21 @@ end module
   associated objects and do not elicit errors about improper redeclarations
   of implicitly typed entities.
 
+* Standard Fortran allows forward references to derived types, which
+  can lead to ambiguity when combined with host association.
+  Some Fortran compilers resolve the type name to the host type,
+  others to the forward-referenced local type; this compiler diagnoses
+  an error.
+```
+module m
+  type ambiguous; integer n; end type
+ contains
+  subroutine s
+    type(ambiguous), pointer :: ptr
+    type ambiguous; real a; end type
+  end
+end
+```
 
 ## De Facto Standard Features
 
diff --git a/flang/docs/GettingStarted.md b/flang/docs/GettingStarted.md
index 3b3f14d4fe..c6cd18db2e 100644
--- a/flang/docs/GettingStarted.md
+++ b/flang/docs/GettingStarted.md
@@ -53,6 +53,7 @@ First, create the root directory and `cd` into it.
 ```bash
 mkdir root
 cd root
+```
 
 Now clone the source:
 ```bash
@@ -233,6 +234,7 @@ cmake \
   -DCMAKE_CUDA_COMPILER=nvcc \
   -DCMAKE_CUDA_HOST_COMPILER=clang++ \
   ../runtime/
+
 make -j FortranRuntime
 ```
 
@@ -275,7 +277,7 @@ Normal `make -j check-flang` will work with such CMake configuration.
 ##### OpenMP target offload build
 Only Clang compiler is currently supported.
 
-```
+```bash
 cd llvm-project/flang
 rm -rf build_flang_runtime
 mkdir build_flang_runtime
@@ -287,6 +289,7 @@ cmake \
   -DCMAKE_CXX_COMPILER=clang++ \
   -DFLANG_OMP_DEVICE_ARCHITECTURES="all" \
   ../runtime/
+
 make -j FortranRuntime
 ```
 
@@ -338,6 +341,7 @@ and the GCC library and tools that were used to build clang++.
 
 CXX should include the full path to clang++
 or clang++ should be found on your PATH.
+
 ```bash
 export CXX=clang++
 ```
diff --git a/flang/docs/OpenACC.md b/flang/docs/OpenACC.md
index 80258041a6..e29c5f89f9 100644
--- a/flang/docs/OpenACC.md
+++ b/flang/docs/OpenACC.md
@@ -21,3 +21,5 @@ local:
 * `!$acc end loop` does not trigger a parsing error and is just ignored.
 * The restriction on `!$acc data` required clauses is emitted as a portability
   warning instead of an error as other compiler accepts it.
+* The `if` clause accepts scalar integer expression in addition to scalar
+  logical expression.
diff --git a/flang/include/flang/Common/real.h b/flang/include/flang/Common/real.h
index 036f665d3d..50aab7d89a 100644
--- a/flang/include/flang/Common/real.h
+++ b/flang/include/flang/Common/real.h
@@ -63,6 +63,10 @@ static constexpr int MaxDecimalConversionDigits(int binaryPrecision) {
   }
 }
 
+static constexpr int MaxHexadecimalConversionDigits(int binaryPrecision) {
+  return binaryPrecision >= 0 ? (binaryPrecision + 3) / 4 : binaryPrecision;
+}
+
 static constexpr int RealKindForPrecision(int binaryPrecision) {
   switch (binaryPrecision) {
   case 8: // IEEE single (truncated): 1+8+7 with implicit bit
@@ -132,6 +136,9 @@ template <int BINARY_PRECISION> class RealDetails {
   static constexpr int maxDecimalConversionDigits{
       MaxDecimalConversionDigits(binaryPrecision)};
 
+  static constexpr int maxHexadecimalConversionDigits{
+      MaxHexadecimalConversionDigits(binaryPrecision)};
+
   static_assert(binaryPrecision > 0);
   static_assert(exponentBits > 1);
   static_assert(exponentBits <= 15);
diff --git a/flang/include/flang/Decimal/binary-floating-point.h b/flang/include/flang/Decimal/binary-floating-point.h
index 28346e7182..b9346a8585 100644
--- a/flang/include/flang/Decimal/binary-floating-point.h
+++ b/flang/include/flang/Decimal/binary-floating-point.h
@@ -21,10 +21,19 @@
 
 namespace Fortran::decimal {
 
+enum FortranRounding {
+  RoundNearest, /* RN and RP */
+  RoundUp, /* RU */
+  RoundDown, /* RD */
+  RoundToZero, /* RZ - no rounding */
+  RoundCompatible, /* RC: like RN, but ties go away from 0 */
+};
+
 template <int BINARY_PRECISION>
 class BinaryFloatingPointNumber : public common::RealDetails<BINARY_PRECISION> {
 public:
   using Details = common::RealDetails<BINARY_PRECISION>;
+  using Details::binaryPrecision;
   using Details::bits;
   using Details::decimalPrecision;
   using Details::decimalRange;
@@ -33,6 +42,7 @@ class BinaryFloatingPointNumber : public common::RealDetails<BINARY_PRECISION> {
   using Details::isImplicitMSB;
   using Details::maxDecimalConversionDigits;
   using Details::maxExponent;
+  using Details::maxHexadecimalConversionDigits;
   using Details::significandBits;
 
   using RawType = common::HostUnsignedIntType<bits>;
@@ -120,6 +130,55 @@ class BinaryFloatingPointNumber : public common::RealDetails<BINARY_PRECISION> {
     InsertExplicitMSB();
   }
 
+  static constexpr BinaryFloatingPointNumber Infinity(bool isNegative) {
+    RawType result{RawType{maxExponent} << significandBits};
+    if (isNegative) {
+      result |= RawType{1} << (bits - 1);
+    }
+    return BinaryFloatingPointNumber{result};
+  }
+
+  // Returns true when the result is exact
+  constexpr bool RoundToBits(int keepBits, enum FortranRounding mode) {
+    if (IsNaN() || IsInfinite() || keepBits >= binaryPrecision) {
+      return true;
+    }
+    int lostBits{binaryPrecision - keepBits};
+    RawType lostMask{static_cast<RawType>((RawType{1} << lostBits) - 1)};
+    if (RawType lost{static_cast<RawType>(raw_ & lostMask)}; lost != 0) {
+      bool increase{false};
+      switch (mode) {
+      case RoundNearest:
+        if (lost >> (lostBits - 1) != 0) { // >= tie
+          if ((lost & (lostMask >> 1)) != 0) {
+            increase = true; // > tie
+          } else {
+            increase = ((raw_ >> lostBits) & 1) != 0; // tie to even
+          }
+        }
+        break;
+      case RoundUp:
+        increase = !IsNegative();
+        break;
+      case RoundDown:
+        increase = IsNegative();
+        break;
+      case RoundToZero:
+        break;
+      case RoundCompatible:
+        increase = lost >> (lostBits - 1) != 0; // >= tie
+        break;
+      }
+      if (increase) {
+        raw_ |= lostMask;
+        Next();
+      }
+      return false; // inexact
+    } else {
+      return true; // exact
+    }
+  }
+
 private:
   constexpr void RemoveExplicitMSB() {
     if constexpr (!isImplicitMSB) {
diff --git a/flang/include/flang/Decimal/decimal.h b/flang/include/flang/Decimal/decimal.h
index b9ac6b71cd..a4e0ee7c84 100644
--- a/flang/include/flang/Decimal/decimal.h
+++ b/flang/include/flang/Decimal/decimal.h
@@ -43,14 +43,6 @@ struct ConversionToDecimalResult {
   enum ConversionResultFlags flags;
 };
 
-enum FortranRounding {
-  RoundNearest, /* RN and RP */
-  RoundUp, /* RU */
-  RoundDown, /* RD */
-  RoundToZero, /* RZ - no rounding */
-  RoundCompatible, /* RC: like RN, but ties go away from 0 */
-};
-
 /* The "minimize" flag causes the fewest number of output digits
  * to be emitted such that reading them back into the same binary
  * floating-point format with RoundNearest will return the same
diff --git a/flang/include/flang/Evaluate/characteristics.h b/flang/include/flang/Evaluate/characteristics.h
index bcb1543203..d685d250bf 100644
--- a/flang/include/flang/Evaluate/characteristics.h
+++ b/flang/include/flang/Evaluate/characteristics.h
@@ -81,18 +81,19 @@ class TypeAndShape {
   bool operator!=(const TypeAndShape &that) const { return !(*this == that); }
 
   static std::optional<TypeAndShape> Characterize(
-      const semantics::Symbol &, FoldingContext &, bool invariantOnly = false);
+      const semantics::Symbol &, FoldingContext &, bool invariantOnly = true);
   static std::optional<TypeAndShape> Characterize(
       const semantics::DeclTypeSpec &, FoldingContext &,
-      bool invariantOnly = false);
+      bool invariantOnly = true);
   static std::optional<TypeAndShape> Characterize(
-      const ActualArgument &, FoldingContext &, bool invariantOnly = false);
+      const ActualArgument &, FoldingContext &, bool invariantOnly = true);
 
   // General case for Expr<T>, &c.
   template <typename A>
   static std::optional<TypeAndShape> Characterize(
-      const A &x, FoldingContext &context, bool invariantOnly = false) {
-    if (const auto *symbol{UnwrapWholeSymbolDataRef(x)}) {
+      const A &x, FoldingContext &context, bool invariantOnly = true) {
+    const auto *symbol{UnwrapWholeSymbolOrComponentDataRef(x)};
+    if (symbol && !symbol->owner().IsDerivedType()) { // Whole variable
       if (auto result{Characterize(*symbol, context, invariantOnly)}) {
         return result;
       }
@@ -106,6 +107,9 @@ class TypeAndShape {
           }
         }
       }
+      if (symbol) { // component
+        result.AcquireAttrs(*symbol);
+      }
       return std::move(result.Rewrite(context));
     }
     return std::nullopt;
@@ -116,15 +120,21 @@ class TypeAndShape {
   static std::optional<TypeAndShape> Characterize(
       const Designator<Type<TypeCategory::Character, KIND>> &x,
       FoldingContext &context, bool invariantOnly = true) {
-    if (const auto *symbol{UnwrapWholeSymbolDataRef(x)}) {
+    const auto *symbol{UnwrapWholeSymbolOrComponentDataRef(x)};
+    if (symbol && !symbol->owner().IsDerivedType()) { // Whole variable
       if (auto result{Characterize(*symbol, context, invariantOnly)}) {
         return result;
       }
     }
     if (auto type{x.GetType()}) {
       TypeAndShape result{*type, GetShape(context, x, invariantOnly)};
-      if (auto length{x.LEN()}) {
-        result.set_LEN(std::move(*length));
+      if (type->category() == TypeCategory::Character) {
+        if (auto length{x.LEN()}) {
+          result.set_LEN(std::move(*length));
+        }
+      }
+      if (symbol) { // component
+        result.AcquireAttrs(*symbol);
       }
       return std::move(result.Rewrite(context));
     }
@@ -133,7 +143,7 @@ class TypeAndShape {
 
   template <typename A>
   static std::optional<TypeAndShape> Characterize(const std::optional<A> &x,
-      FoldingContext &context, bool invariantOnly = false) {
+      FoldingContext &context, bool invariantOnly = true) {
     if (x) {
       return Characterize(*x, context, invariantOnly);
     } else {
@@ -142,7 +152,7 @@ class TypeAndShape {
   }
   template <typename A>
   static std::optional<TypeAndShape> Characterize(
-      A *ptr, FoldingContext &context, bool invariantOnly = false) {
+      A *ptr, FoldingContext &context, bool invariantOnly = true) {
     if (ptr) {
       return Characterize(std::as_const(*ptr), context, invariantOnly);
     } else {
diff --git a/flang/include/flang/Frontend/CodeGenOptions.def b/flang/include/flang/Frontend/CodeGenOptions.def
index c3a04108aa..1e350869f1 100644
--- a/flang/include/flang/Frontend/CodeGenOptions.def
+++ b/flang/include/flang/Frontend/CodeGenOptions.def
@@ -32,6 +32,7 @@ CODEGENOPT(PrepareForThinLTO , 1, 0) ///< Set when -flto=thin is enabled on the
                                      ///< compile step.
 CODEGENOPT(StackArrays, 1, 0) ///< -fstack-arrays (enable the stack-arrays pass)
 CODEGENOPT(LoopVersioning, 1, 0) ///< Enable loop versioning.
+CODEGENOPT(AliasAnalysis, 1, 0) ///< Enable alias analysis pass
 
 CODEGENOPT(Underscoring, 1, 1)
 ENUM_CODEGENOPT(RelocationModel, llvm::Reloc::Model, 3, llvm::Reloc::PIC_) ///< Name of the relocation model to use.
diff --git a/flang/include/flang/ISO_Fortran_binding.h b/flang/include/flang/ISO_Fortran_binding.h
index 2893fd46c2..51d6219427 100644
--- a/flang/include/flang/ISO_Fortran_binding.h
+++ b/flang/include/flang/ISO_Fortran_binding.h
@@ -169,7 +169,7 @@ template <int r> struct CdescStorage : public CFI_cdesc_t {
 template <> struct CdescStorage<1> : public CFI_cdesc_t {};
 template <> struct CdescStorage<0> : public CFI_cdesc_t {};
 } // namespace cfi_internal
-#define CFI_CDESC_T(rank) cfi_internal::CdescStorage<rank>
+#define CFI_CDESC_T(rank) ::Fortran::ISO::cfi_internal::CdescStorage<rank>
 #else
 #define CFI_CDESC_T(_RANK) \
   struct { \
@@ -200,8 +200,8 @@ RT_API_ATTRS int CFI_setpointer(
 #ifdef __cplusplus
 } // extern "C"
 } // inline namespace Fortran_2018
-}
-}
+} // namespace ISO
+} // namespace Fortran
 #endif
 
 #endif /* CFI_ISO_FORTRAN_BINDING_H_ */
diff --git a/flang/include/flang/Lower/ConvertType.h b/flang/include/flang/Lower/ConvertType.h
index f86cc00235..7a3f92649a 100644
--- a/flang/include/flang/Lower/ConvertType.h
+++ b/flang/include/flang/Lower/ConvertType.h
@@ -48,6 +48,8 @@ struct SomeType;
 namespace semantics {
 class Symbol;
 class DerivedTypeSpec;
+class DerivedTypeDetails;
+class Scope;
 } // namespace semantics
 
 namespace lower {
@@ -97,6 +99,34 @@ class TypeBuilder {
 using namespace evaluate;
 FOR_EACH_SPECIFIC_TYPE(extern template class TypeBuilder, )
 
+/// A helper class to reverse iterate through the component names of a derived
+/// type, including the parent component and the component of the parents. This
+/// is useful to deal with StructureConstructor lowering.
+class ComponentReverseIterator {
+public:
+  ComponentReverseIterator(const Fortran::semantics::DerivedTypeSpec &derived) {
+    setCurrentType(derived);
+  }
+  /// Does the current type has a component with \name (does not look-up the
+  /// components of the parent if any)? If there is a match, the iterator
+  /// is advanced to the search result.
+  bool lookup(const Fortran::parser::CharBlock &name) {
+    componentIt = std::find(componentIt, componentItEnd, name);
+    return componentIt != componentItEnd;
+  };
+
+  /// Advance iterator to the last components of the current type parent.
+  const Fortran::semantics::DerivedTypeSpec &advanceToParentType();
+
+private:
+  void setCurrentType(const Fortran::semantics::DerivedTypeSpec &derived);
+  const Fortran::semantics::DerivedTypeSpec *currentParentType = nullptr;
+  const Fortran::semantics::DerivedTypeDetails *currentTypeDetails = nullptr;
+  using name_iterator =
+      std::list<Fortran::parser::CharBlock>::const_reverse_iterator;
+  name_iterator componentIt{};
+  name_iterator componentItEnd{};
+};
 } // namespace lower
 } // namespace Fortran
 
diff --git a/flang/include/flang/Optimizer/Analysis/TBAAForest.h b/flang/include/flang/Optimizer/Analysis/TBAAForest.h
new file mode 100644
index 0000000000..a024544e50
--- /dev/null
+++ b/flang/include/flang/Optimizer/Analysis/TBAAForest.h
@@ -0,0 +1,104 @@
+//===-- TBAAForest.h - A TBAA tree for each function -----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_OPTIMIZER_ANALYSIS_TBAA_FOREST_H
+#define FORTRAN_OPTIMIZER_ANALYSIS_TBAA_FOREST_H
+
+#include "mlir/Dialect/Func/IR/FuncOps.h"
+#include "mlir/Dialect/LLVMIR/LLVMAttrs.h"
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/IR/MLIRContext.h"
+#include "llvm/ADT/DenseMap.h"
+#include <string>
+
+namespace fir {
+
+//===----------------------------------------------------------------------===//
+// TBAATree
+//===----------------------------------------------------------------------===//
+/// Per-function TBAA tree. Each tree contains branches for data (of various
+/// kinds) and descriptor access
+struct TBAATree {
+  //===----------------------------------------------------------------------===//
+  // TBAAForrest::TBAATree::SubtreeState
+  //===----------------------------------------------------------------------===//
+  /// This contains a TBAA subtree based on some parent. New tags can be added
+  /// under the parent using getTag.
+  class SubtreeState {
+    friend TBAATree; // only allow construction by TBAATree
+  public:
+    SubtreeState() = delete;
+    SubtreeState(const SubtreeState &) = delete;
+    SubtreeState(SubtreeState &&) = default;
+
+    mlir::LLVM::TBAATagAttr getTag(llvm::StringRef uniqueId) const;
+
+  private:
+    SubtreeState(mlir::MLIRContext *ctx, std::string name,
+                 mlir::LLVM::TBAANodeAttr grandParent)
+        : parentId{std::move(name)}, context(ctx) {
+      parent = mlir::LLVM::TBAATypeDescriptorAttr::get(
+          context, parentId, mlir::LLVM::TBAAMemberAttr::get(grandParent, 0));
+    }
+
+    const std::string parentId;
+    mlir::MLIRContext *const context;
+    mlir::LLVM::TBAATypeDescriptorAttr parent;
+    llvm::DenseMap<llvm::StringRef, mlir::LLVM::TBAATagAttr> tagDedup;
+  };
+
+  SubtreeState globalDataTree;
+  SubtreeState allocatedDataTree;
+  SubtreeState dummyArgDataTree;
+  mlir::LLVM::TBAATypeDescriptorAttr anyAccessDesc;
+  mlir::LLVM::TBAATypeDescriptorAttr boxMemberTypeDesc;
+  mlir::LLVM::TBAATypeDescriptorAttr anyDataTypeDesc;
+
+  static TBAATree buildTree(mlir::StringAttr functionName);
+
+private:
+  TBAATree(mlir::LLVM::TBAATypeDescriptorAttr anyAccess,
+           mlir::LLVM::TBAATypeDescriptorAttr dataRoot,
+           mlir::LLVM::TBAATypeDescriptorAttr boxMemberTypeDesc);
+};
+
+//===----------------------------------------------------------------------===//
+// TBAAForrest
+//===----------------------------------------------------------------------===//
+/// Collection of TBAATrees, usually indexed by function (so that each function
+/// has a different TBAATree)
+class TBAAForrest {
+public:
+  explicit TBAAForrest(bool separatePerFunction = true)
+      : separatePerFunction{separatePerFunction} {}
+
+  inline const TBAATree &operator[](mlir::func::FuncOp func) {
+    return getFuncTree(func.getSymNameAttr());
+  }
+  inline const TBAATree &operator[](mlir::LLVM::LLVMFuncOp func) {
+    return getFuncTree(func.getSymNameAttr());
+  }
+
+private:
+  const TBAATree &getFuncTree(mlir::StringAttr symName) {
+    if (!separatePerFunction)
+      symName = mlir::StringAttr::get(symName.getContext(), "");
+    if (!trees.contains(symName))
+      trees.insert({symName, TBAATree::buildTree(symName)});
+    return trees.at(symName);
+  }
+
+  // Should each function use a different tree?
+  const bool separatePerFunction;
+  // TBAA tree per function
+  llvm::DenseMap<mlir::StringAttr, TBAATree> trees;
+};
+
+} // namespace fir
+
+#endif // FORTRAN_OPTIMIZER_ANALYSIS_TBAA_FOREST_H
diff --git a/flang/include/flang/Optimizer/Builder/HLFIRTools.h b/flang/include/flang/Optimizer/Builder/HLFIRTools.h
index 8cf838ea19..f0b66baddd 100644
--- a/flang/include/flang/Optimizer/Builder/HLFIRTools.h
+++ b/flang/include/flang/Optimizer/Builder/HLFIRTools.h
@@ -172,6 +172,11 @@ class Entity : public mlir::Value {
     return varIface ? varIface.isAllocatable() : false;
   }
 
+  bool isPointer() const {
+    auto varIface = getIfVariableInterface();
+    return varIface ? varIface.isPointer() : false;
+  }
+
   // Get the entity as an mlir SSA value containing all the shape, type
   // parameters and dynamic shape information.
   mlir::Value getBase() const { return *this; }
diff --git a/flang/include/flang/Optimizer/CodeGen/CodeGen.h b/flang/include/flang/Optimizer/CodeGen/CodeGen.h
index 64747b871d..7d8e548d89 100644
--- a/flang/include/flang/Optimizer/CodeGen/CodeGen.h
+++ b/flang/include/flang/Optimizer/CodeGen/CodeGen.h
@@ -54,6 +54,9 @@ struct FIRToLLVMPassOptions {
 
   // Generate TBAA information for FIR types and memory accessing operations.
   bool applyTBAA = false;
+
+  // Force the usage of a unified tbaa tree in TBAABuilder.
+  bool forceUnifiedTBAATree = false;
 };
 
 /// Convert FIR to the LLVM IR dialect with default options.
diff --git a/flang/include/flang/Optimizer/CodeGen/TBAABuilder.h b/flang/include/flang/Optimizer/CodeGen/TBAABuilder.h
index 3aeb124f91..5420e48146 100644
--- a/flang/include/flang/Optimizer/CodeGen/TBAABuilder.h
+++ b/flang/include/flang/Optimizer/CodeGen/TBAABuilder.h
@@ -13,8 +13,8 @@
 #ifndef FORTRAN_OPTIMIZER_CODEGEN_TBAABUILDER_H
 #define FORTRAN_OPTIMIZER_CODEGEN_TBAABUILDER_H
 
+#include "flang/Optimizer/Analysis/TBAAForest.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
-#include "mlir/IR/BuiltinAttributes.h"
 
 namespace fir {
 
@@ -25,9 +25,9 @@ namespace fir {
 //
 // TBAA type information is represented with LLVM::MetadataOp operation
 // with specific symbol name `TBAABuilder::tbaaMetaOpName`. The basic
-// TBAA tree used for Flang consists of the following nodes:
+// TBAA trees used for Flang consists of the following nodes:
 //   llvm.metadata @__flang_tbaa {
-//     llvm.tbaa_root @root_0 {id = "Flang Type TBAA Root"}
+//     llvm.tbaa_root @root_0 {id = "Flang Type TBAA Function Root funcName"}
 //     llvm.tbaa_type_desc @type_desc_1 {id = "any access",
 //                                       members = {<@root_0, 0>}}
 //     llvm.tbaa_type_desc @type_desc_2 {id = "any data access",
@@ -162,9 +162,14 @@ namespace fir {
 // Given the storage association, all non-box accesses are represented
 // with the conservative data access tag:
 //   < `<any data access>`, `<any data access>`, 0 >
+
+// additional tags are added in flang/Optimizer/Transforms/AddAliasTags.cpp
+// (before CodeGen)
 class TBAABuilder {
 public:
-  TBAABuilder(mlir::MLIRContext *context, bool applyTBAA);
+  /// if forceUnifiedTree is true, functions will not have different TBAA trees
+  TBAABuilder(mlir::MLIRContext *context, bool applyTBAA,
+              bool forceUnifiedTree = false);
   TBAABuilder(TBAABuilder const &) = delete;
   TBAABuilder &operator=(TBAABuilder const &) = delete;
 
@@ -184,13 +189,13 @@ class TBAABuilder {
 
   // Returns TBAATagAttr representing access tag:
   //   < <descriptor member>, <descriptor member>, 0 >
-  mlir::LLVM::TBAATagAttr getAnyBoxAccessTag();
+  mlir::LLVM::TBAATagAttr getAnyBoxAccessTag(mlir::LLVM::LLVMFuncOp func);
   // Returns TBAATagAttr representing access tag:
   //   < <any data access>, <any data access>, 0 >
-  mlir::LLVM::TBAATagAttr getAnyDataAccessTag();
+  mlir::LLVM::TBAATagAttr getAnyDataAccessTag(mlir::LLVM::LLVMFuncOp func);
   // Returns TBAATagAttr representing access tag:
   //   < <any access>, <any access>, 0 >
-  mlir::LLVM::TBAATagAttr getAnyAccessTag();
+  mlir::LLVM::TBAATagAttr getAnyAccessTag(mlir::LLVM::LLVMFuncOp func);
 
   // Returns TBAATagAttr representing access tag described by the base and
   // access FIR types and the LLVM::GepOp representing the access in terms of
@@ -198,7 +203,8 @@ class TBAABuilder {
   // fir::BaseBoxType.
   mlir::LLVM::TBAATagAttr getBoxAccessTag(mlir::Type baseFIRType,
                                           mlir::Type accessFIRType,
-                                          mlir::LLVM::GEPOp gep);
+                                          mlir::LLVM::GEPOp gep,
+                                          mlir::LLVM::LLVMFuncOp func);
 
   // Returns TBAATagAttr representing access tag described by the base and
   // access FIR types and the LLVM::GepOp representing the access in terms of
@@ -206,34 +212,13 @@ class TBAABuilder {
   // "data" access, i.e. not an access of any box/descriptor member.
   mlir::LLVM::TBAATagAttr getDataAccessTag(mlir::Type baseFIRType,
                                            mlir::Type accessFIRType,
-                                           mlir::LLVM::GEPOp gep);
+                                           mlir::LLVM::GEPOp gep,
+                                           mlir::LLVM::LLVMFuncOp func);
 
   // Set to true, if TBAA builder is active, otherwise, all public
   // methods are no-ops.
   bool enableTBAA;
 
-  // LLVM::TBAARootAttr identifying Flang's TBAA root.
-  mlir::LLVM::TBAARootAttr flangTBAARoot;
-  // Identity string for Flang's TBAA root.
-  static constexpr llvm::StringRef flangTBAARootId = "Flang Type TBAA Root";
-
-  // LLVM::TBAATypeDescriptorAttr identifying "any access".
-  mlir::LLVM::TBAATypeDescriptorAttr anyAccessTypeDesc;
-  // Identity string for "any access" type descriptor.
-  static constexpr llvm::StringRef anyAccessTypeDescId = "any access";
-
-  // LLVM::TBAATypeDescriptorAttr identifying "any data access" (i.e. non-box
-  // memory access).
-  mlir::LLVM::TBAATypeDescriptorAttr anyDataAccessTypeDesc;
-  // Identity string for "any data access" type descriptor.
-  static constexpr llvm::StringRef anyDataAccessTypeDescId = "any data access";
-
-  // LLVM::TBAATypeDescriptorAttr identifying "descriptor member" access, i.e.
-  // any access within the bounds of a box/descriptor.
-  mlir::LLVM::TBAATypeDescriptorAttr boxMemberTypeDesc;
-  // Identity string for "descriptor member" type descriptor.
-  static constexpr llvm::StringRef boxMemberTypeDescId = "descriptor member";
-
   // Number of attached TBAA tags (used for debugging).
   unsigned tagAttachmentCounter = 0;
 
@@ -247,6 +232,8 @@ class TBAABuilder {
       std::tuple<mlir::LLVM::TBAANodeAttr, mlir::LLVM::TBAANodeAttr, int64_t>,
       mlir::LLVM::TBAATagAttr>
       tagsMap;
+
+  TBAAForrest trees;
 };
 
 } // namespace fir
diff --git a/flang/include/flang/Optimizer/CodeGen/TypeConverter.h b/flang/include/flang/Optimizer/CodeGen/TypeConverter.h
index 5b8b51b5a3..29d0a902f5 100644
--- a/flang/include/flang/Optimizer/CodeGen/TypeConverter.h
+++ b/flang/include/flang/Optimizer/CodeGen/TypeConverter.h
@@ -45,7 +45,8 @@ namespace fir {
 /// This converts FIR types to LLVM types (for now)
 class LLVMTypeConverter : public mlir::LLVMTypeConverter {
 public:
-  LLVMTypeConverter(mlir::ModuleOp module, bool applyTBAA);
+  LLVMTypeConverter(mlir::ModuleOp module, bool applyTBAA,
+                    bool forceUnifiedTBAATree);
 
   // i32 is used here because LLVM wants i32 constants when indexing into struct
   // types. Indexing into other aggregate types is more flexible.
diff --git a/flang/include/flang/Optimizer/Dialect/CMakeLists.txt b/flang/include/flang/Optimizer/Dialect/CMakeLists.txt
index d657e3f166..fe9864a262 100644
--- a/flang/include/flang/Optimizer/Dialect/CMakeLists.txt
+++ b/flang/include/flang/Optimizer/Dialect/CMakeLists.txt
@@ -18,6 +18,10 @@ set(LLVM_TARGET_DEFINITIONS FortranVariableInterface.td)
 mlir_tablegen(FortranVariableInterface.h.inc -gen-op-interface-decls)
 mlir_tablegen(FortranVariableInterface.cpp.inc -gen-op-interface-defs)
 
+set(LLVM_TARGET_DEFINITIONS FirAliasTagOpInterface.td)
+mlir_tablegen(FirAliasTagOpInterface.h.inc -gen-op-interface-decls)
+mlir_tablegen(FirAliasTagOpInterface.cpp.inc -gen-op-interface-defs)
+
 set(LLVM_TARGET_DEFINITIONS CanonicalizationPatterns.td)
 mlir_tablegen(CanonicalizationPatterns.inc -gen-rewriters)
 add_public_tablegen_target(CanonicalizationPatternsIncGen)
diff --git a/flang/include/flang/Optimizer/Dialect/FIRDialect.td b/flang/include/flang/Optimizer/Dialect/FIRDialect.td
index d0735bbeb2..b366b6d40e 100644
--- a/flang/include/flang/Optimizer/Dialect/FIRDialect.td
+++ b/flang/include/flang/Optimizer/Dialect/FIRDialect.td
@@ -30,7 +30,9 @@ def fir_Dialect : Dialect {
   let dependentDialects = [
     // Arith dialect provides FastMathFlagsAttr
     // supported by some FIR operations.
-    "arith::ArithDialect"
+    "arith::ArithDialect",
+    // TBAA Tag types
+    "LLVM::LLVMDialect"
   ];
 }
 
diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.h b/flang/include/flang/Optimizer/Dialect/FIROps.h
index 8f03dc5cf7..87196dbf9b 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.h
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.h
@@ -11,6 +11,7 @@
 
 #include "flang/Optimizer/Dialect/FIRAttr.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
+#include "flang/Optimizer/Dialect/FirAliasTagOpInterface.h"
 #include "flang/Optimizer/Dialect/FortranVariableInterface.h"
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index cb39619f52..80d1635e50 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -16,10 +16,12 @@
 
 include "mlir/Dialect/Arith/IR/ArithBase.td"
 include "mlir/Dialect/Arith/IR/ArithOpsInterfaces.td"
+include "mlir/Dialect/LLVMIR/LLVMAttrDefs.td"
 include "flang/Optimizer/Dialect/FIRDialect.td"
 include "flang/Optimizer/Dialect/FIRTypes.td"
 include "flang/Optimizer/Dialect/FIRAttr.td"
 include "flang/Optimizer/Dialect/FortranVariableInterface.td"
+include "flang/Optimizer/Dialect/FirAliasTagOpInterface.td"
 include "mlir/IR/BuiltinAttributes.td"
 
 // Base class for FIR operations.
@@ -258,7 +260,7 @@ def fir_FreeMemOp : fir_Op<"freemem", [MemoryEffects<[MemFree]>]> {
   let assemblyFormat = "$heapref attr-dict `:` qualified(type($heapref))";
 }
 
-def fir_LoadOp : fir_OneResultOp<"load", []> {
+def fir_LoadOp : fir_OneResultOp<"load", [FirAliasTagOpInterface]> {
   let summary = "load a value from a memory reference";
   let description = [{
     Load a value from a memory reference into an ssa-value (virtual register).
@@ -274,9 +276,11 @@ def fir_LoadOp : fir_OneResultOp<"load", []> {
     or null.
   }];
 
-  let arguments = (ins Arg<AnyReferenceLike, "", [MemRead]>:$memref);
+  let arguments = (ins Arg<AnyReferenceLike, "", [MemRead]>:$memref,
+                  OptionalAttr<LLVM_TBAATagArrayAttr>:$tbaa);
 
-  let builders = [OpBuilder<(ins "mlir::Value":$refVal)>];
+  let builders = [OpBuilder<(ins "mlir::Value":$refVal)>,
+                  OpBuilder<(ins "mlir::Type":$resTy, "mlir::Value":$refVal)>];
 
   let hasCustomAssemblyFormat = 1;
 
@@ -285,7 +289,7 @@ def fir_LoadOp : fir_OneResultOp<"load", []> {
   }];
 }
 
-def fir_StoreOp : fir_Op<"store", []> {
+def fir_StoreOp : fir_Op<"store", [FirAliasTagOpInterface]> {
   let summary = "store an SSA-value to a memory location";
 
   let description = [{
@@ -305,7 +309,10 @@ def fir_StoreOp : fir_Op<"store", []> {
   }];
 
   let arguments = (ins AnyType:$value,
-                   Arg<AnyReferenceLike, "", [MemWrite]>:$memref);
+                   Arg<AnyReferenceLike, "", [MemWrite]>:$memref,
+                   OptionalAttr<LLVM_TBAATagArrayAttr>:$tbaa);
+
+  let builders = [OpBuilder<(ins "mlir::Value":$value, "mlir::Value":$memref)>];
 
   let hasCustomAssemblyFormat = 1;
   let hasVerifier = 1;
@@ -2112,6 +2119,8 @@ def fir_DoLoopOp : region_Op<"do_loop",
     mlir::Operation::operand_range getIterOperands() {
       return getOperands().drop_front(getNumControlOperands());
     }
+    mlir::OperandRange getInits() { return getIterOperands(); }
+    mlir::ValueRange getYieldedValues();
 
     void setLowerBound(mlir::Value bound) { (*this)->setOperand(0, bound); }
     void setUpperBound(mlir::Value bound) { (*this)->setOperand(1, bound); }
@@ -2263,6 +2272,8 @@ def fir_IterWhileOp : region_Op<"iterate_while",
     mlir::Operation::operand_range getIterOperands() {
       return getOperands().drop_front(getNumControlOperands());
     }
+    mlir::OperandRange getInits() { return getIterOperands(); }
+    mlir::ValueRange getYieldedValues();
 
     void setLowerBound(mlir::Value bound) { (*this)->setOperand(0, bound); }
     void setUpperBound(mlir::Value bound) { (*this)->setOperand(1, bound); }
diff --git a/flang/include/flang/Optimizer/Dialect/FirAliasTagOpInterface.h b/flang/include/flang/Optimizer/Dialect/FirAliasTagOpInterface.h
new file mode 100644
index 0000000000..f2d5b39acf
--- /dev/null
+++ b/flang/include/flang/Optimizer/Dialect/FirAliasTagOpInterface.h
@@ -0,0 +1,27 @@
+//===- FirAliasTagOpInterface.h ---------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains an interface for adding alias analysis information to
+// loads and stores
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_OPTIMIZER_DIALECT_FIR_ALIAS_TAG_OP_INTERFACE_H
+#define FORTRAN_OPTIMIZER_DIALECT_FIR_ALIAS_TAG_OP_INTERFACE_H
+
+#include "mlir/IR/OpDefinition.h"
+#include "mlir/IR/Operation.h"
+#include "mlir/Support/LogicalResult.h"
+
+namespace fir::detail {
+mlir::LogicalResult verifyFirAliasTagOpInterface(mlir::Operation *op);
+} // namespace fir::detail
+
+#include "flang/Optimizer/Dialect/FirAliasTagOpInterface.h.inc"
+
+#endif // FORTRAN_OPTIMIZER_DIALECT_FIR_ALIAS_TAG_OP_INTERFACE_H
diff --git a/flang/include/flang/Optimizer/Dialect/FirAliasTagOpInterface.td b/flang/include/flang/Optimizer/Dialect/FirAliasTagOpInterface.td
new file mode 100644
index 0000000000..9ce720f3b1
--- /dev/null
+++ b/flang/include/flang/Optimizer/Dialect/FirAliasTagOpInterface.td
@@ -0,0 +1,59 @@
+//===-- FirAliasTagOpInterface.td --------------------------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+include "mlir/IR/Interfaces.td"
+
+def FirAliasTagOpInterface : OpInterface<"FirAliasTagOpInterface"> {
+  let description = [{
+    An interface for memory operations that can carry alias analysis metadata.
+    It provides setters and getters for the operation's alias analysis
+    attributes. The default implementations of the interface methods expect
+    the operation to have an attribute of type ArrayAttr named tbaa.
+    Unlike the mlir::LLVM::AliasAnalysisOpInterface, this only supports tbaa.
+  }];
+
+  let cppNamespace = "::fir";
+  let verify = [{ return detail::verifyFirAliasTagOpInterface($_op); }];
+
+  let methods = [
+    InterfaceMethod<
+      /*desc=*/        "Returns the tbaa attribute or nullptr",
+      /*returnType=*/  "mlir::ArrayAttr",
+      /*methodName=*/  "getTBAATagsOrNull",
+      /*args=*/        (ins),
+      /*methodBody=*/  [{}],
+      /*defaultImpl=*/ [{
+        auto op = mlir::cast<ConcreteOp>(this->getOperation());
+        return op.getTbaaAttr();
+      }]
+      >,
+    InterfaceMethod<
+      /*desc=*/        "Sets the tbaa attribute",
+      /*returnType=*/  "void",
+      /*methodName=*/  "setTBAATags",
+      /*args=*/        (ins "const mlir::ArrayAttr":$attr),
+      /*methodBody=*/  [{}],
+      /*defaultImpl=*/ [{
+        auto op = mlir::cast<ConcreteOp>(this->getOperation());
+        op.setTbaaAttr(attr);
+      }]
+      >,
+    InterfaceMethod<
+      /*desc=*/        "Returns a list of all pointer operands accessed by the "
+                       "operation",
+      /*returnType=*/  "::llvm::SmallVector<::mlir::Value>",
+      /*methodName=*/  "getAccessedOperands",
+      /*args=*/        (ins),
+      /*methodBody=*/  [{}],
+      /*defaultImpl=*/ [{
+        auto op = mlir::cast<ConcreteOp>(this->getOperation());
+        return {op.getMemref()};
+      }]
+      >
+  ];
+}
diff --git a/flang/include/flang/Optimizer/Transforms/Passes.h b/flang/include/flang/Optimizer/Transforms/Passes.h
index 64882c8ec4..30d97be380 100644
--- a/flang/include/flang/Optimizer/Transforms/Passes.h
+++ b/flang/include/flang/Optimizer/Transforms/Passes.h
@@ -61,6 +61,7 @@ std::unique_ptr<mlir::Pass> createMemDataFlowOptPass();
 std::unique_ptr<mlir::Pass> createPromoteToAffinePass();
 std::unique_ptr<mlir::Pass> createMemoryAllocationPass();
 std::unique_ptr<mlir::Pass> createStackArraysPass();
+std::unique_ptr<mlir::Pass> createAliasTagsPass();
 std::unique_ptr<mlir::Pass> createSimplifyIntrinsicsPass();
 std::unique_ptr<mlir::Pass> createAddDebugFoundationPass();
 std::unique_ptr<mlir::Pass> createLoopVersioningPass();
diff --git a/flang/include/flang/Optimizer/Transforms/Passes.td b/flang/include/flang/Optimizer/Transforms/Passes.td
index 80da485392..6d211a535b 100644
--- a/flang/include/flang/Optimizer/Transforms/Passes.td
+++ b/flang/include/flang/Optimizer/Transforms/Passes.td
@@ -252,6 +252,26 @@ def StackArrays : Pass<"stack-arrays", "mlir::ModuleOp"> {
   let constructor = "::fir::createStackArraysPass()";
 }
 
+def AddAliasTags : Pass<"fir-add-alias-tags", "mlir::ModuleOp"> {
+  let summary = "Add tbaa tags to operations that implement FirAliasAnalysisOpInterface";
+  let description = [{
+    TBAA (type based alias analysis) is one method to pass pointer alias information
+    from language frontends to LLVM. This pass uses fir::AliasAnalysis to add this
+    information to fir.load and fir.store operations.
+    Additional tags are added during codegen. See fir::TBAABuilder.
+    This needs to be a separate pass so that it happens before structured control
+    flow operations are lowered to branches and basic blocks (this makes tracing
+    the source of values much eaiser). The other TBAA tags need to be applied to
+    box loads and stores which are implicit in FIR and so cannot be annotated
+    until codegen.
+    TODO: this is currently a pass on mlir::ModuleOp to avoid parallelism. In
+    theory, each operation could be considered in prallel, so long as there
+    aren't races adding new tags to the mlir context.
+  }];
+  let dependentDialects = [ "fir::FIROpsDialect" ];
+  let constructor = "::fir::createAliasTagsPass()";
+}
+
 def SimplifyRegionLite : Pass<"simplify-region-lite", "mlir::ModuleOp"> {
   let summary = "Region simplification";
   let description = [{
diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index e7d74dda71..494e54faa6 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -568,6 +568,7 @@ class ParseTreeDumper {
   NODE(parser, OpenACCCombinedConstruct)
   NODE(parser, OpenACCConstruct)
   NODE(parser, OpenACCDeclarativeConstruct)
+  NODE(parser, OpenACCEndConstruct)
   NODE(parser, OpenACCLoopConstruct)
   NODE(parser, OpenACCRoutineConstruct)
   NODE(parser, OpenACCStandaloneDeclarativeConstruct)
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index cb4bb59bf3..a51921f256 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -2259,15 +2259,18 @@ struct LoopControl {
 };
 
 // R1121 label-do-stmt -> [do-construct-name :] DO label [loop-control]
+// A label-do-stmt with a do-construct-name is parsed as a non-label-do-stmt.
 struct LabelDoStmt {
   TUPLE_CLASS_BOILERPLATE(LabelDoStmt);
-  std::tuple<std::optional<Name>, Label, std::optional<LoopControl>> t;
+  std::tuple<Label, std::optional<LoopControl>> t;
 };
 
 // R1122 nonlabel-do-stmt -> [do-construct-name :] DO [loop-control]
 struct NonLabelDoStmt {
   TUPLE_CLASS_BOILERPLATE(NonLabelDoStmt);
-  std::tuple<std::optional<Name>, std::optional<LoopControl>> t;
+  std::tuple<std::optional<Name>, std::optional<Label>,
+      std::optional<LoopControl>>
+      t;
 };
 
 // R1132 end-do-stmt -> END DO [do-construct-name]
@@ -4254,6 +4257,11 @@ struct OpenACCLoopConstruct {
       t;
 };
 
+struct OpenACCEndConstruct {
+  WRAPPER_CLASS_BOILERPLATE(OpenACCEndConstruct, llvm::acc::Directive);
+  CharBlock source;
+};
+
 struct OpenACCStandaloneConstruct {
   TUPLE_CLASS_BOILERPLATE(OpenACCStandaloneConstruct);
   CharBlock source;
@@ -4264,7 +4272,7 @@ struct OpenACCConstruct {
   UNION_CLASS_BOILERPLATE(OpenACCConstruct);
   std::variant<OpenACCBlockConstruct, OpenACCCombinedConstruct,
       OpenACCLoopConstruct, OpenACCStandaloneConstruct, OpenACCCacheConstruct,
-      OpenACCWaitConstruct, OpenACCAtomicConstruct>
+      OpenACCWaitConstruct, OpenACCAtomicConstruct, OpenACCEndConstruct>
       u;
 };
 
diff --git a/flang/include/flang/Runtime/descriptor.h b/flang/include/flang/Runtime/descriptor.h
index c9a3b1b031..fa68d97769 100644
--- a/flang/include/flang/Runtime/descriptor.h
+++ b/flang/include/flang/Runtime/descriptor.h
@@ -390,14 +390,21 @@ class Descriptor {
     if (leadingDimensions > raw_.rank) {
       leadingDimensions = raw_.rank;
     }
+    bool stridesAreContiguous{true};
     for (int j{0}; j < leadingDimensions; ++j) {
       const Dimension &dim{GetDimension(j)};
-      if (bytes != dim.ByteStride()) {
-        return false;
-      }
+      stridesAreContiguous &=
+          (bytes == dim.ByteStride()) || (dim.Extent() == 1);
       bytes *= dim.Extent();
     }
-    return true;
+    // One and zero element arrays are contiguous even if the descriptor
+    // byte strides are not perfect multiples.
+    // Arrays with more than 2 elements may also be contiguous even if a
+    // byte stride in one dimension is not a perfect multiple, as long as
+    // this is the last dimension, or if the dimension has one extent and
+    // the following dimension have either one extents or contiguous byte
+    // strides.
+    return stridesAreContiguous || bytes == 0;
   }
 
   // Establishes a pointer to a section or element.
@@ -406,6 +413,8 @@ class Descriptor {
       const SubscriptValue *upper = nullptr,
       const SubscriptValue *stride = nullptr);
 
+  RT_API_ATTRS void ApplyMold(const Descriptor &, int rank);
+
   RT_API_ATTRS void Check() const;
 
   void Dump(FILE * = stdout) const;
diff --git a/flang/include/flang/Semantics/tools.h b/flang/include/flang/Semantics/tools.h
index 0b5c3dde2e..e3deb2da1b 100644
--- a/flang/include/flang/Semantics/tools.h
+++ b/flang/include/flang/Semantics/tools.h
@@ -509,7 +509,7 @@ template <ComponentKind componentKind> class ComponentIterator {
       explicit ComponentPathNode(const DerivedTypeSpec &derived)
           : derived_{derived} {
         if constexpr (componentKind == ComponentKind::Scope) {
-          const Scope &scope{DEREF(derived.scope())};
+          const Scope &scope{DEREF(derived.GetScope())};
           nameIterator_ = scope.cbegin();
           nameEnd_ = scope.cend();
         } else {
diff --git a/flang/include/flang/Semantics/type.h b/flang/include/flang/Semantics/type.h
index 5228c15066..8965d29d88 100644
--- a/flang/include/flang/Semantics/type.h
+++ b/flang/include/flang/Semantics/type.h
@@ -261,6 +261,8 @@ class DerivedTypeSpec {
   const SourceName &name() const { return name_; }
   const Symbol &typeSymbol() const { return typeSymbol_; }
   const Scope *scope() const { return scope_; }
+  // Return scope_ if it is set, or the typeSymbol_ scope otherwise.
+  const Scope *GetScope() const;
   void set_scope(const Scope &);
   void ReplaceScope(const Scope &);
   const RawParameters &rawParameters() const { return rawParameters_; }
diff --git a/flang/include/flang/Tools/CLOptions.inc b/flang/include/flang/Tools/CLOptions.inc
index 76d18b73ae..2ed716382f 100644
--- a/flang/include/flang/Tools/CLOptions.inc
+++ b/flang/include/flang/Tools/CLOptions.inc
@@ -65,6 +65,12 @@ DisableOption(FirAvc, "avc", "array value copy analysis and transformation");
 DisableOption(
     FirMao, "memory-allocation-opt", "memory allocation optimization");
 
+DisableOption(FirAliasTags, "fir-alias-tags", "fir alias analysis");
+static llvm::cl::opt<bool> useOldAliasTags("use-old-alias-tags",
+    llvm::cl::desc("Use a single TBAA tree for all functions and do not use "
+                   "the FIR alias tags pass"),
+    llvm::cl::init(false), llvm::cl::Hidden);
+
 /// CodeGen Passes
 #if !defined(FLANG_EXCLUDE_CODEGEN)
 DisableOption(CodeGenRewrite, "codegen-rewrite", "rewrite FIR for codegen");
@@ -156,6 +162,7 @@ inline void addFIRToLLVMPass(
   fir::FIRToLLVMPassOptions options;
   options.ignoreMissingTypeDescriptors = ignoreMissingTypeDescriptors;
   options.applyTBAA = optLevel.isOptimizingForSpeed();
+  options.forceUnifiedTBAATree = useOldAliasTags;
   addPassConditionally(pm, disableFirToLlvmIr,
       [&]() { return fir::createFIRToLLVMPass(options); });
 }
@@ -220,6 +227,9 @@ inline void createDefaultFIROptimizerPassPipeline(
   // Polymorphic types
   pm.addPass(fir::createPolymorphicOpConversionPass());
 
+  if (pc.AliasAnalysis && !disableFirAliasTags && !useOldAliasTags)
+    pm.addPass(fir::createAliasTagsPass());
+
   // convert control flow to CFG form
   fir::addCfgConversionPass(pm);
   pm.addPass(mlir::createConvertSCFToCFPass());
diff --git a/flang/include/flang/Tools/CrossToolHelpers.h b/flang/include/flang/Tools/CrossToolHelpers.h
index 6245a2f137..ddec70fa98 100644
--- a/flang/include/flang/Tools/CrossToolHelpers.h
+++ b/flang/include/flang/Tools/CrossToolHelpers.h
@@ -34,12 +34,14 @@ struct MLIRToLLVMPassPipelineConfig {
     Underscoring = opts.Underscoring;
     LoopVersioning = opts.LoopVersioning;
     DebugInfo = opts.getDebugInfo();
+    AliasAnalysis = opts.AliasAnalysis;
   }
 
   llvm::OptimizationLevel OptLevel; ///< optimisation level
   bool StackArrays = false; ///< convert memory allocations to alloca.
   bool Underscoring = true; ///< add underscores to function names.
   bool LoopVersioning = false; ///< Run the version loop pass.
+  bool AliasAnalysis = false; ///< Add TBAA tags to generated LLVMIR
   llvm::codegenoptions::DebugInfoKind DebugInfo =
       llvm::codegenoptions::NoDebugInfo; ///< Debug info generation.
   unsigned VScaleMin = 0; ///< SVE vector range minimum.
diff --git a/flang/lib/Decimal/binary-to-decimal.cpp b/flang/lib/Decimal/binary-to-decimal.cpp
index 7b31d02b29..55fc548a69 100644
--- a/flang/lib/Decimal/binary-to-decimal.cpp
+++ b/flang/lib/Decimal/binary-to-decimal.cpp
@@ -373,7 +373,8 @@ STREAM &BigRadixFloatingPointNumber<PREC, LOG10RADIX>::Dump(STREAM &o) const {
   if (isNegative_) {
     o << '-';
   }
-  o << "10**(" << exponent_ << ") * ...\n";
+  o << "10**(" << exponent_ << ") * ...  (rounding "
+    << static_cast<int>(rounding_) << ")\n";
   for (int j{digits_}; --j >= 0;) {
     std::string str{std::to_string(digit_[j])};
     o << std::string(20 - str.size(), ' ') << str << " [" << j << ']';
diff --git a/flang/lib/Decimal/decimal-to-binary.cpp b/flang/lib/Decimal/decimal-to-binary.cpp
index c8c7b23329..d5b66b9fb9 100644
--- a/flang/lib/Decimal/decimal-to-binary.cpp
+++ b/flang/lib/Decimal/decimal-to-binary.cpp
@@ -257,13 +257,20 @@ ConversionToBinaryResult<PREC> IntermediateFloat<PREC>::ToBinary(
     flags |= Inexact;
   }
   if (fraction == 0 && guard <= oneHalf) {
-    return {Binary{}, static_cast<enum ConversionResultFlags>(flags)};
-  }
-  // The value is nonzero; normalize it.
-  while (fraction < topBit && expo > 1) {
-    --expo;
-    fraction = fraction * 2 + (guard >> (guardBits - 2));
-    guard = (((guard >> (guardBits - 2)) & 1) << (guardBits - 1)) | (guard & 1);
+    if ((!isNegative && rounding == RoundUp) ||
+        (isNegative && rounding == RoundDown)) {
+      // round to minimum nonzero value
+    } else {
+      return {Binary{}, static_cast<enum ConversionResultFlags>(flags)};
+    }
+  } else {
+    // The value is nonzero; normalize it.
+    while (fraction < topBit && expo > 1) {
+      --expo;
+      fraction = fraction * 2 + (guard >> (guardBits - 2));
+      guard =
+          (((guard >> (guardBits - 2)) & 1) << (guardBits - 1)) | (guard & 1);
+    }
   }
   // Apply rounding
   bool incr{false};
@@ -330,8 +337,13 @@ BigRadixFloatingPointNumber<PREC, LOG10RADIX>::ConvertToBinary() {
   exponent_ += digits_ * log10Radix;
   // Sanity checks for ridiculous exponents
   static constexpr int crazy{2 * Real::decimalRange + log10Radix};
-  if (exponent_ < -crazy) { // underflow to +/-0.
-    return {Real{SignBit()}, Inexact};
+  if (exponent_ < -crazy) {
+    if ((!isNegative_ && rounding_ == RoundUp) ||
+        (isNegative_ && rounding_ == RoundDown)) {
+      return {Real{Raw{1} | SignBit()}}; // return least nonzero value
+    } else { // underflow to +/-0.
+      return {Real{SignBit()}, Inexact};
+    }
   } else if (exponent_ > crazy) { // overflow to +/-Inf.
     return {Real{Infinity()}, Overflow};
   }
diff --git a/flang/lib/Evaluate/check-expression.cpp b/flang/lib/Evaluate/check-expression.cpp
index 29bd6eaa46..531fc5ccc5 100644
--- a/flang/lib/Evaluate/check-expression.cpp
+++ b/flang/lib/Evaluate/check-expression.cpp
@@ -821,8 +821,13 @@ class IsContiguousHelper
     if (x.base().Rank() == 0) {
       return (*this)(x.GetLastSymbol());
     } else {
-      // TODO could be true if base contiguous and this is only component, or
-      // if base has only one element?
+      if (Result baseIsContiguous{(*this)(x.base())}) {
+        if (!*baseIsContiguous) {
+          return false;
+        }
+        // TODO could be true if base contiguous and this is only component, or
+        // if base has only one element?
+      }
       return std::nullopt;
     }
   }
diff --git a/flang/lib/Evaluate/constant.cpp b/flang/lib/Evaluate/constant.cpp
index 084836b4ec..0e0d412118 100644
--- a/flang/lib/Evaluate/constant.cpp
+++ b/flang/lib/Evaluate/constant.cpp
@@ -36,11 +36,11 @@ ConstantSubscripts ConstantBounds::ComputeUbounds(
     std::optional<int> dim) const {
   if (dim) {
     CHECK(*dim < Rank());
-    return {lbounds_[*dim] + shape_[*dim] - 1};
+    return {lbounds_[*dim] + (shape_[*dim] - 1)};
   } else {
     ConstantSubscripts ubounds(Rank());
     for (int i{0}; i < Rank(); ++i) {
-      ubounds[i] = lbounds_[i] + shape_[i] - 1;
+      ubounds[i] = lbounds_[i] + (shape_[i] - 1);
     }
     return ubounds;
   }
@@ -73,7 +73,7 @@ ConstantSubscript ConstantBounds::SubscriptsToOffset(
   for (auto j : index) {
     auto lb{lbounds_[dim]};
     auto extent{shape_[dim++]};
-    CHECK(j >= lb && j < lb + extent);
+    CHECK(j >= lb && j - lb < extent);
     offset += stride * (j - lb);
     stride *= extent;
   }
@@ -93,10 +93,10 @@ bool ConstantBounds::IncrementSubscripts(
     ConstantSubscript k{dimOrder ? (*dimOrder)[j] : j};
     auto lb{lbounds_[k]};
     CHECK(indices[k] >= lb);
-    if (++indices[k] < lb + shape_[k]) {
+    if (++indices[k] - lb < shape_[k]) {
       return true;
     } else {
-      CHECK(indices[k] == lb + std::max<ConstantSubscript>(shape_[k], 1));
+      CHECK(indices[k] - lb == std::max<ConstantSubscript>(shape_[k], 1));
       indices[k] = lb;
     }
   }
diff --git a/flang/lib/Evaluate/fold-real.cpp b/flang/lib/Evaluate/fold-real.cpp
index 8e3ab1d8fd..6bcc3ec739 100644
--- a/flang/lib/Evaluate/fold-real.cpp
+++ b/flang/lib/Evaluate/fold-real.cpp
@@ -52,15 +52,28 @@ template <int KIND> class Norm2Accumulator {
       const Constant<T> &array, const Constant<T> &maxAbs, Rounding rounding)
       : array_{array}, maxAbs_{maxAbs}, rounding_{rounding} {};
   void operator()(Scalar<T> &element, const ConstantSubscripts &at) {
-    // Kahan summation of scaled elements
+    // Kahan summation of scaled elements:
+    // Naively,
+    //   NORM2(A(:)) = SQRT(SUM(A(:)**2))
+    // For any T > 0, we have mathematically
+    //   SQRT(SUM(A(:)**2))
+    //     = SQRT(T**2 * (SUM(A(:)**2) / T**2))
+    //     = SQRT(T**2 * SUM(A(:)**2 / T**2))
+    //     = SQRT(T**2 * SUM((A(:)/T)**2))
+    //     = SQRT(T**2) * SQRT(SUM((A(:)/T)**2))
+    //     = T * SQRT(SUM((A(:)/T)**2))
+    // By letting T = MAXVAL(ABS(A)), we ensure that
+    // ALL(ABS(A(:)/T) <= 1), so ALL((A(:)/T)**2 <= 1), and the SUM will
+    // not overflow unless absolutely necessary.
     auto scale{maxAbs_.At(maxAbsAt_)};
     if (scale.IsZero()) {
-      // If maxAbs is zero, so are all elements, and result
+      // Maximum value is zero, and so will the result be.
+      // Avoid division by zero below.
       element = scale;
     } else {
       auto item{array_.At(at)};
       auto scaled{item.Divide(scale).value};
-      auto square{item.Multiply(scaled).value};
+      auto square{scaled.Multiply(scaled).value};
       auto next{square.Add(correction_, rounding_)};
       overflow_ |= next.flags.test(RealFlag::Overflow);
       auto sum{element.Add(next.value, rounding_)};
@@ -73,13 +86,16 @@ template <int KIND> class Norm2Accumulator {
   }
   bool overflow() const { return overflow_; }
   void Done(Scalar<T> &result) {
+    // result+correction == SUM((data(:)/maxAbs)**2)
+    // result = maxAbs * SQRT(result+correction)
     auto corrected{result.Add(correction_, rounding_)};
     overflow_ |= corrected.flags.test(RealFlag::Overflow);
     correction_ = Scalar<T>{};
-    auto rescaled{corrected.value.Multiply(maxAbs_.At(maxAbsAt_))};
+    auto root{corrected.value.SQRT().value};
+    auto product{root.Multiply(maxAbs_.At(maxAbsAt_))};
     maxAbs_.IncrementSubscripts(maxAbsAt_);
-    overflow_ |= rescaled.flags.test(RealFlag::Overflow);
-    result = rescaled.value.SQRT().value;
+    overflow_ |= product.flags.test(RealFlag::Overflow);
+    result = product.value;
   }
 
 private:
diff --git a/flang/lib/Evaluate/fold-reduction.h b/flang/lib/Evaluate/fold-reduction.h
index cff7f54c60..0dd55124e6 100644
--- a/flang/lib/Evaluate/fold-reduction.h
+++ b/flang/lib/Evaluate/fold-reduction.h
@@ -228,7 +228,7 @@ template <typename T, bool ABS = false> class MaxvalMinvalAccumulator {
         test.Rewrite(context_, std::move(test)))};
     CHECK(folded.has_value());
     if (folded->IsTrue()) {
-      element = array_.At(at);
+      element = aAt;
     }
   }
   void Done(Scalar<T> &) const {}
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 37315e0e4d..ba2ecab374 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -214,6 +214,11 @@ static void parseCodeGenArgs(Fortran::frontend::CodeGenOptions &opts,
                    clang::driver::options::OPT_fno_loop_versioning, false))
     opts.LoopVersioning = 1;
 
+  opts.AliasAnalysis =
+      args.hasFlag(clang::driver::options::OPT_falias_analysis,
+                   clang::driver::options::OPT_fno_alias_analysis,
+                   /*default=*/false);
+
   for (auto *a : args.filtered(clang::driver::options::OPT_fpass_plugin_EQ))
     opts.LLVMPassPlugins.push_back(a->getValue());
 
diff --git a/flang/lib/Lower/Allocatable.cpp b/flang/lib/Lower/Allocatable.cpp
index a05f06aead..898f34786a 100644
--- a/flang/lib/Lower/Allocatable.cpp
+++ b/flang/lib/Lower/Allocatable.cpp
@@ -462,7 +462,7 @@ class AllocateStmtHelper {
     errorManager.genStatCheck(builder, loc);
     genAllocateObjectInit(box);
     if (alloc.hasCoarraySpec())
-      TODO(loc, "coarray allocation");
+      TODO(loc, "coarray: allocation of a coarray object");
     if (alloc.type.IsPolymorphic())
       genSetType(alloc, box, loc);
     genSetDeferredLengthParameters(alloc, box);
@@ -582,7 +582,7 @@ class AllocateStmtHelper {
     errorManager.genStatCheck(builder, loc);
     genAllocateObjectInit(box);
     if (alloc.hasCoarraySpec())
-      TODO(loc, "coarray allocation");
+      TODO(loc, "coarray: allocation of a coarray object");
     // Set length of the allocate object if it has. Otherwise, get the length
     // from source for the deferred length parameter.
     if (lenParams.empty() && box.isCharacter() &&
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 5ac4d822fa..c3afd91d74 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -719,46 +719,88 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     const Fortran::semantics::Symbol &hsym = sym.GetUltimate();
     Fortran::lower::SymbolBox hsb = lookupOneLevelUpSymbol(hsym);
     assert(hsb && "Host symbol box not found");
-    fir::ExtendedValue hexv = symBoxToExtendedValue(hsb);
 
     // 2) Fetch the copied one that will mask the original.
     Fortran::lower::SymbolBox sb = shallowLookupSymbol(sym);
     assert(sb && "Host-associated symbol box not found");
     assert(hsb.getAddr() != sb.getAddr() &&
            "Host and associated symbol boxes are the same");
-    fir::ExtendedValue exv = symBoxToExtendedValue(sb);
 
     // 3) Perform the assignment.
     mlir::OpBuilder::InsertPoint insPt = builder->saveInsertionPoint();
     if (copyAssignIP && copyAssignIP->isSet())
       builder->restoreInsertionPoint(*copyAssignIP);
     else
-      builder->setInsertionPointAfter(fir::getBase(exv).getDefiningOp());
+      builder->setInsertionPointAfter(sb.getAddr().getDefiningOp());
 
-    fir::ExtendedValue lhs, rhs;
+    Fortran::lower::SymbolBox *lhs_sb, *rhs_sb;
     if (copyAssignIP && copyAssignIP->isSet() &&
         sym.test(Fortran::semantics::Symbol::Flag::OmpLastPrivate)) {
       // lastprivate case
-      lhs = hexv;
-      rhs = exv;
+      lhs_sb = &hsb;
+      rhs_sb = &sb;
     } else {
-      lhs = exv;
-      rhs = hexv;
+      lhs_sb = &sb;
+      rhs_sb = &hsb;
     }
 
     mlir::Location loc = genLocation(sym.name());
-    mlir::Type symType = genType(sym);
 
-    if (auto seqTy = symType.dyn_cast<fir::SequenceType>()) {
-      Fortran::lower::StatementContext stmtCtx;
-      Fortran::lower::createSomeArrayAssignment(*this, lhs, rhs, localSymbols,
-                                                stmtCtx);
-      stmtCtx.finalizeAndReset();
-    } else if (hexv.getBoxOf<fir::CharBoxValue>()) {
-      fir::factory::CharacterExprHelper{*builder, loc}.createAssign(lhs, rhs);
+    if (lowerToHighLevelFIR()) {
+      hlfir::Entity lhs{lhs_sb->getAddr()};
+      hlfir::Entity rhs{rhs_sb->getAddr()};
+      // Temporary_lhs is set to true in hlfir.assign below to avoid user
+      // assignment to be used and finalization to be called on the LHS.
+      // This may or may not be correct but mimics the current behaviour
+      // without HLFIR.
+      auto copyData = [&](hlfir::Entity l, hlfir::Entity r) {
+        // Dereference RHS and load it if trivial scalar.
+        r = hlfir::loadTrivialScalar(loc, *builder, r);
+        builder->create<hlfir::AssignOp>(
+            loc, r, l,
+            /*isWholeAllocatableAssignment=*/false,
+            /*keepLhsLengthInAllocatableAssignment=*/false,
+            /*temporary_lhs=*/true);
+      };
+      if (lhs.isAllocatable()) {
+        // Deep copy allocatable if it is allocated.
+        // Note that when allocated, the RHS is already allocated with the LHS
+        // shape for copy on entry in createHostAssociateVarClone.
+        // For lastprivate, this assumes that the RHS was not reallocated in
+        // the OpenMP region.
+        lhs = hlfir::derefPointersAndAllocatables(loc, *builder, lhs);
+        mlir::Value addr = hlfir::genVariableRawAddress(loc, *builder, lhs);
+        mlir::Value isAllocated = builder->genIsNotNullAddr(loc, addr);
+        builder->genIfThen(loc, isAllocated)
+            .genThen([&]() {
+              // Copy the DATA, not the descriptors.
+              copyData(lhs, rhs);
+            })
+            .end();
+      } else if (lhs.isPointer()) {
+        // Set LHS target to the target of RHS (do not copy the RHS
+        // target data into the LHS target storage).
+        auto loadVal = builder->create<fir::LoadOp>(loc, rhs);
+        builder->create<fir::StoreOp>(loc, loadVal, lhs);
+      } else {
+        // Non ALLOCATABLE/POINTER variable. Simple DATA copy.
+        copyData(lhs, rhs);
+      }
     } else {
-      auto loadVal = builder->create<fir::LoadOp>(loc, fir::getBase(rhs));
-      builder->create<fir::StoreOp>(loc, loadVal, fir::getBase(lhs));
+      fir::ExtendedValue lhs = symBoxToExtendedValue(*lhs_sb);
+      fir::ExtendedValue rhs = symBoxToExtendedValue(*rhs_sb);
+      mlir::Type symType = genType(sym);
+      if (auto seqTy = symType.dyn_cast<fir::SequenceType>()) {
+        Fortran::lower::StatementContext stmtCtx;
+        Fortran::lower::createSomeArrayAssignment(*this, lhs, rhs, localSymbols,
+                                                  stmtCtx);
+        stmtCtx.finalizeAndReset();
+      } else if (lhs.getBoxOf<fir::CharBoxValue>()) {
+        fir::factory::CharacterExprHelper{*builder, loc}.createAssign(lhs, rhs);
+      } else {
+        auto loadVal = builder->create<fir::LoadOp>(loc, fir::getBase(rhs));
+        builder->create<fir::StoreOp>(loc, loadVal, fir::getBase(lhs));
+      }
     }
 
     if (copyAssignIP && copyAssignIP->isSet() &&
@@ -2610,8 +2652,12 @@ class FirConverter : public Fortran::lower::AbstractConverter {
         scopeBlockIdMap.try_emplace(&scope, ++blockId);
         Fortran::lower::AggregateStoreMap storeMap;
         for (const Fortran::lower::pft::Variable &var :
-             Fortran::lower::pft::getScopeVariableList(scope))
-          instantiateVar(var, storeMap);
+             Fortran::lower::pft::getScopeVariableList(scope)) {
+          // Do no instantiate again variables from the block host
+          // that appears in specification of block variables.
+          if (!var.hasSymbol() || !lookupSymbol(var.getSymbol()))
+            instantiateVar(var, storeMap);
+        }
       } else if (e.getIf<Fortran::parser::EndBlockStmt>()) {
         if (eval.lowerAsUnstructured())
           maybeStartBlock(e.block);
@@ -2625,35 +2671,35 @@ class FirConverter : public Fortran::lower::AbstractConverter {
   }
 
   void genFIR(const Fortran::parser::ChangeTeamConstruct &construct) {
-    TODO(toLocation(), "ChangeTeamConstruct implementation");
+    TODO(toLocation(), "coarray: ChangeTeamConstruct");
   }
   void genFIR(const Fortran::parser::ChangeTeamStmt &stmt) {
-    TODO(toLocation(), "ChangeTeamStmt implementation");
+    TODO(toLocation(), "coarray: ChangeTeamStmt");
   }
   void genFIR(const Fortran::parser::EndChangeTeamStmt &stmt) {
-    TODO(toLocation(), "EndChangeTeamStmt implementation");
+    TODO(toLocation(), "coarray: EndChangeTeamStmt");
   }
 
   void genFIR(const Fortran::parser::CriticalConstruct &criticalConstruct) {
     setCurrentPositionAt(criticalConstruct);
-    TODO(toLocation(), "CriticalConstruct implementation");
+    TODO(toLocation(), "coarray: CriticalConstruct");
   }
   void genFIR(const Fortran::parser::CriticalStmt &) {
-    TODO(toLocation(), "CriticalStmt implementation");
+    TODO(toLocation(), "coarray: CriticalStmt");
   }
   void genFIR(const Fortran::parser::EndCriticalStmt &) {
-    TODO(toLocation(), "EndCriticalStmt implementation");
+    TODO(toLocation(), "coarray: EndCriticalStmt");
   }
 
   void genFIR(const Fortran::parser::SelectRankConstruct &selectRankConstruct) {
     setCurrentPositionAt(selectRankConstruct);
-    TODO(toLocation(), "SelectRankConstruct implementation");
+    TODO(toLocation(), "coarray: SelectRankConstruct");
   }
   void genFIR(const Fortran::parser::SelectRankStmt &) {
-    TODO(toLocation(), "SelectRankStmt implementation");
+    TODO(toLocation(), "coarray: SelectRankStmt");
   }
   void genFIR(const Fortran::parser::SelectRankCaseStmt &) {
-    TODO(toLocation(), "SelectRankCaseStmt implementation");
+    TODO(toLocation(), "coarray: SelectRankCaseStmt");
   }
 
   void genFIR(const Fortran::parser::SelectTypeConstruct &selectTypeConstruct) {
diff --git a/flang/lib/Lower/CallInterface.cpp b/flang/lib/Lower/CallInterface.cpp
index 5299347e56..ea38b737a3 100644
--- a/flang/lib/Lower/CallInterface.cpp
+++ b/flang/lib/Lower/CallInterface.cpp
@@ -952,7 +952,7 @@ class Fortran::lower::CallInterfaceImpl {
     if (shapeAttrs.test(ShapeAttr::AssumedRank))
       TODO(loc, "assumed rank in procedure interface");
     if (shapeAttrs.test(ShapeAttr::Coarray))
-      TODO(loc, "coarray in procedure interface");
+      TODO(loc, "coarray: dummy argument coarray in procedure interface");
 
     // So far assume that if the argument cannot be passed by implicit interface
     // it must be by box. That may no be always true (e.g for simple optionals)
diff --git a/flang/lib/Lower/Coarray.cpp b/flang/lib/Lower/Coarray.cpp
index b5ab7b51fb..a84f65a5c4 100644
--- a/flang/lib/Lower/Coarray.cpp
+++ b/flang/lib/Lower/Coarray.cpp
@@ -27,27 +27,27 @@ void Fortran::lower::genChangeTeamConstruct(
     Fortran::lower::AbstractConverter &converter,
     Fortran::lower::pft::Evaluation &,
     const Fortran::parser::ChangeTeamConstruct &) {
-  TODO(converter.getCurrentLocation(), "CHANGE TEAM construct");
+  TODO(converter.getCurrentLocation(), "coarray: CHANGE TEAM construct");
 }
 
 void Fortran::lower::genChangeTeamStmt(
     Fortran::lower::AbstractConverter &converter,
     Fortran::lower::pft::Evaluation &,
     const Fortran::parser::ChangeTeamStmt &) {
-  TODO(converter.getCurrentLocation(), "CHANGE TEAM stmt");
+  TODO(converter.getCurrentLocation(), "coarray: CHANGE TEAM statement");
 }
 
 void Fortran::lower::genEndChangeTeamStmt(
     Fortran::lower::AbstractConverter &converter,
     Fortran::lower::pft::Evaluation &,
     const Fortran::parser::EndChangeTeamStmt &) {
-  TODO(converter.getCurrentLocation(), "END CHANGE TEAM");
+  TODO(converter.getCurrentLocation(), "coarray: END CHANGE TEAM statement");
 }
 
 void Fortran::lower::genFormTeamStatement(
     Fortran::lower::AbstractConverter &converter,
     Fortran::lower::pft::Evaluation &, const Fortran::parser::FormTeamStmt &) {
-  TODO(converter.getCurrentLocation(), "FORM TEAM");
+  TODO(converter.getCurrentLocation(), "coarray: FORM TEAM statement");
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/flang/lib/Lower/ConvertCall.cpp b/flang/lib/Lower/ConvertCall.cpp
index 90025ba9c6..bc9426827c 100644
--- a/flang/lib/Lower/ConvertCall.cpp
+++ b/flang/lib/Lower/ConvertCall.cpp
@@ -392,9 +392,14 @@ fir::ExtendedValue Fortran::lower::genCallOpAndResult(
     fir::DispatchOp dispatch;
     if (std::optional<unsigned> passArg = caller.getPassArgIndex()) {
       // PASS, PASS(arg-name)
+      // Note that caller.getInputs is used instead of operands to get the
+      // passed object because interface mismatch issues may have inserted a
+      // cast to the operand with a different declared type, which would break
+      // later type bound call resolution in the FIR to FIR pass.
       dispatch = builder.create<fir::DispatchOp>(
           loc, funcType.getResults(), builder.getStringAttr(procName),
-          operands[*passArg], operands, builder.getI32IntegerAttr(*passArg));
+          caller.getInputs()[*passArg], operands,
+          builder.getI32IntegerAttr(*passArg));
     } else {
       // NOPASS
       const Fortran::evaluate::Component *component =
diff --git a/flang/lib/Lower/ConvertConstant.cpp b/flang/lib/Lower/ConvertConstant.cpp
index 6e7a60e42a..423a14de4b 100644
--- a/flang/lib/Lower/ConvertConstant.cpp
+++ b/flang/lib/Lower/ConvertConstant.cpp
@@ -347,6 +347,83 @@ genConstantValue(Fortran::lower::AbstractConverter &converter,
                  mlir::Location loc,
                  const Fortran::lower::SomeExpr &constantExpr);
 
+static mlir::Value genStructureComponentInit(
+    Fortran::lower::AbstractConverter &converter, mlir::Location loc,
+    const Fortran::semantics::Symbol &sym, const Fortran::lower::SomeExpr &expr,
+    mlir::Value res) {
+  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
+  fir::RecordType recTy = mlir::cast<fir::RecordType>(res.getType());
+  std::string name = converter.getRecordTypeFieldName(sym);
+  mlir::Type componentTy = recTy.getType(name);
+  auto fieldTy = fir::FieldType::get(recTy.getContext());
+  assert(componentTy && "failed to retrieve component");
+  // FIXME: type parameters must come from the derived-type-spec
+  auto field = builder.create<fir::FieldIndexOp>(
+      loc, fieldTy, name, recTy,
+      /*typeParams=*/mlir::ValueRange{} /*TODO*/);
+
+  if (Fortran::semantics::IsAllocatable(sym))
+    TODO(loc, "allocatable component in structure constructor");
+
+  if (Fortran::semantics::IsPointer(sym)) {
+    mlir::Value initialTarget =
+        Fortran::lower::genInitialDataTarget(converter, loc, componentTy, expr);
+    res = builder.create<fir::InsertValueOp>(
+        loc, recTy, res, initialTarget,
+        builder.getArrayAttr(field.getAttributes()));
+    return res;
+  }
+
+  if (Fortran::lower::isDerivedTypeWithLenParameters(sym))
+    TODO(loc, "component with length parameters in structure constructor");
+
+  // Special handling for scalar c_ptr/c_funptr constants. The array constant
+  // must fall through to genConstantValue() below.
+  if (Fortran::semantics::IsBuiltinCPtr(sym) && sym.Rank() == 0 &&
+      (Fortran::evaluate::GetLastSymbol(expr) ||
+       Fortran::evaluate::IsNullPointer(expr))) {
+    // Builtin c_ptr and c_funptr have special handling because designators
+    // and NULL() are handled as initial values for them as an extension
+    // (otherwise only c_ptr_null/c_funptr_null are allowed and these are
+    // replaced by structure constructors by semantics, so GetLastSymbol
+    // returns nothing).
+
+    // The Ev::Expr is an initializer that is a pointer target (e.g., 'x' or
+    // NULL()) that must be inserted into an intermediate cptr record value's
+    // address field, which ought to be an intptr_t on the target.
+    mlir::Value addr = fir::getBase(
+        Fortran::lower::genExtAddrInInitializer(converter, loc, expr));
+    if (addr.getType().isa<fir::BoxProcType>())
+      addr = builder.create<fir::BoxAddrOp>(loc, addr);
+    assert((fir::isa_ref_type(addr.getType()) ||
+            addr.getType().isa<mlir::FunctionType>()) &&
+           "expect reference type for address field");
+    assert(fir::isa_derived(componentTy) &&
+           "expect C_PTR, C_FUNPTR to be a record");
+    auto cPtrRecTy = componentTy.cast<fir::RecordType>();
+    llvm::StringRef addrFieldName = Fortran::lower::builtin::cptrFieldName;
+    mlir::Type addrFieldTy = cPtrRecTy.getType(addrFieldName);
+    auto addrField = builder.create<fir::FieldIndexOp>(
+        loc, fieldTy, addrFieldName, componentTy,
+        /*typeParams=*/mlir::ValueRange{});
+    mlir::Value castAddr = builder.createConvert(loc, addrFieldTy, addr);
+    auto undef = builder.create<fir::UndefOp>(loc, componentTy);
+    addr = builder.create<fir::InsertValueOp>(
+        loc, componentTy, undef, castAddr,
+        builder.getArrayAttr(addrField.getAttributes()));
+    res = builder.create<fir::InsertValueOp>(
+        loc, recTy, res, addr, builder.getArrayAttr(field.getAttributes()));
+    return res;
+  }
+
+  mlir::Value val = fir::getBase(genConstantValue(converter, loc, expr));
+  assert(!fir::isa_ref_type(val.getType()) && "expecting a constant value");
+  mlir::Value castVal = builder.createConvert(loc, componentTy, val);
+  res = builder.create<fir::InsertValueOp>(
+      loc, recTy, res, castVal, builder.getArrayAttr(field.getAttributes()));
+  return res;
+}
+
 // Generate a StructureConstructor inlined (returns raw fir.type<T> value,
 // not the address of a global constant).
 static mlir::Value genInlinedStructureCtorLitImpl(
@@ -354,84 +431,76 @@ static mlir::Value genInlinedStructureCtorLitImpl(
     const Fortran::evaluate::StructureConstructor &ctor, mlir::Type type) {
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
   auto recTy = type.cast<fir::RecordType>();
-  auto fieldTy = fir::FieldType::get(type.getContext());
-  mlir::Value res = builder.create<fir::UndefOp>(loc, recTy);
-
-  for (const auto &[sym, expr] : ctor.values()) {
-    // Parent components need more work because they do not appear in the
-    // fir.rec type.
-    if (sym->test(Fortran::semantics::Symbol::Flag::ParentComp))
-      TODO(loc, "parent component in structure constructor");
-
-    std::string name = converter.getRecordTypeFieldName(sym);
-    mlir::Type componentTy = recTy.getType(name);
-    assert(componentTy && "failed to retrieve component");
-    // FIXME: type parameters must come from the derived-type-spec
-    auto field = builder.create<fir::FieldIndexOp>(
-        loc, fieldTy, name, type,
-        /*typeParams=*/mlir::ValueRange{} /*TODO*/);
 
-    if (Fortran::semantics::IsAllocatable(sym))
-      TODO(loc, "allocatable component in structure constructor");
+  if (!converter.getLoweringOptions().getLowerToHighLevelFIR()) {
+    mlir::Value res = builder.create<fir::UndefOp>(loc, recTy);
+    for (const auto &[sym, expr] : ctor.values()) {
+      // Parent components need more work because they do not appear in the
+      // fir.rec type.
+      if (sym->test(Fortran::semantics::Symbol::Flag::ParentComp))
+        TODO(loc, "parent component in structure constructor");
+      res = genStructureComponentInit(converter, loc, sym, expr.value(), res);
+    }
+    return res;
+  }
 
-    if (Fortran::semantics::IsPointer(sym)) {
-      mlir::Value initialTarget = Fortran::lower::genInitialDataTarget(
-          converter, loc, componentTy, expr.value());
+  auto fieldTy = fir::FieldType::get(recTy.getContext());
+  mlir::Value res{};
+  // When the first structure component values belong to some parent type PT
+  // and the next values belong to a type extension ET, a new undef for ET must
+  // be created and the previous PT value inserted into it. There may
+  // be empty parent types in between ET and PT, hence the list and while loop.
+  auto insertParentValueIntoExtension = [&](mlir::Type typeExtension) {
+    assert(res && "res must be set");
+    llvm::SmallVector<mlir::Type> parentTypes = {typeExtension};
+    while (true) {
+      fir::RecordType last = mlir::cast<fir::RecordType>(parentTypes.back());
+      mlir::Type next =
+          last.getType(0); // parent components are first in HLFIR.
+      if (next != res.getType())
+        parentTypes.push_back(next);
+      else
+        break;
+    }
+    for (mlir::Type parentType : llvm::reverse(parentTypes)) {
+      auto undef = builder.create<fir::UndefOp>(loc, parentType);
+      fir::RecordType parentRecTy = mlir::cast<fir::RecordType>(parentType);
+      auto field = builder.create<fir::FieldIndexOp>(
+          loc, fieldTy, parentRecTy.getTypeList()[0].first, parentType,
+          /*typeParams=*/mlir::ValueRange{} /*TODO*/);
       res = builder.create<fir::InsertValueOp>(
-          loc, recTy, res, initialTarget,
+          loc, parentRecTy, undef, res,
           builder.getArrayAttr(field.getAttributes()));
-      continue;
     }
+  };
 
-    if (Fortran::lower::isDerivedTypeWithLenParameters(sym))
-      TODO(loc, "component with length parameters in structure constructor");
-
-    // Special handling for scalar c_ptr/c_funptr constants. The array constant
-    // must fall through to genConstantValue() below.
-    if (Fortran::semantics::IsBuiltinCPtr(sym) && sym->Rank() == 0 &&
-        (Fortran::evaluate::GetLastSymbol(expr.value()) ||
-         Fortran::evaluate::IsNullPointer(expr.value()))) {
-      // Builtin c_ptr and c_funptr have special handling because designators
-      // and NULL() are handled as initial values for them as an extension
-      // (otherwise only c_ptr_null/c_funptr_null are allowed and these are
-      // replaced by structure constructors by semantics, so GetLastSymbol
-      // returns nothing).
-
-      // The Ev::Expr is an initializer that is a pointer target (e.g., 'x' or
-      // NULL()) that must be inserted into an intermediate cptr record value's
-      // address field, which ought to be an intptr_t on the target.
-      mlir::Value addr = fir::getBase(Fortran::lower::genExtAddrInInitializer(
-          converter, loc, expr.value()));
-      if (addr.getType().isa<fir::BoxProcType>())
-        addr = builder.create<fir::BoxAddrOp>(loc, addr);
-      assert((fir::isa_ref_type(addr.getType()) ||
-              addr.getType().isa<mlir::FunctionType>()) &&
-             "expect reference type for address field");
-      assert(fir::isa_derived(componentTy) &&
-             "expect C_PTR, C_FUNPTR to be a record");
-      auto cPtrRecTy = componentTy.cast<fir::RecordType>();
-      llvm::StringRef addrFieldName = Fortran::lower::builtin::cptrFieldName;
-      mlir::Type addrFieldTy = cPtrRecTy.getType(addrFieldName);
-      auto addrField = builder.create<fir::FieldIndexOp>(
-          loc, fieldTy, addrFieldName, componentTy,
-          /*typeParams=*/mlir::ValueRange{});
-      mlir::Value castAddr = builder.createConvert(loc, addrFieldTy, addr);
-      auto undef = builder.create<fir::UndefOp>(loc, componentTy);
-      addr = builder.create<fir::InsertValueOp>(
-          loc, componentTy, undef, castAddr,
-          builder.getArrayAttr(addrField.getAttributes()));
-      res = builder.create<fir::InsertValueOp>(
-          loc, recTy, res, addr, builder.getArrayAttr(field.getAttributes()));
-      continue;
+  const Fortran::semantics::DerivedTypeSpec *curentType = nullptr;
+  for (const auto &[sym, expr] : ctor.values()) {
+    // This TODO is not needed here anymore, but should be removed in a separate
+    // patch.
+    if (sym->test(Fortran::semantics::Symbol::Flag::ParentComp))
+      TODO(loc, "parent component in structure constructor");
+    const Fortran::semantics::DerivedTypeSpec *componentParentType =
+        sym->owner().derivedTypeSpec();
+    assert(componentParentType && "failed to retrieve component parent type");
+    if (!res) {
+      mlir::Type parentType = converter.genType(*componentParentType);
+      curentType = componentParentType;
+      res = builder.create<fir::UndefOp>(loc, parentType);
+    } else if (*componentParentType != *curentType) {
+      mlir::Type parentType = converter.genType(*componentParentType);
+      insertParentValueIntoExtension(parentType);
+      curentType = componentParentType;
     }
-
-    mlir::Value val =
-        fir::getBase(genConstantValue(converter, loc, expr.value()));
-    assert(!fir::isa_ref_type(val.getType()) && "expecting a constant value");
-    mlir::Value castVal = builder.createConvert(loc, componentTy, val);
-    res = builder.create<fir::InsertValueOp>(
-        loc, recTy, res, castVal, builder.getArrayAttr(field.getAttributes()));
+    res = genStructureComponentInit(converter, loc, sym, expr.value(), res);
   }
+
+  if (!res) // structure constructor for empty type.
+    return builder.create<fir::UndefOp>(loc, recTy);
+
+  // The last component may belong to a parent type.
+  if (res.getType() != recTy)
+    insertParentValueIntoExtension(recTy);
   return res;
 }
 
diff --git a/flang/lib/Lower/ConvertExpr.cpp b/flang/lib/Lower/ConvertExpr.cpp
index 8788e82b59..6d2ac62b61 100644
--- a/flang/lib/Lower/ConvertExpr.cpp
+++ b/flang/lib/Lower/ConvertExpr.cpp
@@ -3814,7 +3814,7 @@ class ArrayExprLowering {
     return false;
   }
   bool genShapeFromDataRef(const Fortran::evaluate::CoarrayRef &) {
-    TODO(getLoc(), "coarray ref");
+    TODO(getLoc(), "coarray: reference to a coarray in an expression");
     return false;
   }
   bool genShapeFromDataRef(const Fortran::evaluate::Component &x) {
@@ -7091,7 +7091,7 @@ class ArrayExprLowering {
   }
 
   CC genarr(const Fortran::evaluate::CoarrayRef &x, ComponentPath &components) {
-    TODO(getLoc(), "coarray reference");
+    TODO(getLoc(), "coarray: reference to a coarray in an expression");
   }
 
   CC genarr(const Fortran::evaluate::NamedEntity &x,
diff --git a/flang/lib/Lower/ConvertExprToHLFIR.cpp b/flang/lib/Lower/ConvertExprToHLFIR.cpp
index 44c9eb1e91..4cf29c9aec 100644
--- a/flang/lib/Lower/ConvertExprToHLFIR.cpp
+++ b/flang/lib/Lower/ConvertExprToHLFIR.cpp
@@ -303,63 +303,19 @@ class HlfirDesignatorBuilder {
   }
 
   fir::FortranVariableOpInterface
-  gen(const Fortran::evaluate::Component &component,
-      bool skipParentComponent = false) {
+  gen(const Fortran::evaluate::Component &component) {
     if (Fortran::semantics::IsAllocatableOrPointer(component.GetLastSymbol()))
       return genWholeAllocatableOrPointerComponent(component);
-    if (component.GetLastSymbol().test(
-            Fortran::semantics::Symbol::Flag::ParentComp)) {
-      if (skipParentComponent)
-        // Inner parent components can be skipped: x%parent_comp%i is equivalent
-        // to "x%i" in FIR (all the parent components are part of the FIR type
-        // of "x").
-        return genDataRefAndSkipParentComponents(component.base());
-      // This is a leaf "x%parent_comp" or "x(subscripts)%parent_comp" and
-      // cannot be skipped: the designator must be lowered to the parent type.
-      // This cannot be represented with an hlfir.designate since "parent_comp"
-      // name is meaningless in the fir.record type of "x". Instead, an
-      // hlfir.parent_comp is generated.
-      fir::FirOpBuilder &builder = getBuilder();
-      hlfir::Entity base = genDataRefAndSkipParentComponents(component.base());
-      base = derefPointersAndAllocatables(loc, builder, base);
-      mlir::Value shape;
-      if (base.isArray())
-        shape = hlfir::genShape(loc, builder, base);
-      const Fortran::semantics::DeclTypeSpec *declTypeSpec =
-          component.GetLastSymbol().GetType();
-      assert(declTypeSpec && declTypeSpec->AsDerived() &&
-             "parent component symbols must have a derived type");
-      mlir::Type componentType = Fortran::lower::translateDerivedTypeToFIRType(
-          getConverter(), *declTypeSpec->AsDerived());
-      mlir::Type resultType =
-          changeElementType(base.getElementOrSequenceType(), componentType);
-      // Note that the result is monomorphic even if the base is polymorphic:
-      // the dynamic type of the parent component reference is the parent type.
-      // If the base is an array, it is however most likely not contiguous.
-      if (base.isArray() || fir::isRecordWithTypeParameters(componentType))
-        resultType = fir::BoxType::get(resultType);
-      else
-        resultType = fir::ReferenceType::get(resultType);
-      if (fir::isRecordWithTypeParameters(componentType))
-        TODO(loc, "parent component reference with a parametrized parent type");
-      auto parentComp = builder.create<hlfir::ParentComponentOp>(
-          loc, resultType, base, shape, /*typeParams=*/mlir::ValueRange{});
-      return mlir::cast<fir::FortranVariableOpInterface>(
-          parentComp.getOperation());
-    }
     PartInfo partInfo;
     mlir::Type resultType = visit(component, partInfo);
     return genDesignate(resultType, partInfo, component);
   }
 
   fir::FortranVariableOpInterface
-  genDataRefAndSkipParentComponents(const Fortran::evaluate::DataRef &dataRef) {
-    return std::visit(Fortran::common::visitors{
-                          [&](const Fortran::evaluate::Component &component) {
-                            return gen(component, /*skipParentComponent=*/true);
-                          },
-                          [&](const auto &x) { return gen(x); }},
-                      dataRef.u);
+  gen(const Fortran::evaluate::DataRef &dataRef) {
+    return std::visit(
+        Fortran::common::visitors{[&](const auto &x) { return gen(x); }},
+        dataRef.u);
   }
 
   fir::FortranVariableOpInterface
@@ -371,11 +327,11 @@ class HlfirDesignatorBuilder {
 
   fir::FortranVariableOpInterface
   gen(const Fortran::evaluate::CoarrayRef &coarrayRef) {
-    TODO(getLoc(), "lowering CoarrayRef to HLFIR");
+    TODO(getLoc(), "coarray: lowering a reference to a coarray object");
   }
 
   mlir::Type visit(const Fortran::evaluate::CoarrayRef &, PartInfo &) {
-    TODO(getLoc(), "lowering CoarrayRef to HLFIR");
+    TODO(getLoc(), "coarray: lowering a reference to a coarray object");
   }
 
   fir::FortranVariableOpInterface
@@ -725,7 +681,7 @@ class HlfirDesignatorBuilder {
     // coarray-ref, or another component, this creates another hlfir.designate
     // for it.  hlfir.designate is not meant to represent more than one
     // part-ref.
-    partInfo.base = genDataRefAndSkipParentComponents(component.base());
+    partInfo.base = gen(component.base());
     // If the base is an allocatable/pointer, dereference it here since the
     // component ref designates its target.
     partInfo.base =
@@ -739,9 +695,6 @@ class HlfirDesignatorBuilder {
     // Lower the information about the component (type, length parameters and
     // shape).
     const Fortran::semantics::Symbol &componentSym = component.GetLastSymbol();
-    assert(
-        !componentSym.test(Fortran::semantics::Symbol::Flag::ParentComp) &&
-        "parent components are skipped and must not reach visitComponentImpl");
     partInfo.componentName = converter.getRecordTypeFieldName(componentSym);
     auto recordType =
         hlfir::getFortranElementType(baseType).cast<fir::RecordType>();
@@ -1697,7 +1650,7 @@ class HlfirBuilder {
   // Construct an entity holding the value specified by the
   // StructureConstructor. The initialization of the temporary entity
   // is done component by component with the help of HLFIR operations
-  // ParentComponentOp, DesignateOp and AssignOp.
+  // DesignateOp and AssignOp.
   hlfir::EntityWithAttributes
   gen(const Fortran::evaluate::StructureConstructor &ctor) {
     mlir::Location loc = getLoc();
@@ -1720,35 +1673,58 @@ class HlfirBuilder {
     mlir::Value box = builder.createBox(loc, fir::ExtendedValue{varOp});
     fir::runtime::genDerivedTypeInitialize(builder, loc, box);
 
+    // StructureConstructor values may relate to name of components in parent
+    // types. These components cannot be addressed directly, the parent
+    // components must be addressed first. The loop below creates all the
+    // required chains of hlfir.designate to address the parent components so
+    // that the StructureConstructor can later be lowered by addressing these
+    // parent components if needed. Note: the front-end orders the components in
+    // structure constructors. The code below relies on the component to appear
+    // in order.
+    using ValueAndParent = std::tuple<const Fortran::lower::SomeExpr &,
+                                      const Fortran::semantics::Symbol &,
+                                      hlfir::EntityWithAttributes>;
+    llvm::SmallVector<ValueAndParent> valuesAndParents;
+    Fortran::lower::ComponentReverseIterator compIterator(
+        ctor.result().derivedTypeSpec());
+    hlfir::EntityWithAttributes currentParent = varOp;
+    for (const auto &value : llvm::reverse(ctor.values())) {
+      const Fortran::semantics::Symbol &compSym = *value.first;
+      while (!compIterator.lookup(compSym.name())) {
+        const auto &parentType = compIterator.advanceToParentType();
+        llvm::StringRef parentName = toStringRef(parentType.name());
+        auto baseRecTy = mlir::cast<fir::RecordType>(
+            hlfir::getFortranElementType(currentParent.getType()));
+        auto parentCompType = baseRecTy.getType(parentName);
+        assert(parentCompType && "failed to retrieve parent component type");
+        mlir::Type designatorType = builder.getRefType(parentCompType);
+        mlir::Value newParent = builder.create<hlfir::DesignateOp>(
+            loc, designatorType, currentParent, parentName,
+            /*compShape=*/mlir::Value{}, hlfir::DesignateOp::Subscripts{},
+            /*substring=*/mlir::ValueRange{},
+            /*complexPart=*/std::nullopt,
+            /*shape=*/mlir::Value{}, /*typeParams=*/mlir::ValueRange{},
+            fir::FortranVariableFlagsAttr{});
+        currentParent = hlfir::EntityWithAttributes{newParent};
+      }
+      valuesAndParents.emplace_back(
+          ValueAndParent{value.second.value(), compSym, currentParent});
+    }
+
     HlfirDesignatorBuilder designatorBuilder(loc, converter, symMap, stmtCtx);
-    for (const auto &value : ctor.values()) {
-      const Fortran::semantics::Symbol &sym = *value.first;
-      const Fortran::lower::SomeExpr &expr = value.second.value();
+    for (const auto &iter : llvm::reverse(valuesAndParents)) {
+      auto &sym = std::get<const Fortran::semantics::Symbol &>(iter);
+      auto &expr = std::get<const Fortran::lower::SomeExpr &>(iter);
+      auto &baseOp = std::get<hlfir::EntityWithAttributes>(iter);
       std::string name = converter.getRecordTypeFieldName(sym);
-      if (sym.test(Fortran::semantics::Symbol::Flag::ParentComp)) {
-        const Fortran::semantics::DeclTypeSpec *declTypeSpec = sym.GetType();
-        assert(declTypeSpec && declTypeSpec->AsDerived() &&
-               "parent component symbol must have a derived type");
-        mlir::Type compType = Fortran::lower::translateDerivedTypeToFIRType(
-            converter, *declTypeSpec->AsDerived());
-        if (fir::isRecordWithTypeParameters(compType))
-          TODO(loc,
-               "parent component reference with a parameterized parent type");
-        mlir::Type resultType = builder.getRefType(compType);
-        auto lhs = builder.create<hlfir::ParentComponentOp>(
-            loc, resultType, varOp, /*shape=*/nullptr,
-            /*typeparams=*/mlir::ValueRange{});
-        auto rhs = gen(expr);
-        builder.create<hlfir::AssignOp>(loc, rhs, lhs, /*realloc=*/false,
-                                        /*keep_lhs_length_if_realloc=*/false,
-                                        /*temporary_lhs=*/true);
-        continue;
-      }
 
       // Generate DesignateOp for the component.
       // The designator's result type is just a reference to the component type,
       // because the whole component is being designated.
-      auto compType = recTy.getType(name);
+      auto baseRecTy = mlir::cast<fir::RecordType>(
+          hlfir::getFortranElementType(baseOp.getType()));
+      auto compType = baseRecTy.getType(name);
+      assert(compType && "failed to retrieve component type");
       mlir::Value compShape =
           designatorBuilder.genComponentShape(sym, compType);
       mlir::Type designatorType = builder.getRefType(compType);
@@ -1771,7 +1747,7 @@ class HlfirBuilder {
 
       // Get the component designator.
       auto lhs = builder.create<hlfir::DesignateOp>(
-          loc, designatorType, varOp, name, compShape,
+          loc, designatorType, baseOp, name, compShape,
           hlfir::DesignateOp::Subscripts{},
           /*substring=*/mlir::ValueRange{},
           /*complexPart=*/std::nullopt,
diff --git a/flang/lib/Lower/ConvertType.cpp b/flang/lib/Lower/ConvertType.cpp
index 22b83efe86..1ed3b60262 100644
--- a/flang/lib/Lower/ConvertType.cpp
+++ b/flang/lib/Lower/ConvertType.cpp
@@ -376,31 +376,56 @@ struct TypeBuilderImpl {
       return genVectorType(tySpec);
     }
 
+    const Fortran::semantics::Scope &derivedScope = DEREF(tySpec.GetScope());
+
     auto rec = fir::RecordType::get(context, converter.mangleName(tySpec));
     // Maintain the stack of types for recursive references.
     derivedTypeInConstruction.emplace_back(typeSymbol, rec);
 
     // Gather the record type fields.
     // (1) The data components.
-    for (const auto &component :
-         Fortran::semantics::OrderedComponentIterator(tySpec)) {
-      // Lowering is assuming non deferred component lower bounds are always 1.
-      // Catch any situations where this is not true for now.
-      if (!converter.getLoweringOptions().getLowerToHighLevelFIR() &&
-          componentHasNonDefaultLowerBounds(component))
-        TODO(converter.genLocation(component.name()),
-             "derived type components with non default lower bounds");
-      if (IsProcedure(component))
-        TODO(converter.genLocation(component.name()), "procedure components");
-      mlir::Type ty = genSymbolType(component);
-      // Do not add the parent component (component of the parents are
-      // added and should be sufficient, the parent component would
-      // duplicate the fields). Note that genSymbolType must be called above on
-      // it so that the dispatch table for the parent type still gets emitted
-      // as needed.
-      if (component.test(Fortran::semantics::Symbol::Flag::ParentComp))
-        continue;
-      cs.emplace_back(converter.getRecordTypeFieldName(component), ty);
+    if (converter.getLoweringOptions().getLowerToHighLevelFIR()) {
+      // In HLFIR the parent component is the first fir.type component.
+      for (const auto &componentName :
+           typeSymbol.get<Fortran::semantics::DerivedTypeDetails>()
+               .componentNames()) {
+        auto scopeIter = derivedScope.find(componentName);
+        assert(scopeIter != derivedScope.cend() &&
+               "failed to find derived type component symbol");
+        const Fortran::semantics::Symbol &component = scopeIter->second.get();
+        if (IsProcedure(component))
+          TODO(converter.genLocation(component.name()), "procedure components");
+        mlir::Type ty = genSymbolType(component);
+        cs.emplace_back(converter.getRecordTypeFieldName(component), ty);
+      }
+    } else {
+      for (const auto &component :
+           Fortran::semantics::OrderedComponentIterator(tySpec)) {
+        // In the lowering to FIR the parent component does not appear in the
+        // fir.type and its components are inlined at the beginning of the
+        // fir.type<>.
+        // FIXME: this strategy leads to bugs because padding should be inserted
+        // after the component of the parents so that the next components do not
+        // end-up in the parent storage if the sum of the parent's component
+        // storage size is not a multiple of the parent type storage alignment.
+
+        // Lowering is assuming non deferred component lower bounds are
+        // always 1. Catch any situations where this is not true for now.
+        if (componentHasNonDefaultLowerBounds(component))
+          TODO(converter.genLocation(component.name()),
+               "derived type components with non default lower bounds");
+        if (IsProcedure(component))
+          TODO(converter.genLocation(component.name()), "procedure components");
+        mlir::Type ty = genSymbolType(component);
+        // Do not add the parent component (component of the parents are
+        // added and should be sufficient, the parent component would
+        // duplicate the fields). Note that genSymbolType must be called above
+        // on it so that the dispatch table for the parent type still gets
+        // emitted as needed.
+        if (component.test(Fortran::semantics::Symbol::Flag::ParentComp))
+          continue;
+        cs.emplace_back(converter.getRecordTypeFieldName(component), ty);
+      }
     }
 
     mlir::Location loc = converter.genLocation(typeSymbol.name());
@@ -427,11 +452,9 @@ struct TypeBuilderImpl {
     LLVM_DEBUG(llvm::dbgs() << "derived type: " << rec << '\n');
 
     // Generate the type descriptor object if any
-    if (const Fortran::semantics::Scope *derivedScope =
-            tySpec.scope() ? tySpec.scope() : tySpec.typeSymbol().scope())
-      if (const Fortran::semantics::Symbol *typeInfoSym =
-              derivedScope->runtimeDerivedTypeDescription())
-        converter.registerTypeInfo(loc, *typeInfoSym, tySpec, rec);
+    if (const Fortran::semantics::Symbol *typeInfoSym =
+            derivedScope.runtimeDerivedTypeDescription())
+      converter.registerTypeInfo(loc, *typeInfoSym, tySpec, rec);
     return rec;
   }
 
@@ -596,6 +619,25 @@ mlir::Type Fortran::lower::TypeBuilder<T>::genType(
   return TypeBuilderImpl{converter}.genExprType(funcRef);
 }
 
+const Fortran::semantics::DerivedTypeSpec &
+Fortran::lower::ComponentReverseIterator::advanceToParentType() {
+  const Fortran::semantics::Scope *scope = currentParentType->GetScope();
+  auto parentComp =
+      DEREF(scope).find(currentTypeDetails->GetParentComponentName().value());
+  assert(parentComp != scope->cend() && "failed to get parent component");
+  setCurrentType(parentComp->second->GetType()->derivedTypeSpec());
+  return *currentParentType;
+}
+
+void Fortran::lower::ComponentReverseIterator::setCurrentType(
+    const Fortran::semantics::DerivedTypeSpec &derived) {
+  currentParentType = &derived;
+  currentTypeDetails = &currentParentType->typeSymbol()
+                            .get<Fortran::semantics::DerivedTypeDetails>();
+  componentIt = currentTypeDetails->componentNames().crbegin();
+  componentItEnd = currentTypeDetails->componentNames().crend();
+}
+
 using namespace Fortran::evaluate;
 using namespace Fortran::common;
 FOR_EACH_SPECIFIC_TYPE(template class Fortran::lower::TypeBuilder, )
diff --git a/flang/lib/Lower/ConvertVariable.cpp b/flang/lib/Lower/ConvertVariable.cpp
index 46a59b38ae..57fb9fc432 100644
--- a/flang/lib/Lower/ConvertVariable.cpp
+++ b/flang/lib/Lower/ConvertVariable.cpp
@@ -309,6 +309,74 @@ mlir::Value Fortran::lower::genInitialDataTarget(
                                       /*slice=*/mlir::Value{});
 }
 
+/// Generate default initial value for a derived type object \p sym with mlir
+/// type \p symTy.
+static mlir::Value genDefaultInitializerValue(
+    Fortran::lower::AbstractConverter &converter, mlir::Location loc,
+    const Fortran::semantics::Symbol &sym, mlir::Type symTy,
+    Fortran::lower::StatementContext &stmtCtx);
+
+/// Generate the initial value of a derived component \p component and insert
+/// it into the derived type initial value \p insertInto of type \p recTy.
+/// Return the new derived type initial value after the insertion.
+static mlir::Value genComponentDefaultInit(
+    Fortran::lower::AbstractConverter &converter, mlir::Location loc,
+    const Fortran::semantics::Symbol &component, fir::RecordType recTy,
+    mlir::Value insertInto, Fortran::lower::StatementContext &stmtCtx) {
+  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
+  std::string name = converter.getRecordTypeFieldName(component);
+  mlir::Type componentTy = recTy.getType(name);
+  assert(componentTy && "component not found in type");
+  mlir::Value componentValue;
+  if (const auto *object{
+          component.detailsIf<Fortran::semantics::ObjectEntityDetails>()}) {
+    if (const auto &init = object->init()) {
+      // Component has explicit initialization.
+      if (Fortran::semantics::IsPointer(component))
+        // Initial data target.
+        componentValue =
+            genInitialDataTarget(converter, loc, componentTy, *init);
+      else
+        // Initial value.
+        componentValue = fir::getBase(
+            genInitializerExprValue(converter, loc, *init, stmtCtx));
+    } else if (Fortran::semantics::IsAllocatableOrPointer(component)) {
+      // Pointer or allocatable without initialization.
+      // Create deallocated/disassociated value.
+      // From a standard point of view, pointer without initialization do not
+      // need to be disassociated, but for sanity and simplicity, do it in
+      // global constructor since this has no runtime cost.
+      componentValue = fir::factory::createUnallocatedBox(
+          builder, loc, componentTy, std::nullopt);
+    } else if (hasDefaultInitialization(component)) {
+      // Component type has default initialization.
+      componentValue = genDefaultInitializerValue(converter, loc, component,
+                                                  componentTy, stmtCtx);
+    } else {
+      // Component has no initial value. Set its bits to zero by extension
+      // to match what is expected because other compilers are doing it.
+      componentValue = builder.create<fir::ZeroOp>(loc, componentTy);
+    }
+  } else if (const auto *proc{
+                 component
+                     .detailsIf<Fortran::semantics::ProcEntityDetails>()}) {
+    if (proc->init().has_value())
+      TODO(loc, "procedure pointer component default initialization");
+    else
+      componentValue = builder.create<fir::ZeroOp>(loc, componentTy);
+  }
+  assert(componentValue && "must have been computed");
+  componentValue = builder.createConvert(loc, componentTy, componentValue);
+  auto fieldTy = fir::FieldType::get(recTy.getContext());
+  // FIXME: type parameters must come from the derived-type-spec
+  auto field = builder.create<fir::FieldIndexOp>(
+      loc, fieldTy, name, recTy,
+      /*typeParams=*/mlir::ValueRange{} /*TODO*/);
+  return builder.create<fir::InsertValueOp>(
+      loc, recTy, insertInto, componentValue,
+      builder.getArrayAttr(field.getAttributes()));
+}
+
 static mlir::Value genDefaultInitializerValue(
     Fortran::lower::AbstractConverter &converter, mlir::Location loc,
     const Fortran::semantics::Symbol &sym, mlir::Type symTy,
@@ -323,67 +391,40 @@ static mlir::Value genDefaultInitializerValue(
   // Build a scalar default value of the symbol type, looping through the
   // components to build each component initial value.
   auto recTy = scalarType.cast<fir::RecordType>();
-  auto fieldTy = fir::FieldType::get(scalarType.getContext());
   mlir::Value initialValue = builder.create<fir::UndefOp>(loc, scalarType);
   const Fortran::semantics::DeclTypeSpec *declTy = sym.GetType();
   assert(declTy && "var with default initialization must have a type");
-  Fortran::semantics::OrderedComponentIterator components(
-      declTy->derivedTypeSpec());
-  for (const auto &component : components) {
-    // Skip parent components, the sub-components of parent types are part of
-    // components and will be looped through right after.
-    if (component.test(Fortran::semantics::Symbol::Flag::ParentComp))
-      continue;
-    mlir::Value componentValue;
-    std::string name = converter.getRecordTypeFieldName(component);
-    mlir::Type componentTy = recTy.getType(name);
-    assert(componentTy && "component not found in type");
-    if (const auto *object{
-            component.detailsIf<Fortran::semantics::ObjectEntityDetails>()}) {
-      if (const auto &init = object->init()) {
-        // Component has explicit initialization.
-        if (Fortran::semantics::IsPointer(component))
-          // Initial data target.
-          componentValue =
-              genInitialDataTarget(converter, loc, componentTy, *init);
-        else
-          // Initial value.
-          componentValue = fir::getBase(
-              genInitializerExprValue(converter, loc, *init, stmtCtx));
-      } else if (Fortran::semantics::IsAllocatableOrPointer(component)) {
-        // Pointer or allocatable without initialization.
-        // Create deallocated/disassociated value.
-        // From a standard point of view, pointer without initialization do not
-        // need to be disassociated, but for sanity and simplicity, do it in
-        // global constructor since this has no runtime cost.
-        componentValue = fir::factory::createUnallocatedBox(
-            builder, loc, componentTy, std::nullopt);
-      } else if (hasDefaultInitialization(component)) {
-        // Component type has default initialization.
-        componentValue = genDefaultInitializerValue(converter, loc, component,
-                                                    componentTy, stmtCtx);
-      } else {
-        // Component has no initial value. Set its bits to zero by extension
-        // to match what is expected because other compilers are doing it.
-        componentValue = builder.create<fir::ZeroOp>(loc, componentTy);
-      }
-    } else if (const auto *proc{
-                   component
-                       .detailsIf<Fortran::semantics::ProcEntityDetails>()}) {
-      if (proc->init().has_value())
-        TODO(loc, "procedure pointer component default initialization");
-      else
-        componentValue = builder.create<fir::ZeroOp>(loc, componentTy);
+
+  if (converter.getLoweringOptions().getLowerToHighLevelFIR()) {
+    // In HLFIR, the parent type is the first component, while in FIR there is
+    // not parent component in the fir.type and the component of the parent are
+    // "inlined" at the beginning of the fir.type.
+    const Fortran::semantics::Symbol &typeSymbol =
+        declTy->derivedTypeSpec().typeSymbol();
+    const Fortran::semantics::Scope *derivedScope =
+        declTy->derivedTypeSpec().GetScope();
+    assert(derivedScope && "failed to retrieve derived type scope");
+    for (const auto &componentName :
+         typeSymbol.get<Fortran::semantics::DerivedTypeDetails>()
+             .componentNames()) {
+      auto scopeIter = derivedScope->find(componentName);
+      assert(scopeIter != derivedScope->cend() &&
+             "failed to find derived type component symbol");
+      const Fortran::semantics::Symbol &component = scopeIter->second.get();
+      initialValue = genComponentDefaultInit(converter, loc, component, recTy,
+                                             initialValue, stmtCtx);
+    }
+  } else {
+    Fortran::semantics::OrderedComponentIterator components(
+        declTy->derivedTypeSpec());
+    for (const auto &component : components) {
+      // Skip parent components, the sub-components of parent types are part of
+      // components and will be looped through right after.
+      if (component.test(Fortran::semantics::Symbol::Flag::ParentComp))
+        continue;
+      initialValue = genComponentDefaultInit(converter, loc, component, recTy,
+                                             initialValue, stmtCtx);
     }
-    assert(componentValue && "must have been computed");
-    componentValue = builder.createConvert(loc, componentTy, componentValue);
-    // FIXME: type parameters must come from the derived-type-spec
-    auto field = builder.create<fir::FieldIndexOp>(
-        loc, fieldTy, name, scalarType,
-        /*typeParams=*/mlir::ValueRange{} /*TODO*/);
-    initialValue = builder.create<fir::InsertValueOp>(
-        loc, recTy, initialValue, componentValue,
-        builder.getArrayAttr(field.getAttributes()));
   }
 
   if (sequenceType) {
@@ -714,7 +755,10 @@ needDummyIntentoutFinalization(const Fortran::lower::pft::Variable &var) {
     return true;
   // Intent(out) dummies must be finalized at runtime if their type has a
   // finalization.
-  return hasFinalization(sym);
+  // Allocatable components of INTENT(OUT) dummies must be deallocated (9.7.3.2
+  // p6). Calling finalization runtime for this works even if the components
+  // have no final procedures.
+  return hasFinalization(sym) || hasAllocatableDirectComponent(sym);
 }
 
 /// Call default initialization runtime routine to initialize \p var.
@@ -747,6 +791,9 @@ static void finalizeAtRuntime(Fortran::lower::AbstractConverter &converter,
 // is deallocated; any allocated allocatable object that is a subobject of an
 // actual argument corresponding to an INTENT(OUT) dummy argument is
 // deallocated.
+// Note that allocatable components of non-ALLOCATABLE INTENT(OUT) dummy
+// arguments are dealt with needDummyIntentoutFinalization (finalization runtime
+// is called to reach the intended component deallocation effect).
 static void deallocateIntentOut(Fortran::lower::AbstractConverter &converter,
                                 const Fortran::lower::pft::Variable &var,
                                 Fortran::lower::SymMap &symMap) {
diff --git a/flang/lib/Lower/DirectivesCommon.h b/flang/lib/Lower/DirectivesCommon.h
index 535ec1c03b..ed44598bc9 100644
--- a/flang/lib/Lower/DirectivesCommon.h
+++ b/flang/lib/Lower/DirectivesCommon.h
@@ -879,8 +879,17 @@ mlir::Value gatherDataOperandAddrAndBounds(
                       builder, operandLocation, converter, compExv, baseAddr);
                 asFortran << (*expr).AsFortran();
 
+                if (auto loadOp = mlir::dyn_cast_or_null<fir::LoadOp>(
+                        baseAddr.getDefiningOp())) {
+                  if (fir::isAllocatableType(loadOp.getType()) ||
+                      fir::isPointerType(loadOp.getType()))
+                    baseAddr = builder.create<fir::BoxAddrOp>(operandLocation,
+                                                              baseAddr);
+                }
+
                 // If the component is an allocatable or pointer the result of
-                // genExprAddr will be the result of a fir.box_addr operation.
+                // genExprAddr will be the result of a fir.box_addr operation or
+                // a fir.box_addr has been inserted just before.
                 // Retrieve the box so we handle it like other descriptor.
                 if (auto boxAddrOp = mlir::dyn_cast_or_null<fir::BoxAddrOp>(
                         baseAddr.getDefiningOp())) {
diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index 61a1b9fd86..4fafcebc30 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -463,7 +463,7 @@ bool isConstantBound(mlir::acc::DataBoundsOp &op) {
 }
 
 /// Return true iff all the bounds are expressed with constant values.
-bool areAllBoundConstant(llvm::SmallVector<mlir::Value> &bounds) {
+bool areAllBoundConstant(const llvm::SmallVector<mlir::Value> &bounds) {
   for (auto bound : bounds) {
     auto dataBound =
         mlir::dyn_cast<mlir::acc::DataBoundsOp>(bound.getDefiningOp());
@@ -474,27 +474,6 @@ bool areAllBoundConstant(llvm::SmallVector<mlir::Value> &bounds) {
   return true;
 }
 
-static fir::ShapeOp
-genShapeFromBounds(mlir::Location loc, fir::FirOpBuilder &builder,
-                   const llvm::SmallVector<mlir::Value> &args) {
-  assert(args.size() % 3 == 0 && "Triplets must be a multiple of 3");
-  llvm::SmallVector<mlir::Value> extents;
-  mlir::Type idxTy = builder.getIndexType();
-  mlir::Value one = builder.createIntegerConstant(loc, idxTy, 1);
-  mlir::Value zero = builder.createIntegerConstant(loc, idxTy, 0);
-  for (unsigned i = 0; i < args.size(); i += 3) {
-    mlir::Value s1 =
-        builder.create<mlir::arith::SubIOp>(loc, args[i + 1], args[0]);
-    mlir::Value s2 = builder.create<mlir::arith::AddIOp>(loc, s1, one);
-    mlir::Value s3 = builder.create<mlir::arith::DivSIOp>(loc, s2, args[i + 2]);
-    mlir::Value cmp = builder.create<mlir::arith::CmpIOp>(
-        loc, mlir::arith::CmpIPredicate::sgt, s3, zero);
-    mlir::Value ext = builder.create<mlir::arith::SelectOp>(loc, cmp, s3, zero);
-    extents.push_back(ext);
-  }
-  return builder.create<fir::ShapeOp>(loc, extents);
-}
-
 static llvm::SmallVector<mlir::Value>
 genConstantBounds(fir::FirOpBuilder &builder, mlir::Location loc,
                   mlir::acc::DataBoundsOp &dataBound) {
@@ -520,6 +499,63 @@ genConstantBounds(fir::FirOpBuilder &builder, mlir::Location loc,
   return {lb, ub, step};
 }
 
+static fir::ShapeOp genShapeFromBoundsOrArgs(
+    mlir::Location loc, fir::FirOpBuilder &builder, fir::SequenceType seqTy,
+    const llvm::SmallVector<mlir::Value> &bounds, mlir::ValueRange arguments) {
+  llvm::SmallVector<mlir::Value> args;
+  if (areAllBoundConstant(bounds)) {
+    for (auto bound : llvm::reverse(bounds)) {
+      auto dataBound =
+          mlir::cast<mlir::acc::DataBoundsOp>(bound.getDefiningOp());
+      args.append(genConstantBounds(builder, loc, dataBound));
+    }
+  } else {
+    assert(((arguments.size() - 2) / 3 == seqTy.getDimension()) &&
+           "Expect 3 block arguments per dimension");
+    for (auto arg : arguments.drop_front(2))
+      args.push_back(arg);
+  }
+
+  assert(args.size() % 3 == 0 && "Triplets must be a multiple of 3");
+  llvm::SmallVector<mlir::Value> extents;
+  mlir::Type idxTy = builder.getIndexType();
+  mlir::Value one = builder.createIntegerConstant(loc, idxTy, 1);
+  mlir::Value zero = builder.createIntegerConstant(loc, idxTy, 0);
+  for (unsigned i = 0; i < args.size(); i += 3) {
+    mlir::Value s1 =
+        builder.create<mlir::arith::SubIOp>(loc, args[i + 1], args[0]);
+    mlir::Value s2 = builder.create<mlir::arith::AddIOp>(loc, s1, one);
+    mlir::Value s3 = builder.create<mlir::arith::DivSIOp>(loc, s2, args[i + 2]);
+    mlir::Value cmp = builder.create<mlir::arith::CmpIOp>(
+        loc, mlir::arith::CmpIPredicate::sgt, s3, zero);
+    mlir::Value ext = builder.create<mlir::arith::SelectOp>(loc, cmp, s3, zero);
+    extents.push_back(ext);
+  }
+  return builder.create<fir::ShapeOp>(loc, extents);
+}
+
+static hlfir::DesignateOp::Subscripts
+getSubscriptsFromArgs(mlir::ValueRange args) {
+  hlfir::DesignateOp::Subscripts triplets;
+  for (unsigned i = 2; i < args.size(); i += 3)
+    triplets.emplace_back(
+        hlfir::DesignateOp::Triplet{args[i], args[i + 1], args[i + 2]});
+  return triplets;
+}
+
+static hlfir::Entity genDesignateWithTriplets(
+    fir::FirOpBuilder &builder, mlir::Location loc, hlfir::Entity &entity,
+    hlfir::DesignateOp::Subscripts &triplets, mlir::Value shape) {
+  llvm::SmallVector<mlir::Value> lenParams;
+  hlfir::genLengthParameters(loc, builder, entity, lenParams);
+  auto designate = builder.create<hlfir::DesignateOp>(
+      loc, entity.getBase().getType(), entity, /*component=*/"",
+      /*componentShape=*/mlir::Value{}, triplets,
+      /*substring=*/mlir::ValueRange{}, /*complexPartAttr=*/std::nullopt, shape,
+      lenParams);
+  return hlfir::Entity{designate.getResult()};
+}
+
 mlir::acc::FirstprivateRecipeOp Fortran::lower::createOrGetFirstprivateRecipe(
     mlir::OpBuilder &builder, llvm::StringRef recipeName, mlir::Location loc,
     mlir::Type ty, llvm::SmallVector<mlir::Value> &bounds) {
@@ -533,8 +569,20 @@ mlir::acc::FirstprivateRecipeOp Fortran::lower::createOrGetFirstprivateRecipe(
   mlir::OpBuilder modBuilder(mod.getBodyRegion());
   auto recipe =
       modBuilder.create<mlir::acc::FirstprivateRecipeOp>(loc, recipeName, ty);
+  llvm::SmallVector<mlir::Type> initArgsTy{ty};
+  llvm::SmallVector<mlir::Location> initArgsLoc{loc};
+  auto refTy = fir::unwrapRefType(ty);
+  if (auto seqTy = mlir::dyn_cast_or_null<fir::SequenceType>(refTy)) {
+    if (seqTy.hasDynamicExtents()) {
+      mlir::Type idxTy = builder.getIndexType();
+      for (unsigned i = 0; i < seqTy.getDimension(); ++i) {
+        initArgsTy.push_back(idxTy);
+        initArgsLoc.push_back(loc);
+      }
+    }
+  }
   builder.createBlock(&recipe.getInitRegion(), recipe.getInitRegion().end(),
-                      {ty}, {loc});
+                      initArgsTy, initArgsLoc);
   builder.setInsertionPointToEnd(&recipe.getInitRegion().back());
   genPrivateLikeInitRegion<mlir::acc::FirstprivateRecipeOp>(builder, recipe, ty,
                                                             loc);
@@ -565,32 +613,28 @@ mlir::acc::FirstprivateRecipeOp Fortran::lower::createOrGetFirstprivateRecipe(
     builder.create<fir::StoreOp>(loc, initValue,
                                  recipe.getCopyRegion().front().getArgument(1));
   } else if (auto seqTy = mlir::dyn_cast_or_null<fir::SequenceType>(ty)) {
-    if (seqTy.hasDynamicExtents())
-      TODO(loc, "firstprivate recipe of array with dynamic extents");
-    mlir::Type idxTy = builder.getIndexType();
-    mlir::Type refTy = fir::ReferenceType::get(seqTy.getEleTy());
-    mlir::Value arraySrc = recipe.getCopyRegion().front().getArgument(0);
-    mlir::Value arrayDst = recipe.getCopyRegion().front().getArgument(1);
-    llvm::SmallVector<fir::DoLoopOp> loops;
-    llvm::SmallVector<mlir::Value> ivs;
-    for (auto ext : llvm::reverse(seqTy.getShape())) {
-      auto lb = builder.create<mlir::arith::ConstantOp>(
-          loc, idxTy, builder.getIntegerAttr(idxTy, 0));
-      auto ub = builder.create<mlir::arith::ConstantOp>(
-          loc, idxTy, builder.getIntegerAttr(idxTy, ext - 1));
-      auto step = builder.create<mlir::arith::ConstantOp>(
-          loc, idxTy, builder.getIntegerAttr(idxTy, 1));
-      auto loop = builder.create<fir::DoLoopOp>(loc, lb, ub, step,
-                                                /*unordered=*/false);
-      builder.setInsertionPointToStart(loop.getBody());
-      loops.push_back(loop);
-      ivs.push_back(loop.getInductionVar());
-    }
-    auto addr1 = builder.create<fir::CoordinateOp>(loc, refTy, arraySrc, ivs);
-    auto addr2 = builder.create<fir::CoordinateOp>(loc, refTy, arrayDst, ivs);
-    auto loadedValue = builder.create<fir::LoadOp>(loc, addr1);
-    builder.create<fir::StoreOp>(loc, loadedValue, addr2);
-    builder.setInsertionPointAfter(loops[0]);
+    fir::FirOpBuilder firBuilder{builder, recipe.getOperation()};
+    auto shape = genShapeFromBoundsOrArgs(
+        loc, firBuilder, seqTy, bounds, recipe.getCopyRegion().getArguments());
+
+    auto leftDeclOp = builder.create<hlfir::DeclareOp>(
+        loc, recipe.getCopyRegion().getArgument(0), llvm::StringRef{}, shape,
+        llvm::ArrayRef<mlir::Value>{}, fir::FortranVariableFlagsAttr{});
+    auto rightDeclOp = builder.create<hlfir::DeclareOp>(
+        loc, recipe.getCopyRegion().getArgument(1), llvm::StringRef{}, shape,
+        llvm::ArrayRef<mlir::Value>{}, fir::FortranVariableFlagsAttr{});
+
+    hlfir::DesignateOp::Subscripts triplets =
+        getSubscriptsFromArgs(recipe.getCopyRegion().getArguments());
+    auto leftEntity = hlfir::Entity{leftDeclOp.getBase()};
+    auto left =
+        genDesignateWithTriplets(firBuilder, loc, leftEntity, triplets, shape);
+    auto rightEntity = hlfir::Entity{rightDeclOp.getBase()};
+    auto right =
+        genDesignateWithTriplets(firBuilder, loc, rightEntity, triplets, shape);
+
+    firBuilder.create<hlfir::AssignOp>(loc, left, right);
+
   } else if (auto boxTy = mlir::dyn_cast_or_null<fir::BaseBoxType>(ty)) {
     fir::FirOpBuilder firBuilder{builder, recipe.getOperation()};
     llvm::SmallVector<mlir::Value> tripletArgs;
@@ -600,47 +644,16 @@ mlir::acc::FirstprivateRecipeOp Fortran::lower::createOrGetFirstprivateRecipe(
     if (!seqTy)
       TODO(loc, "Unsupported boxed type in OpenACC firstprivate");
 
-    if (allConstantBound) {
-      for (auto bound : llvm::reverse(bounds)) {
-        auto dataBound =
-            mlir::cast<mlir::acc::DataBoundsOp>(bound.getDefiningOp());
-        tripletArgs.append(genConstantBounds(firBuilder, loc, dataBound));
-      }
-    } else {
-      assert(((recipe.getCopyRegion().getArguments().size() - 2) / 3 ==
-              seqTy.getDimension()) &&
-             "Expect 3 block arguments per dimension");
-      for (auto arg : recipe.getCopyRegion().getArguments().drop_front(2))
-        tripletArgs.push_back(arg);
-    }
-    auto shape = genShapeFromBounds(loc, firBuilder, tripletArgs);
-    hlfir::DesignateOp::Subscripts triplets;
-    for (unsigned i = 2; i < recipe.getCopyRegion().getArguments().size();
-         i += 3)
-      triplets.emplace_back(hlfir::DesignateOp::Triplet{
-          recipe.getCopyRegion().getArgument(i),
-          recipe.getCopyRegion().getArgument(i + 1),
-          recipe.getCopyRegion().getArgument(i + 2)});
-
-    llvm::SmallVector<mlir::Value> lenParamsLeft;
+    auto shape = genShapeFromBoundsOrArgs(
+        loc, firBuilder, seqTy, bounds, recipe.getCopyRegion().getArguments());
+    hlfir::DesignateOp::Subscripts triplets =
+        getSubscriptsFromArgs(recipe.getCopyRegion().getArguments());
     auto leftEntity = hlfir::Entity{recipe.getCopyRegion().getArgument(0)};
-    hlfir::genLengthParameters(loc, firBuilder, leftEntity, lenParamsLeft);
-    auto leftDesignate = firBuilder.create<hlfir::DesignateOp>(
-        loc, leftEntity.getBase().getType(), leftEntity, /*component=*/"",
-        /*componentShape=*/mlir::Value{}, triplets,
-        /*substring=*/mlir::ValueRange{}, /*complexPartAttr=*/std::nullopt,
-        shape, lenParamsLeft);
-    auto left = hlfir::Entity{leftDesignate.getResult()};
-
-    llvm::SmallVector<mlir::Value> lenParamsRight;
+    auto left =
+        genDesignateWithTriplets(firBuilder, loc, leftEntity, triplets, shape);
     auto rightEntity = hlfir::Entity{recipe.getCopyRegion().getArgument(1)};
-    hlfir::genLengthParameters(loc, firBuilder, rightEntity, lenParamsRight);
-    auto rightDesignate = firBuilder.create<hlfir::DesignateOp>(
-        loc, rightEntity.getBase().getType(), rightEntity, /*component=*/"",
-        /*componentShape=*/mlir::Value{}, triplets,
-        /*substring=*/mlir::ValueRange{}, /*complexPartAttr=*/std::nullopt,
-        shape, lenParamsRight);
-    auto right = hlfir::Entity{rightDesignate.getResult()};
+    auto right =
+        genDesignateWithTriplets(firBuilder, loc, rightEntity, triplets, shape);
     firBuilder.create<hlfir::AssignOp>(loc, left, right);
   }
 
@@ -915,18 +928,27 @@ static mlir::Value genReductionInitRegion(fir::FirOpBuilder &builder,
                                  declareOp.getBase());
     return declareOp.getBase();
   } else if (auto seqTy = mlir::dyn_cast_or_null<fir::SequenceType>(ty)) {
-    if (seqTy.hasDynamicExtents())
-      TODO(loc, "reduction recipe of array with dynamic extents");
     if (fir::isa_trivial(seqTy.getEleTy())) {
-      mlir::Value alloca = builder.create<fir::AllocaOp>(loc, seqTy);
-      auto shapeOp = genShapeOp(builder, seqTy, loc);
+      mlir::Value shape;
+      auto extents = builder.getBlock()->getArguments().drop_front(1);
+      if (seqTy.hasDynamicExtents())
+        shape = builder.create<fir::ShapeOp>(loc, extents);
+      else
+        shape = genShapeOp(builder, seqTy, loc);
+      mlir::Value alloca = builder.create<fir::AllocaOp>(
+          loc, seqTy, /*typeparams=*/mlir::ValueRange{}, extents);
       auto declareOp = builder.create<hlfir::DeclareOp>(
-          loc, alloca, accReductionInitName, shapeOp,
+          loc, alloca, accReductionInitName, shape,
           llvm::ArrayRef<mlir::Value>{}, fir::FortranVariableFlagsAttr{});
       mlir::Type idxTy = builder.getIndexType();
       mlir::Type refTy = fir::ReferenceType::get(seqTy.getEleTy());
       llvm::SmallVector<fir::DoLoopOp> loops;
       llvm::SmallVector<mlir::Value> ivs;
+
+      if (seqTy.hasDynamicExtents()) {
+        builder.create<hlfir::AssignOp>(loc, initValue, declareOp.getBase());
+        return declareOp.getBase();
+      }
       for (auto ext : llvm::reverse(seqTy.getShape())) {
         auto lb = builder.createIntegerConstant(loc, idxTy, 0);
         auto ub = builder.createIntegerConstant(loc, idxTy, ext - 1);
@@ -1047,6 +1069,18 @@ static mlir::Value genScalarCombiner(fir::FirOpBuilder &builder,
   TODO(loc, "reduction operator");
 }
 
+static hlfir::DesignateOp::Subscripts
+getTripletsFromArgs(mlir::acc::ReductionRecipeOp recipe) {
+  hlfir::DesignateOp::Subscripts triplets;
+  for (unsigned i = 2; i < recipe.getCombinerRegion().getArguments().size();
+       i += 3)
+    triplets.emplace_back(hlfir::DesignateOp::Triplet{
+        recipe.getCombinerRegion().getArgument(i),
+        recipe.getCombinerRegion().getArgument(i + 1),
+        recipe.getCombinerRegion().getArgument(i + 2)});
+  return triplets;
+}
+
 static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
                         mlir::acc::ReductionOperator op, mlir::Type ty,
                         mlir::Value value1, mlir::Value value2,
@@ -1056,11 +1090,60 @@ static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
   ty = fir::unwrapRefType(ty);
 
   if (auto seqTy = mlir::dyn_cast<fir::SequenceType>(ty)) {
-    assert(!seqTy.hasDynamicExtents() &&
-           "Assumed shaped array should be boxed for reduction");
     mlir::Type refTy = fir::ReferenceType::get(seqTy.getEleTy());
     llvm::SmallVector<fir::DoLoopOp> loops;
     llvm::SmallVector<mlir::Value> ivs;
+    if (seqTy.hasDynamicExtents()) {
+      auto shape =
+          genShapeFromBoundsOrArgs(loc, builder, seqTy, bounds,
+                                   recipe.getCombinerRegion().getArguments());
+      auto v1DeclareOp = builder.create<hlfir::DeclareOp>(
+          loc, value1, llvm::StringRef{}, shape, llvm::ArrayRef<mlir::Value>{},
+          fir::FortranVariableFlagsAttr{});
+      auto v2DeclareOp = builder.create<hlfir::DeclareOp>(
+          loc, value2, llvm::StringRef{}, shape, llvm::ArrayRef<mlir::Value>{},
+          fir::FortranVariableFlagsAttr{});
+      hlfir::DesignateOp::Subscripts triplets = getTripletsFromArgs(recipe);
+
+      llvm::SmallVector<mlir::Value> lenParamsLeft;
+      auto leftEntity = hlfir::Entity{v1DeclareOp.getBase()};
+      hlfir::genLengthParameters(loc, builder, leftEntity, lenParamsLeft);
+      auto leftDesignate = builder.create<hlfir::DesignateOp>(
+          loc, v1DeclareOp.getBase().getType(), v1DeclareOp.getBase(),
+          /*component=*/"",
+          /*componentShape=*/mlir::Value{}, triplets,
+          /*substring=*/mlir::ValueRange{}, /*complexPartAttr=*/std::nullopt,
+          shape, lenParamsLeft);
+      auto left = hlfir::Entity{leftDesignate.getResult()};
+
+      llvm::SmallVector<mlir::Value> lenParamsRight;
+      auto rightEntity = hlfir::Entity{v2DeclareOp.getBase()};
+      hlfir::genLengthParameters(loc, builder, rightEntity, lenParamsLeft);
+      auto rightDesignate = builder.create<hlfir::DesignateOp>(
+          loc, v2DeclareOp.getBase().getType(), v2DeclareOp.getBase(),
+          /*component=*/"",
+          /*componentShape=*/mlir::Value{}, triplets,
+          /*substring=*/mlir::ValueRange{}, /*complexPartAttr=*/std::nullopt,
+          shape, lenParamsRight);
+      auto right = hlfir::Entity{rightDesignate.getResult()};
+
+      llvm::SmallVector<mlir::Value, 1> typeParams;
+      auto genKernel = [&builder, &loc, op, seqTy, &left, &right](
+                           mlir::Location l, fir::FirOpBuilder &b,
+                           mlir::ValueRange oneBasedIndices) -> hlfir::Entity {
+        auto leftElement = hlfir::getElementAt(l, b, left, oneBasedIndices);
+        auto rightElement = hlfir::getElementAt(l, b, right, oneBasedIndices);
+        auto leftVal = hlfir::loadTrivialScalar(l, b, leftElement);
+        auto rightVal = hlfir::loadTrivialScalar(l, b, rightElement);
+        return hlfir::Entity{genScalarCombiner(
+            builder, loc, op, seqTy.getEleTy(), leftVal, rightVal)};
+      };
+      mlir::Value elemental = hlfir::genElementalOp(
+          loc, builder, seqTy.getEleTy(), shape, typeParams, genKernel,
+          /*isUnordered=*/true);
+      builder.create<hlfir::AssignOp>(loc, elemental, v1DeclareOp.getBase());
+      return;
+    }
     if (allConstantBound) {
       // Use the constant bound directly in the combiner region so they do not
       // need to be passed as block argument.
@@ -1103,55 +1186,22 @@ static void genCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
     builder.create<fir::StoreOp>(loc, res, addr1);
     builder.setInsertionPointAfter(loops[0]);
   } else if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(ty)) {
-    llvm::SmallVector<mlir::Value> tripletArgs;
     mlir::Type innerTy = extractSequenceType(boxTy);
     fir::SequenceType seqTy =
         mlir::dyn_cast_or_null<fir::SequenceType>(innerTy);
     if (!seqTy)
       TODO(loc, "Unsupported boxed type in OpenACC reduction");
 
-    if (allConstantBound) {
-      for (auto bound : llvm::reverse(bounds)) {
-        auto dataBound =
-            mlir::cast<mlir::acc::DataBoundsOp>(bound.getDefiningOp());
-        tripletArgs.append(genConstantBounds(builder, loc, dataBound));
-      }
-    } else {
-      assert(((recipe.getCombinerRegion().getArguments().size() - 2) / 3 ==
-              seqTy.getDimension()) &&
-             "Expect 3 block arguments per dimension");
-      for (auto arg : recipe.getCombinerRegion().getArguments().drop_front(2))
-        tripletArgs.push_back(arg);
-    }
-    auto shape = genShapeFromBounds(loc, builder, tripletArgs);
-
-    hlfir::DesignateOp::Subscripts triplets;
-    for (unsigned i = 2; i < recipe.getCombinerRegion().getArguments().size();
-         i += 3)
-      triplets.emplace_back(hlfir::DesignateOp::Triplet{
-          recipe.getCombinerRegion().getArgument(i),
-          recipe.getCombinerRegion().getArgument(i + 1),
-          recipe.getCombinerRegion().getArgument(i + 2)});
-
-    llvm::SmallVector<mlir::Value> lenParamsLeft;
+    auto shape = genShapeFromBoundsOrArgs(
+        loc, builder, seqTy, bounds, recipe.getCombinerRegion().getArguments());
+    hlfir::DesignateOp::Subscripts triplets =
+        getSubscriptsFromArgs(recipe.getCombinerRegion().getArguments());
     auto leftEntity = hlfir::Entity{value1};
-    hlfir::genLengthParameters(loc, builder, leftEntity, lenParamsLeft);
-    auto leftDesignate = builder.create<hlfir::DesignateOp>(
-        loc, value1.getType(), leftEntity, /*component=*/"",
-        /*componentShape=*/mlir::Value{}, triplets,
-        /*substring=*/mlir::ValueRange{}, /*complexPartAttr=*/std::nullopt,
-        shape, lenParamsLeft);
-    auto left = hlfir::Entity{leftDesignate.getResult()};
-
-    llvm::SmallVector<mlir::Value> lenParamsRight;
+    auto left =
+        genDesignateWithTriplets(builder, loc, leftEntity, triplets, shape);
     auto rightEntity = hlfir::Entity{value2};
-    hlfir::genLengthParameters(loc, builder, rightEntity, lenParamsRight);
-    auto rightDesignate = builder.create<hlfir::DesignateOp>(
-        loc, value2.getType(), rightEntity, /*component=*/"",
-        /*componentShape=*/mlir::Value{}, triplets,
-        /*substring=*/mlir::ValueRange{}, /*complexPartAttr=*/std::nullopt,
-        shape, lenParamsRight);
-    auto right = hlfir::Entity{rightDesignate.getResult()};
+    auto right =
+        genDesignateWithTriplets(builder, loc, rightEntity, triplets, shape);
 
     llvm::SmallVector<mlir::Value, 1> typeParams;
     auto genKernel = [&builder, &loc, op, seqTy, &left, &right](
@@ -1187,8 +1237,20 @@ mlir::acc::ReductionRecipeOp Fortran::lower::createOrGetReductionRecipe(
   mlir::OpBuilder modBuilder(mod.getBodyRegion());
   auto recipe =
       modBuilder.create<mlir::acc::ReductionRecipeOp>(loc, recipeName, ty, op);
+  llvm::SmallVector<mlir::Type> initArgsTy{ty};
+  llvm::SmallVector<mlir::Location> initArgsLoc{loc};
+  mlir::Type refTy = fir::unwrapRefType(ty);
+  if (auto seqTy = mlir::dyn_cast_or_null<fir::SequenceType>(refTy)) {
+    if (seqTy.hasDynamicExtents()) {
+      mlir::Type idxTy = builder.getIndexType();
+      for (unsigned i = 0; i < seqTy.getDimension(); ++i) {
+        initArgsTy.push_back(idxTy);
+        initArgsLoc.push_back(loc);
+      }
+    }
+  }
   builder.createBlock(&recipe.getInitRegion(), recipe.getInitRegion().end(),
-                      {ty}, {loc});
+                      initArgsTy, initArgsLoc);
   builder.setInsertionPointToEnd(&recipe.getInitRegion().back());
   mlir::Value initValue = genReductionInitRegion(builder, loc, ty, op);
   builder.create<mlir::acc::YieldOp>(loc, initValue);
@@ -1792,9 +1854,10 @@ createComputeOp(Fortran::lower::AbstractConverter &converter,
     } else if (const auto *privateClause =
                    std::get_if<Fortran::parser::AccClause::Private>(
                        &clause.u)) {
-      genPrivatizations<mlir::acc::PrivateRecipeOp>(
-          privateClause->v, converter, semanticsContext, stmtCtx,
-          privateOperands, privatizations);
+      if (!outerCombined)
+        genPrivatizations<mlir::acc::PrivateRecipeOp>(
+            privateClause->v, converter, semanticsContext, stmtCtx,
+            privateOperands, privatizations);
     } else if (const auto *firstprivateClause =
                    std::get_if<Fortran::parser::AccClause::Firstprivate>(
                        &clause.u)) {
@@ -1804,8 +1867,9 @@ createComputeOp(Fortran::lower::AbstractConverter &converter,
     } else if (const auto *reductionClause =
                    std::get_if<Fortran::parser::AccClause::Reduction>(
                        &clause.u)) {
-      genReductions(reductionClause->v, converter, semanticsContext, stmtCtx,
-                    reductionOperands, reductionRecipes);
+      if (!outerCombined)
+        genReductions(reductionClause->v, converter, semanticsContext, stmtCtx,
+                      reductionOperands, reductionRecipes);
     } else if (const auto *defaultClause =
                    std::get_if<Fortran::parser::AccClause::Default>(
                        &clause.u)) {
@@ -3294,6 +3358,9 @@ void Fortran::lower::genOpenACCConstruct(
           [&](const Fortran::parser::OpenACCAtomicConstruct &atomicConstruct) {
             genACC(converter, eval, atomicConstruct);
           },
+          [&](const Fortran::parser::OpenACCEndConstruct &) {
+            // No op
+          },
       },
       accConstruct.u);
 }
diff --git a/flang/lib/Lower/Runtime.cpp b/flang/lib/Lower/Runtime.cpp
index 2cf1e522d3..8855cab8b5 100644
--- a/flang/lib/Lower/Runtime.cpp
+++ b/flang/lib/Lower/Runtime.cpp
@@ -140,49 +140,49 @@ void Fortran::lower::genFailImageStatement(
 void Fortran::lower::genEventPostStatement(
     Fortran::lower::AbstractConverter &converter,
     const Fortran::parser::EventPostStmt &) {
-  TODO(converter.getCurrentLocation(), "EVENT POST runtime");
+  TODO(converter.getCurrentLocation(), "coarray: EVENT POST runtime");
 }
 
 void Fortran::lower::genEventWaitStatement(
     Fortran::lower::AbstractConverter &converter,
     const Fortran::parser::EventWaitStmt &) {
-  TODO(converter.getCurrentLocation(), "EVENT WAIT runtime");
+  TODO(converter.getCurrentLocation(), "coarray: EVENT WAIT runtime");
 }
 
 void Fortran::lower::genLockStatement(
     Fortran::lower::AbstractConverter &converter,
     const Fortran::parser::LockStmt &) {
-  TODO(converter.getCurrentLocation(), "LOCK runtime");
+  TODO(converter.getCurrentLocation(), "coarray: LOCK runtime");
 }
 
 void Fortran::lower::genUnlockStatement(
     Fortran::lower::AbstractConverter &converter,
     const Fortran::parser::UnlockStmt &) {
-  TODO(converter.getCurrentLocation(), "UNLOCK runtime");
+  TODO(converter.getCurrentLocation(), "coarray: UNLOCK runtime");
 }
 
 void Fortran::lower::genSyncAllStatement(
     Fortran::lower::AbstractConverter &converter,
     const Fortran::parser::SyncAllStmt &) {
-  TODO(converter.getCurrentLocation(), "SYNC ALL runtime");
+  TODO(converter.getCurrentLocation(), "coarray: SYNC ALL runtime");
 }
 
 void Fortran::lower::genSyncImagesStatement(
     Fortran::lower::AbstractConverter &converter,
     const Fortran::parser::SyncImagesStmt &) {
-  TODO(converter.getCurrentLocation(), "SYNC IMAGES runtime");
+  TODO(converter.getCurrentLocation(), "coarray: SYNC IMAGES runtime");
 }
 
 void Fortran::lower::genSyncMemoryStatement(
     Fortran::lower::AbstractConverter &converter,
     const Fortran::parser::SyncMemoryStmt &) {
-  TODO(converter.getCurrentLocation(), "SYNC MEMORY runtime");
+  TODO(converter.getCurrentLocation(), "coarray: SYNC MEMORY runtime");
 }
 
 void Fortran::lower::genSyncTeamStatement(
     Fortran::lower::AbstractConverter &converter,
     const Fortran::parser::SyncTeamStmt &) {
-  TODO(converter.getCurrentLocation(), "SYNC TEAM runtime");
+  TODO(converter.getCurrentLocation(), "coarray: SYNC TEAM runtime");
 }
 
 void Fortran::lower::genPauseStatement(
diff --git a/flang/lib/Lower/VectorSubscripts.cpp b/flang/lib/Lower/VectorSubscripts.cpp
index ca5dfc836e..7439b9f7df 100644
--- a/flang/lib/Lower/VectorSubscripts.cpp
+++ b/flang/lib/Lower/VectorSubscripts.cpp
@@ -212,7 +212,8 @@ class VectorSubscriptBoxBuilder {
 
   mlir::Type gen(const Fortran::evaluate::CoarrayRef &) {
     // Is this possible/legal ?
-    TODO(loc, "coarray ref with vector subscript in IO input");
+    TODO(loc, "coarray: reference to coarray object with vector subscript in "
+              "IO input");
   }
 
   template <typename A>
diff --git a/flang/lib/Optimizer/Analysis/CMakeLists.txt b/flang/lib/Optimizer/Analysis/CMakeLists.txt
index 19dadf72cf..436d4d3f18 100644
--- a/flang/lib/Optimizer/Analysis/CMakeLists.txt
+++ b/flang/lib/Optimizer/Analysis/CMakeLists.txt
@@ -1,5 +1,6 @@
 add_flang_library(FIRAnalysis
   AliasAnalysis.cpp
+  TBAAForest.cpp
 
   DEPENDS
   FIRDialect
diff --git a/flang/lib/Optimizer/Analysis/TBAAForest.cpp b/flang/lib/Optimizer/Analysis/TBAAForest.cpp
new file mode 100644
index 0000000000..070e2be670
--- /dev/null
+++ b/flang/lib/Optimizer/Analysis/TBAAForest.cpp
@@ -0,0 +1,60 @@
+//===- TBAAForest.cpp - Per-functon TBAA Trees ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Optimizer/Analysis/TBAAForest.h"
+#include <mlir/Dialect/LLVMIR/LLVMAttrs.h>
+
+mlir::LLVM::TBAATagAttr
+fir::TBAATree::SubtreeState::getTag(llvm::StringRef uniqueName) const {
+  // mlir::LLVM::TBAATagAttr &tag = tagDedup[uniqueName];
+  // if (tag)
+  //   return tag;
+  std::string id = (parentId + "/" + uniqueName).str();
+  mlir::LLVM::TBAATypeDescriptorAttr type =
+      mlir::LLVM::TBAATypeDescriptorAttr::get(
+          context, id, mlir::LLVM::TBAAMemberAttr::get(parent, 0));
+  return mlir::LLVM::TBAATagAttr::get(type, type, 0);
+  // return tag;
+}
+
+fir::TBAATree fir::TBAATree::buildTree(mlir::StringAttr func) {
+  llvm::StringRef funcName = func.getValue();
+  std::string rootId = ("Flang function root " + funcName).str();
+  mlir::MLIRContext *ctx = func.getContext();
+  mlir::LLVM::TBAARootAttr funcRoot =
+      mlir::LLVM::TBAARootAttr::get(ctx, mlir::StringAttr::get(ctx, rootId));
+
+  static constexpr llvm::StringRef anyAccessTypeDescId = "any access";
+  mlir::LLVM::TBAATypeDescriptorAttr anyAccess =
+      mlir::LLVM::TBAATypeDescriptorAttr::get(
+          ctx, anyAccessTypeDescId,
+          mlir::LLVM::TBAAMemberAttr::get(funcRoot, 0));
+
+  static constexpr llvm::StringRef anyDataAccessTypeDescId = "any data access";
+  mlir::LLVM::TBAATypeDescriptorAttr dataRoot =
+      mlir::LLVM::TBAATypeDescriptorAttr::get(
+          ctx, anyDataAccessTypeDescId,
+          mlir::LLVM::TBAAMemberAttr::get(anyAccess, 0));
+
+  static constexpr llvm::StringRef boxMemberTypeDescId = "descriptor member";
+  mlir::LLVM::TBAATypeDescriptorAttr boxMemberTypeDesc =
+      mlir::LLVM::TBAATypeDescriptorAttr::get(
+          ctx, boxMemberTypeDescId,
+          mlir::LLVM::TBAAMemberAttr::get(anyAccess, 0));
+
+  return TBAATree{anyAccess, dataRoot, boxMemberTypeDesc};
+}
+
+fir::TBAATree::TBAATree(mlir::LLVM::TBAATypeDescriptorAttr anyAccess,
+                        mlir::LLVM::TBAATypeDescriptorAttr dataRoot,
+                        mlir::LLVM::TBAATypeDescriptorAttr boxMemberTypeDesc)
+    : globalDataTree(dataRoot.getContext(), "global data", dataRoot),
+      allocatedDataTree(dataRoot.getContext(), "allocated data", dataRoot),
+      dummyArgDataTree(dataRoot.getContext(), "dummy arg data", dataRoot),
+      anyAccessDesc(anyAccess), boxMemberTypeDesc(boxMemberTypeDesc),
+      anyDataTypeDesc(dataRoot) {}
diff --git a/flang/lib/Optimizer/CodeGen/CMakeLists.txt b/flang/lib/Optimizer/CodeGen/CMakeLists.txt
index 0b2e3943c1..0daa97b00d 100644
--- a/flang/lib/Optimizer/CodeGen/CMakeLists.txt
+++ b/flang/lib/Optimizer/CodeGen/CMakeLists.txt
@@ -14,6 +14,7 @@ add_flang_library(FIRCodeGen
   CGOpsIncGen
 
   LINK_LIBS
+  FIRAnalysis
   FIRBuilder
   FIRDialect
   FIRDialectSupport
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 5bf6b87615..f17df3f5c6 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -3086,7 +3086,10 @@ struct LoadOpConversion : public FIROpConversion<fir::LoadOp> {
       auto boxValue = rewriter.create<mlir::LLVM::LoadOp>(
           loc, boxPtrTy.cast<mlir::LLVM::LLVMPointerType>().getElementType(),
           inputBoxStorage);
-      attachTBAATag(boxValue, boxTy, boxTy, nullptr);
+      if (std::optional<mlir::ArrayAttr> optionalTag = load.getTbaa())
+        boxValue.setTBAATags(*optionalTag);
+      else
+        attachTBAATag(boxValue, boxTy, boxTy, nullptr);
       auto newBoxStorage =
           genAllocaWithType(loc, boxPtrTy, defaultAlign, rewriter);
       auto storeOp =
@@ -3097,7 +3100,10 @@ struct LoadOpConversion : public FIROpConversion<fir::LoadOp> {
       mlir::Type loadTy = convertType(load.getType());
       auto loadOp = rewriter.create<mlir::LLVM::LoadOp>(
           load.getLoc(), loadTy, adaptor.getOperands(), load->getAttrs());
-      attachTBAATag(loadOp, load.getType(), load.getType(), nullptr);
+      if (std::optional<mlir::ArrayAttr> optionalTag = load.getTbaa())
+        loadOp.setTBAATags(*optionalTag);
+      else
+        attachTBAATag(loadOp, load.getType(), load.getType(), nullptr);
       rewriter.replaceOp(load, loadOp.getResult());
     }
     return mlir::success();
@@ -3341,7 +3347,10 @@ struct StoreOpConversion : public FIROpConversion<fir::StoreOp> {
       newStoreOp = rewriter.create<mlir::LLVM::StoreOp>(
           loc, adaptor.getOperands()[0], adaptor.getOperands()[1]);
     }
-    attachTBAATag(newStoreOp, storeTy, storeTy, nullptr);
+    if (std::optional<mlir::ArrayAttr> optionalTag = store.getTbaa())
+      newStoreOp.setTBAATags(*optionalTag);
+    else
+      attachTBAATag(newStoreOp, storeTy, storeTy, nullptr);
     rewriter.eraseOp(store);
     return mlir::success();
   }
@@ -3778,7 +3787,8 @@ class FIRToLLVMLowering
 
     auto *context = getModule().getContext();
     fir::LLVMTypeConverter typeConverter{getModule(),
-                                         options.applyTBAA || applyTBAA};
+                                         options.applyTBAA || applyTBAA,
+                                         options.forceUnifiedTBAATree};
     mlir::RewritePatternSet pattern(context);
     pattern.insert<
         AbsentOpConversion, AddcOpConversion, AddrOfOpConversion,
diff --git a/flang/lib/Optimizer/CodeGen/TBAABuilder.cpp b/flang/lib/Optimizer/CodeGen/TBAABuilder.cpp
index 1a5ae8cf7a..8e7f59f763 100644
--- a/flang/lib/Optimizer/CodeGen/TBAABuilder.cpp
+++ b/flang/lib/Optimizer/CodeGen/TBAABuilder.cpp
@@ -15,6 +15,9 @@
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
+#include <mlir/Dialect/LLVMIR/LLVMAttrs.h>
+#include <mlir/Dialect/LLVMIR/LLVMDialect.h>
+#include <mlir/Dialect/LLVMIR/LLVMTypes.h>
 
 #define DEBUG_TYPE "flang-tbaa-builder"
 
@@ -27,6 +30,13 @@ static llvm::cl::opt<bool> disableTBAA(
                    "to override default Flang behavior"),
     llvm::cl::init(false));
 
+// disabling this will play badly with the FIR TBAA pass, leading to worse
+// performance
+static llvm::cl::opt<bool> perFunctionTBAATrees(
+    "per-function-tbaa-trees",
+    llvm::cl::desc("Give each function an independent TBAA tree (default)"),
+    llvm::cl::init(true), llvm::cl::Hidden);
+
 // tagAttachmentLimit is a debugging option that allows limiting
 // the number of TBAA access tag attributes attached to operations.
 // It is set to kTagAttachmentUnlimited by default denoting "no limit".
@@ -38,27 +48,12 @@ static llvm::cl::opt<unsigned>
 
 namespace fir {
 
-TBAABuilder::TBAABuilder(MLIRContext *context, bool applyTBAA)
-    : enableTBAA(applyTBAA && !disableTBAA) {
+TBAABuilder::TBAABuilder(MLIRContext *context, bool applyTBAA,
+                         bool forceUnifiedTree)
+    : enableTBAA(applyTBAA && !disableTBAA),
+      trees(/*separatePerFunction=*/perFunctionTBAATrees && !forceUnifiedTree) {
   if (!enableTBAA)
     return;
-
-  // Root node.
-  flangTBAARoot =
-      TBAARootAttr::get(context, StringAttr::get(context, flangTBAARootId));
-
-  // Any access node.
-  anyAccessTypeDesc = TBAATypeDescriptorAttr::get(
-      context, anyAccessTypeDescId, TBAAMemberAttr::get(flangTBAARoot, 0));
-
-  // Any data access node.
-  anyDataAccessTypeDesc =
-      TBAATypeDescriptorAttr::get(context, anyDataAccessTypeDescId,
-                                  TBAAMemberAttr::get(anyAccessTypeDesc, 0));
-
-  // Box member access node.
-  boxMemberTypeDesc = TBAATypeDescriptorAttr::get(
-      context, boxMemberTypeDescId, TBAAMemberAttr::get(anyAccessTypeDesc, 0));
 }
 
 TBAATagAttr TBAABuilder::getAccessTag(TBAATypeDescriptorAttr baseTypeDesc,
@@ -73,26 +68,31 @@ TBAATagAttr TBAABuilder::getAccessTag(TBAATypeDescriptorAttr baseTypeDesc,
   return tag;
 }
 
-TBAATagAttr TBAABuilder::getAnyBoxAccessTag() {
+TBAATagAttr TBAABuilder::getAnyBoxAccessTag(mlir::LLVM::LLVMFuncOp func) {
+  TBAATypeDescriptorAttr boxMemberTypeDesc = trees[func].boxMemberTypeDesc;
   return getAccessTag(boxMemberTypeDesc, boxMemberTypeDesc, /*offset=*/0);
 }
 
 TBAATagAttr TBAABuilder::getBoxAccessTag(Type baseFIRType, Type accessFIRType,
-                                         GEPOp gep) {
-  return getAnyBoxAccessTag();
+                                         GEPOp gep,
+                                         mlir::LLVM::LLVMFuncOp func) {
+  return getAnyBoxAccessTag(func);
 }
 
-TBAATagAttr TBAABuilder::getAnyDataAccessTag() {
+TBAATagAttr TBAABuilder::getAnyDataAccessTag(mlir::LLVM::LLVMFuncOp func) {
+  TBAATypeDescriptorAttr anyDataAccessTypeDesc = trees[func].anyDataTypeDesc;
   return getAccessTag(anyDataAccessTypeDesc, anyDataAccessTypeDesc,
                       /*offset=*/0);
 }
 
 TBAATagAttr TBAABuilder::getDataAccessTag(Type baseFIRType, Type accessFIRType,
-                                          GEPOp gep) {
-  return getAnyDataAccessTag();
+                                          GEPOp gep,
+                                          mlir::LLVM::LLVMFuncOp func) {
+  return getAnyDataAccessTag(func);
 }
 
-TBAATagAttr TBAABuilder::getAnyAccessTag() {
+TBAATagAttr TBAABuilder::getAnyAccessTag(mlir::LLVM::LLVMFuncOp func) {
+  TBAATypeDescriptorAttr anyAccessTypeDesc = trees[func].anyAccessDesc;
   return getAccessTag(anyAccessTypeDesc, anyAccessTypeDesc, /*offset=*/0);
 }
 
@@ -101,6 +101,9 @@ void TBAABuilder::attachTBAATag(AliasAnalysisOpInterface op, Type baseFIRType,
   if (!enableTBAA)
     return;
 
+  mlir::LLVM::LLVMFuncOp func = op->getParentOfType<mlir::LLVM::LLVMFuncOp>();
+  assert(func && "func.func should have already been converted to llvm.func");
+
   ++tagAttachmentCounter;
   if (tagAttachmentLimit != kTagAttachmentUnlimited &&
       tagAttachmentCounter > tagAttachmentLimit)
@@ -115,11 +118,11 @@ void TBAABuilder::attachTBAATag(AliasAnalysisOpInterface op, Type baseFIRType,
     // a mix of data members and descriptor members may alias
     // with both data and descriptor accesses.
     // Conservatively set any-access tag if there is any descriptor member.
-    tbaaTagSym = getAnyAccessTag();
+    tbaaTagSym = getAnyAccessTag(func);
   } else if (baseFIRType.isa<fir::BaseBoxType>()) {
-    tbaaTagSym = getBoxAccessTag(baseFIRType, accessFIRType, gep);
+    tbaaTagSym = getBoxAccessTag(baseFIRType, accessFIRType, gep, func);
   } else {
-    tbaaTagSym = getDataAccessTag(baseFIRType, accessFIRType, gep);
+    tbaaTagSym = getDataAccessTag(baseFIRType, accessFIRType, gep, func);
   }
 
   if (!tbaaTagSym)
diff --git a/flang/lib/Optimizer/CodeGen/TypeConverter.cpp b/flang/lib/Optimizer/CodeGen/TypeConverter.cpp
index 77e94c00ec..104018030b 100644
--- a/flang/lib/Optimizer/CodeGen/TypeConverter.cpp
+++ b/flang/lib/Optimizer/CodeGen/TypeConverter.cpp
@@ -26,7 +26,8 @@
 
 namespace fir {
 
-LLVMTypeConverter::LLVMTypeConverter(mlir::ModuleOp module, bool applyTBAA)
+LLVMTypeConverter::LLVMTypeConverter(mlir::ModuleOp module, bool applyTBAA,
+                                     bool forceUnifiedTBAATree)
     : mlir::LLVMTypeConverter(module.getContext(),
                               [&] {
                                 mlir::LowerToLLVMOptions options(
@@ -38,8 +39,8 @@ LLVMTypeConverter::LLVMTypeConverter(mlir::ModuleOp module, bool applyTBAA)
       specifics(CodeGenSpecifics::get(module.getContext(),
                                       getTargetTriple(module),
                                       getKindMapping(module))),
-      tbaaBuilder(
-          std::make_unique<TBAABuilder>(module->getContext(), applyTBAA)) {
+      tbaaBuilder(std::make_unique<TBAABuilder>(module->getContext(), applyTBAA,
+                                                forceUnifiedTBAATree)) {
   LLVM_DEBUG(llvm::dbgs() << "FIR type converter\n");
 
   // Each conversion should return a value of type mlir::Type.
diff --git a/flang/lib/Optimizer/Dialect/CMakeLists.txt b/flang/lib/Optimizer/Dialect/CMakeLists.txt
index fe5edb54a7..745439b7e1 100644
--- a/flang/lib/Optimizer/Dialect/CMakeLists.txt
+++ b/flang/lib/Optimizer/Dialect/CMakeLists.txt
@@ -6,6 +6,7 @@ add_flang_library(FIRDialect
   FIROps.cpp
   FIRType.cpp
   FortranVariableInterface.cpp
+  FirAliasTagOpInterface.cpp
   Inliner.cpp
 
   DEPENDS
diff --git a/flang/lib/Optimizer/Dialect/FIRDialect.cpp b/flang/lib/Optimizer/Dialect/FIRDialect.cpp
index c2377f112b..997a6c90ad 100644
--- a/flang/lib/Optimizer/Dialect/FIRDialect.cpp
+++ b/flang/lib/Optimizer/Dialect/FIRDialect.cpp
@@ -14,6 +14,7 @@
 #include "flang/Optimizer/Dialect/FIRAttr.h"
 #include "flang/Optimizer/Dialect/FIROps.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Transforms/InliningUtils.h"
 
 using namespace fir;
@@ -58,6 +59,7 @@ struct FIRInlinerInterface : public mlir::DialectInlinerInterface {
 
 fir::FIROpsDialect::FIROpsDialect(mlir::MLIRContext *ctx)
     : mlir::Dialect("fir", ctx, mlir::TypeID::get<FIROpsDialect>()) {
+  getContext()->loadDialect<mlir::LLVM::LLVMDialect>();
   registerTypes();
   registerAttributes();
   addOperations<
diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index d60e5e9657..38311832f2 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -1933,6 +1933,12 @@ mlir::Value fir::IterWhileOp::blockArgToSourceOp(unsigned blockArgNum) {
   return {};
 }
 
+mlir::ValueRange fir::IterWhileOp::getYieldedValues() {
+  auto *term = getRegion().front().getTerminator();
+  return getFinalValue() ? term->getOperands().drop_front()
+                         : term->getOperands();
+}
+
 //===----------------------------------------------------------------------===//
 // LenParamIndexOp
 //===----------------------------------------------------------------------===//
@@ -1977,8 +1983,18 @@ void fir::LoadOp::build(mlir::OpBuilder &builder, mlir::OperationState &result,
     mlir::emitError(result.location, "not a memory reference type");
     return;
   }
+  build(builder, result, eleTy, refVal);
+}
+
+void fir::LoadOp::build(mlir::OpBuilder &builder, mlir::OperationState &result,
+                        mlir::Type resTy, mlir::Value refVal) {
+
+  if (!refVal) {
+    mlir::emitError(result.location, "LoadOp has null argument");
+    return;
+  }
   result.addOperands(refVal);
-  result.addTypes(eleTy);
+  result.addTypes(resTy);
 }
 
 mlir::ParseResult fir::LoadOp::getElementOf(mlir::Type &ele, mlir::Type ref) {
@@ -2228,6 +2244,12 @@ mlir::Value fir::DoLoopOp::blockArgToSourceOp(unsigned blockArgNum) {
   return {};
 }
 
+mlir::ValueRange fir::DoLoopOp::getYieldedValues() {
+  auto *term = getRegion().front().getTerminator();
+  return getFinalValue() ? term->getOperands().drop_front()
+                         : term->getOperands();
+}
+
 //===----------------------------------------------------------------------===//
 // DTEntryOp
 //===----------------------------------------------------------------------===//
@@ -3249,6 +3271,11 @@ mlir::LogicalResult fir::StoreOp::verify() {
   return mlir::success();
 }
 
+void fir::StoreOp::build(mlir::OpBuilder &builder, mlir::OperationState &result,
+                         mlir::Value value, mlir::Value memref) {
+  build(builder, result, value, memref, {});
+}
+
 //===----------------------------------------------------------------------===//
 // StringLitOp
 //===----------------------------------------------------------------------===//
diff --git a/flang/lib/Optimizer/Dialect/FirAliasTagOpInterface.cpp b/flang/lib/Optimizer/Dialect/FirAliasTagOpInterface.cpp
new file mode 100644
index 0000000000..648f490f63
--- /dev/null
+++ b/flang/lib/Optimizer/Dialect/FirAliasTagOpInterface.cpp
@@ -0,0 +1,31 @@
+//===-- FirAliasTagOpInterface.cpp ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Optimizer/Dialect/FirAliasTagOpInterface.h"
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+
+#include "flang/Optimizer/Dialect/FirAliasTagOpInterface.cpp.inc"
+
+mlir::LogicalResult
+fir::detail::verifyFirAliasTagOpInterface(mlir::Operation *op) {
+  auto iface = mlir::cast<FirAliasTagOpInterface>(op);
+
+  mlir::ArrayAttr tags = iface.getTBAATagsOrNull();
+  if (!tags)
+    return mlir::success();
+
+  for (mlir::Attribute iter : tags)
+    if (!mlir::isa<mlir::LLVM::TBAATagAttr>(iter))
+      return op->emitOpError("expected op to return array of ")
+             << mlir::LLVM::TBAATagAttr::getMnemonic() << " attributes";
+  return mlir::success();
+}
diff --git a/flang/lib/Optimizer/Transforms/AddAliasTags.cpp b/flang/lib/Optimizer/Transforms/AddAliasTags.cpp
new file mode 100644
index 0000000000..25439837ac
--- /dev/null
+++ b/flang/lib/Optimizer/Transforms/AddAliasTags.cpp
@@ -0,0 +1,210 @@
+//===- AddAliasTags.cpp ---------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+/// \file
+/// Adds TBAA alias tags to fir loads and stores, based on information from
+/// fir::AliasAnalysis. More are added later in CodeGen - see fir::TBAABuilder
+//===----------------------------------------------------------------------===//
+
+#include "flang/Optimizer/Analysis/AliasAnalysis.h"
+#include "flang/Optimizer/Analysis/TBAAForest.h"
+#include "flang/Optimizer/Dialect/FIRDialect.h"
+#include "flang/Optimizer/Dialect/FirAliasTagOpInterface.h"
+#include "flang/Optimizer/Transforms/Passes.h"
+#include "mlir/Pass/Pass.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+#include <optional>
+
+namespace fir {
+#define GEN_PASS_DEF_ADDALIASTAGS
+#include "flang/Optimizer/Transforms/Passes.h.inc"
+} // namespace fir
+
+#define DEBUG_TYPE "fir-add-alias-tags"
+
+static llvm::cl::opt<bool>
+    enableDummyArgs("dummy-arg-tbaa", llvm::cl::init(true), llvm::cl::Hidden,
+                    llvm::cl::desc("Add TBAA tags to dummy arguments"));
+// These two are **known unsafe** (misscompare in spec2017/wrf_r). They should
+// not be enabled by default.
+// The code is kept so that these may be tried with new benchmarks to see if
+// this is worth fixing in the future.
+static llvm::cl::opt<bool>
+    enableGlobals("globals-tbaa", llvm::cl::init(false), llvm::cl::Hidden,
+                  llvm::cl::desc("Add TBAA tags to global variables. UNSAFE."));
+static llvm::cl::opt<bool> enableLocalAllocs(
+    "local-alloc-tbaa", llvm::cl::init(false), llvm::cl::Hidden,
+    llvm::cl::desc("Add TBAA tags to local allocations. UNSAFE."));
+
+namespace {
+
+/// Shared state per-module
+class PassState {
+public:
+  /// memoised call to fir::AliasAnalysis::getSource
+  inline const fir::AliasAnalysis::Source &getSource(mlir::Value value) {
+    if (!analysisCache.contains(value))
+      analysisCache.insert({value, analysis.getSource(value)});
+    return analysisCache[value];
+  }
+
+  /// get the per-function TBAATree for this function
+  inline const fir::TBAATree &getFuncTree(mlir::func::FuncOp func) {
+    return forrest[func];
+  }
+
+private:
+  fir::AliasAnalysis analysis;
+  llvm::DenseMap<mlir::Value, fir::AliasAnalysis::Source> analysisCache;
+  fir::TBAAForrest forrest;
+};
+
+class AddAliasTagsPass : public fir::impl::AddAliasTagsBase<AddAliasTagsPass> {
+public:
+  void runOnOperation() override;
+
+private:
+  /// The real workhorse of the pass. This is a runOnOperation() which
+  /// operates on fir::FirAliasTagOpInterface, using some extra state
+  void runOnAliasInterface(fir::FirAliasTagOpInterface op, PassState &state);
+};
+
+} // namespace
+
+static fir::DeclareOp getDeclareOp(mlir::Value arg) {
+  for (mlir::Operation *use : arg.getUsers())
+    if (fir::DeclareOp declare = mlir::dyn_cast<fir::DeclareOp>(use))
+      return declare;
+  return nullptr;
+}
+
+/// Get the name of a function argument using the "fir.bindc_name" attribute,
+/// or ""
+static std::string getFuncArgName(mlir::Value arg) {
+  // first try getting the name from the hlfir.declare
+  if (fir::DeclareOp declare = getDeclareOp(arg))
+    return declare.getUniqName().str();
+
+  // get from attribute on function argument
+  // always succeeds because arg is a function argument
+  mlir::BlockArgument blockArg = mlir::cast<mlir::BlockArgument>(arg);
+  assert(blockArg.getOwner() && blockArg.getOwner()->isEntryBlock() &&
+         "arg is a function argument");
+  mlir::FunctionOpInterface func = mlir::dyn_cast<mlir::FunctionOpInterface>(
+      blockArg.getOwner()->getParentOp());
+  mlir::StringAttr attr = func.getArgAttrOfType<mlir::StringAttr>(
+      blockArg.getArgNumber(), "fir.bindc_name");
+  if (!attr)
+    return "";
+  return attr.str();
+}
+
+void AddAliasTagsPass::runOnAliasInterface(fir::FirAliasTagOpInterface op,
+                                           PassState &state) {
+  mlir::func::FuncOp func = op->getParentOfType<mlir::func::FuncOp>();
+  if (!func)
+    return;
+
+  llvm::SmallVector<mlir::Value> accessedOperands = op.getAccessedOperands();
+  assert(accessedOperands.size() == 1 &&
+         "load and store only access one address");
+  mlir::Value memref = accessedOperands.front();
+
+  // skip boxes. These get an "any descriptor access" tag in TBAABuilder
+  // (CodeGen). I didn't see any speedup from giving each box a separate TBAA
+  // type.
+  if (mlir::isa<fir::BaseBoxType>(fir::unwrapRefType(memref.getType())))
+    return;
+  LLVM_DEBUG(llvm::dbgs() << "Analysing " << op << "\n");
+
+  const fir::AliasAnalysis::Source &source = state.getSource(memref);
+  if (source.isTargetOrPointer()) {
+    LLVM_DEBUG(llvm::dbgs().indent(2) << "Skipping TARGET/POINTER\n");
+    // These will get an "any data access" tag in TBAABuilder (CodeGen): causing
+    // them to "MayAlias" with all non-box accesses
+    return;
+  }
+
+  mlir::LLVM::TBAATagAttr tag;
+  // TBAA for dummy arguments
+  if (enableDummyArgs &&
+      source.kind == fir::AliasAnalysis::SourceKind::Argument) {
+    LLVM_DEBUG(llvm::dbgs().indent(2)
+               << "Found reference to dummy argument at " << *op << "\n");
+    std::string name = getFuncArgName(source.u.get<mlir::Value>());
+    if (!name.empty())
+      tag = state.getFuncTree(func).dummyArgDataTree.getTag(name);
+    else
+      LLVM_DEBUG(llvm::dbgs().indent(2)
+                 << "WARN: couldn't find a name for dummy argument " << *op
+                 << "\n");
+
+    // TBAA for global variables
+  } else if (enableGlobals &&
+             source.kind == fir::AliasAnalysis::SourceKind::Global) {
+    mlir::SymbolRefAttr glbl = source.u.get<mlir::SymbolRefAttr>();
+    const char *name = glbl.getRootReference().data();
+    LLVM_DEBUG(llvm::dbgs().indent(2) << "Found reference to global " << name
+                                      << " at " << *op << "\n");
+    tag = state.getFuncTree(func).globalDataTree.getTag(name);
+
+    // TBAA for local allocations
+  } else if (enableLocalAllocs &&
+             source.kind == fir::AliasAnalysis::SourceKind::Allocate) {
+    std::optional<llvm::StringRef> name;
+    mlir::Operation *sourceOp = source.u.get<mlir::Value>().getDefiningOp();
+    if (auto alloc = mlir::dyn_cast_or_null<fir::AllocaOp>(sourceOp))
+      name = alloc.getUniqName();
+    else if (auto alloc = mlir::dyn_cast_or_null<fir::AllocMemOp>(sourceOp))
+      name = alloc.getUniqName();
+    if (name) {
+      LLVM_DEBUG(llvm::dbgs().indent(2) << "Found reference to allocation "
+                                        << name << " at " << *op << "\n");
+      tag = state.getFuncTree(func).allocatedDataTree.getTag(*name);
+    } else {
+      LLVM_DEBUG(llvm::dbgs().indent(2)
+                 << "WARN: couldn't find a name for allocation " << *op
+                 << "\n");
+    }
+  } else {
+    if (source.kind != fir::AliasAnalysis::SourceKind::Argument &&
+        source.kind != fir::AliasAnalysis::SourceKind::Allocate &&
+        source.kind != fir::AliasAnalysis::SourceKind::Global)
+      LLVM_DEBUG(llvm::dbgs().indent(2)
+                 << "WARN: unsupported value: " << source << "\n");
+  }
+
+  if (tag)
+    op.setTBAATags(mlir::ArrayAttr::get(&getContext(), tag));
+}
+
+void AddAliasTagsPass::runOnOperation() {
+  LLVM_DEBUG(llvm::dbgs() << "=== Begin " DEBUG_TYPE " ===\n");
+
+  // MLIR forbids storing state in a pass because different instances might be
+  // used in different threads.
+  // Instead this pass stores state per mlir::ModuleOp (which is what MLIR
+  // thinks the pass operates on), then the real work of the pass is done in
+  // runOnAliasInterface
+  PassState state;
+
+  mlir::ModuleOp mod = getOperation();
+  mod.walk(
+      [&](fir::FirAliasTagOpInterface op) { runOnAliasInterface(op, state); });
+
+  LLVM_DEBUG(llvm::dbgs() << "=== End " DEBUG_TYPE " ===\n");
+}
+
+std::unique_ptr<mlir::Pass> fir::createAliasTagsPass() {
+  return std::make_unique<AddAliasTagsPass>();
+}
diff --git a/flang/lib/Optimizer/Transforms/CMakeLists.txt b/flang/lib/Optimizer/Transforms/CMakeLists.txt
index 428c4c2a1e..98314fa7a2 100644
--- a/flang/lib/Optimizer/Transforms/CMakeLists.txt
+++ b/flang/lib/Optimizer/Transforms/CMakeLists.txt
@@ -1,5 +1,6 @@
 add_flang_library(FIRTransforms
   AbstractResult.cpp
+  AddAliasTags.cpp
   AffinePromotion.cpp
   AffineDemotion.cpp
   AnnotateConstant.cpp
@@ -27,6 +28,7 @@ add_flang_library(FIRTransforms
   HLFIROpsIncGen
 
   LINK_LIBS
+  FIRAnalysis
   FIRBuilder
   FIRCodeGen
   FIRDialect
diff --git a/flang/lib/Optimizer/Transforms/LoopVersioning.cpp b/flang/lib/Optimizer/Transforms/LoopVersioning.cpp
index 4d3ea51ae1..d030a3b383 100644
--- a/flang/lib/Optimizer/Transforms/LoopVersioning.cpp
+++ b/flang/lib/Optimizer/Transforms/LoopVersioning.cpp
@@ -51,6 +51,7 @@
 #include "flang/Optimizer/Dialect/Support/KindMapping.h"
 #include "flang/Optimizer/Transforms/Passes.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/IR/Dominance.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/Pass/Pass.h"
@@ -292,6 +293,8 @@ void LoopVersioningPass::runOnOperation() {
   // immediately nested in a loop.
   llvm::DenseMap<fir::DoLoopOp, ArgsUsageInLoop> argsInLoops;
 
+  auto &domInfo = getAnalysis<mlir::DominanceInfo>();
+
   // Traverse the loops in post-order and see
   // if those arguments are used inside any loop.
   func.walk([&](fir::DoLoopOp loop) {
@@ -309,12 +312,15 @@ void LoopVersioningPass::runOnOperation() {
       for (auto a : argsOfInterest) {
         if (a.arg == normaliseVal(operand)) {
           // Use the reboxed value, not the block arg when re-creating the loop.
-          // TODO: should we check that the operand dominates the loop?
-          // If this might be a case, we should record such operands in
-          // argsInLoop.cannotTransform, so that they disable the transformation
-          // for the parent loops as well.
           a.arg = operand;
 
+          // Check that the operand dominates the loop?
+          // If this is the case, record such operands in argsInLoop.cannot-
+          // Transform, so that they disable the transformation for the parent
+          /// loops as well.
+          if (!domInfo.dominates(a.arg, loop))
+            argsInLoop.cannotTransform.insert(a.arg);
+
           // No support currently for sliced arrays.
           // This means that we cannot transform properly
           // instructions referencing a.arg in the whole loop
diff --git a/flang/lib/Parser/executable-parsers.cpp b/flang/lib/Parser/executable-parsers.cpp
index a61bf2ce15..892c612d0c 100644
--- a/flang/lib/Parser/executable-parsers.cpp
+++ b/flang/lib/Parser/executable-parsers.cpp
@@ -279,13 +279,17 @@ TYPE_CONTEXT_PARSER("loop control"_en_US,
                 many(Parser<LocalitySpec>{})))))
 
 // R1121 label-do-stmt -> [do-construct-name :] DO label [loop-control]
+// A label-do-stmt with a do-construct-name is parsed as a nonlabel-do-stmt
+// with an optional label.
 TYPE_CONTEXT_PARSER("label DO statement"_en_US,
-    construct<LabelDoStmt>(
-        maybe(name / ":"), "DO" >> label, maybe(loopControl)))
+    construct<LabelDoStmt>("DO" >> label, maybe(loopControl)))
 
 // R1122 nonlabel-do-stmt -> [do-construct-name :] DO [loop-control]
 TYPE_CONTEXT_PARSER("nonlabel DO statement"_en_US,
-    construct<NonLabelDoStmt>(maybe(name / ":"), "DO" >> maybe(loopControl)))
+    construct<NonLabelDoStmt>(
+        name / ":", "DO" >> maybe(label), maybe(loopControl)) ||
+        construct<NonLabelDoStmt>(construct<std::optional<Name>>(),
+            construct<std::optional<Label>>(), "DO" >> maybe(loopControl)))
 
 // R1132 end-do-stmt -> END DO [do-construct-name]
 TYPE_CONTEXT_PARSER("END DO statement"_en_US,
diff --git a/flang/lib/Parser/openacc-parsers.cpp b/flang/lib/Parser/openacc-parsers.cpp
index 09b30e679d..bd5dcd8405 100644
--- a/flang/lib/Parser/openacc-parsers.cpp
+++ b/flang/lib/Parser/openacc-parsers.cpp
@@ -150,11 +150,12 @@ TYPE_PARSER(sourced(construct<AccLoopDirective>(
 TYPE_PARSER(construct<AccBeginLoopDirective>(
     sourced(Parser<AccLoopDirective>{}), Parser<AccClauseList>{}))
 
-TYPE_PARSER(construct<AccEndLoop>(startAccLine >> "END LOOP"_tok))
+TYPE_PARSER(construct<AccEndLoop>("END LOOP"_tok))
 
 TYPE_PARSER(construct<OpenACCLoopConstruct>(
     sourced(Parser<AccBeginLoopDirective>{} / endAccLine),
-    maybe(Parser<DoConstruct>{}), maybe(Parser<AccEndLoop>{} / endAccLine)))
+    maybe(Parser<DoConstruct>{}),
+    maybe(startAccLine >> Parser<AccEndLoop>{} / endAccLine)))
 
 // 2.15.1 Routine directive
 TYPE_PARSER(sourced(construct<OpenACCRoutineConstruct>(verbatim("ROUTINE"_tok),
@@ -227,22 +228,29 @@ TYPE_PARSER(construct<OpenACCStandaloneConstruct>(
 TYPE_PARSER(construct<OpenACCStandaloneDeclarativeConstruct>(
     sourced(Parser<AccDeclarativeDirective>{}), Parser<AccClauseList>{}))
 
-TYPE_PARSER(
-    startAccLine >> first(sourced(construct<OpenACCDeclarativeConstruct>(
-                              Parser<OpenACCStandaloneDeclarativeConstruct>{})),
-                        sourced(construct<OpenACCDeclarativeConstruct>(
-                            Parser<OpenACCRoutineConstruct>{}))))
+TYPE_PARSER(startAccLine >>
+    withMessage("expected OpenACC directive"_err_en_US,
+        first(sourced(construct<OpenACCDeclarativeConstruct>(
+                  Parser<OpenACCStandaloneDeclarativeConstruct>{})),
+            sourced(construct<OpenACCDeclarativeConstruct>(
+                Parser<OpenACCRoutineConstruct>{})))))
+
+TYPE_PARSER(sourced(construct<OpenACCEndConstruct>(
+    "END"_tok >> "LOOP"_tok >> pure(llvm::acc::Directive::ACCD_loop))))
 
 // OpenACC constructs
 TYPE_CONTEXT_PARSER("OpenACC construct"_en_US,
     startAccLine >>
-        first(construct<OpenACCConstruct>(Parser<OpenACCBlockConstruct>{}),
-            construct<OpenACCConstruct>(Parser<OpenACCCombinedConstruct>{}),
-            construct<OpenACCConstruct>(Parser<OpenACCLoopConstruct>{}),
-            construct<OpenACCConstruct>(Parser<OpenACCStandaloneConstruct>{}),
-            construct<OpenACCConstruct>(Parser<OpenACCCacheConstruct>{}),
-            construct<OpenACCConstruct>(Parser<OpenACCWaitConstruct>{}),
-            construct<OpenACCConstruct>(Parser<OpenACCAtomicConstruct>{})))
+        withMessage("expected OpenACC directive"_err_en_US,
+            first(construct<OpenACCConstruct>(Parser<OpenACCBlockConstruct>{}),
+                construct<OpenACCConstruct>(Parser<OpenACCCombinedConstruct>{}),
+                construct<OpenACCConstruct>(Parser<OpenACCLoopConstruct>{}),
+                construct<OpenACCConstruct>(
+                    Parser<OpenACCStandaloneConstruct>{}),
+                construct<OpenACCConstruct>(Parser<OpenACCCacheConstruct>{}),
+                construct<OpenACCConstruct>(Parser<OpenACCWaitConstruct>{}),
+                construct<OpenACCConstruct>(Parser<OpenACCAtomicConstruct>{}),
+                construct<OpenACCConstruct>(Parser<OpenACCEndConstruct>{}))))
 
 TYPE_PARSER(startAccLine >>
     sourced(construct<AccEndCombinedDirective>(sourced("END"_tok >>
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index 6cac3fb585..6d9d176216 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -1001,13 +1001,14 @@ class UnparseVisitor {
     Walk(std::get<Statement<EndDoStmt>>(x.t));
   }
   void Unparse(const LabelDoStmt &x) { // R1121
-    Walk(std::get<std::optional<Name>>(x.t), ": ");
     Word("DO "), Walk(std::get<Label>(x.t));
     Walk(" ", std::get<std::optional<LoopControl>>(x.t));
   }
   void Unparse(const NonLabelDoStmt &x) { // R1122
     Walk(std::get<std::optional<Name>>(x.t), ": ");
-    Word("DO "), Walk(std::get<std::optional<LoopControl>>(x.t));
+    Word("DO ");
+    Walk(std::get<std::optional<Label>>(x.t), " ");
+    Walk(std::get<std::optional<LoopControl>>(x.t));
   }
   void Unparse(const LoopControl &x) { // R1123
     common::visit(common::visitors{
diff --git a/flang/lib/Semantics/CMakeLists.txt b/flang/lib/Semantics/CMakeLists.txt
index bfd2f2b979..809206565f 100644
--- a/flang/lib/Semantics/CMakeLists.txt
+++ b/flang/lib/Semantics/CMakeLists.txt
@@ -36,6 +36,7 @@ add_flang_library(FortranSemantics
   resolve-directives.cpp
   resolve-names-utils.cpp
   resolve-names.cpp
+  rewrite-directives.cpp
   rewrite-parse-tree.cpp
   runtime-type-info.cpp
   scope.cpp
diff --git a/flang/lib/Semantics/canonicalize-do.cpp b/flang/lib/Semantics/canonicalize-do.cpp
index 6aab4b7bc4..ef20cffc3e 100644
--- a/flang/lib/Semantics/canonicalize-do.cpp
+++ b/flang/lib/Semantics/canonicalize-do.cpp
@@ -117,16 +117,15 @@ class CanonicalizationOfDoLoops {
             std::get<ExecutableConstruct>(doLoop->u).u)};
         auto &loopControl{
             std::get<std::optional<LoopControl>>(labelDo.statement.value().t)};
-        auto &name{std::get<std::optional<Name>>(labelDo.statement.value().t)};
         Statement<NonLabelDoStmt> nonLabelDoStmt{std::move(labelDo.label),
-            NonLabelDoStmt{
-                std::make_tuple(common::Clone(name), std::move(loopControl))}};
+            NonLabelDoStmt{std::make_tuple(std::optional<Name>{},
+                std::optional<Label>{}, std::move(loopControl))}};
         nonLabelDoStmt.source = originalSource;
         std::get<ExecutableConstruct>(doLoop->u).u =
             common::Indirection<DoConstruct>{
                 std::make_tuple(std::move(nonLabelDoStmt), std::move(block),
-                    Statement<EndDoStmt>{
-                        std::optional<Label>{}, EndDoStmt{std::move(name)}})};
+                    Statement<EndDoStmt>{std::optional<Label>{},
+                        EndDoStmt{std::optional<Name>{}}})};
         stack.pop_back();
       } while (!stack.empty() && stack.back().label == currentLabel);
       i = --next;
diff --git a/flang/lib/Semantics/check-acc-structure.cpp b/flang/lib/Semantics/check-acc-structure.cpp
index 3c9e89940d..763418ede2 100644
--- a/flang/lib/Semantics/check-acc-structure.cpp
+++ b/flang/lib/Semantics/check-acc-structure.cpp
@@ -376,19 +376,16 @@ CHECK_SIMPLE_CLAUSE(DeviceNum, ACCC_device_num)
 CHECK_SIMPLE_CLAUSE(Finalize, ACCC_finalize)
 CHECK_SIMPLE_CLAUSE(Firstprivate, ACCC_firstprivate)
 CHECK_SIMPLE_CLAUSE(Host, ACCC_host)
-CHECK_SIMPLE_CLAUSE(If, ACCC_if)
 CHECK_SIMPLE_CLAUSE(IfPresent, ACCC_if_present)
 CHECK_SIMPLE_CLAUSE(Independent, ACCC_independent)
 CHECK_SIMPLE_CLAUSE(NoCreate, ACCC_no_create)
 CHECK_SIMPLE_CLAUSE(Nohost, ACCC_nohost)
-CHECK_SIMPLE_CLAUSE(NumWorkers, ACCC_num_workers)
 CHECK_SIMPLE_CLAUSE(Private, ACCC_private)
 CHECK_SIMPLE_CLAUSE(Read, ACCC_read)
 CHECK_SIMPLE_CLAUSE(Seq, ACCC_seq)
 CHECK_SIMPLE_CLAUSE(Tile, ACCC_tile)
 CHECK_SIMPLE_CLAUSE(UseDevice, ACCC_use_device)
 CHECK_SIMPLE_CLAUSE(Vector, ACCC_vector)
-CHECK_SIMPLE_CLAUSE(VectorLength, ACCC_vector_length)
 CHECK_SIMPLE_CLAUSE(Wait, ACCC_wait)
 CHECK_SIMPLE_CLAUSE(Worker, ACCC_worker)
 CHECK_SIMPLE_CLAUSE(Write, ACCC_write)
@@ -542,13 +539,30 @@ void AccStructureChecker::Enter(const parser::AccClause::Gang &g) {
 }
 
 void AccStructureChecker::Enter(const parser::AccClause::NumGangs &n) {
-  CheckAllowed(llvm::acc::Clause::ACCC_num_gangs);
+  CheckAllowed(llvm::acc::Clause::ACCC_num_gangs,
+      /*warnInsteadOfError=*/GetContext().directive ==
+              llvm::acc::Directive::ACCD_serial ||
+          GetContext().directive == llvm::acc::Directive::ACCD_serial_loop);
 
   if (n.v.size() > 3)
     context_.Say(GetContext().clauseSource,
         "NUM_GANGS clause accepts a maximum of 3 arguments"_err_en_US);
 }
 
+void AccStructureChecker::Enter(const parser::AccClause::NumWorkers &n) {
+  CheckAllowed(llvm::acc::Clause::ACCC_num_workers,
+      /*warnInsteadOfError=*/GetContext().directive ==
+              llvm::acc::Directive::ACCD_serial ||
+          GetContext().directive == llvm::acc::Directive::ACCD_serial_loop);
+}
+
+void AccStructureChecker::Enter(const parser::AccClause::VectorLength &n) {
+  CheckAllowed(llvm::acc::Clause::ACCC_vector_length,
+      /*warnInsteadOfError=*/GetContext().directive ==
+              llvm::acc::Directive::ACCD_serial ||
+          GetContext().directive == llvm::acc::Directive::ACCD_serial_loop);
+}
+
 void AccStructureChecker::Enter(const parser::AccClause::Reduction &reduction) {
   CheckAllowed(llvm::acc::Clause::ACCC_reduction);
 
@@ -660,6 +674,24 @@ void AccStructureChecker::Enter(const parser::AccClause::Link &x) {
   CheckMultipleOccurrenceInDeclare(x.v, llvm::acc::Clause::ACCC_link);
 }
 
+void AccStructureChecker::Enter(const parser::AccClause::If &x) {
+  CheckAllowed(llvm::acc::Clause::ACCC_if);
+  if (const auto *expr{GetExpr(x.v)}) {
+    if (auto type{expr->GetType()}) {
+      if (type->category() == TypeCategory::Integer ||
+          type->category() == TypeCategory::Logical) {
+        return; // LOGICAL and INTEGER type supported for the if clause.
+      }
+    }
+  }
+  context_.Say(
+      GetContext().clauseSource, "Must have LOGICAL or INTEGER type"_err_en_US);
+}
+
+void AccStructureChecker::Enter(const parser::OpenACCEndConstruct &x) {
+  context_.Say(x.source, "Misplaced OpenACC end directive"_warn_en_US);
+}
+
 void AccStructureChecker::Enter(const parser::Module &) {
   declareSymbols.clear();
 }
diff --git a/flang/lib/Semantics/check-acc-structure.h b/flang/lib/Semantics/check-acc-structure.h
index 8b87b8ddc5..2a09b7a393 100644
--- a/flang/lib/Semantics/check-acc-structure.h
+++ b/flang/lib/Semantics/check-acc-structure.h
@@ -63,6 +63,7 @@ class AccStructureChecker
   void Enter(const parser::OpenACCCacheConstruct &);
   void Leave(const parser::OpenACCCacheConstruct &);
   void Enter(const parser::AccAtomicUpdate &);
+  void Enter(const parser::OpenACCEndConstruct &);
 
   // Clauses
   void Leave(const parser::AccClauseList &);
diff --git a/flang/lib/Semantics/check-directive-structure.h b/flang/lib/Semantics/check-directive-structure.h
index 14a3151e67..9c3aa47e19 100644
--- a/flang/lib/Semantics/check-directive-structure.h
+++ b/flang/lib/Semantics/check-directive-structure.h
@@ -333,7 +333,7 @@ class DirectiveStructureChecker : public virtual BaseChecker {
 
   void CheckRequireAtLeastOneOf(bool warnInsteadOfError = false);
 
-  void CheckAllowed(C clause);
+  void CheckAllowed(C clause, bool warnInsteadOfError = false);
 
   void CheckAtLeastOneClause();
 
@@ -452,15 +452,21 @@ std::string DirectiveStructureChecker<D, C, PC,
 // Check that clauses present on the directive are allowed clauses.
 template <typename D, typename C, typename PC, std::size_t ClauseEnumSize>
 void DirectiveStructureChecker<D, C, PC, ClauseEnumSize>::CheckAllowed(
-    C clause) {
+    C clause, bool warnInsteadOfError) {
   if (!GetContext().allowedClauses.test(clause) &&
       !GetContext().allowedOnceClauses.test(clause) &&
       !GetContext().allowedExclusiveClauses.test(clause) &&
       !GetContext().requiredClauses.test(clause)) {
-    context_.Say(GetContext().clauseSource,
-        "%s clause is not allowed on the %s directive"_err_en_US,
-        parser::ToUpperCaseLetters(getClauseName(clause).str()),
-        parser::ToUpperCaseLetters(GetContext().directiveSource.ToString()));
+    if (warnInsteadOfError)
+      context_.Say(GetContext().clauseSource,
+          "%s clause is not allowed on the %s directive and will be ignored"_port_en_US,
+          parser::ToUpperCaseLetters(getClauseName(clause).str()),
+          parser::ToUpperCaseLetters(GetContext().directiveSource.ToString()));
+    else
+      context_.Say(GetContext().clauseSource,
+          "%s clause is not allowed on the %s directive"_err_en_US,
+          parser::ToUpperCaseLetters(getClauseName(clause).str()),
+          parser::ToUpperCaseLetters(GetContext().directiveSource.ToString()));
     return;
   }
   if ((GetContext().allowedOnceClauses.test(clause) ||
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index b642cb7678..12f54fbd51 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -1879,9 +1879,6 @@ void OmpStructureChecker::CheckAtomicMemoryOrderClause(
   if (rightHandClauseList) {
     checkForValidMemoryOrderClause(rightHandClauseList);
   }
-  if (numMemoryOrderClause == 0) {
-    atomicDirectiveDefaultOrderFound_ = true;
-  }
 }
 
 void OmpStructureChecker::Enter(const parser::OpenMPAtomicConstruct &x) {
@@ -3219,16 +3216,7 @@ void OmpStructureChecker::Enter(
 void OmpStructureChecker::CheckAllowedRequiresClause(llvmOmpClause clause) {
   CheckAllowed(clause);
 
-  if (clause == llvm::omp::Clause::OMPC_atomic_default_mem_order) {
-    // Check that it does not appear after an atomic operation without memory
-    // order
-    if (atomicDirectiveDefaultOrderFound_) {
-      context_.Say(GetContext().clauseSource,
-          "REQUIRES directive with '%s' clause found lexically after atomic "
-          "operation without a memory order clause"_err_en_US,
-          parser::ToUpperCaseLetters(getClauseName(clause).str()));
-    }
-  } else {
+  if (clause != llvm::omp::Clause::OMPC_atomic_default_mem_order) {
     // Check that it does not appear after a device construct
     if (deviceConstructFound_) {
       context_.Say(GetContext().clauseSource,
diff --git a/flang/lib/Semantics/check-omp-structure.h b/flang/lib/Semantics/check-omp-structure.h
index cdff890ddd..78892d7bd2 100644
--- a/flang/lib/Semantics/check-omp-structure.h
+++ b/flang/lib/Semantics/check-omp-structure.h
@@ -210,7 +210,6 @@ class OmpStructureChecker
 
   void CheckAllowedRequiresClause(llvmOmpClause clause);
   bool deviceConstructFound_{false};
-  bool atomicDirectiveDefaultOrderFound_{false};
 
   void EnterDirectiveNest(const int index) { directiveNest_[index]++; }
   void ExitDirectiveNest(const int index) { directiveNest_[index]--; }
diff --git a/flang/lib/Semantics/compute-offsets.cpp b/flang/lib/Semantics/compute-offsets.cpp
index 139a8eb7c8..375d2e3f7d 100644
--- a/flang/lib/Semantics/compute-offsets.cpp
+++ b/flang/lib/Semantics/compute-offsets.cpp
@@ -116,6 +116,8 @@ void ComputeOffsetsHelper::Compute(Scope &scope) {
       DoSymbol(*symbol);
     }
   }
+  // Ensure that the size is a multiple of the alignment
+  offset_ = Align(offset_, alignment_);
   scope.set_size(offset_);
   scope.SetAlignment(alignment_);
   // Assign offsets in COMMON blocks, unless this scope is a BLOCK construct,
diff --git a/flang/lib/Semantics/data-to-inits.cpp b/flang/lib/Semantics/data-to-inits.cpp
index bc0355a2c5..fe90e0e58e 100644
--- a/flang/lib/Semantics/data-to-inits.cpp
+++ b/flang/lib/Semantics/data-to-inits.cpp
@@ -81,7 +81,7 @@ template <typename DSV = parser::DataStmtValue> class ValueListIterator {
 };
 
 template <typename DSV> void ValueListIterator<DSV>::SetRepetitionCount() {
-  for (repetitionsRemaining_ = 1; at_ != end_; ++at_) {
+  for (; at_ != end_; ++at_) {
     auto repetitions{GetValue().repetitions};
     if (repetitions < 0) {
       hasFatalError_ = true;
@@ -335,10 +335,15 @@ bool DataInitializationCompiler<DSV>::InitElement(
     }
   }};
   const auto GetImage{[&]() -> evaluate::InitialImage & {
-    auto iter{inits_.emplace(&symbol, symbol.size())};
-    auto &symbolInit{iter.first->second};
-    symbolInit.initializedRanges.emplace_back(
-        offsetSymbol.offset(), offsetSymbol.size());
+    // This could be (and was) written to always call std::map<>::emplace(),
+    // which should handle duplicate entries gracefully, but it was still
+    // causing memory allocation & deallocation with gcc.
+    auto iter{inits_.find(&symbol)};
+    if (iter == inits_.end()) {
+      iter = inits_.emplace(&symbol, symbol.size()).first;
+    }
+    auto &symbolInit{iter->second};
+    symbolInit.NoteInitializedRange(offsetSymbol);
     return symbolInit.image;
   }};
   const auto OutOfRangeError{[&]() {
@@ -590,8 +595,7 @@ static void PopulateWithComponentDefaults(SymbolDataInitialization &init,
           }
         }
         if (initialized) {
-          init.initializedRanges.emplace_back(
-              componentOffset, component.size());
+          init.NoteInitializedRange(componentOffset, component.size());
         }
       }
     } else if (const auto *proc{component.detailsIf<ProcEntityDetails>()}) {
@@ -599,8 +603,7 @@ static void PopulateWithComponentDefaults(SymbolDataInitialization &init,
         SomeExpr procPtrInit{evaluate::ProcedureDesignator{**proc->init()}};
         auto extant{init.image.AsConstantPointer(componentOffset)};
         if (!extant || !(*extant == procPtrInit)) {
-          init.initializedRanges.emplace_back(
-              componentOffset, component.size());
+          init.NoteInitializedRange(componentOffset, component.size());
           init.image.AddPointer(componentOffset, std::move(procPtrInit));
         }
       }
@@ -651,7 +654,7 @@ static void IncorporateExplicitInitialization(
   if (iter != inits.end()) { // DATA statement initialization
     for (const auto &range : iter->second.initializedRanges) {
       auto at{offset + range.start()};
-      combined.initializedRanges.emplace_back(at, range.size());
+      combined.NoteInitializedRange(at, range.size());
       combined.image.Incorporate(
           at, iter->second.image, range.start(), range.size());
     }
@@ -663,7 +666,7 @@ static void IncorporateExplicitInitialization(
     if (IsPointer(mutableSymbol)) {
       if (auto *object{mutableSymbol.detailsIf<ObjectEntityDetails>()}) {
         if (object->init()) {
-          combined.initializedRanges.emplace_back(offset, mutableSymbol.size());
+          combined.NoteInitializedRange(offset, mutableSymbol.size());
           combined.image.AddPointer(offset, *object->init());
           if (removeOriginalInits) {
             object->init().reset();
@@ -671,7 +674,7 @@ static void IncorporateExplicitInitialization(
         }
       } else if (auto *proc{mutableSymbol.detailsIf<ProcEntityDetails>()}) {
         if (proc->init() && *proc->init()) {
-          combined.initializedRanges.emplace_back(offset, mutableSymbol.size());
+          combined.NoteInitializedRange(offset, mutableSymbol.size());
           combined.image.AddPointer(
               offset, SomeExpr{evaluate::ProcedureDesignator{**proc->init()}});
           if (removeOriginalInits) {
@@ -681,7 +684,7 @@ static void IncorporateExplicitInitialization(
       }
     } else if (auto *object{mutableSymbol.detailsIf<ObjectEntityDetails>()}) {
       if (!IsNamedConstant(mutableSymbol) && object->init()) {
-        combined.initializedRanges.emplace_back(offset, mutableSymbol.size());
+        combined.NoteInitializedRange(offset, mutableSymbol.size());
         combined.image.Add(
             offset, mutableSymbol.size(), *object->init(), foldingContext);
         if (removeOriginalInits) {
@@ -770,8 +773,7 @@ static bool CombineEquivalencedInitialization(
     return false;
   }
   // If the items are in static storage, save the final initialization.
-  if (std::find_if(associated.begin(), associated.end(),
-          [](SymbolRef ref) { return IsSaved(*ref); }) != associated.end()) {
+  if (llvm::any_of(associated, [](SymbolRef ref) { return IsSaved(*ref); })) {
     // Create a compiler array temp that overlaps all the items.
     SourceName name{exprAnalyzer.context().GetTempName(scope)};
     auto emplaced{
diff --git a/flang/lib/Semantics/data-to-inits.h b/flang/lib/Semantics/data-to-inits.h
index 10d850d23d..d8cc4601de 100644
--- a/flang/lib/Semantics/data-to-inits.h
+++ b/flang/lib/Semantics/data-to-inits.h
@@ -11,6 +11,7 @@
 
 #include "flang/Common/default-kinds.h"
 #include "flang/Common/interval.h"
+#include "flang/Evaluate/fold-designator.h"
 #include "flang/Evaluate/initial-image.h"
 #include <list>
 #include <map>
@@ -30,6 +31,21 @@ struct SymbolDataInitialization {
   using Range = common::Interval<common::ConstantSubscript>;
   explicit SymbolDataInitialization(std::size_t bytes) : image{bytes} {}
   SymbolDataInitialization(SymbolDataInitialization &&) = default;
+
+  void NoteInitializedRange(Range range) {
+    if (initializedRanges.empty() ||
+        !initializedRanges.back().AnnexIfPredecessor(range)) {
+      initializedRanges.emplace_back(range);
+    }
+  }
+  void NoteInitializedRange(
+      common::ConstantSubscript offset, std::size_t size) {
+    NoteInitializedRange(Range{offset, size});
+  }
+  void NoteInitializedRange(evaluate::OffsetSymbol offsetSymbol) {
+    NoteInitializedRange(offsetSymbol.offset(), offsetSymbol.size());
+  }
+
   evaluate::InitialImage image;
   std::list<Range> initializedRanges;
 };
diff --git a/flang/lib/Semantics/mod-file.cpp b/flang/lib/Semantics/mod-file.cpp
index cee267a894..8684eb1fbd 100644
--- a/flang/lib/Semantics/mod-file.cpp
+++ b/flang/lib/Semantics/mod-file.cpp
@@ -1183,30 +1183,49 @@ Scope *ModFileReader::Read(const SourceName &name,
   }
   Scope &topScope{isIntrinsic.value_or(false) ? context_.intrinsicModulesScope()
                                               : context_.globalScope()};
-  if (!ancestor) {
+  Symbol *moduleSymbol{nullptr};
+  if (!ancestor) { // module, not submodule
     parentScope = &topScope;
+    auto pair{parentScope->try_emplace(name, UnknownDetails{})};
+    if (!pair.second) {
+      return nullptr;
+    }
+    moduleSymbol = &*pair.first->second;
+    moduleSymbol->set(Symbol::Flag::ModFile);
   } else if (std::optional<SourceName> parent{GetSubmoduleParent(parseTree)}) {
+    // submodule with submodule parent
     parentScope = Read(*parent, false /*not intrinsic*/, ancestor, silent);
   } else {
+    // submodule with module parent
     parentScope = ancestor;
   }
-  auto pair{parentScope->try_emplace(name, UnknownDetails{})};
-  if (!pair.second) {
-    return nullptr;
-  }
   // Process declarations from the module file
-  Symbol &modSymbol{*pair.first->second};
-  modSymbol.set(Symbol::Flag::ModFile);
   bool wasInModuleFile{context_.foldingContext().inModuleFile()};
   context_.foldingContext().set_inModuleFile(true);
   ResolveNames(context_, parseTree, topScope);
   context_.foldingContext().set_inModuleFile(wasInModuleFile);
-  CHECK(modSymbol.has<ModuleDetails>());
-  CHECK(modSymbol.test(Symbol::Flag::ModFile));
-  if (isIntrinsic.value_or(false)) {
-    modSymbol.attrs().set(Attr::INTRINSIC);
+  if (!moduleSymbol) {
+    // Submodule symbols' storage are owned by their parents' scopes,
+    // but their names are not in their parents' dictionaries -- we
+    // don't want to report bogus errors about clashes between submodule
+    // names and other objects in the parent scopes.
+    if (Scope * submoduleScope{ancestor->FindSubmodule(name)}) {
+      moduleSymbol = submoduleScope->symbol();
+      if (moduleSymbol) {
+        moduleSymbol->set(Symbol::Flag::ModFile);
+      }
+    }
+  }
+  if (moduleSymbol) {
+    CHECK(moduleSymbol->has<ModuleDetails>());
+    CHECK(moduleSymbol->test(Symbol::Flag::ModFile));
+    if (isIntrinsic.value_or(false)) {
+      moduleSymbol->attrs().set(Attr::INTRINSIC);
+    }
+    return moduleSymbol->scope();
+  } else {
+    return nullptr;
   }
-  return modSymbol.scope();
 }
 
 parser::Message &ModFileReader::Say(const SourceName &name,
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 794a784088..f7720fcf43 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -254,9 +254,6 @@ class AccAttributeVisitor : DirectiveAttributeVisitor<llvm::acc::Directive> {
       Symbol::Flag::AccCopyIn, Symbol::Flag::AccCopyOut,
       Symbol::Flag::AccDelete, Symbol::Flag::AccPresent};
 
-  Symbol::Flags accFlagsRequireNewSymbol{Symbol::Flag::AccPrivate,
-      Symbol::Flag::AccFirstPrivate, Symbol::Flag::AccReduction};
-
   Symbol::Flags accDataMvtFlags{
       Symbol::Flag::AccDevice, Symbol::Flag::AccHost, Symbol::Flag::AccSelf};
 
@@ -266,7 +263,7 @@ class AccAttributeVisitor : DirectiveAttributeVisitor<llvm::acc::Directive> {
       Symbol::Flag::AccDevicePtr, Symbol::Flag::AccDeviceResident,
       Symbol::Flag::AccLink, Symbol::Flag::AccPresent};
 
-  void PrivatizeAssociatedLoopIndex(const parser::OpenACCLoopConstruct &);
+  void CheckAssociatedLoopIndex(const parser::OpenACCLoopConstruct &);
   void ResolveAccObjectList(const parser::AccObjectList &, Symbol::Flag);
   void ResolveAccObject(const parser::AccObject &, Symbol::Flag);
   Symbol *ResolveAcc(const parser::Name &, Symbol::Flag, Scope &);
@@ -877,7 +874,7 @@ bool AccAttributeVisitor::Pre(const parser::OpenACCLoopConstruct &x) {
   }
   ClearDataSharingAttributeObjects();
   SetContextAssociatedLoopLevel(GetAssociatedLoopLevelFromClauses(clauseList));
-  PrivatizeAssociatedLoopIndex(x);
+  CheckAssociatedLoopIndex(x);
   return true;
 }
 
@@ -916,6 +913,10 @@ Symbol *AccAttributeVisitor::ResolveFctName(const parser::Name &name) {
   Symbol *prev{currScope().FindSymbol(name.source)};
   if (!prev || (prev && prev->IsFuncResult())) {
     prev = currScope().parent().FindSymbol(name.source);
+    if (!prev) {
+      prev = &context_.globalScope().MakeSymbol(
+          name.source, Attrs{}, ProcEntityDetails{});
+    }
   }
   if (prev != name.symbol) {
     name.symbol = prev;
@@ -968,7 +969,7 @@ void AccAttributeVisitor::AddRoutineInfoToSymbol(
                      std::get_if<Fortran::parser::AccClause::Bind>(&clause.u)) {
         if (const auto *name =
                 std::get_if<Fortran::parser::Name>(&bindClause->v.u)) {
-          if (Symbol *sym = ResolveName(*name, true)) {
+          if (Symbol *sym = ResolveFctName(*name)) {
             info.set_bindName(sym->name().ToString());
           } else {
             context_.Say((*name).source,
@@ -1011,7 +1012,7 @@ bool AccAttributeVisitor::Pre(const parser::OpenACCRoutineConstruct &x) {
 
 bool AccAttributeVisitor::Pre(const parser::AccBindClause &x) {
   if (const auto *name{std::get_if<parser::Name>(&x.u)}) {
-    if (!ResolveName(*name, true)) {
+    if (!ResolveFctName(*name)) {
       context_.Say(name->source,
           "No function or subroutine declared for '%s'"_err_en_US,
           name->source);
@@ -1141,13 +1142,12 @@ std::int64_t AccAttributeVisitor::GetAssociatedLoopLevelFromClauses(
   return 1; // default is outermost loop
 }
 
-void AccAttributeVisitor::PrivatizeAssociatedLoopIndex(
+void AccAttributeVisitor::CheckAssociatedLoopIndex(
     const parser::OpenACCLoopConstruct &x) {
   std::int64_t level{GetContext().associatedLoopLevel};
-  if (level <= 0) { // collpase value was negative or 0
+  if (level <= 0) { // collapse value was negative or 0
     return;
   }
-  Symbol::Flag ivDSA{Symbol::Flag::AccPrivate};
 
   const auto getNextDoConstruct =
       [this](const parser::Block &block) -> const parser::DoConstruct * {
@@ -1166,16 +1166,8 @@ void AccAttributeVisitor::PrivatizeAssociatedLoopIndex(
 
   const auto &outer{std::get<std::optional<parser::DoConstruct>>(x.t)};
   for (const parser::DoConstruct *loop{&*outer}; loop && level > 0; --level) {
-    // go through all the nested do-loops and resolve index variables
-    const parser::Name *iv{GetLoopIndex(*loop)};
-    if (iv) {
-      if (auto *symbol{ResolveAcc(*iv, ivDSA, currScope())}) {
-        symbol->set(Symbol::Flag::AccPreDetermined);
-        iv->symbol = symbol; // adjust the symbol within region
-        AddToContextObjectWithDSA(*symbol, ivDSA);
-      }
-    }
-
+    // Go through all nested loops to ensure index variable exists.
+    GetLoopIndex(*loop);
     const auto &block{std::get<parser::Block>(loop->t)};
     loop = getNextDoConstruct(block);
   }
@@ -1328,20 +1320,12 @@ void AccAttributeVisitor::ResolveAccObject(
 
 Symbol *AccAttributeVisitor::ResolveAcc(
     const parser::Name &name, Symbol::Flag accFlag, Scope &scope) {
-  if (accFlagsRequireNewSymbol.test(accFlag)) {
-    return DeclarePrivateAccessEntity(name, accFlag, scope);
-  } else {
-    return DeclareOrMarkOtherAccessEntity(name, accFlag);
-  }
+  return DeclareOrMarkOtherAccessEntity(name, accFlag);
 }
 
 Symbol *AccAttributeVisitor::ResolveAcc(
     Symbol &symbol, Symbol::Flag accFlag, Scope &scope) {
-  if (accFlagsRequireNewSymbol.test(accFlag)) {
-    return DeclarePrivateAccessEntity(symbol, accFlag, scope);
-  } else {
-    return DeclareOrMarkOtherAccessEntity(symbol, accFlag);
-  }
+  return DeclareOrMarkOtherAccessEntity(symbol, accFlag);
 }
 
 Symbol *AccAttributeVisitor::DeclareOrMarkOtherAccessEntity(
@@ -1374,11 +1358,6 @@ static bool WithMultipleAppearancesAccException(
 void AccAttributeVisitor::CheckMultipleAppearances(
     const parser::Name &name, const Symbol &symbol, Symbol::Flag accFlag) {
   const auto *target{&symbol};
-  if (accFlagsRequireNewSymbol.test(accFlag)) {
-    if (const auto *details{symbol.detailsIf<HostAssocDetails>()}) {
-      target = &details->symbol();
-    }
-  }
   if (HasDataSharingAttributeObject(*target) &&
       !WithMultipleAppearancesAccException(symbol, accFlag)) {
     context_.Say(name.source,
@@ -1542,7 +1521,10 @@ void OmpAttributeVisitor::ResolveSeqLoopIndexInParallelOrTaskConstruct(
   }
   if (auto *symbol{ResolveOmp(iv, Symbol::Flag::OmpPrivate, targetIt->scope)}) {
     targetIt++;
-    symbol->set(Symbol::Flag::OmpPreDetermined);
+    // If this object already had a DSA then it is not predetermined
+    if (!IsObjectWithDSA(*symbol)) {
+      symbol->set(Symbol::Flag::OmpPreDetermined);
+    }
     iv.symbol = symbol; // adjust the symbol within region
     for (auto it{dirContext_.rbegin()}; it != targetIt; ++it) {
       AddToContextObjectWithDSA(*symbol, Symbol::Flag::OmpPrivate, *it);
@@ -1944,18 +1926,19 @@ void OmpAttributeVisitor::ResolveOmpNameList(
 
 Symbol *OmpAttributeVisitor::ResolveOmpCommonBlockName(
     const parser::Name *name) {
-  if (auto *prev{name
-              ? GetContext().scope.parent().FindCommonBlock(name->source)
-              : nullptr}) {
+  if (!name) {
+    return nullptr;
+  }
+  // First check if the Common Block is declared in the current scope
+  if (auto *cur{GetContext().scope.FindCommonBlock(name->source)}) {
+    name->symbol = cur;
+    return cur;
+  }
+  // Then check parent scope
+  if (auto *prev{GetContext().scope.parent().FindCommonBlock(name->source)}) {
     name->symbol = prev;
     return prev;
   }
-  // Check if the Common Block is declared in the current scope
-  if (auto *commonBlockSymbol{
-          name ? GetContext().scope.FindCommonBlock(name->source) : nullptr}) {
-    name->symbol = commonBlockSymbol;
-    return commonBlockSymbol;
-  }
   return nullptr;
 }
 
diff --git a/flang/lib/Semantics/resolve-labels.cpp b/flang/lib/Semantics/resolve-labels.cpp
index ac02801999..d04b8f3eb5 100644
--- a/flang/lib/Semantics/resolve-labels.cpp
+++ b/flang/lib/Semantics/resolve-labels.cpp
@@ -275,7 +275,29 @@ class ParseTreeAnalyzer {
     return PushConstructName(criticalConstruct);
   }
   bool Pre(const parser::DoConstruct &doConstruct) {
-    return PushConstructName(doConstruct);
+    const auto &optionalName{std::get<std::optional<parser::Name>>(
+        std::get<parser::Statement<parser::NonLabelDoStmt>>(doConstruct.t)
+            .statement.t)};
+    if (optionalName) {
+      constructNames_.emplace_back(optionalName->ToString());
+    }
+    // Allow FORTRAN '66 extended DO ranges
+    PushScope().isExteriorGotoFatal = false;
+    // Process labels of the DO and END DO statements, but not the
+    // statements themselves, so that a non-construct END DO
+    // can be distinguished (below).
+    Pre(std::get<parser::Statement<parser::NonLabelDoStmt>>(doConstruct.t));
+    Walk(std::get<parser::Block>(doConstruct.t), *this);
+    Pre(std::get<parser::Statement<parser::EndDoStmt>>(doConstruct.t));
+    PopConstructName(doConstruct);
+    return false;
+  }
+  void Post(const parser::EndDoStmt &endDoStmt) {
+    // Visited only for non-construct labeled DO termination
+    if (const auto &name{endDoStmt.v}) {
+      context_.Say(name->source, "Unexpected DO construct name '%s'"_err_en_US,
+          name->source);
+    }
   }
   bool Pre(const parser::IfConstruct &ifConstruct) {
     return PushConstructName(ifConstruct);
@@ -330,9 +352,6 @@ class ParseTreeAnalyzer {
   void Post(const parser::CriticalConstruct &criticalConstruct) {
     PopConstructName(criticalConstruct);
   }
-  void Post(const parser::DoConstruct &doConstruct) {
-    PopConstructName(doConstruct);
-  }
   void Post(const parser::IfConstruct &ifConstruct) {
     PopConstructName(ifConstruct);
   }
@@ -716,6 +735,22 @@ class ParseTreeAnalyzer {
   // C1131
   void CheckName(const parser::DoConstruct &doConstruct) {
     CheckEndName<parser::NonLabelDoStmt, parser::EndDoStmt>("DO", doConstruct);
+    if (auto label{std::get<std::optional<parser::Label>>(
+            std::get<parser::Statement<parser::NonLabelDoStmt>>(doConstruct.t)
+                .statement.t)}) {
+      const auto &endDoStmt{
+          std::get<parser::Statement<parser::EndDoStmt>>(doConstruct.t)};
+      if (!endDoStmt.label || *endDoStmt.label != *label) {
+        context_
+            .Say(endDoStmt.source,
+                "END DO statement must have the label '%d' matching its DO statement"_err_en_US,
+                *label)
+            .Attach(std::get<parser::Statement<parser::NonLabelDoStmt>>(
+                        doConstruct.t)
+                        .source,
+                "corresponding DO statement"_en_US);
+      }
+    }
   }
   // C1035
   void CheckName(const parser::ForallConstruct &forallConstruct) {
diff --git a/flang/lib/Semantics/resolve-names-utils.cpp b/flang/lib/Semantics/resolve-names-utils.cpp
index ebc7aab374..b901080e28 100644
--- a/flang/lib/Semantics/resolve-names-utils.cpp
+++ b/flang/lib/Semantics/resolve-names-utils.cpp
@@ -779,6 +779,7 @@ class SymbolMapper : public evaluate::AnyTraverse<SymbolMapper, bool> {
     return false;
   }
   void MapSymbolExprs(Symbol &);
+  Symbol *CopySymbol(const Symbol *);
 
 private:
   void MapParamValue(ParamValue &param) const { (*this)(param.GetExplicit()); }
@@ -797,16 +798,44 @@ class SymbolMapper : public evaluate::AnyTraverse<SymbolMapper, bool> {
   SymbolAndTypeMappings &map_;
 };
 
-void SymbolMapper::MapSymbolExprs(Symbol &symbol) {
-  if (auto *object{symbol.detailsIf<ObjectEntityDetails>()}) {
-    if (const DeclTypeSpec *type{object->type()}) {
-      if (const DeclTypeSpec *newType{MapType(*type)}) {
-        object->ReplaceType(*newType);
+Symbol *SymbolMapper::CopySymbol(const Symbol *symbol) {
+  if (symbol) {
+    if (auto *subp{symbol->detailsIf<SubprogramDetails>()}) {
+      if (subp->isInterface()) {
+        if (auto pair{scope_.try_emplace(symbol->name(), symbol->attrs())};
+            pair.second) {
+          Symbol &copy{*pair.first->second};
+          map_.symbolMap[symbol] = &copy;
+          copy.set(symbol->test(Symbol::Flag::Subroutine)
+                  ? Symbol::Flag::Subroutine
+                  : Symbol::Flag::Function);
+          Scope &newScope{scope_.MakeScope(Scope::Kind::Subprogram, &copy)};
+          copy.set_scope(&newScope);
+          copy.set_details(SubprogramDetails{});
+          auto &newSubp{copy.get<SubprogramDetails>()};
+          newSubp.set_isInterface(true);
+          newSubp.set_isDummy(subp->isDummy());
+          newSubp.set_defaultIgnoreTKR(subp->defaultIgnoreTKR());
+          MapSubprogramToNewSymbols(*symbol, copy, newScope, &map_);
+          return &copy;
+        }
       }
+    } else if (Symbol * copy{scope_.CopySymbol(*symbol)}) {
+      map_.symbolMap[symbol] = copy;
+      return copy;
     }
   }
+  return nullptr;
+}
+
+void SymbolMapper::MapSymbolExprs(Symbol &symbol) {
   common::visit(
       common::visitors{[&](ObjectEntityDetails &object) {
+                         if (const DeclTypeSpec * type{object.type()}) {
+                           if (const DeclTypeSpec * newType{MapType(*type)}) {
+                             object.ReplaceType(*newType);
+                           }
+                         }
                          for (ShapeSpec &spec : object.shape()) {
                            MapShapeSpec(spec);
                          }
@@ -892,13 +921,7 @@ const Symbol *SymbolMapper::MapInterface(const Symbol *interface) {
       return interface;
     } else if (const auto *subp{interface->detailsIf<SubprogramDetails>()};
                subp && subp->isInterface()) {
-      if (Symbol *newSymbol{scope_.CopySymbol(*interface)}) {
-        newSymbol->get<SubprogramDetails>().set_isInterface(true);
-        map_.symbolMap[interface] = newSymbol;
-        Scope &newScope{scope_.MakeScope(Scope::Kind::Subprogram, newSymbol)};
-        MapSubprogramToNewSymbols(*interface, *newSymbol, newScope, &map_);
-        return newSymbol;
-      }
+      return CopySymbol(interface);
     }
   }
   return nullptr;
@@ -913,10 +936,11 @@ void MapSubprogramToNewSymbols(const Symbol &oldSymbol, Symbol &newSymbol,
   mappings->symbolMap[&oldSymbol] = &newSymbol;
   const auto &oldDetails{oldSymbol.get<SubprogramDetails>()};
   auto &newDetails{newSymbol.get<SubprogramDetails>()};
+  SymbolMapper mapper{newScope, *mappings};
   for (const Symbol *dummyArg : oldDetails.dummyArgs()) {
     if (!dummyArg) {
       newDetails.add_alternateReturn();
-    } else if (Symbol *copy{newScope.CopySymbol(*dummyArg)}) {
+    } else if (Symbol * copy{mapper.CopySymbol(dummyArg)}) {
       copy->set(Symbol::Flag::Implicit, false);
       newDetails.add_dummyArg(*copy);
       mappings->symbolMap[dummyArg] = copy;
@@ -924,12 +948,12 @@ void MapSubprogramToNewSymbols(const Symbol &oldSymbol, Symbol &newSymbol,
   }
   if (oldDetails.isFunction()) {
     newScope.erase(newSymbol.name());
-    if (Symbol *copy{newScope.CopySymbol(oldDetails.result())}) {
+    const Symbol &result{oldDetails.result()};
+    if (Symbol * copy{mapper.CopySymbol(&result)}) {
       newDetails.set_result(*copy);
-      mappings->symbolMap[&oldDetails.result()] = copy;
+      mappings->symbolMap[&result] = copy;
     }
   }
-  SymbolMapper mapper{newScope, *mappings};
   for (auto &[_, ref] : newScope) {
     mapper.MapSymbolExprs(*ref);
   }
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index 40f5ab9eb6..90c14806af 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -3230,7 +3230,11 @@ bool ModuleVisitor::BeginSubmodule(
 }
 
 void ModuleVisitor::BeginModule(const parser::Name &name, bool isSubmodule) {
-  auto &symbol{MakeSymbol(name, ModuleDetails{isSubmodule})};
+  // Submodule symbols are not visible in their parents' scopes.
+  Symbol &symbol{isSubmodule ? Resolve(name,
+                                   currScope().MakeSymbol(name.source, Attrs{},
+                                       ModuleDetails{true}))
+                             : MakeSymbol(name, ModuleDetails{false})};
   auto &details{symbol.get<ModuleDetails>()};
   PushScope(Scope::Kind::Module, &symbol);
   details.set_scope(&currScope());
@@ -6425,6 +6429,11 @@ std::optional<DerivedTypeSpec> DeclarationVisitor::ResolveDerivedType(
       Say(name, "Derived type '%s' not found"_err_en_US);
       return std::nullopt;
     }
+  } else if (&DEREF(symbol).owner() != &outer &&
+      !ultimate->has<GenericDetails>()) {
+    // Prevent a later declaration in this scope of a host-associated
+    // type name.
+    outer.add_importName(name.source);
   }
   if (CheckUseError(name)) {
     return std::nullopt;
@@ -8092,7 +8101,7 @@ void ResolveNamesVisitor::CheckImport(
     const Symbol &ultimate{symbol->GetUltimate()};
     if (&ultimate.owner() == &currScope()) {
       Say(location, "'%s' from host is not accessible"_err_en_US, name)
-          .Attach(symbol->name(), "'%s' is hidden by this entity"_en_US,
+          .Attach(symbol->name(), "'%s' is hidden by this entity"_because_en_US,
               symbol->name());
     }
   }
diff --git a/flang/lib/Semantics/rewrite-directives.cpp b/flang/lib/Semantics/rewrite-directives.cpp
new file mode 100644
index 0000000000..bab91d2530
--- /dev/null
+++ b/flang/lib/Semantics/rewrite-directives.cpp
@@ -0,0 +1,177 @@
+//===-- lib/Semantics/rewrite-directives.cpp ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "rewrite-directives.h"
+#include "flang/Parser/parse-tree-visitor.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Semantics/semantics.h"
+#include "flang/Semantics/symbol.h"
+#include "llvm/Frontend/OpenMP/OMP.h.inc"
+#include <list>
+
+namespace Fortran::semantics {
+
+using namespace parser::literals;
+
+class DirectiveRewriteMutator {
+public:
+  explicit DirectiveRewriteMutator(SemanticsContext &context)
+      : context_{context} {}
+
+  // Default action for a parse tree node is to visit children.
+  template <typename T> bool Pre(T &) { return true; }
+  template <typename T> void Post(T &) {}
+
+protected:
+  SemanticsContext &context_;
+};
+
+// Rewrite atomic constructs to add an explicit memory ordering to all that do
+// not specify it, honoring in this way the `atomic_default_mem_order` clause of
+// the REQUIRES directive.
+class OmpRewriteMutator : public DirectiveRewriteMutator {
+public:
+  explicit OmpRewriteMutator(SemanticsContext &context)
+      : DirectiveRewriteMutator(context) {}
+
+  template <typename T> bool Pre(T &) { return true; }
+  template <typename T> void Post(T &) {}
+
+  bool Pre(parser::OpenMPAtomicConstruct &);
+  bool Pre(parser::OpenMPRequiresConstruct &);
+
+private:
+  bool atomicDirectiveDefaultOrderFound_{false};
+};
+
+bool OmpRewriteMutator::Pre(parser::OpenMPAtomicConstruct &x) {
+  // Find top-level parent of the operation.
+  Symbol *topLevelParent{common::visit(
+      [&](auto &atomic) {
+        Symbol *symbol{nullptr};
+        Scope *scope{
+            &context_.FindScope(std::get<parser::Verbatim>(atomic.t).source)};
+        do {
+          if (Symbol * parent{scope->symbol()}) {
+            symbol = parent;
+          }
+          scope = &scope->parent();
+        } while (!scope->IsGlobal());
+
+        assert(symbol &&
+            "Atomic construct must be within a scope associated with a symbol");
+        return symbol;
+      },
+      x.u)};
+
+  // Get the `atomic_default_mem_order` clause from the top-level parent.
+  std::optional<common::OmpAtomicDefaultMemOrderType> defaultMemOrder;
+  common::visit(
+      [&](auto &details) {
+        if constexpr (std::is_convertible_v<decltype(&details),
+                          WithOmpDeclarative *>) {
+          if (details.has_ompAtomicDefaultMemOrder()) {
+            defaultMemOrder = *details.ompAtomicDefaultMemOrder();
+          }
+        }
+      },
+      topLevelParent->details());
+
+  if (!defaultMemOrder) {
+    return false;
+  }
+
+  auto findMemOrderClause =
+      [](const std::list<parser::OmpAtomicClause> &clauses) {
+        return std::find_if(
+                   clauses.begin(), clauses.end(), [](const auto &clause) {
+                     return std::get_if<parser::OmpMemoryOrderClause>(
+                         &clause.u);
+                   }) != clauses.end();
+      };
+
+  // Get the clause list to which the new memory order clause must be added,
+  // only if there are no other memory order clauses present for this atomic
+  // directive.
+  std::list<parser::OmpAtomicClause> *clauseList = common::visit(
+      common::visitors{[&](parser::OmpAtomic &atomicConstruct) {
+                         // OmpAtomic only has a single list of clauses.
+                         auto &clauses{std::get<parser::OmpAtomicClauseList>(
+                             atomicConstruct.t)};
+                         return !findMemOrderClause(clauses.v) ? &clauses.v
+                                                               : nullptr;
+                       },
+          [&](auto &atomicConstruct) {
+            // All other atomic constructs have two lists of clauses.
+            auto &clausesLhs{std::get<0>(atomicConstruct.t)};
+            auto &clausesRhs{std::get<2>(atomicConstruct.t)};
+            return !findMemOrderClause(clausesLhs.v) &&
+                    !findMemOrderClause(clausesRhs.v)
+                ? &clausesRhs.v
+                : nullptr;
+          }},
+      x.u);
+
+  // Add a memory order clause to the atomic directive.
+  if (clauseList) {
+    atomicDirectiveDefaultOrderFound_ = true;
+    switch (*defaultMemOrder) {
+    case common::OmpAtomicDefaultMemOrderType::AcqRel:
+      clauseList->emplace_back<parser::OmpMemoryOrderClause>(common::visit(
+          common::visitors{[](parser::OmpAtomicRead &) -> parser::OmpClause {
+                             return parser::OmpClause::Acquire{};
+                           },
+              [](parser::OmpAtomicCapture &) -> parser::OmpClause {
+                return parser::OmpClause::AcqRel{};
+              },
+              [](auto &) -> parser::OmpClause {
+                // parser::{OmpAtomic, OmpAtomicUpdate, OmpAtomicWrite}
+                return parser::OmpClause::Release{};
+              }},
+          x.u));
+      break;
+    case common::OmpAtomicDefaultMemOrderType::Relaxed:
+      clauseList->emplace_back<parser::OmpMemoryOrderClause>(
+          parser::OmpClause{parser::OmpClause::Relaxed{}});
+      break;
+    case common::OmpAtomicDefaultMemOrderType::SeqCst:
+      clauseList->emplace_back<parser::OmpMemoryOrderClause>(
+          parser::OmpClause{parser::OmpClause::SeqCst{}});
+      break;
+    }
+  }
+
+  return false;
+}
+
+bool OmpRewriteMutator::Pre(parser::OpenMPRequiresConstruct &x) {
+  for (parser::OmpClause &clause : std::get<parser::OmpClauseList>(x.t).v) {
+    if (std::holds_alternative<parser::OmpClause::AtomicDefaultMemOrder>(
+            clause.u) &&
+        atomicDirectiveDefaultOrderFound_) {
+      context_.Say(clause.source,
+          "REQUIRES directive with '%s' clause found lexically after atomic "
+          "operation without a memory order clause"_err_en_US,
+          parser::ToUpperCaseLetters(llvm::omp::getOpenMPClauseName(
+              llvm::omp::OMPC_atomic_default_mem_order)
+                                         .str()));
+    }
+  }
+  return false;
+}
+
+bool RewriteOmpParts(SemanticsContext &context, parser::Program &program) {
+  if (!context.IsEnabled(common::LanguageFeature::OpenMP)) {
+    return true;
+  }
+  OmpRewriteMutator ompMutator{context};
+  parser::Walk(program, ompMutator);
+  return !context.AnyFatalError();
+}
+
+} // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/rewrite-directives.h b/flang/lib/Semantics/rewrite-directives.h
new file mode 100644
index 0000000000..6759621928
--- /dev/null
+++ b/flang/lib/Semantics/rewrite-directives.h
@@ -0,0 +1,24 @@
+//===-- lib/Semantics/rewrite-directives.h ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_SEMANTICS_REWRITE_DIRECTIVES_H_
+#define FORTRAN_SEMANTICS_REWRITE_DIRECTIVES_H_
+
+namespace Fortran::parser {
+struct Program;
+} // namespace Fortran::parser
+
+namespace Fortran::semantics {
+class SemanticsContext;
+} // namespace Fortran::semantics
+
+namespace Fortran::semantics {
+bool RewriteOmpParts(SemanticsContext &, parser::Program &);
+} // namespace Fortran::semantics
+
+#endif // FORTRAN_SEMANTICS_REWRITE_DIRECTIVES_H_
diff --git a/flang/lib/Semantics/rewrite-parse-tree.cpp b/flang/lib/Semantics/rewrite-parse-tree.cpp
index 05df1f4080..b4fb72ce21 100644
--- a/flang/lib/Semantics/rewrite-parse-tree.cpp
+++ b/flang/lib/Semantics/rewrite-parse-tree.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "rewrite-parse-tree.h"
+#include "rewrite-directives.h"
 #include "flang/Common/indirection.h"
 #include "flang/Parser/parse-tree-visitor.h"
 #include "flang/Parser/parse-tree.h"
@@ -175,7 +176,7 @@ void RewriteMutator::Post(parser::WriteStmt &x) {
 bool RewriteParseTree(SemanticsContext &context, parser::Program &program) {
   RewriteMutator mutator{context};
   parser::Walk(program, mutator);
-  return !context.AnyFatalError();
+  return !context.AnyFatalError() && RewriteOmpParts(context, program);
 }
 
 } // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/type.cpp b/flang/lib/Semantics/type.cpp
index a72c2e8ea2..e812283fc6 100644
--- a/flang/lib/Semantics/type.cpp
+++ b/flang/lib/Semantics/type.cpp
@@ -37,6 +37,10 @@ void DerivedTypeSpec::ReplaceScope(const Scope &scope) {
   scope_ = &scope;
 }
 
+const Scope *DerivedTypeSpec::GetScope() const {
+  return scope_ ? scope_ : typeSymbol_.scope();
+}
+
 void DerivedTypeSpec::AddRawParamValue(
     const parser::Keyword *keyword, ParamValue &&value) {
   CHECK(parameters_.empty());
diff --git a/flang/module/__fortran_ieee_exceptions.f90 b/flang/module/__fortran_ieee_exceptions.f90
index 77dc6f8551..785c4adaec 100644
--- a/flang/module/__fortran_ieee_exceptions.f90
+++ b/flang/module/__fortran_ieee_exceptions.f90
@@ -27,10 +27,8 @@
     ieee_denorm = ieee_flag_type(32) ! PGI extension
 
   type(ieee_flag_type), parameter :: &
-    ieee_usual(*) = [ &
-      ieee_overflow, ieee_divide_by_zero, ieee_invalid ], &
-    ieee_all(*) = [ &
-      ieee_usual, ieee_underflow, ieee_inexact, ieee_denorm ]
+    ieee_usual(*) = [ ieee_overflow, ieee_divide_by_zero, ieee_invalid ], &
+    ieee_all(*) = [ ieee_usual, ieee_underflow, ieee_inexact ]
 
   type :: ieee_modes_type ! Fortran 2018, 17.7
     private
diff --git a/flang/runtime/ISO_Fortran_binding.cpp b/flang/runtime/ISO_Fortran_binding.cpp
index 15743be88d..ce14684453 100644
--- a/flang/runtime/ISO_Fortran_binding.cpp
+++ b/flang/runtime/ISO_Fortran_binding.cpp
@@ -125,14 +125,18 @@ RT_API_ATTRS int CFI_establish(CFI_cdesc_t *descriptor, void *base_addr,
 }
 
 RT_API_ATTRS int CFI_is_contiguous(const CFI_cdesc_t *descriptor) {
+  // See Descriptor::IsContiguous for the rationale.
+  bool stridesAreContiguous{true};
   CFI_index_t bytes = descriptor->elem_len;
   for (int j{0}; j < descriptor->rank; ++j) {
-    if (bytes != descriptor->dim[j].sm) {
-      return 0;
-    }
+    stridesAreContiguous &=
+        (bytes == descriptor->dim[j].sm) || (descriptor->dim[j].extent == 1);
     bytes *= descriptor->dim[j].extent;
   }
-  return 1;
+  if (stridesAreContiguous || bytes == 0) {
+    return 1;
+  }
+  return 0;
 }
 
 RT_API_ATTRS int CFI_section(CFI_cdesc_t *result, const CFI_cdesc_t *source,
diff --git a/flang/runtime/allocatable.cpp b/flang/runtime/allocatable.cpp
index 4b9e438e8a..409255aaa2 100644
--- a/flang/runtime/allocatable.cpp
+++ b/flang/runtime/allocatable.cpp
@@ -130,17 +130,7 @@ void RTNAME(AllocatableApplyMold)(
     // 9.7.1.3 Return so the error can be emitted by AllocatableAllocate.
     return;
   }
-  descriptor = mold;
-  descriptor.set_base_addr(nullptr);
-  descriptor.raw().attribute = CFI_attribute_allocatable;
-  descriptor.raw().rank = rank;
-  if (auto *descAddendum{descriptor.Addendum()}) {
-    if (const auto *moldAddendum{mold.Addendum()}) {
-      if (const auto *derived{moldAddendum->derivedType()}) {
-        descAddendum->set_derivedType(derived);
-      }
-    }
-  }
+  descriptor.ApplyMold(mold, rank);
 }
 
 int RTNAME(AllocatableAllocate)(Descriptor &descriptor, bool hasStat,
@@ -198,14 +188,15 @@ int RTNAME(AllocatableDeallocatePolymorphic)(Descriptor &descriptor,
   int stat{RTNAME(AllocatableDeallocate)(
       descriptor, hasStat, errMsg, sourceFile, sourceLine)};
   if (stat == StatOk) {
-    DescriptorAddendum *addendum{descriptor.Addendum()};
-    if (addendum) {
+    if (DescriptorAddendum * addendum{descriptor.Addendum()}) {
       addendum->set_derivedType(derivedType);
+      descriptor.raw().type = derivedType ? CFI_type_struct : CFI_type_other;
     } else {
       // Unlimited polymorphic descriptors initialized with
       // AllocatableInitIntrinsic do not have an addendum. Make sure the
       // derivedType is null in that case.
       INTERNAL_CHECK(!derivedType);
+      descriptor.raw().type = CFI_type_other;
     }
   }
   return stat;
diff --git a/flang/runtime/derived-api.cpp b/flang/runtime/derived-api.cpp
index 66123030f9..39bf0521e7 100644
--- a/flang/runtime/derived-api.cpp
+++ b/flang/runtime/derived-api.cpp
@@ -101,73 +101,55 @@ static const typeInfo::DerivedType *GetDerivedType(const Descriptor &desc) {
 }
 
 bool RTNAME(SameTypeAs)(const Descriptor &a, const Descriptor &b) {
-  // Unlimited polymorphic with intrinsic dynamic type.
-  if (a.raw().type != CFI_type_struct && a.raw().type != CFI_type_other &&
-      b.raw().type != CFI_type_struct && b.raw().type != CFI_type_other)
-    return a.raw().type == b.raw().type;
-
-  const typeInfo::DerivedType *derivedTypeA{GetDerivedType(a)};
-  const typeInfo::DerivedType *derivedTypeB{GetDerivedType(b)};
-
-  // No dynamic type in one or both descriptor.
-  if (derivedTypeA == nullptr || derivedTypeB == nullptr) {
-    return false;
-  }
-
-  // Exact match of derived type.
-  if (derivedTypeA == derivedTypeB) {
-    return true;
+  auto aType{a.raw().type};
+  auto bType{b.raw().type};
+  if ((aType != CFI_type_struct && aType != CFI_type_other) ||
+      (bType != CFI_type_struct && bType != CFI_type_other)) {
+    // If either type is intrinsic, they must match.
+    return aType == bType;
+  } else {
+    const typeInfo::DerivedType *derivedTypeA{GetDerivedType(a)};
+    const typeInfo::DerivedType *derivedTypeB{GetDerivedType(b)};
+    if (derivedTypeA == nullptr || derivedTypeB == nullptr) {
+      // Unallocated/disassociated CLASS(*) never matches.
+      return false;
+    } else if (derivedTypeA == derivedTypeB) {
+      // Exact match of derived type.
+      return true;
+    } else {
+      // Otherwise compare with the name. Note 16.29 kind type parameters are
+      // not considered in the test.
+      return CompareDerivedTypeNames(
+          derivedTypeA->name(), derivedTypeB->name());
+    }
   }
-  // Otherwise compare with the name. Note 16.29 kind type parameters are not
-  // considered in the test.
-  return CompareDerivedTypeNames(derivedTypeA->name(), derivedTypeB->name());
 }
 
 bool RTNAME(ExtendsTypeOf)(const Descriptor &a, const Descriptor &mold) {
-  if (a.raw().type != CFI_type_struct && a.raw().type != CFI_type_other &&
-      mold.raw().type != CFI_type_struct && mold.raw().type != CFI_type_other)
-    return a.raw().type == mold.raw().type;
-
-  const typeInfo::DerivedType *derivedTypeA{GetDerivedType(a)};
-  const typeInfo::DerivedType *derivedTypeMold{GetDerivedType(mold)};
-
-  // If MOLD is unlimited polymorphic and is either a disassociated pointer or
-  // unallocated allocatable, the result is true.
-  // Unlimited polymorphic descriptors are initialized with a CFI_type_other
-  // type.
-  if (mold.type().raw() == CFI_type_other &&
-      (mold.IsAllocatable() || mold.IsPointer()) &&
-      derivedTypeMold == nullptr) {
-    return true;
-  }
-
-  // If A is unlimited polymorphic and is either a disassociated pointer or
-  // unallocated allocatable, the result is false.
-  // Unlimited polymorphic descriptors are initialized with a CFI_type_other
-  // type.
-  if (a.type().raw() == CFI_type_other &&
-      (a.IsAllocatable() || a.IsPointer()) && derivedTypeA == nullptr) {
-    return false;
-  }
-
-  if (derivedTypeA == nullptr || derivedTypeMold == nullptr) {
+  auto aType{a.raw().type};
+  auto moldType{mold.raw().type};
+  if ((aType != CFI_type_struct && aType != CFI_type_other) ||
+      (moldType != CFI_type_struct && moldType != CFI_type_other)) {
+    // If either type is intrinsic, they must match.
+    return aType == moldType;
+  } else if (const typeInfo::DerivedType *
+      derivedTypeMold{GetDerivedType(mold)}) {
+    // If A is unlimited polymorphic and is either a disassociated pointer or
+    // unallocated allocatable, the result is false.
+    // Otherwise if the dynamic type of A or MOLD is extensible, the result is
+    // true if and only if the dynamic type of A is an extension type of the
+    // dynamic type of MOLD.
+    for (const typeInfo::DerivedType *derivedTypeA{GetDerivedType(a)};
+         derivedTypeA; derivedTypeA = derivedTypeA->GetParentType()) {
+      if (CompareDerivedType(derivedTypeA, derivedTypeMold)) {
+        return true;
+      }
+    }
     return false;
-  }
-
-  // Otherwise if the dynamic type of A or MOLD is extensible, the result is
-  // true if and only if the dynamic type of A is an extension type of the
-  // dynamic type of MOLD.
-  if (CompareDerivedType(derivedTypeA, derivedTypeMold)) {
+  } else {
+    // MOLD is unlimited polymorphic and unallocated/disassociated.
     return true;
   }
-  const typeInfo::DerivedType *parent{derivedTypeA->GetParentType()};
-  while (parent) {
-    if (CompareDerivedType(parent, derivedTypeMold)) {
-      return true;
-    }
-    parent = parent->GetParentType();
-  }
-  return false;
 }
 
 void RTNAME(DestroyWithoutFinalization)(const Descriptor &descriptor) {
diff --git a/flang/runtime/descriptor-io.h b/flang/runtime/descriptor-io.h
index 840d73b8e8..e20f31e9b4 100644
--- a/flang/runtime/descriptor-io.h
+++ b/flang/runtime/descriptor-io.h
@@ -288,7 +288,11 @@ static bool DefaultComponentwiseIO(IoStatementState &io,
           *compArray.Element<typeInfo::Component>(at)};
       if (!DefaultComponentIO<DIR>(
               io, component, descriptor, subscripts, handler, table)) {
-        return false;
+        // Truncated nonempty namelist input sequence?
+        auto *listInput{
+            io.get_if<ListDirectedStatementState<Direction::Input>>()};
+        return DIR == Direction::Input && (j > 0 || k > 0) && listInput &&
+            listInput->inNamelistSequence();
       }
     }
   }
diff --git a/flang/runtime/descriptor.cpp b/flang/runtime/descriptor.cpp
index 8dd3f21527..34ca33a6a8 100644
--- a/flang/runtime/descriptor.cpp
+++ b/flang/runtime/descriptor.cpp
@@ -243,6 +243,22 @@ RT_API_ATTRS bool Descriptor::EstablishPointerSection(const Descriptor &source,
   return CFI_section(&raw_, &source.raw_, lower, upper, stride) == CFI_SUCCESS;
 }
 
+RT_API_ATTRS void Descriptor::ApplyMold(const Descriptor &mold, int rank) {
+  raw_.elem_len = mold.raw_.elem_len;
+  raw_.rank = rank;
+  raw_.type = mold.raw_.type;
+  for (int j{0}; j < rank && j < mold.raw_.rank; ++j) {
+    GetDimension(j) = mold.GetDimension(j);
+  }
+  if (auto *addendum{Addendum()}) {
+    if (auto *moldAddendum{mold.Addendum()}) {
+      *addendum = *moldAddendum;
+    } else {
+      INTERNAL_CHECK(!addendum->derivedType());
+    }
+  }
+}
+
 RT_API_ATTRS void Descriptor::Check() const {
   // TODO
 }
diff --git a/flang/runtime/edit-input.cpp b/flang/runtime/edit-input.cpp
index 1861c9f849..4e8c9aa868 100644
--- a/flang/runtime/edit-input.cpp
+++ b/flang/runtime/edit-input.cpp
@@ -125,7 +125,7 @@ static bool EditBOZInput(
   return CheckCompleteListDirectedField(io, edit);
 }
 
-static inline char32_t GetDecimalPoint(const DataEdit &edit) {
+static inline char32_t GetRadixPointChar(const DataEdit &edit) {
   return edit.modes.editingFlags & decimalComma ? char32_t{','} : char32_t{'.'};
 }
 
@@ -229,17 +229,22 @@ bool EditIntegerInput(
 
 // Parses a REAL input number from the input source as a normalized
 // fraction into a supplied buffer -- there's an optional '-', a
-// decimal point, and at least one digit.  The adjusted exponent value
-// is returned in a reference argument.  The returned value is the number
-// of characters that (should) have been written to the buffer -- this can
-// be larger than the buffer size and can indicate overflow.  Replaces
-// blanks with zeroes if appropriate.
-static int ScanRealInput(char *buffer, int bufferSize, IoStatementState &io,
-    const DataEdit &edit, int &exponent) {
+// decimal point when the input is not hexadecimal, and at least one
+// digit.  Replaces blanks with zeroes where appropriate.
+struct ScannedRealInput {
+  // Number of characters that (should) have been written to the
+  // buffer -- this can be larger than the buffer size, which
+  // indicates buffer overflow.  Zero indicates an error.
+  int got{0};
+  int exponent{0}; // adjusted as necessary; binary if isHexadecimal
+  bool isHexadecimal{false}; // 0X...
+};
+static ScannedRealInput ScanRealInput(
+    char *buffer, int bufferSize, IoStatementState &io, const DataEdit &edit) {
   std::optional<int> remaining;
   std::optional<char32_t> next;
   int got{0};
-  std::optional<int> decimalPoint;
+  std::optional<int> radixPointOffset;
   auto Put{[&](char ch) -> void {
     if (got < bufferSize) {
       buffer[got] = ch;
@@ -251,6 +256,7 @@ static int ScanRealInput(char *buffer, int bufferSize, IoStatementState &io,
     Put('-');
   }
   bool bzMode{(edit.modes.editingFlags & blankZero) != 0};
+  int exponent{0};
   if (!next || (!bzMode && *next == ' ')) {
     if (!edit.IsListDirected() && !io.GetConnectionState().IsAtEOF()) {
       // An empty/blank field means zero when not list-directed.
@@ -259,10 +265,11 @@ static int ScanRealInput(char *buffer, int bufferSize, IoStatementState &io,
       // required to pass FCVS.
       Put('0');
     }
-    return got;
+    return {got, exponent, false};
   }
-  char32_t decimal{GetDecimalPoint(edit)};
+  char32_t radixPointChar{GetRadixPointChar(edit)};
   char32_t first{*next >= 'a' && *next <= 'z' ? *next + 'A' - 'a' : *next};
+  bool isHexadecimal{false};
   if (first == 'N' || first == 'I') {
     // NaN or infinity - convert to upper case
     // Subtle: a blank field of digits could be followed by 'E' or 'D',
@@ -283,7 +290,7 @@ static int ScanRealInput(char *buffer, int bufferSize, IoStatementState &io,
         if (depth == 0) {
           break;
         } else if (!next) {
-          return 0; // error
+          return {}; // error
         } else if (*next == '(') {
           ++depth;
         } else if (*next == ')') {
@@ -292,34 +299,51 @@ static int ScanRealInput(char *buffer, int bufferSize, IoStatementState &io,
         Put(*next);
       }
     }
-    exponent = 0;
-  } else if (first == decimal || (first >= '0' && first <= '9') ||
+  } else if (first == radixPointChar || (first >= '0' && first <= '9') ||
       (bzMode && (first == ' ' || first == '\t')) || first == 'E' ||
       first == 'D' || first == 'Q') {
-    Put('.'); // input field is normalized to a fraction
+    if (first == '0') {
+      next = io.NextInField(remaining, edit);
+      if (next && (*next == 'x' || *next == 'X')) { // 0X...
+        isHexadecimal = true;
+        next = io.NextInField(remaining, edit);
+      } else {
+        Put('0');
+      }
+    }
+    // input field is normalized to a fraction
+    if (!isHexadecimal) {
+      Put('.');
+    }
     auto start{got};
     for (; next; next = io.NextInField(remaining, edit)) {
       char32_t ch{*next};
       if (ch == ' ' || ch == '\t') {
-        if (bzMode) {
+        if (isHexadecimal) {
+          return {}; // error
+        } else if (bzMode) {
           ch = '0'; // BZ mode - treat blank as if it were zero
         } else {
-          continue;
+          continue; // ignore blank in fixed field
         }
       }
-      if (ch == '0' && got == start && !decimalPoint) {
-        // omit leading zeroes before the decimal
+      if (ch == '0' && got == start && !radixPointOffset) {
+        // omit leading zeroes before the radix point
       } else if (ch >= '0' && ch <= '9') {
         Put(ch);
-      } else if (ch == decimal && !decimalPoint) {
-        // the decimal point is *not* copied to the buffer
-        decimalPoint = got - start; // # of digits before the decimal point
+      } else if (ch == radixPointChar && !radixPointOffset) {
+        // The radix point character is *not* copied to the buffer.
+        radixPointOffset = got - start; // # of digits before the radix point
+      } else if (isHexadecimal && ch >= 'A' && ch <= 'F') {
+        Put(ch);
+      } else if (isHexadecimal && ch >= 'a' && ch <= 'f') {
+        Put(ch - 'a' + 'A'); // normalize to capitals
       } else {
         break;
       }
     }
     if (got == start) {
-      // Nothing but zeroes and maybe a decimal point.  F'2018 requires
+      // Nothing but zeroes and maybe a radix point.  F'2018 requires
       // at least one digit, but F'77 did not, and a bare "." shows up in
       // the FCVS suite.
       Put('0'); // emit at least one digit
@@ -328,17 +352,22 @@ static int ScanRealInput(char *buffer, int bufferSize, IoStatementState &io,
     auto nextBeforeExponent{next};
     auto startExponent{io.GetConnectionState().positionInRecord};
     bool hasGoodExponent{false};
-    if (next &&
-        (*next == 'e' || *next == 'E' || *next == 'd' || *next == 'D' ||
-            *next == 'q' || *next == 'Q')) {
-      // Optional exponent letter.  Blanks are allowed between the
-      // optional exponent letter and the exponent value.
-      io.SkipSpaces(remaining);
-      next = io.NextInField(remaining, edit);
+    if (next) {
+      if (isHexadecimal) {
+        if (*next == 'p' || *next == 'P') {
+          next = io.NextInField(remaining, edit);
+        } else {
+          // The binary exponent is not optional in the standard.
+          return {}; // error
+        }
+      } else if (*next == 'e' || *next == 'E' || *next == 'd' || *next == 'D' ||
+          *next == 'q' || *next == 'Q') {
+        // Optional exponent letter.  Blanks are allowed between the
+        // optional exponent letter and the exponent value.
+        io.SkipSpaces(remaining);
+        next = io.NextInField(remaining, edit);
+      }
     }
-    // The default exponent is -kP, but the scale factor doesn't affect
-    // an explicit exponent.
-    exponent = -edit.modes.scale;
     if (next &&
         (*next == '-' || *next == '+' || (*next >= '0' && *next <= '9') ||
             *next == ' ' || *next == '\t')) {
@@ -346,14 +375,16 @@ static int ScanRealInput(char *buffer, int bufferSize, IoStatementState &io,
       if (negExpo || *next == '+') {
         next = io.NextInField(remaining, edit);
       }
-      for (exponent = 0; next; next = io.NextInField(remaining, edit)) {
+      for (; next; next = io.NextInField(remaining, edit)) {
         if (*next >= '0' && *next <= '9') {
           hasGoodExponent = true;
           if (exponent < 10000) {
             exponent = 10 * exponent + *next - '0';
           }
         } else if (*next == ' ' || *next == '\t') {
-          if (bzMode) {
+          if (isHexadecimal) {
+            break;
+          } else if (bzMode) {
             hasGoodExponent = true;
             exponent = 10 * exponent;
           }
@@ -366,23 +397,29 @@ static int ScanRealInput(char *buffer, int bufferSize, IoStatementState &io,
       }
     }
     if (!hasGoodExponent) {
+      if (isHexadecimal) {
+        return {}; // error
+      }
       // There isn't a good exponent; do not consume it.
       next = nextBeforeExponent;
       io.HandleAbsolutePosition(startExponent);
-    }
-    if (decimalPoint) {
-      exponent += *decimalPoint;
+      // The default exponent is -kP, but the scale factor doesn't affect
+      // an explicit exponent.
+      exponent = -edit.modes.scale;
+    }
+    // Adjust exponent by number of digits before the radix point.
+    if (isHexadecimal) {
+      // Exponents for hexadecimal input are binary.
+      exponent += radixPointOffset.value_or(got - start) * 4;
+    } else if (radixPointOffset) {
+      exponent += *radixPointOffset;
     } else {
-      // When no decimal point (or comma) appears in the value, the 'd'
+      // When no redix point (or comma) appears in the value, the 'd'
       // part of the edit descriptor must be interpreted as the number of
       // digits in the value to be interpreted as being to the *right* of
-      // the assumed decimal point (13.7.2.3.2)
+      // the assumed radix point (13.7.2.3.2)
       exponent += got - start - edit.digits.value_or(0);
     }
-  } else {
-    // TODO: hex FP input
-    exponent = 0;
-    return 0;
   }
   // Consume the trailing ')' of a list-directed or NAMELIST complex
   // input value.
@@ -403,10 +440,10 @@ static int ScanRealInput(char *buffer, int bufferSize, IoStatementState &io,
       next = io.NextInField(remaining, edit);
     }
     if (next) {
-      return 0; // error: unused nonblank character in fixed-width field
+      return {}; // error: unused nonblank character in fixed-width field
     }
   }
-  return got;
+  return {got, exponent, isHexadecimal};
 }
 
 static void RaiseFPExceptions(decimal::ConversionResultFlags flags) {
@@ -433,7 +470,7 @@ static void RaiseFPExceptions(decimal::ConversionResultFlags flags) {
 // converter without modification, this fast path for real input
 // saves time by avoiding memory copies and reformatting of the exponent.
 template <int PRECISION>
-static bool TryFastPathRealInput(
+static bool TryFastPathRealDecimalInput(
     IoStatementState &io, const DataEdit &edit, void *n) {
   if (edit.modes.editingFlags & (blankZero | decimalComma)) {
     return false;
@@ -504,10 +541,103 @@ static bool TryFastPathRealInput(
   return true;
 }
 
+template <int binaryPrecision>
+decimal::ConversionToBinaryResult<binaryPrecision> ConvertHexadecimal(
+    const char *&p, enum decimal::FortranRounding rounding, int expo) {
+  using RealType = decimal::BinaryFloatingPointNumber<binaryPrecision>;
+  using RawType = typename RealType::RawType;
+  bool isNegative{*p == '-'};
+  constexpr RawType one{1};
+  RawType signBit{0};
+  if (isNegative) {
+    ++p;
+    signBit = one << (RealType::bits - 1);
+  }
+  RawType fraction{0};
+  // Adjust the incoming binary P+/- exponent to shift the radix point
+  // to below the LSB and add in the bias.
+  expo += binaryPrecision - 1 + RealType::exponentBias;
+  // Input the fraction.
+  int roundingBit{0};
+  int guardBit{0};
+  for (; *p; ++p) {
+    fraction <<= 4;
+    expo -= 4;
+    if (*p >= '0' && *p <= '9') {
+      fraction |= *p - '0';
+    } else if (*p >= 'A' && *p <= 'F') {
+      fraction |= *p - 'A' + 10; // data were normalized to capitals
+    } else {
+      break;
+    }
+    while (fraction >> binaryPrecision) {
+      guardBit |= roundingBit;
+      roundingBit = (int)fraction & 1;
+      fraction >>= 1;
+      ++expo;
+    }
+  }
+  if (fraction) {
+    // Boost biased expo if too small
+    while (expo < 1) {
+      guardBit |= roundingBit;
+      roundingBit = (int)fraction & 1;
+      fraction >>= 1;
+      ++expo;
+    }
+    // Normalize
+    while (expo > 1 && !(fraction >> (binaryPrecision - 1))) {
+      fraction <<= 1;
+      --expo;
+    }
+    // Rounding
+    bool increase{false};
+    switch (rounding) {
+    case decimal::RoundNearest: // RN & RP
+      increase = roundingBit && (guardBit | ((int)fraction & 1));
+      break;
+    case decimal::RoundUp: // RU
+      increase = !isNegative && (roundingBit | guardBit);
+      break;
+    case decimal::RoundDown: // RD
+      increase = isNegative && (roundingBit | guardBit);
+      break;
+    case decimal::RoundToZero: // RZ
+      break;
+    case decimal::RoundCompatible: // RC
+      increase = roundingBit != 0;
+      break;
+    }
+    if (increase) {
+      ++fraction;
+      if (fraction >> binaryPrecision) {
+        fraction >>= 1;
+        ++expo;
+      }
+    }
+  }
+  // Package & return result
+  constexpr RawType significandMask{(one << RealType::significandBits) - 1};
+  if (!fraction) {
+    expo = 0;
+  } else if (expo == 1 && !(fraction >> (binaryPrecision - 1))) {
+    expo = 0; // subnormal
+  } else if (expo >= RealType::maxExponent) {
+    expo = RealType::maxExponent; // +/-Inf
+    fraction = 0;
+  } else {
+    fraction &= significandMask; // remove explicit normalization unless x87
+  }
+  return decimal::ConversionToBinaryResult<binaryPrecision>{
+      RealType{static_cast<RawType>(signBit |
+          static_cast<RawType>(expo) << RealType::significandBits | fraction)},
+      (roundingBit | guardBit) ? decimal::Inexact : decimal::Exact};
+}
+
 template <int KIND>
 bool EditCommonRealInput(IoStatementState &io, const DataEdit &edit, void *n) {
   constexpr int binaryPrecision{common::PrecisionOfRealKind(KIND)};
-  if (TryFastPathRealInput<binaryPrecision>(io, edit, n)) {
+  if (TryFastPathRealDecimalInput<binaryPrecision>(io, edit, n)) {
     return CheckCompleteListDirectedField(io, edit);
   }
   // Fast path wasn't available or didn't work; go the more general route
@@ -515,8 +645,8 @@ bool EditCommonRealInput(IoStatementState &io, const DataEdit &edit, void *n) {
       common::MaxDecimalConversionDigits(binaryPrecision)};
   static constexpr int bufferSize{maxDigits + 18};
   char buffer[bufferSize];
-  int exponent{0};
-  int got{ScanRealInput(buffer, maxDigits + 2, io, edit, exponent)};
+  auto scanned{ScanRealInput(buffer, maxDigits + 2, io, edit)};
+  int got{scanned.got};
   if (got >= maxDigits + 2) {
     io.GetIoErrorHandler().Crash("EditCommonRealInput: buffer was too small");
     return false;
@@ -529,48 +659,55 @@ bool EditCommonRealInput(IoStatementState &io, const DataEdit &edit, void *n) {
         static_cast<int>(connection.currentRecordNumber));
     return false;
   }
-  bool hadExtra{got > maxDigits};
-  if (exponent != 0) {
-    buffer[got++] = 'e';
-    if (exponent < 0) {
-      buffer[got++] = '-';
-      exponent = -exponent;
-    }
-    if (exponent > 9999) {
-      exponent = 9999; // will convert to +/-Inf
-    }
-    if (exponent > 999) {
-      int dig{exponent / 1000};
-      buffer[got++] = '0' + dig;
-      int rest{exponent - 1000 * dig};
-      dig = rest / 100;
-      buffer[got++] = '0' + dig;
-      rest -= 100 * dig;
-      dig = rest / 10;
-      buffer[got++] = '0' + dig;
-      buffer[got++] = '0' + (rest - 10 * dig);
-    } else if (exponent > 99) {
-      int dig{exponent / 100};
-      buffer[got++] = '0' + dig;
-      int rest{exponent - 100 * dig};
-      dig = rest / 10;
-      buffer[got++] = '0' + dig;
-      buffer[got++] = '0' + (rest - 10 * dig);
-    } else if (exponent > 9) {
-      int dig{exponent / 10};
-      buffer[got++] = '0' + dig;
-      buffer[got++] = '0' + (exponent - 10 * dig);
-    } else {
-      buffer[got++] = '0' + exponent;
-    }
-  }
-  buffer[got] = '\0';
+  decimal::ConversionToBinaryResult<binaryPrecision> converted;
   const char *p{buffer};
-  decimal::ConversionToBinaryResult<binaryPrecision> converted{
-      decimal::ConvertToBinary<binaryPrecision>(p, edit.modes.round)};
-  if (hadExtra) {
-    converted.flags = static_cast<enum decimal::ConversionResultFlags>(
-        converted.flags | decimal::Inexact);
+  if (scanned.isHexadecimal) {
+    buffer[got] = '\0';
+    converted = ConvertHexadecimal<binaryPrecision>(
+        p, edit.modes.round, scanned.exponent);
+  } else {
+    bool hadExtra{got > maxDigits};
+    int exponent{scanned.exponent};
+    if (exponent != 0) {
+      buffer[got++] = 'e';
+      if (exponent < 0) {
+        buffer[got++] = '-';
+        exponent = -exponent;
+      }
+      if (exponent > 9999) {
+        exponent = 9999; // will convert to +/-Inf
+      }
+      if (exponent > 999) {
+        int dig{exponent / 1000};
+        buffer[got++] = '0' + dig;
+        int rest{exponent - 1000 * dig};
+        dig = rest / 100;
+        buffer[got++] = '0' + dig;
+        rest -= 100 * dig;
+        dig = rest / 10;
+        buffer[got++] = '0' + dig;
+        buffer[got++] = '0' + (rest - 10 * dig);
+      } else if (exponent > 99) {
+        int dig{exponent / 100};
+        buffer[got++] = '0' + dig;
+        int rest{exponent - 100 * dig};
+        dig = rest / 10;
+        buffer[got++] = '0' + dig;
+        buffer[got++] = '0' + (rest - 10 * dig);
+      } else if (exponent > 9) {
+        int dig{exponent / 10};
+        buffer[got++] = '0' + dig;
+        buffer[got++] = '0' + (exponent - 10 * dig);
+      } else {
+        buffer[got++] = '0' + exponent;
+      }
+    }
+    buffer[got] = '\0';
+    converted = decimal::ConvertToBinary<binaryPrecision>(p, edit.modes.round);
+    if (hadExtra) {
+      converted.flags = static_cast<enum decimal::ConversionResultFlags>(
+          converted.flags | decimal::Inexact);
+    }
   }
   if (*p) { // unprocessed junk after value
     const auto &connection{io.GetConnectionState()};
diff --git a/flang/runtime/edit-output.cpp b/flang/runtime/edit-output.cpp
index be0bb07f08..18b209bc67 100644
--- a/flang/runtime/edit-output.cpp
+++ b/flang/runtime/edit-output.cpp
@@ -205,13 +205,20 @@ const char *RealOutputEditingBase::FormatExponent(
     } else if (exponent == eEnd) {
       *--exponent = '0'; // Ew.dE0 with zero-valued exponent
     }
-  } else { // ensure at least two exponent digits
+  } else if (edit.variation == 'X') {
+    if (expo == 0) {
+      *--exponent = '0'; // EX without Ee and zero-valued exponent
+    }
+  } else {
+    // Ensure at least two exponent digits unless EX
     while (exponent + 2 > eEnd) {
       *--exponent = '0';
     }
   }
   *--exponent = expo < 0 ? '-' : '+';
-  if (edit.expoDigits || edit.IsListDirected() || exponent + 3 == eEnd) {
+  if (edit.variation == 'X') {
+    *--exponent = 'P';
+  } else if (edit.expoDigits || edit.IsListDirected() || exponent + 3 == eEnd) {
     *--exponent = edit.descriptor == 'D' ? 'D' : 'E'; // not 'G' or 'Q'
   }
   length = eEnd - exponent;
@@ -251,19 +258,32 @@ bool RealOutputEditingBase::EmitSuffix(const DataEdit &edit) {
 }
 
 template <int KIND>
-decimal::ConversionToDecimalResult RealOutputEditing<KIND>::Convert(
+decimal::ConversionToDecimalResult RealOutputEditing<KIND>::ConvertToDecimal(
     int significantDigits, enum decimal::FortranRounding rounding, int flags) {
   auto converted{decimal::ConvertToDecimal<binaryPrecision>(buffer_,
       sizeof buffer_, static_cast<enum decimal::DecimalConversionFlags>(flags),
       significantDigits, rounding, x_)};
   if (!converted.str) { // overflow
     io_.GetIoErrorHandler().Crash(
-        "RealOutputEditing::Convert : buffer size %zd was insufficient",
+        "RealOutputEditing::ConvertToDecimal: buffer size %zd was insufficient",
         sizeof buffer_);
   }
   return converted;
 }
 
+static bool IsInfOrNaN(const char *p, int length) {
+  if (!p || length < 1) {
+    return false;
+  }
+  if (*p == '-' || *p == '+') {
+    if (length == 1) {
+      return false;
+    }
+    ++p;
+  }
+  return *p == 'I' || *p == 'N';
+}
+
 // 13.7.2.3.3 in F'2018
 template <int KIND>
 bool RealOutputEditing<KIND>::EditEorDOutput(const DataEdit &edit) {
@@ -275,7 +295,6 @@ bool RealOutputEditing<KIND>::EditEorDOutput(const DataEdit &edit) {
   if (edit.modes.editingFlags & signPlus) {
     flags |= decimal::AlwaysSign;
   }
-  bool noLeadingSpaces{editWidth == 0};
   int scale{edit.modes.scale}; // 'kP' value
   if (editWidth == 0) { // "the processor selects the field width"
     if (edit.digits.has_value()) { // E0.d
@@ -319,8 +338,8 @@ bool RealOutputEditing<KIND>::EditEorDOutput(const DataEdit &edit) {
   // In EN editing, multiple attempts may be necessary, so this is a loop.
   while (true) {
     decimal::ConversionToDecimalResult converted{
-        Convert(significantDigits, edit.modes.round, flags)};
-    if (IsInfOrNaN(converted)) {
+        ConvertToDecimal(significantDigits, edit.modes.round, flags)};
+    if (IsInfOrNaN(converted.str, static_cast<int>(converted.length))) {
       return editWidth > 0 &&
               converted.length > static_cast<std::size_t>(editWidth)
           ? EmitRepeated(io_, '*', editWidth)
@@ -380,7 +399,7 @@ bool RealOutputEditing<KIND>::EditEorDOutput(const DataEdit &edit) {
       zeroesBeforePoint = 1;
       ++totalLength;
     }
-    if (totalLength < width && noLeadingSpaces) {
+    if (totalLength < width && editWidth == 0) {
       width = totalLength;
     }
     return EmitPrefix(edit, totalLength, width) &&
@@ -418,8 +437,8 @@ bool RealOutputEditing<KIND>::EditFOutput(const DataEdit &edit) {
   bool canIncrease{true};
   while (true) {
     decimal::ConversionToDecimalResult converted{
-        Convert(extraDigits + fracDigits, rounding, flags)};
-    if (IsInfOrNaN(converted)) {
+        ConvertToDecimal(extraDigits + fracDigits, rounding, flags)};
+    if (IsInfOrNaN(converted.str, static_cast<int>(converted.length))) {
       return editWidth > 0 &&
               converted.length > static_cast<std::size_t>(editWidth)
           ? EmitRepeated(io_, '*', editWidth)
@@ -521,8 +540,8 @@ DataEdit RealOutputEditing<KIND>::EditForGOutput(DataEdit edit) {
     flags |= decimal::AlwaysSign;
   }
   decimal::ConversionToDecimalResult converted{
-      Convert(significantDigits, edit.modes.round, flags)};
-  if (IsInfOrNaN(converted)) {
+      ConvertToDecimal(significantDigits, edit.modes.round, flags)};
+  if (IsInfOrNaN(converted.str, static_cast<int>(converted.length))) {
     return edit; // Inf/Nan -> Ew.d (same as Fw.d)
   }
   int expo{IsZero() ? 1 : converted.decimalExponent}; // 's'
@@ -549,8 +568,9 @@ DataEdit RealOutputEditing<KIND>::EditForGOutput(DataEdit edit) {
 // 13.10.4 in F'2018
 template <int KIND>
 bool RealOutputEditing<KIND>::EditListDirectedOutput(const DataEdit &edit) {
-  decimal::ConversionToDecimalResult converted{Convert(1, edit.modes.round)};
-  if (IsInfOrNaN(converted)) {
+  decimal::ConversionToDecimalResult converted{
+      ConvertToDecimal(1, edit.modes.round)};
+  if (IsInfOrNaN(converted.str, static_cast<int>(converted.length))) {
     return EditEorDOutput(edit);
   }
   int expo{converted.decimalExponent};
@@ -567,11 +587,120 @@ bool RealOutputEditing<KIND>::EditListDirectedOutput(const DataEdit &edit) {
   return EditFOutput(edit);
 }
 
-// 13.7.5.2.6 in F'2018
+// 13.7.2.3.6 in F'2023
+// The specification for hexadecimal output, unfortunately for implementors,
+// leaves as "implementation dependent" the choice of how to emit values
+// with multiple hexadecimal output possibilities that are numerically
+// equivalent.  The one working implementation of EX output that I can find
+// apparently chooses to frame the nybbles from most to least significant,
+// rather than trying to minimize the magnitude of the binary exponent.
+// E.g., 2. is edited into 0X8.0P-2 rather than 0X2.0P0.  This implementation
+// follows that precedent so as to avoid a gratuitous incompatibility.
 template <int KIND>
-bool RealOutputEditing<KIND>::EditEXOutput(const DataEdit &) {
-  io_.GetIoErrorHandler().Crash(
-      "not yet implemented: EX output editing"); // TODO
+auto RealOutputEditing<KIND>::ConvertToHexadecimal(
+    int significantDigits, enum decimal::FortranRounding rounding, int flags)
+    -> ConvertToHexadecimalResult {
+  if (x_.IsNaN() || x_.IsInfinite()) {
+    auto converted{ConvertToDecimal(significantDigits, rounding, flags)};
+    return {converted.str, static_cast<int>(converted.length), 0};
+  }
+  x_.RoundToBits(4 * significantDigits, rounding);
+  if (x_.IsInfinite()) { // rounded away to +/-Inf
+    auto converted{ConvertToDecimal(significantDigits, rounding, flags)};
+    return {converted.str, static_cast<int>(converted.length), 0};
+  }
+  int len{0};
+  if (x_.IsNegative()) {
+    buffer_[len++] = '-';
+  } else if (flags & decimal::AlwaysSign) {
+    buffer_[len++] = '+';
+  }
+  auto fraction{x_.Fraction()};
+  if (fraction == 0) {
+    buffer_[len++] = '0';
+    return {buffer_, len, 0};
+  } else {
+    // Ensure that the MSB is set.
+    int expo{x_.UnbiasedExponent() - 3};
+    while (!(fraction >> (x_.binaryPrecision - 1))) {
+      fraction <<= 1;
+      --expo;
+    }
+    // This is initially the right shift count needed to bring the
+    // most-significant hexadecimal digit's bits into the LSBs.
+    // x_.binaryPrecision is constant, so / can be used for readability.
+    int shift{x_.binaryPrecision - 4};
+    typename BinaryFloatingPoint::RawType one{1};
+    auto remaining{(one << shift) - one};
+    for (int digits{0}; digits < significantDigits; ++digits) {
+      if ((flags & decimal::Minimize) && !(fraction & remaining)) {
+        break;
+      }
+      int hexDigit{0};
+      if (shift >= 0) {
+        hexDigit = int(fraction >> shift) & 0xf;
+      } else if (shift >= -3) {
+        hexDigit = int(fraction << -shift) & 0xf;
+      }
+      if (hexDigit >= 10) {
+        buffer_[len++] = 'A' + hexDigit - 10;
+      } else {
+        buffer_[len++] = '0' + hexDigit;
+      }
+      shift -= 4;
+      remaining >>= 4;
+    }
+    return {buffer_, len, expo};
+  }
+}
+
+template <int KIND>
+bool RealOutputEditing<KIND>::EditEXOutput(const DataEdit &edit) {
+  addSpaceBeforeCharacter(io_);
+  int editDigits{edit.digits.value_or(0)}; // 'd' field
+  int significantDigits{editDigits + 1};
+  int flags{0};
+  if (edit.modes.editingFlags & signPlus) {
+    flags |= decimal::AlwaysSign;
+  }
+  int editWidth{edit.width.value_or(0)}; // 'w' field
+  if (editWidth == 0 && !edit.digits) { // EX0 (no .d)
+    flags |= decimal::Minimize;
+    significantDigits = 28; // enough for 128-bit F.P.
+  }
+  auto converted{
+      ConvertToHexadecimal(significantDigits, edit.modes.round, flags)};
+  if (IsInfOrNaN(converted.str, converted.length)) {
+    return editWidth > 0 && converted.length > editWidth
+        ? EmitRepeated(io_, '*', editWidth)
+        : (editWidth <= converted.length ||
+              EmitRepeated(io_, ' ', editWidth - converted.length)) &&
+            EmitAscii(io_, converted.str, converted.length);
+  }
+  int signLength{converted.length > 0 &&
+              (converted.str[0] == '-' || converted.str[0] == '+')
+          ? 1
+          : 0};
+  int convertedDigits{converted.length - signLength};
+  int expoLength{0};
+  const char *exponent{FormatExponent(converted.exponent, edit, expoLength)};
+  int trailingZeroes{flags & decimal::Minimize
+          ? 0
+          : std::max(0, significantDigits - convertedDigits)};
+  int totalLength{converted.length + trailingZeroes + expoLength + 3 /*0X.*/};
+  int width{editWidth > 0 ? editWidth : totalLength};
+  return totalLength > width || !exponent
+      ? EmitRepeated(io_, '*', width)
+      : EmitRepeated(io_, ' ', width - totalLength) &&
+          EmitAscii(io_, converted.str, signLength) &&
+          EmitAscii(io_, "0X", 2) &&
+          EmitAscii(io_, converted.str + signLength, 1) &&
+          EmitAscii(
+              io_, edit.modes.editingFlags & decimalComma ? "," : ".", 1) &&
+          EmitAscii(io_, converted.str + signLength + 1,
+              converted.length - (signLength + 1)) &&
+          EmitRepeated(io_, '0', trailingZeroes) &&
+          EmitAscii(io_, exponent, expoLength);
 }
 
 template <int KIND> bool RealOutputEditing<KIND>::Edit(const DataEdit &edit) {
diff --git a/flang/runtime/edit-output.h b/flang/runtime/edit-output.h
index 765e41f898..4e6d6b25b4 100644
--- a/flang/runtime/edit-output.h
+++ b/flang/runtime/edit-output.h
@@ -38,20 +38,6 @@ class RealOutputEditingBase {
 protected:
   explicit RealOutputEditingBase(IoStatementState &io) : io_{io} {}
 
-  static bool IsInfOrNaN(const decimal::ConversionToDecimalResult &res) {
-    const char *p{res.str};
-    if (!p || res.length < 1) {
-      return false;
-    }
-    if (*p == '-' || *p == '+') {
-      if (res.length == 1) {
-        return false;
-      }
-      ++p;
-    }
-    return *p < '0' || *p > '9';
-  }
-
   // Returns null when the exponent overflows a fixed-size output field.
   const char *FormatExponent(int, const DataEdit &edit, int &length);
   bool EmitPrefix(const DataEdit &, std::size_t length, std::size_t width);
@@ -84,7 +70,15 @@ template <int KIND> class RealOutputEditing : public RealOutputEditingBase {
 
   bool IsZero() const { return x_.IsZero(); }
 
-  decimal::ConversionToDecimalResult Convert(
+  decimal::ConversionToDecimalResult ConvertToDecimal(
+      int significantDigits, enum decimal::FortranRounding, int flags = 0);
+
+  struct ConvertToHexadecimalResult {
+    const char *str;
+    int length;
+    int exponent;
+  };
+  ConvertToHexadecimalResult ConvertToHexadecimal(
       int significantDigits, enum decimal::FortranRounding, int flags = 0);
 
   BinaryFloatingPoint x_;
diff --git a/flang/runtime/io-stmt.h b/flang/runtime/io-stmt.h
index fa432d07a6..d4ceb83265 100644
--- a/flang/runtime/io-stmt.h
+++ b/flang/runtime/io-stmt.h
@@ -295,8 +295,7 @@ template <>
 class ListDirectedStatementState<Direction::Input>
     : public FormattedIoStatementState<Direction::Input> {
 public:
-  bool inNamelistArray() const { return inNamelistArray_; }
-  void set_inNamelistArray(bool yes = true) { inNamelistArray_ = yes; }
+  bool inNamelistSequence() const { return inNamelistSequence_; }
 
   // Skips value separators, handles repetition and null values.
   // Vacant when '/' appears; present with descriptor == ListDirectedNullValue
@@ -308,11 +307,11 @@ class ListDirectedStatementState<Direction::Input>
   // input statement.  This member function resets some state so that
   // repetition and null values work correctly for each successive
   // NAMELIST input item.
-  void ResetForNextNamelistItem(bool inNamelistArray) {
+  void ResetForNextNamelistItem(bool inNamelistSequence) {
     remaining_ = 0;
     eatComma_ = false;
     realPart_ = imaginaryPart_ = false;
-    inNamelistArray_ = inNamelistArray;
+    inNamelistSequence_ = inNamelistSequence;
   }
 
 private:
@@ -322,7 +321,7 @@ class ListDirectedStatementState<Direction::Input>
   bool hitSlash_{false}; // once '/' is seen, nullify further items
   bool realPart_{false};
   bool imaginaryPart_{false};
-  bool inNamelistArray_{false};
+  bool inNamelistSequence_{false};
 };
 
 template <Direction DIR>
diff --git a/flang/runtime/namelist.cpp b/flang/runtime/namelist.cpp
index 1b3207ef2f..61815a7cc8 100644
--- a/flang/runtime/namelist.cpp
+++ b/flang/runtime/namelist.cpp
@@ -522,15 +522,18 @@ bool IONAME(InputNamelist)(Cookie cookie, const NamelistGroup &group) {
     }
     io.HandleRelativePosition(byteCount);
     // Read the values into the descriptor.  An array can be short.
-    listInput->ResetForNextNamelistItem(useDescriptor->rank() > 0);
     if (const auto *addendum{useDescriptor->Addendum()};
         addendum && addendum->derivedType()) {
       const NonTbpDefinedIoTable *table{group.nonTbpDefinedIo};
+      listInput->ResetForNextNamelistItem(/*inNamelistSequence=*/true);
       if (!IONAME(InputDerivedType)(cookie, *useDescriptor, table)) {
         return false;
       }
-    } else if (!descr::DescriptorIO<Direction::Input>(io, *useDescriptor)) {
-      return false;
+    } else {
+      listInput->ResetForNextNamelistItem(useDescriptor->rank() > 0);
+      if (!descr::DescriptorIO<Direction::Input>(io, *useDescriptor)) {
+        return false;
+      }
     }
     next = io.GetNextNonBlank(byteCount);
     if (next && *next == comma) {
@@ -549,7 +552,7 @@ bool IONAME(InputNamelist)(Cookie cookie, const NamelistGroup &group) {
 bool IsNamelistNameOrSlash(IoStatementState &io) {
   if (auto *listInput{
           io.get_if<ListDirectedStatementState<Direction::Input>>()}) {
-    if (listInput->inNamelistArray()) {
+    if (listInput->inNamelistSequence()) {
       SavedPosition savedPosition{io};
       std::size_t byteCount{0};
       if (auto ch{io.GetNextNonBlank(byteCount)}) {
diff --git a/flang/runtime/pointer.cpp b/flang/runtime/pointer.cpp
index 0320468ffd..b0003add7b 100644
--- a/flang/runtime/pointer.cpp
+++ b/flang/runtime/pointer.cpp
@@ -56,17 +56,7 @@ void RTNAME(PointerSetDerivedLength)(
 
 void RTNAME(PointerApplyMold)(
     Descriptor &pointer, const Descriptor &mold, int rank) {
-  pointer = mold;
-  pointer.set_base_addr(nullptr);
-  pointer.raw().attribute = CFI_attribute_pointer;
-  pointer.raw().rank = rank;
-  if (auto *pointerAddendum{pointer.Addendum()}) {
-    if (const auto *moldAddendum{mold.Addendum()}) {
-      if (const auto *derived{moldAddendum->derivedType()}) {
-        pointerAddendum->set_derivedType(derived);
-      }
-    }
-  }
+  pointer.ApplyMold(mold, rank);
 }
 
 void RTNAME(PointerAssociateScalar)(Descriptor &pointer, void *target) {
@@ -183,14 +173,15 @@ int RTNAME(PointerDeallocatePolymorphic)(Descriptor &pointer,
   int stat{RTNAME(PointerDeallocate)(
       pointer, hasStat, errMsg, sourceFile, sourceLine)};
   if (stat == StatOk) {
-    DescriptorAddendum *addendum{pointer.Addendum()};
-    if (addendum) {
+    if (DescriptorAddendum * addendum{pointer.Addendum()}) {
       addendum->set_derivedType(derivedType);
+      pointer.raw().type = derivedType ? CFI_type_struct : CFI_type_other;
     } else {
       // Unlimited polymorphic descriptors initialized with
       // PointerNullifyIntrinsic do not have an addendum. Make sure the
       // derivedType is null in that case.
       INTERNAL_CHECK(!derivedType);
+      pointer.raw().type = CFI_type_other;
     }
   }
   return stat;
diff --git a/flang/runtime/random.cpp b/flang/runtime/random.cpp
index b7175d6b63..8b00cfd1ca 100644
--- a/flang/runtime/random.cpp
+++ b/flang/runtime/random.cpp
@@ -20,10 +20,10 @@
 #include <algorithm>
 #include <cmath>
 #include <cstdint>
-#include <ctime>
 #include <limits>
 #include <memory>
 #include <random>
+#include <time.h>
 
 namespace Fortran::runtime {
 
@@ -100,7 +100,13 @@ void RTNAME(RandomInit)(bool repeatable, bool /*image_distinct*/) {
     if (repeatable) {
       generator.seed(0);
     } else {
-      generator.seed(std::time(nullptr));
+#ifdef CLOCK_REALTIME
+      timespec ts;
+      clock_gettime(CLOCK_REALTIME, &ts);
+      generator.seed(ts.tv_sec & ts.tv_nsec);
+#else
+      generator.seed(time(nullptr));
+#endif
     }
   }
 }
diff --git a/flang/runtime/type-info.cpp b/flang/runtime/type-info.cpp
index baf446e0c7..b30a2c832a 100644
--- a/flang/runtime/type-info.cpp
+++ b/flang/runtime/type-info.cpp
@@ -251,10 +251,12 @@ FILE *DerivedType::Dump(FILE *f) const {
   std::fprintf(
       f, "\n  special descriptor (byteSize 0x%zx): ", special_.byteSize);
   specialDesc.Dump(f);
-  std::size_t specials{specialDesc.Elements()};
-  for (std::size_t j{0}; j < specials; ++j) {
-    std::fprintf(f, "  [%3zd] ", j);
-    specialDesc.ZeroBasedIndexedElement<SpecialBinding>(j)->Dump(f);
+  if (specialDesc.IsAllocated()) {
+    std::size_t specials{specialDesc.Elements()};
+    for (std::size_t j{0}; j < specials; ++j) {
+      std::fprintf(f, "  [%3zd] ", j);
+      specialDesc.ZeroBasedIndexedElement<SpecialBinding>(j)->Dump(f);
+    }
   }
   return f;
 }
diff --git a/flang/test/Driver/driver-help-hidden.f90 b/flang/test/Driver/driver-help-hidden.f90
index 807b0f938d..0edb84d25f 100644
--- a/flang/test/Driver/driver-help-hidden.f90
+++ b/flang/test/Driver/driver-help-hidden.f90
@@ -24,6 +24,7 @@
 ! CHECK-NEXT: -D <macro>=<value>      Define <macro> to <value> (or 1 if <value> omitted)
 ! CHECK-NEXT: -emit-llvm              Use the LLVM representation for assembler and object files
 ! CHECK-NEXT: -E                      Only run the preprocessor
+! CHECK-NEXT: -falias-analysis        Pass alias information on to LLVM (default when optimizing for speed)
 ! CHECK-NEXT: -falternative-parameter-statement
 ! CHECK-NEXT:                         Enable the old style PARAMETER statement
 ! CHECK-NEXT: -fapprox-func           Allow certain math function calls to be replaced with an approximately equivalent calculation
@@ -56,6 +57,7 @@
 ! CHECK-NEXT: -flto=jobserver         Enable LTO in 'full' mode
 ! CHECK-NEXT: -flto=<value>           Set LTO mode
 ! CHECK-NEXT: -flto                   Enable LTO in 'full' mode
+! CHECK-NEXT: -fno-alias-analysis     Do not pass alias information on to LLVM (default for unoptimized builds)
 ! CHECK-NEXT: -fno-automatic          Implies the SAVE attribute for non-automatic local objects in subprograms unless RECURSIVE
 ! CHECK-NEXT: -fno-color-diagnostics  Disable colors in diagnostics
 ! CHECK-NEXT: -fno-integrated-as      Disable the integrated assembler
diff --git a/flang/test/Driver/driver-help.f90 b/flang/test/Driver/driver-help.f90
index 4894f90f53..53b05a5ac1 100644
--- a/flang/test/Driver/driver-help.f90
+++ b/flang/test/Driver/driver-help.f90
@@ -20,6 +20,7 @@
 ! HELP-NEXT: -D <macro>=<value>      Define <macro> to <value> (or 1 if <value> omitted)
 ! HELP-NEXT: -emit-llvm              Use the LLVM representation for assembler and object files
 ! HELP-NEXT: -E                      Only run the preprocessor
+! HELP-NEXT: -falias-analysis        Pass alias information on to LLVM (default when optimizing for speed)
 ! HELP-NEXT: -falternative-parameter-statement
 ! HELP-NEXT:                         Enable the old style PARAMETER statement
 ! HELP-NEXT: -fapprox-func           Allow certain math function calls to be replaced with an approximately equivalent calculation
@@ -48,6 +49,7 @@
 ! HELP-NEXT: -flto=jobserver         Enable LTO in 'full' mode
 ! HELP-NEXT: -flto=<value>           Set LTO mode
 ! HELP-NEXT: -flto                   Enable LTO in 'full' mode
+! HELP-NEXT: -fno-alias-analysis     Do not pass alias information on to LLVM (default for unoptimized builds)
 ! HELP-NEXT: -fno-automatic          Implies the SAVE attribute for non-automatic local objects in subprograms unless RECURSIVE
 ! HELP-NEXT: -fno-color-diagnostics  Disable colors in diagnostics
 ! HELP-NEXT: -fno-integrated-as      Disable the integrated assembler
@@ -141,6 +143,7 @@
 ! HELP-FC1-NEXT: -emit-llvm              Use the LLVM representation for assembler and object files
 ! HELP-FC1-NEXT: -emit-obj               Emit native object files
 ! HELP-FC1-NEXT: -E                      Only run the preprocessor
+! HELP-FC1-NEXT: -falias-analysis        Pass alias information on to LLVM (default when optimizing for speed)
 ! HELP-FC1-NEXT: -falternative-parameter-statement
 ! HELP-FC1-NEXT:                         Enable the old style PARAMETER statement
 ! HELP-FC1-NEXT: -fapprox-func           Allow certain math function calls to be replaced with an approximately equivalent calculation
@@ -187,6 +190,7 @@
 ! HELP-FC1-NEXT: -flogical-abbreviations Enable logical abbreviations
 ! HELP-FC1-NEXT: -flto=<value>           Set LTO mode
 ! HELP-FC1-NEXT: -flto                   Enable LTO in 'full' mode
+! HELP-FC1-NEXT: -fno-alias-analysis     Do not pass alias information on to LLVM (default for unoptimized builds)
 ! HELP-FC1-NEXT: -fno-analyzed-objects-for-unparse
 ! HELP-FC1-NEXT:                         Do not use the analyzed objects when unparsing
 ! HELP-FC1-NEXT: -fno-automatic          Implies the SAVE attribute for non-automatic local objects in subprograms unless RECURSIVE
diff --git a/flang/test/Driver/falias-analysis.f90 b/flang/test/Driver/falias-analysis.f90
new file mode 100644
index 0000000000..f2c5dbde6d
--- /dev/null
+++ b/flang/test/Driver/falias-analysis.f90
@@ -0,0 +1,21 @@
+! Check that -falias-analysis and -fno-alias-analysis work as expected
+! See flang/test/Fir/tbaa-codegen.fir for a test that the output is correct
+
+! RUN: %flang -c -emit-llvm -falias-analysis %s -o - | llvm-dis | FileCheck %s --check-prefix=CHECK-AA --check-prefix=CHECK-ALL
+! RUN: %flang -c -emit-llvm -falias-analysis -fno-alias-analysis %s -o - | llvm-dis | FileCheck %s --check-prefix=CHECK-NOAA --check-prefix=CHECK-ALL
+! RUN: %flang -c -emit-llvm %s -o - | llvm-dis | FileCheck %s --check-prefix=CHECK-NOAA --check-prefix=CHECK-ALL
+
+! RUN: %flang -fc1 -emit-llvm -falias-analysis %s -o - | FileCheck %s --check-prefix=CHECK-AA --check-prefix=CHECK-ALL
+! RUN: %flang -fc1 -emit-llvm -falias-analysis -fno-alias-analysis %s -o - | FileCheck %s --check-prefix=CHECK-NOAA --check-prefix=CHECK-ALL
+! RUN: %flang -fc1 -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK-NOAA --check-prefix=CHECK-ALL
+
+subroutine simple(a)
+  integer, intent(inout) :: a(:)
+  a(1) = a(2)
+end subroutine
+! CHECK-ALL-LABEL: define void @simple
+! CHECK-ALL:       ret
+! CHECK-ALL:     }
+
+! CHECK-AA: ![[ROOT:.*]] = !{!"Flang function root _QPsimple"}
+! CHECK-NOAA-NOT: ![[ROOT:.*]] = !{!"Flang function root _QPsimple"}
diff --git a/flang/test/Driver/linker-flags.f90 b/flang/test/Driver/linker-flags.f90
index 09b8a224df..213bc032d9 100644
--- a/flang/test/Driver/linker-flags.f90
+++ b/flang/test/Driver/linker-flags.f90
@@ -2,15 +2,16 @@
 ! invocation. These libraries are added on top of other standard runtime
 ! libraries that the Clang driver will include.
 
-! RUN: %flang -### -target ppc64le-linux-gnu %S/Inputs/hello.f90 2>&1 | FileCheck %s --check-prefixes=CHECK,GNU
-! RUN: %flang -### -target aarch64-apple-darwin %S/Inputs/hello.f90 2>&1 | FileCheck %s --check-prefixes=CHECK,DARWIN
-! RUN: %flang -### -target x86_64-windows-gnu %S/Inputs/hello.f90 2>&1 | FileCheck %s --check-prefixes=CHECK,MINGW
+! RUN: %flang -### --target=ppc64le-linux-gnu %S/Inputs/hello.f90 2>&1 | FileCheck %s --check-prefixes=CHECK,UNIX
+! RUN: %flang -### --target=aarch64-apple-darwin %S/Inputs/hello.f90 2>&1 | FileCheck %s --check-prefixes=CHECK,DARWIN
+! RUN: %flang -### --target=sparc-sun-solaris2.11 %S/Inputs/hello.f90 2>&1 | FileCheck %s --check-prefixes=CHECK,UNIX
+! RUN: %flang -### --target=x86_64-windows-gnu %S/Inputs/hello.f90 2>&1 | FileCheck %s --check-prefixes=CHECK,MINGW
 
 ! NOTE: Clang's driver library, clangDriver, usually adds 'libcmt' and
 !       'oldnames' on Windows, but they are not needed when compiling
 !       Fortran code and they might bring in additional dependencies.
 !       Make sure they're not added.
-! RUN: %flang -### -target aarch64-windows-msvc -fuse-ld= %S/Inputs/hello.f90 2>&1 | FileCheck %s --check-prefixes=CHECK,MSVC --implicit-check-not libcmt --implicit-check-not oldnames
+! RUN: %flang -### --target=aarch64-windows-msvc -fuse-ld= %S/Inputs/hello.f90 2>&1 | FileCheck %s --check-prefixes=CHECK,MSVC --implicit-check-not libcmt --implicit-check-not oldnames
 
 ! Compiler invocation to generate the object file
 ! CHECK-LABEL: {{.*}} "-emit-obj"
@@ -21,12 +22,9 @@
 !       run on any other platform, such as Windows that use a .exe
 !       suffix. Clang's driver will try to resolve the path to the ld
 !       executable and may find the GNU linker from MinGW or Cygwin.
-! GNU-LABEL:  "{{.*}}ld{{(\.exe)?}}"
-! GNU-SAME: "[[object_file]]"
-! GNU-SAME: -lFortran_main
-! GNU-SAME: -lFortranRuntime
-! GNU-SAME: -lFortranDecimal
-! GNU-SAME: -lm
+! UNIX-LABEL:  "{{.*}}ld{{(\.exe)?}}"
+! UNIX-SAME: "[[object_file]]"
+! UNIX-SAME: "-lFortran_main" "-lFortranRuntime" "-lFortranDecimal" "-lm"
 
 ! DARWIN-LABEL:  "{{.*}}ld{{(\.exe)?}}"
 ! DARWIN-SAME: "[[object_file]]"
diff --git a/flang/test/Evaluate/fold-norm2.f90 b/flang/test/Evaluate/fold-norm2.f90
index 30d5289b5a..370532bafa 100644
--- a/flang/test/Evaluate/fold-norm2.f90
+++ b/flang/test/Evaluate/fold-norm2.f90
@@ -17,13 +17,20 @@ module m
   real(dp), parameter :: a(3,4) = &
     reshape([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], shape(a))
   real(dp), parameter :: nAll = norm2(a)
-  real(dp), parameter :: check_nAll = sqrt(sum(a * a))
+  real(dp), parameter :: check_nAll = 11._dp * sqrt(sum((a/11._dp)**2))
   logical, parameter :: test_all = nAll == check_nAll
   real(dp), parameter :: norms1(4) = norm2(a, dim=1)
-  real(dp), parameter :: check_norms1(4) = sqrt(sum(a * a, dim=1))
+  real(dp), parameter :: check_norms1(4) = [ &
+    2.236067977499789805051477742381393909454345703125_8, &
+    7.07106781186547550532850436866283416748046875_8, &
+    1.2206555615733702069292121450416743755340576171875e1_8, &
+    1.7378147196982769884243680280633270740509033203125e1_8 ]
   logical, parameter :: test_norms1 = all(norms1 == check_norms1)
   real(dp), parameter :: norms2(3) = norm2(a, dim=2)
-  real(dp), parameter :: check_norms2(3) = sqrt(sum(a * a, dim=2))
+  real(dp), parameter :: check_norms2(3) = [ &
+    1.1224972160321822656214862945489585399627685546875e1_8, &
+    1.28840987267251261272349438513629138469696044921875e1_8, &
+    1.4628738838327791427218471653759479522705078125e1_8 ]
   logical, parameter :: test_norms2 = all(norms2 == check_norms2)
   logical, parameter :: test_normZ = norm2([0.,0.,0.]) == 0.
 end
diff --git a/flang/test/Evaluate/folding08.f90 b/flang/test/Evaluate/folding08.f90
index 8c5296e889..1b2e5605e8 100644
--- a/flang/test/Evaluate/folding08.f90
+++ b/flang/test/Evaluate/folding08.f90
@@ -146,4 +146,10 @@ subroutine test4_bound_parentheses
     logical, parameter :: test_ubpa4_dim = ubound((pa4), 1) == 5 .and. &
          ubound((pa4), 2) == 4
   end
+  subroutine test5_max_ubound
+    ! Test maximum ubound value
+    integer(8), parameter :: I64_MAX = INT(z'7fffffffffffffff', kind=8)
+    integer, parameter :: a5(I64_MAX - 2 : I64_MAX) = [1, 2, 3]
+    logical, parameter :: test_uba5 = ubound(a5, 1, kind=8) == I64_MAX
+  end subroutine
 end
diff --git a/flang/test/Evaluate/folding09.f90 b/flang/test/Evaluate/folding09.f90
index e796ed3d51..863b5e873a 100644
--- a/flang/test/Evaluate/folding09.f90
+++ b/flang/test/Evaluate/folding09.f90
@@ -56,4 +56,13 @@ subroutine test(arr1, arr2, arr3, mat, alloc)
       end block
     end associate
   end subroutine
+  subroutine test2(x, vec)
+    type t
+      integer :: i
+    end type
+    type(t) :: x(100)
+    integer(8) :: vec(10)
+    integer(kind=merge(1,-1, .not. is_contiguous(x(1:50:2)%i)))    t01
+    integer(kind=merge(1,-1, .not. is_contiguous(x(vec)%i)))       t02
+  end subroutine
 end module
diff --git a/flang/test/Fir/invalid.fir b/flang/test/Fir/invalid.fir
index c3bfb6922d..824aeec28b 100644
--- a/flang/test/Fir/invalid.fir
+++ b/flang/test/Fir/invalid.fir
@@ -690,7 +690,7 @@ func.func @bad_array_modify(%arr1 : !fir.ref<!fir.array<?x?xf32>>, %m : index, %
 func.func @slice_must_be_integral() {
   %0 = arith.constant 42 : i32
   %1 = fir.field_index field, !fir.type<t(param:i32){field:i32}> (%0 : i32)
-  // expected-error@+1 {{'fir.slice' op operand #0 must be any integer, but got '!fir.field'}}
+  // expected-error@+1 {{'fir.slice' op operand #0 must be variadic of any integer, but got '!fir.field'}}
   %2 = fir.slice %1, %1, %1 : (!fir.field, !fir.field, !fir.field) -> !fir.slice<1>
   return
 }
diff --git a/flang/test/Fir/tbaa-codegen.fir b/flang/test/Fir/tbaa-codegen.fir
new file mode 100644
index 0000000000..fd0eb9c730
--- /dev/null
+++ b/flang/test/Fir/tbaa-codegen.fir
@@ -0,0 +1,47 @@
+// Test that tbaa attributes can be added to fir.load and fir.store
+// and that these attributes are propagated to LLVMIR
+
+// RUN: tco %s | FileCheck %s
+
+// subroutine simple(a)
+//   integer, intent(inout) :: a(:)
+//   a(1) = a(2)
+// end subroutine
+#tbaa_root = #llvm.tbaa_root<id = "Flang function root _QPsimple">
+#tbaa_type_desc = #llvm.tbaa_type_desc<id = "any access", members = {<#tbaa_root, 0>}>
+#tbaa_type_desc1 = #llvm.tbaa_type_desc<id = "any data access", members = {<#tbaa_type_desc, 0>}>
+#tbaa_type_desc2 = #llvm.tbaa_type_desc<id = "dummy arg data", members = {<#tbaa_type_desc1, 0>}>
+#tbaa_type_desc3 = #llvm.tbaa_type_desc<id = "dummy arg data/_QFfuncEa", members = {<#tbaa_type_desc2, 0>}>
+#tbaa_tag = #llvm.tbaa_tag<base_type = #tbaa_type_desc3, access_type = #tbaa_type_desc3, offset = 0>
+module attributes {fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", llvm.target_triple = "aarch64-unknown-linux-gnu"} {
+  func.func @_QPsimple(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"}) {
+    %c1 = arith.constant 1 : index
+    %c2 = arith.constant 2 : index
+    %0 = fir.declare %arg0 {fortran_attrs = #fir.var_attrs<intent_inout>, uniq_name = "_QFfuncEa"} : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+    %1 = fir.rebox %0 : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+    %2 = fir.array_coor %1 %c2 : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+    %3 = fir.load %2 {tbaa = [#tbaa_tag]} : !fir.ref<i32>
+    %4 = fir.array_coor %1 %c1 : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+    fir.store %3 to %4 {tbaa = [#tbaa_tag]} : !fir.ref<i32>
+    return
+  }
+}
+
+// CHECK-LABEL: define void @_QPsimple(
+// CHECK-SAME:      ptr %[[ARG0:.*]]) {
+// [...]
+// load  a(2):
+// CHECK:  %[[VAL20:.*]] = getelementptr i8, ptr %{{.*}}, i64 %{{.*}}
+// CHECK:  %[[A2:.*]] = load i32, ptr %[[VAL20]], align 4, !tbaa ![[A_ACCESS_TAG:.*]]
+// [...]
+// store a(2) to a(1):
+// CHECK:  %[[A1:.*]] = getelementptr i8, ptr %{{.*}}, i64 %{{.*}}
+// CHECK:  store i32 %[[A2]], ptr %[[A1]], align 4, !tbaa ![[A_ACCESS_TAG]]
+// CHECK:  ret void
+// CHECK: }
+// CHECK: ![[ANY_ACCESS_TYPE:.*]] = !{!"any access", ![[ROOT:.*]], i64 0}
+// CHECK: ![[ROOT]] = !{!"Flang function root _QPsimple"}
+// CHECK: ![[A_ACCESS_TAG]] = !{![[A_ACCESS_TYPE:.*]], ![[A_ACCESS_TYPE]], i64 0}
+// CHECK: ![[A_ACCESS_TYPE]] = !{!"dummy arg data/_QFfuncEa", ![[DUMMY_ARG_TYPE:.*]], i64 0}
+// CHECK: ![[DUMMY_ARG_TYPE]] = !{!"dummy arg data", ![[DATA_ACCESS_TYPE:.*]], i64 0}
+// CHECK: ![[DATA_ACCESS_TYPE]] = !{!"any data access", ![[ANY_ACCESS_TYPE]], i64 0}
diff --git a/flang/test/Fir/tbaa-codegen2.fir b/flang/test/Fir/tbaa-codegen2.fir
new file mode 100644
index 0000000000..f79a6108fb
--- /dev/null
+++ b/flang/test/Fir/tbaa-codegen2.fir
@@ -0,0 +1,114 @@
+// test that tbaa attributes can be added to fir.load and fir.store
+// and that these attributes are propagated to LLVMIR and that these
+// interoperrate with tbaa tags added during codegen
+
+// RUN: tco %s | FileCheck %s
+
+// subroutine func(a)
+//   integer, intent(inout) :: a(:)
+//   a = a+1
+//   a(1) = a(2)
+#tbaa_root = #llvm.tbaa_root<id = "Flang function root _QPfunc">
+#tbaa_type_desc = #llvm.tbaa_type_desc<id = "any access", members = {<#tbaa_root, 0>}>
+#tbaa_type_desc1 = #llvm.tbaa_type_desc<id = "any data access", members = {<#tbaa_type_desc, 0>}>
+#tbaa_type_desc2 = #llvm.tbaa_type_desc<id = "dummy arg data", members = {<#tbaa_type_desc1, 0>}>
+#tbaa_type_desc3 = #llvm.tbaa_type_desc<id = "dummy arg data/_QFfuncEa", members = {<#tbaa_type_desc2, 0>}>
+#tbaa_tag = #llvm.tbaa_tag<base_type = #tbaa_type_desc3, access_type = #tbaa_type_desc3, offset = 0>
+module attributes {fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", llvm.target_triple = "aarch64-unknown-linux-gnu"} {
+  func.func @_QPfunc(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"}) {
+    %c3_i32 = arith.constant 3 : i32
+    %c1_i32 = arith.constant 1 : i32
+    %c0 = arith.constant 0 : index
+    %c2 = arith.constant 2 : index
+    %c1 = arith.constant 1 : index
+    %0 = fir.alloca !fir.box<!fir.array<?xi32>>
+    %1 = fir.declare %arg0 {fortran_attrs = #fir.var_attrs<intent_inout>, uniq_name = "_QFfuncEa"} : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+    %2 = fir.rebox %1 : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+    %3:3 = fir.box_dims %2, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+    %4 = fir.shape %3#1 : (index) -> !fir.shape<1>
+    %5 = fir.allocmem !fir.array<?xi32>, %3#1 {bindc_name = ".tmp.array", uniq_name = ""}
+    %6 = fir.declare %5(%4) {uniq_name = ".tmp.array"} : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.heap<!fir.array<?xi32>>
+    %7 = fir.embox %6(%4) : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
+    fir.do_loop %arg1 = %c1 to %3#1 step %c1 unordered {
+      %16 = fir.array_coor %2 %arg1 : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+      // load with tbaa
+      %17 = fir.load %16 {tbaa = [#tbaa_tag]} : !fir.ref<i32>
+      %18 = arith.addi %17, %c1_i32 : i32
+      %19 = fir.array_coor %6(%4) %arg1 : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>, index) -> !fir.ref<i32>
+      // store without tbaa
+      fir.store %18 to %19 : !fir.ref<i32>
+    }
+    fir.store %2 to %0 : !fir.ref<!fir.box<!fir.array<?xi32>>>
+    %8 = fir.address_of(@_QQcl.2F746D702F73696D706C652E66393000) : !fir.ref<!fir.char<1,16>>
+    %9 = fir.convert %0 : (!fir.ref<!fir.box<!fir.array<?xi32>>>) -> !fir.ref<!fir.box<none>>
+    %10 = fir.convert %7 : (!fir.box<!fir.array<?xi32>>) -> !fir.box<none>
+    %11 = fir.convert %8 : (!fir.ref<!fir.char<1,16>>) -> !fir.ref<i8>
+    %12 = fir.call @_FortranAAssign(%9, %10, %11, %c3_i32) : (!fir.ref<!fir.box<none>>, !fir.box<none>, !fir.ref<i8>, i32) -> none
+    fir.freemem %6 : !fir.heap<!fir.array<?xi32>>
+    %13 = fir.array_coor %2 %c2 : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+    // load modified not to have tbaa
+    %14 = fir.load %13 : !fir.ref<i32>
+    %15 = fir.array_coor %2 %c1 : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+    // store with tbaa
+    fir.store %14 to %15 {tbaa = [#tbaa_tag]} : !fir.ref<i32>
+    return
+  }
+  func.func private @_FortranAAssign(!fir.ref<!fir.box<none>>, !fir.box<none>, !fir.ref<i8>, i32) -> none attributes {fir.runtime}
+  fir.global linkonce @_QQcl.2F746D702F73696D706C652E66393000 constant : !fir.char<1,16> {
+    %0 = fir.string_lit "/tmp/simple.f90\00"(16) : !fir.char<1,16>
+    fir.has_value %0 : !fir.char<1,16>
+  }
+}
+// CHECK-LABEL: define void @_QPfunc(
+// CHECK-SAME:      ptr %[[ARG0:.*]]) {
+// [...]
+// CHECK:  %[[VAL5:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %[[ARG0]], i32 0, i32 7, i32 0, i32 0
+// box access:
+// CHECK:  %[[VAL6:.*]] = load i64, ptr %[[VAL5]], align 4, !tbaa ![[BOX_ACCESS_TAG:.*]]
+// CHECK:  %[[VAL7:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %0, i32 0, i32 7, i32 0, i32 1
+// box access:
+// CHECK:  %[[VAL8:.*]] = load i64, ptr %[[VAL7]], align 4, !tbaa ![[BOX_ACCESS_TAG]]
+// CHECK:  %[[VAL9:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %[[ARG0]], i32 0, i32 7, i32 0, i32 2
+// box access:
+// CHECK:  %[[VAL10:.*]] = load i64, ptr %[[VAL9]], align 4, !tbaa ![[BOX_ACCESS_TAG]]
+// CHECK:  %[[VAL11:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %[[ARG0]], i32 0, i32 0
+// box access:
+// CHECK:  %[[VAL12:.*]] = load ptr, ptr %[[VAL11]], align 8, !tbaa ![[BOX_ACCESS_TAG]]
+// CHECK:  %[[VAL15:.*]] = insertvalue { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] } %14, ptr %[[VAL12]], 0
+// CHECK:  store { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] } %[[VAL15]], ptr %{{.*}}, align 8, !tbaa ![[BOX_ACCESS_TAG]]
+// CHECK:  %[[VAL16:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %{{.*}}, i32 0, i32 7, i64 0, i32 0
+// box access:
+// CHECK:  %[[VAL17:.*]] = load i64, ptr %[[VAL16]], align 4, !tbaa ![[BOX_ACCESS_TAG]]
+// CHECK:  %[[VAL18:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %{{.*}}, i32 0, i32 7, i64 0, i32 1
+// box access:
+// CHECK:  %[[VAL19:.*]] = load i64, ptr %[[VAL18]], align 4, !tbaa ![[BOX_ACCESS_TAG]]
+// CHECK:  %[[VAL20:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %{{.*}}, i32 0, i32 7, i64 0, i32 2
+// box access:
+// CHECK:  %[[VAL21:.*]] = load i64, ptr %[[VAL20]], align 4, !tbaa ![[BOX_ACCESS_TAG]]
+// [...]
+// box access:
+// CHECK:  store { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] } %{{.*}}, ptr %{{.*}}, align 8, !tbaa ![[BOX_ACCESS_TAG]]
+// [...]
+
+// [...]
+// CHECK:  %[[VAL40:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %{{.*}}, i32 0, i32 0
+// box access:
+// CHECK:  %[[VAL41:.*]] = load ptr, ptr %[[VAL40]], align 8, !tbaa ![[BOX_ACCESS_TAG]]
+// CHECK:  %[[VAL42:.*]] = getelementptr i8, ptr %[[VAL41]], i64 %{{.*}}
+// access to 'a':
+// CHECK:  %[[VAL43:.*]] = load i32, ptr %[[VAL42]], align 4, !tbaa ![[A_ACCESS_TAG:.*]]
+// [...]
+// CHECK:  %[[VAL50:.*]] = getelementptr i32, ptr %{{.*}}, i64 %{{.*}}
+// store to the temporary:
+// CHECK:  store i32 %{{.*}}, ptr %[[VAL50]], align 4, !tbaa ![[DATA_ACCESS_TAG:.*]]
+// [...]
+
+// CHECK: [[BOX_ACCESS_TAG]] = !{![[BOX_ACCESS_TYPE:.*]], ![[BOX_ACCESS_TYPE]], i64 0}
+// CHECK: ![[BOX_ACCESS_TYPE]] = !{!"descriptor member", ![[ANY_ACCESS_TYPE:.*]], i64 0}
+// CHECK: ![[ANY_ACCESS_TYPE]] = !{!"any access", ![[ROOT_TYPE:.*]], i64 0}
+// CHECK: ![[ROOT_TYPE]] = !{!"Flang function root _QPfunc"}
+// CHECK: ![[A_ACCESS_TAG]] = !{![[A_ACCESS_TYPE:.*]], ![[A_ACCESS_TYPE]], i64 0}
+// CHECK: ![[A_ACCESS_TYPE]] = !{!"dummy arg data/_QFfuncEa", ![[ARG_ACCESS_TYPE:.*]], i64 0}
+// CHECK: ![[ARG_ACCESS_TYPE]] = !{!"dummy arg data", ![[DATA_ACCESS_TYPE:.*]], i64 0}
+// CHECK: ![[DATA_ACCESS_TYPE]] = !{!"any data access", ![[ANY_ACCESS_TYPE]], i64 0}
+// CHECK: ![[DATA_ACCESS_TAG]] = !{![[DATA_ACCESS_TYPE]], ![[DATA_ACCESS_TYPE]], i64 0}
diff --git a/flang/test/Fir/tbaa.fir b/flang/test/Fir/tbaa.fir
index eabc9f3012..d260e4f4ae 100644
--- a/flang/test/Fir/tbaa.fir
+++ b/flang/test/Fir/tbaa.fir
@@ -1,5 +1,7 @@
-// RUN: fir-opt %s --split-input-file --fir-to-llvm-ir="target=x86_64-unknown-linux-gnu apply-tbaa=true" | FileCheck %s
-// RUN: fir-opt %s --split-input-file --fir-to-llvm-ir="target=aarch64-unknown-linux-gnu apply-tbaa=true" | FileCheck %s
+// test without per-function tbaa trees so that this functionality does not bitrot
+// per-function tbaa tbaa-codegen2.fir
+// RUN: fir-opt %s --split-input-file --fir-to-llvm-ir="target=x86_64-unknown-linux-gnu apply-tbaa=true" --per-function-tbaa-trees=false | FileCheck %s
+// RUN: fir-opt %s --split-input-file --fir-to-llvm-ir="target=aarch64-unknown-linux-gnu apply-tbaa=true" --per-function-tbaa-trees=false | FileCheck %s
 
 module {
   func.func @tbaa(%arg0: !fir.class<!fir.array<?xnone>> {fir.bindc_name = "a"}) {
@@ -20,7 +22,7 @@ module {
   }
 }
 
-// CHECK-DAG:     #[[ROOT:.*]] = #llvm.tbaa_root<id = "Flang Type TBAA Root">
+// CHECK-DAG:     #[[ROOT:.*]] = #llvm.tbaa_root<id = "Flang function root ">
 // CHECK-DAG:     #[[ANYACC:.*]] = #llvm.tbaa_type_desc<id = "any access", members = {<#[[ROOT]], 0>}>
 // CHECK-DAG:     #[[ANYDACC:.*]] = #llvm.tbaa_type_desc<id = "any data access", members = {<#[[ANYACC]], 0>}>
 // CHECK-DAG:     #[[BOXMEM:.*]] = #llvm.tbaa_type_desc<id = "descriptor member", members = {<#[[ANYACC]], 0>}>
@@ -119,7 +121,7 @@ module {
   }
 }
 
-// CHECK-DAG:     #[[ROOT:.*]] = #llvm.tbaa_root<id = "Flang Type TBAA Root">
+// CHECK-DAG:     #[[ROOT:.*]] = #llvm.tbaa_root<id = "Flang function root ">
 // CHECK-DAG:     #[[ANYACC:.*]] = #llvm.tbaa_type_desc<id = "any access", members = {<#[[ROOT]], 0>}>
 // CHECK-DAG:     #[[BOXMEM:.*]] = #llvm.tbaa_type_desc<id = "descriptor member", members = {<#[[ANYACC]], 0>}>
 // CHECK-DAG:     #[[$BOXT:.*]] = #llvm.tbaa_tag<base_type = #[[BOXMEM]], access_type = #[[BOXMEM]], offset = 0>
@@ -245,7 +247,7 @@ func.func @tbaa(%arg0: !fir.box<!fir.array<*:f64>>) -> i32 {
   return %0 : i32
 }
 
-// CHECK-DAG:     #[[ROOT:.*]] = #llvm.tbaa_root<id = "Flang Type TBAA Root">
+// CHECK-DAG:     #[[ROOT:.*]] = #llvm.tbaa_root<id = "Flang function root ">
 // CHECK-DAG:     #[[ANYACC:.*]] = #llvm.tbaa_type_desc<id = "any access", members = {<#[[ROOT]], 0>}>
 // CHECK-DAG:     #[[BOXMEM:.*]] = #llvm.tbaa_type_desc<id = "descriptor member", members = {<#[[ANYACC]], 0>}>
 // CHECK-DAG:     #[[$BOXT:.*]] = #llvm.tbaa_tag<base_type = #[[BOXMEM]], access_type = #[[BOXMEM]], offset = 0>
@@ -264,7 +266,7 @@ func.func @tbaa(%arg0: !fir.box<!fir.array<*:f64>>) -> i1 {
   return %0 : i1
 }
 
-// CHECK-DAG:     #[[ROOT:.*]] = #llvm.tbaa_root<id = "Flang Type TBAA Root">
+// CHECK-DAG:     #[[ROOT:.*]] = #llvm.tbaa_root<id = "Flang function root ">
 // CHECK-DAG:     #[[ANYACC:.*]] = #llvm.tbaa_type_desc<id = "any access", members = {<#[[ROOT]], 0>}>
 // CHECK-DAG:     #[[BOXMEM:.*]] = #llvm.tbaa_type_desc<id = "descriptor member", members = {<#[[ANYACC]], 0>}>
 // CHECK-DAG:     #[[$BOXT:.*]] = #llvm.tbaa_tag<base_type = #[[BOXMEM]], access_type = #[[BOXMEM]], offset = 0>
@@ -285,7 +287,7 @@ func.func @tbaa(%arg0: !fir.box<f32>) -> i32 {
   return %0 : i32
 }
 
-// CHECK-DAG:     #[[ROOT:.*]] = #llvm.tbaa_root<id = "Flang Type TBAA Root">
+// CHECK-DAG:     #[[ROOT:.*]] = #llvm.tbaa_root<id = "Flang function root ">
 // CHECK-DAG:     #[[ANYACC:.*]] = #llvm.tbaa_type_desc<id = "any access", members = {<#[[ROOT]], 0>}>
 // CHECK-DAG:     #[[BOXMEM:.*]] = #llvm.tbaa_type_desc<id = "descriptor member", members = {<#[[ANYACC]], 0>}>
 // CHECK-DAG:     #[[$BOXT:.*]] = #llvm.tbaa_tag<base_type = #[[BOXMEM]], access_type = #[[BOXMEM]], offset = 0>
@@ -304,7 +306,7 @@ func.func @tbaa(%arg0: !fir.box<!fir.array<*:f64>>) -> i1 {
   return %0 : i1
 }
 
-// CHECK-DAG:     #[[ROOT:.*]] = #llvm.tbaa_root<id = "Flang Type TBAA Root">
+// CHECK-DAG:     #[[ROOT:.*]] = #llvm.tbaa_root<id = "Flang function root ">
 // CHECK-DAG:     #[[ANYACC:.*]] = #llvm.tbaa_type_desc<id = "any access", members = {<#[[ROOT]], 0>}>
 // CHECK-DAG:     #[[BOXMEM:.*]] = #llvm.tbaa_type_desc<id = "descriptor member", members = {<#[[ANYACC]], 0>}>
 // CHECK-DAG:     #[[$BOXT:.*]] = #llvm.tbaa_tag<base_type = #[[BOXMEM]], access_type = #[[BOXMEM]], offset = 0>
@@ -328,7 +330,7 @@ func.func @tbaa(%arg0: !fir.box<!fir.array<?xi32>>) {
   return
 }
 
-// CHECK-DAG:     #[[ROOT:.*]] = #llvm.tbaa_root<id = "Flang Type TBAA Root">
+// CHECK-DAG:     #[[ROOT:.*]] = #llvm.tbaa_root<id = "Flang function root ">
 // CHECK-DAG:     #[[ANYACC:.*]] = #llvm.tbaa_type_desc<id = "any access", members = {<#[[ROOT]], 0>}>
 // CHECK-DAG:     #[[BOXMEM:.*]] = #llvm.tbaa_type_desc<id = "descriptor member", members = {<#[[ANYACC]], 0>}>
 // CHECK-DAG:     #[[$BOXT:.*]] = #llvm.tbaa_tag<base_type = #[[BOXMEM]], access_type = #[[BOXMEM]], offset = 0>
@@ -356,7 +358,7 @@ func.func @tbaa(%arg0: !fir.box<!fir.array<?xi32>>) {
 
 // Check that the scalar aggregate load/store with a descriptor member
 // is mapped to any-access.
-// CHECK-DAG:     #[[ROOT:.*]] = #llvm.tbaa_root<id = "Flang Type TBAA Root">
+// CHECK-DAG:     #[[ROOT:.*]] = #llvm.tbaa_root<id = "Flang function root ">
 // CHECK-DAG:     #[[ANYACC:.*]] = #llvm.tbaa_type_desc<id = "any access", members = {<#[[ROOT]], 0>}>
 // CHECK-DAG:     #[[$ANYT:.*]] = #llvm.tbaa_tag<base_type = #[[ANYACC]], access_type = #[[ANYACC]], offset = 0>
 
@@ -373,7 +375,7 @@ func.func @tbaa(%arg0: !fir.ref<!fir.type<_QMtypesTt{x:!fir.box<!fir.heap<f32>>}
 
 // Check that the array aggregate load/store with a descriptor member
 // is mapped to any-access.
-// CHECK-DAG:     #[[ROOT:.*]] = #llvm.tbaa_root<id = "Flang Type TBAA Root">
+// CHECK-DAG:     #[[ROOT:.*]] = #llvm.tbaa_root<id = "Flang function root ">
 // CHECK-DAG:     #[[ANYACC:.*]] = #llvm.tbaa_type_desc<id = "any access", members = {<#[[ROOT]], 0>}>
 // CHECK-DAG:     #[[$ANYT:.*]] = #llvm.tbaa_tag<base_type = #[[ANYACC]], access_type = #[[ANYACC]], offset = 0>
 
diff --git a/flang/test/HLFIR/invalid.fir b/flang/test/HLFIR/invalid.fir
index 49b6c1852b..09165f0976 100644
--- a/flang/test/HLFIR/invalid.fir
+++ b/flang/test/HLFIR/invalid.fir
@@ -267,7 +267,7 @@ func.func @bad_concat(%arg0: !fir.ref<!fir.char<1,10>>, %arg1: !fir.ref<!fir.cha
 // -----
 func.func @bad_concat_2(%arg0: !fir.ref<!fir.array<100x!fir.char<1,10>>>, %arg1: !fir.ref<!fir.array<100x!fir.char<1,20>>>) {
   %c30 = arith.constant 30 : index
-  // expected-error@+1 {{'hlfir.concat' op operand #0 must be any character scalar type, but got '!fir.ref<!fir.array<100x!fir.char<1,10>>>'}}
+  // expected-error@+1 {{'hlfir.concat' op operand #0 must be variadic of any character scalar type, but got '!fir.ref<!fir.array<100x!fir.char<1,10>>>'}}
   %0 = hlfir.concat %arg0, %arg1 len %c30 : (!fir.ref<!fir.array<100x!fir.char<1,10>>>, !fir.ref<!fir.array<100x!fir.char<1,20>>>, index) -> (!hlfir.expr<100x!fir.char<1,30>>)
   return
 }
@@ -275,7 +275,7 @@ func.func @bad_concat_2(%arg0: !fir.ref<!fir.array<100x!fir.char<1,10>>>, %arg1:
 // -----
 func.func @bad_concat_3(%arg0: !fir.ref<!fir.char<1,10>>, %arg1: !fir.ref<i32>) {
   %c30 = arith.constant 30 : index
-  // expected-error@+1 {{'hlfir.concat' op operand #1 must be any character scalar type, but got '!fir.ref<i32>'}}
+  // expected-error@+1 {{'hlfir.concat' op operand #1 must be variadic of any character scalar type, but got '!fir.ref<i32>'}}
   %0 = hlfir.concat %arg0, %arg1 len %c30 : (!fir.ref<!fir.char<1,10>>, !fir.ref<i32>, index) -> (!hlfir.expr<!fir.char<1,30>>)
   return
 }
diff --git a/flang/test/Lower/HLFIR/calls-constant-expr-arg-polymorphic.f90 b/flang/test/Lower/HLFIR/calls-constant-expr-arg-polymorphic.f90
index e1de5f7653..04d1354596 100644
--- a/flang/test/Lower/HLFIR/calls-constant-expr-arg-polymorphic.f90
+++ b/flang/test/Lower/HLFIR/calls-constant-expr-arg-polymorphic.f90
@@ -21,9 +21,9 @@ subroutine foo(x)
 end
 ! CHECK-LABEL:   func.func @_QQmain() {
 ! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %{{.*}}(%[[VAL_2:.*]]) {fortran_attrs = #fir.var_attrs<parameter>, uniq_name = "_QQro.1x_QFTt2.0"}
-! CHECK:           %[[VAL_4:.*]] = hlfir.as_expr %[[VAL_3]]#0 : (!fir.ref<!fir.array<1x!fir.type<_QFTt2{i:i32,j:i32}>>>) -> !hlfir.expr<1x!fir.type<_QFTt2{i:i32,j:i32}>>
-! CHECK:           %[[VAL_5:.*]]:3 = hlfir.associate %[[VAL_4]](%[[VAL_2]]) {uniq_name = "adapt.valuebyref"} : (!hlfir.expr<1x!fir.type<_QFTt2{i:i32,j:i32}>>, !fir.shape<1>) -> (!fir.ref<!fir.array<1x!fir.type<_QFTt2{i:i32,j:i32}>>>, !fir.ref<!fir.array<1x!fir.type<_QFTt2{i:i32,j:i32}>>>, i1)
-! CHECK:           %[[VAL_6:.*]] = fir.embox %[[VAL_5]]#0(%[[VAL_2]]) : (!fir.ref<!fir.array<1x!fir.type<_QFTt2{i:i32,j:i32}>>>, !fir.shape<1>) -> !fir.box<!fir.array<1x!fir.type<_QFTt2{i:i32,j:i32}>>>
-! CHECK:           %[[VAL_7:.*]] = fir.convert %[[VAL_6]] : (!fir.box<!fir.array<1x!fir.type<_QFTt2{i:i32,j:i32}>>>) -> !fir.class<!fir.array<?x!fir.type<_QFTt1{i:i32}>>>
+! CHECK:           %[[VAL_4:.*]] = hlfir.as_expr %[[VAL_3]]#0 : (!fir.ref<!fir.array<1x!fir.type<_QFTt2{t1:!fir.type<_QFTt1{i:i32}>,j:i32}>>>) -> !hlfir.expr<1x!fir.type<_QFTt2{t1:!fir.type<_QFTt1{i:i32}>,j:i32}>>
+! CHECK:           %[[VAL_5:.*]]:3 = hlfir.associate %[[VAL_4]](%[[VAL_2]]) {uniq_name = "adapt.valuebyref"} : (!hlfir.expr<1x!fir.type<_QFTt2{t1:!fir.type<_QFTt1{i:i32}>,j:i32}>>, !fir.shape<1>) -> (!fir.ref<!fir.array<1x!fir.type<_QFTt2{t1:!fir.type<_QFTt1{i:i32}>,j:i32}>>>, !fir.ref<!fir.array<1x!fir.type<_QFTt2{t1:!fir.type<_QFTt1{i:i32}>,j:i32}>>>, i1)
+! CHECK:           %[[VAL_6:.*]] = fir.embox %[[VAL_5]]#0(%[[VAL_2]]) : (!fir.ref<!fir.array<1x!fir.type<_QFTt2{t1:!fir.type<_QFTt1{i:i32}>,j:i32}>>>, !fir.shape<1>) -> !fir.box<!fir.array<1x!fir.type<_QFTt2{t1:!fir.type<_QFTt1{i:i32}>,j:i32}>>>
+! CHECK:           %[[VAL_7:.*]] = fir.convert %[[VAL_6]] : (!fir.box<!fir.array<1x!fir.type<_QFTt2{t1:!fir.type<_QFTt1{i:i32}>,j:i32}>>>) -> !fir.class<!fir.array<?x!fir.type<_QFTt1{i:i32}>>>
 ! CHECK:           fir.call @_QPfoo(%[[VAL_7]]) {{.*}}: (!fir.class<!fir.array<?x!fir.type<_QFTt1{i:i32}>>>) -> ()
-! CHECK:           hlfir.end_associate %[[VAL_5]]#1, %[[VAL_5]]#2 : !fir.ref<!fir.array<1x!fir.type<_QFTt2{i:i32,j:i32}>>>, i1
+! CHECK:           hlfir.end_associate %[[VAL_5]]#1, %[[VAL_5]]#2 : !fir.ref<!fir.array<1x!fir.type<_QFTt2{t1:!fir.type<_QFTt1{i:i32}>,j:i32}>>>, i1
diff --git a/flang/test/Lower/HLFIR/convert-variable-block.f90 b/flang/test/Lower/HLFIR/convert-variable-block.f90
new file mode 100644
index 0000000000..30f8eacaae
--- /dev/null
+++ b/flang/test/Lower/HLFIR/convert-variable-block.f90
@@ -0,0 +1,25 @@
+! Test that hlfir.declare is not created again for dummy arguments
+! used in specifications of BLOCK variables.
+! RUN: bbc -emit-hlfir %s -o - | FileCheck %s
+
+subroutine test(n)
+  integer(8) :: n
+  call before_block()
+  block
+    real :: x(n)
+    call foo(x)
+  end block
+end subroutine
+! CHECK-LABEL:   func.func @_QPtest(
+! CHECK-SAME:                       %[[VAL_0:.*]]: !fir.ref<i64> {fir.bindc_name = "n"}) {
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtestEn"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
+! CHECK:           fir.call @_QPbefore_block() {{.*}}: () -> ()
+! CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<i64>
+! CHECK:           %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (i64) -> index
+! CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
+! CHECK:           %[[VAL_6:.*]] = arith.cmpi sgt, %[[VAL_4]], %[[VAL_5]] : index
+! CHECK:           %[[VAL_7:.*]] = arith.select %[[VAL_6]], %[[VAL_4]], %[[VAL_5]] : index
+! CHECK:           %[[VAL_8:.*]] = fir.alloca !fir.array<?xf32>, %[[VAL_7]] {bindc_name = "x", uniq_name = "_QFtestB1Ex"}
+! CHECK:           %[[VAL_9:.*]] = fir.shape %[[VAL_7]] : (index) -> !fir.shape<1>
+! CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_8]](%[[VAL_9]]) {uniq_name = "_QFtestB1Ex"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xf32>>, !fir.ref<!fir.array<?xf32>>)
+! CHECK:           fir.call @_QPfoo(%[[VAL_10]]#1) {{.*}}: (!fir.ref<!fir.array<?xf32>>) -> ()
diff --git a/flang/test/Lower/HLFIR/intentout-allocatable-components.f90 b/flang/test/Lower/HLFIR/intentout-allocatable-components.f90
new file mode 100644
index 0000000000..932fafd322
--- /dev/null
+++ b/flang/test/Lower/HLFIR/intentout-allocatable-components.f90
@@ -0,0 +1,32 @@
+! Test that allocatable components of non pointer/non allocatable INTENT(OUT)
+! dummy arguments are deallocated.
+! RUN: bbc -emit-hlfir -polymorphic-type %s -o - -I nowhere | FileCheck %s
+
+subroutine test_intentout_component_deallocate(a)
+  type :: t
+    integer, allocatable :: x
+  end type
+  type(t), intent(out) :: a
+end subroutine
+! CHECK-LABEL:   func.func @_QPtest_intentout_component_deallocate(
+! CHECK-SAME:      %[[VAL_0:.*]]: !fir.ref<!fir.type<_QFtest_intentout_component_deallocateTt{x:!fir.box<!fir.heap<i32>>}>>
+! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<intent_out>, uniq_name = "_QFtest_intentout_component_deallocateEa"}
+! CHECK:  %[[VAL_2:.*]] = fir.embox %[[VAL_1]]#1 : (!fir.ref<!fir.type<_QFtest_intentout_component_deallocateTt{x:!fir.box<!fir.heap<i32>>}>>) -> !fir.box<!fir.type<_QFtest_intentout_component_deallocateTt{x:!fir.box<!fir.heap<i32>>}>>
+! CHECK:  %[[VAL_3:.*]] = fir.convert %[[VAL_2]] : (!fir.box<!fir.type<_QFtest_intentout_component_deallocateTt{x:!fir.box<!fir.heap<i32>>}>>) -> !fir.box<none>
+! CHECK:  %[[VAL_4:.*]] = fir.call @_FortranADestroy(%[[VAL_3]]) fastmath<contract> : (!fir.box<none>) -> none
+
+subroutine test_intentout_optional_component_deallocate(a)
+  type :: t
+    integer, allocatable :: x
+  end type
+  type(t), optional, intent(out) :: a
+end subroutine
+! CHECK-LABEL:   func.func @_QPtest_intentout_optional_component_deallocate(
+! CHECK-SAME:      %[[VAL_0:.*]]: !fir.ref<!fir.type<_QFtest_intentout_optional_component_deallocateTt{x:!fir.box<!fir.heap<i32>>}>>
+! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<intent_out, optional>, uniq_name = "_QFtest_intentout_optional_component_deallocateEa"}
+! CHECK:  %[[VAL_2:.*]] = fir.is_present %[[VAL_1]]#1 : (!fir.ref<!fir.type<_QFtest_intentout_optional_component_deallocateTt{x:!fir.box<!fir.heap<i32>>}>>) -> i1
+! CHECK:  fir.if %[[VAL_2]] {
+! CHECK:    %[[VAL_3:.*]] = fir.embox %[[VAL_1]]#1 : (!fir.ref<!fir.type<_QFtest_intentout_optional_component_deallocateTt{x:!fir.box<!fir.heap<i32>>}>>) -> !fir.box<!fir.type<_QFtest_intentout_optional_component_deallocateTt{x:!fir.box<!fir.heap<i32>>}>>
+! CHECK:    %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (!fir.box<!fir.type<_QFtest_intentout_optional_component_deallocateTt{x:!fir.box<!fir.heap<i32>>}>>) -> !fir.box<none>
+! CHECK:    %[[VAL_5:.*]] = fir.call @_FortranADestroy(%[[VAL_4]]) fastmath<contract> : (!fir.box<none>) -> none
+! CHECK:  }
diff --git a/flang/test/Lower/HLFIR/local-end-of-scope-component-dealloc.f90 b/flang/test/Lower/HLFIR/local-end-of-scope-component-dealloc.f90
index b63026b1e3..06884138d2 100644
--- a/flang/test/Lower/HLFIR/local-end-of-scope-component-dealloc.f90
+++ b/flang/test/Lower/HLFIR/local-end-of-scope-component-dealloc.f90
@@ -62,7 +62,7 @@ subroutine test3()
 end subroutine test3
 ! CHECK-LABEL:   func.func @_QPtest3() {
 ! CHECK-DAG:       %[[VAL_10:.*]] = fir.call @_FortranADestroy(%[[VAL_9:.*]]) fastmath<contract> : (!fir.box<none>) -> none
-! CHECK-DAG:       %[[VAL_9]] = fir.convert %{{.*}} : (!fir.box<!fir.type<_QMtypesTt3{x:!fir.box<!fir.heap<f32>>}>>) -> !fir.box<none>
+! CHECK-DAG:       %[[VAL_9]] = fir.convert %{{.*}} : (!fir.box<!fir.type<_QMtypesTt3{t1:!fir.type<_QMtypesTt1{x:!fir.box<!fir.heap<f32>>}>}>>) -> !fir.box<none>
 
 subroutine test3b()
   use types
@@ -72,7 +72,7 @@ subroutine test3b()
 end subroutine test3b
 ! CHECK-LABEL:   func.func @_QPtest3b() {
 ! CHECK-DAG:       %[[VAL_11:.*]] = fir.call @_FortranADestroy(%[[VAL_10:.*]]) fastmath<contract> : (!fir.box<none>) -> none
-! CHECK-DAG:       %[[VAL_10]] = fir.convert %{{.*}} : (!fir.box<!fir.type<_QMtypesTt3{x:!fir.box<!fir.heap<f32>>}>>) -> !fir.box<none>
+! CHECK-DAG:       %[[VAL_10]] = fir.convert %{{.*}} : (!fir.box<!fir.type<_QMtypesTt3{t1:!fir.type<_QMtypesTt1{x:!fir.box<!fir.heap<f32>>}>}>>) -> !fir.box<none>
 
 subroutine test4()
   use types
@@ -80,7 +80,7 @@ subroutine test4()
 end subroutine test4
 ! CHECK-LABEL:   func.func @_QPtest4() {
 ! CHECK-DAG:       %[[VAL_10:.*]] = fir.call @_FortranADestroy(%[[VAL_9:.*]]) fastmath<contract> : (!fir.box<none>) -> none
-! CHECK-DAG:       %[[VAL_9]] = fir.convert %{{.*}} : (!fir.box<!fir.type<_QMtypesTt4{x:!fir.box<!fir.heap<f32>>}>>) -> !fir.box<none>
+! CHECK-DAG:       %[[VAL_9]] = fir.convert %{{.*}} : (!fir.box<!fir.type<_QMtypesTt4{t3:!fir.type<_QMtypesTt3{t1:!fir.type<_QMtypesTt1{x:!fir.box<!fir.heap<f32>>}>}>}>>) -> !fir.box<none>
 
 subroutine test4b()
   use types
@@ -90,7 +90,7 @@ subroutine test4b()
 end subroutine test4b
 ! CHECK-LABEL:   func.func @_QPtest4b() {
 ! CHECK-DAG:       %[[VAL_11:.*]] = fir.call @_FortranADestroy(%[[VAL_10:.*]]) fastmath<contract> : (!fir.box<none>) -> none
-! CHECK-DAG:       %[[VAL_10]] = fir.convert %{{.*}} : (!fir.box<!fir.type<_QMtypesTt4{x:!fir.box<!fir.heap<f32>>}>>) -> !fir.box<none>
+! CHECK-DAG:       %[[VAL_10]] = fir.convert %{{.*}} : (!fir.box<!fir.type<_QMtypesTt4{t3:!fir.type<_QMtypesTt3{t1:!fir.type<_QMtypesTt1{x:!fir.box<!fir.heap<f32>>}>}>}>>) -> !fir.box<none>
 
 subroutine test5()
   use types
@@ -98,7 +98,7 @@ subroutine test5()
 end subroutine test5
 ! CHECK-LABEL:   func.func @_QPtest5() {
 ! CHECK-DAG:       %[[VAL_10:.*]] = fir.call @_FortranADestroy(%[[VAL_9:.*]]) fastmath<contract> : (!fir.box<none>) -> none
-! CHECK-DAG:       %[[VAL_9]] = fir.convert %{{.*}} : (!fir.box<!fir.type<_QMtypesTt5{x:!fir.type<_QMtypesTt1{x:!fir.box<!fir.heap<f32>>}>}>>) -> !fir.box<none>
+! CHECK-DAG:       %[[VAL_9]] = fir.convert %{{.*}} : (!fir.box<!fir.type<_QMtypesTt5{t2:!fir.type<_QMtypesTt2{x:!fir.type<_QMtypesTt1{x:!fir.box<!fir.heap<f32>>}>}>}>>) -> !fir.box<none>
 
 subroutine test5b()
   use types
@@ -108,4 +108,4 @@ subroutine test5b()
 end subroutine test5b
 ! CHECK-LABEL:   func.func @_QPtest5b() {
 ! CHECK-DAG:       %[[VAL_11:.*]] = fir.call @_FortranADestroy(%[[VAL_10:.*]]) fastmath<contract> : (!fir.box<none>) -> none
-! CHECK-DAG:       %[[VAL_10]] = fir.convert %{{.*}} : (!fir.box<!fir.type<_QMtypesTt5{x:!fir.type<_QMtypesTt1{x:!fir.box<!fir.heap<f32>>}>}>>) -> !fir.box<none>
+! CHECK-DAG:       %[[VAL_10]] = fir.convert %{{.*}} : (!fir.box<!fir.type<_QMtypesTt5{t2:!fir.type<_QMtypesTt2{x:!fir.type<_QMtypesTt1{x:!fir.box<!fir.heap<f32>>}>}>}>>) -> !fir.box<none>
diff --git a/flang/test/Lower/HLFIR/parent-component-ref.f90 b/flang/test/Lower/HLFIR/parent-component-ref.f90
index ceba56e730..b08d8f450e 100644
--- a/flang/test/Lower/HLFIR/parent-component-ref.f90
+++ b/flang/test/Lower/HLFIR/parent-component-ref.f90
@@ -23,28 +23,30 @@ subroutine takes_int_array(x)
 end interface
 end module
 
-subroutine test_ignored_inner_parent_comp(x)
+subroutine test_inner_parent_comp(x)
  use pc_types
  type(t2) :: x
  call takes_int(x%t%i)
 end subroutine
-! CHECK-LABEL: func.func @_QPtest_ignored_inner_parent_comp(
+! CHECK-LABEL: func.func @_QPtest_inner_parent_comp(
 ! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare {{.*}}Ex"
-! CHECK:  %[[VAL_2:.*]] = hlfir.designate %[[VAL_1]]#0{"i"}   : (!fir.ref<!fir.type<_QMpc_typesTt2{i:i32,j:i32}>>) -> !fir.ref<i32>
-! CHECK:  fir.call @_QPtakes_int(%[[VAL_2]])
+! CHECK:  %[[VAL_2:.*]] = hlfir.designate %[[VAL_1]]#0{"t"}   : (!fir.ref<!fir.type<_QMpc_typesTt2{t:!fir.type<_QMpc_typesTt{i:i32}>,j:i32}>>) -> !fir.ref<!fir.type<_QMpc_typesTt{i:i32}>>
+! CHECK:  %[[VAL_3:.*]] = hlfir.designate %[[VAL_2]]{"i"}   : (!fir.ref<!fir.type<_QMpc_typesTt{i:i32}>>) -> !fir.ref<i32>
+! CHECK:  fir.call @_QPtakes_int(%[[VAL_3]])
 
-subroutine test_ignored_inner_parent_comp_2(x)
+subroutine test_inner_parent_comp_2(x)
  use pc_types
  type(t2) :: x(:)
  call takes_int_array(x%t%i)
 end subroutine
-! CHECK-LABEL: func.func @_QPtest_ignored_inner_parent_comp_2(
+! CHECK-LABEL: func.func @_QPtest_inner_parent_comp_2(
 ! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare {{.*}}Ex"
 ! CHECK:  %[[VAL_2:.*]] = arith.constant 0 : index
-! CHECK:  %[[VAL_3:.*]]:3 = fir.box_dims %[[VAL_1]]#0, %[[VAL_2]] : (!fir.box<!fir.array<?x!fir.type<_QMpc_typesTt2{i:i32,j:i32}>>>, index) -> (index, index, index)
+! CHECK:  %[[VAL_3:.*]]:3 = fir.box_dims %[[VAL_1]]#0, %[[VAL_2]] : (!fir.box<!fir.array<?x!fir.type<_QMpc_typesTt2{t:!fir.type<_QMpc_typesTt{i:i32}>,j:i32}>>>, index) -> (index, index, index)
 ! CHECK:  %[[VAL_4:.*]] = fir.shape %[[VAL_3]]#1 : (index) -> !fir.shape<1>
-! CHECK:  %[[VAL_5:.*]] = hlfir.designate %[[VAL_1]]#0{"i"}   shape %[[VAL_4]] : (!fir.box<!fir.array<?x!fir.type<_QMpc_typesTt2{i:i32,j:i32}>>>, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
-! CHECK:  fir.call @_QPtakes_int_array(%[[VAL_5]])
+! CHECK:  %[[VAL_5:.*]] = hlfir.designate %[[VAL_1]]#0{"t"}   shape %[[VAL_4]] : (!fir.box<!fir.array<?x!fir.type<_QMpc_typesTt2{t:!fir.type<_QMpc_typesTt{i:i32}>,j:i32}>>>, !fir.shape<1>) -> !fir.box<!fir.array<?x!fir.type<_QMpc_typesTt{i:i32}>>>
+! CHECK:  %[[VAL_6:.*]] = hlfir.designate %[[VAL_5]]{"i"}   shape %[[VAL_4]] : (!fir.box<!fir.array<?x!fir.type<_QMpc_typesTt{i:i32}>>>, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
+! CHECK:  fir.call @_QPtakes_int_array(%[[VAL_6]])
 
 subroutine test_leaf_parent_ref(x)
  use pc_types
@@ -53,7 +55,7 @@ subroutine test_leaf_parent_ref(x)
 end subroutine
 ! CHECK-LABEL: func.func @_QPtest_leaf_parent_ref(
 ! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare {{.*}}Ex"
-! CHECK:  %[[VAL_2:.*]] = hlfir.parent_comp %[[VAL_1]]#0 : (!fir.ref<!fir.type<_QMpc_typesTt2{i:i32,j:i32}>>) -> !fir.ref<!fir.type<_QMpc_typesTt{i:i32}>>
+! CHECK:  %[[VAL_2:.*]] = hlfir.designate %[[VAL_1]]#0{"t"}   : (!fir.ref<!fir.type<_QMpc_typesTt2{t:!fir.type<_QMpc_typesTt{i:i32}>,j:i32}>>) -> !fir.ref<!fir.type<_QMpc_typesTt{i:i32}>>
 ! CHECK:  fir.call @_QPtakes_parent(%[[VAL_2]])
 
 subroutine test_leaf_parent_ref_array(x)
@@ -63,12 +65,12 @@ subroutine test_leaf_parent_ref_array(x)
 ! CHECK:  %[[VAL_4:.*]]:2 = hlfir.declare {{.*}}Ex"
  call takes_t_type_array(x%t)
 ! CHECK:  %[[VAL_5:.*]] = arith.constant 0 : index
-! CHECK:  %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_4]]#0, %[[VAL_5]] : (!fir.class<!fir.array<?x!fir.type<_QMpc_typesTt2{i:i32,j:i32}>>>, index) -> (index, index, index)
+! CHECK:  %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_4]]#0, %[[VAL_5]] : (!fir.class<!fir.array<?x!fir.type<_QMpc_typesTt2{t:!fir.type<_QMpc_typesTt{i:i32}>,j:i32}>>>, index) -> (index, index, index)
 ! CHECK:  %[[VAL_7:.*]] = fir.shape %[[VAL_6]]#1 : (index) -> !fir.shape<1>
-! CHECK:  %[[VAL_8:.*]] = hlfir.parent_comp %[[VAL_4]]#0 shape %[[VAL_7]] : (!fir.class<!fir.array<?x!fir.type<_QMpc_typesTt2{i:i32,j:i32}>>>, !fir.shape<1>) -> !fir.box<!fir.array<?x!fir.type<_QMpc_typesTt{i:i32}>>>
+! CHECK:  %[[VAL_8:.*]] = hlfir.designate %[[VAL_4]]#0{"t"}   shape %[[VAL_7]] : (!fir.class<!fir.array<?x!fir.type<_QMpc_typesTt2{t:!fir.type<_QMpc_typesTt{i:i32}>,j:i32}>>>, !fir.shape<1>) -> !fir.box<!fir.array<?x!fir.type<_QMpc_typesTt{i:i32}>>>
 ! CHECK:  fir.call @_QPtakes_t_type_array(%[[VAL_8]])
  call takes_t_class_array(x%t)
-! CHECK:  %[[VAL_12:.*]] = hlfir.parent_comp %[[VAL_4]]#0 shape %{{.*}} : (!fir.class<!fir.array<?x!fir.type<_QMpc_typesTt2{i:i32,j:i32}>>>, !fir.shape<1>) -> !fir.box<!fir.array<?x!fir.type<_QMpc_typesTt{i:i32}>>>
+! CHECK:  %[[VAL_12:.*]] = hlfir.designate %[[VAL_4]]#0{"t"}   shape %{{.*}} : (!fir.class<!fir.array<?x!fir.type<_QMpc_typesTt2{t:!fir.type<_QMpc_typesTt{i:i32}>,j:i32}>>>, !fir.shape<1>) -> !fir.box<!fir.array<?x!fir.type<_QMpc_typesTt{i:i32}>>>
 ! CHECK:  %[[VAL_13:.*]] = fir.convert %[[VAL_12]] : (!fir.box<!fir.array<?x!fir.type<_QMpc_typesTt{i:i32}>>>) -> !fir.class<!fir.array<?x!fir.type<_QMpc_typesTt{i:i32}>>>
 ! CHECK:  fir.call @_QPtakes_t_class_array(%[[VAL_13]])
 end subroutine
@@ -80,8 +82,8 @@ subroutine test_parent_section_leaf_array(x)
 end subroutine
 ! CHECK-LABEL: func.func @_QPtest_parent_section_leaf_array(
 ! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare {{.*}}Ex"
-! CHECK:  %[[VAL_7:.*]] = hlfir.designate %[[VAL_1]]#0 ({{.*}})  shape %[[VAL_6:.*]] : (!fir.class<!fir.array<?x!fir.type<_QMpc_typesTt2{i:i32,j:i32}>>>, index, index, index, !fir.shape<1>) -> !fir.class<!fir.array<10x!fir.type<_QMpc_typesTt2{i:i32,j:i32}>>>
-! CHECK:  %[[VAL_8:.*]] = hlfir.parent_comp %[[VAL_7]] shape %[[VAL_6]] : (!fir.class<!fir.array<10x!fir.type<_QMpc_typesTt2{i:i32,j:i32}>>>, !fir.shape<1>) -> !fir.box<!fir.array<10x!fir.type<_QMpc_typesTt{i:i32}>>>
+! CHECK:  %[[VAL_7:.*]] = hlfir.designate %[[VAL_1]]#0 ({{.*}})  shape %[[VAL_6:.*]] : (!fir.class<!fir.array<?x!fir.type<_QMpc_typesTt2{t:!fir.type<_QMpc_typesTt{i:i32}>,j:i32}>>>, index, index, index, !fir.shape<1>) -> !fir.class<!fir.array<10x!fir.type<_QMpc_typesTt2{t:!fir.type<_QMpc_typesTt{i:i32}>,j:i32}>>>
+! CHECK:  %[[VAL_8:.*]] = hlfir.designate %[[VAL_7]]{"t"}   shape %[[VAL_6]] : (!fir.class<!fir.array<10x!fir.type<_QMpc_typesTt2{t:!fir.type<_QMpc_typesTt{i:i32}>,j:i32}>>>, !fir.shape<1>) -> !fir.box<!fir.array<10x!fir.type<_QMpc_typesTt{i:i32}>>>
 ! CHECK:  %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (!fir.box<!fir.array<10x!fir.type<_QMpc_typesTt{i:i32}>>>) -> !fir.box<!fir.array<?x!fir.type<_QMpc_typesTt{i:i32}>>>
 ! CHECK:  fir.call @_QPtakes_t_type_array(%[[VAL_9]])
 
@@ -92,8 +94,9 @@ subroutine test_pointer_leaf_parent_ref_array(x)
 end subroutine
 ! CHECK-LABEL: func.func @_QPtest_pointer_leaf_parent_ref_array(
 ! CHECK:  %[[VAL_1:.*]]:2 = hlfir.declare {{.*}}Ex"
-! CHECK:  %[[VAL_2:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.class<!fir.ptr<!fir.array<?x!fir.type<_QMpc_typesTt2{i:i32,j:i32}>>>>>
+! CHECK:  %[[VAL_2:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.class<!fir.ptr<!fir.array<?x!fir.type<_QMpc_typesTt2{t:!fir.type<_QMpc_typesTt{i:i32}>,j:i32}>>>>>
 ! CHECK:  %[[VAL_3:.*]] = arith.constant 0 : index
-! CHECK:  %[[VAL_4:.*]]:3 = fir.box_dims %[[VAL_2]], %[[VAL_3]] : (!fir.class<!fir.ptr<!fir.array<?x!fir.type<_QMpc_typesTt2{i:i32,j:i32}>>>>, index) -> (index, index, index)
+! CHECK:  %[[VAL_4:.*]]:3 = fir.box_dims %[[VAL_2]], %[[VAL_3]] : (!fir.class<!fir.ptr<!fir.array<?x!fir.type<_QMpc_typesTt2{t:!fir.type<_QMpc_typesTt{i:i32}>,j:i32}>>>>, index) -> (index, index, index)
 ! CHECK:  %[[VAL_5:.*]] = fir.shape %[[VAL_4]]#1 : (index) -> !fir.shape<1>
-! CHECK:  %[[VAL_6:.*]] = hlfir.parent_comp %[[VAL_2]] shape %[[VAL_5]] : (!fir.class<!fir.ptr<!fir.array<?x!fir.type<_QMpc_typesTt2{i:i32,j:i32}>>>>, !fir.shape<1>) -> !fir.box<!fir.array<?x!fir.type<_QMpc_typesTt{i:i32}>>>
+! CHECK:  %[[VAL_6:.*]] = hlfir.designate %[[VAL_2]]{"t"}   shape %[[VAL_5]] : (!fir.class<!fir.ptr<!fir.array<?x!fir.type<_QMpc_typesTt2{t:!fir.type<_QMpc_typesTt{i:i32}>,j:i32}>>>>, !fir.shape<1>) -> !fir.box<!fir.array<?x!fir.type<_QMpc_typesTt{i:i32}>>>
+! CHECK:  fir.call @_QPtakes_t_type_array(%[[VAL_6]])
diff --git a/flang/test/Lower/HLFIR/private-components.f90 b/flang/test/Lower/HLFIR/private-components.f90
index bf231c52f5..23c1c7402c 100644
--- a/flang/test/Lower/HLFIR/private-components.f90
+++ b/flang/test/Lower/HLFIR/private-components.f90
@@ -11,7 +11,7 @@ module name_clash
 end module
 
 !CHECK-LABEL: func.func @_QPuser_clash(
-!CHECK-SAME: !fir.ref<!fir.type<_QFuser_clashTt2{_QMname_clashTt.i:i32,i:i32}>>
+!CHECK-SAME: !fir.ref<!fir.type<_QFuser_clashTt2{t:!fir.type<_QMname_clashTt{_QMname_clashTt.i:i32}>,i:i32}>>
 !CHECK-SAME: !fir.ref<!fir.type<_QMname_clashTt{_QMname_clashTt.i:i32}>>
 subroutine user_clash(a, at)
   use name_clash
@@ -26,7 +26,7 @@ subroutine user_clash(a, at)
 end subroutine
 
 ! CHECK-LABEL: func.func @_QPclash_with_intrinsic_module(
-! CHECK-SAME: !fir.ref<!fir.type<_QFclash_with_intrinsic_moduleTmy_class{_QMieee_arithmeticTieee_class_type.which:i8,which:i8}>>
+! CHECK-SAME: !fir.ref<!fir.type<_QFclash_with_intrinsic_moduleTmy_class{ieee_class_type:!fir.type<_QMieee_arithmeticTieee_class_type{_QMieee_arithmeticTieee_class_type.which:i8}>,which:i8}>>
 subroutine clash_with_intrinsic_module(a)
  use ieee_arithmetic
  type, extends(ieee_class_type) :: my_class
diff --git a/flang/test/Lower/HLFIR/structure-constructor.f90 b/flang/test/Lower/HLFIR/structure-constructor.f90
index 797f801ad6..8bf140abaa 100644
--- a/flang/test/Lower/HLFIR/structure-constructor.f90
+++ b/flang/test/Lower/HLFIR/structure-constructor.f90
@@ -41,9 +41,9 @@ end subroutine test1
 ! CHECK:           %[[VAL_2:.*]] = fir.alloca !fir.type<_QMtypesTt1{c:!fir.char<1,4>}> {bindc_name = "res", uniq_name = "_QFtest1Eres"}
 ! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFtest1Eres"} : (!fir.ref<!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>) -> (!fir.ref<!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>, !fir.ref<!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>)
 ! CHECK:           %[[VAL_4:.*]]:2 = fir.unboxchar %[[VAL_0]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK:           %[[VAL_6:.*]] = fir.convert %[[VAL_4]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,4>>
-! CHECK:           %[[VAL_5:.*]] = arith.constant 4 : index
-! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] typeparams %[[VAL_5]] {uniq_name = "_QFtest1Ex"} : (!fir.ref<!fir.char<1,4>>, index) -> (!fir.ref<!fir.char<1,4>>, !fir.ref<!fir.char<1,4>>)
+! CHECK:           %[[VAL_5:.*]] = fir.convert %[[VAL_4]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,4>>
+! CHECK:           %[[VAL_6:.*]] = arith.constant 4 : index
+! CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_5]] typeparams %[[VAL_6]] {uniq_name = "_QFtest1Ex"} : (!fir.ref<!fir.char<1,4>>, index) -> (!fir.ref<!fir.char<1,4>>, !fir.ref<!fir.char<1,4>>)
 ! CHECK:           %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "ctor.temp"} : (!fir.ref<!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>) -> (!fir.ref<!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>, !fir.ref<!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>)
 ! CHECK:           %[[VAL_9:.*]] = fir.embox %[[VAL_8]]#0 : (!fir.ref<!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>) -> !fir.box<!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>
 ! CHECK:           %[[VAL_10:.*]] = fir.address_of(@_QQcl.{{.*}}) : !fir.ref<!fir.char<1,{{[0-9]*}}>>
@@ -218,28 +218,28 @@ end subroutine test6
 ! CHECK:           %[[VAL_3:.*]] = fir.alloca !fir.array<10xi64> {bindc_name = ".rt.arrayctor.vector"}
 ! CHECK:           %[[VAL_4:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>>> {bindc_name = ".tmp.arrayctor"}
 ! CHECK:           %[[VAL_5:.*]] = fir.alloca !fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>
-! CHECK:           %[[VAL_6:.*]] = fir.alloca !fir.type<_QMtypesTt6{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>
+! CHECK:           %[[VAL_6:.*]] = fir.alloca !fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>
 ! CHECK:           %[[VAL_7:.*]]:2 = fir.unboxchar %[[VAL_1]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_7]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,4>>
-! CHECK:           %[[VAL_8:.*]] = arith.constant 4 : index
-! CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_9]] typeparams %[[VAL_8]] {uniq_name = "_QFtest6Ec"} : (!fir.ref<!fir.char<1,4>>, index) -> (!fir.ref<!fir.char<1,4>>, !fir.ref<!fir.char<1,4>>)
-! CHECK:           %[[VAL_11:.*]] = fir.alloca !fir.type<_QMtypesTt6{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}> {bindc_name = "res", uniq_name = "_QFtest6Eres"}
-! CHECK:           %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_11]] {uniq_name = "_QFtest6Eres"} : (!fir.ref<!fir.type<_QMtypesTt6{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>) -> (!fir.ref<!fir.type<_QMtypesTt6{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>, !fir.ref<!fir.type<_QMtypesTt6{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>)
-! CHECK:           %[[VAL_13:.*]] = fir.embox %[[VAL_12]]#1 : (!fir.ref<!fir.type<_QMtypesTt6{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>) -> !fir.box<!fir.type<_QMtypesTt6{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>
+! CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_7]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,4>>
+! CHECK:           %[[VAL_9:.*]] = arith.constant 4 : index
+! CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_8]] typeparams %[[VAL_9]] {uniq_name = "_QFtest6Ec"} : (!fir.ref<!fir.char<1,4>>, index) -> (!fir.ref<!fir.char<1,4>>, !fir.ref<!fir.char<1,4>>)
+! CHECK:           %[[VAL_11:.*]] = fir.alloca !fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}> {bindc_name = "res", uniq_name = "_QFtest6Eres"}
+! CHECK:           %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_11]] {uniq_name = "_QFtest6Eres"} : (!fir.ref<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>) -> (!fir.ref<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>, !fir.ref<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>)
+! CHECK:           %[[VAL_13:.*]] = fir.embox %[[VAL_12]]#1 : (!fir.ref<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>) -> !fir.box<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>
 ! CHECK:           %[[VAL_14:.*]] = fir.address_of(@_QQcl.{{.*}}) : !fir.ref<!fir.char<1,{{[0-9]*}}>>
 ! CHECK:           %[[VAL_15:.*]] = arith.constant {{[0-9]*}} : i32
-! CHECK:           %[[VAL_16:.*]] = fir.convert %[[VAL_13]] : (!fir.box<!fir.type<_QMtypesTt6{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>) -> !fir.box<none>
+! CHECK:           %[[VAL_16:.*]] = fir.convert %[[VAL_13]] : (!fir.box<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>) -> !fir.box<none>
 ! CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_14]] : (!fir.ref<!fir.char<1,{{[0-9]*}}>>) -> !fir.ref<i8>
 ! CHECK:           %[[VAL_18:.*]] = fir.call @_FortranAInitialize(%[[VAL_16]], %[[VAL_17]], %[[VAL_15]]) fastmath<contract> : (!fir.box<none>, !fir.ref<i8>, i32) -> none
 ! CHECK:           %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest6Ex"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>>)
-! CHECK:           %[[VAL_20:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "ctor.temp"} : (!fir.ref<!fir.type<_QMtypesTt6{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>) -> (!fir.ref<!fir.type<_QMtypesTt6{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>, !fir.ref<!fir.type<_QMtypesTt6{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>)
-! CHECK:           %[[VAL_21:.*]] = fir.embox %[[VAL_20]]#0 : (!fir.ref<!fir.type<_QMtypesTt6{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>) -> !fir.box<!fir.type<_QMtypesTt6{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>
+! CHECK:           %[[VAL_20:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "ctor.temp"} : (!fir.ref<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>) -> (!fir.ref<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>, !fir.ref<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>)
+! CHECK:           %[[VAL_21:.*]] = fir.embox %[[VAL_20]]#0 : (!fir.ref<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>) -> !fir.box<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>
 ! CHECK:           %[[VAL_22:.*]] = fir.address_of(@_QQcl.{{.*}}) : !fir.ref<!fir.char<1,{{[0-9]*}}>>
 ! CHECK:           %[[VAL_23:.*]] = arith.constant {{[0-9]*}} : i32
-! CHECK:           %[[VAL_24:.*]] = fir.convert %[[VAL_21]] : (!fir.box<!fir.type<_QMtypesTt6{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>) -> !fir.box<none>
+! CHECK:           %[[VAL_24:.*]] = fir.convert %[[VAL_21]] : (!fir.box<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>) -> !fir.box<none>
 ! CHECK:           %[[VAL_25:.*]] = fir.convert %[[VAL_22]] : (!fir.ref<!fir.char<1,{{[0-9]*}}>>) -> !fir.ref<i8>
 ! CHECK:           %[[VAL_26:.*]] = fir.call @_FortranAInitialize(%[[VAL_24]], %[[VAL_25]], %[[VAL_23]]) fastmath<contract> : (!fir.box<none>, !fir.ref<i8>, i32) -> none
-! CHECK:           %[[VAL_27:.*]] = hlfir.parent_comp %[[VAL_20]]#0 : (!fir.ref<!fir.type<_QMtypesTt6{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>) -> !fir.ref<!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>>
+! CHECK:           %[[VAL_27:.*]] = hlfir.designate %[[VAL_20]]#0{"t5"}   : (!fir.ref<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>) -> !fir.ref<!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>>
 ! CHECK:           %[[VAL_28:.*]]:2 = hlfir.declare %[[VAL_5]] {uniq_name = "ctor.temp"} : (!fir.ref<!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>>) -> (!fir.ref<!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>>, !fir.ref<!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>>)
 ! CHECK:           %[[VAL_29:.*]] = fir.embox %[[VAL_28]]#0 : (!fir.ref<!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>>) -> !fir.box<!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>>
 ! CHECK:           %[[VAL_30:.*]] = fir.address_of(@_QQcl.{{.*}}) : !fir.ref<!fir.char<1,{{[0-9]*}}>>
@@ -260,7 +260,7 @@ end subroutine test6
 ! CHECK:           hlfir.assign %[[VAL_28]]#0 to %[[VAL_27]] temporary_lhs : !fir.ref<!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>>, !fir.ref<!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>>
 ! CHECK:           %[[VAL_42:.*]] = arith.constant 1 : index
 ! CHECK:           %[[VAL_43:.*]] = fir.shape %[[VAL_42]] : (index) -> !fir.shape<1>
-! CHECK:           %[[VAL_44:.*]] = hlfir.designate %[[VAL_20]]#0{"t6m"} <%[[VAL_43]]>   shape %[[VAL_43]] : (!fir.ref<!fir.type<_QMtypesTt6{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>, !fir.shape<1>, !fir.shape<1>) -> !fir.ref<!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>>
+! CHECK:           %[[VAL_44:.*]] = hlfir.designate %[[VAL_20]]#0{"t6m"} <%[[VAL_43]]>   shape %[[VAL_43]] : (!fir.ref<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>, !fir.shape<1>, !fir.shape<1>) -> !fir.ref<!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>>
 ! CHECK:           %[[VAL_45:.*]] = arith.constant 1 : index
 ! CHECK:           %[[VAL_46:.*]] = fir.allocmem !fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>> {bindc_name = ".tmp.arrayctor", uniq_name = ""}
 ! CHECK:           %[[VAL_47:.*]] = fir.shape %[[VAL_45]] : (index) -> !fir.shape<1>
@@ -290,8 +290,7 @@ end subroutine test6
 ! CHECK:           %[[VAL_69:.*]] = arith.constant true
 ! CHECK:           %[[VAL_70:.*]] = hlfir.as_expr %[[VAL_48]]#0 move %[[VAL_69]] : (!fir.heap<!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>>, i1) -> !hlfir.expr<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>
 ! CHECK:           hlfir.assign %[[VAL_70]] to %[[VAL_44]] temporary_lhs : !hlfir.expr<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>, !fir.ref<!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>>
-! CHECK:           hlfir.assign %[[VAL_20]]#0 to %[[VAL_12]]#0 : !fir.ref<!fir.type<_QMtypesTt6{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>, !fir.ref<!fir.type<_QMtypesTt6{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>
-! CHECK:           hlfir.destroy %[[VAL_70]] : !hlfir.expr<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>
+! CHECK:           hlfir.assign %[[VAL_20]]#0 to %[[VAL_12]]#0 : !fir.ref<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>, !fir.ref<!fir.type<_QMtypesTt6{t5:!fir.type<_QMtypesTt5{t5m:!fir.box<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt4{c:!fir.box<!fir.heap<!fir.array<?x!fir.char<1,2>>>>}>>>>}>,t6m:!fir.array<1x!fir.type<_QMtypesTt1{c:!fir.char<1,4>}>>}>>
 ! CHECK:           return
 ! CHECK:         }
 
@@ -307,7 +306,7 @@ subroutine test7(n)
   x = t7(n)
 end subroutine test7
 ! CHECK-LABEL:   func.func @_QPtest7(
-! CHECK-SAME:      %[[VAL_0:.*]]: !fir.ref<i32> {fir.bindc_name = "n"}) {
+! CHECK-SAME:                        %[[VAL_0:.*]]: !fir.ref<i32> {fir.bindc_name = "n"}) {
 ! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.type<_QMtypesTt7{c1:i32,c2:!fir.box<!fir.heap<!fir.array<?xf32>>>}>
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QFtest7En"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:           %[[VAL_3:.*]] = fir.alloca !fir.type<_QMtypesTt7{c1:i32,c2:!fir.box<!fir.heap<!fir.array<?xf32>>>}> {bindc_name = "x", uniq_name = "_QFtest7Ex"}
diff --git a/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90 b/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
new file mode 100644
index 0000000000..6794d11ece
--- /dev/null
+++ b/flang/test/Lower/HLFIR/type-bound-call-mismatch.f90
@@ -0,0 +1,39 @@
+! Test interface that lowering handles small interface mismatch with
+! type bound procedures.
+! RUN: bbc -emit-hlfir --polymorphic-type %s -o - -I nw | FileCheck %s
+
+module dispatch_mismatch
+type t
+  integer :: i
+end type
+type, extends(t) :: t2
+  contains
+    procedure :: proc => foo
+end type
+
+interface
+  subroutine foo(x)
+    import :: t2
+    class(t2) :: x
+  end subroutine
+end interface
+
+end module
+
+subroutine foo(x)
+  use dispatch_mismatch, only : t
+  ! mistmatch compared to the interface, but OK from an ABI
+  ! point of view, and OKI because args compatible with t2 are
+  ! compatible with t.
+  class(t) :: x
+end subroutine
+
+subroutine test(x)
+  use dispatch_mismatch, only : t2
+  class(t2) :: x
+  call x%proc()
+end subroutine
+!CHECK-LABEL:  func.func @_QPtest(
+!CHECK:    %[[X:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFtestEx"}
+!CHECK:    %[[CAST:.*]] = fir.convert %[[X]]#0 : (!fir.class<!fir.type<_QMdispatch_mismatchTt2{t:!fir.type<_QMdispatch_mismatchTt{i:i32}>}>>) -> !fir.class<!fir.type<_QMdispatch_mismatchTt{i:i32}>>
+!CHECK:    fir.dispatch "proc"(%[[X]]#0 : !fir.class<!fir.type<_QMdispatch_mismatchTt2{t:!fir.type<_QMdispatch_mismatchTt{i:i32}>}>>) (%[[CAST]] : !fir.class<!fir.type<_QMdispatch_mismatchTt{i:i32}>>) {pass_arg_pos = 0 : i32}
diff --git a/flang/test/Lower/HLFIR/type-info.f90 b/flang/test/Lower/HLFIR/type-info.f90
index e0716fd069..f1792dc0c0 100644
--- a/flang/test/Lower/HLFIR/type-info.f90
+++ b/flang/test/Lower/HLFIR/type-info.f90
@@ -54,7 +54,7 @@ subroutine needs_final_final(x)
 ! CHECK-DAG:  fir.type_info @_QMtyinfoTneeds_init1 nodestroy nofinal : !fir.type<_QMtyinfoTneeds_init1{i:i32}>
 ! CHECK-DAG:  fir.type_info @_QMtyinfoTneeds_init_and_destroy nofinal : !fir.type<_QMtyinfoTneeds_init_and_destroy{x:!fir.box<!fir.heap<i32>>}>
 ! CHECK-DAG:  fir.type_info @_QMtyinfoTneeds_all : !fir.type<_QMtyinfoTneeds_all{x:!fir.type<_QMtyinfoTneeds_final>,y:!fir.type<_QMtyinfoTneeds_init_and_destroy{x:!fir.box<!fir.heap<i32>>}>}>
-! CHECK-DAG:  fir.type_info @_QMtyinfoTinherits_final noinit extends !fir.type<_QMtyinfoTneeds_final> : !fir.type<_QMtyinfoTinherits_final>
-! CHECK-DAG:  fir.type_info @_QMtyinfoTinherits_init nodestroy nofinal extends !fir.type<_QMtyinfoTneeds_init1{i:i32}> : !fir.type<_QMtyinfoTinherits_init{i:i32}>
-! CHECK-DAG:  fir.type_info @_QMtyinfoTinherits_init_and_destroy nofinal extends !fir.type<_QMtyinfoTneeds_init_and_destroy{x:!fir.box<!fir.heap<i32>>}> : !fir.type<_QMtyinfoTinherits_init_and_destroy{x:!fir.box<!fir.heap<i32>>}>
-! CHECK-DAG:  fir.type_info @_QMtyinfoTinherits_all extends !fir.type<_QMtyinfoTneeds_all{x:!fir.type<_QMtyinfoTneeds_final>,y:!fir.type<_QMtyinfoTneeds_init_and_destroy{x:!fir.box<!fir.heap<i32>>}>}> : !fir.type<_QMtyinfoTinherits_all{x:!fir.type<_QMtyinfoTneeds_final>,y:!fir.type<_QMtyinfoTneeds_init_and_destroy{x:!fir.box<!fir.heap<i32>>}>}>
+! CHECK-DAG:  fir.type_info @_QMtyinfoTinherits_final noinit extends !fir.type<_QMtyinfoTneeds_final> : !fir.type<_QMtyinfoTinherits_final{needs_final:!fir.type<_QMtyinfoTneeds_final>}>
+! CHECK-DAG:  fir.type_info @_QMtyinfoTinherits_init nodestroy nofinal extends !fir.type<_QMtyinfoTneeds_init1{i:i32}> : !fir.type<_QMtyinfoTinherits_init{needs_init1:!fir.type<_QMtyinfoTneeds_init1{i:i32}>}>
+! CHECK-DAG:  fir.type_info @_QMtyinfoTinherits_init_and_destroy nofinal extends !fir.type<_QMtyinfoTneeds_init_and_destroy{x:!fir.box<!fir.heap<i32>>}> : !fir.type<_QMtyinfoTinherits_init_and_destroy{needs_init_and_destroy:!fir.type<_QMtyinfoTneeds_init_and_destroy{x:!fir.box<!fir.heap<i32>>}>}>
+! CHECK-DAG:  fir.type_info @_QMtyinfoTinherits_all extends !fir.type<_QMtyinfoTneeds_all{x:!fir.type<_QMtyinfoTneeds_final>,y:!fir.type<_QMtyinfoTneeds_init_and_destroy{x:!fir.box<!fir.heap<i32>>}>}> : !fir.type<_QMtyinfoTinherits_all{needs_all:!fir.type<_QMtyinfoTneeds_all{x:!fir.type<_QMtyinfoTneeds_final>,y:!fir.type<_QMtyinfoTneeds_init_and_destroy{x:!fir.box<!fir.heap<i32>>}>}>}>
diff --git a/flang/test/Lower/OpenACC/acc-bounds.f90 b/flang/test/Lower/OpenACC/acc-bounds.f90
new file mode 100644
index 0000000000..c63c9aacf5
--- /dev/null
+++ b/flang/test/Lower/OpenACC/acc-bounds.f90
@@ -0,0 +1,89 @@
+! This test checks lowering of OpenACC data bounds operation.
+
+! RUN: bbc -fopenacc -emit-fir %s -o - | FileCheck %s --check-prefixes=CHECK,FIR
+! RUN: bbc -fopenacc -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,HLFIR
+
+module openacc_bounds
+
+type t1
+  integer, pointer, dimension(:) :: array_comp
+end type
+
+type t2
+  integer, dimension(10) :: array_comp
+end type
+
+type t3
+  integer, allocatable, dimension(:) :: array_comp
+end type
+
+contains
+  subroutine acc_derived_type_component_pointer_array()
+    type(t1) :: d
+    !$acc enter data create(d%array_comp)
+  end subroutine
+
+! CHECK-LABEL: func.func @_QMopenacc_boundsPacc_derived_type_component_pointer_array() {
+! CHECK: %[[D:.*]] = fir.alloca !fir.type<_QMopenacc_boundsTt1{array_comp:!fir.box<!fir.ptr<!fir.array<?xi32>>>}> {bindc_name = "d", uniq_name = "_QMopenacc_boundsFacc_derived_type_component_pointer_arrayEd"}
+! HLFIR: %[[DECL_D:.*]]:2 = hlfir.declare %[[D]] {uniq_name = "_QMopenacc_boundsFacc_derived_type_component_pointer_arrayEd"} : (!fir.ref<!fir.type<_QMopenacc_boundsTt1{array_comp:!fir.box<!fir.ptr<!fir.array<?xi32>>>}>>) -> (!fir.ref<!fir.type<_QMopenacc_boundsTt1{array_comp:!fir.box<!fir.ptr<!fir.array<?xi32>>>}>>, !fir.ref<!fir.type<_QMopenacc_boundsTt1{array_comp:!fir.box<!fir.ptr<!fir.array<?xi32>>>}>>)
+! FIR: %[[FIELD:.*]] = fir.field_index array_comp, !fir.type<_QMopenacc_boundsTt1{array_comp:!fir.box<!fir.ptr<!fir.array<?xi32>>>}>
+! FIR: %[[COORD:.*]] = fir.coordinate_of %[[D]], %[[FIELD]] : (!fir.ref<!fir.type<_QMopenacc_boundsTt1{array_comp:!fir.box<!fir.ptr<!fir.array<?xi32>>>}>>, !fir.field) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
+! HLFIR: %[[COORD:.*]] = hlfir.designate %[[DECL_D]]#0{"array_comp"}   {fortran_attrs = #fir.var_attrs<pointer>} : (!fir.ref<!fir.type<_QMopenacc_boundsTt1{array_comp:!fir.box<!fir.ptr<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
+! CHECK: %[[LOAD:.*]] = fir.load %[[COORD]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
+! CHECK: %[[BOX_DIMS0:.*]]:3 = fir.box_dims %[[LOAD]], %c0{{.*}} : (!fir.box<!fir.ptr<!fir.array<?xi32>>>, index) -> (index, index, index)
+! CHECK: %[[C1:.*]] = arith.constant 1 : index
+! CHECK: %[[BOX_DIMS1:.*]]:3 = fir.box_dims %[[LOAD]], %c0{{.*}} : (!fir.box<!fir.ptr<!fir.array<?xi32>>>, index) -> (index, index, index)
+! CHECK: %[[UB:.*]] = arith.subi %[[BOX_DIMS1]]#1, %[[C1]] : index
+! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%c0{{.*}} : index) upperbound(%[[UB]] : index) stride(%[[BOX_DIMS1]]#2 : index) startIdx(%[[BOX_DIMS0]]#0 : index) {strideInBytes = true}
+! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[LOAD]] : (!fir.box<!fir.ptr<!fir.array<?xi32>>>) -> !fir.ptr<!fir.array<?xi32>>
+! CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.ptr<!fir.array<?xi32>>) bounds(%[[BOUND]]) -> !fir.ptr<!fir.array<?xi32>> {name = "d%array_comp", structured = false}
+! CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ptr<!fir.array<?xi32>>)
+! CHECK: return
+! CHECK: }
+
+  subroutine acc_derived_type_component_array()
+    type(t2) :: d
+    !$acc enter data create(d%array_comp)
+  end subroutine
+
+! CHECK-LABEL: func.func @_QMopenacc_boundsPacc_derived_type_component_array()
+! CHECK: %[[D:.*]] = fir.alloca !fir.type<_QMopenacc_boundsTt2{array_comp:!fir.array<10xi32>}> {bindc_name = "d", uniq_name = "_QMopenacc_boundsFacc_derived_type_component_arrayEd"}
+! HLFIR: %[[DECL_D:.*]]:2 = hlfir.declare %[[D]] {uniq_name = "_QMopenacc_boundsFacc_derived_type_component_arrayEd"} : (!fir.ref<!fir.type<_QMopenacc_boundsTt2{array_comp:!fir.array<10xi32>}>>) -> (!fir.ref<!fir.type<_QMopenacc_boundsTt2{array_comp:!fir.array<10xi32>}>>, !fir.ref<!fir.type<_QMopenacc_boundsTt2{array_comp:!fir.array<10xi32>}>>)
+! FIR:   %[[FIELD:.*]] = fir.field_index array_comp, !fir.type<_QMopenacc_boundsTt2{array_comp:!fir.array<10xi32>}>
+! FIR:   %[[COORD:.*]] = fir.coordinate_of %[[D]], %[[FIELD]] : (!fir.ref<!fir.type<_QMopenacc_boundsTt2{array_comp:!fir.array<10xi32>}>>, !fir.field) -> !fir.ref<!fir.array<10xi32>>
+! CHECK: %[[C10:.*]] = arith.constant 10 : index
+! HLFIR: %[[SHAPE:.*]] = fir.shape %[[C10]] : (index) -> !fir.shape<1>
+! HLFIR: %[[COORD:.*]] = hlfir.designate %[[DECL_D]]#0{"array_comp"} shape %[[SHAPE]] : (!fir.ref<!fir.type<_QMopenacc_boundsTt2{array_comp:!fir.array<10xi32>}>>, !fir.shape<1>) -> !fir.ref<!fir.array<10xi32>>
+! CHECK: %[[C1:.*]] = arith.constant 1 : index
+! CHECK: %[[C0:.*]] = arith.constant 0 : index
+! CHECK: %[[UB:.*]] = arith.subi %[[C10]], %[[C1]] : index
+! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[C0]] : index) upperbound(%[[UB]] : index) extent(%[[C10]] : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
+! CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[COORD]] : !fir.ref<!fir.array<10xi32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<10xi32>> {name = "d%array_comp", structured = false}
+! CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.ref<!fir.array<10xi32>>)
+! CHECK: return
+! CHECK: }
+
+  subroutine acc_derived_type_component_allocatable_array()
+    type(t3) :: d
+    !$acc enter data create(d%array_comp)
+  end subroutine
+
+! CHECK-LABEL: func.func @_QMopenacc_boundsPacc_derived_type_component_allocatable_array() {
+! CHECK: %[[D:.*]] = fir.alloca !fir.type<_QMopenacc_boundsTt3{array_comp:!fir.box<!fir.heap<!fir.array<?xi32>>>}> {bindc_name = "d", uniq_name = "_QMopenacc_boundsFacc_derived_type_component_allocatable_arrayEd"}
+! HLFIR: %[[DECL_D:.*]]:2 = hlfir.declare %[[D]] {uniq_name = "_QMopenacc_boundsFacc_derived_type_component_allocatable_arrayEd"} : (!fir.ref<!fir.type<_QMopenacc_boundsTt3{array_comp:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> (!fir.ref<!fir.type<_QMopenacc_boundsTt3{array_comp:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>, !fir.ref<!fir.type<_QMopenacc_boundsTt3{array_comp:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>)
+! FIR: %[[FIELD:.*]] = fir.field_index array_comp, !fir.type<_QMopenacc_boundsTt3{array_comp:!fir.box<!fir.heap<!fir.array<?xi32>>>}>
+! FIR: %[[COORD:.*]] = fir.coordinate_of %[[D]], %[[FIELD]] : (!fir.ref<!fir.type<_QMopenacc_boundsTt3{array_comp:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>, !fir.field) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! HLFIR: %[[COORD:.*]] = hlfir.designate %[[DECL_D]]#0{"array_comp"}   {fortran_attrs = #fir.var_attrs<allocatable>} : (!fir.ref<!fir.type<_QMopenacc_boundsTt3{array_comp:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK: %[[LOAD:.*]] = fir.load %[[COORD]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK: %[[BOX_DIMS0:.*]]:3 = fir.box_dims %[[LOAD]], %c0{{.*}} : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
+! CHECK: %[[C1:.*]] = arith.constant 1 : index
+! CHECK: %[[BOX_DIMS1:.*]]:3 = fir.box_dims %[[LOAD]], %c0{{.*}} : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
+! CHECK: %[[UB:.*]] = arith.subi %[[BOX_DIMS1]]#1, %[[C1]] : index
+! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%c0{{.*}} : index) upperbound(%[[UB]] : index) stride(%[[BOX_DIMS1]]#2 : index) startIdx(%[[BOX_DIMS0]]#0 : index) {strideInBytes = true}
+! CHECK: %[[BOX_ADDR:.*]] = fir.box_addr %[[LOAD]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.heap<!fir.array<?xi32>>
+! CHECK: %[[CREATE:.*]] = acc.create varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xi32>>) bounds(%[[BOUND]]) -> !fir.heap<!fir.array<?xi32>> {name = "d%array_comp", structured = false}
+! CHECK: acc.enter_data dataOperands(%[[CREATE]] : !fir.heap<!fir.array<?xi32>>)
+! CHECK: return
+! CHECK: }
+
+end module
diff --git a/flang/test/Lower/OpenACC/acc-init.f90 b/flang/test/Lower/OpenACC/acc-init.f90
index 9132d41ccb..de940426b6 100644
--- a/flang/test/Lower/OpenACC/acc-init.f90
+++ b/flang/test/Lower/OpenACC/acc-init.f90
@@ -5,6 +5,7 @@
 
 subroutine acc_init
   logical :: ifCondition = .TRUE.
+  integer :: ifInt = 1
 
   !$acc init
 !CHECK: acc.init{{ *}}{{$}}
@@ -28,4 +29,9 @@ subroutine acc_init
 !CHECK: [[DEVTYPE2:%.*]] = arith.constant 2 : i32
 !CHECK: acc.init device_type([[DEVTYPE1]], [[DEVTYPE2]] : i32, i32) device_num([[DEVNUM]] : i32){{$}}
 
+  !$acc init if(ifInt)
+!CHECK: %[[IFINT:.*]] = fir.load %{{.*}} : !fir.ref<i32>
+!CHECK: %[[CONV:.*]] = fir.convert %[[IFINT]] : (i32) -> i1
+!CHECK: acc.init if(%[[CONV]])
+
 end subroutine acc_init
diff --git a/flang/test/Lower/OpenACC/acc-parallel-loop.f90 b/flang/test/Lower/OpenACC/acc-parallel-loop.f90
index 9a27a359e8..eea4950b6d 100644
--- a/flang/test/Lower/OpenACC/acc-parallel-loop.f90
+++ b/flang/test/Lower/OpenACC/acc-parallel-loop.f90
@@ -3,33 +3,6 @@
 ! RUN: bbc -fopenacc -emit-fir %s -o - | FileCheck %s --check-prefixes=CHECK,FIR
 ! RUN: bbc -fopenacc -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,HLFIR
 
-! CHECK-LABEL: acc.firstprivate.recipe @firstprivatization_section_ext10_ref_10xf32 : !fir.ref<!fir.array<10xf32>> init {
-! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.array<10xf32>>):
-! HLFIR:   %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
-! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<10xf32>
-! HLFIR:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.private.init"} : (!fir.ref<!fir.array<10xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
-! HLFIR:   acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.array<10xf32>>
-! CHECK: } copy {
-! CHECK: ^bb0(%[[SRC:.*]]: !fir.ref<!fir.array<10xf32>>, %[[DST:.*]]: !fir.ref<!fir.array<10xf32>>):
-! CHECK:   %[[LB0:.*]] = arith.constant 0 : index
-! CHECK:   %[[UB0:.*]] = arith.constant 9 : index
-! CHECK:   %[[STEP0:.*]] = arith.constant 1 : index
-! CHECK:   fir.do_loop %[[IV0:.*]] = %[[LB0]] to %[[UB0]] step %[[STEP0]] {
-! CHECK:     %[[COORD0:.*]] = fir.coordinate_of %[[SRC]], %[[IV0]] : (!fir.ref<!fir.array<10xf32>>, index) -> !fir.ref<f32>
-! CHECK:     %[[COORD1:.*]] = fir.coordinate_of %[[DST]], %[[IV0]] : (!fir.ref<!fir.array<10xf32>>, index) -> !fir.ref<f32>
-! CHECK:     %[[LOAD:.*]] = fir.load %[[COORD0]] : !fir.ref<f32>
-! CHECK:     fir.store %[[LOAD]] to %[[COORD1]] : !fir.ref<f32>
-! CHECK:   }
-! CHECK:   acc.terminator
-! CHECK: }
-
-! CHECK-LABEL: acc.private.recipe @privatization_ref_10xf32 : !fir.ref<!fir.array<10xf32>> init {
-! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.array<10xf32>>):
-! HLFIR:   %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
-! HLFIR:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.private.init"} : (!fir.ref<!fir.array<10xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
-! HLFIR:   acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.array<10xf32>>
-! CHECK: }
-
 ! CHECK-LABEL: func.func @_QPacc_parallel_loop()
 
 subroutine acc_parallel_loop
@@ -510,11 +483,9 @@ subroutine acc_parallel_loop
     a(i) = b(i)
   END DO
 
-! FIR:        %[[ACC_PRIVATE_A:.*]] = acc.private varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! HLFIR:      %[[ACC_PRIVATE_A:.*]] = acc.private varPtr(%[[DECLA]]#1 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
 ! FIR:        %[[ACC_PRIVATE_B:.*]] = acc.firstprivate varPtr(%[[B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
 ! HLFIR:      %[[ACC_PRIVATE_B:.*]] = acc.firstprivate varPtr(%[[DECLB]]#1 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
-! CHECK:      acc.parallel firstprivate(@firstprivatization_section_ext10_ref_10xf32 -> %[[ACC_PRIVATE_B]] : !fir.ref<!fir.array<10xf32>>) private(@privatization_ref_10xf32 -> %[[ACC_PRIVATE_A]] : !fir.ref<!fir.array<10xf32>>) {
+! CHECK:      acc.parallel firstprivate(@firstprivatization_section_ext10_ref_10xf32 -> %[[ACC_PRIVATE_B]] : !fir.ref<!fir.array<10xf32>>) {
 ! FIR:        %[[ACC_PRIVATE_A:.*]] = acc.private varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
 ! HLFIR:      %[[ACC_PRIVATE_A:.*]] = acc.private varPtr(%[[DECLA]]#1 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
 ! CHECK:        acc.loop private(@privatization_ref_10xf32 -> %[[ACC_PRIVATE_A]] : !fir.ref<!fir.array<10xf32>>) {
@@ -799,7 +770,7 @@ subroutine acc_parallel_loop
     reduction_i = 1
   end do
 
-! CHECK:      acc.parallel reduction(@reduction_add_ref_f32 -> %{{.*}} : !fir.ref<f32>, @reduction_mul_ref_i32 -> %{{.*}} : !fir.ref<i32>) {
+! CHECK:      acc.parallel {
 ! CHECK:        acc.loop reduction(@reduction_add_ref_f32 -> %{{.*}} : !fir.ref<f32>, @reduction_mul_ref_i32 -> %{{.*}} : !fir.ref<i32>) {
 ! CHECK:          fir.do_loop
 ! CHECK:          acc.yield
diff --git a/flang/test/Lower/OpenACC/acc-private.f90 b/flang/test/Lower/OpenACC/acc-private.f90
index 6adbad754d..80b474b348 100644
--- a/flang/test/Lower/OpenACC/acc-private.f90
+++ b/flang/test/Lower/OpenACC/acc-private.f90
@@ -3,6 +3,23 @@
 ! RUN: bbc -fopenacc -emit-fir %s -o - | FileCheck %s --check-prefixes=CHECK,FIR
 ! RUN: bbc -fopenacc -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,HLFIR
 
+! CHECK-LABEL: acc.firstprivate.recipe @firstprivatization_ref_UxUx2xi32 : !fir.ref<!fir.array<?x?x2xi32>> init {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<!fir.array<?x?x2xi32>>, %[[ARG1:.*]]: index, %[[ARG2:.*]]: index, %[[ARG3:.*]]: index):
+! HLFIR:   %[[SHAPE:.*]] = fir.shape %[[ARG1]], %[[ARG2]], %[[ARG3]] : (index, index, index) -> !fir.shape<3>
+! HLFIR:   %[[TEMP:.*]] = fir.alloca !fir.array<?x?x2xi32>, %[[ARG1]], %[[ARG2]], %[[ARG3]]
+! HLFIR:   %[[DECL:.*]]:2 = hlfir.declare %[[TEMP]](%[[SHAPE]]) {uniq_name = "acc.private.init"} : (!fir.ref<!fir.array<?x?x2xi32>>, !fir.shape<3>) -> (!fir.box<!fir.array<?x?x2xi32>>, !fir.ref<!fir.array<?x?x2xi32>>)
+! HLFIR:   acc.yield %[[DECL]]#0 : !fir.box<!fir.array<?x?x2xi32>>
+! CHECK: } copy {
+! CHECK: ^bb0(%[[SRC:.*]]: !fir.ref<!fir.array<?x?x2xi32>>, %[[DST:.*]]: !fir.ref<!fir.array<?x?x2xi32>>, %[[LB0:.*]]: index, %[[UB0:.*]]: index, %[[STEP0:.*]]: index, %[[LB1:.*]]: index, %[[UB1:.*]]: index, %[[STEP1:.*]]: index, %[[LB2:.*]]: index, %[[UB2:.*]]: index, %[[STEP2:.*]]: index):
+! HLFIR:   %[[SHAPE:.*]] = fir.shape %{{.*}}, %{{.*}}, %{{.*}} : (index, index, index) -> !fir.shape<3>
+! HLFIR:   %[[DECL_SRC:.*]]:2 = hlfir.declare %[[SRC]](%[[SHAPE]]) {uniq_name = ""} : (!fir.ref<!fir.array<?x?x2xi32>>, !fir.shape<3>) -> (!fir.box<!fir.array<?x?x2xi32>>, !fir.ref<!fir.array<?x?x2xi32>>)
+! HLFIR:   %[[DECL_DST:.*]]:2 = hlfir.declare %[[DST]](%[[SHAPE]]) {uniq_name = ""} : (!fir.ref<!fir.array<?x?x2xi32>>, !fir.shape<3>) -> (!fir.box<!fir.array<?x?x2xi32>>, !fir.ref<!fir.array<?x?x2xi32>>)
+! HLFIR:   %[[DES_SRC:.*]] = hlfir.designate %[[DECL_SRC]]#0 (%[[LB0]]:%[[UB0]]:%[[STEP0]], %[[LB1]]:%[[UB1]]:%[[STEP1]], %[[LB2]]:%[[UB2]]:%[[STEP2]]) shape %[[SHAPE]] : (!fir.box<!fir.array<?x?x2xi32>>, index, index, index, index, index, index, index, index, index, !fir.shape<3>) -> !fir.box<!fir.array<?x?x2xi32>>
+! HLFIR:   %[[DES_DST:.*]] = hlfir.designate %[[DECL_DST]]#0 (%[[LB0]]:%[[UB0]]:%[[STEP0]], %[[LB1]]:%[[UB1]]:%[[STEP1]], %[[LB2]]:%[[UB2]]:%[[STEP2]]) shape %[[SHAPE]] : (!fir.box<!fir.array<?x?x2xi32>>, index, index, index, index, index, index, index, index, index, !fir.shape<3>) -> !fir.box<!fir.array<?x?x2xi32>>
+! HLFIR:   hlfir.assign %[[DES_SRC]] to %[[DES_DST]] : !fir.box<!fir.array<?x?x2xi32>>, !fir.box<!fir.array<?x?x2xi32>>
+! HLFIR:   acc.terminator
+! CHECK: }
+
 ! CHECK-LABEL: acc.firstprivate.recipe @firstprivatization_section_lb4.ub9_box_Uxi32 : !fir.box<!fir.array<?xi32>> init {
 ! CHECK: ^bb0(%{{.*}}: !fir.box<!fir.array<?xi32>>):
 ! CHECK: } copy {
@@ -87,16 +104,12 @@
 ! HLFIR:   acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.array<50xf32>>
 ! CHECK: } copy {
 ! CHECK: ^bb0(%[[SRC:.*]]: !fir.ref<!fir.array<50xf32>>, %[[DST:.*]]: !fir.ref<!fir.array<50xf32>>):
-! CHECK:   %[[LB0:.*]] = arith.constant 0 : index
-! CHECK:   %[[UB0:.*]] = arith.constant 49 : index
-! CHECK:   %[[STEP0:.*]] = arith.constant 1 : index
-! CHECK:   fir.do_loop %[[IV0:.*]] = %[[LB0]] to %[[UB0]] step %[[STEP0]] {
-! CHECK:     %[[COORD0:.*]] = fir.coordinate_of %[[SRC]], %[[IV0]] : (!fir.ref<!fir.array<50xf32>>, index) -> !fir.ref<f32>
-! CHECK:     %[[COORD1:.*]] = fir.coordinate_of %[[DST]], %[[IV0]] : (!fir.ref<!fir.array<50xf32>>, index) -> !fir.ref<f32>
-! CHECK:     %[[VALUE:.*]] = fir.load %[[COORD0]] : !fir.ref<f32>
-! CHECK:     fir.store %[[VALUE]] to %[[COORD1]] : !fir.ref<f32>
-! CHECK:   }
-! CHECK:   acc.terminator
+! HLFIR:   %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
+! HLFIR:   %[[DECL_SRC:.*]]:2 = hlfir.declare %[[SRC]](%[[SHAPE]]) {uniq_name = ""} : (!fir.ref<!fir.array<50xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<50xf32>>, !fir.ref<!fir.array<50xf32>>)
+! HLFIR:   %[[DECL_DST:.*]]:2 = hlfir.declare %[[DST]](%[[SHAPE]]) {uniq_name = ""} : (!fir.ref<!fir.array<50xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<50xf32>>, !fir.ref<!fir.array<50xf32>>)
+! HLFIR:   %[[DES_SRC:.*]] = hlfir.designate %[[DECL_SRC]]#0 shape %[[SHAPE:.*]] : (!fir.ref<!fir.array<50xf32>>, !fir.shape<1>) -> !fir.ref<!fir.array<50xf32>>
+! HLFIR:   %[[DES_DST:.*]] = hlfir.designate %[[DECL_DST]]#0 shape %[[SHAPE:.*]] : (!fir.ref<!fir.array<50xf32>>, !fir.shape<1>) -> !fir.ref<!fir.array<50xf32>>
+! HLFIR:   hlfir.assign %[[DES_SRC]] to %[[DES_DST]] : !fir.ref<!fir.array<50xf32>>, !fir.ref<!fir.array<50xf32>>
 ! CHECK: }
 
 ! CHECK-LABEL: acc.firstprivate.recipe @firstprivatization_section_ext100_ref_100xf32 : !fir.ref<!fir.array<100xf32>> init {
@@ -107,15 +120,12 @@
 ! HLFIR:   acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.array<100xf32>>
 ! CHECK: } copy {
 ! CHECK: ^bb0(%[[SRC:.*]]: !fir.ref<!fir.array<100xf32>>, %[[DST:.*]]: !fir.ref<!fir.array<100xf32>>):
-! CHECK:   %[[LB0:.*]] = arith.constant 0 : index
-! CHECK:   %[[UB0:.*]] = arith.constant 99 : index
-! CHECK:   %[[STEP1:.*]] = arith.constant 1 : index
-! CHECK:   fir.do_loop %[[IV0:.*]] = %c0 to %c99 step %c1 {
-! CHECK:     %[[COORD0:.*]] = fir.coordinate_of %[[SRC]], %[[IV0]] : (!fir.ref<!fir.array<100xf32>>, index) -> !fir.ref<f32>
-! CHECK:     %[[COORD1:.*]] = fir.coordinate_of %[[DST]], %[[IV0]] : (!fir.ref<!fir.array<100xf32>>, index) -> !fir.ref<f32>
-! CHECK:     %[[VALUE:.*]] = fir.load %[[COORD0]] : !fir.ref<f32>
-! CHECK:     fir.store %[[VALUE]] to %[[COORD1]] : !fir.ref<f32>
-! CHECK:   }
+! HLFIR:   %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
+! HLFIR:   %[[DECL_SRC:.*]]:2 = hlfir.declare %[[SRC]](%[[SHAPE]]) {uniq_name = ""} : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xf32>>, !fir.ref<!fir.array<100xf32>>)
+! HLFIR:   %[[DECL_DST:.*]]:2 = hlfir.declare %[[DST]](%[[SHAPE]]) {uniq_name = ""} : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xf32>>, !fir.ref<!fir.array<100xf32>>)
+! HLFIR:   %[[DES_SRC:.*]] = hlfir.designate %[[DECL_SRC]]#0 shape %[[SHAPE]] : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>) -> !fir.ref<!fir.array<100xf32>>
+! HLFIR:   %[[DES_DST:.*]] = hlfir.designate %[[DECL_DST]]#0 shape %[[SHAPE]] : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>) -> !fir.ref<!fir.array<100xf32>>
+! HLFIR:   hlfir.assign %[[DES_SRC]] to %[[DES_DST]] : !fir.ref<!fir.array<100xf32>>, !fir.ref<!fir.array<100xf32>>
 ! CHECK:   acc.terminator
 ! CHECK: }
 
@@ -258,9 +268,10 @@ subroutine acc_private_assumed_shape(a, n)
 ! CHECK-LABEL: func.func @_QPacc_private_assumed_shape(
 ! CHECK-SAME:    %[[ARG0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"}
 ! HLFIR: %[[DECL_A:.*]]:2 = hlfir.declare %[[ARG0]] {uniq_name = "_QFacc_private_assumed_shapeEa"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+! HLFIR: acc.parallel {
 ! HLFIR: %[[ADDR:.*]] = fir.box_addr %[[DECL_A]]#1 : (!fir.box<!fir.array<?xi32>>) -> !fir.ref<!fir.array<?xi32>>
 ! HLFIR: %[[PRIVATE:.*]] = acc.private varPtr(%[[ADDR]] : !fir.ref<!fir.array<?xi32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<?xi32>> {name = "a"}
-! HLFIR: acc.parallel private(@privatization_box_Uxi32 -> %[[PRIVATE]] : !fir.ref<!fir.array<?xi32>>) {
+! HLFIR: acc.loop private(@privatization_box_Uxi32 -> %[[PRIVATE]] : !fir.ref<!fir.array<?xi32>>) {
 
 subroutine acc_private_allocatable_array(a, n)
   integer, allocatable :: a(:)
@@ -270,15 +281,21 @@ subroutine acc_private_allocatable_array(a, n)
   do i = 1, n
     a(i) = i
   end do
+
+  !$acc serial private(a)
+  a(i) = 1
+  !$acc end serial
 end subroutine
 
 ! CHECK-LABEL: func.func @_QPacc_private_allocatable_array(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {fir.bindc_name = "a"}
 ! HLFIR: %[[DECLA_A:.*]]:2 = hlfir.declare %[[ARG0]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFacc_private_allocatable_arrayEa"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
+! HLFIR: acc.parallel {
 ! HLFIR: %[[BOX:.*]] = fir.load %[[DECLA_A]]#1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 ! HLFIR: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.heap<!fir.array<?xi32>>
 ! HLFIR: %[[PRIVATE:.*]] = acc.private varPtr(%[[BOX_ADDR]] : !fir.heap<!fir.array<?xi32>>) bounds(%{{.*}}) -> !fir.heap<!fir.array<?xi32>> {name = "a"}
-! HLFIR: acc.parallel private(@privatization_box_heap_Uxi32 -> %[[PRIVATE]] : !fir.heap<!fir.array<?xi32>>) {
+! HLFIR: acc.loop private(@privatization_box_heap_Uxi32 -> %[[PRIVATE]] : !fir.heap<!fir.array<?xi32>>)
+! HLFIR: acc.serial private(@privatization_box_heap_Uxi32 -> %{{.*}} : !fir.heap<!fir.array<?xi32>>)
 
 subroutine acc_private_pointer_array(a, n)
   integer, pointer :: a(:)
@@ -293,10 +310,11 @@ subroutine acc_private_pointer_array(a, n)
 ! CHECK-LABEL: func.func @_QPacc_private_pointer_array(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>> {fir.bindc_name = "a"}, %arg1: !fir.ref<i32> {fir.bindc_name = "n"}) {
 ! HLFIR: %[[DECL_A:.*]]:2 = hlfir.declare %arg0 {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFacc_private_pointer_arrayEa"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>)
+! HLFIR: acc.parallel {
 ! HLFIR: %[[BOX:.*]] = fir.load %[[DECLA_A]]#1 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
 ! HLFIR: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX]] : (!fir.box<!fir.ptr<!fir.array<?xi32>>>) -> !fir.ptr<!fir.array<?xi32>>
 ! HLFIR: %[[PRIVATE:.*]] = acc.private varPtr(%[[BOX_ADDR]] : !fir.ptr<!fir.array<?xi32>>) bounds(%{{.*}}) -> !fir.ptr<!fir.array<?xi32>> {name = "a"}
-! HLFIR: acc.parallel private(@privatization_box_ptr_Uxi32 -> %[[PRIVATE]] : !fir.ptr<!fir.array<?xi32>>)
+! HLFIR: acc.loop private(@privatization_box_ptr_Uxi32 -> %[[PRIVATE]] : !fir.ptr<!fir.array<?xi32>>)
 
 subroutine acc_private_dynamic_extent(a, n)
   integer :: n, i
@@ -312,8 +330,9 @@ subroutine acc_private_dynamic_extent(a, n)
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.array<?x?x2xi32>> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<i32> {fir.bindc_name = "n"}) {
 ! HLFIR: %[[DECL_N:.*]]:2 = hlfir.declare %arg1 {uniq_name = "_QFacc_private_dynamic_extentEn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! HLFIR: %[[DECL_A:.*]]:2 = hlfir.declare %[[ARG0]](%16) {uniq_name = "_QFacc_private_dynamic_extentEa"} : (!fir.ref<!fir.array<?x?x2xi32>>, !fir.shape<3>) -> (!fir.box<!fir.array<?x?x2xi32>>, !fir.ref<!fir.array<?x?x2xi32>>)
+! HLFIR: acc.parallel {
 ! HLFIR: %[[PRIV:.*]] = acc.private varPtr(%[[DECL_A]]#1 : !fir.ref<!fir.array<?x?x2xi32>>) bounds(%{{.*}}, %{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<?x?x2xi32>> {name = "a"}
-! HLFIR: acc.parallel private(@privatization_ref_UxUx2xi32 -> %[[PRIV]] : !fir.ref<!fir.array<?x?x2xi32>>)
+! HLFIR: acc.loop private(@privatization_ref_UxUx2xi32 -> %[[PRIV]] : !fir.ref<!fir.array<?x?x2xi32>>)
 
 subroutine acc_firstprivate_assumed_shape(a, n)
   integer :: a(:), i, n
@@ -332,3 +351,15 @@ subroutine acc_firstprivate_assumed_shape_with_section(a, n)
     a(i) = i
   end do
 end subroutine
+
+subroutine acc_firstprivate_dynamic_extent(a, n)
+  integer :: n, i
+  integer :: a(n, n, 2)
+
+  !$acc parallel loop firstprivate(a)
+  do i = 1, n
+    a(i, i, 1) = i
+  end do
+end subroutine
+
+! CHECK: acc.parallel firstprivate(@firstprivatization_ref_UxUx2xi32 -> %{{.*}} : !fir.ref<!fir.array<?x?x2xi32>>)
diff --git a/flang/test/Lower/OpenACC/acc-reduction.f90 b/flang/test/Lower/OpenACC/acc-reduction.f90
index 0797944539..b874d52196 100644
--- a/flang/test/Lower/OpenACC/acc-reduction.f90
+++ b/flang/test/Lower/OpenACC/acc-reduction.f90
@@ -3,6 +3,35 @@
 ! RUN: bbc -fopenacc -emit-fir %s -o - | FileCheck %s --check-prefixes=CHECK,FIR
 ! RUN: bbc -fopenacc -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,HLFIR
 
+! CHECK-LABEL: acc.reduction.recipe @reduction_max_ref_UxUxf32 : !fir.ref<!fir.array<?x?xf32>> reduction_operator <max> init {
+! CHECK: ^bb0(%[[ARG0:.*]]: !fir.ref<!fir.array<?x?xf32>>, %[[ARG1:.*]]: index, %[[ARG2:.*]]: index):
+! HLFIR:   %[[CST:.*]] = arith.constant -1.401300e-45 : f32
+! HLFIR:   %[[SHAPE:.*]] = fir.shape %arg1, %arg2 : (index, index) -> !fir.shape<2>
+! HLFIR:   %[[TEMP:.*]] = fir.alloca !fir.array<?x?xf32>, %arg1, %arg2
+! HLFIR:   %[[DECL:.*]]:2 = hlfir.declare %[[TEMP]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref<!fir.array<?x?xf32>>, !fir.shape<2>) -> (!fir.box<!fir.array<?x?xf32>>, !fir.ref<!fir.array<?x?xf32>>)
+! HLFIR:   hlfir.assign %[[CST]] to %[[DECL]]#0 : f32, !fir.box<!fir.array<?x?xf32>>
+! HLFIR:   acc.yield %[[DECL]]#0 : !fir.box<!fir.array<?x?xf32>>
+! CHECK: } combiner {
+! CHECK: ^bb0(%[[V1:.*]]: !fir.ref<!fir.array<?x?xf32>>, %[[V2:.*]]: !fir.ref<!fir.array<?x?xf32>>, %[[LB0:.*]]: index, %[[UB0:.*]]: index, %[[STEP0:.*]]: index, %[[LB1:.*]]: index, %[[UB1:.*]]: index, %[[STEP1:.*]]: index):
+! HLFIR:   %[[SHAPE:.*]] = fir.shape %{{.*}}, %{{.*}} : (index, index) -> !fir.shape<2>
+! HLFIR:   %[[DECL_V1:.*]]:2 = hlfir.declare %[[V1]](%[[SHAPE]]) {uniq_name = ""} : (!fir.ref<!fir.array<?x?xf32>>, !fir.shape<2>) -> (!fir.box<!fir.array<?x?xf32>>, !fir.ref<!fir.array<?x?xf32>>)
+! HLFIR:   %[[DECL_V2:.*]]:2 = hlfir.declare %[[V2]](%[[SHAPE]]) {uniq_name = ""} : (!fir.ref<!fir.array<?x?xf32>>, !fir.shape<2>) -> (!fir.box<!fir.array<?x?xf32>>, !fir.ref<!fir.array<?x?xf32>>)
+! HLFIR:   %[[DES_V1:.*]] = hlfir.designate %[[DECL_V1]]#0 (%arg2:%arg3:%arg4, %arg5:%arg6:%arg7)  shape %10 : (!fir.box<!fir.array<?x?xf32>>, index, index, index, index, index, index, !fir.shape<2>) -> !fir.box<!fir.array<?x?xf32>>
+! HLFIR:   %[[DES_V2:.*]] = hlfir.designate %[[DECL_V2]]#0 (%arg2:%arg3:%arg4, %arg5:%arg6:%arg7)  shape %10 : (!fir.box<!fir.array<?x?xf32>>, index, index, index, index, index, index, !fir.shape<2>) -> !fir.box<!fir.array<?x?xf32>>
+! HLFIR:   %[[ELEMENTAL:.*]] = hlfir.elemental %[[SHAPE]] unordered : (!fir.shape<2>) -> !hlfir.expr<?x?xf32> {
+! HLFIR:   ^bb0(%[[ARG0:.*]]: index, %[[ARG1:.*]]: index):
+! HLFIR:     %[[D1:.*]] = hlfir.designate %13 (%[[ARG0]], %[[ARG1]])  : (!fir.box<!fir.array<?x?xf32>>, index, index) -> !fir.ref<f32>
+! HLFIR:     %[[D2:.*]] = hlfir.designate %14 (%[[ARG0]], %[[ARG1]])  : (!fir.box<!fir.array<?x?xf32>>, index, index) -> !fir.ref<f32>
+! HLFIR:     %[[LOAD1:.*]] = fir.load %[[D1]] : !fir.ref<f32>
+! HLFIR:     %[[LOAD2:.*]] = fir.load %[[D2]] : !fir.ref<f32>
+! HLFIR:     %[[CMP:.*]] = arith.cmpf ogt, %[[LOAD1]], %[[LOAD2]] : f32
+! HLFIR:     %[[SELECT:.*]] = arith.select %[[CMP]], %[[LOAD1]], %[[LOAD2]] : f32
+! HLFIR:     hlfir.yield_element %[[SELECT]] : f32
+! HLFIR:   }
+! HLFIR:   hlfir.assign %[[ELEMENTAL]] to %[[DECL_V1]]#0 : !hlfir.expr<?x?xf32>, !fir.box<!fir.array<?x?xf32>>
+! HLFIR:   acc.yield %[[V1]] : !fir.ref<!fir.array<?x?xf32>>
+! CHECK: }
+
 ! CHECK-LABEL: acc.reduction.recipe @reduction_max_box_ptr_Uxf32 : !fir.box<!fir.ptr<!fir.array<?xf32>>> reduction_operator <max> init {
 ! CHECK: ^bb0(%{{.*}}: !fir.box<!fir.ptr<!fir.array<?xf32>>>):
 ! CHECK: } combiner {
@@ -290,8 +319,8 @@
 ! CHECK-LABEL: acc.reduction.recipe @reduction_max_section_ext100_ref_100xf32 : !fir.ref<!fir.array<100xf32>> reduction_operator <max> init {
 ! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.array<100xf32>>):
 ! CHECK:   %[[INIT:.*]] = arith.constant -1.401300e-45 : f32
-! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<100xf32>
 ! HLFIR:   %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<100xf32>
 ! HLFIR:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xf32>>, !fir.ref<!fir.array<100xf32>>)
 ! CHECK:   %[[LB:.*]] = arith.constant 0 : index
 ! CHECK:   %[[UB:.*]] = arith.constant 99 : index
@@ -338,8 +367,8 @@
 ! CHECK-LABEL: acc.reduction.recipe @reduction_max_section_ext100xext10_ref_100x10xi32 : !fir.ref<!fir.array<100x10xi32>> reduction_operator <max> init {
 ! CHECK: ^bb0(%arg0: !fir.ref<!fir.array<100x10xi32>>):
 ! CHECK:   %[[INIT:.*]] = arith.constant -2147483648 : i32
-! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<100x10xi32>
 ! HLFIR:   %[[SHAPE:.*]] = fir.shape %{{.*}}, %{{.*}} : (index, index) -> !fir.shape<2>
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<100x10xi32>
 ! HLFIR:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref<!fir.array<100x10xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<100x10xi32>>, !fir.ref<!fir.array<100x10xi32>>)
 ! HLFIR:   acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.array<100x10xi32>>
 ! CHECK: } combiner {
@@ -384,8 +413,8 @@
 ! CHECK-LABEL: acc.reduction.recipe @reduction_min_section_ext100xext10_ref_100x10xf32 : !fir.ref<!fir.array<100x10xf32>> reduction_operator <min> init {
 ! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.array<100x10xf32>>):
 ! CHECK:   %[[INIT:.*]] = arith.constant 3.40282347E+38 : f32
-! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<100x10xf32>
 ! HLFIR:   %[[SHAPE:.*]] = fir.shape %{{.*}}, %{{.*}} : (index, index) -> !fir.shape<2>
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<100x10xf32>
 ! HLFIR:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref<!fir.array<100x10xf32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<100x10xf32>>, !fir.ref<!fir.array<100x10xf32>>)
 ! HLFIR:   acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.array<100x10xf32>>
 ! CHECK: } combiner {
@@ -430,8 +459,8 @@
 ! CHECK-LABEL: acc.reduction.recipe @reduction_min_section_ext100_ref_100xi32 : !fir.ref<!fir.array<100xi32>> reduction_operator <min> init {
 ! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.array<100xi32>>):
 ! CHECK:   %[[INIT:.*]] = arith.constant 2147483647 : i32
-! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<100xi32>
 ! HLFIR:   %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<100xi32>
 ! HLFIR:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
 ! HLFIR:   acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.array<100xi32>>
 ! CHECK: } combiner {
@@ -487,8 +516,8 @@
 ! CHECK-LABEL: acc.reduction.recipe @reduction_mul_section_ext100_ref_100xi32 : !fir.ref<!fir.array<100xi32>> reduction_operator <mul> init {
 ! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.array<100xi32>>):
 ! CHECK:   %[[INIT:.*]] = arith.constant 1 : i32
-! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<100xi32>
 ! HLFIR:   %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<100xi32>
 ! HLFIR:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
 ! HLFIR:   acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.array<100xi32>>
 ! CHECK: } combiner {
@@ -526,8 +555,8 @@
 ! CHECK-LABEL: acc.reduction.recipe @reduction_add_section_ext100_ref_100xf32 : !fir.ref<!fir.array<100xf32>> reduction_operator <add> init {
 ! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.array<100xf32>>):
 ! CHECK:   %[[INIT:.*]] = arith.constant 0.000000e+00 : f32
-! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<100xf32>
 ! HLFIR:   %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<100xf32>
 ! HLFIR:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref<!fir.array<100xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xf32>>, !fir.ref<!fir.array<100xf32>>)
 ! HLFIR:   acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.array<100xf32>>
 ! CHECK: } combiner {
@@ -565,8 +594,8 @@
 ! CHECK-LABEL: acc.reduction.recipe @reduction_add_section_ext100xext10xext2_ref_100x10x2xi32 : !fir.ref<!fir.array<100x10x2xi32>> reduction_operator <add> init {
 ! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.array<100x10x2xi32>>):
 ! CHECK:   %[[INIT:.*]] = arith.constant 0 : i32
-! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<100x10x2xi32>
 ! HLFIR:   %[[SHAPE:.*]] = fir.shape %{{.*}}, %{{.*}}, %{{.*}} : (index, index, index) -> !fir.shape<3>
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<100x10x2xi32>
 ! HLFIR:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref<!fir.array<100x10x2xi32>>, !fir.shape<3>) -> (!fir.ref<!fir.array<100x10x2xi32>>, !fir.ref<!fir.array<100x10x2xi32>>)
 ! HLFIR:   acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.array<100x10x2xi32>>
 ! CHECK: } combiner {
@@ -598,8 +627,8 @@
 ! CHECK-LABEL: acc.reduction.recipe @reduction_add_section_ext100xext10_ref_100x10xi32 : !fir.ref<!fir.array<100x10xi32>> reduction_operator <add> init {
 ! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.array<100x10xi32>>):
 ! CHECK:   %[[INIT:.*]] = arith.constant 0 : i32
-! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<100x10xi32>
 ! HLFIR:   %[[SHAPE:.*]] = fir.shape %{{.*}}, %{{.*}} : (index, index) -> !fir.shape<2>
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<100x10xi32>
 ! HLFIR:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref<!fir.array<100x10xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<100x10xi32>>, !fir.ref<!fir.array<100x10xi32>>)
 ! HLFIR:   acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.array<100x10xi32>>
 ! CHECK: } combiner {
@@ -626,8 +655,8 @@
 ! CHECK-LABEL: acc.reduction.recipe @reduction_add_section_ext100_ref_100xi32 : !fir.ref<!fir.array<100xi32>> reduction_operator <add> init {
 ! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.array<100xi32>>):
 ! CHECK:   %[[INIT:.*]] = arith.constant 0 : i32
-! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<100xi32>
 ! HLFIR:   %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<100xi32>
 ! HLFIR:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.reduction.init"} : (!fir.ref<!fir.array<100xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<100xi32>>, !fir.ref<!fir.array<100xi32>>)
 ! HFLIR:   acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.array<100xi32>>
 ! CHECK: } combiner {
@@ -1134,13 +1163,13 @@ subroutine acc_reduction_add_dynamic_extent_add(a)
 ! HLFIR: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<!fir.array<?xi32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<?xi32>> {name = "a"}
 ! HLFIR: acc.parallel reduction(@reduction_add_box_Uxi32 -> %[[RED:.*]] : !fir.ref<!fir.array<?xi32>>)
 
-subroutine acc_reduction_add_dynamic_extent_max(a)
+subroutine acc_reduction_add_assumed_shape_max(a)
   real :: a(:)
   !$acc parallel reduction(max:a)
   !$acc end parallel
 end subroutine
 
-! CHECK-LABEL: func.func @_QPacc_reduction_add_dynamic_extent_max(
+! CHECK-LABEL: func.func @_QPacc_reduction_add_assumed_shape_max(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "a"})
 ! HLFIR: %[[DECLARG0:.*]]:2 = hlfir.declare %[[ARG0]]
 ! HLFIR: %[[RED:.*]] = acc.reduction varPtr(%{{.*}} : !fir.ref<!fir.array<?xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<?xf32>> {name = "a"}
@@ -1189,3 +1218,16 @@ subroutine acc_reduction_add_pointer_array(a)
 ! HLFIR: %[[BOX_ADDR:.*]] = fir.box_addr %[[BOX]] : (!fir.box<!fir.ptr<!fir.array<?xf32>>>) -> !fir.ptr<!fir.array<?xf32>>
 ! HLFIR: %[[RED:.*]] = acc.reduction varPtr(%[[BOX_ADDR]] : !fir.ptr<!fir.array<?xf32>>) bounds(%[[BOUND]]) -> !fir.ptr<!fir.array<?xf32>> {name = "a"}
 ! HLFIR: acc.parallel reduction(@reduction_max_box_ptr_Uxf32 -> %[[RED]] : !fir.ptr<!fir.array<?xf32>>)
+
+subroutine acc_reduction_max_dynamic_extent_max(a, n)
+  integer :: n
+  real :: a(n, n)
+  !$acc parallel reduction(max:a)
+  !$acc end parallel
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_max_dynamic_extent_max(
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.array<?x?xf32>> {fir.bindc_name = "a"}, %{{.*}}: !fir.ref<i32> {fir.bindc_name = "n"})
+! HLFIR: %[[DECL_A:.*]]:2 = hlfir.declare %[[ARG0]](%{{.*}}) {uniq_name = "_QFacc_reduction_max_dynamic_extent_maxEa"} : (!fir.ref<!fir.array<?x?xf32>>, !fir.shape<2>) -> (!fir.box<!fir.array<?x?xf32>>, !fir.ref<!fir.array<?x?xf32>>)
+! HLFIR: %[[RED:.*]] = acc.reduction varPtr(%[[DECL_A]]#1 : !fir.ref<!fir.array<?x?xf32>>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref<!fir.array<?x?xf32>> {name = "a"}
+! HLFIR: acc.parallel reduction(@reduction_max_ref_UxUxf32 -> %[[RED]] : !fir.ref<!fir.array<?x?xf32>>)
diff --git a/flang/test/Lower/OpenACC/acc-routine.f90 b/flang/test/Lower/OpenACC/acc-routine.f90
index 7514e0a881..ffa8897309 100644
--- a/flang/test/Lower/OpenACC/acc-routine.f90
+++ b/flang/test/Lower/OpenACC/acc-routine.f90
@@ -96,3 +96,14 @@ subroutine acc_routine11(a)
   end interface
 
 end subroutine
+
+subroutine acc_routine13()
+  !$acc routine bind(acc_routine14)
+end subroutine
+
+subroutine acc_routine14()
+end subroutine
+
+subroutine acc_routine15()
+  !$acc routine bind(acc_routine16)
+end subroutine
diff --git a/flang/test/Lower/OpenACC/acc-serial-loop.f90 b/flang/test/Lower/OpenACC/acc-serial-loop.f90
index 466c679320..fb7e3615b6 100644
--- a/flang/test/Lower/OpenACC/acc-serial-loop.f90
+++ b/flang/test/Lower/OpenACC/acc-serial-loop.f90
@@ -3,22 +3,23 @@
 ! RUN: bbc -fopenacc -emit-fir %s -o - | FileCheck %s --check-prefixes=CHECK,FIR
 ! RUN: bbc -fopenacc -emit-hlfir %s -o - | FileCheck %s --check-prefixes=CHECK,HLFIR
 
-! CHECK-LABEL: acc.firstprivate.recipe @firstprivatization_section_ext10_ref_10xf32 : !fir.ref<!fir.array<10xf32>> init {
+! CHECK-LABEL: acc.private.recipe @privatization_ref_10xf32 : !fir.ref<!fir.array<10xf32>> init {
 ! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.array<10xf32>>):
 ! HLFIR:   %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
-! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<10xf32>
+! HLFIR:   %[[ALLOCA:.*]] = fir.alloca !fir.array<10xf32>
 ! HLFIR:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.private.init"} : (!fir.ref<!fir.array<10xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
 ! HLFIR:   acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.array<10xf32>>
-! CHECK: } copy {
-! CHECK:  ^bb0(%arg0: !fir.ref<!fir.array<10xf32>>, %arg1: !fir.ref<!fir.array<10xf32>>):
-! CHECK:   acc.terminator
 ! CHECK: }
 
-! CHECK-LABEL: acc.private.recipe @privatization_ref_10xf32 : !fir.ref<!fir.array<10xf32>> init {
+! CHECK-LABEL: acc.firstprivate.recipe @firstprivatization_section_ext10_ref_10xf32 : !fir.ref<!fir.array<10xf32>> init {
 ! CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.array<10xf32>>):
 ! HLFIR:   %[[SHAPE:.*]] = fir.shape %{{.*}} : (index) -> !fir.shape<1>
+! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.array<10xf32>
 ! HLFIR:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]](%[[SHAPE]]) {uniq_name = "acc.private.init"} : (!fir.ref<!fir.array<10xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<10xf32>>, !fir.ref<!fir.array<10xf32>>)
 ! HLFIR:   acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.array<10xf32>>
+! CHECK: } copy {
+! CHECK:  ^bb0(%arg0: !fir.ref<!fir.array<10xf32>>, %arg1: !fir.ref<!fir.array<10xf32>>):
+! CHECK:   acc.terminator
 ! CHECK: }
 
 ! CHECK-LABEL: func.func @_QPacc_serial_loop()
@@ -417,11 +418,9 @@ subroutine acc_serial_loop
     a(i) = b(i)
   END DO
 
-! FIR:        %[[ACC_PRIVATE_A:.*]] = acc.private varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
-! HLFIR:      %[[ACC_PRIVATE_A:.*]] = acc.private varPtr(%[[DECLA]]#1 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
 ! FIR:        %[[ACC_FPRIVATE_B:.*]] = acc.firstprivate varPtr(%[[B]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
 ! HLFIR:      %[[ACC_FPRIVATE_B:.*]] = acc.firstprivate varPtr(%[[DECLB]]#1 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "b"}
-! CHECK:      acc.serial firstprivate(@firstprivatization_section_ext10_ref_10xf32 -> %[[ACC_FPRIVATE_B]] : !fir.ref<!fir.array<10xf32>>) private(@privatization_ref_10xf32 -> %[[ACC_PRIVATE_A]] : !fir.ref<!fir.array<10xf32>>) {
+! CHECK:      acc.serial firstprivate(@firstprivatization_section_ext10_ref_10xf32 -> %[[ACC_FPRIVATE_B]] : !fir.ref<!fir.array<10xf32>>) {
 ! FIR:        %[[ACC_PRIVATE_A:.*]] = acc.private varPtr(%[[A]] : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
 ! HLFIR:      %[[ACC_PRIVATE_A:.*]] = acc.private varPtr(%[[DECLA]]#1 : !fir.ref<!fir.array<10xf32>>) bounds(%{{.*}}) -> !fir.ref<!fir.array<10xf32>> {name = "a"}
 ! CHECK:        acc.loop private(@privatization_ref_10xf32 -> %[[ACC_PRIVATE_A]] : !fir.ref<!fir.array<10xf32>>) {
@@ -706,7 +705,7 @@ subroutine acc_serial_loop
     reduction_i = 1
   end do
 
-! CHECK:      acc.serial reduction(@reduction_add_ref_f32 -> %{{.*}} : !fir.ref<f32>, @reduction_mul_ref_i32 -> %{{.*}} : !fir.ref<i32>) {
+! CHECK:      acc.serial {
 ! CHECK:        acc.loop reduction(@reduction_add_ref_f32 -> %{{.*}} : !fir.ref<f32>, @reduction_mul_ref_i32 -> %{{.*}} : !fir.ref<i32>) {
 ! CHECK:          fir.do_loop
 ! CHECK:          acc.yield
diff --git a/flang/test/Lower/OpenMP/FIR/atomic-read.f90 b/flang/test/Lower/OpenMP/FIR/atomic-read.f90
index ff2b651953..0079b347fa 100644
--- a/flang/test/Lower/OpenMP/FIR/atomic-read.f90
+++ b/flang/test/Lower/OpenMP/FIR/atomic-read.f90
@@ -14,7 +14,7 @@
 !CHECK: %[[VAR_X:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFEx"}
 !CHECK: %[[VAR_Y:.*]] = fir.alloca i32 {bindc_name = "y", uniq_name = "_QFEy"}
 !CHECK: omp.atomic.read %[[VAR_X]] = %[[VAR_Y]] memory_order(acquire)  hint(uncontended) : !fir.ref<i32>, i32
-!CHECK: omp.atomic.read %[[VAR_A]] = %[[VAR_B]] memory_order(relaxed) hint(none)  : !fir.ref<!fir.char<1>>, !fir.char<1>
+!CHECK: omp.atomic.read %[[VAR_A]] = %[[VAR_B]] memory_order(relaxed) : !fir.ref<!fir.char<1>>, !fir.char<1>
 !CHECK: omp.atomic.read %[[VAR_C]] = %[[VAR_D]] memory_order(seq_cst)  hint(contended) : !fir.ref<!fir.logical<4>>, !fir.logical<4>
 !CHECK: omp.atomic.read %[[VAR_E]] = %[[VAR_F]] hint(speculative) : !fir.ref<!fir.char<1,8>>, !fir.char<1,8>
 !CHECK: omp.atomic.read %[[VAR_G]] = %[[VAR_H]] hint(nonspeculative) : !fir.ref<f32>, f32
diff --git a/flang/test/Lower/OpenMP/FIR/critical.f90 b/flang/test/Lower/OpenMP/FIR/critical.f90
index c6ac818fe2..b86729f8a9 100644
--- a/flang/test/Lower/OpenMP/FIR/critical.f90
+++ b/flang/test/Lower/OpenMP/FIR/critical.f90
@@ -2,7 +2,7 @@
 !RUN: %flang_fc1 -emit-fir -fopenmp %s -o - | fir-opt --fir-to-llvm-ir | FileCheck %s --check-prefix="OMPDialect"
 !RUN: %flang_fc1 -emit-fir -fopenmp %s -o - | fir-opt --fir-to-llvm-ir | tco | FileCheck %s --check-prefix="LLVMIR"
 
-!OMPDialect: omp.critical.declare @help2 hint(none)
+!OMPDialect: omp.critical.declare @help2
 !OMPDialect: omp.critical.declare @help1 hint(contended)
 
 subroutine omp_critical()
diff --git a/flang/test/Lower/OpenMP/FIR/parallel-private-clause-fixes.f90 b/flang/test/Lower/OpenMP/FIR/parallel-private-clause-fixes.f90
index 3152f9c44d..beb94bfc91 100644
--- a/flang/test/Lower/OpenMP/FIR/parallel-private-clause-fixes.f90
+++ b/flang/test/Lower/OpenMP/FIR/parallel-private-clause-fixes.f90
@@ -7,9 +7,9 @@
 ! CHECK:         %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "j", uniq_name = "_QFmultiple_private_fixEj"}
 ! CHECK:         %[[VAL_2:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFmultiple_private_fixEx"}
 ! CHECK:         omp.parallel {
-! CHECK:           %[[PRIV_J:.*]] = fir.alloca i32 {bindc_name = "j", pinned
-! CHECK:           %[[PRIV_I:.*]] = fir.alloca i32 {adapt.valuebyref, pinned
-! CHECK:           %[[PRIV_X:.*]] = fir.alloca i32 {bindc_name = "x", pinned
+! CHECK-DAG:           %[[PRIV_J:.*]] = fir.alloca i32 {bindc_name = "j", pinned
+! CHECK-DAG:           %[[PRIV_I:.*]] = fir.alloca i32 {adapt.valuebyref, pinned
+! CHECK-DAG:           %[[PRIV_X:.*]] = fir.alloca i32 {bindc_name = "x", pinned
 ! CHECK:           %[[ONE:.*]] = arith.constant 1 : i32
 ! CHECK:           %[[VAL_3:.*]] = fir.load %[[VAL_4:.*]] : !fir.ref<i32>
 ! CHECK:           %[[VAL_5:.*]] = arith.constant 1 : i32
diff --git a/flang/test/Lower/OpenMP/atomic-read.f90 b/flang/test/Lower/OpenMP/atomic-read.f90
new file mode 100644
index 0000000000..97a3777bd3
--- /dev/null
+++ b/flang/test/Lower/OpenMP/atomic-read.f90
@@ -0,0 +1,89 @@
+! RUN: bbc -fopenmp -emit-hlfir %s -o - | FileCheck %s
+
+! This test checks the lowering of atomic read
+
+!CHECK: func @_QQmain() attributes {fir.bindc_name = "ompatomic"} {
+!CHECK:    %[[A_C1:.*]] = arith.constant 1 : index
+!CHECK:    %[[A_REF:.*]] = fir.alloca !fir.char<1> {bindc_name = "a", uniq_name = "_QFEa"}
+!CHECK:    %[[A_DECL:.*]]:2 = hlfir.declare %[[A_REF]] typeparams %[[A_C1]] {uniq_name = "_QFEa"} : (!fir.ref<!fir.char<1>>, index) -> (!fir.ref<!fir.char<1>>, !fir.ref<!fir.char<1>>)
+!CHECK:    %[[B_C1:.*]] = arith.constant 1 : index
+!CHECK:    %[[B_REF:.*]] = fir.alloca !fir.char<1> {bindc_name = "b", uniq_name = "_QFEb"}
+!CHECK:    %[[B_DECL:.*]]:2 = hlfir.declare %[[B_REF]] typeparams %[[B_C1]] {uniq_name = "_QFEb"} : (!fir.ref<!fir.char<1>>, index) -> (!fir.ref<!fir.char<1>>, !fir.ref<!fir.char<1>>)
+!CHECK:    %[[C_REF:.*]] = fir.alloca !fir.logical<4> {bindc_name = "c", uniq_name = "_QFEc"}
+!CHECK:    %[[C_DECL:.*]]:2 = hlfir.declare %[[C_REF]] {uniq_name = "_QFEc"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+!CHECK:    %[[D_REF:.*]] = fir.alloca !fir.logical<4> {bindc_name = "d", uniq_name = "_QFEd"}
+!CHECK:    %[[D_DECL:.*]]:2 = hlfir.declare %[[D_REF]] {uniq_name = "_QFEd"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+!CHECK:    %[[E_C8:.*]] = arith.constant 8 : index
+!CHECK:    %[[E_REF:.*]] = fir.alloca !fir.char<1,8> {bindc_name = "e", uniq_name = "_QFEe"}
+!CHECK:    %[[E_DECL:.*]]:2 = hlfir.declare %[[E_REF]] typeparams %[[E_C8]] {uniq_name = "_QFEe"} : (!fir.ref<!fir.char<1,8>>, index) -> (!fir.ref<!fir.char<1,8>>, !fir.ref<!fir.char<1,8>>)
+!CHECK:    %[[F_C8:.*]] = arith.constant 8 : index
+!CHECK:    %[[F_REF:.*]] = fir.alloca !fir.char<1,8> {bindc_name = "f", uniq_name = "_QFEf"}
+!CHECK:    %[[F_DECL:.*]]:2 = hlfir.declare %[[F_REF]] typeparams %[[F_C8]] {uniq_name = "_QFEf"} : (!fir.ref<!fir.char<1,8>>, index) -> (!fir.ref<!fir.char<1,8>>, !fir.ref<!fir.char<1,8>>)
+!CHECK:    %[[G_REF:.*]] = fir.alloca f32 {bindc_name = "g", uniq_name = "_QFEg"}
+!CHECK:    %[[G_DECL:.*]]:2 = hlfir.declare %[[G_REF]] {uniq_name = "_QFEg"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+!CHECK:    %[[H_REF:.*]] = fir.alloca f32 {bindc_name = "h", uniq_name = "_QFEh"}
+!CHECK:    %[[H_DECL:.*]]:2 = hlfir.declare %[[H_REF]] {uniq_name = "_QFEh"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+!CHECK:    %[[X_REF:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFEx"}
+!CHECK:    %[[X_DECL:.*]]:2 = hlfir.declare %[[X_REF]] {uniq_name = "_QFEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK:    %[[Y_REF:.*]] = fir.alloca i32 {bindc_name = "y", uniq_name = "_QFEy"}
+!CHECK:    %[[Y_DECL:.*]]:2 = hlfir.declare %[[Y_REF]] {uniq_name = "_QFEy"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK:    omp.atomic.read %[[X_DECL]]#1 = %[[Y_DECL]]#1   memory_order(acquire) hint(uncontended) : !fir.ref<i32>, i32
+!CHECK:    omp.atomic.read %[[A_DECL]]#1 = %[[B_DECL]]#1   memory_order(relaxed) : !fir.ref<!fir.char<1>>, !fir.char<1>
+!CHECK:    omp.atomic.read %[[C_DECL]]#1 = %[[D_DECL]]#1   memory_order(seq_cst) hint(contended) : !fir.ref<!fir.logical<4>>, !fir.logical<4>
+!CHECK:    omp.atomic.read %[[E_DECL]]#1 = %[[F_DECL]]#1   hint(speculative) : !fir.ref<!fir.char<1,8>>, !fir.char<1,8>
+!CHECK:    omp.atomic.read %[[G_DECL]]#1 = %[[H_DECL]]#1   hint(nonspeculative) : !fir.ref<f32>, f32
+!CHECK:    omp.atomic.read %[[G_DECL]]#1 = %[[H_DECL]]#1   : !fir.ref<f32>, f32
+
+program OmpAtomic
+
+    use omp_lib
+    integer :: x, y
+    character :: a, b
+    logical :: c, d
+    character(8) :: e, f
+    real g, h
+    !$omp atomic acquire read hint(omp_sync_hint_uncontended)
+       x = y
+    !$omp atomic relaxed read hint(omp_sync_hint_none)
+       a = b
+    !$omp atomic read seq_cst hint(omp_sync_hint_contended)
+       c = d
+    !$omp atomic read hint(omp_sync_hint_speculative)
+       e = f
+    !$omp atomic read hint(omp_sync_hint_nonspeculative)
+       g = h
+    !$omp atomic read
+       g = h
+end program OmpAtomic
+
+! Test lowering atomic read for pointer variables.
+! Please notice to use %[[VAL_4]] and %[[VAL_1]] for operands of atomic
+! operation, instead of %[[VAL_3]] and %[[VAL_0]].
+
+!CHECK-LABEL: func.func @_QPatomic_read_pointer() {
+!CHECK:    %[[X_REF:.*]] = fir.alloca !fir.box<!fir.ptr<i32>> {bindc_name = "x", uniq_name = "_QFatomic_read_pointerEx"}
+!CHECK:    fir.store %2 to %0 : !fir.ref<!fir.box<!fir.ptr<i32>>>
+!CHECK:    %[[X_DECL:.*]]:2 = hlfir.declare %[[X_REF]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFatomic_read_pointerEx"} : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>)
+!CHECK:    %[[Y_REF:.*]] = fir.alloca !fir.box<!fir.ptr<i32>> {bindc_name = "y", uniq_name = "_QFatomic_read_pointerEy"}
+!CHECK:    %[[Y_DECL:.*]]:2 = hlfir.declare %[[Y_REF]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFatomic_read_pointerEy"} : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>)
+!CHECK:    %[[X_ADDR:.*]] = fir.load %[[X_DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<i32>>>
+!CHECK:    %[[X_POINTEE_ADDR:.*]] = fir.box_addr %[[X_ADDR]] : (!fir.box<!fir.ptr<i32>>) -> !fir.ptr<i32>
+!CHECK:    %[[Y_ADDR:.*]] = fir.load %[[Y_DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<i32>>>
+!CHECK:    %[[Y_POINTEE_ADDR:.*]] = fir.box_addr %[[Y_ADDR]] : (!fir.box<!fir.ptr<i32>>) -> !fir.ptr<i32>
+!CHECK:    omp.atomic.read %[[Y_POINTEE_ADDR]] = %[[X_POINTEE_ADDR]]   : !fir.ptr<i32>, i32
+!CHECK:    %[[Y_ADDR:.*]] = fir.load %[[Y_DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<i32>>>
+!CHECK:    %[[Y_POINTEE_ADDR:.*]] = fir.box_addr %[[Y_ADDR]] : (!fir.box<!fir.ptr<i32>>) -> !fir.ptr<i32>
+!CHECK:    %[[Y_POINTEE_VAL:.*]] = fir.load %[[Y_POINTEE_ADDR]] : !fir.ptr<i32>
+!CHECK:    %[[X_ADDR:.*]] = fir.load %[[X_DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<i32>>>
+!CHECK:    %[[X_POINTEE_ADDR:.*]] = fir.box_addr %[[X_ADDR]] : (!fir.box<!fir.ptr<i32>>
+!CHECK:    hlfir.assign %[[Y_POINTEE_VAL]] to %[[X_POINTEE_ADDR]] : i32, !fir.ptr<i32>
+
+subroutine atomic_read_pointer()
+  integer, pointer :: x, y
+
+  !$omp atomic read
+    y = x
+
+  x = y
+end
+
diff --git a/flang/test/Lower/OpenMP/atomic-write.f90 b/flang/test/Lower/OpenMP/atomic-write.f90
new file mode 100644
index 0000000000..119f60c1a9
--- /dev/null
+++ b/flang/test/Lower/OpenMP/atomic-write.f90
@@ -0,0 +1,73 @@
+! RUN: bbc -fopenmp -emit-hlfir %s -o - | FileCheck %s
+
+! This test checks the lowering of atomic write
+
+!CHECK: func @_QQmain() attributes {fir.bindc_name = "ompatomicwrite"} {
+!CHECK:    %[[X_REF:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFEx"}
+!CHECK:    %[[X_DECL:.*]]:2 = hlfir.declare %[[X_REF]] {uniq_name = "_QFEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK:    %[[Y_REF:.*]] = fir.alloca i32 {bindc_name = "y", uniq_name = "_QFEy"}
+!CHECK:    %[[Y_DECL:.*]]:2 = hlfir.declare %[[Y_REF]] {uniq_name = "_QFEy"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK:    %[[Z_REF:.*]] = fir.alloca i32 {bindc_name = "z", uniq_name = "_QFEz"}
+!CHECK:    %[[Z_DECL:.*]]:2 = hlfir.declare %[[Z_REF]] {uniq_name = "_QFEz"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK:    %[[C44:.*]] = arith.constant 44 : i32
+!CHECK:    omp.atomic.write %[[X_DECL:.*]]#1 = %[[C44]]   hint(uncontended) memory_order(seq_cst) : !fir.ref<i32>, i32
+!CHECK:    %[[C7:.*]] = arith.constant 7 : i32
+!CHECK:    %[[Y_VAL:.*]] = fir.load %[[Y_DECL]]#0 : !fir.ref<i32>
+!CHECK:    %[[SEVEN_Y_VAL:.*]] = arith.muli %[[C7]], %[[Y_VAL]] : i32
+!CHECK:    omp.atomic.write %[[X_DECL]]#1 = %[[SEVEN_Y_VAL]]   memory_order(relaxed) : !fir.ref<i32>, i32
+!CHECK:    %[[C10:.*]] = arith.constant 10 : i32
+!CHECK:    %[[X_VAL:.*]] = fir.load %[[X_DECL]]#0 : !fir.ref<i32>
+!CHECK:    %[[TEN_X:.*]] = arith.muli %[[C10]], %[[X_VAL]] : i32
+!CHECK:    %[[Z_VAL:.*]] = fir.load %[[Z_DECL]]#0 : !fir.ref<i32>
+!CHECK:    %[[C2:.*]] = arith.constant 2 : i32
+!CHECK:    %[[Z_DIV_2:.*]] = arith.divsi %[[Z_VAL]], %[[C2]] : i32
+!CHECK:    %172 = arith.addi %[[TEN_X]], %[[Z_DIV_2]] : i32
+!CHECK:    omp.atomic.write %163#1 = %172   hint(speculative) memory_order(release) : !fir.ref<i32>, i32
+
+program OmpAtomicWrite
+    use omp_lib
+    integer :: x, y, z
+    !$omp atomic seq_cst write hint(omp_sync_hint_uncontended)
+        x = 8*4 + 12
+
+    !$omp atomic write relaxed
+        x = 7 * y
+
+    !$omp atomic write release hint(omp_sync_hint_speculative)
+        y = 10*x + z/2
+end program OmpAtomicWrite
+
+! Test lowering atomic read for pointer variables.
+
+!CHECK-LABEL: func.func @_QPatomic_write_pointer() {
+!CHECK:    %[[X_REF:.*]] = fir.alloca !fir.box<!fir.ptr<i32>> {bindc_name = "x", uniq_name = "_QFatomic_write_pointerEx"}
+!CHECK:    %[[X_DECL:.*]]:2 = hlfir.declare %[[X_REF]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFatomic_write_pointerEx"} : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> (!fir.ref<!fir.box<!fir.ptr<i32>>>, !fir.ref<!fir.box<!fir.ptr<i32>>>)
+!CHECK:    %[[C1:.*]] = arith.constant 1 : i32
+!CHECK:    %[[X_ADDR_BOX:.*]] = fir.load %[[X_DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<i32>>>
+!CHECK:    %[[X_POINTEE_ADDR:.*]] = fir.box_addr %[[X_ADDR_BOX]] : (!fir.box<!fir.ptr<i32>>) -> !fir.ptr<i32>
+!CHECK:    omp.atomic.write %[[X_POINTEE_ADDR]] = %[[C1]]   : !fir.ptr<i32>, i32
+!CHECK:    %[[C2:.*]] = arith.constant 2 : i32
+!CHECK:    %[[X_ADDR_BOX:.*]] = fir.load %[[X_DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<i32>>>
+!CHECK:    %[[X_POINTEE_ADDR:.*]] = fir.box_addr %[[X_ADDR_BOX]] : (!fir.box<!fir.ptr<i32>>) -> !fir.ptr<i32>
+!CHECK:    hlfir.assign %[[C2]] to %[[X_POINTEE_ADDR]] : i32, !fir.ptr<i32>
+
+subroutine atomic_write_pointer()
+  integer, pointer :: x
+
+  !$omp atomic write
+    x = 1
+
+  x = 2
+end
+
+!CHECK-LABEL: func.func @_QPatomic_write_typed_assign
+!CHECK:    %[[R2_REF:.*]] = fir.alloca f32 {bindc_name = "r2", uniq_name = "_QFatomic_write_typed_assignEr2"}
+!CHECK:    %[[R2_DECL:.*]]:2 = hlfir.declare %[[R2_REF]] {uniq_name = "_QFatomic_write_typed_assignEr2"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+!CHECK:    %[[C0:.*]] = arith.constant 0.000000e+00 : f32
+!CHECK:    omp.atomic.write %[[R2_DECL]]#1 = %[[C0]]   : !fir.ref<f32>, f32
+
+subroutine atomic_write_typed_assign
+  real :: r2
+  !$omp atomic write
+  r2 = 0
+end subroutine
diff --git a/flang/test/Lower/OpenMP/critical.f90 b/flang/test/Lower/OpenMP/critical.f90
index 9fbd172df9..5a4d2e4815 100644
--- a/flang/test/Lower/OpenMP/critical.f90
+++ b/flang/test/Lower/OpenMP/critical.f90
@@ -1,6 +1,6 @@
 !RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
 
-!CHECK: omp.critical.declare @help2 hint(none)
+!CHECK: omp.critical.declare @help2
 !CHECK: omp.critical.declare @help1 hint(contended)
 
 subroutine omp_critical()
diff --git a/flang/test/Lower/OpenMP/firstprivate-commonblock.f90 b/flang/test/Lower/OpenMP/firstprivate-commonblock.f90
new file mode 100644
index 0000000000..d0fcdac76a
--- /dev/null
+++ b/flang/test/Lower/OpenMP/firstprivate-commonblock.f90
@@ -0,0 +1,34 @@
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+
+!CHECK: func.func @_QPfirstprivate_common() {
+!CHECK: %[[val_0:.*]] = fir.address_of(@c_) : !fir.ref<!fir.array<8xi8>>
+!CHECK: %[[val_1:.*]] = fir.convert %[[val_0]] : (!fir.ref<!fir.array<8xi8>>) -> !fir.ref<!fir.array<?xi8>>
+!CHECK: %[[val_c0:.*]] = arith.constant 0 : index
+!CHECK: %[[val_2:.*]] = fir.coordinate_of %[[val_1]], %[[val_c0]] : (!fir.ref<!fir.array<?xi8>>, index) -> !fir.ref<i8>
+!CHECK: %[[val_3:.*]] = fir.convert %[[val_2]] : (!fir.ref<i8>) -> !fir.ref<f32>
+!CHECK: %[[VAL_3_DECL:.*]]:2 = hlfir.declare %[[val_3]] {uniq_name = "_QFfirstprivate_commonEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+!CHECK: %[[val_4:.*]] = fir.convert %[[val_0]] : (!fir.ref<!fir.array<8xi8>>) -> !fir.ref<!fir.array<?xi8>>
+!CHECK: %[[val_c4:.*]] = arith.constant 4 : index
+!CHECK: %[[val_5:.*]] = fir.coordinate_of %[[val_4]], %[[val_c4]] : (!fir.ref<!fir.array<?xi8>>, index) -> !fir.ref<i8>
+!CHECK: %[[val_6:.*]] = fir.convert %[[val_5]] : (!fir.ref<i8>) -> !fir.ref<f32>
+!CHECK: %[[VAL_6_DECL:.*]]:2 = hlfir.declare %[[val_6]] {uniq_name = "_QFfirstprivate_commonEy"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+!CHECK: omp.parallel {
+!CHECK: %[[val_7:.*]] = fir.alloca f32 {bindc_name = "x", pinned, uniq_name = "_QFfirstprivate_commonEx"}
+!CHECK: %[[VAL_7_DECL:.*]]:2 = hlfir.declare %[[val_7]] {uniq_name = "_QFfirstprivate_commonEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+!CHECK: %[[val_8:.*]] = fir.load %[[VAL_3_DECL]]#0 : !fir.ref<f32>
+!CHECK: hlfir.assign %[[val_8]] to %[[VAL_7_DECL]]#0 temporary_lhs : f32, !fir.ref<f32>
+!CHECK: %[[val_9:.*]] = fir.alloca f32 {bindc_name = "y", pinned, uniq_name = "_QFfirstprivate_commonEy"}
+!CHECK: %[[VAL_9_DECL:.*]]:2 = hlfir.declare %[[val_9]] {uniq_name = "_QFfirstprivate_commonEy"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+!CHECK: %[[val_10:.*]] = fir.load %[[VAL_6_DECL]]#0 : !fir.ref<f32>
+!CHECK: hlfir.assign %[[val_10]] to %[[VAL_9_DECL]]#0 temporary_lhs : f32, !fir.ref<f32>
+!CHECK: omp.terminator
+!CHECK: }
+!CHECK: return
+!CHECK: }
+
+subroutine firstprivate_common
+  common /c/ x, y
+  real x, y
+  !$omp parallel firstprivate(/c/)
+  !$omp end parallel
+end subroutine
diff --git a/flang/test/Lower/OpenMP/lastprivate-commonblock.f90 b/flang/test/Lower/OpenMP/lastprivate-commonblock.f90
index 15b76bb7bc..06fa0a1210 100644
--- a/flang/test/Lower/OpenMP/lastprivate-commonblock.f90
+++ b/flang/test/Lower/OpenMP/lastprivate-commonblock.f90
@@ -18,10 +18,10 @@
 !CHECK:    omp.wsloop   for  (%[[I:.*]]) : i32 = (%{{.*}}) to (%{{.*}}) inclusive step (%{{.*}}) {
 !CHECK:      %[[LAST_ITER:.*]] = arith.cmpi eq, %[[I]], %{{.*}} : i32
 !CHECK:      fir.if %[[LAST_ITER]] {
-!CHECK:        %[[PRIVATE_X_VAL:.*]] = fir.load %[[PRIVATE_X_DECL]]#1 : !fir.ref<f32>
-!CHECK:        fir.store %[[PRIVATE_X_VAL]] to %[[X_DECL]]#1 : !fir.ref<f32>
-!CHECK:        %[[PRIVATE_Y_VAL:.*]] = fir.load %[[PRIVATE_Y_DECL]]#1 : !fir.ref<f32>
-!CHECK:        fir.store %[[PRIVATE_Y_VAL]] to %[[Y_DECL]]#1 : !fir.ref<f32>
+!CHECK:        %[[PRIVATE_X_VAL:.*]] = fir.load %[[PRIVATE_X_DECL]]#0 : !fir.ref<f32>
+!CHECK:        hlfir.assign %[[PRIVATE_X_VAL]] to %[[X_DECL]]#0 temporary_lhs : f32, !fir.ref<f32>
+!CHECK:        %[[PRIVATE_Y_VAL:.*]] = fir.load %[[PRIVATE_Y_DECL]]#0 : !fir.ref<f32>
+!CHECK:        hlfir.assign %[[PRIVATE_Y_VAL]] to %[[Y_DECL]]#0 temporary_lhs : f32, !fir.ref<f32>
 !CHECK:      }
 !CHECK:      omp.yield
 !CHECK:    }
diff --git a/flang/test/Lower/OpenMP/parallel-private-clause-fixes.f90 b/flang/test/Lower/OpenMP/parallel-private-clause-fixes.f90
new file mode 100644
index 0000000000..d713129675
--- /dev/null
+++ b/flang/test/Lower/OpenMP/parallel-private-clause-fixes.f90
@@ -0,0 +1,92 @@
+! This test checks a few bug fixes in the PRIVATE clause lowering
+
+! RUN: bbc -fopenmp -emit-hlfir %s -o - | FileCheck %s
+
+! CHECK-LABEL: multiple_private_fix
+! CHECK-SAME:  %[[GAMA:.*]]: !fir.ref<i32> {fir.bindc_name = "gama"}
+! CHECK:         %[[GAMA_DECL:.*]]:2 = hlfir.declare %[[GAMA]] {uniq_name = "_QFmultiple_private_fixEgama"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:         %[[VAL_0:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFmultiple_private_fixEi"}
+! CHECK:         %[[I_DECL:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFmultiple_private_fixEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:         %[[VAL_1:.*]] = fir.alloca i32 {bindc_name = "j", uniq_name = "_QFmultiple_private_fixEj"}
+! CHECK:         %[[J_DECL:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QFmultiple_private_fixEj"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:         %[[VAL_2:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFmultiple_private_fixEx"}
+! CHECK:         %[[X_DECL:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFmultiple_private_fixEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:         omp.parallel {
+! CHECK:           %[[PRIV_I:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+! CHECK:           %[[PRIV_I_DECL:.*]]:2 = hlfir.declare %[[PRIV_I]] {uniq_name = "_QFmultiple_private_fixEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[PRIV_J:.*]] = fir.alloca i32 {bindc_name = "j", pinned, uniq_name = "_QFmultiple_private_fixEj"}
+! CHECK:           %[[PRIV_J_DECL:.*]]:2 = hlfir.declare %[[PRIV_J]] {uniq_name = "_QFmultiple_private_fixEj"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[PRIV_X:.*]] = fir.alloca i32 {bindc_name = "x", pinned
+! CHECK:           %[[PRIV_X_DECL:.*]]:2 = hlfir.declare %[[PRIV_X]] {uniq_name = "_QFmultiple_private_fixEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[ONE:.*]] = arith.constant 1 : i32
+! CHECK:           %[[VAL_3:.*]] = fir.load %[[GAMA_DECL]]#0 : !fir.ref<i32>
+! CHECK:           %[[VAL_5:.*]] = arith.constant 1 : i32
+! CHECK:           omp.wsloop for (%[[VAL_6:.*]]) : i32 = (%[[ONE]]) to (%[[VAL_3]]) inclusive step (%[[VAL_5]]) {
+! CHECK:             fir.store %[[VAL_6]] to %[[PRIV_I_DECL]]#1 : !fir.ref<i32>
+! CHECK:             %[[VAL_7:.*]] = arith.constant 1 : i32
+! CHECK:             %[[VAL_8:.*]] = fir.convert %[[VAL_7]] : (i32) -> index
+! CHECK:             %[[VAL_9:.*]] = fir.load %[[GAMA_DECL]]#0 : !fir.ref<i32>
+! CHECK:             %[[VAL_10:.*]] = fir.convert %[[VAL_9]] : (i32) -> index
+! CHECK:             %[[VAL_11:.*]] = arith.constant 1 : index
+! CHECK:             %[[LB:.*]] = fir.convert %[[VAL_8]] : (index) -> i32
+! CHECK:             %[[VAL_12:.*]]:2 = fir.do_loop %[[VAL_13:[^ ]*]] =
+! CHECK-SAME:            %[[VAL_8]] to %[[VAL_10]] step %[[VAL_11]]
+! CHECK-SAME:            iter_args(%[[IV:.*]] = %[[LB]]) -> (index, i32) {
+! CHECK:               fir.store %[[IV]] to %[[PRIV_J_DECL]]#1 : !fir.ref<i32>
+! CHECK:               %[[LOAD:.*]] = fir.load %[[PRIV_I_DECL]]#0 : !fir.ref<i32>
+! CHECK:               %[[VAL_15:.*]] = fir.load %[[PRIV_J_DECL]]#0 : !fir.ref<i32>
+! CHECK:               %[[VAL_16:.*]] = arith.addi %[[LOAD]], %[[VAL_15]] : i32
+! CHECK:               hlfir.assign %[[VAL_16]] to %[[PRIV_X_DECL]]#0 : i32, !fir.ref<i32>
+! CHECK:               %[[VAL_17:.*]] = arith.addi %[[VAL_13]], %[[VAL_11]] : index
+! CHECK:               %[[STEPCAST:.*]] = fir.convert %[[VAL_11]] : (index) -> i32
+! CHECK:               %[[IVLOAD:.*]] = fir.load %[[PRIV_J_DECL]]#1 : !fir.ref<i32>
+! CHECK:               %[[IVINC:.*]] = arith.addi %[[IVLOAD]], %[[STEPCAST]]
+! CHECK:               fir.result %[[VAL_17]], %[[IVINC]] : index, i32
+! CHECK:             }
+! CHECK:             fir.store %[[VAL_12]]#1 to %[[PRIV_J_DECL]]#1 : !fir.ref<i32>
+! CHECK:             omp.yield
+! CHECK:           }
+! CHECK:           omp.terminator
+! CHECK:         }
+! CHECK:         return
+subroutine multiple_private_fix(gama)
+        integer :: i, j, x, gama
+!$OMP PARALLEL DO PRIVATE(j,x)
+        do i = 1, gama
+          do j = 1, gama
+            x = i + j
+          end do
+        end do
+!$OMP END PARALLEL DO
+end subroutine
+
+! CHECK-LABEL: multiple_private_fix2
+! CHECK:  %[[X1:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFmultiple_private_fix2Ex"}
+! CHECK:  %[[X1_DECL:.*]]:2 = hlfir.declare %[[X1]] {uniq_name = "_QFmultiple_private_fix2Ex"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:  omp.parallel  {
+! CHECK:    %[[X2:.*]] = fir.alloca i32 {bindc_name = "x", pinned, uniq_name = "_QFmultiple_private_fix2Ex"}
+! CHECK:    %[[X2_DECL:.*]]:2 = hlfir.declare %[[X2]] {uniq_name = "_QFmultiple_private_fix2Ex"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:    omp.parallel  {
+! CHECK:      %[[X3:.*]] = fir.alloca i32 {bindc_name = "x", pinned, uniq_name = "_QFmultiple_private_fix2Ex"}
+! CHECK:      %[[X3_DECL:.*]]:2 = hlfir.declare %[[X3]] {uniq_name = "_QFmultiple_private_fix2Ex"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:      %[[C3:.*]] = arith.constant 1 : i32
+! CHECK:      hlfir.assign %[[C3]] to %[[X3_DECL]]#0 : i32, !fir.ref<i32>
+! CHECK:      omp.terminator
+! CHECK:    }
+! CHECK:      %[[C2:.*]] = arith.constant 1 : i32
+! CHECK:      hlfir.assign %[[C2]] to %[[X2_DECL]]#0 : i32, !fir.ref<i32>
+! CHECK:    omp.terminator
+! CHECK:  }
+! CHECK:      %[[C1:.*]] = arith.constant 1 : i32
+! CHECK:      hlfir.assign %[[C1]] to %[[X1_DECL]]#0 : i32, !fir.ref<i32>
+! CHECK:  return
+subroutine multiple_private_fix2()
+   integer :: x
+   !$omp parallel private(x)
+   !$omp parallel private(x)
+      x = 1
+   !$omp end parallel
+      x = 1
+   !$omp end parallel
+      x = 1
+end subroutine
diff --git a/flang/test/Lower/OpenMP/parallel-reduction-add.f90 b/flang/test/Lower/OpenMP/parallel-reduction-add.f90
new file mode 100644
index 0000000000..81a93aebbd
--- /dev/null
+++ b/flang/test/Lower/OpenMP/parallel-reduction-add.f90
@@ -0,0 +1,101 @@
+! RUN: bbc -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+
+!CHECK-LABEL: omp.reduction.declare
+!CHECK-SAME: @[[RED_F32_NAME:.*]] : f32 init {
+!CHECK: ^bb0(%{{.*}}: f32):
+!CHECK:  %[[C0_1:.*]] = arith.constant 0.000000e+00 : f32
+!CHECK:  omp.yield(%[[C0_1]] : f32)
+!CHECK: } combiner {
+!CHECK: ^bb0(%[[ARG0:.*]]: f32, %[[ARG1:.*]]: f32):
+!CHECK:  %[[RES:.*]] = arith.addf %[[ARG0]], %[[ARG1]] {{.*}}: f32
+!CHECK:  omp.yield(%[[RES]] : f32)
+!CHECK: }
+
+!CHECK-LABEL: omp.reduction.declare
+!CHECK-SAME: @[[RED_I32_NAME:.*]] : i32 init {
+!CHECK: ^bb0(%{{.*}}: i32):
+!CHECK:  %[[C0_1:.*]] = arith.constant 0 : i32
+!CHECK:  omp.yield(%[[C0_1]] : i32)
+!CHECK: } combiner {
+!CHECK: ^bb0(%[[ARG0:.*]]: i32, %[[ARG1:.*]]: i32):
+!CHECK:  %[[RES:.*]] = arith.addi %[[ARG0]], %[[ARG1]] : i32
+!CHECK:  omp.yield(%[[RES]] : i32)
+!CHECK: }
+
+!CHECK-LABEL: func.func @_QPsimple_int_add
+!CHECK:  %[[IREF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFsimple_int_addEi"}
+!CHECK:  %[[I_DECL:.*]]:2 = hlfir.declare %[[IREF]] {uniq_name = "_QFsimple_int_addEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK:  %[[I_START:.*]] = arith.constant 0 : i32
+!CHECK:  hlfir.assign %[[I_START]] to %[[I_DECL]]#0 : i32, !fir.ref<i32>
+!CHECK:  omp.parallel   reduction(@[[RED_I32_NAME]] -> %[[I_DECL]]#0 : !fir.ref<i32>) {
+!CHECK:    %[[I_INCR:.*]] = arith.constant 1 : i32
+!CHECK:    omp.reduction %[[I_INCR]], %[[I_DECL]]#0 : i32, !fir.ref<i32>
+!CHECK:    omp.terminator
+!CHECK:  }
+!CHECK: return
+subroutine simple_int_add
+    integer :: i
+    i = 0
+
+    !$omp parallel reduction(+:i)
+    i = i + 1
+    !$omp end parallel
+
+    print *, i
+end subroutine
+
+!CHECK-LABEL: func.func @_QPsimple_real_add
+!CHECK:  %[[RREF:.*]] = fir.alloca f32 {bindc_name = "r", uniq_name = "_QFsimple_real_addEr"}
+!CHECK:  %[[R_DECL:.*]]:2 = hlfir.declare %[[RREF]] {uniq_name = "_QFsimple_real_addEr"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+!CHECK:  %[[R_START:.*]] = arith.constant 0.000000e+00 : f32
+!CHECK:  hlfir.assign %[[R_START]] to %[[R_DECL]]#0 : f32, !fir.ref<f32>
+!CHECK:  omp.parallel   reduction(@[[RED_F32_NAME]] -> %[[R_DECL]]#0 : !fir.ref<f32>) {
+!CHECK:    %[[R_INCR:.*]] = arith.constant 1.500000e+00 : f32
+!CHECK:    omp.reduction %[[R_INCR]], %[[R_DECL]]#0 : f32, !fir.ref<f32>
+!CHECK:    omp.terminator
+!CHECK:  }
+!CHECK: return
+subroutine simple_real_add
+    real :: r
+    r = 0.0
+
+    !$omp parallel reduction(+:r)
+    r = r + 1.5
+    !$omp end parallel
+
+    print *, r
+end subroutine
+
+!CHECK-LABEL: func.func @_QPint_real_add
+!CHECK:  %[[IREF:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFint_real_addEi"}
+!CHECK:  %[[I_DECL:.*]]:2 = hlfir.declare %[[IREF]] {uniq_name = "_QFint_real_addEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK:  %[[RREF:.*]] = fir.alloca f32 {bindc_name = "r", uniq_name = "_QFint_real_addEr"}
+!CHECK:  %[[R_DECL:.*]]:2 = hlfir.declare %[[RREF]] {uniq_name = "_QFint_real_addEr"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+!CHECK:  %[[R_START:.*]] = arith.constant 0.000000e+00 : f32
+!CHECK:  hlfir.assign %[[R_START]] to %[[R_DECL]]#0 : f32, !fir.ref<f32>
+!CHECK:  %[[I_START:.*]] = arith.constant 0 : i32
+!CHECK:  hlfir.assign %[[I_START]] to %[[I_DECL]]#0 : i32, !fir.ref<i32>
+!CHECK:  omp.parallel   reduction(@[[RED_I32_NAME]] -> %[[I_DECL]]#0 : !fir.ref<i32>, @[[RED_F32_NAME]] -> %[[R_DECL]]#0 : !fir.ref<f32>) {
+!CHECK:    %[[R_INCR:.*]] = arith.constant 1.500000e+00 : f32
+!CHECK:    omp.reduction %[[R_INCR]], %[[R_DECL]]#0 : f32, !fir.ref<f32>
+!CHECK:    %[[I_INCR:.*]] = arith.constant 3 : i32
+!CHECK:    omp.reduction %[[I_INCR]], %[[I_DECL]]#0 : i32, !fir.ref<i32>
+!CHECK:    omp.terminator
+!CHECK:  }
+!CHECK: return
+subroutine int_real_add
+    real :: r
+    integer :: i
+
+    r = 0.0
+    i = 0
+
+    !$omp parallel reduction(+:i,r)
+    r = 1.5 + r
+    i = i + 3
+    !$omp end parallel
+
+    print *, r
+    print *, i
+end subroutine
diff --git a/flang/test/Lower/OpenMP/parallel-sections.f90 b/flang/test/Lower/OpenMP/parallel-sections.f90
new file mode 100644
index 0000000000..77139e40ed
--- /dev/null
+++ b/flang/test/Lower/OpenMP/parallel-sections.f90
@@ -0,0 +1,58 @@
+!RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
+
+!===============================================================================
+! Parallel sections construct
+!===============================================================================
+
+!CHECK: func @_QPomp_parallel_sections
+subroutine omp_parallel_sections(x, y)
+  integer, intent(inout) :: x, y
+  !CHECK: omp.parallel {
+  !CHECK: omp.sections {
+  !$omp parallel sections
+    !CHECK: omp.section {
+    !$omp section
+      !CHECK: fir.load
+      !CHECK: arith.addi
+      !CHECK: hlfir.assign
+      x = x + 12
+      !CHECK: omp.terminator
+    !CHECK: omp.section {
+    !$omp section
+      !CHECK: fir.load
+      !CHECK: arith.subi
+      !CHECK: hlfir.assign
+      y = y - 5
+      !CHECK: omp.terminator
+  !CHECK: omp.terminator
+  !CHECK: omp.terminator
+  !$omp end parallel sections
+end subroutine omp_parallel_sections
+
+!===============================================================================
+! Parallel sections construct with allocate clause
+!===============================================================================
+
+!CHECK: func @_QPomp_parallel_sections
+subroutine omp_parallel_sections_allocate(x, y)
+  use omp_lib
+  integer, intent(inout) :: x, y
+  !CHECK: %[[allocator_1:.*]] = arith.constant 1 : i32
+  !CHECK: %[[allocator_2:.*]] = arith.constant 1 : i32
+  !CHECK: omp.parallel allocate(
+  !CHECK: %[[allocator_2]] : i32 -> %{{.*}} : !fir.ref<i32>) {
+  !CHECK: omp.sections allocate(
+  !CHECK: %[[allocator_1]] : i32 -> %{{.*}} : !fir.ref<i32>) {
+  !$omp parallel sections allocate(omp_high_bw_mem_alloc: x)
+    !CHECK: omp.section {
+    !$omp section
+      x = x + 12
+      !CHECK: omp.terminator
+    !CHECK: omp.section {
+    !$omp section
+      y = y + 5
+      !CHECK: omp.terminator
+  !CHECK: omp.terminator
+  !CHECK: omp.terminator
+  !$omp end parallel sections
+end subroutine omp_parallel_sections_allocate
diff --git a/flang/test/Lower/OpenMP/parallel-wsloop-firstpriv.f90 b/flang/test/Lower/OpenMP/parallel-wsloop-firstpriv.f90
new file mode 100644
index 0000000000..716a7d71bb
--- /dev/null
+++ b/flang/test/Lower/OpenMP/parallel-wsloop-firstpriv.f90
@@ -0,0 +1,69 @@
+! This test checks lowering of OpenMP parallel DO, with the loop bound being
+! a firstprivate variable
+
+! RUN: bbc -fopenmp -emit-hlfir %s -o - | FileCheck %s
+
+! CHECK: func @_QPomp_do_firstprivate(%[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "a"}) 
+subroutine omp_do_firstprivate(a)
+  ! CHECK: %[[ARG0_DECL:.*]]:2 = hlfir.declare %[[ARG0]] {uniq_name = "_QFomp_do_firstprivateEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  integer::a
+  integer::n
+  n = a+1
+  !$omp parallel do firstprivate(a)
+  ! CHECK:  omp.parallel {
+  ! CHECK: %[[I_PVT_REF:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+  ! CHECK: %[[I_PVT_DECL:.*]]:2 = hlfir.declare %[[I_PVT_REF]] {uniq_name = "_QFomp_do_firstprivateEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  ! CHECK: %[[A_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "a", pinned, uniq_name = "_QFomp_do_firstprivateEa"}
+  ! CHECK: %[[A_PVT_DECL:.*]]:2 = hlfir.declare %[[A_PVT_REF]] {uniq_name = "_QFomp_do_firstprivateEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  ! CHECK-NEXT: %[[LD:.*]] = fir.load %[[ARG0_DECL]]#0 : !fir.ref<i32>
+  ! CHECK-NEXT: hlfir.assign %[[LD]] to %[[A_PVT_DECL]]#0 temporary_lhs : i32, !fir.ref<i32>
+  ! CHECK: %[[LB:.*]] = arith.constant 1 : i32
+  ! CHECK-NEXT: %[[UB:.*]] = fir.load %[[A_PVT_DECL]]#0 : !fir.ref<i32>
+  ! CHECK-NEXT: %[[STEP:.*]] = arith.constant 1 : i32
+  ! CHECK-NEXT: omp.wsloop   for  (%[[ARG1:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]])
+  ! CHECK-NEXT: fir.store %[[ARG1]] to %[[I_PVT_DECL]]#1 : !fir.ref<i32>
+  ! CHECK-NEXT: fir.call @_QPfoo(%[[I_PVT_DECL]]#1, %[[A_PVT_DECL]]#1) {{.*}}: (!fir.ref<i32>, !fir.ref<i32>) -> ()
+  ! CHECK-NEXT: omp.yield
+    do i=1, a
+      call foo(i, a)
+    end do
+  !$omp end parallel do
+  !CHECK: fir.call @_QPbar(%[[ARG0_DECL]]#1) {{.*}}: (!fir.ref<i32>) -> ()
+  call bar(a)
+end subroutine omp_do_firstprivate
+
+! CHECK: func @_QPomp_do_firstprivate2(%[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "a"}, %[[ARG1:.*]]: !fir.ref<i32> {fir.bindc_name = "n"}) 
+subroutine omp_do_firstprivate2(a, n)
+  ! CHECK:  %[[ARG0_DECL:.*]]:2 = hlfir.declare %[[ARG0]] {uniq_name = "_QFomp_do_firstprivate2Ea"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  ! CHECK:  %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] {uniq_name = "_QFomp_do_firstprivate2En"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  integer::a
+  integer::n
+  n = a+1
+  !$omp parallel do firstprivate(a, n)
+  ! CHECK:  omp.parallel {
+  ! CHECK: %[[I_PVT_REF:.*]] = fir.alloca i32 {adapt.valuebyref, pinned}
+  ! CHECK: %[[I_PVT_DECL:.*]]:2 = hlfir.declare %[[I_PVT_REF]] {uniq_name = "_QFomp_do_firstprivate2Ei"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  ! CHECK: %[[A_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "a", pinned
+  ! CHECK: %[[A_PVT_DECL:.*]]:2 = hlfir.declare %[[A_PVT_REF]] {uniq_name = "_QFomp_do_firstprivate2Ea"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  ! CHECK: %[[LD:.*]] = fir.load %[[ARG0_DECL]]#0 : !fir.ref<i32>
+  ! CHECK: hlfir.assign %[[LD]] to %[[A_PVT_DECL]]#0 temporary_lhs : i32, !fir.ref<i32>
+  ! CHECK: %[[N_PVT_REF:.*]] = fir.alloca i32 {bindc_name = "n", pinned, uniq_name = "_QFomp_do_firstprivate2En"}
+  ! CHECK: %[[N_PVT_DECL:.*]]:2 = hlfir.declare %[[N_PVT_REF]] {uniq_name = "_QFomp_do_firstprivate2En"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  ! CHECK: %[[LD1:.*]] = fir.load %[[ARG1_DECL]]#0 : !fir.ref<i32>
+  ! CHECK: hlfir.assign %[[LD1]] to %[[N_PVT_DECL]]#0 temporary_lhs : i32, !fir.ref<i32>
+
+
+  ! CHECK: %[[LB:.*]] = fir.load %[[A_PVT_DECL]]#0 : !fir.ref<i32>
+  ! CHECK: %[[UB:.*]] = fir.load %[[N_PVT_DECL]]#0 : !fir.ref<i32>
+  ! CHECK: %[[STEP:.*]] = arith.constant 1 : i32
+  ! CHECK: omp.wsloop   for  (%[[ARG2:.*]]) : i32 = (%[[LB]]) to (%[[UB]]) inclusive step (%[[STEP]])
+  ! CHECK: fir.store %[[ARG2]] to %[[I_PVT_DECL]]#1 : !fir.ref<i32>
+  ! CHECK: fir.call @_QPfoo(%[[I_PVT_DECL]]#1, %[[A_PVT_DECL]]#1) {{.*}}: (!fir.ref<i32>, !fir.ref<i32>) -> ()
+  ! CHECK: omp.yield
+    do i= a, n
+      call foo(i, a)
+    end do
+  !$omp end parallel do
+  !CHECK: fir.call @_QPbar(%[[ARG1_DECL]]#1) {{.*}}: (!fir.ref<i32>) -> ()
+  call bar(n)
+end subroutine omp_do_firstprivate2
diff --git a/flang/test/Lower/OpenMP/sections.f90 b/flang/test/Lower/OpenMP/sections.f90
index 7abbe32eaa..16426d070d 100644
--- a/flang/test/Lower/OpenMP/sections.f90
+++ b/flang/test/Lower/OpenMP/sections.f90
@@ -90,8 +90,8 @@ end program sample
 !CHECK:     omp.section  {
 !CHECK:         %[[PRIVATE_ALPHA:.*]] = fir.alloca f32 {bindc_name = "alpha", pinned, uniq_name = "_QFfirstprivateEalpha"}
 !CHECK:         %[[PRIVATE_ALPHA_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_ALPHA]] {uniq_name = "_QFfirstprivateEalpha"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-!CHECK:         %[[TEMP:.*]] = fir.load %[[ARG_DECL]]#1 : !fir.ref<f32>
-!CHECK:         fir.store %[[TEMP]] to %[[PRIVATE_ALPHA_DECL]]#1 : !fir.ref<f32>
+!CHECK:         %[[TEMP:.*]] = fir.load %[[ARG_DECL]]#0 : !fir.ref<f32>
+!CHECK:         hlfir.assign %[[TEMP]] to %[[PRIVATE_ALPHA_DECL]]#0 temporary_lhs : f32, !fir.ref<f32>
 !CHECK:       omp.terminator
 !CHECK:     }
 !CHECK:     omp.terminator
@@ -147,8 +147,8 @@ subroutine lastprivate()
 !CHECK: %[[RESULT:.*]] = arith.addi %[[TEMP]], %[[CONST]] : i32
 !CHECK: hlfir.assign %[[RESULT]] to %[[PRIVATE_X_DECL]]#0 : i32, !fir.ref<i32>
 !CHECK: fir.if %[[TRUE]] {
-!CHECK: %[[TEMP1:.*]] = fir.load %[[PRIVATE_X_DECL]]#1 : !fir.ref<i32>
-!CHECK: fir.store %[[TEMP1]] to %[[X_DECL]]#1 : !fir.ref<i32>
+!CHECK: %[[TEMP1:.*]] = fir.load %[[PRIVATE_X_DECL]]#0 : !fir.ref<i32>
+!CHECK: hlfir.assign %[[TEMP1]] to %[[X_DECL]]#0 temporary_lhs : i32, !fir.ref<i32>
 !CHECK: }
 !CHECK: omp.terminator
 !CHECK: }
@@ -163,8 +163,8 @@ subroutine lastprivate()
 !CHECK: omp.section {
 !CHECK: %[[PRIVATE_X:.*]] = fir.alloca i32 {bindc_name = "x", pinned, uniq_name = "_QFlastprivateEx"}
 !CHECK: %[[PRIVATE_X_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_X]] {uniq_name = "_QFlastprivateEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK: %[[TEMP:.*]] = fir.load %[[X_DECL]]#1 : !fir.ref<i32>
-!CHECK: fir.store %[[TEMP]] to %[[PRIVATE_X_DECL]]#1 : !fir.ref<i32>
+!CHECK: %[[TEMP:.*]] = fir.load %[[X_DECL]]#0 : !fir.ref<i32>
+!CHECK: hlfir.assign %[[TEMP]] to %[[PRIVATE_X_DECL]]#0 temporary_lhs : i32, !fir.ref<i32>
 !CHECK: omp.barrier
 !CHECK: %[[CONST:.*]] = arith.constant 10 : i32
 !CHECK: %[[TEMP:.*]] = fir.load %[[PRIVATE_X_DECL]]#0 : !fir.ref<i32>
@@ -177,8 +177,8 @@ subroutine lastprivate()
 !CHECK: omp.section {
 !CHECK: %[[PRIVATE_X:.*]] = fir.alloca i32 {bindc_name = "x", pinned, uniq_name = "_QFlastprivateEx"}
 !CHECK: %[[PRIVATE_X_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_X]] {uniq_name = "_QFlastprivateEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK: %[[TEMP:.*]] = fir.load %[[X_DECL]]#1 : !fir.ref<i32>
-!CHECK: fir.store %[[TEMP]] to %[[PRIVATE_X_DECL]]#1 : !fir.ref<i32>
+!CHECK: %[[TEMP:.*]] = fir.load %[[X_DECL]]#0 : !fir.ref<i32>
+!CHECK: hlfir.assign %[[TEMP]] to %[[PRIVATE_X_DECL]]#0 temporary_lhs : i32, !fir.ref<i32>
 !CHECK: omp.barrier
 !CHECK: %[[TRUE:.*]] = arith.constant true
 !CHECK: %[[TEMP:.*]] = fir.load %[[PRIVATE_X_DECL]]#0 : !fir.ref<i32>
@@ -186,8 +186,8 @@ subroutine lastprivate()
 !CHECK: %[[RESULT:.*]] = arith.addi %[[TEMP]], %[[CONST]] : i32
 !CHECK: hlfir.assign %[[RESULT]] to %[[PRIVATE_X_DECL]]#0 : i32, !fir.ref<i32>
 !CHECK: fir.if %[[TRUE]] {
-!CHECK: %[[TEMP:.*]] = fir.load %[[PRIVATE_X_DECL]]#1 : !fir.ref<i32>
-!CHECK: fir.store %[[TEMP]] to %[[X_DECL]]#1 : !fir.ref<i32>
+!CHECK: %[[TEMP:.*]] = fir.load %[[PRIVATE_X_DECL]]#0 : !fir.ref<i32>
+!CHECK: hlfir.assign %[[TEMP]] to %[[X_DECL]]#0 temporary_lhs : i32, !fir.ref<i32>
 !CHECK: }
 !CHECK: omp.terminator
 !CHECK: }
@@ -202,8 +202,8 @@ subroutine lastprivate()
 !CHECK: omp.section {
 !CHECK: %[[PRIVATE_X:.*]] = fir.alloca i32 {bindc_name = "x", pinned, uniq_name = "_QFlastprivateEx"}
 !CHECK: %[[PRIVATE_X_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_X]] {uniq_name = "_QFlastprivateEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK: %[[TEMP:.*]] = fir.load %[[X_DECL]]#1 : !fir.ref<i32>
-!CHECK: fir.store %[[TEMP]] to %[[PRIVATE_X_DECL]]#1 : !fir.ref<i32>
+!CHECK: %[[TEMP:.*]] = fir.load %[[X_DECL]]#0 : !fir.ref<i32>
+!CHECK: hlfir.assign %[[TEMP]] to %[[PRIVATE_X_DECL]]#0 temporary_lhs : i32, !fir.ref<i32>
 !CHECK: omp.barrier
 !CHECK: %[[CONST:.*]] = arith.constant 10 : i32
 !CHECK: %[[TEMP:.*]] = fir.load %[[PRIVATE_X_DECL]]#0 : !fir.ref<i32>
@@ -216,8 +216,8 @@ subroutine lastprivate()
 !CHECK: omp.section {
 !CHECK: %[[PRIVATE_X:.*]] = fir.alloca i32 {bindc_name = "x", pinned, uniq_name = "_QFlastprivateEx"}
 !CHECK: %[[PRIVATE_X_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_X]] {uniq_name = "_QFlastprivateEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK: %[[TEMP:.*]] = fir.load %[[X_DECL]]#1 : !fir.ref<i32>
-!CHECK: fir.store %[[TEMP]] to %[[PRIVATE_X_DECL]]#1 : !fir.ref<i32>
+!CHECK: %[[TEMP:.*]] = fir.load %[[X_DECL]]#0 : !fir.ref<i32>
+!CHECK: hlfir.assign %[[TEMP]] to %[[PRIVATE_X_DECL]]#0 temporary_lhs : i32, !fir.ref<i32>
 !CHECK: omp.barrier
 !CHECK: %[[TRUE:.*]] = arith.constant true
 !CHECK: %[[TEMP:.*]] = fir.load %[[PRIVATE_X_DECL]]#0 : !fir.ref<i32>
@@ -225,8 +225,8 @@ subroutine lastprivate()
 !CHECK: %[[RESULT:.*]] = arith.addi %[[TEMP]], %[[CONST]] : i32
 !CHECK: hlfir.assign %[[RESULT]] to %[[PRIVATE_X_DECL]]#0 : i32, !fir.ref<i32>
 !CHECK: fir.if %[[TRUE]] {
-!CHECK: %[[TEMP:.*]] = fir.load %[[PRIVATE_X_DECL]]#1 : !fir.ref<i32>
-!CHECK: fir.store %[[TEMP]] to %[[X_DECL]]#1 : !fir.ref<i32>
+!CHECK: %[[TEMP:.*]] = fir.load %[[PRIVATE_X_DECL]]#0 : !fir.ref<i32>
+!CHECK: hlfir.assign %[[TEMP]] to %[[X_DECL]]#0 temporary_lhs : i32, !fir.ref<i32>
 !CHECK: omp.barrier
 !CHECK: }
 !CHECK: omp.terminator
@@ -247,8 +247,8 @@ subroutine lastprivate()
 !CHECK: %[[CONST:.*]] = arith.constant 1 : i32
 !CHECK: %[[RESULT:.*]] = arith.addi %[[INNER_PRIVATE_X]], %[[CONST]] : i32
 !CHECK: hlfir.assign %[[RESULT]] to %[[PRIVATE_X_DECL]]#0 : i32, !fir.ref<i32>
-!CHECK: %[[LOADED_VALUE:.*]] = fir.load %[[PRIVATE_X_DECL]]#1 : !fir.ref<i32>
-!CHECK: fir.store %[[LOADED_VALUE]] to %[[X_DECL]]#1 : !fir.ref<i32>
+!CHECK: %[[LOADED_VALUE:.*]] = fir.load %[[PRIVATE_X_DECL]]#0 : !fir.ref<i32>
+!CHECK: hlfir.assign %[[LOADED_VALUE]] to %[[X_DECL]]#0 temporary_lhs : i32, !fir.ref<i32>
 !CHECK: omp.terminator
 !CHECK: }
 !CHECK: omp.terminator
@@ -290,8 +290,8 @@ subroutine unstructured_sections_privatization()
 !CHECK: omp.section {
 !CHECK: %[[PRIVATE_X:.*]] = fir.alloca f32 {bindc_name = "x", pinned, uniq_name = "_QFunstructured_sections_privatizationEx"}
 !CHECK: %[[PRIVATE_X_DECL:.*]]:2 = hlfir.declare %[[PRIVATE_X]] {uniq_name = "_QFunstructured_sections_privatizationEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
-!CHECK: %[[TEMP:.*]] = fir.load %[[X_DECL]]#1 : !fir.ref<f32>
-!CHECK: fir.store %[[TEMP]] to %[[PRIVATE_X_DECL]]#1 : !fir.ref<f32>
+!CHECK: %[[TEMP:.*]] = fir.load %[[X_DECL]]#0 : !fir.ref<f32>
+!CHECK: hlfir.assign %[[TEMP]] to %[[PRIVATE_X_DECL]]#0 temporary_lhs : f32, !fir.ref<f32>
 !CHECK: cf.br ^bb1
 !CHECK: ^bb1:
 !CHECK: %[[INNER_PRIVATE_X:.*]] = fir.load %[[PRIVATE_X_DECL]]#0 : !fir.ref<f32>
diff --git a/flang/test/Lower/OpenMP/unstructured.f90 b/flang/test/Lower/OpenMP/unstructured.f90
new file mode 100644
index 0000000000..e5bf980ce9
--- /dev/null
+++ b/flang/test/Lower/OpenMP/unstructured.f90
@@ -0,0 +1,348 @@
+! Test unstructured code adjacent to and inside OpenMP constructs.
+
+! RUN: bbc %s -fopenmp -emit-hlfir -o "-" | FileCheck %s
+
+! CHECK-LABEL: func @_QPss1{{.*}} {
+! CHECK:   br ^bb1
+! CHECK: ^bb1:  // 2 preds: ^bb0, ^bb4
+! CHECK:   cond_br %{{[0-9]*}}, ^bb2, ^bb5
+! CHECK: ^bb2:  // pred: ^bb1
+! CHECK:   cond_br %{{[0-9]*}}, ^bb3, ^bb4
+! CHECK: ^bb4:  // pred: ^bb2
+! CHECK:   fir.call @_FortranAioBeginExternalListOutput
+! CHECK:   br ^bb1
+! CHECK: ^bb5:  // 2 preds: ^bb1, ^bb3
+! CHECK:   omp.master  {
+! CHECK:     @_FortranAioBeginExternalListOutput
+! CHECK:     omp.terminator
+! CHECK:   }
+! CHECK:   @_FortranAioBeginExternalListOutput
+! CHECK: }
+subroutine ss1(n) ! unstructured code followed by a structured OpenMP construct
+  do i = 1, 3
+    if (i .eq. n) exit
+    print*, 'ss1-A', i
+  enddo
+  !$omp master
+    print*, 'ss1-B', i
+  !$omp end master
+  print*
+end
+
+! CHECK-LABEL: func @_QPss2{{.*}} {
+! CHECK:   omp.master  {
+! CHECK:     @_FortranAioBeginExternalListOutput
+! CHECK:     br ^bb1
+! CHECK:   ^bb1:  // 2 preds: ^bb0, ^bb4
+! CHECK:     cond_br %{{[0-9]*}}, ^bb2, ^bb5
+! CHECK:   ^bb2:  // pred: ^bb1
+! CHECK:     cond_br %{{[0-9]*}}, ^bb3, ^bb4
+! CHECK:   ^bb3:  // pred: ^bb2
+! CHECK:     @_FortranAioBeginExternalListOutput
+! CHECK:     br ^bb1
+! CHECK:   ^bb5:  // 2 preds: ^bb1, ^bb3
+! CHECK:     omp.terminator
+! CHECK:   }
+! CHECK:   @_FortranAioBeginExternalListOutput
+! CHECK:   @_FortranAioBeginExternalListOutput
+! CHECK: }
+subroutine ss2(n) ! unstructured OpenMP construct; loop exit inside construct
+  !$omp master
+    print*, 'ss2-A', n
+    do i = 1, 3
+      if (i .eq. n) exit
+      print*, 'ss2-B', i
+    enddo
+  !$omp end master
+  print*, 'ss2-C', i
+  print*
+end
+
+! CHECK-LABEL: func @_QPss3{{.*}} {
+! CHECK:   omp.parallel {
+! CHECK:     %[[ALLOCA_K:.*]] = fir.alloca i32 {bindc_name = "k", pinned}
+! CHECK:     %[[K_DECL:.*]]:2 = hlfir.declare %[[ALLOCA_K]] {uniq_name = "_QFss3Ek"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:     %[[ALLOCA_1:.*]] = fir.alloca i32 {{{.*}}, pinned}
+! CHECK:     %[[OMP_LOOP_J_DECL:.*]]:2 = hlfir.declare %[[ALLOCA_1]] {uniq_name = "_QFss3Ej"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:     %[[ALLOCA_2:.*]] = fir.alloca i32 {{{.*}}, pinned}
+! CHECK:     %[[OMP_LOOP_K_DECL:.*]]:2 = hlfir.declare %[[ALLOCA_2]] {uniq_name = "_QFss3Ek"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:     br ^bb1
+! CHECK:   ^bb1:  // 2 preds: ^bb0, ^bb3
+! CHECK:     cond_br %{{[0-9]*}}, ^bb2, ^bb4
+! CHECK:   ^bb2:  // pred: ^bb1
+! CHECK:     omp.wsloop for (%[[ARG1:.*]]) : {{.*}} {
+! CHECK:       fir.store %[[ARG1]] to %[[OMP_LOOP_K_DECL]]#1 : !fir.ref<i32>
+! CHECK:     @_FortranAioBeginExternalListOutput
+! CHECK:       %[[LOAD_1:.*]] = fir.load %[[OMP_LOOP_K_DECL]]#0 : !fir.ref<i32>
+! CHECK:     @_FortranAioOutputInteger32(%{{.*}}, %[[LOAD_1]])
+! CHECK:       omp.yield
+! CHECK:     }
+! CHECK:     omp.wsloop for (%[[ARG2:.*]]) : {{.*}} {
+! CHECK:       fir.store %[[ARG2]] to %[[OMP_LOOP_J_DECL]]#1 : !fir.ref<i32>
+! CHECK:       br ^bb1
+! CHECK:     ^bb2:  // 2 preds: ^bb1, ^bb5
+! CHECK:       cond_br %{{[0-9]*}}, ^bb3, ^bb6
+! CHECK:     ^bb3:  // pred: ^bb2
+! CHECK:       cond_br %{{[0-9]*}}, ^bb4, ^bb5
+! CHECK:     ^bb4:  // pred: ^bb3
+! CHECK:       @_FortranAioBeginExternalListOutput
+! CHECK:       %[[LOAD_2:.*]] = fir.load %[[K_DECL]]#0 : !fir.ref<i32>
+! CHECK:     @_FortranAioOutputInteger32(%{{.*}}, %[[LOAD_2]])
+! CHECK:       br ^bb2
+! CHECK:     ^bb6:  // 2 preds: ^bb2, ^bb4
+! CHECK:       omp.yield
+! CHECK:     }
+! CHECK:     br ^bb1
+! CHECK:   ^bb4:  // pred: ^bb1
+! CHECK:     omp.terminator
+! CHECK:   }
+! CHECK: }
+subroutine ss3(n) ! nested unstructured OpenMP constructs
+  !$omp parallel
+    do i = 1, 3
+      !$omp do
+        do k = 1, 3
+          print*, 'ss3-A', k
+        enddo
+      !$omp end do
+      !$omp do
+        do j = 1, 3
+          do k = 1, 3
+            if (k .eq. n) exit
+            print*, 'ss3-B', k
+          enddo
+        enddo
+      !$omp end do
+    enddo
+  !$omp end parallel
+end
+
+! CHECK-LABEL: func @_QPss4{{.*}} {
+! CHECK:       omp.parallel {
+! CHECK:         %[[ALLOCA:.*]] = fir.alloca i32 {{{.*}}, pinned}
+! CHECK:         %[[OMP_LOOP_J_DECL:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "_QFss4Ej"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:         omp.wsloop for (%[[ARG:.*]]) : {{.*}} {
+! CHECK:           fir.store %[[ARG]] to %[[OMP_LOOP_J_DECL]]#1 : !fir.ref<i32>
+! CHECK:           %[[COND:.*]] = arith.cmpi eq, %{{.*}}, %{{.*}}
+! CHECK:           %[[COND_XOR:.*]] = arith.xori %[[COND]], %{{.*}}
+! CHECK:          fir.if %[[COND_XOR]] {
+! CHECK:           @_FortranAioBeginExternalListOutput
+! CHECK:           %[[LOAD:.*]] = fir.load %[[OMP_LOOP_J_DECL]]#0 : !fir.ref<i32>
+! CHECK:           @_FortranAioOutputInteger32(%{{.*}}, %[[LOAD]])
+! CHECK:          } else {
+! CHECK:          }
+! CHECK-NEXT:      omp.yield
+! CHECK-NEXT:  }
+! CHECK:       omp.terminator
+! CHECK-NEXT:}
+subroutine ss4(n) ! CYCLE in OpenMP wsloop constructs
+  !$omp parallel
+    do i = 1, 3
+      !$omp do
+        do j = 1, 3
+           if (j .eq. n) cycle
+           print*, 'ss4', j
+        enddo
+      !$omp end do
+    enddo
+  !$omp end parallel
+end
+
+! CHECK-LABEL: func @_QPss5() {
+! CHECK:  omp.parallel  {
+! CHECK:    omp.wsloop {{.*}} {
+! CHECK:      br ^[[BB1:.*]]
+! CHECK:    ^[[BB1]]:
+! CHECK:      br ^[[BB2:.*]]
+! CHECK:    ^[[BB2]]:
+! CHECK:      cond_br %{{.*}}, ^[[BB3:.*]], ^[[BB6:.*]]
+! CHECK:    ^[[BB3]]:
+! CHECK:      cond_br %{{.*}}, ^[[BB4:.*]], ^[[BB3:.*]]
+! CHECK:    ^[[BB4]]:
+! CHECK:      br ^[[BB6]]
+! CHECK:    ^[[BB3]]:
+! CHECK:      br ^[[BB2]]
+! CHECK:    ^[[BB6]]:
+! CHECK:      omp.yield
+! CHECK:    }
+! CHECK:    omp.terminator
+! CHECK:  }
+subroutine ss5() ! EXIT inside OpenMP wsloop (inside parallel)
+  integer :: x
+  !$omp parallel private(x)
+    !$omp do
+      do j = 1, 3
+        x = j * i
+        do k = 1, 3
+          if (k .eq. n) exit
+          x = k
+          x = x + k
+        enddo
+        x = j - 222
+      enddo
+    !$omp end do
+  !$omp end parallel
+end
+
+! CHECK-LABEL: func @_QPss6() {
+! CHECK:  omp.parallel  {
+! CHECK:    br ^[[BB1_OUTER:.*]]
+! CHECK:  ^[[BB1_OUTER]]:
+! CHECK:    cond_br %{{.*}}, ^[[BB2_OUTER:.*]], ^[[BB3_OUTER:.*]]
+! CHECK:  ^[[BB2_OUTER]]:
+! CHECK:    omp.wsloop {{.*}} {
+! CHECK:      br ^[[BB1:.*]]
+! CHECK:    ^[[BB1]]:
+! CHECK:      br ^[[BB2:.*]]
+! CHECK:    ^[[BB2]]:
+! CHECK:      cond_br %{{.*}}, ^[[BB3:.*]], ^[[BB6:.*]]
+! CHECK:    ^[[BB3]]:
+! CHECK:      cond_br %{{.*}}, ^[[BB4:.*]], ^[[BB5:.*]]
+! CHECK:    ^[[BB4]]:
+! CHECK:      br ^[[BB6]]
+! CHECK:    ^[[BB5]]
+! CHECK:      br ^[[BB2]]
+! CHECK:    ^[[BB6]]:
+! CHECK:      omp.yield
+! CHECK:    }
+! CHECK:    br ^[[BB1_OUTER]]
+! CHECK:  ^[[BB3_OUTER]]:
+! CHECK:    omp.terminator
+! CHECK:  }
+subroutine ss6() ! EXIT inside OpenMP wsloop in a do loop (inside parallel)
+  integer :: x
+  !$omp parallel private(x)
+    do i = 1, 3
+      !$omp do
+        do j = 1, 3
+          x = j * i
+          do k = 1, 3
+            if (k .eq. n) exit
+            x = k
+            x = x + k
+          enddo
+          x = j - 222
+        enddo
+      !$omp end do
+    enddo
+  !$omp end parallel
+end
+
+! CHECK-LABEL: func @_QPss7() {
+! CHECK: br ^[[BB1_OUTER:.*]]
+! CHECK: ^[[BB1_OUTER]]:
+! CHECK:   cond_br %{{.*}}, ^[[BB2_OUTER:.*]], ^[[BB3_OUTER:.*]]
+! CHECK-NEXT: ^[[BB2_OUTER:.*]]:
+! CHECK:   omp.parallel  {
+! CHECK:     omp.wsloop {{.*}} {
+! CHECK:       br ^[[BB1:.*]]
+! CHECK-NEXT:     ^[[BB1]]:
+! CHECK:       br ^[[BB2:.*]]
+! CHECK-NEXT:     ^[[BB2]]:
+! CHECK:       cond_br %{{.*}}, ^[[BB3:.*]], ^[[BB6:.*]]
+! CHECK-NEXT:     ^[[BB3]]:
+! CHECK:       cond_br %{{.*}}, ^[[BB4:.*]], ^[[BB5:.*]]
+! CHECK-NEXT:     ^[[BB4]]:
+! CHECK:       br ^[[BB6]]
+! CHECK-NEXT:     ^[[BB5]]:
+! CHECK:       br ^[[BB2]]
+! CHECK-NEXT:     ^[[BB6]]:
+! CHECK:       omp.yield
+! CHECK:     }
+! CHECK:     omp.terminator
+! CHECK:   }
+! CHECK:   br ^[[BB1_OUTER]]
+! CHECK-NEXT: ^[[BB3_OUTER]]:
+! CHECK-NEXT:   return
+subroutine ss7() ! EXIT inside OpenMP parallel do (inside do loop)
+  integer :: x
+    do i = 1, 3
+      !$omp parallel do private(x)
+        do j = 1, 3
+          x = j * i
+          do k = 1, 3
+            if (k .eq. n) exit
+            x = k
+            x = x + k
+          enddo
+        enddo
+      !$omp end parallel do
+    enddo
+end
+
+! CHECK-LABEL: func @_QPss8() {
+! CHECK:  omp.parallel  {
+! CHECK:    omp.wsloop {{.*}} {
+! CHECK:      br ^[[BB1:.*]]
+! CHECK-NEXT:    ^[[BB1]]:
+! CHECK:      br ^[[BB2:.*]]
+! CHECK:    ^[[BB2]]:
+! CHECK:      cond_br %{{.*}}, ^[[BB3:.*]], ^[[BB6:.*]]
+! CHECK:    ^[[BB3]]:
+! CHECK:      cond_br %{{.*}}, ^[[BB4:.*]], ^[[BB5:.*]]
+! CHECK:    ^[[BB4]]:
+! CHECK-NEXT:    br ^[[BB6]]
+! CHECK:    ^[[BB5]]:
+! CHECK:      br ^[[BB2]]
+! CHECK-NEXT:    ^[[BB6]]:
+! CHECK:      omp.yield
+! CHECK:    }
+! CHECK:    omp.terminator
+! CHECK:  }
+subroutine ss8() ! EXIT inside OpenMP parallel do
+  integer :: x
+      !$omp parallel do private(x)
+        do j = 1, 3
+          x = j * i
+          do k = 1, 3
+            if (k .eq. n) exit
+            x = k
+            x = x + k
+          enddo
+        enddo
+      !$omp end parallel do
+end
+
+! CHECK-LABEL: func @_QPss9() {
+! CHECK:  omp.parallel  {
+! CHECK-NEXT:    omp.parallel  {
+! CHECK:      br ^[[BB1:.*]]
+! CHECK:         ^[[BB1]]:
+! CHECK:      cond_br %{{.*}}, ^[[BB2:.*]], ^[[BB5:.*]]
+! CHECK-NEXT:    ^[[BB2]]:
+! CHECK:      cond_br %{{.*}}, ^[[BB3:.*]], ^[[BB4:.*]]
+! CHECK-NEXT:    ^[[BB3]]:
+! CHECK-NEXT:    br ^[[BB5]]
+! CHECK-NEXT:    ^[[BB4]]:
+! CHECK:      br ^[[BB1]]
+! CHECK-NEXT:    ^[[BB5]]:
+! CHECK:      omp.terminator
+! CHECK-NEXT:    }
+! CHECK:    omp.terminator
+! CHECK-NEXT  }
+! CHECK: }
+subroutine ss9() ! EXIT inside OpenMP parallel (inside parallel)
+  integer :: x
+  !$omp parallel
+  !$omp parallel private(x)
+    do k = 1, 3
+      if (k .eq. n) exit
+      x = k
+      x = x + k
+    end do
+  !$omp end parallel
+  !$omp end parallel
+end
+
+! CHECK-LABEL: func @_QQmain
+program p
+  call ss1(2)
+  call ss2(2)
+  call ss3(2)
+  call ss4(2)
+  call ss5()
+  call ss6()
+  call ss7()
+  call ss8()
+  call ss9()
+end
diff --git a/flang/test/Lower/OpenMP/wsloop.f90 b/flang/test/Lower/OpenMP/wsloop.f90
new file mode 100644
index 0000000000..4068f715c3
--- /dev/null
+++ b/flang/test/Lower/OpenMP/wsloop.f90
@@ -0,0 +1,75 @@
+! This test checks lowering of OpenMP DO Directive (Worksharing).
+
+! RUN: bbc -fopenmp -emit-hlfir %s -o - | FileCheck %s
+
+!CHECK-LABEL: func @_QPsimple_loop()
+subroutine simple_loop
+  integer :: i
+  ! CHECK:  omp.parallel
+  !$OMP PARALLEL
+  ! CHECK:     %[[ALLOCA_IV:.*]] = fir.alloca i32 {{{.*}}, pinned}
+  ! CHECK:     %[[IV_DECL:.*]]:2 = hlfir.declare %[[ALLOCA_IV]] {uniq_name = "_QFsimple_loopEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  ! CHECK:     %[[WS_LB:.*]] = arith.constant 1 : i32
+  ! CHECK:     %[[WS_UB:.*]] = arith.constant 9 : i32
+  ! CHECK:     %[[WS_STEP:.*]] = arith.constant 1 : i32
+  ! CHECK:     omp.wsloop for (%[[I:.*]]) : i32 = (%[[WS_LB]]) to (%[[WS_UB]]) inclusive step (%[[WS_STEP]])
+  !$OMP DO
+  do i=1, 9
+  ! CHECK:             fir.store %[[I]] to %[[IV_DECL:.*]]#1 : !fir.ref<i32>
+  ! CHECK:             %[[LOAD_IV:.*]] = fir.load %[[IV_DECL]]#0 : !fir.ref<i32>
+  ! CHECK:    fir.call @_FortranAioOutputInteger32({{.*}}, %[[LOAD_IV]]) {{.*}}: (!fir.ref<i8>, i32) -> i1
+    print*, i
+  end do
+  ! CHECK:       omp.yield
+  !$OMP END DO
+  ! CHECK:       omp.terminator
+  !$OMP END PARALLEL
+end subroutine
+
+!CHECK-LABEL: func @_QPsimple_loop_with_step()
+subroutine simple_loop_with_step
+  integer :: i
+  ! CHECK:  omp.parallel
+  !$OMP PARALLEL
+  ! CHECK:     %[[ALLOCA_IV:.*]] = fir.alloca i32 {{{.*}}, pinned}
+  ! CHECK:     %[[IV_DECL:.*]]:2 = hlfir.declare %[[ALLOCA_IV]] {uniq_name = "_QFsimple_loop_with_stepEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  ! CHECK:     %[[WS_LB:.*]] = arith.constant 1 : i32
+  ! CHECK:     %[[WS_UB:.*]] = arith.constant 9 : i32
+  ! CHECK:     %[[WS_STEP:.*]] = arith.constant 2 : i32
+  ! CHECK:     omp.wsloop for (%[[I:.*]]) : i32 = (%[[WS_LB]]) to (%[[WS_UB]]) inclusive step (%[[WS_STEP]])
+  ! CHECK:       fir.store %[[I]] to %[[IV_DECL]]#1 : !fir.ref<i32>
+  ! CHECK:       %[[LOAD_IV:.*]] = fir.load %[[IV_DECL]]#0 : !fir.ref<i32>
+  !$OMP DO
+  do i=1, 9, 2
+  ! CHECK:    fir.call @_FortranAioOutputInteger32({{.*}}, %[[LOAD_IV]]) {{.*}}: (!fir.ref<i8>, i32) -> i1
+    print*, i
+  end do
+  ! CHECK:       omp.yield
+  !$OMP END DO
+  ! CHECK:       omp.terminator
+  !$OMP END PARALLEL
+end subroutine
+
+!CHECK-LABEL: func @_QPloop_with_schedule_nowait()
+subroutine loop_with_schedule_nowait
+  integer :: i
+  ! CHECK:  omp.parallel
+  !$OMP PARALLEL
+  ! CHECK:     %[[ALLOCA_IV:.*]] = fir.alloca i32 {{{.*}}, pinned}
+  ! CHECK:     %[[IV_DECL:.*]]:2 = hlfir.declare %[[ALLOCA_IV]] {uniq_name = "_QFloop_with_schedule_nowaitEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  ! CHECK:     %[[WS_LB:.*]] = arith.constant 1 : i32
+  ! CHECK:     %[[WS_UB:.*]] = arith.constant 9 : i32
+  ! CHECK:     %[[WS_STEP:.*]] = arith.constant 1 : i32
+  ! CHECK:     omp.wsloop schedule(runtime) nowait for (%[[I:.*]]) : i32 = (%[[WS_LB]]) to (%[[WS_UB]]) inclusive step (%[[WS_STEP]])
+  !$OMP DO SCHEDULE(runtime)
+  do i=1, 9
+  ! CHECK:       fir.store %[[I]] to %[[IV_DECL]]#1 : !fir.ref<i32>
+  ! CHECK:       %[[LOAD_IV:.*]] = fir.load %[[IV_DECL]]#0 : !fir.ref<i32>
+  ! CHECK:    fir.call @_FortranAioOutputInteger32({{.*}}, %[[LOAD_IV]]) {{.*}}: (!fir.ref<i8>, i32) -> i1
+    print*, i
+  end do
+  ! CHECK:       omp.yield
+  !$OMP END DO NOWAIT
+  ! CHECK:       omp.terminator
+  !$OMP END PARALLEL
+end subroutine
diff --git a/flang/test/Lower/allocatable-polymorphic.f90 b/flang/test/Lower/allocatable-polymorphic.f90
index 53b257d2ea..f53834b13d 100644
--- a/flang/test/Lower/allocatable-polymorphic.f90
+++ b/flang/test/Lower/allocatable-polymorphic.f90
@@ -135,12 +135,12 @@ subroutine test_pointer()
 ! call c1%proc2()
 ! CHECK: %[[C1_LOAD:.*]] = fir.load %[[C1_DESC]] : !fir.ref<!fir.class<!fir.ptr<!fir.type<_QMpolyTp1{a:i32,b:i32}>>>>
 ! CHECK: %[[C1_REBOX:.*]] = fir.rebox %[[C1_LOAD]] : (!fir.class<!fir.ptr<!fir.type<_QMpolyTp1{a:i32,b:i32}>>>) -> !fir.class<!fir.type<_QMpolyTp1{a:i32,b:i32}>>
-! CHECK: fir.dispatch "proc2"(%[[C1_REBOX]] : !fir.class<!fir.type<_QMpolyTp1{a:i32,b:i32}>>) (%[[C1_REBOX]] : !fir.class<!fir.type<_QMpolyTp1{a:i32,b:i32}>>) {pass_arg_pos = 0 : i32}
+! CHECK: fir.dispatch "proc2"(%[[C1_LOAD]] : !fir.class<!fir.ptr<!fir.type<_QMpolyTp1{a:i32,b:i32}>>>) (%[[C1_REBOX]] : !fir.class<!fir.type<_QMpolyTp1{a:i32,b:i32}>>) {pass_arg_pos = 0 : i32}
 
 ! call c2%proc2()
 ! CHECK: %[[C2_LOAD:.*]] = fir.load %[[C2_DESC]] : !fir.ref<!fir.class<!fir.ptr<!fir.type<_QMpolyTp1{a:i32,b:i32}>>>>
 ! CHECK: %[[C2_REBOX:.*]] = fir.rebox %[[C2_LOAD]] : (!fir.class<!fir.ptr<!fir.type<_QMpolyTp1{a:i32,b:i32}>>>) -> !fir.class<!fir.type<_QMpolyTp1{a:i32,b:i32}>>
-! CHECK: fir.dispatch "proc2"(%[[C2_REBOX]] : !fir.class<!fir.type<_QMpolyTp1{a:i32,b:i32}>>) (%[[C2_REBOX]] : !fir.class<!fir.type<_QMpolyTp1{a:i32,b:i32}>>) {pass_arg_pos = 0 : i32}
+! CHECK: fir.dispatch "proc2"(%[[C2_LOAD]] : !fir.class<!fir.ptr<!fir.type<_QMpolyTp1{a:i32,b:i32}>>>) (%[[C2_REBOX]] : !fir.class<!fir.type<_QMpolyTp1{a:i32,b:i32}>>) {pass_arg_pos = 0 : i32}
 
 ! CHECK: %[[TYPE_DESC_P1:.*]] = fir.type_desc !fir.type<_QMpolyTp1{a:i32,b:i32}>
 ! CHECK: %[[C3_CAST:.*]] = fir.convert %[[C3_DESC]] : (!fir.ref<!fir.class<!fir.ptr<!fir.array<?x!fir.type<_QMpolyTp1{a:i32,b:i32}>>>>>) -> !fir.ref<!fir.box<none>>
@@ -319,11 +319,11 @@ subroutine test_allocatable()
 
 ! CHECK: %[[C1_LOAD2:.*]] = fir.load %[[C1_DESC]] : !fir.ref<!fir.class<!fir.heap<!fir.type<_QMpolyTp1{a:i32,b:i32}>>>>
 ! CHECK: %[[C1_REBOX:.*]] = fir.rebox %[[C1_LOAD2]] : (!fir.class<!fir.heap<!fir.type<_QMpolyTp1{a:i32,b:i32}>>>) -> !fir.class<!fir.type<_QMpolyTp1{a:i32,b:i32}>>
-! CHECK: fir.dispatch "proc2"(%[[C1_REBOX]] : !fir.class<!fir.type<_QMpolyTp1{a:i32,b:i32}>>) (%[[C1_REBOX]] : !fir.class<!fir.type<_QMpolyTp1{a:i32,b:i32}>>) {pass_arg_pos = 0 : i32}
+! CHECK: fir.dispatch "proc2"(%[[C1_LOAD2]] : !fir.class<!fir.heap<!fir.type<_QMpolyTp1{a:i32,b:i32}>>>) (%[[C1_REBOX]] : !fir.class<!fir.type<_QMpolyTp1{a:i32,b:i32}>>) {pass_arg_pos = 0 : i32}
 
 ! CHECK: %[[C2_LOAD2:.*]] = fir.load %[[C2_DESC]] : !fir.ref<!fir.class<!fir.heap<!fir.type<_QMpolyTp1{a:i32,b:i32}>>>>
 ! CHECK: %[[C2_REBOX:.*]] = fir.rebox %[[C2_LOAD2]] : (!fir.class<!fir.heap<!fir.type<_QMpolyTp1{a:i32,b:i32}>>>) -> !fir.class<!fir.type<_QMpolyTp1{a:i32,b:i32}>>
-! CHECK: fir.dispatch "proc2"(%[[C2_REBOX]] : !fir.class<!fir.type<_QMpolyTp1{a:i32,b:i32}>>) (%[[C2_REBOX]] : !fir.class<!fir.type<_QMpolyTp1{a:i32,b:i32}>>) {pass_arg_pos = 0 : i32}
+! CHECK: fir.dispatch "proc2"(%[[C2_LOAD2]] : !fir.class<!fir.heap<!fir.type<_QMpolyTp1{a:i32,b:i32}>>>) (%[[C2_REBOX]] : !fir.class<!fir.type<_QMpolyTp1{a:i32,b:i32}>>) {pass_arg_pos = 0 : i32}
 
 ! CHECK-LABEL: %{{.*}} = fir.do_loop
 ! CHECK: %[[C3_LOAD:.*]] = fir.load %[[C3_DESC]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMpolyTp1{a:i32,b:i32}>>>>>
diff --git a/flang/test/Lower/dispatch.f90 b/flang/test/Lower/dispatch.f90
index 0331bfb084..71150ef9b3 100644
--- a/flang/test/Lower/dispatch.f90
+++ b/flang/test/Lower/dispatch.f90
@@ -182,7 +182,7 @@ subroutine check_dispatch_scalar_allocatable(p)
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.class<!fir.heap<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>> {fir.bindc_name = "p"}) {
 ! CHECK: %[[LOAD:.*]] = fir.load %[[ARG0]] : !fir.ref<!fir.class<!fir.heap<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>
 ! CHECK: %[[REBOX:.*]] = fir.rebox %[[LOAD]] : (!fir.class<!fir.heap<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>) -> !fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>
-! CHECK: fir.dispatch "tbp_pass"(%[[REBOX]] : !fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>) (%1 : !fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>) {pass_arg_pos = 0 : i32}
+! CHECK: fir.dispatch "tbp_pass"(%[[LOAD]] : !fir.class<!fir.heap<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>) (%1 : !fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>) {pass_arg_pos = 0 : i32}
 
     subroutine check_dispatch_scalar_pointer(p)
       class(p1), pointer :: p
@@ -193,7 +193,7 @@ subroutine check_dispatch_scalar_pointer(p)
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.class<!fir.ptr<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>> {fir.bindc_name = "p"}) {
 ! CHECK: %[[LOAD:.*]] = fir.load %[[ARG0]] : !fir.ref<!fir.class<!fir.ptr<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>>
 ! CHECK: %[[REBOX:.*]] = fir.rebox %[[LOAD]] : (!fir.class<!fir.ptr<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>) -> !fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>
-! CHECK: fir.dispatch "tbp_pass"(%[[REBOX]] : !fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>) (%1 : !fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>) {pass_arg_pos = 0 : i32}
+! CHECK: fir.dispatch "tbp_pass"(%[[LOAD]] : !fir.class<!fir.ptr<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>>) (%1 : !fir.class<!fir.type<_QMcall_dispatchTp1{a:i32,b:i32}>>) {pass_arg_pos = 0 : i32}
 
     subroutine check_dispatch_static_array(p, t)
       class(p1) :: p(10)
diff --git a/flang/test/Lower/pointer-association-polymorphic.f90 b/flang/test/Lower/pointer-association-polymorphic.f90
index fa3091d9ff..0f5fdd66aa 100644
--- a/flang/test/Lower/pointer-association-polymorphic.f90
+++ b/flang/test/Lower/pointer-association-polymorphic.f90
@@ -90,7 +90,7 @@ subroutine test_pointer()
 ! CHECK: %{{.*}} = fir.call @_FortranAPointerAssociate(%[[P_CONV]], %[[C1_DESC_CONV]]) {{.*}} : (!fir.ref<!fir.box<none>>, !fir.box<none>) -> none
 ! CHECK: %[[P_DESC_LOAD:.*]] = fir.load %[[P_DESC]] : !fir.ref<!fir.class<!fir.ptr<!fir.type<_QMpolyTp1{a:i32,b:i32}>>>>
 ! CHECK: %[[P_REBOX:.*]] = fir.rebox %[[P_DESC_LOAD]] : (!fir.class<!fir.ptr<!fir.type<_QMpolyTp1{a:i32,b:i32}>>>) -> !fir.class<!fir.type<_QMpolyTp1{a:i32,b:i32}>>
-! CHECK: fir.dispatch "proc"(%[[P_REBOX]] : !fir.class<!fir.type<_QMpolyTp1{a:i32,b:i32}>>) (%[[P_REBOX]] : !fir.class<!fir.type<_QMpolyTp1{a:i32,b:i32}>>) {pass_arg_pos = 0 : i32}
+! CHECK: fir.dispatch "proc"(%[[P_DESC_LOAD]] : !fir.class<!fir.ptr<!fir.type<_QMpolyTp1{a:i32,b:i32}>>>) (%[[P_REBOX]] : !fir.class<!fir.type<_QMpolyTp1{a:i32,b:i32}>>) {pass_arg_pos = 0 : i32}
 
 ! CHECK: %[[C2_DESC_LOAD:.*]] = fir.load %[[C2_DESC]] : !fir.ref<!fir.class<!fir.heap<!fir.type<_QMpolyTp1{a:i32,b:i32}>>>>
 ! CHECK: %[[P_CONV:.*]] = fir.convert %[[P_DESC]] : (!fir.ref<!fir.class<!fir.ptr<!fir.type<_QMpolyTp1{a:i32,b:i32}>>>>) -> !fir.ref<!fir.box<none>>
@@ -98,7 +98,7 @@ subroutine test_pointer()
 ! CHECK: %{{.*}} = fir.call @_FortranAPointerAssociate(%[[P_CONV]], %[[C2_DESC_CONV]]) {{.*}} : (!fir.ref<!fir.box<none>>, !fir.box<none>) -> none
 ! CHECK: %[[P_DESC_LOAD:.*]] = fir.load %[[P_DESC]] : !fir.ref<!fir.class<!fir.ptr<!fir.type<_QMpolyTp1{a:i32,b:i32}>>>>
 ! CHECK: %[[P_REBOX:.*]] = fir.rebox %[[P_DESC_LOAD]] : (!fir.class<!fir.ptr<!fir.type<_QMpolyTp1{a:i32,b:i32}>>>) -> !fir.class<!fir.type<_QMpolyTp1{a:i32,b:i32}>>
-! CHECK: fir.dispatch "proc"(%[[P_REBOX]] : !fir.class<!fir.type<_QMpolyTp1{a:i32,b:i32}>>) (%[[P_REBOX]] : !fir.class<!fir.type<_QMpolyTp1{a:i32,b:i32}>>) {pass_arg_pos = 0 : i32}
+! CHECK: fir.dispatch "proc"(%[[P_DESC_LOAD]] : !fir.class<!fir.ptr<!fir.type<_QMpolyTp1{a:i32,b:i32}>>>) (%[[P_REBOX]] : !fir.class<!fir.type<_QMpolyTp1{a:i32,b:i32}>>) {pass_arg_pos = 0 : i32}
 
 ! CHECK: %[[C3_LOAD:.*]] = fir.load %[[C3_DESC]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMpolyTp1{a:i32,b:i32}>>>>>
 ! CHECK: %[[C0:.*]] = arith.constant 0 : index
@@ -113,7 +113,7 @@ subroutine test_pointer()
 ! CHECK: %{{.*}} = fir.call @_FortranAPointerAssociate(%[[P_CONV]], %[[C3_EMBOX_CONV]]) {{.*}} : (!fir.ref<!fir.box<none>>, !fir.box<none>) -> none
 ! CHECK: %[[P_DESC_LOAD:.*]] = fir.load %[[P_DESC]] : !fir.ref<!fir.class<!fir.ptr<!fir.type<_QMpolyTp1{a:i32,b:i32}>>>>
 ! CHECK: %[[P_REBOX:.*]] = fir.rebox %[[P_DESC_LOAD]] : (!fir.class<!fir.ptr<!fir.type<_QMpolyTp1{a:i32,b:i32}>>>) -> !fir.class<!fir.type<_QMpolyTp1{a:i32,b:i32}>>
-! CHECK: fir.dispatch "proc"(%[[P_REBOX]] : !fir.class<!fir.type<_QMpolyTp1{a:i32,b:i32}>>) (%[[P_REBOX]] : !fir.class<!fir.type<_QMpolyTp1{a:i32,b:i32}>>) {pass_arg_pos = 0 : i32}
+! CHECK: fir.dispatch "proc"(%[[P_DESC_LOAD]] : !fir.class<!fir.ptr<!fir.type<_QMpolyTp1{a:i32,b:i32}>>>) (%[[P_REBOX]] : !fir.class<!fir.type<_QMpolyTp1{a:i32,b:i32}>>) {pass_arg_pos = 0 : i32}
 
 ! CHECK: %[[C4_LOAD:.*]] = fir.load %[[C4_DESC]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMpolyTp1{a:i32,b:i32}>>>>>
 ! CHECK: %[[C0:.*]] = arith.constant 0 : index
@@ -128,7 +128,7 @@ subroutine test_pointer()
 ! CHECK: %{{.*}} = fir.call @_FortranAPointerAssociate(%[[P_CONV]], %[[C4_EMBOX_CONV]]) {{.*}} : (!fir.ref<!fir.box<none>>, !fir.box<none>) -> none
 ! CHECK: %[[P_DESC_LOAD:.*]] = fir.load %[[P_DESC]] : !fir.ref<!fir.class<!fir.ptr<!fir.type<_QMpolyTp1{a:i32,b:i32}>>>>
 ! CHECK: %[[P_REBOX:.*]] = fir.rebox %[[P_DESC_LOAD]] : (!fir.class<!fir.ptr<!fir.type<_QMpolyTp1{a:i32,b:i32}>>>) -> !fir.class<!fir.type<_QMpolyTp1{a:i32,b:i32}>>
-! CHECK: fir.dispatch "proc"(%[[P_REBOX]] : !fir.class<!fir.type<_QMpolyTp1{a:i32,b:i32}>>) (%[[P_REBOX]] : !fir.class<!fir.type<_QMpolyTp1{a:i32,b:i32}>>) {pass_arg_pos = 0 : i32}
+! CHECK: fir.dispatch "proc"(%[[P_DESC_LOAD]] : !fir.class<!fir.ptr<!fir.type<_QMpolyTp1{a:i32,b:i32}>>>) (%[[P_REBOX]] : !fir.class<!fir.type<_QMpolyTp1{a:i32,b:i32}>>) {pass_arg_pos = 0 : i32}
 
 ! CHECK: %[[C3_LOAD:.*]] = fir.load %[[C3_DESC]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMpolyTp1{a:i32,b:i32}>>>>>
 ! CHECK: %[[C3_REBOX:.*]] = fir.rebox %[[C3_LOAD]](%{{.*}}) : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMpolyTp1{a:i32,b:i32}>>>>, !fir.shift<1>) -> !fir.class<!fir.array<?x!fir.type<_QMpolyTp1{a:i32,b:i32}>>>
diff --git a/flang/test/Semantics/OpenACC/acc-error.f90 b/flang/test/Semantics/OpenACC/acc-error.f90
new file mode 100644
index 0000000000..69ee59f97e
--- /dev/null
+++ b/flang/test/Semantics/OpenACC/acc-error.f90
@@ -0,0 +1,29 @@
+! RUN: %python %S/../test_errors.py %s %flang -fopenacc
+
+! Check parser specific error for OpenACC
+
+
+subroutine test(a, n)
+    integer :: a(n)
+    !ERROR: expected OpenACC directive
+    !$acc p
+    integer :: i,j
+ 
+    i = 0
+    !ERROR: expected OpenACC directive
+    !$acc p
+  end subroutine
+
+subroutine test2(a, n)
+  integer :: a(n)
+  integer :: i
+
+  !$acc parallel
+  !$acc loop
+  DO i = 1, n
+  END DO
+  !$acc end parallel
+  !WARN: Misplaced OpenACC end directive
+  !$acc end loop
+
+end subroutine
diff --git a/flang/test/Semantics/OpenACC/acc-init-validity.f90 b/flang/test/Semantics/OpenACC/acc-init-validity.f90
index 278211492c..f54898f73f 100644
--- a/flang/test/Semantics/OpenACC/acc-init-validity.f90
+++ b/flang/test/Semantics/OpenACC/acc-init-validity.f90
@@ -10,11 +10,14 @@ program openacc_init_validity
   integer :: i, j
   integer, parameter :: N = 256
   logical :: ifCondition = .TRUE.
+  integer :: ifInt
+  real :: ifReal
   real(8), dimension(N) :: a
 
   !$acc init
   !$acc init if(.TRUE.)
   !$acc init if(ifCondition)
+  !$acc init if(ifInt)
   !$acc init device_num(1)
   !$acc init device_num(i)
   !$acc init device_type(i)
@@ -93,4 +96,7 @@ program openacc_init_validity
   !ERROR: At most one DEVICE_TYPE clause can appear on the INIT directive
   !$acc init device_type(2) device_type(i, j)
 
+  !ERROR: Must have LOGICAL or INTEGER type
+  !$acc init if(ifReal)
+
 end program openacc_init_validity
diff --git a/flang/test/Semantics/OpenACC/acc-routine-validity.f90 b/flang/test/Semantics/OpenACC/acc-routine-validity.f90
index 5b8ecdec4f..c135c2b86a 100644
--- a/flang/test/Semantics/OpenACC/acc-routine-validity.f90
+++ b/flang/test/Semantics/OpenACC/acc-routine-validity.f90
@@ -15,7 +15,6 @@ module openacc_routine_validity
   !ERROR: ROUTINE directive without name must appear within the specification part of a subroutine or function definition, or within an interface body for a subroutine or function in an interface block
   !$acc routine seq
 
-  !ERROR: No function or subroutine declared for 'dummy'
   !$acc routine(dummy) seq
 
 contains
@@ -70,7 +69,6 @@ end function fct4
 
   subroutine sub6(a)
     real :: a(:)
-    !ERROR: No function or subroutine declared for 'dummy_sub'
     !$acc routine seq bind(dummy_sub)
   end subroutine sub6
 
diff --git a/flang/test/Semantics/OpenACC/acc-serial.f90 b/flang/test/Semantics/OpenACC/acc-serial.f90
index a052ef4e47..afcfc00a40 100644
--- a/flang/test/Semantics/OpenACC/acc-serial.f90
+++ b/flang/test/Semantics/OpenACC/acc-serial.f90
@@ -77,15 +77,15 @@ program openacc_serial_validity
   !$acc serial wait(wait1) wait(wait2)
   !$acc end serial
 
-  !ERROR: NUM_GANGS clause is not allowed on the SERIAL directive
+  !PORTABILITY: NUM_GANGS clause is not allowed on the SERIAL directive and will be ignored
   !$acc serial num_gangs(8)
   !$acc end serial
 
-  !ERROR: NUM_WORKERS clause is not allowed on the SERIAL directive
+  !PORTABILITY: NUM_WORKERS clause is not allowed on the SERIAL directive and will be ignored
   !$acc serial num_workers(8)
   !$acc end serial
 
-  !ERROR: VECTOR_LENGTH clause is not allowed on the SERIAL directive
+  !PORTABILITY: VECTOR_LENGTH clause is not allowed on the SERIAL directive and will be ignored
   !$acc serial vector_length(128)
   !$acc end serial
 
diff --git a/flang/test/Semantics/OpenACC/acc-symbols01.f90 b/flang/test/Semantics/OpenACC/acc-symbols01.f90
index ddb87711ee..375445bad1 100644
--- a/flang/test/Semantics/OpenACC/acc-symbols01.f90
+++ b/flang/test/Semantics/OpenACC/acc-symbols01.f90
@@ -14,11 +14,11 @@ program mm
   b = 2
  !$acc parallel present(c) firstprivate(b) private(a)
  !$acc loop
-  !DEF: /mm/OtherConstruct1/i (AccPrivate, AccPreDetermined) HostAssoc INTEGER(4)
+  !REF: /mm/i
   do i=1,10
-   !DEF: /mm/OtherConstruct1/a (AccPrivate) HostAssoc INTEGER(4)
-   !REF: /mm/OtherConstruct1/i
-   !DEF: /mm/OtherConstruct1/b (AccFirstPrivate) HostAssoc INTEGER(4)
+   !REF: /mm/a
+   !REF: /mm/i
+   !REF: /mm/b
    a(i) = b(i)
   end do
  !$acc end parallel
diff --git a/flang/test/Semantics/OpenMP/do05-positivecase.f90 b/flang/test/Semantics/OpenMP/do05-positivecase.f90
index b1d97b4456..4e02235f58 100644
--- a/flang/test/Semantics/OpenMP/do05-positivecase.f90
+++ b/flang/test/Semantics/OpenMP/do05-positivecase.f90
@@ -33,4 +33,13 @@ program omp_do
   !$omp end do
   !$omp end parallel
 
+  !$omp parallel private(i)
+  !DEF: /omp_do/OtherConstruct3/i (OmpPrivate) HostAssoc INTEGER(4)
+  do i=1,10
+     !$omp single
+     print *, "hello"
+     !$omp end single
+  end do
+  !$omp end parallel
+
 end program omp_do
diff --git a/flang/test/Semantics/OpenMP/requires-atomic01.f90 b/flang/test/Semantics/OpenMP/requires-atomic01.f90
new file mode 100644
index 0000000000..b39c9cdcc0
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/requires-atomic01.f90
@@ -0,0 +1,109 @@
+! RUN: %flang_fc1 -fopenmp -fdebug-dump-parse-tree %s 2>&1 | FileCheck %s
+! Ensure that requires atomic_default_mem_order is used to update atomic
+! operations with no explicit memory order set.
+program requires
+  implicit none
+  !$omp requires atomic_default_mem_order(seq_cst)
+  integer :: i, j
+
+  ! ----------------------------------------------------------------------------
+  ! READ
+  ! ----------------------------------------------------------------------------
+
+  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicRead
+  ! CHECK: OmpMemoryOrderClause -> OmpClause -> SeqCst
+  !$omp atomic read
+  i = j
+
+  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicRead
+  ! CHECK-NOT: OmpMemoryOrderClause -> OmpClause -> SeqCst
+  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Relaxed
+  !$omp atomic relaxed read
+  i = j
+
+  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicRead
+  ! CHECK-NOT: OmpMemoryOrderClause -> OmpClause -> SeqCst
+  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Relaxed
+  !$omp atomic read relaxed
+  i = j
+  
+  ! ----------------------------------------------------------------------------
+  ! WRITE
+  ! ----------------------------------------------------------------------------
+
+  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicWrite
+  ! CHECK: OmpMemoryOrderClause -> OmpClause -> SeqCst
+  !$omp atomic write
+  i = j
+
+  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicWrite
+  ! CHECK-NOT: OmpMemoryOrderClause -> OmpClause -> SeqCst
+  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Relaxed
+  !$omp atomic relaxed write
+  i = j
+
+  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicWrite
+  ! CHECK-NOT: OmpMemoryOrderClause -> OmpClause -> SeqCst
+  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Relaxed
+  !$omp atomic write relaxed
+  i = j
+
+  ! ----------------------------------------------------------------------------
+  ! UPDATE
+  ! ----------------------------------------------------------------------------
+
+  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicUpdate
+  ! CHECK: OmpMemoryOrderClause -> OmpClause -> SeqCst
+  !$omp atomic update
+  i = i + j
+
+  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicUpdate
+  ! CHECK-NOT: OmpMemoryOrderClause -> OmpClause -> SeqCst
+  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Relaxed
+  !$omp atomic relaxed update
+  i = i + j
+
+  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicUpdate
+  ! CHECK-NOT: OmpMemoryOrderClause -> OmpClause -> SeqCst
+  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Relaxed
+  !$omp atomic update relaxed
+  i = i + j
+
+  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomic
+  ! CHECK: OmpMemoryOrderClause -> OmpClause -> SeqCst
+  !$omp atomic
+  i = i + j
+
+  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomic
+  ! CHECK-NOT: OmpMemoryOrderClause -> OmpClause -> SeqCst
+  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Relaxed
+  !$omp atomic relaxed
+  i = i + j
+
+  ! ----------------------------------------------------------------------------
+  ! CAPTURE
+  ! ----------------------------------------------------------------------------
+
+  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicCapture
+  ! CHECK: OmpMemoryOrderClause -> OmpClause -> SeqCst
+  !$omp atomic capture
+  i = j
+  i = j
+  !$omp end atomic
+
+  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicCapture
+  ! CHECK-NOT: OmpMemoryOrderClause -> OmpClause -> SeqCst
+  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Relaxed
+  !$omp atomic relaxed capture
+  i = j
+  i = j
+  !$omp end atomic
+
+  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicCapture
+  ! CHECK-NOT: OmpMemoryOrderClause -> OmpClause -> SeqCst
+  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Relaxed
+  !$omp atomic capture relaxed
+  i = j
+  i = j
+  !$omp end atomic
+end program requires
diff --git a/flang/test/Semantics/OpenMP/requires-atomic02.f90 b/flang/test/Semantics/OpenMP/requires-atomic02.f90
new file mode 100644
index 0000000000..3af83970e7
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/requires-atomic02.f90
@@ -0,0 +1,109 @@
+! RUN: %flang_fc1 -fopenmp -fdebug-dump-parse-tree %s 2>&1 | FileCheck %s
+! Ensure that requires atomic_default_mem_order is used to update atomic
+! operations with no explicit memory order set. ACQ_REL clause tested here.
+program requires
+  implicit none
+  !$omp requires atomic_default_mem_order(acq_rel)
+  integer :: i, j
+
+  ! ----------------------------------------------------------------------------
+  ! READ
+  ! ----------------------------------------------------------------------------
+
+  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicRead
+  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Acquire
+  !$omp atomic read
+  i = j
+
+  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicRead
+  ! CHECK-NOT: OmpMemoryOrderClause -> OmpClause -> Acquire
+  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Relaxed
+  !$omp atomic relaxed read
+  i = j
+
+  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicRead
+  ! CHECK-NOT: OmpMemoryOrderClause -> OmpClause -> Acquire
+  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Relaxed
+  !$omp atomic read relaxed
+  i = j
+  
+  ! ----------------------------------------------------------------------------
+  ! WRITE
+  ! ----------------------------------------------------------------------------
+
+  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicWrite
+  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Release
+  !$omp atomic write
+  i = j
+
+  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicWrite
+  ! CHECK-NOT: OmpMemoryOrderClause -> OmpClause -> Release
+  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Relaxed
+  !$omp atomic relaxed write
+  i = j
+
+  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicWrite
+  ! CHECK-NOT: OmpMemoryOrderClause -> OmpClause -> Release
+  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Relaxed
+  !$omp atomic write relaxed
+  i = j
+
+  ! ----------------------------------------------------------------------------
+  ! UPDATE
+  ! ----------------------------------------------------------------------------
+
+  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicUpdate
+  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Release
+  !$omp atomic update
+  i = i + j
+
+  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicUpdate
+  ! CHECK-NOT: OmpMemoryOrderClause -> OmpClause -> Release
+  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Relaxed
+  !$omp atomic relaxed update
+  i = i + j
+
+  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicUpdate
+  ! CHECK-NOT: OmpMemoryOrderClause -> OmpClause -> Release
+  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Relaxed
+  !$omp atomic update relaxed
+  i = i + j
+
+  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomic
+  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Release
+  !$omp atomic
+  i = i + j
+
+  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomic
+  ! CHECK-NOT: OmpMemoryOrderClause -> OmpClause -> Release
+  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Relaxed
+  !$omp atomic relaxed
+  i = i + j
+
+  ! ----------------------------------------------------------------------------
+  ! CAPTURE
+  ! ----------------------------------------------------------------------------
+
+  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicCapture
+  ! CHECK: OmpMemoryOrderClause -> OmpClause -> AcqRel
+  !$omp atomic capture
+  i = j
+  i = j
+  !$omp end atomic
+
+  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicCapture
+  ! CHECK-NOT: OmpMemoryOrderClause -> OmpClause -> AcqRel
+  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Relaxed
+  !$omp atomic relaxed capture
+  i = j
+  i = j
+  !$omp end atomic
+
+  ! CHECK-LABEL: OpenMPAtomicConstruct -> OmpAtomicCapture
+  ! CHECK-NOT: OmpMemoryOrderClause -> OmpClause -> AcqRel
+  ! CHECK: OmpMemoryOrderClause -> OmpClause -> Relaxed
+  !$omp atomic capture relaxed
+  i = j
+  i = j
+  !$omp end atomic
+end program requires
diff --git a/flang/test/Semantics/OpenMP/symbol08.f90 b/flang/test/Semantics/OpenMP/symbol08.f90
index b80c94a9f1..50f34b736c 100644
--- a/flang/test/Semantics/OpenMP/symbol08.f90
+++ b/flang/test/Semantics/OpenMP/symbol08.f90
@@ -67,7 +67,7 @@ subroutine test_pardo
     do j=6,10
    !REF: /test_pardo/a
    a(1,1,1) = 0.
-   !DEF: /test_pardo/OtherConstruct1/k (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+   !DEF: /test_pardo/OtherConstruct1/k (OmpPrivate) HostAssoc INTEGER(4)
    do k=11,15
     !REF: /test_pardo/a
     !REF: /test_pardo/OtherConstruct1/k
@@ -91,7 +91,7 @@ subroutine test_taskloop
 !$omp taskloop  private(j)
  !DEF: /test_taskloop/OtherConstruct1/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
  do i=1,5
-  !DEF: /test_taskloop/OtherConstruct1/j (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  !DEF: /test_taskloop/OtherConstruct1/j (OmpPrivate) HostAssoc INTEGER(4)
   !REF: /test_taskloop/OtherConstruct1/i
   do j=1,i
    !REF: /test_taskloop/a
diff --git a/flang/test/Semantics/OpenMP/threadprivate06.f90 b/flang/test/Semantics/OpenMP/threadprivate06.f90
new file mode 100644
index 0000000000..f31c38f6f2
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/threadprivate06.f90
@@ -0,0 +1,30 @@
+! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+! OpenMP Version 5.1
+! Check OpenMP construct validity for the following directives:
+! 2.21.2 Threadprivate Directive
+
+program main
+  call sub1()
+  print *, 'pass'
+end program main
+
+subroutine sub1()
+  common /c/ a
+  !$omp threadprivate(/c/)
+  integer :: a
+
+  a = 100
+  call sub2()
+  if (a .ne. 101) print *, 'err'
+
+contains
+  subroutine sub2()
+    common /c/ a
+    !$omp threadprivate(/c/)
+    integer :: a
+
+    !$omp parallel copyin(/c/)
+      a = a + 1
+    !$omp end parallel
+  end subroutine
+end subroutine
diff --git a/flang/test/Semantics/dosemantics13.f90 b/flang/test/Semantics/dosemantics13.f90
new file mode 100644
index 0000000000..c2f4376dee
--- /dev/null
+++ b/flang/test/Semantics/dosemantics13.f90
@@ -0,0 +1,29 @@
+! RUN: %python %S/test_errors.py %s %flang_fc1
+program main
+
+  integer j, k
+
+  lab1: do j=1,10
+    cycle lab1
+    exit lab1
+  end do lab1
+
+  lab2: do 2 j=1,10
+    cycle lab2
+    exit lab2
+  2 end do lab2
+
+  lab3: do 3 j=1,10
+    cycle lab3
+    exit lab3
+    !ERROR: DO construct name required but missing
+  3 end do
+
+  do 4 j=1,10
+  !ERROR: Unexpected DO construct name 'lab4'
+  4 end do lab4
+
+  lab5: do 5 j=1,10
+  !ERROR: END DO statement must have the label '5' matching its DO statement
+  666 end do lab5
+end
diff --git a/flang/test/Semantics/dosemantics14.f90 b/flang/test/Semantics/dosemantics14.f90
new file mode 100644
index 0000000000..e1631a749e
--- /dev/null
+++ b/flang/test/Semantics/dosemantics14.f90
@@ -0,0 +1,12 @@
+! RUN: %python %S/test_errors.py %s %flang_fc1
+subroutine bad1
+  lab1: do 1 j=1,10
+  1 continue
+!ERROR: expected 'END DO'
+end
+
+subroutine bad2
+  lab2: do 2 j=1,10
+  2 k = j
+!ERROR: expected 'END DO'
+end
diff --git a/flang/test/Semantics/modproc01.f90 b/flang/test/Semantics/modproc01.f90
index c7d0578333..5652e15750 100644
--- a/flang/test/Semantics/modproc01.f90
+++ b/flang/test/Semantics/modproc01.f90
@@ -22,11 +22,12 @@ module subroutine ms(f)
       procedure(mf) :: f
     end subroutine
   end interface
+  integer sm
 end module
 !CHECK:    mf, MODULE, PUBLIC (Function): Subprogram isInterface result:TYPE(pdt2(k2=2_4,l2=n)) res (INTEGER(4) n,CHARACTER(n,1) str,TYPE(pdt1(k1=1_4,l1=n)) x1)
 !CHECK:    pdt1, PUBLIC: DerivedType components: a1
 !CHECK:    pdt2, PUBLIC: DerivedType components: j2,a2
-!CHECK:    sm: Module (m)
+!CHECK:    sm, PUBLIC size=4 offset=0: ObjectEntity type: INTEGER(4)
 !CHECK:    DerivedType scope: pdt1
 !CHECK:      a1, ALLOCATABLE: ObjectEntity type: TYPE(pdt2(int(k1,kind=4),int(l1,kind=4)))
 !CHECK:      k1: TypeParam type:INTEGER(4) Kind
@@ -128,6 +129,7 @@ program test
 !CHECK:    mf, MODULE (Function): Use from mf in m
 !CHECK:    pdt1: Use from pdt1 in m
 !CHECK:    pdt2: Use from pdt2 in m
+!CHECK:    sm: Use from sm in m
 !CHECK:    x size=88 offset=0: ObjectEntity type: TYPE(pdt2(k2=2_4,l2=3_4))
 !CHECK:    DerivedType scope: size=88 alignment=8 instantiation of pdt2(k2=2_4,l2=3_4)
 !CHECK:      a2 size=80 offset=8: ObjectEntity type: TYPE(pdt1(k1=2_4,l1=3_4)) shape: 1_8:2_8
diff --git a/flang/test/Semantics/modproc02.f90 b/flang/test/Semantics/modproc02.f90
index 229ef72e6b..f47f473f08 100644
--- a/flang/test/Semantics/modproc02.f90
+++ b/flang/test/Semantics/modproc02.f90
@@ -16,7 +16,6 @@ module subroutine s(x) ! implicitly typed
 
 !CHECK:  Module scope: m size=0 alignment=1 sourceRange=63 bytes
 !CHECK:    s, MODULE, PUBLIC (Subroutine): Subprogram isInterface (REAL(4) x)
-!CHECK:    sm: Module (m)
 !CHECK:    Subprogram scope: s size=4 alignment=4 sourceRange=26 bytes
 !CHECK:      s (Subroutine): HostAssoc
 !CHECK:      x (Implicit) size=4 offset=0: ObjectEntity dummy type: REAL(4)
diff --git a/flang/test/Semantics/offsets02.f90 b/flang/test/Semantics/offsets02.f90
index 387bbac5ff..11e086cf68 100644
--- a/flang/test/Semantics/offsets02.f90
+++ b/flang/test/Semantics/offsets02.f90
@@ -8,11 +8,17 @@ subroutine s1
     real(8) :: a
     real(4) :: b
   end type
-  !CHECK: x1 size=12 offset=0:
-  !CHECK: y1 size=12 offset=16:
+  type t2
+    type(t1) c
+    real(4) d
+  end type
+  !CHECK: x1 size=16 offset=0:
+  !CHECK: y1 size=16 offset=16:
   type(t1) :: x1, y1
   !CHECK: z1 size=160 offset=32:
   type(t1) :: z1(10)
+  !CHECK: z2 size=24 offset=192
+  type(t2) z2
 end
 
 ! Like t1 but t2 does not need to be aligned on 64-bit boundary
diff --git a/flang/test/Semantics/reshape.f90 b/flang/test/Semantics/reshape.f90
index 2e9b5adf3f..fb5e0023e2 100644
--- a/flang/test/Semantics/reshape.f90
+++ b/flang/test/Semantics/reshape.f90
@@ -44,6 +44,11 @@ program reshaper
   type(dType), parameter :: array19(*) = [dType::dType(field=[1,2])]
   logical, parameter :: lVar = all(array19(:)%field(1) == [2])
 
+  ! RESHAPE on array with maximum valid upper bound
+  integer(8), parameter :: I64_MAX = INT(z'7fffffffffffffff', kind=8)
+  integer, parameter :: array21(I64_MAX - 2 : I64_MAX) = [1, 2, 3]
+  integer, parameter :: array22(2) = RESHAPE(array21, [2])
+
   !ERROR: Size of 'shape=' argument must not be greater than 15
   CALL ext_sub(RESHAPE([(n, n=1,20)], &
     [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]))
diff --git a/flang/test/Semantics/resolve29.f90 b/flang/test/Semantics/resolve29.f90
index ea4642c1b3..3e6a8a0ba6 100644
--- a/flang/test/Semantics/resolve29.f90
+++ b/flang/test/Semantics/resolve29.f90
@@ -9,6 +9,7 @@ subroutine s1(x)
       !ERROR: 't1' from host is not accessible
       import :: t1
       type(t1) :: x
+      !BECAUSE: 't1' is hidden by this entity
       integer :: t1
     end subroutine
     subroutine s2()
@@ -24,6 +25,7 @@ subroutine s4(x, y)
       import, all
       type(t1) :: x
       type(t3) :: y
+      !BECAUSE: 't3' is hidden by this entity
       integer :: t3
     end subroutine
   end interface
@@ -41,6 +43,27 @@ subroutine s7()
     !ERROR: 's5' is an external procedure without the EXTERNAL attribute in a scope with IMPLICIT NONE(EXTERNAL)
     call s5()
   end
+  subroutine s8()
+    !This case is a dangerous ambiguity allowed by the standard.
+    !ERROR: 't1' from host is not accessible
+    type(t1), pointer :: p
+    !BECAUSE: 't1' is hidden by this entity
+    type t1
+      integer n(2)
+    end type
+  end
+  subroutine s9()
+    !This case is a dangerous ambiguity allowed by the standard.
+    type t2
+      !ERROR: 't1' from host is not accessible
+      type(t1), pointer :: p
+    end type
+    !BECAUSE: 't1' is hidden by this entity
+    type t1
+      integer n(2)
+    end type
+    type(t2) x
+  end
 end module
 module m2
   integer, parameter :: ck = kind('a')
diff --git a/flang/test/Semantics/separate-mp05.f90 b/flang/test/Semantics/separate-mp05.f90
new file mode 100644
index 0000000000..5b7e2523a2
--- /dev/null
+++ b/flang/test/Semantics/separate-mp05.f90
@@ -0,0 +1,40 @@
+! RUN: %python %S/test_symbols.py %s %flang_fc1
+! Ensure that SMPs work with dummy procedures declared as interfaces
+!DEF: /m Module
+module m
+ implicit none
+ interface
+  !DEF: /m/smp MODULE, PUBLIC, PURE (Function) Subprogram REAL(4)
+  !DEF: /m/smp/f EXTERNAL, PURE (Function) Subprogram REAL(4)
+  !DEF: /m/smp/x INTENT(IN) ObjectEntity REAL(4)
+  !DEF: /m/smp/res (Implicit) ObjectEntity REAL(4)
+  pure module function smp(f, x) result(res)
+   interface
+    !REF: /m/smp/f
+    !DEF: /m/smp/f/x INTENT(IN) ObjectEntity REAL(4)
+    !DEF: /m/smp/f/r ObjectEntity REAL(4)
+    pure function f(x) result(r)
+     !REF: /m/smp/f/x
+     real, intent(in) :: x
+     !REF: /m/smp/f/r
+     real r
+    end function
+   end interface
+   !REF: /m/smp/x
+   real, intent(in) :: x
+  end function
+ end interface
+end module
+!REF: /m
+!DEF: /m/sm Module
+submodule (m)sm
+ implicit none
+contains
+ !DEF: /m/sm/smp MODULE, PUBLIC, PURE (Function) Subprogram REAL(4)
+ module procedure smp
+  !DEF: /m/sm/smp/res (Implicit) ObjectEntity REAL(4)
+  !DEF: /m/sm/smp/f EXTERNAL, PURE (Function) Subprogram REAL(4)
+  !DEF: /m/sm/smp/x INTENT(IN) ObjectEntity REAL(4)
+  res = f(x)
+ end procedure
+end submodule
diff --git a/flang/test/Transforms/loop-versioning.fir b/flang/test/Transforms/loop-versioning.fir
index f2768d7325..6313bc2ac3 100644
--- a/flang/test/Transforms/loop-versioning.fir
+++ b/flang/test/Transforms/loop-versioning.fir
@@ -1434,4 +1434,48 @@ func.func @_QPtest_loop_nest(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name
 // CHECK: fir.if
 // CHECK-NOT: fir.if
 
+
+//-----
+
+// Check that a non-dominating operand isn't causing a problem.
+// Just check it compiles, and doesn't version this loop.
+func.func @sum1drebox(%arg0: !fir.box<!fir.array<?xf64>> {fir.bindc_name = "a"}, %arg1: !fir.ref<i32> {fir.bindc_name = "n"}) {
+  %decl = fir.declare %arg0 {uniq_name = "a"} : (!fir.box<!fir.array<?xf64>>) -> !fir.box<!fir.array<?xf64>>
+  %0 = fir.alloca i32 {bindc_name = "i", uniq_name = "_QMmoduleFsum1dEi"}
+  %1 = fir.alloca f64 {bindc_name = "sum", uniq_name = "_QMmoduleFsum1dEsum"}
+  %cst = arith.constant 0.000000e+00 : f64
+  fir.store %cst to %1 : !fir.ref<f64>
+  %c1_i32 = arith.constant 1 : i32
+  %2 = fir.convert %c1_i32 : (i32) -> index
+  %3 = fir.load %arg1 : !fir.ref<i32>
+  %4 = fir.convert %3 : (i32) -> index
+  %c1 = arith.constant 1 : index
+  %5 = fir.convert %2 : (index) -> i32
+  %6:2 = fir.do_loop %arg2 = %2 to %4 step %c1 iter_args(%arg3 = %5) -> (index, i32) {
+    // rebox is not dominating the loop.
+    %rebox = fir.rebox %decl : (!fir.box<!fir.array<?xf64>>) -> !fir.box<!fir.array<?xf64>>
+    fir.store %arg3 to %0 : !fir.ref<i32>
+    %7 = fir.load %1 : !fir.ref<f64>
+    %8 = fir.load %0 : !fir.ref<i32>
+    %9 = fir.convert %8 : (i32) -> i64
+    %c1_i64 = arith.constant 1 : i64
+    %10 = arith.subi %9, %c1_i64 : i64
+    %11 = fir.coordinate_of %rebox, %10 : (!fir.box<!fir.array<?xf64>>, i64) -> !fir.ref<f64>
+    %12 = fir.load %11 : !fir.ref<f64>
+    %13 = arith.addf %7, %12 fastmath<contract> : f64
+    fir.store %13 to %1 : !fir.ref<f64>
+    %14 = arith.addi %arg2, %c1 : index
+    %15 = fir.convert %c1 : (index) -> i32
+    %16 = fir.load %0 : !fir.ref<i32>
+    %17 = arith.addi %16, %15 : i32
+    fir.result %14, %17 : index, i32
+  }
+  fir.store %6#1 to %0 : !fir.ref<i32>
+  return
+}
+// CHECK-LABEL: func @sum1drebox
+// No versioning -> no if-operation.
+// CHECK-NOT: fir.if
+
+
 } // End module
diff --git a/flang/test/Transforms/tbaa.fir b/flang/test/Transforms/tbaa.fir
new file mode 100644
index 0000000000..7825ae60c7
--- /dev/null
+++ b/flang/test/Transforms/tbaa.fir
@@ -0,0 +1,175 @@
+// RUN: fir-opt --split-input-file --fir-add-alias-tags %s | FileCheck %s
+
+// subroutine oneArg(a)
+//   integer :: a(:)
+//   a(1) = a(2)
+// end subroutine
+  func.func @_QPonearg(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"}) {
+    %c1 = arith.constant 1 : index
+    %c2 = arith.constant 2 : index
+    %0 = fir.declare %arg0 {uniq_name = "_QFoneargEa"} : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+    %1 = fir.rebox %0 : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+    %2 = fir.array_coor %1 %c2 : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+    %3 = fir.load %2 : !fir.ref<i32>
+    %4 = fir.array_coor %1 %c1 : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+    fir.store %3 to %4 : !fir.ref<i32>
+    return
+  }
+
+// CHECK: #[[ONE_ARG_ROOT:.+]] = #llvm.tbaa_root<id = "Flang function root _QPonearg">
+// CHECK: #[[ONE_ARG_ANY_ACCESS:.+]] = #llvm.tbaa_type_desc<id = "any access", members = {<#[[ONE_ARG_ROOT]], 0>}>
+// CHECK: #[[ONE_ARG_ANY_DATA:.+]] = #llvm.tbaa_type_desc<id = "any data access", members = {<#[[ONE_ARG_ANY_ACCESS]], 0>}>
+// CHECK: #[[ONE_ARG_ANY_ARG:.+]] = #llvm.tbaa_type_desc<id = "dummy arg data", members = {<#[[ONE_ARG_ANY_DATA]], 0>}>
+// CHECK: #[[ONE_ARG_A:.+]] = #llvm.tbaa_type_desc<id = "dummy arg data/_QFoneargEa", members = {<#[[ONE_ARG_ANY_ARG]], 0>}>
+// CHECK: #[[ONE_ARG_A_TAG:.+]] = #llvm.tbaa_tag<base_type = #[[ONE_ARG_A]], access_type = #[[ONE_ARG_A]], offset = 0>
+
+// CHECK-LABEL:   func.func @_QPonearg(
+// CHECK-SAME:                         %[[VAL_0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"}) {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant 2 : index
+// CHECK:           %[[VAL_3:.*]] = fir.declare %[[VAL_0]] {uniq_name = "_QFoneargEa"} : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+// CHECK:           %[[VAL_4:.*]] = fir.rebox %[[VAL_3]] : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+// CHECK:           %[[VAL_5:.*]] = fir.array_coor %[[VAL_4]] %[[VAL_2]] : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:           %[[VAL_6:.*]] = fir.load %[[VAL_5]] {tbaa = [#[[ONE_ARG_A_TAG]]]} : !fir.ref<i32>
+// CHECK:           %[[VAL_7:.*]] = fir.array_coor %[[VAL_4]] %[[VAL_1]] : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:           fir.store %[[VAL_6]] to %[[VAL_7]] {tbaa = [#[[ONE_ARG_A_TAG]]]} : !fir.ref<i32>
+// CHECK:           return
+// CHECK:         }
+
+// -----
+
+// subroutine twoArg(a, b)
+//   integer :: a(:), b(:)
+//   a(1) = b(1)
+// end subroutine
+  func.func @_QPtwoarg(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"}, %arg1: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "b"}) {
+    %c1 = arith.constant 1 : index
+    %0 = fir.declare %arg0 {uniq_name = "_QFtwoargEa"} : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+    %1 = fir.rebox %0 : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+    %2 = fir.declare %arg1 {uniq_name = "_QFtwoargEb"} : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+    %3 = fir.rebox %2 : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+    %4 = fir.array_coor %3 %c1 : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+    %5 = fir.load %4 : !fir.ref<i32>
+    %6 = fir.array_coor %1 %c1 : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+    fir.store %5 to %6 : !fir.ref<i32>
+    return
+  }
+
+// CHECK: #[[TWO_ARG_ROOT:.+]] = #llvm.tbaa_root<id = "Flang function root _QPtwoarg">
+// CHECK: #[[TWO_ARG_ANY_ACCESS:.+]] = #llvm.tbaa_type_desc<id = "any access", members = {<#[[TWO_ARG_ROOT]], 0>}>
+// CHECK: #[[TWO_ARG_ANY_DATA:.+]] = #llvm.tbaa_type_desc<id = "any data access", members = {<#[[TWO_ARG_ANY_ACCESS]], 0>}>
+// CHECK: #[[TWO_ARG_ANY_ARG:.+]] = #llvm.tbaa_type_desc<id = "dummy arg data", members = {<#[[TWO_ARG_ANY_DATA]], 0>}>
+// CHECK: #[[TWO_ARG_B:.+]] = #llvm.tbaa_type_desc<id = "dummy arg data/_QFtwoargEb", members = {<#[[TWO_ARG_ANY_ARG]], 0>}>
+// CHECK: #[[TWO_ARG_A:.+]] = #llvm.tbaa_type_desc<id = "dummy arg data/_QFtwoargEa", members = {<#[[TWO_ARG_ANY_ARG]], 0>}>
+// CHECK: #[[TWO_ARG_B_TAG:.+]] = #llvm.tbaa_tag<base_type = #[[TWO_ARG_B]], access_type = #[[TWO_ARG_B]], offset = 0>
+// CHECK: #[[TWO_ARG_A_TAG:.+]] = #llvm.tbaa_tag<base_type = #[[TWO_ARG_A]], access_type = #[[TWO_ARG_A]], offset = 0>
+
+// CHECK-LABEL:   func.func @_QPtwoarg(
+// CHECK-SAME:                         %[[VAL_0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"},
+// CHECK-SAME:                         %[[VAL_1:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "b"}) {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_3:.*]] = fir.declare %[[VAL_0]] {uniq_name = "_QFtwoargEa"} : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+// CHECK:           %[[VAL_4:.*]] = fir.rebox %[[VAL_3]] : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+// CHECK:           %[[VAL_5:.*]] = fir.declare %[[VAL_1]] {uniq_name = "_QFtwoargEb"} : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+// CHECK:           %[[VAL_6:.*]] = fir.rebox %[[VAL_5]] : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+// CHECK:           %[[VAL_7:.*]] = fir.array_coor %[[VAL_6]] %[[VAL_2]] : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:           %[[VAL_8:.*]] = fir.load %[[VAL_7]] {tbaa = [#[[TWO_ARG_B_TAG]]]} : !fir.ref<i32>
+// CHECK:           %[[VAL_9:.*]] = fir.array_coor %[[VAL_4]] %[[VAL_2]] : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:           fir.store %[[VAL_8]] to %[[VAL_9]] {tbaa = [#[[TWO_ARG_A_TAG]]]} : !fir.ref<i32>
+// CHECK:           return
+// CHECK:         }
+
+// -----
+
+// subroutine targetArg(a, b)
+//   integer, target :: a(:)
+//   integer :: b(:)
+//   a(1) = b(1)
+// end subroutine
+  func.func @_QPtargetarg(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a", fir.target}, %arg1: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "b"}) {
+    %c1 = arith.constant 1 : index
+    %0 = fir.declare %arg0 {fortran_attrs = #fir.var_attrs<target>, uniq_name = "_QFtargetargEa"} : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+    %1 = fir.rebox %0 : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+    %2 = fir.declare %arg1 {uniq_name = "_QFtargetargEb"} : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+    %3 = fir.rebox %2 : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+    %4 = fir.array_coor %3 %c1 : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+    %5 = fir.load %4 : !fir.ref<i32>
+    %6 = fir.array_coor %1 %c1 : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+    fir.store %5 to %6 : !fir.ref<i32>
+    return
+  }
+
+// CHECK: #[[TARGET_ROOT:.+]] = #llvm.tbaa_root<id = "Flang function root _QPtargetarg">
+// CHECK: #[[TARGET_ANY_ACCESS:.+]] = #llvm.tbaa_type_desc<id = "any access", members = {<#[[TARGET_ROOT]], 0>}>
+// CHECK: #[[TARGET_ANY_DATA:.+]] = #llvm.tbaa_type_desc<id = "any data access", members = {<#[[TARGET_ANY_ACCESS]], 0>}>
+// CHECK: #[[TARGET_ANY_ARG:.+]] = #llvm.tbaa_type_desc<id = "dummy arg data", members = {<#[[TARGET_ANY_DATA]], 0>}>
+// CHECK: #[[TARGET_B:.+]] = #llvm.tbaa_type_desc<id = "dummy arg data/_QFtargetargEb", members = {<#[[TARGET_ANY_ARG]], 0>}>
+// CHECK: #[[TARGET_B_TAG:.+]] = #llvm.tbaa_tag<base_type = #[[TARGET_B]], access_type = #[[TARGET_B]], offset = 0>
+// No entry for "dummy arg data/a" because that pointer should get "any data access" becase it has the TARGET attribute
+
+// CHECK-LABEL:   func.func @_QPtargetarg(
+// CHECK-SAME:                            %[[VAL_0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a", fir.target},
+// CHECK-SAME:                            %[[VAL_1:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "b"}) {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_3:.*]] = fir.declare %[[VAL_0]] {fortran_attrs = #{{.*}}<target>, uniq_name = "_QFtargetargEa"} : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+// CHECK:           %[[VAL_4:.*]] = fir.rebox %[[VAL_3]] : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+// CHECK:           %[[VAL_5:.*]] = fir.declare %[[VAL_1]] {uniq_name = "_QFtargetargEb"} : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+// CHECK:           %[[VAL_6:.*]] = fir.rebox %[[VAL_5]] : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+// CHECK:           %[[VAL_7:.*]] = fir.array_coor %[[VAL_6]] %[[VAL_2]] : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:           %[[VAL_8:.*]] = fir.load %[[VAL_7]] {tbaa = [#[[TARGET_B_TAG]]]} : !fir.ref<i32>
+// CHECK:           %[[VAL_9:.*]] = fir.array_coor %[[VAL_4]] %[[VAL_2]] : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// "any data access" tag is added by TBAABuilder during CodeGen
+// CHECK:           fir.store %[[VAL_8]] to %[[VAL_9]] : !fir.ref<i32>
+// CHECK:           return
+// CHECK:         }
+
+// -----
+
+// subroutine pointerArg(a, b)
+//   integer, pointer :: a(:)
+//   integer :: b(:)
+//   a(1) = b(1)
+// end subroutine
+  func.func @_QPpointerarg(%arg0: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>> {fir.bindc_name = "a"}, %arg1: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "b"}) {
+    %c0 = arith.constant 0 : index
+    %c1 = arith.constant 1 : index
+    %0 = fir.declare %arg0 {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFpointerargEa"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
+    %1 = fir.declare %arg1 {uniq_name = "_QFpointerargEb"} : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+    %2 = fir.rebox %1 : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+    %3 = fir.array_coor %2 %c1 : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+    %4 = fir.load %3 : !fir.ref<i32>
+    %5 = fir.load %0 : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
+    %6:3 = fir.box_dims %5, %c0 : (!fir.box<!fir.ptr<!fir.array<?xi32>>>, index) -> (index, index, index)
+    %7 = fir.shift %6#0 : (index) -> !fir.shift<1>
+    %8 = fir.array_coor %5(%7) %c1 : (!fir.box<!fir.ptr<!fir.array<?xi32>>>, !fir.shift<1>, index) -> !fir.ref<i32>
+    fir.store %4 to %8 : !fir.ref<i32>
+    return
+  }
+
+// CHECK: #[[POINTER_ROOT:.+]] = #llvm.tbaa_root<id = "Flang function root _QPpointerarg">
+// CHECK: #[[POINTER_ANY_ACCESS:.+]] = #llvm.tbaa_type_desc<id = "any access", members = {<#[[POINTER_ROOT]], 0>}>
+// CHECK: #[[POINTER_ANY_DATA:.+]] = #llvm.tbaa_type_desc<id = "any data access", members = {<#[[POINTER_ANY_ACCESS]], 0>}>
+// CHECK: #[[POINTER_ANY_ARG:.+]] = #llvm.tbaa_type_desc<id = "dummy arg data", members = {<#[[POINTER_ANY_DATA]], 0>}>
+// CHECK: #[[POINTER_B:.+]] = #llvm.tbaa_type_desc<id = "dummy arg data/_QFpointerargEb", members = {<#[[POINTER_ANY_ARG]], 0>}>
+// CHECK: #[[POINTER_B_TAG:.+]] = #llvm.tbaa_tag<base_type = #[[POINTER_B]], access_type = #[[POINTER_B]], offset = 0>
+// No entry for "dummy arg data/a" because that pointer should get "any data access" becase it has the POINTER attribute
+
+// CHECK-LABEL:   func.func @_QPpointerarg(
+// CHECK-SAME:                             %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>> {fir.bindc_name = "a"},
+// CHECK-SAME:                             %[[VAL_1:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "b"}) {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_4:.*]] = fir.declare %[[VAL_0]] {fortran_attrs = #{{.*}}<pointer>, uniq_name = "_QFpointerargEa"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>) -> !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
+// CHECK:           %[[VAL_5:.*]] = fir.declare %[[VAL_1]] {uniq_name = "_QFpointerargEb"} : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+// CHECK:           %[[VAL_6:.*]] = fir.rebox %[[VAL_5]] : (!fir.box<!fir.array<?xi32>>) -> !fir.box<!fir.array<?xi32>>
+// CHECK:           %[[VAL_7:.*]] = fir.array_coor %[[VAL_6]] %[[VAL_3]] : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:           %[[VAL_8:.*]] = fir.load %[[VAL_7]] {tbaa = [#[[POINTER_B_TAG]]]} : !fir.ref<i32>
+// "any descriptor access" tag is added by TBAABuilder during CodeGen
+// CHECK:           %[[VAL_9:.*]] = fir.load %[[VAL_4]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>
+// CHECK:           %[[VAL_10:.*]]:3 = fir.box_dims %[[VAL_9]], %[[VAL_2]] : (!fir.box<!fir.ptr<!fir.array<?xi32>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_11:.*]] = fir.shift %[[VAL_10]]#0 : (index) -> !fir.shift<1>
+// CHECK:           %[[VAL_12:.*]] = fir.array_coor %[[VAL_9]](%[[VAL_11]]) %[[VAL_3]] : (!fir.box<!fir.ptr<!fir.array<?xi32>>>, !fir.shift<1>, index) -> !fir.ref<i32>
+// "any data access" tag is added by TBAABuilder during CodeGen
+// CHECK:           fir.store %[[VAL_8]] to %[[VAL_12]] : !fir.ref<i32>
+// CHECK:           return
+// CHECK:         }
diff --git a/flang/test/Transforms/tbaa2.fir b/flang/test/Transforms/tbaa2.fir
new file mode 100644
index 0000000000..84ba281cce
--- /dev/null
+++ b/flang/test/Transforms/tbaa2.fir
@@ -0,0 +1,386 @@
+// Test fir alias analysis pass on a larger real life code example (from the RFC)
+// RUN: fir-opt --fir-add-alias-tags %s | FileCheck %s
+
+  fir.global @_QMmodEa : !fir.box<!fir.heap<!fir.array<?xf32>>> {
+    %c0 = arith.constant 0 : index
+    %0 = fir.zero_bits !fir.heap<!fir.array<?xf32>>
+    %1 = fir.shape %c0 : (index) -> !fir.shape<1>
+    %2 = fir.embox %0(%1) : (!fir.heap<!fir.array<?xf32>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xf32>>>
+    fir.has_value %2 : !fir.box<!fir.heap<!fir.array<?xf32>>>
+  }
+  fir.global @_QMmodEb : !fir.box<!fir.heap<!fir.array<?xf32>>> {
+    %c0 = arith.constant 0 : index
+    %0 = fir.zero_bits !fir.heap<!fir.array<?xf32>>
+    %1 = fir.shape %c0 : (index) -> !fir.shape<1>
+    %2 = fir.embox %0(%1) : (!fir.heap<!fir.array<?xf32>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xf32>>>
+    fir.has_value %2 : !fir.box<!fir.heap<!fir.array<?xf32>>>
+  }
+  fir.global @_QMmodEdxinv : f32 {
+    %0 = fir.zero_bits f32
+    fir.has_value %0 : f32
+  }
+  fir.global @_QMmodEdyinv : f32 {
+    %0 = fir.zero_bits f32
+    fir.has_value %0 : f32
+  }
+  fir.global @_QMmodExstart : i32 {
+    %0 = fir.zero_bits i32
+    fir.has_value %0 : i32
+  }
+  fir.global @_QMmodEystart : i32 {
+    %0 = fir.zero_bits i32
+    fir.has_value %0 : i32
+  }
+  fir.global @_QMmodEystop : i32 {
+    %0 = fir.zero_bits i32
+    fir.has_value %0 : i32
+  }
+  fir.global @_QMmodEzstart : i32 {
+    %0 = fir.zero_bits i32
+    fir.has_value %0 : i32
+  }
+  fir.global @_QMmodEzstop : i32 {
+    %0 = fir.zero_bits i32
+    fir.has_value %0 : i32
+  }
+
+// CHECK: #[[ROOT:.+]] = #llvm.tbaa_root<id = "Flang function root _QMmodPcallee">
+// CHECK: #[[ANY_ACCESS:.+]] = #llvm.tbaa_type_desc<id = "any access", members = {<#[[ROOT]], 0>}>
+// CHECK: #[[ANY_DATA:.+]] = #llvm.tbaa_type_desc<id = "any data access", members = {<#[[ANY_ACCESS]], 0>}>
+// CHECK: #[[ANY_ARG:.+]] = #llvm.tbaa_type_desc<id = "dummy arg data", members = {<#[[ANY_DATA]], 0>}>
+// CHECK: #[[ARG_LOW:.+]] = #llvm.tbaa_type_desc<id = "dummy arg data/_QMmodFcalleeElow", members = {<#[[ANY_ARG]], 0>}>
+// CHECK: #[[ARG_Z:.+]] = #llvm.tbaa_type_desc<id = "dummy arg data/_QMmodFcalleeEz", members = {<#[[ANY_ARG]], 0>}>
+// CHECK: #[[ARG_Y:.+]] = #llvm.tbaa_type_desc<id = "dummy arg data/_QMmodFcalleeEy", members = {<#[[ANY_ARG]], 0>}>
+// CHECK: #[[ARG_LOW_TAG:.+]] = #llvm.tbaa_tag<base_type = #[[ARG_LOW]], access_type = #[[ARG_LOW]], offset = 0>
+// CHECK: #[[ARG_Z_TAG:.+]] = #llvm.tbaa_tag<base_type = #[[ARG_Z]], access_type = #[[ARG_Z]], offset = 0>
+// CHECK: #[[ARG_Y_TAG:.+]] = #llvm.tbaa_tag<base_type = #[[ARG_Y]], access_type = #[[ARG_Y]], offset = 0>
+
+  func.func @_QMmodPcallee(%arg0: !fir.box<!fir.array<?x?x?xf32>> {fir.bindc_name = "z"}, %arg1: !fir.box<!fir.array<?x?x?xf32>> {fir.bindc_name = "y"}, %arg2: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xf32>>>> {fir.bindc_name = "low"}) {
+    %c2 = arith.constant 2 : index
+    %c0 = arith.constant 0 : index
+    %c1 = arith.constant 1 : index
+    %c1_i32 = arith.constant 1 : i32
+    %0 = fir.address_of(@_QMmodEa) : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+    %1 = fir.declare %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMmodEa"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+    %2 = fir.address_of(@_QMmodEb) : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+    %3 = fir.declare %2 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMmodEb"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+    %4 = fir.address_of(@_QMmodEdxinv) : !fir.ref<f32>
+    %5 = fir.declare %4 {uniq_name = "_QMmodEdxinv"} : (!fir.ref<f32>) -> !fir.ref<f32>
+    %6 = fir.address_of(@_QMmodEdyinv) : !fir.ref<f32>
+    %7 = fir.declare %6 {uniq_name = "_QMmodEdyinv"} : (!fir.ref<f32>) -> !fir.ref<f32>
+    %8 = fir.address_of(@_QMmodExstart) : !fir.ref<i32>
+    %9 = fir.declare %8 {uniq_name = "_QMmodExstart"} : (!fir.ref<i32>) -> !fir.ref<i32>
+    %10 = fir.address_of(@_QMmodEystart) : !fir.ref<i32>
+    %11 = fir.declare %10 {uniq_name = "_QMmodEystart"} : (!fir.ref<i32>) -> !fir.ref<i32>
+    %12 = fir.address_of(@_QMmodEystop) : !fir.ref<i32>
+    %13 = fir.declare %12 {uniq_name = "_QMmodEystop"} : (!fir.ref<i32>) -> !fir.ref<i32>
+    %14 = fir.address_of(@_QMmodEzstart) : !fir.ref<i32>
+    %15 = fir.declare %14 {uniq_name = "_QMmodEzstart"} : (!fir.ref<i32>) -> !fir.ref<i32>
+    %16 = fir.address_of(@_QMmodEzstop) : !fir.ref<i32>
+    %17 = fir.declare %16 {uniq_name = "_QMmodEzstop"} : (!fir.ref<i32>) -> !fir.ref<i32>
+    %18 = fir.alloca f32 {bindc_name = "dxold", uniq_name = "_QMmodFcalleeEdxold"}
+    %19 = fir.declare %18 {uniq_name = "_QMmodFcalleeEdxold"} : (!fir.ref<f32>) -> !fir.ref<f32>
+    %20 = fir.alloca f32 {bindc_name = "dzinv", uniq_name = "_QMmodFcalleeEdzinv"}
+    %21 = fir.declare %20 {uniq_name = "_QMmodFcalleeEdzinv"} : (!fir.ref<f32>) -> !fir.ref<f32>
+    %22 = fir.alloca i32 {bindc_name = "i", uniq_name = "_QMmodFcalleeEi"}
+    %23 = fir.declare %22 {uniq_name = "_QMmodFcalleeEi"} : (!fir.ref<i32>) -> !fir.ref<i32>
+    %24 = fir.alloca i32 {bindc_name = "j", uniq_name = "_QMmodFcalleeEj"}
+    %25 = fir.declare %24 {uniq_name = "_QMmodFcalleeEj"} : (!fir.ref<i32>) -> !fir.ref<i32>
+    %26 = fir.alloca i32 {bindc_name = "k", uniq_name = "_QMmodFcalleeEk"}
+    %27 = fir.declare %26 {uniq_name = "_QMmodFcalleeEk"} : (!fir.ref<i32>) -> !fir.ref<i32>
+    %28 = fir.declare %arg2 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMmodFcalleeElow"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xf32>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xf32>>>>
+    %29 = fir.declare %arg1 {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QMmodFcalleeEy"} : (!fir.box<!fir.array<?x?x?xf32>>) -> !fir.box<!fir.array<?x?x?xf32>>
+    %30 = fir.rebox %29 : (!fir.box<!fir.array<?x?x?xf32>>) -> !fir.box<!fir.array<?x?x?xf32>>
+    %31 = fir.declare %arg0 {fortran_attrs = #fir.var_attrs<intent_in>, uniq_name = "_QMmodFcalleeEz"} : (!fir.box<!fir.array<?x?x?xf32>>) -> !fir.box<!fir.array<?x?x?xf32>>
+    %32 = fir.rebox %31 : (!fir.box<!fir.array<?x?x?xf32>>) -> !fir.box<!fir.array<?x?x?xf32>>
+    %33 = fir.load %15 : !fir.ref<i32>
+    %34 = arith.addi %33, %c1_i32 : i32
+    %35 = fir.convert %34 : (i32) -> index
+    %36 = fir.load %17 : !fir.ref<i32>
+    %37 = fir.convert %36 : (i32) -> index
+    %38 = fir.convert %35 : (index) -> i32
+    %39:2 = fir.do_loop %arg3 = %35 to %37 step %c1 iter_args(%arg4 = %38) -> (index, i32) {
+      fir.store %arg4 to %27 : !fir.ref<i32>
+      %40 = fir.load %11 : !fir.ref<i32>
+      %41 = arith.addi %40, %c1_i32 : i32
+      %42 = fir.convert %41 : (i32) -> index
+      %43 = fir.load %13 : !fir.ref<i32>
+      %44 = fir.convert %43 : (i32) -> index
+      %45 = fir.convert %42 : (index) -> i32
+      %46:2 = fir.do_loop %arg5 = %42 to %44 step %c1 iter_args(%arg6 = %45) -> (index, i32) {
+        fir.store %arg6 to %25 : !fir.ref<i32>
+        %51 = fir.load %9 : !fir.ref<i32>
+        %52 = arith.addi %51, %c1_i32 : i32
+        %53 = fir.convert %52 : (i32) -> index
+        %54 = fir.convert %53 : (index) -> i32
+        %55:2 = fir.do_loop %arg7 = %53 to %c0 step %c1 iter_args(%arg8 = %54) -> (index, i32) {
+          fir.store %arg8 to %23 : !fir.ref<i32>
+          %60 = fir.load %28 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xf32>>>>
+          %61 = fir.load %23 : !fir.ref<i32>
+          %62 = fir.convert %61 : (i32) -> i64
+          %63 = fir.load %25 : !fir.ref<i32>
+          %64 = fir.convert %63 : (i32) -> i64
+          %65 = fir.load %27 : !fir.ref<i32>
+          %66 = fir.convert %65 : (i32) -> i64
+          %67 = fir.box_addr %60 : (!fir.box<!fir.heap<!fir.array<?x?x?xf32>>>) -> !fir.heap<!fir.array<?x?x?xf32>>
+          %68:3 = fir.box_dims %60, %c0 : (!fir.box<!fir.heap<!fir.array<?x?x?xf32>>>, index) -> (index, index, index)
+          %69:3 = fir.box_dims %60, %c1 : (!fir.box<!fir.heap<!fir.array<?x?x?xf32>>>, index) -> (index, index, index)
+          %70:3 = fir.box_dims %60, %c2 : (!fir.box<!fir.heap<!fir.array<?x?x?xf32>>>, index) -> (index, index, index)
+          %71 = fir.shape_shift %68#0, %68#1, %69#0, %69#1, %70#0, %70#1 : (index, index, index, index, index, index) -> !fir.shapeshift<3>
+          %72 = fir.array_coor %67(%71) %62, %64, %66 : (!fir.heap<!fir.array<?x?x?xf32>>, !fir.shapeshift<3>, i64, i64, i64) -> !fir.ref<f32>
+          %73 = fir.load %72 : !fir.ref<f32>
+          fir.store %73 to %19 : !fir.ref<f32>
+          %74 = fir.load %1 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+          %75 = fir.load %25 : !fir.ref<i32>
+          %76 = fir.convert %75 : (i32) -> i64
+          %77 = fir.box_addr %74 : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
+          %78:3 = fir.box_dims %74, %c0 : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+          %79 = fir.shape_shift %78#0, %78#1 : (index, index) -> !fir.shapeshift<1>
+          %80 = fir.array_coor %77(%79) %76 : (!fir.heap<!fir.array<?xf32>>, !fir.shapeshift<1>, i64) -> !fir.ref<f32>
+          %81 = fir.load %80 : !fir.ref<f32>
+          %82 = fir.load %28 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xf32>>>>
+          %83 = fir.load %23 : !fir.ref<i32>
+          %84 = fir.convert %83 : (i32) -> i64
+          %85 = fir.load %27 : !fir.ref<i32>
+          %86 = fir.convert %85 : (i32) -> i64
+          %87 = fir.box_addr %82 : (!fir.box<!fir.heap<!fir.array<?x?x?xf32>>>) -> !fir.heap<!fir.array<?x?x?xf32>>
+          %88:3 = fir.box_dims %82, %c0 : (!fir.box<!fir.heap<!fir.array<?x?x?xf32>>>, index) -> (index, index, index)
+          %89:3 = fir.box_dims %82, %c1 : (!fir.box<!fir.heap<!fir.array<?x?x?xf32>>>, index) -> (index, index, index)
+          %90:3 = fir.box_dims %82, %c2 : (!fir.box<!fir.heap<!fir.array<?x?x?xf32>>>, index) -> (index, index, index)
+          %91 = fir.shape_shift %88#0, %88#1, %89#0, %89#1, %90#0, %90#1 : (index, index, index, index, index, index) -> !fir.shapeshift<3>
+          %92 = fir.array_coor %87(%91) %84, %76, %86 : (!fir.heap<!fir.array<?x?x?xf32>>, !fir.shapeshift<3>, i64, i64, i64) -> !fir.ref<f32>
+          %93 = fir.load %92 : !fir.ref<f32>
+          %94 = arith.mulf %81, %93 fastmath<contract> : f32
+          %95 = fir.load %3 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+          %96 = fir.box_addr %95 : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
+          %97:3 = fir.box_dims %95, %c0 : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+          %98 = fir.shape_shift %97#0, %97#1 : (index, index) -> !fir.shapeshift<1>
+          %99 = fir.array_coor %96(%98) %76 : (!fir.heap<!fir.array<?xf32>>, !fir.shapeshift<1>, i64) -> !fir.ref<f32>
+          %100 = fir.load %99 : !fir.ref<f32>
+          %101 = fir.array_coor %32 %84, %76, %86 : (!fir.box<!fir.array<?x?x?xf32>>, i64, i64, i64) -> !fir.ref<f32>
+          %102 = fir.load %101 : !fir.ref<f32>
+          %103 = arith.subi %75, %c1_i32 : i32
+          %104 = fir.convert %103 : (i32) -> i64
+          %105 = fir.array_coor %32 %84, %104, %86 : (!fir.box<!fir.array<?x?x?xf32>>, i64, i64, i64) -> !fir.ref<f32>
+          %106 = fir.load %105 : !fir.ref<f32>
+          %107 = arith.subf %102, %106 fastmath<contract> : f32
+          %108 = fir.no_reassoc %107 : f32
+          %109 = fir.load %7 : !fir.ref<f32>
+          %110 = arith.mulf %108, %109 fastmath<contract> : f32
+          %111 = arith.subi %85, %c1_i32 : i32
+          %112 = fir.convert %111 : (i32) -> i64
+          %113 = fir.array_coor %30 %84, %76, %112 : (!fir.box<!fir.array<?x?x?xf32>>, i64, i64, i64) -> !fir.ref<f32>
+          %114 = fir.load %113 : !fir.ref<f32>
+          %115 = fir.array_coor %30 %84, %76, %86 : (!fir.box<!fir.array<?x?x?xf32>>, i64, i64, i64) -> !fir.ref<f32>
+          %116 = fir.load %115 : !fir.ref<f32>
+          %117 = arith.subf %114, %116 fastmath<contract> : f32
+          %118 = fir.no_reassoc %117 : f32
+          %119 = fir.load %21 : !fir.ref<f32>
+          %120 = arith.mulf %118, %119 fastmath<contract> : f32
+          %121 = arith.addf %110, %120 fastmath<contract> : f32
+          %122 = fir.no_reassoc %121 : f32
+          %123 = arith.mulf %100, %122 fastmath<contract> : f32
+          %124 = arith.addf %94, %123 fastmath<contract> : f32
+          fir.store %124 to %92 : !fir.ref<f32>
+          %125 = arith.addi %arg7, %c1 : index
+          %126 = fir.convert %c1 : (index) -> i32
+          %127 = fir.load %23 : !fir.ref<i32>
+          %128 = arith.addi %127, %126 : i32
+          fir.result %125, %128 : index, i32
+        }
+        fir.store %55#1 to %23 : !fir.ref<i32>
+        %56 = arith.addi %arg5, %c1 : index
+        %57 = fir.convert %c1 : (index) -> i32
+        %58 = fir.load %25 : !fir.ref<i32>
+        %59 = arith.addi %58, %57 : i32
+        fir.result %56, %59 : index, i32
+      }
+      fir.store %46#1 to %25 : !fir.ref<i32>
+      %47 = arith.addi %arg3, %c1 : index
+      %48 = fir.convert %c1 : (index) -> i32
+      %49 = fir.load %27 : !fir.ref<i32>
+      %50 = arith.addi %49, %48 : i32
+      fir.result %47, %50 : index, i32
+    }
+    fir.store %39#1 to %27 : !fir.ref<i32>
+    return
+  }
+// CHECK-LABEL:   func.func @_QMmodPcallee(
+// CHECK-SAME:                             %[[VAL_0:.*]]: !fir.box<!fir.array<?x?x?xf32>> {fir.bindc_name = "z"},
+// CHECK-SAME:                             %[[VAL_1:.*]]: !fir.box<!fir.array<?x?x?xf32>> {fir.bindc_name = "y"},
+// CHECK-SAME:                             %[[VAL_2:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xf32>>>> {fir.bindc_name = "low"}) {
+// CHECK:           %[[VAL_3:.*]] = arith.constant 2 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_5:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_6:.*]] = arith.constant 1 : i32
+// CHECK:           %[[VAL_7:.*]] = fir.address_of(@_QMmodEa) : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+// CHECK:           %[[VAL_8:.*]] = fir.declare %[[VAL_7]] {fortran_attrs = #{{.*}}<allocatable>, uniq_name = "_QMmodEa"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+// CHECK:           %[[VAL_9:.*]] = fir.address_of(@_QMmodEb) : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+// CHECK:           %[[VAL_10:.*]] = fir.declare %[[VAL_9]] {fortran_attrs = #{{.*}}<allocatable>, uniq_name = "_QMmodEb"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+// CHECK:           %[[VAL_11:.*]] = fir.address_of(@_QMmodEdxinv) : !fir.ref<f32>
+// CHECK:           %[[VAL_12:.*]] = fir.declare %[[VAL_11]] {uniq_name = "_QMmodEdxinv"} : (!fir.ref<f32>) -> !fir.ref<f32>
+// CHECK:           %[[VAL_13:.*]] = fir.address_of(@_QMmodEdyinv) : !fir.ref<f32>
+// CHECK:           %[[VAL_14:.*]] = fir.declare %[[VAL_13]] {uniq_name = "_QMmodEdyinv"} : (!fir.ref<f32>) -> !fir.ref<f32>
+// CHECK:           %[[VAL_15:.*]] = fir.address_of(@_QMmodExstart) : !fir.ref<i32>
+// CHECK:           %[[VAL_16:.*]] = fir.declare %[[VAL_15]] {uniq_name = "_QMmodExstart"} : (!fir.ref<i32>) -> !fir.ref<i32>
+// CHECK:           %[[VAL_17:.*]] = fir.address_of(@_QMmodEystart) : !fir.ref<i32>
+// CHECK:           %[[VAL_18:.*]] = fir.declare %[[VAL_17]] {uniq_name = "_QMmodEystart"} : (!fir.ref<i32>) -> !fir.ref<i32>
+// CHECK:           %[[VAL_19:.*]] = fir.address_of(@_QMmodEystop) : !fir.ref<i32>
+// CHECK:           %[[VAL_20:.*]] = fir.declare %[[VAL_19]] {uniq_name = "_QMmodEystop"} : (!fir.ref<i32>) -> !fir.ref<i32>
+// CHECK:           %[[VAL_21:.*]] = fir.address_of(@_QMmodEzstart) : !fir.ref<i32>
+// CHECK:           %[[VAL_22:.*]] = fir.declare %[[VAL_21]] {uniq_name = "_QMmodEzstart"} : (!fir.ref<i32>) -> !fir.ref<i32>
+// CHECK:           %[[VAL_23:.*]] = fir.address_of(@_QMmodEzstop) : !fir.ref<i32>
+// CHECK:           %[[VAL_24:.*]] = fir.declare %[[VAL_23]] {uniq_name = "_QMmodEzstop"} : (!fir.ref<i32>) -> !fir.ref<i32>
+// CHECK:           %[[VAL_25:.*]] = fir.alloca f32 {bindc_name = "dxold", uniq_name = "_QMmodFcalleeEdxold"}
+// CHECK:           %[[VAL_26:.*]] = fir.declare %[[VAL_25]] {uniq_name = "_QMmodFcalleeEdxold"} : (!fir.ref<f32>) -> !fir.ref<f32>
+// CHECK:           %[[VAL_27:.*]] = fir.alloca f32 {bindc_name = "dzinv", uniq_name = "_QMmodFcalleeEdzinv"}
+// CHECK:           %[[VAL_28:.*]] = fir.declare %[[VAL_27]] {uniq_name = "_QMmodFcalleeEdzinv"} : (!fir.ref<f32>) -> !fir.ref<f32>
+// CHECK:           %[[VAL_29:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QMmodFcalleeEi"}
+// CHECK:           %[[VAL_30:.*]] = fir.declare %[[VAL_29]] {uniq_name = "_QMmodFcalleeEi"} : (!fir.ref<i32>) -> !fir.ref<i32>
+// CHECK:           %[[VAL_31:.*]] = fir.alloca i32 {bindc_name = "j", uniq_name = "_QMmodFcalleeEj"}
+// CHECK:           %[[VAL_32:.*]] = fir.declare %[[VAL_31]] {uniq_name = "_QMmodFcalleeEj"} : (!fir.ref<i32>) -> !fir.ref<i32>
+// CHECK:           %[[VAL_33:.*]] = fir.alloca i32 {bindc_name = "k", uniq_name = "_QMmodFcalleeEk"}
+// CHECK:           %[[VAL_34:.*]] = fir.declare %[[VAL_33]] {uniq_name = "_QMmodFcalleeEk"} : (!fir.ref<i32>) -> !fir.ref<i32>
+// CHECK:           %[[VAL_35:.*]] = fir.declare %[[VAL_2]] {fortran_attrs = #{{.*}}<allocatable>, uniq_name = "_QMmodFcalleeElow"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xf32>>>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xf32>>>>
+// CHECK:           %[[VAL_36:.*]] = fir.declare %[[VAL_1]] {fortran_attrs = #{{.*}}<intent_in>, uniq_name = "_QMmodFcalleeEy"} : (!fir.box<!fir.array<?x?x?xf32>>) -> !fir.box<!fir.array<?x?x?xf32>>
+// CHECK:           %[[VAL_37:.*]] = fir.rebox %[[VAL_36]] : (!fir.box<!fir.array<?x?x?xf32>>) -> !fir.box<!fir.array<?x?x?xf32>>
+// CHECK:           %[[VAL_38:.*]] = fir.declare %[[VAL_0]] {fortran_attrs = #{{.*}}<intent_in>, uniq_name = "_QMmodFcalleeEz"} : (!fir.box<!fir.array<?x?x?xf32>>) -> !fir.box<!fir.array<?x?x?xf32>>
+// CHECK:           %[[VAL_39:.*]] = fir.rebox %[[VAL_38]] : (!fir.box<!fir.array<?x?x?xf32>>) -> !fir.box<!fir.array<?x?x?xf32>>
+// TODO: read from global assumed to always alias
+// CHECK:           %[[VAL_40:.*]] = fir.load %[[VAL_22]] : !fir.ref<i32>
+// CHECK:           %[[VAL_41:.*]] = arith.addi %[[VAL_40]], %[[VAL_6]] : i32
+// CHECK:           %[[VAL_42:.*]] = fir.convert %[[VAL_41]] : (i32) -> index
+// TODO: read from global assumed to always alias
+// CHECK:           %[[VAL_43:.*]] = fir.load %[[VAL_24]] : !fir.ref<i32>
+// CHECK:           %[[VAL_44:.*]] = fir.convert %[[VAL_43]] : (i32) -> index
+// CHECK:           %[[VAL_45:.*]] = fir.convert %[[VAL_42]] : (index) -> i32
+// CHECK:           %[[VAL_46:.*]]:2 = fir.do_loop %[[VAL_47:.*]] = %[[VAL_42]] to %[[VAL_44]] step %[[VAL_5]] iter_args(%[[VAL_48:.*]] = %[[VAL_45]]) -> (index, i32) {
+// CHECK:             fir.store %[[VAL_48]] to %[[VAL_34]] : !fir.ref<i32>
+// TODO: read from global assumed to always alias
+// CHECK:             %[[VAL_49:.*]] = fir.load %[[VAL_18]] : !fir.ref<i32>
+// CHECK:             %[[VAL_50:.*]] = arith.addi %[[VAL_49]], %[[VAL_6]] : i32
+// CHECK:             %[[VAL_51:.*]] = fir.convert %[[VAL_50]] : (i32) -> index
+// TODO: read from global assumed to always alias
+// CHECK:             %[[VAL_52:.*]] = fir.load %[[VAL_20]] : !fir.ref<i32>
+// CHECK:             %[[VAL_53:.*]] = fir.convert %[[VAL_52]] : (i32) -> index
+// CHECK:             %[[VAL_54:.*]] = fir.convert %[[VAL_51]] : (index) -> i32
+// CHECK:             %[[VAL_55:.*]]:2 = fir.do_loop %[[VAL_56:.*]] = %[[VAL_51]] to %[[VAL_53]] step %[[VAL_5]] iter_args(%[[VAL_57:.*]] = %[[VAL_54]]) -> (index, i32) {
+// CHECK:               fir.store %[[VAL_57]] to %[[VAL_32]] : !fir.ref<i32>
+// TODO: read from global assumed to always alias
+// CHECK:               %[[VAL_58:.*]] = fir.load %[[VAL_16]] : !fir.ref<i32>
+// CHECK:               %[[VAL_59:.*]] = arith.addi %[[VAL_58]], %[[VAL_6]] : i32
+// CHECK:               %[[VAL_60:.*]] = fir.convert %[[VAL_59]] : (i32) -> index
+// CHECK:               %[[VAL_61:.*]] = fir.convert %[[VAL_60]] : (index) -> i32
+// CHECK:               %[[VAL_62:.*]]:2 = fir.do_loop %[[VAL_63:.*]] = %[[VAL_60]] to %[[VAL_4]] step %[[VAL_5]] iter_args(%[[VAL_64:.*]] = %[[VAL_61]]) -> (index, i32) {
+// TODO: local allocation assumed to always alias
+// CHECK:                 fir.store %[[VAL_64]] to %[[VAL_30]] : !fir.ref<i32>
+// load from box tagged in CodeGen
+// CHECK:                 %[[VAL_65:.*]] = fir.load %[[VAL_35]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xf32>>>>
+// TODO: local allocation assumed to always alias
+// CHECK:                 %[[VAL_66:.*]] = fir.load %[[VAL_30]] : !fir.ref<i32>
+// CHECK:                 %[[VAL_67:.*]] = fir.convert %[[VAL_66]] : (i32) -> i64
+// TODO: local allocation assumed to always alias
+// CHECK:                 %[[VAL_68:.*]] = fir.load %[[VAL_32]] : !fir.ref<i32>
+// CHECK:                 %[[VAL_69:.*]] = fir.convert %[[VAL_68]] : (i32) -> i64
+// TODO: local allocation assumed to always alias
+// CHECK:                 %[[VAL_70:.*]] = fir.load %[[VAL_34]] : !fir.ref<i32>
+// CHECK:                 %[[VAL_71:.*]] = fir.convert %[[VAL_70]] : (i32) -> i64
+// CHECK:                 %[[VAL_72:.*]] = fir.box_addr %[[VAL_65]] : (!fir.box<!fir.heap<!fir.array<?x?x?xf32>>>) -> !fir.heap<!fir.array<?x?x?xf32>>
+// CHECK:                 %[[VAL_73:.*]]:3 = fir.box_dims %[[VAL_65]], %[[VAL_4]] : (!fir.box<!fir.heap<!fir.array<?x?x?xf32>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_74:.*]]:3 = fir.box_dims %[[VAL_65]], %[[VAL_5]] : (!fir.box<!fir.heap<!fir.array<?x?x?xf32>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_75:.*]]:3 = fir.box_dims %[[VAL_65]], %[[VAL_3]] : (!fir.box<!fir.heap<!fir.array<?x?x?xf32>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_76:.*]] = fir.shape_shift %[[VAL_73]]#0, %[[VAL_73]]#1, %[[VAL_74]]#0, %[[VAL_74]]#1, %[[VAL_75]]#0, %[[VAL_75]]#1 : (index, index, index, index, index, index) -> !fir.shapeshift<3>
+// CHECK:                 %[[VAL_77:.*]] = fir.array_coor %[[VAL_72]](%[[VAL_76]]) %[[VAL_67]], %[[VAL_69]], %[[VAL_71]] : (!fir.heap<!fir.array<?x?x?xf32>>, !fir.shapeshift<3>, i64, i64, i64) -> !fir.ref<f32>
+// CHECK:                 %[[VAL_78:.*]] = fir.load %[[VAL_77]] {tbaa = [#[[ARG_LOW_TAG]]]} : !fir.ref<f32>
+// CHECK:                 fir.store %[[VAL_78]] to %[[VAL_26]] : !fir.ref<f32>
+// load from box tagged in CodeGen
+// CHECK:                 %[[VAL_79:.*]] = fir.load %[[VAL_8]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+// TODO: local allocation assumed to always alias
+// CHECK:                 %[[VAL_80:.*]] = fir.load %[[VAL_32]] : !fir.ref<i32>
+// CHECK:                 %[[VAL_81:.*]] = fir.convert %[[VAL_80]] : (i32) -> i64
+// CHECK:                 %[[VAL_82:.*]] = fir.box_addr %[[VAL_79]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
+// CHECK:                 %[[VAL_83:.*]]:3 = fir.box_dims %[[VAL_79]], %[[VAL_4]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_84:.*]] = fir.shape_shift %[[VAL_83]]#0, %[[VAL_83]]#1 : (index, index) -> !fir.shapeshift<1>
+// CHECK:                 %[[VAL_85:.*]] = fir.array_coor %[[VAL_82]](%[[VAL_84]]) %[[VAL_81]] : (!fir.heap<!fir.array<?xf32>>, !fir.shapeshift<1>, i64) -> !fir.ref<f32>
+// load from global variable
+// CHECK:                 %[[VAL_86:.*]] = fir.load %[[VAL_85]] : !fir.ref<f32>
+// load from box
+// CHECK:                 %[[VAL_87:.*]] = fir.load %[[VAL_35]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?x?xf32>>>>
+// load from local allocation
+// CHECK:                 %[[VAL_88:.*]] = fir.load %[[VAL_30]] : !fir.ref<i32>
+// CHECK:                 %[[VAL_89:.*]] = fir.convert %[[VAL_88]] : (i32) -> i64
+// load from local allocation
+// CHECK:                 %[[VAL_90:.*]] = fir.load %[[VAL_34]] : !fir.ref<i32>
+// CHECK:                 %[[VAL_91:.*]] = fir.convert %[[VAL_90]] : (i32) -> i64
+// CHECK:                 %[[VAL_92:.*]] = fir.box_addr %[[VAL_87]] : (!fir.box<!fir.heap<!fir.array<?x?x?xf32>>>) -> !fir.heap<!fir.array<?x?x?xf32>>
+// CHECK:                 %[[VAL_93:.*]]:3 = fir.box_dims %[[VAL_87]], %[[VAL_4]] : (!fir.box<!fir.heap<!fir.array<?x?x?xf32>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_94:.*]]:3 = fir.box_dims %[[VAL_87]], %[[VAL_5]] : (!fir.box<!fir.heap<!fir.array<?x?x?xf32>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_95:.*]]:3 = fir.box_dims %[[VAL_87]], %[[VAL_3]] : (!fir.box<!fir.heap<!fir.array<?x?x?xf32>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_96:.*]] = fir.shape_shift %[[VAL_93]]#0, %[[VAL_93]]#1, %[[VAL_94]]#0, %[[VAL_94]]#1, %[[VAL_95]]#0, %[[VAL_95]]#1 : (index, index, index, index, index, index) -> !fir.shapeshift<3>
+// CHECK:                 %[[VAL_97:.*]] = fir.array_coor %[[VAL_92]](%[[VAL_96]]) %[[VAL_89]], %[[VAL_81]], %[[VAL_91]] : (!fir.heap<!fir.array<?x?x?xf32>>, !fir.shapeshift<3>, i64, i64, i64) -> !fir.ref<f32>
+// CHECK:                 %[[VAL_98:.*]] = fir.load %[[VAL_97]] {tbaa = [#[[ARG_LOW_TAG]]]} : !fir.ref<f32>
+// CHECK:                 %[[VAL_99:.*]] = arith.mulf %[[VAL_86]], %[[VAL_98]] fastmath<contract> : f32
+// load from box
+// CHECK:                 %[[VAL_100:.*]] = fir.load %[[VAL_10]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>
+// CHECK:                 %[[VAL_101:.*]] = fir.box_addr %[[VAL_100]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>) -> !fir.heap<!fir.array<?xf32>>
+// CHECK:                 %[[VAL_102:.*]]:3 = fir.box_dims %[[VAL_100]], %[[VAL_4]] : (!fir.box<!fir.heap<!fir.array<?xf32>>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_103:.*]] = fir.shape_shift %[[VAL_102]]#0, %[[VAL_102]]#1 : (index, index) -> !fir.shapeshift<1>
+// CHECK:                 %[[VAL_104:.*]] = fir.array_coor %[[VAL_101]](%[[VAL_103]]) %[[VAL_81]] : (!fir.heap<!fir.array<?xf32>>, !fir.shapeshift<1>, i64) -> !fir.ref<f32>
+// load from global variable
+// CHECK:                 %[[VAL_105:.*]] = fir.load %[[VAL_104]] : !fir.ref<f32>
+// CHECK:                 %[[VAL_106:.*]] = fir.array_coor %[[VAL_39]] %[[VAL_89]], %[[VAL_81]], %[[VAL_91]] : (!fir.box<!fir.array<?x?x?xf32>>, i64, i64, i64) -> !fir.ref<f32>
+// CHECK:                 %[[VAL_107:.*]] = fir.load %[[VAL_106]] {tbaa = [#[[ARG_Z_TAG]]]} : !fir.ref<f32>
+// CHECK:                 %[[VAL_108:.*]] = arith.subi %[[VAL_80]], %[[VAL_6]] : i32
+// CHECK:                 %[[VAL_109:.*]] = fir.convert %[[VAL_108]] : (i32) -> i64
+// CHECK:                 %[[VAL_110:.*]] = fir.array_coor %[[VAL_39]] %[[VAL_89]], %[[VAL_109]], %[[VAL_91]] : (!fir.box<!fir.array<?x?x?xf32>>, i64, i64, i64) -> !fir.ref<f32>
+// CHECK:                 %[[VAL_111:.*]] = fir.load %[[VAL_110]] {tbaa = [#[[ARG_Z_TAG]]]} : !fir.ref<f32>
+// CHECK:                 %[[VAL_112:.*]] = arith.subf %[[VAL_107]], %[[VAL_111]] fastmath<contract> : f32
+// CHECK:                 %[[VAL_113:.*]] = fir.no_reassoc %[[VAL_112]] : f32
+// load from global variable
+// CHECK:                 %[[VAL_114:.*]] = fir.load %[[VAL_14]] : !fir.ref<f32>
+// CHECK:                 %[[VAL_115:.*]] = arith.mulf %[[VAL_113]], %[[VAL_114]] fastmath<contract> : f32
+// CHECK:                 %[[VAL_116:.*]] = arith.subi %[[VAL_90]], %[[VAL_6]] : i32
+// CHECK:                 %[[VAL_117:.*]] = fir.convert %[[VAL_116]] : (i32) -> i64
+// CHECK:                 %[[VAL_118:.*]] = fir.array_coor %[[VAL_37]] %[[VAL_89]], %[[VAL_81]], %[[VAL_117]] : (!fir.box<!fir.array<?x?x?xf32>>, i64, i64, i64) -> !fir.ref<f32>
+// CHECK:                 %[[VAL_119:.*]] = fir.load %[[VAL_118]] {tbaa = [#[[ARG_Y_TAG]]]} : !fir.ref<f32>
+// CHECK:                 %[[VAL_120:.*]] = fir.array_coor %[[VAL_37]] %[[VAL_89]], %[[VAL_81]], %[[VAL_91]] : (!fir.box<!fir.array<?x?x?xf32>>, i64, i64, i64) -> !fir.ref<f32>
+// CHECK:                 %[[VAL_121:.*]] = fir.load %[[VAL_120]] {tbaa = [#[[ARG_Y_TAG]]]} : !fir.ref<f32>
+// CHECK:                 %[[VAL_122:.*]] = arith.subf %[[VAL_119]], %[[VAL_121]] fastmath<contract> : f32
+// CHECK:                 %[[VAL_123:.*]] = fir.no_reassoc %[[VAL_122]] : f32
+// load from local allocation
+// CHECK:                 %[[VAL_124:.*]] = fir.load %[[VAL_28]] : !fir.ref<f32>
+// CHECK:                 %[[VAL_125:.*]] = arith.mulf %[[VAL_123]], %[[VAL_124]] fastmath<contract> : f32
+// CHECK:                 %[[VAL_126:.*]] = arith.addf %[[VAL_115]], %[[VAL_125]] fastmath<contract> : f32
+// CHECK:                 %[[VAL_127:.*]] = fir.no_reassoc %[[VAL_126]] : f32
+// CHECK:                 %[[VAL_128:.*]] = arith.mulf %[[VAL_105]], %[[VAL_127]] fastmath<contract> : f32
+// CHECK:                 %[[VAL_129:.*]] = arith.addf %[[VAL_99]], %[[VAL_128]] fastmath<contract> : f32
+// CHECK:                 fir.store %[[VAL_129]] to %[[VAL_97]] {tbaa = [#[[ARG_LOW_TAG]]]} : !fir.ref<f32>
+// CHECK:                 %[[VAL_130:.*]] = arith.addi %[[VAL_63]], %[[VAL_5]] : index
+// CHECK:                 %[[VAL_131:.*]] = fir.convert %[[VAL_5]] : (index) -> i32
+// load from local allocation
+// CHECK:                 %[[VAL_132:.*]] = fir.load %[[VAL_30]] : !fir.ref<i32>
+// CHECK:                 %[[VAL_133:.*]] = arith.addi %[[VAL_132]], %[[VAL_131]] : i32
+// CHECK:                 fir.result %[[VAL_130]], %[[VAL_133]] : index, i32
+// CHECK:               }
+// store to local allocation
+// CHECK:               fir.store %[[VAL_134:.*]]#1 to %[[VAL_30]] : !fir.ref<i32>
+// CHECK:               %[[VAL_135:.*]] = arith.addi %[[VAL_56]], %[[VAL_5]] : index
+// CHECK:               %[[VAL_136:.*]] = fir.convert %[[VAL_5]] : (index) -> i32
+// local allocation:
+// CHECK:               %[[VAL_137:.*]] = fir.load %[[VAL_32]] : !fir.ref<i32>
+// CHECK:               %[[VAL_138:.*]] = arith.addi %[[VAL_137]], %[[VAL_136]] : i32
+// CHECK:               fir.result %[[VAL_135]], %[[VAL_138]] : index, i32
+// CHECK:             }
+// local allocation:
+// CHECK:             fir.store %[[VAL_139:.*]]#1 to %[[VAL_32]] : !fir.ref<i32>
+// CHECK:             %[[VAL_140:.*]] = arith.addi %[[VAL_47]], %[[VAL_5]] : index
+// CHECK:             %[[VAL_141:.*]] = fir.convert %[[VAL_5]] : (index) -> i32
+// local allocation:
+// CHECK:             %[[VAL_142:.*]] = fir.load %[[VAL_34]] : !fir.ref<i32>
+// CHECK:             %[[VAL_143:.*]] = arith.addi %[[VAL_142]], %[[VAL_141]] : i32
+// CHECK:             fir.result %[[VAL_140]], %[[VAL_143]] : index, i32
+// CHECK:           }
+// local allocation:
+// CHECK:           fir.store %[[VAL_144:.*]]#1 to %[[VAL_34]] : !fir.ref<i32>
+// CHECK:           return
+// CHECK:         }
diff --git a/flang/tools/flang-driver/driver.cpp b/flang/tools/flang-driver/driver.cpp
index c7c39a731c..99fa66b0dc 100644
--- a/flang/tools/flang-driver/driver.cpp
+++ b/flang/tools/flang-driver/driver.cpp
@@ -32,8 +32,6 @@
 #include "llvm/TargetParser/Host.h"
 #include <stdlib.h>
 
-using llvm::StringRef;
-
 // main frontend method. Lives inside fc1_main.cpp
 extern int fc1_main(llvm::ArrayRef<const char *> argv, const char *argv0);
 
diff --git a/flang/unittests/Evaluate/ISO-Fortran-binding.cpp b/flang/unittests/Evaluate/ISO-Fortran-binding.cpp
index 09a51e6cea..3c98363f90 100644
--- a/flang/unittests/Evaluate/ISO-Fortran-binding.cpp
+++ b/flang/unittests/Evaluate/ISO-Fortran-binding.cpp
@@ -643,13 +643,140 @@ static void run_CFI_setpointer_tests() {
   }
 }
 
+static void run_CFI_is_contiguous_tests() {
+  // INTEGER :: A(0:3,0:3)
+  constexpr CFI_rank_t rank{2};
+  CFI_index_t extents[rank] = {4, 4};
+  CFI_CDESC_T(rank) dv_storage;
+  CFI_cdesc_t *dv{&dv_storage};
+  Descriptor *dvDesc{reinterpret_cast<Descriptor *>(dv)};
+  char base;
+  void *base_addr{&base};
+  int retCode{CFI_establish(dv, base_addr, CFI_attribute_other, CFI_type_int,
+      /*elem_len=*/0, rank, extents)};
+  MATCH(retCode == CFI_SUCCESS, true);
+
+  MATCH(true, CFI_is_contiguous(dv) == 1);
+  MATCH(true, dvDesc->IsContiguous());
+
+  CFI_CDESC_T(rank) sectionDescriptorStorage;
+  CFI_cdesc_t *section{&sectionDescriptorStorage};
+  Descriptor *sectionDesc{reinterpret_cast<Descriptor *>(section)};
+  retCode = CFI_establish(section, base_addr, CFI_attribute_other, CFI_type_int,
+      /*elem_len=*/0, rank, extents);
+  MATCH(retCode == CFI_SUCCESS, true);
+
+  // Test empty section B = A(0:3:2,0:3:-2) is contiguous.
+  CFI_index_t lb[rank] = {0, 0};
+  CFI_index_t ub[rank] = {3, 3};
+  CFI_index_t strides[rank] = {2, -2};
+  retCode = CFI_section(section, dv, lb, ub, strides);
+  MATCH(true, retCode == CFI_SUCCESS);
+  MATCH(true, CFI_is_contiguous(section) == 1);
+  MATCH(true, sectionDesc->IsContiguous());
+
+  // Test 1 element section B = A(0:1:2,0:1:2) is contiguous.
+  lb[0] = 0;
+  lb[1] = 0;
+  ub[0] = 1;
+  ub[1] = 1;
+  strides[0] = 2;
+  strides[1] = 2;
+  retCode = CFI_section(section, dv, lb, ub, strides);
+  MATCH(true, retCode == CFI_SUCCESS);
+  MATCH(true, CFI_is_contiguous(section) == 1);
+  MATCH(true, sectionDesc->IsContiguous());
+
+  // Test section B = A(0:3:1,0:2:1) is contiguous.
+  lb[0] = 0;
+  lb[1] = 0;
+  ub[0] = 3;
+  ub[1] = 2;
+  strides[0] = 1;
+  strides[1] = 1;
+  retCode = CFI_section(section, dv, lb, ub, strides);
+  sectionDesc->Dump();
+  MATCH(true, retCode == CFI_SUCCESS);
+  MATCH(true, CFI_is_contiguous(section) == 1);
+  MATCH(true, sectionDesc->IsContiguous());
+
+  // Test section B = A(0:2:1,0:2:1) is not contiguous.
+  lb[0] = 0;
+  lb[1] = 0;
+  ub[0] = 2;
+  ub[1] = 2;
+  strides[0] = 1;
+  strides[1] = 1;
+  retCode = CFI_section(section, dv, lb, ub, strides);
+  sectionDesc->Dump();
+  MATCH(true, retCode == CFI_SUCCESS);
+  MATCH(true, CFI_is_contiguous(section) == 0);
+  MATCH(false, sectionDesc->IsContiguous());
+
+  // Test section B = A(0:3:2,0:3:1) is not contiguous.
+  lb[0] = 0;
+  lb[1] = 0;
+  ub[0] = 3;
+  ub[1] = 3;
+  strides[0] = 2;
+  strides[1] = 1;
+  retCode = CFI_section(section, dv, lb, ub, strides);
+  MATCH(true, retCode == CFI_SUCCESS);
+  MATCH(true, CFI_is_contiguous(section) == 0);
+  MATCH(false, sectionDesc->IsContiguous());
+
+  // Test section B = A(0:3:1,0:3:2) is not contiguous.
+  lb[0] = 0;
+  lb[1] = 0;
+  ub[0] = 3;
+  ub[1] = 3;
+  strides[0] = 1;
+  strides[1] = 2;
+  retCode = CFI_section(section, dv, lb, ub, strides);
+  MATCH(true, retCode == CFI_SUCCESS);
+  MATCH(true, CFI_is_contiguous(section) == 0);
+  MATCH(false, sectionDesc->IsContiguous());
+
+  // Test section B = A(0:3:1,0:0:2) is contiguous.
+  lb[0] = 0;
+  lb[1] = 0;
+  ub[0] = 3;
+  ub[1] = 0;
+  strides[0] = 1;
+  strides[1] = 2;
+  retCode = CFI_section(section, dv, lb, ub, strides);
+  MATCH(true, retCode == CFI_SUCCESS);
+  MATCH(true, CFI_is_contiguous(section) == 1);
+  MATCH(true, sectionDesc->IsContiguous());
+
+  // INTEGER :: C(0:0, 0:3)
+  CFI_index_t c_extents[rank] = {1, 4};
+  CFI_CDESC_T(rank) c_dv_storage;
+  CFI_cdesc_t *cdv{&c_dv_storage};
+  retCode = CFI_establish(cdv, base_addr, CFI_attribute_other, CFI_type_int,
+      /*elem_len=*/0, rank, c_extents);
+  MATCH(retCode == CFI_SUCCESS, true);
+
+  // Test section B = C(0:0:2, 0:3:1) is contiguous.
+  lb[0] = 0;
+  lb[1] = 0;
+  ub[0] = 0;
+  ub[1] = 3;
+  strides[0] = 2;
+  strides[1] = 1;
+  retCode = CFI_section(section, cdv, lb, ub, strides);
+  MATCH(true, retCode == CFI_SUCCESS);
+  MATCH(true, CFI_is_contiguous(section) == 1);
+  MATCH(true, sectionDesc->IsContiguous());
+}
+
 int main() {
   TestCdescMacroForAllRanksSmallerThan<CFI_MAX_RANK>();
   run_CFI_establish_tests();
   run_CFI_address_tests();
   run_CFI_allocate_tests();
   // TODO: test CFI_deallocate
-  // TODO: test CFI_is_contiguous
+  run_CFI_is_contiguous_tests();
   run_CFI_section_tests();
   run_CFI_select_part_tests();
   run_CFI_setpointer_tests();
diff --git a/flang/unittests/Optimizer/InternalNamesTest.cpp b/flang/unittests/Optimizer/InternalNamesTest.cpp
index 28e49fc515..058bbeef9b 100644
--- a/flang/unittests/Optimizer/InternalNamesTest.cpp
+++ b/flang/unittests/Optimizer/InternalNamesTest.cpp
@@ -13,7 +13,6 @@
 
 using namespace fir;
 using llvm::SmallVector;
-using llvm::StringRef;
 
 struct DeconstructedName {
   DeconstructedName(llvm::StringRef name) : name{name} {}
diff --git a/flang/unittests/Runtime/NumericalFormatTest.cpp b/flang/unittests/Runtime/NumericalFormatTest.cpp
index 833b16be0f..219947fe4f 100644
--- a/flang/unittests/Runtime/NumericalFormatTest.cpp
+++ b/flang/unittests/Runtime/NumericalFormatTest.cpp
@@ -290,6 +290,8 @@ TEST(IOApiTests, FormatZeroes) {
       {"(1P,G32.17,';')", "          0.0000000000000000    ;"},
       {"(2P,E32.17,';')", "         00.0000000000000000E+00;"},
       {"(-1P,E32.17,';')", "         0.00000000000000000E+00;"},
+      {"(EX32.17,';')", "        0X0.00000000000000000P+0;"},
+      {"(DC,EX32.17,';')", "        0X0,00000000000000000P+0;"},
       {"(G0,';')", "0.;"},
   };
 
@@ -321,6 +323,8 @@ TEST(IOApiTests, FormatOnes) {
       {"(2P,G32.17,';')", "          1.0000000000000000    ;"},
       {"(-1P,E32.17,';')", "         0.01000000000000000E+02;"},
       {"(-1P,G32.17,';')", "          1.0000000000000000    ;"},
+      {"(EX32.17,';')", "        0X8.00000000000000000P-3;"},
+      {"(DC,EX32.17,';')", "        0X8,00000000000000000P-3;"},
       {"(G0,';')", "1.;"},
   };
 
@@ -337,6 +341,7 @@ TEST(IOApiTests, FormatNegativeOnes) {
       {"(E32.17,';')", "        -0.10000000000000000E+01;"},
       {"(F32.17,';')", "            -1.00000000000000000;"},
       {"(G32.17,';')", "         -1.0000000000000000    ;"},
+      {"(EX32.17,';')", "       -0X8.00000000000000000P-3;"},
       {"(G0,';')", "-1.;"},
   };
   for (auto const &[format, expect] : negOnes) {
@@ -365,6 +370,7 @@ TEST(IOApiTests, FormatDoubleValues) {
               {"(G8.1,';')", " -0.    ;"},
               {"(G0,';')", "-0.;"},
               {"(E9.1,';')", " -0.0E+00;"},
+              {"(EX9.1,';')", "-0X0.0P+0;"},
           }},
       {// +Inf
           0x7ff0000000000000,
@@ -372,9 +378,11 @@ TEST(IOApiTests, FormatDoubleValues) {
               {"(E9.1,';')", "      Inf;"},
               {"(F9.1,';')", "      Inf;"},
               {"(G9.1,';')", "      Inf;"},
+              {"(EX9.1,';')", "      Inf;"},
               {"(SP,E9.1,';')", "     +Inf;"},
               {"(SP,F9.1,';')", "     +Inf;"},
               {"(SP,G9.1,';')", "     +Inf;"},
+              {"(SP,EX9.1,';')", "     +Inf;"},
               {"(G0,';')", "Inf;"},
           }},
       {// -Inf
@@ -383,6 +391,7 @@ TEST(IOApiTests, FormatDoubleValues) {
               {"(E9.1,';')", "     -Inf;"},
               {"(F9.1,';')", "     -Inf;"},
               {"(G9.1,';')", "     -Inf;"},
+              {"(EX9.1,';')", "     -Inf;"},
               {"(G0,';')", "-Inf;"},
           }},
       {// NaN
@@ -391,6 +400,7 @@ TEST(IOApiTests, FormatDoubleValues) {
               {"(E9.1,';')", "      NaN;"},
               {"(F9.1,';')", "      NaN;"},
               {"(G9.1,';')", "      NaN;"},
+              {"(EX9.1,';')", "      NaN;"},
               {"(G0,';')", "NaN;"},
           }},
       {// NaN (sign irrelevant)
@@ -402,6 +412,7 @@ TEST(IOApiTests, FormatDoubleValues) {
               {"(SP,E9.1,';')", "      NaN;"},
               {"(SP,F9.1,';')", "      NaN;"},
               {"(SP,G9.1,';')", "      NaN;"},
+              {"(SP,EX9.1,';')", "      NaN;"},
               {"(G0,';')", "NaN;"},
           }},
       {// 0.1 rounded
@@ -429,6 +440,7 @@ TEST(IOApiTests, FormatDoubleValues) {
               {"(G0.55,';')",
                   ".1000000000000000055511151231257827021181583404541015625;"},
               {"(G0,';')", ".1;"},
+              {"(EX20.12,';')", " 0XC.CCCCCCCCCCCDP-7;"},
           }},
       {// 1.5
           0x3ff8000000000000,
@@ -436,6 +448,7 @@ TEST(IOApiTests, FormatDoubleValues) {
               {"(E9.2,';')", " 0.15E+01;"},
               {"(F4.1,';')", " 1.5;"},
               {"(G7.1,';')", " 2.    ;"},
+              {"(EX9.1,';')", " 0XC.0P-3;"},
               {"(RN,E8.1,';')", " 0.2E+01;"},
               {"(RN,F3.0,';')", " 2.;"},
               {"(RN,G7.0,';')", " 0.E+01;"},
@@ -465,6 +478,7 @@ TEST(IOApiTests, FormatDoubleValues) {
               {"(RU,E8.1,';')", "-0.1E+01;"},
               {"(RZ,E8.1,';')", "-0.1E+01;"},
               {"(RC,E8.1,';')", "-0.2E+01;"},
+              {"(EX9.1,';')", "-0XC.0P-3;"},
           }},
       {// 2.5
           0x4004000000000000,
@@ -475,6 +489,7 @@ TEST(IOApiTests, FormatDoubleValues) {
               {"(RU,E8.1,';')", " 0.3E+01;"},
               {"(RZ,E8.1,';')", " 0.2E+01;"},
               {"(RC,E8.1,';')", " 0.3E+01;"},
+              {"(EX9.1,';')", " 0XA.0P-2;"},
           }},
       {// -2.5
           0xc004000000000000,
@@ -485,6 +500,7 @@ TEST(IOApiTests, FormatDoubleValues) {
               {"(RU,E8.1,';')", "-0.2E+01;"},
               {"(RZ,E8.1,';')", "-0.2E+01;"},
               {"(RC,E8.1,';')", "-0.3E+01;"},
+              {"(EX9.1,';')", "-0XA.0P-2;"},
           }},
       {// least positive nonzero subnormal
           1,
@@ -583,6 +599,7 @@ TEST(IOApiTests, FormatDoubleValues) {
                   "701797267771758512566055119913150489110145103786273816725095"
                   "583738973359899366480994116420570263709027924276754456522908"
                   "753868250641971826553344726563-323;"},
+              {"(EX24.13,';')", " 0X8.0000000000000P-1077;"},
           }},
       {// least positive nonzero normal
           0x10000000000000,
@@ -603,6 +620,7 @@ TEST(IOApiTests, FormatDoubleValues) {
                   "61364675687023986783152906809846172109246253967285156250-"
                   "307;"},
               {"(G0,';')", ".22250738585072014E-307;"},
+              {"(EX24.13,';')", " 0X8.0000000000000P-1025;"},
           }},
       {// greatest finite
           0x7fefffffffffffffuLL,
@@ -633,6 +651,31 @@ TEST(IOApiTests, FormatDoubleValues) {
                   "123348274797826204144723168738177180919299881250404026184124"
                   "8583680000+306;"},
               {"(G0,';')", ".17976931348623157E+309;"},
+              {"(EX24.13,';')", " 0XF.FFFFFFFFFFFF8P+1020;"},
+          }},
+      {// EX rounding
+          0x3ff1000000000000uLL, // 1.0625
+          {
+              {"(F7.4,';')", " 1.0625;"},
+              {"(EX9.1,';')", " 0X8.8P-3;"},
+              {"(EX9.0,';')", "  0X8.P-3;"},
+              {"(RN,EX9.0,';')", "  0X8.P-3;"},
+              {"(RU,EX9.0,';')", "  0X9.P-3;"},
+              {"(RD,EX9.0,';')", "  0X8.P-3;"},
+              {"(RZ,EX9.0,';')", "  0X8.P-3;"},
+              {"(RC,EX9.0,';')", "  0X9.P-3;"},
+          }},
+      {// EX rounding
+          0xbff1000000000000uLL, // -1.0625
+          {
+              {"(F7.4,';')", "-1.0625;"},
+              {"(EX9.1,';')", "-0X8.8P-3;"},
+              {"(EX9.0,';')", " -0X8.P-3;"},
+              {"(RN,EX9.0,';')", " -0X8.P-3;"},
+              {"(RU,EX9.0,';')", " -0X8.P-3;"},
+              {"(RD,EX9.0,';')", " -0X9.P-3;"},
+              {"(RZ,EX9.0,';')", " -0X8.P-3;"},
+              {"(RC,EX9.0,';')", " -0X9.P-3;"},
           }},
   };
 
@@ -775,11 +818,11 @@ TEST(IOApiTests, FormatIntegerValues) {
 }
 
 //------------------------------------------------------------------------------
-/// Tests for input formatting real values
+/// Tests for input editing real values
 //------------------------------------------------------------------------------
 
 // Ensure double input values correctly map to raw uint64 values
-TEST(IOApiTests, FormatDoubleInputValues) {
+TEST(IOApiTests, EditDoubleInputValues) {
   using TestCaseTy = std::tuple<const char *, const char *, std::uint64_t>;
   static const std::vector<TestCaseTy> testCases{
       {"(F18.0)", "                 0", 0x0},
@@ -806,6 +849,21 @@ TEST(IOApiTests, FormatDoubleInputValues) {
       {"(BZ,F18.0)", "           .      ", 0x0},
       {"(BZ,F18.0)", "           . e +1 ", 0x0},
       {"(DC,F18.0)", "              12,5", 0x4029000000000000},
+      {"(EX22.0)", "0X0P0                 ", 0x0}, // +0.
+      {"(EX22.0)", "-0X0P0                ", 0x8000000000000000}, // -0.
+      {"(EX22.0)", "0X.8P1                ", 0x3ff0000000000000}, // 1.0
+      {"(EX22.0)", "0X8.P-3               ", 0x3ff0000000000000}, // 1.0
+      {"(EX22.0)", "0X.1P4                ", 0x3ff0000000000000}, // 1.0
+      {"(EX22.0)", "0X10.P-4              ", 0x3ff0000000000000}, // 1.0
+      {"(EX22.0)", "0X8.00P-3             ", 0x3ff0000000000000}, // 1.0
+      {"(EX22.0)", "0X80.0P-6             ", 0x4000000000000000}, // 2.0
+      {"(EX22.0)", "0XC.CCCCCCCCCCCDP-7   ", 0x3fb999999999999a}, // 0.1
+      {"(EX22.0)", "0X.8P-1021            ", 0x0010000000000000}, // min normal
+      {"(EX22.0)", "0X.8P-1022            ", 0x0008000000000000}, // subnormal
+      {"(EX22.0)", "0X.8P-1073            ", 0x0000000000000001}, // min subn.
+      {"(EX22.0)", "0X.FFFFFFFFFFFFF8P1024", 0x7fefffffffffffff}, // max finite
+      {"(EX22.0)", "0X.8P1025             ", 0x7ff0000000000000}, // +Inf
+      {"(EX22.0)", "-0X.8P1025            ", 0xfff0000000000000}, // -Inf
   };
   for (auto const &[format, data, want] : testCases) {
     auto cookie{IONAME(BeginInternalFormattedInput)(
diff --git a/libc/config/gpu/entrypoints.txt b/libc/config/gpu/entrypoints.txt
index ad68216a76..756cddb0e8 100644
--- a/libc/config/gpu/entrypoints.txt
+++ b/libc/config/gpu/entrypoints.txt
@@ -63,6 +63,8 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.stdlib.lldiv
     libc.src.stdlib.qsort
     libc.src.stdlib.qsort_r
+    libc.src.stdlib.rand
+    libc.src.stdlib.srand
     libc.src.stdlib.strtod
     libc.src.stdlib.strtof
     libc.src.stdlib.strtol
@@ -104,6 +106,7 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.stdio.fgetc
     libc.src.stdio.getc
     libc.src.stdio.getchar
+    libc.src.stdio.ungetc
     libc.src.stdio.stdin
     libc.src.stdio.stdout
     libc.src.stdio.stderr
@@ -124,11 +127,19 @@ set(TARGET_LIBC_ENTRYPOINTS
 
 set(TARGET_LIBM_ENTRYPOINTS
     # math.h entrypoints
+    libc.src.math.acos
     libc.src.math.acosf
+    libc.src.math.acosh
     libc.src.math.acoshf
+    libc.src.math.asin
     libc.src.math.asinf
+    libc.src.math.asinh
     libc.src.math.asinhf
+    libc.src.math.atan
     libc.src.math.atanf
+    libc.src.math.atan2
+    libc.src.math.atan2f
+    libc.src.math.atanh
     libc.src.math.atanhf
     libc.src.math.ceil
     libc.src.math.ceilf
@@ -138,9 +149,15 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.cosf
     libc.src.math.cosh
     libc.src.math.coshf
+    libc.src.math.erf
+    libc.src.math.erff
+    libc.src.math.exp10
     libc.src.math.exp10f
+    libc.src.math.exp2
     libc.src.math.exp2f
+    libc.src.math.exp
     libc.src.math.expf
+    libc.src.math.expm1
     libc.src.math.expm1f
     libc.src.math.fabs
     libc.src.math.fabsf
@@ -168,15 +185,26 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.llrintf
     libc.src.math.llround
     libc.src.math.llroundf
-    libc.src.math.pow
-    libc.src.math.powf
-    libc.src.math.sin
+    libc.src.math.log10
+    libc.src.math.log10f
+    libc.src.math.log1p
+    libc.src.math.log1pf
+    libc.src.math.log2
+    libc.src.math.log2f
+    libc.src.math.log
+    libc.src.math.logf
+    libc.src.math.lrint
+    libc.src.math.lrintf
+    libc.src.math.lround
+    libc.src.math.lroundf
     libc.src.math.modf
     libc.src.math.modff
     libc.src.math.nearbyint
     libc.src.math.nearbyintf
     libc.src.math.nextafter
     libc.src.math.nextafterf
+    libc.src.math.pow
+    libc.src.math.powf
     libc.src.math.remainder
     libc.src.math.remainderf
     libc.src.math.remquo
@@ -187,6 +215,10 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.roundf
     libc.src.math.scalbn
     libc.src.math.scalbnf
+    libc.src.math.sin
+    libc.src.math.sinf
+    libc.src.math.sincos
+    libc.src.math.sincosf
     libc.src.math.sinh
     libc.src.math.sinhf
     libc.src.math.sqrt
@@ -195,6 +227,8 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.tanf
     libc.src.math.tanh
     libc.src.math.tanhf
+    libc.src.math.tgamma
+    libc.src.math.tgammaf
     libc.src.math.trunc
     libc.src.math.truncf
 )
diff --git a/libc/config/linux/aarch64/headers.txt b/libc/config/linux/aarch64/headers.txt
index 6e30c7c29b..c47e05c924 100644
--- a/libc/config/linux/aarch64/headers.txt
+++ b/libc/config/linux/aarch64/headers.txt
@@ -2,6 +2,7 @@ set(TARGET_PUBLIC_HEADERS
     libc.include.assert
     libc.include.ctype
     libc.include.errno
+    libc.include.features
     libc.include.fenv
     libc.include.inttypes
     libc.include.math
diff --git a/libc/config/linux/riscv/headers.txt b/libc/config/linux/riscv/headers.txt
index aaa75a9dd0..24247ee581 100644
--- a/libc/config/linux/riscv/headers.txt
+++ b/libc/config/linux/riscv/headers.txt
@@ -4,6 +4,7 @@ set(TARGET_PUBLIC_HEADERS
     libc.include.dirent
     libc.include.errno
     libc.include.fcntl
+    libc.include.features
     libc.include.fenv
     libc.include.inttypes
     libc.include.math
diff --git a/libc/config/linux/x86_64/headers.txt b/libc/config/linux/x86_64/headers.txt
index aaa75a9dd0..24247ee581 100644
--- a/libc/config/linux/x86_64/headers.txt
+++ b/libc/config/linux/x86_64/headers.txt
@@ -4,6 +4,7 @@ set(TARGET_PUBLIC_HEADERS
     libc.include.dirent
     libc.include.errno
     libc.include.fcntl
+    libc.include.features
     libc.include.fenv
     libc.include.inttypes
     libc.include.math
diff --git a/libc/docs/gpu/support.rst b/libc/docs/gpu/support.rst
index fd27273ed5..806af5f219 100644
--- a/libc/docs/gpu/support.rst
+++ b/libc/docs/gpu/support.rst
@@ -134,6 +134,7 @@ ftell          |check|    |check|
 fflush         |check|    |check|
 fgetc          |check|    |check|
 fgets          |check|    |check|
+ungetc         |check|    |check|
 getc           |check|    |check|
 getchar        |check|    |check|
 puts           |check|    |check|
diff --git a/libc/fuzzing/stdio/printf_float_conv_fuzz.cpp b/libc/fuzzing/stdio/printf_float_conv_fuzz.cpp
index dd3902eebd..798e1a3866 100644
--- a/libc/fuzzing/stdio/printf_float_conv_fuzz.cpp
+++ b/libc/fuzzing/stdio/printf_float_conv_fuzz.cpp
@@ -29,6 +29,14 @@ inline bool simple_streq(char *first, char *second, int length) {
   return true;
 }
 
+inline int simple_strlen(const char *str) {
+  int i = 0;
+  for (; *str; ++str, ++i) {
+    ;
+  }
+  return i;
+}
+
 enum class TestResult {
   Success,
   BufferSizeFailed,
@@ -36,7 +44,8 @@ enum class TestResult {
   StringsNotEqual,
 };
 
-inline TestResult test_vals(const char *fmt, double num, int prec, int width) {
+template <typename F>
+inline TestResult test_vals(const char *fmt, F num, int prec, int width) {
   // Call snprintf on a nullptr to get the buffer size.
   int buffer_size = LIBC_NAMESPACE::snprintf(nullptr, 0, fmt, width, prec, num);
 
@@ -70,10 +79,7 @@ inline TestResult test_vals(const char *fmt, double num, int prec, int width) {
 }
 
 constexpr char const *fmt_arr[] = {
-    "%*.*f",
-    "%*.*e",
-    "%*.*g",
-    "%*.*a",
+    "%*.*f", "%*.*e", "%*.*g", "%*.*a", "%*.*Lf", "%*.*Le", "%*.*Lg", "%*.*La",
 };
 
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
@@ -100,6 +106,12 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
 
   num = LIBC_NAMESPACE::fputil::FPBits<double>(raw_num).get_val();
 
+  // While we could create a "ld_raw_num" from additional bytes, it's much
+  // easier to stick with simply casting num to long double. This avoids the
+  // issues around 80 bit long doubles, especially unnormal and pseudo-denormal
+  // numbers, which MPFR doesn't handle well.
+  long double ld_num = static_cast<long double>(num);
+
   if (width > MAX_SIZE) {
     width = MAX_SIZE;
   } else if (width < -MAX_SIZE) {
@@ -114,7 +126,13 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
 
   for (size_t cur_fmt = 0; cur_fmt < sizeof(fmt_arr) / sizeof(char *);
        ++cur_fmt) {
-    TestResult result = test_vals(fmt_arr[cur_fmt], num, prec, width);
+    int fmt_len = simple_strlen(fmt_arr[cur_fmt]);
+    TestResult result;
+    if (fmt_arr[cur_fmt][fmt_len - 2] == 'L') {
+      result = test_vals<long double>(fmt_arr[cur_fmt], ld_num, prec, width);
+    } else {
+      result = test_vals<double>(fmt_arr[cur_fmt], num, prec, width);
+    }
     if (result != TestResult::Success) {
       __builtin_trap();
     }
diff --git a/libc/include/CMakeLists.txt b/libc/include/CMakeLists.txt
index fd2cb93514..9d170603ff 100644
--- a/libc/include/CMakeLists.txt
+++ b/libc/include/CMakeLists.txt
@@ -46,6 +46,15 @@ add_gen_header(
     .llvm-libc-types.mode_t
 )
 
+add_gen_header(
+  features
+  DEF_FILE features.h.def
+  GEN_HDR features.h
+  DEPENDS
+    .llvm_libc_common_h
+    .llvm-libc-macros.features_macros
+)
+
 add_gen_header(
   fenv
   DEF_FILE fenv.h.def
diff --git a/libc/include/features.h.def b/libc/include/features.h.def
new file mode 100644
index 0000000000..64205f57ac
--- /dev/null
+++ b/libc/include/features.h.def
@@ -0,0 +1,17 @@
+//===-- C standard library header features.h ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_FEATURES_H
+#define LLVM_LIBC_FEATURES_H
+
+#include <__llvm-libc-common.h>
+#include <llvm-libc-macros/features-macros.h>
+
+%%public_api()
+
+#endif // LLVM_LIBC_FEATURES_H
diff --git a/libc/include/llvm-libc-macros/CMakeLists.txt b/libc/include/llvm-libc-macros/CMakeLists.txt
index 8a225889a6..9c9e6bfd12 100644
--- a/libc/include/llvm-libc-macros/CMakeLists.txt
+++ b/libc/include/llvm-libc-macros/CMakeLists.txt
@@ -46,6 +46,12 @@ add_macro_header(
     fcntl-macros.h
 )
 
+add_macro_header(
+  features_macros
+  HDR
+    features-macros.h
+)
+
 add_macro_header(
   fenv_macros
   HDR
diff --git a/libc/include/llvm-libc-macros/features-macros.h b/libc/include/llvm-libc-macros/features-macros.h
new file mode 100644
index 0000000000..2938b3ccb9
--- /dev/null
+++ b/libc/include/llvm-libc-macros/features-macros.h
@@ -0,0 +1,14 @@
+//===-- Definition of macros from features.h ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __LLVM_LIBC_MACROS_FEATURES_MACROS_H
+#define __LLVM_LIBC_MACROS_FEATURES_MACROS_H
+
+#define __LLVM_LIBC__ 1
+
+#endif // __LLVM_LIBC_MACROS_FEATURES_MACROS_H
diff --git a/libc/include/llvm-libc-types/rpc_opcodes_t.h b/libc/include/llvm-libc-types/rpc_opcodes_t.h
index 61e17756fa..7b85428dd3 100644
--- a/libc/include/llvm-libc-types/rpc_opcodes_t.h
+++ b/libc/include/llvm-libc-types/rpc_opcodes_t.h
@@ -17,6 +17,7 @@ typedef enum {
   RPC_WRITE_TO_STREAM,
   RPC_WRITE_TO_STDOUT_NEWLINE,
   RPC_READ_FROM_STREAM,
+  RPC_READ_FGETS,
   RPC_OPEN_FILE,
   RPC_CLOSE_FILE,
   RPC_MALLOC,
@@ -29,6 +30,7 @@ typedef enum {
   RPC_FSEEK,
   RPC_FTELL,
   RPC_FFLUSH,
+  RPC_UNGETC,
   RPC_LAST = 0xFFFF,
 } rpc_opcode_t;
 
diff --git a/libc/src/__support/FPUtil/FPBits.h b/libc/src/__support/FPUtil/FPBits.h
index 8d3b87c70e..b423ba41f1 100644
--- a/libc/src/__support/FPUtil/FPBits.h
+++ b/libc/src/__support/FPUtil/FPBits.h
@@ -51,23 +51,23 @@ template <typename T> struct FPBits {
 
   UIntType bits;
 
-  LIBC_INLINE void set_mantissa(UIntType mantVal) {
+  LIBC_INLINE constexpr void set_mantissa(UIntType mantVal) {
     mantVal &= (FloatProp::MANTISSA_MASK);
     bits &= ~(FloatProp::MANTISSA_MASK);
     bits |= mantVal;
   }
 
-  LIBC_INLINE UIntType get_mantissa() const {
+  LIBC_INLINE constexpr UIntType get_mantissa() const {
     return bits & FloatProp::MANTISSA_MASK;
   }
 
-  LIBC_INLINE void set_unbiased_exponent(UIntType expVal) {
+  LIBC_INLINE constexpr void set_unbiased_exponent(UIntType expVal) {
     expVal = (expVal << (FloatProp::MANTISSA_WIDTH)) & FloatProp::EXPONENT_MASK;
     bits &= ~(FloatProp::EXPONENT_MASK);
     bits |= expVal;
   }
 
-  LIBC_INLINE uint16_t get_unbiased_exponent() const {
+  LIBC_INLINE constexpr uint16_t get_unbiased_exponent() const {
     return uint16_t((bits & FloatProp::EXPONENT_MASK) >>
                     (FloatProp::MANTISSA_WIDTH));
   }
@@ -81,13 +81,13 @@ template <typename T> struct FPBits {
            (FloatProp::MANTISSA_MASK & bits);
   }
 
-  LIBC_INLINE void set_sign(bool signVal) {
+  LIBC_INLINE constexpr void set_sign(bool signVal) {
     bits |= FloatProp::SIGN_MASK;
     if (!signVal)
       bits -= FloatProp::SIGN_MASK;
   }
 
-  LIBC_INLINE bool get_sign() const {
+  LIBC_INLINE constexpr bool get_sign() const {
     return (bits & FloatProp::SIGN_MASK) != 0;
   }
 
@@ -118,13 +118,15 @@ template <typename T> struct FPBits {
 
   LIBC_INLINE constexpr T get_val() const { return cpp::bit_cast<T>(bits); }
 
-  LIBC_INLINE void set_val(T value) { bits = cpp::bit_cast<UIntType>(value); }
+  LIBC_INLINE constexpr void set_val(T value) {
+    bits = cpp::bit_cast<UIntType>(value);
+  }
 
-  LIBC_INLINE explicit operator T() const { return get_val(); }
+  LIBC_INLINE constexpr explicit operator T() const { return get_val(); }
 
-  LIBC_INLINE UIntType uintval() const { return bits; }
+  LIBC_INLINE constexpr UIntType uintval() const { return bits; }
 
-  LIBC_INLINE int get_exponent() const {
+  LIBC_INLINE constexpr int get_exponent() const {
     return int(get_unbiased_exponent()) - EXPONENT_BIAS;
   }
 
@@ -134,7 +136,7 @@ template <typename T> struct FPBits {
   // values are calculated from the exponent, since just subtracting the bias
   // will give a slightly incorrect result. Additionally, zero has an exponent
   // of zero, and that should actually be treated as zero.
-  LIBC_INLINE int get_explicit_exponent() const {
+  LIBC_INLINE constexpr int get_explicit_exponent() const {
     const int unbiased_exp = int(get_unbiased_exponent());
     if (is_zero()) {
       return 0;
@@ -145,25 +147,25 @@ template <typename T> struct FPBits {
     }
   }
 
-  LIBC_INLINE bool is_zero() const {
+  LIBC_INLINE constexpr bool is_zero() const {
     // Remove sign bit by shift
     return (bits << 1) == 0;
   }
 
-  LIBC_INLINE bool is_inf() const {
+  LIBC_INLINE constexpr bool is_inf() const {
     return (bits & FloatProp::EXP_MANT_MASK) == FloatProp::EXPONENT_MASK;
   }
 
-  LIBC_INLINE bool is_nan() const {
+  LIBC_INLINE constexpr bool is_nan() const {
     return (bits & FloatProp::EXP_MANT_MASK) > FloatProp::EXPONENT_MASK;
   }
 
-  LIBC_INLINE bool is_quiet_nan() const {
+  LIBC_INLINE constexpr bool is_quiet_nan() const {
     return (bits & FloatProp::EXP_MANT_MASK) ==
            (FloatProp::EXPONENT_MASK | FloatProp::QUIET_NAN_MASK);
   }
 
-  LIBC_INLINE bool is_inf_or_nan() const {
+  LIBC_INLINE constexpr bool is_inf_or_nan() const {
     return (bits & FloatProp::EXPONENT_MASK) == FloatProp::EXPONENT_MASK;
   }
 
@@ -226,8 +228,8 @@ template <typename T> struct FPBits {
     return result;
   }
 
-  LIBC_INLINE static FPBits<T> create_value(bool sign, UIntType unbiased_exp,
-                                            UIntType mantissa) {
+  LIBC_INLINE static constexpr FPBits<T>
+  create_value(bool sign, UIntType unbiased_exp, UIntType mantissa) {
     FPBits<T> result;
     result.set_sign(sign);
     result.set_unbiased_exponent(unbiased_exp);
diff --git a/libc/src/math/CMakeLists.txt b/libc/src/math/CMakeLists.txt
index 00e0e4e6a5..f1f7271498 100644
--- a/libc/src/math/CMakeLists.txt
+++ b/libc/src/math/CMakeLists.txt
@@ -54,14 +54,23 @@ function(add_math_entrypoint_object name)
   )
 endfunction()
 
+add_math_entrypoint_object(acos)
 add_math_entrypoint_object(acosf)
+add_math_entrypoint_object(acosh)
 add_math_entrypoint_object(acoshf)
 
+add_math_entrypoint_object(asin)
 add_math_entrypoint_object(asinf)
+add_math_entrypoint_object(asinh)
 add_math_entrypoint_object(asinhf)
 
+add_math_entrypoint_object(atan)
 add_math_entrypoint_object(atanf)
 
+add_math_entrypoint_object(atan2)
+add_math_entrypoint_object(atan2f)
+
+add_math_entrypoint_object(atanh)
 add_math_entrypoint_object(atanhf)
 
 add_math_entrypoint_object(ceil)
@@ -77,6 +86,7 @@ add_math_entrypoint_object(cosf)
 add_math_entrypoint_object(cosh)
 add_math_entrypoint_object(coshf)
 
+add_math_entrypoint_object(erf)
 add_math_entrypoint_object(erff)
 
 add_math_entrypoint_object(exp)
@@ -199,6 +209,7 @@ add_math_entrypoint_object(scalbn)
 add_math_entrypoint_object(scalbnf)
 add_math_entrypoint_object(scalbnl)
 
+add_math_entrypoint_object(sincos)
 add_math_entrypoint_object(sincosf)
 
 add_math_entrypoint_object(sin)
@@ -217,6 +228,9 @@ add_math_entrypoint_object(tanf)
 add_math_entrypoint_object(tanh)
 add_math_entrypoint_object(tanhf)
 
+add_math_entrypoint_object(tgamma)
+add_math_entrypoint_object(tgammaf)
+
 add_math_entrypoint_object(trunc)
 add_math_entrypoint_object(truncf)
 add_math_entrypoint_object(truncl)
diff --git a/libc/src/math/acos.h b/libc/src/math/acos.h
new file mode 100644
index 0000000000..da89a9dd3d
--- /dev/null
+++ b/libc/src/math/acos.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for acos --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_ACOS_H
+#define LLVM_LIBC_SRC_MATH_ACOS_H
+
+namespace LIBC_NAMESPACE {
+
+double acos(double x);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_ACOS_H
diff --git a/libc/src/math/acosh.h b/libc/src/math/acosh.h
new file mode 100644
index 0000000000..a5bbd82c12
--- /dev/null
+++ b/libc/src/math/acosh.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for acosh -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_ACOSH_H
+#define LLVM_LIBC_SRC_MATH_ACOSH_H
+
+namespace LIBC_NAMESPACE {
+
+double acosh(double x);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_ACOSH_H
diff --git a/libc/src/math/asin.h b/libc/src/math/asin.h
new file mode 100644
index 0000000000..e443776e50
--- /dev/null
+++ b/libc/src/math/asin.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for asin --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_ASIN_H
+#define LLVM_LIBC_SRC_MATH_ASIN_H
+
+namespace LIBC_NAMESPACE {
+
+double asin(double x);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_ASIN_H
diff --git a/libc/src/math/asinh.h b/libc/src/math/asinh.h
new file mode 100644
index 0000000000..418bf96179
--- /dev/null
+++ b/libc/src/math/asinh.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for asinh -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_ASINH_H
+#define LLVM_LIBC_SRC_MATH_ASINH_H
+
+namespace LIBC_NAMESPACE {
+
+double asinh(double x);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_ASINH_H
diff --git a/libc/src/math/atan.h b/libc/src/math/atan.h
new file mode 100644
index 0000000000..bcbc97a52e
--- /dev/null
+++ b/libc/src/math/atan.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for atan --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_ATAN_H
+#define LLVM_LIBC_SRC_MATH_ATAN_H
+
+namespace LIBC_NAMESPACE {
+
+double atan(double x);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_ATAN_H
diff --git a/libc/src/math/atan2.h b/libc/src/math/atan2.h
new file mode 100644
index 0000000000..024bbfb5fe
--- /dev/null
+++ b/libc/src/math/atan2.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for atan2 -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_ATAN2_H
+#define LLVM_LIBC_SRC_MATH_ATAN2_H
+
+namespace LIBC_NAMESPACE {
+
+double atan2(double x, double y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_ATAN2_H
diff --git a/libc/src/math/atan2f.h b/libc/src/math/atan2f.h
new file mode 100644
index 0000000000..25d2de0934
--- /dev/null
+++ b/libc/src/math/atan2f.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for atan2f ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_ATAN2F_H
+#define LLVM_LIBC_SRC_MATH_ATAN2F_H
+
+namespace LIBC_NAMESPACE {
+
+float atan2f(float x, float y);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_ATAN2F_H
diff --git a/libc/src/math/atanh.h b/libc/src/math/atanh.h
new file mode 100644
index 0000000000..dc84d07c02
--- /dev/null
+++ b/libc/src/math/atanh.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for atanh -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_ATANH_H
+#define LLVM_LIBC_SRC_MATH_ATANH_H
+
+namespace LIBC_NAMESPACE {
+
+double atanh(double x);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_ATANH_H
diff --git a/libc/src/math/erf.h b/libc/src/math/erf.h
new file mode 100644
index 0000000000..a38c92410d
--- /dev/null
+++ b/libc/src/math/erf.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for erf ---------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_ERF_H
+#define LLVM_LIBC_SRC_MATH_ERF_H
+
+namespace LIBC_NAMESPACE {
+
+double erf(double x);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_ERF_H
diff --git a/libc/src/math/gpu/CMakeLists.txt b/libc/src/math/gpu/CMakeLists.txt
index cee7b7d9db..75a916e2a0 100644
--- a/libc/src/math/gpu/CMakeLists.txt
+++ b/libc/src/math/gpu/CMakeLists.txt
@@ -183,6 +183,46 @@ add_math_entrypoint_gpu_object(
     -O2
 )
 
+add_math_entrypoint_gpu_object(
+  lround
+  SRCS
+    lround.cpp
+  HDRS
+    ../lround.h
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_math_entrypoint_gpu_object(
+  lroundf
+  SRCS
+    lroundf.cpp
+  HDRS
+    ../lroundf.h
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_math_entrypoint_gpu_object(
+  llround
+  SRCS
+    llround.cpp
+  HDRS
+    ../llround.h
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_math_entrypoint_gpu_object(
+  llroundf
+  SRCS
+    llroundf.cpp
+  HDRS
+    ../llroundf.h
+  COMPILE_OPTIONS
+    -O2
+)
+
 add_math_entrypoint_gpu_object(
   modf
   SRCS
@@ -283,16 +323,6 @@ add_math_entrypoint_gpu_object(
     -O2
 )
 
-add_math_entrypoint_gpu_object(
-  sinhf
-  SRCS
-    sinhf.cpp
-  HDRS
-    ../sinhf.h
-  COMPILE_OPTIONS
-    -O2
-)
-
 add_math_entrypoint_gpu_object(
   sqrt
   SRCS
@@ -323,16 +353,6 @@ add_math_entrypoint_gpu_object(
     -O2
 )
 
-add_math_entrypoint_gpu_object(
-  tanf
-  SRCS
-    tanf.cpp
-  HDRS
-    ../tanf.h
-  COMPILE_OPTIONS
-    -O2
-)
-
 add_math_entrypoint_gpu_object(
   tanh
   SRCS
@@ -343,16 +363,6 @@ add_math_entrypoint_gpu_object(
     -O2
 )
 
-add_math_entrypoint_gpu_object(
-  tanhf
-  SRCS
-    tanhf.cpp
-  HDRS
-    ../tanhf.h
-  COMPILE_OPTIONS
-    -O2
-)
-
 add_math_entrypoint_gpu_object(
   trunc
   SRCS
diff --git a/libc/src/math/gpu/vendor/llround.cpp b/libc/src/math/gpu/llround.cpp
similarity index 80%
rename from libc/src/math/gpu/vendor/llround.cpp
rename to libc/src/math/gpu/llround.cpp
index e53701bda7..afd9830873 100644
--- a/libc/src/math/gpu/vendor/llround.cpp
+++ b/libc/src/math/gpu/llround.cpp
@@ -1,4 +1,4 @@
-//===-- Implementation of the llround function for GPU --------------------===//
+//===-- Implementation of the GPU llround function ------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -9,12 +9,10 @@
 #include "src/math/llround.h"
 #include "src/__support/common.h"
 
-#include "common.h"
-
 namespace LIBC_NAMESPACE {
 
 LLVM_LIBC_FUNCTION(long long, llround, (double x)) {
-  return internal::llround(x);
+  return __builtin_llround(x);
 }
 
 } // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/gpu/vendor/llroundf.cpp b/libc/src/math/gpu/llroundf.cpp
similarity index 80%
rename from libc/src/math/gpu/vendor/llroundf.cpp
rename to libc/src/math/gpu/llroundf.cpp
index ddb5ec00a0..897ed15b69 100644
--- a/libc/src/math/gpu/vendor/llroundf.cpp
+++ b/libc/src/math/gpu/llroundf.cpp
@@ -1,4 +1,4 @@
-//===-- Implementation of the llroundf function for GPU -------------------===//
+//===-- Implementation of the GPU llroundf function -----------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -9,12 +9,10 @@
 #include "src/math/llroundf.h"
 #include "src/__support/common.h"
 
-#include "common.h"
-
 namespace LIBC_NAMESPACE {
 
 LLVM_LIBC_FUNCTION(long long, llroundf, (float x)) {
-  return internal::llroundf(x);
+  return __builtin_lroundf(x);
 }
 
 } // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/gpu/sinhf.cpp b/libc/src/math/gpu/lround.cpp
similarity index 70%
rename from libc/src/math/gpu/sinhf.cpp
rename to libc/src/math/gpu/lround.cpp
index ed69dffe3e..51e8f2245a 100644
--- a/libc/src/math/gpu/sinhf.cpp
+++ b/libc/src/math/gpu/lround.cpp
@@ -1,4 +1,4 @@
-//===-- Implementation of the GPU sinhf function --------------------------===//
+//===-- Implementation of the GPU lround function -------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,11 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/math/sinhf.h"
+#include "src/math/lround.h"
 #include "src/__support/common.h"
 
 namespace LIBC_NAMESPACE {
 
-LLVM_LIBC_FUNCTION(float, sinhf, (float x)) { return __builtin_sinhf(x); }
+LLVM_LIBC_FUNCTION(long, lround, (double x)) { return __builtin_lround(x); }
 
 } // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/gpu/tanf.cpp b/libc/src/math/gpu/lroundf.cpp
similarity index 69%
rename from libc/src/math/gpu/tanf.cpp
rename to libc/src/math/gpu/lroundf.cpp
index da7bd54ab0..2a6fe7200d 100644
--- a/libc/src/math/gpu/tanf.cpp
+++ b/libc/src/math/gpu/lroundf.cpp
@@ -1,4 +1,4 @@
-//===-- Implementation of the GPU tanf function ---------------------------===//
+//===-- Implementation of the GPU lroundf function ------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,11 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/math/tanf.h"
+#include "src/math/lroundf.h"
 #include "src/__support/common.h"
 
 namespace LIBC_NAMESPACE {
 
-LLVM_LIBC_FUNCTION(float, tanf, (float x)) { return __builtin_tanf(x); }
+LLVM_LIBC_FUNCTION(long, lroundf, (float x)) { return __builtin_lroundf(x); }
 
 } // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/gpu/vendor/CMakeLists.txt b/libc/src/math/gpu/vendor/CMakeLists.txt
index 2ee74a06a0..f699ca103b 100644
--- a/libc/src/math/gpu/vendor/CMakeLists.txt
+++ b/libc/src/math/gpu/vendor/CMakeLists.txt
@@ -20,7 +20,7 @@ if(CUDAToolkit_FOUND)
          "SHELL:-Xclang -mlink-builtin-bitcode -Xclang ${libdevice_path}")
   endif()
 else()
-  message(STATUS "Could not find the ROCm device library. Unimplemented "
+  message(STATUS "Could not find the CUDA device library. Unimplemented "
                  "functions will be an external reference to the vendor libraries.")
 endif()
 
@@ -29,6 +29,17 @@ endif()
 # will link in identity metadata from both libraries. This silences the warning.
 list(APPEND bitcode_link_flags "-Wno-linker-warnings")
 
+add_entrypoint_object(
+  acos
+  SRCS
+    acos.cpp
+  HDRS
+    ../../acos.h
+  COMPILE_OPTIONS
+    ${bitcode_link_flags}
+    -O2
+)
+
 add_entrypoint_object(
   acosf
   SRCS
@@ -40,6 +51,17 @@ add_entrypoint_object(
     -O2
 )
 
+add_entrypoint_object(
+  acosh
+  SRCS
+    acosh.cpp
+  HDRS
+    ../../acosh.h
+  COMPILE_OPTIONS
+    ${bitcode_link_flags}
+    -O2
+)
+
 add_entrypoint_object(
   acoshf
   SRCS
@@ -51,6 +73,17 @@ add_entrypoint_object(
     -O2
 )
 
+add_entrypoint_object(
+  asin
+  SRCS
+    asin.cpp
+  HDRS
+    ../../asin.h
+  COMPILE_OPTIONS
+    ${bitcode_link_flags}
+    -O2
+)
+
 add_entrypoint_object(
   asinf
   SRCS
@@ -63,11 +96,22 @@ add_entrypoint_object(
 )
 
 add_entrypoint_object(
-  asinhf
+  asinh
   SRCS
-    asinhf.cpp
+    asinh.cpp
   HDRS
-    ../../asinhf.h
+    ../../asinh.h
+  COMPILE_OPTIONS
+    ${bitcode_link_flags}
+    -O2
+)
+
+add_entrypoint_object(
+  atan
+  SRCS
+    atan.cpp
+  HDRS
+    ../../atan.h
   COMPILE_OPTIONS
     ${bitcode_link_flags}
     -O2
@@ -84,6 +128,39 @@ add_entrypoint_object(
     -O2
 )
 
+add_entrypoint_object(
+  atan2
+  SRCS
+    atan2.cpp
+  HDRS
+    ../../atan2.h
+  COMPILE_OPTIONS
+    ${bitcode_link_flags}
+    -O2
+)
+
+add_entrypoint_object(
+  atan2f
+  SRCS
+    atan2f.cpp
+  HDRS
+    ../../atan2f.h
+  COMPILE_OPTIONS
+    ${bitcode_link_flags}
+    -O2
+)
+
+add_entrypoint_object(
+  atanh
+  SRCS
+    atanh.cpp
+  HDRS
+    ../../atanh.h
+  COMPILE_OPTIONS
+    ${bitcode_link_flags}
+    -O2
+)
+
 add_entrypoint_object(
   atanhf
   SRCS
@@ -139,6 +216,50 @@ add_entrypoint_object(
     -O2
 )
 
+add_entrypoint_object(
+  erf
+  SRCS
+    erf.cpp
+  HDRS
+    ../../erf.h
+  COMPILE_OPTIONS
+    ${bitcode_link_flags}
+    -O2
+)
+
+add_entrypoint_object(
+  erff
+  SRCS
+    erff.cpp
+  HDRS
+    ../../erff.h
+  COMPILE_OPTIONS
+    ${bitcode_link_flags}
+    -O2
+)
+
+add_entrypoint_object(
+  exp
+  SRCS
+    exp.cpp
+  HDRS
+    ../../exp.h
+  COMPILE_OPTIONS
+    ${bitcode_link_flags}
+    -O2
+)
+
+add_entrypoint_object(
+  exp10
+  SRCS
+    exp10.cpp
+  HDRS
+    ../../exp10.h
+  COMPILE_OPTIONS
+    ${bitcode_link_flags}
+    -O2
+)
+
 add_entrypoint_object(
   exp10f
   SRCS
@@ -150,6 +271,17 @@ add_entrypoint_object(
     -O2
 )
 
+add_entrypoint_object(
+  exp2
+  SRCS
+    exp2.cpp
+  HDRS
+    ../../exp2.h
+  COMPILE_OPTIONS
+    ${bitcode_link_flags}
+    -O2
+)
+
 add_entrypoint_object(
   exp2f
   SRCS
@@ -172,6 +304,17 @@ add_entrypoint_object(
     -O2
 )
 
+add_entrypoint_object(
+  expm1
+  SRCS
+    expm1.cpp
+  HDRS
+    ../../expm1.h
+  COMPILE_OPTIONS
+    ${bitcode_link_flags}
+    -O2
+)
+
 add_entrypoint_object(
   expm1f
   SRCS
@@ -249,6 +392,94 @@ add_entrypoint_object(
     -O2
 )
 
+add_entrypoint_object(
+  log10
+  SRCS
+    log10.cpp
+  HDRS
+    ../../log10.h
+  COMPILE_OPTIONS
+    ${bitcode_link_flags}
+    -O2
+)
+
+add_entrypoint_object(
+  log10f
+  SRCS
+    log10f.cpp
+  HDRS
+    ../../log10f.h
+  COMPILE_OPTIONS
+    ${bitcode_link_flags}
+    -O2
+)
+
+add_entrypoint_object(
+  log2
+  SRCS
+    log2.cpp
+  HDRS
+    ../../log2.h
+  COMPILE_OPTIONS
+    ${bitcode_link_flags}
+    -O2
+)
+
+add_entrypoint_object(
+  log2f
+  SRCS
+    log2f.cpp
+  HDRS
+    ../../log2f.h
+  COMPILE_OPTIONS
+    ${bitcode_link_flags}
+    -O2
+)
+
+add_entrypoint_object(
+  log
+  SRCS
+    log.cpp
+  HDRS
+    ../../log.h
+  COMPILE_OPTIONS
+    ${bitcode_link_flags}
+    -O2
+)
+
+add_entrypoint_object(
+  logf
+  SRCS
+    logf.cpp
+  HDRS
+    ../../logf.h
+  COMPILE_OPTIONS
+    ${bitcode_link_flags}
+    -O2
+)
+
+add_entrypoint_object(
+  lrint
+  SRCS
+    lrint.cpp
+  HDRS
+    ../../lrint.h
+  COMPILE_OPTIONS
+    ${bitcode_link_flags}
+    -O2
+)
+
+add_entrypoint_object(
+  lrintf
+  SRCS
+    lrintf.cpp
+  HDRS
+    ../../lrintf.h
+  COMPILE_OPTIONS
+    ${bitcode_link_flags}
+    -O2
+)
+
 add_entrypoint_object(
   ldexp
   SRCS
@@ -272,67 +503,66 @@ add_entrypoint_object(
 )
 
 add_entrypoint_object(
-  llrint
+  log1p
   SRCS
-    llrint.cpp
+    log1p.cpp
   HDRS
-    ../../llrint.h
+    ../../log1p.h
   COMPILE_OPTIONS
     ${bitcode_link_flags}
     -O2
 )
 
 add_entrypoint_object(
-  llrintf
+  log1pf
   SRCS
-    llrintf.cpp
+    log1pf.cpp
   HDRS
-    ../../llrintf.h
+    ../../log1pf.h
   COMPILE_OPTIONS
     ${bitcode_link_flags}
     -O2
 )
 
 add_entrypoint_object(
-  remquo
+  llrint
   SRCS
-    remquo.cpp
+    llrint.cpp
   HDRS
-    ../../remquo.h
+    ../../llrint.h
   COMPILE_OPTIONS
     ${bitcode_link_flags}
     -O2
 )
 
 add_entrypoint_object(
-  remquof
+  llrintf
   SRCS
-    remquof.cpp
+    llrintf.cpp
   HDRS
-    ../../remquof.h
+    ../../llrintf.h
   COMPILE_OPTIONS
     ${bitcode_link_flags}
     -O2
 )
 
-
 add_entrypoint_object(
-  llround
+  remquo
   SRCS
-    llround.cpp
+    remquo.cpp
   HDRS
-    ../../llround.h
+    ../../remquo.h
   COMPILE_OPTIONS
     ${bitcode_link_flags}
     -O2
 )
 
 add_entrypoint_object(
-  llroundf
+  remquof
   SRCS
-    llroundf.cpp
+    remquof.cpp
   HDRS
-    ../../llroundf.h
+    ../../remquof.h
   COMPILE_OPTIONS
     ${bitcode_link_flags}
     -O2
@@ -515,6 +745,28 @@ add_entrypoint_object(
     -O2
 )
 
+add_entrypoint_object(
+  tgamma
+  SRCS
+    tgamma.cpp
+  HDRS
+    ../../tgamma.h
+  COMPILE_OPTIONS
+    ${bitcode_link_flags}
+    -O2
+)
+
+add_entrypoint_object(
+  tgammaf
+  SRCS
+    tgammaf.cpp
+  HDRS
+    ../../tgammaf.h
+  COMPILE_OPTIONS
+    ${bitcode_link_flags}
+    -O2
+)
+
 add_entrypoint_object(
   frexp
   SRCS
diff --git a/libc/src/math/gpu/vendor/acos.cpp b/libc/src/math/gpu/vendor/acos.cpp
new file mode 100644
index 0000000000..83b674fa6a
--- /dev/null
+++ b/libc/src/math/gpu/vendor/acos.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of the GPU acos function ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/acos.h"
+#include "src/__support/common.h"
+
+#include "common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(double, acos, (double x)) { return internal::acos(x); }
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/gpu/vendor/acosh.cpp b/libc/src/math/gpu/vendor/acosh.cpp
new file mode 100644
index 0000000000..cc1b8b572b
--- /dev/null
+++ b/libc/src/math/gpu/vendor/acosh.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of the GPU acosh function --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/acosh.h"
+#include "src/__support/common.h"
+
+#include "common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(double, acosh, (double x)) { return internal::acosh(x); }
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/gpu/vendor/amdgpu/amdgpu.h b/libc/src/math/gpu/vendor/amdgpu/amdgpu.h
index 280ae49989..43961fc759 100644
--- a/libc/src/math/gpu/vendor/amdgpu/amdgpu.h
+++ b/libc/src/math/gpu/vendor/amdgpu/amdgpu.h
@@ -16,19 +16,33 @@
 
 namespace LIBC_NAMESPACE {
 namespace internal {
+LIBC_INLINE double acos(double x) { return __ocml_acos_f64(x); }
 LIBC_INLINE float acosf(float x) { return __ocml_acos_f32(x); }
+LIBC_INLINE double acosh(double x) { return __ocml_acosh_f64(x); }
 LIBC_INLINE float acoshf(float x) { return __ocml_acosh_f32(x); }
+LIBC_INLINE double asin(double x) { return __ocml_asin_f64(x); }
 LIBC_INLINE float asinf(float x) { return __ocml_asin_f32(x); }
+LIBC_INLINE double asinh(double x) { return __ocml_asinh_f64(x); }
 LIBC_INLINE float asinhf(float x) { return __ocml_asinh_f32(x); }
+LIBC_INLINE double atan(double x) { return __ocml_atan_f64(x); }
 LIBC_INLINE float atanf(float x) { return __ocml_atan_f32(x); }
+LIBC_INLINE double atan2(double x, double y) { return __ocml_atan2_f64(x, y); }
+LIBC_INLINE float atan2f(float x, float y) { return __ocml_atan2_f32(x, y); }
+LIBC_INLINE double atanh(double x) { return __ocml_atanh_f64(x); }
 LIBC_INLINE float atanhf(float x) { return __ocml_atanh_f32(x); }
 LIBC_INLINE double cos(double x) { return __ocml_cos_f64(x); }
 LIBC_INLINE float cosf(float x) { return __ocml_cos_f32(x); }
 LIBC_INLINE double cosh(double x) { return __ocml_cosh_f64(x); }
 LIBC_INLINE float coshf(float x) { return __ocml_cosh_f32(x); }
+LIBC_INLINE double erf(double x) { return __ocml_erf_f64(x); }
+LIBC_INLINE float erff(float x) { return __ocml_erf_f32(x); }
+LIBC_INLINE double exp(double x) { return __builtin_exp(x); }
 LIBC_INLINE float expf(float x) { return __builtin_expf(x); }
-LIBC_INLINE float exp2f(float x) { return __builtin_exp2f(x); }
+LIBC_INLINE double exp2(double x) { return __ocml_exp2_f64(x); }
+LIBC_INLINE float exp2f(float x) { return __ocml_exp2_f32(x); }
+LIBC_INLINE double exp10(double x) { return __ocml_exp10_f64(x); }
 LIBC_INLINE float exp10f(float x) { return __ocml_exp10_f32(x); }
+LIBC_INLINE double expm1(double x) { return __ocml_expm1_f64(x); }
 LIBC_INLINE float expm1f(float x) { return __ocml_expm1_f32(x); }
 LIBC_INLINE double fdim(double x, double y) { return __ocml_fdim_f64(x, y); }
 LIBC_INLINE float fdimf(float x, float y) { return __ocml_fdim_f32(x, y); }
@@ -44,11 +58,19 @@ LIBC_INLINE long long llrint(double x) {
 LIBC_INLINE long long llrintf(float x) {
   return static_cast<long long>(__builtin_rintf(x));
 }
-LIBC_INLINE long long llround(double x) {
-  return static_cast<long long>(__builtin_round(x));
+LIBC_INLINE double log10(double x) { return __ocml_log10_f64(x); }
+LIBC_INLINE float log10f(float x) { return __ocml_log10_f32(x); }
+LIBC_INLINE double log1p(double x) { return __ocml_log1p_f64(x); }
+LIBC_INLINE float log1pf(float x) { return __ocml_log1p_f32(x); }
+LIBC_INLINE double log2(double x) { return __ocml_log2_f64(x); }
+LIBC_INLINE float log2f(float x) { return __ocml_log2_f32(x); }
+LIBC_INLINE double log(double x) { return __ocml_log_f64(x); }
+LIBC_INLINE float logf(float x) { return __ocml_log_f32(x); }
+LIBC_INLINE long lrint(double x) {
+  return static_cast<long>(__builtin_rint(x));
 }
-LIBC_INLINE long long llroundf(float x) {
-  return static_cast<long long>(__builtin_roundf(x));
+LIBC_INLINE long lrintf(float x) {
+  return static_cast<long>(__builtin_rintf(x));
 }
 LIBC_INLINE double nextafter(double x, double y) {
   return __ocml_nextafter_f64(x, y);
@@ -96,6 +118,8 @@ LIBC_INLINE float remquof(float x, float y, int *q) {
   *q = tmp;
   return r;
 }
+LIBC_INLINE double tgamma(double x) { return __ocml_tgamma_f64(x); }
+LIBC_INLINE float tgammaf(float x) { return __ocml_tgamma_f32(x); }
 
 } // namespace internal
 } // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/gpu/vendor/amdgpu/declarations.h b/libc/src/math/gpu/vendor/amdgpu/declarations.h
index b07c079682..7a01cbc6ab 100644
--- a/libc/src/math/gpu/vendor/amdgpu/declarations.h
+++ b/libc/src/math/gpu/vendor/amdgpu/declarations.h
@@ -15,35 +15,54 @@ namespace LIBC_NAMESPACE {
 
 extern "C" {
 float __ocml_acos_f32(float);
+double __ocml_acos_f64(double);
 float __ocml_acosh_f32(float);
+double __ocml_acosh_f64(double);
 float __ocml_asin_f32(float);
+double __ocml_asin_f64(double);
 float __ocml_asinh_f32(float);
+double __ocml_asinh_f64(double);
 float __ocml_atan_f32(float);
+double __ocml_atan_f64(double);
+float __ocml_atan2_f32(float, float);
+double __ocml_atan2_f64(double, double);
 float __ocml_atanh_f32(float);
+double __ocml_atanh_f64(double);
 float __ocml_cos_f32(float);
 double __ocml_cos_f64(double);
 float __ocml_cosh_f32(float);
 double __ocml_cosh_f64(double);
+float __ocml_erf_f32(float);
+double __ocml_erf_f64(double);
 float __ocml_exp_f32(float);
+double __ocml_exp_f64(double);
 float __ocml_exp2_f32(float);
+double __ocml_exp2_f64(double);
 float __ocml_exp10_f32(float);
+double __ocml_exp10_f64(double);
+double __ocml_exp2_f64(double);
 float __ocml_expm1_f32(float);
+double __ocml_expm1_f64(double);
 float __ocml_fdim_f32(float, float);
 double __ocml_fdim_f64(double, double);
-double __ocml_hypot_f64(double, double);
 float __ocml_hypot_f32(float, float);
+double __ocml_hypot_f64(double, double);
 int __ocml_ilogb_f64(double);
 int __ocml_ilogb_f32(float);
 float __ocml_ldexp_f32(float, int);
 double __ocml_ldexp_f64(double, int);
+float __ocml_log10_f32(float);
+double __ocml_log10_f64(double);
+float __ocml_log1p_f32(float);
+double __ocml_log1p_f64(double);
+float __ocml_log2_f32(float);
+double __ocml_log2_f64(double);
+float __ocml_log_f32(float);
+double __ocml_log_f64(double);
 float __ocml_nextafter_f32(float, float);
 double __ocml_nextafter_f64(double, double);
 float __ocml_pow_f32(float, float);
 double __ocml_pow_f64(double, double);
-double __ocml_rint_f64(double);
-float __ocml_rint_f32(float);
-double __ocml_round_f64(double);
-float __ocml_round_f32(float);
 float __ocml_sin_f32(float);
 double __ocml_sin_f64(double);
 float __ocml_sincos_f32(float, float *);
@@ -56,6 +75,8 @@ float __ocml_tanh_f32(float);
 double __ocml_tanh_f64(double);
 float __ocml_remquo_f32(float, float, gpu::Private<int> *);
 double __ocml_remquo_f64(double, double, gpu::Private<int> *);
+double __ocml_tgamma_f64(double);
+float __ocml_tgamma_f32(float);
 }
 
 } // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/gpu/vendor/asin.cpp b/libc/src/math/gpu/vendor/asin.cpp
new file mode 100644
index 0000000000..24a8a136e8
--- /dev/null
+++ b/libc/src/math/gpu/vendor/asin.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of the GPU asin function ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/asin.h"
+#include "src/__support/common.h"
+
+#include "common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(double, asin, (double x)) { return internal::asin(x); }
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/gpu/vendor/asinh.cpp b/libc/src/math/gpu/vendor/asinh.cpp
new file mode 100644
index 0000000000..f417d9fd8c
--- /dev/null
+++ b/libc/src/math/gpu/vendor/asinh.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of the GPU asinh function --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/asinh.h"
+#include "src/__support/common.h"
+
+#include "common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(double, asinh, (double x)) { return internal::asinh(x); }
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/gpu/vendor/atan.cpp b/libc/src/math/gpu/vendor/atan.cpp
new file mode 100644
index 0000000000..45d7f02c02
--- /dev/null
+++ b/libc/src/math/gpu/vendor/atan.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of the GPU atan function ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/atan.h"
+#include "src/__support/common.h"
+
+#include "common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(double, atan, (double x)) { return internal::atan(x); }
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/gpu/vendor/atan2.cpp b/libc/src/math/gpu/vendor/atan2.cpp
new file mode 100644
index 0000000000..94e215e52c
--- /dev/null
+++ b/libc/src/math/gpu/vendor/atan2.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of the GPU atan2 function --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/atan2.h"
+#include "src/__support/common.h"
+
+#include "common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(double, atan2, (double x, double y)) {
+  return internal::atan2(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/gpu/vendor/atan2f.cpp b/libc/src/math/gpu/vendor/atan2f.cpp
new file mode 100644
index 0000000000..70caa568e3
--- /dev/null
+++ b/libc/src/math/gpu/vendor/atan2f.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of the GPU atan2f function -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/atan2f.h"
+#include "src/__support/common.h"
+
+#include "common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float, atan2f, (float x, float y)) {
+  return internal::atan2f(x, y);
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/gpu/vendor/atanh.cpp b/libc/src/math/gpu/vendor/atanh.cpp
new file mode 100644
index 0000000000..07a75fcbbf
--- /dev/null
+++ b/libc/src/math/gpu/vendor/atanh.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of the GPU atanh function --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/atanh.h"
+#include "src/__support/common.h"
+
+#include "common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(double, atanh, (double x)) { return internal::atanh(x); }
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/gpu/tanhf.cpp b/libc/src/math/gpu/vendor/erf.cpp
similarity index 68%
rename from libc/src/math/gpu/tanhf.cpp
rename to libc/src/math/gpu/vendor/erf.cpp
index be666fd9f8..190321ca25 100644
--- a/libc/src/math/gpu/tanhf.cpp
+++ b/libc/src/math/gpu/vendor/erf.cpp
@@ -1,4 +1,4 @@
-//===-- Implementation of the GPU tanhf function --------------------------===//
+//===-- Implementation of the GPU erf function ----------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,11 +6,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/math/tanhf.h"
+#include "src/math/erf.h"
 #include "src/__support/common.h"
 
+#include "common.h"
+
 namespace LIBC_NAMESPACE {
 
-LLVM_LIBC_FUNCTION(float, tanhf, (float x)) { return __builtin_tanhf(x); }
+LLVM_LIBC_FUNCTION(double, erf, (double x)) { return internal::erf(x); }
 
 } // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/gpu/vendor/erff.cpp b/libc/src/math/gpu/vendor/erff.cpp
new file mode 100644
index 0000000000..a5a08be54b
--- /dev/null
+++ b/libc/src/math/gpu/vendor/erff.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of the GPU erff function ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/erff.h"
+#include "src/__support/common.h"
+
+#include "common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float, erff, (float x)) { return internal::erff(x); }
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/gpu/vendor/exp.cpp b/libc/src/math/gpu/vendor/exp.cpp
new file mode 100644
index 0000000000..ee5a22019f
--- /dev/null
+++ b/libc/src/math/gpu/vendor/exp.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of the GPU exp function ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/exp.h"
+#include "src/__support/common.h"
+
+#include "common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(double, exp, (double x)) { return internal::exp(x); }
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/gpu/vendor/exp10.cpp b/libc/src/math/gpu/vendor/exp10.cpp
new file mode 100644
index 0000000000..8557a33f01
--- /dev/null
+++ b/libc/src/math/gpu/vendor/exp10.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of the GPU exp10 function --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/exp10.h"
+#include "src/__support/common.h"
+
+#include "common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(double, exp10, (double x)) { return internal::exp10(x); }
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/gpu/vendor/exp2.cpp b/libc/src/math/gpu/vendor/exp2.cpp
new file mode 100644
index 0000000000..ffa23d810a
--- /dev/null
+++ b/libc/src/math/gpu/vendor/exp2.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of the GPU exp2 function ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/exp2.h"
+#include "src/__support/common.h"
+
+#include "common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(double, exp2, (double x)) { return internal::exp2(x); }
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/gpu/vendor/expm1.cpp b/libc/src/math/gpu/vendor/expm1.cpp
new file mode 100644
index 0000000000..6ac5f753b9
--- /dev/null
+++ b/libc/src/math/gpu/vendor/expm1.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of the GPU expm1 function --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/expm1.h"
+#include "src/__support/common.h"
+
+#include "common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(double, expm1, (double x)) { return internal::expm1(x); }
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/gpu/vendor/log.cpp b/libc/src/math/gpu/vendor/log.cpp
new file mode 100644
index 0000000000..a97689abcc
--- /dev/null
+++ b/libc/src/math/gpu/vendor/log.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of the GPU log function ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/log.h"
+#include "src/__support/common.h"
+
+#include "common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(double, log, (double x)) { return internal::log(x); }
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/gpu/vendor/log10.cpp b/libc/src/math/gpu/vendor/log10.cpp
new file mode 100644
index 0000000000..c7a917a75c
--- /dev/null
+++ b/libc/src/math/gpu/vendor/log10.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of the GPU log10 function --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/log10.h"
+#include "src/__support/common.h"
+
+#include "common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(double, log10, (double x)) { return internal::log10(x); }
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/gpu/vendor/log10f.cpp b/libc/src/math/gpu/vendor/log10f.cpp
new file mode 100644
index 0000000000..489f5f558b
--- /dev/null
+++ b/libc/src/math/gpu/vendor/log10f.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of the GPU log10f function -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/log10f.h"
+#include "src/__support/common.h"
+
+#include "common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float, log10f, (float x)) { return internal::log10f(x); }
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/gpu/vendor/log1p.cpp b/libc/src/math/gpu/vendor/log1p.cpp
new file mode 100644
index 0000000000..720d23e2f9
--- /dev/null
+++ b/libc/src/math/gpu/vendor/log1p.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of the GPU log1p function --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/log1p.h"
+#include "src/__support/common.h"
+
+#include "common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(double, log1p, (double x)) { return internal::log1p(x); }
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/gpu/vendor/log1pf.cpp b/libc/src/math/gpu/vendor/log1pf.cpp
new file mode 100644
index 0000000000..96ad48b529
--- /dev/null
+++ b/libc/src/math/gpu/vendor/log1pf.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of the GPU log1pf function -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/log1pf.h"
+#include "src/__support/common.h"
+
+#include "common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float, log1pf, (float x)) { return internal::log1pf(x); }
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/gpu/vendor/log2.cpp b/libc/src/math/gpu/vendor/log2.cpp
new file mode 100644
index 0000000000..9fc8a81e7e
--- /dev/null
+++ b/libc/src/math/gpu/vendor/log2.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of the GPU log2 function ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/log2.h"
+#include "src/__support/common.h"
+
+#include "common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(double, log2, (double x)) { return internal::log2(x); }
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/gpu/vendor/log2f.cpp b/libc/src/math/gpu/vendor/log2f.cpp
new file mode 100644
index 0000000000..62df41b69b
--- /dev/null
+++ b/libc/src/math/gpu/vendor/log2f.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of the GPU log2f function --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/log2f.h"
+#include "src/__support/common.h"
+
+#include "common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float, log2f, (float x)) { return internal::log2f(x); }
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/gpu/vendor/logb.cpp b/libc/src/math/gpu/vendor/logb.cpp
new file mode 100644
index 0000000000..5dea57d41b
--- /dev/null
+++ b/libc/src/math/gpu/vendor/logb.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of the GPU logb function ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/logb.h"
+#include "src/__support/common.h"
+
+#include "common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(double, logb, (double x)) { return internal::logb(x); }
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/gpu/vendor/logbf.cpp b/libc/src/math/gpu/vendor/logbf.cpp
new file mode 100644
index 0000000000..1a59df3e09
--- /dev/null
+++ b/libc/src/math/gpu/vendor/logbf.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of the GPU logbf function --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/logbf.h"
+#include "src/__support/common.h"
+
+#include "common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float, logbf, (float x)) { return internal::logbf(x); }
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/gpu/vendor/logf.cpp b/libc/src/math/gpu/vendor/logf.cpp
new file mode 100644
index 0000000000..527b028e10
--- /dev/null
+++ b/libc/src/math/gpu/vendor/logf.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of the GPU logf function ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/logf.h"
+#include "src/__support/common.h"
+
+#include "common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float, logf, (float x)) { return internal::logf(x); }
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/gpu/vendor/lrint.cpp b/libc/src/math/gpu/vendor/lrint.cpp
new file mode 100644
index 0000000000..a08996b755
--- /dev/null
+++ b/libc/src/math/gpu/vendor/lrint.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of the lrint function for GPU ----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/lrint.h"
+#include "src/__support/common.h"
+
+#include "common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(long, lrint, (double x)) { return internal::lrint(x); }
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/gpu/vendor/lrintf.cpp b/libc/src/math/gpu/vendor/lrintf.cpp
new file mode 100644
index 0000000000..695a9b8202
--- /dev/null
+++ b/libc/src/math/gpu/vendor/lrintf.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of the lrintf function for GPU ---------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/lrintf.h"
+#include "src/__support/common.h"
+
+#include "common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(long, lrintf, (float x)) { return internal::lrintf(x); }
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/gpu/vendor/nvptx/declarations.h b/libc/src/math/gpu/vendor/nvptx/declarations.h
index 003a9a8a5d..9cb2be67b8 100644
--- a/libc/src/math/gpu/vendor/nvptx/declarations.h
+++ b/libc/src/math/gpu/vendor/nvptx/declarations.h
@@ -12,19 +12,33 @@
 namespace LIBC_NAMESPACE {
 
 extern "C" {
+double __nv_acos(double);
 float __nv_acosf(float);
+double __nv_acosh(double);
 float __nv_acoshf(float);
+double __nv_asin(double);
 float __nv_asinf(float);
+double __nv_asinh(double);
 float __nv_asinhf(float);
+double __nv_atan(double);
 float __nv_atanf(float);
+double __nv_atan2(double, double);
+float __nv_atan2f(float, float);
+double __nv_atanh(double);
 float __nv_atanhf(float);
 double __nv_cos(double);
 float __nv_cosf(float);
 double __nv_cosh(double);
 float __nv_coshf(float);
+double __nv_erf(double);
+float __nv_erff(float);
+double __nv_exp(double);
 float __nv_expf(float);
+double __nv_exp2(double);
 float __nv_exp2f(float);
+double __nv_exp10(double);
 float __nv_exp10f(float);
+double __nv_expm1(double);
 float __nv_expm1f(float);
 double __nv_fdim(double, double);
 float __nv_fdimf(float, float);
@@ -36,8 +50,16 @@ double __nv_ldexp(double, int);
 float __nv_ldexpf(float, int);
 long long __nv_llrint(double);
 long long __nv_llrintf(float);
-long long __nv_llround(double);
-long long __nv_llroundf(float);
+long __nv_lrint(double);
+long __nv_lrintf(float);
+double __nv_log10(double);
+float __nv_log10f(float);
+double __nv_log1p(double);
+float __nv_log1pf(float);
+double __nv_log2(double);
+float __nv_log2f(float);
+double __nv_log(double);
+float __nv_logf(float);
 double __nv_nextafter(double, double);
 float __nv_nextafterf(float, float);
 double __nv_pow(double, double);
@@ -58,6 +80,8 @@ double __nv_scalbn(double, int);
 float __nv_scalbnf(float, int);
 double __nv_remquo(double, double, int *);
 float __nv_remquof(float, float, int *);
+double __nv_tgamma(double);
+float __nv_tgammaf(float);
 }
 
 } // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/gpu/vendor/nvptx/nvptx.h b/libc/src/math/gpu/vendor/nvptx/nvptx.h
index 4298d56db6..110d570a84 100644
--- a/libc/src/math/gpu/vendor/nvptx/nvptx.h
+++ b/libc/src/math/gpu/vendor/nvptx/nvptx.h
@@ -15,19 +15,33 @@
 
 namespace LIBC_NAMESPACE {
 namespace internal {
+LIBC_INLINE double acos(double x) { return __nv_acos(x); }
 LIBC_INLINE float acosf(float x) { return __nv_acosf(x); }
+LIBC_INLINE double acosh(double x) { return __nv_acosh(x); }
 LIBC_INLINE float acoshf(float x) { return __nv_acoshf(x); }
+LIBC_INLINE double asin(double x) { return __nv_asin(x); }
 LIBC_INLINE float asinf(float x) { return __nv_asinf(x); }
+LIBC_INLINE double asinh(double x) { return __nv_asinh(x); }
 LIBC_INLINE float asinhf(float x) { return __nv_asinhf(x); }
+LIBC_INLINE double atan2(double x, double y) { return __nv_atan2(x, y); }
+LIBC_INLINE float atan2f(float x, float y) { return __nv_atan2f(x, y); }
+LIBC_INLINE double atan(double x) { return __nv_atan(x); }
 LIBC_INLINE float atanf(float x) { return __nv_atanf(x); }
+LIBC_INLINE double atanh(double x) { return __nv_atanh(x); }
 LIBC_INLINE float atanhf(float x) { return __nv_atanhf(x); }
 LIBC_INLINE double cos(double x) { return __nv_cos(x); }
 LIBC_INLINE float cosf(float x) { return __nv_cosf(x); }
 LIBC_INLINE double cosh(double x) { return __nv_cosh(x); }
 LIBC_INLINE float coshf(float x) { return __nv_coshf(x); }
+LIBC_INLINE double erf(double x) { return __nv_erf(x); }
+LIBC_INLINE float erff(float x) { return __nv_erff(x); }
+LIBC_INLINE double exp(double x) { return __nv_exp(x); }
 LIBC_INLINE float expf(float x) { return __nv_expf(x); }
+LIBC_INLINE double exp2(double x) { return __nv_exp2(x); }
 LIBC_INLINE float exp2f(float x) { return __nv_exp2f(x); }
+LIBC_INLINE double exp10(double x) { return __nv_exp10(x); }
 LIBC_INLINE float exp10f(float x) { return __nv_exp10f(x); }
+LIBC_INLINE double expm1(double x) { return __nv_expm1(x); }
 LIBC_INLINE float expm1f(float x) { return __nv_expm1f(x); }
 LIBC_INLINE double fdim(double x, double y) { return __nv_fdim(x, y); }
 LIBC_INLINE float fdimf(float x, float y) { return __nv_fdimf(x, y); }
@@ -39,8 +53,16 @@ LIBC_INLINE double ldexp(double x, int i) { return __nv_ldexp(x, i); }
 LIBC_INLINE float ldexpf(float x, int i) { return __nv_ldexpf(x, i); }
 LIBC_INLINE long long llrint(double x) { return __nv_llrint(x); }
 LIBC_INLINE long long llrintf(float x) { return __nv_llrintf(x); }
-LIBC_INLINE long long llround(double x) { return __nv_llround(x); }
-LIBC_INLINE long long llroundf(float x) { return __nv_llroundf(x); }
+LIBC_INLINE double log10(double x) { return __nv_log10(x); }
+LIBC_INLINE float log10f(float x) { return __nv_log10f(x); }
+LIBC_INLINE double log1p(double x) { return __nv_log1p(x); }
+LIBC_INLINE float log1pf(float x) { return __nv_log1pf(x); }
+LIBC_INLINE double log2(double x) { return __nv_log2(x); }
+LIBC_INLINE float log2f(float x) { return __nv_log2f(x); }
+LIBC_INLINE double log(double x) { return __nv_log(x); }
+LIBC_INLINE float logf(float x) { return __nv_logf(x); }
+LIBC_INLINE long lrint(double x) { return __nv_lrint(x); }
+LIBC_INLINE long lrintf(float x) { return __nv_lrintf(x); }
 LIBC_INLINE double nextafter(double x, double y) {
   return __nv_nextafter(x, y);
 }
@@ -71,6 +93,8 @@ LIBC_INLINE double remquo(double x, double y, int *i) {
 LIBC_INLINE float remquof(float x, float y, int *i) {
   return __nv_remquof(x, y, i);
 }
+LIBC_INLINE double tgamma(double x) { return __nv_tgamma(x); }
+LIBC_INLINE float tgammaf(float x) { return __nv_tgammaf(x); }
 
 } // namespace internal
 } // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/gpu/vendor/tgamma.cpp b/libc/src/math/gpu/vendor/tgamma.cpp
new file mode 100644
index 0000000000..e86116a2b0
--- /dev/null
+++ b/libc/src/math/gpu/vendor/tgamma.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of the GPU tgamma function -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/tgamma.h"
+#include "src/__support/common.h"
+
+#include "common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(double, tgamma, (double x)) { return internal::tgamma(x); }
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/gpu/vendor/tgammaf.cpp b/libc/src/math/gpu/vendor/tgammaf.cpp
new file mode 100644
index 0000000000..552919bae4
--- /dev/null
+++ b/libc/src/math/gpu/vendor/tgammaf.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of the GPU tgammaf function ------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/tgammaf.h"
+#include "src/__support/common.h"
+
+#include "common.h"
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(float, tgammaf, (float x)) { return internal::tgammaf(x); }
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/math/sincos.h b/libc/src/math/sincos.h
new file mode 100644
index 0000000000..6235a7c5c0
--- /dev/null
+++ b/libc/src/math/sincos.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for sincos ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_SINCOS_H
+#define LLVM_LIBC_SRC_MATH_SINCOS_H
+
+namespace LIBC_NAMESPACE {
+
+void sincos(double x, double *sinx, double *cosx);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_SINCOS_H
diff --git a/libc/src/math/tgamma.h b/libc/src/math/tgamma.h
new file mode 100644
index 0000000000..24590ed1a4
--- /dev/null
+++ b/libc/src/math/tgamma.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for tgamma ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_TGAMMA_H
+#define LLVM_LIBC_SRC_MATH_TGAMMA_H
+
+namespace LIBC_NAMESPACE {
+
+double tgamma(double x);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_TGAMMA_H
diff --git a/libc/src/math/tgammaf.h b/libc/src/math/tgammaf.h
new file mode 100644
index 0000000000..d28e4c3251
--- /dev/null
+++ b/libc/src/math/tgammaf.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for tgammaf -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_TGAMMAF_H
+#define LLVM_LIBC_SRC_MATH_TGAMMAF_H
+
+namespace LIBC_NAMESPACE {
+
+float tgammaf(float x);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_MATH_TGAMMAF_H
diff --git a/libc/src/stdio/CMakeLists.txt b/libc/src/stdio/CMakeLists.txt
index 169bc592de..380474ce27 100644
--- a/libc/src/stdio/CMakeLists.txt
+++ b/libc/src/stdio/CMakeLists.txt
@@ -54,18 +54,6 @@ add_entrypoint_object(
     libc.src.__support.File.platform_file
 )
 
-add_entrypoint_object(
-  ungetc
-  SRCS
-    ungetc.cpp
-  HDRS
-    ungetc.h
-  DEPENDS
-    libc.include.stdio
-    libc.src.__support.File.file
-    libc.src.__support.File.platform_file
-)
-
 add_entrypoint_object(
   fopencookie
   SRCS
@@ -286,6 +274,7 @@ add_stdio_entrypoint_object(getc_unlocked)
 add_stdio_entrypoint_object(getchar)
 add_stdio_entrypoint_object(getchar_unlocked)
 add_stdio_entrypoint_object(fgets)
+add_stdio_entrypoint_object(ungetc)
 add_stdio_entrypoint_object(stdin)
 add_stdio_entrypoint_object(stdout)
 add_stdio_entrypoint_object(stderr)
diff --git a/libc/src/stdio/generic/CMakeLists.txt b/libc/src/stdio/generic/CMakeLists.txt
index 282d056bba..2ecef879eb 100644
--- a/libc/src/stdio/generic/CMakeLists.txt
+++ b/libc/src/stdio/generic/CMakeLists.txt
@@ -342,6 +342,18 @@ add_entrypoint_object(
     libc.src.__support.File.platform_file
 )
 
+add_entrypoint_object(
+  ungetc
+  SRCS
+    ungetc.cpp
+  HDRS
+    ../ungetc.h
+  DEPENDS
+    libc.include.stdio
+    libc.src.__support.File.file
+    libc.src.__support.File.platform_file
+)
+
 add_entrypoint_object(
   stdin
   SRCS
diff --git a/libc/src/stdio/ungetc.cpp b/libc/src/stdio/generic/ungetc.cpp
similarity index 100%
rename from libc/src/stdio/ungetc.cpp
rename to libc/src/stdio/generic/ungetc.cpp
diff --git a/libc/src/stdio/gpu/CMakeLists.txt b/libc/src/stdio/gpu/CMakeLists.txt
index 047b68931b..1b1e2a903c 100644
--- a/libc/src/stdio/gpu/CMakeLists.txt
+++ b/libc/src/stdio/gpu/CMakeLists.txt
@@ -251,6 +251,17 @@ add_entrypoint_object(
     .ferror
 )
 
+add_entrypoint_object(
+  ungetc
+  SRCS
+    ungetc.cpp
+  HDRS
+    ../ungetc.h
+  DEPENDS
+    libc.include.stdio
+    .gpu_file
+)
+
 add_entrypoint_object(
   stdin
   SRCS
diff --git a/libc/src/stdio/gpu/fgets.cpp b/libc/src/stdio/gpu/fgets.cpp
index 4cabd15001..5ea4bdcdc9 100644
--- a/libc/src/stdio/gpu/fgets.cpp
+++ b/libc/src/stdio/gpu/fgets.cpp
@@ -22,24 +22,20 @@ LLVM_LIBC_FUNCTION(char *, fgets,
   if (count < 1)
     return nullptr;
 
-  // This implementation is very slow as it makes multiple RPC calls.
-  unsigned char c = '\0';
-  int i = 0;
-  for (; i < count - 1 && c != '\n'; ++i) {
-    auto r = file::read(stream, &c, 1);
-    if (r != 1)
-      break;
-
-    str[i] = c;
-  }
-
-  bool has_error = LIBC_NAMESPACE::ferror(stream);
-  bool has_eof = LIBC_NAMESPACE::feof(stream);
-
-  if (has_error || (i == 0 && has_eof))
+  uint64_t recv_size;
+  void *buf = nullptr;
+  rpc::Client::Port port = rpc::client.open<RPC_READ_FGETS>();
+  port.send([=](rpc::Buffer *buffer) {
+    buffer->data[0] = count;
+    buffer->data[1] = file::from_stream(stream);
+  });
+  port.recv_n(&buf, &recv_size,
+              [&](uint64_t) { return reinterpret_cast<void *>(str); });
+  port.close();
+
+  if (recv_size == 0)
     return nullptr;
 
-  str[i] = '\0';
   return str;
 }
 
diff --git a/libc/src/stdio/gpu/ungetc.cpp b/libc/src/stdio/gpu/ungetc.cpp
new file mode 100644
index 0000000000..373164a0c5
--- /dev/null
+++ b/libc/src/stdio/gpu/ungetc.cpp
@@ -0,0 +1,29 @@
+//===-- Implementation of ungetc ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/stdio/ungetc.h"
+#include "file.h"
+
+#include <stdio.h>
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(int, ungetc, (int c, ::FILE *stream)) {
+  int ret;
+  rpc::Client::Port port = rpc::client.open<RPC_UNGETC>();
+  port.send_and_recv(
+      [=](rpc::Buffer *buffer) {
+        buffer->data[0] = c;
+        buffer->data[1] = file::from_stream(stream);
+      },
+      [&](rpc::Buffer *buffer) { ret = static_cast<int>(buffer->data[0]); });
+  port.close();
+  return ret;
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/stdio/printf_core/core_structs.h b/libc/src/stdio/printf_core/core_structs.h
index e96bc4dc0f..88a13703fd 100644
--- a/libc/src/stdio/printf_core/core_structs.h
+++ b/libc/src/stdio/printf_core/core_structs.h
@@ -53,7 +53,7 @@ struct FormatSection {
 
   // This operator is only used for testing and should be automatically
   // optimized out for release builds.
-  bool operator==(const FormatSection &other) {
+  bool operator==(const FormatSection &other) const {
     if (has_conv != other.has_conv)
       return false;
 
diff --git a/libc/src/stdio/printf_core/float_hex_converter.h b/libc/src/stdio/printf_core/float_hex_converter.h
index e264af9844..6a980a74d4 100644
--- a/libc/src/stdio/printf_core/float_hex_converter.h
+++ b/libc/src/stdio/printf_core/float_hex_converter.h
@@ -75,8 +75,9 @@ LIBC_INLINE int convert_float_hex_exp(Writer *writer,
 
   // This is to handle situations where the mantissa isn't an even number of hex
   // digits. This is primarily relevant for x86 80 bit long doubles, which have
-  // 63 bit mantissas.
-  if (mantissa_width % BITS_IN_HEX_DIGIT != 0) {
+  // 63 bit mantissas. In the case where the mantissa is 0, however, the
+  // exponent should stay as 0.
+  if (mantissa_width % BITS_IN_HEX_DIGIT != 0 && mantissa > 0) {
     exponent -= mantissa_width % BITS_IN_HEX_DIGIT;
   }
 
diff --git a/libc/src/stdlib/rand.cpp b/libc/src/stdlib/rand.cpp
index c5737a8b84..ad543b4048 100644
--- a/libc/src/stdlib/rand.cpp
+++ b/libc/src/stdlib/rand.cpp
@@ -15,10 +15,12 @@ namespace LIBC_NAMESPACE {
 // An implementation of the xorshift64star pseudo random number generator. This
 // is a good general purpose generator for most non-cryptographics applications.
 LLVM_LIBC_FUNCTION(int, rand, (void)) {
-  rand_next ^= rand_next >> 12;
-  rand_next ^= rand_next << 25;
-  rand_next ^= rand_next >> 27;
-  return static_cast<int>((rand_next * 0x2545F4914F6CDD1Dul) >> 32) & RAND_MAX;
+  unsigned long x = rand_next;
+  x ^= x >> 12;
+  x ^= x << 25;
+  x ^= x >> 27;
+  rand_next = x;
+  return static_cast<int>((x * 0x2545F4914F6CDD1Dul) >> 32) & RAND_MAX;
 }
 
 } // namespace LIBC_NAMESPACE
diff --git a/libc/src/stdlib/rand_util.cpp b/libc/src/stdlib/rand_util.cpp
index fc2cd304e7..1f3dbce9c7 100644
--- a/libc/src/stdlib/rand_util.cpp
+++ b/libc/src/stdlib/rand_util.cpp
@@ -11,8 +11,14 @@
 
 namespace LIBC_NAMESPACE {
 
+#ifdef LIBC_TARGET_ARCH_IS_GPU
+// FIXME: Local GPU memory cannot be initialized so we cannot currently provide
+// a standard compliant default value.
+ThreadLocal<unsigned long> rand_next;
+#else
 // C standard 7.10p2: If 'rand' is called before 'srand' it is to proceed as if
 // the 'srand' function was called with a value of '1'.
 LIBC_THREAD_LOCAL unsigned long rand_next = 1;
+#endif
 
 } // namespace LIBC_NAMESPACE
diff --git a/libc/src/stdlib/rand_util.h b/libc/src/stdlib/rand_util.h
index 55c36a45bb..cadd6b5cdc 100644
--- a/libc/src/stdlib/rand_util.h
+++ b/libc/src/stdlib/rand_util.h
@@ -9,11 +9,33 @@
 #ifndef LLVM_LIBC_SRC_STDLIB_RAND_UTIL_H
 #define LLVM_LIBC_SRC_STDLIB_RAND_UTIL_H
 
+#include "src/__support/GPU/utils.h"
 #include "src/__support/macros/attributes.h"
 
 namespace LIBC_NAMESPACE {
 
+#ifdef LIBC_TARGET_ARCH_IS_GPU
+// Implement thread local storage on the GPU using local memory. Each thread
+// gets its slot in the local memory array and is private to the group.
+// TODO: We need to implement the 'thread_local' keyword on the GPU. This is an
+// inefficient and incomplete stand-in until that is done.
+template <typename T> class ThreadLocal {
+private:
+  static constexpr long MAX_THREADS = 1024;
+  [[clang::loader_uninitialized]] static inline gpu::Local<T>
+      storage[MAX_THREADS];
+
+public:
+  LIBC_INLINE operator T() const { return storage[gpu::get_thread_id()]; }
+  LIBC_INLINE void operator=(const T &value) {
+    storage[gpu::get_thread_id()] = value;
+  }
+};
+
+extern ThreadLocal<unsigned long> rand_next;
+#else
 extern LIBC_THREAD_LOCAL unsigned long rand_next;
+#endif
 
 } // namespace LIBC_NAMESPACE
 
diff --git a/libc/src/time/gpu/time_utils.cpp b/libc/src/time/gpu/time_utils.cpp
index 935a5394e5..300a72b102 100644
--- a/libc/src/time/gpu/time_utils.cpp
+++ b/libc/src/time/gpu/time_utils.cpp
@@ -16,7 +16,7 @@ namespace LIBC_NAMESPACE {
 // TODO: Once we have another use-case for this we should put it in a common
 // device environment struct.
 extern "C" [[gnu::visibility("protected")]] uint64_t
-    [[clang::address_space(4)]] LIBC_NAMESPACE_clock_freq = clock_freq;
+    [[clang::address_space(4)]] __llvm_libc_clock_freq = clock_freq;
 #endif
 
 } // namespace LIBC_NAMESPACE
diff --git a/libc/src/time/gpu/time_utils.h b/libc/src/time/gpu/time_utils.h
index 075fcb6176..53548181e1 100644
--- a/libc/src/time/gpu/time_utils.h
+++ b/libc/src/time/gpu/time_utils.h
@@ -39,8 +39,8 @@ constexpr uint64_t clock_freq = 0;
 // We provide an externally visible symbol such that the runtime can set this to
 // the correct value. If it is not set we try to default to the known values.
 extern "C" [[gnu::visibility("protected")]] uint64_t
-    [[clang::address_space(4)]] LIBC_NAMESPACE_clock_freq;
-#define GPU_CLOCKS_PER_SEC static_cast<clock_t>(LIBC_NAMESPACE_clock_freq)
+    [[clang::address_space(4)]] __llvm_libc_clock_freq;
+#define GPU_CLOCKS_PER_SEC static_cast<clock_t>(__llvm_libc_clock_freq)
 
 #elif defined(LIBC_TARGET_ARCH_IS_NVPTX)
 // NPVTX uses a single 1 GHz fixed frequency clock for all target architectures.
diff --git a/libc/src/unistd/linux/syscall.cpp b/libc/src/unistd/linux/syscall.cpp
index e1b13c9c14..e0070fe6d8 100644
--- a/libc/src/unistd/linux/syscall.cpp
+++ b/libc/src/unistd/linux/syscall.cpp
@@ -16,7 +16,7 @@
 
 namespace LIBC_NAMESPACE {
 
-LLVM_LIBC_FUNCTION(long, LIBC_NAMESPACE_syscall,
+LLVM_LIBC_FUNCTION(long, __llvm_libc_syscall,
                    (long number, long arg1, long arg2, long arg3, long arg4,
                     long arg5, long arg6)) {
   long ret = LIBC_NAMESPACE::syscall_impl<long>(number, arg1, arg2, arg3, arg4,
diff --git a/libc/src/unistd/syscall.h b/libc/src/unistd/syscall.h
index 255459cadb..a41d9778e0 100644
--- a/libc/src/unistd/syscall.h
+++ b/libc/src/unistd/syscall.h
@@ -14,8 +14,8 @@
 
 namespace LIBC_NAMESPACE {
 
-long LIBC_NAMESPACE_syscall(long number, long arg1, long arg2, long arg3,
-                            long arg4, long arg5, long arg6);
+long __llvm_libc_syscall(long number, long arg1, long arg2, long arg3,
+                         long arg4, long arg5, long arg6);
 
 } // namespace LIBC_NAMESPACE
 
diff --git a/libc/startup/gpu/amdgpu/start.cpp b/libc/startup/gpu/amdgpu/start.cpp
index 9e642dbe3f..89b0be208d 100644
--- a/libc/startup/gpu/amdgpu/start.cpp
+++ b/libc/startup/gpu/amdgpu/start.cpp
@@ -19,7 +19,7 @@ namespace LIBC_NAMESPACE {
 // real time. However, the frequency of this clock varies between cards and can
 // only be obtained via the driver. The loader will set this so we can use it.
 extern "C" [[gnu::visibility("protected")]] uint64_t
-    [[clang::address_space(4)]] LIBC_NAMESPACE_clock_freq = 0;
+    [[clang::address_space(4)]] __llvm_libc_clock_freq = 0;
 
 extern "C" uintptr_t __init_array_start[];
 extern "C" uintptr_t __init_array_end[];
diff --git a/libc/test/UnitTest/FPMatcher.h b/libc/test/UnitTest/FPMatcher.h
index fb60916a04..14c8a85ba7 100644
--- a/libc/test/UnitTest/FPMatcher.h
+++ b/libc/test/UnitTest/FPMatcher.h
@@ -59,6 +59,16 @@ template <TestCond C, typename T> FPMatcher<T, C> getMatcher(T expectedValue) {
   return FPMatcher<T, C>(expectedValue);
 }
 
+template <typename T> struct FPTest : public Test {
+  using FPBits = LIBC_NAMESPACE::fputil::FPBits<T>;
+  using UIntType = typename FPBits::UIntType;
+  static constexpr T zero = T(FPBits::zero());
+  static constexpr T neg_zero = T(FPBits::neg_zero());
+  static constexpr T aNaN = T(FPBits::build_quiet_nan(1));
+  static constexpr T inf = T(FPBits::inf());
+  static constexpr T neg_inf = T(FPBits::neg_inf());
+};
+
 } // namespace testing
 } // namespace LIBC_NAMESPACE
 
diff --git a/libc/test/UnitTest/LibcTest.cpp b/libc/test/UnitTest/LibcTest.cpp
index b01f40f052..5c27e2e69b 100644
--- a/libc/test/UnitTest/LibcTest.cpp
+++ b/libc/test/UnitTest/LibcTest.cpp
@@ -23,8 +23,8 @@ static long clock() { return LIBC_NAMESPACE::gpu::fixed_frequency_clock(); }
 #else
 // The AMDGPU loader needs to initialize this at runtime by querying the driver.
 extern "C" [[gnu::visibility("protected")]] uint64_t
-    [[clang::address_space(4)]] LIBC_NAMESPACE_clock_freq;
-#define CLOCKS_PER_SEC LIBC_NAMESPACE_clock_freq
+    [[clang::address_space(4)]] __llvm_libc_clock_freq;
+#define CLOCKS_PER_SEC __llvm_libc_clock_freq
 #endif
 #else
 static long clock() { return 0; }
diff --git a/libc/test/src/math/acosf_test.cpp b/libc/test/src/math/acosf_test.cpp
index cb1a07448b..409cf2bc89 100644
--- a/libc/test/src/math/acosf_test.cpp
+++ b/libc/test/src/math/acosf_test.cpp
@@ -17,13 +17,11 @@
 #include <errno.h>
 #include <stdint.h>
 
-using FPBits = LIBC_NAMESPACE::fputil::FPBits<float>;
-
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
-DECLARE_SPECIAL_CONSTANTS(float)
+using LlvmLibcAcosfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
-TEST(LlvmLibcAcosfTest, SpecialNumbers) {
+TEST_F(LlvmLibcAcosfTest, SpecialNumbers) {
   libc_errno = 0;
 
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::acosf(aNaN));
@@ -45,7 +43,7 @@ TEST(LlvmLibcAcosfTest, SpecialNumbers) {
   EXPECT_MATH_ERRNO(EDOM);
 }
 
-TEST(LlvmLibcAcosfTest, InFloatRange) {
+TEST_F(LlvmLibcAcosfTest, InFloatRange) {
   constexpr uint32_t COUNT = 100'000;
   constexpr uint32_t STEP = UINT32_MAX / COUNT;
   for (uint32_t i = 0, v = 0; i <= COUNT; ++i, v += STEP) {
@@ -57,7 +55,7 @@ TEST(LlvmLibcAcosfTest, InFloatRange) {
   }
 }
 
-TEST(LlvmLibcAcosfTest, SpecificBitPatterns) {
+TEST_F(LlvmLibcAcosfTest, SpecificBitPatterns) {
   constexpr int N = 13;
   constexpr uint32_t INPUTS[N] = {
       0x3f000000, // x = 0.5f
diff --git a/libc/test/src/math/acoshf_test.cpp b/libc/test/src/math/acoshf_test.cpp
index 846f5033fb..fe8d76918d 100644
--- a/libc/test/src/math/acoshf_test.cpp
+++ b/libc/test/src/math/acoshf_test.cpp
@@ -17,13 +17,11 @@
 #include <errno.h>
 #include <stdint.h>
 
-using FPBits_t = LIBC_NAMESPACE::fputil::FPBits<float>;
+using LlvmLibcAcoshfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
-DECLARE_SPECIAL_CONSTANTS(float)
-
-TEST(LlvmLibcAcoshfTest, SpecialNumbers) {
+TEST_F(LlvmLibcAcoshfTest, SpecialNumbers) {
   libc_errno = 0;
 
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::acoshf(aNaN));
@@ -42,11 +40,11 @@ TEST(LlvmLibcAcoshfTest, SpecialNumbers) {
   EXPECT_MATH_ERRNO(EDOM);
 }
 
-TEST(LlvmLibcAcoshfTest, InFloatRange) {
+TEST_F(LlvmLibcAcoshfTest, InFloatRange) {
   constexpr uint32_t COUNT = 100'000;
   constexpr uint32_t STEP = UINT32_MAX / COUNT;
   for (uint32_t i = 0, v = 0; i <= COUNT; ++i, v += STEP) {
-    float x = float(FPBits_t(v));
+    float x = float(FPBits(v));
     if (isnan(x) || isinf(x))
       continue;
     ASSERT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Acosh, x,
@@ -54,7 +52,7 @@ TEST(LlvmLibcAcoshfTest, InFloatRange) {
   }
 }
 
-TEST(LlvmLibcAcoshfTest, SpecificBitPatterns) {
+TEST_F(LlvmLibcAcoshfTest, SpecificBitPatterns) {
   constexpr int N = 12;
   constexpr uint32_t INPUTS[N] = {
       0x3f800000, // x = 1.0f
@@ -72,7 +70,7 @@ TEST(LlvmLibcAcoshfTest, SpecificBitPatterns) {
   };
 
   for (int i = 0; i < N; ++i) {
-    float x = float(FPBits_t(INPUTS[i]));
+    float x = float(FPBits(INPUTS[i]));
     EXPECT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Acosh, x,
                                    LIBC_NAMESPACE::acoshf(x), 0.5);
   }
diff --git a/libc/test/src/math/asinf_test.cpp b/libc/test/src/math/asinf_test.cpp
index 49dcd38e82..db9dd0d784 100644
--- a/libc/test/src/math/asinf_test.cpp
+++ b/libc/test/src/math/asinf_test.cpp
@@ -18,13 +18,11 @@
 #include <errno.h>
 #include <stdint.h>
 
-using FPBits = LIBC_NAMESPACE::fputil::FPBits<float>;
+using LlvmLibcAsinfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
-DECLARE_SPECIAL_CONSTANTS(float)
-
-TEST(LlvmLibcAsinfTest, SpecialNumbers) {
+TEST_F(LlvmLibcAsinfTest, SpecialNumbers) {
   libc_errno = 0;
 
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::asinf(aNaN));
@@ -43,7 +41,7 @@ TEST(LlvmLibcAsinfTest, SpecialNumbers) {
   EXPECT_MATH_ERRNO(EDOM);
 }
 
-TEST(LlvmLibcAsinfTest, InFloatRange) {
+TEST_F(LlvmLibcAsinfTest, InFloatRange) {
   constexpr uint32_t COUNT = 100'000;
   constexpr uint32_t STEP = UINT32_MAX / COUNT;
   for (uint32_t i = 0, v = 0; i <= COUNT; ++i, v += STEP) {
@@ -55,7 +53,7 @@ TEST(LlvmLibcAsinfTest, InFloatRange) {
   }
 }
 
-TEST(LlvmLibcAsinfTest, SpecificBitPatterns) {
+TEST_F(LlvmLibcAsinfTest, SpecificBitPatterns) {
   constexpr int N = 11;
   constexpr uint32_t INPUTS[N] = {
       0x3f000000, // x = 0.5f
diff --git a/libc/test/src/math/asinhf_test.cpp b/libc/test/src/math/asinhf_test.cpp
index 0bbd5db031..2afb5b3a9f 100644
--- a/libc/test/src/math/asinhf_test.cpp
+++ b/libc/test/src/math/asinhf_test.cpp
@@ -17,13 +17,11 @@
 #include <errno.h>
 #include <stdint.h>
 
-using FPBits_t = LIBC_NAMESPACE::fputil::FPBits<float>;
+using LlvmLibcAsinhfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
-DECLARE_SPECIAL_CONSTANTS(float)
-
-TEST(LlvmLibcAsinhfTest, SpecialNumbers) {
+TEST_F(LlvmLibcAsinhfTest, SpecialNumbers) {
   libc_errno = 0;
 
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::asinhf(aNaN));
@@ -42,11 +40,11 @@ TEST(LlvmLibcAsinhfTest, SpecialNumbers) {
   EXPECT_MATH_ERRNO(0);
 }
 
-TEST(LlvmLibcAsinhfTest, InFloatRange) {
+TEST_F(LlvmLibcAsinhfTest, InFloatRange) {
   constexpr uint32_t COUNT = 1'001;
   constexpr uint32_t STEP = UINT32_MAX / COUNT;
   for (uint32_t i = 0, v = 0; i <= COUNT; ++i, v += STEP) {
-    float x = float(FPBits_t(v));
+    float x = float(FPBits(v));
     if (isnan(x) || isinf(x))
       continue;
     ASSERT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Asinh, x,
@@ -56,7 +54,7 @@ TEST(LlvmLibcAsinhfTest, InFloatRange) {
   }
 }
 
-TEST(LlvmLibcAsinhfTest, SpecificBitPatterns) {
+TEST_F(LlvmLibcAsinhfTest, SpecificBitPatterns) {
   constexpr int N = 11;
   constexpr uint32_t INPUTS[N] = {
       0x45abaf26, // |x| = 0x1.575e4cp12f
@@ -73,7 +71,7 @@ TEST(LlvmLibcAsinhfTest, SpecificBitPatterns) {
   };
 
   for (int i = 0; i < N; ++i) {
-    float x = float(FPBits_t(INPUTS[i]));
+    float x = float(FPBits(INPUTS[i]));
     EXPECT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Asinh, x,
                                    LIBC_NAMESPACE::asinhf(x), 0.5);
     EXPECT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Asinh, -x,
diff --git a/libc/test/src/math/atanf_test.cpp b/libc/test/src/math/atanf_test.cpp
index c7eab7b668..cd4f7b89ee 100644
--- a/libc/test/src/math/atanf_test.cpp
+++ b/libc/test/src/math/atanf_test.cpp
@@ -19,13 +19,11 @@
 
 #include <initializer_list>
 
-using FPBits = LIBC_NAMESPACE::fputil::FPBits<float>;
+using LlvmLibcAtanfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
-DECLARE_SPECIAL_CONSTANTS(float)
-
-TEST(LlvmLibcAtanfTest, SpecialNumbers) {
+TEST_F(LlvmLibcAtanfTest, SpecialNumbers) {
   libc_errno = 0;
   LIBC_NAMESPACE::fputil::clear_except(FE_ALL_EXCEPT);
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::atanf(aNaN));
@@ -43,7 +41,7 @@ TEST(LlvmLibcAtanfTest, SpecialNumbers) {
   EXPECT_MATH_ERRNO(0);
 }
 
-TEST(LlvmLibcAtanfTest, InFloatRange) {
+TEST_F(LlvmLibcAtanfTest, InFloatRange) {
   constexpr uint32_t COUNT = 100'000;
   const uint32_t STEP = FPBits(inf).uintval() / COUNT;
   for (uint32_t i = 0, v = 0; i <= COUNT; ++i, v += STEP) {
@@ -56,7 +54,7 @@ TEST(LlvmLibcAtanfTest, InFloatRange) {
 }
 
 // For small values, tanh(x) is x.
-TEST(LlvmLibcAtanfTest, SpecialValues) {
+TEST_F(LlvmLibcAtanfTest, SpecialValues) {
   for (uint32_t v : {0x3d8d6b23U, 0x3feefcfbU, 0xbd8d6b23U, 0xbfeefcfbU,
                      0x7F800000U, 0xFF800000U}) {
     float x = float(FPBits(v));
diff --git a/libc/test/src/math/atanhf_test.cpp b/libc/test/src/math/atanhf_test.cpp
index c1a8d19971..f62830ac9d 100644
--- a/libc/test/src/math/atanhf_test.cpp
+++ b/libc/test/src/math/atanhf_test.cpp
@@ -17,13 +17,11 @@
 #include <errno.h>
 #include <stdint.h>
 
-using FPBits = LIBC_NAMESPACE::fputil::FPBits<float>;
+using LlvmLibcAtanhfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
-DECLARE_SPECIAL_CONSTANTS(float)
-
-TEST(LlvmLibcAtanhfTest, SpecialNumbers) {
+TEST_F(LlvmLibcAtanhfTest, SpecialNumbers) {
   libc_errno = 0;
   LIBC_NAMESPACE::fputil::clear_except(FE_ALL_EXCEPT);
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::atanhf(aNaN));
@@ -85,7 +83,7 @@ TEST(LlvmLibcAtanhfTest, SpecialNumbers) {
   EXPECT_MATH_ERRNO(EDOM);
 }
 
-TEST(LlvmLibcAtanhfTest, InFloatRange) {
+TEST_F(LlvmLibcAtanhfTest, InFloatRange) {
   constexpr uint32_t COUNT = 100'000;
   const uint32_t STEP = FPBits(1.0f).uintval() / COUNT;
   for (uint32_t i = 0, v = 0; i <= COUNT; ++i, v += STEP) {
@@ -98,7 +96,7 @@ TEST(LlvmLibcAtanhfTest, InFloatRange) {
 }
 
 // For small values, atanh(x) is x.
-TEST(LlvmLibcAtanhfTest, SmallValues) {
+TEST_F(LlvmLibcAtanhfTest, SmallValues) {
   float x = float(FPBits(uint32_t(0x17800000)));
   float result = LIBC_NAMESPACE::atanhf(x);
   EXPECT_MPFR_MATCH(mpfr::Operation::Atanh, x, result, 0.5);
diff --git a/libc/test/src/math/cos_test.cpp b/libc/test/src/math/cos_test.cpp
index 6d8aeaf223..738d66034f 100644
--- a/libc/test/src/math/cos_test.cpp
+++ b/libc/test/src/math/cos_test.cpp
@@ -13,11 +13,11 @@
 
 #include <math.h>
 
-namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
+using LlvmLibcCosTest = LIBC_NAMESPACE::testing::FPTest<double>;
 
-DECLARE_SPECIAL_CONSTANTS(double)
+namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
-TEST(LlvmLibccosTest, Range) {
+TEST_F(LlvmLibcCosTest, Range) {
   static constexpr double _2pi = 6.283185307179586;
   constexpr UIntType COUNT = 100'000;
   constexpr UIntType STEP = UIntType(-1) / COUNT;
diff --git a/libc/test/src/math/cosf_test.cpp b/libc/test/src/math/cosf_test.cpp
index 7b2fdf9f1b..5a16520439 100644
--- a/libc/test/src/math/cosf_test.cpp
+++ b/libc/test/src/math/cosf_test.cpp
@@ -19,13 +19,11 @@
 #include <stdint.h>
 
 using LIBC_NAMESPACE::testing::SDCOMP26094_VALUES;
-using FPBits = LIBC_NAMESPACE::fputil::FPBits<float>;
+using LlvmLibcCosfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
-DECLARE_SPECIAL_CONSTANTS(float)
-
-TEST(LlvmLibcCosfTest, SpecialNumbers) {
+TEST_F(LlvmLibcCosfTest, SpecialNumbers) {
   libc_errno = 0;
 
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::cosf(aNaN));
@@ -44,7 +42,7 @@ TEST(LlvmLibcCosfTest, SpecialNumbers) {
   EXPECT_MATH_ERRNO(EDOM);
 }
 
-TEST(LlvmLibcCosfTest, InFloatRange) {
+TEST_F(LlvmLibcCosfTest, InFloatRange) {
   constexpr uint32_t COUNT = 100'000;
   constexpr uint32_t STEP = UINT32_MAX / COUNT;
   for (uint32_t i = 0, v = 0; i <= COUNT; ++i, v += STEP) {
@@ -56,7 +54,7 @@ TEST(LlvmLibcCosfTest, InFloatRange) {
   }
 }
 
-TEST(LlvmLibcCosfTest, SpecificBitPatterns) {
+TEST_F(LlvmLibcCosfTest, SpecificBitPatterns) {
   constexpr int N = 42;
   constexpr uint32_t INPUTS[N] = {
       0x3f06'0a92U, // x = pi/6
@@ -114,7 +112,7 @@ TEST(LlvmLibcCosfTest, SpecificBitPatterns) {
 
 // SDCOMP-26094: check cosf in the cases for which the range reducer
 // returns values furthest beyond its nominal upper bound of pi/4.
-TEST(LlvmLibcCosfTest, SDCOMP_26094) {
+TEST_F(LlvmLibcCosfTest, SDCOMP_26094) {
   for (uint32_t v : SDCOMP26094_VALUES) {
     float x = float(FPBits(v));
     ASSERT_MPFR_MATCH(mpfr::Operation::Cos, x, LIBC_NAMESPACE::cosf(x), 0.5);
diff --git a/libc/test/src/math/coshf_test.cpp b/libc/test/src/math/coshf_test.cpp
index f45f35ad3e..797cfec566 100644
--- a/libc/test/src/math/coshf_test.cpp
+++ b/libc/test/src/math/coshf_test.cpp
@@ -18,13 +18,11 @@
 #include <errno.h>
 #include <stdint.h>
 
-using FPBits = LIBC_NAMESPACE::fputil::FPBits<float>;
+using LlvmLibcCoshfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
-DECLARE_SPECIAL_CONSTANTS(float)
-
-TEST(LlvmLibcCoshfTest, SpecialNumbers) {
+TEST_F(LlvmLibcCoshfTest, SpecialNumbers) {
   libc_errno = 0;
 
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::coshf(aNaN));
@@ -43,7 +41,7 @@ TEST(LlvmLibcCoshfTest, SpecialNumbers) {
   EXPECT_MATH_ERRNO(0);
 }
 
-TEST(LlvmLibcCoshfTest, Overflow) {
+TEST_F(LlvmLibcCoshfTest, Overflow) {
   libc_errno = 0;
   EXPECT_FP_EQ_WITH_EXCEPTION(
       inf, LIBC_NAMESPACE::coshf(float(FPBits(0x7f7fffffU))), FE_OVERFLOW);
@@ -58,7 +56,7 @@ TEST(LlvmLibcCoshfTest, Overflow) {
   EXPECT_MATH_ERRNO(ERANGE);
 }
 
-TEST(LlvmLibcCoshfTest, InFloatRange) {
+TEST_F(LlvmLibcCoshfTest, InFloatRange) {
   constexpr uint32_t COUNT = 100'000;
   constexpr uint32_t STEP = UINT32_MAX / COUNT;
   for (uint32_t i = 0, v = 0; i <= COUNT; ++i, v += STEP) {
@@ -69,7 +67,7 @@ TEST(LlvmLibcCoshfTest, InFloatRange) {
   }
 }
 
-TEST(LlvmLibcCoshfTest, SmallValues) {
+TEST_F(LlvmLibcCoshfTest, SmallValues) {
   float x = float(FPBits(0x17800000U));
   float result = LIBC_NAMESPACE::coshf(x);
   EXPECT_MPFR_MATCH(mpfr::Operation::Cosh, x, result, 0.5);
diff --git a/libc/test/src/math/erff_test.cpp b/libc/test/src/math/erff_test.cpp
index 3cb24778d9..933f77c0b5 100644
--- a/libc/test/src/math/erff_test.cpp
+++ b/libc/test/src/math/erff_test.cpp
@@ -16,12 +16,12 @@
 #include <errno.h>
 #include <stdint.h>
 
+using LlvmLibcErffTest = LIBC_NAMESPACE::testing::FPTest<float>;
+
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 using LIBC_NAMESPACE::testing::tlog;
 
-DECLARE_SPECIAL_CONSTANTS(float)
-
-TEST(LlvmLibcErffTest, SpecialNumbers) {
+TEST_F(LlvmLibcErffTest, SpecialNumbers) {
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::erff(aNaN));
   EXPECT_FP_EQ_ALL_ROUNDING(1.0f, LIBC_NAMESPACE::erff(inf));
   EXPECT_FP_EQ_ALL_ROUNDING(-1.0f, LIBC_NAMESPACE::erff(neg_inf));
@@ -29,7 +29,7 @@ TEST(LlvmLibcErffTest, SpecialNumbers) {
   EXPECT_FP_EQ_ALL_ROUNDING(neg_zero, LIBC_NAMESPACE::erff(neg_zero));
 }
 
-TEST(LlvmLibcErffTest, TrickyInputs) {
+TEST_F(LlvmLibcErffTest, TrickyInputs) {
   constexpr int N = 2;
   constexpr uint32_t INPUTS[N] = {
       0x3f65'9229U, // |x| = 0x1.cb2452p-1f
@@ -44,7 +44,7 @@ TEST(LlvmLibcErffTest, TrickyInputs) {
   }
 }
 
-TEST(LlvmLibcErffTest, InFloatRange) {
+TEST_F(LlvmLibcErffTest, InFloatRange) {
   constexpr uint32_t COUNT = 234561;
   constexpr uint32_t START = 0;           // 0
   constexpr uint32_t STOP = 0x4080'0000U; // 4.0f
diff --git a/libc/test/src/math/exp10_test.cpp b/libc/test/src/math/exp10_test.cpp
index 86d902eaad..ec3925846d 100644
--- a/libc/test/src/math/exp10_test.cpp
+++ b/libc/test/src/math/exp10_test.cpp
@@ -17,12 +17,12 @@
 #include <errno.h>
 #include <stdint.h>
 
+using LlvmLibcExp10Test = LIBC_NAMESPACE::testing::FPTest<double>;
+
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 using LIBC_NAMESPACE::testing::tlog;
 
-DECLARE_SPECIAL_CONSTANTS(double)
-
-TEST(LlvmLibcExp10Test, SpecialNumbers) {
+TEST_F(LlvmLibcExp10Test, SpecialNumbers) {
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::exp10(aNaN));
   EXPECT_FP_EQ(inf, LIBC_NAMESPACE::exp10(inf));
   EXPECT_FP_EQ_ALL_ROUNDING(zero, LIBC_NAMESPACE::exp10(neg_inf));
@@ -34,7 +34,7 @@ TEST(LlvmLibcExp10Test, SpecialNumbers) {
   EXPECT_FP_EQ_ALL_ROUNDING(1.0, LIBC_NAMESPACE::exp10(-0.0));
 }
 
-TEST(LlvmLibcExp10Test, TrickyInputs) {
+TEST_F(LlvmLibcExp10Test, TrickyInputs) {
   constexpr int N = 41;
   constexpr uint64_t INPUTS[N] = {
       0x40033093317082F8, 0x3FD79289C6E6A5C0,
@@ -85,7 +85,7 @@ TEST(LlvmLibcExp10Test, TrickyInputs) {
   }
 }
 
-TEST(LlvmLibcExp10Test, InDoubleRange) {
+TEST_F(LlvmLibcExp10Test, InDoubleRange) {
   constexpr uint64_t COUNT = 1'231;
   uint64_t START = LIBC_NAMESPACE::fputil::FPBits<double>(0.25).uintval();
   uint64_t STOP = LIBC_NAMESPACE::fputil::FPBits<double>(4.0).uintval();
diff --git a/libc/test/src/math/exp10f_test.cpp b/libc/test/src/math/exp10f_test.cpp
index ac8f551534..e3151dafa9 100644
--- a/libc/test/src/math/exp10f_test.cpp
+++ b/libc/test/src/math/exp10f_test.cpp
@@ -16,11 +16,11 @@
 
 #include <stdint.h>
 
-namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
+using LlvmLibcExp10fTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
-DECLARE_SPECIAL_CONSTANTS(float)
+namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
-TEST(LlvmLibcExp10fTest, SpecialNumbers) {
+TEST_F(LlvmLibcExp10fTest, SpecialNumbers) {
   libc_errno = 0;
 
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::exp10f(aNaN));
@@ -39,7 +39,7 @@ TEST(LlvmLibcExp10fTest, SpecialNumbers) {
   EXPECT_MATH_ERRNO(0);
 }
 
-TEST(LlvmLibcExp10fTest, Overflow) {
+TEST_F(LlvmLibcExp10fTest, Overflow) {
   libc_errno = 0;
   EXPECT_FP_EQ_WITH_EXCEPTION(
       inf, LIBC_NAMESPACE::exp10f(float(FPBits(0x7f7fffffU))), FE_OVERFLOW);
@@ -54,7 +54,7 @@ TEST(LlvmLibcExp10fTest, Overflow) {
   EXPECT_MATH_ERRNO(ERANGE);
 }
 
-TEST(LlvmLibcExp10fTest, Underflow) {
+TEST_F(LlvmLibcExp10fTest, Underflow) {
   libc_errno = 0;
   EXPECT_FP_EQ_WITH_EXCEPTION(
       0.0f, LIBC_NAMESPACE::exp10f(float(FPBits(0xff7fffffU))), FE_UNDERFLOW);
@@ -71,7 +71,7 @@ TEST(LlvmLibcExp10fTest, Underflow) {
   EXPECT_MATH_ERRNO(ERANGE);
 }
 
-TEST(LlvmLibcExp10fTest, TrickyInputs) {
+TEST_F(LlvmLibcExp10fTest, TrickyInputs) {
   constexpr int N = 20;
   constexpr uint32_t INPUTS[N] = {
       0x325e5bd8, // x = 0x1.bcb7bp-27f
@@ -105,7 +105,7 @@ TEST(LlvmLibcExp10fTest, TrickyInputs) {
   }
 }
 
-TEST(LlvmLibcExp10fTest, InFloatRange) {
+TEST_F(LlvmLibcExp10fTest, InFloatRange) {
   constexpr uint32_t COUNT = 100'000;
   constexpr uint32_t STEP = UINT32_MAX / COUNT;
   for (uint32_t i = 0, v = 0; i <= COUNT; ++i, v += STEP) {
diff --git a/libc/test/src/math/exp2_test.cpp b/libc/test/src/math/exp2_test.cpp
index 6a90c9ba91..539ec7b936 100644
--- a/libc/test/src/math/exp2_test.cpp
+++ b/libc/test/src/math/exp2_test.cpp
@@ -17,12 +17,12 @@
 #include <errno.h>
 #include <stdint.h>
 
+using LlvmLibcExp2Test = LIBC_NAMESPACE::testing::FPTest<double>;
+
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 using LIBC_NAMESPACE::testing::tlog;
 
-DECLARE_SPECIAL_CONSTANTS(double)
-
-TEST(LlvmLibcExp2Test, SpecialNumbers) {
+TEST_F(LlvmLibcExp2Test, SpecialNumbers) {
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::exp2(aNaN));
   EXPECT_FP_EQ(inf, LIBC_NAMESPACE::exp2(inf));
   EXPECT_FP_EQ_ALL_ROUNDING(zero, LIBC_NAMESPACE::exp2(neg_inf));
@@ -33,7 +33,7 @@ TEST(LlvmLibcExp2Test, SpecialNumbers) {
   EXPECT_FP_EQ_ALL_ROUNDING(1.0, LIBC_NAMESPACE::exp2(-0.0));
 }
 
-TEST(LlvmLibcExp2Test, TrickyInputs) {
+TEST_F(LlvmLibcExp2Test, TrickyInputs) {
   constexpr int N = 16;
   constexpr uint64_t INPUTS[N] = {
       0x3FD79289C6E6A5C0,
@@ -60,7 +60,7 @@ TEST(LlvmLibcExp2Test, TrickyInputs) {
   }
 }
 
-TEST(LlvmLibcExp2Test, InDoubleRange) {
+TEST_F(LlvmLibcExp2Test, InDoubleRange) {
   constexpr uint64_t COUNT = 1'231;
   uint64_t START = LIBC_NAMESPACE::fputil::FPBits<double>(0.25).uintval();
   uint64_t STOP = LIBC_NAMESPACE::fputil::FPBits<double>(4.0).uintval();
diff --git a/libc/test/src/math/exp2f_test.cpp b/libc/test/src/math/exp2f_test.cpp
index 987e1b9f59..4c69c1d48d 100644
--- a/libc/test/src/math/exp2f_test.cpp
+++ b/libc/test/src/math/exp2f_test.cpp
@@ -17,11 +17,11 @@
 
 #include <stdint.h>
 
-namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
+using LlvmLibcExp2fTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
-DECLARE_SPECIAL_CONSTANTS(float)
+namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
-TEST(LlvmLibcExp2fTest, SpecialNumbers) {
+TEST_F(LlvmLibcExp2fTest, SpecialNumbers) {
   libc_errno = 0;
 
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::exp2f(aNaN));
@@ -40,7 +40,7 @@ TEST(LlvmLibcExp2fTest, SpecialNumbers) {
   EXPECT_MATH_ERRNO(0);
 }
 
-TEST(LlvmLibcExp2fTest, Overflow) {
+TEST_F(LlvmLibcExp2fTest, Overflow) {
   libc_errno = 0;
   EXPECT_FP_EQ_WITH_EXCEPTION(
       inf, LIBC_NAMESPACE::exp2f(float(FPBits(0x7f7fffffU))), FE_OVERFLOW);
@@ -55,7 +55,7 @@ TEST(LlvmLibcExp2fTest, Overflow) {
   EXPECT_MATH_ERRNO(ERANGE);
 }
 
-TEST(LlvmLibcExp2fTest, TrickyInputs) {
+TEST_F(LlvmLibcExp2fTest, TrickyInputs) {
   constexpr int N = 12;
   constexpr uint32_t INPUTS[N] = {
       0x3b429d37U, /*0x1.853a6ep-9f*/
@@ -80,7 +80,7 @@ TEST(LlvmLibcExp2fTest, TrickyInputs) {
   }
 }
 
-TEST(LlvmLibcExp2fTest, Underflow) {
+TEST_F(LlvmLibcExp2fTest, Underflow) {
   libc_errno = 0;
   EXPECT_FP_EQ_WITH_EXCEPTION(
       0.0f, LIBC_NAMESPACE::exp2f(float(FPBits(0xff7fffffU))), FE_UNDERFLOW);
@@ -102,7 +102,7 @@ TEST(LlvmLibcExp2fTest, Underflow) {
   EXPECT_MATH_ERRNO(ERANGE);
 }
 
-TEST(LlvmLibcExp2fTest, InFloatRange) {
+TEST_F(LlvmLibcExp2fTest, InFloatRange) {
   constexpr uint32_t COUNT = 100'000;
   constexpr uint32_t STEP = UINT32_MAX / COUNT;
   for (uint32_t i = 0, v = 0; i <= COUNT; ++i, v += STEP) {
diff --git a/libc/test/src/math/exp_test.cpp b/libc/test/src/math/exp_test.cpp
index 06894bce70..1de2d7507a 100644
--- a/libc/test/src/math/exp_test.cpp
+++ b/libc/test/src/math/exp_test.cpp
@@ -17,12 +17,12 @@
 #include <errno.h>
 #include <stdint.h>
 
+using LlvmLibcExpTest = LIBC_NAMESPACE::testing::FPTest<double>;
+
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 using LIBC_NAMESPACE::testing::tlog;
 
-DECLARE_SPECIAL_CONSTANTS(double)
-
-TEST(LlvmLibcExpTest, SpecialNumbers) {
+TEST_F(LlvmLibcExpTest, SpecialNumbers) {
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::exp(aNaN));
   EXPECT_FP_EQ(inf, LIBC_NAMESPACE::exp(inf));
   EXPECT_FP_EQ_ALL_ROUNDING(zero, LIBC_NAMESPACE::exp(neg_inf));
@@ -33,7 +33,7 @@ TEST(LlvmLibcExpTest, SpecialNumbers) {
   EXPECT_FP_EQ_ALL_ROUNDING(1.0, LIBC_NAMESPACE::exp(-0.0));
 }
 
-TEST(LlvmLibcExpTest, TrickyInputs) {
+TEST_F(LlvmLibcExpTest, TrickyInputs) {
   constexpr int N = 14;
   constexpr uint64_t INPUTS[N] = {
       0x3FD79289C6E6A5C0,
@@ -58,7 +58,7 @@ TEST(LlvmLibcExpTest, TrickyInputs) {
   }
 }
 
-TEST(LlvmLibcExpTest, InDoubleRange) {
+TEST_F(LlvmLibcExpTest, InDoubleRange) {
   constexpr uint64_t COUNT = 1'231;
   uint64_t START = LIBC_NAMESPACE::fputil::FPBits<double>(0.25).uintval();
   uint64_t STOP = LIBC_NAMESPACE::fputil::FPBits<double>(4.0).uintval();
diff --git a/libc/test/src/math/expf_test.cpp b/libc/test/src/math/expf_test.cpp
index f015e46d50..521eba705b 100644
--- a/libc/test/src/math/expf_test.cpp
+++ b/libc/test/src/math/expf_test.cpp
@@ -16,11 +16,11 @@
 
 #include <stdint.h>
 
-namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
+using LlvmLibcExpfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
-DECLARE_SPECIAL_CONSTANTS(float)
+namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
-TEST(LlvmLibcExpfTest, SpecialNumbers) {
+TEST_F(LlvmLibcExpfTest, SpecialNumbers) {
   libc_errno = 0;
 
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::expf(aNaN));
@@ -39,7 +39,7 @@ TEST(LlvmLibcExpfTest, SpecialNumbers) {
   EXPECT_MATH_ERRNO(0);
 }
 
-TEST(LlvmLibcExpfTest, Overflow) {
+TEST_F(LlvmLibcExpfTest, Overflow) {
   libc_errno = 0;
   EXPECT_FP_EQ_WITH_EXCEPTION(
       inf, LIBC_NAMESPACE::expf(float(FPBits(0x7f7fffffU))), FE_OVERFLOW);
@@ -54,7 +54,7 @@ TEST(LlvmLibcExpfTest, Overflow) {
   EXPECT_MATH_ERRNO(ERANGE);
 }
 
-TEST(LlvmLibcExpfTest, Underflow) {
+TEST_F(LlvmLibcExpfTest, Underflow) {
   libc_errno = 0;
   EXPECT_FP_EQ_WITH_EXCEPTION(
       0.0f, LIBC_NAMESPACE::expf(float(FPBits(0xff7fffffU))), FE_UNDERFLOW);
@@ -73,7 +73,7 @@ TEST(LlvmLibcExpfTest, Underflow) {
 
 // Test with inputs which are the borders of underflow/overflow but still
 // produce valid results without setting errno.
-TEST(LlvmLibcExpfTest, Borderline) {
+TEST_F(LlvmLibcExpfTest, Borderline) {
   float x;
 
   libc_errno = 0;
@@ -103,7 +103,7 @@ TEST(LlvmLibcExpfTest, Borderline) {
   EXPECT_MATH_ERRNO(0);
 }
 
-TEST(LlvmLibcExpfTest, InFloatRange) {
+TEST_F(LlvmLibcExpfTest, InFloatRange) {
   constexpr uint32_t COUNT = 100'000;
   constexpr uint32_t STEP = UINT32_MAX / COUNT;
   for (uint32_t i = 0, v = 0; i <= COUNT; ++i, v += STEP) {
diff --git a/libc/test/src/math/explogxf_test.cpp b/libc/test/src/math/explogxf_test.cpp
index 7b402e17fb..24f9f3c0f8 100644
--- a/libc/test/src/math/explogxf_test.cpp
+++ b/libc/test/src/math/explogxf_test.cpp
@@ -14,9 +14,9 @@
 #include "utils/MPFRWrapper/MPFRUtils.h"
 #include <math.h>
 
-namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
+using LlvmLibcExplogfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
-DECLARE_SPECIAL_CONSTANTS(float)
+namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
 constexpr int def_count = 100003;
 constexpr float def_prec = 0.500001f;
@@ -25,7 +25,7 @@ auto f_normal = [](float x) -> bool {
   return !(isnan(x) || isinf(x) || fabs(x) < 2E-38);
 };
 
-TEST(LlvmLibcExpxfTest, InFloatRange) {
+TEST_F(LlvmLibcExplogfTest, ExpInFloatRange) {
   auto fx = [](float x) -> float {
     auto result = LIBC_NAMESPACE::exp_b_range_reduc<LIBC_NAMESPACE::ExpBase>(x);
     double r = LIBC_NAMESPACE::ExpBase::powb_lo(result.lo);
@@ -39,12 +39,12 @@ TEST(LlvmLibcExpxfTest, InFloatRange) {
              def_prec);
 }
 
-TEST(LlvmLibcLog2xfTest, InFloatRange) {
+TEST_F(LlvmLibcExplogfTest, Log2InFloatRange) {
   CHECK_DATA(0.0f, inf, mpfr::Operation::Log2, LIBC_NAMESPACE::log2_eval,
              f_normal, def_count, def_prec);
 }
 
-TEST(LlvmLibcLogxfTest, InFloatRange) {
+TEST_F(LlvmLibcExplogfTest, LogInFloatRange) {
   CHECK_DATA(0.0f, inf, mpfr::Operation::Log, LIBC_NAMESPACE::log_eval,
              f_normal, def_count, def_prec);
 }
diff --git a/libc/test/src/math/expm1_test.cpp b/libc/test/src/math/expm1_test.cpp
index 76e715e00c..ad53ffb6e8 100644
--- a/libc/test/src/math/expm1_test.cpp
+++ b/libc/test/src/math/expm1_test.cpp
@@ -17,12 +17,12 @@
 #include <errno.h>
 #include <stdint.h>
 
+using LlvmLibcExpm1Test = LIBC_NAMESPACE::testing::FPTest<double>;
+
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 using LIBC_NAMESPACE::testing::tlog;
 
-DECLARE_SPECIAL_CONSTANTS(double)
-
-TEST(LlvmLibcExpm1Test, TrickyInputs) {
+TEST_F(LlvmLibcExpm1Test, TrickyInputs) {
   constexpr int N = 21;
   constexpr uint64_t INPUTS[N] = {
       0x3FD79289C6E6A5C0, // x=0x1.79289c6e6a5cp-2
@@ -54,7 +54,7 @@ TEST(LlvmLibcExpm1Test, TrickyInputs) {
   }
 }
 
-TEST(LlvmLibcExpm1Test, InDoubleRange) {
+TEST_F(LlvmLibcExpm1Test, InDoubleRange) {
   constexpr uint64_t COUNT = 1'231;
   uint64_t START = LIBC_NAMESPACE::fputil::FPBits<double>(0.25).uintval();
   uint64_t STOP = LIBC_NAMESPACE::fputil::FPBits<double>(4.0).uintval();
diff --git a/libc/test/src/math/expm1f_test.cpp b/libc/test/src/math/expm1f_test.cpp
index f63c9f464c..a9eaa4dbe5 100644
--- a/libc/test/src/math/expm1f_test.cpp
+++ b/libc/test/src/math/expm1f_test.cpp
@@ -16,11 +16,11 @@
 
 #include <stdint.h>
 
-namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
+using LlvmLibcExpm1fTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
-DECLARE_SPECIAL_CONSTANTS(float)
+namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
-TEST(LlvmLibcExpm1fTest, SpecialNumbers) {
+TEST_F(LlvmLibcExpm1fTest, SpecialNumbers) {
   libc_errno = 0;
 
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::expm1f(aNaN));
@@ -39,7 +39,7 @@ TEST(LlvmLibcExpm1fTest, SpecialNumbers) {
   EXPECT_MATH_ERRNO(0);
 }
 
-TEST(LlvmLibcExpm1fTest, Overflow) {
+TEST_F(LlvmLibcExpm1fTest, Overflow) {
   libc_errno = 0;
   EXPECT_FP_EQ_WITH_EXCEPTION(
       inf, LIBC_NAMESPACE::expm1f(float(FPBits(0x7f7fffffU))), FE_OVERFLOW);
@@ -54,7 +54,7 @@ TEST(LlvmLibcExpm1fTest, Overflow) {
   EXPECT_MATH_ERRNO(ERANGE);
 }
 
-TEST(LlvmLibcExpm1fTest, Underflow) {
+TEST_F(LlvmLibcExpm1fTest, Underflow) {
   libc_errno = 0;
   EXPECT_FP_EQ(-1.0f, LIBC_NAMESPACE::expm1f(float(FPBits(0xff7fffffU))));
 
@@ -67,7 +67,7 @@ TEST(LlvmLibcExpm1fTest, Underflow) {
 
 // Test with inputs which are the borders of underflow/overflow but still
 // produce valid results without setting errno.
-TEST(LlvmLibcExpm1fTest, Borderline) {
+TEST_F(LlvmLibcExpm1fTest, Borderline) {
   float x;
 
   libc_errno = 0;
@@ -112,7 +112,7 @@ TEST(LlvmLibcExpm1fTest, Borderline) {
   EXPECT_MATH_ERRNO(0);
 }
 
-TEST(LlvmLibcExpm1fTest, InFloatRange) {
+TEST_F(LlvmLibcExpm1fTest, InFloatRange) {
   constexpr uint32_t COUNT = 100'000;
   constexpr uint32_t STEP = UINT32_MAX / COUNT;
   for (uint32_t i = 0, v = 0; i <= COUNT; ++i, v += STEP) {
diff --git a/libc/test/src/math/inv_trigf_utils_test.cpp b/libc/test/src/math/inv_trigf_utils_test.cpp
index 4f50027bf8..23420edcd0 100644
--- a/libc/test/src/math/inv_trigf_utils_test.cpp
+++ b/libc/test/src/math/inv_trigf_utils_test.cpp
@@ -14,21 +14,21 @@
 #include "utils/MPFRWrapper/MPFRUtils.h"
 #include <math.h>
 
-namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
+using LlvmLibcAtanfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
-DECLARE_SPECIAL_CONSTANTS(float)
+namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
 constexpr int def_count = 100003;
 constexpr float def_prec = 0.500001f;
 
 auto f_normal = [](float x) -> bool { return !(isnan(x) || isinf(x)); };
 
-TEST(LlvmLibcAtanfPosTest, InFloatRange) {
+TEST_F(LlvmLibcAtanfTest, InPositiveRange) {
   CHECK_DATA(0.0f, inf, mpfr::Operation::Atan, LIBC_NAMESPACE::atan_eval,
              f_normal, def_count, def_prec);
 }
 
-TEST(LlvmLibcAtanfNegTest, InFloatRange) {
+TEST_F(LlvmLibcAtanfTest, InNegativeRange) {
   CHECK_DATA(-0.0f, neg_inf, mpfr::Operation::Atan, LIBC_NAMESPACE::atan_eval,
              f_normal, def_count, def_prec);
 }
diff --git a/libc/test/src/math/log10_test.cpp b/libc/test/src/math/log10_test.cpp
index 46458c3246..72224c2471 100644
--- a/libc/test/src/math/log10_test.cpp
+++ b/libc/test/src/math/log10_test.cpp
@@ -17,12 +17,12 @@
 #include <errno.h>
 #include <stdint.h>
 
+using LlvmLibcLog10Test = LIBC_NAMESPACE::testing::FPTest<double>;
+
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 using LIBC_NAMESPACE::testing::tlog;
 
-DECLARE_SPECIAL_CONSTANTS(double)
-
-TEST(LlvmLibcLog10Test, SpecialNumbers) {
+TEST_F(LlvmLibcLog10Test, SpecialNumbers) {
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::log10(aNaN));
   EXPECT_FP_EQ(inf, LIBC_NAMESPACE::log10(inf));
   EXPECT_FP_IS_NAN_WITH_EXCEPTION(LIBC_NAMESPACE::log10(neg_inf), FE_INVALID);
@@ -34,7 +34,7 @@ TEST(LlvmLibcLog10Test, SpecialNumbers) {
   EXPECT_FP_EQ_ALL_ROUNDING(zero, LIBC_NAMESPACE::log10(1.0));
 }
 
-TEST(LlvmLibcLog10Test, TrickyInputs) {
+TEST_F(LlvmLibcLog10Test, TrickyInputs) {
   constexpr int N = 36;
   constexpr uint64_t INPUTS[N] = {
       0x3ff0000000000000, // x = 1.0
@@ -72,7 +72,7 @@ TEST(LlvmLibcLog10Test, TrickyInputs) {
   }
 }
 
-TEST(LlvmLibcLog10Test, AllExponents) {
+TEST_F(LlvmLibcLog10Test, AllExponents) {
   double x = 0x1.0p-1074;
   for (int i = -1074; i < 1024; ++i, x *= 2.0) {
     ASSERT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Log10, x,
@@ -80,7 +80,7 @@ TEST(LlvmLibcLog10Test, AllExponents) {
   }
 }
 
-TEST(LlvmLibcLog10Test, InDoubleRange) {
+TEST_F(LlvmLibcLog10Test, InDoubleRange) {
   constexpr uint64_t COUNT = 1'001;
   constexpr uint64_t START = 0x3FD0'0000'0000'0000ULL; // 0.25
   constexpr uint64_t STOP = 0x4010'0000'0000'0000ULL;  // 4.0
diff --git a/libc/test/src/math/log10f_test.cpp b/libc/test/src/math/log10f_test.cpp
index 05c42b9265..3448ea7570 100644
--- a/libc/test/src/math/log10f_test.cpp
+++ b/libc/test/src/math/log10f_test.cpp
@@ -16,11 +16,11 @@
 #include <errno.h>
 #include <stdint.h>
 
-namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
+using LlvmLibcLog10fTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
-DECLARE_SPECIAL_CONSTANTS(float)
+namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
-TEST(LlvmLibcLog10fTest, SpecialNumbers) {
+TEST_F(LlvmLibcLog10fTest, SpecialNumbers) {
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::log10f(aNaN));
   EXPECT_FP_EQ(inf, LIBC_NAMESPACE::log10f(inf));
   EXPECT_FP_IS_NAN_WITH_EXCEPTION(LIBC_NAMESPACE::log10f(neg_inf), FE_INVALID);
@@ -32,7 +32,7 @@ TEST(LlvmLibcLog10fTest, SpecialNumbers) {
   EXPECT_FP_EQ_ALL_ROUNDING(zero, LIBC_NAMESPACE::log10f(1.0f));
 }
 
-TEST(LlvmLibcLog10fTest, TrickyInputs) {
+TEST_F(LlvmLibcLog10fTest, TrickyInputs) {
   constexpr int N = 21;
   constexpr uint32_t INPUTS[N] = {
       0x3f800000U /*1.0f*/,
@@ -65,7 +65,7 @@ TEST(LlvmLibcLog10fTest, TrickyInputs) {
   }
 }
 
-TEST(LlvmLibcLog10fTest, InFloatRange) {
+TEST_F(LlvmLibcLog10fTest, InFloatRange) {
   constexpr uint32_t COUNT = 100'000;
   constexpr uint32_t STEP = UINT32_MAX / COUNT;
   for (uint32_t i = 0, v = 0; i <= COUNT; ++i, v += STEP) {
diff --git a/libc/test/src/math/log1p_test.cpp b/libc/test/src/math/log1p_test.cpp
index b677e7b416..5bec911937 100644
--- a/libc/test/src/math/log1p_test.cpp
+++ b/libc/test/src/math/log1p_test.cpp
@@ -17,12 +17,12 @@
 #include <errno.h>
 #include <stdint.h>
 
+using LlvmLibcLog1pTest = LIBC_NAMESPACE::testing::FPTest<double>;
+
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 using LIBC_NAMESPACE::testing::tlog;
 
-DECLARE_SPECIAL_CONSTANTS(double)
-
-TEST(LlvmLibcLog1pTest, SpecialNumbers) {
+TEST_F(LlvmLibcLog1pTest, SpecialNumbers) {
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::log1p(aNaN));
   EXPECT_FP_EQ(inf, LIBC_NAMESPACE::log1p(inf));
   EXPECT_FP_IS_NAN_WITH_EXCEPTION(LIBC_NAMESPACE::log1p(neg_inf), FE_INVALID);
@@ -33,7 +33,7 @@ TEST(LlvmLibcLog1pTest, SpecialNumbers) {
                               FE_DIVBYZERO);
 }
 
-TEST(LlvmLibcLog1pTest, TrickyInputs) {
+TEST_F(LlvmLibcLog1pTest, TrickyInputs) {
   constexpr int N = 41;
   constexpr uint64_t INPUTS[N] = {
       0x3ff0000000000000, // x = 1.0
@@ -73,7 +73,7 @@ TEST(LlvmLibcLog1pTest, TrickyInputs) {
   }
 }
 
-TEST(LlvmLibcLog1pTest, AllExponents) {
+TEST_F(LlvmLibcLog1pTest, AllExponents) {
   double x = 0x1.0p-1074;
   for (int i = -1074; i < 1024; ++i, x *= 2.0) {
     ASSERT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Log1p, x,
@@ -81,7 +81,7 @@ TEST(LlvmLibcLog1pTest, AllExponents) {
   }
 }
 
-TEST(LlvmLibcLog1pTest, InDoubleRange) {
+TEST_F(LlvmLibcLog1pTest, InDoubleRange) {
   constexpr uint64_t COUNT = 4501;
 
   auto test = [&](uint64_t start, uint64_t stop,
diff --git a/libc/test/src/math/log1pf_test.cpp b/libc/test/src/math/log1pf_test.cpp
index d8132e479e..16cdc4704c 100644
--- a/libc/test/src/math/log1pf_test.cpp
+++ b/libc/test/src/math/log1pf_test.cpp
@@ -17,11 +17,11 @@
 #include <errno.h>
 #include <stdint.h>
 
-namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
+using LlvmLibcLog1pfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
-DECLARE_SPECIAL_CONSTANTS(float)
+namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
-TEST(LlvmLibclog1pfTest, SpecialNumbers) {
+TEST_F(LlvmLibcLog1pfTest, SpecialNumbers) {
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::log1pf(aNaN));
   EXPECT_FP_EQ(inf, LIBC_NAMESPACE::log1pf(inf));
   EXPECT_FP_IS_NAN_WITH_EXCEPTION(LIBC_NAMESPACE::log1pf(neg_inf), FE_INVALID);
@@ -31,7 +31,7 @@ TEST(LlvmLibclog1pfTest, SpecialNumbers) {
                               FE_DIVBYZERO);
 }
 
-TEST(LlvmLibclog1pfTest, TrickyInputs) {
+TEST_F(LlvmLibcLog1pfTest, TrickyInputs) {
   constexpr int N = 27;
   constexpr uint32_t INPUTS[N] = {
       0x35c00006U, /*0x1.80000cp-20f*/
@@ -69,7 +69,7 @@ TEST(LlvmLibclog1pfTest, TrickyInputs) {
   }
 }
 
-TEST(LlvmLibclog1pfTest, InFloatRange) {
+TEST_F(LlvmLibcLog1pfTest, InFloatRange) {
   constexpr uint32_t COUNT = 100'000;
   constexpr uint32_t STEP = UINT32_MAX / COUNT;
   for (uint32_t i = 0, v = 0; i <= COUNT; ++i, v += STEP) {
diff --git a/libc/test/src/math/log2_test.cpp b/libc/test/src/math/log2_test.cpp
index 42643fa708..b471b8eb54 100644
--- a/libc/test/src/math/log2_test.cpp
+++ b/libc/test/src/math/log2_test.cpp
@@ -17,12 +17,12 @@
 #include <errno.h>
 #include <stdint.h>
 
+using LlvmLibcLog2Test = LIBC_NAMESPACE::testing::FPTest<double>;
+
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 using LIBC_NAMESPACE::testing::tlog;
 
-DECLARE_SPECIAL_CONSTANTS(double)
-
-TEST(LlvmLibcLog2Test, SpecialNumbers) {
+TEST_F(LlvmLibcLog2Test, SpecialNumbers) {
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::log2(aNaN));
   EXPECT_FP_EQ(inf, LIBC_NAMESPACE::log2(inf));
   EXPECT_FP_IS_NAN_WITH_EXCEPTION(LIBC_NAMESPACE::log2(neg_inf), FE_INVALID);
@@ -33,7 +33,7 @@ TEST(LlvmLibcLog2Test, SpecialNumbers) {
   EXPECT_FP_EQ_ALL_ROUNDING(zero, LIBC_NAMESPACE::log2(1.0));
 }
 
-TEST(LlvmLibcLog2Test, TrickyInputs) {
+TEST_F(LlvmLibcLog2Test, TrickyInputs) {
   constexpr int N = 30;
   constexpr uint64_t INPUTS[N] = {
       0x3ff0000000000000, // x = 1.0
@@ -70,7 +70,7 @@ TEST(LlvmLibcLog2Test, TrickyInputs) {
   }
 }
 
-TEST(LlvmLibcLog2Test, AllExponents) {
+TEST_F(LlvmLibcLog2Test, AllExponents) {
   double x = 0x1.0p-1074;
   for (int i = -1074; i < 1024; ++i, x *= 2.0) {
     ASSERT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Log2, x,
@@ -78,7 +78,7 @@ TEST(LlvmLibcLog2Test, AllExponents) {
   }
 }
 
-TEST(LlvmLibcLog2Test, InDoubleRange) {
+TEST_F(LlvmLibcLog2Test, InDoubleRange) {
   constexpr uint64_t COUNT = 1'001;
   constexpr uint64_t START = 0x3FD0'0000'0000'0000ULL; // 0.25
   constexpr uint64_t STOP = 0x4010'0000'0000'0000ULL;  // 4.0
diff --git a/libc/test/src/math/log2f_test.cpp b/libc/test/src/math/log2f_test.cpp
index d4f4b937c9..aaa6320fb2 100644
--- a/libc/test/src/math/log2f_test.cpp
+++ b/libc/test/src/math/log2f_test.cpp
@@ -16,11 +16,11 @@
 
 #include <stdint.h>
 
-namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
+using LlvmLibcLog2fTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
-DECLARE_SPECIAL_CONSTANTS(float)
+namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
-TEST(LlvmLibcLog2fTest, SpecialNumbers) {
+TEST_F(LlvmLibcLog2fTest, SpecialNumbers) {
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::log2f(aNaN));
   EXPECT_FP_EQ(inf, LIBC_NAMESPACE::log2f(inf));
   EXPECT_FP_IS_NAN_WITH_EXCEPTION(LIBC_NAMESPACE::log2f(neg_inf), FE_INVALID);
@@ -32,7 +32,7 @@ TEST(LlvmLibcLog2fTest, SpecialNumbers) {
   EXPECT_FP_EQ_ALL_ROUNDING(zero, LIBC_NAMESPACE::log2f(1.0f));
 }
 
-TEST(LlvmLibcLog2fTest, TrickyInputs) {
+TEST_F(LlvmLibcLog2fTest, TrickyInputs) {
   constexpr int N = 10;
   constexpr uint32_t INPUTS[N] = {
       0x3f7d57f5U, 0x3f7e3274U, 0x3f7ed848U, 0x3f7fd6ccU, 0x3f7fffffU,
@@ -45,7 +45,7 @@ TEST(LlvmLibcLog2fTest, TrickyInputs) {
   }
 }
 
-TEST(LlvmLibcLog2fTest, InFloatRange) {
+TEST_F(LlvmLibcLog2fTest, InFloatRange) {
   constexpr uint32_t COUNT = 100'000;
   constexpr uint32_t STEP = UINT32_MAX / COUNT;
   for (uint32_t i = 0, v = 0; i <= COUNT; ++i, v += STEP) {
diff --git a/libc/test/src/math/log_test.cpp b/libc/test/src/math/log_test.cpp
index caa274828f..e3f41e1e26 100644
--- a/libc/test/src/math/log_test.cpp
+++ b/libc/test/src/math/log_test.cpp
@@ -17,12 +17,12 @@
 #include <errno.h>
 #include <stdint.h>
 
+using LlvmLibcLogTest = LIBC_NAMESPACE::testing::FPTest<double>;
+
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 using LIBC_NAMESPACE::testing::tlog;
 
-DECLARE_SPECIAL_CONSTANTS(double)
-
-TEST(LlvmLibcLogTest, SpecialNumbers) {
+TEST_F(LlvmLibcLogTest, SpecialNumbers) {
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::log(aNaN));
   EXPECT_FP_EQ(inf, LIBC_NAMESPACE::log(inf));
   EXPECT_FP_IS_NAN_WITH_EXCEPTION(LIBC_NAMESPACE::log(neg_inf), FE_INVALID);
@@ -32,7 +32,7 @@ TEST(LlvmLibcLogTest, SpecialNumbers) {
   EXPECT_FP_EQ_ALL_ROUNDING(zero, LIBC_NAMESPACE::log(1.0));
 }
 
-TEST(LlvmLibcLogTest, TrickyInputs) {
+TEST_F(LlvmLibcLogTest, TrickyInputs) {
   constexpr int N = 30;
   constexpr uint64_t INPUTS[N] = {
       0x3ff0000000000000, // x = 1.0
@@ -69,7 +69,7 @@ TEST(LlvmLibcLogTest, TrickyInputs) {
   }
 }
 
-TEST(LlvmLibcLogTest, AllExponents) {
+TEST_F(LlvmLibcLogTest, AllExponents) {
   double x = 0x1.0p-1074;
   for (int i = -1074; i < 1024; ++i, x *= 2.0) {
     ASSERT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Log, x,
@@ -77,7 +77,7 @@ TEST(LlvmLibcLogTest, AllExponents) {
   }
 }
 
-TEST(LlvmLibcLogTest, InDoubleRange) {
+TEST_F(LlvmLibcLogTest, InDoubleRange) {
   constexpr uint64_t COUNT = 234561;
   constexpr uint64_t START = 0x3FD0'0000'0000'0000ULL; // 0.25
   constexpr uint64_t STOP = 0x4010'0000'0000'0000ULL;  // 4.0
diff --git a/libc/test/src/math/logf_test.cpp b/libc/test/src/math/logf_test.cpp
index 832bdd21ed..c02e95cb6f 100644
--- a/libc/test/src/math/logf_test.cpp
+++ b/libc/test/src/math/logf_test.cpp
@@ -15,11 +15,11 @@
 
 #include <stdint.h>
 
-namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
+using LlvmLibcLogfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
-DECLARE_SPECIAL_CONSTANTS(float)
+namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
-TEST(LlvmLibcLogfTest, SpecialNumbers) {
+TEST_F(LlvmLibcLogfTest, SpecialNumbers) {
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::logf(aNaN));
   EXPECT_FP_EQ(inf, LIBC_NAMESPACE::logf(inf));
   EXPECT_FP_IS_NAN_WITH_EXCEPTION(LIBC_NAMESPACE::logf(neg_inf), FE_INVALID);
@@ -31,7 +31,7 @@ TEST(LlvmLibcLogfTest, SpecialNumbers) {
   EXPECT_FP_EQ_ALL_ROUNDING(zero, LIBC_NAMESPACE::logf(1.0f));
 }
 
-TEST(LlvmLibcLogfTest, TrickyInputs) {
+TEST_F(LlvmLibcLogfTest, TrickyInputs) {
   constexpr int N = 35;
   constexpr uint32_t INPUTS[N] = {
       0x1b7679ffU, /*0x1.ecf3fep-73f*/
@@ -77,7 +77,7 @@ TEST(LlvmLibcLogfTest, TrickyInputs) {
   }
 }
 
-TEST(LlvmLibcLogfTest, InFloatRange) {
+TEST_F(LlvmLibcLogfTest, InFloatRange) {
   constexpr uint32_t COUNT = 100'000;
   constexpr uint32_t STEP = UINT32_MAX / COUNT;
   for (uint32_t i = 0, v = 0; i <= COUNT; ++i, v += STEP) {
diff --git a/libc/test/src/math/sin_test.cpp b/libc/test/src/math/sin_test.cpp
index ee8451f500..b18219577a 100644
--- a/libc/test/src/math/sin_test.cpp
+++ b/libc/test/src/math/sin_test.cpp
@@ -14,11 +14,11 @@
 
 #include <math.h>
 
-namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
+using LlvmLibcSinTest = LIBC_NAMESPACE::testing::FPTest<double>;
 
-DECLARE_SPECIAL_CONSTANTS(double)
+namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
-TEST(LlvmLibcSinTest, Range) {
+TEST_F(LlvmLibcSinTest, Range) {
   static constexpr double _2pi = 6.283185307179586;
   constexpr UIntType COUNT = 100'000;
   constexpr UIntType STEP = UIntType(-1) / COUNT;
diff --git a/libc/test/src/math/sincosf_test.cpp b/libc/test/src/math/sincosf_test.cpp
index f67fba1d31..fde707f892 100644
--- a/libc/test/src/math/sincosf_test.cpp
+++ b/libc/test/src/math/sincosf_test.cpp
@@ -18,14 +18,13 @@
 #include <errno.h>
 #include <stdint.h>
 
+using LlvmLibcSinCosfTest = LIBC_NAMESPACE::testing::FPTest<float>;
+
 using LIBC_NAMESPACE::testing::SDCOMP26094_VALUES;
-using FPBits = LIBC_NAMESPACE::fputil::FPBits<float>;
 
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
-DECLARE_SPECIAL_CONSTANTS(float)
-
-TEST(LlvmLibcSinCosfTest, SpecialNumbers) {
+TEST_F(LlvmLibcSinCosfTest, SpecialNumbers) {
   libc_errno = 0;
   float sin, cos;
 
@@ -97,7 +96,7 @@ TEST(LlvmLibcSinCosfTest, SpecialNumbers) {
     }                                                                          \
   }
 
-TEST(LlvmLibcSinCosfTest, InFloatRange) {
+TEST_F(LlvmLibcSinCosfTest, InFloatRange) {
   constexpr uint32_t COUNT = 1'001;
   constexpr uint32_t STEP = UINT32_MAX / COUNT;
   for (uint32_t i = 0, v = 0; i <= COUNT; ++i, v += STEP) {
@@ -111,7 +110,7 @@ TEST(LlvmLibcSinCosfTest, InFloatRange) {
 }
 
 // For hard to round inputs.
-TEST(LlvmLibcSinCosfTest, SpecialValues) {
+TEST_F(LlvmLibcSinCosfTest, SpecialValues) {
   constexpr int N = 43;
   constexpr uint32_t INPUTS[N] = {
       0x3b56'37f5U, // x = 0x1.ac6feap-9f
@@ -168,7 +167,7 @@ TEST(LlvmLibcSinCosfTest, SpecialValues) {
 
 // SDCOMP-26094: check sinf in the cases for which the range reducer
 // returns values furthest beyond its nominal upper bound of pi/4.
-TEST(LlvmLibcSinCosfTest, SDCOMP_26094) {
+TEST_F(LlvmLibcSinCosfTest, SDCOMP_26094) {
   for (uint32_t v : SDCOMP26094_VALUES) {
     float x = float(FPBits((v)));
     EXPECT_SINCOS_MATCH_ALL_ROUNDING(x);
diff --git a/libc/test/src/math/sinf_test.cpp b/libc/test/src/math/sinf_test.cpp
index 2425110442..107b015512 100644
--- a/libc/test/src/math/sinf_test.cpp
+++ b/libc/test/src/math/sinf_test.cpp
@@ -18,14 +18,13 @@
 #include <errno.h>
 #include <stdint.h>
 
+using LlvmLibcSinfTest = LIBC_NAMESPACE::testing::FPTest<float>;
+
 using LIBC_NAMESPACE::testing::SDCOMP26094_VALUES;
-using FPBits = LIBC_NAMESPACE::fputil::FPBits<float>;
 
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
-DECLARE_SPECIAL_CONSTANTS(float)
-
-TEST(LlvmLibcSinfTest, SpecialNumbers) {
+TEST_F(LlvmLibcSinfTest, SpecialNumbers) {
   libc_errno = 0;
 
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::sinf(aNaN));
@@ -44,7 +43,7 @@ TEST(LlvmLibcSinfTest, SpecialNumbers) {
   EXPECT_MATH_ERRNO(EDOM);
 }
 
-TEST(LlvmLibcSinfTest, InFloatRange) {
+TEST_F(LlvmLibcSinfTest, InFloatRange) {
   constexpr uint32_t COUNT = 100'000;
   constexpr uint32_t STEP = UINT32_MAX / COUNT;
   for (uint32_t i = 0, v = 0; i <= COUNT; ++i, v += STEP) {
@@ -56,7 +55,7 @@ TEST(LlvmLibcSinfTest, InFloatRange) {
   }
 }
 
-TEST(LlvmLibcSinfTest, SpecificBitPatterns) {
+TEST_F(LlvmLibcSinfTest, SpecificBitPatterns) {
   constexpr int N = 36;
   constexpr uint32_t INPUTS[N] = {
       0x3f06'0a92U, // x = pi/6
@@ -107,7 +106,7 @@ TEST(LlvmLibcSinfTest, SpecificBitPatterns) {
 }
 
 // For small values, sin(x) is x.
-TEST(LlvmLibcSinfTest, SmallValues) {
+TEST_F(LlvmLibcSinfTest, SmallValues) {
   float x = float(FPBits(0x1780'0000U));
   EXPECT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Sin, x,
                                  LIBC_NAMESPACE::sinf(x), 0.5);
@@ -119,7 +118,7 @@ TEST(LlvmLibcSinfTest, SmallValues) {
 
 // SDCOMP-26094: check sinf in the cases for which the range reducer
 // returns values furthest beyond its nominal upper bound of pi/4.
-TEST(LlvmLibcSinfTest, SDCOMP_26094) {
+TEST_F(LlvmLibcSinfTest, SDCOMP_26094) {
   for (uint32_t v : SDCOMP26094_VALUES) {
     float x = float(FPBits((v)));
     EXPECT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Sin, x,
diff --git a/libc/test/src/math/sinhf_test.cpp b/libc/test/src/math/sinhf_test.cpp
index 92284ca7d0..dc80a1b7ae 100644
--- a/libc/test/src/math/sinhf_test.cpp
+++ b/libc/test/src/math/sinhf_test.cpp
@@ -18,13 +18,11 @@
 #include <errno.h>
 #include <stdint.h>
 
-using FPBits = LIBC_NAMESPACE::fputil::FPBits<float>;
+using LlvmLibcSinhfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
-DECLARE_SPECIAL_CONSTANTS(float)
-
-TEST(LlvmLibcSinhfTest, SpecialNumbers) {
+TEST_F(LlvmLibcSinhfTest, SpecialNumbers) {
   libc_errno = 0;
 
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::sinhf(aNaN));
@@ -43,7 +41,7 @@ TEST(LlvmLibcSinhfTest, SpecialNumbers) {
   EXPECT_MATH_ERRNO(0);
 }
 
-TEST(LlvmLibcSinhfTest, InFloatRange) {
+TEST_F(LlvmLibcSinhfTest, InFloatRange) {
   constexpr uint32_t COUNT = 100'000;
   constexpr uint32_t STEP = UINT32_MAX / COUNT;
   for (uint32_t i = 0, v = 0; i <= COUNT; ++i, v += STEP) {
@@ -55,7 +53,7 @@ TEST(LlvmLibcSinhfTest, InFloatRange) {
 }
 
 // For small values, sinh(x) is x.
-TEST(LlvmLibcSinhfTest, SmallValues) {
+TEST_F(LlvmLibcSinhfTest, SmallValues) {
   float x = float(FPBits(uint32_t(0x17800000)));
   float result = LIBC_NAMESPACE::sinhf(x);
   EXPECT_MPFR_MATCH(mpfr::Operation::Sinh, x, result, 0.5);
@@ -67,7 +65,7 @@ TEST(LlvmLibcSinhfTest, SmallValues) {
   EXPECT_FP_EQ(x, result);
 }
 
-TEST(LlvmLibcSinhfTest, Overflow) {
+TEST_F(LlvmLibcSinhfTest, Overflow) {
   libc_errno = 0;
   EXPECT_FP_EQ_WITH_EXCEPTION(
       inf, LIBC_NAMESPACE::sinhf(float(FPBits(0x7f7fffffU))), FE_OVERFLOW);
@@ -82,7 +80,7 @@ TEST(LlvmLibcSinhfTest, Overflow) {
   EXPECT_MATH_ERRNO(ERANGE);
 }
 
-TEST(LlvmLibcSinhfTest, ExceptionalValues) {
+TEST_F(LlvmLibcSinhfTest, ExceptionalValues) {
   float x = float(FPBits(uint32_t(0x3a12'85ffU)));
   EXPECT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Sinh, x,
                                  LIBC_NAMESPACE::sinhf(x), 0.5);
diff --git a/libc/test/src/math/smoke/acosf_test.cpp b/libc/test/src/math/smoke/acosf_test.cpp
index b8ba3d5f85..2d8fba9671 100644
--- a/libc/test/src/math/smoke/acosf_test.cpp
+++ b/libc/test/src/math/smoke/acosf_test.cpp
@@ -16,11 +16,9 @@
 #include <errno.h>
 #include <stdint.h>
 
-using FPBits = LIBC_NAMESPACE::fputil::FPBits<float>;
+using LlvmLibcAcosfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
-DECLARE_SPECIAL_CONSTANTS(float)
-
-TEST(LlvmLibcAcosfTest, SpecialNumbers) {
+TEST_F(LlvmLibcAcosfTest, SpecialNumbers) {
   libc_errno = 0;
 
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::acosf(aNaN));
diff --git a/libc/test/src/math/smoke/acoshf_test.cpp b/libc/test/src/math/smoke/acoshf_test.cpp
index b8d7453bfd..85c24e08ab 100644
--- a/libc/test/src/math/smoke/acoshf_test.cpp
+++ b/libc/test/src/math/smoke/acoshf_test.cpp
@@ -16,11 +16,9 @@
 #include <errno.h>
 #include <stdint.h>
 
-using FPBits_t = LIBC_NAMESPACE::fputil::FPBits<float>;
+using LlvmLibcAcoshfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
-DECLARE_SPECIAL_CONSTANTS(float)
-
-TEST(LlvmLibcAcoshfTest, SpecialNumbers) {
+TEST_F(LlvmLibcAcoshfTest, SpecialNumbers) {
   libc_errno = 0;
 
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::acoshf(aNaN));
diff --git a/libc/test/src/math/smoke/asinf_test.cpp b/libc/test/src/math/smoke/asinf_test.cpp
index e2eca352e1..57bb7dcf6e 100644
--- a/libc/test/src/math/smoke/asinf_test.cpp
+++ b/libc/test/src/math/smoke/asinf_test.cpp
@@ -16,11 +16,9 @@
 #include <errno.h>
 #include <stdint.h>
 
-using FPBits = LIBC_NAMESPACE::fputil::FPBits<float>;
+using LlvmLibcAsinfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
-DECLARE_SPECIAL_CONSTANTS(float)
-
-TEST(LlvmLibcAsinfTest, SpecialNumbers) {
+TEST_F(LlvmLibcAsinfTest, SpecialNumbers) {
   libc_errno = 0;
 
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::asinf(aNaN));
diff --git a/libc/test/src/math/smoke/asinhf_test.cpp b/libc/test/src/math/smoke/asinhf_test.cpp
index f10cf934c5..0c6ac537ef 100644
--- a/libc/test/src/math/smoke/asinhf_test.cpp
+++ b/libc/test/src/math/smoke/asinhf_test.cpp
@@ -16,11 +16,9 @@
 #include <errno.h>
 #include <stdint.h>
 
-using FPBits_t = LIBC_NAMESPACE::fputil::FPBits<float>;
+using LlvmLibcAsinhfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
-DECLARE_SPECIAL_CONSTANTS(float)
-
-TEST(LlvmLibcAsinhfTest, SpecialNumbers) {
+TEST_F(LlvmLibcAsinhfTest, SpecialNumbers) {
   libc_errno = 0;
 
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::asinhf(aNaN));
diff --git a/libc/test/src/math/smoke/atanf_test.cpp b/libc/test/src/math/smoke/atanf_test.cpp
index 6e6f854002..156ce0744b 100644
--- a/libc/test/src/math/smoke/atanf_test.cpp
+++ b/libc/test/src/math/smoke/atanf_test.cpp
@@ -16,11 +16,9 @@
 #include <errno.h>
 #include <stdint.h>
 
-using FPBits = LIBC_NAMESPACE::fputil::FPBits<float>;
+using LlvmLibcAtanfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
-DECLARE_SPECIAL_CONSTANTS(float)
-
-TEST(LlvmLibcAtanfTest, SpecialNumbers) {
+TEST_F(LlvmLibcAtanfTest, SpecialNumbers) {
   libc_errno = 0;
 
   LIBC_NAMESPACE::fputil::clear_except(FE_ALL_EXCEPT);
diff --git a/libc/test/src/math/smoke/atanhf_test.cpp b/libc/test/src/math/smoke/atanhf_test.cpp
index 841cb884c3..0f7dc0d306 100644
--- a/libc/test/src/math/smoke/atanhf_test.cpp
+++ b/libc/test/src/math/smoke/atanhf_test.cpp
@@ -16,11 +16,9 @@
 #include <errno.h>
 #include <stdint.h>
 
-using FPBits = LIBC_NAMESPACE::fputil::FPBits<float>;
+using LlvmLibcAtanhfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
-DECLARE_SPECIAL_CONSTANTS(float)
-
-TEST(LlvmLibcAtanhfTest, SpecialNumbers) {
+TEST_F(LlvmLibcAtanhfTest, SpecialNumbers) {
   libc_errno = 0;
 
   LIBC_NAMESPACE::fputil::clear_except(FE_ALL_EXCEPT);
diff --git a/libc/test/src/math/smoke/cosf_test.cpp b/libc/test/src/math/smoke/cosf_test.cpp
index 6a2c6a5b25..c9efc79188 100644
--- a/libc/test/src/math/smoke/cosf_test.cpp
+++ b/libc/test/src/math/smoke/cosf_test.cpp
@@ -16,11 +16,9 @@
 #include <errno.h>
 #include <stdint.h>
 
-using FPBits = LIBC_NAMESPACE::fputil::FPBits<float>;
+using LlvmLibcCosfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
-DECLARE_SPECIAL_CONSTANTS(float)
-
-TEST(LlvmLibcCosfTest, SpecialNumbers) {
+TEST_F(LlvmLibcCosfTest, SpecialNumbers) {
   libc_errno = 0;
 
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::cosf(aNaN));
diff --git a/libc/test/src/math/smoke/coshf_test.cpp b/libc/test/src/math/smoke/coshf_test.cpp
index dd08f1924a..93fdd12689 100644
--- a/libc/test/src/math/smoke/coshf_test.cpp
+++ b/libc/test/src/math/smoke/coshf_test.cpp
@@ -17,11 +17,9 @@
 #include <errno.h>
 #include <stdint.h>
 
-using FPBits = LIBC_NAMESPACE::fputil::FPBits<float>;
+using LlvmLibcCoshfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
-DECLARE_SPECIAL_CONSTANTS(float)
-
-TEST(LlvmLibcCoshfTest, SpecialNumbers) {
+TEST_F(LlvmLibcCoshfTest, SpecialNumbers) {
   libc_errno = 0;
 
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::coshf(aNaN));
@@ -40,7 +38,7 @@ TEST(LlvmLibcCoshfTest, SpecialNumbers) {
   EXPECT_MATH_ERRNO(0);
 }
 
-TEST(LlvmLibcCoshfTest, Overflow) {
+TEST_F(LlvmLibcCoshfTest, Overflow) {
   libc_errno = 0;
   EXPECT_FP_EQ_WITH_EXCEPTION(
       inf, LIBC_NAMESPACE::coshf(float(FPBits(0x7f7fffffU))), FE_OVERFLOW);
diff --git a/libc/test/src/math/smoke/erff_test.cpp b/libc/test/src/math/smoke/erff_test.cpp
index 9c9105167b..843cc7f4c8 100644
--- a/libc/test/src/math/smoke/erff_test.cpp
+++ b/libc/test/src/math/smoke/erff_test.cpp
@@ -15,11 +15,9 @@
 #include <errno.h>
 #include <stdint.h>
 
-using LIBC_NAMESPACE::testing::tlog;
+using LlvmLibcErffTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
-DECLARE_SPECIAL_CONSTANTS(float)
-
-TEST(LlvmLibcErffTest, SpecialNumbers) {
+TEST_F(LlvmLibcErffTest, SpecialNumbers) {
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::erff(aNaN));
   EXPECT_FP_EQ_ALL_ROUNDING(1.0f, LIBC_NAMESPACE::erff(inf));
   EXPECT_FP_EQ_ALL_ROUNDING(-1.0f, LIBC_NAMESPACE::erff(neg_inf));
diff --git a/libc/test/src/math/smoke/exp10_test.cpp b/libc/test/src/math/smoke/exp10_test.cpp
index cf699bab6e..daa7b5b488 100644
--- a/libc/test/src/math/smoke/exp10_test.cpp
+++ b/libc/test/src/math/smoke/exp10_test.cpp
@@ -16,11 +16,9 @@
 #include <errno.h>
 #include <stdint.h>
 
-using LIBC_NAMESPACE::testing::tlog;
+using LlvmLibcExp10Test = LIBC_NAMESPACE::testing::FPTest<double>;
 
-DECLARE_SPECIAL_CONSTANTS(double)
-
-TEST(LlvmLibcExp10Test, SpecialNumbers) {
+TEST_F(LlvmLibcExp10Test, SpecialNumbers) {
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::exp10(aNaN));
   EXPECT_FP_EQ(inf, LIBC_NAMESPACE::exp10(inf));
   EXPECT_FP_EQ_ALL_ROUNDING(zero, LIBC_NAMESPACE::exp10(neg_inf));
diff --git a/libc/test/src/math/smoke/exp10f_test.cpp b/libc/test/src/math/smoke/exp10f_test.cpp
index c4d3cb6002..8b758dd3f6 100644
--- a/libc/test/src/math/smoke/exp10f_test.cpp
+++ b/libc/test/src/math/smoke/exp10f_test.cpp
@@ -15,9 +15,9 @@
 
 #include <stdint.h>
 
-DECLARE_SPECIAL_CONSTANTS(float)
+using LlvmLibcExp10fTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
-TEST(LlvmLibcExp10fTest, SpecialNumbers) {
+TEST_F(LlvmLibcExp10fTest, SpecialNumbers) {
   libc_errno = 0;
 
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::exp10f(aNaN));
@@ -40,7 +40,7 @@ TEST(LlvmLibcExp10fTest, SpecialNumbers) {
   EXPECT_FP_EQ_ALL_ROUNDING(1000.0f, LIBC_NAMESPACE::exp10f(3.0f));
 }
 
-TEST(LlvmLibcExp10fTest, Overflow) {
+TEST_F(LlvmLibcExp10fTest, Overflow) {
   libc_errno = 0;
   EXPECT_FP_EQ_WITH_EXCEPTION(
       inf, LIBC_NAMESPACE::exp10f(float(FPBits(0x7f7fffffU))), FE_OVERFLOW);
diff --git a/libc/test/src/math/smoke/exp2_test.cpp b/libc/test/src/math/smoke/exp2_test.cpp
index ec076c15fe..7bcc2a3cb6 100644
--- a/libc/test/src/math/smoke/exp2_test.cpp
+++ b/libc/test/src/math/smoke/exp2_test.cpp
@@ -16,11 +16,9 @@
 #include <errno.h>
 #include <stdint.h>
 
-using LIBC_NAMESPACE::testing::tlog;
+using LlvmLibcExp2Test = LIBC_NAMESPACE::testing::FPTest<double>;
 
-DECLARE_SPECIAL_CONSTANTS(double)
-
-TEST(LlvmLibcExp2Test, SpecialNumbers) {
+TEST_F(LlvmLibcExp2Test, SpecialNumbers) {
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::exp2(aNaN));
   EXPECT_FP_EQ(inf, LIBC_NAMESPACE::exp2(inf));
   EXPECT_FP_EQ_ALL_ROUNDING(zero, LIBC_NAMESPACE::exp2(neg_inf));
diff --git a/libc/test/src/math/smoke/exp2f_test.cpp b/libc/test/src/math/smoke/exp2f_test.cpp
index f26f155cc8..6623d449ae 100644
--- a/libc/test/src/math/smoke/exp2f_test.cpp
+++ b/libc/test/src/math/smoke/exp2f_test.cpp
@@ -16,9 +16,9 @@
 
 #include <stdint.h>
 
-DECLARE_SPECIAL_CONSTANTS(float)
+using LlvmLibcExp2fTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
-TEST(LlvmLibcExp2fTest, SpecialNumbers) {
+TEST_F(LlvmLibcExp2fTest, SpecialNumbers) {
   libc_errno = 0;
 
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::exp2f(aNaN));
@@ -42,7 +42,7 @@ TEST(LlvmLibcExp2fTest, SpecialNumbers) {
   EXPECT_FP_EQ_ALL_ROUNDING(0.25f, LIBC_NAMESPACE::exp2f(-2.0f));
 }
 
-TEST(LlvmLibcExp2fTest, Overflow) {
+TEST_F(LlvmLibcExp2fTest, Overflow) {
   libc_errno = 0;
   EXPECT_FP_EQ_WITH_EXCEPTION(
       inf, LIBC_NAMESPACE::exp2f(float(FPBits(0x7f7fffffU))), FE_OVERFLOW);
diff --git a/libc/test/src/math/smoke/exp_test.cpp b/libc/test/src/math/smoke/exp_test.cpp
index 7973c37257..455cb2ea13 100644
--- a/libc/test/src/math/smoke/exp_test.cpp
+++ b/libc/test/src/math/smoke/exp_test.cpp
@@ -16,11 +16,9 @@
 #include <errno.h>
 #include <stdint.h>
 
-using LIBC_NAMESPACE::testing::tlog;
+using LlvmLibcExpTest = LIBC_NAMESPACE::testing::FPTest<double>;
 
-DECLARE_SPECIAL_CONSTANTS(double)
-
-TEST(LlvmLibcExpTest, SpecialNumbers) {
+TEST_F(LlvmLibcExpTest, SpecialNumbers) {
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::exp(aNaN));
   EXPECT_FP_EQ(inf, LIBC_NAMESPACE::exp(inf));
   EXPECT_FP_EQ_ALL_ROUNDING(zero, LIBC_NAMESPACE::exp(neg_inf));
diff --git a/libc/test/src/math/smoke/expf_test.cpp b/libc/test/src/math/smoke/expf_test.cpp
index e5c5afbd18..ff33e95f90 100644
--- a/libc/test/src/math/smoke/expf_test.cpp
+++ b/libc/test/src/math/smoke/expf_test.cpp
@@ -15,9 +15,9 @@
 
 #include <stdint.h>
 
-DECLARE_SPECIAL_CONSTANTS(float)
+using LlvmLibcExpfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
-TEST(LlvmLibcExpfTest, SpecialNumbers) {
+TEST_F(LlvmLibcExpfTest, SpecialNumbers) {
   libc_errno = 0;
 
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::expf(aNaN));
@@ -36,7 +36,7 @@ TEST(LlvmLibcExpfTest, SpecialNumbers) {
   EXPECT_MATH_ERRNO(0);
 }
 
-TEST(LlvmLibcExpfTest, Overflow) {
+TEST_F(LlvmLibcExpfTest, Overflow) {
   libc_errno = 0;
   EXPECT_FP_EQ_WITH_EXCEPTION(
       inf, LIBC_NAMESPACE::expf(float(FPBits(0x7f7fffffU))), FE_OVERFLOW);
diff --git a/libc/test/src/math/smoke/expm1_test.cpp b/libc/test/src/math/smoke/expm1_test.cpp
index 8bb4a11ed6..4581060075 100644
--- a/libc/test/src/math/smoke/expm1_test.cpp
+++ b/libc/test/src/math/smoke/expm1_test.cpp
@@ -11,18 +11,14 @@
 #include "src/math/expm1.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
-#include "utils/MPFRWrapper/MPFRUtils.h"
 #include <math.h>
 
 #include <errno.h>
 #include <stdint.h>
 
-namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
-using LIBC_NAMESPACE::testing::tlog;
+using LlvmLibcExpm1Test = LIBC_NAMESPACE::testing::FPTest<double>;
 
-DECLARE_SPECIAL_CONSTANTS(double)
-
-TEST(LlvmLibcExpm1Test, SpecialNumbers) {
+TEST_F(LlvmLibcExpm1Test, SpecialNumbers) {
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::expm1(aNaN));
   EXPECT_FP_EQ(inf, LIBC_NAMESPACE::expm1(inf));
   EXPECT_FP_EQ_ALL_ROUNDING(-1.0, LIBC_NAMESPACE::expm1(neg_inf));
diff --git a/libc/test/src/math/smoke/expm1f_test.cpp b/libc/test/src/math/smoke/expm1f_test.cpp
index 03fe1413e9..6999a987f9 100644
--- a/libc/test/src/math/smoke/expm1f_test.cpp
+++ b/libc/test/src/math/smoke/expm1f_test.cpp
@@ -15,9 +15,9 @@
 
 #include <stdint.h>
 
-DECLARE_SPECIAL_CONSTANTS(float)
+using LlvmLibcExpm1fTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
-TEST(LlvmLibcExpm1fTest, SpecialNumbers) {
+TEST_F(LlvmLibcExpm1fTest, SpecialNumbers) {
   libc_errno = 0;
 
   EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::expm1f(aNaN));
@@ -36,7 +36,7 @@ TEST(LlvmLibcExpm1fTest, SpecialNumbers) {
   EXPECT_MATH_ERRNO(0);
 }
 
-TEST(LlvmLibcExpm1fTest, Overflow) {
+TEST_F(LlvmLibcExpm1fTest, Overflow) {
   libc_errno = 0;
   EXPECT_FP_EQ_WITH_EXCEPTION(
       inf, LIBC_NAMESPACE::expm1f(float(FPBits(0x7f7fffffU))), FE_OVERFLOW);
diff --git a/libc/test/src/math/smoke/log10_test.cpp b/libc/test/src/math/smoke/log10_test.cpp
index be4900a6a4..c1658a3d5c 100644
--- a/libc/test/src/math/smoke/log10_test.cpp
+++ b/libc/test/src/math/smoke/log10_test.cpp
@@ -16,11 +16,9 @@
 #include <errno.h>
 #include <stdint.h>
 
-using LIBC_NAMESPACE::testing::tlog;
+using LlvmLibcLog10Test = LIBC_NAMESPACE::testing::FPTest<double>;
 
-DECLARE_SPECIAL_CONSTANTS(double)
-
-TEST(LlvmLibcLog10Test, SpecialNumbers) {
+TEST_F(LlvmLibcLog10Test, SpecialNumbers) {
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::log10(aNaN));
   EXPECT_FP_EQ(inf, LIBC_NAMESPACE::log10(inf));
   EXPECT_FP_IS_NAN_WITH_EXCEPTION(LIBC_NAMESPACE::log10(neg_inf), FE_INVALID);
diff --git a/libc/test/src/math/smoke/log10f_test.cpp b/libc/test/src/math/smoke/log10f_test.cpp
index 2369ab0458..53950233ca 100644
--- a/libc/test/src/math/smoke/log10f_test.cpp
+++ b/libc/test/src/math/smoke/log10f_test.cpp
@@ -15,9 +15,9 @@
 #include <errno.h>
 #include <stdint.h>
 
-DECLARE_SPECIAL_CONSTANTS(float)
+using LlvmLibcLog10fTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
-TEST(LlvmLibcLog10fTest, SpecialNumbers) {
+TEST_F(LlvmLibcLog10fTest, SpecialNumbers) {
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::log10f(aNaN));
   EXPECT_FP_EQ(inf, LIBC_NAMESPACE::log10f(inf));
   EXPECT_FP_IS_NAN_WITH_EXCEPTION(LIBC_NAMESPACE::log10f(neg_inf), FE_INVALID);
diff --git a/libc/test/src/math/smoke/log1p_test.cpp b/libc/test/src/math/smoke/log1p_test.cpp
index 9cb0cf17ff..e1966c2839 100644
--- a/libc/test/src/math/smoke/log1p_test.cpp
+++ b/libc/test/src/math/smoke/log1p_test.cpp
@@ -16,11 +16,9 @@
 #include <errno.h>
 #include <stdint.h>
 
-using LIBC_NAMESPACE::testing::tlog;
+using LlvmLibcLog1pTest = LIBC_NAMESPACE::testing::FPTest<double>;
 
-DECLARE_SPECIAL_CONSTANTS(double)
-
-TEST(LlvmLibcLog1pTest, SpecialNumbers) {
+TEST_F(LlvmLibcLog1pTest, SpecialNumbers) {
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::log1p(aNaN));
   EXPECT_FP_EQ(inf, LIBC_NAMESPACE::log1p(inf));
   EXPECT_FP_IS_NAN_WITH_EXCEPTION(LIBC_NAMESPACE::log1p(neg_inf), FE_INVALID);
diff --git a/libc/test/src/math/smoke/log1pf_test.cpp b/libc/test/src/math/smoke/log1pf_test.cpp
index 9836010243..377a46adef 100644
--- a/libc/test/src/math/smoke/log1pf_test.cpp
+++ b/libc/test/src/math/smoke/log1pf_test.cpp
@@ -16,9 +16,9 @@
 #include <errno.h>
 #include <stdint.h>
 
-DECLARE_SPECIAL_CONSTANTS(float)
+using LlvmLibcLog1pfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
-TEST(LlvmLibclog1pfTest, SpecialNumbers) {
+TEST_F(LlvmLibcLog1pfTest, SpecialNumbers) {
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::log1pf(aNaN));
   EXPECT_FP_EQ(inf, LIBC_NAMESPACE::log1pf(inf));
   EXPECT_FP_IS_NAN_WITH_EXCEPTION(LIBC_NAMESPACE::log1pf(neg_inf), FE_INVALID);
diff --git a/libc/test/src/math/smoke/log2_test.cpp b/libc/test/src/math/smoke/log2_test.cpp
index dab8929115..826c51eb8c 100644
--- a/libc/test/src/math/smoke/log2_test.cpp
+++ b/libc/test/src/math/smoke/log2_test.cpp
@@ -16,11 +16,9 @@
 #include <errno.h>
 #include <stdint.h>
 
-using LIBC_NAMESPACE::testing::tlog;
+using LlvmLibcLog2Test = LIBC_NAMESPACE::testing::FPTest<double>;
 
-DECLARE_SPECIAL_CONSTANTS(double)
-
-TEST(LlvmLibcLog2Test, SpecialNumbers) {
+TEST_F(LlvmLibcLog2Test, SpecialNumbers) {
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::log2(aNaN));
   EXPECT_FP_EQ(inf, LIBC_NAMESPACE::log2(inf));
   EXPECT_FP_IS_NAN_WITH_EXCEPTION(LIBC_NAMESPACE::log2(neg_inf), FE_INVALID);
diff --git a/libc/test/src/math/smoke/log2f_test.cpp b/libc/test/src/math/smoke/log2f_test.cpp
index d388230a80..2387ff59d7 100644
--- a/libc/test/src/math/smoke/log2f_test.cpp
+++ b/libc/test/src/math/smoke/log2f_test.cpp
@@ -15,9 +15,9 @@
 
 #include <stdint.h>
 
-DECLARE_SPECIAL_CONSTANTS(float)
+using LlvmLibcLog2fTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
-TEST(LlvmLibcLog2fTest, SpecialNumbers) {
+TEST_F(LlvmLibcLog2fTest, SpecialNumbers) {
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::log2f(aNaN));
   EXPECT_FP_EQ(inf, LIBC_NAMESPACE::log2f(inf));
   EXPECT_FP_IS_NAN_WITH_EXCEPTION(LIBC_NAMESPACE::log2f(neg_inf), FE_INVALID);
diff --git a/libc/test/src/math/smoke/log_test.cpp b/libc/test/src/math/smoke/log_test.cpp
index 34d608a81a..423b9d8bb8 100644
--- a/libc/test/src/math/smoke/log_test.cpp
+++ b/libc/test/src/math/smoke/log_test.cpp
@@ -16,11 +16,9 @@
 #include <errno.h>
 #include <stdint.h>
 
-using LIBC_NAMESPACE::testing::tlog;
+using LlvmLibcLogTest = LIBC_NAMESPACE::testing::FPTest<double>;
 
-DECLARE_SPECIAL_CONSTANTS(double)
-
-TEST(LlvmLibcLogTest, SpecialNumbers) {
+TEST_F(LlvmLibcLogTest, SpecialNumbers) {
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::log(aNaN));
   EXPECT_FP_EQ(inf, LIBC_NAMESPACE::log(inf));
   EXPECT_FP_IS_NAN_WITH_EXCEPTION(LIBC_NAMESPACE::log(neg_inf), FE_INVALID);
diff --git a/libc/test/src/math/smoke/logf_test.cpp b/libc/test/src/math/smoke/logf_test.cpp
index 0edaa8da55..21b39ee689 100644
--- a/libc/test/src/math/smoke/logf_test.cpp
+++ b/libc/test/src/math/smoke/logf_test.cpp
@@ -14,9 +14,9 @@
 
 #include <stdint.h>
 
-DECLARE_SPECIAL_CONSTANTS(float)
+using LlvmLibcLogfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
-TEST(LlvmLibcLogfTest, SpecialNumbers) {
+TEST_F(LlvmLibcLogfTest, SpecialNumbers) {
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::logf(aNaN));
   EXPECT_FP_EQ(inf, LIBC_NAMESPACE::logf(inf));
   EXPECT_FP_IS_NAN_WITH_EXCEPTION(LIBC_NAMESPACE::logf(neg_inf), FE_INVALID);
diff --git a/libc/test/src/math/smoke/sincosf_test.cpp b/libc/test/src/math/smoke/sincosf_test.cpp
index 4e2b1c69e4..0de47f3307 100644
--- a/libc/test/src/math/smoke/sincosf_test.cpp
+++ b/libc/test/src/math/smoke/sincosf_test.cpp
@@ -16,11 +16,9 @@
 #include <errno.h>
 #include <stdint.h>
 
-using FPBits = LIBC_NAMESPACE::fputil::FPBits<float>;
+using LlvmLibcSinCosfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
-DECLARE_SPECIAL_CONSTANTS(float)
-
-TEST(LlvmLibcSinCosfTest, SpecialNumbers) {
+TEST_F(LlvmLibcSinCosfTest, SpecialNumbers) {
   libc_errno = 0;
   float sin, cos;
 
diff --git a/libc/test/src/math/smoke/sinf_test.cpp b/libc/test/src/math/smoke/sinf_test.cpp
index b93011bbdc..bbd7634e00 100644
--- a/libc/test/src/math/smoke/sinf_test.cpp
+++ b/libc/test/src/math/smoke/sinf_test.cpp
@@ -16,11 +16,9 @@
 #include <errno.h>
 #include <stdint.h>
 
-using FPBits = LIBC_NAMESPACE::fputil::FPBits<float>;
+using LlvmLibcSinfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
-DECLARE_SPECIAL_CONSTANTS(float)
-
-TEST(LlvmLibcSinfTest, SpecialNumbers) {
+TEST_F(LlvmLibcSinfTest, SpecialNumbers) {
   libc_errno = 0;
 
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::sinf(aNaN));
diff --git a/libc/test/src/math/smoke/sinhf_test.cpp b/libc/test/src/math/smoke/sinhf_test.cpp
index b2b6bddf91..aa11ed9cbe 100644
--- a/libc/test/src/math/smoke/sinhf_test.cpp
+++ b/libc/test/src/math/smoke/sinhf_test.cpp
@@ -17,11 +17,9 @@
 #include <errno.h>
 #include <stdint.h>
 
-using FPBits = LIBC_NAMESPACE::fputil::FPBits<float>;
+using LlvmLibcSinhfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
-DECLARE_SPECIAL_CONSTANTS(float)
-
-TEST(LlvmLibcSinhfTest, SpecialNumbers) {
+TEST_F(LlvmLibcSinhfTest, SpecialNumbers) {
   libc_errno = 0;
 
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::sinhf(aNaN));
@@ -41,7 +39,7 @@ TEST(LlvmLibcSinhfTest, SpecialNumbers) {
 }
 
 // For small values, sinh(x) is x.
-TEST(LlvmLibcSinhfTest, SmallValues) {
+TEST_F(LlvmLibcSinhfTest, SmallValues) {
   float x = float(FPBits(uint32_t(0x17800000)));
   float result = LIBC_NAMESPACE::sinhf(x);
   EXPECT_FP_EQ(x, result);
@@ -51,7 +49,7 @@ TEST(LlvmLibcSinhfTest, SmallValues) {
   EXPECT_FP_EQ(x, result);
 }
 
-TEST(LlvmLibcSinhfTest, Overflow) {
+TEST_F(LlvmLibcSinhfTest, Overflow) {
   libc_errno = 0;
   EXPECT_FP_EQ_WITH_EXCEPTION(
       inf, LIBC_NAMESPACE::sinhf(float(FPBits(0x7f7fffffU))), FE_OVERFLOW);
diff --git a/libc/test/src/math/smoke/tanf_test.cpp b/libc/test/src/math/smoke/tanf_test.cpp
index 60444b3ad1..fa93da2982 100644
--- a/libc/test/src/math/smoke/tanf_test.cpp
+++ b/libc/test/src/math/smoke/tanf_test.cpp
@@ -16,11 +16,9 @@
 #include <errno.h>
 #include <stdint.h>
 
-using FPBits = LIBC_NAMESPACE::fputil::FPBits<float>;
+using LlvmLibcTanfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
-DECLARE_SPECIAL_CONSTANTS(float)
-
-TEST(LlvmLibcTanfTest, SpecialNumbers) {
+TEST_F(LlvmLibcTanfTest, SpecialNumbers) {
   libc_errno = 0;
 
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::tanf(aNaN));
diff --git a/libc/test/src/math/smoke/tanhf_test.cpp b/libc/test/src/math/smoke/tanhf_test.cpp
index db20be724b..f32e8cc959 100644
--- a/libc/test/src/math/smoke/tanhf_test.cpp
+++ b/libc/test/src/math/smoke/tanhf_test.cpp
@@ -16,11 +16,9 @@
 #include <errno.h>
 #include <stdint.h>
 
-using FPBits = LIBC_NAMESPACE::fputil::FPBits<float>;
+using LlvmLibcTanhfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
-DECLARE_SPECIAL_CONSTANTS(float)
-
-TEST(LlvmLibcTanhfTest, SpecialNumbers) {
+TEST_F(LlvmLibcTanhfTest, SpecialNumbers) {
   libc_errno = 0;
 
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::tanhf(aNaN));
diff --git a/libc/test/src/math/tan_test.cpp b/libc/test/src/math/tan_test.cpp
index a6f7dcb432..b35658cc50 100644
--- a/libc/test/src/math/tan_test.cpp
+++ b/libc/test/src/math/tan_test.cpp
@@ -13,11 +13,11 @@
 
 #include <math.h>
 
-namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
+using LlvmLibcTanTest = LIBC_NAMESPACE::testing::FPTest<double>;
 
-DECLARE_SPECIAL_CONSTANTS(double)
+namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
-TEST(LlvmLibctanTest, Range) {
+TEST_F(LlvmLibcTanTest, Range) {
   static constexpr double _2pi = 6.283185307179586;
   constexpr UIntType COUNT = 100'000;
   constexpr UIntType STEP = UIntType(-1) / COUNT;
diff --git a/libc/test/src/math/tanf_test.cpp b/libc/test/src/math/tanf_test.cpp
index 3c3694a5eb..6005202e75 100644
--- a/libc/test/src/math/tanf_test.cpp
+++ b/libc/test/src/math/tanf_test.cpp
@@ -18,14 +18,13 @@
 #include <errno.h>
 #include <stdint.h>
 
+using LlvmLibcTanfTest = LIBC_NAMESPACE::testing::FPTest<float>;
+
 using LIBC_NAMESPACE::testing::SDCOMP26094_VALUES;
-using FPBits = LIBC_NAMESPACE::fputil::FPBits<float>;
 
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
-DECLARE_SPECIAL_CONSTANTS(float)
-
-TEST(LlvmLibcTanfTest, SpecialNumbers) {
+TEST_F(LlvmLibcTanfTest, SpecialNumbers) {
   libc_errno = 0;
 
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::tanf(aNaN));
@@ -44,7 +43,7 @@ TEST(LlvmLibcTanfTest, SpecialNumbers) {
   EXPECT_MATH_ERRNO(EDOM);
 }
 
-TEST(LlvmLibcTanfTest, InFloatRange) {
+TEST_F(LlvmLibcTanfTest, InFloatRange) {
   constexpr uint32_t COUNT = 100'000;
   constexpr uint32_t STEP = UINT32_MAX / COUNT;
   for (uint32_t i = 0, v = 0; i <= COUNT; ++i, v += STEP) {
@@ -56,7 +55,7 @@ TEST(LlvmLibcTanfTest, InFloatRange) {
   }
 }
 
-TEST(LlvmLibcTanfTest, SpecificBitPatterns) {
+TEST_F(LlvmLibcTanfTest, SpecificBitPatterns) {
   constexpr int N = 54;
   constexpr uint32_t INPUTS[N] = {
       0x3a7a'8d2fU, // x = 0x1.f51a5ep-11f
@@ -126,7 +125,7 @@ TEST(LlvmLibcTanfTest, SpecificBitPatterns) {
 
 // SDCOMP-26094: check tanf in the cases for which the range reducer
 // returns values furthest beyond its nominal upper bound of pi/4.
-TEST(LlvmLibcTanfTest, SDCOMP_26094) {
+TEST_F(LlvmLibcTanfTest, SDCOMP_26094) {
   for (uint32_t v : SDCOMP26094_VALUES) {
     float x = float(FPBits(v));
     ASSERT_MPFR_MATCH(mpfr::Operation::Tan, x, LIBC_NAMESPACE::tanf(x), 0.5);
diff --git a/libc/test/src/math/tanhf_test.cpp b/libc/test/src/math/tanhf_test.cpp
index 06ad02c337..1dc736d369 100644
--- a/libc/test/src/math/tanhf_test.cpp
+++ b/libc/test/src/math/tanhf_test.cpp
@@ -17,13 +17,11 @@
 #include <errno.h>
 #include <stdint.h>
 
-using FPBits = LIBC_NAMESPACE::fputil::FPBits<float>;
+using LlvmLibcTanhfTest = LIBC_NAMESPACE::testing::FPTest<float>;
 
 namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
 
-DECLARE_SPECIAL_CONSTANTS(float)
-
-TEST(LlvmLibcTanhfTest, SpecialNumbers) {
+TEST_F(LlvmLibcTanhfTest, SpecialNumbers) {
   libc_errno = 0;
 
   EXPECT_FP_EQ(aNaN, LIBC_NAMESPACE::tanhf(aNaN));
@@ -42,7 +40,7 @@ TEST(LlvmLibcTanhfTest, SpecialNumbers) {
   EXPECT_MATH_ERRNO(0);
 }
 
-TEST(LlvmLibcTanhfTest, InFloatRange) {
+TEST_F(LlvmLibcTanhfTest, InFloatRange) {
   constexpr uint32_t COUNT = 100'001;
   constexpr uint32_t STEP = UINT32_MAX / COUNT;
   for (uint32_t i = 0, v = 0; i <= COUNT; ++i, v += STEP) {
@@ -54,7 +52,7 @@ TEST(LlvmLibcTanhfTest, InFloatRange) {
   }
 }
 
-TEST(LlvmLibcTanhfTest, ExceptionalValues) {
+TEST_F(LlvmLibcTanhfTest, ExceptionalValues) {
   constexpr int N = 4;
   constexpr uint32_t INPUTS[N] = {
       0x0040'0000,
diff --git a/libc/test/src/stdio/sprintf_test.cpp b/libc/test/src/stdio/sprintf_test.cpp
index b7e8b75485..f3d5dd698c 100644
--- a/libc/test/src/stdio/sprintf_test.cpp
+++ b/libc/test/src/stdio/sprintf_test.cpp
@@ -748,6 +748,9 @@ TEST_F(LlvmLibcSPrintfTest, FloatHexExpConv) {
   written = LIBC_NAMESPACE::sprintf(buff, "%.5a", nan);
   ASSERT_STREQ_LEN(written, buff, "nan");
 
+  written = LIBC_NAMESPACE::sprintf(buff, "%La", 0.0L);
+  ASSERT_STREQ_LEN(written, buff, "0x0p+0");
+
   written = LIBC_NAMESPACE::sprintf(buff, "%.1La", 0.1L);
 #if defined(SPECIAL_X86_LONG_DOUBLE)
   ASSERT_STREQ_LEN(written, buff, "0xc.dp-7");
diff --git a/libc/test/src/stdio/ungetc_test.cpp b/libc/test/src/stdio/ungetc_test.cpp
index 75eecc87ef..c98995ff08 100644
--- a/libc/test/src/stdio/ungetc_test.cpp
+++ b/libc/test/src/stdio/ungetc_test.cpp
@@ -24,12 +24,16 @@ TEST(LlvmLibcUngetcTest, UngetAndReadBack) {
   constexpr size_t CONTENT_SIZE = sizeof(CONTENT);
   ASSERT_EQ(CONTENT_SIZE,
             LIBC_NAMESPACE::fwrite(CONTENT, 1, CONTENT_SIZE, file));
+#ifndef LIBC_TARGET_ARCH_IS_GPU // Behavior varies between libc implementations.
   // Cannot unget to an un-readable file.
   ASSERT_EQ(EOF, LIBC_NAMESPACE::ungetc('1', file));
+#endif
   ASSERT_EQ(0, LIBC_NAMESPACE::fclose(file));
 
   file = LIBC_NAMESPACE::fopen(FILENAME, "r+");
   ASSERT_FALSE(file == nullptr);
+  // Calling with an EOF should always return EOF without doing anything.
+  ASSERT_EQ(EOF, LIBC_NAMESPACE::ungetc(EOF, file));
   char c;
   ASSERT_EQ(LIBC_NAMESPACE::fread(&c, 1, 1, file), size_t(1));
   ASSERT_EQ(c, CONTENT[0]);
@@ -43,8 +47,10 @@ TEST(LlvmLibcUngetcTest, UngetAndReadBack) {
   // ungetc should not fail after a seek operation.
   int unget_char = 'z';
   ASSERT_EQ(unget_char, LIBC_NAMESPACE::ungetc(unget_char, file));
+#ifndef LIBC_TARGET_ARCH_IS_GPU // Behavior varies between libc implementations.
   // Another unget should fail.
   ASSERT_EQ(EOF, LIBC_NAMESPACE::ungetc(unget_char, file));
+#endif
   // ungetting a char at the beginning of the file will allow us to fetch
   // one additional character.
   char new_data[CONTENT_SIZE + 1];
@@ -53,8 +59,10 @@ TEST(LlvmLibcUngetcTest, UngetAndReadBack) {
   ASSERT_STREQ("zabcdef", new_data);
 
   ASSERT_EQ(size_t(1), LIBC_NAMESPACE::fwrite("x", 1, 1, file));
+#ifndef LIBC_TARGET_ARCH_IS_GPU // Behavior varies between libc implementations.
   // unget should fail after a write operation.
   ASSERT_EQ(EOF, LIBC_NAMESPACE::ungetc('1', file));
+#endif
 
   ASSERT_EQ(0, LIBC_NAMESPACE::fclose(file));
 }
diff --git a/libc/test/src/stdlib/rand_test.cpp b/libc/test/src/stdlib/rand_test.cpp
index 6f25708e53..7934dc16aa 100644
--- a/libc/test/src/stdlib/rand_test.cpp
+++ b/libc/test/src/stdlib/rand_test.cpp
@@ -23,12 +23,15 @@ TEST(LlvmLibcRandTest, UnsetSeed) {
     vals[i] = val;
   }
 
+  // FIXME: The GPU implementation cannot initialize the seed correctly.
+#ifndef LIBC_TARGET_ARCH_IS_GPU
   // The C standard specifies that if 'srand' is never called it should behave
   // as if 'srand' was called with a value of 1. If we seed the value with 1 we
   // should get the same sequence as the unseeded version.
   LIBC_NAMESPACE::srand(1);
   for (size_t i = 0; i < 1000; ++i)
     ASSERT_EQ(LIBC_NAMESPACE::rand(), vals[i]);
+#endif
 }
 
 TEST(LlvmLibcRandTest, SetSeed) {
diff --git a/libc/test/src/time/CMakeLists.txt b/libc/test/src/time/CMakeLists.txt
index fd8f976264..10b63ce6f3 100644
--- a/libc/test/src/time/CMakeLists.txt
+++ b/libc/test/src/time/CMakeLists.txt
@@ -56,16 +56,9 @@ add_libc_unittest(
     libc_time_unittests
   SRCS
     gettimeofday_test.cpp
-  HDRS
-    TmHelper.h
-    TmMatcher.h
-  CXX_STANDARD
-    20
   DEPENDS
     libc.include.time
     libc.src.time.gettimeofday
-    libc.src.time.nanosleep
-    libc.src.errno.errno
 )
 
 add_libc_unittest(
diff --git a/libc/test/src/time/gettimeofday_test.cpp b/libc/test/src/time/gettimeofday_test.cpp
index 44250787c5..ee934b7f3a 100644
--- a/libc/test/src/time/gettimeofday_test.cpp
+++ b/libc/test/src/time/gettimeofday_test.cpp
@@ -9,33 +9,10 @@
 #include <time.h>
 
 #include "src/time/gettimeofday.h"
-#include "src/time/nanosleep.h"
-#include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
-namespace cpp = LIBC_NAMESPACE::cpp;
-
 TEST(LlvmLibcGettimeofday, SmokeTest) {
-  using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds;
-  void *tz = nullptr;
-  struct timeval tv;
-
-  int sleep_times[2] = {200, 1000};
-  for (int i = 0; i < 2; i++) {
-    int ret = LIBC_NAMESPACE::gettimeofday(&tv, tz);
-    ASSERT_EQ(ret, 0);
-
-    int sleep_time = sleep_times[i];
-    // Sleep for {sleep_time} microsceconds.
-    struct timespec tim = {0, sleep_time * 1000};
-    struct timespec tim2 = {0, 0};
-    ret = LIBC_NAMESPACE::nanosleep(&tim, &tim2);
-
-    // Call gettimeofday again and verify that it is more {sleep_time}
-    // microscecods.
-    struct timeval tv1;
-    ret = LIBC_NAMESPACE::gettimeofday(&tv1, tz);
-    ASSERT_EQ(ret, 0);
-    ASSERT_GE(tv1.tv_usec - tv.tv_usec, sleep_time);
-  }
+  timeval tv;
+  int ret = LIBC_NAMESPACE::gettimeofday(&tv, nullptr);
+  ASSERT_EQ(ret, 0);
 }
diff --git a/libc/test/src/unistd/CMakeLists.txt b/libc/test/src/unistd/CMakeLists.txt
index f2e2293e02..2fb11fed92 100644
--- a/libc/test/src/unistd/CMakeLists.txt
+++ b/libc/test/src/unistd/CMakeLists.txt
@@ -414,7 +414,7 @@ add_libc_unittest(
     libc.include.unistd
     libc.include.fcntl
     libc.include.sys_syscall
-    libc.src.unistd.LIBC_NAMESPACE_syscall
+    libc.src.unistd.__llvm_libc_syscall
     libc.test.errno_setter_matcher
 )
 
diff --git a/libc/test/src/unistd/syscall_test.cpp b/libc/test/src/unistd/syscall_test.cpp
index 4a53f674b8..211b27c318 100644
--- a/libc/test/src/unistd/syscall_test.cpp
+++ b/libc/test/src/unistd/syscall_test.cpp
@@ -24,7 +24,7 @@ using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds;
 // There is no function named "syscall" in llvm-libc, we instead use a macro to
 // set up the arguments properly. We still need to specify the namespace though
 // because the macro generates a call to the actual internal function
-// (LIBC_NAMESPACE_syscall) which is inside the namespace.
+// (__llvm_libc_syscall) which is inside the namespace.
 TEST(LlvmLibcSyscallTest, TrivialCall) {
   libc_errno = 0;
 
diff --git a/libc/utils/gpu/loader/amdgpu/Loader.cpp b/libc/utils/gpu/loader/amdgpu/Loader.cpp
index 1fb67e841c..1d0247a6dc 100644
--- a/libc/utils/gpu/loader/amdgpu/Loader.cpp
+++ b/libc/utils/gpu/loader/amdgpu/Loader.cpp
@@ -477,7 +477,7 @@ int load(int argc, char **argv, char **envp, void *image, size_t size,
   // If the clock_freq symbol is missing, no work to do.
   hsa_executable_symbol_t freq_sym;
   if (HSA_STATUS_SUCCESS ==
-      hsa_executable_get_symbol_by_name(executable, "LIBC_NAMESPACE_clock_freq",
+      hsa_executable_get_symbol_by_name(executable, "__llvm_libc_clock_freq",
                                         &dev_agent, &freq_sym)) {
 
     void *host_clock_freq;
diff --git a/libc/utils/gpu/server/rpc_server.cpp b/libc/utils/gpu/server/rpc_server.cpp
index 1c1c9f1ae9..05e900edc6 100644
--- a/libc/utils/gpu/server/rpc_server.cpp
+++ b/libc/utils/gpu/server/rpc_server.cpp
@@ -101,6 +101,22 @@ struct Server {
       });
       break;
     }
+    case RPC_READ_FGETS: {
+      uint64_t sizes[lane_size] = {0};
+      void *data[lane_size] = {nullptr};
+      port->recv([&](rpc::Buffer *buffer, uint32_t id) {
+        data[id] = new char[buffer->data[0]];
+        const char *str =
+            fgets(reinterpret_cast<char *>(data[id]), buffer->data[0],
+                  file::to_stream(buffer->data[1]));
+        sizes[id] = !str ? 0 : std::strlen(str) + 1;
+      });
+      port->send_n(data, sizes);
+      for (uint32_t id = 0; id < lane_size; ++id)
+        if (data[id])
+          delete[] reinterpret_cast<uint8_t *>(data[id]);
+      break;
+    }
     case RPC_OPEN_FILE: {
       uint64_t sizes[lane_size] = {0};
       void *paths[lane_size] = {nullptr};
@@ -186,6 +202,13 @@ struct Server {
       });
       break;
     }
+    case RPC_UNGETC: {
+      port->recv_and_send([](rpc::Buffer *buffer) {
+        buffer->data[0] = ungetc(static_cast<int>(buffer->data[0]),
+                                 file::to_stream(buffer->data[1]));
+      });
+      break;
+    }
     case RPC_NOOP: {
       port->recv([](rpc::Buffer *) {});
       break;
diff --git a/libcxx/CMakeLists.txt b/libcxx/CMakeLists.txt
index 16540caf68..d03421afde 100644
--- a/libcxx/CMakeLists.txt
+++ b/libcxx/CMakeLists.txt
@@ -642,18 +642,8 @@ get_sanitizer_flags(SANITIZER_FLAGS "${LLVM_USE_SANITIZER}")
 
 # Link system libraries =======================================================
 function(cxx_link_system_libraries target)
-
-# In order to remove just libc++ from the link step
-# we need to use -nostdlib++ whenever it is supported.
-# Unfortunately this cannot be used universally because for example g++ supports
-# only -nodefaultlibs in which case all libraries will be removed and
-# all libraries but c++ have to be added in manually.
-  if (CXX_SUPPORTS_NOSTDLIBXX_FLAG)
-    target_add_link_flags_if_supported(${target} PRIVATE "-nostdlib++")
-  else()
-    target_add_link_flags_if_supported(${target} PRIVATE "-nodefaultlibs")
-    target_add_compile_flags_if_supported(${target} PRIVATE "/Zl")
-    target_add_link_flags_if_supported(${target} PRIVATE "/nodefaultlib")
+  if (NOT MSVC)
+    target_link_libraries(${target} PRIVATE "-nostdlib++")
   endif()
 
   if (CXX_SUPPORTS_UNWINDLIB_EQ_NONE_FLAG AND LIBCXXABI_USE_LLVM_UNWINDER)
@@ -663,24 +653,6 @@ function(cxx_link_system_libraries target)
     target_add_link_flags_if_supported(${target} PRIVATE "--unwindlib=none")
   endif()
 
-  if (NOT APPLE) # On Apple platforms, we always use -nostdlib++ so we don't need to re-add other libraries
-    if (LIBCXX_HAS_PTHREAD_LIB)
-      target_link_libraries(${target} PRIVATE pthread)
-    endif()
-
-    if (LIBCXX_HAS_C_LIB)
-      target_link_libraries(${target} PRIVATE c)
-    endif()
-
-    if (LIBCXX_HAS_M_LIB)
-      target_link_libraries(${target} PRIVATE m)
-    endif()
-
-    if (LIBCXX_HAS_RT_LIB)
-      target_link_libraries(${target} PRIVATE rt)
-    endif()
-  endif()
-
   if (LIBCXX_USE_COMPILER_RT)
     find_compiler_rt_library(builtins LIBCXX_BUILTINS_LIBRARY)
     if (LIBCXX_BUILTINS_LIBRARY)
diff --git a/libcxx/benchmarks/CMakeLists.txt b/libcxx/benchmarks/CMakeLists.txt
index 1a4d634500..7591f34d93 100644
--- a/libcxx/benchmarks/CMakeLists.txt
+++ b/libcxx/benchmarks/CMakeLists.txt
@@ -122,7 +122,7 @@ endif()
 add_library(           cxx-benchmarks-flags-libcxx INTERFACE)
 target_link_libraries( cxx-benchmarks-flags-libcxx INTERFACE cxx-benchmarks-flags)
 target_compile_options(cxx-benchmarks-flags-libcxx INTERFACE ${SANITIZER_FLAGS} -Wno-user-defined-literals -Wno-suggest-override)
-target_link_options(   cxx-benchmarks-flags-libcxx INTERFACE -nodefaultlibs "-L${BENCHMARK_LIBCXX_INSTALL}/lib" "-L${BENCHMARK_LIBCXX_INSTALL}/lib64" ${SANITIZER_FLAGS})
+target_link_options(   cxx-benchmarks-flags-libcxx INTERFACE -nostdlib++ "-L${BENCHMARK_LIBCXX_INSTALL}/lib" "-L${BENCHMARK_LIBCXX_INSTALL}/lib64" ${SANITIZER_FLAGS})
 
 set(libcxx_benchmark_targets)
 
@@ -213,6 +213,7 @@ set(BENCHMARK_TESTS
     map.bench.cpp
     monotonic_buffer.bench.cpp
     ordered_set.bench.cpp
+    stop_token.bench.cpp
     std_format_spec_string_unicode.bench.cpp
     string.bench.cpp
     stringstream.bench.cpp
diff --git a/libcxx/benchmarks/stop_token.bench.cpp b/libcxx/benchmarks/stop_token.bench.cpp
new file mode 100644
index 0000000000..293d55ed82
--- /dev/null
+++ b/libcxx/benchmarks/stop_token.bench.cpp
@@ -0,0 +1,108 @@
+//===----------------------------------------------------------------------===//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <numeric>
+#include <stop_token>
+#include <thread>
+
+#include "benchmark/benchmark.h"
+#include "make_test_thread.h"
+
+using namespace std::chrono_literals;
+
+// We have a single thread created by std::jthread consuming the stop_token:
+// registering/deregistering callbacks, one at a time.
+void BM_stop_token_single_thread_reg_unreg_callback(benchmark::State& state) {
+  auto thread_func = [&](std::stop_token st, std::atomic<std::uint64_t>* reg_count) {
+    while (!st.stop_requested()) {
+      std::stop_callback cb{st, [&]() noexcept {}};
+      benchmark::DoNotOptimize(cb);
+      reg_count->fetch_add(1, std::memory_order_relaxed);
+    }
+  };
+
+  std::atomic<std::uint64_t> reg_count(0);
+  std::uint64_t total_reg_test_param = state.range(0);
+
+  auto thread = support::make_test_jthread(thread_func, &reg_count);
+
+  for (auto _ : state) {
+    auto start_total = reg_count.load(std::memory_order_relaxed);
+
+    while (reg_count.load(std::memory_order_relaxed) - start_total < total_reg_test_param) {
+      std::this_thread::yield();
+    }
+  }
+}
+BENCHMARK(BM_stop_token_single_thread_reg_unreg_callback)->RangeMultiplier(2)->Range(1 << 10, 1 << 24);
+
+// At startup, it creates a single stop_source which it will then pass an associated stop_token to every
+// request.
+//
+// Assume a thread-pool handles these requests and for each request it polls for stop_requested(), then attaches a
+// stop-callback, does some work, then detaches the stop-callback some time later. The lifetime of requests/callbacks
+// would overlap with other requests/callback from the same thread.
+//
+// Say something like each thread keeping a circular buffer of N stop-callbacks and destroying the stop-callbacks in
+// FIFO order
+void BM_stop_token_async_reg_unreg_callback(benchmark::State& state) {
+  struct dummy_stop_callback {
+    void operator()() const noexcept {}
+  };
+
+  constexpr size_t thread_count             = 20;
+  constexpr size_t concurrent_request_count = 1000;
+  std::atomic<bool> start{false};
+
+  std::uint64_t total_reg_test_param = state.range(0);
+
+  std::stop_source ss;
+  std::vector<std::jthread> threads;
+  threads.reserve(thread_count);
+  std::vector<std::atomic<std::uint64_t>> reg_counts(thread_count);
+
+  auto thread_func = [&start](std::atomic<std::uint64_t>* count, std::stop_token st) {
+    std::vector<std::optional<std::stop_callback<dummy_stop_callback>>> cbs(concurrent_request_count);
+
+    start.wait(false);
+
+    std::uint32_t index = 0;
+    while (!st.stop_requested()) {
+      cbs[index].emplace(st, dummy_stop_callback{});
+      index = (index + 1) % concurrent_request_count;
+      count->fetch_add(1, std::memory_order_relaxed);
+    }
+  };
+
+  for (size_t i = 0; i < thread_count; ++i) {
+    threads.emplace_back(support::make_test_jthread(thread_func, &reg_counts[i], ss.get_token()));
+  }
+
+  auto get_total_reg = [&] {
+    std::uint64_t total = 0;
+    for (const auto& reg_counts : reg_counts) {
+      total += reg_counts.load(std::memory_order_relaxed);
+    }
+    return total;
+  };
+
+  start = true;
+  start.notify_all();
+
+  for (auto _ : state) {
+    auto start_total = get_total_reg();
+
+    while (get_total_reg() - start_total < total_reg_test_param) {
+      std::this_thread::yield();
+    }
+  }
+
+  ss.request_stop();
+}
+BENCHMARK(BM_stop_token_async_reg_unreg_callback)->RangeMultiplier(2)->Range(1 << 10, 1 << 24);
+
+BENCHMARK_MAIN();
diff --git a/libcxx/cmake/caches/AndroidNDK.cmake b/libcxx/cmake/caches/AndroidNDK.cmake
new file mode 100644
index 0000000000..86c5219b8b
--- /dev/null
+++ b/libcxx/cmake/caches/AndroidNDK.cmake
@@ -0,0 +1,38 @@
+# Build libc++abi and libc++ closely resembling what is shipped in the Android
+# NDK.
+
+# The NDK names the libraries libc++_shared.so and libc++_static.a. Using the
+# libc++_shared.so soname ensures that the library doesn't interact with the
+# libc++.so in /system/lib[64].
+set(LIBCXX_SHARED_OUTPUT_NAME c++_shared CACHE STRING "")
+set(LIBCXX_STATIC_OUTPUT_NAME c++_static CACHE STRING "")
+
+# The NDK libc++ uses a special namespace to help isolate its symbols from those
+# in the platform's STL (e.g. /system/lib[64]/libc++.so, but possibly stlport on
+# older versions of Android).
+set(LIBCXX_ABI_VERSION 1 CACHE STRING "")
+set(LIBCXX_ABI_NAMESPACE __ndk1 CACHE STRING "")
+
+# CMake doesn't add a version suffix to an Android shared object filename,
+# (because CMAKE_PLATFORM_NO_VERSIONED_SONAME is set), so it writes both a
+# libc++_shared.so ELF file and a libc++_shared.so linker script to the same
+# output path (the script clobbers the binary). Turn off the linker script.
+set(LIBCXX_ENABLE_ABI_LINKER_SCRIPT OFF CACHE BOOL "")
+
+set(LIBCXX_STATICALLY_LINK_ABI_IN_SHARED_LIBRARY ON CACHE BOOL "")
+set(LIBCXXABI_ENABLE_SHARED OFF CACHE BOOL "")
+
+# Clang links libc++ by default, but it doesn't exist yet. The libc++ CMake
+# files specify -nostdlib++ to avoid this problem, but CMake's default "compiler
+# works" testing doesn't pass that flag, so force those tests to pass.
+set(CMAKE_C_COMPILER_WORKS ON CACHE BOOL "")
+set(CMAKE_CXX_COMPILER_WORKS ON CACHE BOOL "")
+
+# Use adb to push tests to a locally-connected device (e.g. emulator) and run
+# them.
+set(LIBCXX_TEST_CONFIG "llvm-libc++-android-ndk.cfg.in" CACHE STRING "")
+set(LIBCXXABI_TEST_CONFIG "llvm-libc++abi-android-ndk.cfg.in" CACHE STRING "")
+
+# CMAKE_SOURCE_DIR refers to the "<monorepo>/runtimes" directory.
+set(LIBCXX_EXECUTOR "${CMAKE_SOURCE_DIR}/../libcxx/utils/adb_run.py" CACHE STRING "")
+set(LIBCXXABI_EXECUTOR "${LIBCXX_EXECUTOR}" CACHE STRING "")
diff --git a/libcxx/cmake/config-ix.cmake b/libcxx/cmake/config-ix.cmake
index 9962d848d8..9fed861a4e 100644
--- a/libcxx/cmake/config-ix.cmake
+++ b/libcxx/cmake/config-ix.cmake
@@ -14,14 +14,6 @@ include(CheckCSourceCompiles)
 # link with --uwnindlib=none. Check if that option works.
 llvm_check_compiler_linker_flag(C "--unwindlib=none" CXX_SUPPORTS_UNWINDLIB_EQ_NONE_FLAG)
 
-if(WIN32 AND NOT MINGW)
-  # NOTE(compnerd) this is technically a lie, there is msvcrt, but for now, lets
-  # let the default linking take care of that.
-  set(LIBCXX_HAS_C_LIB NO)
-else()
-  check_library_exists(c fopen "" LIBCXX_HAS_C_LIB)
-endif()
-
 if (NOT LIBCXX_USE_COMPILER_RT)
   if(WIN32 AND NOT MINGW)
     set(LIBCXX_HAS_GCC_S_LIB NO)
@@ -54,9 +46,6 @@ else()
 endif()
 
 if (CXX_SUPPORTS_NOSTDLIBXX_FLAG OR C_SUPPORTS_NODEFAULTLIBS_FLAG)
-  if (LIBCXX_HAS_C_LIB)
-    list(APPEND CMAKE_REQUIRED_LIBRARIES c)
-  endif ()
   if (LIBCXX_USE_COMPILER_RT)
     include(HandleCompilerRT)
     find_compiler_rt_library(builtins LIBCXX_BUILTINS_LIBRARY
@@ -108,22 +97,18 @@ if(WIN32 AND NOT MINGW)
   # TODO(compnerd) do we want to support an emulation layer that allows for the
   # use of pthread-win32 or similar libraries to emulate pthreads on Windows?
   set(LIBCXX_HAS_PTHREAD_LIB NO)
-  set(LIBCXX_HAS_M_LIB NO)
   set(LIBCXX_HAS_RT_LIB NO)
   set(LIBCXX_HAS_ATOMIC_LIB NO)
 elseif(APPLE)
   set(LIBCXX_HAS_PTHREAD_LIB NO)
-  set(LIBCXX_HAS_M_LIB NO)
   set(LIBCXX_HAS_RT_LIB NO)
   set(LIBCXX_HAS_ATOMIC_LIB NO)
 elseif(FUCHSIA)
-  set(LIBCXX_HAS_M_LIB NO)
   set(LIBCXX_HAS_PTHREAD_LIB NO)
   set(LIBCXX_HAS_RT_LIB NO)
   check_library_exists(atomic __atomic_fetch_add_8 "" LIBCXX_HAS_ATOMIC_LIB)
 else()
   check_library_exists(pthread pthread_create "" LIBCXX_HAS_PTHREAD_LIB)
-  check_library_exists(m ccos "" LIBCXX_HAS_M_LIB)
   check_library_exists(rt clock_gettime "" LIBCXX_HAS_RT_LIB)
   check_library_exists(atomic __atomic_fetch_add_8 "" LIBCXX_HAS_ATOMIC_LIB)
 endif()
diff --git a/libcxx/docs/BuildingLibcxx.rst b/libcxx/docs/BuildingLibcxx.rst
index e919e1e6f3..2cee97c03c 100644
--- a/libcxx/docs/BuildingLibcxx.rst
+++ b/libcxx/docs/BuildingLibcxx.rst
@@ -145,9 +145,6 @@ should add e.g. ``-DCMAKE_CXX_COMPILER_TARGET=x86_64-windows-msvc`` (replacing
 line above. This will instruct ``check-cxx`` to use the right target triple
 when invoking ``clang++``.
 
-Also note that if not building in Release mode, a failed assert in the tests
-pops up a blocking dialog box, making it hard to run a larger number of tests.
-
 CMake + ninja (MinGW)
 ---------------------
 
@@ -160,19 +157,13 @@ e.g. the ``mingw-w64-x86_64-clang`` package), together with CMake and ninja.
   > cmake -G Ninja -S runtimes -B build                                               \
           -DCMAKE_C_COMPILER=clang                                                    \
           -DCMAKE_CXX_COMPILER=clang++                                                \
-          -DLLVM_ENABLE_RUNTIMES=libcxx                                               \
-          -DLIBCXX_CXX_ABI=libstdc++
+          -DLLVM_ENABLE_LLD=ON                                                        \
+          -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi"                                   \
+          -DLIBCXXABI_ENABLE_SHARED=OFF                                               \
+          -DLIBCXX_ENABLE_STATIC_ABI_LIBRARY=ON
   > ninja -C build cxx
-  > cp /mingw64/bin/{libstdc++-6,libgcc_s_seh-1,libwinpthread-1}.dll lib
   > ninja -C build check-cxx
 
-As this build configuration ends up depending on a couple other DLLs that
-aren't available in path while running tests, copy them into the same
-directory as the tested libc++ DLL.
-
-(Building a libc++ that depends on libstdc++ isn't necessarily a config one
-would want to deploy, but it simplifies the config for testing purposes.)
-
 .. _`libc++abi`: http://libcxxabi.llvm.org/
 
 
diff --git a/libcxx/docs/Contributing.rst b/libcxx/docs/Contributing.rst
index 5790c0c41f..ae88384185 100644
--- a/libcxx/docs/Contributing.rst
+++ b/libcxx/docs/Contributing.rst
@@ -8,22 +8,17 @@ This file contains notes about various tasks and processes specific to contribut
 to libc++. If this is your first time contributing, please also read `this document
 <https://www.llvm.org/docs/Contributing.html>`__ on general rules for contributing to LLVM.
 
-For libc++, please make sure you follow `these instructions <https://www.llvm.org/docs/Phabricator.html#requesting-a-review-via-the-command-line>`_
-for submitting a code review from the command-line using ``arc``, since we have some
-automation (e.g. CI) that depends on the review being submitted that way.
-
 If you plan on contributing to libc++, it can be useful to join the ``#libcxx`` channel
 on `LLVM's Discord server <https://discord.gg/jzUbyP26tQ>`__.
 
-Looking for pre-existing reviews
-================================
+Looking for pre-existing pull requests
+======================================
 
-Before you start working on any feature, please take a look at the open reviews
-to avoid duplicating someone else's work. You can do that by going to the website
-where code reviews are held, `Differential <https://reviews.llvm.org/differential>`__,
-and clicking on ``Libc++ Open Reviews`` in the sidebar to the left. If you see
-that your feature is already being worked on, please consider chiming in instead
-of duplicating work!
+Before you start working on any feature, please take a look at the open libc++ pull
+requests to avoid duplicating someone else's work. You can do that on GitHub by
+filtering pull requests `tagged with libc++ <https://github.com/llvm/llvm-project/pulls?q=is%3Apr+is%3Aopen+label%3Alibc%2B%2B>`__.
+If you see that your feature is already being worked on, please consider chiming in
+and helping review the code instead of duplicating work!
 
 RFCs for significant user-affecting changes
 ===========================================
@@ -212,14 +207,14 @@ Introduction
 
 Unlike most parts of the LLVM project, libc++ uses a pre-commit CI [#]_. This
 CI is hosted on `Buildkite <https://buildkite.com/llvm-project/libcxx-ci>`__ and
-the build results are visible in the review on Phabricator. Please make sure
+the build results are visible in the review on GitHub. Please make sure
 the CI is green before committing a patch.
 
 The CI tests libc++ for all :ref:`supported platforms <SupportedPlatforms>`.
-The build is started for every diff uploaded to Phabricator. A complete CI run
-takes approximately one hour. To reduce the load:
+The build is started for every commit added to a Pull Request. A complete CI
+run takes approximately one hour. To reduce the load:
 
-* The build is cancelled when a new diff for the same revision is uploaded.
+* The build is cancelled when a new commit is pushed to a PR that is already running CI.
 * The build is done in several stages and cancelled when a stage fails.
 
 Typically, the libc++ jobs use a Ubuntu Docker image. This image contains
diff --git a/libcxx/docs/FeatureTestMacroTable.rst b/libcxx/docs/FeatureTestMacroTable.rst
index 2b21ec3d2f..10afd64e0a 100644
--- a/libcxx/docs/FeatureTestMacroTable.rst
+++ b/libcxx/docs/FeatureTestMacroTable.rst
@@ -332,7 +332,7 @@ Status
     --------------------------------------------------- -----------------
     ``__cpp_lib_is_scoped_enum``                        ``202011L``
     --------------------------------------------------- -----------------
-    ``__cpp_lib_mdspan``                                *unimplemented*
+    ``__cpp_lib_mdspan``                                ``202207L``
     --------------------------------------------------- -----------------
     ``__cpp_lib_move_only_function``                    *unimplemented*
     --------------------------------------------------- -----------------
diff --git a/libcxx/docs/ReleaseNotes/18.rst b/libcxx/docs/ReleaseNotes/18.rst
index 5f43d2f2af..ac78563aa7 100644
--- a/libcxx/docs/ReleaseNotes/18.rst
+++ b/libcxx/docs/ReleaseNotes/18.rst
@@ -133,6 +133,13 @@ ABI Affecting Changes
   results in an ABI break, however in practice we expect uses of ``std::projected`` in ABI-sensitive places to be
   extremely rare. Any error resulting from this change should result in a link-time error.
 
+- Under the unstable ABI, the internal alignment requirements for heap allocations
+  inside ``std::string`` has decreased from 16 to 8 This save memory since string requests fewer additional
+  bytes than it did previously. However, this also changes the return value of ``std::string::max_size``
+  and can cause code compiled against older libc++ versions but linked at runtime to a new version
+  to throw a different exception when attempting allocations that are too large
+  (``std::bad_alloc`` vs ``std::length_error``).
+
 Build System Changes
 --------------------
 
diff --git a/libcxx/docs/Status/Cxx23.rst b/libcxx/docs/Status/Cxx23.rst
index 5b4d9a6fe9..839640a7c7 100644
--- a/libcxx/docs/Status/Cxx23.rst
+++ b/libcxx/docs/Status/Cxx23.rst
@@ -40,7 +40,6 @@ Paper Status
 
 .. note::
 
-   .. [#note-P0009R18] P0009R18: ``extents``, ``dextents``, ``layout_left``, ``layout_right``, and ``default_accessor`` are implemented.
    .. [#note-P0533R9] P0533R9: ``isfinite``, ``isinf``, ``isnan`` and ``isnormal`` are implemented.
    .. [#note-P1413R3] P1413R3: ``std::aligned_storage_t`` and ``std::aligned_union_t`` are marked deprecated, but
       clang doesn't issue a diagnostic for deprecated using template declarations.
diff --git a/libcxx/docs/Status/Cxx23Papers.csv b/libcxx/docs/Status/Cxx23Papers.csv
index 35319fb757..0513a4a6e9 100644
--- a/libcxx/docs/Status/Cxx23Papers.csv
+++ b/libcxx/docs/Status/Cxx23Papers.csv
@@ -51,7 +51,7 @@
 "`P2442R1 <https://wg21.link/P2442R1>`__","LWG","Windowing range adaptors: ``views::chunk`` and ``views::slide``","February 2022","","","|ranges|"
 "`P2443R1 <https://wg21.link/P2443R1>`__","LWG","``views::chunk_by``","February 2022","|Complete|","18.0","|ranges|"
 "","","","","","",""
-"`P0009R18 <https://wg21.link/P0009R18>`__","LWG","mdspan: A Non-Owning Multidimensional Array Reference","July 2022","|In progress| [#note-P0009R18]_",""
+"`P0009R18 <https://wg21.link/P0009R18>`__","LWG","mdspan: A Non-Owning Multidimensional Array Reference","July 2022","|Complete|","18.0"
 "`P0429R9 <https://wg21.link/P0429R9>`__","LWG","A Standard ``flat_map``","July 2022","",""
 "`P1169R4 <https://wg21.link/P1169R4>`__","LWG","``static operator()``","July 2022","|Complete|","16.0"
 "`P1222R4 <https://wg21.link/P1222R4>`__","LWG","A Standard ``flat_set``","July 2022","",""
@@ -89,9 +89,9 @@
 "`P2549R1 <https://wg21.link/P2549R1>`__","LWG","``std::unexpected`` should have ``error()`` as member accessor","July 2022","|Complete|","16.0"
 "`P2585R0 <https://wg21.link/P2585R0>`__","LWG","Improving default container formatting","July 2022","|Complete|","17.0"
 "`P2590R2 <https://wg21.link/P2590R2>`__","LWG","Explicit lifetime management","July 2022","",""
-"`P2599R2 <https://wg21.link/P2599R2>`__","LWG","``mdspan::size_type`` should be ``index_type``","July 2022","",""
-"`P2604R0 <https://wg21.link/P2604R0>`__","LWG","mdspan: rename pointer and contiguous","July 2022","",""
-"`P2613R1 <https://wg21.link/P2613R1>`__","LWG","Add the missing ``empty`` to ``mdspan``","July 2022","",""
+"`P2599R2 <https://wg21.link/P2599R2>`__","LWG","``mdspan::size_type`` should be ``index_type``","July 2022","|Complete|","18.0"
+"`P2604R0 <https://wg21.link/P2604R0>`__","LWG","mdspan: rename pointer and contiguous","July 2022","|Complete|","18.0"
+"`P2613R1 <https://wg21.link/P2613R1>`__","LWG","Add the missing ``empty`` to ``mdspan``","July 2022","|Complete|","18.0"
 "","","","","","",""
 "`P1202R5 <https://wg21.link/P1202R5>`__","LWG", "Asymmetric Fences", "November 2022","","","|concurrency TS|"
 "`P1264R2 <https://wg21.link/P1264R2>`__","LWG", "Revising the wording of ``stream`` input operations", "November 2022","|Complete|","9.0",""
diff --git a/libcxx/docs/Status/PSTLPaper.csv b/libcxx/docs/Status/PSTLPaper.csv
index f4e130a5de..1ca0a6ed35 100644
--- a/libcxx/docs/Status/PSTLPaper.csv
+++ b/libcxx/docs/Status/PSTLPaper.csv
@@ -36,7 +36,7 @@ Section,Description,Assignee,Complete
 | `[alg.min.max] <https://wg21.link/alg.min.max>`_,std::min_element,Nikolas Klauser,|Not Started|
 | `[alg.min.max] <https://wg21.link/alg.min.max>`_,std::minmax_element,Nikolas Klauser,|Not Started|
 | `[mismatch] <https://wg21.link/mismatch>`_,std::mismatch,Nikolas Klauser,|Not Started|
-| `[alg.move] <https://wg21.link/alg.move>`_,std::move,Nikolas Klauser,|In Progress|
+| `[alg.move] <https://wg21.link/alg.move>`_,std::move,Nikolas Klauser,|Complete|
 | `[alg.none.of] <https://wg21.link/alg.none.of>`_,std::none_of,Nikolas Klauser,|Complete|
 | `[alg.nth.element] <https://wg21.link/alg.nth.element>`_,std::nth_element,Nikolas Klauser,|Not Started|
 | `[alg.sort] <https://wg21.link/alg.sort>`_,std::partial_sort,Nikolas Klauser,|Not Started|
diff --git a/libcxx/docs/Status/ParallelismProjects.csv b/libcxx/docs/Status/ParallelismProjects.csv
index 8ce6403c27..be3663bf08 100644
--- a/libcxx/docs/Status/ParallelismProjects.csv
+++ b/libcxx/docs/Status/ParallelismProjects.csv
@@ -17,6 +17,7 @@ Section,Description,Dependencies,Assignee,Complete
 | `[parallel.simd.class] <https://wg21.link/N4808>`_, "`Class template simd declaration and alias <https://reviews.llvm.org/D144362>`_", [parallel.simd.abi], Yin Zhang, |Complete|
 | `[parallel.simd.class] <https://wg21.link/N4808>`_, "`simd<>::size() <https://reviews.llvm.org/D144363>`_", [parallel.simd.traits] simd_size[_v], Yin Zhang, |Complete|
 | `[parallel.simd.class] <https://wg21.link/N4808>`_, "`simd broadcast constructor <https://reviews.llvm.org/D156225>`_", None, Yin Zhang, |Complete|
+| `[parallel.simd.class] <https://wg21.link/N4808>`_, "`simd generate constructor <https://reviews.llvm.org/D159442>`_", None, Yin Zhang, |Complete|
 | `[parallel.simd.class] <https://wg21.link/N4808>`_, "Class template simd implementation", None, Yin Zhang, |In Progress|
 | `[parallel.simd.nonmembers] <https://wg21.link/N4808>`_, "simd non-member operations", None, Yin Zhang, |In Progress|
 | `[parallel.simd.mask.class] <https://wg21.link/N4808>`_, "`Class template simd_mask declaration and alias <https://reviews.llvm.org/D144362>`_", [parallel.simd.abi], Yin Zhang, |Complete|
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index 4d98b8eed1..e425af4fda 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -94,6 +94,7 @@ set(files
   __algorithm/pstl_generate.h
   __algorithm/pstl_is_partitioned.h
   __algorithm/pstl_merge.h
+  __algorithm/pstl_move.h
   __algorithm/pstl_replace.h
   __algorithm/pstl_sort.h
   __algorithm/pstl_stable_sort.h
@@ -508,6 +509,7 @@ set(files
   __mdspan/extents.h
   __mdspan/layout_left.h
   __mdspan/layout_right.h
+  __mdspan/layout_stride.h
   __mdspan/mdspan.h
   __memory/addressof.h
   __memory/align.h
@@ -977,7 +979,6 @@ set(files
   scoped_allocator
   semaphore
   set
-  setjmp.h
   shared_mutex
   source_location
   span
diff --git a/libcxx/include/__algorithm/nth_element.h b/libcxx/include/__algorithm/nth_element.h
index dbacf58f9e..ebd1cbf761 100644
--- a/libcxx/include/__algorithm/nth_element.h
+++ b/libcxx/include/__algorithm/nth_element.h
@@ -13,6 +13,7 @@
 #include <__algorithm/comp_ref_type.h>
 #include <__algorithm/iterator_operations.h>
 #include <__algorithm/sort.h>
+#include <__assert>
 #include <__config>
 #include <__debug_utils/randomize_range.h>
 #include <__iterator/iterator_traits.h>
@@ -42,6 +43,7 @@ __nth_element_find_guard(_RandomAccessIterator& __i, _RandomAccessIterator& __j,
 
 template <class _AlgPolicy, class _Compare, class _RandomAccessIterator>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
+// NOLINTNEXTLINE(readability-function-cognitive-complexity)
 __nth_element(_RandomAccessIterator __first, _RandomAccessIterator __nth, _RandomAccessIterator __last, _Compare __comp)
 {
     using _Ops = _IterOps<_AlgPolicy>;
@@ -116,10 +118,18 @@ __nth_element(_RandomAccessIterator __first, _RandomAccessIterator __nth, _Rando
                     return;
                 }
                 while (true) {
-                    while (!__comp(*__first, *__i))
+                    while (!__comp(*__first, *__i)) {
                         ++__i;
-                    while (__comp(*__first, *--__j))
-                        ;
+                        _LIBCPP_ASSERT_UNCATEGORIZED(
+                            __i != __last,
+                            "Would read out of bounds, does your comparator satisfy the strict-weak ordering requirement?");
+                    }
+                    do {
+                        _LIBCPP_ASSERT_UNCATEGORIZED(
+                            __j != __first,
+                            "Would read out of bounds, does your comparator satisfy the strict-weak ordering requirement?");
+                        --__j;
+                    } while (__comp(*__first, *__j));
                     if (__i >= __j)
                         break;
                     _Ops::iter_swap(__i, __j);
@@ -146,11 +156,19 @@ __nth_element(_RandomAccessIterator __first, _RandomAccessIterator __nth, _Rando
             while (true)
             {
                 // __m still guards upward moving __i
-                while (__comp(*__i, *__m))
+                while (__comp(*__i, *__m)) {
                     ++__i;
+                    _LIBCPP_ASSERT_UNCATEGORIZED(
+                        __i != __last,
+                        "Would read out of bounds, does your comparator satisfy the strict-weak ordering requirement?");
+                }
                 // It is now known that a guard exists for downward moving __j
-                while (!__comp(*--__j, *__m))
-                    ;
+                do {
+                    _LIBCPP_ASSERT_UNCATEGORIZED(
+                        __j != __first,
+                        "Would read out of bounds, does your comparator satisfy the strict-weak ordering requirement?");
+                    --__j;
+                } while (!__comp(*__j, *__m));
                 if (__i >= __j)
                     break;
                 _Ops::iter_swap(__i, __j);
diff --git a/libcxx/include/__algorithm/pstl_backend.h b/libcxx/include/__algorithm/pstl_backend.h
index 94644e5c47..1f5c13d49e 100644
--- a/libcxx/include/__algorithm/pstl_backend.h
+++ b/libcxx/include/__algorithm/pstl_backend.h
@@ -41,13 +41,13 @@ A PSTL parallel backend is a tag type to which the following functions are assoc
             class _ForwardIterator2,
             class _ForwardOutIterator,
             class _Comp>
-  _ForwardOutIterator __pstl_merge(_Backend,
-                                   _ForwardIterator1 __first1,
-                                   _ForwardIterator1 __last1,
-                                   _ForwardIterator2 __first2,
-                                   _ForwardIterator2 __last2,
-                                   _ForwardOutIterator __result,
-                                   _Comp __comp);
+  optional<_ForwardOutIterator> __pstl_merge(_Backend,
+                                             _ForwardIterator1 __first1,
+                                             _ForwardIterator1 __last1,
+                                             _ForwardIterator2 __first2,
+                                             _ForwardIterator2 __last2,
+                                             _ForwardOutIterator __result,
+                                             _Comp __comp);
 
   template <class _ExecutionPolicy, class _InIterator, class _OutIterator, class _UnaryOperation>
   optional<_OutIterator>
@@ -131,6 +131,9 @@ implemented, all the algorithms will eventually forward to the basis algorithms
                                       _OutIterator __result,
                                       _Comp __comp);
 
+  template <class _ExecutionPolicy, class _Iterator, class _OutIterator>
+  optional<_OutIterator> __pstl_move(_Backend, _Iterator __first, _Iterator __last, _OutIterator __result);
+
   template <class _ExecutionPolicy, class _Iterator, class _Tp, class _BinaryOperation>
   optional<_Tp> __pstl_reduce(_Backend, _Iterator __first, _Iterator __last, _Tp __init, _BinaryOperation __op);
 
diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/for_each.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/for_each.h
index 6cfef932aa..81fd4526b8 100644
--- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/for_each.h
+++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/for_each.h
@@ -26,7 +26,7 @@
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _Iterator, class _DifferenceType, class _Function>
-_LIBCPP_HIDE_FROM_ABI _Iterator __simd_walk_1(_Iterator __first, _DifferenceType __n, _Function __f) noexcept {
+_LIBCPP_HIDE_FROM_ABI _Iterator __simd_walk(_Iterator __first, _DifferenceType __n, _Function __f) noexcept {
   _PSTL_PRAGMA_SIMD
   for (_DifferenceType __i = 0; __i < __n; ++__i)
     __f(__first[__i]);
@@ -47,7 +47,7 @@ __pstl_for_each(__cpu_backend_tag, _ForwardIterator __first, _ForwardIterator __
         });
   } else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> &&
                        __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) {
-    std::__simd_walk_1(__first, __last - __first, __func);
+    std::__simd_walk(__first, __last - __first, __func);
     return __empty{};
   } else {
     std::for_each(__first, __last, __func);
diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform.h
index 2c7647d61a..fdf1a2e78d 100644
--- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform.h
+++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform.h
@@ -32,7 +32,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _Iterator1, class _DifferenceType, class _Iterator2, class _Function>
 _LIBCPP_HIDE_FROM_ABI _Iterator2
-__simd_walk_2(_Iterator1 __first1, _DifferenceType __n, _Iterator2 __first2, _Function __f) noexcept {
+__simd_walk(_Iterator1 __first1, _DifferenceType __n, _Iterator2 __first2, _Function __f) noexcept {
   _PSTL_PRAGMA_SIMD
   for (_DifferenceType __i = 0; __i < __n; ++__i)
     __f(__first1[__i], __first2[__i]);
@@ -60,7 +60,7 @@ _LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator> __pstl_transform(
   } else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> &&
                        __has_random_access_iterator_category_or_concept<_ForwardIterator>::value &&
                        __has_random_access_iterator_category_or_concept<_ForwardOutIterator>::value) {
-    return std::__simd_walk_2(
+    return std::__simd_walk(
         __first,
         __last - __first,
         __result,
@@ -73,7 +73,7 @@ _LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator> __pstl_transform(
 }
 
 template <class _Iterator1, class _DifferenceType, class _Iterator2, class _Iterator3, class _Function>
-_LIBCPP_HIDE_FROM_ABI _Iterator3 __simd_walk_3(
+_LIBCPP_HIDE_FROM_ABI _Iterator3 __simd_walk(
     _Iterator1 __first1, _DifferenceType __n, _Iterator2 __first2, _Iterator3 __first3, _Function __f) noexcept {
   _PSTL_PRAGMA_SIMD
   for (_DifferenceType __i = 0; __i < __n; ++__i)
@@ -116,7 +116,7 @@ _LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator> __pstl_transform(
                        __has_random_access_iterator_category_or_concept<_ForwardIterator1>::value &&
                        __has_random_access_iterator_category_or_concept<_ForwardIterator2>::value &&
                        __has_random_access_iterator_category_or_concept<_ForwardOutIterator>::value) {
-    return std::__simd_walk_3(
+    return std::__simd_walk(
         __first1,
         __last1 - __first1,
         __first2,
diff --git a/libcxx/include/__algorithm/pstl_move.h b/libcxx/include/__algorithm/pstl_move.h
new file mode 100644
index 0000000000..775dc26049
--- /dev/null
+++ b/libcxx/include/__algorithm/pstl_move.h
@@ -0,0 +1,78 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_PSTL_MOVE_H
+#define _LIBCPP___ALGORITHM_PSTL_MOVE_H
+
+#include <__algorithm/copy_n.h>
+#include <__algorithm/pstl_frontend_dispatch.h>
+#include <__algorithm/pstl_transform.h>
+#include <__config>
+#include <__functional/identity.h>
+#include <__iterator/iterator_traits.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/is_constant_evaluated.h>
+#include <__type_traits/is_execution_policy.h>
+#include <__type_traits/is_trivially_copyable.h>
+#include <__type_traits/remove_cvref.h>
+#include <optional>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#if !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// TODO: Use the std::copy/move shenanigans to forward to std::memmove
+//       Investigate whether we want to still forward to std::transform(policy)
+//       in that case for the execution::par part, or whether we actually want
+//       to run everything serially in that case.
+
+template <class>
+void __pstl_move();
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _ForwardOutIterator,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator>
+__move(_ExecutionPolicy&& __policy,
+       _ForwardIterator&& __first,
+       _ForwardIterator&& __last,
+       _ForwardOutIterator&& __result) noexcept {
+  return std::__pstl_frontend_dispatch(
+      _LIBCPP_PSTL_CUSTOMIZATION_POINT(__pstl_move, _RawPolicy),
+      [&__policy](_ForwardIterator __g_first, _ForwardIterator __g_last, _ForwardOutIterator __g_result) {
+        return std::__transform(__policy, __g_first, __g_last, __g_result, [](auto&& __v) { return std::move(__v); });
+      },
+      std::move(__first),
+      std::move(__last),
+      std::move(__result));
+}
+
+template <class _ExecutionPolicy,
+          class _ForwardIterator,
+          class _ForwardOutIterator,
+          class _RawPolicy                                    = __remove_cvref_t<_ExecutionPolicy>,
+          enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
+_LIBCPP_HIDE_FROM_ABI _ForwardOutIterator
+move(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _ForwardOutIterator __result) {
+  auto __res = std::__move(__policy, std::move(__first), std::move(__last), std::move(__result));
+  if (!__res)
+    std::__throw_bad_alloc();
+  return *__res;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17
+
+#endif // _LIBCPP___ALGORITHM_PSTL_MOVE_H
diff --git a/libcxx/include/__atomic/atomic_base.h b/libcxx/include/__atomic/atomic_base.h
index 87100ba5d8..775d06d757 100644
--- a/libcxx/include/__atomic/atomic_base.h
+++ b/libcxx/include/__atomic/atomic_base.h
@@ -39,7 +39,7 @@ struct __atomic_base  // false
 
     _LIBCPP_HIDE_FROM_ABI
     bool is_lock_free() const volatile _NOEXCEPT
-        {return __cxx_atomic_is_lock_free(sizeof(_Tp));}
+        {return __cxx_atomic_is_lock_free(sizeof(__cxx_atomic_impl<_Tp>));}
     _LIBCPP_HIDE_FROM_ABI
     bool is_lock_free() const _NOEXCEPT
         {return static_cast<__atomic_base const volatile*>(this)->is_lock_free();}
diff --git a/libcxx/include/__concepts/swappable.h b/libcxx/include/__concepts/swappable.h
index c1969de345..cdffe34205 100644
--- a/libcxx/include/__concepts/swappable.h
+++ b/libcxx/include/__concepts/swappable.h
@@ -92,7 +92,7 @@ struct __fn {
   // 2.3   Otherwise, if `E1` and `E2` are lvalues of the same type `T` that models...
   template <__exchangeable _Tp>
   _LIBCPP_HIDE_FROM_ABI constexpr void operator()(_Tp& __x, _Tp& __y) const
-      noexcept(is_nothrow_move_constructible_v<_Tp>&& is_nothrow_move_assignable_v<_Tp>) {
+      noexcept(is_nothrow_move_constructible_v<_Tp> && is_nothrow_move_assignable_v<_Tp>) {
     __y = _VSTD::exchange(__x, _VSTD::move(__y));
   }
 };
diff --git a/libcxx/include/__config b/libcxx/include/__config
index 55d9f1c737..65ce6d6a27 100644
--- a/libcxx/include/__config
+++ b/libcxx/include/__config
@@ -167,6 +167,11 @@
 // The implementation moved to the header, but we still export the symbols from
 // the dylib for backwards compatibility.
 #    define _LIBCPP_ABI_DO_NOT_EXPORT_TO_CHARS_BASE_10
+// Save memory by providing the allocator more freedom to allocate the most
+// efficient size class by dropping the alignment requirements for std::string's
+// pointer from 16 to 8. This changes the output of std::string::max_size,
+// which makes it ABI breaking
+#    define _LIBCPP_ABI_STRING_8_BYTE_ALIGNMENT
 #  elif _LIBCPP_ABI_VERSION == 1
 #    if !(defined(_LIBCPP_OBJECT_FORMAT_COFF) || defined(_LIBCPP_OBJECT_FORMAT_XCOFF))
 // Enable compiling copies of now inline methods into the dylib to support
diff --git a/libcxx/include/__fwd/mdspan.h b/libcxx/include/__fwd/mdspan.h
index a3628c2d60..8889567a04 100644
--- a/libcxx/include/__fwd/mdspan.h
+++ b/libcxx/include/__fwd/mdspan.h
@@ -42,14 +42,11 @@ struct layout_right {
   class mapping;
 };
 
-/*
-// Will be implemented with follow on revision
 // Layout policy with a unique mapping where strides are arbitrary
 struct layout_stride {
-  template<class Extents>
-    class mapping;
+  template <class _Extents>
+  class mapping;
 };
-*/
 
 #endif // _LIBCPP_STD_VER >= 23
 
diff --git a/libcxx/include/__hash_table b/libcxx/include/__hash_table
index 98337abe55..1732c82178 100644
--- a/libcxx/include/__hash_table
+++ b/libcxx/include/__hash_table
@@ -21,6 +21,7 @@
 #include <__memory/addressof.h>
 #include <__memory/allocator_traits.h>
 #include <__memory/compressed_pair.h>
+#include <__memory/construct_at.h>
 #include <__memory/pointer_traits.h>
 #include <__memory/swap_allocator.h>
 #include <__memory/unique_ptr.h>
@@ -45,6 +46,7 @@
 #include <cmath>
 #include <cstring>
 #include <initializer_list>
+#include <new> // __launder
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -107,19 +109,44 @@ struct __hash_node_base
     }
 
     _LIBCPP_INLINE_VISIBILITY __hash_node_base() _NOEXCEPT : __next_(nullptr) {}
+    _LIBCPP_HIDE_FROM_ABI explicit __hash_node_base(__next_pointer __next) _NOEXCEPT : __next_(__next) {}
 };
 
 template <class _Tp, class _VoidPtr>
-struct _LIBCPP_STANDALONE_DEBUG __hash_node
+struct __hash_node
     : public __hash_node_base
              <
                  __rebind_pointer_t<_VoidPtr, __hash_node<_Tp, _VoidPtr> >
              >
 {
     typedef _Tp __node_value_type;
+    using _Base = __hash_node_base<__rebind_pointer_t<_VoidPtr, __hash_node<_Tp, _VoidPtr> > >;
+    using __next_pointer = typename _Base::__next_pointer;
 
     size_t            __hash_;
-    __node_value_type __value_;
+
+    // We allow starting the lifetime of nodes without initializing the value held by the node,
+    // since that is handled by the hash table itself in order to be allocator-aware.
+#ifndef _LIBCPP_CXX03_LANG
+private:
+    union {
+        _Tp __value_;
+    };
+
+public:
+    _LIBCPP_HIDE_FROM_ABI _Tp& __get_value() { return __value_; }
+#else
+private:
+    _ALIGNAS_TYPE(_Tp) char __buffer_[sizeof(_Tp)];
+
+public:
+    _LIBCPP_HIDE_FROM_ABI _Tp& __get_value() {
+        return *std::__launder(reinterpret_cast<_Tp*>(&__buffer_));
+    }
+#endif
+
+    _LIBCPP_HIDE_FROM_ABI explicit __hash_node(__next_pointer __next, size_t __hash) : _Base(__next), __hash_(__hash) {}
+    _LIBCPP_HIDE_FROM_ABI ~__hash_node() {}
 };
 
 inline _LIBCPP_INLINE_VISIBILITY
@@ -311,12 +338,12 @@ public:
 
     _LIBCPP_INLINE_VISIBILITY
     reference operator*() const {
-        return __node_->__upcast()->__value_;
+        return __node_->__upcast()->__get_value();
     }
 
     _LIBCPP_INLINE_VISIBILITY
     pointer operator->() const {
-        return pointer_traits<pointer>::pointer_to(__node_->__upcast()->__value_);
+        return pointer_traits<pointer>::pointer_to(__node_->__upcast()->__get_value());
     }
 
     _LIBCPP_INLINE_VISIBILITY
@@ -387,11 +414,11 @@ public:
 
     _LIBCPP_INLINE_VISIBILITY
     reference operator*() const {
-        return __node_->__upcast()->__value_;
+        return __node_->__upcast()->__get_value();
     }
     _LIBCPP_INLINE_VISIBILITY
     pointer operator->() const {
-        return pointer_traits<pointer>::pointer_to(__node_->__upcast()->__value_);
+        return pointer_traits<pointer>::pointer_to(__node_->__upcast()->__get_value());
     }
 
     _LIBCPP_INLINE_VISIBILITY
@@ -453,12 +480,12 @@ public:
 
     _LIBCPP_INLINE_VISIBILITY
     reference operator*() const {
-        return __node_->__upcast()->__value_;
+        return __node_->__upcast()->__get_value();
     }
 
     _LIBCPP_INLINE_VISIBILITY
     pointer operator->() const {
-        return pointer_traits<pointer>::pointer_to(__node_->__upcast()->__value_);
+        return pointer_traits<pointer>::pointer_to(__node_->__upcast()->__get_value());
     }
 
     _LIBCPP_INLINE_VISIBILITY
@@ -543,12 +570,12 @@ public:
 
     _LIBCPP_INLINE_VISIBILITY
     reference operator*() const {
-        return __node_->__upcast()->__value_;
+        return __node_->__upcast()->__get_value();
     }
 
     _LIBCPP_INLINE_VISIBILITY
     pointer operator->() const {
-        return pointer_traits<pointer>::pointer_to(__node_->__upcast()->__value_);
+        return pointer_traits<pointer>::pointer_to(__node_->__upcast()->__get_value());
     }
 
     _LIBCPP_INLINE_VISIBILITY
@@ -670,8 +697,10 @@ public:
     _LIBCPP_INLINE_VISIBILITY
     void operator()(pointer __p) _NOEXCEPT
     {
-        if (__value_constructed)
-            __alloc_traits::destroy(__na_, _NodeTypes::__get_ptr(__p->__value_));
+        if (__value_constructed) {
+            __alloc_traits::destroy(__na_, _NodeTypes::__get_ptr(__p->__get_value()));
+            std::__destroy_at(std::addressof(*__p));
+        }
         if (__p)
             __alloc_traits::deallocate(__na_, __p, 1);
     }
@@ -1365,7 +1394,8 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::__deallocate_node(__next_pointer __np)
     {
         __next_pointer __next = __np->__next_;
         __node_pointer __real_np = __np->__upcast();
-        __node_traits::destroy(__na, _NodeTypes::__get_ptr(__real_np->__value_));
+        __node_traits::destroy(__na, _NodeTypes::__get_ptr(__real_np->__get_value()));
+        std::__destroy_at(std::addressof(*__real_np));
         __node_traits::deallocate(__na, __real_np, 1);
         __np = __next;
     }
@@ -1434,8 +1464,8 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::__move_assign(
                 const_iterator __i = __u.begin();
                 while (__cache != nullptr && __u.size() != 0)
                 {
-                    __cache->__upcast()->__value_ =
-                        _VSTD::move(__u.remove(__i++)->__value_);
+                    __cache->__upcast()->__get_value() =
+                        _VSTD::move(__u.remove(__i++)->__get_value());
                     __next_pointer __next = __cache->__next_;
                     __node_insert_multi(__cache->__upcast());
                     __cache = __next;
@@ -1453,7 +1483,7 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::__move_assign(
         const_iterator __i = __u.begin();
         while (__u.size() != 0)
         {
-            __node_holder __h = __construct_node(_NodeTypes::__move(__u.remove(__i++)->__value_));
+            __node_holder __h = __construct_node(_NodeTypes::__move(__u.remove(__i++)->__get_value()));
             __node_insert_multi(__h.get());
             __h.release();
         }
@@ -1495,7 +1525,7 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::__assign_unique(_InputIterator __first
 #endif // _LIBCPP_HAS_NO_EXCEPTIONS
             for (; __cache != nullptr && __first != __last; ++__first)
             {
-                __cache->__upcast()->__value_ = *__first;
+                __cache->__upcast()->__get_value() = *__first;
                 __next_pointer __next = __cache->__next_;
                 __node_insert_unique(__cache->__upcast());
                 __cache = __next;
@@ -1535,7 +1565,7 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::__assign_multi(_InputIterator __first,
 #endif // _LIBCPP_HAS_NO_EXCEPTIONS
             for (; __cache != nullptr && __first != __last; ++__first)
             {
-                __cache->__upcast()->__value_ = *__first;
+                __cache->__upcast()->__get_value() = *__first;
                 __next_pointer __next = __cache->__next_;
                 __node_insert_multi(__cache->__upcast());
                 __cache = __next;
@@ -1629,7 +1659,7 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_unique_prepare(
                                                      __ndptr = __ndptr->__next_)
             {
                 if ((__ndptr->__hash() == __hash) &&
-                    key_eq()(__ndptr->__upcast()->__value_, __value))
+                    key_eq()(__ndptr->__upcast()->__get_value(), __value))
                     return __ndptr;
             }
         }
@@ -1678,9 +1708,9 @@ template <class _Tp, class _Hash, class _Equal, class _Alloc>
 pair<typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator, bool>
 __hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_unique(__node_pointer __nd)
 {
-    __nd->__hash_ = hash_function()(__nd->__value_);
+    __nd->__hash_ = hash_function()(__nd->__get_value());
     __next_pointer __existing_node =
-        __node_insert_unique_prepare(__nd->__hash(), __nd->__value_);
+        __node_insert_unique_prepare(__nd->__hash(), __nd->__get_value());
 
     // Insert the node, unless it already exists in the container.
     bool __inserted = false;
@@ -1726,7 +1756,7 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_multi_prepare(
             //      false       true        set __found to true
             //      true        false       break
             if (__found != (__pn->__next_->__hash() == __cp_hash &&
-                            key_eq()(__pn->__next_->__upcast()->__value_, __cp_val)))
+                            key_eq()(__pn->__next_->__upcast()->__get_value(), __cp_val)))
             {
                 if (!__found)
                     __found = true;
@@ -1780,8 +1810,8 @@ template <class _Tp, class _Hash, class _Equal, class _Alloc>
 typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator
 __hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_multi(__node_pointer __cp)
 {
-    __cp->__hash_ = hash_function()(__cp->__value_);
-    __next_pointer __pn = __node_insert_multi_prepare(__cp->__hash(), __cp->__value_);
+    __cp->__hash_ = hash_function()(__cp->__get_value());
+    __next_pointer __pn = __node_insert_multi_prepare(__cp->__hash(), __cp->__get_value());
     __node_insert_multi_perform(__cp, __pn);
 
     return iterator(__cp->__ptr());
@@ -1792,7 +1822,7 @@ typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator
 __hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_multi(
         const_iterator __p, __node_pointer __cp)
 {
-    if (__p != end() && key_eq()(*__p, __cp->__value_))
+    if (__p != end() && key_eq()(*__p, __cp->__get_value()))
     {
         __next_pointer __np = __p.__node_;
         __cp->__hash_ = __np->__hash();
@@ -1839,7 +1869,7 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::__emplace_unique_key_args(_Key const&
                                                            __nd = __nd->__next_)
             {
                 if ((__nd->__hash() == __hash) &&
-                    key_eq()(__nd->__upcast()->__value_, __k))
+                    key_eq()(__nd->__upcast()->__get_value(), __k))
                     goto __done;
             }
         }
@@ -1983,9 +2013,9 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_handle_merge_unique(
          __it != __source.end();)
     {
         __node_pointer __src_ptr = __it.__node_->__upcast();
-        size_t __hash = hash_function()(__src_ptr->__value_);
+        size_t __hash = hash_function()(__src_ptr->__get_value());
         __next_pointer __existing_node =
-            __node_insert_unique_prepare(__hash, __src_ptr->__value_);
+            __node_insert_unique_prepare(__hash, __src_ptr->__get_value());
         auto __prev_iter = __it++;
         if (__existing_node == nullptr)
         {
@@ -2037,9 +2067,9 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_handle_merge_multi(
          __it != __source.end();)
     {
         __node_pointer __src_ptr = __it.__node_->__upcast();
-        size_t __src_hash = hash_function()(__src_ptr->__value_);
+        size_t __src_hash = hash_function()(__src_ptr->__get_value());
         __next_pointer __pn =
-            __node_insert_multi_prepare(__src_hash, __src_ptr->__value_);
+            __node_insert_multi_prepare(__src_hash, __src_ptr->__get_value());
         (void)__source.remove(__it++).release();
         __src_ptr->__hash_ = __src_hash;
         __node_insert_multi_perform(__src_ptr, __pn);
@@ -2113,8 +2143,8 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::__do_rehash(size_type __nbc)
                         if _LIBCPP_CONSTEXPR_SINCE_CXX17 (!_UniqueKeys)
                         {
                             for (; __np->__next_ != nullptr &&
-                                   key_eq()(__cp->__upcast()->__value_,
-                                            __np->__next_->__upcast()->__value_);
+                                   key_eq()(__cp->__upcast()->__get_value(),
+                                            __np->__next_->__upcast()->__get_value());
                                                                __np = __np->__next_)
                                 ;
                         }
@@ -2148,7 +2178,7 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::find(const _Key& __k)
                                                            __nd = __nd->__next_)
             {
                 if ((__nd->__hash() == __hash)
-                    && key_eq()(__nd->__upcast()->__value_, __k))
+                    && key_eq()(__nd->__upcast()->__get_value(), __k))
                     return iterator(__nd);
             }
         }
@@ -2175,7 +2205,7 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::find(const _Key& __k) const
                                                            __nd = __nd->__next_)
             {
                 if ((__nd->__hash() == __hash)
-                    && key_eq()(__nd->__upcast()->__value_, __k))
+                    && key_eq()(__nd->__upcast()->__get_value(), __k))
                     return const_iterator(__nd);
             }
         }
@@ -2193,10 +2223,20 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::__construct_node(_Args&& ...__args)
                   "Construct cannot be called with a hash value type");
     __node_allocator& __na = __node_alloc();
     __node_holder __h(__node_traits::allocate(__na, 1), _Dp(__na));
-    __node_traits::construct(__na, _NodeTypes::__get_ptr(__h->__value_), _VSTD::forward<_Args>(__args)...);
+
+    // Begin the lifetime of the node itself. Note that this doesn't begin the lifetime of the value
+    // held inside the node, since we need to use the allocator's construct() method for that.
+    //
+    // We don't use the allocator's construct() method to construct the node itself since the
+    // Cpp17FooInsertable named requirements don't require the allocator's construct() method
+    // to work on anything other than the value_type.
+    std::__construct_at(std::addressof(*__h), /* next = */nullptr, /* hash = */0);
+
+    // Now construct the value_type using the allocator's construct() method.
+    __node_traits::construct(__na, _NodeTypes::__get_ptr(__h->__get_value()), _VSTD::forward<_Args>(__args)...);
     __h.get_deleter().__value_constructed = true;
-    __h->__hash_ = hash_function()(__h->__value_);
-    __h->__next_ = nullptr;
+
+    __h->__hash_ = hash_function()(__h->__get_value());
     return __h;
 }
 
@@ -2210,12 +2250,11 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::__construct_node_hash(
                   "Construct cannot be called with a hash value type");
     __node_allocator& __na = __node_alloc();
     __node_holder __h(__node_traits::allocate(__na, 1), _Dp(__na));
-    __node_traits::construct(__na, _NodeTypes::__get_ptr(__h->__value_),
+    std::__construct_at(std::addressof(*__h), /* next = */nullptr, /* hash = */__hash);
+    __node_traits::construct(__na, _NodeTypes::__get_ptr(__h->__get_value()),
                              _VSTD::forward<_First>(__f),
                              _VSTD::forward<_Rest>(__rest)...);
     __h.get_deleter().__value_constructed = true;
-    __h->__hash_ = __hash;
-    __h->__next_ = nullptr;
     return __h;
 }
 
diff --git a/libcxx/include/__mdspan/extents.h b/libcxx/include/__mdspan/extents.h
index a510220d40..f6bcd940ee 100644
--- a/libcxx/include/__mdspan/extents.h
+++ b/libcxx/include/__mdspan/extents.h
@@ -456,7 +456,7 @@ using dextents = typename __mdspan_detail::__make_dextents<_IndexType, _Rank>::t
 
 // Deduction guide for extents
 template <class... _IndexTypes>
-extents(_IndexTypes...) -> extents<size_t, size_t((_IndexTypes(), dynamic_extent))...>;
+extents(_IndexTypes...) -> extents<size_t, size_t(((void)sizeof(_IndexTypes), dynamic_extent))...>;
 
 namespace __mdspan_detail {
 
diff --git a/libcxx/include/__mdspan/layout_left.h b/libcxx/include/__mdspan/layout_left.h
index 5faae597f6..fd644fa0c5 100644
--- a/libcxx/include/__mdspan/layout_left.h
+++ b/libcxx/include/__mdspan/layout_left.h
@@ -110,12 +110,27 @@ class layout_left::mapping {
         "layout_left::mapping converting ctor: other.required_span_size() must be representable as index_type.");
   }
 
-// FIXME: add when we add other layouts
-#  if 0
-    template<class _OtherExtents>
-      constexpr explicit(extents_type::rank() > 0)
-        mapping(const layout_stride::mapping_<OtherExtents>&) noexcept;
-#  endif
+  template <class _OtherExtents>
+    requires(is_constructible_v<extents_type, _OtherExtents>)
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit(extents_type::rank() > 0)
+      mapping(const layout_stride::mapping<_OtherExtents>& __other) noexcept
+      : __extents_(__other.extents()) {
+    if constexpr (extents_type::rank() > 0) {
+      _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+          ([&]() {
+            using _CommonType = common_type_t<typename extents_type::index_type, typename _OtherExtents::index_type>;
+            for (rank_type __r = 0; __r < extents_type::rank(); __r++)
+              if (static_cast<_CommonType>(stride(__r)) != static_cast<_CommonType>(__other.stride(__r)))
+                return false;
+            return true;
+          }()),
+          "layout_left::mapping from layout_stride ctor: strides are not compatible with layout_left.");
+      _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+          __mdspan_detail::__is_representable_as<index_type>(__other.required_span_size()),
+          "layout_left::mapping from layout_stride ctor: other.required_span_size() must be representable as "
+          "index_type.");
+    }
+  }
 
   _LIBCPP_HIDE_FROM_ABI constexpr mapping& operator=(const mapping&) noexcept = default;
 
diff --git a/libcxx/include/__mdspan/layout_right.h b/libcxx/include/__mdspan/layout_right.h
index 4f95789a2f..8e64d07dd5 100644
--- a/libcxx/include/__mdspan/layout_right.h
+++ b/libcxx/include/__mdspan/layout_right.h
@@ -109,12 +109,27 @@ class layout_right::mapping {
         "layout_right::mapping converting ctor: other.required_span_size() must be representable as index_type.");
   }
 
-// FIXME: add when we add other layouts
-#  if 0
-    template<class _OtherExtents>
-      constexpr explicit(extents_type::rank() > 0)
-        mapping(const layout_stride::mapping_<OtherExtents>&) noexcept;
-#  endif
+  template <class _OtherExtents>
+    requires(is_constructible_v<extents_type, _OtherExtents>)
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit(extents_type::rank() > 0)
+      mapping(const layout_stride::mapping<_OtherExtents>& __other) noexcept
+      : __extents_(__other.extents()) {
+    if constexpr (extents_type::rank() > 0) {
+      _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+          ([&]() {
+            using _CommonType = common_type_t<typename extents_type::index_type, typename _OtherExtents::index_type>;
+            for (rank_type __r = 0; __r < extents_type::rank(); __r++)
+              if (static_cast<_CommonType>(stride(__r)) != static_cast<_CommonType>(__other.stride(__r)))
+                return false;
+            return true;
+          }()),
+          "layout_right::mapping from layout_stride ctor: strides are not compatible with layout_right.");
+      _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+          __mdspan_detail::__is_representable_as<index_type>(__other.required_span_size()),
+          "layout_right::mapping from layout_stride ctor: other.required_span_size() must be representable as "
+          "index_type.");
+    }
+  }
 
   _LIBCPP_HIDE_FROM_ABI constexpr mapping& operator=(const mapping&) noexcept = default;
 
diff --git a/libcxx/include/__mdspan/layout_stride.h b/libcxx/include/__mdspan/layout_stride.h
new file mode 100644
index 0000000000..77934bfa11
--- /dev/null
+++ b/libcxx/include/__mdspan/layout_stride.h
@@ -0,0 +1,366 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//                        Kokkos v. 4.0
+//       Copyright (2022) National Technology & Engineering
+//               Solutions of Sandia, LLC (NTESS).
+//
+// Under the terms of Contract DE-NA0003525 with NTESS,
+// the U.S. Government retains certain rights in this software.
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MDSPAN_LAYOUT_STRIDE_H
+#define _LIBCPP___MDSPAN_LAYOUT_STRIDE_H
+
+#include <__assert>
+#include <__config>
+#include <__fwd/mdspan.h>
+#include <__mdspan/extents.h>
+#include <__type_traits/is_constructible.h>
+#include <__type_traits/is_convertible.h>
+#include <__type_traits/is_nothrow_constructible.h>
+#include <__utility/as_const.h>
+#include <__utility/integer_sequence.h>
+#include <__utility/swap.h>
+#include <array>
+#include <cinttypes>
+#include <cstddef>
+#include <limits>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+#if _LIBCPP_STD_VER >= 23
+
+namespace __mdspan_detail {
+template <class _Layout, class _Mapping>
+constexpr bool __is_mapping_of =
+    is_same_v<typename _Layout::template mapping<typename _Mapping::extents_type>, _Mapping>;
+
+template <class _Mapping>
+concept __layout_mapping_alike = requires {
+  requires __is_mapping_of<typename _Mapping::layout_type, _Mapping>;
+  requires __is_extents_v<typename _Mapping::extents_type>;
+  { _Mapping::is_always_strided() } -> same_as<bool>;
+  { _Mapping::is_always_exhaustive() } -> same_as<bool>;
+  { _Mapping::is_always_unique() } -> same_as<bool>;
+  bool_constant<_Mapping::is_always_strided()>::value;
+  bool_constant<_Mapping::is_always_exhaustive()>::value;
+  bool_constant<_Mapping::is_always_unique()>::value;
+};
+} // namespace __mdspan_detail
+
+template <class _Extents>
+class layout_stride::mapping {
+public:
+  static_assert(__mdspan_detail::__is_extents<_Extents>::value,
+                "layout_stride::mapping template argument must be a specialization of extents.");
+
+  using extents_type = _Extents;
+  using index_type   = typename extents_type::index_type;
+  using size_type    = typename extents_type::size_type;
+  using rank_type    = typename extents_type::rank_type;
+  using layout_type  = layout_stride;
+
+private:
+  static constexpr rank_type __rank_ = extents_type::rank();
+
+  // Used for default construction check and mandates
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool __required_span_size_is_representable(const extents_type& __ext) {
+    if constexpr (__rank_ == 0)
+      return true;
+
+    index_type __prod = __ext.extent(0);
+    for (rank_type __r = 1; __r < __rank_; __r++) {
+      bool __overflowed = __builtin_mul_overflow(__prod, __ext.extent(__r), &__prod);
+      if (__overflowed)
+        return false;
+    }
+    return true;
+  }
+
+  template <class _OtherIndexType>
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool
+  __required_span_size_is_representable(const extents_type& __ext, span<_OtherIndexType, __rank_> __strides) {
+    if constexpr (__rank_ == 0)
+      return true;
+
+    index_type __size = 1;
+    for (rank_type __r = 0; __r < __rank_; __r++) {
+      // We can only check correct conversion of _OtherIndexType if it is an integral
+      if constexpr (is_integral_v<_OtherIndexType>) {
+        using _CommonType = common_type_t<index_type, _OtherIndexType>;
+        if (static_cast<_CommonType>(__strides[__r]) > static_cast<_CommonType>(numeric_limits<index_type>::max()))
+          return false;
+      }
+      if (__ext.extent(__r) == static_cast<index_type>(0))
+        return true;
+      index_type __prod     = (__ext.extent(__r) - 1);
+      bool __overflowed_mul = __builtin_mul_overflow(__prod, static_cast<index_type>(__strides[__r]), &__prod);
+      if (__overflowed_mul)
+        return false;
+      bool __overflowed_add = __builtin_add_overflow(__size, __prod, &__size);
+      if (__overflowed_add)
+        return false;
+    }
+    return true;
+  }
+
+  // compute offset of a strided layout mapping
+  template <class _StridedMapping>
+  _LIBCPP_HIDE_FROM_ABI static constexpr index_type __offset(const _StridedMapping& __mapping) {
+    if constexpr (_StridedMapping::extents_type::rank() == 0) {
+      return static_cast<index_type>(__mapping());
+    } else if (__mapping.required_span_size() == static_cast<typename _StridedMapping::index_type>(0)) {
+      return static_cast<index_type>(0);
+    } else {
+      return [&]<size_t... _Pos>(index_sequence<_Pos...>) {
+        return static_cast<index_type>(__mapping((_Pos ? 0 : 0)...));
+      }(make_index_sequence<__rank_>());
+    }
+  }
+
+  // compute the permutation for sorting the stride array
+  // we never actually sort the stride array
+  _LIBCPP_HIDE_FROM_ABI constexpr void __bubble_sort_by_strides(array<rank_type, __rank_>& __permute) const {
+    for (rank_type __i = __rank_ - 1; __i > 0; __i--) {
+      for (rank_type __r = 0; __r < __i; __r++) {
+        if (__strides_[__permute[__r]] > __strides_[__permute[__r + 1]]) {
+          swap(__permute[__r], __permute[__r + 1]);
+        } else {
+          // if two strides are the same then one of the associated extents must be 1 or 0
+          // both could be, but you can't have one larger than 1 come first
+          if ((__strides_[__permute[__r]] == __strides_[__permute[__r + 1]]) &&
+              (__extents_.extent(__permute[__r]) > static_cast<index_type>(1)))
+            swap(__permute[__r], __permute[__r + 1]);
+        }
+      }
+    }
+  }
+
+  static_assert((extents_type::rank_dynamic() > 0) || __required_span_size_is_representable(extents_type()),
+                "layout_stride::mapping product of static extents must be representable as index_type.");
+
+public:
+  // [mdspan.layout.stride.cons], constructors
+  _LIBCPP_HIDE_FROM_ABI constexpr mapping() noexcept : __extents_(extents_type()) {
+    // Note the nominal precondition is covered by above static assert since
+    // if rank_dynamic is != 0 required_span_size is zero for default construction
+    if constexpr (__rank_ > 0) {
+      index_type __stride = 1;
+      for (rank_type __r = __rank_ - 1; __r > static_cast<rank_type>(0); __r--) {
+        __strides_[__r] = __stride;
+        __stride *= __extents_.extent(__r);
+      }
+      __strides_[0] = __stride;
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr mapping(const mapping&) noexcept = default;
+
+  template <class _OtherIndexType>
+    requires(is_convertible_v<const _OtherIndexType&, index_type> &&
+             is_nothrow_constructible_v<index_type, const _OtherIndexType&>)
+  _LIBCPP_HIDE_FROM_ABI constexpr mapping(const extents_type& __ext, span<_OtherIndexType, __rank_> __strides) noexcept
+      : __extents_(__ext), __strides_([&]<size_t... _Pos>(index_sequence<_Pos...>) {
+          return __mdspan_detail::__possibly_empty_array<index_type, __rank_>{
+              static_cast<index_type>(std::as_const(__strides[_Pos]))...};
+        }(make_index_sequence<__rank_>())) {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        ([&]<size_t... _Pos>(index_sequence<_Pos...>) {
+          // For integrals we can do a pre-conversion check, for other types not
+          if constexpr (is_integral_v<_OtherIndexType>) {
+            return ((__strides[_Pos] > static_cast<_OtherIndexType>(0)) && ... && true);
+          } else {
+            return ((static_cast<index_type>(__strides[_Pos]) > static_cast<index_type>(0)) && ... && true);
+          }
+        }(make_index_sequence<__rank_>())),
+        "layout_stride::mapping ctor: all strides must be greater than 0");
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __required_span_size_is_representable(__ext, __strides),
+        "layout_stride::mapping ctor: required span size is not representable as index_type.");
+    if constexpr (__rank_ > 1) {
+      _LIBCPP_ASSERT_UNCATEGORIZED(
+          ([&]<size_t... _Pos>(index_sequence<_Pos...>) {
+            // basically sort the dimensions based on strides and extents, sorting is represented in permute array
+            array<rank_type, __rank_> __permute{_Pos...};
+            __bubble_sort_by_strides(__permute);
+
+            // check that this permutations represents a growing set
+            for (rank_type __i = 1; __i < __rank_; __i++)
+              if (static_cast<index_type>(__strides[__permute[__i]]) <
+                  static_cast<index_type>(__strides[__permute[__i - 1]]) * __extents_.extent(__permute[__i - 1]))
+                return false;
+            return true;
+          }(make_index_sequence<__rank_>())),
+          "layout_stride::mapping ctor: the provided extents and strides lead to a non-unique mapping");
+    }
+  }
+
+  template <class _OtherIndexType>
+    requires(is_convertible_v<const _OtherIndexType&, index_type> &&
+             is_nothrow_constructible_v<index_type, const _OtherIndexType&>)
+  _LIBCPP_HIDE_FROM_ABI constexpr mapping(const extents_type& __ext,
+                                          const array<_OtherIndexType, __rank_>& __strides) noexcept
+      : mapping(__ext, span(__strides)) {}
+
+  template <class _StridedLayoutMapping>
+    requires(__mdspan_detail::__layout_mapping_alike<_StridedLayoutMapping> &&
+             is_constructible_v<extents_type, typename _StridedLayoutMapping::extents_type> &&
+             _StridedLayoutMapping::is_always_unique() && _StridedLayoutMapping::is_always_strided())
+  _LIBCPP_HIDE_FROM_ABI constexpr explicit(
+      !(is_convertible_v<typename _StridedLayoutMapping::extents_type, extents_type> &&
+        (__mdspan_detail::__is_mapping_of<layout_left, _StridedLayoutMapping> ||
+         __mdspan_detail::__is_mapping_of<layout_right, _StridedLayoutMapping> ||
+         __mdspan_detail::__is_mapping_of<layout_stride, _StridedLayoutMapping>)))
+      mapping(const _StridedLayoutMapping& __other) noexcept
+      : __extents_(__other.extents()), __strides_([&]<size_t... _Pos>(index_sequence<_Pos...>) {
+          // stride() only compiles for rank > 0
+          if constexpr (__rank_ > 0) {
+            return __mdspan_detail::__possibly_empty_array<index_type, __rank_>{
+                static_cast<index_type>(__other.stride(_Pos))...};
+          } else {
+            return __mdspan_detail::__possibly_empty_array<index_type, 0>{};
+          }
+        }(make_index_sequence<__rank_>())) {
+    // stride() only compiles for rank > 0
+    if constexpr (__rank_ > 0) {
+      _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+          ([&]<size_t... _Pos>(index_sequence<_Pos...>) {
+            return ((static_cast<index_type>(__other.stride(_Pos)) > static_cast<index_type>(0)) && ... && true);
+          }(make_index_sequence<__rank_>())),
+          "layout_stride::mapping converting ctor: all strides must be greater than 0");
+    }
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __mdspan_detail::__is_representable_as<index_type>(__other.required_span_size()),
+        "layout_stride::mapping converting ctor: other.required_span_size() must be representable as index_type.");
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(static_cast<index_type>(0) == __offset(__other),
+                                        "layout_stride::mapping converting ctor: base offset of mapping must be zero.");
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr mapping& operator=(const mapping&) noexcept = default;
+
+  // [mdspan.layout.stride.obs], observers
+  _LIBCPP_HIDE_FROM_ABI constexpr const extents_type& extents() const noexcept { return __extents_; }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr array<index_type, __rank_> strides() const noexcept {
+    return [&]<size_t... _Pos>(index_sequence<_Pos...>) {
+      return array<index_type, __rank_>{__strides_[_Pos]...};
+    }(make_index_sequence<__rank_>());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr index_type required_span_size() const noexcept {
+    if constexpr (__rank_ == 0) {
+      return static_cast<index_type>(1);
+    } else {
+      return [&]<size_t... _Pos>(index_sequence<_Pos...>) {
+        if ((__extents_.extent(_Pos) * ... * 1) == 0)
+          return static_cast<index_type>(0);
+        else
+          return static_cast<index_type>(
+              static_cast<index_type>(1) +
+              (((__extents_.extent(_Pos) - static_cast<index_type>(1)) * __strides_[_Pos]) + ... +
+               static_cast<index_type>(0)));
+      }(make_index_sequence<__rank_>());
+    }
+  }
+
+  template <class... _Indices>
+    requires((sizeof...(_Indices) == __rank_) && (is_convertible_v<_Indices, index_type> && ...) &&
+             (is_nothrow_constructible_v<index_type, _Indices> && ...))
+  _LIBCPP_HIDE_FROM_ABI constexpr index_type operator()(_Indices... __idx) const noexcept {
+    // Mappings are generally meant to be used for accessing allocations and are meant to guarantee to never
+    // return a value exceeding required_span_size(), which is used to know how large an allocation one needs
+    // Thus, this is a canonical point in multi-dimensional data structures to make invalid element access checks
+    // However, mdspan does check this on its own, so for now we avoid double checking in hardened mode
+    _LIBCPP_ASSERT_UNCATEGORIZED(__mdspan_detail::__is_multidimensional_index_in(__extents_, __idx...),
+                                 "layout_stride::mapping: out of bounds indexing");
+    return [&]<size_t... _Pos>(index_sequence<_Pos...>) {
+      return ((static_cast<index_type>(__idx) * __strides_[_Pos]) + ... + index_type(0));
+    }(make_index_sequence<sizeof...(_Indices)>());
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool is_always_unique() noexcept { return true; }
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool is_always_exhaustive() noexcept { return false; }
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool is_always_strided() noexcept { return true; }
+
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool is_unique() noexcept { return true; }
+  // The answer of this function is fairly complex in the case where one or more
+  // extents are zero.
+  // Technically it is meaningless to query is_exhaustive() in that case, but unfortunately
+  // the way the standard defines this function, we can't give a simple true or false then.
+  _LIBCPP_HIDE_FROM_ABI constexpr bool is_exhaustive() const noexcept {
+    if constexpr (__rank_ == 0)
+      return true;
+    else {
+      index_type __span_size = required_span_size();
+      if (__span_size == static_cast<index_type>(0)) {
+        if constexpr (__rank_ == 1)
+          return __strides_[0] == 1;
+        else {
+          rank_type __r_largest = 0;
+          for (rank_type __r = 1; __r < __rank_; __r++)
+            if (__strides_[__r] > __strides_[__r_largest])
+              __r_largest = __r;
+          for (rank_type __r = 0; __r < __rank_; __r++)
+            if (__extents_.extent(__r) == 0 && __r != __r_largest)
+              return false;
+          return true;
+        }
+      } else {
+        return required_span_size() == [&]<size_t... _Pos>(index_sequence<_Pos...>) {
+          return (__extents_.extent(_Pos) * ... * static_cast<index_type>(1));
+        }(make_index_sequence<__rank_>());
+      }
+    }
+  }
+  _LIBCPP_HIDE_FROM_ABI static constexpr bool is_strided() noexcept { return true; }
+
+  // according to the standard layout_stride does not have a constraint on stride(r) for rank>0
+  // it still has the precondition though
+  _LIBCPP_HIDE_FROM_ABI constexpr index_type stride(rank_type __r) const noexcept {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__r < __rank_, "layout_stride::mapping::stride(): invalid rank index");
+    return __strides_[__r];
+  }
+
+  template <class _OtherMapping>
+    requires(__mdspan_detail::__layout_mapping_alike<_OtherMapping> &&
+             (_OtherMapping::extents_type::rank() == __rank_) && _OtherMapping::is_always_strided())
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const mapping& __lhs, const _OtherMapping& __rhs) noexcept {
+    if (__offset(__rhs))
+      return false;
+    if constexpr (__rank_ == 0)
+      return true;
+    else {
+      return __lhs.extents() == __rhs.extents() && [&]<size_t... _Pos>(index_sequence<_Pos...>) {
+        // avoid warning when comparing signed and unsigner integers and pick the wider of two types
+        using _CommonType = common_type_t<index_type, typename _OtherMapping::index_type>;
+        return ((static_cast<_CommonType>(__lhs.stride(_Pos)) == static_cast<_CommonType>(__rhs.stride(_Pos))) && ... &&
+                true);
+      }(make_index_sequence<__rank_>());
+    }
+  }
+
+private:
+  _LIBCPP_NO_UNIQUE_ADDRESS extents_type __extents_{};
+  _LIBCPP_NO_UNIQUE_ADDRESS __mdspan_detail::__possibly_empty_array<index_type, __rank_> __strides_{};
+};
+
+#endif // _LIBCPP_STD_VER >= 23
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___MDSPAN_LAYOUT_STRIDE_H
diff --git a/libcxx/include/__memory/ranges_uninitialized_algorithms.h b/libcxx/include/__memory/ranges_uninitialized_algorithms.h
index 01c3e01003..108f653753 100644
--- a/libcxx/include/__memory/ranges_uninitialized_algorithms.h
+++ b/libcxx/include/__memory/ranges_uninitialized_algorithms.h
@@ -196,8 +196,9 @@ struct __fn {
   operator()(_InputIterator __ifirst, _Sentinel1 __ilast, _OutputIterator __ofirst, _Sentinel2 __olast) const {
     using _ValueType = remove_reference_t<iter_reference_t<_OutputIterator>>;
 
-    auto __result = _VSTD::__uninitialized_copy<_ValueType>(_VSTD::move(__ifirst), _VSTD::move(__ilast),
-                                                            _VSTD::move(__ofirst), _VSTD::move(__olast));
+    auto __stop_copying = [&__olast](auto&& __out_iter) { return __out_iter == __olast; };
+    auto __result       = std::__uninitialized_copy<_ValueType>(
+        std::move(__ifirst), std::move(__ilast), std::move(__ofirst), __stop_copying);
     return {_VSTD::move(__result.first), _VSTD::move(__result.second)};
   }
 
@@ -232,8 +233,9 @@ struct __fn {
   operator()(_InputIterator __ifirst, iter_difference_t<_InputIterator> __n,
              _OutputIterator __ofirst, _Sentinel __olast) const {
     using _ValueType = remove_reference_t<iter_reference_t<_OutputIterator>>;
-    auto __result = _VSTD::__uninitialized_copy_n<_ValueType>(_VSTD::move(__ifirst), __n,
-                                                              _VSTD::move(__ofirst), _VSTD::move(__olast));
+    auto __stop_copying = [&__olast](auto&& __out_iter) { return __out_iter == __olast; };
+    auto __result =
+        std::__uninitialized_copy_n<_ValueType>(std::move(__ifirst), __n, std::move(__ofirst), __stop_copying);
     return {_VSTD::move(__result.first), _VSTD::move(__result.second)};
   }
 };
@@ -261,8 +263,9 @@ struct __fn {
   operator()(_InputIterator __ifirst, _Sentinel1 __ilast, _OutputIterator __ofirst, _Sentinel2 __olast) const {
     using _ValueType = remove_reference_t<iter_reference_t<_OutputIterator>>;
     auto __iter_move = [](auto&& __iter) -> decltype(auto) { return ranges::iter_move(__iter); };
-    auto __result = _VSTD::__uninitialized_move<_ValueType>(_VSTD::move(__ifirst), _VSTD::move(__ilast),
-                                                            _VSTD::move(__ofirst), _VSTD::move(__olast), __iter_move);
+    auto __stop_moving = [&__olast](auto&& __out_iter) { return __out_iter == __olast; };
+    auto __result      = std::__uninitialized_move<_ValueType>(
+        std::move(__ifirst), std::move(__ilast), std::move(__ofirst), __stop_moving, __iter_move);
     return {_VSTD::move(__result.first), _VSTD::move(__result.second)};
   }
 
@@ -298,8 +301,9 @@ struct __fn {
              _OutputIterator __ofirst, _Sentinel __olast) const {
     using _ValueType = remove_reference_t<iter_reference_t<_OutputIterator>>;
     auto __iter_move = [](auto&& __iter) -> decltype(auto) { return ranges::iter_move(__iter); };
-    auto __result = _VSTD::__uninitialized_move_n<_ValueType>(_VSTD::move(__ifirst), __n,
-                                                              _VSTD::move(__ofirst), _VSTD::move(__olast), __iter_move);
+    auto __stop_moving = [&__olast](auto&& __out_iter) { return __out_iter == __olast; };
+    auto __result      = std::__uninitialized_move_n<_ValueType>(
+        std::move(__ifirst), __n, std::move(__ofirst), __stop_moving, __iter_move);
     return {_VSTD::move(__result.first), _VSTD::move(__result.second)};
   }
 };
diff --git a/libcxx/include/__memory/uninitialized_algorithms.h b/libcxx/include/__memory/uninitialized_algorithms.h
index 2b68df8e6d..af0d83c97c 100644
--- a/libcxx/include/__memory/uninitialized_algorithms.h
+++ b/libcxx/include/__memory/uninitialized_algorithms.h
@@ -44,26 +44,23 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-// This is a simplified version of C++20 `unreachable_sentinel` that doesn't use concepts and thus can be used in any
-// language mode.
-struct __unreachable_sentinel {
-  template <class _Iter>
-  _LIBCPP_HIDE_FROM_ABI friend _LIBCPP_CONSTEXPR bool operator!=(const _Iter&, __unreachable_sentinel) _NOEXCEPT {
-    return true;
+struct __always_false {
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool operator()(_Args&&...) const _NOEXCEPT {
+    return false;
   }
 };
 
 // uninitialized_copy
 
-template <class _ValueType, class _InputIterator, class _Sentinel1, class _ForwardIterator, class _Sentinel2>
-inline _LIBCPP_HIDE_FROM_ABI pair<_InputIterator, _ForwardIterator>
-__uninitialized_copy(_InputIterator __ifirst, _Sentinel1 __ilast,
-                     _ForwardIterator __ofirst, _Sentinel2 __olast) {
+template <class _ValueType, class _InputIterator, class _Sentinel1, class _ForwardIterator, class _EndPredicate>
+inline _LIBCPP_HIDE_FROM_ABI pair<_InputIterator, _ForwardIterator> __uninitialized_copy(
+    _InputIterator __ifirst, _Sentinel1 __ilast, _ForwardIterator __ofirst, _EndPredicate __stop_copying) {
   _ForwardIterator __idx = __ofirst;
 #ifndef _LIBCPP_HAS_NO_EXCEPTIONS
   try {
 #endif
-    for (; __ifirst != __ilast && __idx != __olast; ++__ifirst, (void)++__idx)
+    for (; __ifirst != __ilast && !__stop_copying(__idx); ++__ifirst, (void)++__idx)
       ::new (_VSTD::__voidify(*__idx)) _ValueType(*__ifirst);
 #ifndef _LIBCPP_HAS_NO_EXCEPTIONS
   } catch (...) {
@@ -80,22 +77,21 @@ _LIBCPP_HIDE_FROM_ABI
 _ForwardIterator uninitialized_copy(_InputIterator __ifirst, _InputIterator __ilast,
                                     _ForwardIterator __ofirst) {
   typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
-  auto __result = _VSTD::__uninitialized_copy<_ValueType>(_VSTD::move(__ifirst), _VSTD::move(__ilast),
-                                                          _VSTD::move(__ofirst), __unreachable_sentinel());
+  auto __result = std::__uninitialized_copy<_ValueType>(
+      std::move(__ifirst), std::move(__ilast), std::move(__ofirst), __always_false());
   return _VSTD::move(__result.second);
 }
 
 // uninitialized_copy_n
 
-template <class _ValueType, class _InputIterator, class _Size, class _ForwardIterator, class _Sentinel>
-inline _LIBCPP_HIDE_FROM_ABI pair<_InputIterator, _ForwardIterator>
-__uninitialized_copy_n(_InputIterator __ifirst, _Size __n,
-                       _ForwardIterator __ofirst, _Sentinel __olast) {
+template <class _ValueType, class _InputIterator, class _Size, class _ForwardIterator, class _EndPredicate>
+inline _LIBCPP_HIDE_FROM_ABI pair<_InputIterator, _ForwardIterator> __uninitialized_copy_n(
+    _InputIterator __ifirst, _Size __n, _ForwardIterator __ofirst, _EndPredicate __stop_copying) {
   _ForwardIterator __idx = __ofirst;
 #ifndef _LIBCPP_HAS_NO_EXCEPTIONS
   try {
 #endif
-    for (; __n > 0 && __idx != __olast; ++__ifirst, (void)++__idx, (void)--__n)
+    for (; __n > 0 && !__stop_copying(__idx); ++__ifirst, (void)++__idx, (void)--__n)
       ::new (_VSTD::__voidify(*__idx)) _ValueType(*__ifirst);
 #ifndef _LIBCPP_HAS_NO_EXCEPTIONS
   } catch (...) {
@@ -111,8 +107,8 @@ template <class _InputIterator, class _Size, class _ForwardIterator>
 inline _LIBCPP_HIDE_FROM_ABI _ForwardIterator uninitialized_copy_n(_InputIterator __ifirst, _Size __n,
                                                                    _ForwardIterator __ofirst) {
   typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
-  auto __result = _VSTD::__uninitialized_copy_n<_ValueType>(_VSTD::move(__ifirst), __n, _VSTD::move(__ofirst),
-                                                            __unreachable_sentinel());
+  auto __result =
+      std::__uninitialized_copy_n<_ValueType>(std::move(__ifirst), __n, std::move(__ofirst), __always_false());
   return _VSTD::move(__result.second);
 }
 
@@ -300,16 +296,23 @@ _ForwardIterator uninitialized_value_construct_n(_ForwardIterator __first, _Size
 
 // uninitialized_move
 
-template <class _ValueType, class _InputIterator, class _Sentinel1, class _ForwardIterator, class _Sentinel2,
+template <class _ValueType,
+          class _InputIterator,
+          class _Sentinel1,
+          class _ForwardIterator,
+          class _EndPredicate,
           class _IterMove>
-inline _LIBCPP_HIDE_FROM_ABI pair<_InputIterator, _ForwardIterator>
-__uninitialized_move(_InputIterator __ifirst, _Sentinel1 __ilast,
-                     _ForwardIterator __ofirst, _Sentinel2 __olast, _IterMove __iter_move) {
+inline _LIBCPP_HIDE_FROM_ABI pair<_InputIterator, _ForwardIterator> __uninitialized_move(
+    _InputIterator __ifirst,
+    _Sentinel1 __ilast,
+    _ForwardIterator __ofirst,
+    _EndPredicate __stop_moving,
+    _IterMove __iter_move) {
   auto __idx = __ofirst;
 #ifndef _LIBCPP_HAS_NO_EXCEPTIONS
   try {
 #endif
-    for (; __ifirst != __ilast && __idx != __olast; ++__idx, (void)++__ifirst) {
+    for (; __ifirst != __ilast && !__stop_moving(__idx); ++__idx, (void)++__ifirst) {
       ::new (_VSTD::__voidify(*__idx)) _ValueType(__iter_move(__ifirst));
     }
 #ifndef _LIBCPP_HAS_NO_EXCEPTIONS
@@ -328,22 +331,26 @@ inline _LIBCPP_HIDE_FROM_ABI _ForwardIterator uninitialized_move(_InputIterator
   using _ValueType = typename iterator_traits<_ForwardIterator>::value_type;
   auto __iter_move = [](auto&& __iter) -> decltype(auto) { return _VSTD::move(*__iter); };
 
-  auto __result = _VSTD::__uninitialized_move<_ValueType>(_VSTD::move(__ifirst), _VSTD::move(__ilast),
-                                                          _VSTD::move(__ofirst), __unreachable_sentinel(), __iter_move);
+  auto __result = std::__uninitialized_move<_ValueType>(
+      std::move(__ifirst), std::move(__ilast), std::move(__ofirst), __always_false(), __iter_move);
   return _VSTD::move(__result.second);
 }
 
 // uninitialized_move_n
 
-template <class _ValueType, class _InputIterator, class _Size, class _ForwardIterator, class _Sentinel, class _IterMove>
-inline _LIBCPP_HIDE_FROM_ABI pair<_InputIterator, _ForwardIterator>
-__uninitialized_move_n(_InputIterator __ifirst, _Size __n,
-                       _ForwardIterator __ofirst, _Sentinel __olast, _IterMove __iter_move) {
+template <class _ValueType,
+          class _InputIterator,
+          class _Size,
+          class _ForwardIterator,
+          class _EndPredicate,
+          class _IterMove>
+inline _LIBCPP_HIDE_FROM_ABI pair<_InputIterator, _ForwardIterator> __uninitialized_move_n(
+    _InputIterator __ifirst, _Size __n, _ForwardIterator __ofirst, _EndPredicate __stop_moving, _IterMove __iter_move) {
   auto __idx = __ofirst;
 #ifndef _LIBCPP_HAS_NO_EXCEPTIONS
   try {
 #endif
-    for (; __n > 0 && __idx != __olast; ++__idx, (void)++__ifirst, --__n)
+    for (; __n > 0 && !__stop_moving(__idx); ++__idx, (void)++__ifirst, --__n)
       ::new (_VSTD::__voidify(*__idx)) _ValueType(__iter_move(__ifirst));
 #ifndef _LIBCPP_HAS_NO_EXCEPTIONS
   } catch (...) {
@@ -361,8 +368,8 @@ uninitialized_move_n(_InputIterator __ifirst, _Size __n, _ForwardIterator __ofir
   using _ValueType = typename iterator_traits<_ForwardIterator>::value_type;
   auto __iter_move = [](auto&& __iter) -> decltype(auto) { return _VSTD::move(*__iter); };
 
-  return _VSTD::__uninitialized_move_n<_ValueType>(_VSTD::move(__ifirst), __n, _VSTD::move(__ofirst),
-                                                   __unreachable_sentinel(), __iter_move);
+  return std::__uninitialized_move_n<_ValueType>(
+      std::move(__ifirst), __n, std::move(__ofirst), __always_false(), __iter_move);
 }
 
 // TODO: Rewrite this to iterate left to right and use reverse_iterators when calling
diff --git a/libcxx/include/__node_handle b/libcxx/include/__node_handle
index cc4eaf73c0..b3cc3619dd 100644
--- a/libcxx/include/__node_handle
+++ b/libcxx/include/__node_handle
@@ -209,7 +209,7 @@ struct __set_node_handle_specifics
     _LIBCPP_INLINE_VISIBILITY
     value_type& value() const
     {
-        return static_cast<_Derived const*>(this)->__ptr_->__value_;
+        return static_cast<_Derived const*>(this)->__ptr_->__get_value();
     }
 };
 
@@ -223,14 +223,14 @@ struct __map_node_handle_specifics
     key_type& key() const
     {
         return static_cast<_Derived const*>(this)->
-            __ptr_->__value_.__ref().first;
+            __ptr_->__get_value().__ref().first;
     }
 
     _LIBCPP_INLINE_VISIBILITY
     mapped_type& mapped() const
     {
         return static_cast<_Derived const*>(this)->
-            __ptr_->__value_.__ref().second;
+            __ptr_->__get_value().__ref().second;
     }
 };
 
diff --git a/libcxx/include/__ranges/to.h b/libcxx/include/__ranges/to.h
index a519662e75..cf162100ee 100644
--- a/libcxx/include/__ranges/to.h
+++ b/libcxx/include/__ranges/to.h
@@ -207,13 +207,11 @@ _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr auto to(_Args&&... __args)
   static_assert(
       !is_volatile_v<_Container>, "The target container cannot be volatile-qualified, please remove the volatile");
 
-  auto __to_func = []<input_range _Range, class... _Tail>(_Range && __range, _Tail && ... __tail)
+  auto __to_func = []<input_range _Range, class... _Tail>(_Range&& __range, _Tail&&... __tail)
     requires requires { //
       /**/ ranges::to<_Container>(std::forward<_Range>(__range), std::forward<_Tail>(__tail)...);
     }
-  {
-    return ranges::to<_Container>(std::forward<_Range>(__range), std::forward<_Tail>(__tail)...);
-  };
+  { return ranges::to<_Container>(std::forward<_Range>(__range), std::forward<_Tail>(__tail)...); };
 
   return __range_adaptor_closure_t(std::__bind_back(__to_func, std::forward<_Args>(__args)...));
 }
diff --git a/libcxx/include/__std_clang_module b/libcxx/include/__std_clang_module
index 2644ea98b4..e2e9e85ffc 100644
--- a/libcxx/include/__std_clang_module
+++ b/libcxx/include/__std_clang_module
@@ -169,7 +169,6 @@
 #  include <semaphore>
 #endif
 #include <set>
-#include <setjmp.h>
 #if !defined(_LIBCPP_HAS_NO_THREADS)
 #  include <shared_mutex>
 #endif
diff --git a/libcxx/include/__tree b/libcxx/include/__tree
index 54ce71e442..eccadea8a0 100644
--- a/libcxx/include/__tree
+++ b/libcxx/include/__tree
@@ -774,6 +774,8 @@ public:
 
     __node_value_type __value_;
 
+    _LIBCPP_HIDE_FROM_ABI _Tp& __get_value() { return __value_; }
+
 private:
   ~__tree_node() = delete;
   __tree_node(__tree_node const&) = delete;
diff --git a/libcxx/include/__type_traits/promote.h b/libcxx/include/__type_traits/promote.h
index 018bd69df2..e22b4a422c 100644
--- a/libcxx/include/__type_traits/promote.h
+++ b/libcxx/include/__type_traits/promote.h
@@ -50,7 +50,7 @@ struct __numeric_type<void> {
 template <class _A1,
           class _A2 = void,
           class _A3 = void,
-          bool      = __numeric_type<_A1>::value&& __numeric_type<_A2>::value&& __numeric_type<_A3>::value>
+          bool      = __numeric_type<_A1>::value && __numeric_type<_A2>::value && __numeric_type<_A3>::value>
 class __promote_imp {
 public:
   static const bool value = false;
diff --git a/libcxx/include/algorithm b/libcxx/include/algorithm
index 69ba9537dd..ac335cb39e 100644
--- a/libcxx/include/algorithm
+++ b/libcxx/include/algorithm
@@ -1833,6 +1833,7 @@ template <class BidirectionalIterator, class Compare>
 #include <__algorithm/pstl_generate.h>
 #include <__algorithm/pstl_is_partitioned.h>
 #include <__algorithm/pstl_merge.h>
+#include <__algorithm/pstl_move.h>
 #include <__algorithm/pstl_replace.h>
 #include <__algorithm/pstl_sort.h>
 #include <__algorithm/pstl_stable_sort.h>
diff --git a/libcxx/include/csetjmp b/libcxx/include/csetjmp
index 4c64e8327e..d219c8e6cb 100644
--- a/libcxx/include/csetjmp
+++ b/libcxx/include/csetjmp
@@ -35,14 +35,6 @@ void longjmp(jmp_buf env, int val);
 
 #include <setjmp.h>
 
-#ifndef _LIBCPP_SETJMP_H
-#   error <csetjmp> tried including <setjmp.h> but didn't find libc++'s <setjmp.h> header. \
-          This usually means that your header search paths are not configured properly. \
-          The header search paths should contain the C++ Standard Library headers before \
-          any C Standard Library, and you are probably using compiler flags that make that \
-          not be the case.
-#endif
-
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
 #endif
diff --git a/libcxx/include/experimental/__simd/scalar.h b/libcxx/include/experimental/__simd/scalar.h
index 341292612e..479809392f 100644
--- a/libcxx/include/experimental/__simd/scalar.h
+++ b/libcxx/include/experimental/__simd/scalar.h
@@ -46,6 +46,11 @@ struct __simd_operations<_Tp, simd_abi::__scalar> {
   using _MaskStorage = __mask_storage<_Tp, simd_abi::__scalar>;
 
   static _LIBCPP_HIDE_FROM_ABI _SimdStorage __broadcast(_Tp __v) noexcept { return {__v}; }
+
+  template <class _Generator>
+  static _LIBCPP_HIDE_FROM_ABI _SimdStorage __generate(_Generator&& __g) noexcept {
+    return {__g(std::integral_constant<size_t, 0>())};
+  }
 };
 
 template <class _Tp>
diff --git a/libcxx/include/experimental/__simd/simd.h b/libcxx/include/experimental/__simd/simd.h
index 30856f4b03..954f94c907 100644
--- a/libcxx/include/experimental/__simd/simd.h
+++ b/libcxx/include/experimental/__simd/simd.h
@@ -44,6 +44,10 @@ class simd {
   template <class _Up, enable_if_t<__can_broadcast_v<value_type, __remove_cvref_t<_Up>>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI simd(_Up&& __v) noexcept : __s_(_Impl::__broadcast(static_cast<value_type>(__v))) {}
 
+  // generator constructor
+  template <class _Generator, enable_if_t<__can_generate_v<value_type, _Generator, size()>, int> = 0>
+  explicit _LIBCPP_HIDE_FROM_ABI simd(_Generator&& __g) : __s_(_Impl::__generate(std::forward<_Generator>(__g))) {}
+
   // scalar access [simd.subscr]
   // Add operator[] temporarily to test braodcast. Add test for it in later patch.
   _LIBCPP_HIDE_FROM_ABI value_type operator[](size_t __i) const { return __s_.__get(__i); }
diff --git a/libcxx/include/experimental/__simd/utility.h b/libcxx/include/experimental/__simd/utility.h
index 0dba345e05..847d006629 100644
--- a/libcxx/include/experimental/__simd/utility.h
+++ b/libcxx/include/experimental/__simd/utility.h
@@ -19,6 +19,7 @@
 #include <__type_traits/is_volatile.h>
 #include <__type_traits/void_t.h>
 #include <__utility/declval.h>
+#include <__utility/integer_sequence.h>
 #include <cstdint>
 #include <limits>
 
@@ -71,6 +72,25 @@ inline constexpr bool __can_broadcast_v =
     (!__is_vectorizable_v<_Up> && is_convertible_v<_Up, _Tp>) || is_same_v<_Up, int> ||
     (is_same_v<_Up, unsigned int> && is_unsigned_v<_Tp>);
 
+template <class _Tp, class _Generator, std::size_t _Idx, class = void>
+inline constexpr bool __is_well_formed = false;
+
+template <class _Tp, class _Generator, std::size_t _Idx>
+inline constexpr bool
+    __is_well_formed<_Tp,
+                     _Generator,
+                     _Idx,
+                     std::void_t<decltype(std::declval<_Generator>()(integral_constant<size_t, _Idx>()))>> =
+        __can_broadcast_v<_Tp, decltype(std::declval<_Generator>()(integral_constant<size_t, _Idx>()))>;
+
+template <class _Tp, class _Generator, std::size_t... _Idxes>
+_LIBCPP_HIDE_FROM_ABI constexpr bool __can_generate(index_sequence<_Idxes...>) {
+  return (true && ... && __is_well_formed<_Tp, _Generator, _Idxes>);
+}
+
+template <class _Tp, class _Generator, std::size_t _Size>
+inline constexpr bool __can_generate_v = experimental::__can_generate<_Tp, _Generator>(make_index_sequence<_Size>());
+
 } // namespace parallelism_v2
 _LIBCPP_END_NAMESPACE_EXPERIMENTAL
 
diff --git a/libcxx/include/experimental/__simd/vec_ext.h b/libcxx/include/experimental/__simd/vec_ext.h
index 44a1f481ab..4b23bfe384 100644
--- a/libcxx/include/experimental/__simd/vec_ext.h
+++ b/libcxx/include/experimental/__simd/vec_ext.h
@@ -11,6 +11,7 @@
 #define _LIBCPP_EXPERIMENTAL___SIMD_VEC_EXT_H
 
 #include <__bit/bit_ceil.h>
+#include <__utility/forward.h>
 #include <cstddef>
 #include <experimental/__simd/internal_declaration.h>
 #include <experimental/__simd/utility.h>
@@ -56,6 +57,16 @@ struct __simd_operations<_Tp, simd_abi::__vec_ext<_Np>> {
     }
     return __result;
   }
+
+  template <class _Generator, size_t... _Is>
+  static _LIBCPP_HIDE_FROM_ABI _SimdStorage __generate_init(_Generator&& __g, std::index_sequence<_Is...>) {
+    return _SimdStorage{{__g(std::integral_constant<size_t, _Is>())...}};
+  }
+
+  template <class _Generator>
+  static _LIBCPP_HIDE_FROM_ABI _SimdStorage __generate(_Generator&& __g) noexcept {
+    return __generate_init(std::forward<_Generator>(__g), std::make_index_sequence<_Np>());
+  }
 };
 
 template <class _Tp, int _Np>
diff --git a/libcxx/include/ext/hash_map b/libcxx/include/ext/hash_map
index 116b6a72f2..de963675eb 100644
--- a/libcxx/include/ext/hash_map
+++ b/libcxx/include/ext/hash_map
@@ -357,9 +357,9 @@ public:
     void operator()(pointer __p)
     {
         if (__second_constructed)
-            __alloc_traits::destroy(__na_, _VSTD::addressof(__p->__value_.second));
+            __alloc_traits::destroy(__na_, _VSTD::addressof(__p->__get_value().second));
         if (__first_constructed)
-            __alloc_traits::destroy(__na_, _VSTD::addressof(__p->__value_.first));
+            __alloc_traits::destroy(__na_, _VSTD::addressof(__p->__get_value().first));
         if (__p)
             __alloc_traits::deallocate(__na_, __p, 1);
     }
@@ -667,9 +667,9 @@ hash_map<_Key, _Tp, _Hash, _Pred, _Alloc>::__construct_node(const key_type& __k)
 {
     __node_allocator& __na = __table_.__node_alloc();
     __node_holder __h(__node_traits::allocate(__na, 1), _Dp(__na));
-    __node_traits::construct(__na, _VSTD::addressof(__h->__value_.first), __k);
+    __node_traits::construct(__na, _VSTD::addressof(__h->__get_value().first), __k);
     __h.get_deleter().__first_constructed = true;
-    __node_traits::construct(__na, _VSTD::addressof(__h->__value_.second));
+    __node_traits::construct(__na, _VSTD::addressof(__h->__get_value().second));
     __h.get_deleter().__second_constructed = true;
     return __h;
 }
diff --git a/libcxx/include/forward_list b/libcxx/include/forward_list
index 75ac685cc0..09338ab695 100644
--- a/libcxx/include/forward_list
+++ b/libcxx/include/forward_list
@@ -211,6 +211,7 @@ template <class T, class Allocator, class Predicate>
 #include <__memory/allocator.h>
 #include <__memory/allocator_traits.h>
 #include <__memory/compressed_pair.h>
+#include <__memory/construct_at.h>
 #include <__memory/pointer_traits.h>
 #include <__memory/swap_allocator.h>
 #include <__memory_resource/polymorphic_allocator.h>
@@ -230,6 +231,7 @@ template <class T, class Allocator, class Predicate>
 #include <__utility/forward.h>
 #include <__utility/move.h>
 #include <limits>
+#include <new> // __launder
 #include <version>
 
 // standard-mandated includes
@@ -318,17 +320,35 @@ template <class _Tp, class _VoidPtr>
 using __begin_node_of = __forward_begin_node<__rebind_pointer_t<_VoidPtr, __forward_list_node<_Tp, _VoidPtr> > >;
 
 template <class _Tp, class _VoidPtr>
-struct _LIBCPP_STANDALONE_DEBUG __forward_list_node
+struct __forward_list_node
     : public __begin_node_of<_Tp, _VoidPtr>
 {
     typedef _Tp value_type;
     typedef __begin_node_of<_Tp, _VoidPtr> _Base;
     typedef typename _Base::pointer _NodePtr;
 
-    value_type __value_;
+    // We allow starting the lifetime of nodes without initializing the value held by the node,
+    // since that is handled by the list itself in order to be allocator-aware.
+#ifndef _LIBCPP_CXX03_LANG
+private:
+    union {
+        _Tp __value_;
+    };
+
+public:
+    _LIBCPP_HIDE_FROM_ABI _Tp& __get_value() { return __value_; }
+#else
+private:
+    _ALIGNAS_TYPE(_Tp) char __buffer_[sizeof(_Tp)];
+
+public:
+    _LIBCPP_HIDE_FROM_ABI _Tp& __get_value() {
+        return *std::__launder(reinterpret_cast<_Tp*>(&__buffer_));
+    }
+#endif
 
-    _LIBCPP_HIDE_FROM_ABI __forward_list_node() = default;
-    _LIBCPP_HIDE_FROM_ABI __forward_list_node(const value_type& __v, _NodePtr __next) : _Base(__next), __value_(__v) {}
+    _LIBCPP_HIDE_FROM_ABI explicit __forward_list_node(_NodePtr __next) : _Base(__next) {}
+    _LIBCPP_HIDE_FROM_ABI ~__forward_list_node() {}
 };
 
 
@@ -383,10 +403,10 @@ public:
     __forward_list_iterator() _NOEXCEPT : __ptr_(nullptr) {}
 
     _LIBCPP_INLINE_VISIBILITY
-    reference operator*() const {return __get_unsafe_node_pointer()->__value_;}
+    reference operator*() const {return __get_unsafe_node_pointer()->__get_value();}
     _LIBCPP_INLINE_VISIBILITY
     pointer operator->() const {
-        return pointer_traits<pointer>::pointer_to(__get_unsafe_node_pointer()->__value_);
+        return pointer_traits<pointer>::pointer_to(__get_unsafe_node_pointer()->__get_value());
     }
 
     _LIBCPP_INLINE_VISIBILITY
@@ -468,10 +488,10 @@ public:
         : __ptr_(__p.__ptr_) {}
 
     _LIBCPP_INLINE_VISIBILITY
-    reference operator*() const {return __get_unsafe_node_pointer()->__value_;}
+    reference operator*() const {return __get_unsafe_node_pointer()->__get_value();}
     _LIBCPP_INLINE_VISIBILITY
     pointer operator->() const {return pointer_traits<pointer>::pointer_to(
-                __get_unsafe_node_pointer()->__value_);}
+                __get_unsafe_node_pointer()->__get_value());}
 
     _LIBCPP_INLINE_VISIBILITY
     __forward_list_const_iterator& operator++()
@@ -577,15 +597,26 @@ protected:
     _LIBCPP_HIDE_FROM_ABI __node_pointer __create_node(__node_pointer __next, _Args&& ...__args) {
         __node_allocator& __a = __alloc();
         __allocation_guard<__node_allocator> __guard(__a, 1);
-        __guard.__get()->__next_ = __next;
-        __node_traits::construct(__a, std::addressof(__guard.__get()->__value_), std::forward<_Args>(__args)...);
+        // Begin the lifetime of the node itself. Note that this doesn't begin the lifetime of the value
+        // held inside the node, since we need to use the allocator's construct() method for that.
+        //
+        // We don't use the allocator's construct() method to construct the node itself since the
+        // Cpp17FooInsertable named requirements don't require the allocator's construct() method
+        // to work on anything other than the value_type.
+        std::__construct_at(std::addressof(*__guard.__get()), __next);
+
+        // Now construct the value_type using the allocator's construct() method.
+        __node_traits::construct(__a, std::addressof(__guard.__get()->__get_value()), std::forward<_Args>(__args)...);
         return __guard.__release_ptr();
     }
 
     template <class ..._Args>
     _LIBCPP_HIDE_FROM_ABI void __delete_node(__node_pointer __node) {
+        // For the same reason as above, we use the allocator's destroy() method for the value_type,
+        // but not for the node itself.
         __node_allocator& __a = __alloc();
-        __node_traits::destroy(__a, std::addressof(__node->__value_));
+        __node_traits::destroy(__a, std::addressof(__node->__get_value()));
+        std::__destroy_at(std::addressof(*__node));
         __node_traits::deallocate(__a, __node, 1);
     }
 
@@ -847,9 +878,9 @@ public:
     }
 
     _LIBCPP_INLINE_VISIBILITY
-    reference       front()       {return base::__before_begin()->__next_->__value_;}
+    reference       front()       {return base::__before_begin()->__next_->__get_value();}
     _LIBCPP_INLINE_VISIBILITY
-    const_reference front() const {return base::__before_begin()->__next_->__value_;}
+    const_reference front() const {return base::__before_begin()->__next_->__get_value();}
 
 #ifndef _LIBCPP_CXX03_LANG
 #if _LIBCPP_STD_VER >= 17
@@ -1227,7 +1258,7 @@ forward_list<_Tp, _Alloc>::emplace_front(_Args&&... __args)
 {
     base::__before_begin()->__next_ = this->__create_node(/* next = */base::__before_begin()->__next_, std::forward<_Args>(__args)...);
 #if _LIBCPP_STD_VER >= 17
-    return base::__before_begin()->__next_->__value_;
+    return base::__before_begin()->__next_->__get_value();
 #endif
 }
 
@@ -1556,7 +1587,7 @@ forward_list<_Tp, _Alloc>::remove(const value_type& __v)
     const iterator __e = end();
     for (iterator __i = before_begin(); __i.__get_begin()->__next_ != nullptr;)
     {
-        if (__i.__get_begin()->__next_->__value_ == __v)
+        if (__i.__get_begin()->__next_->__get_value() == __v)
         {
             ++__count_removed;
             iterator __j = _VSTD::next(__i, 2);
@@ -1584,7 +1615,7 @@ forward_list<_Tp, _Alloc>::remove_if(_Predicate __pred)
     const iterator __e = end();
     for (iterator __i = before_begin(); __i.__get_begin()->__next_ != nullptr;)
     {
-        if (__pred(__i.__get_begin()->__next_->__value_))
+        if (__pred(__i.__get_begin()->__next_->__get_value()))
         {
             ++__count_removed;
             iterator __j = _VSTD::next(__i, 2);
@@ -1647,11 +1678,11 @@ forward_list<_Tp, _Alloc>::__merge(__node_pointer __f1, __node_pointer __f2,
     if (__f2 == nullptr)
         return __f1;
     __node_pointer __r;
-    if (__comp(__f2->__value_, __f1->__value_))
+    if (__comp(__f2->__get_value(), __f1->__get_value()))
     {
         __node_pointer __t = __f2;
         while (__t->__next_ != nullptr &&
-                             __comp(__t->__next_->__value_, __f1->__value_))
+                             __comp(__t->__next_->__get_value(), __f1->__get_value()))
             __t = __t->__next_;
         __r = __f2;
         __f2 = __t->__next_;
@@ -1663,11 +1694,11 @@ forward_list<_Tp, _Alloc>::__merge(__node_pointer __f1, __node_pointer __f2,
     __f1 = __f1->__next_;
     while (__f1 != nullptr && __f2 != nullptr)
     {
-        if (__comp(__f2->__value_, __f1->__value_))
+        if (__comp(__f2->__get_value(), __f1->__get_value()))
         {
             __node_pointer __t = __f2;
             while (__t->__next_ != nullptr &&
-                                 __comp(__t->__next_->__value_, __f1->__value_))
+                                 __comp(__t->__next_->__get_value(), __f1->__get_value()))
                 __t = __t->__next_;
             __p->__next_ = __f2;
             __f2 = __t->__next_;
@@ -1703,7 +1734,7 @@ forward_list<_Tp, _Alloc>::__sort(__node_pointer __f1, difference_type __sz,
     case 1:
         return __f1;
     case 2:
-        if (__comp(__f1->__next_->__value_, __f1->__value_))
+        if (__comp(__f1->__next_->__get_value(), __f1->__get_value()))
         {
             __node_pointer __t = __f1->__next_;
             __t->__next_ = __f1;
diff --git a/libcxx/include/functional b/libcxx/include/functional
index 5c60b9fc39..5f9e8fa82a 100644
--- a/libcxx/include/functional
+++ b/libcxx/include/functional
@@ -34,7 +34,7 @@ struct binary_function
 template <class T>
 class reference_wrapper
     : public unary_function<T1, R> // if wrapping a unary functor
-    : public binary_function<T1, T2, R> // if wraping a binary functor
+    : public binary_function<T1, T2, R> // if wrapping a binary functor
 {
 public:
     // types
diff --git a/libcxx/include/list b/libcxx/include/list
index b02599bc3f..e5b524b883 100644
--- a/libcxx/include/list
+++ b/libcxx/include/list
@@ -217,6 +217,7 @@ template <class T, class Allocator, class Predicate>
 #include <__memory/allocator.h>
 #include <__memory/allocator_traits.h>
 #include <__memory/compressed_pair.h>
+#include <__memory/construct_at.h>
 #include <__memory/pointer_traits.h>
 #include <__memory/swap_allocator.h>
 #include <__memory_resource/polymorphic_allocator.h>
@@ -237,6 +238,7 @@ template <class T, class Allocator, class Predicate>
 #include <__utility/swap.h>
 #include <cstring>
 #include <limits>
+#include <new> // __launder
 #include <version>
 
 // standard-mandated includes
@@ -308,6 +310,9 @@ struct __list_node_base
     __list_node_base() : __prev_(_NodeTraits::__unsafe_link_pointer_cast(__self())),
                          __next_(_NodeTraits::__unsafe_link_pointer_cast(__self())) {}
 
+    _LIBCPP_HIDE_FROM_ABI explicit __list_node_base(__link_pointer __prev, __link_pointer __next)
+        : __prev_(__prev), __next_(__next) {}
+
     _LIBCPP_INLINE_VISIBILITY
     __base_pointer __self() {
         return pointer_traits<__base_pointer>::pointer_to(*this);
@@ -320,14 +325,35 @@ struct __list_node_base
 };
 
 template <class _Tp, class _VoidPtr>
-struct _LIBCPP_STANDALONE_DEBUG __list_node
+struct __list_node
     : public __list_node_base<_Tp, _VoidPtr>
 {
-    _Tp __value_;
+    // We allow starting the lifetime of nodes without initializing the value held by the node,
+    // since that is handled by the list itself in order to be allocator-aware.
+#ifndef _LIBCPP_CXX03_LANG
+private:
+    union {
+        _Tp __value_;
+    };
+
+public:
+    _LIBCPP_HIDE_FROM_ABI _Tp& __get_value() { return __value_; }
+#else
+private:
+    _ALIGNAS_TYPE(_Tp) char __buffer_[sizeof(_Tp)];
+
+public:
+    _LIBCPP_HIDE_FROM_ABI _Tp& __get_value() {
+        return *std::__launder(reinterpret_cast<_Tp*>(&__buffer_));
+    }
+#endif
 
     typedef __list_node_base<_Tp, _VoidPtr> __base;
     typedef typename __base::__link_pointer __link_pointer;
 
+    _LIBCPP_HIDE_FROM_ABI explicit __list_node(__link_pointer __prev, __link_pointer __next) : __base(__prev, __next) {}
+    _LIBCPP_HIDE_FROM_ABI ~__list_node() {}
+
     _LIBCPP_INLINE_VISIBILITY
     __link_pointer __as_link() {
         return static_cast<__link_pointer>(__base::__self());
@@ -370,12 +396,12 @@ public:
     _LIBCPP_INLINE_VISIBILITY
     reference operator*() const
     {
-        return __ptr_->__as_node()->__value_;
+        return __ptr_->__as_node()->__get_value();
     }
     _LIBCPP_INLINE_VISIBILITY
     pointer operator->() const
     {
-        return pointer_traits<pointer>::pointer_to(__ptr_->__as_node()->__value_);
+        return pointer_traits<pointer>::pointer_to(__ptr_->__as_node()->__get_value());
     }
 
     _LIBCPP_INLINE_VISIBILITY
@@ -442,12 +468,12 @@ public:
     _LIBCPP_INLINE_VISIBILITY
     reference operator*() const
     {
-        return __ptr_->__as_node()->__value_;
+        return __ptr_->__as_node()->__get_value();
     }
     _LIBCPP_INLINE_VISIBILITY
     pointer operator->() const
     {
-        return pointer_traits<pointer>::pointer_to(__ptr_->__as_node()->__value_);
+        return pointer_traits<pointer>::pointer_to(__ptr_->__as_node()->__get_value());
     }
 
     _LIBCPP_INLINE_VISIBILITY
@@ -600,16 +626,26 @@ protected:
     _LIBCPP_HIDE_FROM_ABI __node_pointer __create_node(__link_pointer __prev, __link_pointer __next, _Args&& ...__args) {
         __node_allocator& __alloc = __node_alloc();
         __allocation_guard<__node_allocator> __guard(__alloc, 1);
-        __guard.__get()->__prev_ = __prev;
-        __guard.__get()->__next_ = __next;
-        __node_alloc_traits::construct(__alloc, std::addressof(__guard.__get()->__value_), std::forward<_Args>(__args)...);
+        // Begin the lifetime of the node itself. Note that this doesn't begin the lifetime of the value
+        // held inside the node, since we need to use the allocator's construct() method for that.
+        //
+        // We don't use the allocator's construct() method to construct the node itself since the
+        // Cpp17FooInsertable named requirements don't require the allocator's construct() method
+        // to work on anything other than the value_type.
+        std::__construct_at(std::addressof(*__guard.__get()), __prev, __next);
+
+        // Now construct the value_type using the allocator's construct() method.
+        __node_alloc_traits::construct(__alloc, std::addressof(__guard.__get()->__get_value()), std::forward<_Args>(__args)...);
         return __guard.__release_ptr();
     }
 
     template <class ..._Args>
     _LIBCPP_HIDE_FROM_ABI void __delete_node(__node_pointer __node) {
+        // For the same reason as above, we use the allocator's destroy() method for the value_type,
+        // but not for the node itself.
         __node_allocator& __alloc = __node_alloc();
-        __node_alloc_traits::destroy(__alloc, std::addressof(__node->__value_));
+        __node_alloc_traits::destroy(__alloc, std::addressof(__node->__get_value()));
+        std::__destroy_at(std::addressof(*__node));
         __node_alloc_traits::deallocate(__alloc, __node, 1);
     }
 
@@ -894,25 +930,25 @@ public:
     reference front()
     {
         _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "list::front called on empty list");
-        return base::__end_.__next_->__as_node()->__value_;
+        return base::__end_.__next_->__as_node()->__get_value();
     }
     _LIBCPP_INLINE_VISIBILITY
     const_reference front() const
     {
         _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "list::front called on empty list");
-        return base::__end_.__next_->__as_node()->__value_;
+        return base::__end_.__next_->__as_node()->__get_value();
     }
     _LIBCPP_INLINE_VISIBILITY
     reference back()
     {
         _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "list::back called on empty list");
-        return base::__end_.__prev_->__as_node()->__value_;
+        return base::__end_.__prev_->__as_node()->__get_value();
     }
     _LIBCPP_INLINE_VISIBILITY
     const_reference back() const
     {
         _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "list::back called on empty list");
-        return base::__end_.__prev_->__as_node()->__value_;
+        return base::__end_.__prev_->__as_node()->__get_value();
     }
 
 #ifndef _LIBCPP_CXX03_LANG
@@ -1502,7 +1538,7 @@ list<_Tp, _Alloc>::emplace_front(_Args&&... __args)
     __link_nodes_at_front(__nl, __nl);
     ++base::__sz();
 #if _LIBCPP_STD_VER >= 17
-    return __node->__value_;
+    return __node->__get_value();
 #endif
 }
 
@@ -1520,7 +1556,7 @@ list<_Tp, _Alloc>::emplace_back(_Args&&... __args)
     __link_nodes_at_back(__nl, __nl);
     ++base::__sz();
 #if _LIBCPP_STD_VER >= 17
-    return __node->__value_;
+    return __node->__get_value();
 #endif
 }
 
diff --git a/libcxx/include/mdspan b/libcxx/include/mdspan
index e388166577..05f612593f 100644
--- a/libcxx/include/mdspan
+++ b/libcxx/include/mdspan
@@ -190,6 +190,63 @@ namespace std {
   };
 }
 
+// layout_stride synopsis
+
+namespace std {
+  template<class Extents>
+  class layout_stride::mapping {
+  public:
+    using extents_type = Extents;
+    using index_type = typename extents_type::index_type;
+    using size_type = typename extents_type::size_type;
+    using rank_type = typename extents_type::rank_type;
+    using layout_type = layout_stride;
+
+  private:
+    static constexpr rank_type rank_ = extents_type::rank();    // exposition only
+
+  public:
+    // [mdspan.layout.stride.cons], constructors
+    constexpr mapping() noexcept;
+    constexpr mapping(const mapping&) noexcept = default;
+    template<class OtherIndexType>
+      constexpr mapping(const extents_type&, span<OtherIndexType, rank_>) noexcept;
+    template<class OtherIndexType>
+      constexpr mapping(const extents_type&, const array<OtherIndexType, rank_>&) noexcept;
+
+    template<class StridedLayoutMapping>
+      constexpr explicit(see below) mapping(const StridedLayoutMapping&) noexcept;
+
+    constexpr mapping& operator=(const mapping&) noexcept = default;
+
+    // [mdspan.layout.stride.obs], observers
+    constexpr const extents_type& extents() const noexcept { return extents_; }
+    constexpr array<index_type, rank_> strides() const noexcept { return strides_; }
+
+    constexpr index_type required_span_size() const noexcept;
+
+    template<class... Indices>
+      constexpr index_type operator()(Indices...) const noexcept;
+
+    static constexpr bool is_always_unique() noexcept { return true; }
+    static constexpr bool is_always_exhaustive() noexcept { return false; }
+    static constexpr bool is_always_strided() noexcept { return true; }
+
+    static constexpr bool is_unique() noexcept { return true; }
+    constexpr bool is_exhaustive() const noexcept;
+    static constexpr bool is_strided() noexcept { return true; }
+
+    constexpr index_type stride(rank_type i) const noexcept { return strides_[i]; }
+
+    template<class OtherMapping>
+      friend constexpr bool operator==(const mapping&, const OtherMapping&) noexcept;
+
+  private:
+    extents_type extents_{};                    // exposition only
+    array<index_type, rank_> strides_{};        // exposition only
+  };
+}
+
 // default_accessor synopsis
 
 namespace std {
@@ -348,7 +405,9 @@ namespace std {
 #include <__mdspan/extents.h>
 #include <__mdspan/layout_left.h>
 #include <__mdspan/layout_right.h>
+#include <__mdspan/layout_stride.h>
 #include <__mdspan/mdspan.h>
+#include <version>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
diff --git a/libcxx/include/module.modulemap.in b/libcxx/include/module.modulemap.in
index 09184af273..41158212c9 100644
--- a/libcxx/include/module.modulemap.in
+++ b/libcxx/include/module.modulemap.in
@@ -456,10 +456,7 @@ module std_math_h [system] {
   header "math.h"
   export *
 }
-module std_setjmp_h [system] {
-  header "setjmp.h"
-  export *
-}
+// <setjmp.h> provided by C library.
 // <signal.h> provided by C library.
 // FIXME: <stdalign.h> is missing.
 // <stdarg.h> provided by compiler.
@@ -1513,6 +1510,7 @@ module std_private_mdspan_extents          [system] {
 }
 module std_private_mdspan_layout_left      [system] { header "__mdspan/layout_left.h" }
 module std_private_mdspan_layout_right     [system] { header "__mdspan/layout_right.h" }
+module std_private_mdspan_layout_stride    [system] { header "__mdspan/layout_stride.h" }
 module std_private_mdspan_mdspan           [system] { header "__mdspan/mdspan.h" }
 module std_private_mdspan_mdspan_fwd       [system] { header "__fwd/mdspan.h" }
 
diff --git a/libcxx/include/setjmp.h b/libcxx/include/setjmp.h
deleted file mode 100644
index f4a2bbcb0b..0000000000
--- a/libcxx/include/setjmp.h
+++ /dev/null
@@ -1,46 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _LIBCPP_SETJMP_H
-#define _LIBCPP_SETJMP_H
-
-/*
-    setjmp.h synopsis
-
-Macros:
-
-    setjmp
-
-Types:
-
-    jmp_buf
-
-void longjmp(jmp_buf env, int val);
-
-*/
-
-#include <__config>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
-
-#if __has_include_next(<setjmp.h>)
-#  include_next <setjmp.h>
-#endif
-
-#ifdef __cplusplus
-
-#ifndef setjmp
-#define setjmp(env) setjmp(env)
-#endif
-
-#endif // __cplusplus
-
-#endif // _LIBCPP_SETJMP_H
diff --git a/libcxx/include/sstream b/libcxx/include/sstream
index 47c2d0553a..4fec465d57 100644
--- a/libcxx/include/sstream
+++ b/libcxx/include/sstream
@@ -267,6 +267,7 @@ typedef basic_stringstream<wchar_t> wstringstream;
 */
 
 #include <__assert> // all public C++ headers provide the assertion handler
+#include <__availability>
 #include <__config>
 #include <__fwd/sstream.h>
 #include <__utility/swap.h>
@@ -399,12 +400,12 @@ public:
     _LIBCPP_HIDE_FROM_ABI_SSTREAM string_type str() const & { return str(__str_.get_allocator()); }
 
     _LIBCPP_HIDE_FROM_ABI_SSTREAM string_type str() && {
-        string_type __result;
         const basic_string_view<_CharT, _Traits> __view = view();
-        if (!__view.empty()) {
-            auto __pos = __view.data() - __str_.data();
-            __result.assign(std::move(__str_), __pos, __view.size());
-        }
+        typename string_type::size_type __pos = __view.empty() ? 0 : __view.data() - __str_.data();
+        // In C++23, this is just string_type(std::move(__str_), __pos, __view.size(), __str_.get_allocator());
+        // But we need something that works in C++20 also.
+        string_type __result(__str_.get_allocator());
+        __result.__move_assign(std::move(__str_), __pos, __view.size());
         __str_.clear();
         __init_buf_ptrs();
         return __result;
diff --git a/libcxx/include/string b/libcxx/include/string
index 33e87406a1..91935162f0 100644
--- a/libcxx/include/string
+++ b/libcxx/include/string
@@ -979,12 +979,7 @@ public:
 
     auto __len = std::min<size_type>(__n, __str.size() - __pos);
     if (__alloc_traits::is_always_equal::value || __alloc == __str.__alloc()) {
-      __r_.first() = __str.__r_.first();
-      __str.__r_.first() = __rep();
-
-      _Traits::move(data(), data() + __pos, __len);
-      __set_size(__len);
-      _Traits::assign(data()[__len], value_type());
+      __move_assign(std::move(__str), __pos, __len);
     } else {
       // Perform a copy because the allocators are not compatible.
       __init(__str.data() + __pos, __len);
@@ -1329,6 +1324,20 @@ public:
     return assign(__sv.data(), __sv.size());
   }
 
+#if _LIBCPP_STD_VER >= 20
+  _LIBCPP_HIDE_FROM_ABI constexpr
+  void __move_assign(basic_string&& __str, size_type __pos, size_type __len) {
+    // Pilfer the allocation from __str.
+    _LIBCPP_ASSERT_INTERNAL(__alloc() == __str.__alloc(), "__move_assign called with wrong allocator");
+    __r_.first() = __str.__r_.first();
+    __str.__r_.first() = __rep();
+
+    _Traits::move(data(), data() + __pos, __len);
+    __set_size(__len);
+    _Traits::assign(data()[__len], value_type());
+  }
+#endif
+
     _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
     basic_string& assign(const basic_string& __str) { return *this = __str; }
 #ifndef _LIBCPP_CXX03_LANG
@@ -1851,7 +1860,14 @@ private:
         _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
         size_type __align_it(size_type __s) _NOEXCEPT
             {return (__s + (__a-1)) & ~(__a-1);}
-    enum {__alignment = 16};
+    enum {
+      __alignment =
+#ifdef _LIBCPP_ABI_STRING_8_BYTE_ALIGNMENT
+      8
+#else
+      16
+#endif
+    };
     static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
     size_type __recommend(size_type __s) _NOEXCEPT
     {
diff --git a/libcxx/include/tuple b/libcxx/include/tuple
index e7fc1e28fb..138c132ff1 100644
--- a/libcxx/include/tuple
+++ b/libcxx/include/tuple
@@ -592,51 +592,31 @@ class _LIBCPP_TEMPLATE_VIS tuple
 public:
     // [tuple.cnstr]
 
-    // tuple() constructors (including allocator_arg_t variants)
-    template <template<class...> class _IsImpDefault = __is_implicitly_default_constructible, __enable_if_t<
-        _And<
-            _IsImpDefault<_Tp>... // explicit check
-        >::value
-    , int> = 0>
-    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR
-    tuple()
-        _NOEXCEPT_(_And<is_nothrow_default_constructible<_Tp>...>::value)
-    { }
+_LIBCPP_DIAGNOSTIC_PUSH
+_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wc++20-extensions")
+_LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wc++20-extensions")
 
+    // tuple() constructors (including allocator_arg_t variants)
     template <template<class...> class _IsImpDefault = __is_implicitly_default_constructible,
               template<class...> class _IsDefault = is_default_constructible, __enable_if_t<
         _And<
-            _IsDefault<_Tp>...,
-            _Not<_Lazy<_And, _IsImpDefault<_Tp>...> > // explicit check
+            _IsDefault<_Tp>...
         >::value
     , int> = 0>
     _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR
-    explicit tuple()
+    explicit(_Not<_Lazy<_And, _IsImpDefault<_Tp>...> >::value) tuple()
         _NOEXCEPT_(_And<is_nothrow_default_constructible<_Tp>...>::value)
     { }
 
-    template <class _Alloc, template<class...> class _IsImpDefault = __is_implicitly_default_constructible, __enable_if_t<
-        _And<
-            _IsImpDefault<_Tp>... // explicit check
-        >::value
-    , int> = 0>
-    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX20
-    tuple(allocator_arg_t, _Alloc const& __a)
-      : __base_(allocator_arg_t(), __a,
-                    __tuple_indices<>(), __tuple_types<>(),
-                    typename __make_tuple_indices<sizeof...(_Tp), 0>::type(),
-                    __tuple_types<_Tp...>()) {}
-
     template <class _Alloc,
               template<class...> class _IsImpDefault = __is_implicitly_default_constructible,
               template<class...> class _IsDefault = is_default_constructible, __enable_if_t<
         _And<
-            _IsDefault<_Tp>...,
-            _Not<_Lazy<_And, _IsImpDefault<_Tp>...> > // explicit check
+            _IsDefault<_Tp>...
         >::value
     , int> = 0>
     _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX20
-    explicit tuple(allocator_arg_t, _Alloc const& __a)
+    explicit(_Not<_Lazy<_And, _IsImpDefault<_Tp>...> >::value) tuple(allocator_arg_t, _Alloc const& __a)
       : __base_(allocator_arg_t(), __a,
                     __tuple_indices<>(), __tuple_types<>(),
                     typename __make_tuple_indices<sizeof...(_Tp), 0>::type(),
@@ -646,29 +626,11 @@ public:
     template <template<class...> class _And = _And, __enable_if_t<
         _And<
             _BoolConstant<sizeof...(_Tp) >= 1>,
-            is_copy_constructible<_Tp>...,
-            is_convertible<const _Tp&, _Tp>... // explicit check
-        >::value
-    , int> = 0>
-    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX14
-    tuple(const _Tp& ... __t)
-        _NOEXCEPT_(_And<is_nothrow_copy_constructible<_Tp>...>::value)
-        : __base_(typename __make_tuple_indices<sizeof...(_Tp)>::type(),
-                typename __make_tuple_types<tuple, sizeof...(_Tp)>::type(),
-                typename __make_tuple_indices<0>::type(),
-                typename __make_tuple_types<tuple, 0>::type(),
-                __t...
-               ) {}
-
-    template <template<class...> class _And = _And, __enable_if_t<
-        _And<
-            _BoolConstant<sizeof...(_Tp) >= 1>,
-            is_copy_constructible<_Tp>...,
-            _Not<_Lazy<_And, is_convertible<const _Tp&, _Tp>...> > // explicit check
+            is_copy_constructible<_Tp>...
         >::value
     , int> = 0>
     _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX14
-    explicit tuple(const _Tp& ... __t)
+    explicit(_Not<_Lazy<_And, is_convertible<const _Tp&, _Tp>...> >::value) tuple(const _Tp& ... __t)
         _NOEXCEPT_(_And<is_nothrow_copy_constructible<_Tp>...>::value)
         : __base_(typename __make_tuple_indices<sizeof...(_Tp)>::type(),
                 typename __make_tuple_types<tuple, sizeof...(_Tp)>::type(),
@@ -680,29 +642,11 @@ public:
     template <class _Alloc, template<class...> class _And = _And, __enable_if_t<
         _And<
             _BoolConstant<sizeof...(_Tp) >= 1>,
-            is_copy_constructible<_Tp>...,
-            is_convertible<const _Tp&, _Tp>... // explicit check
+            is_copy_constructible<_Tp>...
         >::value
     , int> = 0>
     _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX20
-    tuple(allocator_arg_t, const _Alloc& __a, const _Tp& ... __t)
-        : __base_(allocator_arg_t(), __a,
-                typename __make_tuple_indices<sizeof...(_Tp)>::type(),
-                typename __make_tuple_types<tuple, sizeof...(_Tp)>::type(),
-                typename __make_tuple_indices<0>::type(),
-                typename __make_tuple_types<tuple, 0>::type(),
-                __t...
-               ) {}
-
-    template <class _Alloc, template<class...> class _And = _And, __enable_if_t<
-        _And<
-            _BoolConstant<sizeof...(_Tp) >= 1>,
-            is_copy_constructible<_Tp>...,
-            _Not<_Lazy<_And, is_convertible<const _Tp&, _Tp>...> > // explicit check
-        >::value
-    , int> = 0>
-    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX20
-    explicit tuple(allocator_arg_t, const _Alloc& __a, const _Tp& ... __t)
+    explicit(_Not<_Lazy<_And, is_convertible<const _Tp&, _Tp>...> >::value) tuple(allocator_arg_t, const _Alloc& __a, const _Tp& ... __t)
         : __base_(allocator_arg_t(), __a,
                 typename __make_tuple_indices<sizeof...(_Tp)>::type(),
                 typename __make_tuple_types<tuple, sizeof...(_Tp)>::type(),
@@ -725,12 +669,11 @@ public:
     template <class ..._Up, __enable_if_t<
         _And<
             _BoolConstant<sizeof...(_Up) == sizeof...(_Tp)>,
-            _EnableUTypesCtor<_Up...>,
-            is_convertible<_Up, _Tp>... // explicit check
+            _EnableUTypesCtor<_Up...>
         >::value
     , int> = 0>
     _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX14
-    tuple(_Up&&... __u)
+    explicit(_Not<_Lazy<_And, is_convertible<_Up, _Tp>...> >::value) tuple(_Up&&... __u)
         _NOEXCEPT_((_And<is_nothrow_constructible<_Tp, _Up>...>::value))
         : __base_(typename __make_tuple_indices<sizeof...(_Up)>::type(),
                     typename __make_tuple_types<tuple, sizeof...(_Up)>::type(),
@@ -738,47 +681,14 @@ public:
                     typename __make_tuple_types<tuple, sizeof...(_Tp), sizeof...(_Up)>::type(),
                     _VSTD::forward<_Up>(__u)...) {}
 
-    template <class ..._Up, __enable_if_t<
-        _And<
-            _BoolConstant<sizeof...(_Up) == sizeof...(_Tp)>,
-            _EnableUTypesCtor<_Up...>,
-            _Not<_Lazy<_And, is_convertible<_Up, _Tp>...> > // explicit check
-        >::value
-    , int> = 0>
-    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX14
-    explicit tuple(_Up&&... __u)
-        _NOEXCEPT_((_And<is_nothrow_constructible<_Tp, _Up>...>::value))
-        : __base_(typename __make_tuple_indices<sizeof...(_Up)>::type(),
-                    typename __make_tuple_types<tuple, sizeof...(_Up)>::type(),
-                    typename __make_tuple_indices<sizeof...(_Tp), sizeof...(_Up)>::type(),
-                    typename __make_tuple_types<tuple, sizeof...(_Tp), sizeof...(_Up)>::type(),
-                    _VSTD::forward<_Up>(__u)...) {}
-
-    template <class _Alloc, class ..._Up, __enable_if_t<
-        _And<
-            _BoolConstant<sizeof...(_Up) == sizeof...(_Tp)>,
-            _EnableUTypesCtor<_Up...>,
-            is_convertible<_Up, _Tp>... // explicit check
-        >::value
-    , int> = 0>
-    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX20
-    tuple(allocator_arg_t, const _Alloc& __a, _Up&&... __u)
-        : __base_(allocator_arg_t(), __a,
-                    typename __make_tuple_indices<sizeof...(_Up)>::type(),
-                    typename __make_tuple_types<tuple, sizeof...(_Up)>::type(),
-                    typename __make_tuple_indices<sizeof...(_Tp), sizeof...(_Up)>::type(),
-                    typename __make_tuple_types<tuple, sizeof...(_Tp), sizeof...(_Up)>::type(),
-                    _VSTD::forward<_Up>(__u)...) {}
-
     template <class _Alloc, class ..._Up, __enable_if_t<
         _And<
             _BoolConstant<sizeof...(_Up) == sizeof...(_Tp)>,
-            _EnableUTypesCtor<_Up...>,
-            _Not<_Lazy<_And, is_convertible<_Up, _Tp>...> > // explicit check
+            _EnableUTypesCtor<_Up...>
         >::value
     , int> = 0>
     _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX20
-    explicit tuple(allocator_arg_t, const _Alloc& __a, _Up&&... __u)
+    explicit(_Not<_Lazy<_And, is_convertible<_Up, _Tp>...> >::value) tuple(allocator_arg_t, const _Alloc& __a, _Up&&... __u)
         : __base_(allocator_arg_t(), __a,
                     typename __make_tuple_indices<sizeof...(_Up)>::type(),
                     typename __make_tuple_types<tuple, sizeof...(_Up)>::type(),
@@ -833,47 +743,22 @@ public:
 
     template <class ..._Up, __enable_if_t<
         _And<
-            _EnableCtorFromUTypesTuple<const tuple<_Up...>&>,
-            is_convertible<const _Up&, _Tp>... // explicit check
+            _EnableCtorFromUTypesTuple<const tuple<_Up...>&>
         >::value
     , int> = 0>
     _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX14
-    tuple(const tuple<_Up...>& __t)
-        _NOEXCEPT_((_And<is_nothrow_constructible<_Tp, const _Up&>...>::value))
-        : __base_(__t)
-    { }
-
-    template <class ..._Up, __enable_if_t<
-        _And<
-            _EnableCtorFromUTypesTuple<const tuple<_Up...>&>,
-            _Not<_Lazy<_And, is_convertible<const _Up&, _Tp>...> > // explicit check
-        >::value
-    , int> = 0>
-    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX14
-    explicit tuple(const tuple<_Up...>& __t)
+    explicit(_Not<_Lazy<_And, is_convertible<const _Up&, _Tp>...> >::value) tuple(const tuple<_Up...>& __t)
         _NOEXCEPT_((_And<is_nothrow_constructible<_Tp, const _Up&>...>::value))
         : __base_(__t)
     { }
 
     template <class ..._Up, class _Alloc, __enable_if_t<
         _And<
-            _EnableCtorFromUTypesTuple<const tuple<_Up...>&>,
-            is_convertible<const _Up&, _Tp>... // explicit check
+            _EnableCtorFromUTypesTuple<const tuple<_Up...>&>
         >::value
     , int> = 0>
     _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX20
-    tuple(allocator_arg_t, const _Alloc& __a, const tuple<_Up...>& __t)
-        : __base_(allocator_arg_t(), __a, __t)
-    { }
-
-    template <class ..._Up, class _Alloc, __enable_if_t<
-        _And<
-            _EnableCtorFromUTypesTuple<const tuple<_Up...>&>,
-            _Not<_Lazy<_And, is_convertible<const _Up&, _Tp>...> > // explicit check
-        >::value
-    , int> = 0>
-    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX20
-    explicit tuple(allocator_arg_t, const _Alloc& __a, const tuple<_Up...>& __t)
+    explicit(_Not<_Lazy<_And, is_convertible<const _Up&, _Tp>...> >::value) tuple(allocator_arg_t, const _Alloc& __a, const tuple<_Up...>& __t)
         : __base_(allocator_arg_t(), __a, __t)
     { }
 
@@ -894,50 +779,24 @@ public:
 #endif // _LIBCPP_STD_VER >= 23
 
     // tuple(tuple<U...>&&) constructors (including allocator_arg_t variants)
-
     template <class ..._Up, __enable_if_t<
         _And<
-            _EnableCtorFromUTypesTuple<tuple<_Up...>&&>,
-            is_convertible<_Up, _Tp>... // explicit check
+            _EnableCtorFromUTypesTuple<tuple<_Up...>&&>
         >::value
     , int> = 0>
     _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX14
-    tuple(tuple<_Up...>&& __t)
+    explicit(_Not<_Lazy<_And, is_convertible<_Up, _Tp>...> >::value) tuple(tuple<_Up...>&& __t)
         _NOEXCEPT_((_And<is_nothrow_constructible<_Tp, _Up>...>::value))
         : __base_(_VSTD::move(__t))
     { }
 
-    template <class ..._Up, __enable_if_t<
-        _And<
-            _EnableCtorFromUTypesTuple<tuple<_Up...>&&>,
-            _Not<_Lazy<_And, is_convertible<_Up, _Tp>...> > // explicit check
-        >::value
-    , int> = 0>
-    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX14
-    explicit tuple(tuple<_Up...>&& __t)
-        _NOEXCEPT_((_And<is_nothrow_constructible<_Tp, _Up>...>::value))
-        : __base_(_VSTD::move(__t))
-    { }
-
-    template <class _Alloc, class ..._Up, __enable_if_t<
-        _And<
-            _EnableCtorFromUTypesTuple<tuple<_Up...>&&>,
-            is_convertible<_Up, _Tp>... // explicit check
-        >::value
-    , int> = 0>
-    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX20
-    tuple(allocator_arg_t, const _Alloc& __a, tuple<_Up...>&& __t)
-        : __base_(allocator_arg_t(), __a, _VSTD::move(__t))
-    { }
-
     template <class _Alloc, class ..._Up, __enable_if_t<
         _And<
-            _EnableCtorFromUTypesTuple<tuple<_Up...>&&>,
-            _Not<_Lazy<_And, is_convertible<_Up, _Tp>...> > // explicit check
+            _EnableCtorFromUTypesTuple<tuple<_Up...>&&>
         >::value
     , int> = 0>
     _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX20
-    explicit tuple(allocator_arg_t, const _Alloc& __a, tuple<_Up...>&& __t)
+    explicit(_Not<_Lazy<_And, is_convertible<_Up, _Tp>...> >::value) tuple(allocator_arg_t, const _Alloc& __a, tuple<_Up...>&& __t)
         : __base_(allocator_arg_t(), __a, _VSTD::move(__t))
     { }
 
@@ -986,47 +845,22 @@ public:
 
     template <class _Up1, class _Up2, template<class...> class _And = _And, __enable_if_t<
         _And<
-            _EnableCtorFromPair<const pair<_Up1, _Up2>&>,
-            _BothImplicitlyConvertible<const pair<_Up1, _Up2>&> // explicit check
+            _EnableCtorFromPair<const pair<_Up1, _Up2>&>
         >::value
     , int> = 0>
     _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX14
-    tuple(const pair<_Up1, _Up2>& __p)
-        _NOEXCEPT_((_NothrowConstructibleFromPair<const pair<_Up1, _Up2>&>::value))
-        : __base_(__p)
-    { }
-
-    template <class _Up1, class _Up2, template<class...> class _And = _And, __enable_if_t<
-        _And<
-            _EnableCtorFromPair<const pair<_Up1, _Up2>&>,
-            _Not<_BothImplicitlyConvertible<const pair<_Up1, _Up2>&> > // explicit check
-        >::value
-    , int> = 0>
-    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX14
-    explicit tuple(const pair<_Up1, _Up2>& __p)
+    explicit(_Not<_BothImplicitlyConvertible<const pair<_Up1, _Up2>&> >::value) tuple(const pair<_Up1, _Up2>& __p)
         _NOEXCEPT_((_NothrowConstructibleFromPair<const pair<_Up1, _Up2>&>::value))
         : __base_(__p)
     { }
 
     template <class _Alloc, class _Up1, class _Up2, template<class...> class _And = _And, __enable_if_t<
         _And<
-            _EnableCtorFromPair<const pair<_Up1, _Up2>&>,
-            _BothImplicitlyConvertible<const pair<_Up1, _Up2>&> // explicit check
+            _EnableCtorFromPair<const pair<_Up1, _Up2>&>
         >::value
     , int> = 0>
     _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX20
-    tuple(allocator_arg_t, const _Alloc& __a, const pair<_Up1, _Up2>& __p)
-        : __base_(allocator_arg_t(), __a, __p)
-    { }
-
-    template <class _Alloc, class _Up1, class _Up2, template<class...> class _And = _And, __enable_if_t<
-        _And<
-            _EnableCtorFromPair<const pair<_Up1, _Up2>&>,
-            _Not<_BothImplicitlyConvertible<const pair<_Up1, _Up2>&> > // explicit check
-        >::value
-    , int> = 0>
-    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX20
-    explicit tuple(allocator_arg_t, const _Alloc& __a, const pair<_Up1, _Up2>& __p)
+    explicit(_Not<_BothImplicitlyConvertible<const pair<_Up1, _Up2>&> >::value) tuple(allocator_arg_t, const _Alloc& __a, const pair<_Up1, _Up2>& __p)
         : __base_(allocator_arg_t(), __a, __p)
     { }
 
@@ -1050,47 +884,22 @@ public:
 
     template <class _Up1, class _Up2, template<class...> class _And = _And, __enable_if_t<
         _And<
-            _EnableCtorFromPair<pair<_Up1, _Up2>&&>,
-            _BothImplicitlyConvertible<pair<_Up1, _Up2>&&> // explicit check
+            _EnableCtorFromPair<pair<_Up1, _Up2>&&>
         >::value
     , int> = 0>
     _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX14
-    tuple(pair<_Up1, _Up2>&& __p)
-        _NOEXCEPT_((_NothrowConstructibleFromPair<pair<_Up1, _Up2>&&>::value))
-        : __base_(_VSTD::move(__p))
-    { }
-
-    template <class _Up1, class _Up2, template<class...> class _And = _And, __enable_if_t<
-        _And<
-            _EnableCtorFromPair<pair<_Up1, _Up2>&&>,
-            _Not<_BothImplicitlyConvertible<pair<_Up1, _Up2>&&> > // explicit check
-        >::value
-    , int> = 0>
-    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX14
-    explicit tuple(pair<_Up1, _Up2>&& __p)
+    explicit(_Not<_BothImplicitlyConvertible<pair<_Up1, _Up2>&&> >::value) tuple(pair<_Up1, _Up2>&& __p)
         _NOEXCEPT_((_NothrowConstructibleFromPair<pair<_Up1, _Up2>&&>::value))
         : __base_(_VSTD::move(__p))
     { }
 
     template <class _Alloc, class _Up1, class _Up2, template<class...> class _And = _And, __enable_if_t<
         _And<
-            _EnableCtorFromPair<pair<_Up1, _Up2>&&>,
-            _BothImplicitlyConvertible<pair<_Up1, _Up2>&&> // explicit check
+            _EnableCtorFromPair<pair<_Up1, _Up2>&&>
         >::value
     , int> = 0>
     _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX20
-    tuple(allocator_arg_t, const _Alloc& __a, pair<_Up1, _Up2>&& __p)
-        : __base_(allocator_arg_t(), __a, _VSTD::move(__p))
-    { }
-
-    template <class _Alloc, class _Up1, class _Up2, template<class...> class _And = _And, __enable_if_t<
-        _And<
-            _EnableCtorFromPair<pair<_Up1, _Up2>&&>,
-            _Not<_BothImplicitlyConvertible<pair<_Up1, _Up2>&&> > // explicit check
-        >::value
-    , int> = 0>
-    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX20
-    explicit tuple(allocator_arg_t, const _Alloc& __a, pair<_Up1, _Up2>&& __p)
+    explicit(_Not<_BothImplicitlyConvertible<pair<_Up1, _Up2>&&> >::value) tuple(allocator_arg_t, const _Alloc& __a, pair<_Up1, _Up2>&& __p)
         : __base_(allocator_arg_t(), __a, _VSTD::move(__p))
     { }
 
@@ -1111,6 +920,8 @@ public:
         : __base_(allocator_arg_t(), __alloc, std::move(__p)) {}
 #endif // _LIBCPP_STD_VER >= 23
 
+_LIBCPP_DIAGNOSTIC_POP
+
     // [tuple.assign]
     _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX20
     tuple& operator=(_If<_And<is_copy_assignable<_Tp>...>::value, tuple, __nat> const& __tuple)
diff --git a/libcxx/include/unordered_map b/libcxx/include/unordered_map
index 8d83063bbe..e5c58feee5 100644
--- a/libcxx/include/unordered_map
+++ b/libcxx/include/unordered_map
@@ -874,9 +874,9 @@ public:
     void operator()(pointer __p) _NOEXCEPT
     {
         if (__second_constructed)
-            __alloc_traits::destroy(__na_, _VSTD::addressof(__p->__value_.__get_value().second));
+            __alloc_traits::destroy(__na_, _VSTD::addressof(__p->__get_value().__get_value().second));
         if (__first_constructed)
-            __alloc_traits::destroy(__na_, _VSTD::addressof(__p->__value_.__get_value().first));
+            __alloc_traits::destroy(__na_, _VSTD::addressof(__p->__get_value().__get_value().first));
         if (__p)
             __alloc_traits::deallocate(__na_, __p, 1);
     }
@@ -1828,7 +1828,7 @@ unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(
         iterator __i = __u.begin();
         while (__u.size() != 0) {
             __table_.__emplace_unique(
-                __u.__table_.remove((__i++).__i_)->__value_.__move());
+                __u.__table_.remove((__i++).__i_)->__get_value().__move());
         }
     }
 }
@@ -1920,9 +1920,9 @@ unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::__construct_node_with_key(const
 {
     __node_allocator& __na = __table_.__node_alloc();
     __node_holder __h(__node_traits::allocate(__na, 1), _Dp(__na));
-    __node_traits::construct(__na, _VSTD::addressof(__h->__value_.__get_value().first), __k);
+    __node_traits::construct(__na, _VSTD::addressof(__h->__get_value().__get_value().first), __k);
     __h.get_deleter().__first_constructed = true;
-    __node_traits::construct(__na, _VSTD::addressof(__h->__value_.__get_value().second));
+    __node_traits::construct(__na, _VSTD::addressof(__h->__get_value().__get_value().second));
     __h.get_deleter().__second_constructed = true;
     return __h;
 }
@@ -2653,7 +2653,7 @@ unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(
         while (__u.size() != 0)
         {
             __table_.__insert_multi(
-                __u.__table_.remove((__i++).__i_)->__value_.__move());
+                __u.__table_.remove((__i++).__i_)->__get_value().__move());
         }
     }
 }
diff --git a/libcxx/include/unordered_set b/libcxx/include/unordered_set
index 5e47f12446..f1b4104df4 100644
--- a/libcxx/include/unordered_set
+++ b/libcxx/include/unordered_set
@@ -1150,7 +1150,7 @@ unordered_set<_Value, _Hash, _Pred, _Alloc>::unordered_set(
     {
         iterator __i = __u.begin();
         while (__u.size() != 0)
-            __table_.__insert_unique(_VSTD::move(__u.__table_.remove(__i++)->__value_));
+            __table_.__insert_unique(_VSTD::move(__u.__table_.remove(__i++)->__get_value()));
     }
 }
 
@@ -1835,7 +1835,7 @@ unordered_multiset<_Value, _Hash, _Pred, _Alloc>::unordered_multiset(
     {
         iterator __i = __u.begin();
         while (__u.size() != 0)
-            __table_.__insert_multi(_VSTD::move(__u.__table_.remove(__i++)->__value_));
+            __table_.__insert_multi(_VSTD::move(__u.__table_.remove(__i++)->__get_value()));
     }
 }
 
diff --git a/libcxx/include/version b/libcxx/include/version
index 4fdfec4f34..c41ce745b5 100644
--- a/libcxx/include/version
+++ b/libcxx/include/version
@@ -427,7 +427,7 @@ __cpp_lib_within_lifetime                               202306L <type_traits>
 # define __cpp_lib_forward_like                         202207L
 # define __cpp_lib_invoke_r                             202106L
 # define __cpp_lib_is_scoped_enum                       202011L
-// # define __cpp_lib_mdspan                               202207L
+# define __cpp_lib_mdspan                               202207L
 // # define __cpp_lib_move_only_function                   202110L
 # undef  __cpp_lib_optional
 # define __cpp_lib_optional                             202110L
diff --git a/libcxx/modules/std/mdspan.inc b/libcxx/modules/std/mdspan.inc
index 888c94ad59..ba3f3926c3 100644
--- a/libcxx/modules/std/mdspan.inc
+++ b/libcxx/modules/std/mdspan.inc
@@ -18,7 +18,7 @@ export namespace std {
   // [mdspan.layout], layout mapping
   using std::layout_left;
   using std::layout_right;
-  // using std::layout_stride;
+  using std::layout_stride;
 
   // [mdspan.accessor.default], class template default_accessor
   using std::default_accessor;
diff --git a/libcxx/src/new.cpp b/libcxx/src/new.cpp
index c435c5ffc8..033bba5c1f 100644
--- a/libcxx/src/new.cpp
+++ b/libcxx/src/new.cpp
@@ -20,239 +20,155 @@
 // in this shared library, so that they can be overridden by programs
 // that define non-weak copies of the functions.
 
-_LIBCPP_WEAK
-void *
-operator new(std::size_t size) _THROW_BAD_ALLOC
-{
-    if (size == 0)
-        size = 1;
-    void* p;
-    while ((p = std::malloc(size)) == nullptr)
-    {
-        // If malloc fails and there is a new_handler,
-        // call it to try free up memory.
-        std::new_handler nh = std::get_new_handler();
-        if (nh)
-            nh();
-        else
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
-            throw std::bad_alloc();
-#else
-            break;
-#endif
-    }
-    return p;
+static void* operator_new_impl(std::size_t size) noexcept {
+  if (size == 0)
+    size = 1;
+  void* p;
+  while ((p = std::malloc(size)) == nullptr) {
+    // If malloc fails and there is a new_handler,
+    // call it to try free up memory.
+    std::new_handler nh = std::get_new_handler();
+    if (nh)
+      nh();
+    else
+      break;
+  }
+  return p;
+}
+
+_LIBCPP_WEAK void* operator new(std::size_t size) _THROW_BAD_ALLOC {
+  void* p = operator_new_impl(size);
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  if (p == nullptr)
+    throw std::bad_alloc();
+#  endif
+  return p;
+}
+
+_LIBCPP_WEAK void* operator new(size_t size, const std::nothrow_t&) noexcept {
+  void* p = nullptr;
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    p = ::operator new(size);
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+  }
+#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  return p;
+}
+
+_LIBCPP_WEAK void* operator new[](size_t size) _THROW_BAD_ALLOC { return ::operator new(size); }
+
+_LIBCPP_WEAK void* operator new[](size_t size, const std::nothrow_t&) noexcept {
+  void* p = nullptr;
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    p = ::operator new[](size);
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+  }
+#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  return p;
 }
 
-_LIBCPP_WEAK
-void*
-operator new(size_t size, const std::nothrow_t&) noexcept
-{
-    void* p = nullptr;
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
-    try
-    {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
-        p = ::operator new(size);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
-    }
-    catch (...)
-    {
-    }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
-    return p;
-}
+_LIBCPP_WEAK void operator delete(void* ptr) noexcept { std::free(ptr); }
 
-_LIBCPP_WEAK
-void*
-operator new[](size_t size) _THROW_BAD_ALLOC
-{
-    return ::operator new(size);
-}
+_LIBCPP_WEAK void operator delete(void* ptr, const std::nothrow_t&) noexcept { ::operator delete(ptr); }
 
-_LIBCPP_WEAK
-void*
-operator new[](size_t size, const std::nothrow_t&) noexcept
-{
-    void* p = nullptr;
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
-    try
-    {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
-        p = ::operator new[](size);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
-    }
-    catch (...)
-    {
-    }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
-    return p;
-}
+_LIBCPP_WEAK void operator delete(void* ptr, size_t) noexcept { ::operator delete(ptr); }
 
-_LIBCPP_WEAK
-void
-operator delete(void* ptr) noexcept
-{
-    std::free(ptr);
-}
+_LIBCPP_WEAK void operator delete[](void* ptr) noexcept { ::operator delete(ptr); }
 
-_LIBCPP_WEAK
-void
-operator delete(void* ptr, const std::nothrow_t&) noexcept
-{
-    ::operator delete(ptr);
-}
+_LIBCPP_WEAK void operator delete[](void* ptr, const std::nothrow_t&) noexcept { ::operator delete[](ptr); }
 
-_LIBCPP_WEAK
-void
-operator delete(void* ptr, size_t) noexcept
-{
-    ::operator delete(ptr);
-}
+_LIBCPP_WEAK void operator delete[](void* ptr, size_t) noexcept { ::operator delete[](ptr); }
 
-_LIBCPP_WEAK
-void
-operator delete[] (void* ptr) noexcept
-{
-    ::operator delete(ptr);
-}
+#  if !defined(_LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION)
 
-_LIBCPP_WEAK
-void
-operator delete[] (void* ptr, const std::nothrow_t&) noexcept
-{
-    ::operator delete[](ptr);
-}
+static void* operator_new_aligned_impl(std::size_t size, std::align_val_t alignment) noexcept {
+  if (size == 0)
+    size = 1;
+  if (static_cast<size_t>(alignment) < sizeof(void*))
+    alignment = std::align_val_t(sizeof(void*));
 
-_LIBCPP_WEAK
-void
-operator delete[] (void* ptr, size_t) noexcept
-{
-    ::operator delete[](ptr);
+  // Try allocating memory. If allocation fails and there is a new_handler,
+  // call it to try free up memory, and try again until it succeeds, or until
+  // the new_handler decides to terminate.
+  void* p;
+  while ((p = std::__libcpp_aligned_alloc(static_cast<std::size_t>(alignment), size)) == nullptr) {
+    std::new_handler nh = std::get_new_handler();
+    if (nh)
+      nh();
+    else
+      break;
+  }
+  return p;
 }
 
-#if !defined(_LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION)
-
-_LIBCPP_WEAK
-void *
-operator new(std::size_t size, std::align_val_t alignment) _THROW_BAD_ALLOC
-{
-    if (size == 0)
-        size = 1;
-    if (static_cast<size_t>(alignment) < sizeof(void*))
-      alignment = std::align_val_t(sizeof(void*));
-
-    // Try allocating memory. If allocation fails and there is a new_handler,
-    // call it to try free up memory, and try again until it succeeds, or until
-    // the new_handler decides to terminate.
-    //
-    // If allocation fails and there is no new_handler, we throw bad_alloc
-    // (or return nullptr if exceptions are disabled).
-    void* p;
-    while ((p = std::__libcpp_aligned_alloc(static_cast<std::size_t>(alignment), size)) == nullptr)
-    {
-        std::new_handler nh = std::get_new_handler();
-        if (nh)
-            nh();
-        else {
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
-            throw std::bad_alloc();
-#else
-            break;
-#endif
-        }
-    }
-    return p;
+_LIBCPP_WEAK void* operator new(std::size_t size, std::align_val_t alignment) _THROW_BAD_ALLOC {
+  void* p = operator_new_aligned_impl(size, alignment);
+#    ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  if (p == nullptr)
+    throw std::bad_alloc();
+#    endif
+  return p;
 }
 
-_LIBCPP_WEAK
-void*
-operator new(size_t size, std::align_val_t alignment, const std::nothrow_t&) noexcept
-{
-    void* p = nullptr;
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
-    try
-    {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
-        p = ::operator new(size, alignment);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
-    }
-    catch (...)
-    {
-    }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
-    return p;
+_LIBCPP_WEAK void* operator new(size_t size, std::align_val_t alignment, const std::nothrow_t&) noexcept {
+  void* p = nullptr;
+#    ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#    endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    p = ::operator new(size, alignment);
+#    ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+  }
+#    endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  return p;
 }
 
-_LIBCPP_WEAK
-void*
-operator new[](size_t size, std::align_val_t alignment) _THROW_BAD_ALLOC
-{
-    return ::operator new(size, alignment);
+_LIBCPP_WEAK void* operator new[](size_t size, std::align_val_t alignment) _THROW_BAD_ALLOC {
+  return ::operator new(size, alignment);
 }
 
-_LIBCPP_WEAK
-void*
-operator new[](size_t size, std::align_val_t alignment, const std::nothrow_t&) noexcept
-{
-    void* p = nullptr;
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
-    try
-    {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
-        p = ::operator new[](size, alignment);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
-    }
-    catch (...)
-    {
-    }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
-    return p;
+_LIBCPP_WEAK void* operator new[](size_t size, std::align_val_t alignment, const std::nothrow_t&) noexcept {
+  void* p = nullptr;
+#    ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#    endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    p = ::operator new[](size, alignment);
+#    ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+  }
+#    endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  return p;
 }
 
-_LIBCPP_WEAK
-void
-operator delete(void* ptr, std::align_val_t) noexcept
-{
-    std::__libcpp_aligned_free(ptr);
-}
+_LIBCPP_WEAK void operator delete(void* ptr, std::align_val_t) noexcept { std::__libcpp_aligned_free(ptr); }
 
-_LIBCPP_WEAK
-void
-operator delete(void* ptr, std::align_val_t alignment, const std::nothrow_t&) noexcept
-{
-    ::operator delete(ptr, alignment);
+_LIBCPP_WEAK void operator delete(void* ptr, std::align_val_t alignment, const std::nothrow_t&) noexcept {
+  ::operator delete(ptr, alignment);
 }
 
-_LIBCPP_WEAK
-void
-operator delete(void* ptr, size_t, std::align_val_t alignment) noexcept
-{
-    ::operator delete(ptr, alignment);
+_LIBCPP_WEAK void operator delete(void* ptr, size_t, std::align_val_t alignment) noexcept {
+  ::operator delete(ptr, alignment);
 }
 
-_LIBCPP_WEAK
-void
-operator delete[] (void* ptr, std::align_val_t alignment) noexcept
-{
-    ::operator delete(ptr, alignment);
+_LIBCPP_WEAK void operator delete[](void* ptr, std::align_val_t alignment) noexcept {
+  ::operator delete(ptr, alignment);
 }
 
-_LIBCPP_WEAK
-void
-operator delete[] (void* ptr, std::align_val_t alignment, const std::nothrow_t&) noexcept
-{
-    ::operator delete[](ptr, alignment);
+_LIBCPP_WEAK void operator delete[](void* ptr, std::align_val_t alignment, const std::nothrow_t&) noexcept {
+  ::operator delete[](ptr, alignment);
 }
 
-_LIBCPP_WEAK
-void
-operator delete[] (void* ptr, size_t, std::align_val_t alignment) noexcept
-{
-    ::operator delete[](ptr, alignment);
+_LIBCPP_WEAK void operator delete[](void* ptr, size_t, std::align_val_t alignment) noexcept {
+  ::operator delete[](ptr, alignment);
 }
 
-#endif // !_LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION
+#  endif // !_LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION
 // ------------------ END COPY ------------------
 
 #endif // !__GLIBCXX__ && !_LIBCPP_ABI_VCRUNTIME
diff --git a/libcxx/test/configs/llvm-libc++-android-ndk.cfg.in b/libcxx/test/configs/llvm-libc++-android-ndk.cfg.in
new file mode 100644
index 0000000000..1be85279a1
--- /dev/null
+++ b/libcxx/test/configs/llvm-libc++-android-ndk.cfg.in
@@ -0,0 +1,47 @@
+# This testing configuration handles running the test suite against LLVM's
+# libc++ using adb and a libc++_shared.so library on Android.
+
+lit_config.load_config(config, '@CMAKE_CURRENT_BINARY_DIR@/cmake-bridge.cfg')
+
+import re
+import site
+
+site.addsitedir(os.path.join('@LIBCXX_SOURCE_DIR@', 'utils'))
+
+import libcxx.test.android
+import libcxx.test.config
+import libcxx.test.params
+
+config.substitutions.append(('%{flags}',
+    '--sysroot @CMAKE_SYSROOT@' if '@CMAKE_SYSROOT@' else ''
+))
+
+compile_flags = '-nostdinc++ -I %{include} -I %{target-include} -I %{libcxx}/test/support'
+if re.match(r'i686-linux-android(21|22|23)$', config.target_triple):
+    # 32-bit x86 Android has a bug where the stack is sometimes misaligned.
+    # The problem appears limited to versions before Android N (API 24) and only
+    # __attribute__((constructor)) functions. Compile with -mstackrealign to
+    # work around the bug.
+    # TODO: Consider automatically doing something like this in Clang itself (LIBCXX-ANDROID-FIXME)
+    # See https://github.com/android/ndk/issues/693.
+    compile_flags += ' -mstackrealign'
+config.substitutions.append(('%{compile_flags}', compile_flags))
+
+# The NDK library is called "libc++_shared.so". Use LD_LIBRARY_PATH to find
+# libc++_shared.so because older Bionic dynamic loaders don't support rpath
+# lookup.
+config.substitutions.append(('%{link_flags}',
+    '-nostdlib++ -L %{lib} -lc++_shared'
+))
+config.substitutions.append(('%{exec}',
+    '%{executor}' +
+    ' --job-limit-socket ' + libcxx.test.android.adb_job_limit_socket() +
+    ' --prepend-path-env LD_LIBRARY_PATH /data/local/tmp/libc++ --execdir %T -- '
+))
+
+libcxx.test.config.configure(
+    libcxx.test.params.DEFAULT_PARAMETERS,
+    libcxx.test.features.DEFAULT_FEATURES,
+    config,
+    lit_config
+)
diff --git a/libcxx/test/libcxx/algorithms/alg.sorting/assert.sort.invalid_comparator.pass.cpp b/libcxx/test/libcxx/algorithms/alg.sorting/assert.sort.invalid_comparator.pass.cpp
index e5e417fe7b..96c2821c4a 100644
--- a/libcxx/test/libcxx/algorithms/alg.sorting/assert.sort.invalid_comparator.pass.cpp
+++ b/libcxx/test/libcxx/algorithms/alg.sorting/assert.sort.invalid_comparator.pass.cpp
@@ -50,24 +50,34 @@
 #include "bad_comparator_values.h"
 #include "check_assertion.h"
 
-void check_oob_sort_read() {
-    std::map<std::size_t, std::map<std::size_t, bool>> comparison_results; // terrible for performance, but really convenient
-    for (auto line : std::views::split(DATA, '\n') | std::views::filter([](auto const& line) { return !line.empty(); })) {
-        auto values = std::views::split(line, ' ');
-        auto it = values.begin();
-        std::size_t left = std::stol(std::string((*it).data(), (*it).size()));
-        it = std::next(it);
-        std::size_t right = std::stol(std::string((*it).data(), (*it).size()));
-        it = std::next(it);
-        bool result = static_cast<bool>(std::stol(std::string((*it).data(), (*it).size())));
-        comparison_results[left][right] = result;
-    }
-    auto predicate = [&](std::size_t* left, std::size_t* right) {
+class ComparisonResults {
+public:
+    explicit ComparisonResults(std::string_view data) {
+        for (auto line : std::views::split(data, '\n') | std::views::filter([](auto const& line) { return !line.empty(); })) {
+            auto values = std::views::split(line, ' ');
+            auto it = values.begin();
+            std::size_t left = std::stol(std::string((*it).data(), (*it).size()));
+            it = std::next(it);
+            std::size_t right = std::stol(std::string((*it).data(), (*it).size()));
+            it = std::next(it);
+            bool result = static_cast<bool>(std::stol(std::string((*it).data(), (*it).size())));
+            comparison_results[left][right] = result;
+        }
+    }
+
+    bool compare(size_t* left, size_t* right) const {
         assert(left != nullptr && right != nullptr && "something is wrong with the test");
-        assert(comparison_results.contains(*left) && comparison_results[*left].contains(*right) && "malformed input data?");
-        return comparison_results[*left][*right];
-    };
+        assert(comparison_results.contains(*left) && comparison_results.at(*left).contains(*right) && "malformed input data?");
+        return comparison_results.at(*left).at(*right);
+    }
 
+    size_t size() const { return comparison_results.size(); }
+private:
+    std::map<std::size_t, std::map<std::size_t, bool>> comparison_results; // terrible for performance, but really convenient
+};
+
+void check_oob_sort_read() {
+    ComparisonResults comparison_results(SORT_DATA);
     std::vector<std::unique_ptr<std::size_t>> elements;
     std::set<std::size_t*> valid_ptrs;
     for (std::size_t i = 0; i != comparison_results.size(); ++i) {
@@ -81,7 +91,7 @@ void check_oob_sort_read() {
         // because we're reading OOB.
         assert(valid_ptrs.contains(left));
         assert(valid_ptrs.contains(right));
-        return predicate(left, right);
+        return comparison_results.compare(left, right);
     };
 
     // Check the classic sorting algorithms
@@ -117,12 +127,6 @@ void check_oob_sort_read() {
         std::vector<std::size_t*> results(copy.size(), nullptr);
        TEST_LIBCPP_ASSERT_FAILURE(std::partial_sort_copy(copy.begin(), copy.end(), results.begin(), results.end(), checked_predicate), "not a valid strict-weak ordering");
     }
-    {
-        std::vector<std::size_t*> copy;
-        for (auto const& e : elements)
-            copy.push_back(e.get());
-        std::nth_element(copy.begin(), copy.end(), copy.end(), checked_predicate); // doesn't go OOB even with invalid comparator
-    }
 
     // Check the Ranges sorting algorithms
     {
@@ -157,11 +161,38 @@ void check_oob_sort_read() {
         std::vector<std::size_t*> results(copy.size(), nullptr);
         TEST_LIBCPP_ASSERT_FAILURE(std::ranges::partial_sort_copy(copy, results, checked_predicate), "not a valid strict-weak ordering");
     }
+}
+
+void check_oob_nth_element_read() {
+    ComparisonResults results(NTH_ELEMENT_DATA);
+    std::vector<std::unique_ptr<std::size_t>> elements;
+    std::set<std::size_t*> valid_ptrs;
+    for (std::size_t i = 0; i != results.size(); ++i) {
+        elements.push_back(std::make_unique<std::size_t>(i));
+        valid_ptrs.insert(elements.back().get());
+    }
+
+    auto checked_predicate = [&](size_t* left, size_t* right) {
+        // If the pointers passed to the comparator are not in the set of pointers we
+        // set up above, then we're being passed garbage values from the algorithm
+        // because we're reading OOB.
+        assert(valid_ptrs.contains(left));
+        assert(valid_ptrs.contains(right));
+        return results.compare(left, right);
+    };
+
     {
         std::vector<std::size_t*> copy;
         for (auto const& e : elements)
             copy.push_back(e.get());
-        std::ranges::nth_element(copy, copy.end(), checked_predicate); // doesn't go OOB even with invalid comparator
+        TEST_LIBCPP_ASSERT_FAILURE(std::nth_element(copy.begin(), copy.begin(), copy.end(), checked_predicate), "Would read out of bounds");
+    }
+
+    {
+        std::vector<std::size_t*> copy;
+        for (auto const& e : elements)
+            copy.push_back(e.get());
+        TEST_LIBCPP_ASSERT_FAILURE(std::ranges::nth_element(copy, copy.begin(), checked_predicate), "Would read out of bounds");
     }
 }
 
@@ -214,6 +245,8 @@ int main(int, char**) {
 
     check_oob_sort_read();
 
+    check_oob_nth_element_read();
+
     check_nan_floats();
 
     check_irreflexive();
diff --git a/libcxx/test/libcxx/algorithms/alg.sorting/bad_comparator_values.h b/libcxx/test/libcxx/algorithms/alg.sorting/bad_comparator_values.h
index 19ea023419..c0ffd16cd4 100644
--- a/libcxx/test/libcxx/algorithms/alg.sorting/bad_comparator_values.h
+++ b/libcxx/test/libcxx/algorithms/alg.sorting/bad_comparator_values.h
@@ -11,7 +11,74 @@
 
 #include <string_view>
 
-inline constexpr std::string_view DATA = R"(
+inline constexpr std::string_view NTH_ELEMENT_DATA = R"(
+0 0 0
+0 1 0
+0 2 0
+0 3 0
+0 4 1
+0 5 0
+0 6 0
+0 7 0
+1 0 0
+1 1 0
+1 2 0
+1 3 1
+1 4 1
+1 5 1
+1 6 1
+1 7 1
+2 0 1
+2 1 1
+2 2 1
+2 3 1
+2 4 1
+2 5 1
+2 6 1
+2 7 1
+3 0 1
+3 1 1
+3 2 1
+3 3 1
+3 4 1
+3 5 1
+3 6 1
+3 7 1
+4 0 1
+4 1 1
+4 2 1
+4 3 1
+4 4 1
+4 5 1
+4 6 1
+4 7 1
+5 0 1
+5 1 1
+5 2 1
+5 3 1
+5 4 1
+5 5 1
+5 6 1
+5 7 1
+6 0 1
+6 1 1
+6 2 1
+6 3 1
+6 4 1
+6 5 1
+6 6 1
+6 7 1
+7 0 1
+7 1 1
+7 2 1
+7 3 1
+7 4 1
+7 5 1
+7 6 1
+7 7 1
+)";
+
+inline constexpr std::string_view SORT_DATA = R"(
 0 0 0
 0 1 1
 0 2 1
diff --git a/libcxx/test/libcxx/algorithms/pstl.robust_against_customization_points_not_working.pass.cpp b/libcxx/test/libcxx/algorithms/pstl.robust_against_customization_points_not_working.pass.cpp
index e3733bf468..071fc50b2f 100644
--- a/libcxx/test/libcxx/algorithms/pstl.robust_against_customization_points_not_working.pass.cpp
+++ b/libcxx/test/libcxx/algorithms/pstl.robust_against_customization_points_not_working.pass.cpp
@@ -172,6 +172,15 @@ optional<__empty> __pstl_fill_n(TestBackend, ForwardIterator, Size, Func) {
   return __empty{};
 }
 
+bool pstl_move_called = false;
+
+template <class, class ForwardIterator, class Size, class Func>
+ForwardIterator __pstl_move(TestBackend, ForwardIterator, Size, Func) {
+  assert(!pstl_move_called);
+  pstl_move_called = true;
+  return 0;
+}
+
 bool pstl_is_partitioned_called = false;
 
 template <class, class ForwardIterator, class Func>
@@ -352,7 +361,9 @@ int main(int, char**) {
   (void)std::generate_n(TestPolicy{}, std::begin(a), std::size(a), pred);
   assert(std::pstl_generate_n_called);
   (void)std::is_partitioned(TestPolicy{}, std::begin(a), std::end(a), pred);
-  assert(std::pstl_generate_n_called);
+  assert(std::pstl_is_partitioned_called);
+  (void)std::move(TestPolicy{}, std::begin(a), std::end(a), std::begin(a));
+  assert(std::pstl_move_called);
   (void)std::replace(TestPolicy{}, std::begin(a), std::end(a), 0, 0);
   assert(std::pstl_replace_called);
   (void)std::replace_if(TestPolicy{}, std::begin(a), std::end(a), pred, 0);
diff --git a/libcxx/test/libcxx/atomics/atomics.align/align.pass.cpp b/libcxx/test/libcxx/atomics/atomics.align/align.pass.cpp
index 495d02fbe5..f9e01bd5d0 100644
--- a/libcxx/test/libcxx/atomics/atomics.align/align.pass.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.align/align.pass.cpp
@@ -100,6 +100,7 @@ int main(int, char**) {
   CHECK_ALIGNMENT(struct Empty {});
   CHECK_ALIGNMENT(struct OneInt { int i; });
   CHECK_ALIGNMENT(struct IntArr2 { int i[2]; });
+  CHECK_ALIGNMENT(struct FloatArr3 { float i[3]; });
   CHECK_ALIGNMENT(struct LLIArr2 { long long int i[2]; });
   CHECK_ALIGNMENT(struct LLIArr4 { long long int i[4]; });
   CHECK_ALIGNMENT(struct LLIArr8 { long long int i[8]; });
diff --git a/libcxx/test/libcxx/clang_modules_include.gen.py b/libcxx/test/libcxx/clang_modules_include.gen.py
index b4c5c79a47..04c8e03aff 100644
--- a/libcxx/test/libcxx/clang_modules_include.gen.py
+++ b/libcxx/test/libcxx/clang_modules_include.gen.py
@@ -34,7 +34,7 @@
 // UNSUPPORTED{BLOCKLIT}: LIBCXX-AIX-FIXME
 
 // The Android headers don't appear to be compatible with modules yet
-// XFAIL{BLOCKLIT}: LIBCXX-ANDROID-FIXME
+// UNSUPPORTED{BLOCKLIT}: LIBCXX-ANDROID-FIXME
 
 // TODO: Investigate this failure
 // UNSUPPORTED{BLOCKLIT}: LIBCXX-FREEBSD-FIXME
@@ -64,7 +64,7 @@
 // UNSUPPORTED{BLOCKLIT}: LIBCXX-AIX-FIXME
 
 // The Android headers don't appear to be compatible with modules yet
-// XFAIL{BLOCKLIT}: LIBCXX-ANDROID-FIXME
+// UNSUPPORTED{BLOCKLIT}: LIBCXX-ANDROID-FIXME
 
 // TODO: Investigate this failure
 // UNSUPPORTED{BLOCKLIT}: LIBCXX-FREEBSD-FIXME
diff --git a/libcxx/test/libcxx/containers/views/mdspan/layout_left/assert.ctor.layout_stride.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/layout_left/assert.ctor.layout_stride.pass.cpp
new file mode 100644
index 0000000000..25ffa46a90
--- /dev/null
+++ b/libcxx/test/libcxx/containers/views/mdspan/layout_left/assert.ctor.layout_stride.pass.cpp
@@ -0,0 +1,83 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: has-unix-headers
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+// UNSUPPORTED: libcpp-hardening-mode=unchecked
+// XFAIL: availability-verbose_abort-missing
+// ADDITIONAL_COMPILE_FLAGS: -Wno-ctad-maybe-unsupported
+
+// FIXME: https://github.com/llvm/llvm-project/issues/64719
+// There appear to be some issues around ctad which make it
+// currently impossible to get this code warning free.
+// Thus added the additional compile flag above
+
+// <mdspan>
+
+// template<class OtherExtents>
+//   constexpr explicit(extents_type::rank() > 0)
+//     mapping(const layout_stride::mapping<OtherExtents>& other);
+//
+// Constraints: is_constructible_v<extents_type, OtherExtents> is true.
+//
+// Preconditions:
+//   - If extents_type::rank() > 0 is true, then for all r in the range [0, extents_type::rank()),
+//     other.stride(r) equals other.extents().fwd-prod-of-extents(r), and
+//   - other.required_span_size() is representable as a value of type index_type ([basic.fundamental]).
+//
+// Effects: Direct-non-list-initializes extents_ with other.extents().
+
+#include <mdspan>
+#include <cassert>
+
+#include "check_assertion.h"
+
+int main(int, char**) {
+  constexpr size_t D = std::dynamic_extent;
+
+  // working case
+  {
+    std::layout_stride::mapping arg(std::extents<int, D>(5), std::array<int, 1>{1});
+    [[maybe_unused]] std::layout_left::mapping<std::extents<size_t, 5>> m(arg); // should work
+  }
+  // mismatch of static extent
+  {
+    std::layout_stride::mapping arg(std::extents<int, D>(5), std::array<int, 1>{1});
+    TEST_LIBCPP_ASSERT_FAILURE(([=] { std::layout_left::mapping<std::extents<int, 3>> m(arg); }()),
+                               "extents construction: mismatch of provided arguments with static extents.");
+  }
+  // non-representability of extents itself
+  {
+    std::layout_stride::mapping arg(std::extents<int, D>(500), std::array<int, 1>{1});
+    TEST_LIBCPP_ASSERT_FAILURE(([=] { std::layout_left::mapping<std::extents<char, D>> m(arg); }()),
+                               "extents ctor: arguments must be representable as index_type and nonnegative");
+  }
+  // non-representability of required span size
+  {
+    std::layout_stride::mapping arg(std::extents<int, D, D>(100, 3), std::array<int, 2>{1, 100});
+    TEST_LIBCPP_ASSERT_FAILURE(
+        ([=] { std::layout_left::mapping<std::extents<char, D, D>> m(arg); }()),
+        "layout_left::mapping from layout_stride ctor: other.required_span_size() must be "
+        "representable as index_type.");
+  }
+  // strides are not layout_left compatible
+  {
+    std::layout_stride::mapping arg(std::extents<int, D>(5), std::array<int, 1>{2});
+    TEST_LIBCPP_ASSERT_FAILURE(
+        ([=] { std::layout_left::mapping<std::extents<size_t, 5>> m(arg); }()),
+        "layout_left::mapping from layout_stride ctor: strides are not compatible with layout_left.");
+  }
+  {
+    std::layout_stride::mapping arg(std::extents<int, D, D>(100, 3), std::array<int, 2>{2, 200});
+    TEST_LIBCPP_ASSERT_FAILURE(
+        ([=] { std::layout_left::mapping<std::extents<int, D, D>> m(arg); }()),
+        "layout_left::mapping from layout_stride ctor: strides are not compatible with layout_left.");
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/libcxx/containers/views/mdspan/layout_right/assert.ctor.layout_stride.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/layout_right/assert.ctor.layout_stride.pass.cpp
new file mode 100644
index 0000000000..b8b78fceeb
--- /dev/null
+++ b/libcxx/test/libcxx/containers/views/mdspan/layout_right/assert.ctor.layout_stride.pass.cpp
@@ -0,0 +1,83 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: has-unix-headers
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+// UNSUPPORTED: libcpp-hardening-mode=unchecked
+// XFAIL: availability-verbose_abort-missing
+// ADDITIONAL_COMPILE_FLAGS: -Wno-ctad-maybe-unsupported
+
+// FIXME: https://github.com/llvm/llvm-project/issues/64719
+// There appear to be some issues around ctad which make it
+// currently impossible to get this code warning free.
+// Thus added the additional compile flag above
+
+// <mdspan>
+
+// template<class OtherExtents>
+//   constexpr explicit(extents_type::rank() > 0)
+//     mapping(const layout_stride::mapping<OtherExtents>& other);
+//
+// Constraints: is_constructible_v<extents_type, OtherExtents> is true.
+//
+// Preconditions:
+//   - If extents_type::rank() > 0 is true, then for all r in the range [0, extents_type::rank()),
+//     other.stride(r) equals other.extents().fwd-prod-of-extents(r), and
+//   - other.required_span_size() is representable as a value of type index_type ([basic.fundamental]).
+//
+// Effects: Direct-non-list-initializes extents_ with other.extents().
+
+#include <mdspan>
+#include <cassert>
+
+#include "check_assertion.h"
+
+int main(int, char**) {
+  constexpr size_t D = std::dynamic_extent;
+
+  // working case
+  {
+    std::layout_stride::mapping arg(std::extents<int, D>(5), std::array<int, 1>{1});
+    [[maybe_unused]] std::layout_right::mapping<std::extents<size_t, 5>> m(arg); // should work
+  }
+  // mismatch of static extent
+  {
+    std::layout_stride::mapping arg(std::extents<int, D>(5), std::array<int, 1>{1});
+    TEST_LIBCPP_ASSERT_FAILURE(([=] { std::layout_right::mapping<std::extents<int, 3>> m(arg); }()),
+                               "extents construction: mismatch of provided arguments with static extents.");
+  }
+  // non-representability of extents itself
+  {
+    std::layout_stride::mapping arg(std::extents<int, D>(500), std::array<int, 1>{1});
+    TEST_LIBCPP_ASSERT_FAILURE(([=] { std::layout_right::mapping<std::extents<char, D>> m(arg); }()),
+                               "extents ctor: arguments must be representable as index_type and nonnegative");
+  }
+  // non-representability of required span size
+  {
+    std::layout_stride::mapping arg(std::extents<int, D, D>(100, 3), std::array<int, 2>{3, 1});
+    TEST_LIBCPP_ASSERT_FAILURE(
+        ([=] { std::layout_right::mapping<std::extents<char, D, D>> m(arg); }()),
+        "layout_right::mapping from layout_stride ctor: other.required_span_size() must be "
+        "representable as index_type.");
+  }
+  // strides are not layout_right compatible
+  {
+    std::layout_stride::mapping arg(std::extents<int, D>(5), std::array<int, 1>{2});
+    TEST_LIBCPP_ASSERT_FAILURE(
+        ([=] { std::layout_right::mapping<std::extents<size_t, 5>> m(arg); }()),
+        "layout_right::mapping from layout_stride ctor: strides are not compatible with layout_right.");
+  }
+  {
+    std::layout_stride::mapping arg(std::extents<int, D, D>(100, 3), std::array<int, 2>{6, 2});
+    TEST_LIBCPP_ASSERT_FAILURE(
+        ([=] { std::layout_right::mapping<std::extents<int, D, D>> m(arg); }()),
+        "layout_right::mapping from layout_stride ctor: strides are not compatible with layout_right.");
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.conversion.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.conversion.pass.cpp
new file mode 100644
index 0000000000..e8bad8b0d5
--- /dev/null
+++ b/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.conversion.pass.cpp
@@ -0,0 +1,112 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: has-unix-headers
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+// UNSUPPORTED: libcpp-hardening-mode=unchecked
+// XFAIL: availability-verbose_abort-missing
+
+// <mdspan>
+
+// template<class StridedLayoutMapping>
+//   constexpr explicit(see below)
+//     mapping(const StridedLayoutMapping& other) noexcept;
+//
+// Constraints:
+//   - layout-mapping-alike<StridedLayoutMapping> is satisfied.
+//   - is_constructible_v<extents_type, typename StridedLayoutMapping::extents_type> is true.
+//   - StridedLayoutMapping::is_always_unique() is true.
+//   - StridedLayoutMapping::is_always_strided() is true.
+//
+// Preconditions:
+//   - StridedLayoutMapping meets the layout mapping requirements ([mdspan.layout.policy.reqmts]),
+//   - other.stride(r) > 0 is true for every rank index r of extents(),
+//   - other.required_span_size() is representable as a value of type index_type ([basic.fundamental]), and
+//   - OFFSET(other) == 0 is true.
+//
+// Effects: Direct-non-list-initializes extents_ with other.extents(), and for all d in the range [0, rank_),
+//          direct-non-list-initializes strides_[d] with other.stride(d).
+//
+// Remarks: The expression inside explicit is equivalent to:
+//   - !(is_convertible_v<typename StridedLayoutMapping::extents_type, extents_type> &&
+//       (is-mapping-of<layout_left, LayoutStrideMapping> ||
+//        is-mapping-of<layout_right, LayoutStrideMapping> ||
+//        is-mapping-of<layout_stride, LayoutStrideMapping>))
+
+#include <mdspan>
+#include <cassert>
+
+#include "check_assertion.h"
+#include "../../../../../std/containers/views/mdspan/CustomTestLayouts.h"
+
+int main(int, char**) {
+  constexpr size_t D = std::dynamic_extent;
+
+  // working case
+  {
+    std::extents<int, D, D> arg_exts{100, 5};
+    std::layout_stride::mapping<std::extents<int, D, D>> arg(arg_exts, std::array<int, 2>{1, 100});
+    [[maybe_unused]] std::layout_stride::mapping<std::extents<size_t, D, 5>> m(arg); // should work
+  }
+  // mismatch of static extent
+  {
+    std::extents<int, D, D> arg_exts{100, 5};
+    std::layout_stride::mapping<std::extents<int, D, D>> arg(arg_exts, std::array<int, 2>{1, 100});
+    TEST_LIBCPP_ASSERT_FAILURE(([=] { std::layout_stride::mapping<std::extents<int, D, 3>> m(arg); }()),
+                               "extents construction: mismatch of provided arguments with static extents.");
+  }
+  // non-representability of extents itself
+  {
+    TEST_LIBCPP_ASSERT_FAILURE(
+        ([=] {
+          std::layout_stride::mapping<std::extents<char, D>> m(
+              std::layout_stride::mapping<std::extents<int, D>>(std::extents<int, D>(500), std::array<int, 1>{1}));
+        }()),
+        "extents ctor: arguments must be representable as index_type and nonnegative");
+  }
+  // all strides must be larger than zero
+  {
+    always_convertible_layout::mapping<std::dextents<int, 2>> offset_map(std::dextents<int, 2>{10, 10}, 100, -1);
+    TEST_LIBCPP_ASSERT_FAILURE(
+        ([=] { std::layout_stride::template mapping<std::extents<char, D, D>> m(offset_map); }()),
+        "layout_stride::mapping converting ctor: all strides must be greater than 0");
+  }
+  // required_span_size not representable, while individual extents are
+  {
+    std::extents<int, D, D> arg_exts{100, 5};
+    std::layout_stride::mapping<std::extents<int, D, D>> arg(arg_exts, std::array<int, 2>{1, 100});
+    // check extents would be constructible
+    [[maybe_unused]] std::extents<char, D, 5> e(arg_exts);
+    // but the product is not, so we can't use it for layout_stride
+    TEST_LIBCPP_ASSERT_FAILURE(
+        ([=] { std::layout_stride::template mapping<std::extents<char, D, 5>> m(arg); }()),
+        "layout_stride::mapping converting ctor: other.required_span_size() must be representable as index_type.");
+  }
+  // required_span_size not representable, while individual extents are, edge case
+  {
+    // required span size = (3-1)*50 + (10-1) * 3 + 1 = 128
+    std::extents<int, D, D> arg_exts{3, 10};
+    std::layout_stride::mapping<std::extents<int, D, D>> arg(arg_exts, std::array<int, 2>{50, 3});
+    // sanity check:
+    assert(arg.required_span_size() == 128);
+    // check extents would be constructible
+    [[maybe_unused]] std::extents<signed char, D, 10> e(arg_exts);
+    // but the product is not, so we can't use it for layout_stride
+    TEST_LIBCPP_ASSERT_FAILURE(
+        ([=] { std::layout_stride::template mapping<std::extents<signed char, D, 10>> m(arg); }()),
+        "layout_stride::mapping converting ctor: other.required_span_size() must be representable as index_type.");
+  }
+  // base offset must be 0 (i.e. mapping(0,...,0)==0) for a strided layout with positiv strides
+  {
+    always_convertible_layout::mapping<std::dextents<int, 2>> offset_map(std::dextents<int, 2>{10, 10}, 3);
+    TEST_LIBCPP_ASSERT_FAILURE(
+        ([=] { std::layout_stride::template mapping<std::extents<char, D, D>> m(offset_map); }()),
+        "layout_stride::mapping converting ctor: base offset of mapping must be zero.");
+  }
+  return 0;
+}
diff --git a/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_array.non_unique.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_array.non_unique.pass.cpp
new file mode 100644
index 0000000000..ba1c81d35d
--- /dev/null
+++ b/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_array.non_unique.pass.cpp
@@ -0,0 +1,67 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: has-unix-headers
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+// UNSUPPORTED: !libcpp-hardening-mode=debug
+// XFAIL: availability-verbose_abort-missing
+
+// <mdspan>
+
+// template<class OtherIndexType>
+//   constexpr mapping(const extents_type& e, const array<OtherIndexType, rank_>& s) noexcept;
+//
+// Constraints:
+//   - is_convertible_v<const OtherIndexType&, index_type> is true, and
+//   - is_nothrow_constructible_v<index_type, const OtherIndexType&> is true.
+//
+// Preconditions:
+//   - s[i] > 0 is true for all i in the range [0, rank_).
+//   - REQUIRED-SPAN-SIZE(e, s) is representable as a value of type index_type ([basic.fundamental]).
+//   - If rank_ is greater than 0, then there exists a permutation P of the integers in the range [0, rank_),
+//     such that s[pi] >= s[pi_1] * e.extent(pi_1) is true for all i in the range [1, rank_), where
+//     pi is the ith element of P.
+//     [Note 1: For layout_stride, this condition is necessary and sufficient for is_unique() to be true. end note]
+//
+// Effects: Direct-non-list-initializes extents_ with e, and for all d in the range [0, rank_), direct-non-list-initializes strides_[d] with as_const(s[d]).
+
+#include <mdspan>
+#include <cassert>
+
+#include "check_assertion.h"
+
+int main(int, char**) {
+  constexpr size_t D = std::dynamic_extent;
+
+  // overlapping strides
+  {
+    TEST_LIBCPP_ASSERT_FAILURE(
+        ([=] {
+          std::layout_stride::template mapping<std::extents<unsigned, D, 5, 7>> m(
+              std::extents<unsigned, D, 5, 7>(20), std::array<unsigned, 3>{4, 1, 200});
+        }()),
+        "layout_stride::mapping ctor: the provided extents and strides lead to a non-unique mapping");
+  }
+  // equal strides
+  {
+    // should work because one of the equal strides is associated with an extent of 1
+    [[maybe_unused]] std::layout_stride::template mapping<std::extents<unsigned, D, 5, 1>> m1(
+        std::extents<unsigned, D, 5, 1>(2), std::array<unsigned, 3>{5, 1, 5});
+    [[maybe_unused]] std::layout_stride::template mapping<std::extents<unsigned, D, 5, 2>> m2(
+        std::extents<unsigned, D, 5, 2>(1), std::array<unsigned, 3>{5, 1, 5});
+
+    // will fail because neither of the equal strides is associated with an extent of 1
+    TEST_LIBCPP_ASSERT_FAILURE(
+        ([=] {
+          std::layout_stride::template mapping<std::extents<unsigned, D, 5, 2>> m3(
+              std::extents<unsigned, D, 5, 2>(2), std::array<unsigned, 3>{5, 1, 5});
+        }()),
+        "layout_stride::mapping ctor: the provided extents and strides lead to a non-unique mapping");
+  }
+  return 0;
+}
diff --git a/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_array.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_array.pass.cpp
new file mode 100644
index 0000000000..6eea40c239
--- /dev/null
+++ b/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_array.pass.cpp
@@ -0,0 +1,73 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: has-unix-headers
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+// UNSUPPORTED: libcpp-hardening-mode=unchecked
+// XFAIL: availability-verbose_abort-missing
+
+// <mdspan>
+
+// template<class OtherIndexType>
+//   constexpr mapping(const extents_type& e, const array<OtherIndexType, rank_>& s) noexcept;
+//
+// Constraints:
+//   - is_convertible_v<const OtherIndexType&, index_type> is true, and
+//   - is_nothrow_constructible_v<index_type, const OtherIndexType&> is true.
+//
+// Preconditions:
+//   - s[i] > 0 is true for all i in the range [0, rank_).
+//   - REQUIRED-SPAN-SIZE(e, s) is representable as a value of type index_type ([basic.fundamental]).
+//   - If rank_ is greater than 0, then there exists a permutation P of the integers in the range [0, rank_),
+//     such that s[pi] >= s[pi_1] * e.extent(pi_1) is true for all i in the range [1, rank_), where
+//     pi is the ith element of P.
+//     [Note 1: For layout_stride, this condition is necessary and sufficient for is_unique() to be true. end note]
+//
+// Effects: Direct-non-list-initializes extents_ with e, and for all d in the range [0, rank_), direct-non-list-initializes strides_[d] with as_const(s[d]).
+
+#include <mdspan>
+#include <cassert>
+
+#include "check_assertion.h"
+
+int main(int, char**) {
+  constexpr size_t D = std::dynamic_extent;
+
+  // the extents are representable but the product with strides is not, so we can't use it for layout_stride
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([=] {
+        std::layout_stride::template mapping<std::extents<char, D, 5>> m(
+            std::extents<char, D, 5>(20), std::array<int, 2>{20, 1});
+      }()),
+      "layout_stride::mapping ctor: required span size is not representable as index_type.");
+
+  // check that if we first overflow in strides conversion we also fail
+  static_assert(static_cast<unsigned char>(257u) == 1);
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([=] {
+        std::layout_stride::template mapping<std::extents<unsigned char, D, 5>> m(
+            std::extents<unsigned char, D, 5>(20), std::array<unsigned, 2>{257, 1});
+      }()),
+      "layout_stride::mapping ctor: required span size is not representable as index_type.");
+
+  // negative strides are not allowed, check with unsigned index_type so we make sure we catch that
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([=] {
+        std::layout_stride::template mapping<std::extents<unsigned, D, 5>> m(
+            std::extents<unsigned, D, 5>(20), std::array<int, 2>{20, -1});
+      }()),
+      "layout_stride::mapping ctor: all strides must be greater than 0");
+  // zero strides are not allowed, check with unsigned index_type so we make sure we catch that
+  TEST_LIBCPP_ASSERT_FAILURE(
+      ([=] {
+        std::layout_stride::template mapping<std::extents<unsigned, D, 5>> m(
+            std::extents<unsigned, D, 5>(20), std::array<unsigned, 2>{20, 0});
+      }()),
+      "layout_stride::mapping ctor: all strides must be greater than 0");
+  return 0;
+}
diff --git a/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_span.non_unique.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_span.non_unique.pass.cpp
new file mode 100644
index 0000000000..b20f657624
--- /dev/null
+++ b/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_span.non_unique.pass.cpp
@@ -0,0 +1,70 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: has-unix-headers
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+// UNSUPPORTED: !libcpp-hardening-mode=debug
+// XFAIL: availability-verbose_abort-missing
+
+// <mdspan>
+
+// template<class OtherIndexType>
+//   constexpr mapping(const extents_type& e, span<OtherIndexType, rank_> s) noexcept;
+//
+// Constraints:
+//   - is_convertible_v<const OtherIndexType&, index_type> is true, and
+//   - is_nothrow_constructible_v<index_type, const OtherIndexType&> is true.
+//
+// Preconditions:
+//   - s[i] > 0 is true for all i in the range [0, rank_).
+//   - REQUIRED-SPAN-SIZE(e, s) is representable as a value of type index_type ([basic.fundamental]).
+//   - If rank_ is greater than 0, then there exists a permutation P of the integers in the range [0, rank_),
+//     such that s[pi] >= s[pi_1] * e.extent(pi_1) is true for all i in the range [1, rank_), where
+//     pi is the ith element of P.
+//     [Note 1: For layout_stride, this condition is necessary and sufficient for is_unique() to be true. end note]
+//
+// Effects: Direct-non-list-initializes extents_ with e, and for all d in the range [0, rank_), direct-non-list-initializes strides_[d] with as_const(s[d]).
+
+#include <mdspan>
+#include <cassert>
+
+#include "check_assertion.h"
+
+int main(int, char**) {
+  constexpr size_t D = std::dynamic_extent;
+
+  // overlapping strides
+  {
+    TEST_LIBCPP_ASSERT_FAILURE(
+        ([=] {
+          std::array<unsigned, 3> strides{4, 1, 200};
+          std::layout_stride::template mapping<std::extents<unsigned, D, 5, 7>> m(
+              std::extents<unsigned, D, 5, 7>(20), std::span(strides));
+        }()),
+        "layout_stride::mapping ctor: the provided extents and strides lead to a non-unique mapping");
+  }
+
+  // equal strides
+  {
+    // should work because one of the equal strides is associated with an extent of 1
+    std::array<unsigned, 3> strides{5, 1, 5};
+    [[maybe_unused]] std::layout_stride::template mapping<std::extents<unsigned, D, 5, 1>> m1(
+        std::extents<unsigned, D, 5, 1>(2), std::span(strides));
+    [[maybe_unused]] std::layout_stride::template mapping<std::extents<unsigned, D, 5, 2>> m2(
+        std::extents<unsigned, D, 5, 2>(1), std::span(strides));
+
+    // will fail because neither of the equal strides is associated with an extent of 1
+    TEST_LIBCPP_ASSERT_FAILURE(
+        ([=] {
+          std::layout_stride::template mapping<std::extents<unsigned, D, 5, 2>> m3(
+              std::extents<unsigned, D, 5, 2>(2), std::span(strides));
+        }()),
+        "layout_stride::mapping ctor: the provided extents and strides lead to a non-unique mapping");
+  }
+  return 0;
+}
diff --git a/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_span.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_span.pass.cpp
new file mode 100644
index 0000000000..cd06bbb545
--- /dev/null
+++ b/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.ctor.extents_span.pass.cpp
@@ -0,0 +1,80 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: has-unix-headers
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+// UNSUPPORTED: libcpp-hardening-mode=unchecked
+// XFAIL: availability-verbose_abort-missing
+
+// <mdspan>
+
+// template<class OtherIndexType>
+//   constexpr mapping(const extents_type& e, span<OtherIndexType, rank_> s) noexcept;
+//
+// Constraints:
+//   - is_convertible_v<const OtherIndexType&, index_type> is true, and
+//   - is_nothrow_constructible_v<index_type, const OtherIndexType&> is true.
+//
+// Preconditions:
+//   - s[i] > 0 is true for all i in the range [0, rank_).
+//   - REQUIRED-SPAN-SIZE(e, s) is representable as a value of type index_type ([basic.fundamental]).
+//   - If rank_ is greater than 0, then there exists a permutation P of the integers in the range [0, rank_),
+//     such that s[pi] >= s[pi_1] * e.extent(pi_1) is true for all i in the range [1, rank_), where
+//     pi is the ith element of P.
+//     [Note 1: For layout_stride, this condition is necessary and sufficient for is_unique() to be true. end note]
+//
+// Effects: Direct-non-list-initializes extents_ with e, and for all d in the range [0, rank_), direct-non-list-initializes strides_[d] with as_const(s[d]).
+
+#include <mdspan>
+#include <cassert>
+
+#include "check_assertion.h"
+
+int main(int, char**) {
+  constexpr size_t D = std::dynamic_extent;
+
+  // value out of range
+  {
+    // the extents are representable but the product with strides is not, so we can't use it for layout_stride
+    TEST_LIBCPP_ASSERT_FAILURE(
+        ([=] {
+          std::array<int, 2> strides{20, 1};
+          std::layout_stride::template mapping<std::extents<char, D, 5>> m(
+              std::extents<char, D, 5>(20), std::span(strides));
+        }()),
+        "layout_stride::mapping ctor: required span size is not representable as index_type.");
+
+    // check that if we first overflow in strides conversion we also fail
+    static_assert(static_cast<unsigned char>(257u) == 1);
+    TEST_LIBCPP_ASSERT_FAILURE(
+        ([=] {
+          std::array<unsigned, 2> strides{257, 1};
+          std::layout_stride::template mapping<std::extents<unsigned char, D, 5>> m(
+              std::extents<unsigned char, D, 5>(20), std::span(strides));
+        }()),
+        "layout_stride::mapping ctor: required span size is not representable as index_type.");
+
+    // negative strides are not allowed, check with unsigned index_type so we make sure we catch that
+    TEST_LIBCPP_ASSERT_FAILURE(
+        ([=] {
+          std::array<int, 2> strides{20, -1};
+          std::layout_stride::template mapping<std::extents<unsigned, D, 5>> m(
+              std::extents<unsigned, D, 5>(20), std::span(strides));
+        }()),
+        "layout_stride::mapping ctor: all strides must be greater than 0");
+    // zero strides are not allowed, check with unsigned index_type so we make sure we catch that
+    TEST_LIBCPP_ASSERT_FAILURE(
+        ([=] {
+          std::array<unsigned, 2> strides{20, 0};
+          std::layout_stride::template mapping<std::extents<unsigned, D, 5>> m(
+              std::extents<unsigned, D, 5>(20), std::span(strides));
+        }()),
+        "layout_stride::mapping ctor: all strides must be greater than 0");
+  }
+  return 0;
+}
diff --git a/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.index_operator.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.index_operator.pass.cpp
new file mode 100644
index 0000000000..72e44bd979
--- /dev/null
+++ b/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.index_operator.pass.cpp
@@ -0,0 +1,88 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: has-unix-headers
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+// UNSUPPORTED: !libcpp-hardening-mode=debug
+// XFAIL: availability-verbose_abort-missing
+
+// <mdspan>
+
+// template<class... Indices>
+//   constexpr index_type operator()(Indices... i) const noexcept;
+//
+// Constraints:
+//   - sizeof...(Indices) == extents_type::rank() is true,
+//   - (is_convertible_v<Indices, index_type> && ...) is true, and
+//   - (is_nothrow_constructible_v<index_type, Indices> && ...) is true.
+//
+// Preconditions: extents_type::index-cast(i) is a multidimensional index in extents_ ([mdspan.overview]).
+
+#include <mdspan>
+#include <cassert>
+
+#include "check_assertion.h"
+
+int main(int, char**) {
+  // value out of range
+  {
+    std::layout_stride::template mapping<std::extents<unsigned char, 5>> m;
+    TEST_LIBCPP_ASSERT_FAILURE(m(-1), "layout_stride::mapping: out of bounds indexing");
+    TEST_LIBCPP_ASSERT_FAILURE(m(-130), "layout_stride::mapping: out of bounds indexing");
+    TEST_LIBCPP_ASSERT_FAILURE(m(5), "layout_stride::mapping: out of bounds indexing");
+    TEST_LIBCPP_ASSERT_FAILURE(m(1000), "layout_stride::mapping: out of bounds indexing");
+  }
+  {
+    std::layout_stride::template mapping<std::extents<signed char, 5>> m;
+    TEST_LIBCPP_ASSERT_FAILURE(m(-1), "layout_stride::mapping: out of bounds indexing");
+    TEST_LIBCPP_ASSERT_FAILURE(m(-130), "layout_stride::mapping: out of bounds indexing");
+    TEST_LIBCPP_ASSERT_FAILURE(m(5), "layout_stride::mapping: out of bounds indexing");
+    TEST_LIBCPP_ASSERT_FAILURE(m(1000), "layout_stride::mapping: out of bounds indexing");
+  }
+  {
+    std::layout_stride::template mapping<std::dextents<unsigned char, 1>> m(
+        std::dextents<unsigned char, 1>(5), std::array<int, 1>{1});
+    TEST_LIBCPP_ASSERT_FAILURE(m(-1), "layout_stride::mapping: out of bounds indexing");
+    TEST_LIBCPP_ASSERT_FAILURE(m(-130), "layout_stride::mapping: out of bounds indexing");
+    TEST_LIBCPP_ASSERT_FAILURE(m(5), "layout_stride::mapping: out of bounds indexing");
+    TEST_LIBCPP_ASSERT_FAILURE(m(1000), "layout_stride::mapping: out of bounds indexing");
+  }
+  {
+    std::layout_stride::template mapping<std::dextents<signed char, 1>> m(
+        std::dextents<signed char, 1>(5), std::array<int, 1>{1});
+    TEST_LIBCPP_ASSERT_FAILURE(m(-1), "layout_stride::mapping: out of bounds indexing");
+    TEST_LIBCPP_ASSERT_FAILURE(m(-130), "layout_stride::mapping: out of bounds indexing");
+    TEST_LIBCPP_ASSERT_FAILURE(m(5), "layout_stride::mapping: out of bounds indexing");
+    TEST_LIBCPP_ASSERT_FAILURE(m(1000), "layout_stride::mapping: out of bounds indexing");
+  }
+  {
+    std::layout_stride::template mapping<std::dextents<int, 3>> m(
+        std::dextents<int, 3>(5, 7, 9), std::array<int, 3>{1, 10, 100});
+    TEST_LIBCPP_ASSERT_FAILURE(m(-1, -1, -1), "layout_stride::mapping: out of bounds indexing");
+    TEST_LIBCPP_ASSERT_FAILURE(m(-1, 0, 0), "layout_stride::mapping: out of bounds indexing");
+    TEST_LIBCPP_ASSERT_FAILURE(m(0, -1, 0), "layout_stride::mapping: out of bounds indexing");
+    TEST_LIBCPP_ASSERT_FAILURE(m(0, 0, -1), "layout_stride::mapping: out of bounds indexing");
+    TEST_LIBCPP_ASSERT_FAILURE(m(5, 3, 3), "layout_stride::mapping: out of bounds indexing");
+    TEST_LIBCPP_ASSERT_FAILURE(m(3, 7, 3), "layout_stride::mapping: out of bounds indexing");
+    TEST_LIBCPP_ASSERT_FAILURE(m(3, 3, 9), "layout_stride::mapping: out of bounds indexing");
+    TEST_LIBCPP_ASSERT_FAILURE(m(5, 7, 9), "layout_stride::mapping: out of bounds indexing");
+  }
+  {
+    std::layout_stride::template mapping<std::dextents<unsigned, 3>> m(
+        std::dextents<int, 3>(5, 7, 9), std::array<int, 3>{1, 10, 100});
+    TEST_LIBCPP_ASSERT_FAILURE(m(-1, -1, -1), "layout_stride::mapping: out of bounds indexing");
+    TEST_LIBCPP_ASSERT_FAILURE(m(-1, 0, 0), "layout_stride::mapping: out of bounds indexing");
+    TEST_LIBCPP_ASSERT_FAILURE(m(0, -1, 0), "layout_stride::mapping: out of bounds indexing");
+    TEST_LIBCPP_ASSERT_FAILURE(m(0, 0, -1), "layout_stride::mapping: out of bounds indexing");
+    TEST_LIBCPP_ASSERT_FAILURE(m(5, 3, 3), "layout_stride::mapping: out of bounds indexing");
+    TEST_LIBCPP_ASSERT_FAILURE(m(3, 7, 3), "layout_stride::mapping: out of bounds indexing");
+    TEST_LIBCPP_ASSERT_FAILURE(m(3, 3, 9), "layout_stride::mapping: out of bounds indexing");
+    TEST_LIBCPP_ASSERT_FAILURE(m(5, 7, 9), "layout_stride::mapping: out of bounds indexing");
+  }
+  return 0;
+}
diff --git a/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.stride.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.stride.pass.cpp
new file mode 100644
index 0000000000..8a42861a0b
--- /dev/null
+++ b/libcxx/test/libcxx/containers/views/mdspan/layout_stride/assert.stride.pass.cpp
@@ -0,0 +1,36 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: has-unix-headers
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+// UNSUPPORTED: libcpp-hardening-mode=unchecked
+// XFAIL: availability-verbose_abort-missing
+
+// <mdspan>
+
+// constexpr index_type stride(rank_type i) const noexcept { return strides_[i]; }
+
+// We intercept this inside layout_stride to give a consistent error message with
+// layout_left and layout_right, technically the precondition is coming from the
+// array access.
+
+#include <mdspan>
+#include <cassert>
+
+#include "check_assertion.h"
+
+int main(int, char**) {
+  // value out of range
+  {
+    std::layout_stride::template mapping<std::dextents<int, 3>> m(
+        std::dextents<int, 3>(100, 100, 100), std::array<int, 3>{1, 100, 10000});
+
+    TEST_LIBCPP_ASSERT_FAILURE(m.stride(4), "invalid rank index");
+  }
+  return 0;
+}
diff --git a/libcxx/test/libcxx/containers/views/mdspan/mdspan/assert.conversion.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/mdspan/assert.conversion.pass.cpp
index cb0b2ed15f..8caaf2295c 100644
--- a/libcxx/test/libcxx/containers/views/mdspan/mdspan/assert.conversion.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/mdspan/mdspan/assert.conversion.pass.cpp
@@ -42,7 +42,7 @@
 #include <mdspan>
 
 #include "check_assertion.h"
-#include "../../../../../std/containers/views/mdspan/mdspan/CustomTestLayouts.h"
+#include "../../../../../std/containers/views/mdspan/CustomTestLayouts.h"
 
 // We use a funky mapping in this test that doesn't check the dynamic/static extents mismatch itself
 int main(int, char**) {
diff --git a/libcxx/test/libcxx/containers/views/mdspan/mdspan/assert.size.pass.cpp b/libcxx/test/libcxx/containers/views/mdspan/mdspan/assert.size.pass.cpp
index a10d89bc86..9b5b872bb3 100644
--- a/libcxx/test/libcxx/containers/views/mdspan/mdspan/assert.size.pass.cpp
+++ b/libcxx/test/libcxx/containers/views/mdspan/mdspan/assert.size.pass.cpp
@@ -22,7 +22,7 @@
 #include <mdspan>
 
 #include "check_assertion.h"
-#include "../../../../../std/containers/views/mdspan/mdspan/CustomTestLayouts.h"
+#include "../../../../../std/containers/views/mdspan/CustomTestLayouts.h"
 
 // We use a funky mapping in this test where required_span_size is much smaller than the size of the index space
 int main(int, char**) {
diff --git a/libcxx/test/libcxx/gdb/gdb_pretty_printer_test.sh.cpp b/libcxx/test/libcxx/gdb/gdb_pretty_printer_test.sh.cpp
index 394a3369e0..29315e76ec 100644
--- a/libcxx/test/libcxx/gdb/gdb_pretty_printer_test.sh.cpp
+++ b/libcxx/test/libcxx/gdb/gdb_pretty_printer_test.sh.cpp
@@ -17,6 +17,12 @@
 // TODO: Investigate this failure on GCC 13 (in Ubuntu Jammy)
 // UNSUPPORTED: gcc-13
 
+// The Android libc++ tests are run on a non-Android host, connected to an
+// Android device over adb. gdb needs special support to make this work (e.g.
+// gdbclient.py, ndk-gdb.py, gdbserver), and the Android organization doesn't
+// support gdb anymore, favoring lldb instead.
+// UNSUPPORTED: android
+
 // RUN: %{cxx} %{flags} %s -o %t.exe %{compile_flags} -g %{link_flags}
 // Ensure locale-independence for unicode tests.
 // RUN: env LANG=en_US.UTF-8 %{gdb} -nx -batch -iex "set autoload off" -ex "source %S/../../../utils/gdb/libcxx/printers.py" -ex "python register_libcxx_printer_loader()" -ex "source %S/gdb_pretty_printer_test.py" %t.exe
diff --git a/libcxx/test/libcxx/input.output/iostream.format/print.fun/vprint_unicode_posix.pass.cpp b/libcxx/test/libcxx/input.output/iostream.format/print.fun/vprint_unicode_posix.pass.cpp
index dd4314f4f5..9a50770d97 100644
--- a/libcxx/test/libcxx/input.output/iostream.format/print.fun/vprint_unicode_posix.pass.cpp
+++ b/libcxx/test/libcxx/input.output/iostream.format/print.fun/vprint_unicode_posix.pass.cpp
@@ -11,6 +11,9 @@
 
 // XFAIL: availability-fp_to_chars-missing
 
+// fmemopen is available starting in Android M (API 23)
+// XFAIL: target={{.+}}-android{{(eabi)?(21|22)}}
+
 // REQUIRES: has-unix-headers
 
 // <print>
diff --git a/libcxx/test/libcxx/selftest/gen.cpp/empty.gen.cpp b/libcxx/test/libcxx/selftest/gen.cpp/empty.gen.cpp
index 1915bede47..e0a36db257 100644
--- a/libcxx/test/libcxx/selftest/gen.cpp/empty.gen.cpp
+++ b/libcxx/test/libcxx/selftest/gen.cpp/empty.gen.cpp
@@ -8,4 +8,4 @@
 
 // Make sure we can generate no tests at all
 
-// RUN: true
+// RUN: :
diff --git a/libcxx/test/libcxx/selftest/sh.cpp/run-success.sh.cpp b/libcxx/test/libcxx/selftest/sh.cpp/run-success.sh.cpp
index 1d8747c719..724d03ddb8 100644
--- a/libcxx/test/libcxx/selftest/sh.cpp/run-success.sh.cpp
+++ b/libcxx/test/libcxx/selftest/sh.cpp/run-success.sh.cpp
@@ -8,4 +8,4 @@
 
 // Make sure the test passes if it succeeds to run
 
-// RUN: true
+// RUN: :
diff --git a/libcxx/test/libcxx/strings/basic.string/string.capacity/allocation_size.pass.cpp b/libcxx/test/libcxx/strings/basic.string/string.capacity/allocation_size.pass.cpp
new file mode 100644
index 0000000000..c7df56c815
--- /dev/null
+++ b/libcxx/test/libcxx/strings/basic.string/string.capacity/allocation_size.pass.cpp
@@ -0,0 +1,45 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <string>
+
+// This test demonstrates the smaller allocation sizes when the alignment
+// requirements of std::string are dropped from 16 to 8.
+#include <algorithm>
+#include <cassert>
+#include <cstddef>
+#include <string>
+
+#include "test_macros.h"
+
+// alignment of the string heap buffer is hardcoded to either 16 or 8
+
+const std::size_t alignment =
+#ifdef _LIBCPP_ABI_STRING_8_BYTE_ALIGNMENT
+    8;
+#else
+    16;
+#endif
+
+int main(int, char**) {
+  std::string input_string;
+  input_string.resize(64, 'a');
+
+  // Call a constructor which selects its size using __recommend.
+  std::string test_string(input_string.data());
+  const std::size_t expected_align8_size = 71;
+
+  // Demonstrate the lesser capacity/allocation size when the alignment requirement is 8.
+  if (alignment == 8) {
+    assert(test_string.capacity() == expected_align8_size);
+  } else {
+    assert(test_string.capacity() == expected_align8_size + 8);
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/libcxx/strings/basic.string/string.capacity/max_size.pass.cpp b/libcxx/test/libcxx/strings/basic.string/string.capacity/max_size.pass.cpp
index 5af9cab0be..a3cb79522f 100644
--- a/libcxx/test/libcxx/strings/basic.string/string.capacity/max_size.pass.cpp
+++ b/libcxx/test/libcxx/strings/basic.string/string.capacity/max_size.pass.cpp
@@ -18,7 +18,13 @@
 #include "test_macros.h"
 
 // alignment of the string heap buffer is hardcoded to 16
-static const std::size_t alignment = 16;
+
+static const std::size_t alignment =
+#ifdef _LIBCPP_ABI_STRING_8_BYTE_ALIGNMENT
+    8;
+#else
+    16;
+#endif
 
 template <class = int>
 TEST_CONSTEXPR_CXX20 void full_size() {
diff --git a/libcxx/test/libcxx/strings/basic.string/string.cons/constinit_sso_string.compile.pass.cpp b/libcxx/test/libcxx/strings/basic.string/string.cons/constinit_sso_string.compile.pass.cpp
index 5c0114c029..9fd82f2e2a 100644
--- a/libcxx/test/libcxx/strings/basic.string/string.cons/constinit_sso_string.compile.pass.cpp
+++ b/libcxx/test/libcxx/strings/basic.string/string.cons/constinit_sso_string.compile.pass.cpp
@@ -14,14 +14,12 @@
 #include <string>
 
 #if __SIZE_WIDTH__ == 64
-#define LONGEST_STR "0123456789012345678901"
+#  define LONGEST_STR "0123456789012345678901"
 #elif __SIZE_WIDTH__ == 32
-#define LONGEST_STR "0123456789"
+#  define LONGEST_STR "0123456789"
 #else
 #  error "std::size_t has an unexpected size"
 #endif
 
 constinit std::string g_str = LONGEST_STR;
-void fn() {
-  constexpr std::string l_str = LONGEST_STR;
-}
+void fn() { constexpr std::string l_str = LONGEST_STR; }
diff --git a/libcxx/test/libcxx/transitive_includes/cxx03.csv b/libcxx/test/libcxx/transitive_includes/cxx03.csv
index f6aeb837a7..153335c351 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx03.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx03.csv
@@ -526,6 +526,7 @@ mdspan concepts
 mdspan cstddef
 mdspan limits
 mdspan span
+mdspan version
 memory atomic
 memory compare
 memory concepts
diff --git a/libcxx/test/libcxx/transitive_includes/cxx11.csv b/libcxx/test/libcxx/transitive_includes/cxx11.csv
index 08fd94393f..f3f8956df6 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx11.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx11.csv
@@ -530,6 +530,7 @@ mdspan concepts
 mdspan cstddef
 mdspan limits
 mdspan span
+mdspan version
 memory atomic
 memory compare
 memory concepts
diff --git a/libcxx/test/libcxx/transitive_includes/cxx14.csv b/libcxx/test/libcxx/transitive_includes/cxx14.csv
index 33384a628e..fb90523d55 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx14.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx14.csv
@@ -532,6 +532,7 @@ mdspan concepts
 mdspan cstddef
 mdspan limits
 mdspan span
+mdspan version
 memory atomic
 memory compare
 memory concepts
diff --git a/libcxx/test/libcxx/transitive_includes/cxx17.csv b/libcxx/test/libcxx/transitive_includes/cxx17.csv
index 33384a628e..fb90523d55 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx17.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx17.csv
@@ -532,6 +532,7 @@ mdspan concepts
 mdspan cstddef
 mdspan limits
 mdspan span
+mdspan version
 memory atomic
 memory compare
 memory concepts
diff --git a/libcxx/test/libcxx/transitive_includes/cxx20.csv b/libcxx/test/libcxx/transitive_includes/cxx20.csv
index 8f6e8dd646..63ea6c09b1 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx20.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx20.csv
@@ -537,6 +537,7 @@ mdspan concepts
 mdspan cstddef
 mdspan limits
 mdspan span
+mdspan version
 memory atomic
 memory compare
 memory concepts
diff --git a/libcxx/test/libcxx/transitive_includes/cxx23.csv b/libcxx/test/libcxx/transitive_includes/cxx23.csv
index d0d858056b..236dedd186 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx23.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx23.csv
@@ -386,6 +386,7 @@ mdspan concepts
 mdspan cstddef
 mdspan limits
 mdspan span
+mdspan version
 memory compare
 memory cstddef
 memory cstdint
diff --git a/libcxx/test/libcxx/transitive_includes/cxx26.csv b/libcxx/test/libcxx/transitive_includes/cxx26.csv
index d0d858056b..236dedd186 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx26.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx26.csv
@@ -386,6 +386,7 @@ mdspan concepts
 mdspan cstddef
 mdspan limits
 mdspan span
+mdspan version
 memory compare
 memory cstddef
 memory cstdint
diff --git a/libcxx/test/std/algorithms/alg.modifying.operations/alg.move/pstl.exception_handling.pass.cpp b/libcxx/test/std/algorithms/alg.modifying.operations/alg.move/pstl.exception_handling.pass.cpp
new file mode 100644
index 0000000000..ffe4f1866e
--- /dev/null
+++ b/libcxx/test/std/algorithms/alg.modifying.operations/alg.move/pstl.exception_handling.pass.cpp
@@ -0,0 +1,39 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14
+// UNSUPPORTED: no-exceptions
+// REQUIRES: has-unix-headers
+
+// UNSUPPORTED: libcpp-has-no-incomplete-pstl
+
+// check that std::move(ExecutionPolicy) terminates on user-thrown exceptions
+
+#include <algorithm>
+
+#include "check_assertion.h"
+#include "test_execution_policies.h"
+#include "test_iterators.h"
+
+int main(int, char**) {
+  test_execution_policies([](auto&& policy) {
+    EXPECT_STD_TERMINATE([&] {
+      try {
+        int a[] = {1, 2};
+        int b[] = {1, 2};
+        (void)std::move(policy,
+                        util::throw_on_move_iterator(std::begin(a), 1),
+                        util::throw_on_move_iterator(std::end(a), 1),
+                        util::throw_on_move_iterator(std::begin(b), 1));
+      } catch (const util::iterator_error&) {
+        assert(false);
+      }
+      std::terminate(); // make the test pass in case the algorithm didn't move the iterator
+    });
+  });
+}
diff --git a/libcxx/test/std/algorithms/alg.modifying.operations/alg.move/pstl.move.pass.cpp b/libcxx/test/std/algorithms/alg.modifying.operations/alg.move/pstl.move.pass.cpp
new file mode 100644
index 0000000000..e4cc5649ce
--- /dev/null
+++ b/libcxx/test/std/algorithms/alg.modifying.operations/alg.move/pstl.move.pass.cpp
@@ -0,0 +1,99 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14
+
+// UNSUPPORTED: libcpp-has-no-incomplete-pstl
+
+// <algorithm>
+
+// template<class ExecutionPolicy, class ForwardIterator1, class ForwardIterator2>
+//   ForwardIterator2 move(ExecutionPolicy&& policy,
+//                         ForwardIterator1 first, ForwardIterator1 last,
+//                         ForwardIterator2 result);
+
+#include <algorithm>
+#include <vector>
+
+#include "test_macros.h"
+#include "test_execution_policies.h"
+#include "test_iterators.h"
+
+EXECUTION_POLICY_SFINAE_TEST(move);
+
+static_assert(sfinae_test_move<int, int*, int*, int*>);
+static_assert(!sfinae_test_move<std::execution::parallel_policy, int*, int*, int*>);
+
+template <class Iter1, class Iter2>
+struct TestInt {
+  template <class Policy>
+  void operator()(Policy&& policy) {
+    // simple test
+    for (const int size : {0, 1, 2, 100, 350}) {
+      std::vector<int> a(size);
+      for (int i = 0; i != size; ++i)
+        a[i] = i + 1;
+
+      std::vector<int> out(std::size(a));
+      decltype(auto) ret =
+          std::move(policy, Iter1(std::data(a)), Iter1(std::data(a) + std::size(a)), Iter2(std::data(out)));
+      static_assert(std::is_same_v<decltype(ret), Iter2>);
+      assert(base(ret) == std::data(out) + std::size(out));
+      for (int i = 0; i != size; ++i)
+        assert(out[i] == i + 1);
+    }
+  }
+};
+
+struct MovedToTester {
+  bool moved_to   = false;
+  MovedToTester() = default;
+  MovedToTester(MovedToTester&&) {}
+  MovedToTester& operator=(MovedToTester&&) {
+    assert(!moved_to);
+    moved_to = true;
+    return *this;
+  }
+  ~MovedToTester() = default;
+};
+
+template <class Iter1, class Iter2>
+struct TestNonTrivial {
+  template <class Policy>
+  void operator()(Policy&& policy) {
+    // simple test
+    for (const int size : {0, 1, 2, 100, 350}) {
+      std::vector<MovedToTester> a(size);
+
+      std::vector<MovedToTester> out(std::size(a));
+      auto ret = std::move(policy, Iter1(std::data(a)), Iter1(std::data(a) + std::size(a)), Iter2(std::data(out)));
+      assert(base(ret) == std::data(out) + std::size(out));
+      assert(std::all_of(std::begin(out), std::end(out), [](MovedToTester& t) { return t.moved_to; }));
+      assert(std::none_of(std::begin(a), std::end(a), [](MovedToTester& t) { return t.moved_to; }));
+    }
+  }
+};
+
+int main(int, char**) {
+  types::for_each(types::forward_iterator_list<int*>{}, types::apply_type_identity{[](auto v) {
+                    using Iter = typename decltype(v)::type;
+                    types::for_each(
+                        types::forward_iterator_list<int*>{},
+                        TestIteratorWithPolicies< types::partial_instantiation<TestInt, Iter>::template apply>{});
+                  }});
+
+  types::for_each(
+      types::forward_iterator_list<MovedToTester*>{}, types::apply_type_identity{[](auto v) {
+        using Iter = typename decltype(v)::type;
+        types::for_each(
+            types::forward_iterator_list<MovedToTester*>{},
+            TestIteratorWithPolicies< types::partial_instantiation<TestNonTrivial, Iter>::template apply>{});
+      }});
+
+  return 0;
+}
diff --git a/libcxx/test/std/atomics/atomics.lockfree/isalwayslockfree.pass.cpp b/libcxx/test/std/atomics/atomics.lockfree/isalwayslockfree.pass.cpp
index b2d83f0a6f..6d6e6477bc 100644
--- a/libcxx/test/std/atomics/atomics.lockfree/isalwayslockfree.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.lockfree/isalwayslockfree.pass.cpp
@@ -21,7 +21,6 @@
 template <typename T>
 void checkAlwaysLockFree() {
   if (std::atomic<T>::is_always_lock_free) {
-    LIBCPP_ASSERT(sizeof(std::atomic<T>) == sizeof(T)); // technically not required, but libc++ does it that way
     assert(std::atomic<T>().is_lock_free());
   }
 }
@@ -79,6 +78,7 @@ void run()
     CHECK_ALWAYS_LOCK_FREE(struct Empty {});
     CHECK_ALWAYS_LOCK_FREE(struct OneInt { int i; });
     CHECK_ALWAYS_LOCK_FREE(struct IntArr2 { int i[2]; });
+    CHECK_ALWAYS_LOCK_FREE(struct FloatArr3 { float i[3]; });
     CHECK_ALWAYS_LOCK_FREE(struct LLIArr2 { long long int i[2]; });
     CHECK_ALWAYS_LOCK_FREE(struct LLIArr4 { long long int i[4]; });
     CHECK_ALWAYS_LOCK_FREE(struct LLIArr8 { long long int i[8]; });
diff --git a/libcxx/test/std/atomics/atomics.types.generic/address.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/address.pass.cpp
index b3aa1fc476..0926628a2e 100644
--- a/libcxx/test/std/atomics/atomics.types.generic/address.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.types.generic/address.pass.cpp
@@ -80,8 +80,14 @@ do_test()
     typedef typename std::remove_pointer<T>::type X;
     A obj(T(0));
     assert(obj == T(0));
-    bool b0 = obj.is_lock_free();
-    ((void)b0); // mark as unused
+    {
+        bool lockfree = obj.is_lock_free();
+        (void)lockfree;
+#if TEST_STD_VER >= 17
+        if (A::is_always_lock_free)
+            assert(lockfree);
+#endif
+    }
     obj.store(T(0));
     assert(obj == T(0));
     obj.store(T(1), std::memory_order_release);
diff --git a/libcxx/test/std/atomics/atomics.types.generic/bool.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/bool.pass.cpp
index 78234ae6d9..2609e5b56e 100644
--- a/libcxx/test/std/atomics/atomics.types.generic/bool.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.types.generic/bool.pass.cpp
@@ -61,8 +61,14 @@ int main(int, char**)
     {
         volatile std::atomic<bool> obj(true);
         assert(obj == true);
-        bool b0 = obj.is_lock_free();
-        (void)b0; // to placate scan-build
+        {
+            bool lockfree = obj.is_lock_free();
+            (void)lockfree;
+#if TEST_STD_VER >= 17
+            if (std::atomic<bool>::is_always_lock_free)
+                assert(lockfree);
+#endif
+        }
         obj.store(false);
         assert(obj == false);
         obj.store(true, std::memory_order_release);
@@ -112,8 +118,14 @@ int main(int, char**)
     {
         std::atomic<bool> obj(true);
         assert(obj == true);
-        bool b0 = obj.is_lock_free();
-        (void)b0; // to placate scan-build
+        {
+            bool lockfree = obj.is_lock_free();
+            (void)lockfree;
+#if TEST_STD_VER >= 17
+            if (std::atomic<bool>::is_always_lock_free)
+                assert(lockfree);
+#endif
+        }
         obj.store(false);
         assert(obj == false);
         obj.store(true, std::memory_order_release);
@@ -163,8 +175,14 @@ int main(int, char**)
     {
         std::atomic_bool obj(true);
         assert(obj == true);
-        bool b0 = obj.is_lock_free();
-        (void)b0; // to placate scan-build
+        {
+            bool lockfree = obj.is_lock_free();
+            (void)lockfree;
+#if TEST_STD_VER >= 17
+            if (std::atomic_bool::is_always_lock_free)
+                assert(lockfree);
+#endif
+        }
         obj.store(false);
         assert(obj == false);
         obj.store(true, std::memory_order_release);
diff --git a/libcxx/test/std/atomics/atomics.types.generic/integral.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/integral.pass.cpp
index 058db2dc3a..2695ff94da 100644
--- a/libcxx/test/std/atomics/atomics.types.generic/integral.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.types.generic/integral.pass.cpp
@@ -98,8 +98,14 @@ do_test()
 {
     A obj(T(0));
     assert(obj == T(0));
-    bool b0 = obj.is_lock_free();
-    ((void)b0); // mark as unused
+    {
+        bool lockfree = obj.is_lock_free();
+        (void)lockfree;
+#if TEST_STD_VER >= 17
+        if (A::is_always_lock_free)
+            assert(lockfree);
+#endif
+    }
     obj.store(T(0));
     assert(obj == T(0));
     obj.store(T(1), std::memory_order_release);
diff --git a/libcxx/test/std/atomics/atomics.types.operations/atomics.types.operations.req/atomic_is_lock_free.pass.cpp b/libcxx/test/std/atomics/atomics.types.operations/atomics.types.operations.req/atomic_is_lock_free.pass.cpp
index 8b838f62ab..39fa837f48 100644
--- a/libcxx/test/std/atomics/atomics.types.operations/atomics.types.operations.req/atomic_is_lock_free.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.types.operations/atomics.types.operations.req/atomic_is_lock_free.pass.cpp
@@ -27,8 +27,12 @@ struct TestFn {
   void operator()() const {
     typedef std::atomic<T> A;
     T t = T();
+
     A a(t);
     bool b1 = std::atomic_is_lock_free(static_cast<const A*>(&a));
+    if (A::is_always_lock_free)
+      assert(b1);
+
     volatile A va(t);
     bool b2 = std::atomic_is_lock_free(static_cast<const volatile A*>(&va));
     assert(b1 == b2);
diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/CustomTestLayouts.h b/libcxx/test/std/containers/views/mdspan/CustomTestLayouts.h
similarity index 87%
rename from libcxx/test/std/containers/views/mdspan/mdspan/CustomTestLayouts.h
rename to libcxx/test/std/containers/views/mdspan/CustomTestLayouts.h
index e1d6e80afb..302eb2f5ed 100644
--- a/libcxx/test/std/containers/views/mdspan/mdspan/CustomTestLayouts.h
+++ b/libcxx/test/std/containers/views/mdspan/CustomTestLayouts.h
@@ -14,8 +14,8 @@
 //
 //===---------------------------------------------------------------------===//
 
-#ifndef TEST_STD_CONTAINERS_VIEWS_MDSPAN_MDSPAN_CUSTOM_TEST_LAYOUTS_H
-#define TEST_STD_CONTAINERS_VIEWS_MDSPAN_MDSPAN_CUSTOM_TEST_LAYOUTS_H
+#ifndef TEST_STD_CONTAINERS_VIEWS_MDSPAN_CUSTOM_TEST_LAYOUTS_H
+#define TEST_STD_CONTAINERS_VIEWS_MDSPAN_CUSTOM_TEST_LAYOUTS_H
 
 #include <algorithm>
 #include <array>
@@ -205,9 +205,9 @@ constexpr auto construct_mapping(layout_wrapping_integral<Wraps>, Extents exts)
   return typename layout_wrapping_integral<Wraps>::template mapping<Extents>(exts, not_extents_constructible_tag{});
 }
 
-
 // This layout does not check convertibility of extents for its conversion ctor
 // Allows triggering mdspan's ctor static assertion on convertibility of extents
+// It also allows for negative strides and offsets via runtime arguments
 class always_convertible_layout {
 public:
   template <class Extents>
@@ -239,8 +239,10 @@ class always_convertible_layout::mapping {
 
 public:
   constexpr mapping() noexcept = delete;
-  constexpr mapping(const mapping& other) noexcept : extents_(other.extents()){};
-  constexpr mapping(const extents_type& ext) noexcept : extents_(ext){};
+  constexpr mapping(const mapping& other) noexcept
+      : extents_(other.extents_), offset_(other.offset_), scaling_(other.scaling_){};
+  constexpr mapping(const extents_type& ext, index_type offset = 0, index_type scaling = 1) noexcept
+      : extents_(ext), offset_(offset), scaling_(scaling){};
 
   template <class OtherExtents>
   constexpr mapping(const mapping<OtherExtents>& other) noexcept {
@@ -256,10 +258,14 @@ class always_convertible_layout::mapping {
     } else {
       extents_ = extents_type();
     }
+    offset_  = other.offset_;
+    scaling_ = other.scaling_;
   }
 
   constexpr mapping& operator=(const mapping& other) noexcept {
     extents_ = other.extents_;
+    offset_  = other.offset_;
+    scaling_ = other.scaling_;
     return *this;
   };
 
@@ -269,7 +275,7 @@ class always_convertible_layout::mapping {
     index_type size = 1;
     for (size_t r = 0; r < extents_type::rank(); r++)
       size *= extents_.extent(r);
-    return size;
+    return std::max(size * scaling_ + offset_, offset_);
   }
 
   template <std::integral... Indices>
@@ -277,16 +283,18 @@ class always_convertible_layout::mapping {
              (std::is_nothrow_constructible_v<index_type, Indices> && ...))
   constexpr index_type operator()(Indices... idx) const noexcept {
     std::array<index_type, extents_type::rank()> idx_a{static_cast<index_type>(static_cast<index_type>(idx))...};
-    return [&]<size_t... Pos>(std::index_sequence<Pos...>) {
-      index_type res = 0;
-      ((res = idx_a[extents_type::rank() - 1 - Pos] + extents_.extent(extents_type::rank() - 1 - Pos) * res), ...);
-      return res;
-    }(std::make_index_sequence<sizeof...(Indices)>());
+    return offset_ +
+           scaling_ * ([&]<size_t... Pos>(std::index_sequence<Pos...>) {
+             index_type res = 0;
+             ((res = idx_a[extents_type::rank() - 1 - Pos] + extents_.extent(extents_type::rank() - 1 - Pos) * res),
+              ...);
+             return res;
+           }(std::make_index_sequence<sizeof...(Indices)>()));
   }
 
-  static constexpr bool is_always_unique() noexcept { return false; }
+  static constexpr bool is_always_unique() noexcept { return true; }
   static constexpr bool is_always_exhaustive() noexcept { return true; }
-  static constexpr bool is_always_strided() noexcept { return false; }
+  static constexpr bool is_always_strided() noexcept { return true; }
 
   static constexpr bool is_unique() noexcept { return true; }
   static constexpr bool is_exhaustive() noexcept { return true; }
@@ -296,15 +304,15 @@ class always_convertible_layout::mapping {
     requires(extents_type::rank() > 0)
   {
     index_type s = 1;
-    for (rank_type i = extents_type::rank() - 1; i > r; i--)
+    for (rank_type i = 0; i < r; i++)
       s *= extents_.extent(i);
-    return s;
+    return s * scaling_;
   }
 
   template <class OtherExtents>
     requires(OtherExtents::rank() == extents_type::rank())
   friend constexpr bool operator==(const mapping& lhs, const mapping<OtherExtents>& rhs) noexcept {
-    return lhs.extents() == rhs.extents();
+    return lhs.extents() == rhs.extents() && lhs.offset_ == rhs.offset && lhs.scaling_ == rhs.scaling_;
   }
 
   friend constexpr void swap(mapping& x, mapping& y) noexcept {
@@ -320,6 +328,11 @@ class always_convertible_layout::mapping {
   }
 
 private:
+  template <class>
+  friend class mapping;
+
   extents_type extents_{};
+  index_type offset_{};
+  index_type scaling_{};
 };
-#endif                     // TEST_STD_CONTAINERS_VIEWS_MDSPAN_MDSPAN_CUSTOM_TEST_LAYOUTS_H
+#endif // TEST_STD_CONTAINERS_VIEWS_MDSPAN_CUSTOM_TEST_LAYOUTS_H
diff --git a/libcxx/test/std/containers/views/mdspan/extents/ctad.pass.cpp b/libcxx/test/std/containers/views/mdspan/extents/ctad.pass.cpp
index 2a3da30bb9..3fc7c707f0 100644
--- a/libcxx/test/std/containers/views/mdspan/extents/ctad.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/extents/ctad.pass.cpp
@@ -21,6 +21,13 @@
 #include "../ConvertibleToIntegral.h"
 #include "test_macros.h"
 
+struct NoDefaultCtorIndex {
+  size_t value;
+  constexpr NoDefaultCtorIndex() = delete;
+  constexpr NoDefaultCtorIndex(size_t val) : value(val){};
+  constexpr operator size_t() const noexcept { return value; }
+};
+
 template <class E, class Expected>
 constexpr void test(E e, Expected expected) {
   ASSERT_SAME_TYPE(E, Expected);
@@ -35,6 +42,7 @@ constexpr bool test() {
   test(std::extents(1, 2u), std::extents<std::size_t, D, D>(1, 2u));
   test(std::extents(1, 2u, 3, 4, 5, 6, 7, 8, 9),
        std::extents<std::size_t, D, D, D, D, D, D, D, D, D>(1, 2u, 3, 4, 5, 6, 7, 8, 9));
+  test(std::extents(NoDefaultCtorIndex{1}, NoDefaultCtorIndex{2}), std::extents<std::size_t, D, D>(1, 2));
   return true;
 }
 
diff --git a/libcxx/test/std/containers/views/mdspan/layout_left/ctor.layout_stride.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_left/ctor.layout_stride.pass.cpp
new file mode 100644
index 0000000000..e85270ad02
--- /dev/null
+++ b/libcxx/test/std/containers/views/mdspan/layout_left/ctor.layout_stride.pass.cpp
@@ -0,0 +1,114 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <mdspan>
+
+// template<class OtherExtents>
+//   constexpr explicit(extents_type::rank() > 0)
+//     mapping(const layout_stride::mapping<OtherExtents>& other);
+//
+// Constraints: is_constructible_v<extents_type, OtherExtents> is true.
+//
+// Preconditions:
+//   - If extents_type::rank() > 0 is true, then for all r in the range [0, extents_type::rank()),
+//     other.stride(r) equals other.extents().fwd-prod-of-extents(r), and
+//   - other.required_span_size() is representable as a value of type index_type ([basic.fundamental]).
+//
+// Effects: Direct-non-list-initializes extents_ with other.extents().
+
+#include <mdspan>
+#include <type_traits>
+#include <cassert>
+#include <limits>
+
+#include "test_macros.h"
+
+template <bool implicit, class ToE, class FromE>
+constexpr void test_conversion(FromE src_exts) {
+  using To   = std::layout_left::mapping<ToE>;
+  using From = std::layout_stride::mapping<FromE>;
+  std::array<typename FromE::index_type, FromE::rank()> strides;
+  if constexpr (FromE::rank() > 0) {
+    strides[0] = 1;
+    for (size_t r = 1; r < FromE::rank(); r++)
+      strides[r] = src_exts.extent(r - 1) * strides[r - 1];
+  }
+  From src(src_exts, strides);
+
+  ASSERT_NOEXCEPT(To(src));
+  To dest(src);
+  assert(dest == src);
+
+  if constexpr (implicit) {
+    To dest_implicit = src;
+    assert(dest_implicit == src);
+  } else {
+    assert((!std::is_convertible_v<From, To>));
+  }
+}
+
+template <class T1, class T2>
+constexpr void test_conversion() {
+  constexpr size_t D = std::dynamic_extent;
+
+  // clang-format off
+  test_conversion<true,  std::extents<T1>>(std::extents<T2>());
+  test_conversion<false, std::extents<T1, D>>(std::extents<T2, D>(5));
+  test_conversion<false, std::extents<T1, 5>>(std::extents<T2, D>(5));
+  test_conversion<false, std::extents<T1, 5>>(std::extents<T2, 5>());
+  test_conversion<false, std::extents<T1, 5, D>>(std::extents<T2, D, D>(5, 5));
+  test_conversion<false, std::extents<T1, D, D>>(std::extents<T2, D, D>(5, 5));
+  test_conversion<false, std::extents<T1, D, D>>(std::extents<T2, D, 7>(5));
+  test_conversion<false, std::extents<T1, 5, 7>>(std::extents<T2, 5, 7>());
+  test_conversion<false, std::extents<T1, 5, D, 8, D, D>>(std::extents<T2, D, D, 8, 9, 1>(5, 7));
+  test_conversion<false, std::extents<T1, D, D, D, D, D>>(
+                         std::extents<T2, D, D, D, D, D>(5, 7, 8, 9, 1));
+  test_conversion<false, std::extents<T1, D, D, 8, 9, D>>(std::extents<T2, D, 7, 8, 9, 1>(5));
+  test_conversion<false, std::extents<T1, 5, 7, 8, 9, 1>>(std::extents<T2, 5, 7, 8, 9, 1>());
+  // clang-format on
+}
+
+template <class IdxT, size_t... Extents>
+using ll_mapping_t = typename std::layout_left::template mapping<std::extents<IdxT, Extents...>>;
+template <class IdxT, size_t... Extents>
+using ls_mapping_t = typename std::layout_stride::template mapping<std::extents<IdxT, Extents...>>;
+
+constexpr void test_rank_mismatch() {
+  constexpr size_t D = std::dynamic_extent;
+
+  static_assert(!std::is_constructible_v<ll_mapping_t<int, D>, ls_mapping_t<int>>);
+  static_assert(!std::is_constructible_v<ll_mapping_t<int>, ls_mapping_t<int, D, D>>);
+  static_assert(!std::is_constructible_v<ll_mapping_t<int, D>, ls_mapping_t<int, D, D>>);
+  static_assert(!std::is_constructible_v<ll_mapping_t<int, D, D, D>, ls_mapping_t<int, D, D>>);
+}
+
+constexpr void test_static_extent_mismatch() {
+  constexpr size_t D = std::dynamic_extent;
+
+  static_assert(!std::is_constructible_v<ll_mapping_t<int, D, 5>, ls_mapping_t<int, D, 4>>);
+  static_assert(!std::is_constructible_v<ll_mapping_t<int, 5>, ls_mapping_t<int, 4>>);
+  static_assert(!std::is_constructible_v<ll_mapping_t<int, 5, D>, ls_mapping_t<int, 4, D>>);
+}
+
+constexpr bool test() {
+  test_conversion<int, int>();
+  test_conversion<int, size_t>();
+  test_conversion<size_t, int>();
+  test_conversion<size_t, long>();
+  test_rank_mismatch();
+  test_static_extent_mismatch();
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test());
+  return 0;
+}
diff --git a/libcxx/test/std/containers/views/mdspan/layout_right/ctor.layout_stride.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_right/ctor.layout_stride.pass.cpp
new file mode 100644
index 0000000000..9d9677d540
--- /dev/null
+++ b/libcxx/test/std/containers/views/mdspan/layout_right/ctor.layout_stride.pass.cpp
@@ -0,0 +1,114 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <mdspan>
+
+// template<class OtherExtents>
+//   constexpr explicit(extents_type::rank() > 0)
+//     mapping(const layout_stride::mapping<OtherExtents>& other);
+//
+// Constraints: is_constructible_v<extents_type, OtherExtents> is true.
+//
+// Preconditions:
+//   - If extents_type::rank() > 0 is true, then for all r in the range [0, extents_type::rank()),
+//     other.stride(r) equals other.extents().fwd-prod-of-extents(r), and
+//   - other.required_span_size() is representable as a value of type index_type ([basic.fundamental]).
+//
+// Effects: Direct-non-list-initializes extents_ with other.extents().
+
+#include <mdspan>
+#include <type_traits>
+#include <cassert>
+#include <limits>
+
+#include "test_macros.h"
+
+template <bool implicit, class ToE, class FromE>
+constexpr void test_conversion(FromE src_exts) {
+  using To   = std::layout_left::mapping<ToE>;
+  using From = std::layout_stride::mapping<FromE>;
+  std::array<typename FromE::index_type, FromE::rank()> strides;
+  if constexpr (FromE::rank() > 0) {
+    strides[0] = 1;
+    for (size_t r = 1; r < FromE::rank(); r++)
+      strides[r] = src_exts.extent(r - 1) * strides[r - 1];
+  }
+  From src(src_exts, strides);
+
+  ASSERT_NOEXCEPT(To(src));
+  To dest(src);
+  assert(dest == src);
+
+  if constexpr (implicit) {
+    To dest_implicit = src;
+    assert(dest_implicit == src);
+  } else {
+    assert((!std::is_convertible_v<From, To>));
+  }
+}
+
+template <class T1, class T2>
+constexpr void test_conversion() {
+  constexpr size_t D = std::dynamic_extent;
+
+  // clang-format off
+  test_conversion<true,  std::extents<T1>>(std::extents<T2>());
+  test_conversion<false, std::extents<T1, D>>(std::extents<T2, D>(5));
+  test_conversion<false, std::extents<T1, 5>>(std::extents<T2, D>(5));
+  test_conversion<false, std::extents<T1, 5>>(std::extents<T2, 5>());
+  test_conversion<false, std::extents<T1, 5, D>>(std::extents<T2, D, D>(5, 5));
+  test_conversion<false, std::extents<T1, D, D>>(std::extents<T2, D, D>(5, 5));
+  test_conversion<false, std::extents<T1, D, D>>(std::extents<T2, D, 7>(5));
+  test_conversion<false, std::extents<T1, 5, 7>>(std::extents<T2, 5, 7>());
+  test_conversion<false, std::extents<T1, 5, D, 8, D, D>>(std::extents<T2, D, D, 8, 9, 1>(5, 7));
+  test_conversion<false, std::extents<T1, D, D, D, D, D>>(
+                         std::extents<T2, D, D, D, D, D>(5, 7, 8, 9, 1));
+  test_conversion<false, std::extents<T1, D, D, 8, 9, D>>(std::extents<T2, D, 7, 8, 9, 1>(5));
+  test_conversion<false, std::extents<T1, 5, 7, 8, 9, 1>>(std::extents<T2, 5, 7, 8, 9, 1>());
+  // clang-format on
+}
+
+template <class IdxT, size_t... Extents>
+using lr_mapping_t = typename std::layout_right::template mapping<std::extents<IdxT, Extents...>>;
+template <class IdxT, size_t... Extents>
+using ls_mapping_t = typename std::layout_stride::template mapping<std::extents<IdxT, Extents...>>;
+
+constexpr void test_rank_mismatch() {
+  constexpr size_t D = std::dynamic_extent;
+
+  static_assert(!std::is_constructible_v<lr_mapping_t<int, D>, ls_mapping_t<int>>);
+  static_assert(!std::is_constructible_v<lr_mapping_t<int>, ls_mapping_t<int, D, D>>);
+  static_assert(!std::is_constructible_v<lr_mapping_t<int, D>, ls_mapping_t<int, D, D>>);
+  static_assert(!std::is_constructible_v<lr_mapping_t<int, D, D, D>, ls_mapping_t<int, D, D>>);
+}
+
+constexpr void test_static_extent_mismatch() {
+  constexpr size_t D = std::dynamic_extent;
+
+  static_assert(!std::is_constructible_v<lr_mapping_t<int, D, 5>, ls_mapping_t<int, D, 4>>);
+  static_assert(!std::is_constructible_v<lr_mapping_t<int, 5>, ls_mapping_t<int, 4>>);
+  static_assert(!std::is_constructible_v<lr_mapping_t<int, 5, D>, ls_mapping_t<int, 4, D>>);
+}
+
+constexpr bool test() {
+  test_conversion<int, int>();
+  test_conversion<int, size_t>();
+  test_conversion<size_t, int>();
+  test_conversion<size_t, long>();
+  test_rank_mismatch();
+  test_static_extent_mismatch();
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test());
+  return 0;
+}
diff --git a/libcxx/test/std/containers/views/mdspan/layout_stride/comparison.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_stride/comparison.pass.cpp
new file mode 100644
index 0000000000..e624e49df5
--- /dev/null
+++ b/libcxx/test/std/containers/views/mdspan/layout_stride/comparison.pass.cpp
@@ -0,0 +1,198 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <mdspan>
+
+// template<class OtherMapping>
+//   friend constexpr bool operator==(const mapping& x, const OtherMapping& y) noexcept;
+//
+// Constraints:
+//   - layout-mapping-alike<OtherMapping> is satisfied.
+//   - rank_ == OtherMapping::extents_type::rank() is true.
+//   - OtherMapping::is_always_strided() is true.
+//
+// Preconditions: OtherMapping meets the layout mapping requirements ([mdspan.layout.policy.reqmts]).
+//
+// Returns: true if x.extents() == y.extents() is true, OFFSET(y) == 0 is true, and each of x.stride(r) == y.stride(r) is true for r in the range [0, x.extents().rank()). Otherwise, false.
+
+#include <mdspan>
+#include <type_traits>
+#include <concepts>
+#include <cassert>
+
+#include "test_macros.h"
+
+#include "../CustomTestLayouts.h"
+
+template <class E1, class E2>
+concept layout_mapping_comparable = requires(
+    E1 e1,
+    E2 e2,
+    std::array<typename E1::index_type, E1::rank()> s1,
+    std::array<typename E1::index_type, E1::rank()> s2) {
+  std::layout_stride::mapping<E1>(e1, s1) == std::layout_stride::mapping<E2>(e2, s2);
+};
+
+template <class T1, class T2>
+constexpr void test_comparison_different_rank() {
+  constexpr size_t D = std::dynamic_extent;
+
+  // sanity check same rank
+  static_assert(layout_mapping_comparable<std::extents<T1, D>, std::extents<T2, D>>);
+  static_assert(layout_mapping_comparable<std::extents<T1, 5>, std::extents<T2, D>>);
+  static_assert(layout_mapping_comparable<std::extents<T1, D>, std::extents<T2, 5>>);
+  static_assert(layout_mapping_comparable<std::extents<T1, 5>, std::extents<T2, 5>>);
+
+  // not equality comparable when rank is not the same
+  static_assert(!layout_mapping_comparable<std::extents<T1>, std::extents<T2, D>>);
+  static_assert(!layout_mapping_comparable<std::extents<T1>, std::extents<T2, 1>>);
+  static_assert(!layout_mapping_comparable<std::extents<T1, D>, std::extents<T2>>);
+  static_assert(!layout_mapping_comparable<std::extents<T1, 1>, std::extents<T2>>);
+  static_assert(!layout_mapping_comparable<std::extents<T1, D>, std::extents<T2, D, D>>);
+  static_assert(!layout_mapping_comparable<std::extents<T1, 5>, std::extents<T2, 5, D>>);
+  static_assert(!layout_mapping_comparable<std::extents<T1, 5>, std::extents<T2, 5, 1>>);
+  static_assert(!layout_mapping_comparable<std::extents<T1, D, D>, std::extents<T2, D>>);
+  static_assert(!layout_mapping_comparable<std::extents<T1, 5, D>, std::extents<T2, 5>>);
+  static_assert(!layout_mapping_comparable<std::extents<T1, 5, 1>, std::extents<T2, 5>>);
+}
+
+template <class To, class From>
+constexpr void test_comparison(
+    bool equal,
+    To dest_exts,
+    From src_exts,
+    std::array<int, To::rank()> dest_strides,
+    std::array<int, From::rank()> src_strides) {
+  std::layout_stride::mapping<To> dest(dest_exts, dest_strides);
+  std::layout_stride::mapping<From> src(src_exts, src_strides);
+  ASSERT_NOEXCEPT(dest == src);
+  assert((dest == src) == equal);
+  assert((dest != src) == !equal);
+}
+
+template <class T1, class T2>
+constexpr void test_comparison_same_rank() {
+  constexpr size_t D = std::dynamic_extent;
+
+  test_comparison(true, std::extents<T1>(), std::extents<T2>(), std::array<int, 0>{}, std::array<int, 0>{});
+
+  test_comparison(true, std::extents<T1, D>(5), std::extents<T2, D>(5), std::array<int, 1>{1}, std::array<int, 1>{1});
+  test_comparison(true, std::extents<T1, D>(0), std::extents<T2, D>(0), std::array<int, 1>{1}, std::array<int, 1>{1});
+  test_comparison(true, std::extents<T1, 5>(), std::extents<T2, D>(5), std::array<int, 1>{3}, std::array<int, 1>{3});
+  test_comparison(true, std::extents<T1, D>(5), std::extents<T2, 5>(), std::array<int, 1>{1}, std::array<int, 1>{1});
+  test_comparison(true, std::extents<T1, 5>(), std::extents< T2, 5>(), std::array<int, 1>{1}, std::array<int, 1>{1});
+  test_comparison(false, std::extents<T1, 5>(), std::extents<T2, D>(5), std::array<int, 1>{2}, std::array<int, 1>{1});
+  test_comparison(false, std::extents<T1, D>(5), std::extents<T2, D>(5), std::array<int, 1>{2}, std::array<int, 1>{1});
+  test_comparison(false, std::extents<T1, D>(5), std::extents<T2, D>(7), std::array<int, 1>{1}, std::array<int, 1>{1});
+  test_comparison(false, std::extents<T1, 5>(), std::extents<T2, D>(7), std::array<int, 1>{1}, std::array<int, 1>{1});
+  test_comparison(false, std::extents<T1, D>(5), std::extents<T2, 7>(), std::array<int, 1>{1}, std::array<int, 1>{1});
+  test_comparison(false, std::extents<T1, 5>(), std::extents<T2, 7>(), std::array<int, 1>{1}, std::array<int, 1>{1});
+
+  test_comparison(
+      true,
+      std::extents<T1, D, D, D, D, D>(5, 6, 7, 8, 9),
+      std::extents<T2, D, D, D, D, D>(5, 6, 7, 8, 9),
+      std::array<int, 5>{2, 20, 200, 2000, 20000},
+      std::array<int, 5>{2, 20, 200, 2000, 20000});
+  test_comparison(
+      true,
+      std::extents<T1, D, 6, D, 8, D>(5, 7, 9),
+      std::extents<T2, 5, D, D, 8, 9>(6, 7),
+      std::array<int, 5>{2, 20, 200, 2000, 20000},
+      std::array<int, 5>{2, 20, 200, 2000, 20000});
+  test_comparison(
+      true,
+      std::extents<T1, 5, 6, 7, 8, 9>(5, 6, 7, 8, 9),
+      std::extents<T2, 5, 6, 7, 8, 9>(),
+      std::array<int, 5>{2, 20, 200, 2000, 20000},
+      std::array<int, 5>{2, 20, 200, 2000, 20000});
+  test_comparison(
+      false,
+      std::extents<T1, 5, 6, 7, 8, 9>(5, 6, 7, 8, 9),
+      std::extents<T2, 5, 6, 7, 8, 9>(),
+      std::array<int, 5>{2, 20, 200, 20000, 2000},
+      std::array<int, 5>{2, 20, 200, 2000, 20000});
+  test_comparison(
+      false,
+      std::extents<T1, D, D, D, D, D>(5, 6, 7, 8, 9),
+      std::extents<T2, D, D, D, D, D>(5, 6, 3, 8, 9),
+      std::array<int, 5>{2, 20, 200, 2000, 20000},
+      std::array<int, 5>{2, 20, 200, 2000, 20000});
+  test_comparison(
+      false,
+      std::extents<T1, D, 6, D, 8, D>(5, 7, 9),
+      std::extents<T2, 5, D, D, 3, 9>(6, 7),
+      std::array<int, 5>{2, 20, 200, 2000, 20000},
+      std::array<int, 5>{2, 20, 200, 2000, 20000});
+  test_comparison(
+      false,
+      std::extents<T1, 5, 6, 7, 8, 9>(5, 6, 7, 8, 9),
+      std::extents<T2, 5, 6, 7, 3, 9>(),
+      std::array<int, 5>{2, 20, 200, 2000, 20000},
+      std::array<int, 5>{2, 20, 200, 2000, 20000});
+}
+
+template <class OtherLayout, class E1, class E2, class... OtherArgs>
+constexpr void test_comparison_with(
+    bool expect_equal, E1 e1, std::array<typename E1::index_type, E1::rank()> strides, E2 e2, OtherArgs... other_args) {
+  typename std::layout_stride::template mapping<E1> map(e1, strides);
+  typename OtherLayout::template mapping<E2> other_map(e2, other_args...);
+
+  assert((map == other_map) == expect_equal);
+}
+
+template <class OtherLayout>
+constexpr void test_comparison_with() {
+  constexpr size_t D = std::dynamic_extent;
+  bool is_left_based =
+      std::is_same_v<OtherLayout, std::layout_left> || std::is_same_v<OtherLayout, always_convertible_layout>;
+  test_comparison_with<OtherLayout>(true, std::extents<int>(), std::array<int, 0>{}, std::extents<unsigned>());
+  test_comparison_with<OtherLayout>(true, std::extents<int, 5>(), std::array<int, 1>{1}, std::extents<unsigned, 5>());
+  test_comparison_with<OtherLayout>(true, std::extents<int, D>(5), std::array<int, 1>{1}, std::extents<unsigned, 5>());
+  test_comparison_with<OtherLayout>(false, std::extents<int, D>(5), std::array<int, 1>{2}, std::extents<unsigned, 5>());
+  test_comparison_with<OtherLayout>(
+      is_left_based, std::extents<int, D, D>(5, 7), std::array<int, 2>{1, 5}, std::extents<unsigned, D, D>(5, 7));
+  test_comparison_with<OtherLayout>(
+      !is_left_based, std::extents<int, D, D>(5, 7), std::array<int, 2>{7, 1}, std::extents<unsigned, D, D>(5, 7));
+  test_comparison_with<OtherLayout>(
+      false, std::extents<int, D, D>(5, 7), std::array<int, 2>{8, 1}, std::extents<unsigned, D, D>(5, 7));
+
+  if constexpr (std::is_same_v<OtherLayout, always_convertible_layout>) {
+    // test layout with strides not equal to product of extents
+    test_comparison_with<OtherLayout>(
+        true, std::extents<int, D, D>(5, 7), std::array<int, 2>{2, 10}, std::extents<unsigned, D, D>(5, 7), 0, 2);
+    // make sure that offset != 0 results in false
+    test_comparison_with<OtherLayout>(
+        false, std::extents<int, D, D>(5, 7), std::array<int, 2>{2, 10}, std::extents<unsigned, D, D>(5, 7), 1, 2);
+  }
+}
+
+template <class T1, class T2>
+constexpr void test_comparison_index_type() {
+  test_comparison_same_rank<T1, T2>();
+  test_comparison_different_rank<T1, T2>();
+  test_comparison_with<std::layout_right>();
+  test_comparison_with<std::layout_left>();
+  test_comparison_with<always_convertible_layout>();
+}
+
+constexpr bool test() {
+  test_comparison_index_type<int, int>();
+  test_comparison_index_type<int, size_t>();
+  test_comparison_index_type<size_t, int>();
+  test_comparison_index_type<size_t, long>();
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test());
+  return 0;
+}
diff --git a/libcxx/test/std/containers/views/mdspan/layout_stride/ctor.default.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_stride/ctor.default.pass.cpp
new file mode 100644
index 0000000000..2fa8866058
--- /dev/null
+++ b/libcxx/test/std/containers/views/mdspan/layout_stride/ctor.default.pass.cpp
@@ -0,0 +1,73 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <mdspan>
+
+// Test default construction:
+//
+// constexpr mapping() noexcept;
+//
+//
+// Preconditions: layout_right::mapping<extents_type>().required_span_size() is representable as a value of type index_type ([basic.fundamental]).
+//
+// Effects: Direct-non-list-initializes extents_ with extents_type(), and for all d in the range [0, rank_),
+//          direct-non-list-initializes strides_[d] with layout_right::mapping<extents_type>().stride(d).
+
+#include <mdspan>
+#include <cassert>
+#include <cstdint>
+
+#include "test_macros.h"
+
+template <class E>
+constexpr void test_construction() {
+  using M = std::layout_stride::mapping<E>;
+  ASSERT_NOEXCEPT(M{});
+  M m;
+  E e;
+
+  // check correct extents are returned
+  ASSERT_NOEXCEPT(m.extents());
+  assert(m.extents() == e);
+
+  // check required_span_size()
+  typename E::index_type expected_size = 1;
+  for (typename E::rank_type r = 0; r < E::rank(); r++)
+    expected_size *= e.extent(r);
+  assert(m.required_span_size() == expected_size);
+
+  // check strides: node stride function is constrained on rank>0, e.extent(r) is not
+  auto strides = m.strides();
+  ASSERT_NOEXCEPT(m.strides());
+  if constexpr (E::rank() > 0) {
+    std::layout_right::mapping<E> m_right;
+    for (typename E::rank_type r = 0; r < E::rank(); r++) {
+      assert(m.stride(r) == m_right.stride(r));
+      assert(strides[r] == m.stride(r));
+    }
+  }
+}
+
+constexpr bool test() {
+  constexpr size_t D = std::dynamic_extent;
+  test_construction<std::extents<int>>();
+  test_construction<std::extents<unsigned, D>>();
+  test_construction<std::extents<unsigned, 7>>();
+  test_construction<std::extents<unsigned, 0>>();
+  test_construction<std::extents<unsigned, 7, 8>>();
+  test_construction<std::extents<int64_t, D, 8, D, D>>();
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test());
+  return 0;
+}
diff --git a/libcxx/test/std/containers/views/mdspan/layout_stride/ctor.extents_array.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_stride/ctor.extents_array.pass.cpp
new file mode 100644
index 0000000000..f9157f8c6e
--- /dev/null
+++ b/libcxx/test/std/containers/views/mdspan/layout_stride/ctor.extents_array.pass.cpp
@@ -0,0 +1,138 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <mdspan>
+
+// template<class OtherIndexType>
+//  constexpr mapping(const extents_type& e, array<OtherIndexType, rank_> s) noexcept;
+//
+// Constraints:
+//    - is_convertible_v<const OtherIndexType&, index_type> is true, and
+//    - is_nothrow_constructible_v<index_type, const OtherIndexType&> is true.
+//
+// Preconditions:
+//    - s[i] > 0 is true for all i in the range [0, rank_).
+//    - REQUIRED-SPAN-SIZE(e, s) is representable as a value of type index_type ([basic.fundamental]).
+//    - If rank_ is greater than 0, then there exists a permutation P of the integers in the range [0, rank_),
+//      such that s[pi] >= s[pi_1] * e.extent(pi_1) is true for all i in the range [1, rank_), where pi is the ith element of P.
+//     Note 1: For layout_stride, this condition is necessary and sufficient for is_unique() to be true.
+//
+// Effects: Direct-non-list-initializes extents_ with e, and for all d in the range [0, rank_),
+//         direct-non-list-initializes strides_[d] with as_const(s[d]).
+
+#include <mdspan>
+#include <cassert>
+#include <cstdint>
+
+#include "test_macros.h"
+#include "../ConvertibleToIntegral.h"
+
+template <class E, class S>
+constexpr void test_construction(E e, S s) {
+  using M = std::layout_stride::mapping<E>;
+  ASSERT_NOEXCEPT(M{e, s});
+  M m(e, s);
+
+  // check correct extents are returned
+  ASSERT_NOEXCEPT(m.extents());
+  assert(m.extents() == e);
+
+  // check required_span_size()
+  typename E::index_type expected_size = 1;
+  for (typename E::rank_type r = 0; r < E::rank(); r++) {
+    if (e.extent(r) == 0) {
+      expected_size = 0;
+      break;
+    }
+    expected_size += (e.extent(r) - 1) * static_cast<typename E::index_type>(s[r]);
+  }
+  assert(m.required_span_size() == expected_size);
+
+  // check strides: node stride function is constrained on rank>0, e.extent(r) is not
+  auto strides = m.strides();
+  ASSERT_NOEXCEPT(m.strides());
+  if constexpr (E::rank() > 0) {
+    for (typename E::rank_type r = 0; r < E::rank(); r++) {
+      assert(m.stride(r) == static_cast<typename E::index_type>(s[r]));
+      assert(strides[r] == m.stride(r));
+    }
+  }
+}
+
+constexpr bool test() {
+  constexpr size_t D = std::dynamic_extent;
+  {
+    std::array<int, 0> s{};
+    test_construction(std::extents<int>(), s);
+  }
+  {
+    std::array<int, 1> s{1};
+    test_construction(std::extents<unsigned, D>(7), s);
+  }
+  {
+    std::array<int, 1> s{1};
+    test_construction(std::extents<unsigned, D>(0), s);
+  }
+  {
+    std::array<int, 1> s{2};
+    test_construction(std::extents<unsigned, 7>(), s);
+  }
+  {
+    std::array<IntType, 1> s{1};
+    test_construction(std::extents<int, D>(7), s);
+  }
+  {
+    std::array<int, 2> s{3, 30};
+    test_construction(std::extents<unsigned, 7, 8>(), s);
+  }
+  {
+    std::array<int, 4> s{20, 2, 200, 2000};
+    test_construction(std::extents<int64_t, D, 8, D, D>(7, 9, 10), s);
+    test_construction(std::extents<int64_t, D, 8, D, D>(0, 9, 10), s);
+    test_construction(std::extents<int64_t, D, 8, D, D>(0, 8, 0), s);
+  }
+  {
+    std::array<int, 4> s{200, 20, 20, 2000};
+    test_construction(std::extents<int64_t, D, D, D, D>(7, 0, 8, 9), s);
+    test_construction(std::extents<int64_t, D, D, D, D>(7, 8, 0, 9), s);
+    test_construction(std::extents<int64_t, D, D, D, D>(7, 1, 8, 9), s);
+    test_construction(std::extents<int64_t, D, D, D, D>(7, 8, 1, 9), s);
+    test_construction(std::extents<int64_t, D, D, D, D>(7, 1, 1, 9), s);
+    test_construction(std::extents<int64_t, D, D, D, D>(7, 0, 0, 9), s);
+    test_construction(std::extents<int64_t, D, D, D, D>(7, 1, 1, 9), s);
+    test_construction(std::extents<int64_t, D, D, D, D>(7, 1, 0, 9), s);
+    test_construction(std::extents<int64_t, D, D, D, D>(7, 0, 1, 9), s);
+  }
+
+  {
+    using mapping_t = std::layout_stride::mapping<std::dextents<unsigned, 2>>;
+    // wrong strides size
+    static_assert(!std::is_constructible_v<mapping_t, std::dextents<int, 2>, std::array<int, 3>>);
+    static_assert(!std::is_constructible_v<mapping_t, std::dextents<int, 2>, std::array<int, 1>>);
+    // wrong extents rank
+    static_assert(!std::is_constructible_v<mapping_t, std::dextents<int, 3>, std::array<int, 2>>);
+    // none-convertible strides
+    static_assert(!std::is_constructible_v<mapping_t, std::dextents<int, 2>, std::array<IntType, 2>>);
+  }
+  {
+    // not no-throw constructible index_type from stride
+    using mapping_t = std::layout_stride::mapping<std::dextents<unsigned char, 2>>;
+    static_assert(std::is_convertible_v<IntType, unsigned char>);
+    static_assert(!std::is_constructible_v<mapping_t, std::dextents<int, 2>, std::array<IntType, 2>>);
+  }
+
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test());
+  return 0;
+}
diff --git a/libcxx/test/std/containers/views/mdspan/layout_stride/ctor.extents_span.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_stride/ctor.extents_span.pass.cpp
new file mode 100644
index 0000000000..36a87ae7a9
--- /dev/null
+++ b/libcxx/test/std/containers/views/mdspan/layout_stride/ctor.extents_span.pass.cpp
@@ -0,0 +1,141 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <mdspan>
+
+// template<class OtherIndexType>
+//  constexpr mapping(const extents_type& e, span<OtherIndexType, rank_> s) noexcept;
+//
+// Constraints:
+//    - is_convertible_v<const OtherIndexType&, index_type> is true, and
+//    - is_nothrow_constructible_v<index_type, const OtherIndexType&> is true.
+//
+// Preconditions:
+//    - s[i] > 0 is true for all i in the range [0, rank_).
+//    - REQUIRED-SPAN-SIZE(e, s) is representable as a value of type index_type ([basic.fundamental]).
+//    - If rank_ is greater than 0, then there exists a permutation P of the integers in the range [0, rank_),
+//      such that s[pi] >= s[pi_1] * e.extent(pi_1) is true for all i in the range [1, rank_), where pi is the ith element of P.
+//     Note 1: For layout_stride, this condition is necessary and sufficient for is_unique() to be true.
+//
+// Effects: Direct-non-list-initializes extents_ with e, and for all d in the range [0, rank_),
+//         direct-non-list-initializes strides_[d] with as_const(s[d]).
+
+#include <mdspan>
+#include <cassert>
+#include <cstdint>
+
+#include "test_macros.h"
+#include "../ConvertibleToIntegral.h"
+
+template <class E, class S>
+constexpr void test_construction(E e, S s) {
+  using M = std::layout_stride::mapping<E>;
+  ASSERT_NOEXCEPT(M{e, s});
+  M m(e, s);
+
+  // check correct extents are returned
+  ASSERT_NOEXCEPT(m.extents());
+  assert(m.extents() == e);
+
+  // check required_span_size()
+  typename E::index_type expected_size = 1;
+  for (typename E::rank_type r = 0; r < E::rank(); r++) {
+    if (e.extent(r) == 0) {
+      expected_size = 0;
+      break;
+    }
+    expected_size += (e.extent(r) - 1) * static_cast<typename E::index_type>(s[r]);
+  }
+  assert(m.required_span_size() == expected_size);
+
+  // check strides: node stride function is constrained on rank>0, e.extent(r) is not
+  auto strides = m.strides();
+  ASSERT_NOEXCEPT(m.strides());
+  if constexpr (E::rank() > 0) {
+    for (typename E::rank_type r = 0; r < E::rank(); r++) {
+      assert(m.stride(r) == static_cast<typename E::index_type>(s[r]));
+      assert(strides[r] == m.stride(r));
+    }
+  }
+}
+
+constexpr bool test() {
+  constexpr size_t D = std::dynamic_extent;
+  {
+    std::array<int, 0> s{};
+    test_construction(std::extents<int>(), std::span(s));
+  }
+  {
+    std::array<int, 1> s{1};
+    test_construction(std::extents<unsigned, D>(7), std::span(s));
+  }
+  {
+    std::array<int, 1> s{1};
+    test_construction(std::extents<unsigned, D>(0), std::span(s));
+  }
+  {
+    std::array<int, 1> s{2};
+    test_construction(std::extents<unsigned, 7>(), std::span(s));
+  }
+  {
+    std::array<IntType, 1> s{1};
+    test_construction(std::extents<int, D>(7), std::span(s));
+  }
+  {
+    std::array<int, 2> s{3, 30};
+    test_construction(std::extents<unsigned, 7, 8>(), std::span(s));
+  }
+  {
+    std::array<int, 4> s{20, 2, 200, 2000};
+    test_construction(std::extents<int64_t, D, 8, D, D>(7, 9, 10), std::span(s));
+  }
+  {
+    std::array<int, 4> s{20, 2, 200, 2000};
+    test_construction(std::extents<int64_t, D, 8, D, D>(7, 0, 10), std::span(s));
+    test_construction(std::extents<int64_t, D, 8, D, D>(0, 9, 10), std::span(s));
+    test_construction(std::extents<int64_t, D, 8, D, D>(0, 8, 0), std::span(s));
+  }
+  {
+    std::array<int, 4> s{200, 20, 20, 2000};
+    test_construction(std::extents<int64_t, D, D, D, D>(7, 0, 8, 9), std::span(s));
+    test_construction(std::extents<int64_t, D, D, D, D>(7, 8, 0, 9), std::span(s));
+    test_construction(std::extents<int64_t, D, D, D, D>(7, 1, 8, 9), std::span(s));
+    test_construction(std::extents<int64_t, D, D, D, D>(7, 8, 1, 9), std::span(s));
+    test_construction(std::extents<int64_t, D, D, D, D>(7, 1, 1, 9), std::span(s));
+    test_construction(std::extents<int64_t, D, D, D, D>(7, 0, 0, 9), std::span(s));
+    test_construction(std::extents<int64_t, D, D, D, D>(7, 1, 1, 9), std::span(s));
+    test_construction(std::extents<int64_t, D, D, D, D>(7, 1, 0, 9), std::span(s));
+    test_construction(std::extents<int64_t, D, D, D, D>(7, 0, 1, 9), std::span(s));
+  }
+
+  {
+    using mapping_t = std::layout_stride::mapping<std::dextents<unsigned, 2>>;
+    // wrong strides size
+    static_assert(!std::is_constructible_v<mapping_t, std::dextents<int, 2>, std::span<int, 3>>);
+    static_assert(!std::is_constructible_v<mapping_t, std::dextents<int, 2>, std::span<int, 1>>);
+    // wrong extents rank
+    static_assert(!std::is_constructible_v<mapping_t, std::dextents<int, 3>, std::span<int, 2>>);
+    // none-convertible strides
+    static_assert(!std::is_constructible_v<mapping_t, std::dextents<int, 2>, std::span<IntType, 2>>);
+  }
+  {
+    // not no-throw constructible index_type from stride
+    using mapping_t = std::layout_stride::mapping<std::dextents<unsigned char, 2>>;
+    static_assert(std::is_convertible_v<IntType, unsigned char>);
+    static_assert(!std::is_constructible_v<mapping_t, std::dextents<int, 2>, std::span<IntType, 2>>);
+  }
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test());
+  return 0;
+}
diff --git a/libcxx/test/std/containers/views/mdspan/layout_stride/ctor.strided_mapping.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_stride/ctor.strided_mapping.pass.cpp
new file mode 100644
index 0000000000..7c9b3f34a4
--- /dev/null
+++ b/libcxx/test/std/containers/views/mdspan/layout_stride/ctor.strided_mapping.pass.cpp
@@ -0,0 +1,187 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <mdspan>
+
+// template<class StridedLayoutMapping>
+//   constexpr explicit(see below)
+//     mapping(const StridedLayoutMapping& other) noexcept;
+//
+// Constraints:
+//   - layout-mapping-alike<StridedLayoutMapping> is satisfied.
+//   - is_constructible_v<extents_type, typename StridedLayoutMapping::extents_type> is true.
+//   - StridedLayoutMapping::is_always_unique() is true.
+//   - StridedLayoutMapping::is_always_strided() is true.
+//
+// Preconditions:
+//   - StridedLayoutMapping meets the layout mapping requirements ([mdspan.layout.policy.reqmts]),
+//   - other.stride(r) > 0 is true for every rank index r of extents(),
+//   - other.required_span_size() is representable as a value of type index_type ([basic.fundamental]), and
+//   - OFFSET(other) == 0 is true.
+//
+// Effects: Direct-non-list-initializes extents_ with other.extents(), and for all d in the range [0, rank_),
+//          direct-non-list-initializes strides_[d] with other.stride(d).
+//
+// Remarks: The expression inside explicit is equivalent to:
+//   - !(is_convertible_v<typename StridedLayoutMapping::extents_type, extents_type> &&
+//       (is-mapping-of<layout_left, LayoutStrideMapping> ||
+//        is-mapping-of<layout_right, LayoutStrideMapping> ||
+//        is-mapping-of<layout_stride, LayoutStrideMapping>))
+
+#include <mdspan>
+#include <type_traits>
+#include <cassert>
+#include <limits>
+
+#include "test_macros.h"
+
+#include "../CustomTestLayouts.h"
+
+template <bool implicit, class FromL, class ToE, class FromE>
+constexpr void test_conversion(FromE src_exts) {
+  using To   = std::layout_stride::mapping<ToE>;
+  using From = typename FromL::template mapping<FromE>;
+
+  From src([&]() {
+    if constexpr (std::is_same_v<FromL, std::layout_stride>) {
+      // just construct some strides which aren't layout_left/layout_right
+      std::array<size_t, FromE::rank()> strides;
+      size_t stride = 2;
+      for (size_t r = 0; r < FromE::rank(); r++) {
+        strides[r] = stride;
+        stride *= src_exts.extent(r);
+      }
+      return From(src_exts, strides);
+    } else {
+      return From(src_exts);
+    }
+  }());
+
+  ASSERT_NOEXCEPT(To(src));
+  To dest(src);
+  assert(dest == src);
+
+  if constexpr (implicit) {
+    To dest_implicit = src;
+    assert(dest_implicit == src);
+  } else {
+    assert((!std::is_convertible_v<From, To>));
+  }
+}
+
+template <class FromL, class T1, class T2>
+constexpr void test_conversion() {
+  constexpr size_t D = std::dynamic_extent;
+  constexpr bool idx_convertible =
+      static_cast<size_t>(std::numeric_limits<T1>::max()) >= static_cast<size_t>(std::numeric_limits<T2>::max());
+  constexpr bool l_convertible =
+      std::is_same_v<FromL, std::layout_right> || std::is_same_v<FromL, std::layout_left> ||
+      std::is_same_v<FromL, std::layout_stride>;
+  constexpr bool idx_l_convertible = idx_convertible && l_convertible;
+
+  // clang-format off
+  // adding extents convertibility expectation
+  test_conversion<idx_l_convertible && true,  FromL, std::extents<T1>>(std::extents<T2>());
+  test_conversion<idx_l_convertible && true,  FromL, std::extents<T1, D>>(std::extents<T2, D>(0));
+  test_conversion<idx_l_convertible && true,  FromL, std::extents<T1, D>>(std::extents<T2, D>(5));
+  test_conversion<idx_l_convertible && false, FromL, std::extents<T1, 5>>(std::extents<T2, D>(5));
+  test_conversion<idx_l_convertible && true,  FromL, std::extents<T1, 5>>(std::extents<T2, 5>());
+  test_conversion<idx_l_convertible && false, FromL, std::extents<T1, 5, D>>(std::extents<T2, D, D>(5, 5));
+  test_conversion<idx_l_convertible && true,  FromL, std::extents<T1, D, D>>(std::extents<T2, D, D>(5, 5));
+  test_conversion<idx_l_convertible && true,  FromL, std::extents<T1, D, D>>(std::extents<T2, D, 7>(5));
+  test_conversion<idx_l_convertible && true,  FromL, std::extents<T1, 5, 7>>(std::extents<T2, 5, 7>());
+  test_conversion<idx_l_convertible && false, FromL, std::extents<T1, 5, D, 8, D, D>>(std::extents<T2, D, D, 8, 9, 1>(5, 7));
+  test_conversion<idx_l_convertible && true,  FromL, std::extents<T1, D, D, D, D, D>>(
+                                                     std::extents<T2, D, D, D, D, D>(5, 7, 8, 9, 1));
+  test_conversion<idx_l_convertible && true,  FromL, std::extents<T1, D, D, 8, 9, D>>(std::extents<T2, D, 7, 8, 9, 1>(5));
+  test_conversion<idx_l_convertible && true,  FromL, std::extents<T1, 5, 7, 8, 9, 1>>(std::extents<T2, 5, 7, 8, 9, 1>());
+  // clang-format on
+}
+
+template <class IdxT, size_t... Extents>
+using ToM = typename std::layout_stride::template mapping<std::extents<IdxT, Extents...>>;
+
+template <class FromL, class IdxT, size_t... Extents>
+using FromM = typename FromL::template mapping<std::extents<IdxT, Extents...>>;
+
+template <class FromL>
+constexpr void test_no_implicit_conversion() {
+  constexpr size_t D = std::dynamic_extent;
+
+  // Sanity check that one static to dynamic conversion works
+  static_assert(std::is_constructible_v<ToM<int, D>, FromM<FromL, int, 5>>);
+  static_assert(std::is_convertible_v<FromM<FromL, int, 5>, ToM<int, D>>);
+
+  // Check that dynamic to static conversion only works explicitly
+  static_assert(std::is_constructible_v<ToM<int, 5>, FromM<FromL, int, D>>);
+  static_assert(!std::is_convertible_v<FromM<FromL, int, D>, ToM<int, 5>>);
+
+  // Sanity check that one static to dynamic conversion works
+  static_assert(std::is_constructible_v<ToM<int, D, 7>, FromM<FromL, int, 5, 7>>);
+  static_assert(std::is_convertible_v<FromM<FromL, int, 5, 7>, ToM<int, D, 7>>);
+
+  // Check that dynamic to static conversion only works explicitly
+  static_assert(std::is_constructible_v<ToM<int, 5, 7>, FromM<FromL, int, D, 7>>);
+  static_assert(!std::is_convertible_v<FromM<FromL, int, D, 7>, ToM<int, 5, 7>>);
+
+  // Sanity check that smaller index_type to larger index_type conversion works
+  static_assert(std::is_constructible_v<ToM<size_t, 5>, FromM<FromL, int, 5>>);
+  static_assert(std::is_convertible_v<FromM<FromL, int, 5>, ToM<size_t, 5>>);
+
+  // Check that larger index_type to smaller index_type conversion works explicitly only
+  static_assert(std::is_constructible_v<ToM<int, 5>, FromM<FromL, size_t, 5>>);
+  static_assert(!std::is_convertible_v<FromM<FromL, size_t, 5>, ToM<int, 5>>);
+}
+
+template <class FromL>
+constexpr void test_rank_mismatch() {
+  constexpr size_t D = std::dynamic_extent;
+
+  static_assert(!std::is_constructible_v<ToM<int, D>, FromM<FromL, int>>);
+  static_assert(!std::is_constructible_v<ToM<int>, FromM<FromL, int, D, D>>);
+  static_assert(!std::is_constructible_v<ToM<int, D>, FromM<FromL, int, D, D>>);
+  static_assert(!std::is_constructible_v<ToM<int, D, D, D>, FromM<FromL, int, D, D>>);
+}
+
+template <class FromL>
+constexpr void test_static_extent_mismatch() {
+  constexpr size_t D = std::dynamic_extent;
+
+  static_assert(!std::is_constructible_v<ToM<int, D, 5>, FromM<FromL, int, D, 4>>);
+  static_assert(!std::is_constructible_v<ToM<int, 5>, FromM<FromL, int, 4>>);
+  static_assert(!std::is_constructible_v<ToM<int, 5, D>, FromM<FromL, int, 4, D>>);
+}
+
+template <class FromL>
+constexpr void test_layout() {
+  test_conversion<FromL, int, int>();
+  test_conversion<FromL, int, size_t>();
+  test_conversion<FromL, size_t, int>();
+  test_conversion<FromL, size_t, long>();
+  // the implicit convertibility test doesn't apply to non std::layouts
+  if constexpr (!std::is_same_v<FromL, always_convertible_layout>)
+    test_no_implicit_conversion<FromL>();
+  test_rank_mismatch<FromL>();
+  test_static_extent_mismatch<FromL>();
+}
+
+constexpr bool test() {
+  test_layout<std::layout_right>();
+  test_layout<std::layout_left>();
+  test_layout<std::layout_stride>();
+  test_layout<always_convertible_layout>();
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test());
+  return 0;
+}
diff --git a/libcxx/test/std/containers/views/mdspan/layout_stride/deduction.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_stride/deduction.pass.cpp
new file mode 100644
index 0000000000..ebe31d9d0e
--- /dev/null
+++ b/libcxx/test/std/containers/views/mdspan/layout_stride/deduction.pass.cpp
@@ -0,0 +1,55 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+// ADDITIONAL_COMPILE_FLAGS: -Wno-ctad-maybe-unsupported
+
+// <mdspan>
+
+#include <mdspan>
+#include <type_traits>
+#include <concepts>
+#include <cassert>
+
+#include "test_macros.h"
+
+// mdspan
+
+// layout_stride::mapping does not have explicit deduction guides,
+// but implicit deduction guides for constructor taking extents and strides
+// should work
+
+constexpr bool test() {
+  constexpr size_t D = std::dynamic_extent;
+
+  ASSERT_SAME_TYPE(decltype(std::layout_stride::mapping(std::extents<int>(), std::array<unsigned, 0>())),
+                   std::layout_stride::template mapping<std::extents<int>>);
+  ASSERT_SAME_TYPE(decltype(std::layout_stride::mapping(std::extents<int, 4>(), std::array<char, 1>{1})),
+                   std::layout_stride::template mapping<std::extents<int, 4>>);
+  ASSERT_SAME_TYPE(decltype(std::layout_stride::mapping(std::extents<int, D>(), std::array<char, 1>{1})),
+                   std::layout_stride::template mapping<std::extents<int, D>>);
+  ASSERT_SAME_TYPE(
+      decltype(std::layout_stride::mapping(std::extents<unsigned, D, 3>(), std::array<int64_t, 2>{3, 100})),
+      std::layout_stride::template mapping<std::extents<unsigned, D, 3>>);
+
+  ASSERT_SAME_TYPE(decltype(std::layout_stride::mapping(std::extents<int>(), std::span<unsigned, 0>())),
+                   std::layout_stride::template mapping<std::extents<int>>);
+  ASSERT_SAME_TYPE(decltype(std::layout_stride::mapping(std::extents<int, 4>(), std::declval<std::span<char, 1>>())),
+                   std::layout_stride::template mapping<std::extents<int, 4>>);
+  ASSERT_SAME_TYPE(decltype(std::layout_stride::mapping(std::extents<int, D>(), std::declval<std::span<char, 1>>())),
+                   std::layout_stride::template mapping<std::extents<int, D>>);
+  ASSERT_SAME_TYPE(
+      decltype(std::layout_stride::mapping(std::extents<unsigned, D, 3>(), std::declval<std::span<int64_t, 2>>())),
+      std::layout_stride::template mapping<std::extents<unsigned, D, 3>>);
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test());
+  return 0;
+}
diff --git a/libcxx/test/std/containers/views/mdspan/layout_stride/extents.verify.cpp b/libcxx/test/std/containers/views/mdspan/layout_stride/extents.verify.cpp
new file mode 100644
index 0000000000..4742527f7a
--- /dev/null
+++ b/libcxx/test/std/containers/views/mdspan/layout_stride/extents.verify.cpp
@@ -0,0 +1,33 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <mdspan>
+
+// template<class Extents>
+// class layout_stride::mapping;
+
+// If Extents is not a specialization of extents, then the program is
+// ill-formed.
+
+// Mandates: If Extents::rank_dynamic() == 0 is true, then the size of the
+// multidimensional index space Extents() is representable as a value of type
+// typename Extents::index_type.
+
+#include <mdspan>
+
+void not_extents() {
+  // expected-error-re@*:* {{{{(static_assert|static assertion)}} failed {{.*}}layout_stride::mapping template argument must be a specialization of extents}}
+  [[maybe_unused]] std::layout_stride::mapping<void> mapping;
+}
+
+void representable() {
+  // expected-error-re@*:* {{{{(static_assert|static assertion)}} failed {{.*}}layout_stride::mapping product of static extents must be representable as index_type.}}
+  [[maybe_unused]] std::layout_stride::mapping<std::extents<char, 20, 20>> mapping;
+}
diff --git a/libcxx/test/std/containers/views/mdspan/layout_stride/index_operator.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_stride/index_operator.pass.cpp
new file mode 100644
index 0000000000..dbd56a5475
--- /dev/null
+++ b/libcxx/test/std/containers/views/mdspan/layout_stride/index_operator.pass.cpp
@@ -0,0 +1,121 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <mdspan>
+
+// Test default iteration:
+//
+// template<class... Indices>
+//   constexpr index_type operator()(Indices...) const noexcept;
+//
+// Constraints:
+//   * sizeof...(Indices) == extents_type::rank() is true,
+//   * (is_convertible_v<Indices, index_type> && ...) is true, and
+//   * (is_nothrow_constructible_v<index_type, Indices> && ...) is true.
+//
+// Preconditions:
+//   * extents_type::index-cast(i) is a multidimensional index in extents_.
+
+#include <mdspan>
+#include <cassert>
+#include <cstdint>
+
+#include "test_macros.h"
+
+#include "../ConvertibleToIntegral.h"
+
+template <class Mapping, class... Indices>
+concept operator_constraints = requires(Mapping m, Indices... idxs) {
+  { std::is_same_v<decltype(m(idxs...)), typename Mapping::index_type> };
+};
+
+template <class Mapping, class... Indices>
+  requires(operator_constraints<Mapping, Indices...>)
+constexpr bool check_operator_constraints(Mapping m, Indices... idxs) {
+  (void)m(idxs...);
+  return true;
+}
+
+template <class Mapping, class... Indices>
+constexpr bool check_operator_constraints(Mapping, Indices...) {
+  return false;
+}
+
+template <class M, class... Args>
+constexpr void iterate_stride(M m, const std::array<int, M::extents_type::rank()>& strides, Args... args) {
+  constexpr int r = static_cast<int>(M::extents_type::rank()) - 1 - static_cast<int>(sizeof...(Args));
+  if constexpr (-1 == r) {
+    ASSERT_NOEXCEPT(m(args...));
+    size_t expected_val = [&]<size_t... Pos>(std::index_sequence<Pos...>) {
+      return ((args * strides[Pos]) + ... + 0);
+    }(std::make_index_sequence<M::extents_type::rank()>());
+    assert(expected_val == static_cast<size_t>(m(args...)));
+  } else {
+    for (typename M::index_type i = 0; i < m.extents().extent(r); i++) {
+      iterate_stride(m, strides, i, args...);
+    }
+  }
+}
+
+template <class E, class... Args>
+constexpr void test_iteration(std::array<int, E::rank()> strides, Args... args) {
+  using M = std::layout_stride::mapping<E>;
+  M m(E(args...), strides);
+
+  iterate_stride(m, strides);
+}
+
+constexpr bool test() {
+  constexpr size_t D = std::dynamic_extent;
+  test_iteration<std::extents<int>>(std::array<int, 0>{});
+  test_iteration<std::extents<unsigned, D>>(std::array<int, 1>{2}, 1);
+  test_iteration<std::extents<unsigned, D>>(std::array<int, 1>{3}, 7);
+  test_iteration<std::extents<unsigned, 7>>(std::array<int, 1>{4});
+  test_iteration<std::extents<unsigned, 7, 8>>(std::array<int, 2>{25, 3});
+  test_iteration<std::extents<char, D, D, D, D>>(std::array<int, 4>{1, 1, 1, 1}, 1, 1, 1, 1);
+
+  // Check operator constraint for number of arguments
+  static_assert(check_operator_constraints(
+      std::layout_stride::mapping<std::extents<int, D>>(std::extents<int, D>(1), std::array{1}), 0));
+  static_assert(!check_operator_constraints(
+      std::layout_stride::mapping<std::extents<int, D>>(std::extents<int, D>(1), std::array{1}), 0, 0));
+
+  // Check operator constraint for convertibility of arguments to index_type
+  static_assert(check_operator_constraints(
+      std::layout_stride::mapping<std::extents<int, D>>(std::extents<int, D>(1), std::array{1}), IntType(0)));
+  static_assert(!check_operator_constraints(
+      std::layout_stride::mapping<std::extents<unsigned, D>>(std::extents<unsigned, D>(1), std::array{1}), IntType(0)));
+
+  // Check operator constraint for no-throw-constructibility of index_type from arguments
+  static_assert(!check_operator_constraints(
+      std::layout_stride::mapping<std::extents<unsigned char, D>>(std::extents<unsigned char, D>(1), std::array{1}),
+      IntType(0)));
+
+  return true;
+}
+
+constexpr bool test_large() {
+  constexpr size_t D = std::dynamic_extent;
+  test_iteration<std::extents<int64_t, D, 8, D, D>>(std::array<int, 4>{2000, 2, 20, 200}, 7, 9, 10);
+  test_iteration<std::extents<int64_t, D, 8, 1, D>>(std::array<int, 4>{2000, 20, 20, 200}, 7, 10);
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test());
+
+  // The large test iterates over ~10k loop indices.
+  // With assertions enabled this triggered the maximum default limit
+  // for steps in consteval expressions. Assertions roughly double the
+  // total number of instructions, so this was already close to the maximum.
+  test_large();
+  return 0;
+}
diff --git a/libcxx/test/std/containers/views/mdspan/layout_stride/is_exhaustive_corner_case.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_stride/is_exhaustive_corner_case.pass.cpp
new file mode 100644
index 0000000000..d13db91ac5
--- /dev/null
+++ b/libcxx/test/std/containers/views/mdspan/layout_stride/is_exhaustive_corner_case.pass.cpp
@@ -0,0 +1,54 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <mdspan>
+
+// constexpr bool is_exhaustive() const noexcept;
+//
+// Returns:
+//   - true if rank_ is 0.
+//   - Otherwise, true if there is a permutation P of the integers in the range [0, rank_) such that
+//     stride(p0) equals 1, and stride(pi) equals stride(pi_1) * extents().extent(pi_1) for i in the
+//     range [1, rank_), where pi is the ith element of P.
+//   - Otherwise, false.
+
+#include <mdspan>
+#include <type_traits>
+#include <concepts>
+#include <cassert>
+
+#include "test_macros.h"
+
+template <class E>
+constexpr void
+test_layout_mapping_stride(E ext, std::array<typename E::index_type, E::rank()> strides, bool exhaustive) {
+  using M = std::layout_stride::template mapping<E>;
+  M m(ext, strides);
+  assert(m.is_exhaustive() == exhaustive);
+}
+
+constexpr bool test() {
+  constexpr size_t D = std::dynamic_extent;
+  test_layout_mapping_stride(std::extents<int, 0>(), std::array<int, 1>{1}, true);
+  test_layout_mapping_stride(std::extents<unsigned, D>(0), std::array<unsigned, 1>{3}, false);
+  test_layout_mapping_stride(std::extents<int, 0, 3>(), std::array<int, 2>{6, 2}, true);
+  test_layout_mapping_stride(std::extents<int, D, D>(3, 0), std::array<int, 2>{6, 2}, false);
+  test_layout_mapping_stride(std::extents<int, D, D>(0, 0), std::array<int, 2>{6, 2}, false);
+  test_layout_mapping_stride(
+      std::extents<unsigned, D, D, D, D>(3, 3, 0, 3), std::array<unsigned, 4>{3, 1, 27, 9}, true);
+  test_layout_mapping_stride(std::extents<int, D, D, D, D>(0, 3, 3, 3), std::array<int, 4>{3, 1, 27, 9}, false);
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test());
+  return 0;
+}
diff --git a/libcxx/test/std/containers/views/mdspan/layout_stride/properties.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_stride/properties.pass.cpp
new file mode 100644
index 0000000000..d3d0aa6b3c
--- /dev/null
+++ b/libcxx/test/std/containers/views/mdspan/layout_stride/properties.pass.cpp
@@ -0,0 +1,116 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <mdspan>
+
+// namespace std {
+//   template<class Extents>
+//   class layout_stride::mapping {
+//
+//     ...
+//     static constexpr bool is_always_unique() noexcept { return true; }
+//     static constexpr bool is_always_exhaustive() noexcept { return false; }
+//     static constexpr bool is_always_strided() noexcept { return true; }
+//
+//     static constexpr bool is_unique() noexcept { return true; }
+//     static constexpr bool is_exhaustive() noexcept;
+//     static constexpr bool is_strided() noexcept { return true; }
+//     ...
+//   };
+// }
+//
+//
+// layout_stride::mapping<E> is a trivially copyable type that models regular for each E.
+//
+// constexpr bool is_exhaustive() const noexcept;
+//
+// Returns:
+//   - true if rank_ is 0.
+//   - Otherwise, true if there is a permutation P of the integers in the range [0, rank_) such that
+//     stride(p0) equals 1, and stride(pi) equals stride(pi_1) * extents().extent(pi_1) for i in the
+//     range [1, rank_), where pi is the ith element of P.
+//   - Otherwise, false.
+
+#include <mdspan>
+#include <type_traits>
+#include <concepts>
+#include <cassert>
+
+#include "test_macros.h"
+
+template <class E>
+constexpr void
+test_layout_mapping_stride(E ext, std::array<typename E::index_type, E::rank()> strides, bool exhaustive) {
+  using M = std::layout_stride::template mapping<E>;
+  M m(ext, strides);
+  const M c_m = m;
+  assert(m.strides() == strides);
+  assert(c_m.strides() == strides);
+  assert(m.extents() == ext);
+  assert(c_m.extents() == ext);
+  assert(M::is_unique() == true);
+  assert(m.is_exhaustive() == exhaustive);
+  assert(c_m.is_exhaustive() == exhaustive);
+  assert(M::is_strided() == true);
+  assert(M::is_always_unique() == true);
+  assert(M::is_always_exhaustive() == false);
+  assert(M::is_always_strided() == true);
+
+  ASSERT_NOEXCEPT(m.strides());
+  ASSERT_NOEXCEPT(c_m.strides());
+  ASSERT_NOEXCEPT(m.extents());
+  ASSERT_NOEXCEPT(c_m.extents());
+  ASSERT_NOEXCEPT(M::is_unique());
+  ASSERT_NOEXCEPT(m.is_exhaustive());
+  ASSERT_NOEXCEPT(c_m.is_exhaustive());
+  ASSERT_NOEXCEPT(M::is_strided());
+  ASSERT_NOEXCEPT(M::is_always_unique());
+  ASSERT_NOEXCEPT(M::is_always_exhaustive());
+  ASSERT_NOEXCEPT(M::is_always_strided());
+
+  for (typename E::rank_type r = 0; r < E::rank(); r++) {
+    assert(m.stride(r) == strides[r]);
+    assert(c_m.stride(r) == strides[r]);
+    ASSERT_NOEXCEPT(m.stride(r));
+    ASSERT_NOEXCEPT(c_m.stride(r));
+  }
+
+  typename E::index_type expected_size = 1;
+  for (typename E::rank_type r = 0; r < E::rank(); r++) {
+    if (ext.extent(r) == 0) {
+      expected_size = 0;
+      break;
+    }
+    expected_size += (ext.extent(r) - 1) * static_cast<typename E::index_type>(strides[r]);
+  }
+  assert(m.required_span_size() == expected_size);
+  assert(c_m.required_span_size() == expected_size);
+  ASSERT_NOEXCEPT(m.required_span_size());
+  ASSERT_NOEXCEPT(c_m.required_span_size());
+
+  static_assert(std::is_trivially_copyable_v<M>);
+  static_assert(std::regular<M>);
+}
+
+constexpr bool test() {
+  constexpr size_t D = std::dynamic_extent;
+  test_layout_mapping_stride(std::extents<int>(), std::array<int, 0>{}, true);
+  test_layout_mapping_stride(std::extents<char, 4, 5>(), std::array<char, 2>{1, 4}, true);
+  test_layout_mapping_stride(std::extents<char, 4, 5>(), std::array<char, 2>{1, 5}, false);
+  test_layout_mapping_stride(std::extents<unsigned, D, 4>(7), std::array<unsigned, 2>{20, 2}, false);
+  test_layout_mapping_stride(std::extents<size_t, D, D, D, D>(3, 3, 3, 3), std::array<size_t, 4>{3, 1, 9, 27}, true);
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test());
+  return 0;
+}
diff --git a/libcxx/test/std/containers/views/mdspan/layout_stride/required_span_size.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_stride/required_span_size.pass.cpp
new file mode 100644
index 0000000000..3b6af9f6c1
--- /dev/null
+++ b/libcxx/test/std/containers/views/mdspan/layout_stride/required_span_size.pass.cpp
@@ -0,0 +1,56 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <mdspan>
+
+// Let REQUIRED-SPAN-SIZE(e, strides) be:
+//    - 1, if e.rank() == 0 is true,
+//    - otherwise 0, if the size of the multidimensional index space e is 0,
+//    - otherwise 1 plus the sum of products of (e.extent(r) - 1) and strides[r] for all r in the range [0, e.rank()).
+
+// constexpr index_type required_span_size() const noexcept;
+//
+//   Returns: REQUIRED-SPAN-SIZE(extents(), strides_).
+
+#include <mdspan>
+#include <cassert>
+#include <cstdint>
+
+#include "test_macros.h"
+
+template <class E>
+constexpr void test_required_span_size(E e, std::array<int, E::rank()> strides, typename E::index_type expected_size) {
+  using M = std::layout_stride::mapping<E>;
+  const M m(e, strides);
+
+  ASSERT_NOEXCEPT(m.required_span_size());
+  assert(m.required_span_size() == expected_size);
+}
+
+constexpr bool test() {
+  constexpr size_t D = std::dynamic_extent;
+  test_required_span_size(std::extents<int>(), std::array<int, 0>{}, 1);
+  test_required_span_size(std::extents<unsigned, D>(0), std::array<int, 1>{5}, 0);
+  test_required_span_size(std::extents<unsigned, D>(1), std::array<int, 1>{5}, 1);
+  test_required_span_size(std::extents<unsigned, D>(7), std::array<int, 1>{5}, 31);
+  test_required_span_size(std::extents<unsigned, 7>(), std::array<int, 1>{5}, 31);
+  test_required_span_size(std::extents<unsigned, 7, 8>(), std::array<int, 2>{20, 2}, 135);
+  test_required_span_size(
+      std::extents<int64_t, D, 8, D, D>(7, 9, 10), std::array<int, 4>{1, 7, 7 * 8, 7 * 8 * 9}, 5040);
+  test_required_span_size(std::extents<int64_t, 1, 8, D, D>(9, 10), std::array<int, 4>{1, 7, 7 * 8, 7 * 8 * 9}, 5034);
+  test_required_span_size(std::extents<int64_t, 1, 0, D, D>(9, 10), std::array<int, 4>{1, 7, 7 * 8, 7 * 8 * 9}, 0);
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test());
+  return 0;
+}
diff --git a/libcxx/test/std/containers/views/mdspan/layout_stride/static_requirements.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_stride/static_requirements.pass.cpp
new file mode 100644
index 0000000000..112f4b9e5a
--- /dev/null
+++ b/libcxx/test/std/containers/views/mdspan/layout_stride/static_requirements.pass.cpp
@@ -0,0 +1,131 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <mdspan>
+
+// A type M meets the layout mapping requirements if
+//    - M models copyable and equality_comparable,
+//    - is_nothrow_move_constructible_v<M> is true,
+//    - is_nothrow_move_assignable_v<M> is true,
+//    - is_nothrow_swappable_v<M> is true, and
+//
+// the following types and expressions are well-formed and have the specified semantics.
+//
+//  typename M::extents_type
+//    Result: A type that is a specialization of extents.
+//
+//  typename M::index_type
+//    Result: typename M::extents_type::index_type.
+//
+//  typename M::rank_type
+//    Result: typename M::extents_type::rank_type.
+//
+//  typename M::layout_type
+//    Result: A type MP that meets the layout mapping policy requirements ([mdspan.layout.policy.reqmts]) and for which is-mapping-of<MP, M> is true.
+//
+//  m.extents()
+//    Result: const typename M::extents_type&
+//
+//  m(i...)
+//    Result: typename M::index_type
+//    Returns: A nonnegative integer less than numeric_limits<typename M::index_type>::max() and less than or equal to numeric_limits<size_t>::max().
+//
+//  m(i...) == m(static_cast<typename M::index_type>(i)...)
+//    Result: bool
+//    Returns: true
+//
+//  m.required_span_size()
+//    Result: typename M::index_type
+//    Returns: If the size of the multidimensional index space m.extents() is 0, then 0, else 1 plus the maximum value of m(i...) for all i.
+//
+//  m.is_unique()
+//    Result: bool
+//    Returns: true only if for every i and j where (i != j || ...) is true, m(i...) != m(j...) is true.
+//
+//  m.is_exhaustive()
+//    Result: bool
+//    Returns: true only if for all k in the range [0, m.required_span_size()) there exists an i such that m(i...) equals k.
+//
+//  m.is_strided()
+//    Result: bool
+//    Returns: true only if for every rank index r of m.extents() there exists an integer
+//             sr such that, for all i where (i+dr) is a multidimensional index in m.extents() ([mdspan.overview]),
+//             m((i + dr)...) - m(i...) equals sr
+//
+//  m.stride(r)
+//    Preconditions: m.is_strided() is true.
+//    Result: typename M::index_type
+//    Returns: sr as defined in m.is_strided() above.
+//
+//  M::is_always_unique()
+//    Result: A constant expression ([expr.const]) of type bool.
+//    Returns: true only if m.is_unique() is true for all possible objects m of type M.
+//
+//  M::is_always_exhaustive()
+//    Result: A constant expression ([expr.const]) of type bool.
+//    Returns: true only if m.is_exhaustive() is true for all possible objects m of type M.
+//
+//  M::is_always_strided()
+//    Result: A constant expression ([expr.const]) of type bool.
+//    Returns: true only if m.is_strided() is true for all possible objects m of type M.
+
+#include <mdspan>
+#include <type_traits>
+#include <concepts>
+#include <cassert>
+
+#include "test_macros.h"
+
+// Common requirements of all layout mappings
+template <class M, size_t... Idxs>
+void test_mapping_requirements(std::index_sequence<Idxs...>) {
+  using E = typename M::extents_type;
+  static_assert(std::__mdspan_detail::__is_extents_v<E>);
+  static_assert(std::is_copy_constructible_v<M>);
+  static_assert(std::is_nothrow_move_constructible_v<M>);
+  static_assert(std::is_nothrow_move_assignable_v<M>);
+  static_assert(std::is_nothrow_swappable_v<M>);
+  ASSERT_SAME_TYPE(typename M::index_type, typename E::index_type);
+  ASSERT_SAME_TYPE(typename M::size_type, typename E::size_type);
+  ASSERT_SAME_TYPE(typename M::rank_type, typename E::rank_type);
+  ASSERT_SAME_TYPE(typename M::layout_type, std::layout_stride);
+  ASSERT_SAME_TYPE(typename M::layout_type::template mapping<E>, M);
+  static_assert(std::is_same_v<decltype(std::declval<M>().extents()), const E&>);
+  static_assert(std::is_same_v<decltype(std::declval<M>().strides()), std::array<typename M::index_type, E::rank()>>);
+  static_assert(std::is_same_v<decltype(std::declval<M>()(Idxs...)), typename M::index_type>);
+  static_assert(std::is_same_v<decltype(std::declval<M>().required_span_size()), typename M::index_type>);
+  static_assert(std::is_same_v<decltype(std::declval<M>().is_unique()), bool>);
+  static_assert(std::is_same_v<decltype(std::declval<M>().is_exhaustive()), bool>);
+  static_assert(std::is_same_v<decltype(std::declval<M>().is_strided()), bool>);
+  static_assert(std::is_same_v<decltype(std::declval<M>().stride(0)), typename M::index_type>);
+  static_assert(std::is_same_v<decltype(M::is_always_unique()), bool>);
+  static_assert(std::is_same_v<decltype(M::is_always_exhaustive()), bool>);
+  static_assert(std::is_same_v<decltype(M::is_always_strided()), bool>);
+}
+
+template <class L, class E>
+void test_layout_mapping_requirements() {
+  using M = typename L::template mapping<E>;
+  test_mapping_requirements<M>(std::make_index_sequence<E::rank()>());
+}
+
+template <class E>
+void test_layout_mapping_stride() {
+  test_layout_mapping_requirements<std::layout_stride, E>();
+}
+
+int main(int, char**) {
+  constexpr size_t D = std::dynamic_extent;
+  test_layout_mapping_stride<std::extents<int>>();
+  test_layout_mapping_stride<std::extents<char, 4, 5>>();
+  test_layout_mapping_stride<std::extents<unsigned, D, 4>>();
+  test_layout_mapping_stride<std::extents<size_t, D, D, D, D>>();
+  return 0;
+}
diff --git a/libcxx/test/std/containers/views/mdspan/layout_stride/stride.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_stride/stride.pass.cpp
new file mode 100644
index 0000000000..07a5199e5b
--- /dev/null
+++ b/libcxx/test/std/containers/views/mdspan/layout_stride/stride.pass.cpp
@@ -0,0 +1,57 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+// <mdspan>
+
+// constexpr index_type stride(rank_type i) const noexcept;
+//
+//   Constraints: extents_type::rank() > 0 is true.
+//
+//   Preconditions: i < extents_type::rank() is true.
+//
+//   Returns: extents().rev-prod-of-extents(i).
+
+#include <mdspan>
+#include <cassert>
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include "test_macros.h"
+
+template <class E, class... Args>
+constexpr void test_stride(std::array<typename E::index_type, E::rank()> strides, Args... args) {
+  using M = std::layout_stride::mapping<E>;
+  M m(E(args...), strides);
+
+  ASSERT_NOEXCEPT(m.stride(0));
+  for (size_t r = 0; r < E::rank(); r++)
+    assert(strides[r] == m.stride(r));
+
+  ASSERT_NOEXCEPT(m.strides());
+  auto strides_out = m.strides();
+  static_assert(std::is_same_v<decltype(strides_out), std::array<typename E::index_type, E::rank()>>);
+  for (size_t r = 0; r < E::rank(); r++)
+    assert(strides[r] == strides_out[r]);
+}
+
+constexpr bool test() {
+  constexpr size_t D = std::dynamic_extent;
+  test_stride<std::extents<unsigned, D>>(std::array<unsigned, 1>{1}, 7);
+  test_stride<std::extents<unsigned, 7>>(std::array<unsigned, 1>{1});
+  test_stride<std::extents<unsigned, 7, 8>>(std::array<unsigned, 2>{8, 1});
+  test_stride<std::extents<int64_t, D, 8, D, D>>(std::array<int64_t, 4>{720, 90, 10, 1}, 7, 9, 10);
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test());
+  return 0;
+}
diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/assign.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/assign.pass.cpp
index a6f436fd44..8741c79315 100644
--- a/libcxx/test/std/containers/views/mdspan/mdspan/assign.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/mdspan/assign.pass.cpp
@@ -19,7 +19,7 @@
 #include "test_macros.h"
 
 #include "../MinimalElementType.h"
-#include "CustomTestLayouts.h"
+#include "../CustomTestLayouts.h"
 #include "CustomTestAccessors.h"
 
 template <class H, class M, class A>
diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/conversion.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/conversion.pass.cpp
index bb59276333..abb647e960 100644
--- a/libcxx/test/std/containers/views/mdspan/mdspan/conversion.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/mdspan/conversion.pass.cpp
@@ -43,7 +43,7 @@
 #include "test_macros.h"
 
 #include "../MinimalElementType.h"
-#include "CustomTestLayouts.h"
+#include "../CustomTestLayouts.h"
 #include "CustomTestAccessors.h"
 
 template <class ToMDS, class FromMDS>
diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/conversion.verify.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/conversion.verify.cpp
index c9fcdb11f8..e3f03ebd07 100644
--- a/libcxx/test/std/containers/views/mdspan/mdspan/conversion.verify.cpp
+++ b/libcxx/test/std/containers/views/mdspan/mdspan/conversion.verify.cpp
@@ -37,7 +37,7 @@
 
 #include <mdspan>
 #include "CustomTestAccessors.h"
-#include "CustomTestLayouts.h"
+#include "../CustomTestLayouts.h"
 
 void cant_construct_data_handle_type() {
   int data;
diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.copy.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.copy.pass.cpp
index 813250d8ce..7fb85ed845 100644
--- a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.copy.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.copy.pass.cpp
@@ -21,7 +21,7 @@
 #include "test_macros.h"
 
 #include "../MinimalElementType.h"
-#include "CustomTestLayouts.h"
+#include "../CustomTestLayouts.h"
 #include "CustomTestAccessors.h"
 
 template <class H, class M, class A>
diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.default.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.default.pass.cpp
index f42a4dbf1b..8ad12944df 100644
--- a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.default.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.default.pass.cpp
@@ -29,7 +29,7 @@
 #include "test_macros.h"
 
 #include "../MinimalElementType.h"
-#include "CustomTestLayouts.h"
+#include "../CustomTestLayouts.h"
 #include "CustomTestAccessors.h"
 
 template <bool hc, bool mc, bool ac, class H, class M, class A>
diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_array.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_array.pass.cpp
index e7fd81b3ab..e4f88d6035 100644
--- a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_array.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_array.pass.cpp
@@ -38,7 +38,7 @@
 
 #include "../ConvertibleToIntegral.h"
 #include "../MinimalElementType.h"
-#include "CustomTestLayouts.h"
+#include "../CustomTestLayouts.h"
 #include "CustomTestAccessors.h"
 
 template <class Extents, size_t... Idxs>
diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_extents.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_extents.pass.cpp
index c194fdaeff..c15d1dc47a 100644
--- a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_extents.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_extents.pass.cpp
@@ -31,7 +31,7 @@
 #include "test_macros.h"
 
 #include "../MinimalElementType.h"
-#include "CustomTestLayouts.h"
+#include "../CustomTestLayouts.h"
 #include "CustomTestAccessors.h"
 
 template <bool mec, bool ac, class H, class M, class A>
diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_integers.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_integers.pass.cpp
index 1bd9e488ad..0288e63106 100644
--- a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_integers.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_integers.pass.cpp
@@ -38,7 +38,7 @@
 #include "test_macros.h"
 
 #include "../MinimalElementType.h"
-#include "CustomTestLayouts.h"
+#include "../CustomTestLayouts.h"
 #include "CustomTestAccessors.h"
 
 template <class MDS, class... Args>
diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_map.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_map.pass.cpp
index 9b1c6bd066..9fe79ad460 100644
--- a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_map.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_map.pass.cpp
@@ -29,7 +29,7 @@
 #include "test_macros.h"
 
 #include "../MinimalElementType.h"
-#include "CustomTestLayouts.h"
+#include "../CustomTestLayouts.h"
 #include "CustomTestAccessors.h"
 
 template <bool ac, class H, class M, class A>
diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_map_acc.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_map_acc.pass.cpp
index 7a94e8b040..3be47ae920 100644
--- a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_map_acc.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_map_acc.pass.cpp
@@ -26,7 +26,7 @@
 #include "test_macros.h"
 
 #include "../MinimalElementType.h"
-#include "CustomTestLayouts.h"
+#include "../CustomTestLayouts.h"
 #include "CustomTestAccessors.h"
 
 template <class H, class M, class A>
diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_span.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_span.pass.cpp
index f76c909783..30876abea5 100644
--- a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_span.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_span.pass.cpp
@@ -38,7 +38,7 @@
 
 #include "../ConvertibleToIntegral.h"
 #include "../MinimalElementType.h"
-#include "CustomTestLayouts.h"
+#include "../CustomTestLayouts.h"
 #include "CustomTestAccessors.h"
 
 template <class Extents, size_t... Idxs>
diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.move.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.move.pass.cpp
index aaaa4bc857..10bd5114bd 100644
--- a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.move.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.move.pass.cpp
@@ -21,7 +21,7 @@
 #include "test_macros.h"
 
 #include "../MinimalElementType.h"
-#include "CustomTestLayouts.h"
+#include "../CustomTestLayouts.h"
 #include "CustomTestAccessors.h"
 
 template <class H, class M, class A>
diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/deduction.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/deduction.pass.cpp
index 69b2f7e398..d83299c2c5 100644
--- a/libcxx/test/std/containers/views/mdspan/mdspan/deduction.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/mdspan/deduction.pass.cpp
@@ -55,7 +55,7 @@
 #include "test_macros.h"
 
 #include "../MinimalElementType.h"
-#include "CustomTestLayouts.h"
+#include "../CustomTestLayouts.h"
 #include "CustomTestAccessors.h"
 
 template <class H, class M, class A>
diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/index_operator.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/index_operator.pass.cpp
index 9e622d1c01..5af2b3a458 100644
--- a/libcxx/test/std/containers/views/mdspan/mdspan/index_operator.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/mdspan/index_operator.pass.cpp
@@ -36,7 +36,7 @@
 #include "test_macros.h"
 
 #include "../ConvertibleToIntegral.h"
-#include "CustomTestLayouts.h"
+#include "../CustomTestLayouts.h"
 
 // Clang 16 does not support argument packs as input to operator []
 #if defined(__clang_major__) && __clang_major__ < 17
diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/move.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/move.pass.cpp
index 544a5b8aa1..4729bd171d 100644
--- a/libcxx/test/std/containers/views/mdspan/mdspan/move.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/mdspan/move.pass.cpp
@@ -21,7 +21,7 @@
 #include "test_macros.h"
 
 #include "../MinimalElementType.h"
-#include "CustomTestLayouts.h"
+#include "../CustomTestLayouts.h"
 #include "CustomTestAccessors.h"
 
 template <class H, class M, class A>
diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/properties.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/properties.pass.cpp
index c822c3ad9c..8f2bc42c8c 100644
--- a/libcxx/test/std/containers/views/mdspan/mdspan/properties.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/mdspan/properties.pass.cpp
@@ -58,7 +58,7 @@
 #include "test_macros.h"
 
 #include "../MinimalElementType.h"
-#include "CustomTestLayouts.h"
+#include "../CustomTestLayouts.h"
 
 template <class H, class M, class A>
 constexpr void test_mdspan_types(const H& handle, const M& map, const A& acc) {
diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/swap.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/swap.pass.cpp
index dd1336c7af..9b54da3252 100644
--- a/libcxx/test/std/containers/views/mdspan/mdspan/swap.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/mdspan/swap.pass.cpp
@@ -24,7 +24,7 @@
 #include "test_macros.h"
 
 #include "../MinimalElementType.h"
-#include "CustomTestLayouts.h"
+#include "../CustomTestLayouts.h"
 
 template <class MDS>
 constexpr void test_swap(MDS a, MDS b) {
diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/types.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/types.pass.cpp
index 5effea554c..e99f8a3c52 100644
--- a/libcxx/test/std/containers/views/mdspan/mdspan/types.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/mdspan/types.pass.cpp
@@ -36,7 +36,7 @@
 
 #include "../MinimalElementType.h"
 #include "CustomTestAccessors.h"
-#include "CustomTestLayouts.h"
+#include "../CustomTestLayouts.h"
 
 // Calculated expected size of an mdspan
 // Note this expectes that only default_accessor is empty
diff --git a/libcxx/test/std/depr/depr.c.headers/setjmp_h.compile.pass.cpp b/libcxx/test/std/depr/depr.c.headers/setjmp_h.compile.pass.cpp
index 7a49a85510..eaaeecbeb7 100644
--- a/libcxx/test/std/depr/depr.c.headers/setjmp_h.compile.pass.cpp
+++ b/libcxx/test/std/depr/depr.c.headers/setjmp_h.compile.pass.cpp
@@ -7,14 +7,15 @@
 //===----------------------------------------------------------------------===//
 
 // test <setjmp.h>
+//
+// Even though <setjmp.h> is not provided by libc++, we still test that
+// using it with libc++ on the search path will work.
 
 #include <setjmp.h>
 
 #include "test_macros.h"
 
-#ifndef setjmp
-#error setjmp not defined
-#endif
-
 jmp_buf jb;
 ASSERT_SAME_TYPE(void, decltype(longjmp(jb, 0)));
+
+void f() { setjmp(jb); }
diff --git a/libcxx/test/std/depr/depr.numeric.imits.has.denorm/deprecated.verify.cpp b/libcxx/test/std/depr/depr.numeric.limits.has.denorm/deprecated.verify.cpp
similarity index 100%
rename from libcxx/test/std/depr/depr.numeric.imits.has.denorm/deprecated.verify.cpp
rename to libcxx/test/std/depr/depr.numeric.limits.has.denorm/deprecated.verify.cpp
diff --git a/libcxx/test/std/experimental/simd/simd.class/simd_ctor_generator.pass.cpp b/libcxx/test/std/experimental/simd/simd.class/simd_ctor_generator.pass.cpp
new file mode 100644
index 0000000000..6349d2f1a1
--- /dev/null
+++ b/libcxx/test/std/experimental/simd/simd.class/simd_ctor_generator.pass.cpp
@@ -0,0 +1,56 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14
+
+// <experimental/simd>
+//
+// [simd.class]
+// template<class G> explicit simd(G&& gen) noexcept;
+
+#include "../test_utils.h"
+#include <experimental/simd>
+
+namespace ex = std::experimental::parallelism_v2;
+
+template <class T, std::size_t>
+struct CheckGenerateSimdCtor {
+  template <class SimdAbi>
+  void operator()() {
+    ex::simd<T, SimdAbi> origin_simd([](T i) { return i; });
+    constexpr size_t array_size = origin_simd.size();
+    std::array<T, array_size> expected_value;
+    for (size_t i = 0; i < array_size; ++i)
+      expected_value[i] = static_cast<T>(i);
+    assert_simd_values_equal<array_size>(origin_simd, expected_value);
+  }
+};
+
+template <class U, class T, class SimdAbi = ex::simd_abi::compatible<T>, class = void>
+struct has_generate_ctor : std::false_type {};
+
+template <class U, class T, class SimdAbi>
+struct has_generate_ctor<U, T, SimdAbi, std::void_t<decltype(ex::simd<T, SimdAbi>(std::declval<U>()))>>
+    : std::true_type {};
+
+template <class T, std::size_t>
+struct CheckGenerateCtorTraits {
+  template <class SimdAbi>
+  void operator()() {
+    static_assert(!has_generate_ctor<SimdAbi, T, SimdAbi>::value);
+
+    auto func = [](T i) { return i; };
+    static_assert(has_generate_ctor<decltype(func), T, SimdAbi>::value);
+  }
+};
+
+int main(int, char**) {
+  test_all_simd_abi<CheckGenerateSimdCtor>();
+  test_all_simd_abi<CheckGenerateCtorTraits>();
+  return 0;
+}
diff --git a/libcxx/test/std/input.output/file.streams/fstreams/filebuf.members/close.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/filebuf.members/close.pass.cpp
index b545041e42..e0338e6f61 100644
--- a/libcxx/test/std/input.output/file.streams/fstreams/filebuf.members/close.pass.cpp
+++ b/libcxx/test/std/input.output/file.streams/fstreams/filebuf.members/close.pass.cpp
@@ -10,6 +10,11 @@
 
 // basic_filebuf<charT,traits>* close();
 
+// This test closes an fd that belongs to a std::filebuf, and Bionic's fdsan
+// detects this and aborts the process, starting in Android R (API 30).
+// See D137129.
+// XFAIL: LIBCXX-ANDROID-FIXME && !android-device-api={{2[1-9]}}
+
 #include <fstream>
 #include <cassert>
 #if defined(__unix__)
diff --git a/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/file_size.pass.cpp b/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/file_size.pass.cpp
index fa105a74b6..3a17e89d2e 100644
--- a/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/file_size.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/file_size.pass.cpp
@@ -12,6 +12,10 @@
 // against already-released libc++'s.
 // XFAIL: stdlib=apple-libc++ && target={{.+}}-apple-macosx{{10.15|11.0}}
 
+// Starting in Android N (API 24), SELinux policy prevents the shell user from
+// creating a FIFO file.
+// XFAIL: LIBCXX-ANDROID-FIXME && !android-device-api={{21|22|23}}
+
 // <filesystem>
 
 // class directory_entry
diff --git a/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/file_type_obs.pass.cpp b/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/file_type_obs.pass.cpp
index 6591d5191d..b490f3379e 100644
--- a/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/file_type_obs.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/file_type_obs.pass.cpp
@@ -8,6 +8,10 @@
 
 // UNSUPPORTED: c++03
 
+// Starting in Android N (API 24), SELinux policy prevents the shell user from
+// creating a hard link.
+// XFAIL: LIBCXX-ANDROID-FIXME && !android-device-api={{21|22|23}}
+
 // <filesystem>
 
 // class directory_entry
diff --git a/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/hard_link_count.pass.cpp b/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/hard_link_count.pass.cpp
index e8c4742290..ebf200df40 100644
--- a/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/hard_link_count.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.directory_entry/directory_entry.obs/hard_link_count.pass.cpp
@@ -12,6 +12,10 @@
 // against already-released libc++'s.
 // XFAIL: stdlib=apple-libc++ && target={{.+}}-apple-macosx{{10.15|11.0}}
 
+// Starting in Android N (API 24), SELinux policy prevents the shell user from
+// creating a hard link.
+// XFAIL: LIBCXX-ANDROID-FIXME && !android-device-api={{21|22|23}}
+
 // <filesystem>
 
 // class directory_entry
diff --git a/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/increment.pass.cpp b/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/increment.pass.cpp
index da982918fb..244f6a295f 100644
--- a/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/increment.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/rec.dir.itr.members/increment.pass.cpp
@@ -10,6 +10,10 @@
 // UNSUPPORTED: no-filesystem
 // UNSUPPORTED: availability-filesystem-missing
 
+// On Android L, ~scoped_test_env() is unable to delete the temp dir using
+// chmod+rm because chmod is too broken.
+// XFAIL: LIBCXX-ANDROID-FIXME && android-device-api={{21|22}}
+
 // <filesystem>
 
 // class recursive_directory_iterator
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.copy/copy.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.copy/copy.pass.cpp
index afcd5ab01c..09e00450dc 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.copy/copy.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.copy/copy.pass.cpp
@@ -10,6 +10,10 @@
 // UNSUPPORTED: no-filesystem
 // UNSUPPORTED: availability-filesystem-missing
 
+// Starting in Android N (API 24), SELinux policy prevents the shell user from
+// creating a FIFO file.
+// XFAIL: LIBCXX-ANDROID-FIXME && !android-device-api={{21|22|23}}
+
 // <filesystem>
 
 // void copy(const path& from, const path& to);
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.copy_file/copy_file.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.copy_file/copy_file.pass.cpp
index 3083d41850..717f689016 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.copy_file/copy_file.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.copy_file/copy_file.pass.cpp
@@ -14,6 +14,10 @@
 // against already-released libc++'s.
 // XFAIL: stdlib=apple-libc++ && target={{.+}}-apple-macosx{{10.15|11.0}}
 
+// Starting in Android N (API 24), SELinux policy prevents the shell user from
+// creating a FIFO file.
+// XFAIL: LIBCXX-ANDROID-FIXME && !android-device-api={{21|22|23}}
+
 // <filesystem>
 
 // bool copy_file(const path& from, const path& to);
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.create_hard_link/create_hard_link.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.create_hard_link/create_hard_link.pass.cpp
index 171405979c..36a8216f1f 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.create_hard_link/create_hard_link.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.create_hard_link/create_hard_link.pass.cpp
@@ -10,6 +10,10 @@
 // UNSUPPORTED: no-filesystem
 // UNSUPPORTED: availability-filesystem-missing
 
+// Starting in Android N (API 24), SELinux policy prevents the shell user from
+// creating a hard link.
+// XFAIL: LIBCXX-ANDROID-FIXME && !android-device-api={{21|22|23}}
+
 // <filesystem>
 
 // void create_hard_link(const path& existing_symlink, const path& new_symlink);
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.equivalent/equivalent.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.equivalent/equivalent.pass.cpp
index 7fd1bf32e1..f227cec8f2 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.equivalent/equivalent.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.equivalent/equivalent.pass.cpp
@@ -10,6 +10,10 @@
 // UNSUPPORTED: no-filesystem
 // UNSUPPORTED: availability-filesystem-missing
 
+// Starting in Android N (API 24), SELinux policy prevents the shell user from
+// creating a hard link.
+// XFAIL: LIBCXX-ANDROID-FIXME && !android-device-api={{21|22|23}}
+
 // <filesystem>
 
 // bool equivalent(path const& lhs, path const& rhs);
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.hard_lk_ct/hard_link_count.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.hard_lk_ct/hard_link_count.pass.cpp
index 65a42e9793..41cbf82684 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.hard_lk_ct/hard_link_count.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.hard_lk_ct/hard_link_count.pass.cpp
@@ -10,6 +10,10 @@
 // UNSUPPORTED: no-filesystem
 // UNSUPPORTED: availability-filesystem-missing
 
+// Starting in Android N (API 24), SELinux policy prevents the shell user from
+// creating a hard link.
+// XFAIL: LIBCXX-ANDROID-FIXME && !android-device-api={{21|22|23}}
+
 // <filesystem>
 
 // uintmax_t hard_link_count(const path& p);
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_empty/is_empty.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_empty/is_empty.pass.cpp
index 74040ba102..424fd1b84b 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_empty/is_empty.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.is_empty/is_empty.pass.cpp
@@ -10,6 +10,10 @@
 // UNSUPPORTED: no-filesystem
 // UNSUPPORTED: availability-filesystem-missing
 
+// Starting in Android N (API 24), SELinux policy prevents the shell user from
+// creating a FIFO file.
+// XFAIL: LIBCXX-ANDROID-FIXME && !android-device-api={{21|22|23}}
+
 // <filesystem>
 
 // bool is_empty(path const& p);
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.permissions/permissions.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.permissions/permissions.pass.cpp
index ff1493fa44..36e91548a8 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.permissions/permissions.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.permissions/permissions.pass.cpp
@@ -10,6 +10,11 @@
 // UNSUPPORTED: no-filesystem
 // UNSUPPORTED: availability-filesystem-missing
 
+// Android's fchmodat seems broken on various OS versions -- see D140183. This
+// test probably passes on new-enough phones (not the emulator).
+// XFAIL: LIBCXX-ANDROID-FIXME && target={{i686|x86_64}}-{{.+}}-android{{.*}}
+// XFAIL: LIBCXX-ANDROID-FIXME && android-device-api={{21|22}}
+
 // <filesystem>
 
 // void permissions(const path& p, perms prms,
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.status/status.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.status/status.pass.cpp
index 6d31191144..64c05cf415 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.status/status.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.status/status.pass.cpp
@@ -10,6 +10,10 @@
 // UNSUPPORTED: no-filesystem
 // UNSUPPORTED: availability-filesystem-missing
 
+// Starting in Android N (API 24), SELinux policy prevents the shell user from
+// creating a FIFO file.
+// XFAIL: LIBCXX-ANDROID-FIXME && !android-device-api={{21|22|23}}
+
 // <filesystem>
 
 // file_status status(const path& p);
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.symlink_status/symlink_status.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.symlink_status/symlink_status.pass.cpp
index a58d35756a..81e8389fb2 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.symlink_status/symlink_status.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.symlink_status/symlink_status.pass.cpp
@@ -10,6 +10,10 @@
 // UNSUPPORTED: no-filesystem
 // UNSUPPORTED: availability-filesystem-missing
 
+// Starting in Android N (API 24), SELinux policy prevents the shell user from
+// creating a FIFO file.
+// XFAIL: LIBCXX-ANDROID-FIXME && !android-device-api={{21|22|23}}
+
 // <filesystem>
 
 // file_status symlink_status(const path& p);
diff --git a/libcxx/test/std/input.output/iostream.format/ext.manip/get_money.pass.cpp b/libcxx/test/std/input.output/iostream.format/ext.manip/get_money.pass.cpp
index 92b82e5f02..d468d7154d 100644
--- a/libcxx/test/std/input.output/iostream.format/ext.manip/get_money.pass.cpp
+++ b/libcxx/test/std/input.output/iostream.format/ext.manip/get_money.pass.cpp
@@ -10,6 +10,9 @@
 
 // template <class moneyT> T7 get_money(moneyT& mon, bool intl = false);
 
+// Bionic has minimal locale support, investigate this later.
+// XFAIL: LIBCXX-ANDROID-FIXME
+
 // REQUIRES: locale.en_US.UTF-8
 
 #include <iomanip>
diff --git a/libcxx/test/std/input.output/iostream.format/ext.manip/put_money.pass.cpp b/libcxx/test/std/input.output/iostream.format/ext.manip/put_money.pass.cpp
index 0cfbcee0c4..d0ff2c8768 100644
--- a/libcxx/test/std/input.output/iostream.format/ext.manip/put_money.pass.cpp
+++ b/libcxx/test/std/input.output/iostream.format/ext.manip/put_money.pass.cpp
@@ -10,6 +10,9 @@
 
 // template <class charT, class moneyT> T8 put_money(const moneyT& mon, bool intl = false);
 
+// Bionic has minimal locale support, investigate this later.
+// XFAIL: LIBCXX-ANDROID-FIXME
+
 // REQUIRES: locale.en_US.UTF-8
 
 #include <iomanip>
diff --git a/libcxx/test/std/input.output/iostream.format/print.fun/no_file_description.pass.cpp b/libcxx/test/std/input.output/iostream.format/print.fun/no_file_description.pass.cpp
index ee23f72846..c9297318cd 100644
--- a/libcxx/test/std/input.output/iostream.format/print.fun/no_file_description.pass.cpp
+++ b/libcxx/test/std/input.output/iostream.format/print.fun/no_file_description.pass.cpp
@@ -12,6 +12,9 @@
 // XFAIL: msvc, target={{.+}}-windows-gnu
 // XFAIL: availability-fp_to_chars-missing
 
+// fmemopen is available starting in Android M (API 23)
+// XFAIL: target={{.+}}-android{{(eabi)?(21|22)}}
+
 // <print>
 
 // The FILE returned by fmemopen does not have file descriptor.
diff --git a/libcxx/test/std/input.output/iostream.format/print.fun/print.file.pass.cpp b/libcxx/test/std/input.output/iostream.format/print.fun/print.file.pass.cpp
index 9fc2fcc4ca..4a397d3e3d 100644
--- a/libcxx/test/std/input.output/iostream.format/print.fun/print.file.pass.cpp
+++ b/libcxx/test/std/input.output/iostream.format/print.fun/print.file.pass.cpp
@@ -11,6 +11,9 @@
 
 // XFAIL: availability-fp_to_chars-missing
 
+// The error exception has no system error string.
+// XFAIL: LIBCXX-ANDROID-FIXME
+
 // <print>
 
 // template<class... Args>
diff --git a/libcxx/test/std/input.output/iostream.format/print.fun/println.file.pass.cpp b/libcxx/test/std/input.output/iostream.format/print.fun/println.file.pass.cpp
index 7c46d35215..ebdddd074f 100644
--- a/libcxx/test/std/input.output/iostream.format/print.fun/println.file.pass.cpp
+++ b/libcxx/test/std/input.output/iostream.format/print.fun/println.file.pass.cpp
@@ -11,6 +11,9 @@
 
 // XFAIL: availability-fp_to_chars-missing
 
+// The error exception has no system error string.
+// XFAIL: LIBCXX-ANDROID-FIXME
+
 // <print>
 
 // template<class... Args>
diff --git a/libcxx/test/std/input.output/iostream.format/print.fun/vprint_nonunicode.file.pass.cpp b/libcxx/test/std/input.output/iostream.format/print.fun/vprint_nonunicode.file.pass.cpp
index b851c3bb7c..9eb85f3b7b 100644
--- a/libcxx/test/std/input.output/iostream.format/print.fun/vprint_nonunicode.file.pass.cpp
+++ b/libcxx/test/std/input.output/iostream.format/print.fun/vprint_nonunicode.file.pass.cpp
@@ -16,6 +16,9 @@
 
 // XFAIL: availability-fp_to_chars-missing
 
+// The error exception has no system error string.
+// XFAIL: LIBCXX-ANDROID-FIXME
+
 // <print>
 
 // void vprint_nonunicode(FILE* stream, string_view fmt, format_args args);
diff --git a/libcxx/test/std/input.output/iostream.format/print.fun/vprint_unicode.file.pass.cpp b/libcxx/test/std/input.output/iostream.format/print.fun/vprint_unicode.file.pass.cpp
index 63b33f2db1..28379b9db5 100644
--- a/libcxx/test/std/input.output/iostream.format/print.fun/vprint_unicode.file.pass.cpp
+++ b/libcxx/test/std/input.output/iostream.format/print.fun/vprint_unicode.file.pass.cpp
@@ -17,6 +17,9 @@
 
 // XFAIL: availability-fp_to_chars-missing
 
+// The error exception has no system error string.
+// XFAIL: LIBCXX-ANDROID-FIXME
+
 // <print>
 
 // void vprint_unicode(FILE* stream, string_view fmt, format_args args);
diff --git a/libcxx/test/std/input.output/iostream.objects/narrow.stream.objects/cin.sh.cpp b/libcxx/test/std/input.output/iostream.objects/narrow.stream.objects/cin.sh.cpp
index b39cd57ab2..28ea650e58 100644
--- a/libcxx/test/std/input.output/iostream.objects/narrow.stream.objects/cin.sh.cpp
+++ b/libcxx/test/std/input.output/iostream.objects/narrow.stream.objects/cin.sh.cpp
@@ -9,6 +9,10 @@
 // TODO: Investigate
 // UNSUPPORTED: LIBCXX-AIX-FIXME
 
+// This test hangs on Android devices that lack shell_v2, which was added in
+// Android N (API 24).
+// UNSUPPORTED: LIBCXX-ANDROID-FIXME && android-device-api={{2[1-3]}}
+
 // <iostream>
 
 // istream cin;
diff --git a/libcxx/test/std/input.output/iostream.objects/wide.stream.objects/wcin-imbue.sh.cpp b/libcxx/test/std/input.output/iostream.objects/wide.stream.objects/wcin-imbue.sh.cpp
index 6bdffc93f3..027e4fa936 100644
--- a/libcxx/test/std/input.output/iostream.objects/wide.stream.objects/wcin-imbue.sh.cpp
+++ b/libcxx/test/std/input.output/iostream.objects/wide.stream.objects/wcin-imbue.sh.cpp
@@ -9,6 +9,10 @@
 // TODO: Investigate
 // UNSUPPORTED: LIBCXX-AIX-FIXME
 
+// This test hangs on Android devices that lack shell_v2, which was added in
+// Android N (API 24).
+// UNSUPPORTED: LIBCXX-ANDROID-FIXME && android-device-api={{2[1-3]}}
+
 // <iostream>
 
 // wistream wcin;
diff --git a/libcxx/test/std/input.output/iostream.objects/wide.stream.objects/wcin.sh.cpp b/libcxx/test/std/input.output/iostream.objects/wide.stream.objects/wcin.sh.cpp
index c0f2c3258b..30972da2f0 100644
--- a/libcxx/test/std/input.output/iostream.objects/wide.stream.objects/wcin.sh.cpp
+++ b/libcxx/test/std/input.output/iostream.objects/wide.stream.objects/wcin.sh.cpp
@@ -9,6 +9,10 @@
 // TODO: Investigate
 // UNSUPPORTED: LIBCXX-AIX-FIXME
 
+// This test hangs on Android devices that lack shell_v2, which was added in
+// Android N (API 24).
+// UNSUPPORTED: LIBCXX-ANDROID-FIXME && android-device-api={{2[1-3]}}
+
 // <iostream>
 
 // wistream wcin;
diff --git a/libcxx/test/std/input.output/stream.buffers/streambuf/streambuf.protected/streambuf.put.area/pbump2gig.pass.cpp b/libcxx/test/std/input.output/stream.buffers/streambuf/streambuf.protected/streambuf.put.area/pbump2gig.pass.cpp
index 4c63cd1ab3..6c747964e3 100644
--- a/libcxx/test/std/input.output/stream.buffers/streambuf/streambuf.protected/streambuf.put.area/pbump2gig.pass.cpp
+++ b/libcxx/test/std/input.output/stream.buffers/streambuf/streambuf.protected/streambuf.put.area/pbump2gig.pass.cpp
@@ -22,9 +22,8 @@
 // UNSUPPORTED: no-exceptions
 
 // Android devices frequently don't have enough memory to run this test. Rather
-// than throw std::bad_alloc, exhausting memory tends to trigger the OOM Killer
-// and/or crash the device (killing adb, rebooting it, etc).
-// UNSUPPORTED: android
+// than throw std::bad_alloc, exhausting memory triggers the OOM Killer.
+// UNSUPPORTED: LIBCXX-ANDROID-FIXME
 
 #include <sstream>
 #include <cassert>
diff --git a/libcxx/test/std/input.output/string.streams/istringstream/istringstream.members/str.allocator_propagation.pass.cpp b/libcxx/test/std/input.output/string.streams/istringstream/istringstream.members/str.allocator_propagation.pass.cpp
new file mode 100644
index 0000000000..ab41103aa8
--- /dev/null
+++ b/libcxx/test/std/input.output/string.streams/istringstream/istringstream.members/str.allocator_propagation.pass.cpp
@@ -0,0 +1,144 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// TODO: Change to XFAIL once https://github.com/llvm/llvm-project/issues/40340 is fixed
+// UNSUPPORTED: availability-pmr-missing
+
+// This test ensures that we properly propagate allocators from istringstream's
+// inner string object to the new string returned from .str().
+// `str() const&` is specified to preserve the allocator (not copy the string).
+// `str() &&` isn't specified, but should preserve the allocator (move the string).
+
+#include <cassert>
+#include <memory>
+#include <memory_resource>
+#include <sstream>
+#include <string>
+#include <string_view>
+#include <type_traits>
+
+#include "make_string.h"
+#include "test_allocator.h"
+#include "test_macros.h"
+
+template <class CharT>
+void test_soccc_behavior() {
+  using Alloc = SocccAllocator<CharT>;
+  using SS    = std::basic_istringstream<CharT, std::char_traits<CharT>, Alloc>;
+  using S     = std::basic_string<CharT, std::char_traits<CharT>, Alloc>;
+  {
+    SS ss = SS(std::ios_base::in, Alloc(10));
+
+    // [stringbuf.members]/6 specifies that the allocator is copied,
+    // not select_on_container_copy_construction'ed.
+    //
+    S copied = ss.str();
+    assert(copied.get_allocator().count_ == 10);
+    assert(ss.rdbuf()->get_allocator().count_ == 10);
+    assert(copied.empty());
+
+    // sanity-check that SOCCC does in fact work
+    assert(S(copied).get_allocator().count_ == 11);
+
+    // [stringbuf.members]/10 doesn't specify the allocator to use,
+    // but copying the allocator as-if-by moving the string makes sense.
+    //
+    S moved = std::move(ss).str();
+    assert(moved.get_allocator().count_ == 10);
+    assert(ss.rdbuf()->get_allocator().count_ == 10);
+    assert(moved.empty());
+  }
+}
+
+template <class CharT,
+          class Base = std::basic_stringbuf<CharT, std::char_traits<CharT>, std::pmr::polymorphic_allocator<CharT>>>
+struct StringBuf : Base {
+  explicit StringBuf(std::pmr::memory_resource* mr) : Base(std::ios_base::in, mr) {}
+  void public_setg(int a, int b, int c) {
+    CharT* p = this->eback();
+    assert(this->view().data() == p);
+    this->setg(p + a, p + b, p + c);
+    assert(this->eback() == p + a);
+    assert(this->view().data() == p + a);
+  }
+};
+
+template <class CharT>
+void test_allocation_is_pilfered() {
+  using SS = std::basic_istringstream<CharT, std::char_traits<CharT>, std::pmr::polymorphic_allocator<CharT>>;
+  using S  = std::pmr::basic_string<CharT>;
+  alignas(void*) char buf[80 * sizeof(CharT)];
+  const CharT* initial =
+      MAKE_CSTRING(CharT, "a very long string that exceeds the small string optimization buffer length");
+  {
+    std::pmr::set_default_resource(std::pmr::null_memory_resource());
+    auto mr1 = std::pmr::monotonic_buffer_resource(buf, sizeof(buf), std::pmr::null_memory_resource());
+    SS ss    = SS(S(initial, &mr1));
+    S s      = std::move(ss).str();
+    assert(s == initial);
+  }
+  {
+    // Try moving-out-of a stringbuf whose view() is not the entire string.
+    // This is libc++'s behavior; libstdc++ doesn't allow such stringbufs to be created.
+    //
+    std::pmr::set_default_resource(std::pmr::null_memory_resource());
+    auto mr1 = std::pmr::monotonic_buffer_resource(buf, sizeof(buf), std::pmr::null_memory_resource());
+    auto src = StringBuf<CharT>(&mr1);
+    src.str(S(initial, &mr1));
+    src.public_setg(2, 6, 40);
+    SS ss(std::ios_base::in, &mr1);
+    *ss.rdbuf() = std::move(src);
+    LIBCPP_ASSERT(ss.view() == std::basic_string_view<CharT>(initial).substr(2, 38));
+    S s = std::move(ss).str();
+    LIBCPP_ASSERT(s == std::basic_string_view<CharT>(initial).substr(2, 38));
+  }
+}
+
+template <class CharT>
+void test_no_foreign_allocations() {
+  using SS = std::basic_istringstream<CharT, std::char_traits<CharT>, std::pmr::polymorphic_allocator<CharT>>;
+  using S  = std::pmr::basic_string<CharT>;
+  const CharT* initial =
+      MAKE_CSTRING(CharT, "a very long string that exceeds the small string optimization buffer length");
+  {
+    std::pmr::set_default_resource(std::pmr::null_memory_resource());
+    auto mr1 = std::pmr::monotonic_buffer_resource(std::pmr::new_delete_resource());
+    auto ss  = SS(S(initial, &mr1));
+    assert(ss.rdbuf()->get_allocator().resource() == &mr1);
+
+    // [stringbuf.members]/6 specifies that the result of `str() const &`
+    // does NOT use the default allocator; it uses the original allocator.
+    //
+    S copied = ss.str();
+    assert(copied.get_allocator().resource() == &mr1);
+    assert(ss.rdbuf()->get_allocator().resource() == &mr1);
+    assert(copied == initial);
+
+    // [stringbuf.members]/10 doesn't specify the allocator to use,
+    // but copying the allocator as-if-by moving the string makes sense.
+    //
+    S moved = std::move(ss).str();
+    assert(moved.get_allocator().resource() == &mr1);
+    assert(ss.rdbuf()->get_allocator().resource() == &mr1);
+    assert(moved == initial);
+  }
+}
+
+int main(int, char**) {
+  test_soccc_behavior<char>();
+  test_allocation_is_pilfered<char>();
+  test_no_foreign_allocations<char>();
+#ifndef TEST_HAS_NO_WIDE_CHARACTERS
+  test_soccc_behavior<wchar_t>();
+  test_allocation_is_pilfered<wchar_t>();
+  test_no_foreign_allocations<wchar_t>();
+#endif
+
+  return 0;
+}
diff --git a/libcxx/test/std/input.output/string.streams/istringstream/istringstream.members/str.move.pass.cpp b/libcxx/test/std/input.output/string.streams/istringstream/istringstream.members/str.move.pass.cpp
index 546f82166a..0bd076af5e 100644
--- a/libcxx/test/std/input.output/string.streams/istringstream/istringstream.members/str.move.pass.cpp
+++ b/libcxx/test/std/input.output/string.streams/istringstream/istringstream.members/str.move.pass.cpp
@@ -37,6 +37,14 @@ static void test() {
     assert(s.empty());
     assert(ss.view().empty());
   }
+  {
+    std::basic_istringstream<CharT> ss(
+        STR("a very long string that exceeds the small string optimization buffer length"));
+    const CharT* p             = ss.view().data();
+    std::basic_string<CharT> s = std::move(ss).str();
+    assert(s.data() == p); // the allocation was pilfered
+    assert(ss.view().empty());
+  }
 }
 
 int main(int, char**) {
diff --git a/libcxx/test/std/input.output/string.streams/ostringstream/ostringstream.members/str.allocator_propagation.pass.cpp b/libcxx/test/std/input.output/string.streams/ostringstream/ostringstream.members/str.allocator_propagation.pass.cpp
new file mode 100644
index 0000000000..a5ee2afab1
--- /dev/null
+++ b/libcxx/test/std/input.output/string.streams/ostringstream/ostringstream.members/str.allocator_propagation.pass.cpp
@@ -0,0 +1,115 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// TODO: Change to XFAIL once https://github.com/llvm/llvm-project/issues/40340 is fixed
+// UNSUPPORTED: availability-pmr-missing
+
+// This test ensures that we properly propagate allocators from ostringstream's
+// inner string object to the new string returned from .str().
+// `str() const&` is specified to preserve the allocator (not copy the string).
+// `str() &&` isn't specified, but should preserve the allocator (move the string).
+
+#include <cassert>
+#include <memory>
+#include <memory_resource>
+#include <sstream>
+#include <string>
+#include <type_traits>
+
+#include "make_string.h"
+#include "test_allocator.h"
+#include "test_macros.h"
+
+template <class CharT>
+void test_soccc_behavior() {
+  using Alloc = SocccAllocator<CharT>;
+  using SS    = std::basic_ostringstream<CharT, std::char_traits<CharT>, Alloc>;
+  using S     = std::basic_string<CharT, std::char_traits<CharT>, Alloc>;
+  {
+    SS ss = SS(std::ios_base::out, Alloc(10));
+
+    // [stringbuf.members]/6 specifies that the allocator is copied,
+    // not select_on_container_copy_construction'ed.
+    //
+    S copied = ss.str();
+    assert(copied.get_allocator().count_ == 10);
+    assert(ss.rdbuf()->get_allocator().count_ == 10);
+    assert(copied.empty());
+
+    // sanity-check that SOCCC does in fact work
+    assert(S(copied).get_allocator().count_ == 11);
+
+    // [stringbuf.members]/10 doesn't specify the allocator to use,
+    // but copying the allocator as-if-by moving the string makes sense.
+    //
+    S moved = std::move(ss).str();
+    assert(moved.get_allocator().count_ == 10);
+    assert(ss.rdbuf()->get_allocator().count_ == 10);
+    assert(moved.empty());
+  }
+}
+
+template <class CharT>
+void test_allocation_is_pilfered() {
+  using SS = std::basic_ostringstream<CharT, std::char_traits<CharT>, std::pmr::polymorphic_allocator<CharT>>;
+  using S  = std::pmr::basic_string<CharT>;
+  alignas(void*) char buf[80 * sizeof(CharT)];
+  const CharT* initial =
+      MAKE_CSTRING(CharT, "a very long string that exceeds the small string optimization buffer length");
+  {
+    std::pmr::set_default_resource(std::pmr::null_memory_resource());
+    auto mr1 = std::pmr::monotonic_buffer_resource(buf, sizeof(buf), std::pmr::null_memory_resource());
+    SS ss    = SS(S(initial, &mr1));
+    S s      = std::move(ss).str();
+    assert(s == initial);
+  }
+}
+
+template <class CharT>
+void test_no_foreign_allocations() {
+  using SS = std::basic_ostringstream<CharT, std::char_traits<CharT>, std::pmr::polymorphic_allocator<CharT>>;
+  using S  = std::pmr::basic_string<CharT>;
+  const CharT* initial =
+      MAKE_CSTRING(CharT, "a very long string that exceeds the small string optimization buffer length");
+  {
+    std::pmr::set_default_resource(std::pmr::null_memory_resource());
+    auto mr1 = std::pmr::monotonic_buffer_resource(std::pmr::new_delete_resource());
+    auto ss  = SS(S(initial, &mr1));
+    assert(ss.rdbuf()->get_allocator().resource() == &mr1);
+
+    // [stringbuf.members]/6 specifies that the result of `str() const &`
+    // does NOT use the default allocator; it uses the original allocator.
+    //
+    S copied = ss.str();
+    assert(copied.get_allocator().resource() == &mr1);
+    assert(ss.rdbuf()->get_allocator().resource() == &mr1);
+    assert(copied == initial);
+
+    // [stringbuf.members]/10 doesn't specify the allocator to use,
+    // but copying the allocator as-if-by moving the string makes sense.
+    //
+    S moved = std::move(ss).str();
+    assert(moved.get_allocator().resource() == &mr1);
+    assert(ss.rdbuf()->get_allocator().resource() == &mr1);
+    assert(moved == initial);
+  }
+}
+
+int main(int, char**) {
+  test_soccc_behavior<char>();
+  test_allocation_is_pilfered<char>();
+  test_no_foreign_allocations<char>();
+#ifndef TEST_HAS_NO_WIDE_CHARACTERS
+  test_soccc_behavior<wchar_t>();
+  test_allocation_is_pilfered<wchar_t>();
+  test_no_foreign_allocations<wchar_t>();
+#endif
+
+  return 0;
+}
diff --git a/libcxx/test/std/input.output/string.streams/ostringstream/ostringstream.members/str.move.pass.cpp b/libcxx/test/std/input.output/string.streams/ostringstream/ostringstream.members/str.move.pass.cpp
index 57f2384bae..0e1c06f191 100644
--- a/libcxx/test/std/input.output/string.streams/ostringstream/ostringstream.members/str.move.pass.cpp
+++ b/libcxx/test/std/input.output/string.streams/ostringstream/ostringstream.members/str.move.pass.cpp
@@ -37,6 +37,14 @@ static void test() {
     assert(s.empty());
     assert(ss.view().empty());
   }
+  {
+    std::basic_ostringstream<CharT> ss(
+        STR("a very long string that exceeds the small string optimization buffer length"));
+    const CharT* p             = ss.view().data();
+    std::basic_string<CharT> s = std::move(ss).str();
+    assert(s.data() == p); // the allocation was pilfered
+    assert(ss.view().empty());
+  }
 }
 
 int main(int, char**) {
diff --git a/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.assign/move.pass.cpp b/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.assign/move.pass.cpp
index 3ae562e17e..e1c71829aa 100644
--- a/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.assign/move.pass.cpp
+++ b/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.assign/move.pass.cpp
@@ -6,6 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+// UNSUPPORTED: c++03
+
 // <sstream>
 
 // template <class charT, class traits = char_traits<charT>, class Allocator = allocator<charT> >
@@ -15,49 +17,75 @@
 
 #include <sstream>
 #include <cassert>
+#include <utility>
 
+#include "make_string.h"
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    {
-        std::stringbuf buf1("testing");
-        std::stringbuf buf;
-        buf = std::move(buf1);
-        assert(buf.str() == "testing");
-    }
+#define STR(S) MAKE_STRING(CharT, S)
+
+template <class CharT>
+struct test_stringbuf : std::basic_stringbuf<CharT> {
+  using std::basic_stringbuf<CharT>::basic_stringbuf;
+
+  // Checks the following requirement after being moved from:
+  //    The six pointers of std::basic_streambuf in *this are guaranteed to be different
+  //    from the corresponding pointers in the moved-from rhs unless null.
+  void check_different_pointers(test_stringbuf<CharT> const& other) const {
+    assert(this->eback() == nullptr || this->eback() != other.eback());
+    assert(this->gptr() == nullptr || this->gptr() != other.gptr());
+    assert(this->egptr() == nullptr || this->egptr() != other.egptr());
+    assert(this->pbase() == nullptr || this->pbase() != other.pbase());
+    assert(this->pptr() == nullptr || this->pptr() != other.pptr());
+    assert(this->epptr() == nullptr || this->epptr() != other.epptr());
+  }
+};
+
+template <class CharT>
+void test() {
+  std::basic_string<CharT> strings[] = {STR(""), STR("short"), STR("loooooooooooooooooooong")};
+  for (std::basic_string<CharT> const& s : strings) {
     {
-        std::stringbuf buf1("testing", std::ios_base::in);
-        std::stringbuf buf;
-        buf = std::move(buf1);
-        assert(buf.str() == "testing");
+      test_stringbuf<CharT> buf1(s);
+      test_stringbuf<CharT> buf;
+      buf = std::move(buf1);
+      assert(buf.str() == s);
+      buf.check_different_pointers(buf1);
     }
     {
-        std::stringbuf buf1("testing", std::ios_base::out);
-        std::stringbuf buf;
-        buf = std::move(buf1);
-        assert(buf.str() == "testing");
+      test_stringbuf<CharT> buf1(s, std::ios_base::in);
+      test_stringbuf<CharT> buf;
+      buf = std::move(buf1);
+      assert(buf.str() == s);
+      buf.check_different_pointers(buf1);
     }
-#ifndef TEST_HAS_NO_WIDE_CHARACTERS
     {
-        std::wstringbuf buf1(L"testing");
-        std::wstringbuf buf;
-        buf = std::move(buf1);
-        assert(buf.str() == L"testing");
+      test_stringbuf<CharT> buf1(s, std::ios_base::out);
+      test_stringbuf<CharT> buf;
+      buf = std::move(buf1);
+      assert(buf.str() == s);
+      buf.check_different_pointers(buf1);
     }
     {
-        std::wstringbuf buf1(L"testing", std::ios_base::in);
-        std::wstringbuf buf;
-        buf = std::move(buf1);
-        assert(buf.str() == L"testing");
+      test_stringbuf<CharT> buf1;
+      test_stringbuf<CharT> buf;
+      buf = std::move(buf1);
+      buf.check_different_pointers(buf1);
     }
+    // Use the assignment operator on an actual std::stringbuf, not test_stringbuf
     {
-        std::wstringbuf buf1(L"testing", std::ios_base::out);
-        std::wstringbuf buf;
-        buf = std::move(buf1);
-        assert(buf.str() == L"testing");
+      std::basic_stringbuf<CharT> buf1(s);
+      std::basic_stringbuf<CharT> buf;
+      buf = std::move(buf1);
+      assert(buf.str() == s);
     }
-#endif // TEST_HAS_NO_WIDE_CHARACTERS
+  }
+}
 
+int main(int, char**) {
+  test<char>();
+#ifndef TEST_HAS_NO_WIDE_CHARACTERS
+  test<wchar_t>();
+#endif
   return 0;
 }
diff --git a/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.cons/move.alloc.pass.cpp b/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.cons/move.alloc.pass.cpp
index ac90d42337..c652fa1012 100644
--- a/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.cons/move.alloc.pass.cpp
+++ b/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.cons/move.alloc.pass.cpp
@@ -17,6 +17,7 @@
 
 #include <sstream>
 #include <cassert>
+#include <utility>
 
 #include "make_string.h"
 #include "test_allocator.h"
@@ -26,12 +27,73 @@
 #define SV(S) MAKE_STRING_VIEW(CharT, S)
 
 template <class CharT>
-static void test() {
-  std::basic_stringbuf<CharT, std::char_traits<CharT>, test_allocator<CharT>> buf1(STR("testing"));
-  const test_allocator<CharT> a(2);
-  const std::basic_stringbuf<CharT, std::char_traits<CharT>, test_allocator<CharT>> buf(std::move(buf1), a);
-  assert(buf.get_allocator() == a);
-  assert(buf.view() == SV("testing"));
+struct test_stringbuf : std::basic_stringbuf<CharT, std::char_traits<CharT>, test_allocator<CharT>> {
+  using std::basic_stringbuf<CharT, std::char_traits<CharT>, test_allocator<CharT>>::basic_stringbuf;
+
+  // Checks the following requirement after being moved from:
+  //    The six pointers of std::basic_streambuf in *this are guaranteed to be different
+  //    from the corresponding pointers in the moved-from rhs unless null.
+  void check_different_pointers(test_stringbuf<CharT> const& other) const {
+    assert(this->eback() == nullptr || this->eback() != other.eback());
+    assert(this->gptr() == nullptr || this->gptr() != other.gptr());
+    assert(this->egptr() == nullptr || this->egptr() != other.egptr());
+    assert(this->pbase() == nullptr || this->pbase() != other.pbase());
+    assert(this->pptr() == nullptr || this->pptr() != other.pptr());
+    assert(this->epptr() == nullptr || this->epptr() != other.epptr());
+  }
+};
+
+template <class CharT>
+void test() {
+  std::basic_string<CharT> strings[] = {STR(""), STR("short"), STR("loooooooooooooooooooong")};
+  for (std::basic_string<CharT> const& s : strings) {
+    using StringBuf = std::basic_stringbuf<CharT, std::char_traits<CharT>, test_allocator<CharT>>;
+    {
+      test_stringbuf<CharT> buf1(s);
+      const test_allocator<CharT> a(2);
+      test_stringbuf<CharT> buf(std::move(static_cast<StringBuf&>(buf1)), a);
+      assert(buf.get_allocator() == a);
+      assert(buf.view() == s);
+      assert(buf1.view().empty());
+      buf.check_different_pointers(buf1);
+    }
+    {
+      test_stringbuf<CharT> buf1(s, std::ios_base::in);
+      const test_allocator<CharT> a(2);
+      test_stringbuf<CharT> buf(std::move(static_cast<StringBuf&>(buf1)), a);
+      assert(buf.get_allocator() == a);
+      assert(buf.view() == s);
+      assert(buf1.view().empty());
+      buf.check_different_pointers(buf1);
+    }
+    {
+      test_stringbuf<CharT> buf1(s, std::ios_base::out);
+      const test_allocator<CharT> a(2);
+      test_stringbuf<CharT> buf(std::move(static_cast<StringBuf&>(buf1)), a);
+      assert(buf.get_allocator() == a);
+      assert(buf.view() == s);
+      assert(buf1.view().empty());
+      buf.check_different_pointers(buf1);
+    }
+    {
+      test_stringbuf<CharT> buf1;
+      const test_allocator<CharT> a(2);
+      test_stringbuf<CharT> buf(std::move(static_cast<StringBuf&>(buf1)), a);
+      assert(buf.get_allocator() == a);
+      assert(buf.view().empty());
+      assert(buf1.view().empty());
+      buf.check_different_pointers(buf1);
+    }
+    // Use the constructor from an actual std::stringbuf, not test_stringbuf
+    {
+      StringBuf buf1(s);
+      const test_allocator<CharT> a(2);
+      StringBuf buf(std::move(buf1), a);
+      assert(buf.get_allocator() == a);
+      assert(buf.view() == s);
+      assert(buf1.view().empty());
+    }
+  }
 }
 
 int main(int, char**) {
diff --git a/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.cons/move.pass.cpp b/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.cons/move.pass.cpp
index c4846c01d4..65767a95ad 100644
--- a/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.cons/move.pass.cpp
+++ b/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.cons/move.pass.cpp
@@ -6,6 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+// UNSUPPORTED: c++03
+
 // <sstream>
 
 // template <class charT, class traits = char_traits<charT>, class Allocator = allocator<charT> >
@@ -15,43 +17,79 @@
 
 #include <sstream>
 #include <cassert>
+#include <utility>
 
+#include "make_string.h"
 #include "test_macros.h"
 
-int main(int, char**)
-{
-    {
-        std::stringbuf buf1("testing");
-        std::stringbuf buf(std::move(buf1));
-        assert(buf.str() == "testing");
-    }
+#define STR(S) MAKE_STRING(CharT, S)
+
+template <class CharT>
+struct test_stringbuf : std::basic_stringbuf<CharT> {
+  using std::basic_stringbuf<CharT>::basic_stringbuf;
+
+  test_stringbuf(std::basic_stringbuf<CharT>&& other) : std::basic_stringbuf<CharT>(std::move(other)) {}
+
+  // Checks the following requirement after being moved from:
+  //    The six pointers of std::basic_streambuf in *this are guaranteed to be different
+  //    from the corresponding pointers in the moved-from rhs unless null.
+  void check_different_pointers(test_stringbuf<CharT> const& other) const {
+    assert(this->eback() == nullptr || this->eback() != other.eback());
+    assert(this->gptr() == nullptr || this->gptr() != other.gptr());
+    assert(this->egptr() == nullptr || this->egptr() != other.egptr());
+    assert(this->pbase() == nullptr || this->pbase() != other.pbase());
+    assert(this->pptr() == nullptr || this->pptr() != other.pptr());
+    assert(this->epptr() == nullptr || this->epptr() != other.epptr());
+  }
+};
+
+template <class CharT>
+void test() {
+  std::basic_string<CharT> strings[] = {STR(""), STR("short"), STR("loooooooooooooooooooong")};
+  for (std::basic_string<CharT> const& s : strings) {
+    using StringBuf = std::basic_stringbuf<CharT>;
     {
-        std::stringbuf buf1("testing", std::ios_base::in);
-        std::stringbuf buf(std::move(buf1));
-        assert(buf.str() == "testing");
+      test_stringbuf<CharT> buf1(s);
+      test_stringbuf<CharT> buf(std::move(static_cast<StringBuf&>(buf1)));
+      assert(buf.str() == s);
+      assert(buf1.str().empty());
+      buf.check_different_pointers(buf1);
     }
     {
-        std::stringbuf buf1("testing", std::ios_base::out);
-        std::stringbuf buf(std::move(buf1));
-        assert(buf.str() == "testing");
+      test_stringbuf<CharT> buf1(s, std::ios_base::in);
+      test_stringbuf<CharT> buf(std::move(static_cast<StringBuf&>(buf1)));
+      assert(buf.str() == s);
+      assert(buf1.str().empty());
+      buf.check_different_pointers(buf1);
     }
-#ifndef TEST_HAS_NO_WIDE_CHARACTERS
     {
-        std::wstringbuf buf1(L"testing");
-        std::wstringbuf buf(std::move(buf1));
-        assert(buf.str() == L"testing");
+      test_stringbuf<CharT> buf1(s, std::ios_base::out);
+      test_stringbuf<CharT> buf(std::move(static_cast<StringBuf&>(buf1)));
+      assert(buf.str() == s);
+      assert(buf1.str().empty());
+      buf.check_different_pointers(buf1);
     }
     {
-        std::wstringbuf buf1(L"testing", std::ios_base::in);
-        std::wstringbuf buf(std::move(buf1));
-        assert(buf.str() == L"testing");
+      test_stringbuf<CharT> buf1;
+      test_stringbuf<CharT> buf(std::move(static_cast<StringBuf&>(buf1)));
+      assert(buf.str().empty());
+      assert(buf1.str().empty());
+      buf.check_different_pointers(buf1);
     }
+    // Use the constructor from an actual std::stringbuf, not test_stringbuf
     {
-        std::wstringbuf buf1(L"testing", std::ios_base::out);
-        std::wstringbuf buf(std::move(buf1));
-        assert(buf.str() == L"testing");
+      StringBuf buf1(s);
+      StringBuf buf(std::move(buf1));
+      assert(buf.str() == s);
+      assert(buf1.str().empty());
     }
-#endif
+  }
+}
 
+int main(int, char**) {
+  test<char>();
+#ifndef TEST_HAS_NO_WIDE_CHARACTERS
+  test<wchar_t>();
+#endif
   return 0;
 }
diff --git a/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.members/str.move.pass.cpp b/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.members/str.move.pass.cpp
index 0f0f540a9c..9d75bf938a 100644
--- a/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.members/str.move.pass.cpp
+++ b/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.members/str.move.pass.cpp
@@ -37,6 +37,48 @@ static void test() {
     assert(s.empty());
     assert(buf.view().empty());
   }
+  {
+    std::basic_stringbuf<CharT> buf(STR("a very long string that exceeds the small string optimization buffer length"));
+    const CharT* p             = buf.view().data();
+    std::basic_string<CharT> s = std::move(buf).str();
+    assert(s.data() == p); // the allocation was pilfered
+    assert(buf.view().empty());
+  }
+}
+
+struct StringBuf : std::stringbuf {
+  using basic_stringbuf::basic_stringbuf;
+  void public_setg(int a, int b, int c) {
+    char* p = eback();
+    this->setg(p + a, p + b, p + c);
+  }
+};
+
+static void test_altered_sequence_pointers() {
+  {
+    auto src = StringBuf("hello world", std::ios_base::in);
+    src.public_setg(4, 6, 9);
+    std::stringbuf dest;
+    dest             = std::move(src);
+    std::string view = std::string(dest.view());
+    std::string str  = std::move(dest).str();
+    assert(view == str);
+    LIBCPP_ASSERT(str == "o wor");
+    assert(dest.str().empty());
+    assert(dest.view().empty());
+  }
+  {
+    auto src = StringBuf("hello world", std::ios_base::in);
+    src.public_setg(4, 6, 9);
+    std::stringbuf dest;
+    dest.swap(src);
+    std::string view = std::string(dest.view());
+    std::string str  = std::move(dest).str();
+    assert(view == str);
+    LIBCPP_ASSERT(str == "o wor");
+    assert(dest.str().empty());
+    assert(dest.view().empty());
+  }
 }
 
 int main(int, char**) {
@@ -44,5 +86,6 @@ int main(int, char**) {
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
   test<wchar_t>();
 #endif
+  test_altered_sequence_pointers();
   return 0;
 }
diff --git a/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.members/str.pass.cpp b/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.members/str.pass.cpp
index 18a2337f6b..8cd3840b68 100644
--- a/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.members/str.pass.cpp
+++ b/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.members/str.pass.cpp
@@ -14,18 +14,51 @@
 // void str(const basic_string<charT,traits,Allocator>& s);
 
 #include <sstream>
+#include <string>
 #include <cassert>
 
 #include "test_macros.h"
 
+struct StringBuf : std::stringbuf {
+  explicit StringBuf(const char* s, std::ios_base::openmode mode) : basic_stringbuf(s, mode) {}
+  void public_setg(int a, int b, int c) {
+    char* p = eback();
+    this->setg(p + a, p + b, p + c);
+  }
+};
+
+static void test_altered_sequence_pointers() {
+  {
+    StringBuf src("hello world", std::ios_base::in);
+    src.public_setg(4, 6, 9);
+    std::stringbuf dest;
+    dest            = std::move(src);
+    std::string str = dest.str();
+    assert(5 <= str.size() && str.size() <= 11);
+    LIBCPP_ASSERT(str == "o wor");
+    LIBCPP_ASSERT(dest.str() == "o wor");
+  }
+  {
+    StringBuf src("hello world", std::ios_base::in);
+    src.public_setg(4, 6, 9);
+    std::stringbuf dest;
+    dest.swap(src);
+    std::string str = dest.str();
+    assert(5 <= str.size() && str.size() <= 11);
+    LIBCPP_ASSERT(str == "o wor");
+    LIBCPP_ASSERT(dest.str() == "o wor");
+  }
+}
+
 int main(int, char**)
 {
-    {
-        std::stringbuf buf("testing");
-        assert(buf.str() == "testing");
-        buf.str("another test");
-        assert(buf.str() == "another test");
-    }
+  test_altered_sequence_pointers();
+  {
+    std::stringbuf buf("testing");
+    assert(buf.str() == "testing");
+    buf.str("another test");
+    assert(buf.str() == "another test");
+  }
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
     {
         std::wstringbuf buf(L"testing");
diff --git a/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.members/view.pass.cpp b/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.members/view.pass.cpp
index 4aa2e4ab23..67ff506bb9 100644
--- a/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.members/view.pass.cpp
+++ b/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.members/view.pass.cpp
@@ -50,10 +50,38 @@ static void test() {
   static_assert(std::is_same_v<decltype(tbuf.view()), std::basic_string_view<CharT, my_char_traits<CharT>>>);
 }
 
+struct StringBuf : std::stringbuf {
+  using basic_stringbuf::basic_stringbuf;
+  void public_setg(int a, int b, int c) {
+    char* p = eback();
+    this->setg(p + a, p + b, p + c);
+  }
+};
+
+static void test_altered_sequence_pointers() {
+  {
+    auto src = StringBuf("hello world", std::ios_base::in);
+    src.public_setg(4, 6, 9);
+    std::stringbuf dest;
+    dest = std::move(src);
+    assert(dest.view() == dest.str());
+    LIBCPP_ASSERT(dest.view() == "o wor");
+  }
+  {
+    auto src = StringBuf("hello world", std::ios_base::in);
+    src.public_setg(4, 6, 9);
+    std::stringbuf dest;
+    dest.swap(src);
+    assert(dest.view() == dest.str());
+    LIBCPP_ASSERT(dest.view() == "o wor");
+  }
+}
+
 int main(int, char**) {
   test<char>();
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
   test<wchar_t>();
 #endif
+  test_altered_sequence_pointers();
   return 0;
 }
diff --git a/libcxx/test/std/input.output/string.streams/stringstream/stringstream.members/gcount.pass.cpp b/libcxx/test/std/input.output/string.streams/stringstream/stringstream.members/gcount.pass.cpp
index 3a5edac6c5..a9079dc63b 100644
--- a/libcxx/test/std/input.output/string.streams/stringstream/stringstream.members/gcount.pass.cpp
+++ b/libcxx/test/std/input.output/string.streams/stringstream/stringstream.members/gcount.pass.cpp
@@ -7,6 +7,11 @@
 //===----------------------------------------------------------------------===//
 
 // UNSUPPORTED: 32-bit-pointer
+// REQUIRES: large_tests
+
+// Android devices frequently don't have enough memory to run this test. Rather
+// than throw std::bad_alloc, exhausting memory triggers the OOM Killer.
+// UNSUPPORTED: LIBCXX-ANDROID-FIXME
 
 // Test that tellp() does not break the stringstream after INT_MAX, due to use
 // of pbump() that accept int.
diff --git a/libcxx/test/std/input.output/string.streams/stringstream/stringstream.members/str.allocator_propagation.pass.cpp b/libcxx/test/std/input.output/string.streams/stringstream/stringstream.members/str.allocator_propagation.pass.cpp
new file mode 100644
index 0000000000..46a9213eaf
--- /dev/null
+++ b/libcxx/test/std/input.output/string.streams/stringstream/stringstream.members/str.allocator_propagation.pass.cpp
@@ -0,0 +1,144 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// TODO: Change to XFAIL once https://github.com/llvm/llvm-project/issues/40340 is fixed
+// UNSUPPORTED: availability-pmr-missing
+
+// This test ensures that we properly propagate allocators from stringstream's
+// inner string object to the new string returned from .str().
+// `str() const&` is specified to preserve the allocator (not copy the string).
+// `str() &&` isn't specified, but should preserve the allocator (move the string).
+
+#include <cassert>
+#include <memory>
+#include <memory_resource>
+#include <sstream>
+#include <string>
+#include <string_view>
+#include <type_traits>
+
+#include "make_string.h"
+#include "test_allocator.h"
+#include "test_macros.h"
+
+template <class CharT>
+void test_soccc_behavior() {
+  using Alloc = SocccAllocator<CharT>;
+  using SS    = std::basic_stringstream<CharT, std::char_traits<CharT>, Alloc>;
+  using S     = std::basic_string<CharT, std::char_traits<CharT>, Alloc>;
+  {
+    SS ss = SS(std::ios_base::out, Alloc(10));
+
+    // [stringbuf.members]/6 specifies that the allocator is copied,
+    // not select_on_container_copy_construction'ed.
+    //
+    S copied = ss.str();
+    assert(copied.get_allocator().count_ == 10);
+    assert(ss.rdbuf()->get_allocator().count_ == 10);
+    assert(copied.empty());
+
+    // sanity-check that SOCCC does in fact work
+    assert(S(copied).get_allocator().count_ == 11);
+
+    // [stringbuf.members]/10 doesn't specify the allocator to use,
+    // but copying the allocator as-if-by moving the string makes sense.
+    //
+    S moved = std::move(ss).str();
+    assert(moved.get_allocator().count_ == 10);
+    assert(ss.rdbuf()->get_allocator().count_ == 10);
+    assert(moved.empty());
+  }
+}
+
+template <class CharT,
+          class Base = std::basic_stringbuf<CharT, std::char_traits<CharT>, std::pmr::polymorphic_allocator<CharT>>>
+struct StringBuf : Base {
+  explicit StringBuf(std::pmr::memory_resource* mr) : Base(std::ios_base::in, mr) {}
+  void public_setg(int a, int b, int c) {
+    CharT* p = this->eback();
+    assert(this->view().data() == p);
+    this->setg(p + a, p + b, p + c);
+    assert(this->eback() == p + a);
+    assert(this->view().data() == p + a);
+  }
+};
+
+template <class CharT>
+void test_allocation_is_pilfered() {
+  using SS = std::basic_stringstream<CharT, std::char_traits<CharT>, std::pmr::polymorphic_allocator<CharT>>;
+  using S  = std::pmr::basic_string<CharT>;
+  alignas(void*) char buf[80 * sizeof(CharT)];
+  const CharT* initial =
+      MAKE_CSTRING(CharT, "a very long string that exceeds the small string optimization buffer length");
+  {
+    std::pmr::set_default_resource(std::pmr::null_memory_resource());
+    auto mr1 = std::pmr::monotonic_buffer_resource(buf, sizeof(buf), std::pmr::null_memory_resource());
+    SS ss    = SS(S(initial, &mr1));
+    S s      = std::move(ss).str();
+    assert(s == initial);
+  }
+  {
+    // Try moving-out-of a stringbuf whose view() is not the entire string.
+    // This is libc++'s behavior; libstdc++ doesn't allow such stringbufs to be created.
+    //
+    std::pmr::set_default_resource(std::pmr::null_memory_resource());
+    auto mr1 = std::pmr::monotonic_buffer_resource(buf, sizeof(buf), std::pmr::null_memory_resource());
+    auto src = StringBuf<CharT>(&mr1);
+    src.str(S(initial, &mr1));
+    src.public_setg(2, 6, 40);
+    SS ss(std::ios_base::in, &mr1);
+    *ss.rdbuf() = std::move(src);
+    LIBCPP_ASSERT(ss.view() == std::basic_string_view<CharT>(initial).substr(2, 38));
+    S s = std::move(ss).str();
+    LIBCPP_ASSERT(s == std::basic_string_view<CharT>(initial).substr(2, 38));
+  }
+}
+
+template <class CharT>
+void test_no_foreign_allocations() {
+  using SS = std::basic_stringstream<CharT, std::char_traits<CharT>, std::pmr::polymorphic_allocator<CharT>>;
+  using S  = std::pmr::basic_string<CharT>;
+  const CharT* initial =
+      MAKE_CSTRING(CharT, "a very long string that exceeds the small string optimization buffer length");
+  {
+    std::pmr::set_default_resource(std::pmr::null_memory_resource());
+    auto mr1 = std::pmr::monotonic_buffer_resource(std::pmr::new_delete_resource());
+    auto ss  = SS(S(initial, &mr1));
+    assert(ss.rdbuf()->get_allocator().resource() == &mr1);
+
+    // [stringbuf.members]/6 specifies that the result of `str() const &`
+    // does NOT use the default allocator; it uses the original allocator.
+    //
+    S copied = ss.str();
+    assert(copied.get_allocator().resource() == &mr1);
+    assert(ss.rdbuf()->get_allocator().resource() == &mr1);
+    assert(copied == initial);
+
+    // [stringbuf.members]/10 doesn't specify the allocator to use,
+    // but copying the allocator as-if-by moving the string makes sense.
+    //
+    S moved = std::move(ss).str();
+    assert(moved.get_allocator().resource() == &mr1);
+    assert(ss.rdbuf()->get_allocator().resource() == &mr1);
+    assert(moved == initial);
+  }
+}
+
+int main(int, char**) {
+  test_soccc_behavior<char>();
+  test_allocation_is_pilfered<char>();
+  test_no_foreign_allocations<char>();
+#ifndef TEST_HAS_NO_WIDE_CHARACTERS
+  test_soccc_behavior<wchar_t>();
+  test_allocation_is_pilfered<wchar_t>();
+  test_no_foreign_allocations<wchar_t>();
+#endif
+
+  return 0;
+}
diff --git a/libcxx/test/std/input.output/string.streams/stringstream/stringstream.members/str.move.pass.cpp b/libcxx/test/std/input.output/string.streams/stringstream/stringstream.members/str.move.pass.cpp
index 35349c9c28..56a0d84fb6 100644
--- a/libcxx/test/std/input.output/string.streams/stringstream/stringstream.members/str.move.pass.cpp
+++ b/libcxx/test/std/input.output/string.streams/stringstream/stringstream.members/str.move.pass.cpp
@@ -37,6 +37,14 @@ static void test() {
     assert(s.empty());
     assert(ss.view().empty());
   }
+  {
+    std::basic_stringstream<CharT> ss(
+        STR("a very long string that exceeds the small string optimization buffer length"));
+    const CharT* p             = ss.view().data();
+    std::basic_string<CharT> s = std::move(ss).str();
+    assert(s.data() == p); // the allocation was pilfered
+    assert(ss.view().empty());
+  }
 }
 
 int main(int, char**) {
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/mdspan.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/mdspan.version.compile.pass.cpp
index e47d678b25..a44f2b6a91 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/mdspan.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/mdspan.version.compile.pass.cpp
@@ -65,17 +65,11 @@
 
 #elif TEST_STD_VER == 23
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_mdspan
-#     error "__cpp_lib_mdspan should be defined in c++23"
-#   endif
-#   if __cpp_lib_mdspan != 202207L
-#     error "__cpp_lib_mdspan should have the value 202207L in c++23"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_mdspan
-#     error "__cpp_lib_mdspan should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_mdspan
+#   error "__cpp_lib_mdspan should be defined in c++23"
+# endif
+# if __cpp_lib_mdspan != 202207L
+#   error "__cpp_lib_mdspan should have the value 202207L in c++23"
 # endif
 
 # ifdef __cpp_lib_submdspan
@@ -84,17 +78,11 @@
 
 #elif TEST_STD_VER > 23
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_mdspan
-#     error "__cpp_lib_mdspan should be defined in c++26"
-#   endif
-#   if __cpp_lib_mdspan != 202207L
-#     error "__cpp_lib_mdspan should have the value 202207L in c++26"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_mdspan
-#     error "__cpp_lib_mdspan should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_mdspan
+#   error "__cpp_lib_mdspan should be defined in c++26"
+# endif
+# if __cpp_lib_mdspan != 202207L
+#   error "__cpp_lib_mdspan should have the value 202207L in c++26"
 # endif
 
 # if !defined(_LIBCPP_VERSION)
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
index 3804a8b2bd..ed86c55522 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
@@ -4804,17 +4804,11 @@
 #   endif
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_mdspan
-#     error "__cpp_lib_mdspan should be defined in c++23"
-#   endif
-#   if __cpp_lib_mdspan != 202207L
-#     error "__cpp_lib_mdspan should have the value 202207L in c++23"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_mdspan
-#     error "__cpp_lib_mdspan should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_mdspan
+#   error "__cpp_lib_mdspan should be defined in c++23"
+# endif
+# if __cpp_lib_mdspan != 202207L
+#   error "__cpp_lib_mdspan should have the value 202207L in c++23"
 # endif
 
 # if !defined(_LIBCPP_AVAILABILITY_HAS_NO_PMR)
@@ -6345,17 +6339,11 @@
 #   endif
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_mdspan
-#     error "__cpp_lib_mdspan should be defined in c++26"
-#   endif
-#   if __cpp_lib_mdspan != 202207L
-#     error "__cpp_lib_mdspan should have the value 202207L in c++26"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_mdspan
-#     error "__cpp_lib_mdspan should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_mdspan
+#   error "__cpp_lib_mdspan should be defined in c++26"
+# endif
+# if __cpp_lib_mdspan != 202207L
+#   error "__cpp_lib_mdspan should have the value 202207L in c++26"
 # endif
 
 # if !defined(_LIBCPP_AVAILABILITY_HAS_NO_PMR)
diff --git a/libcxx/test/std/language.support/support.runtime/csetjmp.pass.cpp b/libcxx/test/std/language.support/support.runtime/csetjmp.pass.cpp
index 0e3d8f69e9..d6d32c371b 100644
--- a/libcxx/test/std/language.support/support.runtime/csetjmp.pass.cpp
+++ b/libcxx/test/std/language.support/support.runtime/csetjmp.pass.cpp
@@ -9,20 +9,28 @@
 // test <csetjmp>
 
 #include <csetjmp>
+#include <cassert>
 #include <type_traits>
 
-#include "test_macros.h"
+int main(int, char**) {
+  std::jmp_buf jb;
 
-#ifndef setjmp
-#error setjmp not defined
-#endif
+  switch (setjmp(jb)) {
+  // First time we set the buffer, the function should return 0
+  case 0:
+    break;
 
-int main(int, char**)
-{
-    std::jmp_buf jb;
-    ((void)jb); // Prevent unused warning
-    static_assert((std::is_same<decltype(std::longjmp(jb, 0)), void>::value),
-                  "std::is_same<decltype(std::longjmp(jb, 0)), void>::value");
+  // If it returned 42, then we're coming from the std::longjmp call below
+  case 42:
+    return 0;
 
-  return 0;
+  // Otherwise, something is wrong
+  default:
+    return 1;
+  }
+
+  std::longjmp(jb, 42);
+  static_assert(std::is_same<decltype(std::longjmp(jb, 0)), void>::value, "");
+
+  return 1;
 }
diff --git a/libcxx/test/std/localization/locale.categories/category.collate/locale.collate.byname/compare.pass.cpp b/libcxx/test/std/localization/locale.categories/category.collate/locale.collate.byname/compare.pass.cpp
index 69f41c8074..158bd5182e 100644
--- a/libcxx/test/std/localization/locale.categories/category.collate/locale.collate.byname/compare.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.collate/locale.collate.byname/compare.pass.cpp
@@ -6,6 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
+// Bionic has minimal locale support, investigate this later.
+// XFAIL: LIBCXX-ANDROID-FIXME
+
 // REQUIRES: locale.en_US.UTF-8
 
 // <locale>
diff --git a/libcxx/test/std/localization/locale.categories/category.collate/locale.collate.byname/transform.pass.cpp b/libcxx/test/std/localization/locale.categories/category.collate/locale.collate.byname/transform.pass.cpp
index 8d076fdeba..4978db116e 100644
--- a/libcxx/test/std/localization/locale.categories/category.collate/locale.collate.byname/transform.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.collate/locale.collate.byname/transform.pass.cpp
@@ -9,6 +9,9 @@
 // NetBSD does not support LC_COLLATE at the moment
 // XFAIL: netbsd
 
+// Bionic has minimal locale support, investigate this later.
+// XFAIL: LIBCXX-ANDROID-FIXME
+
 // <locale>
 
 // template <class charT> class collate_byname
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/is_1.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/is_1.pass.cpp
index d08a71219f..5cebc079c4 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/is_1.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/is_1.pass.cpp
@@ -15,6 +15,9 @@
 // REQUIRES: locale.en_US.UTF-8
 // XFAIL: no-wide-characters
 
+// Bionic has minimal locale support, investigate this later.
+// XFAIL: LIBCXX-ANDROID-FIXME
+
 #include <locale>
 #include <type_traits>
 #include <cassert>
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/is_many.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/is_many.pass.cpp
index 8aa1aa85d9..2869378b42 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/is_many.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/is_many.pass.cpp
@@ -15,6 +15,9 @@
 // REQUIRES: locale.en_US.UTF-8
 // XFAIL: no-wide-characters
 
+// Bionic has minimal locale support, investigate this later.
+// XFAIL: LIBCXX-ANDROID-FIXME
+
 #include <locale>
 #include <string>
 #include <vector>
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/scan_is.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/scan_is.pass.cpp
index 540efde1ea..ea03f042f2 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/scan_is.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/scan_is.pass.cpp
@@ -15,6 +15,9 @@
 // REQUIRES: locale.en_US.UTF-8
 // XFAIL: no-wide-characters
 
+// Bionic has minimal locale support, investigate this later.
+// XFAIL: LIBCXX-ANDROID-FIXME
+
 #include <locale>
 #include <string>
 #include <vector>
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/scan_not.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/scan_not.pass.cpp
index 73e3c13857..2f7acb843e 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/scan_not.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/scan_not.pass.cpp
@@ -15,6 +15,9 @@
 // REQUIRES: locale.en_US.UTF-8
 // XFAIL: no-wide-characters
 
+// Bionic has minimal locale support, investigate this later.
+// XFAIL: LIBCXX-ANDROID-FIXME
+
 #include <locale>
 #include <string>
 #include <vector>
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/tolower_1.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/tolower_1.pass.cpp
index 49b56aa931..447353de8a 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/tolower_1.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/tolower_1.pass.cpp
@@ -9,6 +9,9 @@
 // REQUIRES: locale.en_US.UTF-8
 // XFAIL: win32-broken-utf8-wchar-ctype
 
+// Bionic has minimal locale support, investigate this later.
+// XFAIL: LIBCXX-ANDROID-FIXME
+
 // <locale>
 
 // template <class charT> class ctype_byname;
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/tolower_many.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/tolower_many.pass.cpp
index 952dcf4a6b..94ecc6c8fa 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/tolower_many.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/tolower_many.pass.cpp
@@ -9,6 +9,9 @@
 // REQUIRES: locale.en_US.UTF-8
 // XFAIL: win32-broken-utf8-wchar-ctype
 
+// Bionic has minimal locale support, investigate this later.
+// XFAIL: LIBCXX-ANDROID-FIXME
+
 // <locale>
 
 // template <class charT> class ctype_byname;
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/toupper_1.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/toupper_1.pass.cpp
index 24f8b3d93b..2b37737a26 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/toupper_1.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/toupper_1.pass.cpp
@@ -9,6 +9,9 @@
 // REQUIRES: locale.en_US.UTF-8
 // XFAIL: win32-broken-utf8-wchar-ctype
 
+// Bionic has minimal locale support, investigate this later.
+// XFAIL: LIBCXX-ANDROID-FIXME
+
 // <locale>
 
 // template <class charT> class ctype_byname;
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/toupper_many.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/toupper_many.pass.cpp
index f507763425..016d8d736a 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/toupper_many.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.ctype.byname/toupper_many.pass.cpp
@@ -9,6 +9,9 @@
 // REQUIRES: locale.en_US.UTF-8
 // XFAIL: win32-broken-utf8-wchar-ctype
 
+// Bionic has minimal locale support, investigate this later.
+// XFAIL: LIBCXX-ANDROID-FIXME
+
 // <locale>
 
 // template <class charT> class ctype_byname;
diff --git a/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_long_double_en_US.pass.cpp b/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_long_double_en_US.pass.cpp
index 21117da3c2..9997b07134 100644
--- a/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_long_double_en_US.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_long_double_en_US.pass.cpp
@@ -13,6 +13,9 @@
 // iter_type get(iter_type b, iter_type e, bool intl, ios_base& iob,
 //               ios_base::iostate& err, long double& v) const;
 
+// Bionic has minimal locale support, investigate this later.
+// XFAIL: LIBCXX-ANDROID-FIXME
+
 // REQUIRES: locale.en_US.UTF-8
 
 #include <locale>
diff --git a/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_string_en_US.pass.cpp b/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_string_en_US.pass.cpp
index 80e84a425e..478df7964f 100644
--- a/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_string_en_US.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_string_en_US.pass.cpp
@@ -13,6 +13,9 @@
 // iter_type get(iter_type b, iter_type e, bool intl, ios_base& iob,
 //               ios_base::iostate& err, string_type& v) const;
 
+// Bionic has minimal locale support, investigate this later.
+// XFAIL: LIBCXX-ANDROID-FIXME
+
 // REQUIRES: locale.en_US.UTF-8
 
 #include <locale>
diff --git a/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_long_double_en_US.pass.cpp b/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_long_double_en_US.pass.cpp
index 7092c236eb..4b767fae87 100644
--- a/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_long_double_en_US.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_long_double_en_US.pass.cpp
@@ -13,6 +13,9 @@
 // iter_type put(iter_type s, bool intl, ios_base& f, char_type fill,
 //               long double units) const;
 
+// Bionic has minimal locale support, investigate this later.
+// XFAIL: LIBCXX-ANDROID-FIXME
+
 // REQUIRES: locale.en_US.UTF-8
 
 #include <locale>
diff --git a/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_string_en_US.pass.cpp b/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_string_en_US.pass.cpp
index bb7047a162..1c8710a008 100644
--- a/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_string_en_US.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_string_en_US.pass.cpp
@@ -13,6 +13,9 @@
 // iter_type put(iter_type s, bool intl, ios_base& f, char_type fill,
 //               const string_type& units) const;
 
+// Bionic has minimal locale support, investigate this later.
+// XFAIL: LIBCXX-ANDROID-FIXME
+
 // REQUIRES: locale.en_US.UTF-8
 
 #include <locale>
diff --git a/libcxx/test/std/localization/locale.categories/category.numeric/locale.nm.put/facet.num.put.members/put_long_double.pass.cpp b/libcxx/test/std/localization/locale.categories/category.numeric/locale.nm.put/facet.num.put.members/put_long_double.pass.cpp
index 3327a35a60..8bb9ff1c20 100644
--- a/libcxx/test/std/localization/locale.categories/category.numeric/locale.nm.put/facet.num.put.members/put_long_double.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.numeric/locale.nm.put/facet.num.put.members/put_long_double.pass.cpp
@@ -14,6 +14,10 @@
 
 // XFAIL: win32-broken-printf-g-precision
 
+// Needs more investigation, but this is probably failing on Android M (API 23)
+// and up because the printf formatting of NAN changed.
+// XFAIL: LIBCXX-ANDROID-FIXME && !android-device-api={{21|22}}
+
 #include <locale>
 #include <ios>
 #include <cassert>
diff --git a/libcxx/test/std/re/re.traits/translate_nocase.pass.cpp b/libcxx/test/std/re/re.traits/translate_nocase.pass.cpp
index 1048e58597..be81286e72 100644
--- a/libcxx/test/std/re/re.traits/translate_nocase.pass.cpp
+++ b/libcxx/test/std/re/re.traits/translate_nocase.pass.cpp
@@ -15,6 +15,10 @@
 // REQUIRES: locale.en_US.UTF-8
 // XFAIL: win32-broken-utf8-wchar-ctype
 
+// Prior to Android O (API 26), in the "en_US.UTF-8" locale, towlower(L'\xDA')
+// returned 0xDA instead of 0xFA.
+// XFAIL: LIBCXX-ANDROID-FIXME && android-device-api={{21|22|23|24|25}}
+
 #include <regex>
 #include <cassert>
 
diff --git a/libcxx/test/std/strings/basic.string/string.cons/implicit_deduction_guides.pass.cpp b/libcxx/test/std/strings/basic.string/string.cons/implicit_deduction_guides.pass.cpp
index e0fa234639..f9bd18a663 100644
--- a/libcxx/test/std/strings/basic.string/string.cons/implicit_deduction_guides.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.cons/implicit_deduction_guides.pass.cpp
@@ -48,15 +48,17 @@ using BStr = std::basic_string<T, std::char_traits<T>, Alloc>;
 TEST_CONSTEXPR_CXX20 bool test() {
   using TestSizeT = test_allocator<char>::size_type;
   {
-      // Testing (1)
-      // Nothing to do. Cannot deduce without any arguments.
-  } {
-      // Testing (2)
-      // This overload isn't compatible with implicit deduction guides as
-      // specified in the standard.
-      // const test_allocator<char> alloc{};
-      // std::basic_string s(alloc);
-  } { // Testing (3) w/o allocator
+    // Testing (1)
+    // Nothing to do. Cannot deduce without any arguments.
+  }
+  {
+    // Testing (2)
+    // This overload isn't compatible with implicit deduction guides as
+    // specified in the standard.
+    // const test_allocator<char> alloc{};
+    // std::basic_string s(alloc);
+  }
+  { // Testing (3) w/o allocator
     std::basic_string s(6ull, 'a');
     ASSERT_SAME_TYPE(decltype(s), std::string);
     assert(s == "aaaaaa");
@@ -187,162 +189,163 @@ TEST_CONSTEXPR_CXX20 bool test() {
     ASSERT_SAME_TYPE(decltype(s), std::string);
     assert(s == "abc");
   }
-  {// (8) w/ allocator
-   {using Expect = std::basic_string<char, std::char_traits<char>, test_allocator<char>>;
-  using It          = cpp17_input_iterator<const char*>;
-  const char* input = "abcdef";
-  std::basic_string s(It(input), It(input + 3), test_allocator<char>{});
-  ASSERT_SAME_TYPE(decltype(s), Expect);
-  assert(s == "abc");
-}
+  { // (8) w/ allocator
+    {
+      using Expect      = std::basic_string<char, std::char_traits<char>, test_allocator<char>>;
+      using It          = cpp17_input_iterator<const char*>;
+      const char* input = "abcdef";
+      std::basic_string s(It(input), It(input + 3), test_allocator<char>{});
+      ASSERT_SAME_TYPE(decltype(s), Expect);
+      assert(s == "abc");
+    }
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
-{
-  using ExpectW        = std::basic_string<wchar_t, std::char_traits<wchar_t>, test_allocator<wchar_t>>;
-  using It             = cpp17_input_iterator<const wchar_t*>;
-  const wchar_t* input = L"abcdef";
-  std::basic_string s(It(input), It(input + 3), test_allocator<wchar_t>{});
-  ASSERT_SAME_TYPE(decltype(s), ExpectW);
-  assert(s == L"abc");
-}
+    {
+      using ExpectW        = std::basic_string<wchar_t, std::char_traits<wchar_t>, test_allocator<wchar_t>>;
+      using It             = cpp17_input_iterator<const wchar_t*>;
+      const wchar_t* input = L"abcdef";
+      std::basic_string s(It(input), It(input + 3), test_allocator<wchar_t>{});
+      ASSERT_SAME_TYPE(decltype(s), ExpectW);
+      assert(s == L"abc");
+    }
 #endif
-}
-{ // Testing (9)
-  const std::string sin("abc");
-  std::basic_string s(sin);
-  ASSERT_SAME_TYPE(decltype(s), std::string);
-  assert(s == "abc");
+  }
+  { // Testing (9)
+    const std::string sin("abc");
+    std::basic_string s(sin);
+    ASSERT_SAME_TYPE(decltype(s), std::string);
+    assert(s == "abc");
 
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
-  using WStr = std::basic_string<wchar_t, constexpr_char_traits<wchar_t>, test_allocator<wchar_t>>;
-  const WStr win(L"abcdef");
-  std::basic_string w(win);
-  ASSERT_SAME_TYPE(decltype(w), WStr);
-  assert(w == L"abcdef");
+    using WStr = std::basic_string<wchar_t, constexpr_char_traits<wchar_t>, test_allocator<wchar_t>>;
+    const WStr win(L"abcdef");
+    std::basic_string w(win);
+    ASSERT_SAME_TYPE(decltype(w), WStr);
+    assert(w == L"abcdef");
 #endif
-}
-{ // Testing (10)
-  const std::string sin("abc");
-  std::basic_string s(sin, std::allocator<char>{});
-  ASSERT_SAME_TYPE(decltype(s), std::string);
-  assert(s == "abc");
+  }
+  { // Testing (10)
+    const std::string sin("abc");
+    std::basic_string s(sin, std::allocator<char>{});
+    ASSERT_SAME_TYPE(decltype(s), std::string);
+    assert(s == "abc");
 
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
-  using WStr = std::basic_string<wchar_t, constexpr_char_traits<wchar_t>, test_allocator<wchar_t>>;
-  const WStr win(L"abcdef");
-  std::basic_string w(win, test_allocator<wchar_t>{});
-  ASSERT_SAME_TYPE(decltype(w), WStr);
-  assert(w == L"abcdef");
+    using WStr = std::basic_string<wchar_t, constexpr_char_traits<wchar_t>, test_allocator<wchar_t>>;
+    const WStr win(L"abcdef");
+    std::basic_string w(win, test_allocator<wchar_t>{});
+    ASSERT_SAME_TYPE(decltype(w), WStr);
+    assert(w == L"abcdef");
 #endif
-}
-{ // Testing (11)
-  std::string sin("abc");
-  std::basic_string s(std::move(sin));
-  ASSERT_SAME_TYPE(decltype(s), std::string);
-  assert(s == "abc");
+  }
+  { // Testing (11)
+    std::string sin("abc");
+    std::basic_string s(std::move(sin));
+    ASSERT_SAME_TYPE(decltype(s), std::string);
+    assert(s == "abc");
 
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
-  using WStr = std::basic_string<wchar_t, constexpr_char_traits<wchar_t>, test_allocator<wchar_t>>;
-  WStr win(L"abcdef");
-  std::basic_string w(std::move(win));
-  ASSERT_SAME_TYPE(decltype(w), WStr);
-  assert(w == L"abcdef");
+    using WStr = std::basic_string<wchar_t, constexpr_char_traits<wchar_t>, test_allocator<wchar_t>>;
+    WStr win(L"abcdef");
+    std::basic_string w(std::move(win));
+    ASSERT_SAME_TYPE(decltype(w), WStr);
+    assert(w == L"abcdef");
 #endif
-}
-{ // Testing (12)
-  std::string sin("abc");
-  std::basic_string s(std::move(sin), std::allocator<char>{});
-  ASSERT_SAME_TYPE(decltype(s), std::string);
-  assert(s == "abc");
+  }
+  { // Testing (12)
+    std::string sin("abc");
+    std::basic_string s(std::move(sin), std::allocator<char>{});
+    ASSERT_SAME_TYPE(decltype(s), std::string);
+    assert(s == "abc");
 
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
-  using WStr = std::basic_string<wchar_t, constexpr_char_traits<wchar_t>, test_allocator<wchar_t>>;
-  WStr win(L"abcdef");
-  std::basic_string w(std::move(win), test_allocator<wchar_t>{});
-  ASSERT_SAME_TYPE(decltype(w), WStr);
-  assert(w == L"abcdef");
+    using WStr = std::basic_string<wchar_t, constexpr_char_traits<wchar_t>, test_allocator<wchar_t>>;
+    WStr win(L"abcdef");
+    std::basic_string w(std::move(win), test_allocator<wchar_t>{});
+    ASSERT_SAME_TYPE(decltype(w), WStr);
+    assert(w == L"abcdef");
 #endif
-}
-{ // Testing (13) w/o allocator
-  std::basic_string s({'a', 'b', 'c'});
-  ASSERT_SAME_TYPE(decltype(s), std::string);
-  assert(s == "abc");
+  }
+  { // Testing (13) w/o allocator
+    std::basic_string s({'a', 'b', 'c'});
+    ASSERT_SAME_TYPE(decltype(s), std::string);
+    assert(s == "abc");
 
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
-  std::basic_string w({L'a', L'b', L'c'});
-  ASSERT_SAME_TYPE(decltype(w), std::wstring);
-  assert(w == L"abc");
+    std::basic_string w({L'a', L'b', L'c'});
+    ASSERT_SAME_TYPE(decltype(w), std::wstring);
+    assert(w == L"abc");
 #endif
-}
-{ // Testing (13) w/ allocator
-  std::basic_string s({'a', 'b', 'c'}, test_allocator<char>{});
-  ASSERT_SAME_TYPE(decltype(s), BStr<char, test_allocator<char>>);
-  assert(s == "abc");
+  }
+  { // Testing (13) w/ allocator
+    std::basic_string s({'a', 'b', 'c'}, test_allocator<char>{});
+    ASSERT_SAME_TYPE(decltype(s), BStr<char, test_allocator<char>>);
+    assert(s == "abc");
 
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
-  std::basic_string w({L'a', L'b', L'c'}, test_allocator<wchar_t>{});
-  ASSERT_SAME_TYPE(decltype(w), BStr<wchar_t, test_allocator<wchar_t>>);
-  assert(w == L"abc");
+    std::basic_string w({L'a', L'b', L'c'}, test_allocator<wchar_t>{});
+    ASSERT_SAME_TYPE(decltype(w), BStr<wchar_t, test_allocator<wchar_t>>);
+    assert(w == L"abc");
 #endif
-}
-{ // Testing (14) w/o allocator
-  std::string_view sv("abc");
-  std::basic_string s(sv);
-  ASSERT_SAME_TYPE(decltype(s), std::string);
-  assert(s == "abc");
+  }
+  { // Testing (14) w/o allocator
+    std::string_view sv("abc");
+    std::basic_string s(sv);
+    ASSERT_SAME_TYPE(decltype(s), std::string);
+    assert(s == "abc");
 
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
-  using Expect = std::basic_string<wchar_t, constexpr_char_traits<wchar_t>>;
-  std::basic_string_view<wchar_t, constexpr_char_traits<wchar_t>> BSV(L"abcdef");
-  std::basic_string w(BSV);
-  ASSERT_SAME_TYPE(decltype(w), Expect);
-  assert(w == L"abcdef");
+    using Expect = std::basic_string<wchar_t, constexpr_char_traits<wchar_t>>;
+    std::basic_string_view<wchar_t, constexpr_char_traits<wchar_t>> BSV(L"abcdef");
+    std::basic_string w(BSV);
+    ASSERT_SAME_TYPE(decltype(w), Expect);
+    assert(w == L"abcdef");
 #endif
-}
-{ // Testing (14) w/ allocator
-  using ExpectS = std::basic_string<char, std::char_traits<char>, test_allocator<char>>;
-  std::string_view sv("abc");
-  std::basic_string s(sv, test_allocator<char>{});
-  ASSERT_SAME_TYPE(decltype(s), ExpectS);
-  assert(s == "abc");
+  }
+  { // Testing (14) w/ allocator
+    using ExpectS = std::basic_string<char, std::char_traits<char>, test_allocator<char>>;
+    std::string_view sv("abc");
+    std::basic_string s(sv, test_allocator<char>{});
+    ASSERT_SAME_TYPE(decltype(s), ExpectS);
+    assert(s == "abc");
 
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
-  using ExpectW = std::basic_string<wchar_t, constexpr_char_traits<wchar_t>, test_allocator<wchar_t>>;
-  std::basic_string_view<wchar_t, constexpr_char_traits<wchar_t>> BSV(L"abcdef");
-  std::basic_string w(BSV, test_allocator<wchar_t>{});
-  ASSERT_SAME_TYPE(decltype(w), ExpectW);
-  assert(w == L"abcdef");
+    using ExpectW = std::basic_string<wchar_t, constexpr_char_traits<wchar_t>, test_allocator<wchar_t>>;
+    std::basic_string_view<wchar_t, constexpr_char_traits<wchar_t>> BSV(L"abcdef");
+    std::basic_string w(BSV, test_allocator<wchar_t>{});
+    ASSERT_SAME_TYPE(decltype(w), ExpectW);
+    assert(w == L"abcdef");
 #endif
-}
-{ // Testing (15) w/o allocator
-  std::string s0("abc");
-  std::basic_string s(s0, 1, 1);
-  ASSERT_SAME_TYPE(decltype(s), std::string);
-  assert(s == "b");
+  }
+  { // Testing (15) w/o allocator
+    std::string s0("abc");
+    std::basic_string s(s0, 1, 1);
+    ASSERT_SAME_TYPE(decltype(s), std::string);
+    assert(s == "b");
 
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
-  std::wstring w0(L"abcdef");
-  std::basic_string w(w0, 2, 2);
-  ASSERT_SAME_TYPE(decltype(w), std::wstring);
-  assert(w == L"cd");
+    std::wstring w0(L"abcdef");
+    std::basic_string w(w0, 2, 2);
+    ASSERT_SAME_TYPE(decltype(w), std::wstring);
+    assert(w == L"cd");
 #endif
-}
-{ // Testing (15) w/ allocator
-  using ExpectS = std::basic_string<char, std::char_traits<char>, test_allocator<char>>;
-  ExpectS s0("abc");
-  std::basic_string s(s0, 1, 1, test_allocator<char>{4});
-  ASSERT_SAME_TYPE(decltype(s), ExpectS);
-  assert(s == "b");
+  }
+  { // Testing (15) w/ allocator
+    using ExpectS = std::basic_string<char, std::char_traits<char>, test_allocator<char>>;
+    ExpectS s0("abc");
+    std::basic_string s(s0, 1, 1, test_allocator<char>{4});
+    ASSERT_SAME_TYPE(decltype(s), ExpectS);
+    assert(s == "b");
 
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
-  using ExpectW = std::basic_string<wchar_t, std::char_traits<wchar_t>, test_allocator<wchar_t>>;
-  ExpectW w0(L"abcdef");
-  std::basic_string w(w0, 2, 2, test_allocator<wchar_t>{6});
-  ASSERT_SAME_TYPE(decltype(w), ExpectW);
-  assert(w == L"cd");
+    using ExpectW = std::basic_string<wchar_t, std::char_traits<wchar_t>, test_allocator<wchar_t>>;
+    ExpectW w0(L"abcdef");
+    std::basic_string w(w0, 2, 2, test_allocator<wchar_t>{6});
+    ASSERT_SAME_TYPE(decltype(w), ExpectW);
+    assert(w == L"cd");
 #endif
-}
+  }
 
-return true;
+  return true;
 }
 
 int main(int, char**) {
diff --git a/libcxx/test/std/strings/string.view/string.view.cons/implicit_deduction_guides.pass.cpp b/libcxx/test/std/strings/string.view/string.view.cons/implicit_deduction_guides.pass.cpp
index 534b024480..53caed5506 100644
--- a/libcxx/test/std/strings/string.view/string.view.cons/implicit_deduction_guides.pass.cpp
+++ b/libcxx/test/std/strings/string.view/string.view.cons/implicit_deduction_guides.pass.cpp
@@ -27,9 +27,10 @@
 // (4)  basic_string_view(const CharT*)
 int main(int, char**) {
   {
-      // Testing (1)
-      // Nothing to do. Cannot deduce without any arguments.
-  } { // Testing (2)
+    // Testing (1)
+    // Nothing to do. Cannot deduce without any arguments.
+  }
+  { // Testing (2)
     const std::string_view sin("abc");
     std::basic_string_view s(sin);
     ASSERT_SAME_TYPE(decltype(s), std::string_view);
diff --git a/libcxx/test/std/thread/thread.condition/thread.condition.condvar/notify_one.pass.cpp b/libcxx/test/std/thread/thread.condition/thread.condition.condvar/notify_one.pass.cpp
index b0c476fa19..bc6c3016eb 100644
--- a/libcxx/test/std/thread/thread.condition/thread.condition.condvar/notify_one.pass.cpp
+++ b/libcxx/test/std/thread/thread.condition/thread.condition.condvar/notify_one.pass.cpp
@@ -8,6 +8,9 @@
 //
 // UNSUPPORTED: no-threads
 
+// This test occasionally fails on Android.
+// UNSUPPORTED: LIBCXX-ANDROID-FIXME
+
 // <condition_variable>
 
 // class condition_variable;
diff --git a/libcxx/test/std/thread/thread.jthread/assign.move.pass.cpp b/libcxx/test/std/thread/thread.jthread/assign.move.pass.cpp
index b932ac39d2..89521ad766 100644
--- a/libcxx/test/std/thread/thread.jthread/assign.move.pass.cpp
+++ b/libcxx/test/std/thread/thread.jthread/assign.move.pass.cpp
@@ -23,6 +23,7 @@
 #include <utility>
 #include <vector>
 
+#include "make_test_thread.h"
 #include "test_macros.h"
 
 static_assert(std::is_nothrow_move_assignable_v<std::jthread>);
@@ -30,10 +31,10 @@ static_assert(std::is_nothrow_move_assignable_v<std::jthread>);
 int main(int, char**) {
   // If &x == this is true, there are no effects.
   {
-    std::jthread j([] {});
-    auto id      = j.get_id();
-    auto ssource = j.get_stop_source();
-    j            = std::move(j);
+    std::jthread j = support::make_test_jthread([] {});
+    auto id        = j.get_id();
+    auto ssource   = j.get_stop_source();
+    j              = std::move(j);
     assert(j.get_id() == id);
     assert(j.get_stop_source() == ssource);
   }
@@ -41,12 +42,12 @@ int main(int, char**) {
   // if joinable() is true, calls request_stop() and then join()
   // request_stop is called
   {
-    std::jthread j1([] {});
-    bool called = false;
+    std::jthread j1 = support::make_test_jthread([] {});
+    bool called     = false;
     std::stop_callback cb(j1.get_stop_token(), [&called] { called = true; });
 
-    std::jthread j2([] {});
-    j1 = std::move(j2);
+    std::jthread j2 = support::make_test_jthread([] {});
+    j1              = std::move(j2);
     assert(called);
   }
 
@@ -58,10 +59,10 @@ int main(int, char**) {
     constexpr auto numberOfThreads = 10u;
     jts.reserve(numberOfThreads);
     for (auto i = 0u; i < numberOfThreads; ++i) {
-      jts.emplace_back([&] {
+      jts.emplace_back(support::make_test_jthread([&] {
         std::this_thread::sleep_for(std::chrono::milliseconds(2));
         calledTimes.fetch_add(1, std::memory_order_relaxed);
-      });
+      }));
     }
 
     for (auto i = 0u; i < numberOfThreads; ++i) {
@@ -79,10 +80,10 @@ int main(int, char**) {
 
   // then assigns the state of x to *this
   {
-    std::jthread j1([] {});
-    std::jthread j2([] {});
-    auto id2      = j2.get_id();
-    auto ssource2 = j2.get_stop_source();
+    std::jthread j1 = support::make_test_jthread([] {});
+    std::jthread j2 = support::make_test_jthread([] {});
+    auto id2        = j2.get_id();
+    auto ssource2   = j2.get_stop_source();
 
     j1 = std::move(j2);
 
@@ -92,9 +93,9 @@ int main(int, char**) {
 
   // sets x to a default constructed state
   {
-    std::jthread j1([] {});
-    std::jthread j2([] {});
-    j1 = std::move(j2);
+    std::jthread j1 = support::make_test_jthread([] {});
+    std::jthread j2 = support::make_test_jthread([] {});
+    j1              = std::move(j2);
 
     assert(j2.get_id() == std::jthread::id());
     assert(!j2.get_stop_source().stop_possible());
@@ -103,7 +104,7 @@ int main(int, char**) {
   // joinable is false
   {
     std::jthread j1;
-    std::jthread j2([] {});
+    std::jthread j2 = support::make_test_jthread([] {});
 
     auto j2Id = j2.get_id();
 
diff --git a/libcxx/test/std/thread/thread.jthread/cons.move.pass.cpp b/libcxx/test/std/thread/thread.jthread/cons.move.pass.cpp
index 9eacf8971c..c3c0446770 100644
--- a/libcxx/test/std/thread/thread.jthread/cons.move.pass.cpp
+++ b/libcxx/test/std/thread/thread.jthread/cons.move.pass.cpp
@@ -19,6 +19,7 @@
 #include <type_traits>
 #include <utility>
 
+#include "make_test_thread.h"
 #include "test_macros.h"
 
 static_assert(std::is_nothrow_move_constructible_v<std::jthread>);
@@ -27,8 +28,8 @@ int main(int, char**) {
   {
     // x.get_id() == id() and get_id() returns the value of x.get_id() prior
     // to the start of construction.
-    std::jthread j1{[] {}};
-    auto id1 = j1.get_id();
+    std::jthread j1 = support::make_test_jthread([] {});
+    auto id1        = j1.get_id();
 
     std::jthread j2(std::move(j1));
     assert(j1.get_id() == std::jthread::id());
@@ -38,8 +39,8 @@ int main(int, char**) {
   {
     // ssource has the value of x.ssource prior to the start of construction
     // and x.ssource.stop_possible() is false.
-    std::jthread j1{[] {}};
-    auto ss1 = j1.get_stop_source();
+    std::jthread j1 = support::make_test_jthread([] {});
+    auto ss1        = j1.get_stop_source();
 
     std::jthread j2(std::move(j1));
     assert(ss1 == j2.get_stop_source());
diff --git a/libcxx/test/std/thread/thread.jthread/detach.pass.cpp b/libcxx/test/std/thread/thread.jthread/detach.pass.cpp
index ee48d2691e..54fd5fd93b 100644
--- a/libcxx/test/std/thread/thread.jthread/detach.pass.cpp
+++ b/libcxx/test/std/thread/thread.jthread/detach.pass.cpp
@@ -23,6 +23,7 @@
 #include <thread>
 #include <type_traits>
 
+#include "make_test_thread.h"
 #include "test_macros.h"
 
 int main(int, char**) {
@@ -30,10 +31,10 @@ int main(int, char**) {
   {
     std::atomic_bool start{false};
     std::atomic_bool done{false};
-    std::optional<std::jthread> jt{[&start, &done] {
+    std::optional<std::jthread> jt = support::make_test_jthread([&start, &done] {
       start.wait(false);
       done = true;
-    }};
+    });
 
     // If it blocks, it will deadlock here
     jt->detach();
@@ -49,7 +50,7 @@ int main(int, char**) {
 
   // Postconditions: get_id() == id().
   {
-    std::jthread jt{[] {}};
+    std::jthread jt = support::make_test_jthread([] {});
     assert(jt.get_id() != std::jthread::id());
     jt.detach();
     assert(jt.get_id() == std::jthread::id());
diff --git a/libcxx/test/std/thread/thread.jthread/dtor.pass.cpp b/libcxx/test/std/thread/thread.jthread/dtor.pass.cpp
index 47ee62023f..35be0f6c0d 100644
--- a/libcxx/test/std/thread/thread.jthread/dtor.pass.cpp
+++ b/libcxx/test/std/thread/thread.jthread/dtor.pass.cpp
@@ -20,6 +20,8 @@
 #include <thread>
 #include <type_traits>
 #include <vector>
+
+#include "make_test_thread.h"
 #include "test_macros.h"
 
 int main(int, char**) {
@@ -32,8 +34,8 @@ int main(int, char**) {
   // If joinable() is true, calls request_stop() and then join().
   // request_stop is called
   {
-    std::optional<std::jthread> jt([] {});
-    bool called = false;
+    std::optional<std::jthread> jt = support::make_test_jthread([] {});
+    bool called                    = false;
     std::stop_callback cb(jt->get_stop_token(), [&called] { called = true; });
     jt.reset();
     assert(called);
@@ -48,10 +50,10 @@ int main(int, char**) {
     constexpr auto numberOfThreads = 10u;
     jts.reserve(numberOfThreads);
     for (auto i = 0u; i < numberOfThreads; ++i) {
-      jts.emplace_back([&calledTimes] {
+      jts.emplace_back(support::make_test_jthread([&calledTimes] {
         std::this_thread::sleep_for(std::chrono::milliseconds{2});
         calledTimes.fetch_add(1, std::memory_order_relaxed);
-      });
+      }));
     }
     jts.clear();
 
diff --git a/libcxx/test/std/thread/thread.jthread/get_id.pass.cpp b/libcxx/test/std/thread/thread.jthread/get_id.pass.cpp
index f92472d3d8..b3a2beff9f 100644
--- a/libcxx/test/std/thread/thread.jthread/get_id.pass.cpp
+++ b/libcxx/test/std/thread/thread.jthread/get_id.pass.cpp
@@ -18,6 +18,7 @@
 #include <thread>
 #include <type_traits>
 
+#include "make_test_thread.h"
 #include "test_macros.h"
 
 static_assert(noexcept(std::declval<const std::jthread&>().get_id()));
@@ -32,7 +33,7 @@ int main(int, char**) {
 
   // Represents a thread
   {
-    const std::jthread jt{[] {}};
+    const std::jthread jt                                = support::make_test_jthread([] {});
     std::same_as<std::jthread::id> decltype(auto) result = jt.get_id();
     assert(result != std::jthread::id());
   }
diff --git a/libcxx/test/std/thread/thread.jthread/get_stop_source.pass.cpp b/libcxx/test/std/thread/thread.jthread/get_stop_source.pass.cpp
index 41df2d894f..8f35db297b 100644
--- a/libcxx/test/std/thread/thread.jthread/get_stop_source.pass.cpp
+++ b/libcxx/test/std/thread/thread.jthread/get_stop_source.pass.cpp
@@ -19,6 +19,7 @@
 #include <thread>
 #include <type_traits>
 
+#include "make_test_thread.h"
 #include "test_macros.h"
 
 static_assert(noexcept(std::declval<std::jthread&>().get_stop_source()));
@@ -26,7 +27,7 @@ static_assert(noexcept(std::declval<std::jthread&>().get_stop_source()));
 int main(int, char**) {
   // Represents a thread
   {
-    std::jthread jt{[] {}};
+    std::jthread jt                                      = support::make_test_jthread([] {});
     std::same_as<std::stop_source> decltype(auto) result = jt.get_stop_source();
     assert(result.stop_possible());
   }
diff --git a/libcxx/test/std/thread/thread.jthread/get_stop_token.pass.cpp b/libcxx/test/std/thread/thread.jthread/get_stop_token.pass.cpp
index c65d39b3cd..070761e0a3 100644
--- a/libcxx/test/std/thread/thread.jthread/get_stop_token.pass.cpp
+++ b/libcxx/test/std/thread/thread.jthread/get_stop_token.pass.cpp
@@ -20,6 +20,7 @@
 #include <type_traits>
 #include <utility>
 
+#include "make_test_thread.h"
 #include "test_macros.h"
 
 static_assert(noexcept(std::declval<const std::jthread&>().get_stop_token()));
@@ -27,7 +28,7 @@ static_assert(noexcept(std::declval<const std::jthread&>().get_stop_token()));
 int main(int, char**) {
   // Represents a thread
   {
-    std::jthread jt{[] {}};
+    std::jthread jt                                 = support::make_test_jthread([] {});
     auto ss                                         = jt.get_stop_source();
     std::same_as<std::stop_token> decltype(auto) st = std::as_const(jt).get_stop_token();
 
diff --git a/libcxx/test/std/thread/thread.jthread/join.deadlock.pass.cpp b/libcxx/test/std/thread/thread.jthread/join.deadlock.pass.cpp
index aa5cdf2783..8e2f1e5f5d 100644
--- a/libcxx/test/std/thread/thread.jthread/join.deadlock.pass.cpp
+++ b/libcxx/test/std/thread/thread.jthread/join.deadlock.pass.cpp
@@ -31,6 +31,7 @@
 #include <type_traits>
 #include <vector>
 
+#include "make_test_thread.h"
 #include "test_macros.h"
 
 int main(int, char**) {
@@ -40,12 +41,12 @@ int main(int, char**) {
     std::atomic_bool start = false;
     std::atomic_bool done  = false;
 
-    std::jthread jt{[&] {
+    std::jthread jt = support::make_test_jthread([&] {
       start.wait(false);
       f();
       done = true;
       done.notify_all();
-    }};
+    });
 
     f = [&] {
       try {
diff --git a/libcxx/test/std/thread/thread.jthread/join.pass.cpp b/libcxx/test/std/thread/thread.jthread/join.pass.cpp
index 38986bdfed..2bafd86338 100644
--- a/libcxx/test/std/thread/thread.jthread/join.pass.cpp
+++ b/libcxx/test/std/thread/thread.jthread/join.pass.cpp
@@ -23,6 +23,7 @@
 #include <type_traits>
 #include <vector>
 
+#include "make_test_thread.h"
 #include "test_macros.h"
 
 int main(int, char**) {
@@ -33,10 +34,10 @@ int main(int, char**) {
     constexpr auto numberOfThreads = 10u;
     jts.reserve(numberOfThreads);
     for (auto i = 0u; i < numberOfThreads; ++i) {
-      jts.emplace_back([&] {
+      jts.emplace_back(support::make_test_jthread([&] {
         std::this_thread::sleep_for(std::chrono::milliseconds(2));
         calledTimes.fetch_add(1, std::memory_order_relaxed);
-      });
+      }));
     }
 
     for (auto i = 0u; i < numberOfThreads; ++i) {
@@ -55,15 +56,15 @@ int main(int, char**) {
   // Synchronization: The completion of the thread represented by *this synchronizes with
   // ([intro.multithread]) the corresponding successful join() return.
   {
-    bool flag = false;
-    std::jthread jt{[&] { flag = true; }};
+    bool flag       = false;
+    std::jthread jt = support::make_test_jthread([&] { flag = true; });
     jt.join();
     assert(flag); // non atomic write is visible to the current thread
   }
 
   // Postconditions: The thread represented by *this has completed. get_id() == id().
   {
-    std::jthread jt{[] {}};
+    std::jthread jt = support::make_test_jthread([] {});
     assert(jt.get_id() != std::jthread::id());
     jt.join();
     assert(jt.get_id() == std::jthread::id());
diff --git a/libcxx/test/std/thread/thread.jthread/joinable.pass.cpp b/libcxx/test/std/thread/thread.jthread/joinable.pass.cpp
index 5c0fbece4c..3a88100d93 100644
--- a/libcxx/test/std/thread/thread.jthread/joinable.pass.cpp
+++ b/libcxx/test/std/thread/thread.jthread/joinable.pass.cpp
@@ -19,6 +19,7 @@
 #include <thread>
 #include <type_traits>
 
+#include "make_test_thread.h"
 #include "test_macros.h"
 
 static_assert(noexcept(std::declval<const std::jthread&>().joinable()));
@@ -33,7 +34,7 @@ int main(int, char**) {
 
   // Non-default constructed
   {
-    const std::jthread jt{[] {}};
+    const std::jthread jt                    = support::make_test_jthread([] {});
     std::same_as<bool> decltype(auto) result = jt.joinable();
     assert(result);
   }
@@ -41,8 +42,8 @@ int main(int, char**) {
   // Non-default constructed
   // the thread of execution has not finished
   {
-    std::atomic_bool done = false;
-    const std::jthread jt{[&done] { done.wait(false); }};
+    std::atomic_bool done                    = false;
+    const std::jthread jt                    = support::make_test_jthread([&done] { done.wait(false); });
     std::same_as<bool> decltype(auto) result = jt.joinable();
     done                                     = true;
     done.notify_all();
diff --git a/libcxx/test/std/thread/thread.jthread/request_stop.pass.cpp b/libcxx/test/std/thread/thread.jthread/request_stop.pass.cpp
index f1109561cf..ccbea9f145 100644
--- a/libcxx/test/std/thread/thread.jthread/request_stop.pass.cpp
+++ b/libcxx/test/std/thread/thread.jthread/request_stop.pass.cpp
@@ -19,6 +19,7 @@
 #include <thread>
 #include <type_traits>
 
+#include "make_test_thread.h"
 #include "test_macros.h"
 
 static_assert(noexcept(std::declval<std::jthread&>().request_stop()));
@@ -26,8 +27,8 @@ static_assert(noexcept(std::declval<std::jthread&>().request_stop()));
 int main(int, char**) {
   // Represents a thread
   {
-    std::jthread jt{[] {}};
-    auto st = jt.get_stop_token();
+    std::jthread jt = support::make_test_jthread([] {});
+    auto st         = jt.get_stop_token();
     assert(!st.stop_requested());
     std::same_as<bool> decltype(auto) result = jt.request_stop();
     assert(result);
diff --git a/libcxx/test/std/thread/thread.jthread/swap.free.pass.cpp b/libcxx/test/std/thread/thread.jthread/swap.free.pass.cpp
index 776537cdff..01c8ccd659 100644
--- a/libcxx/test/std/thread/thread.jthread/swap.free.pass.cpp
+++ b/libcxx/test/std/thread/thread.jthread/swap.free.pass.cpp
@@ -17,6 +17,7 @@
 #include <thread>
 #include <type_traits>
 
+#include "make_test_thread.h"
 #include "test_macros.h"
 
 template <class T>
@@ -30,7 +31,7 @@ int main(int, char**) {
   // x is default constructed
   {
     std::jthread t1;
-    std::jthread t2{[] {}};
+    std::jthread t2        = support::make_test_jthread([] {});
     const auto originalId2 = t2.get_id();
     swap(t1, t2);
 
@@ -40,7 +41,7 @@ int main(int, char**) {
 
   // y is default constructed
   {
-    std::jthread t1([] {});
+    std::jthread t1 = support::make_test_jthread([] {});
     std::jthread t2{};
     const auto originalId1 = t1.get_id();
     swap(t1, t2);
@@ -51,8 +52,8 @@ int main(int, char**) {
 
   // both not default constructed
   {
-    std::jthread t1([] {});
-    std::jthread t2{[] {}};
+    std::jthread t1        = support::make_test_jthread([] {});
+    std::jthread t2        = support::make_test_jthread([] {});
     const auto originalId1 = t1.get_id();
     const auto originalId2 = t2.get_id();
     swap(t1, t2);
diff --git a/libcxx/test/std/thread/thread.jthread/swap.member.pass.cpp b/libcxx/test/std/thread/thread.jthread/swap.member.pass.cpp
index 614e3ac831..8ae17f435a 100644
--- a/libcxx/test/std/thread/thread.jthread/swap.member.pass.cpp
+++ b/libcxx/test/std/thread/thread.jthread/swap.member.pass.cpp
@@ -17,6 +17,7 @@
 #include <thread>
 #include <type_traits>
 
+#include "make_test_thread.h"
 #include "test_macros.h"
 
 template <class T>
@@ -30,7 +31,7 @@ int main(int, char**) {
   // this is default constructed
   {
     std::jthread t1;
-    std::jthread t2{[] {}};
+    std::jthread t2        = support::make_test_jthread([] {});
     const auto originalId2 = t2.get_id();
     t1.swap(t2);
 
@@ -40,7 +41,7 @@ int main(int, char**) {
 
   // that is default constructed
   {
-    std::jthread t1([] {});
+    std::jthread t1 = support::make_test_jthread([] {});
     std::jthread t2{};
     const auto originalId1 = t1.get_id();
     t1.swap(t2);
@@ -51,8 +52,8 @@ int main(int, char**) {
 
   // both not default constructed
   {
-    std::jthread t1([] {});
-    std::jthread t2{[] {}};
+    std::jthread t1        = support::make_test_jthread([] {});
+    std::jthread t2        = support::make_test_jthread([] {});
     const auto originalId1 = t1.get_id();
     const auto originalId2 = t2.get_id();
     t1.swap(t2);
diff --git a/libcxx/test/std/utilities/format/format.functions/locale-specific_form.pass.cpp b/libcxx/test/std/utilities/format/format.functions/locale-specific_form.pass.cpp
index 44bf84d55e..b25fd25f4e 100644
--- a/libcxx/test/std/utilities/format/format.functions/locale-specific_form.pass.cpp
+++ b/libcxx/test/std/utilities/format/format.functions/locale-specific_form.pass.cpp
@@ -11,6 +11,9 @@
 
 // XFAIL: availability-fp_to_chars-missing
 
+// Bionic has minimal locale support, investigate this later.
+// XFAIL: LIBCXX-ANDROID-FIXME
+
 // REQUIRES: locale.en_US.UTF-8
 
 // <format>
diff --git a/libcxx/test/std/utilities/memory/specialized.algorithms/overload_compare_iterator.h b/libcxx/test/std/utilities/memory/specialized.algorithms/overload_compare_iterator.h
new file mode 100644
index 0000000000..d5dcb08c37
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/specialized.algorithms/overload_compare_iterator.h
@@ -0,0 +1,73 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LIBCPP_TEST_STD_UTILITIES_MEMORY_SPECIALIZED_ALGORITHMS_OVERLOAD_COMPARE_ITERATOR_H
+#define LIBCPP_TEST_STD_UTILITIES_MEMORY_SPECIALIZED_ALGORITHMS_OVERLOAD_COMPARE_ITERATOR_H
+
+#include <iterator>
+#include <memory>
+
+#include "test_macros.h"
+
+// An iterator type that overloads operator== and operator!= without any constraints, which
+// can trip up some algorithms if we compare iterators against types that we're not allowed to.
+//
+// See https://github.com/llvm/llvm-project/issues/69334 for details.
+template <class Iterator>
+struct overload_compare_iterator {
+  using value_type        = typename std::iterator_traits<Iterator>::value_type;
+  using difference_type   = typename std::iterator_traits<Iterator>::difference_type;
+  using reference         = typename std::iterator_traits<Iterator>::reference;
+  using pointer           = typename std::iterator_traits<Iterator>::pointer;
+  using iterator_category = typename std::iterator_traits<Iterator>::iterator_category;
+
+  overload_compare_iterator() = default;
+
+  explicit overload_compare_iterator(Iterator it) : it_(it) {}
+
+  overload_compare_iterator(overload_compare_iterator const&)            = default;
+  overload_compare_iterator(overload_compare_iterator&&)                 = default;
+  overload_compare_iterator& operator=(overload_compare_iterator const&) = default;
+  overload_compare_iterator& operator=(overload_compare_iterator&&)      = default;
+
+  reference operator*() const TEST_NOEXCEPT { return *it_; }
+
+  pointer operator->() const TEST_NOEXCEPT { return std::addressof(*it_); }
+
+  overload_compare_iterator& operator++() TEST_NOEXCEPT {
+    ++it_;
+    return *this;
+  }
+
+  overload_compare_iterator operator++(int) const TEST_NOEXCEPT {
+    overload_compare_iterator old(*this);
+    ++(*this);
+    return old;
+  }
+
+  bool operator==(overload_compare_iterator const& other) const TEST_NOEXCEPT { return this->it_ == other.it_; }
+
+  bool operator!=(overload_compare_iterator const& other) const TEST_NOEXCEPT { return !this->operator==(other); }
+
+  // Hostile overloads
+  template <class Sentinel>
+  friend bool operator==(overload_compare_iterator const& lhs, Sentinel const& rhs) TEST_NOEXCEPT {
+    return static_cast<Iterator const&>(lhs) == rhs;
+  }
+
+  template <class Sentinel>
+  friend bool operator!=(overload_compare_iterator const& lhs, Sentinel const& rhs) TEST_NOEXCEPT {
+    return static_cast<Iterator const&>(lhs) != rhs;
+  }
+
+private:
+  Iterator it_;
+};
+
+#endif // LIBCPP_TEST_STD_UTILITIES_MEMORY_SPECIALIZED_ALGORITHMS_OVERLOAD_COMPARE_ITERATOR_H
diff --git a/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.copy/ranges_uninitialized_copy.pass.cpp b/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.copy/ranges_uninitialized_copy.pass.cpp
index fb3ae6f4ae..92dc380728 100644
--- a/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.copy/ranges_uninitialized_copy.pass.cpp
+++ b/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.copy/ranges_uninitialized_copy.pass.cpp
@@ -27,6 +27,7 @@
 
 #include "../buffer.h"
 #include "../counted.h"
+#include "../overload_compare_iterator.h"
 #include "test_macros.h"
 #include "test_iterators.h"
 
@@ -396,5 +397,36 @@ int main(int, char**) {
     }
   }
 
+  // Test with an iterator that overloads operator== and operator!= as the input and output iterators
+  {
+    using T        = int;
+    using Iterator = overload_compare_iterator<T*>;
+    const int N    = 5;
+
+    // input
+    {
+      char pool[sizeof(T) * N] = {0};
+      T* p                     = reinterpret_cast<T*>(pool);
+      T* p_end                 = reinterpret_cast<T*>(pool) + N;
+      T array[N]               = {1, 2, 3, 4, 5};
+      std::ranges::uninitialized_copy(Iterator(array), Iterator(array + N), p, p_end);
+      for (int i = 0; i != N; ++i) {
+        assert(array[i] == p[i]);
+      }
+    }
+
+    // output
+    {
+      char pool[sizeof(T) * N] = {0};
+      T* p                     = reinterpret_cast<T*>(pool);
+      T* p_end                 = reinterpret_cast<T*>(pool) + N;
+      T array[N]               = {1, 2, 3, 4, 5};
+      std::ranges::uninitialized_copy(array, array + N, Iterator(p), Iterator(p_end));
+      for (int i = 0; i != N; ++i) {
+        assert(array[i] == p[i]);
+      }
+    }
+  }
+
   return 0;
 }
diff --git a/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.copy/ranges_uninitialized_copy_n.pass.cpp b/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.copy/ranges_uninitialized_copy_n.pass.cpp
index 097f88dec0..80082eb3b9 100644
--- a/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.copy/ranges_uninitialized_copy_n.pass.cpp
+++ b/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.copy/ranges_uninitialized_copy_n.pass.cpp
@@ -24,6 +24,7 @@
 
 #include "../buffer.h"
 #include "../counted.h"
+#include "../overload_compare_iterator.h"
 #include "test_macros.h"
 #include "test_iterators.h"
 
@@ -161,5 +162,36 @@ int main(int, char**) {
     std::ranges::uninitialized_copy_n(std::move(in), N, out.begin(), out.end());
   }
 
+  // Test with an iterator that overloads operator== and operator!= as the input and output iterators
+  {
+    using T        = int;
+    using Iterator = overload_compare_iterator<T*>;
+    const int N    = 5;
+
+    // input
+    {
+      char pool[sizeof(T) * N] = {0};
+      T* p                     = reinterpret_cast<T*>(pool);
+      T* p_end                 = reinterpret_cast<T*>(pool) + N;
+      T array[N]               = {1, 2, 3, 4, 5};
+      std::ranges::uninitialized_copy_n(Iterator(array), N, p, p_end);
+      for (int i = 0; i != N; ++i) {
+        assert(array[i] == p[i]);
+      }
+    }
+
+    // output
+    {
+      char pool[sizeof(T) * N] = {0};
+      T* p                     = reinterpret_cast<T*>(pool);
+      T* p_end                 = reinterpret_cast<T*>(pool) + N;
+      T array[N]               = {1, 2, 3, 4, 5};
+      std::ranges::uninitialized_copy_n(array, N, Iterator(p), Iterator(p_end));
+      for (int i = 0; i != N; ++i) {
+        assert(array[i] == p[i]);
+      }
+    }
+  }
+
   return 0;
 }
diff --git a/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.copy/uninitialized_copy.pass.cpp b/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.copy/uninitialized_copy.pass.cpp
index c3b6e18090..dddc550e0e 100644
--- a/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.copy/uninitialized_copy.pass.cpp
+++ b/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.copy/uninitialized_copy.pass.cpp
@@ -17,6 +17,7 @@
 #include <cassert>
 
 #include "test_macros.h"
+#include "../overload_compare_iterator.h"
 
 struct B
 {
@@ -85,6 +86,34 @@ int main(int, char**)
         }
     }
 
+    // Test with an iterator that overloads operator== and operator!= as the input and output iterators
+    {
+        using T = int;
+        using Iterator = overload_compare_iterator<T*>;
+        const int N = 5;
+
+        // input
+        {
+            char pool[sizeof(T) * N] = {0};
+            T* p = reinterpret_cast<T*>(pool);
+            T array[N] = {1, 2, 3, 4, 5};
+            std::uninitialized_copy(Iterator(array), Iterator(array + N), p);
+            for (int i = 0; i != N; ++i) {
+                assert(array[i] == p[i]);
+            }
+        }
+
+        // output
+        {
+            char pool[sizeof(T) * N] = {0};
+            T* p = reinterpret_cast<T*>(pool);
+            T array[N] = {1, 2, 3, 4, 5};
+            std::uninitialized_copy(array, array + N, Iterator(p));
+            for (int i = 0; i != N; ++i) {
+                assert(array[i] == p[i]);
+            }
+        }
+    }
 
   return 0;
 }
diff --git a/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.copy/uninitialized_copy_n.pass.cpp b/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.copy/uninitialized_copy_n.pass.cpp
index 9d0fae3a8e..ddaf02c184 100644
--- a/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.copy/uninitialized_copy_n.pass.cpp
+++ b/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.copy/uninitialized_copy_n.pass.cpp
@@ -17,6 +17,7 @@
 #include <cassert>
 
 #include "test_macros.h"
+#include "../overload_compare_iterator.h"
 
 struct B
 {
@@ -85,5 +86,34 @@ int main(int, char**)
     }
     }
 
+    // Test with an iterator that overloads operator== and operator!= as the input and output iterators
+    {
+        using T = int;
+        using Iterator = overload_compare_iterator<T*>;
+        const int N = 5;
+
+        // input
+        {
+            char pool[sizeof(T) * N] = {0};
+            T* p = reinterpret_cast<T*>(pool);
+            T array[N] = {1, 2, 3, 4, 5};
+            std::uninitialized_copy_n(Iterator(array), N, p);
+            for (int i = 0; i != N; ++i) {
+                assert(array[i] == p[i]);
+            }
+        }
+
+        // output
+        {
+            char pool[sizeof(T) * N] = {0};
+            T* p = reinterpret_cast<T*>(pool);
+            T array[N] = {1, 2, 3, 4, 5};
+            std::uninitialized_copy_n(array, N, Iterator(p));
+            for (int i = 0; i != N; ++i) {
+                assert(array[i] == p[i]);
+            }
+        }
+    }
+
   return 0;
 }
diff --git a/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.move/ranges_uninitialized_move.pass.cpp b/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.move/ranges_uninitialized_move.pass.cpp
index 57be8c49ca..56dd25c66e 100644
--- a/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.move/ranges_uninitialized_move.pass.cpp
+++ b/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.move/ranges_uninitialized_move.pass.cpp
@@ -27,6 +27,7 @@
 
 #include "../buffer.h"
 #include "../counted.h"
+#include "../overload_compare_iterator.h"
 #include "MoveOnly.h"
 #include "test_macros.h"
 #include "test_iterators.h"
@@ -435,5 +436,36 @@ int main(int, char**) {
     }
   }
 
+  // Test with an iterator that overloads operator== and operator!= as the input and output iterators
+  {
+    using T        = int;
+    using Iterator = overload_compare_iterator<T*>;
+    const int N    = 5;
+
+    // input
+    {
+      char pool[sizeof(T) * N] = {0};
+      T* p                     = reinterpret_cast<T*>(pool);
+      T* p_end                 = reinterpret_cast<T*>(pool) + N;
+      T array[N]               = {1, 2, 3, 4, 5};
+      std::ranges::uninitialized_move(Iterator(array), Iterator(array + N), p, p_end);
+      for (int i = 0; i != N; ++i) {
+        assert(array[i] == p[i]);
+      }
+    }
+
+    // output
+    {
+      char pool[sizeof(T) * N] = {0};
+      T* p                     = reinterpret_cast<T*>(pool);
+      T* p_end                 = reinterpret_cast<T*>(pool) + N;
+      T array[N]               = {1, 2, 3, 4, 5};
+      std::ranges::uninitialized_move(array, array + N, Iterator(p), Iterator(p_end));
+      for (int i = 0; i != N; ++i) {
+        assert(array[i] == p[i]);
+      }
+    }
+  }
+
   return 0;
 }
diff --git a/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.move/ranges_uninitialized_move_n.pass.cpp b/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.move/ranges_uninitialized_move_n.pass.cpp
index df79987920..162b4a4853 100644
--- a/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.move/ranges_uninitialized_move_n.pass.cpp
+++ b/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.move/ranges_uninitialized_move_n.pass.cpp
@@ -24,6 +24,7 @@
 
 #include "../buffer.h"
 #include "../counted.h"
+#include "../overload_compare_iterator.h"
 #include "MoveOnly.h"
 #include "test_iterators.h"
 #include "test_macros.h"
@@ -191,5 +192,36 @@ int main(int, char**) {
     }
   }
 
+  // Test with an iterator that overloads operator== and operator!= as the input and output iterators
+  {
+    using T        = int;
+    using Iterator = overload_compare_iterator<T*>;
+    const int N    = 5;
+
+    // input
+    {
+      char pool[sizeof(T) * N] = {0};
+      T* p                     = reinterpret_cast<T*>(pool);
+      T* p_end                 = reinterpret_cast<T*>(pool) + N;
+      T array[N]               = {1, 2, 3, 4, 5};
+      std::ranges::uninitialized_move_n(Iterator(array), N, p, p_end);
+      for (int i = 0; i != N; ++i) {
+        assert(array[i] == p[i]);
+      }
+    }
+
+    // output
+    {
+      char pool[sizeof(T) * N] = {0};
+      T* p                     = reinterpret_cast<T*>(pool);
+      T* p_end                 = reinterpret_cast<T*>(pool) + N;
+      T array[N]               = {1, 2, 3, 4, 5};
+      std::ranges::uninitialized_move_n(array, N, Iterator(p), Iterator(p_end));
+      for (int i = 0; i != N; ++i) {
+        assert(array[i] == p[i]);
+      }
+    }
+  }
+
   return 0;
 }
diff --git a/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.move/uninitialized_move.pass.cpp b/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.move/uninitialized_move.pass.cpp
index 625d30f64b..f77cbea19b 100644
--- a/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.move/uninitialized_move.pass.cpp
+++ b/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.move/uninitialized_move.pass.cpp
@@ -19,6 +19,7 @@
 
 #include "test_macros.h"
 #include "test_iterators.h"
+#include "../overload_compare_iterator.h"
 
 struct Counted {
   static int count;
@@ -111,5 +112,34 @@ int main(int, char**) {
     test_counted();
     test_ctor_throws();
 
+    // Test with an iterator that overloads operator== and operator!= as the input and output iterators
+    {
+        using T = int;
+        using Iterator = overload_compare_iterator<T*>;
+        const int N = 5;
+
+        // input
+        {
+            char pool[sizeof(T) * N] = {0};
+            T* p = reinterpret_cast<T*>(pool);
+            T array[N] = {1, 2, 3, 4, 5};
+            std::uninitialized_move(Iterator(array), Iterator(array + N), p);
+            for (int i = 0; i != N; ++i) {
+                assert(array[i] == p[i]);
+            }
+        }
+
+        // output
+        {
+            char pool[sizeof(T) * N] = {0};
+            T* p = reinterpret_cast<T*>(pool);
+            T array[N] = {1, 2, 3, 4, 5};
+            std::uninitialized_move(array, array + N, Iterator(p));
+            for (int i = 0; i != N; ++i) {
+                assert(array[i] == p[i]);
+            }
+        }
+    }
+
   return 0;
 }
diff --git a/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.move/uninitialized_move_n.pass.cpp b/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.move/uninitialized_move_n.pass.cpp
index 7dcdcf0c63..7cdfb4da08 100644
--- a/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.move/uninitialized_move_n.pass.cpp
+++ b/libcxx/test/std/utilities/memory/specialized.algorithms/uninitialized.move/uninitialized_move_n.pass.cpp
@@ -19,6 +19,7 @@
 
 #include "test_macros.h"
 #include "test_iterators.h"
+#include "../overload_compare_iterator.h"
 
 struct Counted {
   static int count;
@@ -114,5 +115,34 @@ int main(int, char**)
     test_counted();
     test_ctor_throws();
 
+    // Test with an iterator that overloads operator== and operator!= as the input and output iterators
+    {
+        using T = int;
+        using Iterator = overload_compare_iterator<T*>;
+        const int N = 5;
+
+        // input
+        {
+            char pool[sizeof(T) * N] = {0};
+            T* p = reinterpret_cast<T*>(pool);
+            T array[N] = {1, 2, 3, 4, 5};
+            std::uninitialized_move_n(Iterator(array), N, p);
+            for (int i = 0; i != N; ++i) {
+                assert(array[i] == p[i]);
+            }
+        }
+
+        // output
+        {
+            char pool[sizeof(T) * N] = {0};
+            T* p = reinterpret_cast<T*>(pool);
+            T array[N] = {1, 2, 3, 4, 5};
+            std::uninitialized_move_n(array, N, Iterator(p));
+            for (int i = 0; i != N; ++i) {
+                assert(array[i] == p[i]);
+            }
+        }
+    }
+
   return 0;
 }
diff --git a/libcxx/test/support/counting_projection.h b/libcxx/test/support/counting_projection.h
index 1af2c80f24..ad549c749a 100644
--- a/libcxx/test/support/counting_projection.h
+++ b/libcxx/test/support/counting_projection.h
@@ -26,7 +26,7 @@ class counting_projection {
   constexpr counting_projection(Proj proj, int& count) : proj_(std::move(proj)), count_(&count) {}
 
   template <class T>
-  constexpr decltype(auto) operator()(T&& value) const {
+  constexpr decltype(auto) operator()(T && value) const {
     ++(*count_);
     return std::invoke(proj_, std::forward<T>(value));
   }
diff --git a/libcxx/test/support/filesystem_test_helper.h b/libcxx/test/support/filesystem_test_helper.h
index d63b1e61b5..f924f909fe 100644
--- a/libcxx/test/support/filesystem_test_helper.h
+++ b/libcxx/test/support/filesystem_test_helper.h
@@ -180,13 +180,22 @@ struct scoped_test_env
         std::string cmd = "chmod -R 777 " + test_root.string();
 #endif // defined(__MVS__)
         int ret = std::system(cmd.c_str());
-#if !defined(_AIX)
+#  if !defined(_AIX) && !defined(__ANDROID__)
         // On AIX the chmod command will return non-zero when trying to set
         // the permissions on a directory that contains a bad symlink. This triggers
         // the assert, despite being able to delete everything with the following
         // `rm -r` command.
+        //
+        // Android's chmod was buggy in old OSs, but skipping this assert is
+        // sufficient to ensure that the `rm -rf` succeeds for almost all tests:
+        //  - Android L: chmod aborts after one error
+        //  - Android L and M: chmod -R tries to set permissions of a symlink
+        //    target.
+        // LIBCXX-ANDROID-FIXME: Other fixes to consider: place a toybox chmod
+        // onto old devices, re-enable this assert for devices running Android N
+        // and up, rewrite this chmod+rm in C or C++.
         assert(ret == 0);
-#endif
+#  endif
 
         cmd = "rm -rf " + test_root.string();
         ret = std::system(cmd.c_str());
diff --git a/libcxx/test/support/make_test_thread.h b/libcxx/test/support/make_test_thread.h
index eaf967e218..cd548fd909 100644
--- a/libcxx/test/support/make_test_thread.h
+++ b/libcxx/test/support/make_test_thread.h
@@ -12,13 +12,40 @@
 #include <thread>
 #include <utility>
 
+#include "test_macros.h"
+
 namespace support {
 
+// These functions are used to mock the creation of threads within the test suite.
+//
+// This provides a vendor-friendly way of making the test suite work even on platforms
+// where the standard thread constructors don't work (e.g. embedded environments where
+// creating a thread requires additional information like setting attributes).
+//
+// Vendors can keep a downstream diff in this file to create threads however they
+// need on their platform, and the majority of the test suite will work out of the
+// box. Of course, tests that exercise the standard thread constructors won't work,
+// but any other test that only creates threads as a side effect of testing should
+// work if they use the utilities in this file.
+
 template <class F, class ...Args>
 std::thread make_test_thread(F&& f, Args&& ...args) {
     return std::thread(std::forward<F>(f), std::forward<Args>(args)...);
 }
 
+#if TEST_STD_VER >= 20 && !defined(_LIBCPP_HAS_NO_EXPERIMENTAL_STOP_TOKEN)
+#  ifdef _LIBCPP_VERSION
+#    define TEST_AVAILABILITY_SYNC _LIBCPP_AVAILABILITY_SYNC
+#  else
+#    define TEST_AVAILABILITY_SYNC
+#  endif
+
+template <class F, class... Args>
+TEST_AVAILABILITY_SYNC std::jthread make_test_jthread(F&& f, Args&&... args) {
+  return std::jthread(std::forward<F>(f), std::forward<Args>(args)...);
+}
+#endif
+
 } // end namespace support
 
 #endif // TEST_SUPPORT_MAKE_TEST_THREAD_H
diff --git a/libcxx/test/support/test_allocator.h b/libcxx/test/support/test_allocator.h
index 9330150a83..3bde73183a 100644
--- a/libcxx/test/support/test_allocator.h
+++ b/libcxx/test/support/test_allocator.h
@@ -475,4 +475,29 @@ TEST_CONSTEXPR inline bool operator!=(limited_allocator<T, N> const& LHS, limite
   return !(LHS == RHS);
 }
 
+// Track the "provenance" of this allocator instance: how many times was
+// select_on_container_copy_construction called in order to produce it?
+//
+template <class T>
+struct SocccAllocator {
+  using value_type = T;
+
+  int count_ = 0;
+  explicit SocccAllocator(int i) : count_(i) {}
+
+  template <class U>
+  SocccAllocator(const SocccAllocator<U>& a) : count_(a.count_) {}
+
+  T* allocate(std::size_t n) { return std::allocator<T>().allocate(n); }
+  void deallocate(T* p, std::size_t n) { std::allocator<T>().deallocate(p, n); }
+
+  SocccAllocator select_on_container_copy_construction() const { return SocccAllocator(count_ + 1); }
+
+  bool operator==(const SocccAllocator&) const { return true; }
+
+  using propagate_on_container_copy_assignment = std::false_type;
+  using propagate_on_container_move_assignment = std::false_type;
+  using propagate_on_container_swap            = std::false_type;
+};
+
 #endif // TEST_ALLOCATOR_H
diff --git a/libcxx/utils/adb_run.py b/libcxx/utils/adb_run.py
new file mode 100755
index 0000000000..b54198fed4
--- /dev/null
+++ b/libcxx/utils/adb_run.py
@@ -0,0 +1,256 @@
+#!/usr/bin/env python3
+#===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+#===----------------------------------------------------------------------===##
+
+"""adb_run.py is a utility for running a libc++ test program via adb.
+"""
+
+import argparse
+import hashlib
+import os
+import re
+import shlex
+import socket
+import subprocess
+import sys
+from typing import List, Optional, Tuple
+
+
+# Sync a host file /path/to/dir/file to ${REMOTE_BASE_DIR}/run-${HASH}/dir/file.
+REMOTE_BASE_DIR = "/data/local/tmp/adb_run"
+
+g_job_limit_socket = None
+g_verbose = False
+
+
+def run_adb_sync_command(command: List[str]) -> None:
+    """Run an adb command and discard the output, unless the command fails. If
+    the command fails, dump the output instead, and exit the script with
+    failure.
+    """
+    if g_verbose:
+        sys.stderr.write(f"running: {shlex.join(command)}\n")
+    proc = subprocess.run(command, universal_newlines=True,
+                          stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
+                          stderr=subprocess.STDOUT, encoding="utf-8")
+    if proc.returncode != 0:
+        # adb's stdout (e.g. for adb push) should normally be discarded, but
+        # on failure, it should be shown. Print it to stderr because it's
+        # unrelated to the test program's stdout output. A common error caught
+        # here is "No space left on device".
+        sys.stderr.write(f"{proc.stdout}\n"
+                         f"error: adb command exited with {proc.returncode}: "
+                         f"{shlex.join(command)}\n")
+        sys.exit(proc.returncode)
+
+
+def sync_test_dir(local_dir: str, remote_dir: str) -> None:
+    """Sync the libc++ test directory on the host to the remote device."""
+
+    # Optimization: The typical libc++ test directory has only a single
+    # *.tmp.exe file in it. In that case, skip the `mkdir` command, which is
+    # normally necessary because we don't know if the target directory already
+    # exists on the device.
+    local_files = os.listdir(local_dir)
+    if len(local_files) == 1:
+        local_file = os.path.join(local_dir, local_files[0])
+        remote_file = os.path.join(remote_dir, local_files[0])
+        if not os.path.islink(local_file) and os.path.isfile(local_file):
+            run_adb_sync_command(["adb", "push", "--sync", local_file,
+                                  remote_file])
+            return
+
+    assert os.path.basename(local_dir) == os.path.basename(remote_dir)
+    run_adb_sync_command(["adb", "shell", "mkdir", "-p", remote_dir])
+    run_adb_sync_command(["adb", "push", "--sync", local_dir,
+                          os.path.dirname(remote_dir)])
+
+
+def build_env_arg(env_args: List[str], prepend_path_args: List[Tuple[str, str]]) -> str:
+    components = []
+    for arg in env_args:
+        k, v = arg.split("=", 1)
+        components.append(f"export {k}={shlex.quote(v)}; ")
+    for k, v in prepend_path_args:
+        components.append(f"export {k}={shlex.quote(v)}${{{k}:+:${k}}}; ")
+    return "".join(components)
+
+
+def run_command(args: argparse.Namespace) -> int:
+    local_dir = args.execdir
+    assert local_dir.startswith("/")
+    assert not local_dir.endswith("/")
+
+    # Copy each execdir to a subdir of REMOTE_BASE_DIR. Name the directory using
+    # a hash of local_dir so that concurrent adb_run invocations don't create
+    # the same intermediate parent directory. At least `adb push` has trouble
+    # with concurrent mkdir syscalls on common parent directories. (Somehow
+    # mkdir fails with EAGAIN/EWOULDBLOCK, see internal Google bug,
+    # b/289311228.)
+    local_dir_hash = hashlib.sha1(local_dir.encode()).hexdigest()
+    remote_dir = f"{REMOTE_BASE_DIR}/run-{local_dir_hash}/{os.path.basename(local_dir)}"
+    sync_test_dir(local_dir, remote_dir)
+
+    adb_shell_command = (
+        # Set the environment early so that PATH can be overridden. Overriding
+        # PATH is useful for:
+        #  - Replacing older shell utilities with toybox (e.g. on old devices).
+        #  - Adding a `bash` command that delegates to `sh` (mksh).
+        f"{build_env_arg(args.env, args.prepend_path_env)}"
+
+        # Set a high oom_score_adj so that, if the test program uses too much
+        # memory, it is killed before anything else on the device. The default
+        # oom_score_adj is -1000, so a test using too much memory typically
+        # crashes the device.
+        "echo 1000 >/proc/self/oom_score_adj; "
+
+        # If we're running as root, switch to the shell user. The libc++
+        # filesystem tests require running without root permissions. Some x86
+        # emulator devices (before Android N) do not have a working `adb unroot`
+        # and always run as root. Non-debug builds typically lack `su` and only
+        # run as the shell user.
+        #
+        # Some libc++ tests create temporary files in the working directory,
+        # which might be owned by root. Before switching to shell, make the
+        # cwd writable (and readable+executable) to every user.
+        #
+        # N.B.:
+        #  - Avoid "id -u" because it wasn't supported until Android M.
+        #  - The `env` and `which` commands were also added in Android M.
+        #  - Starting in Android M, su from root->shell resets PATH, so we need
+        #    to modify it again in the new environment.
+        #  - Avoid chmod's "a+rwx" syntax because it's not supported until
+        #    Android N.
+        #  - Defining this function allows specifying the arguments to the test
+        #    program (i.e. "$@") only once.
+        "run_without_root() {"
+        "  chmod 777 .;"
+        "  case \"$(id)\" in"
+        "    *\"uid=0(root)\"*)"
+        "    if command -v env >/dev/null; then"
+        "      su shell \"$(command -v env)\" PATH=\"$PATH\" \"$@\";"
+        "    else"
+        "      su shell \"$@\";"
+        "    fi;;"
+        "    *) \"$@\";;"
+        "  esac;"
+        "}; "
+    )
+
+    # Older versions of Bionic limit the length of argv[0] to 127 bytes
+    # (SOINFO_NAME_LEN-1), and the path to libc++ tests tend to exceed this
+    # limit. Changing the working directory works around this limit. The limit
+    # is increased to 4095 (PATH_MAX-1) in Android M (API 23).
+    command_line = [arg.replace(local_dir + "/", "./") for arg in args.command]
+
+    # Prior to the adb feature "shell_v2" (added in Android N), `adb shell`
+    # always created a pty:
+    #  - This merged stdout and stderr together.
+    #  - The pty converts LF to CRLF.
+    #  - The exit code of the shell command wasn't propagated.
+    # Work around all three limitations, unless "shell_v2" is present.
+    proc = subprocess.run(["adb", "features"], check=True,
+                          stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
+                          encoding="utf-8")
+    adb_features = set(proc.stdout.strip().split())
+    has_shell_v2 = "shell_v2" in adb_features
+    if has_shell_v2:
+        adb_shell_command += (
+            f"cd {remote_dir} && run_without_root {shlex.join(command_line)}"
+        )
+    else:
+        adb_shell_command += (
+            f"{{"
+            f"  stdout=$("
+            f"    cd {remote_dir} && run_without_root {shlex.join(command_line)};"
+            f"    echo -n __libcxx_adb_exit__=$?"
+            f"  ); "
+            f"}} 2>&1; "
+            f"echo -n __libcxx_adb_stdout__\"$stdout\""
+        )
+
+    adb_command_line = ["adb", "shell", adb_shell_command]
+    if g_verbose:
+        sys.stderr.write(f"running: {shlex.join(adb_command_line)}\n")
+
+    if has_shell_v2:
+        proc = subprocess.run(adb_command_line, shell=False, check=False,
+                              encoding="utf-8")
+        return proc.returncode
+    else:
+        proc = subprocess.run(adb_command_line, shell=False, check=False,
+                              stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+                              encoding="utf-8")
+        # The old `adb shell` mode used a pty, which converted LF to CRLF.
+        # Convert it back.
+        output = proc.stdout.replace("\r\n", "\n")
+
+        if proc.returncode:
+            sys.stderr.write(f"error: adb failed:\n"
+                             f"  command: {shlex.join(adb_command_line)}\n"
+                             f"  output: {output}\n")
+            return proc.returncode
+
+        match = re.match(r"(.*)__libcxx_adb_stdout__(.*)__libcxx_adb_exit__=(\d+)$",
+                     output, re.DOTALL)
+        if not match:
+            sys.stderr.write(f"error: could not parse adb output:\n"
+                             f"  command: {shlex.join(adb_command_line)}\n"
+                             f"  output: {output}\n")
+            return 1
+
+        sys.stderr.write(match.group(1))
+        sys.stdout.write(match.group(2))
+        return int(match.group(3))
+
+
+def connect_to_job_limiter_server(sock_addr: str) -> None:
+    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+
+    try:
+        sock.connect(sock_addr)
+    except (FileNotFoundError, ConnectionRefusedError) as e:
+        # Copying-and-pasting an adb_run.py command-line from a lit test failure
+        # is likely to fail because the socket no longer exists (or is
+        # inactive), so just give a warning.
+        sys.stderr.write(f"warning: could not connect to {sock_addr}: {e}\n")
+        return
+
+    # The connect call can succeed before the server has called accept, because
+    # of the listen backlog, so wait for the server to send a byte.
+    sock.recv(1)
+
+    # Keep the socket open until this process ends, then let the OS close the
+    # connection automatically.
+    global g_job_limit_socket
+    g_job_limit_socket = sock
+
+
+def main() -> int:
+    """Main function (pylint wants this docstring)."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--execdir", type=str, required=True)
+    parser.add_argument("--env", type=str, required=False, action="append",
+                        default=[], metavar="NAME=VALUE")
+    parser.add_argument("--prepend-path-env", type=str, nargs=2, required=False,
+                        action="append", default=[],
+                        metavar=("NAME", "PATH"))
+    parser.add_argument("--job-limit-socket")
+    parser.add_argument("--verbose", "-v", default=False, action="store_true")
+    parser.add_argument("command", nargs=argparse.ONE_OR_MORE)
+    args = parser.parse_args()
+
+    global g_verbose
+    g_verbose = args.verbose
+    if args.job_limit_socket is not None:
+        connect_to_job_limiter_server(args.job_limit_socket)
+    return run_command(args)
+
+
+if __name__ == '__main__':
+    sys.exit(main())
diff --git a/libcxx/utils/ci/BOT_OWNERS.txt b/libcxx/utils/ci/BOT_OWNERS.txt
index f6eb91cdac..721b19e52d 100644
--- a/libcxx/utils/ci/BOT_OWNERS.txt
+++ b/libcxx/utils/ci/BOT_OWNERS.txt
@@ -15,3 +15,8 @@ D: Armv7, Armv8, AArch64
 N: LLVM on Power
 E: powerllvm@ca.ibm.com
 D: AIX, ppc64le
+
+N: Android libc++
+E: rprichard@google.com
+H: rprichard
+D: Emulator-based x86[-64] libc++ CI testing
diff --git a/libcxx/utils/ci/buildkite-pipeline.yml b/libcxx/utils/ci/buildkite-pipeline.yml
index ebfb35eee9..460c5a8c43 100644
--- a/libcxx/utils/ci/buildkite-pipeline.yml
+++ b/libcxx/utils/ci/buildkite-pipeline.yml
@@ -26,31 +26,12 @@ env:
     # LLVM POST-BRANCH bump version
     # LLVM POST-BRANCH add compiler test for ToT - 1, e.g. "Clang 17"
     # LLVM RELEASE bump remove compiler ToT - 3, e.g. "Clang 15"
-    LLVM_STABLE_VERSION: "16" # Used for tooling, update after the RELEASE.
     LLVM_HEAD_VERSION: "18"   # Used compiler, update POST-BRANCH.
     GCC_STABLE_VERSION: "13"
 steps:
   #
   # Light pre-commit tests for things like forgetting to update generated files.
   #
-  - label: "Generated output"
-    command: "libcxx/utils/ci/run-buildbot check-generated-output"
-    artifact_paths:
-      - "**/generated_output.patch"
-      - "**/generated_output.status"
-    env:
-        CC: "clang-${LLVM_HEAD_VERSION}"
-        CXX: "clang++-${LLVM_HEAD_VERSION}"
-        CLANG_FORMAT: "/usr/bin/clang-format-${LLVM_STABLE_VERSION}"
-    agents:
-      queue: "libcxx-builders"
-      os: "linux"
-    retry:
-      automatic:
-        - exit_status: -1  # Agent was lost
-          limit: 2
-    timeout_in_minutes: 120
-
   - label: "Documentation"
     command: "libcxx/utils/ci/run-buildbot documentation"
     artifact_paths:
diff --git a/libcxx/utils/ci/run-buildbot b/libcxx/utils/ci/run-buildbot
index a71318123d..be96bb65ef 100755
--- a/libcxx/utils/ci/run-buildbot
+++ b/libcxx/utils/ci/run-buildbot
@@ -156,6 +156,13 @@ function generate-cmake-libcxx-win() {
           "${@}"
 }
 
+function generate-cmake-android() {
+    generate-cmake-base \
+          -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi" \
+          -DLIBCXX_CXX_ABI=libcxxabi \
+          "${@}"
+}
+
 function check-runtimes() {
     echo "+++ Running the libc++ tests"
     ${NINJA} -vC "${BUILD_DIR}" check-cxx
@@ -209,6 +216,8 @@ check-generated-output)
     clean
     generate-cmake
 
+    set +x # Printing all the commands below just creates extremely confusing output
+
     # Reject patches that forgot to re-run the generator scripts.
     echo "+++ Making sure the generator scripts were run"
     ${NINJA} -vC "${BUILD_DIR}" libcxx-generate-files
@@ -222,20 +231,23 @@ check-generated-output)
         false
     fi
 
+    echo "+++ Making sure libcxx/utils/data/ignore_format.txt was updated appropriately"
+    cp ${MONOREPO_ROOT}/libcxx/utils/data/ignore_format.txt ${BUILD_DIR}/before.txt
     ${MONOREPO_ROOT}/libcxx/utils/generate_ignore_format.sh
-    git diff | tee ${BUILD_DIR}/generated_output.patch
-    git ls-files -o --exclude-standard | tee ${BUILD_DIR}/generated_output.status
-    ! grep -q '^--- a' ${BUILD_DIR}/generated_output.patch || false
-    if [ -s ${BUILD_DIR}/generated_output.status ]; then
+    diff ${BUILD_DIR}/before.txt ${MONOREPO_ROOT}/libcxx/utils/data/ignore_format.txt | tee ${BUILD_DIR}/ignore_format.diff || true
+    if [ -s ${BUILD_DIR}/ignore_format.diff ]; then
         echo "It looks like the list of not formatted files has changed."
         echo "If a file is now properly formatted with clang-format, remove the file name from "
         echo "libcxx/utils/data/ignore_format.txt. Otherwise you have to fix the"
-        echo "formatting of some of the changed files."
+        echo "formatting of some of the changed files. The diff above represents the "
+        echo "changes that would be needed to ignore_format.txt to keep it representative "
+        echo "of which files are mis-formatted in the project."
         false
     fi
 
     # Reject patches that introduce non-ASCII characters or hard tabs.
     # Depends on LC_COLLATE set at the top of this script.
+    set -x
     ! grep -rn '[^ -~]' libcxx/include libcxx/src libcxx/test libcxx/benchmarks \
            --exclude '*.dat' \
            --exclude '*unicode*.cpp' \
@@ -701,6 +713,48 @@ aix)
     check-abi-list
     check-runtimes
 ;;
+android-ndk-*)
+    clean
+
+    ANDROID_EMU_IMG="${BUILDER#android-ndk-}"
+    . "${MONOREPO_ROOT}/libcxx/utils/ci/vendor/android/emulator-functions.sh"
+    if ! validate_emu_img "${ANDROID_EMU_IMG}"; then
+        echo "error: android-ndk suffix must be a valid emulator image (${ANDROID_EMU_IMG})" >&2
+        exit 1
+    fi
+    ARCH=$(arch_of_emu_img ${ANDROID_EMU_IMG})
+
+    # Use the Android compiler by default.
+    export CC=${CC:-/opt/android/clang/clang-current/bin/clang}
+    export CXX=${CXX:-/opt/android/clang/clang-current/bin/clang++}
+
+    # The NDK libc++_shared.so is always built against the oldest supported API
+    # level. When tests are run against a device with a newer API level, test
+    # programs can be built for any supported API level, but building for the
+    # newest API (i.e. the system image's API) is probably the most interesting.
+    PARAMS="target_triple=$(triple_of_arch ${ARCH})$(api_of_emu_img ${ANDROID_EMU_IMG})"
+    generate-cmake-android -C "${MONOREPO_ROOT}/runtimes/cmake/android/Arch-${ARCH}.cmake" \
+                           -C "${MONOREPO_ROOT}/libcxx/cmake/caches/AndroidNDK.cmake" \
+                           -DCMAKE_SYSROOT=/opt/android/ndk/sysroot \
+                           -DLIBCXX_TEST_PARAMS="${PARAMS}" \
+                           -DLIBCXXABI_TEST_PARAMS="${PARAMS}"
+    ${NINJA} -vC "${BUILD_DIR}" install-cxx install-cxxabi
+
+    # Start the emulator and make sure we can connect to the adb server running
+    # inside of it.
+    "${MONOREPO_ROOT}/libcxx/utils/ci/vendor/android/start-emulator.sh" ${ANDROID_EMU_IMG}
+    trap "${MONOREPO_ROOT}/libcxx/utils/ci/vendor/android/stop-emulator.sh" EXIT
+    . "${MONOREPO_ROOT}/libcxx/utils/ci/vendor/android/setup-env-for-emulator.sh"
+
+    # Create adb_run early to avoid concurrent `mkdir -p` of common parent
+    # directories.
+    adb shell mkdir -p /data/local/tmp/adb_run
+    adb push "${BUILD_DIR}/lib/libc++_shared.so" /data/local/tmp/libc++/libc++_shared.so
+    echo "+++ Running the libc++ tests"
+    ${NINJA} -vC "${BUILD_DIR}" check-cxx
+    echo "+++ Running the libc++abi tests"
+    ${NINJA} -vC "${BUILD_DIR}" check-cxxabi
+;;
 #################################################################
 # Insert vendor-specific internal configurations below.
 #
diff --git a/libcxx/utils/ci/vendor/android/Dockerfile b/libcxx/utils/ci/vendor/android/Dockerfile
new file mode 100644
index 0000000000..0acfff8e03
--- /dev/null
+++ b/libcxx/utils/ci/vendor/android/Dockerfile
@@ -0,0 +1,83 @@
+#===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+#===----------------------------------------------------------------------===##
+
+#
+# This Dockerfile extends ldionne/libcxx-builder with Android support, including
+# Android Clang and sysroot, Android platform-tools, and the Docker client.
+#
+#   $ docker build -t libcxx-builder-android libcxx/utils/ci/vendor/android
+#
+
+FROM ldionne/libcxx-builder
+
+# Switch back to the root user to install things into /opt and /usr.
+USER root
+WORKDIR /
+
+# Install the Android platform tools (e.g. adb) into /opt/android/sdk.
+RUN apt-get update && apt-get install -y unzip
+RUN mkdir -p /opt/android/sdk && cd /opt/android/sdk && \
+    curl -LO https://dl.google.com/android/repository/platform-tools-latest-linux.zip && \
+    unzip platform-tools-latest-linux.zip && \
+    rm platform-tools-latest-linux.zip
+
+# Install the current Android compiler. Specify the prebuilts commit to retrieve
+# this compiler version even after it's removed from HEAD.
+ENV ANDROID_CLANG_VERSION=r498229b
+ENV ANDROID_CLANG_PREBUILTS_COMMIT=5186d132c99aa75dc25207c392e3ea5b93d0107e
+RUN git clone --filter=blob:none --sparse \
+        https://android.googlesource.com/platform/prebuilts/clang/host/linux-x86 \
+        /opt/android/clang && \
+    git -C /opt/android/clang checkout ${ANDROID_CLANG_PREBUILTS_COMMIT} && \
+    git -C /opt/android/clang sparse-checkout add clang-${ANDROID_CLANG_VERSION} && \
+    rm -fr /opt/android/clang/.git && \
+    ln -sf /opt/android/clang/clang-${ANDROID_CLANG_VERSION} /opt/android/clang/clang-current && \
+    # The "git sparse-checkout" and "ln" commands succeed even if nothing was
+    # checked out, so use this "ls" command to fix that.
+    ls /opt/android/clang/clang-current/bin/clang
+
+# Install an Android sysroot. New AOSP sysroots are available at
+# https://ci.android.com/builds/branches/aosp-main/grid, the "ndk" target. The
+# NDK also makes its sysroot prebuilt available at
+# https://android.googlesource.com/platform/prebuilts/ndk/+/refs/heads/dev/platform/sysroot.
+ENV ANDROID_SYSROOT_BID=10957860
+RUN cd /opt/android && \
+    curl -L -o ndk_platform.tar.bz2 \
+        https://androidbuildinternal.googleapis.com/android/internal/build/v3/builds/${ANDROID_SYSROOT_BID}/ndk/attempts/latest/artifacts/ndk_platform.tar.bz2/url && \
+    tar xf ndk_platform.tar.bz2 && \
+    rm ndk_platform.tar.bz2
+
+# Install Docker. Mark the binary setuid so it can be run without prefixing it
+# with sudo. Adding the container user to the docker group doesn't work because
+# /var/run/docker.sock is owned by the host's docker GID, not the container's
+# docker GID.
+RUN apt-get update && apt-get install -y gpg && \
+    install -m 0755 -d /etc/apt/keyrings && \
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg && \
+    chmod a+r /etc/apt/keyrings/docker.gpg && \
+    echo "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
+         "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \
+         tee /etc/apt/sources.list.d/docker.list > /dev/null && \
+    apt-get update && apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin && \
+    chmod u+s /usr/bin/docker
+
+COPY ./container-setup.sh /opt/android/container-setup.sh
+
+USER libcxx-builder
+WORKDIR /home/libcxx-builder
+
+# Add Android platform-tools to the PATH.
+ENV PATH="/opt/android/sdk/platform-tools:${PATH}"
+
+# Reset the buildkite-agent.cfg file. The tags are provided by an environment
+# variable instead.
+RUN cp /home/libcxx-builder/.buildkite-agent/buildkite-agent.dist.cfg \
+       /home/libcxx-builder/.buildkite-agent/buildkite-agent.cfg
+
+# Modify the Buildkite agent cmdline to do Android setup stuff first.
+CMD /opt/android/container-setup.sh && buildkite-agent start
diff --git a/libcxx/utils/ci/vendor/android/Dockerfile.emulator b/libcxx/utils/ci/vendor/android/Dockerfile.emulator
new file mode 100644
index 0000000000..54953f4001
--- /dev/null
+++ b/libcxx/utils/ci/vendor/android/Dockerfile.emulator
@@ -0,0 +1,59 @@
+#===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+#===----------------------------------------------------------------------===##
+
+FROM ubuntu:jammy
+
+RUN apt-get update && apt-get install -y \
+    curl \
+    netcat-openbsd \
+    openjdk-11-jdk \
+    sudo \
+    unzip \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV ANDROID_HOME /opt/android/sdk
+
+RUN curl -sL https://dl.google.com/android/repository/commandlinetools-linux-9477386_latest.zip -o cmdline-tools.zip && \
+    mkdir -p ${ANDROID_HOME} && \
+    unzip cmdline-tools.zip -d ${ANDROID_HOME}/cmdline-tools && \
+    mv ${ANDROID_HOME}/cmdline-tools/cmdline-tools ${ANDROID_HOME}/cmdline-tools/latest && \
+    rm cmdline-tools.zip
+ENV PATH="${ANDROID_HOME}/cmdline-tools/latest/bin:${PATH}"
+
+RUN yes | sdkmanager --licenses
+RUN sdkmanager --install emulator
+ENV PATH="${ANDROID_HOME}/emulator:${PATH}"
+
+ARG API  # e.g. 21
+RUN sdkmanager --install "platforms;android-${API}"
+
+ARG TYPE  # one of: default, google_apis, or google_apis_playstore
+ARG ABI   # e.g. armeabi-v7a, x86
+ENV EMU_PACKAGE_NAME="system-images;android-${API};${TYPE};${ABI}"
+RUN sdkmanager --install "${EMU_PACKAGE_NAME}"
+
+COPY ./emulator-entrypoint.sh /opt/emulator/bin/emulator-entrypoint.sh
+COPY ./emulator-wait-for-ready.sh /opt/emulator/bin/emulator-wait-for-ready.sh
+ENV PATH="/opt/emulator/bin:${PATH}"
+ENV PATH="${ANDROID_HOME}/platform-tools:${PATH}"
+
+# Setup password-less sudo so that /dev/kvm permissions can be changed. Run the
+# emulator in an unprivileged user for reliability (and it might require it?)
+RUN echo "ALL ALL = (ALL) NOPASSWD: ALL" >> /etc/sudoers
+RUN useradd --create-home emulator
+USER emulator
+WORKDIR /home/emulator
+
+# Size of emulator /data partition in megabytes.
+ENV EMU_PARTITION_SIZE=8192
+
+EXPOSE 5037
+
+HEALTHCHECK CMD emulator-wait-for-ready.sh 5
+
+ENTRYPOINT ["emulator-entrypoint.sh"]
diff --git a/libcxx/utils/ci/vendor/android/build-emulator-images.sh b/libcxx/utils/ci/vendor/android/build-emulator-images.sh
new file mode 100755
index 0000000000..f467ffc623
--- /dev/null
+++ b/libcxx/utils/ci/vendor/android/build-emulator-images.sh
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+#===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+#===----------------------------------------------------------------------===##
+
+set -e
+
+THIS_DIR="$(cd "$(dirname "$0")" && pwd)"
+. "${THIS_DIR}/emulator-functions.sh"
+
+build_image() {
+    local EMU_IMG="$1"
+    validate_emu_img_syntax "${EMU_IMG}"
+    docker build -t $(docker_image_of_emu_img ${EMU_IMG}) \
+        -f Dockerfile.emulator . \
+        --build-arg API=$(api_of_emu_img ${EMU_IMG}) \
+        --build-arg TYPE=$(type_of_emu_img ${EMU_IMG}) \
+        --build-arg ABI=$(abi_of_arch $(arch_of_emu_img ${EMU_IMG}))
+}
+
+cd "${THIS_DIR}"
+
+build_image 21-def-x86
+build_image 33-goog-x86_64
diff --git a/libcxx/utils/ci/vendor/android/container-setup.sh b/libcxx/utils/ci/vendor/android/container-setup.sh
new file mode 100755
index 0000000000..56bc232fef
--- /dev/null
+++ b/libcxx/utils/ci/vendor/android/container-setup.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+#===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+#===----------------------------------------------------------------------===##
+
+set -e
+
+# Different versions of adb can sometimes be incompatible (i.e. "adb server
+# version (nn) doesn't match this client (mm); killing..."). Ensure that the adb
+# in the main builder image matches that in the emulator by sharing the
+# platform-tools from the main image.
+if [ -d /mnt/android-platform-tools ]; then
+    sudo rm -fr /mnt/android-platform-tools/platform-tools
+    sudo cp -r /opt/android/sdk/platform-tools /mnt/android-platform-tools
+fi
diff --git a/libcxx/utils/ci/vendor/android/emulator-entrypoint.sh b/libcxx/utils/ci/vendor/android/emulator-entrypoint.sh
new file mode 100755
index 0000000000..e453869726
--- /dev/null
+++ b/libcxx/utils/ci/vendor/android/emulator-entrypoint.sh
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+#===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+#===----------------------------------------------------------------------===##
+
+# This script is the entrypoint of an Android Emulator Docker container.
+
+set -e
+
+# The container's /dev/kvm has the same UID+GID as the host device. Changing the
+# ownership inside the container doesn't affect the UID+GID on the host.
+sudo chown emulator:emulator /dev/kvm
+
+# Always use a copy of platform-tools provided by the host to ensure that the
+# versions of adb match between the host and the emulator.
+if [ ! -x /mnt/android-platform-tools/platform-tools/adb ]; then
+    echo "error: This image requires platform-tools mounted at" \
+         "/mnt/android-platform-tools containing platform-tools/adb" >&2
+    exit 1
+fi
+sudo cp -r /mnt/android-platform-tools/platform-tools /opt/android/sdk
+
+# Start an adb host server. `adb start-server` blocks until the port is ready.
+# Use ADB_REJECT_KILL_SERVER=1 to ensure that an adb protocol version mismatch
+# doesn't kill the adb server.
+ADB_REJECT_KILL_SERVER=1 adb -a start-server
+
+# This syntax (using an IP address of 127.0.0.1 rather than localhost) seems to
+# prevent the adb client from ever spawning an adb host server.
+export ADB_SERVER_SOCKET=tcp:127.0.0.1:5037
+
+# The AVD could already exist if the Docker container were stopped and then
+# restarted.
+if [ ! -d ~/.android/avd/emulator.avd ]; then
+    # N.B. AVD creation takes a few seconds and creates a mostly-empty
+    # multi-gigabyte userdata disk image. (It's not useful to create the AVDs in
+    # advance.)
+    avdmanager --verbose create avd --name emulator \
+        --package "${EMU_PACKAGE_NAME}" --device pixel_5
+fi
+
+# Use exec so that the emulator is PID 1, so that `docker stop` kills the
+# emulator.
+exec emulator @emulator -no-audio -no-window \
+    -partition-size "${EMU_PARTITION_SIZE}"
diff --git a/libcxx/utils/ci/vendor/android/emulator-functions.sh b/libcxx/utils/ci/vendor/android/emulator-functions.sh
new file mode 100644
index 0000000000..27eea2af15
--- /dev/null
+++ b/libcxx/utils/ci/vendor/android/emulator-functions.sh
@@ -0,0 +1,110 @@
+#===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+#===----------------------------------------------------------------------===##
+
+# Bash functions for managing the names of emulator system images.
+
+# Parse the image name and set variables: API, TYPE, and ARCH.
+__parse_emu_img() {
+    if [[ "${1}" =~ ([0-9]+)-(def|goog|play)-(arm|arm64|x86|x86_64)$ ]]; then
+        API=${BASH_REMATCH[1]}
+        case ${BASH_REMATCH[2]} in
+            def) TYPE=default ;;
+            goog) TYPE=google_apis ;;
+            play) TYPE=google_apis_playstore ;;
+        esac
+        ARCH=${BASH_REMATCH[3]}
+        return 0
+    else
+        return 1
+    fi
+}
+
+# Check that the emulator image name has valid syntax.
+validate_emu_img_syntax() {
+    local EMU_IMG="${1}"
+    local API TYPE ARCH
+    if ! __parse_emu_img "${EMU_IMG}"; then
+        echo "\
+error: invalid emulator image name: ${EMU_IMG}
+  expected \"\${API}-\${TYPE}-\${ARCH}\" where API is a number, TYPE is one of
+  (def|goog|play), and ARCH is one of arm, arm64, x86, or x86_64." >&2
+        return 1
+    fi
+}
+
+docker_image_of_emu_img() {
+    echo "android-emulator-${1}"
+}
+
+# Check that the emulator image name has valid syntax and that the Docker image
+# is present. On failure, writes an error to stderr and exits the script.
+validate_emu_img() {
+    local EMU_IMG="${1}"
+    if ! validate_emu_img_syntax "${EMU_IMG}"; then
+        return 1
+    fi
+    # Make sure Docker is working before trusting other Docker commands.
+    # Temporarily suppress command echoing so we only show 'docker info' output
+    # on failure, and only once.
+    if (set +x; !(docker info &>/dev/null || docker info)); then
+        echo "error: Docker is required for emulator usage but 'docker info' failed" >&2
+        return 1
+    fi
+    local DOCKER_IMAGE=$(docker_image_of_emu_img ${EMU_IMG})
+    if ! docker image inspect ${DOCKER_IMAGE} &>/dev/null; then
+        echo "error: emulator Docker image (${DOCKER_IMAGE}) is not installed" >&2
+        return 1
+    fi
+}
+
+api_of_emu_img() {
+    local API TYPE ARCH
+    __parse_emu_img "${1}"
+    echo ${API}
+}
+
+type_of_emu_img() {
+    local API TYPE ARCH
+    __parse_emu_img "${1}"
+    echo ${TYPE}
+}
+
+arch_of_emu_img() {
+    local API TYPE ARCH
+    __parse_emu_img "${1}"
+    echo ${ARCH}
+}
+
+# Expand the short emu_img string into the full SDK package string identifying
+# the system image.
+sdk_package_of_emu_img() {
+    local API TYPE ARCH
+    __parse_emu_img "${1}"
+    echo "system-images;android-${API};${TYPE};$(abi_of_arch ${ARCH})"
+}
+
+# Return the Android ABI string for an architecture.
+abi_of_arch() {
+    case "${1}" in
+        arm) echo armeabi-v7a ;;
+        arm64) echo aarch64-v8a ;;
+        x86) echo x86 ;;
+        x86_64) echo x86_64 ;;
+        *) echo "error: unhandled arch ${1}" >&2; exit 1 ;;
+    esac
+}
+
+triple_of_arch() {
+    case "${1}" in
+        arm) echo armv7a-linux-androideabi ;;
+        arm64) echo aarch64-linux-android ;;
+        x86) echo i686-linux-android ;;
+        x86_64) echo x86_64-linux-android ;;
+        *) echo "error: unhandled arch ${1}" >&2; exit 1 ;;
+    esac
+}
diff --git a/libcxx/utils/ci/vendor/android/emulator-wait-for-ready.sh b/libcxx/utils/ci/vendor/android/emulator-wait-for-ready.sh
new file mode 100755
index 0000000000..0c35794792
--- /dev/null
+++ b/libcxx/utils/ci/vendor/android/emulator-wait-for-ready.sh
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+#===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+#===----------------------------------------------------------------------===##
+
+set -ex
+
+# Time to wait in seconds. The emulator ought to start in 5-15 seconds or so,
+# so add a safety factor in case something takes longer in CI.
+TIMEOUT=${1-300}
+
+# This syntax (using an IP address of 127.0.0.1 rather than localhost) seems to
+# prevent the adb client from ever spawning an adb host server.
+export ADB_SERVER_SOCKET=tcp:127.0.0.1:5037
+
+# Invoke nc first to ensure that something is listening to port 5037. Otherwise,
+# invoking adb might fork an adb server.
+#
+# TODO: Consider waiting for `adb shell getprop dev.bootcomplete 2>/dev/null
+# | grep 1 >/dev/null` as well. It adds ~4 seconds to 21-def-x86 and ~15 seconds
+# to 33-goog-x86_64 and doesn't seem to be necessary for running libc++ tests.
+timeout ${TIMEOUT} bash -c '
+until (nc -z localhost 5037 && adb wait-for-device); do
+    sleep 0.5
+done
+'
diff --git a/libcxx/utils/ci/vendor/android/run-buildbot-container b/libcxx/utils/ci/vendor/android/run-buildbot-container
new file mode 100755
index 0000000000..4ab83194c0
--- /dev/null
+++ b/libcxx/utils/ci/vendor/android/run-buildbot-container
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+# Similar to libcxx/utils/ci/run-buildbot-container, but adds additional options
+# needed for running Android tests.
+
+set -e
+
+MONOREPO_ROOT="$(git rev-parse --show-toplevel)"
+if [[ ! -d "${MONOREPO_ROOT}/libcxx/utils/ci/vendor/android" ]]; then
+    echo "Was unable to find the root of the LLVM monorepo; are you running from within the monorepo?"
+    exit 1
+fi
+
+DOCKER_OPTIONS=(-it)
+DOCKER_OPTIONS+=(--volume "${MONOREPO_ROOT}:/llvm")
+DOCKER_OPTIONS+=(--workdir "/llvm")
+DOCKER_OPTIONS+=(--cap-add=SYS_PTRACE)
+
+# Mount this volume to allow the main image to share its copy of the Android
+# platform tools with the emulator image, ensuring that the adb versions match.
+# This argument will create a new volume if it doesn't already exist.
+DOCKER_OPTIONS+=(--volume android-platform-tools:/mnt/android-platform-tools)
+
+# Pass through the Docker socket so that the buildbot can start a sibling
+# container running an Android emulator.
+if [ -S /var/run/docker.sock ]; then
+    DOCKER_OPTIONS+=(--volume /var/run/docker.sock:/var/run/docker.sock)
+fi
+
+docker run "${DOCKER_OPTIONS[@]}" libcxx-builder-android \
+    bash -c 'git config --global --add safe.directory /llvm; (/opt/android/container-setup.sh && exec bash)'
diff --git a/libcxx/utils/ci/vendor/android/setup-env-for-emulator.sh b/libcxx/utils/ci/vendor/android/setup-env-for-emulator.sh
new file mode 100644
index 0000000000..7de6cde7a7
--- /dev/null
+++ b/libcxx/utils/ci/vendor/android/setup-env-for-emulator.sh
@@ -0,0 +1,13 @@
+#===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+#===----------------------------------------------------------------------===##
+
+export ADB_SERVER_SOCKET="tcp:$(docker inspect \
+    -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' \
+    libcxx-ci-android-emulator):5037"
+
+echo "setup-env-for-emulator.sh: setting ADB_SERVER_SOCKET to ${ADB_SERVER_SOCKET}"
diff --git a/libcxx/utils/ci/vendor/android/start-emulator.sh b/libcxx/utils/ci/vendor/android/start-emulator.sh
new file mode 100755
index 0000000000..2d6e272675
--- /dev/null
+++ b/libcxx/utils/ci/vendor/android/start-emulator.sh
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+#===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+#===----------------------------------------------------------------------===##
+
+# Starts a new Docker container using a Docker image containing the Android
+# Emulator and an OS image. Stops and removes the old container if it exists
+# already.
+
+set -e
+
+THIS_DIR="$(cd "$(dirname "$0")" && pwd)"
+. "${THIS_DIR}/emulator-functions.sh"
+
+EMU_IMG="${1}"
+if ! validate_emu_img "${EMU_IMG}"; then
+    echo "error: The first argument must be a valid emulator image." >&2
+    exit 1
+fi
+
+"${THIS_DIR}/stop-emulator.sh"
+
+# Start the container.
+docker run --name libcxx-ci-android-emulator --detach --device /dev/kvm \
+    -eEMU_PARTITION_SIZE=8192 \
+    --volume android-platform-tools:/mnt/android-platform-tools \
+    $(docker_image_of_emu_img ${EMU_IMG})
+ERR=0
+docker exec libcxx-ci-android-emulator emulator-wait-for-ready.sh || ERR=${?}
+echo "Emulator container initial logs:"
+docker logs libcxx-ci-android-emulator
+if [ ${ERR} != 0 ]; then
+    exit ${ERR}
+fi
+
+# Make sure the device is accessible from outside the emulator container and
+# advertise to the user that this script exists.
+. "${THIS_DIR}/setup-env-for-emulator.sh"
+adb wait-for-device
diff --git a/libcxx/utils/ci/vendor/android/stop-emulator.sh b/libcxx/utils/ci/vendor/android/stop-emulator.sh
new file mode 100755
index 0000000000..b5797ccb34
--- /dev/null
+++ b/libcxx/utils/ci/vendor/android/stop-emulator.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+#===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+#===----------------------------------------------------------------------===##
+
+set -e
+
+THIS_DIR="$(cd "$(dirname "$0")" && pwd)"
+. "${THIS_DIR}/emulator-functions.sh"
+
+# Cleanup the emulator if it's already running.
+if docker container inspect libcxx-ci-android-emulator &>/dev/null; then
+    echo "Stopping existing emulator container..."
+    docker stop libcxx-ci-android-emulator
+
+    echo "Emulator container final logs:"
+    docker logs libcxx-ci-android-emulator
+
+    echo "Removing existing emulator container..."
+    docker rm libcxx-ci-android-emulator
+fi
diff --git a/libcxx/utils/data/ignore_format.txt b/libcxx/utils/data/ignore_format.txt
index 01038c75f4..d4afd8c873 100644
--- a/libcxx/utils/data/ignore_format.txt
+++ b/libcxx/utils/data/ignore_format.txt
@@ -418,7 +418,6 @@ libcxx/include/regex
 libcxx/include/scoped_allocator
 libcxx/include/semaphore
 libcxx/include/set
-libcxx/include/setjmp.h
 libcxx/include/span
 libcxx/include/__split_buffer
 libcxx/include/sstream
@@ -526,7 +525,6 @@ libcxx/src/locale.cpp
 libcxx/src/memory.cpp
 libcxx/src/mutex.cpp
 libcxx/src/mutex_destructor.cpp
-libcxx/src/new.cpp
 libcxx/src/optional.cpp
 libcxx/src/pstl/libdispatch.cpp
 libcxx/src/random.cpp
@@ -2656,7 +2654,6 @@ libcxx/test/std/containers/views/mdspan/layout_left/index_operator.pass.cpp
 libcxx/test/std/containers/views/mdspan/layout_left/required_span_size.pass.cpp
 libcxx/test/std/containers/views/mdspan/layout_right/index_operator.pass.cpp
 libcxx/test/std/containers/views/mdspan/layout_right/required_span_size.pass.cpp
-libcxx/test/std/containers/views/mdspan/mdspan/CustomTestLayouts.h
 libcxx/test/std/containers/views/mdspan/mdspan/conversion.pass.cpp
 libcxx/test/std/containers/views/mdspan/mdspan/types.pass.cpp
 libcxx/test/std/containers/views/views.span/range_concept_conformance.compile.pass.cpp
@@ -2725,7 +2722,6 @@ libcxx/test/std/depr/depr.c.headers/inttypes_h.compile.pass.cpp
 libcxx/test/std/depr/depr.c.headers/limits_h.compile.pass.cpp
 libcxx/test/std/depr/depr.c.headers/locale_h.compile.pass.cpp
 libcxx/test/std/depr/depr.c.headers/math_h.pass.cpp
-libcxx/test/std/depr/depr.c.headers/setjmp_h.compile.pass.cpp
 libcxx/test/std/depr/depr.c.headers/signal_h.compile.pass.cpp
 libcxx/test/std/depr/depr.c.headers/stdarg_h.compile.pass.cpp
 libcxx/test/std/depr/depr.c.headers/stdbool_h.compile.pass.cpp
@@ -3480,10 +3476,8 @@ libcxx/test/std/input.output/string.streams/ostringstream/ostringstream.cons/str
 libcxx/test/std/input.output/string.streams/ostringstream/ostringstream.members/str.pass.cpp
 libcxx/test/std/input.output/string.streams/ostringstream/types.pass.cpp
 libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.assign/member_swap.pass.cpp
-libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.assign/move.pass.cpp
 libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.assign/nonmember_swap.pass.cpp
 libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.cons/default.pass.cpp
-libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.cons/move.pass.cpp
 libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.cons/string.pass.cpp
 libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.members/str.pass.cpp
 libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.virtuals/overflow.pass.cpp
@@ -3941,7 +3935,6 @@ libcxx/test/std/language.support/support.rtti/bad.typeid/bad_typeid.pass.cpp
 libcxx/test/std/language.support/support.rtti/type.info/type_info.equal.pass.cpp
 libcxx/test/std/language.support/support.rtti/type.info/type_info_hash.pass.cpp
 libcxx/test/std/language.support/support.rtti/type.info/type_info.pass.cpp
-libcxx/test/std/language.support/support.runtime/csetjmp.pass.cpp
 libcxx/test/std/language.support/support.runtime/csignal.pass.cpp
 libcxx/test/std/language.support/support.runtime/cstdarg.pass.cpp
 libcxx/test/std/language.support/support.runtime/cstdbool.pass.cpp
@@ -5265,7 +5258,6 @@ libcxx/test/std/ranges/range.adaptors/range.lazy.split/range.lazy.split.inner/ba
 libcxx/test/std/ranges/range.adaptors/range.lazy.split/range.lazy.split.inner/ctor.default.pass.cpp
 libcxx/test/std/ranges/range.adaptors/range.lazy.split/range.lazy.split.inner/ctor.outer_iterator.pass.cpp
 libcxx/test/std/ranges/range.adaptors/range.lazy.split/range.lazy.split.inner/equal.pass.cpp
-libcxx/test/std/ranges/range.adaptors/range.lazy.split/range.lazy.split.inner/increment.pass.cpp
 libcxx/test/std/ranges/range.adaptors/range.lazy.split/range.lazy.split.inner/iter_move.pass.cpp
 libcxx/test/std/ranges/range.adaptors/range.lazy.split/range.lazy.split.inner/iter_swap.pass.cpp
 libcxx/test/std/ranges/range.adaptors/range.lazy.split/range.lazy.split.inner/types.compile.pass.cpp
@@ -5275,11 +5267,8 @@ libcxx/test/std/ranges/range.adaptors/range.lazy.split/range.lazy.split.outer/ct
 libcxx/test/std/ranges/range.adaptors/range.lazy.split/range.lazy.split.outer/equal.pass.cpp
 libcxx/test/std/ranges/range.adaptors/range.lazy.split/range.lazy.split.outer/types.compile.pass.cpp
 libcxx/test/std/ranges/range.adaptors/range.lazy.split/range.lazy.split.outer.value/ctor.default.pass.cpp
-libcxx/test/std/ranges/range.adaptors/range.lazy.split/range.lazy.split.outer.value/view_interface.pass.cpp
 libcxx/test/std/ranges/range.adaptors/range.lazy.split/types.h
-libcxx/test/std/ranges/range.adaptors/range.lazy.split/view_interface.pass.cpp
 libcxx/test/std/ranges/range.adaptors/range.reverse/adaptor.pass.cpp
-libcxx/test/std/ranges/range.adaptors/range.reverse/base.pass.cpp
 libcxx/test/std/ranges/range.adaptors/range.reverse/begin.pass.cpp
 libcxx/test/std/ranges/range.adaptors/range.reverse/borrowing.compile.pass.cpp
 libcxx/test/std/ranges/range.adaptors/range.reverse/ctad.compile.pass.cpp
@@ -5300,7 +5289,6 @@ libcxx/test/std/ranges/range.adaptors/range.take/ctad.compile.pass.cpp
 libcxx/test/std/ranges/range.adaptors/range.take/ctor.default.pass.cpp
 libcxx/test/std/ranges/range.adaptors/range.take/range_concept_conformance.compile.pass.cpp
 libcxx/test/std/ranges/range.adaptors/range.take/sentinel/ctor.pass.cpp
-libcxx/test/std/ranges/range.adaptors/range.take/sentinel/eq.pass.cpp
 libcxx/test/std/ranges/range.adaptors/range.take/size.pass.cpp
 libcxx/test/std/ranges/range.adaptors/range.take/types.h
 libcxx/test/std/ranges/range.adaptors/range.take.while/adaptor.pass.cpp
@@ -5357,7 +5345,6 @@ libcxx/test/std/ranges/range.factories/range.iota.view/iterator/ctor.value.pass.
 libcxx/test/std/ranges/range.factories/range.iota.view/iterator/decrement.pass.cpp
 libcxx/test/std/ranges/range.factories/range.iota.view/iterator/increment.pass.cpp
 libcxx/test/std/ranges/range.factories/range.iota.view/iterator/member_typedefs.compile.pass.cpp
-libcxx/test/std/ranges/range.factories/range.iota.view/iterator/minus.pass.cpp
 libcxx/test/std/ranges/range.factories/range.iota.view/iterator/star.pass.cpp
 libcxx/test/std/ranges/range.factories/range.iota.view/iterator/subscript.pass.cpp
 libcxx/test/std/ranges/range.factories/range.iota.view/range_concept_conformance.compile.pass.cpp
@@ -7310,4 +7297,6 @@ libcxx/utils/ci/Dockerfile
 libcxx/utils/ci/macos-ci-setup
 libcxx/utils/ci/run-buildbot
 libcxx/utils/ci/run-buildbot-container
+libcxx/utils/ci/vendor/android/Dockerfile
+libcxx/utils/ci/vendor/android/run-buildbot-container
 libcxx/utils/libcxx-lit
diff --git a/libcxx/utils/generate_feature_test_macro_components.py b/libcxx/utils/generate_feature_test_macro_components.py
index 4b114d3824..76faa275d2 100755
--- a/libcxx/utils/generate_feature_test_macro_components.py
+++ b/libcxx/utils/generate_feature_test_macro_components.py
@@ -689,7 +689,6 @@ def add_version_header(tc):
             "name": "__cpp_lib_mdspan",
             "values": {"c++23": 202207},
             "headers": ["mdspan"],
-            "unimplemented": True,
         },
         {
             "name": "__cpp_lib_memory_resource",
diff --git a/libcxx/utils/libcxx/test/android.py b/libcxx/utils/libcxx/test/android.py
new file mode 100644
index 0000000000..29c681581b
--- /dev/null
+++ b/libcxx/utils/libcxx/test/android.py
@@ -0,0 +1,97 @@
+#===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+#===----------------------------------------------------------------------===##
+
+import atexit
+import os
+import re
+import select
+import socket
+import subprocess
+import tempfile
+import threading
+from typing import List
+
+
+def _get_cpu_count() -> int:
+    # Determine the number of cores by listing a /sys directory. Older devices
+    # lack `nproc`. Even if a static toybox binary is pushed to the device, it may
+    # return an incorrect value. (e.g. On a Nexus 7 running Android 5.0, toybox
+    # nproc returns 1 even though the device has 4 CPUs.)
+    job = subprocess.run(["adb", "shell", "ls /sys/devices/system/cpu"],
+                         encoding="utf8", check=False,
+                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    if job.returncode == 1:
+        # Maybe adb is missing, maybe ANDROID_SERIAL needs to be defined, maybe the
+        # /sys subdir isn't there. Most errors will be handled later, just use one
+        # job. (N.B. The adb command still succeeds even if ls fails on older
+        # devices that lack the shell_v2 adb feature.)
+        return 1
+    # Make sure there are no CR characters in the output. Pre-shell_v2, the adb
+    # stdout comes from a master pty so newlines are CRLF-delimited. On Windows,
+    # LF might also get expanded to CRLF.
+    cpu_listing = job.stdout.replace('\r', '\n')
+
+    # Count lines that match "cpu${DIGITS}".
+    result = len([line for line in cpu_listing.splitlines()
+                  if re.match(r'cpu(\d)+$', line)])
+
+    # Restrict the result to something reasonable.
+    if result < 1:
+        result = 1
+    if result > 1024:
+        result = 1024
+
+    return result
+
+
+def _job_limit_socket_thread(temp_dir: tempfile.TemporaryDirectory,
+                             server: socket.socket, job_count: int) -> None:
+    """Service the job limit server socket, accepting only as many connections
+    as there should be concurrent jobs.
+    """
+    clients: List[socket.socket] = []
+    while True:
+        rlist = list(clients)
+        if len(clients) < job_count:
+            rlist.append(server)
+        rlist, _, _ = select.select(rlist, [], [])
+        for sock in rlist:
+            if sock == server:
+                new_client, _ = server.accept()
+                new_client.send(b"x")
+                clients.append(new_client)
+            else:
+                sock.close()
+                clients.remove(sock)
+
+
+def adb_job_limit_socket() -> str:
+    """An Android device can frequently have many fewer cores than the host
+    (e.g. 4 versus 128). We want to exploit all the device cores without
+    overburdening it.
+
+    Create a Unix domain socket that only allows as many connections as CPUs on
+    the Android device.
+    """
+
+    # Create the job limit server socket.
+    temp_dir = tempfile.TemporaryDirectory(prefix="libcxx_")
+    sock_addr = temp_dir.name + "/adb_job.sock"
+    server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    server.bind(sock_addr)
+    server.listen(1)
+
+    # Spawn a thread to service the socket. As a daemon thread, its existence
+    # won't prevent interpreter shutdown. The temp dir will still be removed on
+    # shutdown.
+    cpu_count = _get_cpu_count()
+    threading.Thread(target=_job_limit_socket_thread,
+                     args=(temp_dir, server, cpu_count),
+                     daemon=True).start()
+
+    return sock_addr
diff --git a/libcxx/utils/libcxx/test/features.py b/libcxx/utils/libcxx/test/features.py
index 3779af1094..11f60a9620 100644
--- a/libcxx/utils/libcxx/test/features.py
+++ b/libcxx/utils/libcxx/test/features.py
@@ -43,6 +43,22 @@ def _getSuitableClangTidy(cfg):
         return None
 
 
+def _getAndroidDeviceApi(cfg):
+    return int(
+        programOutput(
+            cfg,
+            r"""
+                #include <android/api-level.h>
+                #include <stdio.h>
+                int main() {
+                    printf("%d\n", android_get_device_api_level());
+                    return 0;
+                }
+            """,
+        )
+    )
+
+
 DEFAULT_FEATURES = [
     Feature(
         name="thread-safety",
@@ -387,6 +403,15 @@ def _getSuitableClangTidy(cfg):
         actions=[AddCompileFlag("-DTEST_WINDOWS_DLL")],
     ),
     Feature(name="linux", when=lambda cfg: "__linux__" in compilerMacros(cfg)),
+    Feature(name="android", when=lambda cfg: "__ANDROID__" in compilerMacros(cfg)),
+    Feature(
+        name=lambda cfg: "android-device-api={}".format(_getAndroidDeviceApi(cfg)),
+        when=lambda cfg: "__ANDROID__" in compilerMacros(cfg),
+    ),
+    Feature(
+        name="LIBCXX-ANDROID-FIXME",
+        when=lambda cfg: "__ANDROID__" in compilerMacros(cfg),
+    ),
     Feature(name="netbsd", when=lambda cfg: "__NetBSD__" in compilerMacros(cfg)),
     Feature(name="freebsd", when=lambda cfg: "__FreeBSD__" in compilerMacros(cfg)),
     Feature(
diff --git a/libcxx/utils/libcxx/test/format.py b/libcxx/utils/libcxx/test/format.py
index c7c0bad681..4106cd83db 100644
--- a/libcxx/utils/libcxx/test/format.py
+++ b/libcxx/utils/libcxx/test/format.py
@@ -45,11 +45,12 @@ def _executeScriptInternal(test, litConfig, commands):
 
     _, tmpBase = _getTempPaths(test)
     execDir = os.path.dirname(test.getExecPath())
-    res = lit.TestRunner.executeScriptInternal(
-        test, litConfig, tmpBase, parsedCommands, execDir, debug=False
-    )
-    if isinstance(res, lit.Test.Result):  # Handle failure to parse the Lit test
-        res = ("", res.output, 127, None)
+    try:
+        res = lit.TestRunner.executeScriptInternal(
+            test, litConfig, tmpBase, parsedCommands, execDir, debug=False
+        )
+    except lit.TestRunner.ScriptFatal as e:
+        res = ("", str(e), 127, None)
     (out, err, exitCode, timeoutInfo) = res
 
     return (out, err, exitCode, timeoutInfo, parsedCommands)
diff --git a/libcxx/utils/libcxx/test/params.py b/libcxx/utils/libcxx/test/params.py
index 456794b9b1..e34fd0387f 100644
--- a/libcxx/utils/libcxx/test/params.py
+++ b/libcxx/utils/libcxx/test/params.py
@@ -77,9 +77,6 @@
 
 
 def getStdFlag(cfg, std):
-    # TODO(LLVM-17) Remove this clang-tidy-16 work-around
-    if std == "c++23":
-        std = "c++2b"
     if hasCompileFlag(cfg, "-std=" + std):
         return "-std=" + std
     # TODO(LLVM-19) Remove the fallbacks needed for Clang 16.
@@ -279,6 +276,14 @@ def getStdFlag(cfg, std):
         help="Whether to enable tests that take longer to run. This can be useful when running on a very slow device.",
         actions=lambda enabled: [] if not enabled else [AddFeature("long_tests")],
     ),
+    Parameter(
+        name="large_tests",
+        choices=[True, False],
+        type=bool,
+        default=True,
+        help="Whether to enable tests that use a lot of memory. This can be useful when running on a device with limited amounts of memory.",
+        actions=lambda enabled: [] if not enabled else [AddFeature("large_tests")],
+    ),
     Parameter(
         name="hardening_mode",
         choices=["unchecked", "hardened", "safe", "debug"],
diff --git a/libcxxabi/src/stdlib_new_delete.cpp b/libcxxabi/src/stdlib_new_delete.cpp
index 080f932ccc..6c9990f063 100644
--- a/libcxxabi/src/stdlib_new_delete.cpp
+++ b/libcxxabi/src/stdlib_new_delete.cpp
@@ -30,237 +30,167 @@
 // in this shared library, so that they can be overridden by programs
 // that define non-weak copies of the functions.
 
-_LIBCPP_WEAK
-void *
-operator new(std::size_t size) _THROW_BAD_ALLOC
-{
-    if (size == 0)
-        size = 1;
-    void* p;
-    while ((p = std::malloc(size)) == nullptr)
-    {
-        // If malloc fails and there is a new_handler,
-        // call it to try free up memory.
-        std::new_handler nh = std::get_new_handler();
-        if (nh)
-            nh();
-        else
+static void* operator_new_impl(std::size_t size) noexcept {
+  if (size == 0)
+    size = 1;
+  void* p;
+  while ((p = std::malloc(size)) == nullptr) {
+    // If malloc fails and there is a new_handler,
+    // call it to try free up memory.
+    std::new_handler nh = std::get_new_handler();
+    if (nh)
+      nh();
+    else
+      break;
+  }
+  return p;
+}
+
+_LIBCPP_WEAK
+void* operator new(std::size_t size) _THROW_BAD_ALLOC {
+  void* p = operator_new_impl(size);
 #ifndef _LIBCPP_HAS_NO_EXCEPTIONS
-            throw std::bad_alloc();
-#else
-            break;
+  if (p == nullptr)
+    throw std::bad_alloc();
 #endif
-    }
-    return p;
+  return p;
 }
 
 _LIBCPP_WEAK
-void*
-operator new(size_t size, const std::nothrow_t&) noexcept
-{
-    void* p = nullptr;
+void* operator new(size_t size, const std::nothrow_t&) noexcept {
+  void* p = nullptr;
 #ifndef _LIBCPP_HAS_NO_EXCEPTIONS
-    try
-    {
+  try {
 #endif // _LIBCPP_HAS_NO_EXCEPTIONS
-        p = ::operator new(size);
+    p = ::operator new(size);
 #ifndef _LIBCPP_HAS_NO_EXCEPTIONS
-    }
-    catch (...)
-    {
-    }
+  } catch (...) {
+  }
 #endif // _LIBCPP_HAS_NO_EXCEPTIONS
-    return p;
+  return p;
 }
 
 _LIBCPP_WEAK
-void*
-operator new[](size_t size) _THROW_BAD_ALLOC
-{
-    return ::operator new(size);
-}
+void* operator new[](size_t size) _THROW_BAD_ALLOC { return ::operator new(size); }
 
 _LIBCPP_WEAK
-void*
-operator new[](size_t size, const std::nothrow_t&) noexcept
-{
-    void* p = nullptr;
+void* operator new[](size_t size, const std::nothrow_t&) noexcept {
+  void* p = nullptr;
 #ifndef _LIBCPP_HAS_NO_EXCEPTIONS
-    try
-    {
+  try {
 #endif // _LIBCPP_HAS_NO_EXCEPTIONS
-        p = ::operator new[](size);
+    p = ::operator new[](size);
 #ifndef _LIBCPP_HAS_NO_EXCEPTIONS
-    }
-    catch (...)
-    {
-    }
+  } catch (...) {
+  }
 #endif // _LIBCPP_HAS_NO_EXCEPTIONS
-    return p;
+  return p;
 }
 
 _LIBCPP_WEAK
-void
-operator delete(void* ptr) noexcept
-{
-    std::free(ptr);
-}
+void operator delete(void* ptr) noexcept { std::free(ptr); }
 
 _LIBCPP_WEAK
-void
-operator delete(void* ptr, const std::nothrow_t&) noexcept
-{
-    ::operator delete(ptr);
-}
+void operator delete(void* ptr, const std::nothrow_t&) noexcept { ::operator delete(ptr); }
 
 _LIBCPP_WEAK
-void
-operator delete(void* ptr, size_t) noexcept
-{
-    ::operator delete(ptr);
-}
+void operator delete(void* ptr, size_t) noexcept { ::operator delete(ptr); }
 
 _LIBCPP_WEAK
-void
-operator delete[] (void* ptr) noexcept
-{
-    ::operator delete(ptr);
-}
+void operator delete[](void* ptr) noexcept { ::operator delete(ptr); }
 
 _LIBCPP_WEAK
-void
-operator delete[] (void* ptr, const std::nothrow_t&) noexcept
-{
-    ::operator delete[](ptr);
-}
+void operator delete[](void* ptr, const std::nothrow_t&) noexcept { ::operator delete[](ptr); }
 
 _LIBCPP_WEAK
-void
-operator delete[] (void* ptr, size_t) noexcept
-{
-    ::operator delete[](ptr);
-}
+void operator delete[](void* ptr, size_t) noexcept { ::operator delete[](ptr); }
 
 #if !defined(_LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION)
 
-_LIBCPP_WEAK
-void *
-operator new(std::size_t size, std::align_val_t alignment) _THROW_BAD_ALLOC
-{
-    if (size == 0)
-        size = 1;
-    if (static_cast<size_t>(alignment) < sizeof(void*))
-      alignment = std::align_val_t(sizeof(void*));
+static void* operator_new_aligned_impl(std::size_t size, std::align_val_t alignment) noexcept {
+  if (size == 0)
+    size = 1;
+  if (static_cast<size_t>(alignment) < sizeof(void*))
+    alignment = std::align_val_t(sizeof(void*));
 
-    // Try allocating memory. If allocation fails and there is a new_handler,
-    // call it to try free up memory, and try again until it succeeds, or until
-    // the new_handler decides to terminate.
-    //
-    // If allocation fails and there is no new_handler, we throw bad_alloc
-    // (or return nullptr if exceptions are disabled).
-    void* p;
-    while ((p = std::__libcpp_aligned_alloc(static_cast<std::size_t>(alignment), size)) == nullptr)
-    {
-        std::new_handler nh = std::get_new_handler();
-        if (nh)
-            nh();
-        else {
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
-            throw std::bad_alloc();
-#else
-            break;
-#endif
-        }
-    }
-    return p;
+  // Try allocating memory. If allocation fails and there is a new_handler,
+  // call it to try free up memory, and try again until it succeeds, or until
+  // the new_handler decides to terminate.
+  void* p;
+  while ((p = std::__libcpp_aligned_alloc(static_cast<std::size_t>(alignment), size)) == nullptr) {
+    std::new_handler nh = std::get_new_handler();
+    if (nh)
+      nh();
+    else
+      break;
+  }
+  return p;
 }
 
 _LIBCPP_WEAK
-void*
-operator new(size_t size, std::align_val_t alignment, const std::nothrow_t&) noexcept
-{
-    void* p = nullptr;
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
-    try
-    {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
-        p = ::operator new(size, alignment);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
-    }
-    catch (...)
-    {
-    }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
-    return p;
+void* operator new(std::size_t size, std::align_val_t alignment) _THROW_BAD_ALLOC {
+  void* p = operator_new_aligned_impl(size, alignment);
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  if (p == nullptr)
+    throw std::bad_alloc();
+#  endif
+  return p;
 }
 
 _LIBCPP_WEAK
-void*
-operator new[](size_t size, std::align_val_t alignment) _THROW_BAD_ALLOC
-{
-    return ::operator new(size, alignment);
+void* operator new(size_t size, std::align_val_t alignment, const std::nothrow_t&) noexcept {
+  void* p = nullptr;
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    p = ::operator new(size, alignment);
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+  }
+#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  return p;
 }
 
 _LIBCPP_WEAK
-void*
-operator new[](size_t size, std::align_val_t alignment, const std::nothrow_t&) noexcept
-{
-    void* p = nullptr;
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
-    try
-    {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
-        p = ::operator new[](size, alignment);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
-    }
-    catch (...)
-    {
-    }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
-    return p;
+void* operator new[](size_t size, std::align_val_t alignment) _THROW_BAD_ALLOC {
+  return ::operator new(size, alignment);
 }
 
 _LIBCPP_WEAK
-void
-operator delete(void* ptr, std::align_val_t) noexcept
-{
-    std::__libcpp_aligned_free(ptr);
+void* operator new[](size_t size, std::align_val_t alignment, const std::nothrow_t&) noexcept {
+  void* p = nullptr;
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    p = ::operator new[](size, alignment);
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+  }
+#  endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  return p;
 }
 
 _LIBCPP_WEAK
-void
-operator delete(void* ptr, std::align_val_t alignment, const std::nothrow_t&) noexcept
-{
-    ::operator delete(ptr, alignment);
-}
+void operator delete(void* ptr, std::align_val_t) noexcept { std::__libcpp_aligned_free(ptr); }
 
 _LIBCPP_WEAK
-void
-operator delete(void* ptr, size_t, std::align_val_t alignment) noexcept
-{
-    ::operator delete(ptr, alignment);
+void operator delete(void* ptr, std::align_val_t alignment, const std::nothrow_t&) noexcept {
+  ::operator delete(ptr, alignment);
 }
 
 _LIBCPP_WEAK
-void
-operator delete[] (void* ptr, std::align_val_t alignment) noexcept
-{
-    ::operator delete(ptr, alignment);
-}
+void operator delete(void* ptr, size_t, std::align_val_t alignment) noexcept { ::operator delete(ptr, alignment); }
 
 _LIBCPP_WEAK
-void
-operator delete[] (void* ptr, std::align_val_t alignment, const std::nothrow_t&) noexcept
-{
-    ::operator delete[](ptr, alignment);
-}
+void operator delete[](void* ptr, std::align_val_t alignment) noexcept { ::operator delete(ptr, alignment); }
 
 _LIBCPP_WEAK
-void
-operator delete[] (void* ptr, size_t, std::align_val_t alignment) noexcept
-{
-    ::operator delete[](ptr, alignment);
+void operator delete[](void* ptr, std::align_val_t alignment, const std::nothrow_t&) noexcept {
+  ::operator delete[](ptr, alignment);
 }
 
+_LIBCPP_WEAK
+void operator delete[](void* ptr, size_t, std::align_val_t alignment) noexcept { ::operator delete[](ptr, alignment); }
+
 #endif // !_LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION
 // ------------------ END COPY ------------------
diff --git a/libcxxabi/test/configs/llvm-libc++abi-android-ndk.cfg.in b/libcxxabi/test/configs/llvm-libc++abi-android-ndk.cfg.in
new file mode 100644
index 0000000000..f2cb62a32d
--- /dev/null
+++ b/libcxxabi/test/configs/llvm-libc++abi-android-ndk.cfg.in
@@ -0,0 +1,40 @@
+# This testing configuration handles running the test suite against LLVM's
+# libc++abi using adb and a libc++_shared.so library on Android.
+
+lit_config.load_config(config, '@CMAKE_CURRENT_BINARY_DIR@/cmake-bridge.cfg')
+
+import re
+import site
+
+site.addsitedir(os.path.join('@LIBCXX_SOURCE_DIR@', 'utils'))
+
+import libcxx.test.android
+import libcxx.test.config
+import libcxx.test.params
+
+config.substitutions.append(('%{flags}',
+    '--sysroot @CMAKE_SYSROOT@' if '@CMAKE_SYSROOT@' else ''
+))
+config.substitutions.append(('%{compile_flags}',
+    '-nostdinc++ -I %{include} -I %{cxx-include} -I %{cxx-target-include} %{maybe-include-libunwind} -I %{libcxx}/test/support -I %{libcxx}/src -D_LIBCPP_ENABLE_CXX17_REMOVED_UNEXPECTED_FUNCTIONS'
+))
+
+# The NDK library is called "libc++_shared.so". Use LD_LIBRARY_PATH to find
+# libc++_shared.so because older Bionic dynamic loaders don't support rpath
+# lookup. The Android libc++ shared library exports libc++abi, so we don't need
+# to link with -lc++abi.
+config.substitutions.append(('%{link_flags}',
+    '-nostdlib++ -L %{lib} -lc++_shared'
+))
+config.substitutions.append(('%{exec}',
+    '%{executor}' +
+    ' --job-limit-socket ' + libcxx.test.android.adb_job_limit_socket() +
+    ' --prepend-path-env LD_LIBRARY_PATH /data/local/tmp/libc++ --execdir %T -- '
+))
+
+libcxx.test.config.configure(
+    libcxx.test.params.DEFAULT_PARAMETERS,
+    libcxx.test.features.DEFAULT_FEATURES,
+    config,
+    lit_config
+)
diff --git a/libcxxabi/test/test_demangle.pass.cpp b/libcxxabi/test/test_demangle.pass.cpp
index df7bedc73c..77741a9528 100644
--- a/libcxxabi/test/test_demangle.pass.cpp
+++ b/libcxxabi/test/test_demangle.pass.cpp
@@ -14,6 +14,10 @@
 // UNSUPPORTED: stdlib=apple-libc++ && target={{.+}}-apple-macosx10.{{9|10|11|12|13|14|15}}
 // UNSUPPORTED: stdlib=apple-libc++ && target={{.+}}-apple-macosx11.0
 
+// Android's long double on x86[-64] is (64/128)-bits instead of Linux's usual
+// 80-bit format, and this demangling test is failing on it.
+// XFAIL: LIBCXX-ANDROID-FIXME && target={{i686|x86_64}}-{{.+}}-android{{.*}}
+
 #include "support/timer.h"
 #include <algorithm>
 #include <cassert>
diff --git a/libunwind/src/AddressSpace.hpp b/libunwind/src/AddressSpace.hpp
index 1abbc82254..5551c7d4be 100644
--- a/libunwind/src/AddressSpace.hpp
+++ b/libunwind/src/AddressSpace.hpp
@@ -414,8 +414,8 @@ static bool checkForUnwindInfoSegment(const Elf_Phdr *phdr, size_t image_base,
     cbdata->sects->dwarf_index_section = eh_frame_hdr_start;
     cbdata->sects->dwarf_index_section_length = phdr->p_memsz;
     if (EHHeaderParser<LocalAddressSpace>::decodeEHHdr(
-            *cbdata->addressSpace, eh_frame_hdr_start, phdr->p_memsz,
-            hdrInfo)) {
+            *cbdata->addressSpace, eh_frame_hdr_start,
+            eh_frame_hdr_start + phdr->p_memsz, hdrInfo)) {
       // .eh_frame_hdr records the start of .eh_frame, but not its size.
       // Rely on a zero terminator to find the end of the section.
       cbdata->sects->dwarf_section = hdrInfo.eh_frame_ptr;
@@ -638,7 +638,8 @@ inline bool LocalAddressSpace::findUnwindSections(pint_t targetAddr,
     info.dwarf_index_section_length = SIZE_MAX;
     EHHeaderParser<LocalAddressSpace>::EHHeaderInfo hdrInfo;
     if (!EHHeaderParser<LocalAddressSpace>::decodeEHHdr(
-            *this, info.dwarf_index_section, info.dwarf_index_section_length,
+            *this, info.dwarf_index_section,
+            info.dwarf_index_section + info.dwarf_index_section_length,
             hdrInfo)) {
       return false;
     }
diff --git a/libunwind/src/EHHeaderParser.hpp b/libunwind/src/EHHeaderParser.hpp
index ed4317c890..0662a1321e 100644
--- a/libunwind/src/EHHeaderParser.hpp
+++ b/libunwind/src/EHHeaderParser.hpp
@@ -55,6 +55,19 @@ template <typename A>
 bool EHHeaderParser<A>::decodeEHHdr(A &addressSpace, pint_t ehHdrStart,
                                     pint_t ehHdrEnd, EHHeaderInfo &ehHdrInfo) {
   pint_t p = ehHdrStart;
+
+  // Ensure that we don't read data beyond the end of .eh_frame_hdr
+  if (ehHdrEnd - ehHdrStart < 4) {
+    // Don't print a message for an empty .eh_frame_hdr (this can happen if
+    // the linker script defines symbols for it even in the empty case).
+    if (ehHdrEnd == ehHdrStart)
+      return false;
+    _LIBUNWIND_LOG("unsupported .eh_frame_hdr at %" PRIx64
+                   ": need at least 4 bytes of data but only got %zd",
+                   static_cast<uint64_t>(ehHdrStart),
+                   static_cast<size_t>(ehHdrEnd - ehHdrStart));
+    return false;
+  }
   uint8_t version = addressSpace.get8(p++);
   if (version != 1) {
     _LIBUNWIND_LOG("unsupported .eh_frame_hdr version: %" PRIu8 " at %" PRIx64,
diff --git a/libunwind/src/Registers.hpp b/libunwind/src/Registers.hpp
index fb6e04e50f..d11ddb3426 100644
--- a/libunwind/src/Registers.hpp
+++ b/libunwind/src/Registers.hpp
@@ -619,6 +619,8 @@ class _LIBUNWIND_HIDDEN Registers_ppc {
   void      setIP(uint32_t value) { _registers.__srr0 = value; }
   uint64_t  getCR() const         { return _registers.__cr; }
   void      setCR(uint32_t value) { _registers.__cr = value; }
+  uint64_t  getLR() const         { return _registers.__lr; }
+  void      setLR(uint32_t value) { _registers.__lr = value; }
 
 private:
   struct ppc_thread_state_t {
@@ -1189,6 +1191,8 @@ class _LIBUNWIND_HIDDEN Registers_ppc64 {
   void      setIP(uint64_t value) { _registers.__srr0 = value; }
   uint64_t  getCR() const         { return _registers.__cr; }
   void      setCR(uint64_t value) { _registers.__cr = value; }
+  uint64_t  getLR() const         { return _registers.__lr; }
+  void      setLR(uint64_t value) { _registers.__lr = value; }
 
 private:
   struct ppc64_thread_state_t {
diff --git a/libunwind/src/UnwindCursor.hpp b/libunwind/src/UnwindCursor.hpp
index dde94773bc..f89c5b2c2f 100644
--- a/libunwind/src/UnwindCursor.hpp
+++ b/libunwind/src/UnwindCursor.hpp
@@ -2301,27 +2301,39 @@ int UnwindCursor<A, R>::stepWithTBTable(pint_t pc, tbtable *TBTable,
     if (!getFunctionName(functionBuf, sizeof(functionBuf), &offset)) {
       functionName = ".anonymous.";
     }
-    _LIBUNWIND_TRACE_UNWINDING("%s: Look up traceback table of func=%s at %p",
-                               __func__, functionName,
-                               reinterpret_cast<void *>(TBTable));
+    _LIBUNWIND_TRACE_UNWINDING(
+        "%s: Look up traceback table of func=%s at %p, pc=%p, "
+        "SP=%p, saves_lr=%d, stores_bc=%d",
+        __func__, functionName, reinterpret_cast<void *>(TBTable),
+        reinterpret_cast<void *>(pc),
+        reinterpret_cast<void *>(registers.getSP()), TBTable->tb.saves_lr,
+        TBTable->tb.stores_bc);
   }
 
 #if defined(__powerpc64__)
-  // Instruction to reload TOC register "l r2,40(r1)"
+  // Instruction to reload TOC register "ld r2,40(r1)"
   const uint32_t loadTOCRegInst = 0xe8410028;
   const int32_t unwPPCF0Index = UNW_PPC64_F0;
   const int32_t unwPPCV0Index = UNW_PPC64_V0;
 #else
-  // Instruction to reload TOC register "l r2,20(r1)"
+  // Instruction to reload TOC register "lwz r2,20(r1)"
   const uint32_t loadTOCRegInst = 0x80410014;
   const int32_t unwPPCF0Index = UNW_PPC_F0;
   const int32_t unwPPCV0Index = UNW_PPC_V0;
 #endif
 
+  // lastStack points to the stack frame of the next routine up.
+  pint_t curStack = static_cast<pint_t>(registers.getSP());
+  pint_t lastStack = *reinterpret_cast<pint_t *>(curStack);
+
+  if (lastStack == 0)
+    return UNW_STEP_END;
+
   R newRegisters = registers;
 
-  // lastStack points to the stack frame of the next routine up.
-  pint_t lastStack = *(reinterpret_cast<pint_t *>(registers.getSP()));
+  // If backchain is not stored, use the current stack frame.
+  if (!TBTable->tb.stores_bc)
+    lastStack = curStack;
 
   // Return address is the address after call site instruction.
   pint_t returnAddress;
@@ -2331,33 +2343,41 @@ int UnwindCursor<A, R>::stepWithTBTable(pint_t pc, tbtable *TBTable,
                                reinterpret_cast<void *>(lastStack));
 
     sigcontext *sigContext = reinterpret_cast<sigcontext *>(
-        reinterpret_cast<char *>(lastStack) + STKMIN);
+        reinterpret_cast<char *>(lastStack) + STKMINALIGN);
     returnAddress = sigContext->sc_jmpbuf.jmp_context.iar;
 
-    _LIBUNWIND_TRACE_UNWINDING("From sigContext=%p, returnAddress=%p\n",
-                               reinterpret_cast<void *>(sigContext),
-                               reinterpret_cast<void *>(returnAddress));
-
+    bool useSTKMIN = false;
     if (returnAddress < 0x10000000) {
-      // Try again using STKMINALIGN
+      // Try again using STKMIN.
       sigContext = reinterpret_cast<sigcontext *>(
-          reinterpret_cast<char *>(lastStack) + STKMINALIGN);
+          reinterpret_cast<char *>(lastStack) + STKMIN);
       returnAddress = sigContext->sc_jmpbuf.jmp_context.iar;
       if (returnAddress < 0x10000000) {
-        _LIBUNWIND_TRACE_UNWINDING("Bad returnAddress=%p\n",
-                                   reinterpret_cast<void *>(returnAddress));
+        _LIBUNWIND_TRACE_UNWINDING("Bad returnAddress=%p from sigcontext=%p",
+                                   reinterpret_cast<void *>(returnAddress),
+                                   reinterpret_cast<void *>(sigContext));
         return UNW_EBADFRAME;
-      } else {
-        _LIBUNWIND_TRACE_UNWINDING("Tried again using STKMINALIGN: "
-                                   "sigContext=%p, returnAddress=%p. "
-                                   "Seems to be a valid address\n",
-                                   reinterpret_cast<void *>(sigContext),
-                                   reinterpret_cast<void *>(returnAddress));
       }
+      useSTKMIN = true;
     }
+    _LIBUNWIND_TRACE_UNWINDING("Returning from a signal handler %s: "
+                               "sigContext=%p, returnAddress=%p. "
+                               "Seems to be a valid address",
+                               useSTKMIN ? "STKMIN" : "STKMINALIGN",
+                               reinterpret_cast<void *>(sigContext),
+                               reinterpret_cast<void *>(returnAddress));
+
     // Restore the condition register from sigcontext.
     newRegisters.setCR(sigContext->sc_jmpbuf.jmp_context.cr);
 
+    // Save the LR in sigcontext for stepping up when the function that
+    // raised the signal is a leaf function. This LR has the return address
+    // to the caller of the leaf function.
+    newRegisters.setLR(sigContext->sc_jmpbuf.jmp_context.lr);
+    _LIBUNWIND_TRACE_UNWINDING(
+        "Save LR=%p from sigcontext",
+        reinterpret_cast<void *>(sigContext->sc_jmpbuf.jmp_context.lr));
+
     // Restore GPRs from sigcontext.
     for (int i = 0; i < 32; ++i)
       newRegisters.setRegister(i, sigContext->sc_jmpbuf.jmp_context.gpr[i]);
@@ -2380,13 +2400,26 @@ int UnwindCursor<A, R>::stepWithTBTable(pint_t pc, tbtable *TBTable,
     }
   } else {
     // Step up a normal frame.
-    returnAddress = reinterpret_cast<pint_t *>(lastStack)[2];
 
-    _LIBUNWIND_TRACE_UNWINDING("Extract info from lastStack=%p, "
-                               "returnAddress=%p\n",
-                               reinterpret_cast<void *>(lastStack),
-                               reinterpret_cast<void *>(returnAddress));
-    _LIBUNWIND_TRACE_UNWINDING("fpr_regs=%d, gpr_regs=%d, saves_cr=%d\n",
+    if (!TBTable->tb.saves_lr && registers.getLR()) {
+      // This case should only occur if we were called from a signal handler
+      // and the signal occurred in a function that doesn't save the LR.
+      returnAddress = registers.getLR();
+      _LIBUNWIND_TRACE_UNWINDING("Use saved LR=%p",
+                                 reinterpret_cast<void *>(returnAddress));
+    } else {
+      // Otherwise, use the LR value in the stack link area.
+      returnAddress = reinterpret_cast<pint_t *>(lastStack)[2];
+    }
+
+    // Reset LR in the current context.
+    newRegisters.setLR(NULL);
+
+    _LIBUNWIND_TRACE_UNWINDING(
+        "Extract info from lastStack=%p, returnAddress=%p",
+        reinterpret_cast<void *>(lastStack),
+        reinterpret_cast<void *>(returnAddress));
+    _LIBUNWIND_TRACE_UNWINDING("fpr_regs=%d, gpr_regs=%d, saves_cr=%d",
                                TBTable->tb.fpr_saved, TBTable->tb.gpr_saved,
                                TBTable->tb.saves_cr);
 
@@ -2450,7 +2483,7 @@ int UnwindCursor<A, R>::stepWithTBTable(pint_t pc, tbtable *TBTable,
 
       struct vec_ext *vec_ext = reinterpret_cast<struct vec_ext *>(charPtr);
 
-      _LIBUNWIND_TRACE_UNWINDING("vr_saved=%d\n", vec_ext->vr_saved);
+      _LIBUNWIND_TRACE_UNWINDING("vr_saved=%d", vec_ext->vr_saved);
 
       // Restore vector register(s) if saved on the stack.
       if (vec_ext->vr_saved) {
@@ -2480,11 +2513,11 @@ int UnwindCursor<A, R>::stepWithTBTable(pint_t pc, tbtable *TBTable,
 
     // Do we need to set the TOC register?
     _LIBUNWIND_TRACE_UNWINDING(
-        "Current gpr2=%p\n",
+        "Current gpr2=%p",
         reinterpret_cast<void *>(newRegisters.getRegister(2)));
     if (firstInstruction == loadTOCRegInst) {
       _LIBUNWIND_TRACE_UNWINDING(
-          "Set gpr2=%p from frame\n",
+          "Set gpr2=%p from frame",
           reinterpret_cast<void *>(reinterpret_cast<pint_t *>(lastStack)[5]));
       newRegisters.setRegister(2, reinterpret_cast<pint_t *>(lastStack)[5]);
     }
@@ -2516,7 +2549,6 @@ int UnwindCursor<A, R>::stepWithTBTable(pint_t pc, tbtable *TBTable,
   } else {
     isSignalFrame = false;
   }
-
   return UNW_STEP_SUCCESS;
 }
 #endif // defined(_LIBUNWIND_SUPPORT_TBTAB_UNWIND)
diff --git a/libunwind/src/UnwindRegistersSave.S b/libunwind/src/UnwindRegistersSave.S
index 5534d1734b..dc0f7da31c 100644
--- a/libunwind/src/UnwindRegistersSave.S
+++ b/libunwind/src/UnwindRegistersSave.S
@@ -352,7 +352,12 @@ LnoR2Fix:
   std   0,  PPC64_OFFS_CR(3)
   mfxer 0
   std   0,  PPC64_OFFS_XER(3)
+#if defined(_AIX)
+  // LR value saved from the register is not used, initialize it to 0.
+  li    0,  0
+#else
   mflr  0
+#endif
   std   0,  PPC64_OFFS_LR(3)
   mfctr 0
   std   0,  PPC64_OFFS_CTR(3)
@@ -565,8 +570,8 @@ DEFINE_LIBUNWIND_FUNCTION(__unw_getcontext)
   // is called from a different module. Save the original TOC register
   // in the context if this is the case.
   mflr    4
-  lwz     4,   0(4)     // Get the instruction at the return address.
-  xoris   0,  4, 0x8041 // Is it reloading the TOC register "ld 2,40(1)"?
+  lwz     4,  0(4)      // Get the instruction at the return address.
+  xoris   0,  4, 0x8041 // Is it reloading the TOC register "lwz 2,20(1)"?
   cmplwi  0,  0x14
   bne     0,  LnoR2Fix  // No need to fix up r2 if it is not.
   lwz     2,  20(1)     // Use the saved TOC register in the stack.
@@ -610,6 +615,11 @@ LnoR2Fix:
   // save CR registers
   mfcr    0
   stw     0, 136(3)
+#if defined(_AIX)
+  // LR value from the register is not used, initialize it to 0.
+  li      0, 0
+  stw     0, 144(3)
+#endif
   // save CTR register
   mfctr   0
   stw     0, 148(3)
diff --git a/libunwind/test/aix_signal_unwind.pass.sh.S b/libunwind/test/aix_signal_unwind.pass.sh.S
new file mode 100644
index 0000000000..9ca18e9481
--- /dev/null
+++ b/libunwind/test/aix_signal_unwind.pass.sh.S
@@ -0,0 +1,245 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Test that _Unwind_Backtrace() walks up from a signal handler and produces
+// a correct traceback when the function raising the signal does not save
+// the link register or does not store the stack back chain.
+
+// REQUIRES: target=powerpc{{(64)?}}-ibm-aix
+
+// Test when the function raising the signal does not save the link register
+// RUN: %{cxx} -x c++ %s -o %t.exe -DCXX_CODE %{flags} %{compile_flags}
+// RUN: %{exec} %t.exe
+
+// Test when the function raising the signal does not store stack back chain.
+// RUN: %{cxx} -x c++ -c %s -o %t1.o -DCXX_CODE -DNOBACKCHAIN %{flags} \
+// RUN:   %{compile_flags}
+// RUN: %{cxx} -c %s -o %t2.o %{flags} %{compile_flags}
+// RUN: %{cxx} -o %t1.exe %t1.o %t2.o %{flags} %{link_flags}
+// RUN: %{exec} %t1.exe
+
+#ifdef CXX_CODE
+
+#undef NDEBUG
+#include <assert.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/debug.h>
+#include <unwind.h>
+
+#define NAME_ARRAY_SIZE 10
+#define NAMES_EXPECTED   6
+
+const char* namesExpected[] = {"handler", "abc", "bar", "foo", "main",
+                               "__start"};
+char *namesObtained[NAME_ARRAY_SIZE] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+
+int funcIndex = 0;
+
+// Get the function name from traceback table.
+char *getFuncName(uintptr_t pc, uint16_t *nameLen) {
+  uint32_t *p = reinterpret_cast<uint32_t *>(pc);
+
+  // Keep looking forward until a word of 0 is found. The traceback
+  // table starts at the following word.
+  while (*p)
+    ++p;
+  tbtable *TBTable = reinterpret_cast<tbtable *>(p + 1);
+
+  if (!TBTable->tb.name_present)
+    return NULL;
+
+  // Get to the optional portion of the traceback table.
+  p = reinterpret_cast<uint32_t *>(&TBTable->tb_ext);
+
+  // Skip field parminfo if it exists.
+  if (TBTable->tb.fixedparms || TBTable->tb.floatparms)
+    ++p;
+
+  // Skip field tb_offset if it exists.
+  if (TBTable->tb.has_tboff)
+    ++p;
+
+  // Skip field hand_mask if it exists.
+  if (TBTable->tb.int_hndl)
+    ++p;
+
+  // Skip fields ctl_info and ctl_info_disp if they exist.
+  if (TBTable->tb.has_ctl)
+    p += 1 + *p;
+
+  *nameLen = *reinterpret_cast<uint16_t *>(p);
+  return reinterpret_cast<char *>(p) + sizeof(uint16_t);
+}
+
+_Unwind_Reason_Code callBack(struct _Unwind_Context *uc, void *arg) {
+  (void)arg;
+  uint16_t nameLen;
+  uintptr_t ip = _Unwind_GetIP(uc);
+  if (funcIndex < NAME_ARRAY_SIZE)
+    namesObtained[funcIndex++] = strndup(getFuncName(ip, &nameLen), nameLen);
+  return _URC_NO_REASON;
+}
+
+extern "C" void handler(int signum) {
+  (void)signum;
+  // Walk stack frames for traceback.
+  _Unwind_Backtrace(callBack, NULL);
+
+  // Verify the traceback.
+  assert(funcIndex <= NAMES_EXPECTED && "Obtained names more than expected");
+  for (int i = 0; i < funcIndex; ++i) {
+    assert(!strcmp(namesExpected[i], namesObtained[i]) &&
+           "Function names do not match");
+    free(namesObtained[i]);
+  }
+  exit(0);
+}
+
+#ifdef NOBACKCHAIN
+// abc() is in assembly. It raises signal SIGSEGV and does not store
+// the stack back chain.
+extern "C" void abc();
+
+#else
+volatile int *null = 0;
+
+// abc() raises signal SIGSEGV and does not save the link register.
+extern "C" __attribute__((noinline)) void abc() {
+  // Produce a SIGSEGV.
+  *null = 0;
+}
+#endif
+
+extern "C" __attribute__((noinline)) void bar() {
+  abc();
+}
+
+extern "C" __attribute__((noinline)) void foo() {
+  bar();
+}
+
+int main() {
+  // Set signal handler for SIGSEGV.
+  signal(SIGSEGV, handler);
+  foo();
+}
+
+#else // Assembly code for abc().
+// This assembly code is similar to the following C code but it saves the
+// link register.
+//
+// int *badp = 0;
+// void abc() {
+//   *badp = 0;
+// }
+
+#ifdef __64BIT__
+        .csect [PR],5
+        .file   "abc.c"
+        .globl  abc[DS]                 # -- Begin function abc
+        .globl  .abc
+        .align  4
+        .csect abc[DS],3
+        .vbyte  8, .abc                 # @abc
+        .vbyte  8, TOC[TC0]
+        .vbyte  8, 0
+        .csect [PR],5
+.abc:
+# %bb.0:                                # %entry
+        mflr 0
+        std 0, 16(1)
+        ld 3, L..C0(2)                  # @badp
+        bl $+4
+        ld 4, 0(3)
+        li 3, 0
+        stw 3, 0(4)
+        ld 0, 16(1)
+        mtlr 0
+        blr
+L..abc0:
+        .vbyte  4, 0x00000000           # Traceback table begin
+        .byte   0x00                    # Version = 0
+        .byte   0x09                    # Language = CPlusPlus
+        .byte   0x20                    # -IsGlobaLinkage, -IsOutOfLineEpilogOrPrologue
+                                        # +HasTraceBackTableOffset, -IsInternalProcedure
+                                        # -HasControlledStorage, -IsTOCless
+                                        # -IsFloatingPointPresent
+                                        # -IsFloatingPointOperationLogOrAbortEnabled
+        .byte   0x61                    # -IsInterruptHandler, +IsFunctionNamePresent, +IsAllocaUsed
+                                        # OnConditionDirective = 0, -IsCRSaved, +IsLRSaved
+        .byte   0x00                    # -IsBackChainStored, -IsFixup, NumOfFPRsSaved = 0
+        .byte   0x01                    # -HasExtensionTable, -HasVectorInfo, NumOfGPRsSaved = 1
+        .byte   0x00                    # NumberOfFixedParms = 0
+        .byte   0x01                    # NumberOfFPParms = 0, +HasParmsOnStack
+        .vbyte  4, L..abc0-.abc         # Function size
+        .vbyte  2, 0x0003               # Function name len = 3
+        .byte   "abc"                   # Function Name
+        .byte   0x1f                    # AllocaUsed
+                                        # -- End function
+        .csect badp[RW],3
+        .globl  badp[RW]                # @badp
+        .align  3
+        .vbyte  8, 0
+        .toc
+L..C0:
+        .tc badp[TC],badp[RW]
+#else
+        .csect [PR],5
+        .file   "abc.c"
+        .globl  abc[DS]                 # -- Begin function abc
+        .globl  .abc
+        .align  4
+        .csect abc[DS],2
+        .vbyte  4, .abc                 # @abc
+        .vbyte  4, TOC[TC0]
+        .vbyte  4, 0
+        .csect [PR],5
+.abc:
+# %bb.0:                                # %entry
+        mflr 0
+        stw 0, 8(1)
+        lwz 3, L..C0(2)                 # @badp
+        bl $+4
+        lwz 4, 0(3)
+        li 3, 0
+        stw 3, 0(4)
+        lwz 0, 8(1)
+        mtlr 0
+        blr
+L..abc0:
+        .vbyte  4, 0x00000000           # Traceback table begin
+        .byte   0x00                    # Version = 0
+        .byte   0x09                    # Language = CPlusPlus
+        .byte   0x20                    # -IsGlobaLinkage, -IsOutOfLineEpilogOrPrologue
+                                        # +HasTraceBackTableOffset, -IsInternalProcedure
+                                        # -HasControlledStorage, -IsTOCless
+                                        # -IsFloatingPointPresent
+                                        # -IsFloatingPointOperationLogOrAbortEnabled
+        .byte   0x61                    # -IsInterruptHandler, +IsFunctionNamePresent, +IsAllocaUsed
+                                        # OnConditionDirective = 0, -IsCRSaved, +IsLRSaved
+        .byte   0x00                    # -IsBackChainStored, -IsFixup, NumOfFPRsSaved = 0
+        .byte   0x01                    # -HasExtensionTable, -HasVectorInfo, NumOfGPRsSaved = 1
+        .byte   0x00                    # NumberOfFixedParms = 0
+        .byte   0x01                    # NumberOfFPParms = 0, +HasParmsOnStack
+        .vbyte  4, L..abc0-.abc         # Function size
+        .vbyte  2, 0x0003               # Function name len = 3
+        .byte   "abc"                   # Function Name
+        .byte   0x1f                    # AllocaUsed
+                                        # -- End function
+        .csect badp[RW],2
+        .globl  badp[RW]                # @badp
+        .align  2
+        .vbyte  4, 0
+        .toc
+L..C0:
+        .tc badp[TC],badp[RW]
+#endif // __64BIT__
+#endif // CXX_CODE
diff --git a/lld/COFF/Chunks.cpp b/lld/COFF/Chunks.cpp
index e17b64df86..4e845afa89 100644
--- a/lld/COFF/Chunks.cpp
+++ b/lld/COFF/Chunks.cpp
@@ -437,7 +437,7 @@ void SectionChunk::applyRelocation(uint8_t *off,
   // Compute the RVA of the relocation for relative relocations.
   uint64_t p = rva + rel.VirtualAddress;
   uint64_t imageBase = file->ctx.config.imageBase;
-  switch (file->ctx.config.machine) {
+  switch (getMachine()) {
   case AMD64:
     applyRelX64(off, rel.Type, os, s, p, imageBase);
     break;
@@ -551,7 +551,7 @@ static uint8_t getBaserelType(const coff_relocation &rel,
 // Only called when base relocation is enabled.
 void SectionChunk::getBaserels(std::vector<Baserel> *res) {
   for (const coff_relocation &rel : getRelocs()) {
-    uint8_t ty = getBaserelType(rel, file->ctx.config.machine);
+    uint8_t ty = getBaserelType(rel, getMachine());
     if (ty == IMAGE_REL_BASED_ABSOLUTE)
       continue;
     Symbol *target = file->getSymbol(rel.SymbolTableIndex);
diff --git a/lld/COFF/Chunks.h b/lld/COFF/Chunks.h
index 3d605e6ab1..cbfeb5c025 100644
--- a/lld/COFF/Chunks.h
+++ b/lld/COFF/Chunks.h
@@ -24,10 +24,11 @@
 namespace lld::coff {
 
 using llvm::COFF::ImportDirectoryTableEntry;
-using llvm::object::COFFSymbolRef;
-using llvm::object::SectionRef;
+using llvm::object::chpe_range_type;
 using llvm::object::coff_relocation;
 using llvm::object::coff_section;
+using llvm::object::COFFSymbolRef;
+using llvm::object::SectionRef;
 
 class Baserel;
 class Defined;
@@ -114,6 +115,9 @@ class Chunk {
   // synthesized by the linker.
   bool isHotPatchable() const;
 
+  MachineTypes getMachine() const;
+  chpe_range_type getArm64ECRangeType() const;
+
 protected:
   Chunk(Kind k = OtherKind) : chunkKind(k), hasData(true), p2Align(0) {}
 
@@ -164,6 +168,8 @@ class NonSectionChunk : public Chunk {
   // Collect all locations that contain absolute addresses for base relocations.
   virtual void getBaserels(std::vector<Baserel> *res) {}
 
+  virtual MachineTypes getMachine() const { return IMAGE_FILE_MACHINE_UNKNOWN; }
+
   // Returns a human-readable name of this chunk. Chunks are unnamed chunks of
   // bytes, so this is used only for logging or debugging.
   virtual StringRef getDebugName() const { return ""; }
@@ -219,6 +225,8 @@ class SectionChunk final : public Chunk {
   ArrayRef<uint8_t> getContents() const;
   void writeTo(uint8_t *buf) const;
 
+  MachineTypes getMachine() const { return file->getMachineType(); }
+
   // Defend against unsorted relocations. This may be overly conservative.
   void sortRelocations();
 
@@ -378,16 +386,13 @@ class SectionChunk final : public Chunk {
 inline size_t Chunk::getSize() const {
   if (isa<SectionChunk>(this))
     return static_cast<const SectionChunk *>(this)->getSize();
-  else
-    return static_cast<const NonSectionChunk *>(this)->getSize();
+  return static_cast<const NonSectionChunk *>(this)->getSize();
 }
 
 inline uint32_t Chunk::getOutputCharacteristics() const {
   if (isa<SectionChunk>(this))
     return static_cast<const SectionChunk *>(this)->getOutputCharacteristics();
-  else
-    return static_cast<const NonSectionChunk *>(this)
-        ->getOutputCharacteristics();
+  return static_cast<const NonSectionChunk *>(this)->getOutputCharacteristics();
 }
 
 inline void Chunk::writeTo(uint8_t *buf) const {
@@ -400,8 +405,7 @@ inline void Chunk::writeTo(uint8_t *buf) const {
 inline StringRef Chunk::getSectionName() const {
   if (isa<SectionChunk>(this))
     return static_cast<const SectionChunk *>(this)->getSectionName();
-  else
-    return static_cast<const NonSectionChunk *>(this)->getSectionName();
+  return static_cast<const NonSectionChunk *>(this)->getSectionName();
 }
 
 inline void Chunk::getBaserels(std::vector<Baserel> *res) {
@@ -414,8 +418,24 @@ inline void Chunk::getBaserels(std::vector<Baserel> *res) {
 inline StringRef Chunk::getDebugName() const {
   if (isa<SectionChunk>(this))
     return static_cast<const SectionChunk *>(this)->getDebugName();
-  else
-    return static_cast<const NonSectionChunk *>(this)->getDebugName();
+  return static_cast<const NonSectionChunk *>(this)->getDebugName();
+}
+
+inline MachineTypes Chunk::getMachine() const {
+  if (isa<SectionChunk>(this))
+    return static_cast<const SectionChunk *>(this)->getMachine();
+  return static_cast<const NonSectionChunk *>(this)->getMachine();
+}
+
+inline chpe_range_type Chunk::getArm64ECRangeType() const {
+  switch (getMachine()) {
+  case AMD64:
+    return chpe_range_type::Amd64;
+  case ARM64EC:
+    return chpe_range_type::Arm64EC;
+  default:
+    return chpe_range_type::Arm64;
+  }
 }
 
 // This class is used to implement an lld-specific feature (not implemented in
@@ -504,6 +524,7 @@ class ImportThunkChunkX64 : public ImportThunkChunk {
   explicit ImportThunkChunkX64(COFFLinkerContext &ctx, Defined *s);
   size_t getSize() const override { return sizeof(importThunkX86); }
   void writeTo(uint8_t *buf) const override;
+  MachineTypes getMachine() const override { return AMD64; }
 };
 
 class ImportThunkChunkX86 : public ImportThunkChunk {
@@ -513,6 +534,7 @@ class ImportThunkChunkX86 : public ImportThunkChunk {
   size_t getSize() const override { return sizeof(importThunkX86); }
   void getBaserels(std::vector<Baserel> *res) override;
   void writeTo(uint8_t *buf) const override;
+  MachineTypes getMachine() const override { return I386; }
 };
 
 class ImportThunkChunkARM : public ImportThunkChunk {
@@ -524,6 +546,7 @@ class ImportThunkChunkARM : public ImportThunkChunk {
   size_t getSize() const override { return sizeof(importThunkARM); }
   void getBaserels(std::vector<Baserel> *res) override;
   void writeTo(uint8_t *buf) const override;
+  MachineTypes getMachine() const override { return ARMNT; }
 };
 
 class ImportThunkChunkARM64 : public ImportThunkChunk {
@@ -534,6 +557,7 @@ class ImportThunkChunkARM64 : public ImportThunkChunk {
   }
   size_t getSize() const override { return sizeof(importThunkARM64); }
   void writeTo(uint8_t *buf) const override;
+  MachineTypes getMachine() const override { return ARM64; }
 };
 
 class RangeExtensionThunkARM : public NonSectionChunk {
@@ -544,6 +568,7 @@ class RangeExtensionThunkARM : public NonSectionChunk {
   }
   size_t getSize() const override;
   void writeTo(uint8_t *buf) const override;
+  MachineTypes getMachine() const override { return ARMNT; }
 
   Defined *target;
 
@@ -559,6 +584,7 @@ class RangeExtensionThunkARM64 : public NonSectionChunk {
   }
   size_t getSize() const override;
   void writeTo(uint8_t *buf) const override;
+  MachineTypes getMachine() const override { return ARM64; }
 
   Defined *target;
 
diff --git a/lld/COFF/Config.h b/lld/COFF/Config.h
index 2d5d4d63d6..1c338cc63f 100644
--- a/lld/COFF/Config.h
+++ b/lld/COFF/Config.h
@@ -316,6 +316,7 @@ struct Configuration {
   bool stdcallFixup = false;
   bool writeCheckSum = false;
   EmitKind emit = EmitKind::Obj;
+  bool allowDuplicateWeak = false;
 };
 
 } // namespace lld::coff
diff --git a/lld/COFF/DLL.cpp b/lld/COFF/DLL.cpp
index 5977970104..0b337a209c 100644
--- a/lld/COFF/DLL.cpp
+++ b/lld/COFF/DLL.cpp
@@ -318,6 +318,7 @@ class ThunkChunkX64 : public NonSectionChunk {
   ThunkChunkX64(Defined *i, Chunk *tm) : imp(i), tailMerge(tm) {}
 
   size_t getSize() const override { return sizeof(thunkX64); }
+  MachineTypes getMachine() const override { return AMD64; }
 
   void writeTo(uint8_t *buf) const override {
     memcpy(buf, thunkX64, sizeof(thunkX64));
@@ -334,6 +335,7 @@ class TailMergeChunkX64 : public NonSectionChunk {
   TailMergeChunkX64(Chunk *d, Defined *h) : desc(d), helper(h) {}
 
   size_t getSize() const override { return sizeof(tailMergeX64); }
+  MachineTypes getMachine() const override { return AMD64; }
 
   void writeTo(uint8_t *buf) const override {
     memcpy(buf, tailMergeX64, sizeof(tailMergeX64));
@@ -386,6 +388,7 @@ class ThunkChunkX86 : public NonSectionChunk {
       : imp(i), tailMerge(tm), ctx(ctx) {}
 
   size_t getSize() const override { return sizeof(thunkX86); }
+  MachineTypes getMachine() const override { return I386; }
 
   void writeTo(uint8_t *buf) const override {
     memcpy(buf, thunkX86, sizeof(thunkX86));
@@ -410,6 +413,7 @@ class TailMergeChunkX86 : public NonSectionChunk {
       : desc(d), helper(h), ctx(ctx) {}
 
   size_t getSize() const override { return sizeof(tailMergeX86); }
+  MachineTypes getMachine() const override { return I386; }
 
   void writeTo(uint8_t *buf) const override {
     memcpy(buf, tailMergeX86, sizeof(tailMergeX86));
@@ -436,6 +440,7 @@ class ThunkChunkARM : public NonSectionChunk {
   }
 
   size_t getSize() const override { return sizeof(thunkARM); }
+  MachineTypes getMachine() const override { return ARMNT; }
 
   void writeTo(uint8_t *buf) const override {
     memcpy(buf, thunkARM, sizeof(thunkARM));
@@ -462,6 +467,7 @@ class TailMergeChunkARM : public NonSectionChunk {
   }
 
   size_t getSize() const override { return sizeof(tailMergeARM); }
+  MachineTypes getMachine() const override { return ARMNT; }
 
   void writeTo(uint8_t *buf) const override {
     memcpy(buf, tailMergeARM, sizeof(tailMergeARM));
@@ -487,6 +493,7 @@ class ThunkChunkARM64 : public NonSectionChunk {
   }
 
   size_t getSize() const override { return sizeof(thunkARM64); }
+  MachineTypes getMachine() const override { return ARM64; }
 
   void writeTo(uint8_t *buf) const override {
     memcpy(buf, thunkARM64, sizeof(thunkARM64));
@@ -506,6 +513,7 @@ class TailMergeChunkARM64 : public NonSectionChunk {
   }
 
   size_t getSize() const override { return sizeof(tailMergeARM64); }
+  MachineTypes getMachine() const override { return ARM64; }
 
   void writeTo(uint8_t *buf) const override {
     memcpy(buf, tailMergeARM64, sizeof(tailMergeARM64));
diff --git a/lld/COFF/DebugTypes.cpp b/lld/COFF/DebugTypes.cpp
index 5071b7b79d..a4c808e4c9 100644
--- a/lld/COFF/DebugTypes.cpp
+++ b/lld/COFF/DebugTypes.cpp
@@ -311,7 +311,7 @@ Error TpiSource::mergeDebugT(TypeMerger *m) {
          "use remapTpiWithGHashes when ghash is enabled");
 
   CVTypeArray types;
-  BinaryStreamReader reader(file->debugTypes, support::little);
+  BinaryStreamReader reader(file->debugTypes, llvm::endianness::little);
   cantFail(reader.readArray(types, reader.getLength()));
 
   // When dealing with PCH.OBJ, some indices were already merged.
@@ -588,7 +588,7 @@ void TpiSource::loadGHashes() {
     ownedGHashes = false;
   } else {
     CVTypeArray types;
-    BinaryStreamReader reader(file->debugTypes, support::little);
+    BinaryStreamReader reader(file->debugTypes, llvm::endianness::little);
     cantFail(reader.readArray(types, reader.getLength()));
     assignGHashesFromVector(GloballyHashedType::hashTypes(types));
   }
diff --git a/lld/COFF/Driver.cpp b/lld/COFF/Driver.cpp
index c7f4e1f767..5613c2e699 100644
--- a/lld/COFF/Driver.cpp
+++ b/lld/COFF/Driver.cpp
@@ -1210,7 +1210,7 @@ static void readCallGraphsFromObjectFiles(COFFLinkerContext &ctx) {
       ArrayRef<uint8_t> contents;
       cantFail(
           obj->getCOFFObj()->getSectionContents(obj->callgraphSec, contents));
-      BinaryStreamReader reader(contents, support::little);
+      BinaryStreamReader reader(contents, llvm::endianness::little);
       while (!reader.empty()) {
         uint32_t fromIndex, toIndex;
         uint64_t count;
@@ -1588,13 +1588,25 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
   {
     llvm::TimeTraceScope timeScope2("Search paths");
     searchPaths.emplace_back("");
-    // Prefer the Clang provided builtins over the ones bundled with MSVC.
-    addClangLibSearchPaths(argsArr[0]);
+    if (!config->mingw) {
+      // Prefer the Clang provided builtins over the ones bundled with MSVC.
+      // In MinGW mode, the compiler driver passes the necessary libpath
+      // options explicitly.
+      addClangLibSearchPaths(argsArr[0]);
+    }
     for (auto *arg : args.filtered(OPT_libpath))
       searchPaths.push_back(arg->getValue());
-    detectWinSysRoot(args);
-    if (!args.hasArg(OPT_lldignoreenv) && !args.hasArg(OPT_winsysroot))
-      addLibSearchPaths();
+    if (!config->mingw) {
+      // Don't automatically deduce the lib path from the environment or MSVC
+      // installations when operating in mingw mode. (This also makes LLD ignore
+      // winsysroot and vctoolsdir arguments.)
+      detectWinSysRoot(args);
+      if (!args.hasArg(OPT_lldignoreenv) && !args.hasArg(OPT_winsysroot))
+        addLibSearchPaths();
+    } else {
+      if (args.hasArg(OPT_vctoolsdir, OPT_winsysroot))
+        warn("ignoring /vctoolsdir or /winsysroot flags in MinGW mode");
+    }
   }
 
   // Handle /ignore
@@ -2028,6 +2040,9 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
   config->stdcallFixup =
       args.hasFlag(OPT_stdcall_fixup, OPT_stdcall_fixup_no, config->mingw);
   config->warnStdcallFixup = !args.hasArg(OPT_stdcall_fixup);
+  config->allowDuplicateWeak =
+      args.hasFlag(OPT_lld_allow_duplicate_weak,
+                   OPT_lld_allow_duplicate_weak_no, config->mingw);
 
   if (args.hasFlag(OPT_inferasanlibs, OPT_inferasanlibs_no, false))
     warn("ignoring '/inferasanlibs', this flag is not supported");
@@ -2181,7 +2196,7 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
       args.hasFlag(OPT_highentropyva, OPT_highentropyva_no, true);
 
   if (!config->dynamicBase &&
-      (config->machine == ARMNT || config->machine == ARM64))
+      (config->machine == ARMNT || isAnyArm64(config->machine)))
     error("/dynamicbase:no is not compatible with " +
           machineToStr(config->machine));
 
diff --git a/lld/COFF/InputFiles.cpp b/lld/COFF/InputFiles.cpp
index a7a08fb2fa..38ce29e6ab 100644
--- a/lld/COFF/InputFiles.cpp
+++ b/lld/COFF/InputFiles.cpp
@@ -81,7 +81,7 @@ static void checkAndSetWeakAlias(COFFLinkerContext &ctx, InputFile *f,
       // of another symbol emitted near the weak symbol.
       // Just use the definition from the first object file that defined
       // this weak symbol.
-      if (ctx.config.mingw)
+      if (ctx.config.allowDuplicateWeak)
         return;
       ctx.symtab.reportDuplicate(source, f);
     }
@@ -709,7 +709,7 @@ void ObjFile::initializeFlags() {
 
   DebugSubsectionArray subsections;
 
-  BinaryStreamReader reader(data, support::little);
+  BinaryStreamReader reader(data, llvm::endianness::little);
   ExitOnError exitOnErr;
   exitOnErr(reader.readArray(subsections, data.size()));
 
@@ -775,7 +775,7 @@ void ObjFile::initializeDependencies() {
   // Get the first type record. It will indicate if this object uses a type
   // server (/Zi) or a PCH file (/Yu).
   CVTypeArray types;
-  BinaryStreamReader reader(data, support::little);
+  BinaryStreamReader reader(data, llvm::endianness::little);
   cantFail(reader.readArray(types, reader.getLength()));
   CVTypeArray::Iterator firstType = types.begin();
   if (firstType == types.end())
diff --git a/lld/COFF/Options.td b/lld/COFF/Options.td
index e7bb952808..977657a433 100644
--- a/lld/COFF/Options.td
+++ b/lld/COFF/Options.td
@@ -232,6 +232,7 @@ defm demangle : B<"demangle",
 def include_optional : Joined<["/", "-", "/?", "-?"], "includeoptional:">,
     HelpText<"Add symbol as undefined, but allow it to remain undefined">;
 def kill_at : F<"kill-at">;
+defm lld_allow_duplicate_weak : B_priv<"lld-allow-duplicate-weak">;
 def lldemit : P<"lldemit", "Specify output type">;
 def lldmingw : F<"lldmingw">;
 def noseh : F<"noseh">;
diff --git a/lld/COFF/PDB.cpp b/lld/COFF/PDB.cpp
index 0c4e0a80cf..f77ff0d4ea 100644
--- a/lld/COFF/PDB.cpp
+++ b/lld/COFF/PDB.cpp
@@ -656,7 +656,7 @@ Error PDBLinker::writeAllModuleSymbolRecords(ObjFile *file,
     auto contents =
         SectionChunk::consumeDebugMagic(sectionContents, ".debug$S");
     DebugSubsectionArray subsections;
-    BinaryStreamReader reader(contents, support::little);
+    BinaryStreamReader reader(contents, llvm::endianness::little);
     exitOnErr(reader.readArray(subsections, contents.size()));
 
     uint32_t nextRelocIndex = 0;
@@ -758,7 +758,7 @@ void DebugSHandler::handleDebugS(SectionChunk *debugChunk) {
   ArrayRef<uint8_t> contents = debugChunk->getContents();
   contents = SectionChunk::consumeDebugMagic(contents, ".debug$S");
   DebugSubsectionArray subsections;
-  BinaryStreamReader reader(contents, support::little);
+  BinaryStreamReader reader(contents, llvm::endianness::little);
   ExitOnError exitOnErr;
   exitOnErr(reader.readArray(subsections, contents.size()));
   debugChunk->sortRelocations();
@@ -868,7 +868,7 @@ Error UnrelocatedDebugSubsection::commit(BinaryStreamWriter &writer) const {
       debugChunk->file->debugTypesObj) {
     TpiSource *source = debugChunk->file->debugTypesObj;
     DebugInlineeLinesSubsectionRef inlineeLines;
-    BinaryStreamReader storageReader(relocatedBytes, support::little);
+    BinaryStreamReader storageReader(relocatedBytes, llvm::endianness::little);
     ExitOnError exitOnErr;
     exitOnErr(inlineeLines.initialize(storageReader));
     for (const InlineeSourceLine &line : inlineeLines) {
@@ -962,7 +962,7 @@ void DebugSHandler::finish() {
     // Copy each frame data record, add in rvaStart, translate string table
     // indices, and add the record to the PDB.
     DebugFrameDataSubsectionRef fds;
-    BinaryStreamReader reader(subsecData, support::little);
+    BinaryStreamReader reader(subsecData, llvm::endianness::little);
     exitOnErr(fds.initialize(reader));
     for (codeview::FrameData fd : fds) {
       fd.RvaStart += rvaStart;
@@ -1050,7 +1050,8 @@ void PDBLinker::addDebugSymbols(TpiSource *source) {
       ArrayRef<uint8_t> relocatedDebugContents =
           relocateDebugChunk(*debugChunk);
       FixedStreamArray<object::FpoData> fpoRecords;
-      BinaryStreamReader reader(relocatedDebugContents, support::little);
+      BinaryStreamReader reader(relocatedDebugContents,
+                                llvm::endianness::little);
       uint32_t count = relocatedDebugContents.size() / sizeof(object::FpoData);
       exitOnErr(reader.readArray(fpoRecords, count));
 
@@ -1772,7 +1773,7 @@ static bool findLineTable(const SectionChunk *c, uint32_t addr,
     ArrayRef<uint8_t> contents =
         SectionChunk::consumeDebugMagic(dbgC->getContents(), ".debug$S");
     DebugSubsectionArray subsections;
-    BinaryStreamReader reader(contents, support::little);
+    BinaryStreamReader reader(contents, llvm::endianness::little);
     exitOnErr(reader.readArray(subsections, contents.size()));
 
     for (const DebugSubsectionRecord &ss : subsections) {
diff --git a/lld/COFF/Writer.cpp b/lld/COFF/Writer.cpp
index 4f6c2a57f5..43d8e7c1d5 100644
--- a/lld/COFF/Writer.cpp
+++ b/lld/COFF/Writer.cpp
@@ -212,6 +212,7 @@ class Writer {
   void locateImportTables();
   void createExportTable();
   void mergeSections();
+  void sortECChunks();
   void removeUnusedSections();
   void assignAddresses();
   bool isInRange(uint16_t relType, uint64_t s, uint64_t p, int margin);
@@ -676,6 +677,7 @@ void Writer::run() {
     createMiscChunks();
     createExportTable();
     mergeSections();
+    sortECChunks();
     removeUnusedSections();
     finalizeAddresses();
     removeEmptySections();
@@ -1377,6 +1379,21 @@ void Writer::mergeSections() {
   }
 }
 
+// EC targets may have chunks of various architectures mixed together at this
+// point. Group code chunks of the same architecture together by sorting chunks
+// by their EC range type.
+void Writer::sortECChunks() {
+  if (!isArm64EC(ctx.config.machine))
+    return;
+
+  for (OutputSection *sec : ctx.outputSections) {
+    if (sec->isCodeSection())
+      llvm::stable_sort(sec->chunks, [=](const Chunk *a, const Chunk *b) {
+        return a->getArm64ECRangeType() < b->getArm64ECRangeType();
+      });
+  }
+}
+
 // Visits all sections to assign incremental, non-overlapping RVAs and
 // file offsets.
 void Writer::assignAddresses() {
@@ -1403,11 +1420,7 @@ void Writer::assignAddresses() {
 
     // If /FUNCTIONPADMIN is used, functions are padded in order to create a
     // hotpatchable image.
-    const bool isCodeSection =
-        (sec->header.Characteristics & IMAGE_SCN_CNT_CODE) &&
-        (sec->header.Characteristics & IMAGE_SCN_MEM_READ) &&
-        (sec->header.Characteristics & IMAGE_SCN_MEM_EXECUTE);
-    uint32_t padding = isCodeSection ? config->functionPadMin : 0;
+    uint32_t padding = sec->isCodeSection() ? config->functionPadMin : 0;
 
     for (Chunk *c : sec->chunks) {
       if (padding && c->isHotPatchable())
diff --git a/lld/COFF/Writer.h b/lld/COFF/Writer.h
index 4a74aa7ada..9004bb310d 100644
--- a/lld/COFF/Writer.h
+++ b/lld/COFF/Writer.h
@@ -64,6 +64,12 @@ class OutputSection {
   // Used only when the name is longer than 8 bytes.
   void setStringTableOff(uint32_t v) { stringTableOff = v; }
 
+  bool isCodeSection() const {
+    return (header.Characteristics & llvm::COFF::IMAGE_SCN_CNT_CODE) &&
+           (header.Characteristics & llvm::COFF::IMAGE_SCN_MEM_READ) &&
+           (header.Characteristics & llvm::COFF::IMAGE_SCN_MEM_EXECUTE);
+  }
+
   // N.B. The section index is one based.
   uint32_t sectionIndex = 0;
 
diff --git a/lld/ELF/Arch/Mips.cpp b/lld/ELF/Arch/Mips.cpp
index d5a335c659..d6c70aeba9 100644
--- a/lld/ELF/Arch/Mips.cpp
+++ b/lld/ELF/Arch/Mips.cpp
@@ -219,7 +219,7 @@ template <endianness E> static uint32_t readShuffle(const uint8_t *loc) {
   // words in a big-endian order. That is why we have to swap these
   // words to get a correct value.
   uint32_t v = read32(loc);
-  if (E == support::little)
+  if (E == llvm::endianness::little)
     return (v << 16) | (v >> 16);
   return v;
 }
@@ -237,12 +237,12 @@ static void writeShuffleValue(uint8_t *loc, uint64_t v, uint8_t bitsSize,
                               uint8_t shift) {
   // See comments in readShuffle for purpose of this code.
   uint16_t *words = (uint16_t *)loc;
-  if (E == support::little)
+  if (E == llvm::endianness::little)
     std::swap(words[0], words[1]);
 
   writeValue(loc, v, bitsSize, shift);
 
-  if (E == support::little)
+  if (E == llvm::endianness::little)
     std::swap(words[0], words[1]);
 }
 
diff --git a/lld/ELF/Arch/RISCV.cpp b/lld/ELF/Arch/RISCV.cpp
index d0d75118e3..6413dcd7dc 100644
--- a/lld/ELF/Arch/RISCV.cpp
+++ b/lld/ELF/Arch/RISCV.cpp
@@ -933,7 +933,7 @@ mergeAttributesSection(const SmallVector<InputSectionBase *, 0> &sections) {
   const auto &attributesTags = RISCVAttrs::getRISCVAttributeTags();
   for (const InputSectionBase *sec : sections) {
     RISCVAttributeParser parser;
-    if (Error e = parser.parse(sec->content(), support::little))
+    if (Error e = parser.parse(sec->content(), llvm::endianness::little))
       warn(toString(sec) + ": " + llvm::toString(std::move(e)));
     for (const auto &tag : attributesTags) {
       switch (RISCVAttrs::AttrType(tag.attr)) {
diff --git a/lld/ELF/DWARF.h b/lld/ELF/DWARF.h
index 9a7993903d..e1688fef9c 100644
--- a/lld/ELF/DWARF.h
+++ b/lld/ELF/DWARF.h
@@ -73,7 +73,7 @@ template <class ELFT> class LLDDwarfObj final : public llvm::DWARFObject {
   StringRef getLineStrSection() const override { return lineStrSection; }
 
   bool isLittleEndian() const override {
-    return ELFT::TargetEndianness == llvm::support::little;
+    return ELFT::TargetEndianness == llvm::endianness::little;
   }
 
   std::optional<llvm::RelocAddrEntry> find(const llvm::DWARFSection &sec,
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index f581529427..5f88389a58 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -1095,7 +1095,7 @@ static void ltoValidateAllVtablesHaveTypeInfos(opt::InputArgList &args) {
 }
 
 static CGProfileSortKind getCGProfileSortKind(opt::InputArgList &args) {
-  StringRef s = args.getLastArgValue(OPT_call_graph_profile_sort, "cdsort");
+  StringRef s = args.getLastArgValue(OPT_call_graph_profile_sort, "hfsort");
   if (s == "hfsort")
     return CGProfileSortKind::Hfsort;
   if (s == "cdsort")
@@ -2248,24 +2248,6 @@ static void replaceCommonSymbols() {
   }
 }
 
-// If all references to a DSO happen to be weak, the DSO is not added to
-// DT_NEEDED. If that happens, replace ShardSymbol with Undefined to avoid
-// dangling references to an unneeded DSO. Use a weak binding to avoid
-// --no-allow-shlib-undefined diagnostics. Similarly, demote lazy symbols.
-static void demoteSharedAndLazySymbols() {
-  llvm::TimeTraceScope timeScope("Demote shared and lazy symbols");
-  for (Symbol *sym : symtab.getSymbols()) {
-    auto *s = dyn_cast<SharedSymbol>(sym);
-    if (!(s && !cast<SharedFile>(s->file)->isNeeded) && !sym->isLazy())
-      continue;
-
-    uint8_t binding = sym->isLazy() ? sym->binding : uint8_t(STB_WEAK);
-    Undefined(nullptr, sym->getName(), binding, sym->stOther, sym->type)
-        .overwrite(*sym);
-    sym->versionId = VER_NDX_GLOBAL;
-  }
-}
-
 // The section referred to by `s` is considered address-significant. Set the
 // keepUnique flag on the section if appropriate.
 static void markAddrsig(Symbol *s) {
@@ -3023,7 +3005,6 @@ void LinkerDriver::link(opt::InputArgList &args) {
 
   // Garbage collection and removal of shared symbols from unused shared objects.
   invokeELFT(markLive,);
-  demoteSharedAndLazySymbols();
 
   // Make copies of any input sections that need to be copied into each
   // partition.
diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp
index 3413586f6b..a0d4be8ff9 100644
--- a/lld/ELF/InputFiles.cpp
+++ b/lld/ELF/InputFiles.cpp
@@ -604,9 +604,9 @@ template <class ELFT> void ObjFile<ELFT>::parse(bool ignoreComdats) {
           check(this->getObj().getSectionContents(sec));
       StringRef name = check(obj.getSectionName(sec, shstrtab));
       this->sections[i] = &InputSection::discarded;
-      if (Error e =
-              attributes.parse(contents, ekind == ELF32LEKind ? support::little
-                                                              : support::big)) {
+      if (Error e = attributes.parse(contents, ekind == ELF32LEKind
+                                                   ? llvm::endianness::little
+                                                   : llvm::endianness::big)) {
         InputSection isec(*this, sec, name);
         warn(toString(&isec) + ": " + llvm::toString(std::move(e)));
       } else {
diff --git a/lld/ELF/InputSection.cpp b/lld/ELF/InputSection.cpp
index 413fa3856a..02394cbae9 100644
--- a/lld/ELF/InputSection.cpp
+++ b/lld/ELF/InputSection.cpp
@@ -435,10 +435,7 @@ void InputSection::copyRelocations(uint8_t *buf,
         continue;
       }
       SectionBase *section = d->section;
-      if (!section->isLive()) {
-        p->setSymbolAndType(0, 0, false);
-        continue;
-      }
+      assert(section->isLive());
 
       int64_t addend = rel.addend;
       const uint8_t *bufLoc = sec->content().begin() + rel.offset;
diff --git a/lld/ELF/MapFile.cpp b/lld/ELF/MapFile.cpp
index 1b6dfcc571..8b10ae183a 100644
--- a/lld/ELF/MapFile.cpp
+++ b/lld/ELF/MapFile.cpp
@@ -229,7 +229,7 @@ static void writeCref(raw_fd_ostream &os) {
       if (isa<SharedSymbol>(sym))
         map[sym].insert(file);
       if (auto *d = dyn_cast<Defined>(sym))
-        if (!d->isLocal() && (!d->section || d->section->isLive()))
+        if (!d->isLocal())
           map[d].insert(file);
     }
   }
diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp
index ee27cc15e0..f3fb0c71a8 100644
--- a/lld/ELF/Relocations.cpp
+++ b/lld/ELF/Relocations.cpp
@@ -507,8 +507,7 @@ int64_t RelocationScanner::computeMipsAddend(const RelTy &rel, RelExpr expr,
 template <class ELFT>
 static std::string maybeReportDiscarded(Undefined &sym) {
   auto *file = dyn_cast_or_null<ObjFile<ELFT>>(sym.file);
-  if (!file || !sym.discardedSecIdx ||
-      file->getSections()[sym.discardedSecIdx] != &InputSection::discarded)
+  if (!file || !sym.discardedSecIdx)
     return "";
   ArrayRef<typename ELFT::Shdr> objSections =
       file->template getELFShdrs<ELFT>();
@@ -1575,7 +1574,8 @@ template <class ELFT> void elf::scanRelocations() {
         scanner.template scanSection<ELFT>(*sec);
       if (part.armExidx && part.armExidx->isLive())
         for (InputSection *sec : part.armExidx->exidxSections)
-          scanner.template scanSection<ELFT>(*sec);
+          if (sec->isLive())
+            scanner.template scanSection<ELFT>(*sec);
     }
   });
 }
diff --git a/lld/ELF/Symbols.cpp b/lld/ELF/Symbols.cpp
index 07061d3a12..671eb56f34 100644
--- a/lld/ELF/Symbols.cpp
+++ b/lld/ELF/Symbols.cpp
@@ -316,12 +316,13 @@ void elf::maybeWarnUnorderableSymbol(const Symbol *sym) {
   if (!config->warnSymbolOrdering)
     return;
 
-  // If UnresolvedPolicy::Ignore is used, no "undefined symbol" error/warning
-  // is emitted. It makes sense to not warn on undefined symbols.
+  // If UnresolvedPolicy::Ignore is used, no "undefined symbol" error/warning is
+  // emitted. It makes sense to not warn on undefined symbols (excluding those
+  // demoted by demoteSymbols).
   //
   // Note, ld.bfd --symbol-ordering-file= does not warn on undefined symbols,
   // but we don't have to be compatible here.
-  if (sym->isUndefined() &&
+  if (sym->isUndefined() && !cast<Undefined>(sym)->discardedSecIdx &&
       config->unresolvedSymbols == UnresolvedPolicy::Ignore)
     return;
 
@@ -330,9 +331,12 @@ void elf::maybeWarnUnorderableSymbol(const Symbol *sym) {
 
   auto report = [&](StringRef s) { warn(toString(file) + s + sym->getName()); };
 
-  if (sym->isUndefined())
-    report(": unable to order undefined symbol: ");
-  else if (sym->isShared())
+  if (sym->isUndefined()) {
+    if (cast<Undefined>(sym)->discardedSecIdx)
+      report(": unable to order discarded symbol: ");
+    else
+      report(": unable to order undefined symbol: ");
+  } else if (sym->isShared())
     report(": unable to order shared symbol: ");
   else if (d && !d->section)
     report(": unable to order absolute symbol: ");
diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp
index 5077c97265..57e1aa06c6 100644
--- a/lld/ELF/Writer.cpp
+++ b/lld/ELF/Writer.cpp
@@ -53,7 +53,6 @@ template <class ELFT> class Writer {
   void run();
 
 private:
-  void copyLocalSymbols();
   void addSectionSymbols();
   void sortSections();
   void resolveShfLinkOrder();
@@ -251,6 +250,47 @@ void elf::addReservedSymbols() {
   ElfSym::edata2 = add("_edata", -1);
 }
 
+static void demoteDefined(Defined &sym, DenseMap<SectionBase *, size_t> &map) {
+  if (map.empty())
+    for (auto [i, sec] : llvm::enumerate(sym.file->getSections()))
+      map.try_emplace(sec, i);
+  // Change WEAK to GLOBAL so that if a scanned relocation references sym,
+  // maybeReportUndefined will report an error.
+  uint8_t binding = sym.isWeak() ? uint8_t(STB_GLOBAL) : sym.binding;
+  Undefined(sym.file, sym.getName(), binding, sym.stOther, sym.type,
+            /*discardedSecIdx=*/map.lookup(sym.section))
+      .overwrite(sym);
+}
+
+// If all references to a DSO happen to be weak, the DSO is not added to
+// DT_NEEDED. If that happens, replace ShardSymbol with Undefined to avoid
+// dangling references to an unneeded DSO. Use a weak binding to avoid
+// --no-allow-shlib-undefined diagnostics. Similarly, demote lazy symbols.
+//
+// In addition, demote symbols defined in discarded sections, so that
+// references to /DISCARD/ discarded symbols will lead to errors.
+static void demoteSymbolsAndComputeIsPreemptible() {
+  llvm::TimeTraceScope timeScope("Demote symbols");
+  DenseMap<InputFile *, DenseMap<SectionBase *, size_t>> sectionIndexMap;
+  for (Symbol *sym : symtab.getSymbols()) {
+    if (auto *d = dyn_cast<Defined>(sym)) {
+      if (d->section && !d->section->isLive())
+        demoteDefined(*d, sectionIndexMap[d->file]);
+    } else {
+      auto *s = dyn_cast<SharedSymbol>(sym);
+      if (sym->isLazy() || (s && !cast<SharedFile>(s->file)->isNeeded)) {
+        uint8_t binding = sym->isLazy() ? sym->binding : uint8_t(STB_WEAK);
+        Undefined(nullptr, sym->getName(), binding, sym->stOther, sym->type)
+            .overwrite(*sym);
+        sym->versionId = VER_NDX_GLOBAL;
+      }
+    }
+
+    if (config->hasDynSymTab)
+      sym->isPreemptible = computeIsPreemptible(*sym);
+  }
+}
+
 // Fully static executables don't support MTE globals at this point in time, as
 // we currently rely on:
 //   - A dynamic loader to process relocations, and
@@ -545,11 +585,6 @@ template <class ELFT> void elf::createSyntheticSections() {
 
 // The main function of the writer.
 template <class ELFT> void Writer<ELFT>::run() {
-  copyLocalSymbols();
-
-  if (config->copyRelocs)
-    addSectionSymbols();
-
   // Now that we have a complete set of output sections. This function
   // completes section contents. For example, we need to add strings
   // to the string table, and add entries to .got and .plt.
@@ -698,31 +733,33 @@ bool lld::elf::includeInSymtab(const Symbol &b) {
     SectionBase *sec = d->section;
     if (!sec)
       return true;
+    assert(sec->isLive());
 
     if (auto *s = dyn_cast<MergeInputSection>(sec))
       return s->getSectionPiece(d->value).live;
-    return sec->isLive();
+    return true;
   }
   return b.used || !config->gcSections;
 }
 
-// Local symbols are not in the linker's symbol table. This function scans
-// each object file's symbol table to copy local symbols to the output.
-template <class ELFT> void Writer<ELFT>::copyLocalSymbols() {
-  if (!in.symTab)
-    return;
+// Scan local symbols to:
+//
+// - demote symbols defined relative to /DISCARD/ discarded input sections so
+//   that relocations referencing them will lead to errors.
+// - copy eligible symbols to .symTab
+static void demoteAndCopyLocalSymbols() {
   llvm::TimeTraceScope timeScope("Add local symbols");
-  if (config->copyRelocs && config->discard != DiscardPolicy::None)
-    markUsedLocalSymbols<ELFT>();
   for (ELFFileBase *file : ctx.objectFiles) {
+    DenseMap<SectionBase *, size_t> sectionIndexMap;
     for (Symbol *b : file->getLocalSymbols()) {
       assert(b->isLocal() && "should have been caught in initializeSymbols()");
       auto *dr = dyn_cast<Defined>(b);
-
-      // No reason to keep local undefined symbol in symtab.
       if (!dr)
         continue;
-      if (includeInSymtab(*b) && shouldKeepInSymtab(*dr))
+
+      if (dr->section && !dr->section->isLive())
+        demoteDefined(*dr, sectionIndexMap);
+      else if (in.symTab && includeInSymtab(*b) && shouldKeepInSymtab(*dr))
         in.symTab->addSymbol(b);
     }
   }
@@ -1935,14 +1972,17 @@ template <class ELFT> void Writer<ELFT>::finalizeSections() {
       for (Partition &part : partitions)
         finalizeSynthetic(part.ehFrame.get());
     }
-
-    if (config->hasDynSymTab) {
-      parallelForEach(symtab.getSymbols(), [](Symbol *sym) {
-        sym->isPreemptible = computeIsPreemptible(*sym);
-      });
-    }
   }
 
+  demoteSymbolsAndComputeIsPreemptible();
+
+  if (config->copyRelocs && config->discard != DiscardPolicy::None)
+    markUsedLocalSymbols<ELFT>();
+  demoteAndCopyLocalSymbols();
+
+  if (config->copyRelocs)
+    addSectionSymbols();
+
   // Change values of linker-script-defined symbols from placeholders (assigned
   // by declareSymbols) to actual definitions.
   script->processSymbolAssignments();
diff --git a/lld/MachO/InputFiles.cpp b/lld/MachO/InputFiles.cpp
index 8f737beee7..09c6ea9b19 100644
--- a/lld/MachO/InputFiles.cpp
+++ b/lld/MachO/InputFiles.cpp
@@ -320,7 +320,7 @@ static std::optional<size_t> getRecordSize(StringRef segname, StringRef name) {
 static Error parseCallGraph(ArrayRef<uint8_t> data,
                             std::vector<CallGraphEntry> &callGraph) {
   TimeTraceScope timeScope("Parsing call graph section");
-  BinaryStreamReader reader(data, support::little);
+  BinaryStreamReader reader(data, llvm::endianness::little);
   while (!reader.empty()) {
     uint32_t fromIndex, toIndex;
     uint64_t count;
diff --git a/lld/docs/ld.lld.1 b/lld/docs/ld.lld.1
index 12b17dd377..2e46fc1813 100644
--- a/lld/docs/ld.lld.1
+++ b/lld/docs/ld.lld.1
@@ -128,9 +128,9 @@ may be:
 .It Cm none
 Ignore call graph profile.
 .It Cm hfsort
-Use hfsort.
+Use hfsort (default).
 .It Cm cdsort
-Use cdsort (default).
+Use cdsort.
 .El
 .Pp
 .It Fl -color-diagnostics Ns = Ns Ar value
diff --git a/lld/test/COFF/Inputs/loadconfig-arm64ec.s b/lld/test/COFF/Inputs/loadconfig-arm64ec.s
new file mode 100644
index 0000000000..1efd02406c
--- /dev/null
+++ b/lld/test/COFF/Inputs/loadconfig-arm64ec.s
@@ -0,0 +1,97 @@
+        .section .rdata,"dr"
+        .globl _load_config_used
+        .p2align 3, 0
+_load_config_used:
+        .word 0x140
+        .fill 0x54, 1, 0
+        .xword __security_cookie
+        .fill 0x10, 1, 0
+        .xword __guard_check_icall_fptr
+        .xword __guard_dispatch_icall_fptr
+        .xword __guard_fids_table
+        .xword __guard_fids_count
+        .xword __guard_flags
+        .xword 0
+        .xword __guard_iat_table
+        .xword __guard_iat_count
+        .xword __guard_longjmp_table
+        .xword __guard_longjmp_count
+        .xword 0
+        .xword __chpe_metadata
+        .fill 0x78, 1, 0
+
+__guard_check_icall_fptr:
+        .xword 0
+__guard_dispatch_icall_fptr:
+        .xword 0
+__os_arm64x_dispatch_call_no_redirect:
+        .xword 0
+__os_arm64x_dispatch_ret:
+        .xword 0
+__os_arm64x_check_call:
+        .xword 0
+__os_arm64x_check_icall:
+        .xword 0
+__os_arm64x_get_x64_information:
+        .xword 0
+__os_arm64x_set_x64_information:
+        .xword 0
+__os_arm64x_check_icall_cfg:
+        .xword 0
+__os_arm64x_dispatch_fptr:
+        .xword 0
+__os_arm64x_helper0:
+        .xword 0
+__os_arm64x_helper1:
+        .xword 0
+__os_arm64x_helper2:
+        .xword 0
+__os_arm64x_helper3:
+        .xword 0
+__os_arm64x_helper4:
+        .xword 0
+__os_arm64x_helper5:
+        .xword 0
+__os_arm64x_helper6:
+        .xword 0
+__os_arm64x_helper7:
+        .xword 0
+__os_arm64x_helper8:
+        .xword 0
+
+        .data
+        .globl __chpe_metadata
+        .p2align 3, 0
+__chpe_metadata:
+        .word 1
+        .rva code_map
+        .word code_map_count
+        .word 0 // __x64_code_ranges_to_entry_points
+        .word 0 //__arm64x_redirection_metadata
+        .rva __os_arm64x_dispatch_call_no_redirect
+        .rva __os_arm64x_dispatch_ret
+        .rva __os_arm64x_check_call
+        .rva __os_arm64x_check_icall
+        .rva __os_arm64x_check_icall_cfg
+        .word 0 // __arm64x_native_entrypoint
+        .word 0 // __hybrid_auxiliary_iat
+        .word 0 // __x64_code_ranges_to_entry_points_count
+        .word 0 // __arm64x_redirection_metadata_count
+        .rva __os_arm64x_get_x64_information
+        .rva __os_arm64x_set_x64_information
+        .word 0 // __arm64x_extra_rfe_table
+        .word 0 // __arm64x_extra_rfe_table_size
+        .rva __os_arm64x_dispatch_fptr
+        .word 0 // __hybrid_auxiliary_iat_copy
+        .rva __os_arm64x_helper0
+        .rva __os_arm64x_helper1
+        .rva __os_arm64x_helper2
+        .rva __os_arm64x_helper3
+        .rva __os_arm64x_helper4
+        .rva __os_arm64x_helper5
+        .rva __os_arm64x_helper6
+        .rva __os_arm64x_helper7
+        .rva __os_arm64x_helper8
+
+__security_cookie:
+        .xword 0
diff --git a/lld/test/COFF/arm64-dynamicbase.s b/lld/test/COFF/arm64-dynamicbase.s
index d4cb12c1c8..6ccce26a2f 100644
--- a/lld/test/COFF/arm64-dynamicbase.s
+++ b/lld/test/COFF/arm64-dynamicbase.s
@@ -1,8 +1,16 @@
 // REQUIRES: aarch64
 // RUN: llvm-mc -filetype=obj -triple=aarch64-windows %s -o %t.obj
 // RUN: not lld-link -entry:_start -subsystem:console %t.obj -out:%t.exe -dynamicbase:no 2>&1 | FileCheck %s
+
+// RUN: llvm-mc -filetype=obj -triple=arm64ec-windows %s -o %t.obj
+// RUN: not lld-link -entry:_start -subsystem:console %t.obj -out:%t.exe -dynamicbase:no -machine:arm64ec 2>&1 \
+// RUN:              | FileCheck %s -check-prefix=ARM64EC
+// RUN: not lld-link -entry:_start -subsystem:console %t.obj -out:%t.exe -dynamicbase:no -machine:arm64x -dll -noentry 2>&1 \
+// RUN:              | FileCheck %s -check-prefix=ARM64X
  .globl _start
 _start:
  ret
 
 # CHECK: dynamicbase:no is not compatible with arm64
+# ARM64EC: dynamicbase:no is not compatible with arm64ec
+# ARM64X: dynamicbase:no is not compatible with arm64x
diff --git a/lld/test/COFF/arm64ec-codemap.test b/lld/test/COFF/arm64ec-codemap.test
new file mode 100644
index 0000000000..424456a6de
--- /dev/null
+++ b/lld/test/COFF/arm64ec-codemap.test
@@ -0,0 +1,195 @@
+REQUIRES: aarch64, x86
+RUN: split-file %s %t.dir && cd %t.dir
+
+RUN: llvm-mc -filetype=obj -triple=arm64-windows arm64-func-sym.s -o arm64-func-sym.obj
+RUN: llvm-mc -filetype=obj -triple=arm64ec-windows arm64ec-func-sym.s -o arm64ec-func-sym.obj
+RUN: llvm-mc -filetype=obj -triple=x86_64-windows x86_64-func-sym.s -o x86_64-func-sym.obj
+RUN: llvm-mc -filetype=obj -triple=arm64ec-windows codemap.s -o codemap.obj
+RUN: llvm-mc -filetype=obj -triple=arm64ec-windows codemap2.s -o codemap2.obj
+RUN: llvm-mc -filetype=obj -triple=arm64ec-windows codemap3.s -o codemap3.obj
+RUN: llvm-mc -filetype=obj -triple=arm64ec-windows %S/Inputs/loadconfig-arm64ec.s -o loadconfig-arm64ec.obj
+
+Link ARM64EC DLL and verify that the code is arranged as expected.
+
+RUN: lld-link -out:test.dll -machine:arm64ec arm64ec-func-sym.obj x86_64-func-sym.obj \
+RUN:          codemap.obj loadconfig-arm64ec.obj -dll -noentry
+
+RUN: llvm-readobj --coff-load-config test.dll | FileCheck -check-prefix=CODEMAP %s
+CODEMAP:       CodeMap [
+CODEMAP-NEXT:    0x1000 - 0x1008  ARM64EC
+CODEMAP-NEXT:    0x2000 - 0x2006  X64
+CODEMAP-NEXT:    0x5000 - 0x5008  ARM64EC
+CODEMAP-NEXT:    0x6000 - 0x6006  X64
+CODEMAP-NEXT:  ]
+
+RUN: llvm-objdump -d test.dll | FileCheck -check-prefix=DISASM %s
+DISASM:      Disassembly of section .text:
+DISASM-EMPTY:
+DISASM-NEXT: 0000000180001000 <.text>:
+DISASM-NEXT: 180001000: 52800040     mov     w0, #0x2
+DISASM-NEXT: 180001004: d65f03c0     ret
+DISASM-NEXT:                 ...
+DISASM-NEXT: 180002000: b8 03 00 00 00               movl    $0x3, %eax
+DISASM-NEXT: 180002005: c3                           retq
+DISASM-EMPTY:
+DISASM-NEXT: Disassembly of section test:
+DISASM-EMPTY:
+DISASM-NEXT: 0000000180005000 <test>:
+DISASM-NEXT: 180005000: 528000a0     mov     w0, #0x5
+DISASM-NEXT: 180005004: d65f03c0     ret
+DISASM-NEXT:                 ...
+DISASM-NEXT: 180006000: b8 06 00 00 00               movl    $0x6, %eax
+DISASM-NEXT: 180006005: c3                           retq
+
+Order of arguments doesn't matter in this case, chunks are sorted by target type anyway.
+
+RUN: lld-link -out:test2.dll -machine:arm64ec x86_64-func-sym.obj arm64ec-func-sym.obj \
+RUN:          codemap.obj loadconfig-arm64ec.obj -dll -noentry
+RUN: llvm-readobj --coff-load-config test2.dll | FileCheck -check-prefix=CODEMAP %s
+RUN: llvm-objdump -d test2.dll | FileCheck -check-prefix=DISASM %s
+
+RUN: lld-link -out:testx.dll -machine:arm64x arm64-func-sym.obj arm64ec-func-sym.obj \
+RUN:          x86_64-func-sym.obj codemap2.obj loadconfig-arm64ec.obj -dll -noentry
+
+Do the same with ARM64X target.
+
+RUN: llvm-readobj --coff-load-config testx.dll | FileCheck -check-prefix=CODEMAPX %s
+CODEMAPX:       CodeMap [
+CODEMAPX-NEXT:    0x1000 - 0x1008  ARM64
+CODEMAPX-NEXT:    0x2000 - 0x2008  ARM64EC
+CODEMAPX-NEXT:    0x3000 - 0x3006  X64
+CODEMAPX-NEXT:    0x6000 - 0x6008  ARM64EC
+CODEMAPX-NEXT:    0x7000 - 0x7006  X64
+CODEMAPX-NEXT:  ]
+
+RUN: llvm-objdump -d testx.dll | FileCheck -check-prefix=DISASMX %s
+DISASMX:      Disassembly of section .text:
+DISASMX-EMPTY:
+DISASMX-NEXT: 0000000180001000 <.text>:
+DISASMX-NEXT: 180001000: 528000e0     mov     w0, #0x7
+DISASMX-NEXT: 180001004: d65f03c0     ret
+DISASMX-NEXT:                 ...
+DISASMX-NEXT: 180002000: 52800040     mov     w0, #0x2
+DISASMX-NEXT: 180002004: d65f03c0     ret
+DISASMX-NEXT:                 ...
+DISASMX-NEXT: 180003000: b8 03 00 00 00               movl    $0x3, %eax
+DISASMX-NEXT: 180003005: c3                           retq
+DISASMX-EMPTY:
+DISASMX-NEXT: Disassembly of section test:
+DISASMX-EMPTY:
+DISASMX-NEXT: 0000000180006000 <test>:
+DISASMX-NEXT: 180006000: 528000a0     mov     w0, #0x5
+DISASMX-NEXT: 180006004: d65f03c0     ret
+DISASMX-NEXT:                 ...
+DISASMX-NEXT: 180007000: b8 06 00 00 00               movl    $0x6, %eax
+DISASMX-NEXT: 180007005: c3                           retq
+
+Test merged sections.
+
+RUN: lld-link -out:testm.dll -machine:arm64ec arm64ec-func-sym.obj x86_64-func-sym.obj \
+RUN:          codemap3.obj loadconfig-arm64ec.obj -dll -noentry -merge:test=.text
+
+RUN: llvm-readobj --coff-load-config testm.dll | FileCheck -check-prefix=CODEMAPM %s
+CODEMAPM:      CodeMap [
+CODEMAPM-NEXT:   0x1000 - 0x1010  ARM64EC
+CODEMAPM-NEXT:   0x2000 - 0x3004  X64
+CODEMAPM-NEXT: ]
+
+RUN: llvm-objdump -d testm.dll | FileCheck -check-prefix=DISASMM %s
+DISASMM:      Disassembly of section .text:
+DISASMM-EMPTY:
+DISASMM-NEXT: 0000000180001000 <.text>:
+DISASMM-NEXT: 180001000: 52800040     mov     w0, #0x2
+DISASMM-NEXT: 180001004: d65f03c0     ret
+DISASMM-NEXT: 180001008: 528000a0     mov     w0, #0x5
+DISASMM-NEXT: 18000100c: d65f03c0     ret
+DISASMM-NEXT:                 ...
+DISASMM-NEXT: 180002000: b8 03 00 00 00               movl    $0x3, %eax
+DISASMM-NEXT: 180002005: c3                           retq
+DISASMM-NEXT:                 ...
+DISASMM-NEXT: 180002ffe: 00 00                        addb    %al, (%rax)
+DISASMM-NEXT: 180003000: b8 06 00 00 00               movl    $0x6, %eax
+
+#--- arm64-func-sym.s
+    .text
+    .globl arm64_func_sym
+    .p2align 2, 0x0
+arm64_func_sym:
+    mov w0, #7
+    ret
+
+#--- arm64ec-func-sym.s
+    .text
+    .globl arm64ec_func_sym
+    .p2align 12, 0x0
+arm64ec_func_sym:
+    mov w0, #2
+    ret
+
+    .section test, "xr"
+    .globl arm64ec_func_sym2
+    .p2align 2, 0x0
+arm64ec_func_sym2:
+    mov w0, #5
+    ret
+
+#--- x86_64-func-sym.s
+    .text
+    .globl x86_64_func_sym
+    .p2align 12, 0x0
+x86_64_func_sym:
+    movl $3, %eax
+    retq
+
+    .section test, "xr"
+    .globl x86_64_func_sym2
+    .p2align 12, 0x0
+x86_64_func_sym2:
+    movl $6, %eax
+    retq
+
+#--- codemap.s
+    .section .rdata,"dr"
+    .globl code_map
+code_map:
+    .rva arm64ec_func_sym + 1
+    .word 8
+    .rva x86_64_func_sym + 2
+    .word 6
+    .rva arm64ec_func_sym2 + 1
+    .word 8
+    .rva x86_64_func_sym2 + 2
+    .word 6
+
+    .globl code_map_count
+code_map_count = 4
+
+#--- codemap2.s
+    .section .rdata,"dr"
+    .globl code_map
+code_map:
+    .rva arm64_func_sym
+    .word 8
+    .rva arm64ec_func_sym + 1
+    .word 8
+    .rva x86_64_func_sym + 2
+    .word 6
+    .rva arm64ec_func_sym2 + 1
+    .word 8
+    .rva x86_64_func_sym2 + 2
+    .word 6
+
+    .globl code_map_count
+code_map_count = 5
+
+#--- codemap3.s
+    .section .rdata,"dr"
+    .globl code_map
+code_map:
+    .rva arm64ec_func_sym + 1
+    .word 16
+    .rva x86_64_func_sym + 2
+    .word 0x1004
+
+    .globl code_map_count
+code_map_count = 2
diff --git a/lld/test/COFF/arm64ec-reloc.test b/lld/test/COFF/arm64ec-reloc.test
new file mode 100644
index 0000000000..3060891bfe
--- /dev/null
+++ b/lld/test/COFF/arm64ec-reloc.test
@@ -0,0 +1,37 @@
+REQUIRES: aarch64, x86
+RUN: split-file %s %t.dir && cd %t.dir
+
+Link a mix of ARM64EC and x86_64 data and check that relocations work.
+
+RUN: llvm-mc -filetype=obj -triple=arm64ec-windows arm64ec-data-sym.s -o arm64ec-data-sym.obj
+RUN: llvm-mc -filetype=obj -triple=x86_64-windows x86_64-data-sym.s -o x86_64-data-sym.obj
+RUN: lld-link -out:test.dll -machine:arm64ec arm64ec-data-sym.obj x86_64-data-sym.obj -dll -noentry
+
+RUN: llvm-readobj --hex-dump=.data test.dll | FileCheck -check-prefix=ARM64EC-DATA %s
+ARM64EC-DATA: 0x180001000 00100080 01000000 08100080 01000000
+
+RUN: llvm-readobj --coff-basereloc test.dll | FileCheck -check-prefix=RELOCS %s
+RELOCS:      BaseReloc [
+RELOCS-NEXT:   Entry {
+RELOCS-NEXT:     Type: DIR64
+RELOCS-NEXT:     Address: 0x1000
+RELOCS-NEXT:   }
+RELOCS-NEXT:   Entry {
+RELOCS-NEXT:     Type: DIR64
+RELOCS-NEXT:     Address: 0x1008
+RELOCS-NEXT:   }
+RELOCS-NEXT: ]
+
+#--- arm64ec-data-sym.s
+        .data
+        .globl arm64ec_data_sym
+        .p2align 2, 0x0
+arm64ec_data_sym:
+        .xword arm64ec_data_sym
+
+#--- x86_64-data-sym.s
+        .data
+        .globl x86_64_data_sym
+        .p2align 2, 0x0
+x86_64_data_sym:
+        .quad x86_64_data_sym
diff --git a/lld/test/COFF/gnu-weak.test b/lld/test/COFF/gnu-weak.test
index 20284d7f02..08e59734e0 100644
--- a/lld/test/COFF/gnu-weak.test
+++ b/lld/test/COFF/gnu-weak.test
@@ -1,4 +1,9 @@
 RUN: lld-link -lldmingw %S/Inputs/gnu-weak.o %S/Inputs/gnu-weak2.o -out:%t.exe
+RUN: lld-link -lld-allow-duplicate-weak %S/Inputs/gnu-weak.o %S/Inputs/gnu-weak2.o -out:%t.exe
+RUN: not lld-link %S/Inputs/gnu-weak.o %S/Inputs/gnu-weak2.o -out:%t.exe 2>&1 | FileCheck %s --check-prefix=DEFAULT-ERROR
+
+DEFAULT-ERROR: error: duplicate symbol: weakfunc
+
 
 GNU ld can handle several definitions of the same weak symbol, and
 unless there is a strong definition of it, it just picks the first
diff --git a/lld/test/COFF/winsysroot.test b/lld/test/COFF/winsysroot.test
index 29df204fa9..e29e6ca115 100644
--- a/lld/test/COFF/winsysroot.test
+++ b/lld/test/COFF/winsysroot.test
@@ -49,5 +49,12 @@ NO64: could not open 'std64.lib'
 NO32: could not open 'std32.lib'
 
 Check that when /winsysroot is specified, %LIB% is ignored.
-# RUN: env LIB=foo.dir/sysroot/VC/Tools/MSVC/1.1.1.1/lib/x86 not lld-link %t.obj /winsysroot:%t.dir/doesnotexist /defaultlib:std32 2>&1 | FileCheck -check-prefix=LIBIGNORED %s
+# RUN: env LIB=%t.dir/sysroot/VC/Tools/MSVC/1.1.1.1/lib/x86 not lld-link %t.obj /winsysroot:%t.dir/doesnotexist /defaultlib:std32 2>&1 | FileCheck -check-prefix=LIBIGNORED %s
 LIBIGNORED: could not open 'std32.lib'
+
+Check that when -lldmingw is specified, %LIB% is ignored.
+# RUN: env LIB=%t.dir/sysroot/VC/Tools/MSVC/1.1.1.1/lib/x86 not lld-link -lldmingw %t.obj /defaultlib:std32 2>&1 | FileCheck -check-prefix=LIBIGNORED_MINGW %s
+LIBIGNORED_MINGW: could not open 'libstd32.a'
+
+# RUN: not lld-link -lldmingw %t.obj /defaultlib:std32 /winsysroot:%t.dir/sysroot 2>&1 | FileCheck -check-prefix=IGNORED_ARG %s
+IGNORED_ARG: warning: ignoring /vctoolsdir or /winsysroot flags in MinGW mode
diff --git a/lld/test/ELF/allow-shlib-undefined.s b/lld/test/ELF/allow-shlib-undefined.s
index 515fa618e7..03f047b02d 100644
--- a/lld/test/ELF/allow-shlib-undefined.s
+++ b/lld/test/ELF/allow-shlib-undefined.s
@@ -37,6 +37,17 @@
 # RUN: not ld.lld %t.o %t.so %t2.so -o /dev/null 2>&1 | \
 # RUN:   FileCheck %s --check-prefixes=CHECK,CHECK2
 
+## Test some cases where relocatable object files provide a hidden definition.
+# RUN: echo '.globl _unresolved; _unresolved:' | llvm-mc -filetype=obj -triple=x86_64 -o %tdef.o
+# RUN: echo '.globl _unresolved; .hidden _unresolved; _unresolved:' | llvm-mc -filetype=obj -triple=x86_64 -o %tdef-hidden.o
+# RUN: ld.lld %t.o %t.so %tdef-hidden.o -o /dev/null 2>&1 | count 0
+
+## The section containing the definition is discarded, and we report an error.
+# RUN: not ld.lld --gc-sections %t.o %t.so %tdef-hidden.o -o /dev/null 2>&1 | FileCheck %s
+## The definition %tdef.so is ignored.
+# RUN: ld.lld -shared -soname=tdef.so %tdef.o -o %tdef.so
+# RUN: not ld.lld --gc-sections %t.o %t.so %tdef.so %tdef-hidden.o -o /dev/null 2>&1 | FileCheck %s
+
 .globl _start
 _start:
   callq _shared@PLT
diff --git a/lld/test/ELF/cgprofile-txt.s b/lld/test/ELF/cgprofile-txt.s
index cf5b17627c..c9194bbbc4 100644
--- a/lld/test/ELF/cgprofile-txt.s
+++ b/lld/test/ELF/cgprofile-txt.s
@@ -26,12 +26,12 @@
 # RUN: echo "TooManyPreds10 TooManyPreds 11" >> %t.call_graph
 # RUN: ld.lld -e A %t --call-graph-ordering-file %t.call_graph --call-graph-profile-sort=hfsort -o %t2
 # RUN: llvm-readobj --symbols %t2 | FileCheck %s
+## --call-graph-profile-sort=hfsort is the default.
+# RUN: ld.lld -e A %t --call-graph-ordering-file %t.call_graph -o %t2b
+# RUN: cmp %t2 %t2b
 
 # RUN: ld.lld -e A %t --call-graph-ordering-file %t.call_graph --call-graph-profile-sort=cdsort -o %t2
 # RUN: llvm-readobj --symbols %t2 | FileCheck %s --check-prefix=CDSORT
-## --call-graph-profile-sort=cdsort is the default.
-# RUN: ld.lld -e A %t --call-graph-ordering-file %t.call_graph -o %t2b
-# RUN: cmp %t2 %t2b
 
 # RUN: not ld.lld -e A %t --call-graph-ordering-file %t.call_graph --call-graph-profile-sort=sort \
 # RUN:   -o /dev/null 2>&1 | FileCheck %s --check-prefix=UNKNOWN
diff --git a/lld/test/ELF/debug-dead-reloc.s b/lld/test/ELF/debug-dead-reloc.s
index e1adf4e2a2..fcf5320507 100644
--- a/lld/test/ELF/debug-dead-reloc.s
+++ b/lld/test/ELF/debug-dead-reloc.s
@@ -5,20 +5,47 @@
 
 # RUN: echo '.globl _start; _start: call group' | llvm-mc -filetype=obj -triple=x86_64 - -o %t.o
 # RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t1.o
-# RUN: ld.lld --gc-sections %t.o %t1.o %t1.o -o %t
+# RUN: ld.lld --emit-relocs --gc-sections %t.o %t1.o %t1.o -o %t
 # RUN: llvm-objdump -s %t | FileCheck %s
+# RUN: llvm-readobj -r %t | FileCheck %s --check-prefix=REL
 
 # CHECK:      Contents of section .debug_loc:
 # CHECK-NEXT:  0000 01000000 00000000 01000000 00000000
-# CHECK-NEXT: Contents of section .debug_ranges:
+# CHECK:      Contents of section .debug_ranges:
 # CHECK-NEXT:  0000 01000000 00000000 01000000 00000000
-# CHECK-NEXT: Contents of section .debug_addr:
+# CHECK:      Contents of section .debug_addr:
 # CHECK-NEXT:  0000 {{.*}}000 00000000 {{.*}}000 00000000
 # CHECK-NEXT:  0010 00000000  00000000 {{.*}}000 00000000
-# CHECK-NEXT: Contents of section .debug_foo:
+# CHECK:      Contents of section .debug_foo:
 # CHECK-NEXT:  0000 00000000 00000000 08000000 00000000
 # CHECK-NEXT:  0010 00000000 00000000 08000000 00000000
 
+# REL:      Relocations [
+# REL-NEXT:   .rela.text {
+# REL-NEXT:     0x201121 R_X86_64_PLT32 group 0xFFFFFFFFFFFFFFFC
+# REL-NEXT:   }
+# REL-NEXT:   .rela.debug_loc {
+# REL-NEXT:     0x0 R_X86_64_NONE - 0x8
+# REL-NEXT:     0x8 R_X86_64_NONE - 0x8
+# REL-NEXT:   }
+# REL-NEXT:   .rela.debug_ranges {
+# REL-NEXT:     0x0 R_X86_64_NONE - 0x10
+# REL-NEXT:     0x8 R_X86_64_NONE - 0x10
+# REL-NEXT:   }
+# REL-NEXT:   .rela.debug_addr {
+# REL-NEXT:     0x0 R_X86_64_64 .text 0x1D
+# REL-NEXT:     0x8 R_X86_64_64 group 0x20
+# REL-NEXT:     0x10 R_X86_64_NONE - 0x18
+# REL-NEXT:     0x18 R_X86_64_64 group 0x20
+# REL-NEXT:   }
+# REL-NEXT:   .rela.debug_foo {
+# REL-NEXT:     0x0 R_X86_64_NONE - 0x8
+# REL-NEXT:     0x8 R_X86_64_NONE - 0x8
+# REL-NEXT:     0x10 R_X86_64_NONE - 0x8
+# REL-NEXT:     0x18 R_X86_64_NONE - 0x8
+# REL-NEXT:   }
+# REL-NEXT: ]
+
 ## -z dead-reloc-in-nonalloc= can override the tombstone value.
 # RUN: ld.lld --gc-sections -z dead-reloc-in-nonalloc=.debug_loc=42 %t.o %t1.o %t1.o -o %t42
 # RUN: llvm-objdump -s %t42 | FileCheck %s --check-prefix=OVERRIDE
diff --git a/lld/test/ELF/gc-sections-tls.s b/lld/test/ELF/gc-sections-tls.s
index ccc9ac3c74..3036a676dd 100644
--- a/lld/test/ELF/gc-sections-tls.s
+++ b/lld/test/ELF/gc-sections-tls.s
@@ -1,23 +1,25 @@
 # REQUIRES: x86
 # RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t.o
 
-## Relocation in a non .debug_* referencing a discarded TLS symbol is invalid.
-## If we happen to have no PT_TLS, we will emit an error.
-# RUN: not ld.lld %t.o --gc-sections -o /dev/null 2>&1 | FileCheck %s --check-prefix=ERR
-
-# ERR: error: {{.*}}.o has an STT_TLS symbol but doesn't have an SHF_TLS section
+## When a TLS section is discarded, we will resolve the relocation in a non-SHF_ALLOC
+## section to the addend. Technically, we can emit an error in this case as the
+## relocation type is not TLS.
+# RUN: ld.lld %t.o --gc-sections -o %t
+# RUN: llvm-readelf -x .noalloc %t | FileCheck %s
 
-## If we happen to have a PT_TLS, we will resolve the relocation to
-## an arbitrary value (current implementation uses a negative value).
 # RUN: echo '.section .tbss,"awT"; .globl root; root: .long 0' | \
 # RUN:   llvm-mc -filetype=obj -triple=x86_64 - -o %t1.o
 # RUN: ld.lld --gc-sections -u root %t.o %t1.o -o %t
 # RUN: llvm-readelf -x .noalloc %t | FileCheck %s
 
 # CHECK:      Hex dump of section '.noalloc':
-# CHECK-NEXT: 0x00000000 {{[0-9a-f]+}} ffffffff
+# CHECK-NEXT: 0x00000000 00800000 00000000
+
+.globl _start
+_start:
 
 .section .tbss,"awT",@nobits
+  .long 0
 tls:
   .long 0
 
diff --git a/lld/test/ELF/linkerscript/discard-section.s b/lld/test/ELF/linkerscript/discard-section.s
index fbdff5dfbe..24f3b2b73e 100644
--- a/lld/test/ELF/linkerscript/discard-section.s
+++ b/lld/test/ELF/linkerscript/discard-section.s
@@ -1,14 +1,83 @@
 # REQUIRES: x86
-# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t
-# RUN: echo "SECTIONS { /DISCARD/ : { *(.aaa*) } }" > %t.script
-# RUN: ld.lld -o %t1 --script %t.script %t
-# RUN: llvm-objdump --section-headers %t1 | FileCheck %s
+## Test relocations referencing symbols defined relative to sections discarded by /DISCARD/.
 
-# CHECK-NOT: .aaa
+# RUN: rm -rf %t && split-file %s %t && cd %t
+# RUN: llvm-mc -filetype=obj -triple=x86_64 a.s -o a.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64 b.s -o b.o
+# RUN: not ld.lld --threads=1 -T a.lds a.o b.o -z undefs -o /dev/null 2>&1 | FileCheck %s --check-prefix=LOCAL --implicit-check-not=error:
+# RUN: not ld.lld --threads=1 -T a.lds a.o b.o -o /dev/null 2>&1 | FileCheck %s --check-prefixes=LOCAL,NONLOCAL --implicit-check-not=error:
+# RUN: ld.lld -r -T a.lds a.o b.o -o a.ro 2>&1 | FileCheck %s --check-prefix=WARNING --implicit-check-not=warning:
+# RUN: llvm-readelf -r -s a.ro | FileCheck %s --check-prefix=RELOC
+
+# LOCAL:      error: relocation refers to a discarded section: .aaa
+# LOCAL-NEXT: >>> defined in a.o
+# LOCAL-NEXT: >>> referenced by a.o:(.bbb+0x0)
+
+# NONLOCAL:      error: relocation refers to a symbol in a discarded section: global
+# NONLOCAL-NEXT: >>> defined in a.o
+# NONLOCAL-NEXT: >>> referenced by b.o:(.data+0x0)
+
+# NONLOCAL:      error: relocation refers to a symbol in a discarded section: weak
+# NONLOCAL-NEXT: >>> defined in a.o
+# NONLOCAL-NEXT: >>> referenced by b.o:(.data+0x8)
+
+# NONLOCAL:      error: relocation refers to a symbol in a discarded section: weakref1
+# NONLOCAL-NEXT: >>> defined in a.o
+# NONLOCAL-NEXT: >>> referenced by b.o:(.data+0x10)
+
+# NONLOCAL:      error: relocation refers to a symbol in a discarded section: weakref2
+# NONLOCAL-NEXT: >>> defined in a.o
+# NONLOCAL-NEXT: >>> referenced by b.o:(.data+0x18)
+
+# WARNING:      warning: relocation refers to a discarded section: .aaa
+# WARNING-NEXT: >>> referenced by a.o:(.rela.bbb+0x0)
+
+# RELOC:      Relocation section '.rela.bbb' at offset {{.*}} contains 1 entries:
+# RELOC-NEXT:     Offset             Info             Type               Symbol's Value  Symbol's Name + Addend
+# RELOC-NEXT: 0000000000000000  0000000000000000 R_X86_64_NONE                             0
+# RELOC-EMPTY:
+# RELOC-NEXT: Relocation section '.rela.data' at offset {{.*}} contains 4 entries:
+# RELOC-NEXT:     Offset             Info             Type               Symbol's Value  Symbol's Name + Addend
+# RELOC-NEXT: 0000000000000000  0000000500000001 R_X86_64_64            0000000000000000 global + 0
+# RELOC-NEXT: 0000000000000008  0000000700000001 R_X86_64_64            0000000000000000 weak + 0
+# RELOC-NEXT: 0000000000000010  0000000600000001 R_X86_64_64            0000000000000000 weakref1 + 0
+# RELOC-NEXT: 0000000000000018  0000000800000001 R_X86_64_64            0000000000000000 weakref2 + 0
+
+# RELOC:      Num:    Value          Size Type    Bind   Vis      Ndx Name
+# RELOC-NEXT:   0: 0000000000000000     0 NOTYPE  LOCAL  DEFAULT  UND
+# RELOC-NEXT:   1: 0000000000000000     0 SECTION LOCAL  DEFAULT    1 .text
+# RELOC-NEXT:   2: 0000000000000000     0 SECTION LOCAL  DEFAULT    2 .bbb
+# RELOC-NEXT:   3: 0000000000000000     0 SECTION LOCAL  DEFAULT    4 .data
+# RELOC-NEXT:   4: 0000000000000000     0 NOTYPE  GLOBAL DEFAULT    1 _start
+# RELOC-NEXT:   5: 0000000000000000     0 NOTYPE  GLOBAL DEFAULT   UND global
+# RELOC-NEXT:   6: 0000000000000000     0 NOTYPE  GLOBAL DEFAULT   UND weakref1
+# RELOC-NEXT:   7: 0000000000000000     0 NOTYPE  GLOBAL DEFAULT   UND weak
+# RELOC-NEXT:   8: 0000000000000000     0 NOTYPE  GLOBAL DEFAULT   UND weakref2
+# RELOC-EMPTY:
+
+#--- a.s
+.globl _start
+_start:
 
 .section .aaa,"a"
-aab:
+.globl global, weakref1
+.weak weak, weakref2
+global:
+weak:
+weakref1:
+weakref2:
   .quad 0
 
-.section .zzz,"a"
-  .quad aab
+.section .bbb,"aw"
+  .quad .aaa
+
+#--- b.s
+.weak weakref1, weakref2
+.section .data,"aw"
+  .quad global
+  .quad weak
+  .quad weakref1
+  .quad weakref2
+
+#--- a.lds
+SECTIONS { /DISCARD/ : { *(.aaa) } }
diff --git a/lld/wasm/WriterUtils.cpp b/lld/wasm/WriterUtils.cpp
index ead22291b5..cc8ed0b1de 100644
--- a/lld/wasm/WriterUtils.cpp
+++ b/lld/wasm/WriterUtils.cpp
@@ -111,12 +111,12 @@ void writeU8(raw_ostream &os, uint8_t byte, const Twine &msg) {
 
 void writeU32(raw_ostream &os, uint32_t number, const Twine &msg) {
   debugWrite(os.tell(), msg + "[0x" + utohexstr(number) + "]");
-  support::endian::write(os, number, support::little);
+  support::endian::write(os, number, llvm::endianness::little);
 }
 
 void writeU64(raw_ostream &os, uint64_t number, const Twine &msg) {
   debugWrite(os.tell(), msg + "[0x" + utohexstr(number) + "]");
-  support::endian::write(os, number, support::little);
+  support::endian::write(os, number, llvm::endianness::little);
 }
 
 void writeValueType(raw_ostream &os, ValType type, const Twine &msg) {
diff --git a/lldb/CodeOwners.rst b/lldb/CodeOwners.rst
index 5635f186fa..52e3e55052 100644
--- a/lldb/CodeOwners.rst
+++ b/lldb/CodeOwners.rst
@@ -234,8 +234,8 @@ lldb-server
 | Pavel Labath
 | pavel\@labath.sk (email), labath (Phabricator), labath (GitHub), labath (Discourse)
 
-lldb-vscode
-~~~~~~~~~~~
+lldb-dap
+~~~~~~~~
 | Greg Clayton
 | gclayton\@fb.com (email), clayborg (Phabricator), clayborg (GitHub), clayborg (Discourse)
 
diff --git a/lldb/bindings/interface/SBTypeDocstrings.i b/lldb/bindings/interface/SBTypeDocstrings.i
index 96421a6aa2..c49e9647ba 100644
--- a/lldb/bindings/interface/SBTypeDocstrings.i
+++ b/lldb/bindings/interface/SBTypeDocstrings.i
@@ -720,6 +720,14 @@ SBType supports the eq/ne operator. For example,::
     "
 ) lldb::SBType::GetTypeFlags;
 
+%feature("docstring",
+    "Searches for a directly nested type that has the provided name.
+
+    Returns the type if it was found.
+    Returns invalid type if nothing was found.
+    "
+) lldb::SBType::FindDirectNestedType;
+
 %feature("docstring",
 "Represents a list of :py:class:`SBType` s.
 
diff --git a/lldb/docs/index.rst b/lldb/docs/index.rst
index 2eb57cefbd..2fff25b27b 100644
--- a/lldb/docs/index.rst
+++ b/lldb/docs/index.rst
@@ -125,6 +125,7 @@ interesting areas to contribute to lldb.
    use/qemu-testing
    use/intel_pt
    use/ondemand
+   use/aarch64-linux
    use/troubleshooting
    use/links
    Man Page <man/lldb>
diff --git a/lldb/docs/lldb-gdb-remote.txt b/lldb/docs/lldb-gdb-remote.txt
index 6eed70eab0..58269e4c2b 100644
--- a/lldb/docs/lldb-gdb-remote.txt
+++ b/lldb/docs/lldb-gdb-remote.txt
@@ -581,11 +581,15 @@ read packet: <binary data>/E<error code>;AAAAAAAAA
 //  will get picked up automatically, and allows registers to change
 //  depending on the actual CPU type that is used.
 //
-//  NB: As of summer 2015, lldb can get register information from the
-//  "qXfer:features:read:target.xml" FSF gdb standard register packet
-//  where the stub provides register definitions in an XML file.
+//  NB: qRegisterInfo is deprecated in favor of the standard gdb remote
+//  serial protocol register description method, 
+//  "qXfer:features:read:target.xml".
 //  If qXfer:features:read:target.xml is supported, qRegisterInfo does
-//  not need to be implemented.
+//  not need to be implemented.  The target.xml format is used by most
+//  gdb RSP stubs whereas qRegisterInfo was an lldb-only design.
+//  qRegisterInfo requires one packet per register and can have undesirable
+//  performance costs at the start of a debug session, whereas target.xml
+//  may be able to describe all registers in a single packet.
 //----------------------------------------------------------------------
 
 With LLDB, for register information, remote GDB servers can add
diff --git a/lldb/docs/use/aarch64-linux.rst b/lldb/docs/use/aarch64-linux.rst
new file mode 100644
index 0000000000..707087a9bd
--- /dev/null
+++ b/lldb/docs/use/aarch64-linux.rst
@@ -0,0 +1,204 @@
+Using LLDB On AArch64 Linux
+===========================
+
+This page explains the details of debugging certain AArch64 extensions using
+LLDB. If something is not mentioned here, it likely works as you would expect.
+
+This is not a replacement for ptrace and Linux Kernel documentation. This covers
+how LLDB has chosen to use those things and how that effects your experience as
+a user.
+
+Scalable Vector Extension (SVE)
+-------------------------------
+
+See `here <https://developer.arm.com/Architectures/Scalable%20Vector%20Extensions>`__
+to learn about the extension and `here <https://kernel.org/doc/html/latest/arch/arm64/sve.html>`__
+for the Linux Kernel's handling of it.
+
+In LLDB you will be able to see the following new registers:
+
+* ``z0-z31`` vector registers, each one has size equal to the vector length.
+* ``p0-p15`` predicate registers, each one containing 1 bit per byte in the vector
+  length. Making each one vector length / 8 sized.
+* ``ffr`` the first fault register, same size as a predicate register.
+* ``vg``, the vector length in "granules". Each granule is 8 bytes.
+
+.. code-block::
+
+       Scalable Vector Extension Registers:
+             vg = 0x0000000000000002
+             z0 = {0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x01 <...> }
+           <...>
+             p0 = {0xff 0xff}
+           <...>
+            ffr = {0xff 0xff}
+
+The example above has a vector length of 16 bytes. Within LLDB you will always
+see "vg" as in the ``vg`` register, which is 2 in this case (8*2 = 16).
+Elsewhere in kernel code or applications, you may see "vq" which is the vector
+length in quadwords (16 bytes). Where you see "vl", it is in bytes.
+
+While you can count the size of a P or Z register, it is intended that ``vg`` be
+used to find the current vector length.
+
+Changing the Vector Length
+..........................
+
+The ``vg`` register can be written during a debug session. Writing the current
+vector length changes nothing. If you increase the vector length, the registers
+will likely be reset to 0. If you decrease it, LLDB will truncate the Z
+registers but everything else will be reset to 0.
+
+You should not assume that SVE state after changing the vector length is in any
+way the same as it was previously. Whether that is done from within the
+debuggee, or by LLDB. If you need to change the vector length, do so before a
+function's first use of SVE.
+
+Z Register Presentation
+.......................
+
+LLDB makes no attempt to predict how SVE Z registers will be used. Since LLDB
+does not know what sort of elements future instructions will interpret the
+register as. It therefore does not change the visualisation of the register
+and always defaults to showing a vector of byte sized elements.
+
+If you know what format you are going to use, give a format option::
+
+  (lldb) register read z0 -f uint32_t[]
+      z0 = {0x01010101 0x01010101 0x01010101 0x01010101}
+
+FPSIMD and SVE Modes
+....................
+
+Prior to the debugee's first use of SVE, it is in what the Linux Kernel terms
+SIMD mode. Only the FPU is being used. In this state LLDB will still show the
+SVE registers however the values are simply the FPU values zero extended up to
+the vector length.
+
+On first access to SVE, the process goes into SVE mode. Now the Z values are
+in the real Z registers.
+
+You can also trigger this with LLDB by writing to an SVE register. Note that
+there is no way to undo this change from within LLDB. However, the debugee
+itself could do something to end up back in SIMD mode.
+
+Expression evaluation
+.....................
+
+If you evaluate an expression, all SVE state is saved prior to, and restored
+after the expression has been evaluated. Including the register values and
+vector length.
+
+Scalable Matrix Extension (SME)
+-------------------------------
+
+See `here <https://community.arm.com/arm-community-blogs/b/architectures-and-processors-blog/posts/scalable-matrix-extension-armv9-a-architecture>`__
+to learn about the extension and `here <https://kernel.org/doc/html/latest/arch/arm64/sme.html>`__
+for the Linux Kernel's handling of it.
+
+SME adds a "Streaming Mode" to SVE, and this mode has its own vector length
+known as the "Streaming Vector Length".
+
+In LLDB you will see the following new registers:
+
+* ``tpidr2``, an extra per thread pointer reserved for use by the SME ABI.
+  This is not scalable, just pointer sized aka 64 bit.
+* ``z0-z31`` streaming SVE registers. These have the same names as the
+  non-streaming registers and therefore you will only see the active set in
+  LLDB. You cannot read or write the inactive mode's registers. Their size
+  is the same as the streaming vector length.
+* ``za`` the Array Storage register. The "Matrix" part of "Scalable Matrix
+  Extension". This is a square made up of rows of length equal to the streaming
+  vector length (svl). Meaning that the total size is svl * svl.
+* ``svcr`` the Streaming Vector Control Register. This is actually a pseduo
+  register but it matches the content of the architecturaly defined ``SVCR``.
+  This is the register you should use to check whether streaming mode and/or
+  ``za`` is active. This register is read only.
+* ``svg`` the streaming vector length in granules. This value is not connected
+  to the vector length of non-streaming mode and may change independently. This
+  register is read only.
+
+.. note::
+  While in non-streaming mode, the ``vg`` register shows the non-streaming
+  vector length, and the ``svg`` register shows the streaming vector length.
+  When in streaming mode, both ``vg`` and ``svg`` show the streaming mode vector
+  length. Therefore it is not possible at this time to read the non-streaming
+  vector length within LLDB, while in streaming mode. This is a limitation of
+  the LLDB implementation not the architecture, which stores both lengths
+  independently.
+
+In the example below, the streaming vector length is 16 bytes and we are in
+streaming mode. Note that bits 0 and 1 of ``svcr`` are set, indicating that we
+are in streaming mode and ZA is active. ``vg`` and ``svg`` report the same value
+as ``vg`` is showing the streaming mode vector length::
+
+  Scalable Vector Extension Registers:
+        vg = 0x0000000000000002
+        z0 = {0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x01 <...> }
+        <...>
+        p0 = {0xff 0xff}
+        <...>
+       ffr = {0xff 0xff}
+
+  <...>
+
+  Thread Local Storage Registers:
+       tpidr = 0x0000fffff7ff4320
+      tpidr2 = 0x1122334455667788
+
+  Scalable Matrix Array Storage Registers:
+          za = {0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x01 <...> }
+
+  Scalable Matrix Extension Registers:
+         svg = 0x0000000000000002
+        svcr = 0x0000000000000003
+
+Changing the Streaming Vector Length
+....................................
+
+To reduce complexity for LLDB, ``svg`` is read only. This means that you can
+only change the streaming vector length using LLDB when the debugee is in
+streaming mode.
+
+As for non-streaming SVE, doing so will essentially make the content of the SVE
+registers undefined. It will also disable ZA, which follows what the Linux
+Kernel does.
+
+Visibility of an Inactive ZA Register
+.....................................
+
+LLDB does not handle registers that can come and go at runtime (SVE changes
+size but it does not dissappear). Therefore when ``za`` is not enabled, LLDB
+will return a block of 0s instead. This block will match the expected size of
+``za``::
+
+  (lldb) register read za svg svcr
+      za = {0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 <...> }
+     svg = 0x0000000000000002
+    svcr = 0x0000000000000001
+
+Note that ``svcr`` bit 2 is not set, meaning ``za`` is inactive.
+
+If you were to write to ``za`` from LLDB, ``za`` will be made active. There is
+no way from within LLDB to reverse this change. As for changing the vector
+length, the debugee could still do something that would disable ``za`` again.
+
+If you want to know whether ``za`` is active or not, refer to bit 2 of the
+``svcr`` register, otherwise known as ``SVCR.ZA``.
+
+ZA Register Presentation
+........................
+
+As for SVE, LLDB does not know how the debugee will use ``za``, and therefore
+does not know how it would be best to display it. At any time any given
+instrucion could interpret its contents as many kinds and sizes of data.
+
+So LLDB will default to showing  ``za`` as one large vector of individual bytes.
+You can override this with a format option (see the SVE example above).
+
+Expression evaluation
+.....................
+
+The mode (streaming or non-streaming), streaming vector length and ZA state will
+be restored after expression evaluation. On top of all the things saved for SVE
+in general.
diff --git a/lldb/include/lldb/API/SBType.h b/lldb/include/lldb/API/SBType.h
index 5962f0c50d..9980fe1218 100644
--- a/lldb/include/lldb/API/SBType.h
+++ b/lldb/include/lldb/API/SBType.h
@@ -215,6 +215,8 @@ class SBType {
   bool GetDescription(lldb::SBStream &description,
                       lldb::DescriptionLevel description_level);
 
+  lldb::SBType FindDirectNestedType(const char *name);
+
   lldb::SBType &operator=(const lldb::SBType &rhs);
 
   bool operator==(lldb::SBType &rhs);
diff --git a/lldb/include/lldb/Expression/DWARFExpression.h b/lldb/include/lldb/Expression/DWARFExpression.h
index 5e03f539a2..1d85308d1c 100644
--- a/lldb/include/lldb/Expression/DWARFExpression.h
+++ b/lldb/include/lldb/Expression/DWARFExpression.h
@@ -18,10 +18,14 @@
 #include "llvm/DebugInfo/DWARF/DWARFLocationExpression.h"
 #include <functional>
 
-class DWARFUnit;
-
 namespace lldb_private {
 
+namespace plugin {
+namespace dwarf {
+class DWARFUnit;
+} // namespace dwarf
+} // namespace plugin
+
 /// \class DWARFExpression DWARFExpression.h
 /// "lldb/Expression/DWARFExpression.h" Encapsulates a DWARF location
 /// expression and interprets it.
@@ -64,18 +68,20 @@ class DWARFExpression {
   /// \return
   ///     The address specified by the operation, if the operation exists, or
   ///     LLDB_INVALID_ADDRESS otherwise.
-  lldb::addr_t GetLocation_DW_OP_addr(const DWARFUnit *dwarf_cu,
+  lldb::addr_t GetLocation_DW_OP_addr(const plugin::dwarf::DWARFUnit *dwarf_cu,
                                       bool &error) const;
 
-  bool Update_DW_OP_addr(const DWARFUnit *dwarf_cu, lldb::addr_t file_addr);
+  bool Update_DW_OP_addr(const plugin::dwarf::DWARFUnit *dwarf_cu,
+                         lldb::addr_t file_addr);
 
   void UpdateValue(uint64_t const_value, lldb::offset_t const_value_byte_size,
                    uint8_t addr_byte_size);
 
-  bool ContainsThreadLocalStorage(const DWARFUnit *dwarf_cu) const;
+  bool
+  ContainsThreadLocalStorage(const plugin::dwarf::DWARFUnit *dwarf_cu) const;
 
   bool LinkThreadLocalStorage(
-      const DWARFUnit *dwarf_cu,
+      const plugin::dwarf::DWARFUnit *dwarf_cu,
       std::function<lldb::addr_t(lldb::addr_t file_addr)> const
           &link_address_callback);
 
@@ -128,13 +134,13 @@ class DWARFExpression {
   ///     details of the failure are provided through it.
   static bool Evaluate(ExecutionContext *exe_ctx, RegisterContext *reg_ctx,
                        lldb::ModuleSP module_sp, const DataExtractor &opcodes,
-                       const DWARFUnit *dwarf_cu,
+                       const plugin::dwarf::DWARFUnit *dwarf_cu,
                        const lldb::RegisterKind reg_set,
                        const Value *initial_value_ptr,
                        const Value *object_address_ptr, Value &result,
                        Status *error_ptr);
 
-  static bool ParseDWARFLocationList(const DWARFUnit *dwarf_cu,
+  static bool ParseDWARFLocationList(const plugin::dwarf::DWARFUnit *dwarf_cu,
                                      const DataExtractor &data,
                                      DWARFExpressionList *loc_list);
 
diff --git a/lldb/include/lldb/Expression/DWARFExpressionList.h b/lldb/include/lldb/Expression/DWARFExpressionList.h
index c093964705..c2218ad4af 100644
--- a/lldb/include/lldb/Expression/DWARFExpressionList.h
+++ b/lldb/include/lldb/Expression/DWARFExpressionList.h
@@ -13,10 +13,14 @@
 #include "lldb/Utility/RangeMap.h"
 #include "lldb/lldb-private.h"
 
-class DWARFUnit;
-
 namespace lldb_private {
 
+namespace plugin {
+namespace dwarf {
+class DWARFUnit;
+} // namespace dwarf
+} // namespace plugin
+
 /// \class DWARFExpressionList DWARFExpressionList.h
 /// "lldb/Expression/DWARFExpressionList.h" Encapsulates a range map from file
 /// address range to a single DWARF location expression.
@@ -24,13 +28,14 @@ class DWARFExpressionList {
 public:
   DWARFExpressionList() = default;
 
-  DWARFExpressionList(lldb::ModuleSP module_sp, const DWARFUnit *dwarf_cu,
+  DWARFExpressionList(lldb::ModuleSP module_sp,
+                      const plugin::dwarf::DWARFUnit *dwarf_cu,
                       lldb::addr_t func_file_addr)
       : m_module_wp(module_sp), m_dwarf_cu(dwarf_cu),
         m_func_file_addr(func_file_addr) {}
 
   DWARFExpressionList(lldb::ModuleSP module_sp, DWARFExpression expr,
-                      const DWARFUnit *dwarf_cu)
+                      const plugin::dwarf::DWARFUnit *dwarf_cu)
       : m_module_wp(module_sp), m_dwarf_cu(dwarf_cu) {
     AddExpression(0, LLDB_INVALID_ADDRESS, expr);
   }
@@ -136,7 +141,7 @@ class DWARFExpressionList {
   /// The DWARF compile unit this expression belongs to. It is used to evaluate
   /// values indexing into the .debug_addr section (e.g. DW_OP_GNU_addr_index,
   /// DW_OP_GNU_const_index)
-  const DWARFUnit *m_dwarf_cu = nullptr;
+  const plugin::dwarf::DWARFUnit *m_dwarf_cu = nullptr;
 
   // Function base file address.
   lldb::addr_t m_func_file_addr = LLDB_INVALID_ADDRESS;
diff --git a/lldb/include/lldb/Symbol/CompileUnit.h b/lldb/include/lldb/Symbol/CompileUnit.h
index 229ee2d27a..93f191b499 100644
--- a/lldb/include/lldb/Symbol/CompileUnit.h
+++ b/lldb/include/lldb/Symbol/CompileUnit.h
@@ -331,8 +331,7 @@ class CompileUnit : public std::enable_shared_from_this<CompileUnit>,
   ///     A line table object pointer that this object now owns.
   void SetLineTable(LineTable *line_table);
 
-  void SetSupportFiles(const FileSpecList &support_files);
-  void SetSupportFiles(FileSpecList &&support_files);
+  void SetSupportFiles(FileSpecList support_files);
 
   void SetDebugMacros(const DebugMacrosSP &debug_macros);
 
diff --git a/lldb/include/lldb/Symbol/CompilerType.h b/lldb/include/lldb/Symbol/CompilerType.h
index 2d7092d2c9..0a9533a1ac 100644
--- a/lldb/include/lldb/Symbol/CompilerType.h
+++ b/lldb/include/lldb/Symbol/CompilerType.h
@@ -42,10 +42,7 @@ class CompilerType {
   ///
   /// \see lldb_private::TypeSystemClang::GetType(clang::QualType)
   CompilerType(lldb::TypeSystemWP type_system,
-               lldb::opaque_compiler_type_t type)
-    : m_type_system(type_system), m_type(type) {
-    assert(Verify() && "verification failed");
-  }
+               lldb::opaque_compiler_type_t type);
 
   /// This is a minimal wrapper of a TypeSystem shared pointer as
   /// returned by CompilerType which conventien dyn_cast support.
@@ -88,10 +85,8 @@ class CompilerType {
     lldb::TypeSystemSP GetSharedPointer() const { return m_typesystem_sp; }
   };
 
-  CompilerType(TypeSystemSPWrapper type_system, lldb::opaque_compiler_type_t type)
-    : m_type_system(type_system.GetSharedPointer()), m_type(type) {
-    assert(Verify() && "verification failed");
-  }
+  CompilerType(TypeSystemSPWrapper type_system,
+               lldb::opaque_compiler_type_t type);
 
   CompilerType(const CompilerType &rhs)
       : m_type_system(rhs.m_type_system), m_type(rhs.m_type) {}
@@ -146,8 +141,6 @@ class CompilerType {
 
   bool IsConst() const;
 
-  bool IsCStringType(uint32_t &length) const;
-
   bool IsDefined() const;
 
   bool IsFloatingPointType(uint32_t &count, bool &is_complex) const;
@@ -437,21 +430,11 @@ class CompilerType {
   LLVM_DUMP_METHOD void dump() const;
 #endif
 
-  void DumpValue(ExecutionContext *exe_ctx, Stream *s, lldb::Format format,
-                 const DataExtractor &data, lldb::offset_t data_offset,
-                 size_t data_byte_size, uint32_t bitfield_bit_size,
-                 uint32_t bitfield_bit_offset, bool show_types,
-                 bool show_summary, bool verbose, uint32_t depth);
-
   bool DumpTypeValue(Stream *s, lldb::Format format, const DataExtractor &data,
                      lldb::offset_t data_offset, size_t data_byte_size,
                      uint32_t bitfield_bit_size, uint32_t bitfield_bit_offset,
                      ExecutionContextScope *exe_scope);
 
-  void DumpSummary(ExecutionContext *exe_ctx, Stream *s,
-                   const DataExtractor &data, lldb::offset_t data_offset,
-                   size_t data_byte_size);
-
   /// Dump to stdout.
   void DumpTypeDescription(lldb::DescriptionLevel level =
                            lldb::eDescriptionLevelFull) const;
diff --git a/lldb/include/lldb/Symbol/SymbolFile.h b/lldb/include/lldb/Symbol/SymbolFile.h
index 8de752816c..512dd9acb8 100644
--- a/lldb/include/lldb/Symbol/SymbolFile.h
+++ b/lldb/include/lldb/Symbol/SymbolFile.h
@@ -22,6 +22,7 @@
 #include "lldb/Symbol/TypeList.h"
 #include "lldb/Symbol/TypeSystem.h"
 #include "lldb/Target/Statistics.h"
+#include "lldb/Utility/StructuredData.h"
 #include "lldb/Utility/XcodeSDK.h"
 #include "lldb/lldb-private.h"
 #include "llvm/ADT/DenseSet.h"
@@ -434,6 +435,18 @@ class SymbolFile : public PluginInterface {
   virtual bool GetDebugInfoHadFrameVariableErrors() const = 0;
   virtual void SetDebugInfoHadFrameVariableErrors() = 0;
 
+  /// Return true if separate debug info files are supported and this function
+  /// succeeded, false otherwise.
+  ///
+  /// \param[out] d
+  ///     If this function succeeded, then this will be a dictionary that
+  ///     contains the keys "type", "symfile", and "separate-debug-info-files".
+  ///     "type" can be used to assume the structure of each object in
+  ///     "separate-debug-info-files".
+  virtual bool GetSeparateDebugInfo(StructuredData::Dictionary &d) {
+    return false;
+  };
+
   virtual lldb::TypeSP
   MakeType(lldb::user_id_t uid, ConstString name,
            std::optional<uint64_t> byte_size, SymbolContextScope *context,
diff --git a/lldb/include/lldb/Symbol/Type.h b/lldb/include/lldb/Symbol/Type.h
index 046501931d..c505262cd9 100644
--- a/lldb/include/lldb/Symbol/Type.h
+++ b/lldb/include/lldb/Symbol/Type.h
@@ -148,15 +148,6 @@ class Type : public std::enable_shared_from_this<Type>, public UserID {
 
   ConstString GetQualifiedName();
 
-  void DumpValue(ExecutionContext *exe_ctx, Stream *s,
-                 const DataExtractor &data, uint32_t data_offset,
-                 bool show_type, bool show_summary, bool verbose,
-                 lldb::Format format = lldb::eFormatDefault);
-
-  bool DumpValueInMemory(ExecutionContext *exe_ctx, Stream *s,
-                         lldb::addr_t address, AddressType address_type,
-                         bool show_types, bool show_summary, bool verbose);
-
   bool ReadFromMemory(ExecutionContext *exe_ctx, lldb::addr_t address,
                       AddressType address_type, DataExtractor &data);
 
@@ -313,6 +304,8 @@ class TypeImpl {
   bool GetDescription(lldb_private::Stream &strm,
                       lldb::DescriptionLevel description_level);
 
+  CompilerType FindDirectNestedType(llvm::StringRef name);
+
 private:
   bool CheckModule(lldb::ModuleSP &module_sp) const;
   bool CheckExeModule(lldb::ModuleSP &module_sp) const;
diff --git a/lldb/include/lldb/Symbol/TypeSystem.h b/lldb/include/lldb/Symbol/TypeSystem.h
index eb6e453e1a..5ac16be334 100644
--- a/lldb/include/lldb/Symbol/TypeSystem.h
+++ b/lldb/include/lldb/Symbol/TypeSystem.h
@@ -28,11 +28,17 @@
 #include "lldb/Symbol/CompilerDeclContext.h"
 #include "lldb/lldb-private.h"
 
-class DWARFDIE;
-class DWARFASTParser;
 class PDBASTParser;
 
 namespace lldb_private {
+
+namespace plugin {
+namespace dwarf {
+class DWARFDIE;
+class DWARFASTParser;
+} // namespace dwarf
+} // namespace plugin
+
 namespace npdb {
   class PdbAstBuilder;
 } // namespace npdb
@@ -93,7 +99,8 @@ class TypeSystem : public PluginInterface,
   /// removing all the TypeSystems from the TypeSystemMap.
   virtual void Finalize() {}
 
-  virtual DWARFASTParser *GetDWARFParser() { return nullptr; }
+  virtual plugin::dwarf::DWARFASTParser *GetDWARFParser() { return nullptr; }
+
   virtual PDBASTParser *GetPDBParser() { return nullptr; }
   virtual npdb::PdbAstBuilder *GetNativePDBParser() { return nullptr; }
 
@@ -135,6 +142,10 @@ class TypeSystem : public PluginInterface,
 
   virtual lldb::LanguageType DeclContextGetLanguage(void *opaque_decl_ctx) = 0;
 
+  /// Returns the direct parent context of specified type
+  virtual CompilerDeclContext
+  GetCompilerDeclContextForType(const CompilerType &type);
+
   // Tests
 #ifndef NDEBUG
   /// Verify the integrity of the type to catch CompilerTypes that mix
@@ -384,14 +395,6 @@ class TypeSystem : public PluginInterface,
   dump(lldb::opaque_compiler_type_t type) const = 0;
 #endif
 
-  virtual void DumpValue(lldb::opaque_compiler_type_t type,
-                         ExecutionContext *exe_ctx, Stream &s,
-                         lldb::Format format, const DataExtractor &data,
-                         lldb::offset_t data_offset, size_t data_byte_size,
-                         uint32_t bitfield_bit_size,
-                         uint32_t bitfield_bit_offset, bool show_types,
-                         bool show_summary, bool verbose, uint32_t depth) = 0;
-
   virtual bool DumpTypeValue(lldb::opaque_compiler_type_t type, Stream &s,
                              lldb::Format format, const DataExtractor &data,
                              lldb::offset_t data_offset, size_t data_byte_size,
@@ -418,16 +421,9 @@ class TypeSystem : public PluginInterface,
   /// This should not modify the state of the TypeSystem if possible.
   virtual void Dump(llvm::raw_ostream &output) = 0;
 
-  // TODO: These methods appear unused. Should they be removed?
-
+  /// This is used by swift.
   virtual bool IsRuntimeGeneratedType(lldb::opaque_compiler_type_t type) = 0;
 
-  virtual void DumpSummary(lldb::opaque_compiler_type_t type,
-                           ExecutionContext *exe_ctx, Stream &s,
-                           const DataExtractor &data,
-                           lldb::offset_t data_offset,
-                           size_t data_byte_size) = 0;
-
   // TODO: Determine if these methods should move to TypeSystemClang.
 
   virtual bool IsPointerOrReferenceType(lldb::opaque_compiler_type_t type,
@@ -435,9 +431,6 @@ class TypeSystem : public PluginInterface,
 
   virtual unsigned GetTypeQualifiers(lldb::opaque_compiler_type_t type) = 0;
 
-  virtual bool IsCStringType(lldb::opaque_compiler_type_t type,
-                             uint32_t &length) = 0;
-
   virtual std::optional<size_t>
   GetTypeBitAlign(lldb::opaque_compiler_type_t type,
                   ExecutionContextScope *exe_scope) = 0;
@@ -581,6 +574,6 @@ class TypeSystemMap {
       std::optional<CreateCallback> create_callback = std::nullopt);
   };
 
-} // namespace lldb_private
+  } // namespace lldb_private
 
 #endif // LLDB_SYMBOL_TYPESYSTEM_H
diff --git a/lldb/include/lldb/Target/Target.h b/lldb/include/lldb/Target/Target.h
index 8752b42a95..8204598801 100644
--- a/lldb/include/lldb/Target/Target.h
+++ b/lldb/include/lldb/Target/Target.h
@@ -683,7 +683,7 @@ class Target : public std::enable_shared_from_this<Target>,
   // Use this to create a breakpoint from a load address and a module file spec
   lldb::BreakpointSP CreateAddressInModuleBreakpoint(lldb::addr_t file_addr,
                                                      bool internal,
-                                                     const FileSpec *file_spec,
+                                                     const FileSpec &file_spec,
                                                      bool request_hardware);
 
   // Use this to create Address breakpoints:
diff --git a/lldb/include/lldb/Utility/FileSpecList.h b/lldb/include/lldb/Utility/FileSpecList.h
index 6b88cd2f30..77587aa917 100644
--- a/lldb/include/lldb/Utility/FileSpecList.h
+++ b/lldb/include/lldb/Utility/FileSpecList.h
@@ -155,19 +155,6 @@ class FileSpecList {
   ///     returned.
   const FileSpec &GetFileSpecAtIndex(size_t idx) const;
 
-  /// Get file specification pointer at index.
-  ///
-  /// Gets a file from the file list. The file objects that are returned can
-  /// be tested using FileSpec::operator void*().
-  ///
-  /// \param[in] idx
-  ///     An index into the file list.
-  ///
-  /// \return
-  ///     A pointer to a contained FileSpec object at index \a idx.
-  ///     If \a idx is out of range, then an NULL is returned.
-  const FileSpec *GetFileSpecPointerAtIndex(size_t idx) const;
-
   /// Get the memory cost of this object.
   ///
   /// Return the size in bytes that this object takes in memory. This returns
@@ -213,9 +200,6 @@ class FileSpecList {
     return false;
   }
 
-  static size_t GetFilesMatchingPartialPath(const char *path, bool dir_okay,
-                                            FileSpecList &matches);
-
   const_iterator begin() const { return m_files.begin(); }
   const_iterator end() const { return m_files.end(); }
 
diff --git a/lldb/include/lldb/Utility/Scalar.h b/lldb/include/lldb/Utility/Scalar.h
index 34c2111ae0..8e087a5dde 100644
--- a/lldb/include/lldb/Utility/Scalar.h
+++ b/lldb/include/lldb/Utility/Scalar.h
@@ -101,7 +101,7 @@ class Scalar {
 
   const char *GetTypeAsCString() const { return GetValueTypeAsCString(m_type); }
 
-  void GetValue(Stream *s, bool show_type) const;
+  void GetValue(Stream &s, bool show_type) const;
 
   bool IsValid() const { return (m_type >= e_int) && (m_type <= e_float); }
 
diff --git a/lldb/packages/Python/lldbsuite/test/dotest.py b/lldb/packages/Python/lldbsuite/test/dotest.py
index a27d8cf321..a639714480 100644
--- a/lldb/packages/Python/lldbsuite/test/dotest.py
+++ b/lldb/packages/Python/lldbsuite/test/dotest.py
@@ -485,15 +485,15 @@ def setupSysPath():
     os.environ["LLDB_SRC"] = lldbsuite.lldb_root
 
     pluginPath = os.path.join(scriptPath, "plugins")
-    toolsLLDBVSCode = os.path.join(scriptPath, "tools", "lldb-vscode")
+    toolsLLDBDAP = os.path.join(scriptPath, "tools", "lldb-dap")
     toolsLLDBServerPath = os.path.join(scriptPath, "tools", "lldb-server")
     intelpt = os.path.join(scriptPath, "tools", "intelpt")
 
     # Insert script dir, plugin dir and lldb-server dir to the sys.path.
     sys.path.insert(0, pluginPath)
-    # Adding test/tools/lldb-vscode to the path makes it easy to
-    # "import lldb_vscode_testcase" from the VSCode tests
-    sys.path.insert(0, toolsLLDBVSCode)
+    # Adding test/tools/lldb-dap to the path makes it easy to
+    # "import lldb_dap_testcase" from the DAP tests
+    sys.path.insert(0, toolsLLDBDAP)
     # Adding test/tools/lldb-server to the path makes it easy
     # to "import lldbgdbserverutils" from the lldb-server tests
     sys.path.insert(0, toolsLLDBServerPath)
@@ -538,15 +538,15 @@ def setupSysPath():
 
     lldbDir = os.path.dirname(lldbtest_config.lldbExec)
 
-    lldbVSCodeExec = os.path.join(lldbDir, "lldb-vscode")
-    if is_exe(lldbVSCodeExec):
-        os.environ["LLDBVSCODE_EXEC"] = lldbVSCodeExec
+    lldbDAPExec = os.path.join(lldbDir, "lldb-dap")
+    if is_exe(lldbDAPExec):
+        os.environ["LLDBDAP_EXEC"] = lldbDAPExec
     else:
-        if not configuration.shouldSkipBecauseOfCategories(["lldb-vscode"]):
+        if not configuration.shouldSkipBecauseOfCategories(["lldb-dap"]):
             print(
-                "The 'lldb-vscode' executable cannot be located.  The lldb-vscode tests can not be run as a result."
+                "The 'lldb-dap' executable cannot be located.  The lldb-dap tests can not be run as a result."
             )
-            configuration.skip_categories.append("lldb-vscode")
+            configuration.skip_categories.append("lldb-dap")
 
     lldbPythonDir = None  # The directory that contains 'lldb/__init__.py'
 
diff --git a/lldb/packages/Python/lldbsuite/test/lldbtest.py b/lldb/packages/Python/lldbsuite/test/lldbtest.py
index c8670b208e..6d736a56ec 100644
--- a/lldb/packages/Python/lldbsuite/test/lldbtest.py
+++ b/lldb/packages/Python/lldbsuite/test/lldbtest.py
@@ -801,10 +801,10 @@ def setUp(self):
         else:
             self.libcxxPath = None
 
-        if "LLDBVSCODE_EXEC" in os.environ:
-            self.lldbVSCodeExec = os.environ["LLDBVSCODE_EXEC"]
+        if "LLDBDAP_EXEC" in os.environ:
+            self.lldbDAPExec = os.environ["LLDBDAP_EXEC"]
         else:
-            self.lldbVSCodeExec = None
+            self.lldbDAPExec = None
 
         self.lldbOption = " ".join("-o '" + s + "'" for s in self.setUpCommands())
 
diff --git a/lldb/packages/Python/lldbsuite/test/test_categories.py b/lldb/packages/Python/lldbsuite/test/test_categories.py
index 3883c4de5e..3f8de175e2 100644
--- a/lldb/packages/Python/lldbsuite/test/test_categories.py
+++ b/lldb/packages/Python/lldbsuite/test/test_categories.py
@@ -31,7 +31,7 @@
     "libc++": "Test for libc++ data formatters",
     "libstdcxx": "Test for libstdcxx data formatters",
     "lldb-server": "Tests related to lldb-server",
-    "lldb-vscode": "Visual Studio Code debug adaptor tests",
+    "lldb-dap": "Tests for the Debug Adaptor Protocol with lldb-dap",
     "llgs": "Tests for the gdb-server functionality of lldb-server",
     "objc": "Tests related to the Objective-C programming language support",
     "pyapi": "Tests related to the Python API",
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/vscode.py b/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py
similarity index 98%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/vscode.py
rename to lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py
index 5ee0800b27..8c645e0fdc 100644
--- a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/vscode.py
+++ b/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py
@@ -80,7 +80,7 @@ def read_packet(f, verbose=False, trace_file=None):
         # Decode the JSON bytes into a python dictionary
         return json.loads(json_str)
 
-    raise Exception("unexpected malformed message from lldb-vscode: " + line)
+    raise Exception("unexpected malformed message from lldb-dap: " + line)
 
 
 def packet_type_is(packet, packet_type):
@@ -104,7 +104,7 @@ def read_packet_thread(vs_comm, log_file):
             packet = read_packet(vs_comm.recv, trace_file=vs_comm.trace_file)
             # `packet` will be `None` on EOF. We want to pass it down to
             # handle_recv_packet anyway so the main thread can handle unexpected
-            # termination of lldb-vscode and stop waiting for new packets.
+            # termination of lldb-dap and stop waiting for new packets.
             done = not vs_comm.handle_recv_packet(packet)
     finally:
         dump_dap_log(log_file)
@@ -1013,7 +1013,7 @@ def terminate(self):
         # self.recv.close()
 
 
-class DebugAdaptor(DebugCommunication):
+class DebugAdaptorServer(DebugCommunication):
     def __init__(
         self, executable=None, port=None, init_commands=[], log_file=None, env=None
     ):
@@ -1024,7 +1024,7 @@ def __init__(
                 adaptor_env.update(env)
 
             if log_file:
-                adaptor_env["LLDBVSCODE_LOG"] = log_file
+                adaptor_env["LLDBDAP_LOG"] = log_file
             self.process = subprocess.Popen(
                 [executable],
                 stdin=subprocess.PIPE,
@@ -1048,7 +1048,7 @@ def get_pid(self):
         return -1
 
     def terminate(self):
-        super(DebugAdaptor, self).terminate()
+        super(DebugAdaptorServer, self).terminate()
         if self.process is not None:
             self.process.terminate()
             self.process.wait()
@@ -1172,7 +1172,7 @@ def main():
         dest="debuggerRoot",
         default=None,
         help=(
-            "Set the working directory for lldb-vscode for any object files "
+            "Set the working directory for lldb-dap for any object files "
             "with relative paths in the Mach-o debug map."
         ),
     )
@@ -1203,7 +1203,7 @@ def main():
         action="store_true",
         dest="sourceInitFile",
         default=False,
-        help="Whether lldb-vscode should source .lldbinit file or not",
+        help="Whether lldb-dap should source .lldbinit file or not",
     )
 
     parser.add_option(
@@ -1360,11 +1360,11 @@ def main():
         print(
             "error: must either specify a path to a Visual Studio Code "
             "Debug Adaptor vscode executable path using the --vscode "
-            "option, or a port to attach to for an existing lldb-vscode "
+            "option, or a port to attach to for an existing lldb-dap "
             "using the --port option"
         )
         return
-    dbg = DebugAdaptor(executable=options.vscode_path, port=options.port)
+    dbg = DebugAdaptorServer(executable=options.vscode_path, port=options.port)
     if options.debug:
         raw_input('Waiting for debugger to attach pid "%i"' % (dbg.get_pid()))
     if options.replay:
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/lldbvscode_testcase.py b/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/lldbdap_testcase.py
similarity index 82%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/lldbvscode_testcase.py
rename to lldb/packages/Python/lldbsuite/test/tools/lldb-dap/lldbdap_testcase.py
index 8cd4e8454c..3094b66af4 100644
--- a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/lldbvscode_testcase.py
+++ b/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/lldbdap_testcase.py
@@ -1,29 +1,29 @@
 import os
 import time
 
-import vscode
+import dap_server
 from lldbsuite.test.lldbtest import *
 
 
-class VSCodeTestCaseBase(TestBase):
+class DAPTestCaseBase(TestBase):
     NO_DEBUG_INFO_TESTCASE = True
 
-    def create_debug_adaptor(self, lldbVSCodeEnv=None):
+    def create_debug_adaptor(self, lldbDAPEnv=None):
         """Create the Visual Studio Code debug adaptor"""
         self.assertTrue(
-            is_exe(self.lldbVSCodeExec), "lldb-vscode must exist and be executable"
+            is_exe(self.lldbDAPExec), "lldb-dap must exist and be executable"
         )
-        log_file_path = self.getBuildArtifact("vscode.txt")
-        self.vscode = vscode.DebugAdaptor(
-            executable=self.lldbVSCodeExec,
+        log_file_path = self.getBuildArtifact("dap.txt")
+        self.dap_server = dap_server.DebugAdaptorServer(
+            executable=self.lldbDAPExec,
             init_commands=self.setUpCommands(),
             log_file=log_file_path,
-            env=lldbVSCodeEnv,
+            env=lldbDAPEnv,
         )
 
-    def build_and_create_debug_adaptor(self, lldbVSCodeEnv=None):
+    def build_and_create_debug_adaptor(self, lldbDAPEnv=None):
         self.build()
-        self.create_debug_adaptor(lldbVSCodeEnv)
+        self.create_debug_adaptor(lldbDAPEnv)
 
     def set_source_breakpoints(self, source_path, lines, data=None):
         """Sets source breakpoints and returns an array of strings containing
@@ -32,7 +32,7 @@ def set_source_breakpoints(self, source_path, lines, data=None):
         Each object in data is 1:1 mapping with the entry in lines.
         It contains optional location/hitCondition/logMessage parameters.
         """
-        response = self.vscode.request_setBreakpoints(source_path, lines, data)
+        response = self.dap_server.request_setBreakpoints(source_path, lines, data)
         if response is None:
             return []
         breakpoints = response["body"]["breakpoints"]
@@ -46,7 +46,7 @@ def set_function_breakpoints(self, functions, condition=None, hitCondition=None)
         and returns an array of strings containing the breakpoint IDs
         ("1", "2") for each breakpoint that was set.
         """
-        response = self.vscode.request_setFunctionBreakpoints(
+        response = self.dap_server.request_setFunctionBreakpoints(
             functions, condition=condition, hitCondition=hitCondition
         )
         if response is None:
@@ -70,7 +70,7 @@ def verify_breakpoint_hit(self, breakpoint_ids):
         "breakpoint_ids" should be a list of breakpoint ID strings
         (["1", "2"]). The return value from self.set_source_breakpoints()
         or self.set_function_breakpoints() can be passed to this function"""
-        stopped_events = self.vscode.wait_for_stopped()
+        stopped_events = self.dap_server.wait_for_stopped()
         for stopped_event in stopped_events:
             if "body" in stopped_event:
                 body = stopped_event["body"]
@@ -83,7 +83,7 @@ def verify_breakpoint_hit(self, breakpoint_ids):
                 # Descriptions for breakpoints will be in the form
                 # "breakpoint 1.1", so look for any description that matches
                 # ("breakpoint 1.") in the description field as verification
-                # that one of the breakpoint locations was hit. VSCode doesn't
+                # that one of the breakpoint locations was hit. DAP doesn't
                 # allow breakpoints to have multiple locations, but LLDB does.
                 # So when looking at the description we just want to make sure
                 # the right breakpoint matches and not worry about the actual
@@ -100,7 +100,7 @@ def verify_stop_exception_info(self, expected_description):
         reason is 'exception' and that the description matches
         'expected_description'
         """
-        stopped_events = self.vscode.wait_for_stopped()
+        stopped_events = self.dap_server.wait_for_stopped()
         for stopped_event in stopped_events:
             if "body" in stopped_event:
                 body = stopped_event["body"]
@@ -150,7 +150,7 @@ def get_dict_value(self, d, key_path):
     def get_stackFrames_and_totalFramesCount(
         self, threadId=None, startFrame=None, levels=None, dump=False
     ):
-        response = self.vscode.request_stackTrace(
+        response = self.dap_server.request_stackTrace(
             threadId=threadId, startFrame=startFrame, levels=levels, dump=dump
         )
         if response:
@@ -185,16 +185,16 @@ def get_source_and_line(self, threadId=None, frameIndex=0):
         return ("", 0)
 
     def get_stdout(self, timeout=0.0):
-        return self.vscode.get_output("stdout", timeout=timeout)
+        return self.dap_server.get_output("stdout", timeout=timeout)
 
     def get_console(self, timeout=0.0):
-        return self.vscode.get_output("console", timeout=timeout)
+        return self.dap_server.get_output("console", timeout=timeout)
 
     def collect_console(self, duration):
-        return self.vscode.collect_output("console", duration=duration)
+        return self.dap_server.collect_output("console", duration=duration)
 
     def get_local_as_int(self, name, threadId=None):
-        value = self.vscode.get_local_variable_value(name, threadId=threadId)
+        value = self.dap_server.get_local_variable_value(name, threadId=threadId)
         if value.startswith("0x"):
             return int(value, 16)
         elif value.startswith("0"):
@@ -204,48 +204,48 @@ def get_local_as_int(self, name, threadId=None):
 
     def set_local(self, name, value, id=None):
         """Set a top level local variable only."""
-        return self.vscode.request_setVariable(1, name, str(value), id=id)
+        return self.dap_server.request_setVariable(1, name, str(value), id=id)
 
     def set_global(self, name, value, id=None):
         """Set a top level global variable only."""
-        return self.vscode.request_setVariable(2, name, str(value), id=id)
+        return self.dap_server.request_setVariable(2, name, str(value), id=id)
 
     def stepIn(self, threadId=None, waitForStop=True):
-        self.vscode.request_stepIn(threadId=threadId)
+        self.dap_server.request_stepIn(threadId=threadId)
         if waitForStop:
-            return self.vscode.wait_for_stopped()
+            return self.dap_server.wait_for_stopped()
         return None
 
     def stepOver(self, threadId=None, waitForStop=True):
-        self.vscode.request_next(threadId=threadId)
+        self.dap_server.request_next(threadId=threadId)
         if waitForStop:
-            return self.vscode.wait_for_stopped()
+            return self.dap_server.wait_for_stopped()
         return None
 
     def stepOut(self, threadId=None, waitForStop=True):
-        self.vscode.request_stepOut(threadId=threadId)
+        self.dap_server.request_stepOut(threadId=threadId)
         if waitForStop:
-            return self.vscode.wait_for_stopped()
+            return self.dap_server.wait_for_stopped()
         return None
 
     def continue_to_next_stop(self):
-        self.vscode.request_continue()
-        return self.vscode.wait_for_stopped()
+        self.dap_server.request_continue()
+        return self.dap_server.wait_for_stopped()
 
     def continue_to_breakpoints(self, breakpoint_ids):
-        self.vscode.request_continue()
+        self.dap_server.request_continue()
         self.verify_breakpoint_hit(breakpoint_ids)
 
     def continue_to_exception_breakpoint(self, filter_label):
-        self.vscode.request_continue()
+        self.dap_server.request_continue()
         self.assertTrue(
             self.verify_stop_exception_info(filter_label),
             'verify we got "%s"' % (filter_label),
         )
 
     def continue_to_exit(self, exitCode=0):
-        self.vscode.request_continue()
-        stopped_events = self.vscode.wait_for_stopped()
+        self.dap_server.request_continue()
+        stopped_events = self.dap_server.wait_for_stopped()
         self.assertEquals(
             len(stopped_events), 1, "stopped_events = {}".format(stopped_events)
         )
@@ -266,10 +266,10 @@ def disassemble(self, threadId=None, frameIndex=None):
         memoryReference = stackFrames[0]["instructionPointerReference"]
         self.assertIsNotNone(memoryReference)
 
-        if memoryReference not in self.vscode.disassembled_instructions:
-            self.vscode.request_disassemble(memoryReference=memoryReference)
+        if memoryReference not in self.dap_server.disassembled_instructions:
+            self.dap_server.request_disassemble(memoryReference=memoryReference)
 
-        return self.vscode.disassembled_instructions[memoryReference]
+        return self.dap_server.disassembled_instructions[memoryReference]
 
     def attach(
         self,
@@ -289,22 +289,22 @@ def attach(
         sourceMap=None,
         sourceInitFile=False,
     ):
-        """Build the default Makefile target, create the VSCode debug adaptor,
+        """Build the default Makefile target, create the DAP debug adaptor,
         and attach to the process.
         """
 
-        # Make sure we disconnect and terminate the VSCode debug adaptor even
+        # Make sure we disconnect and terminate the DAP debug adaptor even
         # if we throw an exception during the test case.
         def cleanup():
             if disconnectAutomatically:
-                self.vscode.request_disconnect(terminateDebuggee=True)
-            self.vscode.terminate()
+                self.dap_server.request_disconnect(terminateDebuggee=True)
+            self.dap_server.terminate()
 
         # Execute the cleanup function during test case tear down.
         self.addTearDownHook(cleanup)
         # Initialize and launch the program
-        self.vscode.request_initialize(sourceInitFile)
-        response = self.vscode.request_attach(
+        self.dap_server.request_initialize(sourceInitFile)
+        response = self.dap_server.request_attach(
             program=program,
             pid=pid,
             waitFor=waitFor,
@@ -352,21 +352,21 @@ def launch(
         enableAutoVariableSummaries=False,
         enableSyntheticChildDebugging=False,
     ):
-        """Sending launch request to vscode"""
+        """Sending launch request to dap"""
 
-        # Make sure we disconnect and terminate the VSCode debug adapter,
+        # Make sure we disconnect and terminate the DAP debug adapter,
         # if we throw an exception during the test case
         def cleanup():
             if disconnectAutomatically:
-                self.vscode.request_disconnect(terminateDebuggee=True)
-            self.vscode.terminate()
+                self.dap_server.request_disconnect(terminateDebuggee=True)
+            self.dap_server.terminate()
 
         # Execute the cleanup function during test case tear down.
         self.addTearDownHook(cleanup)
 
         # Initialize and launch the program
-        self.vscode.request_initialize(sourceInitFile)
-        response = self.vscode.request_launch(
+        self.dap_server.request_initialize(sourceInitFile)
+        response = self.dap_server.request_launch(
             program,
             args=args,
             cwd=cwd,
@@ -422,14 +422,14 @@ def build_and_launch(
         runInTerminal=False,
         disconnectAutomatically=True,
         postRunCommands=None,
-        lldbVSCodeEnv=None,
+        lldbDAPEnv=None,
         enableAutoVariableSummaries=False,
         enableSyntheticChildDebugging=False,
     ):
-        """Build the default Makefile target, create the VSCode debug adaptor,
+        """Build the default Makefile target, create the DAP debug adaptor,
         and launch the process.
         """
-        self.build_and_create_debug_adaptor(lldbVSCodeEnv)
+        self.build_and_create_debug_adaptor(lldbDAPEnv)
         self.assertTrue(os.path.exists(program), "executable must exist")
 
         return self.launch(
diff --git a/lldb/source/API/SBType.cpp b/lldb/source/API/SBType.cpp
index ee5b644742..ac0e56303f 100644
--- a/lldb/source/API/SBType.cpp
+++ b/lldb/source/API/SBType.cpp
@@ -586,6 +586,14 @@ lldb::TemplateArgumentKind SBType::GetTemplateArgumentKind(uint32_t idx) {
   return eTemplateArgumentKindNull;
 }
 
+SBType SBType::FindDirectNestedType(const char *name) {
+  LLDB_INSTRUMENT_VA(this, name);
+
+  if (!IsValid())
+    return SBType();
+  return SBType(m_opaque_sp->FindDirectNestedType(name));
+}
+
 SBTypeList::SBTypeList() : m_opaque_up(new TypeListImpl()) {
   LLDB_INSTRUMENT_VA(this);
 }
diff --git a/lldb/source/Commands/CommandObjectBreakpoint.cpp b/lldb/source/Commands/CommandObjectBreakpoint.cpp
index 327dae4fd2..18cbb9528b 100644
--- a/lldb/source/Commands/CommandObjectBreakpoint.cpp
+++ b/lldb/source/Commands/CommandObjectBreakpoint.cpp
@@ -603,8 +603,8 @@ class CommandObjectBreakpointSet : public CommandObjectParsed {
       //  will track the load location of the library.
       size_t num_modules_specified = m_options.m_modules.GetSize();
       if (num_modules_specified == 1) {
-        const FileSpec *file_spec =
-            m_options.m_modules.GetFileSpecPointerAtIndex(0);
+        const FileSpec &file_spec =
+            m_options.m_modules.GetFileSpecAtIndex(0);
         bp_sp = target.CreateAddressInModuleBreakpoint(
             m_options.m_load_addr, internal, file_spec, m_options.m_hardware);
       } else if (num_modules_specified == 0) {
diff --git a/lldb/source/Commands/CommandObjectDWIMPrint.cpp b/lldb/source/Commands/CommandObjectDWIMPrint.cpp
index 7b168eab9e..bdc17c9cff 100644
--- a/lldb/source/Commands/CommandObjectDWIMPrint.cpp
+++ b/lldb/source/Commands/CommandObjectDWIMPrint.cpp
@@ -172,8 +172,19 @@ bool CommandObjectDWIMPrint::DoExecute(StringRef command,
   {
     auto *exe_scope = m_exe_ctx.GetBestExecutionContextScope();
     ValueObjectSP valobj_sp;
-    ExpressionResults expr_result =
-        target.EvaluateExpression(expr, exe_scope, valobj_sp, eval_options);
+    std::string fixed_expression;
+
+    ExpressionResults expr_result = target.EvaluateExpression(
+        expr, exe_scope, valobj_sp, eval_options, &fixed_expression);
+
+    // Only mention Fix-Its if the expression evaluator applied them.
+    // Compiler errors refer to the final expression after applying Fix-It(s).
+    if (!fixed_expression.empty() && target.GetEnableNotifyAboutFixIts()) {
+      Stream &error_stream = result.GetErrorStream();
+      error_stream << "  Evaluated this expression after applying Fix-It(s):\n";
+      error_stream << "    " << fixed_expression << "\n";
+    }
+
     if (expr_result == eExpressionCompleted) {
       if (verbosity != eDWIMPrintVerbosityNone) {
         StringRef flags;
diff --git a/lldb/source/Commands/CommandObjectExpression.cpp b/lldb/source/Commands/CommandObjectExpression.cpp
index e7e6e3820b..2834be660a 100644
--- a/lldb/source/Commands/CommandObjectExpression.cpp
+++ b/lldb/source/Commands/CommandObjectExpression.cpp
@@ -439,11 +439,11 @@ bool CommandObjectExpression::EvaluateExpression(llvm::StringRef expr,
   ExpressionResults success = target.EvaluateExpression(
       expr, frame, result_valobj_sp, eval_options, &m_fixed_expression);
 
-  // We only tell you about the FixIt if we applied it.  The compiler errors
-  // will suggest the FixIt if it parsed.
+  // Only mention Fix-Its if the expression evaluator applied them.
+  // Compiler errors refer to the final expression after applying Fix-It(s).
   if (!m_fixed_expression.empty() && target.GetEnableNotifyAboutFixIts()) {
-    error_stream.Printf("  Fix-it applied, fixed expression was: \n    %s\n",
-                        m_fixed_expression.c_str());
+    error_stream << "  Evaluated this expression after applying Fix-It(s):\n";
+    error_stream << "    " << m_fixed_expression << "\n";
   }
 
   if (result_valobj_sp) {
diff --git a/lldb/source/Commands/CommandObjectSource.h b/lldb/source/Commands/CommandObjectSource.h
index f2117bd6ca..d552508f44 100644
--- a/lldb/source/Commands/CommandObjectSource.h
+++ b/lldb/source/Commands/CommandObjectSource.h
@@ -1,5 +1,4 @@
-//===-- CommandObjectSource.h.h -----------------------------------*- C++
-//-*-===//
+//===-- CommandObjectSource.h -----------------------------------*- C++-*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -14,8 +13,6 @@
 
 namespace lldb_private {
 
-// CommandObjectMultiwordSource
-
 class CommandObjectMultiwordSource : public CommandObjectMultiword {
 public:
   CommandObjectMultiwordSource(CommandInterpreter &interpreter);
diff --git a/lldb/source/Commands/CommandObjectTarget.cpp b/lldb/source/Commands/CommandObjectTarget.cpp
index 33330ef092..7c20893db2 100644
--- a/lldb/source/Commands/CommandObjectTarget.cpp
+++ b/lldb/source/Commands/CommandObjectTarget.cpp
@@ -52,6 +52,7 @@
 #include "lldb/Utility/FileSpec.h"
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/State.h"
+#include "lldb/Utility/StructuredData.h"
 #include "lldb/Utility/Timer.h"
 #include "lldb/lldb-enumerations.h"
 #include "lldb/lldb-private-enumerations.h"
@@ -61,6 +62,7 @@
 #include "clang/Frontend/CompilerInvocation.h"
 #include "clang/Frontend/FrontendActions.h"
 #include "llvm/ADT/ScopeExit.h"
+#include "llvm/ADT/StringRef.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/FormatAdapters.h"
 
@@ -1462,6 +1464,87 @@ static bool DumpModuleSymbolFile(Stream &strm, Module *module) {
   return false;
 }
 
+static bool GetSeparateDebugInfoList(StructuredData::Array &list,
+                                     Module *module) {
+  if (module) {
+    if (SymbolFile *symbol_file = module->GetSymbolFile(/*can_create=*/true)) {
+      StructuredData::Dictionary d;
+      if (symbol_file->GetSeparateDebugInfo(d)) {
+        list.AddItem(
+            std::make_shared<StructuredData::Dictionary>(std::move(d)));
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
+static void DumpDwoFilesTable(Stream &strm,
+                              StructuredData::Array &dwo_listings) {
+  strm.PutCString("Dwo ID             Err Dwo Path");
+  strm.EOL();
+  strm.PutCString(
+      "------------------ --- -----------------------------------------");
+  strm.EOL();
+  dwo_listings.ForEach([&strm](StructuredData::Object *dwo) {
+    StructuredData::Dictionary *dict = dwo->GetAsDictionary();
+    if (!dict)
+      return false;
+
+    uint64_t dwo_id;
+    if (dict->GetValueForKeyAsInteger("dwo_id", dwo_id))
+      strm.Printf("0x%16.16" PRIx64 " ", dwo_id);
+    else
+      strm.Printf("0x???????????????? ");
+
+    llvm::StringRef error;
+    if (dict->GetValueForKeyAsString("error", error))
+      strm << "E   " << error;
+    else {
+      llvm::StringRef resolved_dwo_path;
+      if (dict->GetValueForKeyAsString("resolved_dwo_path",
+                                       resolved_dwo_path)) {
+        strm << "    " << resolved_dwo_path;
+        if (resolved_dwo_path.ends_with(".dwp")) {
+          llvm::StringRef dwo_name;
+          if (dict->GetValueForKeyAsString("dwo_name", dwo_name))
+            strm << "(" << dwo_name << ")";
+        }
+      }
+    }
+    strm.EOL();
+    return true;
+  });
+}
+
+static void DumpOsoFilesTable(Stream &strm,
+                              StructuredData::Array &oso_listings) {
+  strm.PutCString("Mod Time           Err Oso Path");
+  strm.EOL();
+  strm.PutCString("------------------ --- ---------------------");
+  strm.EOL();
+  oso_listings.ForEach([&strm](StructuredData::Object *oso) {
+    StructuredData::Dictionary *dict = oso->GetAsDictionary();
+    if (!dict)
+      return false;
+
+    uint32_t oso_mod_time;
+    if (dict->GetValueForKeyAsInteger("oso_mod_time", oso_mod_time))
+      strm.Printf("0x%16.16" PRIx32 " ", oso_mod_time);
+
+    llvm::StringRef error;
+    if (dict->GetValueForKeyAsString("error", error))
+      strm << "E   " << error;
+    else {
+      llvm::StringRef oso_path;
+      if (dict->GetValueForKeyAsString("oso_path", oso_path))
+        strm << "    " << oso_path;
+    }
+    strm.EOL();
+    return true;
+  });
+}
+
 static void DumpAddress(ExecutionContextScope *exe_scope,
                         const Address &so_addr, bool verbose, bool all_ranges,
                         Stream &strm) {
@@ -2462,6 +2545,178 @@ class CommandObjectTargetModulesDumpLineTable
   CommandOptions m_options;
 };
 
+#pragma mark CommandObjectTargetModulesDumpSeparateDebugInfoFiles
+#define LLDB_OPTIONS_target_modules_dump_separate_debug_info
+#include "CommandOptions.inc"
+
+// Image debug separate debug info dumping command
+
+class CommandObjectTargetModulesDumpSeparateDebugInfoFiles
+    : public CommandObjectTargetModulesModuleAutoComplete {
+public:
+  CommandObjectTargetModulesDumpSeparateDebugInfoFiles(
+      CommandInterpreter &interpreter)
+      : CommandObjectTargetModulesModuleAutoComplete(
+            interpreter, "target modules dump separate-debug-info",
+            "List the separate debug info symbol files for one or more target "
+            "modules.",
+            nullptr, eCommandRequiresTarget) {}
+
+  ~CommandObjectTargetModulesDumpSeparateDebugInfoFiles() override = default;
+
+  Options *GetOptions() override { return &m_options; }
+
+  class CommandOptions : public Options {
+  public:
+    CommandOptions() = default;
+
+    ~CommandOptions() override = default;
+
+    Status SetOptionValue(uint32_t option_idx, llvm::StringRef option_arg,
+                          ExecutionContext *execution_context) override {
+      Status error;
+      const int short_option = m_getopt_table[option_idx].val;
+
+      switch (short_option) {
+      case 'j':
+        m_json.SetCurrentValue(true);
+        m_json.SetOptionWasSet();
+        break;
+
+      default:
+        llvm_unreachable("Unimplemented option");
+      }
+      return error;
+    }
+
+    void OptionParsingStarting(ExecutionContext *execution_context) override {
+      m_json.Clear();
+    }
+
+    llvm::ArrayRef<OptionDefinition> GetDefinitions() override {
+      return llvm::ArrayRef(g_target_modules_dump_separate_debug_info_options);
+    }
+
+    OptionValueBoolean m_json = false;
+  };
+
+protected:
+  bool DoExecute(Args &command, CommandReturnObject &result) override {
+    Target &target = GetSelectedTarget();
+    uint32_t num_dumped = 0;
+
+    uint32_t addr_byte_size = target.GetArchitecture().GetAddressByteSize();
+    result.GetOutputStream().SetAddressByteSize(addr_byte_size);
+    result.GetErrorStream().SetAddressByteSize(addr_byte_size);
+
+    StructuredData::Array separate_debug_info_lists_by_module;
+    if (command.GetArgumentCount() == 0) {
+      // Dump all sections for all modules images
+      const ModuleList &target_modules = target.GetImages();
+      std::lock_guard<std::recursive_mutex> guard(target_modules.GetMutex());
+      const size_t num_modules = target_modules.GetSize();
+      if (num_modules == 0) {
+        result.AppendError("the target has no associated executable images");
+        return false;
+      }
+      for (ModuleSP module_sp : target_modules.ModulesNoLocking()) {
+        if (INTERRUPT_REQUESTED(
+                GetDebugger(),
+                "Interrupted in dumping all "
+                "separate debug info with {0} of {1} modules dumped",
+                num_dumped, num_modules))
+          break;
+
+        if (GetSeparateDebugInfoList(separate_debug_info_lists_by_module,
+                                     module_sp.get()))
+          num_dumped++;
+      }
+    } else {
+      // Dump specified images (by basename or fullpath)
+      const char *arg_cstr;
+      for (int arg_idx = 0;
+           (arg_cstr = command.GetArgumentAtIndex(arg_idx)) != nullptr;
+           ++arg_idx) {
+        ModuleList module_list;
+        const size_t num_matches =
+            FindModulesByName(&target, arg_cstr, module_list, true);
+        if (num_matches > 0) {
+          for (size_t i = 0; i < num_matches; ++i) {
+            if (INTERRUPT_REQUESTED(GetDebugger(),
+                                    "Interrupted dumping {0} "
+                                    "of {1} requested modules",
+                                    i, num_matches))
+              break;
+            Module *module = module_list.GetModulePointerAtIndex(i);
+            if (GetSeparateDebugInfoList(separate_debug_info_lists_by_module,
+                                         module))
+              num_dumped++;
+          }
+        } else
+          result.AppendWarningWithFormat(
+              "Unable to find an image that matches '%s'.\n", arg_cstr);
+      }
+    }
+
+    if (num_dumped > 0) {
+      Stream &strm = result.GetOutputStream();
+      if (m_options.m_json) {
+        separate_debug_info_lists_by_module.Dump(strm,
+                                                 /*pretty_print=*/true);
+      } else {
+        // List the debug info files in human readable form.
+        separate_debug_info_lists_by_module.ForEach(
+            [&result, &strm](StructuredData::Object *obj) {
+              if (!obj) {
+                return false;
+              }
+
+              // Each item in `separate_debug_info_lists_by_module` should be a
+              // valid structured data dictionary.
+              StructuredData::Dictionary *separate_debug_info_list =
+                  obj->GetAsDictionary();
+              if (!separate_debug_info_list) {
+                return false;
+              }
+
+              llvm::StringRef type;
+              llvm::StringRef symfile;
+              StructuredData::Array *files;
+              if (!(separate_debug_info_list->GetValueForKeyAsString("type",
+                                                                     type) &&
+                    separate_debug_info_list->GetValueForKeyAsString("symfile",
+                                                                     symfile) &&
+                    separate_debug_info_list->GetValueForKeyAsArray(
+                        "separate-debug-info-files", files))) {
+                assert(false);
+              }
+
+              strm << "Symbol file: " << symfile;
+              strm.EOL();
+              strm << "Type: \"" << type << "\"";
+              strm.EOL();
+              if (type == "dwo") {
+                DumpDwoFilesTable(strm, *files);
+              } else if (type == "oso") {
+                DumpOsoFilesTable(strm, *files);
+              } else {
+                result.AppendWarningWithFormat(
+                    "Found unsupported debug info type '%s'.\n",
+                    type.str().c_str());
+              }
+              return true;
+            });
+      }
+      result.SetStatus(eReturnStatusSuccessFinishResult);
+    } else {
+      result.AppendError("no matching executable images found");
+    }
+    return result.Succeeded();
+  }
+
+  CommandOptions m_options;
+};
+
 #pragma mark CommandObjectTargetModulesDump
 
 // Dump multi-word command for target modules
@@ -2475,7 +2730,8 @@ class CommandObjectTargetModulesDump : public CommandObjectMultiword {
             "Commands for dumping information about one or more target "
             "modules.",
             "target modules dump "
-            "[objfile|symtab|sections|ast|symfile|line-table|pcm-info] "
+            "[objfile|symtab|sections|ast|symfile|line-table|pcm-info|separate-"
+            "debug-info] "
             "[<file1> <file2> ...]") {
     LoadSubCommand("objfile",
                    CommandObjectSP(
@@ -2499,6 +2755,10 @@ class CommandObjectTargetModulesDump : public CommandObjectMultiword {
         "pcm-info",
         CommandObjectSP(
             new CommandObjectTargetModulesDumpClangPCMInfo(interpreter)));
+    LoadSubCommand("separate-debug-info",
+                   CommandObjectSP(
+                       new CommandObjectTargetModulesDumpSeparateDebugInfoFiles(
+                           interpreter)));
   }
 
   ~CommandObjectTargetModulesDump() override = default;
diff --git a/lldb/source/Commands/Options.td b/lldb/source/Commands/Options.td
index 7af20e93a1..078b23e09e 100644
--- a/lldb/source/Commands/Options.td
+++ b/lldb/source/Commands/Options.td
@@ -8,6 +8,11 @@ let Command = "target modules dump symtab" in {
     Desc<"Do not demangle symbol names before showing them.">;
 }
 
+let Command = "target modules dump separate debug info" in {
+  def tm_json : Option<"json", "j">, Group<1>,
+  Desc<"Output the details in JSON format.">;
+}
+
 let Command = "help" in {
   def help_hide_aliases : Option<"hide-aliases", "a">,
     Desc<"Hide aliases in the command list.">;
diff --git a/lldb/source/Core/CMakeLists.txt b/lldb/source/Core/CMakeLists.txt
index d7b4f2587a..9073e3e9b2 100644
--- a/lldb/source/Core/CMakeLists.txt
+++ b/lldb/source/Core/CMakeLists.txt
@@ -12,7 +12,7 @@ set(LLDB_LIBEDIT_LIBS)
 if (LLDB_ENABLE_CURSES)
   list(APPEND LLDB_CURSES_LIBS ${PANEL_LIBRARIES} ${CURSES_LIBRARIES})
   if(LLVM_ENABLE_TERMINFO)
-    list(APPEND LLDB_CURSES_LIBS ${TERMINFO_LIB})
+    list(APPEND LLDB_CURSES_LIBS ${Terminfo_LIBRARIES})
   endif()
   if (LLVM_BUILD_STATIC)
     list(APPEND LLDB_CURSES_LIBS gpm)
diff --git a/lldb/source/Core/Value.cpp b/lldb/source/Core/Value.cpp
index 8efcfd3b4a..995cc934c8 100644
--- a/lldb/source/Core/Value.cpp
+++ b/lldb/source/Core/Value.cpp
@@ -98,7 +98,9 @@ void Value::AppendBytes(const void *bytes, int len) {
 }
 
 void Value::Dump(Stream *strm) {
-  m_value.GetValue(strm, true);
+  if (!strm)
+    return;
+  m_value.GetValue(*strm, true);
   strm->Printf(", value_type = %s, context = %p, context_type = %s",
                Value::GetValueTypeAsCString(m_value_type), m_context,
                Value::GetContextTypeAsCString(m_context_type));
diff --git a/lldb/source/DataFormatters/TypeFormat.cpp b/lldb/source/DataFormatters/TypeFormat.cpp
index 5ee89fc0d5..126240aeca 100644
--- a/lldb/source/DataFormatters/TypeFormat.cpp
+++ b/lldb/source/DataFormatters/TypeFormat.cpp
@@ -81,9 +81,9 @@ bool TypeFormatImpl_Format::FormatObject(ValueObject *valobj,
               WritableDataBufferSP buffer_sp(
                   new DataBufferHeap(max_len + 1, 0));
               Address address(valobj->GetPointerValue());
-              if (target_sp->ReadCStringFromMemory(
-                      address, (char *)buffer_sp->GetBytes(), max_len, error) &&
-                  error.Success())
+              target_sp->ReadCStringFromMemory(
+                  address, (char *)buffer_sp->GetBytes(), max_len, error);
+              if (error.Success())
                 data.SetData(buffer_sp);
             }
           }
diff --git a/lldb/source/DataFormatters/VectorType.cpp b/lldb/source/DataFormatters/VectorType.cpp
index 4afcfa2e8e..57dae0b2c7 100644
--- a/lldb/source/DataFormatters/VectorType.cpp
+++ b/lldb/source/DataFormatters/VectorType.cpp
@@ -169,21 +169,49 @@ static lldb::Format GetItemFormatForFormat(lldb::Format format,
   }
 }
 
-static size_t CalculateNumChildren(
-    CompilerType container_type, CompilerType element_type,
-    lldb_private::ExecutionContextScope *exe_scope =
-        nullptr // does not matter here because all we trade in are basic types
-    ) {
-  std::optional<uint64_t> container_size =
-      container_type.GetByteSize(exe_scope);
-  std::optional<uint64_t> element_size = element_type.GetByteSize(exe_scope);
-
-  if (container_size && element_size && *element_size) {
-    if (*container_size % *element_size)
-      return 0;
-    return *container_size / *element_size;
-  }
-  return 0;
+/// Calculates the number of elements stored in a container (with
+/// element type 'container_elem_type') as if it had elements of type
+/// 'element_type'.
+///
+/// For example, a container of type
+/// `uint8_t __attribute__((vector_size(16)))` has 16 elements.
+/// But calling `CalculateNumChildren` with an 'element_type'
+/// of `float` (4-bytes) will return `4` because we are interpreting
+/// the byte-array as a `float32[]`.
+///
+/// \param[in] container_elem_type The type of the elements stored
+/// in the container we are calculating the children of.
+///
+/// \param[in] num_elements Number of 'container_elem_type's our
+/// container stores.
+///
+/// \param[in] element_type The type of elements we interpret
+/// container_type to contain for the purposes of calculating
+/// the number of children.
+///
+/// \returns The number of elements stored in a container of
+/// type 'element_type'. Returns a std::nullopt if the
+/// size of the container is not a multiple of 'element_type'
+/// or if an error occurs.
+static std::optional<size_t>
+CalculateNumChildren(CompilerType container_elem_type, uint64_t num_elements,
+                     CompilerType element_type) {
+  std::optional<uint64_t> container_elem_size =
+      container_elem_type.GetByteSize(/* exe_scope */ nullptr);
+  if (!container_elem_size)
+    return {};
+
+  auto container_size = *container_elem_size * num_elements;
+
+  std::optional<uint64_t> element_size =
+      element_type.GetByteSize(/* exe_scope */ nullptr);
+  if (!element_size || !*element_size)
+    return {};
+
+  if (container_size % *element_size)
+    return {};
+
+  return container_size / *element_size;
 }
 
 namespace lldb_private {
@@ -221,11 +249,14 @@ class VectorTypeSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
     m_parent_format = m_backend.GetFormat();
     CompilerType parent_type(m_backend.GetCompilerType());
     CompilerType element_type;
-    parent_type.IsVectorType(&element_type);
+    uint64_t num_elements;
+    parent_type.IsVectorType(&element_type, &num_elements);
     m_child_type = ::GetCompilerTypeForFormat(
         m_parent_format, element_type,
         parent_type.GetTypeSystem().GetSharedPointer());
-    m_num_children = ::CalculateNumChildren(parent_type, m_child_type);
+    m_num_children =
+        ::CalculateNumChildren(element_type, num_elements, m_child_type)
+            .value_or(0);
     m_item_format = GetItemFormatForFormat(m_parent_format, m_child_type);
     return false;
   }
diff --git a/lldb/source/Expression/DWARFExpression.cpp b/lldb/source/Expression/DWARFExpression.cpp
index 93fcf0579b..fe4928d4f4 100644
--- a/lldb/source/Expression/DWARFExpression.cpp
+++ b/lldb/source/Expression/DWARFExpression.cpp
@@ -45,6 +45,7 @@
 using namespace lldb;
 using namespace lldb_private;
 using namespace lldb_private::dwarf;
+using namespace lldb_private::plugin::dwarf;
 
 // DWARFExpression constructor
 DWARFExpression::DWARFExpression() : m_data() {}
diff --git a/lldb/source/Interpreter/CommandInterpreter.cpp b/lldb/source/Interpreter/CommandInterpreter.cpp
index dcff53ff84..e1275ce711 100644
--- a/lldb/source/Interpreter/CommandInterpreter.cpp
+++ b/lldb/source/Interpreter/CommandInterpreter.cpp
@@ -1773,7 +1773,7 @@ CommandInterpreter::PreprocessToken(std::string &expr_str) {
 
       StreamString value_strm;
       const bool show_type = false;
-      scalar.GetValue(&value_strm, show_type);
+      scalar.GetValue(value_strm, show_type);
       size_t value_string_size = value_strm.GetSize();
       if (value_string_size) {
         expr_str = value_strm.GetData();
diff --git a/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/DynamicLoaderPOSIXDYLD.cpp b/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/DynamicLoaderPOSIXDYLD.cpp
index 85d7ae9dac..c427b47608 100644
--- a/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/DynamicLoaderPOSIXDYLD.cpp
+++ b/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/DynamicLoaderPOSIXDYLD.cpp
@@ -337,29 +337,20 @@ bool DynamicLoaderPOSIXDYLD::SetRendezvousBreakpoint() {
     };
 
     ModuleSP interpreter = LoadInterpreterModule();
-    if (!interpreter) {
-      FileSpecList containingModules;
+    FileSpecList containingModules;
+    if (interpreter)
+      containingModules.Append(interpreter->GetFileSpec());
+    else
       containingModules.Append(
           m_process->GetTarget().GetExecutableModulePointer()->GetFileSpec());
 
-      dyld_break = target.CreateBreakpoint(
-          &containingModules, /*containingSourceFiles=*/nullptr,
-          DebugStateCandidates, eFunctionNameTypeFull, eLanguageTypeC,
-          /*m_offset=*/0,
-          /*skip_prologue=*/eLazyBoolNo,
-          /*internal=*/true,
-          /*request_hardware=*/false);
-    } else {
-      FileSpecList containingModules;
-      containingModules.Append(interpreter->GetFileSpec());
-      dyld_break = target.CreateBreakpoint(
-          &containingModules, /*containingSourceFiles=*/nullptr,
-          DebugStateCandidates, eFunctionNameTypeFull, eLanguageTypeC,
-          /*m_offset=*/0,
-          /*skip_prologue=*/eLazyBoolNo,
-          /*internal=*/true,
-          /*request_hardware=*/false);
-    }
+    dyld_break = target.CreateBreakpoint(
+        &containingModules, /*containingSourceFiles=*/nullptr,
+        DebugStateCandidates, eFunctionNameTypeFull, eLanguageTypeC,
+        /*m_offset=*/0,
+        /*skip_prologue=*/eLazyBoolNo,
+        /*internal=*/true,
+        /*request_hardware=*/false);
   }
 
   if (dyld_break->GetNumResolvedLocations() != 1) {
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp
index 14776cdf80..2e8da396a4 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp
@@ -162,10 +162,27 @@ lldb::ValueObjectSP lldb_private::formatters::
       if (!node_sp || error.Fail())
           return nullptr;
 
-      value_sp = node_sp->GetChildMemberWithName("__value_");
       hash_sp = node_sp->GetChildMemberWithName("__hash_");
-      if (!value_sp || !hash_sp)
+      if (!hash_sp)
         return nullptr;
+
+      value_sp = node_sp->GetChildMemberWithName("__value_");
+      if (!value_sp) {
+        // clang-format off
+        // Since D101206 (ba79fb2e1f), libc++ wraps the `__value_` in an
+        // anonymous union.
+        // Child 0: __hash_node_base base class
+        // Child 1: __hash_
+        // Child 2: anonymous union
+        // clang-format on
+        auto anon_union_sp = node_sp->GetChildAtIndex(2);
+        if (!anon_union_sp)
+          return nullptr;
+
+        value_sp = anon_union_sp->GetChildMemberWithName("__value_");
+        if (!value_sp)
+          return nullptr;
+      }
     }
     m_elements_cache.push_back(
         {value_sp.get(), hash_sp->GetValueAsUnsigned(0)});
diff --git a/lldb/source/Plugins/ObjectFile/PDB/ObjectFilePDB.cpp b/lldb/source/Plugins/ObjectFile/PDB/ObjectFilePDB.cpp
index a3b91fc37d..f0832dbf07 100644
--- a/lldb/source/Plugins/ObjectFile/PDB/ObjectFilePDB.cpp
+++ b/lldb/source/Plugins/ObjectFile/PDB/ObjectFilePDB.cpp
@@ -179,7 +179,7 @@ ObjectFilePDB::loadPDBFile(std::string PdbPath,
 
   llvm::StringRef Path = Buffer->getBufferIdentifier();
   auto Stream = std::make_unique<llvm::MemoryBufferByteStream>(
-      std::move(Buffer), llvm::support::little);
+      std::move(Buffer), llvm::endianness::little);
 
   auto File = std::make_unique<PDBFile>(Path, std::move(Stream), Allocator);
   if (auto EC = File->parseFileHeaders()) {
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformAppleSimulator.cpp b/lldb/source/Plugins/Platform/MacOSX/PlatformAppleSimulator.cpp
index 1348eca05e..dc59d904c7 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformAppleSimulator.cpp
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformAppleSimulator.cpp
@@ -335,8 +335,7 @@ PlatformSP PlatformAppleSimulator::CreateInstance(
 
   bool create = force;
   if (!create && arch && arch->IsValid()) {
-    if (std::count(supported_arch.begin(), supported_arch.end(),
-                   arch->GetMachine())) {
+    if (llvm::is_contained(supported_arch, arch->GetMachine())) {
       const llvm::Triple &triple = arch->GetTriple();
       switch (triple.getVendor()) {
       case llvm::Triple::Apple:
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/AppleDWARFIndex.cpp b/lldb/source/Plugins/SymbolFile/DWARF/AppleDWARFIndex.cpp
index 34fb98b5a9..325517ca1d 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/AppleDWARFIndex.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/AppleDWARFIndex.cpp
@@ -18,6 +18,7 @@
 using namespace lldb_private;
 using namespace lldb;
 using namespace lldb_private::dwarf;
+using namespace lldb_private::plugin::dwarf;
 
 std::unique_ptr<AppleDWARFIndex> AppleDWARFIndex::Create(
     Module &module, DWARFDataExtractor apple_names,
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/AppleDWARFIndex.h b/lldb/source/Plugins/SymbolFile/DWARF/AppleDWARFIndex.h
index 6b948e0798..a1fb99700d 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/AppleDWARFIndex.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/AppleDWARFIndex.h
@@ -12,7 +12,8 @@
 #include "Plugins/SymbolFile/DWARF/DWARFIndex.h"
 #include "llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h"
 
-namespace lldb_private {
+namespace lldb_private::plugin {
+namespace dwarf {
 class AppleDWARFIndex : public DWARFIndex {
 public:
   static std::unique_ptr<AppleDWARFIndex>
@@ -77,6 +78,7 @@ class AppleDWARFIndex : public DWARFIndex {
                  std::optional<dw_tag_t> search_for_tag = std::nullopt,
                  std::optional<uint32_t> search_for_qualhash = std::nullopt);
 };
-} // namespace lldb_private
+} // namespace dwarf
+} // namespace lldb_private::plugin
 
 #endif // LLDB_SOURCE_PLUGINS_SYMBOLFILE_DWARF_APPLEDWARFINDEX_H
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DIERef.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DIERef.cpp
index 88a5e60275..163e9f4c08 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DIERef.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DIERef.cpp
@@ -14,6 +14,7 @@
 
 using namespace lldb;
 using namespace lldb_private;
+using namespace lldb_private::plugin::dwarf;
 
 void llvm::format_provider<DIERef>::format(const DIERef &ref, raw_ostream &OS,
                                            StringRef Style) {
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DIERef.h b/lldb/source/Plugins/SymbolFile/DWARF/DIERef.h
index b5a5cfe263..ad443aacb4 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DIERef.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DIERef.h
@@ -14,6 +14,8 @@
 #include <cassert>
 #include <optional>
 
+namespace lldb_private::plugin {
+namespace dwarf {
 /// Identifies a DWARF debug info entry within a given Module. It contains three
 /// "coordinates":
 /// - file_index: identifies the separate stand alone debug info file
@@ -93,7 +95,7 @@ class DIERef {
   /// \return
   ///   Returns a valid DIERef if decoding succeeded, std::nullopt if there was
   ///   unsufficient or invalid values that were decoded.
-  static std::optional<DIERef> Decode(const lldb_private::DataExtractor &data,
+  static std::optional<DIERef> Decode(const DataExtractor &data,
                                       lldb::offset_t *offset_ptr);
 
   /// Encode this object into a data encoder object.
@@ -103,7 +105,7 @@ class DIERef {
   /// \param encoder
   ///   A data encoder object that serialized bytes will be encoded into.
   ///
-  void Encode(lldb_private::DataEncoder &encoder) const;
+  void Encode(DataEncoder &encoder) const;
 
   static constexpr uint64_t k_die_offset_bit_size = DW_DIE_OFFSET_MAX_BITSIZE;
   static constexpr uint64_t k_file_index_bit_size =
@@ -131,10 +133,13 @@ class DIERef {
 static_assert(sizeof(DIERef) == 8);
 
 typedef std::vector<DIERef> DIEArray;
+} // namespace dwarf
+} // namespace lldb_private::plugin
 
 namespace llvm {
-template<> struct format_provider<DIERef> {
-  static void format(const DIERef &ref, raw_ostream &OS, StringRef Style);
+template <> struct format_provider<lldb_private::plugin::dwarf::DIERef> {
+  static void format(const lldb_private::plugin::dwarf::DIERef &ref,
+                     raw_ostream &OS, StringRef Style);
 };
 } // namespace llvm
 
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParser.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParser.cpp
index a68b7cd110..1fe0cadecc 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParser.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParser.cpp
@@ -18,6 +18,7 @@
 using namespace lldb;
 using namespace lldb_private;
 using namespace lldb_private::dwarf;
+using namespace lldb_private::plugin::dwarf;
 
 std::optional<SymbolFile::ArrayInfo>
 DWARFASTParser::ParseChildArrayInfo(const DWARFDIE &parent_die,
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParser.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParser.h
index 18825ae060..eaafbe169c 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParser.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParser.h
@@ -17,53 +17,53 @@
 #include "lldb/lldb-enumerations.h"
 #include <optional>
 
-class DWARFDIE;
 namespace lldb_private {
 class CompileUnit;
 class ExecutionContext;
 }
+
+namespace lldb_private::plugin {
+namespace dwarf {
+class DWARFDIE;
 class SymbolFileDWARF;
 
 class DWARFASTParser {
 public:
   virtual ~DWARFASTParser() = default;
 
-  virtual lldb::TypeSP ParseTypeFromDWARF(const lldb_private::SymbolContext &sc,
+  virtual lldb::TypeSP ParseTypeFromDWARF(const SymbolContext &sc,
                                           const DWARFDIE &die,
                                           bool *type_is_new_ptr) = 0;
 
-  virtual lldb_private::ConstString
-  ConstructDemangledNameFromDWARF(const DWARFDIE &die) = 0;
+  virtual ConstString ConstructDemangledNameFromDWARF(const DWARFDIE &die) = 0;
 
-  virtual lldb_private::Function *
-  ParseFunctionFromDWARF(lldb_private::CompileUnit &comp_unit,
-                         const DWARFDIE &die,
-                         const lldb_private::AddressRange &range) = 0;
+  virtual Function *ParseFunctionFromDWARF(CompileUnit &comp_unit,
+                                           const DWARFDIE &die,
+                                           const AddressRange &range) = 0;
 
-  virtual bool
-  CompleteTypeFromDWARF(const DWARFDIE &die, lldb_private::Type *type,
-                        lldb_private::CompilerType &compiler_type) = 0;
+  virtual bool CompleteTypeFromDWARF(const DWARFDIE &die, Type *type,
+                                     CompilerType &compiler_type) = 0;
 
-  virtual lldb_private::CompilerDecl
-  GetDeclForUIDFromDWARF(const DWARFDIE &die) = 0;
+  virtual CompilerDecl GetDeclForUIDFromDWARF(const DWARFDIE &die) = 0;
 
-  virtual lldb_private::CompilerDeclContext
+  virtual CompilerDeclContext
   GetDeclContextForUIDFromDWARF(const DWARFDIE &die) = 0;
 
-  virtual lldb_private::CompilerDeclContext
+  virtual CompilerDeclContext
   GetDeclContextContainingUIDFromDWARF(const DWARFDIE &die) = 0;
 
   virtual void EnsureAllDIEsInDeclContextHaveBeenParsed(
-      lldb_private::CompilerDeclContext decl_context) = 0;
+      CompilerDeclContext decl_context) = 0;
 
-  virtual lldb_private::ConstString
-  GetDIEClassTemplateParams(const DWARFDIE &die) = 0;
+  virtual ConstString GetDIEClassTemplateParams(const DWARFDIE &die) = 0;
 
-  static std::optional<lldb_private::SymbolFile::ArrayInfo>
+  static std::optional<SymbolFile::ArrayInfo>
   ParseChildArrayInfo(const DWARFDIE &parent_die,
-                      const lldb_private::ExecutionContext *exe_ctx = nullptr);
+                      const ExecutionContext *exe_ctx = nullptr);
 
   static lldb::AccessType GetAccessTypeFromDWARF(uint32_t dwarf_accessibility);
 };
+} // namespace dwarf
+} // namespace lldb_private::plugin
 
 #endif // LLDB_SOURCE_PLUGINS_SYMBOLFILE_DWARF_DWARFASTPARSER_H
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
index d0065896b0..545a5dcc7d 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
@@ -60,6 +60,8 @@
 using namespace lldb;
 using namespace lldb_private;
 using namespace lldb_private::dwarf;
+using namespace lldb_private::plugin::dwarf;
+
 DWARFASTParserClang::DWARFASTParserClang(TypeSystemClang &ast)
     : m_ast(ast), m_die_to_decl_ctx(), m_decl_ctx_to_die() {}
 
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h
index 88bfc490e8..3d6912cf56 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h
@@ -31,45 +31,51 @@
 namespace lldb_private {
 class CompileUnit;
 }
+namespace lldb_private::plugin {
+namespace dwarf {
 class DWARFDebugInfoEntry;
 class SymbolFileDWARF;
+} // namespace dwarf
+} // namespace lldb_private::plugin
 
 struct ParsedDWARFTypeAttributes;
 
-class DWARFASTParserClang : public DWARFASTParser {
+class DWARFASTParserClang : public lldb_private::plugin::dwarf::DWARFASTParser {
 public:
   DWARFASTParserClang(lldb_private::TypeSystemClang &ast);
 
   ~DWARFASTParserClang() override;
 
   // DWARFASTParser interface.
-  lldb::TypeSP ParseTypeFromDWARF(const lldb_private::SymbolContext &sc,
-                                  const DWARFDIE &die,
-                                  bool *type_is_new_ptr) override;
+  lldb::TypeSP
+  ParseTypeFromDWARF(const lldb_private::SymbolContext &sc,
+                     const lldb_private::plugin::dwarf::DWARFDIE &die,
+                     bool *type_is_new_ptr) override;
 
-  lldb_private::ConstString
-  ConstructDemangledNameFromDWARF(const DWARFDIE &die) override;
+  lldb_private::ConstString ConstructDemangledNameFromDWARF(
+      const lldb_private::plugin::dwarf::DWARFDIE &die) override;
 
   lldb_private::Function *
   ParseFunctionFromDWARF(lldb_private::CompileUnit &comp_unit,
-                         const DWARFDIE &die,
+                         const lldb_private::plugin::dwarf::DWARFDIE &die,
                          const lldb_private::AddressRange &func_range) override;
 
   bool
-  CompleteTypeFromDWARF(const DWARFDIE &die, lldb_private::Type *type,
+  CompleteTypeFromDWARF(const lldb_private::plugin::dwarf::DWARFDIE &die,
+                        lldb_private::Type *type,
                         lldb_private::CompilerType &compiler_type) override;
 
-  lldb_private::CompilerDecl
-  GetDeclForUIDFromDWARF(const DWARFDIE &die) override;
+  lldb_private::CompilerDecl GetDeclForUIDFromDWARF(
+      const lldb_private::plugin::dwarf::DWARFDIE &die) override;
 
   void EnsureAllDIEsInDeclContextHaveBeenParsed(
       lldb_private::CompilerDeclContext decl_context) override;
 
-  lldb_private::CompilerDeclContext
-  GetDeclContextForUIDFromDWARF(const DWARFDIE &die) override;
+  lldb_private::CompilerDeclContext GetDeclContextForUIDFromDWARF(
+      const lldb_private::plugin::dwarf::DWARFDIE &die) override;
 
-  lldb_private::CompilerDeclContext
-  GetDeclContextContainingUIDFromDWARF(const DWARFDIE &die) override;
+  lldb_private::CompilerDeclContext GetDeclContextContainingUIDFromDWARF(
+      const lldb_private::plugin::dwarf::DWARFDIE &die) override;
 
   lldb_private::ClangASTImporter &GetClangASTImporter();
 
@@ -85,9 +91,9 @@ class DWARFASTParserClang : public DWARFASTParser {
   ///         DWARFFormValue with the bit width of the given integer type.
   ///         Returns an error if the value in the DWARFFormValue does not fit
   ///         into the given integer type or the integer type isn't supported.
-  llvm::Expected<llvm::APInt>
-  ExtractIntFromFormValue(const lldb_private::CompilerType &int_type,
-                          const DWARFFormValue &form_value) const;
+  llvm::Expected<llvm::APInt> ExtractIntFromFormValue(
+      const lldb_private::CompilerType &int_type,
+      const lldb_private::plugin::dwarf::DWARFFormValue &form_value) const;
 
   /// Returns the template parameters of a class DWARFDIE as a string.
   ///
@@ -99,8 +105,8 @@ class DWARFASTParserClang : public DWARFASTParser {
   /// \return A string, including surrounding '<>', of the template parameters.
   /// If the DIE's name already has '<>', returns an empty ConstString because
   /// it's assumed that the caller is using the DIE name anyway.
-  lldb_private::ConstString
-  GetDIEClassTemplateParams(const DWARFDIE &die) override;
+  lldb_private::ConstString GetDIEClassTemplateParams(
+      const lldb_private::plugin::dwarf::DWARFDIE &die) override;
 
 protected:
   /// Protected typedefs and members.
@@ -108,14 +114,19 @@ class DWARFASTParserClang : public DWARFASTParser {
   class DelayedAddObjCClassProperty;
   typedef std::vector<DelayedAddObjCClassProperty> DelayedPropertyList;
 
-  typedef llvm::DenseMap<const DWARFDebugInfoEntry *, clang::DeclContext *>
+  typedef llvm::DenseMap<
+      const lldb_private::plugin::dwarf::DWARFDebugInfoEntry *,
+      clang::DeclContext *>
       DIEToDeclContextMap;
-  typedef std::multimap<const clang::DeclContext *, const DWARFDIE>
+  typedef std::multimap<const clang::DeclContext *,
+                        const lldb_private::plugin::dwarf::DWARFDIE>
       DeclContextToDIEMap;
-  typedef llvm::DenseMap<const DWARFDebugInfoEntry *,
-                         lldb_private::OptionalClangModuleID>
+  typedef llvm::DenseMap<
+      const lldb_private::plugin::dwarf::DWARFDebugInfoEntry *,
+      lldb_private::OptionalClangModuleID>
       DIEToModuleMap;
-  typedef llvm::DenseMap<const DWARFDebugInfoEntry *, clang::Decl *>
+  typedef llvm::DenseMap<
+      const lldb_private::plugin::dwarf::DWARFDebugInfoEntry *, clang::Decl *>
       DIEToDeclMap;
 
   lldb_private::TypeSystemClang &m_ast;
@@ -126,11 +137,14 @@ class DWARFASTParserClang : public DWARFASTParser {
   std::unique_ptr<lldb_private::ClangASTImporter> m_clang_ast_importer_up;
   /// @}
 
-  clang::DeclContext *GetDeclContextForBlock(const DWARFDIE &die);
+  clang::DeclContext *
+  GetDeclContextForBlock(const lldb_private::plugin::dwarf::DWARFDIE &die);
 
-  clang::BlockDecl *ResolveBlockDIE(const DWARFDIE &die);
+  clang::BlockDecl *
+  ResolveBlockDIE(const lldb_private::plugin::dwarf::DWARFDIE &die);
 
-  clang::NamespaceDecl *ResolveNamespaceDIE(const DWARFDIE &die);
+  clang::NamespaceDecl *
+  ResolveNamespaceDIE(const lldb_private::plugin::dwarf::DWARFDIE &die);
 
   /// Returns the namespace decl that a DW_TAG_imported_declaration imports.
   ///
@@ -141,82 +155,98 @@ class DWARFASTParserClang : public DWARFASTParser {
   ///          'die' imports. If the imported entity is not a namespace
   ///          or another import declaration, returns nullptr. If an error
   ///          occurs, returns nullptr.
-  clang::NamespaceDecl *ResolveImportedDeclarationDIE(const DWARFDIE &die);
+  clang::NamespaceDecl *ResolveImportedDeclarationDIE(
+      const lldb_private::plugin::dwarf::DWARFDIE &die);
 
-  bool ParseTemplateDIE(const DWARFDIE &die,
+  bool ParseTemplateDIE(const lldb_private::plugin::dwarf::DWARFDIE &die,
                         lldb_private::TypeSystemClang::TemplateParameterInfos
                             &template_param_infos);
 
   bool ParseTemplateParameterInfos(
-      const DWARFDIE &parent_die,
+      const lldb_private::plugin::dwarf::DWARFDIE &parent_die,
       lldb_private::TypeSystemClang::TemplateParameterInfos
           &template_param_infos);
 
-  std::string GetCPlusPlusQualifiedName(const DWARFDIE &die);
+  std::string
+  GetCPlusPlusQualifiedName(const lldb_private::plugin::dwarf::DWARFDIE &die);
 
   bool ParseChildMembers(
-      const DWARFDIE &die, lldb_private::CompilerType &class_compiler_type,
+      const lldb_private::plugin::dwarf::DWARFDIE &die,
+      lldb_private::CompilerType &class_compiler_type,
       std::vector<std::unique_ptr<clang::CXXBaseSpecifier>> &base_classes,
-      std::vector<DWARFDIE> &member_function_dies,
+      std::vector<lldb_private::plugin::dwarf::DWARFDIE> &member_function_dies,
       DelayedPropertyList &delayed_properties,
       const lldb::AccessType default_accessibility,
       lldb_private::ClangASTImporter::LayoutInfo &layout_info);
 
   size_t
   ParseChildParameters(clang::DeclContext *containing_decl_ctx,
-                       const DWARFDIE &parent_die, bool skip_artificial,
-                       bool &is_static, bool &is_variadic,
+                       const lldb_private::plugin::dwarf::DWARFDIE &parent_die,
+                       bool skip_artificial, bool &is_static, bool &is_variadic,
                        bool &has_template_params,
                        std::vector<lldb_private::CompilerType> &function_args,
                        std::vector<clang::ParmVarDecl *> &function_param_decls,
                        unsigned &type_quals);
 
-  size_t ParseChildEnumerators(lldb_private::CompilerType &compiler_type,
-                               bool is_signed, uint32_t enumerator_byte_size,
-                               const DWARFDIE &parent_die);
+  size_t ParseChildEnumerators(
+      lldb_private::CompilerType &compiler_type, bool is_signed,
+      uint32_t enumerator_byte_size,
+      const lldb_private::plugin::dwarf::DWARFDIE &parent_die);
 
   /// Parse a structure, class, or union type DIE.
-  lldb::TypeSP ParseStructureLikeDIE(const lldb_private::SymbolContext &sc,
-                                     const DWARFDIE &die,
-                                     ParsedDWARFTypeAttributes &attrs);
+  lldb::TypeSP
+  ParseStructureLikeDIE(const lldb_private::SymbolContext &sc,
+                        const lldb_private::plugin::dwarf::DWARFDIE &die,
+                        ParsedDWARFTypeAttributes &attrs);
 
-  lldb_private::Type *GetTypeForDIE(const DWARFDIE &die);
+  lldb_private::Type *
+  GetTypeForDIE(const lldb_private::plugin::dwarf::DWARFDIE &die);
 
-  clang::Decl *GetClangDeclForDIE(const DWARFDIE &die);
+  clang::Decl *
+  GetClangDeclForDIE(const lldb_private::plugin::dwarf::DWARFDIE &die);
 
-  clang::DeclContext *GetClangDeclContextForDIE(const DWARFDIE &die);
+  clang::DeclContext *
+  GetClangDeclContextForDIE(const lldb_private::plugin::dwarf::DWARFDIE &die);
 
-  clang::DeclContext *GetClangDeclContextContainingDIE(const DWARFDIE &die,
-                                                       DWARFDIE *decl_ctx_die);
-  lldb_private::OptionalClangModuleID GetOwningClangModule(const DWARFDIE &die);
+  clang::DeclContext *GetClangDeclContextContainingDIE(
+      const lldb_private::plugin::dwarf::DWARFDIE &die,
+      lldb_private::plugin::dwarf::DWARFDIE *decl_ctx_die);
+  lldb_private::OptionalClangModuleID
+  GetOwningClangModule(const lldb_private::plugin::dwarf::DWARFDIE &die);
 
-  bool CopyUniqueClassMethodTypes(const DWARFDIE &src_class_die,
-                                  const DWARFDIE &dst_class_die,
-                                  lldb_private::Type *class_type,
-                                  std::vector<DWARFDIE> &failures);
+  bool CopyUniqueClassMethodTypes(
+      const lldb_private::plugin::dwarf::DWARFDIE &src_class_die,
+      const lldb_private::plugin::dwarf::DWARFDIE &dst_class_die,
+      lldb_private::Type *class_type,
+      std::vector<lldb_private::plugin::dwarf::DWARFDIE> &failures);
 
-  clang::DeclContext *GetCachedClangDeclContextForDIE(const DWARFDIE &die);
+  clang::DeclContext *GetCachedClangDeclContextForDIE(
+      const lldb_private::plugin::dwarf::DWARFDIE &die);
 
-  void LinkDeclContextToDIE(clang::DeclContext *decl_ctx, const DWARFDIE &die);
+  void LinkDeclContextToDIE(clang::DeclContext *decl_ctx,
+                            const lldb_private::plugin::dwarf::DWARFDIE &die);
 
-  void LinkDeclToDIE(clang::Decl *decl, const DWARFDIE &die);
+  void LinkDeclToDIE(clang::Decl *decl,
+                     const lldb_private::plugin::dwarf::DWARFDIE &die);
 
   /// If \p type_sp is valid, calculate and set its symbol context scope, and
   /// update the type list for its backing symbol file.
   ///
   /// Returns \p type_sp.
-  lldb::TypeSP
-  UpdateSymbolContextScopeForType(const lldb_private::SymbolContext &sc,
-                                  const DWARFDIE &die, lldb::TypeSP type_sp);
+  lldb::TypeSP UpdateSymbolContextScopeForType(
+      const lldb_private::SymbolContext &sc,
+      const lldb_private::plugin::dwarf::DWARFDIE &die, lldb::TypeSP type_sp);
 
   /// Follow Clang Module Skeleton CU references to find a type definition.
-  lldb::TypeSP ParseTypeFromClangModule(const lldb_private::SymbolContext &sc,
-                                        const DWARFDIE &die,
-                                        lldb_private::Log *log);
+  lldb::TypeSP
+  ParseTypeFromClangModule(const lldb_private::SymbolContext &sc,
+                           const lldb_private::plugin::dwarf::DWARFDIE &die,
+                           lldb_private::Log *log);
 
   // Return true if this type is a declaration to a type in an external
   // module.
-  lldb::ModuleSP GetModuleForType(const DWARFDIE &die);
+  lldb::ModuleSP
+  GetModuleForType(const lldb_private::plugin::dwarf::DWARFDIE &die);
 
 private:
   struct FieldInfo {
@@ -268,33 +298,41 @@ class DWARFASTParserClang : public DWARFASTParser {
   /// created property.
   /// \param delayed_properties The list of delayed properties that the result
   /// will be appended to.
-  void ParseObjCProperty(const DWARFDIE &die, const DWARFDIE &parent_die,
-                         const lldb_private::CompilerType &class_clang_type,
-                         DelayedPropertyList &delayed_properties);
+  void
+  ParseObjCProperty(const lldb_private::plugin::dwarf::DWARFDIE &die,
+                    const lldb_private::plugin::dwarf::DWARFDIE &parent_die,
+                    const lldb_private::CompilerType &class_clang_type,
+                    DelayedPropertyList &delayed_properties);
 
   void
-  ParseSingleMember(const DWARFDIE &die, const DWARFDIE &parent_die,
+  ParseSingleMember(const lldb_private::plugin::dwarf::DWARFDIE &die,
+                    const lldb_private::plugin::dwarf::DWARFDIE &parent_die,
                     const lldb_private::CompilerType &class_clang_type,
                     lldb::AccessType default_accessibility,
                     lldb_private::ClangASTImporter::LayoutInfo &layout_info,
                     FieldInfo &last_field_info);
 
-  bool CompleteRecordType(const DWARFDIE &die, lldb_private::Type *type,
+  bool CompleteRecordType(const lldb_private::plugin::dwarf::DWARFDIE &die,
+                          lldb_private::Type *type,
                           lldb_private::CompilerType &clang_type);
-  bool CompleteEnumType(const DWARFDIE &die, lldb_private::Type *type,
+  bool CompleteEnumType(const lldb_private::plugin::dwarf::DWARFDIE &die,
+                        lldb_private::Type *type,
                         lldb_private::CompilerType &clang_type);
 
-  lldb::TypeSP ParseTypeModifier(const lldb_private::SymbolContext &sc,
-                                 const DWARFDIE &die,
-                                 ParsedDWARFTypeAttributes &attrs);
+  lldb::TypeSP
+  ParseTypeModifier(const lldb_private::SymbolContext &sc,
+                    const lldb_private::plugin::dwarf::DWARFDIE &die,
+                    ParsedDWARFTypeAttributes &attrs);
   lldb::TypeSP ParseEnum(const lldb_private::SymbolContext &sc,
-                         const DWARFDIE &die, ParsedDWARFTypeAttributes &attrs);
-  lldb::TypeSP ParseSubroutine(const DWARFDIE &die,
+                         const lldb_private::plugin::dwarf::DWARFDIE &die,
+                         ParsedDWARFTypeAttributes &attrs);
+  lldb::TypeSP ParseSubroutine(const lldb_private::plugin::dwarf::DWARFDIE &die,
                                ParsedDWARFTypeAttributes &attrs);
-  lldb::TypeSP ParseArrayType(const DWARFDIE &die,
+  lldb::TypeSP ParseArrayType(const lldb_private::plugin::dwarf::DWARFDIE &die,
                               const ParsedDWARFTypeAttributes &attrs);
-  lldb::TypeSP ParsePointerToMemberType(const DWARFDIE &die,
-                                        const ParsedDWARFTypeAttributes &attrs);
+  lldb::TypeSP
+  ParsePointerToMemberType(const lldb_private::plugin::dwarf::DWARFDIE &die,
+                           const ParsedDWARFTypeAttributes &attrs);
 
   /// Parses a DW_TAG_inheritance DIE into a base/super class.
   ///
@@ -311,7 +349,8 @@ class DWARFASTParserClang : public DWARFASTParser {
   /// \param layout_info The layout information that will be updated for C++
   /// base classes with the base offset.
   void ParseInheritance(
-      const DWARFDIE &die, const DWARFDIE &parent_die,
+      const lldb_private::plugin::dwarf::DWARFDIE &die,
+      const lldb_private::plugin::dwarf::DWARFDIE &parent_die,
       const lldb_private::CompilerType class_clang_type,
       const lldb::AccessType default_accessibility,
       const lldb::ModuleSP &module_sp,
@@ -328,7 +367,8 @@ class DWARFASTParserClang : public DWARFASTParser {
   /// \param layout_info The layout information that will be updated for
   //   base classes with the base offset
   void
-  ParseRustVariantPart(DWARFDIE &die, const DWARFDIE &parent_die,
+  ParseRustVariantPart(lldb_private::plugin::dwarf::DWARFDIE &die,
+                       const lldb_private::plugin::dwarf::DWARFDIE &parent_die,
                        lldb_private::CompilerType &class_clang_type,
                        const lldb::AccessType default_accesibility,
                        lldb_private::ClangASTImporter::LayoutInfo &layout_info);
@@ -338,7 +378,8 @@ class DWARFASTParserClang : public DWARFASTParser {
 /// Some attributes are relevant for all kinds of types (declaration), while
 /// others are only meaningful to a specific type (is_virtual)
 struct ParsedDWARFTypeAttributes {
-  explicit ParsedDWARFTypeAttributes(const DWARFDIE &die);
+  explicit ParsedDWARFTypeAttributes(
+      const lldb_private::plugin::dwarf::DWARFDIE &die);
 
   lldb::AccessType accessibility = lldb::eAccessNone;
   bool is_artificial = false;
@@ -355,12 +396,12 @@ struct ParsedDWARFTypeAttributes {
   const char *mangled_name = nullptr;
   lldb_private::ConstString name;
   lldb_private::Declaration decl;
-  DWARFDIE object_pointer;
-  DWARFFormValue abstract_origin;
-  DWARFFormValue containing_type;
-  DWARFFormValue signature;
-  DWARFFormValue specification;
-  DWARFFormValue type;
+  lldb_private::plugin::dwarf::DWARFDIE object_pointer;
+  lldb_private::plugin::dwarf::DWARFFormValue abstract_origin;
+  lldb_private::plugin::dwarf::DWARFFormValue containing_type;
+  lldb_private::plugin::dwarf::DWARFFormValue signature;
+  lldb_private::plugin::dwarf::DWARFFormValue specification;
+  lldb_private::plugin::dwarf::DWARFFormValue type;
   lldb::LanguageType class_language = lldb::eLanguageTypeUnknown;
   std::optional<uint64_t> byte_size;
   size_t calling_convention = llvm::dwarf::DW_CC_normal;
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFAttribute.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFAttribute.cpp
index 00b56537ae..3d35775e08 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFAttribute.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFAttribute.cpp
@@ -11,6 +11,7 @@
 #include "DWARFDebugInfo.h"
 
 using namespace lldb_private::dwarf;
+using namespace lldb_private::plugin::dwarf;
 
 DWARFAttributes::DWARFAttributes() : m_infos() {}
 
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFAttribute.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFAttribute.h
index 90e12fa024..e05ccc980d 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFAttribute.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFAttribute.h
@@ -14,6 +14,8 @@
 #include "llvm/ADT/SmallVector.h"
 #include <vector>
 
+namespace lldb_private::plugin {
+namespace dwarf {
 class DWARFUnit;
 
 class DWARFAttribute {
@@ -31,6 +33,7 @@ class DWARFAttribute {
     form = m_form;
     val = m_value;
   }
+
 protected:
   dw_attr_t m_attr;
   dw_form_t m_form;
@@ -72,5 +75,7 @@ class DWARFAttributes {
   typedef llvm::SmallVector<AttributeValue, 8> collection;
   collection m_infos;
 };
+} // namespace dwarf
+} // namespace lldb_private::plugin
 
 #endif // LLDB_SOURCE_PLUGINS_SYMBOLFILE_DWARF_DWARFATTRIBUTE_H
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFBaseDIE.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFBaseDIE.cpp
index 37a917c3a7..3a3b05acd2 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFBaseDIE.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFBaseDIE.cpp
@@ -18,6 +18,7 @@
 #include <optional>
 
 using namespace lldb_private;
+using namespace lldb_private::plugin::dwarf;
 
 std::optional<DIERef> DWARFBaseDIE::GetDIERef() const {
   if (!IsValid())
@@ -35,7 +36,7 @@ dw_tag_t DWARFBaseDIE::Tag() const {
 }
 
 const char *DWARFBaseDIE::GetTagAsCString() const {
-  return lldb_private::DW_TAG_value_to_name(Tag());
+  return DW_TAG_value_to_name(Tag());
 }
 
 const char *DWARFBaseDIE::GetAttributeValueAsString(const dw_attr_t attr,
@@ -120,6 +121,8 @@ DWARFAttributes DWARFBaseDIE::GetAttributes(Recurse recurse) const {
   return DWARFAttributes();
 }
 
+namespace lldb_private::plugin {
+namespace dwarf {
 bool operator==(const DWARFBaseDIE &lhs, const DWARFBaseDIE &rhs) {
   return lhs.GetDIE() == rhs.GetDIE() && lhs.GetCU() == rhs.GetCU();
 }
@@ -127,6 +130,8 @@ bool operator==(const DWARFBaseDIE &lhs, const DWARFBaseDIE &rhs) {
 bool operator!=(const DWARFBaseDIE &lhs, const DWARFBaseDIE &rhs) {
   return !(lhs == rhs);
 }
+} // namespace dwarf
+} // namespace lldb_private::plugin
 
 const DWARFDataExtractor &DWARFBaseDIE::GetData() const {
   // Clients must check if this DIE is valid before calling this function.
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFBaseDIE.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFBaseDIE.h
index 8bcf807ad1..75c822703c 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFBaseDIE.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFBaseDIE.h
@@ -15,6 +15,8 @@
 #include "llvm/Support/Error.h"
 #include <optional>
 
+namespace lldb_private::plugin {
+namespace dwarf {
 class DIERef;
 class DWARFASTParser;
 class DWARFAttributes;
@@ -78,7 +80,7 @@ class DWARFBaseDIE {
   // correct section data.
   //
   // Clients must validate that this object is valid before calling this.
-  const lldb_private::DWARFDataExtractor &GetData() const;
+  const DWARFDataExtractor &GetData() const;
 
   // Accessing information about a DIE
   dw_tag_t Tag() const;
@@ -124,5 +126,7 @@ class DWARFBaseDIE {
 
 bool operator==(const DWARFBaseDIE &lhs, const DWARFBaseDIE &rhs);
 bool operator!=(const DWARFBaseDIE &lhs, const DWARFBaseDIE &rhs);
+} // namespace dwarf
+} // namespace lldb_private::plugin
 
 #endif // LLDB_SOURCE_PLUGINS_SYMBOLFILE_DWARF_DWARFBASEDIE_H
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFCompileUnit.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFCompileUnit.cpp
index f839a59bf6..ec4c297cf7 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFCompileUnit.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFCompileUnit.cpp
@@ -16,6 +16,7 @@
 
 using namespace lldb;
 using namespace lldb_private;
+using namespace lldb_private::plugin::dwarf;
 
 void DWARFCompileUnit::Dump(Stream *s) const {
   s->Format(
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFCompileUnit.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFCompileUnit.h
index 65debac4c7..dd130977d4 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFCompileUnit.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFCompileUnit.h
@@ -14,13 +14,15 @@
 
 namespace llvm {
 class DWARFAbbreviationDeclarationSet;
-}
+} // namespace llvm
 
+namespace lldb_private::plugin {
+namespace dwarf {
 class DWARFCompileUnit : public DWARFUnit {
 public:
   void BuildAddressRangeTable(DWARFDebugAranges *debug_aranges) override;
 
-  void Dump(lldb_private::Stream *s) const override;
+  void Dump(Stream *s) const override;
 
   static bool classof(const DWARFUnit *unit) { return !unit->IsTypeUnit(); }
 
@@ -40,5 +42,7 @@ class DWARFCompileUnit : public DWARFUnit {
 
   friend class DWARFUnit;
 };
+} // namespace dwarf
+} // namespace lldb_private::plugin
 
 #endif // LLDB_SOURCE_PLUGINS_SYMBOLFILE_DWARF_DWARFCOMPILEUNIT_H
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFContext.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFContext.cpp
index f72dad88e1..ee347036db 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFContext.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFContext.cpp
@@ -13,6 +13,7 @@
 
 using namespace lldb;
 using namespace lldb_private;
+using namespace lldb_private::plugin::dwarf;
 
 static DWARFDataExtractor LoadSection(SectionList *section_list,
                                       SectionType section_type) {
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFContext.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFContext.h
index 7df776b5f5..87c6eb2093 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFContext.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFContext.h
@@ -16,7 +16,8 @@
 #include <memory>
 #include <optional>
 
-namespace lldb_private {
+namespace lldb_private::plugin {
+namespace dwarf {
 class DWARFContext {
 private:
   SectionList *m_main_section_list;
@@ -78,6 +79,7 @@ class DWARFContext {
 
   llvm::DWARFContext &GetAsLLVM();
 };
-} // namespace lldb_private
+} // namespace dwarf
+} // namespace lldb_private::plugin
 
 #endif
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.cpp
index b31c5dcac9..d43c2ac276 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.cpp
@@ -18,6 +18,7 @@
 
 using namespace lldb_private;
 using namespace lldb_private::dwarf;
+using namespace lldb_private::plugin::dwarf;
 
 namespace {
 
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.h
index 031ea26ad4..25b313bf09 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.h
@@ -13,6 +13,8 @@
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/iterator_range.h"
 
+namespace lldb_private::plugin {
+namespace dwarf {
 class DWARFDIE : public DWARFBaseDIE {
 public:
   class child_iterator;
@@ -31,14 +33,14 @@ class DWARFDIE : public DWARFBaseDIE {
   const char *GetPubname() const;
 
   using DWARFBaseDIE::GetName;
-  void GetName(lldb_private::Stream &s) const;
+  void GetName(Stream &s) const;
 
-  void AppendTypeName(lldb_private::Stream &s) const;
+  void AppendTypeName(Stream &s) const;
 
-  lldb_private::Type *ResolveType() const;
+  Type *ResolveType() const;
 
   // Resolve a type by UID using this DIE's DWARF file
-  lldb_private::Type *ResolveTypeUID(const DWARFDIE &die) const;
+  Type *ResolveTypeUID(const DWARFDIE &die) const;
 
   // Functions for obtaining DIE relations and references
 
@@ -72,8 +74,7 @@ class DWARFDIE : public DWARFBaseDIE {
 
   /// Return this DIE's decl context as it is needed to look up types
   /// in Clang's -gmodules debug info format.
-  void GetDeclContext(
-      llvm::SmallVectorImpl<lldb_private::CompilerContext> &context) const;
+  void GetDeclContext(llvm::SmallVectorImpl<CompilerContext> &context) const;
 
   // Getting attribute values from the DIE.
   //
@@ -88,7 +89,7 @@ class DWARFDIE : public DWARFBaseDIE {
       std::optional<int> &decl_file, std::optional<int> &decl_line,
       std::optional<int> &decl_column, std::optional<int> &call_file,
       std::optional<int> &call_line, std::optional<int> &call_column,
-      lldb_private::DWARFExpressionList *frame_base) const;
+      DWARFExpressionList *frame_base) const;
 
   /// The range of all the children of this DIE.
   llvm::iterator_range<child_iterator> children() const;
@@ -126,5 +127,7 @@ class DWARFDIE::child_iterator
     return *this;
   }
 };
+} // namespace dwarf
+} // namespace lldb_private::plugin
 
 #endif // LLDB_SOURCE_PLUGINS_SYMBOLFILE_DWARF_DWARFDIE_H
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDataExtractor.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDataExtractor.h
index b9526b079c..41b8e9ad02 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDataExtractor.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDataExtractor.h
@@ -33,6 +33,6 @@ class DWARFDataExtractor : public DataExtractor {
   llvm::DWARFDataExtractor GetAsLLVMDWARF() const;
   llvm::DataExtractor GetAsLLVM() const;
 };
-}
+} // namespace lldb_private
 
 #endif // LLDB_SOURCE_PLUGINS_SYMBOLFILE_DWARF_DWARFDATAEXTRACTOR_H
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugArangeSet.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugArangeSet.cpp
index 03cbfd28ae..8461b94abc 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugArangeSet.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugArangeSet.cpp
@@ -13,6 +13,7 @@
 #include <cassert>
 
 using namespace lldb_private;
+using namespace lldb_private::plugin::dwarf;
 
 DWARFDebugArangeSet::DWARFDebugArangeSet()
     : m_offset(DW_INVALID_OFFSET), m_next_offset(DW_INVALID_OFFSET) {}
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugArangeSet.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugArangeSet.h
index 3c8633eaa3..ecdbe953f5 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugArangeSet.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugArangeSet.h
@@ -13,6 +13,8 @@
 #include <cstdint>
 #include <vector>
 
+namespace lldb_private::plugin {
+namespace dwarf {
 class DWARFDebugArangeSet {
 public:
   struct Header {
@@ -42,7 +44,7 @@ class DWARFDebugArangeSet {
   DWARFDebugArangeSet();
   void Clear();
   void SetOffset(uint32_t offset) { m_offset = offset; }
-  llvm::Error extract(const lldb_private::DWARFDataExtractor &data,
+  llvm::Error extract(const DWARFDataExtractor &data,
                       lldb::offset_t *offset_ptr);
   dw_offset_t FindAddress(dw_addr_t address) const;
   size_t NumDescriptors() const { return m_arange_descriptors.size(); }
@@ -62,5 +64,7 @@ class DWARFDebugArangeSet {
   Header m_header;
   DescriptorColl m_arange_descriptors;
 };
+} // namespace dwarf
+} // namespace lldb_private::plugin
 
 #endif // LLDB_SOURCE_PLUGINS_SYMBOLFILE_DWARF_DWARFDEBUGARANGESET_H
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugAranges.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugAranges.cpp
index b38dd2b88c..da73891f66 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugAranges.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugAranges.cpp
@@ -15,6 +15,7 @@
 
 using namespace lldb;
 using namespace lldb_private;
+using namespace lldb_private::plugin::dwarf;
 
 // Constructor
 DWARFDebugAranges::DWARFDebugAranges() : m_aranges() {}
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugAranges.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugAranges.h
index 5ff37e400c..99e2108b85 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugAranges.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugAranges.h
@@ -13,10 +13,11 @@
 #include "lldb/Utility/RangeMap.h"
 #include "llvm/Support/Error.h"
 
+namespace lldb_private::plugin {
+namespace dwarf {
 class DWARFDebugAranges {
 protected:
-  typedef lldb_private::RangeDataVector<dw_addr_t, uint32_t, dw_offset_t>
-      RangeToDIE;
+  typedef RangeDataVector<dw_addr_t, uint32_t, dw_offset_t> RangeToDIE;
 
 public:
   typedef RangeToDIE::Entry Range;
@@ -26,14 +27,14 @@ class DWARFDebugAranges {
 
   void Clear() { m_aranges.Clear(); }
 
-  void extract(const lldb_private::DWARFDataExtractor &debug_aranges_data);
+  void extract(const DWARFDataExtractor &debug_aranges_data);
 
   // Use append range multiple times and then call sort
   void AppendRange(dw_offset_t cu_offset, dw_addr_t low_pc, dw_addr_t high_pc);
 
   void Sort(bool minimize);
 
-  void Dump(lldb_private::Log *log) const;
+  void Dump(Log *log) const;
 
   dw_offset_t FindAddress(dw_addr_t address) const;
 
@@ -50,5 +51,7 @@ class DWARFDebugAranges {
 protected:
   RangeToDIE m_aranges;
 };
+} // namespace dwarf
+} // namespace lldb_private::plugin
 
 #endif // LLDB_SOURCE_PLUGINS_SYMBOLFILE_DWARF_DWARFDEBUGARANGES_H
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfo.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfo.cpp
index 9a33d6338b..553b6a4c55 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfo.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfo.cpp
@@ -27,10 +27,10 @@
 
 using namespace lldb;
 using namespace lldb_private;
+using namespace lldb_private::plugin::dwarf;
 
 // Constructor
-DWARFDebugInfo::DWARFDebugInfo(SymbolFileDWARF &dwarf,
-                               lldb_private::DWARFContext &context)
+DWARFDebugInfo::DWARFDebugInfo(SymbolFileDWARF &dwarf, DWARFContext &context)
     : m_dwarf(dwarf), m_context(context), m_units(), m_cu_aranges_up() {}
 
 const DWARFDebugAranges &DWARFDebugInfo::GetCompileUnitAranges() {
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfo.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfo.h
index c990ac9fbe..d5e48f312e 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfo.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfo.h
@@ -19,20 +19,18 @@
 #include "lldb/lldb-private.h"
 #include "llvm/Support/Error.h"
 
-namespace lldb_private {
+namespace lldb_private::plugin {
+namespace dwarf {
 class DWARFContext;
-}
 
 class DWARFDebugInfo {
 public:
-  typedef dw_offset_t (*Callback)(SymbolFileDWARF *dwarf2Data,
-                                  DWARFUnit *cu,
+  typedef dw_offset_t (*Callback)(SymbolFileDWARF *dwarf2Data, DWARFUnit *cu,
                                   DWARFDebugInfoEntry *die,
                                   const dw_offset_t next_offset,
                                   const uint32_t depth, void *userData);
 
-  explicit DWARFDebugInfo(SymbolFileDWARF &dwarf,
-                          lldb_private::DWARFContext &context);
+  explicit DWARFDebugInfo(SymbolFileDWARF &dwarf, DWARFContext &context);
 
   size_t GetNumUnits();
   DWARFUnit *GetUnitAtIndex(size_t idx);
@@ -58,7 +56,7 @@ class DWARFDebugInfo {
   typedef std::vector<DWARFUnitSP> UnitColl;
 
   SymbolFileDWARF &m_dwarf;
-  lldb_private::DWARFContext &m_context;
+  DWARFContext &m_context;
 
   llvm::once_flag m_units_once_flag;
   UnitColl m_units;
@@ -80,5 +78,7 @@ class DWARFDebugInfo {
   DWARFDebugInfo(const DWARFDebugInfo &) = delete;
   const DWARFDebugInfo &operator=(const DWARFDebugInfo &) = delete;
 };
+} // namespace dwarf
+} // namespace lldb_private::plugin
 
 #endif // LLDB_SOURCE_PLUGINS_SYMBOLFILE_DWARF_DWARFDEBUGINFO_H
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfoEntry.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfoEntry.cpp
index a6ab837009..1b0fefedf9 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfoEntry.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfoEntry.cpp
@@ -35,6 +35,7 @@
 
 using namespace lldb_private;
 using namespace lldb_private::dwarf;
+using namespace lldb_private::plugin::dwarf;
 extern int g_verbose;
 
 // Extract a debug info entry for a given DWARFUnit from the data
@@ -49,16 +50,12 @@ bool DWARFDebugInfoEntry::Extract(const DWARFDataExtractor &data,
   lldbassert(abbr_idx <= UINT16_MAX);
   m_abbr_idx = abbr_idx;
 
-  // assert (fixed_form_sizes);  // For best performance this should be
-  // specified!
-
   if (m_abbr_idx == 0) {
     m_tag = llvm::dwarf::DW_TAG_null;
     m_has_children = false;
     return true; // NULL debug tag entry
   }
 
-  lldb::offset_t offset = *offset_ptr;
   const auto *abbrevDecl = GetAbbreviationDeclarationPtr(cu);
   if (abbrevDecl == nullptr) {
     cu->GetSymbolFileDWARF().GetObjectFile()->GetModule()->ReportError(
@@ -73,137 +70,18 @@ bool DWARFDebugInfoEntry::Extract(const DWARFDataExtractor &data,
   m_tag = abbrevDecl->getTag();
   m_has_children = abbrevDecl->hasChildren();
   // Skip all data in the .debug_info or .debug_types for the attributes
-  dw_form_t form;
   for (const auto &attribute : abbrevDecl->attributes()) {
-    form = attribute.Form;
-    std::optional<uint8_t> fixed_skip_size =
-        DWARFFormValue::GetFixedSize(form, cu);
-    if (fixed_skip_size)
-      offset += *fixed_skip_size;
-    else {
-      bool form_is_indirect = false;
-      do {
-        form_is_indirect = false;
-        uint32_t form_size = 0;
-        switch (form) {
-        // Blocks if inlined data that have a length field and the data bytes
-        // inlined in the .debug_info/.debug_types
-        case DW_FORM_exprloc:
-        case DW_FORM_block:
-          form_size = data.GetULEB128(&offset);
-          break;
-        case DW_FORM_block1:
-          form_size = data.GetU8_unchecked(&offset);
-          break;
-        case DW_FORM_block2:
-          form_size = data.GetU16_unchecked(&offset);
-          break;
-        case DW_FORM_block4:
-          form_size = data.GetU32_unchecked(&offset);
-          break;
-
-        // Inlined NULL terminated C-strings
-        case DW_FORM_string:
-          data.GetCStr(&offset);
-          break;
-
-        // Compile unit address sized values
-        case DW_FORM_addr:
-          form_size = cu->GetAddressByteSize();
-          break;
-        case DW_FORM_ref_addr:
-          if (cu->GetVersion() <= 2)
-            form_size = cu->GetAddressByteSize();
-          else
-            form_size = 4;
-          break;
+    if (DWARFFormValue::SkipValue(attribute.Form, data, offset_ptr, cu))
+      continue;
 
-        // 0 sized form
-        case DW_FORM_flag_present:
-          form_size = 0;
-          break;
-
-        // 1 byte values
-        case DW_FORM_addrx1:
-        case DW_FORM_data1:
-        case DW_FORM_flag:
-        case DW_FORM_ref1:
-        case DW_FORM_strx1:
-          form_size = 1;
-          break;
-
-        // 2 byte values
-        case DW_FORM_addrx2:
-        case DW_FORM_data2:
-        case DW_FORM_ref2:
-        case DW_FORM_strx2:
-          form_size = 2;
-          break;
-
-        // 3 byte values
-        case DW_FORM_addrx3:
-        case DW_FORM_strx3:
-          form_size = 3;
-          break;
-
-        // 4 byte values
-        case DW_FORM_addrx4:
-        case DW_FORM_data4:
-        case DW_FORM_ref4:
-        case DW_FORM_strx4:
-          form_size = 4;
-          break;
-
-        // 8 byte values
-        case DW_FORM_data8:
-        case DW_FORM_ref8:
-        case DW_FORM_ref_sig8:
-          form_size = 8;
-          break;
-
-        // signed or unsigned LEB 128 values
-        case DW_FORM_addrx:
-        case DW_FORM_loclistx:
-        case DW_FORM_rnglistx:
-        case DW_FORM_sdata:
-        case DW_FORM_udata:
-        case DW_FORM_ref_udata:
-        case DW_FORM_GNU_addr_index:
-        case DW_FORM_GNU_str_index:
-        case DW_FORM_strx:
-          data.Skip_LEB128(&offset);
-          break;
-
-        case DW_FORM_indirect:
-          form_is_indirect = true;
-          form = static_cast<dw_form_t>(data.GetULEB128(&offset));
-          break;
-
-        case DW_FORM_strp:
-        case DW_FORM_line_strp:
-        case DW_FORM_sec_offset:
-          data.GetU32(&offset);
-          break;
-
-        case DW_FORM_implicit_const:
-          form_size = 0;
-          break;
-
-        default:
-          cu->GetSymbolFileDWARF().GetObjectFile()->GetModule()->ReportError(
-              "[{0:x16}]: Unsupported DW_FORM_{1:x}, please file a bug "
-              "and "
-              "attach the file at the start of this error message",
-              (uint64_t)m_offset, (unsigned)form);
-          *offset_ptr = m_offset;
-          return false;
-        }
-        offset += form_size;
-
-      } while (form_is_indirect);
-    }
+    cu->GetSymbolFileDWARF().GetObjectFile()->GetModule()->ReportError(
+        "[{0:x16}]: Unsupported DW_FORM_{1:x}, please file a bug "
+        "and "
+        "attach the file at the start of this error message",
+        (uint64_t)m_offset, (unsigned)attribute.Form);
+    *offset_ptr = m_offset;
+    return false;
   }
-  *offset_ptr = offset;
   return true;
 }
 
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfoEntry.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfoEntry.h
index 29db44a16b..c19fa74285 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfoEntry.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfoEntry.h
@@ -22,6 +22,8 @@
 
 #include "llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h"
 
+namespace lldb_private::plugin {
+namespace dwarf {
 class DWARFDeclContext;
 
 #define DIE_SIBLING_IDX_BITSIZE 31
@@ -47,8 +49,8 @@ class DWARFDebugInfoEntry {
   void BuildFunctionAddressRangeTable(DWARFUnit *cu,
                                       DWARFDebugAranges *debug_aranges) const;
 
-  bool Extract(const lldb_private::DWARFDataExtractor &data,
-               const DWARFUnit *cu, lldb::offset_t *offset_ptr);
+  bool Extract(const DWARFDataExtractor &data, const DWARFUnit *cu,
+               lldb::offset_t *offset_ptr);
 
   using Recurse = DWARFBaseDIE::Recurse;
   DWARFAttributes GetAttributes(DWARFUnit *cu,
@@ -104,13 +106,15 @@ class DWARFDebugInfoEntry {
 
   const char *GetPubname(const DWARFUnit *cu) const;
 
-  bool GetDIENamesAndRanges(
-      DWARFUnit *cu, const char *&name, const char *&mangled,
-      DWARFRangeList &rangeList, std::optional<int> &decl_file,
-      std::optional<int> &decl_line, std::optional<int> &decl_column,
-      std::optional<int> &call_file, std::optional<int> &call_line,
-      std::optional<int> &call_column,
-      lldb_private::DWARFExpressionList *frame_base = nullptr) const;
+  bool GetDIENamesAndRanges(DWARFUnit *cu, const char *&name,
+                            const char *&mangled, DWARFRangeList &rangeList,
+                            std::optional<int> &decl_file,
+                            std::optional<int> &decl_line,
+                            std::optional<int> &decl_column,
+                            std::optional<int> &call_file,
+                            std::optional<int> &call_line,
+                            std::optional<int> &call_column,
+                            DWARFExpressionList *frame_base = nullptr) const;
 
   const llvm::DWARFAbbreviationDeclaration *
   GetAbbreviationDeclarationPtr(const DWARFUnit *cu) const;
@@ -190,5 +194,7 @@ class DWARFDebugInfoEntry {
   void GetAttributes(DWARFUnit *cu, DWARFAttributes &attrs, Recurse recurse,
                      uint32_t curr_depth) const;
 };
+} // namespace dwarf
+} // namespace lldb_private::plugin
 
 #endif // LLDB_SOURCE_PLUGINS_SYMBOLFILE_DWARF_DWARFDEBUGINFOENTRY_H
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugMacro.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugMacro.cpp
index 19c6448c4e..2cd84bc55b 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugMacro.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugMacro.cpp
@@ -15,6 +15,7 @@
 
 using namespace lldb_private;
 using namespace lldb_private::dwarf;
+using namespace lldb_private::plugin::dwarf;
 
 DWARFDebugMacroHeader
 DWARFDebugMacroHeader::ParseHeader(const DWARFDataExtractor &debug_macro_data,
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugMacro.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugMacro.h
index cbf7624583..67d1cde8d5 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugMacro.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugMacro.h
@@ -17,11 +17,11 @@
 #include "lldb/lldb-types.h"
 
 namespace lldb_private {
-
 class DWARFDataExtractor;
+}
 
-} // namespace lldb_private
-
+namespace lldb_private::plugin {
+namespace dwarf {
 class SymbolFileDWARF;
 
 class DWARFDebugMacroHeader {
@@ -33,15 +33,14 @@ class DWARFDebugMacroHeader {
   };
 
   static DWARFDebugMacroHeader
-  ParseHeader(const lldb_private::DWARFDataExtractor &debug_macro_data,
+  ParseHeader(const DWARFDataExtractor &debug_macro_data,
               lldb::offset_t *offset);
 
   bool OffsetIs64Bit() const { return m_offset_is_64_bit; }
 
 private:
-  static void
-  SkipOperandTable(const lldb_private::DWARFDataExtractor &debug_macro_data,
-                   lldb::offset_t *offset);
+  static void SkipOperandTable(const DWARFDataExtractor &debug_macro_data,
+                               lldb::offset_t *offset);
 
   uint16_t m_version = 0;
   bool m_offset_is_64_bit = false;
@@ -50,12 +49,14 @@ class DWARFDebugMacroHeader {
 
 class DWARFDebugMacroEntry {
 public:
-  static void
-  ReadMacroEntries(const lldb_private::DWARFDataExtractor &debug_macro_data,
-                   const lldb_private::DWARFDataExtractor &debug_str_data,
-                   const bool offset_is_64_bit, lldb::offset_t *sect_offset,
-                   SymbolFileDWARF *sym_file_dwarf,
-                   lldb_private::DebugMacrosSP &debug_macros_sp);
+  static void ReadMacroEntries(const DWARFDataExtractor &debug_macro_data,
+                               const DWARFDataExtractor &debug_str_data,
+                               const bool offset_is_64_bit,
+                               lldb::offset_t *sect_offset,
+                               SymbolFileDWARF *sym_file_dwarf,
+                               DebugMacrosSP &debug_macros_sp);
 };
+} // namespace dwarf
+} // namespace lldb_private::plugin
 
 #endif // LLDB_SOURCE_PLUGINS_SYMBOLFILE_DWARF_DWARFDEBUGMACRO_H
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugRanges.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugRanges.cpp
index 0b5bb23a49..fd8f4e12ff 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugRanges.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugRanges.cpp
@@ -11,6 +11,7 @@
 #include "llvm/DebugInfo/DWARF/DWARFDebugRangeList.h"
 
 using namespace lldb_private;
+using namespace lldb_private::plugin::dwarf;
 
 DWARFDebugRanges::DWARFDebugRanges() : m_range_map() {}
 
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugRanges.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugRanges.h
index 2e06cd5daf..a04fcf59d5 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugRanges.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugRanges.h
@@ -12,21 +12,23 @@
 #include "lldb/Core/dwarf.h"
 #include <map>
 
+namespace lldb_private::plugin {
+namespace dwarf {
 class DWARFUnit;
-namespace lldb_private {
 class DWARFContext;
-}
 
 class DWARFDebugRanges {
 public:
   DWARFDebugRanges();
 
-  void Extract(lldb_private::DWARFContext &context);
+  void Extract(DWARFContext &context);
   DWARFRangeList FindRanges(const DWARFUnit *cu,
                             dw_offset_t debug_ranges_offset) const;
 
 protected:
   std::map<dw_offset_t, DWARFRangeList> m_range_map;
 };
+} // namespace dwarf
+} // namespace lldb_private::plugin
 
 #endif // LLDB_SOURCE_PLUGINS_SYMBOLFILE_DWARF_DWARFDEBUGRANGES_H
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDeclContext.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDeclContext.cpp
index 393de0038e..44e7602279 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDeclContext.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDeclContext.cpp
@@ -9,6 +9,7 @@
 #include "DWARFDeclContext.h"
 
 using namespace lldb_private::dwarf;
+using namespace lldb_private::plugin::dwarf;
 
 const char *DWARFDeclContext::GetQualifiedName() const {
   if (m_qualified_name.empty()) {
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDeclContext.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDeclContext.h
index 13e3dfb70c..a20a862d34 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDeclContext.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDeclContext.h
@@ -16,6 +16,8 @@
 #include <string>
 #include <vector>
 
+namespace lldb_private::plugin {
+namespace dwarf {
 // DWARFDeclContext
 //
 // A class that represents a declaration context all the way down to a
@@ -68,8 +70,8 @@ class DWARFDeclContext {
 
   // Same as GetQualifiedName, but the life time of the returned string will
   // be that of the LLDB session.
-  lldb_private::ConstString GetQualifiedNameAsConstString() const {
-    return lldb_private::ConstString(GetQualifiedName());
+  ConstString GetQualifiedNameAsConstString() const {
+    return ConstString(GetQualifiedName());
   }
 
   void Clear() {
@@ -82,5 +84,7 @@ class DWARFDeclContext {
   collection m_entries;
   mutable std::string m_qualified_name;
 };
+} // namespace dwarf
+} // namespace lldb_private::plugin
 
 #endif // LLDB_SOURCE_PLUGINS_SYMBOLFILE_DWARF_DWARFDECLCONTEXT_H
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDefines.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDefines.cpp
index 4e99a295ce..9a88aed85e 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDefines.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDefines.cpp
@@ -12,7 +12,8 @@
 #include <cstring>
 #include <string>
 
-namespace lldb_private {
+namespace lldb_private::plugin {
+namespace dwarf {
 
 const char *DW_TAG_value_to_name(uint32_t val) {
   static char invalid[100];
@@ -88,4 +89,5 @@ const char *DW_LNS_value_to_name(uint32_t val) {
   return llvmstr.data();
 }
 
-} // namespace lldb_private
+} // namespace dwarf
+} // namespace lldb_private::plugin
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDefines.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDefines.h
index 2afdbb4738..3ed92cc203 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDefines.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDefines.h
@@ -12,7 +12,8 @@
 #include "lldb/Core/dwarf.h"
 #include <cstdint>
 
-namespace lldb_private {
+namespace lldb_private::plugin {
+namespace dwarf {
 
 typedef uint32_t DRC_class; // Holds DRC_* class bitfields
 
@@ -30,6 +31,7 @@ const char *DW_LANG_value_to_name(uint32_t val);
 
 const char *DW_LNS_value_to_name(uint32_t val);
 
-} // namespace lldb_private
+} // namespace dwarf
+} // namespace lldb_private::plugin
 
 #endif // LLDB_SOURCE_PLUGINS_SYMBOLFILE_DWARF_DWARFDEFINES_H
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFFormValue.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFFormValue.cpp
index 6ca17dcf47..0a7029a55c 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFFormValue.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFFormValue.cpp
@@ -22,6 +22,7 @@ class DWARFUnit;
 
 using namespace lldb_private;
 using namespace lldb_private::dwarf;
+using namespace lldb_private::plugin::dwarf;
 
 void DWARFFormValue::Clear() {
   m_unit = nullptr;
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFFormValue.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFFormValue.h
index 2a8843c1a0..445749a6aa 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFFormValue.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFFormValue.h
@@ -13,6 +13,8 @@
 #include <cstddef>
 #include <optional>
 
+namespace lldb_private::plugin {
+namespace dwarf {
 class DWARFUnit;
 class SymbolFileDWARF;
 class DWARFDIE;
@@ -51,9 +53,8 @@ class DWARFFormValue {
   ValueType &ValueRef() { return m_value; }
   void SetValue(const ValueType &val) { m_value = val; }
 
-  void Dump(lldb_private::Stream &s) const;
-  bool ExtractValue(const lldb_private::DWARFDataExtractor &data,
-                    lldb::offset_t *offset_ptr);
+  void Dump(Stream &s) const;
+  bool ExtractValue(const DWARFDataExtractor &data, lldb::offset_t *offset_ptr);
   const uint8_t *BlockData() const;
   static std::optional<uint8_t> GetFixedSize(dw_form_t form,
                                              const DWARFUnit *u);
@@ -68,10 +69,10 @@ class DWARFFormValue {
   const char *AsCString() const;
   dw_addr_t Address() const;
   bool IsValid() const { return m_form != 0; }
-  bool SkipValue(const lldb_private::DWARFDataExtractor &debug_info_data,
+  bool SkipValue(const DWARFDataExtractor &debug_info_data,
                  lldb::offset_t *offset_ptr) const;
   static bool SkipValue(const dw_form_t form,
-                        const lldb_private::DWARFDataExtractor &debug_info_data,
+                        const DWARFDataExtractor &debug_info_data,
                         lldb::offset_t *offset_ptr, const DWARFUnit *unit);
   static bool IsBlockForm(const dw_form_t form);
   static bool IsDataForm(const dw_form_t form);
@@ -84,7 +85,9 @@ class DWARFFormValue {
   // It may be different from compile unit where m_value refers to.
   const DWARFUnit *m_unit = nullptr; // Unit for this form
   dw_form_t m_form = dw_form_t(0);   // Form for this value
-  ValueType m_value;            // Contains all data for the form
+  ValueType m_value;                 // Contains all data for the form
 };
+} // namespace dwarf
+} // namespace lldb_private::plugin
 
 #endif // LLDB_SOURCE_PLUGINS_SYMBOLFILE_DWARF_DWARFFORMVALUE_H
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFIndex.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFIndex.cpp
index 779b52481b..b1c323b101 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFIndex.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFIndex.cpp
@@ -17,6 +17,7 @@
 
 using namespace lldb_private;
 using namespace lldb;
+using namespace lldb_private::plugin::dwarf;
 
 DWARFIndex::~DWARFIndex() = default;
 
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFIndex.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFIndex.h
index 13fe96dae2..9aadeddbb2 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFIndex.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFIndex.h
@@ -17,10 +17,11 @@
 #include "lldb/Core/Module.h"
 #include "lldb/Target/Statistics.h"
 
+namespace lldb_private::plugin {
+namespace dwarf {
 class DWARFDeclContext;
 class DWARFDIE;
 
-namespace lldb_private {
 class DWARFIndex {
 public:
   DWARFIndex(Module &module) : m_module(module) {}
@@ -102,6 +103,7 @@ class DWARFIndex {
 
   void ReportInvalidDIERef(DIERef ref, llvm::StringRef name) const;
 };
-} // namespace lldb_private
+} // namespace dwarf
+} // namespace lldb_private::plugin
 
 #endif // LLDB_SOURCE_PLUGINS_SYMBOLFILE_DWARF_DWARFINDEX_H
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFTypeUnit.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFTypeUnit.cpp
index 87af7177ca..4f3a3f5446 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFTypeUnit.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFTypeUnit.cpp
@@ -13,6 +13,7 @@
 
 using namespace lldb;
 using namespace lldb_private;
+using namespace lldb_private::plugin::dwarf;
 
 void DWARFTypeUnit::Dump(Stream *s) const {
   s->Format("{0:x16}: Type Unit: length = {1:x8}, version = {2:x4}, "
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFTypeUnit.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFTypeUnit.h
index 5d939582a3..7b58c632c6 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFTypeUnit.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFTypeUnit.h
@@ -14,13 +14,15 @@
 
 namespace llvm {
 class DWARFAbbreviationDeclarationSet;
-}
+} // namespace llvm
 
+namespace lldb_private::plugin {
+namespace dwarf {
 class DWARFTypeUnit : public DWARFUnit {
 public:
   void BuildAddressRangeTable(DWARFDebugAranges *debug_aranges) override {}
 
-  void Dump(lldb_private::Stream *s) const override;
+  void Dump(Stream *s) const override;
 
   uint64_t GetTypeHash() { return m_header.GetTypeHash(); }
 
@@ -37,5 +39,7 @@ class DWARFTypeUnit : public DWARFUnit {
 
   friend class DWARFUnit;
 };
+} // namespace dwarf
+} // namespace lldb_private::plugin
 
 #endif // LLDB_SOURCE_PLUGINS_SYMBOLFILE_DWARF_DWARFTYPEUNIT_H
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp
index a09c68087c..6f771c66a7 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp
@@ -28,6 +28,7 @@
 using namespace lldb;
 using namespace lldb_private;
 using namespace lldb_private::dwarf;
+using namespace lldb_private::plugin::dwarf;
 
 extern int g_verbose;
 
@@ -201,8 +202,8 @@ DWARFUnit::ScopedExtractDIEs::ScopedExtractDIEs(ScopedExtractDIEs &&rhs)
   rhs.m_cu = nullptr;
 }
 
-DWARFUnit::ScopedExtractDIEs &DWARFUnit::ScopedExtractDIEs::operator=(
-    DWARFUnit::ScopedExtractDIEs &&rhs) {
+DWARFUnit::ScopedExtractDIEs &
+DWARFUnit::ScopedExtractDIEs::operator=(DWARFUnit::ScopedExtractDIEs &&rhs) {
   m_cu = rhs.m_cu;
   rhs.m_cu = nullptr;
   m_clear_dies = rhs.m_clear_dies;
@@ -311,9 +312,9 @@ void DWARFUnit::ExtractDIEsRWLocked() {
   }
 
   if (!m_die_array.empty()) {
-    // The last die cannot have children (if it did, it wouldn't be the last one).
-    // This only makes a difference for malformed dwarf that does not have a
-    // terminating null die.
+    // The last die cannot have children (if it did, it wouldn't be the last
+    // one). This only makes a difference for malformed dwarf that does not have
+    // a terminating null die.
     m_die_array.back().SetHasChildren(false);
 
     if (m_first_die) {
@@ -720,7 +721,7 @@ void DWARFUnit::ParseProducerInfo() {
 
   llvm::SmallVector<llvm::StringRef, 3> matches;
   if (g_swiftlang_version_regex.Execute(producer, &matches)) {
-      m_producer_version.tryParse(matches[1]);
+    m_producer_version.tryParse(matches[1]);
     m_producer = eProducerSwift;
   } else if (producer.contains("clang")) {
     if (g_clang_version_regex.Execute(producer, &matches))
@@ -905,9 +906,10 @@ llvm::Error DWARFUnitHeader::ApplyIndexEntry(
   return llvm::Error::success();
 }
 
-llvm::Expected<DWARFUnitHeader> DWARFUnitHeader::extract(
-    const DWARFDataExtractor &data, DIERef::Section section,
-    lldb_private::DWARFContext &context, lldb::offset_t *offset_ptr) {
+llvm::Expected<DWARFUnitHeader>
+DWARFUnitHeader::extract(const DWARFDataExtractor &data,
+                         DIERef::Section section, DWARFContext &context,
+                         lldb::offset_t *offset_ptr) {
   DWARFUnitHeader header;
   header.m_offset = *offset_ptr;
   header.m_length = data.GetDWARFInitialLength(offset_ptr);
@@ -1086,22 +1088,20 @@ DWARFUnit::FindRnglistFromOffset(dw_offset_t offset) {
   return ranges;
 }
 
-llvm::Expected<DWARFRangeList>
-DWARFUnit::FindRnglistFromIndex(uint32_t index) {
+llvm::Expected<DWARFRangeList> DWARFUnit::FindRnglistFromIndex(uint32_t index) {
   llvm::Expected<uint64_t> maybe_offset = GetRnglistOffset(index);
   if (!maybe_offset)
     return maybe_offset.takeError();
   return FindRnglistFromOffset(*maybe_offset);
 }
 
-
 bool DWARFUnit::HasAny(llvm::ArrayRef<dw_tag_t> tags) {
   ExtractUnitDIEIfNeeded();
   if (m_dwo)
     return m_dwo->HasAny(tags);
 
-  for (const auto &die: m_die_array) {
-    for (const auto tag: tags) {
+  for (const auto &die : m_die_array) {
+    for (const auto tag : tags) {
       if (tag == die.Tag())
         return true;
     }
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.h
index 20871d805e..3aef03712d 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.h
@@ -19,6 +19,8 @@
 #include <atomic>
 #include <optional>
 
+namespace lldb_private::plugin {
+namespace dwarf {
 class DWARFUnit;
 class DWARFCompileUnit;
 class NameToDIE;
@@ -78,21 +80,21 @@ class DWARFUnitHeader {
 
   llvm::Error ApplyIndexEntry(const llvm::DWARFUnitIndex::Entry *index_entry);
 
-  static llvm::Expected<DWARFUnitHeader>
-  extract(const lldb_private::DWARFDataExtractor &data, DIERef::Section section,
-          lldb_private::DWARFContext &dwarf_context,
-          lldb::offset_t *offset_ptr);
+  static llvm::Expected<DWARFUnitHeader> extract(const DWARFDataExtractor &data,
+                                                 DIERef::Section section,
+                                                 DWARFContext &dwarf_context,
+                                                 lldb::offset_t *offset_ptr);
 };
 
-class DWARFUnit : public lldb_private::UserID {
+class DWARFUnit : public UserID {
   using die_iterator_range =
       llvm::iterator_range<DWARFDebugInfoEntry::collection::iterator>;
 
 public:
   static llvm::Expected<DWARFUnitSP>
   extract(SymbolFileDWARF &dwarf2Data, lldb::user_id_t uid,
-          const lldb_private::DWARFDataExtractor &debug_info,
-          DIERef::Section section, lldb::offset_t *offset_ptr);
+          const DWARFDataExtractor &debug_info, DIERef::Section section,
+          lldb::offset_t *offset_ptr);
   virtual ~DWARFUnit();
 
   bool IsDWOUnit() { return m_is_dwo; }
@@ -104,6 +106,7 @@ class DWARFUnit : public lldb_private::UserID {
 
   class ScopedExtractDIEs {
     DWARFUnit *m_cu;
+
   public:
     bool m_clear_dies = false;
     ScopedExtractDIEs(DWARFUnit &cu);
@@ -115,8 +118,8 @@ class DWARFUnit : public lldb_private::UserID {
   };
   ScopedExtractDIEs ExtractDIEsScoped();
 
-  bool Verify(lldb_private::Stream *s) const;
-  virtual void Dump(lldb_private::Stream *s) const = 0;
+  bool Verify(Stream *s) const;
+  virtual void Dump(Stream *s) const = 0;
   /// Get the data that contains the DIE information for this unit.
   ///
   /// This will return the correct bytes that contain the data for
@@ -125,7 +128,7 @@ class DWARFUnit : public lldb_private::UserID {
   ///
   /// \return
   ///   The correct data for the DIE information in this unit.
-  const lldb_private::DWARFDataExtractor &GetData() const;
+  const DWARFDataExtractor &GetData() const;
 
   /// Get the size in bytes of the unit header.
   ///
@@ -210,10 +213,10 @@ class DWARFUnit : public lldb_private::UserID {
 
   bool GetIsOptimized();
 
-  const lldb_private::FileSpec &GetCompilationDirectory();
-  const lldb_private::FileSpec &GetAbsolutePath();
-  lldb_private::FileSpec GetFile(size_t file_idx);
-  lldb_private::FileSpec::Style GetPathStyle();
+  const FileSpec &GetCompilationDirectory();
+  const FileSpec &GetAbsolutePath();
+  FileSpec GetFile(size_t file_idx);
+  FileSpec::Style GetPathStyle();
 
   SymbolFileDWARFDwo *GetDwoSymbolFile();
 
@@ -227,7 +230,9 @@ class DWARFUnit : public lldb_private::UserID {
   uint8_t GetUnitType() const { return m_header.GetUnitType(); }
   bool IsTypeUnit() const { return m_header.IsTypeUnit(); }
   /// Note that this check only works for DWARF5+.
-  bool IsSkeletonUnit() const { return GetUnitType() == llvm::dwarf::DW_UT_skeleton; }
+  bool IsSkeletonUnit() const {
+    return GetUnitType() == llvm::dwarf::DW_UT_skeleton;
+  }
 
   std::optional<uint64_t> GetStringOffsetSectionItem(uint32_t index) const;
 
@@ -259,9 +264,9 @@ class DWARFUnit : public lldb_private::UserID {
   /// Return the location table for parsing the given location list data. The
   /// format is chosen according to the unit type. Never returns null.
   std::unique_ptr<llvm::DWARFLocationTable>
-  GetLocationTable(const lldb_private::DataExtractor &data) const;
+  GetLocationTable(const DataExtractor &data) const;
 
-  lldb_private::DWARFDataExtractor GetLocationData() const;
+  DWARFDataExtractor GetLocationData() const;
 
   /// Returns true if any DIEs in the unit match any DW_TAG values in \a tags.
   ///
@@ -272,7 +277,6 @@ class DWARFUnit : public lldb_private::UserID {
   ///   True if any DIEs match any tag in \a tags, false otherwise.
   bool HasAny(llvm::ArrayRef<dw_tag_t> tags);
 
-
   /// Get the fission .dwo file specific error for this compile unit.
   ///
   /// The skeleton compile unit only can have a DWO error. Any other type
@@ -281,7 +285,7 @@ class DWARFUnit : public lldb_private::UserID {
   /// \returns
   ///   A valid DWO error if there is a problem with anything in the
   ///   locating or parsing inforamtion in the .dwo file
-  const lldb_private::Status &GetDwoError() const { return m_dwo_error; }
+  const Status &GetDwoError() const { return m_dwo_error; }
 
   /// Set the fission .dwo file specific error for this compile unit.
   ///
@@ -289,7 +293,7 @@ class DWARFUnit : public lldb_private::UserID {
   /// .dwo file. Things like a missing .dwo file, DWO ID mismatch, and other
   /// .dwo errors can be stored in each compile unit so the issues can be
   /// communicated to the user.
-  void SetDwoError(const lldb_private::Status &error) { m_dwo_error = error; }
+  void SetDwoError(const Status &error) { m_dwo_error = error; }
 
 protected:
   DWARFUnit(SymbolFileDWARF &dwarf, lldb::user_id_t uid,
@@ -298,7 +302,7 @@ class DWARFUnit : public lldb_private::UserID {
             DIERef::Section section, bool is_dwo);
 
   llvm::Error ExtractHeader(SymbolFileDWARF &dwarf,
-                            const lldb_private::DWARFDataExtractor &data,
+                            const DWARFDataExtractor &data,
                             lldb::offset_t *offset_ptr);
 
   // Get the DWARF unit DWARF debug information entry. Parse the single DIE
@@ -321,7 +325,7 @@ class DWARFUnit : public lldb_private::UserID {
 
   const std::optional<llvm::DWARFDebugRnglistTable> &GetRnglistTable();
 
-  lldb_private::DWARFDataExtractor GetRnglistData() const;
+  DWARFDataExtractor GetRnglistData() const;
 
   SymbolFileDWARF &m_dwarf;
   std::shared_ptr<DWARFUnit> m_dwo;
@@ -348,12 +352,12 @@ class DWARFUnit : public lldb_private::UserID {
   DWARFProducer m_producer = eProducerInvalid;
   llvm::VersionTuple m_producer_version;
   std::optional<uint64_t> m_language_type;
-  lldb_private::LazyBool m_is_optimized = lldb_private::eLazyBoolCalculate;
-  std::optional<lldb_private::FileSpec> m_comp_dir;
-  std::optional<lldb_private::FileSpec> m_file_spec;
-  std::optional<dw_addr_t> m_addr_base;  ///< Value of DW_AT_addr_base.
-  dw_addr_t m_loclists_base = 0;         ///< Value of DW_AT_loclists_base.
-  dw_addr_t m_ranges_base = 0;           ///< Value of DW_AT_rnglists_base.
+  LazyBool m_is_optimized = eLazyBoolCalculate;
+  std::optional<FileSpec> m_comp_dir;
+  std::optional<FileSpec> m_file_spec;
+  std::optional<dw_addr_t> m_addr_base; ///< Value of DW_AT_addr_base.
+  dw_addr_t m_loclists_base = 0;        ///< Value of DW_AT_loclists_base.
+  dw_addr_t m_ranges_base = 0;          ///< Value of DW_AT_rnglists_base.
   std::optional<uint64_t> m_gnu_addr_base;
   std::optional<uint64_t> m_gnu_ranges_base;
 
@@ -374,7 +378,7 @@ class DWARFUnit : public lldb_private::UserID {
   /// If we get an error when trying to load a .dwo file, save that error here.
   /// Errors include .dwo/.dwp file not found, or the .dwp/.dwp file was found
   /// but DWO ID doesn't match, etc.
-  lldb_private::Status m_dwo_error;
+  Status m_dwo_error;
 
 private:
   void ParseProducerInfo();
@@ -390,5 +394,7 @@ class DWARFUnit : public lldb_private::UserID {
   DWARFUnit(const DWARFUnit &) = delete;
   const DWARFUnit &operator=(const DWARFUnit &) = delete;
 };
+} // namespace dwarf
+} // namespace lldb_private::plugin
 
 #endif // LLDB_SOURCE_PLUGINS_SYMBOLFILE_DWARF_DWARFUNIT_H
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.cpp
index af2d6c5541..292ea2806c 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.cpp
@@ -18,6 +18,7 @@
 using namespace lldb_private;
 using namespace lldb;
 using namespace lldb_private::dwarf;
+using namespace lldb_private::plugin::dwarf;
 
 llvm::Expected<std::unique_ptr<DebugNamesDWARFIndex>>
 DebugNamesDWARFIndex::Create(Module &module, DWARFDataExtractor debug_names,
@@ -227,7 +228,7 @@ void DebugNamesDWARFIndex::GetNamespaces(
     ConstString name, llvm::function_ref<bool(DWARFDIE die)> callback) {
   for (const DebugNames::Entry &entry :
        m_debug_names_up->equal_range(name.GetStringRef())) {
-    dwarf::Tag entry_tag = entry.tag();
+    lldb_private::dwarf::Tag entry_tag = entry.tag();
     if (entry_tag == DW_TAG_namespace ||
         entry_tag == DW_TAG_imported_declaration) {
       if (!ProcessEntry(entry, callback))
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.h b/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.h
index abbd700f16..7ce630a561 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DebugNamesDWARFIndex.h
@@ -17,7 +17,8 @@
 #include "llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h"
 #include <optional>
 
-namespace lldb_private {
+namespace lldb_private::plugin {
+namespace dwarf {
 class DebugNamesDWARFIndex : public DWARFIndex {
 public:
   static llvm::Expected<std::unique_ptr<DebugNamesDWARFIndex>>
@@ -89,6 +90,7 @@ class DebugNamesDWARFIndex : public DWARFIndex {
   static llvm::DenseSet<dw_offset_t> GetUnits(const DebugNames &debug_names);
 };
 
-} // namespace lldb_private
+} // namespace dwarf
+} // namespace lldb_private::plugin
 
 #endif // LLDB_SOURCE_PLUGINS_SYMBOLFILE_DWARF_DEBUGNAMESDWARFINDEX_H
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/ManualDWARFIndex.cpp b/lldb/source/Plugins/SymbolFile/DWARF/ManualDWARFIndex.cpp
index 90f18c96af..16ff5f7d48 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/ManualDWARFIndex.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/ManualDWARFIndex.cpp
@@ -28,6 +28,7 @@
 using namespace lldb_private;
 using namespace lldb;
 using namespace lldb_private::dwarf;
+using namespace lldb_private::plugin::dwarf;
 
 void ManualDWARFIndex::Index() {
   if (m_indexed)
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/ManualDWARFIndex.h b/lldb/source/Plugins/SymbolFile/DWARF/ManualDWARFIndex.h
index d95cf501fa..0126e587e5 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/ManualDWARFIndex.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/ManualDWARFIndex.h
@@ -13,10 +13,11 @@
 #include "Plugins/SymbolFile/DWARF/NameToDIE.h"
 #include "llvm/ADT/DenseSet.h"
 
+namespace lldb_private::plugin {
+namespace dwarf {
 class DWARFDebugInfo;
 class SymbolFileDWARFDwo;
 
-namespace lldb_private {
 class ManualDWARFIndex : public DWARFIndex {
 public:
   ManualDWARFIndex(Module &module, SymbolFileDWARF &dwarf,
@@ -173,6 +174,7 @@ class ManualDWARFIndex : public DWARFIndex {
   IndexSet m_set;
   bool m_indexed = false;
 };
-} // namespace lldb_private
+} // namespace dwarf
+} // namespace lldb_private::plugin
 
 #endif // LLDB_SOURCE_PLUGINS_SYMBOLFILE_DWARF_MANUALDWARFINDEX_H
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/NameToDIE.cpp b/lldb/source/Plugins/SymbolFile/DWARF/NameToDIE.cpp
index 89e628f5ea..44d9064870 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/NameToDIE.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/NameToDIE.cpp
@@ -20,6 +20,7 @@
 
 using namespace lldb;
 using namespace lldb_private;
+using namespace lldb_private::plugin::dwarf;
 
 void NameToDIE::Finalize() {
   m_map.Sort(std::less<DIERef>());
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/NameToDIE.h b/lldb/source/Plugins/SymbolFile/DWARF/NameToDIE.h
index 61df1a628a..90eac1fa37 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/NameToDIE.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/NameToDIE.h
@@ -16,6 +16,8 @@
 #include "lldb/Core/dwarf.h"
 #include "lldb/lldb-defines.h"
 
+namespace lldb_private::plugin {
+namespace dwarf {
 class DWARFUnit;
 
 class NameToDIE {
@@ -24,18 +26,18 @@ class NameToDIE {
 
   ~NameToDIE() = default;
 
-  void Dump(lldb_private::Stream *s);
+  void Dump(Stream *s);
 
-  void Insert(lldb_private::ConstString name, const DIERef &die_ref);
+  void Insert(ConstString name, const DIERef &die_ref);
 
   void Append(const NameToDIE &other);
 
   void Finalize();
 
-  bool Find(lldb_private::ConstString name,
+  bool Find(ConstString name,
             llvm::function_ref<bool(DIERef ref)> callback) const;
 
-  bool Find(const lldb_private::RegularExpression &regex,
+  bool Find(const RegularExpression &regex,
             llvm::function_ref<bool(DIERef ref)> callback) const;
 
   /// \a unit must be the skeleton unit if possible, not GetNonSkeletonUnit().
@@ -44,8 +46,7 @@ class NameToDIE {
                         llvm::function_ref<bool(DIERef ref)> callback) const;
 
   void
-  ForEach(std::function<bool(lldb_private::ConstString name,
-                             const DIERef &die_ref)> const
+  ForEach(std::function<bool(ConstString name, const DIERef &die_ref)> const
               &callback) const;
 
   /// Decode a serialized version of this object from data.
@@ -61,9 +62,8 @@ class NameToDIE {
   ///   All strings in cache files are put into string tables for efficiency
   ///   and cache file size reduction. Strings are stored as uint32_t string
   ///   table offsets in the cache data.
-  bool Decode(const lldb_private::DataExtractor &data,
-              lldb::offset_t *offset_ptr,
-              const lldb_private::StringTableReader &strtab);
+  bool Decode(const DataExtractor &data, lldb::offset_t *offset_ptr,
+              const StringTableReader &strtab);
 
   /// Encode this object into a data encoder object.
   ///
@@ -76,8 +76,7 @@ class NameToDIE {
   ///   All strings in cache files are put into string tables for efficiency
   ///   and cache file size reduction. Strings are stored as uint32_t string
   ///   table offsets in the cache data.
-  void Encode(lldb_private::DataEncoder &encoder,
-              lldb_private::ConstStringTable &strtab) const;
+  void Encode(DataEncoder &encoder, ConstStringTable &strtab) const;
 
   /// Used for unit testing the encoding and decoding.
   bool operator==(const NameToDIE &rhs) const;
@@ -87,7 +86,9 @@ class NameToDIE {
   void Clear() { m_map.Clear(); }
 
 protected:
-  lldb_private::UniqueCStringMap<DIERef> m_map;
+  UniqueCStringMap<DIERef> m_map;
 };
+} // namespace dwarf
+} // namespace lldb_private::plugin
 
 #endif // LLDB_SOURCE_PLUGINS_SYMBOLFILE_DWARF_NAMETODIE_H
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
index e472074545..3f7248c397 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
@@ -10,6 +10,7 @@
 
 #include "llvm/DebugInfo/DWARF/DWARFDebugLoc.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/Format.h"
 #include "llvm/Support/Threading.h"
 
 #include "lldb/Core/Module.h"
@@ -24,6 +25,7 @@
 #include "lldb/Utility/RegularExpression.h"
 #include "lldb/Utility/Scalar.h"
 #include "lldb/Utility/StreamString.h"
+#include "lldb/Utility/StructuredData.h"
 #include "lldb/Utility/Timer.h"
 
 #include "Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.h"
@@ -97,6 +99,7 @@
 using namespace lldb;
 using namespace lldb_private;
 using namespace lldb_private::dwarf;
+using namespace lldb_private::plugin::dwarf;
 
 LLDB_PLUGIN_DEFINE(SymbolFileDWARF)
 
@@ -136,9 +139,8 @@ static PluginProperties &GetGlobalPluginProperties() {
 }
 
 static const llvm::DWARFDebugLine::LineTable *
-ParseLLVMLineTable(lldb_private::DWARFContext &context,
-                   llvm::DWARFDebugLine &line, dw_offset_t line_offset,
-                   dw_offset_t unit_offset) {
+ParseLLVMLineTable(DWARFContext &context, llvm::DWARFDebugLine &line,
+                   dw_offset_t line_offset, dw_offset_t unit_offset) {
   Log *log = GetLog(DWARFLog::DebugInfo);
 
   llvm::DWARFDataExtractor data = context.getOrLoadLineData().GetAsLLVMDWARF();
@@ -159,7 +161,7 @@ ParseLLVMLineTable(lldb_private::DWARFContext &context,
   return *line_table;
 }
 
-static bool ParseLLVMLineTablePrologue(lldb_private::DWARFContext &context,
+static bool ParseLLVMLineTablePrologue(DWARFContext &context,
                                        llvm::DWARFDebugLine::Prologue &prologue,
                                        dw_offset_t line_offset,
                                        dw_offset_t unit_offset) {
@@ -1752,11 +1754,10 @@ SymbolFileDWARF::GetDwoSymbolFileForCompileUnit(
   // it. Or it's absolute.
   found = FileSystem::Instance().Exists(dwo_file);
 
+  const char *comp_dir =
+      cu_die.GetAttributeValueAsString(dwarf_cu, DW_AT_comp_dir, nullptr);
   if (!found) {
     // It could be a relative path that also uses DW_AT_COMP_DIR.
-    const char *comp_dir =
-        cu_die.GetAttributeValueAsString(dwarf_cu, DW_AT_comp_dir, nullptr);
-
     if (comp_dir) {
       dwo_file.SetFile(comp_dir, FileSpec::Style::native);
       if (!dwo_file.IsRelative()) {
@@ -1854,10 +1855,16 @@ SymbolFileDWARF::GetDwoSymbolFileForCompileUnit(
   }
 
   if (!found) {
+    FileSpec error_dwo_path(dwo_name);
+    FileSystem::Instance().Resolve(error_dwo_path);
+    if (error_dwo_path.IsRelative() && comp_dir != nullptr) {
+      error_dwo_path.PrependPathComponent(comp_dir);
+      FileSystem::Instance().Resolve(error_dwo_path);
+    }
     unit.SetDwoError(Status::createWithFormat(
         "unable to locate .dwo debug file \"{0}\" for skeleton DIE "
         "{1:x16}",
-        dwo_file.GetPath().c_str(), cu_die.GetOffset()));
+        error_dwo_path.GetPath().c_str(), cu_die.GetOffset()));
 
     if (m_dwo_warning_issued.test_and_set(std::memory_order_relaxed) == false) {
       GetObjectFile()->GetModule()->ReportWarning(
@@ -2428,7 +2435,7 @@ bool SymbolFileDWARF::DIEInDeclContext(const CompilerDeclContext &decl_ctx,
     // ...But if we are only checking root decl contexts, confirm that the
     // 'die' is a top-level context.
     if (only_root_namespaces)
-      return die.GetParent().Tag() == dwarf::DW_TAG_compile_unit;
+      return die.GetParent().Tag() == llvm::dwarf::DW_TAG_compile_unit;
 
     return true;
   }
@@ -4226,6 +4233,70 @@ void SymbolFileDWARF::DumpClangAST(Stream &s) {
   clang->Dump(s.AsRawOstream());
 }
 
+bool SymbolFileDWARF::GetSeparateDebugInfo(StructuredData::Dictionary &d) {
+  StructuredData::Array separate_debug_info_files;
+  DWARFDebugInfo &info = DebugInfo();
+  const size_t num_cus = info.GetNumUnits();
+  for (size_t cu_idx = 0; cu_idx < num_cus; cu_idx++) {
+    DWARFUnit *unit = info.GetUnitAtIndex(cu_idx);
+    DWARFCompileUnit *dwarf_cu = llvm::dyn_cast<DWARFCompileUnit>(unit);
+    if (dwarf_cu == nullptr)
+      continue;
+
+    // Check if this is a DWO unit by checking if it has a DWO ID.
+    // NOTE: it seems that `DWARFUnit::IsDWOUnit` is always false?
+    if (!dwarf_cu->GetDWOId().has_value())
+      continue;
+
+    StructuredData::DictionarySP dwo_data =
+        std::make_shared<StructuredData::Dictionary>();
+    const uint64_t dwo_id = dwarf_cu->GetDWOId().value();
+    dwo_data->AddIntegerItem("dwo_id", dwo_id);
+
+    if (const DWARFBaseDIE die = dwarf_cu->GetUnitDIEOnly()) {
+      const char *dwo_name = GetDWOName(*dwarf_cu, *die.GetDIE());
+      if (dwo_name) {
+        dwo_data->AddStringItem("dwo_name", dwo_name);
+      } else {
+        dwo_data->AddStringItem("error", "missing dwo name");
+      }
+
+      const char *comp_dir = die.GetDIE()->GetAttributeValueAsString(
+          dwarf_cu, DW_AT_comp_dir, nullptr);
+      if (comp_dir) {
+        dwo_data->AddStringItem("comp_dir", comp_dir);
+      }
+    } else {
+      dwo_data->AddStringItem(
+          "error",
+          llvm::formatv("unable to get unit DIE for DWARFUnit at {0:x}",
+                        dwarf_cu->GetOffset())
+              .str());
+    }
+
+    // If we have a DWO symbol file, that means we were able to successfully
+    // load it.
+    SymbolFile *dwo_symfile = dwarf_cu->GetDwoSymbolFile();
+    if (dwo_symfile) {
+      dwo_data->AddStringItem(
+          "resolved_dwo_path",
+          dwo_symfile->GetObjectFile()->GetFileSpec().GetPath());
+    } else {
+      dwo_data->AddStringItem("error",
+                              dwarf_cu->GetDwoError().AsCString("unknown"));
+    }
+    dwo_data->AddBooleanItem("loaded", dwo_symfile != nullptr);
+    separate_debug_info_files.AddItem(dwo_data);
+  }
+
+  d.AddStringItem("type", "dwo");
+  d.AddStringItem("symfile", GetMainObjectFile()->GetFileSpec().GetPath());
+  d.AddItem("separate-debug-info-files",
+            std::make_shared<StructuredData::Array>(
+                std::move(separate_debug_info_files)));
+  return true;
+}
+
 SymbolFileDWARFDebugMap *SymbolFileDWARF::GetDebugMapSymfile() {
   if (m_debug_map_symfile == nullptr) {
     lldb::ModuleSP module_sp(m_debug_map_module_wp.lock());
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
index 5aaf8bd270..646d5d9a47 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
@@ -30,6 +30,7 @@
 #include "lldb/Utility/ConstString.h"
 #include "lldb/Utility/Flags.h"
 #include "lldb/Utility/RangeMap.h"
+#include "lldb/Utility/StructuredData.h"
 #include "lldb/lldb-private.h"
 
 #include "DWARFContext.h"
@@ -38,6 +39,14 @@
 #include "DWARFIndex.h"
 #include "UniqueDWARFASTType.h"
 
+class DWARFASTParserClang;
+
+namespace llvm {
+class DWARFDebugAbbrev;
+} // namespace llvm
+
+namespace lldb_private::plugin {
+namespace dwarf {
 // Forward Declarations for this DWARF plugin
 class DebugMapModule;
 class DWARFCompileUnit;
@@ -52,15 +61,10 @@ class DWARFTypeUnit;
 class SymbolFileDWARFDebugMap;
 class SymbolFileDWARFDwo;
 class SymbolFileDWARFDwp;
-class UserID;
-
-namespace llvm {
-class DWARFDebugAbbrev;
-}
 
 #define DIE_IS_BEING_PARSED ((lldb_private::Type *)1)
 
-class SymbolFileDWARF : public lldb_private::SymbolFileCommon {
+class SymbolFileDWARF : public SymbolFileCommon {
   /// LLVM RTTI support.
   static char ID;
 
@@ -78,26 +82,24 @@ class SymbolFileDWARF : public lldb_private::SymbolFileCommon {
   friend class DebugMapModule;
   friend class DWARFCompileUnit;
   friend class DWARFDIE;
-  friend class DWARFASTParserClang;
+  friend class ::DWARFASTParserClang;
 
   // Static Functions
   static void Initialize();
 
   static void Terminate();
 
-  static void DebuggerInitialize(lldb_private::Debugger &debugger);
+  static void DebuggerInitialize(Debugger &debugger);
 
   static llvm::StringRef GetPluginNameStatic() { return "dwarf"; }
 
   static llvm::StringRef GetPluginDescriptionStatic();
 
-  static lldb_private::SymbolFile *
-  CreateInstance(lldb::ObjectFileSP objfile_sp);
+  static SymbolFile *CreateInstance(lldb::ObjectFileSP objfile_sp);
 
   // Constructors and Destructors
 
-  SymbolFileDWARF(lldb::ObjectFileSP objfile_sp,
-                  lldb_private::SectionList *dwo_section_list);
+  SymbolFileDWARF(lldb::ObjectFileSP objfile_sp, SectionList *dwo_section_list);
 
   ~SymbolFileDWARF() override;
 
@@ -107,118 +109,99 @@ class SymbolFileDWARF : public lldb_private::SymbolFileCommon {
 
   // Compile Unit function calls
 
-  lldb::LanguageType
-  ParseLanguage(lldb_private::CompileUnit &comp_unit) override;
+  lldb::LanguageType ParseLanguage(CompileUnit &comp_unit) override;
 
-  lldb_private::XcodeSDK
-  ParseXcodeSDK(lldb_private::CompileUnit &comp_unit) override;
+  XcodeSDK ParseXcodeSDK(CompileUnit &comp_unit) override;
 
-  size_t ParseFunctions(lldb_private::CompileUnit &comp_unit) override;
+  size_t ParseFunctions(CompileUnit &comp_unit) override;
 
-  bool ParseLineTable(lldb_private::CompileUnit &comp_unit) override;
+  bool ParseLineTable(CompileUnit &comp_unit) override;
 
-  bool ParseDebugMacros(lldb_private::CompileUnit &comp_unit) override;
+  bool ParseDebugMacros(CompileUnit &comp_unit) override;
 
-  bool ForEachExternalModule(
-      lldb_private::CompileUnit &, llvm::DenseSet<lldb_private::SymbolFile *> &,
-      llvm::function_ref<bool(lldb_private::Module &)>) override;
+  bool ForEachExternalModule(CompileUnit &, llvm::DenseSet<SymbolFile *> &,
+                             llvm::function_ref<bool(Module &)>) override;
 
-  bool ParseSupportFiles(lldb_private::CompileUnit &comp_unit,
-                         lldb_private::FileSpecList &support_files) override;
+  bool ParseSupportFiles(CompileUnit &comp_unit,
+                         FileSpecList &support_files) override;
 
-  bool ParseIsOptimized(lldb_private::CompileUnit &comp_unit) override;
+  bool ParseIsOptimized(CompileUnit &comp_unit) override;
 
-  size_t ParseTypes(lldb_private::CompileUnit &comp_unit) override;
+  size_t ParseTypes(CompileUnit &comp_unit) override;
 
-  bool ParseImportedModules(
-      const lldb_private::SymbolContext &sc,
-      std::vector<lldb_private::SourceModule> &imported_modules) override;
+  bool
+  ParseImportedModules(const SymbolContext &sc,
+                       std::vector<SourceModule> &imported_modules) override;
 
-  size_t ParseBlocksRecursive(lldb_private::Function &func) override;
+  size_t ParseBlocksRecursive(Function &func) override;
 
-  size_t
-  ParseVariablesForContext(const lldb_private::SymbolContext &sc) override;
+  size_t ParseVariablesForContext(const SymbolContext &sc) override;
 
-  lldb_private::Type *ResolveTypeUID(lldb::user_id_t type_uid) override;
-  std::optional<ArrayInfo> GetDynamicArrayInfoForUID(
-      lldb::user_id_t type_uid,
-      const lldb_private::ExecutionContext *exe_ctx) override;
+  Type *ResolveTypeUID(lldb::user_id_t type_uid) override;
+  std::optional<ArrayInfo>
+  GetDynamicArrayInfoForUID(lldb::user_id_t type_uid,
+                            const ExecutionContext *exe_ctx) override;
 
-  bool CompleteType(lldb_private::CompilerType &compiler_type) override;
+  bool CompleteType(CompilerType &compiler_type) override;
 
-  lldb_private::Type *ResolveType(const DWARFDIE &die,
-                                  bool assert_not_being_parsed = true,
-                                  bool resolve_function_context = false);
+  Type *ResolveType(const DWARFDIE &die, bool assert_not_being_parsed = true,
+                    bool resolve_function_context = false);
 
-  lldb_private::CompilerDecl GetDeclForUID(lldb::user_id_t uid) override;
+  CompilerDecl GetDeclForUID(lldb::user_id_t uid) override;
 
-  lldb_private::CompilerDeclContext
-  GetDeclContextForUID(lldb::user_id_t uid) override;
+  CompilerDeclContext GetDeclContextForUID(lldb::user_id_t uid) override;
 
-  lldb_private::CompilerDeclContext
-  GetDeclContextContainingUID(lldb::user_id_t uid) override;
+  CompilerDeclContext GetDeclContextContainingUID(lldb::user_id_t uid) override;
 
-  void
-  ParseDeclsForContext(lldb_private::CompilerDeclContext decl_ctx) override;
+  void ParseDeclsForContext(CompilerDeclContext decl_ctx) override;
 
-  uint32_t ResolveSymbolContext(const lldb_private::Address &so_addr,
+  uint32_t ResolveSymbolContext(const Address &so_addr,
                                 lldb::SymbolContextItem resolve_scope,
-                                lldb_private::SymbolContext &sc) override;
-
-  lldb_private::Status
-  CalculateFrameVariableError(lldb_private::StackFrame &frame) override;
+                                SymbolContext &sc) override;
 
-  uint32_t ResolveSymbolContext(
-      const lldb_private::SourceLocationSpec &src_location_spec,
-      lldb::SymbolContextItem resolve_scope,
-      lldb_private::SymbolContextList &sc_list) override;
+  Status CalculateFrameVariableError(StackFrame &frame) override;
 
-  void
-  FindGlobalVariables(lldb_private::ConstString name,
-                      const lldb_private::CompilerDeclContext &parent_decl_ctx,
-                      uint32_t max_matches,
-                      lldb_private::VariableList &variables) override;
+  uint32_t ResolveSymbolContext(const SourceLocationSpec &src_location_spec,
+                                lldb::SymbolContextItem resolve_scope,
+                                SymbolContextList &sc_list) override;
 
-  void FindGlobalVariables(const lldb_private::RegularExpression &regex,
+  void FindGlobalVariables(ConstString name,
+                           const CompilerDeclContext &parent_decl_ctx,
                            uint32_t max_matches,
-                           lldb_private::VariableList &variables) override;
+                           VariableList &variables) override;
 
-  void FindFunctions(const lldb_private::Module::LookupInfo &lookup_info,
-                     const lldb_private::CompilerDeclContext &parent_decl_ctx,
-                     bool include_inlines,
-                     lldb_private::SymbolContextList &sc_list) override;
+  void FindGlobalVariables(const RegularExpression &regex, uint32_t max_matches,
+                           VariableList &variables) override;
 
-  void FindFunctions(const lldb_private::RegularExpression &regex,
-                     bool include_inlines,
-                     lldb_private::SymbolContextList &sc_list) override;
+  void FindFunctions(const Module::LookupInfo &lookup_info,
+                     const CompilerDeclContext &parent_decl_ctx,
+                     bool include_inlines, SymbolContextList &sc_list) override;
 
-  void GetMangledNamesForFunction(
-      const std::string &scope_qualified_name,
-      std::vector<lldb_private::ConstString> &mangled_names) override;
+  void FindFunctions(const RegularExpression &regex, bool include_inlines,
+                     SymbolContextList &sc_list) override;
 
   void
-  FindTypes(lldb_private::ConstString name,
-            const lldb_private::CompilerDeclContext &parent_decl_ctx,
-            uint32_t max_matches,
-            llvm::DenseSet<lldb_private::SymbolFile *> &searched_symbol_files,
-            lldb_private::TypeMap &types) override;
-
-  void FindTypes(llvm::ArrayRef<lldb_private::CompilerContext> pattern,
-                 lldb_private::LanguageSet languages,
+  GetMangledNamesForFunction(const std::string &scope_qualified_name,
+                             std::vector<ConstString> &mangled_names) override;
+
+  void FindTypes(ConstString name, const CompilerDeclContext &parent_decl_ctx,
+                 uint32_t max_matches,
                  llvm::DenseSet<SymbolFile *> &searched_symbol_files,
-                 lldb_private::TypeMap &types) override;
+                 TypeMap &types) override;
 
-  void GetTypes(lldb_private::SymbolContextScope *sc_scope,
-                lldb::TypeClass type_mask,
-                lldb_private::TypeList &type_list) override;
+  void FindTypes(llvm::ArrayRef<CompilerContext> pattern, LanguageSet languages,
+                 llvm::DenseSet<SymbolFile *> &searched_symbol_files,
+                 TypeMap &types) override;
+
+  void GetTypes(SymbolContextScope *sc_scope, lldb::TypeClass type_mask,
+                TypeList &type_list) override;
 
   llvm::Expected<lldb::TypeSystemSP>
   GetTypeSystemForLanguage(lldb::LanguageType language) override;
 
-  lldb_private::CompilerDeclContext
-  FindNamespace(lldb_private::ConstString name,
-                const lldb_private::CompilerDeclContext &parent_decl_ctx,
-                bool only_root_namespaces) override;
+  CompilerDeclContext FindNamespace(ConstString name,
+                                    const CompilerDeclContext &parent_decl_ctx,
+                                    bool only_root_namespaces) override;
 
   void PreloadSymbols() override;
 
@@ -238,25 +221,22 @@ class SymbolFileDWARF : public lldb_private::SymbolFileCommon {
   DWARFDIE
   GetDeclContextDIEContainingDIE(const DWARFDIE &die);
 
-  bool
-  HasForwardDeclForClangType(const lldb_private::CompilerType &compiler_type);
+  bool HasForwardDeclForClangType(const CompilerType &compiler_type);
 
-  lldb_private::CompileUnit *
-  GetCompUnitForDWARFCompUnit(DWARFCompileUnit &dwarf_cu);
+  CompileUnit *GetCompUnitForDWARFCompUnit(DWARFCompileUnit &dwarf_cu);
 
-  virtual void GetObjCMethods(lldb_private::ConstString class_name,
+  virtual void GetObjCMethods(ConstString class_name,
                               llvm::function_ref<bool(DWARFDIE die)> callback);
 
   bool Supports_DW_AT_APPLE_objc_complete_type(DWARFUnit *cu);
 
-  lldb_private::DebugMacrosSP ParseDebugMacros(lldb::offset_t *offset);
+  DebugMacrosSP ParseDebugMacros(lldb::offset_t *offset);
 
   static DWARFDIE GetParentSymbolContextDIE(const DWARFDIE &die);
 
-  lldb::ModuleSP GetExternalModule(lldb_private::ConstString name);
+  lldb::ModuleSP GetExternalModule(ConstString name);
 
-  typedef std::map<lldb_private::ConstString, lldb::ModuleSP>
-      ExternalTypeModuleMap;
+  typedef std::map<ConstString, lldb::ModuleSP> ExternalTypeModuleMap;
 
   /// Return the list of Clang modules imported by this SymbolFile.
   const ExternalTypeModuleMap &getExternalTypeModules() const {
@@ -274,22 +254,25 @@ class SymbolFileDWARF : public lldb_private::SymbolFileCommon {
   /// If this is a DWARF object with a single CU, return its DW_AT_dwo_id.
   std::optional<uint64_t> GetDWOId();
 
-  static bool
-  DIEInDeclContext(const lldb_private::CompilerDeclContext &parent_decl_ctx,
-                   const DWARFDIE &die, bool only_root_namespaces = false);
+  static bool DIEInDeclContext(const CompilerDeclContext &parent_decl_ctx,
+                               const DWARFDIE &die,
+                               bool only_root_namespaces = false);
+
+  std::vector<std::unique_ptr<CallEdge>>
+  ParseCallEdgesInFunction(UserID func_id) override;
 
-  std::vector<std::unique_ptr<lldb_private::CallEdge>>
-  ParseCallEdgesInFunction(lldb_private::UserID func_id) override;
+  void Dump(Stream &s) override;
 
-  void Dump(lldb_private::Stream &s) override;
+  void DumpClangAST(Stream &s) override;
 
-  void DumpClangAST(lldb_private::Stream &s) override;
+  /// List separate dwo files.
+  bool GetSeparateDebugInfo(StructuredData::Dictionary &d) override;
 
-  lldb_private::DWARFContext &GetDWARFContext() { return m_context; }
+  DWARFContext &GetDWARFContext() { return m_context; }
 
   const std::shared_ptr<SymbolFileDWARFDwo> &GetDwpSymbolFile();
 
-  lldb_private::FileSpec GetFile(DWARFUnit &unit, size_t file_idx);
+  FileSpec GetFile(DWARFUnit &unit, size_t file_idx);
 
   static llvm::Expected<lldb::TypeSystemSP> GetTypeSystem(DWARFUnit &unit);
 
@@ -297,12 +280,11 @@ class SymbolFileDWARF : public lldb_private::SymbolFileCommon {
 
   // CompilerDecl related functions
 
-  static lldb_private::CompilerDecl GetDecl(const DWARFDIE &die);
+  static CompilerDecl GetDecl(const DWARFDIE &die);
 
-  static lldb_private::CompilerDeclContext GetDeclContext(const DWARFDIE &die);
+  static CompilerDeclContext GetDeclContext(const DWARFDIE &die);
 
-  static lldb_private::CompilerDeclContext
-  GetContainingDeclContext(const DWARFDIE &die);
+  static CompilerDeclContext GetContainingDeclContext(const DWARFDIE &die);
 
   static DWARFDeclContext GetDWARFDeclContext(const DWARFDIE &die);
 
@@ -312,39 +294,34 @@ class SymbolFileDWARF : public lldb_private::SymbolFileCommon {
   /// Same as GetLanguage() but reports all C++ versions as C++ (no version).
   static lldb::LanguageType GetLanguageFamily(DWARFUnit &unit);
 
-  lldb_private::StatsDuration::Duration GetDebugInfoParseTime() override {
+  StatsDuration::Duration GetDebugInfoParseTime() override {
     return m_parse_time;
   }
-  lldb_private::StatsDuration::Duration GetDebugInfoIndexTime() override;
+  StatsDuration::Duration GetDebugInfoIndexTime() override;
 
-  lldb_private::StatsDuration &GetDebugInfoParseTimeRef() {
-    return m_parse_time;
-  }
+  StatsDuration &GetDebugInfoParseTimeRef() { return m_parse_time; }
 
   virtual lldb::offset_t
-  GetVendorDWARFOpcodeSize(const lldb_private::DataExtractor &data,
+  GetVendorDWARFOpcodeSize(const DataExtractor &data,
                            const lldb::offset_t data_offset,
                            const uint8_t op) const {
     return LLDB_INVALID_OFFSET;
   }
 
-  virtual bool
-  ParseVendorDWARFOpcode(uint8_t op, const lldb_private::DataExtractor &opcodes,
-                         lldb::offset_t &offset,
-                         std::vector<lldb_private::Value> &stack) const {
+  virtual bool ParseVendorDWARFOpcode(uint8_t op, const DataExtractor &opcodes,
+                                      lldb::offset_t &offset,
+                                      std::vector<Value> &stack) const {
     return false;
   }
 
-  lldb_private::ConstString ConstructFunctionDemangledName(const DWARFDIE &die);
+  ConstString ConstructFunctionDemangledName(const DWARFDIE &die);
 
   std::optional<uint64_t> GetFileIndex() const { return m_file_index; }
   void SetFileIndex(std::optional<uint64_t> file_index) {
     m_file_index = file_index;
   }
 
-protected:
-  typedef llvm::DenseMap<const DWARFDebugInfoEntry *, lldb_private::Type *>
-      DIEToTypePtr;
+  typedef llvm::DenseMap<const DWARFDebugInfoEntry *, Type *> DIEToTypePtr;
   typedef llvm::DenseMap<const DWARFDebugInfoEntry *, lldb::VariableSP>
       DIEToVariableSP;
   typedef llvm::DenseMap<const DWARFDebugInfoEntry *,
@@ -356,69 +333,64 @@ class SymbolFileDWARF : public lldb_private::SymbolFileCommon {
   const SymbolFileDWARF &operator=(const SymbolFileDWARF &) = delete;
 
   virtual void LoadSectionData(lldb::SectionType sect_type,
-                               lldb_private::DWARFDataExtractor &data);
+                               DWARFDataExtractor &data);
 
-  bool DeclContextMatchesThisSymbolFile(
-      const lldb_private::CompilerDeclContext &decl_ctx);
+  bool DeclContextMatchesThisSymbolFile(const CompilerDeclContext &decl_ctx);
 
   uint32_t CalculateNumCompileUnits() override;
 
   lldb::CompUnitSP ParseCompileUnitAtIndex(uint32_t index) override;
 
-  lldb_private::TypeList &GetTypeList() override;
+  TypeList &GetTypeList() override;
 
   lldb::CompUnitSP ParseCompileUnit(DWARFCompileUnit &dwarf_cu);
 
-  virtual DWARFCompileUnit *
-  GetDWARFCompileUnit(lldb_private::CompileUnit *comp_unit);
+  virtual DWARFCompileUnit *GetDWARFCompileUnit(CompileUnit *comp_unit);
 
   DWARFUnit *GetNextUnparsedDWARFCompileUnit(DWARFUnit *prev_cu);
 
-  bool GetFunction(const DWARFDIE &die, lldb_private::SymbolContext &sc);
+  bool GetFunction(const DWARFDIE &die, SymbolContext &sc);
 
-  lldb_private::Function *ParseFunction(lldb_private::CompileUnit &comp_unit,
-                                        const DWARFDIE &die);
+  Function *ParseFunction(CompileUnit &comp_unit, const DWARFDIE &die);
 
-  size_t ParseBlocksRecursive(lldb_private::CompileUnit &comp_unit,
-                              lldb_private::Block *parent_block,
+  size_t ParseBlocksRecursive(CompileUnit &comp_unit, Block *parent_block,
                               const DWARFDIE &die,
                               lldb::addr_t subprogram_low_pc, uint32_t depth);
 
-  size_t ParseTypes(const lldb_private::SymbolContext &sc, const DWARFDIE &die,
+  size_t ParseTypes(const SymbolContext &sc, const DWARFDIE &die,
                     bool parse_siblings, bool parse_children);
 
-  lldb::TypeSP ParseType(const lldb_private::SymbolContext &sc,
-                         const DWARFDIE &die, bool *type_is_new);
+  lldb::TypeSP ParseType(const SymbolContext &sc, const DWARFDIE &die,
+                         bool *type_is_new);
 
   bool ParseSupportFiles(DWARFUnit &dwarf_cu, const lldb::ModuleSP &module,
-                         lldb_private::FileSpecList &support_files);
+                         FileSpecList &support_files);
 
-  lldb_private::Type *ResolveTypeUID(const DWARFDIE &die,
-                                     bool assert_not_being_parsed);
+  Type *ResolveTypeUID(const DWARFDIE &die, bool assert_not_being_parsed);
 
-  lldb_private::Type *ResolveTypeUID(const DIERef &die_ref);
+  Type *ResolveTypeUID(const DIERef &die_ref);
 
-  lldb::VariableSP ParseVariableDIE(const lldb_private::SymbolContext &sc,
+  lldb::VariableSP ParseVariableDIE(const SymbolContext &sc,
                                     const DWARFDIE &die,
                                     const lldb::addr_t func_low_pc);
-  lldb::VariableSP ParseVariableDIECached(const lldb_private::SymbolContext &sc,
+  lldb::VariableSP ParseVariableDIECached(const SymbolContext &sc,
                                           const DWARFDIE &die);
 
-  void
-  ParseAndAppendGlobalVariable(const lldb_private::SymbolContext &sc,
-                               const DWARFDIE &die,
-                               lldb_private::VariableList &cc_variable_list);
+  void ParseAndAppendGlobalVariable(const SymbolContext &sc,
+                                    const DWARFDIE &die,
+                                    VariableList &cc_variable_list);
 
-  size_t ParseVariablesInFunctionContext(const lldb_private::SymbolContext &sc,
+  size_t ParseVariablesInFunctionContext(const SymbolContext &sc,
                                          const DWARFDIE &die,
                                          const lldb::addr_t func_low_pc);
 
-  size_t ParseVariablesInFunctionContextRecursive(
-      const lldb_private::SymbolContext &sc, const DWARFDIE &die,
-      lldb::addr_t func_low_pc, DIEArray &accumulator);
+  size_t ParseVariablesInFunctionContextRecursive(const SymbolContext &sc,
+                                                  const DWARFDIE &die,
+                                                  lldb::addr_t func_low_pc,
+                                                  DIEArray &accumulator);
 
-  size_t PopulateBlockVariableList(lldb_private::VariableList &variable_list,
-                                   const lldb_private::SymbolContext &sc,
+  size_t PopulateBlockVariableList(VariableList &variable_list,
+                                   const SymbolContext &sc,
                                    llvm::ArrayRef<DIERef> variable_dies,
                                    lldb::addr_t func_low_pc);
 
@@ -429,25 +401,22 @@ class SymbolFileDWARF : public lldb_private::SymbolFileCommon {
 
   // Given a die_offset, figure out the symbol context representing that die.
   bool ResolveFunction(const DWARFDIE &die, bool include_inlines,
-                       lldb_private::SymbolContextList &sc_list);
+                       SymbolContextList &sc_list);
 
   /// Resolve functions and (possibly) blocks for the given file address and a
   /// compile unit. The compile unit comes from the sc argument and it must be
   /// set. The results of the lookup (if any) are written back to the symbol
   /// context.
   void ResolveFunctionAndBlock(lldb::addr_t file_vm_addr, bool lookup_block,
-                               lldb_private::SymbolContext &sc);
+                               SymbolContext &sc);
 
   virtual lldb::TypeSP
   FindDefinitionTypeForDWARFDeclContext(const DWARFDIE &die);
 
-  virtual lldb::TypeSP
-  FindCompleteObjCDefinitionTypeForDIE(const DWARFDIE &die,
-                                       lldb_private::ConstString type_name,
-                                       bool must_be_implementation);
+  virtual lldb::TypeSP FindCompleteObjCDefinitionTypeForDIE(
+      const DWARFDIE &die, ConstString type_name, bool must_be_implementation);
 
-  lldb_private::Symbol *
-  GetObjCClassSymbol(lldb_private::ConstString objc_class_name);
+  Symbol *GetObjCClassSymbol(ConstString objc_class_name);
 
   lldb::TypeSP GetTypeForDIE(const DWARFDIE &die,
                              bool resolve_function_context = false);
@@ -470,12 +439,11 @@ class SymbolFileDWARF : public lldb_private::SymbolFileCommon {
 
   bool DIEDeclContextsMatch(const DWARFDIE &die1, const DWARFDIE &die2);
 
-  bool ClassContainsSelector(const DWARFDIE &class_die,
-                             lldb_private::ConstString selector);
+  bool ClassContainsSelector(const DWARFDIE &class_die, ConstString selector);
 
   /// Parse call site entries (DW_TAG_call_site), including any nested call site
   /// parameters (DW_TAG_call_site_parameter).
-  std::vector<std::unique_ptr<lldb_private::CallEdge>>
+  std::vector<std::unique_ptr<CallEdge>>
   CollectCallEdges(lldb::ModuleSP module, DWARFDIE function_die);
 
   /// If this symbol file is linked to by a debug map (see
@@ -485,16 +453,15 @@ class SymbolFileDWARF : public lldb_private::SymbolFileCommon {
   /// needed, on success and LLDB_INVALID_ADDRESS otherwise.
   lldb::addr_t FixupAddress(lldb::addr_t file_addr);
 
-  bool FixupAddress(lldb_private::Address &addr);
+  bool FixupAddress(Address &addr);
 
-  typedef llvm::SetVector<lldb_private::Type *> TypeSet;
+  typedef llvm::SetVector<Type *> TypeSet;
 
   void GetTypes(const DWARFDIE &die, dw_offset_t min_die_offset,
                 dw_offset_t max_die_offset, uint32_t type_mask,
                 TypeSet &type_set);
 
-  typedef lldb_private::RangeDataVector<lldb::addr_t, lldb::addr_t,
-                                        lldb_private::Variable *>
+  typedef RangeDataVector<lldb::addr_t, lldb::addr_t, Variable *>
       GlobalVariableMap;
 
   GlobalVariableMap &GetGlobalAranges();
@@ -518,15 +485,14 @@ class SymbolFileDWARF : public lldb_private::SymbolFileCommon {
 
   void FindDwpSymbolFile();
 
-  const lldb_private::FileSpecList &GetTypeUnitSupportFiles(DWARFTypeUnit &tu);
+  const FileSpecList &GetTypeUnitSupportFiles(DWARFTypeUnit &tu);
 
-  void InitializeFirstCodeAddressRecursive(
-      const lldb_private::SectionList &section_list);
+  void InitializeFirstCodeAddressRecursive(const SectionList &section_list);
 
   void InitializeFirstCodeAddress();
 
-  void GetCompileOptions(
-      std::unordered_map<lldb::CompUnitSP, lldb_private::Args> &args) override;
+  void
+  GetCompileOptions(std::unordered_map<lldb::CompUnitSP, Args> &args) override;
 
   lldb::ModuleWP m_debug_map_module_wp;
   SymbolFileDWARFDebugMap *m_debug_map_symfile;
@@ -534,7 +500,7 @@ class SymbolFileDWARF : public lldb_private::SymbolFileCommon {
   llvm::once_flag m_dwp_symfile_once_flag;
   std::shared_ptr<SymbolFileDWARFDwo> m_dwp_symfile;
 
-  lldb_private::DWARFContext m_context;
+  DWARFContext m_context;
 
   llvm::once_flag m_info_once_flag;
   std::unique_ptr<DWARFDebugInfo> m_info;
@@ -542,14 +508,13 @@ class SymbolFileDWARF : public lldb_private::SymbolFileCommon {
   std::unique_ptr<llvm::DWARFDebugAbbrev> m_abbr;
   std::unique_ptr<GlobalVariableMap> m_global_aranges_up;
 
-  typedef std::unordered_map<lldb::offset_t, lldb_private::DebugMacrosSP>
-      DebugMacrosMap;
+  typedef std::unordered_map<lldb::offset_t, DebugMacrosSP> DebugMacrosMap;
   DebugMacrosMap m_debug_macros_map;
 
   ExternalTypeModuleMap m_external_type_modules;
-  std::unique_ptr<lldb_private::DWARFIndex> m_index;
+  std::unique_ptr<DWARFIndex> m_index;
   bool m_fetched_external_modules : 1;
-  lldb_private::LazyBool m_supports_DW_AT_APPLE_objc_complete_type;
+  LazyBool m_supports_DW_AT_APPLE_objc_complete_type;
 
   typedef std::set<DIERef> DIERefSet;
   typedef llvm::StringMap<DIERefSet> NameToOffsetMap;
@@ -560,8 +525,7 @@ class SymbolFileDWARF : public lldb_private::SymbolFileCommon {
   DIEToVariableSP m_die_to_variable_sp;
   DIEToClangType m_forward_decl_die_to_clang_type;
   ClangTypeToDIE m_forward_decl_clang_type_to_die;
-  llvm::DenseMap<dw_offset_t, lldb_private::FileSpecList>
-      m_type_unit_support_files;
+  llvm::DenseMap<dw_offset_t, FileSpecList> m_type_unit_support_files;
   std::vector<uint32_t> m_lldb_cu_to_dwarf_unit;
   /// DWARF does not provide a good way for traditional (concatenating) linkers
   /// to invalidate debug info describing dead-stripped code. These linkers will
@@ -570,7 +534,7 @@ class SymbolFileDWARF : public lldb_private::SymbolFileCommon {
   /// Try to filter out this debug info by comparing it to the lowest code
   /// address in the module.
   lldb::addr_t m_first_code_address = LLDB_INVALID_ADDRESS;
-  lldb_private::StatsDuration m_parse_time;
+  StatsDuration m_parse_time;
   std::atomic_flag m_dwo_warning_issued = ATOMIC_FLAG_INIT;
   /// If this DWARF file a .DWO file or a DWARF .o file on mac when
   /// no dSYM file is being used, this file index will be set to a
@@ -578,5 +542,7 @@ class SymbolFileDWARF : public lldb_private::SymbolFileCommon {
   /// an index that identifies the .DWO or .o file.
   std::optional<uint64_t> m_file_index;
 };
+} // namespace dwarf
+} // namespace lldb_private::plugin
 
 #endif // LLDB_SOURCE_PLUGINS_SYMBOLFILE_DWARF_SYMBOLFILEDWARF_H
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.cpp
index eadedd32e1..f789cbac9a 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.cpp
@@ -18,8 +18,9 @@
 #include "lldb/Host/FileSystem.h"
 #include "lldb/Utility/RangeMap.h"
 #include "lldb/Utility/RegularExpression.h"
-#include "lldb/Utility/Timer.h"
 #include "lldb/Utility/StreamString.h"
+#include "lldb/Utility/StructuredData.h"
+#include "lldb/Utility/Timer.h"
 
 //#define DEBUG_OSO_DMAP // DO NOT CHECKIN WITH THIS NOT COMMENTED OUT
 
@@ -42,6 +43,7 @@
 
 using namespace lldb;
 using namespace lldb_private;
+using namespace lldb_private::plugin::dwarf;
 
 char SymbolFileDWARFDebugMap::ID;
 
@@ -166,6 +168,8 @@ SymbolFileDWARFDebugMap::CompileUnitInfo::GetFileRangeMap(
   return file_range_map;
 }
 
+namespace lldb_private::plugin {
+namespace dwarf {
 class DebugMapModule : public Module {
 public:
   DebugMapModule(const ModuleSP &exe_module_sp, uint32_t cu_idx,
@@ -222,6 +226,8 @@ class DebugMapModule : public Module {
   ModuleWP m_exe_module_wp;
   const uint32_t m_cu_idx;
 };
+} // namespace dwarf
+} // namespace lldb_private::plugin
 
 void SymbolFileDWARFDebugMap::Initialize() {
   PluginManager::RegisterPlugin(GetPluginNameStatic(),
@@ -1271,6 +1277,42 @@ void SymbolFileDWARFDebugMap::DumpClangAST(Stream &s) {
   });
 }
 
+bool SymbolFileDWARFDebugMap::GetSeparateDebugInfo(
+    lldb_private::StructuredData::Dictionary &d) {
+  StructuredData::Array separate_debug_info_files;
+  const uint32_t cu_count = GetNumCompileUnits();
+  for (uint32_t cu_idx = 0; cu_idx < cu_count; ++cu_idx) {
+    const auto &info = m_compile_unit_infos[cu_idx];
+    StructuredData::DictionarySP oso_data =
+        std::make_shared<StructuredData::Dictionary>();
+    oso_data->AddStringItem("so_file", info.so_file.GetPath());
+    oso_data->AddStringItem("oso_path", info.oso_path);
+    oso_data->AddIntegerItem("oso_mod_time",
+                             (uint32_t)llvm::sys::toTimeT(info.oso_mod_time));
+
+    bool loaded_successfully = false;
+    if (GetModuleByOSOIndex(cu_idx)) {
+      // If we have a valid pointer to the module, we successfully
+      // loaded the oso if there are no load errors.
+      if (!info.oso_load_error.Fail()) {
+        loaded_successfully = true;
+      }
+    }
+    if (!loaded_successfully) {
+      oso_data->AddStringItem("error", info.oso_load_error.AsCString());
+    }
+    oso_data->AddBooleanItem("loaded", loaded_successfully);
+    separate_debug_info_files.AddItem(oso_data);
+  }
+
+  d.AddStringItem("type", "oso");
+  d.AddStringItem("symfile", GetMainObjectFile()->GetFileSpec().GetPath());
+  d.AddItem("separate-debug-info-files",
+            std::make_shared<StructuredData::Array>(
+                std::move(separate_debug_info_files)));
+  return true;
+}
+
 lldb::CompUnitSP
 SymbolFileDWARFDebugMap::GetCompileUnit(SymbolFileDWARF *oso_dwarf, DWARFCompileUnit &dwarf_cu) {
   if (oso_dwarf) {
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h
index 881fd4c45f..52fa1dca3d 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h
@@ -19,13 +19,18 @@
 #include <vector>
 
 #include "UniqueDWARFASTType.h"
+#include "lldb/Utility/StructuredData.h"
 
+class DWARFASTParserClang;
+
+namespace lldb_private::plugin {
+namespace dwarf {
 class SymbolFileDWARF;
 class DWARFCompileUnit;
 class DWARFDebugAranges;
 class DWARFDeclContext;
 
-class SymbolFileDWARFDebugMap : public lldb_private::SymbolFileCommon {
+class SymbolFileDWARFDebugMap : public SymbolFileCommon {
   /// LLVM RTTI support.
   static char ID;
 
@@ -47,8 +52,7 @@ class SymbolFileDWARFDebugMap : public lldb_private::SymbolFileCommon {
 
   static llvm::StringRef GetPluginDescriptionStatic();
 
-  static lldb_private::SymbolFile *
-  CreateInstance(lldb::ObjectFileSP objfile_sp);
+  static SymbolFile *CreateInstance(lldb::ObjectFileSP objfile_sp);
 
   // Constructors and Destructors
   SymbolFileDWARFDebugMap(lldb::ObjectFileSP objfile_sp);
@@ -58,110 +62,94 @@ class SymbolFileDWARFDebugMap : public lldb_private::SymbolFileCommon {
   void InitializeObject() override;
 
   // Compile Unit function calls
-  lldb::LanguageType
-  ParseLanguage(lldb_private::CompileUnit &comp_unit) override;
-  lldb_private::XcodeSDK
-  ParseXcodeSDK(lldb_private::CompileUnit &comp_unit) override;
+  lldb::LanguageType ParseLanguage(CompileUnit &comp_unit) override;
+  XcodeSDK ParseXcodeSDK(CompileUnit &comp_unit) override;
   llvm::SmallSet<lldb::LanguageType, 4>
-  ParseAllLanguages(lldb_private::CompileUnit &comp_unit) override;
-  size_t ParseFunctions(lldb_private::CompileUnit &comp_unit) override;
-  bool ParseLineTable(lldb_private::CompileUnit &comp_unit) override;
-  bool ParseDebugMacros(lldb_private::CompileUnit &comp_unit) override;
-
-  bool ForEachExternalModule(
-      lldb_private::CompileUnit &, llvm::DenseSet<lldb_private::SymbolFile *> &,
-      llvm::function_ref<bool(lldb_private::Module &)>) override;
-
-  bool ParseSupportFiles(lldb_private::CompileUnit &comp_unit,
-                         lldb_private::FileSpecList &support_files) override;
-
-  bool ParseIsOptimized(lldb_private::CompileUnit &comp_unit) override;
-
-  size_t ParseTypes(lldb_private::CompileUnit &comp_unit) override;
-
-  bool ParseImportedModules(
-      const lldb_private::SymbolContext &sc,
-      std::vector<lldb_private::SourceModule> &imported_modules) override;
-  size_t ParseBlocksRecursive(lldb_private::Function &func) override;
-  size_t
-  ParseVariablesForContext(const lldb_private::SymbolContext &sc) override;
-
-  lldb_private::Type *ResolveTypeUID(lldb::user_id_t type_uid) override;
-  std::optional<ArrayInfo> GetDynamicArrayInfoForUID(
-      lldb::user_id_t type_uid,
-      const lldb_private::ExecutionContext *exe_ctx) override;
-
-  lldb_private::CompilerDeclContext
-  GetDeclContextForUID(lldb::user_id_t uid) override;
-  lldb_private::CompilerDeclContext
-  GetDeclContextContainingUID(lldb::user_id_t uid) override;
-  void
-  ParseDeclsForContext(lldb_private::CompilerDeclContext decl_ctx) override;
+  ParseAllLanguages(CompileUnit &comp_unit) override;
+  size_t ParseFunctions(CompileUnit &comp_unit) override;
+  bool ParseLineTable(CompileUnit &comp_unit) override;
+  bool ParseDebugMacros(CompileUnit &comp_unit) override;
+
+  bool ForEachExternalModule(CompileUnit &, llvm::DenseSet<SymbolFile *> &,
+                             llvm::function_ref<bool(Module &)>) override;
+
+  bool ParseSupportFiles(CompileUnit &comp_unit,
+                         FileSpecList &support_files) override;
+
+  bool ParseIsOptimized(CompileUnit &comp_unit) override;
+
+  size_t ParseTypes(CompileUnit &comp_unit) override;
+
+  bool
+  ParseImportedModules(const SymbolContext &sc,
+                       std::vector<SourceModule> &imported_modules) override;
+  size_t ParseBlocksRecursive(Function &func) override;
+  size_t ParseVariablesForContext(const SymbolContext &sc) override;
+
+  Type *ResolveTypeUID(lldb::user_id_t type_uid) override;
+  std::optional<ArrayInfo>
+  GetDynamicArrayInfoForUID(lldb::user_id_t type_uid,
+                            const ExecutionContext *exe_ctx) override;
 
-  bool CompleteType(lldb_private::CompilerType &compiler_type) override;
-  uint32_t ResolveSymbolContext(const lldb_private::Address &so_addr,
+  CompilerDeclContext GetDeclContextForUID(lldb::user_id_t uid) override;
+  CompilerDeclContext GetDeclContextContainingUID(lldb::user_id_t uid) override;
+  void ParseDeclsForContext(CompilerDeclContext decl_ctx) override;
+
+  bool CompleteType(CompilerType &compiler_type) override;
+  uint32_t ResolveSymbolContext(const Address &so_addr,
+                                lldb::SymbolContextItem resolve_scope,
+                                SymbolContext &sc) override;
+  uint32_t ResolveSymbolContext(const SourceLocationSpec &src_location_spec,
                                 lldb::SymbolContextItem resolve_scope,
-                                lldb_private::SymbolContext &sc) override;
-  uint32_t ResolveSymbolContext(
-      const lldb_private::SourceLocationSpec &src_location_spec,
-      lldb::SymbolContextItem resolve_scope,
-      lldb_private::SymbolContextList &sc_list) override;
+                                SymbolContextList &sc_list) override;
 
-  lldb_private::Status
-  CalculateFrameVariableError(lldb_private::StackFrame &frame) override;
+  Status CalculateFrameVariableError(StackFrame &frame) override;
 
-  void
-  FindGlobalVariables(lldb_private::ConstString name,
-                      const lldb_private::CompilerDeclContext &parent_decl_ctx,
-                      uint32_t max_matches,
-                      lldb_private::VariableList &variables) override;
-  void FindGlobalVariables(const lldb_private::RegularExpression &regex,
+  void FindGlobalVariables(ConstString name,
+                           const CompilerDeclContext &parent_decl_ctx,
                            uint32_t max_matches,
-                           lldb_private::VariableList &variables) override;
-  void FindFunctions(const lldb_private::Module::LookupInfo &lookup_info,
-                     const lldb_private::CompilerDeclContext &parent_decl_ctx,
-                     bool include_inlines,
-                     lldb_private::SymbolContextList &sc_list) override;
-  void FindFunctions(const lldb_private::RegularExpression &regex,
-                     bool include_inlines,
-                     lldb_private::SymbolContextList &sc_list) override;
-  void
-  FindTypes(lldb_private::ConstString name,
-            const lldb_private::CompilerDeclContext &parent_decl_ctx,
-            uint32_t max_matches,
-            llvm::DenseSet<lldb_private::SymbolFile *> &searched_symbol_files,
-            lldb_private::TypeMap &types) override;
-  void
-  FindTypes(llvm::ArrayRef<lldb_private::CompilerContext> context,
-            lldb_private::LanguageSet languages,
-            llvm::DenseSet<lldb_private::SymbolFile *> &searched_symbol_files,
-            lldb_private::TypeMap &types) override;
-  lldb_private::CompilerDeclContext
-  FindNamespace(lldb_private::ConstString name,
-                const lldb_private::CompilerDeclContext &parent_decl_ctx,
-                bool only_root_namespaces) override;
-  void GetTypes(lldb_private::SymbolContextScope *sc_scope,
-                lldb::TypeClass type_mask,
-                lldb_private::TypeList &type_list) override;
-  std::vector<std::unique_ptr<lldb_private::CallEdge>>
-  ParseCallEdgesInFunction(lldb_private::UserID func_id) override;
-
-  void DumpClangAST(lldb_private::Stream &s) override;
+                           VariableList &variables) override;
+  void FindGlobalVariables(const RegularExpression &regex, uint32_t max_matches,
+                           VariableList &variables) override;
+  void FindFunctions(const Module::LookupInfo &lookup_info,
+                     const CompilerDeclContext &parent_decl_ctx,
+                     bool include_inlines, SymbolContextList &sc_list) override;
+  void FindFunctions(const RegularExpression &regex, bool include_inlines,
+                     SymbolContextList &sc_list) override;
+  void FindTypes(ConstString name, const CompilerDeclContext &parent_decl_ctx,
+                 uint32_t max_matches,
+                 llvm::DenseSet<SymbolFile *> &searched_symbol_files,
+                 TypeMap &types) override;
+  void FindTypes(llvm::ArrayRef<CompilerContext> context, LanguageSet languages,
+                 llvm::DenseSet<SymbolFile *> &searched_symbol_files,
+                 TypeMap &types) override;
+  CompilerDeclContext FindNamespace(ConstString name,
+                                    const CompilerDeclContext &parent_decl_ctx,
+                                    bool only_root_namespaces) override;
+  void GetTypes(SymbolContextScope *sc_scope, lldb::TypeClass type_mask,
+                TypeList &type_list) override;
+  std::vector<std::unique_ptr<CallEdge>>
+  ParseCallEdgesInFunction(UserID func_id) override;
+
+  void DumpClangAST(Stream &s) override;
+
+  /// List separate oso files.
+  bool GetSeparateDebugInfo(StructuredData::Dictionary &d) override;
 
   // PluginInterface protocol
   llvm::StringRef GetPluginName() override { return GetPluginNameStatic(); }
 
   // Statistics overrides.
-  lldb_private::ModuleList GetDebugInfoModules() override;
+  ModuleList GetDebugInfoModules() override;
 
-  void GetCompileOptions(
-      std::unordered_map<lldb::CompUnitSP, lldb_private::Args> &args) override;
+  void
+  GetCompileOptions(std::unordered_map<lldb::CompUnitSP, Args> &args) override;
 
 protected:
   enum { kHaveInitializedOSOs = (1 << 0), kNumFlags };
 
   friend class DebugMapModule;
-  friend class DWARFASTParserClang;
+  friend class ::DWARFASTParserClang;
   friend class DWARFCompileUnit;
   friend class SymbolFileDWARF;
   struct OSOInfo {
@@ -172,16 +160,15 @@ class SymbolFileDWARFDebugMap : public lldb_private::SymbolFileCommon {
 
   typedef std::shared_ptr<OSOInfo> OSOInfoSP;
 
-  typedef lldb_private::RangeDataVector<lldb::addr_t, lldb::addr_t,
-                                        lldb::addr_t>
+  typedef RangeDataVector<lldb::addr_t, lldb::addr_t, lldb::addr_t>
       FileRangeMap;
 
   // Class specific types
   struct CompileUnitInfo {
-    lldb_private::FileSpec so_file;
-    lldb_private::ConstString oso_path;
+    FileSpec so_file;
+    ConstString oso_path;
     llvm::sys::TimePoint<> oso_mod_time;
-    lldb_private::Status oso_load_error;
+    Status oso_load_error;
     OSOInfoSP oso_sp;
     /// The compile units that an object file contains.
     llvm::SmallVector<lldb::CompUnitSP, 2> compile_units_sps;
@@ -223,28 +210,26 @@ class SymbolFileDWARFDebugMap : public lldb_private::SymbolFileCommon {
 
   static SymbolFileDWARF *GetSymbolFileAsSymbolFileDWARF(SymbolFile *sym_file);
 
-  bool GetFileSpecForSO(uint32_t oso_idx, lldb_private::FileSpec &file_spec);
+  bool GetFileSpecForSO(uint32_t oso_idx, FileSpec &file_spec);
 
-  CompileUnitInfo *GetCompUnitInfo(const lldb_private::SymbolContext &sc);
-  CompileUnitInfo *GetCompUnitInfo(const lldb_private::CompileUnit &comp_unit);
+  CompileUnitInfo *GetCompUnitInfo(const SymbolContext &sc);
+  CompileUnitInfo *GetCompUnitInfo(const CompileUnit &comp_unit);
 
-  size_t GetCompUnitInfosForModule(const lldb_private::Module *oso_module,
+  size_t GetCompUnitInfosForModule(const Module *oso_module,
                                    std::vector<CompileUnitInfo *> &cu_infos);
 
-  lldb_private::Module *
-  GetModuleByCompUnitInfo(CompileUnitInfo *comp_unit_info);
+  Module *GetModuleByCompUnitInfo(CompileUnitInfo *comp_unit_info);
 
-  lldb_private::Module *GetModuleByOSOIndex(uint32_t oso_idx);
+  Module *GetModuleByOSOIndex(uint32_t oso_idx);
 
-  lldb_private::ObjectFile *
-  GetObjectFileByCompUnitInfo(CompileUnitInfo *comp_unit_info);
+  ObjectFile *GetObjectFileByCompUnitInfo(CompileUnitInfo *comp_unit_info);
 
-  lldb_private::ObjectFile *GetObjectFileByOSOIndex(uint32_t oso_idx);
+  ObjectFile *GetObjectFileByOSOIndex(uint32_t oso_idx);
 
   uint32_t GetCompUnitInfoIndex(const CompileUnitInfo *comp_unit_info);
 
-  SymbolFileDWARF *GetSymbolFile(const lldb_private::SymbolContext &sc);
-  SymbolFileDWARF *GetSymbolFile(const lldb_private::CompileUnit &comp_unit);
+  SymbolFileDWARF *GetSymbolFile(const SymbolContext &sc);
+  SymbolFileDWARF *GetSymbolFile(const CompileUnit &comp_unit);
 
   SymbolFileDWARF *GetSymbolFileByCompUnitInfo(CompileUnitInfo *comp_unit_info);
 
@@ -275,11 +260,11 @@ class SymbolFileDWARFDebugMap : public lldb_private::SymbolFileCommon {
   static int SymbolContainsSymbolWithID(lldb::user_id_t *symbol_idx_ptr,
                                         const CompileUnitInfo *comp_unit_info);
 
-  void PrivateFindGlobalVariables(
-      lldb_private::ConstString name,
-      const lldb_private::CompilerDeclContext &parent_decl_ctx,
-      const std::vector<uint32_t> &name_symbol_indexes, uint32_t max_matches,
-      lldb_private::VariableList &variables);
+  void
+  PrivateFindGlobalVariables(ConstString name,
+                             const CompilerDeclContext &parent_decl_ctx,
+                             const std::vector<uint32_t> &name_symbol_indexes,
+                             uint32_t max_matches, VariableList &variables);
 
   void SetCompileUnit(SymbolFileDWARF *oso_dwarf,
                       const lldb::CompUnitSP &cu_sp);
@@ -297,8 +282,7 @@ class SymbolFileDWARFDebugMap : public lldb_private::SymbolFileCommon {
   bool Supports_DW_AT_APPLE_objc_complete_type(SymbolFileDWARF *skip_dwarf_oso);
 
   lldb::TypeSP FindCompleteObjCDefinitionTypeForDIE(
-      const DWARFDIE &die, lldb_private::ConstString type_name,
-      bool must_be_implementation);
+      const DWARFDIE &die, ConstString type_name, bool must_be_implementation);
 
   UniqueDWARFASTTypeMap &GetUniqueDWARFASTTypeMap() {
     return m_unique_ast_type_map;
@@ -329,19 +313,16 @@ class SymbolFileDWARFDebugMap : public lldb_private::SymbolFileCommon {
     lldb::addr_t m_oso_file_addr = LLDB_INVALID_ADDRESS;
   };
 
-  typedef lldb_private::RangeDataVector<lldb::addr_t, lldb::addr_t, OSOEntry>
-      DebugMap;
+  typedef RangeDataVector<lldb::addr_t, lldb::addr_t, OSOEntry> DebugMap;
 
   // Member Variables
   std::bitset<kNumFlags> m_flags;
   std::vector<CompileUnitInfo> m_compile_unit_infos;
   std::vector<uint32_t> m_func_indexes; // Sorted by address
   std::vector<uint32_t> m_glob_indexes;
-  std::map<std::pair<lldb_private::ConstString, llvm::sys::TimePoint<>>,
-           OSOInfoSP>
-      m_oso_map;
+  std::map<std::pair<ConstString, llvm::sys::TimePoint<>>, OSOInfoSP> m_oso_map;
   UniqueDWARFASTTypeMap m_unique_ast_type_map;
-  lldb_private::LazyBool m_supports_DW_AT_APPLE_objc_complete_type;
+  LazyBool m_supports_DW_AT_APPLE_objc_complete_type;
   DebugMap m_debug_map;
 
   // When an object file from the debug map gets parsed in
@@ -365,7 +346,7 @@ class SymbolFileDWARFDebugMap : public lldb_private::SymbolFileCommon {
   /// \return
   ///     Returns true if \a addr was converted to be an executable
   ///     section/offset address, false otherwise.
-  bool LinkOSOAddress(lldb_private::Address &addr);
+  bool LinkOSOAddress(Address &addr);
 
   /// Convert a .o file "file address" to an executable "file address".
   ///
@@ -396,12 +377,13 @@ class SymbolFileDWARFDebugMap : public lldb_private::SymbolFileCommon {
   ///     Returns a valid line table full of linked addresses, or NULL
   ///     if none of the line table addresses exist in the main
   ///     executable.
-  lldb_private::LineTable *
-  LinkOSOLineTable(SymbolFileDWARF *oso_symfile,
-                   lldb_private::LineTable *line_table);
+  LineTable *LinkOSOLineTable(SymbolFileDWARF *oso_symfile,
+                              LineTable *line_table);
 
   size_t AddOSOARanges(SymbolFileDWARF *dwarf2Data,
                        DWARFDebugAranges *debug_aranges);
 };
+} // namespace dwarf
+} // namespace lldb_private::plugin
 
 #endif // LLDB_SOURCE_PLUGINS_SYMBOLFILE_DWARF_SYMBOLFILEDWARFDEBUGMAP_H
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.cpp
index 78c3c19684..60313ca3a0 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.cpp
@@ -21,6 +21,7 @@
 
 using namespace lldb;
 using namespace lldb_private;
+using namespace lldb_private::plugin::dwarf;
 
 char SymbolFileDWARFDwo::ID;
 
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.h b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.h
index e98ea49d93..8408264c34 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.h
@@ -12,6 +12,8 @@
 #include "SymbolFileDWARF.h"
 #include <optional>
 
+namespace lldb_private::plugin {
+namespace dwarf {
 class SymbolFileDWARFDwo : public SymbolFileDWARF {
   /// LLVM RTTI support.
   static char ID;
@@ -32,7 +34,7 @@ class SymbolFileDWARFDwo : public SymbolFileDWARF {
 
   DWARFCompileUnit *GetDWOCompileUnitForHash(uint64_t hash);
 
-  void GetObjCMethods(lldb_private::ConstString class_name,
+  void GetObjCMethods(ConstString class_name,
                       llvm::function_ref<bool(DWARFDIE die)> callback) override;
 
   llvm::Expected<lldb::TypeSystemSP>
@@ -41,15 +43,13 @@ class SymbolFileDWARFDwo : public SymbolFileDWARF {
   DWARFDIE
   GetDIE(const DIERef &die_ref) override;
 
-  lldb::offset_t
-  GetVendorDWARFOpcodeSize(const lldb_private::DataExtractor &data,
-                           const lldb::offset_t data_offset,
-                           const uint8_t op) const override;
+  lldb::offset_t GetVendorDWARFOpcodeSize(const DataExtractor &data,
+                                          const lldb::offset_t data_offset,
+                                          const uint8_t op) const override;
 
-  bool ParseVendorDWARFOpcode(
-      uint8_t op, const lldb_private::DataExtractor &opcodes,
-      lldb::offset_t &offset,
-      std::vector<lldb_private::Value> &stack) const override;
+  bool ParseVendorDWARFOpcode(uint8_t op, const DataExtractor &opcodes,
+                              lldb::offset_t &offset,
+                              std::vector<Value> &stack) const override;
 
 protected:
   DIEToTypePtr &GetDIEToType() override;
@@ -65,9 +65,10 @@ class SymbolFileDWARFDwo : public SymbolFileDWARF {
   lldb::TypeSP
   FindDefinitionTypeForDWARFDeclContext(const DWARFDIE &die) override;
 
-  lldb::TypeSP FindCompleteObjCDefinitionTypeForDIE(
-      const DWARFDIE &die, lldb_private::ConstString type_name,
-      bool must_be_implementation) override;
+  lldb::TypeSP
+  FindCompleteObjCDefinitionTypeForDIE(const DWARFDIE &die,
+                                       ConstString type_name,
+                                       bool must_be_implementation) override;
 
   SymbolFileDWARF &GetBaseSymbolFile() const { return m_base_symbol_file; }
 
@@ -77,5 +78,7 @@ class SymbolFileDWARFDwo : public SymbolFileDWARF {
 
   SymbolFileDWARF &m_base_symbol_file;
 };
+} // namespace dwarf
+} // namespace lldb_private::plugin
 
 #endif // LLDB_SOURCE_PLUGINS_SYMBOLFILE_DWARF_SYMBOLFILEDWARFDWO_H
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.cpp b/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.cpp
index 22a921cf61..223518f0ae 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.cpp
@@ -11,6 +11,7 @@
 #include "lldb/Core/Declaration.h"
 
 using namespace lldb_private::dwarf;
+using namespace lldb_private::plugin::dwarf;
 
 bool UniqueDWARFASTTypeList::Find(const DWARFDIE &die,
                                   const lldb_private::Declaration &decl,
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.h b/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.h
index 0947d1e581..bf3cbae55e 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.h
@@ -16,13 +16,15 @@
 #include "DWARFDIE.h"
 #include "lldb/Core/Declaration.h"
 
+namespace lldb_private::plugin {
+namespace dwarf {
 class UniqueDWARFASTType {
 public:
   // Constructors and Destructors
   UniqueDWARFASTType() : m_type_sp(), m_die(), m_declaration() {}
 
   UniqueDWARFASTType(lldb::TypeSP &type_sp, const DWARFDIE &die,
-                     const lldb_private::Declaration &decl, int32_t byte_size)
+                     const Declaration &decl, int32_t byte_size)
       : m_type_sp(type_sp), m_die(die), m_declaration(decl),
         m_byte_size(byte_size) {}
 
@@ -44,7 +46,7 @@ class UniqueDWARFASTType {
 
   lldb::TypeSP m_type_sp;
   DWARFDIE m_die;
-  lldb_private::Declaration m_declaration;
+  Declaration m_declaration;
   int32_t m_byte_size = -1;
 };
 
@@ -60,7 +62,7 @@ class UniqueDWARFASTTypeList {
     m_collection.push_back(entry);
   }
 
-  bool Find(const DWARFDIE &die, const lldb_private::Declaration &decl,
+  bool Find(const DWARFDIE &die, const Declaration &decl,
             const int32_t byte_size, UniqueDWARFASTType &entry) const;
 
 protected:
@@ -74,14 +76,12 @@ class UniqueDWARFASTTypeMap {
 
   ~UniqueDWARFASTTypeMap() = default;
 
-  void Insert(lldb_private::ConstString name,
-              const UniqueDWARFASTType &entry) {
+  void Insert(ConstString name, const UniqueDWARFASTType &entry) {
     m_collection[name.GetCString()].Append(entry);
   }
 
-  bool Find(lldb_private::ConstString name, const DWARFDIE &die,
-            const lldb_private::Declaration &decl, const int32_t byte_size,
-            UniqueDWARFASTType &entry) const {
+  bool Find(ConstString name, const DWARFDIE &die, const Declaration &decl,
+            const int32_t byte_size, UniqueDWARFASTType &entry) const {
     const char *unique_name_cstr = name.GetCString();
     collection::const_iterator pos = m_collection.find(unique_name_cstr);
     if (pos != m_collection.end()) {
@@ -95,5 +95,7 @@ class UniqueDWARFASTTypeMap {
   typedef llvm::DenseMap<const char *, UniqueDWARFASTTypeList> collection;
   collection m_collection;
 };
+} // namespace dwarf
+} // namespace lldb_private::plugin
 
 #endif // LLDB_SOURCE_PLUGINS_SYMBOLFILE_DWARF_UNIQUEDWARFASTTYPE_H
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
index 69cff0f35a..f1353db263 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
@@ -86,6 +86,7 @@
 using namespace lldb;
 using namespace lldb_private;
 using namespace lldb_private::dwarf;
+using namespace lldb_private::plugin::dwarf;
 using namespace clang;
 using llvm::StringSwitch;
 
@@ -2636,6 +2637,13 @@ TypeSystemClang::GetDeclContextForType(const CompilerType &type) {
   return GetDeclContextForType(ClangUtil::GetQualType(type));
 }
 
+CompilerDeclContext
+TypeSystemClang::GetCompilerDeclContextForType(const CompilerType &type) {
+  if (auto *decl_context = GetDeclContextForType(type))
+    return CreateDeclContext(decl_context);
+  return CompilerDeclContext();
+}
+
 /// Aggressively desugar the provided type, skipping past various kinds of
 /// syntactic sugar and other constructs one typically wants to ignore.
 /// The \p mask argument allows one to skip certain kinds of simplifications,
@@ -8512,380 +8520,6 @@ void TypeSystemClang::DumpFromSymbolFile(Stream &s,
   }
 }
 
-void TypeSystemClang::DumpValue(
-    lldb::opaque_compiler_type_t type, ExecutionContext *exe_ctx, Stream &s,
-    lldb::Format format, const lldb_private::DataExtractor &data,
-    lldb::offset_t data_byte_offset, size_t data_byte_size,
-    uint32_t bitfield_bit_size, uint32_t bitfield_bit_offset, bool show_types,
-    bool show_summary, bool verbose, uint32_t depth) {
-  if (!type)
-    return;
-
-  clang::QualType qual_type(GetQualType(type));
-  switch (qual_type->getTypeClass()) {
-  case clang::Type::Record:
-    if (GetCompleteType(type)) {
-      const clang::RecordType *record_type =
-          llvm::cast<clang::RecordType>(qual_type.getTypePtr());
-      const clang::RecordDecl *record_decl = record_type->getDecl();
-      assert(record_decl);
-      uint32_t field_bit_offset = 0;
-      uint32_t field_byte_offset = 0;
-      const clang::ASTRecordLayout &record_layout =
-          getASTContext().getASTRecordLayout(record_decl);
-      uint32_t child_idx = 0;
-
-      const clang::CXXRecordDecl *cxx_record_decl =
-          llvm::dyn_cast<clang::CXXRecordDecl>(record_decl);
-      if (cxx_record_decl) {
-        // We might have base classes to print out first
-        clang::CXXRecordDecl::base_class_const_iterator base_class,
-            base_class_end;
-        for (base_class = cxx_record_decl->bases_begin(),
-            base_class_end = cxx_record_decl->bases_end();
-             base_class != base_class_end; ++base_class) {
-          const clang::CXXRecordDecl *base_class_decl =
-              llvm::cast<clang::CXXRecordDecl>(
-                  base_class->getType()->getAs<clang::RecordType>()->getDecl());
-
-          // Skip empty base classes
-          if (!verbose && !TypeSystemClang::RecordHasFields(base_class_decl))
-            continue;
-
-          if (base_class->isVirtual())
-            field_bit_offset =
-                record_layout.getVBaseClassOffset(base_class_decl)
-                    .getQuantity() *
-                8;
-          else
-            field_bit_offset = record_layout.getBaseClassOffset(base_class_decl)
-                                   .getQuantity() *
-                               8;
-          field_byte_offset = field_bit_offset / 8;
-          assert(field_bit_offset % 8 == 0);
-          if (child_idx == 0)
-            s.PutChar('{');
-          else
-            s.PutChar(',');
-
-          clang::QualType base_class_qual_type = base_class->getType();
-          std::string base_class_type_name(base_class_qual_type.getAsString());
-
-          // Indent and print the base class type name
-          s.Format("\n{0}{1}", llvm::fmt_repeat(" ", depth + DEPTH_INCREMENT),
-                    base_class_type_name);
-
-          clang::TypeInfo base_class_type_info =
-              getASTContext().getTypeInfo(base_class_qual_type);
-
-          // Dump the value of the member
-          CompilerType base_clang_type = GetType(base_class_qual_type);
-          base_clang_type.DumpValue(
-              exe_ctx,
-              &s, // Stream to dump to
-              base_clang_type
-                  .GetFormat(), // The format with which to display the member
-              data, // Data buffer containing all bytes for this type
-              data_byte_offset + field_byte_offset, // Offset into "data" where
-                                                    // to grab value from
-              base_class_type_info.Width / 8, // Size of this type in bytes
-              0,                              // Bitfield bit size
-              0,                              // Bitfield bit offset
-              show_types,   // Boolean indicating if we should show the variable
-                            // types
-              show_summary, // Boolean indicating if we should show a summary
-                            // for the current type
-              verbose,      // Verbose output?
-              depth + DEPTH_INCREMENT); // Scope depth for any types that have
-                                        // children
-
-          ++child_idx;
-        }
-      }
-      uint32_t field_idx = 0;
-      clang::RecordDecl::field_iterator field, field_end;
-      for (field = record_decl->field_begin(),
-          field_end = record_decl->field_end();
-           field != field_end; ++field, ++field_idx, ++child_idx) {
-        // Print the starting squiggly bracket (if this is the first member) or
-        // comma (for member 2 and beyond) for the struct/union/class member.
-        if (child_idx == 0)
-          s.PutChar('{');
-        else
-          s.PutChar(',');
-
-        // Indent
-        s.Printf("\n%*s", depth + DEPTH_INCREMENT, "");
-
-        clang::QualType field_type = field->getType();
-        // Print the member type if requested
-        // Figure out the type byte size (field_type_info.first) and alignment
-        // (field_type_info.second) from the AST context.
-        clang::TypeInfo field_type_info =
-            getASTContext().getTypeInfo(field_type);
-        assert(field_idx < record_layout.getFieldCount());
-        // Figure out the field offset within the current struct/union/class
-        // type
-        field_bit_offset = record_layout.getFieldOffset(field_idx);
-        field_byte_offset = field_bit_offset / 8;
-        uint32_t field_bitfield_bit_size = 0;
-        uint32_t field_bitfield_bit_offset = 0;
-        if (FieldIsBitfield(*field, field_bitfield_bit_size))
-          field_bitfield_bit_offset = field_bit_offset % 8;
-
-        if (show_types) {
-          std::string field_type_name(field_type.getAsString());
-          if (field_bitfield_bit_size > 0)
-            s.Printf("(%s:%u) ", field_type_name.c_str(),
-                     field_bitfield_bit_size);
-          else
-            s.Printf("(%s) ", field_type_name.c_str());
-        }
-        // Print the member name and equal sign
-        s.Printf("%s = ", field->getNameAsString().c_str());
-
-        // Dump the value of the member
-        CompilerType field_clang_type = GetType(field_type);
-        field_clang_type.DumpValue(
-            exe_ctx,
-            &s, // Stream to dump to
-            field_clang_type
-                .GetFormat(), // The format with which to display the member
-            data,             // Data buffer containing all bytes for this type
-            data_byte_offset + field_byte_offset, // Offset into "data" where to
-                                                  // grab value from
-            field_type_info.Width / 8,            // Size of this type in bytes
-            field_bitfield_bit_size,              // Bitfield bit size
-            field_bitfield_bit_offset,            // Bitfield bit offset
-            show_types,   // Boolean indicating if we should show the variable
-                          // types
-            show_summary, // Boolean indicating if we should show a summary for
-                          // the current type
-            verbose,      // Verbose output?
-            depth + DEPTH_INCREMENT); // Scope depth for any types that have
-                                      // children
-      }
-
-      // Indent the trailing squiggly bracket
-      if (child_idx > 0)
-        s.Printf("\n%*s}", depth, "");
-    }
-    return;
-
-  case clang::Type::Enum:
-    if (GetCompleteType(type)) {
-      const clang::EnumType *enutype =
-          llvm::cast<clang::EnumType>(qual_type.getTypePtr());
-      const clang::EnumDecl *enum_decl = enutype->getDecl();
-      assert(enum_decl);
-      clang::EnumDecl::enumerator_iterator enum_pos, enum_end_pos;
-      lldb::offset_t offset = data_byte_offset;
-      const int64_t enum_value = data.GetMaxU64Bitfield(
-          &offset, data_byte_size, bitfield_bit_size, bitfield_bit_offset);
-      for (enum_pos = enum_decl->enumerator_begin(),
-          enum_end_pos = enum_decl->enumerator_end();
-           enum_pos != enum_end_pos; ++enum_pos) {
-        if (enum_pos->getInitVal() == enum_value) {
-          s.Printf("%s", enum_pos->getNameAsString().c_str());
-          return;
-        }
-      }
-      // If we have gotten here we didn't get find the enumerator in the enum
-      // decl, so just print the integer.
-      s.Printf("%" PRIi64, enum_value);
-    }
-    return;
-
-  case clang::Type::ConstantArray: {
-    const clang::ConstantArrayType *array =
-        llvm::cast<clang::ConstantArrayType>(qual_type.getTypePtr());
-    bool is_array_of_characters = false;
-    clang::QualType element_qual_type = array->getElementType();
-
-    const clang::Type *canonical_type =
-        element_qual_type->getCanonicalTypeInternal().getTypePtr();
-    if (canonical_type)
-      is_array_of_characters = canonical_type->isCharType();
-
-    const uint64_t element_count = array->getSize().getLimitedValue();
-
-    clang::TypeInfo field_type_info =
-        getASTContext().getTypeInfo(element_qual_type);
-
-    uint32_t element_idx = 0;
-    uint32_t element_offset = 0;
-    uint64_t element_byte_size = field_type_info.Width / 8;
-    uint32_t element_stride = element_byte_size;
-
-    if (is_array_of_characters) {
-      s.PutChar('"');
-      DumpDataExtractor(data, &s, data_byte_offset, lldb::eFormatChar,
-                        element_byte_size, element_count, UINT32_MAX,
-                        LLDB_INVALID_ADDRESS, 0, 0);
-      s.PutChar('"');
-      return;
-    } else {
-      CompilerType element_clang_type = GetType(element_qual_type);
-      lldb::Format element_format = element_clang_type.GetFormat();
-
-      for (element_idx = 0; element_idx < element_count; ++element_idx) {
-        // Print the starting squiggly bracket (if this is the first member) or
-        // comman (for member 2 and beyong) for the struct/union/class member.
-        if (element_idx == 0)
-          s.PutChar('{');
-        else
-          s.PutChar(',');
-
-        // Indent and print the index
-        s.Printf("\n%*s[%u] ", depth + DEPTH_INCREMENT, "", element_idx);
-
-        // Figure out the field offset within the current struct/union/class
-        // type
-        element_offset = element_idx * element_stride;
-
-        // Dump the value of the member
-        element_clang_type.DumpValue(
-            exe_ctx,
-            &s,             // Stream to dump to
-            element_format, // The format with which to display the element
-            data,           // Data buffer containing all bytes for this type
-            data_byte_offset +
-                element_offset, // Offset into "data" where to grab value from
-            element_byte_size,  // Size of this type in bytes
-            0,                  // Bitfield bit size
-            0,                  // Bitfield bit offset
-            show_types,   // Boolean indicating if we should show the variable
-                          // types
-            show_summary, // Boolean indicating if we should show a summary for
-                          // the current type
-            verbose,      // Verbose output?
-            depth + DEPTH_INCREMENT); // Scope depth for any types that have
-                                      // children
-      }
-
-      // Indent the trailing squiggly bracket
-      if (element_idx > 0)
-        s.Printf("\n%*s}", depth, "");
-    }
-  }
-    return;
-
-  case clang::Type::Typedef: {
-    clang::QualType typedef_qual_type =
-        llvm::cast<clang::TypedefType>(qual_type)
-            ->getDecl()
-            ->getUnderlyingType();
-
-    CompilerType typedef_clang_type = GetType(typedef_qual_type);
-    lldb::Format typedef_format = typedef_clang_type.GetFormat();
-    clang::TypeInfo typedef_type_info =
-        getASTContext().getTypeInfo(typedef_qual_type);
-    uint64_t typedef_byte_size = typedef_type_info.Width / 8;
-
-    return typedef_clang_type.DumpValue(
-        exe_ctx,
-        &s,                  // Stream to dump to
-        typedef_format,      // The format with which to display the element
-        data,                // Data buffer containing all bytes for this type
-        data_byte_offset,    // Offset into "data" where to grab value from
-        typedef_byte_size,   // Size of this type in bytes
-        bitfield_bit_size,   // Bitfield bit size
-        bitfield_bit_offset, // Bitfield bit offset
-        show_types,   // Boolean indicating if we should show the variable types
-        show_summary, // Boolean indicating if we should show a summary for the
-                      // current type
-        verbose,      // Verbose output?
-        depth);       // Scope depth for any types that have children
-  } break;
-
-  case clang::Type::Auto: {
-    clang::QualType elaborated_qual_type =
-        llvm::cast<clang::AutoType>(qual_type)->getDeducedType();
-    CompilerType elaborated_clang_type = GetType(elaborated_qual_type);
-    lldb::Format elaborated_format = elaborated_clang_type.GetFormat();
-    clang::TypeInfo elaborated_type_info =
-        getASTContext().getTypeInfo(elaborated_qual_type);
-    uint64_t elaborated_byte_size = elaborated_type_info.Width / 8;
-
-    return elaborated_clang_type.DumpValue(
-        exe_ctx,
-        &s,                   // Stream to dump to
-        elaborated_format,    // The format with which to display the element
-        data,                 // Data buffer containing all bytes for this type
-        data_byte_offset,     // Offset into "data" where to grab value from
-        elaborated_byte_size, // Size of this type in bytes
-        bitfield_bit_size,    // Bitfield bit size
-        bitfield_bit_offset,  // Bitfield bit offset
-        show_types,   // Boolean indicating if we should show the variable types
-        show_summary, // Boolean indicating if we should show a summary for the
-                      // current type
-        verbose,      // Verbose output?
-        depth);       // Scope depth for any types that have children
-  } break;
-
-  case clang::Type::Elaborated: {
-    clang::QualType elaborated_qual_type =
-        llvm::cast<clang::ElaboratedType>(qual_type)->getNamedType();
-    CompilerType elaborated_clang_type = GetType(elaborated_qual_type);
-    lldb::Format elaborated_format = elaborated_clang_type.GetFormat();
-    clang::TypeInfo elaborated_type_info =
-        getASTContext().getTypeInfo(elaborated_qual_type);
-    uint64_t elaborated_byte_size = elaborated_type_info.Width / 8;
-
-    return elaborated_clang_type.DumpValue(
-        exe_ctx,
-        &s,                   // Stream to dump to
-        elaborated_format,    // The format with which to display the element
-        data,                 // Data buffer containing all bytes for this type
-        data_byte_offset,     // Offset into "data" where to grab value from
-        elaborated_byte_size, // Size of this type in bytes
-        bitfield_bit_size,    // Bitfield bit size
-        bitfield_bit_offset,  // Bitfield bit offset
-        show_types,   // Boolean indicating if we should show the variable types
-        show_summary, // Boolean indicating if we should show a summary for the
-                      // current type
-        verbose,      // Verbose output?
-        depth);       // Scope depth for any types that have children
-  } break;
-
-  case clang::Type::Paren: {
-    clang::QualType desugar_qual_type =
-        llvm::cast<clang::ParenType>(qual_type)->desugar();
-    CompilerType desugar_clang_type = GetType(desugar_qual_type);
-
-    lldb::Format desugar_format = desugar_clang_type.GetFormat();
-    clang::TypeInfo desugar_type_info =
-        getASTContext().getTypeInfo(desugar_qual_type);
-    uint64_t desugar_byte_size = desugar_type_info.Width / 8;
-
-    return desugar_clang_type.DumpValue(
-        exe_ctx,
-        &s,                  // Stream to dump to
-        desugar_format,      // The format with which to display the element
-        data,                // Data buffer containing all bytes for this type
-        data_byte_offset,    // Offset into "data" where to grab value from
-        desugar_byte_size,   // Size of this type in bytes
-        bitfield_bit_size,   // Bitfield bit size
-        bitfield_bit_offset, // Bitfield bit offset
-        show_types,   // Boolean indicating if we should show the variable types
-        show_summary, // Boolean indicating if we should show a summary for the
-                      // current type
-        verbose,      // Verbose output?
-        depth);       // Scope depth for any types that have children
-  } break;
-
-  default:
-    // We are down to a scalar type that we just need to display.
-    DumpDataExtractor(data, &s, data_byte_offset, format, data_byte_size, 1,
-                      UINT32_MAX, LLDB_INVALID_ADDRESS, bitfield_bit_size,
-                      bitfield_bit_offset);
-
-    if (show_summary)
-      DumpSummary(type, exe_ctx, s, data, data_byte_offset, data_byte_size);
-    break;
-  }
-}
-
 static bool DumpEnumValue(const clang::QualType &qual_type, Stream &s,
                           const DataExtractor &data, lldb::offset_t byte_offset,
                           size_t byte_size, uint32_t bitfield_bit_offset,
@@ -9091,51 +8725,6 @@ bool TypeSystemClang::DumpTypeValue(
   return false;
 }
 
-void TypeSystemClang::DumpSummary(lldb::opaque_compiler_type_t type,
-                                  ExecutionContext *exe_ctx, Stream &s,
-                                  const lldb_private::DataExtractor &data,
-                                  lldb::offset_t data_byte_offset,
-                                  size_t data_byte_size) {
-  uint32_t length = 0;
-  if (IsCStringType(type, length)) {
-    if (exe_ctx) {
-      Process *process = exe_ctx->GetProcessPtr();
-      if (process) {
-        lldb::offset_t offset = data_byte_offset;
-        lldb::addr_t pointer_address = data.GetMaxU64(&offset, data_byte_size);
-        std::vector<uint8_t> buf;
-        if (length > 0)
-          buf.resize(length);
-        else
-          buf.resize(256);
-
-        DataExtractor cstr_data(&buf.front(), buf.size(),
-                                process->GetByteOrder(), 4);
-        buf.back() = '\0';
-        size_t bytes_read;
-        size_t total_cstr_len = 0;
-        Status error;
-        while ((bytes_read = process->ReadMemory(pointer_address, &buf.front(),
-                                                 buf.size(), error)) > 0) {
-          const size_t len = strlen((const char *)&buf.front());
-          if (len == 0)
-            break;
-          if (total_cstr_len == 0)
-            s.PutCString(" \"");
-          DumpDataExtractor(cstr_data, &s, 0, lldb::eFormatChar, 1, len,
-                            UINT32_MAX, LLDB_INVALID_ADDRESS, 0, 0);
-          total_cstr_len += len;
-          if (len < buf.size())
-            break;
-          pointer_address += total_cstr_len;
-        }
-        if (total_cstr_len > 0)
-          s.PutChar('"');
-      }
-    }
-  }
-}
-
 void TypeSystemClang::DumpTypeDescription(lldb::opaque_compiler_type_t type,
                                           lldb::DescriptionLevel level) {
   StreamFile s(stdout, false);
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
index 0544de3cd3..66e59ec985 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
@@ -219,6 +219,9 @@ class TypeSystemClang : public TypeSystem {
 
   static clang::DeclContext *GetDeclContextForType(const CompilerType &type);
 
+  CompilerDeclContext
+  GetCompilerDeclContextForType(const CompilerType &type) override;
+
   uint32_t GetPointerByteSize() override;
 
   clang::TranslationUnitDecl *GetTranslationUnitDecl() {
@@ -514,7 +517,7 @@ class TypeSystemClang : public TypeSystem {
                                               size_t bit_size);
 
   // TypeSystem methods
-  DWARFASTParser *GetDWARFParser() override;
+  plugin::dwarf::DWARFASTParser *GetDWARFParser() override;
   PDBASTParser *GetPDBParser() override;
   npdb::PdbAstBuilder *GetNativePDBParser() override;
 
@@ -632,8 +635,7 @@ class TypeSystemClang : public TypeSystem {
 
   bool IsConst(lldb::opaque_compiler_type_t type) override;
 
-  bool IsCStringType(lldb::opaque_compiler_type_t type,
-                     uint32_t &length) override;
+  bool IsCStringType(lldb::opaque_compiler_type_t type, uint32_t &length);
 
   static bool IsCXXClassType(const CompilerType &type);
 
@@ -1029,23 +1031,12 @@ class TypeSystemClang : public TypeSystem {
   ///       The name of the symbol to dump, if it is empty dump all the symbols
   void DumpFromSymbolFile(Stream &s, llvm::StringRef symbol_name);
 
-  void DumpValue(lldb::opaque_compiler_type_t type, ExecutionContext *exe_ctx,
-                 Stream &s, lldb::Format format, const DataExtractor &data,
-                 lldb::offset_t data_offset, size_t data_byte_size,
-                 uint32_t bitfield_bit_size, uint32_t bitfield_bit_offset,
-                 bool show_types, bool show_summary, bool verbose,
-                 uint32_t depth) override;
-
   bool DumpTypeValue(lldb::opaque_compiler_type_t type, Stream &s,
                      lldb::Format format, const DataExtractor &data,
                      lldb::offset_t data_offset, size_t data_byte_size,
                      uint32_t bitfield_bit_size, uint32_t bitfield_bit_offset,
                      ExecutionContextScope *exe_scope) override;
 
-  void DumpSummary(lldb::opaque_compiler_type_t type, ExecutionContext *exe_ctx,
-                   Stream &s, const DataExtractor &data,
-                   lldb::offset_t data_offset, size_t data_byte_size) override;
-
   void DumpTypeDescription(
       lldb::opaque_compiler_type_t type,
       lldb::DescriptionLevel level = lldb::eDescriptionLevelFull) override;
diff --git a/lldb/source/Symbol/CompileUnit.cpp b/lldb/source/Symbol/CompileUnit.cpp
index 280425d587..c979697394 100644
--- a/lldb/source/Symbol/CompileUnit.cpp
+++ b/lldb/source/Symbol/CompileUnit.cpp
@@ -178,11 +178,7 @@ void CompileUnit::SetLineTable(LineTable *line_table) {
   m_line_table_up.reset(line_table);
 }
 
-void CompileUnit::SetSupportFiles(const FileSpecList &support_files) {
-  m_support_files = support_files;
-}
-
-void CompileUnit::SetSupportFiles(FileSpecList &&support_files) {
+void CompileUnit::SetSupportFiles(FileSpecList support_files) {
   m_support_files = std::move(support_files);
 }
 
diff --git a/lldb/source/Symbol/CompilerType.cpp b/lldb/source/Symbol/CompilerType.cpp
index 33f7e63d9b..78cc8dad94 100644
--- a/lldb/source/Symbol/CompilerType.cpp
+++ b/lldb/source/Symbol/CompilerType.cpp
@@ -108,13 +108,6 @@ bool CompilerType::IsConst() const {
   return false;
 }
 
-bool CompilerType::IsCStringType(uint32_t &length) const {
-  if (IsValid())
-    if (auto type_system_sp = GetTypeSystem())
-      return type_system_sp->IsCStringType(m_type, length);
-  return false;
-}
-
 bool CompilerType::IsFunctionType() const {
   if (IsValid())
     if (auto type_system_sp = GetTypeSystem())
@@ -821,20 +814,6 @@ CompilerType::GetIndexOfChildWithName(llvm::StringRef name,
 
 // Dumping types
 
-void CompilerType::DumpValue(ExecutionContext *exe_ctx, Stream *s,
-                             lldb::Format format, const DataExtractor &data,
-                             lldb::offset_t data_byte_offset,
-                             size_t data_byte_size, uint32_t bitfield_bit_size,
-                             uint32_t bitfield_bit_offset, bool show_types,
-                             bool show_summary, bool verbose, uint32_t depth) {
-  if (!IsValid())
-    if (auto type_system_sp = GetTypeSystem())
-      type_system_sp->DumpValue(m_type, exe_ctx, *s, format, data,
-                                data_byte_offset, data_byte_size,
-                                bitfield_bit_size, bitfield_bit_offset,
-                                show_types, show_summary, verbose, depth);
-}
-
 bool CompilerType::DumpTypeValue(Stream *s, lldb::Format format,
                                  const DataExtractor &data,
                                  lldb::offset_t byte_offset, size_t byte_size,
@@ -849,16 +828,6 @@ bool CompilerType::DumpTypeValue(Stream *s, lldb::Format format,
   return false;
 }
 
-void CompilerType::DumpSummary(ExecutionContext *exe_ctx, Stream *s,
-                               const DataExtractor &data,
-                               lldb::offset_t data_byte_offset,
-                               size_t data_byte_size) {
-  if (IsValid())
-    if (auto type_system_sp = GetTypeSystem())
-      type_system_sp->DumpSummary(m_type, exe_ctx, *s, data, data_byte_offset,
-                                  data_byte_size);
-}
-
 void CompilerType::DumpTypeDescription(lldb::DescriptionLevel level) const {
   if (IsValid())
     if (auto type_system_sp = GetTypeSystem())
@@ -982,6 +951,18 @@ bool CompilerType::GetValueAsScalar(const lldb_private::DataExtractor &data,
   return false;
 }
 
+CompilerType::CompilerType(CompilerType::TypeSystemSPWrapper type_system,
+                           lldb::opaque_compiler_type_t type)
+    : m_type_system(type_system.GetSharedPointer()), m_type(type) {
+  assert(Verify() && "verification failed");
+}
+
+CompilerType::CompilerType(lldb::TypeSystemWP type_system,
+                           lldb::opaque_compiler_type_t type)
+    : m_type_system(type_system), m_type(type) {
+  assert(Verify() && "verification failed");
+}
+
 #ifndef NDEBUG
 bool CompilerType::Verify() const {
   if (!IsValid())
diff --git a/lldb/source/Symbol/SymbolFile.cpp b/lldb/source/Symbol/SymbolFile.cpp
index b271efd07b..7dcee8ced0 100644
--- a/lldb/source/Symbol/SymbolFile.cpp
+++ b/lldb/source/Symbol/SymbolFile.cpp
@@ -18,6 +18,7 @@
 #include "lldb/Symbol/VariableList.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/StreamString.h"
+#include "lldb/Utility/StructuredData.h"
 #include "lldb/lldb-private.h"
 
 #include <future>
diff --git a/lldb/source/Symbol/Type.cpp b/lldb/source/Symbol/Type.cpp
index 66284eb73c..548300d570 100644
--- a/lldb/source/Symbol/Type.cpp
+++ b/lldb/source/Symbol/Type.cpp
@@ -312,30 +312,6 @@ ConstString Type::GetBaseName() {
 
 void Type::DumpTypeName(Stream *s) { GetName().Dump(s, "<invalid-type-name>"); }
 
-void Type::DumpValue(ExecutionContext *exe_ctx, Stream *s,
-                     const DataExtractor &data, uint32_t data_byte_offset,
-                     bool show_types, bool show_summary, bool verbose,
-                     lldb::Format format) {
-  if (ResolveCompilerType(ResolveState::Forward)) {
-    if (show_types) {
-      s->PutChar('(');
-      if (verbose)
-        s->Printf("Type{0x%8.8" PRIx64 "} ", GetID());
-      DumpTypeName(s);
-      s->PutCString(") ");
-    }
-
-    GetForwardCompilerType().DumpValue(
-        exe_ctx, s, format == lldb::eFormatDefault ? GetFormat() : format, data,
-        data_byte_offset,
-        GetByteSize(exe_ctx ? exe_ctx->GetBestExecutionContextScope() : nullptr)
-            .value_or(0),
-        0, // Bitfield bit size
-        0, // Bitfield bit offset
-        show_types, show_summary, verbose, 0);
-  }
-}
-
 Type *Type::GetEncodingType() {
   if (m_encoding_type == nullptr && m_encoding_uid != LLDB_INVALID_UID)
     m_encoding_type = m_symbol_file->ResolveTypeUID(m_encoding_uid);
@@ -416,24 +392,6 @@ lldb::Encoding Type::GetEncoding(uint64_t &count) {
   return GetForwardCompilerType().GetEncoding(count);
 }
 
-bool Type::DumpValueInMemory(ExecutionContext *exe_ctx, Stream *s,
-                             lldb::addr_t address, AddressType address_type,
-                             bool show_types, bool show_summary, bool verbose) {
-  if (address != LLDB_INVALID_ADDRESS) {
-    DataExtractor data;
-    Target *target = nullptr;
-    if (exe_ctx)
-      target = exe_ctx->GetTargetPtr();
-    if (target)
-      data.SetByteOrder(target->GetArchitecture().GetByteOrder());
-    if (ReadFromMemory(exe_ctx, address, address_type, data)) {
-      DumpValue(exe_ctx, s, data, 0, show_types, show_summary, verbose);
-      return true;
-    }
-  }
-  return false;
-}
-
 bool Type::ReadFromMemory(ExecutionContext *exe_ctx, lldb::addr_t addr,
                           AddressType address_type, DataExtractor &data) {
   if (address_type == eAddressTypeFile) {
@@ -1082,6 +1040,23 @@ bool TypeImpl::GetDescription(lldb_private::Stream &strm,
   return true;
 }
 
+CompilerType TypeImpl::FindDirectNestedType(llvm::StringRef name) {
+  if (name.empty())
+    return CompilerType();
+  auto type_system = GetTypeSystem(/*prefer_dynamic*/ false);
+  auto *symbol_file = type_system->GetSymbolFile();
+  auto decl_context = type_system->GetCompilerDeclContextForType(m_static_type);
+  if (!decl_context.IsValid())
+    return CompilerType();
+  llvm::DenseSet<lldb_private::SymbolFile *> searched_symbol_files;
+  TypeMap search_result;
+  symbol_file->FindTypes(ConstString(name), decl_context, /*max_matches*/ 1,
+                         searched_symbol_files, search_result);
+  if (search_result.Empty())
+    return CompilerType();
+  return search_result.GetTypeAtIndex(0)->GetFullCompilerType();
+}
+
 bool TypeMemberFunctionImpl::IsValid() {
   return m_type.IsValid() && m_kind != lldb::eMemberFunctionKindUnknown;
 }
diff --git a/lldb/source/Symbol/TypeSystem.cpp b/lldb/source/Symbol/TypeSystem.cpp
index 24f2029305..874f12573e 100644
--- a/lldb/source/Symbol/TypeSystem.cpp
+++ b/lldb/source/Symbol/TypeSystem.cpp
@@ -186,6 +186,11 @@ std::optional<llvm::json::Value> TypeSystem::ReportStatistics() {
   return std::nullopt;
 }
 
+CompilerDeclContext
+TypeSystem::GetCompilerDeclContextForType(const CompilerType &type) {
+  return CompilerDeclContext();
+}
+
 #pragma mark TypeSystemMap
 
 TypeSystemMap::TypeSystemMap() : m_mutex(), m_map() {}
diff --git a/lldb/source/Target/Target.cpp b/lldb/source/Target/Target.cpp
index 069b7bcdc4..5f8756c576 100644
--- a/lldb/source/Target/Target.cpp
+++ b/lldb/source/Target/Target.cpp
@@ -480,12 +480,12 @@ BreakpointSP Target::CreateBreakpoint(const Address &addr, bool internal,
 
 lldb::BreakpointSP
 Target::CreateAddressInModuleBreakpoint(lldb::addr_t file_addr, bool internal,
-                                        const FileSpec *file_spec,
+                                        const FileSpec &file_spec,
                                         bool request_hardware) {
   SearchFilterSP filter_sp(
       new SearchFilterForUnconstrainedSearches(shared_from_this()));
   BreakpointResolverSP resolver_sp(new BreakpointResolverAddress(
-      nullptr, file_addr, file_spec ? *file_spec : FileSpec()));
+      nullptr, file_addr, file_spec));
   return CreateBreakpoint(filter_sp, resolver_sp, internal, request_hardware,
                           false);
 }
diff --git a/lldb/source/Utility/Diagnostics.cpp b/lldb/source/Utility/Diagnostics.cpp
index 1632ae0f9d..b2a08165dd 100644
--- a/lldb/source/Utility/Diagnostics.cpp
+++ b/lldb/source/Utility/Diagnostics.cpp
@@ -52,10 +52,8 @@ Diagnostics::CallbackID Diagnostics::AddCallback(Callback callback) {
 
 void Diagnostics::RemoveCallback(CallbackID id) {
   std::lock_guard<std::mutex> guard(m_callbacks_mutex);
-  m_callbacks.erase(
-      std::remove_if(m_callbacks.begin(), m_callbacks.end(),
-                     [id](const CallbackEntry &e) { return e.id == id; }),
-      m_callbacks.end());
+  llvm::erase_if(m_callbacks,
+                 [id](const CallbackEntry &e) { return e.id == id; });
 }
 
 bool Diagnostics::Dump(raw_ostream &stream) {
diff --git a/lldb/source/Utility/FileSpecList.cpp b/lldb/source/Utility/FileSpecList.cpp
index e5e0ac3e59..d5369ac4bb 100644
--- a/lldb/source/Utility/FileSpecList.cpp
+++ b/lldb/source/Utility/FileSpecList.cpp
@@ -140,12 +140,6 @@ const FileSpec &FileSpecList::GetFileSpecAtIndex(size_t idx) const {
   return g_empty_file_spec;
 }
 
-const FileSpec *FileSpecList::GetFileSpecPointerAtIndex(size_t idx) const {
-  if (idx < m_files.size())
-    return &m_files[idx];
-  return nullptr;
-}
-
 // Return the size in bytes that this object takes in memory. This returns the
 // size in bytes of this object's member variables and any FileSpec objects its
 // member variables contain, the result doesn't not include the string values
@@ -162,9 +156,3 @@ size_t FileSpecList::MemorySize() const {
 
 // Return the number of files in the file spec list.
 size_t FileSpecList::GetSize() const { return m_files.size(); }
-
-size_t FileSpecList::GetFilesMatchingPartialPath(const char *path,
-                                                 bool dir_okay,
-                                                 FileSpecList &matches) {
-  return 0;
-}
diff --git a/lldb/source/Utility/Scalar.cpp b/lldb/source/Utility/Scalar.cpp
index 791c0fb743..5ad68065bc 100644
--- a/lldb/source/Utility/Scalar.cpp
+++ b/lldb/source/Utility/Scalar.cpp
@@ -153,20 +153,20 @@ bool Scalar::IsZero() const {
   return false;
 }
 
-void Scalar::GetValue(Stream *s, bool show_type) const {
+void Scalar::GetValue(Stream &s, bool show_type) const {
   if (show_type)
-    s->Printf("(%s) ", GetTypeAsCString());
+    s.Printf("(%s) ", GetTypeAsCString());
 
   switch (m_type) {
   case e_void:
     break;
   case e_int:
-    s->PutCString(llvm::toString(m_integer, 10));
+    s.PutCString(llvm::toString(m_integer, 10));
     break;
   case e_float:
     llvm::SmallString<24> string;
     m_float.toString(string);
-    s->PutCString(string);
+    s.PutCString(string);
     break;
   }
 }
@@ -894,6 +894,6 @@ bool Scalar::SetBit(uint32_t bit) {
 
 llvm::raw_ostream &lldb_private::operator<<(llvm::raw_ostream &os, const Scalar &scalar) {
   StreamString s;
-  scalar.GetValue(&s, /*show_type*/ true);
+  scalar.GetValue(s, /*show_type*/ true);
   return os << s.GetString();
 }
diff --git a/lldb/test/API/commands/expression/fixits/TestFixIts.py b/lldb/test/API/commands/expression/fixits/TestFixIts.py
index 3bdeb84b4e..38b242838c 100644
--- a/lldb/test/API/commands/expression/fixits/TestFixIts.py
+++ b/lldb/test/API/commands/expression/fixits/TestFixIts.py
@@ -22,7 +22,9 @@ def test_with_dummy_target(self):
         self.assertEqual(
             result, lldb.eReturnStatusSuccessFinishResult, ret_val.GetError()
         )
-        self.assertIn("Fix-it applied", ret_val.GetError())
+        self.assertIn(
+            "Evaluated this expression after applying Fix-It(s):", ret_val.GetError()
+        )
 
     def test_with_target(self):
         """Test calling expressions with errors that can be fixed by the FixIts."""
@@ -99,7 +101,9 @@ def test_with_target_error_applies_fixit(self):
         )
         self.assertEqual(result, lldb.eReturnStatusFailed, ret_val.GetError())
 
-        self.assertIn("Fix-it applied, fixed expression was:", ret_val.GetError())
+        self.assertIn(
+            "Evaluated this expression after applying Fix-It(s):", ret_val.GetError()
+        )
         self.assertIn("null_pointer->first", ret_val.GetError())
 
     # The final function call runs into SIGILL on aarch64-linux.
diff --git a/lldb/test/API/commands/expression/import-std-module/queue/TestQueueFromStdModule.py b/lldb/test/API/commands/expression/import-std-module/queue/TestQueueFromStdModule.py
index 84e8e3cfb8..b08a53855e 100644
--- a/lldb/test/API/commands/expression/import-std-module/queue/TestQueueFromStdModule.py
+++ b/lldb/test/API/commands/expression/import-std-module/queue/TestQueueFromStdModule.py
@@ -10,6 +10,11 @@
 class TestQueue(TestBase):
     @add_test_categories(["libc++"])
     @skipIf(compiler=no_match("clang"))
+    @skipIf(
+        compiler="clang",
+        compiler_version=[">", "16.0"],
+        bugnumber="https://github.com/llvm/llvm-project/issues/68968",
+    )
     def test(self):
         self.build()
 
diff --git a/lldb/test/API/commands/target/dump-separate-debug-info/dwo/Makefile b/lldb/test/API/commands/target/dump-separate-debug-info/dwo/Makefile
new file mode 100644
index 0000000000..3b6d788b2b
--- /dev/null
+++ b/lldb/test/API/commands/target/dump-separate-debug-info/dwo/Makefile
@@ -0,0 +1,4 @@
+CXX_SOURCES := main.cpp foo.cpp
+CFLAGS_EXTRAS := -gsplit-dwarf
+
+include Makefile.rules
diff --git a/lldb/test/API/commands/target/dump-separate-debug-info/dwo/TestDumpDwo.py b/lldb/test/API/commands/target/dump-separate-debug-info/dwo/TestDumpDwo.py
new file mode 100644
index 0000000000..3d9d8e8e77
--- /dev/null
+++ b/lldb/test/API/commands/target/dump-separate-debug-info/dwo/TestDumpDwo.py
@@ -0,0 +1,126 @@
+"""
+Test 'target modules dump separate-debug-info' for dwo files.
+"""
+
+import json
+import os
+
+from lldbsuite.test import lldbtest, lldbutil
+from lldbsuite.test.decorators import *
+
+
+class TestDumpDWO(lldbtest.TestBase):
+    NO_DEBUG_INFO_TESTCASE = True
+
+    def get_dwos_from_json(self):
+        """Returns a dictionary of `symfile` -> {`dwo_name` -> dwo_info object}."""
+        result = {}
+        output = json.loads(self.res.GetOutput())
+        for symfile_entry in output:
+            dwo_dict = {}
+            for dwo_entry in symfile_entry["separate-debug-info-files"]:
+                dwo_dict[dwo_entry["dwo_name"]] = dwo_entry
+            result[symfile_entry["symfile"]] = dwo_dict
+        return result
+
+    @skipIfRemote
+    @skipIfDarwin
+    @skipIfWindows
+    def test_dwos_loaded_json_output(self):
+        self.build()
+        exe = self.getBuildArtifact("a.out")
+        main_dwo = self.getBuildArtifact("main.dwo")
+        foo_dwo = self.getBuildArtifact("foo.dwo")
+
+        # Make sure dwo files exist
+        self.assertTrue(os.path.exists(main_dwo), f'Make sure "{main_dwo}" file exists')
+        self.assertTrue(os.path.exists(foo_dwo), f'Make sure "{foo_dwo}" file exists')
+
+        target = self.dbg.CreateTarget(exe)
+        self.assertTrue(target, lldbtest.VALID_TARGET)
+
+        self.runCmd("target modules dump separate-debug-info --json")
+
+        # Check the output
+        output = self.get_dwos_from_json()
+        self.assertTrue(output[exe]["main.dwo"]["loaded"])
+        self.assertTrue(output[exe]["foo.dwo"]["loaded"])
+
+    @skipIfRemote
+    @skipIfDarwin
+    @skipIfWindows
+    def test_dwos_not_loaded_json_output(self):
+        self.build()
+        exe = self.getBuildArtifact("a.out")
+        main_dwo = self.getBuildArtifact("main.dwo")
+        foo_dwo = self.getBuildArtifact("foo.dwo")
+
+        # REMOVE the dwo files
+        os.unlink(main_dwo)
+        os.unlink(foo_dwo)
+
+        target = self.dbg.CreateTarget(exe)
+        self.assertTrue(target, lldbtest.VALID_TARGET)
+
+        self.runCmd("target modules dump separate-debug-info --json")
+
+        # Check the output
+        output = self.get_dwos_from_json()
+        self.assertFalse(output[exe]["main.dwo"]["loaded"])
+        self.assertFalse(output[exe]["foo.dwo"]["loaded"])
+        self.assertIn("error", output[exe]["main.dwo"])
+        self.assertIn("error", output[exe]["foo.dwo"])
+
+    @skipIfRemote
+    @skipIfDarwin
+    @skipIfWindows
+    def test_dwos_loaded_table_output(self):
+        self.build()
+        exe = self.getBuildArtifact("a.out")
+        main_dwo = self.getBuildArtifact("main.dwo")
+        foo_dwo = self.getBuildArtifact("foo.dwo")
+
+        # Make sure dwo files exist
+        self.assertTrue(os.path.exists(main_dwo), f'Make sure "{main_dwo}" file exists')
+        self.assertTrue(os.path.exists(foo_dwo), f'Make sure "{foo_dwo}" file exists')
+
+        target = self.dbg.CreateTarget(exe)
+        self.assertTrue(target, lldbtest.VALID_TARGET)
+
+        self.expect(
+            "target modules dump separate-debug-info",
+            patterns=[
+                "Symbol file: .*?a\.out",
+                'Type: "dwo"',
+                "Dwo ID\s+Err\s+Dwo Path",
+                "0x[a-zA-Z0-9]{16}\s+.*main\.dwo",
+                "0x[a-zA-Z0-9]{16}\s+.*foo\.dwo",
+            ],
+        )
+
+    @skipIfRemote
+    @skipIfDarwin
+    @skipIfWindows
+    def test_dwos_not_loaded_table_output(self):
+        self.build()
+        exe = self.getBuildArtifact("a.out")
+        main_dwo = self.getBuildArtifact("main.dwo")
+        foo_dwo = self.getBuildArtifact("foo.dwo")
+
+        # REMOVE the dwo files
+        os.unlink(main_dwo)
+        os.unlink(foo_dwo)
+
+        target = self.dbg.CreateTarget(exe)
+        self.assertTrue(target, lldbtest.VALID_TARGET)
+
+        self.expect(
+            "target modules dump separate-debug-info",
+            patterns=[
+                "Symbol file: .*?a\.out",
+                'Type: "dwo"',
+                "Dwo ID\s+Err\s+Dwo Path",
+                "0x[a-zA-Z0-9]{16}\s+E\s+.*main\.dwo",
+                "0x[a-zA-Z0-9]{16}\s+E\s+.*foo\.dwo",
+            ],
+        )
diff --git a/lldb/test/API/commands/target/dump-separate-debug-info/dwo/foo.cpp b/lldb/test/API/commands/target/dump-separate-debug-info/dwo/foo.cpp
new file mode 100644
index 0000000000..28e2b6e768
--- /dev/null
+++ b/lldb/test/API/commands/target/dump-separate-debug-info/dwo/foo.cpp
@@ -0,0 +1,3 @@
+#include "foo.h"
+
+int foo() { return 1; }
diff --git a/lldb/test/API/commands/target/dump-separate-debug-info/dwo/foo.h b/lldb/test/API/commands/target/dump-separate-debug-info/dwo/foo.h
new file mode 100644
index 0000000000..4ec598ad51
--- /dev/null
+++ b/lldb/test/API/commands/target/dump-separate-debug-info/dwo/foo.h
@@ -0,0 +1,6 @@
+#ifndef FOO_H
+#define FOO_H
+
+int foo();
+
+#endif
diff --git a/lldb/test/API/commands/target/dump-separate-debug-info/dwo/main.cpp b/lldb/test/API/commands/target/dump-separate-debug-info/dwo/main.cpp
new file mode 100644
index 0000000000..8087e68243
--- /dev/null
+++ b/lldb/test/API/commands/target/dump-separate-debug-info/dwo/main.cpp
@@ -0,0 +1,3 @@
+#include "foo.h"
+
+int main() { return foo(); }
diff --git a/lldb/test/API/tools/lldb-vscode/evaluate/Makefile b/lldb/test/API/commands/target/dump-separate-debug-info/oso/Makefile
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/evaluate/Makefile
rename to lldb/test/API/commands/target/dump-separate-debug-info/oso/Makefile
diff --git a/lldb/test/API/commands/target/dump-separate-debug-info/oso/TestDumpOso.py b/lldb/test/API/commands/target/dump-separate-debug-info/oso/TestDumpOso.py
new file mode 100644
index 0000000000..05beed0eac
--- /dev/null
+++ b/lldb/test/API/commands/target/dump-separate-debug-info/oso/TestDumpOso.py
@@ -0,0 +1,120 @@
+"""
+Test 'target modules dump separate-debug-info' for oso files.
+"""
+
+import json
+import os
+
+from lldbsuite.test import lldbtest, lldbutil
+from lldbsuite.test.decorators import *
+
+
+class TestDumpOso(lldbtest.TestBase):
+    NO_DEBUG_INFO_TESTCASE = True
+
+    def get_osos_from_json(self):
+        """Returns a dictionary of `symfile` -> {`OSO_PATH` -> oso_info object}."""
+        result = {}
+        output = json.loads(self.res.GetOutput())
+        for symfile_entry in output:
+            oso_dict = {}
+            for oso_entry in symfile_entry["separate-debug-info-files"]:
+                oso_dict[oso_entry["oso_path"]] = oso_entry
+            result[symfile_entry["symfile"]] = oso_dict
+        return result
+
+    @skipIfRemote
+    @skipUnlessDarwin
+    def test_shows_oso_loaded_json_output(self):
+        self.build(debug_info="dwarf")
+        exe = self.getBuildArtifact("a.out")
+        main_o = self.getBuildArtifact("main.o")
+        foo_o = self.getBuildArtifact("foo.o")
+
+        # Make sure o files exist
+        self.assertTrue(os.path.exists(main_o), f'Make sure "{main_o}" file exists')
+        self.assertTrue(os.path.exists(foo_o), f'Make sure "{foo_o}" file exists')
+
+        target = self.dbg.CreateTarget(exe)
+        self.assertTrue(target, lldbtest.VALID_TARGET)
+
+        self.runCmd("target modules dump separate-debug-info --json")
+
+        # Check the output
+        osos = self.get_osos_from_json()
+        self.assertTrue(osos[exe][main_o]["loaded"])
+        self.assertTrue(osos[exe][foo_o]["loaded"])
+
+    @skipIfRemote
+    @skipUnlessDarwin
+    def test_shows_oso_not_loaded_json_output(self):
+        self.build(debug_info="dwarf")
+        exe = self.getBuildArtifact("a.out")
+        main_o = self.getBuildArtifact("main.o")
+        foo_o = self.getBuildArtifact("foo.o")
+
+        # REMOVE the o files
+        os.unlink(main_o)
+        os.unlink(foo_o)
+
+        target = self.dbg.CreateTarget(exe)
+        self.assertTrue(target, lldbtest.VALID_TARGET)
+
+        self.runCmd("target modules dump separate-debug-info --json")
+
+        # Check the output
+        osos = self.get_osos_from_json()
+        self.assertFalse(osos[exe][main_o]["loaded"])
+        self.assertFalse(osos[exe][foo_o]["loaded"])
+
+    @skipIfRemote
+    @skipUnlessDarwin
+    def test_shows_oso_loaded_table_output(self):
+        self.build(debug_info="dwarf")
+        exe = self.getBuildArtifact("a.out")
+        main_o = self.getBuildArtifact("main.o")
+        foo_o = self.getBuildArtifact("foo.o")
+
+        # Make sure o files exist
+        self.assertTrue(os.path.exists(main_o), f'Make sure "{main_o}" file exists')
+        self.assertTrue(os.path.exists(foo_o), f'Make sure "{foo_o}" file exists')
+
+        target = self.dbg.CreateTarget(exe)
+        self.assertTrue(target, lldbtest.VALID_TARGET)
+
+        self.expect(
+            "target modules dump separate-debug-info",
+            patterns=[
+                "Symbol file: .*?a\.out",
+                'Type: "oso"',
+                "Mod Time\s+Err\s+Oso Path",
+                "0x[a-zA-Z0-9]{16}\s+.*main\.o",
+                "0x[a-zA-Z0-9]{16}\s+.*foo\.o",
+            ],
+        )
+
+    @skipIfRemote
+    @skipUnlessDarwin
+    def test_shows_oso_not_loaded_table_output(self):
+        self.build(debug_info="dwarf")
+        exe = self.getBuildArtifact("a.out")
+        main_o = self.getBuildArtifact("main.o")
+        foo_o = self.getBuildArtifact("foo.o")
+
+        # REMOVE the o files
+        os.unlink(main_o)
+        os.unlink(foo_o)
+
+        target = self.dbg.CreateTarget(exe)
+        self.assertTrue(target, lldbtest.VALID_TARGET)
+
+        self.expect(
+            "target modules dump separate-debug-info",
+            patterns=[
+                "Symbol file: .*?a\.out",
+                'Type: "oso"',
+                "Mod Time\s+Err\s+Oso Path",
+                "0x[a-zA-Z0-9]{16}\s+E\s+.*main\.o",
+                "0x[a-zA-Z0-9]{16}\s+E\s+.*foo\.o",
+            ],
+        )
diff --git a/lldb/test/API/commands/target/dump-separate-debug-info/oso/foo.cpp b/lldb/test/API/commands/target/dump-separate-debug-info/oso/foo.cpp
new file mode 100644
index 0000000000..28e2b6e768
--- /dev/null
+++ b/lldb/test/API/commands/target/dump-separate-debug-info/oso/foo.cpp
@@ -0,0 +1,3 @@
+#include "foo.h"
+
+int foo() { return 1; }
diff --git a/lldb/test/API/commands/target/dump-separate-debug-info/oso/foo.h b/lldb/test/API/commands/target/dump-separate-debug-info/oso/foo.h
new file mode 100644
index 0000000000..4ec598ad51
--- /dev/null
+++ b/lldb/test/API/commands/target/dump-separate-debug-info/oso/foo.h
@@ -0,0 +1,6 @@
+#ifndef FOO_H
+#define FOO_H
+
+int foo();
+
+#endif
diff --git a/lldb/test/API/commands/target/dump-separate-debug-info/oso/main.cpp b/lldb/test/API/commands/target/dump-separate-debug-info/oso/main.cpp
new file mode 100644
index 0000000000..8087e68243
--- /dev/null
+++ b/lldb/test/API/commands/target/dump-separate-debug-info/oso/main.cpp
@@ -0,0 +1,3 @@
+#include "foo.h"
+
+int main() { return foo(); }
diff --git a/lldb/test/API/functionalities/data-formatter/builtin-formats/TestBuiltinFormats.py b/lldb/test/API/functionalities/data-formatter/builtin-formats/TestBuiltinFormats.py
index aa768c158b..4e0f14d039 100644
--- a/lldb/test/API/functionalities/data-formatter/builtin-formats/TestBuiltinFormats.py
+++ b/lldb/test/API/functionalities/data-formatter/builtin-formats/TestBuiltinFormats.py
@@ -3,9 +3,9 @@
 """
 
 import lldb
+from lldbsuite.test import lldbutil
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
-from lldbsuite.test import lldbutil
 
 
 class TestCase(TestBase):
@@ -19,6 +19,20 @@ def getFormatted(self, format, expr):
         self.assertTrue(result.Succeeded(), result.GetError())
         return result.GetOutput()
 
+    @no_debug_info_test
+    @skipIfWindows
+    def testAllPlatforms(self):
+        self.build()
+        lldbutil.run_to_source_breakpoint(
+            self, "// break here", lldb.SBFileSpec("main.cpp")
+        )
+        # We can dump correctly non char* c-strings with explicit formatting.
+        self.assertIn(' = ""', self.getFormatted("c-string", "void_empty_cstring"))
+        self.assertIn(' = ""', self.getFormatted("c-string", "empty_cstring"))
+
+    # TODO: Move as many asserts as possible within this function to `testAllPlatforms`.
+    # Currently `arm` is being skipped even though many asserts would effectively
+    # pass.
     @no_debug_info_test
     @skipIfWindows
     # uint128_t not available on arm.
diff --git a/lldb/test/API/functionalities/data-formatter/builtin-formats/main.cpp b/lldb/test/API/functionalities/data-formatter/builtin-formats/main.cpp
index 58b8116dfa..573c111306 100644
--- a/lldb/test/API/functionalities/data-formatter/builtin-formats/main.cpp
+++ b/lldb/test/API/functionalities/data-formatter/builtin-formats/main.cpp
@@ -1,8 +1,10 @@
 #include <cstdint>
 
 const char cstring[15] = " \033\a\b\f\n\r\t\vaA09\0";
+const char *empty_cstring = "";
 
 int main() {
   int use = *cstring;
+  void *void_empty_cstring = (void *)empty_cstring;
   return use; // break here
 }
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCNSDate.py b/lldb/test/API/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCNSDate.py
index c4aec10505..a1ffe84ad5 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCNSDate.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCNSDate.py
@@ -53,12 +53,12 @@ def nsdate_data_formatter_commands(self):
 
         self.expect(
             "frame variable cupertino home europe",
-            substrs=['@"America/Los_Angeles"', '@"Europe/Rome"', '@"Europe/Paris"'],
+            substrs=['"America/Los_Angeles"', '"Europe/Rome"', '"Europe/Paris"'],
         )
 
         self.expect(
             "frame variable cupertino_ns home_ns europe_ns",
-            substrs=['@"America/Los_Angeles"', '@"Europe/Rome"', '@"Europe/Paris"'],
+            substrs=['"America/Los_Angeles"', '"Europe/Rome"', '"Europe/Paris"'],
         )
 
         self.expect(
diff --git a/lldb/test/API/functionalities/data-formatter/vector-types/TestVectorTypesFormatting.py b/lldb/test/API/functionalities/data-formatter/vector-types/TestVectorTypesFormatting.py
index 4103d62878..1839c28aeb 100644
--- a/lldb/test/API/functionalities/data-formatter/vector-types/TestVectorTypesFormatting.py
+++ b/lldb/test/API/functionalities/data-formatter/vector-types/TestVectorTypesFormatting.py
@@ -86,3 +86,10 @@ def cleanup():
         v.SetFormat(lldb.eFormatVectorOfFloat32)
         oldValueAgain = v.GetChildAtIndex(0).GetValue()
         self.assertEqual(oldValue, oldValueAgain, "same format but different values")
+
+        # Test formatter for vector types whose size is not a power-of-2
+        f3 = self.frame().FindVariable("f3")
+        self.assertEqual(f3.GetNumChildren(), 3)
+        self.assertEqual(f3.GetChildAtIndex(0).GetData().float[0], 1.25)
+        self.assertEqual(f3.GetChildAtIndex(1).GetData().float[0], 2.50)
+        self.assertEqual(f3.GetChildAtIndex(2).GetData().float[0], 2.50)
diff --git a/lldb/test/API/functionalities/data-formatter/vector-types/main.cpp b/lldb/test/API/functionalities/data-formatter/vector-types/main.cpp
index ef0a227560..7f2309e776 100644
--- a/lldb/test/API/functionalities/data-formatter/vector-types/main.cpp
+++ b/lldb/test/API/functionalities/data-formatter/vector-types/main.cpp
@@ -1,8 +1,10 @@
 typedef float float4 __attribute__((ext_vector_type(4)));
-typedef  unsigned char vec __attribute__((ext_vector_type(16)));
+typedef unsigned char vec __attribute__((ext_vector_type(16)));
+typedef float float3 __attribute__((ext_vector_type(3)));
 
 int main() {
     float4 f4 = {1.25, 1.25, 2.50, 2.50};
     vec v = (vec)f4;
+    float3 f3 = f4.gba;
     return 0; // break here
 }
diff --git a/lldb/test/API/tools/lldb-vscode/completions/Makefile b/lldb/test/API/lang/cpp/fixits/Makefile
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/completions/Makefile
rename to lldb/test/API/lang/cpp/fixits/Makefile
diff --git a/lldb/test/API/lang/cpp/fixits/TestCppFixIts.py b/lldb/test/API/lang/cpp/fixits/TestCppFixIts.py
new file mode 100644
index 0000000000..34b52f796d
--- /dev/null
+++ b/lldb/test/API/lang/cpp/fixits/TestCppFixIts.py
@@ -0,0 +1,44 @@
+"""
+Tests a C++ fixit for the `expr` command and
+`po` alias (aka DWIM aka "do what I mean") alias.
+"""
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+
+class TestCase(TestBase):
+    def test_fixit_with_dwim(self):
+        """Confirms `po` shows an expression after applying Fix-It(s)."""
+
+        self.build()
+        lldbutil.run_to_source_breakpoint(
+            self, "// break here", lldb.SBFileSpec("main.cpp")
+        )
+
+        self.expect(
+            "dwim-print -O -- class C { int i; void f() { []() { ++i; }(); } }; 42",
+            error=True,
+            substrs=[
+                "Evaluated this expression after applying Fix-It(s)",
+                "class C { int i; void f() { [this]() { ++i; }(); } }",
+            ],
+        )
+
+    def test_fixit_with_expression(self):
+        """Confirms `expression` shows an expression after applying Fix-It(s)."""
+
+        self.build()
+        lldbutil.run_to_source_breakpoint(
+            self, "// break here", lldb.SBFileSpec("main.cpp")
+        )
+
+        self.expect(
+            "expr class C { int i; void f() { []() { ++i; }(); } }; 42",
+            error=True,
+            substrs=[
+                "Evaluated this expression after applying Fix-It(s)",
+                "class C { int i; void f() { [this]() { ++i; }(); } }",
+            ],
+        )
diff --git a/lldb/test/API/lang/cpp/fixits/main.cpp b/lldb/test/API/lang/cpp/fixits/main.cpp
new file mode 100644
index 0000000000..e9cf11d18a
--- /dev/null
+++ b/lldb/test/API/lang/cpp/fixits/main.cpp
@@ -0,0 +1,5 @@
+int main() {
+  long foo = 1234;
+
+  return 0; // break here
+}
diff --git a/lldb/test/API/lit.cfg.py b/lldb/test/API/lit.cfg.py
index c306c5b3c0..8b35e8b2a2 100644
--- a/lldb/test/API/lit.cfg.py
+++ b/lldb/test/API/lit.cfg.py
@@ -248,7 +248,7 @@ def delete_module_cache(path):
     "lldb-repro-capture" in config.available_features
     or "lldb-repro-replay" in config.available_features
 ):
-    dotest_cmd += ["--skip-category=lldb-vscode", "--skip-category=std-module"]
+    dotest_cmd += ["--skip-category=lldb-dap", "--skip-category=std-module"]
 
 if "lldb-simulator-ios" in config.available_features:
     dotest_cmd += ["--apple-sdk", "iphonesimulator", "--platform-name", "ios-simulator"]
diff --git a/lldb/test/API/python_api/type/TestTypeList.py b/lldb/test/API/python_api/type/TestTypeList.py
index c2fcadc46e..c267defb58 100644
--- a/lldb/test/API/python_api/type/TestTypeList.py
+++ b/lldb/test/API/python_api/type/TestTypeList.py
@@ -119,6 +119,37 @@ def test(self):
 
         self.assertEqual(task_type, task_head_pointee_type)
 
+        # Check whether we can find a directly nested type by name
+        name_type = task_type.FindDirectNestedType("name")
+        self.assertTrue(name_type)
+        self.DebugSBType(name_type)
+
+        enum_type = task_type.FindDirectNestedType("E")
+        self.assertTrue(enum_type)
+        self.DebugSBType(enum_type)
+
+        union_type = task_type.FindDirectNestedType("U")
+        self.assertTrue(union_type)
+        self.DebugSBType(union_type)
+
+        # Check that we don't find indirectly nested types
+        self.assertTrue(enum_type.size == 1)
+
+        invalid_type = task_type.FindDirectNestedType("E2")
+        self.assertFalse(invalid_type)
+
+        # Check that FindDirectNestedType handles types without DeclContext
+        # and other errorneous inputs
+        task_ptr_type = task_type.GetPointerType()
+        invalid_type = task_ptr_type.FindDirectNestedType("name")
+        self.assertFalse(invalid_type)
+
+        invalid_type = task_type.FindDirectNestedType("")
+        self.assertFalse(invalid_type)
+
+        invalid_type = task_type.FindDirectNestedType(None)
+        self.assertFalse(invalid_type)
+
         # We'll now get the child member 'id' from 'task_head'.
         id = task_head.GetChildMemberWithName("id")
         self.DebugSBValue(id)
diff --git a/lldb/test/API/python_api/type/main.cpp b/lldb/test/API/python_api/type/main.cpp
index b1ef625283..98de9707d8 100644
--- a/lldb/test/API/python_api/type/main.cpp
+++ b/lldb/test/API/python_api/type/main.cpp
@@ -21,7 +21,12 @@ class Task {
     } my_type_is_nameless;
     struct name {
       int x;
+      enum E : int {} e;
+      enum E2 {} e2;
     } my_type_is_named;
+    enum E : unsigned char {} e;
+    union U {
+    } u;
     Task(int i, Task *n):
         id(i),
         next(n),
diff --git a/lldb/test/API/tools/lldb-vscode/attach/Makefile b/lldb/test/API/tools/lldb-dap/attach/Makefile
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/attach/Makefile
rename to lldb/test/API/tools/lldb-dap/attach/Makefile
diff --git a/lldb/test/API/tools/lldb-vscode/attach/TestVSCode_attach.py b/lldb/test/API/tools/lldb-dap/attach/TestDAP_attach.py
similarity index 96%
rename from lldb/test/API/tools/lldb-vscode/attach/TestVSCode_attach.py
rename to lldb/test/API/tools/lldb-dap/attach/TestDAP_attach.py
index a53dac77e3..04017a0521 100644
--- a/lldb/test/API/tools/lldb-vscode/attach/TestVSCode_attach.py
+++ b/lldb/test/API/tools/lldb-dap/attach/TestDAP_attach.py
@@ -1,13 +1,13 @@
 """
-Test lldb-vscode setBreakpoints request
+Test lldb-dap setBreakpoints request
 """
 
 
-import vscode
+import dap_server
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
-import lldbvscode_testcase
+import lldbdap_testcase
 import os
 import shutil
 import subprocess
@@ -25,7 +25,7 @@ def spawn_and_wait(program, delay):
     process.wait()
 
 
-class TestVSCode_attach(lldbvscode_testcase.VSCodeTestCaseBase):
+class TestDAP_attach(lldbdap_testcase.DAPTestCaseBase):
     def set_and_hit_breakpoint(self, continueToExit=True):
         source = "main.c"
         breakpoint1_line = line_number(source, "// breakpoint 1")
@@ -190,10 +190,10 @@ def test_commands(self):
         # Continue after launch and hit the "pause()" call and stop the target.
         # Get output from the console. This should contain both the
         # "stopCommands" that were run after we stop.
-        self.vscode.request_continue()
+        self.dap_server.request_continue()
         time.sleep(0.5)
-        self.vscode.request_pause()
-        self.vscode.wait_for_stopped()
+        self.dap_server.request_pause()
+        self.dap_server.wait_for_stopped()
         output = self.get_console(timeout=1.0)
         self.verify_commands("stopCommands", output, stopCommands)
 
@@ -236,6 +236,6 @@ def test_terminate_commands(self):
         self.get_console()
         # Once it's disconnected the console should contain the
         # "terminateCommands"
-        self.vscode.request_disconnect(terminateDebuggee=True)
+        self.dap_server.request_disconnect(terminateDebuggee=True)
         output = self.collect_console(duration=1.0)
         self.verify_commands("terminateCommands", output, terminateCommands)
diff --git a/lldb/test/API/tools/lldb-vscode/attach/main.c b/lldb/test/API/tools/lldb-dap/attach/main.c
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/attach/main.c
rename to lldb/test/API/tools/lldb-dap/attach/main.c
diff --git a/lldb/test/API/tools/lldb-vscode/breakpoint-events/Makefile b/lldb/test/API/tools/lldb-dap/breakpoint-events/Makefile
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/breakpoint-events/Makefile
rename to lldb/test/API/tools/lldb-dap/breakpoint-events/Makefile
diff --git a/lldb/test/API/tools/lldb-vscode/breakpoint-events/TestVSCode_breakpointEvents.py b/lldb/test/API/tools/lldb-dap/breakpoint-events/TestDAP_breakpointEvents.py
similarity index 83%
rename from lldb/test/API/tools/lldb-vscode/breakpoint-events/TestVSCode_breakpointEvents.py
rename to lldb/test/API/tools/lldb-dap/breakpoint-events/TestDAP_breakpointEvents.py
index 51996d03d9..a20384b75f 100644
--- a/lldb/test/API/tools/lldb-vscode/breakpoint-events/TestVSCode_breakpointEvents.py
+++ b/lldb/test/API/tools/lldb-dap/breakpoint-events/TestDAP_breakpointEvents.py
@@ -1,17 +1,17 @@
 """
-Test lldb-vscode setBreakpoints request
+Test lldb-dap setBreakpoints request
 """
 
 
-import vscode
+import dap_server
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
-import lldbvscode_testcase
+import lldbdap_testcase
 import os
 
 
-class TestVSCode_breakpointEvents(lldbvscode_testcase.VSCodeTestCaseBase):
+class TestDAP_breakpointEvents(lldbdap_testcase.DAPTestCaseBase):
     @skipIfWindows
     @skipUnlessDarwin
     @expectedFailureAll(macos_version=[">=", "13.0"])
@@ -48,7 +48,7 @@ def test_breakpoint_events(self):
 
         # Set a breakpoint after creating the target by running a command line
         # command. It will eventually resolve and cause a breakpoint changed
-        # event to be sent to lldb-vscode. We want to make sure we don't send a
+        # event to be sent to lldb-dap. We want to make sure we don't send a
         # breakpoint any breakpoints that were set from the command line.
         # Breakpoints that are set via the VS code DAP packets will be
         # registered and marked with a special keyword to ensure we deliver
@@ -59,24 +59,28 @@ def test_breakpoint_events(self):
         main_bp_id = 0
         foo_bp_id = 0
         # Set breakpoints and verify that they got set correctly
-        vscode_breakpoint_ids = []
-        response = self.vscode.request_setBreakpoints(main_source_path, [main_bp_line])
+        dap_breakpoint_ids = []
+        response = self.dap_server.request_setBreakpoints(
+            main_source_path, [main_bp_line]
+        )
         if response:
             breakpoints = response["body"]["breakpoints"]
             for breakpoint in breakpoints:
                 main_bp_id = breakpoint["id"]
-                vscode_breakpoint_ids.append("%i" % (main_bp_id))
+                dap_breakpoint_ids.append("%i" % (main_bp_id))
                 # line = breakpoint['line']
                 self.assertTrue(
                     breakpoint["verified"], "expect main breakpoint to be verified"
                 )
 
-        response = self.vscode.request_setBreakpoints(foo_source_path, [foo_bp1_line])
+        response = self.dap_server.request_setBreakpoints(
+            foo_source_path, [foo_bp1_line]
+        )
         if response:
             breakpoints = response["body"]["breakpoints"]
             for breakpoint in breakpoints:
                 foo_bp_id = breakpoint["id"]
-                vscode_breakpoint_ids.append("%i" % (foo_bp_id))
+                dap_breakpoint_ids.append("%i" % (foo_bp_id))
                 self.assertFalse(
                     breakpoint["verified"], "expect foo breakpoint to not be verified"
                 )
@@ -88,21 +92,23 @@ def test_breakpoint_events(self):
         # libraries are not loaded yet (at least on macOS they aren't) and any
         # breakpoints set in foo.cpp should not be resolved.
         self.assertEqual(
-            len(self.vscode.breakpoint_events),
+            len(self.dap_server.breakpoint_events),
             0,
             "no breakpoint events when stopped at entry point",
         )
 
         # Continue to the breakpoint
-        self.continue_to_breakpoints(vscode_breakpoint_ids)
+        self.continue_to_breakpoints(dap_breakpoint_ids)
 
         # Make sure we only get an event for the breakpoint we set via a call
-        # to self.vscode.request_setBreakpoints(...), not the breakpoint
+        # to self.dap_server.request_setBreakpoints(...), not the breakpoint
         # we set with with a LLDB command in preRunCommands.
         self.assertEqual(
-            len(self.vscode.breakpoint_events), 1, "make sure we got a breakpoint event"
+            len(self.dap_server.breakpoint_events),
+            1,
+            "make sure we got a breakpoint event",
         )
-        event = self.vscode.breakpoint_events[0]
+        event = self.dap_server.breakpoint_events[0]
         # Verify the details of the breakpoint changed notification.
         body = event["body"]
         self.assertEqual(
diff --git a/lldb/test/API/tools/lldb-vscode/breakpoint-events/foo.cpp b/lldb/test/API/tools/lldb-dap/breakpoint-events/foo.cpp
similarity index 91%
rename from lldb/test/API/tools/lldb-vscode/breakpoint-events/foo.cpp
rename to lldb/test/API/tools/lldb-dap/breakpoint-events/foo.cpp
index fa4f5657e6..7a4f90d7dd 100644
--- a/lldb/test/API/tools/lldb-vscode/breakpoint-events/foo.cpp
+++ b/lldb/test/API/tools/lldb-dap/breakpoint-events/foo.cpp
@@ -7,5 +7,5 @@ static void unique_function_name() {
 int foo(int x) {
   // foo breakpoint 1
   unique_function_name();
-  return x+42;
+  return x + 42;
 }
diff --git a/lldb/test/API/tools/lldb-vscode/breakpoint-events/foo.h b/lldb/test/API/tools/lldb-dap/breakpoint-events/foo.h
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/breakpoint-events/foo.h
rename to lldb/test/API/tools/lldb-dap/breakpoint-events/foo.h
diff --git a/lldb/test/API/tools/lldb-vscode/breakpoint-events/main.cpp b/lldb/test/API/tools/lldb-dap/breakpoint-events/main.cpp
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/breakpoint-events/main.cpp
rename to lldb/test/API/tools/lldb-dap/breakpoint-events/main.cpp
diff --git a/lldb/test/API/tools/lldb-vscode/breakpoint/Makefile b/lldb/test/API/tools/lldb-dap/breakpoint/Makefile
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/breakpoint/Makefile
rename to lldb/test/API/tools/lldb-dap/breakpoint/Makefile
diff --git a/lldb/test/API/tools/lldb-vscode/breakpoint/TestVSCode_logpoints.py b/lldb/test/API/tools/lldb-dap/breakpoint/TestDAP_logpoints.py
similarity index 94%
rename from lldb/test/API/tools/lldb-vscode/breakpoint/TestVSCode_logpoints.py
rename to lldb/test/API/tools/lldb-dap/breakpoint/TestDAP_logpoints.py
index f530542749..25e794a49d 100644
--- a/lldb/test/API/tools/lldb-vscode/breakpoint/TestVSCode_logpoints.py
+++ b/lldb/test/API/tools/lldb-dap/breakpoint/TestDAP_logpoints.py
@@ -1,20 +1,20 @@
 """
-Test lldb-vscode logpoints feature.
+Test lldb-dap logpoints feature.
 """
 
 
-import vscode
+import dap_server
 import shutil
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
-import lldbvscode_testcase
+import lldbdap_testcase
 import os
 
 
-class TestVSCode_logpoints(lldbvscode_testcase.VSCodeTestCaseBase):
+class TestDAP_logpoints(lldbdap_testcase.DAPTestCaseBase):
     def setUp(self):
-        lldbvscode_testcase.VSCodeTestCaseBase.setUp(self)
+        lldbdap_testcase.DAPTestCaseBase.setUp(self)
 
         self.main_basename = "main-copy.cpp"
         self.main_path = os.path.realpath(self.getBuildArtifact(self.main_basename))
@@ -36,7 +36,7 @@ def test_logmessage_basic(self):
         )
         self.assertEquals(len(before_loop_breakpoint_ids), 1, "expect one breakpoint")
 
-        self.vscode.request_continue()
+        self.dap_server.request_continue()
 
         # Verify we hit the breakpoint before loop line
         self.verify_breakpoint_hit(before_loop_breakpoint_ids)
@@ -56,7 +56,7 @@ def test_logmessage_basic(self):
         )
 
         # Continue to trigger the breakpoint with log messages
-        self.vscode.request_continue()
+        self.dap_server.request_continue()
 
         # Verify we hit the breakpoint after loop line
         self.verify_breakpoint_hit([post_loop_breakpoint_id])
@@ -94,7 +94,7 @@ def test_logmessage_advanced(self):
         )
         self.assertEquals(len(before_loop_breakpoint_ids), 1, "expect one breakpoint")
 
-        self.vscode.request_continue()
+        self.dap_server.request_continue()
 
         # Verify we hit the breakpoint before loop line
         self.verify_breakpoint_hit(before_loop_breakpoint_ids)
@@ -117,7 +117,7 @@ def test_logmessage_advanced(self):
         )
 
         # Continue to trigger the breakpoint with log messages
-        self.vscode.request_continue()
+        self.dap_server.request_continue()
 
         # Verify we hit the breakpoint after loop line
         self.verify_breakpoint_hit([post_loop_breakpoint_id])
@@ -157,7 +157,7 @@ def test_logmessage_format(self):
         )
         self.assertEquals(len(before_loop_breakpoint_ids), 1, "expect one breakpoint")
 
-        self.vscode.request_continue()
+        self.dap_server.request_continue()
 
         # Verify we hit the breakpoint before loop line
         self.verify_breakpoint_hit(before_loop_breakpoint_ids)
@@ -179,7 +179,7 @@ def test_logmessage_format(self):
         )
 
         # Continue to trigger the breakpoint with log messages
-        self.vscode.request_continue()
+        self.dap_server.request_continue()
 
         # Verify we hit the breakpoint after loop line
         self.verify_breakpoint_hit([post_loop_breakpoint_id])
@@ -222,7 +222,7 @@ def test_logmessage_format_failure(self):
         )
         self.assertEquals(len(before_loop_breakpoint_ids), 1, "expect one breakpoint")
 
-        self.vscode.request_continue()
+        self.dap_server.request_continue()
 
         # Verify we hit the breakpoint before loop line
         self.verify_breakpoint_hit(before_loop_breakpoint_ids)
@@ -244,7 +244,7 @@ def test_logmessage_format_failure(self):
         )
 
         # Continue to trigger the breakpoint with log messages
-        self.vscode.request_continue()
+        self.dap_server.request_continue()
 
         # Verify we hit logpoint breakpoint if it's format has error.
         self.verify_breakpoint_hit([loop_breakpoint_id])
diff --git a/lldb/test/API/tools/lldb-vscode/breakpoint/TestVSCode_setBreakpoints.py b/lldb/test/API/tools/lldb-dap/breakpoint/TestDAP_setBreakpoints.py
similarity index 89%
rename from lldb/test/API/tools/lldb-vscode/breakpoint/TestVSCode_setBreakpoints.py
rename to lldb/test/API/tools/lldb-dap/breakpoint/TestDAP_setBreakpoints.py
index a918ed42f5..3d3252a78b 100644
--- a/lldb/test/API/tools/lldb-vscode/breakpoint/TestVSCode_setBreakpoints.py
+++ b/lldb/test/API/tools/lldb-dap/breakpoint/TestDAP_setBreakpoints.py
@@ -1,20 +1,20 @@
 """
-Test lldb-vscode setBreakpoints request
+Test lldb-dap setBreakpoints request
 """
 
 
-import vscode
+import dap_server
 import shutil
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
-import lldbvscode_testcase
+import lldbdap_testcase
 import os
 
 
-class TestVSCode_setBreakpoints(lldbvscode_testcase.VSCodeTestCaseBase):
+class TestDAP_setBreakpoints(lldbdap_testcase.DAPTestCaseBase):
     def setUp(self):
-        lldbvscode_testcase.VSCodeTestCaseBase.setUp(self)
+        lldbdap_testcase.DAPTestCaseBase.setUp(self)
 
         self.main_basename = "main-copy.cpp"
         self.main_path = os.path.realpath(self.getBuildArtifact(self.main_basename))
@@ -58,7 +58,7 @@ def test_source_map(self):
         self.launch(program, sourceMap=source_map)
 
         # breakpoint in main.cpp
-        response = self.vscode.request_setBreakpoints(new_main_path, [main_line])
+        response = self.dap_server.request_setBreakpoints(new_main_path, [main_line])
         breakpoints = response["body"]["breakpoints"]
         self.assertEquals(len(breakpoints), 1)
         breakpoint = breakpoints[0]
@@ -68,7 +68,7 @@ def test_source_map(self):
         self.assertEqual(new_main_path, breakpoint["source"]["path"])
 
         # 2nd breakpoint, which is from a dynamically loaded library
-        response = self.vscode.request_setBreakpoints(new_other_path, [other_line])
+        response = self.dap_server.request_setBreakpoints(new_other_path, [other_line])
         breakpoints = response["body"]["breakpoints"]
         breakpoint = breakpoints[0]
         self.assertEqual(breakpoint["line"], other_line)
@@ -77,11 +77,11 @@ def test_source_map(self):
         self.assertEqual(new_other_path, breakpoint["source"]["path"])
         other_breakpoint_id = breakpoint["id"]
 
-        self.vscode.request_continue()
+        self.dap_server.request_continue()
         self.verify_breakpoint_hit([other_breakpoint_id])
 
         # 2nd breakpoint again, which should be valid at this point
-        response = self.vscode.request_setBreakpoints(new_other_path, [other_line])
+        response = self.dap_server.request_setBreakpoints(new_other_path, [other_line])
         breakpoints = response["body"]["breakpoints"]
         breakpoint = breakpoints[0]
         self.assertEqual(breakpoint["line"], other_line)
@@ -90,7 +90,7 @@ def test_source_map(self):
         self.assertEqual(new_other_path, breakpoint["source"]["path"])
 
         # now we check the stack trace making sure that we got mapped source paths
-        frames = self.vscode.request_stackTrace()["body"]["stackFrames"]
+        frames = self.dap_server.request_stackTrace()["body"]["stackFrames"]
 
         self.assertEqual(frames[0]["source"]["name"], other_basename)
         self.assertEqual(frames[0]["source"]["path"], new_other_path)
@@ -125,7 +125,7 @@ def test_set_and_clear(self):
         self.build_and_launch(program)
 
         # Set 3 breakpoints and verify that they got set correctly
-        response = self.vscode.request_setBreakpoints(self.main_path, lines)
+        response = self.dap_server.request_setBreakpoints(self.main_path, lines)
         line_to_id = {}
         if response:
             breakpoints = response["body"]["breakpoints"]
@@ -152,7 +152,7 @@ def test_set_and_clear(self):
         lines.remove(second_line)
         # Set 2 breakpoints and verify that the previous breakpoints that were
         # set above are still set.
-        response = self.vscode.request_setBreakpoints(self.main_path, lines)
+        response = self.dap_server.request_setBreakpoints(self.main_path, lines)
         if response:
             breakpoints = response["body"]["breakpoints"]
             self.assertEquals(
@@ -179,7 +179,7 @@ def test_set_and_clear(self):
         # we have only 2 breakpoints set. The response above could have told
         # us about 2 breakpoints, but we want to make sure we don't have the
         # third one still set in the target
-        response = self.vscode.request_testGetTargetBreakpoints()
+        response = self.dap_server.request_testGetTargetBreakpoints()
         if response:
             breakpoints = response["body"]["breakpoints"]
             self.assertEquals(
@@ -204,7 +204,7 @@ def test_set_and_clear(self):
         # Now clear all breakpoints for the source file by passing down an
         # empty lines array
         lines = []
-        response = self.vscode.request_setBreakpoints(self.main_path, lines)
+        response = self.dap_server.request_setBreakpoints(self.main_path, lines)
         if response:
             breakpoints = response["body"]["breakpoints"]
             self.assertEquals(
@@ -214,7 +214,7 @@ def test_set_and_clear(self):
             )
 
         # Verify with the target that all breakpoints have been cleared
-        response = self.vscode.request_testGetTargetBreakpoints()
+        response = self.dap_server.request_testGetTargetBreakpoints()
         if response:
             breakpoints = response["body"]["breakpoints"]
             self.assertEquals(
@@ -226,7 +226,7 @@ def test_set_and_clear(self):
         # Now set a breakpoint again in the same source file and verify it
         # was added.
         lines = [second_line]
-        response = self.vscode.request_setBreakpoints(self.main_path, lines)
+        response = self.dap_server.request_setBreakpoints(self.main_path, lines)
         if response:
             breakpoints = response["body"]["breakpoints"]
             self.assertEquals(
@@ -245,7 +245,7 @@ def test_set_and_clear(self):
         # we have only 2 breakpoints set. The response above could have told
         # us about 2 breakpoints, but we want to make sure we don't have the
         # third one still set in the target
-        response = self.vscode.request_testGetTargetBreakpoints()
+        response = self.dap_server.request_testGetTargetBreakpoints()
         if response:
             breakpoints = response["body"]["breakpoints"]
             self.assertEquals(
@@ -278,7 +278,7 @@ def test_clear_breakpoints_unset_breakpoints(self):
         self.build_and_launch(program)
 
         # Set one breakpoint and verify that it got set correctly.
-        response = self.vscode.request_setBreakpoints(self.main_path, lines)
+        response = self.dap_server.request_setBreakpoints(self.main_path, lines)
         line_to_id = {}
         breakpoints = response["body"]["breakpoints"]
         self.assertEquals(
@@ -295,12 +295,12 @@ def test_clear_breakpoints_unset_breakpoints(self):
         # Now clear all breakpoints for the source file by not setting the
         # lines array.
         lines = None
-        response = self.vscode.request_setBreakpoints(self.main_path, lines)
+        response = self.dap_server.request_setBreakpoints(self.main_path, lines)
         breakpoints = response["body"]["breakpoints"]
         self.assertEquals(len(breakpoints), 0, "expect no source breakpoints")
 
         # Verify with the target that all breakpoints have been cleared.
-        response = self.vscode.request_testGetTargetBreakpoints()
+        response = self.dap_server.request_testGetTargetBreakpoints()
         breakpoints = response["body"]["breakpoints"]
         self.assertEquals(len(breakpoints), 0, "expect no source breakpoints")
 
@@ -317,13 +317,13 @@ def test_functionality(self):
         # hitCondition
         breakpoint_ids = self.set_source_breakpoints(self.main_path, [loop_line])
         self.assertEquals(len(breakpoint_ids), 1, "expect one breakpoint")
-        self.vscode.request_continue()
+        self.dap_server.request_continue()
 
         # Verify we hit the breakpoint we just set
         self.verify_breakpoint_hit(breakpoint_ids)
 
         # Make sure i is zero at first breakpoint
-        i = int(self.vscode.get_local_variable_value("i"))
+        i = int(self.dap_server.get_local_variable_value("i"))
         self.assertEquals(i, 0, "i != 0 after hitting breakpoint")
 
         # Update the condition on our breakpoint
@@ -337,7 +337,7 @@ def test_functionality(self):
         )
 
         self.continue_to_breakpoints(breakpoint_ids)
-        i = int(self.vscode.get_local_variable_value("i"))
+        i = int(self.dap_server.get_local_variable_value("i"))
         self.assertEquals(i, 4, "i != 4 showing conditional works")
 
         new_breakpoint_ids = self.set_source_breakpoints(
@@ -352,11 +352,11 @@ def test_functionality(self):
 
         # Continue with a hitCondition of 2 and expect it to skip 1 value
         self.continue_to_breakpoints(breakpoint_ids)
-        i = int(self.vscode.get_local_variable_value("i"))
+        i = int(self.dap_server.get_local_variable_value("i"))
         self.assertEquals(i, 6, "i != 6 showing hitCondition works")
 
         # continue after hitting our hitCondition and make sure it only goes
         # up by 1
         self.continue_to_breakpoints(breakpoint_ids)
-        i = int(self.vscode.get_local_variable_value("i"))
+        i = int(self.dap_server.get_local_variable_value("i"))
         self.assertEquals(i, 7, "i != 7 showing post hitCondition hits every time")
diff --git a/lldb/test/API/tools/lldb-vscode/breakpoint/TestVSCode_setExceptionBreakpoints.py b/lldb/test/API/tools/lldb-dap/breakpoint/TestDAP_setExceptionBreakpoints.py
similarity index 86%
rename from lldb/test/API/tools/lldb-vscode/breakpoint/TestVSCode_setExceptionBreakpoints.py
rename to lldb/test/API/tools/lldb-dap/breakpoint/TestDAP_setExceptionBreakpoints.py
index 9ce9b1144d..84d3f12490 100644
--- a/lldb/test/API/tools/lldb-vscode/breakpoint/TestVSCode_setExceptionBreakpoints.py
+++ b/lldb/test/API/tools/lldb-dap/breakpoint/TestDAP_setExceptionBreakpoints.py
@@ -1,16 +1,16 @@
 """
-Test lldb-vscode setBreakpoints request
+Test lldb-dap setBreakpoints request
 """
 
 
-import vscode
+import dap_server
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
-import lldbvscode_testcase
+import lldbdap_testcase
 
 
-class TestVSCode_setExceptionBreakpoints(lldbvscode_testcase.VSCodeTestCaseBase):
+class TestDAP_setExceptionBreakpoints(lldbdap_testcase.DAPTestCaseBase):
     @skipIfWindows
     @skipIfRemote
     def test_functionality(self):
@@ -34,7 +34,7 @@ def test_functionality(self):
         self.build_and_launch(program)
 
         filters = ["cpp_throw", "cpp_catch"]
-        response = self.vscode.request_setExceptionBreakpoints(filters)
+        response = self.dap_server.request_setExceptionBreakpoints(filters)
         if response:
             self.assertTrue(response["success"])
 
diff --git a/lldb/test/API/tools/lldb-vscode/breakpoint/TestVSCode_setFunctionBreakpoints.py b/lldb/test/API/tools/lldb-dap/breakpoint/TestDAP_setFunctionBreakpoints.py
similarity index 88%
rename from lldb/test/API/tools/lldb-vscode/breakpoint/TestVSCode_setFunctionBreakpoints.py
rename to lldb/test/API/tools/lldb-dap/breakpoint/TestDAP_setFunctionBreakpoints.py
index 8748d0526a..f90dc0f041 100644
--- a/lldb/test/API/tools/lldb-vscode/breakpoint/TestVSCode_setFunctionBreakpoints.py
+++ b/lldb/test/API/tools/lldb-dap/breakpoint/TestDAP_setFunctionBreakpoints.py
@@ -1,16 +1,16 @@
 """
-Test lldb-vscode setBreakpoints request
+Test lldb-dap setBreakpoints request
 """
 
 
-import vscode
+import dap_server
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
-import lldbvscode_testcase
+import lldbdap_testcase
 
 
-class TestVSCode_setFunctionBreakpoints(lldbvscode_testcase.VSCodeTestCaseBase):
+class TestDAP_setFunctionBreakpoints(lldbdap_testcase.DAPTestCaseBase):
     @skipIfWindows
     @skipIfRemote
     def test_set_and_clear(self):
@@ -34,7 +34,7 @@ def test_set_and_clear(self):
         bp_id_12 = None
         functions = ["twelve"]
         # Set a function breakpoint at 'twelve'
-        response = self.vscode.request_setFunctionBreakpoints(functions)
+        response = self.dap_server.request_setFunctionBreakpoints(functions)
         if response:
             breakpoints = response["body"]["breakpoints"]
             self.assertEquals(
@@ -48,7 +48,7 @@ def test_set_and_clear(self):
 
         # Add an extra name and make sure we have two breakpoints after this
         functions.append("thirteen")
-        response = self.vscode.request_setFunctionBreakpoints(functions)
+        response = self.dap_server.request_setFunctionBreakpoints(functions)
         if response:
             breakpoints = response["body"]["breakpoints"]
             self.assertEquals(
@@ -62,7 +62,7 @@ def test_set_and_clear(self):
         # There is no breakpoint delete packet, clients just send another
         # setFunctionBreakpoints packet with the different function names.
         functions.remove("thirteen")
-        response = self.vscode.request_setFunctionBreakpoints(functions)
+        response = self.dap_server.request_setFunctionBreakpoints(functions)
         if response:
             breakpoints = response["body"]["breakpoints"]
             self.assertEquals(
@@ -83,7 +83,7 @@ def test_set_and_clear(self):
         # we have only 1 breakpoints set. The response above could have told
         # us about 1 breakpoints, but we want to make sure we don't have the
         # second one still set in the target
-        response = self.vscode.request_testGetTargetBreakpoints()
+        response = self.dap_server.request_testGetTargetBreakpoints()
         if response:
             breakpoints = response["body"]["breakpoints"]
             self.assertEquals(
@@ -103,7 +103,7 @@ def test_set_and_clear(self):
         # Now clear all breakpoints for the source file by passing down an
         # empty lines array
         functions = []
-        response = self.vscode.request_setFunctionBreakpoints(functions)
+        response = self.dap_server.request_setFunctionBreakpoints(functions)
         if response:
             breakpoints = response["body"]["breakpoints"]
             self.assertEquals(
@@ -113,7 +113,7 @@ def test_set_and_clear(self):
             )
 
         # Verify with the target that all breakpoints have been cleared
-        response = self.vscode.request_testGetTargetBreakpoints()
+        response = self.dap_server.request_testGetTargetBreakpoints()
         if response:
             breakpoints = response["body"]["breakpoints"]
             self.assertEquals(
@@ -140,7 +140,7 @@ def test_functionality(self):
         self.continue_to_breakpoints(breakpoint_ids)
 
         # Make sure i is zero at first breakpoint
-        i = int(self.vscode.get_local_variable_value("i"))
+        i = int(self.dap_server.get_local_variable_value("i"))
         self.assertEquals(i, 0, "i != 0 after hitting breakpoint")
 
         # Update the condition on our breakpoint
@@ -152,7 +152,7 @@ def test_functionality(self):
         )
 
         self.continue_to_breakpoints(breakpoint_ids)
-        i = int(self.vscode.get_local_variable_value("i"))
+        i = int(self.dap_server.get_local_variable_value("i"))
         self.assertEquals(i, 4, "i != 4 showing conditional works")
         new_breakpoint_ids = self.set_function_breakpoints(functions, hitCondition="2")
 
@@ -164,11 +164,11 @@ def test_functionality(self):
 
         # Continue with a hitCondition of 2 and expect it to skip 1 value
         self.continue_to_breakpoints(breakpoint_ids)
-        i = int(self.vscode.get_local_variable_value("i"))
+        i = int(self.dap_server.get_local_variable_value("i"))
         self.assertEquals(i, 6, "i != 6 showing hitCondition works")
 
         # continue after hitting our hitCondition and make sure it only goes
         # up by 1
         self.continue_to_breakpoints(breakpoint_ids)
-        i = int(self.vscode.get_local_variable_value("i"))
+        i = int(self.dap_server.get_local_variable_value("i"))
         self.assertEquals(i, 7, "i != 7 showing post hitCondition hits every time")
diff --git a/lldb/test/API/tools/lldb-vscode/breakpoint/main.cpp b/lldb/test/API/tools/lldb-dap/breakpoint/main.cpp
similarity index 82%
rename from lldb/test/API/tools/lldb-vscode/breakpoint/main.cpp
rename to lldb/test/API/tools/lldb-dap/breakpoint/main.cpp
index d4e0ac26dd..935a63fab6 100644
--- a/lldb/test/API/tools/lldb-vscode/breakpoint/main.cpp
+++ b/lldb/test/API/tools/lldb-dap/breakpoint/main.cpp
@@ -11,10 +11,10 @@ int thirteen(int i) {
 }
 
 namespace a {
-  int fourteen(int i) {
-    return 14 + i; // break 14
-  }
+int fourteen(int i) {
+  return 14 + i; // break 14
 }
+} // namespace a
 int main(int argc, char const *argv[]) {
 #if defined(__APPLE__)
   const char *libother_name = "libother.dylib";
@@ -35,11 +35,11 @@ int main(int argc, char const *argv[]) {
   }
   foo(12); // before loop
 
-  for (int i=0; i<10; ++i) {
+  for (int i = 0; i < 10; ++i) {
     int x = twelve(i) + thirteen(i) + a::fourteen(i); // break loop
   }
   try {
-    throw std::invalid_argument( "throwing exception for testing" );
+    throw std::invalid_argument("throwing exception for testing");
   } catch (...) {
     puts("caught exception...");
   }
diff --git a/lldb/test/API/tools/lldb-vscode/breakpoint/other.c b/lldb/test/API/tools/lldb-dap/breakpoint/other.c
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/breakpoint/other.c
rename to lldb/test/API/tools/lldb-dap/breakpoint/other.c
diff --git a/lldb/test/API/tools/lldb-dap/categories b/lldb/test/API/tools/lldb-dap/categories
new file mode 100644
index 0000000000..0902bbb595
--- /dev/null
+++ b/lldb/test/API/tools/lldb-dap/categories
@@ -0,0 +1 @@
+lldb-dap
diff --git a/lldb/test/API/tools/lldb-vscode/console/Makefile b/lldb/test/API/tools/lldb-dap/completions/Makefile
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/console/Makefile
rename to lldb/test/API/tools/lldb-dap/completions/Makefile
diff --git a/lldb/test/API/tools/lldb-vscode/completions/TestVSCode_completions.py b/lldb/test/API/tools/lldb-dap/completions/TestDAP_completions.py
similarity index 81%
rename from lldb/test/API/tools/lldb-vscode/completions/TestVSCode_completions.py
rename to lldb/test/API/tools/lldb-dap/completions/TestDAP_completions.py
index 1b44c3c993..5f6d63392f 100644
--- a/lldb/test/API/tools/lldb-vscode/completions/TestVSCode_completions.py
+++ b/lldb/test/API/tools/lldb-dap/completions/TestDAP_completions.py
@@ -1,16 +1,16 @@
 """
-Test lldb-vscode completions request
+Test lldb-dap completions request
 """
 
 
-import lldbvscode_testcase
-import vscode
+import lldbdap_testcase
+import dap_server
 from lldbsuite.test import lldbutil
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 
 
-class TestVSCode_completions(lldbvscode_testcase.VSCodeTestCaseBase):
+class TestDAP_completions(lldbdap_testcase.DAPTestCaseBase):
     def verify_completions(self, actual_list, expected_list, not_expected_list=[]):
         for expected_item in expected_list:
             self.assertIn(expected_item, actual_list)
@@ -36,7 +36,7 @@ def test_completions(self):
 
         # shouldn't see variables inside main
         self.verify_completions(
-            self.vscode.get_completions("var"),
+            self.dap_server.get_completions("var"),
             [
                 {
                     "text": "var",
@@ -54,7 +54,7 @@ def test_completions(self):
 
         # should see global keywords but not variables inside main
         self.verify_completions(
-            self.vscode.get_completions("str"),
+            self.dap_server.get_completions("str"),
             [{"text": "struct", "label": "struct"}],
             [{"text": "str1", "label": "str1 -- std::string &"}],
         )
@@ -63,7 +63,7 @@ def test_completions(self):
 
         # should see variables from main but not from the other function
         self.verify_completions(
-            self.vscode.get_completions("var"),
+            self.dap_server.get_completions("var"),
             [
                 {"text": "var1", "label": "var1 -- int &"},
                 {"text": "var2", "label": "var2 -- int &"},
@@ -77,7 +77,7 @@ def test_completions(self):
         )
 
         self.verify_completions(
-            self.vscode.get_completions("str"),
+            self.dap_server.get_completions("str"),
             [
                 {"text": "struct", "label": "struct"},
                 {"text": "str1", "label": "str1 -- string &"},
@@ -86,19 +86,19 @@ def test_completions(self):
 
         # should complete arbitrary commands including word starts
         self.verify_completions(
-            self.vscode.get_completions("`log enable  "),
+            self.dap_server.get_completions("`log enable  "),
             [{"text": "gdb-remote", "label": "gdb-remote"}],
         )
 
         # should complete expressions with quotes inside
         self.verify_completions(
-            self.vscode.get_completions('`expr " "; typed'),
+            self.dap_server.get_completions('`expr " "; typed'),
             [{"text": "typedef", "label": "typedef"}],
         )
 
         # should complete an incomplete quoted token
         self.verify_completions(
-            self.vscode.get_completions('`setting "se'),
+            self.dap_server.get_completions('`setting "se'),
             [
                 {
                     "text": "set",
@@ -107,7 +107,7 @@ def test_completions(self):
             ],
         )
         self.verify_completions(
-            self.vscode.get_completions("`'comm"),
+            self.dap_server.get_completions("`'comm"),
             [
                 {
                     "text": "command",
@@ -117,34 +117,34 @@ def test_completions(self):
         )
 
         self.verify_completions(
-            self.vscode.get_completions("foo1.v"),
+            self.dap_server.get_completions("foo1.v"),
             [{"text": "var1", "label": "foo1.var1 -- int"}],
         )
 
         self.verify_completions(
-            self.vscode.get_completions("foo1.my_bar_object.v"),
+            self.dap_server.get_completions("foo1.my_bar_object.v"),
             [{"text": "var1", "label": "foo1.my_bar_object.var1 -- int"}],
         )
 
         self.verify_completions(
-            self.vscode.get_completions("foo1.var1 + foo1.v"),
+            self.dap_server.get_completions("foo1.var1 + foo1.v"),
             [{"text": "var1", "label": "foo1.var1 -- int"}],
         )
 
         self.verify_completions(
-            self.vscode.get_completions("foo1.var1 + v"),
+            self.dap_server.get_completions("foo1.var1 + v"),
             [{"text": "var1", "label": "var1 -- int &"}],
         )
 
         # should correctly handle spaces between objects and member operators
         self.verify_completions(
-            self.vscode.get_completions("foo1 .v"),
+            self.dap_server.get_completions("foo1 .v"),
             [{"text": "var1", "label": ".var1 -- int"}],
             [{"text": "var2", "label": ".var2 -- int"}],
         )
 
         self.verify_completions(
-            self.vscode.get_completions("foo1 . v"),
+            self.dap_server.get_completions("foo1 . v"),
             [{"text": "var1", "label": "var1 -- int"}],
             [{"text": "var2", "label": "var2 -- int"}],
         )
diff --git a/lldb/test/API/tools/lldb-vscode/completions/main.cpp b/lldb/test/API/tools/lldb-dap/completions/main.cpp
similarity index 81%
rename from lldb/test/API/tools/lldb-vscode/completions/main.cpp
rename to lldb/test/API/tools/lldb-dap/completions/main.cpp
index dccf43ff6f..4314067cfe 100644
--- a/lldb/test/API/tools/lldb-vscode/completions/main.cpp
+++ b/lldb/test/API/tools/lldb-dap/completions/main.cpp
@@ -7,9 +7,9 @@ struct bar {
 
 struct foo {
   int var1;
-  bar* my_bar_pointer;
+  bar *my_bar_pointer;
   bar my_bar_object;
-  foo* next_foo;
+  foo *next_foo;
 };
 
 struct baz {
@@ -28,7 +28,7 @@ int main(int argc, char const *argv[]) {
   std::vector<baz> vec;
   fun(vec);
   bar bar1 = {2};
-  bar* bar2 = &bar1;
-  foo foo1 = {3,&bar1, bar1, NULL};
+  bar *bar2 = &bar1;
+  foo foo1 = {3, &bar1, bar1, NULL};
   return 0; // breakpoint 2
 }
diff --git a/lldb/test/API/tools/lldb-vscode/disconnect/Makefile b/lldb/test/API/tools/lldb-dap/console/Makefile
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/disconnect/Makefile
rename to lldb/test/API/tools/lldb-dap/console/Makefile
diff --git a/lldb/test/API/tools/lldb-vscode/console/TestVSCode_console.py b/lldb/test/API/tools/lldb-dap/console/TestDAP_console.py
similarity index 83%
rename from lldb/test/API/tools/lldb-vscode/console/TestVSCode_console.py
rename to lldb/test/API/tools/lldb-dap/console/TestDAP_console.py
index d28e98b37c..47c706f2ca 100644
--- a/lldb/test/API/tools/lldb-vscode/console/TestVSCode_console.py
+++ b/lldb/test/API/tools/lldb-dap/console/TestDAP_console.py
@@ -1,17 +1,19 @@
 """
-Test lldb-vscode setBreakpoints request
+Test lldb-dap setBreakpoints request
 """
 
-import vscode
+import dap_server
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
-import lldbvscode_testcase
+import lldbdap_testcase
 
 
-class TestVSCode_console(lldbvscode_testcase.VSCodeTestCaseBase):
+class TestDAP_console(lldbdap_testcase.DAPTestCaseBase):
     def check_lldb_command(self, lldb_command, contains_string, assert_msg):
-        response = self.vscode.request_evaluate("`%s" % (lldb_command), context="repl")
+        response = self.dap_server.request_evaluate(
+            "`%s" % (lldb_command), context="repl"
+        )
         output = response["body"]["result"]
         self.assertIn(
             contains_string,
@@ -29,8 +31,8 @@ def test_scopes_variables_setVariable_evaluate(self):
         """
         Tests that the "scopes" request causes the currently selected
         thread and frame to be updated. There are no DAP packets that tell
-        lldb-vscode which thread and frame are selected other than the
-        "scopes" request. lldb-vscode will now select the thread and frame
+        lldb-dap which thread and frame are selected other than the
+        "scopes" request. lldb-dap will now select the thread and frame
         for the latest "scopes" request that it receives.
 
         The LLDB command interpreter needs to have the right thread and
@@ -52,7 +54,7 @@ def test_scopes_variables_setVariable_evaluate(self):
         self.continue_to_breakpoints(breakpoint_ids)
         # Cause a "scopes" to be sent for frame zero which should update the
         # selected thread and frame to frame 0.
-        self.vscode.get_local_variables(frameIndex=0)
+        self.dap_server.get_local_variables(frameIndex=0)
         # Verify frame #0 is selected in the command interpreter by running
         # the "frame select" command with no frame index which will print the
         # currently selected frame.
@@ -60,7 +62,7 @@ def test_scopes_variables_setVariable_evaluate(self):
 
         # Cause a "scopes" to be sent for frame one which should update the
         # selected thread and frame to frame 1.
-        self.vscode.get_local_variables(frameIndex=1)
+        self.dap_server.get_local_variables(frameIndex=1)
         # Verify frame #1 is selected in the command interpreter by running
         # the "frame select" command with no frame index which will print the
         # currently selected frame.
diff --git a/lldb/test/API/tools/lldb-vscode/console/TestVSCode_redirection_to_console.py b/lldb/test/API/tools/lldb-dap/console/TestDAP_redirection_to_console.py
similarity index 66%
rename from lldb/test/API/tools/lldb-vscode/console/TestVSCode_redirection_to_console.py
rename to lldb/test/API/tools/lldb-dap/console/TestDAP_redirection_to_console.py
index 27b0215c54..85911a449e 100644
--- a/lldb/test/API/tools/lldb-vscode/console/TestVSCode_redirection_to_console.py
+++ b/lldb/test/API/tools/lldb-dap/console/TestDAP_redirection_to_console.py
@@ -1,12 +1,12 @@
-import vscode
+import dap_server
 import json
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
-import lldbvscode_testcase
+import lldbdap_testcase
 
 
-class TestVSCode_redirection_to_console(lldbvscode_testcase.VSCodeTestCaseBase):
+class TestDAP_redirection_to_console(lldbdap_testcase.DAPTestCaseBase):
     @skipIfWindows
     @skipIfRemote
     def test(self):
@@ -14,11 +14,11 @@ def test(self):
         Without proper stderr and stdout redirection, the following code would throw an
         exception, like the following:
 
-            Exception: unexpected malformed message from lldb-vscode
+            Exception: unexpected malformed message from lldb-dap
         """
         program = self.getBuildArtifact("a.out")
         self.build_and_launch(
-            program, lldbVSCodeEnv={"LLDB_VSCODE_TEST_STDOUT_STDERR_REDIRECTION": ""}
+            program, lldbDAPEnv={"LLDB_DAP_TEST_STDOUT_STDERR_REDIRECTION": ""}
         )
 
         source = "main.cpp"
@@ -29,4 +29,6 @@ def test(self):
         self.assertEqual(len(breakpoint_ids), 1, "expect correct number of breakpoints")
         self.continue_to_breakpoints(breakpoint_ids)
 
-        self.assertIn("argc", json.dumps(self.vscode.get_local_variables(frameIndex=1)))
+        self.assertIn(
+            "argc", json.dumps(self.dap_server.get_local_variables(frameIndex=1))
+        )
diff --git a/lldb/test/API/tools/lldb-vscode/console/main.cpp b/lldb/test/API/tools/lldb-dap/console/main.cpp
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/console/main.cpp
rename to lldb/test/API/tools/lldb-dap/console/main.cpp
diff --git a/lldb/test/API/tools/lldb-vscode/coreFile/TestVSCode_coreFile.py b/lldb/test/API/tools/lldb-dap/coreFile/TestDAP_coreFile.py
similarity index 92%
rename from lldb/test/API/tools/lldb-vscode/coreFile/TestVSCode_coreFile.py
rename to lldb/test/API/tools/lldb-dap/coreFile/TestDAP_coreFile.py
index 8cbdb8fa7e..2ac1f39772 100644
--- a/lldb/test/API/tools/lldb-vscode/coreFile/TestVSCode_coreFile.py
+++ b/lldb/test/API/tools/lldb-dap/coreFile/TestDAP_coreFile.py
@@ -1,17 +1,17 @@
 """
-Test lldb-vscode coreFile attaching
+Test lldb-dap coreFile attaching
 """
 
 
-import vscode
+import dap_server
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
-import lldbvscode_testcase
+import lldbdap_testcase
 import os
 
 
-class TestVSCode_coreFile(lldbvscode_testcase.VSCodeTestCaseBase):
+class TestDAP_coreFile(lldbdap_testcase.DAPTestCaseBase):
     @skipIfWindows
     @skipIfRemote
     @skipIfLLVMTargetMissing("X86")
@@ -53,7 +53,7 @@ def test_core_file(self):
         self.continue_to_next_stop()
         self.assertEquals(self.get_stackFrames(), expected_frames)
 
-        self.vscode.request_next(threadId=32259)
+        self.dap_server.request_next(threadId=32259)
         self.assertEquals(self.get_stackFrames(), expected_frames)
 
     @skipIfWindows
diff --git a/lldb/test/API/tools/lldb-vscode/coreFile/linux-x86_64.core b/lldb/test/API/tools/lldb-dap/coreFile/linux-x86_64.core
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/coreFile/linux-x86_64.core
rename to lldb/test/API/tools/lldb-dap/coreFile/linux-x86_64.core
diff --git a/lldb/test/API/tools/lldb-vscode/coreFile/linux-x86_64.out b/lldb/test/API/tools/lldb-dap/coreFile/linux-x86_64.out
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/coreFile/linux-x86_64.out
rename to lldb/test/API/tools/lldb-dap/coreFile/linux-x86_64.out
diff --git a/lldb/test/API/tools/lldb-vscode/coreFile/main.c b/lldb/test/API/tools/lldb-dap/coreFile/main.c
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/coreFile/main.c
rename to lldb/test/API/tools/lldb-dap/coreFile/main.c
diff --git a/lldb/test/API/tools/lldb-vscode/correct-thread/Makefile b/lldb/test/API/tools/lldb-dap/correct-thread/Makefile
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/correct-thread/Makefile
rename to lldb/test/API/tools/lldb-dap/correct-thread/Makefile
diff --git a/lldb/test/API/tools/lldb-vscode/correct-thread/TestVSCode_correct_thread.py b/lldb/test/API/tools/lldb-dap/correct-thread/TestDAP_correct_thread.py
similarity index 86%
rename from lldb/test/API/tools/lldb-vscode/correct-thread/TestVSCode_correct_thread.py
rename to lldb/test/API/tools/lldb-dap/correct-thread/TestDAP_correct_thread.py
index 5318163465..43d3b02e40 100644
--- a/lldb/test/API/tools/lldb-vscode/correct-thread/TestVSCode_correct_thread.py
+++ b/lldb/test/API/tools/lldb-dap/correct-thread/TestDAP_correct_thread.py
@@ -1,15 +1,15 @@
 """
-Test lldb-vscode setBreakpoints request
+Test lldb-dap setBreakpoints request
 """
 
-import vscode
+import dap_server
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
-import lldbvscode_testcase
+import lldbdap_testcase
 
 
-class TestVSCode_correct_thread(lldbvscode_testcase.VSCodeTestCaseBase):
+class TestDAP_correct_thread(lldbdap_testcase.DAPTestCaseBase):
     @skipIfWindows
     @skipIfRemote
     def test_correct_thread(self):
@@ -33,8 +33,8 @@ def test_correct_thread(self):
         # We're now stopped at the breakpoint in the first thread, thread #2.
         # Continue to join the first thread and hit the breakpoint in the
         # second thread, thread #3.
-        self.vscode.request_continue()
-        stopped_event = self.vscode.wait_for_stopped()
+        self.dap_server.request_continue()
+        stopped_event = self.dap_server.wait_for_stopped()
         # Verify that the description is the relevant breakpoint,
         # preserveFocusHint is False and threadCausedFocus is True
         self.assertTrue(
diff --git a/lldb/test/API/tools/lldb-vscode/correct-thread/main.c b/lldb/test/API/tools/lldb-dap/correct-thread/main.c
similarity index 84%
rename from lldb/test/API/tools/lldb-vscode/correct-thread/main.c
rename to lldb/test/API/tools/lldb-dap/correct-thread/main.c
index 157c3f994d..3eeb1ef02c 100644
--- a/lldb/test/API/tools/lldb-vscode/correct-thread/main.c
+++ b/lldb/test/API/tools/lldb-dap/correct-thread/main.c
@@ -3,14 +3,12 @@
 
 int state_var;
 
-void *thread (void *in)
-{
+void *thread(void *in) {
   state_var++; // break here
   return NULL;
 }
 
-int main(int argc, char **argv)
-{
+int main(int argc, char **argv) {
   pthread_t t1, t2;
 
   pthread_create(&t1, NULL, *thread, NULL);
diff --git a/lldb/test/API/tools/lldb-vscode/disassemble/Makefile b/lldb/test/API/tools/lldb-dap/disassemble/Makefile
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/disassemble/Makefile
rename to lldb/test/API/tools/lldb-dap/disassemble/Makefile
diff --git a/lldb/test/API/tools/lldb-vscode/disassemble/TestVSCode_disassemble.py b/lldb/test/API/tools/lldb-dap/disassemble/TestDAP_disassemble.py
similarity index 89%
rename from lldb/test/API/tools/lldb-vscode/disassemble/TestVSCode_disassemble.py
rename to lldb/test/API/tools/lldb-dap/disassemble/TestDAP_disassemble.py
index 521e8bf56a..cb4e946c52 100644
--- a/lldb/test/API/tools/lldb-vscode/disassemble/TestVSCode_disassemble.py
+++ b/lldb/test/API/tools/lldb-dap/disassemble/TestDAP_disassemble.py
@@ -1,17 +1,17 @@
 """
-Test lldb-vscode disassemble request
+Test lldb-dap disassemble request
 """
 
 
-import vscode
+import dap_server
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
-import lldbvscode_testcase
+import lldbdap_testcase
 import os
 
 
-class TestVSCode_disassemble(lldbvscode_testcase.VSCodeTestCaseBase):
+class TestDAP_disassemble(lldbdap_testcase.DAPTestCaseBase):
     @skipIfWindows
     @skipIfRemote
     def test_disassemble(self):
diff --git a/lldb/test/API/tools/lldb-dap/disassemble/main.c b/lldb/test/API/tools/lldb-dap/disassemble/main.c
new file mode 100644
index 0000000000..6609a4c37a
--- /dev/null
+++ b/lldb/test/API/tools/lldb-dap/disassemble/main.c
@@ -0,0 +1,30 @@
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+int compare_ints(const void *a, const void *b) {
+  int arg1 = *(const int *)a;
+  int arg2 = *(const int *)b;
+
+  // breakpoint 1
+
+  if (arg1 < arg2)
+    return -1;
+  if (arg1 > arg2)
+    return 1;
+  return 0;
+}
+
+int main(void) {
+  int ints[] = {-2, 99, 0, -743, 2, INT_MIN, 4};
+  int size = sizeof ints / sizeof *ints;
+
+  qsort(ints, size, sizeof(int), compare_ints);
+
+  for (int i = 0; i < size; i++) {
+    printf("%d ", ints[i]);
+  }
+
+  printf("\n");
+  return 0;
+}
\ No newline at end of file
diff --git a/lldb/test/API/tools/lldb-vscode/exception/Makefile b/lldb/test/API/tools/lldb-dap/disconnect/Makefile
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/exception/Makefile
rename to lldb/test/API/tools/lldb-dap/disconnect/Makefile
diff --git a/lldb/test/API/tools/lldb-vscode/disconnect/TestVSCode_disconnect.py b/lldb/test/API/tools/lldb-dap/disconnect/TestDAP_disconnect.py
similarity index 87%
rename from lldb/test/API/tools/lldb-vscode/disconnect/TestVSCode_disconnect.py
rename to lldb/test/API/tools/lldb-dap/disconnect/TestDAP_disconnect.py
index bc7b74e503..e5aab88c7f 100644
--- a/lldb/test/API/tools/lldb-vscode/disconnect/TestVSCode_disconnect.py
+++ b/lldb/test/API/tools/lldb-dap/disconnect/TestDAP_disconnect.py
@@ -1,23 +1,23 @@
 """
-Test lldb-vscode disconnect request
+Test lldb-dap disconnect request
 """
 
 
-import vscode
+import dap_server
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
-import lldbvscode_testcase
+import lldbdap_testcase
 import subprocess
 import time
 import os
 
 
-class TestVSCode_launch(lldbvscode_testcase.VSCodeTestCaseBase):
+class TestDAP_launch(lldbdap_testcase.DAPTestCaseBase):
     source = "main.cpp"
 
     def disconnect_and_assert_no_output_printed(self):
-        self.vscode.request_disconnect()
+        self.dap_server.request_disconnect()
         # verify we didn't get any input after disconnect
         time.sleep(2)
         output = self.get_stdout()
@@ -40,7 +40,7 @@ def test_launch(self):
         )
         self.continue_to_next_stop()
 
-        self.vscode.request_disconnect()
+        self.dap_server.request_disconnect()
         # verify we didn't produce the side effect file
         time.sleep(1)
         self.assertFalse(os.path.exists(program + ".side_effect"))
@@ -69,13 +69,13 @@ def test_attach(self):
         lldbutil.wait_for_file_on_target(self, sync_file_path)
 
         self.attach(pid=self.process.pid, disconnectAutomatically=False)
-        response = self.vscode.request_evaluate("wait_for_attach = false;")
+        response = self.dap_server.request_evaluate("wait_for_attach = false;")
         self.assertTrue(response["success"])
 
         # verify we haven't produced the side effect file yet
         self.assertFalse(os.path.exists(program + ".side_effect"))
 
-        self.vscode.request_disconnect()
+        self.dap_server.request_disconnect()
         time.sleep(2)
         # verify we produced the side effect file, as the program continued after disconnecting
         self.assertTrue(os.path.exists(program + ".side_effect"))
diff --git a/lldb/test/API/tools/lldb-vscode/disconnect/main.cpp b/lldb/test/API/tools/lldb-dap/disconnect/main.cpp
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/disconnect/main.cpp
rename to lldb/test/API/tools/lldb-dap/disconnect/main.cpp
diff --git a/lldb/test/API/tools/lldb-dap/evaluate/Makefile b/lldb/test/API/tools/lldb-dap/evaluate/Makefile
new file mode 100644
index 0000000000..7df22699c5
--- /dev/null
+++ b/lldb/test/API/tools/lldb-dap/evaluate/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp foo.cpp
+
+include Makefile.rules
diff --git a/lldb/test/API/tools/lldb-vscode/evaluate/TestVSCode_evaluate.py b/lldb/test/API/tools/lldb-dap/evaluate/TestDAP_evaluate.py
similarity index 95%
rename from lldb/test/API/tools/lldb-vscode/evaluate/TestVSCode_evaluate.py
rename to lldb/test/API/tools/lldb-dap/evaluate/TestDAP_evaluate.py
index d1b73e1a05..de9d2c93a1 100644
--- a/lldb/test/API/tools/lldb-vscode/evaluate/TestVSCode_evaluate.py
+++ b/lldb/test/API/tools/lldb-dap/evaluate/TestDAP_evaluate.py
@@ -1,19 +1,19 @@
 """
-Test lldb-vscode completions request
+Test lldb-dap completions request
 """
 
 
-import lldbvscode_testcase
-import vscode
+import lldbdap_testcase
+import dap_server
 from lldbsuite.test import lldbutil
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 
 
-class TestVSCode_variables(lldbvscode_testcase.VSCodeTestCaseBase):
+class TestDAP_variables(lldbdap_testcase.DAPTestCaseBase):
     def assertEvaluate(self, expression, regex):
         self.assertRegexpMatches(
-            self.vscode.request_evaluate(expression, context=self.context)["body"][
+            self.dap_server.request_evaluate(expression, context=self.context)["body"][
                 "result"
             ],
             regex,
@@ -22,7 +22,7 @@ def assertEvaluate(self, expression, regex):
     def assertEvaluateFailure(self, expression):
         self.assertNotIn(
             "result",
-            self.vscode.request_evaluate(expression, context=self.context)["body"],
+            self.dap_server.request_evaluate(expression, context=self.context)["body"],
         )
 
     def isExpressionParsedExpected(self):
diff --git a/lldb/test/API/tools/lldb-vscode/evaluate/foo.cpp b/lldb/test/API/tools/lldb-dap/evaluate/foo.cpp
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/evaluate/foo.cpp
rename to lldb/test/API/tools/lldb-dap/evaluate/foo.cpp
diff --git a/lldb/test/API/tools/lldb-vscode/evaluate/foo.h b/lldb/test/API/tools/lldb-dap/evaluate/foo.h
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/evaluate/foo.h
rename to lldb/test/API/tools/lldb-dap/evaluate/foo.h
diff --git a/lldb/test/API/tools/lldb-vscode/evaluate/main.cpp b/lldb/test/API/tools/lldb-dap/evaluate/main.cpp
similarity index 94%
rename from lldb/test/API/tools/lldb-vscode/evaluate/main.cpp
rename to lldb/test/API/tools/lldb-dap/evaluate/main.cpp
index f09d00e644..ca27b5ba5c 100644
--- a/lldb/test/API/tools/lldb-vscode/evaluate/main.cpp
+++ b/lldb/test/API/tools/lldb-dap/evaluate/main.cpp
@@ -1,7 +1,7 @@
 #include "foo.h"
 
-#include <vector>
 #include <map>
+#include <vector>
 
 static int static_int = 42;
 
@@ -43,7 +43,7 @@ int main(int argc, char const *argv[]) {
   std::vector<bool> my_bool_vec;
   my_bool_vec.push_back(true);
   my_bool_vec.push_back(false); // breakpoint 6
-  my_bool_vec.push_back(true); // breakpoint 7
+  my_bool_vec.push_back(true);  // breakpoint 7
 
   return 0;
 }
diff --git a/lldb/test/API/tools/lldb-vscode/variables/Makefile b/lldb/test/API/tools/lldb-dap/exception/Makefile
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/variables/Makefile
rename to lldb/test/API/tools/lldb-dap/exception/Makefile
diff --git a/lldb/test/API/tools/lldb-vscode/exception/TestVSCode_exception.py b/lldb/test/API/tools/lldb-dap/exception/TestDAP_exception.py
similarity index 74%
rename from lldb/test/API/tools/lldb-vscode/exception/TestVSCode_exception.py
rename to lldb/test/API/tools/lldb-dap/exception/TestDAP_exception.py
index f52daaa713..8c2c0154ba 100644
--- a/lldb/test/API/tools/lldb-vscode/exception/TestVSCode_exception.py
+++ b/lldb/test/API/tools/lldb-dap/exception/TestDAP_exception.py
@@ -1,14 +1,14 @@
 """
-Test exception behavior in VSCode
+Test exception behavior in DAP
 """
 
 
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
-import lldbvscode_testcase
+import lldbdap_testcase
 
 
-class TestVSCode_exception(lldbvscode_testcase.VSCodeTestCaseBase):
+class TestDAP_exception(lldbdap_testcase.DAPTestCaseBase):
     @skipIfWindows
     def test_stopped_description(self):
         """
@@ -19,5 +19,5 @@ def test_stopped_description(self):
         print("test_stopped_description called", flush=True)
         self.build_and_launch(program)
 
-        self.vscode.request_continue()
+        self.dap_server.request_continue()
         self.assertTrue(self.verify_stop_exception_info("signal SIGABRT"))
diff --git a/lldb/test/API/tools/lldb-vscode/exception/main.cpp b/lldb/test/API/tools/lldb-dap/exception/main.cpp
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/exception/main.cpp
rename to lldb/test/API/tools/lldb-dap/exception/main.cpp
diff --git a/lldb/test/API/tools/lldb-vscode/launch/Makefile b/lldb/test/API/tools/lldb-dap/launch/Makefile
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/launch/Makefile
rename to lldb/test/API/tools/lldb-dap/launch/Makefile
diff --git a/lldb/test/API/tools/lldb-vscode/launch/TestVSCode_launch.py b/lldb/test/API/tools/lldb-dap/launch/TestDAP_launch.py
similarity index 94%
rename from lldb/test/API/tools/lldb-vscode/launch/TestVSCode_launch.py
rename to lldb/test/API/tools/lldb-dap/launch/TestDAP_launch.py
index 3984546888..7895861225 100644
--- a/lldb/test/API/tools/lldb-vscode/launch/TestVSCode_launch.py
+++ b/lldb/test/API/tools/lldb-dap/launch/TestDAP_launch.py
@@ -1,18 +1,18 @@
 """
-Test lldb-vscode setBreakpoints request
+Test lldb-dap setBreakpoints request
 """
 
 
-import vscode
+import dap_server
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
-import lldbvscode_testcase
+import lldbdap_testcase
 import time
 import os
 
 
-class TestVSCode_launch(lldbvscode_testcase.VSCodeTestCaseBase):
+class TestDAP_launch(lldbdap_testcase.DAPTestCaseBase):
     @skipIfWindows
     @skipIfRemote
     def test_default(self):
@@ -33,22 +33,22 @@ def test_default(self):
     @skipIfRemote
     def test_termination(self):
         """
-        Tests the correct termination of lldb-vscode upon a 'disconnect'
+        Tests the correct termination of lldb-dap upon a 'disconnect'
         request.
         """
         self.create_debug_adaptor()
-        # The underlying lldb-vscode process must be alive
-        self.assertEqual(self.vscode.process.poll(), None)
+        # The underlying lldb-dap process must be alive
+        self.assertEqual(self.dap_server.process.poll(), None)
 
-        # The lldb-vscode process should finish even though
+        # The lldb-dap process should finish even though
         # we didn't close the communication socket explicitly
-        self.vscode.request_disconnect()
+        self.dap_server.request_disconnect()
 
-        # Wait until the underlying lldb-vscode process dies.
-        self.vscode.process.wait(timeout=10)
+        # Wait until the underlying lldb-dap process dies.
+        self.dap_server.process.wait(timeout=10)
 
         # Check the return code
-        self.assertEqual(self.vscode.process.poll(), 0)
+        self.assertEqual(self.dap_server.process.poll(), 0)
 
     @skipIfWindows
     @skipIfRemote
@@ -102,7 +102,7 @@ def test_cwd(self):
     def test_debuggerRoot(self):
         """
         Tests the "debuggerRoot" will change the working directory of
-        the lldb-vscode debug adaptor.
+        the lldb-dap debug adaptor.
         """
         program = self.getBuildArtifact("a.out")
         program_parent_dir = os.path.realpath(os.path.dirname(os.path.dirname(program)))
@@ -121,10 +121,10 @@ def test_debuggerRoot(self):
                 self.assertEquals(
                     program_parent_dir,
                     line[len(prefix) :],
-                    "lldb-vscode working dir '%s' == '%s'"
+                    "lldb-dap working dir '%s' == '%s'"
                     % (program_parent_dir, line[6:]),
                 )
-        self.assertTrue(found, "verified lldb-vscode working directory")
+        self.assertTrue(found, "verified lldb-dap working directory")
         self.continue_to_exit()
 
     @skipIfWindows
@@ -148,7 +148,7 @@ def test_sourcePath(self):
                 self.assertEquals(
                     quoted_path,
                     line[len(prefix) :],
-                    "lldb-vscode working dir %s == %s" % (quoted_path, line[6:]),
+                    "lldb-dap working dir %s == %s" % (quoted_path, line[6:]),
                 )
         self.assertTrue(found, 'found "sourcePath" in console output')
         self.continue_to_exit()
@@ -440,6 +440,6 @@ def test_terminate_commands(self):
         self.get_console()
         # Once it's disconnected the console should contain the
         # "terminateCommands"
-        self.vscode.request_disconnect(terminateDebuggee=True)
+        self.dap_server.request_disconnect(terminateDebuggee=True)
         output = self.collect_console(duration=1.0)
         self.verify_commands("terminateCommands", output, terminateCommands)
diff --git a/lldb/test/API/tools/lldb-vscode/launch/main.c b/lldb/test/API/tools/lldb-dap/launch/main.c
similarity index 77%
rename from lldb/test/API/tools/lldb-vscode/launch/main.c
rename to lldb/test/API/tools/lldb-dap/launch/main.c
index aed2af9828..01e209ac12 100644
--- a/lldb/test/API/tools/lldb-vscode/launch/main.c
+++ b/lldb/test/API/tools/lldb-dap/launch/main.c
@@ -3,13 +3,13 @@
 #include <unistd.h>
 
 int main(int argc, char const *argv[], char const *envp[]) {
-  for (int i=0; i<argc; ++i)
+  for (int i = 0; i < argc; ++i)
     printf("arg[%i] = \"%s\"\n", i, argv[i]);
-  for (int i=0; envp[i]; ++i)
+  for (int i = 0; envp[i]; ++i)
     printf("env[%i] = \"%s\"\n", i, envp[i]);
   char *cwd = getcwd(NULL, 0);
   printf("cwd = \"%s\"\n", cwd); // breakpoint 1
   free(cwd);
   cwd = NULL;
-  return 0;  // breakpoint 2
+  return 0; // breakpoint 2
 }
diff --git a/lldb/test/API/tools/lldb-vscode/module/Makefile b/lldb/test/API/tools/lldb-dap/module/Makefile
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/module/Makefile
rename to lldb/test/API/tools/lldb-dap/module/Makefile
diff --git a/lldb/test/API/tools/lldb-vscode/module/TestVSCode_module.py b/lldb/test/API/tools/lldb-dap/module/TestDAP_module.py
similarity index 88%
rename from lldb/test/API/tools/lldb-vscode/module/TestVSCode_module.py
rename to lldb/test/API/tools/lldb-dap/module/TestDAP_module.py
index 5ddad1beab..91321a00ed 100644
--- a/lldb/test/API/tools/lldb-vscode/module/TestVSCode_module.py
+++ b/lldb/test/API/tools/lldb-dap/module/TestDAP_module.py
@@ -1,16 +1,16 @@
 """
-Test lldb-vscode setBreakpoints request
+Test lldb-dap setBreakpoints request
 """
 
-import vscode
+import dap_server
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
-import lldbvscode_testcase
+import lldbdap_testcase
 import re
 
 
-class TestVSCode_module(lldbvscode_testcase.VSCodeTestCaseBase):
+class TestDAP_module(lldbdap_testcase.DAPTestCaseBase):
     def run_test(self, symbol_basename, expect_debug_info_size):
         program_basename = "a.out.stripped"
         program = self.getBuildArtifact(program_basename)
@@ -19,7 +19,7 @@ def run_test(self, symbol_basename, expect_debug_info_size):
         breakpoint_ids = self.set_function_breakpoints(functions)
         self.assertEquals(len(breakpoint_ids), len(functions), "expect one breakpoint")
         self.continue_to_breakpoints(breakpoint_ids)
-        active_modules = self.vscode.get_modules()
+        active_modules = self.dap_server.get_modules()
         program_module = active_modules[program_basename]
         self.assertIn(
             program_basename,
@@ -36,13 +36,13 @@ def run_test(self, symbol_basename, expect_debug_info_size):
             "Make sure a.out.stripped has no debug info",
         )
         symbols_path = self.getBuildArtifact(symbol_basename)
-        self.vscode.request_evaluate(
+        self.dap_server.request_evaluate(
             "`%s" % ('target symbols add -s "%s" "%s"' % (program, symbols_path)),
             context="repl",
         )
 
         def checkSymbolsLoadedWithSize():
-            active_modules = self.vscode.get_modules()
+            active_modules = self.dap_server.get_modules()
             program_module = active_modules[program_basename]
             self.assertIn("symbolFilePath", program_module)
             self.assertIn(symbols_path, program_module["symbolFilePath"])
@@ -51,7 +51,7 @@ def checkSymbolsLoadedWithSize():
 
         if expect_debug_info_size:
             self.waitUntil(checkSymbolsLoadedWithSize)
-        active_modules = self.vscode.get_modules()
+        active_modules = self.dap_server.get_modules()
         program_module = active_modules[program_basename]
         self.assertEqual(program_basename, program_module["name"])
         self.assertEqual(program, program_module["path"])
@@ -95,8 +95,8 @@ def test_compile_units(self):
         lines = [breakpoint1_line]
         breakpoint_ids = self.set_source_breakpoints(source, lines)
         self.continue_to_breakpoints(breakpoint_ids)
-        moduleId = self.vscode.get_modules()["a.out"]["id"]
-        response = self.vscode.request_compileUnits(moduleId)
+        moduleId = self.dap_server.get_modules()["a.out"]["id"]
+        response = self.dap_server.request_compileUnits(moduleId)
         self.assertTrue(response["body"])
         cu_paths = [cu["compileUnitPath"] for cu in response["body"]["compileUnits"]]
         self.assertIn(main_source_path, cu_paths, "Real path to main.cpp matches")
diff --git a/lldb/test/API/tools/lldb-dap/module/foo.cpp b/lldb/test/API/tools/lldb-dap/module/foo.cpp
new file mode 100644
index 0000000000..b6f33b8e07
--- /dev/null
+++ b/lldb/test/API/tools/lldb-dap/module/foo.cpp
@@ -0,0 +1 @@
+int foo() { return 12; }
diff --git a/lldb/test/API/tools/lldb-vscode/module/foo.h b/lldb/test/API/tools/lldb-dap/module/foo.h
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/module/foo.h
rename to lldb/test/API/tools/lldb-dap/module/foo.h
diff --git a/lldb/test/API/tools/lldb-vscode/module/main.cpp b/lldb/test/API/tools/lldb-dap/module/main.cpp
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/module/main.cpp
rename to lldb/test/API/tools/lldb-dap/module/main.cpp
diff --git a/lldb/test/API/tools/lldb-vscode/optimized/Makefile b/lldb/test/API/tools/lldb-dap/optimized/Makefile
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/optimized/Makefile
rename to lldb/test/API/tools/lldb-dap/optimized/Makefile
diff --git a/lldb/test/API/tools/lldb-vscode/optimized/TestVSCode_optimized.py b/lldb/test/API/tools/lldb-dap/optimized/TestDAP_optimized.py
similarity index 81%
rename from lldb/test/API/tools/lldb-vscode/optimized/TestVSCode_optimized.py
rename to lldb/test/API/tools/lldb-dap/optimized/TestDAP_optimized.py
index 3f2d0dcab7..cc544919b6 100644
--- a/lldb/test/API/tools/lldb-vscode/optimized/TestVSCode_optimized.py
+++ b/lldb/test/API/tools/lldb-dap/optimized/TestDAP_optimized.py
@@ -1,15 +1,15 @@
 """
-Test lldb-vscode variables/stackTrace request for optimized code
+Test lldb-dap variables/stackTrace request for optimized code
 """
 
-import vscode
+import dap_server
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
-import lldbvscode_testcase
+import lldbdap_testcase
 
 
-class TestVSCode_optimized(lldbvscode_testcase.VSCodeTestCaseBase):
+class TestDAP_optimized(lldbdap_testcase.DAPTestCaseBase):
     @skipIfWindows
     @skipIfRemote
     def test_stack_frame_name(self):
@@ -24,9 +24,9 @@ def test_stack_frame_name(self):
             len(breakpoint_ids), len(lines), "expect correct number of breakpoints"
         )
         self.continue_to_breakpoints(breakpoint_ids)
-        leaf_frame = self.vscode.get_stackFrame(frameIndex=0)
+        leaf_frame = self.dap_server.get_stackFrame(frameIndex=0)
         self.assertTrue(leaf_frame["name"].endswith(" [opt]"))
-        parent_frame = self.vscode.get_stackFrame(frameIndex=1)
+        parent_frame = self.dap_server.get_stackFrame(frameIndex=1)
         self.assertTrue(parent_frame["name"].endswith(" [opt]"))
 
     @skipIfWindows
@@ -44,6 +44,6 @@ def test_optimized_variable(self):
             len(breakpoint_ids), len(lines), "expect correct number of breakpoints"
         )
         self.continue_to_breakpoints(breakpoint_ids)
-        optimized_variable = self.vscode.get_local_variable("argc")
+        optimized_variable = self.dap_server.get_local_variable("argc")
 
         self.assertTrue(optimized_variable["value"].startswith("<error:"))
diff --git a/lldb/test/API/tools/lldb-vscode/optimized/main.cpp b/lldb/test/API/tools/lldb-dap/optimized/main.cpp
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/optimized/main.cpp
rename to lldb/test/API/tools/lldb-dap/optimized/main.cpp
diff --git a/lldb/test/API/tools/lldb-vscode/restart/Makefile b/lldb/test/API/tools/lldb-dap/restart/Makefile
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/restart/Makefile
rename to lldb/test/API/tools/lldb-dap/restart/Makefile
diff --git a/lldb/test/API/tools/lldb-vscode/restart/TestVSCode_restart.py b/lldb/test/API/tools/lldb-dap/restart/TestDAP_restart.py
similarity index 79%
rename from lldb/test/API/tools/lldb-vscode/restart/TestVSCode_restart.py
rename to lldb/test/API/tools/lldb-dap/restart/TestDAP_restart.py
index ec541a84f7..b622792db5 100644
--- a/lldb/test/API/tools/lldb-vscode/restart/TestVSCode_restart.py
+++ b/lldb/test/API/tools/lldb-dap/restart/TestDAP_restart.py
@@ -1,13 +1,13 @@
 """
-Test lldb-vscode RestartRequest.
+Test lldb-dap RestartRequest.
 """
 
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import line_number
-import lldbvscode_testcase
+import lldbdap_testcase
 
 
-class TestVSCode_restart(lldbvscode_testcase.VSCodeTestCaseBase):
+class TestDAP_restart(lldbdap_testcase.DAPTestCaseBase):
     @skipIfWindows
     @skipIfRemote
     def test_basic_functionality(self):
@@ -23,23 +23,23 @@ def test_basic_functionality(self):
         [bp_A, bp_B] = self.set_source_breakpoints("main.c", [line_A, line_B])
 
         # Verify we hit A, then B.
-        self.vscode.request_configurationDone()
+        self.dap_server.request_configurationDone()
         self.verify_breakpoint_hit([bp_A])
-        self.vscode.request_continue()
+        self.dap_server.request_continue()
         self.verify_breakpoint_hit([bp_B])
 
         # Make sure i has been modified from its initial value of 0.
         self.assertEquals(
-            int(self.vscode.get_local_variable_value("i")),
+            int(self.dap_server.get_local_variable_value("i")),
             1234,
             "i != 1234 after hitting breakpoint B",
         )
 
         # Restart then check we stop back at A and program state has been reset.
-        self.vscode.request_restart()
+        self.dap_server.request_restart()
         self.verify_breakpoint_hit([bp_A])
         self.assertEquals(
-            int(self.vscode.get_local_variable_value("i")),
+            int(self.dap_server.get_local_variable_value("i")),
             0,
             "i != 0 after hitting breakpoint A on restart",
         )
@@ -53,11 +53,11 @@ def test_stopOnEntry(self):
         program = self.getBuildArtifact("a.out")
         self.build_and_launch(program, stopOnEntry=True)
         [bp_main] = self.set_function_breakpoints(["main"])
-        self.vscode.request_configurationDone()
+        self.dap_server.request_configurationDone()
 
         # Once the "configuration done" event is sent, we should get a stopped
         # event immediately because of stopOnEntry.
-        stopped_events = self.vscode.wait_for_stopped()
+        stopped_events = self.dap_server.wait_for_stopped()
         for stopped_event in stopped_events:
             if "body" in stopped_event:
                 body = stopped_event["body"]
@@ -68,13 +68,13 @@ def test_stopOnEntry(self):
                     )
 
         # Then, if we continue, we should hit the breakpoint at main.
-        self.vscode.request_continue()
+        self.dap_server.request_continue()
         self.verify_breakpoint_hit([bp_main])
 
         # Restart and check that we still get a stopped event before reaching
         # main.
-        self.vscode.request_restart()
-        stopped_events = self.vscode.wait_for_stopped()
+        self.dap_server.request_restart()
+        stopped_events = self.dap_server.wait_for_stopped()
         for stopped_event in stopped_events:
             if "body" in stopped_event:
                 body = stopped_event["body"]
@@ -90,7 +90,7 @@ def test_stopOnEntry(self):
     @skipIfRemote
     def test_arguments(self):
         """
-        Tests that lldb-vscode will use updated launch arguments included
+        Tests that lldb-dap will use updated launch arguments included
         with a restart request.
         """
         line_A = line_number("main.c", "// breakpoint A")
@@ -100,20 +100,20 @@ def test_arguments(self):
         [bp_A] = self.set_source_breakpoints("main.c", [line_A])
 
         # Verify we hit A, then B.
-        self.vscode.request_configurationDone()
+        self.dap_server.request_configurationDone()
         self.verify_breakpoint_hit([bp_A])
 
         # We don't set any arguments in the initial launch request, so argc
         # should be 1.
         self.assertEquals(
-            int(self.vscode.get_local_variable_value("argc")),
+            int(self.dap_server.get_local_variable_value("argc")),
             1,
             "argc != 1 before restart",
         )
 
         # Restart with some extra 'args' and check that the new argc reflects
         # the updated launch config.
-        self.vscode.request_restart(
+        self.dap_server.request_restart(
             restartArguments={
                 "arguments": {
                     "program": program,
@@ -123,7 +123,7 @@ def test_arguments(self):
         )
         self.verify_breakpoint_hit([bp_A])
         self.assertEquals(
-            int(self.vscode.get_local_variable_value("argc")),
+            int(self.dap_server.get_local_variable_value("argc")),
             5,
             "argc != 5 after restart",
         )
diff --git a/lldb/test/API/tools/lldb-vscode/restart/TestVSCode_restart_runInTerminal.py b/lldb/test/API/tools/lldb-dap/restart/TestDAP_restart_runInTerminal.py
similarity index 80%
rename from lldb/test/API/tools/lldb-vscode/restart/TestVSCode_restart_runInTerminal.py
rename to lldb/test/API/tools/lldb-dap/restart/TestDAP_restart_runInTerminal.py
index 38eebc92bc..789f7b817d 100644
--- a/lldb/test/API/tools/lldb-vscode/restart/TestVSCode_restart_runInTerminal.py
+++ b/lldb/test/API/tools/lldb-dap/restart/TestDAP_restart_runInTerminal.py
@@ -1,22 +1,22 @@
 """
-Test lldb-vscode RestartRequest.
+Test lldb-dap RestartRequest.
 """
 
 import os
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import line_number
-import lldbvscode_testcase
+import lldbdap_testcase
 
 
-class TestVSCode_restart_runInTerminal(lldbvscode_testcase.VSCodeTestCaseBase):
+class TestDAP_restart_runInTerminal(lldbdap_testcase.DAPTestCaseBase):
     def isTestSupported(self):
         try:
             # We skip this test for debug builds because it takes too long
             # parsing lldb's own debug info. Release builds are fine.
-            # Checking the size of the lldb-vscode binary seems to be a decent
+            # Checking the size of the lldb-dap binary seems to be a decent
             # proxy for a quick detection. It should be far less than 1 MB in
             # Release builds.
-            return os.path.getsize(os.environ["LLDBVSCODE_EXEC"]) < 1000000
+            return os.path.getsize(os.environ["LLDBDAP_EXEC"]) < 1000000
         except:
             return False
 
@@ -38,25 +38,25 @@ def test_basic_functionality(self):
         [bp_A, bp_B] = self.set_source_breakpoints("main.c", [line_A, line_B])
 
         # Verify we hit A, then B.
-        self.vscode.request_configurationDone()
+        self.dap_server.request_configurationDone()
         self.verify_breakpoint_hit([bp_A])
-        self.vscode.request_continue()
+        self.dap_server.request_continue()
         self.verify_breakpoint_hit([bp_B])
 
         # Make sure i has been modified from its initial value of 0.
         self.assertEquals(
-            int(self.vscode.get_local_variable_value("i")),
+            int(self.dap_server.get_local_variable_value("i")),
             1234,
             "i != 1234 after hitting breakpoint B",
         )
 
         # Restart.
-        self.vscode.request_restart()
+        self.dap_server.request_restart()
 
         # Finally, check we stop back at A and program state has been reset.
         self.verify_breakpoint_hit([bp_A])
         self.assertEquals(
-            int(self.vscode.get_local_variable_value("i")),
+            int(self.dap_server.get_local_variable_value("i")),
             0,
             "i != 0 after hitting breakpoint A on restart",
         )
@@ -76,11 +76,11 @@ def test_stopOnEntry(self):
         program = self.getBuildArtifact("a.out")
         self.build_and_launch(program, runInTerminal=True, stopOnEntry=True)
         [bp_main] = self.set_function_breakpoints(["main"])
-        self.vscode.request_configurationDone()
+        self.dap_server.request_configurationDone()
 
         # When using stopOnEntry, configurationDone doesn't result in a running
         # process, we should immediately get a stopped event instead.
-        stopped_events = self.vscode.wait_for_stopped()
+        stopped_events = self.dap_server.wait_for_stopped()
         # We should be stopped at the entry point.
         for stopped_event in stopped_events:
             if "body" in stopped_event:
@@ -92,13 +92,13 @@ def test_stopOnEntry(self):
                     )
 
         # Then, if we continue, we should hit the breakpoint at main.
-        self.vscode.request_continue()
+        self.dap_server.request_continue()
         self.verify_breakpoint_hit([bp_main])
 
         # Restart and check that we still get a stopped event before reaching
         # main.
-        self.vscode.request_restart()
-        stopped_events = self.vscode.wait_for_stopped()
+        self.dap_server.request_restart()
+        stopped_events = self.dap_server.wait_for_stopped()
         for stopped_event in stopped_events:
             if "body" in stopped_event:
                 body = stopped_event["body"]
diff --git a/lldb/test/API/tools/lldb-vscode/restart/main.c b/lldb/test/API/tools/lldb-dap/restart/main.c
similarity index 86%
rename from lldb/test/API/tools/lldb-vscode/restart/main.c
rename to lldb/test/API/tools/lldb-dap/restart/main.c
index 710af8e975..b02928a694 100644
--- a/lldb/test/API/tools/lldb-vscode/restart/main.c
+++ b/lldb/test/API/tools/lldb-dap/restart/main.c
@@ -5,5 +5,5 @@ int main(int argc, char const *argv[], char const *envp[]) {
   printf("Do something\n"); // breakpoint A
   printf("Do something else\n");
   i = 1234;
-  return 0;  // breakpoint B
+  return 0; // breakpoint B
 }
diff --git a/lldb/test/API/tools/lldb-vscode/runInTerminal/Makefile b/lldb/test/API/tools/lldb-dap/runInTerminal/Makefile
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/runInTerminal/Makefile
rename to lldb/test/API/tools/lldb-dap/runInTerminal/Makefile
diff --git a/lldb/test/API/tools/lldb-vscode/runInTerminal/TestVSCode_runInTerminal.py b/lldb/test/API/tools/lldb-dap/runInTerminal/TestDAP_runInTerminal.py
similarity index 84%
rename from lldb/test/API/tools/lldb-vscode/runInTerminal/TestVSCode_runInTerminal.py
rename to lldb/test/API/tools/lldb-dap/runInTerminal/TestDAP_runInTerminal.py
index a6919f1f63..a72571898a 100644
--- a/lldb/test/API/tools/lldb-vscode/runInTerminal/TestVSCode_runInTerminal.py
+++ b/lldb/test/API/tools/lldb-dap/runInTerminal/TestDAP_runInTerminal.py
@@ -1,13 +1,13 @@
 """
-Test lldb-vscode runInTerminal reverse request
+Test lldb-dap runInTerminal reverse request
 """
 
 
-import vscode
+import dap_server
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
-import lldbvscode_testcase
+import lldbdap_testcase
 import time
 import os
 import subprocess
@@ -16,7 +16,7 @@
 from threading import Thread
 
 
-class TestVSCode_runInTerminal(lldbvscode_testcase.VSCodeTestCaseBase):
+class TestDAP_runInTerminal(lldbdap_testcase.DAPTestCaseBase):
     def readPidMessage(self, fifo_file):
         with open(fifo_file, "r") as file:
             self.assertIn("pid", file.readline())
@@ -36,9 +36,9 @@ def isTestSupported(self):
         try:
             # We skip this test for debug builds because it takes too long parsing lldb's own
             # debug info. Release builds are fine.
-            # Checking the size of the lldb-vscode binary seems to be a decent proxy for a quick
+            # Checking the size of the lldb-dap binary seems to be a decent proxy for a quick
             # detection. It should be far less than 1 MB in Release builds.
-            if os.path.getsize(os.environ["LLDBVSCODE_EXEC"]) < 1000000:
+            if os.path.getsize(os.environ["LLDBDAP_EXEC"]) < 1000000:
                 return True
         except:
             return False
@@ -60,11 +60,13 @@ def test_runInTerminal(self):
         )
 
         self.assertEqual(
-            len(self.vscode.reverse_requests), 1, "make sure we got a reverse request"
+            len(self.dap_server.reverse_requests),
+            1,
+            "make sure we got a reverse request",
         )
 
-        request = self.vscode.reverse_requests[0]
-        self.assertIn(self.lldbVSCodeExec, request["arguments"]["args"])
+        request = self.dap_server.reverse_requests[0]
+        self.assertIn(self.lldbDAPExec, request["arguments"]["args"])
         self.assertIn(program, request["arguments"]["args"])
         self.assertIn("foobar", request["arguments"]["args"])
         self.assertIn("FOO", request["arguments"]["env"])
@@ -75,18 +77,18 @@ def test_runInTerminal(self):
         self.continue_to_next_stop()
 
         # We verify we actually stopped inside the loop
-        counter = int(self.vscode.get_local_variable_value("counter"))
+        counter = int(self.dap_server.get_local_variable_value("counter"))
         self.assertTrue(counter > 0)
 
         # We verify we were able to set the launch arguments
-        argc = int(self.vscode.get_local_variable_value("argc"))
+        argc = int(self.dap_server.get_local_variable_value("argc"))
         self.assertEqual(argc, 2)
 
-        argv1 = self.vscode.request_evaluate("argv[1]")["body"]["result"]
+        argv1 = self.dap_server.request_evaluate("argv[1]")["body"]["result"]
         self.assertIn("foobar", argv1)
 
         # We verify we were able to set the environment
-        env = self.vscode.request_evaluate("foo")["body"]["result"]
+        env = self.dap_server.request_evaluate("foo")["body"]["result"]
         self.assertIn("bar", env)
 
     @skipIfWindows
@@ -116,7 +118,7 @@ def test_missingArgInRunInTerminalLauncher(self):
         if not self.isTestSupported():
             return
         proc = subprocess.run(
-            [self.lldbVSCodeExec, "--launch-target", "INVALIDPROGRAM"],
+            [self.lldbDAPExec, "--launch-target", "INVALIDPROGRAM"],
             capture_output=True,
             universal_newlines=True,
         )
@@ -136,7 +138,7 @@ def test_FakeAttachedRunInTerminalLauncherWithInvalidProgram(self):
 
         proc = subprocess.Popen(
             [
-                self.lldbVSCodeExec,
+                self.lldbDAPExec,
                 "--comm-file",
                 comm_file,
                 "--launch-target",
@@ -164,7 +166,7 @@ def test_FakeAttachedRunInTerminalLauncherWithValidProgram(self):
 
         proc = subprocess.Popen(
             [
-                self.lldbVSCodeExec,
+                self.lldbDAPExec,
                 "--comm-file",
                 comm_file,
                 "--launch-target",
@@ -191,7 +193,7 @@ def test_FakeAttachedRunInTerminalLauncherAndCheckEnvironment(self):
         os.mkfifo(comm_file)
 
         proc = subprocess.Popen(
-            [self.lldbVSCodeExec, "--comm-file", comm_file, "--launch-target", "env"],
+            [self.lldbDAPExec, "--comm-file", comm_file, "--launch-target", "env"],
             universal_newlines=True,
             stdout=subprocess.PIPE,
             env={**os.environ, "FOO": "BAR"},
@@ -214,7 +216,7 @@ def test_NonAttachedRunInTerminalLauncher(self):
 
         proc = subprocess.Popen(
             [
-                self.lldbVSCodeExec,
+                self.lldbDAPExec,
                 "--comm-file",
                 comm_file,
                 "--launch-target",
@@ -223,7 +225,7 @@ def test_NonAttachedRunInTerminalLauncher(self):
             ],
             universal_newlines=True,
             stderr=subprocess.PIPE,
-            env={**os.environ, "LLDB_VSCODE_RIT_TIMEOUT_IN_MS": "1000"},
+            env={**os.environ, "LLDB_DAP_RIT_TIMEOUT_IN_MS": "1000"},
         )
 
         self.readPidMessage(comm_file)
diff --git a/lldb/test/API/tools/lldb-vscode/runInTerminal/main.c b/lldb/test/API/tools/lldb-dap/runInTerminal/main.c
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/runInTerminal/main.c
rename to lldb/test/API/tools/lldb-dap/runInTerminal/main.c
diff --git a/lldb/test/API/tools/lldb-vscode/stackTrace/Makefile b/lldb/test/API/tools/lldb-dap/stackTrace/Makefile
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/stackTrace/Makefile
rename to lldb/test/API/tools/lldb-dap/stackTrace/Makefile
diff --git a/lldb/test/API/tools/lldb-vscode/stackTrace/TestVSCode_stackTrace.py b/lldb/test/API/tools/lldb-dap/stackTrace/TestDAP_stackTrace.py
similarity index 97%
rename from lldb/test/API/tools/lldb-vscode/stackTrace/TestVSCode_stackTrace.py
rename to lldb/test/API/tools/lldb-dap/stackTrace/TestDAP_stackTrace.py
index fa9e9f30ce..245b3f34b7 100644
--- a/lldb/test/API/tools/lldb-vscode/stackTrace/TestVSCode_stackTrace.py
+++ b/lldb/test/API/tools/lldb-dap/stackTrace/TestDAP_stackTrace.py
@@ -1,17 +1,17 @@
 """
-Test lldb-vscode setBreakpoints request
+Test lldb-dap setBreakpoints request
 """
 
 
-import vscode
+import dap_server
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
-import lldbvscode_testcase
+import lldbdap_testcase
 import os
 
 
-class TestVSCode_stackTrace(lldbvscode_testcase.VSCodeTestCaseBase):
+class TestDAP_stackTrace(lldbdap_testcase.DAPTestCaseBase):
     name_key_path = ["name"]
     source_key_path = ["source", "path"]
     line_key_path = ["line"]
diff --git a/lldb/test/API/tools/lldb-vscode/stackTrace/main.c b/lldb/test/API/tools/lldb-dap/stackTrace/main.c
similarity index 65%
rename from lldb/test/API/tools/lldb-vscode/stackTrace/main.c
rename to lldb/test/API/tools/lldb-dap/stackTrace/main.c
index 85b41c4928..862473a3e6 100644
--- a/lldb/test/API/tools/lldb-vscode/stackTrace/main.c
+++ b/lldb/test/API/tools/lldb-dap/stackTrace/main.c
@@ -3,8 +3,8 @@
 
 int recurse(int x) {
   if (x <= 1)
-    return 1; // recurse end
-  return recurse(x-1) + x; // recurse call
+    return 1;                // recurse end
+  return recurse(x - 1) + x; // recurse call
 }
 
 int main(int argc, char const *argv[]) {
diff --git a/lldb/test/API/tools/lldb-vscode/stackTraceMissingFunctionName/Makefile b/lldb/test/API/tools/lldb-dap/stackTraceMissingFunctionName/Makefile
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/stackTraceMissingFunctionName/Makefile
rename to lldb/test/API/tools/lldb-dap/stackTraceMissingFunctionName/Makefile
diff --git a/lldb/test/API/tools/lldb-vscode/stackTraceMissingFunctionName/TestVSCode_stackTraceMissingFunctionName.py b/lldb/test/API/tools/lldb-dap/stackTraceMissingFunctionName/TestDAP_stackTraceMissingFunctionName.py
similarity index 77%
rename from lldb/test/API/tools/lldb-vscode/stackTraceMissingFunctionName/TestVSCode_stackTraceMissingFunctionName.py
rename to lldb/test/API/tools/lldb-dap/stackTraceMissingFunctionName/TestDAP_stackTraceMissingFunctionName.py
index 8e7da6386d..344629c1d5 100644
--- a/lldb/test/API/tools/lldb-vscode/stackTraceMissingFunctionName/TestVSCode_stackTraceMissingFunctionName.py
+++ b/lldb/test/API/tools/lldb-dap/stackTraceMissingFunctionName/TestDAP_stackTraceMissingFunctionName.py
@@ -1,17 +1,17 @@
 """
-Test lldb-vscode stack trace response
+Test lldb-dap stack trace response
 """
 
 
-import vscode
+import dap_server
 from lldbsuite.test.decorators import *
 import os
 
-import lldbvscode_testcase
+import lldbdap_testcase
 from lldbsuite.test import lldbtest, lldbutil
 
 
-class TestVSCode_stackTraceMissingFunctionName(lldbvscode_testcase.VSCodeTestCaseBase):
+class TestDAP_stackTraceMissingFunctionName(lldbdap_testcase.DAPTestCaseBase):
     @skipIfWindows
     @skipIfRemote
     def test_missingFunctionName(self):
diff --git a/lldb/test/API/tools/lldb-vscode/stackTraceMissingFunctionName/main.cpp b/lldb/test/API/tools/lldb-dap/stackTraceMissingFunctionName/main.cpp
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/stackTraceMissingFunctionName/main.cpp
rename to lldb/test/API/tools/lldb-dap/stackTraceMissingFunctionName/main.cpp
diff --git a/lldb/test/API/tools/lldb-vscode/startDebugging/Makefile b/lldb/test/API/tools/lldb-dap/startDebugging/Makefile
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/startDebugging/Makefile
rename to lldb/test/API/tools/lldb-dap/startDebugging/Makefile
diff --git a/lldb/test/API/tools/lldb-vscode/startDebugging/TestVSCode_startDebugging.py b/lldb/test/API/tools/lldb-dap/startDebugging/TestDAP_startDebugging.py
similarity index 64%
rename from lldb/test/API/tools/lldb-vscode/startDebugging/TestVSCode_startDebugging.py
rename to lldb/test/API/tools/lldb-dap/startDebugging/TestDAP_startDebugging.py
index d8bc910c35..fd48e69cae 100644
--- a/lldb/test/API/tools/lldb-vscode/startDebugging/TestVSCode_startDebugging.py
+++ b/lldb/test/API/tools/lldb-dap/startDebugging/TestDAP_startDebugging.py
@@ -1,16 +1,16 @@
 """
-Test lldb-vscode startDebugging reverse request
+Test lldb-dap startDebugging reverse request
 """
 
 
-import vscode
+import dap_server
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
-import lldbvscode_testcase
+import lldbdap_testcase
 
 
-class TestVSCode_startDebugging(lldbvscode_testcase.VSCodeTestCaseBase):
+class TestDAP_startDebugging(lldbdap_testcase.DAPTestCaseBase):
     def test_startDebugging(self):
         """
         Tests the "startDebugging" reverse request. It makes sure that the IDE can
@@ -24,16 +24,18 @@ def test_startDebugging(self):
 
         self.set_source_breakpoints(source, [breakpoint_line])
         self.continue_to_next_stop()
-        self.vscode.request_evaluate(
-            "`lldb-vscode startDebugging attach '{\"pid\":321}'", context="repl"
+        self.dap_server.request_evaluate(
+            "`lldb-dap startDebugging attach '{\"pid\":321}'", context="repl"
         )
 
         self.continue_to_exit()
 
         self.assertEqual(
-            len(self.vscode.reverse_requests), 1, "make sure we got a reverse request"
+            len(self.dap_server.reverse_requests),
+            1,
+            "make sure we got a reverse request",
         )
 
-        request = self.vscode.reverse_requests[0]
+        request = self.dap_server.reverse_requests[0]
         self.assertEqual(request["arguments"]["configuration"]["pid"], 321)
         self.assertEqual(request["arguments"]["request"], "attach")
diff --git a/lldb/test/API/tools/lldb-vscode/startDebugging/main.c b/lldb/test/API/tools/lldb-dap/startDebugging/main.c
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/startDebugging/main.c
rename to lldb/test/API/tools/lldb-dap/startDebugging/main.c
diff --git a/lldb/test/API/tools/lldb-vscode/step/Makefile b/lldb/test/API/tools/lldb-dap/step/Makefile
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/step/Makefile
rename to lldb/test/API/tools/lldb-dap/step/Makefile
diff --git a/lldb/test/API/tools/lldb-vscode/step/TestVSCode_step.py b/lldb/test/API/tools/lldb-dap/step/TestDAP_step.py
similarity index 94%
rename from lldb/test/API/tools/lldb-vscode/step/TestVSCode_step.py
rename to lldb/test/API/tools/lldb-dap/step/TestDAP_step.py
index 14dd20d0b8..578e64e36e 100644
--- a/lldb/test/API/tools/lldb-vscode/step/TestVSCode_step.py
+++ b/lldb/test/API/tools/lldb-dap/step/TestDAP_step.py
@@ -1,16 +1,16 @@
 """
-Test lldb-vscode setBreakpoints request
+Test lldb-dap setBreakpoints request
 """
 
 
-import vscode
+import dap_server
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
-import lldbvscode_testcase
+import lldbdap_testcase
 
 
-class TestVSCode_step(lldbvscode_testcase.VSCodeTestCaseBase):
+class TestDAP_step(lldbdap_testcase.DAPTestCaseBase):
     @skipIfWindows
     @skipIfRemote
     def test_step(self):
@@ -29,7 +29,7 @@ def test_step(self):
             len(breakpoint_ids), len(lines), "expect correct number of breakpoints"
         )
         self.continue_to_breakpoints(breakpoint_ids)
-        threads = self.vscode.get_threads()
+        threads = self.dap_server.get_threads()
         for thread in threads:
             if "reason" in thread:
                 reason = thread["reason"]
diff --git a/lldb/test/API/tools/lldb-dap/step/main.cpp b/lldb/test/API/tools/lldb-dap/step/main.cpp
new file mode 100644
index 0000000000..8905beb5e7
--- /dev/null
+++ b/lldb/test/API/tools/lldb-dap/step/main.cpp
@@ -0,0 +1,8 @@
+int function(int x) {
+  if ((x % 2) == 0)
+    return function(x - 1) + x; // breakpoint 1
+  else
+    return x;
+}
+
+int main(int argc, char const *argv[]) { return function(2); }
diff --git a/lldb/test/API/tools/lldb-vscode/stop-hooks/Makefile b/lldb/test/API/tools/lldb-dap/stop-hooks/Makefile
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/stop-hooks/Makefile
rename to lldb/test/API/tools/lldb-dap/stop-hooks/Makefile
diff --git a/lldb/test/API/tools/lldb-vscode/stop-hooks/TestVSCode_stop_hooks.py b/lldb/test/API/tools/lldb-dap/stop-hooks/TestDAP_stop_hooks.py
similarity index 82%
rename from lldb/test/API/tools/lldb-vscode/stop-hooks/TestVSCode_stop_hooks.py
rename to lldb/test/API/tools/lldb-dap/stop-hooks/TestDAP_stop_hooks.py
index 16a09f9d14..c538e8002a 100644
--- a/lldb/test/API/tools/lldb-vscode/stop-hooks/TestVSCode_stop_hooks.py
+++ b/lldb/test/API/tools/lldb-dap/stop-hooks/TestDAP_stop_hooks.py
@@ -5,14 +5,14 @@
 
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
-import lldbvscode_testcase
+import lldbdap_testcase
 
 
-class TestVSCode_stop_hooks(lldbvscode_testcase.VSCodeTestCaseBase):
+class TestDAP_stop_hooks(lldbdap_testcase.DAPTestCaseBase):
     @skipIfRemote
     def test_stop_hooks_before_run(self):
         """
-        Test that there is no race condition between lldb-vscode and
+        Test that there is no race condition between lldb-dap and
         stop hooks executor
         """
         program = self.getBuildArtifact("a.out")
@@ -24,9 +24,9 @@ def test_stop_hooks_before_run(self):
 
         breakpoint_ids = self.set_function_breakpoints(["main"])
         # This request hangs if the race happens, because, in that case, the
-        # command interpreter is in synchronous mode while lldb-vscode expects
+        # command interpreter is in synchronous mode while lldb-dap expects
         # it to be in asynchronous mode, so, the process doesn't send the stop
-        # event to "lldb.Debugger" listener (which is monitored by lldb-vscode).
+        # event to "lldb.Debugger" listener (which is monitored by lldb-dap).
         self.continue_to_breakpoints(breakpoint_ids)
 
         self.continue_to_exit()
diff --git a/lldb/test/API/tools/lldb-vscode/stop-hooks/main.c b/lldb/test/API/tools/lldb-dap/stop-hooks/main.c
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/stop-hooks/main.c
rename to lldb/test/API/tools/lldb-dap/stop-hooks/main.c
diff --git a/lldb/test/API/tools/lldb-vscode/terminated-event/Makefile b/lldb/test/API/tools/lldb-dap/terminated-event/Makefile
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/terminated-event/Makefile
rename to lldb/test/API/tools/lldb-dap/terminated-event/Makefile
diff --git a/lldb/test/API/tools/lldb-vscode/terminated-event/TestVSCode_terminatedEvent.py b/lldb/test/API/tools/lldb-dap/terminated-event/TestDAP_terminatedEvent.py
similarity index 89%
rename from lldb/test/API/tools/lldb-vscode/terminated-event/TestVSCode_terminatedEvent.py
rename to lldb/test/API/tools/lldb-dap/terminated-event/TestDAP_terminatedEvent.py
index 525de0b120..78bc14c380 100644
--- a/lldb/test/API/tools/lldb-vscode/terminated-event/TestVSCode_terminatedEvent.py
+++ b/lldb/test/API/tools/lldb-dap/terminated-event/TestDAP_terminatedEvent.py
@@ -1,17 +1,17 @@
 """
-Test lldb-vscode terminated event
+Test lldb-dap terminated event
 """
 
-import vscode
+import dap_server
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
-import lldbvscode_testcase
+import lldbdap_testcase
 import re
 import json
 
 
-class TestVSCode_terminatedEvent(lldbvscode_testcase.VSCodeTestCaseBase):
+class TestDAP_terminatedEvent(lldbdap_testcase.DAPTestCaseBase):
     @skipIfWindows
     @skipIfRemote
     def test_terminated_event(self):
@@ -44,7 +44,7 @@ def test_terminated_event(self):
         self.continue_to_breakpoints(breakpoint_ids)
         self.continue_to_exit()
 
-        statistics = self.vscode.wait_for_terminated()["statistics"]
+        statistics = self.dap_server.wait_for_terminated()["statistics"]
         self.assertTrue(statistics["totalDebugInfoByteSize"] > 0)
         self.assertTrue(statistics["totalDebugInfoEnabled"] > 0)
         self.assertTrue(statistics["totalModuleCountHasDebugInfo"] > 0)
@@ -52,7 +52,7 @@ def test_terminated_event(self):
         self.assertIsNotNone(statistics["memory"])
         self.assertNotIn("modules", statistics.keys())
 
-        # lldb-vscode debugs one target at a time
+        # lldb-dap debugs one target at a time
         target = json.loads(statistics["targets"])[0]
         self.assertTrue(target["totalBreakpointResolveTime"] > 0)
 
diff --git a/lldb/test/API/tools/lldb-dap/terminated-event/foo.cpp b/lldb/test/API/tools/lldb-dap/terminated-event/foo.cpp
new file mode 100644
index 0000000000..b6f33b8e07
--- /dev/null
+++ b/lldb/test/API/tools/lldb-dap/terminated-event/foo.cpp
@@ -0,0 +1 @@
+int foo() { return 12; }
diff --git a/lldb/test/API/tools/lldb-vscode/terminated-event/foo.h b/lldb/test/API/tools/lldb-dap/terminated-event/foo.h
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/terminated-event/foo.h
rename to lldb/test/API/tools/lldb-dap/terminated-event/foo.h
diff --git a/lldb/test/API/tools/lldb-vscode/terminated-event/main.cpp b/lldb/test/API/tools/lldb-dap/terminated-event/main.cpp
similarity index 100%
rename from lldb/test/API/tools/lldb-vscode/terminated-event/main.cpp
rename to lldb/test/API/tools/lldb-dap/terminated-event/main.cpp
index cd984e560e..50dd77c0a9 100644
--- a/lldb/test/API/tools/lldb-vscode/terminated-event/main.cpp
+++ b/lldb/test/API/tools/lldb-dap/terminated-event/main.cpp
@@ -1,5 +1,5 @@
-#include <iostream>
 #include "foo.h"
+#include <iostream>
 
 int main(int argc, char const *argv[]) {
   std::cout << "Hello World!" << std::endl; // main breakpoint 1
diff --git a/lldb/test/API/tools/lldb-dap/variables/Makefile b/lldb/test/API/tools/lldb-dap/variables/Makefile
new file mode 100644
index 0000000000..99998b20bc
--- /dev/null
+++ b/lldb/test/API/tools/lldb-dap/variables/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+
+include Makefile.rules
diff --git a/lldb/test/API/tools/lldb-vscode/variables/TestVSCode_variables.py b/lldb/test/API/tools/lldb-dap/variables/TestDAP_variables.py
similarity index 89%
rename from lldb/test/API/tools/lldb-vscode/variables/TestVSCode_variables.py
rename to lldb/test/API/tools/lldb-dap/variables/TestDAP_variables.py
index fc24b3b34e..11d9bf18db 100644
--- a/lldb/test/API/tools/lldb-vscode/variables/TestVSCode_variables.py
+++ b/lldb/test/API/tools/lldb-dap/variables/TestDAP_variables.py
@@ -1,11 +1,11 @@
 """
-Test lldb-vscode setBreakpoints request
+Test lldb-dap setBreakpoints request
 """
 
 import os
 
-import lldbvscode_testcase
-import vscode
+import lldbdap_testcase
+import dap_server
 from lldbsuite.test import lldbutil
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
@@ -18,7 +18,7 @@ def make_buffer_verify_dict(start_idx, count, offset=0):
     return verify_dict
 
 
-class TestVSCode_variables(lldbvscode_testcase.VSCodeTestCaseBase):
+class TestDAP_variables(lldbdap_testcase.DAPTestCaseBase):
     def verify_values(self, verify_dict, actual, varref_dict=None, expression=None):
         if "equals" in verify_dict:
             verify = verify_dict["equals"]
@@ -79,7 +79,7 @@ def verify_values(self, verify_dict, actual, varref_dict=None, expression=None):
                 ("children verify values specified for " "variable without children"),
             )
 
-            response = self.vscode.request_variables(varRef)
+            response = self.dap_server.request_variables(varRef)
             self.verify_variables(
                 verify_dict["children"], response["body"]["variables"], varref_dict
             )
@@ -111,7 +111,7 @@ def darwin_dwarf_missing_obj(self, initCommands):
         self.assertEquals(len(breakpoint_ids), len(functions), "expect one breakpoint")
         self.continue_to_breakpoints(breakpoint_ids)
 
-        locals = self.vscode.get_local_variables()
+        locals = self.dap_server.get_local_variables()
 
         verify_locals = {
             "<error>": {
@@ -147,8 +147,8 @@ def do_test_scopes_variables_setVariable_evaluate(
             len(breakpoint_ids), len(lines), "expect correct number of breakpoints"
         )
         self.continue_to_breakpoints(breakpoint_ids)
-        locals = self.vscode.get_local_variables()
-        globals = self.vscode.get_global_variables()
+        locals = self.dap_server.get_local_variables()
+        globals = self.dap_server.get_global_variables()
         buffer_children = make_buffer_verify_dict(0, 32)
         verify_locals = {
             "argc": {"equals": {"type": "int", "value": "1"}},
@@ -181,28 +181,28 @@ def do_test_scopes_variables_setVariable_evaluate(
         # has optional parameters like "start" and "count" to limit the number
         # of variables that are fetched
         varRef = varref_dict["pt.buffer"]
-        response = self.vscode.request_variables(varRef)
+        response = self.dap_server.request_variables(varRef)
         self.verify_variables(buffer_children, response["body"]["variables"])
         # Verify setting start=0 in the arguments still gets all children
-        response = self.vscode.request_variables(varRef, start=0)
+        response = self.dap_server.request_variables(varRef, start=0)
         self.verify_variables(buffer_children, response["body"]["variables"])
         # Verify setting count=0 in the arguments still gets all children.
         # If count is zero, it means to get all children.
-        response = self.vscode.request_variables(varRef, count=0)
+        response = self.dap_server.request_variables(varRef, count=0)
         self.verify_variables(buffer_children, response["body"]["variables"])
         # Verify setting count to a value that is too large in the arguments
         # still gets all children, and no more
-        response = self.vscode.request_variables(varRef, count=1000)
+        response = self.dap_server.request_variables(varRef, count=1000)
         self.verify_variables(buffer_children, response["body"]["variables"])
         # Verify setting the start index and count gets only the children we
         # want
-        response = self.vscode.request_variables(varRef, start=5, count=5)
+        response = self.dap_server.request_variables(varRef, start=5, count=5)
         self.verify_variables(
             make_buffer_verify_dict(5, 5), response["body"]["variables"]
         )
         # Verify setting the start index to a value that is out of range
         # results in an empty list
-        response = self.vscode.request_variables(varRef, start=32, count=1)
+        response = self.dap_server.request_variables(varRef, start=32, count=1)
         self.assertEqual(
             len(response["body"]["variables"]),
             0,
@@ -253,7 +253,7 @@ def do_test_scopes_variables_setVariable_evaluate(
             },
         }
         for expression in expressions:
-            response = self.vscode.request_evaluate(expression)
+            response = self.dap_server.request_evaluate(expression)
             self.verify_values(expressions[expression], response["body"])
 
         # Test setting variables
@@ -269,8 +269,8 @@ def do_test_scopes_variables_setVariable_evaluate(
 
         # Set a variable value whose name is synthetic, like a variable index
         # and verify the value by reading it
-        self.vscode.request_setVariable(varRef, "[0]", 100)
-        response = self.vscode.request_variables(varRef, start=0, count=1)
+        self.dap_server.request_setVariable(varRef, "[0]", 100)
+        response = self.dap_server.request_variables(varRef, start=0, count=1)
         self.verify_variables(
             make_buffer_verify_dict(0, 1, 100), response["body"]["variables"]
         )
@@ -278,8 +278,8 @@ def do_test_scopes_variables_setVariable_evaluate(
         # Set a variable value whose name is a real child value, like "pt.x"
         # and verify the value by reading it
         varRef = varref_dict["pt"]
-        self.vscode.request_setVariable(varRef, "x", 111)
-        response = self.vscode.request_variables(varRef, start=0, count=1)
+        self.dap_server.request_setVariable(varRef, "x", 111)
+        response = self.dap_server.request_variables(varRef, start=0, count=1)
         value = response["body"]["variables"][0]["value"]
         self.assertEqual(
             value, "111", "verify pt.x got set to 111 (111 != %s)" % (value)
@@ -301,40 +301,42 @@ def do_test_scopes_variables_setVariable_evaluate(
         verify_locals["x @ main.cpp:19"] = {"equals": {"type": "int", "value": "42"}}
         verify_locals["x @ main.cpp:21"] = {"equals": {"type": "int", "value": "72"}}
 
-        self.verify_variables(verify_locals, self.vscode.get_local_variables())
+        self.verify_variables(verify_locals, self.dap_server.get_local_variables())
 
         # Now we verify that we correctly change the name of a variable with and without differentiator suffix
-        self.assertFalse(self.vscode.request_setVariable(1, "x2", 9)["success"])
+        self.assertFalse(self.dap_server.request_setVariable(1, "x2", 9)["success"])
         self.assertFalse(
-            self.vscode.request_setVariable(1, "x @ main.cpp:0", 9)["success"]
+            self.dap_server.request_setVariable(1, "x @ main.cpp:0", 9)["success"]
         )
 
         self.assertTrue(
-            self.vscode.request_setVariable(1, "x @ main.cpp:17", 17)["success"]
+            self.dap_server.request_setVariable(1, "x @ main.cpp:17", 17)["success"]
         )
         self.assertTrue(
-            self.vscode.request_setVariable(1, "x @ main.cpp:19", 19)["success"]
+            self.dap_server.request_setVariable(1, "x @ main.cpp:19", 19)["success"]
         )
         self.assertTrue(
-            self.vscode.request_setVariable(1, "x @ main.cpp:21", 21)["success"]
+            self.dap_server.request_setVariable(1, "x @ main.cpp:21", 21)["success"]
         )
 
         # The following should have no effect
         self.assertFalse(
-            self.vscode.request_setVariable(1, "x @ main.cpp:21", "invalid")["success"]
+            self.dap_server.request_setVariable(1, "x @ main.cpp:21", "invalid")[
+                "success"
+            ]
         )
 
         verify_locals["x @ main.cpp:17"]["equals"]["value"] = "17"
         verify_locals["x @ main.cpp:19"]["equals"]["value"] = "19"
         verify_locals["x @ main.cpp:21"]["equals"]["value"] = "21"
 
-        self.verify_variables(verify_locals, self.vscode.get_local_variables())
+        self.verify_variables(verify_locals, self.dap_server.get_local_variables())
 
         # The plain x variable shold refer to the innermost x
-        self.assertTrue(self.vscode.request_setVariable(1, "x", 22)["success"])
+        self.assertTrue(self.dap_server.request_setVariable(1, "x", 22)["success"])
         verify_locals["x @ main.cpp:21"]["equals"]["value"] = "22"
 
-        self.verify_variables(verify_locals, self.vscode.get_local_variables())
+        self.verify_variables(verify_locals, self.dap_server.get_local_variables())
 
         # In breakpoint 3, there should be no shadowed variables
         breakpoint3_line = line_number(source, "// breakpoint 3")
@@ -345,7 +347,7 @@ def do_test_scopes_variables_setVariable_evaluate(
         )
         self.continue_to_breakpoints(breakpoint_ids)
 
-        locals = self.vscode.get_local_variables()
+        locals = self.dap_server.get_local_variables()
         names = [var["name"] for var in locals]
         # The first shadowed x shouldn't have a suffix anymore
         verify_locals["x"] = {"equals": {"type": "int", "value": "17"}}
@@ -389,7 +391,7 @@ def do_test_scopes_and_evaluate_expansion(self, enableAutoVariableSummaries: boo
         self.continue_to_breakpoints(breakpoint_ids)
 
         # Verify locals
-        locals = self.vscode.get_local_variables()
+        locals = self.dap_server.get_local_variables()
         buffer_children = make_buffer_verify_dict(0, 32)
         verify_locals = {
             "argc": {
@@ -451,7 +453,7 @@ def do_test_scopes_and_evaluate_expansion(self, enableAutoVariableSummaries: boo
 
         # Evaluate from permanent UI.
         permanent_expr_varref_dict = {}
-        response = self.vscode.request_evaluate(
+        response = self.dap_server.request_evaluate(
             expandable_expression["name"], frameIndex=0, threadId=None, context="repl"
         )
         self.verify_values(
@@ -463,7 +465,7 @@ def do_test_scopes_and_evaluate_expansion(self, enableAutoVariableSummaries: boo
 
         # Evaluate from temporary UI.
         temporary_expr_varref_dict = {}
-        response = self.vscode.request_evaluate(expandable_expression["name"])
+        response = self.dap_server.request_evaluate(expandable_expression["name"])
         self.verify_values(
             expandable_expression["response"],
             response["body"],
@@ -472,13 +474,13 @@ def do_test_scopes_and_evaluate_expansion(self, enableAutoVariableSummaries: boo
         )
 
         # Evaluate locals again.
-        locals = self.vscode.get_local_variables()
+        locals = self.dap_server.get_local_variables()
         self.verify_variables(verify_locals, locals)
 
         # Verify the evaluated expressions before second locals evaluation
         # can be expanded.
         var_ref = temporary_expr_varref_dict[expandable_expression["name"]]
-        response = self.vscode.request_variables(var_ref)
+        response = self.dap_server.request_variables(var_ref)
         self.verify_variables(
             expandable_expression["children"], response["body"]["variables"]
         )
@@ -494,14 +496,14 @@ def do_test_scopes_and_evaluate_expansion(self, enableAutoVariableSummaries: boo
         self.continue_to_breakpoints(breakpoint_ids)
 
         var_ref = permanent_expr_varref_dict[expandable_expression["name"]]
-        response = self.vscode.request_variables(var_ref)
+        response = self.dap_server.request_variables(var_ref)
         self.verify_variables(
             expandable_expression["children"], response["body"]["variables"]
         )
 
         # Test that frame scopes have corresponding presentation hints.
-        frame_id = self.vscode.get_stackFrame()["id"]
-        scopes = self.vscode.request_scopes(frame_id)["body"]["scopes"]
+        frame_id = self.dap_server.get_stackFrame()["id"]
+        scopes = self.dap_server.request_scopes(frame_id)["body"]["scopes"]
 
         scope_names = [scope["name"] for scope in scopes]
         self.assertIn("Locals", scope_names)
@@ -544,7 +546,7 @@ def do_test_indexedVariables(self, enableSyntheticChildDebugging: bool):
         self.continue_to_breakpoints(breakpoint_ids)
 
         # Verify locals
-        locals = self.vscode.get_local_variables()
+        locals = self.dap_server.get_local_variables()
         # The vector variables might have one additional entry from the fake
         # "[raw]" child.
         raw_child_count = 1 if enableSyntheticChildDebugging else 0
@@ -569,7 +571,7 @@ def do_test_indexedVariables(self, enableSyntheticChildDebugging: bool):
         if enableSyntheticChildDebugging:
             verify_children["[raw]"] = ({"contains": {"type": ["vector"]}},)
 
-        children = self.vscode.request_variables(locals[2]["variablesReference"])[
+        children = self.dap_server.request_variables(locals[2]["variablesReference"])[
             "body"
         ]["variables"]
         self.verify_variables(verify_children, children)
@@ -620,11 +622,11 @@ def test_registers(self):
         if pc_name is None:
             return
         # Verify locals
-        reg_sets = self.vscode.get_registers()
+        reg_sets = self.dap_server.get_registers()
         for reg_set in reg_sets:
             if reg_set["name"] == "General Purpose Registers":
                 varRef = reg_set["variablesReference"]
-                regs = self.vscode.request_variables(varRef)["body"]["variables"]
+                regs = self.dap_server.request_variables(varRef)["body"]["variables"]
                 for reg in regs:
                     if reg["name"] == pc_name:
                         value = reg["value"]
diff --git a/lldb/test/API/tools/lldb-vscode/variables/main.cpp b/lldb/test/API/tools/lldb-dap/variables/main.cpp
similarity index 90%
rename from lldb/test/API/tools/lldb-vscode/variables/main.cpp
rename to lldb/test/API/tools/lldb-dap/variables/main.cpp
index d81a9a2054..da09ecb474 100644
--- a/lldb/test/API/tools/lldb-vscode/variables/main.cpp
+++ b/lldb/test/API/tools/lldb-dap/variables/main.cpp
@@ -11,8 +11,8 @@ static int s_global = 234;
 int test_indexedVariables();
 int main(int argc, char const *argv[]) {
   static float s_local = 2.25;
-  PointType pt = { 11,22, {0}};
-  for (int i=0; i<BUFFER_SIZE; ++i)
+  PointType pt = {11, 22, {0}};
+  for (int i = 0; i < BUFFER_SIZE; ++i)
     pt.buffer[i] = i;
   int x = s_global - g_global - pt.y; // breakpoint 1
   {
diff --git a/lldb/test/API/tools/lldb-vscode/categories b/lldb/test/API/tools/lldb-vscode/categories
deleted file mode 100644
index ce2cfd048e..0000000000
--- a/lldb/test/API/tools/lldb-vscode/categories
+++ /dev/null
@@ -1 +0,0 @@
-lldb-vscode
diff --git a/lldb/test/API/tools/lldb-vscode/disassemble/main.c b/lldb/test/API/tools/lldb-vscode/disassemble/main.c
deleted file mode 100644
index 8dd32f263c..0000000000
--- a/lldb/test/API/tools/lldb-vscode/disassemble/main.c
+++ /dev/null
@@ -1,30 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <limits.h>
- 
-int compare_ints(const void* a, const void* b)
-{
-    int arg1 = *(const int*)a;
-    int arg2 = *(const int*)b; 
-    
-    // breakpoint 1
- 
-    if (arg1 < arg2) return -1;
-    if (arg1 > arg2) return 1;
-    return 0;
-}
- 
-int main(void)
-{
-    int ints[] = { -2, 99, 0, -743, 2, INT_MIN, 4 };
-    int size = sizeof ints / sizeof *ints;
- 
-    qsort(ints, size, sizeof(int), compare_ints);
- 
-    for (int i = 0; i < size; i++) {
-        printf("%d ", ints[i]);
-    }
- 
-    printf("\n");
-    return 0;
-}
\ No newline at end of file
diff --git a/lldb/test/API/tools/lldb-vscode/module/foo.cpp b/lldb/test/API/tools/lldb-vscode/module/foo.cpp
deleted file mode 100644
index 9dba85a9cc..0000000000
--- a/lldb/test/API/tools/lldb-vscode/module/foo.cpp
+++ /dev/null
@@ -1,3 +0,0 @@
-int foo() {
-    return 12;
-}
diff --git a/lldb/test/API/tools/lldb-vscode/step/main.cpp b/lldb/test/API/tools/lldb-vscode/step/main.cpp
deleted file mode 100644
index 3027551972..0000000000
--- a/lldb/test/API/tools/lldb-vscode/step/main.cpp
+++ /dev/null
@@ -1,10 +0,0 @@
-int function(int x) {
-  if ((x % 2) == 0)
-    return function(x-1) + x; // breakpoint 1
-  else
-    return x;
-}
-
-int main(int argc, char const *argv[]) {
-  return function(2);
-}
diff --git a/lldb/test/API/tools/lldb-vscode/terminated-event/foo.cpp b/lldb/test/API/tools/lldb-vscode/terminated-event/foo.cpp
deleted file mode 100644
index 9dba85a9cc..0000000000
--- a/lldb/test/API/tools/lldb-vscode/terminated-event/foo.cpp
+++ /dev/null
@@ -1,3 +0,0 @@
-int foo() {
-    return 12;
-}
diff --git a/lldb/test/CMakeLists.txt b/lldb/test/CMakeLists.txt
index f215704db1..59f87fe066 100644
--- a/lldb/test/CMakeLists.txt
+++ b/lldb/test/CMakeLists.txt
@@ -114,8 +114,8 @@ if(TARGET lldb-server)
   add_lldb_test_dependency(lldb-server)
 endif()
 
-if(TARGET lldb-vscode)
-  add_lldb_test_dependency(lldb-vscode)
+if(TARGET lldb-dap)
+  add_lldb_test_dependency(lldb-dap)
 endif()
 
 if(TARGET liblldb)
diff --git a/lldb/test/Shell/Breakpoint/breakpoint-command.test b/lldb/test/Shell/Breakpoint/breakpoint-command.test
index 2bca6a97e6..8031be22dc 100644
--- a/lldb/test/Shell/Breakpoint/breakpoint-command.test
+++ b/lldb/test/Shell/Breakpoint/breakpoint-command.test
@@ -1,7 +1,7 @@
 # REQUIRES: lua || python
 
 # RUN: %build %p/Inputs/dummy-target.c -o %t.out
-# RUN: %lldb %t.out -o 'b main' -o 'break command add 1 -o "script print(95000 + 126)"' -o 'r' | FileCheck %s
+# RUN: %lldb %t.out -o 'b main' -o 'break command add 1 -o "script print(\"A R4Nd0m\" + \" Str1ng\")"' -o 'r' | FileCheck %s
 
-# CHECK: 95126
-# CHECK-NOT: 95126
+# CHECK: A R4Nd0m Str1ng
+# CHECK-NOT: A R4Nd0m Str1ng
diff --git a/lldb/test/Shell/VSCode/TestOptions.test b/lldb/test/Shell/DAP/TestOptions.test
similarity index 70%
rename from lldb/test/Shell/VSCode/TestOptions.test
rename to lldb/test/Shell/DAP/TestOptions.test
index 05588ecd9d..e37e9116e3 100644
--- a/lldb/test/Shell/VSCode/TestOptions.test
+++ b/lldb/test/Shell/DAP/TestOptions.test
@@ -1,4 +1,4 @@
-# RUN: lldb-vscode --help | FileCheck %s
+# RUN: lldb-dap --help | FileCheck %s
 # CHECK: -g
 # CHECK: --help
 # CHECK: -h
diff --git a/lldb/test/Shell/SymbolFile/DWARF/Inputs/dwo-missing-error.c b/lldb/test/Shell/SymbolFile/DWARF/Inputs/dwo-missing-error.c
new file mode 100644
index 0000000000..78f2de106c
--- /dev/null
+++ b/lldb/test/Shell/SymbolFile/DWARF/Inputs/dwo-missing-error.c
@@ -0,0 +1 @@
+int main(void) { return 0; }
diff --git a/lldb/test/Shell/SymbolFile/DWARF/dwo-missing-error.test b/lldb/test/Shell/SymbolFile/DWARF/dwo-missing-error.test
new file mode 100644
index 0000000000..4f7e70e36f
--- /dev/null
+++ b/lldb/test/Shell/SymbolFile/DWARF/dwo-missing-error.test
@@ -0,0 +1,34 @@
+# Check that LLDB prints an error message containing the DWO_AT_comp_dir and
+# DW_AT_dwo_name when it can't find a DWO and the DW_AT_comp_dir and
+# DW_AT_dwo_name are relative.
+
+# -gsplit-dwarf is supported only on Linux.
+# REQUIRES: system-linux
+
+# Test the error message with a relative DW_AT_comp_dir and DW_AT_dwo_name.
+# Creating and compiling to %t.compdir makes it easy to remove the dwo files.
+# DW_AT_comp_dir should be "./a/b/", and DW_AT_dwo_name should be
+# "a.out-dwo-missing-error.dwo".
+# since %T is deprecated.
+# RUN: rm -rf %t.compdir/
+# RUN: mkdir -p %t.compdir/a/b/
+# RUN: cd %t.compdir/a/b/
+# RUN: %clang_host %S/Inputs/dwo-missing-error.c -glldb -gdwarf-5 \
+# RUN:     -gsplit-dwarf -fdebug-prefix-map=%t.compdir=. -o a.out
+# RUN: rm *.dwo
+# RUN: %lldb a.out -s %s -o exit 2>&1 | FileCheck %s 
+# RUN: cd -
+
+# Test the error message with an absolute DW_AT_comp_dir and DW_AT_dwo_name.
+# RUN: rm -rf %t.compdir/
+# RUN: mkdir -p %t.compdir/a/b/
+# RUN: %clang_host %S/Inputs/dwo-missing-error.c -glldb -gdwarf-5 \
+# RUN:     -gsplit-dwarf -o %t.compdir/a/b/a.out
+# RUN: rm %t.compdir/a/b/*.dwo
+# RUN: %lldb %t.compdir/a/b/a.out -s %s -o exit 2>&1 | FileCheck %s 
+
+b main
+run
+
+fr v
+# CHECK: error: unable to locate .dwo debug file "{{.*}}a/b/a.out-dwo-missing-error.dwo" for skeleton DIE {{.*}}
diff --git a/lldb/test/Shell/helper/toolchain.py b/lldb/test/Shell/helper/toolchain.py
index 8b56c659b1..255955fc70 100644
--- a/lldb/test/Shell/helper/toolchain.py
+++ b/lldb/test/Shell/helper/toolchain.py
@@ -90,7 +90,7 @@ def use_lldb_substitutions(config):
             unresolved="ignore",
         ),
         "lldb-test",
-        "lldb-vscode",
+        "lldb-dap",
         ToolSubst(
             "%build", command="'" + sys.executable + "'", extra_args=build_script_args
         ),
diff --git a/lldb/tools/CMakeLists.txt b/lldb/tools/CMakeLists.txt
index 16a2c7956a..6804dc2345 100644
--- a/lldb/tools/CMakeLists.txt
+++ b/lldb/tools/CMakeLists.txt
@@ -9,7 +9,7 @@ add_subdirectory(lldb-test EXCLUDE_FROM_ALL)
 add_subdirectory(lldb-fuzzer EXCLUDE_FROM_ALL)
 
 add_lldb_tool_subdirectory(lldb-instr)
-add_lldb_tool_subdirectory(lldb-vscode)
+add_lldb_tool_subdirectory(lldb-dap)
 
 if (CMAKE_SYSTEM_NAME MATCHES "Darwin")
   add_lldb_tool_subdirectory(darwin-debug)
diff --git a/lldb/tools/lldb-vscode/BreakpointBase.cpp b/lldb/tools/lldb-dap/BreakpointBase.cpp
similarity index 97%
rename from lldb/tools/lldb-vscode/BreakpointBase.cpp
rename to lldb/tools/lldb-dap/BreakpointBase.cpp
index 9ec392c9af..cd12f97fb1 100644
--- a/lldb/tools/lldb-vscode/BreakpointBase.cpp
+++ b/lldb/tools/lldb-dap/BreakpointBase.cpp
@@ -7,10 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "BreakpointBase.h"
-#include "VSCode.h"
+#include "DAP.h"
 #include "llvm/ADT/StringExtras.h"
 
-using namespace lldb_vscode;
+using namespace lldb_dap;
 
 BreakpointBase::BreakpointBase(const llvm::json::Object &obj)
     : condition(std::string(GetString(obj, "condition"))),
@@ -270,7 +270,7 @@ void BreakpointBase::SetLogMessage() {
 void BreakpointBase::NotifyLogMessageError(llvm::StringRef error) {
   std::string message = "Log message has error: ";
   message += error;
-  g_vsc.SendOutput(OutputType::Console, message);
+  g_dap.SendOutput(OutputType::Console, message);
 }
 
 /*static*/
@@ -304,7 +304,7 @@ bool BreakpointBase::BreakpointHitCallback(
   }
   if (!output.empty() && output.back() != '\n')
     output.push_back('\n'); // Ensure log message has line break.
-  g_vsc.SendOutput(OutputType::Console, output.c_str());
+  g_dap.SendOutput(OutputType::Console, output.c_str());
 
   // Do not stop.
   return false;
@@ -328,7 +328,7 @@ void BreakpointBase::UpdateBreakpoint(const BreakpointBase &request_bp) {
 const char *BreakpointBase::GetBreakpointLabel() {
   // Breakpoints in LLDB can have names added to them which are kind of like
   // labels or categories. All breakpoints that are set through the IDE UI get
-  // sent through the various VS code DAP set*Breakpoint packets, and these
+  // sent through the various DAP set*Breakpoint packets, and these
   // breakpoints will be labeled with this name so if breakpoint update events
   // come in for breakpoints that the IDE doesn't know about, like if a
   // breakpoint is set manually using the debugger console, we won't report any
@@ -338,5 +338,5 @@ const char *BreakpointBase::GetBreakpointLabel() {
   // in via LLDB breakpoint changed events and check the breakpoint by calling
   // "bool lldb::SBBreakpoint::MatchesName(const char *)" to check if a
   // breakpoint in one of the UI breakpoints that we should report changes for.
-  return "vscode";
+  return "dap";
 }
diff --git a/lldb/tools/lldb-vscode/BreakpointBase.h b/lldb/tools/lldb-dap/BreakpointBase.h
similarity index 93%
rename from lldb/tools/lldb-vscode/BreakpointBase.h
rename to lldb/tools/lldb-dap/BreakpointBase.h
index 91e0dd6001..41787f7861 100644
--- a/lldb/tools/lldb-vscode/BreakpointBase.h
+++ b/lldb/tools/lldb-dap/BreakpointBase.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLDB_TOOLS_LLDB_VSCODE_BREAKPOINTBASE_H
-#define LLDB_TOOLS_LLDB_VSCODE_BREAKPOINTBASE_H
+#ifndef LLDB_TOOLS_LLDB_DAP_BREAKPOINTBASE_H
+#define LLDB_TOOLS_LLDB_DAP_BREAKPOINTBASE_H
 
 #include "JSONUtils.h"
 #include "lldb/API/SBBreakpoint.h"
@@ -15,7 +15,7 @@
 #include <string>
 #include <vector>
 
-namespace lldb_vscode {
+namespace lldb_dap {
 
 struct BreakpointBase {
   // logMessage part can be either a raw text or an expression.
@@ -58,6 +58,6 @@ struct BreakpointBase {
                                     lldb::SBBreakpointLocation &location);
 };
 
-} // namespace lldb_vscode
+} // namespace lldb_dap
 
 #endif
diff --git a/lldb/tools/lldb-vscode/CMakeLists.txt b/lldb/tools/lldb-dap/CMakeLists.txt
similarity index 79%
rename from lldb/tools/lldb-vscode/CMakeLists.txt
rename to lldb/tools/lldb-dap/CMakeLists.txt
index b68a4f3f67..c71c809818 100644
--- a/lldb/tools/lldb-vscode/CMakeLists.txt
+++ b/lldb/tools/lldb-dap/CMakeLists.txt
@@ -9,11 +9,11 @@ endif ()
 
 if(APPLE)
   configure_file(
-    ${CMAKE_CURRENT_SOURCE_DIR}/lldb-vscode-Info.plist.in
-    ${CMAKE_CURRENT_BINARY_DIR}/lldb-vscode-Info.plist
+    ${CMAKE_CURRENT_SOURCE_DIR}/lldb-dap-Info.plist.in
+    ${CMAKE_CURRENT_BINARY_DIR}/lldb-dap-Info.plist
     )
   # Inline info plist in binary (use target_link_options for this as soon as CMake 3.13 is available)
-  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-sectcreate,__TEXT,__info_plist,${CMAKE_CURRENT_BINARY_DIR}/lldb-vscode-Info.plist")
+  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-sectcreate,__TEXT,__info_plist,${CMAKE_CURRENT_BINARY_DIR}/lldb-dap-Info.plist")
 endif()
 
 # We need to include the llvm components we depend on manually, as liblldb does
@@ -21,9 +21,9 @@ endif()
 set(LLVM_LINK_COMPONENTS Support)
 set(LLVM_TARGET_DEFINITIONS Options.td)
 tablegen(LLVM Options.inc -gen-opt-parser-defs)
-add_public_tablegen_target(LLDBVSCodeOptionsTableGen)
-add_lldb_tool(lldb-vscode
-  lldb-vscode.cpp
+add_public_tablegen_target(LLDBDAPOptionsTableGen)
+add_lldb_tool(lldb-dap
+  lldb-dap.cpp
   BreakpointBase.cpp
   ExceptionBreakpoint.cpp
   FifoFiles.cpp
@@ -35,7 +35,7 @@ add_lldb_tool(lldb-vscode
   ProgressEvent.cpp
   RunInTerminal.cpp
   SourceBreakpoint.cpp
-  VSCode.cpp
+  DAP.cpp
 
   LINK_LIBS
     liblldb
@@ -49,7 +49,7 @@ add_lldb_tool(lldb-vscode
 if(LLDB_BUILD_FRAMEWORK)
   # In the build-tree, we know the exact path to the framework directory.
   # The installed framework can be in different locations.
-  lldb_setup_rpaths(lldb-vscode
+  lldb_setup_rpaths(lldb-dap
     BUILD_RPATH
       "${LLDB_FRAMEWORK_ABSOLUTE_BUILD_DIR}"
     INSTALL_RPATH
diff --git a/lldb/tools/lldb-vscode/VSCode.cpp b/lldb/tools/lldb-dap/DAP.cpp
similarity index 86%
rename from lldb/tools/lldb-vscode/VSCode.cpp
rename to lldb/tools/lldb-dap/DAP.cpp
index 1384604c48..af20e64606 100644
--- a/lldb/tools/lldb-vscode/VSCode.cpp
+++ b/lldb/tools/lldb-dap/DAP.cpp
@@ -1,4 +1,4 @@
-//===-- VSCode.cpp ----------------------------------------------*- C++ -*-===//
+//===-- DAP.cpp -------------------------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -12,8 +12,8 @@
 #include <mutex>
 #include <sstream>
 
+#include "DAP.h"
 #include "LLDBUtils.h"
-#include "VSCode.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/FormatVariadic.h"
 
@@ -24,14 +24,14 @@
 #include <windows.h>
 #endif
 
-using namespace lldb_vscode;
+using namespace lldb_dap;
 
-namespace lldb_vscode {
+namespace lldb_dap {
 
-VSCode g_vsc;
+DAP g_dap;
 
-VSCode::VSCode()
-    : broadcaster("lldb-vscode"),
+DAP::DAP()
+    : broadcaster("lldb-dap"),
       exception_breakpoints(
           {{"cpp_catch", "C++ Catch", lldb::eLanguageTypeC_plus_plus},
            {"cpp_throw", "C++ Throw", lldb::eLanguageTypeC_plus_plus},
@@ -49,7 +49,7 @@ VSCode::VSCode()
           [&](const ProgressEvent &event) { SendJSON(event.ToJSON()); }),
       reverse_request_seq(0), repl_mode(ReplMode::Auto),
       auto_repl_mode_collision_warning(false) {
-  const char *log_file_path = getenv("LLDBVSCODE_LOG");
+  const char *log_file_path = getenv("LLDBDAP_LOG");
 #if defined(_WIN32)
   // Windows opens stdout and stdin in text mode which converts \n to 13,10
   // while the value is just 10 on Darwin/Linux. Setting the file mode to binary
@@ -64,9 +64,9 @@ VSCode::VSCode()
     log.reset(new std::ofstream(log_file_path));
 }
 
-VSCode::~VSCode() = default;
+DAP::~DAP() = default;
 
-ExceptionBreakpoint *VSCode::GetExceptionBreakpoint(const std::string &filter) {
+ExceptionBreakpoint *DAP::GetExceptionBreakpoint(const std::string &filter) {
   for (auto &bp : exception_breakpoints) {
     if (bp.filter == filter)
       return &bp;
@@ -74,8 +74,7 @@ ExceptionBreakpoint *VSCode::GetExceptionBreakpoint(const std::string &filter) {
   return nullptr;
 }
 
-ExceptionBreakpoint *
-VSCode::GetExceptionBreakpoint(const lldb::break_id_t bp_id) {
+ExceptionBreakpoint *DAP::GetExceptionBreakpoint(const lldb::break_id_t bp_id) {
   for (auto &bp : exception_breakpoints) {
     if (bp.bp.GetID() == bp_id)
       return &bp;
@@ -86,7 +85,7 @@ VSCode::GetExceptionBreakpoint(const lldb::break_id_t bp_id) {
 // Send the JSON in "json_str" to the "out" stream. Correctly send the
 // "Content-Length:" field followed by the length, followed by the raw
 // JSON bytes.
-void VSCode::SendJSON(const std::string &json_str) {
+void DAP::SendJSON(const std::string &json_str) {
   output.write_full("Content-Length: ");
   output.write_full(llvm::utostr(json_str.size()));
   output.write_full("\r\n\r\n");
@@ -95,7 +94,7 @@ void VSCode::SendJSON(const std::string &json_str) {
 
 // Serialize the JSON value into a string and send the JSON packet to
 // the "out" stream.
-void VSCode::SendJSON(const llvm::json::Value &json) {
+void DAP::SendJSON(const llvm::json::Value &json) {
   std::string s;
   llvm::raw_string_ostream strm(s);
   strm << json;
@@ -112,7 +111,7 @@ void VSCode::SendJSON(const llvm::json::Value &json) {
 }
 
 // Read a JSON packet from the "in" stream.
-std::string VSCode::ReadJSON() {
+std::string DAP::ReadJSON() {
   std::string length_str;
   std::string json_str;
   int length;
@@ -198,7 +197,7 @@ std::string VSCode::ReadJSON() {
 //     "required": [ "event", "body" ]
 //   }]
 // }
-void VSCode::SendOutput(OutputType o, const llvm::StringRef output) {
+void DAP::SendOutput(OutputType o, const llvm::StringRef output) {
   if (output.empty())
     return;
 
@@ -318,13 +317,13 @@ void VSCode::SendOutput(OutputType o, const llvm::StringRef output) {
 //   };
 // }
 
-void VSCode::SendProgressEvent(uint64_t progress_id, const char *message,
-                               uint64_t completed, uint64_t total) {
+void DAP::SendProgressEvent(uint64_t progress_id, const char *message,
+                            uint64_t completed, uint64_t total) {
   progress_event_reporter.Push(progress_id, message, completed, total);
 }
 
 void __attribute__((format(printf, 3, 4)))
-VSCode::SendFormattedOutput(OutputType o, const char *format, ...) {
+DAP::SendFormattedOutput(OutputType o, const char *format, ...) {
   char buffer[1024];
   va_list args;
   va_start(args, format);
@@ -334,8 +333,7 @@ VSCode::SendFormattedOutput(OutputType o, const char *format, ...) {
       o, llvm::StringRef(buffer, std::min<int>(actual_length, sizeof(buffer))));
 }
 
-ExceptionBreakpoint *
-VSCode::GetExceptionBPFromStopReason(lldb::SBThread &thread) {
+ExceptionBreakpoint *DAP::GetExceptionBPFromStopReason(lldb::SBThread &thread) {
   const auto num = thread.GetStopReasonDataCount();
   // Check to see if have hit an exception breakpoint and change the
   // reason to "exception", but only do so if all breakpoints that were
@@ -356,12 +354,12 @@ VSCode::GetExceptionBPFromStopReason(lldb::SBThread &thread) {
   return exc_bp;
 }
 
-lldb::SBThread VSCode::GetLLDBThread(const llvm::json::Object &arguments) {
+lldb::SBThread DAP::GetLLDBThread(const llvm::json::Object &arguments) {
   auto tid = GetSigned(arguments, "threadId", LLDB_INVALID_THREAD_ID);
   return target.GetProcess().GetThreadByID(tid);
 }
 
-lldb::SBFrame VSCode::GetLLDBFrame(const llvm::json::Object &arguments) {
+lldb::SBFrame DAP::GetLLDBFrame(const llvm::json::Object &arguments) {
   const uint64_t frame_id = GetUnsigned(arguments, "frameId", UINT64_MAX);
   lldb::SBProcess process = target.GetProcess();
   // Upper 32 bits is the thread index ID
@@ -371,19 +369,19 @@ lldb::SBFrame VSCode::GetLLDBFrame(const llvm::json::Object &arguments) {
   return thread.GetFrameAtIndex(GetLLDBFrameID(frame_id));
 }
 
-llvm::json::Value VSCode::CreateTopLevelScopes() {
+llvm::json::Value DAP::CreateTopLevelScopes() {
   llvm::json::Array scopes;
   scopes.emplace_back(CreateScope("Locals", VARREF_LOCALS,
-                                  g_vsc.variables.locals.GetSize(), false));
+                                  g_dap.variables.locals.GetSize(), false));
   scopes.emplace_back(CreateScope("Globals", VARREF_GLOBALS,
-                                  g_vsc.variables.globals.GetSize(), false));
+                                  g_dap.variables.globals.GetSize(), false));
   scopes.emplace_back(CreateScope("Registers", VARREF_REGS,
-                                  g_vsc.variables.registers.GetSize(), false));
+                                  g_dap.variables.registers.GetSize(), false));
   return llvm::json::Value(std::move(scopes));
 }
 
-ExpressionContext VSCode::DetectExpressionContext(lldb::SBFrame &frame,
-                                                  std::string &text) {
+ExpressionContext DAP::DetectExpressionContext(lldb::SBFrame &frame,
+                                               std::string &text) {
   // Include ` as an escape hatch.
   if (!text.empty() && text[0] == '`') {
     text = text.substr(1);
@@ -433,35 +431,35 @@ ExpressionContext VSCode::DetectExpressionContext(lldb::SBFrame &frame,
   return ExpressionContext::Variable;
 }
 
-void VSCode::RunLLDBCommands(llvm::StringRef prefix,
-                             const std::vector<std::string> &commands) {
+void DAP::RunLLDBCommands(llvm::StringRef prefix,
+                          const std::vector<std::string> &commands) {
   SendOutput(OutputType::Console,
              llvm::StringRef(::RunLLDBCommands(prefix, commands)));
 }
 
-void VSCode::RunInitCommands() {
+void DAP::RunInitCommands() {
   RunLLDBCommands("Running initCommands:", init_commands);
 }
 
-void VSCode::RunPreRunCommands() {
+void DAP::RunPreRunCommands() {
   RunLLDBCommands("Running preRunCommands:", pre_run_commands);
 }
 
-void VSCode::RunStopCommands() {
+void DAP::RunStopCommands() {
   RunLLDBCommands("Running stopCommands:", stop_commands);
 }
 
-void VSCode::RunExitCommands() {
+void DAP::RunExitCommands() {
   RunLLDBCommands("Running exitCommands:", exit_commands);
 }
 
-void VSCode::RunTerminateCommands() {
+void DAP::RunTerminateCommands() {
   RunLLDBCommands("Running terminateCommands:", terminate_commands);
 }
 
 lldb::SBTarget
-VSCode::CreateTargetFromArguments(const llvm::json::Object &arguments,
-                                  lldb::SBError &error) {
+DAP::CreateTargetFromArguments(const llvm::json::Object &arguments,
+                               lldb::SBError &error) {
   // Grab the name of the program we need to debug and create a target using
   // the given program as an argument. Executable file can be a source of target
   // architecture and platform, if they differ from the host. Setting exe path
@@ -490,7 +488,7 @@ VSCode::CreateTargetFromArguments(const llvm::json::Object &arguments,
   return target;
 }
 
-void VSCode::SetTarget(const lldb::SBTarget target) {
+void DAP::SetTarget(const lldb::SBTarget target) {
   this->target = target;
 
   if (target.IsValid()) {
@@ -504,7 +502,7 @@ void VSCode::SetTarget(const lldb::SBTarget target) {
   }
 }
 
-PacketStatus VSCode::GetNextObject(llvm::json::Object &object) {
+PacketStatus DAP::GetNextObject(llvm::json::Object &object) {
   std::string json = ReadJSON();
   if (json.empty())
     return PacketStatus::EndOfFile;
@@ -538,7 +536,7 @@ PacketStatus VSCode::GetNextObject(llvm::json::Object &object) {
   return PacketStatus::Success;
 }
 
-bool VSCode::HandleObject(const llvm::json::Object &object) {
+bool DAP::HandleObject(const llvm::json::Object &object) {
   const auto packet_type = GetString(object, "type");
   if (packet_type == "request") {
     const auto command = GetString(object, "command");
@@ -591,16 +589,16 @@ bool VSCode::HandleObject(const llvm::json::Object &object) {
   return false;
 }
 
-llvm::Error VSCode::Loop() {
+llvm::Error DAP::Loop() {
   while (!sent_terminated_event) {
     llvm::json::Object object;
-    lldb_vscode::PacketStatus status = GetNextObject(object);
+    lldb_dap::PacketStatus status = GetNextObject(object);
 
-    if (status == lldb_vscode::PacketStatus::EndOfFile) {
+    if (status == lldb_dap::PacketStatus::EndOfFile) {
       break;
     }
 
-    if (status != lldb_vscode::PacketStatus::Success) {
+    if (status != lldb_dap::PacketStatus::Success) {
       return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                      "failed to send packet");
     }
@@ -614,9 +612,9 @@ llvm::Error VSCode::Loop() {
   return llvm::Error::success();
 }
 
-void VSCode::SendReverseRequest(llvm::StringRef command,
-                                llvm::json::Value arguments,
-                                ResponseCallback callback) {
+void DAP::SendReverseRequest(llvm::StringRef command,
+                             llvm::json::Value arguments,
+                             ResponseCallback callback) {
   int64_t id;
   {
     std::lock_guard<std::mutex> locker(call_mutex);
@@ -632,12 +630,12 @@ void VSCode::SendReverseRequest(llvm::StringRef command,
   });
 }
 
-void VSCode::RegisterRequestCallback(std::string request,
-                                     RequestCallback callback) {
+void DAP::RegisterRequestCallback(std::string request,
+                                  RequestCallback callback) {
   request_handlers[request] = callback;
 }
 
-lldb::SBError VSCode::WaitForProcessToStop(uint32_t seconds) {
+lldb::SBError DAP::WaitForProcessToStop(uint32_t seconds) {
   lldb::SBError error;
   lldb::SBProcess process = target.GetProcess();
   if (!process.IsValid()) {
@@ -649,26 +647,26 @@ lldb::SBError VSCode::WaitForProcessToStop(uint32_t seconds) {
   while (std::chrono::steady_clock::now() < timeout_time) {
     const auto state = process.GetState();
     switch (state) {
-      case lldb::eStateAttaching:
-      case lldb::eStateConnected:
-      case lldb::eStateInvalid:
-      case lldb::eStateLaunching:
-      case lldb::eStateRunning:
-      case lldb::eStateStepping:
-      case lldb::eStateSuspended:
-        break;
-      case lldb::eStateDetached:
-        error.SetErrorString("process detached during launch or attach");
-        return error;
-      case lldb::eStateExited:
-        error.SetErrorString("process exited during launch or attach");
-        return error;
-      case lldb::eStateUnloaded:
-        error.SetErrorString("process unloaded during launch or attach");
-        return error;
-      case lldb::eStateCrashed:
-      case lldb::eStateStopped:
-        return lldb::SBError(); // Success!
+    case lldb::eStateAttaching:
+    case lldb::eStateConnected:
+    case lldb::eStateInvalid:
+    case lldb::eStateLaunching:
+    case lldb::eStateRunning:
+    case lldb::eStateStepping:
+    case lldb::eStateSuspended:
+      break;
+    case lldb::eStateDetached:
+      error.SetErrorString("process detached during launch or attach");
+      return error;
+    case lldb::eStateExited:
+      error.SetErrorString("process exited during launch or attach");
+      return error;
+    case lldb::eStateUnloaded:
+      error.SetErrorString("process unloaded during launch or attach");
+      return error;
+    case lldb::eStateCrashed:
+    case lldb::eStateStopped:
+      return lldb::SBError(); // Success!
     }
     std::this_thread::sleep_for(std::chrono::microseconds(250));
   }
@@ -759,7 +757,7 @@ bool StartDebuggingRequestHandler::DoExecute(
     return false;
   }
 
-  g_vsc.SendReverseRequest(
+  g_dap.SendReverseRequest(
       "startDebugging",
       llvm::json::Object{{"request", request},
                          {"configuration", std::move(*configuration)}},
@@ -783,7 +781,7 @@ bool ReplModeRequestHandler::DoExecute(lldb::SBDebugger debugger,
   // If a new mode is not specified report the current mode.
   if (!command || llvm::StringRef(command[0]).empty()) {
     std::string mode;
-    switch (g_vsc.repl_mode) {
+    switch (g_dap.repl_mode) {
     case ReplMode::Variable:
       mode = "variable";
       break;
@@ -795,7 +793,7 @@ bool ReplModeRequestHandler::DoExecute(lldb::SBDebugger debugger,
       break;
     }
 
-    result.Printf("lldb-vscode repl-mode %s.\n", mode.c_str());
+    result.Printf("lldb-dap repl-mode %s.\n", mode.c_str());
     result.SetStatus(lldb::eReturnStatusSuccessFinishResult);
 
     return true;
@@ -804,11 +802,11 @@ bool ReplModeRequestHandler::DoExecute(lldb::SBDebugger debugger,
   llvm::StringRef new_mode{command[0]};
 
   if (new_mode == "variable") {
-    g_vsc.repl_mode = ReplMode::Variable;
+    g_dap.repl_mode = ReplMode::Variable;
   } else if (new_mode == "command") {
-    g_vsc.repl_mode = ReplMode::Command;
+    g_dap.repl_mode = ReplMode::Command;
   } else if (new_mode == "auto") {
-    g_vsc.repl_mode = ReplMode::Auto;
+    g_dap.repl_mode = ReplMode::Auto;
   } else {
     lldb::SBStream error_message;
     error_message.Printf("Invalid repl-mode '%s'. Expected one of 'variable', "
@@ -818,9 +816,9 @@ bool ReplModeRequestHandler::DoExecute(lldb::SBDebugger debugger,
     return false;
   }
 
-  result.Printf("lldb-vscode repl-mode %s set.\n", new_mode.data());
+  result.Printf("lldb-dap repl-mode %s set.\n", new_mode.data());
   result.SetStatus(lldb::eReturnStatusSuccessFinishNoResult);
   return true;
 }
 
-} // namespace lldb_vscode
+} // namespace lldb_dap
diff --git a/lldb/tools/lldb-vscode/VSCode.h b/lldb/tools/lldb-dap/DAP.h
similarity index 96%
rename from lldb/tools/lldb-vscode/VSCode.h
rename to lldb/tools/lldb-dap/DAP.h
index 59bb11c71e..e13d91a9df 100644
--- a/lldb/tools/lldb-vscode/VSCode.h
+++ b/lldb/tools/lldb-dap/DAP.h
@@ -1,4 +1,4 @@
-//===-- VSCode.h ------------------------------------------------*- C++ -*-===//
+//===-- DAP.h ---------------------------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLDB_TOOLS_LLDB_VSCODE_VSCODE_H
-#define LLDB_TOOLS_LLDB_VSCODE_VSCODE_H
+#ifndef LLDB_TOOLS_LLDB_DAP_DAP_H
+#define LLDB_TOOLS_LLDB_DAP_DAP_H
 
 #include "llvm/Config/llvm-config.h" // for LLVM_ON_UNIX
 
@@ -62,13 +62,13 @@
 #define VARREF_FIRST_VAR_IDX (int64_t)4
 #define NO_TYPENAME "<no-type>"
 
-namespace lldb_vscode {
+namespace lldb_dap {
 
 typedef llvm::DenseMap<uint32_t, SourceBreakpoint> SourceBreakpointMap;
 typedef llvm::StringMap<FunctionBreakpoint> FunctionBreakpointMap;
 enum class OutputType { Console, Stdout, Stderr, Telemetry };
 
-enum VSCodeBroadcasterBits {
+enum DAPBroadcasterBits {
   eBroadcastBitStopEventThread = 1u << 0,
   eBroadcastBitStopProgressThread = 1u << 1
 };
@@ -141,7 +141,7 @@ struct ReplModeRequestHandler : public lldb::SBCommandPluginInterface {
                  lldb::SBCommandReturnObject &result) override;
 };
 
-struct VSCode {
+struct DAP {
   std::string debug_adaptor_path;
   InputStream input;
   OutputStream output;
@@ -189,10 +189,10 @@ struct VSCode {
   ReplMode repl_mode;
   bool auto_repl_mode_collision_warning;
 
-  VSCode();
-  ~VSCode();
-  VSCode(const VSCode &rhs) = delete;
-  void operator=(const VSCode &rhs) = delete;
+  DAP();
+  ~DAP();
+  DAP(const DAP &rhs) = delete;
+  void operator=(const DAP &rhs) = delete;
   ExceptionBreakpoint *GetExceptionBreakpoint(const std::string &filter);
   ExceptionBreakpoint *GetExceptionBreakpoint(const lldb::break_id_t bp_id);
 
@@ -245,7 +245,7 @@ struct VSCode {
   lldb::SBTarget CreateTargetFromArguments(const llvm::json::Object &arguments,
                                            lldb::SBError &error);
 
-  /// Set given target object as a current target for lldb-vscode and start
+  /// Set given target object as a current target for lldb-dap and start
   /// listeing for its breakpoint events.
   void SetTarget(const lldb::SBTarget target);
 
@@ -311,8 +311,8 @@ struct VSCode {
   void SendJSON(const std::string &json_str);
 };
 
-extern VSCode g_vsc;
+extern DAP g_dap;
 
-} // namespace lldb_vscode
+} // namespace lldb_dap
 
 #endif
diff --git a/lldb/tools/lldb-vscode/VSCodeForward.h b/lldb/tools/lldb-dap/DAPForward.h
similarity index 81%
rename from lldb/tools/lldb-vscode/VSCodeForward.h
rename to lldb/tools/lldb-dap/DAPForward.h
index 92eb5757d1..fffff1e3f7 100644
--- a/lldb/tools/lldb-vscode/VSCodeForward.h
+++ b/lldb/tools/lldb-dap/DAPForward.h
@@ -1,4 +1,4 @@
-//===-- VSCodeForward.h -----------------------------------------*- C++ -*-===//
+//===-- DAPForward.h --------------------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,15 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLDB_TOOLS_LLDB_VSCODE_VSCODEFORWARD_H
-#define LLDB_TOOLS_LLDB_VSCODE_VSCODEFORWARD_H
+#ifndef LLDB_TOOLS_LLDB_DAP_DAPFORWARD_H
+#define LLDB_TOOLS_LLDB_DAP_DAPFORWARD_H
 
-namespace lldb_vscode {
+namespace lldb_dap {
 struct BreakpointBase;
 struct ExceptionBreakpoint;
 struct FunctionBreakpoint;
 struct SourceBreakpoint;
-} // namespace lldb_vscode
+} // namespace lldb_dap
 
 namespace lldb {
 class SBAttachInfo;
diff --git a/lldb/tools/lldb-vscode/ExceptionBreakpoint.cpp b/lldb/tools/lldb-dap/ExceptionBreakpoint.cpp
similarity index 84%
rename from lldb/tools/lldb-vscode/ExceptionBreakpoint.cpp
rename to lldb/tools/lldb-dap/ExceptionBreakpoint.cpp
index 9324e66d8b..130c237e65 100644
--- a/lldb/tools/lldb-vscode/ExceptionBreakpoint.cpp
+++ b/lldb/tools/lldb-dap/ExceptionBreakpoint.cpp
@@ -8,16 +8,16 @@
 
 #include "ExceptionBreakpoint.h"
 #include "BreakpointBase.h"
-#include "VSCode.h"
+#include "DAP.h"
 
-namespace lldb_vscode {
+namespace lldb_dap {
 
 void ExceptionBreakpoint::SetBreakpoint() {
   if (bp.IsValid())
     return;
   bool catch_value = filter.find("_catch") != std::string::npos;
   bool throw_value = filter.find("_throw") != std::string::npos;
-  bp = g_vsc.target.BreakpointCreateForException(language, catch_value,
+  bp = g_dap.target.BreakpointCreateForException(language, catch_value,
                                                  throw_value);
   // See comments in BreakpointBase::GetBreakpointLabel() for details of why
   // we add a label to our breakpoints.
@@ -27,8 +27,8 @@ void ExceptionBreakpoint::SetBreakpoint() {
 void ExceptionBreakpoint::ClearBreakpoint() {
   if (!bp.IsValid())
     return;
-  g_vsc.target.BreakpointDelete(bp.GetID());
+  g_dap.target.BreakpointDelete(bp.GetID());
   bp = lldb::SBBreakpoint();
 }
 
-} // namespace lldb_vscode
+} // namespace lldb_dap
diff --git a/lldb/tools/lldb-vscode/ExceptionBreakpoint.h b/lldb/tools/lldb-dap/ExceptionBreakpoint.h
similarity index 83%
rename from lldb/tools/lldb-vscode/ExceptionBreakpoint.h
rename to lldb/tools/lldb-dap/ExceptionBreakpoint.h
index 203630ccf4..7b81d845cb 100644
--- a/lldb/tools/lldb-vscode/ExceptionBreakpoint.h
+++ b/lldb/tools/lldb-dap/ExceptionBreakpoint.h
@@ -6,14 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLDB_TOOLS_LLDB_VSCODE_EXCEPTIONBREAKPOINT_H
-#define LLDB_TOOLS_LLDB_VSCODE_EXCEPTIONBREAKPOINT_H
+#ifndef LLDB_TOOLS_LLDB_DAP_EXCEPTIONBREAKPOINT_H
+#define LLDB_TOOLS_LLDB_DAP_EXCEPTIONBREAKPOINT_H
 
 #include <string>
 
 #include "lldb/API/SBBreakpoint.h"
 
-namespace lldb_vscode {
+namespace lldb_dap {
 
 struct ExceptionBreakpoint {
   std::string filter;
@@ -29,6 +29,6 @@ struct ExceptionBreakpoint {
   void ClearBreakpoint();
 };
 
-} // namespace lldb_vscode
+} // namespace lldb_dap
 
 #endif
diff --git a/lldb/tools/lldb-vscode/FifoFiles.cpp b/lldb/tools/lldb-dap/FifoFiles.cpp
similarity index 98%
rename from lldb/tools/lldb-vscode/FifoFiles.cpp
rename to lldb/tools/lldb-dap/FifoFiles.cpp
index a5330d58c3..9a6423f794 100644
--- a/lldb/tools/lldb-vscode/FifoFiles.cpp
+++ b/lldb/tools/lldb-dap/FifoFiles.cpp
@@ -26,7 +26,7 @@
 
 using namespace llvm;
 
-namespace lldb_vscode {
+namespace lldb_dap {
 
 FifoFile::FifoFile(StringRef path) : m_path(path) {}
 
@@ -102,4 +102,4 @@ Error FifoFileIO::SendJSON(const json::Value &json,
   return Error::success();
 }
 
-} // namespace lldb_vscode
+} // namespace lldb_dap
diff --git a/lldb/tools/lldb-vscode/FifoFiles.h b/lldb/tools/lldb-dap/FifoFiles.h
similarity index 93%
rename from lldb/tools/lldb-vscode/FifoFiles.h
rename to lldb/tools/lldb-dap/FifoFiles.h
index a0c4562b5a..02a97cd5cb 100644
--- a/lldb/tools/lldb-vscode/FifoFiles.h
+++ b/lldb/tools/lldb-dap/FifoFiles.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLDB_TOOLS_LLDB_VSCODE_FIFOFILES_H
-#define LLDB_TOOLS_LLDB_VSCODE_FIFOFILES_H
+#ifndef LLDB_TOOLS_LLDB_DAP_FIFOFILES_H
+#define LLDB_TOOLS_LLDB_DAP_FIFOFILES_H
 
 #include "llvm/Config/llvm-config.h" // for LLVM_ON_UNIX
 #include "llvm/Support/Error.h"
@@ -16,7 +16,7 @@
 
 #include <chrono>
 
-namespace lldb_vscode {
+namespace lldb_dap {
 
 /// Struct that controls the life of a fifo file in the filesystem.
 ///
@@ -82,6 +82,6 @@ class FifoFileIO {
   std::string m_other_endpoint_name;
 };
 
-} // namespace lldb_vscode
+} // namespace lldb_dap
 
-#endif // LLDB_TOOLS_LLDB_VSCODE_FIFOFILES_H
+#endif // LLDB_TOOLS_LLDB_DAP_FIFOFILES_H
diff --git a/lldb/tools/lldb-vscode/FunctionBreakpoint.cpp b/lldb/tools/lldb-dap/FunctionBreakpoint.cpp
similarity index 87%
rename from lldb/tools/lldb-vscode/FunctionBreakpoint.cpp
rename to lldb/tools/lldb-dap/FunctionBreakpoint.cpp
index b7a8705d85..d4bdb97650 100644
--- a/lldb/tools/lldb-vscode/FunctionBreakpoint.cpp
+++ b/lldb/tools/lldb-dap/FunctionBreakpoint.cpp
@@ -7,9 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "FunctionBreakpoint.h"
-#include "VSCode.h"
+#include "DAP.h"
 
-namespace lldb_vscode {
+namespace lldb_dap {
 
 FunctionBreakpoint::FunctionBreakpoint(const llvm::json::Object &obj)
     : BreakpointBase(obj), functionName(std::string(GetString(obj, "name"))) {}
@@ -17,7 +17,7 @@ FunctionBreakpoint::FunctionBreakpoint(const llvm::json::Object &obj)
 void FunctionBreakpoint::SetBreakpoint() {
   if (functionName.empty())
     return;
-  bp = g_vsc.target.BreakpointCreateByName(functionName.c_str());
+  bp = g_dap.target.BreakpointCreateByName(functionName.c_str());
   // See comments in BreakpointBase::GetBreakpointLabel() for details of why
   // we add a label to our breakpoints.
   bp.AddName(GetBreakpointLabel());
@@ -29,4 +29,4 @@ void FunctionBreakpoint::SetBreakpoint() {
     SetLogMessage();
 }
 
-} // namespace lldb_vscode
+} // namespace lldb_dap
diff --git a/lldb/tools/lldb-vscode/FunctionBreakpoint.h b/lldb/tools/lldb-dap/FunctionBreakpoint.h
similarity index 80%
rename from lldb/tools/lldb-vscode/FunctionBreakpoint.h
rename to lldb/tools/lldb-dap/FunctionBreakpoint.h
index 82eec89cd1..fc23e94e12 100644
--- a/lldb/tools/lldb-vscode/FunctionBreakpoint.h
+++ b/lldb/tools/lldb-dap/FunctionBreakpoint.h
@@ -6,12 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLDB_TOOLS_LLDB_VSCODE_FUNCTIONBREAKPOINT_H
-#define LLDB_TOOLS_LLDB_VSCODE_FUNCTIONBREAKPOINT_H
+#ifndef LLDB_TOOLS_LLDB_DAP_FUNCTIONBREAKPOINT_H
+#define LLDB_TOOLS_LLDB_DAP_FUNCTIONBREAKPOINT_H
 
 #include "BreakpointBase.h"
 
-namespace lldb_vscode {
+namespace lldb_dap {
 
 struct FunctionBreakpoint : public BreakpointBase {
   std::string functionName;
@@ -23,6 +23,6 @@ struct FunctionBreakpoint : public BreakpointBase {
   void SetBreakpoint();
 };
 
-} // namespace lldb_vscode
+} // namespace lldb_dap
 
 #endif
diff --git a/lldb/tools/lldb-vscode/IOStream.cpp b/lldb/tools/lldb-dap/IOStream.cpp
similarity index 99%
rename from lldb/tools/lldb-vscode/IOStream.cpp
rename to lldb/tools/lldb-dap/IOStream.cpp
index 6e2a89c5df..897ab791ed 100644
--- a/lldb/tools/lldb-vscode/IOStream.cpp
+++ b/lldb/tools/lldb-dap/IOStream.cpp
@@ -20,7 +20,7 @@
 #include <string>
 #include <vector>
 
-using namespace lldb_vscode;
+using namespace lldb_dap;
 
 StreamDescriptor::StreamDescriptor() = default;
 
diff --git a/lldb/tools/lldb-vscode/IOStream.h b/lldb/tools/lldb-dap/IOStream.h
similarity index 93%
rename from lldb/tools/lldb-vscode/IOStream.h
rename to lldb/tools/lldb-dap/IOStream.h
index 0eb9b6fefb..b625024191 100644
--- a/lldb/tools/lldb-vscode/IOStream.h
+++ b/lldb/tools/lldb-dap/IOStream.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLDB_TOOLS_LLDB_VSCODE_IOSTREAM_H
-#define LLDB_TOOLS_LLDB_VSCODE_IOSTREAM_H
+#ifndef LLDB_TOOLS_LLDB_DAP_IOSTREAM_H
+#define LLDB_TOOLS_LLDB_DAP_IOSTREAM_H
 
 #include "llvm/Config/llvm-config.h" // for LLVM_ON_UNIX
 
@@ -32,7 +32,7 @@ typedef int SOCKET;
 // types of files, so we can't simply have one code path that just uses read
 // and write everywhere.  So we need an abstraction in order to allow us to
 // treat them identically.
-namespace lldb_vscode {
+namespace lldb_dap {
 struct StreamDescriptor {
   StreamDescriptor();
   ~StreamDescriptor();
@@ -66,6 +66,6 @@ struct OutputStream {
 
   bool write_full(llvm::StringRef str);
 };
-} // namespace lldb_vscode
+} // namespace lldb_dap
 
 #endif
diff --git a/lldb/tools/lldb-vscode/JSONUtils.cpp b/lldb/tools/lldb-dap/JSONUtils.cpp
similarity index 98%
rename from lldb/tools/lldb-vscode/JSONUtils.cpp
rename to lldb/tools/lldb-dap/JSONUtils.cpp
index 6cf753170d..0daa8c11c1 100644
--- a/lldb/tools/lldb-vscode/JSONUtils.cpp
+++ b/lldb/tools/lldb-dap/JSONUtils.cpp
@@ -24,12 +24,12 @@
 #include "lldb/API/SBValue.h"
 #include "lldb/Host/PosixApi.h"
 
+#include "DAP.h"
 #include "ExceptionBreakpoint.h"
 #include "JSONUtils.h"
 #include "LLDBUtils.h"
-#include "VSCode.h"
 
-namespace lldb_vscode {
+namespace lldb_dap {
 
 void EmplaceSafeString(llvm::json::Object &obj, llvm::StringRef key,
                        llvm::StringRef str) {
@@ -140,7 +140,7 @@ TryCreateAutoSummaryForContainer(lldb::SBValue &v) {
   // We gate this feature because it performs GetNumChildren(), which can
   // cause performance issues because LLDB needs to complete possibly huge
   // types.
-  if (!g_vsc.enable_auto_variable_summaries)
+  if (!g_dap.enable_auto_variable_summaries)
     return std::nullopt;
 
   if (!v.MightHaveChildren())
@@ -197,7 +197,7 @@ TryCreateAutoSummaryForContainer(lldb::SBValue &v) {
 /// Try to create a summary string for the given value that doesn't have a
 /// summary of its own.
 static std::optional<std::string> TryCreateAutoSummary(lldb::SBValue value) {
-  if (!g_vsc.enable_auto_variable_summaries)
+  if (!g_dap.enable_auto_variable_summaries)
     return std::nullopt;
 
   // We use the dereferenced value for generating the summary.
@@ -326,7 +326,7 @@ llvm::json::Value CreateScope(const llvm::StringRef name,
   llvm::json::Object object;
   EmplaceSafeString(object, "name", name.str());
 
-  // TODO: Support "arguments" scope. At the moment lldb-vscode includes the
+  // TODO: Support "arguments" scope. At the moment lldb-dap includes the
   // arguments into the "locals" scope.
   if (variablesReference == VARREF_LOCALS) {
     object.try_emplace("presentationHint", "locals");
@@ -425,7 +425,7 @@ llvm::json::Value CreateBreakpoint(lldb::SBBreakpoint &bp,
 
   if (bp_addr.IsValid()) {
     std::string formatted_addr =
-        "0x" + llvm::utohexstr(bp_addr.GetLoadAddress(g_vsc.target));
+        "0x" + llvm::utohexstr(bp_addr.GetLoadAddress(g_dap.target));
     object.try_emplace("instructionReference", formatted_addr);
     auto line_entry = bp_addr.GetLineEntry();
     const auto line = line_entry.GetLine();
@@ -513,7 +513,7 @@ llvm::json::Value CreateModule(lldb::SBModule &module) {
     object.try_emplace("symbolStatus", "Symbols not found.");
   }
   std::string loaded_addr = std::to_string(
-      module.GetObjectFileHeaderAddress().GetLoadAddress(g_vsc.target));
+      module.GetObjectFileHeaderAddress().GetLoadAddress(g_dap.target));
   object.try_emplace("addressRange", loaded_addr);
   std::string version_str;
   uint32_t version_nums[3];
@@ -775,7 +775,7 @@ std::optional<llvm::json::Value> CreateSource(lldb::SBFrame &frame) {
 // }
 llvm::json::Value CreateStackFrame(lldb::SBFrame &frame) {
   llvm::json::Object object;
-  int64_t frame_id = MakeVSCodeFrameID(frame);
+  int64_t frame_id = MakeDAPFrameID(frame);
   object.try_emplace("id", frame_id);
 
   // `function_name` can be a nullptr, which throws an error when assigned to an
@@ -932,7 +932,7 @@ llvm::json::Value CreateThreadStopped(lldb::SBThread &thread,
     body.try_emplace("reason", "step");
     break;
   case lldb::eStopReasonBreakpoint: {
-    ExceptionBreakpoint *exc_bp = g_vsc.GetExceptionBPFromStopReason(thread);
+    ExceptionBreakpoint *exc_bp = g_dap.GetExceptionBPFromStopReason(thread);
     if (exc_bp) {
       body.try_emplace("reason", "exception");
       EmplaceSafeString(body, "description", exc_bp->label);
@@ -990,10 +990,10 @@ llvm::json::Value CreateThreadStopped(lldb::SBThread &thread,
     }
   }
   // "threadCausedFocus" is used in tests to validate breaking behavior.
-  if (tid == g_vsc.focus_tid) {
+  if (tid == g_dap.focus_tid) {
     body.try_emplace("threadCausedFocus", true);
   }
-  body.try_emplace("preserveFocusHint", tid != g_vsc.focus_tid);
+  body.try_emplace("preserveFocusHint", tid != g_dap.focus_tid);
   body.try_emplace("allThreadsStopped", true);
   event.try_emplace("body", std::move(body));
   return llvm::json::Value(std::move(event));
@@ -1119,7 +1119,7 @@ llvm::json::Value CreateVariable(lldb::SBValue v, int64_t variablesReference,
     // We create a "[raw]" fake child for each synthetic type, so we have to
     // account for it when returning indexed variables. We don't need to do this
     // for non-indexed ones.
-    bool has_raw_child = is_synthetic && g_vsc.enable_synthetic_child_debugging;
+    bool has_raw_child = is_synthetic && g_dap.enable_synthetic_child_debugging;
     int actual_num_children = num_children + (has_raw_child ? 1 : 0);
     if (is_array) {
       object.try_emplace("indexedVariables", actual_num_children);
@@ -1170,8 +1170,8 @@ CreateRunInTerminalReverseRequest(const llvm::json::Object &launch_request,
 
   auto launch_request_arguments = launch_request.getObject("arguments");
   // The program path must be the first entry in the "args" field
-  std::vector<std::string> args = {
-      debug_adaptor_path.str(), "--comm-file", comm_file.str()};
+  std::vector<std::string> args = {debug_adaptor_path.str(), "--comm-file",
+                                   comm_file.str()};
   if (debugger_pid != LLDB_INVALID_PROCESS_ID) {
     args.push_back("--debugger-pid");
     args.push_back(std::to_string(debugger_pid));
@@ -1248,7 +1248,7 @@ void FilterAndGetValueForKey(const lldb::SBStructuredData data, const char *key,
 }
 
 void addStatistic(llvm::json::Object &event) {
-  lldb::SBStructuredData statistics = g_vsc.target.GetStatistics();
+  lldb::SBStructuredData statistics = g_dap.target.GetStatistics();
   bool is_dictionary =
       statistics.GetType() == lldb::eStructuredDataTypeDictionary;
   if (!is_dictionary)
@@ -1279,4 +1279,4 @@ std::string JSONToString(const llvm::json::Value &json) {
   return data;
 }
 
-} // namespace lldb_vscode
+} // namespace lldb_dap
diff --git a/lldb/tools/lldb-vscode/JSONUtils.h b/lldb/tools/lldb-dap/JSONUtils.h
similarity index 94%
rename from lldb/tools/lldb-vscode/JSONUtils.h
rename to lldb/tools/lldb-dap/JSONUtils.h
index 2013147f5d..ade8ed6822 100644
--- a/lldb/tools/lldb-vscode/JSONUtils.h
+++ b/lldb/tools/lldb-dap/JSONUtils.h
@@ -6,17 +6,17 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLDB_TOOLS_LLDB_VSCODE_JSONUTILS_H
-#define LLDB_TOOLS_LLDB_VSCODE_JSONUTILS_H
+#ifndef LLDB_TOOLS_LLDB_DAP_JSONUTILS_H
+#define LLDB_TOOLS_LLDB_DAP_JSONUTILS_H
 
-#include "VSCodeForward.h"
+#include "DAPForward.h"
 #include "lldb/API/SBModule.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/JSON.h"
 #include <cstdint>
 #include <optional>
 
-namespace lldb_vscode {
+namespace lldb_dap {
 
 /// Emplace a StringRef in a json::Object after enusring that the
 /// string is valid UTF8. If not, first call llvm::json::fixUTF8
@@ -154,7 +154,7 @@ std::vector<std::string> GetStrings(const llvm::json::Object *obj,
 /// and "success" set to true.
 ///
 /// \param[in] request
-///     The request object received from a call to VSCode::ReadJSON().
+///     The request object received from a call to DAP::ReadJSON().
 ///
 /// \param[in,out] response
 ///     An empty llvm::json::Object object that will be filled
@@ -214,7 +214,7 @@ void AppendBreakpoint(
     std::optional<llvm::StringRef> request_path = std::nullopt,
     std::optional<uint32_t> request_line = std::nullopt);
 
-/// Converts breakpoint location to a Visual Studio Code "Breakpoint"
+/// Converts breakpoint location to a debug adaptor protocol "Breakpoint".
 ///
 /// \param[in] bp
 ///     A LLDB breakpoint object to convert into a JSON value
@@ -233,11 +233,10 @@ void AppendBreakpoint(
 ///     fallback.
 ///
 /// \param[in] request_column
-///     An optional column to use when creating the resulting "Breakpoint" object.
-///     It is used if the breakpoint has no valid locations.
-///     It is useful to ensure the same column
-///     provided by the setBreakpoints request are returned to the IDE as a
-///     fallback.
+///     An optional column to use when creating the resulting "Breakpoint"
+///     object. It is used if the breakpoint has no valid locations. It is
+///     useful to ensure the same column provided by the setBreakpoints request
+///     are returned to the IDE as a fallback.
 ///
 /// \return
 ///     A "Breakpoint" JSON object with that follows the formal JSON
@@ -269,7 +268,7 @@ llvm::json::Value CreateModule(lldb::SBModule &module);
 llvm::json::Object CreateEventObject(const llvm::StringRef event_name);
 
 /// Create a "ExceptionBreakpointsFilter" JSON object as described in
-/// the Visual Studio Code debug adaptor definition.
+/// the debug adaptor definition.
 ///
 /// \param[in] bp
 ///     The exception breakpoint object to use
@@ -280,8 +279,7 @@ llvm::json::Object CreateEventObject(const llvm::StringRef event_name);
 llvm::json::Value
 CreateExceptionBreakpointFilter(const ExceptionBreakpoint &bp);
 
-/// Create a "Scope" JSON object as described in the Visual Studio Code
-/// debug adaptor definition.
+/// Create a "Scope" JSON object as described in the debug adaptor definition.
 ///
 /// \param[in] name
 ///     The value to place into the "name" key
@@ -302,8 +300,7 @@ llvm::json::Value CreateScope(const llvm::StringRef name,
                               int64_t variablesReference,
                               int64_t namedVariables, bool expensive);
 
-/// Create a "Source" JSON object as described in the Visual Studio Code
-/// debug adaptor definition.
+/// Create a "Source" JSON object as described in the debug adaptor definition.
 ///
 /// \param[in] line_entry
 ///     The LLDB line table to use when populating out the "Source"
@@ -330,7 +327,7 @@ llvm::json::Value CreateSource(llvm::StringRef source_path);
 /// object:
 ///   "id" - the stack frame ID as an integer
 ///   "name" - the function name as a string
-///   "source" - source file information as a "Source" VSCode object
+///   "source" - source file information as a "Source" DAP object
 ///   "line" - the source file line number as an integer
 ///   "column" - the source file column number as an integer
 ///
@@ -404,7 +401,7 @@ std::string CreateUniqueVariableNameForDisplay(lldb::SBValue v,
 ///   "value" - the value of the variable as a string
 ///   "type" - the typename of the variable as a string
 ///   "id" - a unique identifier for a value in case there are multiple
-///          variables with the same name. Other parts of the VSCode
+///          variables with the same name. Other parts of the DAP
 ///          protocol refer to values by name so this can help
 ///          disambiguate such cases if a IDE passes this "id" value
 ///          back down.
@@ -468,7 +465,7 @@ llvm::json::Value CreateCompileUnit(lldb::SBCompileUnit unit);
 ///     The fifo file used to communicate the with the target launcher.
 ///
 /// \param[in] debugger_pid
-///     The PID of the lldb-vscode instance that will attach to the target. The
+///     The PID of the lldb-dap instance that will attach to the target. The
 ///     launcher uses it on Linux tell the kernel that it should allow the
 ///     debugger process to attach.
 ///
@@ -490,6 +487,6 @@ llvm::json::Object CreateTerminatedEventObject();
 /// Convert a given JSON object to a string.
 std::string JSONToString(const llvm::json::Value &json);
 
-} // namespace lldb_vscode
+} // namespace lldb_dap
 
 #endif
diff --git a/lldb/tools/lldb-vscode/LLDBUtils.cpp b/lldb/tools/lldb-dap/LLDBUtils.cpp
similarity index 92%
rename from lldb/tools/lldb-vscode/LLDBUtils.cpp
rename to lldb/tools/lldb-dap/LLDBUtils.cpp
index 464195bdc6..955c11f636 100644
--- a/lldb/tools/lldb-vscode/LLDBUtils.cpp
+++ b/lldb/tools/lldb-dap/LLDBUtils.cpp
@@ -7,16 +7,16 @@
 //===----------------------------------------------------------------------===//
 
 #include "LLDBUtils.h"
-#include "VSCode.h"
+#include "DAP.h"
 
-namespace lldb_vscode {
+namespace lldb_dap {
 
 void RunLLDBCommands(llvm::StringRef prefix,
                      const llvm::ArrayRef<std::string> &commands,
                      llvm::raw_ostream &strm) {
   if (commands.empty())
     return;
-  lldb::SBCommandInterpreter interp = g_vsc.debugger.GetCommandInterpreter();
+  lldb::SBCommandInterpreter interp = g_dap.debugger.GetCommandInterpreter();
   if (!prefix.empty())
     strm << prefix << "\n";
   for (const auto &command : commands) {
@@ -78,9 +78,9 @@ uint32_t GetLLDBFrameID(uint64_t dap_frame_id) {
   return dap_frame_id & ((1u << THREAD_INDEX_SHIFT) - 1);
 }
 
-int64_t MakeVSCodeFrameID(lldb::SBFrame &frame) {
+int64_t MakeDAPFrameID(lldb::SBFrame &frame) {
   return ((int64_t)frame.GetThread().GetIndexID() << THREAD_INDEX_SHIFT) |
          frame.GetFrameID();
 }
 
-} // namespace lldb_vscode
+} // namespace lldb_dap
diff --git a/lldb/tools/lldb-vscode/LLDBUtils.h b/lldb/tools/lldb-dap/LLDBUtils.h
similarity index 82%
rename from lldb/tools/lldb-vscode/LLDBUtils.h
rename to lldb/tools/lldb-dap/LLDBUtils.h
index 8867589b18..a99f798835 100644
--- a/lldb/tools/lldb-vscode/LLDBUtils.h
+++ b/lldb/tools/lldb-dap/LLDBUtils.h
@@ -6,17 +6,17 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLDB_TOOLS_LLDB_VSCODE_LLDBUTILS_H
-#define LLDB_TOOLS_LLDB_VSCODE_LLDBUTILS_H
+#ifndef LLDB_TOOLS_LLDB_DAP_LLDBUTILS_H
+#define LLDB_TOOLS_LLDB_DAP_LLDBUTILS_H
 
-#include "VSCodeForward.h"
+#include "DAPForward.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/raw_ostream.h"
 #include <string>
 #include <vector>
 
-namespace lldb_vscode {
+namespace lldb_dap {
 
 /// Run a list of LLDB commands in the LLDB command interpreter.
 ///
@@ -68,7 +68,7 @@ bool ThreadHasStopReason(lldb::SBThread &thread);
 /// Given a LLDB frame, make a frame ID that is unique to a specific
 /// thread and frame.
 ///
-/// VSCode requires a Stackframe "id" to be unique, so we use the frame
+/// DAP requires a Stackframe "id" to be unique, so we use the frame
 /// index in the lower 32 bits and the thread index ID in the upper 32
 /// bits.
 ///
@@ -78,34 +78,34 @@ bool ThreadHasStopReason(lldb::SBThread &thread);
 /// \return
 ///     A unique integer that allows us to easily find the right
 ///     stack frame within a thread on subsequent VS code requests.
-int64_t MakeVSCodeFrameID(lldb::SBFrame &frame);
+int64_t MakeDAPFrameID(lldb::SBFrame &frame);
 
-/// Given a VSCode frame ID, convert to a LLDB thread index id.
+/// Given a DAP frame ID, convert to a LLDB thread index id.
 ///
-/// VSCode requires a Stackframe "id" to be unique, so we use the frame
+/// DAP requires a Stackframe "id" to be unique, so we use the frame
 /// index in the lower THREAD_INDEX_SHIFT bits and the thread index ID in
 /// the upper 32 - THREAD_INDEX_SHIFT bits.
 ///
 /// \param[in] dap_frame_id
-///     The VSCode frame ID to convert to a thread index ID.
+///     The DAP frame ID to convert to a thread index ID.
 ///
 /// \return
 ///     The LLDB thread index ID.
 uint32_t GetLLDBThreadIndexID(uint64_t dap_frame_id);
 
-/// Given a VSCode frame ID, convert to a LLDB frame ID.
+/// Given a DAP frame ID, convert to a LLDB frame ID.
 ///
-/// VSCode requires a Stackframe "id" to be unique, so we use the frame
+/// DAP requires a Stackframe "id" to be unique, so we use the frame
 /// index in the lower THREAD_INDEX_SHIFT bits and the thread index ID in
 /// the upper 32 - THREAD_INDEX_SHIFT bits.
 ///
 /// \param[in] dap_frame_id
-///     The VSCode frame ID to convert to a frame ID.
+///     The DAP frame ID to convert to a frame ID.
 ///
 /// \return
 ///     The LLDB frame index ID.
 uint32_t GetLLDBFrameID(uint64_t dap_frame_id);
 
-} // namespace lldb_vscode
+} // namespace lldb_dap
 
 #endif
diff --git a/lldb/tools/lldb-vscode/Options.td b/lldb/tools/lldb-dap/Options.td
similarity index 85%
rename from lldb/tools/lldb-vscode/Options.td
rename to lldb/tools/lldb-dap/Options.td
index a6ba0a318f..571967b232 100644
--- a/lldb/tools/lldb-vscode/Options.td
+++ b/lldb/tools/lldb-dap/Options.td
@@ -6,7 +6,7 @@ class R<list<string> prefixes, string name>
   : Option<prefixes, name, KIND_REMAINING_ARGS>;
 
 def help: F<"help">,
-  HelpText<"Prints out the usage information for the LLDB VSCode tool.">;
+  HelpText<"Prints out the usage information for the lldb-dap tool.">;
 def: Flag<["-"], "h">,
   Alias<help>,
   HelpText<"Alias for --help">;
@@ -19,7 +19,7 @@ def: Flag<["-"], "g">,
 
 def port: S<"port">,
   MetaVarName<"<port>">,
-  HelpText<"Communicate with the lldb-vscode tool over the defined port.">;
+  HelpText<"Communicate with the lldb-dap tool over the defined port.">;
 def: Separate<["-"], "p">,
   Alias<port>,
   HelpText<"Alias for --port">;
@@ -37,7 +37,7 @@ def comm_file: S<"comm-file">,
 
 def debugger_pid: S<"debugger-pid">,
   MetaVarName<"<pid>">,
-  HelpText<"The PID of the lldb-vscode instance that sent the launchInTerminal "
+  HelpText<"The PID of the lldb-dap instance that sent the launchInTerminal "
     "request when using --launch-target.">;
 
 def repl_mode: S<"repl-mode">,
diff --git a/lldb/tools/lldb-vscode/OutputRedirector.cpp b/lldb/tools/lldb-dap/OutputRedirector.cpp
similarity index 96%
rename from lldb/tools/lldb-vscode/OutputRedirector.cpp
rename to lldb/tools/lldb-dap/OutputRedirector.cpp
index 9243915f7d..4e6907ce6c 100644
--- a/lldb/tools/lldb-vscode/OutputRedirector.cpp
+++ b/lldb/tools/lldb-dap/OutputRedirector.cpp
@@ -18,7 +18,7 @@
 
 using namespace llvm;
 
-namespace lldb_vscode {
+namespace lldb_dap {
 
 Error RedirectFd(int fd, std::function<void(llvm::StringRef)> callback) {
   int new_fd[2];
@@ -59,4 +59,4 @@ Error RedirectFd(int fd, std::function<void(llvm::StringRef)> callback) {
   return Error::success();
 }
 
-} // namespace lldb_vscode
+} // namespace lldb_dap
diff --git a/lldb/tools/lldb-vscode/OutputRedirector.h b/lldb/tools/lldb-dap/OutputRedirector.h
similarity index 77%
rename from lldb/tools/lldb-vscode/OutputRedirector.h
rename to lldb/tools/lldb-dap/OutputRedirector.h
index c728367a21..dba5101677 100644
--- a/lldb/tools/lldb-vscode/OutputRedirector.h
+++ b/lldb/tools/lldb-dap/OutputRedirector.h
@@ -6,15 +6,15 @@
 //
 //===----------------------------------------------------------------------===/
 
-#ifndef LLDB_TOOLS_LLDB_VSCODE_OUTPUT_REDIRECTOR_H
-#define LLDB_TOOLS_LLDB_VSCODE_OUTPUT_REDIRECTOR_H
+#ifndef LLDB_TOOLS_LLDB_DAP_OUTPUT_REDIRECTOR_H
+#define LLDB_TOOLS_LLDB_DAP_OUTPUT_REDIRECTOR_H
 
 #include <thread>
 
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Error.h"
 
-namespace lldb_vscode {
+namespace lldb_dap {
 
 /// Redirects the output of a given file descriptor to a callback.
 ///
@@ -23,6 +23,6 @@ namespace lldb_vscode {
 ///     otherwise.
 llvm::Error RedirectFd(int fd, std::function<void(llvm::StringRef)> callback);
 
-} // namespace lldb_vscode
+} // namespace lldb_dap
 
-#endif // LLDB_TOOLS_LLDB_VSCODE_OUTPUT_REDIRECTOR_H
+#endif // LLDB_TOOLS_LLDB_DAP_OUTPUT_REDIRECTOR_H
diff --git a/lldb/tools/lldb-vscode/ProgressEvent.cpp b/lldb/tools/lldb-dap/ProgressEvent.cpp
similarity index 99%
rename from lldb/tools/lldb-vscode/ProgressEvent.cpp
rename to lldb/tools/lldb-dap/ProgressEvent.cpp
index 2930b5c1e2..8a660b50af 100644
--- a/lldb/tools/lldb-vscode/ProgressEvent.cpp
+++ b/lldb/tools/lldb-dap/ProgressEvent.cpp
@@ -12,7 +12,7 @@
 #include "llvm/Support/ErrorHandling.h"
 #include <optional>
 
-using namespace lldb_vscode;
+using namespace lldb_dap;
 using namespace llvm;
 
 // The minimum duration of an event for it to be reported
diff --git a/lldb/tools/lldb-vscode/ProgressEvent.h b/lldb/tools/lldb-dap/ProgressEvent.h
similarity index 97%
rename from lldb/tools/lldb-vscode/ProgressEvent.h
rename to lldb/tools/lldb-dap/ProgressEvent.h
index 85006e4c0a..dac21977ad 100644
--- a/lldb/tools/lldb-vscode/ProgressEvent.h
+++ b/lldb/tools/lldb-dap/ProgressEvent.h
@@ -12,17 +12,13 @@
 #include <queue>
 #include <thread>
 
-#include "VSCodeForward.h"
+#include "DAPForward.h"
 
 #include "llvm/Support/JSON.h"
 
-namespace lldb_vscode {
+namespace lldb_dap {
 
-enum ProgressEventType {
-  progressStart,
-  progressUpdate,
-  progressEnd
-};
+enum ProgressEventType { progressStart, progressUpdate, progressEnd };
 
 class ProgressEvent;
 using ProgressEventReportCallback = std::function<void(ProgressEvent &)>;
@@ -157,4 +153,4 @@ class ProgressEventReporter {
   std::mutex m_mutex;
 };
 
-} // namespace lldb_vscode
+} // namespace lldb_dap
diff --git a/lldb/tools/lldb-vscode/README.md b/lldb/tools/lldb-dap/README.md
similarity index 64%
rename from lldb/tools/lldb-vscode/README.md
rename to lldb/tools/lldb-dap/README.md
index 154ccefc5f..00ceb0bedc 100644
--- a/lldb/tools/lldb-vscode/README.md
+++ b/lldb/tools/lldb-dap/README.md
@@ -1,91 +1,99 @@
 
 # Table of Contents
 
-- [Introduction](#Introduction)
-- [Installation](#Installation-Visual-Studio-Code)
+- [Table of Contents](#table-of-contents)
+- [Introduction](#introduction)
+- [Installation for Visual Studio Code](#installation-for-visual-studio-code)
 - [Configurations](#configurations)
-	- [Launch Configuration Settings](#launch-configuration-settings)
-	- [Attach Configuration Settings](#attach-configuration-settings)
-	- [Example configurations](#example-configurations)
-		- [Launching](#launching)
-		- [Attach to process using process ID](#attach-using-pid)
-		- [Attach to process by name](#attach-by-name)
-		- [Loading a core file](#loading-a-core-file)
-- [Custom Debugger Commands](#custom-debugger-commands)
-  - [startDebugging](#startDebugging)
+  - [Launch Configuration Settings](#launch-configuration-settings)
+  - [Attaching Settings](#attaching-settings)
+  - [Example configurations](#example-configurations)
+    - [Launching](#launching)
+    - [Attach using PID](#attach-using-pid)
+    - [Attach by Name](#attach-by-name)
+    - [Loading a Core File](#loading-a-core-file)
+- [Custom debugger commands](#custom-debugger-commands)
+  - [startDebugging](#startdebugging)
+  - [repl-mode](#repl-mode)
 
 # Introduction
 
-The `lldb-vscode` tool creates a command line tool that implements the [Visual
-Studio Code Debug API](https://code.visualstudio.com/docs/extensionAPI/api-debugging).
-It can be installed as an extension for the Visual Studio Code and Nuclide IDE.
+The `lldb-dap` tool (formerly `lldb-vscode`) creates a command line tool that
+implements the [Debug Adapter
+Protocol](https://microsoft.github.io/debug-adapter-protocol/). It can be
+installed as an extension for Visual Studio Code and other IDEs supporting DAP.
 The protocol is easy to run remotely and also can allow other tools and IDEs to
 get a full featured debugger with a well defined protocol.
 
 # Installation for Visual Studio Code
 
-Installing the plug-in involves creating a directory in the `~/.vscode/extensions` folder and copying the package.json file that is in the same directory as this
-documentation into it, and copying to symlinking a lldb-vscode binary into
-the `bin` directory inside the plug-in directory.
-
-If you want to make a stand alone plug-in that you can send to others on unix systems:
-
-```
-$ mkdir -p ~/.vscode/extensions/llvm-org.lldb-vscode-0.1.0/bin
-$ cp package.json ~/.vscode/extensions/llvm-org.lldb-vscode-0.1.0
-$ cd ~/.vscode/extensions/llvm-org.lldb-vscode-0.1.0/bin
-$ cp /path/to/a/built/lldb-vscode .
-$ cp /path/to/a/built/liblldb.so .
+Installing the plug-in involves creating a directory in any location outside of
+`~/.vscode/extensions`. For example, `~/vscode-lldb` is a valid one. You'll also
+need a subfolder `bin`, e.g. `~/vscode-lldb/bin`. Then copy the `package.json`
+file that is in the same directory as this documentation into it, and symlink
+the `lldb-dap` binary into the `bin` directory inside the plug-in directory.
+
+Finally, on VS Code, execute the command
+`Developer: Install Extension from Location` and pick the folder you just
+created, which would be `~/vscode-lldb` following the example above.
+
+If you want to make a stand alone plug-in that you can send to others on UNIX
+systems:
+
+```bash
+mkdir -p ~/llvm-org.lldb-dap-0.1.0/bin
+cp package.json ~/llvm-org.lldb-dap-0.1.0
+cd ~/llvm-org.lldb-dap-0.1.0/bin
+cp /path/to/a/built/lldb-dap .
+cp /path/to/a/built/liblldb.so .
 ```
 
-It is important to note that the directory `~/.vscode/extensions` works for users logged in locally to the machine. If you are remoting into the box using Visual Studio Code's Remote plugins (SSH, WSL, Docker) it will look for extensions on `~/.vscode-server/extensions` only and you will not see your just installed lldb-vscode plug-in. If you want this plugin to be visible to remoting users, you will need to either repeat the process above for the `~/.vscode-server` folder or create a symbolic link from it to `~/.vscode/extensions`:
+If you want to make a stand alone plug-in that you can send to others on macOS
+systems:
 
+```bash
+mkdir -p ~/llvm-org.lldb-dap-0.1.0/bin
+cp package.json ~/llvm-org.lldb-dap-0.1.0
+cd ~/llvm-org.lldb-dap-0.1.0/bin
+cp /path/to/a/built/lldb-dap .
+rsync -av /path/to/a/built/LLDB.framework LLDB.framework
 ```
-$ cd ~/.vscode-server/extensions
-$ ln -s ~/.vscode/extensions/llvm-org.lldb-vscode-0.1.0  llvm-org.lldb-vscode-0.1.0
-```
-
-If you want to make a stand alone plug-in that you can send to others on macOS systems:
 
+You might need to create additional directories for the `liblldb.so` or
+`LLDB.framework` inside or next to the `bin` folder depending on how the
+[rpath](https://en.wikipedia.org/wiki/Rpath) is set in your `lldb-dap`
+binary. By default the `Debug` builds of LLDB usually includes
+the current executable directory in the rpath, so these steps should work for
+most people.
+
+To create a plug-in that symlinks into your `lldb-dap` in your build
+directory:
+
+```bash
+mkdir -p ~/llvm-org.lldb-dap-0.1.0/bin
+cp package.json ~/llvm-org.lldb-dap-0.1.0
+cd ~/llvm-org.lldb-dap-0.1.0/bin
+ln -s /path/to/a/built/lldb-dap
 ```
-$ mkdir -p ~/.vscode/extensions/llvm-org.lldb-vscode-0.1.0/bin
-$ cp package.json ~/.vscode/extensions/llvm-org.lldb-vscode-0.1.0
-$ cd ~/.vscode/extensions/llvm-org.lldb-vscode-0.1.0/bin
-$ cp /path/to/a/built/lldb-vscode .
-$ rsync -av /path/to/a/built/LLDB.framework LLDB.framework
-```
-
-You might need to create additional directories for the `liblldb.so` or `LLDB.framework` inside or next to the `bin` folder depending on how the [rpath](https://en.wikipedia.org/wiki/Rpath) is set in your `lldb-vscode` binary. By default the `Debug` builds of LLDB usually includes
-the current executable directory in the rpath, so these steps should work for most people.
-
-To create a plug-in that symlinks into your `lldb-vscode` in your build directory:
-
-```
-$ mkdir -p ~/.vscode/extensions/llvm-org.lldb-vscode-0.1.0/bin
-$ cp package.json ~/.vscode/extensions/llvm-org.lldb-vscode-0.1.0
-$ cd ~/.vscode/extensions/llvm-org.lldb-vscode-0.1.0/bin
-$ ln -s /path/to/a/built/lldb-vscode
-```
-
-This is handy if you want to debug and develope the `lldb-vscode` executable when adding features or fixing bugs.
-
 
+This is handy if you want to debug and develop the `lldb-dap` executable
+when adding features or fixing bugs.
 
 # Configurations
 
 Launching to attaching require you to create a [launch configuration](https://code.visualstudio.com/Docs/editor/debugging#_launch-configurations). This file
-defines arguments that get passed to `lldb-vscode` and the configuration settings
+defines arguments that get passed to `lldb-dap` and the configuration settings
 control how the launch or attach happens.
 
 ## Launch Configuration Settings
 
 When you launch a program with Visual Studio Code you will need to create a [launch.json](https://code.visualstudio.com/Docs/editor/debugging#_launch-configurations)
-file that defines how your program will be run. The JSON configuration file can contain the following `lldb-vscode` specific launch key/value pairs:
+file that defines how your program will be run. The JSON configuration file can contain the following `lldb-dap` specific launch key/value pairs:
 
 |parameter          |type|req |         |
 |-------------------|----|:--:|---------|
 |**name**           |string|Y| A configuration name that will be displayed in the IDE.
-|**type**           |string|Y| Must be "lldb-vscode".
+|**type**           |string|Y| Must be "lldb-dap".
 |**request**        |string|Y| Must be "launch".
 |**program**        |string|Y| Path to the executable to launch.
 |**args**           |[string]|| An array of command line argument strings to be passed to the program being launched.
@@ -99,7 +107,7 @@ file that defines how your program will be run. The JSON configuration file can
 |**exitCommands**   |[string]| | LLDB commands executed when the program exits. Commands and command output will be sent to the debugger console when they are executed.
 |**terminateCommands** |[string]| | LLDB commands executed when the debugging session ends. Commands and command output will be sent to the debugger console when they are executed.
 |**sourceMap**      |[string[2]]| | Specify an array of path re-mappings. Each element in the array must be a two element array containing a source and destination pathname.
-|**debuggerRoot**   | string| |Specify a working directory to use when launching lldb-vscode. If the debug information in your executable contains relative paths, this option can be used so that `lldb-vscode` can find source files and object files that have relative paths.
+|**debuggerRoot**   | string| |Specify a working directory to use when launching lldb-dap. If the debug information in your executable contains relative paths, this option can be used so that `lldb-dap` can find source files and object files that have relative paths.
 
 ## Attaching Settings
 
@@ -109,12 +117,12 @@ When attaching to a process using LLDB you can attach in a few ways
 2. Attach to an existing process by name
 3. Attach by name by waiting for the next instance of a process to launch
 
-The JSON configuration file can contain the following `lldb-vscode` specific launch key/value pairs:
+The JSON configuration file can contain the following `lldb-dap` specific launch key/value pairs:
 
 |parameter          |type    |req |         |
 |-------------------|--------|:--:|---------|
 |**name**           |string  |Y| A configuration name that will be displayed in the IDE.
-|**type**           |string  |Y| Must be "lldb-vscode".
+|**type**           |string  |Y| Must be "lldb-dap".
 |**request**        |string  |Y| Must be "attach".
 |**program**        |string  | | Path to the executable to attach to. This value is optional but can help to resolve breakpoints prior the attaching to the program.
 |**pid**            |number  | | The process id of the process you wish to attach to. If **pid** is omitted, the debugger will attempt to attach to the program by finding a process whose file name matches the file name from **porgram**. Setting this value to `${command:pickMyProcess}` will allow interactive process selection in the IDE.
@@ -127,7 +135,6 @@ The JSON configuration file can contain the following `lldb-vscode` specific lau
 |**terminateCommands** |[string]| | LLDB commands executed when the debugging session ends. Commands and command output will be sent to the debugger console when they are executed.
 |**attachCommands** |[string]| | LLDB commands that will be executed after **preRunCommands** which take place of the code that normally does the attach. The commands can create a new target and attach or launch it however desired. This allows custom launch and attach configurations. Core files can use `target create --core /path/to/core` to attach to core files.
 
-
 ## Example configurations
 
 ### Launching
@@ -137,7 +144,7 @@ adds `FOO=1` and `bar` to the environment:
 
 ```javascript
 {
-  "type": "lldb-vscode",
+  "type": "lldb-dap",
   "request": "launch",
   "name": "Debug",
   "program": "/tmp/a.out",
@@ -152,7 +159,7 @@ This will attach to a process `a.out` whose process ID is 123:
 
 ```javascript
 {
-  "type": "lldb-vscode",
+  "type": "lldb-dap",
   "request": "attach",
   "name": "Attach to PID",
   "program": "/tmp/a.out",
@@ -169,7 +176,7 @@ above configuration:
 ```javascript
 {
   "name": "Attach to Name",
-  "type": "lldb-vscode",
+  "type": "lldb-dap",
   "request": "attach",
   "program": "/tmp/a.out",
 }
@@ -181,7 +188,7 @@ to be launched you can add the "waitFor" key value pair:
 ```javascript
 {
   "name": "Attach to Name (wait)",
-  "type": "lldb-vscode",
+  "type": "lldb-dap",
   "request": "attach",
   "program": "/tmp/a.out",
   "waitFor": true
@@ -191,7 +198,6 @@ to be launched you can add the "waitFor" key value pair:
 This will work as long as the architecture, vendor and OS supports waiting
 for processes. Currently MacOS is the only platform that supports this.
 
-
 ### Loading a Core File
 
 This loads the coredump file `/cores/123.core` associated with the program
@@ -200,21 +206,53 @@ This loads the coredump file `/cores/123.core` associated with the program
 ```javascript
 {
   "name": "Load coredump",
-  "type": "lldb-vscode",
+  "type": "lldb-dap",
   "request": "attach",
   "coreFile": "/cores/123.core",
   "program": "/tmp/a.out"
 }
 ```
 
+### Connect to a Debug Server on the Current Machine
+
+This connects to a debug server (e.g. `lldb-server`, `gdbserver`) on
+the current machine, that is debugging the program `/tmp/a.out` and listening
+locally on port `2345`.
+
+```javascript
+{
+  "name": "Local Debug Server",
+  "type": "lldb-dap",
+  "request": "attach",
+  "program": "/tmp/a.out",
+  "attachCommands": ["gdb-remote 2345"],
+}
+```
+
+### Connect to a Debug Server on Another Machine
+
+This connects to a debug server running on another machine with hostname
+`hostnmame`. Which is debugging the program `/tmp/a.out` and listening on
+port `5678` of that other machine.
+
+```javascript
+{
+  "name": "Remote Debug Server",
+  "type": "lldb-dap",
+  "request": "attach",
+  "program": "/tmp/a.out",
+  "attachCommands": ["gdb-remote hostname:5678"],
+}
+```
+
 # Custom debugger commands
 
-The `lldb-vscode` tool includes additional custom commands to support the Debug
+The `lldb-dap` tool includes additional custom commands to support the Debug
 Adapter Protocol features.
 
 ## startDebugging
 
-Using the command `lldb-vscode startDebugging` it is possible to trigger a
+Using the command `lldb-dap startDebugging` it is possible to trigger a
 reverse request to the client requesting a child debug session with the
 specified configuration. For example, this can be used to attached to forked or
 spawned processes. For more information see
@@ -223,7 +261,7 @@ spawned processes. For more information see
 The custom command has the following format:
 
 ```
-lldb-vscode startDebugging <launch|attach> <configuration>
+lldb-dap startDebugging <launch|attach> <configuration>
 ```
 
 This will launch a server and then request a child debug session for a client.
@@ -232,26 +270,26 @@ This will launch a server and then request a child debug session for a client.
 {
   "program": "server",
   "postRunCommand": [
-    "lldb-vscode startDebugging launch '{\"program\":\"client\"}'"
+    "lldb-dap startDebugging launch '{\"program\":\"client\"}'"
   ]
 }
 ```
 
 ## repl-mode
 
-Inspect or adjust the behavior of lldb-vscode repl evaluation requests. The
+Inspect or adjust the behavior of lldb-dap repl evaluation requests. The
 supported modes are `variable`, `command` and `auto`.
 
-*  `variable` - Variable mode expressions are evaluated in the context of the
+- `variable` - Variable mode expressions are evaluated in the context of the
    current frame. Use a `\`` prefix on the command to run an lldb command.
-*  `command` - Command mode expressions are evaluated as lldb commands, as a
+- `command` - Command mode expressions are evaluated as lldb commands, as a
    result, values printed by lldb are always stringified representations of the
    expression output.
-*  `auto` - Auto mode will attempt to infer if the expression represents an lldb
+- `auto` - Auto mode will attempt to infer if the expression represents an lldb
    command or a variable expression. A heuristic is used to infer if the input
    represents a variable or a command. Use a `\`` prefix to ensure an expression
    is evaluated as a command.
 
 The initial repl-mode can be configured with the cli flag `--repl-mode=<mode>`
 and may also be adjusted at runtime using the lldb command
-`lldb-vscode repl-mode <mode>`.
+`lldb-dap repl-mode <mode>`.
diff --git a/lldb/tools/lldb-vscode/RunInTerminal.cpp b/lldb/tools/lldb-dap/RunInTerminal.cpp
similarity index 98%
rename from lldb/tools/lldb-vscode/RunInTerminal.cpp
rename to lldb/tools/lldb-dap/RunInTerminal.cpp
index 25260afa64..ad019b8a56 100644
--- a/lldb/tools/lldb-vscode/RunInTerminal.cpp
+++ b/lldb/tools/lldb-dap/RunInTerminal.cpp
@@ -25,7 +25,7 @@
 
 using namespace llvm;
 
-namespace lldb_vscode {
+namespace lldb_dap {
 
 const RunInTerminalMessagePid *RunInTerminalMessage::GetAsPidMessage() const {
   return static_cast<const RunInTerminalMessagePid *>(this);
@@ -163,11 +163,11 @@ std::string RunInTerminalDebugAdapterCommChannel::GetLauncherError() {
 Expected<std::shared_ptr<FifoFile>> CreateRunInTerminalCommFile() {
   SmallString<256> comm_file;
   if (std::error_code EC = sys::fs::getPotentiallyUniqueTempFileName(
-          "lldb-vscode-run-in-terminal-comm", "", comm_file))
+          "lldb-dap-run-in-terminal-comm", "", comm_file))
     return createStringError(EC, "Error making unique file name for "
                                  "runInTerminal communication files");
 
   return CreateFifoFile(comm_file.str());
 }
 
-} // namespace lldb_vscode
+} // namespace lldb_dap
diff --git a/lldb/tools/lldb-vscode/RunInTerminal.h b/lldb/tools/lldb-dap/RunInTerminal.h
similarity index 94%
rename from lldb/tools/lldb-vscode/RunInTerminal.h
rename to lldb/tools/lldb-dap/RunInTerminal.h
index cdccdd6bcf..2fbe3acbb4 100644
--- a/lldb/tools/lldb-vscode/RunInTerminal.h
+++ b/lldb/tools/lldb-dap/RunInTerminal.h
@@ -6,15 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLDB_TOOLS_LLDB_VSCODE_RUNINTERMINAL_H
-#define LLDB_TOOLS_LLDB_VSCODE_RUNINTERMINAL_H
+#ifndef LLDB_TOOLS_LLDB_DAP_RUNINTERMINAL_H
+#define LLDB_TOOLS_LLDB_DAP_RUNINTERMINAL_H
 
 #include "FifoFiles.h"
 
 #include <future>
 #include <thread>
 
-namespace lldb_vscode {
+namespace lldb_dap {
 
 enum RunInTerminalMessageKind {
   eRunInTerminalMessageKindPID = 0,
@@ -124,6 +124,6 @@ class RunInTerminalDebugAdapterCommChannel {
 /// the runInTerminal launcher.
 llvm::Expected<std::shared_ptr<FifoFile>> CreateRunInTerminalCommFile();
 
-} // namespace lldb_vscode
+} // namespace lldb_dap
 
-#endif // LLDB_TOOLS_LLDB_VSCODE_RUNINTERMINAL_H
+#endif // LLDB_TOOLS_LLDB_DAP_RUNINTERMINAL_H
diff --git a/lldb/tools/lldb-vscode/SourceBreakpoint.cpp b/lldb/tools/lldb-dap/SourceBreakpoint.cpp
similarity index 88%
rename from lldb/tools/lldb-vscode/SourceBreakpoint.cpp
rename to lldb/tools/lldb-dap/SourceBreakpoint.cpp
index 7c57bf7d7c..3bd83c0a68 100644
--- a/lldb/tools/lldb-vscode/SourceBreakpoint.cpp
+++ b/lldb/tools/lldb-dap/SourceBreakpoint.cpp
@@ -7,9 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "SourceBreakpoint.h"
-#include "VSCode.h"
+#include "DAP.h"
 
-namespace lldb_vscode {
+namespace lldb_dap {
 
 SourceBreakpoint::SourceBreakpoint(const llvm::json::Object &obj)
     : BreakpointBase(obj), line(GetUnsigned(obj, "line", 0)),
@@ -17,7 +17,7 @@ SourceBreakpoint::SourceBreakpoint(const llvm::json::Object &obj)
 
 void SourceBreakpoint::SetBreakpoint(const llvm::StringRef source_path) {
   lldb::SBFileSpecList module_list;
-  bp = g_vsc.target.BreakpointCreateByLocation(source_path.str().c_str(), line,
+  bp = g_dap.target.BreakpointCreateByLocation(source_path.str().c_str(), line,
                                                column, 0, module_list);
   // See comments in BreakpointBase::GetBreakpointLabel() for details of why
   // we add a label to our breakpoints.
@@ -30,4 +30,4 @@ void SourceBreakpoint::SetBreakpoint(const llvm::StringRef source_path) {
     SetLogMessage();
 }
 
-} // namespace lldb_vscode
+} // namespace lldb_dap
diff --git a/lldb/tools/lldb-vscode/SourceBreakpoint.h b/lldb/tools/lldb-dap/SourceBreakpoint.h
similarity index 87%
rename from lldb/tools/lldb-vscode/SourceBreakpoint.h
rename to lldb/tools/lldb-dap/SourceBreakpoint.h
index 2891d57ee8..f4b54a44fc 100644
--- a/lldb/tools/lldb-vscode/SourceBreakpoint.h
+++ b/lldb/tools/lldb-dap/SourceBreakpoint.h
@@ -6,13 +6,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLDB_TOOLS_LLDB_VSCODE_SOURCEBREAKPOINT_H
-#define LLDB_TOOLS_LLDB_VSCODE_SOURCEBREAKPOINT_H
+#ifndef LLDB_TOOLS_LLDB_DAP_SOURCEBREAKPOINT_H
+#define LLDB_TOOLS_LLDB_DAP_SOURCEBREAKPOINT_H
 
 #include "BreakpointBase.h"
 #include "llvm/ADT/StringRef.h"
 
-namespace lldb_vscode {
+namespace lldb_dap {
 
 struct SourceBreakpoint : public BreakpointBase {
 
@@ -33,6 +33,6 @@ inline bool operator<(const SourceBreakpoint &lhs,
   return lhs.line < rhs.line;
 }
 
-} // namespace lldb_vscode
+} // namespace lldb_dap
 
 #endif
diff --git a/lldb/tools/lldb-vscode/lldb-vscode-Info.plist.in b/lldb/tools/lldb-dap/lldb-dap-Info.plist.in
similarity index 88%
rename from lldb/tools/lldb-vscode/lldb-vscode-Info.plist.in
rename to lldb/tools/lldb-dap/lldb-dap-Info.plist.in
index 2098e190d6..7d01d3145d 100644
--- a/lldb/tools/lldb-vscode/lldb-vscode-Info.plist.in
+++ b/lldb/tools/lldb-dap/lldb-dap-Info.plist.in
@@ -5,11 +5,11 @@
 	<key>CFBundleDevelopmentRegion</key>
 	<string>English</string>
 	<key>CFBundleIdentifier</key>
-	<string>com.apple.lldb-vscode</string>
+	<string>com.apple.lldb-dap</string>
 	<key>CFBundleInfoDictionaryVersion</key>
 	<string>6.0</string>
 	<key>CFBundleName</key>
-	<string>lldb-vscode</string>
+	<string>lldb-dap</string>
 	<key>CFBundleVersion</key>
 	<string>${LLDB_VERSION}</string>
 	<key>SecTaskAccess</key>
diff --git a/lldb/tools/lldb-vscode/lldb-vscode.cpp b/lldb/tools/lldb-dap/lldb-dap.cpp
similarity index 88%
rename from lldb/tools/lldb-vscode/lldb-vscode.cpp
rename to lldb/tools/lldb-dap/lldb-dap.cpp
index 3904d430c4..a218e85664 100644
--- a/lldb/tools/lldb-vscode/lldb-vscode.cpp
+++ b/lldb/tools/lldb-dap/lldb-dap.cpp
@@ -1,4 +1,4 @@
-//===-- lldb-vscode.cpp -----------------------------------------*- C++ -*-===//
+//===-- lldb-dap.cpp -----------------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "VSCode.h"
+#include "DAP.h"
 
 #include <cassert>
 #include <climits>
@@ -73,7 +73,7 @@
 typedef int socklen_t;
 #endif
 
-using namespace lldb_vscode;
+using namespace lldb_dap;
 
 namespace {
 using namespace llvm::opt;
@@ -97,9 +97,9 @@ static constexpr llvm::opt::OptTable::Info InfoTable[] = {
 #include "Options.inc"
 #undef OPTION
 };
-class LLDBVSCodeOptTable : public llvm::opt::GenericOptTable {
+class LLDBDAPOptTable : public llvm::opt::GenericOptTable {
 public:
-  LLDBVSCodeOptTable() : llvm::opt::GenericOptTable(InfoTable, true) {}
+  LLDBDAPOptTable() : llvm::opt::GenericOptTable(InfoTable, true) {}
 };
 
 typedef void (*RequestCallback)(const llvm::json::Object &command);
@@ -109,11 +109,11 @@ enum LaunchMethod { Launch, Attach, AttachForSuspendedLaunch };
 lldb::SBValueList *GetTopLevelScope(int64_t variablesReference) {
   switch (variablesReference) {
   case VARREF_LOCALS:
-    return &g_vsc.variables.locals;
+    return &g_dap.variables.locals;
   case VARREF_GLOBALS:
-    return &g_vsc.variables.globals;
+    return &g_dap.variables.globals;
   case VARREF_REGS:
-    return &g_vsc.variables.registers;
+    return &g_dap.variables.registers;
   default:
     return nullptr;
   }
@@ -125,8 +125,8 @@ SOCKET AcceptConnection(int portno) {
   struct sockaddr_in serv_addr, cli_addr;
   SOCKET sockfd = socket(AF_INET, SOCK_STREAM, 0);
   if (sockfd < 0) {
-    if (g_vsc.log)
-      *g_vsc.log << "error: opening socket (" << strerror(errno) << ")"
+    if (g_dap.log)
+      *g_dap.log << "error: opening socket (" << strerror(errno) << ")"
                  << std::endl;
   } else {
     memset((char *)&serv_addr, 0, sizeof(serv_addr));
@@ -135,8 +135,8 @@ SOCKET AcceptConnection(int portno) {
     serv_addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
     serv_addr.sin_port = htons(portno);
     if (bind(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) {
-      if (g_vsc.log)
-        *g_vsc.log << "error: binding socket (" << strerror(errno) << ")"
+      if (g_dap.log)
+        *g_dap.log << "error: binding socket (" << strerror(errno) << ")"
                    << std::endl;
     } else {
       listen(sockfd, 5);
@@ -145,8 +145,8 @@ SOCKET AcceptConnection(int portno) {
           llvm::sys::RetryAfterSignal(static_cast<SOCKET>(-1), accept, sockfd,
                                       (struct sockaddr *)&cli_addr, &clilen);
       if (newsockfd < 0)
-        if (g_vsc.log)
-          *g_vsc.log << "error: accept (" << strerror(errno) << ")"
+        if (g_dap.log)
+          *g_dap.log << "error: accept (" << strerror(errno) << ")"
                      << std::endl;
     }
 #if defined(_WIN32)
@@ -176,7 +176,7 @@ void SendProcessExitedEvent(lldb::SBProcess &process) {
   llvm::json::Object body;
   body.try_emplace("exitCode", (int64_t)process.GetExitStatus());
   event.try_emplace("body", std::move(body));
-  g_vsc.SendJSON(llvm::json::Value(std::move(event)));
+  g_dap.SendJSON(llvm::json::Value(std::move(event)));
 }
 
 void SendThreadExitedEvent(lldb::tid_t tid) {
@@ -185,29 +185,29 @@ void SendThreadExitedEvent(lldb::tid_t tid) {
   body.try_emplace("reason", "exited");
   body.try_emplace("threadId", (int64_t)tid);
   event.try_emplace("body", std::move(body));
-  g_vsc.SendJSON(llvm::json::Value(std::move(event)));
+  g_dap.SendJSON(llvm::json::Value(std::move(event)));
 }
 
 // Send a "continued" event to indicate the process is in the running state.
 void SendContinuedEvent() {
-  lldb::SBProcess process = g_vsc.target.GetProcess();
+  lldb::SBProcess process = g_dap.target.GetProcess();
   if (!process.IsValid()) {
     return;
   }
 
   // If the focus thread is not set then we haven't reported any thread status
   // to the client, so nothing to report.
-  if (!g_vsc.configuration_done_sent ||
-      g_vsc.focus_tid == LLDB_INVALID_THREAD_ID) {
+  if (!g_dap.configuration_done_sent ||
+      g_dap.focus_tid == LLDB_INVALID_THREAD_ID) {
     return;
   }
 
   llvm::json::Object event(CreateEventObject("continued"));
   llvm::json::Object body;
-  body.try_emplace("threadId", (int64_t)g_vsc.focus_tid);
+  body.try_emplace("threadId", (int64_t)g_dap.focus_tid);
   body.try_emplace("allThreadsContinued", true);
   event.try_emplace("body", std::move(body));
-  g_vsc.SendJSON(llvm::json::Value(std::move(event)));
+  g_dap.SendJSON(llvm::json::Value(std::move(event)));
 }
 
 // Send a "terminated" event to indicate the process is done being
@@ -217,7 +217,7 @@ void SendTerminatedEvent() {
   // the threads executing EventThreadFunction and request_discontinue
   // respectively may call SendTerminatedEvent simultaneously. Without any
   // synchronization, the thread executing EventThreadFunction may set
-  // g_vsc.sent_terminated_event before the thread executing
+  // g_dap.sent_terminated_event before the thread executing
   // request_discontinue has had a chance to test it, in which case the latter
   // would move ahead to issue a response to the disconnect request. Said
   // response may get dispatched ahead of the terminated event compelling the
@@ -226,24 +226,24 @@ void SendTerminatedEvent() {
   // synchronize simultaneous calls to SendTerminatedEvent.
   static std::mutex mutex;
   std::lock_guard<std::mutex> locker(mutex);
-  if (!g_vsc.sent_terminated_event) {
-    g_vsc.sent_terminated_event = true;
-    g_vsc.RunTerminateCommands();
+  if (!g_dap.sent_terminated_event) {
+    g_dap.sent_terminated_event = true;
+    g_dap.RunTerminateCommands();
     // Send a "terminated" event
     llvm::json::Object event(CreateTerminatedEventObject());
-    g_vsc.SendJSON(llvm::json::Value(std::move(event)));
+    g_dap.SendJSON(llvm::json::Value(std::move(event)));
   }
 }
 
 // Send a thread stopped event for all threads as long as the process
 // is stopped.
 void SendThreadStoppedEvent() {
-  lldb::SBProcess process = g_vsc.target.GetProcess();
+  lldb::SBProcess process = g_dap.target.GetProcess();
   if (process.IsValid()) {
     auto state = process.GetState();
     if (state == lldb::eStateStopped) {
       llvm::DenseSet<lldb::tid_t> old_thread_ids;
-      old_thread_ids.swap(g_vsc.thread_ids);
+      old_thread_ids.swap(g_dap.thread_ids);
       uint32_t stop_id = process.GetStopID();
       const uint32_t num_threads = process.GetNumThreads();
 
@@ -259,10 +259,10 @@ void SendThreadStoppedEvent() {
         const lldb::tid_t tid = thread.GetThreadID();
         const bool has_reason = ThreadHasStopReason(thread);
         // If the focus thread doesn't have a stop reason, clear the thread ID
-        if (tid == g_vsc.focus_tid) {
+        if (tid == g_dap.focus_tid) {
           focus_thread_exists = true;
           if (!has_reason)
-            g_vsc.focus_tid = LLDB_INVALID_THREAD_ID;
+            g_dap.focus_tid = LLDB_INVALID_THREAD_ID;
         }
         if (has_reason) {
           ++num_threads_with_reason;
@@ -271,47 +271,47 @@ void SendThreadStoppedEvent() {
         }
       }
 
-      // We will have cleared g_vsc.focus_tid if the focus thread doesn't have
+      // We will have cleared g_dap.focus_tid if the focus thread doesn't have
       // a stop reason, so if it was cleared, or wasn't set, or doesn't exist,
       // then set the focus thread to the first thread with a stop reason.
-      if (!focus_thread_exists || g_vsc.focus_tid == LLDB_INVALID_THREAD_ID)
-        g_vsc.focus_tid = first_tid_with_reason;
+      if (!focus_thread_exists || g_dap.focus_tid == LLDB_INVALID_THREAD_ID)
+        g_dap.focus_tid = first_tid_with_reason;
 
       // If no threads stopped with a reason, then report the first one so
       // we at least let the UI know we stopped.
       if (num_threads_with_reason == 0) {
         lldb::SBThread thread = process.GetThreadAtIndex(0);
-        g_vsc.focus_tid = thread.GetThreadID();
-        g_vsc.SendJSON(CreateThreadStopped(thread, stop_id));
+        g_dap.focus_tid = thread.GetThreadID();
+        g_dap.SendJSON(CreateThreadStopped(thread, stop_id));
       } else {
         for (uint32_t thread_idx = 0; thread_idx < num_threads; ++thread_idx) {
           lldb::SBThread thread = process.GetThreadAtIndex(thread_idx);
-          g_vsc.thread_ids.insert(thread.GetThreadID());
+          g_dap.thread_ids.insert(thread.GetThreadID());
           if (ThreadHasStopReason(thread)) {
-            g_vsc.SendJSON(CreateThreadStopped(thread, stop_id));
+            g_dap.SendJSON(CreateThreadStopped(thread, stop_id));
           }
         }
       }
 
       for (auto tid : old_thread_ids) {
-        auto end = g_vsc.thread_ids.end();
-        auto pos = g_vsc.thread_ids.find(tid);
+        auto end = g_dap.thread_ids.end();
+        auto pos = g_dap.thread_ids.find(tid);
         if (pos == end)
           SendThreadExitedEvent(tid);
       }
     } else {
-      if (g_vsc.log)
-        *g_vsc.log << "error: SendThreadStoppedEvent() when process"
+      if (g_dap.log)
+        *g_dap.log << "error: SendThreadStoppedEvent() when process"
                       " isn't stopped ("
                    << lldb::SBDebugger::StateAsCString(state) << ')'
                    << std::endl;
     }
   } else {
-    if (g_vsc.log)
-      *g_vsc.log << "error: SendThreadStoppedEvent() invalid process"
+    if (g_dap.log)
+      *g_dap.log << "error: SendThreadStoppedEvent() invalid process"
                  << std::endl;
   }
-  g_vsc.RunStopCommands();
+  g_dap.RunStopCommands();
 }
 
 // "ProcessEvent": {
@@ -369,13 +369,13 @@ void SendThreadStoppedEvent() {
 //   ]
 // }
 void SendProcessEvent(LaunchMethod launch_method) {
-  lldb::SBFileSpec exe_fspec = g_vsc.target.GetExecutable();
+  lldb::SBFileSpec exe_fspec = g_dap.target.GetExecutable();
   char exe_path[PATH_MAX];
   exe_fspec.GetPath(exe_path, sizeof(exe_path));
   llvm::json::Object event(CreateEventObject("process"));
   llvm::json::Object body;
   EmplaceSafeString(body, "name", std::string(exe_path));
-  const auto pid = g_vsc.target.GetProcess().GetProcessID();
+  const auto pid = g_dap.target.GetProcess().GetProcessID();
   body.try_emplace("systemProcessId", (int64_t)pid);
   body.try_emplace("isLocalProcess", true);
   const char *startMethod = nullptr;
@@ -392,7 +392,7 @@ void SendProcessEvent(LaunchMethod launch_method) {
   }
   body.try_emplace("startMethod", startMethod);
   event.try_emplace("body", std::move(body));
-  g_vsc.SendJSON(llvm::json::Value(std::move(event)));
+  g_dap.SendJSON(llvm::json::Value(std::move(event)));
 }
 
 // Grab any STDOUT and STDERR from the process and send it up to VS Code
@@ -401,22 +401,22 @@ void SendStdOutStdErr(lldb::SBProcess &process) {
   char buffer[1024];
   size_t count;
   while ((count = process.GetSTDOUT(buffer, sizeof(buffer))) > 0)
-    g_vsc.SendOutput(OutputType::Stdout, llvm::StringRef(buffer, count));
+    g_dap.SendOutput(OutputType::Stdout, llvm::StringRef(buffer, count));
   while ((count = process.GetSTDERR(buffer, sizeof(buffer))) > 0)
-    g_vsc.SendOutput(OutputType::Stderr, llvm::StringRef(buffer, count));
+    g_dap.SendOutput(OutputType::Stderr, llvm::StringRef(buffer, count));
 }
 
 void ProgressEventThreadFunction() {
-  lldb::SBListener listener("lldb-vscode.progress.listener");
-  g_vsc.debugger.GetBroadcaster().AddListener(
+  lldb::SBListener listener("lldb-dap.progress.listener");
+  g_dap.debugger.GetBroadcaster().AddListener(
       listener, lldb::SBDebugger::eBroadcastBitProgress);
-  g_vsc.broadcaster.AddListener(listener, eBroadcastBitStopProgressThread);
+  g_dap.broadcaster.AddListener(listener, eBroadcastBitStopProgressThread);
   lldb::SBEvent event;
   bool done = false;
   while (!done) {
     if (listener.WaitForEvent(1, event)) {
       const auto event_mask = event.GetType();
-      if (event.BroadcasterMatchesRef(g_vsc.broadcaster)) {
+      if (event.BroadcasterMatchesRef(g_dap.broadcaster)) {
         if (event_mask & eBroadcastBitStopProgressThread) {
           done = true;
         }
@@ -428,7 +428,7 @@ void ProgressEventThreadFunction() {
         const char *message = lldb::SBDebugger::GetProgressFromEvent(
             event, progress_id, completed, total, is_debugger_specific);
         if (message)
-          g_vsc.SendProgressEvent(progress_id, message, completed, total);
+          g_dap.SendProgressEvent(progress_id, message, completed, total);
       }
     }
   }
@@ -441,7 +441,7 @@ void ProgressEventThreadFunction() {
 // is required.
 void EventThreadFunction() {
   lldb::SBEvent event;
-  lldb::SBListener listener = g_vsc.debugger.GetListener();
+  lldb::SBListener listener = g_dap.debugger.GetListener();
   bool done = false;
   while (!done) {
     if (listener.WaitForEvent(1, event)) {
@@ -477,7 +477,7 @@ void EventThreadFunction() {
             // stop events which we do not want to send an event for. We will
             // manually send a stopped event in request_configurationDone(...)
             // so don't send any before then.
-            if (g_vsc.configuration_done_sent) {
+            if (g_dap.configuration_done_sent) {
               // Only report a stopped event if the process was not
               // automatically restarted.
               if (!lldb::SBProcess::GetRestartedFromEvent(event)) {
@@ -487,7 +487,7 @@ void EventThreadFunction() {
             }
             break;
           case lldb::eStateRunning:
-            g_vsc.WillContinue();
+            g_dap.WillContinue();
             SendContinuedEvent();
             break;
           case lldb::eStateExited:
@@ -495,12 +495,12 @@ void EventThreadFunction() {
             // just killed with the old PID, or even with no PID. In that case
             // we don't have to terminate the session.
             if (process.GetProcessID() == LLDB_INVALID_PROCESS_ID ||
-                process.GetProcessID() == g_vsc.restarting_process_id) {
-              g_vsc.restarting_process_id = LLDB_INVALID_PROCESS_ID;
+                process.GetProcessID() == g_dap.restarting_process_id) {
+              g_dap.restarting_process_id = LLDB_INVALID_PROCESS_ID;
             } else {
               // Run any exit LLDB commands the user specified in the
               // launch.json
-              g_vsc.RunExitCommands();
+              g_dap.RunExitCommands();
               SendProcessExitedEvent(process);
               SendTerminatedEvent();
               done = true;
@@ -537,10 +537,10 @@ void EventThreadFunction() {
             body.try_emplace("breakpoint", source_bp);
             body.try_emplace("reason", "changed");
             bp_event.try_emplace("body", std::move(body));
-            g_vsc.SendJSON(llvm::json::Value(std::move(bp_event)));
+            g_dap.SendJSON(llvm::json::Value(std::move(bp_event)));
           }
         }
-      } else if (event.BroadcasterMatchesRef(g_vsc.broadcaster)) {
+      } else if (event.BroadcasterMatchesRef(g_dap.broadcaster)) {
         if (event_mask & eBroadcastBitStopEventThread) {
           done = true;
         }
@@ -569,7 +569,7 @@ void SetSourceMapFromArguments(const llvm::json::Object &arguments) {
       if (mapping == nullptr || mapping->size() != 2 ||
           (*mapping)[0].kind() != llvm::json::Value::String ||
           (*mapping)[1].kind() != llvm::json::Value::String) {
-        g_vsc.SendOutput(OutputType::Console, llvm::StringRef(sourceMapHelp));
+        g_dap.SendOutput(OutputType::Console, llvm::StringRef(sourceMapHelp));
         return;
       }
       auto mapFrom = GetAsString((*mapping)[0]);
@@ -578,7 +578,7 @@ void SetSourceMapFromArguments(const llvm::json::Object &arguments) {
     }
   } else {
     if (ObjectContainsKey(arguments, "sourceMap")) {
-      g_vsc.SendOutput(OutputType::Console, llvm::StringRef(sourceMapHelp));
+      g_dap.SendOutput(OutputType::Console, llvm::StringRef(sourceMapHelp));
       return;
     }
     if (sourcePath.empty())
@@ -588,7 +588,7 @@ void SetSourceMapFromArguments(const llvm::json::Object &arguments) {
   }
   strm.flush();
   if (!sourceMapCommand.empty()) {
-    g_vsc.RunLLDBCommands("Setting source map:", {sourceMapCommand});
+    g_dap.RunLLDBCommands("Setting source map:", {sourceMapCommand});
   }
 }
 
@@ -621,8 +621,8 @@ void SetSourceMapFromArguments(const llvm::json::Object &arguments) {
 //   }]
 // }
 void request_attach(const llvm::json::Object &request) {
-  g_vsc.is_attach = true;
-  g_vsc.last_launch_or_attach_request = request;
+  g_dap.is_attach = true;
+  g_dap.last_launch_or_attach_request = request;
   llvm::json::Object response;
   lldb::SBError error;
   FillResponse(request, response);
@@ -634,83 +634,83 @@ void request_attach(const llvm::json::Object &request) {
     attach_info.SetProcessID(pid);
   const auto wait_for = GetBoolean(arguments, "waitFor", false);
   attach_info.SetWaitForLaunch(wait_for, false /*async*/);
-  g_vsc.init_commands = GetStrings(arguments, "initCommands");
-  g_vsc.pre_run_commands = GetStrings(arguments, "preRunCommands");
-  g_vsc.stop_commands = GetStrings(arguments, "stopCommands");
-  g_vsc.exit_commands = GetStrings(arguments, "exitCommands");
-  g_vsc.terminate_commands = GetStrings(arguments, "terminateCommands");
+  g_dap.init_commands = GetStrings(arguments, "initCommands");
+  g_dap.pre_run_commands = GetStrings(arguments, "preRunCommands");
+  g_dap.stop_commands = GetStrings(arguments, "stopCommands");
+  g_dap.exit_commands = GetStrings(arguments, "exitCommands");
+  g_dap.terminate_commands = GetStrings(arguments, "terminateCommands");
   auto attachCommands = GetStrings(arguments, "attachCommands");
   llvm::StringRef core_file = GetString(arguments, "coreFile");
   const uint64_t timeout_seconds = GetUnsigned(arguments, "timeout", 30);
-  g_vsc.stop_at_entry =
+  g_dap.stop_at_entry =
       core_file.empty() ? GetBoolean(arguments, "stopOnEntry", false) : true;
   std::vector<std::string> postRunCommands =
       GetStrings(arguments, "postRunCommands");
   const llvm::StringRef debuggerRoot = GetString(arguments, "debuggerRoot");
-  g_vsc.enable_auto_variable_summaries =
+  g_dap.enable_auto_variable_summaries =
       GetBoolean(arguments, "enableAutoVariableSummaries", false);
-  g_vsc.enable_synthetic_child_debugging =
+  g_dap.enable_synthetic_child_debugging =
       GetBoolean(arguments, "enableSyntheticChildDebugging", false);
 
   // This is a hack for loading DWARF in .o files on Mac where the .o files
   // in the debug map of the main executable have relative paths which require
-  // the lldb-vscode binary to have its working directory set to that relative
+  // the lldb-dap binary to have its working directory set to that relative
   // root for the .o files in order to be able to load debug info.
   if (!debuggerRoot.empty())
     llvm::sys::fs::set_current_path(debuggerRoot);
 
   // Run any initialize LLDB commands the user specified in the launch.json
-  g_vsc.RunInitCommands();
+  g_dap.RunInitCommands();
 
   SetSourceMapFromArguments(*arguments);
 
   lldb::SBError status;
-  g_vsc.SetTarget(g_vsc.CreateTargetFromArguments(*arguments, status));
+  g_dap.SetTarget(g_dap.CreateTargetFromArguments(*arguments, status));
   if (status.Fail()) {
     response["success"] = llvm::json::Value(false);
     EmplaceSafeString(response, "message", status.GetCString());
-    g_vsc.SendJSON(llvm::json::Value(std::move(response)));
+    g_dap.SendJSON(llvm::json::Value(std::move(response)));
     return;
   }
 
   // Run any pre run LLDB commands the user specified in the launch.json
-  g_vsc.RunPreRunCommands();
+  g_dap.RunPreRunCommands();
 
   if (pid == LLDB_INVALID_PROCESS_ID && wait_for) {
     char attach_msg[256];
     auto attach_msg_len = snprintf(attach_msg, sizeof(attach_msg),
                                    "Waiting to attach to \"%s\"...",
-                                   g_vsc.target.GetExecutable().GetFilename());
-    g_vsc.SendOutput(OutputType::Console,
+                                   g_dap.target.GetExecutable().GetFilename());
+    g_dap.SendOutput(OutputType::Console,
                      llvm::StringRef(attach_msg, attach_msg_len));
   }
   if (attachCommands.empty()) {
     // No "attachCommands", just attach normally.
     // Disable async events so the attach will be successful when we return from
     // the launch call and the launch will happen synchronously
-    g_vsc.debugger.SetAsync(false);
+    g_dap.debugger.SetAsync(false);
     if (core_file.empty())
-      g_vsc.target.Attach(attach_info, error);
+      g_dap.target.Attach(attach_info, error);
     else
-      g_vsc.target.LoadCore(core_file.data(), error);
+      g_dap.target.LoadCore(core_file.data(), error);
     // Reenable async events
-    g_vsc.debugger.SetAsync(true);
+    g_dap.debugger.SetAsync(true);
   } else {
     // We have "attachCommands" that are a set of commands that are expected
     // to execute the commands after which a process should be created. If there
     // is no valid process after running these commands, we have failed.
-    g_vsc.RunLLDBCommands("Running attachCommands:", attachCommands);
+    g_dap.RunLLDBCommands("Running attachCommands:", attachCommands);
     // The custom commands might have created a new target so we should use the
     // selected target after these commands are run.
-    g_vsc.target = g_vsc.debugger.GetSelectedTarget();
+    g_dap.target = g_dap.debugger.GetSelectedTarget();
 
     // Make sure the process is attached and stopped before proceeding as the
     // the launch commands are not run using the synchronous mode.
-    error = g_vsc.WaitForProcessToStop(timeout_seconds);
+    error = g_dap.WaitForProcessToStop(timeout_seconds);
   }
 
   if (error.Success() && core_file.empty()) {
-    auto attached_pid = g_vsc.target.GetProcess().GetProcessID();
+    auto attached_pid = g_dap.target.GetProcess().GetProcessID();
     if (attached_pid == LLDB_INVALID_PROCESS_ID) {
       if (attachCommands.empty())
         error.SetErrorString("failed to attach to a process");
@@ -723,13 +723,13 @@ void request_attach(const llvm::json::Object &request) {
     response["success"] = llvm::json::Value(false);
     EmplaceSafeString(response, "message", std::string(error.GetCString()));
   } else {
-    g_vsc.RunLLDBCommands("Running postRunCommands:", postRunCommands);
+    g_dap.RunLLDBCommands("Running postRunCommands:", postRunCommands);
   }
 
-  g_vsc.SendJSON(llvm::json::Value(std::move(response)));
+  g_dap.SendJSON(llvm::json::Value(std::move(response)));
   if (error.Success()) {
     SendProcessEvent(Attach);
-    g_vsc.SendJSON(CreateEventObject("initialized"));
+    g_dap.SendJSON(CreateEventObject("initialized"));
   }
 }
 
@@ -790,12 +790,12 @@ void request_attach(const llvm::json::Object &request) {
 void request_continue(const llvm::json::Object &request) {
   llvm::json::Object response;
   FillResponse(request, response);
-  lldb::SBProcess process = g_vsc.target.GetProcess();
+  lldb::SBProcess process = g_dap.target.GetProcess();
   lldb::SBError error = process.Continue();
   llvm::json::Object body;
   body.try_emplace("allThreadsContinued", true);
   response.try_emplace("body", std::move(body));
-  g_vsc.SendJSON(llvm::json::Value(std::move(response)));
+  g_dap.SendJSON(llvm::json::Value(std::move(response)));
 }
 
 // "ConfigurationDoneRequest": {
@@ -832,12 +832,12 @@ void request_continue(const llvm::json::Object &request) {
 void request_configurationDone(const llvm::json::Object &request) {
   llvm::json::Object response;
   FillResponse(request, response);
-  g_vsc.SendJSON(llvm::json::Value(std::move(response)));
-  g_vsc.configuration_done_sent = true;
-  if (g_vsc.stop_at_entry)
+  g_dap.SendJSON(llvm::json::Value(std::move(response)));
+  g_dap.configuration_done_sent = true;
+  if (g_dap.stop_at_entry)
     SendThreadStoppedEvent();
   else
-    g_vsc.target.GetProcess().Continue();
+    g_dap.target.GetProcess().Continue();
 }
 
 // "DisconnectRequest": {
@@ -889,10 +889,10 @@ void request_disconnect(const llvm::json::Object &request) {
   FillResponse(request, response);
   auto arguments = request.getObject("arguments");
 
-  bool defaultTerminateDebuggee = g_vsc.is_attach ? false : true;
+  bool defaultTerminateDebuggee = g_dap.is_attach ? false : true;
   bool terminateDebuggee =
       GetBoolean(arguments, "terminateDebuggee", defaultTerminateDebuggee);
-  lldb::SBProcess process = g_vsc.target.GetProcess();
+  lldb::SBProcess process = g_dap.target.GetProcess();
   auto state = process.GetState();
   switch (state) {
   case lldb::eStateInvalid:
@@ -908,22 +908,22 @@ void request_disconnect(const llvm::json::Object &request) {
   case lldb::eStateSuspended:
   case lldb::eStateStopped:
   case lldb::eStateRunning:
-    g_vsc.debugger.SetAsync(false);
+    g_dap.debugger.SetAsync(false);
     lldb::SBError error = terminateDebuggee ? process.Kill() : process.Detach();
     if (!error.Success())
       response.try_emplace("error", error.GetCString());
-    g_vsc.debugger.SetAsync(true);
+    g_dap.debugger.SetAsync(true);
     break;
   }
   SendTerminatedEvent();
-  g_vsc.SendJSON(llvm::json::Value(std::move(response)));
-  if (g_vsc.event_thread.joinable()) {
-    g_vsc.broadcaster.BroadcastEventByType(eBroadcastBitStopEventThread);
-    g_vsc.event_thread.join();
+  g_dap.SendJSON(llvm::json::Value(std::move(response)));
+  if (g_dap.event_thread.joinable()) {
+    g_dap.broadcaster.BroadcastEventByType(eBroadcastBitStopEventThread);
+    g_dap.event_thread.join();
   }
-  if (g_vsc.progress_event_thread.joinable()) {
-    g_vsc.broadcaster.BroadcastEventByType(eBroadcastBitStopProgressThread);
-    g_vsc.progress_event_thread.join();
+  if (g_dap.progress_event_thread.joinable()) {
+    g_dap.broadcaster.BroadcastEventByType(eBroadcastBitStopProgressThread);
+    g_dap.progress_event_thread.join();
   }
 }
 
@@ -932,13 +932,13 @@ void request_exceptionInfo(const llvm::json::Object &request) {
   FillResponse(request, response);
   auto arguments = request.getObject("arguments");
   llvm::json::Object body;
-  lldb::SBThread thread = g_vsc.GetLLDBThread(*arguments);
+  lldb::SBThread thread = g_dap.GetLLDBThread(*arguments);
   if (thread.IsValid()) {
     auto stopReason = thread.GetStopReason();
     if (stopReason == lldb::eStopReasonSignal)
       body.try_emplace("exceptionId", "signal");
     else if (stopReason == lldb::eStopReasonBreakpoint) {
-      ExceptionBreakpoint *exc_bp = g_vsc.GetExceptionBPFromStopReason(thread);
+      ExceptionBreakpoint *exc_bp = g_dap.GetExceptionBPFromStopReason(thread);
       if (exc_bp) {
         EmplaceSafeString(body, "exceptionId", exc_bp->filter);
         EmplaceSafeString(body, "description", exc_bp->label);
@@ -963,7 +963,7 @@ void request_exceptionInfo(const llvm::json::Object &request) {
     response["success"] = llvm::json::Value(false);
   }
   response.try_emplace("body", std::move(body));
-  g_vsc.SendJSON(llvm::json::Value(std::move(response)));
+  g_dap.SendJSON(llvm::json::Value(std::move(response)));
 }
 
 // "CompletionsRequest": {
@@ -1088,7 +1088,7 @@ void request_completions(const llvm::json::Object &request) {
   auto arguments = request.getObject("arguments");
 
   // If we have a frame, try to set the context for variable completions.
-  lldb::SBFrame frame = g_vsc.GetLLDBFrame(*arguments);
+  lldb::SBFrame frame = g_dap.GetLLDBFrame(*arguments);
   if (frame.IsValid()) {
     frame.GetThread().GetProcess().SetSelectedThread(frame.GetThread());
     frame.GetThread().SetSelectedFrame(frame.GetFrameID());
@@ -1107,7 +1107,7 @@ void request_completions(const llvm::json::Object &request) {
   }
   llvm::json::Array targets;
 
-  if (g_vsc.DetectExpressionContext(frame, text) ==
+  if (g_dap.DetectExpressionContext(frame, text) ==
       ExpressionContext::Variable) {
     char command[] = "expression -- ";
     text = command + text;
@@ -1116,7 +1116,7 @@ void request_completions(const llvm::json::Object &request) {
   lldb::SBStringList matches;
   lldb::SBStringList descriptions;
 
-  if (g_vsc.debugger.GetCommandInterpreter().HandleCompletionWithDescriptions(
+  if (g_dap.debugger.GetCommandInterpreter().HandleCompletionWithDescriptions(
           text.c_str(), offset, 0, 100, matches, descriptions)) {
     // The first element is the common substring after the cursor position for
     // all the matches. The rest of the elements are the matches so ignore the
@@ -1146,7 +1146,7 @@ void request_completions(const llvm::json::Object &request) {
 
   body.try_emplace("targets", std::move(targets));
   response.try_emplace("body", std::move(body));
-  g_vsc.SendJSON(llvm::json::Value(std::move(response)));
+  g_dap.SendJSON(llvm::json::Value(std::move(response)));
 }
 
 //  "EvaluateRequest": {
@@ -1255,16 +1255,16 @@ void request_evaluate(const llvm::json::Object &request) {
   FillResponse(request, response);
   llvm::json::Object body;
   auto arguments = request.getObject("arguments");
-  lldb::SBFrame frame = g_vsc.GetLLDBFrame(*arguments);
+  lldb::SBFrame frame = g_dap.GetLLDBFrame(*arguments);
   std::string expression = GetString(arguments, "expression").str();
   llvm::StringRef context = GetString(arguments, "context");
 
-  if (context == "repl" && g_vsc.DetectExpressionContext(frame, expression) ==
+  if (context == "repl" && g_dap.DetectExpressionContext(frame, expression) ==
                                ExpressionContext::Command) {
     // If we're evaluating a command relative to the current frame, set the
     // focus_tid to the current frame for any thread related events.
     if (frame.IsValid()) {
-      g_vsc.focus_tid = frame.GetThread().GetThreadID();
+      g_dap.focus_tid = frame.GetThread().GetThreadID();
     }
     auto result = RunLLDBCommands(llvm::StringRef(), {std::string(expression)});
     EmplaceSafeString(body, "result", result);
@@ -1302,7 +1302,7 @@ void request_evaluate(const llvm::json::Object &request) {
       EmplaceSafeString(body, "type",
                         value_typename ? value_typename : NO_TYPENAME);
       if (value.MightHaveChildren()) {
-        auto variableReference = g_vsc.variables.InsertExpandableVariable(
+        auto variableReference = g_dap.variables.InsertExpandableVariable(
             value, /*is_permanent=*/context == "repl");
         body.try_emplace("variablesReference", variableReference);
       } else {
@@ -1311,7 +1311,7 @@ void request_evaluate(const llvm::json::Object &request) {
     }
   }
   response.try_emplace("body", std::move(body));
-  g_vsc.SendJSON(llvm::json::Value(std::move(response)));
+  g_dap.SendJSON(llvm::json::Value(std::move(response)));
 }
 
 // "compileUnitsRequest": {
@@ -1361,9 +1361,9 @@ void request_compileUnits(const llvm::json::Object &request) {
   llvm::json::Array units;
   auto arguments = request.getObject("arguments");
   std::string module_id = std::string(GetString(arguments, "moduleId"));
-  int num_modules = g_vsc.target.GetNumModules();
+  int num_modules = g_dap.target.GetNumModules();
   for (int i = 0; i < num_modules; i++) {
-    auto curr_module = g_vsc.target.GetModuleAtIndex(i);
+    auto curr_module = g_dap.target.GetModuleAtIndex(i);
     if (module_id == curr_module.GetUUIDString()) {
       int num_units = curr_module.GetNumCompileUnits();
       for (int j = 0; j < num_units; j++) {
@@ -1375,7 +1375,7 @@ void request_compileUnits(const llvm::json::Object &request) {
     }
   }
   response.try_emplace("body", std::move(body));
-  g_vsc.SendJSON(llvm::json::Value(std::move(response)));
+  g_dap.SendJSON(llvm::json::Value(std::move(response)));
 }
 
 // "modulesRequest": {
@@ -1409,15 +1409,15 @@ void request_modules(const llvm::json::Object &request) {
   FillResponse(request, response);
 
   llvm::json::Array modules;
-  for (size_t i = 0; i < g_vsc.target.GetNumModules(); i++) {
-    lldb::SBModule module = g_vsc.target.GetModuleAtIndex(i);
+  for (size_t i = 0; i < g_dap.target.GetNumModules(); i++) {
+    lldb::SBModule module = g_dap.target.GetModuleAtIndex(i);
     modules.emplace_back(CreateModule(module));
   }
 
   llvm::json::Object body;
   body.try_emplace("modules", std::move(modules));
   response.try_emplace("body", std::move(body));
-  g_vsc.SendJSON(llvm::json::Value(std::move(response)));
+  g_dap.SendJSON(llvm::json::Value(std::move(response)));
 }
 
 // "InitializeRequest": {
@@ -1498,7 +1498,7 @@ void request_modules(const llvm::json::Object &request) {
 // }
 void request_initialize(const llvm::json::Object &request) {
   auto log_cb = [](const char *buf, void *baton) -> void {
-    g_vsc.SendOutput(OutputType::Console, llvm::StringRef{buf});
+    g_dap.SendOutput(OutputType::Console, llvm::StringRef{buf});
   };
 
   auto arguments = request.getObject("arguments");
@@ -1507,25 +1507,24 @@ void request_initialize(const llvm::json::Object &request) {
   // which may affect the outcome of tests.
   bool source_init_file = GetBoolean(arguments, "sourceInitFile", true);
 
-  g_vsc.debugger =
-      lldb::SBDebugger::Create(source_init_file, log_cb, nullptr);
-  auto cmd = g_vsc.debugger.GetCommandInterpreter().AddMultiwordCommand(
-      "lldb-vscode", "Commands for managing lldb-vscode.");
+  g_dap.debugger = lldb::SBDebugger::Create(source_init_file, log_cb, nullptr);
+  auto cmd = g_dap.debugger.GetCommandInterpreter().AddMultiwordCommand(
+      "lldb-dap", "Commands for managing lldb-dap.");
   if (GetBoolean(arguments, "supportsStartDebuggingRequest", false)) {
     cmd.AddCommand(
-        "startDebugging", &g_vsc.start_debugging_request_handler,
+        "startDebugging", &g_dap.start_debugging_request_handler,
         "Sends a startDebugging request from the debug adapter to the client "
         "to start a child debug session of the same type as the caller.");
   }
   cmd.AddCommand(
-      "repl-mode", &g_vsc.repl_mode_request_handler,
-      "Get or set the repl behavior of vscode-lldb evaluation requests.");
+      "repl-mode", &g_dap.repl_mode_request_handler,
+      "Get or set the repl behavior of lldb-dap evaluation requests.");
 
-  g_vsc.progress_event_thread = std::thread(ProgressEventThreadFunction);
+  g_dap.progress_event_thread = std::thread(ProgressEventThreadFunction);
 
   // Start our event thread so we can receive events from the debugger, target,
   // process and more.
-  g_vsc.event_thread = std::thread(EventThreadFunction);
+  g_dap.event_thread = std::thread(EventThreadFunction);
 
   llvm::json::Object response;
   FillResponse(request, response);
@@ -1544,7 +1543,7 @@ void request_initialize(const llvm::json::Object &request) {
   body.try_emplace("supportsEvaluateForHovers", true);
   // Available filters or options for the setExceptionBreakpoints request.
   llvm::json::Array filters;
-  for (const auto &exc_bp : g_vsc.exception_breakpoints) {
+  for (const auto &exc_bp : g_dap.exception_breakpoints) {
     filters.emplace_back(CreateExceptionBreakpointFilter(exc_bp));
   }
   body.try_emplace("exceptionBreakpointFilters", std::move(filters));
@@ -1607,12 +1606,12 @@ void request_initialize(const llvm::json::Object &request) {
   body.try_emplace("supportsLogPoints", true);
 
   response.try_emplace("body", std::move(body));
-  g_vsc.SendJSON(llvm::json::Value(std::move(response)));
+  g_dap.SendJSON(llvm::json::Value(std::move(response)));
 }
 
 llvm::Error request_runInTerminal(const llvm::json::Object &launch_request,
                                   const uint64_t timeout_seconds) {
-  g_vsc.is_attach = true;
+  g_dap.is_attach = true;
   lldb::SBAttachInfo attach_info;
 
   llvm::Expected<std::shared_ptr<FifoFile>> comm_file_or_err =
@@ -1628,8 +1627,8 @@ llvm::Error request_runInTerminal(const llvm::json::Object &launch_request,
   debugger_pid = getpid();
 #endif
   llvm::json::Object reverse_request = CreateRunInTerminalReverseRequest(
-      launch_request, g_vsc.debug_adaptor_path, comm_file.m_path, debugger_pid);
-  g_vsc.SendReverseRequest("runInTerminal", std::move(reverse_request),
+      launch_request, g_dap.debug_adaptor_path, comm_file.m_path, debugger_pid);
+  g_dap.SendReverseRequest("runInTerminal", std::move(reverse_request),
                            [](llvm::Expected<llvm::json::Value> value) {
                              if (!value) {
                                llvm::Error err = value.takeError();
@@ -1644,9 +1643,9 @@ llvm::Error request_runInTerminal(const llvm::json::Object &launch_request,
   else
     return pid.takeError();
 
-  g_vsc.debugger.SetAsync(false);
+  g_dap.debugger.SetAsync(false);
   lldb::SBError error;
-  g_vsc.target.Attach(attach_info, error);
+  g_dap.target.Attach(attach_info, error);
 
   if (error.Fail())
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
@@ -1664,11 +1663,11 @@ llvm::Error request_runInTerminal(const llvm::json::Object &launch_request,
   // process right in the middle of the exec. To the user, what we are doing is
   // transparent, as they will only be able to see the process since the exec,
   // completely unaware of the preparatory work.
-  g_vsc.target.GetProcess().Continue();
+  g_dap.target.GetProcess().Continue();
 
   // Now that the actual target is just starting (i.e. exec was just invoked),
   // we return the debugger to its async state.
-  g_vsc.debugger.SetAsync(true);
+  g_dap.debugger.SetAsync(true);
 
   // If sending the notification failed, the launcher should be dead by now and
   // the async didAttach notification should have an error message, so we
@@ -1691,7 +1690,7 @@ lldb::SBError LaunchProcess(const llvm::json::Object &request) {
   auto launchCommands = GetStrings(arguments, "launchCommands");
 
   // Instantiate a launch info instance for the target.
-  auto launch_info = g_vsc.target.GetLaunchInfo();
+  auto launch_info = g_dap.target.GetLaunchInfo();
 
   // Grab the current working directory if there is one and set it in the
   // launch info.
@@ -1730,21 +1729,21 @@ lldb::SBError LaunchProcess(const llvm::json::Object &request) {
   } else if (launchCommands.empty()) {
     // Disable async events so the launch will be successful when we return from
     // the launch call and the launch will happen synchronously
-    g_vsc.debugger.SetAsync(false);
-    g_vsc.target.Launch(launch_info, error);
-    g_vsc.debugger.SetAsync(true);
+    g_dap.debugger.SetAsync(false);
+    g_dap.target.Launch(launch_info, error);
+    g_dap.debugger.SetAsync(true);
   } else {
     // Set the launch info so that run commands can access the configured
     // launch details.
-    g_vsc.target.SetLaunchInfo(launch_info);
-    g_vsc.RunLLDBCommands("Running launchCommands:", launchCommands);
+    g_dap.target.SetLaunchInfo(launch_info);
+    g_dap.RunLLDBCommands("Running launchCommands:", launchCommands);
     // The custom commands might have created a new target so we should use the
     // selected target after these commands are run.
-    g_vsc.target = g_vsc.debugger.GetSelectedTarget();
+    g_dap.target = g_dap.debugger.GetSelectedTarget();
     // Make sure the process is launched and stopped at the entry point before
     // proceeding as the launch commands are not run using the synchronous
     // mode.
-    error = g_vsc.WaitForProcessToStop(timeout_seconds);
+    error = g_dap.WaitForProcessToStop(timeout_seconds);
   }
   return error;
 }
@@ -1784,28 +1783,28 @@ lldb::SBError LaunchProcess(const llvm::json::Object &request) {
 //   }]
 // }
 void request_launch(const llvm::json::Object &request) {
-  g_vsc.is_attach = false;
-  g_vsc.last_launch_or_attach_request = request;
+  g_dap.is_attach = false;
+  g_dap.last_launch_or_attach_request = request;
   llvm::json::Object response;
   FillResponse(request, response);
   auto arguments = request.getObject("arguments");
-  g_vsc.init_commands = GetStrings(arguments, "initCommands");
-  g_vsc.pre_run_commands = GetStrings(arguments, "preRunCommands");
-  g_vsc.stop_commands = GetStrings(arguments, "stopCommands");
-  g_vsc.exit_commands = GetStrings(arguments, "exitCommands");
-  g_vsc.terminate_commands = GetStrings(arguments, "terminateCommands");
+  g_dap.init_commands = GetStrings(arguments, "initCommands");
+  g_dap.pre_run_commands = GetStrings(arguments, "preRunCommands");
+  g_dap.stop_commands = GetStrings(arguments, "stopCommands");
+  g_dap.exit_commands = GetStrings(arguments, "exitCommands");
+  g_dap.terminate_commands = GetStrings(arguments, "terminateCommands");
   std::vector<std::string> postRunCommands =
       GetStrings(arguments, "postRunCommands");
-  g_vsc.stop_at_entry = GetBoolean(arguments, "stopOnEntry", false);
+  g_dap.stop_at_entry = GetBoolean(arguments, "stopOnEntry", false);
   const llvm::StringRef debuggerRoot = GetString(arguments, "debuggerRoot");
-  g_vsc.enable_auto_variable_summaries =
+  g_dap.enable_auto_variable_summaries =
       GetBoolean(arguments, "enableAutoVariableSummaries", false);
-  g_vsc.enable_synthetic_child_debugging =
+  g_dap.enable_synthetic_child_debugging =
       GetBoolean(arguments, "enableSyntheticChildDebugging", false);
 
   // This is a hack for loading DWARF in .o files on Mac where the .o files
   // in the debug map of the main executable have relative paths which require
-  // the lldb-vscode binary to have its working directory set to that relative
+  // the lldb-dap binary to have its working directory set to that relative
   // root for the .o files in order to be able to load debug info.
   if (!debuggerRoot.empty())
     llvm::sys::fs::set_current_path(debuggerRoot);
@@ -1813,21 +1812,21 @@ void request_launch(const llvm::json::Object &request) {
   // Run any initialize LLDB commands the user specified in the launch.json.
   // This is run before target is created, so commands can't do anything with
   // the targets - preRunCommands are run with the target.
-  g_vsc.RunInitCommands();
+  g_dap.RunInitCommands();
 
   SetSourceMapFromArguments(*arguments);
 
   lldb::SBError status;
-  g_vsc.SetTarget(g_vsc.CreateTargetFromArguments(*arguments, status));
+  g_dap.SetTarget(g_dap.CreateTargetFromArguments(*arguments, status));
   if (status.Fail()) {
     response["success"] = llvm::json::Value(false);
     EmplaceSafeString(response, "message", status.GetCString());
-    g_vsc.SendJSON(llvm::json::Value(std::move(response)));
+    g_dap.SendJSON(llvm::json::Value(std::move(response)));
     return;
   }
 
   // Run any pre run LLDB commands the user specified in the launch.json
-  g_vsc.RunPreRunCommands();
+  g_dap.RunPreRunCommands();
 
   status = LaunchProcess(request);
 
@@ -1835,18 +1834,18 @@ void request_launch(const llvm::json::Object &request) {
     response["success"] = llvm::json::Value(false);
     EmplaceSafeString(response, "message", std::string(status.GetCString()));
   } else {
-    g_vsc.RunLLDBCommands("Running postRunCommands:", postRunCommands);
+    g_dap.RunLLDBCommands("Running postRunCommands:", postRunCommands);
   }
 
-  g_vsc.SendJSON(llvm::json::Value(std::move(response)));
+  g_dap.SendJSON(llvm::json::Value(std::move(response)));
 
   if (!status.Fail()) {
-    if (g_vsc.is_attach)
-      SendProcessEvent(Attach);  // this happens when doing runInTerminal
+    if (g_dap.is_attach)
+      SendProcessEvent(Attach); // this happens when doing runInTerminal
     else
       SendProcessEvent(Launch);
   }
-  g_vsc.SendJSON(CreateEventObject("initialized"));
+  g_dap.SendJSON(CreateEventObject("initialized"));
 }
 
 // "NextRequest": {
@@ -1891,16 +1890,16 @@ void request_next(const llvm::json::Object &request) {
   llvm::json::Object response;
   FillResponse(request, response);
   auto arguments = request.getObject("arguments");
-  lldb::SBThread thread = g_vsc.GetLLDBThread(*arguments);
+  lldb::SBThread thread = g_dap.GetLLDBThread(*arguments);
   if (thread.IsValid()) {
     // Remember the thread ID that caused the resume so we can set the
     // "threadCausedFocus" boolean value in the "stopped" events.
-    g_vsc.focus_tid = thread.GetThreadID();
+    g_dap.focus_tid = thread.GetThreadID();
     thread.StepOver();
   } else {
     response["success"] = llvm::json::Value(false);
   }
-  g_vsc.SendJSON(llvm::json::Value(std::move(response)));
+  g_dap.SendJSON(llvm::json::Value(std::move(response)));
 }
 
 // "PauseRequest": {
@@ -1942,12 +1941,11 @@ void request_next(const llvm::json::Object &request) {
 void request_pause(const llvm::json::Object &request) {
   llvm::json::Object response;
   FillResponse(request, response);
-  lldb::SBProcess process = g_vsc.target.GetProcess();
+  lldb::SBProcess process = g_dap.target.GetProcess();
   lldb::SBError error = process.Stop();
-  g_vsc.SendJSON(llvm::json::Value(std::move(response)));
+  g_dap.SendJSON(llvm::json::Value(std::move(response)));
 }
 
-
 // "RestartRequest": {
 //   "allOf": [ { "$ref": "#/definitions/Request" }, {
 //     "type": "object",
@@ -1992,11 +1990,11 @@ void request_pause(const llvm::json::Object &request) {
 void request_restart(const llvm::json::Object &request) {
   llvm::json::Object response;
   FillResponse(request, response);
-  if (!g_vsc.last_launch_or_attach_request) {
+  if (!g_dap.last_launch_or_attach_request) {
     response["success"] = llvm::json::Value(false);
     EmplaceSafeString(response, "message",
                       "Restart request received but no process was launched.");
-    g_vsc.SendJSON(llvm::json::Value(std::move(response)));
+    g_dap.SendJSON(llvm::json::Value(std::move(response)));
     return;
   }
   // Check if we were in a "launch" session or an "attach" session.
@@ -2008,12 +2006,12 @@ void request_restart(const llvm::json::Object &request) {
   // Note that when using runInTerminal we're technically attached, but it's an
   // implementation detail. The adapter *did* launch the process in response to
   // a "launch" command, so we can still stop it and re-run it. This is why we
-  // don't just check `g_vsc.is_attach`.
-  if (GetString(*g_vsc.last_launch_or_attach_request, "command") == "attach") {
+  // don't just check `g_dap.is_attach`.
+  if (GetString(*g_dap.last_launch_or_attach_request, "command") == "attach") {
     response["success"] = llvm::json::Value(false);
     EmplaceSafeString(response, "message",
                       "Restarting an \"attach\" session is not supported.");
-    g_vsc.SendJSON(llvm::json::Value(std::move(response)));
+    g_dap.SendJSON(llvm::json::Value(std::move(response)));
     return;
   }
 
@@ -2023,20 +2021,20 @@ void request_restart(const llvm::json::Object &request) {
   if (restart_arguments) {
     auto launch_request_arguments = restart_arguments->getObject("arguments");
     if (launch_request_arguments) {
-      (*g_vsc.last_launch_or_attach_request)["arguments"] =
+      (*g_dap.last_launch_or_attach_request)["arguments"] =
           llvm::json::Value(llvm::json::Object(*launch_request_arguments));
     }
   }
 
   // Keep track of the old PID so when we get a "process exited" event from the
   // killed process we can detect it and not shut down the whole session.
-  lldb::SBProcess process = g_vsc.target.GetProcess();
-  g_vsc.restarting_process_id = process.GetProcessID();
+  lldb::SBProcess process = g_dap.target.GetProcess();
+  g_dap.restarting_process_id = process.GetProcessID();
 
   // Stop the current process if necessary. The logic here is similar to
   // CommandObjectProcessLaunchOrAttach::StopProcessIfNecessary, except that
   // we don't ask the user for confirmation.
-  g_vsc.debugger.SetAsync(false);
+  g_dap.debugger.SetAsync(false);
   if (process.IsValid()) {
     lldb::StateType state = process.GetState();
     if (state != lldb::eStateConnected) {
@@ -2044,21 +2042,21 @@ void request_restart(const llvm::json::Object &request) {
     }
     // Clear the list of thread ids to avoid sending "thread exited" events
     // for threads of the process we are terminating.
-    g_vsc.thread_ids.clear();
+    g_dap.thread_ids.clear();
   }
-  g_vsc.debugger.SetAsync(true);
-  LaunchProcess(*g_vsc.last_launch_or_attach_request);
+  g_dap.debugger.SetAsync(true);
+  LaunchProcess(*g_dap.last_launch_or_attach_request);
 
   // This is normally done after receiving a "configuration done" request.
   // Because we're restarting, configuration has already happened so we can
   // continue the process right away.
-  if (g_vsc.stop_at_entry) {
+  if (g_dap.stop_at_entry) {
     SendThreadStoppedEvent();
   } else {
-    g_vsc.target.GetProcess().Continue();
+    g_dap.target.GetProcess().Continue();
   }
 
-  g_vsc.SendJSON(llvm::json::Value(std::move(response)));
+  g_dap.SendJSON(llvm::json::Value(std::move(response)));
 }
 
 // "ScopesRequest": {
@@ -2117,7 +2115,7 @@ void request_scopes(const llvm::json::Object &request) {
   FillResponse(request, response);
   llvm::json::Object body;
   auto arguments = request.getObject("arguments");
-  lldb::SBFrame frame = g_vsc.GetLLDBFrame(*arguments);
+  lldb::SBFrame frame = g_dap.GetLLDBFrame(*arguments);
   // As the user selects different stack frames in the GUI, a "scopes" request
   // will be sent to the DAP. This is the only way we know that the user has
   // selected a frame in a thread. There are no other notifications that are
@@ -2137,18 +2135,18 @@ void request_scopes(const llvm::json::Object &request) {
     frame.GetThread().SetSelectedFrame(frame.GetFrameID());
   }
 
-  g_vsc.variables.locals = frame.GetVariables(/*arguments=*/true,
+  g_dap.variables.locals = frame.GetVariables(/*arguments=*/true,
                                               /*locals=*/true,
                                               /*statics=*/false,
                                               /*in_scope_only=*/true);
-  g_vsc.variables.globals = frame.GetVariables(/*arguments=*/false,
+  g_dap.variables.globals = frame.GetVariables(/*arguments=*/false,
                                                /*locals=*/false,
                                                /*statics=*/true,
                                                /*in_scope_only=*/true);
-  g_vsc.variables.registers = frame.GetRegisters();
-  body.try_emplace("scopes", g_vsc.CreateTopLevelScopes());
+  g_dap.variables.registers = frame.GetRegisters();
+  body.try_emplace("scopes", g_dap.CreateTopLevelScopes());
   response.try_emplace("body", std::move(body));
-  g_vsc.SendJSON(llvm::json::Value(std::move(response)));
+  g_dap.SendJSON(llvm::json::Value(std::move(response)));
 }
 
 // "SetBreakpointsRequest": {
@@ -2283,8 +2281,8 @@ void request_setBreakpoints(const llvm::json::Object &request) {
         request_bps[src_bp.line] = src_bp;
 
         // We check if this breakpoint already exists to update it
-        auto existing_source_bps = g_vsc.source_breakpoints.find(path);
-        if (existing_source_bps != g_vsc.source_breakpoints.end()) {
+        auto existing_source_bps = g_dap.source_breakpoints.find(path);
+        if (existing_source_bps != g_dap.source_breakpoints.end()) {
           const auto &existing_bp =
               existing_source_bps->second.find(src_bp.line);
           if (existing_bp != existing_source_bps->second.end()) {
@@ -2295,8 +2293,8 @@ void request_setBreakpoints(const llvm::json::Object &request) {
           }
         }
         // At this point the breakpoint is new
-        g_vsc.source_breakpoints[path][src_bp.line] = src_bp;
-        SourceBreakpoint &new_bp = g_vsc.source_breakpoints[path][src_bp.line];
+        g_dap.source_breakpoints[path][src_bp.line] = src_bp;
+        SourceBreakpoint &new_bp = g_dap.source_breakpoints[path][src_bp.line];
         new_bp.SetBreakpoint(path.data());
         AppendBreakpoint(new_bp.bp, response_breakpoints, path, new_bp.line);
       }
@@ -2306,13 +2304,13 @@ void request_setBreakpoints(const llvm::json::Object &request) {
   // Delete any breakpoints in this source file that aren't in the
   // request_bps set. There is no call to remove breakpoints other than
   // calling this function with a smaller or empty "breakpoints" list.
-  auto old_src_bp_pos = g_vsc.source_breakpoints.find(path);
-  if (old_src_bp_pos != g_vsc.source_breakpoints.end()) {
+  auto old_src_bp_pos = g_dap.source_breakpoints.find(path);
+  if (old_src_bp_pos != g_dap.source_breakpoints.end()) {
     for (auto &old_bp : old_src_bp_pos->second) {
       auto request_pos = request_bps.find(old_bp.first);
       if (request_pos == request_bps.end()) {
         // This breakpoint no longer exists in this source file, delete it
-        g_vsc.target.BreakpointDelete(old_bp.second.bp.GetID());
+        g_dap.target.BreakpointDelete(old_bp.second.bp.GetID());
         old_src_bp_pos->second.erase(old_bp.first);
       }
     }
@@ -2321,7 +2319,7 @@ void request_setBreakpoints(const llvm::json::Object &request) {
   llvm::json::Object body;
   body.try_emplace("breakpoints", std::move(response_breakpoints));
   response.try_emplace("body", std::move(body));
-  g_vsc.SendJSON(llvm::json::Value(std::move(response)));
+  g_dap.SendJSON(llvm::json::Value(std::move(response)));
 }
 
 // "SetExceptionBreakpointsRequest": {
@@ -2380,23 +2378,23 @@ void request_setExceptionBreakpoints(const llvm::json::Object &request) {
   // Keep a list of any exception breakpoint filter names that weren't set
   // so we can clear any exception breakpoints if needed.
   std::set<std::string> unset_filters;
-  for (const auto &bp : g_vsc.exception_breakpoints)
+  for (const auto &bp : g_dap.exception_breakpoints)
     unset_filters.insert(bp.filter);
 
   for (const auto &value : *filters) {
     const auto filter = GetAsString(value);
-    auto exc_bp = g_vsc.GetExceptionBreakpoint(std::string(filter));
+    auto exc_bp = g_dap.GetExceptionBreakpoint(std::string(filter));
     if (exc_bp) {
       exc_bp->SetBreakpoint();
       unset_filters.erase(std::string(filter));
     }
   }
   for (const auto &filter : unset_filters) {
-    auto exc_bp = g_vsc.GetExceptionBreakpoint(filter);
+    auto exc_bp = g_dap.GetExceptionBreakpoint(filter);
     if (exc_bp)
       exc_bp->ClearBreakpoint();
   }
-  g_vsc.SendJSON(llvm::json::Value(std::move(response)));
+  g_dap.SendJSON(llvm::json::Value(std::move(response)));
 }
 
 // "SetFunctionBreakpointsRequest": {
@@ -2497,11 +2495,11 @@ void request_setFunctionBreakpoints(const llvm::json::Object &request) {
   // Disable any function breakpoints that aren't in the request_bps.
   // There is no call to remove function breakpoints other than calling this
   // function with a smaller or empty "breakpoints" list.
-  for (auto &pair : g_vsc.function_breakpoints) {
+  for (auto &pair : g_dap.function_breakpoints) {
     auto request_pos = request_bps.find(pair.first());
     if (request_pos == request_bps.end()) {
       // This function breakpoint no longer exists delete it from LLDB
-      g_vsc.target.BreakpointDelete(pair.second.bp.GetID());
+      g_dap.target.BreakpointDelete(pair.second.bp.GetID());
       remove_names.push_back(pair.first());
     } else {
       // Update the existing breakpoint as any setting withing the function
@@ -2516,14 +2514,14 @@ void request_setFunctionBreakpoints(const llvm::json::Object &request) {
   }
   // Remove any breakpoints that are no longer in our list
   for (const auto &name : remove_names)
-    g_vsc.function_breakpoints.erase(name);
+    g_dap.function_breakpoints.erase(name);
 
   // Any breakpoints that are left in "request_bps" are breakpoints that
   // need to be set.
   for (auto &pair : request_bps) {
     // Add this breakpoint info to the response
-    g_vsc.function_breakpoints[pair.first()] = std::move(pair.second);
-    FunctionBreakpoint &new_bp = g_vsc.function_breakpoints[pair.first()];
+    g_dap.function_breakpoints[pair.first()] = std::move(pair.second);
+    FunctionBreakpoint &new_bp = g_dap.function_breakpoints[pair.first()];
     new_bp.SetBreakpoint();
     AppendBreakpoint(new_bp.bp, response_breakpoints);
   }
@@ -2531,7 +2529,7 @@ void request_setFunctionBreakpoints(const llvm::json::Object &request) {
   llvm::json::Object body;
   body.try_emplace("breakpoints", std::move(response_breakpoints));
   response.try_emplace("body", std::move(body));
-  g_vsc.SendJSON(llvm::json::Value(std::move(response)));
+  g_dap.SendJSON(llvm::json::Value(std::move(response)));
 }
 
 // "SourceRequest": {
@@ -2597,7 +2595,7 @@ void request_source(const llvm::json::Object &request) {
   FillResponse(request, response);
   llvm::json::Object body{{"content", ""}};
   response.try_emplace("body", std::move(body));
-  g_vsc.SendJSON(llvm::json::Value(std::move(response)));
+  g_dap.SendJSON(llvm::json::Value(std::move(response)));
 }
 
 // "StackTraceRequest": {
@@ -2675,7 +2673,7 @@ void request_stackTrace(const llvm::json::Object &request) {
   FillResponse(request, response);
   lldb::SBError error;
   auto arguments = request.getObject("arguments");
-  lldb::SBThread thread = g_vsc.GetLLDBThread(*arguments);
+  lldb::SBThread thread = g_dap.GetLLDBThread(*arguments);
   llvm::json::Array stackFrames;
   llvm::json::Object body;
 
@@ -2753,7 +2751,7 @@ void request_stackTrace(const llvm::json::Object &request) {
   }
   body.try_emplace("stackFrames", std::move(stackFrames));
   response.try_emplace("body", std::move(body));
-  g_vsc.SendJSON(llvm::json::Value(std::move(response)));
+  g_dap.SendJSON(llvm::json::Value(std::move(response)));
 }
 
 // "StepInRequest": {
@@ -2805,16 +2803,16 @@ void request_stepIn(const llvm::json::Object &request) {
   llvm::json::Object response;
   FillResponse(request, response);
   auto arguments = request.getObject("arguments");
-  lldb::SBThread thread = g_vsc.GetLLDBThread(*arguments);
+  lldb::SBThread thread = g_dap.GetLLDBThread(*arguments);
   if (thread.IsValid()) {
     // Remember the thread ID that caused the resume so we can set the
     // "threadCausedFocus" boolean value in the "stopped" events.
-    g_vsc.focus_tid = thread.GetThreadID();
+    g_dap.focus_tid = thread.GetThreadID();
     thread.StepInto();
   } else {
     response["success"] = llvm::json::Value(false);
   }
-  g_vsc.SendJSON(llvm::json::Value(std::move(response)));
+  g_dap.SendJSON(llvm::json::Value(std::move(response)));
 }
 
 // "StepOutRequest": {
@@ -2857,16 +2855,16 @@ void request_stepOut(const llvm::json::Object &request) {
   llvm::json::Object response;
   FillResponse(request, response);
   auto arguments = request.getObject("arguments");
-  lldb::SBThread thread = g_vsc.GetLLDBThread(*arguments);
+  lldb::SBThread thread = g_dap.GetLLDBThread(*arguments);
   if (thread.IsValid()) {
     // Remember the thread ID that caused the resume so we can set the
     // "threadCausedFocus" boolean value in the "stopped" events.
-    g_vsc.focus_tid = thread.GetThreadID();
+    g_dap.focus_tid = thread.GetThreadID();
     thread.StepOut();
   } else {
     response["success"] = llvm::json::Value(false);
   }
-  g_vsc.SendJSON(llvm::json::Value(std::move(response)));
+  g_dap.SendJSON(llvm::json::Value(std::move(response)));
 }
 
 // "ThreadsRequest": {
@@ -2906,7 +2904,7 @@ void request_stepOut(const llvm::json::Object &request) {
 // }
 void request_threads(const llvm::json::Object &request) {
 
-  lldb::SBProcess process = g_vsc.target.GetProcess();
+  lldb::SBProcess process = g_dap.target.GetProcess();
   llvm::json::Object response;
   FillResponse(request, response);
 
@@ -2922,7 +2920,7 @@ void request_threads(const llvm::json::Object &request) {
   llvm::json::Object body;
   body.try_emplace("threads", std::move(threads));
   response.try_emplace("body", std::move(body));
-  g_vsc.SendJSON(llvm::json::Value(std::move(response)));
+  g_dap.SendJSON(llvm::json::Value(std::move(response)));
 }
 
 // "SetVariableRequest": {
@@ -3034,10 +3032,10 @@ void request_setVariable(const llvm::json::Object &request) {
   // only specifies the variable reference of the enclosing scope/variable, and
   // the name of the variable. We could have two shadowed variables with the
   // same name in "Locals" or "Globals". In our case the "id" absolute index
-  // of the variable within the g_vsc.variables list.
+  // of the variable within the g_dap.variables list.
   const auto id_value = GetUnsigned(arguments, "id", UINT64_MAX);
   if (id_value != UINT64_MAX) {
-    variable = g_vsc.variables.GetVariable(id_value);
+    variable = g_dap.variables.GetVariable(id_value);
   } else if (lldb::SBValueList *top_scope =
                  GetTopLevelScope(variablesReference)) {
     // variablesReference is one of our scopes, not an actual variable it is
@@ -3060,7 +3058,7 @@ void request_setVariable(const llvm::json::Object &request) {
 
     // We have a named item within an actual variable so we need to find it
     // withing the container variable by name.
-    lldb::SBValue container = g_vsc.variables.GetVariable(variablesReference);
+    lldb::SBValue container = g_dap.variables.GetVariable(variablesReference);
     variable = container.GetChildMemberWithName(name.data());
     if (!variable.IsValid()) {
       if (name.startswith("[")) {
@@ -3081,12 +3079,12 @@ void request_setVariable(const llvm::json::Object &request) {
       SetValueForKey(variable, body, "value");
       EmplaceSafeString(body, "type", variable.GetType().GetDisplayTypeName());
 
-      // We don't know the index of the variable in our g_vsc.variables
+      // We don't know the index of the variable in our g_dap.variables
       // so always insert a new one to get its variablesReference.
       // is_permanent is false because debug console does not support
       // setVariable request.
       if (variable.MightHaveChildren())
-        newVariablesReference = g_vsc.variables.InsertExpandableVariable(
+        newVariablesReference = g_dap.variables.InsertExpandableVariable(
             variable, /*is_permanent=*/false);
 
       body.try_emplace("variablesReference", newVariablesReference);
@@ -3099,7 +3097,7 @@ void request_setVariable(const llvm::json::Object &request) {
   }
 
   response.try_emplace("body", std::move(body));
-  g_vsc.SendJSON(llvm::json::Value(std::move(response)));
+  g_dap.SendJSON(llvm::json::Value(std::move(response)));
 }
 
 // "VariablesRequest": {
@@ -3201,8 +3199,8 @@ void request_variables(const llvm::json::Object &request) {
       // and resolve what the pointer resolves to. Only change the format if the
       // format was set to the default format or if it was hex as some registers
       // have formats set for them.
-      const uint32_t addr_size = g_vsc.target.GetProcess().GetAddressByteSize();
-      lldb::SBValue reg_set = g_vsc.variables.registers.GetValueAtIndex(0);
+      const uint32_t addr_size = g_dap.target.GetProcess().GetAddressByteSize();
+      lldb::SBValue reg_set = g_dap.variables.registers.GetValueAtIndex(0);
       const uint32_t num_regs = reg_set.GetNumChildren();
       for (uint32_t reg_idx = 0; reg_idx < num_regs; ++reg_idx) {
         lldb::SBValue reg = reg_set.GetChildAtIndex(reg_idx);
@@ -3260,7 +3258,7 @@ void request_variables(const llvm::json::Object &request) {
 
       int64_t var_ref = 0;
       if (variable.MightHaveChildren() || variable.IsSynthetic()) {
-        var_ref = g_vsc.variables.InsertExpandableVariable(
+        var_ref = g_dap.variables.InsertExpandableVariable(
             variable, /*is_permanent=*/false);
       }
       variables.emplace_back(CreateVariable(
@@ -3270,7 +3268,7 @@ void request_variables(const llvm::json::Object &request) {
   } else {
     // We are expanding a variable that has children, so we will return its
     // children.
-    lldb::SBValue variable = g_vsc.variables.GetVariable(variablesReference);
+    lldb::SBValue variable = g_dap.variables.GetVariable(variablesReference);
     if (variable.IsValid()) {
       auto addChild = [&](lldb::SBValue child,
                           std::optional<std::string> custom_name = {}) {
@@ -3278,9 +3276,9 @@ void request_variables(const llvm::json::Object &request) {
           return;
         if (child.MightHaveChildren()) {
           auto is_permanent =
-              g_vsc.variables.IsPermanentVariableReference(variablesReference);
+              g_dap.variables.IsPermanentVariableReference(variablesReference);
           auto childVariablesReferences =
-              g_vsc.variables.InsertExpandableVariable(child, is_permanent);
+              g_dap.variables.InsertExpandableVariable(child, is_permanent);
           variables.emplace_back(CreateVariable(
               child, childVariablesReferences, childVariablesReferences, hex,
               /*is_name_duplicated=*/false, custom_name));
@@ -3300,7 +3298,7 @@ void request_variables(const llvm::json::Object &request) {
       // "[raw]" child that can be used to inspect the raw version of a
       // synthetic member. That eliminates the need for the user to go to the
       // debug console and type `frame var <variable> to get these values.
-      if (g_vsc.enable_synthetic_child_debugging && variable.IsSynthetic() &&
+      if (g_dap.enable_synthetic_child_debugging && variable.IsSynthetic() &&
           i == num_children)
         addChild(variable.GetNonSyntheticValue(), "[raw]");
     }
@@ -3308,7 +3306,7 @@ void request_variables(const llvm::json::Object &request) {
   llvm::json::Object body;
   body.try_emplace("variables", std::move(variables));
   response.try_emplace("body", std::move(body));
-  g_vsc.SendJSON(llvm::json::Value(std::move(response)));
+  g_dap.SendJSON(llvm::json::Value(std::move(response)));
 }
 
 // "DisassembleRequest": {
@@ -3394,27 +3392,27 @@ void request_disassemble(const llvm::json::Object &request) {
     response["success"] = false;
     response["message"] =
         "Malformed memory reference: " + memoryReference.str();
-    g_vsc.SendJSON(llvm::json::Value(std::move(response)));
+    g_dap.SendJSON(llvm::json::Value(std::move(response)));
     return;
   }
 
   addr_ptr += GetSigned(arguments, "instructionOffset", 0);
-  lldb::SBAddress addr(addr_ptr, g_vsc.target);
+  lldb::SBAddress addr(addr_ptr, g_dap.target);
   if (!addr.IsValid()) {
     response["success"] = false;
     response["message"] = "Memory reference not found in the current binary.";
-    g_vsc.SendJSON(llvm::json::Value(std::move(response)));
+    g_dap.SendJSON(llvm::json::Value(std::move(response)));
     return;
   }
 
   const auto inst_count = GetUnsigned(arguments, "instructionCount", 0);
   lldb::SBInstructionList insts =
-      g_vsc.target.ReadInstructions(addr, inst_count);
+      g_dap.target.ReadInstructions(addr, inst_count);
 
   if (!insts.IsValid()) {
     response["success"] = false;
     response["message"] = "Failed to find instructions for memory address.";
-    g_vsc.SendJSON(llvm::json::Value(std::move(response)));
+    g_dap.SendJSON(llvm::json::Value(std::move(response)));
     return;
   }
 
@@ -3424,11 +3422,11 @@ void request_disassemble(const llvm::json::Object &request) {
   for (size_t i = 0; i < num_insts; ++i) {
     lldb::SBInstruction inst = insts.GetInstructionAtIndex(i);
     auto addr = inst.GetAddress();
-    const auto inst_addr = addr.GetLoadAddress(g_vsc.target);
-    const char *m = inst.GetMnemonic(g_vsc.target);
-    const char *o = inst.GetOperands(g_vsc.target);
-    const char *c = inst.GetComment(g_vsc.target);
-    auto d = inst.GetData(g_vsc.target);
+    const auto inst_addr = addr.GetLoadAddress(g_dap.target);
+    const char *m = inst.GetMnemonic(g_dap.target);
+    const char *o = inst.GetOperands(g_dap.target);
+    const char *c = inst.GetComment(g_dap.target);
+    auto d = inst.GetData(g_dap.target);
 
     std::string bytes;
     llvm::raw_string_ostream sb(bytes);
@@ -3514,7 +3512,7 @@ void request_disassemble(const llvm::json::Object &request) {
   llvm::json::Object body;
   body.try_emplace("instructions", std::move(instructions));
   response.try_emplace("body", std::move(body));
-  g_vsc.SendJSON(llvm::json::Value(std::move(response)));
+  g_dap.SendJSON(llvm::json::Value(std::move(response)));
 }
 // A request used in testing to get the details on all breakpoints that are
 // currently set in the target. This helps us to test "setBreakpoints" and
@@ -3524,76 +3522,76 @@ void request__testGetTargetBreakpoints(const llvm::json::Object &request) {
   llvm::json::Object response;
   FillResponse(request, response);
   llvm::json::Array response_breakpoints;
-  for (uint32_t i = 0; g_vsc.target.GetBreakpointAtIndex(i).IsValid(); ++i) {
-    auto bp = g_vsc.target.GetBreakpointAtIndex(i);
+  for (uint32_t i = 0; g_dap.target.GetBreakpointAtIndex(i).IsValid(); ++i) {
+    auto bp = g_dap.target.GetBreakpointAtIndex(i);
     AppendBreakpoint(bp, response_breakpoints);
   }
   llvm::json::Object body;
   body.try_emplace("breakpoints", std::move(response_breakpoints));
   response.try_emplace("body", std::move(body));
-  g_vsc.SendJSON(llvm::json::Value(std::move(response)));
+  g_dap.SendJSON(llvm::json::Value(std::move(response)));
 }
 
 void RegisterRequestCallbacks() {
-  g_vsc.RegisterRequestCallback("attach", request_attach);
-  g_vsc.RegisterRequestCallback("completions", request_completions);
-  g_vsc.RegisterRequestCallback("continue", request_continue);
-  g_vsc.RegisterRequestCallback("configurationDone", request_configurationDone);
-  g_vsc.RegisterRequestCallback("disconnect", request_disconnect);
-  g_vsc.RegisterRequestCallback("evaluate", request_evaluate);
-  g_vsc.RegisterRequestCallback("exceptionInfo", request_exceptionInfo);
-  g_vsc.RegisterRequestCallback("initialize", request_initialize);
-  g_vsc.RegisterRequestCallback("launch", request_launch);
-  g_vsc.RegisterRequestCallback("next", request_next);
-  g_vsc.RegisterRequestCallback("pause", request_pause);
-  g_vsc.RegisterRequestCallback("restart", request_restart);
-  g_vsc.RegisterRequestCallback("scopes", request_scopes);
-  g_vsc.RegisterRequestCallback("setBreakpoints", request_setBreakpoints);
-  g_vsc.RegisterRequestCallback("setExceptionBreakpoints",
+  g_dap.RegisterRequestCallback("attach", request_attach);
+  g_dap.RegisterRequestCallback("completions", request_completions);
+  g_dap.RegisterRequestCallback("continue", request_continue);
+  g_dap.RegisterRequestCallback("configurationDone", request_configurationDone);
+  g_dap.RegisterRequestCallback("disconnect", request_disconnect);
+  g_dap.RegisterRequestCallback("evaluate", request_evaluate);
+  g_dap.RegisterRequestCallback("exceptionInfo", request_exceptionInfo);
+  g_dap.RegisterRequestCallback("initialize", request_initialize);
+  g_dap.RegisterRequestCallback("launch", request_launch);
+  g_dap.RegisterRequestCallback("next", request_next);
+  g_dap.RegisterRequestCallback("pause", request_pause);
+  g_dap.RegisterRequestCallback("restart", request_restart);
+  g_dap.RegisterRequestCallback("scopes", request_scopes);
+  g_dap.RegisterRequestCallback("setBreakpoints", request_setBreakpoints);
+  g_dap.RegisterRequestCallback("setExceptionBreakpoints",
                                 request_setExceptionBreakpoints);
-  g_vsc.RegisterRequestCallback("setFunctionBreakpoints",
+  g_dap.RegisterRequestCallback("setFunctionBreakpoints",
                                 request_setFunctionBreakpoints);
-  g_vsc.RegisterRequestCallback("setVariable", request_setVariable);
-  g_vsc.RegisterRequestCallback("source", request_source);
-  g_vsc.RegisterRequestCallback("stackTrace", request_stackTrace);
-  g_vsc.RegisterRequestCallback("stepIn", request_stepIn);
-  g_vsc.RegisterRequestCallback("stepOut", request_stepOut);
-  g_vsc.RegisterRequestCallback("threads", request_threads);
-  g_vsc.RegisterRequestCallback("variables", request_variables);
-  g_vsc.RegisterRequestCallback("disassemble", request_disassemble);
+  g_dap.RegisterRequestCallback("setVariable", request_setVariable);
+  g_dap.RegisterRequestCallback("source", request_source);
+  g_dap.RegisterRequestCallback("stackTrace", request_stackTrace);
+  g_dap.RegisterRequestCallback("stepIn", request_stepIn);
+  g_dap.RegisterRequestCallback("stepOut", request_stepOut);
+  g_dap.RegisterRequestCallback("threads", request_threads);
+  g_dap.RegisterRequestCallback("variables", request_variables);
+  g_dap.RegisterRequestCallback("disassemble", request_disassemble);
   // Custom requests
-  g_vsc.RegisterRequestCallback("compileUnits", request_compileUnits);
-  g_vsc.RegisterRequestCallback("modules", request_modules);
+  g_dap.RegisterRequestCallback("compileUnits", request_compileUnits);
+  g_dap.RegisterRequestCallback("modules", request_modules);
   // Testing requests
-  g_vsc.RegisterRequestCallback("_testGetTargetBreakpoints",
+  g_dap.RegisterRequestCallback("_testGetTargetBreakpoints",
                                 request__testGetTargetBreakpoints);
 }
 
 } // anonymous namespace
 
-static void printHelp(LLDBVSCodeOptTable &table, llvm::StringRef tool_name) {
+static void printHelp(LLDBDAPOptTable &table, llvm::StringRef tool_name) {
   std::string usage_str = tool_name.str() + " options";
-  table.printHelp(llvm::outs(), usage_str.c_str(), "LLDB VSCode", false);
+  table.printHelp(llvm::outs(), usage_str.c_str(), "LLDB DAP", false);
 
   std::string examples = R"___(
 EXAMPLES:
   The debug adapter can be started in two modes.
 
-  Running lldb-vscode without any arguments will start communicating with the
-  parent over stdio. Passing a port number causes lldb-vscode to start listening
+  Running lldb-dap without any arguments will start communicating with the
+  parent over stdio. Passing a port number causes lldb-dap to start listening
   for connections on that port.
 
-    lldb-vscode -p <port>
+    lldb-dap -p <port>
 
   Passing --wait-for-debugger will pause the process at startup and wait for a
   debugger to attach to the process.
 
-    lldb-vscode -g
+    lldb-dap -g
   )___";
   llvm::outs() << examples;
 }
 
-// If --launch-target is provided, this instance of lldb-vscode becomes a
+// If --launch-target is provided, this instance of lldb-dap becomes a
 // runInTerminal launcher. It will ultimately launch the program specified in
 // the --launch-target argument, which is the original program the user wanted
 // to debug. This is done in such a way that the actual debug adaptor can
@@ -3638,7 +3636,7 @@ void LaunchRunInTerminalTarget(llvm::opt::Arg &target_arg,
   // using a signal to prevent being paused forever.
 
   // This env var should be used only for tests.
-  const char *timeout_env_var = getenv("LLDB_VSCODE_RIT_TIMEOUT_IN_MS");
+  const char *timeout_env_var = getenv("LLDB_DAP_RIT_TIMEOUT_IN_MS");
   int timeout_in_ms =
       timeout_env_var != nullptr ? atoi(timeout_env_var) : 20000;
   if (llvm::Error err = comm_channel.WaitUntilDebugAdaptorAttaches(
@@ -3676,26 +3674,26 @@ int SetupStdoutStderrRedirection() {
   int stdoutfd = fileno(stdout);
   int new_stdout_fd = dup(stdoutfd);
   auto output_callback_stderr = [](llvm::StringRef data) {
-    g_vsc.SendOutput(OutputType::Stderr, data);
+    g_dap.SendOutput(OutputType::Stderr, data);
   };
   auto output_callback_stdout = [](llvm::StringRef data) {
-    g_vsc.SendOutput(OutputType::Stdout, data);
+    g_dap.SendOutput(OutputType::Stdout, data);
   };
   if (llvm::Error err = RedirectFd(stdoutfd, output_callback_stdout)) {
     std::string error_message = llvm::toString(std::move(err));
-    if (g_vsc.log)
-      *g_vsc.log << error_message << std::endl;
+    if (g_dap.log)
+      *g_dap.log << error_message << std::endl;
     output_callback_stderr(error_message);
   }
   if (llvm::Error err = RedirectFd(fileno(stderr), output_callback_stderr)) {
     std::string error_message = llvm::toString(std::move(err));
-    if (g_vsc.log)
-      *g_vsc.log << error_message << std::endl;
+    if (g_dap.log)
+      *g_dap.log << error_message << std::endl;
     output_callback_stderr(error_message);
   }
 
   /// used only by TestVSCode_redirection_to_console.py
-  if (getenv("LLDB_VSCODE_TEST_STDOUT_STDERR_REDIRECTION") != nullptr)
+  if (getenv("LLDB_DAP_TEST_STDOUT_STDERR_REDIRECTION") != nullptr)
     redirection_test();
   return new_stdout_fd;
 }
@@ -3706,9 +3704,9 @@ int main(int argc, char *argv[]) {
 
   llvm::SmallString<256> program_path(argv[0]);
   llvm::sys::fs::make_absolute(program_path);
-  g_vsc.debug_adaptor_path = program_path.str().str();
+  g_dap.debug_adaptor_path = program_path.str().str();
 
-  LLDBVSCodeOptTable T;
+  LLDBDAPOptTable T;
   unsigned MAI, MAC;
   llvm::ArrayRef<const char *> ArgsArr = llvm::ArrayRef(argv + 1, argc);
   llvm::opt::InputArgList input_args = T.ParseArgs(ArgsArr, MAI, MAC);
@@ -3722,11 +3720,11 @@ int main(int argc, char *argv[]) {
     llvm::opt::Arg *repl_mode = input_args.getLastArg(OPT_repl_mode);
     llvm::StringRef repl_mode_value = repl_mode->getValue();
     if (repl_mode_value == "auto") {
-      g_vsc.repl_mode = ReplMode::Auto;
+      g_dap.repl_mode = ReplMode::Auto;
     } else if (repl_mode_value == "variable") {
-      g_vsc.repl_mode = ReplMode::Variable;
+      g_dap.repl_mode = ReplMode::Variable;
     } else if (repl_mode_value == "command") {
-      g_vsc.repl_mode = ReplMode::Command;
+      g_dap.repl_mode = ReplMode::Command;
     } else {
       llvm::errs()
           << "'" << repl_mode_value
@@ -3742,7 +3740,8 @@ int main(int argc, char *argv[]) {
       if (debugger_pid) {
         llvm::StringRef debugger_pid_value = debugger_pid->getValue();
         if (debugger_pid_value.getAsInteger(10, pid)) {
-          llvm::errs() << "'" << debugger_pid_value << "' is not a valid "
+          llvm::errs() << "'" << debugger_pid_value
+                       << "' is not a valid "
                           "PID\n";
           return EXIT_FAILURE;
         }
@@ -3796,20 +3795,20 @@ int main(int argc, char *argv[]) {
     printf("Listening on port %i...\n", portno);
     SOCKET socket_fd = AcceptConnection(portno);
     if (socket_fd >= 0) {
-      g_vsc.input.descriptor = StreamDescriptor::from_socket(socket_fd, true);
-      g_vsc.output.descriptor = StreamDescriptor::from_socket(socket_fd, false);
+      g_dap.input.descriptor = StreamDescriptor::from_socket(socket_fd, true);
+      g_dap.output.descriptor = StreamDescriptor::from_socket(socket_fd, false);
     } else {
       return EXIT_FAILURE;
     }
   } else {
-    g_vsc.input.descriptor = StreamDescriptor::from_file(fileno(stdin), false);
-    g_vsc.output.descriptor = StreamDescriptor::from_file(new_stdout_fd, false);
+    g_dap.input.descriptor = StreamDescriptor::from_file(fileno(stdin), false);
+    g_dap.output.descriptor = StreamDescriptor::from_file(new_stdout_fd, false);
   }
 
   bool CleanExit = true;
-  if (auto Err = g_vsc.Loop()) {
-    if (g_vsc.log)
-      *g_vsc.log << "Transport Error: " << llvm::toString(std::move(Err))
+  if (auto Err = g_dap.Loop()) {
+    if (g_dap.log)
+      *g_dap.log << "Transport Error: " << llvm::toString(std::move(Err))
                  << "\n";
     CleanExit = false;
   }
diff --git a/lldb/tools/lldb-vscode/package.json b/lldb/tools/lldb-dap/package.json
similarity index 97%
rename from lldb/tools/lldb-vscode/package.json
rename to lldb/tools/lldb-dap/package.json
index 1b3d452f92..79954cde2b 100644
--- a/lldb/tools/lldb-vscode/package.json
+++ b/lldb/tools/lldb-dap/package.json
@@ -1,5 +1,5 @@
 {
-	"name": "lldb-vscode",
+	"name": "lldb-dap",
 	"displayName": "LLDB VSCode",
 	"version": "0.1.0",
 	"publisher": "llvm",
@@ -103,7 +103,7 @@
 		],
 		"debuggers": [
 			{
-				"type": "lldb-vscode",
+				"type": "lldb-dap",
 				"label": "Native LLDB Debugger",
 				"enableBreakpointsFor": {
 					"languageIds": [
@@ -124,9 +124,9 @@
 						"swift"
 					]
 				},
-				"program": "./bin/lldb-vscode",
+				"program": "./bin/lldb-dap",
 				"windows": {
-					"program": "./bin/lldb-vscode.exe"
+					"program": "./bin/lldb-dap.exe"
 				},
 				"configurationAttributes": {
 					"launch": {
@@ -219,7 +219,7 @@
 							},
 							"launchCommands": {
 								"type": "array",
-								"description": "Custom commands that are executed instead of launching a process. A target will be created with the launch arguments prior to executing these commands. The commands may optionally create a new target and must perform a launch. A valid process must exist after these commands complete or the \"launch\" will fail. Launch the process with \"process launch -s\" to make the process to at the entry point since lldb-vscode will auto resume if necessary.",
+								"description": "Custom commands that are executed instead of launching a process. A target will be created with the launch arguments prior to executing these commands. The commands may optionally create a new target and must perform a launch. A valid process must exist after these commands complete or the \"launch\" will fail. Launch the process with \"process launch -s\" to make the process to at the entry point since lldb-dap will auto resume if necessary.",
 								"default": []
 							},
 							"stopCommands": {
@@ -345,7 +345,7 @@
 				},
 				"initialConfigurations": [
 					{
-						"type": "lldb-vscode",
+						"type": "lldb-dap",
 						"request": "launch",
 						"name": "Debug",
 						"program": "${workspaceRoot}/<your program>",
@@ -359,7 +359,7 @@
 						"label": "LLDB: Launch",
 						"description": "",
 						"body": {
-							"type": "lldb-vscode",
+							"type": "lldb-dap",
 							"request": "launch",
 							"name": "${2:Launch}",
 							"program": "^\"\\${workspaceRoot}/${1:<your program>}\"",
diff --git a/lldb/tools/lldb-vscode/syntaxes/arm.disasm b/lldb/tools/lldb-dap/syntaxes/arm.disasm
similarity index 100%
rename from lldb/tools/lldb-vscode/syntaxes/arm.disasm
rename to lldb/tools/lldb-dap/syntaxes/arm.disasm
diff --git a/lldb/tools/lldb-vscode/syntaxes/arm64.disasm b/lldb/tools/lldb-dap/syntaxes/arm64.disasm
similarity index 100%
rename from lldb/tools/lldb-vscode/syntaxes/arm64.disasm
rename to lldb/tools/lldb-dap/syntaxes/arm64.disasm
diff --git a/lldb/tools/lldb-vscode/syntaxes/disassembly.json b/lldb/tools/lldb-dap/syntaxes/disassembly.json
similarity index 100%
rename from lldb/tools/lldb-vscode/syntaxes/disassembly.json
rename to lldb/tools/lldb-dap/syntaxes/disassembly.json
diff --git a/lldb/tools/lldb-vscode/syntaxes/x86.disasm b/lldb/tools/lldb-dap/syntaxes/x86.disasm
similarity index 100%
rename from lldb/tools/lldb-vscode/syntaxes/x86.disasm
rename to lldb/tools/lldb-dap/syntaxes/x86.disasm
diff --git a/lldb/tools/lldb-vscode b/lldb/tools/lldb-vscode
new file mode 120000
index 0000000000..46b4004408
--- /dev/null
+++ b/lldb/tools/lldb-vscode
@@ -0,0 +1 @@
+lldb-dap
\ No newline at end of file
diff --git a/lldb/unittests/Expression/DWARFExpressionTest.cpp b/lldb/unittests/Expression/DWARFExpressionTest.cpp
index b8b5b39422..8d77d6b258 100644
--- a/lldb/unittests/Expression/DWARFExpressionTest.cpp
+++ b/lldb/unittests/Expression/DWARFExpressionTest.cpp
@@ -25,6 +25,7 @@
 
 using namespace lldb_private;
 using namespace lldb_private::dwarf;
+using namespace lldb_private::plugin::dwarf;
 
 static llvm::Expected<Scalar> Evaluate(llvm::ArrayRef<uint8_t> expr,
                                        lldb::ModuleSP module_sp = {},
diff --git a/lldb/unittests/ScriptInterpreter/Python/PythonDataObjectsTests.cpp b/lldb/unittests/ScriptInterpreter/Python/PythonDataObjectsTests.cpp
index 3450712395..0b816a8ae0 100644
--- a/lldb/unittests/ScriptInterpreter/Python/PythonDataObjectsTests.cpp
+++ b/lldb/unittests/ScriptInterpreter/Python/PythonDataObjectsTests.cpp
@@ -23,7 +23,6 @@
 
 using namespace lldb_private;
 using namespace lldb_private::python;
-using llvm::Error;
 using llvm::Expected;
 
 class PythonDataObjectsTest : public PythonTestSuite {
diff --git a/lldb/unittests/Symbol/SymtabTest.cpp b/lldb/unittests/Symbol/SymtabTest.cpp
index e6b78eb5cd..7b8892e5b5 100644
--- a/lldb/unittests/Symbol/SymtabTest.cpp
+++ b/lldb/unittests/Symbol/SymtabTest.cpp
@@ -27,6 +27,7 @@
 
 using namespace lldb;
 using namespace lldb_private;
+using namespace lldb_private::plugin::dwarf;
 
 class SymtabTest : public testing::Test {
   SubsystemRAII<FileSystem, HostInfo, ObjectFileMachO, SymbolFileDWARF,
diff --git a/lldb/unittests/Symbol/TestLineEntry.cpp b/lldb/unittests/Symbol/TestLineEntry.cpp
index d8b64c525e..592504bc21 100644
--- a/lldb/unittests/Symbol/TestLineEntry.cpp
+++ b/lldb/unittests/Symbol/TestLineEntry.cpp
@@ -27,8 +27,9 @@
 #include "llvm/Support/Program.h"
 #include "llvm/Testing/Support/Error.h"
 
-using namespace lldb_private;
 using namespace lldb;
+using namespace lldb_private;
+using namespace lldb_private::plugin::dwarf;
 
 class LineEntryTest : public testing::Test {
   SubsystemRAII<FileSystem, HostInfo, ObjectFileMachO, SymbolFileDWARF,
diff --git a/lldb/unittests/SymbolFile/DWARF/DWARFASTParserClangTests.cpp b/lldb/unittests/SymbolFile/DWARF/DWARFASTParserClangTests.cpp
index 20a085741f..0c2a0d735d 100644
--- a/lldb/unittests/SymbolFile/DWARF/DWARFASTParserClangTests.cpp
+++ b/lldb/unittests/SymbolFile/DWARF/DWARFASTParserClangTests.cpp
@@ -18,6 +18,7 @@
 using namespace lldb;
 using namespace lldb_private;
 using namespace lldb_private::dwarf;
+using namespace lldb_private::plugin::dwarf;
 
 namespace {
 static std::once_flag debugger_initialize_flag;
diff --git a/lldb/unittests/SymbolFile/DWARF/DWARFDIETest.cpp b/lldb/unittests/SymbolFile/DWARF/DWARFDIETest.cpp
index 7fc37cefea..8497855b2f 100644
--- a/lldb/unittests/SymbolFile/DWARF/DWARFDIETest.cpp
+++ b/lldb/unittests/SymbolFile/DWARF/DWARFDIETest.cpp
@@ -14,6 +14,7 @@
 
 using namespace lldb;
 using namespace lldb_private;
+using namespace lldb_private::plugin::dwarf;
 
 TEST(DWARFDIETest, ChildIteration) {
   // Tests DWARFDIE::child_iterator.
diff --git a/lldb/unittests/SymbolFile/DWARF/DWARFIndexCachingTest.cpp b/lldb/unittests/SymbolFile/DWARF/DWARFIndexCachingTest.cpp
index e5d2b8eda0..c5f25780a1 100644
--- a/lldb/unittests/SymbolFile/DWARF/DWARFIndexCachingTest.cpp
+++ b/lldb/unittests/SymbolFile/DWARF/DWARFIndexCachingTest.cpp
@@ -21,6 +21,7 @@
 
 using namespace lldb;
 using namespace lldb_private;
+using namespace lldb_private::plugin::dwarf;
 
 static void EncodeDecode(const DIERef &object, ByteOrder byte_order) {
   const uint8_t addr_size = 8;
diff --git a/lldb/unittests/SymbolFile/DWARF/DWARFUnitTest.cpp b/lldb/unittests/SymbolFile/DWARF/DWARFUnitTest.cpp
index 123acb6ea8..9135449415 100644
--- a/lldb/unittests/SymbolFile/DWARF/DWARFUnitTest.cpp
+++ b/lldb/unittests/SymbolFile/DWARF/DWARFUnitTest.cpp
@@ -13,6 +13,7 @@
 
 using namespace lldb;
 using namespace lldb_private;
+using namespace lldb_private::plugin::dwarf;
 
 TEST(DWARFUnitTest, NullUnitDie) {
   // Make sure we don't crash parsing a null unit DIE.
diff --git a/lldb/unittests/SymbolFile/DWARF/SymbolFileDWARFTests.cpp b/lldb/unittests/SymbolFile/DWARF/SymbolFileDWARFTests.cpp
index 9cbdeffc59..3192efe8b3 100644
--- a/lldb/unittests/SymbolFile/DWARF/SymbolFileDWARFTests.cpp
+++ b/lldb/unittests/SymbolFile/DWARF/SymbolFileDWARFTests.cpp
@@ -38,6 +38,7 @@
 using namespace lldb;
 using namespace lldb_private;
 using namespace lldb_private::dwarf;
+using namespace lldb_private::plugin::dwarf;
 
 class SymbolFileDWARFTests : public testing::Test {
   SubsystemRAII<FileSystem, HostInfo, ObjectFilePECOFF, SymbolFileDWARF,
diff --git a/lldb/unittests/SymbolFile/DWARF/XcodeSDKModuleTests.cpp b/lldb/unittests/SymbolFile/DWARF/XcodeSDKModuleTests.cpp
index ca4c29ebb9..c37da89ff7 100644
--- a/lldb/unittests/SymbolFile/DWARF/XcodeSDKModuleTests.cpp
+++ b/lldb/unittests/SymbolFile/DWARF/XcodeSDKModuleTests.cpp
@@ -19,6 +19,7 @@
 
 using namespace lldb;
 using namespace lldb_private;
+using namespace lldb_private::plugin::dwarf;
 
 #ifdef __APPLE__
 namespace {
diff --git a/lldb/unittests/SymbolFile/PDB/SymbolFilePDBTests.cpp b/lldb/unittests/SymbolFile/PDB/SymbolFilePDBTests.cpp
index bd69d24607..acd381ccad 100644
--- a/lldb/unittests/SymbolFile/PDB/SymbolFilePDBTests.cpp
+++ b/lldb/unittests/SymbolFile/PDB/SymbolFilePDBTests.cpp
@@ -53,7 +53,7 @@ class SymbolFilePDBTests : public testing::Test {
     FileSystem::Initialize();
     HostInfo::Initialize();
     ObjectFilePECOFF::Initialize();
-    SymbolFileDWARF::Initialize();
+    plugin::dwarf::SymbolFileDWARF::Initialize();
     TypeSystemClang::Initialize();
     SymbolFilePDB::Initialize();
 
@@ -64,7 +64,7 @@ class SymbolFilePDBTests : public testing::Test {
   void TearDown() override {
     SymbolFilePDB::Terminate();
     TypeSystemClang::Initialize();
-    SymbolFileDWARF::Terminate();
+    plugin::dwarf::SymbolFileDWARF::Terminate();
     ObjectFilePECOFF::Terminate();
     HostInfo::Terminate();
     FileSystem::Terminate();
diff --git a/lldb/unittests/TestingSupport/Symbol/YAMLModuleTester.cpp b/lldb/unittests/TestingSupport/Symbol/YAMLModuleTester.cpp
index 14e7c7d520..e9fdbcbafe 100644
--- a/lldb/unittests/TestingSupport/Symbol/YAMLModuleTester.cpp
+++ b/lldb/unittests/TestingSupport/Symbol/YAMLModuleTester.cpp
@@ -13,6 +13,7 @@
 #include "llvm/ObjectYAML/DWARFEmitter.h"
 
 using namespace lldb_private;
+using namespace lldb_private::plugin::dwarf;
 
 YAMLModuleTester::YAMLModuleTester(llvm::StringRef yaml_data, size_t cu_index) {
   llvm::Expected<TestFile> File = TestFile::fromYaml(yaml_data);
diff --git a/lldb/unittests/TestingSupport/Symbol/YAMLModuleTester.h b/lldb/unittests/TestingSupport/Symbol/YAMLModuleTester.h
index 3021ca7c9f..37d9025cbc 100644
--- a/lldb/unittests/TestingSupport/Symbol/YAMLModuleTester.h
+++ b/lldb/unittests/TestingSupport/Symbol/YAMLModuleTester.h
@@ -26,16 +26,16 @@ namespace lldb_private {
 class YAMLModuleTester {
 protected:
   SubsystemRAII<FileSystem, HostInfo, TypeSystemClang, ObjectFileELF,
-                SymbolFileDWARF>
+                plugin::dwarf::SymbolFileDWARF>
       subsystems;
   std::optional<TestFile> m_file;
   lldb::ModuleSP m_module_sp;
-  DWARFUnit *m_dwarf_unit;
+  plugin::dwarf::DWARFUnit *m_dwarf_unit;
 
 public:
   /// Parse the debug info sections from the YAML description.
   YAMLModuleTester(llvm::StringRef yaml_data, size_t cu_index = 0);
-  DWARFUnit *GetDwarfUnit() const { return m_dwarf_unit; }
+  plugin::dwarf::DWARFUnit *GetDwarfUnit() const { return m_dwarf_unit; }
   lldb::ModuleSP GetModule() const { return m_module_sp; }
 };
 
diff --git a/lldb/unittests/Utility/ScalarTest.cpp b/lldb/unittests/Utility/ScalarTest.cpp
index 17dfc689dd..29a4bcd356 100644
--- a/lldb/unittests/Utility/ScalarTest.cpp
+++ b/lldb/unittests/Utility/ScalarTest.cpp
@@ -241,7 +241,7 @@ TEST(ScalarTest, ExtractBitfield) {
 
 template <typename T> static std::string ScalarGetValue(T value) {
   StreamString stream;
-  Scalar(value).GetValue(&stream, false);
+  Scalar(value).GetValue(stream, false);
   return std::string(stream.GetString());
 }
 
diff --git a/lldb/unittests/tools/lldb-server/tests/MessageObjects.cpp b/lldb/unittests/tools/lldb-server/tests/MessageObjects.cpp
index 2c6b9a2299..da4dc10d4b 100644
--- a/lldb/unittests/tools/lldb-server/tests/MessageObjects.cpp
+++ b/lldb/unittests/tools/lldb-server/tests/MessageObjects.cpp
@@ -42,9 +42,9 @@ Expected<ProcessInfo> ProcessInfo::create(StringRef response) {
   process_info.m_triple = fromHex(elements["triple"]);
   StringRef endian_str = elements["endian"];
   if (endian_str == "little")
-    process_info.m_endian = support::little;
+    process_info.m_endian = llvm::endianness::little;
   else if (endian_str == "big")
-    process_info.m_endian = support::big;
+    process_info.m_endian = llvm::endianness::big;
   else
     return make_parsing_error("ProcessInfo: endian");
 
@@ -53,7 +53,7 @@ Expected<ProcessInfo> ProcessInfo::create(StringRef response) {
 
 lldb::pid_t ProcessInfo::GetPid() const { return m_pid; }
 
-support::endianness ProcessInfo::GetEndian() const { return m_endian; }
+llvm::endianness ProcessInfo::GetEndian() const { return m_endian; }
 
 //====== ThreadInfo ============================================================
 ThreadInfo::ThreadInfo(StringRef name, StringRef reason, RegisterMap registers,
@@ -84,7 +84,7 @@ JThreadsInfo::parseRegisters(const StructuredData::Dictionary &Dict,
       return make_parsing_error("JThreadsInfo: register key[{0}]", i);
 
     auto RegValOr =
-        parseRegisterValue(RegInfos[Register], ValueStr, support::big);
+        parseRegisterValue(RegInfos[Register], ValueStr, llvm::endianness::big);
     if (!RegValOr)
       return RegValOr.takeError();
     Result[Register] = std::move(*RegValOr);
@@ -214,9 +214,10 @@ Expected<RegisterValue> parseRegisterValue(const RegisterInfo &Info,
   StringExtractor(HexValue).GetHexBytes(Bytes, '\xcc');
   RegisterValue Value;
   Status ST;
-  Value.SetFromMemoryData(
-      Info, Bytes.data(), Bytes.size(),
-      Endian == support::little ? eByteOrderLittle : eByteOrderBig, ST);
+  Value.SetFromMemoryData(Info, Bytes.data(), Bytes.size(),
+                          Endian == llvm::endianness::little ? eByteOrderLittle
+                                                             : eByteOrderBig,
+                          ST);
   if (ST.Fail())
     return ST.ToError();
   return Value;
@@ -237,7 +238,7 @@ StopReply::create(StringRef Response, llvm::endianness Endian,
 
 Expected<RegisterMap> StopReplyStop::parseRegisters(
     const StringMap<SmallVector<StringRef, 2>> &Elements,
-    support::endianness Endian, ArrayRef<lldb_private::RegisterInfo> RegInfos) {
+    llvm::endianness Endian, ArrayRef<lldb_private::RegisterInfo> RegInfos) {
 
   RegisterMap Result;
   for (const auto &E : Elements) {
@@ -263,7 +264,7 @@ Expected<RegisterMap> StopReplyStop::parseRegisters(
 }
 
 Expected<std::unique_ptr<StopReplyStop>>
-StopReplyStop::create(StringRef Response, support::endianness Endian,
+StopReplyStop::create(StringRef Response, llvm::endianness Endian,
                       ArrayRef<RegisterInfo> RegInfos) {
   unsigned int Signal;
   StringRef SignalStr = Response.take_front(2);
diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt
index 103c08ffbe..82d4beea91 100644
--- a/llvm/CMakeLists.txt
+++ b/llvm/CMakeLists.txt
@@ -643,6 +643,9 @@ option(LLVM_USE_OPROFILE
 option(LLVM_EXTERNALIZE_DEBUGINFO
   "Generate dSYM files and strip executables and libraries (Darwin Only)" OFF)
 
+option(LLVM_EXPERIMENTAL_DEBUGINFO_ITERATORS
+  "Add extra Booleans to ilist_iterators to communicate facts for debug-info" OFF)
+
 set(LLVM_CODESIGNING_IDENTITY "" CACHE STRING
   "Sign executables and dylibs with the given identity or skip if empty (Darwin Only)")
 
@@ -1216,6 +1219,9 @@ if( LLVM_INCLUDE_EXAMPLES )
 endif()
 
 if( LLVM_INCLUDE_TESTS )
+  set(LLVM_GTEST_RUN_UNDER
+    "" CACHE STRING
+    "Define the wrapper program that LLVM unit tests should be run under.")
   if(EXISTS ${LLVM_MAIN_SRC_DIR}/projects/test-suite AND TARGET clang)
     include(LLVMExternalProjectUtils)
     llvm_ExternalProject_Add(test-suite ${LLVM_MAIN_SRC_DIR}/projects/test-suite
diff --git a/llvm/CODE_OWNERS.TXT b/llvm/CODE_OWNERS.TXT
index 2a9fe11942..35874381d7 100644
--- a/llvm/CODE_OWNERS.TXT
+++ b/llvm/CODE_OWNERS.TXT
@@ -131,6 +131,10 @@ N: Benjamin Kramer
 E: benny.kra@gmail.com
 D: DWARF Parser
 
+N: Sundeep Kushwaha
+E: sundeepk@quicinc.com
+D: Hexagon Backend
+
 N: Florian Hahn
 E: flo@fhahn.com
 D: Loop Vectorizer
@@ -172,10 +176,6 @@ N: Diego Novillo
 E: dnovillo@google.com
 D: SampleProfile and related parts of ProfileData
 
-N: Krzysztof Parzyszek
-E: kparzysz@quicinc.com
-D: Hexagon Backend
-
 N: Nigel Perks
 E: nigelp@xmos.com
 D: XCore Backend
diff --git a/llvm/cmake/modules/AddLLVM.cmake b/llvm/cmake/modules/AddLLVM.cmake
index 93011522e4..05e4d6c823 100644
--- a/llvm/cmake/modules/AddLLVM.cmake
+++ b/llvm/cmake/modules/AddLLVM.cmake
@@ -726,6 +726,8 @@ function(llvm_add_library name)
     endforeach()
   endif()
 
+  add_custom_linker_flags(${name})
+
   if(ARG_SHARED OR ARG_MODULE)
     llvm_externalize_debuginfo(${name})
     llvm_codesign(${name} ENTITLEMENTS ${ARG_ENTITLEMENTS} BUNDLE_PATH ${ARG_BUNDLE_PATH})
@@ -1019,6 +1021,8 @@ macro(add_llvm_executable name)
     endforeach()
   endif( LLVM_COMMON_DEPENDS )
 
+  add_custom_linker_flags(${name})
+
   if(NOT ARG_IGNORE_EXTERNALIZE_DEBUGINFO)
     llvm_externalize_debuginfo(${name})
   endif()
@@ -1524,6 +1528,13 @@ macro(add_llvm_tool_subdirectory name)
   add_llvm_external_project(${name})
 endmacro(add_llvm_tool_subdirectory)
 
+macro(add_custom_linker_flags name)
+  if (LLVM_${name}_LINKER_FLAGS)
+    message(DEBUG "Applying ${LLVM_${name}_LINKER_FLAGS} to ${name}")
+    target_link_options(${name} PRIVATE ${LLVM_${name}_LINKER_FLAGS})
+  endif()
+endmacro()
+
 function(get_project_name_from_src_var var output)
   string(REGEX MATCH "LLVM_EXTERNAL_(.*)_SOURCE_DIR"
          MACHED_TOOL "${var}")
diff --git a/llvm/cmake/modules/FindFFI.cmake b/llvm/cmake/modules/FindFFI.cmake
index a493a89d63..c9ba104601 100644
--- a/llvm/cmake/modules/FindFFI.cmake
+++ b/llvm/cmake/modules/FindFFI.cmake
@@ -38,15 +38,27 @@ find_library(FFI_LIBRARIES ffi PATHS ${FFI_LIBRARY_DIR})
 
 if(FFI_LIBRARIES)
   include(CMakePushCheckState)
-  include(CheckCSourceCompiles)
   cmake_push_check_state()
   list(APPEND CMAKE_REQUIRED_LIBRARIES ${FFI_LIBRARIES})
-  check_c_source_compiles("
+  set(HAVE_FFI_CALL_SRC [=[
+    #ifdef __cplusplus
+    extern "C" {
+    #endif
     struct ffi_cif;
     typedef struct ffi_cif ffi_cif;
     void ffi_call(ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue);
-    int main(void) { ffi_call(0, 0, 0, 0); }"
-    HAVE_FFI_CALL)
+    #ifdef __cplusplus
+    }
+    #endif
+    int main(void) { ffi_call(0, 0, 0, 0); }
+    ]=])
+  if(DEFINED CMAKE_C_COMPILER)
+    include(CheckCSourceCompiles)
+    check_c_source_compiles("${HAVE_FFI_CALL_SRC}" HAVE_FFI_CALL)
+  else()
+    include(CheckCXXSourceCompiles)
+    check_cxx_source_compiles("${HAVE_FFI_CALL_SRC}" HAVE_FFI_CALL)
+  endif()
   cmake_pop_check_state()
 endif()
 
diff --git a/llvm/cmake/modules/FindTerminfo.cmake b/llvm/cmake/modules/FindTerminfo.cmake
index eef1f95853..163af66970 100644
--- a/llvm/cmake/modules/FindTerminfo.cmake
+++ b/llvm/cmake/modules/FindTerminfo.cmake
@@ -15,13 +15,25 @@ find_library(Terminfo_LIBRARIES NAMES terminfo tinfo curses ncurses ncursesw)
 
 if(Terminfo_LIBRARIES)
   include(CMakePushCheckState)
-  include(CheckCSourceCompiles)
   cmake_push_check_state()
   list(APPEND CMAKE_REQUIRED_LIBRARIES ${Terminfo_LIBRARIES})
-  check_c_source_compiles("
+  set(Terminfo_LINKABLE_SRC [=[
+    #ifdef __cplusplus
+    extern "C" {
+    #endif
     int setupterm(char *term, int filedes, int *errret);
-    int main(void) { return setupterm(0, 0, 0); }"
-    Terminfo_LINKABLE)
+    #ifdef __cplusplus
+    }
+    #endif
+    int main(void) { return setupterm(0, 0, 0); }
+    ]=])
+  if(DEFINED CMAKE_C_COMPILER)
+    include(CheckCSourceCompiles)
+    check_c_source_compiles("${Terminfo_LINKABLE_SRC}" Terminfo_LINKABLE)
+  else()
+    include(CheckCXXSourceCompiles)
+    check_cxx_source_compiles("${Terminfo_LINKABLE_SRC}" Terminfo_LINKABLE)
+  endif()
   cmake_pop_check_state()
 endif()
 
diff --git a/llvm/cmake/modules/HandleLLVMOptions.cmake b/llvm/cmake/modules/HandleLLVMOptions.cmake
index c5142c9e66..ca7cedd6e4 100644
--- a/llvm/cmake/modules/HandleLLVMOptions.cmake
+++ b/llvm/cmake/modules/HandleLLVMOptions.cmake
@@ -36,6 +36,21 @@ string(TOUPPER "${LLVM_ENABLE_LTO}" uppercase_LLVM_ENABLE_LTO)
 # The following only works with the Ninja generator in CMake >= 3.0.
 set(LLVM_PARALLEL_COMPILE_JOBS "" CACHE STRING
   "Define the maximum number of concurrent compilation jobs (Ninja only).")
+if(LLVM_RAM_PER_COMPILE_JOB OR LLVM_RAM_PER_LINK_JOB)
+  cmake_host_system_information(RESULT available_physical_memory QUERY AVAILABLE_PHYSICAL_MEMORY)
+  cmake_host_system_information(RESULT number_of_logical_cores QUERY NUMBER_OF_LOGICAL_CORES)
+endif()
+if(LLVM_RAM_PER_COMPILE_JOB)
+  math(EXPR jobs_with_sufficient_memory "${available_physical_memory} / ${LLVM_RAM_PER_COMPILE_JOB}" OUTPUT_FORMAT DECIMAL)
+  if (jobs_with_sufficient_memory LESS 1)
+    set(jobs_with_sufficient_memory 1)
+  endif()
+  if (jobs_with_sufficient_memory LESS number_of_logical_cores)
+    set(LLVM_PARALLEL_COMPILE_JOBS "${jobs_with_sufficient_memory}")
+  else()
+    set(LLVM_PARALLEL_COMPILE_JOBS "${number_of_logical_cores}")
+  endif()
+endif()
 if(LLVM_PARALLEL_COMPILE_JOBS)
   if(NOT CMAKE_GENERATOR MATCHES "Ninja")
     message(WARNING "Job pooling is only available with Ninja generators.")
@@ -47,6 +62,17 @@ endif()
 
 set(LLVM_PARALLEL_LINK_JOBS "" CACHE STRING
   "Define the maximum number of concurrent link jobs (Ninja only).")
+if(LLVM_RAM_PER_LINK_JOB)
+  math(EXPR jobs_with_sufficient_memory "${available_physical_memory} / ${LLVM_RAM_PER_LINK_JOB}" OUTPUT_FORMAT DECIMAL)
+  if (jobs_with_sufficient_memory LESS 1)
+    set(jobs_with_sufficient_memory 1)
+  endif()
+  if (jobs_with_sufficient_memory LESS number_of_logical_cores)
+    set(LLVM_PARALLEL_LINK_JOBS "${jobs_with_sufficient_memory}")
+  else()
+    set(LLVM_PARALLEL_LINK_JOBS "${number_of_logical_cores}")
+  endif()
+endif()
 if(CMAKE_GENERATOR MATCHES "Ninja")
   if(NOT LLVM_PARALLEL_LINK_JOBS AND uppercase_LLVM_ENABLE_LTO STREQUAL "THIN")
     message(STATUS "ThinLTO provides its own parallel linking - limiting parallel link jobs to 2.")
@@ -85,8 +111,13 @@ if( LLVM_ENABLE_ASSERTIONS )
   endif()
   # Enable assertions in libstdc++.
   add_compile_definitions(_GLIBCXX_ASSERTIONS)
-  # Enable the hardened mode in libc++.
-  add_compile_definitions(_LIBCPP_ENABLE_HARDENED_MODE)
+  # Cautiously enable the safe hardened mode in libc++.
+  if((DEFINED LIBCXX_HARDENING_MODE) AND
+     (NOT LIBCXX_HARDENING_MODE STREQUAL "safe"))
+    message(WARNING "LLVM_ENABLE_ASSERTIONS implies LIBCXX_HARDENING_MODE \"safe\" but is overriden from command line with value \"${LIBCXX_HARDENING_MODE}\".")
+  else()
+    set(LIBCXX_HARDENING_MODE "safe")
+  endif()
 endif()
 
 if(LLVM_ENABLE_EXPENSIVE_CHECKS)
@@ -109,6 +140,10 @@ if(LLVM_ENABLE_EXPENSIVE_CHECKS)
   endif()
 endif()
 
+if(LLVM_EXPERIMENTAL_DEBUGINFO_ITERATORS)
+  add_compile_definitions(EXPERIMENTAL_DEBUGINFO_ITERATORS)
+endif()
+
 if (LLVM_ENABLE_STRICT_FIXED_SIZE_VECTORS)
   add_compile_definitions(STRICT_FIXED_SIZE_VECTORS)
 endif()
@@ -1007,7 +1042,7 @@ if (LLVM_USE_SPLIT_DWARF AND
   # Limit to clang and gcc so far. Add compilers supporting this option.
   if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR
       CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-    add_compile_options(-gsplit-dwarf)
+    add_compile_options($<$<COMPILE_LANGUAGE:C,CXX>:-gsplit-dwarf>)
     include(LLVMCheckLinkerFlag)
     llvm_check_linker_flag(CXX "-Wl,--gdb-index" LINKER_SUPPORTS_GDB_INDEX)
     append_if(LINKER_SUPPORTS_GDB_INDEX "-Wl,--gdb-index"
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index 8022816d7e..9427df94e1 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -360,7 +360,7 @@ Every processor supports every OS ABI (see :ref:`amdgpu-os`) with the following
      ``gfx90a``                  ``amdgcn``   dGPU  - sramecc         - Absolute      - *rocm-amdhsa* *TBA*
                                                     - tgsplit           flat
                                                     - xnack             scratch                       .. TODO::
-                                                                      - Packed
+                                                    - kernarg preload - Packed
                                                                         work-item                       Add product
                                                                         IDs                             names.
 
@@ -381,21 +381,21 @@ Every processor supports every OS ABI (see :ref:`amdgpu-os`) with the following
      ``gfx940``                  ``amdgcn``   dGPU  - sramecc         - Architected                   *TBA*
                                                     - tgsplit           flat
                                                     - xnack             scratch                       .. TODO::
-                                                                      - Packed
+                                                    - kernarg preload - Packed
                                                                         work-item                       Add product
                                                                         IDs                             names.
 
      ``gfx941``                  ``amdgcn``   dGPU  - sramecc         - Architected                   *TBA*
                                                     - tgsplit           flat
                                                     - xnack             scratch                       .. TODO::
-                                                                      - Packed
+                                                    - kernarg preload - Packed
                                                                         work-item                       Add product
                                                                         IDs                             names.
 
      ``gfx942``                  ``amdgcn``   dGPU  - sramecc         - Architected                   *TBA*
                                                     - tgsplit           flat
                                                     - xnack             scratch                       .. TODO::
-                                                                      - Packed
+                                                    - kernarg preload - Packed
                                                                         work-item                       Add product
                                                                         IDs                             names.
 
@@ -4375,12 +4375,24 @@ The fields used by CP for code objects before V3 also match those specified in
                                                      dynamically sized stack.
                                                      This is only set in code
                                                      object v5 and later.
-     463:460 1 bit                                   Reserved, must be 0.
-     464     1 bit   RESERVED_464                    Deprecated, must be 0.
-     467:465 3 bits                                  Reserved, must be 0.
-     468     1 bit   RESERVED_468                    Deprecated, must be 0.
-     469:471 3 bits                                  Reserved, must be 0.
-     511:472 5 bytes                                 Reserved, must be 0.
+     463:460 4 bits                                  Reserved, must be 0.
+     470:464 7 bits  KERNARG_PRELOAD_SPEC_LENGTH     GFX6-GFX9
+                                                       - Reserved, must be 0.
+                                                     GFX90A, GFX940
+                                                       - The number of dwords from
+                                                         the kernarg segment to preload
+                                                         into User SGPRs before kernel
+                                                         execution. (see
+                                                         :ref:`amdgpu-amdhsa-kernarg-preload`).
+     479:471 9 bits  KERNARG_PRELOAD_SPEC_OFFSET     GFX6-GFX9
+                                                       - Reserved, must be 0.
+                                                     GFX90A, GFX940
+                                                       - An offset in dwords into the
+                                                         kernarg segment to begin
+                                                         preloading data into User
+                                                         SGPRs. (see
+                                                         :ref:`amdgpu-amdhsa-kernarg-preload`).
+     511:480 4 bytes                                 Reserved, must be 0.
      512     **Total size 64 bytes.**
      ======= ====================================================================
 
@@ -5002,7 +5014,7 @@ for enabled registers are dense starting at SGPR0: the first enabled register is
 SGPR0, the next enabled register is SGPR1 etc.; disabled registers do not have
 an SGPR number.
 
-The initial SGPRs comprise up to 16 User SRGPs that are set by CP and apply to
+The initial SGPRs comprise up to 16 User SGPRs that are set by CP and apply to
 all wavefronts of the grid. It is possible to specify more than 16 User SGPRs
 using the ``enable_sgpr_*`` bit fields, in which case only the first 16 are
 actually initialized. These are then immediately followed by the System SGPRs
@@ -5045,6 +5057,9 @@ SGPR register initial state is defined in
      then       Flat Scratch Init          2      See
                 (enable_sgpr_flat_scratch         :ref:`amdgpu-amdhsa-kernel-prolog-flat-scratch`.
                 _init)
+     then       Preloaded Kernargs         N/A    See
+                (kernarg_preload_spec             :ref:`amdgpu-amdhsa-kernarg-preload`.
+                _length)
      then       Private Segment Size       1      The 32-bit byte size of a
                 (enable_sgpr_private              single work-item's memory
                 _segment_size)                    allocation. This is the
@@ -5177,6 +5192,31 @@ following properties:
 * MTYPE set to support memory coherence that matches the runtime (such as CC for
   APU and NC for dGPU).
 
+.. _amdgpu-amdhsa-kernarg-preload:
+
+Preloaded Kernel Arguments
+++++++++++++++++++++++++++
+
+On hardware that supports this feature, kernel arguments can be preloaded into
+User SGPRs, up to the maximum number of User SGPRs available. The allocation of
+Preload SGPRs occurs directly after the last enabled non-kernarg preload User
+SGPR. (See :ref:`amdgpu-amdhsa-initial-kernel-execution-state`)
+
+The data preloaded is copied from the kernarg segment, the amount of data is
+determined by the value specified in the kernarg_preload_spec_length field of
+the kernel descriptor. This data is then loaded into consecutive User SGPRs. The
+number of SGPRs receiving preloaded kernarg data corresponds with the value
+given by kernarg_preload_spec_length. The preloading starts at the dword offset
+within the kernarg segment, which is specified by the
+kernarg_preload_spec_offset field.
+
+If the kernarg_preload_spec_length is non-zero, the CP firmware will append an
+additional 256 bytes to the kernel_code_entry_byte_offset. This addition
+facilitates the incorporation of a prologue to the kernel entry to handle cases
+where code designed for kernarg preloading is executed on hardware equipped with
+incompatible firmware. If hardware has compatible firmware the 256 bytes at the
+start of the kernel entry will be skipped.
+
 .. _amdgpu-amdhsa-kernel-prolog:
 
 Kernel Prolog
@@ -15352,6 +15392,10 @@ terminated by an ``.end_amdhsa_kernel`` directive.
                                                                                                :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx11-table`.
      ``.amdhsa_exception_int_div_zero``                       0                   GFX6-GFX11   Controls ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO in
                                                                                                :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx11-table`.
+     ``.amdhsa_user_sgpr_kernarg_preload_length``             0                   GFX90A,      Controls KERNARG_PRELOAD_SPEC_LENGTH in
+                                                                                  GFX940       :ref:`amdgpu-amdhsa-kernel-descriptor-v3-table`.
+     ``.amdhsa_user_sgpr_kernarg_preload_offset``             0                   GFX90A,      Controls KERNARG_PRELOAD_SPEC_OFFSET in
+                                                                                  GFX940       :ref:`amdgpu-amdhsa-kernel-descriptor-v3-table`.
      ======================================================== =================== ============ ===================
 
 .amdgpu_metadata
diff --git a/llvm/docs/BitCodeFormat.rst b/llvm/docs/BitCodeFormat.rst
index 70be73abef..5742f8594e 100644
--- a/llvm/docs/BitCodeFormat.rst
+++ b/llvm/docs/BitCodeFormat.rst
@@ -1085,6 +1085,15 @@ The integer codes are mapped to well-known attributes as follows.
 * code 77: ``elementtype``
 * code 78: ``disable_sanitizer_instrumentation``
 * code 79: ``nosanitize_bounds``
+* code 80: ``allocalign``
+* code 81: ``allocptr``
+* code 82: ``allockind``
+* code 83: ``presplitcoroutine``
+* code 84: ``fn_ret_thunk_extern``
+* code 85: ``skipprofile``
+* code 86: ``memory``
+* code 87: ``nofpclass``
+* code 88: ``optdebug``
 
 .. note::
   The ``allocsize`` attribute has a special encoding for its arguments. Its two
diff --git a/llvm/docs/CMake.rst b/llvm/docs/CMake.rst
index c70b6b8206..17b76996af 100644
--- a/llvm/docs/CMake.rst
+++ b/llvm/docs/CMake.rst
@@ -428,6 +428,9 @@ enabled sub-projects. Nearly all of these variable names begin with
   $CMAKE_INSTALL_PREFIX/Toolchains containing an xctoolchain directory which can
   be used to override the default system tools.
 
+**LLVM_<target>_LINKER_FLAGS**:STRING
+  Defines the set of linker flags that should be applied to a <target>.
+
 **LLVM_DEFAULT_TARGET_TRIPLE**:STRING
   LLVM target to use for code generation when no target is explicitly specified.
   It defaults to "host", meaning that it shall pick the architecture
@@ -752,6 +755,19 @@ enabled sub-projects. Nearly all of these variable names begin with
 **LLVM_PARALLEL_LINK_JOBS**:STRING
   Define the maximum number of concurrent link jobs.
 
+**LLVM_RAM_PER_COMPILE_JOB**:STRING
+  Calculates the amount of Ninja compile jobs according to available resources.
+  Value has to be in MB, overwrites LLVM_PARALLEL_COMPILE_JOBS. Compile jobs 
+  will be between one and amount of logical cores.
+
+**LLVM_RAM_PER_LINK_JOB**:STRING
+  Calculates the amount of Ninja link jobs according to available resources.
+  Value has to be in MB, overwrites LLVM_PARALLEL_LINK_JOBS. Link jobs will 
+  be between one and amount of logical cores. Link jobs will not run 
+  exclusively therefore you should add an offset of one or two compile jobs 
+  to be sure its not terminated in your memory restricted environment. On ELF
+  platforms also consider ``LLVM_USE_SPLIT_DWARF`` in Debug build.
+
 **LLVM_PROFDATA_FILE**:PATH
   Path to a profdata file to pass into clang's -fprofile-instr-use flag. This
   can only be specified if you're building with clang.
diff --git a/llvm/docs/CodingStandards.rst b/llvm/docs/CodingStandards.rst
index ed7faa8b56..3268cf1a94 100644
--- a/llvm/docs/CodingStandards.rst
+++ b/llvm/docs/CodingStandards.rst
@@ -148,7 +148,7 @@ For example:
 
 .. code-block:: bash
 
-   $ darker -r HEAD^ $(git diff --name-only HEAD^)
+   $ darker -r HEAD^ $(git diff --name-only --diff-filter=d HEAD^)
 
 Mechanical Source Issues
 ========================
diff --git a/llvm/docs/CommandGuide/llvm-remarkutil.rst b/llvm/docs/CommandGuide/llvm-remarkutil.rst
index d13a51707a..6fd739e844 100644
--- a/llvm/docs/CommandGuide/llvm-remarkutil.rst
+++ b/llvm/docs/CommandGuide/llvm-remarkutil.rst
@@ -75,7 +75,7 @@ CSV format is as follows:
   Function,InstructionCount
   foo,123
 
-if `--use-debug-loc` is passed then the CSV will include the source path, line number and column. 
+if `--use-debug-loc` is passed then the CSV will include the source path, line number and column.
 
 ::
   Source,Function,InstructionCount
@@ -104,12 +104,86 @@ CSV format is as follows:
   Function,Count
   foo,123
 
-if `--use-debug-loc` is passed then the CSV will include the source path, line number and column. 
+if `--use-debug-loc` is passed then the CSV will include the source path, line number and column.
 
 ::
   Source,Function,Count
   path:line:column,foo,3
 
+.. _count_subcommand:
+
+count
+~~~~~
+
+..program:: llvm-remarkutil count
+
+USAGE: :program:`llvm-remarkutil` count [*options*] <input file>
+
+Summary
+^^^^^^^
+
+:program:`llvm-remarkutil count` counts `remarks <https://llvm.org/docs/Remarks.html>` based on specified properties.
+By default the tool counts remarks based on how many occour in a source file or function or total for the generated remark file.
+The tool also supports collecting count based on specific remark arguments. The specified arguments should have an integer value to be able to report a count.
+
+The tool contains utilities to filter the remark count based on remark name, pass name, argument value and remark type.
+OPTIONS
+-------
+
+.. option:: --parser=<yaml|bitstream>
+
+  Select the type of input remark parser. Required.
+  * ``yaml``: The tool will parse YAML remarks.
+  * ``bitstream``: The tool will parse bitstream remarks.
+
+.. option:: --count-by<value>
+  Select option to collect remarks by.
+  * ``remark-name``: count how many individual remarks exist.
+  * ``arg``: count remarks based on specified arguments passed by --(r)args. The argument value must be a number.
+
+.. option:: --group-by=<value>
+  group count of remarks by property.
+  * ``source``: Count will be collected per source path. Remarks with no debug location will not be counted.
+  * ``function``: Count is collected per function.
+  * ``function-with-loc``: Count is collected per function per source. Remarks with no debug location will not be counted.
+  * ``Total``: Report a count for the provided remark file.
+
+.. option:: --args[=arguments]
+  If `count-by` is set to `arg` this flag can be used to collect from specified remark arguments represented as a comma seperated string.
+  The arguments must have a numeral value to be able to count remarks by
+
+.. option:: --rargs[=arguments]
+  If `count-by` is set to `arg` this flag can be used to collect from specified remark arguments using regular expression.
+  The arguments must have a numeral value to be able to count remarks by
+
+.. option:: --pass-name[=<string>]
+  Filter count by pass name.
+
+.. option:: --rpass-name[=<string>]
+  Filter count by pass name using regular expressions.
+
+.. option:: --remark-name[=<string>]
+  Filter count by remark name.
+
+.. option:: --rremark-name[=<string>]
+  Filter count by remark name using regular expressions.
+
+.. option:: --filter-arg-by[=<string>]
+  Filter count by argument value.
+
+.. option:: --rfilter-arg-by[=<string>]
+  Filter count by argument value using regular expressions.
+
+.. option:: --remark-type=<value>
+  Filter remarks by type with the following options.
+  * ``unknown``
+  * ``passed``
+  * ``missed``
+  * ``analysis``
+  * ``analysis-fp-commute``
+  * ``analysis-aliasing``
+  * ``failure``
+
 .. _size-diff_subcommand:
 
 size-diff
diff --git a/llvm/docs/GettingInvolved.rst b/llvm/docs/GettingInvolved.rst
index 4ffdcfa2b6..9ace5b7cbb 100644
--- a/llvm/docs/GettingInvolved.rst
+++ b/llvm/docs/GettingInvolved.rst
@@ -318,6 +318,11 @@ don't find anyone present, chances are they happen to be off that day.
     - Monthly, 3rd Wednesday of the month at 8:30am Beijing time, for 30 minutes.
     - `MS Teams <https://teams.microsoft.com/l/meetup-join/19%3ameeting_NWQ0MjU0NjYtZjUyMi00YTU3LThmM2EtY2Y2YTE4NGM3NmFi%40thread.v2/0?context=%7b%22Tid%22%3a%2246c98d88-e344-4ed4-8496-4ed7712e255d%22%2c%22Oid%22%3a%227b309d9c-a9bb-44c8-a940-ab97eef42d4d%22%7d>`__
     - English, Chinese
+  * - Amara Emerson
+    - GlobalISel questions.
+    - Monthly, 4th Wednesday of the month at 9:30am PT, for 30 minutes.
+    - `Google meet <https://meet.google.com/pdd-dibg-cwv>`__
+    - English
 
 
 Guidance for office hours hosts
diff --git a/llvm/docs/GitRepositoryPolicy.md b/llvm/docs/GitRepositoryPolicy.md
index f6442e4e11..f2449251a2 100644
--- a/llvm/docs/GitRepositoryPolicy.md
+++ b/llvm/docs/GitRepositoryPolicy.md
@@ -20,7 +20,7 @@ Requirements for *new* repositories as part of the
 
 If you want to integrate your project as part of the Monorepo, please take a
 look at the
-[Developer Policy](DeveloperPolicy.html#adding-an-established-project-to-the-llvm-monorepo>).
+[Developer Policy](project:DeveloperPolicy.rst#Adding an Established Project To the LLVM Monorepo).
 
 To request a new repository, please create an issue with the
 [Infrastructure Working Group](https://github.com/llvm/llvm-iwg/issues).
diff --git a/llvm/docs/GlobalISel/MIRPatterns.rst b/llvm/docs/GlobalISel/MIRPatterns.rst
index 51d1850a12..fa70311f48 100644
--- a/llvm/docs/GlobalISel/MIRPatterns.rst
+++ b/llvm/docs/GlobalISel/MIRPatterns.rst
@@ -257,8 +257,8 @@ Common Pattern #3: Emitting a Constant Value
 When an immediate operand appears in an 'apply' pattern, the behavior
 depends on whether it's typed or not.
 
-* If the immediate is typed, a ``G_CONSTANT`` is implicitly emitted
-  (= a register operand is added to the instruction).
+* If the immediate is typed, ``MachineIRBuilder::buildConstant`` is used
+  to create a ``G_CONSTANT``. A ``G_BUILD_VECTOR`` will be used for vectors.
 * If the immediate is untyped, a simple immediate is added
   (``MachineInstrBuilder::addImm``).
 
diff --git a/llvm/docs/InstrRefDebugInfo.md b/llvm/docs/InstrRefDebugInfo.md
index e0180ea11e..3917989e40 100644
--- a/llvm/docs/InstrRefDebugInfo.md
+++ b/llvm/docs/InstrRefDebugInfo.md
@@ -191,4 +191,4 @@ instruction number of any cloned instruction, to avoid duplicate numbers
 appearing to `LiveDebugValues`. Dealing with duplicated instructions is a
 natural extension to instruction referencing that's currently unimplemented.
 
-[LiveDebugValues]: SourceLevelDebugging.html#livedebugvalues-expansion-of-variable-locations
+[LiveDebugValues]: project:SourceLevelDebugging.rst#LiveDebugValues expansion of variable locations
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 1883e9f629..64c61dd8c0 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -309,7 +309,7 @@ added in the future:
     prototype of all callees to exactly match the prototype of the
     function definition. Furthermore the inliner doesn't consider such function
     calls for inlining.
-"``cc 10``" - GHC convention
+"``ghccc``" - GHC convention
     This calling convention has been implemented specifically for use by
     the `Glasgow Haskell Compiler (GHC) <http://www.haskell.org/ghc>`_.
     It passes everything in registers, going to extremes to achieve this
@@ -1872,6 +1872,8 @@ example:
     passes make choices that keep the code size of this function as small
     as possible and perform optimizations that may sacrifice runtime
     performance in order to minimize the size of the generated code.
+    This attribute is incompatible with the ``optdebug`` and ``optnone``
+    attributes.
 ``naked``
     This attribute disables prologue / epilogue emission for the
     function. This can have very system-specific consequences.
@@ -2024,6 +2026,12 @@ example:
    Note: Comparing address of a global variable to ``null`` may still
    evaluate to false because of a limitation in querying this attribute inside
    constant expressions.
+``optdebug``
+    This attribute suggests that optimization passes and code generator passes
+    should make choices that try to preserve debug info without significantly
+    degrading runtime performance.
+    This attribute is incompatible with the ``minsize``, ``optsize``, and
+    ``optnone`` attributes.
 ``optforfuzzing``
     This attribute indicates that this function should be optimized
     for maximum fuzzing signal.
@@ -2033,7 +2041,7 @@ example:
     Code generation defaults to the "fast" instruction selector.
     This attribute cannot be used together with the ``alwaysinline``
     attribute; this attribute is also incompatible
-    with the ``minsize`` attribute and the ``optsize`` attribute.
+    with the ``minsize``, ``optsize``, and ``optdebug`` attributes.
 
     This attribute requires the ``noinline`` attribute to be specified on
     the function as well, so the function is never inlined into any caller.
@@ -2044,6 +2052,8 @@ example:
     passes make choices that keep the code size of this function low,
     and otherwise do optimizations specifically to reduce code size as
     long as they do not significantly impact runtime performance.
+    This attribute is incompatible with the ``optdebug`` and ``optnone``
+    attributes.
 ``"patchable-function"``
     This attribute tells the code generator that the code
     generated for this function needs to follow certain conventions that
@@ -3888,7 +3898,7 @@ integer to memory.
 
 A bitcast from a vector type to a scalar integer type will see the elements
 being packed together (without padding). The order in which elements are
-inserted in the integer depends on endianess. For little endian element zero
+inserted in the integer depends on endianness. For little endian element zero
 is put in the least significant bits of the integer, and for big endian
 element zero is put in the most significant bits.
 
@@ -5282,7 +5292,6 @@ X86:
 - ``O``: An immediate integer between 0 and 127.
 - ``e``: An immediate 32-bit signed integer.
 - ``Z``: An immediate 32-bit unsigned integer.
-- ``o``, ``v``: Treated the same as ``m``, at the moment.
 - ``q``: An 8, 16, 32, or 64-bit register which can be accessed as an 8-bit
   ``l`` integer register. On X86-32, this is the ``a``, ``b``, ``c``, and ``d``
   registers, and on X86-64, it is all of the integer registers.
@@ -5293,10 +5302,13 @@ X86:
   existed since i386, and can be accessed without the REX prefix.
 - ``f``: A 32, 64, or 80-bit '387 FPU stack pseudo-register.
 - ``y``: A 64-bit MMX register, if MMX is enabled.
-- ``x``: If SSE is enabled: a 32 or 64-bit scalar operand, or 128-bit vector
+- ``v``: If SSE is enabled: a 32 or 64-bit scalar operand, or 128-bit vector
   operand in a SSE register. If AVX is also enabled, can also be a 256-bit
   vector operand in an AVX register. If AVX-512 is also enabled, can also be a
-  512-bit vector operand in an AVX512 register, Otherwise, an error.
+  512-bit vector operand in an AVX512 register. Otherwise, an error.
+- ``x``: The same as ``v``, except that when AVX-512 is enabled, the ``x`` code
+  only allocates into the first 16 AVX-512 registers, while the ``v`` code
+  allocates into any of the 32 AVX-512 registers.
 - ``Y``: The same as ``x``, if *SSE2* is enabled, otherwise an error.
 - ``A``: Special case: allocates EAX first, then EDX, for a single operand (in
   32-bit mode, a 64-bit integer operand will get split into two registers). It
@@ -11677,7 +11689,7 @@ To convert pointers to other types, use the :ref:`inttoptr <i_inttoptr>`
 or :ref:`ptrtoint <i_ptrtoint>` instructions first.
 
 There is a caveat for bitcasts involving vector types in relation to
-endianess. For example ``bitcast <2 x i8> <value> to i16`` puts element zero
+endianness. For example ``bitcast <2 x i8> <value> to i16`` puts element zero
 of the vector in the least significant bits of the i16 for little-endian while
 element zero ends up in the most significant bits for big-endian.
 
@@ -11686,9 +11698,9 @@ Example:
 
 .. code-block:: text
 
-      %X = bitcast i8 255 to i8          ; yields i8 :-1
-      %Y = bitcast i32* %x to i16*       ; yields i16*:%x
-      %Z = bitcast <2 x i32> %V to i64;  ; yields i64: %V (depends on endianess)
+      %X = bitcast i8 255 to i8         ; yields i8 :-1
+      %Y = bitcast i32* %x to i16*      ; yields i16*:%x
+      %Z = bitcast <2 x i32> %V to i64; ; yields i64: %V (depends on endianness)
       %Z = bitcast <2 x i32*> %V to <2 x i64*> ; yields <2 x i64*>
 
 .. _i_addrspacecast:
@@ -15748,7 +15760,8 @@ Syntax:
 """""""
 
 This is an overloaded intrinsic. You can use ``llvm.lrint`` on any
-floating-point type. Not all targets support all types however.
+floating-point type or vector of floating-point type. Not all targets
+support all types however.
 
 ::
 
@@ -15792,7 +15805,8 @@ Syntax:
 """""""
 
 This is an overloaded intrinsic. You can use ``llvm.llrint`` on any
-floating-point type. Not all targets support all types however.
+floating-point type or vector of floating-point type. Not all targets
+support all types however.
 
 ::
 
@@ -18357,8 +18371,8 @@ For example:
 
 .. code-block:: text
 
- llvm.experimental.vector.splice(<A,B,C,D>, <E,F,G,H>, 1)  ==> <B, C, D, E> ; index
- llvm.experimental.vector.splice(<A,B,C,D>, <E,F,G,H>, -3) ==> <B, C, D, E> ; trailing elements
+ llvm.experimental.vector.splice(<A,B,C,D>, <E,F,G,H>, 1);  ==> <B, C, D, E> index
+ llvm.experimental.vector.splice(<A,B,C,D>, <E,F,G,H>, -3); ==> <B, C, D, E> trailing elements
 
 
 Arguments:
@@ -21517,8 +21531,8 @@ Examples:
 
 .. code-block:: text
 
- llvm.experimental.vp.splice(<A,B,C,D>, <E,F,G,H>, 1, 2, 3)  ==> <B, E, F, poison> ; index
- llvm.experimental.vp.splice(<A,B,C,D>, <E,F,G,H>, -2, 3, 2) ==> <B, C, poison, poison> ; trailing elements
+ llvm.experimental.vp.splice(<A,B,C,D>, <E,F,G,H>, 1, 2, 3);  ==> <B, E, F, poison> index
+ llvm.experimental.vp.splice(<A,B,C,D>, <E,F,G,H>, -2, 3, 2); ==> <B, C, poison, poison> trailing elements
 
 
 .. _int_vp_load:
diff --git a/llvm/docs/MarkdownQuickstartTemplate.md b/llvm/docs/MarkdownQuickstartTemplate.md
index add5985d5b..8b03b4e26d 100644
--- a/llvm/docs/MarkdownQuickstartTemplate.md
+++ b/llvm/docs/MarkdownQuickstartTemplate.md
@@ -158,4 +158,4 @@ integration documentation can be found in the [myst-parser docs].
 
 ## Generating the documentation
 
-see [Sphinx Quickstart Template](SphinxQuickstartTemplate.html#generating-the-documentation)
+see [Sphinx Quickstart Template](project:SphinxQuickstartTemplate.rst#Generating the documentation)
diff --git a/llvm/docs/NewPassManager.rst b/llvm/docs/NewPassManager.rst
index 7464110daa..4554d80043 100644
--- a/llvm/docs/NewPassManager.rst
+++ b/llvm/docs/NewPassManager.rst
@@ -17,6 +17,8 @@ Just Tell Me How To Run The Default Optimization Pipeline With The New Pass Mana
 .. code-block:: c++
 
   // Create the analysis managers.
+  // These must be declared in this order so that they are destroyed in the
+  // correct order due to inter-analysis-manager references.
   LoopAnalysisManager LAM;
   FunctionAnalysisManager FAM;
   CGSCCAnalysisManager CGAM;
diff --git a/llvm/docs/PointerAuth.md b/llvm/docs/PointerAuth.md
index c5f195faaa..41266b43bc 100644
--- a/llvm/docs/PointerAuth.md
+++ b/llvm/docs/PointerAuth.md
@@ -303,8 +303,7 @@ instructions as such:
 
 #### Assembly Representation
 
-At the assembly level,
-[Authenticated Relocations](#authenticated-global-relocation) are represented
+At the assembly level, authenticated relocations are represented
 using the `@AUTH` modifier:
 
 ```asm
@@ -328,8 +327,7 @@ For example:
 
 #### ELF Object File Representation
 
-At the object file level,
-[Authenticated Relocations](#authenticated-global-relocation) are represented
+At the object file level, authenticated relocations are represented
 using the `R_AARCH64_AUTH_ABS64` relocation kind (with value `0xE100`).
 
 The signing schema is encoded in the place of relocation to be applied
diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst
index f0d4b5c5df..bdd250657b 100644
--- a/llvm/docs/ReleaseNotes.rst
+++ b/llvm/docs/ReleaseNotes.rst
@@ -69,6 +69,13 @@ Changes to building LLVM
 Changes to TableGen
 -------------------
 
+* Added constructs for debugging TableGen files:
+
+  * `dump` keyword to dump messages to standard error, see
+     https://github.com/llvm/llvm-project/pull/68793.
+  * `!repr` bang operator to inspect the content of values, see
+     https://github.com/llvm/llvm-project/pull/68716.
+
 Changes to Interprocedural Optimizations
 ----------------------------------------
 
@@ -129,6 +136,14 @@ Changes to the Windows Target
 Changes to the X86 Backend
 --------------------------
 
+* The ``i128`` type now matches GCC and clang's ``__int128`` type. This mainly
+  benefits external projects such as Rust which aim to be binary compatible
+  with C, but also fixes code generation where LLVM already assumed that the
+  type matched and called into libgcc helper functions.
+* Support ISA of ``USER_MSR``.
+* Support ISA of ``AVX10.1-256`` and ``AVX10.1-512``.
+* ``-mcpu=pantherlake`` and ``-mcpu=clearwaterforest`` are now supported.
+
 Changes to the OCaml bindings
 -----------------------------
 
@@ -186,6 +201,17 @@ Changes to LLDB
 
 * Methods in SBHostOS related to threads have had their implementations
   removed. These methods will return a value indicating failure.
+* ``SBType::FindDirectNestedType`` function is added. It's useful
+  for formatters to quickly find directly nested type when it's known
+  where to search for it, avoiding more expensive global search via
+  ``SBTarget::FindFirstType``.
+* ``lldb-vscode`` was renamed to ``lldb-dap`` and and its installation
+  instructions have been updated to reflect this. The underlying functionality
+  remains unchanged.
+
+* LLDB now supports debugging the Scalable Matrix Extension (SME) on AArch64
+  Linux for both running processes and core files. For details refer to the
+  `AArch64 Linux documentation <https://lldb.llvm.org/use/aarch64-linux.html>`_.
 
 Changes to Sanitizers
 ---------------------
diff --git a/llvm/docs/TableGen/ProgRef.rst b/llvm/docs/TableGen/ProgRef.rst
index 92cf5d146f..e5420a05da 100644
--- a/llvm/docs/TableGen/ProgRef.rst
+++ b/llvm/docs/TableGen/ProgRef.rst
@@ -202,10 +202,10 @@ TableGen has the following reserved keywords, which cannot be used as
 identifiers::
 
    assert     bit           bits          class         code
-   dag        def           else          false         foreach
-   defm       defset        defvar        field         if
-   in         include       int           let           list
-   multiclass string        then          true
+   dag        def           dump          else          false
+   foreach    defm          defset        defvar        field
+   if         in            include       int           let
+   list       multiclass    string        then          true
 
 .. warning::
   The ``field`` reserved word is deprecated, except when used with the
@@ -225,10 +225,10 @@ TableGen provides "bang operators" that have a wide variety of uses:
                : !getdagname  !getdagop    !gt          !head        !if
                : !interleave  !isa         !le          !listconcat  !listremove
                : !listsplat   !logtwo      !lt          !mul         !ne
-               : !not         !or          !range       !setdagarg   !setdagname
-               : !setdagop    !shl         !size        !sra         !srl
-               : !strconcat   !sub         !subst       !substr      !tail
-               : !tolower     !toupper     !xor
+               : !not         !or          !range       !repr        !setdagarg
+               : !setdagname  !setdagop    !shl         !size        !sra
+               : !srl         !strconcat   !sub         !subst       !substr
+               : !tail        !tolower     !toupper     !xor
 
 The ``!cond`` operator has a slightly different
 syntax compared to other bang operators, so it is defined separately:
@@ -571,7 +571,7 @@ files.
    TableGenFile: (`Statement` | `IncludeDirective`
             :| `PreprocessorDirective`)*
    Statement: `Assert` | `Class` | `Def` | `Defm` | `Defset` | `Defvar`
-            :| `Foreach` | `If` | `Let` | `MultiClass`
+            :| `Dump`  | `Foreach` | `If` | `Let` | `MultiClass`
 
 The following sections describe each of these top-level statements.
 
@@ -1275,6 +1275,29 @@ be nested.
 This loop defines records named ``R0``, ``R1``, ``R2``, and ``R3``, along
 with ``F0``, ``F1``, ``F2``, and ``F3``.
 
+``dump`` --- print messages to stderr
+-------------------------------------
+
+A ``dump`` statement prints the input string to standard error
+output. It is intended for debugging purpose.
+
+* At top level, the message is printed immediately.
+
+* Within a record/class/multiclass, `dump` gets evaluated at each
+  instantiation point of the containing record.
+
+.. productionlist::
+   Dump: "dump"  `string` ";"
+
+For example, it can be used in combination with `!repr` to investigate
+the values passed to a multiclass:
+
+.. code-block:: text
+
+  multiclass MC<dag s> {
+    dump "s = " # !repr(s);
+  }
+
 
 ``if`` --- select statements based on a test
 --------------------------------------------
@@ -1850,6 +1873,10 @@ and non-0 as true.
 ``!range(``\ *list*\ ``)``
     Equivalent to ``!range(0, !size(list))``.
 
+``!repr(``\ *value*\ ``)``
+    Represents *value* as a string. String format for the value is not
+    guaranteed to be stable. Intended for debugging purposes only.
+
 ``!setdagarg(``\ *dag*\ ``,``\ *key*\ ``,``\ *arg*\ ``)``
     This operator produces a DAG node with the same operator and arguments as
     *dag*, but replacing the value of the argument specified by the *key* with
diff --git a/llvm/docs/TestSuiteGuide.md b/llvm/docs/TestSuiteGuide.md
index e09d7d8803..85623da080 100644
--- a/llvm/docs/TestSuiteGuide.md
+++ b/llvm/docs/TestSuiteGuide.md
@@ -177,7 +177,7 @@ benchmarks. CMake can print a list of them:
 - `TEST_SUITE_FORTRAN`
 
   Activate that Fortran tests. This is a work in progress. More information can be
-  found in the [Flang documentation](https://flang.llvm.org/docs/html/FortranLLVMTestSuite.html)
+  found in the [Flang documentation](https://flang.llvm.org/docs/FortranLLVMTestSuite.html)
 
 - `TEST_SUITE_RUN_UNDER`
 
diff --git a/llvm/docs/WritingAnLLVMPass.rst b/llvm/docs/WritingAnLLVMPass.rst
index 3e6063e583..b94669917c 100644
--- a/llvm/docs/WritingAnLLVMPass.rst
+++ b/llvm/docs/WritingAnLLVMPass.rst
@@ -1020,8 +1020,7 @@ series of passes:
    instead of traversing the entire program.  It reduces the memory consumption
    of compiler, because, for example, only one `DominatorSet
    <https://llvm.org/doxygen/classllvm_1_1DominatorSet.html>`_ needs to be
-   calculated at a time.  This also makes it possible to implement some
-   :ref:`interesting enhancements <writing-an-llvm-pass-SMP>` in the future.
+   calculated at a time.
 
 The effectiveness of the ``PassManager`` is influenced directly by how much
 information it has about the behaviors of the passes it is scheduling.  For
@@ -1350,29 +1349,3 @@ some with solutions, some without.
 Hopefully these tips will help with common case debugging situations.  If you'd
 like to contribute some tips of your own, just contact `Chris
 <mailto:sabre@nondot.org>`_.
-
-Future extensions planned
--------------------------
-
-Although the LLVM Pass Infrastructure is very capable as it stands, and does
-some nifty stuff, there are things we'd like to add in the future.  Here is
-where we are going:
-
-.. _writing-an-llvm-pass-SMP:
-
-Multithreaded LLVM
-^^^^^^^^^^^^^^^^^^
-
-Multiple CPU machines are becoming more common and compilation can never be
-fast enough: obviously we should allow for a multithreaded compiler.  Because
-of the semantics defined for passes above (specifically they cannot maintain
-state across invocations of their ``run*`` methods), a nice clean way to
-implement a multithreaded compiler would be for the ``PassManager`` class to
-create multiple instances of each pass object, and allow the separate instances
-to be hacking on different parts of the program at the same time.
-
-This implementation would prevent each of the passes from having to implement
-multithreaded constructs, requiring only the LLVM core to have locking in a few
-places (for global resources).  Although this is a simple extension, we simply
-haven't had time (or multiprocessor machines, thus a reason) to implement this.
-Despite that, we have kept the LLVM passes SMP ready, and you should too.
diff --git a/llvm/docs/tutorial/MyFirstLanguageFrontend/LangImpl04.rst b/llvm/docs/tutorial/MyFirstLanguageFrontend/LangImpl04.rst
index 79bb1f1c8f..96bccb6440 100644
--- a/llvm/docs/tutorial/MyFirstLanguageFrontend/LangImpl04.rst
+++ b/llvm/docs/tutorial/MyFirstLanguageFrontend/LangImpl04.rst
@@ -94,14 +94,6 @@ use, in the form of "passes".
 LLVM Optimization Passes
 ========================
 
-.. warning::
-
-   Due to the transition to the new PassManager infrastructure this tutorial
-   is based on ``llvm::legacy::FunctionPassManager`` which can be found in
-   `LegacyPassManager.h <https://llvm.org/doxygen/classllvm_1_1legacy_1_1FunctionPassManager.html>`_.
-   For the purpose of the this tutorial the above should be used until
-   the pass manager transition is complete.
-
 LLVM provides many optimization passes, which do many different sorts of
 things and have different tradeoffs. Unlike other systems, LLVM doesn't
 hold to the mistaken notion that one set of optimizations is right for
@@ -127,44 +119,93 @@ in. If we wanted to make a "static Kaleidoscope compiler", we would use
 exactly the code we have now, except that we would defer running the
 optimizer until the entire file has been parsed.
 
+In addition to the distinction between function and module passes, passes can be
+divided into transform and analysis passes. Transform passes mutate the IR, and
+analysis passes compute information that other passes can use. In order to add
+a transform pass, all analysis passes it depends upon must be registered in
+advance.
+
 In order to get per-function optimizations going, we need to set up a
 `FunctionPassManager <../../WritingAnLLVMPass.html#what-passmanager-doesr>`_ to hold
 and organize the LLVM optimizations that we want to run. Once we have
 that, we can add a set of optimizations to run. We'll need a new
 FunctionPassManager for each module that we want to optimize, so we'll
-write a function to create and initialize both the module and pass manager
-for us:
+add to a function created in the previous chapter (``InitializeModule()``):
 
 .. code-block:: c++
 
-    void InitializeModuleAndPassManager(void) {
+    void InitializeModuleAndManagers(void) {
       // Open a new context and module.
-      TheModule = std::make_unique<Module>("my cool jit", *TheContext);
+      TheContext = std::make_unique<LLVMContext>();
+      TheModule = std::make_unique<Module>("KaleidoscopeJIT", *TheContext);
+      TheModule->setDataLayout(TheJIT->getDataLayout());
 
-      // Create a new pass manager attached to it.
-      TheFPM = std::make_unique<legacy::FunctionPassManager>(TheModule.get());
+      // Create a new builder for the module.
+      Builder = std::make_unique<IRBuilder<>>(*TheContext);
+
+      // Create new pass and analysis managers.
+      TheFPM = std::make_unique<FunctionPassManager>();
+      TheFAM = std::make_unique<FunctionAnalysisManager>();
+      TheMAM = std::make_unique<ModuleAnalysisManager>();
+      ThePIC = std::make_unique<PassInstrumentationCallbacks>();
+      TheSI = std::make_unique<StandardInstrumentations>(*TheContext,
+                                                        /*DebugLogging*/ true);
+      TheSI->registerCallbacks(*ThePIC, TheMAM.get());
+      ...
 
+After initializing the global module ``TheModule`` and the FunctionPassManager,
+we need to initialize other parts of the framework. The FunctionAnalysisManager
+and ModuleAnalysisManager allow us to add analysis passes that run across the
+function and the whole module, respectively. PassInstrumentationCallbacks
+and StandardInstrumentations are required for the pass instrumentation
+framework, which allows developers to customize what
+happens between passes.
+
+Once these managers are set up, we use a series of "addPass" calls to add a
+bunch of LLVM transform passes:
+
+.. code-block:: c++
+
+      // Add transform passes.
       // Do simple "peephole" optimizations and bit-twiddling optzns.
-      TheFPM->add(createInstructionCombiningPass());
+      TheFPM->addPass(InstCombinePass());
       // Reassociate expressions.
-      TheFPM->add(createReassociatePass());
+      TheFPM->addPass(ReassociatePass());
       // Eliminate Common SubExpressions.
-      TheFPM->add(createGVNPass());
+      TheFPM->addPass(GVNPass());
       // Simplify the control flow graph (deleting unreachable blocks, etc).
-      TheFPM->add(createCFGSimplificationPass());
-
-      TheFPM->doInitialization();
-    }
-
-This code initializes the global module ``TheModule``, and the function pass
-manager ``TheFPM``, which is attached to ``TheModule``. Once the pass manager is
-set up, we use a series of "add" calls to add a bunch of LLVM passes.
+      TheFPM->addPass(SimplifyCFGPass());
 
 In this case, we choose to add four optimization passes.
 The passes we choose here are a pretty standard set
 of "cleanup" optimizations that are useful for a wide variety of code. I won't
 delve into what they do but, believe me, they are a good starting place :).
 
+Next, we register the analysis passes used by the transform passes. This is
+generally done using ``PassBuilder::register...Analyses()``, but we'll do it
+manually to make clearer what's under the hood.
+
+.. code-block:: c++
+
+      // Register analysis passes used in these transform passes.
+      TheFAM->registerPass([&] { return AAManager(); });
+      TheFAM->registerPass([&] { return AssumptionAnalysis(); });
+      TheFAM->registerPass([&] { return DominatorTreeAnalysis(); });
+      TheFAM->registerPass([&] { return LoopAnalysis(); });
+      TheFAM->registerPass([&] { return MemoryDependenceAnalysis(); });
+      TheFAM->registerPass([&] { return MemorySSAAnalysis(); });
+      TheFAM->registerPass([&] { return OptimizationRemarkEmitterAnalysis(); });
+      TheFAM->registerPass([&] {
+        return OuterAnalysisManagerProxy<ModuleAnalysisManager, Function>(*TheMAM);
+      });
+      TheFAM->registerPass(
+          [&] { return PassInstrumentationAnalysis(ThePIC.get()); });
+      TheFAM->registerPass([&] { return TargetIRAnalysis(); });
+      TheFAM->registerPass([&] { return TargetLibraryAnalysis(); });
+
+      TheMAM->registerPass([&] { return ProfileSummaryAnalysis(); });
+    }
+
 Once the PassManager is set up, we need to make use of it. We do this by
 running it after our newly created function is constructed (in
 ``FunctionAST::codegen()``), but before it is returned to the client:
@@ -179,7 +220,7 @@ running it after our newly created function is constructed (in
         verifyFunction(*TheFunction);
 
         // Optimize the function.
-        TheFPM->run(*TheFunction);
+        TheFPM->run(*TheFunction, *TheFAM);
 
         return TheFunction;
       }
diff --git a/llvm/examples/Kaleidoscope/Chapter4/CMakeLists.txt b/llvm/examples/Kaleidoscope/Chapter4/CMakeLists.txt
index fdc083e076..5281941fec 100644
--- a/llvm/examples/Kaleidoscope/Chapter4/CMakeLists.txt
+++ b/llvm/examples/Kaleidoscope/Chapter4/CMakeLists.txt
@@ -5,6 +5,7 @@ set(LLVM_LINK_COMPONENTS
   InstCombine
   Object
   OrcJIT
+  Passes
   RuntimeDyld
   ScalarOpts
   Support
diff --git a/llvm/examples/Kaleidoscope/Chapter4/toy.cpp b/llvm/examples/Kaleidoscope/Chapter4/toy.cpp
index fb443c7f15..19ec70efd5 100644
--- a/llvm/examples/Kaleidoscope/Chapter4/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter4/toy.cpp
@@ -1,21 +1,32 @@
 #include "../include/KaleidoscopeJIT.h"
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/Analysis/AssumptionCache.h"
+#include "llvm/Analysis/BasicAliasAnalysis.h"
+#include "llvm/Analysis/MemoryDependenceAnalysis.h"
+#include "llvm/Analysis/MemorySSA.h"
+#include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/Analysis/ProfileSummaryInfo.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/LLVMContext.h"
-#include "llvm/IR/LegacyPassManager.h"
 #include "llvm/IR/Module.h"
+#include "llvm/IR/PassManager.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Verifier.h"
+#include "llvm/Passes/PassBuilder.h"
+#include "llvm/Passes/StandardInstrumentations.h"
 #include "llvm/Support/TargetSelect.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Transforms/InstCombine/InstCombine.h"
 #include "llvm/Transforms/Scalar.h"
 #include "llvm/Transforms/Scalar/GVN.h"
+#include "llvm/Transforms/Scalar/Reassociate.h"
+#include "llvm/Transforms/Scalar/SimplifyCFG.h"
 #include <algorithm>
 #include <cassert>
 #include <cctype>
@@ -413,8 +424,12 @@ static std::unique_ptr<LLVMContext> TheContext;
 static std::unique_ptr<Module> TheModule;
 static std::unique_ptr<IRBuilder<>> Builder;
 static std::map<std::string, Value *> NamedValues;
-static std::unique_ptr<legacy::FunctionPassManager> TheFPM;
 static std::unique_ptr<KaleidoscopeJIT> TheJIT;
+static std::unique_ptr<FunctionPassManager> TheFPM;
+static std::unique_ptr<FunctionAnalysisManager> TheFAM;
+static std::unique_ptr<ModuleAnalysisManager> TheMAM;
+static std::unique_ptr<PassInstrumentationCallbacks> ThePIC;
+static std::unique_ptr<StandardInstrumentations> TheSI;
 static std::map<std::string, std::unique_ptr<PrototypeAST>> FunctionProtos;
 static ExitOnError ExitOnErr;
 
@@ -535,7 +550,7 @@ Function *FunctionAST::codegen() {
     verifyFunction(*TheFunction);
 
     // Run the optimizer on the function.
-    TheFPM->run(*TheFunction);
+    TheFPM->run(*TheFunction, *TheFAM);
 
     return TheFunction;
   }
@@ -549,28 +564,51 @@ Function *FunctionAST::codegen() {
 // Top-Level parsing and JIT Driver
 //===----------------------------------------------------------------------===//
 
-static void InitializeModuleAndPassManager() {
+static void InitializeModuleAndManagers() {
   // Open a new context and module.
   TheContext = std::make_unique<LLVMContext>();
-  TheModule = std::make_unique<Module>("my cool jit", *TheContext);
+  TheModule = std::make_unique<Module>("KaleidoscopeJIT", *TheContext);
   TheModule->setDataLayout(TheJIT->getDataLayout());
 
   // Create a new builder for the module.
   Builder = std::make_unique<IRBuilder<>>(*TheContext);
 
-  // Create a new pass manager attached to it.
-  TheFPM = std::make_unique<legacy::FunctionPassManager>(TheModule.get());
+  // Create new pass and analysis managers.
+  TheFPM = std::make_unique<FunctionPassManager>();
+  TheFAM = std::make_unique<FunctionAnalysisManager>();
+  TheMAM = std::make_unique<ModuleAnalysisManager>();
+  ThePIC = std::make_unique<PassInstrumentationCallbacks>();
+  TheSI = std::make_unique<StandardInstrumentations>(*TheContext,
+                                                     /*DebugLogging*/ true);
+  TheSI->registerCallbacks(*ThePIC, TheMAM.get());
 
+  // Add transform passes.
   // Do simple "peephole" optimizations and bit-twiddling optzns.
-  TheFPM->add(createInstructionCombiningPass());
+  TheFPM->addPass(InstCombinePass());
   // Reassociate expressions.
-  TheFPM->add(createReassociatePass());
+  TheFPM->addPass(ReassociatePass());
   // Eliminate Common SubExpressions.
-  TheFPM->add(createGVNPass());
+  TheFPM->addPass(GVNPass());
   // Simplify the control flow graph (deleting unreachable blocks, etc).
-  TheFPM->add(createCFGSimplificationPass());
-
-  TheFPM->doInitialization();
+  TheFPM->addPass(SimplifyCFGPass());
+
+  // Register analysis passes used in these transform passes.
+  TheFAM->registerPass([&] { return AAManager(); });
+  TheFAM->registerPass([&] { return AssumptionAnalysis(); });
+  TheFAM->registerPass([&] { return DominatorTreeAnalysis(); });
+  TheFAM->registerPass([&] { return LoopAnalysis(); });
+  TheFAM->registerPass([&] { return MemoryDependenceAnalysis(); });
+  TheFAM->registerPass([&] { return MemorySSAAnalysis(); });
+  TheFAM->registerPass([&] { return OptimizationRemarkEmitterAnalysis(); });
+  TheFAM->registerPass([&] {
+    return OuterAnalysisManagerProxy<ModuleAnalysisManager, Function>(*TheMAM);
+  });
+  TheFAM->registerPass(
+      [&] { return PassInstrumentationAnalysis(ThePIC.get()); });
+  TheFAM->registerPass([&] { return TargetIRAnalysis(); });
+  TheFAM->registerPass([&] { return TargetLibraryAnalysis(); });
+
+  TheMAM->registerPass([&] { return ProfileSummaryAnalysis(); });
 }
 
 static void HandleDefinition() {
@@ -581,7 +619,7 @@ static void HandleDefinition() {
       fprintf(stderr, "\n");
       ExitOnErr(TheJIT->addModule(
           ThreadSafeModule(std::move(TheModule), std::move(TheContext))));
-      InitializeModuleAndPassManager();
+      InitializeModuleAndManagers();
     }
   } else {
     // Skip token for error recovery.
@@ -613,7 +651,7 @@ static void HandleTopLevelExpression() {
 
       auto TSM = ThreadSafeModule(std::move(TheModule), std::move(TheContext));
       ExitOnErr(TheJIT->addModule(std::move(TSM), RT));
-      InitializeModuleAndPassManager();
+      InitializeModuleAndManagers();
 
       // Search the JIT for the __anon_expr symbol.
       auto ExprSymbol = ExitOnErr(TheJIT->lookup("__anon_expr"));
@@ -699,7 +737,7 @@ int main() {
 
   TheJIT = ExitOnErr(KaleidoscopeJIT::Create());
 
-  InitializeModuleAndPassManager();
+  InitializeModuleAndManagers();
 
   // Run the main "interpreter loop" now.
   MainLoop();
diff --git a/llvm/examples/Kaleidoscope/Chapter5/CMakeLists.txt b/llvm/examples/Kaleidoscope/Chapter5/CMakeLists.txt
index 757d901ef5..35fb1b7911 100644
--- a/llvm/examples/Kaleidoscope/Chapter5/CMakeLists.txt
+++ b/llvm/examples/Kaleidoscope/Chapter5/CMakeLists.txt
@@ -5,6 +5,7 @@ set(LLVM_LINK_COMPONENTS
   InstCombine
   Object
   OrcJIT
+  Passes
   RuntimeDyld
   ScalarOpts
   Support
diff --git a/llvm/examples/Kaleidoscope/Chapter5/toy.cpp b/llvm/examples/Kaleidoscope/Chapter5/toy.cpp
index dc7174aa1c..f41f08de51 100644
--- a/llvm/examples/Kaleidoscope/Chapter5/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter5/toy.cpp
@@ -1,6 +1,13 @@
 #include "../include/KaleidoscopeJIT.h"
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/Analysis/AssumptionCache.h"
+#include "llvm/Analysis/BasicAliasAnalysis.h"
+#include "llvm/Analysis/MemoryDependenceAnalysis.h"
+#include "llvm/Analysis/MemorySSA.h"
+#include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/Analysis/ProfileSummaryInfo.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DerivedTypes.h"
@@ -8,15 +15,19 @@
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/LLVMContext.h"
-#include "llvm/IR/LegacyPassManager.h"
 #include "llvm/IR/Module.h"
+#include "llvm/IR/PassManager.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Verifier.h"
+#include "llvm/Passes/PassBuilder.h"
+#include "llvm/Passes/StandardInstrumentations.h"
 #include "llvm/Support/TargetSelect.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Transforms/InstCombine/InstCombine.h"
 #include "llvm/Transforms/Scalar.h"
 #include "llvm/Transforms/Scalar/GVN.h"
+#include "llvm/Transforms/Scalar/Reassociate.h"
+#include "llvm/Transforms/Scalar/SimplifyCFG.h"
 #include <algorithm>
 #include <cassert>
 #include <cctype>
@@ -540,8 +551,12 @@ static std::unique_ptr<LLVMContext> TheContext;
 static std::unique_ptr<Module> TheModule;
 static std::unique_ptr<IRBuilder<>> Builder;
 static std::map<std::string, Value *> NamedValues;
-static std::unique_ptr<legacy::FunctionPassManager> TheFPM;
 static std::unique_ptr<KaleidoscopeJIT> TheJIT;
+static std::unique_ptr<FunctionPassManager> TheFPM;
+static std::unique_ptr<FunctionAnalysisManager> TheFAM;
+static std::unique_ptr<ModuleAnalysisManager> TheMAM;
+static std::unique_ptr<PassInstrumentationCallbacks> ThePIC;
+static std::unique_ptr<StandardInstrumentations> TheSI;
 static std::map<std::string, std::unique_ptr<PrototypeAST>> FunctionProtos;
 static ExitOnError ExitOnErr;
 
@@ -809,7 +824,7 @@ Function *FunctionAST::codegen() {
     verifyFunction(*TheFunction);
 
     // Run the optimizer on the function.
-    TheFPM->run(*TheFunction);
+    TheFPM->run(*TheFunction, *TheFAM);
 
     return TheFunction;
   }
@@ -823,28 +838,51 @@ Function *FunctionAST::codegen() {
 // Top-Level parsing and JIT Driver
 //===----------------------------------------------------------------------===//
 
-static void InitializeModuleAndPassManager() {
-  // Open a new module.
+static void InitializeModuleAndManagers() {
+  // Open a new context and module.
   TheContext = std::make_unique<LLVMContext>();
-  TheModule = std::make_unique<Module>("my cool jit", *TheContext);
+  TheModule = std::make_unique<Module>("KaleidoscopeJIT", *TheContext);
   TheModule->setDataLayout(TheJIT->getDataLayout());
 
   // Create a new builder for the module.
   Builder = std::make_unique<IRBuilder<>>(*TheContext);
 
-  // Create a new pass manager attached to it.
-  TheFPM = std::make_unique<legacy::FunctionPassManager>(TheModule.get());
+  // Create new pass and analysis managers.
+  TheFPM = std::make_unique<FunctionPassManager>();
+  TheFAM = std::make_unique<FunctionAnalysisManager>();
+  TheMAM = std::make_unique<ModuleAnalysisManager>();
+  ThePIC = std::make_unique<PassInstrumentationCallbacks>();
+  TheSI = std::make_unique<StandardInstrumentations>(*TheContext,
+                                                     /*DebugLogging*/ true);
+  TheSI->registerCallbacks(*ThePIC, TheMAM.get());
 
+  // Add transform passes.
   // Do simple "peephole" optimizations and bit-twiddling optzns.
-  TheFPM->add(createInstructionCombiningPass());
+  TheFPM->addPass(InstCombinePass());
   // Reassociate expressions.
-  TheFPM->add(createReassociatePass());
+  TheFPM->addPass(ReassociatePass());
   // Eliminate Common SubExpressions.
-  TheFPM->add(createGVNPass());
+  TheFPM->addPass(GVNPass());
   // Simplify the control flow graph (deleting unreachable blocks, etc).
-  TheFPM->add(createCFGSimplificationPass());
-
-  TheFPM->doInitialization();
+  TheFPM->addPass(SimplifyCFGPass());
+
+  // Register analysis passes used in these transform passes.
+  TheFAM->registerPass([&] { return AAManager(); });
+  TheFAM->registerPass([&] { return AssumptionAnalysis(); });
+  TheFAM->registerPass([&] { return DominatorTreeAnalysis(); });
+  TheFAM->registerPass([&] { return LoopAnalysis(); });
+  TheFAM->registerPass([&] { return MemoryDependenceAnalysis(); });
+  TheFAM->registerPass([&] { return MemorySSAAnalysis(); });
+  TheFAM->registerPass([&] { return OptimizationRemarkEmitterAnalysis(); });
+  TheFAM->registerPass([&] {
+    return OuterAnalysisManagerProxy<ModuleAnalysisManager, Function>(*TheMAM);
+  });
+  TheFAM->registerPass(
+      [&] { return PassInstrumentationAnalysis(ThePIC.get()); });
+  TheFAM->registerPass([&] { return TargetIRAnalysis(); });
+  TheFAM->registerPass([&] { return TargetLibraryAnalysis(); });
+
+  TheMAM->registerPass([&] { return ProfileSummaryAnalysis(); });
 }
 
 static void HandleDefinition() {
@@ -855,7 +893,7 @@ static void HandleDefinition() {
       fprintf(stderr, "\n");
       ExitOnErr(TheJIT->addModule(
           ThreadSafeModule(std::move(TheModule), std::move(TheContext))));
-      InitializeModuleAndPassManager();
+      InitializeModuleAndManagers();
     }
   } else {
     // Skip token for error recovery.
@@ -887,7 +925,7 @@ static void HandleTopLevelExpression() {
 
       auto TSM = ThreadSafeModule(std::move(TheModule), std::move(TheContext));
       ExitOnErr(TheJIT->addModule(std::move(TSM), RT));
-      InitializeModuleAndPassManager();
+      InitializeModuleAndManagers();
 
       // Search the JIT for the __anon_expr symbol.
       auto ExprSymbol = ExitOnErr(TheJIT->lookup("__anon_expr"));
@@ -973,7 +1011,7 @@ int main() {
 
   TheJIT = ExitOnErr(KaleidoscopeJIT::Create());
 
-  InitializeModuleAndPassManager();
+  InitializeModuleAndManagers();
 
   // Run the main "interpreter loop" now.
   MainLoop();
diff --git a/llvm/examples/Kaleidoscope/Chapter6/CMakeLists.txt b/llvm/examples/Kaleidoscope/Chapter6/CMakeLists.txt
index ad50928a34..627d682bdc 100644
--- a/llvm/examples/Kaleidoscope/Chapter6/CMakeLists.txt
+++ b/llvm/examples/Kaleidoscope/Chapter6/CMakeLists.txt
@@ -5,6 +5,7 @@ set(LLVM_LINK_COMPONENTS
   InstCombine
   Object
   OrcJIT
+  Passes
   RuntimeDyld
   ScalarOpts
   Support
diff --git a/llvm/examples/Kaleidoscope/Chapter6/toy.cpp b/llvm/examples/Kaleidoscope/Chapter6/toy.cpp
index f40eea3c3a..ad275edc68 100644
--- a/llvm/examples/Kaleidoscope/Chapter6/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter6/toy.cpp
@@ -1,6 +1,13 @@
 #include "../include/KaleidoscopeJIT.h"
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/Analysis/AssumptionCache.h"
+#include "llvm/Analysis/BasicAliasAnalysis.h"
+#include "llvm/Analysis/MemoryDependenceAnalysis.h"
+#include "llvm/Analysis/MemorySSA.h"
+#include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/Analysis/ProfileSummaryInfo.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DerivedTypes.h"
@@ -8,15 +15,19 @@
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/LLVMContext.h"
-#include "llvm/IR/LegacyPassManager.h"
 #include "llvm/IR/Module.h"
+#include "llvm/IR/PassManager.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Verifier.h"
+#include "llvm/Passes/PassBuilder.h"
+#include "llvm/Passes/StandardInstrumentations.h"
 #include "llvm/Support/TargetSelect.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Transforms/InstCombine/InstCombine.h"
 #include "llvm/Transforms/Scalar.h"
 #include "llvm/Transforms/Scalar/GVN.h"
+#include "llvm/Transforms/Scalar/Reassociate.h"
+#include "llvm/Transforms/Scalar/SimplifyCFG.h"
 #include <algorithm>
 #include <cassert>
 #include <cctype>
@@ -632,8 +643,12 @@ static std::unique_ptr<LLVMContext> TheContext;
 static std::unique_ptr<Module> TheModule;
 static std::unique_ptr<IRBuilder<>> Builder;
 static std::map<std::string, Value *> NamedValues;
-static std::unique_ptr<legacy::FunctionPassManager> TheFPM;
 static std::unique_ptr<KaleidoscopeJIT> TheJIT;
+static std::unique_ptr<FunctionPassManager> TheFPM;
+static std::unique_ptr<FunctionAnalysisManager> TheFAM;
+static std::unique_ptr<ModuleAnalysisManager> TheMAM;
+static std::unique_ptr<PassInstrumentationCallbacks> ThePIC;
+static std::unique_ptr<StandardInstrumentations> TheSI;
 static std::map<std::string, std::unique_ptr<PrototypeAST>> FunctionProtos;
 static ExitOnError ExitOnErr;
 
@@ -925,7 +940,7 @@ Function *FunctionAST::codegen() {
     verifyFunction(*TheFunction);
 
     // Run the optimizer on the function.
-    TheFPM->run(*TheFunction);
+    TheFPM->run(*TheFunction, *TheFAM);
 
     return TheFunction;
   }
@@ -942,28 +957,51 @@ Function *FunctionAST::codegen() {
 // Top-Level parsing and JIT Driver
 //===----------------------------------------------------------------------===//
 
-static void InitializeModuleAndPassManager() {
-  // Open a new module.
+static void InitializeModuleAndManagers() {
+  // Open a new context and module.
   TheContext = std::make_unique<LLVMContext>();
-  TheModule = std::make_unique<Module>("my cool jit", *TheContext);
+  TheModule = std::make_unique<Module>("KaleidoscopeJIT", *TheContext);
   TheModule->setDataLayout(TheJIT->getDataLayout());
 
   // Create a new builder for the module.
   Builder = std::make_unique<IRBuilder<>>(*TheContext);
 
-  // Create a new pass manager attached to it.
-  TheFPM = std::make_unique<legacy::FunctionPassManager>(TheModule.get());
+  // Create new pass and analysis managers.
+  TheFPM = std::make_unique<FunctionPassManager>();
+  TheFAM = std::make_unique<FunctionAnalysisManager>();
+  TheMAM = std::make_unique<ModuleAnalysisManager>();
+  ThePIC = std::make_unique<PassInstrumentationCallbacks>();
+  TheSI = std::make_unique<StandardInstrumentations>(*TheContext,
+                                                     /*DebugLogging*/ true);
+  TheSI->registerCallbacks(*ThePIC, TheMAM.get());
 
+  // Add transform passes.
   // Do simple "peephole" optimizations and bit-twiddling optzns.
-  TheFPM->add(createInstructionCombiningPass());
+  TheFPM->addPass(InstCombinePass());
   // Reassociate expressions.
-  TheFPM->add(createReassociatePass());
+  TheFPM->addPass(ReassociatePass());
   // Eliminate Common SubExpressions.
-  TheFPM->add(createGVNPass());
+  TheFPM->addPass(GVNPass());
   // Simplify the control flow graph (deleting unreachable blocks, etc).
-  TheFPM->add(createCFGSimplificationPass());
-
-  TheFPM->doInitialization();
+  TheFPM->addPass(SimplifyCFGPass());
+
+  // Register analysis passes used in these transform passes.
+  TheFAM->registerPass([&] { return AAManager(); });
+  TheFAM->registerPass([&] { return AssumptionAnalysis(); });
+  TheFAM->registerPass([&] { return DominatorTreeAnalysis(); });
+  TheFAM->registerPass([&] { return LoopAnalysis(); });
+  TheFAM->registerPass([&] { return MemoryDependenceAnalysis(); });
+  TheFAM->registerPass([&] { return MemorySSAAnalysis(); });
+  TheFAM->registerPass([&] { return OptimizationRemarkEmitterAnalysis(); });
+  TheFAM->registerPass([&] {
+    return OuterAnalysisManagerProxy<ModuleAnalysisManager, Function>(*TheMAM);
+  });
+  TheFAM->registerPass(
+      [&] { return PassInstrumentationAnalysis(ThePIC.get()); });
+  TheFAM->registerPass([&] { return TargetIRAnalysis(); });
+  TheFAM->registerPass([&] { return TargetLibraryAnalysis(); });
+
+  TheMAM->registerPass([&] { return ProfileSummaryAnalysis(); });
 }
 
 static void HandleDefinition() {
@@ -974,7 +1012,7 @@ static void HandleDefinition() {
       fprintf(stderr, "\n");
       ExitOnErr(TheJIT->addModule(
           ThreadSafeModule(std::move(TheModule), std::move(TheContext))));
-      InitializeModuleAndPassManager();
+      InitializeModuleAndManagers();
     }
   } else {
     // Skip token for error recovery.
@@ -1006,7 +1044,7 @@ static void HandleTopLevelExpression() {
 
       auto TSM = ThreadSafeModule(std::move(TheModule), std::move(TheContext));
       ExitOnErr(TheJIT->addModule(std::move(TSM), RT));
-      InitializeModuleAndPassManager();
+      InitializeModuleAndManagers();
 
       // Search the JIT for the __anon_expr symbol.
       auto ExprSymbol = ExitOnErr(TheJIT->lookup("__anon_expr"));
@@ -1092,7 +1130,7 @@ int main() {
 
   TheJIT = ExitOnErr(KaleidoscopeJIT::Create());
 
-  InitializeModuleAndPassManager();
+  InitializeModuleAndManagers();
 
   // Run the main "interpreter loop" now.
   MainLoop();
diff --git a/llvm/examples/Kaleidoscope/Chapter7/CMakeLists.txt b/llvm/examples/Kaleidoscope/Chapter7/CMakeLists.txt
index 03220358ab..f4d8bd9e20 100644
--- a/llvm/examples/Kaleidoscope/Chapter7/CMakeLists.txt
+++ b/llvm/examples/Kaleidoscope/Chapter7/CMakeLists.txt
@@ -5,6 +5,7 @@ set(LLVM_LINK_COMPONENTS
   InstCombine
   Object
   OrcJIT
+  Passes
   RuntimeDyld
   ScalarOpts
   Support
diff --git a/llvm/examples/Kaleidoscope/Chapter7/toy.cpp b/llvm/examples/Kaleidoscope/Chapter7/toy.cpp
index 5bbab8d563..f2954a4cf1 100644
--- a/llvm/examples/Kaleidoscope/Chapter7/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter7/toy.cpp
@@ -1,6 +1,13 @@
 #include "../include/KaleidoscopeJIT.h"
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/Analysis/AssumptionCache.h"
+#include "llvm/Analysis/BasicAliasAnalysis.h"
+#include "llvm/Analysis/MemoryDependenceAnalysis.h"
+#include "llvm/Analysis/MemorySSA.h"
+#include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/Analysis/ProfileSummaryInfo.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DerivedTypes.h"
@@ -8,15 +15,19 @@
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/LLVMContext.h"
-#include "llvm/IR/LegacyPassManager.h"
 #include "llvm/IR/Module.h"
+#include "llvm/IR/PassManager.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Verifier.h"
+#include "llvm/Passes/PassBuilder.h"
+#include "llvm/Passes/StandardInstrumentations.h"
 #include "llvm/Support/TargetSelect.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Transforms/InstCombine/InstCombine.h"
 #include "llvm/Transforms/Scalar.h"
 #include "llvm/Transforms/Scalar/GVN.h"
+#include "llvm/Transforms/Scalar/Reassociate.h"
+#include "llvm/Transforms/Scalar/SimplifyCFG.h"
 #include "llvm/Transforms/Utils.h"
 #include <algorithm>
 #include <cassert>
@@ -705,8 +716,12 @@ static std::unique_ptr<LLVMContext> TheContext;
 static std::unique_ptr<Module> TheModule;
 static std::unique_ptr<IRBuilder<>> Builder;
 static std::map<std::string, AllocaInst *> NamedValues;
-static std::unique_ptr<legacy::FunctionPassManager> TheFPM;
 static std::unique_ptr<KaleidoscopeJIT> TheJIT;
+static std::unique_ptr<FunctionPassManager> TheFPM;
+static std::unique_ptr<FunctionAnalysisManager> TheFAM;
+static std::unique_ptr<ModuleAnalysisManager> TheMAM;
+static std::unique_ptr<PassInstrumentationCallbacks> ThePIC;
+static std::unique_ptr<StandardInstrumentations> TheSI;
 static std::map<std::string, std::unique_ptr<PrototypeAST>> FunctionProtos;
 static ExitOnError ExitOnErr;
 
@@ -1094,7 +1109,7 @@ Function *FunctionAST::codegen() {
     verifyFunction(*TheFunction);
 
     // Run the optimizer on the function.
-    TheFPM->run(*TheFunction);
+    TheFPM->run(*TheFunction, *TheFAM);
 
     return TheFunction;
   }
@@ -1111,30 +1126,51 @@ Function *FunctionAST::codegen() {
 // Top-Level parsing and JIT Driver
 //===----------------------------------------------------------------------===//
 
-static void InitializeModuleAndPassManager() {
-  // Open a new module.
+static void InitializeModuleAndManagers() {
+  // Open a new context and module.
   TheContext = std::make_unique<LLVMContext>();
-  TheModule = std::make_unique<Module>("my cool jit", *TheContext);
+  TheModule = std::make_unique<Module>("KaleidoscopeJIT", *TheContext);
   TheModule->setDataLayout(TheJIT->getDataLayout());
 
   // Create a new builder for the module.
   Builder = std::make_unique<IRBuilder<>>(*TheContext);
 
-  // Create a new pass manager attached to it.
-  TheFPM = std::make_unique<legacy::FunctionPassManager>(TheModule.get());
+  // Create new pass and analysis managers.
+  TheFPM = std::make_unique<FunctionPassManager>();
+  TheFAM = std::make_unique<FunctionAnalysisManager>();
+  TheMAM = std::make_unique<ModuleAnalysisManager>();
+  ThePIC = std::make_unique<PassInstrumentationCallbacks>();
+  TheSI = std::make_unique<StandardInstrumentations>(*TheContext,
+                                                     /*DebugLogging*/ true);
+  TheSI->registerCallbacks(*ThePIC, TheMAM.get());
 
-  // Promote allocas to registers.
-  TheFPM->add(createPromoteMemoryToRegisterPass());
+  // Add transform passes.
   // Do simple "peephole" optimizations and bit-twiddling optzns.
-  TheFPM->add(createInstructionCombiningPass());
+  TheFPM->addPass(InstCombinePass());
   // Reassociate expressions.
-  TheFPM->add(createReassociatePass());
+  TheFPM->addPass(ReassociatePass());
   // Eliminate Common SubExpressions.
-  TheFPM->add(createGVNPass());
+  TheFPM->addPass(GVNPass());
   // Simplify the control flow graph (deleting unreachable blocks, etc).
-  TheFPM->add(createCFGSimplificationPass());
-
-  TheFPM->doInitialization();
+  TheFPM->addPass(SimplifyCFGPass());
+
+  // Register analysis passes used in these transform passes.
+  TheFAM->registerPass([&] { return AAManager(); });
+  TheFAM->registerPass([&] { return AssumptionAnalysis(); });
+  TheFAM->registerPass([&] { return DominatorTreeAnalysis(); });
+  TheFAM->registerPass([&] { return LoopAnalysis(); });
+  TheFAM->registerPass([&] { return MemoryDependenceAnalysis(); });
+  TheFAM->registerPass([&] { return MemorySSAAnalysis(); });
+  TheFAM->registerPass([&] { return OptimizationRemarkEmitterAnalysis(); });
+  TheFAM->registerPass([&] {
+    return OuterAnalysisManagerProxy<ModuleAnalysisManager, Function>(*TheMAM);
+  });
+  TheFAM->registerPass(
+      [&] { return PassInstrumentationAnalysis(ThePIC.get()); });
+  TheFAM->registerPass([&] { return TargetIRAnalysis(); });
+  TheFAM->registerPass([&] { return TargetLibraryAnalysis(); });
+
+  TheMAM->registerPass([&] { return ProfileSummaryAnalysis(); });
 }
 
 static void HandleDefinition() {
@@ -1145,7 +1181,7 @@ static void HandleDefinition() {
       fprintf(stderr, "\n");
       ExitOnErr(TheJIT->addModule(
           ThreadSafeModule(std::move(TheModule), std::move(TheContext))));
-      InitializeModuleAndPassManager();
+      InitializeModuleAndManagers();
     }
   } else {
     // Skip token for error recovery.
@@ -1177,7 +1213,7 @@ static void HandleTopLevelExpression() {
 
       auto TSM = ThreadSafeModule(std::move(TheModule), std::move(TheContext));
       ExitOnErr(TheJIT->addModule(std::move(TSM), RT));
-      InitializeModuleAndPassManager();
+      InitializeModuleAndManagers();
 
       // Search the JIT for the __anon_expr symbol.
       auto ExprSymbol = ExitOnErr(TheJIT->lookup("__anon_expr"));
@@ -1264,7 +1300,7 @@ int main() {
 
   TheJIT = ExitOnErr(KaleidoscopeJIT::Create());
 
-  InitializeModuleAndPassManager();
+  InitializeModuleAndManagers();
 
   // Run the main "interpreter loop" now.
   MainLoop();
diff --git a/llvm/include/llvm/ADT/GenericCycleImpl.h b/llvm/include/llvm/ADT/GenericCycleImpl.h
index 2adec725cd..74faff98b9 100644
--- a/llvm/include/llvm/ADT/GenericCycleImpl.h
+++ b/llvm/include/llvm/ADT/GenericCycleImpl.h
@@ -184,6 +184,24 @@ void GenericCycleInfo<ContextT>::moveTopLevelCycleToNewParent(CycleT *NewParent,
       It.second = NewParent;
 }
 
+template <typename ContextT>
+void GenericCycleInfo<ContextT>::addBlockToCycle(BlockT *Block, CycleT *Cycle) {
+  // FixMe: Appending NewBlock is fine as a set of blocks in a cycle. When
+  // printing, cycle NewBlock is at the end of list but it should be in the
+  // middle to represent actual traversal of a cycle.
+  Cycle->appendBlock(Block);
+  BlockMap.try_emplace(Block, Cycle);
+
+  CycleT *ParentCycle = Cycle->getParentCycle();
+  while (ParentCycle) {
+    Cycle = ParentCycle;
+    Cycle->appendBlock(Block);
+    ParentCycle = Cycle->getParentCycle();
+  }
+
+  BlockMapTopLevel.try_emplace(Block, Cycle);
+}
+
 /// \brief Main function of the cycle info computations.
 template <typename ContextT>
 void GenericCycleInfoCompute<ContextT>::run(BlockT *EntryBlock) {
@@ -368,17 +386,11 @@ void GenericCycleInfo<ContextT>::splitCriticalEdge(BlockT *Pred, BlockT *Succ,
                                                    BlockT *NewBlock) {
   // Edge Pred-Succ is replaced by edges Pred-NewBlock and NewBlock-Succ, all
   // cycles that had blocks Pred and Succ also get NewBlock.
-  CycleT *Cycle = this->getCycle(Pred);
-  if (Cycle && Cycle->contains(Succ)) {
-    while (Cycle) {
-      // FixMe: Appending NewBlock is fine as a set of blocks in a cycle. When
-      // printing cycle NewBlock is at the end of list but it should be in the
-      // middle to represent actual traversal of a cycle.
-      Cycle->appendBlock(NewBlock);
-      BlockMap.try_emplace(NewBlock, Cycle);
-      Cycle = Cycle->getParentCycle();
-    }
-  }
+  CycleT *Cycle = getSmallestCommonCycle(getCycle(Pred), getCycle(Succ));
+  if (!Cycle)
+    return;
+
+  addBlockToCycle(NewBlock, Cycle);
   assert(validateTree());
 }
 
@@ -392,6 +404,35 @@ auto GenericCycleInfo<ContextT>::getCycle(const BlockT *Block) const
   return BlockMap.lookup(Block);
 }
 
+/// \brief Find the innermost cycle containing both given cycles.
+///
+/// \returns the innermost cycle containing both \p A and \p B
+///          or nullptr if there is no such cycle.
+template <typename ContextT>
+auto GenericCycleInfo<ContextT>::getSmallestCommonCycle(CycleT *A,
+                                                        CycleT *B) const
+    -> CycleT * {
+  if (!A || !B)
+    return nullptr;
+
+  // If cycles A and B have different depth replace them with parent cycle
+  // until they have the same depth.
+  while (A->getDepth() > B->getDepth())
+    A = A->getParentCycle();
+  while (B->getDepth() > A->getDepth())
+    B = B->getParentCycle();
+
+  // Cycles A and B are at same depth but may be disjoint, replace them with
+  // parent cycles until we find cycle that contains both or we run out of
+  // parent cycles.
+  while (A != B) {
+    A = A->getParentCycle();
+    B = B->getParentCycle();
+  }
+
+  return A;
+}
+
 /// \brief get the depth for the cycle which containing a given block.
 ///
 /// \returns the depth for the innermost cycle containing \p Block or 0 if it is
diff --git a/llvm/include/llvm/ADT/GenericCycleInfo.h b/llvm/include/llvm/ADT/GenericCycleInfo.h
index 18be7eb4a6..83c4c2759d 100644
--- a/llvm/include/llvm/ADT/GenericCycleInfo.h
+++ b/llvm/include/llvm/ADT/GenericCycleInfo.h
@@ -248,6 +248,12 @@ template <typename ContextT> class GenericCycleInfo {
   /// the subtree.
   void moveTopLevelCycleToNewParent(CycleT *NewParent, CycleT *Child);
 
+  /// Assumes that \p Cycle is the innermost cycle containing \p Block.
+  /// \p Block will be appended to \p Cycle and all of its parent cycles.
+  /// \p Block will be added to BlockMap with \p Cycle and
+  /// BlockMapTopLevel with \p Cycle's top level parent cycle.
+  void addBlockToCycle(BlockT *Block, CycleT *Cycle);
+
 public:
   GenericCycleInfo() = default;
   GenericCycleInfo(GenericCycleInfo &&) = default;
@@ -261,6 +267,7 @@ template <typename ContextT> class GenericCycleInfo {
   const ContextT &getSSAContext() const { return Context; }
 
   CycleT *getCycle(const BlockT *Block) const;
+  CycleT *getSmallestCommonCycle(CycleT *A, CycleT *B) const;
   unsigned getCycleDepth(const BlockT *Block) const;
   CycleT *getTopLevelParentCycle(BlockT *Block);
 
diff --git a/llvm/include/llvm/ADT/GenericUniformityImpl.h b/llvm/include/llvm/ADT/GenericUniformityImpl.h
index ddd0746ccd..b7d0a1228e 100644
--- a/llvm/include/llvm/ADT/GenericUniformityImpl.h
+++ b/llvm/include/llvm/ADT/GenericUniformityImpl.h
@@ -49,8 +49,6 @@
 
 #define DEBUG_TYPE "uniformity"
 
-using namespace llvm;
-
 namespace llvm {
 
 template <typename Range> auto unique(Range &&R) {
diff --git a/llvm/include/llvm/ADT/IndexedMap.h b/llvm/include/llvm/ADT/IndexedMap.h
index 5ac5f79826..b1ebbdd1bf 100644
--- a/llvm/include/llvm/ADT/IndexedMap.h
+++ b/llvm/include/llvm/ADT/IndexedMap.h
@@ -22,6 +22,7 @@
 
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/identity.h"
 #include <cassert>
 
 namespace llvm {
diff --git a/llvm/include/llvm/ADT/STLExtras.h b/llvm/include/llvm/ADT/STLExtras.h
index c7d417324c..d0b79fa91c 100644
--- a/llvm/include/llvm/ADT/STLExtras.h
+++ b/llvm/include/llvm/ADT/STLExtras.h
@@ -21,7 +21,6 @@
 #include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/STLForwardCompat.h"
 #include "llvm/ADT/STLFunctionalExtras.h"
-#include "llvm/ADT/identity.h"
 #include "llvm/ADT/iterator.h"
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/Config/abi-breaking.h"
diff --git a/llvm/include/llvm/CodeGen/StableHashing.h b/llvm/include/llvm/ADT/StableHashing.h
similarity index 95%
rename from llvm/include/llvm/CodeGen/StableHashing.h
rename to llvm/include/llvm/ADT/StableHashing.h
index caf27e152e..884b5752d9 100644
--- a/llvm/include/llvm/CodeGen/StableHashing.h
+++ b/llvm/include/llvm/ADT/StableHashing.h
@@ -1,4 +1,4 @@
-//===- llvm/CodeGen/StableHashing.h - Utilities for stable hashing * C++ *-===//
+//===- llvm/ADT/StableHashing.h - Utilities for stable hashing * C++ *-----===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -12,8 +12,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_CODEGEN_STABLEHASHING_H
-#define LLVM_CODEGEN_STABLEHASHING_H
+#ifndef LLVM_ADT_STABLEHASHING_H
+#define LLVM_ADT_STABLEHASHING_H
 
 #include "llvm/ADT/StringRef.h"
 
diff --git a/llvm/include/llvm/ADT/ilist_iterator.h b/llvm/include/llvm/ADT/ilist_iterator.h
index be87634790..9047b9b739 100644
--- a/llvm/include/llvm/ADT/ilist_iterator.h
+++ b/llvm/include/llvm/ADT/ilist_iterator.h
@@ -175,6 +175,185 @@ class ilist_iterator : ilist_detail::SpecificNodeAccess<OptionsT> {
   bool isEnd() const { return NodePtr ? NodePtr->isSentinel() : false; }
 };
 
+/// Iterator for intrusive lists  based on ilist_node. Much like ilist_iterator,
+/// but with the addition of two bits recording whether this position (when in
+/// a range) is half or fully open.
+template <class OptionsT, bool IsReverse, bool IsConst>
+class ilist_iterator_w_bits : ilist_detail::SpecificNodeAccess<OptionsT> {
+  friend ilist_iterator_w_bits<OptionsT, IsReverse, !IsConst>;
+  friend ilist_iterator_w_bits<OptionsT, !IsReverse, IsConst>;
+  friend ilist_iterator<OptionsT, !IsReverse, !IsConst>;
+
+  using Traits = ilist_detail::IteratorTraits<OptionsT, IsConst>;
+  using Access = ilist_detail::SpecificNodeAccess<OptionsT>;
+
+public:
+  using value_type = typename Traits::value_type;
+  using pointer = typename Traits::pointer;
+  using reference = typename Traits::reference;
+  using difference_type = ptrdiff_t;
+  using iterator_category = std::bidirectional_iterator_tag;
+  using const_pointer = typename OptionsT::const_pointer;
+  using const_reference = typename OptionsT::const_reference;
+
+private:
+  using node_pointer = typename Traits::node_pointer;
+  using node_reference = typename Traits::node_reference;
+
+  node_pointer NodePtr = nullptr;
+
+#ifdef EXPERIMENTAL_DEBUGINFO_ITERATORS
+  // (Default: Off) Allow extra position-information flags to be stored
+  // in iterators, in aid of removing debug-info intrinsics from LLVM.
+
+  /// Is this position intended to contain any debug-info immediately before
+  /// the position?
+  mutable bool HeadInclusiveBit = false;
+  /// Is this position intended to contain any debug-info immediately after
+  /// the position?
+  mutable bool TailInclusiveBit = false;
+#endif
+
+public:
+  /// Create from an ilist_node.
+  explicit ilist_iterator_w_bits(node_reference N) : NodePtr(&N) {}
+
+  explicit ilist_iterator_w_bits(pointer NP)
+      : NodePtr(Access::getNodePtr(NP)) {}
+  explicit ilist_iterator_w_bits(reference NR)
+      : NodePtr(Access::getNodePtr(&NR)) {}
+  ilist_iterator_w_bits() = default;
+
+  // This is templated so that we can allow constructing a const iterator from
+  // a nonconst iterator...
+  template <bool RHSIsConst>
+  ilist_iterator_w_bits(
+      const ilist_iterator_w_bits<OptionsT, IsReverse, RHSIsConst> &RHS,
+      std::enable_if_t<IsConst || !RHSIsConst, void *> = nullptr)
+      : NodePtr(RHS.NodePtr) {
+#ifdef EXPERIMENTAL_DEBUGINFO_ITERATORS
+    HeadInclusiveBit = RHS.HeadInclusiveBit;
+    TailInclusiveBit = RHS.TailInclusiveBit;
+#endif
+  }
+
+  // This is templated so that we can allow assigning to a const iterator from
+  // a nonconst iterator...
+  template <bool RHSIsConst>
+  std::enable_if_t<IsConst || !RHSIsConst, ilist_iterator_w_bits &>
+  operator=(const ilist_iterator_w_bits<OptionsT, IsReverse, RHSIsConst> &RHS) {
+    NodePtr = RHS.NodePtr;
+#ifdef EXPERIMENTAL_DEBUGINFO_ITERATORS
+    HeadInclusiveBit = RHS.HeadInclusiveBit;
+    TailInclusiveBit = RHS.TailInclusiveBit;
+#endif
+    return *this;
+  }
+
+  /// Explicit conversion between forward/reverse iterators.
+  ///
+  /// Translate between forward and reverse iterators without changing range
+  /// boundaries.  The resulting iterator will dereference (and have a handle)
+  /// to the previous node, which is somewhat unexpected; but converting the
+  /// two endpoints in a range will give the same range in reverse.
+  ///
+  /// This matches std::reverse_iterator conversions.
+  explicit ilist_iterator_w_bits(
+      const ilist_iterator_w_bits<OptionsT, !IsReverse, IsConst> &RHS)
+      : ilist_iterator_w_bits(++RHS.getReverse()) {}
+
+  /// Get a reverse iterator to the same node.
+  ///
+  /// Gives a reverse iterator that will dereference (and have a handle) to the
+  /// same node.  Converting the endpoint iterators in a range will give a
+  /// different range; for range operations, use the explicit conversions.
+  ilist_iterator_w_bits<OptionsT, !IsReverse, IsConst> getReverse() const {
+    if (NodePtr)
+      return ilist_iterator_w_bits<OptionsT, !IsReverse, IsConst>(*NodePtr);
+    return ilist_iterator_w_bits<OptionsT, !IsReverse, IsConst>();
+  }
+
+  /// Const-cast.
+  ilist_iterator_w_bits<OptionsT, IsReverse, false> getNonConst() const {
+    if (NodePtr) {
+      auto New = ilist_iterator_w_bits<OptionsT, IsReverse, false>(
+          const_cast<typename ilist_iterator_w_bits<OptionsT, IsReverse,
+                                                    false>::node_reference>(
+              *NodePtr));
+#ifdef EXPERIMENTAL_DEBUGINFO_ITERATORS
+      New.HeadInclusiveBit = HeadInclusiveBit;
+      New.TailInclusiveBit = TailInclusiveBit;
+#endif
+      return New;
+    }
+    return ilist_iterator_w_bits<OptionsT, IsReverse, false>();
+  }
+
+  // Accessors...
+  reference operator*() const {
+    assert(!NodePtr->isKnownSentinel());
+    return *Access::getValuePtr(NodePtr);
+  }
+  pointer operator->() const { return &operator*(); }
+
+  // Comparison operators
+  friend bool operator==(const ilist_iterator_w_bits &LHS,
+                         const ilist_iterator_w_bits &RHS) {
+    return LHS.NodePtr == RHS.NodePtr;
+  }
+  friend bool operator!=(const ilist_iterator_w_bits &LHS,
+                         const ilist_iterator_w_bits &RHS) {
+    return LHS.NodePtr != RHS.NodePtr;
+  }
+
+  // Increment and decrement operators...
+  ilist_iterator_w_bits &operator--() {
+    NodePtr = IsReverse ? NodePtr->getNext() : NodePtr->getPrev();
+#ifdef EXPERIMENTAL_DEBUGINFO_ITERATORS
+    HeadInclusiveBit = false;
+    TailInclusiveBit = false;
+#endif
+    return *this;
+  }
+  ilist_iterator_w_bits &operator++() {
+    NodePtr = IsReverse ? NodePtr->getPrev() : NodePtr->getNext();
+#ifdef EXPERIMENTAL_DEBUGINFO_ITERATORS
+    HeadInclusiveBit = false;
+    TailInclusiveBit = false;
+#endif
+    return *this;
+  }
+  ilist_iterator_w_bits operator--(int) {
+    ilist_iterator_w_bits tmp = *this;
+    --*this;
+    return tmp;
+  }
+  ilist_iterator_w_bits operator++(int) {
+    ilist_iterator_w_bits tmp = *this;
+    ++*this;
+    return tmp;
+  }
+
+  /// Get the underlying ilist_node.
+  node_pointer getNodePtr() const { return static_cast<node_pointer>(NodePtr); }
+
+  /// Check for end.  Only valid if ilist_sentinel_tracking<true>.
+  bool isEnd() const { return NodePtr ? NodePtr->isSentinel() : false; }
+
+#ifdef EXPERIMENTAL_DEBUGINFO_ITERATORS
+  bool getHeadBit() const { return HeadInclusiveBit; }
+  bool getTailBit() const { return TailInclusiveBit; }
+  void setHeadBit(bool SetBit) const { HeadInclusiveBit = SetBit; }
+  void setTailBit(bool SetBit) const { TailInclusiveBit = SetBit; }
+#else
+  // Store and return no information if we're not using this feature.
+  bool getHeadBit() const { return false; }
+  bool getTailBit() const { return false; }
+  void setHeadBit(bool SetBit) const { (void)SetBit; }
+  void setTailBit(bool SetBit) const { (void)SetBit; }
+#endif
+};
+
 template <typename From> struct simplify_type;
 
 /// Allow ilist_iterators to convert into pointers to a node automatically when
@@ -192,6 +371,18 @@ template <class OptionsT, bool IsConst>
 struct simplify_type<const ilist_iterator<OptionsT, false, IsConst>>
     : simplify_type<ilist_iterator<OptionsT, false, IsConst>> {};
 
+// ilist_iterator_w_bits should also be accessible via isa/dyn_cast.
+template <class OptionsT, bool IsConst>
+struct simplify_type<ilist_iterator_w_bits<OptionsT, false, IsConst>> {
+  using iterator = ilist_iterator_w_bits<OptionsT, false, IsConst>;
+  using SimpleType = typename iterator::pointer;
+
+  static SimpleType getSimplifiedValue(const iterator &Node) { return &*Node; }
+};
+template <class OptionsT, bool IsConst>
+struct simplify_type<const ilist_iterator_w_bits<OptionsT, false, IsConst>>
+    : simplify_type<ilist_iterator_w_bits<OptionsT, false, IsConst>> {};
+
 } // end namespace llvm
 
 #endif // LLVM_ADT_ILIST_ITERATOR_H
diff --git a/llvm/include/llvm/ADT/ilist_node.h b/llvm/include/llvm/ADT/ilist_node.h
index 7856b1c0d4..3b6f0dcc7b 100644
--- a/llvm/include/llvm/ADT/ilist_node.h
+++ b/llvm/include/llvm/ADT/ilist_node.h
@@ -27,8 +27,22 @@ struct NodeAccess;
 } // end namespace ilist_detail
 
 template <class OptionsT, bool IsReverse, bool IsConst> class ilist_iterator;
+template <class OptionsT, bool IsReverse, bool IsConst>
+class ilist_iterator_w_bits;
 template <class OptionsT> class ilist_sentinel;
 
+// Selector for which iterator type to pick given the iterator-bits node option.
+template <bool use_iterator_bits, typename Opts, bool arg1, bool arg2>
+class ilist_select_iterator_type {
+public:
+  using type = ilist_iterator<Opts, arg1, arg2>;
+};
+template <typename Opts, bool arg1, bool arg2>
+class ilist_select_iterator_type<true, Opts, arg1, arg2> {
+public:
+  using type = ilist_iterator_w_bits<Opts, arg1, arg2>;
+};
+
 /// Implementation for an ilist node.
 ///
 /// Templated on an appropriate \a ilist_detail::node_options, usually computed
@@ -45,16 +59,29 @@ template <class OptionsT> class ilist_node_impl : OptionsT::node_base_type {
   friend typename OptionsT::list_base_type;
   friend struct ilist_detail::NodeAccess;
   friend class ilist_sentinel<OptionsT>;
+
   friend class ilist_iterator<OptionsT, false, false>;
   friend class ilist_iterator<OptionsT, false, true>;
   friend class ilist_iterator<OptionsT, true, false>;
   friend class ilist_iterator<OptionsT, true, true>;
+  friend class ilist_iterator_w_bits<OptionsT, false, false>;
+  friend class ilist_iterator_w_bits<OptionsT, false, true>;
+  friend class ilist_iterator_w_bits<OptionsT, true, false>;
+  friend class ilist_iterator_w_bits<OptionsT, true, true>;
 
 protected:
-  using self_iterator = ilist_iterator<OptionsT, false, false>;
-  using const_self_iterator = ilist_iterator<OptionsT, false, true>;
-  using reverse_self_iterator = ilist_iterator<OptionsT, true, false>;
-  using const_reverse_self_iterator = ilist_iterator<OptionsT, true, true>;
+  using self_iterator =
+      typename ilist_select_iterator_type<OptionsT::has_iterator_bits, OptionsT,
+                                          false, false>::type;
+  using const_self_iterator =
+      typename ilist_select_iterator_type<OptionsT::has_iterator_bits, OptionsT,
+                                          false, true>::type;
+  using reverse_self_iterator =
+      typename ilist_select_iterator_type<OptionsT::has_iterator_bits, OptionsT,
+                                          true, false>::type;
+  using const_reverse_self_iterator =
+      typename ilist_select_iterator_type<OptionsT::has_iterator_bits, OptionsT,
+                                          true, true>::type;
 
   ilist_node_impl() = default;
 
diff --git a/llvm/include/llvm/ADT/ilist_node_options.h b/llvm/include/llvm/ADT/ilist_node_options.h
index 05340d344e..e6e1068953 100644
--- a/llvm/include/llvm/ADT/ilist_node_options.h
+++ b/llvm/include/llvm/ADT/ilist_node_options.h
@@ -31,6 +31,14 @@ template <bool EnableSentinelTracking> struct ilist_sentinel_tracking {};
 /// simultaneously.  See \a ilist_node for usage examples.
 template <class Tag> struct ilist_tag {};
 
+/// Option to add extra bits to the ilist_iterator.
+///
+/// Some use-cases (debug-info) need to know whether a position is intended
+/// to be half-open or fully open, i.e. whether to include any immediately
+/// adjacent debug-info in an operation. This option adds two bits to the
+/// iterator class to store that information.
+template <bool ExtraIteratorBits> struct ilist_iterator_bits {};
+
 namespace ilist_detail {
 
 /// Helper trait for recording whether an option is specified explicitly.
@@ -91,6 +99,21 @@ template <> struct extract_tag<> {
 };
 template <class Tag> struct is_valid_option<ilist_tag<Tag>> : std::true_type {};
 
+/// Extract iterator bits option.
+///
+/// Look through \p Options for the \a ilist_iterator_bits option. Defaults
+/// to false.
+template <class... Options> struct extract_iterator_bits;
+template <bool IteratorBits, class... Options>
+struct extract_iterator_bits<ilist_iterator_bits<IteratorBits>, Options...>
+    : std::integral_constant<bool, IteratorBits> {};
+template <class Option1, class... Options>
+struct extract_iterator_bits<Option1, Options...>
+    : extract_iterator_bits<Options...> {};
+template <> struct extract_iterator_bits<> : std::false_type, is_implicit {};
+template <bool IteratorBits>
+struct is_valid_option<ilist_iterator_bits<IteratorBits>> : std::true_type {};
+
 /// Check whether options are valid.
 ///
 /// The conjunction of \a is_valid_option on each individual option.
@@ -105,7 +128,7 @@ struct check_options<Option1, Options...>
 ///
 /// This is usually computed via \a compute_node_options.
 template <class T, bool EnableSentinelTracking, bool IsSentinelTrackingExplicit,
-          class TagT>
+          class TagT, bool HasIteratorBits>
 struct node_options {
   typedef T value_type;
   typedef T *pointer;
@@ -115,6 +138,7 @@ struct node_options {
 
   static const bool enable_sentinel_tracking = EnableSentinelTracking;
   static const bool is_sentinel_tracking_explicit = IsSentinelTrackingExplicit;
+  static const bool has_iterator_bits = HasIteratorBits;
   typedef TagT tag;
   typedef ilist_node_base<enable_sentinel_tracking> node_base_type;
   typedef ilist_base<enable_sentinel_tracking> list_base_type;
@@ -123,7 +147,8 @@ struct node_options {
 template <class T, class... Options> struct compute_node_options {
   typedef node_options<T, extract_sentinel_tracking<Options...>::value,
                        extract_sentinel_tracking<Options...>::is_explicit,
-                       typename extract_tag<Options...>::type>
+                       typename extract_tag<Options...>::type,
+                       extract_iterator_bits<Options...>::value>
       type;
 };
 
diff --git a/llvm/include/llvm/ADT/simple_ilist.h b/llvm/include/llvm/ADT/simple_ilist.h
index 3a96e1ba56..7236b3fa5a 100644
--- a/llvm/include/llvm/ADT/simple_ilist.h
+++ b/llvm/include/llvm/ADT/simple_ilist.h
@@ -92,10 +92,18 @@ class simple_ilist
   using reference = typename OptionsT::reference;
   using const_pointer = typename OptionsT::const_pointer;
   using const_reference = typename OptionsT::const_reference;
-  using iterator = ilist_iterator<OptionsT, false, false>;
-  using const_iterator = ilist_iterator<OptionsT, false, true>;
-  using reverse_iterator = ilist_iterator<OptionsT, true, false>;
-  using const_reverse_iterator = ilist_iterator<OptionsT, true, true>;
+  using iterator =
+      typename ilist_select_iterator_type<OptionsT::has_iterator_bits, OptionsT,
+                                          false, false>::type;
+  using const_iterator =
+      typename ilist_select_iterator_type<OptionsT::has_iterator_bits, OptionsT,
+                                          false, true>::type;
+  using reverse_iterator =
+      typename ilist_select_iterator_type<OptionsT::has_iterator_bits, OptionsT,
+                                          true, false>::type;
+  using const_reverse_iterator =
+      typename ilist_select_iterator_type<OptionsT::has_iterator_bits, OptionsT,
+                                          true, true>::type;
   using size_type = size_t;
   using difference_type = ptrdiff_t;
 
diff --git a/llvm/include/llvm/Analysis/AliasAnalysis.h b/llvm/include/llvm/Analysis/AliasAnalysis.h
index 4f06ae1d38..a69c4adf28 100644
--- a/llvm/include/llvm/Analysis/AliasAnalysis.h
+++ b/llvm/include/llvm/Analysis/AliasAnalysis.h
@@ -172,7 +172,7 @@ class SimpleCaptureInfo final : public CaptureInfo {
 /// approximation to a precise "captures before" analysis.
 class EarliestEscapeInfo final : public CaptureInfo {
   DominatorTree &DT;
-  const LoopInfo &LI;
+  const LoopInfo *LI;
 
   /// Map from identified local object to an instruction before which it does
   /// not escape, or nullptr if it never escapes. The "earliest" instruction
@@ -184,11 +184,11 @@ class EarliestEscapeInfo final : public CaptureInfo {
   /// This is used for cache invalidation purposes.
   DenseMap<Instruction *, TinyPtrVector<const Value *>> Inst2Obj;
 
-  const SmallPtrSetImpl<const Value *> &EphValues;
+  const SmallPtrSetImpl<const Value *> *EphValues;
 
 public:
-  EarliestEscapeInfo(DominatorTree &DT, const LoopInfo &LI,
-                     const SmallPtrSetImpl<const Value *> &EphValues)
+  EarliestEscapeInfo(DominatorTree &DT, const LoopInfo *LI = nullptr,
+                     const SmallPtrSetImpl<const Value *> *EphValues = nullptr)
       : DT(DT), LI(LI), EphValues(EphValues) {}
 
   bool isNotCapturedBeforeOrAt(const Value *Object,
diff --git a/llvm/include/llvm/Analysis/CaptureTracking.h b/llvm/include/llvm/Analysis/CaptureTracking.h
index a2d9277745..6589c2b863 100644
--- a/llvm/include/llvm/Analysis/CaptureTracking.h
+++ b/llvm/include/llvm/Analysis/CaptureTracking.h
@@ -83,7 +83,7 @@ namespace llvm {
   Instruction *
   FindEarliestCapture(const Value *V, Function &F, bool ReturnCaptures,
                       bool StoreCaptures, const DominatorTree &DT,
-                      const SmallPtrSetImpl<const Value *> &EphValues,
+                      const SmallPtrSetImpl<const Value *> *EphValues = nullptr,
                       unsigned MaxUsesToExplore = 0);
 
   /// This callback is used in conjunction with PointerMayBeCaptured. In
diff --git a/llvm/include/llvm/Analysis/InlineAdvisor.h b/llvm/include/llvm/Analysis/InlineAdvisor.h
index 53c018d15c..2740106bc7 100644
--- a/llvm/include/llvm/Analysis/InlineAdvisor.h
+++ b/llvm/include/llvm/Analysis/InlineAdvisor.h
@@ -36,9 +36,8 @@ struct ReplayInlinerSettings;
 ///
 /// - Development mode, for training new models.
 /// In this mode, we trade off runtime performance for flexibility. This mode
-/// requires the full C Tensorflow API library, and evaluates models
-/// dynamically. This mode also permits generating training logs, for offline
-/// training.
+/// requires the TFLite library, and evaluates models dynamically. This mode
+/// also permits generating training logs, for offline training.
 ///
 /// - Dynamically load an advisor via a plugin (PluginInlineAdvisorAnalysis)
 enum class InliningAdvisorMode : int { Default, Release, Development };
diff --git a/llvm/include/llvm/Analysis/InlineModelFeatureMaps.h b/llvm/include/llvm/Analysis/InlineModelFeatureMaps.h
index 77ae60059c..06925e620b 100644
--- a/llvm/include/llvm/Analysis/InlineModelFeatureMaps.h
+++ b/llvm/include/llvm/Analysis/InlineModelFeatureMaps.h
@@ -92,8 +92,8 @@ constexpr bool isHeuristicInlineCostFeature(InlineCostFeatureIndex Feature) {
 
 // List of features. Each feature is defined through a triple:
 // - the name of an enum member, which will be the feature index
-// - a textual name, used for Tensorflow model binding (so it needs to match the
-// names used by the Tensorflow model)
+// - a textual name, used for ML model binding (so it needs to match the
+// names used by the ML model).
 // - a documentation description. Currently, that is not used anywhere
 // programmatically, and serves as workaround to inability of inserting comments
 // in macros.
diff --git a/llvm/include/llvm/Analysis/MLModelRunner.h b/llvm/include/llvm/Analysis/MLModelRunner.h
index 903411fbdf..21f155de85 100644
--- a/llvm/include/llvm/Analysis/MLModelRunner.h
+++ b/llvm/include/llvm/Analysis/MLModelRunner.h
@@ -17,7 +17,9 @@ namespace llvm {
 class LLVMContext;
 
 /// MLModelRunner interface: abstraction of a mechanism for evaluating a
-/// tensorflow "saved model".
+/// ML model. More abstractly, evaluating a function that has as tensors as
+/// arguments, described via TensorSpecs, and returns a tensor. Currently, the
+/// latter is assumed to be a scalar, in absence of more elaborate scenarios.
 /// NOTE: feature indices are expected to be consistent all accross
 /// MLModelRunners (pertaining to the same model), and also Loggers (see
 /// TFUtils.h)
diff --git a/llvm/include/llvm/Analysis/ModelUnderTrainingRunner.h b/llvm/include/llvm/Analysis/ModelUnderTrainingRunner.h
index 7e29bbd2be..1a79f2c36c 100644
--- a/llvm/include/llvm/Analysis/ModelUnderTrainingRunner.h
+++ b/llvm/include/llvm/Analysis/ModelUnderTrainingRunner.h
@@ -23,9 +23,10 @@
 
 namespace llvm {
 
-/// ModelUnderTrainingRunner - training mode implementation. It uses TF C APIs
+/// ModelUnderTrainingRunner - training mode implementation. It uses TFLite
 /// to dynamically load and evaluate a TF SavedModel
-/// (https://www.tensorflow.org/guide/saved_model). Runtime performance is
+/// (https://www.tensorflow.org/guide/saved_model) converted to TFLite. see
+/// lib/Analysis/models/saved-model-to-tflite.py. Runtime performance is
 /// sacrificed for ease of use while training.
 class ModelUnderTrainingRunner final : public MLModelRunner {
 public:
diff --git a/llvm/include/llvm/Analysis/TargetLibraryInfo.def b/llvm/include/llvm/Analysis/TargetLibraryInfo.def
index 03ac422d3e..6bd922eed8 100644
--- a/llvm/include/llvm/Analysis/TargetLibraryInfo.def
+++ b/llvm/include/llvm/Analysis/TargetLibraryInfo.def
@@ -1570,7 +1570,7 @@ TLI_DEFINE_SIG_INTERNAL(Int, Int)
 /// long int labs(long int j);
 TLI_DEFINE_ENUM_INTERNAL(labs)
 TLI_DEFINE_STRING_INTERNAL("labs")
-TLI_DEFINE_SIG_INTERNAL(Long, Long)
+TLI_DEFINE_SIG_INTERNAL(Long, Same)
 
 /// int lchown(const char *path, uid_t owner, gid_t group);
 TLI_DEFINE_ENUM_INTERNAL(lchown)
diff --git a/llvm/include/llvm/Analysis/TensorSpec.h b/llvm/include/llvm/Analysis/TensorSpec.h
index c50507b7a6..d6b23cf56f 100644
--- a/llvm/include/llvm/Analysis/TensorSpec.h
+++ b/llvm/include/llvm/Analysis/TensorSpec.h
@@ -26,11 +26,15 @@ namespace llvm {
 /// Machine Learning on Heterogeneous Distributed Systems", section 4.2, para 2:
 /// https://static.googleusercontent.com/media/research.google.com/en//pubs/archive/45166.pdf)
 ///
-/// Known tensor types. The left part is the C type, the right is a name we
-/// can use to identify the type (to implement TensorSpec equality checks), and
-/// to use, if needed, when mapping to an underlying evaluator's type system.
-/// The main requirement is that the C type we use has the same size and
-/// encoding (e.g. endian-ness) as the one used by the evaluator.
+/// Note that the design is motivated by Tensorflow, but it is not intended to
+/// be Tensorflow-specific.
+///
+/// Known tensor types. The left part is the C type, the
+/// right is a name we can use to identify the type (to implement TensorSpec
+/// equality checks), and to use, if needed, when mapping to an underlying
+/// evaluator's type system. The main requirement is that the C type we use has
+/// the same size and encoding (e.g. endian-ness) as the one used by the
+/// evaluator.
 #define SUPPORTED_TENSOR_TYPES(M)                                              \
   M(float, Float)                                                              \
   M(double, Double)                                                            \
diff --git a/llvm/include/llvm/Analysis/Utils/TFUtils.h b/llvm/include/llvm/Analysis/Utils/TFUtils.h
index 04bb8af3a5..817702b869 100644
--- a/llvm/include/llvm/Analysis/Utils/TFUtils.h
+++ b/llvm/include/llvm/Analysis/Utils/TFUtils.h
@@ -1,4 +1,4 @@
-//===- TFUtils.h - utilities for tensorflow C API ---------------*- C++ -*-===//
+//===- TFUtils.h - utilities for TFLite -------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -93,7 +93,7 @@ class TFModelEvaluator final {
     return static_cast<T *>(getUntypedInput(Index));
   }
 
-  /// Returns true if the tensorflow model was loaded successfully, false
+  /// Returns true if the model was loaded successfully, false
   /// otherwise.
   bool isValid() const { return !!Impl; }
 
diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h
index d970ffee5d..0e02d0d5b4 100644
--- a/llvm/include/llvm/Analysis/ValueTracking.h
+++ b/llvm/include/llvm/Analysis/ValueTracking.h
@@ -17,6 +17,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/Analysis/SimplifyQuery.h"
+#include "llvm/Analysis/WithCache.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/FMF.h"
@@ -90,6 +91,12 @@ KnownBits computeKnownBits(const Value *V, const APInt &DemandedElts,
                            const DominatorTree *DT = nullptr,
                            bool UseInstrInfo = true);
 
+KnownBits computeKnownBits(const Value *V, const APInt &DemandedElts,
+                           unsigned Depth, const SimplifyQuery &Q);
+
+KnownBits computeKnownBits(const Value *V, unsigned Depth,
+                           const SimplifyQuery &Q);
+
 /// Compute known bits from the range metadata.
 /// \p KnownZero the set of bits that are known to be zero
 /// \p KnownOne the set of bits that are known to be one
@@ -107,7 +114,8 @@ KnownBits analyzeKnownBitsFromAndXorOr(
     bool UseInstrInfo = true);
 
 /// Return true if LHS and RHS have no common bits set.
-bool haveNoCommonBitsSet(const Value *LHS, const Value *RHS,
+bool haveNoCommonBitsSet(const WithCache<const Value *> &LHSCache,
+                         const WithCache<const Value *> &RHSCache,
                          const SimplifyQuery &SQ);
 
 /// Return true if the given value is known to have exactly one bit set when
@@ -240,10 +248,6 @@ struct KnownFPClass {
   /// definitely set or false if the sign bit is definitely unset.
   std::optional<bool> SignBit;
 
-  bool operator==(KnownFPClass Other) const {
-    return KnownFPClasses == Other.KnownFPClasses && SignBit == Other.SignBit;
-  }
-
   /// Return true if it's known this can never be one of the mask entries.
   bool isKnownNever(FPClassTest Mask) const {
     return (KnownFPClasses & Mask) == fcNone;
@@ -851,9 +855,12 @@ OverflowResult computeOverflowForUnsignedMul(const Value *LHS, const Value *RHS,
                                              const SimplifyQuery &SQ);
 OverflowResult computeOverflowForSignedMul(const Value *LHS, const Value *RHS,
                                            const SimplifyQuery &SQ);
-OverflowResult computeOverflowForUnsignedAdd(const Value *LHS, const Value *RHS,
-                                             const SimplifyQuery &SQ);
-OverflowResult computeOverflowForSignedAdd(const Value *LHS, const Value *RHS,
+OverflowResult
+computeOverflowForUnsignedAdd(const WithCache<const Value *> &LHS,
+                              const WithCache<const Value *> &RHS,
+                              const SimplifyQuery &SQ);
+OverflowResult computeOverflowForSignedAdd(const WithCache<const Value *> &LHS,
+                                           const WithCache<const Value *> &RHS,
                                            const SimplifyQuery &SQ);
 /// This version also leverages the sign bit of Add if known.
 OverflowResult computeOverflowForSignedAdd(const AddOperator *Add,
diff --git a/llvm/include/llvm/Analysis/WithCache.h b/llvm/include/llvm/Analysis/WithCache.h
new file mode 100644
index 0000000000..7bd680bf09
--- /dev/null
+++ b/llvm/include/llvm/Analysis/WithCache.h
@@ -0,0 +1,72 @@
+//===- llvm/Analysis/WithCache.h - KnownBits cache for pointers -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Store a pointer to any type along with the KnownBits information for it
+// that is computed lazily (if required).
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ANALYSIS_WITHCACHE_H
+#define LLVM_ANALYSIS_WITHCACHE_H
+
+#include "llvm/ADT/PointerIntPair.h"
+#include "llvm/IR/Value.h"
+#include "llvm/Support/KnownBits.h"
+#include <type_traits>
+
+namespace llvm {
+struct SimplifyQuery;
+KnownBits computeKnownBits(const Value *V, unsigned Depth,
+                           const SimplifyQuery &Q);
+
+template <typename Arg> class WithCache {
+  static_assert(std::is_pointer_v<Arg>, "WithCache requires a pointer type!");
+
+  using UnderlyingType = std::remove_pointer_t<Arg>;
+  constexpr static bool IsConst = std::is_const_v<Arg>;
+
+  template <typename T, bool Const>
+  using conditionally_const_t = std::conditional_t<Const, const T, T>;
+
+  using PointerType = conditionally_const_t<UnderlyingType *, IsConst>;
+  using ReferenceType = conditionally_const_t<UnderlyingType &, IsConst>;
+
+  // Store the presence of the KnownBits information in one of the bits of
+  // Pointer.
+  // true  -> present
+  // false -> absent
+  mutable PointerIntPair<PointerType, 1, bool> Pointer;
+  mutable KnownBits Known;
+
+  void calculateKnownBits(const SimplifyQuery &Q) const {
+    Known = computeKnownBits(Pointer.getPointer(), 0, Q);
+    Pointer.setInt(true);
+  }
+
+public:
+  WithCache(PointerType Pointer) : Pointer(Pointer, false) {}
+  WithCache(PointerType Pointer, const KnownBits &Known)
+      : Pointer(Pointer, true), Known(Known) {}
+
+  [[nodiscard]] PointerType getValue() const { return Pointer.getPointer(); }
+
+  [[nodiscard]] const KnownBits &getKnownBits(const SimplifyQuery &Q) const {
+    if (!hasKnownBits())
+      calculateKnownBits(Q);
+    return Known;
+  }
+
+  [[nodiscard]] bool hasKnownBits() const { return Pointer.getInt(); }
+
+  operator PointerType() const { return Pointer.getPointer(); }
+  PointerType operator->() const { return Pointer.getPointer(); }
+  ReferenceType operator*() const { return *Pointer.getPointer(); }
+};
+} // namespace llvm
+
+#endif
diff --git a/llvm/include/llvm/AsmParser/LLToken.h b/llvm/include/llvm/AsmParser/LLToken.h
index 673dc58ce6..2d6b8a1940 100644
--- a/llvm/include/llvm/AsmParser/LLToken.h
+++ b/llvm/include/llvm/AsmParser/LLToken.h
@@ -175,6 +175,7 @@ enum Kind {
   kw_amdgpu_kernel,
   kw_amdgpu_gfx,
   kw_tailcc,
+  kw_m68k_rtdcc,
 
   // Attributes:
   kw_attributes,
diff --git a/llvm/include/llvm/BinaryFormat/Dwarf.def b/llvm/include/llvm/BinaryFormat/Dwarf.def
index 40d958c867..fb328a0257 100644
--- a/llvm/include/llvm/BinaryFormat/Dwarf.def
+++ b/llvm/include/llvm/BinaryFormat/Dwarf.def
@@ -1036,6 +1036,7 @@ HANDLE_DW_CC(0xc8, LLVM_Swift)
 HANDLE_DW_CC(0xc9, LLVM_PreserveMost)
 HANDLE_DW_CC(0xca, LLVM_PreserveAll)
 HANDLE_DW_CC(0xcb, LLVM_X86RegCall)
+HANDLE_DW_CC(0xcc, LLVM_M68kRTD)
 // From GCC source code (include/dwarf2.h): This DW_CC_ value is not currently
 // generated by any toolchain.  It is used internally to GDB to indicate OpenCL
 // C functions that have been compiled with the IBM XL C for OpenCL compiler and
diff --git a/llvm/include/llvm/BinaryFormat/ELF.h b/llvm/include/llvm/BinaryFormat/ELF.h
index f5a7cdb387..3596174f74 100644
--- a/llvm/include/llvm/BinaryFormat/ELF.h
+++ b/llvm/include/llvm/BinaryFormat/ELF.h
@@ -1611,6 +1611,7 @@ enum : unsigned {
   NT_ARM_HW_WATCH = 0x403,
   NT_ARM_SVE = 0x405,
   NT_ARM_PAC_MASK = 0x406,
+  NT_ARM_TAGGED_ADDR_CTRL = 0x409,
   NT_ARM_SSVE = 0x40b,
   NT_ARM_ZA = 0x40c,
   NT_ARM_ZT = 0x40d,
diff --git a/llvm/include/llvm/BinaryFormat/MachO.def b/llvm/include/llvm/BinaryFormat/MachO.def
index d841b42ee8..df527b4a8a 100644
--- a/llvm/include/llvm/BinaryFormat/MachO.def
+++ b/llvm/include/llvm/BinaryFormat/MachO.def
@@ -120,5 +120,21 @@ LOAD_COMMAND_STRUCT(fileset_entry_command)
 
 #endif
 
+#ifdef PLATFORM
+// PLATFORM(platform, id, name, build_name, target, tapi_target, marketing)
+PLATFORM(UNKNOWN, 0, unknown, unknown, unknown, unknown, unknown)
+PLATFORM(MACOS, 1, macos, macos, macos, macos, macOS)
+PLATFORM(IOS, 2, ios, ios, ios, ios, iOS)
+PLATFORM(TVOS, 3, tvos, tvos, tvos, tvos, tvOS)
+PLATFORM(WATCHOS, 4, watchos, watchos, watchos, watchos, watchOS)
+PLATFORM(BRIDGEOS, 5, bridgeos, bridgeos, bridgeos, bridgeos, bridgeOS)
+PLATFORM(MACCATALYST, 6, macCatalyst, macCatalyst, ios-macabi, maccatalyst, macCatalyst)
+PLATFORM(IOSSIMULATOR, 7, iossimulator, iossimulator, ios-simulator, ios-simulator, iOS Simulator)
+PLATFORM(TVOSSIMULATOR, 8, tvossimulator, tvossimulator, tvos-simulator, tvos-simulator, tvOS Simulator)
+PLATFORM(WATCHOSSIMULATOR, 9, watchossimulator, watchossimulator, watchos-simulator, watchos-simulator, watchOS Simulator)
+PLATFORM(DRIVERKIT, 10, driverkit, driverkit, driverkit, driverkit, DriverKit)
+#endif
+
 #undef HANDLE_LOAD_COMMAND
 #undef LOAD_COMMAND_STRUCT
+#undef PLATFORM
diff --git a/llvm/include/llvm/BinaryFormat/MachO.h b/llvm/include/llvm/BinaryFormat/MachO.h
index f59cd14c1b..49991ebe7b 100644
--- a/llvm/include/llvm/BinaryFormat/MachO.h
+++ b/llvm/include/llvm/BinaryFormat/MachO.h
@@ -497,17 +497,10 @@ enum { VM_PROT_READ = 0x1, VM_PROT_WRITE = 0x2, VM_PROT_EXECUTE = 0x4 };
 
 // Values for platform field in build_version_command.
 enum PlatformType {
-  PLATFORM_UNKNOWN = 0,
-  PLATFORM_MACOS = 1,
-  PLATFORM_IOS = 2,
-  PLATFORM_TVOS = 3,
-  PLATFORM_WATCHOS = 4,
-  PLATFORM_BRIDGEOS = 5,
-  PLATFORM_MACCATALYST = 6,
-  PLATFORM_IOSSIMULATOR = 7,
-  PLATFORM_TVOSSIMULATOR = 8,
-  PLATFORM_WATCHOSSIMULATOR = 9,
-  PLATFORM_DRIVERKIT = 10,
+#define PLATFORM(platform, id, name, build_name, target, tapi_target,          \
+                 marketing)                                                    \
+  PLATFORM_##platform = id,
+#include "MachO.def"
 };
 
 // Values for tools enum in build_tool_version.
diff --git a/llvm/include/llvm/BinaryFormat/MsgPack.h b/llvm/include/llvm/BinaryFormat/MsgPack.h
index 9fda14b21c..01edae0ee0 100644
--- a/llvm/include/llvm/BinaryFormat/MsgPack.h
+++ b/llvm/include/llvm/BinaryFormat/MsgPack.h
@@ -21,7 +21,7 @@ namespace llvm {
 namespace msgpack {
 
 /// The endianness of all multi-byte encoded values in MessagePack.
-constexpr support::endianness Endianness = support::big;
+constexpr llvm::endianness Endianness = llvm::endianness::big;
 
 /// The first byte identifiers of MessagePack object formats.
 namespace FirstByte {
diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
index 52e76356a8..5d7be5ca93 100644
--- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h
+++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
@@ -713,6 +713,7 @@ enum AttributeKindCodes {
   ATTR_KIND_SKIP_PROFILE = 85,
   ATTR_KIND_MEMORY = 86,
   ATTR_KIND_NOFPCLASS = 87,
+  ATTR_KIND_OPTIMIZE_FOR_DEBUGGING = 88,
 };
 
 enum ComdatSelectionKindCodes {
diff --git a/llvm/include/llvm/Bitstream/BitstreamReader.h b/llvm/include/llvm/Bitstream/BitstreamReader.h
index d271fed137..dbc98d1ad7 100644
--- a/llvm/include/llvm/Bitstream/BitstreamReader.h
+++ b/llvm/include/llvm/Bitstream/BitstreamReader.h
@@ -169,8 +169,7 @@ class SimpleBitstreamCursor {
     if (BitcodeBytes.size() >= NextChar + sizeof(word_t)) {
       BytesRead = sizeof(word_t);
       CurWord =
-          support::endian::read<word_t, support::little, support::unaligned>(
-              NextCharPtr);
+          support::endian::read<word_t, llvm::endianness::little>(NextCharPtr);
     } else {
       // Short read.
       BytesRead = BitcodeBytes.size() - NextChar;
diff --git a/llvm/include/llvm/Bitstream/BitstreamWriter.h b/llvm/include/llvm/Bitstream/BitstreamWriter.h
index 423af4c2cc..f7d362b5d7 100644
--- a/llvm/include/llvm/Bitstream/BitstreamWriter.h
+++ b/llvm/include/llvm/Bitstream/BitstreamWriter.h
@@ -75,7 +75,8 @@ class BitstreamWriter {
   std::vector<BlockInfo> BlockInfoRecords;
 
   void WriteWord(unsigned Value) {
-    Value = support::endian::byte_swap<uint32_t, support::little>(Value);
+    Value =
+        support::endian::byte_swap<uint32_t, llvm::endianness::little>(Value);
     Out.append(reinterpret_cast<const char *>(&Value),
                reinterpret_cast<const char *>(&Value + 1));
   }
@@ -138,10 +139,11 @@ class BitstreamWriter {
     uint64_t NumOfFlushedBytes = GetNumOfFlushedBytes();
 
     if (ByteNo >= NumOfFlushedBytes) {
-      assert((!endian::readAtBitAlignment<uint8_t, little, unaligned>(
+      assert((!endian::readAtBitAlignment<uint8_t, llvm::endianness::little,
+                                          unaligned>(
                  &Out[ByteNo - NumOfFlushedBytes], StartBit)) &&
              "Expected to be patching over 0-value placeholders");
-      endian::writeAtBitAlignment<uint8_t, little, unaligned>(
+      endian::writeAtBitAlignment<uint8_t, llvm::endianness::little, unaligned>(
           &Out[ByteNo - NumOfFlushedBytes], NewByte, StartBit);
       return;
     }
@@ -170,14 +172,14 @@ class BitstreamWriter {
       assert(BytesRead >= 0 && static_cast<size_t>(BytesRead) == BytesFromDisk);
       for (size_t i = 0; i < BytesFromBuffer; ++i)
         Bytes[BytesFromDisk + i] = Out[i];
-      assert((!endian::readAtBitAlignment<uint8_t, little, unaligned>(
-                 Bytes, StartBit)) &&
+      assert((!endian::readAtBitAlignment<uint8_t, llvm::endianness::little,
+                                          unaligned>(Bytes, StartBit)) &&
              "Expected to be patching over 0-value placeholders");
     }
 
     // Update Bytes in terms of bit offset and value.
-    endian::writeAtBitAlignment<uint8_t, little, unaligned>(Bytes, NewByte,
-                                                            StartBit);
+    endian::writeAtBitAlignment<uint8_t, llvm::endianness::little, unaligned>(
+        Bytes, NewByte, StartBit);
 
     // Copy updated data back to the file FS and the buffer Out.
     FS->seek(ByteNo);
diff --git a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
index ad26eee642..6e01dfd11e 100644
--- a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
+++ b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
@@ -28,17 +28,60 @@
 
 namespace llvm {
 
-// The cluster information for a machine basic block.
-struct BBClusterInfo {
-  // Unique ID for this basic block.
+// This structure represents a unique ID for every block specified in the
+// input profile.
+struct ProfileBBID {
+  // Basic block id associated with `MachineBasicBlock::BBID`.
   unsigned BBID;
+  // The clone id associated with the block. This is zero for the original
+  // block. For the cloned ones, it is equal to 1 + index of the associated
+  // path in `FunctionPathAndClusterInfo::ClonePaths`.
+  unsigned CloneID;
+};
+
+// This struct represents the cluster information for a machine basic block,
+// which is specifed by a unique ID. This templated struct is used for both the
+// raw input profile (as `BBClusterInfo<ProfileBBID>`) and the processed profile
+// after applying the clonings (as `BBClusterInfo<unsigned>`).
+template <typename BBIDType> struct BBClusterInfo {
+  // Basic block ID.
+  BBIDType BasicBlockID;
   // Cluster ID this basic block belongs to.
   unsigned ClusterID;
   // Position of basic block within the cluster.
   unsigned PositionInCluster;
 };
 
-using ProgramBBClusterInfoMapTy = StringMap<SmallVector<BBClusterInfo>>;
+// This represents the raw input profile for one function.
+struct FunctionPathAndClusterInfo {
+  // BB Cluster information specified by `ProfileBBID`s (before cloning).
+  SmallVector<BBClusterInfo<ProfileBBID>> ClusterInfo;
+  // Paths to clone. A path a -> b -> c -> d implies cloning b, c, and d along
+  // the edge a -> b (a is not cloned). The index of the path in this vector
+  // determines the `ProfileBBID::CloneID` of the cloned blocks in that path.
+  SmallVector<SmallVector<unsigned>> ClonePaths;
+};
+
+// Provides DenseMapInfo for ProfileBBID.
+template <> struct DenseMapInfo<ProfileBBID> {
+  static inline ProfileBBID getEmptyKey() {
+    unsigned EmptyKey = DenseMapInfo<unsigned>::getEmptyKey();
+    return ProfileBBID{EmptyKey, EmptyKey};
+  }
+  static inline ProfileBBID getTombstoneKey() {
+    unsigned TombstoneKey = DenseMapInfo<unsigned>::getTombstoneKey();
+    return ProfileBBID{TombstoneKey, TombstoneKey};
+  }
+  static unsigned getHashValue(const ProfileBBID &Val) {
+    std::pair<unsigned, unsigned> PairVal =
+        std::make_pair(Val.BBID, Val.CloneID);
+    return DenseMapInfo<std::pair<unsigned, unsigned>>::getHashValue(PairVal);
+  }
+  static bool isEqual(const ProfileBBID &LHS, const ProfileBBID &RHS) {
+    return DenseMapInfo<unsigned>::isEqual(LHS.BBID, RHS.BBID) &&
+           DenseMapInfo<unsigned>::isEqual(LHS.CloneID, RHS.CloneID);
+  }
+};
 
 class BasicBlockSectionsProfileReader : public ImmutablePass {
 public:
@@ -70,11 +113,11 @@ class BasicBlockSectionsProfileReader : public ImmutablePass {
   // function. If the first element is true and the second element is empty, it
   // means unique basic block sections are desired for all basic blocks of the
   // function.
-  std::pair<bool, SmallVector<BBClusterInfo>>
-  getBBClusterInfoForFunction(StringRef FuncName) const;
+  std::pair<bool, FunctionPathAndClusterInfo>
+  getPathAndClusterInfoForFunction(StringRef FuncName) const;
 
   // Initializes the FunctionNameToDIFilename map for the current module and
-  // then reads the profile for matching functions.
+  // then reads the profile for the matching functions.
   bool doInitialization(Module &M) override;
 
 private:
@@ -91,6 +134,12 @@ class BasicBlockSectionsProfileReader : public ImmutablePass {
         inconvertibleErrorCode());
   }
 
+  // Parses a `ProfileBBID` from `S`. `S` must be in the form "<bbid>"
+  // (representing an original block) or "<bbid>.<cloneid>" (representing a
+  // cloned block) where bbid is a non-negative integer and cloneid is a
+  // positive integer.
+  Expected<ProfileBBID> parseProfileBBID(StringRef S) const;
+
   // Reads the basic block sections profile for functions in this module.
   Error ReadProfile();
 
@@ -111,16 +160,16 @@ class BasicBlockSectionsProfileReader : public ImmutablePass {
   // empty string if no debug info is available.
   StringMap<SmallString<128>> FunctionNameToDIFilename;
 
-  // This encapsulates the BB cluster information for the whole program.
+  // This contains the BB cluster information for the whole program.
   //
-  // For every function name, it contains the cluster information for (all or
-  // some of) its basic blocks. The cluster information for every basic block
-  // includes its cluster ID along with the position of the basic block in that
-  // cluster.
-  ProgramBBClusterInfoMapTy ProgramBBClusterInfo;
+  // For every function name, it contains the cloning and cluster information
+  // for (all or some of) its basic blocks. The cluster information for every
+  // basic block includes its cluster ID along with the position of the basic
+  // block in that cluster.
+  StringMap<FunctionPathAndClusterInfo> ProgramPathAndClusterInfo;
 
   // Some functions have alias names. We use this map to find the main alias
-  // name for which we have mapping in ProgramBBClusterInfo.
+  // name which appears in ProgramPathAndClusterInfo as a key.
   StringMap<StringRef> FuncAliasMap;
 };
 
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 3dd16dafe3..752b554d69 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1691,7 +1691,69 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     }
     }
 
-    // Assume that we need to scalarize this intrinsic.
+    // VP Intrinsics should have the same cost as their non-vp counterpart.
+    // TODO: Adjust the cost to make the vp intrinsic cheaper than its non-vp
+    // counterpart when the vector length argument is smaller than the maximum
+    // vector length.
+    // TODO: Support other kinds of VPIntrinsics
+    if (VPIntrinsic::isVPIntrinsic(ICA.getID())) {
+      std::optional<unsigned> FOp =
+          VPIntrinsic::getFunctionalOpcodeForVP(ICA.getID());
+      if (FOp) {
+        if (ICA.getID() == Intrinsic::vp_load) {
+          Align Alignment;
+          if (auto *VPI = dyn_cast_or_null<VPIntrinsic>(ICA.getInst()))
+            Alignment = VPI->getPointerAlignment().valueOrOne();
+          unsigned AS = 0;
+          if (ICA.getArgs().size() > 1)
+            if (auto *PtrTy =
+                    dyn_cast<PointerType>(ICA.getArgs()[0]->getType()))
+              AS = PtrTy->getAddressSpace();
+          return thisT()->getMemoryOpCost(*FOp, ICA.getReturnType(), Alignment,
+                                          AS, CostKind);
+        }
+        if (ICA.getID() == Intrinsic::vp_store) {
+          Align Alignment;
+          if (auto *VPI = dyn_cast_or_null<VPIntrinsic>(ICA.getInst()))
+            Alignment = VPI->getPointerAlignment().valueOrOne();
+          unsigned AS = 0;
+          if (ICA.getArgs().size() >= 2)
+            if (auto *PtrTy =
+                    dyn_cast<PointerType>(ICA.getArgs()[1]->getType()))
+              AS = PtrTy->getAddressSpace();
+          return thisT()->getMemoryOpCost(*FOp, Args[0]->getType(), Alignment,
+                                          AS, CostKind);
+        }
+        if (VPBinOpIntrinsic::isVPBinOp(ICA.getID())) {
+          return thisT()->getArithmeticInstrCost(*FOp, ICA.getReturnType(),
+                                                 CostKind);
+        }
+      }
+
+      std::optional<Intrinsic::ID> FID =
+          VPIntrinsic::getFunctionalIntrinsicIDForVP(ICA.getID());
+      if (FID) {
+        // Non-vp version will have same Args/Tys except mask and vector length.
+        assert(ICA.getArgs().size() >= 2 && ICA.getArgTypes().size() >= 2 &&
+               "Expected VPIntrinsic to have Mask and Vector Length args and "
+               "types");
+        ArrayRef<Type *> NewTys = ArrayRef(ICA.getArgTypes()).drop_back(2);
+
+        // VPReduction intrinsics have a start value argument that their non-vp
+        // counterparts do not have, except for the fadd and fmul non-vp
+        // counterpart.
+        if (VPReductionIntrinsic::isVPReduction(ICA.getID()) &&
+            *FID != Intrinsic::vector_reduce_fadd &&
+            *FID != Intrinsic::vector_reduce_fmul)
+          NewTys = NewTys.drop_front();
+
+        IntrinsicCostAttributes NewICA(*FID, ICA.getReturnType(), NewTys,
+                                       ICA.getFlags());
+        return thisT()->getIntrinsicInstrCost(NewICA, CostKind);
+      }
+    }
+
+    // Assume that we need to scalarize this intrinsic.)
     // Compute the scalarization overhead based on Args for a vector
     // intrinsic.
     InstructionCost ScalarizationCost = InstructionCost::getInvalid();
@@ -1847,6 +1909,12 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     case Intrinsic::rint:
       ISD = ISD::FRINT;
       break;
+    case Intrinsic::lrint:
+      ISD = ISD::LRINT;
+      break;
+    case Intrinsic::llrint:
+      ISD = ISD::LLRINT;
+      break;
     case Intrinsic::round:
       ISD = ISD::FROUND;
       break;
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index d8f19c19ee..d64b414f27 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -194,9 +194,6 @@ class CombinerHelper {
   /// Match (and (load x), mask) -> zextload x
   bool matchCombineLoadWithAndMask(MachineInstr &MI, BuildFnTy &MatchInfo);
 
-  /// Combine \p MI into a pre-indexed or post-indexed load/store operation if
-  /// legal and the surrounding code makes it useful.
-  bool tryCombineIndexedLoadStore(MachineInstr &MI);
   bool matchCombineIndexedLoadStore(MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo);
   void applyCombineIndexedLoadStore(MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo);
 
@@ -378,7 +375,6 @@ class CombinerHelper {
 
   /// Transform anyext(trunc(x)) to x.
   bool matchCombineAnyExtTrunc(MachineInstr &MI, Register &Reg);
-  void applyCombineAnyExtTrunc(MachineInstr &MI, Register &Reg);
 
   /// Transform zext(trunc(x)) to x.
   bool matchCombineZextTrunc(MachineInstr &MI, Register &Reg);
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h
index 2b0733cf93..209f80c6d6 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h
@@ -40,6 +40,7 @@ class APInt;
 class APFloat;
 class GISelKnownBits;
 class MachineInstr;
+class MachineIRBuilder;
 class MachineInstrBuilder;
 class MachineFunction;
 class MachineOperand;
@@ -291,6 +292,11 @@ enum {
   /// - Opcode - The new opcode to use
   GIR_BuildMI,
 
+  /// Builds a constant and stores its result in a TempReg.
+  /// - TempRegID - Temp Register to define.
+  /// - Imm - The immediate to add
+  GIR_BuildConstant,
+
   /// Copy an operand to the specified instruction
   /// - NewInsnID - Instruction ID to modify
   /// - OldInsnID - Instruction ID to copy from
@@ -555,15 +561,15 @@ class GIMatchTableExecutor {
   /// and false otherwise.
   template <class TgtExecutor, class PredicateBitset, class ComplexMatcherMemFn,
             class CustomRendererFn>
-  bool executeMatchTable(
-      TgtExecutor &Exec, NewMIVector &OutMIs, MatcherState &State,
-      const ExecInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn>
-          &ISelInfo,
-      const int64_t *MatchTable, const TargetInstrInfo &TII,
-      MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
-      const RegisterBankInfo &RBI, const PredicateBitset &AvailableFeatures,
-      CodeGenCoverage *CoverageInfo,
-      GISelChangeObserver *Observer = nullptr) const;
+  bool executeMatchTable(TgtExecutor &Exec, MatcherState &State,
+                         const ExecInfoTy<PredicateBitset, ComplexMatcherMemFn,
+                                          CustomRendererFn> &ExecInfo,
+                         MachineIRBuilder &Builder, const int64_t *MatchTable,
+                         const TargetInstrInfo &TII, MachineRegisterInfo &MRI,
+                         const TargetRegisterInfo &TRI,
+                         const RegisterBankInfo &RBI,
+                         const PredicateBitset &AvailableFeatures,
+                         CodeGenCoverage *CoverageInfo) const;
 
   virtual const int64_t *getMatchTable() const {
     llvm_unreachable("Should have been overridden by tablegen if used");
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h
index 883c1ca0fe..fb03d5ec0b 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h
@@ -18,6 +18,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h"
 #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
+#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
 #include "llvm/CodeGen/GlobalISel/Utils.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineOperand.h"
@@ -42,17 +43,20 @@ namespace llvm {
 template <class TgtExecutor, class PredicateBitset, class ComplexMatcherMemFn,
           class CustomRendererFn>
 bool GIMatchTableExecutor::executeMatchTable(
-    TgtExecutor &Exec, NewMIVector &OutMIs, MatcherState &State,
+    TgtExecutor &Exec, MatcherState &State,
     const ExecInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn>
         &ExecInfo,
-    const int64_t *MatchTable, const TargetInstrInfo &TII,
-    MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
-    const RegisterBankInfo &RBI, const PredicateBitset &AvailableFeatures,
-    CodeGenCoverage *CoverageInfo, GISelChangeObserver *Observer) const {
+    MachineIRBuilder &Builder, const int64_t *MatchTable,
+    const TargetInstrInfo &TII, MachineRegisterInfo &MRI,
+    const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI,
+    const PredicateBitset &AvailableFeatures,
+    CodeGenCoverage *CoverageInfo) const {
 
   uint64_t CurrentIdx = 0;
   SmallVector<uint64_t, 4> OnFailResumeAt;
+  NewMIVector OutMIs;
 
+  GISelChangeObserver *Observer = Builder.getObserver();
   // Bypass the flag check on the instruction, and only look at the MCInstrDesc.
   bool NoFPException = !State.MIs[0]->getDesc().mayRaiseFPException();
 
@@ -71,14 +75,18 @@ bool GIMatchTableExecutor::executeMatchTable(
     return RejectAndResume;
   };
 
-  auto propagateFlags = [=](NewMIVector &OutMIs) {
+  const auto propagateFlags = [&]() {
     for (auto MIB : OutMIs) {
       // Set the NoFPExcept flag when no original matched instruction could
       // raise an FP exception, but the new instruction potentially might.
       uint16_t MIBFlags = Flags;
       if (NoFPException && MIB->mayRaiseFPException())
         MIBFlags |= MachineInstr::NoFPExcept;
+      if (Observer)
+        Observer->changingInstr(*MIB);
       MIB.setMIFlags(MIBFlags);
+      if (Observer)
+        Observer->changedInstr(*MIB);
     }
 
     return true;
@@ -898,9 +906,13 @@ bool GIMatchTableExecutor::executeMatchTable(
       if (NewInsnID >= OutMIs.size())
         OutMIs.resize(NewInsnID + 1);
 
-      OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
-                                              State.MIs[OldInsnID]);
+      MachineInstr *OldMI = State.MIs[OldInsnID];
+      if (Observer)
+        Observer->changingInstr(*OldMI);
+      OutMIs[NewInsnID] = MachineInstrBuilder(*OldMI->getMF(), OldMI);
       OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
+      if (Observer)
+        Observer->changedInstr(*OldMI);
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
                              << NewInsnID << "], MIs[" << OldInsnID << "], "
@@ -914,14 +926,23 @@ bool GIMatchTableExecutor::executeMatchTable(
       if (NewInsnID >= OutMIs.size())
         OutMIs.resize(NewInsnID + 1);
 
-      OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
-                                  MIMetadata(*State.MIs[0]), TII.get(Opcode));
+      OutMIs[NewInsnID] = Builder.buildInstr(Opcode);
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
                              << NewInsnID << "], " << Opcode << ")\n");
       break;
     }
 
+    case GIR_BuildConstant: {
+      int64_t TempRegID = MatchTable[CurrentIdx++];
+      int64_t Imm = MatchTable[CurrentIdx++];
+      Builder.buildConstant(State.TempRegisters[TempRegID], Imm);
+      DEBUG_WITH_TYPE(TgtExecutor::getName(),
+                      dbgs() << CurrentIdx << ": GIR_BuildConstant(TempReg["
+                             << TempRegID << "], Imm=" << Imm << ")\n");
+      break;
+    }
+
     case GIR_Copy: {
       int64_t NewInsnID = MatchTable[CurrentIdx++];
       int64_t OldInsnID = MatchTable[CurrentIdx++];
@@ -1239,6 +1260,10 @@ bool GIMatchTableExecutor::executeMatchTable(
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
                              << InsnID << "])\n");
+      // If we're erasing the insertion point, ensure we don't leave a dangling
+      // pointer in the builder.
+      if (Builder.getInsertPt() == MI)
+        Builder.setInsertPt(*MI->getParent(), ++MI->getIterator());
       if (Observer)
         Observer->erasingInstr(*MI);
       MI->eraseFromParent();
@@ -1309,11 +1334,7 @@ bool GIMatchTableExecutor::executeMatchTable(
     case GIR_Done:
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": GIR_Done\n");
-      if (Observer) {
-        for (MachineInstr *MI : OutMIs)
-          Observer->createdInstr(*MI);
-      }
-      propagateFlags(OutMIs);
+      propagateFlags();
       return true;
     default:
       llvm_unreachable("Unexpected command");
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegacyLegalizerInfo.h b/llvm/include/llvm/CodeGen/GlobalISel/LegacyLegalizerInfo.h
index 08233dba20..0b167ce965 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegacyLegalizerInfo.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegacyLegalizerInfo.h
@@ -240,16 +240,6 @@ class LegacyLegalizerInfo {
                                                        Unsupported);
   }
 
-  static SizeAndActionsVec
-  narrowToSmallerAndWidenToSmallest(const SizeAndActionsVec &v) {
-    using namespace LegacyLegalizeActions;
-    assert(v.size() > 0 &&
-           "At least one size that can be legalized towards is needed"
-           " for this SizeChangeStrategy");
-    return decreaseToSmallerTypesAndIncreaseToSmallest(v, NarrowScalar,
-                                                       WidenScalar);
-  }
-
   /// A SizeChangeStrategy for the common case where legalization for a
   /// particular vector operation consists of having more elements in the
   /// vector, to a type that is legal. Unless there is no such type and then
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
index 8516a28d00..86d3cb2bed 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
@@ -384,6 +384,7 @@ class LegalizerHelper {
   LegalizeResult lowerFunnelShiftAsShifts(MachineInstr &MI);
   LegalizeResult lowerFunnelShift(MachineInstr &MI);
   LegalizeResult lowerEXT(MachineInstr &MI);
+  LegalizeResult lowerTRUNC(MachineInstr &MI);
   LegalizeResult lowerRotateWithReverseRotate(MachineInstr &MI);
   LegalizeResult lowerRotate(MachineInstr &MI);
 
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
index bf41c19cd6..3409d44170 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
@@ -298,6 +298,9 @@ class MachineIRBuilder {
   /// Getter for the State
   MachineIRBuilderState &getState() { return State; }
 
+  /// Setter for the State
+  void setState(const MachineIRBuilderState &NewState) { State = NewState; }
+
   /// Getter for the basic block we currently build.
   const MachineBasicBlock &getMBB() const {
     assert(State.MBB && "MachineBasicBlock is not set");
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
index ffb6e53a03..515fd5975d 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
@@ -171,11 +171,13 @@ void reportGISelWarning(MachineFunction &MF, const TargetPassConfig &TPC,
 
 /// If \p VReg is defined by a G_CONSTANT, return the corresponding value.
 std::optional<APInt> getIConstantVRegVal(Register VReg,
-                                         const MachineRegisterInfo &MRI);
+                                         const MachineRegisterInfo &MRI,
+                                         bool LookThroughInstrs = false);
 
 /// If \p VReg is defined by a G_CONSTANT fits in int64_t returns it.
 std::optional<int64_t> getIConstantVRegSExtVal(Register VReg,
-                                               const MachineRegisterInfo &MRI);
+                                               const MachineRegisterInfo &MRI,
+                                               bool LookThroughInstrs = false);
 
 /// Simple struct used to hold a constant integer value and a virtual
 /// register.
diff --git a/llvm/include/llvm/CodeGen/LiveVariables.h b/llvm/include/llvm/CodeGen/LiveVariables.h
index a1ed3c0732..9ed4c7bdf7 100644
--- a/llvm/include/llvm/CodeGen/LiveVariables.h
+++ b/llvm/include/llvm/CodeGen/LiveVariables.h
@@ -118,11 +118,6 @@ class LiveVariables : public MachineFunctionPass {
   ///
   IndexedMap<VarInfo, VirtReg2IndexFunctor> VirtRegInfo;
 
-  /// PHIJoins - list of virtual registers that are PHI joins. These registers
-  /// may have multiple definitions, and they require special handling when
-  /// building live intervals.
-  SparseBitVector<> PHIJoins;
-
 private:   // Intermediate data structures
   MachineFunction *MF = nullptr;
 
@@ -182,10 +177,6 @@ class LiveVariables : public MachineFunctionPass {
 
   bool runOnMachineFunction(MachineFunction &MF) override;
 
-  /// RegisterDefIsDead - Return true if the specified instruction defines the
-  /// specified register, but that definition is dead.
-  bool RegisterDefIsDead(MachineInstr &MI, Register Reg) const;
-
   //===--------------------------------------------------------------------===//
   //  API to update live variable information
 
@@ -306,12 +297,6 @@ class LiveVariables : public MachineFunctionPass {
                    MachineBasicBlock *DomBB,
                    MachineBasicBlock *SuccBB,
                    std::vector<SparseBitVector<>> &LiveInSets);
-
-  /// isPHIJoin - Return true if Reg is a phi join register.
-  bool isPHIJoin(Register Reg) { return PHIJoins.test(Reg.id()); }
-
-  /// setPHIJoin - Mark Reg as a phi join register.
-  void setPHIJoin(Register Reg) { PHIJoins.set(Reg.id()); }
 };
 
 } // End llvm namespace
diff --git a/llvm/include/llvm/CodeGen/MachineInstr.h b/llvm/include/llvm/CodeGen/MachineInstr.h
index 8367f999fc..6afc2d5cbe 100644
--- a/llvm/include/llvm/CodeGen/MachineInstr.h
+++ b/llvm/include/llvm/CodeGen/MachineInstr.h
@@ -1365,12 +1365,6 @@ class MachineInstr
            getOpcode() == TargetOpcode::INLINEASM_BR;
   }
 
-  /// FIXME: Seems like a layering violation that the AsmDialect, which is X86
-  /// specific, be attached to a generic MachineInstr.
-  bool isMSInlineAsm() const {
-    return isInlineAsm() && getInlineAsmDialect() == InlineAsm::AD_Intel;
-  }
-
   bool isStackAligningInlineAsm() const;
   InlineAsm::AsmDialect getInlineAsmDialect() const;
 
diff --git a/llvm/include/llvm/CodeGen/MachineStableHash.h b/llvm/include/llvm/CodeGen/MachineStableHash.h
index 43571b7b8a..743615d136 100644
--- a/llvm/include/llvm/CodeGen/MachineStableHash.h
+++ b/llvm/include/llvm/CodeGen/MachineStableHash.h
@@ -14,7 +14,7 @@
 #ifndef LLVM_CODEGEN_MACHINESTABLEHASH_H
 #define LLVM_CODEGEN_MACHINESTABLEHASH_H
 
-#include "llvm/CodeGen/StableHashing.h"
+#include "llvm/ADT/StableHashing.h"
 
 namespace llvm {
 class MachineBasicBlock;
diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h
index befa8a6eb9..8d14eef949 100644
--- a/llvm/include/llvm/CodeGen/Passes.h
+++ b/llvm/include/llvm/CodeGen/Passes.h
@@ -446,9 +446,6 @@ namespace llvm {
   /// LiveDebugValues pass
   extern char &LiveDebugValuesID;
 
-  /// createJumpInstrTables - This pass creates jump-instruction tables.
-  ModulePass *createJumpInstrTablesPass();
-
   /// InterleavedAccess Pass - This pass identifies and matches interleaved
   /// memory accesses to target specific intrinsics.
   ///
@@ -587,9 +584,6 @@ namespace llvm {
   /// or split the data to two <128 x i32>.
   FunctionPass *createX86LowerAMXTypePass();
 
-  /// The pass insert tile config intrinsics for AMX fast register allocation.
-  FunctionPass *createX86PreAMXConfigPass();
-
   /// The pass transforms amx intrinsics to scalar operation if the function has
   /// optnone attribute or it is O0.
   FunctionPass *createX86LowerAMXIntrinsicsPass();
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index 2c629f3f96..e867448b9d 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -1919,10 +1919,10 @@ class SelectionDAG {
   SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT,
                                  ArrayRef<SDValue> Ops);
 
-  /// Fold floating-point operations with 2 operands when both operands are
-  /// constants and/or undefined.
+  /// Fold floating-point operations when all operands are constants and/or
+  /// undefined.
   SDValue foldConstantFPMath(unsigned Opcode, const SDLoc &DL, EVT VT,
-                             SDValue N1, SDValue N2);
+                             ArrayRef<SDValue> Ops);
 
   /// Constant fold a setcc to true or false.
   SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond,
diff --git a/llvm/include/llvm/CodeGen/SlotIndexes.h b/llvm/include/llvm/CodeGen/SlotIndexes.h
index 1a8d117322..72f4a6876b 100644
--- a/llvm/include/llvm/CodeGen/SlotIndexes.h
+++ b/llvm/include/llvm/CodeGen/SlotIndexes.h
@@ -125,8 +125,7 @@ class raw_ostream;
 
     // Construct a new slot index from the given one, and set the slot.
     SlotIndex(const SlotIndex &li, Slot s) : lie(li.listEntry(), unsigned(s)) {
-      assert(lie.getPointer() != nullptr &&
-             "Attempt to construct index with 0 pointer.");
+      assert(isValid() && "Attempt to construct index with 0 pointer.");
     }
 
     /// Returns true if this is a valid index. Invalid indices do
@@ -178,7 +177,7 @@ class raw_ostream;
 
     /// isSameInstr - Return true if A and B refer to the same instruction.
     static bool isSameInstr(SlotIndex A, SlotIndex B) {
-      return A.lie.getPointer() == B.lie.getPointer();
+      return A.listEntry() == B.listEntry();
     }
 
     /// isEarlierInstr - Return true if A refers to an instruction earlier than
@@ -388,7 +387,7 @@ class raw_ostream;
     /// Returns the instruction for the given index, or null if the given
     /// index has no instruction associated with it.
     MachineInstr* getInstructionFromIndex(SlotIndex index) const {
-      return index.isValid() ? index.listEntry()->getInstr() : nullptr;
+      return index.listEntry()->getInstr();
     }
 
     /// Returns the next non-null index, if one exists.
@@ -473,18 +472,25 @@ class raw_ostream;
     /// begin and basic block)
     using MBBIndexIterator = SmallVectorImpl<IdxMBBPair>::const_iterator;
 
-    /// Move iterator to the next IdxMBBPair where the SlotIndex is greater or
-    /// equal to \p To.
-    MBBIndexIterator advanceMBBIndex(MBBIndexIterator I, SlotIndex To) const {
-      return std::partition_point(
-          I, idx2MBBMap.end(),
-          [=](const IdxMBBPair &IM) { return IM.first < To; });
+    /// Get an iterator pointing to the first IdxMBBPair with SlotIndex greater
+    /// than or equal to \p Idx. If \p Start is provided, only search the range
+    /// from \p Start to the end of the function.
+    MBBIndexIterator getMBBLowerBound(MBBIndexIterator Start,
+                                      SlotIndex Idx) const {
+      return std::lower_bound(
+          Start, MBBIndexEnd(), Idx,
+          [](const IdxMBBPair &IM, SlotIndex Idx) { return IM.first < Idx; });
+    }
+    MBBIndexIterator getMBBLowerBound(SlotIndex Idx) const {
+      return getMBBLowerBound(MBBIndexBegin(), Idx);
     }
 
-    /// Get an iterator pointing to the IdxMBBPair with the biggest SlotIndex
-    /// that is greater or equal to \p Idx.
-    MBBIndexIterator findMBBIndex(SlotIndex Idx) const {
-      return advanceMBBIndex(idx2MBBMap.begin(), Idx);
+    /// Get an iterator pointing to the first IdxMBBPair with SlotIndex greater
+    /// than \p Idx.
+    MBBIndexIterator getMBBUpperBound(SlotIndex Idx) const {
+      return std::upper_bound(
+          MBBIndexBegin(), MBBIndexEnd(), Idx,
+          [](SlotIndex Idx, const IdxMBBPair &IM) { return Idx < IM.first; });
     }
 
     /// Returns an iterator for the begin of the idx2MBBMap.
@@ -502,16 +508,11 @@ class raw_ostream;
       if (MachineInstr *MI = getInstructionFromIndex(index))
         return MI->getParent();
 
-      MBBIndexIterator I = findMBBIndex(index);
-      // Take the pair containing the index
-      MBBIndexIterator J =
-        ((I != MBBIndexEnd() && I->first > index) ||
-         (I == MBBIndexEnd() && !idx2MBBMap.empty())) ? std::prev(I) : I;
-
-      assert(J != MBBIndexEnd() && J->first <= index &&
-             index < getMBBEndIdx(J->second) &&
+      MBBIndexIterator I = std::prev(getMBBUpperBound(index));
+      assert(I != MBBIndexEnd() && I->first <= index &&
+             index < getMBBEndIdx(I->second) &&
              "index does not correspond to an MBB");
-      return J->second;
+      return I->second;
     }
 
     /// Insert the given machine instruction into the mapping. Returns the
diff --git a/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
index 14e27abe88..8e7499ac62 100644
--- a/llvm/include/llvm/CodeGen/TargetInstrInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
@@ -2093,12 +2093,19 @@ class TargetInstrInfo : public MCInstrInfo {
         "Target didn't implement TargetInstrInfo::insertOutlinedCall!");
   }
 
-  /// Insert an architecture-specific instruction to clear a register.
+  /// Insert an architecture-specific instruction to clear a register. If you
+  /// need to avoid sideeffects (e.g. avoid XOR on x86, which sets EFLAGS), set
+  /// \p AllowSideEffects to \p false.
   virtual void buildClearRegister(Register Reg, MachineBasicBlock &MBB,
                                   MachineBasicBlock::iterator Iter,
-                                  DebugLoc &DL) const {
+                                  DebugLoc &DL,
+                                  bool AllowSideEffects = true) const {
+#if 0
+    // FIXME: This should exist once all platforms that use stack protectors
+    // implements it.
     llvm_unreachable(
         "Target didn't implement TargetInstrInfo::buildClearRegister!");
+#endif
   }
 
   /// Return true if the function can safely be outlined from.
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 187e000d02..1494f335e4 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -828,6 +828,24 @@ class TargetLoweringBase {
     return N->getOpcode() == ISD::FDIV;
   }
 
+  // Given:
+  //    (icmp eq/ne (and X, C0), (shift X, C1))
+  // or
+  //    (icmp eq/ne X, (rotate X, CPow2))
+
+  // If C0 is a mask or shifted mask and the shift amt (C1) isolates the
+  // remaining bits (i.e something like `(x64 & UINT32_MAX) == (x64 >> 32)`)
+  // Do we prefer the shift to be shift-right, shift-left, or rotate.
+  // Note: Its only valid to convert the rotate version to the shift version iff
+  // the shift-amt (`C1`) is a power of 2 (including 0).
+  // If ShiftOpc (current Opcode) is returned, do nothing.
+  virtual unsigned preferedOpcodeForCmpEqPiecesOfOperand(
+      EVT VT, unsigned ShiftOpc, bool MayTransformRotate,
+      const APInt &ShiftOrRotateAmt,
+      const std::optional<APInt> &AndMask) const {
+    return ShiftOpc;
+  }
+
   /// These two forms are equivalent:
   ///   sub %y, (xor %x, -1)
   ///   add (add %x, 1), %y
@@ -4397,8 +4415,14 @@ class TargetLowering : public TargetLoweringBase {
     }
 
     CallLoweringInfo &setCallee(CallingConv::ID CC, Type *ResultType,
-                                SDValue Target, ArgListTy &&ArgsList) {
+                                SDValue Target, ArgListTy &&ArgsList,
+                                AttributeSet ResultAttrs = {}) {
       RetTy = ResultType;
+      IsInReg = ResultAttrs.hasAttribute(Attribute::InReg);
+      RetSExt = ResultAttrs.hasAttribute(Attribute::SExt);
+      RetZExt = ResultAttrs.hasAttribute(Attribute::ZExt);
+      NoMerge = ResultAttrs.hasAttribute(Attribute::NoMerge);
+
       Callee = Target;
       CallConv = CC;
       NumFixedArgs = ArgsList.size();
diff --git a/llvm/include/llvm/Config/llvm-config.h.cmake b/llvm/include/llvm/Config/llvm-config.h.cmake
index 8d0a1bc5dc..17b2d47fb6 100644
--- a/llvm/include/llvm/Config/llvm-config.h.cmake
+++ b/llvm/include/llvm/Config/llvm-config.h.cmake
@@ -98,7 +98,7 @@
 /* Define if zstd compression is available */
 #cmakedefine01 LLVM_ENABLE_ZSTD
 
-/* Define if LLVM is using tflite instead of libtensorflow */
+/* Define if LLVM is using tflite */
 #cmakedefine LLVM_HAVE_TFLITE
 
 /* Define to 1 if you have the <sysexits.h> header file. */
diff --git a/llvm/include/llvm/DebugInfo/CodeView/SymbolDeserializer.h b/llvm/include/llvm/DebugInfo/CodeView/SymbolDeserializer.h
index 108abb2914..9461af8a48 100644
--- a/llvm/include/llvm/DebugInfo/CodeView/SymbolDeserializer.h
+++ b/llvm/include/llvm/DebugInfo/CodeView/SymbolDeserializer.h
@@ -24,7 +24,7 @@ class SymbolVisitorDelegate;
 class SymbolDeserializer : public SymbolVisitorCallbacks {
   struct MappingInfo {
     MappingInfo(ArrayRef<uint8_t> RecordData, CodeViewContainer Container)
-        : Stream(RecordData, llvm::support::little), Reader(Stream),
+        : Stream(RecordData, llvm::endianness::little), Reader(Stream),
           Mapping(Reader, Container) {}
 
     BinaryByteStream Stream;
diff --git a/llvm/include/llvm/DebugInfo/CodeView/TypeDeserializer.h b/llvm/include/llvm/DebugInfo/CodeView/TypeDeserializer.h
index 2b17f5ccb1..acea4e341c 100644
--- a/llvm/include/llvm/DebugInfo/CodeView/TypeDeserializer.h
+++ b/llvm/include/llvm/DebugInfo/CodeView/TypeDeserializer.h
@@ -28,7 +28,7 @@ namespace codeview {
 class TypeDeserializer : public TypeVisitorCallbacks {
   struct MappingInfo {
     explicit MappingInfo(ArrayRef<uint8_t> RecordData)
-        : Stream(RecordData, llvm::support::little), Reader(Stream),
+        : Stream(RecordData, llvm::endianness::little), Reader(Stream),
           Mapping(Reader) {}
 
     BinaryByteStream Stream;
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h
index 3c07707874..7084081ce6 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h
@@ -79,8 +79,8 @@ class DWARFUnitHeader {
   /// Note that \p SectionKind is used as a hint to guess the unit type
   /// for DWARF formats prior to DWARFv5. In DWARFv5 the unit type is
   /// explicitly defined in the header and the hint is ignored.
-  bool extract(DWARFContext &Context, const DWARFDataExtractor &debug_info,
-               uint64_t *offset_ptr, DWARFSectionKind SectionKind);
+  Error extract(DWARFContext &Context, const DWARFDataExtractor &debug_info,
+                uint64_t *offset_ptr, DWARFSectionKind SectionKind);
   // For units in DWARF Package File, remember the index entry and update
   // the abbreviation offset read by extract().
   bool applyIndexEntry(const DWARFUnitIndex::Entry *Entry);
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFVerifier.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFVerifier.h
index ac890cdf06..e56d3781e8 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFVerifier.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFVerifier.h
@@ -156,7 +156,6 @@ class DWARFVerifier {
   unsigned verifyUnitSection(const DWARFSection &S);
   unsigned verifyUnits(const DWARFUnitVector &Units);
 
-  unsigned verifyIndexes(const DWARFObject &DObj);
   unsigned verifyIndex(StringRef Name, DWARFSectionKind SectionKind,
                        StringRef Index);
 
diff --git a/llvm/include/llvm/DebugInfo/MSF/MappedBlockStream.h b/llvm/include/llvm/DebugInfo/MSF/MappedBlockStream.h
index b5f0596fce..04a64d28e0 100644
--- a/llvm/include/llvm/DebugInfo/MSF/MappedBlockStream.h
+++ b/llvm/include/llvm/DebugInfo/MSF/MappedBlockStream.h
@@ -54,8 +54,8 @@ class MappedBlockStream : public BinaryStream {
   createDirectoryStream(const MSFLayout &Layout, BinaryStreamRef MsfData,
                         BumpPtrAllocator &Allocator);
 
-  support::endianness getEndian() const override {
-    return support::little;
+  llvm::endianness getEndian() const override {
+    return llvm::endianness::little;
   }
 
   Error readBytes(uint64_t Offset, uint64_t Size,
@@ -121,8 +121,8 @@ class WritableMappedBlockStream : public WritableBinaryStream {
   createFpmStream(const MSFLayout &Layout, WritableBinaryStreamRef MsfData,
                   BumpPtrAllocator &Allocator, bool AltFpm = false);
 
-  support::endianness getEndian() const override {
-    return support::little;
+  llvm::endianness getEndian() const override {
+    return llvm::endianness::little;
   }
 
   Error readBytes(uint64_t Offset, uint64_t Size,
diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/FormatUtil.h b/llvm/include/llvm/DebugInfo/PDB/Native/FormatUtil.h
index ed745eaf97..01de8b49dd 100644
--- a/llvm/include/llvm/DebugInfo/PDB/Native/FormatUtil.h
+++ b/llvm/include/llvm/DebugInfo/PDB/Native/FormatUtil.h
@@ -107,10 +107,9 @@ namespace detail {
 template <typename T>
 struct EndianAdapter final
     : public FormatAdapter<support::detail::packed_endian_specific_integral<
-          T, support::little, support::unaligned>> {
-  using EndianType =
-      support::detail::packed_endian_specific_integral<T, support::little,
-                                                       support::unaligned>;
+          T, llvm::endianness::little, support::unaligned>> {
+  using EndianType = support::detail::packed_endian_specific_integral<
+      T, llvm::endianness::little, support::unaligned>;
 
   explicit EndianAdapter(EndianType &&Item)
       : FormatAdapter<EndianType>(std::move(Item)) {}
@@ -122,10 +121,9 @@ struct EndianAdapter final
 } // namespace detail
 
 template <typename T>
-detail::EndianAdapter<T>
-fmtle(support::detail::packed_endian_specific_integral<T, support::little,
-                                                       support::unaligned>
-          Value) {
+detail::EndianAdapter<T> fmtle(support::detail::packed_endian_specific_integral<
+                               T, llvm::endianness::little, support::unaligned>
+                                   Value) {
   return detail::EndianAdapter<T>(std::move(Value));
 }
 } // namespace pdb
diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h
index 4fcb4656a2..fb758f7a66 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h
@@ -995,14 +995,14 @@ class LinkGraph {
   using GetEdgeKindNameFunction = const char *(*)(Edge::Kind);
 
   LinkGraph(std::string Name, const Triple &TT, SubtargetFeatures Features,
-            unsigned PointerSize, support::endianness Endianness,
+            unsigned PointerSize, llvm::endianness Endianness,
             GetEdgeKindNameFunction GetEdgeKindName)
       : Name(std::move(Name)), TT(TT), Features(std::move(Features)),
         PointerSize(PointerSize), Endianness(Endianness),
         GetEdgeKindName(std::move(GetEdgeKindName)) {}
 
   LinkGraph(std::string Name, const Triple &TT, unsigned PointerSize,
-            support::endianness Endianness,
+            llvm::endianness Endianness,
             GetEdgeKindNameFunction GetEdgeKindName)
       : LinkGraph(std::move(Name), TT, SubtargetFeatures(), PointerSize,
                   Endianness, GetEdgeKindName) {}
@@ -1026,7 +1026,7 @@ class LinkGraph {
   unsigned getPointerSize() const { return PointerSize; }
 
   /// Returns the endianness of content in this graph.
-  support::endianness getEndianness() const { return Endianness; }
+  llvm::endianness getEndianness() const { return Endianness; }
 
   const char *getEdgeKindName(Edge::Kind K) const { return GetEdgeKindName(K); }
 
@@ -1530,7 +1530,7 @@ class LinkGraph {
   Triple TT;
   SubtargetFeatures Features;
   unsigned PointerSize;
-  support::endianness Endianness;
+  llvm::endianness Endianness;
   GetEdgeKindNameFunction GetEdgeKindName = nullptr;
   DenseMap<StringRef, std::unique_ptr<Section>> Sections;
   ExternalSymbolMap ExternalSymbols;
diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/ppc64.h b/llvm/include/llvm/ExecutionEngine/JITLink/ppc64.h
index 88af15d61e..b723914455 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/ppc64.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/ppc64.h
@@ -51,6 +51,7 @@ enum EdgeKind_ppc64 : Edge::Kind {
   TOCDelta16HI,
   TOCDelta16LO,
   TOCDelta16LODS,
+  RequestGOTAndTransformToDelta34,
   CallBranchDelta,
   // Need to restore r2 after the bl, suggesting the bl is followed by a nop.
   CallBranchDeltaRestoreTOC,
@@ -60,6 +61,7 @@ enum EdgeKind_ppc64 : Edge::Kind {
   RequestCallNoTOC,
   RequestTLSDescInGOTAndTransformToTOCDelta16HA,
   RequestTLSDescInGOTAndTransformToTOCDelta16LO,
+  RequestTLSDescInGOTAndTransformToDelta34,
 };
 
 enum PLTCallStubKind {
@@ -88,9 +90,9 @@ struct PLTCallStubInfo {
   SmallVector<PLTCallStubReloc, 2> Relocs;
 };
 
-template <support::endianness Endianness>
+template <llvm::endianness Endianness>
 inline PLTCallStubInfo pickStub(PLTCallStubKind StubKind) {
-  constexpr bool isLE = Endianness == support::endianness::little;
+  constexpr bool isLE = Endianness == llvm::endianness::little;
   switch (StubKind) {
   case LongBranch: {
     ArrayRef<char> Content =
@@ -139,7 +141,7 @@ inline Symbol &createAnonymousPointer(LinkGraph &G, Section &PointerSection,
   return G.addAnonymousSymbol(B, 0, G.getPointerSize(), false, false);
 }
 
-template <support::endianness Endianness>
+template <llvm::endianness Endianness>
 inline Symbol &createAnonymousPointerJumpStub(LinkGraph &G,
                                               Section &StubSection,
                                               Symbol &PointerSymbol,
@@ -152,7 +154,7 @@ inline Symbol &createAnonymousPointerJumpStub(LinkGraph &G,
   return G.addAnonymousSymbol(B, 0, StubInfo.Content.size(), true, false);
 }
 
-template <support::endianness Endianness>
+template <llvm::endianness Endianness>
 class TOCTableManager : public TableManager<TOCTableManager<Endianness>> {
 public:
   // FIXME: `llvm-jitlink -check` relies this name to be $__GOT.
@@ -170,6 +172,10 @@ class TOCTableManager : public TableManager<TOCTableManager<Endianness>> {
       // Create TOC section if TOC relocation, PLT or GOT is used.
       getOrCreateTOCSection(G);
       return false;
+    case RequestGOTAndTransformToDelta34:
+      E.setKind(ppc64::Delta34);
+      E.setTarget(createEntry(G, E.getTarget()));
+      return true;
     default:
       return false;
     }
@@ -190,13 +196,17 @@ class TOCTableManager : public TableManager<TOCTableManager<Endianness>> {
   Section *TOCSection = nullptr;
 };
 
-template <support::endianness Endianness>
+template <llvm::endianness Endianness>
 class PLTTableManager : public TableManager<PLTTableManager<Endianness>> {
 public:
   PLTTableManager(TOCTableManager<Endianness> &TOC) : TOC(TOC) {}
 
   static StringRef getSectionName() { return "$__STUBS"; }
 
+  // FIXME: One external symbol can only have one PLT stub in a object file.
+  // This is a limitation when we need different PLT stubs for the same symbol.
+  // For example, we need two different PLT stubs for `bl __tls_get_addr` and
+  // `bl __tls_get_addr@notoc`.
   bool visitEdge(LinkGraph &G, Block *B, Edge &E) {
     bool isExternal = E.getTarget().isExternal();
     Edge::Kind K = E.getKind();
@@ -271,21 +281,21 @@ inline static uint16_t highesta(uint64_t x) { return (x + 0x8000) >> 48; }
 // the most significant 32 bits belong to the prefix word. The prefix word is at
 // low address for both big/little endian. Byte order in each word still follows
 // its endian.
-template <support::endianness Endianness>
+template <llvm::endianness Endianness>
 inline static uint64_t readPrefixedInstruction(const char *Loc) {
-  constexpr bool isLE = Endianness == support::endianness::little;
+  constexpr bool isLE = Endianness == llvm::endianness::little;
   uint64_t Inst = support::endian::read64<Endianness>(Loc);
   return isLE ? (Inst << 32) | (Inst >> 32) : Inst;
 }
 
-template <support::endianness Endianness>
+template <llvm::endianness Endianness>
 inline static void writePrefixedInstruction(char *Loc, uint64_t Inst) {
-  constexpr bool isLE = Endianness == support::endianness::little;
+  constexpr bool isLE = Endianness == llvm::endianness::little;
   Inst = isLE ? (Inst << 32) | (Inst >> 32) : Inst;
   support::endian::write64<Endianness>(Loc, Inst);
 }
 
-template <support::endianness Endianness>
+template <llvm::endianness Endianness>
 inline Error relocateHalf16(char *FixupPtr, int64_t Value, Edge::Kind K) {
   switch (K) {
   case Delta16:
@@ -343,7 +353,7 @@ inline Error relocateHalf16(char *FixupPtr, int64_t Value, Edge::Kind K) {
 }
 
 /// Apply fixup expression for edge to block content.
-template <support::endianness Endianness>
+template <llvm::endianness Endianness>
 inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E,
                         const Symbol *TOCSymbol) {
   char *BlockWorkingMem = B.getAlreadyMutableContent().data();
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h
index 2f13560061..a6f097da1f 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h
@@ -343,7 +343,7 @@ class DLLImportDefinitionGenerator : public DefinitionGenerator {
       : ES(ES), L(L) {}
 
   static Expected<unsigned> getTargetPointerSize(const Triple &TT);
-  static Expected<support::endianness> getTargetEndianness(const Triple &TT);
+  static Expected<llvm::endianness> getTargetEndianness(const Triple &TT);
   Expected<std::unique_ptr<jitlink::LinkGraph>>
   createStubsGraph(const SymbolMap &Resolved);
 
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/MachOBuilder.h b/llvm/include/llvm/ExecutionEngine/Orc/MachOBuilder.h
index 8cbd9ed244..2bc66b11e2 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/MachOBuilder.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/MachOBuilder.h
@@ -507,7 +507,7 @@ struct MachO64LE {
   using NList = MachO::nlist_64;
   using Relocation = MachO::relocation_info;
 
-  static constexpr support::endianness Endianness = support::little;
+  static constexpr llvm::endianness Endianness = llvm::endianness::little;
   static constexpr uint32_t Magic = MachO::MH_MAGIC_64;
   static constexpr MachO::LoadCommandType SegmentCmd = MachO::LC_SEGMENT_64;
   static constexpr MachO::LoadCommandType SymTabCmd = MachO::LC_SYMTAB;
diff --git a/llvm/include/llvm/ExecutionEngine/RuntimeDyldChecker.h b/llvm/include/llvm/ExecutionEngine/RuntimeDyldChecker.h
index 85df587bff..faca037a56 100644
--- a/llvm/include/llvm/ExecutionEngine/RuntimeDyldChecker.h
+++ b/llvm/include/llvm/ExecutionEngine/RuntimeDyldChecker.h
@@ -163,9 +163,9 @@ class RuntimeDyldChecker {
                      GetSymbolInfoFunction GetSymbolInfo,
                      GetSectionInfoFunction GetSectionInfo,
                      GetStubInfoFunction GetStubInfo,
-                     GetGOTInfoFunction GetGOTInfo,
-                     support::endianness Endianness, Triple TT, StringRef CPU,
-                     SubtargetFeatures TF, raw_ostream &ErrStream);
+                     GetGOTInfoFunction GetGOTInfo, llvm::endianness Endianness,
+                     Triple TT, StringRef CPU, SubtargetFeatures TF,
+                     raw_ostream &ErrStream);
   ~RuntimeDyldChecker();
 
   /// Check a single expression against the attached RuntimeDyld
diff --git a/llvm/include/llvm/Frontend/OpenACC/ACC.td b/llvm/include/llvm/Frontend/OpenACC/ACC.td
index 62b4113be6..692ddefb1b 100644
--- a/llvm/include/llvm/Frontend/OpenACC/ACC.td
+++ b/llvm/include/llvm/Frontend/OpenACC/ACC.td
@@ -159,7 +159,7 @@ def ACCC_Host : Clause<"host"> {
 
 // 2.5.6
 def ACCC_If : Clause <"if"> {
-  let flangClass = "ScalarLogicalExpr";
+  let flangClass = "ScalarExpr";
 }
 
 // 2.14.4
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index b505daae7e..00b4707a7f 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -1917,8 +1917,18 @@ class OpenMPIRBuilder {
   ///
   /// \param Loc The location where the teams construct was encountered.
   /// \param BodyGenCB Callback that will generate the region code.
-  InsertPointTy createTeams(const LocationDescription &Loc,
-                            BodyGenCallbackTy BodyGenCB);
+  /// \param NumTeamsLower Lower bound on number of teams. If this is nullptr,
+  ///        it is as if lower bound is specified as equal to upperbound. If
+  ///        this is non-null, then upperbound must also be non-null.
+  /// \param NumTeamsUpper Upper bound on the number of teams.
+  /// \param ThreadLimit on the number of threads that may participate in a
+  ///        contention group created by each team.
+  /// \param IfExpr is the integer argument value of the if condition on the
+  ///        teams clause.
+  InsertPointTy
+  createTeams(const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
+              Value *NumTeamsLower = nullptr, Value *NumTeamsUpper = nullptr,
+              Value *ThreadLimit = nullptr, Value *IfExpr = nullptr);
 
   /// Generate conditional branch and relevant BasicBlocks through which private
   /// threads copy the 'copyin' variables from Master copy to threadprivate
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
index 176b883fe6..4823c4cc6b 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
@@ -387,6 +387,7 @@ __OMP_RTL(__kmpc_cancellationpoint, false, Int32, IdentPtr, Int32, Int32)
 
 __OMP_RTL(__kmpc_fork_teams, true, Void, IdentPtr, Int32, ParallelTaskPtr)
 __OMP_RTL(__kmpc_push_num_teams, false, Void, IdentPtr, Int32, Int32, Int32)
+__OMP_RTL(__kmpc_push_num_teams_51, false, Void, IdentPtr, Int32, Int32, Int32, Int32)
 __OMP_RTL(__kmpc_set_thread_limit, false, Void, IdentPtr, Int32, Int32)
 
 __OMP_RTL(__kmpc_copyprivate, false, Void, IdentPtr, Int32, SizeTy, VoidPtr,
diff --git a/llvm/include/llvm/IR/Attributes.td b/llvm/include/llvm/IR/Attributes.td
index aba1d718f7..fda79f5f24 100644
--- a/llvm/include/llvm/IR/Attributes.td
+++ b/llvm/include/llvm/IR/Attributes.td
@@ -200,6 +200,9 @@ def NoSanitizeCoverage : EnumAttr<"nosanitize_coverage", [FnAttr]>;
 /// Null pointer in address space zero is valid.
 def NullPointerIsValid : EnumAttr<"null_pointer_is_valid", [FnAttr]>;
 
+/// Select optimizations that give decent debug info.
+def OptimizeForDebugging : EnumAttr<"optdebug", [FnAttr]>;
+
 /// Select optimizations for best fuzzing signal.
 def OptForFuzzing : EnumAttr<"optforfuzzing", [FnAttr]>;
 
diff --git a/llvm/include/llvm/IR/BasicBlock.h b/llvm/include/llvm/IR/BasicBlock.h
index b031f72493..ab291c24e5 100644
--- a/llvm/include/llvm/IR/BasicBlock.h
+++ b/llvm/include/llvm/IR/BasicBlock.h
@@ -55,7 +55,7 @@ class ValueSymbolTable;
 class BasicBlock final : public Value, // Basic blocks are data objects also
                          public ilist_node_with_parent<BasicBlock, Function> {
 public:
-  using InstListType = SymbolTableList<Instruction>;
+  using InstListType = SymbolTableList<Instruction, ilist_iterator_bits<true>>;
 
 private:
   friend class BlockAddress;
@@ -91,11 +91,13 @@ class BasicBlock final : public Value, // Basic blocks are data objects also
 
   // These functions and classes need access to the instruction list.
   friend void Instruction::removeFromParent();
-  friend iplist<Instruction>::iterator Instruction::eraseFromParent();
+  friend BasicBlock::iterator Instruction::eraseFromParent();
   friend BasicBlock::iterator Instruction::insertInto(BasicBlock *BB,
                                                       BasicBlock::iterator It);
-  friend class llvm::SymbolTableListTraits<llvm::Instruction>;
-  friend class llvm::ilist_node_with_parent<llvm::Instruction, llvm::BasicBlock>;
+  friend class llvm::SymbolTableListTraits<llvm::Instruction,
+                                           ilist_iterator_bits<true>>;
+  friend class llvm::ilist_node_with_parent<llvm::Instruction, llvm::BasicBlock,
+                                            ilist_iterator_bits<true>>;
 
   /// Creates a new BasicBlock.
   ///
@@ -178,7 +180,8 @@ class BasicBlock final : public Value, // Basic blocks are data objects also
   InstListType::const_iterator getFirstNonPHIIt() const;
   InstListType::iterator getFirstNonPHIIt() {
     BasicBlock::iterator It =
-      static_cast<const BasicBlock *>(this)->getFirstNonPHIIt().getNonConst();
+        static_cast<const BasicBlock *>(this)->getFirstNonPHIIt().getNonConst();
+    It.setHeadBit(true);
     return It;
   }
 
@@ -332,8 +335,19 @@ class BasicBlock final : public Value, // Basic blocks are data objects also
   //===--------------------------------------------------------------------===//
   /// Instruction iterator methods
   ///
-  inline iterator                begin()       { return InstList.begin(); }
-  inline const_iterator          begin() const { return InstList.begin(); }
+  inline iterator begin() {
+    iterator It = InstList.begin();
+    // Set the head-inclusive bit to indicate that this iterator includes
+    // any debug-info at the start of the block. This is a no-op unless the
+    // appropriate CMake flag is set.
+    It.setHeadBit(true);
+    return It;
+  }
+  inline const_iterator begin() const {
+    const_iterator It = InstList.begin();
+    It.setHeadBit(true);
+    return It;
+  }
   inline iterator                end  ()       { return InstList.end();   }
   inline const_iterator          end  () const { return InstList.end();   }
 
diff --git a/llvm/include/llvm/IR/CallingConv.h b/llvm/include/llvm/IR/CallingConv.h
index e97623b29f..40222fa31d 100644
--- a/llvm/include/llvm/IR/CallingConv.h
+++ b/llvm/include/llvm/IR/CallingConv.h
@@ -245,6 +245,9 @@ namespace CallingConv {
     /// placement. Preserves active lane values for input VGPRs.
     AMDGPU_CS_ChainPreserve = 105,
 
+    /// Used for M68k rtd-based CC (similar to X86's stdcall).
+    M68k_RTD = 106,
+
     /// The highest possible ID. Must be some 2^k - 1.
     MaxID = 1023
   };
diff --git a/llvm/include/llvm/IR/DerivedTypes.h b/llvm/include/llvm/IR/DerivedTypes.h
index 203a73067e..f7a09fda8c 100644
--- a/llvm/include/llvm/IR/DerivedTypes.h
+++ b/llvm/include/llvm/IR/DerivedTypes.h
@@ -132,7 +132,10 @@ class FunctionType : public Type {
   }
 
   /// Parameter type accessors.
-  Type *getParamType(unsigned i) const { return ContainedTys[i+1]; }
+  Type *getParamType(unsigned i) const {
+    assert(i < getNumParams() && "getParamType() out of range!");
+    return ContainedTys[i + 1];
+  }
 
   /// Return the number of fixed parameters this function type requires.
   /// This does not consider varargs.
diff --git a/llvm/include/llvm/IR/GenericConvergenceVerifierImpl.h b/llvm/include/llvm/IR/GenericConvergenceVerifierImpl.h
index 2ba81015cb..e2ece30b18 100644
--- a/llvm/include/llvm/IR/GenericConvergenceVerifierImpl.h
+++ b/llvm/include/llvm/IR/GenericConvergenceVerifierImpl.h
@@ -31,8 +31,6 @@
 #include "llvm/ADT/Twine.h"
 #include "llvm/IR/Intrinsics.h"
 
-using namespace llvm;
-
 #define Check(C, ...)                                                          \
   do {                                                                         \
     if (!(C)) {                                                                \
@@ -49,6 +47,7 @@ using namespace llvm;
     }                                                                          \
   } while (false)
 
+namespace llvm {
 static bool isConvergenceControlIntrinsic(unsigned IntrinsicID) {
   switch (IntrinsicID) {
   default:
@@ -60,7 +59,6 @@ static bool isConvergenceControlIntrinsic(unsigned IntrinsicID) {
   }
 }
 
-namespace llvm {
 template <class ContextT> void GenericConvergenceVerifier<ContextT>::clear() {
   Tokens.clear();
   CI.clear();
diff --git a/llvm/include/llvm/IR/GlobalAlias.h b/llvm/include/llvm/IR/GlobalAlias.h
index de405da5ca..583d66e281 100644
--- a/llvm/include/llvm/IR/GlobalAlias.h
+++ b/llvm/include/llvm/IR/GlobalAlias.h
@@ -23,7 +23,7 @@ namespace llvm {
 
 class Twine;
 class Module;
-template <typename ValueSubClass> class SymbolTableListTraits;
+template <typename ValueSubClass, typename... Args> class SymbolTableListTraits;
 
 class GlobalAlias : public GlobalValue, public ilist_node<GlobalAlias> {
   friend class SymbolTableListTraits<GlobalAlias>;
diff --git a/llvm/include/llvm/IR/GlobalIFunc.h b/llvm/include/llvm/IR/GlobalIFunc.h
index c148ee7907..4d1982da0b 100644
--- a/llvm/include/llvm/IR/GlobalIFunc.h
+++ b/llvm/include/llvm/IR/GlobalIFunc.h
@@ -29,7 +29,7 @@ class Twine;
 class Module;
 
 // Traits class for using GlobalIFunc in symbol table in Module.
-template <typename ValueSubClass> class SymbolTableListTraits;
+template <typename ValueSubClass, typename... Args> class SymbolTableListTraits;
 
 class GlobalIFunc final : public GlobalObject, public ilist_node<GlobalIFunc> {
   friend class SymbolTableListTraits<GlobalIFunc>;
diff --git a/llvm/include/llvm/IR/GlobalVariable.h b/llvm/include/llvm/IR/GlobalVariable.h
index 03c680e4f9..5ddffd16ac 100644
--- a/llvm/include/llvm/IR/GlobalVariable.h
+++ b/llvm/include/llvm/IR/GlobalVariable.h
@@ -33,18 +33,19 @@ namespace llvm {
 class Constant;
 class Module;
 
-template <typename ValueSubClass> class SymbolTableListTraits;
+template <typename ValueSubClass, typename... Args> class SymbolTableListTraits;
 class DIGlobalVariableExpression;
 
 class GlobalVariable : public GlobalObject, public ilist_node<GlobalVariable> {
   friend class SymbolTableListTraits<GlobalVariable>;
 
   AttributeSet Attrs;
-  bool isConstantGlobal : 1;                   // Is this a global constant?
-  bool isExternallyInitializedConstant : 1;    // Is this a global whose value
-                                               // can change from its initial
-                                               // value before global
-                                               // initializers are run?
+
+  // Is this a global constant?
+  bool isConstantGlobal : 1;
+  // Is this a global whose value can change from its initial value before
+  // global initializers are run?
+  bool isExternallyInitializedConstant : 1;
 
 public:
   /// GlobalVariable ctor - If a parent module is specified, the global is
diff --git a/llvm/include/llvm/IR/Instruction.h b/llvm/include/llvm/IR/Instruction.h
index 69c3af5b76..af7aa791cb 100644
--- a/llvm/include/llvm/IR/Instruction.h
+++ b/llvm/include/llvm/IR/Instruction.h
@@ -39,7 +39,11 @@ template <> struct ilist_alloc_traits<Instruction> {
 };
 
 class Instruction : public User,
-                    public ilist_node_with_parent<Instruction, BasicBlock> {
+                    public ilist_node_with_parent<Instruction, BasicBlock,
+                                                  ilist_iterator_bits<true>> {
+public:
+  using InstListType = SymbolTableList<Instruction, ilist_iterator_bits<true>>;
+private:
   BasicBlock *Parent;
   DebugLoc DbgLoc;                         // 'dbg' Metadata cache.
 
@@ -118,12 +122,12 @@ class Instruction : public User,
   /// This method unlinks 'this' from the containing basic block and deletes it.
   ///
   /// \returns an iterator pointing to the element after the erased one
-  SymbolTableList<Instruction>::iterator eraseFromParent();
+  InstListType::iterator eraseFromParent();
 
   /// Insert an unlinked instruction into a basic block immediately before
   /// the specified instruction.
   void insertBefore(Instruction *InsertPos);
-  void insertBefore(SymbolTableList<Instruction>::iterator InsertPos) {
+  void insertBefore(InstListType::iterator InsertPos) {
     insertBefore(&*InsertPos);
   }
 
@@ -133,11 +137,10 @@ class Instruction : public User,
 
   /// Inserts an unlinked instruction into \p ParentBB at position \p It and
   /// returns the iterator of the inserted instruction.
-  SymbolTableList<Instruction>::iterator
-  insertInto(BasicBlock *ParentBB, SymbolTableList<Instruction>::iterator It);
+  InstListType::iterator insertInto(BasicBlock *ParentBB,
+                                    InstListType::iterator It);
 
-  void insertBefore(BasicBlock &BB,
-                    SymbolTableList<Instruction>::iterator InsertPos) {
+  void insertBefore(BasicBlock &BB, InstListType::iterator InsertPos) {
     insertInto(&BB, InsertPos);
   }
 
@@ -157,10 +160,10 @@ class Instruction : public User,
   /// Unlink this instruction and insert into BB before I.
   ///
   /// \pre I is a valid iterator into BB.
-  void moveBefore(BasicBlock &BB, SymbolTableList<Instruction>::iterator I);
+  void moveBefore(BasicBlock &BB, InstListType::iterator I);
 
   /// (See other overload for moveBeforePreserving).
-  void moveBeforePreserving(BasicBlock &BB, SymbolTableList<Instruction>::iterator I) {
+  void moveBeforePreserving(BasicBlock &BB, InstListType::iterator I) {
     moveBefore(BB, I);
   }
 
@@ -902,7 +905,7 @@ class Instruction : public User,
   };
 
 private:
-  friend class SymbolTableListTraits<Instruction>;
+  friend class SymbolTableListTraits<Instruction, ilist_iterator_bits<true>>;
   friend class BasicBlock; // For renumbering.
 
   // Shadow Value::setValueSubclassData with a private forwarding method so that
diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h
index c85727ce30..af6ac566a0 100644
--- a/llvm/include/llvm/IR/Instructions.h
+++ b/llvm/include/llvm/IR/Instructions.h
@@ -3661,7 +3661,7 @@ class SwitchInstProfUpdateWrapper {
 
   /// Delegate the call to the underlying SwitchInst::eraseFromParent() and mark
   /// this object to not touch the underlying SwitchInst in destructor.
-  SymbolTableList<Instruction>::iterator eraseFromParent();
+  Instruction::InstListType::iterator eraseFromParent();
 
   void setSuccessorWeight(unsigned idx, CaseWeightOpt W);
   CaseWeightOpt getSuccessorWeight(unsigned idx);
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index ab15b1f1e0..80c3d8d403 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -720,13 +720,11 @@ def int_gcwrite : Intrinsic<[],
 // eliminate retain and releases where possible.
 
 def int_objc_autorelease                    : Intrinsic<[llvm_ptr_ty],
-                                                        [llvm_ptr_ty],
-                                                        [Returned<ArgIndex<0>>]>;
+                                                        [llvm_ptr_ty]>;
 def int_objc_autoreleasePoolPop             : Intrinsic<[], [llvm_ptr_ty]>;
 def int_objc_autoreleasePoolPush            : Intrinsic<[llvm_ptr_ty], []>;
 def int_objc_autoreleaseReturnValue         : Intrinsic<[llvm_ptr_ty],
-                                                        [llvm_ptr_ty],
-                                                        [Returned<ArgIndex<0>>]>;
+                                                        [llvm_ptr_ty]>;
 def int_objc_copyWeak                       : Intrinsic<[],
                                                         [llvm_ptr_ty,
                                                          llvm_ptr_ty]>;
@@ -743,17 +741,13 @@ def int_objc_moveWeak                       : Intrinsic<[],
                                                          llvm_ptr_ty]>;
 def int_objc_release                        : Intrinsic<[], [llvm_ptr_ty]>;
 def int_objc_retain                         : Intrinsic<[llvm_ptr_ty],
-                                                        [llvm_ptr_ty],
-                                                        [Returned<ArgIndex<0>>]>;
+                                                        [llvm_ptr_ty]>;
 def int_objc_retainAutorelease              : Intrinsic<[llvm_ptr_ty],
-                                                        [llvm_ptr_ty],
-                                                        [Returned<ArgIndex<0>>]>;
+                                                        [llvm_ptr_ty]>;
 def int_objc_retainAutoreleaseReturnValue   : Intrinsic<[llvm_ptr_ty],
-                                                        [llvm_ptr_ty],
-                                                        [Returned<ArgIndex<0>>]>;
+                                                        [llvm_ptr_ty]>;
 def int_objc_retainAutoreleasedReturnValue  : Intrinsic<[llvm_ptr_ty],
-                                                        [llvm_ptr_ty],
-                                                        [Returned<ArgIndex<0>>]>;
+                                                        [llvm_ptr_ty]>;
 def int_objc_retainBlock                    : Intrinsic<[llvm_ptr_ty],
                                                         [llvm_ptr_ty]>;
 def int_objc_storeStrong                    : Intrinsic<[],
@@ -768,8 +762,7 @@ def int_objc_clang_arc_noop_use : DefaultAttrsIntrinsic<[],
                                                         [llvm_vararg_ty],
                                                         [IntrInaccessibleMemOnly]>;
 def int_objc_unsafeClaimAutoreleasedReturnValue : Intrinsic<[llvm_ptr_ty],
-                                                            [llvm_ptr_ty],
-                                                            [Returned<ArgIndex<0>>]>;
+                                                            [llvm_ptr_ty]>;
 def int_objc_retainedObject                 : Intrinsic<[llvm_ptr_ty],
                                                         [llvm_ptr_ty]>;
 def int_objc_unretainedObject               : Intrinsic<[llvm_ptr_ty],
@@ -777,8 +770,7 @@ def int_objc_unretainedObject               : Intrinsic<[llvm_ptr_ty],
 def int_objc_unretainedPointer              : Intrinsic<[llvm_ptr_ty],
                                                         [llvm_ptr_ty]>;
 def int_objc_retain_autorelease             : Intrinsic<[llvm_ptr_ty],
-                                                        [llvm_ptr_ty],
-                                                        [Returned<ArgIndex<0>>]>;
+                                                        [llvm_ptr_ty]>;
 def int_objc_sync_enter                     : Intrinsic<[llvm_i32_ty],
                                                         [llvm_ptr_ty]>;
 def int_objc_sync_exit                      : Intrinsic<[llvm_i32_ty],
@@ -1697,7 +1689,7 @@ def int_experimental_deoptimize : Intrinsic<[llvm_any_ty], [llvm_vararg_ty],
                                             [Throws]>;
 
 // Support for speculative runtime guards
-def int_experimental_guard : DefaultAttrsIntrinsic<[], [llvm_i1_ty, llvm_vararg_ty],
+def int_experimental_guard : Intrinsic<[], [llvm_i1_ty, llvm_vararg_ty],
                                        [Throws]>;
 
 // Supports widenable conditions for guards represented as explicit branches.
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index 557063c881..a42e2c49cb 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -3011,6 +3011,16 @@ let TargetPrefix = "aarch64" in {
                             [llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
                             [IntrNoMem]>;
 
+  class SME2_BFMLS_Intrinsic
+    : DefaultAttrsIntrinsic<[llvm_nxv4f32_ty],
+                            [llvm_nxv4f32_ty, llvm_nxv8bf16_ty, llvm_nxv8bf16_ty],
+                            [IntrNoMem]>;
+
+  class SME2_BFMLS_Lane_Intrinsic
+    : DefaultAttrsIntrinsic<[llvm_nxv4f32_ty],
+                            [llvm_nxv4f32_ty, llvm_nxv8bf16_ty, llvm_nxv8bf16_ty, llvm_i32_ty],
+                            [IntrNoMem, ImmArg<ArgIndex<3>>]>;
+
   class SME2_ZA_ArrayVector_Read_VG2_Intrinsic
     : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>],
                 [llvm_i32_ty],
@@ -3214,6 +3224,12 @@ let TargetPrefix = "aarch64" in {
   def int_aarch64_sme_usmla_za32_lane_vg4x2 : SME2_Matrix_ArrayVector_VG2_Multi_Index_Intrinsic;
   def int_aarch64_sme_usmla_za32_lane_vg4x4 : SME2_Matrix_ArrayVector_VG4_Multi_Index_Intrinsic;
 
+  def int_aarch64_sve_bfmlslb : SME2_BFMLS_Intrinsic;
+  def int_aarch64_sve_bfmlslb_lane : SME2_BFMLS_Lane_Intrinsic;
+
+  def int_aarch64_sve_bfmlslt : SME2_BFMLS_Intrinsic;
+  def int_aarch64_sve_bfmlslt_lane : SME2_BFMLS_Lane_Intrinsic;
+
   // Multi-vector signed saturating doubling multiply high
 
   def int_aarch64_sve_sqdmulh_single_vgx2 : SME2_VG2_Multi_Single_Intrinsic;
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index 0df66e1ffc..5f1d1d932f 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -1174,9 +1174,9 @@ class AMDGPUStructPtrBufferStore<LLVMType data_ty = llvm_any_ty> : DefaultAttrsI
 def int_amdgcn_struct_ptr_buffer_store_format : AMDGPUStructPtrBufferStore;
 def int_amdgcn_struct_ptr_buffer_store : AMDGPUStructPtrBufferStore;
 
-class AMDGPURawBufferAtomic<LLVMType data_ty = llvm_any_ty, bit NoRtn = false> : Intrinsic <
-  !if(NoRtn, [], [data_ty]),
-  [!if(NoRtn, data_ty, LLVMMatchType<0>),  // vdata(VGPR)
+class AMDGPURawBufferAtomic<LLVMType data_ty = llvm_any_ty> : Intrinsic <
+  [data_ty],
+  [LLVMMatchType<0>,  // vdata(VGPR)
    llvm_v4i32_ty,     // rsrc(SGPR)
    llvm_i32_ty,       // offset(VGPR/imm, included in bounds checking and swizzling)
    llvm_i32_ty,       // soffset(SGPR/imm, excluded from bounds checking and swizzling)
@@ -1208,9 +1208,9 @@ def int_amdgcn_raw_buffer_atomic_cmpswap : Intrinsic<
   [ImmArg<ArgIndex<5>>, IntrWillReturn, IntrNoCallback, IntrNoFree], "", [SDNPMemOperand]>,
   AMDGPURsrcIntrinsic<2, 0>;
 
-class AMDGPURawPtrBufferAtomic<LLVMType data_ty = llvm_any_ty, bit NoRtn = false> : Intrinsic <
-  !if(NoRtn, [], [data_ty]),
-  [!if(NoRtn, data_ty, LLVMMatchType<0>),  // vdata(VGPR)
+class AMDGPURawPtrBufferAtomic<LLVMType data_ty = llvm_any_ty> : Intrinsic <
+  [data_ty],
+  [LLVMMatchType<0>,            // vdata(VGPR)
    AMDGPUBufferRsrcTy,          // rsrc(SGPR)
    llvm_i32_ty,                 // offset(VGPR/imm, included in bounds checking and swizzling)
    llvm_i32_ty,                 // soffset(SGPR/imm, excluded from bounds checking and swizzling)
@@ -1249,9 +1249,9 @@ def int_amdgcn_raw_ptr_buffer_atomic_cmpswap : Intrinsic<
 def int_amdgcn_raw_buffer_atomic_fadd : AMDGPURawBufferAtomic<llvm_anyfloat_ty>;
 def int_amdgcn_raw_ptr_buffer_atomic_fadd : AMDGPURawPtrBufferAtomic<llvm_anyfloat_ty>;
 
-class AMDGPUStructBufferAtomic<LLVMType data_ty = llvm_any_ty, bit NoRtn = false> : Intrinsic <
-  !if(NoRtn, [], [data_ty]),
-  [!if(NoRtn, data_ty, LLVMMatchType<0>),  // vdata(VGPR)
+class AMDGPUStructBufferAtomic<LLVMType data_ty = llvm_any_ty> : Intrinsic <
+  [data_ty],
+  [LLVMMatchType<0>,  // vdata(VGPR)
    llvm_v4i32_ty,     // rsrc(SGPR)
    llvm_i32_ty,       // vindex(VGPR)
    llvm_i32_ty,       // offset(VGPR/imm, included in bounds checking and swizzling)
@@ -1283,9 +1283,9 @@ def int_amdgcn_struct_buffer_atomic_cmpswap : Intrinsic<
   [ImmArg<ArgIndex<6>>, IntrWillReturn, IntrNoCallback, IntrNoFree], "", [SDNPMemOperand]>,
   AMDGPURsrcIntrinsic<2, 0>;
 
-class AMDGPUStructPtrBufferAtomic<LLVMType data_ty = llvm_any_ty, bit NoRtn = false> : Intrinsic <
-  !if(NoRtn, [], [data_ty]),
-  [!if(NoRtn, data_ty, LLVMMatchType<0>),  // vdata(VGPR)
+class AMDGPUStructPtrBufferAtomic<LLVMType data_ty = llvm_any_ty> : Intrinsic <
+  [data_ty],
+  [LLVMMatchType<0>,            // vdata(VGPR)
    AMDGPUBufferRsrcTy,          // rsrc(SGPR)
    llvm_i32_ty,                 // vindex(VGPR)
    llvm_i32_ty,                 // offset(VGPR/imm, included in bounds checking and swizzling)
diff --git a/llvm/include/llvm/IR/IntrinsicsRISCVXsf.td b/llvm/include/llvm/IR/IntrinsicsRISCVXsf.td
index c8d24ec7d8..4381c55a3f 100644
--- a/llvm/include/llvm/IR/IntrinsicsRISCVXsf.td
+++ b/llvm/include/llvm/IR/IntrinsicsRISCVXsf.td
@@ -128,8 +128,28 @@ let TargetPrefix = "riscv" in {
     }
   }
 
+  class RISCVSFCustomVMACC
+      : DefaultAttrsIntrinsic< [llvm_anyvector_ty],
+                   [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty,
+                    llvm_anyint_ty, LLVMMatchType<3>],
+                   [ImmArg<ArgIndex<4>>, IntrNoMem] >, RISCVVIntrinsic {
+    let VLOperand = 3;
+  }
+
   defm "" : RISCVSFCustomVC_X<["x", "i"]>;
   defm "" : RISCVSFCustomVC_XV<["x", "i", "v", "f"]>;
   defm "" : RISCVSFCustomVC_XVV<["x", "i", "v", "f"]>;
   defm "" : RISCVSFCustomVC_XVW<["x", "i", "v", "f"]>;
+
+  // XSfvqmaccdod
+  def int_riscv_sf_vqmaccu_2x8x2  : RISCVSFCustomVMACC;
+  def int_riscv_sf_vqmacc_2x8x2   : RISCVSFCustomVMACC;
+  def int_riscv_sf_vqmaccus_2x8x2 : RISCVSFCustomVMACC;
+  def int_riscv_sf_vqmaccsu_2x8x2 : RISCVSFCustomVMACC;
+
+  // XSfvqmaccqoq
+  def int_riscv_sf_vqmaccu_4x8x4  : RISCVSFCustomVMACC;
+  def int_riscv_sf_vqmacc_4x8x4   : RISCVSFCustomVMACC;
+  def int_riscv_sf_vqmaccus_4x8x4 : RISCVSFCustomVMACC;
+  def int_riscv_sf_vqmaccsu_4x8x4 : RISCVSFCustomVMACC;
 } // TargetPrefix = "riscv"
diff --git a/llvm/include/llvm/IR/IntrinsicsX86.td b/llvm/include/llvm/IR/IntrinsicsX86.td
index 57cd1dc47b..fdc2b0fb7f 100644
--- a/llvm/include/llvm/IR/IntrinsicsX86.td
+++ b/llvm/include/llvm/IR/IntrinsicsX86.td
@@ -5673,8 +5673,16 @@ let TargetPrefix = "x86" in {
               Intrinsic<[], [llvm_i64_ty], []>;
 }
 
+let TargetPrefix = "x86" in {
+def int_x86_urdmsr : ClangBuiltin<"__builtin_ia32_urdmsr">,
+        Intrinsic<[llvm_i64_ty], [llvm_i64_ty],
+                  [IntrInaccessibleMemOnly]>;
+def int_x86_uwrmsr : ClangBuiltin<"__builtin_ia32_uwrmsr">,
+        Intrinsic<[], [llvm_i64_ty, llvm_i64_ty],
+                  [IntrInaccessibleMemOnly]>;
+}
+
 //===----------------------------------------------------------------------===//
-// avx512_fp16: vaddph
 let TargetPrefix = "x86" in {
   def int_x86_avx512fp16_add_ph_512
       : ClangBuiltin<"__builtin_ia32_addph512">,
diff --git a/llvm/include/llvm/IR/SymbolTableListTraits.h b/llvm/include/llvm/IR/SymbolTableListTraits.h
index 8af712374b..bd31fca5e5 100644
--- a/llvm/include/llvm/IR/SymbolTableListTraits.h
+++ b/llvm/include/llvm/IR/SymbolTableListTraits.h
@@ -57,15 +57,16 @@ DEFINE_SYMBOL_TABLE_PARENT_TYPE(GlobalAlias, Module)
 DEFINE_SYMBOL_TABLE_PARENT_TYPE(GlobalIFunc, Module)
 #undef DEFINE_SYMBOL_TABLE_PARENT_TYPE
 
-template <typename NodeTy> class SymbolTableList;
+template <typename NodeTy, typename... Args> class SymbolTableList;
 
 // ValueSubClass   - The type of objects that I hold, e.g. Instruction.
 // ItemParentClass - The type of object that owns the list, e.g. BasicBlock.
+// OptionsT        - Extra options to ilist nodes.
 //
-template <typename ValueSubClass>
+template <typename ValueSubClass, typename... Args>
 class SymbolTableListTraits : public ilist_alloc_traits<ValueSubClass> {
-  using ListTy = SymbolTableList<ValueSubClass>;
-  using iterator = typename simple_ilist<ValueSubClass>::iterator;
+  using ListTy = SymbolTableList<ValueSubClass, Args...>;
+  using iterator = typename simple_ilist<ValueSubClass, Args...>::iterator;
   using ItemParentClass =
       typename SymbolTableListParentType<ValueSubClass>::type;
 
@@ -110,9 +111,10 @@ class SymbolTableListTraits : public ilist_alloc_traits<ValueSubClass> {
 /// When nodes are inserted into and removed from this list, the associated
 /// symbol table will be automatically updated.  Similarly, parent links get
 /// updated automatically.
-template <class T>
-class SymbolTableList
-    : public iplist_impl<simple_ilist<T>, SymbolTableListTraits<T>> {};
+template <class T, typename... Args>
+class SymbolTableList : public iplist_impl<simple_ilist<T, Args...>,
+                                           SymbolTableListTraits<T, Args...>> {
+};
 
 } // end namespace llvm
 
diff --git a/llvm/include/llvm/IR/ValueSymbolTable.h b/llvm/include/llvm/IR/ValueSymbolTable.h
index 43d00268f4..6350f6a243 100644
--- a/llvm/include/llvm/IR/ValueSymbolTable.h
+++ b/llvm/include/llvm/IR/ValueSymbolTable.h
@@ -27,8 +27,9 @@ class GlobalAlias;
 class GlobalIFunc;
 class GlobalVariable;
 class Instruction;
+template <bool ExtraIteratorBits> struct ilist_iterator_bits;
 template <unsigned InternalLen> class SmallString;
-template <typename ValueSubClass> class SymbolTableListTraits;
+template <typename ValueSubClass, typename ... Args> class SymbolTableListTraits;
 
 /// This class provides a symbol table of name/value pairs. It is essentially
 /// a std::map<std::string,Value*> but has a controlled interface provided by
@@ -41,7 +42,7 @@ class ValueSymbolTable {
   friend class SymbolTableListTraits<GlobalAlias>;
   friend class SymbolTableListTraits<GlobalIFunc>;
   friend class SymbolTableListTraits<GlobalVariable>;
-  friend class SymbolTableListTraits<Instruction>;
+  friend class SymbolTableListTraits<Instruction, ilist_iterator_bits<true>>;
   friend class Value;
 
 /// @name Types
diff --git a/llvm/include/llvm/IRReader/IRReader.h b/llvm/include/llvm/IRReader/IRReader.h
index 644fea82bf..4d690dcaf1 100644
--- a/llvm/include/llvm/IRReader/IRReader.h
+++ b/llvm/include/llvm/IRReader/IRReader.h
@@ -14,7 +14,6 @@
 #ifndef LLVM_IRREADER_IRREADER_H
 #define LLVM_IRREADER_IRREADER_H
 
-#include "llvm/ADT/STLFunctionalExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Bitcode/BitcodeReader.h"
 #include <memory>
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index db653fff71..9fbd431c22 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -188,7 +188,6 @@ void initializeLoopUnrollPass(PassRegistry&);
 void initializeLowerAtomicLegacyPassPass(PassRegistry&);
 void initializeLowerConstantIntrinsicsPass(PassRegistry&);
 void initializeLowerEmuTLSPass(PassRegistry&);
-void initializeLowerExpectIntrinsicPass(PassRegistry&);
 void initializeLowerGlobalDtorsLegacyPassPass(PassRegistry &);
 void initializeLowerWidenableConditionLegacyPassPass(PassRegistry&);
 void initializeLowerIntrinsicsPass(PassRegistry&);
diff --git a/llvm/include/llvm/LinkAllPasses.h b/llvm/include/llvm/LinkAllPasses.h
index 081a15077e..141b5420b9 100644
--- a/llvm/include/llvm/LinkAllPasses.h
+++ b/llvm/include/llvm/LinkAllPasses.h
@@ -99,7 +99,6 @@ namespace {
       (void) llvm::createLoopUnrollPass();
       (void) llvm::createLoopRotatePass();
       (void) llvm::createLowerConstantIntrinsicsPass();
-      (void) llvm::createLowerExpectIntrinsicPass();
       (void) llvm::createLowerGlobalDtorsLegacyPass();
       (void) llvm::createLowerInvokePass();
       (void) llvm::createLowerSwitchPass();
diff --git a/llvm/include/llvm/MC/MCAsmBackend.h b/llvm/include/llvm/MC/MCAsmBackend.h
index 5e08fb4167..b5c30aa756 100644
--- a/llvm/include/llvm/MC/MCAsmBackend.h
+++ b/llvm/include/llvm/MC/MCAsmBackend.h
@@ -41,15 +41,14 @@ class raw_ostream;
 /// Generic interface to target specific assembler backends.
 class MCAsmBackend {
 protected: // Can only create subclasses.
-  MCAsmBackend(support::endianness Endian,
-               unsigned RelaxFixupKind = MaxFixupKind);
+  MCAsmBackend(llvm::endianness Endian, unsigned RelaxFixupKind = MaxFixupKind);
 
 public:
   MCAsmBackend(const MCAsmBackend &) = delete;
   MCAsmBackend &operator=(const MCAsmBackend &) = delete;
   virtual ~MCAsmBackend();
 
-  const support::endianness Endian;
+  const llvm::endianness Endian;
 
   /// Fixup kind used for linker relaxation. Currently only used by RISC-V.
   const unsigned RelaxFixupKind;
diff --git a/llvm/include/llvm/MC/MCInstrAnalysis.h b/llvm/include/llvm/MC/MCInstrAnalysis.h
index c3c675c39c..e3ddf0b8b8 100644
--- a/llvm/include/llvm/MC/MCInstrAnalysis.h
+++ b/llvm/include/llvm/MC/MCInstrAnalysis.h
@@ -37,6 +37,21 @@ class MCInstrAnalysis {
   MCInstrAnalysis(const MCInstrInfo *Info) : Info(Info) {}
   virtual ~MCInstrAnalysis() = default;
 
+  /// Clear the internal state. See updateState for more information.
+  virtual void resetState() {}
+
+  /// Update internal state with \p Inst at \p Addr.
+  ///
+  /// For some types of analyses, inspecting a single instruction is not
+  /// sufficient. Some examples are auipc/jalr pairs on RISC-V or adrp/ldr pairs
+  /// on AArch64. To support inspecting multiple instructions, targets may keep
+  /// track of an internal state while analysing instructions. Clients should
+  /// call updateState for every instruction which allows later calls to one of
+  /// the analysis functions to take previous instructions into account.
+  /// Whenever state becomes irrelevant (e.g., when starting to disassemble a
+  /// new function), clients should call resetState to clear it.
+  virtual void updateState(const MCInst &Inst, uint64_t Addr) {}
+
   virtual bool isBranch(const MCInst &Inst) const {
     return Info->get(Inst.getOpcode()).isBranch();
   }
diff --git a/llvm/include/llvm/MC/MCMachObjectWriter.h b/llvm/include/llvm/MC/MCMachObjectWriter.h
index 05d816671b..1683543082 100644
--- a/llvm/include/llvm/MC/MCMachObjectWriter.h
+++ b/llvm/include/llvm/MC/MCMachObjectWriter.h
@@ -131,7 +131,8 @@ class MachObjectWriter : public MCObjectWriter {
       : TargetObjectWriter(std::move(MOTW)),
         StringTable(TargetObjectWriter->is64Bit() ? StringTableBuilder::MachO64
                                                   : StringTableBuilder::MachO),
-        W(OS, IsLittleEndian ? support::little : support::big) {}
+        W(OS,
+          IsLittleEndian ? llvm::endianness::little : llvm::endianness::big) {}
 
   support::endian::Writer W;
 
diff --git a/llvm/include/llvm/Object/COFF.h b/llvm/include/llvm/Object/COFF.h
index 24ed885d7b..a548b2c15c 100644
--- a/llvm/include/llvm/Object/COFF.h
+++ b/llvm/include/llvm/Object/COFF.h
@@ -1298,7 +1298,8 @@ class BaseRelocRef {
 class ResourceSectionRef {
 public:
   ResourceSectionRef() = default;
-  explicit ResourceSectionRef(StringRef Ref) : BBS(Ref, support::little) {}
+  explicit ResourceSectionRef(StringRef Ref)
+      : BBS(Ref, llvm::endianness::little) {}
 
   Error load(const COFFObjectFile *O);
   Error load(const COFFObjectFile *O, const SectionRef &S);
diff --git a/llvm/include/llvm/Object/COFFImportFile.h b/llvm/include/llvm/Object/COFFImportFile.h
index 3d148112dc..0fb65fabdb 100644
--- a/llvm/include/llvm/Object/COFFImportFile.h
+++ b/llvm/include/llvm/Object/COFFImportFile.h
@@ -63,6 +63,8 @@ class COFFImportFile : public SymbolicFile {
         Data.getBufferStart());
   }
 
+  uint16_t getMachine() const { return getCOFFImportHeader()->Machine; }
+
 private:
   bool isData() const {
     return getCOFFImportHeader()->getType() == COFF::IMPORT_DATA;
diff --git a/llvm/include/llvm/Object/ELFObjectFile.h b/llvm/include/llvm/Object/ELFObjectFile.h
index dc3d6bb587..d7947d8573 100644
--- a/llvm/include/llvm/Object/ELFObjectFile.h
+++ b/llvm/include/llvm/Object/ELFObjectFile.h
@@ -458,8 +458,9 @@ template <class ELFT> class ELFObjectFile : public ELFObjectFileBase {
 
   bool isDyldType() const { return isDyldELFObject; }
   static bool classof(const Binary *v) {
-    return v->getType() == getELFType(ELFT::TargetEndianness == support::little,
-                                      ELFT::Is64Bits);
+    return v->getType() ==
+           getELFType(ELFT::TargetEndianness == llvm::endianness::little,
+                      ELFT::Is64Bits);
   }
 
   elf_symbol_iterator_range getDynamicSymbolIterators() const override;
@@ -1128,7 +1129,8 @@ ELFObjectFile<ELFT>::ELFObjectFile(MemoryBufferRef Object, ELFFile<ELFT> EF,
                                    const Elf_Shdr *DotSymtabSec,
                                    const Elf_Shdr *DotSymtabShndx)
     : ELFObjectFileBase(
-          getELFType(ELFT::TargetEndianness == support::little, ELFT::Is64Bits),
+          getELFType(ELFT::TargetEndianness == llvm::endianness::little,
+                     ELFT::Is64Bits),
           Object),
       EF(EF), DotDynSymSec(DotDynSymSec), DotSymtabSec(DotSymtabSec),
       DotSymtabShndxSec(DotSymtabShndx) {}
@@ -1197,7 +1199,8 @@ uint8_t ELFObjectFile<ELFT>::getBytesInAddress() const {
 
 template <class ELFT>
 StringRef ELFObjectFile<ELFT>::getFileFormatName() const {
-  constexpr bool IsLittleEndian = ELFT::TargetEndianness == support::little;
+  constexpr bool IsLittleEndian =
+      ELFT::TargetEndianness == llvm::endianness::little;
   switch (EF.getHeader().e_ident[ELF::EI_CLASS]) {
   case ELF::ELFCLASS32:
     switch (EF.getHeader().e_machine) {
@@ -1275,7 +1278,7 @@ StringRef ELFObjectFile<ELFT>::getFileFormatName() const {
 }
 
 template <class ELFT> Triple::ArchType ELFObjectFile<ELFT>::getArch() const {
-  bool IsLittleEndian = ELFT::TargetEndianness == support::little;
+  bool IsLittleEndian = ELFT::TargetEndianness == llvm::endianness::little;
   switch (EF.getHeader().e_machine) {
   case ELF::EM_68K:
     return Triple::m68k;
diff --git a/llvm/include/llvm/Object/ELFTypes.h b/llvm/include/llvm/Object/ELFTypes.h
index 6295341e15..45fc52288b 100644
--- a/llvm/include/llvm/Object/ELFTypes.h
+++ b/llvm/include/llvm/Object/ELFTypes.h
@@ -90,10 +90,10 @@ template <endianness E, bool Is64> struct ELFType {
   using Off = packed<uint>;
 };
 
-using ELF32LE = ELFType<support::little, false>;
-using ELF32BE = ELFType<support::big, false>;
-using ELF64LE = ELFType<support::little, true>;
-using ELF64BE = ELFType<support::big, true>;
+using ELF32LE = ELFType<llvm::endianness::little, false>;
+using ELF32BE = ELFType<llvm::endianness::big, false>;
+using ELF64LE = ELFType<llvm::endianness::little, true>;
+using ELF64BE = ELFType<llvm::endianness::big, true>;
 
 // Use an alignment of 2 for the typedefs since that is the worst case for
 // ELF files in archives.
@@ -740,7 +740,7 @@ template <class ELFT> struct Elf_CGProfile_Impl {
 template <class ELFT>
 struct Elf_Mips_RegInfo;
 
-template <support::endianness TargetEndianness>
+template <llvm::endianness TargetEndianness>
 struct Elf_Mips_RegInfo<ELFType<TargetEndianness, false>> {
   LLVM_ELF_IMPORT_TYPES(TargetEndianness, false)
   Elf_Word ri_gprmask;     // bit-mask of used general registers
@@ -748,7 +748,7 @@ struct Elf_Mips_RegInfo<ELFType<TargetEndianness, false>> {
   Elf_Addr ri_gp_value;    // gp register value
 };
 
-template <support::endianness TargetEndianness>
+template <llvm::endianness TargetEndianness>
 struct Elf_Mips_RegInfo<ELFType<TargetEndianness, true>> {
   LLVM_ELF_IMPORT_TYPES(TargetEndianness, true)
   Elf_Word ri_gprmask;     // bit-mask of used general registers
diff --git a/llvm/include/llvm/Object/FaultMapParser.h b/llvm/include/llvm/Object/FaultMapParser.h
index 4d1f0397a0..028d3900d9 100644
--- a/llvm/include/llvm/Object/FaultMapParser.h
+++ b/llvm/include/llvm/Object/FaultMapParser.h
@@ -42,7 +42,7 @@ class FaultMapParser {
 
   template <typename T> static T read(const uint8_t *P, const uint8_t *E) {
     assert(P + sizeof(T) <= E && "out of bounds read!");
-    return support::endian::read<T, support::little, 1>(P);
+    return support::endian::read<T, llvm::endianness::little>(P);
   }
 
 public:
diff --git a/llvm/include/llvm/Object/GOFF.h b/llvm/include/llvm/Object/GOFF.h
index f4aa04cd99..91762457ae 100644
--- a/llvm/include/llvm/Object/GOFF.h
+++ b/llvm/include/llvm/Object/GOFF.h
@@ -69,8 +69,7 @@ class Record {
   static void get(const uint8_t *Bytes, uint8_t ByteIndex, T &Value) {
     assert(ByteIndex + sizeof(T) <= GOFF::RecordLength &&
            "Byte index out of bounds!");
-    Value = support::endian::read<T, support::big, support::unaligned>(
-        &Bytes[ByteIndex]);
+    Value = support::endian::read<T, llvm::endianness::big>(&Bytes[ByteIndex]);
   }
 };
 
diff --git a/llvm/include/llvm/Object/MachO.h b/llvm/include/llvm/Object/MachO.h
index 894252db53..f91f21d837 100644
--- a/llvm/include/llvm/Object/MachO.h
+++ b/llvm/include/llvm/Object/MachO.h
@@ -789,16 +789,11 @@ class MachOObjectFile : public ObjectFile {
 
   static std::string getBuildPlatform(uint32_t platform) {
     switch (platform) {
-    case MachO::PLATFORM_MACOS: return "macos";
-    case MachO::PLATFORM_IOS: return "ios";
-    case MachO::PLATFORM_TVOS: return "tvos";
-    case MachO::PLATFORM_WATCHOS: return "watchos";
-    case MachO::PLATFORM_BRIDGEOS: return "bridgeos";
-    case MachO::PLATFORM_MACCATALYST: return "macCatalyst";
-    case MachO::PLATFORM_IOSSIMULATOR: return "iossimulator";
-    case MachO::PLATFORM_TVOSSIMULATOR: return "tvossimulator";
-    case MachO::PLATFORM_WATCHOSSIMULATOR: return "watchossimulator";
-    case MachO::PLATFORM_DRIVERKIT: return "driverkit";
+#define PLATFORM(platform, id, name, build_name, target, tapi_target,          \
+                 marketing)                                                    \
+  case MachO::PLATFORM_##platform:                                             \
+    return #name;
+#include "llvm/BinaryFormat/MachO.def"
     default:
       std::string ret;
       raw_string_ostream ss(ret);
diff --git a/llvm/include/llvm/Object/StackMapParser.h b/llvm/include/llvm/Object/StackMapParser.h
index 3560675af5..8853e3656f 100644
--- a/llvm/include/llvm/Object/StackMapParser.h
+++ b/llvm/include/llvm/Object/StackMapParser.h
@@ -21,8 +21,7 @@
 namespace llvm {
 
 /// A parser for the latest stackmap format.  At the moment, latest=V3.
-template <support::endianness Endianness>
-class StackMapParser {
+template <llvm::endianness Endianness> class StackMapParser {
 public:
   template <typename AccessorT>
   class AccessorIterator {
@@ -437,7 +436,7 @@ class StackMapParser {
 private:
   template <typename T>
   static T read(const uint8_t *P) {
-    return support::endian::read<T, Endianness, 1>(P);
+    return support::endian::read<T, Endianness>(P);
   }
 
   static const unsigned HeaderOffset = 0;
diff --git a/llvm/include/llvm/Object/XCOFFObjectFile.h b/llvm/include/llvm/Object/XCOFFObjectFile.h
index 5f51aacfab..63064abb4d 100644
--- a/llvm/include/llvm/Object/XCOFFObjectFile.h
+++ b/llvm/include/llvm/Object/XCOFFObjectFile.h
@@ -15,6 +15,7 @@
 
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/iterator_range.h"
 #include "llvm/BinaryFormat/XCOFF.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/Endian.h"
@@ -23,6 +24,8 @@
 namespace llvm {
 namespace object {
 
+class xcoff_symbol_iterator;
+
 struct XCOFFFileHeader32 {
   support::ubig16_t Magic;
   support::ubig16_t NumberOfSections;
@@ -576,6 +579,10 @@ class XCOFFObjectFile : public ObjectFile {
   Expected<uint32_t> getSymbolFlags(DataRefImpl Symb) const override;
   basic_symbol_iterator symbol_begin() const override;
   basic_symbol_iterator symbol_end() const override;
+
+  using xcoff_symbol_iterator_range = iterator_range<xcoff_symbol_iterator>;
+  xcoff_symbol_iterator_range symbols() const;
+
   bool is64Bit() const override;
   Expected<StringRef> getSymbolName(DataRefImpl Symb) const override;
   Expected<uint64_t> getSymbolAddress(DataRefImpl Symb) const override;
@@ -761,33 +768,48 @@ struct XCOFFSymbolEntry64 {
   uint8_t NumberOfAuxEntries;
 };
 
-class XCOFFSymbolRef {
+class XCOFFSymbolRef : public SymbolRef {
 public:
   enum { NAME_IN_STR_TBL_MAGIC = 0x0 };
 
   XCOFFSymbolRef(DataRefImpl SymEntDataRef,
                  const XCOFFObjectFile *OwningObjectPtr)
-      : OwningObjectPtr(OwningObjectPtr) {
+      : SymbolRef(SymEntDataRef, OwningObjectPtr) {
     assert(OwningObjectPtr && "OwningObjectPtr cannot be nullptr!");
     assert(SymEntDataRef.p != 0 &&
            "Symbol table entry pointer cannot be nullptr!");
+  }
 
-    if (OwningObjectPtr->is64Bit())
-      Entry64 = reinterpret_cast<const XCOFFSymbolEntry64 *>(SymEntDataRef.p);
-    else
-      Entry32 = reinterpret_cast<const XCOFFSymbolEntry32 *>(SymEntDataRef.p);
+  const XCOFFSymbolEntry32 *getSymbol32() const {
+    return reinterpret_cast<const XCOFFSymbolEntry32 *>(getRawDataRefImpl().p);
   }
 
-  const XCOFFSymbolEntry32 *getSymbol32() { return Entry32; }
-  const XCOFFSymbolEntry64 *getSymbol64() { return Entry64; }
+  const XCOFFSymbolEntry64 *getSymbol64() const {
+    return reinterpret_cast<const XCOFFSymbolEntry64 *>(getRawDataRefImpl().p);
+  }
 
-  uint64_t getValue() const { return Entry32 ? getValue32() : getValue64(); }
+  uint64_t getValue() const {
+    return getObject()->is64Bit() ? getValue64() : getValue32();
+  }
 
-  uint32_t getValue32() const { return Entry32->Value; }
+  uint32_t getValue32() const {
+    return reinterpret_cast<const XCOFFSymbolEntry32 *>(getRawDataRefImpl().p)
+        ->Value;
+  }
 
-  uint64_t getValue64() const { return Entry64->Value; }
+  uint64_t getValue64() const {
+    return reinterpret_cast<const XCOFFSymbolEntry64 *>(getRawDataRefImpl().p)
+        ->Value;
+  }
 
-#define GETVALUE(X) Entry32 ? Entry32->X : Entry64->X
+  uint64_t getSize() const {
+    return getObject()->getSymbolSize(getRawDataRefImpl());
+  }
+
+#define GETVALUE(X)                                                            \
+  getObject()->is64Bit()                                                       \
+      ? reinterpret_cast<const XCOFFSymbolEntry64 *>(getRawDataRefImpl().p)->X \
+      : reinterpret_cast<const XCOFFSymbolEntry32 *>(getRawDataRefImpl().p)->X
 
   int16_t getSectionNumber() const { return GETVALUE(SectionNumber); }
 
@@ -812,8 +834,7 @@ class XCOFFSymbolRef {
 #undef GETVALUE
 
   uintptr_t getEntryAddress() const {
-    return Entry32 ? reinterpret_cast<uintptr_t>(Entry32)
-                   : reinterpret_cast<uintptr_t>(Entry64);
+    return getRawDataRefImpl().p;
   }
 
   Expected<StringRef> getName() const;
@@ -822,9 +843,23 @@ class XCOFFSymbolRef {
   Expected<XCOFFCsectAuxRef> getXCOFFCsectAuxRef() const;
 
 private:
-  const XCOFFObjectFile *OwningObjectPtr;
-  const XCOFFSymbolEntry32 *Entry32 = nullptr;
-  const XCOFFSymbolEntry64 *Entry64 = nullptr;
+  const XCOFFObjectFile *getObject() const {
+    return cast<XCOFFObjectFile>(BasicSymbolRef::getObject());
+  }
+};
+
+class xcoff_symbol_iterator : public symbol_iterator {
+public:
+  xcoff_symbol_iterator(const basic_symbol_iterator &B)
+      : symbol_iterator(B) {}
+
+  const XCOFFSymbolRef *operator->() const {
+    return static_cast<const XCOFFSymbolRef *>(symbol_iterator::operator->());
+  }
+
+  const XCOFFSymbolRef &operator*() const {
+    return static_cast<const XCOFFSymbolRef &>(symbol_iterator::operator*());
+  }
 };
 
 class TBVectorExt {
diff --git a/llvm/include/llvm/Option/ArgList.h b/llvm/include/llvm/Option/ArgList.h
index f18585ea49..fcde68e0b7 100644
--- a/llvm/include/llvm/Option/ArgList.h
+++ b/llvm/include/llvm/Option/ArgList.h
@@ -329,12 +329,11 @@ class ArgList {
   /// and not matching any of the excluded ids.
   void AddAllArgsExcept(ArgStringList &Output, ArrayRef<OptSpecifier> Ids,
                         ArrayRef<OptSpecifier> ExcludeIds) const;
-  /// AddAllArgs - Render all arguments matching any of the given ids.
-  void AddAllArgs(ArgStringList &Output, ArrayRef<OptSpecifier> Ids) const;
+  /// Render all arguments matching any of the given ids.
+  void addAllArgs(ArgStringList &Output, ArrayRef<OptSpecifier> Ids) const;
 
   /// AddAllArgs - Render all arguments matching the given ids.
-  void AddAllArgs(ArgStringList &Output, OptSpecifier Id0,
-                  OptSpecifier Id1 = 0U, OptSpecifier Id2 = 0U) const;
+  void AddAllArgs(ArgStringList &Output, OptSpecifier Id0) const;
 
   /// AddAllArgValues - Render the argument values of all arguments
   /// matching the given ids.
diff --git a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h
index b2fdbc1936..948203eea8 100644
--- a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h
+++ b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h
@@ -790,26 +790,26 @@ getLineCoverageStats(const coverage::CoverageData &CD) {
 namespace accessors {
 
 /// Return the structural hash associated with the function.
-template <class FuncRecordTy, support::endianness Endian>
+template <class FuncRecordTy, llvm::endianness Endian>
 uint64_t getFuncHash(const FuncRecordTy *Record) {
   return support::endian::byte_swap<uint64_t, Endian>(Record->FuncHash);
 }
 
 /// Return the coverage map data size for the function.
-template <class FuncRecordTy, support::endianness Endian>
+template <class FuncRecordTy, llvm::endianness Endian>
 uint64_t getDataSize(const FuncRecordTy *Record) {
   return support::endian::byte_swap<uint32_t, Endian>(Record->DataSize);
 }
 
 /// Return the function lookup key. The value is considered opaque.
-template <class FuncRecordTy, support::endianness Endian>
+template <class FuncRecordTy, llvm::endianness Endian>
 uint64_t getFuncNameRef(const FuncRecordTy *Record) {
   return support::endian::byte_swap<uint64_t, Endian>(Record->NameRef);
 }
 
 /// Return the PGO name of the function. Used for formats in which the name is
 /// a hash.
-template <class FuncRecordTy, support::endianness Endian>
+template <class FuncRecordTy, llvm::endianness Endian>
 Error getFuncNameViaRef(const FuncRecordTy *Record,
                         InstrProfSymtab &ProfileNames, StringRef &FuncName) {
   uint64_t NameRef = getFuncNameRef<FuncRecordTy, Endian>(Record);
@@ -820,7 +820,7 @@ Error getFuncNameViaRef(const FuncRecordTy *Record,
 /// Read coverage mapping out-of-line, from \p MappingBuf. This is used when the
 /// coverage mapping is attached to the file header, instead of to the function
 /// record.
-template <class FuncRecordTy, support::endianness Endian>
+template <class FuncRecordTy, llvm::endianness Endian>
 StringRef getCoverageMappingOutOfLine(const FuncRecordTy *Record,
                                       const char *MappingBuf) {
   return {MappingBuf, size_t(getDataSize<FuncRecordTy, Endian>(Record))};
@@ -828,7 +828,7 @@ StringRef getCoverageMappingOutOfLine(const FuncRecordTy *Record,
 
 /// Advance to the next out-of-line coverage mapping and its associated
 /// function record.
-template <class FuncRecordTy, support::endianness Endian>
+template <class FuncRecordTy, llvm::endianness Endian>
 std::pair<const char *, const FuncRecordTy *>
 advanceByOneOutOfLine(const FuncRecordTy *Record, const char *MappingBuf) {
   return {MappingBuf + getDataSize<FuncRecordTy, Endian>(Record), Record + 1};
@@ -847,21 +847,21 @@ struct CovMapFunctionRecordV1 {
 #undef COVMAP_V1
   CovMapFunctionRecordV1() = delete;
 
-  template <support::endianness Endian> uint64_t getFuncHash() const {
+  template <llvm::endianness Endian> uint64_t getFuncHash() const {
     return accessors::getFuncHash<ThisT, Endian>(this);
   }
 
-  template <support::endianness Endian> uint64_t getDataSize() const {
+  template <llvm::endianness Endian> uint64_t getDataSize() const {
     return accessors::getDataSize<ThisT, Endian>(this);
   }
 
   /// Return function lookup key. The value is consider opaque.
-  template <support::endianness Endian> IntPtrT getFuncNameRef() const {
+  template <llvm::endianness Endian> IntPtrT getFuncNameRef() const {
     return support::endian::byte_swap<IntPtrT, Endian>(NamePtr);
   }
 
   /// Return the PGO name of the function.
-  template <support::endianness Endian>
+  template <llvm::endianness Endian>
   Error getFuncName(InstrProfSymtab &ProfileNames, StringRef &FuncName) const {
     IntPtrT NameRef = getFuncNameRef<Endian>();
     uint32_t NameS = support::endian::byte_swap<uint32_t, Endian>(NameSize);
@@ -872,17 +872,17 @@ struct CovMapFunctionRecordV1 {
     return Error::success();
   }
 
-  template <support::endianness Endian>
+  template <llvm::endianness Endian>
   std::pair<const char *, const ThisT *>
   advanceByOne(const char *MappingBuf) const {
     return accessors::advanceByOneOutOfLine<ThisT, Endian>(this, MappingBuf);
   }
 
-  template <support::endianness Endian> uint64_t getFilenamesRef() const {
+  template <llvm::endianness Endian> uint64_t getFilenamesRef() const {
     llvm_unreachable("V1 function format does not contain a filenames ref");
   }
 
-  template <support::endianness Endian>
+  template <llvm::endianness Endian>
   StringRef getCoverageMapping(const char *MappingBuf) const {
     return accessors::getCoverageMappingOutOfLine<ThisT, Endian>(this,
                                                                  MappingBuf);
@@ -898,35 +898,35 @@ struct CovMapFunctionRecordV2 {
 #undef COVMAP_V2
   CovMapFunctionRecordV2() = delete;
 
-  template <support::endianness Endian> uint64_t getFuncHash() const {
+  template <llvm::endianness Endian> uint64_t getFuncHash() const {
     return accessors::getFuncHash<ThisT, Endian>(this);
   }
 
-  template <support::endianness Endian> uint64_t getDataSize() const {
+  template <llvm::endianness Endian> uint64_t getDataSize() const {
     return accessors::getDataSize<ThisT, Endian>(this);
   }
 
-  template <support::endianness Endian> uint64_t getFuncNameRef() const {
+  template <llvm::endianness Endian> uint64_t getFuncNameRef() const {
     return accessors::getFuncNameRef<ThisT, Endian>(this);
   }
 
-  template <support::endianness Endian>
+  template <llvm::endianness Endian>
   Error getFuncName(InstrProfSymtab &ProfileNames, StringRef &FuncName) const {
     return accessors::getFuncNameViaRef<ThisT, Endian>(this, ProfileNames,
                                                        FuncName);
   }
 
-  template <support::endianness Endian>
+  template <llvm::endianness Endian>
   std::pair<const char *, const ThisT *>
   advanceByOne(const char *MappingBuf) const {
     return accessors::advanceByOneOutOfLine<ThisT, Endian>(this, MappingBuf);
   }
 
-  template <support::endianness Endian> uint64_t getFilenamesRef() const {
+  template <llvm::endianness Endian> uint64_t getFilenamesRef() const {
     llvm_unreachable("V2 function format does not contain a filenames ref");
   }
 
-  template <support::endianness Endian>
+  template <llvm::endianness Endian>
   StringRef getCoverageMapping(const char *MappingBuf) const {
     return accessors::getCoverageMappingOutOfLine<ThisT, Endian>(this,
                                                                  MappingBuf);
@@ -942,39 +942,39 @@ struct CovMapFunctionRecordV3 {
 #undef COVMAP_V3
   CovMapFunctionRecordV3() = delete;
 
-  template <support::endianness Endian> uint64_t getFuncHash() const {
+  template <llvm::endianness Endian> uint64_t getFuncHash() const {
     return accessors::getFuncHash<ThisT, Endian>(this);
   }
 
-  template <support::endianness Endian> uint64_t getDataSize() const {
+  template <llvm::endianness Endian> uint64_t getDataSize() const {
     return accessors::getDataSize<ThisT, Endian>(this);
   }
 
-  template <support::endianness Endian> uint64_t getFuncNameRef() const {
+  template <llvm::endianness Endian> uint64_t getFuncNameRef() const {
     return accessors::getFuncNameRef<ThisT, Endian>(this);
   }
 
-  template <support::endianness Endian>
+  template <llvm::endianness Endian>
   Error getFuncName(InstrProfSymtab &ProfileNames, StringRef &FuncName) const {
     return accessors::getFuncNameViaRef<ThisT, Endian>(this, ProfileNames,
                                                        FuncName);
   }
 
   /// Get the filename set reference.
-  template <support::endianness Endian> uint64_t getFilenamesRef() const {
+  template <llvm::endianness Endian> uint64_t getFilenamesRef() const {
     return support::endian::byte_swap<uint64_t, Endian>(FilenamesRef);
   }
 
   /// Read the inline coverage mapping. Ignore the buffer parameter, it is for
   /// out-of-line coverage mapping data only.
-  template <support::endianness Endian>
+  template <llvm::endianness Endian>
   StringRef getCoverageMapping(const char *) const {
     return StringRef(&CoverageMapping, getDataSize<Endian>());
   }
 
   // Advance to the next inline coverage mapping and its associated function
   // record. Ignore the out-of-line coverage mapping buffer.
-  template <support::endianness Endian>
+  template <llvm::endianness Endian>
   std::pair<const char *, const CovMapFunctionRecordV3 *>
   advanceByOne(const char *) const {
     assert(isAddrAligned(Align(8), this) && "Function record not aligned");
@@ -992,19 +992,19 @@ struct CovMapFunctionRecordV3 {
 struct CovMapHeader {
 #define COVMAP_HEADER(Type, LLVMType, Name, Init) Type Name;
 #include "llvm/ProfileData/InstrProfData.inc"
-  template <support::endianness Endian> uint32_t getNRecords() const {
+  template <llvm::endianness Endian> uint32_t getNRecords() const {
     return support::endian::byte_swap<uint32_t, Endian>(NRecords);
   }
 
-  template <support::endianness Endian> uint32_t getFilenamesSize() const {
+  template <llvm::endianness Endian> uint32_t getFilenamesSize() const {
     return support::endian::byte_swap<uint32_t, Endian>(FilenamesSize);
   }
 
-  template <support::endianness Endian> uint32_t getCoverageSize() const {
+  template <llvm::endianness Endian> uint32_t getCoverageSize() const {
     return support::endian::byte_swap<uint32_t, Endian>(CoverageSize);
   }
 
-  template <support::endianness Endian> uint32_t getVersion() const {
+  template <llvm::endianness Endian> uint32_t getVersion() const {
     return support::endian::byte_swap<uint32_t, Endian>(Version);
   }
 };
diff --git a/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h b/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h
index 326c1b0d33..346ca4ad2e 100644
--- a/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h
+++ b/llvm/include/llvm/ProfileData/Coverage/CoverageMappingReader.h
@@ -213,7 +213,7 @@ class BinaryCoverageReader : public CoverageMappingReader {
                                  FuncRecordsStorage &&FuncRecords,
                                  InstrProfSymtab &&ProfileNames,
                                  uint8_t BytesInAddress,
-                                 support::endianness Endian,
+                                 llvm::endianness Endian,
                                  StringRef CompilationDir = "");
 
   Error readNextRecord(CoverageMappingRecord &Record) override;
diff --git a/llvm/include/llvm/ProfileData/FunctionId.h b/llvm/include/llvm/ProfileData/FunctionId.h
new file mode 100644
index 0000000000..0076cdc090
--- /dev/null
+++ b/llvm/include/llvm/ProfileData/FunctionId.h
@@ -0,0 +1,213 @@
+//===--- FunctionId.h - Sample profile function object ----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+///
+/// Defines FunctionId class.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_PROFILEDATA_FUNCTIONID_H
+#define LLVM_PROFILEDATA_FUNCTIONID_H
+
+#include "llvm/ADT/DenseMapInfo.h"
+#include "llvm/ADT/Hashing.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/MD5.h"
+#include "llvm/Support/raw_ostream.h"
+#include <cstdint>
+
+namespace llvm {
+namespace sampleprof {
+
+/// This class represents a function that is read from a sample profile. It
+/// comes with two forms: a string or a hash code. The latter form is the 64-bit
+/// MD5 of the function name for efficient storage supported by ExtBinary
+/// profile format, and when reading the profile, this class can represent it
+/// without converting it to a string first.
+/// When representing a hash code, we utilize the LengthOrHashCode field to
+/// store it, and Name is set to null. When representing a string, it is same as
+/// StringRef.
+class FunctionId {
+
+  const char *Data = nullptr;
+
+  // Use uint64_t instead of size_t so that it can also hold a MD5 value on
+  // 32-bit system.
+  uint64_t LengthOrHashCode = 0;
+
+  /// Extension to memcmp to handle hash code representation. If both are hash
+  /// values, Lhs and Rhs are both null, function returns 0 (and needs an extra
+  /// comparison using getIntValue). If only one is hash code, it is considered
+  /// less than the StringRef one. Otherwise perform normal string comparison.
+  static int compareMemory(const char *Lhs, const char *Rhs, uint64_t Length) {
+    if (Lhs == Rhs)
+      return 0;
+    if (!Lhs)
+      return -1;
+    if (!Rhs)
+      return 1;
+    return ::memcmp(Lhs, Rhs, (size_t)Length);
+  }
+
+public:
+  FunctionId() = default;
+
+  /// Constructor from a StringRef.
+  explicit FunctionId(StringRef Str)
+      : Data(Str.data()), LengthOrHashCode(Str.size()) {
+  }
+
+  /// Constructor from a hash code.
+  explicit FunctionId(uint64_t HashCode)
+      : LengthOrHashCode(HashCode) {
+    assert(HashCode != 0);
+  }
+
+  /// Check for equality. Similar to StringRef::equals, but will also cover for
+  /// the case where one or both are hash codes. Comparing their int values are
+  /// sufficient. A hash code FunctionId is considered not equal to a StringRef
+  /// FunctionId regardless of actual contents.
+  bool equals(const FunctionId &Other) const {
+    return LengthOrHashCode == Other.LengthOrHashCode &&
+           compareMemory(Data, Other.Data, LengthOrHashCode) == 0;
+  }
+
+  /// Total order comparison. If both FunctionId are StringRef, this is the same
+  /// as StringRef::compare. If one of them is StringRef, it is considered
+  /// greater than the hash code FunctionId. Otherwise this is the the same
+  /// as comparing their int values.
+  int compare(const FunctionId &Other) const {
+    auto Res = compareMemory(
+        Data, Other.Data, std::min(LengthOrHashCode, Other.LengthOrHashCode));
+    if (Res != 0)
+      return Res;
+    if (LengthOrHashCode == Other.LengthOrHashCode)
+      return 0;
+    return LengthOrHashCode < Other.LengthOrHashCode ? -1 : 1;
+  }
+
+  /// Convert to a string, usually for output purpose. Use caution on return
+  /// value's lifetime when converting to StringRef.
+  std::string str() const {
+    if (Data)
+      return std::string(Data, LengthOrHashCode);
+    if (LengthOrHashCode != 0)
+      return std::to_string(LengthOrHashCode);
+    return std::string();
+  }
+
+  /// Convert to StringRef. This is only allowed when it is known this object is
+  /// representing a StringRef, not a hash code. Calling this function on a hash
+  /// code is considered an error.
+  StringRef stringRef() const {
+    if (Data)
+      return StringRef(Data, LengthOrHashCode);
+    assert(LengthOrHashCode == 0 &&
+           "Cannot convert MD5 FunctionId to StringRef");
+    return StringRef();
+  }
+
+  friend raw_ostream &operator<<(raw_ostream &OS, const FunctionId &Obj);
+
+  /// Get hash code of this object. Returns this object's hash code if it is
+  /// already representing one, otherwise returns the MD5 of its string content.
+  /// Note that it is not the same as std::hash because we want to keep the
+  /// consistency that the same sample profile function in string form or MD5
+  /// form has the same hash code.
+  uint64_t getHashCode() const {
+    if (Data)
+      return MD5Hash(StringRef(Data, LengthOrHashCode));
+    return LengthOrHashCode;
+  }
+
+  bool empty() const { return LengthOrHashCode == 0; }
+
+  /// Check if this object represents a StringRef, or a hash code.
+  bool isStringRef() const { return Data != nullptr; }
+};
+
+inline bool operator==(const FunctionId &LHS, const FunctionId &RHS) {
+  return LHS.equals(RHS);
+}
+
+inline bool operator!=(const FunctionId &LHS, const FunctionId &RHS) {
+  return !LHS.equals(RHS);
+}
+
+inline bool operator<(const FunctionId &LHS, const FunctionId &RHS) {
+  return LHS.compare(RHS) < 0;
+}
+
+inline bool operator<=(const FunctionId &LHS, const FunctionId &RHS) {
+  return LHS.compare(RHS) <= 0;
+}
+
+inline bool operator>(const FunctionId &LHS, const FunctionId &RHS) {
+  return LHS.compare(RHS) > 0;
+}
+
+inline bool operator>=(const FunctionId &LHS, const FunctionId &RHS) {
+  return LHS.compare(RHS) >= 0;
+}
+
+inline raw_ostream &operator<<(raw_ostream &OS, const FunctionId &Obj) {
+  if (Obj.Data)
+    return OS << StringRef(Obj.Data, Obj.LengthOrHashCode);
+  if (Obj.LengthOrHashCode != 0)
+    return OS << Obj.LengthOrHashCode;
+  return OS;
+}
+
+inline uint64_t MD5Hash(const FunctionId &Obj) {
+  return Obj.getHashCode();
+}
+
+inline uint64_t hash_value(const FunctionId &Obj) {
+  return Obj.getHashCode();
+}
+
+} // end namespace sampleprof
+
+/// Template specialization for FunctionId so that it can be used in LLVM map
+/// containers.
+template <> struct DenseMapInfo<sampleprof::FunctionId, void> {
+
+  static inline sampleprof::FunctionId getEmptyKey() {
+    return sampleprof::FunctionId(~0ULL);
+  }
+
+  static inline sampleprof::FunctionId getTombstoneKey() {
+    return sampleprof::FunctionId(~1ULL);
+  }
+
+  static unsigned getHashValue(const sampleprof::FunctionId &Val) {
+    return Val.getHashCode();
+  }
+
+  static bool isEqual(const sampleprof::FunctionId &LHS,
+                      const sampleprof::FunctionId &RHS) {
+    return LHS == RHS;
+  }
+};
+
+} // end namespace llvm
+
+namespace std {
+
+/// Template specialization for FunctionId so that it can be used in STL
+/// containers.
+template <> struct hash<llvm::sampleprof::FunctionId> {
+  size_t operator()(const llvm::sampleprof::FunctionId &Val) const {
+    return Val.getHashCode();
+  }
+};
+
+} // end namespace std
+
+#endif // LLVM_PROFILEDATA_FUNCTIONID_H
diff --git a/llvm/include/llvm/ProfileData/GCOV.h b/llvm/include/llvm/ProfileData/GCOV.h
index 674260c81f..c8af71dbf6 100644
--- a/llvm/include/llvm/ProfileData/GCOV.h
+++ b/llvm/include/llvm/ProfileData/GCOV.h
@@ -249,7 +249,7 @@ class GCOVFunction {
     return make_range(blocks.begin(), blocks.end());
   }
 
-  uint64_t propagateCounts(const GCOVBlock &v, GCOVArc *pred);
+  void propagateCounts(const GCOVBlock &v, GCOVArc *pred);
   void print(raw_ostream &OS) const;
   void dump() const;
 
diff --git a/llvm/include/llvm/ProfileData/HashKeyMap.h b/llvm/include/llvm/ProfileData/HashKeyMap.h
new file mode 100644
index 0000000000..b2f1bf2221
--- /dev/null
+++ b/llvm/include/llvm/ProfileData/HashKeyMap.h
@@ -0,0 +1,129 @@
+//===--- HashKeyMap.h - Wrapper for maps using hash value key ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+///
+/// Defines HashKeyMap template.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_PROFILEDATA_HASHKEYMAP_H
+#define LLVM_PROFILEDATA_HASHKEYMAP_H
+
+#include "llvm/ADT/Hashing.h"
+#include <iterator>
+#include <utility>
+
+namespace llvm {
+
+namespace sampleprof {
+
+/// This class is a wrapper to associative container MapT<KeyT, ValueT> using
+/// the hash value of the original key as the new key. This greatly improves the
+/// performance of insert and query operations especially when hash values of
+/// keys are available a priori, and reduces memory usage if KeyT has a large
+/// size.
+/// All keys with the same hash value are considered equivalent (i.e. hash
+/// collision is silently ignored). Given such feature this class should only be
+/// used where it does not affect compilation correctness, for example, when
+/// loading a sample profile. The original key is not stored, so if the user
+/// needs to preserve it, it should be stored in the mapped type.
+/// Assuming the hashing algorithm is uniform, we use the formula
+/// 1 - Permute(n, k) / n ^ k where n is the universe size and k is number of
+/// elements chosen at random to calculate the probability of collision. With
+/// 1,000,000 entries the probability is negligible:
+/// 1 - (2^64)!/((2^64-1000000)!*(2^64)^1000000) ~= 3*10^-8.
+/// Source: https://en.wikipedia.org/wiki/Birthday_problem
+///
+/// \param MapT The underlying associative container type.
+/// \param KeyT The original key type, which requires the implementation of
+///   llvm::hash_value(KeyT).
+/// \param ValueT The original mapped type, which has the same requirement as
+///   the underlying container.
+/// \param MapTArgs Additional template parameters passed to the underlying
+///   container.
+template <template <typename, typename, typename...> typename MapT,
+          typename KeyT, typename ValueT, typename... MapTArgs>
+class HashKeyMap :
+    public MapT<decltype(hash_value(KeyT())), ValueT, MapTArgs...> {
+public:
+  using base_type = MapT<decltype(hash_value(KeyT())), ValueT, MapTArgs...>;
+  using key_type = decltype(hash_value(KeyT()));
+  using original_key_type = KeyT;
+  using mapped_type = ValueT;
+  using value_type = typename base_type::value_type;
+
+  using iterator = typename base_type::iterator;
+  using const_iterator = typename base_type::const_iterator;
+
+  template <typename... Ts>
+  std::pair<iterator, bool> try_emplace(const key_type &Hash,
+                                        const original_key_type &Key,
+                                        Ts &&...Args) {
+    assert(Hash == hash_value(Key));
+    return base_type::try_emplace(Hash, std::forward<Ts>(Args)...);
+  }
+
+  template <typename... Ts>
+  std::pair<iterator, bool> try_emplace(const original_key_type &Key,
+                                        Ts &&...Args) {
+    return try_emplace(hash_value(Key), Key, std::forward<Ts>(Args)...);
+  }
+
+  template <typename... Ts> std::pair<iterator, bool> emplace(Ts &&...Args) {
+    return try_emplace(std::forward<Ts>(Args)...);
+  }
+
+  mapped_type &operator[](const original_key_type &Key) {
+    return try_emplace(Key, mapped_type()).first->second;
+  }
+
+  iterator find(const original_key_type &Key) {
+    auto It = base_type::find(hash_value(Key));
+    if (It != base_type::end())
+      return It;
+    return base_type::end();
+  }
+
+  const_iterator find(const original_key_type &Key) const {
+    auto It = base_type::find(hash_value(Key));
+    if (It != base_type::end())
+      return It;
+    return base_type::end();
+  }
+
+  mapped_type lookup(const original_key_type &Key) const {
+    auto It = base_type::find(hash_value(Key));
+    if (It != base_type::end())
+      return It->second;
+    return mapped_type();
+  }
+
+  size_t count(const original_key_type &Key) const {
+    return base_type::count(hash_value(Key));
+  }
+
+  size_t erase(const original_key_type &Ctx) {
+    auto It = find(Ctx);
+    if (It != base_type::end()) {
+      base_type::erase(It);
+      return 1;
+    }
+    return 0;
+  }
+
+  iterator erase(const_iterator It) {
+    return base_type::erase(It);
+  }
+};
+
+}
+
+}
+
+#endif // LLVM_PROFILEDATA_HASHKEYMAP_H
diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc
index 8ba7e186d4..13be2753e5 100644
--- a/llvm/include/llvm/ProfileData/InstrProfData.inc
+++ b/llvm/include/llvm/ProfileData/InstrProfData.inc
@@ -340,7 +340,7 @@ typedef struct ValueProfRecord {
    * Do byte swap for this instance. \c Old is the original order before
    * the swap, and \c New is the New byte order.
    */
-  void swapBytes(support::endianness Old, support::endianness New);
+  void swapBytes(llvm::endianness Old, llvm::endianness New);
 #endif
 } ValueProfRecord;
 
@@ -395,15 +395,15 @@ typedef struct ValueProfData {
   static Expected<std::unique_ptr<ValueProfData>>
   getValueProfData(const unsigned char *SrcBuffer,
                    const unsigned char *const SrcBufferEnd,
-                   support::endianness SrcDataEndianness);
+                   llvm::endianness SrcDataEndianness);
   /*!
    * Swap byte order from \c Endianness order to host byte order.
    */
-  void swapBytesToHost(support::endianness Endianness);
+  void swapBytesToHost(llvm::endianness Endianness);
   /*!
    * Swap byte order from host byte order to \c Endianness order.
    */
-  void swapBytesFromHost(support::endianness Endianness);
+  void swapBytesFromHost(llvm::endianness Endianness);
   /*!
    * Return the total size of \c ValueProfileData.
    */
diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h
index acbc9e21fc..5f54cbeb1b 100644
--- a/llvm/include/llvm/ProfileData/InstrProfReader.h
+++ b/llvm/include/llvm/ProfileData/InstrProfReader.h
@@ -412,13 +412,13 @@ class RawInstrProfReader : public InstrProfReader {
     return ShouldSwapBytes ? llvm::byteswap(Int) : Int;
   }
 
-  support::endianness getDataEndianness() const {
+  llvm::endianness getDataEndianness() const {
     if (!ShouldSwapBytes)
       return llvm::endianness::native;
     if (llvm::endianness::native == llvm::endianness::little)
-      return support::big;
+      return llvm::endianness::big;
     else
-      return support::little;
+      return llvm::endianness::little;
   }
 
   inline uint8_t getNumPaddingBytes(uint64_t SizeInBytes) {
@@ -477,7 +477,7 @@ class InstrProfLookupTrait {
   // Endianness of the input value profile data.
   // It should be LE by default, but can be changed
   // for testing purpose.
-  support::endianness ValueProfDataEndianness = support::little;
+  llvm::endianness ValueProfDataEndianness = llvm::endianness::little;
 
 public:
   InstrProfLookupTrait(IndexedInstrProf::HashT HashType, unsigned FormatVersion)
@@ -500,8 +500,10 @@ class InstrProfLookupTrait {
   ReadKeyDataLength(const unsigned char *&D) {
     using namespace support;
 
-    offset_type KeyLen = endian::readNext<offset_type, little, unaligned>(D);
-    offset_type DataLen = endian::readNext<offset_type, little, unaligned>(D);
+    offset_type KeyLen =
+        endian::readNext<offset_type, llvm::endianness::little, unaligned>(D);
+    offset_type DataLen =
+        endian::readNext<offset_type, llvm::endianness::little, unaligned>(D);
     return std::make_pair(KeyLen, DataLen);
   }
 
@@ -514,7 +516,7 @@ class InstrProfLookupTrait {
   data_type ReadData(StringRef K, const unsigned char *D, offset_type N);
 
   // Used for testing purpose only.
-  void setValueProfDataEndianness(support::endianness Endianness) {
+  void setValueProfDataEndianness(llvm::endianness Endianness) {
     ValueProfDataEndianness = Endianness;
   }
 };
@@ -531,7 +533,7 @@ struct InstrProfReaderIndexBase {
                                      ArrayRef<NamedInstrProfRecord> &Data) = 0;
   virtual void advanceToNextKey() = 0;
   virtual bool atEnd() const = 0;
-  virtual void setValueProfDataEndianness(support::endianness Endianness) = 0;
+  virtual void setValueProfDataEndianness(llvm::endianness Endianness) = 0;
   virtual uint64_t getVersion() const = 0;
   virtual bool isIRLevelProfile() const = 0;
   virtual bool hasCSIRLevelProfile() const = 0;
@@ -580,7 +582,7 @@ class InstrProfReaderIndex : public InstrProfReaderIndexBase {
     return RecordIterator == HashTable->data_end();
   }
 
-  void setValueProfDataEndianness(support::endianness Endianness) override {
+  void setValueProfDataEndianness(llvm::endianness Endianness) override {
     HashTable->getInfoObj().setValueProfDataEndianness(Endianness);
   }
 
@@ -753,7 +755,7 @@ class IndexedInstrProfReader : public InstrProfReader {
          std::unique_ptr<MemoryBuffer> RemappingBuffer = nullptr);
 
   // Used for testing purpose only.
-  void setValueProfDataEndianness(support::endianness Endianness) {
+  void setValueProfDataEndianness(llvm::endianness Endianness) {
     Index->setValueProfDataEndianness(Endianness);
   }
 
diff --git a/llvm/include/llvm/ProfileData/InstrProfWriter.h b/llvm/include/llvm/ProfileData/InstrProfWriter.h
index e50705ee05..311c84a2d5 100644
--- a/llvm/include/llvm/ProfileData/InstrProfWriter.h
+++ b/llvm/include/llvm/ProfileData/InstrProfWriter.h
@@ -170,7 +170,7 @@ class InstrProfWriter {
   InstrProfKind getProfileKind() const { return ProfileKind; }
 
   // Internal interface for testing purpose only.
-  void setValueProfDataEndianness(support::endianness Endianness);
+  void setValueProfDataEndianness(llvm::endianness Endianness);
   void setOutputSparse(bool Sparse);
   // Compute the overlap b/w this object and Other. Program level result is
   // stored in Overlap and function level result is stored in FuncLevelOverlap.
diff --git a/llvm/include/llvm/ProfileData/MemProf.h b/llvm/include/llvm/ProfileData/MemProf.h
index d0ba5b10be..1a066c10c1 100644
--- a/llvm/include/llvm/ProfileData/MemProf.h
+++ b/llvm/include/llvm/ProfileData/MemProf.h
@@ -50,7 +50,7 @@ struct PortableMemInfoBlock {
       switch (Id) {
 #define MIBEntryDef(NameTag, Name, Type)                                       \
   case Meta::Name: {                                                           \
-    Name = endian::readNext<Type, little, unaligned>(Ptr);                     \
+    Name = endian::readNext<Type, llvm::endianness::little, unaligned>(Ptr);   \
   } break;
 #include "llvm/ProfileData/MIBEntryDef.inc"
 #undef MIBEntryDef
@@ -66,7 +66,7 @@ struct PortableMemInfoBlock {
   void serialize(const MemProfSchema &Schema, raw_ostream &OS) const {
     using namespace support;
 
-    endian::Writer LE(OS, little);
+    endian::Writer LE(OS, llvm::endianness::little);
     for (const Meta Id : Schema) {
       switch (Id) {
 #define MIBEntryDef(NameTag, Name, Type)                                       \
@@ -187,7 +187,7 @@ struct Frame {
   void serialize(raw_ostream &OS) const {
     using namespace support;
 
-    endian::Writer LE(OS, little);
+    endian::Writer LE(OS, llvm::endianness::little);
 
     // If the type of the GlobalValue::GUID changes, then we need to update
     // the reader and the writer.
@@ -204,10 +204,14 @@ struct Frame {
   static Frame deserialize(const unsigned char *Ptr) {
     using namespace support;
 
-    const uint64_t F = endian::readNext<uint64_t, little, unaligned>(Ptr);
-    const uint32_t L = endian::readNext<uint32_t, little, unaligned>(Ptr);
-    const uint32_t C = endian::readNext<uint32_t, little, unaligned>(Ptr);
-    const bool I = endian::readNext<bool, little, unaligned>(Ptr);
+    const uint64_t F =
+        endian::readNext<uint64_t, llvm::endianness::little, unaligned>(Ptr);
+    const uint32_t L =
+        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Ptr);
+    const uint32_t C =
+        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Ptr);
+    const bool I =
+        endian::readNext<bool, llvm::endianness::little, unaligned>(Ptr);
     return Frame(/*Function=*/F, /*LineOffset=*/L, /*Column=*/C,
                  /*IsInlineFrame=*/I);
   }
@@ -466,14 +470,17 @@ class RecordLookupTrait {
   ReadKeyDataLength(const unsigned char *&D) {
     using namespace support;
 
-    offset_type KeyLen = endian::readNext<offset_type, little, unaligned>(D);
-    offset_type DataLen = endian::readNext<offset_type, little, unaligned>(D);
+    offset_type KeyLen =
+        endian::readNext<offset_type, llvm::endianness::little, unaligned>(D);
+    offset_type DataLen =
+        endian::readNext<offset_type, llvm::endianness::little, unaligned>(D);
     return std::make_pair(KeyLen, DataLen);
   }
 
   uint64_t ReadKey(const unsigned char *D, offset_type /*Unused*/) {
     using namespace support;
-    return endian::readNext<external_key_type, little, unaligned>(D);
+    return endian::readNext<external_key_type, llvm::endianness::little,
+                            unaligned>(D);
   }
 
   data_type ReadData(uint64_t K, const unsigned char *D,
@@ -514,7 +521,7 @@ class RecordWriterTrait {
   EmitKeyDataLength(raw_ostream &Out, key_type_ref K, data_type_ref V) {
     using namespace support;
 
-    endian::Writer LE(Out, little);
+    endian::Writer LE(Out, llvm::endianness::little);
     offset_type N = sizeof(K);
     LE.write<offset_type>(N);
     offset_type M = V.serializedSize();
@@ -524,7 +531,7 @@ class RecordWriterTrait {
 
   void EmitKey(raw_ostream &Out, key_type_ref K, offset_type /*Unused*/) {
     using namespace support;
-    endian::Writer LE(Out, little);
+    endian::Writer LE(Out, llvm::endianness::little);
     LE.write<uint64_t>(K);
   }
 
@@ -552,7 +559,7 @@ class FrameWriterTrait {
   static std::pair<offset_type, offset_type>
   EmitKeyDataLength(raw_ostream &Out, key_type_ref K, data_type_ref V) {
     using namespace support;
-    endian::Writer LE(Out, little);
+    endian::Writer LE(Out, llvm::endianness::little);
     offset_type N = sizeof(K);
     LE.write<offset_type>(N);
     offset_type M = V.serializedSize();
@@ -562,7 +569,7 @@ class FrameWriterTrait {
 
   void EmitKey(raw_ostream &Out, key_type_ref K, offset_type /*Unused*/) {
     using namespace support;
-    endian::Writer LE(Out, little);
+    endian::Writer LE(Out, llvm::endianness::little);
     LE.write<key_type>(K);
   }
 
@@ -593,14 +600,17 @@ class FrameLookupTrait {
   ReadKeyDataLength(const unsigned char *&D) {
     using namespace support;
 
-    offset_type KeyLen = endian::readNext<offset_type, little, unaligned>(D);
-    offset_type DataLen = endian::readNext<offset_type, little, unaligned>(D);
+    offset_type KeyLen =
+        endian::readNext<offset_type, llvm::endianness::little, unaligned>(D);
+    offset_type DataLen =
+        endian::readNext<offset_type, llvm::endianness::little, unaligned>(D);
     return std::make_pair(KeyLen, DataLen);
   }
 
   uint64_t ReadKey(const unsigned char *D, offset_type /*Unused*/) {
     using namespace support;
-    return endian::readNext<external_key_type, little, unaligned>(D);
+    return endian::readNext<external_key_type, llvm::endianness::little,
+                            unaligned>(D);
   }
 
   data_type ReadData(uint64_t K, const unsigned char *D,
diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h
index 78a16499ac..57ea144532 100644
--- a/llvm/include/llvm/ProfileData/SampleProf.h
+++ b/llvm/include/llvm/ProfileData/SampleProf.h
@@ -21,10 +21,12 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/GlobalValue.h"
+#include "llvm/ProfileData/FunctionId.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorOr.h"
 #include "llvm/Support/MathExtras.h"
+#include "llvm/ProfileData/HashKeyMap.h"
 #include <algorithm>
 #include <cstdint>
 #include <list>
@@ -109,16 +111,6 @@ static inline uint64_t SPMagic(SampleProfileFormat Format = SPF_Binary) {
          uint64_t('2') << (64 - 56) | uint64_t(Format);
 }
 
-/// Get the proper representation of a string according to whether the
-/// current Format uses MD5 to represent the string.
-static inline StringRef getRepInFormat(StringRef Name, bool UseMD5,
-                                       std::string &GUIDBuf) {
-  if (Name.empty() || !UseMD5)
-    return Name;
-  GUIDBuf = std::to_string(Function::getGUID(Name));
-  return GUIDBuf;
-}
-
 static inline uint64_t SPVersion() { return 103; }
 
 // Section Type used by SampleProfileExtBinaryBaseReader and
@@ -305,27 +297,22 @@ struct LineLocation {
     return LineOffset != O.LineOffset || Discriminator != O.Discriminator;
   }
 
+  uint64_t getHashCode() const {
+    return ((uint64_t) Discriminator << 32) | LineOffset;
+  }
+
   uint32_t LineOffset;
   uint32_t Discriminator;
 };
 
 struct LineLocationHash {
   uint64_t operator()(const LineLocation &Loc) const {
-    return std::hash<std::uint64_t>{}((((uint64_t)Loc.LineOffset) << 32) |
-                                      Loc.Discriminator);
+    return Loc.getHashCode();
   }
 };
 
 raw_ostream &operator<<(raw_ostream &OS, const LineLocation &Loc);
 
-static inline uint64_t hashFuncName(StringRef F) {
-  // If function name is already MD5 string, do not hash again.
-  uint64_t Hash;
-  if (F.getAsInteger(10, Hash))
-    Hash = MD5Hash(F);
-  return Hash;
-}
-
 /// Representation of a single sample record.
 ///
 /// A sample record is represented by a positive integer value, which
@@ -338,7 +325,7 @@ static inline uint64_t hashFuncName(StringRef F) {
 /// will be a list of one or more functions.
 class SampleRecord {
 public:
-  using CallTarget = std::pair<StringRef, uint64_t>;
+  using CallTarget = std::pair<FunctionId, uint64_t>;
   struct CallTargetComparator {
     bool operator()(const CallTarget &LHS, const CallTarget &RHS) const {
       if (LHS.second != RHS.second)
@@ -349,7 +336,7 @@ class SampleRecord {
   };
 
   using SortedCallTargetSet = std::set<CallTarget, CallTargetComparator>;
-  using CallTargetMap = StringMap<uint64_t>;
+  using CallTargetMap = std::unordered_map<FunctionId, uint64_t>;
   SampleRecord() = default;
 
   /// Increment the number of samples for this record by \p S.
@@ -378,7 +365,7 @@ class SampleRecord {
   ///
   /// Sample counts accumulate using saturating arithmetic, to avoid wrapping
   /// around unsigned integers.
-  sampleprof_error addCalledTarget(StringRef F, uint64_t S,
+  sampleprof_error addCalledTarget(FunctionId F, uint64_t S,
                                    uint64_t Weight = 1) {
     uint64_t &TargetSamples = CallTargets[F];
     bool Overflowed;
@@ -390,7 +377,7 @@ class SampleRecord {
 
   /// Remove called function from the call target map. Return the target sample
   /// count of the called function.
-  uint64_t removeCalledTarget(StringRef F) {
+  uint64_t removeCalledTarget(FunctionId F) {
     uint64_t Count = 0;
     auto I = CallTargets.find(F);
     if (I != CallTargets.end()) {
@@ -474,18 +461,18 @@ enum ContextAttributeMask {
       0x4, // Leaf of context is duplicated into the base profile
 };
 
-// Represents a context frame with function name and line location
+// Represents a context frame with profile function and line location
 struct SampleContextFrame {
-  StringRef FuncName;
+  FunctionId Func;
   LineLocation Location;
 
   SampleContextFrame() : Location(0, 0) {}
-
-  SampleContextFrame(StringRef FuncName, LineLocation Location)
-      : FuncName(FuncName), Location(Location) {}
+  
+  SampleContextFrame(FunctionId Func, LineLocation Location)
+      : Func(Func), Location(Location) {}
 
   bool operator==(const SampleContextFrame &That) const {
-    return Location == That.Location && FuncName == That.FuncName;
+    return Location == That.Location && Func == That.Func;
   }
 
   bool operator!=(const SampleContextFrame &That) const {
@@ -494,7 +481,7 @@ struct SampleContextFrame {
 
   std::string toString(bool OutputLineLocation) const {
     std::ostringstream OContextStr;
-    OContextStr << FuncName.str();
+    OContextStr << Func.str();
     if (OutputLineLocation) {
       OContextStr << ":" << Location.LineOffset;
       if (Location.Discriminator)
@@ -502,11 +489,16 @@ struct SampleContextFrame {
     }
     return OContextStr.str();
   }
+
+  uint64_t getHashCode() const {
+    uint64_t NameHash = Func.getHashCode();
+    uint64_t LocId = Location.getHashCode();
+    return NameHash + (LocId << 5) + LocId;
+  }
 };
 
 static inline hash_code hash_value(const SampleContextFrame &arg) {
-  return hash_combine(arg.FuncName, arg.Location.LineOffset,
-                      arg.Location.Discriminator);
+  return arg.getHashCode();
 }
 
 using SampleContextFrameVector = SmallVector<SampleContextFrame, 1>;
@@ -533,9 +525,12 @@ class SampleContext {
   SampleContext() : State(UnknownContext), Attributes(ContextNone) {}
 
   SampleContext(StringRef Name)
-      : Name(Name), State(UnknownContext), Attributes(ContextNone) {
+      : Func(Name), State(UnknownContext), Attributes(ContextNone) {
         assert(!Name.empty() && "Name is empty");
       }
+  
+  SampleContext(FunctionId Func)
+      : Func(Func), State(UnknownContext), Attributes(ContextNone) {}
 
   SampleContext(SampleContextFrames Context,
                 ContextStateMask CState = RawContext)
@@ -557,7 +552,7 @@ class SampleContext {
     bool HasContext = ContextStr.startswith("[");
     if (!HasContext) {
       State = UnknownContext;
-      Name = ContextStr;
+      Func = FunctionId(ContextStr);
     } else {
       CSNameTable.emplace_back();
       SampleContextFrameVector &Context = CSNameTable.back();
@@ -574,24 +569,25 @@ class SampleContext {
     ContextStr = ContextStr.substr(1, ContextStr.size() - 2);
     StringRef ContextRemain = ContextStr;
     StringRef ChildContext;
-    StringRef CalleeName;
+    FunctionId Callee;
     while (!ContextRemain.empty()) {
       auto ContextSplit = ContextRemain.split(" @ ");
       ChildContext = ContextSplit.first;
       ContextRemain = ContextSplit.second;
       LineLocation CallSiteLoc(0, 0);
-      decodeContextString(ChildContext, CalleeName, CallSiteLoc);
-      Context.emplace_back(CalleeName, CallSiteLoc);
+      decodeContextString(ChildContext, Callee, CallSiteLoc);
+      Context.emplace_back(Callee, CallSiteLoc);
     }
   }
 
   // Decode context string for a frame to get function name and location.
   // `ContextStr` is in the form of `FuncName:StartLine.Discriminator`.
-  static void decodeContextString(StringRef ContextStr, StringRef &FName,
+  static void decodeContextString(StringRef ContextStr,
+                                  FunctionId &Func,
                                   LineLocation &LineLoc) {
     // Get function name
     auto EntrySplit = ContextStr.split(':');
-    FName = EntrySplit.first;
+    Func = FunctionId(EntrySplit.first);
 
     LineLoc = {0, 0};
     if (!EntrySplit.second.empty()) {
@@ -618,7 +614,7 @@ class SampleContext {
   void clearState(ContextStateMask S) { State &= (uint32_t)~S; }
   bool hasContext() const { return State != UnknownContext; }
   bool isBaseContext() const { return FullContext.size() == 1; }
-  StringRef getName() const { return Name; }
+  FunctionId getFunction() const { return Func; }
   SampleContextFrames getContextFrames() const { return FullContext; }
 
   static std::string getContextString(SampleContextFrames Context,
@@ -636,22 +632,19 @@ class SampleContext {
 
   std::string toString() const {
     if (!hasContext())
-      return Name.str();
+      return Func.str();
     return getContextString(FullContext, false);
   }
 
   uint64_t getHashCode() const {
     if (hasContext())
       return hash_value(getContextFrames());
-
-    // For non-context function name, use its MD5 as hash value, so that it is
-    // consistent with the profile map's key.
-    return hashFuncName(getName());
+    return getFunction().getHashCode();
   }
 
   /// Set the name of the function and clear the current context.
-  void setName(StringRef FunctionName) {
-    Name = FunctionName;
+  void setFunction(FunctionId newFunction) {
+    Func = newFunction;
     FullContext = SampleContextFrames();
     State = UnknownContext;
   }
@@ -660,12 +653,12 @@ class SampleContext {
                   ContextStateMask CState = RawContext) {
     assert(CState != UnknownContext);
     FullContext = Context;
-    Name = Context.back().FuncName;
+    Func = Context.back().Func;
     State = CState;
   }
 
   bool operator==(const SampleContext &That) const {
-    return State == That.State && Name == That.Name &&
+    return State == That.State && Func == That.Func &&
            FullContext == That.FullContext;
   }
 
@@ -676,14 +669,14 @@ class SampleContext {
       return State < That.State;
 
     if (!hasContext()) {
-      return Name < That.Name;
+      return Func < That.Func;
     }
 
     uint64_t I = 0;
     while (I < std::min(FullContext.size(), That.FullContext.size())) {
       auto &Context1 = FullContext[I];
       auto &Context2 = That.FullContext[I];
-      auto V = Context1.FuncName.compare(Context2.FuncName);
+      auto V = Context1.Func.compare(Context2.Func);
       if (V)
         return V < 0;
       if (Context1.Location != Context2.Location)
@@ -707,15 +700,16 @@ class SampleContext {
       return false;
     ThatContext = ThatContext.take_front(ThisContext.size());
     // Compare Leaf frame first
-    if (ThisContext.back().FuncName != ThatContext.back().FuncName)
+    if (ThisContext.back().Func != ThatContext.back().Func)
       return false;
     // Compare leading context
     return ThisContext.drop_back() == ThatContext.drop_back();
   }
 
 private:
-  /// Mangled name of the function.
-  StringRef Name;
+  // The function associated with this context. If CS profile, this is the leaf
+  // function.
+  FunctionId Func;
   // Full context including calling context and leaf function name
   SampleContextFrames FullContext;
   // State of the associated sample profile
@@ -738,7 +732,7 @@ class SampleProfileReaderItaniumRemapper;
 using BodySampleMap = std::map<LineLocation, SampleRecord>;
 // NOTE: Using a StringMap here makes parsed profiles consume around 17% more
 // memory, which is *very* significant for large profiles.
-using FunctionSamplesMap = std::map<std::string, FunctionSamples, std::less<>>;
+using FunctionSamplesMap = std::map<FunctionId, FunctionSamples>;
 using CallsiteSampleMap = std::map<LineLocation, FunctionSamplesMap>;
 using LocToLocMap =
     std::unordered_map<LineLocation, LineLocation, LineLocationHash>;
@@ -790,14 +784,16 @@ class FunctionSamples {
 
   sampleprof_error addCalledTargetSamples(uint32_t LineOffset,
                                           uint32_t Discriminator,
-                                          StringRef FName, uint64_t Num,
+                                          FunctionId Func,
+                                          uint64_t Num,
                                           uint64_t Weight = 1) {
     return BodySamples[LineLocation(LineOffset, Discriminator)].addCalledTarget(
-        FName, Num, Weight);
+        Func, Num, Weight);
   }
 
   sampleprof_error addSampleRecord(LineLocation Location,
-                                   const SampleRecord &SampleRecord, uint64_t Weight = 1) {
+                                   const SampleRecord &SampleRecord,
+                                   uint64_t Weight = 1) {
     return BodySamples[Location].merge(SampleRecord, Weight);
   }
 
@@ -805,11 +801,11 @@ class FunctionSamples {
   // the number of body samples actually decreased.
   uint64_t removeCalledTargetAndBodySample(uint32_t LineOffset,
                                            uint32_t Discriminator,
-                                           StringRef FName) {
+                                           FunctionId Func) {
     uint64_t Count = 0;
     auto I = BodySamples.find(LineLocation(LineOffset, Discriminator));
     if (I != BodySamples.end()) {
-      Count = I->second.removeCalledTarget(FName);
+      Count = I->second.removeCalledTarget(Func);
       Count = I->second.removeSamples(Count);
       if (!I->second.getSamples())
         BodySamples.erase(I);
@@ -1002,7 +998,7 @@ class FunctionSamples {
     sampleprof_error Result = sampleprof_error::success;
     if (!GUIDToFuncNameMap)
       GUIDToFuncNameMap = Other.GUIDToFuncNameMap;
-    if (Context.getName().empty())
+    if (Context.getFunction().empty())
       Context = Other.getContext();
     if (FunctionHash == 0) {
       // Set the function hash code for the target profile.
@@ -1039,25 +1035,26 @@ class FunctionSamples {
   /// GUID to \p S. Also traverse the BodySamples to add hot CallTarget's GUID
   /// to \p S.
   void findInlinedFunctions(DenseSet<GlobalValue::GUID> &S,
-                            const StringMap<Function *> &SymbolMap,
+                            const HashKeyMap<std::unordered_map, FunctionId,
+                                             Function *>  &SymbolMap,
                             uint64_t Threshold) const {
     if (TotalSamples <= Threshold)
       return;
     auto isDeclaration = [](const Function *F) {
       return !F || F->isDeclaration();
     };
-    if (isDeclaration(SymbolMap.lookup(getFuncName()))) {
+    if (isDeclaration(SymbolMap.lookup(getFunction()))) {
       // Add to the import list only when it's defined out of module.
-      S.insert(getGUID(getName()));
+      S.insert(getGUID());
     }
     // Import hot CallTargets, which may not be available in IR because full
     // profile annotation cannot be done until backend compilation in ThinLTO.
     for (const auto &BS : BodySamples)
       for (const auto &TS : BS.second.getCallTargets())
-        if (TS.getValue() > Threshold) {
-          const Function *Callee = SymbolMap.lookup(getFuncName(TS.getKey()));
+        if (TS.second > Threshold) {
+          const Function *Callee = SymbolMap.lookup(TS.first);
           if (isDeclaration(Callee))
-            S.insert(getGUID(TS.getKey()));
+            S.insert(TS.first.getHashCode());
         }
     for (const auto &CS : CallsiteSamples)
       for (const auto &NameFS : CS.second)
@@ -1065,13 +1062,15 @@ class FunctionSamples {
   }
 
   /// Set the name of the function.
-  void setName(StringRef FunctionName) { Context.setName(FunctionName); }
+  void setFunction(FunctionId newFunction) {
+    Context.setFunction(newFunction);
+  }
 
   /// Return the function name.
-  StringRef getName() const { return Context.getName(); }
+  FunctionId getFunction() const { return Context.getFunction(); }
 
   /// Return the original function name.
-  StringRef getFuncName() const { return getFuncName(getName()); }
+  StringRef getFuncName() const { return getFuncName(getFunction()); }
 
   void setFunctionHash(uint64_t Hash) { FunctionHash = Hash; }
 
@@ -1128,19 +1127,19 @@ class FunctionSamples {
     return FnName;
   }
 
-  /// Translate \p Name into its original name.
-  /// When profile doesn't use MD5, \p Name needs no translation.
-  /// When profile uses MD5, \p Name in current FunctionSamples
+  /// Translate \p Func into its original name.
+  /// When profile doesn't use MD5, \p Func needs no translation.
+  /// When profile uses MD5, \p Func in current FunctionSamples
   /// is actually GUID of the original function name. getFuncName will
-  /// translate \p Name in current FunctionSamples into its original name
+  /// translate \p Func in current FunctionSamples into its original name
   /// by looking up in the function map GUIDToFuncNameMap.
   /// If the original name doesn't exist in the map, return empty StringRef.
-  StringRef getFuncName(StringRef Name) const {
+  StringRef getFuncName(FunctionId Func) const {
     if (!UseMD5)
-      return Name;
+      return Func.stringRef();
 
     assert(GUIDToFuncNameMap && "GUIDToFuncNameMap needs to be populated first");
-    return GUIDToFuncNameMap->lookup(std::stoull(Name.data()));
+    return GUIDToFuncNameMap->lookup(Func.getHashCode());
   }
 
   /// Returns the line offset to the start line of the subprogram.
@@ -1156,8 +1155,12 @@ class FunctionSamples {
 
   /// Returns a unique hash code for a combination of a callsite location and
   /// the callee function name.
-  static uint64_t getCallSiteHash(StringRef CalleeName,
-                                  const LineLocation &Callsite);
+  /// Guarantee MD5 and non-MD5 representation of the same function results in
+  /// the same hash.
+  static uint64_t getCallSiteHash(FunctionId Callee,
+                                  const LineLocation &Callsite) {
+    return SampleContextFrame(Callee, Callsite).getHashCode();
+  }
 
   /// Get the FunctionSamples of the inline instance where DIL originates
   /// from.
@@ -1197,16 +1200,15 @@ class FunctionSamples {
   /// all the function symbols defined or declared in current module.
   DenseMap<uint64_t, StringRef> *GUIDToFuncNameMap = nullptr;
 
-  // Assume the input \p Name is a name coming from FunctionSamples itself.
-  // If UseMD5 is true, the name is already a GUID and we
-  // don't want to return the GUID of GUID.
-  static uint64_t getGUID(StringRef Name) {
-    return UseMD5 ? std::stoull(Name.data()) : Function::getGUID(Name);
+  /// Return the GUID of the context's name. If the context is already using
+  /// MD5, don't hash it again.
+  uint64_t getGUID() const {
+    return getFunction().getHashCode();
   }
 
   // Find all the names in the current FunctionSamples including names in
   // all the inline instances and names of call targets.
-  void findAllNames(DenseSet<StringRef> &NameSet) const;
+  void findAllNames(DenseSet<FunctionId> &NameSet) const;
 
   bool operator==(const FunctionSamples &Other) const {
     return (GUIDToFuncNameMap == Other.GUIDToFuncNameMap ||
@@ -1223,9 +1225,6 @@ class FunctionSamples {
     return !(*this == Other);
   }
 
-  template <typename T>
-  const T &getKey() const;
-
 private:
   /// CFG hash value for the function.
   uint64_t FunctionHash = 0;
@@ -1289,90 +1288,16 @@ class FunctionSamples {
   const LocToLocMap *IRToProfileLocationMap = nullptr;
 };
 
-template <>
-inline const SampleContext &FunctionSamples::getKey<SampleContext>() const {
-  return getContext();
+/// Get the proper representation of a string according to whether the
+/// current Format uses MD5 to represent the string.
+static inline FunctionId getRepInFormat(StringRef Name) {
+  if (Name.empty() || !FunctionSamples::UseMD5)
+    return FunctionId(Name);
+  return FunctionId(Function::getGUID(Name));
 }
 
 raw_ostream &operator<<(raw_ostream &OS, const FunctionSamples &FS);
 
-/// This class is a wrapper to associative container MapT<KeyT, ValueT> using
-/// the hash value of the original key as the new key. This greatly improves the
-/// performance of insert and query operations especially when hash values of
-/// keys are available a priori, and reduces memory usage if KeyT has a large
-/// size.
-/// All keys with the same hash value are considered equivalent (i.e. hash
-/// collision is silently ignored). Given such feature this class should only be
-/// used where it does not affect compilation correctness, for example, when
-/// loading a sample profile.
-/// Assuming the hashing algorithm is uniform, we use the formula
-/// 1 - Permute(n, k) / n ^ k where n is the universe size and k is number of
-/// elements chosen at random to calculate the probability of collision. With
-/// 1,000,000 entries the probability is negligible:
-/// 1 - (2^64)!/((2^64-1000000)!*(2^64)^1000000) ~= 3*10^-8.
-/// Source: https://en.wikipedia.org/wiki/Birthday_problem
-template <template <typename, typename, typename...> typename MapT,
-          typename KeyT, typename ValueT, typename... MapTArgs>
-class HashKeyMap : public MapT<hash_code, ValueT, MapTArgs...> {
-public:
-  using base_type = MapT<hash_code, ValueT, MapTArgs...>;
-  using key_type = hash_code;
-  using original_key_type = KeyT;
-  using mapped_type = ValueT;
-  using value_type = typename base_type::value_type;
-
-  using iterator = typename base_type::iterator;
-  using const_iterator = typename base_type::const_iterator;
-
-  template <typename... Ts>
-  std::pair<iterator, bool> try_emplace(const key_type &Hash,
-                                        const original_key_type &Key,
-                                        Ts &&...Args) {
-    assert(Hash == hash_value(Key));
-    return base_type::try_emplace(Hash, std::forward<Ts>(Args)...);
-  }
-
-  template <typename... Ts>
-  std::pair<iterator, bool> try_emplace(const original_key_type &Key,
-                                        Ts &&...Args) {
-    key_type Hash = hash_value(Key);
-    return try_emplace(Hash, Key, std::forward<Ts>(Args)...);
-  }
-
-  template <typename... Ts> std::pair<iterator, bool> emplace(Ts &&...Args) {
-    return try_emplace(std::forward<Ts>(Args)...);
-  }
-
-  mapped_type &operator[](const original_key_type &Key) {
-    return try_emplace(Key, mapped_type()).first->second;
-  }
-
-  iterator find(const original_key_type &Key) {
-    key_type Hash = hash_value(Key);
-    auto It = base_type::find(Hash);
-    if (It != base_type::end())
-      return It;
-    return base_type::end();
-  }
-
-  const_iterator find(const original_key_type &Key) const {
-    key_type Hash = hash_value(Key);
-    auto It = base_type::find(Hash);
-    if (It != base_type::end())
-      return It;
-    return base_type::end();
-  }
-
-  size_t erase(const original_key_type &Ctx) {
-    auto It = find(Ctx);
-    if (It != base_type::end()) {
-      base_type::erase(It);
-      return 1;
-    }
-    return 0;
-  }
-};
-
 /// This class provides operator overloads to the map container using MD5 as the
 /// key type, so that existing code can still work in most cases using
 /// SampleContext as key.
@@ -1400,11 +1325,6 @@ class SampleProfileMap
         Ctx);
   }
 
-  // Overloaded find() to lookup a function by name.
-  iterator find(StringRef Fname) {
-    return base_type::find(hashFuncName(Fname));
-  }
-
   size_t erase(const SampleContext &Ctx) {
     return HashKeyMap<std::unordered_map, SampleContext, FunctionSamples>::
         erase(Ctx);
@@ -1475,7 +1395,7 @@ class ProfileConverter {
   // profile.
   void convertCSProfiles();
   struct FrameNode {
-    FrameNode(StringRef FName = StringRef(),
+    FrameNode(FunctionId FName = FunctionId(),
               FunctionSamples *FSamples = nullptr,
               LineLocation CallLoc = {0, 0})
         : FuncName(FName), FuncSamples(FSamples), CallSiteLoc(CallLoc){};
@@ -1483,14 +1403,14 @@ class ProfileConverter {
     // Map line+discriminator location to child frame
     std::map<uint64_t, FrameNode> AllChildFrames;
     // Function name for current frame
-    StringRef FuncName;
+    FunctionId FuncName;
     // Function Samples for current frame
     FunctionSamples *FuncSamples;
     // Callsite location in parent context
     LineLocation CallSiteLoc;
 
     FrameNode *getOrCreateChildFrame(const LineLocation &CallSite,
-                                     StringRef CalleeName);
+                                     FunctionId CalleeName);
   };
 
   static void flattenProfile(SampleProfileMap &ProfileMap,
@@ -1507,7 +1427,7 @@ class ProfileConverter {
       for (const auto &I : InputProfiles) {
         // Retain the profile name and clear the full context for each function
         // profile.
-        FunctionSamples &FS = OutputProfiles.Create(I.second.getName());
+        FunctionSamples &FS = OutputProfiles.Create(I.second.getFunction());
         FS.merge(I.second);
       }
     } else {
@@ -1553,7 +1473,8 @@ class ProfileConverter {
                                CalleeProfile.getHeadSamplesEstimate());
         // Add callsite sample.
         Profile.addCalledTargetSamples(
-            I.first.LineOffset, I.first.Discriminator, CalleeProfile.getName(),
+            I.first.LineOffset, I.first.Discriminator,
+            CalleeProfile.getFunction(),
             CalleeProfile.getHeadSamplesEstimate());
         // Update total samples.
         TotalSamples = TotalSamples >= CalleeProfile.getTotalSamples()
@@ -1625,7 +1546,9 @@ using namespace sampleprof;
 template <> struct DenseMapInfo<SampleContext> {
   static inline SampleContext getEmptyKey() { return SampleContext(); }
 
-  static inline SampleContext getTombstoneKey() { return SampleContext("@"); }
+  static inline SampleContext getTombstoneKey() {
+    return SampleContext(FunctionId(~1ULL));
+  }
 
   static unsigned getHashValue(const SampleContext &Val) {
     return Val.getHashCode();
diff --git a/llvm/include/llvm/ProfileData/SampleProfReader.h b/llvm/include/llvm/ProfileData/SampleProfReader.h
index a5b1df3ef5..9e8f543909 100644
--- a/llvm/include/llvm/ProfileData/SampleProfReader.h
+++ b/llvm/include/llvm/ProfileData/SampleProfReader.h
@@ -409,13 +409,13 @@ class SampleProfileReader {
 
   /// Return the samples collected for function \p F.
   FunctionSamples *getSamplesFor(StringRef Fname) {
-    auto It = Profiles.find(Fname);
+    auto It = Profiles.find(FunctionId(Fname));
     if (It != Profiles.end())
       return &It->second;
 
     if (Remapper) {
       if (auto NameInProfile = Remapper->lookUpNameInProfile(Fname)) {
-        auto It = Profiles.find(*NameInProfile);
+        auto It = Profiles.find(FunctionId(*NameInProfile));
         if (It != Profiles.end())
           return &It->second;
       }
@@ -474,7 +474,7 @@ class SampleProfileReader {
 
   /// It includes all the names that have samples either in outline instance
   /// or inline instance.
-  virtual std::vector<StringRef> *getNameTable() { return nullptr; }
+  virtual std::vector<FunctionId> *getNameTable() { return nullptr; }
   virtual bool dumpSectionInfo(raw_ostream &OS = dbgs()) { return false; };
 
   /// Return whether names in the profile are all MD5 numbers.
@@ -508,10 +508,6 @@ class SampleProfileReader {
   /// Memory buffer holding the profile file.
   std::unique_ptr<MemoryBuffer> Buffer;
 
-  /// Extra name buffer holding names created on demand.
-  /// This should only be needed for md5 profiles.
-  std::unordered_set<std::string> MD5NameBuffer;
-
   /// Profile summary information.
   std::unique_ptr<ProfileSummary> Summary;
 
@@ -595,7 +591,9 @@ class SampleProfileReaderBinary : public SampleProfileReader {
 
   /// It includes all the names that have samples either in outline instance
   /// or inline instance.
-  std::vector<StringRef> *getNameTable() override { return &NameTable; }
+  std::vector<FunctionId> *getNameTable() override {
+    return &NameTable;
+  }
 
 protected:
   /// Read a numeric value of type T from the profile.
@@ -637,7 +635,7 @@ class SampleProfileReaderBinary : public SampleProfileReader {
   std::error_code readNameTable();
 
   /// Read a string indirectly via the name table. Optionally return the index.
-  ErrorOr<StringRef> readStringFromTable(size_t *RetIdx = nullptr);
+  ErrorOr<FunctionId> readStringFromTable(size_t *RetIdx = nullptr);
 
   /// Read a context indirectly via the CSNameTable. Optionally return the
   /// index.
@@ -654,19 +652,7 @@ class SampleProfileReaderBinary : public SampleProfileReader {
   const uint8_t *End = nullptr;
 
   /// Function name table.
-  std::vector<StringRef> NameTable;
-
-  /// If MD5 is used in NameTable section, the section saves uint64_t data.
-  /// The uint64_t data has to be converted to a string and then the string
-  /// will be used to initialize StringRef in NameTable.
-  /// Note NameTable contains StringRef so it needs another buffer to own
-  /// the string data. MD5StringBuf serves as the string buffer that is
-  /// referenced by NameTable (vector of StringRef). We make sure
-  /// the lifetime of MD5StringBuf is not shorter than that of NameTable.
-  std::vector<std::string> MD5StringBuf;
-
-  /// The starting address of fixed length MD5 name table section.
-  const uint8_t *MD5NameMemStart = nullptr;
+  std::vector<FunctionId> NameTable;
 
   /// CSNameTable is used to save full context vectors. It is the backing buffer
   /// for SampleContextFrames.
diff --git a/llvm/include/llvm/ProfileData/SampleProfWriter.h b/llvm/include/llvm/ProfileData/SampleProfWriter.h
index 1f19283ea1..963a4d4918 100644
--- a/llvm/include/llvm/ProfileData/SampleProfWriter.h
+++ b/llvm/include/llvm/ProfileData/SampleProfWriter.h
@@ -196,20 +196,20 @@ class SampleProfileWriterBinary : public SampleProfileWriter {
   std::error_code writeSample(const FunctionSamples &S) override;
 
 protected:
-  virtual MapVector<StringRef, uint32_t> &getNameTable() { return NameTable; }
+  virtual MapVector<FunctionId, uint32_t> &getNameTable() { return NameTable; }
   virtual std::error_code writeMagicIdent(SampleProfileFormat Format);
   virtual std::error_code writeNameTable();
   std::error_code writeHeader(const SampleProfileMap &ProfileMap) override;
   std::error_code writeSummary();
   virtual std::error_code writeContextIdx(const SampleContext &Context);
-  std::error_code writeNameIdx(StringRef FName);
+  std::error_code writeNameIdx(FunctionId FName);
   std::error_code writeBody(const FunctionSamples &S);
-  inline void stablizeNameTable(MapVector<StringRef, uint32_t> &NameTable,
-                                std::set<StringRef> &V);
-
-  MapVector<StringRef, uint32_t> NameTable;
-
-  void addName(StringRef FName);
+  inline void stablizeNameTable(MapVector<FunctionId, uint32_t> &NameTable,
+                                std::set<FunctionId> &V);
+  
+  MapVector<FunctionId, uint32_t> NameTable;
+  
+  void addName(FunctionId FName);
   virtual void addContext(const SampleContext &Context);
   void addNames(const FunctionSamples &S);
 
diff --git a/llvm/include/llvm/Remarks/Remark.h b/llvm/include/llvm/Remarks/Remark.h
index a66f7ed73f..de81c5a992 100644
--- a/llvm/include/llvm/Remarks/Remark.h
+++ b/llvm/include/llvm/Remarks/Remark.h
@@ -52,6 +52,10 @@ struct Argument {
 
   /// Implement operator<< on Argument.
   void print(raw_ostream &OS) const;
+  /// Return the value of argument as int.
+  std::optional<int> getValAsInt() const;
+  /// Check if the argument value can be parsed as int.
+  bool isValInt() const;
 };
 
 // Create wrappers for C Binding types (see CBindingWrapping.h).
diff --git a/llvm/include/llvm/Support/AMDHSAKernelDescriptor.h b/llvm/include/llvm/Support/AMDHSAKernelDescriptor.h
index 1bd65471d3..0574f96e6e 100644
--- a/llvm/include/llvm/Support/AMDHSAKernelDescriptor.h
+++ b/llvm/include/llvm/Support/AMDHSAKernelDescriptor.h
@@ -79,8 +79,21 @@ enum : uint8_t {
 };
 
 // Compute program resource register 1. Must match hardware definition.
+// GFX6+.
 #define COMPUTE_PGM_RSRC1(NAME, SHIFT, WIDTH) \
   AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC1_ ## NAME, SHIFT, WIDTH)
+// [GFX6-GFX8].
+#define COMPUTE_PGM_RSRC1_GFX6_GFX8(NAME, SHIFT, WIDTH) \
+  AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC1_GFX6_GFX8_ ## NAME, SHIFT, WIDTH)
+// [GFX6-GFX9].
+#define COMPUTE_PGM_RSRC1_GFX6_GFX9(NAME, SHIFT, WIDTH) \
+  AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC1_GFX6_GFX9_ ## NAME, SHIFT, WIDTH)
+// GFX9+.
+#define COMPUTE_PGM_RSRC1_GFX9_PLUS(NAME, SHIFT, WIDTH) \
+  AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC1_GFX9_PLUS_ ## NAME, SHIFT, WIDTH)
+// GFX10+.
+#define COMPUTE_PGM_RSRC1_GFX10_PLUS(NAME, SHIFT, WIDTH) \
+  AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC1_GFX10_PLUS_ ## NAME, SHIFT, WIDTH)
 enum : int32_t {
   COMPUTE_PGM_RSRC1(GRANULATED_WORKITEM_VGPR_COUNT, 0, 6),
   COMPUTE_PGM_RSRC1(GRANULATED_WAVEFRONT_SGPR_COUNT, 6, 4),
@@ -95,11 +108,13 @@ enum : int32_t {
   COMPUTE_PGM_RSRC1(ENABLE_IEEE_MODE, 23, 1),
   COMPUTE_PGM_RSRC1(BULKY, 24, 1),
   COMPUTE_PGM_RSRC1(CDBG_USER, 25, 1),
-  COMPUTE_PGM_RSRC1(FP16_OVFL, 26, 1),    // GFX9+
-  COMPUTE_PGM_RSRC1(RESERVED0, 27, 2),
-  COMPUTE_PGM_RSRC1(WGP_MODE, 29, 1),     // GFX10+
-  COMPUTE_PGM_RSRC1(MEM_ORDERED, 30, 1),  // GFX10+
-  COMPUTE_PGM_RSRC1(FWD_PROGRESS, 31, 1), // GFX10+
+  COMPUTE_PGM_RSRC1_GFX6_GFX8(RESERVED0, 26, 1),
+  COMPUTE_PGM_RSRC1_GFX9_PLUS(FP16_OVFL, 26, 1),
+  COMPUTE_PGM_RSRC1(RESERVED1, 27, 2),
+  COMPUTE_PGM_RSRC1_GFX6_GFX9(RESERVED2, 29, 3),
+  COMPUTE_PGM_RSRC1_GFX10_PLUS(WGP_MODE, 29, 1),
+  COMPUTE_PGM_RSRC1_GFX10_PLUS(MEM_ORDERED, 30, 1),
+  COMPUTE_PGM_RSRC1_GFX10_PLUS(FWD_PROGRESS, 31, 1),
 };
 #undef COMPUTE_PGM_RSRC1
 
@@ -143,15 +158,24 @@ enum : int32_t {
 
 // Compute program resource register 3 for GFX10+. Must match hardware
 // definition.
+// [GFX10].
+#define COMPUTE_PGM_RSRC3_GFX10(NAME, SHIFT, WIDTH) \
+  AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC3_GFX10_ ## NAME, SHIFT, WIDTH)
+// GFX10+.
 #define COMPUTE_PGM_RSRC3_GFX10_PLUS(NAME, SHIFT, WIDTH) \
   AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC3_GFX10_PLUS_ ## NAME, SHIFT, WIDTH)
+// GFX11+.
+#define COMPUTE_PGM_RSRC3_GFX11_PLUS(NAME, SHIFT, WIDTH) \
+  AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC3_GFX11_PLUS_ ## NAME, SHIFT, WIDTH)
 enum : int32_t {
-  COMPUTE_PGM_RSRC3_GFX10_PLUS(SHARED_VGPR_COUNT, 0, 4), // GFX10+
-  COMPUTE_PGM_RSRC3_GFX10_PLUS(INST_PREF_SIZE, 4, 6),    // GFX11+
-  COMPUTE_PGM_RSRC3_GFX10_PLUS(TRAP_ON_START, 10, 1),    // GFX11+
-  COMPUTE_PGM_RSRC3_GFX10_PLUS(TRAP_ON_END, 11, 1),      // GFX11+
-  COMPUTE_PGM_RSRC3_GFX10_PLUS(RESERVED0, 12, 19),
-  COMPUTE_PGM_RSRC3_GFX10_PLUS(IMAGE_OP, 31, 1),         // GFX11+
+  COMPUTE_PGM_RSRC3_GFX10_PLUS(SHARED_VGPR_COUNT, 0, 4),
+  COMPUTE_PGM_RSRC3_GFX10(RESERVED0, 4, 8),
+  COMPUTE_PGM_RSRC3_GFX11_PLUS(INST_PREF_SIZE, 4, 6),
+  COMPUTE_PGM_RSRC3_GFX11_PLUS(TRAP_ON_START, 10, 1),
+  COMPUTE_PGM_RSRC3_GFX11_PLUS(TRAP_ON_END, 11, 1),
+  COMPUTE_PGM_RSRC3_GFX10_PLUS(RESERVED1, 12, 19),
+  COMPUTE_PGM_RSRC3_GFX10(RESERVED2, 31, 1),
+  COMPUTE_PGM_RSRC3_GFX11_PLUS(IMAGE_OP, 31, 1),
 };
 #undef COMPUTE_PGM_RSRC3_GFX10_PLUS
 
diff --git a/llvm/include/llvm/Support/BinaryByteStream.h b/llvm/include/llvm/Support/BinaryByteStream.h
index d313994fd5..ed1a0b2297 100644
--- a/llvm/include/llvm/Support/BinaryByteStream.h
+++ b/llvm/include/llvm/Support/BinaryByteStream.h
@@ -133,7 +133,7 @@ class MutableBinaryByteStream : public WritableBinaryStream {
 /// causing the underlying data to grow.  This class owns the underlying data.
 class AppendingBinaryByteStream : public WritableBinaryStream {
   std::vector<uint8_t> Data;
-  llvm::endianness Endian = llvm::support::little;
+  llvm::endianness Endian = llvm::endianness::little;
 
 public:
   AppendingBinaryByteStream() = default;
diff --git a/llvm/include/llvm/Support/Chrono.h b/llvm/include/llvm/Support/Chrono.h
index 9c2bd45d28..71859af7c7 100644
--- a/llvm/include/llvm/Support/Chrono.h
+++ b/llvm/include/llvm/Support/Chrono.h
@@ -33,6 +33,19 @@ namespace sys {
 template <typename D = std::chrono::nanoseconds>
 using TimePoint = std::chrono::time_point<std::chrono::system_clock, D>;
 
+// utc_clock and utc_time are only available since C++20. Add enough code to
+// support formatting date/time in UTC.
+class UtcClock : public std::chrono::system_clock {};
+
+template <typename D = std::chrono::nanoseconds>
+using UtcTime = std::chrono::time_point<UtcClock, D>;
+
+/// Convert a std::time_t to a UtcTime
+inline UtcTime<std::chrono::seconds> toUtcTime(std::time_t T) {
+  using namespace std::chrono;
+  return UtcTime<seconds>(seconds(T));
+}
+
 /// Convert a TimePoint to std::time_t
 inline std::time_t toTimeT(TimePoint<> TP) {
   using namespace std::chrono;
@@ -40,6 +53,13 @@ inline std::time_t toTimeT(TimePoint<> TP) {
       time_point_cast<system_clock::time_point::duration>(TP));
 }
 
+/// Convert a UtcTime to std::time_t
+inline std::time_t toTimeT(UtcTime<> TP) {
+  using namespace std::chrono;
+  return system_clock::to_time_t(time_point<system_clock, seconds>(
+      duration_cast<seconds>(TP.time_since_epoch())));
+}
+
 /// Convert a std::time_t to a TimePoint
 inline TimePoint<std::chrono::seconds>
 toTimePoint(std::time_t T) {
@@ -58,6 +78,7 @@ toTimePoint(std::time_t T, uint32_t nsec) {
 } // namespace sys
 
 raw_ostream &operator<<(raw_ostream &OS, sys::TimePoint<> TP);
+raw_ostream &operator<<(raw_ostream &OS, sys::UtcTime<> TP);
 
 /// Format provider for TimePoint<>
 ///
@@ -73,6 +94,11 @@ struct format_provider<sys::TimePoint<>> {
                      StringRef Style);
 };
 
+template <> struct format_provider<sys::UtcTime<std::chrono::seconds>> {
+  static void format(const sys::UtcTime<std::chrono::seconds> &TP,
+                     llvm::raw_ostream &OS, StringRef Style);
+};
+
 namespace detail {
 template <typename Period> struct unit { static const char value[]; };
 template <typename Period> const char unit<Period>::value[] = "";
diff --git a/llvm/include/llvm/Support/Debug.h b/llvm/include/llvm/Support/Debug.h
index 5788ab3b21..3e2f0d9b43 100644
--- a/llvm/include/llvm/Support/Debug.h
+++ b/llvm/include/llvm/Support/Debug.h
@@ -53,7 +53,7 @@ void setCurrentDebugType(const char *Type);
 void setCurrentDebugTypes(const char **Types, unsigned Count);
 
 /// DEBUG_WITH_TYPE macro - This macro should be used by passes to emit debug
-/// information.  In the '-debug' option is specified on the commandline, and if
+/// information.  If the '-debug' option is specified on the commandline, and if
 /// this is a debug build, then the code specified as the option to the macro
 /// will be executed.  Otherwise it will not be.  Example:
 ///
@@ -92,7 +92,7 @@ extern bool EnableDebugBuffering;
 raw_ostream &dbgs();
 
 // DEBUG macro - This macro should be used by passes to emit debug information.
-// In the '-debug' option is specified on the commandline, and if this is a
+// If the '-debug' option is specified on the commandline, and if this is a
 // debug build, then the code specified as the option to the macro will be
 // executed.  Otherwise it will not be.  Example:
 //
diff --git a/llvm/include/llvm/Support/ELFAttributeParser.h b/llvm/include/llvm/Support/ELFAttributeParser.h
index 75ed82b3f6..ceb84540d6 100644
--- a/llvm/include/llvm/Support/ELFAttributeParser.h
+++ b/llvm/include/llvm/Support/ELFAttributeParser.h
@@ -58,7 +58,7 @@ class ELFAttributeParser {
   ELFAttributeParser(TagNameMap tagNameMap, StringRef vendor)
       : vendor(vendor), sw(nullptr), tagToStringMap(tagNameMap) {}
 
-  Error parse(ArrayRef<uint8_t> section, support::endianness endian);
+  Error parse(ArrayRef<uint8_t> section, llvm::endianness endian);
 
   std::optional<unsigned> getAttributeValue(unsigned tag) const {
     auto I = attributes.find(tag);
diff --git a/llvm/include/llvm/Support/Endian.h b/llvm/include/llvm/Support/Endian.h
index 808446e615..4c0405cf1e 100644
--- a/llvm/include/llvm/Support/Endian.h
+++ b/llvm/include/llvm/Support/Endian.h
@@ -25,13 +25,6 @@
 namespace llvm {
 namespace support {
 
-// TODO: Remove the following once we are done migrating to llvm::endianness,
-// llvm::endianness::big, etc.
-using endianness = llvm::endianness;
-constexpr llvm::endianness big = llvm::endianness::big;
-constexpr llvm::endianness little = llvm::endianness::little;
-constexpr llvm::endianness native = llvm::endianness::native;
-
 // These are named values for common alignments.
 enum {aligned = 0, unaligned = 1};
 
@@ -47,13 +40,9 @@ struct PickAlignment {
 
 namespace endian {
 
-LLVM_DEPRECATED("Use llvm::endianness::native instead",
-                "llvm::endianness::native")
-constexpr endianness system_endianness() { return llvm::endianness::native; }
-
 template <typename value_type>
 [[nodiscard]] inline value_type byte_swap(value_type value, endianness endian) {
-  if (endian != native)
+  if (endian != llvm::endianness::native)
     sys::swapByteOrder(value);
   return value;
 }
@@ -273,85 +262,120 @@ struct packed_endian_specific_integral {
 } // end namespace detail
 
 using ulittle16_t =
-    detail::packed_endian_specific_integral<uint16_t, little, unaligned>;
+    detail::packed_endian_specific_integral<uint16_t, llvm::endianness::little,
+                                            unaligned>;
 using ulittle32_t =
-    detail::packed_endian_specific_integral<uint32_t, little, unaligned>;
+    detail::packed_endian_specific_integral<uint32_t, llvm::endianness::little,
+                                            unaligned>;
 using ulittle64_t =
-    detail::packed_endian_specific_integral<uint64_t, little, unaligned>;
+    detail::packed_endian_specific_integral<uint64_t, llvm::endianness::little,
+                                            unaligned>;
 
 using little16_t =
-    detail::packed_endian_specific_integral<int16_t, little, unaligned>;
+    detail::packed_endian_specific_integral<int16_t, llvm::endianness::little,
+                                            unaligned>;
 using little32_t =
-    detail::packed_endian_specific_integral<int32_t, little, unaligned>;
+    detail::packed_endian_specific_integral<int32_t, llvm::endianness::little,
+                                            unaligned>;
 using little64_t =
-    detail::packed_endian_specific_integral<int64_t, little, unaligned>;
+    detail::packed_endian_specific_integral<int64_t, llvm::endianness::little,
+                                            unaligned>;
 
 using aligned_ulittle16_t =
-    detail::packed_endian_specific_integral<uint16_t, little, aligned>;
+    detail::packed_endian_specific_integral<uint16_t, llvm::endianness::little,
+                                            aligned>;
 using aligned_ulittle32_t =
-    detail::packed_endian_specific_integral<uint32_t, little, aligned>;
+    detail::packed_endian_specific_integral<uint32_t, llvm::endianness::little,
+                                            aligned>;
 using aligned_ulittle64_t =
-    detail::packed_endian_specific_integral<uint64_t, little, aligned>;
+    detail::packed_endian_specific_integral<uint64_t, llvm::endianness::little,
+                                            aligned>;
 
 using aligned_little16_t =
-    detail::packed_endian_specific_integral<int16_t, little, aligned>;
+    detail::packed_endian_specific_integral<int16_t, llvm::endianness::little,
+                                            aligned>;
 using aligned_little32_t =
-    detail::packed_endian_specific_integral<int32_t, little, aligned>;
+    detail::packed_endian_specific_integral<int32_t, llvm::endianness::little,
+                                            aligned>;
 using aligned_little64_t =
-    detail::packed_endian_specific_integral<int64_t, little, aligned>;
+    detail::packed_endian_specific_integral<int64_t, llvm::endianness::little,
+                                            aligned>;
 
 using ubig16_t =
-    detail::packed_endian_specific_integral<uint16_t, big, unaligned>;
+    detail::packed_endian_specific_integral<uint16_t, llvm::endianness::big,
+                                            unaligned>;
 using ubig32_t =
-    detail::packed_endian_specific_integral<uint32_t, big, unaligned>;
+    detail::packed_endian_specific_integral<uint32_t, llvm::endianness::big,
+                                            unaligned>;
 using ubig64_t =
-    detail::packed_endian_specific_integral<uint64_t, big, unaligned>;
+    detail::packed_endian_specific_integral<uint64_t, llvm::endianness::big,
+                                            unaligned>;
 
 using big16_t =
-    detail::packed_endian_specific_integral<int16_t, big, unaligned>;
+    detail::packed_endian_specific_integral<int16_t, llvm::endianness::big,
+                                            unaligned>;
 using big32_t =
-    detail::packed_endian_specific_integral<int32_t, big, unaligned>;
+    detail::packed_endian_specific_integral<int32_t, llvm::endianness::big,
+                                            unaligned>;
 using big64_t =
-    detail::packed_endian_specific_integral<int64_t, big, unaligned>;
+    detail::packed_endian_specific_integral<int64_t, llvm::endianness::big,
+                                            unaligned>;
 
 using aligned_ubig16_t =
-    detail::packed_endian_specific_integral<uint16_t, big, aligned>;
+    detail::packed_endian_specific_integral<uint16_t, llvm::endianness::big,
+                                            aligned>;
 using aligned_ubig32_t =
-    detail::packed_endian_specific_integral<uint32_t, big, aligned>;
+    detail::packed_endian_specific_integral<uint32_t, llvm::endianness::big,
+                                            aligned>;
 using aligned_ubig64_t =
-    detail::packed_endian_specific_integral<uint64_t, big, aligned>;
+    detail::packed_endian_specific_integral<uint64_t, llvm::endianness::big,
+                                            aligned>;
 
 using aligned_big16_t =
-    detail::packed_endian_specific_integral<int16_t, big, aligned>;
+    detail::packed_endian_specific_integral<int16_t, llvm::endianness::big,
+                                            aligned>;
 using aligned_big32_t =
-    detail::packed_endian_specific_integral<int32_t, big, aligned>;
+    detail::packed_endian_specific_integral<int32_t, llvm::endianness::big,
+                                            aligned>;
 using aligned_big64_t =
-    detail::packed_endian_specific_integral<int64_t, big, aligned>;
+    detail::packed_endian_specific_integral<int64_t, llvm::endianness::big,
+                                            aligned>;
 
 using unaligned_uint16_t =
-    detail::packed_endian_specific_integral<uint16_t, native, unaligned>;
+    detail::packed_endian_specific_integral<uint16_t, llvm::endianness::native,
+                                            unaligned>;
 using unaligned_uint32_t =
-    detail::packed_endian_specific_integral<uint32_t, native, unaligned>;
+    detail::packed_endian_specific_integral<uint32_t, llvm::endianness::native,
+                                            unaligned>;
 using unaligned_uint64_t =
-    detail::packed_endian_specific_integral<uint64_t, native, unaligned>;
+    detail::packed_endian_specific_integral<uint64_t, llvm::endianness::native,
+                                            unaligned>;
 
 using unaligned_int16_t =
-    detail::packed_endian_specific_integral<int16_t, native, unaligned>;
+    detail::packed_endian_specific_integral<int16_t, llvm::endianness::native,
+                                            unaligned>;
 using unaligned_int32_t =
-    detail::packed_endian_specific_integral<int32_t, native, unaligned>;
+    detail::packed_endian_specific_integral<int32_t, llvm::endianness::native,
+                                            unaligned>;
 using unaligned_int64_t =
-    detail::packed_endian_specific_integral<int64_t, native, unaligned>;
+    detail::packed_endian_specific_integral<int64_t, llvm::endianness::native,
+                                            unaligned>;
 
 template <typename T>
-using little_t = detail::packed_endian_specific_integral<T, little, unaligned>;
+using little_t =
+    detail::packed_endian_specific_integral<T, llvm::endianness::little,
+                                            unaligned>;
 template <typename T>
-using big_t = detail::packed_endian_specific_integral<T, big, unaligned>;
+using big_t = detail::packed_endian_specific_integral<T, llvm::endianness::big,
+                                                      unaligned>;
 
 template <typename T>
 using aligned_little_t =
-    detail::packed_endian_specific_integral<T, little, aligned>;
+    detail::packed_endian_specific_integral<T, llvm::endianness::little,
+                                            aligned>;
 template <typename T>
-using aligned_big_t = detail::packed_endian_specific_integral<T, big, aligned>;
+using aligned_big_t =
+    detail::packed_endian_specific_integral<T, llvm::endianness::big, aligned>;
 
 namespace endian {
 
@@ -380,17 +404,23 @@ template <endianness E> [[nodiscard]] inline uint64_t read64(const void *P) {
 }
 
 [[nodiscard]] inline uint16_t read16le(const void *P) {
-  return read16<little>(P);
+  return read16<llvm::endianness::little>(P);
 }
 [[nodiscard]] inline uint32_t read32le(const void *P) {
-  return read32<little>(P);
+  return read32<llvm::endianness::little>(P);
 }
 [[nodiscard]] inline uint64_t read64le(const void *P) {
-  return read64<little>(P);
+  return read64<llvm::endianness::little>(P);
+}
+[[nodiscard]] inline uint16_t read16be(const void *P) {
+  return read16<llvm::endianness::big>(P);
+}
+[[nodiscard]] inline uint32_t read32be(const void *P) {
+  return read32<llvm::endianness::big>(P);
+}
+[[nodiscard]] inline uint64_t read64be(const void *P) {
+  return read64<llvm::endianness::big>(P);
 }
-[[nodiscard]] inline uint16_t read16be(const void *P) { return read16<big>(P); }
-[[nodiscard]] inline uint32_t read32be(const void *P) { return read32<big>(P); }
-[[nodiscard]] inline uint64_t read64be(const void *P) { return read64<big>(P); }
 
 template <typename T, endianness E> inline void write(void *P, T V) {
   *(detail::packed_endian_specific_integral<T, E, unaligned> *)P = V;
@@ -416,12 +446,24 @@ template <endianness E> inline void write64(void *P, uint64_t V) {
   write<uint64_t, E>(P, V);
 }
 
-inline void write16le(void *P, uint16_t V) { write16<little>(P, V); }
-inline void write32le(void *P, uint32_t V) { write32<little>(P, V); }
-inline void write64le(void *P, uint64_t V) { write64<little>(P, V); }
-inline void write16be(void *P, uint16_t V) { write16<big>(P, V); }
-inline void write32be(void *P, uint32_t V) { write32<big>(P, V); }
-inline void write64be(void *P, uint64_t V) { write64<big>(P, V); }
+inline void write16le(void *P, uint16_t V) {
+  write16<llvm::endianness::little>(P, V);
+}
+inline void write32le(void *P, uint32_t V) {
+  write32<llvm::endianness::little>(P, V);
+}
+inline void write64le(void *P, uint64_t V) {
+  write64<llvm::endianness::little>(P, V);
+}
+inline void write16be(void *P, uint16_t V) {
+  write16<llvm::endianness::big>(P, V);
+}
+inline void write32be(void *P, uint32_t V) {
+  write32<llvm::endianness::big>(P, V);
+}
+inline void write64be(void *P, uint64_t V) {
+  write64<llvm::endianness::big>(P, V);
+}
 
 } // end namespace endian
 
diff --git a/llvm/include/llvm/Support/EndianStream.h b/llvm/include/llvm/Support/EndianStream.h
index a1dac5ad9f..9b0a4bf30e 100644
--- a/llvm/include/llvm/Support/EndianStream.h
+++ b/llvm/include/llvm/Support/EndianStream.h
@@ -17,7 +17,6 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/Endian.h"
-#include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 
 namespace llvm {
diff --git a/llvm/include/llvm/Support/HashBuilder.h b/llvm/include/llvm/Support/HashBuilder.h
index ccbfe614eb..0971108744 100644
--- a/llvm/include/llvm/Support/HashBuilder.h
+++ b/llvm/include/llvm/Support/HashBuilder.h
@@ -135,7 +135,7 @@ template <typename HasherT> class HashBuilderBase {
 /// computation (for example when computing `add((int)123)`).
 /// Specifiying a non-`native` `Endianness` template parameter allows to compute
 /// stable hash across platforms with different endianness.
-template <typename HasherT, support::endianness Endianness>
+template <typename HasherT, llvm::endianness Endianness>
 class HashBuilder : public HashBuilderBase<HasherT> {
 public:
   explicit HashBuilder(HasherT &Hasher) : HashBuilderBase<HasherT>(Hasher) {}
@@ -213,7 +213,7 @@ class HashBuilder : public HashBuilderBase<HasherT> {
   /// providing a `addHash` templated function.
   ///
   /// ```
-  /// template <typename HasherT, support::endianness Endianness>
+  /// template <typename HasherT, llvm::endianness Endianness>
   /// void addHash(HashBuilder<HasherT, Endianness> &HBuilder,
   ///              const UserDefinedStruct &Value);
   /// ```
@@ -225,7 +225,7 @@ class HashBuilder : public HashBuilderBase<HasherT> {
   ///   int i;
   /// };
   ///
-  /// template <typename HasherT, support::endianness Endianness>
+  /// template <typename HasherT, llvm::endianness Endianness>
   /// void addHash(HashBuilder<HasherT, Endianness> &HBuilder,
   ///              const SimpleStruct &Value) {
   ///   HBuilder.add(Value.c);
@@ -245,7 +245,7 @@ class HashBuilder : public HashBuilderBase<HasherT> {
   ///
   ///   // If possible, we want to hash both `I` and `C` in a single
   ///   // `update` call for performance concerns.
-  ///   template <typename HasherT, support::endianness Endianness>
+  ///   template <typename HasherT, llvm::endianness Endianness>
   ///   friend void addHash(HashBuilder<HasherT, Endianness> &HBuilder,
   ///                       const StructWithFastHash &Value) {
   ///     if (Endianness == llvm::endianness::native) {
@@ -275,7 +275,7 @@ class HashBuilder : public HashBuilderBase<HasherT> {
   ///     for (size_t I = 0; I != Size; ++I)
   ///       Elements[I] = I;
   ///   }
-  ///   template <typename HasherT, support::endianness Endianness>
+  ///   template <typename HasherT, llvm::endianness Endianness>
   ///   friend void addHash(HashBuilder<HasherT, Endianness> &HBuilder,
   ///                       const CustomContainer &Value) {
   ///     if (Endianness == llvm::endianness::native) {
@@ -348,7 +348,7 @@ class HashBuilder : public HashBuilderBase<HasherT> {
 
   template <typename T>
   using HasByteSwapT = decltype(support::endian::byte_swap(
-      std::declval<T &>(), support::endianness::little));
+      std::declval<T &>(), llvm::endianness::little));
   /// Adjust `Value` for the target endianness and add it to the hash.
   template <typename T>
   std::enable_if_t<is_detected<HasByteSwapT, T>::value, HashBuilder &>
@@ -393,8 +393,8 @@ class HashCodeHasher {
   hash_code Code;
 };
 
-using HashCodeHashBuilder = HashBuilder<hashbuilder_detail::HashCodeHasher,
-                                        support::endianness::native>;
+using HashCodeHashBuilder =
+    HashBuilder<hashbuilder_detail::HashCodeHasher, llvm::endianness::native>;
 } // namespace hashbuilder_detail
 
 /// Provide a default implementation of `hash_value` when `addHash(const T &)`
diff --git a/llvm/include/llvm/Support/KnownBits.h b/llvm/include/llvm/Support/KnownBits.h
index 8462aa1120..fb034e0b9e 100644
--- a/llvm/include/llvm/Support/KnownBits.h
+++ b/llvm/include/llvm/Support/KnownBits.h
@@ -332,6 +332,11 @@ struct KnownBits {
   static KnownBits computeForAddSub(bool Add, bool NSW, const KnownBits &LHS,
                                     KnownBits RHS);
 
+  /// Compute known bits results from subtracting RHS from LHS with 1-bit
+  /// Borrow.
+  static KnownBits computeForSubBorrow(const KnownBits &LHS, KnownBits RHS,
+                                       const KnownBits &Borrow);
+
   /// Compute knownbits resulting from llvm.sadd.sat(LHS, RHS)
   static KnownBits sadd_sat(const KnownBits &LHS, const KnownBits &RHS);
 
diff --git a/llvm/include/llvm/Support/MD5.h b/llvm/include/llvm/Support/MD5.h
index fa2f477261..0e9f22d3bf 100644
--- a/llvm/include/llvm/Support/MD5.h
+++ b/llvm/include/llvm/Support/MD5.h
@@ -47,12 +47,12 @@ class MD5 {
       // Our MD5 implementation returns the result in little endian, so the low
       // word is first.
       using namespace support;
-      return endian::read<uint64_t, little, unaligned>(data());
+      return endian::read<uint64_t, llvm::endianness::little>(data());
     }
 
     uint64_t high() const {
       using namespace support;
-      return endian::read<uint64_t, little, unaligned>(data() + 8);
+      return endian::read<uint64_t, llvm::endianness::little>(data() + 8);
     }
     std::pair<uint64_t, uint64_t> words() const {
       using namespace support;
diff --git a/llvm/include/llvm/Support/OnDiskHashTable.h b/llvm/include/llvm/Support/OnDiskHashTable.h
index bb90d8fc3a..0a8cbbd8b1 100644
--- a/llvm/include/llvm/Support/OnDiskHashTable.h
+++ b/llvm/include/llvm/Support/OnDiskHashTable.h
@@ -149,7 +149,7 @@ template <typename Info> class OnDiskChainedHashTableGenerator {
   /// Uses the provided Info instead of a stack allocated one.
   offset_type Emit(raw_ostream &Out, Info &InfoObj) {
     using namespace llvm::support;
-    endian::Writer LE(Out, little);
+    endian::Writer LE(Out, llvm::endianness::little);
 
     // Now we're done adding entries, resize the bucket list if it's
     // significantly too large. (This only happens if the number of
@@ -304,9 +304,11 @@ template <typename Info> class OnDiskChainedHashTable {
            "buckets should be 4-byte aligned.");
     using namespace llvm::support;
     offset_type NumBuckets =
-        endian::readNext<offset_type, little, aligned>(Buckets);
+        endian::readNext<offset_type, llvm::endianness::little, aligned>(
+            Buckets);
     offset_type NumEntries =
-        endian::readNext<offset_type, little, aligned>(Buckets);
+        endian::readNext<offset_type, llvm::endianness::little, aligned>(
+            Buckets);
     return std::make_pair(NumBuckets, NumEntries);
   }
 
@@ -357,19 +359,23 @@ template <typename Info> class OnDiskChainedHashTable {
     offset_type Idx = KeyHash & (NumBuckets - 1);
     const unsigned char *Bucket = Buckets + sizeof(offset_type) * Idx;
 
-    offset_type Offset = endian::readNext<offset_type, little, aligned>(Bucket);
+    offset_type Offset =
+        endian::readNext<offset_type, llvm::endianness::little, aligned>(
+            Bucket);
     if (Offset == 0)
       return iterator(); // Empty bucket.
     const unsigned char *Items = Base + Offset;
 
     // 'Items' starts with a 16-bit unsigned integer representing the
     // number of items in this bucket.
-    unsigned Len = endian::readNext<uint16_t, little, unaligned>(Items);
+    unsigned Len =
+        endian::readNext<uint16_t, llvm::endianness::little, unaligned>(Items);
 
     for (unsigned i = 0; i < Len; ++i) {
       // Read the hash.
       hash_value_type ItemHash =
-          endian::readNext<hash_value_type, little, unaligned>(Items);
+          endian::readNext<hash_value_type, llvm::endianness::little,
+                           unaligned>(Items);
 
       // Determine the length of the key and the data.
       const std::pair<offset_type, offset_type> &L =
@@ -467,7 +473,8 @@ class OnDiskIterableChainedHashTable : public OnDiskChainedHashTable<Info> {
         // 'Items' starts with a 16-bit unsigned integer representing the
         // number of items in this bucket.
         NumItemsInBucketLeft =
-            endian::readNext<uint16_t, little, unaligned>(Ptr);
+            endian::readNext<uint16_t, llvm::endianness::little, unaligned>(
+                Ptr);
       }
       Ptr += sizeof(hash_value_type); // Skip the hash.
       // Determine the length of the key and the data.
diff --git a/llvm/include/llvm/Support/Regex.h b/llvm/include/llvm/Support/Regex.h
index ae4b9516f1..bb7a8009b6 100644
--- a/llvm/include/llvm/Support/Regex.h
+++ b/llvm/include/llvm/Support/Regex.h
@@ -85,8 +85,9 @@ namespace llvm {
                std::string *Error = nullptr) const;
 
     /// sub - Return the result of replacing the first match of the regex in
-    /// \p String with the \p Repl string. Backreferences like "\0" in the
-    /// replacement string are replaced with the appropriate match substring.
+    /// \p String with the \p Repl string. Backreferences like "\0" and "\g<1>"
+    /// in the replacement string are replaced with the appropriate match
+    /// substring.
     ///
     /// Note that the replacement string has backslash escaping performed on
     /// it. Invalid backreferences are ignored (replaced by empty strings).
diff --git a/llvm/include/llvm/Support/SwapByteOrder.h b/llvm/include/llvm/Support/SwapByteOrder.h
index 8f26af6f68..18fd41ac5b 100644
--- a/llvm/include/llvm/Support/SwapByteOrder.h
+++ b/llvm/include/llvm/Support/SwapByteOrder.h
@@ -15,7 +15,6 @@
 #define LLVM_SUPPORT_SWAPBYTEORDER_H
 
 #include "llvm/ADT/bit.h"
-#include <cstddef>
 #include <cstdint>
 #include <type_traits>
 
diff --git a/llvm/include/llvm/Support/ThreadSafeAllocator.h b/llvm/include/llvm/Support/ThreadSafeAllocator.h
index 3092287e69..8c56bb6e58 100644
--- a/llvm/include/llvm/Support/ThreadSafeAllocator.h
+++ b/llvm/include/llvm/Support/ThreadSafeAllocator.h
@@ -10,7 +10,6 @@
 #define LLVM_SUPPORT_THREADSAFEALLOCATOR_H
 
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/STLFunctionalExtras.h"
 #include "llvm/Support/Allocator.h"
 #include <atomic>
 
diff --git a/llvm/include/llvm/Support/VersionTuple.h b/llvm/include/llvm/Support/VersionTuple.h
index 4d58424657..e56efeae60 100644
--- a/llvm/include/llvm/Support/VersionTuple.h
+++ b/llvm/include/llvm/Support/VersionTuple.h
@@ -22,7 +22,7 @@
 #include <tuple>
 
 namespace llvm {
-template <typename HasherT, support::endianness Endianness> class HashBuilder;
+template <typename HasherT, llvm::endianness Endianness> class HashBuilder;
 class raw_ostream;
 class StringRef;
 
diff --git a/llvm/include/llvm/Support/X86DisassemblerDecoderCommon.h b/llvm/include/llvm/Support/X86DisassemblerDecoderCommon.h
index 169b8e9798..6e08fc6a0c 100644
--- a/llvm/include/llvm/Support/X86DisassemblerDecoderCommon.h
+++ b/llvm/include/llvm/Support/X86DisassemblerDecoderCommon.h
@@ -33,6 +33,7 @@ namespace X86Disassembler {
 #define THREEDNOW_MAP_SYM x86Disassembler3DNowOpcodes
 #define MAP5_SYM          x86DisassemblerMap5Opcodes
 #define MAP6_SYM          x86DisassemblerMap6Opcodes
+#define MAP7_SYM          x86DisassemblerMap7Opcodes
 
 #define INSTRUCTIONS_STR  "x86DisassemblerInstrSpecifiers"
 #define CONTEXTS_STR      "x86DisassemblerContexts"
@@ -46,6 +47,7 @@ namespace X86Disassembler {
 #define THREEDNOW_MAP_STR "x86Disassembler3DNowOpcodes"
 #define MAP5_STR          "x86DisassemblerMap5Opcodes"
 #define MAP6_STR          "x86DisassemblerMap6Opcodes"
+#define MAP7_STR          "x86DisassemblerMap7Opcodes"
 
 // Attributes of an instruction that must be known before the opcode can be
 // processed correctly.  Most of these indicate the presence of particular
@@ -296,7 +298,8 @@ enum OpcodeType {
   XOPA_MAP      = 6,
   THREEDNOW_MAP = 7,
   MAP5          = 8,
-  MAP6          = 9
+  MAP6          = 9,
+  MAP7          = 10
 };
 
 // The following structs are used for the hierarchical decode table.  After
diff --git a/llvm/include/llvm/Support/YAMLTraits.h b/llvm/include/llvm/Support/YAMLTraits.h
index b351ec629c..45bb303409 100644
--- a/llvm/include/llvm/Support/YAMLTraits.h
+++ b/llvm/include/llvm/Support/YAMLTraits.h
@@ -1277,7 +1277,7 @@ struct ScalarTraits<double> {
 // For endian types, we use existing scalar Traits class for the underlying
 // type.  This way endian aware types are supported whenever the traits are
 // defined for the underlying type.
-template <typename value_type, support::endianness endian, size_t alignment>
+template <typename value_type, llvm::endianness endian, size_t alignment>
 struct ScalarTraits<support::detail::packed_endian_specific_integral<
                         value_type, endian, alignment>,
                     std::enable_if_t<has_ScalarTraits<value_type>::value>> {
@@ -1301,7 +1301,7 @@ struct ScalarTraits<support::detail::packed_endian_specific_integral<
   }
 };
 
-template <typename value_type, support::endianness endian, size_t alignment>
+template <typename value_type, llvm::endianness endian, size_t alignment>
 struct ScalarEnumerationTraits<
     support::detail::packed_endian_specific_integral<value_type, endian,
                                                      alignment>,
@@ -1317,7 +1317,7 @@ struct ScalarEnumerationTraits<
   }
 };
 
-template <typename value_type, support::endianness endian, size_t alignment>
+template <typename value_type, llvm::endianness endian, size_t alignment>
 struct ScalarBitSetTraits<
     support::detail::packed_endian_specific_integral<value_type, endian,
                                                      alignment>,
diff --git a/llvm/include/llvm/TableGen/Error.h b/llvm/include/llvm/TableGen/Error.h
index 2e639224c9..04618995e0 100644
--- a/llvm/include/llvm/TableGen/Error.h
+++ b/llvm/include/llvm/TableGen/Error.h
@@ -43,6 +43,7 @@ void PrintError(const RecordVal *RecVal, const Twine &Msg);
 [[noreturn]] void PrintFatalError(const RecordVal *RecVal, const Twine &Msg);
 
 void CheckAssert(SMLoc Loc, Init *Condition, Init *Message);
+void dumpMessage(SMLoc Loc, Init *Message);
 
 extern SourceMgr SrcMgr;
 extern unsigned ErrorsPrinted;
diff --git a/llvm/include/llvm/TableGen/Record.h b/llvm/include/llvm/TableGen/Record.h
index 7d743d24f4..d1023a73d6 100644
--- a/llvm/include/llvm/TableGen/Record.h
+++ b/llvm/include/llvm/TableGen/Record.h
@@ -845,7 +845,8 @@ class UnOpInit : public OpInit, public FoldingSetNode {
     SIZE,
     EMPTY,
     GETDAGOP,
-    LOG2
+    LOG2,
+    REPR
   };
 
 private:
@@ -1640,6 +1641,15 @@ class Record {
         : Loc(Loc), Condition(Condition), Message(Message) {}
   };
 
+  struct DumpInfo {
+    SMLoc Loc;
+    Init *Message;
+
+    // User-defined constructor to support std::make_unique(). It can be
+    // removed in C++20 when braced initialization is supported.
+    DumpInfo(SMLoc Loc, Init *Message) : Loc(Loc), Message(Message) {}
+  };
+
 private:
   Init *Name;
   // Location where record was instantiated, followed by the location of
@@ -1651,6 +1661,7 @@ class Record {
   SmallVector<Init *, 0> TemplateArgs;
   SmallVector<RecordVal, 0> Values;
   SmallVector<AssertionInfo, 0> Assertions;
+  SmallVector<DumpInfo, 0> Dumps;
 
   // All superclasses in the inheritance forest in post-order (yes, it
   // must be a forest; diamond-shaped inheritance is not allowed).
@@ -1741,6 +1752,7 @@ class Record {
   ArrayRef<RecordVal> getValues() const { return Values; }
 
   ArrayRef<AssertionInfo> getAssertions() const { return Assertions; }
+  ArrayRef<DumpInfo> getDumps() const { return Dumps; }
 
   ArrayRef<std::pair<Record *, SMRange>>  getSuperClasses() const {
     return SuperClasses;
@@ -1801,11 +1813,18 @@ class Record {
     Assertions.push_back(AssertionInfo(Loc, Condition, Message));
   }
 
+  void addDump(SMLoc Loc, Init *Message) {
+    Dumps.push_back(DumpInfo(Loc, Message));
+  }
+
   void appendAssertions(const Record *Rec) {
     Assertions.append(Rec->Assertions);
   }
 
+  void appendDumps(const Record *Rec) { Dumps.append(Rec->Dumps); }
+
   void checkRecordAssertions();
+  void emitRecordDumps();
   void checkUnusedTemplateArgs();
 
   bool isSubClassOf(const Record *R) const {
diff --git a/llvm/include/llvm/TargetParser/AArch64TargetParser.h b/llvm/include/llvm/TargetParser/AArch64TargetParser.h
index 616f2d7902..bf14473f13 100644
--- a/llvm/include/llvm/TargetParser/AArch64TargetParser.h
+++ b/llvm/include/llvm/TargetParser/AArch64TargetParser.h
@@ -159,7 +159,8 @@ enum ArchExtKind : unsigned {
   AEK_RASv2 =         55, // FEAT_RASv2
   AEK_ITE =           56, // FEAT_ITE
   AEK_GCS =           57, // FEAT_GCS
-  AEK_NUM_EXTENSIONS =  AEK_GCS + 1
+  AEK_FPMR =          58, // FEAT_FPMR
+  AEK_NUM_EXTENSIONS
 };
 using ExtensionBitset = Bitset<AEK_NUM_EXTENSIONS>;
 // clang-format on
@@ -267,6 +268,7 @@ inline constexpr ExtensionInfo Extensions[] = {
     {"tme", AArch64::AEK_TME, "+tme", "-tme", FEAT_INIT, "", 0},
     {"wfxt", AArch64::AEK_NONE, {}, {}, FEAT_WFXT, "+wfxt", 550},
     {"gcs", AArch64::AEK_GCS, "+gcs", "-gcs", FEAT_INIT, "", 0},
+    {"fpmr", AArch64::AEK_FPMR, "+fpmr", "-fpmr", FEAT_INIT, "", 0},
     // Special cases
     {"none", AArch64::AEK_NONE, {}, {}, FEAT_INIT, "", ExtensionInfo::MaxFMVPriority},
 };
diff --git a/llvm/include/llvm/TargetParser/ARMTargetParser.h b/llvm/include/llvm/TargetParser/ARMTargetParser.h
index b893eab190..c42d66f048 100644
--- a/llvm/include/llvm/TargetParser/ARMTargetParser.h
+++ b/llvm/include/llvm/TargetParser/ARMTargetParser.h
@@ -143,6 +143,14 @@ enum class FPURestriction {
   SP_D16    ///< Only single-precision instructions, with 16 D registers
 };
 
+inline bool isDoublePrecision(const FPURestriction restriction) {
+  return restriction != FPURestriction::SP_D16;
+}
+
+inline bool has32Regs(const FPURestriction restriction) {
+  return restriction == FPURestriction::None;
+}
+
 // An FPU name implies one of three levels of Neon support:
 enum class NeonSupportLevel {
   None = 0, ///< No Neon
diff --git a/llvm/include/llvm/TargetParser/X86TargetParser.def b/llvm/include/llvm/TargetParser/X86TargetParser.def
index 85ff6996d3..4204811a6a 100644
--- a/llvm/include/llvm/TargetParser/X86TargetParser.def
+++ b/llvm/include/llvm/TargetParser/X86TargetParser.def
@@ -48,6 +48,7 @@ X86_CPU_TYPE(AMDFAM19H,           "amdfam19h")
 X86_CPU_TYPE(ZHAOXIN_FAM7H,       "zhaoxin_fam7h")
 X86_CPU_TYPE(INTEL_SIERRAFOREST,  "sierraforest")
 X86_CPU_TYPE(INTEL_GRANDRIDGE,    "grandridge")
+X86_CPU_TYPE(INTEL_CLEARWATERFOREST, "clearwaterforest")
 
 // Alternate names supported by __builtin_cpu_is and target multiversioning.
 X86_CPU_TYPE_ALIAS(INTEL_BONNELL,    "atom")
@@ -102,6 +103,7 @@ X86_CPU_SUBTYPE(INTEL_COREI7_GRANITERAPIDS,  "graniterapids")
 X86_CPU_SUBTYPE(INTEL_COREI7_GRANITERAPIDS_D,"graniterapids-d")
 X86_CPU_SUBTYPE(INTEL_COREI7_ARROWLAKE,      "arrowlake")
 X86_CPU_SUBTYPE(INTEL_COREI7_ARROWLAKE_S,    "arrowlake-s")
+X86_CPU_SUBTYPE(INTEL_COREI7_PANTHERLAKE,    "pantherlake")
 
 // Alternate names supported by __builtin_cpu_is and target multiversioning.
 X86_CPU_SUBTYPE_ALIAS(INTEL_COREI7_ALDERLAKE, "raptorlake")
@@ -241,6 +243,9 @@ X86_FEATURE       (SM3,             "sm3")
 X86_FEATURE       (SM4,             "sm4")
 X86_FEATURE       (AVXVNNIINT16,    "avxvnniint16")
 X86_FEATURE       (EVEX512,         "evex512")
+X86_FEATURE       (AVX10_1,         "avx10.1-256")
+X86_FEATURE       (AVX10_1_512,     "avx10.1-512")
+X86_FEATURE       (USERMSR,         "usermsr")
 // These features aren't really CPU features, but the frontend can set them.
 X86_FEATURE       (RETPOLINE_EXTERNAL_THUNK,    "retpoline-external-thunk")
 X86_FEATURE       (RETPOLINE_INDIRECT_BRANCHES, "retpoline-indirect-branches")
diff --git a/llvm/include/llvm/TargetParser/X86TargetParser.h b/llvm/include/llvm/TargetParser/X86TargetParser.h
index d546564720..2083e585af 100644
--- a/llvm/include/llvm/TargetParser/X86TargetParser.h
+++ b/llvm/include/llvm/TargetParser/X86TargetParser.h
@@ -114,11 +114,13 @@ enum CPUKind {
   CK_Arrowlake,
   CK_ArrowlakeS,
   CK_Lunarlake,
+  CK_Pantherlake,
   CK_Sierraforest,
   CK_Grandridge,
   CK_Graniterapids,
   CK_GraniterapidsD,
   CK_Emeraldrapids,
+  CK_Clearwaterforest,
   CK_KNL,
   CK_KNM,
   CK_Lakemont,
diff --git a/llvm/include/llvm/TextAPI/InterfaceFile.h b/llvm/include/llvm/TextAPI/InterfaceFile.h
index 88139d8506..ae2bf78403 100644
--- a/llvm/include/llvm/TextAPI/InterfaceFile.h
+++ b/llvm/include/llvm/TextAPI/InterfaceFile.h
@@ -352,9 +352,8 @@ class InterfaceFile {
   }
 
   /// Add a symbol to the symbols list or extend an existing one.
-  template <typename RangeT,
-            typename ElT = typename std::remove_reference<
-                decltype(*std::begin(std::declval<RangeT>()))>::type>
+  template <typename RangeT, typename ElT = std::remove_reference_t<
+                                 decltype(*std::begin(std::declval<RangeT>()))>>
   void addSymbol(SymbolKind Kind, StringRef Name, RangeT &&Targets,
                  SymbolFlags Flags = SymbolFlags::None) {
     SymbolsSet->addGlobal(Kind, Name, Flags, Targets);
diff --git a/llvm/include/llvm/TextAPI/SymbolSet.h b/llvm/include/llvm/TextAPI/SymbolSet.h
index 238385178d..5b67ebea11 100644
--- a/llvm/include/llvm/TextAPI/SymbolSet.h
+++ b/llvm/include/llvm/TextAPI/SymbolSet.h
@@ -95,9 +95,8 @@ class SymbolSet {
                     const Target &Targ);
   size_t size() const { return Symbols.size(); }
 
-  template <typename RangeT,
-            typename ElT = typename std::remove_reference<
-                decltype(*std::begin(std::declval<RangeT>()))>::type>
+  template <typename RangeT, typename ElT = std::remove_reference_t<
+                                 decltype(*std::begin(std::declval<RangeT>()))>>
   Symbol *addGlobal(SymbolKind Kind, StringRef Name, SymbolFlags Flags,
                     RangeT &&Targets) {
     auto *Global = addGlobalImpl(Kind, Name, Flags);
diff --git a/llvm/include/llvm/Transforms/HipStdPar/HipStdPar.h b/llvm/include/llvm/Transforms/HipStdPar/HipStdPar.h
new file mode 100644
index 0000000000..9df093d8d5
--- /dev/null
+++ b/llvm/include/llvm/Transforms/HipStdPar/HipStdPar.h
@@ -0,0 +1,46 @@
+//===--------- HipStdPar.h - Standard Parallelism passes --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+///
+/// AcceleratorCodeSelection - Identify all functions reachable from a kernel,
+/// removing those that are unreachable.
+///
+/// AllocationInterposition - Forward calls to allocation / deallocation
+//  functions to runtime provided equivalents that allocate memory that is
+//  accessible for an accelerator
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_HIPSTDPAR_HIPSTDPAR_H
+#define LLVM_TRANSFORMS_HIPSTDPAR_HIPSTDPAR_H
+
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+
+class Module;
+class ModuleAnaysisManager;
+
+class HipStdParAcceleratorCodeSelectionPass
+  : public PassInfoMixin<HipStdParAcceleratorCodeSelectionPass> {
+public:
+  PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
+
+  static bool isRequired() { return true; }
+};
+
+class HipStdParAllocationInterpositionPass
+  : public PassInfoMixin<HipStdParAllocationInterpositionPass> {
+public:
+  PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
+
+  static bool isRequired() { return true; }
+};
+
+} // namespace llvm
+
+#endif // LLVM_TRANSFORMS_HIPSTDPAR_HIPSTDPAR_H
diff --git a/llvm/include/llvm/Transforms/IPO/FunctionSpecialization.h b/llvm/include/llvm/Transforms/IPO/FunctionSpecialization.h
index 59378bc108..50f9aae73d 100644
--- a/llvm/include/llvm/Transforms/IPO/FunctionSpecialization.h
+++ b/llvm/include/llvm/Transforms/IPO/FunctionSpecialization.h
@@ -84,8 +84,6 @@
 #include "llvm/Transforms/Utils/SCCPSolver.h"
 #include "llvm/Transforms/Utils/SizeOpts.h"
 
-using namespace llvm;
-
 namespace llvm {
 // Map of potential specializations for each function. The FunctionSpecializer
 // keeps the discovered specialisation opportunities for the module in a single
diff --git a/llvm/include/llvm/Transforms/IPO/ProfiledCallGraph.h b/llvm/include/llvm/Transforms/IPO/ProfiledCallGraph.h
index bc8360a80b..9436e7b41a 100644
--- a/llvm/include/llvm/Transforms/IPO/ProfiledCallGraph.h
+++ b/llvm/include/llvm/Transforms/IPO/ProfiledCallGraph.h
@@ -52,10 +52,11 @@ struct ProfiledCallGraphNode {
   using edges = std::set<edge, ProfiledCallGraphEdgeComparer>;
   using iterator = edges::iterator;
   using const_iterator = edges::const_iterator;
-
-  ProfiledCallGraphNode(StringRef FName = StringRef()) : Name(FName) {}
-
-  StringRef Name;
+  
+  ProfiledCallGraphNode(FunctionId FName = FunctionId()) : Name(FName)
+  {}
+  
+  FunctionId Name;
   edges Edges;
 };
 
@@ -85,7 +86,7 @@ class ProfiledCallGraph {
     std::queue<ContextTrieNode *> Queue;
     for (auto &Child : ContextTracker.getRootContext().getAllChildContext()) {
       ContextTrieNode *Callee = &Child.second;
-      addProfiledFunction(ContextTracker.getFuncNameFor(Callee));
+      addProfiledFunction(Callee->getFuncName());
       Queue.push(Callee);
     }
 
@@ -102,7 +103,7 @@ class ProfiledCallGraph {
       // context-based one, which may in turn block context-based inlining.
       for (auto &Child : Caller->getAllChildContext()) {
         ContextTrieNode *Callee = &Child.second;
-        addProfiledFunction(ContextTracker.getFuncNameFor(Callee));
+        addProfiledFunction(Callee->getFuncName());
         Queue.push(Callee);
 
         // Fetch edge weight from the profile.
@@ -116,15 +117,14 @@ class ProfiledCallGraph {
           LineLocation Callsite = Callee->getCallSiteLoc();
           if (auto CallTargets = CallerSamples->findCallTargetMapAt(Callsite)) {
             SampleRecord::CallTargetMap &TargetCounts = CallTargets.get();
-            auto It = TargetCounts.find(CalleeSamples->getName());
+            auto It = TargetCounts.find(CalleeSamples->getFunction());
             if (It != TargetCounts.end())
               CallsiteCount = It->second;
           }
           Weight = std::max(CallsiteCount, CalleeEntryCount);
         }
 
-        addProfiledCall(ContextTracker.getFuncNameFor(Caller),
-                        ContextTracker.getFuncNameFor(Callee), Weight);
+        addProfiledCall(Caller->getFuncName(), Callee->getFuncName(), Weight);
       }
     }
 
@@ -136,8 +136,8 @@ class ProfiledCallGraph {
   iterator begin() { return Root.Edges.begin(); }
   iterator end() { return Root.Edges.end(); }
   ProfiledCallGraphNode *getEntryNode() { return &Root; }
-
-  void addProfiledFunction(StringRef Name) {
+  
+  void addProfiledFunction(FunctionId Name) {
     if (!ProfiledFunctions.count(Name)) {
       // Link to synthetic root to make sure every node is reachable
       // from root. This does not affect SCC order.
@@ -147,7 +147,7 @@ class ProfiledCallGraph {
   }
 
 private:
-  void addProfiledCall(StringRef CallerName, StringRef CalleeName,
+  void addProfiledCall(FunctionId CallerName, FunctionId CalleeName,
                        uint64_t Weight = 0) {
     assert(ProfiledFunctions.count(CallerName));
     auto CalleeIt = ProfiledFunctions.find(CalleeName);
@@ -168,19 +168,19 @@ class ProfiledCallGraph {
   }
 
   void addProfiledCalls(const FunctionSamples &Samples) {
-    addProfiledFunction(Samples.getFuncName());
+    addProfiledFunction(Samples.getFunction());
 
     for (const auto &Sample : Samples.getBodySamples()) {
       for (const auto &[Target, Frequency] : Sample.second.getCallTargets()) {
         addProfiledFunction(Target);
-        addProfiledCall(Samples.getFuncName(), Target, Frequency);
+        addProfiledCall(Samples.getFunction(), Target, Frequency);
       }
     }
 
     for (const auto &CallsiteSamples : Samples.getCallsiteSamples()) {
       for (const auto &InlinedSamples : CallsiteSamples.second) {
         addProfiledFunction(InlinedSamples.first);
-        addProfiledCall(Samples.getFuncName(), InlinedSamples.first,
+        addProfiledCall(Samples.getFunction(), InlinedSamples.first,
                         InlinedSamples.second.getHeadSamplesEstimate());
         addProfiledCalls(InlinedSamples.second);
       }
@@ -206,7 +206,8 @@ class ProfiledCallGraph {
   }
 
   ProfiledCallGraphNode Root;
-  StringMap<ProfiledCallGraphNode> ProfiledFunctions;
+  HashKeyMap<std::unordered_map, FunctionId, ProfiledCallGraphNode>
+      ProfiledFunctions;
 };
 
 } // end namespace sampleprof
diff --git a/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h b/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h
index 347dac1e96..f4c999ab93 100644
--- a/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h
+++ b/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h
@@ -35,20 +35,20 @@ class Instruction;
 class ContextTrieNode {
 public:
   ContextTrieNode(ContextTrieNode *Parent = nullptr,
-                  StringRef FName = StringRef(),
+                  FunctionId FName = FunctionId(),
                   FunctionSamples *FSamples = nullptr,
                   LineLocation CallLoc = {0, 0})
       : ParentContext(Parent), FuncName(FName), FuncSamples(FSamples),
         CallSiteLoc(CallLoc){};
   ContextTrieNode *getChildContext(const LineLocation &CallSite,
-                                   StringRef ChildName);
+                                   FunctionId ChildName);
   ContextTrieNode *getHottestChildContext(const LineLocation &CallSite);
   ContextTrieNode *getOrCreateChildContext(const LineLocation &CallSite,
-                                           StringRef ChildName,
+                                           FunctionId ChildName,
                                            bool AllowCreate = true);
-  void removeChildContext(const LineLocation &CallSite, StringRef ChildName);
+  void removeChildContext(const LineLocation &CallSite, FunctionId ChildName);
   std::map<uint64_t, ContextTrieNode> &getAllChildContext();
-  StringRef getFuncName() const;
+  FunctionId getFuncName() const;
   FunctionSamples *getFunctionSamples() const;
   void setFunctionSamples(FunctionSamples *FSamples);
   std::optional<uint32_t> getFunctionSize() const;
@@ -68,7 +68,7 @@ class ContextTrieNode {
   ContextTrieNode *ParentContext;
 
   // Function name for current context
-  StringRef FuncName;
+  FunctionId FuncName;
 
   // Function Samples for current context
   FunctionSamples *FuncSamples;
@@ -118,7 +118,8 @@ class SampleContextTracker {
   FunctionSamples *getBaseSamplesFor(const Function &Func,
                                      bool MergeContext = true);
   // Query base profile for a given function by name.
-  FunctionSamples *getBaseSamplesFor(StringRef Name, bool MergeContext = true);
+  FunctionSamples *getBaseSamplesFor(FunctionId Name,
+                                     bool MergeContext = true);
   // Retrieve the context trie node for given profile context
   ContextTrieNode *getContextFor(const SampleContext &Context);
   // Get real function name for a given trie node.
@@ -129,7 +130,7 @@ class SampleContextTracker {
   void markContextSamplesInlined(const FunctionSamples *InlinedSamples);
   ContextTrieNode &getRootContext();
   void promoteMergeContextSamplesTree(const Instruction &Inst,
-                                      StringRef CalleeName);
+                                      FunctionId CalleeName);
 
   // Create a merged conext-less profile map.
   void createContextLessProfileMap(SampleProfileMap &ContextLessProfiles);
@@ -140,7 +141,8 @@ class SampleContextTracker {
       return nullptr;
     return I->second;
   }
-  StringMap<ContextSamplesTy> &getFuncToCtxtProfiles() {
+  HashKeyMap<std::unordered_map, FunctionId, ContextSamplesTy>
+      &getFuncToCtxtProfiles() {
     return FuncToCtxtProfiles;
   }
 
@@ -189,9 +191,9 @@ class SampleContextTracker {
 private:
   ContextTrieNode *getContextFor(const DILocation *DIL);
   ContextTrieNode *getCalleeContextFor(const DILocation *DIL,
-                                       StringRef CalleeName);
-  ContextTrieNode *getTopLevelContextNode(StringRef FName);
-  ContextTrieNode &addTopLevelContextNode(StringRef FName);
+                                       FunctionId CalleeName);
+  ContextTrieNode *getTopLevelContextNode(FunctionId FName);
+  ContextTrieNode &addTopLevelContextNode(FunctionId FName);
   ContextTrieNode &promoteMergeContextSamplesTree(ContextTrieNode &NodeToPromo);
   void mergeContextNode(ContextTrieNode &FromNode, ContextTrieNode &ToNode);
   ContextTrieNode &
@@ -204,7 +206,8 @@ class SampleContextTracker {
     ProfileToNodeMap[FSample] = Node;
   }
   // Map from function name to context profiles (excluding base profile)
-  StringMap<ContextSamplesTy> FuncToCtxtProfiles;
+  HashKeyMap<std::unordered_map, FunctionId, ContextSamplesTy>
+      FuncToCtxtProfiles;
 
   // Map from current FunctionSample to the belonged context trie.
   std::unordered_map<const FunctionSamples *, ContextTrieNode *>
diff --git a/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h b/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h
index dcfcc8f41d..f8b3874267 100644
--- a/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h
+++ b/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h
@@ -510,15 +510,18 @@ class LLVM_LIBRARY_VISIBILITY InstCombiner {
                                              SQ.getWithInstruction(CxtI));
   }
 
-  OverflowResult computeOverflowForUnsignedAdd(const Value *LHS,
-                                               const Value *RHS,
-                                               const Instruction *CxtI) const {
+  OverflowResult
+  computeOverflowForUnsignedAdd(const WithCache<const Value *> &LHS,
+                                const WithCache<const Value *> &RHS,
+                                const Instruction *CxtI) const {
     return llvm::computeOverflowForUnsignedAdd(LHS, RHS,
                                                SQ.getWithInstruction(CxtI));
   }
 
-  OverflowResult computeOverflowForSignedAdd(const Value *LHS, const Value *RHS,
-                                             const Instruction *CxtI) const {
+  OverflowResult
+  computeOverflowForSignedAdd(const WithCache<const Value *> &LHS,
+                              const WithCache<const Value *> &RHS,
+                              const Instruction *CxtI) const {
     return llvm::computeOverflowForSignedAdd(LHS, RHS,
                                              SQ.getWithInstruction(CxtI));
   }
diff --git a/llvm/include/llvm/Transforms/Instrumentation/CFGMST.h b/llvm/include/llvm/Transforms/Instrumentation/CFGMST.h
index 269441db7a..6ed8a6c6ea 100644
--- a/llvm/include/llvm/Transforms/Instrumentation/CFGMST.h
+++ b/llvm/include/llvm/Transforms/Instrumentation/CFGMST.h
@@ -28,8 +28,6 @@
 
 #define DEBUG_TYPE "cfgmst"
 
-using namespace llvm;
-
 namespace llvm {
 
 /// An union-find based Minimum Spanning Tree for CFG
diff --git a/llvm/include/llvm/Transforms/Scalar.h b/llvm/include/llvm/Transforms/Scalar.h
index 2af6ca8e47..0676f485f3 100644
--- a/llvm/include/llvm/Transforms/Scalar.h
+++ b/llvm/include/llvm/Transforms/Scalar.h
@@ -217,12 +217,6 @@ Pass *createMergeICmpsLegacyPass();
 FunctionPass *createInferAddressSpacesPass(unsigned AddressSpace = ~0u);
 extern char &InferAddressSpacesID;
 
-//===----------------------------------------------------------------------===//
-//
-// LowerExpectIntrinsics - Removes llvm.expect intrinsics and creates
-// "block_weights" metadata.
-FunctionPass *createLowerExpectIntrinsicPass();
-
 //===----------------------------------------------------------------------===//
 //
 // TLSVariableHoist - This pass reduce duplicated TLS address call.
diff --git a/llvm/include/llvm/Transforms/Scalar/GVN.h b/llvm/include/llvm/Transforms/Scalar/GVN.h
index 0a00e3af03..4ba9b74ccb 100644
--- a/llvm/include/llvm/Transforms/Scalar/GVN.h
+++ b/llvm/include/llvm/Transforms/Scalar/GVN.h
@@ -261,7 +261,7 @@ class GVNPass : public PassInfoMixin<GVNPass> {
 
   bool runImpl(Function &F, AssumptionCache &RunAC, DominatorTree &RunDT,
                const TargetLibraryInfo &RunTLI, AAResults &RunAA,
-               MemoryDependenceResults *RunMD, LoopInfo *LI,
+               MemoryDependenceResults *RunMD, LoopInfo &LI,
                OptimizationRemarkEmitter *ORE, MemorySSA *MSSA = nullptr);
 
   /// Push a new Value to the LeaderTable onto the list for its value number.
diff --git a/llvm/include/llvm/Transforms/Utils/CodeExtractor.h b/llvm/include/llvm/Transforms/Utils/CodeExtractor.h
index bb23cf4a9a..27b34ef023 100644
--- a/llvm/include/llvm/Transforms/Utils/CodeExtractor.h
+++ b/llvm/include/llvm/Transforms/Utils/CodeExtractor.h
@@ -114,6 +114,10 @@ class CodeExtractorAnalysisCache {
     // label, if non-empty, otherwise "extracted".
     std::string Suffix;
 
+    // If true, the outlined function has aggregate argument in zero address
+    // space.
+    bool ArgsInZeroAddressSpace;
+
   public:
     /// Create a code extractor for a sequence of blocks.
     ///
@@ -128,13 +132,16 @@ class CodeExtractorAnalysisCache {
     /// Any new allocations will be placed in the AllocationBlock, unless
     /// it is null, in which case it will be placed in the entry block of
     /// the function from which the code is being extracted.
+    /// If ArgsInZeroAddressSpace param is set to true, then the aggregate
+    /// param pointer of the outlined function is declared in zero address
+    /// space.
     CodeExtractor(ArrayRef<BasicBlock *> BBs, DominatorTree *DT = nullptr,
                   bool AggregateArgs = false, BlockFrequencyInfo *BFI = nullptr,
                   BranchProbabilityInfo *BPI = nullptr,
                   AssumptionCache *AC = nullptr, bool AllowVarArgs = false,
                   bool AllowAlloca = false,
                   BasicBlock *AllocationBlock = nullptr,
-                  std::string Suffix = "");
+                  std::string Suffix = "", bool ArgsInZeroAddressSpace = false);
 
     /// Create a code extractor for a loop body.
     ///
diff --git a/llvm/include/llvm/Transforms/Utils/CodeLayout.h b/llvm/include/llvm/Transforms/Utils/CodeLayout.h
index f5127cff24..9d550fae6d 100644
--- a/llvm/include/llvm/Transforms/Utils/CodeLayout.h
+++ b/llvm/include/llvm/Transforms/Utils/CodeLayout.h
@@ -65,6 +65,8 @@ struct CDSortConfig {
   unsigned CacheEntries = 16;
   /// The size of a line in the cache.
   unsigned CacheSize = 2048;
+  /// The maximum size of a chain to create.
+  unsigned MaxChainSize = 128;
   /// The power exponent for the distance-based locality.
   double DistancePower = 0.25;
   /// The scale factor for the frequency-based locality.
diff --git a/llvm/include/llvm/Transforms/Utils/SampleProfileInference.h b/llvm/include/llvm/Transforms/Utils/SampleProfileInference.h
index b69468ee7e..b4ea1ad840 100644
--- a/llvm/include/llvm/Transforms/Utils/SampleProfileInference.h
+++ b/llvm/include/llvm/Transforms/Utils/SampleProfileInference.h
@@ -119,7 +119,7 @@ void applyFlowInference(FlowFunction &Func);
 template <typename FT> class SampleProfileInference {
 public:
   using NodeRef = typename GraphTraits<FT *>::NodeRef;
-  using BasicBlockT = typename std::remove_pointer<NodeRef>::type;
+  using BasicBlockT = std::remove_pointer_t<NodeRef>;
   using FunctionT = FT;
   using Edge = std::pair<const BasicBlockT *, const BasicBlockT *>;
   using BlockWeightMap = DenseMap<const BasicBlockT *, uint64_t>;
diff --git a/llvm/include/llvm/Transforms/Utils/SampleProfileLoaderBaseImpl.h b/llvm/include/llvm/Transforms/Utils/SampleProfileLoaderBaseImpl.h
index 21e85eede7..66814d3952 100644
--- a/llvm/include/llvm/Transforms/Utils/SampleProfileLoaderBaseImpl.h
+++ b/llvm/include/llvm/Transforms/Utils/SampleProfileLoaderBaseImpl.h
@@ -152,7 +152,7 @@ template <typename FT> class SampleProfileLoaderBaseImpl {
   void dump() { Reader->dump(); }
 
   using NodeRef = typename GraphTraits<FT *>::NodeRef;
-  using BT = typename std::remove_pointer<NodeRef>::type;
+  using BT = std::remove_pointer_t<NodeRef>;
   using InstructionT = typename afdo_detail::IRTraits<BT>::InstructionT;
   using BasicBlockT = typename afdo_detail::IRTraits<BT>::BasicBlockT;
   using BlockFrequencyInfoT =
diff --git a/llvm/lib/Analysis/BasicAliasAnalysis.cpp b/llvm/lib/Analysis/BasicAliasAnalysis.cpp
index c162b8f6ed..3e59970816 100644
--- a/llvm/lib/Analysis/BasicAliasAnalysis.cpp
+++ b/llvm/lib/Analysis/BasicAliasAnalysis.cpp
@@ -219,7 +219,7 @@ bool EarliestEscapeInfo::isNotCapturedBeforeOrAt(const Value *Object,
     return true;
 
   return I != Iter.first->second &&
-         !isPotentiallyReachable(Iter.first->second, I, nullptr, &DT, &LI);
+         !isPotentiallyReachable(Iter.first->second, I, nullptr, &DT, LI);
 }
 
 void EarliestEscapeInfo::removeInstruction(Instruction *I) {
@@ -662,6 +662,7 @@ BasicAAResult::DecomposeGEPExpression(const Value *V, const DataLayout &DL,
         if (Decomposed.VarIndices[i].Val.V == LE.Val.V &&
             Decomposed.VarIndices[i].Val.hasSameCastsAs(LE.Val)) {
           Scale += Decomposed.VarIndices[i].Scale;
+          LE.IsNSW = false; // We cannot guarantee nsw for the merge.
           Decomposed.VarIndices.erase(Decomposed.VarIndices.begin() + i);
           break;
         }
diff --git a/llvm/lib/Analysis/CaptureTracking.cpp b/llvm/lib/Analysis/CaptureTracking.cpp
index 2827eb705a..2b93620548 100644
--- a/llvm/lib/Analysis/CaptureTracking.cpp
+++ b/llvm/lib/Analysis/CaptureTracking.cpp
@@ -167,7 +167,7 @@ namespace {
   struct EarliestCaptures : public CaptureTracker {
 
     EarliestCaptures(bool ReturnCaptures, Function &F, const DominatorTree &DT,
-                     const SmallPtrSetImpl<const Value *> &EphValues)
+                     const SmallPtrSetImpl<const Value *> *EphValues)
         : EphValues(EphValues), DT(DT), ReturnCaptures(ReturnCaptures), F(F) {}
 
     void tooManyUses() override {
@@ -180,7 +180,7 @@ namespace {
       if (isa<ReturnInst>(I) && !ReturnCaptures)
         return false;
 
-      if (EphValues.contains(I))
+      if (EphValues && EphValues->contains(I))
         return false;
 
       if (!EarliestCapture)
@@ -194,7 +194,7 @@ namespace {
       return false;
     }
 
-    const SmallPtrSetImpl<const Value *> &EphValues;
+    const SmallPtrSetImpl<const Value *> *EphValues;
 
     Instruction *EarliestCapture = nullptr;
 
@@ -286,8 +286,7 @@ bool llvm::PointerMayBeCapturedBefore(const Value *V, bool ReturnCaptures,
 Instruction *
 llvm::FindEarliestCapture(const Value *V, Function &F, bool ReturnCaptures,
                           bool StoreCaptures, const DominatorTree &DT,
-
-                          const SmallPtrSetImpl<const Value *> &EphValues,
+                          const SmallPtrSetImpl<const Value *> *EphValues,
                           unsigned MaxUsesToExplore) {
   assert(!isa<GlobalValue>(V) &&
          "It doesn't make sense to ask whether a global is captured.");
diff --git a/llvm/lib/Analysis/DevelopmentModeInlineAdvisor.cpp b/llvm/lib/Analysis/DevelopmentModeInlineAdvisor.cpp
index 456d586606..7d51302bcc 100644
--- a/llvm/lib/Analysis/DevelopmentModeInlineAdvisor.cpp
+++ b/llvm/lib/Analysis/DevelopmentModeInlineAdvisor.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This file implements a model runner using Tensorflow C APIs, allowing the
+// This file implements a model runner using TFLite, allowing the
 // loading of a model from a command line option.
 //
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Analysis/FunctionPropertiesAnalysis.cpp b/llvm/lib/Analysis/FunctionPropertiesAnalysis.cpp
index 97c54943ab..e27db66710 100644
--- a/llvm/lib/Analysis/FunctionPropertiesAnalysis.cpp
+++ b/llvm/lib/Analysis/FunctionPropertiesAnalysis.cpp
@@ -25,6 +25,7 @@
 
 using namespace llvm;
 
+namespace llvm {
 cl::opt<bool> EnableDetailedFunctionProperties(
     "enable-detailed-function-properties", cl::Hidden, cl::init(false),
     cl::desc("Whether or not to compute detailed function properties."));
@@ -38,8 +39,9 @@ cl::opt<unsigned> MediumBasicBlockInstructionThreshold(
     "medium-basic-block-instruction-threshold", cl::Hidden, cl::init(15),
     cl::desc("The minimum number of instructions a basic block should contain "
              "before being considered medium-sized."));
+}
 
-cl::opt<unsigned> CallWithManyArgumentsThreshold(
+static cl::opt<unsigned> CallWithManyArgumentsThreshold(
     "call-with-many-arguments-threshold", cl::Hidden, cl::init(4),
     cl::desc("The minimum number of arguments a function call must have before "
              "it is considered having many arguments."));
diff --git a/llvm/lib/Analysis/InlineOrder.cpp b/llvm/lib/Analysis/InlineOrder.cpp
index 503880e3e8..e6157580e2 100644
--- a/llvm/lib/Analysis/InlineOrder.cpp
+++ b/llvm/lib/Analysis/InlineOrder.cpp
@@ -218,15 +218,17 @@ class PriorityInlineOrder : public InlineOrder<std::pair<CallBase *, int>> {
   // A call site could become less desirable for inlining because of the size
   // growth from prior inlining into the callee. This method is used to lazily
   // update the desirability of a call site if it's decreasing. It is only
-  // called on pop() or front(), not every time the desirability changes. When
-  // the desirability of the front call site decreases, an updated one would be
-  // pushed right back into the heap. For simplicity, those cases where
-  // the desirability of a call site increases are ignored here.
+  // called on pop(), not every time the desirability changes. When the
+  // desirability of the front call site decreases, an updated one would be
+  // pushed right back into the heap. For simplicity, those cases where the
+  // desirability of a call site increases are ignored here.
   void adjust() {
-    while (updateAndCheckDecreased(Heap.front())) {
-      std::pop_heap(Heap.begin(), Heap.end(), isLess);
+    std::pop_heap(Heap.begin(), Heap.end(), isLess);
+    while (updateAndCheckDecreased(Heap.back())) {
       std::push_heap(Heap.begin(), Heap.end(), isLess);
+      std::pop_heap(Heap.begin(), Heap.end(), isLess);
     }
+    std::push_heap(Heap.begin(), Heap.end(), isLess);
   }
 
 public:
@@ -253,11 +255,10 @@ class PriorityInlineOrder : public InlineOrder<std::pair<CallBase *, int>> {
     assert(size() > 0);
     adjust();
 
-    CallBase *CB = Heap.front();
+    std::pop_heap(Heap.begin(), Heap.end(), isLess);
+    CallBase *CB = Heap.pop_back_val();
     T Result = std::make_pair(CB, InlineHistoryMap[CB]);
     InlineHistoryMap.erase(CB);
-    std::pop_heap(Heap.begin(), Heap.end(), isLess);
-    Heap.pop_back();
     return Result;
   }
 
@@ -317,4 +318,4 @@ llvm::getInlineOrder(FunctionAnalysisManager &FAM, const InlineParams &Params,
                                                                M);
   }
   return getDefaultInlineOrder(FAM, Params, MAM, M);
-}
\ No newline at end of file
+}
diff --git a/llvm/lib/Analysis/LazyValueInfo.cpp b/llvm/lib/Analysis/LazyValueInfo.cpp
index 789a02ed32..5cb207c803 100644
--- a/llvm/lib/Analysis/LazyValueInfo.cpp
+++ b/llvm/lib/Analysis/LazyValueInfo.cpp
@@ -26,6 +26,7 @@
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/Dominators.h"
+#include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Intrinsics.h"
@@ -1083,6 +1084,26 @@ static ValueLatticeElement getValueFromSimpleICmpCondition(
   return ValueLatticeElement::getRange(TrueValues.subtract(Offset));
 }
 
+static std::optional<ConstantRange>
+getRangeViaSLT(CmpInst::Predicate Pred, APInt RHS,
+               function_ref<std::optional<ConstantRange>(const APInt &)> Fn) {
+  bool Invert = false;
+  if (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SGE) {
+    Pred = ICmpInst::getInversePredicate(Pred);
+    Invert = true;
+  }
+  if (Pred == ICmpInst::ICMP_SLE) {
+    Pred = ICmpInst::ICMP_SLT;
+    if (RHS.isMaxSignedValue())
+      return std::nullopt; // Could also return full/empty here, if we wanted.
+    ++RHS;
+  }
+  assert(Pred == ICmpInst::ICMP_SLT && "Must be signed predicate");
+  if (auto CR = Fn(RHS))
+    return Invert ? CR->inverse() : CR;
+  return std::nullopt;
+}
+
 static ValueLatticeElement getValueFromICmpCondition(Value *Val, ICmpInst *ICI,
                                                      bool isTrueDest) {
   Value *LHS = ICI->getOperand(0);
@@ -1148,6 +1169,25 @@ static ValueLatticeElement getValueFromICmpCondition(Value *Val, ICmpInst *ICI,
           CR.getUnsignedMin().zext(BitWidth), APInt(BitWidth, 0)));
   }
 
+  // Recognize:
+  // icmp slt (ashr X, ShAmtC), C --> icmp slt X, C << ShAmtC
+  // Preconditions: (C << ShAmtC) >> ShAmtC == C
+  const APInt *ShAmtC;
+  if (CmpInst::isSigned(EdgePred) &&
+      match(LHS, m_AShr(m_Specific(Val), m_APInt(ShAmtC))) &&
+      match(RHS, m_APInt(C))) {
+    auto CR = getRangeViaSLT(
+        EdgePred, *C, [&](const APInt &RHS) -> std::optional<ConstantRange> {
+          APInt New = RHS << *ShAmtC;
+          if ((New.ashr(*ShAmtC)) != RHS)
+            return std::nullopt;
+          return ConstantRange::getNonEmpty(
+              APInt::getSignedMinValue(New.getBitWidth()), New);
+        });
+    if (CR)
+      return ValueLatticeElement::getRange(*CR);
+  }
+
   return ValueLatticeElement::getOverdefined();
 }
 
diff --git a/llvm/lib/Analysis/Local.cpp b/llvm/lib/Analysis/Local.cpp
index 5d558de516..672e6a190a 100644
--- a/llvm/lib/Analysis/Local.cpp
+++ b/llvm/lib/Analysis/Local.cpp
@@ -28,6 +28,13 @@ Value *llvm::emitGEPOffset(IRBuilderBase *Builder, const DataLayout &DL,
   // If the GEP is inbounds, we know that none of the addressing operations will
   // overflow in a signed sense.
   bool isInBounds = GEPOp->isInBounds() && !NoAssumptions;
+  auto AddOffset = [&](Value *Offset) {
+    if (Result)
+      Result = Builder->CreateAdd(Result, Offset, GEP->getName() + ".offs",
+                                  false /*NUW*/, isInBounds /*NSW*/);
+    else
+      Result = Offset;
+  };
 
   // Build a mask for high order bits.
   unsigned IntPtrWidth = IntIdxTy->getScalarType()->getIntegerBitWidth();
@@ -39,7 +46,6 @@ Value *llvm::emitGEPOffset(IRBuilderBase *Builder, const DataLayout &DL,
        ++i, ++GTI) {
     Value *Op = *i;
     uint64_t Size = DL.getTypeAllocSize(GTI.getIndexedType()) & PtrSizeMask;
-    Value *Offset;
     if (Constant *OpC = dyn_cast<Constant>(Op)) {
       if (OpC->isZeroValue())
         continue;
@@ -51,42 +57,26 @@ Value *llvm::emitGEPOffset(IRBuilderBase *Builder, const DataLayout &DL,
         if (!Size)
           continue;
 
-        Offset = ConstantInt::get(IntIdxTy, Size);
-      } else {
-        // Splat the constant if needed.
-        if (IntIdxTy->isVectorTy() && !OpC->getType()->isVectorTy())
-          OpC = ConstantVector::getSplat(
-              cast<VectorType>(IntIdxTy)->getElementCount(), OpC);
-
-        Constant *Scale = ConstantInt::get(IntIdxTy, Size);
-        Constant *OC =
-            ConstantExpr::getIntegerCast(OpC, IntIdxTy, true /*SExt*/);
-        Offset =
-            ConstantExpr::getMul(OC, Scale, false /*NUW*/, isInBounds /*NSW*/);
-      }
-    } else {
-      // Splat the index if needed.
-      if (IntIdxTy->isVectorTy() && !Op->getType()->isVectorTy())
-        Op = Builder->CreateVectorSplat(
-            cast<FixedVectorType>(IntIdxTy)->getNumElements(), Op);
-
-      // Convert to correct type.
-      if (Op->getType() != IntIdxTy)
-        Op = Builder->CreateIntCast(Op, IntIdxTy, true, Op->getName() + ".c");
-      if (Size != 1) {
-        // We'll let instcombine(mul) convert this to a shl if possible.
-        Op = Builder->CreateMul(Op, ConstantInt::get(IntIdxTy, Size),
-                                GEP->getName() + ".idx", false /*NUW*/,
-                                isInBounds /*NSW*/);
+        AddOffset(ConstantInt::get(IntIdxTy, Size));
+        continue;
       }
-      Offset = Op;
     }
 
-    if (Result)
-      Result = Builder->CreateAdd(Result, Offset, GEP->getName() + ".offs",
-                                  false /*NUW*/, isInBounds /*NSW*/);
-    else
-      Result = Offset;
+    // Splat the index if needed.
+    if (IntIdxTy->isVectorTy() && !Op->getType()->isVectorTy())
+      Op = Builder->CreateVectorSplat(
+          cast<FixedVectorType>(IntIdxTy)->getNumElements(), Op);
+
+    // Convert to correct type.
+    if (Op->getType() != IntIdxTy)
+      Op = Builder->CreateIntCast(Op, IntIdxTy, true, Op->getName() + ".c");
+    if (Size != 1) {
+      // We'll let instcombine(mul) convert this to a shl if possible.
+      Op = Builder->CreateMul(Op, ConstantInt::get(IntIdxTy, Size),
+                              GEP->getName() + ".idx", false /*NUW*/,
+                              isInBounds /*NSW*/);
+    }
+    AddOffset(Op);
   }
   return Result ? Result : Constant::getNullValue(IntIdxTy);
 }
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 245cce25a3..540b70229b 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -14,7 +14,6 @@
 #include "llvm/Analysis/LoopAccessAnalysis.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/DepthFirstIterator.h"
 #include "llvm/ADT/EquivalenceClasses.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/STLExtras.h"
@@ -22,7 +21,6 @@
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/iterator_range.h"
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/AliasSetTracker.h"
 #include "llvm/Analysis/LoopAnalysisManager.h"
diff --git a/llvm/lib/Analysis/MemoryBuiltins.cpp b/llvm/lib/Analysis/MemoryBuiltins.cpp
index 6ee1984f90..9e6811f3bf 100644
--- a/llvm/lib/Analysis/MemoryBuiltins.cpp
+++ b/llvm/lib/Analysis/MemoryBuiltins.cpp
@@ -282,10 +282,7 @@ static AllocFnKind getAllocFnKind(const Value *V) {
 }
 
 static AllocFnKind getAllocFnKind(const Function *F) {
-  Attribute Attr = F->getFnAttribute(Attribute::AllocKind);
-  if (Attr.isValid())
-    return AllocFnKind(Attr.getValueAsInt());
-  return AllocFnKind::Unknown;
+  return F->getAttributes().getAllocKind();
 }
 
 static bool checkFnAllocKind(const Value *V, AllocFnKind Wanted) {
diff --git a/llvm/lib/Analysis/RegionPrinter.cpp b/llvm/lib/Analysis/RegionPrinter.cpp
index fbd3d17feb..ad59c79f40 100644
--- a/llvm/lib/Analysis/RegionPrinter.cpp
+++ b/llvm/lib/Analysis/RegionPrinter.cpp
@@ -9,7 +9,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Analysis/RegionPrinter.h"
-#include "llvm/ADT/DepthFirstIterator.h"
 #include "llvm/Analysis/DOTGraphTraitsPass.h"
 #include "llvm/Analysis/RegionInfo.h"
 #include "llvm/Analysis/RegionIterator.h"
diff --git a/llvm/lib/Analysis/TFLiteUtils.cpp b/llvm/lib/Analysis/TFLiteUtils.cpp
index b2862033e9..2762e22f28 100644
--- a/llvm/lib/Analysis/TFLiteUtils.cpp
+++ b/llvm/lib/Analysis/TFLiteUtils.cpp
@@ -1,4 +1,4 @@
-//===- TFUtils.cpp - tensorflow evaluation utilities ----------------------===//
+//===- TFUtils.cpp - TFLite-based evaluation utilities --------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This file implements utilities for interfacing with tensorflow C APIs.
+// This file implements utilities for interfacing with TFLite.
 //
 //===----------------------------------------------------------------------===//
 #include "llvm/Config/config.h"
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index e186431934..8824a05e3a 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -33,6 +33,7 @@
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/VectorUtils.h"
+#include "llvm/Analysis/WithCache.h"
 #include "llvm/IR/Argument.h"
 #include "llvm/IR/Attributes.h"
 #include "llvm/IR/BasicBlock.h"
@@ -178,17 +179,11 @@ void llvm::computeKnownBits(const Value *V, const APInt &DemandedElts,
       SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo));
 }
 
-static KnownBits computeKnownBits(const Value *V, const APInt &DemandedElts,
-                                  unsigned Depth, const SimplifyQuery &Q);
-
-static KnownBits computeKnownBits(const Value *V, unsigned Depth,
-                                  const SimplifyQuery &Q);
-
 KnownBits llvm::computeKnownBits(const Value *V, const DataLayout &DL,
                                  unsigned Depth, AssumptionCache *AC,
                                  const Instruction *CxtI,
                                  const DominatorTree *DT, bool UseInstrInfo) {
-  return ::computeKnownBits(
+  return computeKnownBits(
       V, Depth, SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo));
 }
 
@@ -196,13 +191,17 @@ KnownBits llvm::computeKnownBits(const Value *V, const APInt &DemandedElts,
                                  const DataLayout &DL, unsigned Depth,
                                  AssumptionCache *AC, const Instruction *CxtI,
                                  const DominatorTree *DT, bool UseInstrInfo) {
-  return ::computeKnownBits(
+  return computeKnownBits(
       V, DemandedElts, Depth,
       SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo));
 }
 
-bool llvm::haveNoCommonBitsSet(const Value *LHS, const Value *RHS,
+bool llvm::haveNoCommonBitsSet(const WithCache<const Value *> &LHSCache,
+                               const WithCache<const Value *> &RHSCache,
                                const SimplifyQuery &SQ) {
+  const Value *LHS = LHSCache.getValue();
+  const Value *RHS = RHSCache.getValue();
+
   assert(LHS->getType() == RHS->getType() &&
          "LHS and RHS should have the same type");
   assert(LHS->getType()->isIntOrIntVectorTy() &&
@@ -250,12 +249,9 @@ bool llvm::haveNoCommonBitsSet(const Value *LHS, const Value *RHS,
         match(LHS, m_Not(m_c_Or(m_Specific(A), m_Specific(B)))))
       return true;
   }
-  IntegerType *IT = cast<IntegerType>(LHS->getType()->getScalarType());
-  KnownBits LHSKnown(IT->getBitWidth());
-  KnownBits RHSKnown(IT->getBitWidth());
-  ::computeKnownBits(LHS, LHSKnown, 0, SQ);
-  ::computeKnownBits(RHS, RHSKnown, 0, SQ);
-  return KnownBits::haveNoCommonBitsSet(LHSKnown, RHSKnown);
+
+  return KnownBits::haveNoCommonBitsSet(LHSCache.getKnownBits(SQ),
+                                        RHSCache.getKnownBits(SQ));
 }
 
 bool llvm::isOnlyUsedInZeroEqualityComparison(const Instruction *I) {
@@ -567,11 +563,24 @@ static bool cmpExcludesZero(CmpInst::Predicate Pred, const Value *RHS) {
 
   // All other predicates - rely on generic ConstantRange handling.
   const APInt *C;
-  if (!match(RHS, m_APInt(C)))
+  auto Zero = APInt::getZero(RHS->getType()->getScalarSizeInBits());
+  if (match(RHS, m_APInt(C))) {
+    ConstantRange TrueValues = ConstantRange::makeExactICmpRegion(Pred, *C);
+    return !TrueValues.contains(Zero);
+  }
+
+  auto *VC = dyn_cast<ConstantDataVector>(RHS);
+  if (VC == nullptr)
     return false;
 
-  ConstantRange TrueValues = ConstantRange::makeExactICmpRegion(Pred, *C);
-  return !TrueValues.contains(APInt::getZero(C->getBitWidth()));
+  for (unsigned ElemIdx = 0, NElem = VC->getNumElements(); ElemIdx < NElem;
+       ++ElemIdx) {
+    ConstantRange TrueValues = ConstantRange::makeExactICmpRegion(
+        Pred, VC->getElementAsAPInt(ElemIdx));
+    if (TrueValues.contains(Zero))
+      return false;
+  }
+  return true;
 }
 
 static bool isKnownNonZeroFromAssume(const Value *V, const SimplifyQuery &Q) {
@@ -1771,19 +1780,19 @@ static void computeKnownBitsFromOperator(const Operator *I,
 
 /// Determine which bits of V are known to be either zero or one and return
 /// them.
-KnownBits computeKnownBits(const Value *V, const APInt &DemandedElts,
-                           unsigned Depth, const SimplifyQuery &Q) {
+KnownBits llvm::computeKnownBits(const Value *V, const APInt &DemandedElts,
+                                 unsigned Depth, const SimplifyQuery &Q) {
   KnownBits Known(getBitWidth(V->getType(), Q.DL));
-  computeKnownBits(V, DemandedElts, Known, Depth, Q);
+  ::computeKnownBits(V, DemandedElts, Known, Depth, Q);
   return Known;
 }
 
 /// Determine which bits of V are known to be either zero or one and return
 /// them.
-KnownBits computeKnownBits(const Value *V, unsigned Depth,
-                           const SimplifyQuery &Q) {
+KnownBits llvm::computeKnownBits(const Value *V, unsigned Depth,
+                                 const SimplifyQuery &Q) {
   KnownBits Known(getBitWidth(V->getType(), Q.DL));
-  computeKnownBits(V, Known, Depth, Q);
+  ::computeKnownBits(V, Known, Depth, Q);
   return Known;
 }
 
@@ -2048,20 +2057,19 @@ bool isKnownToBeAPowerOfTwo(const Value *V, bool OrZero, unsigned Depth,
       return isKnownToBeAPowerOfTwo(I->getOperand(0), OrZero, Depth, Q);
     return false;
   case Instruction::Mul:
-    return OrZero &&
-           isKnownToBeAPowerOfTwo(I->getOperand(1), OrZero, Depth, Q) &&
-           isKnownToBeAPowerOfTwo(I->getOperand(0), OrZero, Depth, Q);
+    return isKnownToBeAPowerOfTwo(I->getOperand(1), OrZero, Depth, Q) &&
+           isKnownToBeAPowerOfTwo(I->getOperand(0), OrZero, Depth, Q) &&
+           (OrZero || isKnownNonZero(I, Depth, Q));
   case Instruction::And:
-    if (OrZero) {
-      // A power of two and'd with anything is a power of two or zero.
-      if (isKnownToBeAPowerOfTwo(I->getOperand(1), /*OrZero*/ true, Depth, Q) ||
-          isKnownToBeAPowerOfTwo(I->getOperand(0), /*OrZero*/ true, Depth, Q))
-        return true;
-      // X & (-X) is always a power of two or zero.
-      if (match(I->getOperand(0), m_Neg(m_Specific(I->getOperand(1)))) ||
-          match(I->getOperand(1), m_Neg(m_Specific(I->getOperand(0)))))
-        return true;
-    }
+    // A power of two and'd with anything is a power of two or zero.
+    if (OrZero &&
+        (isKnownToBeAPowerOfTwo(I->getOperand(1), /*OrZero*/ true, Depth, Q) ||
+         isKnownToBeAPowerOfTwo(I->getOperand(0), /*OrZero*/ true, Depth, Q)))
+      return true;
+    // X & (-X) is always a power of two or zero.
+    if (match(I->getOperand(0), m_Neg(m_Specific(I->getOperand(1)))) ||
+        match(I->getOperand(1), m_Neg(m_Specific(I->getOperand(0)))))
+      return OrZero || isKnownNonZero(I->getOperand(0), Depth, Q);
     return false;
   case Instruction::Add: {
     // Adding a power-of-two or zero to the same power-of-two or zero yields
@@ -2256,6 +2264,10 @@ static bool isKnownNonNullFromDominatingCondition(const Value *V,
         return true;
     }
 
+    if (match(U, m_IDiv(m_Value(), m_Specific(V))) &&
+        isValidAssumeForContext(cast<Instruction>(U), CtxI, DT))
+      return true;
+
     // Consider only compare instructions uniquely controlling a branch
     Value *RHS;
     CmpInst::Predicate Pred;
@@ -6244,10 +6256,11 @@ static OverflowResult mapOverflowResult(ConstantRange::OverflowResult OR) {
 
 /// Combine constant ranges from computeConstantRange() and computeKnownBits().
 static ConstantRange
-computeConstantRangeIncludingKnownBits(const Value *V, bool ForSigned,
+computeConstantRangeIncludingKnownBits(const WithCache<const Value *> &V,
+                                       bool ForSigned,
                                        const SimplifyQuery &SQ) {
-  KnownBits Known = ::computeKnownBits(V, /*Depth=*/0, SQ);
-  ConstantRange CR1 = ConstantRange::fromKnownBits(Known, ForSigned);
+  ConstantRange CR1 =
+      ConstantRange::fromKnownBits(V.getKnownBits(SQ), ForSigned);
   ConstantRange CR2 = computeConstantRange(V, ForSigned, SQ.IIQ.UseInstrInfo);
   ConstantRange::PreferredRangeType RangeType =
       ForSigned ? ConstantRange::Signed : ConstantRange::Unsigned;
@@ -6257,8 +6270,8 @@ computeConstantRangeIncludingKnownBits(const Value *V, bool ForSigned,
 OverflowResult llvm::computeOverflowForUnsignedMul(const Value *LHS,
                                                    const Value *RHS,
                                                    const SimplifyQuery &SQ) {
-  KnownBits LHSKnown = ::computeKnownBits(LHS, /*Depth=*/0, SQ);
-  KnownBits RHSKnown = ::computeKnownBits(RHS, /*Depth=*/0, SQ);
+  KnownBits LHSKnown = computeKnownBits(LHS, /*Depth=*/0, SQ);
+  KnownBits RHSKnown = computeKnownBits(RHS, /*Depth=*/0, SQ);
   ConstantRange LHSRange = ConstantRange::fromKnownBits(LHSKnown, false);
   ConstantRange RHSRange = ConstantRange::fromKnownBits(RHSKnown, false);
   return mapOverflowResult(LHSRange.unsignedMulMayOverflow(RHSRange));
@@ -6295,17 +6308,18 @@ OverflowResult llvm::computeOverflowForSignedMul(const Value *LHS,
     // product is exactly the minimum negative number.
     // E.g. mul i16 with 17 sign bits: 0xff00 * 0xff80 = 0x8000
     // For simplicity we just check if at least one side is not negative.
-    KnownBits LHSKnown = ::computeKnownBits(LHS, /*Depth=*/0, SQ);
-    KnownBits RHSKnown = ::computeKnownBits(RHS, /*Depth=*/0, SQ);
+    KnownBits LHSKnown = computeKnownBits(LHS, /*Depth=*/0, SQ);
+    KnownBits RHSKnown = computeKnownBits(RHS, /*Depth=*/0, SQ);
     if (LHSKnown.isNonNegative() || RHSKnown.isNonNegative())
       return OverflowResult::NeverOverflows;
   }
   return OverflowResult::MayOverflow;
 }
 
-OverflowResult llvm::computeOverflowForUnsignedAdd(const Value *LHS,
-                                                   const Value *RHS,
-                                                   const SimplifyQuery &SQ) {
+OverflowResult
+llvm::computeOverflowForUnsignedAdd(const WithCache<const Value *> &LHS,
+                                    const WithCache<const Value *> &RHS,
+                                    const SimplifyQuery &SQ) {
   ConstantRange LHSRange =
       computeConstantRangeIncludingKnownBits(LHS, /*ForSigned=*/false, SQ);
   ConstantRange RHSRange =
@@ -6313,10 +6327,10 @@ OverflowResult llvm::computeOverflowForUnsignedAdd(const Value *LHS,
   return mapOverflowResult(LHSRange.unsignedAddMayOverflow(RHSRange));
 }
 
-static OverflowResult computeOverflowForSignedAdd(const Value *LHS,
-                                                  const Value *RHS,
-                                                  const AddOperator *Add,
-                                                  const SimplifyQuery &SQ) {
+static OverflowResult
+computeOverflowForSignedAdd(const WithCache<const Value *> &LHS,
+                            const WithCache<const Value *> &RHS,
+                            const AddOperator *Add, const SimplifyQuery &SQ) {
   if (Add && Add->hasNoSignedWrap()) {
     return OverflowResult::NeverOverflows;
   }
@@ -6932,9 +6946,10 @@ OverflowResult llvm::computeOverflowForSignedAdd(const AddOperator *Add,
                                        Add, SQ);
 }
 
-OverflowResult llvm::computeOverflowForSignedAdd(const Value *LHS,
-                                                 const Value *RHS,
-                                                 const SimplifyQuery &SQ) {
+OverflowResult
+llvm::computeOverflowForSignedAdd(const WithCache<const Value *> &LHS,
+                                  const WithCache<const Value *> &RHS,
+                                  const SimplifyQuery &SQ) {
   return ::computeOverflowForSignedAdd(LHS, RHS, nullptr, SQ);
 }
 
@@ -8492,6 +8507,11 @@ static void setLimitsForBinOp(const BinaryOperator &BO, APInt &Lower,
     if (match(BO.getOperand(1), m_APInt(C)))
       // 'and x, C' produces [0, C].
       Upper = *C + 1;
+    // X & -X is a power of two or zero. So we can cap the value at max power of
+    // two.
+    if (match(BO.getOperand(0), m_Neg(m_Specific(BO.getOperand(1)))) ||
+        match(BO.getOperand(1), m_Neg(m_Specific(BO.getOperand(0)))))
+      Upper = APInt::getSignedMinValue(Width) + 1;
     break;
 
   case Instruction::Or:
@@ -8553,7 +8573,20 @@ static void setLimitsForBinOp(const BinaryOperator &BO, APInt &Lower,
           Lower = *C;
           Upper = C->shl(ShiftAmount) + 1;
         }
+      } else {
+        // If lowbit is set, value can never be zero.
+        if ((*C)[0])
+          Lower = APInt::getOneBitSet(Width, 0);
+        // If we are shifting a constant the largest it can be is if the longest
+        // sequence of consecutive ones is shifted to the highbits (breaking
+        // ties for which sequence is higher). At the moment we take a liberal
+        // upper bound on this by just popcounting the constant.
+        // TODO: There may be a bitwise trick for it longest/highest
+        // consecutative sequence of ones (naive method is O(Width) loop).
+        Upper = APInt::getHighBitsSet(Width, C->popcount()) + 1;
       }
+    } else if (match(BO.getOperand(1), m_APInt(C)) && C->ult(Width)) {
+      Upper = APInt::getBitsSetFrom(Width, C->getZExtValue()) + 1;
     }
     break;
 
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index 9893e23468..be171867a2 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -12,7 +12,6 @@
 
 #include "llvm/Analysis/VectorUtils.h"
 #include "llvm/ADT/EquivalenceClasses.h"
-#include "llvm/ADT/SmallString.h"
 #include "llvm/Analysis/DemandedBits.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/LoopIterator.h"
diff --git a/llvm/lib/AsmParser/LLLexer.cpp b/llvm/lib/AsmParser/LLLexer.cpp
index 466bdebc00..ae46209b30 100644
--- a/llvm/lib/AsmParser/LLLexer.cpp
+++ b/llvm/lib/AsmParser/LLLexer.cpp
@@ -279,7 +279,7 @@ lltok::Kind LLLexer::LexDollar() {
       if (CurChar == '"') {
         StrVal.assign(TokStart + 2, CurPtr - 1);
         UnEscapeLexed(StrVal);
-        if (StringRef(StrVal).find_first_of(0) != StringRef::npos) {
+        if (StringRef(StrVal).contains(0)) {
           Error("Null bytes are not allowed in names");
           return lltok::Error;
         }
@@ -362,7 +362,7 @@ lltok::Kind LLLexer::LexVar(lltok::Kind Var, lltok::Kind VarID) {
       if (CurChar == '"') {
         StrVal.assign(TokStart+2, CurPtr-1);
         UnEscapeLexed(StrVal);
-        if (StringRef(StrVal).find_first_of(0) != StringRef::npos) {
+        if (StringRef(StrVal).contains(0)) {
           Error("Null bytes are not allowed in names");
           return lltok::Error;
         }
@@ -397,7 +397,7 @@ lltok::Kind LLLexer::LexQuote() {
 
   if (CurPtr[0] == ':') {
     ++CurPtr;
-    if (StringRef(StrVal).find_first_of(0) != StringRef::npos) {
+    if (StringRef(StrVal).contains(0)) {
       Error("Null bytes are not allowed in names");
       kind = lltok::Error;
     } else {
@@ -632,6 +632,7 @@ lltok::Kind LLLexer::LexIdentifier() {
   KEYWORD(amdgpu_kernel);
   KEYWORD(amdgpu_gfx);
   KEYWORD(tailcc);
+  KEYWORD(m68k_rtdcc);
 
   KEYWORD(cc);
   KEYWORD(c);
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index 04eabc94cf..e104f8b3d1 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -1999,6 +1999,7 @@ void LLParser::parseOptionalDLLStorageClass(unsigned &Res) {
 ///   ::= 'amdgpu_cs_chain_preserve'
 ///   ::= 'amdgpu_kernel'
 ///   ::= 'tailcc'
+///   ::= 'm68k_rtdcc'
 ///   ::= 'cc' UINT
 ///
 bool LLParser::parseOptionalCallingConv(unsigned &CC) {
@@ -2067,6 +2068,7 @@ bool LLParser::parseOptionalCallingConv(unsigned &CC) {
     break;
   case lltok::kw_amdgpu_kernel:  CC = CallingConv::AMDGPU_KERNEL; break;
   case lltok::kw_tailcc:         CC = CallingConv::Tail; break;
+  case lltok::kw_m68k_rtdcc:     CC = CallingConv::M68k_RTD; break;
   case lltok::kw_cc: {
       Lex.Lex();
       return parseUInt32(CC);
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index 1d1ec988a9..28addb9068 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -1980,6 +1980,8 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) {
     return Attribute::NoSanitizeCoverage;
   case bitc::ATTR_KIND_NULL_POINTER_IS_VALID:
     return Attribute::NullPointerIsValid;
+  case bitc::ATTR_KIND_OPTIMIZE_FOR_DEBUGGING:
+    return Attribute::OptimizeForDebugging;
   case bitc::ATTR_KIND_OPT_FOR_FUZZING:
     return Attribute::OptForFuzzing;
   case bitc::ATTR_KIND_OPTIMIZE_FOR_SIZE:
@@ -2644,7 +2646,7 @@ Expected<Value *> BitcodeReader::recordValue(SmallVectorImpl<uint64_t> &Record,
   Value *V = ValueList[ValueID];
 
   StringRef NameStr(ValueName.data(), ValueName.size());
-  if (NameStr.find_first_of(0) != StringRef::npos)
+  if (NameStr.contains(0))
     return error("Invalid value name");
   V->setName(NameStr);
   auto *GO = dyn_cast<GlobalObject>(V);
diff --git a/llvm/lib/Bitcode/Reader/MetadataLoader.cpp b/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
index 1e9ed5fcaa..1a4b55ea38 100644
--- a/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
+++ b/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
@@ -22,7 +22,6 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/ADT/ilist_iterator.h"
-#include "llvm/ADT/iterator_range.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/Bitcode/BitcodeReader.h"
 #include "llvm/Bitcode/LLVMBitCodes.h"
@@ -705,10 +704,11 @@ class MetadataLoader::MetadataLoaderImpl {
     return Error::success();
   }
 
-  void upgradeDebugInfo() {
+  void upgradeDebugInfo(bool ModuleLevel) {
     upgradeCUSubprograms();
     upgradeCUVariables();
-    upgradeCULocals();
+    if (ModuleLevel)
+      upgradeCULocals();
   }
 
   void callMDTypeCallback(Metadata **Val, unsigned TypeID);
@@ -1085,7 +1085,7 @@ Error MetadataLoader::MetadataLoaderImpl::parseMetadata(bool ModuleLevel) {
       // Reading the named metadata created forward references and/or
       // placeholders, that we flush here.
       resolveForwardRefsAndPlaceholders(Placeholders);
-      upgradeDebugInfo();
+      upgradeDebugInfo(ModuleLevel);
       // Return at the beginning of the block, since it is easy to skip it
       // entirely from there.
       Stream.ReadBlockEnd(); // Pop the abbrev block context.
@@ -1116,7 +1116,7 @@ Error MetadataLoader::MetadataLoaderImpl::parseMetadata(bool ModuleLevel) {
       return error("Malformed block");
     case BitstreamEntry::EndBlock:
       resolveForwardRefsAndPlaceholders(Placeholders);
-      upgradeDebugInfo();
+      upgradeDebugInfo(ModuleLevel);
       return Error::success();
     case BitstreamEntry::Record:
       // The interesting case.
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index e991d055f3..c427459508 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -747,6 +747,8 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
     return bitc::ATTR_KIND_NO_SANITIZE_COVERAGE;
   case Attribute::NullPointerIsValid:
     return bitc::ATTR_KIND_NULL_POINTER_IS_VALID;
+  case Attribute::OptimizeForDebugging:
+    return bitc::ATTR_KIND_OPTIMIZE_FOR_DEBUGGING;
   case Attribute::OptForFuzzing:
     return bitc::ATTR_KIND_OPT_FOR_FUZZING;
   case Attribute::OptimizeForSize:
diff --git a/llvm/lib/CodeGen/AsmPrinter/AccelTable.cpp b/llvm/lib/CodeGen/AsmPrinter/AccelTable.cpp
index 8f936037d1..88d5487427 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AccelTable.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AccelTable.cpp
@@ -13,7 +13,6 @@
 #include "llvm/CodeGen/AccelTable.h"
 #include "DwarfCompileUnit.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/CodeGen/AsmPrinter.h"
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index 97d2fe3426..072c55f79c 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -1524,7 +1524,7 @@ void AsmPrinter::emitPCSections(const MachineFunction &MF) {
         const size_t OptStart = SecWithOpt.find('!'); // likely npos
         const StringRef Sec = SecWithOpt.substr(0, OptStart);
         const StringRef Opts = SecWithOpt.substr(OptStart); // likely empty
-        ConstULEB128 = Opts.find('C') != StringRef::npos;
+        ConstULEB128 = Opts.contains('C');
 #ifndef NDEBUG
         for (char O : Opts)
           assert((O == '!' || O == 'C') && "Invalid !pcsections options");
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
index d30f0ef7af..a5960a5d4a 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
@@ -16,7 +16,6 @@
 #include "DwarfExpression.h"
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/APInt.h"
-#include "llvm/ADT/iterator_range.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
diff --git a/llvm/lib/CodeGen/AsmPrinter/EHStreamer.cpp b/llvm/lib/CodeGen/AsmPrinter/EHStreamer.cpp
index eef6b1d93f..32239535e4 100644
--- a/llvm/lib/CodeGen/AsmPrinter/EHStreamer.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/EHStreamer.cpp
@@ -13,7 +13,6 @@
 #include "EHStreamer.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Twine.h"
-#include "llvm/ADT/iterator_range.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/CodeGen/AsmPrinter.h"
 #include "llvm/CodeGen/MachineFunction.h"
diff --git a/llvm/lib/CodeGen/BasicBlockSections.cpp b/llvm/lib/CodeGen/BasicBlockSections.cpp
index de7c17082f..632fd68d88 100644
--- a/llvm/lib/CodeGen/BasicBlockSections.cpp
+++ b/llvm/lib/CodeGen/BasicBlockSections.cpp
@@ -168,31 +168,6 @@ updateBranches(MachineFunction &MF,
   }
 }
 
-// This function provides the BBCluster information associated with a function.
-// Returns true if a valid association exists and false otherwise.
-bool getBBClusterInfoForFunction(
-    const MachineFunction &MF,
-    BasicBlockSectionsProfileReader *BBSectionsProfileReader,
-    DenseMap<unsigned, BBClusterInfo> &V) {
-
-  // Find the assoicated cluster information.
-  std::pair<bool, SmallVector<BBClusterInfo, 4>> P =
-      BBSectionsProfileReader->getBBClusterInfoForFunction(MF.getName());
-  if (!P.first)
-    return false;
-
-  if (P.second.empty()) {
-    // This indicates that sections are desired for all basic blocks of this
-    // function. We clear the BBClusterInfo vector to denote this.
-    V.clear();
-    return true;
-  }
-
-  for (const BBClusterInfo &BBCI : P.second)
-    V[BBCI.BBID] = BBCI;
-  return true;
-}
-
 // This function sorts basic blocks according to the cluster's information.
 // All explicitly specified clusters of basic blocks will be ordered
 // accordingly. All non-specified BBs go into a separate "Cold" section.
@@ -200,12 +175,12 @@ bool getBBClusterInfoForFunction(
 // clusters, they are moved into a single "Exception" section. Eventually,
 // clusters are ordered in increasing order of their IDs, with the "Exception"
 // and "Cold" succeeding all other clusters.
-// FuncBBClusterInfo represent the cluster information for basic blocks. It
+// ClusterInfoByBBID represents the cluster information for basic blocks. It
 // maps from BBID of basic blocks to their cluster information. If this is
 // empty, it means unique sections for all basic blocks in the function.
-static void
-assignSections(MachineFunction &MF,
-               const DenseMap<unsigned, BBClusterInfo> &FuncBBClusterInfo) {
+static void assignSections(
+    MachineFunction &MF,
+    const DenseMap<unsigned, BBClusterInfo<unsigned>> &ClusterInfoByBBID) {
   assert(MF.hasBBSections() && "BB Sections is not set for function.");
   // This variable stores the section ID of the cluster containing eh_pads (if
   // all eh_pads are one cluster). If more than one cluster contain eh_pads, we
@@ -216,17 +191,17 @@ assignSections(MachineFunction &MF,
     // With the 'all' option, every basic block is placed in a unique section.
     // With the 'list' option, every basic block is placed in a section
     // associated with its cluster, unless we want individual unique sections
-    // for every basic block in this function (if FuncBBClusterInfo is empty).
+    // for every basic block in this function (if ClusterInfoByBBID is empty).
     if (MF.getTarget().getBBSectionsType() == llvm::BasicBlockSection::All ||
-        FuncBBClusterInfo.empty()) {
+        ClusterInfoByBBID.empty()) {
       // If unique sections are desired for all basic blocks of the function, we
       // set every basic block's section ID equal to its original position in
       // the layout (which is equal to its number). This ensures that basic
       // blocks are ordered canonically.
       MBB.setSectionID(MBB.getNumber());
     } else {
-      auto I = FuncBBClusterInfo.find(*MBB.getBBID());
-      if (I != FuncBBClusterInfo.end()) {
+      auto I = ClusterInfoByBBID.find(*MBB.getBBID());
+      if (I != ClusterInfoByBBID.end()) {
         MBB.setSectionID(I->second.ClusterID);
       } else {
         // BB goes into the special cold section if it is not specified in the
@@ -333,16 +308,28 @@ bool BasicBlockSections::runOnMachineFunction(MachineFunction &MF) {
     return true;
   }
 
-  BBSectionsProfileReader = &getAnalysis<BasicBlockSectionsProfileReader>();
+  DenseMap<unsigned, BBClusterInfo<unsigned>> ClusterInfoByBBID;
+  if (BBSectionsType == BasicBlockSection::List) {
+    auto [HasProfile, PathAndClusterInfo] =
+        getAnalysis<BasicBlockSectionsProfileReader>()
+            .getPathAndClusterInfoForFunction(MF.getName());
+    if (!HasProfile)
+      return true;
+    for (const BBClusterInfo<ProfileBBID> &BBP :
+         PathAndClusterInfo.ClusterInfo) {
+      // TODO: Apply the path cloning profile.
+      assert(!BBP.BasicBlockID.CloneID && "Path cloning is not supported yet");
+      const auto [I, Inserted] = ClusterInfoByBBID.try_emplace(
+          BBP.BasicBlockID.BBID,
+          BBClusterInfo<unsigned>{BBP.BasicBlockID.BBID, BBP.ClusterID,
+                                  BBP.PositionInCluster});
+      (void)I;
+      assert(Inserted && "Duplicate BBID found in profile");
+    }
+  }
 
-  // Map from BBID of blocks to their cluster information.
-  DenseMap<unsigned, BBClusterInfo> FuncBBClusterInfo;
-  if (BBSectionsType == BasicBlockSection::List &&
-      !getBBClusterInfoForFunction(MF, BBSectionsProfileReader,
-                                   FuncBBClusterInfo))
-    return true;
   MF.setBBSectionsType(BBSectionsType);
-  assignSections(MF, FuncBBClusterInfo);
+  assignSections(MF, ClusterInfoByBBID);
 
   // We make sure that the cluster including the entry basic block precedes all
   // other clusters.
@@ -376,8 +363,8 @@ bool BasicBlockSections::runOnMachineFunction(MachineFunction &MF) {
     // If the two basic block are in the same section, the order is decided by
     // their position within the section.
     if (XSectionID.Type == MBBSectionID::SectionType::Default)
-      return FuncBBClusterInfo.lookup(*X.getBBID()).PositionInCluster <
-             FuncBBClusterInfo.lookup(*Y.getBBID()).PositionInCluster;
+      return ClusterInfoByBBID.lookup(*X.getBBID()).PositionInCluster <
+             ClusterInfoByBBID.lookup(*Y.getBBID()).PositionInCluster;
     return X.getNumber() < Y.getNumber();
   };
 
diff --git a/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp b/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp
index ef5f1251f5..6bb412a6c7 100644
--- a/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp
+++ b/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp
@@ -13,6 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/CodeGen/BasicBlockSectionsProfileReader.h"
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
@@ -34,17 +35,37 @@ INITIALIZE_PASS(BasicBlockSectionsProfileReader, "bbsections-profile-reader",
                 "Reads and parses a basic block sections profile.", false,
                 false)
 
+Expected<ProfileBBID>
+BasicBlockSectionsProfileReader::parseProfileBBID(StringRef S) const {
+  SmallVector<StringRef, 2> Parts;
+  S.split(Parts, '.');
+  if (Parts.size() > 2)
+    return createProfileParseError(Twine("unable to parse basic block id: '") +
+                                   S + "'");
+  unsigned long long BBID;
+  if (getAsUnsignedInteger(Parts[0], 10, BBID))
+    return createProfileParseError(
+        Twine("unable to parse BB id: '" + Parts[0]) +
+        "': unsigned integer expected");
+  unsigned long long CloneID = 0;
+  if (Parts.size() > 1 && getAsUnsignedInteger(Parts[1], 10, CloneID))
+    return createProfileParseError(Twine("unable to parse clone id: '") +
+                                   Parts[1] + "': unsigned integer expected");
+  return ProfileBBID{static_cast<unsigned>(BBID),
+                     static_cast<unsigned>(CloneID)};
+}
+
 bool BasicBlockSectionsProfileReader::isFunctionHot(StringRef FuncName) const {
-  return getBBClusterInfoForFunction(FuncName).first;
+  return getPathAndClusterInfoForFunction(FuncName).first;
 }
 
-std::pair<bool, SmallVector<BBClusterInfo>>
-BasicBlockSectionsProfileReader::getBBClusterInfoForFunction(
+std::pair<bool, FunctionPathAndClusterInfo>
+BasicBlockSectionsProfileReader::getPathAndClusterInfoForFunction(
     StringRef FuncName) const {
-  auto R = ProgramBBClusterInfo.find(getAliasName(FuncName));
-  return R != ProgramBBClusterInfo.end()
+  auto R = ProgramPathAndClusterInfo.find(getAliasName(FuncName));
+  return R != ProgramPathAndClusterInfo.end()
              ? std::pair(true, R->second)
-             : std::pair(false, SmallVector<BBClusterInfo>{});
+             : std::pair(false, FunctionPathAndClusterInfo());
 }
 
 // Reads the version 1 basic block sections profile. Profile for each function
@@ -61,8 +82,49 @@ BasicBlockSectionsProfileReader::getBBClusterInfoForFunction(
 // aliases. Basic block clusters are specified by 'c' and specify the cluster of
 // basic blocks, and the internal order in which they must be placed in the same
 // section.
+// This profile can also specify cloning paths which instruct the compiler to
+// clone basic blocks along a path. The cloned blocks are then specified in the
+// cluster information.
+// The following profile lists two cloning paths (starting with 'p') for
+// function bar and places the total 9 blocks within two clusters. The first two
+// blocks of a cloning path specify the edge along which the path is cloned. For
+// instance, path 1 (1 -> 3 -> 4) instructs that 3 and 4 must be cloned along
+// the edge 1->3. Within the given clusters, each cloned block is identified by
+// "<original block id>.<clone id>". For instance, 3.1 represents the first
+// clone of block 3. Original blocks are specified just with their block ids. A
+// block cloned multiple times appears with distinct clone ids. The CFG for bar
+// is shown below before and after cloning with its final clusters labeled.
+//
+// f main
+// f bar
+// p 1 3 4           # cloning path 1
+// p 4 2             # cloning path 2
+// c 1 3.1 4.1 6     # basic block cluster 1
+// c 0 2 3 4 2.1 5   # basic block cluster 2
+// ****************************************************************************
+// function bar before and after cloning with basic block clusters shown.
+// ****************************************************************************
+//                                ....      ..............
+//      0 -------+                : 0 :---->: 1 ---> 3.1 :
+//      |        |                : | :     :........ |  :
+//      v        v                : v :             : v  :
+// +--> 2 --> 5  1   ~~~~~~>  +---: 2 :             : 4.1: clsuter 1
+// |    |        |            |   : | :             : |  :
+// |    v        |            |   : v .......       : v  :
+// |    3 <------+            |   : 3 <--+  :       : 6  :
+// |    |                     |   : |    |  :       :....:
+// |    v                     |   : v    |  :
+// +--- 4 ---> 6              |   : 4    |  :
+//                            |   : |    |  :
+//                            |   : v    |  :
+//                            |   :2.1---+  : cluster 2
+//                            |   : | ......:
+//                            |   : v :
+//                            +-->: 5 :
+//                                ....
+// ****************************************************************************
 Error BasicBlockSectionsProfileReader::ReadV1Profile() {
-  auto FI = ProgramBBClusterInfo.end();
+  auto FI = ProgramPathAndClusterInfo.end();
 
   // Current cluster ID corresponding to this function.
   unsigned CurrentCluster = 0;
@@ -71,7 +133,7 @@ Error BasicBlockSectionsProfileReader::ReadV1Profile() {
 
   // Temporary set to ensure every basic block ID appears once in the clusters
   // of a function.
-  SmallSet<unsigned, 4> FuncBBIDs;
+  DenseSet<ProfileBBID> FuncBBIDs;
 
   // Debug-info-based module filename for the current function. Empty string
   // means no filename.
@@ -85,7 +147,7 @@ Error BasicBlockSectionsProfileReader::ReadV1Profile() {
     S.split(Values, ' ');
     switch (Specifier) {
     case '@':
-      break;
+      continue;
     case 'm': // Module name speicifer.
       if (Values.size() != 1) {
         return createProfileParseError(Twine("invalid module name value: '") +
@@ -106,7 +168,7 @@ Error BasicBlockSectionsProfileReader::ReadV1Profile() {
       if (!FunctionFound) {
         // Skip the following profile by setting the profile iterator (FI) to
         // the past-the-end element.
-        FI = ProgramBBClusterInfo.end();
+        FI = ProgramPathAndClusterInfo.end();
         DIFilename = "";
         continue;
       }
@@ -115,7 +177,7 @@ Error BasicBlockSectionsProfileReader::ReadV1Profile() {
 
       // Prepare for parsing clusters of this function name.
       // Start a new cluster map for this function name.
-      auto R = ProgramBBClusterInfo.try_emplace(Values.front());
+      auto R = ProgramPathAndClusterInfo.try_emplace(Values.front());
       // Report error when multiple profiles have been specified for the same
       // function.
       if (!R.second)
@@ -132,38 +194,55 @@ Error BasicBlockSectionsProfileReader::ReadV1Profile() {
     case 'c': // Basic block cluster specifier.
       // Skip the profile when we the profile iterator (FI) refers to the
       // past-the-end element.
-      if (FI == ProgramBBClusterInfo.end())
-        break;
+      if (FI == ProgramPathAndClusterInfo.end())
+        continue;
       // Reset current cluster position.
       CurrentPosition = 0;
-      for (auto BBIDStr : Values) {
-        unsigned long long BBID;
-        if (getAsUnsignedInteger(BBIDStr, 10, BBID))
-          return createProfileParseError(Twine("unsigned integer expected: '") +
-                                         BBIDStr + "'");
-        if (!FuncBBIDs.insert(BBID).second)
+      for (auto BasicBlockIDStr : Values) {
+        auto BasicBlockID = parseProfileBBID(BasicBlockIDStr);
+        if (!BasicBlockID)
+          return BasicBlockID.takeError();
+        if (!FuncBBIDs.insert(*BasicBlockID).second)
           return createProfileParseError(
-              Twine("duplicate basic block id found '") + BBIDStr + "'");
-        if (BBID == 0 && CurrentPosition)
+              Twine("duplicate basic block id found '") + BasicBlockIDStr +
+              "'");
+
+        if (!BasicBlockID->BBID && CurrentPosition)
           return createProfileParseError(
-              "entry BB (0) does not begin a cluster");
+              "entry BB (0) does not begin a cluster.");
 
-        FI->second.emplace_back(
-            BBClusterInfo{((unsigned)BBID), CurrentCluster, CurrentPosition++});
+        FI->second.ClusterInfo.emplace_back(BBClusterInfo<ProfileBBID>{
+            *std::move(BasicBlockID), CurrentCluster, CurrentPosition++});
       }
       CurrentCluster++;
       continue;
+    case 'p': { // Basic block cloning path specifier.
+      SmallSet<unsigned, 5> BBsInPath;
+      FI->second.ClonePaths.push_back({});
+      for (size_t I = 0; I < Values.size(); ++I) {
+        auto BBIDStr = Values[I];
+        unsigned long long BBID = 0;
+        if (getAsUnsignedInteger(BBIDStr, 10, BBID))
+          return createProfileParseError(Twine("unsigned integer expected: '") +
+                                         BBIDStr + "'");
+        if (I != 0 && !BBsInPath.insert(BBID).second)
+          return createProfileParseError(
+              Twine("duplicate cloned block in path: '") + BBIDStr + "'");
+        FI->second.ClonePaths.back().push_back(BBID);
+      }
+      continue;
+    }
     default:
       return createProfileParseError(Twine("invalid specifier: '") +
                                      Twine(Specifier) + "'");
     }
+    llvm_unreachable("should not break from this switch statement");
   }
   return Error::success();
 }
 
 Error BasicBlockSectionsProfileReader::ReadV0Profile() {
-  auto FI = ProgramBBClusterInfo.end();
-
+  auto FI = ProgramPathAndClusterInfo.end();
   // Current cluster ID corresponding to this function.
   unsigned CurrentCluster = 0;
   // Current position in the current cluster.
@@ -184,7 +263,7 @@ Error BasicBlockSectionsProfileReader::ReadV0Profile() {
     if (S.consume_front("!")) {
       // Skip the profile when we the profile iterator (FI) refers to the
       // past-the-end element.
-      if (FI == ProgramBBClusterInfo.end())
+      if (FI == ProgramPathAndClusterInfo.end())
         continue;
       SmallVector<StringRef, 4> BBIDs;
       S.split(BBIDs, ' ');
@@ -202,8 +281,10 @@ Error BasicBlockSectionsProfileReader::ReadV0Profile() {
           return createProfileParseError(
               "entry BB (0) does not begin a cluster");
 
-        FI->second.emplace_back(
-            BBClusterInfo{((unsigned)BBID), CurrentCluster, CurrentPosition++});
+        FI->second.ClusterInfo.emplace_back(
+            BBClusterInfo<ProfileBBID>({{static_cast<unsigned>(BBID), 0},
+                                        CurrentCluster,
+                                        CurrentPosition++}));
       }
       CurrentCluster++;
     } else {
@@ -237,7 +318,7 @@ Error BasicBlockSectionsProfileReader::ReadV0Profile() {
       if (!FunctionFound) {
         // Skip the following profile by setting the profile iterator (FI) to
         // the past-the-end element.
-        FI = ProgramBBClusterInfo.end();
+        FI = ProgramPathAndClusterInfo.end();
         continue;
       }
       for (size_t i = 1; i < Aliases.size(); ++i)
@@ -245,7 +326,7 @@ Error BasicBlockSectionsProfileReader::ReadV0Profile() {
 
       // Prepare for parsing clusters of this function name.
       // Start a new cluster map for this function name.
-      auto R = ProgramBBClusterInfo.try_emplace(Aliases.front());
+      auto R = ProgramPathAndClusterInfo.try_emplace(Aliases.front());
       // Report error when multiple profiles have been specified for the same
       // function.
       if (!R.second)
@@ -261,7 +342,7 @@ Error BasicBlockSectionsProfileReader::ReadV0Profile() {
 
 // Basic Block Sections can be enabled for a subset of machine basic blocks.
 // This is done by passing a file containing names of functions for which basic
-// block sections are desired.  Additionally, machine basic block ids of the
+// block sections are desired. Additionally, machine basic block ids of the
 // functions can also be specified for a finer granularity. Moreover, a cluster
 // of basic blocks could be assigned to the same section.
 // Optionally, a debug-info filename can be specified for each function to allow
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 371f6598e6..187820717b 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -7999,6 +7999,8 @@ static bool tryUnmergingGEPsAcrossIndirectBr(GetElementPtrInst *GEPI,
       return false;
     if (UGEPI->getOperand(0) != GEPIOp)
       return false;
+    if (UGEPI->getSourceElementType() != GEPI->getSourceElementType())
+      return false;
     if (GEPIIdx->getType() !=
         cast<ConstantInt>(UGEPI->getOperand(1))->getType())
       return false;
diff --git a/llvm/lib/CodeGen/EarlyIfConversion.cpp b/llvm/lib/CodeGen/EarlyIfConversion.cpp
index 61867d74bf..31e107ade1 100644
--- a/llvm/lib/CodeGen/EarlyIfConversion.cpp
+++ b/llvm/lib/CodeGen/EarlyIfConversion.cpp
@@ -828,8 +828,6 @@ void updateDomTree(MachineDominatorTree *DomTree, const SSAIfConv &IfConv,
 /// Update LoopInfo after if-conversion.
 void updateLoops(MachineLoopInfo *Loops,
                  ArrayRef<MachineBasicBlock *> Removed) {
-  if (!Loops)
-    return;
   // If-conversion doesn't change loop structure, and it doesn't mess with back
   // edges, so updating LoopInfo is simply removing the dead blocks.
   for (auto *B : Removed)
@@ -1092,7 +1090,7 @@ bool EarlyIfConverter::runOnMachineFunction(MachineFunction &MF) {
   SchedModel = STI.getSchedModel();
   MRI = &MF.getRegInfo();
   DomTree = &getAnalysis<MachineDominatorTree>();
-  Loops = getAnalysisIfAvailable<MachineLoopInfo>();
+  Loops = &getAnalysis<MachineLoopInfo>();
   Traces = &getAnalysis<MachineTraceMetrics>();
   MinInstr = nullptr;
 
@@ -1226,7 +1224,7 @@ bool EarlyIfPredicator::runOnMachineFunction(MachineFunction &MF) {
   MRI = &MF.getRegInfo();
   SchedModel.init(&STI);
   DomTree = &getAnalysis<MachineDominatorTree>();
-  Loops = getAnalysisIfAvailable<MachineLoopInfo>();
+  Loops = &getAnalysis<MachineLoopInfo>();
   MBPI = &getAnalysis<MachineBranchProbabilityInfo>();
 
   bool Changed = false;
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index ec501083aa..108768494c 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -2141,8 +2141,20 @@ LegalizerHelper::widenScalarMulo(MachineInstr &MI, unsigned TypeIdx,
   auto LeftOperand = MIRBuilder.buildInstr(ExtOp, {WideTy}, {LHS});
   auto RightOperand = MIRBuilder.buildInstr(ExtOp, {WideTy}, {RHS});
 
-  auto Mulo = MIRBuilder.buildInstr(MI.getOpcode(), {WideTy, OverflowTy},
-                                    {LeftOperand, RightOperand});
+  // Multiplication cannot overflow if the WideTy is >= 2 * original width,
+  // so we don't need to check the overflow result of larger type Mulo.
+  bool WideMulCanOverflow = WideTy.getScalarSizeInBits() < 2 * SrcBitWidth;
+
+  unsigned MulOpc =
+      WideMulCanOverflow ? MI.getOpcode() : (unsigned)TargetOpcode::G_MUL;
+
+  MachineInstrBuilder Mulo;
+  if (WideMulCanOverflow)
+    Mulo = MIRBuilder.buildInstr(MulOpc, {WideTy, OverflowTy},
+                                 {LeftOperand, RightOperand});
+  else
+    Mulo = MIRBuilder.buildInstr(MulOpc, {WideTy}, {LeftOperand, RightOperand});
+
   auto Mul = Mulo->getOperand(0);
   MIRBuilder.buildTrunc(Result, Mul);
 
@@ -2160,9 +2172,7 @@ LegalizerHelper::widenScalarMulo(MachineInstr &MI, unsigned TypeIdx,
     ExtResult = MIRBuilder.buildZExtInReg(WideTy, Mul, SrcBitWidth);
   }
 
-  // Multiplication cannot overflow if the WideTy is >= 2 * original width,
-  // so we don't need to check the overflow result of larger type Mulo.
-  if (WideTy.getScalarSizeInBits() < 2 * SrcBitWidth) {
+  if (WideMulCanOverflow) {
     auto Overflow =
         MIRBuilder.buildICmp(CmpInst::ICMP_NE, OverflowTy, Mul, ExtResult);
     // Finally check if the multiplication in the larger type itself overflowed.
@@ -3766,6 +3776,8 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT LowerHintTy) {
   case G_SEXT:
   case G_ANYEXT:
     return lowerEXT(MI);
+  case G_TRUNC:
+    return lowerTRUNC(MI);
   GISEL_VECREDUCE_CASES_NONSEQ
     return lowerVectorReduction(MI);
   }
@@ -5110,13 +5122,7 @@ LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx,
     MI.eraseFromParent();
     return Legalized;
   }
-  case TargetOpcode::G_TRUNC: {
-    Observer.changingInstr(MI);
-    moreElementsVectorSrc(MI, MoreTy, 1);
-    moreElementsVectorDst(MI, MoreTy, 0);
-    Observer.changedInstr(MI);
-    return Legalized;
-  }
+  case TargetOpcode::G_TRUNC:
   case TargetOpcode::G_FPTRUNC:
   case TargetOpcode::G_FPEXT: {
     if (TypeIdx != 0)
@@ -6165,6 +6171,63 @@ LegalizerHelper::LegalizeResult LegalizerHelper::lowerEXT(MachineInstr &MI) {
   return UnableToLegalize;
 }
 
+LegalizerHelper::LegalizeResult LegalizerHelper::lowerTRUNC(MachineInstr &MI) {
+  // MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
+  MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
+  // Similar to how operand splitting is done in SelectiondDAG, we can handle
+  // %res(v8s8) = G_TRUNC %in(v8s32) by generating:
+  //   %inlo(<4x s32>), %inhi(<4 x s32>) = G_UNMERGE %in(<8 x s32>)
+  //   %lo16(<4 x s16>) = G_TRUNC %inlo
+  //   %hi16(<4 x s16>) = G_TRUNC %inhi
+  //   %in16(<8 x s16>) = G_CONCAT_VECTORS %lo16, %hi16
+  //   %res(<8 x s8>) = G_TRUNC %in16
+
+  assert(MI.getOpcode() == TargetOpcode::G_TRUNC);
+
+  Register DstReg = MI.getOperand(0).getReg();
+  Register SrcReg = MI.getOperand(1).getReg();
+  LLT DstTy = MRI.getType(DstReg);
+  LLT SrcTy = MRI.getType(SrcReg);
+
+  if (DstTy.isVector() && isPowerOf2_32(DstTy.getNumElements()) &&
+      isPowerOf2_32(DstTy.getScalarSizeInBits()) &&
+      isPowerOf2_32(SrcTy.getNumElements()) &&
+      isPowerOf2_32(SrcTy.getScalarSizeInBits())) {
+    // Split input type.
+    LLT SplitSrcTy = SrcTy.changeElementCount(
+        SrcTy.getElementCount().divideCoefficientBy(2));
+
+    // First, split the source into two smaller vectors.
+    SmallVector<Register, 2> SplitSrcs;
+    extractParts(SrcReg, SplitSrcTy, 2, SplitSrcs);
+
+    // Truncate the splits into intermediate narrower elements.
+    LLT InterTy;
+    if (DstTy.getScalarSizeInBits() * 2 < SrcTy.getScalarSizeInBits())
+      InterTy = SplitSrcTy.changeElementSize(DstTy.getScalarSizeInBits() * 2);
+    else
+      InterTy = SplitSrcTy.changeElementSize(DstTy.getScalarSizeInBits());
+    for (unsigned I = 0; I < SplitSrcs.size(); ++I) {
+      SplitSrcs[I] = MIRBuilder.buildTrunc(InterTy, SplitSrcs[I]).getReg(0);
+    }
+
+    // Combine the new truncates into one vector
+    auto Merge = MIRBuilder.buildMergeLikeInstr(
+        DstTy.changeElementSize(InterTy.getScalarSizeInBits()), SplitSrcs);
+
+    // Truncate the new vector to the final result type
+    if (DstTy.getScalarSizeInBits() * 2 < SrcTy.getScalarSizeInBits())
+      MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), Merge.getReg(0));
+    else
+      MIRBuilder.buildCopy(MI.getOperand(0).getReg(), Merge.getReg(0));
+
+    MI.eraseFromParent();
+
+    return Legalized;
+  }
+  return UnableToLegalize;
+}
+
 LegalizerHelper::LegalizeResult
 LegalizerHelper::lowerRotateWithReverseRotate(MachineInstr &MI) {
   auto [Dst, DstTy, Src, SrcTy, Amt, AmtTy] = MI.getFirst3RegLLTs();
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index 473c3f452f..36de083b31 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -290,9 +290,10 @@ void llvm::reportGISelFailure(MachineFunction &MF, const TargetPassConfig &TPC,
 }
 
 std::optional<APInt> llvm::getIConstantVRegVal(Register VReg,
-                                               const MachineRegisterInfo &MRI) {
-  std::optional<ValueAndVReg> ValAndVReg = getIConstantVRegValWithLookThrough(
-      VReg, MRI, /*LookThroughInstrs*/ false);
+                                               const MachineRegisterInfo &MRI,
+                                               bool LookThroughInstrs) {
+  std::optional<ValueAndVReg> ValAndVReg =
+      getIConstantVRegValWithLookThrough(VReg, MRI, LookThroughInstrs);
   assert((!ValAndVReg || ValAndVReg->VReg == VReg) &&
          "Value found while looking through instrs");
   if (!ValAndVReg)
@@ -301,8 +302,9 @@ std::optional<APInt> llvm::getIConstantVRegVal(Register VReg,
 }
 
 std::optional<int64_t>
-llvm::getIConstantVRegSExtVal(Register VReg, const MachineRegisterInfo &MRI) {
-  std::optional<APInt> Val = getIConstantVRegVal(VReg, MRI);
+llvm::getIConstantVRegSExtVal(Register VReg, const MachineRegisterInfo &MRI,
+                              bool LookThroughInstrs) {
+  std::optional<APInt> Val = getIConstantVRegVal(VReg, MRI, LookThroughInstrs);
   if (Val && Val->getBitWidth() <= 64)
     return Val->getSExtValue();
   return std::nullopt;
diff --git a/llvm/lib/CodeGen/InlineSpiller.cpp b/llvm/lib/CodeGen/InlineSpiller.cpp
index c62f3db9d3..46fcc62e09 100644
--- a/llvm/lib/CodeGen/InlineSpiller.cpp
+++ b/llvm/lib/CodeGen/InlineSpiller.cpp
@@ -1071,8 +1071,7 @@ void InlineSpiller::insertReload(Register NewVReg,
 static bool isRealSpill(const MachineInstr &Def) {
   if (!Def.isImplicitDef())
     return true;
-  assert(Def.getNumOperands() == 1 &&
-         "Implicit def with more than one definition");
+
   // We can say that the VReg defined by Def is undef, only if it is
   // fully defined by Def. Otherwise, some of the lanes may not be
   // undef and the value of the VReg matters.
diff --git a/llvm/lib/CodeGen/LiveIntervalCalc.cpp b/llvm/lib/CodeGen/LiveIntervalCalc.cpp
index ccc5ae9808..ba4ea32e68 100644
--- a/llvm/lib/CodeGen/LiveIntervalCalc.cpp
+++ b/llvm/lib/CodeGen/LiveIntervalCalc.cpp
@@ -12,7 +12,6 @@
 
 #include "llvm/CodeGen/LiveIntervalCalc.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/iterator_range.h"
 #include "llvm/CodeGen/LiveInterval.h"
 #include "llvm/CodeGen/MachineInstr.h"
 #include "llvm/CodeGen/MachineOperand.h"
diff --git a/llvm/lib/CodeGen/LiveVariables.cpp b/llvm/lib/CodeGen/LiveVariables.cpp
index 077276b64a..6b983b6320 100644
--- a/llvm/lib/CodeGen/LiveVariables.cpp
+++ b/llvm/lib/CodeGen/LiveVariables.cpp
@@ -601,7 +601,6 @@ bool LiveVariables::runOnMachineFunction(MachineFunction &mf) {
   PhysRegDef.assign(NumRegs, nullptr);
   PhysRegUse.assign(NumRegs, nullptr);
   PHIVarInfo.resize(MF->getNumBlockIDs());
-  PHIJoins.clear();
 
   // FIXME: LiveIntervals will be updated to remove its dependence on
   // LiveVariables to improve compilation time and eliminate bizarre pass
diff --git a/llvm/lib/CodeGen/MIRFSDiscriminator.cpp b/llvm/lib/CodeGen/MIRFSDiscriminator.cpp
index 8d17cceeb3..f5146f5fee 100644
--- a/llvm/lib/CodeGen/MIRFSDiscriminator.cpp
+++ b/llvm/lib/CodeGen/MIRFSDiscriminator.cpp
@@ -35,10 +35,12 @@ using namespace sampleprofutil;
 
 // TODO(xur): Remove this option and related code once we make true as the
 // default.
+namespace llvm {
 cl::opt<bool> ImprovedFSDiscriminator(
     "improved-fs-discriminator", cl::Hidden, cl::init(false),
     cl::desc("New FS discriminators encoding (incompatible with the original "
              "encoding)"));
+}
 
 char MIRAddFSDiscriminators::ID = 0;
 
diff --git a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
index b2e570c5e6..78d7e62797 100644
--- a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
@@ -13,7 +13,6 @@
 
 #include "llvm/CodeGen/MIRParser/MIRParser.h"
 #include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/AsmParser/Parser.h"
 #include "llvm/AsmParser/SlotMapping.h"
diff --git a/llvm/lib/CodeGen/MIRSampleProfile.cpp b/llvm/lib/CodeGen/MIRSampleProfile.cpp
index 96f8589e68..42d0aba4b1 100644
--- a/llvm/lib/CodeGen/MIRSampleProfile.cpp
+++ b/llvm/lib/CodeGen/MIRSampleProfile.cpp
@@ -61,7 +61,9 @@ static cl::opt<bool> ViewBFIAfter("fs-viewbfi-after", cl::Hidden,
                                   cl::init(false),
                                   cl::desc("View BFI after MIR loader"));
 
+namespace llvm {
 extern cl::opt<bool> ImprovedFSDiscriminator;
+}
 char MIRProfileLoaderPass::ID = 0;
 
 INITIALIZE_PASS_BEGIN(MIRProfileLoaderPass, DEBUG_TYPE,
diff --git a/llvm/lib/CodeGen/MachineBasicBlock.cpp b/llvm/lib/CodeGen/MachineBasicBlock.cpp
index 7d3d8b6fba..5f9e4a66c0 100644
--- a/llvm/lib/CodeGen/MachineBasicBlock.cpp
+++ b/llvm/lib/CodeGen/MachineBasicBlock.cpp
@@ -1097,6 +1097,36 @@ static bool jumpTableHasOtherUses(const MachineFunction &MF,
   return false;
 }
 
+class SlotIndexUpdateDelegate : public MachineFunction::Delegate {
+private:
+  MachineFunction &MF;
+  SlotIndexes *Indexes;
+  SmallSetVector<MachineInstr *, 2> Insertions;
+
+public:
+  SlotIndexUpdateDelegate(MachineFunction &MF, SlotIndexes *Indexes)
+      : MF(MF), Indexes(Indexes) {
+    MF.setDelegate(this);
+  }
+
+  ~SlotIndexUpdateDelegate() {
+    MF.resetDelegate(this);
+    for (auto MI : Insertions)
+      Indexes->insertMachineInstrInMaps(*MI);
+  }
+
+  void MF_HandleInsertion(MachineInstr &MI) override {
+    // This is called before MI is inserted into block so defer index update.
+    if (Indexes)
+      Insertions.insert(&MI);
+  }
+
+  void MF_HandleRemoval(MachineInstr &MI) override {
+    if (Indexes && !Insertions.remove(&MI))
+      Indexes->removeMachineInstrFromMaps(MI);
+  }
+};
+
 MachineBasicBlock *MachineBasicBlock::SplitCriticalEdge(
     MachineBasicBlock *Succ, Pass &P,
     std::vector<SparseBitVector<>> *LiveInSets) {
@@ -1170,51 +1200,23 @@ MachineBasicBlock *MachineBasicBlock::SplitCriticalEdge(
 
   ReplaceUsesOfBlockWith(Succ, NMBB);
 
-  // If updateTerminator() removes instructions, we need to remove them from
-  // SlotIndexes.
-  SmallVector<MachineInstr*, 4> Terminators;
-  if (Indexes) {
-    for (MachineInstr &MI :
-         llvm::make_range(getFirstInstrTerminator(), instr_end()))
-      Terminators.push_back(&MI);
-  }
-
   // Since we replaced all uses of Succ with NMBB, that should also be treated
   // as the fallthrough successor
   if (Succ == PrevFallthrough)
     PrevFallthrough = NMBB;
 
-  if (!ChangedIndirectJump)
+  if (!ChangedIndirectJump) {
+    SlotIndexUpdateDelegate SlotUpdater(*MF, Indexes);
     updateTerminator(PrevFallthrough);
-
-  if (Indexes) {
-    SmallVector<MachineInstr*, 4> NewTerminators;
-    for (MachineInstr &MI :
-         llvm::make_range(getFirstInstrTerminator(), instr_end()))
-      NewTerminators.push_back(&MI);
-
-    for (MachineInstr *Terminator : Terminators) {
-      if (!is_contained(NewTerminators, Terminator))
-        Indexes->removeMachineInstrFromMaps(*Terminator);
-    }
   }
 
   // Insert unconditional "jump Succ" instruction in NMBB if necessary.
   NMBB->addSuccessor(Succ);
   if (!NMBB->isLayoutSuccessor(Succ)) {
+    SlotIndexUpdateDelegate SlotUpdater(*MF, Indexes);
     SmallVector<MachineOperand, 4> Cond;
     const TargetInstrInfo *TII = getParent()->getSubtarget().getInstrInfo();
     TII->insertBranch(*NMBB, Succ, nullptr, Cond, DL);
-
-    if (Indexes) {
-      for (MachineInstr &MI : NMBB->instrs()) {
-        // Some instructions may have been moved to NMBB by updateTerminator(),
-        // so we first remove any instruction that already has an index.
-        if (Indexes->hasIndex(MI))
-          Indexes->removeMachineInstrFromMaps(MI);
-        Indexes->insertMachineInstrInMaps(MI);
-      }
-    }
   }
 
   // Fix PHI nodes in Succ so they refer to NMBB instead of this.
diff --git a/llvm/lib/CodeGen/MachineOperand.cpp b/llvm/lib/CodeGen/MachineOperand.cpp
index 788c134b6e..b6d6a7532d 100644
--- a/llvm/lib/CodeGen/MachineOperand.cpp
+++ b/llvm/lib/CodeGen/MachineOperand.cpp
@@ -11,13 +11,13 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/CodeGen/MachineOperand.h"
+#include "llvm/ADT/StableHashing.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Analysis/Loads.h"
 #include "llvm/CodeGen/MIRFormatter.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineJumpTableInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/StableHashing.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
 #include "llvm/Config/llvm-config.h"
diff --git a/llvm/lib/CodeGen/MachineSink.cpp b/llvm/lib/CodeGen/MachineSink.cpp
index 40f33d6644..d0e119cb74 100644
--- a/llvm/lib/CodeGen/MachineSink.cpp
+++ b/llvm/lib/CodeGen/MachineSink.cpp
@@ -57,7 +57,6 @@
 #include <algorithm>
 #include <cassert>
 #include <cstdint>
-#include <map>
 #include <utility>
 #include <vector>
 
@@ -139,7 +138,7 @@ namespace {
     DenseSet<Register> RegsToClearKillFlags;
 
     using AllSuccsCache =
-        std::map<MachineBasicBlock *, SmallVector<MachineBasicBlock *, 4>>;
+        DenseMap<MachineBasicBlock *, SmallVector<MachineBasicBlock *, 4>>;
 
     /// DBG_VALUE pointer and flag. The flag is true if this DBG_VALUE is
     /// post-dominated by another DBG_VALUE of the same variable location.
@@ -160,14 +159,15 @@ namespace {
     /// current block.
     DenseSet<DebugVariable> SeenDbgVars;
 
-    std::map<std::pair<MachineBasicBlock *, MachineBasicBlock *>, bool>
+    DenseMap<std::pair<MachineBasicBlock *, MachineBasicBlock *>, bool>
         HasStoreCache;
-    std::map<std::pair<MachineBasicBlock *, MachineBasicBlock *>,
-             std::vector<MachineInstr *>>
+
+    DenseMap<std::pair<MachineBasicBlock *, MachineBasicBlock *>,
+             SmallVector<MachineInstr *>>
         StoreInstrCache;
 
     /// Cached BB's register pressure.
-    std::map<const MachineBasicBlock *, std::vector<unsigned>>
+    DenseMap<const MachineBasicBlock *, std::vector<unsigned>>
         CachedRegisterPressure;
 
     bool EnableSinkAndFold;
@@ -539,9 +539,10 @@ bool MachineSinking::PerformSinkAndFold(MachineInstr &MI,
       New = TII->emitLdStWithAddr(*SinkDst, MaybeAM);
     }
     LLVM_DEBUG(dbgs() << "yielding"; New->dump());
+    // Clear the StoreInstrCache, since we may invalidate it by erasing.
+    if (SinkDst->mayStore() && !SinkDst->hasOrderedMemoryRef())
+      StoreInstrCache.clear();
     SinkDst->eraseFromParent();
-    // Clear the StoreInstrCache, since we may have invalidated it by erasing.
-    StoreInstrCache.clear();
   }
 
   // Collect operands that need to be cleaned up because the registers no longer
@@ -1429,11 +1430,11 @@ bool MachineSinking::hasStoreBetween(MachineBasicBlock *From,
 
   // Does these two blocks pair be queried before and have a definite cached
   // result?
-  if (HasStoreCache.find(BlockPair) != HasStoreCache.end())
-    return HasStoreCache[BlockPair];
+  if (auto It = HasStoreCache.find(BlockPair); It != HasStoreCache.end())
+    return It->second;
 
-  if (StoreInstrCache.find(BlockPair) != StoreInstrCache.end())
-    return llvm::any_of(StoreInstrCache[BlockPair], [&](MachineInstr *I) {
+  if (auto It = StoreInstrCache.find(BlockPair); It != StoreInstrCache.end())
+    return llvm::any_of(It->second, [&](MachineInstr *I) {
       return I->mayAlias(AA, MI, false);
     });
 
diff --git a/llvm/lib/CodeGen/MachineStableHash.cpp b/llvm/lib/CodeGen/MachineStableHash.cpp
index 9628e4c5ae..66edac9476 100644
--- a/llvm/lib/CodeGen/MachineStableHash.cpp
+++ b/llvm/lib/CodeGen/MachineStableHash.cpp
@@ -19,9 +19,9 @@
 #include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StableHashing.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/ilist_iterator.h"
-#include "llvm/ADT/iterator_range.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineInstr.h"
@@ -30,7 +30,6 @@
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/Register.h"
-#include "llvm/CodeGen/StableHashing.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/MC/MCSymbol.h"
diff --git a/llvm/lib/CodeGen/PHIElimination.cpp b/llvm/lib/CodeGen/PHIElimination.cpp
index dbb9a9ffdf..10d8378ce5 100644
--- a/llvm/lib/CodeGen/PHIElimination.cpp
+++ b/llvm/lib/CodeGen/PHIElimination.cpp
@@ -330,9 +330,6 @@ void PHIElimination::LowerPHINode(MachineBasicBlock &MBB,
     if (IncomingReg) {
       LiveVariables::VarInfo &VI = LV->getVarInfo(IncomingReg);
 
-      // Increment use count of the newly created virtual register.
-      LV->setPHIJoin(IncomingReg);
-
       MachineInstr *OldKill = nullptr;
       bool IsPHICopyAfterOldKill = false;
 
diff --git a/llvm/lib/CodeGen/PeepholeOptimizer.cpp b/llvm/lib/CodeGen/PeepholeOptimizer.cpp
index a08cc78f11..09ae2680f5 100644
--- a/llvm/lib/CodeGen/PeepholeOptimizer.cpp
+++ b/llvm/lib/CodeGen/PeepholeOptimizer.cpp
@@ -202,7 +202,8 @@ namespace {
     bool isMoveImmediate(MachineInstr &MI, SmallSet<Register, 4> &ImmDefRegs,
                          DenseMap<Register, MachineInstr *> &ImmDefMIs);
     bool foldImmediate(MachineInstr &MI, SmallSet<Register, 4> &ImmDefRegs,
-                       DenseMap<Register, MachineInstr *> &ImmDefMIs);
+                       DenseMap<Register, MachineInstr *> &ImmDefMIs,
+                       bool &Deleted);
 
     /// Finds recurrence cycles, but only ones that formulated around
     /// a def operand and a use operand that are tied. If there is a use
@@ -217,8 +218,11 @@ namespace {
     /// set \p CopyMIs. If this virtual register was previously seen as a
     /// copy, replace the uses of this copy with the previously seen copy's
     /// destination register.
+    /// \p LocalMIs contains all previous seen instructions. An optimized away
+    /// instruction should be deleted from LocalMIs.
     bool foldRedundantCopy(MachineInstr &MI,
-                           DenseMap<RegSubRegPair, MachineInstr *> &CopyMIs);
+                           DenseMap<RegSubRegPair, MachineInstr *> &CopyMIs,
+                           SmallPtrSetImpl<MachineInstr *> &LocalMIs);
 
     /// Is the register \p Reg a non-allocatable physical register?
     bool isNAPhysCopy(Register Reg);
@@ -1351,18 +1355,19 @@ bool PeepholeOptimizer::isMoveImmediate(
     MachineInstr &MI, SmallSet<Register, 4> &ImmDefRegs,
     DenseMap<Register, MachineInstr *> &ImmDefMIs) {
   const MCInstrDesc &MCID = MI.getDesc();
-  if (!MI.isMoveImmediate())
-    return false;
-  if (MCID.getNumDefs() != 1)
+  if (MCID.getNumDefs() != 1 || !MI.getOperand(0).isReg())
     return false;
   Register Reg = MI.getOperand(0).getReg();
-  if (Reg.isVirtual()) {
-    ImmDefMIs.insert(std::make_pair(Reg, &MI));
-    ImmDefRegs.insert(Reg);
-    return true;
-  }
+  if (!Reg.isVirtual())
+    return false;
 
-  return false;
+  int64_t ImmVal;
+  if (!MI.isMoveImmediate() && !TII->getConstValDefinedInReg(MI, Reg, ImmVal))
+    return false;
+
+  ImmDefMIs.insert(std::make_pair(Reg, &MI));
+  ImmDefRegs.insert(Reg);
+  return true;
 }
 
 /// Try folding register operands that are defined by move immediate
@@ -1370,7 +1375,8 @@ bool PeepholeOptimizer::isMoveImmediate(
 /// and only if the def and use are in the same BB.
 bool PeepholeOptimizer::foldImmediate(
     MachineInstr &MI, SmallSet<Register, 4> &ImmDefRegs,
-    DenseMap<Register, MachineInstr *> &ImmDefMIs) {
+    DenseMap<Register, MachineInstr *> &ImmDefMIs, bool &Deleted) {
+  Deleted = false;
   for (unsigned i = 0, e = MI.getDesc().getNumOperands(); i != e; ++i) {
     MachineOperand &MO = MI.getOperand(i);
     if (!MO.isReg() || MO.isDef())
@@ -1384,6 +1390,19 @@ bool PeepholeOptimizer::foldImmediate(
     assert(II != ImmDefMIs.end() && "couldn't find immediate definition");
     if (TII->FoldImmediate(MI, *II->second, Reg, MRI)) {
       ++NumImmFold;
+      // FoldImmediate can delete ImmDefMI if MI was its only user. If ImmDefMI
+      // is not deleted, and we happened to get a same MI, we can delete MI and
+      // replace its users.
+      if (MRI->getVRegDef(Reg) &&
+          MI.isIdenticalTo(*II->second, MachineInstr::IgnoreVRegDefs)) {
+        Register DstReg = MI.getOperand(0).getReg();
+        if (DstReg.isVirtual() &&
+            MRI->getRegClass(DstReg) == MRI->getRegClass(Reg)) {
+          MRI->replaceRegWith(DstReg, Reg);
+          MI.eraseFromParent();
+          Deleted = true;
+        }
+      }
       return true;
     }
   }
@@ -1405,7 +1424,8 @@ bool PeepholeOptimizer::foldImmediate(
 //
 // Should replace %2 uses with %1:sub1
 bool PeepholeOptimizer::foldRedundantCopy(
-    MachineInstr &MI, DenseMap<RegSubRegPair, MachineInstr *> &CopyMIs) {
+    MachineInstr &MI, DenseMap<RegSubRegPair, MachineInstr *> &CopyMIs,
+    SmallPtrSetImpl<MachineInstr *> &LocalMIs) {
   assert(MI.isCopy() && "expected a COPY machine instruction");
 
   Register SrcReg = MI.getOperand(1).getReg();
@@ -1425,6 +1445,10 @@ bool PeepholeOptimizer::foldRedundantCopy(
   }
 
   MachineInstr *PrevCopy = CopyMIs.find(SrcPair)->second;
+  // A COPY instruction can be deleted or changed by other optimizations.
+  // Check if the previous COPY instruction is existing and still a COPY.
+  if (!LocalMIs.count(PrevCopy) || !PrevCopy->isCopy())
+    return false;
 
   assert(SrcSubReg == PrevCopy->getOperand(1).getSubReg() &&
          "Unexpected mismatching subreg!");
@@ -1732,7 +1756,7 @@ bool PeepholeOptimizer::runOnMachineFunction(MachineFunction &MF) {
         continue;
       }
 
-      if (MI->isCopy() && (foldRedundantCopy(*MI, CopySrcMIs) ||
+      if (MI->isCopy() && (foldRedundantCopy(*MI, CopySrcMIs, LocalMIs) ||
                            foldRedundantNAPhysCopy(*MI, NAPhysToVirtMIs))) {
         LocalMIs.erase(MI);
         LLVM_DEBUG(dbgs() << "Deleting redundant copy: " << *MI << "\n");
@@ -1750,8 +1774,14 @@ bool PeepholeOptimizer::runOnMachineFunction(MachineFunction &MF) {
         // next iteration sees the new instructions.
         MII = MI;
         ++MII;
-        if (SeenMoveImm)
-          Changed |= foldImmediate(*MI, ImmDefRegs, ImmDefMIs);
+        if (SeenMoveImm) {
+          bool Deleted;
+          Changed |= foldImmediate(*MI, ImmDefRegs, ImmDefMIs, Deleted);
+          if (Deleted) {
+            LocalMIs.erase(MI);
+            continue;
+          }
+        }
       }
 
       // Check whether MI is a load candidate for folding into a later
diff --git a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
index 9318306779..b6e8e9a8b4 100644
--- a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
+++ b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
@@ -162,16 +162,6 @@ static bool lowerObjCCall(Function &F, const char *NewFn,
     CallInst::TailCallKind TCK = CI->getTailCallKind();
     NewCI->setTailCallKind(std::max(TCK, OverridingTCK));
 
-    // Transfer the 'returned' attribute from the intrinsic to the call site.
-    // By applying this only to intrinsic call sites, we avoid applying it to
-    // non-ARC explicit calls to things like objc_retain which have not been
-    // auto-upgraded to use the intrinsics.
-    unsigned Index;
-    if (F.getAttributes().hasAttrSomewhere(Attribute::Returned, &Index) &&
-        Index)
-      NewCI->addParamAttr(Index - AttributeList::FirstArgIndex,
-                          Attribute::Returned);
-
     if (!CI->use_empty())
       CI->replaceAllUsesWith(NewCI);
     CI->eraseFromParent();
diff --git a/llvm/lib/CodeGen/PrologEpilogInserter.cpp b/llvm/lib/CodeGen/PrologEpilogInserter.cpp
index 9514a0875a..0b8dcdcd6e 100644
--- a/llvm/lib/CodeGen/PrologEpilogInserter.cpp
+++ b/llvm/lib/CodeGen/PrologEpilogInserter.cpp
@@ -337,7 +337,9 @@ bool PEI::runOnMachineFunction(MachineFunction &MF) {
     return MachineOptimizationRemarkAnalysis(DEBUG_TYPE, "StackSize",
                                              MF.getFunction().getSubprogram(),
                                              &MF.front())
-           << ore::NV("NumStackBytes", StackSize) << " stack bytes in function";
+           << ore::NV("NumStackBytes", StackSize)
+           << " stack bytes in function '"
+           << ore::NV("Function", MF.getFunction().getName()) << "'";
   });
 
   delete RS;
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.cpp b/llvm/lib/CodeGen/RegAllocGreedy.cpp
index 4d476924a7..349d8b0975 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.cpp
+++ b/llvm/lib/CodeGen/RegAllocGreedy.cpp
@@ -1217,6 +1217,12 @@ bool RAGreedy::trySplitAroundHintReg(MCPhysReg Hint,
                                      const LiveInterval &VirtReg,
                                      SmallVectorImpl<Register> &NewVRegs,
                                      AllocationOrder &Order) {
+  // Split the VirtReg may generate COPY instructions in multiple cold basic
+  // blocks, and increase code size. So we avoid it when the function is
+  // optimized for size.
+  if (MF->getFunction().hasOptSize())
+    return false;
+
   // Don't allow repeated splitting as a safe guard against looping.
   if (ExtraInfo->getStage(VirtReg) >= RS_Split2)
     return false;
diff --git a/llvm/lib/CodeGen/ScheduleDAG.cpp b/llvm/lib/CodeGen/ScheduleDAG.cpp
index 19ea39f519..de8e6f6379 100644
--- a/llvm/lib/CodeGen/ScheduleDAG.cpp
+++ b/llvm/lib/CodeGen/ScheduleDAG.cpp
@@ -15,7 +15,6 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
-#include "llvm/ADT/iterator_range.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/ScheduleHazardRecognizer.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 1021b07da1..1add486255 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -495,6 +495,7 @@ namespace {
     SDValue visitFSUB(SDNode *N);
     SDValue visitFMUL(SDNode *N);
     template <class MatchContextClass> SDValue visitFMA(SDNode *N);
+    SDValue visitFMAD(SDNode *N);
     SDValue visitFDIV(SDNode *N);
     SDValue visitFREM(SDNode *N);
     SDValue visitFSQRT(SDNode *N);
@@ -504,6 +505,7 @@ namespace {
     SDValue visitUINT_TO_FP(SDNode *N);
     SDValue visitFP_TO_SINT(SDNode *N);
     SDValue visitFP_TO_UINT(SDNode *N);
+    SDValue visitXRINT(SDNode *N);
     SDValue visitFP_ROUND(SDNode *N);
     SDValue visitFP_EXTEND(SDNode *N);
     SDValue visitFNEG(SDNode *N);
@@ -1910,6 +1912,7 @@ void DAGCombiner::Run(CombineLevel AtLevel) {
 }
 
 SDValue DAGCombiner::visit(SDNode *N) {
+  // clang-format off
   switch (N->getOpcode()) {
   default: break;
   case ISD::TokenFactor:        return visitTokenFactor(N);
@@ -2000,6 +2003,7 @@ SDValue DAGCombiner::visit(SDNode *N) {
   case ISD::FSUB:               return visitFSUB(N);
   case ISD::FMUL:               return visitFMUL(N);
   case ISD::FMA:                return visitFMA<EmptyMatchContext>(N);
+  case ISD::FMAD:               return visitFMAD(N);
   case ISD::FDIV:               return visitFDIV(N);
   case ISD::FREM:               return visitFREM(N);
   case ISD::FSQRT:              return visitFSQRT(N);
@@ -2009,6 +2013,8 @@ SDValue DAGCombiner::visit(SDNode *N) {
   case ISD::UINT_TO_FP:         return visitUINT_TO_FP(N);
   case ISD::FP_TO_SINT:         return visitFP_TO_SINT(N);
   case ISD::FP_TO_UINT:         return visitFP_TO_UINT(N);
+  case ISD::LRINT:
+  case ISD::LLRINT:             return visitXRINT(N);
   case ISD::FP_ROUND:           return visitFP_ROUND(N);
   case ISD::FP_EXTEND:          return visitFP_EXTEND(N);
   case ISD::FNEG:               return visitFNEG(N);
@@ -2063,6 +2069,7 @@ SDValue DAGCombiner::visit(SDNode *N) {
 #include "llvm/IR/VPIntrinsics.def"
     return visitVPOp(N);
   }
+  // clang-format on
   return SDValue();
 }
 
@@ -6635,12 +6642,17 @@ bool DAGCombiner::BackwardsPropagateMask(SDNode *N) {
       SDValue Op1 = LogicN->getOperand(1);
 
       if (isa<ConstantSDNode>(Op0))
-          std::swap(Op0, Op1);
+        Op0 =
+            DAG.getNode(ISD::AND, SDLoc(Op0), Op0.getValueType(), Op0, MaskOp);
 
-      SDValue And = DAG.getNode(ISD::AND, SDLoc(Op1), Op1.getValueType(),
-                                Op1, MaskOp);
+      if (isa<ConstantSDNode>(Op1))
+        Op1 =
+            DAG.getNode(ISD::AND, SDLoc(Op1), Op1.getValueType(), Op1, MaskOp);
 
-      DAG.UpdateNodeOperands(LogicN, Op0, And);
+      if (isa<ConstantSDNode>(Op0) && !isa<ConstantSDNode>(Op1))
+        std::swap(Op0, Op1);
+
+      DAG.UpdateNodeOperands(LogicN, Op0, Op1);
     }
 
     // Create narrow loads.
@@ -7071,12 +7083,23 @@ SDValue DAGCombiner::visitAND(SDNode *N) {
             N1, /*AllowUndef=*/false, /*AllowTruncation=*/true)) {
       Constant = C->getAPIntValue();
     } else if (BuildVectorSDNode *Vector = dyn_cast<BuildVectorSDNode>(N1)) {
+      unsigned EltBitWidth = Vector->getValueType(0).getScalarSizeInBits();
       APInt SplatValue, SplatUndef;
       unsigned SplatBitSize;
       bool HasAnyUndefs;
-      bool IsSplat = Vector->isConstantSplat(SplatValue, SplatUndef,
-                                             SplatBitSize, HasAnyUndefs);
-      if (IsSplat) {
+      // Endianness should not matter here. Code below makes sure that we only
+      // use the result if the SplatBitSize is a multiple of the vector element
+      // size. And after that we AND all element sized parts of the splat
+      // together. So the end result should be the same regardless of in which
+      // order we do those operations.
+      const bool IsBigEndian = false;
+      bool IsSplat =
+          Vector->isConstantSplat(SplatValue, SplatUndef, SplatBitSize,
+                                  HasAnyUndefs, EltBitWidth, IsBigEndian);
+
+      // Make sure that variable 'Constant' is only set if 'SplatBitSize' is a
+      // multiple of 'BitWidth'. Otherwise, we could propagate a wrong value.
+      if (IsSplat && (SplatBitSize % EltBitWidth) == 0) {
         // Undef bits can contribute to a possible optimisation if set, so
         // set them.
         SplatValue |= SplatUndef;
@@ -7085,23 +7108,9 @@ SDValue DAGCombiner::visitAND(SDNode *N) {
         // the first vector value and FF for the rest, repeating. We need a mask
         // that will apply equally to all members of the vector, so AND all the
         // lanes of the constant together.
-        unsigned EltBitWidth = Vector->getValueType(0).getScalarSizeInBits();
-
-        // If the splat value has been compressed to a bitlength lower
-        // than the size of the vector lane, we need to re-expand it to
-        // the lane size.
-        if (EltBitWidth > SplatBitSize)
-          for (SplatValue = SplatValue.zextOrTrunc(EltBitWidth);
-               SplatBitSize < EltBitWidth; SplatBitSize = SplatBitSize * 2)
-            SplatValue |= SplatValue.shl(SplatBitSize);
-
-        // Make sure that variable 'Constant' is only set if 'SplatBitSize' is a
-        // multiple of 'BitWidth'. Otherwise, we could propagate a wrong value.
-        if ((SplatBitSize % EltBitWidth) == 0) {
-          Constant = APInt::getAllOnes(EltBitWidth);
-          for (unsigned i = 0, n = (SplatBitSize / EltBitWidth); i < n; ++i)
-            Constant &= SplatValue.extractBits(EltBitWidth, i * EltBitWidth);
-        }
+        Constant = APInt::getAllOnes(EltBitWidth);
+        for (unsigned i = 0, n = (SplatBitSize / EltBitWidth); i < n; ++i)
+          Constant &= SplatValue.extractBits(EltBitWidth, i * EltBitWidth);
       }
     }
 
@@ -12439,27 +12448,127 @@ SDValue DAGCombiner::visitSETCC(SDNode *N) {
 
   ISD::CondCode Cond = cast<CondCodeSDNode>(N->getOperand(2))->get();
   EVT VT = N->getValueType(0);
+  SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
 
-  SDValue Combined = SimplifySetCC(VT, N->getOperand(0), N->getOperand(1), Cond,
-                                   SDLoc(N), !PreferSetCC);
-
-  if (!Combined)
-    return SDValue();
+  SDValue Combined = SimplifySetCC(VT, N0, N1, Cond, SDLoc(N), !PreferSetCC);
 
-  // If we prefer to have a setcc, and we don't, we'll try our best to
-  // recreate one using rebuildSetCC.
-  if (PreferSetCC && Combined.getOpcode() != ISD::SETCC) {
-    SDValue NewSetCC = rebuildSetCC(Combined);
+  if (Combined) {
+    // If we prefer to have a setcc, and we don't, we'll try our best to
+    // recreate one using rebuildSetCC.
+    if (PreferSetCC && Combined.getOpcode() != ISD::SETCC) {
+      SDValue NewSetCC = rebuildSetCC(Combined);
 
-    // We don't have anything interesting to combine to.
-    if (NewSetCC.getNode() == N)
-      return SDValue();
+      // We don't have anything interesting to combine to.
+      if (NewSetCC.getNode() == N)
+        return SDValue();
 
-    if (NewSetCC)
-      return NewSetCC;
+      if (NewSetCC)
+        return NewSetCC;
+    }
+    return Combined;
   }
 
-  return Combined;
+  // Optimize
+  //    1) (icmp eq/ne (and X, C0), (shift X, C1))
+  // or
+  //    2) (icmp eq/ne X, (rotate X, C1))
+  // If C0 is a mask or shifted mask and the shift amt (C1) isolates the
+  // remaining bits (i.e something like `(x64 & UINT32_MAX) == (x64 >> 32)`)
+  // Then:
+  // If C1 is a power of 2, then the rotate and shift+and versions are
+  // equivilent, so we can interchange them depending on target preference.
+  // Otherwise, if we have the shift+and version we can interchange srl/shl
+  // which inturn affects the constant C0. We can use this to get better
+  // constants again determined by target preference.
+  if (Cond == ISD::SETNE || Cond == ISD::SETEQ) {
+    auto IsAndWithShift = [](SDValue A, SDValue B) {
+      return A.getOpcode() == ISD::AND &&
+             (B.getOpcode() == ISD::SRL || B.getOpcode() == ISD::SHL) &&
+             A.getOperand(0) == B.getOperand(0);
+    };
+    auto IsRotateWithOp = [](SDValue A, SDValue B) {
+      return (B.getOpcode() == ISD::ROTL || B.getOpcode() == ISD::ROTR) &&
+             B.getOperand(0) == A;
+    };
+    SDValue AndOrOp = SDValue(), ShiftOrRotate = SDValue();
+    bool IsRotate = false;
+
+    // Find either shift+and or rotate pattern.
+    if (IsAndWithShift(N0, N1)) {
+      AndOrOp = N0;
+      ShiftOrRotate = N1;
+    } else if (IsAndWithShift(N1, N0)) {
+      AndOrOp = N1;
+      ShiftOrRotate = N0;
+    } else if (IsRotateWithOp(N0, N1)) {
+      IsRotate = true;
+      AndOrOp = N0;
+      ShiftOrRotate = N1;
+    } else if (IsRotateWithOp(N1, N0)) {
+      IsRotate = true;
+      AndOrOp = N1;
+      ShiftOrRotate = N0;
+    }
+
+    if (AndOrOp && ShiftOrRotate && ShiftOrRotate.hasOneUse() &&
+        (IsRotate || AndOrOp.hasOneUse())) {
+      EVT OpVT = N0.getValueType();
+      // Get constant shift/rotate amount and possibly mask (if its shift+and
+      // variant).
+      auto GetAPIntValue = [](SDValue Op) -> std::optional<APInt> {
+        ConstantSDNode *CNode = isConstOrConstSplat(Op, /*AllowUndefs*/ false,
+                                                    /*AllowTrunc*/ false);
+        if (CNode == nullptr)
+          return std::nullopt;
+        return CNode->getAPIntValue();
+      };
+      std::optional<APInt> AndCMask =
+          IsRotate ? std::nullopt : GetAPIntValue(AndOrOp.getOperand(1));
+      std::optional<APInt> ShiftCAmt =
+          GetAPIntValue(ShiftOrRotate.getOperand(1));
+      unsigned NumBits = OpVT.getScalarSizeInBits();
+
+      // We found constants.
+      if (ShiftCAmt && (IsRotate || AndCMask) && ShiftCAmt->ult(NumBits)) {
+        unsigned ShiftOpc = ShiftOrRotate.getOpcode();
+        // Check that the constants meet the constraints.
+        bool CanTransform =
+            IsRotate ||
+            (*ShiftCAmt == (~*AndCMask).popcount() && ShiftOpc == ISD::SHL
+                 ? (~*AndCMask).isMask()
+                 : AndCMask->isMask());
+
+        // See if target prefers another shift/rotate opcode.
+        unsigned NewShiftOpc = TLI.preferedOpcodeForCmpEqPiecesOfOperand(
+            OpVT, ShiftOpc, ShiftCAmt->isPowerOf2(), *ShiftCAmt, AndCMask);
+        // Transform is valid and we have a new preference.
+        if (CanTransform && NewShiftOpc != ShiftOpc) {
+          SDLoc DL(N);
+          SDValue NewShiftOrRotate =
+              DAG.getNode(NewShiftOpc, DL, OpVT, ShiftOrRotate.getOperand(0),
+                          ShiftOrRotate.getOperand(1));
+          SDValue NewAndOrOp = SDValue();
+
+          if (NewShiftOpc == ISD::SHL || NewShiftOpc == ISD::SRL) {
+            APInt NewMask =
+                NewShiftOpc == ISD::SHL
+                    ? APInt::getHighBitsSet(NumBits,
+                                            NumBits - ShiftCAmt->getZExtValue())
+                    : APInt::getLowBitsSet(NumBits,
+                                           NumBits - ShiftCAmt->getZExtValue());
+            NewAndOrOp =
+                DAG.getNode(ISD::AND, DL, OpVT, ShiftOrRotate.getOperand(0),
+                            DAG.getConstant(NewMask, DL, OpVT));
+          } else {
+            NewAndOrOp = ShiftOrRotate.getOperand(0);
+          }
+
+          return DAG.getSetCC(DL, VT, NewAndOrOp, NewShiftOrRotate, Cond);
+        }
+      }
+    }
+  }
+  return SDValue();
 }
 
 SDValue DAGCombiner::visitSETCCCARRY(SDNode *N) {
@@ -16750,6 +16859,21 @@ template <class MatchContextClass> SDValue DAGCombiner::visitFMA(SDNode *N) {
   return SDValue();
 }
 
+SDValue DAGCombiner::visitFMAD(SDNode *N) {
+  SDValue N0 = N->getOperand(0);
+  SDValue N1 = N->getOperand(1);
+  SDValue N2 = N->getOperand(2);
+  EVT VT = N->getValueType(0);
+  SDLoc DL(N);
+
+  // Constant fold FMAD.
+  if (isa<ConstantFPSDNode>(N0) && isa<ConstantFPSDNode>(N1) &&
+      isa<ConstantFPSDNode>(N2))
+    return DAG.getNode(ISD::FMAD, DL, VT, N0, N1, N2);
+
+  return SDValue();
+}
+
 // Combine multiple FDIVs with the same divisor into multiple FMULs by the
 // reciprocal.
 // E.g., (a / D; b / D;) -> (recip = 1.0 / D; a * recip; b * recip)
@@ -17361,6 +17485,21 @@ SDValue DAGCombiner::visitFP_TO_UINT(SDNode *N) {
   return FoldIntToFPToInt(N, DAG);
 }
 
+SDValue DAGCombiner::visitXRINT(SDNode *N) {
+  SDValue N0 = N->getOperand(0);
+  EVT VT = N->getValueType(0);
+
+  // fold (lrint|llrint undef) -> undef
+  if (N0.isUndef())
+    return DAG.getUNDEF(VT);
+
+  // fold (lrint|llrint c1fp) -> c1
+  if (DAG.isConstantFPBuildVectorOrConstantFP(N0))
+    return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N0);
+
+  return SDValue();
+}
+
 SDValue DAGCombiner::visitFP_ROUND(SDNode *N) {
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
index 95f1812178..1c429546a8 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
@@ -2198,6 +2198,7 @@ bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) {
   // to use the promoted float operand.  Nodes that produce at least one
   // promotion-requiring floating point result have their operands legalized as
   // a part of PromoteFloatResult.
+  // clang-format off
   switch (N->getOpcode()) {
     default:
   #ifndef NDEBUG
@@ -2209,7 +2210,9 @@ bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) {
     case ISD::BITCAST:    R = PromoteFloatOp_BITCAST(N, OpNo); break;
     case ISD::FCOPYSIGN:  R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break;
     case ISD::FP_TO_SINT:
-    case ISD::FP_TO_UINT: R = PromoteFloatOp_FP_TO_XINT(N, OpNo); break;
+    case ISD::FP_TO_UINT:
+    case ISD::LRINT:
+    case ISD::LLRINT:     R = PromoteFloatOp_UnaryOp(N, OpNo); break;
     case ISD::FP_TO_SINT_SAT:
     case ISD::FP_TO_UINT_SAT:
                           R = PromoteFloatOp_FP_TO_XINT_SAT(N, OpNo); break;
@@ -2218,6 +2221,7 @@ bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) {
     case ISD::SETCC:      R = PromoteFloatOp_SETCC(N, OpNo); break;
     case ISD::STORE:      R = PromoteFloatOp_STORE(N, OpNo); break;
   }
+  // clang-format on
 
   if (R.getNode())
     ReplaceValueWith(SDValue(N, 0), R);
@@ -2251,7 +2255,7 @@ SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) {
 }
 
 // Convert the promoted float value to the desired integer type
-SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT(SDNode *N, unsigned OpNo) {
+SDValue DAGTypeLegalizer::PromoteFloatOp_UnaryOp(SDNode *N, unsigned OpNo) {
   SDValue Op = GetPromotedFloat(N->getOperand(0));
   return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op);
 }
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index c802604a34..3372626775 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -711,7 +711,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
   SDValue PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo);
   SDValue PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo);
   SDValue PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo);
-  SDValue PromoteFloatOp_FP_TO_XINT(SDNode *N, unsigned OpNo);
+  SDValue PromoteFloatOp_UnaryOp(SDNode *N, unsigned OpNo);
   SDValue PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N, unsigned OpNo);
   SDValue PromoteFloatOp_STORE(SDNode *N, unsigned OpNo);
   SDValue PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index ddc3b94e9c..b3f1e5d5a8 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -402,6 +402,8 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
   case ISD::FCEIL:
   case ISD::FTRUNC:
   case ISD::FRINT:
+  case ISD::LRINT:
+  case ISD::LLRINT:
   case ISD::FNEARBYINT:
   case ISD::FROUND:
   case ISD::FROUNDEVEN:
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 1bb6fbbf06..2c5343c3c4 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -101,6 +101,8 @@ void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
   case ISD::FP_TO_SINT:
   case ISD::FP_TO_UINT:
   case ISD::FRINT:
+  case ISD::LRINT:
+  case ISD::LLRINT:
   case ISD::FROUND:
   case ISD::FROUNDEVEN:
   case ISD::FSIN:
@@ -681,6 +683,8 @@ bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
   case ISD::FP_TO_UINT:
   case ISD::SINT_TO_FP:
   case ISD::UINT_TO_FP:
+  case ISD::LRINT:
+  case ISD::LLRINT:
     Res = ScalarizeVecOp_UnaryOp(N);
     break;
   case ISD::STRICT_SINT_TO_FP:
@@ -1097,6 +1101,8 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
   case ISD::VP_FP_TO_UINT:
   case ISD::FRINT:
   case ISD::VP_FRINT:
+  case ISD::LRINT:
+  case ISD::LLRINT:
   case ISD::FROUND:
   case ISD::VP_FROUND:
   case ISD::FROUNDEVEN:
@@ -2974,6 +2980,8 @@ bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
   case ISD::ZERO_EXTEND:
   case ISD::ANY_EXTEND:
   case ISD::FTRUNC:
+  case ISD::LRINT:
+  case ISD::LLRINT:
     Res = SplitVecOp_UnaryOp(N);
     break;
   case ISD::FLDEXP:
@@ -4209,6 +4217,8 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
   case ISD::FLOG2:
   case ISD::FNEARBYINT:
   case ISD::FRINT:
+  case ISD::LRINT:
+  case ISD::LLRINT:
   case ISD::FROUND:
   case ISD::FROUNDEVEN:
   case ISD::FSIN:
@@ -5958,7 +5968,11 @@ bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
   case ISD::STRICT_FSETCCS:     Res = WidenVecOp_STRICT_FSETCC(N); break;
   case ISD::VSELECT:            Res = WidenVecOp_VSELECT(N); break;
   case ISD::FLDEXP:
-  case ISD::FCOPYSIGN:          Res = WidenVecOp_UnrollVectorOp(N); break;
+  case ISD::FCOPYSIGN:
+  case ISD::LRINT:
+  case ISD::LLRINT:
+    Res = WidenVecOp_UnrollVectorOp(N);
+    break;
   case ISD::IS_FPCLASS:         Res = WidenVecOp_IS_FPCLASS(N); break;
 
   case ISD::ANY_EXTEND:
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index e831316eff..1ee2f03373 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -161,8 +161,13 @@ bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) {
   unsigned SplatBitSize;
   bool HasUndefs;
   unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
+  // Endianness does not matter here. We are checking for a splat given the
+  // element size of the vector, and if we find such a splat for little endian
+  // layout, then that should be valid also for big endian (as the full vector
+  // size is known to be a multiple of the element size).
+  const bool IsBigEndian = false;
   return BV->isConstantSplat(SplatVal, SplatUndef, SplatBitSize, HasUndefs,
-                             EltSize) &&
+                             EltSize, IsBigEndian) &&
          EltSize == SplatBitSize;
 }
 
@@ -3736,14 +3741,19 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
     assert(Op.getResNo() == 0 &&
            "We only compute knownbits for the difference here.");
 
-    // TODO: Compute influence of the carry operand.
-    if (Opcode == ISD::USUBO_CARRY || Opcode == ISD::SSUBO_CARRY)
-      break;
+    // With USUBO_CARRY and SSUBO_CARRY a borrow bit may be added in.
+    KnownBits Borrow(1);
+    if (Opcode == ISD::USUBO_CARRY || Opcode == ISD::SSUBO_CARRY) {
+      Borrow = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1);
+      // Borrow has bit width 1
+      Borrow = Borrow.trunc(1);
+    } else {
+      Borrow.setAllZero();
+    }
 
     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
     Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
-    Known = KnownBits::computeForAddSub(/* Add */ false, /* NSW */ false,
-                                        Known, Known2);
+    Known = KnownBits::computeForSubBorrow(Known, Known2, Borrow);
     break;
   }
   case ISD::UADDO:
@@ -3768,15 +3778,13 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
     if (Opcode == ISD::ADDE)
       // Can't track carry from glue, set carry to unknown.
       Carry.resetAll();
-    else if (Opcode == ISD::UADDO_CARRY || Opcode == ISD::SADDO_CARRY)
-      // TODO: Compute known bits for the carry operand. Not sure if it is worth
-      // the trouble (how often will we find a known carry bit). And I haven't
-      // tested this very much yet, but something like this might work:
-      //   Carry = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1);
-      //   Carry = Carry.zextOrTrunc(1, false);
-      Carry.resetAll();
-    else
+    else if (Opcode == ISD::UADDO_CARRY || Opcode == ISD::SADDO_CARRY) {
+      Carry = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1);
+      // Carry has bit width 1
+      Carry = Carry.trunc(1);
+    } else {
       Carry.setAllZero();
+    }
 
     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
     Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
@@ -5003,8 +5011,6 @@ bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
 
   unsigned Opcode = Op.getOpcode();
   switch (Opcode) {
-  case ISD::AssertSext:
-  case ISD::AssertZext:
   case ISD::FREEZE:
   case ISD::CONCAT_VECTORS:
   case ISD::INSERT_SUBVECTOR:
@@ -5127,6 +5133,8 @@ bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const
   case ISD::FROUND:
   case ISD::FROUNDEVEN:
   case ISD::FRINT:
+  case ISD::LRINT:
+  case ISD::LLRINT:
   case ISD::FNEARBYINT:
   case ISD::FLDEXP: {
     if (SNaN)
@@ -6231,7 +6239,7 @@ SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
 
   // Handle binops special cases.
   if (NumOps == 2) {
-    if (SDValue CFP = foldConstantFPMath(Opcode, DL, VT, Ops[0], Ops[1]))
+    if (SDValue CFP = foldConstantFPMath(Opcode, DL, VT, Ops))
       return CFP;
 
     if (auto *C1 = dyn_cast<ConstantSDNode>(Ops[0])) {
@@ -6424,11 +6432,17 @@ SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
 }
 
 SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL,
-                                         EVT VT, SDValue N1, SDValue N2) {
+                                         EVT VT, ArrayRef<SDValue> Ops) {
+  // TODO: Add support for unary/ternary fp opcodes.
+  if (Ops.size() != 2)
+    return SDValue();
+
   // TODO: We don't do any constant folding for strict FP opcodes here, but we
   //       should. That will require dealing with a potentially non-default
   //       rounding mode, checking the "opStatus" return value from the APFloat
   //       math calculations, and possibly other variations.
+  SDValue N1 = Ops[0];
+  SDValue N2 = Ops[1];
   ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N1, /*AllowUndefs*/ false);
   ConstantFPSDNode *N2CFP = isConstOrConstSplatFP(N2, /*AllowUndefs*/ false);
   if (N1CFP && N2CFP) {
@@ -7064,7 +7078,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
          "Operand is DELETED_NODE!");
   // Perform various simplifications.
   switch (Opcode) {
-  case ISD::FMA: {
+  case ISD::FMA:
+  case ISD::FMAD: {
     assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
     assert(N1.getValueType() == VT && N2.getValueType() == VT &&
            N3.getValueType() == VT && "FMA types must match!");
@@ -7075,7 +7090,11 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
       APFloat  V1 = N1CFP->getValueAPF();
       const APFloat &V2 = N2CFP->getValueAPF();
       const APFloat &V3 = N3CFP->getValueAPF();
-      V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven);
+      if (Opcode == ISD::FMAD) {
+        V1.multiply(V2, APFloat::rmNearestTiesToEven);
+        V1.add(V3, APFloat::rmNearestTiesToEven);
+      } else
+        V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven);
       return getConstantFP(V1, DL, VT);
     }
     break;
@@ -9854,6 +9873,27 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
       SDValue ZeroOverFlow = getConstant(0, DL, VTList.VTs[1]);
       return getNode(ISD::MERGE_VALUES, DL, VTList, {N1, ZeroOverFlow}, Flags);
     }
+
+    if (VTList.VTs[0].isVector() &&
+        VTList.VTs[0].getVectorElementType() == MVT::i1 &&
+        VTList.VTs[1].getVectorElementType() == MVT::i1) {
+      SDValue F1 = getFreeze(N1);
+      SDValue F2 = getFreeze(N2);
+      // {vXi1,vXi1} (u/s)addo(vXi1 x, vXi1y) -> {xor(x,y),and(x,y)}
+      if (Opcode == ISD::UADDO || Opcode == ISD::SADDO)
+        return getNode(ISD::MERGE_VALUES, DL, VTList,
+                       {getNode(ISD::XOR, DL, VTList.VTs[0], F1, F2),
+                        getNode(ISD::AND, DL, VTList.VTs[1], F1, F2)},
+                       Flags);
+      // {vXi1,vXi1} (u/s)subo(vXi1 x, vXi1y) -> {xor(x,y),and(~x,y)}
+      if (Opcode == ISD::USUBO || Opcode == ISD::SSUBO) {
+        SDValue NotF1 = getNOT(DL, F1, VTList.VTs[0]);
+        return getNode(ISD::MERGE_VALUES, DL, VTList,
+                       {getNode(ISD::XOR, DL, VTList.VTs[0], F1, F2),
+                        getNode(ISD::AND, DL, VTList.VTs[1], NotF1, F2)},
+                       Flags);
+      }
+    }
     break;
   }
   case ISD::SMUL_LOHI:
@@ -12357,6 +12397,10 @@ bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
 
   // FIXME: This does not work for vectors with elements less than 8 bits.
   while (VecWidth > 8) {
+    // If we can't split in half, stop here.
+    if (VecWidth & 1)
+      break;
+
     unsigned HalfSize = VecWidth / 2;
     APInt HighValue = SplatValue.extractBits(HalfSize, HalfSize);
     APInt LowValue = SplatValue.extractBits(HalfSize, 0);
@@ -12374,6 +12418,12 @@ bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
     VecWidth = HalfSize;
   }
 
+  // FIXME: The loop above only tries to split in halves. But if the input
+  // vector for example is <3 x i16> it wouldn't be able to detect a
+  // SplatBitSize of 16. No idea if that is a design flaw currently limiting
+  // optimizations. I guess that back in the days when this helper was created
+  // vectors normally was power-of-2 sized.
+
   SplatBitSize = VecWidth;
   return true;
 }
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index c5fd56795a..eabc76334f 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -1738,6 +1738,12 @@ SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
     if (const auto *NC = dyn_cast<NoCFIValue>(C))
       return getValue(NC->getGlobalValue());
 
+    if (VT == MVT::aarch64svcount) {
+      assert(C->isNullValue() && "Can only zero this target type!");
+      return DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT,
+                         DAG.getConstant(0, getCurSDLoc(), MVT::nxv16i1));
+    }
+
     VectorType *VecTy = cast<VectorType>(V->getType());
 
     // Now that we know the number and type of the elements, get that number of
@@ -9728,7 +9734,7 @@ SDValue SelectionDAGBuilder::lowerRangeToAssertZExt(SelectionDAG &DAG,
 void SelectionDAGBuilder::populateCallLoweringInfo(
     TargetLowering::CallLoweringInfo &CLI, const CallBase *Call,
     unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy,
-    bool IsPatchPoint) {
+    AttributeSet RetAttrs, bool IsPatchPoint) {
   TargetLowering::ArgListTy Args;
   Args.reserve(NumArgs);
 
@@ -9749,7 +9755,8 @@ void SelectionDAGBuilder::populateCallLoweringInfo(
 
   CLI.setDebugLoc(getCurSDLoc())
       .setChain(getRoot())
-      .setCallee(Call->getCallingConv(), ReturnTy, Callee, std::move(Args))
+      .setCallee(Call->getCallingConv(), ReturnTy, Callee, std::move(Args),
+                 RetAttrs)
       .setDiscardResult(Call->use_empty())
       .setIsPatchPoint(IsPatchPoint)
       .setIsPreallocated(
@@ -9898,7 +9905,7 @@ void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB,
 
   TargetLowering::CallLoweringInfo CLI(DAG);
   populateCallLoweringInfo(CLI, &CB, NumMetaOpers, NumCallArgs, Callee,
-                           ReturnTy, true);
+                           ReturnTy, CB.getAttributes().getRetAttrs(), true);
   std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
 
   SDNode *CallEnd = Result.second.getNode();
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
index ec23445b01..a97884f0ef 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
@@ -426,7 +426,8 @@ class SelectionDAGBuilder {
   void populateCallLoweringInfo(TargetLowering::CallLoweringInfo &CLI,
                                 const CallBase *Call, unsigned ArgIdx,
                                 unsigned NumArgs, SDValue Callee,
-                                Type *ReturnTy, bool IsPatchPoint);
+                                Type *ReturnTy, AttributeSet RetAttrs,
+                                bool IsPatchPoint);
 
   std::pair<SDValue, SDValue>
   lowerInvokable(TargetLowering::CallLoweringInfo &CLI,
diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
index 5ea05028ce..cf32350036 100644
--- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
@@ -62,15 +62,15 @@ STATISTIC(NumOfStatepoints, "Number of statepoint nodes encountered");
 STATISTIC(StatepointMaxSlotsRequired,
           "Maximum number of stack slots required for a singe statepoint");
 
-cl::opt<bool> UseRegistersForDeoptValues(
+static cl::opt<bool> UseRegistersForDeoptValues(
     "use-registers-for-deopt-values", cl::Hidden, cl::init(false),
     cl::desc("Allow using registers for non pointer deopt args"));
 
-cl::opt<bool> UseRegistersForGCPointersInLandingPad(
+static cl::opt<bool> UseRegistersForGCPointersInLandingPad(
     "use-registers-for-gc-values-in-landing-pad", cl::Hidden, cl::init(false),
     cl::desc("Allow using registers for gc pointer in landing pad"));
 
-cl::opt<unsigned> MaxRegistersForGCPointers(
+static cl::opt<unsigned> MaxRegistersForGCPointers(
     "max-registers-for-gc-values", cl::Hidden, cl::init(0),
     cl::desc("Max number of VRegs allowed to pass GC pointer meta args in"));
 
@@ -1033,10 +1033,16 @@ SelectionDAGBuilder::LowerStatepoint(const GCStatepointInst &I,
     ActualCallee = Callee;
   }
 
+  const auto GCResultLocality = getGCResultLocality(I);
+  AttributeSet retAttrs;
+  if (GCResultLocality.first)
+    retAttrs = GCResultLocality.first->getAttributes().getRetAttrs();
+
   StatepointLoweringInfo SI(DAG);
   populateCallLoweringInfo(SI.CLI, &I, GCStatepointInst::CallArgsBeginPos,
                            I.getNumCallArgs(), ActualCallee,
-                           I.getActualReturnType(), false /* IsPatchPoint */);
+                           I.getActualReturnType(), retAttrs,
+                           /*IsPatchPoint=*/false);
 
   // There may be duplication in the gc.relocate list; such as two copies of
   // each relocation on normal and exceptional path for an invoke.  We only
@@ -1092,8 +1098,6 @@ SelectionDAGBuilder::LowerStatepoint(const GCStatepointInst &I,
   SDValue ReturnValue = LowerAsSTATEPOINT(SI);
 
   // Export the result value if needed
-  const auto GCResultLocality = getGCResultLocality(I);
-
   if (!GCResultLocality.first && !GCResultLocality.second) {
     // The return value is not needed, just generate a poison value.
     // Note: This covers the void return case.
@@ -1138,7 +1142,7 @@ void SelectionDAGBuilder::LowerCallSiteWithDeoptBundleImpl(
   populateCallLoweringInfo(
       SI.CLI, Call, ArgBeginIndex, Call->arg_size(), Callee,
       ForceVoidReturnTy ? Type::getVoidTy(*DAG.getContext()) : Call->getType(),
-      false);
+      Call->getAttributes().getRetAttrs(), /*IsPatchPoint=*/false);
   if (!VarArgDisallowed)
     SI.CLI.IsVarArg = Call->getFunctionType()->isVarArg();
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index c0e88051dc..8b4f315949 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -1897,7 +1897,7 @@ bool TargetLowering::SimplifyDemandedBits(
         if (isNarrowingProfitable(VT, HalfVT) &&
             isTypeDesirableForOp(ISD::SRL, HalfVT) &&
             isTruncateFree(VT, HalfVT) && isZExtFree(HalfVT, VT) &&
-            (!TLO.LegalOperations() || isOperationLegal(ISD::SRL, VT)) &&
+            (!TLO.LegalOperations() || isOperationLegal(ISD::SRL, HalfVT)) &&
             ((InDemandedMask.countLeadingZeros() >= (BitWidth / 2)) ||
              TLO.DAG.MaskedValueIsZero(Op0, HiBits))) {
           SDValue NewOp = TLO.DAG.getNode(ISD::TRUNCATE, dl, HalfVT, Op0);
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index 3e4bff5ddc..99eadf4bb9 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -873,13 +873,13 @@ void TargetLoweringBase::initActions() {
 
     // These operations default to expand for vector types.
     if (VT.isVector())
-      setOperationAction({ISD::FCOPYSIGN, ISD::SIGN_EXTEND_INREG,
-                          ISD::ANY_EXTEND_VECTOR_INREG,
-                          ISD::SIGN_EXTEND_VECTOR_INREG,
-                          ISD::ZERO_EXTEND_VECTOR_INREG, ISD::SPLAT_VECTOR},
-                         VT, Expand);
+      setOperationAction(
+          {ISD::FCOPYSIGN, ISD::SIGN_EXTEND_INREG, ISD::ANY_EXTEND_VECTOR_INREG,
+           ISD::SIGN_EXTEND_VECTOR_INREG, ISD::ZERO_EXTEND_VECTOR_INREG,
+           ISD::SPLAT_VECTOR, ISD::LRINT, ISD::LLRINT},
+          VT, Expand);
 
-    // Constrained floating-point operations default to expand.
+      // Constrained floating-point operations default to expand.
 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN)               \
     setOperationAction(ISD::STRICT_##DAGN, VT, Expand);
 #include "llvm/IR/ConstrainedOps.def"
diff --git a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
index 6210e7fc12..f3ba380818 100644
--- a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
@@ -763,6 +763,25 @@ calcUniqueIDUpdateFlagsAndSize(const GlobalObject *GO, StringRef SectionName,
   return NextUniqueID++;
 }
 
+static std::tuple<StringRef, bool, unsigned>
+getGlobalObjectInfo(const GlobalObject *GO, const TargetMachine &TM) {
+  StringRef Group = "";
+  bool IsComdat = false;
+  unsigned Flags = 0;
+  if (const Comdat *C = getELFComdat(GO)) {
+    Flags |= ELF::SHF_GROUP;
+    Group = C->getName();
+    IsComdat = C->getSelectionKind() == Comdat::Any;
+  }
+  if (auto *GV = dyn_cast<GlobalVariable>(GO)) {
+    if (TM.isLargeData(GV)) {
+      assert(TM.getTargetTriple().getArch() == Triple::x86_64);
+      Flags |= ELF::SHF_X86_64_LARGE;
+    }
+  }
+  return {Group, IsComdat, Flags};
+}
+
 static MCSection *selectExplicitSectionGlobal(
     const GlobalObject *GO, SectionKind Kind, const TargetMachine &TM,
     MCContext &Ctx, Mangler &Mang, unsigned &NextUniqueID,
@@ -793,14 +812,9 @@ static MCSection *selectExplicitSectionGlobal(
   // Infer section flags from the section name if we can.
   Kind = getELFKindForNamedSection(SectionName, Kind);
 
-  StringRef Group = "";
-  bool IsComdat = false;
   unsigned Flags = getELFSectionFlags(Kind);
-  if (const Comdat *C = getELFComdat(GO)) {
-    Group = C->getName();
-    IsComdat = C->getSelectionKind() == Comdat::Any;
-    Flags |= ELF::SHF_GROUP;
-  }
+  auto [Group, IsComdat, ExtraFlags] = getGlobalObjectInfo(GO, TM);
+  Flags |= ExtraFlags;
 
   unsigned EntrySize = getEntrySizeForKind(Kind);
   const unsigned UniqueID = calcUniqueIDUpdateFlagsAndSize(
@@ -848,19 +862,8 @@ static MCSectionELF *selectELFSectionForGlobal(
     const TargetMachine &TM, bool EmitUniqueSection, unsigned Flags,
     unsigned *NextUniqueID, const MCSymbolELF *AssociatedSymbol) {
 
-  StringRef Group = "";
-  bool IsComdat = false;
-  if (const Comdat *C = getELFComdat(GO)) {
-    Flags |= ELF::SHF_GROUP;
-    Group = C->getName();
-    IsComdat = C->getSelectionKind() == Comdat::Any;
-  }
-  if (auto *GV = dyn_cast<GlobalVariable>(GO)) {
-    if (TM.isLargeData(GV)) {
-      assert(TM.getTargetTriple().getArch() == Triple::x86_64);
-      Flags |= ELF::SHF_X86_64_LARGE;
-    }
-  }
+  auto [Group, IsComdat, ExtraFlags] = getGlobalObjectInfo(GO, TM);
+  Flags |= ExtraFlags;
 
   // Get the section entry size based on the kind.
   unsigned EntrySize = getEntrySizeForKind(Kind);
diff --git a/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp b/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp
index 4ae396723e..d3f58ae310 100644
--- a/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp
+++ b/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp
@@ -125,6 +125,7 @@ class TwoAddressInstructionPass : public MachineFunctionPass {
   bool isCopyToReg(MachineInstr &MI, Register &SrcReg, Register &DstReg,
                    bool &IsSrcPhys, bool &IsDstPhys) const;
 
+  bool isPlainlyKilled(const MachineInstr *MI, LiveRange &LR) const;
   bool isPlainlyKilled(const MachineInstr *MI, Register Reg) const;
   bool isPlainlyKilled(const MachineOperand &MO) const;
 
@@ -305,27 +306,37 @@ bool TwoAddressInstructionPass::isCopyToReg(MachineInstr &MI, Register &SrcReg,
   return true;
 }
 
+bool TwoAddressInstructionPass::isPlainlyKilled(const MachineInstr *MI,
+                                                LiveRange &LR) const {
+  // This is to match the kill flag version where undefs don't have kill flags.
+  if (!LR.hasAtLeastOneValue())
+    return false;
+
+  SlotIndex useIdx = LIS->getInstructionIndex(*MI);
+  LiveInterval::const_iterator I = LR.find(useIdx);
+  assert(I != LR.end() && "Reg must be live-in to use.");
+  return !I->end.isBlock() && SlotIndex::isSameInstr(I->end, useIdx);
+}
+
 /// Test if the given register value, which is used by the
 /// given instruction, is killed by the given instruction.
 bool TwoAddressInstructionPass::isPlainlyKilled(const MachineInstr *MI,
                                                 Register Reg) const {
-  if (LIS && Reg.isVirtual() && !LIS->isNotInMIMap(*MI)) {
-    // FIXME: Sometimes tryInstructionTransform() will add instructions and
-    // test whether they can be folded before keeping them. In this case it
-    // sets a kill before recursively calling tryInstructionTransform() again.
-    // If there is no interval available, we assume that this instruction is
-    // one of those. A kill flag is manually inserted on the operand so the
-    // check below will handle it.
-    LiveInterval &LI = LIS->getInterval(Reg);
-    // This is to match the kill flag version where undefs don't have kill
-    // flags.
-    if (!LI.hasAtLeastOneValue())
+  // FIXME: Sometimes tryInstructionTransform() will add instructions and
+  // test whether they can be folded before keeping them. In this case it
+  // sets a kill before recursively calling tryInstructionTransform() again.
+  // If there is no interval available, we assume that this instruction is
+  // one of those. A kill flag is manually inserted on the operand so the
+  // check below will handle it.
+  if (LIS && !LIS->isNotInMIMap(*MI)) {
+    if (Reg.isVirtual())
+      return isPlainlyKilled(MI, LIS->getInterval(Reg));
+    // Reserved registers are considered always live.
+    if (MRI->isReserved(Reg))
       return false;
-
-    SlotIndex useIdx = LIS->getInstructionIndex(*MI);
-    LiveInterval::const_iterator I = LI.find(useIdx);
-    assert(I != LI.end() && "Reg must be live-in to use.");
-    return !I->end.isBlock() && SlotIndex::isSameInstr(I->end, useIdx);
+    return all_of(TRI->regunits(Reg), [&](MCRegUnit U) {
+      return isPlainlyKilled(MI, LIS->getRegUnit(U));
+    });
   }
 
   return MI->killsRegister(Reg);
diff --git a/llvm/lib/CodeGen/TypePromotion.cpp b/llvm/lib/CodeGen/TypePromotion.cpp
index 51f77e5fd8..053caf518b 100644
--- a/llvm/lib/CodeGen/TypePromotion.cpp
+++ b/llvm/lib/CodeGen/TypePromotion.cpp
@@ -1016,11 +1016,8 @@ bool TypePromotionLegacy::runOnFunction(Function &F) {
   if (skipFunction(F))
     return false;
 
-  auto *TPC = getAnalysisIfAvailable<TargetPassConfig>();
-  if (!TPC)
-    return false;
-
-  auto *TM = &TPC->getTM<TargetMachine>();
+  auto &TPC = getAnalysis<TargetPassConfig>();
+  auto *TM = &TPC.getTM<TargetMachine>();
   auto &TTI = getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
   auto &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
 
diff --git a/llvm/lib/CodeGen/VirtRegMap.cpp b/llvm/lib/CodeGen/VirtRegMap.cpp
index c1c6ce2273..48f4ee29fb 100644
--- a/llvm/lib/CodeGen/VirtRegMap.cpp
+++ b/llvm/lib/CodeGen/VirtRegMap.cpp
@@ -261,7 +261,7 @@ bool VirtRegRewriter::runOnMachineFunction(MachineFunction &fn) {
   Indexes = &getAnalysis<SlotIndexes>();
   LIS = &getAnalysis<LiveIntervals>();
   VRM = &getAnalysis<VirtRegMap>();
-  DebugVars = getAnalysisIfAvailable<LiveDebugVariables>();
+  DebugVars = &getAnalysis<LiveDebugVariables>();
   LLVM_DEBUG(dbgs() << "********** REWRITE VIRTUAL REGISTERS **********\n"
                     << "********** Function: " << MF->getName() << '\n');
   LLVM_DEBUG(VRM->dump());
@@ -275,7 +275,7 @@ bool VirtRegRewriter::runOnMachineFunction(MachineFunction &fn) {
   // Rewrite virtual registers.
   rewrite();
 
-  if (DebugVars && ClearVirtRegs) {
+  if (ClearVirtRegs) {
     // Write out new DBG_VALUE instructions.
 
     // We only do this if ClearVirtRegs is specified since this should be the
@@ -312,7 +312,7 @@ void VirtRegRewriter::addLiveInsForSubRanges(const LiveInterval &LI,
 
   // Check all mbb start positions between First and Last while
   // simultaneously advancing an iterator for each subrange.
-  for (SlotIndexes::MBBIndexIterator MBBI = Indexes->findMBBIndex(First);
+  for (SlotIndexes::MBBIndexIterator MBBI = Indexes->getMBBLowerBound(First);
        MBBI != Indexes->MBBIndexEnd() && MBBI->first <= Last; ++MBBI) {
     SlotIndex MBBBegin = MBBI->first;
     // Advance all subrange iterators so that their end position is just
@@ -363,7 +363,7 @@ void VirtRegRewriter::addMBBLiveIns() {
       // sorted by slot indexes.
       SlotIndexes::MBBIndexIterator I = Indexes->MBBIndexBegin();
       for (const auto &Seg : LI) {
-        I = Indexes->advanceMBBIndex(I, Seg.start);
+        I = Indexes->getMBBLowerBound(I, Seg.start);
         for (; I != Indexes->MBBIndexEnd() && I->first < Seg.end; ++I) {
           MachineBasicBlock *MBB = I->second;
           MBB->addLiveIn(PhysReg);
diff --git a/llvm/lib/DWARFLinker/DWARFLinker.cpp b/llvm/lib/DWARFLinker/DWARFLinker.cpp
index 5ed4923dc0..2d8360f100 100644
--- a/llvm/lib/DWARFLinker/DWARFLinker.cpp
+++ b/llvm/lib/DWARFLinker/DWARFLinker.cpp
@@ -2682,12 +2682,12 @@ Error DWARFLinker::link() {
       continue;
     }
 
-    // In a first phase, just read in the debug info and load all clang modules.
+    // Clone all the clang modules with requires extracting the DIE units. We
+    // don't need the full debug info until the Analyze phase.
     OptContext.CompileUnits.reserve(
         OptContext.File.Dwarf->getNumCompileUnits());
-
     for (const auto &CU : OptContext.File.Dwarf->compile_units()) {
-      auto CUDie = CU->getUnitDIE(false);
+      auto CUDie = CU->getUnitDIE(/*ExtractUnitDIEOnly=*/true);
       if (Options.Verbose) {
         outs() << "Input compilation unit:";
         DIDumpOptions DumpOpts;
@@ -2728,9 +2728,9 @@ Error DWARFLinker::link() {
       return;
 
     for (const auto &CU : Context.File.Dwarf->compile_units()) {
-      // The !isClangModuleRef condition effectively skips over fully resolved
-      // skeleton units.
-      auto CUDie = CU->getUnitDIE();
+      // Previously we only extracted the unit DIEs. We need the full debug info
+      // now.
+      auto CUDie = CU->getUnitDIE(/*ExtractUnitDIEOnly=*/false);
       std::string PCMFile = getPCMFile(CUDie, Options.ObjectPrefixMap);
 
       if (!CUDie || LLVM_UNLIKELY(Options.Update) ||
diff --git a/llvm/lib/DWARFLinkerParallel/DWARFLinkerCompileUnit.h b/llvm/lib/DWARFLinkerParallel/DWARFLinkerCompileUnit.h
index 3f9ae4ba5d..54666f684f 100644
--- a/llvm/lib/DWARFLinkerParallel/DWARFLinkerCompileUnit.h
+++ b/llvm/lib/DWARFLinkerParallel/DWARFLinkerCompileUnit.h
@@ -55,7 +55,7 @@ class CompileUnit : public DwarfUnit {
   CompileUnit(LinkingGlobalData &GlobalData, unsigned ID,
               StringRef ClangModuleName, DWARFFile &File,
               OffsetToUnitTy UnitFromOffset, dwarf::FormParams Format,
-              support::endianness Endianess)
+              llvm::endianness Endianess)
       : DwarfUnit(GlobalData, ID, ClangModuleName), File(File),
         getUnitFromOffset(UnitFromOffset), Stage(Stage::CreatedNotLoaded) {
     UnitName = File.FileName;
@@ -65,7 +65,7 @@ class CompileUnit : public DwarfUnit {
   CompileUnit(LinkingGlobalData &GlobalData, DWARFUnit &OrigUnit, unsigned ID,
               StringRef ClangModuleName, DWARFFile &File,
               OffsetToUnitTy UnitFromOffset, dwarf::FormParams Format,
-              support::endianness Endianess)
+              llvm::endianness Endianess)
       : DwarfUnit(GlobalData, ID, ClangModuleName), File(File),
         OrigUnit(&OrigUnit), getUnitFromOffset(UnitFromOffset),
         Stage(Stage::CreatedNotLoaded) {
diff --git a/llvm/lib/DWARFLinkerParallel/DWARFLinkerImpl.cpp b/llvm/lib/DWARFLinkerParallel/DWARFLinkerImpl.cpp
index a02326f025..78f553a125 100644
--- a/llvm/lib/DWARFLinkerParallel/DWARFLinkerImpl.cpp
+++ b/llvm/lib/DWARFLinkerParallel/DWARFLinkerImpl.cpp
@@ -64,12 +64,12 @@ Error DWARFLinkerImpl::link() {
 
   dwarf::FormParams GlobalFormat = {GlobalData.getOptions().TargetDWARFVersion,
                                     0, dwarf::DwarfFormat::DWARF32};
-  support::endianness GlobalEndianness = llvm::endianness::native;
+  llvm::endianness GlobalEndianness = llvm::endianness::native;
 
   if (TheDwarfEmitter) {
     GlobalEndianness = TheDwarfEmitter->getTargetTriple().isLittleEndian()
-                           ? support::endianness::little
-                           : support::endianness::big;
+                           ? llvm::endianness::little
+                           : llvm::endianness::big;
   }
 
   for (std::unique_ptr<LinkContext> &Context : ObjectContexts) {
diff --git a/llvm/lib/DWARFLinkerParallel/DWARFLinkerImpl.h b/llvm/lib/DWARFLinkerParallel/DWARFLinkerImpl.h
index f6fc617d30..45dc506b99 100644
--- a/llvm/lib/DWARFLinkerParallel/DWARFLinkerImpl.h
+++ b/llvm/lib/DWARFLinkerParallel/DWARFLinkerImpl.h
@@ -227,8 +227,8 @@ class DWARFLinkerImpl : public DWARFLinker {
         // Set context format&endianness based on the input file.
         Format.Version = File.Dwarf->getMaxVersion();
         Format.AddrSize = File.Dwarf->getCUAddrSize();
-        Endianness = File.Dwarf->isLittleEndian() ? support::endianness::little
-                                                  : support::endianness::big;
+        Endianness = File.Dwarf->isLittleEndian() ? llvm::endianness::little
+                                                  : llvm::endianness::big;
       }
     }
 
diff --git a/llvm/lib/DWARFLinkerParallel/OutputSections.h b/llvm/lib/DWARFLinkerParallel/OutputSections.h
index f34ac25ee3..b101ac6193 100644
--- a/llvm/lib/DWARFLinkerParallel/OutputSections.h
+++ b/llvm/lib/DWARFLinkerParallel/OutputSections.h
@@ -139,7 +139,7 @@ struct SectionDescriptor {
   friend OutputSections;
 
   SectionDescriptor(DebugSectionKind SectionKind, LinkingGlobalData &GlobalData,
-                    dwarf::FormParams Format, support::endianness Endianess)
+                    dwarf::FormParams Format, llvm::endianness Endianess)
       : OS(Contents), GlobalData(GlobalData), SectionKind(SectionKind),
         Format(Format), Endianess(Endianess) {
     ListDebugStrPatch.setAllocator(&GlobalData.getAllocator());
@@ -247,7 +247,7 @@ struct SectionDescriptor {
   const StringLiteral &getName() const { return getSectionName(SectionKind); }
 
   /// Returns endianess used by section.
-  support::endianness getEndianess() const { return Endianess; }
+  llvm::endianness getEndianess() const { return Endianess; }
 
   /// Returns FormParams used by section.
   dwarf::FormParams getFormParams() const { return Format; }
@@ -266,8 +266,7 @@ struct SectionDescriptor {
   void applySLEB128(uint64_t PatchOffset, uint64_t Val);
 
   /// Sets output format.
-  void setOutputFormat(dwarf::FormParams Format,
-                       support::endianness Endianess) {
+  void setOutputFormat(dwarf::FormParams Format, llvm::endianness Endianess) {
     this->Format = Format;
     this->Endianess = Endianess;
   }
@@ -288,7 +287,7 @@ struct SectionDescriptor {
 
   /// Output format.
   dwarf::FormParams Format = {4, 4, dwarf::DWARF32};
-  support::endianness Endianess = support::endianness::little;
+  llvm::endianness Endianess = llvm::endianness::little;
 };
 
 /// This class keeps contents and offsets to the debug sections. Any objects
@@ -299,8 +298,7 @@ class OutputSections {
   OutputSections(LinkingGlobalData &GlobalData) : GlobalData(GlobalData) {}
 
   /// Sets output format for all keeping sections.
-  void setOutputFormat(dwarf::FormParams Format,
-                       support::endianness Endianness) {
+  void setOutputFormat(dwarf::FormParams Format, llvm::endianness Endianness) {
     this->Format = Format;
     this->Endianness = Endianness;
   }
@@ -400,7 +398,7 @@ class OutputSections {
                     StringEntryToDwarfStringPoolEntryMap &DebugLineStrStrings);
 
   /// Endiannes for the sections.
-  support::endianness getEndianness() const { return Endianness; }
+  llvm::endianness getEndianness() const { return Endianness; }
 
   /// Return DWARF version.
   uint16_t getVersion() const { return Format.Version; }
@@ -432,7 +430,7 @@ class OutputSections {
   dwarf::FormParams Format = {4, 4, dwarf::DWARF32};
 
   /// Endiannes for sections.
-  support::endianness Endianness = llvm::endianness::native;
+  llvm::endianness Endianness = llvm::endianness::native;
 
   /// All keeping sections.
   using SectionsSetTy = std::map<DebugSectionKind, SectionDescriptor>;
diff --git a/llvm/lib/DebugInfo/BTF/BTFParser.cpp b/llvm/lib/DebugInfo/BTF/BTFParser.cpp
index d1ed5d097e..4bc0c94340 100644
--- a/llvm/lib/DebugInfo/BTF/BTFParser.cpp
+++ b/llvm/lib/DebugInfo/BTF/BTFParser.cpp
@@ -203,13 +203,12 @@ const BTF::CommonType VoidTypeInst = {0, BTF::BTF_KIND_UNKN << 24, {0}};
 //   `BTFParser::Types` vector and the process stops.
 Error BTFParser::parseTypesInfo(ParseContext &Ctx, uint64_t TypesInfoStart,
                                 StringRef RawData) {
-  using support::big;
-  using support::little;
   using support::endian::byte_swap;
 
   TypesBuffer = OwningArrayRef<uint8_t>(arrayRefFromStringRef(RawData));
   // Switch endianness if necessary.
-  endianness Endianness = Ctx.Obj.isLittleEndian() ? little : big;
+  endianness Endianness = Ctx.Obj.isLittleEndian() ? llvm::endianness::little
+                                                   : llvm::endianness::big;
   uint32_t *TypesBuffer32 = (uint32_t *)TypesBuffer.data();
   for (uint64_t I = 0; I < TypesBuffer.size() / 4; ++I)
     TypesBuffer32[I] = byte_swap(TypesBuffer32[I], Endianness);
diff --git a/llvm/lib/DebugInfo/CodeView/CVTypeVisitor.cpp b/llvm/lib/DebugInfo/CodeView/CVTypeVisitor.cpp
index 689c643a70..3cafa3a93a 100644
--- a/llvm/lib/DebugInfo/CodeView/CVTypeVisitor.cpp
+++ b/llvm/lib/DebugInfo/CodeView/CVTypeVisitor.cpp
@@ -185,7 +185,7 @@ Error CVTypeVisitor::visitFieldListMemberStream(BinaryStreamReader &Reader) {
 struct FieldListVisitHelper {
   FieldListVisitHelper(TypeVisitorCallbacks &Callbacks, ArrayRef<uint8_t> Data,
                        VisitorDataSource Source)
-      : Stream(Data, llvm::support::little), Reader(Stream),
+      : Stream(Data, llvm::endianness::little), Reader(Stream),
         Deserializer(Reader),
         Visitor((Source == VDS_BytesPresent) ? Pipeline : Callbacks) {
     if (Source == VDS_BytesPresent) {
diff --git a/llvm/lib/DebugInfo/CodeView/LazyRandomTypeCollection.cpp b/llvm/lib/DebugInfo/CodeView/LazyRandomTypeCollection.cpp
index 460f95d96a..2343386e03 100644
--- a/llvm/lib/DebugInfo/CodeView/LazyRandomTypeCollection.cpp
+++ b/llvm/lib/DebugInfo/CodeView/LazyRandomTypeCollection.cpp
@@ -69,13 +69,13 @@ void LazyRandomTypeCollection::reset(BinaryStreamReader &Reader,
 }
 
 void LazyRandomTypeCollection::reset(StringRef Data, uint32_t RecordCountHint) {
-  BinaryStreamReader Reader(Data, support::little);
+  BinaryStreamReader Reader(Data, llvm::endianness::little);
   reset(Reader, RecordCountHint);
 }
 
 void LazyRandomTypeCollection::reset(ArrayRef<uint8_t> Data,
                                      uint32_t RecordCountHint) {
-  BinaryStreamReader Reader(Data, support::little);
+  BinaryStreamReader Reader(Data, llvm::endianness::little);
   reset(Reader, RecordCountHint);
 }
 
diff --git a/llvm/lib/DebugInfo/CodeView/RecordName.cpp b/llvm/lib/DebugInfo/CodeView/RecordName.cpp
index 5fbbc4a5d4..e06b036ede 100644
--- a/llvm/lib/DebugInfo/CodeView/RecordName.cpp
+++ b/llvm/lib/DebugInfo/CodeView/RecordName.cpp
@@ -324,7 +324,7 @@ StringRef llvm::codeview::getSymbolName(CVSymbol Sym) {
   if (Sym.kind() == SymbolKind::S_CONSTANT) {
     // S_CONSTANT is preceded by an APSInt, which has a variable length.  So we
     // have to do a full deserialization.
-    BinaryStreamReader Reader(Sym.content(), llvm::support::little);
+    BinaryStreamReader Reader(Sym.content(), llvm::endianness::little);
     // The container doesn't matter for single records.
     SymbolRecordMapping Mapping(Reader, CodeViewContainer::ObjectFile);
     ConstantSym Const(SymbolKind::S_CONSTANT);
diff --git a/llvm/lib/DebugInfo/CodeView/RecordSerialization.cpp b/llvm/lib/DebugInfo/CodeView/RecordSerialization.cpp
index d76905df86..032704478f 100644
--- a/llvm/lib/DebugInfo/CodeView/RecordSerialization.cpp
+++ b/llvm/lib/DebugInfo/CodeView/RecordSerialization.cpp
@@ -103,7 +103,7 @@ Error llvm::codeview::consume(BinaryStreamReader &Reader, APSInt &Num) {
 
 Error llvm::codeview::consume(StringRef &Data, APSInt &Num) {
   ArrayRef<uint8_t> Bytes(Data.bytes_begin(), Data.bytes_end());
-  BinaryByteStream S(Bytes, llvm::support::little);
+  BinaryByteStream S(Bytes, llvm::endianness::little);
   BinaryStreamReader SR(S);
   auto EC = consume(SR, Num);
   Data = Data.take_back(SR.bytesRemaining());
@@ -129,7 +129,7 @@ Error llvm::codeview::consume(BinaryStreamReader &Reader, uint32_t &Item) {
 
 Error llvm::codeview::consume(StringRef &Data, uint32_t &Item) {
   ArrayRef<uint8_t> Bytes(Data.bytes_begin(), Data.bytes_end());
-  BinaryByteStream S(Bytes, llvm::support::little);
+  BinaryByteStream S(Bytes, llvm::endianness::little);
   BinaryStreamReader SR(S);
   auto EC = consume(SR, Item);
   Data = Data.take_back(SR.bytesRemaining());
diff --git a/llvm/lib/DebugInfo/CodeView/SimpleTypeSerializer.cpp b/llvm/lib/DebugInfo/CodeView/SimpleTypeSerializer.cpp
index cf0c877fdb..25725853fb 100644
--- a/llvm/lib/DebugInfo/CodeView/SimpleTypeSerializer.cpp
+++ b/llvm/lib/DebugInfo/CodeView/SimpleTypeSerializer.cpp
@@ -34,7 +34,7 @@ SimpleTypeSerializer::~SimpleTypeSerializer() = default;
 
 template <typename T>
 ArrayRef<uint8_t> SimpleTypeSerializer::serialize(T &Record) {
-  BinaryStreamWriter Writer(ScratchBuffer, support::little);
+  BinaryStreamWriter Writer(ScratchBuffer, llvm::endianness::little);
   TypeRecordMapping Mapping(Writer);
 
   // Write the record prefix first with a dummy length but real kind.
diff --git a/llvm/lib/DebugInfo/CodeView/SymbolSerializer.cpp b/llvm/lib/DebugInfo/CodeView/SymbolSerializer.cpp
index 5fb8d497b9..e52f3e56f1 100644
--- a/llvm/lib/DebugInfo/CodeView/SymbolSerializer.cpp
+++ b/llvm/lib/DebugInfo/CodeView/SymbolSerializer.cpp
@@ -20,8 +20,8 @@ using namespace llvm::codeview;
 
 SymbolSerializer::SymbolSerializer(BumpPtrAllocator &Allocator,
                                    CodeViewContainer Container)
-    : Storage(Allocator), Stream(RecordBuffer, support::little), Writer(Stream),
-      Mapping(Writer, Container) {}
+    : Storage(Allocator), Stream(RecordBuffer, llvm::endianness::little),
+      Writer(Stream), Mapping(Writer, Container) {}
 
 Error SymbolSerializer::visitSymbolBegin(CVSymbol &Record) {
   assert(!CurrentSymbol && "Already in a symbol mapping!");
diff --git a/llvm/lib/DebugInfo/CodeView/TypeIndexDiscovery.cpp b/llvm/lib/DebugInfo/CodeView/TypeIndexDiscovery.cpp
index e903a37a8c..59e2a85c4d 100644
--- a/llvm/lib/DebugInfo/CodeView/TypeIndexDiscovery.cpp
+++ b/llvm/lib/DebugInfo/CodeView/TypeIndexDiscovery.cpp
@@ -470,7 +470,7 @@ static void resolveTypeIndexReferences(ArrayRef<uint8_t> RecordData,
 
   RecordData = RecordData.drop_front(sizeof(RecordPrefix));
 
-  BinaryStreamReader Reader(RecordData, support::little);
+  BinaryStreamReader Reader(RecordData, llvm::endianness::little);
   for (const auto &Ref : Refs) {
     Reader.setOffset(Ref.Offset);
     FixedStreamArray<TypeIndex> Run;
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFContext.cpp b/llvm/lib/DebugInfo/DWARF/DWARFContext.cpp
index 372897835c..57ca11a077 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFContext.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFContext.cpp
@@ -89,9 +89,11 @@ void fixupIndexV4(DWARFContext &C, DWARFUnitIndex &Index) {
     DWARFDataExtractor Data(DObj, S, C.isLittleEndian(), 0);
     while (Data.isValidOffset(Offset)) {
       DWARFUnitHeader Header;
-      if (!Header.extract(C, Data, &Offset, DWARFSectionKind::DW_SECT_INFO)) {
-        logAllUnhandledErrors(
-            createError("Failed to parse CU header in DWP file"), errs());
+      if (Error ExtractionErr = Header.extract(
+              C, Data, &Offset, DWARFSectionKind::DW_SECT_INFO)) {
+        C.getWarningHandler()(
+            createError("Failed to parse CU header in DWP file: " +
+                        toString(std::move(ExtractionErr))));
         Map.clear();
         break;
       }
@@ -149,9 +151,11 @@ void fixupIndexV5(DWARFContext &C, DWARFUnitIndex &Index) {
     uint64_t Offset = 0;
     while (Data.isValidOffset(Offset)) {
       DWARFUnitHeader Header;
-      if (!Header.extract(C, Data, &Offset, DWARFSectionKind::DW_SECT_INFO)) {
-        logAllUnhandledErrors(
-            createError("Failed to parse unit header in DWP file"), errs());
+      if (Error ExtractionErr = Header.extract(
+              C, Data, &Offset, DWARFSectionKind::DW_SECT_INFO)) {
+        C.getWarningHandler()(
+            createError("Failed to parse CU header in DWP file: " +
+                        toString(std::move(ExtractionErr))));
         break;
       }
       bool CU = Header.getUnitType() == DW_UT_split_compile;
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFTypePrinter.cpp b/llvm/lib/DebugInfo/DWARF/DWARFTypePrinter.cpp
index c474de6076..5a5ac28f18 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFTypePrinter.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFTypePrinter.cpp
@@ -620,6 +620,9 @@ void DWARFTypePrinter::appendSubroutineNameAfter(
     case CallingConvention::DW_CC_LLVM_X86RegCall:
       OS << " __attribute__((regcall))";
       break;
+    case CallingConvention::DW_CC_LLVM_M68kRTD:
+      OS << " __attribute__((m68k_rtd))";
+      break;
     }
   }
 
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFUnit.cpp b/llvm/lib/DebugInfo/DWARF/DWARFUnit.cpp
index 0cd45bde3e..9f455fa7e9 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFUnit.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFUnit.cpp
@@ -81,8 +81,11 @@ void DWARFUnitVector::addUnitsImpl(
       if (!Data.isValidOffset(Offset))
         return nullptr;
       DWARFUnitHeader Header;
-      if (!Header.extract(Context, Data, &Offset, SectionKind))
+      if (Error ExtractErr =
+              Header.extract(Context, Data, &Offset, SectionKind)) {
+        Context.getWarningHandler()(std::move(ExtractErr));
         return nullptr;
+      }
       if (!IndexEntry && IsDWO) {
         const DWARFUnitIndex &Index = getDWARFUnitIndex(
             Context, Header.isTypeUnit() ? DW_SECT_EXT_TYPES : DW_SECT_INFO);
@@ -244,10 +247,10 @@ Expected<uint64_t> DWARFUnit::getStringOffsetSectionItem(uint32_t Index) const {
   return DA.getRelocatedValue(ItemSize, &Offset);
 }
 
-bool DWARFUnitHeader::extract(DWARFContext &Context,
-                              const DWARFDataExtractor &debug_info,
-                              uint64_t *offset_ptr,
-                              DWARFSectionKind SectionKind) {
+Error DWARFUnitHeader::extract(DWARFContext &Context,
+                               const DWARFDataExtractor &debug_info,
+                               uint64_t *offset_ptr,
+                               DWARFSectionKind SectionKind) {
   Offset = *offset_ptr;
   Error Err = Error::success();
   IndexEntry = nullptr;
@@ -277,72 +280,58 @@ bool DWARFUnitHeader::extract(DWARFContext &Context,
   } else if (UnitType == DW_UT_split_compile || UnitType == DW_UT_skeleton)
     DWOId = debug_info.getU64(offset_ptr, &Err);
 
-  if (Err) {
-    Context.getWarningHandler()(joinErrors(
+  if (Err)
+    return joinErrors(
         createStringError(
             errc::invalid_argument,
             "DWARF unit at 0x%8.8" PRIx64 " cannot be parsed:", Offset),
-        std::move(Err)));
-    return false;
-  }
+        std::move(Err));
 
   // Header fields all parsed, capture the size of this unit header.
   assert(*offset_ptr - Offset <= 255 && "unexpected header size");
   Size = uint8_t(*offset_ptr - Offset);
   uint64_t NextCUOffset = Offset + getUnitLengthFieldByteSize() + getLength();
 
-  if (!debug_info.isValidOffset(getNextUnitOffset() - 1)) {
-    Context.getWarningHandler()(
-        createStringError(errc::invalid_argument,
-                          "DWARF unit from offset 0x%8.8" PRIx64 " incl. "
-                          "to offset  0x%8.8" PRIx64 " excl. "
-                          "extends past section size 0x%8.8zx",
-                          Offset, NextCUOffset, debug_info.size()));
-    return false;
-  }
+  if (!debug_info.isValidOffset(getNextUnitOffset() - 1))
+    return createStringError(errc::invalid_argument,
+                             "DWARF unit from offset 0x%8.8" PRIx64 " incl. "
+                             "to offset  0x%8.8" PRIx64 " excl. "
+                             "extends past section size 0x%8.8zx",
+                             Offset, NextCUOffset, debug_info.size());
 
-  if (!DWARFContext::isSupportedVersion(getVersion())) {
-    Context.getWarningHandler()(createStringError(
+  if (!DWARFContext::isSupportedVersion(getVersion()))
+    return createStringError(
         errc::invalid_argument,
         "DWARF unit at offset 0x%8.8" PRIx64 " "
         "has unsupported version %" PRIu16 ", supported are 2-%u",
-        Offset, getVersion(), DWARFContext::getMaxSupportedVersion()));
-    return false;
-  }
+        Offset, getVersion(), DWARFContext::getMaxSupportedVersion());
 
   // Type offset is unit-relative; should be after the header and before
   // the end of the current unit.
-  if (isTypeUnit() && TypeOffset < Size) {
-    Context.getWarningHandler()(
-        createStringError(errc::invalid_argument,
-                          "DWARF type unit at offset "
-                          "0x%8.8" PRIx64 " "
-                          "has its relocated type_offset 0x%8.8" PRIx64 " "
-                          "pointing inside the header",
-                          Offset, Offset + TypeOffset));
-    return false;
-  }
-  if (isTypeUnit() &&
-      TypeOffset >= getUnitLengthFieldByteSize() + getLength()) {
-    Context.getWarningHandler()(createStringError(
+  if (isTypeUnit() && TypeOffset < Size)
+    return createStringError(errc::invalid_argument,
+                             "DWARF type unit at offset "
+                             "0x%8.8" PRIx64 " "
+                             "has its relocated type_offset 0x%8.8" PRIx64 " "
+                             "pointing inside the header",
+                             Offset, Offset + TypeOffset);
+
+  if (isTypeUnit() && TypeOffset >= getUnitLengthFieldByteSize() + getLength())
+    return createStringError(
         errc::invalid_argument,
         "DWARF type unit from offset 0x%8.8" PRIx64 " incl. "
         "to offset 0x%8.8" PRIx64 " excl. has its "
         "relocated type_offset 0x%8.8" PRIx64 " pointing past the unit end",
-        Offset, NextCUOffset, Offset + TypeOffset));
-    return false;
-  }
+        Offset, NextCUOffset, Offset + TypeOffset);
 
   if (Error SizeErr = DWARFContext::checkAddressSizeSupported(
           getAddressByteSize(), errc::invalid_argument,
-          "DWARF unit at offset 0x%8.8" PRIx64, Offset)) {
-    Context.getWarningHandler()(std::move(SizeErr));
-    return false;
-  }
+          "DWARF unit at offset 0x%8.8" PRIx64, Offset))
+    return SizeErr;
 
   // Keep track of the highest DWARF version we encounter across all units.
   Context.setMaxVersionIfGreater(getVersion());
-  return true;
+  return Error::success();
 }
 
 bool DWARFUnitHeader::applyIndexEntry(const DWARFUnitIndex::Entry *Entry) {
diff --git a/llvm/lib/DebugInfo/GSYM/DwarfTransformer.cpp b/llvm/lib/DebugInfo/GSYM/DwarfTransformer.cpp
index e38347f15e..d720c1e334 100644
--- a/llvm/lib/DebugInfo/GSYM/DwarfTransformer.cpp
+++ b/llvm/lib/DebugInfo/GSYM/DwarfTransformer.cpp
@@ -132,11 +132,11 @@ static DWARFDie GetParentDeclContextDIE(DWARFDie &Die) {
 static std::optional<uint32_t>
 getQualifiedNameIndex(DWARFDie &Die, uint64_t Language, GsymCreator &Gsym) {
   // If the dwarf has mangled name, use mangled name
-  if (auto LinkageName =
-          dwarf::toString(Die.findRecursively({dwarf::DW_AT_MIPS_linkage_name,
-                                               dwarf::DW_AT_linkage_name}),
-                          nullptr))
-    return Gsym.insertString(LinkageName, /* Copy */ false);
+  if (auto LinkageName = Die.getLinkageName()) {
+    // We have seen cases were linkage name is actually empty.
+    if (strlen(LinkageName) > 0)
+      return Gsym.insertString(LinkageName, /* Copy */ false);
+  }
 
   StringRef ShortName(Die.getName(DINameKind::ShortName));
   if (ShortName.empty())
diff --git a/llvm/lib/DebugInfo/GSYM/GsymReader.cpp b/llvm/lib/DebugInfo/GSYM/GsymReader.cpp
index 7e6eec71d1..1fe90ef579 100644
--- a/llvm/lib/DebugInfo/GSYM/GsymReader.cpp
+++ b/llvm/lib/DebugInfo/GSYM/GsymReader.cpp
@@ -74,7 +74,8 @@ GsymReader::parse() {
       break;
     case GSYM_CIGAM:
       // This is a GSYM file, but not native endianness.
-      Endian = sys::IsBigEndianHost ? support::little : support::big;
+      Endian = sys::IsBigEndianHost ? llvm::endianness::little
+                                    : llvm::endianness::big;
       Swap.reset(new SwappedData);
       break;
     default:
@@ -82,7 +83,7 @@ GsymReader::parse() {
                                "not a GSYM file");
   }
 
-  bool DataIsLittleEndian = HostByteOrder != support::little;
+  bool DataIsLittleEndian = HostByteOrder != llvm::endianness::little;
   // Read a correctly byte swapped header if we need to.
   if (Swap) {
     DataExtractor Data(MemBuffer->getBuffer(), DataIsLittleEndian, 4);
@@ -259,10 +260,11 @@ llvm::Expected<FunctionInfo> GsymReader::getFunctionInfo(uint64_t Addr) const {
   // Address info offsets size should have been checked in parse().
   assert(*AddressIndex < AddrInfoOffsets.size());
   auto AddrInfoOffset = AddrInfoOffsets[*AddressIndex];
-  assert((Endian == support::big || Endian == support::little) &&
-         "Endian must be either big or little");
+  assert(
+      (Endian == llvm::endianness::big || Endian == llvm::endianness::little) &&
+      "Endian must be either big or little");
   DataExtractor Data(MemBuffer->getBuffer().substr(AddrInfoOffset),
-                     Endian == support::little, 4);
+                     Endian == llvm::endianness::little, 4);
   if (std::optional<uint64_t> OptAddr = getAddress(*AddressIndex)) {
     auto ExpectedFI = FunctionInfo::decode(Data, *OptAddr);
     if (ExpectedFI) {
@@ -284,10 +286,11 @@ llvm::Expected<LookupResult> GsymReader::lookup(uint64_t Addr) const {
   // Address info offsets size should have been checked in parse().
   assert(*AddressIndex < AddrInfoOffsets.size());
   auto AddrInfoOffset = AddrInfoOffsets[*AddressIndex];
-  assert((Endian == support::big || Endian == support::little) &&
-         "Endian must be either big or little");
+  assert(
+      (Endian == llvm::endianness::big || Endian == llvm::endianness::little) &&
+      "Endian must be either big or little");
   DataExtractor Data(MemBuffer->getBuffer().substr(AddrInfoOffset),
-                     Endian == support::little, 4);
+                     Endian == llvm::endianness::little, 4);
   if (std::optional<uint64_t> OptAddr = getAddress(*AddressIndex))
     return FunctionInfo::lookup(Data, *this, *OptAddr, Addr);
   return createStringError(std::errc::invalid_argument,
diff --git a/llvm/lib/DebugInfo/LogicalView/Core/LVCompare.cpp b/llvm/lib/DebugInfo/LogicalView/Core/LVCompare.cpp
index 65baf52ffb..3ed0de14f9 100644
--- a/llvm/lib/DebugInfo/LogicalView/Core/LVCompare.cpp
+++ b/llvm/lib/DebugInfo/LogicalView/Core/LVCompare.cpp
@@ -206,11 +206,10 @@ Error LVCompare::execute(LVReader *ReferenceReader, LVReader *TargetReader) {
               updateExpected(Reference);
             Reference->setIsInCompare();
             LVElement *CurrentTarget = nullptr;
-            if (std::any_of(Targets.begin(), Targets.end(),
-                            [&](auto Target) -> bool {
-                              CurrentTarget = Target;
-                              return Reference->equals(Target);
-                            })) {
+            if (llvm::any_of(Targets, [&](auto Target) -> bool {
+                  CurrentTarget = Target;
+                  return Reference->equals(Target);
+                })) {
               if (Pass == LVComparePass::Missing && Reference->getIsScope()) {
                 // If the elements being compared are scopes and are a match,
                 // they are recorded, to be used when creating the augmented
diff --git a/llvm/lib/DebugInfo/LogicalView/Core/LVElement.cpp b/llvm/lib/DebugInfo/LogicalView/Core/LVElement.cpp
index cfe304eead..30ce937cda 100644
--- a/llvm/lib/DebugInfo/LogicalView/Core/LVElement.cpp
+++ b/llvm/lib/DebugInfo/LogicalView/Core/LVElement.cpp
@@ -252,8 +252,7 @@ void LVElement::generateName(std::string &Prefix) const {
   Prefix.append(isLined() ? lineNumberAsString(/*ShowZero=*/true) : "?");
 
   // Remove any whitespaces.
-  Prefix.erase(std::remove_if(Prefix.begin(), Prefix.end(), ::isspace),
-               Prefix.end());
+  llvm::erase_if(Prefix, ::isspace);
 }
 
 // Generate a name for unnamed elements.
diff --git a/llvm/lib/DebugInfo/LogicalView/Core/LVScope.cpp b/llvm/lib/DebugInfo/LogicalView/Core/LVScope.cpp
index 2f26025d01..e2b0d4db3a 100644
--- a/llvm/lib/DebugInfo/LogicalView/Core/LVScope.cpp
+++ b/llvm/lib/DebugInfo/LogicalView/Core/LVScope.cpp
@@ -299,20 +299,12 @@ void LVScope::addMissingElements(LVScope *Reference) {
   LVSymbols References;
   References.append(ReferenceSymbols->begin(), ReferenceSymbols->end());
 
-  auto RemoveSymbol = [&](LVSymbols &Symbols, LVSymbol *Symbol) {
-    LVSymbols::iterator Iter = std::remove_if(
-        Symbols.begin(), Symbols.end(),
-        [Symbol](LVSymbol *Item) -> bool { return Item == Symbol; });
-    if (Iter != Symbols.end())
-      Symbols.erase(Iter, Symbols.end());
-  };
-
   // Erase abstract symbols already in this scope from the collection of
   // symbols in the referenced scope.
   if (getSymbols())
     for (const LVSymbol *Symbol : *getSymbols())
       if (Symbol->getHasReferenceAbstract())
-        RemoveSymbol(References, Symbol->getReference());
+        llvm::erase_value(References, Symbol->getReference());
 
   // If we have elements left in 'References', those are the elements that
   // need to be inserted in the current scope.
diff --git a/llvm/lib/DebugInfo/LogicalView/Readers/LVCodeViewReader.cpp b/llvm/lib/DebugInfo/LogicalView/Readers/LVCodeViewReader.cpp
index d72fe2683f..d1789fe587 100644
--- a/llvm/lib/DebugInfo/LogicalView/Readers/LVCodeViewReader.cpp
+++ b/llvm/lib/DebugInfo/LogicalView/Readers/LVCodeViewReader.cpp
@@ -276,7 +276,7 @@ Error LVCodeViewReader::collectInlineeInfo(
 }
 
 Error LVCodeViewReader::traverseInlineeLines(StringRef Subsection) {
-  BinaryStreamReader SR(Subsection, llvm::support::little);
+  BinaryStreamReader SR(Subsection, llvm::endianness::little);
   DebugInlineeLinesSubsectionRef Lines;
   if (Error E = Lines.initialize(SR))
     return createStringError(errorToErrorCode(std::move(E)), getFileName());
@@ -349,7 +349,7 @@ Error LVCodeViewReader::initializeFileAndStringTables(
     if (Error E = Reader.readFixedString(Contents, SubSectionSize))
       return createStringError(errorToErrorCode(std::move(E)), getFileName());
 
-    BinaryStreamRef ST(Contents, support::little);
+    BinaryStreamRef ST(Contents, llvm::endianness::little);
     switch (DebugSubsectionKind(SubType)) {
     case DebugSubsectionKind::FileChecksums:
       if (Error E = CVFileChecksumTable.initialize(ST))
@@ -478,8 +478,8 @@ Error LVCodeViewReader::loadPrecompiledObject(PrecompRecord &Precomp,
       if (Magic != COFF::DEBUG_SECTION_MAGIC)
         return errorCodeToError(object_error::parse_failed);
 
-      ReaderPrecomp =
-          std::make_unique<BinaryStreamReader>(*DataOrErr, support::little);
+      ReaderPrecomp = std::make_unique<BinaryStreamReader>(
+          *DataOrErr, llvm::endianness::little);
       cantFail(
           ReaderPrecomp->readArray(CVTypesPrecomp, ReaderPrecomp->getLength()));
 
@@ -514,7 +514,7 @@ Error LVCodeViewReader::loadPrecompiledObject(PrecompRecord &Precomp,
       [&](TypeIndex TI, const CVType &Type) { TypeArray.push_back(Type); });
 
   ItemStream =
-      std::make_unique<BinaryItemStream<CVType>>(llvm::support::little);
+      std::make_unique<BinaryItemStream<CVType>>(llvm::endianness::little);
   ItemStream->setItems(TypeArray);
   TypeStream.setUnderlyingStream(*ItemStream);
 
@@ -550,7 +550,7 @@ Error LVCodeViewReader::traverseTypeSection(StringRef SectionName,
   // Get the first type record. It will indicate if this object uses a type
   // server (/Zi) or a PCH file (/Yu).
   CVTypeArray CVTypes;
-  BinaryStreamReader Reader(*DataOrErr, support::little);
+  BinaryStreamReader Reader(*DataOrErr, llvm::endianness::little);
   cantFail(Reader.readArray(CVTypes, Reader.getLength()));
   CVTypeArray::Iterator FirstType = CVTypes.begin();
 
@@ -621,7 +621,7 @@ Error LVCodeViewReader::traverseSymbolsSubsection(StringRef Subsection,
   LVSymbolVisitorDelegate VisitorDelegate(this, Section, &getObj(),
                                           SectionContents);
   CVSymbolArray Symbols;
-  BinaryStreamReader Reader(BinaryData, llvm::support::little);
+  BinaryStreamReader Reader(BinaryData, llvm::endianness::little);
   if (Error E = Reader.readArray(Symbols, Reader.getLength()))
     return createStringError(errorToErrorCode(std::move(E)), getFileName());
 
@@ -664,7 +664,7 @@ Error LVCodeViewReader::traverseSymbolSection(StringRef SectionName,
   if (Magic != COFF::DEBUG_SECTION_MAGIC)
     return createStringError(object_error::parse_failed, getFileName());
 
-  BinaryStreamReader FSReader(Data, support::little);
+  BinaryStreamReader FSReader(Data, llvm::endianness::little);
   if (Error Err = initializeFileAndStringTables(FSReader))
     return Err;
 
@@ -752,7 +752,8 @@ Error LVCodeViewReader::traverseSymbolSection(StringRef SectionName,
       W.printString("Symbol Name", SymbolName);
     });
 
-    BinaryStreamReader Reader(FunctionLineTables[SymbolName], support::little);
+    BinaryStreamReader Reader(FunctionLineTables[SymbolName],
+                              llvm::endianness::little);
 
     DebugLinesSubsectionRef Lines;
     if (Error E = Lines.initialize(Reader))
diff --git a/llvm/lib/DebugInfo/LogicalView/Readers/LVCodeViewVisitor.cpp b/llvm/lib/DebugInfo/LogicalView/Readers/LVCodeViewVisitor.cpp
index 5a6414f556..1d01785328 100644
--- a/llvm/lib/DebugInfo/LogicalView/Readers/LVCodeViewVisitor.cpp
+++ b/llvm/lib/DebugInfo/LogicalView/Readers/LVCodeViewVisitor.cpp
@@ -465,13 +465,10 @@ LVScope *LVNamespaceDeduction::get(LVStringRefs Components) {
 LVScope *LVNamespaceDeduction::get(StringRef ScopedName, bool CheckScope) {
   LVStringRefs Components = getAllLexicalComponents(ScopedName);
   if (CheckScope)
-    Components.erase(std::remove_if(Components.begin(), Components.end(),
-                                    [&](StringRef Component) {
-                                      LookupSet::iterator Iter =
-                                          IdentifiedNamespaces.find(Component);
-                                      return Iter == IdentifiedNamespaces.end();
-                                    }),
-                     Components.end());
+    llvm::erase_if(Components, [&](StringRef Component) {
+      LookupSet::iterator Iter = IdentifiedNamespaces.find(Component);
+      return Iter == IdentifiedNamespaces.end();
+    });
 
   LLVM_DEBUG(
       { dbgs() << formatv("ScopedName: '{0}'\n", ScopedName.str().c_str()); });
@@ -2939,7 +2936,7 @@ Error LVLogicalVisitor::finishVisitation(CVType &Record, TypeIndex TI,
 // Customized version of 'FieldListVisitHelper'.
 Error LVLogicalVisitor::visitFieldListMemberStream(
     TypeIndex TI, LVElement *Element, ArrayRef<uint8_t> FieldList) {
-  BinaryByteStream Stream(FieldList, llvm::support::little);
+  BinaryByteStream Stream(FieldList, llvm::endianness::little);
   BinaryStreamReader Reader(Stream);
   FieldListDeserializer Deserializer(Reader);
   TypeVisitorCallbackPipeline Pipeline;
diff --git a/llvm/lib/DebugInfo/MSF/MSFBuilder.cpp b/llvm/lib/DebugInfo/MSF/MSFBuilder.cpp
index e909e851a9..ed2d14dd79 100644
--- a/llvm/lib/DebugInfo/MSF/MSFBuilder.cpp
+++ b/llvm/lib/DebugInfo/MSF/MSFBuilder.cpp
@@ -386,7 +386,7 @@ Expected<FileBufferByteStream> MSFBuilder::commit(StringRef Path,
     return std::move(EC);
 
   FileBufferByteStream Buffer(std::move(*OutFileOrError),
-                              llvm::support::little);
+                              llvm::endianness::little);
   BinaryStreamWriter Writer(Buffer);
 
   if (auto EC = Writer.writeObject(*Layout.SB))
diff --git a/llvm/lib/DebugInfo/MSF/MappedBlockStream.cpp b/llvm/lib/DebugInfo/MSF/MappedBlockStream.cpp
index 94935d6345..5ebb76994b 100644
--- a/llvm/lib/DebugInfo/MSF/MappedBlockStream.cpp
+++ b/llvm/lib/DebugInfo/MSF/MappedBlockStream.cpp
@@ -10,7 +10,6 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/DebugInfo/MSF/MSFCommon.h"
 #include "llvm/Support/BinaryStreamWriter.h"
-#include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MathExtras.h"
 #include <algorithm>
diff --git a/llvm/lib/DebugInfo/PDB/Native/DbiModuleDescriptor.cpp b/llvm/lib/DebugInfo/PDB/Native/DbiModuleDescriptor.cpp
index 9755f2ca3b..f27a645c77 100644
--- a/llvm/lib/DebugInfo/PDB/Native/DbiModuleDescriptor.cpp
+++ b/llvm/lib/DebugInfo/PDB/Native/DbiModuleDescriptor.cpp
@@ -9,7 +9,6 @@
 #include "llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h"
 #include "llvm/DebugInfo/PDB/Native/RawTypes.h"
 #include "llvm/Support/BinaryStreamReader.h"
-#include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MathExtras.h"
 #include <cstdint>
diff --git a/llvm/lib/DebugInfo/PDB/Native/DbiStreamBuilder.cpp b/llvm/lib/DebugInfo/PDB/Native/DbiStreamBuilder.cpp
index fb8b11b9c1..ad3d09ae50 100644
--- a/llvm/lib/DebugInfo/PDB/Native/DbiStreamBuilder.cpp
+++ b/llvm/lib/DebugInfo/PDB/Native/DbiStreamBuilder.cpp
@@ -188,7 +188,7 @@ Error DbiStreamBuilder::generateFileInfoSubstream() {
   uint32_t NamesOffset = calculateNamesOffset();
 
   FileInfoBuffer = MutableBinaryByteStream(MutableArrayRef<uint8_t>(Data, Size),
-                                           llvm::support::little);
+                                           llvm::endianness::little);
 
   WritableBinaryStreamRef MetadataBuffer =
       WritableBinaryStreamRef(FileInfoBuffer).keep_front(NamesOffset);
diff --git a/llvm/lib/DebugInfo/PDB/Native/GSIStreamBuilder.cpp b/llvm/lib/DebugInfo/PDB/Native/GSIStreamBuilder.cpp
index 9f9bd47aa6..c195754c0c 100644
--- a/llvm/lib/DebugInfo/PDB/Native/GSIStreamBuilder.cpp
+++ b/llvm/lib/DebugInfo/PDB/Native/GSIStreamBuilder.cpp
@@ -394,7 +394,7 @@ static Error writePublics(BinaryStreamWriter &Writer,
 
 static Error writeRecords(BinaryStreamWriter &Writer,
                           ArrayRef<CVSymbol> Records) {
-  BinaryItemStream<CVSymbol> ItemStream(support::endianness::little);
+  BinaryItemStream<CVSymbol> ItemStream(llvm::endianness::little);
   ItemStream.setItems(Records);
   BinaryStreamRef RecordsRef(ItemStream);
   return Writer.writeStreamRef(RecordsRef);
diff --git a/llvm/lib/DebugInfo/PDB/Native/InjectedSourceStream.cpp b/llvm/lib/DebugInfo/PDB/Native/InjectedSourceStream.cpp
index f1e8adeb1b..841068c77d 100644
--- a/llvm/lib/DebugInfo/PDB/Native/InjectedSourceStream.cpp
+++ b/llvm/lib/DebugInfo/PDB/Native/InjectedSourceStream.cpp
@@ -14,7 +14,6 @@
 #include "llvm/DebugInfo/PDB/Native/RawConstants.h"
 #include "llvm/DebugInfo/PDB/Native/RawTypes.h"
 #include "llvm/Support/BinaryStreamReader.h"
-#include "llvm/Support/Endian.h"
 
 using namespace llvm;
 using namespace llvm::msf;
diff --git a/llvm/lib/DebugInfo/PDB/Native/InputFile.cpp b/llvm/lib/DebugInfo/PDB/Native/InputFile.cpp
index 85c22483fa..cddee3e1c2 100644
--- a/llvm/lib/DebugInfo/PDB/Native/InputFile.cpp
+++ b/llvm/lib/DebugInfo/PDB/Native/InputFile.cpp
@@ -107,7 +107,7 @@ static inline bool isCodeViewDebugSubsection(object::SectionRef Section,
     return false;
   }
 
-  Reader = BinaryStreamReader(*ContentsOrErr, support::little);
+  Reader = BinaryStreamReader(*ContentsOrErr, llvm::endianness::little);
   uint32_t Magic;
   if (Reader.bytesRemaining() < sizeof(uint32_t))
     return false;
diff --git a/llvm/lib/DebugInfo/PDB/Native/NativeSession.cpp b/llvm/lib/DebugInfo/PDB/Native/NativeSession.cpp
index 91b428afad..fb0ca2f9cf 100644
--- a/llvm/lib/DebugInfo/PDB/Native/NativeSession.cpp
+++ b/llvm/lib/DebugInfo/PDB/Native/NativeSession.cpp
@@ -74,7 +74,7 @@ Error NativeSession::createFromPdb(std::unique_ptr<MemoryBuffer> Buffer,
                                    std::unique_ptr<IPDBSession> &Session) {
   StringRef Path = Buffer->getBufferIdentifier();
   auto Stream = std::make_unique<MemoryBufferByteStream>(
-      std::move(Buffer), llvm::support::little);
+      std::move(Buffer), llvm::endianness::little);
 
   auto Allocator = std::make_unique<BumpPtrAllocator>();
   auto File = std::make_unique<PDBFile>(Path, std::move(Stream), *Allocator);
@@ -104,8 +104,8 @@ loadPdbFile(StringRef PdbPath, std::unique_ptr<BumpPtrAllocator> &Allocator) {
   if (EC || Magic != file_magic::pdb)
     return make_error<RawError>(EC);
 
-  auto Stream = std::make_unique<MemoryBufferByteStream>(std::move(Buffer),
-                                                         llvm::support::little);
+  auto Stream = std::make_unique<MemoryBufferByteStream>(
+      std::move(Buffer), llvm::endianness::little);
 
   auto File = std::make_unique<PDBFile>(PdbPath, std::move(Stream), *Allocator);
   if (auto EC = File->parseFileHeaders())
diff --git a/llvm/lib/DebugInfo/PDB/Native/PublicsStream.cpp b/llvm/lib/DebugInfo/PDB/Native/PublicsStream.cpp
index c7b9f443da..c350e0e0b3 100644
--- a/llvm/lib/DebugInfo/PDB/Native/PublicsStream.cpp
+++ b/llvm/lib/DebugInfo/PDB/Native/PublicsStream.cpp
@@ -26,7 +26,6 @@
 #include "llvm/DebugInfo/PDB/Native/RawError.h"
 #include "llvm/DebugInfo/PDB/Native/RawTypes.h"
 #include "llvm/Support/BinaryStreamReader.h"
-#include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
 #include <cstdint>
 
diff --git a/llvm/lib/DebugInfo/PDB/Native/SymbolStream.cpp b/llvm/lib/DebugInfo/PDB/Native/SymbolStream.cpp
index 5802d1c775..5dd636f326 100644
--- a/llvm/lib/DebugInfo/PDB/Native/SymbolStream.cpp
+++ b/llvm/lib/DebugInfo/PDB/Native/SymbolStream.cpp
@@ -9,7 +9,6 @@
 #include "llvm/DebugInfo/PDB/Native/SymbolStream.h"
 
 #include "llvm/DebugInfo/MSF/MappedBlockStream.h"
-#include "llvm/Support/Endian.h"
 
 using namespace llvm;
 using namespace llvm::msf;
diff --git a/llvm/lib/DebugInfo/PDB/Native/TpiStreamBuilder.cpp b/llvm/lib/DebugInfo/PDB/Native/TpiStreamBuilder.cpp
index 30d5a563e8..22663f0096 100644
--- a/llvm/lib/DebugInfo/PDB/Native/TpiStreamBuilder.cpp
+++ b/llvm/lib/DebugInfo/PDB/Native/TpiStreamBuilder.cpp
@@ -165,7 +165,7 @@ Error TpiStreamBuilder::finalizeMsfLayout() {
         reinterpret_cast<const uint8_t *>(HashBuffer.data()),
         calculateHashBufferSize());
     HashValueStream =
-        std::make_unique<BinaryByteStream>(Bytes, llvm::support::little);
+        std::make_unique<BinaryByteStream>(Bytes, llvm::endianness::little);
   }
   return Error::success();
 }
diff --git a/llvm/lib/ExecutionEngine/ExecutionEngine.cpp b/llvm/lib/ExecutionEngine/ExecutionEngine.cpp
index 9e81cca0b1..0d4f9835f0 100644
--- a/llvm/lib/ExecutionEngine/ExecutionEngine.cpp
+++ b/llvm/lib/ExecutionEngine/ExecutionEngine.cpp
@@ -618,7 +618,18 @@ GenericValue ExecutionEngine::getConstantValue(const Constant *C) {
       case Type::ScalableVectorTyID:
         report_fatal_error(
             "Scalable vector support not yet implemented in ExecutionEngine");
-      case Type::FixedVectorTyID:
+      case Type::ArrayTyID: {
+        auto *ArrTy = cast<ArrayType>(C->getType());
+        Type *ElemTy = ArrTy->getElementType();
+        unsigned int elemNum = ArrTy->getNumElements();
+        Result.AggregateVal.resize(elemNum);
+        if (ElemTy->isIntegerTy())
+          for (unsigned int i = 0; i < elemNum; ++i)
+            Result.AggregateVal[i].IntVal =
+                APInt(ElemTy->getPrimitiveSizeInBits(), 0);
+        break;
+      }
+      case Type::FixedVectorTyID: {
         // if the whole vector is 'undef' just reserve memory for the value.
         auto *VTy = cast<FixedVectorType>(C->getType());
         Type *ElemTy = VTy->getElementType();
@@ -629,6 +640,7 @@ GenericValue ExecutionEngine::getConstantValue(const Constant *C) {
             Result.AggregateVal[i].IntVal =
                 APInt(ElemTy->getPrimitiveSizeInBits(), 0);
         break;
+      }
     }
     return Result;
   }
diff --git a/llvm/lib/ExecutionEngine/JITLink/COFF.cpp b/llvm/lib/ExecutionEngine/JITLink/COFF.cpp
index fddc9b813f..f4701bc830 100644
--- a/llvm/lib/ExecutionEngine/JITLink/COFF.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/COFF.cpp
@@ -15,7 +15,6 @@
 #include "llvm/BinaryFormat/COFF.h"
 #include "llvm/ExecutionEngine/JITLink/COFF_x86_64.h"
 #include "llvm/Object/COFF.h"
-#include "llvm/Support/Endian.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include <cstring>
diff --git a/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.cpp b/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.cpp
index 060b3d6723..1fd2a33d3f 100644
--- a/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.cpp
@@ -43,9 +43,10 @@ COFFLinkGraphBuilder::getPointerSize(const object::COFFObjectFile &Obj) {
   return Obj.getBytesInAddress();
 }
 
-support::endianness
+llvm::endianness
 COFFLinkGraphBuilder::getEndianness(const object::COFFObjectFile &Obj) {
-  return Obj.isLittleEndian() ? support::little : support::big;
+  return Obj.isLittleEndian() ? llvm::endianness::little
+                              : llvm::endianness::big;
 }
 
 uint64_t COFFLinkGraphBuilder::getSectionSize(const object::COFFObjectFile &Obj,
diff --git a/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.h b/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.h
index e648237595..e5f3ce8c53 100644
--- a/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.h
+++ b/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.h
@@ -14,7 +14,6 @@
 #define LIB_EXECUTIONENGINE_JITLINK_COFFLINKGRAPHBUILDER_H
 
 #include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/StringMap.h"
 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
 #include "llvm/Object/COFF.h"
 
@@ -162,7 +161,7 @@ class COFFLinkGraphBuilder {
                                  const object::coff_section *Section);
   static bool isComdatSection(const object::coff_section *Section);
   static unsigned getPointerSize(const object::COFFObjectFile &Obj);
-  static support::endianness getEndianness(const object::COFFObjectFile &Obj);
+  static llvm::endianness getEndianness(const object::COFFObjectFile &Obj);
   static StringRef getDLLImportStubPrefix() { return "__imp_"; }
   static StringRef getDirectiveSectionName() { return ".drectve"; }
   StringRef getCOFFSectionName(COFFSectionIndex SectionIndex,
diff --git a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp
index 16665f422c..c11577b03f 100644
--- a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp
@@ -126,16 +126,23 @@ Error EHFrameEdgeFixer::processBlock(ParseContext &PC, Block &B) {
   }
 
   // Find the offsets of any existing edges from this block.
-  BlockEdgeMap BlockEdges;
+  BlockEdgesInfo BlockEdges;
   for (auto &E : B.edges())
     if (E.isRelocation()) {
-      if (BlockEdges.count(E.getOffset()))
-        return make_error<JITLinkError>(
-            "Multiple relocations at offset " +
-            formatv("{0:x16}", E.getOffset()) + " in " + EHFrameSectionName +
-            " block at address " + formatv("{0:x16}", B.getAddress()));
-
-      BlockEdges[E.getOffset()] = EdgeTarget(E);
+      // Check if we already saw more than one relocation at this offset.
+      if (BlockEdges.Multiple.contains(E.getOffset()))
+        continue;
+
+      // Otherwise check if we previously had exactly one relocation at this
+      // offset. If so, we now have a second one and move it from the TargetMap
+      // into the Multiple set.
+      auto It = BlockEdges.TargetMap.find(E.getOffset());
+      if (It != BlockEdges.TargetMap.end()) {
+        BlockEdges.TargetMap.erase(It);
+        BlockEdges.Multiple.insert(E.getOffset());
+      } else {
+        BlockEdges.TargetMap[E.getOffset()] = EdgeTarget(E);
+      }
     }
 
   BinaryStreamReader BlockReader(
@@ -172,7 +179,7 @@ Error EHFrameEdgeFixer::processBlock(ParseContext &PC, Block &B) {
 
 Error EHFrameEdgeFixer::processCIE(ParseContext &PC, Block &B,
                                    size_t CIEDeltaFieldOffset,
-                                   const BlockEdgeMap &BlockEdges) {
+                                   const BlockEdgesInfo &BlockEdges) {
 
   LLVM_DEBUG(dbgs() << "    Record is CIE\n");
 
@@ -285,7 +292,7 @@ Error EHFrameEdgeFixer::processCIE(ParseContext &PC, Block &B,
 Error EHFrameEdgeFixer::processFDE(ParseContext &PC, Block &B,
                                    size_t CIEDeltaFieldOffset,
                                    uint32_t CIEDelta,
-                                   const BlockEdgeMap &BlockEdges) {
+                                   const BlockEdgesInfo &BlockEdges) {
   LLVM_DEBUG(dbgs() << "    Record is FDE\n");
 
   orc::ExecutorAddr RecordAddress = B.getAddress();
@@ -303,12 +310,17 @@ Error EHFrameEdgeFixer::processFDE(ParseContext &PC, Block &B,
 
   {
     // Process the CIE pointer field.
-    auto CIEEdgeItr = BlockEdges.find(CIEDeltaFieldOffset);
+    if (BlockEdges.Multiple.contains(CIEDeltaFieldOffset))
+      return make_error<JITLinkError>(
+          "CIE pointer field already has multiple edges at " +
+          formatv("{0:x16}", RecordAddress + CIEDeltaFieldOffset));
+
+    auto CIEEdgeItr = BlockEdges.TargetMap.find(CIEDeltaFieldOffset);
 
     orc::ExecutorAddr CIEAddress =
         RecordAddress + orc::ExecutorAddrDiff(CIEDeltaFieldOffset) -
         orc::ExecutorAddrDiff(CIEDelta);
-    if (CIEEdgeItr == BlockEdges.end()) {
+    if (CIEEdgeItr == BlockEdges.TargetMap.end()) {
       LLVM_DEBUG({
         dbgs() << "        Adding edge at "
                << (RecordAddress + CIEDeltaFieldOffset)
@@ -497,7 +509,7 @@ Error EHFrameEdgeFixer::skipEncodedPointer(uint8_t PointerEncoding,
 }
 
 Expected<Symbol *> EHFrameEdgeFixer::getOrCreateEncodedPointerEdge(
-    ParseContext &PC, const BlockEdgeMap &BlockEdges, uint8_t PointerEncoding,
+    ParseContext &PC, const BlockEdgesInfo &BlockEdges, uint8_t PointerEncoding,
     BinaryStreamReader &RecordReader, Block &BlockToFix,
     size_t PointerFieldOffset, const char *FieldName) {
   using namespace dwarf;
@@ -508,8 +520,8 @@ Expected<Symbol *> EHFrameEdgeFixer::getOrCreateEncodedPointerEdge(
   // If there's already an edge here then just skip the encoded pointer and
   // return the edge's target.
   {
-    auto EdgeI = BlockEdges.find(PointerFieldOffset);
-    if (EdgeI != BlockEdges.end()) {
+    auto EdgeI = BlockEdges.TargetMap.find(PointerFieldOffset);
+    if (EdgeI != BlockEdges.TargetMap.end()) {
       LLVM_DEBUG({
         dbgs() << "      Existing edge at "
                << (BlockToFix.getAddress() + PointerFieldOffset) << " to "
@@ -522,6 +534,10 @@ Expected<Symbol *> EHFrameEdgeFixer::getOrCreateEncodedPointerEdge(
         return std::move(Err);
       return EdgeI->second.Target;
     }
+
+    if (BlockEdges.Multiple.contains(PointerFieldOffset))
+      return make_error<JITLinkError>("Multiple relocations at offset " +
+                                      formatv("{0:x16}", PointerFieldOffset));
   }
 
   // Switch absptr to corresponding udata encoding.
diff --git a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h
index cc3c24cc85..49fbf650e7 100644
--- a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h
+++ b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h
@@ -60,7 +60,11 @@ class EHFrameEdgeFixer {
     Edge::AddendT Addend = 0;
   };
 
-  using BlockEdgeMap = DenseMap<Edge::OffsetT, EdgeTarget>;
+  struct BlockEdgesInfo {
+    DenseMap<Edge::OffsetT, EdgeTarget> TargetMap;
+    DenseSet<Edge::OffsetT> Multiple;
+  };
+
   using CIEInfosMap = DenseMap<orc::ExecutorAddr, CIEInformation>;
 
   struct ParseContext {
@@ -82,9 +86,9 @@ class EHFrameEdgeFixer {
 
   Error processBlock(ParseContext &PC, Block &B);
   Error processCIE(ParseContext &PC, Block &B, size_t CIEDeltaFieldOffset,
-                   const BlockEdgeMap &BlockEdges);
+                   const BlockEdgesInfo &BlockEdges);
   Error processFDE(ParseContext &PC, Block &B, size_t CIEDeltaFieldOffset,
-                   uint32_t CIEDelta, const BlockEdgeMap &BlockEdges);
+                   uint32_t CIEDelta, const BlockEdgesInfo &BlockEdges);
 
   Expected<AugmentationInfo>
   parseAugmentationString(BinaryStreamReader &RecordReader);
@@ -94,9 +98,9 @@ class EHFrameEdgeFixer {
   Error skipEncodedPointer(uint8_t PointerEncoding,
                            BinaryStreamReader &RecordReader);
   Expected<Symbol *> getOrCreateEncodedPointerEdge(
-      ParseContext &PC, const BlockEdgeMap &BlockEdges, uint8_t PointerEncoding,
-      BinaryStreamReader &RecordReader, Block &BlockToFix,
-      size_t PointerFieldOffset, const char *FieldName);
+      ParseContext &PC, const BlockEdgesInfo &BlockEdges,
+      uint8_t PointerEncoding, BinaryStreamReader &RecordReader,
+      Block &BlockToFix, size_t PointerFieldOffset, const char *FieldName);
 
   Expected<Symbol &> getOrCreateSymbol(ParseContext &PC,
                                        orc::ExecutorAddr Addr);
diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF.cpp
index d60f33f5d0..fdcce20cd2 100644
--- a/llvm/lib/ExecutionEngine/JITLink/ELF.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/ELF.cpp
@@ -21,7 +21,6 @@
 #include "llvm/ExecutionEngine/JITLink/ELF_riscv.h"
 #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h"
 #include "llvm/Object/ELF.h"
-#include "llvm/Support/Endian.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include <cstring>
diff --git a/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h b/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h
index 1f545adcdf..56d1efa4bd 100644
--- a/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h
+++ b/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h
@@ -193,7 +193,7 @@ ELFLinkGraphBuilder<ELFT>::ELFLinkGraphBuilder(
     StringRef FileName, LinkGraph::GetEdgeKindNameFunction GetEdgeKindName)
     : ELFLinkGraphBuilderBase(std::make_unique<LinkGraph>(
           FileName.str(), Triple(std::move(TT)), std::move(Features),
-          ELFT::Is64Bits ? 8 : 4, support::endianness(ELFT::TargetEndianness),
+          ELFT::Is64Bits ? 8 : 4, llvm::endianness(ELFT::TargetEndianness),
           std::move(GetEdgeKindName))),
       Obj(Obj) {
   LLVM_DEBUG(
diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch32.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch32.cpp
index 4d2dc68462..23946c7de9 100644
--- a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch32.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch32.cpp
@@ -114,7 +114,7 @@ class ELFJITLinker_aarch32 : public JITLinker<ELFJITLinker_aarch32> {
   }
 };
 
-template <support::endianness DataEndianness>
+template <llvm::endianness DataEndianness>
 class ELFLinkGraphBuilder_aarch32
     : public ELFLinkGraphBuilder<ELFType<DataEndianness, false>> {
 private:
@@ -265,7 +265,7 @@ createLinkGraphFromELFObject_aarch32(MemoryBufferRef ObjectBuffer) {
   case Triple::arm:
   case Triple::thumb: {
     auto &ELFFile = cast<ELFObjectFile<ELF32LE>>(**ELFObj).getELFFile();
-    return ELFLinkGraphBuilder_aarch32<support::little>(
+    return ELFLinkGraphBuilder_aarch32<llvm::endianness::little>(
                (*ELFObj)->getFileName(), ELFFile, TT, std::move(*Features),
                ArmCfg)
         .buildGraph();
@@ -273,7 +273,7 @@ createLinkGraphFromELFObject_aarch32(MemoryBufferRef ObjectBuffer) {
   case Triple::armeb:
   case Triple::thumbeb: {
     auto &ELFFile = cast<ELFObjectFile<ELF32BE>>(**ELFObj).getELFFile();
-    return ELFLinkGraphBuilder_aarch32<support::big>(
+    return ELFLinkGraphBuilder_aarch32<llvm::endianness::big>(
                (*ELFObj)->getFileName(), ELFFile, TT, std::move(*Features),
                ArmCfg)
         .buildGraph();
diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_ppc64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_ppc64.cpp
index 5ce7a5bda8..25b1dd9d3d 100644
--- a/llvm/lib/ExecutionEngine/JITLink/ELF_ppc64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/ELF_ppc64.cpp
@@ -33,7 +33,7 @@ constexpr StringRef TOCSymbolAliasIdent = "__TOC__";
 constexpr uint64_t ELFTOCBaseOffset = 0x8000;
 constexpr StringRef ELFTLSInfoSectionName = "$__TLSINFO";
 
-template <support::endianness Endianness>
+template <llvm::endianness Endianness>
 class TLSInfoTableManager_ELF_ppc64
     : public TableManager<TLSInfoTableManager_ELF_ppc64<Endianness>> {
 public:
@@ -43,17 +43,22 @@ class TLSInfoTableManager_ELF_ppc64
 
   bool visitEdge(LinkGraph &G, Block *B, Edge &E) {
     Edge::Kind K = E.getKind();
-    if (K == ppc64::RequestTLSDescInGOTAndTransformToTOCDelta16HA) {
+    switch (K) {
+    case ppc64::RequestTLSDescInGOTAndTransformToTOCDelta16HA:
       E.setKind(ppc64::TOCDelta16HA);
       E.setTarget(this->getEntryForTarget(G, E.getTarget()));
       return true;
-    }
-    if (K == ppc64::RequestTLSDescInGOTAndTransformToTOCDelta16LO) {
+    case ppc64::RequestTLSDescInGOTAndTransformToTOCDelta16LO:
       E.setKind(ppc64::TOCDelta16LO);
       E.setTarget(this->getEntryForTarget(G, E.getTarget()));
       return true;
+    case ppc64::RequestTLSDescInGOTAndTransformToDelta34:
+      E.setKind(ppc64::Delta34);
+      E.setTarget(this->getEntryForTarget(G, E.getTarget()));
+      return true;
+    default:
+      return false;
     }
-    return false;
   }
 
   Symbol &createEntry(LinkGraph &G, Symbol &Target) {
@@ -84,19 +89,19 @@ class TLSInfoTableManager_ELF_ppc64
 
 template <>
 const uint8_t TLSInfoTableManager_ELF_ppc64<
-    support::endianness::little>::TLSInfoEntryContent[16] = {
+    llvm::endianness::little>::TLSInfoEntryContent[16] = {
     0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /*pthread key */
     0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00  /*data address*/
 };
 
 template <>
 const uint8_t TLSInfoTableManager_ELF_ppc64<
-    support::endianness::big>::TLSInfoEntryContent[16] = {
+    llvm::endianness::big>::TLSInfoEntryContent[16] = {
     0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /*pthread key */
     0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00  /*data address*/
 };
 
-template <support::endianness Endianness>
+template <llvm::endianness Endianness>
 Symbol &createELFGOTHeader(LinkGraph &G,
                            ppc64::TOCTableManager<Endianness> &TOC) {
   Symbol *TOCSymbol = nullptr;
@@ -122,7 +127,7 @@ Symbol &createELFGOTHeader(LinkGraph &G,
 }
 
 // Register preexisting GOT entries with TOC table manager.
-template <support::endianness Endianness>
+template <llvm::endianness Endianness>
 inline void
 registerExistingGOTEntries(LinkGraph &G,
                            ppc64::TOCTableManager<Endianness> &TOC) {
@@ -140,7 +145,7 @@ registerExistingGOTEntries(LinkGraph &G,
   }
 }
 
-template <support::endianness Endianness>
+template <llvm::endianness Endianness>
 Error buildTables_ELF_ppc64(LinkGraph &G) {
   LLVM_DEBUG(dbgs() << "Visiting edges in graph:\n");
   ppc64::TOCTableManager<Endianness> TOC;
@@ -189,7 +194,7 @@ Error buildTables_ELF_ppc64(LinkGraph &G) {
 
 namespace llvm::jitlink {
 
-template <support::endianness Endianness>
+template <llvm::endianness Endianness>
 class ELFLinkGraphBuilder_ppc64
     : public ELFLinkGraphBuilder<object::ELFType<Endianness, true>> {
 private:
@@ -235,6 +240,14 @@ class ELFLinkGraphBuilder_ppc64
       return make_error<StringError>("Local-dynamic TLS model is not supported",
                                      inconvertibleErrorCode());
 
+    if (ELFReloc == ELF::R_PPC64_PCREL_OPT)
+      // TODO: Support PCREL optimization, now ignore it.
+      return Error::success();
+
+    if (ELFReloc == ELF::R_PPC64_TPREL34)
+      return make_error<StringError>("Local-exec TLS model is not supported",
+                                     inconvertibleErrorCode());
+
     auto ObjSymbol = Base::Obj.getRelocationSymbol(Rel, Base::SymTabSec);
     if (!ObjSymbol)
       return ObjSymbol.takeError();
@@ -360,12 +373,18 @@ class ELFLinkGraphBuilder_ppc64
     case ELF::R_PPC64_PCREL34:
       Kind = ppc64::Delta34;
       break;
+    case ELF::R_PPC64_GOT_PCREL34:
+      Kind = ppc64::RequestGOTAndTransformToDelta34;
+      break;
     case ELF::R_PPC64_GOT_TLSGD16_HA:
       Kind = ppc64::RequestTLSDescInGOTAndTransformToTOCDelta16HA;
       break;
     case ELF::R_PPC64_GOT_TLSGD16_LO:
       Kind = ppc64::RequestTLSDescInGOTAndTransformToTOCDelta16LO;
       break;
+    case ELF::R_PPC64_GOT_TLSGD_PCREL34:
+      Kind = ppc64::RequestTLSDescInGOTAndTransformToDelta34;
+      break;
     }
 
     Edge GE(Kind, Offset, *GraphSymbol, Addend);
@@ -381,7 +400,7 @@ class ELFLinkGraphBuilder_ppc64
                                   FileName, ppc64::getEdgeKindName) {}
 };
 
-template <support::endianness Endianness>
+template <llvm::endianness Endianness>
 class ELFJITLinker_ppc64 : public JITLinker<ELFJITLinker_ppc64<Endianness>> {
   using JITLinkerBase = JITLinker<ELFJITLinker_ppc64<Endianness>>;
   friend JITLinkerBase;
@@ -443,7 +462,7 @@ class ELFJITLinker_ppc64 : public JITLinker<ELFJITLinker_ppc64<Endianness>> {
   }
 };
 
-template <support::endianness Endianness>
+template <llvm::endianness Endianness>
 Expected<std::unique_ptr<LinkGraph>>
 createLinkGraphFromELFObject_ppc64(MemoryBufferRef ObjectBuffer) {
   LLVM_DEBUG({
@@ -467,7 +486,7 @@ createLinkGraphFromELFObject_ppc64(MemoryBufferRef ObjectBuffer) {
       .buildGraph();
 }
 
-template <support::endianness Endianness>
+template <llvm::endianness Endianness>
 void link_ELF_ppc64(std::unique_ptr<LinkGraph> G,
                     std::unique_ptr<JITLinkContext> Ctx) {
   PassConfiguration Config;
@@ -500,26 +519,26 @@ void link_ELF_ppc64(std::unique_ptr<LinkGraph> G,
 
 Expected<std::unique_ptr<LinkGraph>>
 createLinkGraphFromELFObject_ppc64(MemoryBufferRef ObjectBuffer) {
-  return createLinkGraphFromELFObject_ppc64<support::big>(
+  return createLinkGraphFromELFObject_ppc64<llvm::endianness::big>(
       std::move(ObjectBuffer));
 }
 
 Expected<std::unique_ptr<LinkGraph>>
 createLinkGraphFromELFObject_ppc64le(MemoryBufferRef ObjectBuffer) {
-  return createLinkGraphFromELFObject_ppc64<support::little>(
+  return createLinkGraphFromELFObject_ppc64<llvm::endianness::little>(
       std::move(ObjectBuffer));
 }
 
 /// jit-link the given object buffer, which must be a ELF ppc64 object file.
 void link_ELF_ppc64(std::unique_ptr<LinkGraph> G,
                     std::unique_ptr<JITLinkContext> Ctx) {
-  return link_ELF_ppc64<support::big>(std::move(G), std::move(Ctx));
+  return link_ELF_ppc64<llvm::endianness::big>(std::move(G), std::move(Ctx));
 }
 
 /// jit-link the given object buffer, which must be a ELF ppc64le object file.
 void link_ELF_ppc64le(std::unique_ptr<LinkGraph> G,
                       std::unique_ptr<JITLinkContext> Ctx) {
-  return link_ELF_ppc64<support::little>(std::move(G), std::move(Ctx));
+  return link_ELF_ppc64<llvm::endianness::little>(std::move(G), std::move(Ctx));
 }
 
 } // end namespace llvm::jitlink
diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp
index 3dc2636a44..46f8064bb1 100644
--- a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp
@@ -16,7 +16,6 @@
 #include "llvm/ExecutionEngine/JITLink/TableManager.h"
 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
 #include "llvm/Object/ELFObjectFile.h"
-#include "llvm/Support/Endian.h"
 
 #include "DefineExternalSectionStartAndEndSymbols.h"
 #include "EHFrameSupportImpl.h"
diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp
index 57e17aa78f..474a0b5160 100644
--- a/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp
@@ -155,8 +155,8 @@ void SimpleSegmentAlloc::Create(JITLinkMemoryManager &MemMgr,
       "__---.finalize", "__R--.finalize", "__-W-.finalize", "__RW-.finalize",
       "__--X.finalize", "__R-X.finalize", "__-WX.finalize", "__RWX.finalize"};
 
-  auto G =
-      std::make_unique<LinkGraph>("", Triple(), 0, support::native, nullptr);
+  auto G = std::make_unique<LinkGraph>("", Triple(), 0,
+                                       llvm::endianness::native, nullptr);
   orc::AllocGroupSmallMap<Block *> ContentBlocks;
 
   orc::ExecutorAddr NextAddr(0x100000);
diff --git a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp
index 0927f2ff4e..bcbc429cae 100644
--- a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp
@@ -106,9 +106,10 @@ MachOLinkGraphBuilder::getPointerSize(const object::MachOObjectFile &Obj) {
   return Obj.is64Bit() ? 8 : 4;
 }
 
-support::endianness
+llvm::endianness
 MachOLinkGraphBuilder::getEndianness(const object::MachOObjectFile &Obj) {
-  return Obj.isLittleEndian() ? support::little : support::big;
+  return Obj.isLittleEndian() ? llvm::endianness::little
+                              : llvm::endianness::big;
 }
 
 Section &MachOLinkGraphBuilder::getCommonSection() {
diff --git a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h
index 2805c2960b..a4ae0ac1ec 100644
--- a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h
+++ b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h
@@ -181,7 +181,7 @@ class MachOLinkGraphBuilder {
 
 private:
   static unsigned getPointerSize(const object::MachOObjectFile &Obj);
-  static support::endianness getEndianness(const object::MachOObjectFile &Obj);
+  static llvm::endianness getEndianness(const object::MachOObjectFile &Obj);
 
   void setCanonicalSymbol(NormalizedSection &NSec, Symbol &Sym) {
     auto *&CanonicalSymEntry = NSec.CanonicalSymbols[Sym.getAddress()];
diff --git a/llvm/lib/ExecutionEngine/JITLink/aarch32.cpp b/llvm/lib/ExecutionEngine/JITLink/aarch32.cpp
index 124955236c..4aed649666 100644
--- a/llvm/lib/ExecutionEngine/JITLink/aarch32.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/aarch32.cpp
@@ -305,7 +305,7 @@ void writeImmediate(WritableArmRelocation &R, uint32_t Imm) {
 }
 
 Expected<int64_t> readAddendData(LinkGraph &G, Block &B, const Edge &E) {
-  support::endianness Endian = G.getEndianness();
+  llvm::endianness Endian = G.getEndianness();
 
   Edge::Kind Kind = E.getKind();
   const char *BlockWorkingMem = B.getContent().data();
@@ -405,10 +405,10 @@ Error applyFixupData(LinkGraph &G, Block &B, const Edge &E) {
   auto Write32 = [FixupPtr, Endian = G.getEndianness()](int64_t Value) {
     assert(isInt<32>(Value) && "Must be in signed 32-bit range");
     uint32_t Imm = static_cast<int32_t>(Value);
-    if (LLVM_LIKELY(Endian == little))
-      endian::write32<little>(FixupPtr, Imm);
+    if (LLVM_LIKELY(Endian == llvm::endianness::little))
+      endian::write32<llvm::endianness::little>(FixupPtr, Imm);
     else
-      endian::write32<big>(FixupPtr, Imm);
+      endian::write32<llvm::endianness::big>(FixupPtr, Imm);
   };
 
   Edge::Kind Kind = E.getKind();
diff --git a/llvm/lib/ExecutionEngine/JITLink/ppc64.cpp b/llvm/lib/ExecutionEngine/JITLink/ppc64.cpp
index b147ffc8da..27484aaf20 100644
--- a/llvm/lib/ExecutionEngine/JITLink/ppc64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/ppc64.cpp
@@ -120,6 +120,8 @@ const char *getEdgeKindName(Edge::Kind K) {
     return "TOCDelta16LO";
   case TOCDelta16LODS:
     return "TOCDelta16LODS";
+  case RequestGOTAndTransformToDelta34:
+    return "RequestGOTAndTransformToDelta34";
   case CallBranchDelta:
     return "CallBranchDelta";
   case CallBranchDeltaRestoreTOC:
@@ -132,6 +134,8 @@ const char *getEdgeKindName(Edge::Kind K) {
     return "RequestTLSDescInGOTAndTransformToTOCDelta16HA";
   case RequestTLSDescInGOTAndTransformToTOCDelta16LO:
     return "RequestTLSDescInGOTAndTransformToTOCDelta16LO";
+  case RequestTLSDescInGOTAndTransformToDelta34:
+    return "RequestTLSDescInGOTAndTransformToDelta34";
   default:
     return getGenericEdgeKindName(static_cast<Edge::Kind>(K));
   }
diff --git a/llvm/lib/ExecutionEngine/Orc/COFFPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/COFFPlatform.cpp
index 7c869bead0..c8f5a99099 100644
--- a/llvm/lib/ExecutionEngine/Orc/COFFPlatform.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/COFFPlatform.cpp
@@ -54,13 +54,13 @@ class COFFHeaderMaterializationUnit : public MaterializationUnit {
 
   void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
     unsigned PointerSize;
-    support::endianness Endianness;
+    llvm::endianness Endianness;
     const auto &TT = CP.getExecutionSession().getTargetTriple();
 
     switch (TT.getArch()) {
     case Triple::x86_64:
       PointerSize = 8;
-      Endianness = support::endianness::little;
+      Endianness = llvm::endianness::little;
       break;
     default:
       llvm_unreachable("Unrecognized architecture");
diff --git a/llvm/lib/ExecutionEngine/Orc/Core.cpp b/llvm/lib/ExecutionEngine/Orc/Core.cpp
index f4c0ecf784..56838e9bc8 100644
--- a/llvm/lib/ExecutionEngine/Orc/Core.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Core.cpp
@@ -1353,7 +1353,7 @@ void JITDylib::addToLinkOrder(const JITDylibSearchOrder &NewLinks) {
   ES.runSessionLocked([&]() {
     for (auto &KV : NewLinks) {
       // Skip elements of NewLinks that are already in the link order.
-      if (llvm::find(LinkOrder, KV) != LinkOrder.end())
+      if (llvm::is_contained(LinkOrder, KV))
         continue;
 
       LinkOrder.push_back(std::move(KV));
diff --git a/llvm/lib/ExecutionEngine/Orc/Debugging/DebugInfoSupport.cpp b/llvm/lib/ExecutionEngine/Orc/Debugging/DebugInfoSupport.cpp
index febd2e73aa..b541db3672 100644
--- a/llvm/lib/ExecutionEngine/Orc/Debugging/DebugInfoSupport.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Debugging/DebugInfoSupport.cpp
@@ -113,8 +113,9 @@ llvm::orc::createDWARFContext(LinkGraph &G) {
           std::make_unique<SmallVectorMemoryBuffer>(std::move(SecData));
     }
   }
-  auto Ctx = DWARFContext::create(DWARFSectionData, G.getPointerSize(),
-                                  G.getEndianness() == support::little);
+  auto Ctx =
+      DWARFContext::create(DWARFSectionData, G.getPointerSize(),
+                           G.getEndianness() == llvm::endianness::little);
   dumpDWARFContext(*Ctx);
   return std::make_pair(std::move(Ctx), std::move(DWARFSectionData));
 }
diff --git a/llvm/lib/ExecutionEngine/Orc/Debugging/DebuggerSupportPlugin.cpp b/llvm/lib/ExecutionEngine/Orc/Debugging/DebuggerSupportPlugin.cpp
index 2f70a11944..cdc1158ce1 100644
--- a/llvm/lib/ExecutionEngine/Orc/Debugging/DebuggerSupportPlugin.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Debugging/DebuggerSupportPlugin.cpp
@@ -158,15 +158,15 @@ class MachODebugObjectSynthesizer : public MachODebugObjectSynthesizerBase {
 
     std::optional<StringRef> FileName;
     if (!DebugLineSectionData.empty()) {
-      assert((G.getEndianness() == support::endianness::big ||
-              G.getEndianness() == support::endianness::little) &&
+      assert((G.getEndianness() == llvm::endianness::big ||
+              G.getEndianness() == llvm::endianness::little) &&
              "G.getEndianness() must be either big or little");
-      auto DWARFCtx = DWARFContext::create(DebugSectionMap, G.getPointerSize(),
-                                           G.getEndianness() ==
-                                               support::endianness::little);
+      auto DWARFCtx =
+          DWARFContext::create(DebugSectionMap, G.getPointerSize(),
+                               G.getEndianness() == llvm::endianness::little);
       DWARFDataExtractor DebugLineData(
-          DebugLineSectionData,
-          G.getEndianness() == support::endianness::little, G.getPointerSize());
+          DebugLineSectionData, G.getEndianness() == llvm::endianness::little,
+          G.getPointerSize());
       uint64_t Offset = 0;
       DWARFDebugLine::LineTable LineTable;
 
@@ -374,7 +374,7 @@ void GDBJITDebugInfoRegistrationPlugin::modifyPassConfigForMachO(
   case Triple::aarch64:
     // Supported, continue.
     assert(LG.getPointerSize() == 8 && "Graph has incorrect pointer size");
-    assert(LG.getEndianness() == support::little &&
+    assert(LG.getEndianness() == llvm::endianness::little &&
            "Graph has incorrect endianness");
     break;
   default:
@@ -384,7 +384,7 @@ void GDBJITDebugInfoRegistrationPlugin::modifyPassConfigForMachO(
              << "MachO graph " << LG.getName()
              << "(triple = " << LG.getTargetTriple().str()
              << ", pointer size = " << LG.getPointerSize() << ", endianness = "
-             << (LG.getEndianness() == support::big ? "big" : "little")
+             << (LG.getEndianness() == llvm::endianness::big ? "big" : "little")
              << ")\n";
     });
     return;
diff --git a/llvm/lib/ExecutionEngine/Orc/Debugging/PerfSupportPlugin.cpp b/llvm/lib/ExecutionEngine/Orc/Debugging/PerfSupportPlugin.cpp
index 1c5b9c6078..fffecfc978 100644
--- a/llvm/lib/ExecutionEngine/Orc/Debugging/PerfSupportPlugin.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Debugging/PerfSupportPlugin.cpp
@@ -29,7 +29,7 @@ namespace {
 // to the start of the .eh_frame section. Absolute injects a 64-bit absolute
 // address space offset 4 bytes from the start instead of 4 bytes
 Expected<std::string> createX64EHFrameHeader(Section &EHFrame,
-                                             support::endianness endianness,
+                                             llvm::endianness endianness,
                                              bool absolute) {
   uint8_t Version = 1;
   uint8_t EhFramePtrEnc = 0;
diff --git a/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp
index c08b8b037f..2b6c4b9e7f 100644
--- a/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp
@@ -40,29 +40,29 @@ class DSOHandleMaterializationUnit : public MaterializationUnit {
 
   void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
     unsigned PointerSize;
-    support::endianness Endianness;
+    llvm::endianness Endianness;
     jitlink::Edge::Kind EdgeKind;
     const auto &TT = ENP.getExecutionSession().getTargetTriple();
 
     switch (TT.getArch()) {
     case Triple::x86_64:
       PointerSize = 8;
-      Endianness = support::endianness::little;
+      Endianness = llvm::endianness::little;
       EdgeKind = jitlink::x86_64::Pointer64;
       break;
     case Triple::aarch64:
       PointerSize = 8;
-      Endianness = support::endianness::little;
+      Endianness = llvm::endianness::little;
       EdgeKind = jitlink::aarch64::Pointer64;
       break;
     case Triple::ppc64:
       PointerSize = 8;
-      Endianness = support::endianness::big;
+      Endianness = llvm::endianness::big;
       EdgeKind = jitlink::ppc64::Pointer64;
       break;
     case Triple::ppc64le:
       PointerSize = 8;
-      Endianness = support::endianness::little;
+      Endianness = llvm::endianness::little;
       EdgeKind = jitlink::ppc64::Pointer64;
       break;
     default:
diff --git a/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
index 8ed9cadd2a..eb8e6825fa 100644
--- a/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
@@ -538,11 +538,11 @@ DLLImportDefinitionGenerator::getTargetPointerSize(const Triple &TT) {
   }
 }
 
-Expected<support::endianness>
+Expected<llvm::endianness>
 DLLImportDefinitionGenerator::getTargetEndianness(const Triple &TT) {
   switch (TT.getArch()) {
   case Triple::x86_64:
-    return support::endianness::little;
+    return llvm::endianness::little;
   default:
     return make_error<StringError>(
         "architecture unsupported by DLLImportDefinitionGenerator",
diff --git a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
index 6b73366f3d..abc0bbbcaa 100644
--- a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp
@@ -64,14 +64,14 @@ namespace {
 std::unique_ptr<jitlink::LinkGraph> createPlatformGraph(MachOPlatform &MOP,
                                                         std::string Name) {
   unsigned PointerSize;
-  support::endianness Endianness;
+  llvm::endianness Endianness;
   const auto &TT = MOP.getExecutionSession().getTargetTriple();
 
   switch (TT.getArch()) {
   case Triple::aarch64:
   case Triple::x86_64:
     PointerSize = 8;
-    Endianness = support::endianness::little;
+    Endianness = llvm::endianness::little;
     break;
   default:
     llvm_unreachable("Unrecognized architecture");
diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp
index e99f637bb4..68497305f0 100644
--- a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp
+++ b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp
@@ -806,7 +806,7 @@ class RuntimeDyldCheckerExprEval {
 RuntimeDyldCheckerImpl::RuntimeDyldCheckerImpl(
     IsSymbolValidFunction IsSymbolValid, GetSymbolInfoFunction GetSymbolInfo,
     GetSectionInfoFunction GetSectionInfo, GetStubInfoFunction GetStubInfo,
-    GetGOTInfoFunction GetGOTInfo, support::endianness Endianness, Triple TT,
+    GetGOTInfoFunction GetGOTInfo, llvm::endianness Endianness, Triple TT,
     StringRef CPU, SubtargetFeatures TF, raw_ostream &ErrStream)
     : IsSymbolValid(std::move(IsSymbolValid)),
       GetSymbolInfo(std::move(GetSymbolInfo)),
@@ -1016,7 +1016,7 @@ std::pair<uint64_t, std::string> RuntimeDyldCheckerImpl::getStubOrGOTAddrFor(
 RuntimeDyldChecker::RuntimeDyldChecker(
     IsSymbolValidFunction IsSymbolValid, GetSymbolInfoFunction GetSymbolInfo,
     GetSectionInfoFunction GetSectionInfo, GetStubInfoFunction GetStubInfo,
-    GetGOTInfoFunction GetGOTInfo, support::endianness Endianness, Triple TT,
+    GetGOTInfoFunction GetGOTInfo, llvm::endianness Endianness, Triple TT,
     StringRef CPU, SubtargetFeatures TF, raw_ostream &ErrStream)
     : Impl(::std::make_unique<RuntimeDyldCheckerImpl>(
           std::move(IsSymbolValid), std::move(GetSymbolInfo),
diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h
index d2a969b0f2..9f44a9389f 100644
--- a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h
+++ b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h
@@ -28,11 +28,13 @@ class RuntimeDyldCheckerImpl {
   using GetGOTInfoFunction = RuntimeDyldChecker::GetGOTInfoFunction;
 
 public:
-  RuntimeDyldCheckerImpl(
-      IsSymbolValidFunction IsSymbolValid, GetSymbolInfoFunction GetSymbolInfo,
-      GetSectionInfoFunction GetSectionInfo, GetStubInfoFunction GetStubInfo,
-      GetGOTInfoFunction GetGOTInfo, support::endianness Endianness, Triple TT,
-      StringRef CPU, SubtargetFeatures TF, llvm::raw_ostream &ErrStream);
+  RuntimeDyldCheckerImpl(IsSymbolValidFunction IsSymbolValid,
+                         GetSymbolInfoFunction GetSymbolInfo,
+                         GetSectionInfoFunction GetSectionInfo,
+                         GetStubInfoFunction GetStubInfo,
+                         GetGOTInfoFunction GetGOTInfo,
+                         llvm::endianness Endianness, Triple TT, StringRef CPU,
+                         SubtargetFeatures TF, llvm::raw_ostream &ErrStream);
 
   bool check(StringRef CheckExpr) const;
   bool checkAllRulesInBuffer(StringRef RulePrefix, MemoryBuffer *MemBuf) const;
@@ -71,7 +73,7 @@ class RuntimeDyldCheckerImpl {
   GetSectionInfoFunction GetSectionInfo;
   GetStubInfoFunction GetStubInfo;
   GetGOTInfoFunction GetGOTInfo;
-  support::endianness Endianness;
+  llvm::endianness Endianness;
   Triple TT;
   std::string CPU;
   SubtargetFeatures TF;
diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp
index d439b1b4eb..9fdabf310d 100644
--- a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp
+++ b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp
@@ -35,7 +35,8 @@ static void or32AArch64Imm(void *L, uint64_t Imm) {
 }
 
 template <class T> static void write(bool isBE, void *P, T V) {
-  isBE ? write<T, support::big>(P, V) : write<T, support::little>(P, V);
+  isBE ? write<T, llvm::endianness::big>(P, V)
+       : write<T, llvm::endianness::little>(P, V);
 }
 
 static void write32AArch64Addr(void *L, uint64_t Imm) {
diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h
index 1ae3ac1797..73e2b365f1 100644
--- a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h
+++ b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h
@@ -318,18 +318,24 @@ class RuntimeDyldImpl {
   std::string ErrorStr;
 
   void writeInt16BE(uint8_t *Addr, uint16_t Value) {
-    llvm::support::endian::write<uint16_t>(
-        Addr, Value, IsTargetLittleEndian ? support::little : support::big);
+    llvm::support::endian::write<uint16_t>(Addr, Value,
+                                           IsTargetLittleEndian
+                                               ? llvm::endianness::little
+                                               : llvm::endianness::big);
   }
 
   void writeInt32BE(uint8_t *Addr, uint32_t Value) {
-    llvm::support::endian::write<uint32_t>(
-        Addr, Value, IsTargetLittleEndian ? support::little : support::big);
+    llvm::support::endian::write<uint32_t>(Addr, Value,
+                                           IsTargetLittleEndian
+                                               ? llvm::endianness::little
+                                               : llvm::endianness::big);
   }
 
   void writeInt64BE(uint8_t *Addr, uint64_t Value) {
-    llvm::support::endian::write<uint64_t>(
-        Addr, Value, IsTargetLittleEndian ? support::little : support::big);
+    llvm::support::endian::write<uint64_t>(Addr, Value,
+                                           IsTargetLittleEndian
+                                               ? llvm::endianness::little
+                                               : llvm::endianness::big);
   }
 
   virtual void setMipsABI(const ObjectFile &Obj) {
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 8e240ba960..5b24e9fe2e 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -5733,7 +5733,9 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCompare(
 
 OpenMPIRBuilder::InsertPointTy
 OpenMPIRBuilder::createTeams(const LocationDescription &Loc,
-                             BodyGenCallbackTy BodyGenCB) {
+                             BodyGenCallbackTy BodyGenCB, Value *NumTeamsLower,
+                             Value *NumTeamsUpper, Value *ThreadLimit,
+                             Value *IfExpr) {
   if (!updateToLocation(Loc))
     return InsertPointTy();
 
@@ -5771,6 +5773,42 @@ OpenMPIRBuilder::createTeams(const LocationDescription &Loc,
   BasicBlock *AllocaBB =
       splitBB(Builder, /*CreateBranch=*/true, "teams.alloca");
 
+  // Push num_teams
+  if (NumTeamsLower || NumTeamsUpper || ThreadLimit || IfExpr) {
+    assert((NumTeamsLower == nullptr || NumTeamsUpper != nullptr) &&
+           "if lowerbound is non-null, then upperbound must also be non-null "
+           "for bounds on num_teams");
+
+    if (NumTeamsUpper == nullptr)
+      NumTeamsUpper = Builder.getInt32(0);
+
+    if (NumTeamsLower == nullptr)
+      NumTeamsLower = NumTeamsUpper;
+
+    if (IfExpr) {
+      assert(IfExpr->getType()->isIntegerTy() &&
+             "argument to if clause must be an integer value");
+
+      // upper = ifexpr ? upper : 1
+      if (IfExpr->getType() != Int1)
+        IfExpr = Builder.CreateICmpNE(IfExpr,
+                                      ConstantInt::get(IfExpr->getType(), 0));
+      NumTeamsUpper = Builder.CreateSelect(
+          IfExpr, NumTeamsUpper, Builder.getInt32(1), "numTeamsUpper");
+
+      // lower = ifexpr ? lower : 1
+      NumTeamsLower = Builder.CreateSelect(
+          IfExpr, NumTeamsLower, Builder.getInt32(1), "numTeamsLower");
+    }
+
+    if (ThreadLimit == nullptr)
+      ThreadLimit = Builder.getInt32(0);
+
+    Value *ThreadNum = getOrCreateThreadID(Ident);
+    Builder.CreateCall(
+        getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_push_num_teams_51),
+        {Ident, ThreadNum, NumTeamsLower, NumTeamsUpper, ThreadLimit});
+  }
   // Generate the body of teams.
   InsertPointTy AllocaIP(AllocaBB, AllocaBB->begin());
   InsertPointTy CodeGenIP(BodyBB, BodyBB->begin());
diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp
index e190d82127..bd8b3e9ad5 100644
--- a/llvm/lib/IR/AsmWriter.cpp
+++ b/llvm/lib/IR/AsmWriter.cpp
@@ -350,6 +350,7 @@ static void PrintCallingConv(unsigned cc, raw_ostream &Out) {
     break;
   case CallingConv::AMDGPU_KERNEL: Out << "amdgpu_kernel"; break;
   case CallingConv::AMDGPU_Gfx:    Out << "amdgpu_gfx"; break;
+  case CallingConv::M68k_RTD:      Out << "m68k_rtdcc"; break;
   }
 }
 
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 304edc2c3a..e102aae525 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -5201,13 +5201,29 @@ std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) {
   // If the datalayout matches the expected format, add pointer size address
   // spaces to the datalayout.
   std::string AddrSpaces = "-p270:32:32-p271:32:32-p272:64:64";
-  if (!DL.contains(AddrSpaces)) {
+  if (StringRef Ref = Res; !Ref.contains(AddrSpaces)) {
     SmallVector<StringRef, 4> Groups;
     Regex R("(e-m:[a-z](-p:32:32)?)(-[if]64:.*$)");
-    if (R.match(DL, &Groups))
+    if (R.match(Res, &Groups))
       Res = (Groups[1] + AddrSpaces + Groups[3]).str();
   }
 
+  // i128 values need to be 16-byte-aligned. LLVM already called into libgcc
+  // for i128 operations prior to this being reflected in the data layout, and
+  // clang mostly produced LLVM IR that already aligned i128 to 16 byte
+  // boundaries, so although this is a breaking change, the upgrade is expected
+  // to fix more IR than it breaks.
+  // Intel MCU is an exception and uses 4-byte-alignment.
+  if (!T.isOSIAMCU()) {
+    std::string I128 = "-i128:128";
+    if (StringRef Ref = Res; !Ref.contains(I128)) {
+      SmallVector<StringRef, 4> Groups;
+      Regex R("^(e(-[mpi][^-]*)*)((-[^mpi][^-]*)*)$");
+      if (R.match(Res, &Groups))
+        Res = (Groups[1] + I128 + Groups[3]).str();
+    }
+  }
+
   // For 32-bit MSVC targets, raise the alignment of f80 values to 16 bytes.
   // Raising the alignment is safe because Clang did not produce f80 values in
   // the MSVC environment before this upgrade was added.
diff --git a/llvm/lib/IR/BasicBlock.cpp b/llvm/lib/IR/BasicBlock.cpp
index d6677aa721..46b1a3b371 100644
--- a/llvm/lib/IR/BasicBlock.cpp
+++ b/llvm/lib/IR/BasicBlock.cpp
@@ -42,7 +42,8 @@ template <> void llvm::invalidateParentIListOrdering(BasicBlock *BB) {
 
 // Explicit instantiation of SymbolTableListTraits since some of the methods
 // are not in the public header file...
-template class llvm::SymbolTableListTraits<Instruction>;
+template class llvm::SymbolTableListTraits<Instruction,
+                                           ilist_iterator_bits<true>>;
 
 BasicBlock::BasicBlock(LLVMContext &C, const Twine &Name, Function *NewParent,
                        BasicBlock *InsertBefore)
@@ -221,7 +222,13 @@ const Instruction* BasicBlock::getFirstNonPHI() const {
 }
 
 BasicBlock::const_iterator BasicBlock::getFirstNonPHIIt() const {
-  return getFirstNonPHI()->getIterator();
+  const Instruction *I = getFirstNonPHI();
+  BasicBlock::const_iterator It = I->getIterator();
+  // Set the head-inclusive bit to indicate that this iterator includes
+  // any debug-info at the start of the block. This is a no-op unless the
+  // appropriate CMake flag is set.
+  It.setHeadBit(true);
+  return It;
 }
 
 const Instruction *BasicBlock::getFirstNonPHIOrDbg(bool SkipPseudoOp) const {
@@ -261,6 +268,10 @@ BasicBlock::const_iterator BasicBlock::getFirstInsertionPt() const {
 
   const_iterator InsertPt = FirstNonPHI->getIterator();
   if (InsertPt->isEHPad()) ++InsertPt;
+  // Set the head-inclusive bit to indicate that this iterator includes
+  // any debug-info at the start of the block. This is a no-op unless the
+  // appropriate CMake flag is set.
+  InsertPt.setHeadBit(true);
   return InsertPt;
 }
 
diff --git a/llvm/lib/IR/Instruction.cpp b/llvm/lib/IR/Instruction.cpp
index b497951a59..9b176eb788 100644
--- a/llvm/lib/IR/Instruction.cpp
+++ b/llvm/lib/IR/Instruction.cpp
@@ -80,7 +80,7 @@ void Instruction::removeFromParent() {
   getParent()->getInstList().remove(getIterator());
 }
 
-iplist<Instruction>::iterator Instruction::eraseFromParent() {
+BasicBlock::iterator Instruction::eraseFromParent() {
   return getParent()->getInstList().erase(getIterator());
 }
 
@@ -114,8 +114,7 @@ void Instruction::moveAfter(Instruction *MovePos) {
   moveBefore(*MovePos->getParent(), ++MovePos->getIterator());
 }
 
-void Instruction::moveBefore(BasicBlock &BB,
-                             SymbolTableList<Instruction>::iterator I) {
+void Instruction::moveBefore(BasicBlock &BB, InstListType::iterator I) {
   assert(I == BB.end() || I->getParent() == &BB);
   BB.splice(I, getParent(), getIterator());
 }
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index ece3b58792..2ea9c05de6 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -4606,7 +4606,7 @@ void SwitchInstProfUpdateWrapper::addCase(
            "num of prof branch_weights must accord with num of successors");
 }
 
-SymbolTableList<Instruction>::iterator
+Instruction::InstListType::iterator
 SwitchInstProfUpdateWrapper::eraseFromParent() {
   // Instruction is erased. Mark as unchanged to not touch it in the destructor.
   Changed = false;
diff --git a/llvm/lib/IR/ReplaceConstant.cpp b/llvm/lib/IR/ReplaceConstant.cpp
index 58aa040eb0..42dec7c723 100644
--- a/llvm/lib/IR/ReplaceConstant.cpp
+++ b/llvm/lib/IR/ReplaceConstant.cpp
@@ -22,24 +22,29 @@ static bool isExpandableUser(User *U) {
   return isa<ConstantExpr>(U) || isa<ConstantAggregate>(U);
 }
 
-static Instruction *expandUser(Instruction *InsertPt, Constant *C) {
+static SmallVector<Instruction *, 4> expandUser(Instruction *InsertPt,
+                                                Constant *C) {
+  SmallVector<Instruction *, 4> NewInsts;
   if (auto *CE = dyn_cast<ConstantExpr>(C)) {
-    return CE->getAsInstruction(InsertPt);
+    NewInsts.push_back(CE->getAsInstruction(InsertPt));
   } else if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
     Value *V = PoisonValue::get(C->getType());
-    for (auto [Idx, Op] : enumerate(C->operands()))
+    for (auto [Idx, Op] : enumerate(C->operands())) {
       V = InsertValueInst::Create(V, Op, Idx, "", InsertPt);
-    return cast<Instruction>(V);
+      NewInsts.push_back(cast<Instruction>(V));
+    }
   } else if (isa<ConstantVector>(C)) {
     Type *IdxTy = Type::getInt32Ty(C->getContext());
     Value *V = PoisonValue::get(C->getType());
-    for (auto [Idx, Op] : enumerate(C->operands()))
+    for (auto [Idx, Op] : enumerate(C->operands())) {
       V = InsertElementInst::Create(V, Op, ConstantInt::get(IdxTy, Idx), "",
                                     InsertPt);
-    return cast<Instruction>(V);
+      NewInsts.push_back(cast<Instruction>(V));
+    }
   } else {
     llvm_unreachable("Not an expandable user");
   }
+  return NewInsts;
 }
 
 bool convertUsersOfConstantsToInstructions(ArrayRef<Constant *> Consts) {
@@ -73,6 +78,7 @@ bool convertUsersOfConstantsToInstructions(ArrayRef<Constant *> Consts) {
   bool Changed = false;
   while (!InstructionWorklist.empty()) {
     Instruction *I = InstructionWorklist.pop_back_val();
+    DebugLoc Loc = I->getDebugLoc();
     for (Use &U : I->operands()) {
       auto *BI = I;
       if (auto *Phi = dyn_cast<PHINode>(I)) {
@@ -85,9 +91,11 @@ bool convertUsersOfConstantsToInstructions(ArrayRef<Constant *> Consts) {
       if (auto *C = dyn_cast<Constant>(U.get())) {
         if (ExpandableUsers.contains(C)) {
           Changed = true;
-          Instruction *NI = expandUser(BI, C);
-          InstructionWorklist.insert(NI);
-          U.set(NI);
+          auto NewInsts = expandUser(BI, C);
+          for (auto *NI : NewInsts)
+            NI->setDebugLoc(Loc);
+          InstructionWorklist.insert(NewInsts.begin(), NewInsts.end());
+          U.set(NewInsts.back());
         }
       }
     }
diff --git a/llvm/lib/IR/SymbolTableListTraitsImpl.h b/llvm/lib/IR/SymbolTableListTraitsImpl.h
index 4283744bd0..990552f9b6 100644
--- a/llvm/lib/IR/SymbolTableListTraitsImpl.h
+++ b/llvm/lib/IR/SymbolTableListTraitsImpl.h
@@ -28,10 +28,10 @@ template <> void invalidateParentIListOrdering(BasicBlock *BB);
 /// setSymTabObject - This is called when (f.e.) the parent of a basic block
 /// changes.  This requires us to remove all the instruction symtab entries from
 /// the current function and reinsert them into the new function.
-template <typename ValueSubClass>
+template <typename ValueSubClass, typename... Args>
 template <typename TPtr>
-void SymbolTableListTraits<ValueSubClass>::setSymTabObject(TPtr *Dest,
-                                                           TPtr Src) {
+void SymbolTableListTraits<ValueSubClass, Args...>::setSymTabObject(TPtr *Dest,
+                                                                    TPtr Src) {
   // Get the old symtab and value list before doing the assignment.
   ValueSymbolTable *OldST = getSymTab(getListOwner());
 
@@ -61,11 +61,11 @@ void SymbolTableListTraits<ValueSubClass>::setSymTabObject(TPtr *Dest,
       if (I->hasName())
         NewST->reinsertValue(&*I);
   }
-
 }
 
-template <typename ValueSubClass>
-void SymbolTableListTraits<ValueSubClass>::addNodeToList(ValueSubClass *V) {
+template <typename ValueSubClass, typename... Args>
+void SymbolTableListTraits<ValueSubClass, Args...>::addNodeToList(
+    ValueSubClass *V) {
   assert(!V->getParent() && "Value already in a container!!");
   ItemParentClass *Owner = getListOwner();
   V->setParent(Owner);
@@ -75,8 +75,8 @@ void SymbolTableListTraits<ValueSubClass>::addNodeToList(ValueSubClass *V) {
       ST->reinsertValue(V);
 }
 
-template <typename ValueSubClass>
-void SymbolTableListTraits<ValueSubClass>::removeNodeFromList(
+template <typename ValueSubClass, typename... Args>
+void SymbolTableListTraits<ValueSubClass, Args...>::removeNodeFromList(
     ValueSubClass *V) {
   V->setParent(nullptr);
   if (V->hasName())
@@ -84,8 +84,8 @@ void SymbolTableListTraits<ValueSubClass>::removeNodeFromList(
       ST->removeValueName(V->getValueName());
 }
 
-template <typename ValueSubClass>
-void SymbolTableListTraits<ValueSubClass>::transferNodesFromList(
+template <typename ValueSubClass, typename... Args>
+void SymbolTableListTraits<ValueSubClass, Args...>::transferNodesFromList(
     SymbolTableListTraits &L2, iterator first, iterator last) {
   // Transfering nodes, even within the same BB, invalidates the ordering. The
   // list that we removed the nodes from still has a valid ordering.
diff --git a/llvm/lib/IR/Type.cpp b/llvm/lib/IR/Type.cpp
index 97febcd99b..006278d164 100644
--- a/llvm/lib/IR/Type.cpp
+++ b/llvm/lib/IR/Type.cpp
@@ -841,7 +841,8 @@ static TargetTypeInfo getTargetTypeInfo(const TargetExtType *Ty) {
 
   // Opaque types in the AArch64 name space.
   if (Name == "aarch64.svcount")
-    return TargetTypeInfo(ScalableVectorType::get(Type::getInt1Ty(C), 16));
+    return TargetTypeInfo(ScalableVectorType::get(Type::getInt1Ty(C), 16),
+                          TargetExtType::HasZeroInit);
 
   return TargetTypeInfo(Type::getVoidTy(C));
 }
diff --git a/llvm/lib/IR/Value.cpp b/llvm/lib/IR/Value.cpp
index 41260a98e3..b485a6275b 100644
--- a/llvm/lib/IR/Value.cpp
+++ b/llvm/lib/IR/Value.cpp
@@ -330,8 +330,7 @@ void Value::setNameImpl(const Twine &NewName) {
 
   SmallString<256> NameData;
   StringRef NameRef = NeedNewName ? NewName.toStringRef(NameData) : "";
-  assert(NameRef.find_first_of(0) == StringRef::npos &&
-         "Null bytes are not allowed in names");
+  assert(!NameRef.contains(0) && "Null bytes are not allowed in names");
 
   // Name isn't changing?
   if (getName() == NameRef)
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 188e4a4a65..6d7d4b758a 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -2118,6 +2118,17 @@ void Verifier::verifyFunctionAttrs(FunctionType *FT, AttributeList Attrs,
 
     Check(!Attrs.hasFnAttr(Attribute::MinSize),
           "Attributes 'minsize and optnone' are incompatible!", V);
+
+    Check(!Attrs.hasFnAttr(Attribute::OptimizeForDebugging),
+          "Attributes 'optdebug and optnone' are incompatible!", V);
+  }
+
+  if (Attrs.hasFnAttr(Attribute::OptimizeForDebugging)) {
+    Check(!Attrs.hasFnAttr(Attribute::OptimizeForSize),
+          "Attributes 'optsize and optdebug' are incompatible!", V);
+
+    Check(!Attrs.hasFnAttr(Attribute::MinSize),
+          "Attributes 'minsize and optdebug' are incompatible!", V);
   }
 
   if (Attrs.hasFnAttr("aarch64_pstate_sm_enabled")) {
@@ -5658,10 +5669,28 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
     }
     break;
   }
-  case Intrinsic::lround:
-  case Intrinsic::llround:
   case Intrinsic::lrint:
   case Intrinsic::llrint: {
+    Type *ValTy = Call.getArgOperand(0)->getType();
+    Type *ResultTy = Call.getType();
+    Check(
+        ValTy->isFPOrFPVectorTy() && ResultTy->isIntOrIntVectorTy(),
+        "llvm.lrint, llvm.llrint: argument must be floating-point or vector "
+        "of floating-points, and result must be integer or vector of integers",
+        &Call);
+    Check(ValTy->isVectorTy() == ResultTy->isVectorTy(),
+          "llvm.lrint, llvm.llrint: argument and result disagree on vector use",
+          &Call);
+    if (ValTy->isVectorTy()) {
+      Check(cast<VectorType>(ValTy)->getElementCount() ==
+                cast<VectorType>(ResultTy)->getElementCount(),
+            "llvm.lrint, llvm.llrint: argument must be same length as result",
+            &Call);
+    }
+    break;
+  }
+  case Intrinsic::lround:
+  case Intrinsic::llround: {
     Type *ValTy = Call.getArgOperand(0)->getType();
     Type *ResultTy = Call.getType();
     Check(!ValTy->isVectorTy() && !ResultTy->isVectorTy(),
@@ -6318,20 +6347,6 @@ void Verifier::visitDbgIntrinsic(StringRef Kind, DbgVariableIntrinsic &DII) {
   CheckDI(isType(Var->getRawType()), "invalid type ref", Var,
           Var->getRawType());
   verifyFnArgs(DII);
-
-  if (auto *Declare = dyn_cast<DbgDeclareInst>(&DII)) {
-    if (auto *Alloca = dyn_cast_or_null<AllocaInst>(Declare->getAddress())) {
-      DIExpression *Expr = Declare->getExpression();
-      std::optional<uint64_t> FragSize = Declare->getFragmentSizeInBits();
-      std::optional<TypeSize> AllocSize = Alloca->getAllocationSizeInBits(DL);
-      if (FragSize && AllocSize && !AllocSize->isScalable() &&
-          !Expr->isComplex()) {
-        CheckDI(*FragSize <= AllocSize->getFixedValue(),
-                "llvm.dbg.declare has larger fragment size than alloca size ",
-                &DII);
-      }
-    }
-  }
 }
 
 void Verifier::visitDbgLabelIntrinsic(StringRef Kind, DbgLabelInst &DLI) {
diff --git a/llvm/lib/InterfaceStub/ELFObjHandler.cpp b/llvm/lib/InterfaceStub/ELFObjHandler.cpp
index 49ed27e265..c1256563d0 100644
--- a/llvm/lib/InterfaceStub/ELFObjHandler.cpp
+++ b/llvm/lib/InterfaceStub/ELFObjHandler.cpp
@@ -57,7 +57,7 @@ static void initELFHeader(typename ELFT::Ehdr &ElfHeader, uint16_t Machine) {
   ElfHeader.e_ident[EI_MAG2] = ElfMagic[EI_MAG2];
   ElfHeader.e_ident[EI_MAG3] = ElfMagic[EI_MAG3];
   ElfHeader.e_ident[EI_CLASS] = ELFT::Is64Bits ? ELFCLASS64 : ELFCLASS32;
-  bool IsLittleEndian = ELFT::TargetEndianness == support::little;
+  bool IsLittleEndian = ELFT::TargetEndianness == llvm::endianness::little;
   ElfHeader.e_ident[EI_DATA] = IsLittleEndian ? ELFDATA2LSB : ELFDATA2MSB;
   ElfHeader.e_ident[EI_VERSION] = EV_CURRENT;
   ElfHeader.e_ident[EI_OSABI] = ELFOSABI_NONE;
diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp
index 4a64aa4593..214c2ef45d 100644
--- a/llvm/lib/LTO/LTO.cpp
+++ b/llvm/lib/LTO/LTO.cpp
@@ -74,7 +74,6 @@ namespace llvm {
 cl::opt<bool> EnableLTOInternalization(
     "enable-lto-internalization", cl::init(true), cl::Hidden,
     cl::desc("Enable global value internalization in LTO"));
-}
 
 /// Indicate we are linking with an allocator that supports hot/cold operator
 /// new interfaces.
@@ -82,6 +81,7 @@ extern cl::opt<bool> SupportsHotColdNew;
 
 /// Enable MemProf context disambiguation for thin link.
 extern cl::opt<bool> EnableMemProfContextDisambiguation;
+} // namespace llvm
 
 // Computes a unique hash for the Module considering the current list of
 // export/import and other global analysis results.
diff --git a/llvm/lib/MC/DXContainerPSVInfo.cpp b/llvm/lib/MC/DXContainerPSVInfo.cpp
index bdc6f79a68..48182fcd31 100644
--- a/llvm/lib/MC/DXContainerPSVInfo.cpp
+++ b/llvm/lib/MC/DXContainerPSVInfo.cpp
@@ -87,15 +87,15 @@ void PSVRuntimeInfo::write(raw_ostream &OS, uint32_t Version) const {
   }
   // Write the size of the info.
 
-  support::endian::write(OS, InfoSize, support::little);
+  support::endian::write(OS, InfoSize, llvm::endianness::little);
   // Write the info itself.
   OS.write(reinterpret_cast<const char *>(&BaseData), InfoSize);
 
   uint32_t ResourceCount = static_cast<uint32_t>(Resources.size());
 
-  support::endian::write(OS, ResourceCount, support::little);
+  support::endian::write(OS, ResourceCount, llvm::endianness::little);
   if (ResourceCount > 0)
-    support::endian::write(OS, BindingSize, support::little);
+    support::endian::write(OS, BindingSize, llvm::endianness::little);
 
   for (const auto &Res : Resources)
     OS.write(reinterpret_cast<const char *>(&Res), BindingSize);
@@ -126,22 +126,22 @@ void PSVRuntimeInfo::write(raw_ostream &OS, uint32_t Version) const {
   }
 
   support::endian::write(OS, static_cast<uint32_t>(StrTabBuilder.getSize()),
-                         support::little);
+                         llvm::endianness::little);
 
   // Write the string table.
   StrTabBuilder.write(OS);
 
   // Write the index table size, then table.
   support::endian::write(OS, static_cast<uint32_t>(IndexBuffer.size()),
-                         support::little);
+                         llvm::endianness::little);
   for (auto I : IndexBuffer)
-    support::endian::write(OS, I, support::little);
+    support::endian::write(OS, I, llvm::endianness::little);
 
   if (SignatureElements.size() > 0) {
     // write the size of the signature elements.
     support::endian::write(OS,
                            static_cast<uint32_t>(sizeof(v0::SignatureElement)),
-                           support::little);
+                           llvm::endianness::little);
 
     // write the signature elements.
     OS.write(reinterpret_cast<const char *>(&SignatureElements[0]),
@@ -150,16 +150,16 @@ void PSVRuntimeInfo::write(raw_ostream &OS, uint32_t Version) const {
 
   for (const auto &MaskVector : OutputVectorMasks)
     support::endian::write_array(OS, ArrayRef<uint32_t>(MaskVector),
-                                 support::little);
+                                 llvm::endianness::little);
   support::endian::write_array(OS, ArrayRef<uint32_t>(PatchOrPrimMasks),
-                               support::little);
+                               llvm::endianness::little);
   for (const auto &MaskVector : InputOutputMap)
     support::endian::write_array(OS, ArrayRef<uint32_t>(MaskVector),
-                                 support::little);
+                                 llvm::endianness::little);
   support::endian::write_array(OS, ArrayRef<uint32_t>(InputPatchMap),
-                               support::little);
+                               llvm::endianness::little);
   support::endian::write_array(OS, ArrayRef<uint32_t>(PatchOutputMap),
-                               support::little);
+                               llvm::endianness::little);
 }
 
 void Signature::write(raw_ostream &OS) {
diff --git a/llvm/lib/MC/ELFObjectWriter.cpp b/llvm/lib/MC/ELFObjectWriter.cpp
index 816aa21321..8490fefe7f 100644
--- a/llvm/lib/MC/ELFObjectWriter.cpp
+++ b/llvm/lib/MC/ELFObjectWriter.cpp
@@ -152,8 +152,9 @@ struct ELFWriter {
 public:
   ELFWriter(ELFObjectWriter &OWriter, raw_pwrite_stream &OS,
             bool IsLittleEndian, DwoMode Mode)
-      : OWriter(OWriter),
-        W(OS, IsLittleEndian ? support::little : support::big), Mode(Mode) {}
+      : OWriter(OWriter), W(OS, IsLittleEndian ? llvm::endianness::little
+                                               : llvm::endianness::big),
+        Mode(Mode) {}
 
   void WriteWord(uint64_t Word) {
     if (is64Bit())
@@ -406,8 +407,8 @@ void ELFWriter::writeHeader(const MCAssembler &Asm) {
   W.OS << char(is64Bit() ? ELF::ELFCLASS64 : ELF::ELFCLASS32); // e_ident[EI_CLASS]
 
   // e_ident[EI_DATA]
-  W.OS << char(W.Endian == support::little ? ELF::ELFDATA2LSB
-                                           : ELF::ELFDATA2MSB);
+  W.OS << char(W.Endian == llvm::endianness::little ? ELF::ELFDATA2LSB
+                                                    : ELF::ELFDATA2MSB);
 
   W.OS << char(ELF::EV_CURRENT);        // e_ident[EI_VERSION]
   // e_ident[EI_OSABI]
diff --git a/llvm/lib/MC/GOFFObjectWriter.cpp b/llvm/lib/MC/GOFFObjectWriter.cpp
index 33244cbf88..addeb6db95 100644
--- a/llvm/lib/MC/GOFFObjectWriter.cpp
+++ b/llvm/lib/MC/GOFFObjectWriter.cpp
@@ -137,7 +137,8 @@ class GOFFOstream : public raw_ostream {
 
   // Support for endian-specific data.
   template <typename value_type> void writebe(value_type Value) {
-    Value = support::endian::byte_swap<value_type>(Value, support::big);
+    Value =
+        support::endian::byte_swap<value_type>(Value, llvm::endianness::big);
     write(reinterpret_cast<const char *>(&Value), sizeof(value_type));
   }
 };
diff --git a/llvm/lib/MC/MCAsmBackend.cpp b/llvm/lib/MC/MCAsmBackend.cpp
index 4ad0e7a809..4b1064a07e 100644
--- a/llvm/lib/MC/MCAsmBackend.cpp
+++ b/llvm/lib/MC/MCAsmBackend.cpp
@@ -23,7 +23,7 @@
 
 using namespace llvm;
 
-MCAsmBackend::MCAsmBackend(support::endianness Endian, unsigned RelaxFixupKind)
+MCAsmBackend::MCAsmBackend(llvm::endianness Endian, unsigned RelaxFixupKind)
     : Endian(Endian), RelaxFixupKind(RelaxFixupKind) {}
 
 MCAsmBackend::~MCAsmBackend() = default;
@@ -33,11 +33,11 @@ MCAsmBackend::createObjectWriter(raw_pwrite_stream &OS) const {
   auto TW = createObjectTargetWriter();
   switch (TW->getFormat()) {
   case Triple::ELF:
-    return createELFObjectWriter(cast<MCELFObjectTargetWriter>(std::move(TW)), OS,
-                                 Endian == support::little);
+    return createELFObjectWriter(cast<MCELFObjectTargetWriter>(std::move(TW)),
+                                 OS, Endian == llvm::endianness::little);
   case Triple::MachO:
     return createMachObjectWriter(cast<MCMachObjectTargetWriter>(std::move(TW)),
-                                  OS, Endian == support::little);
+                                  OS, Endian == llvm::endianness::little);
   case Triple::COFF:
     return createWinCOFFObjectWriter(
         cast<MCWinCOFFObjectTargetWriter>(std::move(TW)), OS);
@@ -72,7 +72,7 @@ MCAsmBackend::createDwoObjectWriter(raw_pwrite_stream &OS,
   case Triple::ELF:
     return createELFDwoObjectWriter(
         cast<MCELFObjectTargetWriter>(std::move(TW)), OS, DwoOS,
-        Endian == support::little);
+        Endian == llvm::endianness::little);
   case Triple::Wasm:
     return createWasmDwoObjectWriter(
         cast<MCWasmObjectTargetWriter>(std::move(TW)), OS, DwoOS);
diff --git a/llvm/lib/MC/MCAsmStreamer.cpp b/llvm/lib/MC/MCAsmStreamer.cpp
index 06de70ad2f..fb809f4010 100644
--- a/llvm/lib/MC/MCAsmStreamer.cpp
+++ b/llvm/lib/MC/MCAsmStreamer.cpp
@@ -629,18 +629,11 @@ void MCAsmStreamer::emitVersionMin(MCVersionMinType Type, unsigned Major,
 
 static const char *getPlatformName(MachO::PlatformType Type) {
   switch (Type) {
-  case MachO::PLATFORM_UNKNOWN: /* silence warning*/
-    break;
-  case MachO::PLATFORM_MACOS:            return "macos";
-  case MachO::PLATFORM_IOS:              return "ios";
-  case MachO::PLATFORM_TVOS:             return "tvos";
-  case MachO::PLATFORM_WATCHOS:          return "watchos";
-  case MachO::PLATFORM_BRIDGEOS:         return "bridgeos";
-  case MachO::PLATFORM_MACCATALYST:      return "macCatalyst";
-  case MachO::PLATFORM_IOSSIMULATOR:     return "iossimulator";
-  case MachO::PLATFORM_TVOSSIMULATOR:    return "tvossimulator";
-  case MachO::PLATFORM_WATCHOSSIMULATOR: return "watchossimulator";
-  case MachO::PLATFORM_DRIVERKIT:        return "driverkit";
+#define PLATFORM(platform, id, name, build_name, target, tapi_target,          \
+                 marketing)                                                    \
+  case MachO::PLATFORM_##platform:                                             \
+    return #build_name;
+#include "llvm/BinaryFormat/MachO.def"
   }
   llvm_unreachable("Invalid Mach-O platform type");
 }
diff --git a/llvm/lib/MC/MCAssembler.cpp b/llvm/lib/MC/MCAssembler.cpp
index 1cc408e114..55558820b6 100644
--- a/llvm/lib/MC/MCAssembler.cpp
+++ b/llvm/lib/MC/MCAssembler.cpp
@@ -521,7 +521,7 @@ static void writeFragment(raw_ostream &OS, const MCAssembler &Asm,
   // FIXME: Embed in fragments instead?
   uint64_t FragmentSize = Asm.computeFragmentSize(Layout, F);
 
-  support::endianness Endian = Asm.getBackend().Endian;
+  llvm::endianness Endian = Asm.getBackend().Endian;
 
   if (const MCEncodedFragment *EF = dyn_cast<MCEncodedFragment>(&F))
     Asm.writeFragmentPadding(OS, *EF, FragmentSize);
@@ -606,7 +606,7 @@ static void writeFragment(raw_ostream &OS, const MCAssembler &Asm,
     // Duplicate V into Data as byte vector to reduce number of
     // writes done. As such, do endian conversion here.
     for (unsigned I = 0; I != VSize; ++I) {
-      unsigned index = Endian == support::little ? I : (VSize - I - 1);
+      unsigned index = Endian == llvm::endianness::little ? I : (VSize - I - 1);
       Data[I] = uint8_t(V >> (index * 8));
     }
     for (unsigned I = VSize; I < MaxChunkSize; ++I)
diff --git a/llvm/lib/MC/MCCodeView.cpp b/llvm/lib/MC/MCCodeView.cpp
index f09997ebdf..d234ce1109 100644
--- a/llvm/lib/MC/MCCodeView.cpp
+++ b/llvm/lib/MC/MCCodeView.cpp
@@ -654,7 +654,7 @@ void CodeViewContext::encodeDefRange(MCAsmLayout &Layout,
     }
     unsigned NumGaps = J - I - 1;
 
-    support::endian::Writer LEWriter(OS, support::little);
+    support::endian::Writer LEWriter(OS, llvm::endianness::little);
 
     unsigned Bias = 0;
     // We must split the range into chunks of MaxDefRange, this is a fundamental
diff --git a/llvm/lib/MC/MCDXContainerWriter.cpp b/llvm/lib/MC/MCDXContainerWriter.cpp
index 028bfe6e79..0580dc7e42 100644
--- a/llvm/lib/MC/MCDXContainerWriter.cpp
+++ b/llvm/lib/MC/MCDXContainerWriter.cpp
@@ -30,7 +30,7 @@ class DXContainerObjectWriter : public MCObjectWriter {
 public:
   DXContainerObjectWriter(std::unique_ptr<MCDXContainerTargetWriter> MOTW,
                           raw_pwrite_stream &OS)
-      : W(OS, support::little), TargetObjectWriter(std::move(MOTW)) {}
+      : W(OS, llvm::endianness::little), TargetObjectWriter(std::move(MOTW)) {}
 
   ~DXContainerObjectWriter() override {}
 
diff --git a/llvm/lib/MC/MCDwarf.cpp b/llvm/lib/MC/MCDwarf.cpp
index 55632f2fe7..7925fba876 100644
--- a/llvm/lib/MC/MCDwarf.cpp
+++ b/llvm/lib/MC/MCDwarf.cpp
@@ -1940,8 +1940,9 @@ void MCDwarfFrameEmitter::encodeAdvanceLoc(MCContext &Context,
   if (AddrDelta == 0)
     return;
 
-  support::endianness E =
-      Context.getAsmInfo()->isLittleEndian() ? support::little : support::big;
+  llvm::endianness E = Context.getAsmInfo()->isLittleEndian()
+                           ? llvm::endianness::little
+                           : llvm::endianness::big;
 
   if (isUIntN(6, AddrDelta)) {
     uint8_t Opcode = dwarf::DW_CFA_advance_loc | AddrDelta;
diff --git a/llvm/lib/MC/MCParser/DarwinAsmParser.cpp b/llvm/lib/MC/MCParser/DarwinAsmParser.cpp
index 7c390041b3..edea5a56be 100644
--- a/llvm/lib/MC/MCParser/DarwinAsmParser.cpp
+++ b/llvm/lib/MC/MCParser/DarwinAsmParser.cpp
@@ -1167,14 +1167,13 @@ bool DarwinAsmParser::parseBuildVersion(StringRef Directive, SMLoc Loc) {
     return TokError("platform name expected");
 
   unsigned Platform = StringSwitch<unsigned>(PlatformName)
-    .Case("macos", MachO::PLATFORM_MACOS)
-    .Case("ios", MachO::PLATFORM_IOS)
-    .Case("tvos", MachO::PLATFORM_TVOS)
-    .Case("watchos", MachO::PLATFORM_WATCHOS)
-    .Case("macCatalyst", MachO::PLATFORM_MACCATALYST)
-    .Case("driverkit", MachO::PLATFORM_DRIVERKIT)
-    .Default(0);
-  if (Platform == 0)
+#define PLATFORM(platform, id, name, build_name, target, tapi_target,          \
+                 marketing)                                                    \
+  .Case(#build_name, MachO::PLATFORM_##platform)
+#include "llvm/BinaryFormat/MachO.def"
+                          .Default(MachO::PLATFORM_UNKNOWN);
+
+  if (Platform == MachO::PLATFORM_UNKNOWN)
     return Error(PlatformLoc, "unknown platform name");
 
   if (getLexer().isNot(AsmToken::Comma))
diff --git a/llvm/lib/MC/MCPseudoProbe.cpp b/llvm/lib/MC/MCPseudoProbe.cpp
index ec9d086588..eb3894dbb3 100644
--- a/llvm/lib/MC/MCPseudoProbe.cpp
+++ b/llvm/lib/MC/MCPseudoProbe.cpp
@@ -343,7 +343,7 @@ template <typename T> ErrorOr<T> MCPseudoProbeDecoder::readUnencodedNumber() {
   if (Data + sizeof(T) > End) {
     return std::error_code();
   }
-  T Val = endian::readNext<T, little, unaligned>(Data);
+  T Val = endian::readNext<T, llvm::endianness::little, unaligned>(Data);
   return ErrorOr<T>(Val);
 }
 
diff --git a/llvm/lib/MC/MCStreamer.cpp b/llvm/lib/MC/MCStreamer.cpp
index 7f9c0c3b0b..2371cb2384 100644
--- a/llvm/lib/MC/MCStreamer.cpp
+++ b/llvm/lib/MC/MCStreamer.cpp
@@ -137,7 +137,7 @@ void MCStreamer::emitIntValue(uint64_t Value, unsigned Size) {
          "Invalid size");
   const bool IsLittleEndian = Context.getAsmInfo()->isLittleEndian();
   uint64_t Swapped = support::endian::byte_swap(
-      Value, IsLittleEndian ? support::little : support::big);
+      Value, IsLittleEndian ? llvm::endianness::little : llvm::endianness::big);
   unsigned Index = IsLittleEndian ? 0 : 8 - Size;
   emitBytes(StringRef(reinterpret_cast<char *>(&Swapped) + Index, Size));
 }
diff --git a/llvm/lib/MC/MachObjectWriter.cpp b/llvm/lib/MC/MachObjectWriter.cpp
index 04097dfe2e..d17e6e125d 100644
--- a/llvm/lib/MC/MachObjectWriter.cpp
+++ b/llvm/lib/MC/MachObjectWriter.cpp
@@ -630,7 +630,7 @@ void MachObjectWriter::computeSymbolTable(
       // Set the Index and the IsExtern bit.
       unsigned Index = Rel.Sym->getIndex();
       assert(isInt<24>(Index));
-      if (W.Endian == support::little)
+      if (W.Endian == llvm::endianness::little)
         Rel.MRE.r_word1 = (Rel.MRE.r_word1 & (~0U << 24)) | Index | (1 << 27);
       else
         Rel.MRE.r_word1 = (Rel.MRE.r_word1 & 0xff) | Index << 8 | (1 << 4);
diff --git a/llvm/lib/MC/SPIRVObjectWriter.cpp b/llvm/lib/MC/SPIRVObjectWriter.cpp
index cb49f5eeca..39856e96e9 100644
--- a/llvm/lib/MC/SPIRVObjectWriter.cpp
+++ b/llvm/lib/MC/SPIRVObjectWriter.cpp
@@ -24,7 +24,7 @@ class SPIRVObjectWriter : public MCObjectWriter {
 public:
   SPIRVObjectWriter(std::unique_ptr<MCSPIRVObjectTargetWriter> MOTW,
                     raw_pwrite_stream &OS)
-      : W(OS, support::little), TargetObjectWriter(std::move(MOTW)) {}
+      : W(OS, llvm::endianness::little), TargetObjectWriter(std::move(MOTW)) {}
 
   ~SPIRVObjectWriter() override {}
 
diff --git a/llvm/lib/MC/WasmObjectWriter.cpp b/llvm/lib/MC/WasmObjectWriter.cpp
index bb8c68410a..b99df3837c 100644
--- a/llvm/lib/MC/WasmObjectWriter.cpp
+++ b/llvm/lib/MC/WasmObjectWriter.cpp
@@ -1438,12 +1438,12 @@ void WasmObjectWriter::prepareImports(
 
 uint64_t WasmObjectWriter::writeObject(MCAssembler &Asm,
                                        const MCAsmLayout &Layout) {
-  support::endian::Writer MainWriter(*OS, support::little);
+  support::endian::Writer MainWriter(*OS, llvm::endianness::little);
   W = &MainWriter;
   if (IsSplitDwarf) {
     uint64_t TotalSize = writeOneObject(Asm, Layout, DwoMode::NonDwoOnly);
     assert(DwoOS);
-    support::endian::Writer DwoWriter(*DwoOS, support::little);
+    support::endian::Writer DwoWriter(*DwoOS, llvm::endianness::little);
     W = &DwoWriter;
     return TotalSize + writeOneObject(Asm, Layout, DwoMode::DwoOnly);
   } else {
diff --git a/llvm/lib/MC/WinCOFFObjectWriter.cpp b/llvm/lib/MC/WinCOFFObjectWriter.cpp
index fd8af1f8cd..1f73cb9884 100644
--- a/llvm/lib/MC/WinCOFFObjectWriter.cpp
+++ b/llvm/lib/MC/WinCOFFObjectWriter.cpp
@@ -260,7 +260,7 @@ void COFFSymbol::set_name_offset(uint32_t Offset) {
 
 WinCOFFWriter::WinCOFFWriter(WinCOFFObjectWriter &OWriter,
                              raw_pwrite_stream &OS, DwoMode Mode)
-    : OWriter(OWriter), W(OS, support::little), Mode(Mode) {
+    : OWriter(OWriter), W(OS, llvm::endianness::little), Mode(Mode) {
   Header.Machine = OWriter.TargetObjectWriter->getMachine();
   // Some relocations on ARM64 (the 21 bit ADRP relocations) have a slightly
   // limited range for the immediate offset (+/- 1 MB); create extra offset
diff --git a/llvm/lib/MC/XCOFFObjectWriter.cpp b/llvm/lib/MC/XCOFFObjectWriter.cpp
index f1cce9b8c9..343e2fc877 100644
--- a/llvm/lib/MC/XCOFFObjectWriter.cpp
+++ b/llvm/lib/MC/XCOFFObjectWriter.cpp
@@ -448,7 +448,7 @@ class XCOFFObjectWriter : public MCObjectWriter {
 
 XCOFFObjectWriter::XCOFFObjectWriter(
     std::unique_ptr<MCXCOFFObjectTargetWriter> MOTW, raw_pwrite_stream &OS)
-    : W(OS, support::big), TargetObjectWriter(std::move(MOTW)),
+    : W(OS, llvm::endianness::big), TargetObjectWriter(std::move(MOTW)),
       Strings(StringTableBuilder::XCOFF),
       Text(".text", XCOFF::STYP_TEXT, /* IsVirtual */ false,
            CsectGroups{&ProgramCodeCsects, &ReadOnlyCsects}),
diff --git a/llvm/lib/ObjCopy/ELF/ELFObject.cpp b/llvm/lib/ObjCopy/ELF/ELFObject.cpp
index eaeef11b12..0a54d3798d 100644
--- a/llvm/lib/ObjCopy/ELF/ELFObject.cpp
+++ b/llvm/lib/ObjCopy/ELF/ELFObject.cpp
@@ -1987,8 +1987,9 @@ template <class ELFT> void ELFWriter<ELFT>::writeEhdr() {
   Ehdr.e_ident[EI_MAG2] = 'L';
   Ehdr.e_ident[EI_MAG3] = 'F';
   Ehdr.e_ident[EI_CLASS] = ELFT::Is64Bits ? ELFCLASS64 : ELFCLASS32;
-  Ehdr.e_ident[EI_DATA] =
-      ELFT::TargetEndianness == support::big ? ELFDATA2MSB : ELFDATA2LSB;
+  Ehdr.e_ident[EI_DATA] = ELFT::TargetEndianness == llvm::endianness::big
+                              ? ELFDATA2MSB
+                              : ELFDATA2LSB;
   Ehdr.e_ident[EI_VERSION] = EV_CURRENT;
   Ehdr.e_ident[EI_OSABI] = Obj.OSABI;
   Ehdr.e_ident[EI_ABIVERSION] = Obj.ABIVersion;
@@ -2695,11 +2696,11 @@ uint64_t IHexWriter::writeEntryPointRecord(uint8_t *Buf) {
   if (Obj.Entry <= 0xFFFFFU) {
     Data[0] = ((Obj.Entry & 0xF0000U) >> 12) & 0xFF;
     support::endian::write(&Data[2], static_cast<uint16_t>(Obj.Entry),
-                           support::big);
+                           llvm::endianness::big);
     HexData = IHexRecord::getLine(IHexRecord::StartAddr80x86, 0, Data);
   } else {
     support::endian::write(Data, static_cast<uint32_t>(Obj.Entry),
-                           support::big);
+                           llvm::endianness::big);
     HexData = IHexRecord::getLine(IHexRecord::StartAddr, 0, Data);
   }
   memcpy(Buf, HexData.data(), HexData.size());
diff --git a/llvm/lib/Object/Archive.cpp b/llvm/lib/Object/Archive.cpp
index bf9a1ce1de..fdd87824e2 100644
--- a/llvm/lib/Object/Archive.cpp
+++ b/llvm/lib/Object/Archive.cpp
@@ -1421,7 +1421,7 @@ BigArchive::BigArchive(MemoryBufferRef Source, Error &Err)
     // 64-bit global symbol tables, we need to merge them into a single table.
     raw_string_ostream Out(MergedGlobalSymtabBuf);
     uint64_t SymNum = SymtabInfos[0].SymNum + SymtabInfos[1].SymNum;
-    write(Out, SymNum, support::big);
+    write(Out, SymNum, llvm::endianness::big);
     // Merge symbol offset.
     Out << SymtabInfos[0].SymbolOffsetTable;
     Out << SymtabInfos[1].SymbolOffsetTable;
diff --git a/llvm/lib/Object/ArchiveWriter.cpp b/llvm/lib/Object/ArchiveWriter.cpp
index aa74a2f74f..2f70c9edd1 100644
--- a/llvm/lib/Object/ArchiveWriter.cpp
+++ b/llvm/lib/Object/ArchiveWriter.cpp
@@ -18,6 +18,7 @@
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/Object/Archive.h"
 #include "llvm/Object/COFF.h"
+#include "llvm/Object/COFFImportFile.h"
 #include "llvm/Object/Error.h"
 #include "llvm/Object/IRObjectFile.h"
 #include "llvm/Object/MachO.h"
@@ -202,11 +203,12 @@ static bool isBSDLike(object::Archive::Kind Kind) {
 template <class T>
 static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val) {
   support::endian::write(Out, Val,
-                         isBSDLike(Kind) ? support::little : support::big);
+                         isBSDLike(Kind) ? llvm::endianness::little
+                                         : llvm::endianness::big);
 }
 
 template <class T> static void printLE(raw_ostream &Out, T Val) {
-  support::endian::write(Out, Val, support::little);
+  support::endian::write(Out, Val, llvm::endianness::little);
 }
 
 static void printRestOfMemberHeader(
@@ -659,6 +661,10 @@ static bool isECObject(object::SymbolicFile &Obj) {
     return cast<llvm::object::COFFObjectFile>(&Obj)->getMachine() !=
            COFF::IMAGE_FILE_MACHINE_ARM64;
 
+  if (Obj.isCOFFImportFile())
+    return cast<llvm::object::COFFImportFile>(&Obj)->getMachine() !=
+           COFF::IMAGE_FILE_MACHINE_ARM64;
+
   if (Obj.isIR()) {
     Expected<std::string> TripleStr =
         getBitcodeTargetTriple(Obj.getMemoryBufferRef());
diff --git a/llvm/lib/Object/COFFImportFile.cpp b/llvm/lib/Object/COFFImportFile.cpp
index a567ecde9b..2cca1f728c 100644
--- a/llvm/lib/Object/COFFImportFile.cpp
+++ b/llvm/lib/Object/COFFImportFile.cpp
@@ -612,7 +612,8 @@ Error writeImportLibrary(StringRef ImportName, StringRef Path,
 
   return writeArchive(Path, Members, SymtabWritingMode::NormalSymtab,
                       MinGW ? object::Archive::K_GNU : object::Archive::K_COFF,
-                      /*Deterministic*/ true, /*Thin*/ false);
+                      /*Deterministic*/ true, /*Thin*/ false,
+                      /*OldArchiveBuf*/ nullptr, isArm64EC(Machine));
 }
 
 } // namespace object
diff --git a/llvm/lib/Object/COFFObjectFile.cpp b/llvm/lib/Object/COFFObjectFile.cpp
index 08eb0d034c..574f7a7cf1 100644
--- a/llvm/lib/Object/COFFObjectFile.cpp
+++ b/llvm/lib/Object/COFFObjectFile.cpp
@@ -1907,7 +1907,7 @@ Error ResourceSectionRef::load(const COFFObjectFile *O, const SectionRef &S) {
   Expected<StringRef> Contents = Section.getContents();
   if (!Contents)
     return Contents.takeError();
-  BBS = BinaryByteStream(*Contents, support::little);
+  BBS = BinaryByteStream(*Contents, llvm::endianness::little);
   const coff_section *COFFSect = Obj->getCOFFSection(Section);
   ArrayRef<coff_relocation> OrigRelocs = Obj->getRelocations(COFFSect);
   Relocs.reserve(OrigRelocs.size());
diff --git a/llvm/lib/Object/ModuleSymbolTable.cpp b/llvm/lib/Object/ModuleSymbolTable.cpp
index 0290a819e5..dc73937863 100644
--- a/llvm/lib/Object/ModuleSymbolTable.cpp
+++ b/llvm/lib/Object/ModuleSymbolTable.cpp
@@ -15,7 +15,6 @@
 #include "llvm/Object/ModuleSymbolTable.h"
 #include "RecordStreamer.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/GlobalAlias.h"
diff --git a/llvm/lib/Object/SymbolSize.cpp b/llvm/lib/Object/SymbolSize.cpp
index f93a5f7d9b..cb20feffb7 100644
--- a/llvm/lib/Object/SymbolSize.cpp
+++ b/llvm/lib/Object/SymbolSize.cpp
@@ -59,6 +59,12 @@ llvm::object::computeSymbolSizes(const ObjectFile &O) {
     return Ret;
   }
 
+  if (const auto *E = dyn_cast<XCOFFObjectFile>(&O)) {
+    for (XCOFFSymbolRef Sym : E->symbols())
+      Ret.push_back({Sym, Sym.getSize()});
+    return Ret;
+  }
+
   // Collect sorted symbol addresses. Include dummy addresses for the end
   // of each section.
   std::vector<SymEntry> Addresses;
diff --git a/llvm/lib/Object/WindowsResource.cpp b/llvm/lib/Object/WindowsResource.cpp
index 0764dc8f75..61ca49e290 100644
--- a/llvm/lib/Object/WindowsResource.cpp
+++ b/llvm/lib/Object/WindowsResource.cpp
@@ -50,7 +50,7 @@ WindowsResource::WindowsResource(MemoryBufferRef Source)
     : Binary(Binary::ID_WinRes, Source) {
   size_t LeadingSize = WIN_RES_MAGIC_SIZE + WIN_RES_NULL_ENTRY_SIZE;
   BBS = BinaryByteStream(Data.getBuffer().drop_front(LeadingSize),
-                         support::little);
+                         llvm::endianness::little);
 }
 
 // static
diff --git a/llvm/lib/Object/XCOFFObjectFile.cpp b/llvm/lib/Object/XCOFFObjectFile.cpp
index fa4917e354..4c192aa37a 100644
--- a/llvm/lib/Object/XCOFFObjectFile.cpp
+++ b/llvm/lib/Object/XCOFFObjectFile.cpp
@@ -689,6 +689,10 @@ basic_symbol_iterator XCOFFObjectFile::symbol_end() const {
   return basic_symbol_iterator(SymbolRef(SymDRI, this));
 }
 
+XCOFFObjectFile::xcoff_symbol_iterator_range XCOFFObjectFile::symbols() const {
+  return xcoff_symbol_iterator_range(symbol_begin(), symbol_end());
+}
+
 section_iterator XCOFFObjectFile::section_begin() const {
   DataRefImpl DRI;
   DRI.p = getSectionHeaderTableAddress();
@@ -1248,7 +1252,7 @@ bool XCOFFSymbolRef::isFunction() const {
     return false;
 
   const int16_t SectNum = getSectionNumber();
-  Expected<DataRefImpl> SI = OwningObjectPtr->getSectionByNum(SectNum);
+  Expected<DataRefImpl> SI = getObject()->getSectionByNum(SectNum);
   if (!SI) {
     // If we could not get the section, then this symbol should not be
     // a function. So consume the error and return `false` to move on.
@@ -1256,7 +1260,7 @@ bool XCOFFSymbolRef::isFunction() const {
     return false;
   }
 
-  return (OwningObjectPtr->getSectionFlags(SI.get()) & XCOFF::STYP_TEXT);
+  return (getObject()->getSectionFlags(SI.get()) & XCOFF::STYP_TEXT);
 }
 
 bool XCOFFSymbolRef::isCsectSymbol() const {
@@ -1275,13 +1279,13 @@ Expected<XCOFFCsectAuxRef> XCOFFSymbolRef::getXCOFFCsectAuxRef() const {
   if (auto Err = NameOrErr.takeError())
     return std::move(Err);
 
-  uint32_t SymbolIdx = OwningObjectPtr->getSymbolIndex(getEntryAddress());
+  uint32_t SymbolIdx = getObject()->getSymbolIndex(getEntryAddress());
   if (!NumberOfAuxEntries) {
     return createError("csect symbol \"" + *NameOrErr + "\" with index " +
                        Twine(SymbolIdx) + " contains no auxiliary entry");
   }
 
-  if (!OwningObjectPtr->is64Bit()) {
+  if (!getObject()->is64Bit()) {
     // In XCOFF32, the csect auxilliary entry is always the last auxiliary
     // entry for the symbol.
     uintptr_t AuxAddr = XCOFFObjectFile::getAdvancedSymbolEntryAddress(
@@ -1294,10 +1298,10 @@ Expected<XCOFFCsectAuxRef> XCOFFSymbolRef::getXCOFFCsectAuxRef() const {
   for (uint8_t Index = NumberOfAuxEntries; Index > 0; --Index) {
     uintptr_t AuxAddr = XCOFFObjectFile::getAdvancedSymbolEntryAddress(
         getEntryAddress(), Index);
-    if (*OwningObjectPtr->getSymbolAuxType(AuxAddr) ==
+    if (*getObject()->getSymbolAuxType(AuxAddr) ==
         XCOFF::SymbolAuxType::AUX_CSECT) {
 #ifndef NDEBUG
-      OwningObjectPtr->checkSymbolEntryPointer(AuxAddr);
+      getObject()->checkSymbolEntryPointer(AuxAddr);
 #endif
       return XCOFFCsectAuxRef(viewAs<XCOFFCsectAuxEnt64>(AuxAddr));
     }
@@ -1314,14 +1318,15 @@ Expected<StringRef> XCOFFSymbolRef::getName() const {
   if (getStorageClass() & 0x80)
     return StringRef("Unimplemented Debug Name");
 
-  if (Entry32) {
-    if (Entry32->NameInStrTbl.Magic != XCOFFSymbolRef::NAME_IN_STR_TBL_MAGIC)
-      return generateXCOFFFixedNameStringRef(Entry32->SymbolName);
+  if (!getObject()->is64Bit()) {
+    if (getSymbol32()->NameInStrTbl.Magic !=
+        XCOFFSymbolRef::NAME_IN_STR_TBL_MAGIC)
+      return generateXCOFFFixedNameStringRef(getSymbol32()->SymbolName);
 
-    return OwningObjectPtr->getStringTableEntry(Entry32->NameInStrTbl.Offset);
+    return getObject()->getStringTableEntry(getSymbol32()->NameInStrTbl.Offset);
   }
 
-  return OwningObjectPtr->getStringTableEntry(Entry64->Offset);
+  return getObject()->getStringTableEntry(getSymbol64()->Offset);
 }
 
 // Explictly instantiate template classes.
diff --git a/llvm/lib/ObjectYAML/COFFEmitter.cpp b/llvm/lib/ObjectYAML/COFFEmitter.cpp
index 2e72e4fa7f..7088223b9b 100644
--- a/llvm/lib/ObjectYAML/COFFEmitter.cpp
+++ b/llvm/lib/ObjectYAML/COFFEmitter.cpp
@@ -182,7 +182,7 @@ toDebugS(ArrayRef<CodeViewYAML::YAMLDebugSubsection> Subsections,
   }
   uint8_t *Buffer = Allocator.Allocate<uint8_t>(Size);
   MutableArrayRef<uint8_t> Output(Buffer, Size);
-  BinaryStreamWriter Writer(Output, support::little);
+  BinaryStreamWriter Writer(Output, llvm::endianness::little);
 
   Err(Writer.writeInteger<uint32_t>(COFF::DEBUG_SECTION_MAGIC));
   for (const auto &B : Builders) {
@@ -314,8 +314,8 @@ template <typename value_type>
 raw_ostream &operator<<(raw_ostream &OS,
                         const binary_le_impl<value_type> &BLE) {
   char Buffer[sizeof(BLE.Value)];
-  support::endian::write<value_type, support::little, support::unaligned>(
-      Buffer, BLE.Value);
+  support::endian::write<value_type, llvm::endianness::little>(Buffer,
+                                                               BLE.Value);
   OS.write(Buffer, sizeof(BLE.Value));
   return OS;
 }
diff --git a/llvm/lib/ObjectYAML/CodeViewYAMLDebugSections.cpp b/llvm/lib/ObjectYAML/CodeViewYAMLDebugSections.cpp
index 02f053bb0e..662eb63f83 100644
--- a/llvm/lib/ObjectYAML/CodeViewYAMLDebugSections.cpp
+++ b/llvm/lib/ObjectYAML/CodeViewYAMLDebugSections.cpp
@@ -900,7 +900,7 @@ YAMLDebugSubsection::fromCodeViewSubection(const StringsAndChecksumsRef &SC,
 std::vector<YAMLDebugSubsection>
 llvm::CodeViewYAML::fromDebugS(ArrayRef<uint8_t> Data,
                                const StringsAndChecksumsRef &SC) {
-  BinaryStreamReader Reader(Data, support::little);
+  BinaryStreamReader Reader(Data, llvm::endianness::little);
   uint32_t Magic;
 
   ExitOnError Err("Invalid .debug$S section!");
diff --git a/llvm/lib/ObjectYAML/CodeViewYAMLTypeHashing.cpp b/llvm/lib/ObjectYAML/CodeViewYAMLTypeHashing.cpp
index e921ae1e7d..0ec8662f42 100644
--- a/llvm/lib/ObjectYAML/CodeViewYAMLTypeHashing.cpp
+++ b/llvm/lib/ObjectYAML/CodeViewYAMLTypeHashing.cpp
@@ -49,7 +49,7 @@ DebugHSection llvm::CodeViewYAML::fromDebugH(ArrayRef<uint8_t> DebugH) {
   assert(DebugH.size() >= 8);
   assert((DebugH.size() - 8) % 8 == 0);
 
-  BinaryStreamReader Reader(DebugH, llvm::support::little);
+  BinaryStreamReader Reader(DebugH, llvm::endianness::little);
   DebugHSection DHS;
   cantFail(Reader.readInteger(DHS.Magic));
   cantFail(Reader.readInteger(DHS.Version));
@@ -69,7 +69,7 @@ ArrayRef<uint8_t> llvm::CodeViewYAML::toDebugH(const DebugHSection &DebugH,
   uint32_t Size = 8 + 8 * DebugH.Hashes.size();
   uint8_t *Data = Alloc.Allocate<uint8_t>(Size);
   MutableArrayRef<uint8_t> Buffer(Data, Size);
-  BinaryStreamWriter Writer(Buffer, llvm::support::little);
+  BinaryStreamWriter Writer(Buffer, llvm::endianness::little);
 
   cantFail(Writer.writeInteger(DebugH.Magic));
   cantFail(Writer.writeInteger(DebugH.Version));
diff --git a/llvm/lib/ObjectYAML/CodeViewYAMLTypes.cpp b/llvm/lib/ObjectYAML/CodeViewYAMLTypes.cpp
index e4e2b2a6d2..99689786a1 100644
--- a/llvm/lib/ObjectYAML/CodeViewYAMLTypes.cpp
+++ b/llvm/lib/ObjectYAML/CodeViewYAMLTypes.cpp
@@ -784,7 +784,7 @@ std::vector<LeafRecord>
 llvm::CodeViewYAML::fromDebugT(ArrayRef<uint8_t> DebugTorP,
                                StringRef SectionName) {
   ExitOnError Err("Invalid " + std::string(SectionName) + " section!");
-  BinaryStreamReader Reader(DebugTorP, support::little);
+  BinaryStreamReader Reader(DebugTorP, llvm::endianness::little);
   CVTypeArray Types;
   uint32_t Magic;
 
@@ -813,7 +813,7 @@ ArrayRef<uint8_t> llvm::CodeViewYAML::toDebugT(ArrayRef<LeafRecord> Leafs,
   }
   uint8_t *ResultBuffer = Alloc.Allocate<uint8_t>(Size);
   MutableArrayRef<uint8_t> Output(ResultBuffer, Size);
-  BinaryStreamWriter Writer(Output, support::little);
+  BinaryStreamWriter Writer(Output, llvm::endianness::little);
   ExitOnError Err("Error writing type record to " + std::string(SectionName) +
                   " section");
   Err(Writer.writeInteger<uint32_t>(COFF::DEBUG_SECTION_MAGIC));
diff --git a/llvm/lib/ObjectYAML/ELFEmitter.cpp b/llvm/lib/ObjectYAML/ELFEmitter.cpp
index a64ab62ef2..40f81f867e 100644
--- a/llvm/lib/ObjectYAML/ELFEmitter.cpp
+++ b/llvm/lib/ObjectYAML/ELFEmitter.cpp
@@ -122,7 +122,7 @@ class ContiguousBlobAccumulator {
     return encodeULEB128(Val, OS);
   }
 
-  template <typename T> void write(T Val, support::endianness E) {
+  template <typename T> void write(T Val, llvm::endianness E) {
     if (checkLimit(sizeof(T)))
       support::endian::write<T>(OS, Val, E);
   }
diff --git a/llvm/lib/ObjectYAML/ELFYAML.cpp b/llvm/lib/ObjectYAML/ELFYAML.cpp
index e92c61d810..872b89420a 100644
--- a/llvm/lib/ObjectYAML/ELFYAML.cpp
+++ b/llvm/lib/ObjectYAML/ELFYAML.cpp
@@ -131,6 +131,7 @@ void ScalarEnumerationTraits<ELFYAML::ELF_NT>::enumeration(
   ECase(NT_ARM_HW_WATCH);
   ECase(NT_ARM_SVE);
   ECase(NT_ARM_PAC_MASK);
+  ECase(NT_ARM_TAGGED_ADDR_CTRL);
   ECase(NT_ARM_SSVE);
   ECase(NT_ARM_ZA);
   ECase(NT_ARM_ZT);
diff --git a/llvm/lib/ObjectYAML/XCOFFEmitter.cpp b/llvm/lib/ObjectYAML/XCOFFEmitter.cpp
index 7ad878f04c..ccf768c06a 100644
--- a/llvm/lib/ObjectYAML/XCOFFEmitter.cpp
+++ b/llvm/lib/ObjectYAML/XCOFFEmitter.cpp
@@ -33,7 +33,7 @@ constexpr uint32_t MaxRawDataSize = UINT32_MAX;
 class XCOFFWriter {
 public:
   XCOFFWriter(XCOFFYAML::Object &Obj, raw_ostream &OS, yaml::ErrorHandler EH)
-      : Obj(Obj), W(OS, support::big), ErrHandler(EH),
+      : Obj(Obj), W(OS, llvm::endianness::big), ErrHandler(EH),
         StrTblBuilder(StringTableBuilder::XCOFF) {
     Is64Bit = Obj.Header.Magic == (llvm::yaml::Hex16)XCOFF::XCOFF64;
   }
diff --git a/llvm/lib/Option/ArgList.cpp b/llvm/lib/Option/ArgList.cpp
index 86f28e578e..8f764ec779 100644
--- a/llvm/lib/Option/ArgList.cpp
+++ b/llvm/lib/Option/ArgList.cpp
@@ -132,17 +132,14 @@ void ArgList::AddAllArgsExcept(ArgStringList &Output,
 }
 
 /// This is a nicer interface when you don't have a list of Ids to exclude.
-void ArgList::AddAllArgs(ArgStringList &Output,
+void ArgList::addAllArgs(ArgStringList &Output,
                          ArrayRef<OptSpecifier> Ids) const {
   ArrayRef<OptSpecifier> Exclude = std::nullopt;
   AddAllArgsExcept(Output, Ids, Exclude);
 }
 
-/// This 3-opt variant of AddAllArgs could be eliminated in favor of one
-/// that accepts a single specifier, given the above which accepts any number.
-void ArgList::AddAllArgs(ArgStringList &Output, OptSpecifier Id0,
-                         OptSpecifier Id1, OptSpecifier Id2) const {
-  for (auto *Arg : filtered(Id0, Id1, Id2)) {
+void ArgList::AddAllArgs(ArgStringList &Output, OptSpecifier Id0) const {
+  for (auto *Arg : filtered(Id0)) {
     Arg->claim();
     Arg->render(*this, Output);
   }
diff --git a/llvm/lib/Passes/CMakeLists.txt b/llvm/lib/Passes/CMakeLists.txt
index 576d0f3ff4..e42edfe949 100644
--- a/llvm/lib/Passes/CMakeLists.txt
+++ b/llvm/lib/Passes/CMakeLists.txt
@@ -19,6 +19,7 @@ add_llvm_component_library(LLVMPasses
   CodeGen
   Core
   Coroutines
+  HipStdPar
   IPO
   InstCombine
   IRPrinter
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 985ff88139..fde759026e 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -94,6 +94,7 @@
 #include "llvm/Transforms/Coroutines/CoroEarly.h"
 #include "llvm/Transforms/Coroutines/CoroElide.h"
 #include "llvm/Transforms/Coroutines/CoroSplit.h"
+#include "llvm/Transforms/HipStdPar/HipStdPar.h"
 #include "llvm/Transforms/IPO/AlwaysInliner.h"
 #include "llvm/Transforms/IPO/Annotation2Metadata.h"
 #include "llvm/Transforms/IPO/ArgumentPromotion.h"
diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp
index 78e0e63530..600f8d43ca 100644
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -37,6 +37,7 @@
 #include "llvm/Transforms/Coroutines/CoroEarly.h"
 #include "llvm/Transforms/Coroutines/CoroElide.h"
 #include "llvm/Transforms/Coroutines/CoroSplit.h"
+#include "llvm/Transforms/HipStdPar/HipStdPar.h"
 #include "llvm/Transforms/IPO/AlwaysInliner.h"
 #include "llvm/Transforms/IPO/Annotation2Metadata.h"
 #include "llvm/Transforms/IPO/ArgumentPromotion.h"
@@ -276,11 +277,13 @@ static cl::opt<bool> UseLoopVersioningLICM(
     "enable-loop-versioning-licm", cl::init(false), cl::Hidden,
     cl::desc("Enable the experimental Loop Versioning LICM pass"));
 
+namespace llvm {
 cl::opt<bool> EnableMemProfContextDisambiguation(
     "enable-memprof-context-disambiguation", cl::init(false), cl::Hidden,
     cl::ZeroOrMore, cl::desc("Enable MemProf context disambiguation"));
 
 extern cl::opt<bool> EnableInferAlignmentPass;
+} // namespace llvm
 
 PipelineTuningOptions::PipelineTuningOptions() {
   LoopInterleaving = true;
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index df9f14920f..91782d661d 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -64,6 +64,9 @@ MODULE_PASS("forceattrs", ForceFunctionAttrsPass())
 MODULE_PASS("function-import", FunctionImportPass())
 MODULE_PASS("globalopt", GlobalOptPass())
 MODULE_PASS("globalsplit", GlobalSplitPass())
+MODULE_PASS("hipstdpar-select-accelerator-code",
+  HipStdParAcceleratorCodeSelectionPass())
+MODULE_PASS("hipstdpar-interpose-alloc", HipStdParAllocationInterpositionPass())
 MODULE_PASS("hotcoldsplit", HotColdSplittingPass())
 MODULE_PASS("inferattrs", InferFunctionAttrsPass())
 MODULE_PASS("inliner-wrapper", ModuleInlinerWrapperPass())
diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp
index 879a5bd805..06cc58c021 100644
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -14,11 +14,11 @@
 
 #include "llvm/Passes/StandardInstrumentations.h"
 #include "llvm/ADT/Any.h"
+#include "llvm/ADT/StableHashing.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Analysis/CallGraphSCCPass.h"
 #include "llvm/Analysis/LazyCallGraph.h"
 #include "llvm/Analysis/LoopInfo.h"
-#include "llvm/CodeGen/StableHashing.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Module.h"
@@ -812,7 +812,8 @@ void PrintIRInstrumentation::printBeforePass(StringRef PassID, Any IR) {
   ++CurrentPassNumber;
 
   if (shouldPrintPassNumbers())
-    dbgs() << " Running pass " << CurrentPassNumber << " " << PassID << "\n";   
+    dbgs() << " Running pass " << CurrentPassNumber << " " << PassID
+           << " on " << getIRName(IR) << "\n";
 
   if (!shouldPrintBeforePass(PassID))
     return;
diff --git a/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp b/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp
index 2a4ed54c2b..380a3aef9b 100644
--- a/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp
+++ b/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp
@@ -15,7 +15,6 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallBitVector.h"
-#include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
diff --git a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
index e468fbf718..857498256e 100644
--- a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
+++ b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
@@ -535,7 +535,7 @@ struct CovMapFuncRecordReader {
                       const char *OutOfLineMappingBuf,
                       const char *OutOfLineMappingBufEnd) = 0;
 
-  template <class IntPtrT, support::endianness Endian>
+  template <class IntPtrT, llvm::endianness Endian>
   static Expected<std::unique_ptr<CovMapFuncRecordReader>>
   get(CovMapVersion Version, InstrProfSymtab &P,
       std::vector<BinaryCoverageReader::ProfileMappingRecord> &R, StringRef D,
@@ -543,7 +543,7 @@ struct CovMapFuncRecordReader {
 };
 
 // A class for reading coverage mapping function records for a module.
-template <CovMapVersion Version, class IntPtrT, support::endianness Endian>
+template <CovMapVersion Version, class IntPtrT, llvm::endianness Endian>
 class VersionedCovMapFuncRecordReader : public CovMapFuncRecordReader {
   using FuncRecordType =
       typename CovMapTraits<Version, IntPtrT>::CovMapFuncRecordType;
@@ -767,7 +767,7 @@ class VersionedCovMapFuncRecordReader : public CovMapFuncRecordReader {
 
 } // end anonymous namespace
 
-template <class IntPtrT, support::endianness Endian>
+template <class IntPtrT, llvm::endianness Endian>
 Expected<std::unique_ptr<CovMapFuncRecordReader>> CovMapFuncRecordReader::get(
     CovMapVersion Version, InstrProfSymtab &P,
     std::vector<BinaryCoverageReader::ProfileMappingRecord> &R, StringRef D,
@@ -805,7 +805,7 @@ Expected<std::unique_ptr<CovMapFuncRecordReader>> CovMapFuncRecordReader::get(
   llvm_unreachable("Unsupported version");
 }
 
-template <typename T, support::endianness Endian>
+template <typename T, llvm::endianness Endian>
 static Error readCoverageMappingData(
     InstrProfSymtab &ProfileNames, StringRef CovMap, StringRef FuncRecords,
     std::vector<BinaryCoverageReader::ProfileMappingRecord> &Records,
@@ -852,30 +852,28 @@ Expected<std::unique_ptr<BinaryCoverageReader>>
 BinaryCoverageReader::createCoverageReaderFromBuffer(
     StringRef Coverage, FuncRecordsStorage &&FuncRecords,
     InstrProfSymtab &&ProfileNames, uint8_t BytesInAddress,
-    support::endianness Endian, StringRef CompilationDir) {
+    llvm::endianness Endian, StringRef CompilationDir) {
   std::unique_ptr<BinaryCoverageReader> Reader(
       new BinaryCoverageReader(std::move(FuncRecords)));
   Reader->ProfileNames = std::move(ProfileNames);
   StringRef FuncRecordsRef = Reader->FuncRecords->getBuffer();
-  if (BytesInAddress == 4 && Endian == support::endianness::little) {
-    if (Error E =
-            readCoverageMappingData<uint32_t, support::endianness::little>(
-                Reader->ProfileNames, Coverage, FuncRecordsRef,
-                Reader->MappingRecords, CompilationDir, Reader->Filenames))
+  if (BytesInAddress == 4 && Endian == llvm::endianness::little) {
+    if (Error E = readCoverageMappingData<uint32_t, llvm::endianness::little>(
+            Reader->ProfileNames, Coverage, FuncRecordsRef,
+            Reader->MappingRecords, CompilationDir, Reader->Filenames))
       return std::move(E);
-  } else if (BytesInAddress == 4 && Endian == support::endianness::big) {
-    if (Error E = readCoverageMappingData<uint32_t, support::endianness::big>(
+  } else if (BytesInAddress == 4 && Endian == llvm::endianness::big) {
+    if (Error E = readCoverageMappingData<uint32_t, llvm::endianness::big>(
             Reader->ProfileNames, Coverage, FuncRecordsRef,
             Reader->MappingRecords, CompilationDir, Reader->Filenames))
       return std::move(E);
-  } else if (BytesInAddress == 8 && Endian == support::endianness::little) {
-    if (Error E =
-            readCoverageMappingData<uint64_t, support::endianness::little>(
-                Reader->ProfileNames, Coverage, FuncRecordsRef,
-                Reader->MappingRecords, CompilationDir, Reader->Filenames))
+  } else if (BytesInAddress == 8 && Endian == llvm::endianness::little) {
+    if (Error E = readCoverageMappingData<uint64_t, llvm::endianness::little>(
+            Reader->ProfileNames, Coverage, FuncRecordsRef,
+            Reader->MappingRecords, CompilationDir, Reader->Filenames))
       return std::move(E);
-  } else if (BytesInAddress == 8 && Endian == support::endianness::big) {
-    if (Error E = readCoverageMappingData<uint64_t, support::endianness::big>(
+  } else if (BytesInAddress == 8 && Endian == llvm::endianness::big) {
+    if (Error E = readCoverageMappingData<uint64_t, llvm::endianness::big>(
             Reader->ProfileNames, Coverage, FuncRecordsRef,
             Reader->MappingRecords, CompilationDir, Reader->Filenames))
       return std::move(E);
@@ -889,7 +887,7 @@ BinaryCoverageReader::createCoverageReaderFromBuffer(
 static Expected<std::unique_ptr<BinaryCoverageReader>>
 loadTestingFormat(StringRef Data, StringRef CompilationDir) {
   uint8_t BytesInAddress = 8;
-  support::endianness Endian = support::endianness::little;
+  llvm::endianness Endian = llvm::endianness::little;
 
   // Read the magic and version.
   Data = Data.substr(sizeof(TestingFormatMagic));
@@ -897,7 +895,7 @@ loadTestingFormat(StringRef Data, StringRef CompilationDir) {
     return make_error<CoverageMapError>(coveragemap_error::malformed,
                                         "the size of data is too small");
   auto TestingVersion =
-      support::endian::byte_swap<uint64_t, support::endianness::little>(
+      support::endian::byte_swap<uint64_t, llvm::endianness::little>(
           *reinterpret_cast<const uint64_t *>(Data.data()));
   Data = Data.substr(sizeof(uint64_t));
 
@@ -957,7 +955,7 @@ loadTestingFormat(StringRef Data, StringRef CompilationDir) {
   auto const *CovHeader = reinterpret_cast<const CovMapHeader *>(
       Data.substr(0, sizeof(CovMapHeader)).data());
   auto Version =
-      CovMapVersion(CovHeader->getVersion<support::endianness::little>());
+      CovMapVersion(CovHeader->getVersion<llvm::endianness::little>());
 
   // In Version1, the size of CoverageMapping is calculated.
   if (TestingVersion == uint64_t(TestingFormatVersion::Version1)) {
@@ -965,7 +963,7 @@ loadTestingFormat(StringRef Data, StringRef CompilationDir) {
       CoverageMappingSize = Data.size();
     } else {
       auto FilenamesSize =
-          CovHeader->getFilenamesSize<support::endianness::little>();
+          CovHeader->getFilenamesSize<llvm::endianness::little>();
       CoverageMappingSize = sizeof(CovMapHeader) + FilenamesSize;
     }
   }
@@ -1046,9 +1044,8 @@ loadBinaryFormat(std::unique_ptr<Binary> Bin, StringRef Arch,
 
   // The coverage uses native pointer sizes for the object it's written in.
   uint8_t BytesInAddress = OF->getBytesInAddress();
-  support::endianness Endian = OF->isLittleEndian()
-                                   ? support::endianness::little
-                                   : support::endianness::big;
+  llvm::endianness Endian =
+      OF->isLittleEndian() ? llvm::endianness::little : llvm::endianness::big;
 
   // Look for the sections that we are interested in.
   auto ObjFormat = OF->getTripleObjectFormat();
@@ -1156,7 +1153,7 @@ BinaryCoverageReader::create(
 
   if (ObjectBuffer.getBuffer().size() > sizeof(TestingFormatMagic)) {
     uint64_t Magic =
-        support::endian::byte_swap<uint64_t, support::endianness::little>(
+        support::endian::byte_swap<uint64_t, llvm::endianness::little>(
             *reinterpret_cast<const uint64_t *>(ObjectBuffer.getBufferStart()));
     if (Magic == TestingFormatMagic) {
       // This is a special format used for testing.
diff --git a/llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp b/llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
index 23d86bd056..2abfbbad16 100644
--- a/llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
+++ b/llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
@@ -252,7 +252,7 @@ void CoverageMappingWriter::write(raw_ostream &OS) {
 
 void TestingFormatWriter::write(raw_ostream &OS, TestingFormatVersion Version) {
   auto ByteSwap = [](uint64_t N) {
-    return support::endian::byte_swap<uint64_t, support::endianness::little>(N);
+    return support::endian::byte_swap<uint64_t, llvm::endianness::little>(N);
   };
 
   // Output a 64bit magic number.
diff --git a/llvm/lib/ProfileData/GCOV.cpp b/llvm/lib/ProfileData/GCOV.cpp
index 0a3330fde1..5fce3dd5f7 100644
--- a/llvm/lib/ProfileData/GCOV.cpp
+++ b/llvm/lib/ProfileData/GCOV.cpp
@@ -365,25 +365,60 @@ GCOVBlock &GCOVFunction::getExitBlock() const {
 // For each basic block, the sum of incoming edge counts equals the sum of
 // outgoing edge counts by Kirchoff's circuit law. If the unmeasured arcs form a
 // spanning tree, the count for each unmeasured arc (GCOV_ARC_ON_TREE) can be
-// uniquely identified.
-uint64_t GCOVFunction::propagateCounts(const GCOVBlock &v, GCOVArc *pred) {
-  // If GCOV_ARC_ON_TREE edges do form a tree, visited is not needed; otherwise
-  // this prevents infinite recursion.
-  if (!visited.insert(&v).second)
-    return 0;
-
-  uint64_t excess = 0;
-  for (GCOVArc *e : v.srcs())
-    if (e != pred)
-      excess += e->onTree() ? propagateCounts(e->src, e) : e->count;
-  for (GCOVArc *e : v.dsts())
-    if (e != pred)
-      excess -= e->onTree() ? propagateCounts(e->dst, e) : e->count;
-  if (int64_t(excess) < 0)
-    excess = -excess;
-  if (pred)
-    pred->count = excess;
-  return excess;
+// uniquely identified. Use an iterative algorithm to decrease stack usage for
+// library users in threads. See the edge propagation algorithm in Optimally
+// Profiling and Tracing Programs, ACM Transactions on Programming Languages and
+// Systems, 1994.
+void GCOVFunction::propagateCounts(const GCOVBlock &v, GCOVArc *pred) {
+  struct Elem {
+    const GCOVBlock &v;
+    GCOVArc *pred;
+    bool inDst;
+    size_t i = 0;
+    uint64_t excess = 0;
+  };
+
+  SmallVector<Elem, 0> stack;
+  stack.push_back({v, pred, false});
+  for (;;) {
+    Elem &u = stack.back();
+    // If GCOV_ARC_ON_TREE edges do form a tree, visited is not needed;
+    // otherwise, this prevents infinite recursion for bad input.
+    if (u.i == 0 && !visited.insert(&u.v).second) {
+      stack.pop_back();
+      if (stack.empty())
+        break;
+      continue;
+    }
+    if (u.i < u.v.pred.size()) {
+      GCOVArc *e = u.v.pred[u.i++];
+      if (e != u.pred) {
+        if (e->onTree())
+          stack.push_back({e->src, e, /*inDst=*/false});
+        else
+          u.excess += e->count;
+      }
+    } else if (u.i < u.v.pred.size() + u.v.succ.size()) {
+      GCOVArc *e = u.v.succ[u.i++ - u.v.pred.size()];
+      if (e != u.pred) {
+        if (e->onTree())
+          stack.push_back({e->dst, e, /*inDst=*/true});
+        else
+          u.excess -= e->count;
+      }
+    } else {
+      uint64_t excess = u.excess;
+      if (static_cast<int64_t>(excess) < 0)
+        excess = -excess;
+      if (u.pred)
+        u.pred->count = excess;
+      bool inDst = u.inDst;
+      stack.pop_back();
+      if (stack.empty())
+        break;
+      stack.back().excess += inDst ? -excess : excess;
+    }
+  }
 }
 
 void GCOVFunction::print(raw_ostream &OS) const {
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index 1375713cb3..ddc1130474 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -982,8 +982,7 @@ void ValueProfRecord::deserializeTo(InstrProfRecord &Record,
 // For writing/serializing,  Old is the host endianness, and  New is
 // byte order intended on disk. For Reading/deserialization, Old
 // is the on-disk source endianness, and New is the host endianness.
-void ValueProfRecord::swapBytes(support::endianness Old,
-                                support::endianness New) {
+void ValueProfRecord::swapBytes(llvm::endianness Old, llvm::endianness New) {
   using namespace support;
 
   if (Old == New)
@@ -1020,13 +1019,13 @@ void ValueProfData::deserializeTo(InstrProfRecord &Record,
 }
 
 template <class T>
-static T swapToHostOrder(const unsigned char *&D, support::endianness Orig) {
+static T swapToHostOrder(const unsigned char *&D, llvm::endianness Orig) {
   using namespace support;
 
-  if (Orig == little)
-    return endian::readNext<T, little, unaligned>(D);
+  if (Orig == llvm::endianness::little)
+    return endian::readNext<T, llvm::endianness::little, unaligned>(D);
   else
-    return endian::readNext<T, big, unaligned>(D);
+    return endian::readNext<T, llvm::endianness::big, unaligned>(D);
 }
 
 static std::unique_ptr<ValueProfData> allocValueProfData(uint32_t TotalSize) {
@@ -1060,7 +1059,7 @@ Error ValueProfData::checkIntegrity() {
 Expected<std::unique_ptr<ValueProfData>>
 ValueProfData::getValueProfData(const unsigned char *D,
                                 const unsigned char *const BufferEnd,
-                                support::endianness Endianness) {
+                                llvm::endianness Endianness) {
   using namespace support;
 
   if (D + sizeof(ValueProfData) > BufferEnd)
@@ -1083,7 +1082,7 @@ ValueProfData::getValueProfData(const unsigned char *D,
   return std::move(VPD);
 }
 
-void ValueProfData::swapBytesToHost(support::endianness Endianness) {
+void ValueProfData::swapBytesToHost(llvm::endianness Endianness) {
   using namespace support;
 
   if (Endianness == llvm::endianness::native)
@@ -1099,7 +1098,7 @@ void ValueProfData::swapBytesToHost(support::endianness Endianness) {
   }
 }
 
-void ValueProfData::swapBytesFromHost(support::endianness Endianness) {
+void ValueProfData::swapBytesFromHost(llvm::endianness Endianness) {
   using namespace support;
 
   if (Endianness == llvm::endianness::native)
@@ -1450,7 +1449,7 @@ static inline uint64_t read(const unsigned char *Buffer, size_t Offset) {
 
 uint64_t Header::formatVersion() const {
   using namespace support;
-  return endian::byte_swap<uint64_t, little>(Version);
+  return endian::byte_swap<uint64_t, llvm::endianness::little>(Version);
 }
 
 Expected<Header> Header::readFromBuffer(const unsigned char *Buffer) {
@@ -1462,7 +1461,8 @@ Expected<Header> Header::readFromBuffer(const unsigned char *Buffer) {
 
   H.Magic = read(Buffer, offsetOf(&Header::Magic));
   // Check the magic number.
-  uint64_t Magic = endian::byte_swap<uint64_t, little>(H.Magic);
+  uint64_t Magic =
+      endian::byte_swap<uint64_t, llvm::endianness::little>(H.Magic);
   if (Magic != IndexedInstrProf::Magic)
     return make_error<InstrProfError>(instrprof_error::bad_magic);
 
diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp
index f3fd7fee4f..a920a31d0a 100644
--- a/llvm/lib/ProfileData/InstrProfReader.cpp
+++ b/llvm/lib/ProfileData/InstrProfReader.cpp
@@ -112,10 +112,11 @@ readBinaryIdsInternal(const MemoryBuffer &DataBuffer,
           "not enough data to read binary id length");
 
     uint64_t BILen = 0;
-    if (Endian == little)
-      BILen = endian::readNext<uint64_t, little, unaligned>(BI);
+    if (Endian == llvm::endianness::little)
+      BILen =
+          endian::readNext<uint64_t, llvm::endianness::little, unaligned>(BI);
     else
-      BILen = endian::readNext<uint64_t, big, unaligned>(BI);
+      BILen = endian::readNext<uint64_t, llvm::endianness::big, unaligned>(BI);
 
     if (BILen == 0)
       return make_error<InstrProfError>(instrprof_error::malformed,
@@ -800,7 +801,8 @@ data_type InstrProfLookupTrait::ReadData(StringRef K, const unsigned char *D,
     // Read hash.
     if (D + sizeof(uint64_t) >= End)
       return data_type();
-    uint64_t Hash = endian::readNext<uint64_t, little, unaligned>(D);
+    uint64_t Hash =
+        endian::readNext<uint64_t, llvm::endianness::little, unaligned>(D);
 
     // Initialize number of counters for GET_VERSION(FormatVersion) == 1.
     uint64_t CountsSize = N / sizeof(uint64_t) - 1;
@@ -808,7 +810,8 @@ data_type InstrProfLookupTrait::ReadData(StringRef K, const unsigned char *D,
     if (GET_VERSION(FormatVersion) != IndexedInstrProf::ProfVersion::Version1) {
       if (D + sizeof(uint64_t) > End)
         return data_type();
-      CountsSize = endian::readNext<uint64_t, little, unaligned>(D);
+      CountsSize =
+          endian::readNext<uint64_t, llvm::endianness::little, unaligned>(D);
     }
     // Read counter values.
     if (D + CountsSize * sizeof(uint64_t) > End)
@@ -817,7 +820,8 @@ data_type InstrProfLookupTrait::ReadData(StringRef K, const unsigned char *D,
     CounterBuffer.clear();
     CounterBuffer.reserve(CountsSize);
     for (uint64_t J = 0; J < CountsSize; ++J)
-      CounterBuffer.push_back(endian::readNext<uint64_t, little, unaligned>(D));
+      CounterBuffer.push_back(
+          endian::readNext<uint64_t, llvm::endianness::little, unaligned>(D));
 
     DataBuffer.emplace_back(K, Hash, std::move(CounterBuffer));
 
@@ -1001,8 +1005,8 @@ bool IndexedInstrProfReader::hasFormat(const MemoryBuffer &DataBuffer) {
 
   if (DataBuffer.getBufferSize() < 8)
     return false;
-  uint64_t Magic =
-      endian::read<uint64_t, little, aligned>(DataBuffer.getBufferStart());
+  uint64_t Magic = endian::read<uint64_t, llvm::endianness::little, aligned>(
+      DataBuffer.getBufferStart());
   // Verify that it's magical.
   return Magic == IndexedInstrProf::Magic;
 }
@@ -1016,10 +1020,10 @@ IndexedInstrProfReader::readSummary(IndexedInstrProf::ProfVersion Version,
   if (Version >= IndexedInstrProf::Version4) {
     const IndexedInstrProf::Summary *SummaryInLE =
         reinterpret_cast<const IndexedInstrProf::Summary *>(Cur);
-    uint64_t NFields =
-        endian::byte_swap<uint64_t, little>(SummaryInLE->NumSummaryFields);
-    uint64_t NEntries =
-        endian::byte_swap<uint64_t, little>(SummaryInLE->NumCutoffEntries);
+    uint64_t NFields = endian::byte_swap<uint64_t, llvm::endianness::little>(
+        SummaryInLE->NumSummaryFields);
+    uint64_t NEntries = endian::byte_swap<uint64_t, llvm::endianness::little>(
+        SummaryInLE->NumCutoffEntries);
     uint32_t SummarySize =
         IndexedInstrProf::Summary::getSize(NFields, NEntries);
     std::unique_ptr<IndexedInstrProf::Summary> SummaryData =
@@ -1028,7 +1032,7 @@ IndexedInstrProfReader::readSummary(IndexedInstrProf::ProfVersion Version,
     const uint64_t *Src = reinterpret_cast<const uint64_t *>(SummaryInLE);
     uint64_t *Dst = reinterpret_cast<uint64_t *>(SummaryData.get());
     for (unsigned I = 0; I < SummarySize / sizeof(uint64_t); I++)
-      Dst[I] = endian::byte_swap<uint64_t, little>(Src[I]);
+      Dst[I] = endian::byte_swap<uint64_t, llvm::endianness::little>(Src[I]);
 
     SummaryEntryVector DetailedSummary;
     for (unsigned I = 0; I < SummaryData->NumCutoffEntries; I++) {
@@ -1085,11 +1089,12 @@ Error IndexedInstrProfReader::readHeader() {
                     /* UseCS */ true);
   // Read the hash type and start offset.
   IndexedInstrProf::HashT HashType = static_cast<IndexedInstrProf::HashT>(
-      endian::byte_swap<uint64_t, little>(Header->HashType));
+      endian::byte_swap<uint64_t, llvm::endianness::little>(Header->HashType));
   if (HashType > IndexedInstrProf::HashT::Last)
     return error(instrprof_error::unsupported_hash_type);
 
-  uint64_t HashOffset = endian::byte_swap<uint64_t, little>(Header->HashOffset);
+  uint64_t HashOffset =
+      endian::byte_swap<uint64_t, llvm::endianness::little>(Header->HashOffset);
 
   // The hash table with profile counts comes next.
   auto IndexPtr = std::make_unique<InstrProfReaderIndex<OnDiskHashTableImplV3>>(
@@ -1100,19 +1105,23 @@ Error IndexedInstrProfReader::readHeader() {
   if (GET_VERSION(Header->formatVersion()) >= 8 &&
       Header->formatVersion() & VARIANT_MASK_MEMPROF) {
     uint64_t MemProfOffset =
-        endian::byte_swap<uint64_t, little>(Header->MemProfOffset);
+        endian::byte_swap<uint64_t, llvm::endianness::little>(
+            Header->MemProfOffset);
 
     const unsigned char *Ptr = Start + MemProfOffset;
     // The value returned from RecordTableGenerator.Emit.
     const uint64_t RecordTableOffset =
-        support::endian::readNext<uint64_t, little, unaligned>(Ptr);
+        support::endian::readNext<uint64_t, llvm::endianness::little,
+                                  unaligned>(Ptr);
     // The offset in the stream right before invoking
     // FrameTableGenerator.Emit.
     const uint64_t FramePayloadOffset =
-        support::endian::readNext<uint64_t, little, unaligned>(Ptr);
+        support::endian::readNext<uint64_t, llvm::endianness::little,
+                                  unaligned>(Ptr);
     // The value returned from FrameTableGenerator.Emit.
     const uint64_t FrameTableOffset =
-        support::endian::readNext<uint64_t, little, unaligned>(Ptr);
+        support::endian::readNext<uint64_t, llvm::endianness::little,
+                                  unaligned>(Ptr);
 
     // Read the schema.
     auto SchemaOr = memprof::readMemProfSchema(Ptr);
@@ -1137,10 +1146,13 @@ Error IndexedInstrProfReader::readHeader() {
   // is higher than 9 (when it was introduced).
   if (GET_VERSION(Header->formatVersion()) >= 9) {
     uint64_t BinaryIdOffset =
-        endian::byte_swap<uint64_t, little>(Header->BinaryIdOffset);
+        endian::byte_swap<uint64_t, llvm::endianness::little>(
+            Header->BinaryIdOffset);
     const unsigned char *Ptr = Start + BinaryIdOffset;
     // Read binary ids size.
-    BinaryIdsSize = support::endian::readNext<uint64_t, little, unaligned>(Ptr);
+    BinaryIdsSize =
+        support::endian::readNext<uint64_t, llvm::endianness::little,
+                                  unaligned>(Ptr);
     if (BinaryIdsSize % sizeof(uint64_t))
       return error(instrprof_error::bad_header);
     // Set the binary ids start.
@@ -1153,31 +1165,37 @@ Error IndexedInstrProfReader::readHeader() {
   if (GET_VERSION(Header->formatVersion()) >= 10 &&
       Header->formatVersion() & VARIANT_MASK_TEMPORAL_PROF) {
     uint64_t TemporalProfTracesOffset =
-        endian::byte_swap<uint64_t, little>(Header->TemporalProfTracesOffset);
+        endian::byte_swap<uint64_t, llvm::endianness::little>(
+            Header->TemporalProfTracesOffset);
     const unsigned char *Ptr = Start + TemporalProfTracesOffset;
     const auto *PtrEnd = (const unsigned char *)DataBuffer->getBufferEnd();
     // Expect at least two 64 bit fields: NumTraces, and TraceStreamSize
     if (Ptr + 2 * sizeof(uint64_t) > PtrEnd)
       return error(instrprof_error::truncated);
     const uint64_t NumTraces =
-        support::endian::readNext<uint64_t, little, unaligned>(Ptr);
+        support::endian::readNext<uint64_t, llvm::endianness::little,
+                                  unaligned>(Ptr);
     TemporalProfTraceStreamSize =
-        support::endian::readNext<uint64_t, little, unaligned>(Ptr);
+        support::endian::readNext<uint64_t, llvm::endianness::little,
+                                  unaligned>(Ptr);
     for (unsigned i = 0; i < NumTraces; i++) {
       // Expect at least two 64 bit fields: Weight and NumFunctions
       if (Ptr + 2 * sizeof(uint64_t) > PtrEnd)
         return error(instrprof_error::truncated);
       TemporalProfTraceTy Trace;
       Trace.Weight =
-          support::endian::readNext<uint64_t, little, unaligned>(Ptr);
+          support::endian::readNext<uint64_t, llvm::endianness::little,
+                                    unaligned>(Ptr);
       const uint64_t NumFunctions =
-          support::endian::readNext<uint64_t, little, unaligned>(Ptr);
+          support::endian::readNext<uint64_t, llvm::endianness::little,
+                                    unaligned>(Ptr);
       // Expect at least NumFunctions 64 bit fields
       if (Ptr + NumFunctions * sizeof(uint64_t) > PtrEnd)
         return error(instrprof_error::truncated);
       for (unsigned j = 0; j < NumFunctions; j++) {
         const uint64_t NameRef =
-            support::endian::readNext<uint64_t, little, unaligned>(Ptr);
+            support::endian::readNext<uint64_t, llvm::endianness::little,
+                                      unaligned>(Ptr);
         Trace.FunctionNameRefs.push_back(NameRef);
       }
       TemporalProfTraces.push_back(std::move(Trace));
@@ -1337,12 +1355,12 @@ Error IndexedInstrProfReader::readNextRecord(NamedInstrProfRecord &Record) {
 Error IndexedInstrProfReader::readBinaryIds(
     std::vector<llvm::object::BuildID> &BinaryIds) {
   return readBinaryIdsInternal(*DataBuffer, BinaryIdsSize, BinaryIdsStart,
-                               BinaryIds, llvm::support::little);
+                               BinaryIds, llvm::endianness::little);
 }
 
 Error IndexedInstrProfReader::printBinaryIds(raw_ostream &OS) {
   return printBinaryIdsInternal(OS, *DataBuffer, BinaryIdsSize, BinaryIdsStart,
-                                llvm::support::little);
+                                llvm::endianness::little);
 }
 
 void InstrProfReader::accumulateCounts(CountSumOrPercent &Sum, bool IsCS) {
diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp
index 2fd09713c9..6892654b00 100644
--- a/llvm/lib/ProfileData/InstrProfWriter.cpp
+++ b/llvm/lib/ProfileData/InstrProfWriter.cpp
@@ -49,9 +49,9 @@ namespace llvm {
 class ProfOStream {
 public:
   ProfOStream(raw_fd_ostream &FD)
-      : IsFDOStream(true), OS(FD), LE(FD, support::little) {}
+      : IsFDOStream(true), OS(FD), LE(FD, llvm::endianness::little) {}
   ProfOStream(raw_string_ostream &STR)
-      : IsFDOStream(false), OS(STR), LE(STR, support::little) {}
+      : IsFDOStream(false), OS(STR), LE(STR, llvm::endianness::little) {}
 
   uint64_t tell() { return OS.tell(); }
   void write(uint64_t V) { LE.write<uint64_t>(V); }
@@ -80,7 +80,8 @@ class ProfOStream {
       std::string &Data = SOStream.str(); // with flush
       for (int K = 0; K < NItems; K++) {
         for (int I = 0; I < P[K].N; I++) {
-          uint64_t Bytes = endian::byte_swap<uint64_t, little>(P[K].D[I]);
+          uint64_t Bytes =
+              endian::byte_swap<uint64_t, llvm::endianness::little>(P[K].D[I]);
           Data.replace(P[K].Pos + I * sizeof(uint64_t), sizeof(uint64_t),
                        (const char *)&Bytes, sizeof(uint64_t));
         }
@@ -106,7 +107,7 @@ class InstrProfRecordWriterTrait {
   using hash_value_type = uint64_t;
   using offset_type = uint64_t;
 
-  support::endianness ValueProfDataEndianness = support::little;
+  llvm::endianness ValueProfDataEndianness = llvm::endianness::little;
   InstrProfSummaryBuilder *SummaryBuilder;
   InstrProfSummaryBuilder *CSSummaryBuilder;
 
@@ -120,7 +121,7 @@ class InstrProfRecordWriterTrait {
   EmitKeyDataLength(raw_ostream &Out, key_type_ref K, data_type_ref V) {
     using namespace support;
 
-    endian::Writer LE(Out, little);
+    endian::Writer LE(Out, llvm::endianness::little);
 
     offset_type N = K.size();
     LE.write<offset_type>(N);
@@ -147,7 +148,7 @@ class InstrProfRecordWriterTrait {
   void EmitData(raw_ostream &Out, key_type_ref, data_type_ref V, offset_type) {
     using namespace support;
 
-    endian::Writer LE(Out, little);
+    endian::Writer LE(Out, llvm::endianness::little);
     for (const auto &ProfileData : *V) {
       const InstrProfRecord &ProfRecord = ProfileData.second;
       if (NamedInstrProfRecord::hasCSFlagInHash(ProfileData.first))
@@ -182,8 +183,7 @@ InstrProfWriter::InstrProfWriter(bool Sparse,
 InstrProfWriter::~InstrProfWriter() { delete InfoObj; }
 
 // Internal interface for testing purpose only.
-void InstrProfWriter::setValueProfDataEndianness(
-    support::endianness Endianness) {
+void InstrProfWriter::setValueProfDataEndianness(llvm::endianness Endianness) {
   InfoObj->ValueProfDataEndianness = Endianness;
 }
 
diff --git a/llvm/lib/ProfileData/MemProf.cpp b/llvm/lib/ProfileData/MemProf.cpp
index 3255cba4dd..db34de704a 100644
--- a/llvm/lib/ProfileData/MemProf.cpp
+++ b/llvm/lib/ProfileData/MemProf.cpp
@@ -13,7 +13,7 @@ void IndexedMemProfRecord::serialize(const MemProfSchema &Schema,
                                      raw_ostream &OS) {
   using namespace support;
 
-  endian::Writer LE(OS, little);
+  endian::Writer LE(OS, llvm::endianness::little);
 
   LE.write<uint64_t>(AllocSites.size());
   for (const IndexedAllocationInfo &N : AllocSites) {
@@ -40,13 +40,15 @@ IndexedMemProfRecord::deserialize(const MemProfSchema &Schema,
   IndexedMemProfRecord Record;
 
   // Read the meminfo nodes.
-  const uint64_t NumNodes = endian::readNext<uint64_t, little, unaligned>(Ptr);
+  const uint64_t NumNodes =
+      endian::readNext<uint64_t, llvm::endianness::little, unaligned>(Ptr);
   for (uint64_t I = 0; I < NumNodes; I++) {
     IndexedAllocationInfo Node;
     const uint64_t NumFrames =
-        endian::readNext<uint64_t, little, unaligned>(Ptr);
+        endian::readNext<uint64_t, llvm::endianness::little, unaligned>(Ptr);
     for (uint64_t J = 0; J < NumFrames; J++) {
-      const FrameId Id = endian::readNext<FrameId, little, unaligned>(Ptr);
+      const FrameId Id =
+          endian::readNext<FrameId, llvm::endianness::little, unaligned>(Ptr);
       Node.CallStack.push_back(Id);
     }
     Node.Info.deserialize(Schema, Ptr);
@@ -55,14 +57,16 @@ IndexedMemProfRecord::deserialize(const MemProfSchema &Schema,
   }
 
   // Read the callsite information.
-  const uint64_t NumCtxs = endian::readNext<uint64_t, little, unaligned>(Ptr);
+  const uint64_t NumCtxs =
+      endian::readNext<uint64_t, llvm::endianness::little, unaligned>(Ptr);
   for (uint64_t J = 0; J < NumCtxs; J++) {
     const uint64_t NumFrames =
-        endian::readNext<uint64_t, little, unaligned>(Ptr);
+        endian::readNext<uint64_t, llvm::endianness::little, unaligned>(Ptr);
     llvm::SmallVector<FrameId> Frames;
     Frames.reserve(NumFrames);
     for (uint64_t K = 0; K < NumFrames; K++) {
-      const FrameId Id = endian::readNext<FrameId, little, unaligned>(Ptr);
+      const FrameId Id =
+          endian::readNext<FrameId, llvm::endianness::little, unaligned>(Ptr);
       Frames.push_back(Id);
     }
     Record.CallSites.push_back(Frames);
@@ -90,7 +94,7 @@ Expected<MemProfSchema> readMemProfSchema(const unsigned char *&Buffer) {
 
   const unsigned char *Ptr = Buffer;
   const uint64_t NumSchemaIds =
-      endian::readNext<uint64_t, little, unaligned>(Ptr);
+      endian::readNext<uint64_t, llvm::endianness::little, unaligned>(Ptr);
   if (NumSchemaIds > static_cast<uint64_t>(Meta::Size)) {
     return make_error<InstrProfError>(instrprof_error::malformed,
                                       "memprof schema invalid");
@@ -98,7 +102,8 @@ Expected<MemProfSchema> readMemProfSchema(const unsigned char *&Buffer) {
 
   MemProfSchema Result;
   for (size_t I = 0; I < NumSchemaIds; I++) {
-    const uint64_t Tag = endian::readNext<uint64_t, little, unaligned>(Ptr);
+    const uint64_t Tag =
+        endian::readNext<uint64_t, llvm::endianness::little, unaligned>(Ptr);
     if (Tag >= static_cast<uint64_t>(Meta::Size)) {
       return make_error<InstrProfError>(instrprof_error::malformed,
                                         "memprof schema invalid");
diff --git a/llvm/lib/ProfileData/RawMemProfReader.cpp b/llvm/lib/ProfileData/RawMemProfReader.cpp
index 0716ec53ce..284e5ec634 100644
--- a/llvm/lib/ProfileData/RawMemProfReader.cpp
+++ b/llvm/lib/ProfileData/RawMemProfReader.cpp
@@ -87,7 +87,7 @@ llvm::SmallVector<SegmentEntry> readSegmentEntries(const char *Ptr) {
   using namespace support;
 
   const uint64_t NumItemsToRead =
-      endian::readNext<uint64_t, little, unaligned>(Ptr);
+      endian::readNext<uint64_t, llvm::endianness::little, unaligned>(Ptr);
   llvm::SmallVector<SegmentEntry> Items;
   for (uint64_t I = 0; I < NumItemsToRead; I++) {
     Items.push_back(*reinterpret_cast<const SegmentEntry *>(
@@ -101,10 +101,11 @@ readMemInfoBlocks(const char *Ptr) {
   using namespace support;
 
   const uint64_t NumItemsToRead =
-      endian::readNext<uint64_t, little, unaligned>(Ptr);
+      endian::readNext<uint64_t, llvm::endianness::little, unaligned>(Ptr);
   llvm::SmallVector<std::pair<uint64_t, MemInfoBlock>> Items;
   for (uint64_t I = 0; I < NumItemsToRead; I++) {
-    const uint64_t Id = endian::readNext<uint64_t, little, unaligned>(Ptr);
+    const uint64_t Id =
+        endian::readNext<uint64_t, llvm::endianness::little, unaligned>(Ptr);
     const MemInfoBlock MIB = *reinterpret_cast<const MemInfoBlock *>(Ptr);
     Items.push_back({Id, MIB});
     // Only increment by size of MIB since readNext implicitly increments.
@@ -117,16 +118,19 @@ CallStackMap readStackInfo(const char *Ptr) {
   using namespace support;
 
   const uint64_t NumItemsToRead =
-      endian::readNext<uint64_t, little, unaligned>(Ptr);
+      endian::readNext<uint64_t, llvm::endianness::little, unaligned>(Ptr);
   CallStackMap Items;
 
   for (uint64_t I = 0; I < NumItemsToRead; I++) {
-    const uint64_t StackId = endian::readNext<uint64_t, little, unaligned>(Ptr);
-    const uint64_t NumPCs = endian::readNext<uint64_t, little, unaligned>(Ptr);
+    const uint64_t StackId =
+        endian::readNext<uint64_t, llvm::endianness::little, unaligned>(Ptr);
+    const uint64_t NumPCs =
+        endian::readNext<uint64_t, llvm::endianness::little, unaligned>(Ptr);
 
     SmallVector<uint64_t> CallStack;
     for (uint64_t J = 0; J < NumPCs; J++) {
-      CallStack.push_back(endian::readNext<uint64_t, little, unaligned>(Ptr));
+      CallStack.push_back(
+          endian::readNext<uint64_t, llvm::endianness::little, unaligned>(Ptr));
     }
 
     Items[StackId] = CallStack;
diff --git a/llvm/lib/ProfileData/SampleProf.cpp b/llvm/lib/ProfileData/SampleProf.cpp
index b14dc01be2..59fa71899e 100644
--- a/llvm/lib/ProfileData/SampleProf.cpp
+++ b/llvm/lib/ProfileData/SampleProf.cpp
@@ -121,7 +121,7 @@ sampleprof_error SampleRecord::merge(const SampleRecord &Other,
   sampleprof_error Result;
   Result = addSamples(Other.getSamples(), Weight);
   for (const auto &I : Other.getCallTargets()) {
-    MergeResult(Result, addCalledTarget(I.first(), I.second, Weight));
+    MergeResult(Result, addCalledTarget(I.first, I.second, Weight));
   }
   return Result;
 }
@@ -181,7 +181,8 @@ void FunctionSamples::print(raw_ostream &OS, unsigned Indent) const {
     for (const auto &CS : SortedCallsiteSamples.get()) {
       for (const auto &FS : CS->second) {
         OS.indent(Indent + 2);
-        OS << CS->first << ": inlined callee: " << FS.second.getName() << ": ";
+        OS << CS->first << ": inlined callee: " << FS.second.getFunction()
+           << ": ";
         FS.second.print(OS, Indent + 4);
       }
     }
@@ -234,14 +235,6 @@ LineLocation FunctionSamples::getCallSiteIdentifier(const DILocation *DIL,
   }
 }
 
-uint64_t FunctionSamples::getCallSiteHash(StringRef CalleeName,
-                                          const LineLocation &Callsite) {
-  uint64_t NameHash = std::hash<std::string>{}(CalleeName.str());
-  uint64_t LocId =
-      (((uint64_t)Callsite.LineOffset) << 32) | Callsite.Discriminator;
-  return NameHash + (LocId << 5) + LocId;
-}
-
 const FunctionSamples *FunctionSamples::findFunctionSamples(
     const DILocation *DIL, SampleProfileReaderItaniumRemapper *Remapper) const {
   assert(DIL);
@@ -268,11 +261,11 @@ const FunctionSamples *FunctionSamples::findFunctionSamples(
   return FS;
 }
 
-void FunctionSamples::findAllNames(DenseSet<StringRef> &NameSet) const {
-  NameSet.insert(getName());
+void FunctionSamples::findAllNames(DenseSet<FunctionId> &NameSet) const {
+  NameSet.insert(getFunction());
   for (const auto &BS : BodySamples)
     for (const auto &TS : BS.second.getCallTargets())
-      NameSet.insert(TS.getKey());
+      NameSet.insert(TS.first);
 
   for (const auto &CS : CallsiteSamples) {
     for (const auto &NameFS : CS.second) {
@@ -287,18 +280,15 @@ const FunctionSamples *FunctionSamples::findFunctionSamplesAt(
     SampleProfileReaderItaniumRemapper *Remapper) const {
   CalleeName = getCanonicalFnName(CalleeName);
 
-  std::string CalleeGUID;
-  CalleeName = getRepInFormat(CalleeName, UseMD5, CalleeGUID);
-
   auto iter = CallsiteSamples.find(mapIRLocToProfileLoc(Loc));
   if (iter == CallsiteSamples.end())
     return nullptr;
-  auto FS = iter->second.find(CalleeName);
+  auto FS = iter->second.find(getRepInFormat(CalleeName));
   if (FS != iter->second.end())
     return &FS->second;
   if (Remapper) {
     if (auto NameInProfile = Remapper->lookUpNameInProfile(CalleeName)) {
-      auto FS = iter->second.find(*NameInProfile);
+      auto FS = iter->second.find(getRepInFormat(*NameInProfile));
       if (FS != iter->second.end())
         return &FS->second;
     }
@@ -422,7 +412,7 @@ void ProfileSymbolList::dump(raw_ostream &OS) const {
 
 ProfileConverter::FrameNode *
 ProfileConverter::FrameNode::getOrCreateChildFrame(const LineLocation &CallSite,
-                                                   StringRef CalleeName) {
+                                                   FunctionId CalleeName) {
   uint64_t Hash = FunctionSamples::getCallSiteHash(CalleeName, CallSite);
   auto It = AllChildFrames.find(Hash);
   if (It != AllChildFrames.end()) {
@@ -450,7 +440,7 @@ ProfileConverter::getOrCreateContextPath(const SampleContext &Context) {
   auto Node = &RootFrame;
   LineLocation CallSiteLoc(0, 0);
   for (auto &Callsite : Context.getContextFrames()) {
-    Node = Node->getOrCreateChildFrame(CallSiteLoc, Callsite.FuncName);
+    Node = Node->getOrCreateChildFrame(CallSiteLoc, Callsite.Func);
     CallSiteLoc = Callsite.Location;
   }
   return Node;
@@ -468,23 +458,23 @@ void ProfileConverter::convertCSProfiles(ProfileConverter::FrameNode &Node) {
     if (!ChildProfile)
       continue;
     SampleContext OrigChildContext = ChildProfile->getContext();
-    hash_code OrigChildContextHash = OrigChildContext.getHashCode();
+    uint64_t OrigChildContextHash = OrigChildContext.getHashCode();
     // Reset the child context to be contextless.
-    ChildProfile->getContext().setName(OrigChildContext.getName());
+    ChildProfile->getContext().setFunction(OrigChildContext.getFunction());
     if (NodeProfile) {
       // Add child profile to the callsite profile map.
       auto &SamplesMap = NodeProfile->functionSamplesAt(ChildNode.CallSiteLoc);
-      SamplesMap.emplace(OrigChildContext.getName().str(), *ChildProfile);
+      SamplesMap.emplace(OrigChildContext.getFunction(), *ChildProfile);
       NodeProfile->addTotalSamples(ChildProfile->getTotalSamples());
       // Remove the corresponding body sample for the callsite and update the
       // total weight.
       auto Count = NodeProfile->removeCalledTargetAndBodySample(
           ChildNode.CallSiteLoc.LineOffset, ChildNode.CallSiteLoc.Discriminator,
-          OrigChildContext.getName());
+          OrigChildContext.getFunction());
       NodeProfile->removeTotalSamples(Count);
     }
 
-    hash_code NewChildProfileHash(0);
+    uint64_t NewChildProfileHash = 0;
     // Separate child profile to be a standalone profile, if the current parent
     // profile doesn't exist. This is a duplicating operation when the child
     // profile is already incorporated into the parent which is still useful and
@@ -498,7 +488,7 @@ void ProfileConverter::convertCSProfiles(ProfileConverter::FrameNode &Node) {
       ProfileMap[ChildProfile->getContext()].merge(*ChildProfile);
       NewChildProfileHash = ChildProfile->getContext().getHashCode();
       auto &SamplesMap = NodeProfile->functionSamplesAt(ChildNode.CallSiteLoc);
-      SamplesMap[ChildProfile->getName().str()].getContext().setAttribute(
+      SamplesMap[ChildProfile->getFunction()].getContext().setAttribute(
           ContextDuplicatedIntoBase);
     }
 
diff --git a/llvm/lib/ProfileData/SampleProfReader.cpp b/llvm/lib/ProfileData/SampleProfReader.cpp
index d632a812b8..a69e9d5849 100644
--- a/llvm/lib/ProfileData/SampleProfReader.cpp
+++ b/llvm/lib/ProfileData/SampleProfReader.cpp
@@ -91,7 +91,7 @@ static void dumpFunctionProfileJson(const FunctionSamples &S,
           JOS.attributeArray("calls", [&] {
             for (const auto &J : CallTargets) {
               JOS.object([&] {
-                JOS.attribute("function", J.first);
+                JOS.attribute("function", J.first.str());
                 JOS.attribute("samples", J.second);
               });
             }
@@ -117,7 +117,7 @@ static void dumpFunctionProfileJson(const FunctionSamples &S,
   };
 
   JOS.object([&] {
-    JOS.attribute("name", S.getName());
+    JOS.attribute("name", S.getFunction().str());
     JOS.attribute("total", S.getTotalSamples());
     if (TopLevel)
       JOS.attribute("head", S.getHeadSamples());
@@ -392,8 +392,8 @@ std::error_code SampleProfileReaderText::readImpl() {
       switch (LineTy) {
       case LineType::CallSiteProfile: {
         FunctionSamples &FSamples = InlineStack.back()->functionSamplesAt(
-            LineLocation(LineOffset, Discriminator))[std::string(FName)];
-        FSamples.setName(FName);
+            LineLocation(LineOffset, Discriminator))[FunctionId(FName)];
+        FSamples.setFunction(FunctionId(FName));
         MergeResult(Result, FSamples.addTotalSamples(NumSamples));
         InlineStack.push_back(&FSamples);
         DepthMetadata = 0;
@@ -406,7 +406,8 @@ std::error_code SampleProfileReaderText::readImpl() {
         FunctionSamples &FProfile = *InlineStack.back();
         for (const auto &name_count : TargetCountMap) {
           MergeResult(Result, FProfile.addCalledTargetSamples(
-                                  LineOffset, Discriminator, name_count.first,
+                                  LineOffset, Discriminator,
+                                  FunctionId(name_count.first),
                                   name_count.second));
         }
         MergeResult(Result, FProfile.addBodySamples(LineOffset, Discriminator,
@@ -502,7 +503,7 @@ ErrorOr<T> SampleProfileReaderBinary::readUnencodedNumber() {
   }
 
   using namespace support;
-  T Val = endian::readNext<T, little, unaligned>(Data);
+  T Val = endian::readNext<T, llvm::endianness::little, unaligned>(Data);
   return Val;
 }
 
@@ -516,28 +517,14 @@ inline ErrorOr<size_t> SampleProfileReaderBinary::readStringIndex(T &Table) {
   return *Idx;
 }
 
-ErrorOr<StringRef>
+ErrorOr<FunctionId>
 SampleProfileReaderBinary::readStringFromTable(size_t *RetIdx) {
   auto Idx = readStringIndex(NameTable);
   if (std::error_code EC = Idx.getError())
     return EC;
-
-  // Lazy loading, if the string has not been materialized from memory storing
-  // MD5 values, then it is default initialized with the null pointer. This can
-  // only happen when using fixed length MD5, that bounds check is performed
-  // while parsing the name table to ensure MD5NameMemStart points to an array
-  // with enough MD5 entries.
-  StringRef &SR = NameTable[*Idx];
-  if (!SR.data()) {
-    assert(MD5NameMemStart);
-    using namespace support;
-    uint64_t FID = endian::read<uint64_t, little, unaligned>(
-       MD5NameMemStart + (*Idx) * sizeof(uint64_t));
-    SR = MD5StringBuf.emplace_back(std::to_string(FID));
-  }
   if (RetIdx)
     *RetIdx = *Idx;
-  return SR;
+  return NameTable[*Idx];
 }
 
 ErrorOr<SampleContextFrames>
@@ -655,8 +642,8 @@ SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) {
     uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask();
 
     FunctionSamples &CalleeProfile = FProfile.functionSamplesAt(
-        LineLocation(*LineOffset, DiscriminatorVal))[std::string(*FName)];
-    CalleeProfile.setName(*FName);
+        LineLocation(*LineOffset, DiscriminatorVal))[*FName];
+    CalleeProfile.setFunction(*FName);
     if (std::error_code EC = readProfile(CalleeProfile))
       return EC;
   }
@@ -894,13 +881,17 @@ std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles() {
       const SampleContext *CommonContext = nullptr;
       for (const auto &NameOffset : FuncOffsetList) {
         const auto &FContext = NameOffset.first;
-        auto FName = FContext.getName();
+        FunctionId FName = FContext.getFunction();
+        StringRef FNameString;
+        if (!useMD5())
+          FNameString = FName.stringRef();
+
         // For function in the current module, keep its farthest ancestor
         // context. This can be used to load itself and its child and
         // sibling contexts.
-        if ((useMD5() && FuncGuidsToUse.count(std::stoull(FName.data()))) ||
-            (!useMD5() && (FuncsToUse.count(FName) ||
-                           (Remapper && Remapper->exist(FName))))) {
+        if ((useMD5() && FuncGuidsToUse.count(FName.getHashCode())) ||
+            (!useMD5() && (FuncsToUse.count(FNameString) ||
+                           (Remapper && Remapper->exist(FNameString))))) {
           if (!CommonContext || !CommonContext->IsPrefixOf(FContext))
             CommonContext = &FContext;
         }
@@ -929,8 +920,9 @@ std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles() {
       assert(useFuncOffsetList());
       for (auto NameOffset : FuncOffsetList) {
         SampleContext FContext(NameOffset.first);
-        auto FuncName = FContext.getName();
-        if (!FuncsToUse.count(FuncName) && !Remapper->exist(FuncName))
+        auto FuncName = FContext.getFunction();
+        StringRef FuncNameStr = FuncName.stringRef();
+        if (!FuncsToUse.count(FuncNameStr) && !Remapper->exist(FuncNameStr))
           continue;
         const uint8_t *FuncProfileAddr = Start + NameOffset.second;
         if (std::error_code EC = readFuncProfile(FuncProfileAddr))
@@ -1062,8 +1054,6 @@ std::error_code SampleProfileReaderBinary::readNameTable() {
   // tables mixing string and MD5, all of them have to be normalized to use MD5,
   // because optimization passes can only handle either type.
   bool UseMD5 = useMD5();
-  if (UseMD5)
-    MD5StringBuf.reserve(MD5StringBuf.size() + *Size);
 
   NameTable.clear();
   NameTable.reserve(*Size);
@@ -1082,12 +1072,12 @@ std::error_code SampleProfileReaderBinary::readNameTable() {
     if (std::error_code EC = Name.getError())
       return EC;
     if (UseMD5) {
-      uint64_t FID = hashFuncName(*Name);
+      FunctionId FID(*Name);
       if (!ProfileIsCS)
-        MD5SampleContextTable.emplace_back(FID);
-      NameTable.emplace_back(MD5StringBuf.emplace_back(std::to_string(FID)));
+        MD5SampleContextTable.emplace_back(FID.getHashCode());
+      NameTable.emplace_back(FID);
     } else
-      NameTable.push_back(*Name);
+      NameTable.push_back(FunctionId(*Name));
   }
   if (!ProfileIsCS)
     MD5SampleContextStart = MD5SampleContextTable.data();
@@ -1110,14 +1100,14 @@ SampleProfileReaderExtBinaryBase::readNameTableSec(bool IsMD5,
     if (Data + (*Size) * sizeof(uint64_t) > End)
       return sampleprof_error::truncated;
 
-    // Preallocate and initialize NameTable so we can check whether a name
-    // index has been read before by checking whether the element in the
-    // NameTable is empty, meanwhile readStringIndex can do the boundary
-    // check using the size of NameTable.
-    MD5StringBuf.reserve(MD5StringBuf.size() + *Size);
     NameTable.clear();
-    NameTable.resize(*Size);
-    MD5NameMemStart = Data;
+    NameTable.reserve(*Size);
+    for (size_t I = 0; I < *Size; ++I) {
+      using namespace support;
+      uint64_t FID = endian::read<uint64_t, endianness::little, unaligned>(
+          Data + I * sizeof(uint64_t));
+      NameTable.emplace_back(FunctionId(FID));
+    }
     if (!ProfileIsCS)
       MD5SampleContextStart = reinterpret_cast<const uint64_t *>(Data);
     Data = Data + (*Size) * sizeof(uint64_t);
@@ -1130,7 +1120,6 @@ SampleProfileReaderExtBinaryBase::readNameTableSec(bool IsMD5,
     if (std::error_code EC = Size.getError())
       return EC;
 
-    MD5StringBuf.reserve(MD5StringBuf.size() + *Size);
     NameTable.clear();
     NameTable.reserve(*Size);
     if (!ProfileIsCS)
@@ -1141,7 +1130,7 @@ SampleProfileReaderExtBinaryBase::readNameTableSec(bool IsMD5,
         return EC;
       if (!ProfileIsCS)
         support::endian::write64le(&MD5SampleContextTable[I], *FID);
-      NameTable.emplace_back(MD5StringBuf.emplace_back(std::to_string(*FID)));
+      NameTable.emplace_back(FunctionId(*FID));
     }
     if (!ProfileIsCS)
       MD5SampleContextStart = MD5SampleContextTable.data();
@@ -1243,7 +1232,7 @@ SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute,
           CalleeProfile = const_cast<FunctionSamples *>(
               &FProfile->functionSamplesAt(LineLocation(
                   *LineOffset,
-                  *Discriminator))[std::string(FContext.getName())]);
+                  *Discriminator))[FContext.getFunction()]);
         }
         if (std::error_code EC =
                 readFuncMetadata(ProfileHasAttribute, CalleeProfile))
@@ -1653,7 +1642,7 @@ std::error_code SampleProfileReaderGCC::readOneFunctionProfile(
     // body, there will be identical replicated profiles for the
     // original function.  In this case, we simply not bother updating
     // the profile of the original function.
-    FProfile = &Profiles[Name];
+    FProfile = &Profiles[FunctionId(Name)];
     FProfile->addHeadSamples(HeadCount);
     if (FProfile->getTotalSamples() > 0)
       Update = false;
@@ -1665,9 +1654,9 @@ std::error_code SampleProfileReaderGCC::readOneFunctionProfile(
     uint32_t LineOffset = Offset >> 16;
     uint32_t Discriminator = Offset & 0xffff;
     FProfile = &CallerProfile->functionSamplesAt(
-        LineLocation(LineOffset, Discriminator))[std::string(Name)];
+        LineLocation(LineOffset, Discriminator))[FunctionId(Name)];
   }
-  FProfile->setName(Name);
+  FProfile->setFunction(FunctionId(Name));
 
   for (uint32_t I = 0; I < NumPosCounts; ++I) {
     uint32_t Offset;
@@ -1723,7 +1712,8 @@ std::error_code SampleProfileReaderGCC::readOneFunctionProfile(
 
       if (Update)
         FProfile->addCalledTargetSamples(LineOffset, Discriminator,
-                                         TargetName, TargetCount);
+                                         FunctionId(TargetName),
+                                         TargetCount);
     }
   }
 
@@ -1784,11 +1774,13 @@ void SampleProfileReaderItaniumRemapper::applyRemapping(LLVMContext &Ctx) {
   // We will need to remap the entire context string.
   assert(Remappings && "should be initialized while creating remapper");
   for (auto &Sample : Reader.getProfiles()) {
-    DenseSet<StringRef> NamesInSample;
+    DenseSet<FunctionId> NamesInSample;
     Sample.second.findAllNames(NamesInSample);
-    for (auto &Name : NamesInSample)
-      if (auto Key = Remappings->insert(Name))
-        NameMap.insert({Key, Name});
+    for (auto &Name : NamesInSample) {
+      StringRef NameStr = Name.stringRef();
+      if (auto Key = Remappings->insert(NameStr))
+        NameMap.insert({Key, NameStr});
+    }
   }
 
   RemappingApplied = true;
diff --git a/llvm/lib/ProfileData/SampleProfWriter.cpp b/llvm/lib/ProfileData/SampleProfWriter.cpp
index 74990238d3..625e523f13 100644
--- a/llvm/lib/ProfileData/SampleProfWriter.cpp
+++ b/llvm/lib/ProfileData/SampleProfWriter.cpp
@@ -240,7 +240,7 @@ std::error_code SampleProfileWriterExtBinaryBase::writeContextIdx(
   if (Context.hasContext())
     return writeCSNameIdx(Context);
   else
-    return SampleProfileWriterBinary::writeNameIdx(Context.getName());
+    return SampleProfileWriterBinary::writeNameIdx(Context.getFunction());
 }
 
 std::error_code
@@ -346,16 +346,16 @@ std::error_code SampleProfileWriterExtBinaryBase::writeNameTable() {
     return SampleProfileWriterBinary::writeNameTable();
 
   auto &OS = *OutputStream;
-  std::set<StringRef> V;
+  std::set<FunctionId> V;
   stablizeNameTable(NameTable, V);
 
   // Write out the MD5 name table. We wrote unencoded MD5 so reader can
   // retrieve the name using the name index without having to read the
   // whole name table.
   encodeULEB128(NameTable.size(), OS);
-  support::endian::Writer Writer(OS, support::little);
+  support::endian::Writer Writer(OS, llvm::endianness::little);
   for (auto N : V)
-    Writer.write(hashFuncName(N));
+    Writer.write(N.getHashCode());
   return sampleprof_error::success;
 }
 
@@ -369,10 +369,13 @@ std::error_code SampleProfileWriterExtBinaryBase::writeNameTableSection(
   // If NameTable contains ".__uniq." suffix, set SecFlagUniqSuffix flag
   // so compiler won't strip the suffix during profile matching after
   // seeing the flag in the profile.
-  for (const auto &I : NameTable) {
-    if (I.first.contains(FunctionSamples::UniqSuffix)) {
-      addSectionFlag(SecNameTable, SecNameTableFlags::SecFlagUniqSuffix);
-      break;
+  // Original names are unavailable if using MD5, so this option has no use.
+  if (!UseMD5) {
+    for (const auto &I : NameTable) {
+      if (I.first.stringRef().contains(FunctionSamples::UniqSuffix)) {
+        addSectionFlag(SecNameTable, SecNameTableFlags::SecFlagUniqSuffix);
+        break;
+      }
     }
   }
 
@@ -394,12 +397,12 @@ std::error_code SampleProfileWriterExtBinaryBase::writeCSNameTableSection() {
 
   auto &OS = *OutputStream;
   encodeULEB128(OrderedContexts.size(), OS);
-  support::endian::Writer Writer(OS, support::little);
+  support::endian::Writer Writer(OS, llvm::endianness::little);
   for (auto Context : OrderedContexts) {
     auto Frames = Context.getContextFrames();
     encodeULEB128(Frames.size(), OS);
     for (auto &Callsite : Frames) {
-      if (std::error_code EC = writeNameIdx(Callsite.FuncName))
+      if (std::error_code EC = writeNameIdx(Callsite.Func))
         return EC;
       encodeULEB128(Callsite.Location.LineOffset, OS);
       encodeULEB128(Callsite.Location.Discriminator, OS);
@@ -567,7 +570,7 @@ std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) {
   if (FunctionSamples::ProfileIsCS)
     OS << "[" << S.getContext().toString() << "]:" << S.getTotalSamples();
   else
-    OS << S.getName() << ":" << S.getTotalSamples();
+    OS << S.getFunction() << ":" << S.getTotalSamples();
 
   if (Indent == 0)
     OS << ":" << S.getHeadSamples();
@@ -627,10 +630,10 @@ std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) {
 std::error_code
 SampleProfileWriterBinary::writeContextIdx(const SampleContext &Context) {
   assert(!Context.hasContext() && "cs profile is not supported");
-  return writeNameIdx(Context.getName());
+  return writeNameIdx(Context.getFunction());
 }
 
-std::error_code SampleProfileWriterBinary::writeNameIdx(StringRef FName) {
+std::error_code SampleProfileWriterBinary::writeNameIdx(FunctionId FName) {
   auto &NTable = getNameTable();
   const auto &Ret = NTable.find(FName);
   if (Ret == NTable.end())
@@ -639,13 +642,13 @@ std::error_code SampleProfileWriterBinary::writeNameIdx(StringRef FName) {
   return sampleprof_error::success;
 }
 
-void SampleProfileWriterBinary::addName(StringRef FName) {
+void SampleProfileWriterBinary::addName(FunctionId FName) {
   auto &NTable = getNameTable();
   NTable.insert(std::make_pair(FName, 0));
 }
 
 void SampleProfileWriterBinary::addContext(const SampleContext &Context) {
-  addName(Context.getName());
+  addName(Context.getFunction());
 }
 
 void SampleProfileWriterBinary::addNames(const FunctionSamples &S) {
@@ -653,14 +656,14 @@ void SampleProfileWriterBinary::addNames(const FunctionSamples &S) {
   for (const auto &I : S.getBodySamples()) {
     const SampleRecord &Sample = I.second;
     for (const auto &J : Sample.getCallTargets())
-      addName(J.first());
+      addName(J.first);
   }
 
   // Recursively add all the names for inlined callsites.
   for (const auto &J : S.getCallsiteSamples())
     for (const auto &FS : J.second) {
       const FunctionSamples &CalleeSamples = FS.second;
-      addName(CalleeSamples.getName());
+      addName(CalleeSamples.getFunction());
       addNames(CalleeSamples);
     }
 }
@@ -669,26 +672,26 @@ void SampleProfileWriterExtBinaryBase::addContext(
     const SampleContext &Context) {
   if (Context.hasContext()) {
     for (auto &Callsite : Context.getContextFrames())
-      SampleProfileWriterBinary::addName(Callsite.FuncName);
+      SampleProfileWriterBinary::addName(Callsite.Func);
     CSNameTable.insert(std::make_pair(Context, 0));
   } else {
-    SampleProfileWriterBinary::addName(Context.getName());
+    SampleProfileWriterBinary::addName(Context.getFunction());
   }
 }
 
 void SampleProfileWriterBinary::stablizeNameTable(
-    MapVector<StringRef, uint32_t> &NameTable, std::set<StringRef> &V) {
+    MapVector<FunctionId, uint32_t> &NameTable, std::set<FunctionId> &V) {
   // Sort the names to make NameTable deterministic.
   for (const auto &I : NameTable)
     V.insert(I.first);
   int i = 0;
-  for (const StringRef &N : V)
+  for (const FunctionId &N : V)
     NameTable[N] = i++;
 }
 
 std::error_code SampleProfileWriterBinary::writeNameTable() {
   auto &OS = *OutputStream;
-  std::set<StringRef> V;
+  std::set<FunctionId> V;
   stablizeNameTable(NameTable, V);
 
   // Write out the name table.
@@ -741,7 +744,7 @@ void SampleProfileWriterExtBinaryBase::setToCompressSection(SecType Type) {
 }
 
 void SampleProfileWriterExtBinaryBase::allocSecHdrTable() {
-  support::endian::Writer Writer(*OutputStream, support::little);
+  support::endian::Writer Writer(*OutputStream, llvm::endianness::little);
 
   Writer.write(static_cast<uint64_t>(SectionHdrLayout.size()));
   SecHdrTableOffset = OutputStream->tell();
@@ -771,7 +774,8 @@ std::error_code SampleProfileWriterExtBinaryBase::writeSecHdrTable() {
   // but it needs to be read before SecLBRProfile (the order in
   // SectionHdrLayout). So we use IndexMap above to switch the order.
   support::endian::SeekableWriter Writer(
-      static_cast<raw_pwrite_stream &>(*OutputStream), support::little);
+      static_cast<raw_pwrite_stream &>(*OutputStream),
+      llvm::endianness::little);
   for (uint32_t LayoutIdx = 0; LayoutIdx < SectionHdrLayout.size();
        LayoutIdx++) {
     assert(IndexMap[LayoutIdx] < SecHdrTable.size() &&
@@ -834,7 +838,7 @@ std::error_code SampleProfileWriterBinary::writeBody(const FunctionSamples &S) {
     encodeULEB128(Sample.getSamples(), OS);
     encodeULEB128(Sample.getCallTargets().size(), OS);
     for (const auto &J : Sample.getSortedCallTargets()) {
-      StringRef Callee = J.first;
+      FunctionId Callee = J.first;
       uint64_t CalleeSamples = J.second;
       if (std::error_code EC = writeNameIdx(Callee))
         return EC;
diff --git a/llvm/lib/Remarks/Remark.cpp b/llvm/lib/Remarks/Remark.cpp
index 1b248db417..ef42271a3c 100644
--- a/llvm/lib/Remarks/Remark.cpp
+++ b/llvm/lib/Remarks/Remark.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Remarks/Remark.h"
+#include "llvm/ADT/APInt.h"
 #include "llvm/ADT/ArrayRef.h"
 #include <optional>
 
@@ -25,6 +26,16 @@ std::string Remark::getArgsAsMsg() const {
   return OS.str();
 }
 
+/// Returns the value of a specified key parsed from StringRef.
+std::optional<int> Argument::getValAsInt() const {
+  APInt KeyVal;
+  if (Val.getAsInteger(10, KeyVal))
+    return std::nullopt;
+  return KeyVal.getSExtValue();
+}
+
+bool Argument::isValInt() const { return getValAsInt().has_value(); }
+
 void RemarkLocation::print(raw_ostream &OS) const {
   OS << "{ "
      << "File: " << SourceFilePath << ", Line: " << SourceLine
diff --git a/llvm/lib/Remarks/YAMLRemarkParser.cpp b/llvm/lib/Remarks/YAMLRemarkParser.cpp
index f5123b0f64..947adbba10 100644
--- a/llvm/lib/Remarks/YAMLRemarkParser.cpp
+++ b/llvm/lib/Remarks/YAMLRemarkParser.cpp
@@ -75,8 +75,7 @@ static Expected<uint64_t> parseVersion(StringRef &Buf) {
                              "Expecting version number.");
 
   uint64_t Version =
-      support::endian::read<uint64_t, support::little, support::unaligned>(
-          Buf.data());
+      support::endian::read<uint64_t, llvm::endianness::little>(Buf.data());
   if (Version != remarks::CurrentRemarkVersion)
     return createStringError(std::errc::illegal_byte_sequence,
                              "Mismatching remark version. Got %" PRId64
@@ -91,8 +90,7 @@ static Expected<uint64_t> parseStrTabSize(StringRef &Buf) {
     return createStringError(std::errc::illegal_byte_sequence,
                              "Expecting string table size.");
   uint64_t StrTabSize =
-      support::endian::read<uint64_t, support::little, support::unaligned>(
-          Buf.data());
+      support::endian::read<uint64_t, llvm::endianness::little>(Buf.data());
   Buf = Buf.drop_front(sizeof(uint64_t));
   return StrTabSize;
 }
diff --git a/llvm/lib/Support/Chrono.cpp b/llvm/lib/Support/Chrono.cpp
index 859ece8f55..993d200675 100644
--- a/llvm/lib/Support/Chrono.cpp
+++ b/llvm/lib/Support/Chrono.cpp
@@ -40,6 +40,24 @@ static inline struct tm getStructTM(TimePoint<> TP) {
   return Storage;
 }
 
+static inline struct tm getStructTMUtc(UtcTime<> TP) {
+  struct tm Storage;
+  std::time_t OurTime = toTimeT(TP);
+
+#if defined(LLVM_ON_UNIX)
+  struct tm *LT = ::gmtime_r(&OurTime, &Storage);
+  assert(LT);
+  (void)LT;
+#endif
+#if defined(_WIN32)
+  int Error = ::gmtime_s(&Storage, &OurTime);
+  assert(!Error);
+  (void)Error;
+#endif
+
+  return Storage;
+}
+
 raw_ostream &operator<<(raw_ostream &OS, TimePoint<> TP) {
   struct tm LT = getStructTM(TP);
   char Buffer[sizeof("YYYY-MM-DD HH:MM:SS")];
@@ -50,12 +68,10 @@ raw_ostream &operator<<(raw_ostream &OS, TimePoint<> TP) {
                                .count()));
 }
 
-void format_provider<TimePoint<>>::format(const TimePoint<> &T, raw_ostream &OS,
-                                          StringRef Style) {
+template <class T>
+static void format(const T &Fractional, struct tm &LT, raw_ostream &OS,
+                   StringRef Style) {
   using namespace std::chrono;
-  TimePoint<seconds> Truncated = time_point_cast<seconds>(T);
-  auto Fractional = T - Truncated;
-  struct tm LT = getStructTM(Truncated);
   // Handle extensions first. strftime mangles unknown %x on some platforms.
   if (Style.empty()) Style = "%Y-%m-%d %H:%M:%S.%N";
   std::string Format;
@@ -90,4 +106,23 @@ void format_provider<TimePoint<>>::format(const TimePoint<> &T, raw_ostream &OS,
   OS << (Len ? Buffer : "BAD-DATE-FORMAT");
 }
 
+void format_provider<UtcTime<std::chrono::seconds>>::format(
+    const UtcTime<std::chrono::seconds> &T, raw_ostream &OS, StringRef Style) {
+  using namespace std::chrono;
+  UtcTime<seconds> Truncated =
+      UtcTime<seconds>(duration_cast<seconds>(T.time_since_epoch()));
+  auto Fractional = T - Truncated;
+  struct tm LT = getStructTMUtc(Truncated);
+  llvm::format(Fractional, LT, OS, Style);
+}
+
+void format_provider<TimePoint<>>::format(const TimePoint<> &T, raw_ostream &OS,
+                                          StringRef Style) {
+  using namespace std::chrono;
+  TimePoint<seconds> Truncated = time_point_cast<seconds>(T);
+  auto Fractional = T - Truncated;
+  struct tm LT = getStructTM(Truncated);
+  llvm::format(Fractional, LT, OS, Style);
+}
+
 } // namespace llvm
diff --git a/llvm/lib/Support/CodeGenCoverage.cpp b/llvm/lib/Support/CodeGenCoverage.cpp
index d5ab77b9c6..0df45b4ff2 100644
--- a/llvm/lib/Support/CodeGenCoverage.cpp
+++ b/llvm/lib/Support/CodeGenCoverage.cpp
@@ -58,7 +58,8 @@ bool CodeGenCoverage::parse(MemoryBuffer &Buffer, StringRef BackendName) {
       if (std::distance(CurPtr, Buffer.getBufferEnd()) < 8)
         return false; // Data is invalid. Not enough bytes for another rule id.
 
-      uint64_t RuleID = support::endian::read64(CurPtr, support::native);
+      uint64_t RuleID =
+          support::endian::read64(CurPtr, llvm::endianness::native);
       CurPtr += 8;
 
       // ~0ull terminates the rule id list.
diff --git a/llvm/lib/Support/ELFAttributeParser.cpp b/llvm/lib/Support/ELFAttributeParser.cpp
index 2e90b70dc8..d3100c9ebb 100644
--- a/llvm/lib/Support/ELFAttributeParser.cpp
+++ b/llvm/lib/Support/ELFAttributeParser.cpp
@@ -189,9 +189,9 @@ Error ELFAttributeParser::parseSubsection(uint32_t length) {
 }
 
 Error ELFAttributeParser::parse(ArrayRef<uint8_t> section,
-                                support::endianness endian) {
+                                llvm::endianness endian) {
   unsigned sectionNumber = 0;
-  de = DataExtractor(section, endian == support::little, 0);
+  de = DataExtractor(section, endian == llvm::endianness::little, 0);
 
   // For early returns, we have more specific errors, consume the Error in
   // cursor.
diff --git a/llvm/lib/Support/KnownBits.cpp b/llvm/lib/Support/KnownBits.cpp
index 097c22d33d..770e4051ca 100644
--- a/llvm/lib/Support/KnownBits.cpp
+++ b/llvm/lib/Support/KnownBits.cpp
@@ -85,6 +85,18 @@ KnownBits KnownBits::computeForAddSub(bool Add, bool NSW,
   return KnownOut;
 }
 
+KnownBits KnownBits::computeForSubBorrow(const KnownBits &LHS, KnownBits RHS,
+                                         const KnownBits &Borrow) {
+  assert(Borrow.getBitWidth() == 1 && "Borrow must be 1-bit");
+
+  // LHS - RHS = LHS + ~RHS + 1
+  // Carry 1 - Borrow in ::computeForAddCarry
+  std::swap(RHS.Zero, RHS.One);
+  return ::computeForAddCarry(LHS, RHS,
+                              /*CarryZero=*/Borrow.One.getBoolValue(),
+                              /*CarryOne=*/Borrow.Zero.getBoolValue());
+}
+
 KnownBits KnownBits::sextInReg(unsigned SrcBitWidth) const {
   unsigned BitWidth = getBitWidth();
   assert(0 < SrcBitWidth && SrcBitWidth <= BitWidth &&
diff --git a/llvm/lib/Support/RISCVISAInfo.cpp b/llvm/lib/Support/RISCVISAInfo.cpp
index 72d33e1e65..22208e2e0c 100644
--- a/llvm/lib/Support/RISCVISAInfo.cpp
+++ b/llvm/lib/Support/RISCVISAInfo.cpp
@@ -73,6 +73,8 @@ static const RISCVSupportedExtension SupportedExtensions[] = {
     {"xcvsimd", RISCVExtensionVersion{1, 0}},
     {"xsfcie", RISCVExtensionVersion{1, 0}},
     {"xsfvcp", RISCVExtensionVersion{1, 0}},
+    {"xsfvqmaccdod", RISCVExtensionVersion{1, 0}},
+    {"xsfvqmaccqoq", RISCVExtensionVersion{1, 0}},
     {"xtheadba", RISCVExtensionVersion{1, 0}},
     {"xtheadbb", RISCVExtensionVersion{1, 0}},
     {"xtheadbs", RISCVExtensionVersion{1, 0}},
@@ -991,6 +993,8 @@ static const char *ImpliedExtsF[] = {"zicsr"};
 static const char *ImpliedExtsV[] = {"zvl128b", "zve64d"};
 static const char *ImpliedExtsXTHeadVdot[] = {"v"};
 static const char *ImpliedExtsXsfvcp[] = {"zve32x"};
+static const char *ImpliedExtsXsfvqmaccdod[] = {"zve32x"};
+static const char *ImpliedExtsXsfvqmaccqoq[] = {"zve32x"};
 static const char *ImpliedExtsZacas[] = {"a"};
 static const char *ImpliedExtsZcb[] = {"zca"};
 static const char *ImpliedExtsZcd[] = {"d", "zca"};
@@ -1058,6 +1062,8 @@ static constexpr ImpliedExtsEntry ImpliedExts[] = {
     {{"f"}, {ImpliedExtsF}},
     {{"v"}, {ImpliedExtsV}},
     {{"xsfvcp"}, {ImpliedExtsXsfvcp}},
+    {{"xsfvqmaccdod"}, {ImpliedExtsXsfvqmaccdod}},
+    {{"xsfvqmaccqoq"}, {ImpliedExtsXsfvqmaccqoq}},
     {{"xtheadvdot"}, {ImpliedExtsXTHeadVdot}},
     {{"zacas"}, {ImpliedExtsZacas}},
     {{"zcb"}, {ImpliedExtsZcb}},
diff --git a/llvm/lib/Support/Regex.cpp b/llvm/lib/Support/Regex.cpp
index dfbd373e4a..8fa71a749c 100644
--- a/llvm/lib/Support/Regex.cpp
+++ b/llvm/lib/Support/Regex.cpp
@@ -163,6 +163,25 @@ std::string Regex::sub(StringRef Repl, StringRef String,
 
     // FIXME: We should have a StringExtras function for mapping C99 escapes.
     switch (Repl[0]) {
+
+      // Backreference with the "\g<ref>" syntax
+    case 'g':
+      if (Repl.size() >= 4 && Repl[1] == '<') {
+        size_t End = Repl.find('>');
+        StringRef Ref = Repl.slice(2, End);
+        unsigned RefValue;
+        if (End != StringRef::npos && !Ref.getAsInteger(10, RefValue)) {
+          Repl = Repl.substr(End + 1);
+          if (RefValue < Matches.size())
+            Res += Matches[RefValue];
+          else if (Error && Error->empty())
+            *Error =
+                ("invalid backreference string 'g<" + Twine(Ref) + ">'").str();
+          break;
+        }
+      }
+      [[fallthrough]];
+
       // Treat all unrecognized characters as self-quoting.
     default:
       Res += Repl[0];
diff --git a/llvm/lib/Support/UnicodeNameToCodepoint.cpp b/llvm/lib/Support/UnicodeNameToCodepoint.cpp
index 0f77ff099c..a09b3ffc4c 100644
--- a/llvm/lib/Support/UnicodeNameToCodepoint.cpp
+++ b/llvm/lib/Support/UnicodeNameToCodepoint.cpp
@@ -387,8 +387,7 @@ static std::optional<char32_t> nameToCodepoint(StringRef Name, bool Strict,
     std::reverse(Buffer.begin(), Buffer.end());
     // UAX44-LM2. Ignore case, whitespace, underscore ('_'), and all medial
     // hyphens except the hyphen in U+1180 HANGUL JUNGSEONG O-E.
-    if (!Strict && Value == 0x116c &&
-        Name.find_insensitive("O-E") != StringRef::npos) {
+    if (!Strict && Value == 0x116c && Name.contains_insensitive("O-E")) {
       Buffer = "HANGUL JUNGSEONG O-E";
       Value = 0x1180;
     }
diff --git a/llvm/lib/Support/Unix/Memory.inc b/llvm/lib/Support/Unix/Memory.inc
index 4c8f6b2ea7..69bd116434 100644
--- a/llvm/lib/Support/Unix/Memory.inc
+++ b/llvm/lib/Support/Unix/Memory.inc
@@ -237,7 +237,8 @@ void Memory::InvalidateInstructionCache(const void *Addr, size_t Len) {
   for (intptr_t Line = StartLine; Line < EndLine; Line += LineSize)
     asm volatile("icbi 0, %0" : : "r"(Line));
   asm volatile("isync");
-#elif (defined(__arm__) || defined(__aarch64__) || defined(__mips__)) &&       \
+#elif (defined(__arm__) || defined(__aarch64__) || defined(__loongarch__) ||   \
+       defined(__mips__)) &&                                                   \
     defined(__GNUC__)
   // FIXME: Can we safely always call this for __GNUC__ everywhere?
   const char *Start = static_cast<const char *>(Addr);
diff --git a/llvm/lib/Support/YAMLParser.cpp b/llvm/lib/Support/YAMLParser.cpp
index 6ac2c6aeeb..1422e40f91 100644
--- a/llvm/lib/Support/YAMLParser.cpp
+++ b/llvm/lib/Support/YAMLParser.cpp
@@ -392,6 +392,10 @@ class Scanner {
   ///        Pos is whitespace or a new line
   bool isBlankOrBreak(StringRef::iterator Position);
 
+  /// Return true if the minimal well-formed code unit subsequence at
+  ///        Pos is considered a "safe" character for plain scalars.
+  bool isPlainSafeNonBlank(StringRef::iterator Position);
+
   /// Return true if the line is a line break, false otherwise.
   bool isLineEmpty(StringRef Line);
 
@@ -545,6 +549,10 @@ class Scanner {
   /// Can the next token be the start of a simple key?
   bool IsSimpleKeyAllowed;
 
+  /// Can the next token be a value indicator even if it does not have a
+  /// trailing space?
+  bool IsAdjacentValueAllowedInFlow;
+
   /// True if an error has occurred.
   bool Failed;
 
@@ -868,6 +876,7 @@ void Scanner::init(MemoryBufferRef Buffer) {
   FlowLevel = 0;
   IsStartOfStream = true;
   IsSimpleKeyAllowed = true;
+  IsAdjacentValueAllowedInFlow = false;
   Failed = false;
   std::unique_ptr<MemoryBuffer> InputBufferOwner =
       MemoryBuffer::getMemBuffer(Buffer, /*RequiresNullTerminator=*/false);
@@ -1049,6 +1058,15 @@ bool Scanner::isBlankOrBreak(StringRef::iterator Position) {
          *Position == '\n';
 }
 
+bool Scanner::isPlainSafeNonBlank(StringRef::iterator Position) {
+  if (Position == End || isBlankOrBreak(Position))
+    return false;
+  if (FlowLevel &&
+      StringRef(Position, 1).find_first_of(",[]{}") != StringRef::npos)
+    return false;
+  return true;
+}
+
 bool Scanner::isLineEmpty(StringRef Line) {
   for (const auto *Position = Line.begin(); Position != Line.end(); ++Position)
     if (!isBlankOrBreak(Position))
@@ -1189,6 +1207,7 @@ bool Scanner::scanStreamEnd() {
   unrollIndent(-1);
   SimpleKeys.clear();
   IsSimpleKeyAllowed = false;
+  IsAdjacentValueAllowedInFlow = false;
 
   Token T;
   T.Kind = Token::TK_StreamEnd;
@@ -1202,6 +1221,7 @@ bool Scanner::scanDirective() {
   unrollIndent(-1);
   SimpleKeys.clear();
   IsSimpleKeyAllowed = false;
+  IsAdjacentValueAllowedInFlow = false;
 
   StringRef::iterator Start = Current;
   consume('%');
@@ -1233,6 +1253,7 @@ bool Scanner::scanDocumentIndicator(bool IsStart) {
   unrollIndent(-1);
   SimpleKeys.clear();
   IsSimpleKeyAllowed = false;
+  IsAdjacentValueAllowedInFlow = false;
 
   Token T;
   T.Kind = IsStart ? Token::TK_DocumentStart : Token::TK_DocumentEnd;
@@ -1255,6 +1276,8 @@ bool Scanner::scanFlowCollectionStart(bool IsSequence) {
 
   // And may also be followed by a simple key.
   IsSimpleKeyAllowed = true;
+  // Adjacent values are allowed in flows only after JSON-style keys.
+  IsAdjacentValueAllowedInFlow = false;
   ++FlowLevel;
   return true;
 }
@@ -1262,6 +1285,7 @@ bool Scanner::scanFlowCollectionStart(bool IsSequence) {
 bool Scanner::scanFlowCollectionEnd(bool IsSequence) {
   removeSimpleKeyCandidatesOnFlowLevel(FlowLevel);
   IsSimpleKeyAllowed = false;
+  IsAdjacentValueAllowedInFlow = true;
   Token T;
   T.Kind = IsSequence ? Token::TK_FlowSequenceEnd
                       : Token::TK_FlowMappingEnd;
@@ -1276,6 +1300,7 @@ bool Scanner::scanFlowCollectionEnd(bool IsSequence) {
 bool Scanner::scanFlowEntry() {
   removeSimpleKeyCandidatesOnFlowLevel(FlowLevel);
   IsSimpleKeyAllowed = true;
+  IsAdjacentValueAllowedInFlow = false;
   Token T;
   T.Kind = Token::TK_FlowEntry;
   T.Range = StringRef(Current, 1);
@@ -1288,6 +1313,7 @@ bool Scanner::scanBlockEntry() {
   rollIndent(Column, Token::TK_BlockSequenceStart, TokenQueue.end());
   removeSimpleKeyCandidatesOnFlowLevel(FlowLevel);
   IsSimpleKeyAllowed = true;
+  IsAdjacentValueAllowedInFlow = false;
   Token T;
   T.Kind = Token::TK_BlockEntry;
   T.Range = StringRef(Current, 1);
@@ -1302,6 +1328,7 @@ bool Scanner::scanKey() {
 
   removeSimpleKeyCandidatesOnFlowLevel(FlowLevel);
   IsSimpleKeyAllowed = !FlowLevel;
+  IsAdjacentValueAllowedInFlow = false;
 
   Token T;
   T.Kind = Token::TK_Key;
@@ -1339,6 +1366,7 @@ bool Scanner::scanValue() {
       rollIndent(Column, Token::TK_BlockMappingStart, TokenQueue.end());
     IsSimpleKeyAllowed = !FlowLevel;
   }
+  IsAdjacentValueAllowedInFlow = false;
 
   Token T;
   T.Kind = Token::TK_Value;
@@ -1420,6 +1448,7 @@ bool Scanner::scanFlowScalar(bool IsDoubleQuoted) {
   saveSimpleKeyCandidate(--TokenQueue.end(), ColStart, false);
 
   IsSimpleKeyAllowed = false;
+  IsAdjacentValueAllowedInFlow = true;
 
   return true;
 }
@@ -1434,21 +1463,9 @@ bool Scanner::scanPlainScalar() {
     if (*Current == '#')
       break;
 
-    while (Current != End && !isBlankOrBreak(Current)) {
-      if (FlowLevel && *Current == ':' &&
-          (Current + 1 == End ||
-           !(isBlankOrBreak(Current + 1) || *(Current + 1) == ','))) {
-        setError("Found unexpected ':' while scanning a plain scalar", Current);
-        return false;
-      }
-
-      // Check for the end of the plain scalar.
-      if (  (*Current == ':' && isBlankOrBreak(Current + 1))
-          || (  FlowLevel
-          && (StringRef(Current, 1).find_first_of(",:?[]{}")
-              != StringRef::npos)))
-        break;
-
+    while (Current != End &&
+           ((*Current != ':' && isPlainSafeNonBlank(Current)) ||
+            (*Current == ':' && isPlainSafeNonBlank(Current + 1)))) {
       StringRef::iterator i = skip_nb_char(Current);
       if (i == Current)
         break;
@@ -1499,6 +1516,7 @@ bool Scanner::scanPlainScalar() {
   saveSimpleKeyCandidate(--TokenQueue.end(), ColStart, false);
 
   IsSimpleKeyAllowed = false;
+  IsAdjacentValueAllowedInFlow = false;
 
   return true;
 }
@@ -1534,6 +1552,7 @@ bool Scanner::scanAliasOrAnchor(bool IsAlias) {
   saveSimpleKeyCandidate(--TokenQueue.end(), ColStart, false);
 
   IsSimpleKeyAllowed = false;
+  IsAdjacentValueAllowedInFlow = false;
 
   return true;
 }
@@ -1766,6 +1785,7 @@ bool Scanner::scanBlockScalar(bool IsLiteral) {
   // New lines may start a simple key.
   if (!FlowLevel)
     IsSimpleKeyAllowed = true;
+  IsAdjacentValueAllowedInFlow = false;
 
   Token T;
   T.Kind = Token::TK_BlockScalar;
@@ -1799,6 +1819,7 @@ bool Scanner::scanTag() {
   saveSimpleKeyCandidate(--TokenQueue.end(), ColStart, false);
 
   IsSimpleKeyAllowed = false;
+  IsAdjacentValueAllowedInFlow = false;
 
   return true;
 }
@@ -1848,13 +1869,14 @@ bool Scanner::fetchMoreTokens() {
   if (*Current == ',')
     return scanFlowEntry();
 
-  if (*Current == '-' && isBlankOrBreak(Current + 1))
+  if (*Current == '-' && (isBlankOrBreak(Current + 1) || Current + 1 == End))
     return scanBlockEntry();
 
-  if (*Current == '?' && (FlowLevel || isBlankOrBreak(Current + 1)))
+  if (*Current == '?' && (Current + 1 == End || isBlankOrBreak(Current + 1)))
     return scanKey();
 
-  if (*Current == ':' && (FlowLevel || isBlankOrBreak(Current + 1)))
+  if (*Current == ':' &&
+      (!isPlainSafeNonBlank(Current + 1) || IsAdjacentValueAllowedInFlow))
     return scanValue();
 
   if (*Current == '*')
@@ -1880,15 +1902,10 @@ bool Scanner::fetchMoreTokens() {
 
   // Get a plain scalar.
   StringRef FirstChar(Current, 1);
-  if (!(isBlankOrBreak(Current)
-        || FirstChar.find_first_of("-?:,[]{}#&*!|>'\"%@`") != StringRef::npos)
-      || (*Current == '-' && !isBlankOrBreak(Current + 1))
-      || (!FlowLevel && (*Current == '?' || *Current == ':')
-          && isBlankOrBreak(Current + 1))
-      || (!FlowLevel && *Current == ':'
-                      && Current + 2 < End
-                      && *(Current + 1) == ':'
-                      && !isBlankOrBreak(Current + 2)))
+  if ((!isBlankOrBreak(Current) &&
+       FirstChar.find_first_of("-?:,[]{}#&*!|>'\"%@`") == StringRef::npos) ||
+      (FirstChar.find_first_of("?:-") != StringRef::npos &&
+       isPlainSafeNonBlank(Current + 1)))
     return scanPlainScalar();
 
   setError("Unrecognized character while tokenizing.", Current);
diff --git a/llvm/lib/Support/YAMLTraits.cpp b/llvm/lib/Support/YAMLTraits.cpp
index 9325a09faa..4aaf59be2c 100644
--- a/llvm/lib/Support/YAMLTraits.cpp
+++ b/llvm/lib/Support/YAMLTraits.cpp
@@ -156,6 +156,8 @@ bool Input::preflightKey(const char *Key, bool Required, bool, bool &UseDefault,
   if (!CurrentNode) {
     if (Required)
       EC = make_error_code(errc::invalid_argument);
+    else
+      UseDefault = true;
     return false;
   }
 
diff --git a/llvm/lib/TableGen/Error.cpp b/llvm/lib/TableGen/Error.cpp
index ebe9129eba..dabb265ef8 100644
--- a/llvm/lib/TableGen/Error.cpp
+++ b/llvm/lib/TableGen/Error.cpp
@@ -170,4 +170,11 @@ void CheckAssert(SMLoc Loc, Init *Condition, Init *Message) {
   }
 }
 
+// Dump a message to stderr.
+void dumpMessage(SMLoc Loc, Init *Message) {
+  auto *MessageInit = dyn_cast<StringInit>(Message);
+  assert(MessageInit && "no debug message to print");
+  PrintNote(Loc, MessageInit->getValue());
+}
+
 } // end namespace llvm
diff --git a/llvm/lib/TableGen/Record.cpp b/llvm/lib/TableGen/Record.cpp
index 8709b176f8..675969003a 100644
--- a/llvm/lib/TableGen/Record.cpp
+++ b/llvm/lib/TableGen/Record.cpp
@@ -797,6 +797,25 @@ void UnOpInit::Profile(FoldingSetNodeID &ID) const {
 Init *UnOpInit::Fold(Record *CurRec, bool IsFinal) const {
   RecordKeeper &RK = getRecordKeeper();
   switch (getOpcode()) {
+  case REPR:
+    if (LHS->isConcrete()) {
+      // If it is a Record, print the full content.
+      if (const auto *Def = dyn_cast<DefInit>(LHS)) {
+        std::string S;
+        raw_string_ostream OS(S);
+        OS << *Def->getDef();
+        OS.flush();
+        return StringInit::get(RK, S);
+      } else {
+        // Otherwise, print the value of the variable.
+        //
+        // NOTE: we could recursively !repr the elements of a list,
+        // but that could produce a lot of output when printing a
+        // defset.
+        return StringInit::get(RK, LHS->getAsString());
+      }
+    }
+    break;
   case TOLOWER:
     if (StringInit *LHSs = dyn_cast<StringInit>(LHS))
       return StringInit::get(RK, LHSs->getValue().lower());
@@ -957,6 +976,9 @@ std::string UnOpInit::getAsString() const {
   case EMPTY: Result = "!empty"; break;
   case GETDAGOP: Result = "!getdagop"; break;
   case LOG2 : Result = "!logtwo"; break;
+  case REPR:
+    Result = "!repr";
+    break;
   case TOLOWER:
     Result = "!tolower";
     break;
@@ -2253,6 +2275,9 @@ DefInit *VarDefInit::instantiate() {
     // Copy assertions from class to instance.
     NewRec->appendAssertions(Class);
 
+    // Copy dumps from class to instance.
+    NewRec->appendDumps(Class);
+
     // Substitute and resolve template arguments
     ArrayRef<Init *> TArgs = Class->getTemplateArgs();
     MapResolver R(NewRec);
@@ -2287,6 +2312,9 @@ DefInit *VarDefInit::instantiate() {
     // Check the assertions.
     NewRec->checkRecordAssertions();
 
+    // Check the assertions.
+    NewRec->emitRecordDumps();
+
     Def = DefInit::get(NewRec);
   }
 
@@ -2844,6 +2872,11 @@ void Record::resolveReferences(Resolver &R, const RecordVal *SkipVal) {
     Value = Assertion.Message->resolveReferences(R);
     Assertion.Message = Value;
   }
+  // Resolve the dump expressions.
+  for (auto &Dump : Dumps) {
+    Init *Value = Dump.Message->resolveReferences(R);
+    Dump.Message = Value;
+  }
 }
 
 void Record::resolveReferences(Init *NewName) {
@@ -3100,6 +3133,16 @@ void Record::checkRecordAssertions() {
   }
 }
 
+void Record::emitRecordDumps() {
+  RecordResolver R(*this);
+  R.setFinal(true);
+
+  for (const auto &Dump : getDumps()) {
+    Init *Message = Dump.Message->resolveReferences(R);
+    dumpMessage(Dump.Loc, Message);
+  }
+}
+
 // Report a warning if the record has unused template arguments.
 void Record::checkUnusedTemplateArgs() {
   for (const Init *TA : getTemplateArgs()) {
diff --git a/llvm/lib/TableGen/TGLexer.cpp b/llvm/lib/TableGen/TGLexer.cpp
index 1730f5d88a..c811a67d93 100644
--- a/llvm/lib/TableGen/TGLexer.cpp
+++ b/llvm/lib/TableGen/TGLexer.cpp
@@ -346,31 +346,32 @@ tgtok::TokKind TGLexer::LexIdentifier() {
   StringRef Str(IdentStart, CurPtr-IdentStart);
 
   tgtok::TokKind Kind = StringSwitch<tgtok::TokKind>(Str)
-    .Case("int", tgtok::Int)
-    .Case("bit", tgtok::Bit)
-    .Case("bits", tgtok::Bits)
-    .Case("string", tgtok::String)
-    .Case("list", tgtok::List)
-    .Case("code", tgtok::Code)
-    .Case("dag", tgtok::Dag)
-    .Case("class", tgtok::Class)
-    .Case("def", tgtok::Def)
-    .Case("true", tgtok::TrueVal)
-    .Case("false", tgtok::FalseVal)
-    .Case("foreach", tgtok::Foreach)
-    .Case("defm", tgtok::Defm)
-    .Case("defset", tgtok::Defset)
-    .Case("multiclass", tgtok::MultiClass)
-    .Case("field", tgtok::Field)
-    .Case("let", tgtok::Let)
-    .Case("in", tgtok::In)
-    .Case("defvar", tgtok::Defvar)
-    .Case("include", tgtok::Include)
-    .Case("if", tgtok::If)
-    .Case("then", tgtok::Then)
-    .Case("else", tgtok::ElseKW)
-    .Case("assert", tgtok::Assert)
-    .Default(tgtok::Id);
+                            .Case("int", tgtok::Int)
+                            .Case("bit", tgtok::Bit)
+                            .Case("bits", tgtok::Bits)
+                            .Case("string", tgtok::String)
+                            .Case("list", tgtok::List)
+                            .Case("code", tgtok::Code)
+                            .Case("dag", tgtok::Dag)
+                            .Case("class", tgtok::Class)
+                            .Case("def", tgtok::Def)
+                            .Case("true", tgtok::TrueVal)
+                            .Case("false", tgtok::FalseVal)
+                            .Case("foreach", tgtok::Foreach)
+                            .Case("defm", tgtok::Defm)
+                            .Case("defset", tgtok::Defset)
+                            .Case("multiclass", tgtok::MultiClass)
+                            .Case("field", tgtok::Field)
+                            .Case("let", tgtok::Let)
+                            .Case("in", tgtok::In)
+                            .Case("defvar", tgtok::Defvar)
+                            .Case("include", tgtok::Include)
+                            .Case("if", tgtok::If)
+                            .Case("then", tgtok::Then)
+                            .Case("else", tgtok::ElseKW)
+                            .Case("assert", tgtok::Assert)
+                            .Case("dump", tgtok::Dump)
+                            .Default(tgtok::Id);
 
   // A couple of tokens require special processing.
   switch (Kind) {
@@ -605,6 +606,7 @@ tgtok::TokKind TGLexer::LexExclaim() {
           .Case("exists", tgtok::XExists)
           .Case("tolower", tgtok::XToLower)
           .Case("toupper", tgtok::XToUpper)
+          .Case("repr", tgtok::XRepr)
           .Default(tgtok::Error);
 
   return Kind != tgtok::Error ? Kind : ReturnError(Start-1, "Unknown operator");
diff --git a/llvm/lib/TableGen/TGLexer.h b/llvm/lib/TableGen/TGLexer.h
index 7c5594bb4d..2e2aa59f34 100644
--- a/llvm/lib/TableGen/TGLexer.h
+++ b/llvm/lib/TableGen/TGLexer.h
@@ -98,6 +98,7 @@ enum TokKind {
   Defm,
   Defset,
   Defvar,
+  Dump,
   Foreach,
   If,
   Let,
@@ -155,7 +156,8 @@ enum TokKind {
   XGetDagName,
   XSetDagArg,
   XSetDagName,
-  BANG_OPERATOR_LAST = XSetDagName,
+  XRepr,
+  BANG_OPERATOR_LAST = XRepr,
 
   // String valued tokens.
   STRING_VALUE_FIRST,
diff --git a/llvm/lib/TableGen/TGParser.cpp b/llvm/lib/TableGen/TGParser.cpp
index 6990a469c6..16854c7d1f 100644
--- a/llvm/lib/TableGen/TGParser.cpp
+++ b/llvm/lib/TableGen/TGParser.cpp
@@ -111,11 +111,10 @@ static void checkConcrete(Record &R) {
 
 /// Return an Init with a qualifier prefix referring
 /// to CurRec's name.
-static Init *QualifyName(Record &CurRec, MultiClass *CurMultiClass, Init *Name,
-                         StringRef Scoper) {
+static Init *QualifyName(Record &CurRec, Init *Name, bool IsMC = false) {
   RecordKeeper &RK = CurRec.getRecords();
-  Init *NewName = BinOpInit::getStrConcat(CurRec.getNameInit(),
-                                          StringInit::get(RK, Scoper));
+  Init *NewName = BinOpInit::getStrConcat(
+      CurRec.getNameInit(), StringInit::get(RK, IsMC ? "::" : ":"));
   NewName = BinOpInit::getStrConcat(NewName, Name);
 
   if (BinOpInit *BinOp = dyn_cast<BinOpInit>(NewName))
@@ -123,18 +122,20 @@ static Init *QualifyName(Record &CurRec, MultiClass *CurMultiClass, Init *Name,
   return NewName;
 }
 
+static Init *QualifyName(MultiClass *MC, Init *Name) {
+  return QualifyName(MC->Rec, Name, /*IsMC=*/true);
+}
+
 /// Return the qualified version of the implicit 'NAME' template argument.
-static Init *QualifiedNameOfImplicitName(Record &Rec,
-                                         MultiClass *MC = nullptr) {
-  return QualifyName(Rec, MC, StringInit::get(Rec.getRecords(), "NAME"),
-                     MC ? "::" : ":");
+static Init *QualifiedNameOfImplicitName(Record &Rec, bool IsMC = false) {
+  return QualifyName(Rec, StringInit::get(Rec.getRecords(), "NAME"), IsMC);
 }
 
 static Init *QualifiedNameOfImplicitName(MultiClass *MC) {
-  return QualifiedNameOfImplicitName(MC->Rec, MC);
+  return QualifiedNameOfImplicitName(MC->Rec, /*IsMC=*/true);
 }
 
-Init *TGVarScope::getVar(RecordKeeper &Records, MultiClass* ParsingMultiClass,
+Init *TGVarScope::getVar(RecordKeeper &Records, MultiClass *ParsingMultiClass,
                          StringInit *Name, SMRange NameLoc,
                          bool TrackReferenceLocs) const {
   // First, we search in local variables.
@@ -142,11 +143,11 @@ Init *TGVarScope::getVar(RecordKeeper &Records, MultiClass* ParsingMultiClass,
   if (It != Vars.end())
     return It->second;
 
-  std::function<Init *(Record *, StringInit *, StringRef)> FindValueInArgs =
-      [&](Record *Rec, StringInit *Name, StringRef Scoper) -> Init * {
+  auto FindValueInArgs = [&](Record *Rec, StringInit *Name,
+                             bool IsMC) -> Init * {
     if (!Rec)
       return nullptr;
-    Init *ArgName = QualifyName(*Rec, ParsingMultiClass, Name, Scoper);
+    Init *ArgName = QualifyName(*Rec, Name, IsMC);
     if (Rec->isTemplateArg(ArgName)) {
       RecordVal *RV = Rec->getValue(ArgName);
       assert(RV && "Template arg doesn't exist??");
@@ -176,7 +177,7 @@ Init *TGVarScope::getVar(RecordKeeper &Records, MultiClass* ParsingMultiClass,
 
       // The variable is a class template argument?
       if (CurRec->isClass())
-        if (auto *V = FindValueInArgs(CurRec, Name, ":"))
+        if (auto *V = FindValueInArgs(CurRec, Name, /*IsMC=*/false))
           return V;
     }
     break;
@@ -193,7 +194,7 @@ Init *TGVarScope::getVar(RecordKeeper &Records, MultiClass* ParsingMultiClass,
   case SK_MultiClass: {
     // The variable is a multiclass template argument?
     if (CurMultiClass)
-      if (auto *V = FindValueInArgs(&CurMultiClass->Rec, Name, "::"))
+      if (auto *V = FindValueInArgs(&CurMultiClass->Rec, Name, /*IsMC=*/true))
         return V;
     break;
   }
@@ -313,6 +314,9 @@ bool TGParser::AddSubClass(Record *CurRec, SubClassReference &SubClass) {
   // Copy the subclass record's assertions to the new record.
   CurRec->appendAssertions(SC);
 
+  // Copy the subclass record's dumps to the new record.
+  CurRec->appendDumps(SC);
+
   Init *Name;
   if (CurRec->isClass())
     Name = VarInit::get(QualifiedNameOfImplicitName(*CurRec),
@@ -376,7 +380,7 @@ bool TGParser::AddSubMultiClass(MultiClass *CurMC,
 
 /// Add a record, foreach loop, or assertion to the current context.
 bool TGParser::addEntry(RecordsEntry E) {
-  assert((!!E.Rec + !!E.Loop + !!E.Assertion) == 1 &&
+  assert((!!E.Rec + !!E.Loop + !!E.Assertion + !!E.Dump) == 1 &&
          "RecordsEntry has invalid number of items");
 
   // If we are parsing a loop, add it to the loop's entries.
@@ -404,6 +408,11 @@ bool TGParser::addEntry(RecordsEntry E) {
     return false;
   }
 
+  if (E.Dump) {
+    dumpMessage(E.Dump->Loc, E.Dump->Message);
+    return false;
+  }
+
   // It must be a record, so finish it off.
   return addDefOne(std::move(E.Rec));
 }
@@ -498,6 +507,18 @@ bool TGParser::resolve(const std::vector<RecordsEntry> &Source,
       else
         CheckAssert(E.Assertion->Loc, Condition, Message);
 
+    } else if (E.Dump) {
+      MapResolver R;
+      for (const auto &S : Substs)
+        R.set(S.first, S.second);
+      Init *Message = E.Dump->Message->resolveReferences(R);
+
+      if (Dest)
+        Dest->push_back(
+            std::make_unique<Record::DumpInfo>(E.Dump->Loc, Message));
+      else
+        dumpMessage(E.Dump->Loc, Message);
+
     } else {
       auto Rec = std::make_unique<Record>(*E.Rec);
       if (Loc)
@@ -545,6 +566,9 @@ bool TGParser::addDefOne(std::unique_ptr<Record> Rec) {
   // Check the assertions.
   Rec->checkRecordAssertions();
 
+  // Run the dumps.
+  Rec->emitRecordDumps();
+
   // If ObjectBody has template arguments, it's an error.
   assert(Rec->getTemplateArgs().empty() && "How'd this get template args?");
 
@@ -1168,6 +1192,7 @@ Init *TGParser::ParseOperation(Record *CurRec, RecTy *ItemType) {
   case tgtok::XSize:
   case tgtok::XEmpty:
   case tgtok::XCast:
+  case tgtok::XRepr:
   case tgtok::XGetDagOp: { // Value ::= !unop '(' Value ')'
     UnOpInit::UnaryOp Code;
     RecTy *Type = nullptr;
@@ -1185,6 +1210,11 @@ Init *TGParser::ParseOperation(Record *CurRec, RecTy *ItemType) {
         return nullptr;
       }
 
+      break;
+    case tgtok::XRepr:
+      Lex.Lex(); // eat the operation
+      Code = UnOpInit::REPR;
+      Type = StringRecTy::get(Records);
       break;
     case tgtok::XToLower:
       Lex.Lex(); // eat the operation
@@ -3162,8 +3192,7 @@ bool TGParser::ParseTemplateArgValueList(
                      "The name of named argument should be a valid identifier");
 
       auto *Name = cast<StringInit>(Value);
-      Init *QualifiedName =
-          QualifyName(*ArgsRec, CurMultiClass, Name, IsDefm ? "::" : ":");
+      Init *QualifiedName = QualifyName(*ArgsRec, Name, /*IsMC=*/IsDefm);
       auto *NamedArg = ArgsRec->getValue(QualifiedName);
       if (!NamedArg)
         return Error(ValueLoc,
@@ -3242,17 +3271,17 @@ Init *TGParser::ParseDeclaration(Record *CurRec,
                         RecordVal(DeclName, IdLoc, Type,
                                   HasField ? RecordVal::FK_NonconcreteOK
                                            : RecordVal::FK_Normal));
-
   } else if (CurRec) { // class template argument
-    DeclName = QualifyName(*CurRec, CurMultiClass, DeclName, ":");
-    BadField = AddValue(CurRec, IdLoc, RecordVal(DeclName, IdLoc, Type,
-                                                 RecordVal::FK_TemplateArg));
-
+    DeclName = QualifyName(*CurRec, DeclName);
+    BadField =
+        AddValue(CurRec, IdLoc,
+                 RecordVal(DeclName, IdLoc, Type, RecordVal::FK_TemplateArg));
   } else { // multiclass template argument
     assert(CurMultiClass && "invalid context for template argument");
-    DeclName = QualifyName(CurMultiClass->Rec, CurMultiClass, DeclName, "::");
-    BadField = AddValue(CurRec, IdLoc, RecordVal(DeclName, IdLoc, Type,
-                                                 RecordVal::FK_TemplateArg));
+    DeclName = QualifyName(CurMultiClass, DeclName);
+    BadField =
+        AddValue(CurRec, IdLoc,
+                 RecordVal(DeclName, IdLoc, Type, RecordVal::FK_TemplateArg));
   }
   if (BadField)
     return nullptr;
@@ -3399,6 +3428,7 @@ bool TGParser::ParseTemplateArgList(Record *CurRec) {
 ///   BodyItem ::= Declaration ';'
 ///   BodyItem ::= LET ID OptionalBitList '=' Value ';'
 ///   BodyItem ::= Defvar
+///   BodyItem ::= Dump
 ///   BodyItem ::= Assert
 ///
 bool TGParser::ParseBodyItem(Record *CurRec) {
@@ -3408,6 +3438,9 @@ bool TGParser::ParseBodyItem(Record *CurRec) {
   if (Lex.getCode() == tgtok::Defvar)
     return ParseDefvar(CurRec);
 
+  if (Lex.getCode() == tgtok::Dump)
+    return ParseDump(nullptr, CurRec);
+
   if (Lex.getCode() != tgtok::Let) {
     if (!ParseDeclaration(CurRec, false))
       return true;
@@ -3504,6 +3537,10 @@ bool TGParser::ApplyLetStack(RecordsEntry &Entry) {
   if (Entry.Assertion)
     return false;
 
+  // Let bindings are not applied to dumps.
+  if (Entry.Dump)
+    return false;
+
   for (auto &E : Entry.Loop->Entries) {
     if (ApplyLetStack(E))
       return true;
@@ -4084,13 +4121,14 @@ bool TGParser::ParseMultiClass() {
     while (Lex.getCode() != tgtok::r_brace) {
       switch (Lex.getCode()) {
       default:
-        return TokError("expected 'assert', 'def', 'defm', 'defvar', "
+        return TokError("expected 'assert', 'def', 'defm', 'defvar', 'dump', "
                         "'foreach', 'if', or 'let' in multiclass body");
 
       case tgtok::Assert:
       case tgtok::Def:
       case tgtok::Defm:
       case tgtok::Defvar:
+      case tgtok::Dump:
       case tgtok::Foreach:
       case tgtok::If:
       case tgtok::Let:
@@ -4234,15 +4272,18 @@ bool TGParser::ParseDefm(MultiClass *CurMultiClass) {
 ///   Object ::= Defset
 ///   Object ::= Defvar
 ///   Object ::= Assert
+///   Object ::= Dump
 bool TGParser::ParseObject(MultiClass *MC) {
   switch (Lex.getCode()) {
   default:
     return TokError(
-               "Expected assert, class, def, defm, defset, foreach, if, or let");
+        "Expected assert, class, def, defm, defset, dump, foreach, if, or let");
   case tgtok::Assert:  return ParseAssert(MC);
   case tgtok::Def:     return ParseDef(MC);
   case tgtok::Defm:    return ParseDefm(MC);
   case tgtok::Defvar:  return ParseDefvar();
+  case tgtok::Dump:
+    return ParseDump(MC);
   case tgtok::Foreach: return ParseForeach(MC);
   case tgtok::If:      return ParseIf(MC);
   case tgtok::Let:     return ParseTopLevelLet(MC);
@@ -4353,3 +4394,30 @@ LLVM_DUMP_METHOD void MultiClass::dump() const {
     E.dump();
 }
 #endif
+
+bool TGParser::ParseDump(MultiClass *CurMultiClass, Record *CurRec) {
+  // Location of the `dump` statement.
+  SMLoc Loc = Lex.getLoc();
+  assert(Lex.getCode() == tgtok::Dump && "Unknown tok");
+  Lex.Lex(); // eat the operation
+
+  Init *Message = ParseValue(CurRec);
+  if (!Message)
+    return true;
+
+  // Allow to use dump directly on `defvar` and `def`, by wrapping
+  // them with a `!repl`.
+  if (isa<DefInit>(Message))
+    Message = UnOpInit::get(UnOpInit::REPR, Message, StringRecTy::get(Records))
+                  ->Fold(CurRec);
+
+  if (!consume(tgtok::semi))
+    return TokError("expected ';'");
+
+  if (CurRec)
+    CurRec->addDump(Loc, Message);
+  else
+    addEntry(std::make_unique<Record::DumpInfo>(Loc, Message));
+
+  return false;
+}
diff --git a/llvm/lib/TableGen/TGParser.h b/llvm/lib/TableGen/TGParser.h
index d42cdad88a..c5365ff270 100644
--- a/llvm/lib/TableGen/TGParser.h
+++ b/llvm/lib/TableGen/TGParser.h
@@ -41,6 +41,7 @@ struct RecordsEntry {
   std::unique_ptr<Record> Rec;
   std::unique_ptr<ForeachLoop> Loop;
   std::unique_ptr<Record::AssertionInfo> Assertion;
+  std::unique_ptr<Record::DumpInfo> Dump;
 
   void dump() const;
 
@@ -49,6 +50,8 @@ struct RecordsEntry {
   RecordsEntry(std::unique_ptr<ForeachLoop> Loop) : Loop(std::move(Loop)) {}
   RecordsEntry(std::unique_ptr<Record::AssertionInfo> Assertion)
       : Assertion(std::move(Assertion)) {}
+  RecordsEntry(std::unique_ptr<Record::DumpInfo> Dump)
+      : Dump(std::move(Dump)) {}
 };
 
 /// ForeachLoop - Record the iteration state associated with a for loop.
@@ -262,6 +265,7 @@ class TGParser {
   bool ParseDef(MultiClass *CurMultiClass);
   bool ParseDefset();
   bool ParseDefvar(Record *CurRec = nullptr);
+  bool ParseDump(MultiClass *CurMultiClass, Record *CurRec = nullptr);
   bool ParseForeach(MultiClass *CurMultiClass);
   bool ParseIf(MultiClass *CurMultiClass);
   bool ParseIfBody(MultiClass *CurMultiClass, StringRef Kind);
diff --git a/llvm/lib/Target/AArch64/AArch64.td b/llvm/lib/Target/AArch64/AArch64.td
index 70973c9230..ced1d43892 100644
--- a/llvm/lib/Target/AArch64/AArch64.td
+++ b/llvm/lib/Target/AArch64/AArch64.td
@@ -127,6 +127,9 @@ def FeatureCCPP : SubtargetFeature<"ccpp", "HasCCPP",
 def FeatureSVE : SubtargetFeature<"sve", "HasSVE", "true",
   "Enable Scalable Vector Extension (SVE) instructions (FEAT_SVE)", [FeatureFullFP16]>;
 
+def FeatureFPMR : SubtargetFeature<"fpmr", "HasFPMR", "true",
+  "Enable FPMR Register (FEAT_FPMR)">;
+
 // This flag is currently still labeled as Experimental, but when fully
 // implemented this should tell the compiler to use the zeroing pseudos to
 // benefit from the reverse instructions (e.g. SUB vs SUBR) if the inactive
diff --git a/llvm/lib/Target/AArch64/AArch64ConditionalCompares.cpp b/llvm/lib/Target/AArch64/AArch64ConditionalCompares.cpp
index 4c8c03a4c6..ad0ff83356 100644
--- a/llvm/lib/Target/AArch64/AArch64ConditionalCompares.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ConditionalCompares.cpp
@@ -935,7 +935,7 @@ bool AArch64ConditionalCompares::runOnMachineFunction(MachineFunction &MF) {
   SchedModel = MF.getSubtarget().getSchedModel();
   MRI = &MF.getRegInfo();
   DomTree = &getAnalysis<MachineDominatorTree>();
-  Loops = getAnalysisIfAvailable<MachineLoopInfo>();
+  Loops = &getAnalysis<MachineLoopInfo>();
   MBPI = &getAnalysis<MachineBranchProbabilityInfo>();
   Traces = &getAnalysis<MachineTraceMetrics>();
   MinInstr = nullptr;
diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index 78f85faaf6..880de7d030 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -269,14 +269,10 @@ STATISTIC(NumRedZoneFunctions, "Number of functions using red zone");
 static int64_t getArgumentStackToRestore(MachineFunction &MF,
                                          MachineBasicBlock &MBB) {
   MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
-  bool IsTailCallReturn = false;
-  if (MBB.end() != MBBI) {
-    unsigned RetOpcode = MBBI->getOpcode();
-    IsTailCallReturn = RetOpcode == AArch64::TCRETURNdi ||
-                       RetOpcode == AArch64::TCRETURNri ||
-                       RetOpcode == AArch64::TCRETURNriBTI;
-  }
   AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
+  bool IsTailCallReturn = (MBB.end() != MBBI)
+                              ? AArch64InstrInfo::isTailCallReturnInst(*MBBI)
+                              : false;
 
   int64_t ArgumentPopSize = 0;
   if (IsTailCallReturn) {
@@ -300,7 +296,6 @@ static int64_t getArgumentStackToRestore(MachineFunction &MF,
 static bool produceCompactUnwindFrame(MachineFunction &MF);
 static bool needsWinCFI(const MachineFunction &MF);
 static StackOffset getSVEStackSize(const MachineFunction &MF);
-static bool needsShadowCallStackPrologueEpilogue(MachineFunction &MF);
 
 /// Returns true if a homogeneous prolog or epilog code can be emitted
 /// for the size optimization. If possible, a frame helper call is injected.
@@ -335,6 +330,23 @@ bool AArch64FrameLowering::homogeneousPrologEpilog(
   if (AFI->hasSwiftAsyncContext())
     return false;
 
+  // If there are an odd number of GPRs before LR and FP in the CSRs list,
+  // they will not be paired into one RegPairInfo, which is incompatible with
+  // the assumption made by the homogeneous prolog epilog pass.
+  const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
+  unsigned NumGPRs = 0;
+  for (unsigned I = 0; CSRegs[I]; ++I) {
+    Register Reg = CSRegs[I];
+    if (Reg == AArch64::LR) {
+      assert(CSRegs[I + 1] == AArch64::FP);
+      if (NumGPRs % 2 != 0)
+        return false;
+      break;
+    }
+    if (AArch64::GPR64RegClass.contains(Reg))
+      ++NumGPRs;
+  }
+
   return true;
 }
 
@@ -617,7 +629,7 @@ void AArch64FrameLowering::resetCFIToInitialState(
   }
 
   // Shadow call stack uses X18, reset it.
-  if (needsShadowCallStackPrologueEpilogue(MF))
+  if (MFI.needsShadowCallStackPrologueEpilogue(MF))
     insertCFISameValue(CFIDesc, MF, MBB, InsertPt,
                        TRI.getDwarfRegNum(AArch64::X18, true));
 
@@ -1290,19 +1302,6 @@ static bool IsSVECalleeSave(MachineBasicBlock::iterator I) {
   }
 }
 
-static bool needsShadowCallStackPrologueEpilogue(MachineFunction &MF) {
-  if (!(llvm::any_of(
-            MF.getFrameInfo().getCalleeSavedInfo(),
-            [](const auto &Info) { return Info.getReg() == AArch64::LR; }) &&
-        MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack)))
-    return false;
-
-  if (!MF.getSubtarget<AArch64Subtarget>().isXRegisterReserved(18))
-    report_fatal_error("Must reserve x18 to use shadow call stack");
-
-  return true;
-}
-
 static void emitShadowCallStackPrologue(const TargetInstrInfo &TII,
                                         MachineFunction &MF,
                                         MachineBasicBlock &MBB,
@@ -1414,7 +1413,7 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
   DebugLoc DL;
 
   const auto &MFnI = *MF.getInfo<AArch64FunctionInfo>();
-  if (needsShadowCallStackPrologueEpilogue(MF))
+  if (MFnI.needsShadowCallStackPrologueEpilogue(MF))
     emitShadowCallStackPrologue(*TII, MF, MBB, MBBI, DL, NeedsWinCFI,
                                 MFnI.needsDwarfUnwindInfo(MF));
 
@@ -1945,7 +1944,7 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
       if (NeedsWinCFI)
         HasWinCFI = true; // AArch64PointerAuth pass will insert SEH_PACSignLR
     }
-    if (needsShadowCallStackPrologueEpilogue(MF))
+    if (AFI->needsShadowCallStackPrologueEpilogue(MF))
       emitShadowCallStackEpilogue(*TII, MF, MBB, MBB.getFirstTerminator(), DL);
     if (EmitCFI)
       emitCalleeSavedGPRRestores(MBB, MBB.getFirstTerminator());
@@ -2768,7 +2767,7 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
       // Update register live in.
       if (!MRI.isReserved(RPI.Reg1))
         MBB.addLiveIn(RPI.Reg1);
-      if (!MRI.isReserved(RPI.Reg2))
+      if (RPI.isPaired() && !MRI.isReserved(RPI.Reg2))
         MBB.addLiveIn(RPI.Reg2);
     }
     return true;
@@ -3018,6 +3017,7 @@ void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF,
                                 : (unsigned)AArch64::NoRegister;
 
   unsigned ExtraCSSpill = 0;
+  bool HasUnpairedGPR64 = false;
   // Figure out which callee-saved registers to save/restore.
   for (unsigned i = 0; CSRegs[i]; ++i) {
     const unsigned Reg = CSRegs[i];
@@ -3028,10 +3028,29 @@ void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF,
 
     bool RegUsed = SavedRegs.test(Reg);
     unsigned PairedReg = AArch64::NoRegister;
-    if (AArch64::GPR64RegClass.contains(Reg) ||
-        AArch64::FPR64RegClass.contains(Reg) ||
-        AArch64::FPR128RegClass.contains(Reg))
-      PairedReg = CSRegs[i ^ 1];
+    const bool RegIsGPR64 = AArch64::GPR64RegClass.contains(Reg);
+    if (RegIsGPR64 || AArch64::FPR64RegClass.contains(Reg) ||
+        AArch64::FPR128RegClass.contains(Reg)) {
+      // Compensate for odd numbers of GP CSRs.
+      // For now, all the known cases of odd number of CSRs are of GPRs.
+      if (HasUnpairedGPR64)
+        PairedReg = CSRegs[i % 2 == 0 ? i - 1 : i + 1];
+      else
+        PairedReg = CSRegs[i ^ 1];
+    }
+
+    // If the function requires all the GP registers to save (SavedRegs),
+    // and there are an odd number of GP CSRs at the same time (CSRegs),
+    // PairedReg could be in a different register class from Reg, which would
+    // lead to a FPR (usually D8) accidentally being marked saved.
+    if (RegIsGPR64 && !AArch64::GPR64RegClass.contains(PairedReg)) {
+      PairedReg = AArch64::NoRegister;
+      HasUnpairedGPR64 = true;
+    }
+    assert(PairedReg == AArch64::NoRegister ||
+           AArch64::GPR64RegClass.contains(Reg, PairedReg) ||
+           AArch64::FPR64RegClass.contains(Reg, PairedReg) ||
+           AArch64::FPR128RegClass.contains(Reg, PairedReg));
 
     if (!RegUsed) {
       if (AArch64::GPR64RegClass.contains(Reg) &&
@@ -3130,12 +3149,21 @@ void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF,
       LLVM_DEBUG(dbgs() << "Spilling " << printReg(UnspilledCSGPR, RegInfo)
                         << " to get a scratch register.\n");
       SavedRegs.set(UnspilledCSGPR);
+      ExtraCSSpill = UnspilledCSGPR;
+
       // MachO's compact unwind format relies on all registers being stored in
       // pairs, so if we need to spill one extra for BigStack, then we need to
       // store the pair.
-      if (producePairRegisters(MF))
-        SavedRegs.set(UnspilledCSGPRPaired);
-      ExtraCSSpill = UnspilledCSGPR;
+      if (producePairRegisters(MF)) {
+        if (UnspilledCSGPRPaired == AArch64::NoRegister) {
+          // Failed to make a pair for compact unwind format, revert spilling.
+          if (produceCompactUnwindFrame(MF)) {
+            SavedRegs.reset(UnspilledCSGPR);
+            ExtraCSSpill = AArch64::NoRegister;
+          }
+        } else
+          SavedRegs.set(UnspilledCSGPRPaired);
+      }
     }
 
     // If we didn't find an extra callee-saved register to spill, create
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index dcd3a9f1e4..a16a102e47 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -14431,6 +14431,31 @@ static bool areOperandsOfVmullHighP64(Value *Op1, Value *Op2) {
   return isOperandOfVmullHighP64(Op1) && isOperandOfVmullHighP64(Op2);
 }
 
+static bool shouldSinkVectorOfPtrs(Value *Ptrs, SmallVectorImpl<Use *> &Ops) {
+  // Restrict ourselves to the form CodeGenPrepare typically constructs.
+  auto *GEP = dyn_cast<GetElementPtrInst>(Ptrs);
+  if (!GEP || GEP->getNumOperands() != 2)
+    return false;
+
+  Value *Base = GEP->getOperand(0);
+  Value *Offsets = GEP->getOperand(1);
+
+  // We only care about scalar_base+vector_offsets.
+  if (Base->getType()->isVectorTy() || !Offsets->getType()->isVectorTy())
+    return false;
+
+  // Sink extends that would allow us to use 32-bit offset vectors.
+  if (isa<SExtInst>(Offsets) || isa<ZExtInst>(Offsets)) {
+    auto *OffsetsInst = cast<Instruction>(Offsets);
+    if (OffsetsInst->getType()->getScalarSizeInBits() > 32 &&
+        OffsetsInst->getOperand(0)->getType()->getScalarSizeInBits() <= 32)
+      Ops.push_back(&GEP->getOperandUse(1));
+  }
+
+  // Sink the GEP.
+  return true;
+}
+
 /// Check if sinking \p I's operands to I's basic block is profitable, because
 /// the operands can be folded into a target instruction, e.g.
 /// shufflevectors extracts and/or sext/zext can be folded into (u,s)subl(2).
@@ -14532,6 +14557,16 @@ bool AArch64TargetLowering::shouldSinkOperands(
       Ops.push_back(&II->getArgOperandUse(0));
       Ops.push_back(&II->getArgOperandUse(1));
       return true;
+    case Intrinsic::masked_gather:
+      if (!shouldSinkVectorOfPtrs(II->getArgOperand(0), Ops))
+        return false;
+      Ops.push_back(&II->getArgOperandUse(0));
+      return true;
+    case Intrinsic::masked_scatter:
+      if (!shouldSinkVectorOfPtrs(II->getArgOperand(1), Ops))
+        return false;
+      Ops.push_back(&II->getArgOperandUse(1));
+      return true;
     default:
       return false;
     }
@@ -19065,9 +19100,14 @@ static SDValue tryCombineShiftImm(unsigned IID, SDNode *N, SelectionDAG &DAG) {
   case Intrinsic::aarch64_neon_sshl:
   case Intrinsic::aarch64_neon_ushl:
     // For positive shift amounts we can use SHL, as ushl/sshl perform a regular
-    // left shift for positive shift amounts. Below, we only replace the current
-    // node with VSHL, if this condition is met.
-    Opcode = AArch64ISD::VSHL;
+    // left shift for positive shift amounts. For negative shifts we can use a
+    // VASHR/VLSHR as appropiate.
+    if (ShiftAmount < 0) {
+      Opcode = IID == Intrinsic::aarch64_neon_sshl ? AArch64ISD::VASHR
+                                                   : AArch64ISD::VLSHR;
+      ShiftAmount = -ShiftAmount;
+    } else
+      Opcode = AArch64ISD::VSHL;
     IsRightShift = false;
     break;
   }
diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
index 57d69ae05c..e5dbfa404b 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
@@ -1517,7 +1517,7 @@ def UImm3s8Operand : UImmScaledMemoryIndexed<3, 8>;
 
 def uimm3s8 : Operand<i64>, ImmLeaf<i64,
 [{ return Imm >= 0 && Imm <= 56 && ((Imm % 8) == 0); }], UImmS8XForm> {
-  let PrintMethod = "printVectorIndex<8>";
+  let PrintMethod = "printMatrixIndex<8>";
   let ParserMatchClass = UImm3s8Operand;
 }
 
@@ -6011,11 +6011,11 @@ multiclass SIMDLogicalThreeVectorTied<bit U, bits<2> size,
 
 // ARMv8.2-A Dot Product Instructions (Vector): These instructions extract
 // bytes from S-sized elements.
-class BaseSIMDThreeSameVectorDot<bit Q, bit U, bit Mixed, string asm, string kind1,
-                                 string kind2, RegisterOperand RegType,
+class BaseSIMDThreeSameVectorDot<bit Q, bit U, bits<2> sz, bits<4> opc, string asm,
+                                 string kind1, string kind2, RegisterOperand RegType,
                                  ValueType AccumType, ValueType InputType,
                                  SDPatternOperator OpNode> :
-        BaseSIMDThreeSameVectorTied<Q, U, 0b100, {0b1001, Mixed}, RegType, asm, kind1,
+        BaseSIMDThreeSameVectorTied<Q, U, {sz, 0b0}, {0b1, opc}, RegType, asm, kind1,
         [(set (AccumType RegType:$dst),
               (OpNode (AccumType RegType:$Rd),
                       (InputType RegType:$Rn),
@@ -6024,9 +6024,9 @@ class BaseSIMDThreeSameVectorDot<bit Q, bit U, bit Mixed, string asm, string kin
 }
 
 multiclass SIMDThreeSameVectorDot<bit U, bit Mixed, string asm, SDPatternOperator OpNode> {
-  def v8i8  : BaseSIMDThreeSameVectorDot<0, U, Mixed, asm, ".2s", ".8b", V64,
+  def v8i8  : BaseSIMDThreeSameVectorDot<0, U, 0b10, {0b001, Mixed}, asm, ".2s", ".8b", V64,
                                          v2i32, v8i8, OpNode>;
-  def v16i8 : BaseSIMDThreeSameVectorDot<1, U, Mixed, asm, ".4s", ".16b", V128,
+  def v16i8 : BaseSIMDThreeSameVectorDot<1, U, 0b10, {0b001, Mixed}, asm, ".4s", ".16b", V128,
                                          v4i32, v16i8, OpNode>;
 }
 
@@ -8482,12 +8482,12 @@ class SIMDThreeSameVectorMatMul<bit B, bit U, string asm, SDPatternOperator OpNo
 
 //----------------------------------------------------------------------------
 // ARMv8.2-A Dot Product Instructions (Indexed)
-class BaseSIMDThreeSameVectorDotIndex<bit Q, bit U, bit Mixed, bits<2> size, string asm,
+class BaseSIMDThreeSameVectorIndexS<bit Q, bit U, bits<2> size, bits<4> opc, string asm,
                                       string dst_kind, string lhs_kind, string rhs_kind,
                                       RegisterOperand RegType,
                                       ValueType AccumType, ValueType InputType,
                                       SDPatternOperator OpNode> :
-        BaseSIMDIndexedTied<Q, U, 0b0, size, {0b111, Mixed}, RegType, RegType, V128,
+        BaseSIMDIndexedTied<Q, U, 0b0, size, opc, RegType, RegType, V128,
                             VectorIndexS, asm, "", dst_kind, lhs_kind, rhs_kind,
         [(set (AccumType RegType:$dst),
               (AccumType (OpNode (AccumType RegType:$Rd),
@@ -8502,20 +8502,20 @@ class BaseSIMDThreeSameVectorDotIndex<bit Q, bit U, bit Mixed, bits<2> size, str
 
 multiclass SIMDThreeSameVectorDotIndex<bit U, bit Mixed, bits<2> size, string asm,
                                        SDPatternOperator OpNode> {
-  def v8i8  : BaseSIMDThreeSameVectorDotIndex<0, U, Mixed, size, asm, ".2s", ".8b", ".4b",
+  def v8i8  : BaseSIMDThreeSameVectorIndexS<0, U, size, {0b111, Mixed}, asm, ".2s", ".8b", ".4b",
                                               V64, v2i32, v8i8, OpNode>;
-  def v16i8 : BaseSIMDThreeSameVectorDotIndex<1, U, Mixed, size, asm, ".4s", ".16b", ".4b",
+  def v16i8 : BaseSIMDThreeSameVectorIndexS<1, U, size, {0b111, Mixed}, asm, ".4s", ".16b", ".4b",
                                               V128, v4i32, v16i8, OpNode>;
 }
 
 // ARMv8.2-A Fused Multiply Add-Long Instructions (Indexed)
 let mayRaiseFPException = 1, Uses = [FPCR] in
-class BaseSIMDThreeSameVectorFMLIndex<bit Q, bit U, bits<4> opc, string asm,
+class BaseSIMDThreeSameVectorIndexH<bit Q, bit U, bits<2> sz, bits<4> opc, string asm,
                                       string dst_kind, string lhs_kind,
                                       string rhs_kind, RegisterOperand RegType,
-                                      ValueType AccumType, ValueType InputType,
-                                      SDPatternOperator OpNode> :
-        BaseSIMDIndexedTied<Q, U, 0, 0b10, opc, RegType, RegType, V128_lo,
+                                      RegisterOperand RegType_lo, ValueType AccumType,
+                                      ValueType InputType, SDPatternOperator OpNode> :
+        BaseSIMDIndexedTied<Q, U, 0, sz, opc, RegType, RegType, RegType_lo,
                             VectorIndexH, asm, "", dst_kind, lhs_kind, rhs_kind,
           [(set (AccumType RegType:$dst),
                 (AccumType (OpNode (AccumType RegType:$Rd),
@@ -8531,10 +8531,10 @@ class BaseSIMDThreeSameVectorFMLIndex<bit Q, bit U, bits<4> opc, string asm,
 
 multiclass SIMDThreeSameVectorFMLIndex<bit U, bits<4> opc, string asm,
                                        SDPatternOperator OpNode> {
-  def v4f16 : BaseSIMDThreeSameVectorFMLIndex<0, U, opc, asm, ".2s", ".2h", ".h",
-                                              V64, v2f32, v4f16, OpNode>;
-  def v8f16 : BaseSIMDThreeSameVectorFMLIndex<1, U, opc, asm, ".4s", ".4h", ".h",
-                                              V128, v4f32, v8f16, OpNode>;
+  def v4f16 : BaseSIMDThreeSameVectorIndexH<0, U, 0b10, opc, asm, ".2s", ".2h", ".h",
+                                              V64, V128_lo, v2f32, v4f16, OpNode>;
+  def v8f16 : BaseSIMDThreeSameVectorIndexH<1, U, 0b10, opc, asm, ".4s", ".4h", ".h",
+                                              V128, V128_lo, v4f32, v8f16, OpNode>;
 }
 
 multiclass SIMDFPIndexed<bit U, bits<4> opc, string asm,
diff --git a/llvm/lib/Target/AArch64/AArch64InstrGISel.td b/llvm/lib/Target/AArch64/AArch64InstrGISel.td
index c6ff7bea4b..27338bd243 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrGISel.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrGISel.td
@@ -511,3 +511,20 @@ let AddedComplexity = 19 in {
   defm : VecROStoreLane64_0Pat<ro16, store, v4i16, i16, hsub, STRHroW, STRHroX>;
   defm : VecROStoreLane64_0Pat<ro32, store, v2i32, i32, ssub, STRSroW, STRSroX>;
 }
+
+def : Pat<(v8i8 (AArch64dup (i8 (load (am_indexed8 GPR64sp:$Rn))))),
+          (LD1Rv8b GPR64sp:$Rn)>;
+def : Pat<(v16i8 (AArch64dup (i8 (load GPR64sp:$Rn)))),
+          (LD1Rv16b GPR64sp:$Rn)>;
+def : Pat<(v4i16 (AArch64dup (i16 (load GPR64sp:$Rn)))),
+          (LD1Rv4h GPR64sp:$Rn)>;
+def : Pat<(v8i16 (AArch64dup (i16 (load GPR64sp:$Rn)))),
+          (LD1Rv8h GPR64sp:$Rn)>;
+def : Pat<(v2i32 (AArch64dup (i32 (load GPR64sp:$Rn)))),
+          (LD1Rv2s GPR64sp:$Rn)>;
+def : Pat<(v4i32 (AArch64dup (i32 (load GPR64sp:$Rn)))),
+          (LD1Rv4s GPR64sp:$Rn)>;
+def : Pat<(v2i64 (AArch64dup (i64 (load GPR64sp:$Rn)))),
+          (LD1Rv2d GPR64sp:$Rn)>;
+def : Pat<(v1i64 (AArch64dup (i64 (load GPR64sp:$Rn)))),
+          (LD1Rv1d GPR64sp:$Rn)>;
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 5a234bceb2..7f1421549b 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -14,6 +14,7 @@
 #include "AArch64InstrInfo.h"
 #include "AArch64FrameLowering.h"
 #include "AArch64MachineFunctionInfo.h"
+#include "AArch64PointerAuth.h"
 #include "AArch64Subtarget.h"
 #include "MCTargetDesc/AArch64AddressingModes.h"
 #include "Utils/AArch64BaseInfo.h"
@@ -2490,6 +2491,20 @@ bool AArch64InstrInfo::isPairableLdStInst(const MachineInstr &MI) {
   }
 }
 
+bool AArch64InstrInfo::isTailCallReturnInst(const MachineInstr &MI) {
+  switch (MI.getOpcode()) {
+  default:
+    assert((!MI.isCall() || !MI.isReturn()) &&
+           "Unexpected instruction - was a new tail call opcode introduced?");
+    return false;
+  case AArch64::TCRETURNdi:
+  case AArch64::TCRETURNri:
+  case AArch64::TCRETURNriBTI:
+  case AArch64::TCRETURNriALL:
+    return true;
+  }
+}
+
 unsigned AArch64InstrInfo::convertToFlagSettingOpc(unsigned Opc) {
   switch (Opc) {
   default:
@@ -2963,7 +2978,10 @@ bool AArch64InstrInfo::canFoldIntoAddrMode(const MachineInstr &MemI,
 
     // Don't fold the add if the result would be slower, unless optimising for
     // size.
-    int64_t Shift = AddrI.getOperand(3).getImm();
+    unsigned Shift = static_cast<unsigned>(AddrI.getOperand(3).getImm());
+    if (AArch64_AM::getShiftType(Shift) != AArch64_AM::ShiftExtendType::LSL)
+      return false;
+    Shift = AArch64_AM::getShiftValue(Shift);
     if (!OptSize) {
       if ((Shift != 2 && Shift != 3) || !Subtarget.hasAddrLSLFast())
         return false;
@@ -5492,42 +5510,14 @@ MachineInstr *AArch64InstrInfo::foldMemoryOperandImpl(
     //
     //   STRXui %xzr, %stack.0
     //
-    if (IsSpill && DstMO.isUndef() && SrcReg.isPhysical()) {
+    if (IsSpill && DstMO.isUndef() && SrcReg == AArch64::WZR &&
+        TRI.getRegSizeInBits(*getRegClass(DstReg)) == 64) {
       assert(SrcMO.getSubReg() == 0 &&
              "Unexpected subreg on physical register");
-      const TargetRegisterClass *SpillRC;
-      unsigned SpillSubreg;
-      switch (DstMO.getSubReg()) {
-      default:
-        SpillRC = nullptr;
-        break;
-      case AArch64::sub_32:
-      case AArch64::ssub:
-        if (AArch64::GPR32RegClass.contains(SrcReg)) {
-          SpillRC = &AArch64::GPR64RegClass;
-          SpillSubreg = AArch64::sub_32;
-        } else if (AArch64::FPR32RegClass.contains(SrcReg)) {
-          SpillRC = &AArch64::FPR64RegClass;
-          SpillSubreg = AArch64::ssub;
-        } else
-          SpillRC = nullptr;
-        break;
-      case AArch64::dsub:
-        if (AArch64::FPR64RegClass.contains(SrcReg)) {
-          SpillRC = &AArch64::FPR128RegClass;
-          SpillSubreg = AArch64::dsub;
-        } else
-          SpillRC = nullptr;
-        break;
-      }
-
-      if (SpillRC)
-        if (unsigned WidenedSrcReg =
-                TRI.getMatchingSuperReg(SrcReg, SpillSubreg, SpillRC)) {
-          storeRegToStackSlot(MBB, InsertPt, WidenedSrcReg, SrcMO.isKill(),
-                              FrameIndex, SpillRC, &TRI, Register());
-          return &*--InsertPt;
-        }
+      storeRegToStackSlot(MBB, InsertPt, AArch64::XZR, SrcMO.isKill(),
+                          FrameIndex, &AArch64::GPR64RegClass, &TRI,
+                          Register());
+      return &*--InsertPt;
     }
 
     // Handle cases like filling use of:
@@ -5594,6 +5584,14 @@ int llvm::isAArch64FrameOffsetLegal(const MachineInstr &MI,
   switch (MI.getOpcode()) {
   default:
     break;
+  case AArch64::LD1Rv1d:
+  case AArch64::LD1Rv2s:
+  case AArch64::LD1Rv2d:
+  case AArch64::LD1Rv4h:
+  case AArch64::LD1Rv4s:
+  case AArch64::LD1Rv8b:
+  case AArch64::LD1Rv8h:
+  case AArch64::LD1Rv16b:
   case AArch64::LD1Twov2d:
   case AArch64::LD1Threev2d:
   case AArch64::LD1Fourv2d:
@@ -8217,12 +8215,24 @@ AArch64InstrInfo::getOutliningCandidateInfo(
   // necessary. However, at this point we don't know if the outlined function
   // will have a RET instruction so we assume the worst.
   const TargetRegisterInfo &TRI = getRegisterInfo();
+  // Performing a tail call may require extra checks when PAuth is enabled.
+  // If PAuth is disabled, set it to zero for uniformity.
+  unsigned NumBytesToCheckLRInTCEpilogue = 0;
   if (FirstCand.getMF()
           ->getInfo<AArch64FunctionInfo>()
           ->shouldSignReturnAddress(true)) {
     // One PAC and one AUT instructions
     NumBytesToCreateFrame += 8;
 
+    // PAuth is enabled - set extra tail call cost, if any.
+    auto LRCheckMethod = Subtarget.getAuthenticatedLRCheckMethod();
+    NumBytesToCheckLRInTCEpilogue =
+        AArch64PAuth::getCheckerSizeInBytes(LRCheckMethod);
+    // Checking the authenticated LR value may significantly impact
+    // SequenceSize, so account for it for more precise results.
+    if (isTailCallReturnInst(*RepeatedSequenceLocs[0].back()))
+      SequenceSize += NumBytesToCheckLRInTCEpilogue;
+
     // We have to check if sp modifying instructions would get outlined.
     // If so we only allow outlining if sp is unchanged overall, so matching
     // sub and add instructions are okay to outline, all other sp modifications
@@ -8393,7 +8403,8 @@ AArch64InstrInfo::getOutliningCandidateInfo(
   if (RepeatedSequenceLocs[0].back()->isTerminator()) {
     FrameID = MachineOutlinerTailCall;
     NumBytesToCreateFrame = 0;
-    SetCandidateCallInfo(MachineOutlinerTailCall, 4);
+    unsigned NumBytesForCall = 4 + NumBytesToCheckLRInTCEpilogue;
+    SetCandidateCallInfo(MachineOutlinerTailCall, NumBytesForCall);
   }
 
   else if (LastInstrOpcode == AArch64::BL ||
@@ -8402,7 +8413,7 @@ AArch64InstrInfo::getOutliningCandidateInfo(
             !HasBTI)) {
     // FIXME: Do we need to check if the code after this uses the value of LR?
     FrameID = MachineOutlinerThunk;
-    NumBytesToCreateFrame = 0;
+    NumBytesToCreateFrame = NumBytesToCheckLRInTCEpilogue;
     SetCandidateCallInfo(MachineOutlinerThunk, 4);
   }
 
@@ -9123,14 +9134,14 @@ bool AArch64InstrInfo::shouldOutlineFromFunctionByDefault(
 
 void AArch64InstrInfo::buildClearRegister(Register Reg, MachineBasicBlock &MBB,
                                           MachineBasicBlock::iterator Iter,
-                                          DebugLoc &DL) const {
+                                          DebugLoc &DL,
+                                          bool AllowSideEffects) const {
   const MachineFunction &MF = *MBB.getParent();
   const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>();
   const AArch64RegisterInfo &TRI = *STI.getRegisterInfo();
 
   if (TRI.isGeneralPurposeRegister(MF, Reg)) {
-    BuildMI(MBB, Iter, DL, get(AArch64::MOVi64imm), Reg)
-      .addImm(0);
+    BuildMI(MBB, Iter, DL, get(AArch64::MOVZXi), Reg).addImm(0).addImm(0);
   } else if (STI.hasSVE()) {
     BuildMI(MBB, Iter, DL, get(AArch64::DUP_ZI_D), Reg)
       .addImm(0)
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
index f5874d7856..a934103c90 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
@@ -129,6 +129,9 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo {
   /// Return true if pairing the given load or store may be paired with another.
   static bool isPairableLdStInst(const MachineInstr &MI);
 
+  /// Returns true if MI is one of the TCRETURN* instructions.
+  static bool isTailCallReturnInst(const MachineInstr &MI);
+
   /// Return the opcode that set flags when possible.  The caller is
   /// responsible for ensuring the opc has a flag setting equivalent.
   static unsigned convertToFlagSettingOpc(unsigned Opc);
@@ -330,8 +333,8 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo {
   bool shouldOutlineFromFunctionByDefault(MachineFunction &MF) const override;
 
   void buildClearRegister(Register Reg, MachineBasicBlock &MBB,
-                          MachineBasicBlock::iterator Iter,
-                          DebugLoc &DL) const override;
+                          MachineBasicBlock::iterator Iter, DebugLoc &DL,
+                          bool AllowSideEffects = true) const override;
 
   /// Returns the vector element size (B, H, S or D) of an SVE opcode.
   uint64_t getElementSizeForOpcode(unsigned Opc) const;
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 5293df90b8..382d3956f1 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -160,6 +160,8 @@ def HasSME2          : Predicate<"Subtarget->hasSME2()">,
                                  AssemblerPredicateWithAll<(all_of FeatureSME2), "sme2">;
 def HasSME2p1        : Predicate<"Subtarget->hasSME2p1()">,
                                  AssemblerPredicateWithAll<(all_of FeatureSME2p1), "sme2p1">;
+def HasFPMR          : Predicate<"Subtarget->hasFPMR()">,
+                                 AssemblerPredicateWithAll<(all_of FeatureFPMR), "fpmr">;
 
 // A subset of SVE(2) instructions are legal in Streaming SVE execution mode,
 // they should be enabled if either has been specified.
@@ -1206,7 +1208,7 @@ defm USDOTlane : SIMDThreeSameVectorDotIndex<0, 1, 0b10, "usdot", int_aarch64_ne
 class BaseSIMDSUDOTIndex<bit Q, string dst_kind, string lhs_kind,
                          string rhs_kind, RegisterOperand RegType,
                          ValueType AccumType, ValueType InputType>
-      : BaseSIMDThreeSameVectorDotIndex<Q, 0, 1, 0b00, "sudot", dst_kind,
+      : BaseSIMDThreeSameVectorIndexS<Q, 0, 0b00, 0b1111, "sudot", dst_kind,
                                         lhs_kind, rhs_kind, RegType, AccumType,
                                         InputType, null_frag> {
   let Pattern = [(set (AccumType RegType:$dst),
diff --git a/llvm/lib/Target/AArch64/AArch64LowerHomogeneousPrologEpilog.cpp b/llvm/lib/Target/AArch64/AArch64LowerHomogeneousPrologEpilog.cpp
index d054fe509b..b8b74ae840 100644
--- a/llvm/lib/Target/AArch64/AArch64LowerHomogeneousPrologEpilog.cpp
+++ b/llvm/lib/Target/AArch64/AArch64LowerHomogeneousPrologEpilog.cpp
@@ -146,8 +146,11 @@ static std::string getFrameHelperName(SmallVectorImpl<unsigned> &Regs,
     break;
   }
 
-  for (auto Reg : Regs)
+  for (auto Reg : Regs) {
+    if (Reg == AArch64::NoRegister)
+      continue;
     RegStream << AArch64InstPrinter::getRegisterName(Reg);
+  }
 
   return RegStream.str();
 }
@@ -195,46 +198,84 @@ static MachineFunction &createFrameHelperMachineFunction(Module *M,
 }
 
 /// Emit a store-pair instruction for frame-setup.
+/// If Reg2 is AArch64::NoRegister, emit STR instead.
 static void emitStore(MachineFunction &MF, MachineBasicBlock &MBB,
                       MachineBasicBlock::iterator Pos,
                       const TargetInstrInfo &TII, unsigned Reg1, unsigned Reg2,
                       int Offset, bool IsPreDec) {
+  assert(Reg1 != AArch64::NoRegister);
+  const bool IsPaired = Reg2 != AArch64::NoRegister;
   bool IsFloat = AArch64::FPR64RegClass.contains(Reg1);
   assert(!(IsFloat ^ AArch64::FPR64RegClass.contains(Reg2)));
   unsigned Opc;
-  if (IsPreDec)
-    Opc = IsFloat ? AArch64::STPDpre : AArch64::STPXpre;
-  else
-    Opc = IsFloat ? AArch64::STPDi : AArch64::STPXi;
+  if (IsPreDec) {
+    if (IsFloat)
+      Opc = IsPaired ? AArch64::STPDpre : AArch64::STRDpre;
+    else
+      Opc = IsPaired ? AArch64::STPXpre : AArch64::STRXpre;
+  } else {
+    if (IsFloat)
+      Opc = IsPaired ? AArch64::STPDi : AArch64::STRDui;
+    else
+      Opc = IsPaired ? AArch64::STPXi : AArch64::STRXui;
+  }
+  // The implicit scale for Offset is 8.
+  TypeSize Scale(0U, false);
+  unsigned Width;
+  int64_t MinOffset, MaxOffset;
+  [[maybe_unused]] bool Success =
+      AArch64InstrInfo::getMemOpInfo(Opc, Scale, Width, MinOffset, MaxOffset);
+  assert(Success && "Invalid Opcode");
+  Offset *= (8 / (int)Scale);
 
   MachineInstrBuilder MIB = BuildMI(MBB, Pos, DebugLoc(), TII.get(Opc));
   if (IsPreDec)
     MIB.addDef(AArch64::SP);
-  MIB.addReg(Reg2)
-      .addReg(Reg1)
+  if (IsPaired)
+    MIB.addReg(Reg2);
+  MIB.addReg(Reg1)
       .addReg(AArch64::SP)
       .addImm(Offset)
       .setMIFlag(MachineInstr::FrameSetup);
 }
 
 /// Emit a load-pair instruction for frame-destroy.
+/// If Reg2 is AArch64::NoRegister, emit LDR instead.
 static void emitLoad(MachineFunction &MF, MachineBasicBlock &MBB,
                      MachineBasicBlock::iterator Pos,
                      const TargetInstrInfo &TII, unsigned Reg1, unsigned Reg2,
                      int Offset, bool IsPostDec) {
+  assert(Reg1 != AArch64::NoRegister);
+  const bool IsPaired = Reg2 != AArch64::NoRegister;
   bool IsFloat = AArch64::FPR64RegClass.contains(Reg1);
   assert(!(IsFloat ^ AArch64::FPR64RegClass.contains(Reg2)));
   unsigned Opc;
-  if (IsPostDec)
-    Opc = IsFloat ? AArch64::LDPDpost : AArch64::LDPXpost;
-  else
-    Opc = IsFloat ? AArch64::LDPDi : AArch64::LDPXi;
+  if (IsPostDec) {
+    if (IsFloat)
+      Opc = IsPaired ? AArch64::LDPDpost : AArch64::LDRDpost;
+    else
+      Opc = IsPaired ? AArch64::LDPXpost : AArch64::LDRXpost;
+  } else {
+    if (IsFloat)
+      Opc = IsPaired ? AArch64::LDPDi : AArch64::LDRDui;
+    else
+      Opc = IsPaired ? AArch64::LDPXi : AArch64::LDRXui;
+  }
+  // The implicit scale for Offset is 8.
+  TypeSize Scale(0U, false);
+  unsigned Width;
+  int64_t MinOffset, MaxOffset;
+  [[maybe_unused]] bool Success =
+      AArch64InstrInfo::getMemOpInfo(Opc, Scale, Width, MinOffset, MaxOffset);
+  assert(Success && "Invalid Opcode");
+  Offset *= (8 / (int)Scale);
 
   MachineInstrBuilder MIB = BuildMI(MBB, Pos, DebugLoc(), TII.get(Opc));
   if (IsPostDec)
     MIB.addDef(AArch64::SP);
-  MIB.addReg(Reg2, getDefRegState(true))
-      .addReg(Reg1, getDefRegState(true))
+  if (IsPaired)
+    MIB.addReg(Reg2, getDefRegState(true));
+  MIB.addReg(Reg1, getDefRegState(true))
       .addReg(AArch64::SP)
       .addImm(Offset)
       .setMIFlag(MachineInstr::FrameDestroy);
@@ -433,9 +474,18 @@ bool AArch64LowerHomogeneousPE::lowerEpilog(
 
   DebugLoc DL = MI.getDebugLoc();
   SmallVector<unsigned, 8> Regs;
+  bool HasUnpairedReg = false;
   for (auto &MO : MI.operands())
-    if (MO.isReg())
+    if (MO.isReg()) {
+      if (!MO.getReg().isValid()) {
+        // For now we are only expecting unpaired GP registers which should
+        // occur exactly once.
+        assert(!HasUnpairedReg);
+        HasUnpairedReg = true;
+      }
       Regs.push_back(MO.getReg());
+    }
+  (void)HasUnpairedReg;
   int Size = (int)Regs.size();
   if (Size == 0)
     return false;
@@ -507,17 +557,26 @@ bool AArch64LowerHomogeneousPE::lowerProlog(
 
   DebugLoc DL = MI.getDebugLoc();
   SmallVector<unsigned, 8> Regs;
+  bool HasUnpairedReg = false;
   int LRIdx = 0;
   std::optional<int> FpOffset;
   for (auto &MO : MI.operands()) {
     if (MO.isReg()) {
-      if (MO.getReg() == AArch64::LR)
-        LRIdx = Regs.size();
+      if (MO.getReg().isValid()) {
+        if (MO.getReg() == AArch64::LR)
+          LRIdx = Regs.size();
+      } else {
+        // For now we are only expecting unpaired GP registers which should
+        // occur exactly once.
+        assert(!HasUnpairedReg);
+        HasUnpairedReg = true;
+      }
       Regs.push_back(MO.getReg());
     } else if (MO.isImm()) {
       FpOffset = MO.getImm();
     }
   }
+  (void)HasUnpairedReg;
   int Size = (int)Regs.size();
   if (Size == 0)
     return false;
diff --git a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.cpp b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.cpp
index 961a19317d..7bb5041b8b 100644
--- a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.cpp
@@ -122,11 +122,27 @@ bool AArch64FunctionInfo::shouldSignReturnAddress(bool SpillsLR) const {
   return SpillsLR;
 }
 
+static bool isLRSpilled(const MachineFunction &MF) {
+  return llvm::any_of(
+      MF.getFrameInfo().getCalleeSavedInfo(),
+      [](const auto &Info) { return Info.getReg() == AArch64::LR; });
+}
+
 bool AArch64FunctionInfo::shouldSignReturnAddress(
     const MachineFunction &MF) const {
-  return shouldSignReturnAddress(llvm::any_of(
-      MF.getFrameInfo().getCalleeSavedInfo(),
-      [](const auto &Info) { return Info.getReg() == AArch64::LR; }));
+  return shouldSignReturnAddress(isLRSpilled(MF));
+}
+
+bool AArch64FunctionInfo::needsShadowCallStackPrologueEpilogue(
+    MachineFunction &MF) const {
+  if (!(isLRSpilled(MF) &&
+        MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack)))
+    return false;
+
+  if (!MF.getSubtarget<AArch64Subtarget>().isXRegisterReserved(18))
+    report_fatal_error("Must reserve x18 to use shadow call stack");
+
+  return true;
 }
 
 bool AArch64FunctionInfo::needsDwarfUnwindInfo(
diff --git a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h
index 8df95ff1e6..0b8bfb04a5 100644
--- a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h
@@ -431,6 +431,8 @@ class AArch64FunctionInfo final : public MachineFunctionInfo {
   bool shouldSignReturnAddress(const MachineFunction &MF) const;
   bool shouldSignReturnAddress(bool SpillsLR) const;
 
+  bool needsShadowCallStackPrologueEpilogue(MachineFunction &MF) const;
+
   bool shouldSignWithBKey() const { return SignWithBKey; }
   bool isMTETagged() const { return IsMTETagged; }
 
diff --git a/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp b/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp
index f8f11b43e3..5d11f0d225 100644
--- a/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp
+++ b/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp
@@ -6,7 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "AArch64PointerAuth.h"
+
 #include "AArch64.h"
+#include "AArch64InstrInfo.h"
 #include "AArch64MachineFunctionInfo.h"
 #include "AArch64Subtarget.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
@@ -14,6 +17,7 @@
 #include "llvm/CodeGen/MachineModuleInfo.h"
 
 using namespace llvm;
+using namespace llvm::AArch64PAuth;
 
 #define AARCH64_POINTER_AUTH_NAME "AArch64 Pointer Authentication"
 
@@ -30,13 +34,19 @@ class AArch64PointerAuth : public MachineFunctionPass {
   StringRef getPassName() const override { return AARCH64_POINTER_AUTH_NAME; }
 
 private:
+  /// An immediate operand passed to BRK instruction, if it is ever emitted.
+  const unsigned BrkOperand = 0xc471;
+
   const AArch64Subtarget *Subtarget = nullptr;
   const AArch64InstrInfo *TII = nullptr;
+  const AArch64RegisterInfo *TRI = nullptr;
 
   void signLR(MachineFunction &MF, MachineBasicBlock::iterator MBBI) const;
 
   void authenticateLR(MachineFunction &MF,
                       MachineBasicBlock::iterator MBBI) const;
+
+  bool checkAuthenticatedLR(MachineBasicBlock::iterator TI) const;
 };
 
 } // end anonymous namespace
@@ -132,22 +142,181 @@ void AArch64PointerAuth::authenticateLR(
   }
 }
 
+namespace {
+
+// Mark dummy LDR instruction as volatile to prevent removing it as dead code.
+MachineMemOperand *createCheckMemOperand(MachineFunction &MF,
+                                         const AArch64Subtarget &Subtarget) {
+  MachinePointerInfo PointerInfo(Subtarget.getAddressCheckPSV());
+  auto MOVolatileLoad =
+      MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile;
+
+  return MF.getMachineMemOperand(PointerInfo, MOVolatileLoad, 4, Align(4));
+}
+
+} // namespace
+
+MachineBasicBlock &llvm::AArch64PAuth::checkAuthenticatedRegister(
+    MachineBasicBlock::iterator MBBI, AuthCheckMethod Method,
+    Register AuthenticatedReg, Register TmpReg, bool UseIKey, unsigned BrkImm) {
+
+  MachineBasicBlock &MBB = *MBBI->getParent();
+  MachineFunction &MF = *MBB.getParent();
+  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
+  const AArch64InstrInfo *TII = Subtarget.getInstrInfo();
+  DebugLoc DL = MBBI->getDebugLoc();
+
+  // First, handle the methods not requiring creating extra MBBs.
+  switch (Method) {
+  default:
+    break;
+  case AuthCheckMethod::None:
+    return MBB;
+  case AuthCheckMethod::DummyLoad:
+    BuildMI(MBB, MBBI, DL, TII->get(AArch64::LDRWui), getWRegFromXReg(TmpReg))
+        .addReg(AArch64::LR)
+        .addImm(0)
+        .addMemOperand(createCheckMemOperand(MF, Subtarget));
+    return MBB;
+  }
+
+  // Control flow has to be changed, so arrange new MBBs.
+
+  // At now, at least an AUT* instruction is expected before MBBI
+  assert(MBBI != MBB.begin() &&
+         "Cannot insert the check at the very beginning of MBB");
+  // The block to insert check into.
+  MachineBasicBlock *CheckBlock = &MBB;
+  // The remaining part of the original MBB that is executed on success.
+  MachineBasicBlock *SuccessBlock = MBB.splitAt(*std::prev(MBBI));
+
+  // The block that explicitly generates a break-point exception on failure.
+  MachineBasicBlock *BreakBlock =
+      MF.CreateMachineBasicBlock(MBB.getBasicBlock());
+  MF.push_back(BreakBlock);
+  MBB.splitSuccessor(SuccessBlock, BreakBlock);
+
+  assert(CheckBlock->getFallThrough() == SuccessBlock);
+  BuildMI(BreakBlock, DL, TII->get(AArch64::BRK)).addImm(BrkImm);
+
+  switch (Method) {
+  case AuthCheckMethod::None:
+  case AuthCheckMethod::DummyLoad:
+    llvm_unreachable("Should be handled above");
+  case AuthCheckMethod::HighBitsNoTBI:
+    BuildMI(CheckBlock, DL, TII->get(AArch64::EORXrs), TmpReg)
+        .addReg(AuthenticatedReg)
+        .addReg(AuthenticatedReg)
+        .addImm(1);
+    BuildMI(CheckBlock, DL, TII->get(AArch64::TBNZX))
+        .addReg(TmpReg)
+        .addImm(62)
+        .addMBB(BreakBlock);
+    return *SuccessBlock;
+  case AuthCheckMethod::XPACHint:
+    assert(AuthenticatedReg == AArch64::LR &&
+           "XPACHint mode is only compatible with checking the LR register");
+    assert(UseIKey && "XPACHint mode is only compatible with I-keys");
+    BuildMI(CheckBlock, DL, TII->get(AArch64::ORRXrs), TmpReg)
+        .addReg(AArch64::XZR)
+        .addReg(AArch64::LR)
+        .addImm(0);
+    BuildMI(CheckBlock, DL, TII->get(AArch64::XPACLRI));
+    BuildMI(CheckBlock, DL, TII->get(AArch64::SUBSXrs), AArch64::XZR)
+        .addReg(TmpReg)
+        .addReg(AArch64::LR)
+        .addImm(0);
+    BuildMI(CheckBlock, DL, TII->get(AArch64::Bcc))
+        .addImm(AArch64CC::NE)
+        .addMBB(BreakBlock);
+    return *SuccessBlock;
+  }
+  llvm_unreachable("Unknown AuthCheckMethod enum");
+}
+
+unsigned llvm::AArch64PAuth::getCheckerSizeInBytes(AuthCheckMethod Method) {
+  switch (Method) {
+  case AuthCheckMethod::None:
+    return 0;
+  case AuthCheckMethod::DummyLoad:
+    return 4;
+  case AuthCheckMethod::HighBitsNoTBI:
+    return 12;
+  case AuthCheckMethod::XPACHint:
+    return 20;
+  }
+  llvm_unreachable("Unknown AuthCheckMethod enum");
+}
+
+bool AArch64PointerAuth::checkAuthenticatedLR(
+    MachineBasicBlock::iterator TI) const {
+  AuthCheckMethod Method = Subtarget->getAuthenticatedLRCheckMethod();
+
+  if (Method == AuthCheckMethod::None)
+    return false;
+
+  // FIXME If FEAT_FPAC is implemented by the CPU, this check can be skipped.
+
+  assert(!TI->getMF()->hasWinCFI() && "WinCFI is not yet supported");
+
+  // The following code may create a signing oracle:
+  //
+  //   <authenticate LR>
+  //   TCRETURN          ; the callee may sign and spill the LR in its prologue
+  //
+  // To avoid generating a signing oracle, check the authenticated value
+  // before possibly re-signing it in the callee, as follows:
+  //
+  //   <authenticate LR>
+  //   <check if LR contains a valid address>
+  //   b.<cond> break_block
+  // ret_block:
+  //   TCRETURN
+  // break_block:
+  //   brk <BrkOperand>
+  //
+  // or just
+  //
+  //   <authenticate LR>
+  //   ldr tmp, [lr]
+  //   TCRETURN
+
+  // TmpReg is chosen assuming X16 and X17 are dead after TI.
+  assert(AArch64InstrInfo::isTailCallReturnInst(*TI) &&
+         "Tail call is expected");
+  Register TmpReg =
+      TI->readsRegister(AArch64::X16, TRI) ? AArch64::X17 : AArch64::X16;
+  assert(!TI->readsRegister(TmpReg, TRI) &&
+         "More than a single register is used by TCRETURN");
+
+  checkAuthenticatedRegister(TI, Method, AArch64::LR, TmpReg, /*UseIKey=*/true,
+                             BrkOperand);
+
+  return true;
+}
+
 bool AArch64PointerAuth::runOnMachineFunction(MachineFunction &MF) {
-  if (!MF.getInfo<AArch64FunctionInfo>()->shouldSignReturnAddress(true))
+  const auto *MFnI = MF.getInfo<AArch64FunctionInfo>();
+  if (!MFnI->shouldSignReturnAddress(true))
     return false;
 
   Subtarget = &MF.getSubtarget<AArch64Subtarget>();
   TII = Subtarget->getInstrInfo();
+  TRI = Subtarget->getRegisterInfo();
 
   SmallVector<MachineBasicBlock::iterator> DeletedInstrs;
+  SmallVector<MachineBasicBlock::iterator> TailCallInstrs;
+
   bool Modified = false;
+  bool HasAuthenticationInstrs = false;
 
   for (auto &MBB : MF) {
     for (auto &MI : MBB) {
       auto It = MI.getIterator();
       switch (MI.getOpcode()) {
       default:
-        // do nothing
+        if (AArch64InstrInfo::isTailCallReturnInst(MI))
+          TailCallInstrs.push_back(It);
         break;
       case AArch64::PAUTH_PROLOGUE:
         signLR(MF, It);
@@ -158,11 +327,20 @@ bool AArch64PointerAuth::runOnMachineFunction(MachineFunction &MF) {
         authenticateLR(MF, It);
         DeletedInstrs.push_back(It);
         Modified = true;
+        HasAuthenticationInstrs = true;
         break;
       }
     }
   }
 
+  // FIXME Do we need to emit any PAuth-related epilogue code at all
+  //       when SCS is enabled?
+  if (HasAuthenticationInstrs &&
+      !MFnI->needsShadowCallStackPrologueEpilogue(MF)) {
+    for (auto TailCall : TailCallInstrs)
+      Modified |= checkAuthenticatedLR(TailCall);
+  }
+
   for (auto MI : DeletedInstrs)
     MI->eraseFromParent();
 
diff --git a/llvm/lib/Target/AArch64/AArch64PointerAuth.h b/llvm/lib/Target/AArch64/AArch64PointerAuth.h
new file mode 100644
index 0000000000..e1ceaed58a
--- /dev/null
+++ b/llvm/lib/Target/AArch64/AArch64PointerAuth.h
@@ -0,0 +1,116 @@
+//===-- AArch64PointerAuth.h -- Harden code using PAuth ---------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AARCH64_AARCH64POINTERAUTH_H
+#define LLVM_LIB_TARGET_AARCH64_AARCH64POINTERAUTH_H
+
+#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/Register.h"
+
+namespace llvm {
+namespace AArch64PAuth {
+
+/// Variants of check performed on an authenticated pointer.
+///
+/// In cases such as authenticating the LR value when performing a tail call
+/// or when re-signing a signed pointer with a different signing schema,
+/// a failed authentication may not generate an exception on its own and may
+/// create an authentication or signing oracle if not checked explicitly.
+///
+/// A number of check methods modify control flow in a similar way by
+/// rewriting the code
+///
+/// ```
+///   <authenticate LR>
+///   <more instructions>
+/// ```
+///
+/// as follows:
+///
+/// ```
+///   <authenticate LR>
+///   <method-specific checker>
+/// ret_block:
+///   <more instructions>
+///   ...
+///
+/// break_block:
+///   brk <code>
+/// ```
+enum class AuthCheckMethod {
+  /// Do not check the value at all
+  None,
+  /// Perform a load to a temporary register
+  DummyLoad,
+  /// Check by comparing bits 62 and 61 of the authenticated address.
+  ///
+  /// This method modifies control flow and inserts the following checker:
+  ///
+  /// ```
+  ///   eor Xtmp, Xn, Xn, lsl #1
+  ///   tbnz Xtmp, #62, break_block
+  /// ```
+  HighBitsNoTBI,
+  /// Check by comparing the authenticated value with an XPAC-ed one without
+  /// using PAuth instructions not encoded as HINT. Can only be applied to LR.
+  ///
+  /// This method modifies control flow and inserts the following checker:
+  ///
+  /// ```
+  ///   mov Xtmp, LR
+  ///   xpaclri           ; encoded as "hint #7"
+  ///   ; Note: at this point, the LR register contains the address as if
+  ///   ; the authentication succeeded and the temporary register contains the
+  ///   ; *real* result of authentication.
+  ///   cmp Xtmp, LR
+  ///   b.ne break_block
+  /// ```
+  XPACHint,
+};
+
+#define AUTH_CHECK_METHOD_CL_VALUES_COMMON                                     \
+      clEnumValN(AArch64PAuth::AuthCheckMethod::None, "none",                  \
+                 "Do not check authenticated address"),                        \
+      clEnumValN(AArch64PAuth::AuthCheckMethod::DummyLoad, "load",             \
+                 "Perform dummy load from authenticated address"),             \
+      clEnumValN(AArch64PAuth::AuthCheckMethod::HighBitsNoTBI,                 \
+                 "high-bits-notbi",                                            \
+                 "Compare bits 62 and 61 of address (TBI should be disabled)")
+
+#define AUTH_CHECK_METHOD_CL_VALUES_LR                                         \
+      AUTH_CHECK_METHOD_CL_VALUES_COMMON,                                      \
+      clEnumValN(AArch64PAuth::AuthCheckMethod::XPACHint, "xpac-hint",         \
+                 "Compare with the result of XPACLRI")
+
+/// Explicitly checks that pointer authentication succeeded.
+///
+/// Assuming AuthenticatedReg contains a value returned by one of the AUT*
+/// instructions, check the value using Method just before the instruction
+/// pointed to by MBBI. If the check succeeds, execution proceeds to the
+/// instruction pointed to by MBBI, otherwise a CPU exception is generated.
+///
+/// Some of the methods may need to know if the pointer was authenticated
+/// using an I-key or D-key and which register can be used as temporary.
+/// If an explicit BRK instruction is used to generate an exception, BrkImm
+/// specifies its immediate operand.
+///
+/// \returns The machine basic block containing the code that is executed
+///          after the check succeeds.
+MachineBasicBlock &checkAuthenticatedRegister(MachineBasicBlock::iterator MBBI,
+                                              AuthCheckMethod Method,
+                                              Register AuthenticatedReg,
+                                              Register TmpReg, bool UseIKey,
+                                              unsigned BrkImm);
+
+/// Returns the number of bytes added by checkAuthenticatedRegister.
+unsigned getCheckerSizeInBytes(AuthCheckMethod Method);
+
+} // end namespace AArch64PAuth
+} // end namespace llvm
+
+#endif
diff --git a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
index f306021dd7..2685f2e3c8 100644
--- a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
@@ -66,8 +66,8 @@ let Predicates = [HasSME] in {
 defm BFMOPA_MPPZZ  : sme_bf16_outer_product<0b000, "bfmopa", int_aarch64_sme_mopa_wide>;
 defm BFMOPS_MPPZZ  : sme_bf16_outer_product<0b001, "bfmops", int_aarch64_sme_mops_wide>;
 
-defm FMOPA_MPPZZ_S : sme_outer_product_fp32<0b0, "fmopa", int_aarch64_sme_mopa>;
-defm FMOPS_MPPZZ_S : sme_outer_product_fp32<0b1, "fmops", int_aarch64_sme_mops>;
+defm FMOPA_MPPZZ_S : sme_outer_product_fp32<0b0, 0b00, ZPR32, "fmopa", int_aarch64_sme_mopa>;
+defm FMOPS_MPPZZ_S : sme_outer_product_fp32<0b1, 0b00, ZPR32, "fmops", int_aarch64_sme_mops>;
 }
 
 let Predicates = [HasSMEF64F64] in {
@@ -216,29 +216,29 @@ def : Pat<(AArch64_smstop (i32 svcr_op:$pstate), (i64 GPR64:$rtpstate), (i64 tim
 let Predicates = [HasSME2] in {
 defm ADD_VG2_M2ZZ_S  : sme2_dot_mla_add_sub_array_vg2_single<"add", 0b0011010, MatrixOp32, ZZ_s, ZPR4b32, nxv4i32, int_aarch64_sme_add_write_single_za_vg1x2>;
 defm ADD_VG4_M4ZZ_S  : sme2_dot_mla_add_sub_array_vg4_single<"add", 0b0111010, MatrixOp32, ZZZZ_s, ZPR4b32, nxv4i32, int_aarch64_sme_add_write_single_za_vg1x4>;
-defm ADD_VG2_M2Z2Z_S : sme2_dot_mla_add_sub_array_vg2_multi<"add", 0b011010, MatrixOp32, ZZ_s_mul_r, nxv4i32, int_aarch64_sme_add_write_za_vg1x2>;
-defm ADD_VG4_M4Z4Z_S : sme2_dot_mla_add_sub_array_vg4_multi<"add", 0b011010, MatrixOp32, ZZZZ_s_mul_r, nxv4i32, int_aarch64_sme_add_write_za_vg1x4>;
+defm ADD_VG2_M2Z2Z_S : sme2_dot_mla_add_sub_array_vg2_multi<"add", 0b0110010, MatrixOp32, ZZ_s_mul_r, nxv4i32, int_aarch64_sme_add_write_za_vg1x2>;
+defm ADD_VG4_M4Z4Z_S : sme2_dot_mla_add_sub_array_vg4_multi<"add", 0b0110010, MatrixOp32, ZZZZ_s_mul_r, nxv4i32, int_aarch64_sme_add_write_za_vg1x4>;
 
 defm ADD_VG2_2ZZ  : sme2_int_sve_destructive_vector_vg2_single<"add", 0b0110000>;
 defm ADD_VG4_4ZZ  : sme2_int_sve_destructive_vector_vg4_single<"add", 0b0110000>;
 
 defm SUB_VG2_M2ZZ_S  : sme2_dot_mla_add_sub_array_vg2_single<"sub", 0b0011011, MatrixOp32, ZZ_s, ZPR4b32, nxv4i32, int_aarch64_sme_sub_write_single_za_vg1x2>;
 defm SUB_VG4_M4ZZ_S  : sme2_dot_mla_add_sub_array_vg4_single<"sub", 0b0111011, MatrixOp32, ZZZZ_s, ZPR4b32, nxv4i32, int_aarch64_sme_sub_write_single_za_vg1x4>;
-defm SUB_VG2_M2Z2Z_S : sme2_dot_mla_add_sub_array_vg2_multi<"sub", 0b011011, MatrixOp32, ZZ_s_mul_r, nxv4i32, int_aarch64_sme_sub_write_za_vg1x2>;
-defm SUB_VG4_M4Z4Z_S : sme2_dot_mla_add_sub_array_vg4_multi<"sub", 0b011011, MatrixOp32, ZZZZ_s_mul_r, nxv4i32, int_aarch64_sme_sub_write_za_vg1x4>;
+defm SUB_VG2_M2Z2Z_S : sme2_dot_mla_add_sub_array_vg2_multi<"sub", 0b0110011, MatrixOp32, ZZ_s_mul_r, nxv4i32, int_aarch64_sme_sub_write_za_vg1x2>;
+defm SUB_VG4_M4Z4Z_S : sme2_dot_mla_add_sub_array_vg4_multi<"sub", 0b0110011, MatrixOp32, ZZZZ_s_mul_r, nxv4i32, int_aarch64_sme_sub_write_za_vg1x4>;
 
 defm FMLA_VG2_M2ZZ_S  : sme2_dot_mla_add_sub_array_vg2_single<"fmla", 0b0011000, MatrixOp32, ZZ_s, ZPR4b32, nxv4f32, int_aarch64_sme_fmla_single_vg1x2>;
 defm FMLA_VG4_M4ZZ_S  : sme2_dot_mla_add_sub_array_vg4_single<"fmla", 0b0111000, MatrixOp32, ZZZZ_s, ZPR4b32, nxv4f32, int_aarch64_sme_fmla_single_vg1x4>;
-defm FMLA_VG2_M2Z2Z_S : sme2_dot_mla_add_sub_array_vg2_multi<"fmla", 0b011000, MatrixOp32, ZZ_s_mul_r, nxv4f32, int_aarch64_sme_fmla_vg1x2>;
-defm FMLA_VG4_M4Z4Z_S : sme2_dot_mla_add_sub_array_vg4_multi<"fmla", 0b011000, MatrixOp32, ZZZZ_s_mul_r, nxv4f32, int_aarch64_sme_fmla_vg1x4>;
-defm FMLA_VG2_M2ZZI_S : sme2_multi_vec_array_vg2_index_32b<"fmla", 0b0000, ZZ_s_mul_r, ZPR4b32, nxv4f32, int_aarch64_sme_fmla_lane_vg1x2>;
+defm FMLA_VG2_M2Z2Z_S : sme2_dot_mla_add_sub_array_vg2_multi<"fmla", 0b0110000, MatrixOp32, ZZ_s_mul_r, nxv4f32, int_aarch64_sme_fmla_vg1x2>;
+defm FMLA_VG4_M4Z4Z_S : sme2_dot_mla_add_sub_array_vg4_multi<"fmla", 0b0110000, MatrixOp32, ZZZZ_s_mul_r, nxv4f32, int_aarch64_sme_fmla_vg1x4>;
+defm FMLA_VG2_M2ZZI_S : sme2_multi_vec_array_vg2_index_32b<"fmla", 0b01, 0b0000, ZZ_s_mul_r, ZPR4b32, nxv4f32, int_aarch64_sme_fmla_lane_vg1x2>;
 defm FMLA_VG4_M4ZZI_S : sme2_multi_vec_array_vg4_index_32b<"fmla", 0b0000, ZZZZ_s_mul_r, ZPR4b32, nxv4f32, int_aarch64_sme_fmla_lane_vg1x4>;
 
 defm FMLS_VG2_M2ZZ_S  : sme2_dot_mla_add_sub_array_vg2_single<"fmls", 0b0011001, MatrixOp32, ZZ_s, ZPR4b32, nxv4f32, int_aarch64_sme_fmls_single_vg1x2>;
 defm FMLS_VG4_M4ZZ_S  : sme2_dot_mla_add_sub_array_vg4_single<"fmls", 0b0111001, MatrixOp32, ZZZZ_s, ZPR4b32, nxv4f32, int_aarch64_sme_fmls_single_vg1x4>;
-defm FMLS_VG2_M2Z2Z_S : sme2_dot_mla_add_sub_array_vg2_multi<"fmls", 0b011001, MatrixOp32, ZZ_s_mul_r, nxv4f32, int_aarch64_sme_fmls_vg1x2>;
-defm FMLS_VG4_M4Z4Z_S : sme2_dot_mla_add_sub_array_vg4_multi<"fmls", 0b011001, MatrixOp32, ZZZZ_s_mul_r, nxv4f32, int_aarch64_sme_fmls_vg1x4>;
-defm FMLS_VG2_M2ZZI_S : sme2_multi_vec_array_vg2_index_32b<"fmls", 0b0010, ZZ_s_mul_r, ZPR4b32, nxv4f32, int_aarch64_sme_fmls_lane_vg1x2>;
+defm FMLS_VG2_M2Z2Z_S : sme2_dot_mla_add_sub_array_vg2_multi<"fmls", 0b0110001, MatrixOp32, ZZ_s_mul_r, nxv4f32, int_aarch64_sme_fmls_vg1x2>;
+defm FMLS_VG4_M4Z4Z_S : sme2_dot_mla_add_sub_array_vg4_multi<"fmls", 0b0110001, MatrixOp32, ZZZZ_s_mul_r, nxv4f32, int_aarch64_sme_fmls_vg1x4>;
+defm FMLS_VG2_M2ZZI_S : sme2_multi_vec_array_vg2_index_32b<"fmls", 0b01, 0b0010, ZZ_s_mul_r, ZPR4b32, nxv4f32, int_aarch64_sme_fmls_lane_vg1x2>;
 defm FMLS_VG4_M4ZZI_S : sme2_multi_vec_array_vg4_index_32b<"fmls", 0b0010, ZZZZ_s_mul_r, ZPR4b32, nxv4f32, int_aarch64_sme_fmls_lane_vg1x4>;
 
 defm ADD_VG2_M2Z_S : sme2_multivec_accum_add_sub_vg2<"add", 0b0010, MatrixOp32, ZZ_s_mul_r, nxv4i32, int_aarch64_sme_add_za32_vg1x2>;
@@ -262,37 +262,37 @@ defm FMLAL_MZZI      : sme2_mla_long_array_index<"fmlal",  0b10,   0b00, nxv8f16
 defm FMLAL_VG2_M2ZZI : sme2_fp_mla_long_array_vg2_index<"fmlal",   0b00, nxv8f16, int_aarch64_sme_fmlal_lane_vg2x2>;
 defm FMLAL_VG4_M4ZZI : sme2_fp_mla_long_array_vg4_index<"fmlal",   0b00, nxv8f16, int_aarch64_sme_fmlal_lane_vg2x4>;
 defm FMLAL_MZZ       : sme2_mla_long_array_single<"fmlal", 0b00,   0b00, nxv8f16, int_aarch64_sme_fmlal_single_vg2x1>;
-defm FMLAL_VG2_M2ZZ  : sme2_fp_mla_long_array_vg2_single<"fmlal",  0b00, nxv8f16, int_aarch64_sme_fmlal_single_vg2x2>;
-defm FMLAL_VG4_M4ZZ  : sme2_fp_mla_long_array_vg4_single<"fmlal",  0b00, nxv8f16, int_aarch64_sme_fmlal_single_vg2x4>;
-defm FMLAL_VG2_M2Z2Z : sme2_fp_mla_long_array_vg2_multi<"fmlal",   0b00, nxv8f16, int_aarch64_sme_fmlal_vg2x2>;
-defm FMLAL_VG4_M4Z4Z : sme2_fp_mla_long_array_vg4_multi<"fmlal",   0b00, nxv8f16, int_aarch64_sme_fmlal_vg2x4>;
+defm FMLAL_VG2_M2ZZ_HtoS  : sme2_fp_mla_long_array_vg2_single<"fmlal",  0b000, MatrixOp32, ZZ_h, ZPR4b16, nxv8f16, int_aarch64_sme_fmlal_single_vg2x2>;
+defm FMLAL_VG4_M4ZZ_HtoS  : sme2_fp_mla_long_array_vg4_single<"fmlal",  0b000, MatrixOp32, ZZZZ_h, ZPR4b16, nxv8f16, int_aarch64_sme_fmlal_single_vg2x4>;
+defm FMLAL_VG2_M2Z2Z_HtoS : sme2_fp_mla_long_array_vg2_multi<"fmlal",   0b000, MatrixOp32, ZZ_h_mul_r, nxv8f16, int_aarch64_sme_fmlal_vg2x2>;
+defm FMLAL_VG4_M4Z4Z_HtoS : sme2_fp_mla_long_array_vg4_multi<"fmlal",   0b000, MatrixOp32, ZZZZ_h_mul_r, nxv8f16, int_aarch64_sme_fmlal_vg2x4>;
 
 defm FMLSL_MZZI      : sme2_mla_long_array_index<"fmlsl",  0b10,   0b01, nxv8f16, int_aarch64_sme_fmlsl_lane_vg2x1>;
 defm FMLSL_VG2_M2ZZI : sme2_fp_mla_long_array_vg2_index<"fmlsl",   0b01, nxv8f16, int_aarch64_sme_fmlsl_lane_vg2x2>;
 defm FMLSL_VG4_M4ZZI : sme2_fp_mla_long_array_vg4_index<"fmlsl",   0b01, nxv8f16, int_aarch64_sme_fmlsl_lane_vg2x4>;
 defm FMLSL_MZZ       : sme2_mla_long_array_single<"fmlsl", 0b00,   0b01, nxv8f16, int_aarch64_sme_fmlsl_single_vg2x1>;
-defm FMLSL_VG2_M2ZZ  : sme2_fp_mla_long_array_vg2_single<"fmlsl",  0b01, nxv8f16, int_aarch64_sme_fmlsl_single_vg2x2>;
-defm FMLSL_VG4_M4ZZ  : sme2_fp_mla_long_array_vg4_single<"fmlsl",  0b01, nxv8f16, int_aarch64_sme_fmlsl_single_vg2x4>;
-defm FMLSL_VG2_M2Z2Z : sme2_fp_mla_long_array_vg2_multi<"fmlsl",   0b01, nxv8f16, int_aarch64_sme_fmlsl_vg2x2>;
-defm FMLSL_VG4_M4Z4Z : sme2_fp_mla_long_array_vg4_multi<"fmlsl",   0b01, nxv8f16, int_aarch64_sme_fmlsl_vg2x4>;
+defm FMLSL_VG2_M2ZZ_HtoS  : sme2_fp_mla_long_array_vg2_single<"fmlsl",  0b010,  MatrixOp32, ZZ_h, ZPR4b16, nxv8f16, int_aarch64_sme_fmlsl_single_vg2x2>;
+defm FMLSL_VG4_M4ZZ_HtoS  : sme2_fp_mla_long_array_vg4_single<"fmlsl",  0b010, MatrixOp32, ZZZZ_h, ZPR4b16, nxv8f16, int_aarch64_sme_fmlsl_single_vg2x4>;
+defm FMLSL_VG2_M2Z2Z_HtoS : sme2_fp_mla_long_array_vg2_multi<"fmlsl",   0b001, MatrixOp32, ZZ_h_mul_r, nxv8f16, int_aarch64_sme_fmlsl_vg2x2>;
+defm FMLSL_VG4_M4Z4Z_HtoS : sme2_fp_mla_long_array_vg4_multi<"fmlsl",   0b001, MatrixOp32, ZZZZ_h_mul_r, nxv8f16, int_aarch64_sme_fmlsl_vg2x4>;
 
 defm BFMLAL_MZZI      : sme2_mla_long_array_index<"bfmlal",  0b10,   0b10, nxv8bf16, int_aarch64_sme_fmlal_lane_vg2x1>;
 defm BFMLAL_VG2_M2ZZI : sme2_fp_mla_long_array_vg2_index<"bfmlal",   0b10, nxv8bf16, int_aarch64_sme_fmlal_lane_vg2x2>;
 defm BFMLAL_VG4_M4ZZI : sme2_fp_mla_long_array_vg4_index<"bfmlal",   0b10, nxv8bf16, int_aarch64_sme_fmlal_lane_vg2x4>;
 defm BFMLAL_MZZ       : sme2_mla_long_array_single<"bfmlal", 0b00,   0b10, nxv8bf16, int_aarch64_sme_fmlal_single_vg2x1>;
-defm BFMLAL_VG2_M2ZZ  : sme2_fp_mla_long_array_vg2_single<"bfmlal",  0b10, nxv8bf16, int_aarch64_sme_fmlal_single_vg2x2>;
-defm BFMLAL_VG4_M4ZZ  : sme2_fp_mla_long_array_vg4_single<"bfmlal",  0b10, nxv8bf16, int_aarch64_sme_fmlal_single_vg2x4>;
-defm BFMLAL_VG2_M2Z2Z : sme2_fp_mla_long_array_vg2_multi<"bfmlal",   0b10, nxv8bf16, int_aarch64_sme_fmlal_vg2x2>;
-defm BFMLAL_VG4_M4Z4Z : sme2_fp_mla_long_array_vg4_multi<"bfmlal",   0b10, nxv8bf16, int_aarch64_sme_fmlal_vg2x4>;
+defm BFMLAL_VG2_M2ZZ_HtoS  : sme2_fp_mla_long_array_vg2_single<"bfmlal",  0b100, MatrixOp32, ZZ_h, ZPR4b16, nxv8bf16, int_aarch64_sme_fmlal_single_vg2x2>;
+defm BFMLAL_VG4_M4ZZ_HtoS  : sme2_fp_mla_long_array_vg4_single<"bfmlal",  0b100, MatrixOp32, ZZZZ_h, ZPR4b16, nxv8bf16, int_aarch64_sme_fmlal_single_vg2x4>;
+defm BFMLAL_VG2_M2Z2Z_HtoS : sme2_fp_mla_long_array_vg2_multi<"bfmlal",   0b010, MatrixOp32, ZZ_h_mul_r, nxv8bf16, int_aarch64_sme_fmlal_vg2x2>;
+defm BFMLAL_VG4_M4Z4Z_HtoS : sme2_fp_mla_long_array_vg4_multi<"bfmlal",   0b010, MatrixOp32, ZZZZ_h_mul_r, nxv8bf16, int_aarch64_sme_fmlal_vg2x4>;
 
 defm BFMLSL_MZZI      : sme2_mla_long_array_index<"bfmlsl",  0b10,   0b11, nxv8bf16, int_aarch64_sme_fmlsl_lane_vg2x1>;
 defm BFMLSL_VG2_M2ZZI : sme2_fp_mla_long_array_vg2_index<"bfmlsl",   0b11, nxv8bf16, int_aarch64_sme_fmlsl_lane_vg2x2>;
 defm BFMLSL_VG4_M4ZZI : sme2_fp_mla_long_array_vg4_index<"bfmlsl",   0b11, nxv8bf16, int_aarch64_sme_fmlsl_lane_vg2x4>;
 defm BFMLSL_MZZ       : sme2_mla_long_array_single<"bfmlsl", 0b00,   0b11, nxv8bf16, int_aarch64_sme_fmlsl_single_vg2x1>;
-defm BFMLSL_VG2_M2ZZ  : sme2_fp_mla_long_array_vg2_single<"bfmlsl",  0b11, nxv8bf16, int_aarch64_sme_fmlsl_single_vg2x2>;
-defm BFMLSL_VG4_M4ZZ  : sme2_fp_mla_long_array_vg4_single<"bfmlsl",  0b11, nxv8bf16, int_aarch64_sme_fmlsl_single_vg2x4>;
-defm BFMLSL_VG2_M2Z2Z : sme2_fp_mla_long_array_vg2_multi<"bfmlsl",   0b11, nxv8bf16, int_aarch64_sme_fmlsl_vg2x2>;
-defm BFMLSL_VG4_M4Z4Z : sme2_fp_mla_long_array_vg4_multi<"bfmlsl",   0b11, nxv8bf16, int_aarch64_sme_fmlsl_vg2x4>;
+defm BFMLSL_VG2_M2ZZ_HtoS  : sme2_fp_mla_long_array_vg2_single<"bfmlsl",  0b110, MatrixOp32, ZZ_h, ZPR4b16, nxv8bf16, int_aarch64_sme_fmlsl_single_vg2x2>;
+defm BFMLSL_VG4_M4ZZ_HtoS  : sme2_fp_mla_long_array_vg4_single<"bfmlsl",  0b110, MatrixOp32, ZZZZ_h, ZPR4b16, nxv8bf16, int_aarch64_sme_fmlsl_single_vg2x4>;
+defm BFMLSL_VG2_M2Z2Z_HtoS : sme2_fp_mla_long_array_vg2_multi<"bfmlsl",   0b011, MatrixOp32, ZZ_h_mul_r, nxv8bf16, int_aarch64_sme_fmlsl_vg2x2>;
+defm BFMLSL_VG4_M4Z4Z_HtoS : sme2_fp_mla_long_array_vg4_multi<"bfmlsl",   0b011, MatrixOp32, ZZZZ_h_mul_r, nxv8bf16, int_aarch64_sme_fmlsl_vg2x4>;
 
 defm SMLAL_MZZI      : sme2_mla_long_array_index<"smlal", 0b11,    0b00, nxv8i16, int_aarch64_sme_smlal_lane_vg2x1>;
 defm SMLAL_VG2_M2ZZI : sme2_int_mla_long_array_vg2_index<"smlal",  0b00, int_aarch64_sme_smlal_lane_vg2x2>;
@@ -413,122 +413,122 @@ defm SCLAMP_VG4_4Z4Z : sme2_int_clamp_vector_vg4_multi<"sclamp", 0b0>;
 defm UCLAMP_VG2_2Z2Z : sme2_int_clamp_vector_vg2_multi<"uclamp", 0b1>;
 defm UCLAMP_VG4_4Z4Z : sme2_int_clamp_vector_vg4_multi<"uclamp", 0b1>;
 
-defm FDOT_VG2_M2ZZI_HtoS : sme2_multi_vec_array_vg2_index_32b<"fdot", 0b1001, ZZ_h_mul_r, ZPR4b16, nxv8f16, int_aarch64_sme_fdot_lane_za32_vg1x2>;
+defm FDOT_VG2_M2ZZI_HtoS : sme2_multi_vec_array_vg2_index_32b<"fdot", 0b01, 0b1001, ZZ_h_mul_r, ZPR4b16, nxv8f16, int_aarch64_sme_fdot_lane_za32_vg1x2>;
 defm FDOT_VG4_M4ZZI_HtoS : sme2_multi_vec_array_vg4_index_32b<"fdot", 0b1001, ZZZZ_h_mul_r, ZPR4b16, nxv8f16, int_aarch64_sme_fdot_lane_za32_vg1x4>;
 defm FDOT_VG2_M2ZZ_HtoS  : sme2_dot_mla_add_sub_array_vg2_single<"fdot", 0b0010000, MatrixOp32, ZZ_h, ZPR4b16, nxv8f16, int_aarch64_sme_fdot_single_za32_vg1x2>;
 defm FDOT_VG4_M4ZZ_HtoS  : sme2_dot_mla_add_sub_array_vg4_single<"fdot", 0b0110000, MatrixOp32, ZZZZ_h, ZPR4b16, nxv8f16, int_aarch64_sme_fdot_single_za32_vg1x4>;
-defm FDOT_VG2_M2Z2Z_HtoS : sme2_dot_mla_add_sub_array_vg2_multi<"fdot",  0b010000, MatrixOp32, ZZ_h_mul_r, nxv8f16, int_aarch64_sme_fdot_za32_vg1x2>;
-defm FDOT_VG4_M4Z4Z_HtoS : sme2_dot_mla_add_sub_array_vg4_multi<"fdot",  0b010000, MatrixOp32, ZZZZ_h_mul_r, nxv8f16, int_aarch64_sme_fdot_za32_vg1x4>;
+defm FDOT_VG2_M2Z2Z_HtoS : sme2_dot_mla_add_sub_array_vg2_multi<"fdot",  0b0100000, MatrixOp32, ZZ_h_mul_r, nxv8f16, int_aarch64_sme_fdot_za32_vg1x2>;
+defm FDOT_VG4_M4Z4Z_HtoS : sme2_dot_mla_add_sub_array_vg4_multi<"fdot",  0b0100000, MatrixOp32, ZZZZ_h_mul_r, nxv8f16, int_aarch64_sme_fdot_za32_vg1x4>;
 
-defm BFDOT_VG2_M2ZZI_HtoS : sme2_multi_vec_array_vg2_index_32b<"bfdot", 0b1011, ZZ_h_mul_r, ZPR4b16, nxv8bf16, int_aarch64_sme_fdot_lane_za32_vg1x2>;
+defm BFDOT_VG2_M2ZZI_HtoS : sme2_multi_vec_array_vg2_index_32b<"bfdot", 0b01, 0b1011, ZZ_h_mul_r, ZPR4b16, nxv8bf16, int_aarch64_sme_fdot_lane_za32_vg1x2>;
 defm BFDOT_VG4_M4ZZI_HtoS : sme2_multi_vec_array_vg4_index_32b<"bfdot", 0b1011, ZZZZ_h_mul_r, ZPR4b16, nxv8bf16, int_aarch64_sme_fdot_lane_za32_vg1x4>;
 defm BFDOT_VG2_M2ZZ_HtoS  : sme2_dot_mla_add_sub_array_vg2_single<"bfdot", 0b0010010, MatrixOp32, ZZ_h, ZPR4b16, nxv8bf16, int_aarch64_sme_fdot_single_za32_vg1x2>;
 defm BFDOT_VG4_M4ZZ_HtoS  : sme2_dot_mla_add_sub_array_vg4_single<"bfdot", 0b0110010, MatrixOp32, ZZZZ_h, ZPR4b16, nxv8bf16, int_aarch64_sme_fdot_single_za32_vg1x4>;
-defm BFDOT_VG2_M2Z2Z_HtoS : sme2_dot_mla_add_sub_array_vg2_multi<"bfdot",  0b010010, MatrixOp32, ZZ_h_mul_r, nxv8bf16, int_aarch64_sme_fdot_za32_vg1x2>;
-defm BFDOT_VG4_M4Z4Z_HtoS : sme2_dot_mla_add_sub_array_vg4_multi<"bfdot",  0b010010, MatrixOp32, ZZZZ_h_mul_r, nxv8bf16, int_aarch64_sme_fdot_za32_vg1x4>;
+defm BFDOT_VG2_M2Z2Z_HtoS : sme2_dot_mla_add_sub_array_vg2_multi<"bfdot",  0b0100010, MatrixOp32, ZZ_h_mul_r, nxv8bf16, int_aarch64_sme_fdot_za32_vg1x2>;
+defm BFDOT_VG4_M4Z4Z_HtoS : sme2_dot_mla_add_sub_array_vg4_multi<"bfdot",  0b0100010, MatrixOp32, ZZZZ_h_mul_r, nxv8bf16, int_aarch64_sme_fdot_za32_vg1x4>;
 
-defm BFVDOT_VG2_M2ZZI_HtoS : sme2_multi_vec_array_vg2_index_32b<"bfvdot", 0b0011, ZZ_h_mul_r, ZPR4b16, nxv8bf16, int_aarch64_sme_fvdot_lane_za32_vg1x2>;
+defm BFVDOT_VG2_M2ZZI_HtoS : sme2_multi_vec_array_vg2_index_32b<"bfvdot", 0b01,  0b0011, ZZ_h_mul_r, ZPR4b16, nxv8bf16, int_aarch64_sme_fvdot_lane_za32_vg1x2>;
 
-defm FVDOT_VG2_M2ZZI_HtoS : sme2_multi_vec_array_vg2_index_32b<"fvdot", 0b0001, ZZ_h_mul_r, ZPR4b16, nxv8f16, int_aarch64_sme_fvdot_lane_za32_vg1x2>;
+defm FVDOT_VG2_M2ZZI_HtoS : sme2_multi_vec_array_vg2_index_32b<"fvdot", 0b01, 0b0001, ZZ_h_mul_r, ZPR4b16, nxv8f16, int_aarch64_sme_fvdot_lane_za32_vg1x2>;
 
-defm SDOT_VG2_M2ZZI_HToS : sme2_multi_vec_array_vg2_index_32b<"sdot", 0b1000, ZZ_h_mul_r, ZPR4b16, nxv8i16, int_aarch64_sme_sdot_lane_za32_vg1x2>;
-defm SDOT_VG2_M2ZZI_BToS : sme2_multi_vec_array_vg2_index_32b<"sdot", 0b1100, ZZ_b_mul_r, ZPR4b8, nxv16i8, int_aarch64_sme_sdot_lane_za32_vg1x2>;
+defm SDOT_VG2_M2ZZI_HToS : sme2_multi_vec_array_vg2_index_32b<"sdot", 0b01,  0b1000, ZZ_h_mul_r, ZPR4b16, nxv8i16, int_aarch64_sme_sdot_lane_za32_vg1x2>;
+defm SDOT_VG2_M2ZZI_BToS : sme2_multi_vec_array_vg2_index_32b<"sdot", 0b01, 0b1100, ZZ_b_mul_r, ZPR4b8, nxv16i8, int_aarch64_sme_sdot_lane_za32_vg1x2>;
 defm SDOT_VG4_M4ZZI_HToS : sme2_multi_vec_array_vg4_index_32b<"sdot", 0b1000, ZZZZ_h_mul_r, ZPR4b16, nxv8i16, int_aarch64_sme_sdot_lane_za32_vg1x4>;
 defm SDOT_VG4_M4ZZI_BToS : sme2_multi_vec_array_vg4_index_32b<"sdot", 0b1100, ZZZZ_b_mul_r, ZPR4b8, nxv16i8, int_aarch64_sme_sdot_lane_za32_vg1x4>;
 defm SDOT_VG2_M2ZZ_HtoS  : sme2_dot_mla_add_sub_array_vg2_single<"sdot", 0b1010101, MatrixOp32, ZZ_h, ZPR4b16, nxv8i16, int_aarch64_sme_sdot_single_za32_vg1x2>;
 defm SDOT_VG4_M4ZZ_HtoS  : sme2_dot_mla_add_sub_array_vg4_single<"sdot", 0b1110101, MatrixOp32, ZZZZ_h, ZPR4b16, nxv8i16, int_aarch64_sme_sdot_single_za32_vg1x4>;
-defm SDOT_VG2_M2Z2Z_HtoS : sme2_dot_mla_add_sub_array_vg2_multi<"sdot", 0b110101, MatrixOp32, ZZ_h_mul_r, nxv8i16, int_aarch64_sme_sdot_za32_vg1x2>;
-defm SDOT_VG4_M4Z4Z_HtoS : sme2_dot_mla_add_sub_array_vg4_multi<"sdot", 0b110101, MatrixOp32, ZZZZ_h_mul_r, nxv8i16, int_aarch64_sme_sdot_za32_vg1x4>;
+defm SDOT_VG2_M2Z2Z_HtoS : sme2_dot_mla_add_sub_array_vg2_multi<"sdot", 0b1101001, MatrixOp32, ZZ_h_mul_r, nxv8i16, int_aarch64_sme_sdot_za32_vg1x2>;
+defm SDOT_VG4_M4Z4Z_HtoS : sme2_dot_mla_add_sub_array_vg4_multi<"sdot", 0b1101001, MatrixOp32, ZZZZ_h_mul_r, nxv8i16, int_aarch64_sme_sdot_za32_vg1x4>;
 defm SDOT_VG2_M2ZZ_BtoS : sme2_dot_mla_add_sub_array_vg2_single<"sdot", 0b0010100, MatrixOp32, ZZ_b, ZPR4b8, nxv16i8, int_aarch64_sme_sdot_single_za32_vg1x2>;
 defm SDOT_VG4_M4ZZ_BtoS : sme2_dot_mla_add_sub_array_vg4_single<"sdot", 0b0110100, MatrixOp32, ZZZZ_b, ZPR4b8, nxv16i8, int_aarch64_sme_sdot_single_za32_vg1x4>;
-defm SDOT_VG2_M2Z2Z_BtoS : sme2_dot_mla_add_sub_array_vg2_multi<"sdot", 0b010100, MatrixOp32, ZZ_b_mul_r, nxv16i8, int_aarch64_sme_sdot_za32_vg1x2>;
-defm SDOT_VG4_M4Z4Z_BtoS : sme2_dot_mla_add_sub_array_vg4_multi<"sdot", 0b010100, MatrixOp32, ZZZZ_b_mul_r, nxv16i8, int_aarch64_sme_sdot_za32_vg1x4>;
+defm SDOT_VG2_M2Z2Z_BtoS : sme2_dot_mla_add_sub_array_vg2_multi<"sdot", 0b0101000, MatrixOp32, ZZ_b_mul_r, nxv16i8, int_aarch64_sme_sdot_za32_vg1x2>;
+defm SDOT_VG4_M4Z4Z_BtoS : sme2_dot_mla_add_sub_array_vg4_multi<"sdot", 0b0101000, MatrixOp32, ZZZZ_b_mul_r, nxv16i8, int_aarch64_sme_sdot_za32_vg1x4>;
 
-defm SUDOT_VG2_M2ZZI_BToS : sme2_multi_vec_array_vg2_index_32b<"sudot", 0b1111, ZZ_b_mul_r, ZPR4b8, nxv16i8, int_aarch64_sme_sudot_lane_za32_vg1x2>;
+defm SUDOT_VG2_M2ZZI_BToS : sme2_multi_vec_array_vg2_index_32b<"sudot", 0b01, 0b1111, ZZ_b_mul_r, ZPR4b8, nxv16i8, int_aarch64_sme_sudot_lane_za32_vg1x2>;
 defm SUDOT_VG4_M4ZZI_BToS : sme2_multi_vec_array_vg4_index_32b<"sudot", 0b1111, ZZZZ_b_mul_r, ZPR4b8, nxv16i8, int_aarch64_sme_sudot_lane_za32_vg1x4>;
 defm SUDOT_VG2_M2ZZ_BToS  : sme2_dot_mla_add_sub_array_vg2_single<"sudot", 0b0010111, MatrixOp32, ZZ_b, ZPR4b8, nxv16i8, int_aarch64_sme_sudot_single_za32_vg1x2>;
 defm SUDOT_VG4_M4ZZ_BToS  : sme2_dot_mla_add_sub_array_vg4_single<"sudot", 0b0110111, MatrixOp32, ZZZZ_b, ZPR4b8, nxv16i8, int_aarch64_sme_sudot_single_za32_vg1x4>;
 
-defm SVDOT_VG2_M2ZZI_HtoS : sme2_multi_vec_array_vg2_index_32b<"svdot", 0b0100, ZZ_h_mul_r, ZPR4b16, nxv8i16, int_aarch64_sme_svdot_lane_za32_vg1x2>;
+defm SVDOT_VG2_M2ZZI_HtoS : sme2_multi_vec_array_vg2_index_32b<"svdot", 0b01, 0b0100, ZZ_h_mul_r, ZPR4b16, nxv8i16, int_aarch64_sme_svdot_lane_za32_vg1x2>;
 defm SVDOT_VG4_M4ZZI_BtoS : sme2_multi_vec_array_vg4_index_32b<"svdot", 0b0100, ZZZZ_b_mul_r, ZPR4b8, nxv16i8, int_aarch64_sme_svdot_lane_za32_vg1x4>;
 
 defm SUVDOT_VG4_M4ZZI_BToS : sme2_multi_vec_array_vg4_index_32b<"suvdot", 0b0111, ZZZZ_b_mul_r, ZPR4b8, nxv16i8, int_aarch64_sme_suvdot_lane_za32_vg1x4>;
 
-defm UDOT_VG2_M2ZZI_HToS : sme2_multi_vec_array_vg2_index_32b<"udot", 0b1010, ZZ_h_mul_r, ZPR4b16, nxv8i16, int_aarch64_sme_udot_lane_za32_vg1x2>;
-defm UDOT_VG2_M2ZZI_BToS : sme2_multi_vec_array_vg2_index_32b<"udot", 0b1110, ZZ_b_mul_r, ZPR4b8, nxv16i8, int_aarch64_sme_udot_lane_za32_vg1x2>;
+defm UDOT_VG2_M2ZZI_HToS : sme2_multi_vec_array_vg2_index_32b<"udot", 0b01, 0b1010, ZZ_h_mul_r, ZPR4b16, nxv8i16, int_aarch64_sme_udot_lane_za32_vg1x2>;
+defm UDOT_VG2_M2ZZI_BToS : sme2_multi_vec_array_vg2_index_32b<"udot", 0b01, 0b1110, ZZ_b_mul_r, ZPR4b8, nxv16i8, int_aarch64_sme_udot_lane_za32_vg1x2>;
 defm UDOT_VG4_M4ZZI_BtoS : sme2_multi_vec_array_vg4_index_32b<"udot", 0b1110, ZZZZ_b_mul_r, ZPR4b8, nxv16i8, int_aarch64_sme_udot_lane_za32_vg1x4>;
 defm UDOT_VG4_M4ZZI_HToS : sme2_multi_vec_array_vg4_index_32b<"udot", 0b1010, ZZZZ_h_mul_r, ZPR4b16, nxv8i16, int_aarch64_sme_udot_lane_za32_vg1x4>;
 defm UDOT_VG2_M2ZZ_HtoS  : sme2_dot_mla_add_sub_array_vg2_single<"udot", 0b1010111, MatrixOp32, ZZ_h, ZPR4b16, nxv8i16, int_aarch64_sme_udot_single_za32_vg1x2>;
 defm UDOT_VG4_M4ZZ_HtoS  : sme2_dot_mla_add_sub_array_vg4_single<"udot", 0b1110111, MatrixOp32, ZZZZ_h, ZPR4b16, nxv8i16, int_aarch64_sme_udot_single_za32_vg1x4>;
-defm UDOT_VG2_M2Z2Z_HtoS : sme2_dot_mla_add_sub_array_vg2_multi<"udot", 0b110111, MatrixOp32, ZZ_h_mul_r, nxv8i16, int_aarch64_sme_udot_za32_vg1x2>;
-defm UDOT_VG4_M4Z4Z_HtoS : sme2_dot_mla_add_sub_array_vg4_multi<"udot", 0b110111, MatrixOp32, ZZZZ_h_mul_r, nxv8i16, int_aarch64_sme_udot_za32_vg1x4>;
+defm UDOT_VG2_M2Z2Z_HtoS : sme2_dot_mla_add_sub_array_vg2_multi<"udot", 0b1101011, MatrixOp32, ZZ_h_mul_r, nxv8i16, int_aarch64_sme_udot_za32_vg1x2>;
+defm UDOT_VG4_M4Z4Z_HtoS : sme2_dot_mla_add_sub_array_vg4_multi<"udot", 0b1101011, MatrixOp32, ZZZZ_h_mul_r, nxv8i16, int_aarch64_sme_udot_za32_vg1x4>;
 defm UDOT_VG2_M2ZZ_BtoS : sme2_dot_mla_add_sub_array_vg2_single<"udot", 0b0010110, MatrixOp32, ZZ_b, ZPR4b8, nxv16i8, int_aarch64_sme_udot_single_za32_vg1x2>;
 defm UDOT_VG4_M4ZZ_BtoS : sme2_dot_mla_add_sub_array_vg4_single<"udot", 0b0110110, MatrixOp32, ZZZZ_b, ZPR4b8, nxv16i8, int_aarch64_sme_udot_single_za32_vg1x4>;
-defm UDOT_VG2_M2Z2Z_BtoS : sme2_dot_mla_add_sub_array_vg2_multi<"udot", 0b010110, MatrixOp32, ZZ_b_mul_r, nxv16i8, int_aarch64_sme_udot_za32_vg1x2>;
-defm UDOT_VG4_M4Z4Z_BtoS : sme2_dot_mla_add_sub_array_vg4_multi<"udot", 0b010110, MatrixOp32, ZZZZ_b_mul_r, nxv16i8, int_aarch64_sme_udot_za32_vg1x4>;
+defm UDOT_VG2_M2Z2Z_BtoS : sme2_dot_mla_add_sub_array_vg2_multi<"udot", 0b0101010, MatrixOp32, ZZ_b_mul_r, nxv16i8, int_aarch64_sme_udot_za32_vg1x2>;
+defm UDOT_VG4_M4Z4Z_BtoS : sme2_dot_mla_add_sub_array_vg4_multi<"udot", 0b0101010, MatrixOp32, ZZZZ_b_mul_r, nxv16i8, int_aarch64_sme_udot_za32_vg1x4>;
 
-defm USDOT_VG2_M2ZZI_BToS: sme2_multi_vec_array_vg2_index_32b<"usdot", 0b1101, ZZ_b_mul_r, ZPR4b8, nxv16i8, int_aarch64_sme_usdot_lane_za32_vg1x2>;
+defm USDOT_VG2_M2ZZI_BToS: sme2_multi_vec_array_vg2_index_32b<"usdot", 0b01, 0b1101, ZZ_b_mul_r, ZPR4b8, nxv16i8, int_aarch64_sme_usdot_lane_za32_vg1x2>;
 defm USDOT_VG4_M4ZZI_BToS: sme2_multi_vec_array_vg4_index_32b<"usdot", 0b1101, ZZZZ_b_mul_r, ZPR4b8, nxv16i8, int_aarch64_sme_usdot_lane_za32_vg1x4>;
 defm USDOT_VG2_M2ZZ_BToS : sme2_dot_mla_add_sub_array_vg2_single<"usdot",  0b0010101, MatrixOp32, ZZ_b, ZPR4b8, nxv16i8, int_aarch64_sme_usdot_single_za32_vg1x2>;
 defm USDOT_VG4_M4ZZ_BToS  : sme2_dot_mla_add_sub_array_vg4_single<"usdot", 0b0110101, MatrixOp32, ZZZZ_b, ZPR4b8, nxv16i8, int_aarch64_sme_usdot_single_za32_vg1x4>;
-defm USDOT_VG2_M2Z2Z_BToS : sme2_dot_mla_add_sub_array_vg2_multi<"usdot", 0b010101, MatrixOp32, ZZ_b_mul_r, nxv16i8, int_aarch64_sme_usdot_za32_vg1x2>;
-defm USDOT_VG4_M4Z4Z_BToS : sme2_dot_mla_add_sub_array_vg4_multi<"usdot", 0b010101, MatrixOp32, ZZZZ_b_mul_r, nxv16i8, int_aarch64_sme_usdot_za32_vg1x4>;
+defm USDOT_VG2_M2Z2Z_BToS : sme2_dot_mla_add_sub_array_vg2_multi<"usdot", 0b0101001, MatrixOp32, ZZ_b_mul_r, nxv16i8, int_aarch64_sme_usdot_za32_vg1x2>;
+defm USDOT_VG4_M4Z4Z_BToS : sme2_dot_mla_add_sub_array_vg4_multi<"usdot", 0b0101001, MatrixOp32, ZZZZ_b_mul_r, nxv16i8, int_aarch64_sme_usdot_za32_vg1x4>;
 
 defm USVDOT_VG4_M4ZZI_BToS : sme2_multi_vec_array_vg4_index_32b<"usvdot", 0b0101, ZZZZ_b_mul_r, ZPR4b8, nxv16i8, int_aarch64_sme_usvdot_lane_za32_vg1x4>;
 
-defm UVDOT_VG2_M2ZZI_HtoS : sme2_multi_vec_array_vg2_index_32b<"uvdot", 0b0110, ZZ_h_mul_r, ZPR4b16, nxv8i16, int_aarch64_sme_uvdot_lane_za32_vg1x2>;
+defm UVDOT_VG2_M2ZZI_HtoS : sme2_multi_vec_array_vg2_index_32b<"uvdot", 0b01, 0b0110, ZZ_h_mul_r, ZPR4b16, nxv8i16, int_aarch64_sme_uvdot_lane_za32_vg1x2>;
 defm UVDOT_VG4_M4ZZI_BtoS : sme2_multi_vec_array_vg4_index_32b<"uvdot", 0b0110, ZZZZ_b_mul_r, ZPR4b8, nxv16i8, int_aarch64_sme_uvdot_lane_za32_vg1x4>;
 
-defm SMLALL_MZZI_BtoS      : sme2_mla_ll_array_index_32b<"smlall", 0b000, int_aarch64_sme_smla_za32_lane_vg4x1>;
-defm SMLALL_VG2_M2ZZI_BtoS : sme2_mla_ll_array_vg2_index_32b<"smlall", 0b000, int_aarch64_sme_smla_za32_lane_vg4x2>;
-defm SMLALL_VG4_M4ZZI_BtoS : sme2_mla_ll_array_vg4_index_32b<"smlall", 0b000, int_aarch64_sme_smla_za32_lane_vg4x4>;
-defm SMLALL_MZZ_BtoS       : sme2_mla_ll_array_single<"smlall", 0b0000, MatrixOp32, ZPR8, ZPR4b8, nxv16i8, int_aarch64_sme_smla_za32_single_vg4x1>;
+defm SMLALL_MZZI_BtoS      : sme2_mla_ll_array_index_32b<"smlall", 0b00, 0b000, int_aarch64_sme_smla_za32_lane_vg4x1>;
+defm SMLALL_VG2_M2ZZI_BtoS : sme2_mla_ll_array_vg2_index_32b<"smlall", 0b00, 0b000, int_aarch64_sme_smla_za32_lane_vg4x2>;
+defm SMLALL_VG4_M4ZZI_BtoS : sme2_mla_ll_array_vg4_index_32b<"smlall", 0b00, 0b0000, int_aarch64_sme_smla_za32_lane_vg4x4>;
+defm SMLALL_MZZ_BtoS       : sme2_mla_ll_array_single<"smlall", 0b00000, MatrixOp32, ZPR8, ZPR4b8, nxv16i8, int_aarch64_sme_smla_za32_single_vg4x1>;
 defm SMLALL_VG2_M2ZZ_BtoS  : sme2_mla_ll_array_vg2_single<"smlall", 0b00000, MatrixOp32, ZZ_b, ZPR4b8, nxv16i8, int_aarch64_sme_smla_za32_single_vg4x2>;
 defm SMLALL_VG4_M4ZZ_BtoS  : sme2_mla_ll_array_vg4_single<"smlall", 0b01000, MatrixOp32, ZZZZ_b, ZPR4b8, nxv16i8, int_aarch64_sme_smla_za32_single_vg4x4>;
-defm SMLALL_VG2_M2Z2Z_BtoS : sme2_mla_ll_array_vg2_multi<"smlall", 0b0000, MatrixOp32, ZZ_b_mul_r, nxv16i8, int_aarch64_sme_smla_za32_vg4x2>;
-defm SMLALL_VG4_M4Z4Z_BtoS : sme2_mla_ll_array_vg4_multi<"smlall", 0b0000, MatrixOp32, ZZZZ_b_mul_r, nxv16i8, int_aarch64_sme_smla_za32_vg4x4>;
+defm SMLALL_VG2_M2Z2Z_BtoS : sme2_mla_ll_array_vg2_multi<"smlall", 0b00000, MatrixOp32, ZZ_b_mul_r, nxv16i8, int_aarch64_sme_smla_za32_vg4x2>;
+defm SMLALL_VG4_M4Z4Z_BtoS : sme2_mla_ll_array_vg4_multi<"smlall", 0b00000, MatrixOp32, ZZZZ_b_mul_r, nxv16i8, int_aarch64_sme_smla_za32_vg4x4>;
 
-defm USMLALL_MZZI_BtoS      : sme2_mla_ll_array_index_32b<"usmlall", 0b001, int_aarch64_sme_usmla_za32_lane_vg4x1>;
-defm USMLALL_VG2_M2ZZI_BtoS : sme2_mla_ll_array_vg2_index_32b<"usmlall", 0b100, int_aarch64_sme_usmla_za32_lane_vg4x2>;
-defm USMLALL_VG4_M4ZZI_BtoS : sme2_mla_ll_array_vg4_index_32b<"usmlall", 0b100, int_aarch64_sme_usmla_za32_lane_vg4x4>;
-defm USMLALL_MZZ_BtoS       : sme2_mla_ll_array_single<"usmlall", 0b0001, MatrixOp32, ZPR8, ZPR4b8, nxv16i8, int_aarch64_sme_usmla_za32_single_vg4x1>;
+defm USMLALL_MZZI_BtoS      : sme2_mla_ll_array_index_32b<"usmlall", 0b00, 0b001, int_aarch64_sme_usmla_za32_lane_vg4x1>;
+defm USMLALL_VG2_M2ZZI_BtoS : sme2_mla_ll_array_vg2_index_32b<"usmlall", 0b00, 0b100, int_aarch64_sme_usmla_za32_lane_vg4x2>;
+defm USMLALL_VG4_M4ZZI_BtoS : sme2_mla_ll_array_vg4_index_32b<"usmlall", 0b00, 0b0100, int_aarch64_sme_usmla_za32_lane_vg4x4>;
+defm USMLALL_MZZ_BtoS       : sme2_mla_ll_array_single<"usmlall", 0b00001, MatrixOp32, ZPR8, ZPR4b8, nxv16i8, int_aarch64_sme_usmla_za32_single_vg4x1>;
 defm USMLALL_VG2_M2ZZ_BtoS  : sme2_mla_ll_array_vg2_single<"usmlall", 0b00001, MatrixOp32, ZZ_b, ZPR4b8, nxv16i8, int_aarch64_sme_usmla_za32_single_vg4x2>;
 defm USMLALL_VG4_M4ZZ_BtoS  : sme2_mla_ll_array_vg4_single<"usmlall", 0b01001, MatrixOp32, ZZZZ_b, ZPR4b8, nxv16i8, int_aarch64_sme_usmla_za32_single_vg4x4>;
-defm USMLALL_VG2_M2Z2Z_BtoS : sme2_mla_ll_array_vg2_multi<"usmlall", 0b0001, MatrixOp32, ZZ_b_mul_r, nxv16i8, int_aarch64_sme_usmla_za32_vg4x2>;
-defm USMLALL_VG4_M4Z4Z_BtoS : sme2_mla_ll_array_vg4_multi<"usmlall", 0b0001, MatrixOp32, ZZZZ_b_mul_r, nxv16i8, int_aarch64_sme_usmla_za32_vg4x4>;
+defm USMLALL_VG2_M2Z2Z_BtoS : sme2_mla_ll_array_vg2_multi<"usmlall", 0b00001, MatrixOp32, ZZ_b_mul_r, nxv16i8, int_aarch64_sme_usmla_za32_vg4x2>;
+defm USMLALL_VG4_M4Z4Z_BtoS : sme2_mla_ll_array_vg4_multi<"usmlall", 0b00001, MatrixOp32, ZZZZ_b_mul_r, nxv16i8, int_aarch64_sme_usmla_za32_vg4x4>;
 
-defm SMLSLL_MZZI_BtoS      : sme2_mla_ll_array_index_32b<"smlsll", 0b010, int_aarch64_sme_smls_za32_lane_vg4x1>;
-defm SMLSLL_VG2_M2ZZI_BtoS : sme2_mla_ll_array_vg2_index_32b<"smlsll", 0b001, int_aarch64_sme_smls_za32_lane_vg4x2>;
-defm SMLSLL_VG4_M4ZZI_BtoS : sme2_mla_ll_array_vg4_index_32b<"smlsll", 0b001, int_aarch64_sme_smls_za32_lane_vg4x4>;
-defm SMLSLL_MZZ_BtoS       : sme2_mla_ll_array_single<"smlsll", 0b0010, MatrixOp32, ZPR8, ZPR4b8, nxv16i8, int_aarch64_sme_smls_za32_single_vg4x1>;
+defm SMLSLL_MZZI_BtoS      : sme2_mla_ll_array_index_32b<"smlsll", 0b00, 0b010, int_aarch64_sme_smls_za32_lane_vg4x1>;
+defm SMLSLL_VG2_M2ZZI_BtoS : sme2_mla_ll_array_vg2_index_32b<"smlsll", 0b00, 0b001, int_aarch64_sme_smls_za32_lane_vg4x2>;
+defm SMLSLL_VG4_M4ZZI_BtoS : sme2_mla_ll_array_vg4_index_32b<"smlsll", 0b00, 0b0001, int_aarch64_sme_smls_za32_lane_vg4x4>;
+defm SMLSLL_MZZ_BtoS       : sme2_mla_ll_array_single<"smlsll", 0b00010, MatrixOp32, ZPR8, ZPR4b8, nxv16i8, int_aarch64_sme_smls_za32_single_vg4x1>;
 defm SMLSLL_VG2_M2ZZ_BtoS  : sme2_mla_ll_array_vg2_single<"smlsll", 0b00010, MatrixOp32, ZZ_b, ZPR4b8, nxv16i8, int_aarch64_sme_smls_za32_single_vg4x2>;
 defm SMLSLL_VG4_M4ZZ_BtoS  : sme2_mla_ll_array_vg4_single<"smlsll", 0b01010, MatrixOp32, ZZZZ_b, ZPR4b8, nxv16i8, int_aarch64_sme_smls_za32_single_vg4x4>;
-defm SMLSLL_VG2_M2Z2Z_BtoS : sme2_mla_ll_array_vg2_multi<"smlsll", 0b0010, MatrixOp32, ZZ_b_mul_r, nxv16i8, int_aarch64_sme_smls_za32_vg4x2>;
-defm SMLSLL_VG4_M4Z4Z_BtoS : sme2_mla_ll_array_vg4_multi<"smlsll", 0b0010, MatrixOp32, ZZZZ_b_mul_r, nxv16i8, int_aarch64_sme_smls_za32_vg4x4>;
+defm SMLSLL_VG2_M2Z2Z_BtoS : sme2_mla_ll_array_vg2_multi<"smlsll", 0b00010, MatrixOp32, ZZ_b_mul_r, nxv16i8, int_aarch64_sme_smls_za32_vg4x2>;
+defm SMLSLL_VG4_M4Z4Z_BtoS : sme2_mla_ll_array_vg4_multi<"smlsll", 0b00010, MatrixOp32, ZZZZ_b_mul_r, nxv16i8, int_aarch64_sme_smls_za32_vg4x4>;
 
-defm UMLALL_MZZI_BtoS      : sme2_mla_ll_array_index_32b<"umlall", 0b100, int_aarch64_sme_umla_za32_lane_vg4x1>;
-defm UMLALL_VG2_M2ZZI_BtoS : sme2_mla_ll_array_vg2_index_32b<"umlall", 0b010, int_aarch64_sme_umla_za32_lane_vg4x2>;
-defm UMLALL_VG4_M4ZZI_BtoS : sme2_mla_ll_array_vg4_index_32b<"umlall", 0b010, int_aarch64_sme_umla_za32_lane_vg4x4>;
-defm UMLALL_MZZ_BtoS       : sme2_mla_ll_array_single<"umlall", 0b0100, MatrixOp32, ZPR8, ZPR4b8, nxv16i8, int_aarch64_sme_umla_za32_single_vg4x1>;
+defm UMLALL_MZZI_BtoS      : sme2_mla_ll_array_index_32b<"umlall", 0b00, 0b100, int_aarch64_sme_umla_za32_lane_vg4x1>;
+defm UMLALL_VG2_M2ZZI_BtoS : sme2_mla_ll_array_vg2_index_32b<"umlall", 0b00, 0b010, int_aarch64_sme_umla_za32_lane_vg4x2>;
+defm UMLALL_VG4_M4ZZI_BtoS : sme2_mla_ll_array_vg4_index_32b<"umlall", 0b00, 0b0010, int_aarch64_sme_umla_za32_lane_vg4x4>;
+defm UMLALL_MZZ_BtoS       : sme2_mla_ll_array_single<"umlall", 0b00100, MatrixOp32, ZPR8, ZPR4b8, nxv16i8, int_aarch64_sme_umla_za32_single_vg4x1>;
 defm UMLALL_VG2_M2ZZ_BtoS  : sme2_mla_ll_array_vg2_single<"umlall", 0b00100, MatrixOp32, ZZ_b, ZPR4b8, nxv16i8, int_aarch64_sme_umla_za32_single_vg4x2>;
 defm UMLALL_VG4_M4ZZ_BtoS  : sme2_mla_ll_array_vg4_single<"umlall", 0b01100, MatrixOp32, ZZZZ_b, ZPR4b8, nxv16i8, int_aarch64_sme_umla_za32_single_vg4x4>;
-defm UMLALL_VG2_M2Z2Z_BtoS : sme2_mla_ll_array_vg2_multi<"umlall", 0b0100, MatrixOp32, ZZ_b_mul_r, nxv16i8, int_aarch64_sme_umla_za32_vg4x2>;
-defm UMLALL_VG4_M4Z4Z_BtoS : sme2_mla_ll_array_vg4_multi<"umlall", 0b0100, MatrixOp32, ZZZZ_b_mul_r, nxv16i8, int_aarch64_sme_umla_za32_vg4x4>;
+defm UMLALL_VG2_M2Z2Z_BtoS : sme2_mla_ll_array_vg2_multi<"umlall", 0b00100, MatrixOp32, ZZ_b_mul_r, nxv16i8, int_aarch64_sme_umla_za32_vg4x2>;
+defm UMLALL_VG4_M4Z4Z_BtoS : sme2_mla_ll_array_vg4_multi<"umlall", 0b00100, MatrixOp32, ZZZZ_b_mul_r, nxv16i8, int_aarch64_sme_umla_za32_vg4x4>;
 
-defm SUMLALL_MZZI_BtoS      : sme2_mla_ll_array_index_32b<"sumlall", 0b101, int_aarch64_sme_sumla_za32_lane_vg4x1>;
-defm SUMLALL_VG2_M2ZZI_BtoS : sme2_mla_ll_array_vg2_index_32b<"sumlall", 0b110, int_aarch64_sme_sumla_za32_lane_vg4x2>;
-defm SUMLALL_VG4_M4ZZI_BtoS : sme2_mla_ll_array_vg4_index_32b<"sumlall", 0b110, int_aarch64_sme_sumla_za32_lane_vg4x4>;
+defm SUMLALL_MZZI_BtoS      : sme2_mla_ll_array_index_32b<"sumlall", 0b00, 0b101, int_aarch64_sme_sumla_za32_lane_vg4x1>;
+defm SUMLALL_VG2_M2ZZI_BtoS : sme2_mla_ll_array_vg2_index_32b<"sumlall", 0b00, 0b110, int_aarch64_sme_sumla_za32_lane_vg4x2>;
+defm SUMLALL_VG4_M4ZZI_BtoS : sme2_mla_ll_array_vg4_index_32b<"sumlall", 0b00, 0b0110, int_aarch64_sme_sumla_za32_lane_vg4x4>;
 defm SUMLALL_VG2_M2ZZ_BtoS  : sme2_mla_ll_array_vg2_single<"sumlall", 0b00101, MatrixOp32, ZZ_b, ZPR4b8, nxv16i8, int_aarch64_sme_sumla_za32_single_vg4x2>;
 defm SUMLALL_VG4_M4ZZ_BtoS  : sme2_mla_ll_array_vg4_single<"sumlall", 0b01101, MatrixOp32, ZZZZ_b, ZPR4b8, nxv16i8, int_aarch64_sme_sumla_za32_single_vg4x4>;
 
-defm UMLSLL_MZZI_BtoS      : sme2_mla_ll_array_index_32b<"umlsll", 0b110, int_aarch64_sme_umls_za32_lane_vg4x1>;
-defm UMLSLL_VG2_M2ZZI_BtoS : sme2_mla_ll_array_vg2_index_32b<"umlsll", 0b011, int_aarch64_sme_umls_za32_lane_vg4x2>;
-defm UMLSLL_VG4_M4ZZI_BtoS : sme2_mla_ll_array_vg4_index_32b<"umlsll", 0b011, int_aarch64_sme_umls_za32_lane_vg4x4>;
-defm UMLSLL_MZZ_BtoS       : sme2_mla_ll_array_single<"umlsll", 0b0110, MatrixOp32, ZPR8, ZPR4b8, nxv16i8, int_aarch64_sme_umls_za32_single_vg4x1>;
+defm UMLSLL_MZZI_BtoS      : sme2_mla_ll_array_index_32b<"umlsll", 0b00, 0b110, int_aarch64_sme_umls_za32_lane_vg4x1>;
+defm UMLSLL_VG2_M2ZZI_BtoS : sme2_mla_ll_array_vg2_index_32b<"umlsll", 0b00, 0b011, int_aarch64_sme_umls_za32_lane_vg4x2>;
+defm UMLSLL_VG4_M4ZZI_BtoS : sme2_mla_ll_array_vg4_index_32b<"umlsll", 0b00, 0b0011, int_aarch64_sme_umls_za32_lane_vg4x4>;
+defm UMLSLL_MZZ_BtoS       : sme2_mla_ll_array_single<"umlsll", 0b00110, MatrixOp32, ZPR8, ZPR4b8, nxv16i8, int_aarch64_sme_umls_za32_single_vg4x1>;
 defm UMLSLL_VG2_M2ZZ_BtoS  : sme2_mla_ll_array_vg2_single<"umlsll", 0b00110, MatrixOp32, ZZ_b, ZPR4b8, nxv16i8, int_aarch64_sme_umls_za32_single_vg4x2>;
 defm UMLSLL_VG4_M4ZZ_BtoS  : sme2_mla_ll_array_vg4_single<"umlsll", 0b01110, MatrixOp32, ZZZZ_b, ZPR4b8, nxv16i8, int_aarch64_sme_umls_za32_single_vg4x4>;
-defm UMLSLL_VG2_M2Z2Z_BtoS : sme2_mla_ll_array_vg2_multi<"umlsll", 0b0110, MatrixOp32, ZZ_b_mul_r, nxv16i8, int_aarch64_sme_umls_za32_vg4x2>;
-defm UMLSLL_VG4_M4Z4Z_BtoS : sme2_mla_ll_array_vg4_multi<"umlsll", 0b0110, MatrixOp32, ZZZZ_b_mul_r, nxv16i8, int_aarch64_sme_umls_za32_vg4x4>;
+defm UMLSLL_VG2_M2Z2Z_BtoS : sme2_mla_ll_array_vg2_multi<"umlsll", 0b00110, MatrixOp32, ZZ_b_mul_r, nxv16i8, int_aarch64_sme_umls_za32_vg4x2>;
+defm UMLSLL_VG4_M4Z4Z_BtoS : sme2_mla_ll_array_vg4_multi<"umlsll", 0b00110, MatrixOp32, ZZZZ_b_mul_r, nxv16i8, int_aarch64_sme_umls_za32_vg4x4>;
 
 defm BMOPA_MPPZZ_S : sme2_int_bmopx_tile<"bmopa", 0b100, int_aarch64_sme_bmopa_za32>;
 defm BMOPS_MPPZZ_S : sme2_int_bmopx_tile<"bmops", 0b101, int_aarch64_sme_bmops_za32>;
@@ -674,13 +674,13 @@ defm STNT1D_4Z_STRIDED_IMM : sme2_st_vector_vg4_multi_scalar_immediate<0b11, 0b1
 let Predicates = [HasSME2, HasSMEI16I64] in {
 defm ADD_VG2_M2ZZ_D  : sme2_dot_mla_add_sub_array_vg2_single<"add", 0b1011010, MatrixOp64, ZZ_d, ZPR4b64, nxv2i64, int_aarch64_sme_add_write_single_za_vg1x2>;
 defm ADD_VG4_M4ZZ_D  : sme2_dot_mla_add_sub_array_vg4_single<"add", 0b1111010, MatrixOp64, ZZZZ_d, ZPR4b64, nxv2i64, int_aarch64_sme_add_write_single_za_vg1x4>;
-defm ADD_VG2_M2Z2Z_D : sme2_dot_mla_add_sub_array_vg2_multi<"add", 0b111010, MatrixOp64, ZZ_d_mul_r, nxv2i64, int_aarch64_sme_add_write_za_vg1x2>;
-defm ADD_VG4_M4Z4Z_D : sme2_dot_mla_add_sub_array_vg4_multi<"add", 0b111010, MatrixOp64, ZZZZ_d_mul_r, nxv2i64, int_aarch64_sme_add_write_za_vg1x4>;
+defm ADD_VG2_M2Z2Z_D : sme2_dot_mla_add_sub_array_vg2_multi<"add", 0b1110010, MatrixOp64, ZZ_d_mul_r, nxv2i64, int_aarch64_sme_add_write_za_vg1x2>;
+defm ADD_VG4_M4Z4Z_D : sme2_dot_mla_add_sub_array_vg4_multi<"add", 0b1110010, MatrixOp64, ZZZZ_d_mul_r, nxv2i64, int_aarch64_sme_add_write_za_vg1x4>;
 
 defm SUB_VG2_M2ZZ_D  : sme2_dot_mla_add_sub_array_vg2_single<"sub", 0b1011011, MatrixOp64, ZZ_d, ZPR4b64, nxv2i64, int_aarch64_sme_sub_write_single_za_vg1x2>;
 defm SUB_VG4_M4ZZ_D  : sme2_dot_mla_add_sub_array_vg4_single<"sub", 0b1111011, MatrixOp64, ZZZZ_d, ZPR4b64, nxv2i64, int_aarch64_sme_sub_write_single_za_vg1x4>;
-defm SUB_VG2_M2Z2Z_D : sme2_dot_mla_add_sub_array_vg2_multi<"sub", 0b111011, MatrixOp64, ZZ_d_mul_r, nxv2i64, int_aarch64_sme_sub_write_za_vg1x2>;
-defm SUB_VG4_M4Z4Z_D : sme2_dot_mla_add_sub_array_vg4_multi<"sub", 0b111011, MatrixOp64, ZZZZ_d_mul_r, nxv2i64, int_aarch64_sme_sub_write_za_vg1x4>;
+defm SUB_VG2_M2Z2Z_D : sme2_dot_mla_add_sub_array_vg2_multi<"sub", 0b1110011, MatrixOp64, ZZ_d_mul_r, nxv2i64, int_aarch64_sme_sub_write_za_vg1x2>;
+defm SUB_VG4_M4Z4Z_D : sme2_dot_mla_add_sub_array_vg4_multi<"sub", 0b1110011, MatrixOp64, ZZZZ_d_mul_r, nxv2i64, int_aarch64_sme_sub_write_za_vg1x4>;
 
 defm ADD_VG2_M2Z_D : sme2_multivec_accum_add_sub_vg2<"add", 0b1010, MatrixOp64, ZZ_d_mul_r, nxv2i64, int_aarch64_sme_add_za64_vg1x2>;
 defm ADD_VG4_M4Z_D : sme2_multivec_accum_add_sub_vg4<"add", 0b1010, MatrixOp64, ZZZZ_d_mul_r, nxv2i64, int_aarch64_sme_add_za64_vg1x4>;
@@ -692,8 +692,8 @@ defm SDOT_VG2_M2ZZI_HtoD : sme2_multi_vec_array_vg2_index_64b<"sdot", 0b01, ZZ_h
 defm SDOT_VG4_M4ZZI_HtoD : sme2_multi_vec_array_vg4_index_64b<"sdot", 0b001, ZZZZ_h_mul_r, ZPR4b16, nxv8i16, int_aarch64_sme_sdot_lane_za64_vg1x4>;
 defm SDOT_VG2_M2ZZ_HtoD : sme2_dot_mla_add_sub_array_vg2_single<"sdot", 0b1010100, MatrixOp64, ZZ_h, ZPR4b16, nxv8i16, int_aarch64_sme_sdot_single_za64_vg1x2>;
 defm SDOT_VG4_M4ZZ_HtoD : sme2_dot_mla_add_sub_array_vg4_single<"sdot", 0b1110100, MatrixOp64, ZZZZ_h, ZPR4b16, nxv8i16, int_aarch64_sme_sdot_single_za64_vg1x4>;
-defm SDOT_VG2_M2Z2Z_HtoD : sme2_dot_mla_add_sub_array_vg2_multi<"sdot", 0b110100, MatrixOp64, ZZ_h_mul_r, nxv8i16, int_aarch64_sme_sdot_za64_vg1x2>;
-defm SDOT_VG4_M4Z4Z_HtoD : sme2_dot_mla_add_sub_array_vg4_multi<"sdot", 0b110100, MatrixOp64, ZZZZ_h_mul_r, nxv8i16, int_aarch64_sme_sdot_za64_vg1x4>;
+defm SDOT_VG2_M2Z2Z_HtoD : sme2_dot_mla_add_sub_array_vg2_multi<"sdot", 0b1101000, MatrixOp64, ZZ_h_mul_r, nxv8i16, int_aarch64_sme_sdot_za64_vg1x2>;
+defm SDOT_VG4_M4Z4Z_HtoD : sme2_dot_mla_add_sub_array_vg4_multi<"sdot", 0b1101000, MatrixOp64, ZZZZ_h_mul_r, nxv8i16, int_aarch64_sme_sdot_za64_vg1x4>;
 
 defm SVDOT_VG4_M4ZZI_HtoD : sme2_multi_vec_array_vg4_index_64b<"svdot", 0b101, ZZZZ_h_mul_r, ZPR4b16, nxv8i16, int_aarch64_sme_svdot_lane_za64_vg1x4>;
 
@@ -701,46 +701,46 @@ defm UDOT_VG2_M2ZZI_HtoD : sme2_multi_vec_array_vg2_index_64b<"udot", 0b11, ZZ_h
 defm UDOT_VG4_M4ZZI_HtoD : sme2_multi_vec_array_vg4_index_64b<"udot", 0b011, ZZZZ_h_mul_r, ZPR4b16, nxv8i16, int_aarch64_sme_udot_lane_za64_vg1x4>;
 defm UDOT_VG2_M2ZZ_HtoD : sme2_dot_mla_add_sub_array_vg2_single<"udot", 0b1010110, MatrixOp64, ZZ_h, ZPR4b16, nxv8i16, int_aarch64_sme_udot_single_za64_vg1x2>;
 defm UDOT_VG4_M4ZZ_HtoD : sme2_dot_mla_add_sub_array_vg4_single<"udot", 0b1110110, MatrixOp64, ZZZZ_h, ZPR4b16, nxv8i16, int_aarch64_sme_udot_single_za64_vg1x4>;
-defm UDOT_VG2_M2Z2Z_HtoD : sme2_dot_mla_add_sub_array_vg2_multi<"udot", 0b110110, MatrixOp64, ZZ_h_mul_r, nxv8i16, int_aarch64_sme_udot_za64_vg1x2>;
-defm UDOT_VG4_M4Z4Z_HtoD : sme2_dot_mla_add_sub_array_vg4_multi<"udot", 0b110110, MatrixOp64, ZZZZ_h_mul_r, nxv8i16, int_aarch64_sme_udot_za64_vg1x4>;
+defm UDOT_VG2_M2Z2Z_HtoD : sme2_dot_mla_add_sub_array_vg2_multi<"udot", 0b1101010, MatrixOp64, ZZ_h_mul_r, nxv8i16, int_aarch64_sme_udot_za64_vg1x2>;
+defm UDOT_VG4_M4Z4Z_HtoD : sme2_dot_mla_add_sub_array_vg4_multi<"udot", 0b1101010, MatrixOp64, ZZZZ_h_mul_r, nxv8i16, int_aarch64_sme_udot_za64_vg1x4>;
 
 defm UVDOT_VG4_M4ZZI_HtoD : sme2_multi_vec_array_vg4_index_64b<"uvdot", 0b111, ZZZZ_h_mul_r, ZPR4b16, nxv8i16, int_aarch64_sme_uvdot_lane_za64_vg1x4>;
 
 defm SMLALL_MZZI_HtoD      : sme2_mla_ll_array_index_64b<"smlall", 0b00, int_aarch64_sme_smla_za64_lane_vg4x1>;
 defm SMLALL_VG2_M2ZZI_HtoD : sme2_mla_ll_array_vg2_index_64b<"smlall", 0b00, int_aarch64_sme_smla_za64_lane_vg4x2>;
 defm SMLALL_VG4_M4ZZI_HtoD : sme2_mla_ll_array_vg4_index_64b<"smlall", 0b00, int_aarch64_sme_smla_za64_lane_vg4x4>;
-defm SMLALL_MZZ_HtoD       : sme2_mla_ll_array_single<"smlall", 0b1000, MatrixOp64, ZPR16, ZPR4b16, nxv8i16, int_aarch64_sme_smla_za64_single_vg4x1>;
+defm SMLALL_MZZ_HtoD       : sme2_mla_ll_array_single<"smlall", 0b10000, MatrixOp64, ZPR16, ZPR4b16, nxv8i16, int_aarch64_sme_smla_za64_single_vg4x1>;
 defm SMLALL_VG2_M2ZZ_HtoD  : sme2_mla_ll_array_vg2_single<"smlall", 0b10000, MatrixOp64, ZZ_h, ZPR4b16, nxv8i16, int_aarch64_sme_smla_za64_single_vg4x2>;
 defm SMLALL_VG4_M4ZZ_HtoD  : sme2_mla_ll_array_vg4_single<"smlall", 0b11000, MatrixOp64, ZZZZ_h, ZPR4b16, nxv8i16, int_aarch64_sme_smla_za64_single_vg4x4>;
-defm SMLALL_VG2_M2Z2Z_HtoD : sme2_mla_ll_array_vg2_multi<"smlall",  0b1000, MatrixOp64, ZZ_h_mul_r, nxv8i16, int_aarch64_sme_smla_za64_vg4x2>;
-defm SMLALL_VG4_M4Z4Z_HtoD : sme2_mla_ll_array_vg4_multi<"smlall",  0b1000, MatrixOp64, ZZZZ_h_mul_r, nxv8i16, int_aarch64_sme_smla_za64_vg4x4>;
+defm SMLALL_VG2_M2Z2Z_HtoD : sme2_mla_ll_array_vg2_multi<"smlall",  0b10000, MatrixOp64, ZZ_h_mul_r, nxv8i16, int_aarch64_sme_smla_za64_vg4x2>;
+defm SMLALL_VG4_M4Z4Z_HtoD : sme2_mla_ll_array_vg4_multi<"smlall",  0b10000, MatrixOp64, ZZZZ_h_mul_r, nxv8i16, int_aarch64_sme_smla_za64_vg4x4>;
 
 defm SMLSLL_MZZI_HtoD      : sme2_mla_ll_array_index_64b<"smlsll", 0b01, int_aarch64_sme_smls_za64_lane_vg4x1>;
 defm SMLSLL_VG2_M2ZZI_HtoD : sme2_mla_ll_array_vg2_index_64b<"smlsll", 0b01, int_aarch64_sme_smls_za64_lane_vg4x2>;
 defm SMLSLL_VG4_M4ZZI_HtoD : sme2_mla_ll_array_vg4_index_64b<"smlsll", 0b01, int_aarch64_sme_smls_za64_lane_vg4x4>;
-defm SMLSLL_MZZ_HtoD       : sme2_mla_ll_array_single<"smlsll", 0b1010, MatrixOp64, ZPR16, ZPR4b16, nxv8i16, int_aarch64_sme_smls_za64_single_vg4x1>;
+defm SMLSLL_MZZ_HtoD       : sme2_mla_ll_array_single<"smlsll", 0b10010, MatrixOp64, ZPR16, ZPR4b16, nxv8i16, int_aarch64_sme_smls_za64_single_vg4x1>;
 defm SMLSLL_VG2_M2ZZ_HtoD  : sme2_mla_ll_array_vg2_single<"smlsll", 0b10010, MatrixOp64, ZZ_h, ZPR4b16, nxv8i16, int_aarch64_sme_smls_za64_single_vg4x2>;
 defm SMLSLL_VG4_M4ZZ_HtoD  : sme2_mla_ll_array_vg4_single<"smlsll", 0b11010, MatrixOp64, ZZZZ_h, ZPR4b16, nxv8i16, int_aarch64_sme_smls_za64_single_vg4x4>;
-defm SMLSLL_VG2_M2Z2Z_HtoD : sme2_mla_ll_array_vg2_multi<"smlsll",  0b1010, MatrixOp64, ZZ_h_mul_r, nxv8i16, int_aarch64_sme_smls_za64_vg4x2>;
-defm SMLSLL_VG4_M4Z4Z_HtoD : sme2_mla_ll_array_vg4_multi<"smlsll",  0b1010, MatrixOp64, ZZZZ_h_mul_r, nxv8i16, int_aarch64_sme_smls_za64_vg4x4>;
+defm SMLSLL_VG2_M2Z2Z_HtoD : sme2_mla_ll_array_vg2_multi<"smlsll",  0b10010, MatrixOp64, ZZ_h_mul_r, nxv8i16, int_aarch64_sme_smls_za64_vg4x2>;
+defm SMLSLL_VG4_M4Z4Z_HtoD : sme2_mla_ll_array_vg4_multi<"smlsll",  0b10010, MatrixOp64, ZZZZ_h_mul_r, nxv8i16, int_aarch64_sme_smls_za64_vg4x4>;
 
 defm UMLALL_MZZI_HtoD      : sme2_mla_ll_array_index_64b<"umlall", 0b10, int_aarch64_sme_umla_za64_lane_vg4x1>;
 defm UMLALL_VG2_M2ZZI_HtoD : sme2_mla_ll_array_vg2_index_64b<"umlall", 0b10, int_aarch64_sme_umla_za64_lane_vg4x2>;
 defm UMLALL_VG4_M4ZZI_HtoD : sme2_mla_ll_array_vg4_index_64b<"umlall", 0b10, int_aarch64_sme_umla_za64_lane_vg4x4>;
-defm UMLALL_MZZ_HtoD       : sme2_mla_ll_array_single<"umlall", 0b1100, MatrixOp64, ZPR16, ZPR4b16, nxv8i16, int_aarch64_sme_umla_za64_single_vg4x1>;
+defm UMLALL_MZZ_HtoD       : sme2_mla_ll_array_single<"umlall", 0b10100, MatrixOp64, ZPR16, ZPR4b16, nxv8i16, int_aarch64_sme_umla_za64_single_vg4x1>;
 defm UMLALL_VG2_M2ZZ_HtoD  : sme2_mla_ll_array_vg2_single<"umlall", 0b10100, MatrixOp64, ZZ_h, ZPR4b16, nxv8i16, int_aarch64_sme_umla_za64_single_vg4x2>;
 defm UMLALL_VG4_M4ZZ_HtoD  : sme2_mla_ll_array_vg4_single<"umlall", 0b11100, MatrixOp64, ZZZZ_h, ZPR4b16, nxv8i16, int_aarch64_sme_umla_za64_single_vg4x4>;
-defm UMLALL_VG2_M2Z2Z_HtoD : sme2_mla_ll_array_vg2_multi<"umlall",  0b1100, MatrixOp64, ZZ_h_mul_r, nxv8i16, int_aarch64_sme_umla_za64_vg4x2>;
-defm UMLALL_VG4_M4Z4Z_HtoD : sme2_mla_ll_array_vg4_multi<"umlall",  0b1100, MatrixOp64, ZZZZ_h_mul_r, nxv8i16, int_aarch64_sme_umla_za64_vg4x4>;
+defm UMLALL_VG2_M2Z2Z_HtoD : sme2_mla_ll_array_vg2_multi<"umlall",  0b10100, MatrixOp64, ZZ_h_mul_r, nxv8i16, int_aarch64_sme_umla_za64_vg4x2>;
+defm UMLALL_VG4_M4Z4Z_HtoD : sme2_mla_ll_array_vg4_multi<"umlall",  0b10100, MatrixOp64, ZZZZ_h_mul_r, nxv8i16, int_aarch64_sme_umla_za64_vg4x4>;
 
 defm UMLSLL_MZZI_HtoD      : sme2_mla_ll_array_index_64b<"umlsll", 0b11, int_aarch64_sme_umls_za64_lane_vg4x1>;
 defm UMLSLL_VG2_M2ZZI_HtoD : sme2_mla_ll_array_vg2_index_64b<"umlsll", 0b11, int_aarch64_sme_umls_za64_lane_vg4x2>;
 defm UMLSLL_VG4_M4ZZI_HtoD : sme2_mla_ll_array_vg4_index_64b<"umlsll", 0b11, int_aarch64_sme_umls_za64_lane_vg4x4>;
-defm UMLSLL_MZZ_HtoD       : sme2_mla_ll_array_single<"umlsll", 0b1110, MatrixOp64, ZPR16, ZPR4b16, nxv8i16, int_aarch64_sme_umls_za64_single_vg4x1>;
+defm UMLSLL_MZZ_HtoD       : sme2_mla_ll_array_single<"umlsll", 0b10110, MatrixOp64, ZPR16, ZPR4b16, nxv8i16, int_aarch64_sme_umls_za64_single_vg4x1>;
 defm UMLSLL_VG2_M2ZZ_HtoD  : sme2_mla_ll_array_vg2_single<"umlsll", 0b10110, MatrixOp64, ZZ_h, ZPR4b16, nxv8i16, int_aarch64_sme_umls_za64_single_vg4x2>;
 defm UMLSLL_VG4_M4ZZ_HtoD  : sme2_mla_ll_array_vg4_single<"umlsll", 0b11110, MatrixOp64, ZZZZ_h, ZPR4b16, nxv8i16, int_aarch64_sme_umls_za64_single_vg4x4>;
-defm UMLSLL_VG2_M2Z2Z_HtoD : sme2_mla_ll_array_vg2_multi<"umlsll",  0b1110, MatrixOp64, ZZ_h_mul_r, nxv8i16, int_aarch64_sme_umls_za64_vg4x2>;
-defm UMLSLL_VG4_M4Z4Z_HtoD : sme2_mla_ll_array_vg4_multi<"umlsll",  0b1110, MatrixOp64, ZZZZ_h_mul_r, nxv8i16, int_aarch64_sme_umls_za64_vg4x4>;
+defm UMLSLL_VG2_M2Z2Z_HtoD : sme2_mla_ll_array_vg2_multi<"umlsll",  0b10110, MatrixOp64, ZZ_h_mul_r, nxv8i16, int_aarch64_sme_umls_za64_vg4x2>;
+defm UMLSLL_VG4_M4Z4Z_HtoD : sme2_mla_ll_array_vg4_multi<"umlsll",  0b10110, MatrixOp64, ZZZZ_h_mul_r, nxv8i16, int_aarch64_sme_umls_za64_vg4x4>;
 }
 
 let Predicates = [HasSME2, HasSMEF64F64] in {
@@ -748,15 +748,15 @@ defm FMLA_VG2_M2ZZI_D : sme2_multi_vec_array_vg2_index_64b<"fmla", 0b00, ZZ_d_mu
 defm FMLA_VG4_M4ZZI_D : sme2_multi_vec_array_vg4_index_64b<"fmla", 0b000, ZZZZ_d_mul_r, ZPR4b64, nxv2f64, int_aarch64_sme_fmla_lane_vg1x4>;
 defm FMLA_VG2_M2ZZ_D  : sme2_dot_mla_add_sub_array_vg2_single<"fmla", 0b1011000, MatrixOp64, ZZ_d, ZPR4b64, nxv2f64, int_aarch64_sme_fmla_single_vg1x2>;
 defm FMLA_VG4_M4ZZ_D  : sme2_dot_mla_add_sub_array_vg4_single<"fmla", 0b1111000, MatrixOp64, ZZZZ_d, ZPR4b64, nxv2f64, int_aarch64_sme_fmla_single_vg1x4>;
-defm FMLA_VG2_M2Z2Z_D : sme2_dot_mla_add_sub_array_vg2_multi<"fmla", 0b111000, MatrixOp64, ZZ_d_mul_r, nxv2f64, int_aarch64_sme_fmla_vg1x2>;
-defm FMLA_VG4_M4Z4Z_D : sme2_dot_mla_add_sub_array_vg4_multi<"fmla", 0b111000, MatrixOp64, ZZZZ_d_mul_r, nxv2f64, int_aarch64_sme_fmla_vg1x4>;
+defm FMLA_VG2_M2Z2Z_D : sme2_dot_mla_add_sub_array_vg2_multi<"fmla", 0b1110000, MatrixOp64, ZZ_d_mul_r, nxv2f64, int_aarch64_sme_fmla_vg1x2>;
+defm FMLA_VG4_M4Z4Z_D : sme2_dot_mla_add_sub_array_vg4_multi<"fmla", 0b1110000, MatrixOp64, ZZZZ_d_mul_r, nxv2f64, int_aarch64_sme_fmla_vg1x4>;
 
 defm FMLS_VG2_M2ZZI_D : sme2_multi_vec_array_vg2_index_64b<"fmls", 0b10, ZZ_d_mul_r, ZPR4b64, nxv2f64, int_aarch64_sme_fmls_lane_vg1x2>;
 defm FMLS_VG4_M4ZZI_D : sme2_multi_vec_array_vg4_index_64b<"fmls", 0b010, ZZZZ_d_mul_r, ZPR4b64, nxv2f64, int_aarch64_sme_fmls_lane_vg1x4>;
 defm FMLS_VG2_M2ZZ_D  : sme2_dot_mla_add_sub_array_vg2_single<"fmls", 0b1011001, MatrixOp64, ZZ_d, ZPR4b64, nxv2f64, int_aarch64_sme_fmls_single_vg1x2>;
 defm FMLS_VG4_M4ZZ_D  : sme2_dot_mla_add_sub_array_vg4_single<"fmls", 0b1111001, MatrixOp64, ZZZZ_d, ZPR4b64, nxv2f64, int_aarch64_sme_fmls_single_vg1x4>;
-defm FMLS_VG2_M2Z2Z_D : sme2_dot_mla_add_sub_array_vg2_multi<"fmls", 0b111001, MatrixOp64, ZZ_d_mul_r, nxv2f64, int_aarch64_sme_fmls_vg1x2>;
-defm FMLS_VG4_M4Z4Z_D : sme2_dot_mla_add_sub_array_vg4_multi<"fmls", 0b111001, MatrixOp64, ZZZZ_d_mul_r, nxv2f64, int_aarch64_sme_fmls_vg1x4>;
+defm FMLS_VG2_M2Z2Z_D : sme2_dot_mla_add_sub_array_vg2_multi<"fmls", 0b1110001, MatrixOp64, ZZ_d_mul_r, nxv2f64, int_aarch64_sme_fmls_vg1x2>;
+defm FMLS_VG4_M4Z4Z_D : sme2_dot_mla_add_sub_array_vg4_multi<"fmls", 0b1110001, MatrixOp64, ZZZZ_d_mul_r, nxv2f64, int_aarch64_sme_fmls_vg1x4>;
 
 defm FADD_VG2_M2Z_D : sme2_multivec_accum_add_sub_vg2<"fadd", 0b1000, MatrixOp64, ZZ_d_mul_r, nxv2f64, int_aarch64_sme_add_za64_vg1x2>;
 defm FADD_VG4_M4Z_D : sme2_multivec_accum_add_sub_vg4<"fadd", 0b1000, MatrixOp64, ZZZZ_d_mul_r, nxv2f64, int_aarch64_sme_add_za64_vg1x4>;
@@ -787,25 +787,25 @@ defm FADD_VG4_M4Z_H : sme2_multivec_accum_add_sub_vg4<"fadd", 0b0100, MatrixOp16
 defm FSUB_VG2_M2Z_H : sme2_multivec_accum_add_sub_vg2<"fsub", 0b0101, MatrixOp16, ZZ_h_mul_r, nxv8f16, null_frag>;
 defm FSUB_VG4_M4Z_H : sme2_multivec_accum_add_sub_vg4<"fsub", 0b0101, MatrixOp16, ZZZZ_h_mul_r, nxv8f16, null_frag>;
 
-defm FMLA_VG2_M2ZZI_H : sme2p1_multi_vec_array_vg2_index_16b<"fmla", 0b00>;
-defm FMLA_VG4_M4ZZI_H : sme2p1_multi_vec_array_vg4_index_16b<"fmla", 0b00>;
+defm FMLA_VG2_M2ZZI_H : sme2p1_multi_vec_array_vg2_index_16b<"fmla", 0b00, 0b100, ZZ_h_mul_r, ZPR4b16>;
+defm FMLA_VG4_M4ZZI_H : sme2p1_multi_vec_array_vg4_index_16b<"fmla", 0b000, ZZZZ_h_mul_r, ZPR4b16>;
 defm FMLA_VG2_M2ZZ_H :  sme2_dot_mla_add_sub_array_vg24_single<"fmla", 0b0011100, MatrixOp16, ZZ_h, ZPR4b16>;
 defm FMLA_VG4_M4ZZ_H :  sme2_dot_mla_add_sub_array_vg24_single<"fmla", 0b0111100, MatrixOp16, ZZZZ_h, ZPR4b16>;
-defm FMLA_VG2_M2Z4Z_H : sme2_dot_mla_add_sub_array_vg2_multi<"fmla", 0b010001, MatrixOp16, ZZ_h_mul_r, nxv8f16, null_frag>;
-defm FMLA_VG4_M4Z4Z_H : sme2_dot_mla_add_sub_array_vg4_multi<"fmla", 0b010001, MatrixOp16, ZZZZ_h_mul_r, nxv8f16, null_frag>;
+defm FMLA_VG2_M2Z4Z_H : sme2_dot_mla_add_sub_array_vg2_multi<"fmla", 0b0100001, MatrixOp16, ZZ_h_mul_r, nxv8f16, null_frag>;
+defm FMLA_VG4_M4Z4Z_H : sme2_dot_mla_add_sub_array_vg4_multi<"fmla", 0b0100001, MatrixOp16, ZZZZ_h_mul_r, nxv8f16, null_frag>;
 
-defm FMLS_VG2_M2ZZI_H : sme2p1_multi_vec_array_vg2_index_16b<"fmls", 0b01>;
-defm FMLS_VG4_M4ZZI_H : sme2p1_multi_vec_array_vg4_index_16b<"fmls", 0b01>;
+defm FMLS_VG2_M2ZZI_H : sme2p1_multi_vec_array_vg2_index_16b<"fmls", 0b00, 0b101, ZZ_h_mul_r, ZPR4b16>;
+defm FMLS_VG4_M4ZZI_H : sme2p1_multi_vec_array_vg4_index_16b<"fmls", 0b001, ZZZZ_h_mul_r, ZPR4b16>;
 defm FMLS_VG2_M2ZZ_H :  sme2_dot_mla_add_sub_array_vg24_single<"fmls", 0b0011101, MatrixOp16, ZZ_h, ZPR4b16>;
 defm FMLS_VG4_M4ZZ_H :  sme2_dot_mla_add_sub_array_vg24_single<"fmls", 0b0111101, MatrixOp16, ZZZZ_h, ZPR4b16>;
-defm FMLS_VG2_M2Z2Z_H : sme2_dot_mla_add_sub_array_vg2_multi<"fmls", 0b010011, MatrixOp16, ZZ_h_mul_r, nxv8f16, null_frag>;
-defm FMLS_VG4_M4Z2Z_H : sme2_dot_mla_add_sub_array_vg4_multi<"fmls", 0b010011, MatrixOp16, ZZZZ_h_mul_r, nxv8f16, null_frag>;
+defm FMLS_VG2_M2Z2Z_H : sme2_dot_mla_add_sub_array_vg2_multi<"fmls", 0b0100011, MatrixOp16, ZZ_h_mul_r, nxv8f16, null_frag>;
+defm FMLS_VG4_M4Z2Z_H : sme2_dot_mla_add_sub_array_vg4_multi<"fmls", 0b0100011, MatrixOp16, ZZZZ_h_mul_r, nxv8f16, null_frag>;
 
 defm FCVT_2ZZ_H  : sme2p1_fp_cvt_vector_vg2_single<"fcvt", 0b0>;
 defm FCVTL_2ZZ_H : sme2p1_fp_cvt_vector_vg2_single<"fcvtl", 0b1>;
 
-defm FMOPA_MPPZZ_H : sme2p1_fmop_tile_fp16<"fmopa", 0b0, 0b0>;
-defm FMOPS_MPPZZ_H : sme2p1_fmop_tile_fp16<"fmops", 0b0, 0b1>;
+defm FMOPA_MPPZZ_H : sme2p1_fmop_tile_fp16<"fmopa", 0b0, 0b0, 0b11, ZPR16>;
+defm FMOPS_MPPZZ_H : sme2p1_fmop_tile_fp16<"fmops", 0b0, 0b1, 0b11, ZPR16>;
 }
 
 let Predicates = [HasSME2p1, HasB16B16] in {
@@ -814,19 +814,19 @@ defm BFADD_VG4_M4Z_H : sme2_multivec_accum_add_sub_vg4<"bfadd", 0b1100, MatrixOp
 defm BFSUB_VG2_M2Z_H : sme2_multivec_accum_add_sub_vg2<"bfsub", 0b1101, MatrixOp16, ZZ_h_mul_r,  nxv8bf16, null_frag>;
 defm BFSUB_VG4_M4Z_H : sme2_multivec_accum_add_sub_vg4<"bfsub", 0b1101, MatrixOp16, ZZZZ_h_mul_r,  nxv8bf16, null_frag>;
 
-defm BFMLA_VG2_M2ZZI : sme2p1_multi_vec_array_vg2_index_16b<"bfmla", 0b10>;
-defm BFMLA_VG4_M4ZZI : sme2p1_multi_vec_array_vg4_index_16b<"bfmla", 0b10>;
+defm BFMLA_VG2_M2ZZI : sme2p1_multi_vec_array_vg2_index_16b<"bfmla", 0b00, 0b110, ZZ_h_mul_r, ZPR4b16>;
+defm BFMLA_VG4_M4ZZI : sme2p1_multi_vec_array_vg4_index_16b<"bfmla", 0b010, ZZZZ_h_mul_r, ZPR4b16>;
 defm BFMLA_VG2_M2ZZ : sme2_dot_mla_add_sub_array_vg24_single<"bfmla", 0b1011100, MatrixOp16, ZZ_h, ZPR4b16>;
 defm BFMLA_VG4_M4ZZ : sme2_dot_mla_add_sub_array_vg24_single<"bfmla", 0b1111100, MatrixOp16, ZZZZ_h, ZPR4b16>;
-defm BFMLA_VG2_M2Z2Z : sme2_dot_mla_add_sub_array_vg2_multi<"bfmla", 0b110001, MatrixOp16, ZZ_h_mul_r, nxv8bf16, null_frag>;
-defm BFMLA_VG4_M4Z4Z : sme2_dot_mla_add_sub_array_vg4_multi<"bfmla", 0b110001, MatrixOp16, ZZZZ_h_mul_r, nxv8bf16, null_frag>;
+defm BFMLA_VG2_M2Z2Z : sme2_dot_mla_add_sub_array_vg2_multi<"bfmla", 0b1100001, MatrixOp16, ZZ_h_mul_r, nxv8bf16, null_frag>;
+defm BFMLA_VG4_M4Z4Z : sme2_dot_mla_add_sub_array_vg4_multi<"bfmla", 0b1100001, MatrixOp16, ZZZZ_h_mul_r, nxv8bf16, null_frag>;
 
-defm BFMLS_VG2_M2ZZI : sme2p1_multi_vec_array_vg2_index_16b<"bfmls", 0b11>;
-defm BFMLS_VG4_M4ZZI : sme2p1_multi_vec_array_vg4_index_16b<"bfmls", 0b11>;
+defm BFMLS_VG2_M2ZZI : sme2p1_multi_vec_array_vg2_index_16b<"bfmls", 0b00, 0b111, ZZ_h_mul_r, ZPR4b16>;
+defm BFMLS_VG4_M4ZZI : sme2p1_multi_vec_array_vg4_index_16b<"bfmls", 0b011, ZZZZ_h_mul_r, ZPR4b16>;
 defm BFMLS_VG2_M2ZZ : sme2_dot_mla_add_sub_array_vg24_single<"bfmls", 0b1011101, MatrixOp16, ZZ_h, ZPR4b16>;
 defm BFMLS_VG4_M4ZZ : sme2_dot_mla_add_sub_array_vg24_single<"bfmls", 0b1111101, MatrixOp16, ZZZZ_h, ZPR4b16>;
-defm BFMLS_VG2_M2Z2Z : sme2_dot_mla_add_sub_array_vg2_multi<"bfmls", 0b110011, MatrixOp16, ZZ_h_mul_r, nxv8bf16, null_frag>;
-defm BFMLS_VG4_M4Z4Z : sme2_dot_mla_add_sub_array_vg4_multi<"bfmls", 0b110011, MatrixOp16, ZZZZ_h_mul_r, nxv8bf16, null_frag>;
+defm BFMLS_VG2_M2Z2Z : sme2_dot_mla_add_sub_array_vg2_multi<"bfmls", 0b1100011, MatrixOp16, ZZ_h_mul_r, nxv8bf16, null_frag>;
+defm BFMLS_VG4_M4Z4Z : sme2_dot_mla_add_sub_array_vg4_multi<"bfmls", 0b1100011, MatrixOp16, ZZZZ_h_mul_r, nxv8bf16, null_frag>;
 
 
 defm BFMAX_VG2_2ZZ  : sme2p1_bf_max_min_vector_vg2_single<"bfmax", 0b0010000>;
@@ -852,6 +852,6 @@ defm BFMINNM_VG4_4Z2Z : sme2p1_bf_max_min_vector_vg4_multi<"bfminnm",  0b0010011
 defm BFCLAMP_VG2_2ZZZ: sme2p1_bfclamp_vector_vg2_multi<"bfclamp">;
 defm BFCLAMP_VG4_4ZZZ: sme2p1_bfclamp_vector_vg4_multi<"bfclamp">;
 
-defm BFMOPA_MPPZZ_H : sme2p1_fmop_tile_fp16<"bfmopa", 0b1, 0b0>;
-defm BFMOPS_MPPZZ_H : sme2p1_fmop_tile_fp16<"bfmops", 0b1, 0b1>;
+defm BFMOPA_MPPZZ_H : sme2p1_fmop_tile_fp16<"bfmopa", 0b1, 0b0, 0b11, ZPR16>;
+defm BFMOPS_MPPZZ_H : sme2p1_fmop_tile_fp16<"bfmops", 0b1, 0b1, 0b11, ZPR16>;
 }
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index a7a64c6b20..d599ac4689 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -2204,8 +2204,8 @@ let Predicates = [HasSVEorSME] in {
 } // End HasSVEorSME
 
 let Predicates = [HasBF16, HasSVEorSME] in {
-  defm BFDOT_ZZZ    : sve_float_dot<0b1, "bfdot", nxv8bf16, int_aarch64_sve_bfdot>;
-  defm BFDOT_ZZI    : sve_float_dot_indexed<0b1, "bfdot", nxv8bf16, int_aarch64_sve_bfdot_lane_v2>;
+  defm BFDOT_ZZZ    : sve_float_dot<0b1, 0b0, ZPR32, ZPR16, "bfdot", nxv8bf16, int_aarch64_sve_bfdot>;
+  defm BFDOT_ZZI    : sve_float_dot_indexed<0b1, 0b00, ZPR16, ZPR3b16, "bfdot", nxv8bf16, int_aarch64_sve_bfdot_lane_v2>;
 } // End HasBF16, HasSVEorSME
 
 let Predicates = [HasBF16, HasSVE] in {
@@ -3753,12 +3753,14 @@ defm PSEL_PPPRI : sve2_int_perm_sel_p<"psel", int_aarch64_sve_psel>;
 
 let Predicates = [HasSVE2p1_or_HasSME2] in {
 defm FCLAMP_ZZZ : sve2p1_fclamp<"fclamp", int_aarch64_sve_fclamp>;
-defm FDOT_ZZZ_S  : sve_float_dot<0b0, "fdot", nxv8f16, int_aarch64_sve_fdot_x2>;
-defm FDOT_ZZZI_S : sve_float_dot_indexed<0b0, "fdot", nxv8f16, int_aarch64_sve_fdot_lane_x2>;
-def BFMLSLB_ZZZ_S : sve2_fp_mla_long<0b110, "bfmlslb">;
-def BFMLSLT_ZZZ_S : sve2_fp_mla_long<0b111, "bfmlslt">;
-def BFMLSLB_ZZZI_S : sve2_fp_mla_long_by_indexed_elem<0b110, "bfmlslb">;
-def BFMLSLT_ZZZI_S : sve2_fp_mla_long_by_indexed_elem<0b111, "bfmlslt">;
+
+defm FDOT_ZZZ_S  : sve_float_dot<0b0, 0b0, ZPR32, ZPR16, "fdot", nxv8f16, int_aarch64_sve_fdot_x2>;
+defm FDOT_ZZZI_S : sve_float_dot_indexed<0b0, 0b00, ZPR16, ZPR3b16, "fdot", nxv8f16, int_aarch64_sve_fdot_lane_x2>;
+
+defm BFMLSLB_ZZZ_S : sve2_fp_mla_long<0b110, "bfmlslb", nxv4f32, nxv8bf16, int_aarch64_sve_bfmlslb>;
+defm BFMLSLT_ZZZ_S : sve2_fp_mla_long<0b111, "bfmlslt", nxv4f32, nxv8bf16, int_aarch64_sve_bfmlslt>;
+defm BFMLSLB_ZZZI_S : sve2_fp_mla_long_by_indexed_elem<0b110, "bfmlslb", nxv4f32, nxv8bf16, int_aarch64_sve_bfmlslb_lane>;
+defm BFMLSLT_ZZZI_S : sve2_fp_mla_long_by_indexed_elem<0b111, "bfmlslt", nxv4f32, nxv8bf16, int_aarch64_sve_bfmlslt_lane>;
 
 defm SDOT_ZZZ_HtoS  : sve2p1_two_way_dot_vv<"sdot", 0b0, int_aarch64_sve_sdot_x2>;
 defm UDOT_ZZZ_HtoS  : sve2p1_two_way_dot_vv<"udot", 0b1, int_aarch64_sve_udot_x2>;
diff --git a/llvm/lib/Target/AArch64/AArch64SchedA57.td b/llvm/lib/Target/AArch64/AArch64SchedA57.td
index 8ce2293740..277ec772cf 100644
--- a/llvm/lib/Target/AArch64/AArch64SchedA57.td
+++ b/llvm/lib/Target/AArch64/AArch64SchedA57.td
@@ -183,165 +183,165 @@ def : InstRW<[A57Write_3cyc_1W], (instregex "^CRC32")>;
 // -----------------------------------------------------------------------------
 
 def : InstRW<[A57Write_8cyc_1L_1V],           (instregex "LD1i(8|16|32)$")>;
-def : InstRW<[A57Write_8cyc_1L_1V, WriteAdr], (instregex "LD1i(8|16|32)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_8cyc_1L_1V], (instregex "LD1i(8|16|32)_POST$")>;
 def : InstRW<[A57Write_5cyc_1L],            (instregex "LD1i(64)$")>;
-def : InstRW<[A57Write_5cyc_1L, WriteAdr],  (instregex "LD1i(64)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_5cyc_1L],  (instregex "LD1i(64)_POST$")>;
 
 def : InstRW<[A57Write_8cyc_1L_1V],           (instregex "LD1Rv(8b|4h|2s)$")>;
-def : InstRW<[A57Write_8cyc_1L_1V, WriteAdr], (instregex "LD1Rv(8b|4h|2s)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_8cyc_1L_1V], (instregex "LD1Rv(8b|4h|2s)_POST$")>;
 def : InstRW<[A57Write_5cyc_1L],            (instregex "LD1Rv(1d)$")>;
-def : InstRW<[A57Write_5cyc_1L, WriteAdr],  (instregex "LD1Rv(1d)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_5cyc_1L],  (instregex "LD1Rv(1d)_POST$")>;
 def : InstRW<[A57Write_8cyc_1L_1V],           (instregex "LD1Rv(16b|8h|4s|2d)$")>;
-def : InstRW<[A57Write_8cyc_1L_1V, WriteAdr], (instregex "LD1Rv(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_8cyc_1L_1V], (instregex "LD1Rv(16b|8h|4s|2d)_POST$")>;
 
 def : InstRW<[A57Write_5cyc_1L],              (instregex "LD1Onev(8b|4h|2s|1d)$")>;
-def : InstRW<[A57Write_5cyc_1L, WriteAdr],    (instregex "LD1Onev(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_5cyc_1L],    (instregex "LD1Onev(8b|4h|2s|1d)_POST$")>;
 def : InstRW<[A57Write_5cyc_1L],              (instregex "LD1Onev(16b|8h|4s|2d)$")>;
-def : InstRW<[A57Write_5cyc_1L, WriteAdr],    (instregex "LD1Onev(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_5cyc_1L],    (instregex "LD1Onev(16b|8h|4s|2d)_POST$")>;
 def : InstRW<[A57Write_5cyc_1L],              (instregex "LD1Twov(8b|4h|2s|1d)$")>;
-def : InstRW<[A57Write_5cyc_1L, WriteAdr],    (instregex "LD1Twov(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_5cyc_1L],    (instregex "LD1Twov(8b|4h|2s|1d)_POST$")>;
 def : InstRW<[A57Write_6cyc_2L],             (instregex "LD1Twov(16b|8h|4s|2d)$")>;
-def : InstRW<[A57Write_6cyc_2L, WriteAdr],   (instregex "LD1Twov(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_6cyc_2L],   (instregex "LD1Twov(16b|8h|4s|2d)_POST$")>;
 def : InstRW<[A57Write_6cyc_2L],             (instregex "LD1Threev(8b|4h|2s|1d)$")>;
-def : InstRW<[A57Write_6cyc_2L, WriteAdr],   (instregex "LD1Threev(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_6cyc_2L],   (instregex "LD1Threev(8b|4h|2s|1d)_POST$")>;
 def : InstRW<[A57Write_7cyc_3L],            (instregex "LD1Threev(16b|8h|4s|2d)$")>;
-def : InstRW<[A57Write_7cyc_3L, WriteAdr],  (instregex "LD1Threev(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_7cyc_3L],  (instregex "LD1Threev(16b|8h|4s|2d)_POST$")>;
 def : InstRW<[A57Write_6cyc_2L],             (instregex "LD1Fourv(8b|4h|2s|1d)$")>;
-def : InstRW<[A57Write_6cyc_2L, WriteAdr],   (instregex "LD1Fourv(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_6cyc_2L],   (instregex "LD1Fourv(8b|4h|2s|1d)_POST$")>;
 def : InstRW<[A57Write_8cyc_4L],           (instregex "LD1Fourv(16b|8h|4s|2d)$")>;
-def : InstRW<[A57Write_8cyc_4L, WriteAdr], (instregex "LD1Fourv(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_8cyc_4L], (instregex "LD1Fourv(16b|8h|4s|2d)_POST$")>;
 
 def : InstRW<[A57Write_8cyc_1L_2V],           (instregex "LD2i(8|16)$")>;
-def : InstRW<[A57Write_8cyc_1L_2V, WriteAdr], (instregex "LD2i(8|16)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_8cyc_1L_2V], (instregex "LD2i(8|16)_POST$")>;
 def : InstRW<[A57Write_6cyc_2L],            (instregex "LD2i(32)$")>;
-def : InstRW<[A57Write_6cyc_2L, WriteAdr],  (instregex "LD2i(32)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_6cyc_2L],  (instregex "LD2i(32)_POST$")>;
 def : InstRW<[A57Write_8cyc_1L_1V],            (instregex "LD2i(64)$")>;
-def : InstRW<[A57Write_8cyc_1L_1V, WriteAdr],  (instregex "LD2i(64)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_8cyc_1L_1V],  (instregex "LD2i(64)_POST$")>;
 
 def : InstRW<[A57Write_8cyc_1L_1V],            (instregex "LD2Rv(8b|4h|2s)$")>;
-def : InstRW<[A57Write_8cyc_1L_1V, WriteAdr],  (instregex "LD2Rv(8b|4h|2s)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_8cyc_1L_1V],  (instregex "LD2Rv(8b|4h|2s)_POST$")>;
 def : InstRW<[A57Write_5cyc_1L],             (instregex "LD2Rv(1d)$")>;
-def : InstRW<[A57Write_5cyc_1L, WriteAdr],   (instregex "LD2Rv(1d)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_5cyc_1L],   (instregex "LD2Rv(1d)_POST$")>;
 def : InstRW<[A57Write_8cyc_1L_2V],           (instregex "LD2Rv(16b|8h|4s|2d)$")>;
-def : InstRW<[A57Write_8cyc_1L_2V, WriteAdr], (instregex "LD2Rv(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_8cyc_1L_2V], (instregex "LD2Rv(16b|8h|4s|2d)_POST$")>;
 
 def : InstRW<[A57Write_8cyc_1L_1V],             (instregex "LD2Twov(8b|4h|2s)$")>;
-def : InstRW<[A57Write_8cyc_1L_1V, WriteAdr],   (instregex "LD2Twov(8b|4h|2s)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_8cyc_1L_1V],   (instregex "LD2Twov(8b|4h|2s)_POST$")>;
 def : InstRW<[A57Write_9cyc_2L_2V],           (instregex "LD2Twov(16b|8h|4s)$")>;
-def : InstRW<[A57Write_9cyc_2L_2V, WriteAdr], (instregex "LD2Twov(16b|8h|4s)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_9cyc_2L_2V], (instregex "LD2Twov(16b|8h|4s)_POST$")>;
 def : InstRW<[A57Write_6cyc_2L],             (instregex "LD2Twov(2d)$")>;
-def : InstRW<[A57Write_6cyc_2L, WriteAdr],   (instregex "LD2Twov(2d)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_6cyc_2L],   (instregex "LD2Twov(2d)_POST$")>;
 
 def : InstRW<[A57Write_9cyc_1L_3V],           (instregex "LD3i(8|16)$")>;
-def : InstRW<[A57Write_9cyc_1L_3V, WriteAdr], (instregex "LD3i(8|16)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_9cyc_1L_3V], (instregex "LD3i(8|16)_POST$")>;
 def : InstRW<[A57Write_8cyc_1L_2V],            (instregex "LD3i(32)$")>;
-def : InstRW<[A57Write_8cyc_1L_2V, WriteAdr],  (instregex "LD3i(32)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_8cyc_1L_2V],  (instregex "LD3i(32)_POST$")>;
 def : InstRW<[A57Write_6cyc_2L],             (instregex "LD3i(64)$")>;
-def : InstRW<[A57Write_6cyc_2L, WriteAdr],   (instregex "LD3i(64)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_6cyc_2L],   (instregex "LD3i(64)_POST$")>;
 
 def : InstRW<[A57Write_8cyc_1L_2V],             (instregex "LD3Rv(8b|4h|2s)$")>;
-def : InstRW<[A57Write_8cyc_1L_2V, WriteAdr],   (instregex "LD3Rv(8b|4h|2s)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_8cyc_1L_2V],   (instregex "LD3Rv(8b|4h|2s)_POST$")>;
 def : InstRW<[A57Write_6cyc_2L],              (instregex "LD3Rv(1d)$")>;
-def : InstRW<[A57Write_6cyc_2L, WriteAdr],    (instregex "LD3Rv(1d)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_6cyc_2L],    (instregex "LD3Rv(1d)_POST$")>;
 def : InstRW<[A57Write_9cyc_1L_3V],            (instregex "LD3Rv(16b|8h|4s)$")>;
-def : InstRW<[A57Write_9cyc_1L_3V, WriteAdr],  (instregex "LD3Rv(16b|8h|4s)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_9cyc_1L_3V],  (instregex "LD3Rv(16b|8h|4s)_POST$")>;
 def : InstRW<[A57Write_9cyc_2L_3V],           (instregex "LD3Rv(2d)$")>;
-def : InstRW<[A57Write_9cyc_2L_3V, WriteAdr], (instregex "LD3Rv(2d)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_9cyc_2L_3V], (instregex "LD3Rv(2d)_POST$")>;
 
 def : InstRW<[A57Write_9cyc_2L_2V],               (instregex "LD3Threev(8b|4h|2s)$")>;
-def : InstRW<[A57Write_9cyc_2L_2V, WriteAdr],     (instregex "LD3Threev(8b|4h|2s)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_9cyc_2L_2V],     (instregex "LD3Threev(8b|4h|2s)_POST$")>;
 def : InstRW<[A57Write_10cyc_3L_4V],           (instregex "LD3Threev(16b|8h|4s)$")>;
-def : InstRW<[A57Write_10cyc_3L_4V, WriteAdr], (instregex "LD3Threev(16b|8h|4s)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_10cyc_3L_4V], (instregex "LD3Threev(16b|8h|4s)_POST$")>;
 def : InstRW<[A57Write_8cyc_4L],               (instregex "LD3Threev(2d)$")>;
-def : InstRW<[A57Write_8cyc_4L, WriteAdr],     (instregex "LD3Threev(2d)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_8cyc_4L],     (instregex "LD3Threev(2d)_POST$")>;
 
 def : InstRW<[A57Write_9cyc_2L_3V],           (instregex "LD4i(8|16)$")>;
-def : InstRW<[A57Write_9cyc_2L_3V, WriteAdr], (instregex "LD4i(8|16)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_9cyc_2L_3V], (instregex "LD4i(8|16)_POST$")>;
 def : InstRW<[A57Write_8cyc_1L_2V],             (instregex "LD4i(32)$")>;
-def : InstRW<[A57Write_8cyc_1L_2V, WriteAdr],   (instregex "LD4i(32)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_8cyc_1L_2V],   (instregex "LD4i(32)_POST$")>;
 def : InstRW<[A57Write_9cyc_2L_3V],           (instregex "LD4i(64)$")>;
-def : InstRW<[A57Write_9cyc_2L_3V, WriteAdr], (instregex "LD4i(64)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_9cyc_2L_3V], (instregex "LD4i(64)_POST$")>;
 
 def : InstRW<[A57Write_8cyc_1L_2V],              (instregex "LD4Rv(8b|4h|2s)$")>;
-def : InstRW<[A57Write_8cyc_1L_2V, WriteAdr],    (instregex "LD4Rv(8b|4h|2s)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_8cyc_1L_2V],    (instregex "LD4Rv(8b|4h|2s)_POST$")>;
 def : InstRW<[A57Write_6cyc_2L],               (instregex "LD4Rv(1d)$")>;
-def : InstRW<[A57Write_6cyc_2L, WriteAdr],     (instregex "LD4Rv(1d)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_6cyc_2L],     (instregex "LD4Rv(1d)_POST$")>;
 def : InstRW<[A57Write_9cyc_2L_3V],            (instregex "LD4Rv(16b|8h|4s)$")>;
-def : InstRW<[A57Write_9cyc_2L_3V, WriteAdr],  (instregex "LD4Rv(16b|8h|4s)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_9cyc_2L_3V],  (instregex "LD4Rv(16b|8h|4s)_POST$")>;
 def : InstRW<[A57Write_9cyc_2L_4V],           (instregex "LD4Rv(2d)$")>;
-def : InstRW<[A57Write_9cyc_2L_4V, WriteAdr], (instregex "LD4Rv(2d)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_9cyc_2L_4V], (instregex "LD4Rv(2d)_POST$")>;
 
 def : InstRW<[A57Write_9cyc_2L_2V],                (instregex "LD4Fourv(8b|4h|2s)$")>;
-def : InstRW<[A57Write_9cyc_2L_2V, WriteAdr],      (instregex "LD4Fourv(8b|4h|2s)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_9cyc_2L_2V],      (instregex "LD4Fourv(8b|4h|2s)_POST$")>;
 def : InstRW<[A57Write_11cyc_4L_4V],           (instregex "LD4Fourv(16b|8h|4s)$")>;
-def : InstRW<[A57Write_11cyc_4L_4V, WriteAdr], (instregex "LD4Fourv(16b|8h|4s)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_11cyc_4L_4V], (instregex "LD4Fourv(16b|8h|4s)_POST$")>;
 def : InstRW<[A57Write_8cyc_4L],                (instregex "LD4Fourv(2d)$")>;
-def : InstRW<[A57Write_8cyc_4L, WriteAdr],      (instregex "LD4Fourv(2d)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_8cyc_4L],      (instregex "LD4Fourv(2d)_POST$")>;
 
 // Vector Store
 // -----------------------------------------------------------------------------
 
 def : InstRW<[A57Write_1cyc_1S],            (instregex "ST1i(8|16|32)$")>;
-def : InstRW<[A57Write_1cyc_1S, WriteAdr],  (instregex "ST1i(8|16|32)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_1cyc_1S],  (instregex "ST1i(8|16|32)_POST$")>;
 def : InstRW<[A57Write_3cyc_1S_1V],           (instregex "ST1i(64)$")>;
-def : InstRW<[A57Write_3cyc_1S_1V, WriteAdr], (instregex "ST1i(64)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_3cyc_1S_1V], (instregex "ST1i(64)_POST$")>;
 
 def : InstRW<[A57Write_1cyc_1S],                  (instregex "ST1Onev(8b|4h|2s|1d)$")>;
-def : InstRW<[A57Write_1cyc_1S, WriteAdr],        (instregex "ST1Onev(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_1cyc_1S],        (instregex "ST1Onev(8b|4h|2s|1d)_POST$")>;
 def : InstRW<[A57Write_2cyc_2S],                 (instregex "ST1Onev(16b|8h|4s|2d)$")>;
-def : InstRW<[A57Write_2cyc_2S, WriteAdr],       (instregex "ST1Onev(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_2cyc_2S],       (instregex "ST1Onev(16b|8h|4s|2d)_POST$")>;
 def : InstRW<[A57Write_2cyc_2S],                 (instregex "ST1Twov(8b|4h|2s|1d)$")>;
-def : InstRW<[A57Write_2cyc_2S, WriteAdr],       (instregex "ST1Twov(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_2cyc_2S],       (instregex "ST1Twov(8b|4h|2s|1d)_POST$")>;
 def : InstRW<[A57Write_4cyc_4S],               (instregex "ST1Twov(16b|8h|4s|2d)$")>;
-def : InstRW<[A57Write_4cyc_4S, WriteAdr],     (instregex "ST1Twov(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_4cyc_4S],     (instregex "ST1Twov(16b|8h|4s|2d)_POST$")>;
 def : InstRW<[A57Write_3cyc_3S],                (instregex "ST1Threev(8b|4h|2s|1d)$")>;
-def : InstRW<[A57Write_3cyc_3S, WriteAdr],      (instregex "ST1Threev(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_3cyc_3S],      (instregex "ST1Threev(8b|4h|2s|1d)_POST$")>;
 def : InstRW<[A57Write_6cyc_6S],             (instregex "ST1Threev(16b|8h|4s|2d)$")>;
-def : InstRW<[A57Write_6cyc_6S, WriteAdr],   (instregex "ST1Threev(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_6cyc_6S],   (instregex "ST1Threev(16b|8h|4s|2d)_POST$")>;
 def : InstRW<[A57Write_4cyc_4S],               (instregex "ST1Fourv(8b|4h|2s|1d)$")>;
-def : InstRW<[A57Write_4cyc_4S, WriteAdr],     (instregex "ST1Fourv(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_4cyc_4S],     (instregex "ST1Fourv(8b|4h|2s|1d)_POST$")>;
 def : InstRW<[A57Write_8cyc_8S],           (instregex "ST1Fourv(16b|8h|4s|2d)$")>;
-def : InstRW<[A57Write_8cyc_8S, WriteAdr], (instregex "ST1Fourv(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_8cyc_8S], (instregex "ST1Fourv(16b|8h|4s|2d)_POST$")>;
 
 def : InstRW<[A57Write_3cyc_1S_1V],           (instregex "ST2i(8|16|32)$")>;
-def : InstRW<[A57Write_3cyc_1S_1V, WriteAdr], (instregex "ST2i(8|16|32)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_3cyc_1S_1V], (instregex "ST2i(8|16|32)_POST$")>;
 def : InstRW<[A57Write_2cyc_2S],           (instregex "ST2i(64)$")>;
-def : InstRW<[A57Write_2cyc_2S, WriteAdr], (instregex "ST2i(64)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_2cyc_2S], (instregex "ST2i(64)_POST$")>;
 
 def : InstRW<[A57Write_3cyc_2S_1V],              (instregex "ST2Twov(8b|4h|2s)$")>;
-def : InstRW<[A57Write_3cyc_2S_1V, WriteAdr],    (instregex "ST2Twov(8b|4h|2s)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_3cyc_2S_1V],    (instregex "ST2Twov(8b|4h|2s)_POST$")>;
 def : InstRW<[A57Write_4cyc_4S_2V],           (instregex "ST2Twov(16b|8h|4s)$")>;
-def : InstRW<[A57Write_4cyc_4S_2V, WriteAdr], (instregex "ST2Twov(16b|8h|4s)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_4cyc_4S_2V], (instregex "ST2Twov(16b|8h|4s)_POST$")>;
 def : InstRW<[A57Write_4cyc_4S],             (instregex "ST2Twov(2d)$")>;
-def : InstRW<[A57Write_4cyc_4S, WriteAdr],   (instregex "ST2Twov(2d)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_4cyc_4S],   (instregex "ST2Twov(2d)_POST$")>;
 
 def : InstRW<[A57Write_3cyc_1S_1V],            (instregex "ST3i(8|16)$")>;
-def : InstRW<[A57Write_3cyc_1S_1V, WriteAdr],  (instregex "ST3i(8|16)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_3cyc_1S_1V],  (instregex "ST3i(8|16)_POST$")>;
 def : InstRW<[A57Write_3cyc_3S],           (instregex "ST3i(32)$")>;
-def : InstRW<[A57Write_3cyc_3S, WriteAdr], (instregex "ST3i(32)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_3cyc_3S], (instregex "ST3i(32)_POST$")>;
 def : InstRW<[A57Write_3cyc_2S_1V],           (instregex "ST3i(64)$")>;
-def : InstRW<[A57Write_3cyc_2S_1V, WriteAdr], (instregex "ST3i(64)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_3cyc_2S_1V], (instregex "ST3i(64)_POST$")>;
 
 def : InstRW<[A57Write_3cyc_3S_2V],                 (instregex "ST3Threev(8b|4h|2s)$")>;
-def : InstRW<[A57Write_3cyc_3S_2V, WriteAdr],       (instregex "ST3Threev(8b|4h|2s)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_3cyc_3S_2V],       (instregex "ST3Threev(8b|4h|2s)_POST$")>;
 def : InstRW<[A57Write_6cyc_6S_4V],           (instregex "ST3Threev(16b|8h|4s)$")>;
-def : InstRW<[A57Write_6cyc_6S_4V, WriteAdr], (instregex "ST3Threev(16b|8h|4s)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_6cyc_6S_4V], (instregex "ST3Threev(16b|8h|4s)_POST$")>;
 def : InstRW<[A57Write_6cyc_6S],                (instregex "ST3Threev(2d)$")>;
-def : InstRW<[A57Write_6cyc_6S, WriteAdr],      (instregex "ST3Threev(2d)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_6cyc_6S],      (instregex "ST3Threev(2d)_POST$")>;
 
 def : InstRW<[A57Write_3cyc_1S_1V],             (instregex "ST4i(8|16)$")>;
-def : InstRW<[A57Write_3cyc_1S_1V, WriteAdr],   (instregex "ST4i(8|16)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_3cyc_1S_1V],   (instregex "ST4i(8|16)_POST$")>;
 def : InstRW<[A57Write_4cyc_4S],           (instregex "ST4i(32)$")>;
-def : InstRW<[A57Write_4cyc_4S, WriteAdr], (instregex "ST4i(32)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_4cyc_4S], (instregex "ST4i(32)_POST$")>;
 def : InstRW<[A57Write_3cyc_2S_1V],            (instregex "ST4i(64)$")>;
-def : InstRW<[A57Write_3cyc_2S_1V, WriteAdr],  (instregex "ST4i(64)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_3cyc_2S_1V],  (instregex "ST4i(64)_POST$")>;
 
 def : InstRW<[A57Write_4cyc_4S_2V],                  (instregex "ST4Fourv(8b|4h|2s)$")>;
-def : InstRW<[A57Write_4cyc_4S_2V, WriteAdr],        (instregex "ST4Fourv(8b|4h|2s)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_4cyc_4S_2V],        (instregex "ST4Fourv(8b|4h|2s)_POST$")>;
 def : InstRW<[A57Write_8cyc_8S_4V],           (instregex "ST4Fourv(16b|8h|4s)$")>;
-def : InstRW<[A57Write_8cyc_8S_4V, WriteAdr], (instregex "ST4Fourv(16b|8h|4s)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_8cyc_8S_4V], (instregex "ST4Fourv(16b|8h|4s)_POST$")>;
 def : InstRW<[A57Write_8cyc_8S],                (instregex "ST4Fourv(2d)$")>;
-def : InstRW<[A57Write_8cyc_8S, WriteAdr],      (instregex "ST4Fourv(2d)_POST$")>;
+def : InstRW<[WriteAdr, A57Write_8cyc_8S],      (instregex "ST4Fourv(2d)_POST$")>;
 
 // Vector - Integer
 // -----------------------------------------------------------------------------
@@ -592,38 +592,38 @@ def : InstRW<[A57Write_5cyc_1L, WriteLDHi], (instrs LDNPDi)>;
 def : InstRW<[A57Write_6cyc_2L, WriteLDHi], (instrs LDNPQi)>;
 def : InstRW<[A57Write_5cyc_1L, WriteLDHi], (instrs LDNPSi)>;
 def : InstRW<[A57Write_5cyc_1L, WriteLDHi], (instrs LDPDi)>;
-def : InstRW<[A57Write_5cyc_1L, WriteLDHi, WriteAdr], (instrs LDPDpost)>;
-def : InstRW<[A57Write_5cyc_1L, WriteLDHi, WriteAdr], (instrs LDPDpre)>;
+def : InstRW<[WriteAdr, A57Write_5cyc_1L, WriteLDHi], (instrs LDPDpost)>;
+def : InstRW<[WriteAdr, A57Write_5cyc_1L, WriteLDHi], (instrs LDPDpre)>;
 def : InstRW<[A57Write_6cyc_2L, WriteLDHi], (instrs LDPQi)>;
-def : InstRW<[A57Write_6cyc_2L, WriteLDHi, WriteAdr], (instrs LDPQpost)>;
-def : InstRW<[A57Write_6cyc_2L, WriteLDHi, WriteAdr], (instrs LDPQpre)>;
+def : InstRW<[WriteAdr, A57Write_6cyc_2L, WriteLDHi], (instrs LDPQpost)>;
+def : InstRW<[WriteAdr, A57Write_6cyc_2L, WriteLDHi], (instrs LDPQpre)>;
 def : InstRW<[A57Write_5cyc_1I_2L, WriteLDHi], (instrs LDPSWi)>;
-def : InstRW<[A57Write_5cyc_1I_2L, WriteLDHi, WriteAdr], (instrs LDPSWpost)>;
-def : InstRW<[A57Write_5cyc_1I_2L, WriteLDHi, WriteAdr], (instrs LDPSWpre)>;
+def : InstRW<[WriteAdr, A57Write_5cyc_1I_2L, WriteLDHi], (instrs LDPSWpost)>;
+def : InstRW<[WriteAdr, A57Write_5cyc_1I_2L, WriteLDHi], (instrs LDPSWpre)>;
 def : InstRW<[A57Write_5cyc_1L, WriteLDHi], (instrs LDPSi)>;
-def : InstRW<[A57Write_5cyc_1L, WriteLDHi, WriteAdr], (instrs LDPSpost)>;
-def : InstRW<[A57Write_5cyc_1L, WriteLDHi, WriteAdr], (instrs LDPSpre)>;
+def : InstRW<[WriteAdr, A57Write_5cyc_1L, WriteLDHi], (instrs LDPSpost)>;
+def : InstRW<[WriteAdr, A57Write_5cyc_1L, WriteLDHi], (instrs LDPSpre)>;
 def : InstRW<[A57Write_5cyc_1L, WriteI], (instrs LDRBpost)>;
-def : InstRW<[A57Write_5cyc_1L, WriteAdr], (instrs LDRBpre)>;
+def : InstRW<[WriteAdr, A57Write_5cyc_1L], (instrs LDRBpre)>;
 def : InstRW<[A57Write_5cyc_1L, ReadAdrBase], (instrs LDRBroW)>;
 def : InstRW<[A57Write_5cyc_1L, ReadAdrBase], (instrs LDRBroX)>;
 def : InstRW<[A57Write_5cyc_1L], (instrs LDRBui)>;
 def : InstRW<[A57Write_5cyc_1L], (instrs LDRDl)>;
 def : InstRW<[A57Write_5cyc_1L, WriteI], (instrs LDRDpost)>;
-def : InstRW<[A57Write_5cyc_1L, WriteAdr], (instrs LDRDpre)>;
+def : InstRW<[WriteAdr, A57Write_5cyc_1L], (instrs LDRDpre)>;
 def : InstRW<[A57Write_5cyc_1L, ReadAdrBase], (instrs LDRDroW)>;
 def : InstRW<[A57Write_5cyc_1L, ReadAdrBase], (instrs LDRDroX)>;
 def : InstRW<[A57Write_5cyc_1L], (instrs LDRDui)>;
 def : InstRW<[A57Write_5cyc_1I_1L, ReadAdrBase], (instrs LDRHHroW)>;
 def : InstRW<[A57Write_5cyc_1I_1L, ReadAdrBase], (instrs LDRHHroX)>;
 def : InstRW<[A57Write_5cyc_1L, WriteI], (instrs LDRHpost)>;
-def : InstRW<[A57Write_5cyc_1L, WriteAdr], (instrs LDRHpre)>;
+def : InstRW<[WriteAdr, A57Write_5cyc_1L], (instrs LDRHpre)>;
 def : InstRW<[A57Write_6cyc_1I_1L, ReadAdrBase], (instrs LDRHroW)>;
 def : InstRW<[A57Write_6cyc_1I_1L, ReadAdrBase], (instrs LDRHroX)>;
 def : InstRW<[A57Write_5cyc_1L], (instrs LDRHui)>;
 def : InstRW<[A57Write_5cyc_1L], (instrs LDRQl)>;
 def : InstRW<[A57Write_5cyc_1L, WriteI], (instrs LDRQpost)>;
-def : InstRW<[A57Write_5cyc_1L, WriteAdr], (instrs LDRQpre)>;
+def : InstRW<[WriteAdr, A57Write_5cyc_1L], (instrs LDRQpre)>;
 def : InstRW<[A57Write_6cyc_1I_1L, ReadAdrBase], (instrs LDRQroW)>;
 def : InstRW<[A57Write_6cyc_1I_1L, ReadAdrBase], (instrs LDRQroX)>;
 def : InstRW<[A57Write_5cyc_1L], (instrs LDRQui)>;
@@ -633,7 +633,7 @@ def : InstRW<[A57Write_5cyc_1I_1L, ReadAdrBase], (instrs LDRSHXroW)>;
 def : InstRW<[A57Write_5cyc_1I_1L, ReadAdrBase], (instrs LDRSHXroX)>;
 def : InstRW<[A57Write_5cyc_1L], (instrs LDRSl)>;
 def : InstRW<[A57Write_5cyc_1L, WriteI], (instrs LDRSpost)>;
-def : InstRW<[A57Write_5cyc_1L, WriteAdr], (instrs LDRSpre)>;
+def : InstRW<[WriteAdr, A57Write_5cyc_1L], (instrs LDRSpre)>;
 def : InstRW<[A57Write_5cyc_1L, ReadAdrBase], (instrs LDRSroW)>;
 def : InstRW<[A57Write_5cyc_1L, ReadAdrBase], (instrs LDRSroX)>;
 def : InstRW<[A57Write_5cyc_1L], (instrs LDRSui)>;
diff --git a/llvm/lib/Target/AArch64/AArch64SchedNeoverseN1.td b/llvm/lib/Target/AArch64/AArch64SchedNeoverseN1.td
index 220bebc668..2ec9600f84 100644
--- a/llvm/lib/Target/AArch64/AArch64SchedNeoverseN1.td
+++ b/llvm/lib/Target/AArch64/AArch64SchedNeoverseN1.td
@@ -377,7 +377,7 @@ def : InstRW<[N1Write_4c_1L, N1Write_0c_0Z], (instrs LDPWi, LDNPWi)>;
 def : InstRW<[N1Write_5c_1I_1L, N1Write_0c_0Z], (instrs LDPSWi)>;
 
 // Load pair, immed post or pre-index, signed words
-def : InstRW<[N1Write_5c_1I_1L, N1Write_0c_0Z, WriteAdr],
+def : InstRW<[WriteAdr, N1Write_5c_1I_1L, N1Write_0c_0Z],
              (instrs LDPSWpost, LDPSWpre)>;
 
 
@@ -477,7 +477,7 @@ def : InstRW<[N1Write_5c_1L, ReadAdrBase], (instregex "^LDR[SDQ]l$",
 
 // Load vector reg, immed post-index
 // Load vector reg, immed pre-index
-def : InstRW<[N1Write_5c_1L, WriteAdr],
+def : InstRW<[WriteAdr, N1Write_5c_1L],
              (instregex "^LDR[BHSDQ](post|pre)$")>;
 
 // Load vector reg, unsigned immed
@@ -501,12 +501,12 @@ def : InstRW<[N1Write_7c_1I_1L, WriteLDHi], (instregex "^LDPN?[HQ]i$")>;
 
 // Load vector pair, immed post-index, S/D-form
 // Load vector pair, immed pre-index, S/D-form
-def : InstRW<[N1Write_5c_1L, WriteLDHi, WriteAdr],
+def : InstRW<[WriteAdr, N1Write_5c_1L, WriteLDHi],
              (instregex "^LDP[SD](pre|post)$")>;
 
 // Load vector pair, immed post-index, Q-form
 // Load vector pair, immed pre-index, Q-form
-def : InstRW<[N1Write_7c_1L, WriteLDHi, WriteAdr],
+def : InstRW<[WriteAdr, N1Write_7c_1L, WriteLDHi],
              (instrs LDPQpost, LDPQpre)>;
 
 
@@ -521,11 +521,11 @@ def : InstRW<[N1Write_2c_2I_2L], (instrs STURQi)>;
 
 // Store vector reg, immed post-index, B/H/S/D-form
 // Store vector reg, immed pre-index, B/H/S/D-form
-def : InstRW<[N1Write_2c_1L_1V, WriteAdr], (instregex "^STR[BHSD](pre|post)$")>;
+def : InstRW<[WriteAdr, N1Write_2c_1L_1V], (instregex "^STR[BHSD](pre|post)$")>;
 
 // Store vector reg, immed pre-index, Q-form
 // Store vector reg, immed post-index, Q-form
-def : InstRW<[N1Write_2c_2L_2V, WriteAdr], (instrs STRQpre, STRQpost)>;
+def : InstRW<[WriteAdr, N1Write_2c_2L_2V], (instrs STRQpre, STRQpost)>;
 
 // Store vector reg, unsigned immed, B/H/S/D-form
 def : InstRW<[N1Write_2c_1L_1V], (instregex "^STR[BHSD]ui$")>;
@@ -562,15 +562,15 @@ def : InstRW<[N1Write_3c_4L_2V], (instrs STPQi, STNPQi)>;
 
 // Store vector pair, immed post-index, S-form
 // Store vector pair, immed pre-index, S-form
-def : InstRW<[N1Write_2c_1L_1V, WriteAdr], (instrs STPSpre, STPSpost)>;
+def : InstRW<[WriteAdr, N1Write_2c_1L_1V], (instrs STPSpre, STPSpost)>;
 
 // Store vector pair, immed post-index, D-form
 // Store vector pair, immed pre-index, D-form
-def : InstRW<[N1Write_2c_2L_2V, WriteAdr], (instrs STPDpre, STPDpost)>;
+def : InstRW<[WriteAdr, N1Write_2c_2L_2V], (instrs STPDpre, STPDpost)>;
 
 // Store vector pair, immed post-index, Q-form
 // Store vector pair, immed pre-index, Q-form
-def : InstRW<[N1Write_3c_4L_2V, WriteAdr], (instrs STPQpre, STPQpost)>;
+def : InstRW<[WriteAdr, N1Write_3c_4L_2V], (instrs STPQpre, STPQpost)>;
 
 
 // ASIMD integer instructions
@@ -818,25 +818,25 @@ def : InstRW<[N1Write_5c_1M_1V], (instregex "^INSvi(8|16|32|64)gpr$")>;
 // ASIMD load, 1 element, multiple, 1 reg
 def : InstRW<[N1Write_5c_1L],
              (instregex "^LD1Onev(8b|16b|4h|8h|2s|4s|1d|2d)$")>;
-def : InstRW<[N1Write_5c_1L, WriteAdr],
+def : InstRW<[WriteAdr, N1Write_5c_1L],
              (instregex "^LD1Onev(8b|16b|4h|8h|2s|4s|1d|2d)_POST$")>;
 
 // ASIMD load, 1 element, multiple, 2 reg
 def : InstRW<[N1Write_5c_2L],
              (instregex "^LD1Twov(8b|16b|4h|8h|2s|4s|1d|2d)$")>;
-def : InstRW<[N1Write_5c_2L, WriteAdr],
+def : InstRW<[WriteAdr, N1Write_5c_2L],
              (instregex "^LD1Twov(8b|16b|4h|8h|2s|4s|1d|2d)_POST$")>;
 
 // ASIMD load, 1 element, multiple, 3 reg
 def : InstRW<[N1Write_6c_3L],
              (instregex "^LD1Threev(8b|16b|4h|8h|2s|4s|1d|2d)$")>;
-def : InstRW<[N1Write_6c_3L, WriteAdr],
+def : InstRW<[WriteAdr, N1Write_6c_3L],
              (instregex "^LD1Threev(8b|16b|4h|8h|2s|4s|1d|2d)_POST$")>;
 
 // ASIMD load, 1 element, multiple, 4 reg
 def : InstRW<[N1Write_6c_4L],
              (instregex "^LD1Fourv(8b|16b|4h|8h|2s|4s|1d|2d)$")>;
-def : InstRW<[N1Write_6c_4L, WriteAdr],
+def : InstRW<[WriteAdr, N1Write_6c_4L],
              (instregex "^LD1Fourv(8b|16b|4h|8h|2s|4s|1d|2d)_POST$")>;
 
 // ASIMD load, 1 element, one lane
@@ -844,7 +844,7 @@ def : InstRW<[N1Write_6c_4L, WriteAdr],
 def : InstRW<[N1Write_7c_1L_1V],
              (instregex "LD1(i|Rv)(8|16|32|64)$",
                         "LD1Rv(8b|16b|4h|8h|2s|4s|1d|2d)$")>;
-def : InstRW<[N1Write_7c_1L_1V, WriteAdr],
+def : InstRW<[WriteAdr, N1Write_7c_1L_1V],
              (instregex "LD1i(8|16|32|64)_POST$",
                         "LD1Rv(8b|16b|4h|8h|2s|4s|1d|2d)_POST$")>;
 
@@ -855,7 +855,7 @@ def : InstRW<[N1Write_7c_2L_2V],
              (instregex "LD2Twov(8b|16b|4h|8h|2s|4s|2d)$",
                         "LD2i(8|16|32|64)$",
                         "LD2Rv(8b|16b|4h|8h|2s|4s|1d|2d)$")>;
-def : InstRW<[N1Write_7c_2L_2V, WriteAdr],
+def : InstRW<[WriteAdr, N1Write_7c_2L_2V],
              (instregex "LD2Twov(8b|16b|4h|8h|2s|4s|2d)_POST$",
                         "LD2i(8|16|32|64)_POST$",
                         "LD2Rv(8b|16b|4h|8h|2s|4s|1d|2d)_POST$")>;
@@ -863,7 +863,7 @@ def : InstRW<[N1Write_7c_2L_2V, WriteAdr],
 // ASIMD load, 3 element, multiple
 def : InstRW<[N1Write_8c_3L_3V],
              (instregex "LD3Threev(8b|16b|4h|8h|2s|4s|2d)$")>;
-def : InstRW<[N1Write_8c_3L_3V, WriteAdr],
+def : InstRW<[WriteAdr, N1Write_8c_3L_3V],
              (instregex "LD3Threev(8b|16b|4h|8h|2s|4s|2d)_POST$")>;
 
 // ASIMD load, 3 element, one lane
@@ -871,20 +871,20 @@ def : InstRW<[N1Write_8c_3L_3V, WriteAdr],
 def : InstRW<[N1Write_7c_2L_3V],
              (instregex "LD3i(8|16|32|64)$",
                         "LD3Rv(8b|16b|4h|8h|2s|4s|1d|2d)$")>;
-def : InstRW<[N1Write_7c_2L_3V, WriteAdr],
+def : InstRW<[WriteAdr, N1Write_7c_2L_3V],
              (instregex "LD3i(8|16|32|64)_POST$",
                         "LD3Rv(8b|16b|4h|8h|2s|4s|1d|2d)_POST$")>;
 
 // ASIMD load, 4 element, multiple, D-form
 def : InstRW<[N1Write_8c_3L_4V],
              (instregex "LD4Fourv(8b|4h|2s)$")>;
-def : InstRW<[N1Write_8c_3L_4V, WriteAdr], 
+def : InstRW<[WriteAdr, N1Write_8c_3L_4V],
              (instregex "LD4Fourv(8b|4h|2s)_POST$")>;
 
 // ASIMD load, 4 element, multiple, Q-form
 def : InstRW<[N1Write_10c_4L_4V],
              (instregex "LD4Fourv(16b|8h|4s|2d)$")>;
-def : InstRW<[N1Write_10c_4L_4V, WriteAdr],
+def : InstRW<[WriteAdr, N1Write_10c_4L_4V],
              (instregex "LD4Fourv(16b|8h|4s|2d)_POST$")>;
 
 // ASIMD load, 4 element, one lane
@@ -892,7 +892,7 @@ def : InstRW<[N1Write_10c_4L_4V, WriteAdr],
 def : InstRW<[N1Write_8c_4L_4V],
              (instregex "LD4i(8|16|32|64)$",
                         "LD4Rv(8b|16b|4h|8h|2s|4s|1d|2d)$")>;
-def : InstRW<[N1Write_8c_4L_4V, WriteAdr],
+def : InstRW<[WriteAdr, N1Write_8c_4L_4V],
              (instregex "LD4i(8|16|32|64)_POST$",
                         "LD4Rv(8b|16b|4h|8h|2s|4s|1d|2d)_POST$")>;
 
@@ -903,127 +903,127 @@ def : InstRW<[N1Write_8c_4L_4V, WriteAdr],
 // ASIMD store, 1 element, multiple, 1 reg, D-form
 def : InstRW<[N1Write_2c_1L_1V],
              (instregex "ST1Onev(8b|4h|2s|1d)$")>;
-def : InstRW<[N1Write_2c_1L_1V, WriteAdr],
+def : InstRW<[WriteAdr, N1Write_2c_1L_1V],
              (instregex "ST1Onev(8b|4h|2s|1d)_POST$")>;
 
 // ASIMD store, 1 element, multiple, 1 reg, Q-form
 def : InstRW<[N1Write_2c_1L_1V],
              (instregex "ST1Onev(16b|8h|4s|2d)$")>;
-def : InstRW<[N1Write_2c_1L_1V, WriteAdr],
+def : InstRW<[WriteAdr, N1Write_2c_1L_1V],
              (instregex "ST1Onev(16b|8h|4s|2d)_POST$")>;
 
 // ASIMD store, 1 element, multiple, 2 reg, D-form
 def : InstRW<[N1Write_2c_1L_2V],
              (instregex "ST1Twov(8b|4h|2s|1d)$")>;
-def : InstRW<[N1Write_2c_1L_2V, WriteAdr],
+def : InstRW<[WriteAdr, N1Write_2c_1L_2V],
              (instregex "ST1Twov(8b|4h|2s|1d)_POST$")>;
 
 // ASIMD store, 1 element, multiple, 2 reg, Q-form
 def : InstRW<[N1Write_3c_2L_2V],
              (instregex "ST1Twov(16b|8h|4s|2d)$")>;
-def : InstRW<[N1Write_3c_2L_2V, WriteAdr],
+def : InstRW<[WriteAdr, N1Write_3c_2L_2V],
              (instregex "ST1Twov(16b|8h|4s|2d)_POST$")>;
 
 // ASIMD store, 1 element, multiple, 3 reg, D-form
 def : InstRW<[N1Write_3c_2L_3V],           
              (instregex "ST1Threev(8b|4h|2s|1d)$")>;
-def : InstRW<[N1Write_3c_2L_3V, WriteAdr], 
+def : InstRW<[WriteAdr, N1Write_3c_2L_3V],
              (instregex "ST1Threev(8b|4h|2s|1d)_POST$")>;
 
 // ASIMD store, 1 element, multiple, 3 reg, Q-form
 def : InstRW<[N1Write_4c_3L_3V],
              (instregex "ST1Threev(16b|8h|4s|2d)$")>;
-def : InstRW<[N1Write_4c_3L_3V, WriteAdr],
+def : InstRW<[WriteAdr, N1Write_4c_3L_3V],
              (instregex "ST1Threev(16b|8h|4s|2d)_POST$")>;
 
 // ASIMD store, 1 element, multiple, 4 reg, D-form
 def : InstRW<[N1Write_3c_2L_2V],
              (instregex "ST1Fourv(8b|4h|2s|1d)$")>;
-def : InstRW<[N1Write_3c_2L_2V, WriteAdr],
+def : InstRW<[WriteAdr, N1Write_3c_2L_2V],
              (instregex "ST1Fourv(8b|4h|2s|1d)_POST$")>;
 
 // ASIMD store, 1 element, multiple, 4 reg, Q-form
 def : InstRW<[N1Write_5c_4L_4V],
              (instregex "ST1Fourv(16b|8h|4s|2d)$")>;
-def : InstRW<[N1Write_5c_4L_4V, WriteAdr],
+def : InstRW<[WriteAdr, N1Write_5c_4L_4V],
              (instregex "ST1Fourv(16b|8h|4s|2d)_POST$")>;
 
 // ASIMD store, 1 element, one lane
 def : InstRW<[N1Write_4c_1L_1V],
              (instregex "ST1i(8|16|32|64)$")>;
-def : InstRW<[N1Write_4c_1L_1V, WriteAdr],
+def : InstRW<[WriteAdr, N1Write_4c_1L_1V],
              (instregex "ST1i(8|16|32|64)_POST$")>;
 
 // ASIMD store, 2 element, multiple, D-form, B/H/S
 def : InstRW<[N1Write_4c_1L_1V],
              (instregex "ST2Twov(8b|4h|2s)$")>;
-def : InstRW<[N1Write_4c_1L_1V, WriteAdr],
+def : InstRW<[WriteAdr, N1Write_4c_1L_1V],
              (instregex "ST2Twov(8b|4h|2s)_POST$")>;
 
 // ASIMD store, 2 element, multiple, Q-form
 def : InstRW<[N1Write_5c_2L_2V],
              (instregex "ST2Twov(16b|8h|4s|2d)$")>;
-def : InstRW<[N1Write_5c_2L_2V, WriteAdr],
+def : InstRW<[WriteAdr, N1Write_5c_2L_2V],
              (instregex "ST2Twov(16b|8h|4s|2d)_POST$")>;
 
 // ASIMD store, 2 element, one lane
 def : InstRW<[N1Write_4c_1L_1V],
              (instregex "ST2i(8|16|32|64)$")>;
-def : InstRW<[N1Write_4c_1L_1V, WriteAdr],
+def : InstRW<[WriteAdr, N1Write_4c_1L_1V],
              (instregex "ST2i(8|16|32|64)_POST$")>;
 
 // ASIMD store, 3 element, multiple, D-form, B/H/S
 def : InstRW<[N1Write_5c_2L_2V],
              (instregex "ST3Threev(8b|4h|2s)$")>;
-def : InstRW<[N1Write_5c_2L_2V, WriteAdr],
+def : InstRW<[WriteAdr, N1Write_5c_2L_2V],
              (instregex "ST3Threev(8b|4h|2s)_POST$")>;
 
 // ASIMD store, 3 element, multiple, Q-form
 def : InstRW<[N1Write_6c_3L_3V],
              (instregex "ST3Threev(16b|8h|4s|2d)$")>;
-def : InstRW<[N1Write_6c_3L_3V, WriteAdr],
+def : InstRW<[WriteAdr, N1Write_6c_3L_3V],
              (instregex "ST3Threev(16b|8h|4s|2d)_POST$")>;
 
 // ASIMD store, 3 element, one lane, B/H/S
 def : InstRW<[N1Write_4c_3L_3V],
              (instregex "ST3i(8|16|32)$")>;
-def : InstRW<[N1Write_4c_3L_3V, WriteAdr],
+def : InstRW<[WriteAdr, N1Write_4c_3L_3V],
              (instregex "ST3i(8|16|32)_POST$")>;
 
 // ASIMD store, 3 element, one lane, D
 def : InstRW<[N1Write_5c_3L_3V],
              (instrs ST3i64)>;
-def : InstRW<[N1Write_5c_3L_3V, WriteAdr],
+def : InstRW<[WriteAdr, N1Write_5c_3L_3V],
              (instrs ST3i64_POST)>;
 
 // ASIMD store, 4 element, multiple, D-form, B/H/S
 def : InstRW<[N1Write_7c_3L_3V],
              (instregex "ST4Fourv(8b|4h|2s)$")>;
-def : InstRW<[N1Write_7c_3L_3V, WriteAdr],
+def : InstRW<[WriteAdr, N1Write_7c_3L_3V],
              (instregex "ST4Fourv(8b|4h|2s)_POST$")>;
 
 // ASIMD store, 4 element, multiple, Q-form, B/H/S
 def : InstRW<[N1Write_9c_6L_6V],
              (instregex "ST4Fourv(16b|8h|4s)$")>;
-def : InstRW<[N1Write_9c_6L_6V, WriteAdr],
+def : InstRW<[WriteAdr, N1Write_9c_6L_6V],
              (instregex "ST4Fourv(16b|8h|4s)_POST$")>;
 
 // ASIMD store, 4 element, multiple, Q-form, D
 def : InstRW<[N1Write_6c_4L_4V],
              (instrs ST4Fourv2d)>;
-def : InstRW<[N1Write_6c_4L_4V, WriteAdr],
+def : InstRW<[WriteAdr, N1Write_6c_4L_4V],
              (instrs ST4Fourv2d_POST)>;
 
 // ASIMD store, 4 element, one lane, B/H/S
 def : InstRW<[N1Write_5c_3L_3V],
              (instregex "ST4i(8|16|32)$")>;
-def : InstRW<[N1Write_5c_3L_3V, WriteAdr],
+def : InstRW<[WriteAdr, N1Write_5c_3L_3V],
              (instregex "ST4i(8|16|32)_POST$")>;
 
 // ASIMD store, 4 element, one lane, D
 def : InstRW<[N1Write_4c_3L_3V],
              (instrs ST4i64)>;
-def : InstRW<[N1Write_4c_3L_3V, WriteAdr],
+def : InstRW<[WriteAdr, N1Write_4c_3L_3V],
              (instrs ST4i64_POST)>;
 
 
diff --git a/llvm/lib/Target/AArch64/AArch64SchedNeoverseN2.td b/llvm/lib/Target/AArch64/AArch64SchedNeoverseN2.td
index a6e28e653e..517d0da7f4 100644
--- a/llvm/lib/Target/AArch64/AArch64SchedNeoverseN2.td
+++ b/llvm/lib/Target/AArch64/AArch64SchedNeoverseN2.td
@@ -742,7 +742,7 @@ def : SchedAlias<WriteLDIdx, N2Write_4cyc_1I_1L>;
 // Load pair, signed immed offset, signed words
 def : InstRW<[N2Write_5cyc_1M0, WriteLDHi], (instrs LDPSWi)>;
 // Load pair, immed post-index or immed pre-index, signed words
-def : InstRW<[N2Write_5cyc_1M0, WriteLDHi, WriteAdr],
+def : InstRW<[WriteAdr, N2Write_5cyc_1M0, WriteLDHi],
              (instregex "^LDPSW(post|pre)$")>;
 
 // Store instructions
@@ -860,7 +860,7 @@ def : InstRW<[N2Write_6cyc_1L], (instregex "^LDR[SDQ]l$",
 // Load vector reg, immed post-index
 def : InstRW<[N2Write_6cyc_1I_1L, WriteI], (instregex "^LDR[BHSDQ]post$")>;
 // Load vector reg, immed pre-index
-def : InstRW<[N2Write_6cyc_1I_1L, WriteAdr], (instregex "^LDR[BHSDQ]pre$")>;
+def : InstRW<[WriteAdr, N2Write_6cyc_1I_1L], (instregex "^LDR[BHSDQ]pre$")>;
 
 // Load vector reg, unsigned immed
 def : InstRW<[N2Write_6cyc_1L], (instregex "^LDR[BHSDQ]ui$")>;
@@ -883,12 +883,12 @@ def : InstRW<[N2Write_6cyc_2L, WriteLDHi], (instrs LDPQi, LDNPQi)>;
 
 // Load vector pair, immed post-index, S/D-form
 // Load vector pair, immed pre-index, S/D-form
-def : InstRW<[N2Write_6cyc_1I_1L, WriteLDHi, WriteAdr],
+def : InstRW<[WriteAdr, N2Write_6cyc_1I_1L, WriteLDHi],
              (instregex "^LDP[SD](pre|post)$")>;
 
 // Load vector pair, immed post-index, Q-form
 // Load vector pair, immed pre-index, Q-form
-def : InstRW<[N2Write_6cyc_2I_2L, WriteLDHi, WriteAdr], (instrs LDPQpost,
+def : InstRW<[WriteAdr, N2Write_6cyc_2I_2L, WriteLDHi], (instrs LDPQpost,
                                                                 LDPQpre)>;
 
 // FP store instructions
@@ -1238,223 +1238,223 @@ def : InstRW<[N2Write_5cyc_1M0_1V], (instregex "^INSvi(8|16|32|64)gpr$")>;
 
 // ASIMD load, 1 element, multiple, 1 reg, D-form
 def : InstRW<[N2Write_6cyc_1L], (instregex "^LD1Onev(8b|4h|2s|1d)$")>;
-def : InstRW<[N2Write_6cyc_1L, WriteAdr],
+def : InstRW<[WriteAdr, N2Write_6cyc_1L],
              (instregex "^LD1Onev(8b|4h|2s|1d)_POST$")>;
 
 // ASIMD load, 1 element, multiple, 1 reg, Q-form
 def : InstRW<[N2Write_6cyc_1L], (instregex "^LD1Onev(16b|8h|4s|2d)$")>;
-def : InstRW<[N2Write_6cyc_1L, WriteAdr],
+def : InstRW<[WriteAdr, N2Write_6cyc_1L],
              (instregex "^LD1Onev(16b|8h|4s|2d)_POST$")>;
 
 // ASIMD load, 1 element, multiple, 2 reg, D-form
 def : InstRW<[N2Write_6cyc_2L], (instregex "^LD1Twov(8b|4h|2s|1d)$")>;
-def : InstRW<[N2Write_6cyc_2L, WriteAdr],
+def : InstRW<[WriteAdr, N2Write_6cyc_2L],
              (instregex "^LD1Twov(8b|4h|2s|1d)_POST$")>;
 
 // ASIMD load, 1 element, multiple, 2 reg, Q-form
 def : InstRW<[N2Write_6cyc_2L], (instregex "^LD1Twov(16b|8h|4s|2d)$")>;
-def : InstRW<[N2Write_6cyc_2L, WriteAdr],
+def : InstRW<[WriteAdr, N2Write_6cyc_2L],
              (instregex "^LD1Twov(16b|8h|4s|2d)_POST$")>;
 
 // ASIMD load, 1 element, multiple, 3 reg, D-form
 def : InstRW<[N2Write_6cyc_3L], (instregex "^LD1Threev(8b|4h|2s|1d)$")>;
-def : InstRW<[N2Write_6cyc_3L, WriteAdr],
+def : InstRW<[WriteAdr, N2Write_6cyc_3L],
              (instregex "^LD1Threev(8b|4h|2s|1d)_POST$")>;
 
 // ASIMD load, 1 element, multiple, 3 reg, Q-form
 def : InstRW<[N2Write_6cyc_3L], (instregex "^LD1Threev(16b|8h|4s|2d)$")>;
-def : InstRW<[N2Write_6cyc_3L, WriteAdr],
+def : InstRW<[WriteAdr, N2Write_6cyc_3L],
              (instregex "^LD1Threev(16b|8h|4s|2d)_POST$")>;
 
 // ASIMD load, 1 element, multiple, 4 reg, D-form
 def : InstRW<[N2Write_7cyc_4L], (instregex "^LD1Fourv(8b|4h|2s|1d)$")>;
-def : InstRW<[N2Write_7cyc_4L, WriteAdr],
+def : InstRW<[WriteAdr, N2Write_7cyc_4L],
              (instregex "^LD1Fourv(8b|4h|2s|1d)_POST$")>;
 
 // ASIMD load, 1 element, multiple, 4 reg, Q-form
 def : InstRW<[N2Write_7cyc_4L], (instregex "^LD1Fourv(16b|8h|4s|2d)$")>;
-def : InstRW<[N2Write_7cyc_4L, WriteAdr],
+def : InstRW<[WriteAdr, N2Write_7cyc_4L],
              (instregex "^LD1Fourv(16b|8h|4s|2d)_POST$")>;
 
 // ASIMD load, 1 element, one lane, B/H/S
 // ASIMD load, 1 element, one lane, D
 def : InstRW<[N2Write_8cyc_1L_1V],           (instregex "LD1i(8|16|32|64)$")>;
-def : InstRW<[N2Write_8cyc_1L_1V, WriteAdr], (instregex "LD1i(8|16|32|64)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_8cyc_1L_1V], (instregex "LD1i(8|16|32|64)_POST$")>;
 
 // ASIMD load, 1 element, all lanes, D-form, B/H/S
 // ASIMD load, 1 element, all lanes, D-form, D
 def : InstRW<[N2Write_8cyc_1L_1V],           (instregex "LD1Rv(8b|4h|2s|1d)$")>;
-def : InstRW<[N2Write_8cyc_1L_1V, WriteAdr], (instregex "LD1Rv(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_8cyc_1L_1V], (instregex "LD1Rv(8b|4h|2s|1d)_POST$")>;
 
 // ASIMD load, 1 element, all lanes, Q-form
 def : InstRW<[N2Write_8cyc_1L_1V],           (instregex "LD1Rv(16b|8h|4s|2d)$")>;
-def : InstRW<[N2Write_8cyc_1L_1V, WriteAdr], (instregex "LD1Rv(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_8cyc_1L_1V], (instregex "LD1Rv(16b|8h|4s|2d)_POST$")>;
 
 // ASIMD load, 2 element, multiple, D-form, B/H/S
 def : InstRW<[N2Write_8cyc_1L_2V],           (instregex "LD2Twov(8b|4h|2s)$")>;
-def : InstRW<[N2Write_8cyc_1L_2V, WriteAdr], (instregex "LD2Twov(8b|4h|2s)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_8cyc_1L_2V], (instregex "LD2Twov(8b|4h|2s)_POST$")>;
 
 // ASIMD load, 2 element, multiple, Q-form, B/H/S
 // ASIMD load, 2 element, multiple, Q-form, D
 def : InstRW<[N2Write_8cyc_2L_2V],           (instregex "LD2Twov(16b|8h|4s|2d)$")>;
-def : InstRW<[N2Write_8cyc_2L_2V, WriteAdr], (instregex "LD2Twov(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_8cyc_2L_2V], (instregex "LD2Twov(16b|8h|4s|2d)_POST$")>;
 
 // ASIMD load, 2 element, one lane, B/H
 // ASIMD load, 2 element, one lane, S
 // ASIMD load, 2 element, one lane, D
 def : InstRW<[N2Write_8cyc_1L_2V],           (instregex "LD2i(8|16|32|64)$")>;
-def : InstRW<[N2Write_8cyc_1L_2V, WriteAdr], (instregex "LD2i(8|16|32|64)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_8cyc_1L_2V], (instregex "LD2i(8|16|32|64)_POST$")>;
 
 // ASIMD load, 2 element, all lanes, D-form, B/H/S
 // ASIMD load, 2 element, all lanes, D-form, D
 def : InstRW<[N2Write_8cyc_1L_2V],            (instregex "LD2Rv(8b|4h|2s|1d)$")>;
-def : InstRW<[N2Write_8cyc_1L_2V, WriteAdr],  (instregex "LD2Rv(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_8cyc_1L_2V],  (instregex "LD2Rv(8b|4h|2s|1d)_POST$")>;
 
 // ASIMD load, 2 element, all lanes, Q-form
 def : InstRW<[N2Write_8cyc_1L_2V],           (instregex "LD2Rv(16b|8h|4s|2d)$")>;
-def : InstRW<[N2Write_8cyc_1L_2V, WriteAdr], (instregex "LD2Rv(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_8cyc_1L_2V], (instregex "LD2Rv(16b|8h|4s|2d)_POST$")>;
 
 // ASIMD load, 3 element, multiple, D-form, B/H/S
 def : InstRW<[N2Write_8cyc_2L_3V],           (instregex "LD3Threev(8b|4h|2s)$")>;
-def : InstRW<[N2Write_8cyc_2L_3V, WriteAdr], (instregex "LD3Threev(8b|4h|2s)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_8cyc_2L_3V], (instregex "LD3Threev(8b|4h|2s)_POST$")>;
 
 // ASIMD load, 3 element, multiple, Q-form, B/H/S
 def : InstRW<[N2Write_8cyc_3L_3V],           (instregex "LD3Threev(16b|8h|4s)$")>;
-def : InstRW<[N2Write_8cyc_3L_3V, WriteAdr], (instregex "LD3Threev(16b|8h|4s)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_8cyc_3L_3V], (instregex "LD3Threev(16b|8h|4s)_POST$")>;
 
 // ASIMD load, 3 element, multiple, Q-form, D
 def : InstRW<[N2Write_8cyc_3L_3V],           (instregex "LD3Threev(2d)$")>;
-def : InstRW<[N2Write_8cyc_3L_3V, WriteAdr], (instregex "LD3Threev(2d)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_8cyc_3L_3V], (instregex "LD3Threev(2d)_POST$")>;
 
 // ASIMD load, 3 element, one lane, B/H
 // ASIMD load, 3 element, one lane, S
 // ASIMD load, 3 element, one lane, D
 def : InstRW<[N2Write_8cyc_2L_3V],           (instregex "LD3i(8|16|32|64)$")>;
-def : InstRW<[N2Write_8cyc_2L_3V, WriteAdr], (instregex "LD3i(8|16|32|64)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_8cyc_2L_3V], (instregex "LD3i(8|16|32|64)_POST$")>;
 
 // ASIMD load, 3 element, all lanes, D-form, B/H/S
 // ASIMD load, 3 element, all lanes, D-form, D
 def : InstRW<[N2Write_8cyc_2L_3V],           (instregex "LD3Rv(8b|4h|2s|1d)$")>;
-def : InstRW<[N2Write_8cyc_2L_3V, WriteAdr], (instregex "LD3Rv(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_8cyc_2L_3V], (instregex "LD3Rv(8b|4h|2s|1d)_POST$")>;
 
 // ASIMD load, 3 element, all lanes, Q-form, B/H/S
 // ASIMD load, 3 element, all lanes, Q-form, D
 def : InstRW<[N2Write_8cyc_3L_3V],           (instregex "LD3Rv(16b|8h|4s|2d)$")>;
-def : InstRW<[N2Write_8cyc_3L_3V, WriteAdr], (instregex "LD3Rv(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_8cyc_3L_3V], (instregex "LD3Rv(16b|8h|4s|2d)_POST$")>;
 
 // ASIMD load, 4 element, multiple, D-form, B/H/S
 def : InstRW<[N2Write_8cyc_3L_4V],           (instregex "LD4Fourv(8b|4h|2s)$")>;
-def : InstRW<[N2Write_8cyc_3L_4V, WriteAdr], (instregex "LD4Fourv(8b|4h|2s)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_8cyc_3L_4V], (instregex "LD4Fourv(8b|4h|2s)_POST$")>;
 
 // ASIMD load, 4 element, multiple, Q-form, B/H/S
 // ASIMD load, 4 element, multiple, Q-form, D
 def : InstRW<[N2Write_9cyc_4L_4V],           (instregex "LD4Fourv(16b|8h|4s|2d)$")>;
-def : InstRW<[N2Write_9cyc_4L_4V, WriteAdr], (instregex "LD4Fourv(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_9cyc_4L_4V], (instregex "LD4Fourv(16b|8h|4s|2d)_POST$")>;
 
 // ASIMD load, 4 element, one lane, B/H
 // ASIMD load, 4 element, one lane, S
 // ASIMD load, 4 element, one lane, D
 def : InstRW<[N2Write_8cyc_3L_4V],           (instregex "LD4i(8|16|32|64)$")>;
-def : InstRW<[N2Write_8cyc_3L_4V, WriteAdr], (instregex "LD4i(8|16|32|64)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_8cyc_3L_4V], (instregex "LD4i(8|16|32|64)_POST$")>;
 
 // ASIMD load, 4 element, all lanes, D-form, B/H/S
 // ASIMD load, 4 element, all lanes, D-form, D
 def : InstRW<[N2Write_8cyc_3L_4V],              (instregex "LD4Rv(8b|4h|2s|1d)$")>;
-def : InstRW<[N2Write_8cyc_3L_4V, WriteAdr],    (instregex "LD4Rv(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_8cyc_3L_4V],    (instregex "LD4Rv(8b|4h|2s|1d)_POST$")>;
 
 // ASIMD load, 4 element, all lanes, Q-form, B/H/S
 // ASIMD load, 4 element, all lanes, Q-form, D
 def : InstRW<[N2Write_8cyc_4L_4V],            (instregex "LD4Rv(16b|8h|4s|2d)$")>;
-def : InstRW<[N2Write_8cyc_4L_4V, WriteAdr],  (instregex "LD4Rv(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_8cyc_4L_4V],  (instregex "LD4Rv(16b|8h|4s|2d)_POST$")>;
 
 // ASIMD store instructions
 // -----------------------------------------------------------------------------
 
 // ASIMD store, 1 element, multiple, 1 reg, D-form
 def : InstRW<[N2Write_2cyc_1L01_1V],           (instregex "ST1Onev(8b|4h|2s|1d)$")>;
-def : InstRW<[N2Write_2cyc_1L01_1V, WriteAdr], (instregex "ST1Onev(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_2cyc_1L01_1V], (instregex "ST1Onev(8b|4h|2s|1d)_POST$")>;
 
 // ASIMD store, 1 element, multiple, 1 reg, Q-form
 def : InstRW<[N2Write_2cyc_1L01_1V],           (instregex "ST1Onev(16b|8h|4s|2d)$")>;
-def : InstRW<[N2Write_2cyc_1L01_1V, WriteAdr], (instregex "ST1Onev(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_2cyc_1L01_1V], (instregex "ST1Onev(16b|8h|4s|2d)_POST$")>;
 
 // ASIMD store, 1 element, multiple, 2 reg, D-form
 def : InstRW<[N2Write_2cyc_1L01_1V],           (instregex "ST1Twov(8b|4h|2s|1d)$")>;
-def : InstRW<[N2Write_2cyc_1L01_1V, WriteAdr], (instregex "ST1Twov(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_2cyc_1L01_1V], (instregex "ST1Twov(8b|4h|2s|1d)_POST$")>;
 
 // ASIMD store, 1 element, multiple, 2 reg, Q-form
 def : InstRW<[N2Write_2cyc_2L01_2V],           (instregex "ST1Twov(16b|8h|4s|2d)$")>;
-def : InstRW<[N2Write_2cyc_2L01_2V, WriteAdr], (instregex "ST1Twov(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_2cyc_2L01_2V], (instregex "ST1Twov(16b|8h|4s|2d)_POST$")>;
 
 // ASIMD store, 1 element, multiple, 3 reg, D-form
 def : InstRW<[N2Write_2cyc_2L01_2V],           (instregex "ST1Threev(8b|4h|2s|1d)$")>;
-def : InstRW<[N2Write_2cyc_2L01_2V, WriteAdr], (instregex "ST1Threev(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_2cyc_2L01_2V], (instregex "ST1Threev(8b|4h|2s|1d)_POST$")>;
 
 // ASIMD store, 1 element, multiple, 3 reg, Q-form
 def : InstRW<[N2Write_2cyc_3L01_3V],           (instregex "ST1Threev(16b|8h|4s|2d)$")>;
-def : InstRW<[N2Write_2cyc_3L01_3V, WriteAdr], (instregex "ST1Threev(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_2cyc_3L01_3V], (instregex "ST1Threev(16b|8h|4s|2d)_POST$")>;
 
 // ASIMD store, 1 element, multiple, 4 reg, D-form
 def : InstRW<[N2Write_2cyc_2L01_2V],           (instregex "ST1Fourv(8b|4h|2s|1d)$")>;
-def : InstRW<[N2Write_2cyc_2L01_2V, WriteAdr], (instregex "ST1Fourv(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_2cyc_2L01_2V], (instregex "ST1Fourv(8b|4h|2s|1d)_POST$")>;
 
 // ASIMD store, 1 element, multiple, 4 reg, Q-form
 def : InstRW<[N2Write_2cyc_4L01_4V],           (instregex "ST1Fourv(16b|8h|4s|2d)$")>;
-def : InstRW<[N2Write_2cyc_4L01_4V, WriteAdr], (instregex "ST1Fourv(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_2cyc_4L01_4V], (instregex "ST1Fourv(16b|8h|4s|2d)_POST$")>;
 
 // ASIMD store, 1 element, one lane, B/H/S
 // ASIMD store, 1 element, one lane, D
 def : InstRW<[N2Write_4cyc_1L01_1V],           (instregex "ST1i(8|16|32|64)$")>;
-def : InstRW<[N2Write_4cyc_1L01_1V, WriteAdr], (instregex "ST1i(8|16|32|64)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_4cyc_1L01_1V], (instregex "ST1i(8|16|32|64)_POST$")>;
 
 // ASIMD store, 2 element, multiple, D-form, B/H/S
 def : InstRW<[N2Write_4cyc_1L01_1V],           (instregex "ST2Twov(8b|4h|2s)$")>;
-def : InstRW<[N2Write_4cyc_1L01_1V, WriteAdr], (instregex "ST2Twov(8b|4h|2s)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_4cyc_1L01_1V], (instregex "ST2Twov(8b|4h|2s)_POST$")>;
 
 // ASIMD store, 2 element, multiple, Q-form, B/H/S
 // ASIMD store, 2 element, multiple, Q-form, D
 def : InstRW<[N2Write_4cyc_2L01_2V],           (instregex "ST2Twov(16b|8h|4s|2d)$")>;
-def : InstRW<[N2Write_4cyc_2L01_2V, WriteAdr], (instregex "ST2Twov(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_4cyc_2L01_2V], (instregex "ST2Twov(16b|8h|4s|2d)_POST$")>;
 
 // ASIMD store, 2 element, one lane, B/H/S
 // ASIMD store, 2 element, one lane, D
 def : InstRW<[N2Write_4cyc_1L01_1V],           (instregex "ST2i(8|16|32|64)$")>;
-def : InstRW<[N2Write_4cyc_1L01_1V, WriteAdr], (instregex "ST2i(8|16|32|64)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_4cyc_1L01_1V], (instregex "ST2i(8|16|32|64)_POST$")>;
 
 // ASIMD store, 3 element, multiple, D-form, B/H/S
 def : InstRW<[N2Write_5cyc_2L01_2V],           (instregex "ST3Threev(8b|4h|2s)$")>;
-def : InstRW<[N2Write_5cyc_2L01_2V, WriteAdr], (instregex "ST3Threev(8b|4h|2s)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_5cyc_2L01_2V], (instregex "ST3Threev(8b|4h|2s)_POST$")>;
 
 // ASIMD store, 3 element, multiple, Q-form, B/H/S
 // ASIMD store, 3 element, multiple, Q-form, D
 def : InstRW<[N2Write_6cyc_3L01_3V],           (instregex "ST3Threev(16b|8h|4s|2d)$")>;
-def : InstRW<[N2Write_6cyc_3L01_3V, WriteAdr], (instregex "ST3Threev(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_6cyc_3L01_3V], (instregex "ST3Threev(16b|8h|4s|2d)_POST$")>;
 
 // ASIMD store, 3 element, one lane, B/H
 // ASIMD store, 3 element, one lane, S
 // ASIMD store, 3 element, one lane, D
 def : InstRW<[N2Write_6cyc_3L01_3V],           (instregex "ST3i(8|16|32|64)$")>;
-def : InstRW<[N2Write_6cyc_3L01_3V, WriteAdr], (instregex "ST3i(8|16|32|64)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_6cyc_3L01_3V], (instregex "ST3i(8|16|32|64)_POST$")>;
 
 // ASIMD store, 4 element, multiple, D-form, B/H/S
 def : InstRW<[N2Write_6cyc_3L01_3V],           (instregex "ST4Fourv(8b|4h|2s)$")>;
-def : InstRW<[N2Write_6cyc_3L01_3V, WriteAdr], (instregex "ST4Fourv(8b|4h|2s)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_6cyc_3L01_3V], (instregex "ST4Fourv(8b|4h|2s)_POST$")>;
 
 // ASIMD store, 4 element, multiple, Q-form, B/H/S
 def : InstRW<[N2Write_7cyc_6L01_6V],           (instregex "ST4Fourv(16b|8h|4s)$")>;
-def : InstRW<[N2Write_7cyc_6L01_6V, WriteAdr], (instregex "ST4Fourv(16b|8h|4s)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_7cyc_6L01_6V], (instregex "ST4Fourv(16b|8h|4s)_POST$")>;
 
 // ASIMD store, 4 element, multiple, Q-form, D
 def : InstRW<[N2Write_5cyc_4L01_4V],           (instregex "ST4Fourv(2d)$")>;
-def : InstRW<[N2Write_5cyc_4L01_4V, WriteAdr], (instregex "ST4Fourv(2d)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_5cyc_4L01_4V], (instregex "ST4Fourv(2d)_POST$")>;
 
 // ASIMD store, 4 element, one lane, B/H/S
 def : InstRW<[N2Write_6cyc_3L01_3V],           (instregex "ST4i(8|16|32)$")>;
-def : InstRW<[N2Write_6cyc_3L01_3V, WriteAdr], (instregex "ST4i(8|16|32)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_6cyc_3L01_3V], (instregex "ST4i(8|16|32)_POST$")>;
 
 // ASIMD store, 4 element, one lane, D
 def : InstRW<[N2Write_4cyc_3L01_3V],            (instregex "ST4i(64)$")>;
-def : InstRW<[N2Write_4cyc_3L01_3V, WriteAdr],  (instregex "ST4i(64)_POST$")>;
+def : InstRW<[WriteAdr, N2Write_4cyc_3L01_3V],  (instregex "ST4i(64)_POST$")>;
 
 // Cryptography extensions
 // -----------------------------------------------------------------------------
diff --git a/llvm/lib/Target/AArch64/AArch64SchedNeoverseV1.td b/llvm/lib/Target/AArch64/AArch64SchedNeoverseV1.td
index 8792aa618a..5c155c936d 100644
--- a/llvm/lib/Target/AArch64/AArch64SchedNeoverseV1.td
+++ b/llvm/lib/Target/AArch64/AArch64SchedNeoverseV1.td
@@ -624,14 +624,14 @@ def : SchedAlias<WriteAdr,   V1Write_1c_1I>;
 // Load pair, immed offset
 def : SchedAlias<WriteLDHi, V1Write_4c_1L>;
 def : InstRW<[V1Write_4c_1L, V1Write_0c_0Z], (instrs LDPWi, LDNPWi)>;
-def : InstRW<[V1Write_4c_1L, V1Write_0c_0Z, WriteAdr],
+def : InstRW<[WriteAdr, V1Write_4c_1L, V1Write_0c_0Z],
              (instrs LDPWpost, LDPWpre)>;
 
 // Load pair, signed immed offset, signed words
 def : InstRW<[V1Write_5c_1I_1L, V1Write_0c_0Z], (instrs LDPSWi)>;
 
 // Load pair, immed post or pre-index, signed words
-def : InstRW<[V1Write_5c_1I_1L, V1Write_0c_0Z, WriteAdr],
+def : InstRW<[WriteAdr, V1Write_5c_1I_1L, V1Write_0c_0Z],
              (instrs LDPSWpost, LDPSWpre)>;
 
 
@@ -735,7 +735,7 @@ def : InstRW<[V1Write_6c_1L, ReadAdrBase], (instregex "^LDR[SDQ]l$",
 
 // Load vector reg, immed post-index
 // Load vector reg, immed pre-index
-def : InstRW<[V1Write_6c_1L, WriteAdr],
+def : InstRW<[WriteAdr, V1Write_6c_1L],
              (instregex "^LDR[BHSDQ](post|pre)$")>;
 
 // Load vector reg, register offset, basic
@@ -756,12 +756,12 @@ def : InstRW<[V1Write_6c_1L, WriteLDHi], (instrs LDPQi, LDNPQi)>;
 
 // Load vector pair, immed post-index, S/D-form
 // Load vector pair, immed pre-index, S/D-form
-def : InstRW<[V1Write_6c_1L, V1Write_0c_0Z, WriteAdr],
+def : InstRW<[WriteAdr, V1Write_6c_1L, V1Write_0c_0Z],
              (instregex "^LDP[SD](pre|post)$")>;
 
 // Load vector pair, immed post-index, Q-form
 // Load vector pair, immed pre-index, Q-form
-def : InstRW<[V1Write_6c_1L, WriteLDHi, WriteAdr],
+def : InstRW<[WriteAdr, V1Write_6c_1L, WriteLDHi],
              (instrs LDPQpost, LDPQpre)>;
 
 
@@ -773,7 +773,7 @@ def : InstRW<[V1Write_2c_1L01_1V01], (instregex "^STUR[BHSDQ]i$")>;
 
 // Store vector reg, immed post-index, B/H/S/D/Q-form
 // Store vector reg, immed pre-index, B/H/S/D/Q-form
-def : InstRW<[V1Write_2c_1L01_1V01, WriteAdr],
+def : InstRW<[WriteAdr, V1Write_2c_1L01_1V01],
              (instregex "^STR[BHSDQ](pre|post)$")>;
 
 // Store vector reg, unsigned immed, B/H/S/D/Q-form
@@ -798,12 +798,12 @@ def : InstRW<[V1Write_2c_1L01_1V01], (instregex "^STN?P[SDQ]i$")>;
 
 // Store vector pair, immed post-index, S/D-form
 // Store vector pair, immed pre-index, S/D-form
-def : InstRW<[V1Write_2c_1L01_1V01, WriteAdr],
+def : InstRW<[WriteAdr, V1Write_2c_1L01_1V01],
              (instregex "^STP[SD](pre|post)$")>;
 
 // Store vector pair, immed post-index, Q-form
 // Store vector pair, immed pre-index, Q-form
-def : InstRW<[V1Write_2c_2L01_1V01, WriteAdr], (instrs STPQpre, STPQpost)>;
+def : InstRW<[WriteAdr, V1Write_2c_2L01_1V01], (instrs STPQpre, STPQpost)>;
 
 
 // ASIMD integer instructions
@@ -1074,31 +1074,31 @@ def : InstRW<[V1Write_5c_1M0_1V], (instregex "^INSvi(8|16|32|64)gpr$")>;
 // ASIMD load, 1 element, multiple, 1 reg
 def : InstRW<[V1Write_6c_1L],
              (instregex "^LD1Onev(8b|16b|4h|8h|2s|4s|1d|2d)$")>;
-def : InstRW<[V1Write_6c_1L, WriteAdr],
+def : InstRW<[WriteAdr, V1Write_6c_1L],
              (instregex "^LD1Onev(8b|16b|4h|8h|2s|4s|1d|2d)_POST$")>;
 
 // ASIMD load, 1 element, multiple, 2 reg
 def : InstRW<[V1Write_6c_2L],
              (instregex "^LD1Twov(8b|16b|4h|8h|2s|4s|1d|2d)$")>;
-def : InstRW<[V1Write_6c_2L, WriteAdr],
+def : InstRW<[WriteAdr, V1Write_6c_2L],
              (instregex "^LD1Twov(8b|16b|4h|8h|2s|4s|1d|2d)_POST$")>;
 
 // ASIMD load, 1 element, multiple, 3 reg
 def : InstRW<[V1Write_6c_3L],
              (instregex "^LD1Threev(8b|16b|4h|8h|2s|4s|1d|2d)$")>;
-def : InstRW<[V1Write_6c_3L, WriteAdr],
+def : InstRW<[WriteAdr, V1Write_6c_3L],
              (instregex "^LD1Threev(8b|16b|4h|8h|2s|4s|1d|2d)_POST$")>;
 
 // ASIMD load, 1 element, multiple, 4 reg, D-form
 def : InstRW<[V1Write_6c_2L],
              (instregex "^LD1Fourv(8b|4h|2s|1d)$")>;
-def : InstRW<[V1Write_6c_2L, WriteAdr],
+def : InstRW<[WriteAdr, V1Write_6c_2L],
              (instregex "^LD1Fourv(8b|4h|2s|1d)_POST$")>;
 
 // ASIMD load, 1 element, multiple, 4 reg, Q-form
 def : InstRW<[V1Write_7c_4L],
              (instregex "^LD1Fourv(16b|8h|4s|2d)$")>;
-def : InstRW<[V1Write_7c_4L, WriteAdr],
+def : InstRW<[WriteAdr, V1Write_7c_4L],
              (instregex "^LD1Fourv(16b|8h|4s|2d)_POST$")>;
 
 // ASIMD load, 1 element, one lane
@@ -1106,20 +1106,20 @@ def : InstRW<[V1Write_7c_4L, WriteAdr],
 def : InstRW<[V1Write_8c_1L_1V],
              (instregex "^LD1(i|Rv)(8|16|32|64)$",
                         "^LD1Rv(8b|16b|4h|8h|2s|4s|1d|2d)$")>;
-def : InstRW<[V1Write_8c_1L_1V, WriteAdr],
+def : InstRW<[WriteAdr, V1Write_8c_1L_1V],
              (instregex "^LD1i(8|16|32|64)_POST$",
                         "^LD1Rv(8b|16b|4h|8h|2s|4s|1d|2d)_POST$")>;
 
 // ASIMD load, 2 element, multiple, D-form
 def : InstRW<[V1Write_8c_1L_2V],
              (instregex "^LD2Twov(8b|4h|2s)$")>;
-def : InstRW<[V1Write_8c_1L_2V, WriteAdr],
+def : InstRW<[WriteAdr, V1Write_8c_1L_2V],
              (instregex "^LD2Twov(8b|4h|2s)_POST$")>;
                         
 // ASIMD load, 2 element, multiple, Q-form
 def : InstRW<[V1Write_8c_2L_2V],
              (instregex "^LD2Twov(16b|8h|4s|2d)$")>;
-def : InstRW<[V1Write_8c_2L_2V, WriteAdr],
+def : InstRW<[WriteAdr, V1Write_8c_2L_2V],
              (instregex "^LD2Twov(16b|8h|4s|2d)_POST$")>;
                         
 // ASIMD load, 2 element, one lane
@@ -1127,7 +1127,7 @@ def : InstRW<[V1Write_8c_2L_2V, WriteAdr],
 def : InstRW<[V1Write_8c_1L_2V],
              (instregex "^LD2i(8|16|32|64)$",
                         "^LD2Rv(8b|16b|4h|8h|2s|4s|1d|2d)$")>;
-def : InstRW<[V1Write_8c_1L_2V, WriteAdr],
+def : InstRW<[WriteAdr, V1Write_8c_1L_2V],
              (instregex "^LD2i(8|16|32|64)_POST$",
                         "^LD2Rv(8b|16b|4h|8h|2s|4s|1d|2d)_POST$")>;
                         
@@ -1138,7 +1138,7 @@ def : InstRW<[V1Write_8c_2L_3V],
              (instregex "^LD3Threev(8b|4h|2s)$",
                         "^LD3i(8|16|32|64)$",
                         "^LD3Rv(8b|16b|4h|8h|2s|4s|1d|2d)$")>;
-def : InstRW<[V1Write_8c_2L_3V, WriteAdr],
+def : InstRW<[WriteAdr, V1Write_8c_2L_3V],
              (instregex "^LD3Threev(8b|4h|2s)_POST$",
                         "^LD3i(8|16|32|64)_POST$",
                         "^LD3Rv(8b|16b|4h|8h|2s|4s|1d|2d)_POST$")>;
@@ -1146,7 +1146,7 @@ def : InstRW<[V1Write_8c_2L_3V, WriteAdr],
 // ASIMD load, 3 element, multiple, Q-form
 def : InstRW<[V1Write_8c_3L_3V],
              (instregex "^LD3Threev(16b|8h|4s|2d)$")>;
-def : InstRW<[V1Write_8c_3L_3V, WriteAdr],
+def : InstRW<[WriteAdr, V1Write_8c_3L_3V],
              (instregex "^LD3Threev(16b|8h|4s|2d)_POST$")>;
 
 // ASIMD load, 4 element, multiple, D-form
@@ -1156,7 +1156,7 @@ def : InstRW<[V1Write_8c_3L_4V],
              (instregex "^LD4Fourv(8b|4h|2s)$",
                         "^LD4i(8|16|32|64)$",
                         "^LD4Rv(8b|16b|4h|8h|2s|4s|1d|2d)$")>;
-def : InstRW<[V1Write_8c_3L_4V, WriteAdr], 
+def : InstRW<[WriteAdr, V1Write_8c_3L_4V],
              (instregex "^LD4Fourv(8b|4h|2s)_POST$",
                         "^LD4i(8|16|32|64)_POST$",
                         "^LD4Rv(8b|16b|4h|8h|2s|4s|1d|2d)_POST$")>;
@@ -1164,7 +1164,7 @@ def : InstRW<[V1Write_8c_3L_4V, WriteAdr],
 // ASIMD load, 4 element, multiple, Q-form
 def : InstRW<[V1Write_9c_4L_4V],
              (instregex "^LD4Fourv(16b|8h|4s|2d)$")>;
-def : InstRW<[V1Write_9c_4L_4V, WriteAdr],
+def : InstRW<[WriteAdr, V1Write_9c_4L_4V],
              (instregex "^LD4Fourv(16b|8h|4s|2d)_POST$")>;
 
 
@@ -1176,7 +1176,7 @@ def : InstRW<[V1Write_9c_4L_4V, WriteAdr],
 def : InstRW<[V1Write_2c_1L01_1V01],
              (instregex "^ST1Onev(8b|16b|4h|8h|2s|4s|1d|2d)$",
                         "^ST1Twov(8b|4h|2s|1d)$")>;
-def : InstRW<[V1Write_2c_1L01_1V01, WriteAdr],
+def : InstRW<[WriteAdr, V1Write_2c_1L01_1V01],
              (instregex "^ST1Onev(8b|16b|4h|8h|2s|4s|1d|2d)_POST$",
                         "^ST1Twov(8b|4h|2s|1d)_POST$")>;
 
@@ -1187,7 +1187,7 @@ def : InstRW<[V1Write_2c_2L01_2V01],
              (instregex "^ST1Twov(16b|8h|4s|2d)$",
                         "^ST1Threev(8b|4h|2s|1d)$",
                         "^ST1Fourv(8b|4h|2s|1d)$")>;
-def : InstRW<[V1Write_2c_2L01_2V01, WriteAdr],
+def : InstRW<[WriteAdr, V1Write_2c_2L01_2V01],
              (instregex "^ST1Twov(16b|8h|4s|2d)_POST$",
                         "^ST1Threev(8b|4h|2s|1d)_POST$",
                         "^ST1Fourv(8b|4h|2s|1d)_POST$")>;
@@ -1195,13 +1195,13 @@ def : InstRW<[V1Write_2c_2L01_2V01, WriteAdr],
 // ASIMD store, 1 element, multiple, 3 reg, Q-form
 def : InstRW<[V1Write_2c_3L01_3V01],
              (instregex "^ST1Threev(16b|8h|4s|2d)$")>;
-def : InstRW<[V1Write_2c_3L01_3V01, WriteAdr],
+def : InstRW<[WriteAdr, V1Write_2c_3L01_3V01],
              (instregex "^ST1Threev(16b|8h|4s|2d)_POST$")>;
 
 // ASIMD store, 1 element, multiple, 4 reg, Q-form
 def : InstRW<[V1Write_2c_4L01_4V01],
              (instregex "^ST1Fourv(16b|8h|4s|2d)$")>;
-def : InstRW<[V1Write_2c_4L01_4V01, WriteAdr],
+def : InstRW<[WriteAdr, V1Write_2c_4L01_4V01],
              (instregex "^ST1Fourv(16b|8h|4s|2d)_POST$")>;
 
 // ASIMD store, 1 element, one lane
@@ -1211,7 +1211,7 @@ def : InstRW<[V1Write_4c_1L01_1V01],
              (instregex "^ST1i(8|16|32|64)$",
                         "^ST2Twov(8b|4h|2s)$",
                         "^ST2i(8|16|32|64)$")>;
-def : InstRW<[V1Write_4c_1L01_1V01, WriteAdr],
+def : InstRW<[WriteAdr, V1Write_4c_1L01_1V01],
              (instregex "^ST1i(8|16|32|64)_POST$",
                         "^ST2Twov(8b|4h|2s)_POST$",
                         "^ST2i(8|16|32|64)_POST$")>;
@@ -1225,7 +1225,7 @@ def : InstRW<[V1Write_4c_2L01_2V01],
                         "^ST3Threev(8b|4h|2s)$",
                         "^ST3i(8|16|32|64)$",
                         "^ST4i64$")>;
-def : InstRW<[V1Write_4c_2L01_2V01, WriteAdr],
+def : InstRW<[WriteAdr, V1Write_4c_2L01_2V01],
              (instregex "^ST2Twov(16b|8h|4s|2d)_POST$",
                         "^ST3Threev(8b|4h|2s)_POST$",
                         "^ST3i(8|16|32|64)_POST$",
@@ -1234,31 +1234,31 @@ def : InstRW<[V1Write_4c_2L01_2V01, WriteAdr],
 // ASIMD store, 3 element, multiple, Q-form
 def : InstRW<[V1Write_5c_3L01_3V01],
              (instregex "^ST3Threev(16b|8h|4s|2d)$")>;
-def : InstRW<[V1Write_5c_3L01_3V01, WriteAdr],
+def : InstRW<[WriteAdr, V1Write_5c_3L01_3V01],
              (instregex "^ST3Threev(16b|8h|4s|2d)_POST$")>;
 
 // ASIMD store, 4 element, multiple, D-form
 def : InstRW<[V1Write_6c_3L01_3V01],
              (instregex "^ST4Fourv(8b|4h|2s)$")>;
-def : InstRW<[V1Write_6c_3L01_3V01, WriteAdr],
+def : InstRW<[WriteAdr, V1Write_6c_3L01_3V01],
              (instregex "^ST4Fourv(8b|4h|2s)_POST$")>;
 
 // ASIMD store, 4 element, multiple, Q-form, B/H/S
 def : InstRW<[V1Write_7c_6L01_6V01],
              (instregex "^ST4Fourv(16b|8h|4s)$")>;
-def : InstRW<[V1Write_7c_6L01_6V01, WriteAdr],
+def : InstRW<[WriteAdr, V1Write_7c_6L01_6V01],
              (instregex "^ST4Fourv(16b|8h|4s)_POST$")>;
 
 // ASIMD store, 4 element, multiple, Q-form, D
 def : InstRW<[V1Write_4c_4L01_4V01],
              (instrs ST4Fourv2d)>;
-def : InstRW<[V1Write_4c_4L01_4V01, WriteAdr],
+def : InstRW<[WriteAdr, V1Write_4c_4L01_4V01],
              (instrs ST4Fourv2d_POST)>;
 
 // ASIMD store, 4 element, one lane, B/H/S
 def : InstRW<[V1Write_6c_3L_3V],
              (instregex "^ST4i(8|16|32)$")>;
-def : InstRW<[V1Write_6c_3L_3V, WriteAdr],
+def : InstRW<[WriteAdr, V1Write_6c_3L_3V],
              (instregex "^ST4i(8|16|32)_POST$")>;
 
 
diff --git a/llvm/lib/Target/AArch64/AArch64SchedTSV110.td b/llvm/lib/Target/AArch64/AArch64SchedTSV110.td
index af4a0176e4..9e5060f1f3 100644
--- a/llvm/lib/Target/AArch64/AArch64SchedTSV110.td
+++ b/llvm/lib/Target/AArch64/AArch64SchedTSV110.td
@@ -443,8 +443,8 @@ def : InstRW<[TSV110Wr_4cyc_1LdSt],     (instrs LDRSWl)>;
 def : InstRW<[TSV110Wr_4cyc_1LdSt],     (instregex "^LDR(BB|HH|W|X)ui$")>;
 def : InstRW<[TSV110Wr_4cyc_1LdSt],     (instregex "^LDRS(BW|BX|HW|HX|W)ui$")>;
 
-def : InstRW<[TSV110Wr_4cyc_1LdSt, WriteAdr],     (instregex "^LDR(BB|HH|W|X)(post|pre)$")>;
-def : InstRW<[TSV110Wr_4cyc_1LdSt, WriteAdr],     (instregex "^LDRS(BW|BX|HW|HX|W)(post|pre)$")>;
+def : InstRW<[WriteAdr, TSV110Wr_4cyc_1LdSt],     (instregex "^LDR(BB|HH|W|X)(post|pre)$")>;
+def : InstRW<[WriteAdr, TSV110Wr_4cyc_1LdSt],     (instregex "^LDRS(BW|BX|HW|HX|W)(post|pre)$")>;
 
 def : InstRW<[TSV110Wr_4cyc_1LdSt],     (instregex "^LDTR(B|H|W|X)i$")>;
 def : InstRW<[TSV110Wr_4cyc_1LdSt],     (instregex "^LDUR(BB|HH|W|X)i$")>;
@@ -453,11 +453,11 @@ def : InstRW<[TSV110Wr_4cyc_1LdSt],     (instregex "^LDURS(BW|BX|HW|HX|W)i$")>;
 
 def : InstRW<[TSV110Wr_4cyc_1LdSt, WriteLDHi],     (instregex "^LDNP(W|X)i$")>;
 def : InstRW<[TSV110Wr_4cyc_1LdSt, WriteLDHi],     (instregex "^LDP(W|X)i$")>;
-def : InstRW<[TSV110Wr_4cyc_1LdSt_1ALUAB, WriteLDHi, WriteAdr],(instregex "^LDP(W|X)(post|pre)$")>;
+def : InstRW<[WriteAdr, TSV110Wr_4cyc_1LdSt_1ALUAB, WriteLDHi],(instregex "^LDP(W|X)(post|pre)$")>;
 
 def : InstRW<[TSV110Wr_4cyc_1LdSt, WriteLDHi],           (instrs LDPSWi)>;
-def : InstRW<[TSV110Wr_4cyc_1LdSt, WriteLDHi, WriteAdr], (instrs LDPSWpost)>;
-def : InstRW<[TSV110Wr_4cyc_1LdSt, WriteLDHi, WriteAdr], (instrs LDPSWpre)>;
+def : InstRW<[WriteAdr, TSV110Wr_4cyc_1LdSt, WriteLDHi], (instrs LDPSWpost)>;
+def : InstRW<[WriteAdr, TSV110Wr_4cyc_1LdSt, WriteLDHi], (instrs LDPSWpre)>;
 
 def : InstRW<[TSV110Wr_4cyc_1LdSt],     (instrs PRFMl)>;
 def : InstRW<[TSV110Wr_4cyc_1LdSt],     (instrs PRFUMi)>;
@@ -469,13 +469,13 @@ def : InstRW<[TSV110Wr_4cyc_1LdSt],     (instregex "^PRFMro(W|X)$")>;
 // -----------------------------------------------------------------------------
 
 def : InstRW<[TSV110Wr_1cyc_1LdSt],            (instregex "^STN?P(W|X)i$")>;
-def : InstRW<[TSV110Wr_1cyc_1LdSt, WriteAdr],  (instregex "^STP(W|X)(post|pre)$")>;
+def : InstRW<[WriteAdr, TSV110Wr_1cyc_1LdSt],  (instregex "^STP(W|X)(post|pre)$")>;
 def : InstRW<[TSV110Wr_1cyc_1LdSt],            (instregex "^STUR(BB|HH|W|X)i$")>;
 def : InstRW<[TSV110Wr_1cyc_1LdSt],            (instregex "^STTR(B|H|W|X)i$")>;
 def : InstRW<[TSV110Wr_1cyc_1LdSt],            (instregex "^STR(BB|HH|W|X)ui$")>;
 
-def : InstRW<[TSV110Wr_1cyc_1LdSt, WriteAdr],  (instregex "^STR(BB|HH|W|X)(post|pre)$")>;
-def : InstRW<[TSV110Wr_1cyc_1LdSt, WriteAdr],  (instregex "^STR(BB|HH|W|X)ro(W|X)$")>;
+def : InstRW<[WriteAdr, TSV110Wr_1cyc_1LdSt],  (instregex "^STR(BB|HH|W|X)(post|pre)$")>;
+def : InstRW<[WriteAdr, TSV110Wr_1cyc_1LdSt],  (instregex "^STR(BB|HH|W|X)ro(W|X)$")>;
 
 
 // FP Data Processing Instructions
@@ -524,11 +524,11 @@ def : InstRW<[TSV110Wr_2cyc_1F],      (instregex "^FMOV[SD][ir]$")>;
 
 def : InstRW<[TSV110Wr_5cyc_1LdSt],                      (instregex "^LDR[DSQ]l")>;
 def : InstRW<[TSV110Wr_5cyc_1LdSt],                      (instregex "^LDUR[BDHSQ]i")>;
-def : InstRW<[TSV110Wr_5cyc_1LdSt, WriteAdr],            (instregex "^LDR[BDHSQ](post|pre)")>;
+def : InstRW<[WriteAdr, TSV110Wr_5cyc_1LdSt],            (instregex "^LDR[BDHSQ](post|pre)")>;
 def : InstRW<[TSV110Wr_5cyc_1LdSt],                      (instregex "^LDR[BDHSQ]ui")>;
 def : InstRW<[TSV110Wr_6cyc_1LdSt_1ALUAB, ReadAdrBase],  (instregex "^LDR(Q|D|H|S|B)ro(W|X)$")>;
 def : InstRW<[TSV110Wr_5cyc_1LdSt, WriteLDHi],           (instregex "^LDN?P[DQS]i")>;
-def : InstRW<[TSV110Wr_5cyc_1LdSt, WriteLDHi, WriteAdr], (instregex "^LDP[DQS](post|pre)")>;
+def : InstRW<[WriteAdr, TSV110Wr_5cyc_1LdSt, WriteLDHi], (instregex "^LDP[DQS](post|pre)")>;
 
 
 // FP Store Instructions
@@ -539,7 +539,7 @@ def : InstRW<[TSV110Wr_1cyc_1LdSt_1ALUAB, ReadAdrBase], (instregex "^STR[BHSDQ](
 def : InstRW<[TSV110Wr_1cyc_1LdSt],                     (instregex "^STR[BHSDQ]ui")>;
 def : InstRW<[TSV110Wr_2cyc_1LdSt_1ALUAB, ReadAdrBase], (instregex "^STR[BHSDQ]ro[WX]")>;
 def : InstRW<[TSV110Wr_2cyc_2LdSt],                     (instregex "^STN?P[SDQ]i")>;
-def : InstRW<[TSV110Wr_2cyc_2LdSt, WriteAdr],           (instregex "^STP[SDQ](post|pre)")>;
+def : InstRW<[WriteAdr, TSV110Wr_2cyc_2LdSt],           (instregex "^STP[SDQ](post|pre)")>;
 
 
 // ASIMD Integer Instructions
@@ -704,70 +704,70 @@ def : InstRW<[TSV110Wr_3cyc_1F], (instregex "^[FU](RECP|RSQRT)(E|X)v")>;
 // -----------------------------------------------------------------------------
 
 def : InstRW<[TSV110Wr_7cyc_1F_1LdSt],            (instregex "^LD1Rv(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
-def : InstRW<[TSV110Wr_7cyc_1F_1LdSt, WriteAdr],  (instregex "^LD1Rv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, TSV110Wr_7cyc_1F_1LdSt],  (instregex "^LD1Rv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
 def : InstRW<[TSV110Wr_7cyc_2F_1LdSt],            (instregex "^LD2Rv(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
-def : InstRW<[TSV110Wr_7cyc_2F_1LdSt, WriteAdr],  (instregex "^LD2Rv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, TSV110Wr_7cyc_2F_1LdSt],  (instregex "^LD2Rv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
 def : InstRW<[TSV110Wr_8cyc_3F_1LdSt],            (instregex "^LD3Rv(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
-def : InstRW<[TSV110Wr_8cyc_3F_1LdSt, WriteAdr],  (instregex "^LD3Rv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, TSV110Wr_8cyc_3F_1LdSt],  (instregex "^LD3Rv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
 def : InstRW<[TSV110Wr_8cyc_3F_2LdSt],            (instregex "^LD4Rv(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
-def : InstRW<[TSV110Wr_8cyc_3F_2LdSt, WriteAdr],  (instregex "^LD4Rv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, TSV110Wr_8cyc_3F_2LdSt],  (instregex "^LD4Rv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
 
 def  : InstRW<[TSV110Wr_7cyc_1F_1LdSt],           (instregex "LD1i(8|16|32|64)$")>;
-def  : InstRW<[TSV110Wr_7cyc_1F_1LdSt, WriteAdr], (instregex "LD1i(8|16|32|64)_POST$")>;
+def  : InstRW<[WriteAdr, TSV110Wr_7cyc_1F_1LdSt], (instregex "LD1i(8|16|32|64)_POST$")>;
 def  : InstRW<[TSV110Wr_7cyc_2F_1LdSt],           (instregex "LD2i(8|16|32|64)$")>;
-def  : InstRW<[TSV110Wr_7cyc_2F_1LdSt, WriteAdr], (instregex "LD2i(8|16|32|64)_POST$")>;
+def  : InstRW<[WriteAdr, TSV110Wr_7cyc_2F_1LdSt], (instregex "LD2i(8|16|32|64)_POST$")>;
 def  : InstRW<[TSV110Wr_8cyc_3F_1LdSt],           (instregex "LD3i(8|16|32|64)$")>;
-def  : InstRW<[TSV110Wr_8cyc_3F_1LdSt, WriteAdr], (instregex "LD3i(8|16|32|64)_POST$")>;
+def  : InstRW<[WriteAdr, TSV110Wr_8cyc_3F_1LdSt], (instregex "LD3i(8|16|32|64)_POST$")>;
 def  : InstRW<[TSV110Wr_8cyc_3F_2LdSt],           (instregex "LD4i(8|16|32|64)$")>;
-def  : InstRW<[TSV110Wr_8cyc_3F_2LdSt, WriteAdr], (instregex "LD4i(8|16|32|64)_POST$")>;
+def  : InstRW<[WriteAdr, TSV110Wr_8cyc_3F_2LdSt], (instregex "LD4i(8|16|32|64)_POST$")>;
 
 def : InstRW<[TSV110Wr_5cyc_1LdSt],               (instregex "^LD1Onev(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
-def : InstRW<[TSV110Wr_5cyc_1LdSt, WriteAdr],     (instregex "^LD1Onev(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, TSV110Wr_5cyc_1LdSt],     (instregex "^LD1Onev(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
 def : InstRW<[TSV110Wr_5cyc_1LdSt],               (instregex "^LD1Twov(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
-def : InstRW<[TSV110Wr_5cyc_1LdSt, WriteAdr],     (instregex "^LD1Twov(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, TSV110Wr_5cyc_1LdSt],     (instregex "^LD1Twov(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
 def : InstRW<[TSV110Wr_6cyc_3LdSt],               (instregex "^LD1Threev(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
-def : InstRW<[TSV110Wr_6cyc_3LdSt, WriteAdr],     (instregex "^LD1Threev(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, TSV110Wr_6cyc_3LdSt],     (instregex "^LD1Threev(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
 def : InstRW<[TSV110Wr_6cyc_2LdSt],               (instregex "^LD1Fourv(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
-def : InstRW<[TSV110Wr_6cyc_2LdSt, WriteAdr],     (instregex "^LD1Fourv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, TSV110Wr_6cyc_2LdSt],     (instregex "^LD1Fourv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
 
 def : InstRW<[TSV110Wr_7cyc_2F_1LdSt],            (instregex "^LD2Twov(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
-def : InstRW<[TSV110Wr_7cyc_2F_1LdSt, WriteAdr],  (instregex "^LD2Twov(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, TSV110Wr_7cyc_2F_1LdSt],  (instregex "^LD2Twov(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
 
 def : InstRW<[TSV110Wr_8cyc_3F_1LdSt],            (instregex "^LD3Threev(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
-def : InstRW<[TSV110Wr_8cyc_3F_1LdSt, WriteAdr],  (instregex "^LD3Threev(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, TSV110Wr_8cyc_3F_1LdSt],  (instregex "^LD3Threev(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
 
 def : InstRW<[TSV110Wr_10cyc_4F_4LdSt],           (instregex "^LD4Fourv(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
-def : InstRW<[TSV110Wr_10cyc_4F_4LdSt, WriteAdr], (instregex "^LD4Fourv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, TSV110Wr_10cyc_4F_4LdSt], (instregex "^LD4Fourv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
 
 
 // ASIMD Store Instructions
 // -----------------------------------------------------------------------------
 
 def  : InstRW<[TSV110Wr_3cyc_1F],             (instregex "ST1i(8|16|32|64)$")>;
-def  : InstRW<[TSV110Wr_3cyc_1F, WriteAdr],   (instregex "ST1i(8|16|32|64)_POST$")>;
+def  : InstRW<[WriteAdr, TSV110Wr_3cyc_1F],   (instregex "ST1i(8|16|32|64)_POST$")>;
 def  : InstRW<[TSV110Wr_4cyc_1F],             (instregex "ST2i(8|16|32|64)$")>;
-def  : InstRW<[TSV110Wr_4cyc_1F, WriteAdr],   (instregex "ST2i(8|16|32|64)_POST$")>;
+def  : InstRW<[WriteAdr, TSV110Wr_4cyc_1F],   (instregex "ST2i(8|16|32|64)_POST$")>;
 def  : InstRW<[TSV110Wr_5cyc_1F],             (instregex "ST3i(8|16|32|64)$")>;
-def  : InstRW<[TSV110Wr_5cyc_1F, WriteAdr],   (instregex "ST3i(8|16|32|64)_POST$")>;
+def  : InstRW<[WriteAdr, TSV110Wr_5cyc_1F],   (instregex "ST3i(8|16|32|64)_POST$")>;
 def  : InstRW<[TSV110Wr_6cyc_1F],             (instregex "ST4i(8|16|32|64)$")>;
-def  : InstRW<[TSV110Wr_6cyc_1F, WriteAdr],   (instregex "ST4i(8|16|32|64)_POST$")>;
+def  : InstRW<[WriteAdr, TSV110Wr_6cyc_1F],   (instregex "ST4i(8|16|32|64)_POST$")>;
 
 def : InstRW<[TSV110Wr_3cyc_1F],              (instregex "^ST1Onev(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
-def : InstRW<[TSV110Wr_3cyc_1F, WriteAdr],    (instregex "^ST1Onev(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, TSV110Wr_3cyc_1F],    (instregex "^ST1Onev(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
 def : InstRW<[TSV110Wr_4cyc_1F],              (instregex "^ST1Twov(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
-def : InstRW<[TSV110Wr_4cyc_1F, WriteAdr],    (instregex "^ST1Twov(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, TSV110Wr_4cyc_1F],    (instregex "^ST1Twov(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
 def : InstRW<[TSV110Wr_5cyc_1F],              (instregex "^ST1Threev(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
-def : InstRW<[TSV110Wr_5cyc_1F, WriteAdr],    (instregex "^ST1Threev(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, TSV110Wr_5cyc_1F],    (instregex "^ST1Threev(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
 def : InstRW<[TSV110Wr_6cyc_1F],              (instregex "^ST1Fourv(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
-def : InstRW<[TSV110Wr_6cyc_1F, WriteAdr],    (instregex "^ST1Fourv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, TSV110Wr_6cyc_1F],    (instregex "^ST1Fourv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
 
 def : InstRW<[TSV110Wr_4cyc_1F],              (instregex "^ST2Twov(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
-def : InstRW<[TSV110Wr_4cyc_1F, WriteAdr],    (instregex "^ST2Twov(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, TSV110Wr_4cyc_1F],    (instregex "^ST2Twov(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
 
 def : InstRW<[TSV110Wr_5cyc_1F],              (instregex "^ST3Threev(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
-def : InstRW<[TSV110Wr_5cyc_1F, WriteAdr],    (instregex "^ST3Threev(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, TSV110Wr_5cyc_1F],    (instregex "^ST3Threev(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
 
 def : InstRW<[TSV110Wr_8cyc_1F],              (instregex "^ST4Fourv(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
-def : InstRW<[TSV110Wr_8cyc_1F, WriteAdr],    (instregex "^ST4Fourv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, TSV110Wr_8cyc_1F],    (instregex "^ST4Fourv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
 
 } // SchedModel = TSV110Model
diff --git a/llvm/lib/Target/AArch64/AArch64StackTagging.cpp b/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
index 63bca97d73..97334193ba 100644
--- a/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
+++ b/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
@@ -12,7 +12,6 @@
 #include "AArch64Subtarget.h"
 #include "AArch64TargetMachine.h"
 #include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/DepthFirstIterator.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
diff --git a/llvm/lib/Target/AArch64/AArch64StackTaggingPreRA.cpp b/llvm/lib/Target/AArch64/AArch64StackTaggingPreRA.cpp
index 41cd405c89..090c63633d 100644
--- a/llvm/lib/Target/AArch64/AArch64StackTaggingPreRA.cpp
+++ b/llvm/lib/Target/AArch64/AArch64StackTaggingPreRA.cpp
@@ -10,7 +10,6 @@
 #include "AArch64.h"
 #include "AArch64MachineFunctionInfo.h"
 #include "AArch64InstrInfo.h"
-#include "llvm/ADT/DepthFirstIterator.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/CodeGen/MachineBranchProbabilityInfo.h"
diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
index 8946c0b71e..e3c3bff8e3 100644
--- a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
+++ b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
@@ -70,6 +70,13 @@ static cl::opt<bool> ForceStreamingCompatibleSVE(
         "Force the use of streaming-compatible SVE code for all functions"),
     cl::Hidden);
 
+static cl::opt<AArch64PAuth::AuthCheckMethod>
+    AuthenticatedLRCheckMethod("aarch64-authenticated-lr-check-method",
+                               cl::Hidden,
+                               cl::desc("Override the variant of check applied "
+                                        "to authenticated LR during tail call"),
+                               cl::values(AUTH_CHECK_METHOD_CL_VALUES_LR));
+
 unsigned AArch64Subtarget::getVectorInsertExtractBaseCost() const {
   if (OverrideVectorInsertExtractBaseCost.getNumOccurrences() > 0)
     return OverrideVectorInsertExtractBaseCost;
@@ -335,6 +342,8 @@ AArch64Subtarget::AArch64Subtarget(const Triple &TT, StringRef CPU,
   // X29 is named FP, so we can't use TRI->getName to check X29.
   if (ReservedRegNames.count("X29") || ReservedRegNames.count("FP"))
     ReserveXRegisterForRA.set(29);
+
+  AddressCheckPSV.reset(new AddressCheckPseudoSourceValue(TM));
 }
 
 const CallLowering *AArch64Subtarget::getCallLowering() const {
@@ -490,3 +499,26 @@ bool AArch64Subtarget::isSVEAvailable() const{
   // as we don't yet support the feature in LLVM.
   return hasSVE() && !isStreaming() && !isStreamingCompatible();
 }
+
+// If return address signing is enabled, tail calls are emitted as follows:
+//
+// ```
+//   <authenticate LR>
+//   <check LR>
+//   TCRETURN          ; the callee may sign and spill the LR in its prologue
+// ```
+//
+// LR may require explicit checking because if FEAT_FPAC is not implemented
+// and LR was tampered with, then `<authenticate LR>` will not generate an
+// exception on its own. Later, if the callee spills the signed LR value and
+// neither FEAT_PAuth2 nor FEAT_EPAC are implemented, the valid PAC replaces
+// the higher bits of LR thus hiding the authentication failure.
+AArch64PAuth::AuthCheckMethod
+AArch64Subtarget::getAuthenticatedLRCheckMethod() const {
+  if (AuthenticatedLRCheckMethod.getNumOccurrences())
+    return AuthenticatedLRCheckMethod;
+
+  // At now, use None by default because checks may introduce an unexpected
+  // performance regression or incompatibility with execute-only mappings.
+  return AArch64PAuth::AuthCheckMethod::None;
+}
diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.h b/llvm/lib/Target/AArch64/AArch64Subtarget.h
index 5d0fd7f9f4..b91c5c81ed 100644
--- a/llvm/lib/Target/AArch64/AArch64Subtarget.h
+++ b/llvm/lib/Target/AArch64/AArch64Subtarget.h
@@ -16,6 +16,7 @@
 #include "AArch64FrameLowering.h"
 #include "AArch64ISelLowering.h"
 #include "AArch64InstrInfo.h"
+#include "AArch64PointerAuth.h"
 #include "AArch64RegisterInfo.h"
 #include "AArch64SelectionDAGInfo.h"
 #include "llvm/CodeGen/GlobalISel/CallLowering.h"
@@ -432,6 +433,32 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo {
       return "__security_check_cookie_arm64ec";
     return "__security_check_cookie";
   }
+
+  /// Choose a method of checking LR before performing a tail call.
+  AArch64PAuth::AuthCheckMethod getAuthenticatedLRCheckMethod() const;
+
+  const PseudoSourceValue *getAddressCheckPSV() const {
+    return AddressCheckPSV.get();
+  }
+
+private:
+  /// Pseudo value representing memory load performed to check an address.
+  ///
+  /// This load operation is solely used for its side-effects: if the address
+  /// is not mapped (or not readable), it triggers CPU exception, otherwise
+  /// execution proceeds and the value is not used.
+  class AddressCheckPseudoSourceValue : public PseudoSourceValue {
+  public:
+    AddressCheckPseudoSourceValue(const TargetMachine &TM)
+        : PseudoSourceValue(TargetCustom, TM) {}
+
+    bool isConstant(const MachineFrameInfo *) const override { return false; }
+    bool isAliased(const MachineFrameInfo *) const override { return true; }
+    bool mayAlias(const MachineFrameInfo *) const override { return true; }
+    void printCustom(raw_ostream &OS) const override { OS << "AddressCheck"; }
+  };
+
+  std::unique_ptr<AddressCheckPseudoSourceValue> AddressCheckPSV;
 };
 } // End llvm namespace
 
diff --git a/llvm/lib/Target/AArch64/AArch64SystemOperands.td b/llvm/lib/Target/AArch64/AArch64SystemOperands.td
index 3e7d4d81b2..96afef6228 100644
--- a/llvm/lib/Target/AArch64/AArch64SystemOperands.td
+++ b/llvm/lib/Target/AArch64/AArch64SystemOperands.td
@@ -743,6 +743,7 @@ def : ROSysReg<"ID_AA64AFR1_EL1",     0b11, 0b000, 0b0000, 0b0101, 0b101>;
 def : ROSysReg<"ID_AA64ISAR0_EL1",    0b11, 0b000, 0b0000, 0b0110, 0b000>;
 def : ROSysReg<"ID_AA64ISAR1_EL1",    0b11, 0b000, 0b0000, 0b0110, 0b001>;
 def : ROSysReg<"ID_AA64ISAR2_EL1",    0b11, 0b000, 0b0000, 0b0110, 0b010>;
+def : ROSysReg<"ID_AA64ISAR3_EL1",    0b11, 0b000, 0b0000, 0b0110, 0b011>;
 def : ROSysReg<"ID_AA64MMFR0_EL1",    0b11, 0b000, 0b0000, 0b0111, 0b000>;
 def : ROSysReg<"ID_AA64MMFR1_EL1",    0b11, 0b000, 0b0000, 0b0111, 0b001>;
 def : ROSysReg<"ID_AA64MMFR2_EL1",    0b11, 0b000, 0b0000, 0b0111, 0b010>;
@@ -1927,3 +1928,12 @@ def : RWSysReg<"PFAR_EL2",          0b11, 0b100, 0b0110, 0b0000, 0b101>;
 // v9.4a Exception-based event profiling (FEAT_EBEP)
 //                                  Op0   Op1    CRn     CRm     Op2
 def : RWSysReg<"PM",                0b11, 0b000, 0b0100, 0b0011, 0b001>;
+
+// 2023 ISA Extension
+// AArch64 Floating-point Mode Register controls behaviors of the FP8
+// instructions (FEAT_FPMR)
+let Requires = [{ {AArch64::FeatureFPMR} }] in {
+//                                 Op0   Op1    CRn     CRm     Op2
+def : ROSysReg<"ID_AA64FPFR0_EL1", 0b11, 0b000, 0b0000, 0b0100, 0b111>;
+def : RWSysReg<"FPMR",             0b11, 0b011, 0b0100, 0b0100, 0b010>;
+}
diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
index 3d818c76bd..fcc30a7cfc 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
@@ -200,7 +200,7 @@ static cl::opt<bool> EnableGISelLoadStoreOptPostLegal(
 static cl::opt<bool>
     EnableSinkFold("aarch64-enable-sink-fold",
                    cl::desc("Enable sinking and folding of instruction copies"),
-                   cl::init(false), cl::Hidden);
+                   cl::init(true), cl::Hidden);
 
 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64Target() {
   // Register the target.
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index cded28054f..d8a0e68d71 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -798,10 +798,31 @@ instCombineConvertFromSVBool(InstCombiner &IC, IntrinsicInst &II) {
   return IC.replaceInstUsesWith(II, EarliestReplacement);
 }
 
+static bool isAllActivePredicate(Value *Pred) {
+  // Look through convert.from.svbool(convert.to.svbool(...) chain.
+  Value *UncastedPred;
+  if (match(Pred, m_Intrinsic<Intrinsic::aarch64_sve_convert_from_svbool>(
+                      m_Intrinsic<Intrinsic::aarch64_sve_convert_to_svbool>(
+                          m_Value(UncastedPred)))))
+    // If the predicate has the same or less lanes than the uncasted
+    // predicate then we know the casting has no effect.
+    if (cast<ScalableVectorType>(Pred->getType())->getMinNumElements() <=
+        cast<ScalableVectorType>(UncastedPred->getType())->getMinNumElements())
+      Pred = UncastedPred;
+
+  return match(Pred, m_Intrinsic<Intrinsic::aarch64_sve_ptrue>(
+                         m_ConstantInt<AArch64SVEPredPattern::all>()));
+}
+
 static std::optional<Instruction *> instCombineSVESel(InstCombiner &IC,
                                                       IntrinsicInst &II) {
-  auto Select = IC.Builder.CreateSelect(II.getOperand(0), II.getOperand(1),
-                                        II.getOperand(2));
+  // svsel(ptrue, x, y) => x
+  auto *OpPredicate = II.getOperand(0);
+  if (isAllActivePredicate(OpPredicate))
+    return IC.replaceInstUsesWith(II, II.getOperand(1));
+
+  auto Select =
+      IC.Builder.CreateSelect(OpPredicate, II.getOperand(1), II.getOperand(2));
   return IC.replaceInstUsesWith(II, Select);
 }
 
@@ -1200,22 +1221,6 @@ instCombineSVEVectorFuseMulAddSub(InstCombiner &IC, IntrinsicInst &II,
   return IC.replaceInstUsesWith(II, Res);
 }
 
-static bool isAllActivePredicate(Value *Pred) {
-  // Look through convert.from.svbool(convert.to.svbool(...) chain.
-  Value *UncastedPred;
-  if (match(Pred, m_Intrinsic<Intrinsic::aarch64_sve_convert_from_svbool>(
-                      m_Intrinsic<Intrinsic::aarch64_sve_convert_to_svbool>(
-                          m_Value(UncastedPred)))))
-    // If the predicate has the same or less lanes than the uncasted
-    // predicate then we know the casting has no effect.
-    if (cast<ScalableVectorType>(Pred->getType())->getMinNumElements() <=
-        cast<ScalableVectorType>(UncastedPred->getType())->getMinNumElements())
-      Pred = UncastedPred;
-
-  return match(Pred, m_Intrinsic<Intrinsic::aarch64_sve_ptrue>(
-                         m_ConstantInt<AArch64SVEPredPattern::all>()));
-}
-
 static std::optional<Instruction *>
 instCombineSVELD1(InstCombiner &IC, IntrinsicInst &II, const DataLayout &DL) {
   Value *Pred = II.getOperand(0);
diff --git a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
index ce59cf8dba..6e70deec3f 100644
--- a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
+++ b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
@@ -3638,6 +3638,7 @@ static const struct Extension {
     {"sb", {AArch64::FeatureSB}},
     {"ssbs", {AArch64::FeatureSSBS}},
     {"tme", {AArch64::FeatureTME}},
+    {"fpmr", {AArch64::FeatureFPMR}},
 };
 
 static void setRequiredFeatureString(FeatureBitset FBS, std::string &Str) {
@@ -4537,6 +4538,8 @@ ParseStatus AArch64AsmParser::tryParseZTOperand(OperandVector &Operands) {
 
   // Check if register is followed by an index
   if (parseOptionalToken(AsmToken::LBrac)) {
+    Operands.push_back(
+        AArch64Operand::CreateToken("[", getLoc(), getContext()));
     const MCExpr *ImmVal;
     if (getParser().parseExpression(ImmVal))
       return ParseStatus::NoMatch;
@@ -4549,6 +4552,8 @@ ParseStatus AArch64AsmParser::tryParseZTOperand(OperandVector &Operands) {
     Operands.push_back(AArch64Operand::CreateImm(
         MCConstantExpr::create(MCE->getValue(), getContext()), StartLoc,
         getLoc(), getContext()));
+    Operands.push_back(
+        AArch64Operand::CreateToken("]", getLoc(), getContext()));
   }
 
   return ParseStatus::Success;
@@ -7484,7 +7489,7 @@ bool AArch64AsmParser::parseAuthExpr(const MCExpr *&Res, SMLoc &EndLoc) {
   // Look for '_sym@AUTH' ...
   if (Tok.is(AsmToken::Identifier) && Tok.getIdentifier().endswith("@AUTH")) {
     StringRef SymName = Tok.getIdentifier().drop_back(strlen("@AUTH"));
-    if (SymName.find('@') != StringRef::npos)
+    if (SymName.contains('@'))
       return TokError(
           "combination of @AUTH with other modifiers not supported");
     Res = MCSymbolRefExpr::create(Ctx.getOrCreateSymbol(SymName), Ctx);
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index 1c7a09696e..f4ef6cffdf 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -102,6 +102,11 @@ class AArch64InstructionSelector : public InstructionSelector {
   // An early selection function that runs before the selectImpl() call.
   bool earlySelect(MachineInstr &I);
 
+  /// Save state that is shared between select calls, call select on \p I and
+  /// then restore the saved state. This can be used to recursively call select
+  /// within a select call.
+  bool selectAndRestoreState(MachineInstr &I);
+
   // Do some preprocessing of G_PHIs before we begin selection.
   void processPHIs(MachineFunction &MF);
 
@@ -3552,6 +3557,13 @@ bool AArch64InstructionSelector::select(MachineInstr &I) {
   return false;
 }
 
+bool AArch64InstructionSelector::selectAndRestoreState(MachineInstr &I) {
+  MachineIRBuilderState OldMIBState = MIB.getState();
+  bool Success = select(I);
+  MIB.setState(OldMIBState);
+  return Success;
+}
+
 bool AArch64InstructionSelector::selectReduction(MachineInstr &I,
                                                  MachineRegisterInfo &MRI) {
   Register VecReg = I.getOperand(1).getReg();
@@ -4749,11 +4761,17 @@ MachineInstr *AArch64InstructionSelector::emitCarryIn(MachineInstr &I,
   // emit a carry generating instruction. E.g. for G_UADDE/G_USUBE sequences
   // generated during legalization of wide add/sub. This optimization depends on
   // these sequences not being interrupted by other instructions.
+  // We have to select the previous instruction before the carry-using
+  // instruction is deleted by the calling function, otherwise the previous
+  // instruction might become dead and would get deleted.
   MachineInstr *SrcMI = MRI->getVRegDef(CarryReg);
   if (SrcMI == I.getPrevNode()) {
     if (auto *CarrySrcMI = dyn_cast<GAddSubCarryOut>(SrcMI)) {
       bool ProducesNegatedCarry = CarrySrcMI->isSub();
-      if (NeedsNegatedCarry == ProducesNegatedCarry && CarrySrcMI->isUnsigned())
+      if (NeedsNegatedCarry == ProducesNegatedCarry &&
+          CarrySrcMI->isUnsigned() &&
+          CarrySrcMI->getCarryOutReg() == CarryReg &&
+          selectAndRestoreState(*SrcMI))
         return nullptr;
     }
   }
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index e75f6e891a..ddc27bebb7 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -229,10 +229,10 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
        .clampScalar(1, s32, s64)
       .widenScalarToNextPow2(0);
 
-  getActionDefinitionsBuilder({G_FADD, G_FSUB, G_FMUL, G_FDIV, G_FNEG, G_FABS,
-                               G_FSQRT, G_FMAXNUM, G_FMINNUM, G_FMAXIMUM,
-                               G_FMINIMUM, G_FCEIL, G_FFLOOR, G_FRINT,
-                               G_FNEARBYINT, G_INTRINSIC_TRUNC,
+  getActionDefinitionsBuilder({G_FADD, G_FSUB, G_FMUL, G_FDIV, G_FMA, G_FNEG,
+                               G_FABS, G_FSQRT, G_FMAXNUM, G_FMINNUM,
+                               G_FMAXIMUM, G_FMINIMUM, G_FCEIL, G_FFLOOR,
+                               G_FRINT, G_FNEARBYINT, G_INTRINSIC_TRUNC,
                                G_INTRINSIC_ROUND, G_INTRINSIC_ROUNDEVEN})
       .legalFor({MinFPScalar, s32, s64, v2s32, v4s32, v2s64})
       .legalIf([=](const LegalityQuery &Query) {
@@ -251,7 +251,7 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
       .minScalar(0, s32)
       .scalarize(0);
 
-  getActionDefinitionsBuilder({G_FMA, G_INTRINSIC_LRINT})
+  getActionDefinitionsBuilder(G_INTRINSIC_LRINT)
       // If we don't have full FP16 support, then scalarize the elements of
       // vectors containing fp16 types.
       .fewerElementsIf(
@@ -536,14 +536,22 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
       });
 
   getActionDefinitionsBuilder(G_TRUNC)
+      .legalFor({{v2s32, v2s64}, {v4s16, v4s32}, {v8s8, v8s16}})
+      .moreElementsToNextPow2(0)
+      .clampMaxNumElements(0, s8, 8)
+      .clampMaxNumElements(0, s16, 4)
+      .clampMaxNumElements(0, s32, 2)
       .minScalarOrEltIf(
           [=](const LegalityQuery &Query) { return Query.Types[0].isVector(); },
           0, s8)
-      .customIf([=](const LegalityQuery &Query) {
+      .lowerIf([=](const LegalityQuery &Query) {
         LLT DstTy = Query.Types[0];
         LLT SrcTy = Query.Types[1];
-        return DstTy == v8s8 && SrcTy.getSizeInBits() > 128;
+        return DstTy.isVector() && (SrcTy.getSizeInBits() > 128 ||
+                                    (DstTy.getScalarSizeInBits() * 2 <
+                                     SrcTy.getScalarSizeInBits()));
       })
+
       .alwaysLegal();
 
   getActionDefinitionsBuilder(G_SEXT_INREG)
@@ -887,6 +895,13 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
       .clampMaxNumElements(1, s16, 8)
       .lower();
 
+  getActionDefinitionsBuilder(G_VECREDUCE_MUL)
+      .clampMaxNumElements(1, s32, 2)
+      .clampMaxNumElements(1, s16, 4)
+      .clampMaxNumElements(1, s8, 8)
+      .scalarize(1)
+      .lower();
+
   getActionDefinitionsBuilder(
       {G_VECREDUCE_OR, G_VECREDUCE_AND, G_VECREDUCE_XOR})
       // Try to break down into smaller vectors as long as they're at least 64
@@ -1002,8 +1017,6 @@ bool AArch64LegalizerInfo::legalizeCustom(LegalizerHelper &Helper,
     return legalizeShlAshrLshr(MI, MRI, MIRBuilder, Observer);
   case TargetOpcode::G_GLOBAL_VALUE:
     return legalizeSmallCMGlobalValue(MI, MRI, MIRBuilder, Observer);
-  case TargetOpcode::G_TRUNC:
-    return legalizeVectorTrunc(MI, Helper);
   case TargetOpcode::G_SBFX:
   case TargetOpcode::G_UBFX:
     return legalizeBitfieldExtract(MI, MRI, Helper);
@@ -1102,54 +1115,6 @@ bool AArch64LegalizerInfo::legalizeRotate(MachineInstr &MI,
   return true;
 }
 
-static void extractParts(Register Reg, MachineRegisterInfo &MRI,
-                         MachineIRBuilder &MIRBuilder, LLT Ty, int NumParts,
-                         SmallVectorImpl<Register> &VRegs) {
-  for (int I = 0; I < NumParts; ++I)
-    VRegs.push_back(MRI.createGenericVirtualRegister(Ty));
-  MIRBuilder.buildUnmerge(VRegs, Reg);
-}
-
-bool AArch64LegalizerInfo::legalizeVectorTrunc(
-    MachineInstr &MI, LegalizerHelper &Helper) const {
-  MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
-  MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
-  // Similar to how operand splitting is done in SelectiondDAG, we can handle
-  // %res(v8s8) = G_TRUNC %in(v8s32) by generating:
-  //   %inlo(<4x s32>), %inhi(<4 x s32>) = G_UNMERGE %in(<8 x s32>)
-  //   %lo16(<4 x s16>) = G_TRUNC %inlo
-  //   %hi16(<4 x s16>) = G_TRUNC %inhi
-  //   %in16(<8 x s16>) = G_CONCAT_VECTORS %lo16, %hi16
-  //   %res(<8 x s8>) = G_TRUNC %in16
-
-  Register DstReg = MI.getOperand(0).getReg();
-  Register SrcReg = MI.getOperand(1).getReg();
-  LLT DstTy = MRI.getType(DstReg);
-  LLT SrcTy = MRI.getType(SrcReg);
-  assert(llvm::has_single_bit<uint32_t>(DstTy.getSizeInBits()) &&
-         llvm::has_single_bit<uint32_t>(SrcTy.getSizeInBits()));
-
-  // Split input type.
-  LLT SplitSrcTy =
-      SrcTy.changeElementCount(SrcTy.getElementCount().divideCoefficientBy(2));
-  // First, split the source into two smaller vectors.
-  SmallVector<Register, 2> SplitSrcs;
-  extractParts(SrcReg, MRI, MIRBuilder, SplitSrcTy, 2, SplitSrcs);
-
-  // Truncate the splits into intermediate narrower elements.
-  LLT InterTy = SplitSrcTy.changeElementSize(DstTy.getScalarSizeInBits() * 2);
-  for (unsigned I = 0; I < SplitSrcs.size(); ++I)
-    SplitSrcs[I] = MIRBuilder.buildTrunc(InterTy, SplitSrcs[I]).getReg(0);
-
-  auto Concat = MIRBuilder.buildConcatVectors(
-      DstTy.changeElementSize(DstTy.getScalarSizeInBits() * 2), SplitSrcs);
-
-  Helper.Observer.changingInstr(MI);
-  MI.getOperand(1).setReg(Concat.getReg(0));
-  Helper.Observer.changedInstr(MI);
-  return true;
-}
-
 bool AArch64LegalizerInfo::legalizeSmallCMGlobalValue(
     MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder,
     GISelChangeObserver &Observer) const {
@@ -1319,6 +1284,9 @@ bool AArch64LegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
 
     return true;
   }
+  case Intrinsic::experimental_vector_reverse:
+    // TODO: Add support for vector_reverse
+    return false;
   }
 
   return true;
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h
index 853d5a2305..e6c9182da9 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h
@@ -46,7 +46,6 @@ class AArch64LegalizerInfo : public LegalizerInfo {
   bool legalizeSmallCMGlobalValue(MachineInstr &MI, MachineRegisterInfo &MRI,
                                   MachineIRBuilder &MIRBuilder,
                                   GISelChangeObserver &Observer) const;
-  bool legalizeVectorTrunc(MachineInstr &MI, LegalizerHelper &Helper) const;
   bool legalizeBitfieldExtract(MachineInstr &MI, MachineRegisterInfo &MRI,
                                LegalizerHelper &Helper) const;
   bool legalizeRotate(MachineInstr &MI, MachineRegisterInfo &MRI,
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
index be248125d8..c7ff14c252 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
@@ -40,7 +40,8 @@ class AArch64AsmBackend : public MCAsmBackend {
 
 public:
   AArch64AsmBackend(const Target &T, const Triple &TT, bool IsLittleEndian)
-      : MCAsmBackend(IsLittleEndian ? support::little : support::big),
+      : MCAsmBackend(IsLittleEndian ? llvm::endianness::little
+                                    : llvm::endianness::big),
         TheTriple(TT) {}
 
   unsigned getNumFixupKinds() const override {
@@ -360,7 +361,7 @@ AArch64AsmBackend::getFixupKind(StringRef Name) const {
 /// getFixupKindContainereSizeInBytes - The number of bytes of the
 /// container involved in big endian or 0 if the item is little endian
 unsigned AArch64AsmBackend::getFixupKindContainereSizeInBytes(unsigned Kind) const {
-  if (Endian == support::little)
+  if (Endian == llvm::endianness::little)
     return 0;
 
   switch (Kind) {
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp
index 5273a02f18..988c786991 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp
@@ -1756,10 +1756,11 @@ void AArch64InstPrinter::printVectorIndex(const MCInst *MI, unsigned OpNum,
   O << "[" << Scale * MI->getOperand(OpNum).getImm() << "]";
 }
 
+template <unsigned Scale>
 void AArch64InstPrinter::printMatrixIndex(const MCInst *MI, unsigned OpNum,
                                           const MCSubtargetInfo &STI,
                                           raw_ostream &O) {
-  O << MI->getOperand(OpNum).getImm();
+  O << Scale * MI->getOperand(OpNum).getImm();
 }
 
 void AArch64InstPrinter::printAlignedLabel(const MCInst *MI, uint64_t Address,
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.h b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.h
index c4c83f0f25..9dccdf4236 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.h
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.h
@@ -173,6 +173,7 @@ class AArch64InstPrinter : public MCInstPrinter {
   template <unsigned Scale = 1>
   void printVectorIndex(const MCInst *MI, unsigned OpNum,
                         const MCSubtargetInfo &STI, raw_ostream &O);
+  template <unsigned Scale = 1>
   void printMatrixIndex(const MCInst *MI, unsigned OpNum,
                         const MCSubtargetInfo &STI, raw_ostream &O);
   void printAdrAdrpLabel(const MCInst *MI, uint64_t Address, unsigned OpNum,
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp
index 727f799098..dbc4323a86 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp
@@ -685,7 +685,7 @@ void AArch64MCCodeEmitter::encodeInstruction(const MCInst &MI,
   }
 
   uint64_t Binary = getBinaryCodeForInstr(MI, Fixups, STI);
-  support::endian::write<uint32_t>(CB, Binary, support::little);
+  support::endian::write<uint32_t>(CB, Binary, llvm::endianness::little);
   ++MCNumEmitted; // Keep track of the # of mi's emitted.
 }
 
diff --git a/llvm/lib/Target/AArch64/SMEInstrFormats.td b/llvm/lib/Target/AArch64/SMEInstrFormats.td
index edd24b4a84..823115c7d0 100644
--- a/llvm/lib/Target/AArch64/SMEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SMEInstrFormats.td
@@ -230,7 +230,7 @@ def : InstAlias<"smstop za",  (MSRpstatesvcrImm1 0b010, 0b0)>;
 // SME Outer Products
 //===----------------------------------------------------------------------===//
 
-class sme_fp_outer_product_inst<bit S, bits<2> sz, bit op, MatrixTileOperand za_ty,
+class sme_fp_outer_product_inst<bit S, bits<2> sz, bits<2> op, MatrixTileOperand za_ty,
                                 ZPRRegOp zpr_ty, string mnemonic>
     : I<(outs za_ty:$ZAda),
       (ins za_ty:$_ZAda, PPR3bAny:$Pn, PPR3bAny:$Pm, zpr_ty:$Zn, zpr_ty:$Zm),
@@ -242,7 +242,7 @@ class sme_fp_outer_product_inst<bit S, bits<2> sz, bit op, MatrixTileOperand za_
   bits<3> Pn;
   bits<5> Zn;
   let Inst{31-25} = 0b1000000;
-  let Inst{24}    = op;
+  let Inst{24}    = op{1};
   let Inst{23}    = 0b1;
   let Inst{22-21} = sz;
   let Inst{20-16} = Zm;
@@ -250,25 +250,25 @@ class sme_fp_outer_product_inst<bit S, bits<2> sz, bit op, MatrixTileOperand za_
   let Inst{12-10} = Pn;
   let Inst{9-5}   = Zn;
   let Inst{4}     = S;
-  let Inst{3}     = op;
+  let Inst{3}     = op{0};
 
   let Constraints = "$ZAda = $_ZAda";
 }
 
-multiclass sme_outer_product_fp32<bit S, string mnemonic, SDPatternOperator op> {
-  def NAME : sme_fp_outer_product_inst<S, 0b00, 0b0, TileOp32, ZPR32, mnemonic>, SMEPseudo2Instr<NAME, 1> {
+multiclass sme_outer_product_fp32<bit S, bits<2> sz, ZPRRegOp zpr_ty, string mnemonic, SDPatternOperator op> {
+  def NAME : sme_fp_outer_product_inst<S, sz, 0b00, TileOp32, zpr_ty, mnemonic>, SMEPseudo2Instr<NAME, 1> {
     bits<2> ZAda;
     let Inst{1-0} = ZAda;
     let Inst{2}   = 0b0;
   }
 
-  def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR32, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>;
+  def NAME # _PSEUDO : sme_outer_product_pseudo<zpr_ty, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>;
 
   def : SME_ZA_Tile_TwoPred_TwoVec_Pat<NAME, op, timm32_0_3, nxv4i1, nxv4f32>;
 }
 
 multiclass sme_outer_product_fp64<bit S, string mnemonic, SDPatternOperator op> {
-  def NAME : sme_fp_outer_product_inst<S, 0b10, 0b0, TileOp64, ZPR64, mnemonic>, SMEPseudo2Instr<NAME, 1> {
+  def NAME : sme_fp_outer_product_inst<S, 0b10, 0b00, TileOp64, ZPR64, mnemonic>, SMEPseudo2Instr<NAME, 1> {
     bits<3> ZAda;
     let Inst{2-0} = ZAda;
   }
@@ -278,8 +278,8 @@ multiclass sme_outer_product_fp64<bit S, string mnemonic, SDPatternOperator op>
   def : SME_ZA_Tile_TwoPred_TwoVec_Pat<NAME, op, timm32_0_7, nxv2i1, nxv2f64>;
 }
 
-multiclass sme2p1_fmop_tile_fp16<string mnemonic, bit bf, bit s>{
-  def NAME : sme_fp_outer_product_inst<s, {0,bf}, 0b1, TileOp16, ZPR16, mnemonic> {
+multiclass sme2p1_fmop_tile_fp16<string mnemonic, bit bf, bit s, bits<2> op, ZPRRegOp zpr_ty>{
+  def NAME : sme_fp_outer_product_inst<s, {0,bf}, op, TileOp16, zpr_ty, mnemonic> {
     bits<1> ZAda;
     let Inst{2-1} = 0b00;
     let Inst{0}   = ZAda;
@@ -1449,7 +1449,7 @@ multiclass sme2_dot_mla_add_sub_array_vg4_single<string mnemonic, bits<7> op,
 
 //===----------------------------------------------------------------------===//
 // SME2 multiple vectors ternary INT/FP  two and four registers
-class sme2_dot_mla_add_sub_array_vg2_multi<bits<6> op,
+class sme2_dot_mla_add_sub_array_vg2_multi<bits<7> op,
                                        MatrixOperand matrix_ty,
                                        RegisterOperand multi_vector_ty,
                                        string mnemonic>
@@ -1463,20 +1463,19 @@ class sme2_dot_mla_add_sub_array_vg2_multi<bits<6> op,
   bits<2> Rv;
   bits<3> imm3;
   let Inst{31-23} = 0b110000011;
-  let Inst{22}    = op{5}; //sz
+  let Inst{22}    = op{6}; //sz
   let Inst{21}    = 0b1;
   let Inst{20-17} = Zm;
   let Inst{16-15} = 0b00;
   let Inst{14-13} = Rv;
-  let Inst{12-10} = op{4-2};
+  let Inst{12-10} = op{5-3};
   let Inst{9-6}   = Zn;
-  let Inst{5}     = 0b0;
-  let Inst{4-3}   = op{1-0};
+  let Inst{5-3}   = op{2-0};
   let Inst{2-0}   = imm3;
   let Constraints = "$ZAd = $_ZAd";
 }
 
-multiclass sme2_dot_mla_add_sub_array_vg2_multi<string mnemonic, bits<6> op,
+multiclass sme2_dot_mla_add_sub_array_vg2_multi<string mnemonic, bits<7> op,
                                             MatrixOperand  matrix_ty,
                                             RegisterOperand multi_vector_ty, ValueType zpr_ty,
                                             SDPatternOperator intrinsic> {
@@ -1490,7 +1489,7 @@ multiclass sme2_dot_mla_add_sub_array_vg2_multi<string mnemonic, bits<6> op,
                   (!cast<Instruction>(NAME) matrix_ty:$ZAd,  MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, multi_vector_ty:$Zm), 0>;
 }
 
-class sme2_dot_mla_add_sub_array_vg4_multi<bits<6> op,
+class sme2_dot_mla_add_sub_array_vg4_multi<bits<7> op,
                                             MatrixOperand matrix_ty,
                                             RegisterOperand multi_vector_ty,
                                             string mnemonic>
@@ -1504,20 +1503,20 @@ class sme2_dot_mla_add_sub_array_vg4_multi<bits<6> op,
   bits<2> Rv;
   bits<3> imm3;
   let Inst{31-23} = 0b110000011;
-  let Inst{22}    = op{5}; //sz
+  let Inst{22}    = op{6}; //sz
   let Inst{21}    = 0b1;
   let Inst{20-18} = Zm;
   let Inst{17-15} = 0b010;
   let Inst{14-13} = Rv;
-  let Inst{12-10} = op{4-2};
+  let Inst{12-10} = op{5-3};
   let Inst{9-7}   = Zn;
-  let Inst{6-5}   = 0b00;
-  let Inst{4-3}   = op{1-0};
+  let Inst{6}     = 0b0;
+  let Inst{5-3}   = op{2-0};
   let Inst{2-0}   = imm3;
   let Constraints = "$ZAd = $_ZAd";
 }
 
-multiclass sme2_dot_mla_add_sub_array_vg4_multi<string mnemonic, bits<6> op,
+multiclass sme2_dot_mla_add_sub_array_vg4_multi<string mnemonic, bits<7> op,
                                             MatrixOperand  matrix_ty,
                                             RegisterOperand multi_vector_ty,
                                             ValueType zpr_ty, SDPatternOperator intrinsic>{
@@ -1794,8 +1793,8 @@ class sme2_mla_long_array_index_base<bits<2> op0, bits<2> op, Operand index_ty,
 }
 
 multiclass sme2_mla_long_array_index<string mnemonic, bits<2> op0, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> {
-  def _S : sme2_mla_long_array_index_base<op0, op, uimm3s2range, ZPR16,
-                                          mnemonic>, SMEPseudo2Instr<NAME # _S, 1> {
+  def _HtoS : sme2_mla_long_array_index_base<op0, op, uimm3s2range, ZPR16,
+                                          mnemonic>, SMEPseudo2Instr<NAME # _HtoS, 1> {
     bits<3> i3;
     bits<5> Zn;
     bits<3> imm;
@@ -1805,9 +1804,9 @@ multiclass sme2_mla_long_array_index<string mnemonic, bits<2> op0, bits<2> op, V
     let Inst{2-0}   = imm;
   }
 
-  def _S_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _S, uimm3s2range, ZPR16, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>;
+  def _HtoS_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _HtoS, uimm3s2range, ZPR16, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>;
 
-  def : SME2_ZA_TwoOp_Multi_Index_Pat<NAME # _S, intrinsic, uimm3s2range, ZPR4b16, zpr_ty, VectorIndexH32b_timm, tileslicerange3s2>;
+  def : SME2_ZA_TwoOp_Multi_Index_Pat<NAME # _HtoS, intrinsic, uimm3s2range, ZPR4b16, zpr_ty, VectorIndexH32b_timm, tileslicerange3s2>;
 }
 
 class sme2_mla_long_array_vg2_index<string mnemonic, bits<2> op0, bits<2> op>
@@ -1825,14 +1824,14 @@ class sme2_mla_long_array_vg2_index<string mnemonic, bits<2> op0, bits<2> op>
 }
 
 multiclass sme2_fp_mla_long_array_vg2_index<string mnemonic, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> {
-  def _S : sme2_mla_long_array_vg2_index<mnemonic, 0b10, op>, SMEPseudo2Instr<NAME # _S, 1>;
+  def _HtoS : sme2_mla_long_array_vg2_index<mnemonic, 0b10, op>, SMEPseudo2Instr<NAME # _HtoS, 1>;
 
-  def _S_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _S, uimm2s2range, ZZ_h_mul_r, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>;
+  def _HtoS_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _HtoS, uimm2s2range, ZZ_h_mul_r, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>;
 
-  def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, zpr_ty, VectorIndexH32b_timm, tileslicerange2s2>;
+  def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME # _HtoS, intrinsic, uimm2s2range, ZPR4b16, zpr_ty, VectorIndexH32b_timm, tileslicerange2s2>;
 
   def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i3",
-                 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i3), 0>;
+                 (!cast<Instruction>(NAME #_HtoS) MatrixOp32:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i3), 0>;
 }
 
 multiclass sme2_int_mla_long_array_vg2_index<string mnemonic, bits<2> op, SDPatternOperator intrinsic> {
@@ -1861,33 +1860,35 @@ class sme2_mla_long_array_vg4_index<string mnemonic, bits<2> op0, bits<2> op>
 }
 
 multiclass sme2_fp_mla_long_array_vg4_index<string mnemonic, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> {
-  def _S : sme2_mla_long_array_vg4_index<mnemonic, 0b10, op>, SMEPseudo2Instr<NAME # _S, 1>;
+  def _HtoS : sme2_mla_long_array_vg4_index<mnemonic, 0b10, op>, SMEPseudo2Instr<NAME # _HtoS, 1>;
 
-  def _S_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _S, uimm2s2range, ZZZZ_h_mul_r, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>;
+  def _HtoS_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _HtoS, uimm2s2range, ZZZZ_h_mul_r, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>;
 
-  def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, zpr_ty, VectorIndexH32b_timm, tileslicerange2s2>;
+  def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME # _HtoS, intrinsic, uimm2s2range, ZPR4b16, zpr_ty, VectorIndexH32b_timm, tileslicerange2s2>;
 
   def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i3",
-                 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i3), 0>;
+                 (!cast<Instruction>(NAME #_HtoS) MatrixOp32:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i3), 0>;
 }
 
 multiclass sme2_int_mla_long_array_vg4_index<string mnemonic, bits<2> op, SDPatternOperator intrinsic> {
-  def _S : sme2_mla_long_array_vg4_index<mnemonic, 0b11, op>, SMEPseudo2Instr<NAME # _S, 1>;
+  def _HtoS : sme2_mla_long_array_vg4_index<mnemonic, 0b11, op>, SMEPseudo2Instr<NAME # _HtoS, 1>;
 
-  def _S_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _S, uimm2s2range, ZZZZ_h_mul_r, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>;
+  def _HtoS_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _HtoS, uimm2s2range, ZZZZ_h_mul_r, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>;
 
-  def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, nxv8i16, VectorIndexH32b_timm, tileslicerange2s2>;
+  def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME # _HtoS, intrinsic, uimm2s2range, ZPR4b16, nxv8i16, VectorIndexH32b_timm, tileslicerange2s2>;
 
   def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i3",
-                 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i3), 0>;
+                 (!cast<Instruction>(NAME #_HtoS) MatrixOp32:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i3), 0>;
 }
 
-class sme2_mla_long_array<bits<2>op0, bits<2> op, Operand index_ty,
+class sme2_mla_long_array<bits<2>op0, bits<2> op,
+                          MatrixOperand matrix_ty,
+                          Operand index_ty,
                           RegisterOperand first_vector_ty,
                           RegisterOperand second_vector_ty,
                           string mnemonic, string vg_acronym="">
-   : I<(outs MatrixOp32:$ZAda),
-       (ins  MatrixOp32:$_ZAda, MatrixIndexGPR32Op8_11:$Rv,
+   : I<(outs matrix_ty:$ZAda),
+       (ins  matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv,
        index_ty:$imm, first_vector_ty:$Zn, second_vector_ty:$Zm),
        mnemonic,"\t$ZAda[$Rv, $imm" # !if(!eq(vg_acronym, ""), "", ", " # vg_acronym) # "], $Zn, $Zm",
        "", []> , Sched<[]> {
@@ -1905,8 +1906,8 @@ class sme2_mla_long_array<bits<2>op0, bits<2> op, Operand index_ty,
 }
 
 multiclass sme2_mla_long_array_single<string mnemonic, bits<2> op0, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> {
-  def _S : sme2_mla_long_array<op0, op, uimm3s2range, ZPR16, ZPR4b16,
-                               mnemonic> , SMEPseudo2Instr<NAME # _S, 1>{
+  def _HtoS : sme2_mla_long_array<op0, op, MatrixOp32, uimm3s2range, ZPR16, ZPR4b16,
+                               mnemonic> , SMEPseudo2Instr<NAME # _HtoS, 1>{
     bits<4> Zm;
     bits<5> Zn;
     bits<3> imm;
@@ -1916,15 +1917,15 @@ multiclass sme2_mla_long_array_single<string mnemonic, bits<2> op0, bits<2> op,
     let Inst{2-0}   = imm;
   }
 
-  def _S_PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME # _S, uimm3s2range, ZPR16, ZPR4b16, SMEMatrixArray>;
+  def _HtoS_PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME # _HtoS, uimm3s2range, ZPR16, ZPR4b16, SMEMatrixArray>;
 
-  def : SME2_ZA_TwoOp_Multi_Single_Pat<NAME # _S, intrinsic, uimm3s2range, ZPR4b16, zpr_ty, tileslicerange3s2>;
+  def : SME2_ZA_TwoOp_Multi_Single_Pat<NAME # _HtoS, intrinsic, uimm3s2range, ZPR4b16, zpr_ty, tileslicerange3s2>;
 }
 
-class sme2_mla_long_array_vg24_single<bits<2> op0, bit vg4, bits<2> op,
-                                      RegisterOperand first_vector_ty,
-                                      string mnemonic, string vg_acronym>
-    : sme2_mla_long_array<op0, op, uimm2s2range, first_vector_ty, ZPR4b16,
+class sme2_mla_long_array_vg24_single<bits<2> op0, bit vg4, bits<2> op, bit o2,
+                                      MatrixOperand matrix_ty, RegisterOperand multi_vector_ty,
+                                      ZPRRegOp zpr_ty, string mnemonic, string vg_acronym>
+    : sme2_mla_long_array<op0, op, matrix_ty, uimm2s2range, multi_vector_ty, zpr_ty,
                           mnemonic, vg_acronym> {
   bits<4> Zm;
   bits<5> Zn;
@@ -1932,96 +1933,117 @@ class sme2_mla_long_array_vg24_single<bits<2> op0, bit vg4, bits<2> op,
   let Inst{20}    = vg4;
   let Inst{19-16} = Zm;
   let Inst{9-5}   = Zn;
-  let Inst{2}     = 0b0;
+  let Inst{2}     = o2;
   let Inst{1-0}   = imm;
 }
 
-multiclass sme2_fp_mla_long_array_vg2_single<string mnemonic, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> {
-  def _S : sme2_mla_long_array_vg24_single<0b00, 0b0, op, ZZ_h, mnemonic,
-                                           "vgx2">, SMEPseudo2Instr<NAME # _S, 1>;
+	
+multiclass sme2_fp_mla_long_array_vg2_single<string mnemonic, bits<3> op, MatrixOperand matrix_ty,
+                                             RegisterOperand multi_vector_ty, ZPRRegOp vector_ty,
+                                             ValueType zpr_ty, SDPatternOperator intrinsic> {
+  def NAME : sme2_mla_long_array_vg24_single<0b00, 0b0, op{2-1}, op{0}, matrix_ty,  multi_vector_ty,
+                                           vector_ty, mnemonic, "vgx2">, SMEPseudo2Instr<NAME, 1>;
 
-  def _S_PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME # _S, uimm2s2range, ZZ_h, ZPR4b16, SMEMatrixArray>;
+  def _PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME, uimm2s2range, multi_vector_ty,
+                                                        vector_ty, SMEMatrixArray>;
 
-  def : SME2_ZA_TwoOp_VG2_Multi_Single_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, zpr_ty, tileslicerange2s2>;
+  def : SME2_ZA_TwoOp_VG2_Multi_Single_Pat<NAME, intrinsic, uimm2s2range, vector_ty, zpr_ty,
+                                           tileslicerange2s2>;
 
   def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm",
-                 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZ_h:$Zn, ZPR4b16:$Zm), 0>;
+                 (!cast<Instruction>(NAME) matrix_ty:$ZAda,  MatrixIndexGPR32Op8_11:$Rv,
+                  uimm2s2range:$imm, multi_vector_ty:$Zn, vector_ty:$Zm), 0>;
 }
 
 multiclass sme2_int_mla_long_array_vg2_single<string mnemonic, bits<2> op, SDPatternOperator intrinsic> {
-  def _S : sme2_mla_long_array_vg24_single<0b01, 0b0, op, ZZ_h, mnemonic,
-                                           "vgx2">, SMEPseudo2Instr<NAME # _S, 1>;
+  def _HtoS : sme2_mla_long_array_vg24_single<0b01, 0b0, op, 0b0, MatrixOp32, ZZ_h, ZPR4b16, mnemonic,
+                                             "vgx2">, SMEPseudo2Instr<NAME # _HtoS, 1>;
 
-  def _S_PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME # _S, uimm2s2range, ZZ_h, ZPR4b16, SMEMatrixArray>;
+  def _HtoS_PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME # _HtoS, uimm2s2range, ZZ_h, ZPR4b16, SMEMatrixArray>;
 
-  def : SME2_ZA_TwoOp_VG2_Multi_Single_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, nxv8i16, tileslicerange2s2>;
+  def : SME2_ZA_TwoOp_VG2_Multi_Single_Pat<NAME # _HtoS, intrinsic, uimm2s2range, ZPR4b16, nxv8i16, tileslicerange2s2>;
 
   def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm",
-                 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZ_h:$Zn, ZPR4b16:$Zm), 0>;
+                 (!cast<Instruction>(NAME #_HtoS) MatrixOp32:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZ_h:$Zn, ZPR4b16:$Zm), 0>;
 }
 
-multiclass sme2_fp_mla_long_array_vg4_single<string mnemonic, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> {
-  def _S : sme2_mla_long_array_vg24_single<0b00, 0b1, op, ZZZZ_h, mnemonic,
-                                           "vgx4">, SMEPseudo2Instr<NAME # _S, 1>;
+multiclass sme2_fp_mla_long_array_vg4_single<string mnemonic, bits<3> op, MatrixOperand matrix_ty,
+                                             RegisterOperand multi_vector_ty, ZPRRegOp vector_ty,
+                                             ValueType zpr_ty, SDPatternOperator intrinsic> {
+  def NAME : sme2_mla_long_array_vg24_single<0b00, 0b1, op{2-1}, op{0}, matrix_ty, multi_vector_ty, 
+                                             vector_ty, mnemonic, "vgx4">, SMEPseudo2Instr<NAME, 1>;
 
-  def _S_PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME # _S, uimm2s2range, ZZZZ_h, ZPR4b16, SMEMatrixArray>;
+  def _PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME, uimm2s2range, multi_vector_ty, vector_ty,
+                                                      SMEMatrixArray>;
 
-  def : SME2_ZA_TwoOp_VG4_Multi_Single_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, zpr_ty, tileslicerange2s2>;
+  def : SME2_ZA_TwoOp_VG4_Multi_Single_Pat<NAME, intrinsic, uimm2s2range, vector_ty, zpr_ty,
+                                           tileslicerange2s2>;
 
   def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm",
-                 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZZZ_h:$Zn, ZPR4b16:$Zm), 0>;
+                 (!cast<Instruction>(NAME) matrix_ty:$ZAda, MatrixIndexGPR32Op8_11:$Rv,
+                  uimm2s2range:$imm, multi_vector_ty:$Zn, vector_ty:$Zm), 0>;
 }
 
 multiclass sme2_int_mla_long_array_vg4_single<string mnemonic, bits<2> op, SDPatternOperator intrinsic> {
-  def _S : sme2_mla_long_array_vg24_single<0b01, 0b1, op, ZZZZ_h, mnemonic,
-                                           "vgx4">, SMEPseudo2Instr<NAME # _S, 1>;
+  def _HtoS : sme2_mla_long_array_vg24_single<0b01, 0b1, op, 0b0, MatrixOp32, ZZZZ_h, ZPR4b16,  mnemonic,
+                                           "vgx4">, SMEPseudo2Instr<NAME # _HtoS, 1>;
 
-  def _S_PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME # _S, uimm2s2range, ZZZZ_h, ZPR4b16, SMEMatrixArray>;
+  def _HtoS_PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME # _HtoS, uimm2s2range, ZZZZ_h, ZPR4b16, SMEMatrixArray>;
 
-  def : SME2_ZA_TwoOp_VG4_Multi_Single_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, nxv8i16, tileslicerange2s2>;
+  def : SME2_ZA_TwoOp_VG4_Multi_Single_Pat<NAME # _HtoS, intrinsic, uimm2s2range, ZPR4b16, nxv8i16, tileslicerange2s2>;
 
   def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm",
-                 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZZZ_h:$Zn, ZPR4b16:$Zm), 0>;
+                 (!cast<Instruction>(NAME #_HtoS) MatrixOp32:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZZZ_h:$Zn, ZPR4b16:$Zm), 0>;
 }
-class sme2_mla_long_array_vg2_multi<string mnemonic, bits<2> op0, bits<2> op>
-   : sme2_mla_long_array<op0, op, uimm2s2range, ZZ_h_mul_r, ZZ_h_mul_r,  mnemonic,
-                         "vgx2"> {
+
+class sme2_mla_long_array_vg2_multi<string mnemonic, bits<2> op0, bits<3> op,
+                                    MatrixOperand matrix_ty, RegisterOperand multi_vector_ty>
+   : sme2_mla_long_array<op0, op{1-0},  matrix_ty, uimm2s2range, multi_vector_ty, multi_vector_ty,
+                        mnemonic, "vgx2"> {
   bits<4> Zm;
   bits<4> Zn;
   bits<2> imm;
   let Inst{20-17} = Zm;
   let Inst{16}    = 0b0;
   let Inst{9-6}   = Zn;
-  let Inst{5}     = 0b0;
+  let Inst{5}     = op{2};  // fp8
   let Inst{2}     = 0b0;
   let Inst{1-0}   = imm;
 }
 
-multiclass sme2_fp_mla_long_array_vg2_multi<string mnemonic, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> {
-  def _S : sme2_mla_long_array_vg2_multi<mnemonic, 0b10, op>, SMEPseudo2Instr<NAME # _S, 1>;
+multiclass sme2_fp_mla_long_array_vg2_multi<string mnemonic, bits<3> op, MatrixOperand matrix_ty,
+                                            RegisterOperand multi_vector_ty,
+                                            ValueType zpr_ty, SDPatternOperator intrinsic> {
+
+  def NAME : sme2_mla_long_array_vg2_multi<mnemonic, 0b10, op, matrix_ty, multi_vector_ty>,
+                                           SMEPseudo2Instr<NAME, 1>;
 
-  def _S_PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME # _S, uimm2s2range, ZZ_h_mul_r, SMEMatrixArray>;
+  def _PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME, uimm2s2range, multi_vector_ty, SMEMatrixArray>;
 
-  def : SME2_ZA_TwoOp_VG2_Multi_Multi_Pat<NAME # _S, intrinsic, uimm2s2range, zpr_ty, tileslicerange2s2>;
+  def : SME2_ZA_TwoOp_VG2_Multi_Multi_Pat<NAME, intrinsic, uimm2s2range, zpr_ty, tileslicerange2s2>;
 
   def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm",
-                 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZ_h_mul_r:$Zn, ZZ_h_mul_r:$Zm), 0>;
+                  (!cast<Instruction>(NAME) matrix_ty:$ZAda,  MatrixIndexGPR32Op8_11:$Rv,
+                  uimm2s2range:$imm, multi_vector_ty:$Zn, multi_vector_ty:$Zm), 0>;
 }
 
 multiclass sme2_int_mla_long_array_vg2_multi<string mnemonic, bits<2> op, SDPatternOperator intrinsic> {
-  def _S : sme2_mla_long_array_vg2_multi<mnemonic, 0b11, op>, SMEPseudo2Instr<NAME # _S, 1>;
+  def _HtoS : sme2_mla_long_array_vg2_multi<mnemonic, 0b11, {0b0, op}, MatrixOp32, ZZ_h_mul_r>,
+                                         SMEPseudo2Instr<NAME # _HtoS, 1>;
 
-  def _S_PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME # _S, uimm2s2range, ZZ_h_mul_r, SMEMatrixArray>;
+  def _HtoS_PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME # _HtoS, uimm2s2range, ZZ_h_mul_r, SMEMatrixArray>;
 
-  def : SME2_ZA_TwoOp_VG2_Multi_Multi_Pat<NAME # _S, intrinsic, uimm2s2range, nxv8i16, tileslicerange2s2>;
+  def : SME2_ZA_TwoOp_VG2_Multi_Multi_Pat<NAME # _HtoS, intrinsic, uimm2s2range, nxv8i16, tileslicerange2s2>;
 
   def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm2], $Zn, $Zm",
-                 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm2, ZZ_h_mul_r:$Zn, ZZ_h_mul_r:$Zm), 0>;
+                 (!cast<Instruction>(NAME #_HtoS) MatrixOp32:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm2, ZZ_h_mul_r:$Zn, ZZ_h_mul_r:$Zm), 0>;
 }
 
-class sme2_mla_long_array_vg4_multi<string mnemonic, bits<2> op0, bits<2> op>
-   : sme2_mla_long_array<op0, op, uimm2s2range, ZZZZ_h_mul_r, ZZZZ_h_mul_r, mnemonic,
-                        "vgx4"> {
+class sme2_mla_long_array_vg4_multi<string mnemonic, bits<2> op0, bits<3> op,
+                                    MatrixOperand matrix_ty,
+                                    RegisterOperand multi_vector_ty>
+   : sme2_mla_long_array<op0, op{1-0}, matrix_ty, uimm2s2range, multi_vector_ty, multi_vector_ty,
+                         mnemonic, "vgx4"> {
   bits<3> Zm;
   bits<3> Zn;
   bits<2> imm;
@@ -2029,31 +2051,37 @@ class sme2_mla_long_array_vg4_multi<string mnemonic, bits<2> op0, bits<2> op>
   let Inst{17}    = 0b0;
   let Inst{16}    = 0b1;
   let Inst{9-7}   = Zn;
-  let Inst{6-5}   = 0b00;
+  let Inst{6}     = 0b0;
+  let Inst{5}     = op{2};  //fp8
   let Inst{2}     = 0b0;
   let Inst{1-0}   = imm;
 }
 
-multiclass sme2_fp_mla_long_array_vg4_multi<string mnemonic, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> {
-  def _S : sme2_mla_long_array_vg4_multi<mnemonic, 0b10, op>, SMEPseudo2Instr<NAME # _S, 1>;
+multiclass sme2_fp_mla_long_array_vg4_multi<string mnemonic, bits<3> op, MatrixOperand matrix_ty,
+                                            RegisterOperand multi_vector_ty, ValueType zpr_ty,
+                                            SDPatternOperator intrinsic> {
+  def NAME : sme2_mla_long_array_vg4_multi<mnemonic, 0b10, op, matrix_ty, multi_vector_ty>,
+                                           SMEPseudo2Instr<NAME, 1>;
 
-  def _S_PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME # _S, uimm2s2range, ZZZZ_h_mul_r, SMEMatrixArray>;
+  def _PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME, uimm2s2range, multi_vector_ty, SMEMatrixArray>;
 
-  def : SME2_ZA_TwoOp_VG4_Multi_Multi_Pat<NAME # _S, intrinsic, uimm2s2range, zpr_ty, tileslicerange2s2>;
+  def : SME2_ZA_TwoOp_VG4_Multi_Multi_Pat<NAME, intrinsic, uimm2s2range, zpr_ty, tileslicerange2s2>;
 
   def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm",
-                 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZZZ_h_mul_r:$Zn, ZZZZ_h_mul_r:$Zm), 0>;
+                 (!cast<Instruction>(NAME) matrix_ty:$ZAda,  MatrixIndexGPR32Op8_11:$Rv,
+                  uimm2s2range:$imm, multi_vector_ty:$Zn, multi_vector_ty:$Zm), 0>;
 }
 
 multiclass sme2_int_mla_long_array_vg4_multi<string mnemonic, bits<2> op, SDPatternOperator intrinsic> {
-  def _S : sme2_mla_long_array_vg4_multi<mnemonic, 0b11, op>, SMEPseudo2Instr<NAME # _S, 1>;
+  def _HtoS : sme2_mla_long_array_vg4_multi<mnemonic, 0b11, {0b0, op}, MatrixOp32, ZZZZ_h_mul_r>,
+                                            SMEPseudo2Instr<NAME # _HtoS, 1>;
 
-  def _S_PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME # _S, uimm2s2range, ZZZZ_h_mul_r, SMEMatrixArray>;
+  def _HtoS_PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME # _HtoS, uimm2s2range, ZZZZ_h_mul_r, SMEMatrixArray>;
 
-  def : SME2_ZA_TwoOp_VG4_Multi_Multi_Pat<NAME # _S, intrinsic, uimm2s2range, nxv8i16, tileslicerange2s2>;
+  def : SME2_ZA_TwoOp_VG4_Multi_Multi_Pat<NAME # _HtoS, intrinsic, uimm2s2range, nxv8i16, tileslicerange2s2>;
 
   def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm2], $Zn, $Zm",
-                 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm2, ZZZZ_h_mul_r:$Zn, ZZZZ_h_mul_r:$Zm), 0>;
+                 (!cast<Instruction>(NAME #_HtoS) MatrixOp32:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm2, ZZZZ_h_mul_r:$Zn, ZZZZ_h_mul_r:$Zm), 0>;
 }
 
 //===----------------------------------------------------------------------===//
@@ -2344,7 +2372,7 @@ multiclass sme2_zip_vector_vg2<string mnemonic, bit op> {
 //===----------------------------------------------------------------------===//
 // SME2 Dot Products and MLA
 
-class sme2_multi_vec_array_vg2_index<bit sz, bits<6> op, MatrixOperand matrix_ty,
+class sme2_multi_vec_array_vg2_index<bits<2> sz, bits<6> op, MatrixOperand matrix_ty,
                                      RegisterOperand multi_vector_ty,
                                      ZPRRegOp vector_ty, Operand index_ty,
                                      string mnemonic>
@@ -2357,8 +2385,8 @@ class sme2_multi_vec_array_vg2_index<bit sz, bits<6> op, MatrixOperand matrix_ty
   bits<2> Rv;
   bits<4> Zn;
   bits<3> imm3;
-  let Inst{31-23} = 0b110000010;
-  let Inst{22}    = sz;
+  let Inst{31-24} = 0b11000001;
+  let Inst{23-22} = sz;
   let Inst{21-20} = 0b01;
   let Inst{19-16} = Zm;
   let Inst{15}    = 0b0;
@@ -2372,11 +2400,11 @@ class sme2_multi_vec_array_vg2_index<bit sz, bits<6> op, MatrixOperand matrix_ty
 }
 
 // SME2 multi-vec ternary indexed two registers 32-bit
-multiclass sme2_multi_vec_array_vg2_index_32b<string mnemonic, bits<4> op,
+multiclass sme2_multi_vec_array_vg2_index_32b<string mnemonic, bits<2> sz, bits<4> op,
                                               RegisterOperand multi_vector_ty,
                                               ZPRRegOp vector_ty, ValueType vt,
                                               SDPatternOperator intrinsic> {
-  def NAME : sme2_multi_vec_array_vg2_index<0b1, {op{3},?,?,op{2-0}}, MatrixOp32, multi_vector_ty, vector_ty,
+  def NAME : sme2_multi_vec_array_vg2_index<sz, {op{3},?,?,op{2-0}}, MatrixOp32, multi_vector_ty, vector_ty,
                                              VectorIndexS32b_timm,  mnemonic>, SMEPseudo2Instr<NAME, 1> {
     bits<2> i;
     let Inst{11-10} = i;
@@ -2392,9 +2420,10 @@ multiclass sme2_multi_vec_array_vg2_index_32b<string mnemonic, bits<4> op,
 }
 
 // SME2.1 multi-vec ternary indexed two registers 16-bit
-multiclass sme2p1_multi_vec_array_vg2_index_16b<string mnemonic, bits<2> op> {
-  def NAME : sme2_multi_vec_array_vg2_index<0b0, {0b1,?,?,op,?}, MatrixOp16,
-                                            ZZ_h_mul_r, ZPR4b16,
+multiclass sme2p1_multi_vec_array_vg2_index_16b<string mnemonic, bits<2> sz, bits<3> op,
+                                                RegisterOperand multi_vector_ty, ZPRRegOp zpr_ty> {
+  def NAME : sme2_multi_vec_array_vg2_index<sz, {op{2},?,?,op{1-0},?}, MatrixOp16,
+                                            multi_vector_ty, zpr_ty,
                                             VectorIndexH, mnemonic> {
     bits<3> i;
     let Inst{11-10} = i{2-1};
@@ -2402,7 +2431,7 @@ multiclass sme2p1_multi_vec_array_vg2_index_16b<string mnemonic, bits<2> op> {
   }
   def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i",
         (!cast<Instruction>(NAME) MatrixOp16:$ZAda,  MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3,
-        ZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH:$i), 0>;
+        multi_vector_ty:$Zn, zpr_ty:$Zm, VectorIndexH:$i), 0>;
 }
 
 // SME2 multi-vec ternary indexed two registers 64-bit
@@ -2451,7 +2480,7 @@ multiclass sme2_multi_vec_array_vg2_index_64b<string mnemonic, bits<2> op,
         multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexD32b_timm:$i1), 0>;
 }
 
-class sme2_multi_vec_array_vg4_index<bit sz, bits<6> op, MatrixOperand matrix_ty,
+class sme2_multi_vec_array_vg4_index<bit sz, bits<7> op, MatrixOperand matrix_ty,
                                      RegisterOperand multi_vector_ty,
                                      ZPRRegOp vector_ty, Operand index_ty,
                                      string mnemonic>
@@ -2470,10 +2499,9 @@ class sme2_multi_vec_array_vg4_index<bit sz, bits<6> op, MatrixOperand matrix_ty
   let Inst{19-16} = Zm;
   let Inst{15}    = 0b1;
   let Inst{14-13} = Rv;
-  let Inst{12-10} = op{5-3};
+  let Inst{12-10} = op{6-4};
   let Inst{9-7}   = Zn;
-  let Inst{6}     = 0b0;
-  let Inst{5-3}   = op{2-0};
+  let Inst{6-3}   = op{3-0};
   let Inst{2-0}   = imm3;
 
   let Constraints = "$ZAda = $_ZAda";
@@ -2484,7 +2512,7 @@ multiclass sme2_multi_vec_array_vg4_index_32b<string mnemonic, bits<4> op,
                                               RegisterOperand multi_vector_ty,
                                               ZPRRegOp vector_ty, ValueType vt,
                                               SDPatternOperator intrinsic> {
-  def NAME : sme2_multi_vec_array_vg4_index<0b1, {op{3},?,?,op{2-0}}, MatrixOp32,  multi_vector_ty,
+  def NAME : sme2_multi_vec_array_vg4_index<0b1, {op{3},?,?,0b0, op{2-0}}, MatrixOp32,  multi_vector_ty,
                                             vector_ty, VectorIndexS32b_timm, mnemonic>, SMEPseudo2Instr<NAME, 1> {
    bits<2> i;
    let Inst{11-10} = i;
@@ -2500,9 +2528,11 @@ multiclass sme2_multi_vec_array_vg4_index_32b<string mnemonic, bits<4> op,
 }
 
 // SME2.1 multi-vec ternary indexed four registers 16-bit
-multiclass sme2p1_multi_vec_array_vg4_index_16b<string mnemonic, bits<2> op> {
+multiclass sme2p1_multi_vec_array_vg4_index_16b<string mnemonic, bits<3> op,
+                                                RegisterOperand multi_vector_ty,
+                                                ZPRRegOp zpr_ty> {
   def NAME : sme2_multi_vec_array_vg4_index<0b0,{0b1,?,?,op,?}, MatrixOp16,
-                                            ZZZZ_h_mul_r, ZPR4b16,
+                                            multi_vector_ty, zpr_ty,
                                             VectorIndexH, mnemonic>{
     bits<3> i;
     let Inst{11-10} = i{2-1};
@@ -2511,7 +2541,7 @@ multiclass sme2p1_multi_vec_array_vg4_index_16b<string mnemonic, bits<2> op> {
 
   def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i",
         (!cast<Instruction>(NAME) MatrixOp16:$ZAda,  MatrixIndexGPR32Op8_11:$Rv,
-        sme_elm_idx0_7:$imm3, ZZZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH:$i), 0>;
+        sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm, VectorIndexH:$i), 0>;
 }
 
 // SME2 multi-vec ternary indexed four registers 64-bit
@@ -2561,7 +2591,7 @@ multiclass sme2_multi_vec_array_vg4_index_64b<string mnemonic, bits<3> op,
 }
 //===----------------------------------------------------------------------===//
 // SME2 multi-vec indexed long long MLA one source 32-bit
-class sme2_mla_ll_array_index_32b<string mnemonic, bits<3> op>
+class sme2_mla_ll_array_index_32b<string mnemonic, bits<2> sz, bits<3> op>
     : I<(outs MatrixOp32:$ZAda),
         (ins MatrixOp32:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s4range:$imm2, ZPR8:$Zn, ZPR4b8:$Zm, VectorIndexB32b_timm:$i),
         mnemonic, "\t$ZAda[$Rv, $imm2], $Zn, $Zm$i",
@@ -2571,7 +2601,9 @@ class sme2_mla_ll_array_index_32b<string mnemonic, bits<3> op>
   bits<4> i;
   bits<5> Zn;
   bits<2> imm2;
-  let Inst{31-20} = 0b110000010000;
+  let Inst{31-24} = 0b11000001;
+  let Inst{23-22} = sz;
+  let Inst{21-20} = 0b00;
   let Inst{19-16} = Zm;
   let Inst{15}    = i{3};
   let Inst{14-13} = Rv;
@@ -2583,8 +2615,8 @@ class sme2_mla_ll_array_index_32b<string mnemonic, bits<3> op>
   let Constraints = "$ZAda = $_ZAda";
 }
 
-multiclass sme2_mla_ll_array_index_32b<string mnemonic, bits<3> op, SDPatternOperator intrinsic> {
-  def NAME : sme2_mla_ll_array_index_32b<mnemonic, op>, SMEPseudo2Instr<NAME, 1>;
+multiclass sme2_mla_ll_array_index_32b<string mnemonic, bits<2> sz, bits<3> op, SDPatternOperator intrinsic> {
+  def NAME : sme2_mla_ll_array_index_32b<mnemonic, sz, op>, SMEPseudo2Instr<NAME, 1>;
 
   def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, uimm2s4range, ZPR8, ZPR4b8, VectorIndexB32b_timm, SMEMatrixArray>;
 
@@ -2625,7 +2657,7 @@ multiclass sme2_mla_ll_array_index_64b<string mnemonic, bits<2> op, SDPatternOpe
   def : SME2_ZA_TwoOp_Multi_Index_Pat<NAME, intrinsic, uimm2s4range, ZPR4b16, nxv8i16, VectorIndexH32b_timm, tileslicerange2s4>;
 }
 
-class sme2_mla_ll_array_vg24_index_32b<bit vg4, bits<3> op,
+class sme2_mla_ll_array_vg24_index_32b<bits<2> sz, bit vg4, bits<3> op,
                                        RegisterOperand vector_ty,
                                        string mnemonic>
     : I<(outs MatrixOp32:$ZAda),
@@ -2637,7 +2669,9 @@ class sme2_mla_ll_array_vg24_index_32b<bit vg4, bits<3> op,
   bits<2> Rv;
   bits<4> i;
   bit     imm;
-  let Inst{31-20} = 0b110000010001;
+  let Inst{31-24} = 0b11000001;
+  let Inst{23-22} = sz;
+  let Inst{21-20} = 0b01;
   let Inst{19-16} = Zm;
   let Inst{15}    = vg4;
   let Inst{14-13} = Rv;
@@ -2652,8 +2686,8 @@ class sme2_mla_ll_array_vg24_index_32b<bit vg4, bits<3> op,
 
 //SME2 multi-vec indexed long long MLA two sources 32-bit
 
-multiclass sme2_mla_ll_array_vg2_index_32b<string mnemonic, bits<3> op, SDPatternOperator intrinsic> {
-  def NAME: sme2_mla_ll_array_vg24_index_32b<0b0, op, ZZ_b_mul_r, mnemonic>, SMEPseudo2Instr<NAME, 1> {
+multiclass sme2_mla_ll_array_vg2_index_32b<string mnemonic, bits<2> sz, bits<3> op, SDPatternOperator intrinsic> {
+  def NAME: sme2_mla_ll_array_vg24_index_32b<sz, 0b0, op, ZZ_b_mul_r, mnemonic>, SMEPseudo2Instr<NAME, 1> {
    bits<4> Zn;
    let Inst{9-6} = Zn;
   }
@@ -2668,11 +2702,11 @@ multiclass sme2_mla_ll_array_vg2_index_32b<string mnemonic, bits<3> op, SDPatter
 
 // SME2 multi-vec indexed long long MLA four sources 32-bit
 
-multiclass sme2_mla_ll_array_vg4_index_32b<string mnemonic, bits<3> op, SDPatternOperator intrinsic> {
-  def NAME: sme2_mla_ll_array_vg24_index_32b<0b1, op, ZZZZ_b_mul_r, mnemonic>, SMEPseudo2Instr<NAME, 1> {
+multiclass sme2_mla_ll_array_vg4_index_32b<string mnemonic, bits<2> sz, bits<4> op, SDPatternOperator intrinsic> {
+  def NAME: sme2_mla_ll_array_vg24_index_32b<sz, 0b1, op{2-0}, ZZZZ_b_mul_r, mnemonic>, SMEPseudo2Instr<NAME, 1> {
    bits<3> Zn;
    let Inst{9-7} = Zn;
-   let Inst{6}   = 0b0;
+   let Inst{6}   = op{3};
   }
 
   def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, uimm1s4range, ZZZZ_b_mul_r, ZPR4b8, VectorIndexB32b_timm, SMEMatrixArray>;
@@ -2744,7 +2778,7 @@ multiclass sme2_mla_ll_array_vg4_index_64b<string mnemonic, bits<2> op, SDPatter
 
 //SME2 multiple and single vector long long FMA one source
 
-class sme2_mla_ll_array_single<string mnemonic, bits<4> op,
+class sme2_mla_ll_array_single<string mnemonic, bits<5> op,
                                MatrixOperand matrix_ty, ZPRRegOp vector_ty,
                                ZPRRegOp zpr_ty>
     : I<(outs matrix_ty:$ZAda),
@@ -2757,8 +2791,9 @@ class sme2_mla_ll_array_single<string mnemonic, bits<4> op,
   bits<5> Zn;
   bits<2> imm;
   let Inst{31-23} = 0b110000010;
-  let Inst{22}    = op{3}; //sz
-  let Inst{21-20} = 0b10;
+  let Inst{22}    = op{4}; //sz
+  let Inst{21}    = 0b1;
+  let Inst{20}    = op{3}; //fp8
   let Inst{19-16} = Zm;
   let Inst{15}    = 0b0;
   let Inst{14-13} = Rv;
@@ -2770,7 +2805,7 @@ class sme2_mla_ll_array_single<string mnemonic, bits<4> op,
   let Constraints = "$ZAda = $_ZAda";
 }
 
-multiclass sme2_mla_ll_array_single<string mnemonic, bits<4> op,
+multiclass sme2_mla_ll_array_single<string mnemonic, bits<5> op,
                                     MatrixOperand matrix_ty, ZPRRegOp vector_ty,
                                     ZPRRegOp zpr_ty, ValueType vt, SDPatternOperator intrinsic> {
   def NAME : sme2_mla_ll_array_single<mnemonic, op, matrix_ty, vector_ty, zpr_ty>, SMEPseudo2Instr<NAME, 1>;
@@ -2780,29 +2815,28 @@ multiclass sme2_mla_ll_array_single<string mnemonic, bits<4> op,
   def : SME2_ZA_TwoOp_Multi_Single_Pat<NAME, intrinsic, uimm2s4range, zpr_ty, vt, tileslicerange2s4>;
 }
 
-class sme2_mla_ll_array_vg24_single<bits<5> op, MatrixOperand matrix_ty,
+class sme2_mla_ll_array_vg24_single<bits<6> op, MatrixOperand matrix_ty,
                                     RegisterOperand vector_ty, ZPRRegOp zpr_ty,
                                     string mnemonic>
     : I<(outs matrix_ty:$ZAda),
         (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm,
              vector_ty:$Zn, zpr_ty:$Zm),
-        mnemonic, "\t$ZAda[$Rv, $imm,  " # !if(op{3}, "vgx4", "vgx2") # "], $Zn, $Zm",
+        mnemonic, "\t$ZAda[$Rv, $imm,  " # !if(op{4}, "vgx4", "vgx2") # "], $Zn, $Zm",
         "", []>, Sched<[]> {
   bits<4> Zm;
   bits<2> Rv;
   bits<5> Zn;
   bit     imm;
   let Inst{31-23} = 0b110000010;
-  let Inst{22}    = op{4}; //sz
+  let Inst{22}    = op{5}; //sz
   let Inst{21}    = 0b1;
-  let Inst{20}    = op{3}; //vg4
+  let Inst{20}    = op{4}; //vg4
   let Inst{19-16} = Zm;
   let Inst{15}    = 0b0;
   let Inst{14-13} = Rv;
   let Inst{12-10} = 0b000;
   let Inst{9-5}   = Zn;
-  let Inst{4-2}   = op{2-0};
-  let Inst{1}     = 0b0;
+  let Inst{4-1}   = op{3-0};
   let Inst{0}     = imm;
 
   let Constraints = "$ZAda = $_ZAda";
@@ -2810,7 +2844,7 @@ class sme2_mla_ll_array_vg24_single<bits<5> op, MatrixOperand matrix_ty,
 
 //SME2 single-multi long long MLA two and four sources
 
-multiclass sme2_mla_ll_array_vg24_single<string mnemonic, bits<5> op,
+multiclass sme2_mla_ll_array_vg24_single<string mnemonic, bits<6> op,
                                           MatrixOperand matrix_ty,
                                           RegisterOperand multi_vector_ty,
                                           ZPRRegOp zpr_ty> {
@@ -2828,7 +2862,7 @@ multiclass sme2_mla_ll_array_vg2_single<string mnemonic, bits<5> op,
                                         RegisterOperand multi_vector_ty,
                                         ZPRRegOp zpr_ty, ValueType vt, SDPatternOperator intrinsic> {
 
-  defm NAME: sme2_mla_ll_array_vg24_single<mnemonic, op, matrix_ty, multi_vector_ty, zpr_ty>;
+  defm NAME: sme2_mla_ll_array_vg24_single<mnemonic, {op, 0b0}, matrix_ty, multi_vector_ty, zpr_ty>;
 
   def : SME2_ZA_TwoOp_VG2_Multi_Single_Pat<NAME, intrinsic, uimm1s4range, zpr_ty, vt, tileslicerange1s4>;
 }
@@ -2837,14 +2871,14 @@ multiclass sme2_mla_ll_array_vg4_single<string mnemonic, bits<5> op,
                                         MatrixOperand matrix_ty,
                                         RegisterOperand multi_vector_ty,
                                         ZPRRegOp zpr_ty, ValueType vt, SDPatternOperator intrinsic> {
-  defm NAME: sme2_mla_ll_array_vg24_single<mnemonic, op, matrix_ty, multi_vector_ty, zpr_ty>;
+  defm NAME: sme2_mla_ll_array_vg24_single<mnemonic, {op, 0b0}, matrix_ty, multi_vector_ty, zpr_ty>;
 
   def : SME2_ZA_TwoOp_VG4_Multi_Single_Pat<NAME, intrinsic, uimm1s4range, zpr_ty, vt, tileslicerange1s4>;
 }
 
 // SME2 multiple vectors long long MLA two sources
 
-class sme2_mla_ll_array_vg2_multi<bits<4> op, MatrixOperand matrix_ty,
+class sme2_mla_ll_array_vg2_multi<bits<5> op, MatrixOperand matrix_ty,
                                   RegisterOperand vector_ty,string mnemonic>
     : I<(outs matrix_ty:$ZAda),
         (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm,
@@ -2856,22 +2890,21 @@ class sme2_mla_ll_array_vg2_multi<bits<4> op, MatrixOperand matrix_ty,
   bits<4> Zn;
   bit     imm;
   let Inst{31-23} = 0b110000011;
-  let Inst{22}    = op{3};  // sz
+  let Inst{22}    = op{4};  // sz
   let Inst{21}    = 0b1;
   let Inst{20-17} = Zm;
   let Inst{16-15} = 0b00;
   let Inst{14-13} = Rv;
   let Inst{12-10} = 0b000;
   let Inst{9-6}   = Zn;
-  let Inst{5}     = 0b0;
-  let Inst{4-2}   = op{2-0};
+  let Inst{5-2}   = op{3-0};
   let Inst{1}     = 0b0;
   let Inst{0}     = imm;
 
   let Constraints = "$ZAda = $_ZAda";
 }
 
-multiclass sme2_mla_ll_array_vg2_multi<string mnemonic, bits<4> op,
+multiclass sme2_mla_ll_array_vg2_multi<string mnemonic, bits<5> op,
                                        MatrixOperand matrix_ty,
                                        RegisterOperand vector_ty,
                                        ValueType vt, SDPatternOperator intrinsic> {
@@ -2887,7 +2920,7 @@ multiclass sme2_mla_ll_array_vg2_multi<string mnemonic, bits<4> op,
 
 // SME2 multiple vectors long long MLA four sources
 
-class sme2_mla_ll_array_vg4_multi<bits<4> op,MatrixOperand matrix_ty,
+class sme2_mla_ll_array_vg4_multi<bits<5> op,MatrixOperand matrix_ty,
                                   RegisterOperand vector_ty,
                                   string mnemonic>
     : I<(outs matrix_ty:$ZAda),
@@ -2900,22 +2933,22 @@ class sme2_mla_ll_array_vg4_multi<bits<4> op,MatrixOperand matrix_ty,
   bits<3> Zn;
   bit     imm;
   let Inst{31-23} = 0b110000011;
-  let Inst{22}    = op{3}; // sz
+  let Inst{22}    = op{4}; // sz
   let Inst{21}    = 0b1;
   let Inst{20-18} = Zm;
   let Inst{17-15} = 0b010;
   let Inst{14-13} = Rv;
   let Inst{12-10} = 0b000;
   let Inst{9-7}   = Zn;
-  let Inst{6-5}   = 0b00;
-  let Inst{4-2}   = op{2-0};
+  let Inst{6}     = 0b0;
+  let Inst{5-2}   = op{3-0};
   let Inst{1}     = 0b0;
   let Inst{0}     = imm;
 
   let Constraints = "$ZAda = $_ZAda";
 }
 
-multiclass sme2_mla_ll_array_vg4_multi<string mnemonic, bits<4> op,
+multiclass sme2_mla_ll_array_vg4_multi<string mnemonic, bits<5> op,
                                        MatrixOperand matrix_ty,
                                        RegisterOperand vector_ty,
                                        ValueType vt, SDPatternOperator intrinsic> {
@@ -2985,7 +3018,7 @@ class sme2_spill_fill_vector<string mnemonic, bits<8> opc>
 // SME2 move to/from lookup table
 class sme2_movt_zt_to_scalar<string mnemonic, bits<7> opc>
     : I<(outs GPR64:$Rt), (ins ZTR:$ZTt, uimm3s8:$imm3),
-         mnemonic, "\t$Rt, $ZTt$imm3",
+         mnemonic, "\t$Rt, $ZTt[$imm3]",
          "", []>, Sched<[]> {
   bits<3> imm3;
   bits<5> Rt;
@@ -2997,7 +3030,7 @@ class sme2_movt_zt_to_scalar<string mnemonic, bits<7> opc>
 
 class sme2_movt_scalar_to_zt<string mnemonic, bits<7> opc>
     : I<(outs ZTR:$ZTt), (ins uimm3s8:$imm3, GPR64:$Rt),
-         mnemonic, "\t$ZTt$imm3, $Rt",
+         mnemonic, "\t$ZTt[$imm3], $Rt",
          "", []>, Sched<[]> {
   bits<3> imm3;
   bits<5> Rt;
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index d56540acf7..7bb457d918 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -8721,8 +8721,8 @@ multiclass sve2_crypto_unary_op<bit opc, string asm, SDPatternOperator op> {
 // SVE BFloat16 Group
 //===----------------------------------------------------------------------===//
 
-class sve_float_dot<bit bf, string asm>
-: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm),
+class sve_float_dot<bit bf, bit o2, ZPRRegOp dst_ty, ZPRRegOp src_ty, string asm>
+: I<(outs dst_ty:$Zda), (ins dst_ty:$_Zda, src_ty:$Zn, src_ty:$Zm),
      asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> {
   bits<5> Zda;
   bits<5> Zn;
@@ -8731,7 +8731,8 @@ class sve_float_dot<bit bf, string asm>
   let Inst{22}    = bf;
   let Inst{21}    = 0b1;
   let Inst{20-16} = Zm;
-  let Inst{15-10} = 0b100000;
+  let Inst{15-11} = 0b10000;
+  let Inst{10}    = o2;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zda;
 
@@ -8741,24 +8742,24 @@ class sve_float_dot<bit bf, string asm>
   let mayRaiseFPException = 1;
 }
 
-multiclass sve_float_dot<bit bf, string asm, ValueType InVT, SDPatternOperator op> {
-  def NAME : sve_float_dot<bf, asm>;
+multiclass sve_float_dot<bit bf, bit o2, ZPRRegOp dst_ty, ZPRRegOp src_ty,
+                         string asm, ValueType InVT, SDPatternOperator op> {
+  def NAME : sve_float_dot<bf, o2, dst_ty, src_ty, asm>;
   def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, InVT, InVT, !cast<Instruction>(NAME)>;
 }
 
-class sve_float_dot_indexed<bit bf, string asm>
-: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, VectorIndexS32b:$iop),
+class sve_float_dot_indexed<bit bf, ZPRRegOp dst_ty, ZPRRegOp src1_ty,
+                            ZPRRegOp src2_ty, Operand iop_ty, string asm>
+: I<(outs dst_ty:$Zda), (ins dst_ty:$_Zda, src1_ty:$Zn, src2_ty:$Zm, iop_ty:$iop),
     asm, "\t$Zda, $Zn, $Zm$iop", "", []>, Sched<[]> {
   bits<5> Zda;
   bits<5> Zn;
   bits<3> Zm;
-  bits<2> iop;
   let Inst{31-23} = 0b011001000;
   let Inst{22}    = bf;
   let Inst{21}    = 0b1;
-  let Inst{20-19} = iop;
   let Inst{18-16} = Zm;
-  let Inst{15-10} = 0b010000;
+  let Inst{15-12} = 0b0100;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zda;
 
@@ -8768,8 +8769,14 @@ class sve_float_dot_indexed<bit bf, string asm>
   let mayRaiseFPException = 1;
 }
 
-multiclass sve_float_dot_indexed<bit bf, string asm, ValueType InVT, SDPatternOperator op> {
-  def NAME : sve_float_dot_indexed<bf, asm>;
+multiclass sve_float_dot_indexed<bit bf, bits<2> opc, ZPRRegOp src1_ty,
+                                 ZPRRegOp src2_ty, string asm, ValueType InVT,
+                                 SDPatternOperator op> {
+  def NAME : sve_float_dot_indexed<bf, ZPR32, src1_ty, src2_ty, VectorIndexS32b, asm> {
+    bits<2> iop;
+    let Inst{20-19} = iop;
+    let Inst{11-10} = opc;
+  }
   def : SVE_4_Op_Imm_Pat<nxv4f32, op, nxv4f32, InVT, InVT, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME)>;
 }
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
index bf5a7b0a96..bf244bb024 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -693,6 +693,13 @@ def FeatureAtomicGlobalPkAddBF16Inst : SubtargetFeature<"atomic-global-pk-add-bf
  [FeatureFlatGlobalInsts]
 >;
 
+def FeatureAtomicCSubNoRtnInsts : SubtargetFeature<"atomic-csub-no-rtn-insts",
+  "HasAtomicCSubNoRtnInsts",
+  "true",
+  "Has buffer_atomic_csub and global_atomic_csub instructions that don't "
+  "return original value"
+>;
+
 def FeatureFlatAtomicFaddF32Inst
   : SubtargetFeature<"flat-atomic-fadd-f32-inst",
   "HasFlatAtomicFaddF32Inst",
@@ -1927,6 +1934,8 @@ def HasGWS : Predicate<"Subtarget->hasGWS()">;
 def HasCvtFP8VOP1Bug : Predicate<"Subtarget->hasCvtFP8VOP1Bug()">;
 def HasNoCvtFP8VOP1Bug : Predicate<"!Subtarget->hasCvtFP8VOP1Bug()">;
 
+def HasAtomicCSubNoRtnInsts : Predicate<"Subtarget->hasAtomicCSubNoRtnInsts()">;
+
 // Include AMDGPU TD files
 include "SISchedule.td"
 include "GCNProcessors.td"
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
index 0f28074ea1..c67a21c639 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
@@ -965,11 +965,10 @@ class MFMASmallGemmSingleWaveOpt final : public IGLPStrategy {
 
       // Does the VALU have a DS_WRITE successor that is the same as other
       // VALU already in the group. The V_PERMs will all share 1 DS_W succ
-      return std::any_of(Cache->begin(), Cache->end(), [&SU](SUnit *Elt) {
-        return std::any_of(SU->Succs.begin(), SU->Succs.end(),
-                           [&Elt](const SDep &ThisSucc) {
-                             return ThisSucc.getSUnit() == Elt;
-                           });
+      return llvm::any_of(*Cache, [&SU](SUnit *Elt) {
+        return llvm::any_of(SU->Succs, [&Elt](const SDep &ThisSucc) {
+          return ThisSucc.getSUnit() == Elt;
+        });
       });
     }
 
@@ -1088,10 +1087,9 @@ class MFMASmallGemmSingleWaveOpt final : public IGLPStrategy {
       auto DAG = SyncPipe[0].DAG;
       // Does the previous DS_WRITE share a V_PERM predecessor with this
       // VMEM_READ
-      return (
-          std::any_of(Cache->begin(), Cache->end(), [&SU, &DAG](SUnit *Elt) {
-            return DAG->IsReachable(const_cast<SUnit *>(SU), Elt);
-          }));
+      return llvm::any_of(*Cache, [&SU, &DAG](SUnit *Elt) {
+        return DAG->IsReachable(const_cast<SUnit *>(SU), Elt);
+      });
     }
     SharesPredWithPrevNthGroup(unsigned Distance, const SIInstrInfo *TII,
                                unsigned SGID, bool NeedsCache = false)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index 607d59db7b..adf4e0139e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -5041,6 +5041,36 @@ SDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N,
     return performAssertSZExtCombine(N, DCI);
   case ISD::INTRINSIC_WO_CHAIN:
     return performIntrinsicWOChainCombine(N, DCI);
+  case AMDGPUISD::FMAD_FTZ: {
+    SDValue N0 = N->getOperand(0);
+    SDValue N1 = N->getOperand(1);
+    SDValue N2 = N->getOperand(2);
+    EVT VT = N->getValueType(0);
+
+    // FMAD_FTZ is a FMAD + flush denormals to zero.
+    // We flush the inputs, the intermediate step, and the output.
+    ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
+    ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
+    ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
+    if (N0CFP && N1CFP && N2CFP) {
+      const auto FTZ = [](const APFloat &V) {
+        if (V.isDenormal()) {
+          APFloat Zero(V.getSemantics(), 0);
+          return V.isNegative() ? -Zero : Zero;
+        }
+        return V;
+      };
+
+      APFloat V0 = FTZ(N0CFP->getValueAPF());
+      APFloat V1 = FTZ(N1CFP->getValueAPF());
+      APFloat V2 = FTZ(N2CFP->getValueAPF());
+      V0.multiply(V1, APFloat::rmNearestTiesToEven);
+      V0 = FTZ(V0);
+      V0.add(V2, APFloat::rmNearestTiesToEven);
+      return DAG.getConstantFP(FTZ(V0), DL, VT);
+    }
+    break;
+  }
   }
   return SDValue();
 }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUImageIntrinsicOptimizer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUImageIntrinsicOptimizer.cpp
index acfd340768..e5fbcca1e7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUImageIntrinsicOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUImageIntrinsicOptimizer.cpp
@@ -108,28 +108,23 @@ void addInstToMergeableList(
     if (IIList.front()->getType() != II->getType())
       continue;
 
-    // Check DMask.
-    Value *DMaskList = IIList.front()->getArgOperand(ImageDimIntr->DMaskIndex);
-    Value *DMask = II->getArgOperand(ImageDimIntr->DMaskIndex);
-    if (DMaskList != DMask)
-      continue;
-
-    // Check VAddr (except FragId).
-    int I = ImageDimIntr->VAddrStart;
-    for (; I < ImageDimIntr->VAddrEnd - 1; ++I) {
-      if (IIList.front()->getArgOperand(I) != II->getArgOperand(I))
-        break;
+    // Check all arguments (DMask, VAddr, RSrc etc).
+    bool AllEqual = true;
+    assert(IIList.front()->arg_size() == II->arg_size());
+    for (int I = 1, E = II->arg_size(); AllEqual && I != E; ++I) {
+      Value *ArgList = IIList.front()->getArgOperand(I);
+      Value *Arg = II->getArgOperand(I);
+      if (I == ImageDimIntr->VAddrEnd - 1) {
+        // Check FragId group.
+        auto FragIdList = cast<ConstantInt>(IIList.front()->getArgOperand(I));
+        auto FragId = cast<ConstantInt>(II->getArgOperand(I));
+        AllEqual = FragIdList->getValue().udiv(4) == FragId->getValue().udiv(4);
+      } else {
+        // Check all arguments except FragId.
+        AllEqual = ArgList == Arg;
+      }
     }
-
-    if (I != ImageDimIntr->VAddrEnd - 1)
-      continue;
-
-    // Check FragId group.
-    const uint8_t FragIdIndex = ImageDimIntr->VAddrEnd - 1;
-    Value *FragIdList = IIList.front()->getArgOperand(FragIdIndex);
-    auto IIListFragId = cast<ConstantInt>(FragIdList);
-    auto IIFragId = cast<ConstantInt>(II->getArgOperand(FragIdIndex));
-    if (IIListFragId->getValue().udiv(4) != IIFragId->getValue().udiv(4))
+    if (!AllEqual)
       continue;
 
     // Add to the list.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
index 12ccfd29f2..23242ad84b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
@@ -125,11 +125,11 @@ def InstFlag : OperandWithDefaultOps <i32, (ops (i32 0))>;
 
 def i1imm_0 : OperandWithDefaultOps<i1, (ops (i1 0))>;
 
-class CustomOperandClass<string name, bit optional, string parserMethod,
-                         string defaultMethod>
+class CustomOperandClass<string name, bit optional, string predicateMethod,
+                         string parserMethod, string defaultMethod>
     : AsmOperandClass {
   let Name = name;
-  let PredicateMethod = "is"#name;
+  let PredicateMethod = predicateMethod;
   let ParserMethod = parserMethod;
   let RenderMethod = "addImmOperands";
   let IsOptional = optional;
@@ -138,6 +138,7 @@ class CustomOperandClass<string name, bit optional, string parserMethod,
 
 class CustomOperandProps<bit optional = 0, string name = NAME> {
   string ImmTy = "ImmTy"#name;
+  string PredicateMethod = "is"#name;
   string ParserMethod = "parse"#name;
   string DefaultValue = "0";
   string DefaultMethod = "[this]() { return "#
@@ -145,7 +146,8 @@ class CustomOperandProps<bit optional = 0, string name = NAME> {
     "AMDGPUOperand::"#ImmTy#"); }";
   string PrintMethod = "print"#name;
   AsmOperandClass ParserMatchClass =
-    CustomOperandClass<name, optional, ParserMethod, DefaultMethod>;
+    CustomOperandClass<name, optional, PredicateMethod, ParserMethod,
+                       DefaultMethod>;
   string OperandType = "OPERAND_IMMEDIATE";
 }
 
@@ -621,6 +623,7 @@ defm int_amdgcn_flat_atomic_fadd : global_addr_space_atomic_op;
 defm int_amdgcn_global_atomic_fadd_v2bf16 : noret_op;
 defm int_amdgcn_global_atomic_fmin : noret_op;
 defm int_amdgcn_global_atomic_fmax : noret_op;
+defm int_amdgcn_global_atomic_csub : noret_op;
 defm int_amdgcn_flat_atomic_fadd : local_addr_space_atomic_op;
 defm int_amdgcn_ds_fadd_v2bf16 : noret_op;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 4797e5a7a6..3d70ed150d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -4641,8 +4641,8 @@ bool AMDGPULegalizerInfo::legalizeFastUnsafeFDIV(MachineInstr &MI,
     }
   }
 
-  // For f16 require arcp only.
-  // For f32 require afn+arcp.
+  // For f16 require afn or arcp.
+  // For f32 require afn.
   if (!AllowInaccurateRcp && (ResTy != LLT::scalar(16) ||
                               !MI.getFlag(MachineInstr::FmArcp)))
     return false;
@@ -5879,31 +5879,23 @@ bool AMDGPULegalizerInfo::legalizeBufferAtomic(MachineInstr &MI,
       IID == Intrinsic::amdgcn_struct_buffer_atomic_cmpswap ||
       IID == Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap ||
       IID == Intrinsic::amdgcn_struct_ptr_buffer_atomic_cmpswap;
-  const bool HasReturn = MI.getNumExplicitDefs() != 0;
-
-  Register Dst;
-
-  int OpOffset = 0;
-  if (HasReturn) {
-    // A few FP atomics do not support return values.
-    Dst = MI.getOperand(0).getReg();
-  } else {
-    OpOffset = -1;
-  }
 
+  Register Dst = MI.getOperand(0).getReg();
   // Since we don't have 128-bit atomics, we don't need to handle the case of
   // p8 argmunents to the atomic itself
-  Register VData = MI.getOperand(2 + OpOffset).getReg();
+  Register VData = MI.getOperand(2).getReg();
+
   Register CmpVal;
+  int OpOffset = 0;
 
   if (IsCmpSwap) {
-    CmpVal = MI.getOperand(3 + OpOffset).getReg();
+    CmpVal = MI.getOperand(3).getReg();
     ++OpOffset;
   }
 
   castBufferRsrcArgToV4I32(MI, B, 3 + OpOffset);
   Register RSrc = MI.getOperand(3 + OpOffset).getReg();
-  const unsigned NumVIndexOps = (IsCmpSwap ? 8 : 7) + HasReturn;
+  const unsigned NumVIndexOps = IsCmpSwap ? 9 : 8;
 
   // The struct intrinsic variants add one additional operand over raw.
   const bool HasVIndex = MI.getNumOperands() == NumVIndexOps;
@@ -5924,12 +5916,9 @@ bool AMDGPULegalizerInfo::legalizeBufferAtomic(MachineInstr &MI,
   unsigned ImmOffset;
   std::tie(VOffset, ImmOffset) = splitBufferOffsets(B, VOffset);
 
-  auto MIB = B.buildInstr(getBufferAtomicPseudo(IID));
-
-  if (HasReturn)
-    MIB.addDef(Dst);
-
-  MIB.addUse(VData); // vdata
+  auto MIB = B.buildInstr(getBufferAtomicPseudo(IID))
+      .addDef(Dst)
+      .addUse(VData); // vdata
 
   if (IsCmpSwap)
     MIB.addReg(CmpVal);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index b7c1f03459..dc7321cd5d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -50,6 +50,7 @@
 #include "llvm/InitializePasses.h"
 #include "llvm/MC/TargetRegistry.h"
 #include "llvm/Passes/PassBuilder.h"
+#include "llvm/Transforms/HipStdPar/HipStdPar.h"
 #include "llvm/Transforms/IPO.h"
 #include "llvm/Transforms/IPO/AlwaysInliner.h"
 #include "llvm/Transforms/IPO/GlobalDCE.h"
@@ -348,6 +349,11 @@ static cl::opt<bool> EnableRewritePartialRegUses(
     cl::desc("Enable rewrite partial reg uses pass"), cl::init(false),
     cl::Hidden);
 
+static cl::opt<bool> EnableHipStdPar(
+  "amdgpu-enable-hipstdpar",
+  cl::desc("Enable HIP Standard Parallelism Offload support"), cl::init(false),
+  cl::Hidden);
+
 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
   // Register the target
   RegisterTargetMachine<R600TargetMachine> X(getTheR600Target());
@@ -699,6 +705,8 @@ void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
         if (EnableLibCallSimplify && Level != OptimizationLevel::O0)
           FPM.addPass(AMDGPUSimplifyLibCallsPass());
         PM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+        if (EnableHipStdPar)
+          PM.addPass(HipStdParAcceleratorCodeSelectionPass());
       });
 
   PB.registerPipelineEarlySimplificationEPCallback(
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 074c862657..cb877a4695 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -650,6 +650,15 @@ InstructionCost GCNTTIImpl::getArithmeticInstrCost(
       return LT.first * Cost * NElts;
     }
 
+    if (SLT == MVT::f32 && ((CxtI && CxtI->hasApproxFunc()) ||
+                            TLI->getTargetMachine().Options.UnsafeFPMath)) {
+      // Fast unsafe fdiv lowering:
+      // f32 rcp
+      // f32 fmul
+      int Cost = getQuarterRateInstrCost(CostKind) + getFullRateInstrCost();
+      return LT.first * Cost * NElts;
+    }
+
     if (SLT == MVT::f32 || SLT == MVT::f16) {
       // 4 more v_cvt_* insts without f16 insts support
       int Cost = (SLT == MVT::f16 ? 14 : 10) * getFullRateInstrCost() +
diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index 35656bcaea..fe2729f979 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -75,6 +75,7 @@ class AMDGPUOperand : public MCParsedAsmOperand {
     bool Abs = false;
     bool Neg = false;
     bool Sext = false;
+    bool Lit = false;
 
     bool hasFPModifiers() const { return Abs || Neg; }
     bool hasIntModifiers() const { return Sext; }
@@ -355,25 +356,20 @@ class AMDGPUOperand : public MCParsedAsmOperand {
     return isImm() && Imm.Type == ImmT;
   }
 
+  template <ImmTy Ty> bool isImmTy() const { return isImmTy(Ty); }
+
   bool isImmLiteral() const { return isImmTy(ImmTyNone); }
 
   bool isImmModifier() const {
     return isImm() && Imm.Type != ImmTyNone;
   }
 
-  bool isClampSI() const { return isImmTy(ImmTyClampSI); }
   bool isOModSI() const { return isImmTy(ImmTyOModSI); }
   bool isDMask() const { return isImmTy(ImmTyDMask); }
   bool isDim() const { return isImmTy(ImmTyDim); }
-  bool isUNorm() const { return isImmTy(ImmTyUNorm); }
-  bool isDA() const { return isImmTy(ImmTyDA); }
   bool isR128A16() const { return isImmTy(ImmTyR128A16); }
-  bool isA16() const { return isImmTy(ImmTyA16); }
-  bool isLWE() const { return isImmTy(ImmTyLWE); }
   bool isOff() const { return isImmTy(ImmTyOff); }
   bool isExpTgt() const { return isImmTy(ImmTyExpTgt); }
-  bool isExpVM() const { return isImmTy(ImmTyExpVM); }
-  bool isExpCompr() const { return isImmTy(ImmTyExpCompr); }
   bool isOffen() const { return isImmTy(ImmTyOffen); }
   bool isIdxen() const { return isImmTy(ImmTyIdxen); }
   bool isAddr64() const { return isImmTy(ImmTyAddr64); }
@@ -386,7 +382,6 @@ class AMDGPUOperand : public MCParsedAsmOperand {
   bool isLDS() const { return isImmTy(ImmTyLDS); }
   bool isCPol() const { return isImmTy(ImmTyCPol); }
   bool isTFE() const { return isImmTy(ImmTyTFE); }
-  bool isD16() const { return isImmTy(ImmTyD16); }
   bool isFORMAT() const { return isImmTy(ImmTyFORMAT) && isUInt<7>(getImm()); }
   bool isDppBankMask() const { return isImmTy(ImmTyDppBankMask); }
   bool isDppRowMask() const { return isImmTy(ImmTyDppRowMask); }
@@ -403,7 +398,6 @@ class AMDGPUOperand : public MCParsedAsmOperand {
   bool isOpSelHi() const { return isImmTy(ImmTyOpSelHi); }
   bool isNegLo() const { return isImmTy(ImmTyNegLo); }
   bool isNegHi() const { return isImmTy(ImmTyNegHi); }
-  bool isHigh() const { return isImmTy(ImmTyHigh); }
 
   bool isRegOrImm() const {
     return isReg() || isImm();
@@ -1583,9 +1577,11 @@ class AMDGPUAsmParser : public MCTargetAsmParser {
   bool isNamedOperandModifier(const AsmToken &Token, const AsmToken &NextToken) const;
   bool isOpcodeModifierWithVal(const AsmToken &Token, const AsmToken &NextToken) const;
   bool parseSP3NegModifier();
-  ParseStatus parseImm(OperandVector &Operands, bool HasSP3AbsModifier = false);
+  ParseStatus parseImm(OperandVector &Operands, bool HasSP3AbsModifier = false,
+                       bool HasLit = false);
   ParseStatus parseReg(OperandVector &Operands);
-  ParseStatus parseRegOrImm(OperandVector &Operands, bool HasSP3AbsMod = false);
+  ParseStatus parseRegOrImm(OperandVector &Operands, bool HasSP3AbsMod = false,
+                            bool HasLit = false);
   ParseStatus parseRegOrImmWithFPInputMods(OperandVector &Operands,
                                            bool AllowImm = true);
   ParseStatus parseRegOrImmWithIntInputMods(OperandVector &Operands,
@@ -2140,9 +2136,10 @@ void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyMo
           const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(Inst.getLoc(),
           "Can't encode literal as exact 64-bit floating-point operand. "
           "Low 32-bits will be set to zero");
+          Val &= 0xffffffff00000000u;
         }
 
-        Inst.addOperand(MCOperand::createImm(Literal.lshr(32).getZExtValue()));
+        Inst.addOperand(MCOperand::createImm(Val));
         setImmKindLiteral();
         return;
       }
@@ -2241,7 +2238,10 @@ void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyMo
       return;
     }
 
-    Inst.addOperand(MCOperand::createImm(Lo_32(Val)));
+    Val = AMDGPU::isSISrcFPOperand(InstDesc, OpNum) ? (uint64_t)Val << 32
+                                                    : Lo_32(Val);
+
+    Inst.addOperand(MCOperand::createImm(Val));
     setImmKindLiteral();
     return;
 
@@ -2906,13 +2906,26 @@ AMDGPUAsmParser::parseRegister(bool RestoreOnFailure) {
 }
 
 ParseStatus AMDGPUAsmParser::parseImm(OperandVector &Operands,
-                                      bool HasSP3AbsModifier) {
+                                      bool HasSP3AbsModifier, bool HasLit) {
   // TODO: add syntactic sugar for 1/(2*PI)
 
   if (isRegister())
     return ParseStatus::NoMatch;
   assert(!isModifier());
 
+  if (!HasLit) {
+    HasLit = trySkipId("lit");
+    if (HasLit) {
+      if (!skipToken(AsmToken::LParen, "expected left paren after lit"))
+        return ParseStatus::Failure;
+      ParseStatus S = parseImm(Operands, HasSP3AbsModifier, HasLit);
+      if (S.isSuccess() &&
+          !skipToken(AsmToken::RParen, "expected closing parentheses"))
+        return ParseStatus::Failure;
+      return S;
+    }
+  }
+
   const auto& Tok = getToken();
   const auto& NextTok = peekToken();
   bool IsReal = Tok.is(AsmToken::Real);
@@ -2925,6 +2938,9 @@ ParseStatus AMDGPUAsmParser::parseImm(OperandVector &Operands,
     Negate = true;
   }
 
+  AMDGPUOperand::Modifiers Mods;
+  Mods.Lit = HasLit;
+
   if (IsReal) {
     // Floating-point expressions are not supported.
     // Can only allow floating-point literals with an
@@ -2943,6 +2959,8 @@ ParseStatus AMDGPUAsmParser::parseImm(OperandVector &Operands,
     Operands.push_back(
       AMDGPUOperand::CreateImm(this, RealVal.bitcastToAPInt().getZExtValue(), S,
                                AMDGPUOperand::ImmTyNone, true));
+    AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
+    Op.setModifiers(Mods);
 
     return ParseStatus::Success;
 
@@ -2969,7 +2987,11 @@ ParseStatus AMDGPUAsmParser::parseImm(OperandVector &Operands,
 
     if (Expr->evaluateAsAbsolute(IntVal)) {
       Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
+      AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
+      Op.setModifiers(Mods);
     } else {
+      if (HasLit)
+        return ParseStatus::NoMatch;
       Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
     }
 
@@ -2992,13 +3014,13 @@ ParseStatus AMDGPUAsmParser::parseReg(OperandVector &Operands) {
 }
 
 ParseStatus AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands,
-                                           bool HasSP3AbsMod) {
+                                           bool HasSP3AbsMod, bool HasLit) {
   ParseStatus Res = parseReg(Operands);
   if (!Res.isNoMatch())
     return Res;
   if (isModifier())
     return ParseStatus::NoMatch;
-  return parseImm(Operands, HasSP3AbsMod);
+  return parseImm(Operands, HasSP3AbsMod, HasLit);
 }
 
 bool
@@ -3094,6 +3116,7 @@ AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands,
                                               bool AllowImm) {
   bool Neg, SP3Neg;
   bool Abs, SP3Abs;
+  bool Lit;
   SMLoc Loc;
 
   // Disable ambiguous constructs like '--1' etc. Should use neg(-1) instead.
@@ -3113,6 +3136,10 @@ AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands,
   if (Abs && !skipToken(AsmToken::LParen, "expected left paren after abs"))
     return ParseStatus::Failure;
 
+  Lit = trySkipId("lit");
+  if (Lit && !skipToken(AsmToken::LParen, "expected left paren after lit"))
+    return ParseStatus::Failure;
+
   Loc = getLoc();
   SP3Abs = trySkipToken(AsmToken::Pipe);
   if (Abs && SP3Abs)
@@ -3120,12 +3147,15 @@ AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands,
 
   ParseStatus Res;
   if (AllowImm) {
-    Res = parseRegOrImm(Operands, SP3Abs);
+    Res = parseRegOrImm(Operands, SP3Abs, Lit);
   } else {
     Res = parseReg(Operands);
   }
   if (!Res.isSuccess())
-    return (SP3Neg || Neg || SP3Abs || Abs) ? ParseStatus::Failure : Res;
+    return (SP3Neg || Neg || SP3Abs || Abs || Lit) ? ParseStatus::Failure : Res;
+
+  if (Lit && !Operands.back()->isImm())
+    Error(Loc, "expected immediate with lit modifier");
 
   if (SP3Abs && !skipToken(AsmToken::Pipe, "expected vertical bar"))
     return ParseStatus::Failure;
@@ -3133,12 +3163,15 @@ AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands,
     return ParseStatus::Failure;
   if (Neg && !skipToken(AsmToken::RParen, "expected closing parentheses"))
     return ParseStatus::Failure;
+  if (Lit && !skipToken(AsmToken::RParen, "expected closing parentheses"))
+    return ParseStatus::Failure;
 
   AMDGPUOperand::Modifiers Mods;
   Mods.Abs = Abs || SP3Abs;
   Mods.Neg = Neg || SP3Neg;
+  Mods.Lit = Lit;
 
-  if (Mods.hasFPModifiers()) {
+  if (Mods.hasFPModifiers() || Lit) {
     AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
     if (Op.isExpr())
       return Error(Op.getStartLoc(), "expected an absolute expression");
@@ -4297,7 +4330,19 @@ bool AMDGPUAsmParser::validateVOPLiteral(const MCInst &Inst,
       continue;
 
     if (MO.isImm() && !isInlineConstant(Inst, OpIdx)) {
-      uint32_t Value = static_cast<uint32_t>(MO.getImm());
+      uint64_t Value = static_cast<uint64_t>(MO.getImm());
+      bool IsFP64 = AMDGPU::isSISrcFPOperand(Desc, OpIdx) &&
+                    AMDGPU::getOperandSize(Desc.operands()[OpIdx]) == 8;
+      bool IsValid32Op = AMDGPU::isValid32BitLiteral(Value, IsFP64);
+
+      if (!IsValid32Op && !isInt<32>(Value) && !isUInt<32>(Value)) {
+        Error(getLitLoc(Operands), "invalid operand for instruction");
+        return false;
+      }
+
+      if (IsFP64 && IsValid32Op)
+        Value = Hi_32(Value);
+
       if (NumLiterals == 0 || LiteralValue != Value) {
         LiteralValue = Value;
         ++NumLiterals;
@@ -5221,7 +5266,7 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
     } else if (ID == ".amdhsa_fp16_overflow") {
       if (IVersion.Major < 9)
         return Error(IDRange.Start, "directive requires gfx9+", IDRange);
-      PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_FP16_OVFL, Val,
+      PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL, Val,
                        ValRange);
     } else if (ID == ".amdhsa_tg_split") {
       if (!isGFX90A())
@@ -5231,17 +5276,17 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
     } else if (ID == ".amdhsa_workgroup_processor_mode") {
       if (IVersion.Major < 10)
         return Error(IDRange.Start, "directive requires gfx10+", IDRange);
-      PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_WGP_MODE, Val,
+      PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, Val,
                        ValRange);
     } else if (ID == ".amdhsa_memory_ordered") {
       if (IVersion.Major < 10)
         return Error(IDRange.Start, "directive requires gfx10+", IDRange);
-      PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_MEM_ORDERED, Val,
+      PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, Val,
                        ValRange);
     } else if (ID == ".amdhsa_forward_progress") {
       if (IVersion.Major < 10)
         return Error(IDRange.Start, "directive requires gfx10+", IDRange);
-      PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_FWD_PROGRESS, Val,
+      PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS, Val,
                        ValRange);
     } else if (ID == ".amdhsa_shared_vgpr_count") {
       if (IVersion.Major < 10)
diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td
index 4297111930..e1bdf0e3bc 100644
--- a/llvm/lib/Target/AMDGPU/BUFInstructions.td
+++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td
@@ -1046,10 +1046,16 @@ defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics <
   "buffer_atomic_dec_x2", VReg_64, i64
 >;
 
-let SubtargetPredicate = HasGFX10_BEncoding in
-defm BUFFER_ATOMIC_CSUB : MUBUF_Pseudo_Atomics_RTN <
-  "buffer_atomic_csub", VGPR_32, i32, int_amdgcn_global_atomic_csub
->;
+let SubtargetPredicate = HasGFX10_BEncoding in {
+  defm BUFFER_ATOMIC_CSUB : MUBUF_Pseudo_Atomics_RTN <
+    "buffer_atomic_csub", VGPR_32, i32, int_amdgcn_global_atomic_csub
+  >;
+
+  let OtherPredicates = [HasAtomicCSubNoRtnInsts] in
+    defm BUFFER_ATOMIC_CSUB : MUBUF_Pseudo_Atomics_NO_RTN <
+      "buffer_atomic_csub", VGPR_32, i32
+    >;
+}
 
 let SubtargetPredicate = isGFX8GFX9 in {
 def BUFFER_STORE_LDS_DWORD : MUBUF_Pseudo_Store_Lds <"buffer_store_lds_dword">;
@@ -1585,6 +1591,9 @@ defm : SIBufferAtomicPat<"SIbuffer_atomic_xor", i64, "BUFFER_ATOMIC_XOR_X2">;
 defm : SIBufferAtomicPat<"SIbuffer_atomic_inc", i64, "BUFFER_ATOMIC_INC_X2">;
 defm : SIBufferAtomicPat<"SIbuffer_atomic_dec", i64, "BUFFER_ATOMIC_DEC_X2">;
 
+let SubtargetPredicate = HasAtomicCSubNoRtnInsts in
+defm : SIBufferAtomicPat<"SIbuffer_atomic_csub", i32, "BUFFER_ATOMIC_CSUB", ["noret"]>;
+
 let SubtargetPredicate = isGFX6GFX7GFX10Plus in {
   defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f32, "BUFFER_ATOMIC_FMIN">;
   defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f32, "BUFFER_ATOMIC_FMAX">;
@@ -2243,7 +2252,7 @@ defm BUFFER_ATOMIC_AND_X2         : MUBUF_Real_Atomic_gfx11_Renamed<0x049, "buff
 defm BUFFER_ATOMIC_CMPSWAP        : MUBUF_Real_Atomic_gfx11_Renamed<0x034, "buffer_atomic_cmpswap_b32">;
 defm BUFFER_ATOMIC_CMPSWAP_X2     : MUBUF_Real_Atomic_gfx11_Renamed<0x042, "buffer_atomic_cmpswap_b64">;
 defm BUFFER_ATOMIC_FCMPSWAP       : MUBUF_Real_Atomic_gfx11_Renamed<0x050, "buffer_atomic_cmpswap_f32">;
-defm BUFFER_ATOMIC_CSUB           : MUBUF_Real_Atomic_gfx11_Renamed_impl<0x037, 1, "buffer_atomic_csub_u32">;
+defm BUFFER_ATOMIC_CSUB           : MUBUF_Real_Atomic_gfx11_Renamed<0x037, "buffer_atomic_csub_u32">;
 def : Pre_gfx11_MUBUF_Name<"buffer_atomic_csub", "buffer_atomic_csub_u32">;
 defm BUFFER_ATOMIC_DEC            : MUBUF_Real_Atomic_gfx11_Renamed<0x040, "buffer_atomic_dec_u32">;
 defm BUFFER_ATOMIC_DEC_X2         : MUBUF_Real_Atomic_gfx11_Renamed<0x04D, "buffer_atomic_dec_u64">;
@@ -2515,7 +2524,7 @@ defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05e>;
 defm BUFFER_ATOMIC_FMIN_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05f>;
 defm BUFFER_ATOMIC_FMAX_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x060>;
 
-defm BUFFER_ATOMIC_CSUB       : MUBUF_Real_Atomics_RTN_gfx10<0x034>;
+defm BUFFER_ATOMIC_CSUB       : MUBUF_Real_Atomics_gfx10<0x034>;
 
 defm BUFFER_WBINVL1_SC        : MUBUF_Real_gfx6<0x070>;
 defm BUFFER_WBINVL1_VOL       : MUBUF_Real_gfx7<0x070>;
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index 8124fdd5dd..0c07208907 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -176,6 +176,7 @@ add_llvm_target(AMDGPUCodeGen
   CodeGenTypes
   Core
   GlobalISel
+  HipStdPar
   IPO
   MC
   MIRParser
diff --git a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
index 439762bc6c..1b301ee5f4 100644
--- a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
+++ b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
@@ -378,6 +378,15 @@ static DecodeStatus decodeOperand_AVLdSt_Any(MCInst &Inst, unsigned Imm,
   return addOperand(Inst, DAsm->decodeSrcOp(Opw, Imm | 256));
 }
 
+static DecodeStatus decodeOperand_VSrc_f64(MCInst &Inst, unsigned Imm,
+                                           uint64_t Addr,
+                                           const MCDisassembler *Decoder) {
+  assert(Imm < (1 << 9) && "9-bit encoding");
+  auto DAsm = static_cast<const AMDGPUDisassembler *>(Decoder);
+  return addOperand(
+      Inst, DAsm->decodeSrcOp(AMDGPUDisassembler::OPW64, Imm, false, 64, true));
+}
+
 static DecodeStatus
 DecodeAVLdSt_32RegisterClass(MCInst &Inst, unsigned Imm, uint64_t Addr,
                              const MCDisassembler *Decoder) {
@@ -428,18 +437,19 @@ DECODE_SDWA(VopcDst)
 
 template <typename T> static inline T eatBytes(ArrayRef<uint8_t>& Bytes) {
   assert(Bytes.size() >= sizeof(T));
-  const auto Res = support::endian::read<T, support::endianness::little>(Bytes.data());
+  const auto Res =
+      support::endian::read<T, llvm::endianness::little>(Bytes.data());
   Bytes = Bytes.slice(sizeof(T));
   return Res;
 }
 
 static inline DecoderUInt128 eat12Bytes(ArrayRef<uint8_t> &Bytes) {
   assert(Bytes.size() >= 12);
-  uint64_t Lo = support::endian::read<uint64_t, support::endianness::little>(
-      Bytes.data());
+  uint64_t Lo =
+      support::endian::read<uint64_t, llvm::endianness::little>(Bytes.data());
   Bytes = Bytes.slice(8);
-  uint64_t Hi = support::endian::read<uint32_t, support::endianness::little>(
-      Bytes.data());
+  uint64_t Hi =
+      support::endian::read<uint32_t, llvm::endianness::little>(Bytes.data());
   Bytes = Bytes.slice(4);
   return DecoderUInt128(Lo, Hi);
 }
@@ -1218,7 +1228,7 @@ AMDGPUDisassembler::decodeMandatoryLiteralConstant(unsigned Val) const {
   return MCOperand::createImm(Literal);
 }
 
-MCOperand AMDGPUDisassembler::decodeLiteralConstant() const {
+MCOperand AMDGPUDisassembler::decodeLiteralConstant(bool ExtendFP64) const {
   // For now all literal constants are supposed to be unsigned integer
   // ToDo: deal with signed/unsigned 64-bit integer constants
   // ToDo: deal with float/double constants
@@ -1228,9 +1238,11 @@ MCOperand AMDGPUDisassembler::decodeLiteralConstant() const {
                         Twine(Bytes.size()));
     }
     HasLiteral = true;
-    Literal = eatBytes<uint32_t>(Bytes);
+    Literal = Literal64 = eatBytes<uint32_t>(Bytes);
+    if (ExtendFP64)
+      Literal64 <<= 32;
   }
-  return MCOperand::createImm(Literal);
+  return MCOperand::createImm(ExtendFP64 ? Literal64 : Literal);
 }
 
 MCOperand AMDGPUDisassembler::decodeIntImmed(unsigned Imm) {
@@ -1447,7 +1459,7 @@ int AMDGPUDisassembler::getTTmpIdx(unsigned Val) const {
 
 MCOperand AMDGPUDisassembler::decodeSrcOp(const OpWidthTy Width, unsigned Val,
                                           bool MandatoryLiteral,
-                                          unsigned ImmWidth) const {
+                                          unsigned ImmWidth, bool IsFP) const {
   using namespace AMDGPU::EncValues;
 
   assert(Val < 1024); // enum10
@@ -1459,13 +1471,15 @@ MCOperand AMDGPUDisassembler::decodeSrcOp(const OpWidthTy Width, unsigned Val,
     return createRegOperand(IsAGPR ? getAgprClassId(Width)
                                    : getVgprClassId(Width), Val - VGPR_MIN);
   }
-  return decodeNonVGPRSrcOp(Width, Val & 0xFF, MandatoryLiteral, ImmWidth);
+  return decodeNonVGPRSrcOp(Width, Val & 0xFF, MandatoryLiteral, ImmWidth,
+                            IsFP);
 }
 
 MCOperand AMDGPUDisassembler::decodeNonVGPRSrcOp(const OpWidthTy Width,
                                                  unsigned Val,
                                                  bool MandatoryLiteral,
-                                                 unsigned ImmWidth) const {
+                                                 unsigned ImmWidth,
+                                                 bool IsFP) const {
   // Cases when Val{8} is 1 (vgpr, agpr or true 16 vgpr) should have been
   // decoded earlier.
   assert(Val < (1 << 8) && "9-bit Src encoding when Val{8} is 0");
@@ -1493,7 +1507,7 @@ MCOperand AMDGPUDisassembler::decodeNonVGPRSrcOp(const OpWidthTy Width,
       // Keep a sentinel value for deferred setting
       return MCOperand::createImm(LITERAL_CONST);
     else
-      return decodeLiteralConstant();
+      return decodeLiteralConstant(IsFP && ImmWidth == 64);
   }
 
   switch (Width) {
@@ -1804,16 +1818,23 @@ MCDisassembler::DecodeStatus AMDGPUDisassembler::decodeCOMPUTE_PGM_RSRC1(
   if (FourByteBuffer & COMPUTE_PGM_RSRC1_CDBG_USER)
     return MCDisassembler::Fail;
 
-  PRINT_DIRECTIVE(".amdhsa_fp16_overflow", COMPUTE_PGM_RSRC1_FP16_OVFL);
+  if (isGFX9Plus())
+    PRINT_DIRECTIVE(".amdhsa_fp16_overflow", COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL);
 
-  if (FourByteBuffer & COMPUTE_PGM_RSRC1_RESERVED0)
+  if (!isGFX9Plus())
+    if (FourByteBuffer & COMPUTE_PGM_RSRC1_GFX6_GFX8_RESERVED0)
+      return MCDisassembler::Fail;
+  if (FourByteBuffer & COMPUTE_PGM_RSRC1_RESERVED1)
     return MCDisassembler::Fail;
+  if (!isGFX10Plus())
+    if (FourByteBuffer & COMPUTE_PGM_RSRC1_GFX6_GFX9_RESERVED2)
+      return MCDisassembler::Fail;
 
   if (isGFX10Plus()) {
     PRINT_DIRECTIVE(".amdhsa_workgroup_processor_mode",
-                    COMPUTE_PGM_RSRC1_WGP_MODE);
-    PRINT_DIRECTIVE(".amdhsa_memory_ordered", COMPUTE_PGM_RSRC1_MEM_ORDERED);
-    PRINT_DIRECTIVE(".amdhsa_forward_progress", COMPUTE_PGM_RSRC1_FWD_PROGRESS);
+                    COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE);
+    PRINT_DIRECTIVE(".amdhsa_memory_ordered", COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED);
+    PRINT_DIRECTIVE(".amdhsa_forward_progress", COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS);
   }
   return MCDisassembler::Success;
 }
@@ -1894,16 +1915,29 @@ MCDisassembler::DecodeStatus AMDGPUDisassembler::decodeCOMPUTE_PGM_RSRC3(
       PRINT_PSEUDO_DIRECTIVE_COMMENT(
           "SHARED_VGPR_COUNT", COMPUTE_PGM_RSRC3_GFX10_PLUS_SHARED_VGPR_COUNT);
     }
-    PRINT_PSEUDO_DIRECTIVE_COMMENT("INST_PREF_SIZE",
-                                   COMPUTE_PGM_RSRC3_GFX10_PLUS_INST_PREF_SIZE);
-    PRINT_PSEUDO_DIRECTIVE_COMMENT("TRAP_ON_START",
-                                   COMPUTE_PGM_RSRC3_GFX10_PLUS_TRAP_ON_START);
-    PRINT_PSEUDO_DIRECTIVE_COMMENT("TRAP_ON_END",
-                                   COMPUTE_PGM_RSRC3_GFX10_PLUS_TRAP_ON_END);
-    if (FourByteBuffer & COMPUTE_PGM_RSRC3_GFX10_PLUS_RESERVED0)
+
+    if (isGFX11Plus()) {
+      PRINT_PSEUDO_DIRECTIVE_COMMENT("INST_PREF_SIZE",
+                                     COMPUTE_PGM_RSRC3_GFX11_PLUS_INST_PREF_SIZE);
+      PRINT_PSEUDO_DIRECTIVE_COMMENT("TRAP_ON_START",
+                                     COMPUTE_PGM_RSRC3_GFX11_PLUS_TRAP_ON_START);
+      PRINT_PSEUDO_DIRECTIVE_COMMENT("TRAP_ON_END",
+                                     COMPUTE_PGM_RSRC3_GFX11_PLUS_TRAP_ON_END);
+    } else {
+      if (FourByteBuffer & COMPUTE_PGM_RSRC3_GFX10_RESERVED0)
+        return MCDisassembler::Fail;
+    }
+
+    if (FourByteBuffer & COMPUTE_PGM_RSRC3_GFX10_PLUS_RESERVED1)
       return MCDisassembler::Fail;
-    PRINT_PSEUDO_DIRECTIVE_COMMENT("IMAGE_OP",
-                                   COMPUTE_PGM_RSRC3_GFX10_PLUS_TRAP_ON_START);
+
+    if (isGFX11Plus()) {
+      PRINT_PSEUDO_DIRECTIVE_COMMENT("IMAGE_OP",
+                                     COMPUTE_PGM_RSRC3_GFX11_PLUS_TRAP_ON_START);
+    } else {
+      if (FourByteBuffer & COMPUTE_PGM_RSRC3_GFX10_RESERVED2)
+        return MCDisassembler::Fail;
+    }
   } else if (FourByteBuffer) {
     return MCDisassembler::Fail;
   }
@@ -2076,7 +2110,7 @@ MCDisassembler::DecodeStatus AMDGPUDisassembler::decodeKernelDescriptor(
   if (isGFX10Plus()) {
     uint16_t KernelCodeProperties =
         support::endian::read16(&Bytes[amdhsa::KERNEL_CODE_PROPERTIES_OFFSET],
-                                support::endianness::little);
+                                llvm::endianness::little);
     EnableWavefrontSize32 =
         AMDHSA_BITS_GET(KernelCodeProperties,
                         amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32);
diff --git a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.h b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.h
index 5f3b277d57..91b73b593d 100644
--- a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.h
+++ b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.h
@@ -97,6 +97,7 @@ class AMDGPUDisassembler : public MCDisassembler {
   const unsigned TargetMaxInstBytes;
   mutable ArrayRef<uint8_t> Bytes;
   mutable uint32_t Literal;
+  mutable uint64_t Literal64;
   mutable bool HasLiteral;
   mutable std::optional<bool> EnableWavefrontSize32;
 
@@ -229,15 +230,15 @@ class AMDGPUDisassembler : public MCDisassembler {
   static MCOperand decodeFPImmed(unsigned ImmWidth, unsigned Imm);
 
   MCOperand decodeMandatoryLiteralConstant(unsigned Imm) const;
-  MCOperand decodeLiteralConstant() const;
+  MCOperand decodeLiteralConstant(bool ExtendFP64) const;
 
   MCOperand decodeSrcOp(const OpWidthTy Width, unsigned Val,
-                        bool MandatoryLiteral = false,
-                        unsigned ImmWidth = 0) const;
+                        bool MandatoryLiteral = false, unsigned ImmWidth = 0,
+                        bool IsFP = false) const;
 
   MCOperand decodeNonVGPRSrcOp(const OpWidthTy Width, unsigned Val,
                                bool MandatoryLiteral = false,
-                               unsigned ImmWidth = 0) const;
+                               unsigned ImmWidth = 0, bool IsFP = false) const;
 
   MCOperand decodeVOPDDstYOp(MCInst &Inst, unsigned Val) const;
   MCOperand decodeSpecialReg32(unsigned Val) const;
diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td
index 7d2286c560..533013a313 100644
--- a/llvm/lib/Target/AMDGPU/FLATInstructions.td
+++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td
@@ -870,9 +870,14 @@ defm GLOBAL_ATOMIC_INC_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_inc_x2",
 defm GLOBAL_ATOMIC_DEC_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_dec_x2",
                               VReg_64, i64>;
 
-let SubtargetPredicate = HasGFX10_BEncoding in
-defm GLOBAL_ATOMIC_CSUB : FLAT_Global_Atomic_Pseudo_RTN <"global_atomic_csub",
-                              VGPR_32, i32>;
+let SubtargetPredicate = HasGFX10_BEncoding in {
+  defm GLOBAL_ATOMIC_CSUB : FLAT_Global_Atomic_Pseudo_RTN <"global_atomic_csub",
+                                VGPR_32, i32>;
+
+  let OtherPredicates = [HasAtomicCSubNoRtnInsts] in
+    defm GLOBAL_ATOMIC_CSUB : FLAT_Global_Atomic_Pseudo_NO_RTN <"global_atomic_csub",
+                                  VGPR_32, i32>;
+}
 
 defm GLOBAL_LOAD_LDS_UBYTE  : FLAT_Global_Load_LDS_Pseudo <"global_load_lds_ubyte">;
 defm GLOBAL_LOAD_LDS_SBYTE  : FLAT_Global_Load_LDS_Pseudo <"global_load_lds_sbyte">;
@@ -1442,6 +1447,9 @@ defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_CMPSWAP", "AMDGPUatomic_cmp_swap_glo
 defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_XOR", "atomic_load_xor_global", i32>;
 defm : GlobalFLATAtomicPatsRtn <"GLOBAL_ATOMIC_CSUB", "int_amdgcn_global_atomic_csub", i32, i32, /* isIntr */ 1>;
 
+let OtherPredicates = [HasAtomicCSubNoRtnInsts] in
+defm : GlobalFLATAtomicPatsNoRtn <"GLOBAL_ATOMIC_CSUB", "int_amdgcn_global_atomic_csub", i32, i32, /* isIntr */ 1>;
+
 defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD_X2", "atomic_load_add_global", i64>;
 defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SUB_X2", "atomic_load_sub_global", i64>;
 defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_INC_X2", "atomic_load_uinc_wrap_global", i64>;
@@ -2102,7 +2110,7 @@ defm GLOBAL_ATOMIC_SWAP         : FLAT_Real_GlblAtomics_gfx10<0x030>;
 defm GLOBAL_ATOMIC_CMPSWAP      : FLAT_Real_GlblAtomics_gfx10<0x031>;
 defm GLOBAL_ATOMIC_ADD          : FLAT_Real_GlblAtomics_gfx10<0x032>;
 defm GLOBAL_ATOMIC_SUB          : FLAT_Real_GlblAtomics_gfx10<0x033>;
-defm GLOBAL_ATOMIC_CSUB         : FLAT_Real_GlblAtomics_RTN_gfx10<0x034>;
+defm GLOBAL_ATOMIC_CSUB         : FLAT_Real_GlblAtomics_gfx10<0x034>;
 defm GLOBAL_ATOMIC_SMIN         : FLAT_Real_GlblAtomics_gfx10<0x035>;
 defm GLOBAL_ATOMIC_UMIN         : FLAT_Real_GlblAtomics_gfx10<0x036>;
 defm GLOBAL_ATOMIC_SMAX         : FLAT_Real_GlblAtomics_gfx10<0x037>;
@@ -2333,7 +2341,7 @@ defm GLOBAL_ATOMIC_SWAP_B32     : FLAT_Real_GlblAtomics_gfx11<0x033, "GLOBAL_ATO
 defm GLOBAL_ATOMIC_CMPSWAP_B32  : FLAT_Real_GlblAtomics_gfx11<0x034, "GLOBAL_ATOMIC_CMPSWAP", "global_atomic_cmpswap_b32", true>;
 defm GLOBAL_ATOMIC_ADD_U32      : FLAT_Real_GlblAtomics_gfx11<0x035, "GLOBAL_ATOMIC_ADD", "global_atomic_add_u32", true>;
 defm GLOBAL_ATOMIC_SUB_U32      : FLAT_Real_GlblAtomics_gfx11<0x036, "GLOBAL_ATOMIC_SUB", "global_atomic_sub_u32", true>;
-defm GLOBAL_ATOMIC_CSUB_U32     : FLAT_Real_GlblAtomics_RTN_gfx11<0x037, "GLOBAL_ATOMIC_CSUB", "global_atomic_csub_u32", true>;
+defm GLOBAL_ATOMIC_CSUB_U32     : FLAT_Real_GlblAtomics_gfx11<0x037, "GLOBAL_ATOMIC_CSUB", "global_atomic_csub_u32", true>;
 defm GLOBAL_ATOMIC_MIN_I32      : FLAT_Real_GlblAtomics_gfx11<0x038, "GLOBAL_ATOMIC_SMIN", "global_atomic_min_i32", true>;
 defm GLOBAL_ATOMIC_MIN_U32      : FLAT_Real_GlblAtomics_gfx11<0x039, "GLOBAL_ATOMIC_UMIN", "global_atomic_min_u32", true>;
 defm GLOBAL_ATOMIC_MAX_I32      : FLAT_Real_GlblAtomics_gfx11<0x03a, "GLOBAL_ATOMIC_SMAX", "global_atomic_max_i32", true>;
diff --git a/llvm/lib/Target/AMDGPU/GCNCreateVOPD.cpp b/llvm/lib/Target/AMDGPU/GCNCreateVOPD.cpp
index c9e0c68495..982367f8f9 100644
--- a/llvm/lib/Target/AMDGPU/GCNCreateVOPD.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNCreateVOPD.cpp
@@ -25,7 +25,6 @@
 #include "Utils/AMDGPUBaseInfo.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
-#include "llvm/ADT/StringMap.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineInstr.h"
 #include "llvm/CodeGen/MachineOperand.h"
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
index ce538f086c..4c624ed594 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
@@ -160,6 +160,7 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
   bool HasAtomicFaddNoRtnInsts = false;
   bool HasAtomicBufferGlobalPkAddF16NoRtnInsts = false;
   bool HasAtomicBufferGlobalPkAddF16Insts = false;
+  bool HasAtomicCSubNoRtnInsts = false;
   bool HasAtomicGlobalPkAddBF16Inst = false;
   bool HasFlatAtomicFaddF32Inst = false;
   bool SupportsSRAMECC = false;
@@ -1203,6 +1204,9 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
   // \returns true if FP8/BF8 VOP1 form of conversion to F32 is unreliable.
   bool hasCvtFP8VOP1Bug() const { return true; }
 
+  // \returns true is CSUB atomics support a no-return form.
+  bool hasAtomicCSubNoRtnInsts() const { return HasAtomicCSubNoRtnInsts; }
+
   /// \returns SGPR allocation granularity supported by the subtarget.
   unsigned getSGPRAllocGranule() const {
     return AMDGPU::IsaInfo::getSGPRAllocGranule(this);
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp
index 65efb553d9..e18c04e623 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp
@@ -28,7 +28,7 @@ namespace {
 
 class AMDGPUAsmBackend : public MCAsmBackend {
 public:
-  AMDGPUAsmBackend(const Target &T) : MCAsmBackend(support::little) {}
+  AMDGPUAsmBackend(const Target &T) : MCAsmBackend(llvm::endianness::little) {}
 
   unsigned getNumFixupKinds() const override { return AMDGPU::NumTargetFixupKinds; };
 
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
index ad4c48a8d6..9459ee088d 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
@@ -426,7 +426,7 @@ void AMDGPUInstPrinter::printImmediate32(uint32_t Imm,
 
 void AMDGPUInstPrinter::printImmediate64(uint64_t Imm,
                                          const MCSubtargetInfo &STI,
-                                         raw_ostream &O) {
+                                         raw_ostream &O, bool IsFP) {
   int64_t SImm = static_cast<int64_t>(Imm);
   if (SImm >= -16 && SImm <= 64) {
     O << SImm;
@@ -454,7 +454,10 @@ void AMDGPUInstPrinter::printImmediate64(uint64_t Imm,
   else if (Imm == 0x3fc45f306dc9c882 &&
            STI.hasFeature(AMDGPU::FeatureInv2PiInlineImm))
     O << "0.15915494309189532";
-  else {
+  else if (IsFP) {
+    assert(AMDGPU::isValid32BitLiteral(Imm, true));
+    O << formatHex(static_cast<uint64_t>(Hi_32(Imm)));
+  } else {
     assert(isUInt<32>(Imm) || isInt<32>(Imm));
 
     // In rare situations, we will have a 32-bit literal in a 64-bit
@@ -605,11 +608,13 @@ void AMDGPUInstPrinter::printRegularOperand(const MCInst *MI, unsigned OpNo,
       printImmediate32(Op.getImm(), STI, O);
       break;
     case AMDGPU::OPERAND_REG_IMM_INT64:
-    case AMDGPU::OPERAND_REG_IMM_FP64:
     case AMDGPU::OPERAND_REG_INLINE_C_INT64:
+      printImmediate64(Op.getImm(), STI, O, false);
+      break;
+    case AMDGPU::OPERAND_REG_IMM_FP64:
     case AMDGPU::OPERAND_REG_INLINE_C_FP64:
     case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
-      printImmediate64(Op.getImm(), STI, O);
+      printImmediate64(Op.getImm(), STI, O, true);
       break;
     case AMDGPU::OPERAND_REG_INLINE_C_INT16:
     case AMDGPU::OPERAND_REG_INLINE_AC_INT16:
@@ -671,7 +676,7 @@ void AMDGPUInstPrinter::printRegularOperand(const MCInst *MI, unsigned OpNo,
       if (RCBits == 32)
         printImmediate32(llvm::bit_cast<uint32_t>((float)Value), STI, O);
       else if (RCBits == 64)
-        printImmediate64(llvm::bit_cast<uint64_t>(Value), STI, O);
+        printImmediate64(llvm::bit_cast<uint64_t>(Value), STI, O, true);
       else
         llvm_unreachable("Invalid register class size");
     }
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h
index 3b14faab13..dc83547a4a 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h
@@ -91,7 +91,7 @@ class AMDGPUInstPrinter : public MCInstPrinter {
   void printImmediate32(uint32_t Imm, const MCSubtargetInfo &STI,
                         raw_ostream &O);
   void printImmediate64(uint64_t Imm, const MCSubtargetInfo &STI,
-                        raw_ostream &O);
+                        raw_ostream &O, bool IsFP);
   void printOperand(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI,
                     raw_ostream &O);
   void printRegularOperand(const MCInst *MI, unsigned OpNo,
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp
index 57ccb523c7..88c1668f62 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp
@@ -411,7 +411,10 @@ void AMDGPUMCCodeEmitter::encodeInstruction(const MCInst &MI,
     } else if (!Op.isExpr()) // Exprs will be replaced with a fixup value.
       llvm_unreachable("Must be immediate or expr");
 
-    support::endian::write<uint32_t>(CB, Imm, support::endianness::little);
+    if (Desc.operands()[i].OperandType == AMDGPU::OPERAND_REG_IMM_FP64)
+      Imm = Hi_32(Imm);
+
+    support::endian::write<uint32_t>(CB, Imm, llvm::endianness::little);
 
     // Only one literal value allowed
     break;
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
index 6b8c03c162..70350b8384 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
@@ -453,7 +453,7 @@ void AMDGPUTargetAsmStreamer::EmitAmdhsaKernelDescriptor(
   if (IVersion.Major >= 9)
     PRINT_FIELD(OS, ".amdhsa_fp16_overflow", KD,
                 compute_pgm_rsrc1,
-                amdhsa::COMPUTE_PGM_RSRC1_FP16_OVFL);
+                amdhsa::COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL);
   if (AMDGPU::isGFX90A(STI))
     PRINT_FIELD(OS, ".amdhsa_tg_split", KD,
                 compute_pgm_rsrc3,
@@ -461,13 +461,13 @@ void AMDGPUTargetAsmStreamer::EmitAmdhsaKernelDescriptor(
   if (IVersion.Major >= 10) {
     PRINT_FIELD(OS, ".amdhsa_workgroup_processor_mode", KD,
                 compute_pgm_rsrc1,
-                amdhsa::COMPUTE_PGM_RSRC1_WGP_MODE);
+                amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE);
     PRINT_FIELD(OS, ".amdhsa_memory_ordered", KD,
                 compute_pgm_rsrc1,
-                amdhsa::COMPUTE_PGM_RSRC1_MEM_ORDERED);
+                amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED);
     PRINT_FIELD(OS, ".amdhsa_forward_progress", KD,
                 compute_pgm_rsrc1,
-                amdhsa::COMPUTE_PGM_RSRC1_FWD_PROGRESS);
+                amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS);
     PRINT_FIELD(OS, ".amdhsa_shared_vgpr_count", KD, compute_pgm_rsrc3,
                 amdhsa::COMPUTE_PGM_RSRC3_GFX10_PLUS_SHARED_VGPR_COUNT);
   }
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/R600MCCodeEmitter.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/R600MCCodeEmitter.cpp
index bbbfbe4faa..6c539df767 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/R600MCCodeEmitter.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/R600MCCodeEmitter.cpp
@@ -142,11 +142,11 @@ void R600MCCodeEmitter::encodeInstruction(const MCInst &MI,
 }
 
 void R600MCCodeEmitter::emit(uint32_t Value, SmallVectorImpl<char> &CB) const {
-  support::endian::write(CB, Value, support::little);
+  support::endian::write(CB, Value, llvm::endianness::little);
 }
 
 void R600MCCodeEmitter::emit(uint64_t Value, SmallVectorImpl<char> &CB) const {
-  support::endian::write(CB, Value, support::little);
+  support::endian::write(CB, Value, llvm::endianness::little);
 }
 
 unsigned R600MCCodeEmitter::getHWReg(unsigned RegNo) const {
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 9bd0f5390b..a7d8e11461 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -769,6 +769,8 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
   // extract of relevant bits.
   setOperationAction(ISD::GET_FPMODE, MVT::i32, Legal);
 
+  setOperationAction(ISD::MUL, MVT::i1, Promote);
+
   setTargetDAGCombine({ISD::ADD,
                        ISD::UADDO_CARRY,
                        ISD::SUB,
@@ -9577,8 +9579,8 @@ SDValue SITargetLowering::lowerFastUnsafeFDIV(SDValue Op,
     }
   }
 
-  // For f16 require arcp only.
-  // For f32 require afn+arcp.
+  // For f16 require afn or arcp.
+  // For f32 require afn.
   if (!AllowInaccurateRcp && (VT != MVT::f16 || !Flags.hasAllowReciprocal()))
     return SDValue();
 
@@ -12829,7 +12831,7 @@ static void placeSources(ByteProvider<SDValue> &Src0,
         return IterElt.first == *BPP.first.Src;
       };
 
-      auto Match = std::find_if(Srcs.begin(), Srcs.end(), MatchesFirst);
+      auto Match = llvm::find_if(Srcs, MatchesFirst);
       if (Match != Srcs.end()) {
         Match->second = addPermMasks(FirstMask, Match->second);
         FirstGroup = I;
@@ -12842,7 +12844,7 @@ static void placeSources(ByteProvider<SDValue> &Src0,
       auto MatchesSecond = [&BPP](std::pair<SDValue, unsigned> IterElt) {
         return IterElt.first == *BPP.second.Src;
       };
-      auto Match = std::find_if(Srcs.begin(), Srcs.end(), MatchesSecond);
+      auto Match = llvm::find_if(Srcs, MatchesSecond);
       if (Match != Srcs.end()) {
         Match->second = addPermMasks(SecondMask, Match->second);
       } else
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 51397cbb79..4ff7b462f0 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -3170,6 +3170,7 @@ bool SIInstrInfo::isFoldableCopy(const MachineInstr &MI) {
   case AMDGPU::V_MOV_B64_e64:
   case AMDGPU::S_MOV_B32:
   case AMDGPU::S_MOV_B64:
+  case AMDGPU::S_MOV_B64_IMM_PSEUDO:
   case AMDGPU::COPY:
   case AMDGPU::WWM_COPY:
   case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
@@ -3203,11 +3204,10 @@ bool SIInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
   switch (DefMI.getOpcode()) {
   default:
     return false;
+  case AMDGPU::V_MOV_B64_e32:
   case AMDGPU::S_MOV_B64:
-    // TODO: We could fold 64-bit immediates, but this get complicated
-    // when there are sub-registers.
-    return false;
-
+  case AMDGPU::V_MOV_B64_PSEUDO:
+  case AMDGPU::S_MOV_B64_IMM_PSEUDO:
   case AMDGPU::V_MOV_B32_e32:
   case AMDGPU::S_MOV_B32:
   case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
@@ -3220,19 +3220,45 @@ bool SIInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
   if (!ImmOp->isImm())
     return false;
 
+  auto getImmFor = [ImmOp](const MachineOperand &UseOp) -> int64_t {
+    int64_t Imm = ImmOp->getImm();
+    switch (UseOp.getSubReg()) {
+    default:
+      return Imm;
+    case AMDGPU::sub0:
+      return Lo_32(Imm);
+    case AMDGPU::sub1:
+      return Hi_32(Imm);
+    case AMDGPU::lo16:
+      return APInt(16, Imm).getSExtValue();
+    case AMDGPU::hi16:
+      return APInt(32, Imm).ashr(16).getSExtValue();
+    case AMDGPU::sub1_lo16:
+      return APInt(16, Hi_32(Imm)).getSExtValue();
+    case AMDGPU::sub1_hi16:
+      return APInt(32, Hi_32(Imm)).ashr(16).getSExtValue();
+    }
+  };
+
+  assert(!DefMI.getOperand(0).getSubReg() && "Expected SSA form");
+
   unsigned Opc = UseMI.getOpcode();
   if (Opc == AMDGPU::COPY) {
+    assert(!UseMI.getOperand(0).getSubReg() && "Expected SSA form");
+
     Register DstReg = UseMI.getOperand(0).getReg();
-    bool Is16Bit = getOpSize(UseMI, 0) == 2;
+    unsigned OpSize = getOpSize(UseMI, 0);
+    bool Is16Bit = OpSize == 2;
+    bool Is64Bit = OpSize == 8;
     bool isVGPRCopy = RI.isVGPR(*MRI, DstReg);
-    unsigned NewOpc = isVGPRCopy ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32;
-    APInt Imm(32, ImmOp->getImm());
-
-    if (UseMI.getOperand(1).getSubReg() == AMDGPU::hi16)
-      Imm = Imm.ashr(16);
+    unsigned NewOpc = isVGPRCopy ? Is64Bit ? AMDGPU::V_MOV_B64_PSEUDO
+                                           : AMDGPU::V_MOV_B32_e32
+                                 : Is64Bit ? AMDGPU::S_MOV_B64_IMM_PSEUDO
+                                           : AMDGPU::S_MOV_B32;
+    APInt Imm(Is64Bit ? 64 : 32, getImmFor(UseMI.getOperand(1)));
 
     if (RI.isAGPR(*MRI, DstReg)) {
-      if (!isInlineConstant(Imm))
+      if (Is64Bit || !isInlineConstant(Imm))
         return false;
       NewOpc = AMDGPU::V_ACCVGPR_WRITE_B32_e64;
     }
@@ -3317,7 +3343,7 @@ bool SIInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
       if (NewOpc == AMDGPU::V_FMAMK_F16_t16)
         return false;
 
-      const int64_t Imm = ImmOp->getImm();
+      const int64_t Imm = getImmFor(RegSrc == Src1 ? *Src0 : *Src1);
 
       // FIXME: This would be a lot easier if we could return a new instruction
       // instead of having to modify in place.
@@ -3401,8 +3427,6 @@ bool SIInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
       if (NewOpc == AMDGPU::V_FMAAK_F16_t16)
         return false;
 
-      const int64_t Imm = ImmOp->getImm();
-
       // FIXME: This would be a lot easier if we could return a new instruction
       // instead of having to modify in place.
 
@@ -3413,7 +3437,7 @@ bool SIInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
             AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
 
       // ChangingToImmediate adds Src2 back to the instruction.
-      Src2->ChangeToImmediate(Imm);
+      Src2->ChangeToImmediate(getImmFor(*Src2));
 
       // These come before src2.
       removeModOperands(UseMI);
@@ -5467,6 +5491,18 @@ bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
     return true;
   }
 
+  if (MO->isImm()) {
+    uint64_t Imm = MO->getImm();
+    bool Is64BitFPOp = OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_FP64;
+    bool Is64BitOp = Is64BitFPOp ||
+                     OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_INT64 ||
+                     OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2INT32 ||
+                     OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2FP32;
+    if (Is64BitOp && !AMDGPU::isValid32BitLiteral(Imm, Is64BitFPOp) &&
+        !AMDGPU::isInlinableLiteral64(Imm, ST.hasInv2PiInlineImm()))
+      return false;
+  }
+
   // Handle non-register types that are treated like immediates.
   assert(MO->isImm() || MO->isTargetIndex() || MO->isFI() || MO->isGlobal());
 
@@ -6056,6 +6092,17 @@ loadMBUFScalarOperandsFromVGPR(const SIInstrInfo &TII, MachineInstr &MI,
   unsigned MovExecOpc = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
   const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
 
+  // Save SCC. Waterfall Loop may overwrite SCC.
+  Register SaveSCCReg;
+  bool SCCNotDead = (MBB.computeRegisterLiveness(TRI, AMDGPU::SCC, MI, 30) !=
+                     MachineBasicBlock::LQR_Dead);
+  if (SCCNotDead) {
+    SaveSCCReg = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
+    BuildMI(MBB, Begin, DL, TII.get(AMDGPU::S_CSELECT_B32), SaveSCCReg)
+        .addImm(1)
+        .addImm(0);
+  }
+
   Register SaveExec = MRI.createVirtualRegister(BoolXExecRC);
 
   // Save the EXEC mask
@@ -6111,8 +6158,15 @@ loadMBUFScalarOperandsFromVGPR(const SIInstrInfo &TII, MachineInstr &MI,
 
   emitLoadScalarOpsFromVGPRLoop(TII, MRI, MBB, *LoopBB, *BodyBB, DL, ScalarOps);
 
-  // Restore the EXEC mask
   MachineBasicBlock::iterator First = RemainderBB->begin();
+  // Restore SCC
+  if (SCCNotDead) {
+    BuildMI(*RemainderBB, First, DL, TII.get(AMDGPU::S_CMP_LG_U32))
+        .addReg(SaveSCCReg, RegState::Kill)
+        .addImm(0);
+  }
+
+  // Restore the EXEC mask
   BuildMI(*RemainderBB, First, DL, TII.get(MovExecOpc), Exec).addReg(SaveExec);
   return BodyBB;
 }
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index 60a6964c75..b0b91d8317 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -158,36 +158,18 @@ def SIbuffer_store_format_d16 : SDNode <"AMDGPUISD::BUFFER_STORE_FORMAT_D16",
                             SDTBufferStore,
                             [SDNPMayStore, SDNPMemOperand, SDNPHasChain]>;
 
-class SDBufferAtomic<string opcode> : SDNode <opcode,
-  SDTypeProfile<1, 8,
-       [SDTCisVT<2, v4i32>, // rsrc
-       SDTCisVT<3, i32>,   // vindex(VGPR)
-       SDTCisVT<4, i32>,   // voffset(VGPR)
-       SDTCisVT<5, i32>,   // soffset(SGPR)
-       SDTCisVT<6, i32>,   // offset(imm)
-       SDTCisVT<7, i32>,   // cachepolicy(imm)
-       SDTCisVT<8, i1>]>,  // idxen(imm)
-  [SDNPMemOperand, SDNPHasChain, SDNPMayLoad, SDNPMayStore]
->;
-
-def SIbuffer_atomic_swap : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_SWAP">;
-def SIbuffer_atomic_add : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_ADD">;
-def SIbuffer_atomic_sub : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_SUB">;
-def SIbuffer_atomic_smin : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_SMIN">;
-def SIbuffer_atomic_umin : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_UMIN">;
-def SIbuffer_atomic_smax : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_SMAX">;
-def SIbuffer_atomic_umax : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_UMAX">;
-def SIbuffer_atomic_and : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_AND">;
-def SIbuffer_atomic_or : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_OR">;
-def SIbuffer_atomic_xor : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_XOR">;
-def SIbuffer_atomic_inc : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_INC">;
-def SIbuffer_atomic_dec : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_DEC">;
-def SIbuffer_atomic_csub : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_CSUB">;
-def SIbuffer_atomic_fadd : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_FADD">;
-def SIbuffer_atomic_fmin : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_FMIN">;
-def SIbuffer_atomic_fmax : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_FMAX">;
-
-multiclass SDBufferAtomicNoRet {
+multiclass SDBufferAtomic<string opcode> {
+  def "" : SDNode <opcode,
+    SDTypeProfile<1, 8,
+         [SDTCisVT<2, v4i32>, // rsrc
+         SDTCisVT<3, i32>,   // vindex(VGPR)
+         SDTCisVT<4, i32>,   // voffset(VGPR)
+         SDTCisVT<5, i32>,   // soffset(SGPR)
+         SDTCisVT<6, i32>,   // offset(imm)
+         SDTCisVT<7, i32>,   // cachepolicy(imm)
+         SDTCisVT<8, i1>]>,  // idxen(imm)
+    [SDNPMemOperand, SDNPHasChain, SDNPMayLoad, SDNPMayStore]
+  >;
   def "_noret" : PatFrag<
     (ops node:$vdata_in, node:$rsrc, node:$vindex, node:$voffset, node:$soffset,
       node:$offset, node:$cachepolicy, node:$idxen),
@@ -198,21 +180,22 @@ multiclass SDBufferAtomicNoRet {
   }
 }
 
-defm SIbuffer_atomic_swap : SDBufferAtomicNoRet;
-defm SIbuffer_atomic_add : SDBufferAtomicNoRet;
-defm SIbuffer_atomic_sub : SDBufferAtomicNoRet;
-defm SIbuffer_atomic_smin : SDBufferAtomicNoRet;
-defm SIbuffer_atomic_umin : SDBufferAtomicNoRet;
-defm SIbuffer_atomic_smax : SDBufferAtomicNoRet;
-defm SIbuffer_atomic_umax : SDBufferAtomicNoRet;
-defm SIbuffer_atomic_and : SDBufferAtomicNoRet;
-defm SIbuffer_atomic_or : SDBufferAtomicNoRet;
-defm SIbuffer_atomic_xor : SDBufferAtomicNoRet;
-defm SIbuffer_atomic_inc : SDBufferAtomicNoRet;
-defm SIbuffer_atomic_dec : SDBufferAtomicNoRet;
-defm SIbuffer_atomic_fadd : SDBufferAtomicNoRet;
-defm SIbuffer_atomic_fmin : SDBufferAtomicNoRet;
-defm SIbuffer_atomic_fmax : SDBufferAtomicNoRet;
+defm SIbuffer_atomic_swap : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_SWAP">;
+defm SIbuffer_atomic_add : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_ADD">;
+defm SIbuffer_atomic_sub : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_SUB">;
+defm SIbuffer_atomic_smin : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_SMIN">;
+defm SIbuffer_atomic_umin : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_UMIN">;
+defm SIbuffer_atomic_smax : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_SMAX">;
+defm SIbuffer_atomic_umax : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_UMAX">;
+defm SIbuffer_atomic_and : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_AND">;
+defm SIbuffer_atomic_or : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_OR">;
+defm SIbuffer_atomic_xor : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_XOR">;
+defm SIbuffer_atomic_inc : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_INC">;
+defm SIbuffer_atomic_dec : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_DEC">;
+defm SIbuffer_atomic_csub : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_CSUB">;
+defm SIbuffer_atomic_fadd : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_FADD">;
+defm SIbuffer_atomic_fmin : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_FMIN">;
+defm SIbuffer_atomic_fmax : SDBufferAtomic <"AMDGPUISD::BUFFER_ATOMIC_FMAX">;
 
 def SIbuffer_atomic_cmpswap : SDNode <"AMDGPUISD::BUFFER_ATOMIC_CMPSWAP",
   SDTypeProfile<1, 9,
@@ -1044,6 +1027,7 @@ class NamedIntOperand<ValueType Type, string Prefix, string Name = NAME,
 
 class NamedBitOperand<string Id, string Name = NAME>
     : CustomOperand<i1, 1, Name> {
+  let PredicateMethod = "isImmTy<AMDGPUOperand::"#ImmTy#">";
   let ParserMethod =
     "[this](OperandVector &Operands) -> ParseStatus { "#
     "return parseNamedBit(\""#Id#"\", Operands, AMDGPUOperand::"#ImmTy#"); }";
@@ -1055,6 +1039,7 @@ class NamedBitOperand<string Id, string Name = NAME>
 class DefaultOperand<CustomOperand Op, int Value>
   : OperandWithDefaultOps<Op.Type, (ops (Op.Type Value))>,
     CustomOperandProps<1, Op.ParserMatchClass.Name> {
+  let PredicateMethod = Op.ParserMatchClass.PredicateMethod;
   let ParserMethod = Op.ParserMatchClass.ParserMethod;
   let PrintMethod = Op.PrintMethod;
 }
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 2e47082055..567f1b812c 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -151,6 +151,7 @@ def S_MOV_B64_IMM_PSEUDO : SPseudoInstSI <(outs SReg_64:$sdst),
   let SchedRW = [WriteSALU, Write64Bit];
   let Size = 16; // Needs maximum 2 s_mov_b32 instructions 8 byte long each.
   let Uses = [];
+  let UseNamedOperandTable = 1;
 }
 
 // Pseudoinstruction for @llvm.amdgcn.wqm. It is turned into a copy after the
@@ -3572,8 +3573,8 @@ def G_AMDGPU_ATOMIC_FMIN : G_ATOMICRMW_OP;
 def G_AMDGPU_ATOMIC_FMAX : G_ATOMICRMW_OP;
 }
 
-class BufferAtomicGenericInstruction<bit NoRtn = 0> : AMDGPUGenericInstruction {
-  let OutOperandList = !if(NoRtn, (outs), (outs type0:$dst));
+class BufferAtomicGenericInstruction : AMDGPUGenericInstruction {
+  let OutOperandList = (outs type0:$dst);
   let InOperandList = (ins type0:$vdata, type1:$rsrc, type2:$vindex, type2:$voffset,
                            type2:$soffset, untyped_imm_0:$offset,
                            untyped_imm_0:$cachepolicy, untyped_imm_0:$idxen);
diff --git a/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp b/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp
index 97b3161c7f..53fc2c0686 100644
--- a/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp
@@ -546,7 +546,8 @@ SIPeepholeSDWA::matchSDWAOperand(MachineInstr &MI) {
 
     MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
     MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst);
-    if (Src1->getReg().isPhysical() || Dst->getReg().isPhysical())
+    if (!Src1->isReg() || Src1->getReg().isPhysical() ||
+        Dst->getReg().isPhysical())
       break;
 
     if (Opcode == AMDGPU::V_LSHLREV_B32_e32 ||
@@ -584,7 +585,8 @@ SIPeepholeSDWA::matchSDWAOperand(MachineInstr &MI) {
     MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
     MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst);
 
-    if (Src1->getReg().isPhysical() || Dst->getReg().isPhysical())
+    if (!Src1->isReg() || Src1->getReg().isPhysical() ||
+        Dst->getReg().isPhysical())
       break;
 
     if (Opcode == AMDGPU::V_LSHLREV_B16_e32 ||
@@ -647,7 +649,8 @@ SIPeepholeSDWA::matchSDWAOperand(MachineInstr &MI) {
     MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
     MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst);
 
-    if (Src0->getReg().isPhysical() || Dst->getReg().isPhysical())
+    if (!Src0->isReg() || Src0->getReg().isPhysical() ||
+        Dst->getReg().isPhysical())
       break;
 
     return std::make_unique<SDWASrcOperand>(
@@ -675,7 +678,8 @@ SIPeepholeSDWA::matchSDWAOperand(MachineInstr &MI) {
 
     MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst);
 
-    if (ValSrc->getReg().isPhysical() || Dst->getReg().isPhysical())
+    if (!ValSrc->isReg() || ValSrc->getReg().isPhysical() ||
+        Dst->getReg().isPhysical())
       break;
 
     return std::make_unique<SDWASrcOperand>(
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.td b/llvm/lib/Target/AMDGPU/SIRegisterInfo.td
index c3c5bfae40..ea06e85fb4 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.td
@@ -1263,7 +1263,9 @@ def VSrc_f32 : RegOrF32 <"VS_32", "OPERAND_REG_IMM">;
 def VSrc_v2b16 : RegOrV2B16 <"VS_32", "OPERAND_REG_IMM">;
 def VSrc_v2f16 : RegOrV2F16 <"VS_32", "OPERAND_REG_IMM">;
 def VSrc_b64 : RegOrB64 <"VS_64", "OPERAND_REG_IMM">;
-def VSrc_f64 : RegOrF64 <"VS_64", "OPERAND_REG_IMM">;
+def VSrc_f64 : RegOrF64 <"VS_64", "OPERAND_REG_IMM"> {
+  let DecoderMethod = "decodeOperand_VSrc_f64";
+}
 def VSrc_v2b32 : RegOrV2B32 <"VS_64", "OPERAND_REG_IMM">;
 def VSrc_v2f32 : RegOrV2F32 <"VS_64", "OPERAND_REG_IMM">;
 
diff --git a/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp b/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp
index 4159dc694c..88c75a0f86 100644
--- a/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp
+++ b/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp
@@ -104,8 +104,7 @@ bool SIShrinkInstructions::foldImmediates(MachineInstr &MI,
         bool ConstantFolded = false;
 
         if (TII->isOperandLegal(MI, Src0Idx, &MovSrc)) {
-          if (MovSrc.isImm() &&
-              (isInt<32>(MovSrc.getImm()) || isUInt<32>(MovSrc.getImm()))) {
+          if (MovSrc.isImm()) {
             Src0.ChangeToImmediate(MovSrc.getImm());
             ConstantFolded = true;
           } else if (MovSrc.isFI()) {
diff --git a/llvm/lib/Target/AMDGPU/SOPInstructions.td b/llvm/lib/Target/AMDGPU/SOPInstructions.td
index f3309049e7..2f3b0ff2f7 100644
--- a/llvm/lib/Target/AMDGPU/SOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SOPInstructions.td
@@ -15,6 +15,7 @@ class SOP_Pseudo<string opName, dag outs, dag ins, string asmOps,
 
   let isPseudo = 1;
   let isCodeGenOnly = 1;
+  let Size = 4;
 
   string Mnemonic = opName;
   string AsmOperands = asmOps;
@@ -36,7 +37,6 @@ class SOP1_Pseudo <string opName, dag outs, dag ins,
   let SALU = 1;
   let SOP1 = 1;
   let SchedRW = [WriteSALU];
-  let Size = 4;
   let UseNamedOperandTable = 1;
 
   bits<1> has_src0 = 1;
@@ -455,7 +455,6 @@ class SOP2_Pseudo<string opName, dag outs, dag ins,
   // let sdst = xxx in {
   // for multiclasses that include both real and pseudo instructions.
   // field bits<7> sdst = 0;
-  // let Size = 4; // Do we need size here?
 }
 
 class SOP2_Real<SOP_Pseudo ps, string real_name = ps.Mnemonic> :
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
index 6d0ad763d9..954f21b5ec 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -1163,10 +1163,10 @@ amdhsa::kernel_descriptor_t getDefaultAmdhsaKernelDescriptor(
                     amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32,
                     STI->getFeatureBits().test(FeatureWavefrontSize32) ? 1 : 0);
     AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
-                    amdhsa::COMPUTE_PGM_RSRC1_WGP_MODE,
+                    amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE,
                     STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1);
     AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
-                    amdhsa::COMPUTE_PGM_RSRC1_MEM_ORDERED, 1);
+                    amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, 1);
   }
   if (AMDGPU::isGFX90A(*STI)) {
     AMDHSA_BITS_SET(KD.compute_pgm_rsrc3,
@@ -2519,6 +2519,13 @@ bool isFoldableLiteralV216(int32_t Literal, bool HasInv2Pi) {
   return Lo16 == Hi16;
 }
 
+bool isValid32BitLiteral(uint64_t Val, bool IsFP64) {
+  if (IsFP64)
+    return !(Val & 0xffffffffu);
+
+  return isUInt<32>(Val) || isInt<32>(Val);
+}
+
 bool isArgPassedInSGPR(const Argument *A) {
   const Function *F = A->getParent();
 
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
index 297a69f54d..bb2964f592 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
@@ -1290,6 +1290,9 @@ bool isInlinableIntLiteralV216(int32_t Literal);
 LLVM_READNONE
 bool isFoldableLiteralV216(int32_t Literal, bool HasInv2Pi);
 
+LLVM_READNONE
+bool isValid32BitLiteral(uint64_t Val, bool IsFP64);
+
 bool isArgPassedInSGPR(const Argument *Arg);
 
 bool isArgPassedInSGPR(const CallBase *CB, unsigned ArgNo);
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.cpp
index 58a26e1a2d..278341944c 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.cpp
@@ -723,7 +723,7 @@ void AMDGPUPALMetadata::toLegacyBlob(std::string &Blob) {
   if (Registers.getMap().empty())
     return;
   raw_string_ostream OS(Blob);
-  support::endian::Writer EW(OS, support::endianness::little);
+  support::endian::Writer EW(OS, llvm::endianness::little);
   for (auto I : Registers.getMap()) {
     EW.write(uint32_t(I.first.getUInt()));
     EW.write(uint32_t(I.second.getUInt()));
diff --git a/llvm/lib/Target/AMDGPU/VOPCInstructions.td b/llvm/lib/Target/AMDGPU/VOPCInstructions.td
index 6fc3d0957d..cbea380ab2 100644
--- a/llvm/lib/Target/AMDGPU/VOPCInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOPCInstructions.td
@@ -1081,6 +1081,8 @@ multiclass FCMP_Pattern <PatFrags cond, Instruction inst, ValueType vt> {
   }
 }
 
+defm : FCMP_Pattern <COND_O, V_CMP_O_F32_e64, f32>;
+defm : FCMP_Pattern <COND_UO, V_CMP_U_F32_e64, f32>;
 defm : FCMP_Pattern <COND_OEQ, V_CMP_EQ_F32_e64, f32>;
 defm : FCMP_Pattern <COND_ONE, V_CMP_NEQ_F32_e64, f32>;
 defm : FCMP_Pattern <COND_OGT, V_CMP_GT_F32_e64, f32>;
@@ -1088,6 +1090,8 @@ defm : FCMP_Pattern <COND_OGE, V_CMP_GE_F32_e64, f32>;
 defm : FCMP_Pattern <COND_OLT, V_CMP_LT_F32_e64, f32>;
 defm : FCMP_Pattern <COND_OLE, V_CMP_LE_F32_e64, f32>;
 
+defm : FCMP_Pattern <COND_O, V_CMP_O_F64_e64, f64>;
+defm : FCMP_Pattern <COND_UO, V_CMP_U_F64_e64, f64>;
 defm : FCMP_Pattern <COND_OEQ, V_CMP_EQ_F64_e64, f64>;
 defm : FCMP_Pattern <COND_ONE, V_CMP_NEQ_F64_e64, f64>;
 defm : FCMP_Pattern <COND_OGT, V_CMP_GT_F64_e64, f64>;
@@ -1110,6 +1114,8 @@ defm : FCMP_Pattern <COND_ULT, V_CMP_NGE_F64_e64, f64>;
 defm : FCMP_Pattern <COND_ULE, V_CMP_NGT_F64_e64, f64>;
 
 let OtherPredicates = [HasTrue16BitInsts] in {
+defm : FCMP_Pattern <COND_O, V_CMP_O_F16_t16_e64, f16>;
+defm : FCMP_Pattern <COND_UO, V_CMP_U_F16_t16_e64, f16>;
 defm : FCMP_Pattern <COND_OEQ, V_CMP_EQ_F16_t16_e64, f16>;
 defm : FCMP_Pattern <COND_ONE, V_CMP_NEQ_F16_t16_e64, f16>;
 defm : FCMP_Pattern <COND_OGT, V_CMP_GT_F16_t16_e64, f16>;
@@ -1126,6 +1132,8 @@ defm : FCMP_Pattern <COND_ULE, V_CMP_NGT_F16_t16_e64, f16>;
 } // End OtherPredicates = [HasTrue16BitInsts]
 
 let OtherPredicates = [NotHasTrue16BitInsts] in {
+defm : FCMP_Pattern <COND_O, V_CMP_O_F16_e64, f16>;
+defm : FCMP_Pattern <COND_UO, V_CMP_U_F16_e64, f16>;
 defm : FCMP_Pattern <COND_OEQ, V_CMP_EQ_F16_e64, f16>;
 defm : FCMP_Pattern <COND_ONE, V_CMP_NEQ_F16_e64, f16>;
 defm : FCMP_Pattern <COND_OGT, V_CMP_GT_F16_e64, f16>;
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 8cca167e32..9fe5dd55a8 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -9082,6 +9082,8 @@ static SDValue LowerCONCAT_VECTORS_i1(SDValue Op, SelectionDAG &DAG,
     EVT Op1VT = V1.getValueType();
     EVT Op2VT = V2.getValueType();
     assert(Op1VT == Op2VT && "Operand types don't match!");
+    assert((Op1VT == MVT::v2i1 || Op1VT == MVT::v4i1 || Op1VT == MVT::v8i1) &&
+           "Unexpected i1 concat operations!");
     EVT VT = Op1VT.getDoubleNumVectorElementsVT(*DAG.getContext());
 
     SDValue NewV1 = PromoteMVEPredVector(dl, V1, Op1VT, DAG);
@@ -9093,37 +9095,44 @@ static SDValue LowerCONCAT_VECTORS_i1(SDValue Op, SelectionDAG &DAG,
         getVectorTyFromPredicateVector(VT).getScalarType().getSimpleVT();
     unsigned NumElts = 2 * Op1VT.getVectorNumElements();
 
+    EVT ConcatVT = MVT::getVectorVT(ElType, NumElts);
+    if (Op1VT == MVT::v4i1 || Op1VT == MVT::v8i1) {
+      // Use MVETRUNC to truncate the combined NewV1::NewV2 into the smaller
+      // ConcatVT.
+      SDValue ConVec =
+          DAG.getNode(ARMISD::MVETRUNC, dl, ConcatVT, NewV1, NewV2);
+      return DAG.getNode(ARMISD::VCMPZ, dl, VT, ConVec,
+                         DAG.getConstant(ARMCC::NE, dl, MVT::i32));
+    }
+
     // Extract the vector elements from Op1 and Op2 one by one and truncate them
     // to be the right size for the destination. For example, if Op1 is v4i1
     // then the promoted vector is v4i32. The result of concatenation gives a
     // v8i1, which when promoted is v8i16. That means each i32 element from Op1
     // needs truncating to i16 and inserting in the result.
-    EVT ConcatVT = MVT::getVectorVT(ElType, NumElts);
-    SDValue ConVec = DAG.getNode(ISD::UNDEF, dl, ConcatVT);
     auto ExtractInto = [&DAG, &dl](SDValue NewV, SDValue ConVec, unsigned &j) {
       EVT NewVT = NewV.getValueType();
       EVT ConcatVT = ConVec.getValueType();
+      unsigned ExtScale = 1;
+      if (NewVT == MVT::v2f64) {
+        NewV = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, NewV);
+        ExtScale = 2;
+      }
       for (unsigned i = 0, e = NewVT.getVectorNumElements(); i < e; i++, j++) {
         SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, NewV,
-                                  DAG.getIntPtrConstant(i, dl));
+                                  DAG.getIntPtrConstant(i * ExtScale, dl));
         ConVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, ConcatVT, ConVec, Elt,
                              DAG.getConstant(j, dl, MVT::i32));
       }
       return ConVec;
     };
     unsigned j = 0;
+    SDValue ConVec = DAG.getNode(ISD::UNDEF, dl, ConcatVT);
     ConVec = ExtractInto(NewV1, ConVec, j);
     ConVec = ExtractInto(NewV2, ConVec, j);
 
     // Now return the result of comparing the subvector with zero, which will
-    // generate a real predicate, i.e. v4i1, v8i1 or v16i1. For a v2i1 we
-    // convert to a v4i1 compare to fill in the two halves of the i64 as i32s.
-    if (VT == MVT::v2i1) {
-      SDValue BC = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, ConVec);
-      SDValue Cmp = DAG.getNode(ARMISD::VCMPZ, dl, MVT::v4i1, BC,
-                                DAG.getConstant(ARMCC::NE, dl, MVT::i32));
-      return DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::v2i1, Cmp);
-    }
+    // generate a real predicate, i.e. v4i1, v8i1 or v16i1.
     return DAG.getNode(ARMISD::VCMPZ, dl, VT, ConVec,
                        DAG.getConstant(ARMCC::NE, dl, MVT::i32));
   };
diff --git a/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp b/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
index 93db983b92..a679699a66 100644
--- a/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
@@ -2829,9 +2829,7 @@ ARMPreAllocLoadStoreOpt::RescheduleLoadStoreInstrs(MachineBasicBlock *MBB) {
               return Var == DbgVar;
             };
 
-            InstrVec.erase(
-                std::remove_if(InstrVec.begin(), InstrVec.end(), IsDbgVar),
-                InstrVec.end());
+            llvm::erase_if(InstrVec, IsDbgVar);
           }
           forEachDbgRegOperand(Instr,
                                [&](MachineOperand &Op) { Op.setReg(0); });
diff --git a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
index 590887b765..373d5b59bc 100644
--- a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
+++ b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
@@ -11981,6 +11981,8 @@ bool ARMAsmParser::parseDirectiveInst(SMLoc Loc, char Suffix) {
     }
 
     getTargetStreamer().emitInst(Value->getValue(), CurSuffix);
+    forwardITPosition();
+    forwardVPTPosition();
     return false;
   };
 
diff --git a/llvm/lib/Target/ARM/Disassembler/ARMDisassembler.cpp b/llvm/lib/Target/ARM/Disassembler/ARMDisassembler.cpp
index 4063a647be..604f22d711 100644
--- a/llvm/lib/Target/ARM/Disassembler/ARMDisassembler.cpp
+++ b/llvm/lib/Target/ARM/Disassembler/ARMDisassembler.cpp
@@ -135,9 +135,9 @@ class ARMDisassembler : public MCDisassembler {
   ARMDisassembler(const MCSubtargetInfo &STI, MCContext &Ctx,
                   const MCInstrInfo *MCII)
       : MCDisassembler(STI, Ctx), MCII(MCII) {
-    InstructionEndianness = STI.hasFeature(ARM::ModeBigEndianInstructions)
-                                ? llvm::support::big
-                                : llvm::support::little;
+        InstructionEndianness = STI.hasFeature(ARM::ModeBigEndianInstructions)
+                                    ? llvm::endianness::big
+                                    : llvm::endianness::little;
   }
 
   ~ARMDisassembler() override = default;
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
index 7016918046..9230ff7bae 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
@@ -196,8 +196,9 @@ const MCFixupKindInfo &ARMAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
 
   assert(unsigned(Kind - FirstTargetFixupKind) < getNumFixupKinds() &&
          "Invalid kind!");
-  return (Endian == support::little ? InfosLE
-                                    : InfosBE)[Kind - FirstTargetFixupKind];
+  return (Endian == llvm::endianness::little
+              ? InfosLE
+              : InfosBE)[Kind - FirstTargetFixupKind];
 }
 
 void ARMAsmBackend::handleAssemblerFlag(MCAssemblerFlag Flag) {
@@ -493,7 +494,7 @@ unsigned ARMAsmBackend::adjustFixupValue(const MCAssembler &Asm,
     // inst{14-12} = Mid3;
     // inst{7-0} = Lo8;
     Value = (Hi4 << 16) | (i << 26) | (Mid3 << 12) | (Lo8);
-    return swapHalfWords(Value, Endian == support::little);
+    return swapHalfWords(Value, Endian == llvm::endianness::little);
   }
   case ARM::fixup_arm_thumb_upper_8_15:
     if (IsResolved || !STI->getTargetTriple().isOSBinFormatELF())
@@ -532,7 +533,7 @@ unsigned ARMAsmBackend::adjustFixupValue(const MCAssembler &Asm,
     // Same addressing mode as fixup_arm_pcrel_10,
     // but with 16-bit halfwords swapped.
     if (Kind == ARM::fixup_t2_ldst_pcrel_12)
-      return swapHalfWords(Value, Endian == support::little);
+      return swapHalfWords(Value, Endian == llvm::endianness::little);
 
     return Value;
   }
@@ -565,7 +566,7 @@ unsigned ARMAsmBackend::adjustFixupValue(const MCAssembler &Asm,
     out |= (Value & 0x700) << 4;
     out |= (Value & 0x0FF);
 
-    return swapHalfWords(out, Endian == support::little);
+    return swapHalfWords(out, Endian == llvm::endianness::little);
   }
 
   case ARM::fixup_arm_condbranch:
@@ -602,7 +603,7 @@ unsigned ARMAsmBackend::adjustFixupValue(const MCAssembler &Asm,
     out |= (Value & 0x1FF800) << 5; // imm6 field
     out |= (Value & 0x0007FF);      // imm11 field
 
-    return swapHalfWords(out, Endian == support::little);
+    return swapHalfWords(out, Endian == llvm::endianness::little);
   }
   case ARM::fixup_t2_condbranch: {
     Value = Value - 4;
@@ -620,7 +621,7 @@ unsigned ARMAsmBackend::adjustFixupValue(const MCAssembler &Asm,
     out |= (Value & 0x1F800) << 5; // imm6 field
     out |= (Value & 0x007FF);      // imm11 field
 
-    return swapHalfWords(out, Endian == support::little);
+    return swapHalfWords(out, Endian == llvm::endianness::little);
   }
   case ARM::fixup_arm_thumb_bl: {
     if (!isInt<25>(Value - 4) ||
@@ -656,7 +657,8 @@ unsigned ARMAsmBackend::adjustFixupValue(const MCAssembler &Asm,
     uint32_t FirstHalf = (((uint16_t)signBit << 10) | (uint16_t)imm10Bits);
     uint32_t SecondHalf = (((uint16_t)J1Bit << 13) | ((uint16_t)J2Bit << 11) |
                            (uint16_t)imm11Bits);
-    return joinHalfWords(FirstHalf, SecondHalf, Endian == support::little);
+    return joinHalfWords(FirstHalf, SecondHalf,
+                         Endian == llvm::endianness::little);
   }
   case ARM::fixup_arm_thumb_blx: {
     // The value doesn't encode the low two bits (always zero) and is offset by
@@ -692,7 +694,8 @@ unsigned ARMAsmBackend::adjustFixupValue(const MCAssembler &Asm,
     uint32_t FirstHalf = (((uint16_t)signBit << 10) | (uint16_t)imm10HBits);
     uint32_t SecondHalf = (((uint16_t)J1Bit << 13) | ((uint16_t)J2Bit << 11) |
                            ((uint16_t)imm10LBits) << 1);
-    return joinHalfWords(FirstHalf, SecondHalf, Endian == support::little);
+    return joinHalfWords(FirstHalf, SecondHalf,
+                         Endian == llvm::endianness::little);
   }
   case ARM::fixup_thumb_adr_pcrel_10:
   case ARM::fixup_arm_thumb_cp:
@@ -783,7 +786,7 @@ unsigned ARMAsmBackend::adjustFixupValue(const MCAssembler &Asm,
     // Same addressing mode as fixup_arm_pcrel_10, but with 16-bit halfwords
     // swapped.
     if (Kind == ARM::fixup_t2_pcrel_10)
-      return swapHalfWords(Value, Endian == support::little);
+      return swapHalfWords(Value, Endian == llvm::endianness::little);
 
     return Value;
   }
@@ -814,7 +817,7 @@ unsigned ARMAsmBackend::adjustFixupValue(const MCAssembler &Asm,
     // Same addressing mode as fixup_arm_pcrel_9, but with 16-bit halfwords
     // swapped.
     if (Kind == ARM::fixup_t2_pcrel_9)
-      return swapHalfWords(Value, Endian == support::little);
+      return swapHalfWords(Value, Endian == llvm::endianness::little);
 
     return Value;
   }
@@ -840,7 +843,7 @@ unsigned ARMAsmBackend::adjustFixupValue(const MCAssembler &Asm,
     EncValue |= (Value & 0x800) << 15;
     EncValue |= (Value & 0x700) << 4;
     EncValue |= (Value & 0xff);
-    return swapHalfWords(EncValue, Endian == support::little);
+    return swapHalfWords(EncValue, Endian == llvm::endianness::little);
   }
   case ARM::fixup_bf_branch: {
     const char *FixupDiagnostic = reasonForFixupRelaxation(Fixup, Value);
@@ -849,7 +852,7 @@ unsigned ARMAsmBackend::adjustFixupValue(const MCAssembler &Asm,
       return 0;
     }
     uint32_t out = (((Value - 4) >> 1) & 0xf) << 23;
-    return swapHalfWords(out, Endian == support::little);
+    return swapHalfWords(out, Endian == llvm::endianness::little);
   }
   case ARM::fixup_bf_target:
   case ARM::fixup_bfl_target:
@@ -865,7 +868,7 @@ unsigned ARMAsmBackend::adjustFixupValue(const MCAssembler &Asm,
     out |= (((Value - 4) >> 1) & 0x1) << 11;
     out |= (((Value - 4) >> 1) & 0x7fe);
     out |= (((Value - 4) >> 1) & HighBitMask) << 5;
-    return swapHalfWords(out, Endian == support::little);
+    return swapHalfWords(out, Endian == llvm::endianness::little);
   }
   case ARM::fixup_bfcsel_else_target: {
     // If this is a fixup of a branch future's else target then it should be a
@@ -879,7 +882,7 @@ unsigned ARMAsmBackend::adjustFixupValue(const MCAssembler &Asm,
       return 0;
     }
     uint32_t out = ((Value >> 2) & 1) << 17;
-    return swapHalfWords(out, Endian == support::little);
+    return swapHalfWords(out, Endian == llvm::endianness::little);
   }
   case ARM::fixup_wls:
   case ARM::fixup_le: {
@@ -894,7 +897,7 @@ unsigned ARMAsmBackend::adjustFixupValue(const MCAssembler &Asm,
       real_value = -real_value;
     out |= ((real_value >> 1) & 0x1) << 11;
     out |= ((real_value >> 1) & 0x7fe);
-    return swapHalfWords(out, Endian == support::little);
+    return swapHalfWords(out, Endian == llvm::endianness::little);
   }
   }
 }
@@ -1089,7 +1092,7 @@ void ARMAsmBackend::applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
 
   // Used to point to big endian bytes.
   unsigned FullSizeBytes;
-  if (Endian == support::big) {
+  if (Endian == llvm::endianness::big) {
     FullSizeBytes = getFixupKindContainerSizeBytes(Kind);
     assert((Offset + FullSizeBytes) <= Data.size() && "Invalid fixup size!");
     assert(NumBytes <= FullSizeBytes && "Invalid fixup size!");
@@ -1099,7 +1102,8 @@ void ARMAsmBackend::applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
   // the fixup value. The Value has been "split up" into the appropriate
   // bitfields above.
   for (unsigned i = 0; i != NumBytes; ++i) {
-    unsigned Idx = Endian == support::little ? i : (FullSizeBytes - 1 - i);
+    unsigned Idx =
+        Endian == llvm::endianness::little ? i : (FullSizeBytes - 1 - i);
     Data[Offset + Idx] |= uint8_t((Value >> (i * 8)) & 0xff);
   }
 }
@@ -1326,7 +1330,7 @@ static MCAsmBackend *createARMAsmBackend(const Target &T,
                                          const MCSubtargetInfo &STI,
                                          const MCRegisterInfo &MRI,
                                          const MCTargetOptions &Options,
-                                         support::endianness Endian) {
+                                         llvm::endianness Endian) {
   const Triple &TheTriple = STI.getTargetTriple();
   switch (TheTriple.getObjectFormat()) {
   default:
@@ -1348,12 +1352,12 @@ MCAsmBackend *llvm::createARMLEAsmBackend(const Target &T,
                                           const MCSubtargetInfo &STI,
                                           const MCRegisterInfo &MRI,
                                           const MCTargetOptions &Options) {
-  return createARMAsmBackend(T, STI, MRI, Options, support::little);
+  return createARMAsmBackend(T, STI, MRI, Options, llvm::endianness::little);
 }
 
 MCAsmBackend *llvm::createARMBEAsmBackend(const Target &T,
                                           const MCSubtargetInfo &STI,
                                           const MCRegisterInfo &MRI,
                                           const MCTargetOptions &Options) {
-  return createARMAsmBackend(T, STI, MRI, Options, support::big);
+  return createARMAsmBackend(T, STI, MRI, Options, llvm::endianness::big);
 }
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h
index 64c78d3528..40d111b797 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h
@@ -20,7 +20,7 @@ namespace llvm {
 class ARMAsmBackend : public MCAsmBackend {
   bool isThumbMode;    // Currently emitting Thumb code.
 public:
-  ARMAsmBackend(const Target &T, bool isThumb, support::endianness Endian)
+  ARMAsmBackend(const Target &T, bool isThumb, llvm::endianness Endian)
       : MCAsmBackend(Endian), isThumbMode(isThumb) {}
 
   unsigned getNumFixupKinds() const override {
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackendDarwin.h b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackendDarwin.h
index ace573c8fa..ac0c9b101c 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackendDarwin.h
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackendDarwin.h
@@ -22,7 +22,8 @@ class ARMAsmBackendDarwin : public ARMAsmBackend {
   const MachO::CPUSubTypeARM Subtype;
   ARMAsmBackendDarwin(const Target &T, const MCSubtargetInfo &STI,
                       const MCRegisterInfo &MRI)
-      : ARMAsmBackend(T, STI.getTargetTriple().isThumb(), support::little),
+      : ARMAsmBackend(T, STI.getTargetTriple().isThumb(),
+                      llvm::endianness::little),
         MRI(MRI), TT(STI.getTargetTriple()),
         Subtype((MachO::CPUSubTypeARM)cantFail(
             MachO::getCPUSubType(STI.getTargetTriple()))) {}
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackendELF.h b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackendELF.h
index 37afdf439e..abbe73e336 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackendELF.h
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackendELF.h
@@ -20,7 +20,7 @@ class ARMAsmBackendELF : public ARMAsmBackend {
 public:
   uint8_t OSABI;
   ARMAsmBackendELF(const Target &T, bool isThumb, uint8_t OSABI,
-                   support::endianness Endian)
+                   llvm::endianness Endian)
       : ARMAsmBackend(T, isThumb, Endian), OSABI(OSABI) {}
 
   std::unique_ptr<MCObjectTargetWriter>
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackendWinCOFF.h b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackendWinCOFF.h
index 6e447df9e4..86ce6efe66 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackendWinCOFF.h
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackendWinCOFF.h
@@ -17,7 +17,7 @@ namespace {
 class ARMAsmBackendWinCOFF : public ARMAsmBackend {
 public:
   ARMAsmBackendWinCOFF(const Target &T, bool isThumb)
-      : ARMAsmBackend(T, isThumb, support::little) {}
+      : ARMAsmBackend(T, isThumb, llvm::endianness::little) {}
   std::unique_ptr<MCObjectTargetWriter>
   createObjectTargetWriter() const override {
     return createARMWinCOFFObjectWriter();
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMMCCodeEmitter.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMMCCodeEmitter.cpp
index 616dd6dba7..3f37acff29 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMMCCodeEmitter.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMMCCodeEmitter.cpp
@@ -1899,7 +1899,8 @@ void ARMMCCodeEmitter::encodeInstruction(const MCInst &MI,
   else
     llvm_unreachable("Unexpected instruction size!");
 
-  auto Endian = IsLittleEndian ? support::little : support::big;
+  auto Endian =
+      IsLittleEndian ? llvm::endianness::little : llvm::endianness::big;
   uint32_t Binary = getBinaryCodeForInstr(MI, Fixups, STI);
   if (Size == 2) {
     support::endian::write<uint16_t>(CB, Binary, Endian);
diff --git a/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h b/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h
index d6a30e4dfa..3081fe1fd5 100644
--- a/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h
+++ b/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h
@@ -29,7 +29,7 @@ struct MCFixupKindInfo;
 class AVRAsmBackend : public MCAsmBackend {
 public:
   AVRAsmBackend(Triple::OSType OSType)
-      : MCAsmBackend(support::little), OSType(OSType) {}
+      : MCAsmBackend(llvm::endianness::little), OSType(OSType) {}
 
   void adjustFixupValue(const MCFixup &Fixup, const MCValue &Target,
                         uint64_t &Value, MCContext *Ctx = nullptr) const;
diff --git a/llvm/lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.cpp b/llvm/lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.cpp
index c08e293d04..7682394e83 100644
--- a/llvm/lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.cpp
+++ b/llvm/lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.cpp
@@ -285,7 +285,7 @@ void AVRMCCodeEmitter::encodeInstruction(const MCInst &MI,
 
   for (int64_t i = Size / 2 - 1; i >= 0; --i) {
     uint16_t Word = (BinaryOpCode >> (i * 16)) & 0xFFFF;
-    support::endian::write(CB, Word, support::endianness::little);
+    support::endian::write(CB, Word, llvm::endianness::little);
   }
 }
 
diff --git a/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp b/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp
index f2445d3695..fccc4ee9f7 100644
--- a/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp
+++ b/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp
@@ -25,7 +25,7 @@ namespace {
 
 class BPFAsmBackend : public MCAsmBackend {
 public:
-  BPFAsmBackend(support::endianness Endian) : MCAsmBackend(Endian) {}
+  BPFAsmBackend(llvm::endianness Endian) : MCAsmBackend(Endian) {}
   ~BPFAsmBackend() override = default;
 
   void applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
@@ -97,7 +97,7 @@ void BPFAsmBackend::applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
     support::endian::write<uint64_t>(&Data[Fixup.getOffset()], Value, Endian);
   } else if (Fixup.getKind() == FK_PCRel_4) {
     Value = (uint32_t)((Value - 8) / 8);
-    if (Endian == support::little) {
+    if (Endian == llvm::endianness::little) {
       Data[Fixup.getOffset() + 1] = 0x10;
       support::endian::write32le(&Data[Fixup.getOffset() + 4], Value);
     } else {
@@ -131,12 +131,12 @@ MCAsmBackend *llvm::createBPFAsmBackend(const Target &T,
                                         const MCSubtargetInfo &STI,
                                         const MCRegisterInfo &MRI,
                                         const MCTargetOptions &) {
-  return new BPFAsmBackend(support::little);
+  return new BPFAsmBackend(llvm::endianness::little);
 }
 
 MCAsmBackend *llvm::createBPFbeAsmBackend(const Target &T,
                                           const MCSubtargetInfo &STI,
                                           const MCRegisterInfo &MRI,
                                           const MCTargetOptions &) {
-  return new BPFAsmBackend(support::big);
+  return new BPFAsmBackend(llvm::endianness::big);
 }
diff --git a/llvm/lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp b/llvm/lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp
index 420a2aad48..b807d69040 100644
--- a/llvm/lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp
+++ b/llvm/lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp
@@ -116,8 +116,8 @@ void BPFMCCodeEmitter::encodeInstruction(const MCInst &MI,
                                          const MCSubtargetInfo &STI) const {
   unsigned Opcode = MI.getOpcode();
   raw_svector_ostream OS(CB);
-  support::endian::Writer OSE(OS,
-                              IsLittleEndian ? support::little : support::big);
+  support::endian::Writer OSE(OS, IsLittleEndian ? llvm::endianness::little
+                                                 : llvm::endianness::big);
 
   if (Opcode == BPF::LD_imm64 || Opcode == BPF::LD_pseudo) {
     uint64_t Value = getBinaryCodeForInstr(MI, Fixups, STI);
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.cpp b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.cpp
index d53d2e9e00..76f5a5fc83 100644
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.cpp
+++ b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.cpp
@@ -223,7 +223,7 @@ void CSKYAsmBackend::applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
 
   // For each byte of the fragment that the fixup touches, mask in the
   // bits from the fixup value.
-  bool IsLittleEndian = (Endian == support::little);
+  bool IsLittleEndian = (Endian == llvm::endianness::little);
   bool IsInstFixup = (Kind >= FirstTargetFixupKind);
 
   if (IsLittleEndian && IsInstFixup && (NumBytes == 4)) {
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.h b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.h
index 09b3ce6cc8..8a2f743bde 100644
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.h
+++ b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.h
@@ -20,7 +20,7 @@ class CSKYAsmBackend : public MCAsmBackend {
 
 public:
   CSKYAsmBackend(const MCSubtargetInfo &STI, const MCTargetOptions &OP)
-      : MCAsmBackend(support::little) {}
+      : MCAsmBackend(llvm::endianness::little) {}
 
   unsigned int getNumFixupKinds() const override {
     return CSKY::NumTargetFixupKinds;
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCCodeEmitter.cpp b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCCodeEmitter.cpp
index 4437c5c998..fc5ddde07f 100644
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCCodeEmitter.cpp
+++ b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCCodeEmitter.cpp
@@ -60,8 +60,9 @@ CSKYMCCodeEmitter::getImmOpValueMSBSize(const MCInst &MI, unsigned Idx,
 static void writeData(uint32_t Bin, unsigned Size, SmallVectorImpl<char> &CB) {
   if (Size == 4)
     support::endian::write(CB, static_cast<uint16_t>(Bin >> 16),
-                           support::little);
-  support::endian::write(CB, static_cast<uint16_t>(Bin), support::little);
+                           llvm::endianness::little);
+  support::endian::write(CB, static_cast<uint16_t>(Bin),
+                         llvm::endianness::little);
 }
 
 void CSKYMCCodeEmitter::expandJBTF(const MCInst &MI, SmallVectorImpl<char> &CB,
diff --git a/llvm/lib/Target/DirectX/MCTargetDesc/DirectXMCTargetDesc.cpp b/llvm/lib/Target/DirectX/MCTargetDesc/DirectXMCTargetDesc.cpp
index 77007d853d..4a73cbbea3 100644
--- a/llvm/lib/Target/DirectX/MCTargetDesc/DirectXMCTargetDesc.cpp
+++ b/llvm/lib/Target/DirectX/MCTargetDesc/DirectXMCTargetDesc.cpp
@@ -72,7 +72,8 @@ class DXILMCCodeEmitter : public MCCodeEmitter {
 class DXILAsmBackend : public MCAsmBackend {
 
 public:
-  DXILAsmBackend(const MCSubtargetInfo &STI) : MCAsmBackend(support::little) {}
+  DXILAsmBackend(const MCSubtargetInfo &STI)
+      : MCAsmBackend(llvm::endianness::little) {}
   ~DXILAsmBackend() override = default;
 
   void applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp
index 12c84ceb5f..76b4dc4e5a 100644
--- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp
+++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp
@@ -62,10 +62,9 @@ class HexagonAsmBackend : public MCAsmBackend {
 public:
   HexagonAsmBackend(const Target &T, const Triple &TT, uint8_t OSABI,
                     StringRef CPU)
-      : MCAsmBackend(support::little), OSABI(OSABI), CPU(CPU), relaxedCnt(0),
-        MCII(T.createMCInstrInfo()), RelaxTarget(new MCInst *),
-        Extender(nullptr), MaxPacketSize(HexagonMCInstrInfo::packetSize(CPU))
-        {}
+      : MCAsmBackend(llvm::endianness::little), OSABI(OSABI), CPU(CPU),
+        relaxedCnt(0), MCII(T.createMCInstrInfo()), RelaxTarget(new MCInst *),
+        Extender(nullptr), MaxPacketSize(HexagonMCInstrInfo::packetSize(CPU)) {}
 
   std::unique_ptr<MCObjectTargetWriter>
   createObjectTargetWriter() const override {
diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCCodeEmitter.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCCodeEmitter.cpp
index 8bf4d0a412..96ec81cd86 100644
--- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCCodeEmitter.cpp
+++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCCodeEmitter.cpp
@@ -443,7 +443,7 @@ void HexagonMCCodeEmitter::encodeSingleInstruction(
 
     Binary |= SubBits0 | (SubBits1 << 16);
   }
-  support::endian::write<uint32_t>(CB, Binary, support::little);
+  support::endian::write<uint32_t>(CB, Binary, llvm::endianness::little);
   ++MCNumEmitted;
 }
 
diff --git a/llvm/lib/Target/Lanai/MCTargetDesc/LanaiAsmBackend.cpp b/llvm/lib/Target/Lanai/MCTargetDesc/LanaiAsmBackend.cpp
index 3c2a3ac692..08ca577a47 100644
--- a/llvm/lib/Target/Lanai/MCTargetDesc/LanaiAsmBackend.cpp
+++ b/llvm/lib/Target/Lanai/MCTargetDesc/LanaiAsmBackend.cpp
@@ -46,7 +46,7 @@ class LanaiAsmBackend : public MCAsmBackend {
 
 public:
   LanaiAsmBackend(const Target &T, Triple::OSType OST)
-      : MCAsmBackend(support::big), OSType(OST) {}
+      : MCAsmBackend(llvm::endianness::big), OSType(OST) {}
 
   void applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
                   const MCValue &Target, MutableArrayRef<char> Data,
diff --git a/llvm/lib/Target/Lanai/MCTargetDesc/LanaiMCCodeEmitter.cpp b/llvm/lib/Target/Lanai/MCTargetDesc/LanaiMCCodeEmitter.cpp
index 5f9c2a1002..d09966e369 100644
--- a/llvm/lib/Target/Lanai/MCTargetDesc/LanaiMCCodeEmitter.cpp
+++ b/llvm/lib/Target/Lanai/MCTargetDesc/LanaiMCCodeEmitter.cpp
@@ -178,7 +178,7 @@ void LanaiMCCodeEmitter::encodeInstruction(
   unsigned Value = getBinaryCodeForInstr(Inst, Fixups, SubtargetInfo);
   ++MCNumEmitted; // Keep track of the number of emitted insns.
 
-  support::endian::write<uint32_t>(CB, Value, support::big);
+  support::endian::write<uint32_t>(CB, Value, llvm::endianness::big);
 }
 
 // Encode Lanai Memory Operand
diff --git a/llvm/lib/Target/LoongArch/LoongArch.h b/llvm/lib/Target/LoongArch/LoongArch.h
index 05f4ac8c92..09ca089c91 100644
--- a/llvm/lib/Target/LoongArch/LoongArch.h
+++ b/llvm/lib/Target/LoongArch/LoongArch.h
@@ -36,9 +36,11 @@ bool lowerLoongArchMachineOperandToMCOperand(const MachineOperand &MO,
 FunctionPass *createLoongArchExpandAtomicPseudoPass();
 FunctionPass *createLoongArchISelDag(LoongArchTargetMachine &TM);
 FunctionPass *createLoongArchPreRAExpandPseudoPass();
+FunctionPass *createLoongArchExpandPseudoPass();
 void initializeLoongArchDAGToDAGISelPass(PassRegistry &);
 void initializeLoongArchExpandAtomicPseudoPass(PassRegistry &);
 void initializeLoongArchPreRAExpandPseudoPass(PassRegistry &);
+void initializeLoongArchExpandPseudoPass(PassRegistry &);
 } // end namespace llvm
 
 #endif // LLVM_LIB_TARGET_LOONGARCH_LOONGARCH_H
diff --git a/llvm/lib/Target/LoongArch/LoongArchExpandAtomicPseudoInsts.cpp b/llvm/lib/Target/LoongArch/LoongArchExpandAtomicPseudoInsts.cpp
index 51df0463e2..18a532b55e 100644
--- a/llvm/lib/Target/LoongArch/LoongArchExpandAtomicPseudoInsts.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchExpandAtomicPseudoInsts.cpp
@@ -153,18 +153,12 @@ static void doAtomicBinOpExpansion(const LoongArchInstrInfo *TII,
   Register ScratchReg = MI.getOperand(1).getReg();
   Register AddrReg = MI.getOperand(2).getReg();
   Register IncrReg = MI.getOperand(3).getReg();
-  AtomicOrdering Ordering =
-      static_cast<AtomicOrdering>(MI.getOperand(4).getImm());
 
   // .loop:
-  //   if(Ordering != AtomicOrdering::Monotonic)
-  //     dbar 0
   //   ll.[w|d] dest, (addr)
   //   binop scratch, dest, val
   //   sc.[w|d] scratch, scratch, (addr)
   //   beqz scratch, loop
-  if (Ordering != AtomicOrdering::Monotonic)
-    BuildMI(LoopMBB, DL, TII->get(LoongArch::DBAR)).addImm(0);
   BuildMI(LoopMBB, DL,
           TII->get(Width == 32 ? LoongArch::LL_W : LoongArch::LL_D), DestReg)
       .addReg(AddrReg)
@@ -251,12 +245,8 @@ static void doMaskedAtomicBinOpExpansion(
   Register AddrReg = MI.getOperand(2).getReg();
   Register IncrReg = MI.getOperand(3).getReg();
   Register MaskReg = MI.getOperand(4).getReg();
-  AtomicOrdering Ordering =
-      static_cast<AtomicOrdering>(MI.getOperand(5).getImm());
 
   // .loop:
-  //   if(Ordering != AtomicOrdering::Monotonic)
-  //     dbar 0
   //   ll.w destreg, (alignedaddr)
   //   binop scratch, destreg, incr
   //   xor scratch, destreg, scratch
@@ -264,8 +254,6 @@ static void doMaskedAtomicBinOpExpansion(
   //   xor scratch, destreg, scratch
   //   sc.w scratch, scratch, (alignedaddr)
   //   beqz scratch, loop
-  if (Ordering != AtomicOrdering::Monotonic)
-    BuildMI(LoopMBB, DL, TII->get(LoongArch::DBAR)).addImm(0);
   BuildMI(LoopMBB, DL, TII->get(LoongArch::LL_W), DestReg)
       .addReg(AddrReg)
       .addImm(0);
@@ -372,23 +360,20 @@ bool LoongArchExpandAtomicPseudo::expandAtomicMinMaxOp(
   auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
   auto LoopIfBodyMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
   auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
-  auto TailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
   auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
 
   // Insert new MBBs.
   MF->insert(++MBB.getIterator(), LoopHeadMBB);
   MF->insert(++LoopHeadMBB->getIterator(), LoopIfBodyMBB);
   MF->insert(++LoopIfBodyMBB->getIterator(), LoopTailMBB);
-  MF->insert(++LoopTailMBB->getIterator(), TailMBB);
-  MF->insert(++TailMBB->getIterator(), DoneMBB);
+  MF->insert(++LoopTailMBB->getIterator(), DoneMBB);
 
   // Set up successors and transfer remaining instructions to DoneMBB.
   LoopHeadMBB->addSuccessor(LoopIfBodyMBB);
   LoopHeadMBB->addSuccessor(LoopTailMBB);
   LoopIfBodyMBB->addSuccessor(LoopTailMBB);
   LoopTailMBB->addSuccessor(LoopHeadMBB);
-  LoopTailMBB->addSuccessor(TailMBB);
-  TailMBB->addSuccessor(DoneMBB);
+  LoopTailMBB->addSuccessor(DoneMBB);
   DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
   DoneMBB->transferSuccessors(&MBB);
   MBB.addSuccessor(LoopHeadMBB);
@@ -402,11 +387,9 @@ bool LoongArchExpandAtomicPseudo::expandAtomicMinMaxOp(
 
   //
   // .loophead:
-  //   dbar 0
   //   ll.w destreg, (alignedaddr)
   //   and scratch2, destreg, mask
   //   move scratch1, destreg
-  BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::DBAR)).addImm(0);
   BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::LL_W), DestReg)
       .addReg(AddrReg)
       .addImm(0);
@@ -463,7 +446,6 @@ bool LoongArchExpandAtomicPseudo::expandAtomicMinMaxOp(
   // .looptail:
   //   sc.w scratch1, scratch1, (addr)
   //   beqz scratch1, loop
-  //   dbar 0x700
   BuildMI(LoopTailMBB, DL, TII->get(LoongArch::SC_W), Scratch1Reg)
       .addReg(Scratch1Reg)
       .addReg(AddrReg)
@@ -472,10 +454,6 @@ bool LoongArchExpandAtomicPseudo::expandAtomicMinMaxOp(
       .addReg(Scratch1Reg)
       .addMBB(LoopHeadMBB);
 
-  // .tail:
-  //   dbar 0x700
-  BuildMI(TailMBB, DL, TII->get(LoongArch::DBAR)).addImm(0x700);
-
   NextMBBI = MBB.end();
   MI.eraseFromParent();
 
@@ -483,7 +461,6 @@ bool LoongArchExpandAtomicPseudo::expandAtomicMinMaxOp(
   computeAndAddLiveIns(LiveRegs, *LoopHeadMBB);
   computeAndAddLiveIns(LiveRegs, *LoopIfBodyMBB);
   computeAndAddLiveIns(LiveRegs, *LoopTailMBB);
-  computeAndAddLiveIns(LiveRegs, *TailMBB);
   computeAndAddLiveIns(LiveRegs, *DoneMBB);
 
   return true;
@@ -535,12 +512,10 @@ bool LoongArchExpandAtomicPseudo::expandAtomicCmpXchg(
         .addReg(CmpValReg)
         .addMBB(TailMBB);
     // .looptail:
-    //   dbar 0
     //   move scratch, newval
     //   sc.[w|d] scratch, scratch, (addr)
     //   beqz scratch, loophead
     //   b done
-    BuildMI(LoopTailMBB, DL, TII->get(LoongArch::DBAR)).addImm(0);
     BuildMI(LoopTailMBB, DL, TII->get(LoongArch::OR), ScratchReg)
         .addReg(NewValReg)
         .addReg(LoongArch::R0);
@@ -573,13 +548,11 @@ bool LoongArchExpandAtomicPseudo::expandAtomicCmpXchg(
         .addMBB(TailMBB);
 
     // .looptail:
-    //   dbar 0
     //   andn scratch, dest, mask
     //   or scratch, scratch, newval
     //   sc.[w|d] scratch, scratch, (addr)
     //   beqz scratch, loophead
     //   b done
-    BuildMI(LoopTailMBB, DL, TII->get(LoongArch::DBAR)).addImm(0);
     BuildMI(LoopTailMBB, DL, TII->get(LoongArch::ANDN), ScratchReg)
         .addReg(DestReg)
         .addReg(MaskReg);
@@ -598,9 +571,24 @@ bool LoongArchExpandAtomicPseudo::expandAtomicCmpXchg(
     BuildMI(LoopTailMBB, DL, TII->get(LoongArch::B)).addMBB(DoneMBB);
   }
 
+  AtomicOrdering FailureOrdering =
+      static_cast<AtomicOrdering>(MI.getOperand(IsMasked ? 6 : 5).getImm());
+  int hint;
+
+  switch (FailureOrdering) {
+  case AtomicOrdering::Acquire:
+  case AtomicOrdering::AcquireRelease:
+  case AtomicOrdering::SequentiallyConsistent:
+    // acquire
+    hint = 0b10100;
+    break;
+  default:
+    hint = 0x700;
+  }
+
   // .tail:
-  //   dbar 0x700
-  BuildMI(TailMBB, DL, TII->get(LoongArch::DBAR)).addImm(0x700);
+  //   dbar 0x700 | acquire
+  BuildMI(TailMBB, DL, TII->get(LoongArch::DBAR)).addImm(hint);
 
   NextMBBI = MBB.end();
   MI.eraseFromParent();
diff --git a/llvm/lib/Target/LoongArch/LoongArchExpandPseudoInsts.cpp b/llvm/lib/Target/LoongArch/LoongArchExpandPseudoInsts.cpp
index dd0b2cfde5..72c1f1cec1 100644
--- a/llvm/lib/Target/LoongArch/LoongArchExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchExpandPseudoInsts.cpp
@@ -29,6 +29,8 @@ using namespace llvm;
 
 #define LOONGARCH_PRERA_EXPAND_PSEUDO_NAME                                     \
   "LoongArch Pre-RA pseudo instruction expansion pass"
+#define LOONGARCH_EXPAND_PSEUDO_NAME                                           \
+  "LoongArch pseudo instruction expansion pass"
 
 namespace {
 
@@ -513,15 +515,134 @@ bool LoongArchPreRAExpandPseudo::expandFunctionCALL(
   return true;
 }
 
+class LoongArchExpandPseudo : public MachineFunctionPass {
+public:
+  const LoongArchInstrInfo *TII;
+  static char ID;
+
+  LoongArchExpandPseudo() : MachineFunctionPass(ID) {
+    initializeLoongArchExpandPseudoPass(*PassRegistry::getPassRegistry());
+  }
+
+  bool runOnMachineFunction(MachineFunction &MF) override;
+
+  StringRef getPassName() const override {
+    return LOONGARCH_EXPAND_PSEUDO_NAME;
+  }
+
+private:
+  bool expandMBB(MachineBasicBlock &MBB);
+  bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
+                MachineBasicBlock::iterator &NextMBBI);
+  bool expandCopyCFR(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
+                     MachineBasicBlock::iterator &NextMBBI);
+};
+
+char LoongArchExpandPseudo::ID = 0;
+
+bool LoongArchExpandPseudo::runOnMachineFunction(MachineFunction &MF) {
+  TII =
+      static_cast<const LoongArchInstrInfo *>(MF.getSubtarget().getInstrInfo());
+
+  bool Modified = false;
+  for (auto &MBB : MF)
+    Modified |= expandMBB(MBB);
+
+  return Modified;
+}
+
+bool LoongArchExpandPseudo::expandMBB(MachineBasicBlock &MBB) {
+  bool Modified = false;
+
+  MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
+  while (MBBI != E) {
+    MachineBasicBlock::iterator NMBBI = std::next(MBBI);
+    Modified |= expandMI(MBB, MBBI, NMBBI);
+    MBBI = NMBBI;
+  }
+
+  return Modified;
+}
+
+bool LoongArchExpandPseudo::expandMI(MachineBasicBlock &MBB,
+                                     MachineBasicBlock::iterator MBBI,
+                                     MachineBasicBlock::iterator &NextMBBI) {
+  switch (MBBI->getOpcode()) {
+  case LoongArch::PseudoCopyCFR:
+    return expandCopyCFR(MBB, MBBI, NextMBBI);
+  }
+
+  return false;
+}
+
+bool LoongArchExpandPseudo::expandCopyCFR(
+    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
+    MachineBasicBlock::iterator &NextMBBI) {
+  MachineFunction *MF = MBB.getParent();
+  MachineInstr &MI = *MBBI;
+  DebugLoc DL = MI.getDebugLoc();
+
+  // Expand:
+  // MBB:
+  //    fcmp.caf.s  $dst, $fa0, $fa0 # set $dst 0(false)
+  //    bceqz $src, SinkBB
+  // FalseBB:
+  //    fcmp.cueq.s $dst, $fa0, $fa0 # set $dst 1(true)
+  // SinkBB:
+  //    fallthrough
+
+  const BasicBlock *LLVM_BB = MBB.getBasicBlock();
+  auto *FalseBB = MF->CreateMachineBasicBlock(LLVM_BB);
+  auto *SinkBB = MF->CreateMachineBasicBlock(LLVM_BB);
+
+  MF->insert(++MBB.getIterator(), FalseBB);
+  MF->insert(++FalseBB->getIterator(), SinkBB);
+
+  Register DestReg = MI.getOperand(0).getReg();
+  Register SrcReg = MI.getOperand(1).getReg();
+  // DestReg = 0
+  BuildMI(MBB, MBBI, DL, TII->get(LoongArch::SET_CFR_FALSE), DestReg);
+  // Insert branch instruction.
+  BuildMI(MBB, MBBI, DL, TII->get(LoongArch::BCEQZ))
+      .addReg(SrcReg)
+      .addMBB(SinkBB);
+  // DestReg = 1
+  BuildMI(FalseBB, DL, TII->get(LoongArch::SET_CFR_TRUE), DestReg);
+
+  FalseBB->addSuccessor(SinkBB);
+
+  SinkBB->splice(SinkBB->end(), &MBB, MI, MBB.end());
+  SinkBB->transferSuccessors(&MBB);
+
+  MBB.addSuccessor(FalseBB);
+  MBB.addSuccessor(SinkBB);
+
+  NextMBBI = MBB.end();
+  MI.eraseFromParent();
+
+  // Make sure live-ins are correctly attached to this new basic block.
+  LivePhysRegs LiveRegs;
+  computeAndAddLiveIns(LiveRegs, *FalseBB);
+  computeAndAddLiveIns(LiveRegs, *SinkBB);
+
+  return true;
+}
+
 } // end namespace
 
 INITIALIZE_PASS(LoongArchPreRAExpandPseudo, "loongarch-prera-expand-pseudo",
                 LOONGARCH_PRERA_EXPAND_PSEUDO_NAME, false, false)
 
+INITIALIZE_PASS(LoongArchExpandPseudo, "loongarch-expand-pseudo",
+                LOONGARCH_EXPAND_PSEUDO_NAME, false, false)
+
 namespace llvm {
 
 FunctionPass *createLoongArchPreRAExpandPseudoPass() {
   return new LoongArchPreRAExpandPseudo();
 }
+FunctionPass *createLoongArchExpandPseudoPass() {
+  return new LoongArchExpandPseudo();
+}
 
 } // end namespace llvm
diff --git a/llvm/lib/Target/LoongArch/LoongArchFloat32InstrInfo.td b/llvm/lib/Target/LoongArch/LoongArchFloat32InstrInfo.td
index d4d8736ec0..6f35609df7 100644
--- a/llvm/lib/Target/LoongArch/LoongArchFloat32InstrInfo.td
+++ b/llvm/lib/Target/LoongArch/LoongArchFloat32InstrInfo.td
@@ -126,6 +126,23 @@ def PseudoST_CFR : Pseudo<(outs),
 let hasSideEffects = 0, mayLoad = 1, mayStore = 0 in
 def PseudoLD_CFR : Pseudo<(outs CFR:$ccd),
                           (ins GPR:$rj, grlenimm:$imm)>;
+
+// SET_CFR_{FALSE,TRUE}
+// These instructions are defined in order to avoid expensive check error if
+// regular instruction patterns are used.
+// fcmp.caf.s $dst, $fa0, $fa0
+def SET_CFR_FALSE : SET_CFR<0x0c100000, "fcmp.caf.s">;
+// fcmp.cueq.s $dst, $fa0, $fa0
+def SET_CFR_TRUE  : SET_CFR<0x0c160000, "fcmp.cueq.s">;
+
+// Pseudo instruction for copying CFRs.
+def PseudoCopyCFR : Pseudo<(outs CFR:$dst), (ins CFR:$src)> {
+  let mayLoad = 0;
+  let mayStore = 0;
+  let hasSideEffects = 0;
+  let Size = 12;
+}
+
 } // Predicates = [HasBasicF]
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/LoongArch/LoongArchFloatInstrFormats.td b/llvm/lib/Target/LoongArch/LoongArchFloatInstrFormats.td
index f853fca5c8..f66f620ca8 100644
--- a/llvm/lib/Target/LoongArch/LoongArchFloatInstrFormats.td
+++ b/llvm/lib/Target/LoongArch/LoongArchFloatInstrFormats.td
@@ -218,3 +218,15 @@ class FP_STORE_2RI12<bits<32> op, RegisterClass rc = FPR32>
     : FPFmt2RI12<op, (outs), (ins rc:$fd, GPR:$rj, simm12:$imm12),
                  "$fd, $rj, $imm12">;
 } // hasSideEffects = 0, mayLoad = 0, mayStore = 1
+
+// This class is used to define `SET_CFR_{FALSE,TRUE}` instructions which are
+// used to expand `PseudoCopyCFR`.
+class SET_CFR<bits<32> op, string opcstr>
+    : FP_CMP<op> {
+  let isCodeGenOnly = 1;
+  let fj = 0; // fa0
+  let fk = 0; // fa0
+  let AsmString = opcstr # "\t$cd, $$fa0, $$fa0";
+  let OutOperandList = (outs CFR:$cd);
+  let InOperandList = (ins);
+}
diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
index 7d01887f24..334daccab1 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
@@ -154,6 +154,8 @@ LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM,
     setLibcallName(RTLIB::MUL_I128, nullptr);
   }
 
+  setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom);
+
   static const ISD::CondCode FPCCToExpand[] = {
       ISD::SETOGT, ISD::SETOGE, ISD::SETUGT, ISD::SETUGE,
       ISD::SETGE,  ISD::SETNE,  ISD::SETGT};
@@ -269,6 +271,8 @@ bool LoongArchTargetLowering::isOffsetFoldingLegal(
 SDValue LoongArchTargetLowering::LowerOperation(SDValue Op,
                                                 SelectionDAG &DAG) const {
   switch (Op.getOpcode()) {
+  case ISD::ATOMIC_FENCE:
+    return lowerATOMIC_FENCE(Op, DAG);
   case ISD::EH_DWARF_CFA:
     return lowerEH_DWARF_CFA(Op, DAG);
   case ISD::GlobalAddress:
@@ -313,6 +317,22 @@ SDValue LoongArchTargetLowering::LowerOperation(SDValue Op,
   return SDValue();
 }
 
+SDValue LoongArchTargetLowering::lowerATOMIC_FENCE(SDValue Op,
+                                                   SelectionDAG &DAG) const {
+  SDLoc DL(Op);
+  SyncScope::ID FenceSSID =
+      static_cast<SyncScope::ID>(Op.getConstantOperandVal(2));
+
+  // singlethread fences only synchronize with signal handlers on the same
+  // thread and thus only need to preserve instruction order, not actually
+  // enforce memory ordering.
+  if (FenceSSID == SyncScope::SingleThread)
+    // MEMBARRIER is a compiler barrier; it codegens to a no-op.
+    return DAG.getNode(ISD::MEMBARRIER, DL, MVT::Other, Op.getOperand(0));
+
+  return Op;
+}
+
 SDValue LoongArchTargetLowering::lowerWRITE_REGISTER(SDValue Op,
                                                      SelectionDAG &DAG) const {
 
@@ -4164,8 +4184,9 @@ LoongArchTargetLowering::shouldExpandAtomicCmpXchgInIR(
 Value *LoongArchTargetLowering::emitMaskedAtomicCmpXchgIntrinsic(
     IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr,
     Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const {
-  Value *Ordering =
-      Builder.getIntN(Subtarget.getGRLen(), static_cast<uint64_t>(Ord));
+  AtomicOrdering FailOrd = CI->getFailureOrdering();
+  Value *FailureOrdering =
+      Builder.getIntN(Subtarget.getGRLen(), static_cast<uint64_t>(FailOrd));
 
   // TODO: Support cmpxchg on LA32.
   Intrinsic::ID CmpXchgIntrID = Intrinsic::loongarch_masked_cmpxchg_i64;
@@ -4176,7 +4197,7 @@ Value *LoongArchTargetLowering::emitMaskedAtomicCmpXchgIntrinsic(
   Function *MaskedCmpXchg =
       Intrinsic::getDeclaration(CI->getModule(), CmpXchgIntrID, Tys);
   Value *Result = Builder.CreateCall(
-      MaskedCmpXchg, {AlignedAddr, CmpVal, NewVal, Mask, Ordering});
+      MaskedCmpXchg, {AlignedAddr, CmpVal, NewVal, Mask, FailureOrdering});
   Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
   return Result;
 }
diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
index 49d563a513..3141286671 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
@@ -262,6 +262,7 @@ class LoongArchTargetLowering : public TargetLowering {
   MachineBasicBlock *
   EmitInstrWithCustomInserter(MachineInstr &MI,
                               MachineBasicBlock *BB) const override;
+  SDValue lowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerEH_DWARF_CFA(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerFP_TO_SINT(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.cpp b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.cpp
index 9fad3377a8..ddd1c9943f 100644
--- a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.cpp
@@ -77,6 +77,12 @@ void LoongArchInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
         .addReg(SrcReg, getKillRegState(KillSrc));
     return;
   }
+  // CFR->CFR copy.
+  if (LoongArch::CFRRegClass.contains(DstReg, SrcReg)) {
+    BuildMI(MBB, MBBI, DL, get(LoongArch::PseudoCopyCFR), DstReg)
+        .addReg(SrcReg, getKillRegState(KillSrc));
+    return;
+  }
 
   // FPR->FPR copies.
   unsigned Opc;
diff --git a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td
index 5d4ed46025..d030fc9aa5 100644
--- a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td
+++ b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td
@@ -1661,7 +1661,29 @@ def : RegRegStPat<store, STX_D, GPR, i64>;
 
 /// Atomic loads and stores
 
-def : Pat<(atomic_fence timm, timm), (DBAR 0)>;
+// DBAR hint encoding for LA664 and later micro-architectures, paraphrased from
+// the Linux patch revealing it [1]:
+//
+// - Bit 4: kind of constraint (0: completion, 1: ordering)
+// - Bit 3: barrier for previous read (0: true, 1: false)
+// - Bit 2: barrier for previous write (0: true, 1: false)
+// - Bit 1: barrier for succeeding read (0: true, 1: false)
+// - Bit 0: barrier for succeeding write (0: true, 1: false)
+//
+// Hint 0x700: barrier for "read after read" from the same address, which is
+// e.g. needed by LL-SC loops on older models. (DBAR 0x700 behaves the same as
+// nop if such reordering is disabled on supporting newer models.)
+//
+// [1]: https://lore.kernel.org/loongarch/20230516124536.535343-1-chenhuacai@loongson.cn/
+//
+// Implementations without support for the finer-granularity hints simply treat
+// all as the full barrier (DBAR 0), so we can unconditionally start emiting the
+// more precise hints right away.
+
+def : Pat<(atomic_fence 4, timm), (DBAR 0b10100)>; // acquire
+def : Pat<(atomic_fence 5, timm), (DBAR 0b10010)>; // release
+def : Pat<(atomic_fence 6, timm), (DBAR 0b10000)>; // acqrel
+def : Pat<(atomic_fence 7, timm), (DBAR 0b10000)>; // seqcst
 
 defm : LdPat<atomic_load_8, LD_B>;
 defm : LdPat<atomic_load_16, LD_H>;
@@ -1792,7 +1814,7 @@ def PseudoMaskedAtomicLoadMin32 : PseudoMaskedAMMinMax;
 
 class PseudoCmpXchg
     : Pseudo<(outs GPR:$res, GPR:$scratch),
-             (ins GPR:$addr, GPR:$cmpval, GPR:$newval)> {
+             (ins GPR:$addr, GPR:$cmpval, GPR:$newval, grlenimm:$fail_order)> {
   let Constraints = "@earlyclobber $res,@earlyclobber $scratch";
   let mayLoad = 1;
   let mayStore = 1;
@@ -1806,7 +1828,7 @@ def PseudoCmpXchg64 : PseudoCmpXchg;
 def PseudoMaskedCmpXchg32
     : Pseudo<(outs GPR:$res, GPR:$scratch),
              (ins GPR:$addr, GPR:$cmpval, GPR:$newval, GPR:$mask,
-              grlenimm:$ordering)> {
+              grlenimm:$fail_order)> {
   let Constraints = "@earlyclobber $res,@earlyclobber $scratch";
   let mayLoad = 1;
   let mayStore = 1;
@@ -1824,6 +1846,43 @@ class AtomicPat<Intrinsic intrin, Pseudo AMInst>
     : Pat<(intrin GPR:$addr, GPR:$incr, GPR:$mask, timm:$ordering),
           (AMInst GPR:$addr, GPR:$incr, GPR:$mask, timm:$ordering)>;
 
+// These atomic cmpxchg PatFrags only care about the failure ordering.
+// The PatFrags defined by multiclass `ternary_atomic_op_ord` in
+// TargetSelectionDAG.td care about the merged memory ordering that is the
+// stronger one between success and failure. But for LoongArch LL-SC we only
+// need to care about the failure ordering as explained in PR #67391. So we
+// define these PatFrags that will be used to define cmpxchg pats below.
+multiclass ternary_atomic_op_failure_ord {
+  def NAME#_failure_monotonic : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
+      (!cast<SDPatternOperator>(NAME) node:$ptr, node:$cmp, node:$val), [{
+    AtomicOrdering Ordering = cast<AtomicSDNode>(N)->getFailureOrdering();
+    return Ordering == AtomicOrdering::Monotonic;
+  }]>;
+  def NAME#_failure_acquire : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
+      (!cast<SDPatternOperator>(NAME) node:$ptr, node:$cmp, node:$val), [{
+    AtomicOrdering Ordering = cast<AtomicSDNode>(N)->getFailureOrdering();
+    return Ordering == AtomicOrdering::Acquire;
+  }]>;
+  def NAME#_failure_release : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
+      (!cast<SDPatternOperator>(NAME) node:$ptr, node:$cmp, node:$val), [{
+    AtomicOrdering Ordering = cast<AtomicSDNode>(N)->getFailureOrdering();
+    return Ordering == AtomicOrdering::Release;
+  }]>;
+  def NAME#_failure_acq_rel : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
+      (!cast<SDPatternOperator>(NAME) node:$ptr, node:$cmp, node:$val), [{
+    AtomicOrdering Ordering = cast<AtomicSDNode>(N)->getFailureOrdering();
+    return Ordering == AtomicOrdering::AcquireRelease;
+  }]>;
+  def NAME#_failure_seq_cst : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
+      (!cast<SDPatternOperator>(NAME) node:$ptr, node:$cmp, node:$val), [{
+    AtomicOrdering Ordering = cast<AtomicSDNode>(N)->getFailureOrdering();
+    return Ordering == AtomicOrdering::SequentiallyConsistent;
+  }]>;
+}
+
+defm atomic_cmp_swap_32 : ternary_atomic_op_failure_ord;
+defm atomic_cmp_swap_64 : ternary_atomic_op_failure_ord;
+
 let Predicates = [IsLA64] in {
 def : AtomicPat<int_loongarch_masked_atomicrmw_xchg_i64,
                 PseudoMaskedAtomicSwap32>;
@@ -1882,14 +1941,28 @@ def : AtomicPat<int_loongarch_masked_atomicrmw_umax_i64,
 def : AtomicPat<int_loongarch_masked_atomicrmw_umin_i64,
                 PseudoMaskedAtomicLoadUMin32>;
 
-def : Pat<(atomic_cmp_swap_64 GPR:$addr, GPR:$cmp, GPR:$new),
-          (PseudoCmpXchg64 GPR:$addr, GPR:$cmp, GPR:$new)>;
+// Ordering constants must be kept in sync with the AtomicOrdering enum in
+// AtomicOrdering.h.
+multiclass PseudoCmpXchgPat<string Op, Pseudo CmpXchgInst,
+                            ValueType vt = GRLenVT> {
+  def : Pat<(vt (!cast<PatFrag>(Op#"_failure_monotonic") GPR:$addr, GPR:$cmp, GPR:$new)),
+            (CmpXchgInst GPR:$addr, GPR:$cmp, GPR:$new, 2)>;
+  def : Pat<(vt (!cast<PatFrag>(Op#"_failure_acquire") GPR:$addr, GPR:$cmp, GPR:$new)),
+            (CmpXchgInst GPR:$addr, GPR:$cmp, GPR:$new, 4)>;
+  def : Pat<(vt (!cast<PatFrag>(Op#"_failure_release") GPR:$addr, GPR:$cmp, GPR:$new)),
+            (CmpXchgInst GPR:$addr, GPR:$cmp, GPR:$new, 5)>;
+  def : Pat<(vt (!cast<PatFrag>(Op#"_failure_acq_rel") GPR:$addr, GPR:$cmp, GPR:$new)),
+            (CmpXchgInst GPR:$addr, GPR:$cmp, GPR:$new, 6)>;
+  def : Pat<(vt (!cast<PatFrag>(Op#"_failure_seq_cst") GPR:$addr, GPR:$cmp, GPR:$new)),
+            (CmpXchgInst GPR:$addr, GPR:$cmp, GPR:$new, 7)>;
+}
+
+defm : PseudoCmpXchgPat<"atomic_cmp_swap_32", PseudoCmpXchg32>;
+defm : PseudoCmpXchgPat<"atomic_cmp_swap_64", PseudoCmpXchg64, i64>;
 def : Pat<(int_loongarch_masked_cmpxchg_i64
-            GPR:$addr, GPR:$cmpval, GPR:$newval, GPR:$mask, timm:$ordering),
+            GPR:$addr, GPR:$cmpval, GPR:$newval, GPR:$mask, timm:$fail_order),
           (PseudoMaskedCmpXchg32
-            GPR:$addr, GPR:$cmpval, GPR:$newval, GPR:$mask, timm:$ordering)>;
-def : Pat<(atomic_cmp_swap_32 GPR:$addr, GPR:$cmp, GPR:$new),
-          (PseudoCmpXchg32 GPR:$addr, GPR:$cmp, GPR:$new)>;
+            GPR:$addr, GPR:$cmpval, GPR:$newval, GPR:$mask, timm:$fail_order)>;
 
 def : PseudoMaskedAMMinMaxPat<int_loongarch_masked_atomicrmw_max_i64,
                               PseudoMaskedAtomicLoadMax32>;
@@ -1919,9 +1992,9 @@ defm : PseudoBinPat<"atomic_load_xor_32", PseudoAtomicLoadXor32>;
 /// Intrinsics
 
 def : Pat<(int_loongarch_cacop_d timm:$op, i64:$rj, timm:$imm12),
-          (CACOP uimm5:$op, GPR:$rj, simm12:$imm12)>;
+          (CACOP timm:$op, GPR:$rj, timm:$imm12)>;
 def : Pat<(int_loongarch_cacop_w i32:$op, i32:$rj, i32:$imm12),
-          (CACOP uimm5:$op, GPR:$rj, simm12:$imm12)>;
+          (CACOP timm:$op, GPR:$rj, timm:$imm12)>;
 def : Pat<(loongarch_dbar uimm15:$imm15), (DBAR uimm15:$imm15)>;
 def : Pat<(loongarch_ibar uimm15:$imm15), (IBAR uimm15:$imm15)>;
 def : Pat<(loongarch_break uimm15:$imm15), (BREAK uimm15:$imm15)>;
@@ -2085,9 +2158,9 @@ def : Pat<(int_loongarch_asrtle_d GPR:$rj, GPR:$rk),
 def : Pat<(int_loongarch_asrtgt_d GPR:$rj, GPR:$rk),
           (ASRTGT_D GPR:$rj, GPR:$rk)>;
 def : Pat<(int_loongarch_lddir_d GPR:$rj, timm:$imm8),
-          (LDDIR GPR:$rj, uimm8:$imm8)>;
+          (LDDIR GPR:$rj, timm:$imm8)>;
 def : Pat<(int_loongarch_ldpte_d GPR:$rj, timm:$imm8),
-          (LDPTE GPR:$rj, uimm8:$imm8)>;
+          (LDPTE GPR:$rj, timm:$imm8)>;
 } // Predicates = [IsLA64]
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/LoongArch/LoongArchRegisterInfo.cpp b/llvm/lib/Target/LoongArch/LoongArchRegisterInfo.cpp
index 4037c4d370..257b947a3c 100644
--- a/llvm/lib/Target/LoongArch/LoongArchRegisterInfo.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchRegisterInfo.cpp
@@ -98,13 +98,6 @@ LoongArchRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
   if (TFI->hasBP(MF))
     markSuperRegs(Reserved, LoongArchABI::getBPReg()); // bp
 
-  // FIXME: To avoid generating COPY instructions between CFRs, only use $fcc0.
-  // This is required to work around the fact that COPY instruction between CFRs
-  // is not provided in LoongArch.
-  if (MF.getSubtarget<LoongArchSubtarget>().hasBasicF())
-    for (size_t Reg = LoongArch::FCC1; Reg <= LoongArch::FCC7; ++Reg)
-      markSuperRegs(Reserved, Reg);
-
   assert(checkAllSuperRegsMarked(Reserved));
   return Reserved;
 }
diff --git a/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp b/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp
index c54a9b9c76..a5a4d78ace 100644
--- a/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchTargetMachine.cpp
@@ -180,6 +180,7 @@ LoongArchTargetMachine::getTargetTransformInfo(const Function &F) const {
 void LoongArchPassConfig::addPreEmitPass() { addPass(&BranchRelaxationPassID); }
 
 void LoongArchPassConfig::addPreEmitPass2() {
+  addPass(createLoongArchExpandPseudoPass());
   // Schedule the expansion of AtomicPseudos at the last possible moment,
   // avoiding the possibility for other passes to break the requirements for
   // forward progress in the LL/SC block.
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h
index ae9bb8af04..f840f9fa2b 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h
@@ -31,8 +31,8 @@ class LoongArchAsmBackend : public MCAsmBackend {
 public:
   LoongArchAsmBackend(const MCSubtargetInfo &STI, uint8_t OSABI, bool Is64Bit,
                       const MCTargetOptions &Options)
-      : MCAsmBackend(support::little), STI(STI), OSABI(OSABI), Is64Bit(Is64Bit),
-        TargetOptions(Options) {}
+      : MCAsmBackend(llvm::endianness::little), STI(STI), OSABI(OSABI),
+        Is64Bit(Is64Bit), TargetOptions(Options) {}
   ~LoongArchAsmBackend() override {}
 
   void applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCCodeEmitter.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCCodeEmitter.cpp
index 03fb9e008a..fbe817a2b5 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCCodeEmitter.cpp
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCCodeEmitter.cpp
@@ -296,7 +296,7 @@ void LoongArchMCCodeEmitter::expandToVectorLDI(
   }
   MCInst TmpInst = MCInstBuilder(Opc).addOperand(MI.getOperand(0)).addImm(Imm);
   uint32_t Binary = getBinaryCodeForInstr(TmpInst, Fixups, STI);
-  support::endian::write(CB, Binary, support::little);
+  support::endian::write(CB, Binary, llvm::endianness::little);
 }
 
 void LoongArchMCCodeEmitter::encodeInstruction(
@@ -326,7 +326,7 @@ void LoongArchMCCodeEmitter::encodeInstruction(
     llvm_unreachable("Unhandled encodeInstruction length!");
   case 4: {
     uint32_t Bits = getBinaryCodeForInstr(MI, Fixups, STI);
-    support::endian::write(CB, Bits, support::little);
+    support::endian::write(CB, Bits, llvm::endianness::little);
     break;
   }
   }
diff --git a/llvm/lib/Target/M68k/M68kExpandPseudo.cpp b/llvm/lib/Target/M68k/M68kExpandPseudo.cpp
index 2f60fc834a..13268d754a 100644
--- a/llvm/lib/Target/M68k/M68kExpandPseudo.cpp
+++ b/llvm/lib/Target/M68k/M68kExpandPseudo.cpp
@@ -258,32 +258,22 @@ bool M68kExpandPseudo::ExpandMI(MachineBasicBlock &MBB,
 
     if (StackAdj == 0) {
       MIB = BuildMI(MBB, MBBI, DL, TII->get(M68k::RTS));
-    } else if (isUInt<16>(StackAdj)) {
-
-      if (STI->atLeastM68020()) {
-        llvm_unreachable("RTD is not implemented");
-      } else {
-        // Copy PC from stack to a free address(A0 or A1) register
-        // TODO check if pseudo expand uses free address register
-        BuildMI(MBB, MBBI, DL, TII->get(M68k::MOV32aj), M68k::A1)
-            .addReg(M68k::SP);
+    } else {
+      // Copy return address from stack to a free address(A0 or A1) register
+      // TODO check if pseudo expand uses free address register
+      BuildMI(MBB, MBBI, DL, TII->get(M68k::MOV32aj), M68k::A1)
+          .addReg(M68k::SP);
 
-        // Adjust SP
-        FL->emitSPUpdate(MBB, MBBI, StackAdj, /*InEpilogue=*/true);
+      // Adjust SP
+      FL->emitSPUpdate(MBB, MBBI, StackAdj, /*InEpilogue=*/true);
 
-        // Put the return address on stack
-        BuildMI(MBB, MBBI, DL, TII->get(M68k::MOV32ja))
-            .addReg(M68k::SP)
-            .addReg(M68k::A1);
+      // Put the return address on stack
+      BuildMI(MBB, MBBI, DL, TII->get(M68k::MOV32ja))
+          .addReg(M68k::SP)
+          .addReg(M68k::A1);
 
-        // RTS
-        BuildMI(MBB, MBBI, DL, TII->get(M68k::RTS));
-      }
-    } else {
-      // TODO: RTD can only handle immediates as big as 2**16-1.
-      // If we need to pop off bytes before the return address, we
-      // must do it manually.
-      llvm_unreachable("Stack adjustment size not supported");
+      // RTS
+      BuildMI(MBB, MBBI, DL, TII->get(M68k::RTS));
     }
 
     // FIXME: Can rest of the operands be ignored, if there is any?
diff --git a/llvm/lib/Target/M68k/M68kISelLowering.cpp b/llvm/lib/Target/M68k/M68kISelLowering.cpp
index d1ed26457f..0830cc7feb 100644
--- a/llvm/lib/Target/M68k/M68kISelLowering.cpp
+++ b/llvm/lib/Target/M68k/M68kISelLowering.cpp
@@ -3050,9 +3050,8 @@ M68kTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
 
 /// Determines whether the callee is required to pop its own arguments.
 /// Callee pop is necessary to support tail calls.
-bool M68k::isCalleePop(CallingConv::ID CallingConv, bool IsVarArg,
-                       bool GuaranteeTCO) {
-  return false;
+bool M68k::isCalleePop(CallingConv::ID CC, bool IsVarArg, bool GuaranteeTCO) {
+  return CC == CallingConv::M68k_RTD && !IsVarArg;
 }
 
 // Return true if it is OK for this CMOV pseudo-opcode to be cascaded
diff --git a/llvm/lib/Target/M68k/M68kInstrInfo.cpp b/llvm/lib/Target/M68k/M68kInstrInfo.cpp
index 8d36e94d8e..d56fef9e90 100644
--- a/llvm/lib/Target/M68k/M68kInstrInfo.cpp
+++ b/llvm/lib/Target/M68k/M68kInstrInfo.cpp
@@ -361,6 +361,7 @@ bool M68kInstrInfo::ExpandMOVX_RR(MachineInstrBuilder &MIB, MVT MVTDst,
 
   assert(RCDst && RCSrc && "Wrong use of MOVX_RR");
   assert(RCDst != RCSrc && "You cannot use the same Reg Classes with MOVX_RR");
+  (void)RCSrc;
 
   // We need to find the super source register that matches the size of Dst
   unsigned SSrc = RI.getMatchingMegaReg(Src, RCDst);
@@ -407,6 +408,7 @@ bool M68kInstrInfo::ExpandMOVSZX_RR(MachineInstrBuilder &MIB, bool IsSigned,
 
   assert(RCDst && RCSrc && "Wrong use of MOVSX_RR");
   assert(RCDst != RCSrc && "You cannot use the same Reg Classes with MOVSX_RR");
+  (void)RCSrc;
 
   // We need to find the super source register that matches the size of Dst
   unsigned SSrc = RI.getMatchingMegaReg(Src, RCDst);
@@ -746,6 +748,7 @@ void M68kInstrInfo::storeRegToStackSlot(
   const MachineFrameInfo &MFI = MBB.getParent()->getFrameInfo();
   assert(MFI.getObjectSize(FrameIndex) >= TRI->getSpillSize(*RC) &&
          "Stack slot is too small to store");
+  (void)MFI;
 
   unsigned Opc = getStoreRegOpcode(SrcReg, RC, TRI, Subtarget);
   DebugLoc DL = MBB.findDebugLoc(MI);
@@ -763,6 +766,7 @@ void M68kInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
   const MachineFrameInfo &MFI = MBB.getParent()->getFrameInfo();
   assert(MFI.getObjectSize(FrameIndex) >= TRI->getSpillSize(*RC) &&
          "Stack slot is too small to load");
+  (void)MFI;
 
   unsigned Opc = getLoadRegOpcode(DstReg, RC, TRI, Subtarget);
   DebugLoc DL = MBB.findDebugLoc(MI);
diff --git a/llvm/lib/Target/M68k/MCTargetDesc/M68kAsmBackend.cpp b/llvm/lib/Target/M68k/MCTargetDesc/M68kAsmBackend.cpp
index b66557ec6c..1b85e6df37 100644
--- a/llvm/lib/Target/M68k/MCTargetDesc/M68kAsmBackend.cpp
+++ b/llvm/lib/Target/M68k/MCTargetDesc/M68kAsmBackend.cpp
@@ -41,7 +41,7 @@ namespace {
 class M68kAsmBackend : public MCAsmBackend {
 
 public:
-  M68kAsmBackend(const Target &T) : MCAsmBackend(support::big) {}
+  M68kAsmBackend(const Target &T) : MCAsmBackend(llvm::endianness::big) {}
 
   unsigned getNumFixupKinds() const override { return 0; }
 
diff --git a/llvm/lib/Target/M68k/MCTargetDesc/M68kBaseInfo.h b/llvm/lib/Target/M68k/MCTargetDesc/M68kBaseInfo.h
index e52b4961e3..1376b06bef 100644
--- a/llvm/lib/Target/M68k/MCTargetDesc/M68kBaseInfo.h
+++ b/llvm/lib/Target/M68k/MCTargetDesc/M68kBaseInfo.h
@@ -82,11 +82,11 @@ template <typename value_t> value_t swapWord(value_t Val) {
   const unsigned NumWords = sizeof(Val) / 2;
   if (NumWords <= 1)
     return Val;
-  Val = support::endian::byte_swap(Val, support::big);
+  Val = support::endian::byte_swap(Val, llvm::endianness::big);
   value_t NewVal = 0;
   for (unsigned i = 0U; i != NumWords; ++i) {
     uint16_t Part = (Val >> (i * 16)) & 0xFFFF;
-    Part = support::endian::byte_swap(Part, support::big);
+    Part = support::endian::byte_swap(Part, llvm::endianness::big);
     NewVal |= (Part << (i * 16));
   }
   return NewVal;
diff --git a/llvm/lib/Target/M68k/MCTargetDesc/M68kMCCodeEmitter.cpp b/llvm/lib/Target/M68k/MCTargetDesc/M68kMCCodeEmitter.cpp
index 7fc5395671..97f5d7a3dc 100644
--- a/llvm/lib/Target/M68k/MCTargetDesc/M68kMCCodeEmitter.cpp
+++ b/llvm/lib/Target/M68k/MCTargetDesc/M68kMCCodeEmitter.cpp
@@ -203,10 +203,9 @@ void M68kMCCodeEmitter::encodeInstruction(const MCInst &MI,
                                           SmallVectorImpl<char> &CB,
                                           SmallVectorImpl<MCFixup> &Fixups,
                                           const MCSubtargetInfo &STI) const {
-  unsigned Opcode = MI.getOpcode();
-
-  LLVM_DEBUG(dbgs() << "EncodeInstruction: " << MCII.getName(Opcode) << "("
-                    << Opcode << ")\n");
+  LLVM_DEBUG(dbgs() << "EncodeInstruction: " << MCII.getName(MI.getOpcode())
+                    << "(" << MI.getOpcode() << ")\n");
+  (void)MCII;
 
   // Try using the new method first.
   APInt EncodedInst(16, 0U);
@@ -218,7 +217,7 @@ void M68kMCCodeEmitter::encodeInstruction(const MCInst &MI,
   for (uint64_t Word : Data) {
     for (int i = 0; i < 4 && InstSize > 0; ++i, InstSize -= 16) {
       support::endian::write<uint16_t>(CB, static_cast<uint16_t>(Word),
-                                       support::big);
+                                       llvm::endianness::big);
       Word >>= 16;
     }
   }
diff --git a/llvm/lib/Target/MSP430/MCTargetDesc/MSP430AsmBackend.cpp b/llvm/lib/Target/MSP430/MCTargetDesc/MSP430AsmBackend.cpp
index a667f457bd..bd9f627944 100644
--- a/llvm/lib/Target/MSP430/MCTargetDesc/MSP430AsmBackend.cpp
+++ b/llvm/lib/Target/MSP430/MCTargetDesc/MSP430AsmBackend.cpp
@@ -34,7 +34,7 @@ class MSP430AsmBackend : public MCAsmBackend {
 
 public:
   MSP430AsmBackend(const MCSubtargetInfo &STI, uint8_t OSABI)
-      : MCAsmBackend(support::little), OSABI(OSABI) {}
+      : MCAsmBackend(llvm::endianness::little), OSABI(OSABI) {}
   ~MSP430AsmBackend() override = default;
 
   void applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
diff --git a/llvm/lib/Target/MSP430/MCTargetDesc/MSP430MCCodeEmitter.cpp b/llvm/lib/Target/MSP430/MCTargetDesc/MSP430MCCodeEmitter.cpp
index 985906a353..51428552d8 100644
--- a/llvm/lib/Target/MSP430/MCTargetDesc/MSP430MCCodeEmitter.cpp
+++ b/llvm/lib/Target/MSP430/MCTargetDesc/MSP430MCCodeEmitter.cpp
@@ -94,7 +94,8 @@ void MSP430MCCodeEmitter::encodeInstruction(const MCInst &MI,
   size_t WordCount = Size / 2;
 
   while (WordCount--) {
-    support::endian::write(CB, (uint16_t)BinaryOpCode, support::little);
+    support::endian::write(CB, (uint16_t)BinaryOpCode,
+                           llvm::endianness::little);
     BinaryOpCode >>= 16;
   }
 }
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp
index 71f333d0d0..7eca49e709 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp
@@ -281,7 +281,7 @@ void MipsAsmBackend::applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
   bool microMipsLEByteOrder = needsMMLEByteOrder((unsigned) Kind);
 
   for (unsigned i = 0; i != NumBytes; ++i) {
-    unsigned Idx = Endian == support::little
+    unsigned Idx = Endian == llvm::endianness::little
                        ? (microMipsLEByteOrder ? calculateMMLEIndex(i) : i)
                        : (FullSize - 1 - i);
     CurVal |= (uint64_t)((uint8_t)Data[Offset + Idx]) << (i*8);
@@ -293,7 +293,7 @@ void MipsAsmBackend::applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
 
   // Write out the fixed up bytes back to the code/data bits.
   for (unsigned i = 0; i != NumBytes; ++i) {
-    unsigned Idx = Endian == support::little
+    unsigned Idx = Endian == llvm::endianness::little
                        ? (microMipsLEByteOrder ? calculateMMLEIndex(i) : i)
                        : (FullSize - 1 - i);
     Data[Offset + Idx] = (uint8_t)((CurVal >> (i*8)) & 0xff);
@@ -519,7 +519,7 @@ getFixupKindInfo(MCFixupKind Kind) const {
   assert(unsigned(Kind - FirstTargetFixupKind) < getNumFixupKinds() &&
           "Invalid kind!");
 
-  if (Endian == support::little)
+  if (Endian == llvm::endianness::little)
     return LittleEndianInfos[Kind - FirstTargetFixupKind];
   return BigEndianInfos[Kind - FirstTargetFixupKind];
 }
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h b/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h
index 749223a6d0..228a0b4c40 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h
@@ -32,7 +32,8 @@ class MipsAsmBackend : public MCAsmBackend {
 public:
   MipsAsmBackend(const Target &T, const MCRegisterInfo &MRI, const Triple &TT,
                  StringRef CPU, bool N32)
-      : MCAsmBackend(TT.isLittleEndian() ? support::little : support::big),
+      : MCAsmBackend(TT.isLittleEndian() ? llvm::endianness::little
+                                         : llvm::endianness::big),
         TheTriple(TT), IsN32(N32) {}
 
   std::unique_ptr<MCObjectTargetWriter>
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp
index 6a81a842be..73ee44eec2 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp
@@ -206,7 +206,8 @@ void MipsMCCodeEmitter::encodeInstruction(const MCInst &MI,
   if (!Size)
     llvm_unreachable("Desc.getSize() returns 0");
 
-  auto Endian = IsLittleEndian ? support::little : support::big;
+  auto Endian =
+      IsLittleEndian ? llvm::endianness::little : llvm::endianness::big;
   if (Size == 2) {
     support::endian::write<uint16_t>(CB, Binary, Endian);
   } else if (IsLittleEndian && isMicroMips(STI)) {
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 36da2e7b40..617009334d 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -504,13 +504,21 @@ NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,
   // Only logical ops can be done on v4i8 directly, others must be done
   // elementwise.
   setOperationAction(
-      {ISD::ADD,       ISD::MUL,        ISD::ABS,        ISD::SMIN,
-       ISD::SMAX,      ISD::UMIN,       ISD::UMAX,       ISD::CTPOP,
-       ISD::CTLZ,      ISD::ADD,        ISD::SUB,        ISD::MUL,
-       ISD::SHL,       ISD::SREM,       ISD::UREM,       ISD::SDIV,
-       ISD::UDIV,      ISD::SRA,        ISD::SRL,        ISD::MULHS,
-       ISD::MULHU,     ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::SINT_TO_FP,
-       ISD::UINT_TO_FP},
+      {ISD::ABS,         ISD::ADD,        ISD::ADDC,        ISD::ADDE,
+       ISD::BITREVERSE,  ISD::CTLZ,       ISD::CTPOP,       ISD::CTTZ,
+       ISD::FP_TO_SINT,  ISD::FP_TO_UINT, ISD::FSHL,        ISD::FSHR,
+       ISD::MUL,         ISD::MULHS,      ISD::MULHU,       ISD::PARITY,
+       ISD::ROTL,        ISD::ROTR,       ISD::SADDO,       ISD::SADDO_CARRY,
+       ISD::SADDSAT,     ISD::SDIV,       ISD::SDIVREM,     ISD::SELECT_CC,
+       ISD::SETCC,       ISD::SHL,        ISD::SINT_TO_FP,  ISD::SMAX,
+       ISD::SMIN,        ISD::SMULO,      ISD::SMUL_LOHI,   ISD::SRA,
+       ISD::SREM,        ISD::SRL,        ISD::SSHLSAT,     ISD::SSUBO,
+       ISD::SSUBO_CARRY, ISD::SSUBSAT,    ISD::SUB,         ISD::SUBC,
+       ISD::SUBE,        ISD::UADDO,      ISD::UADDO_CARRY, ISD::UADDSAT,
+       ISD::UDIV,        ISD::UDIVREM,    ISD::UINT_TO_FP,  ISD::UMAX,
+       ISD::UMIN,        ISD::UMULO,      ISD::UMUL_LOHI,   ISD::UREM,
+       ISD::USHLSAT,     ISD::USUBO,      ISD::USUBO_CARRY, ISD::VSELECT,
+       ISD::USUBSAT},
       MVT::v4i8, Expand);
 
   // Operations not directly supported by NVPTX.
@@ -693,8 +701,8 @@ NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,
   setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand);
 
   // We have some custom DAG combine patterns for these nodes
-  setTargetDAGCombine({ISD::ADD, ISD::AND, ISD::FADD, ISD::MUL, ISD::SHL,
-                       ISD::SREM, ISD::UREM, ISD::EXTRACT_VECTOR_ELT,
+  setTargetDAGCombine({ISD::ADD, ISD::AND, ISD::EXTRACT_VECTOR_ELT, ISD::FADD,
+                       ISD::LOAD, ISD::MUL, ISD::SHL, ISD::SREM, ISD::UREM,
                        ISD::VSELECT});
 
   // setcc for f16x2 and bf16x2 needs special handling to prevent
@@ -5471,6 +5479,45 @@ static SDValue PerformVSELECTCombine(SDNode *N,
   return DCI.DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v4i8, E);
 }
 
+static SDValue PerformLOADCombine(SDNode *N,
+                                  TargetLowering::DAGCombinerInfo &DCI) {
+  SelectionDAG &DAG = DCI.DAG;
+  LoadSDNode *LD = cast<LoadSDNode>(N);
+
+  // Lower a v16i8 load into a LoadV4 operation with i32 results instead of
+  // letting ReplaceLoadVector split it into smaller loads during legalization.
+  // This is done at dag-combine1 time, so that vector operations with i8
+  // elements can be optimised away instead of being needlessly split during
+  // legalization, which involves storing to the stack and loading it back.
+  EVT VT = N->getValueType(0);
+  if (VT != MVT::v16i8)
+    return SDValue();
+
+  SDLoc DL(N);
+
+  // Create a v4i32 vector load operation, effectively <4 x v4i8>.
+  unsigned Opc = NVPTXISD::LoadV4;
+  EVT NewVT = MVT::v4i32;
+  EVT EltVT = NewVT.getVectorElementType();
+  unsigned NumElts = NewVT.getVectorNumElements();
+  EVT RetVTs[] = {EltVT, EltVT, EltVT, EltVT, MVT::Other};
+  SDVTList RetVTList = DAG.getVTList(RetVTs);
+  SmallVector<SDValue, 8> Ops(N->ops());
+  Ops.push_back(DAG.getIntPtrConstant(LD->getExtensionType(), DL));
+  SDValue NewLoad = DAG.getMemIntrinsicNode(Opc, DL, RetVTList, Ops, NewVT,
+                                            LD->getMemOperand());
+  SDValue NewChain = NewLoad.getValue(NumElts);
+
+  // Create a vector of the same type returned by the original load.
+  SmallVector<SDValue, 4> Elts;
+  for (unsigned i = 0; i < NumElts; i++)
+    Elts.push_back(NewLoad.getValue(i));
+  return DCI.DAG.getMergeValues(
+      {DCI.DAG.getBitcast(VT, DCI.DAG.getBuildVector(NewVT, DL, Elts)),
+       NewChain},
+      DL);
+}
+
 SDValue NVPTXTargetLowering::PerformDAGCombine(SDNode *N,
                                                DAGCombinerInfo &DCI) const {
   CodeGenOptLevel OptLevel = getTargetMachine().getOptLevel();
@@ -5490,6 +5537,8 @@ SDValue NVPTXTargetLowering::PerformDAGCombine(SDNode *N,
       return PerformREMCombine(N, DCI, OptLevel);
     case ISD::SETCC:
       return PerformSETCCCombine(N, DCI);
+    case ISD::LOAD:
+      return PerformLOADCombine(N, DCI);
     case NVPTXISD::StoreRetval:
     case NVPTXISD::StoreRetvalV2:
     case NVPTXISD::StoreRetvalV4:
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
index 5c7c10965e..f6932db2ae 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
@@ -586,6 +586,12 @@ class NVPTXTargetLowering : public TargetLowering {
   AtomicExpansionKind
   shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override;
 
+  bool aggressivelyPreferBuildVectorSources(EVT VecVT) const override {
+    // There's rarely any point of packing something into a vector type if we
+    // already have the source data.
+    return true;
+  }
+
 private:
   const NVPTXSubtarget &STI; // cache the subtarget here
   SDValue getParamSymbol(SelectionDAG &DAG, int idx, EVT) const;
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
index 84ed953ad1..b0b96b94a1 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -3485,6 +3485,9 @@ def : Pat<(v2bf16 (build_vector (bf16 Int16Regs:$a), (bf16 Int16Regs:$b))),
 def : Pat<(v2i16 (build_vector (i16 Int16Regs:$a), (i16 Int16Regs:$b))),
           (V2I16toI32 Int16Regs:$a, Int16Regs:$b)>;
 
+def: Pat<(v2i16 (scalar_to_vector (i16 Int16Regs:$a))),
+         (CVT_u32_u16 Int16Regs:$a, CvtNONE)>;
+
 // Count leading zeros
 let hasSideEffects = false in {
   def CLZr32 : NVPTXInst<(outs Int32Regs:$d), (ins Int32Regs:$a),
diff --git a/llvm/lib/Target/NVPTX/NVVMReflect.cpp b/llvm/lib/Target/NVPTX/NVVMReflect.cpp
index 7ff5a5eb79..7d2678ae59 100644
--- a/llvm/lib/Target/NVPTX/NVVMReflect.cpp
+++ b/llvm/lib/Target/NVPTX/NVVMReflect.cpp
@@ -20,7 +20,6 @@
 
 #include "NVPTX.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/StringMap.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/Function.h"
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
index 89d04dbe37..8bd27571a7 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
@@ -87,7 +87,8 @@ class PPCAsmBackend : public MCAsmBackend {
   Triple TT;
 public:
   PPCAsmBackend(const Target &T, const Triple &TT)
-      : MCAsmBackend(TT.isLittleEndian() ? support::little : support::big),
+      : MCAsmBackend(TT.isLittleEndian() ? llvm::endianness::little
+                                         : llvm::endianness::big),
         TT(TT) {}
 
   unsigned getNumFixupKinds() const override {
@@ -132,7 +133,7 @@ class PPCAsmBackend : public MCAsmBackend {
 
     assert(unsigned(Kind - FirstTargetFixupKind) < getNumFixupKinds() &&
            "Invalid kind!");
-    return (Endian == support::little
+    return (Endian == llvm::endianness::little
                 ? InfosLE
                 : InfosBE)[Kind - FirstTargetFixupKind];
   }
@@ -154,7 +155,8 @@ class PPCAsmBackend : public MCAsmBackend {
     // from the fixup value. The Value has been "split up" into the appropriate
     // bitfields above.
     for (unsigned i = 0; i != NumBytes; ++i) {
-      unsigned Idx = Endian == support::little ? i : (NumBytes - 1 - i);
+      unsigned Idx =
+          Endian == llvm::endianness::little ? i : (NumBytes - 1 - i);
       Data[Offset + i] |= uint8_t((Value >> (Idx * 8)) & 0xff);
     }
   }
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp
index 70f1c9676d..910b5892d0 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp
@@ -508,7 +508,8 @@ void PPCMCCodeEmitter::encodeInstruction(const MCInst &MI,
 
   // Output the constant in big/little endian byte order.
   unsigned Size = getInstSizeInBytes(MI);
-  support::endianness E = IsLittleEndian ? support::little : support::big;
+  llvm::endianness E =
+      IsLittleEndian ? llvm::endianness::little : llvm::endianness::big;
   switch (Size) {
   case 0:
     break;
diff --git a/llvm/lib/Target/PowerPC/P10InstrResources.td b/llvm/lib/Target/PowerPC/P10InstrResources.td
index 0827e528a8..66a0509556 100644
--- a/llvm/lib/Target/PowerPC/P10InstrResources.td
+++ b/llvm/lib/Target/PowerPC/P10InstrResources.td
@@ -825,9 +825,7 @@ def : InstRW<[P10W_F2_4C, P10W_DISP_ANY, P10F2_Read, P10F2_Read, P10F2_Read],
 def : InstRW<[P10W_F2_4C, P10W_DISP_EVEN, P10W_DISP_ANY, P10F2_Read],
       (instrs
     SRADI_rec,
-    SRAWI_rec,
-    TABORTDCI,
-    TABORTWCI
+    SRAWI_rec
 )>;
 
 // Single crack instructions
@@ -835,9 +833,7 @@ def : InstRW<[P10W_F2_4C, P10W_DISP_EVEN, P10W_DISP_ANY, P10F2_Read],
 def : InstRW<[P10W_F2_4C, P10W_DISP_EVEN, P10W_DISP_ANY, P10F2_Read, P10F2_Read],
       (instrs
     SRAD_rec,
-    SRAW_rec,
-    TABORTDC,
-    TABORTWC
+    SRAW_rec
 )>;
 
 // 2-way crack instructions
@@ -1130,10 +1126,7 @@ def : InstRW<[P10W_FX_3C, P10W_DISP_EVEN, P10W_DISP_ANY],
     MFFSCRNI,
     MFFSL,
     MFVSCR,
-    MTFSB0,
-    TBEGIN,
-    TRECHKPT,
-    TSR
+    MTFSB0
 )>;
 
 // Single crack instructions
@@ -1153,9 +1146,7 @@ def : InstRW<[P10W_FX_3C, P10W_DISP_EVEN, P10W_DISP_ANY, P10FX_Read],
     SUBFME8_rec, SUBFME_rec,
     SUBFME8O_rec, SUBFMEO_rec,
     SUBFZE8_rec, SUBFZE_rec,
-    SUBFZE8O_rec, SUBFZEO_rec,
-    TABORT,
-    TRECLAIM
+    SUBFZE8O_rec, SUBFZEO_rec
 )>;
 
 // Single crack instructions
@@ -1862,8 +1853,6 @@ def : InstRW<[P10W_ST_3C, P10W_DISP_EVEN, P10W_DISP_ANY],
     EnforceIEIO,
     MSGSYNC,
     SLBSYNC,
-    TCHECK,
-    TEND,
     TLBSYNC
 )>;
 
diff --git a/llvm/lib/Target/PowerPC/PPCExpandAtomicPseudoInsts.cpp b/llvm/lib/Target/PowerPC/PPCExpandAtomicPseudoInsts.cpp
index a9794ddd05..aee57a5075 100644
--- a/llvm/lib/Target/PowerPC/PPCExpandAtomicPseudoInsts.cpp
+++ b/llvm/lib/Target/PowerPC/PPCExpandAtomicPseudoInsts.cpp
@@ -239,23 +239,18 @@ bool PPCExpandAtomicPseudo::expandAtomicCmpSwap128(
   // loop:
   //   old = lqarx ptr
   //   <compare old, cmp>
-  //   bne 0, fail
+  //   bne 0, exit
   // succ:
   //   stqcx new ptr
   //   bne 0, loop
-  //   b exit
-  // fail:
-  //   stqcx old ptr
   // exit:
   //   ....
   MachineFunction::iterator MFI = ++MBB.getIterator();
   MachineBasicBlock *LoopCmpMBB = MF->CreateMachineBasicBlock(BB);
   MachineBasicBlock *CmpSuccMBB = MF->CreateMachineBasicBlock(BB);
-  MachineBasicBlock *CmpFailMBB = MF->CreateMachineBasicBlock(BB);
   MachineBasicBlock *ExitMBB = MF->CreateMachineBasicBlock(BB);
   MF->insert(MFI, LoopCmpMBB);
   MF->insert(MFI, CmpSuccMBB);
-  MF->insert(MFI, CmpFailMBB);
   MF->insert(MFI, ExitMBB);
   ExitMBB->splice(ExitMBB->begin(), &MBB, std::next(MI.getIterator()),
                   MBB.end());
@@ -276,9 +271,9 @@ bool PPCExpandAtomicPseudo::expandAtomicCmpSwap128(
   BuildMI(CurrentMBB, DL, TII->get(PPC::BCC))
       .addImm(PPC::PRED_NE)
       .addReg(PPC::CR0)
-      .addMBB(CmpFailMBB);
+      .addMBB(ExitMBB);
   CurrentMBB->addSuccessor(CmpSuccMBB);
-  CurrentMBB->addSuccessor(CmpFailMBB);
+  CurrentMBB->addSuccessor(ExitMBB);
   // Build succ.
   CurrentMBB = CmpSuccMBB;
   PairedCopy(TII, *CurrentMBB, CurrentMBB->end(), DL, ScratchHi, ScratchLo,
@@ -288,16 +283,11 @@ bool PPCExpandAtomicPseudo::expandAtomicCmpSwap128(
       .addImm(PPC::PRED_NE)
       .addReg(PPC::CR0)
       .addMBB(LoopCmpMBB);
-  BuildMI(CurrentMBB, DL, TII->get(PPC::B)).addMBB(ExitMBB);
   CurrentMBB->addSuccessor(LoopCmpMBB);
   CurrentMBB->addSuccessor(ExitMBB);
-  CurrentMBB = CmpFailMBB;
-  BuildMI(CurrentMBB, DL, SC).addReg(Old).addReg(RA).addReg(RB);
-  CurrentMBB->addSuccessor(ExitMBB);
 
   recomputeLiveIns(*LoopCmpMBB);
   recomputeLiveIns(*CmpSuccMBB);
-  recomputeLiveIns(*CmpFailMBB);
   recomputeLiveIns(*ExitMBB);
   NMBBI = MBB.end();
   MI.eraseFromParent();
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index c6257dcdf7..5e0c2d62f5 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -15588,7 +15588,7 @@ SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N,
       break;
     SDValue ConstOp = DAG.getConstant(Imm, dl, MVT::i32);
     SDValue NarrowAnd = DAG.getNode(ISD::AND, dl, MVT::i32, NarrowOp, ConstOp);
-    return DAG.getAnyExtOrTrunc(NarrowAnd, dl, N->getValueType(0));
+    return DAG.getZExtOrTrunc(NarrowAnd, dl, N->getValueType(0));
   }
   case ISD::SHL:
     return combineSHL(N, DCI);
diff --git a/llvm/lib/Target/PowerPC/PPCInstr64Bit.td b/llvm/lib/Target/PowerPC/PPCInstr64Bit.td
index fd436d4222..0322bb37b1 100644
--- a/llvm/lib/Target/PowerPC/PPCInstr64Bit.td
+++ b/llvm/lib/Target/PowerPC/PPCInstr64Bit.td
@@ -380,7 +380,7 @@ let mayStore = 1, mayLoad = 1,
     Defs = [CR0],
     Constraints = "@earlyclobber $scratch,@earlyclobber $RTp" in {
 // Atomic pseudo instructions expanded post-ra.
-def ATOMIC_SWAP_I128 : AtomicRMW128<"#ATOMIC_SWAP_I128">;
+def ATOMIC_SWAP_I128      : AtomicRMW128<"#ATOMIC_SWAP_I128">;
 def ATOMIC_LOAD_ADD_I128  : AtomicRMW128<"#ATOMIC_LOAD_ADD_I128">;
 def ATOMIC_LOAD_SUB_I128  : AtomicRMW128<"#ATOMIC_LOAD_SUB_I128">;
 def ATOMIC_LOAD_AND_I128  : AtomicRMW128<"#ATOMIC_LOAD_AND_I128">;
@@ -395,48 +395,21 @@ def ATOMIC_CMP_SWAP_I128 : PPCPostRAExpPseudo<
                               "#ATOMIC_CMP_SWAP_I128", []>;
 }
 
-def : Pat<(int_ppc_atomicrmw_add_i128 ForceXForm:$ptr,
-                                      i64:$incr_lo,
-                                      i64:$incr_hi),
-          (SPLIT_QUADWORD (ATOMIC_LOAD_ADD_I128 memrr:$ptr,
-                                                g8rc:$incr_lo,
-                                                g8rc:$incr_hi))>;
-def : Pat<(int_ppc_atomicrmw_sub_i128 ForceXForm:$ptr,
-                                      i64:$incr_lo,
-                                      i64:$incr_hi),
-          (SPLIT_QUADWORD (ATOMIC_LOAD_SUB_I128 memrr:$ptr,
-                                                g8rc:$incr_lo,
-                                                g8rc:$incr_hi))>;
-def : Pat<(int_ppc_atomicrmw_xor_i128 ForceXForm:$ptr,
-                                      i64:$incr_lo,
-                                      i64:$incr_hi),
-          (SPLIT_QUADWORD (ATOMIC_LOAD_XOR_I128 memrr:$ptr,
-                                                g8rc:$incr_lo,
-                                                g8rc:$incr_hi))>;
-def : Pat<(int_ppc_atomicrmw_and_i128 ForceXForm:$ptr,
-                                      i64:$incr_lo,
-                                      i64:$incr_hi),
-          (SPLIT_QUADWORD (ATOMIC_LOAD_AND_I128 memrr:$ptr,
-                                                g8rc:$incr_lo,
-                                                g8rc:$incr_hi))>;
-def : Pat<(int_ppc_atomicrmw_nand_i128 ForceXForm:$ptr,
-                                       i64:$incr_lo,
-                                       i64:$incr_hi),
-          (SPLIT_QUADWORD (ATOMIC_LOAD_NAND_I128 memrr:$ptr,
-                                                 g8rc:$incr_lo,
-                                                 g8rc:$incr_hi))>;
-def : Pat<(int_ppc_atomicrmw_or_i128 ForceXForm:$ptr,
-                                     i64:$incr_lo,
-                                     i64:$incr_hi),
-          (SPLIT_QUADWORD (ATOMIC_LOAD_OR_I128 memrr:$ptr,
-                                               g8rc:$incr_lo,
-                                               g8rc:$incr_hi))>;
-def : Pat<(int_ppc_atomicrmw_xchg_i128 ForceXForm:$ptr,
-                                       i64:$incr_lo,
-                                       i64:$incr_hi),
-          (SPLIT_QUADWORD (ATOMIC_SWAP_I128 memrr:$ptr,
-                                            g8rc:$incr_lo,
-                                            g8rc:$incr_hi))>;
+class PatAtomicRMWI128<SDPatternOperator OpNode, AtomicRMW128 Inst> :
+      Pat<(OpNode ForceXForm:$ptr,
+                  i64:$incr_lo,
+                  i64:$incr_hi),
+          (SPLIT_QUADWORD (Inst memrr:$ptr,
+                                g8rc:$incr_lo,
+                                g8rc:$incr_hi))>;
+
+def : PatAtomicRMWI128<int_ppc_atomicrmw_add_i128,  ATOMIC_LOAD_ADD_I128>;
+def : PatAtomicRMWI128<int_ppc_atomicrmw_sub_i128,  ATOMIC_LOAD_SUB_I128>;
+def : PatAtomicRMWI128<int_ppc_atomicrmw_xor_i128,  ATOMIC_LOAD_XOR_I128>;
+def : PatAtomicRMWI128<int_ppc_atomicrmw_and_i128,  ATOMIC_LOAD_AND_I128>;
+def : PatAtomicRMWI128<int_ppc_atomicrmw_nand_i128, ATOMIC_LOAD_NAND_I128>;
+def : PatAtomicRMWI128<int_ppc_atomicrmw_or_i128,   ATOMIC_LOAD_OR_I128>;
+def : PatAtomicRMWI128<int_ppc_atomicrmw_xchg_i128, ATOMIC_SWAP_I128>;
 def : Pat<(int_ppc_cmpxchg_i128 ForceXForm:$ptr,
                                 i64:$cmp_lo,
                                 i64:$cmp_hi,
diff --git a/llvm/lib/Target/PowerPC/PPCScheduleP10.td b/llvm/lib/Target/PowerPC/PPCScheduleP10.td
index 9bbb605d4b..f922f8a7d9 100644
--- a/llvm/lib/Target/PowerPC/PPCScheduleP10.td
+++ b/llvm/lib/Target/PowerPC/PPCScheduleP10.td
@@ -29,8 +29,8 @@ def P10Model : SchedMachineModel {
   let LoopMicroOpBufferSize = 60;
   let CompleteModel = 1;
 
-  // Do not support SPE (Signal Procesing Engine) on Power 10.
-  let UnsupportedFeatures = [HasSPE, IsE500, IsBookE, IsISAFuture];
+  // Power 10 does not support instructions from SPE, Book E and HTM.
+  let UnsupportedFeatures = [HasSPE, IsE500, IsBookE, IsISAFuture, HasHTM];
 }
 
 let SchedModel = P10Model in {
diff --git a/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp b/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
index 42f052cb15..b09975172b 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
@@ -265,8 +265,9 @@ static PPCTargetMachine::PPCABI computeTargetABI(const Triple &TT,
 
 static Reloc::Model getEffectiveRelocModel(const Triple &TT,
                                            std::optional<Reloc::Model> RM) {
-  assert((!TT.isOSAIX() || !RM || *RM == Reloc::PIC_) &&
-         "Invalid relocation model for AIX.");
+  if (TT.isOSAIX() && RM && *RM != Reloc::PIC_)
+    report_fatal_error("invalid relocation model, AIX only supports PIC",
+                       false);
 
   if (RM)
     return *RM;
diff --git a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
index d561d90d30..e5ce029449 100644
--- a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
+++ b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
@@ -558,6 +558,12 @@ DecodeStatus RISCVDisassembler::getInstruction(MCInst &MI, uint64_t &Size,
                           "XTHeadVdot custom opcode table");
     TRY_TO_DECODE_FEATURE(RISCV::FeatureVendorXSfvcp, DecoderTableXSfvcp32,
                           "SiFive VCIX custom opcode table");
+    TRY_TO_DECODE_FEATURE(
+        RISCV::FeatureVendorXSfvqmaccdod, DecoderTableXSfvqmaccdod32,
+        "SiFive Matrix Multiplication (2x8 and 8x2) Instruction opcode table");
+    TRY_TO_DECODE_FEATURE(
+        RISCV::FeatureVendorXSfvqmaccqoq, DecoderTableXSfvqmaccqoq32,
+        "SiFive Matrix Multiplication (4x8 and 8x4) Instruction opcode table");
     TRY_TO_DECODE_FEATURE(RISCV::FeatureVendorXSfcie, DecoderTableXSfcie32,
                           "Sifive CIE custom opcode table");
     TRY_TO_DECODE_FEATURE(RISCV::FeatureVendorXCVbitmanip,
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVCallLowering.cpp b/llvm/lib/Target/RISCV/GISel/RISCVCallLowering.cpp
index a362a70932..b605f2f621 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVCallLowering.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVCallLowering.cpp
@@ -17,6 +17,7 @@
 #include "RISCVSubtarget.h"
 #include "llvm/CodeGen/Analysis.h"
 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
 
 using namespace llvm;
 
@@ -56,19 +57,38 @@ struct RISCVOutgoingValueAssigner : public CallLowering::OutgoingValueAssigner {
 struct RISCVOutgoingValueHandler : public CallLowering::OutgoingValueHandler {
   RISCVOutgoingValueHandler(MachineIRBuilder &B, MachineRegisterInfo &MRI,
                             MachineInstrBuilder MIB)
-      : OutgoingValueHandler(B, MRI), MIB(MIB) {}
-
-  MachineInstrBuilder MIB;
-
+      : OutgoingValueHandler(B, MRI), MIB(MIB),
+        Subtarget(MIRBuilder.getMF().getSubtarget<RISCVSubtarget>()) {}
   Register getStackAddress(uint64_t MemSize, int64_t Offset,
                            MachinePointerInfo &MPO,
                            ISD::ArgFlagsTy Flags) override {
-    llvm_unreachable("not implemented");
+    MachineFunction &MF = MIRBuilder.getMF();
+    LLT p0 = LLT::pointer(0, Subtarget.getXLen());
+    LLT sXLen = LLT::scalar(Subtarget.getXLen());
+
+    if (!SPReg)
+      SPReg = MIRBuilder.buildCopy(p0, Register(RISCV::X2)).getReg(0);
+
+    auto OffsetReg = MIRBuilder.buildConstant(sXLen, Offset);
+
+    auto AddrReg = MIRBuilder.buildPtrAdd(p0, SPReg, OffsetReg);
+
+    MPO = MachinePointerInfo::getStack(MF, Offset);
+    return AddrReg.getReg(0);
   }
 
   void assignValueToAddress(Register ValVReg, Register Addr, LLT MemTy,
                             MachinePointerInfo &MPO, CCValAssign &VA) override {
-    llvm_unreachable("not implemented");
+    MachineFunction &MF = MIRBuilder.getMF();
+    uint64_t LocMemOffset = VA.getLocMemOffset();
+
+    // TODO: Move StackAlignment to subtarget and share with FrameLowering.
+    auto MMO =
+        MF.getMachineMemOperand(MPO, MachineMemOperand::MOStore, MemTy,
+                                commonAlignment(Align(16), LocMemOffset));
+
+    Register ExtReg = extendRegister(ValVReg, VA);
+    MIRBuilder.buildStore(ExtReg, Addr, *MMO);
   }
 
   void assignValueToReg(Register ValVReg, Register PhysReg,
@@ -77,6 +97,14 @@ struct RISCVOutgoingValueHandler : public CallLowering::OutgoingValueHandler {
     MIRBuilder.buildCopy(PhysReg, ExtReg);
     MIB.addUse(PhysReg, RegState::Implicit);
   }
+
+private:
+  MachineInstrBuilder MIB;
+
+  // Cache the SP register vreg if we need it more than once in this call site.
+  Register SPReg;
+
+  const RISCVSubtarget &Subtarget;
 };
 
 struct RISCVIncomingValueAssigner : public CallLowering::IncomingValueAssigner {
@@ -112,24 +140,50 @@ struct RISCVIncomingValueAssigner : public CallLowering::IncomingValueAssigner {
 
 struct RISCVIncomingValueHandler : public CallLowering::IncomingValueHandler {
   RISCVIncomingValueHandler(MachineIRBuilder &B, MachineRegisterInfo &MRI)
-      : IncomingValueHandler(B, MRI) {}
+      : IncomingValueHandler(B, MRI),
+        Subtarget(MIRBuilder.getMF().getSubtarget<RISCVSubtarget>()) {}
 
   Register getStackAddress(uint64_t MemSize, int64_t Offset,
                            MachinePointerInfo &MPO,
                            ISD::ArgFlagsTy Flags) override {
-    llvm_unreachable("not implemented");
+    MachineFrameInfo &MFI = MIRBuilder.getMF().getFrameInfo();
+
+    int FI = MFI.CreateFixedObject(MemSize, Offset, /*Immutable=*/true);
+    MPO = MachinePointerInfo::getFixedStack(MIRBuilder.getMF(), FI);
+    return MIRBuilder.buildFrameIndex(LLT::pointer(0, Subtarget.getXLen()), FI)
+        .getReg(0);
   }
 
   void assignValueToAddress(Register ValVReg, Register Addr, LLT MemTy,
                             MachinePointerInfo &MPO, CCValAssign &VA) override {
-    llvm_unreachable("not implemented");
+    MachineFunction &MF = MIRBuilder.getMF();
+    auto MMO = MF.getMachineMemOperand(MPO, MachineMemOperand::MOLoad, MemTy,
+                                       inferAlignFromPtrInfo(MF, MPO));
+    MIRBuilder.buildLoad(ValVReg, Addr, *MMO);
   }
 
   void assignValueToReg(Register ValVReg, Register PhysReg,
                         CCValAssign VA) override {
-    // Copy argument received in physical register to desired VReg.
+    markPhysRegUsed(PhysReg);
+    IncomingValueHandler::assignValueToReg(ValVReg, PhysReg, VA);
+  }
+
+  /// How the physical register gets marked varies between formal
+  /// parameters (it's a basic-block live-in), and a call instruction
+  /// (it's an implicit-def of the BL).
+  virtual void markPhysRegUsed(MCRegister PhysReg) = 0;
+
+private:
+  const RISCVSubtarget &Subtarget;
+};
+
+struct RISCVFormalArgHandler : public RISCVIncomingValueHandler {
+  RISCVFormalArgHandler(MachineIRBuilder &B, MachineRegisterInfo &MRI)
+      : RISCVIncomingValueHandler(B, MRI) {}
+
+  void markPhysRegUsed(MCRegister PhysReg) override {
+    MIRBuilder.getMRI()->addLiveIn(PhysReg);
     MIRBuilder.getMBB().addLiveIn(PhysReg);
-    MIRBuilder.buildCopy(ValVReg, PhysReg);
   }
 };
 
@@ -138,14 +192,11 @@ struct RISCVCallReturnHandler : public RISCVIncomingValueHandler {
                          MachineInstrBuilder &MIB)
       : RISCVIncomingValueHandler(B, MRI), MIB(MIB) {}
 
-  MachineInstrBuilder MIB;
-
-  void assignValueToReg(Register ValVReg, Register PhysReg,
-                        CCValAssign VA) override {
-    // Copy argument received in physical register to desired VReg.
+  void markPhysRegUsed(MCRegister PhysReg) override {
     MIB.addDef(PhysReg, RegState::Implicit);
-    MIRBuilder.buildCopy(ValVReg, PhysReg);
   }
+
+  MachineInstrBuilder MIB;
 };
 
 } // namespace
@@ -153,6 +204,40 @@ struct RISCVCallReturnHandler : public RISCVIncomingValueHandler {
 RISCVCallLowering::RISCVCallLowering(const RISCVTargetLowering &TLI)
     : CallLowering(&TLI) {}
 
+// TODO: Support all argument types.
+static bool isSupportedArgumentType(Type *T, const RISCVSubtarget &Subtarget) {
+  // TODO: Integers larger than 2*XLen are passed indirectly which is not
+  // supported yet.
+  if (T->isIntegerTy())
+    return T->getIntegerBitWidth() <= Subtarget.getXLen() * 2;
+  if (T->isPointerTy())
+    return true;
+  return false;
+}
+
+// TODO: Only integer, pointer and aggregate types are supported now.
+static bool isSupportedReturnType(Type *T, const RISCVSubtarget &Subtarget) {
+  // TODO: Integers larger than 2*XLen are passed indirectly which is not
+  // supported yet.
+  if (T->isIntegerTy())
+    return T->getIntegerBitWidth() <= Subtarget.getXLen() * 2;
+  if (T->isPointerTy())
+    return true;
+
+  if (T->isArrayTy())
+    return isSupportedReturnType(T->getArrayElementType(), Subtarget);
+
+  if (T->isStructTy()) {
+    auto StructT = cast<StructType>(T);
+    for (unsigned i = 0, e = StructT->getNumElements(); i != e; ++i)
+      if (!isSupportedReturnType(StructT->getElementType(i), Subtarget))
+        return false;
+    return true;
+  }
+
+  return false;
+}
+
 bool RISCVCallLowering::lowerReturnVal(MachineIRBuilder &MIRBuilder,
                                        const Value *Val,
                                        ArrayRef<Register> VRegs,
@@ -160,8 +245,9 @@ bool RISCVCallLowering::lowerReturnVal(MachineIRBuilder &MIRBuilder,
   if (!Val)
     return true;
 
-  // TODO: Only integer, pointer and aggregate types are supported now.
-  if (!Val->getType()->isIntOrPtrTy() && !Val->getType()->isAggregateType())
+  const RISCVSubtarget &Subtarget =
+      MIRBuilder.getMF().getSubtarget<RISCVSubtarget>();
+  if (!isSupportedReturnType(Val->getType(), Subtarget))
     return false;
 
   MachineFunction &MF = MIRBuilder.getMF();
@@ -208,13 +294,11 @@ bool RISCVCallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder,
   if (F.isVarArg())
     return false;
 
-  // TODO: Support all argument types.
+  const RISCVSubtarget &Subtarget =
+      MIRBuilder.getMF().getSubtarget<RISCVSubtarget>();
   for (auto &Arg : F.args()) {
-    if (Arg.getType()->isIntegerTy())
-      continue;
-    if (Arg.getType()->isPointerTy())
-      continue;
-    return false;
+    if (!isSupportedArgumentType(Arg.getType(), Subtarget))
+      return false;
   }
 
   MachineFunction &MF = MIRBuilder.getMF();
@@ -239,7 +323,7 @@ bool RISCVCallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder,
   RISCVIncomingValueAssigner Assigner(
       CC == CallingConv::Fast ? RISCV::CC_RISCV_FastCC : RISCV::CC_RISCV,
       /*IsRet=*/false);
-  RISCVIncomingValueHandler Handler(MIRBuilder, MF.getRegInfo());
+  RISCVFormalArgHandler Handler(MIRBuilder, MF.getRegInfo());
 
   return determineAndHandleAssignments(Handler, Assigner, SplitArgInfos,
                                        MIRBuilder, CC, F.isVarArg());
@@ -252,15 +336,11 @@ bool RISCVCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
   const Function &F = MF.getFunction();
   CallingConv::ID CC = F.getCallingConv();
 
-  // TODO: Support all argument types.
+  const RISCVSubtarget &Subtarget =
+      MIRBuilder.getMF().getSubtarget<RISCVSubtarget>();
   for (auto &AInfo : Info.OrigArgs) {
-    if (AInfo.Ty->isIntegerTy())
-      continue;
-    if (AInfo.Ty->isPointerTy())
-      continue;
-    if (AInfo.Ty->isFloatingPointTy())
-      continue;
-    return false;
+    if (!isSupportedArgumentType(AInfo.Ty, Subtarget))
+      return false;
   }
 
   SmallVector<ArgInfo, 32> SplitArgInfos;
@@ -297,8 +377,7 @@ bool RISCVCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
   if (Info.OrigRet.Ty->isVoidTy())
     return true;
 
-  // TODO: Only integer, pointer and aggregate types are supported now.
-  if (!Info.OrigRet.Ty->isIntOrPtrTy() && !Info.OrigRet.Ty->isAggregateType())
+  if (!isSupportedReturnType(Info.OrigRet.Ty, Subtarget))
     return false;
 
   SmallVector<ArgInfo, 4> SplitRetInfos;
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
index 59c95f9c74..ab2125a361 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
@@ -18,6 +18,7 @@
 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
+#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
 #include "llvm/IR/IntrinsicsRISCV.h"
 #include "llvm/Support/Debug.h"
@@ -65,17 +66,22 @@ class RISCVInstructionSelector : public InstructionSelector {
   bool selectSelect(MachineInstr &MI, MachineIRBuilder &MIB,
                     MachineRegisterInfo &MRI) const;
 
-  bool earlySelectShift(unsigned Opc, MachineInstr &I, MachineIRBuilder &MIB,
-                        const MachineRegisterInfo &MRI);
-
   ComplexRendererFns selectShiftMask(MachineOperand &Root) const;
   ComplexRendererFns selectAddrRegImm(MachineOperand &Root) const;
 
+  ComplexRendererFns selectSHXADDOp(MachineOperand &Root, unsigned ShAmt) const;
+  template <unsigned ShAmt>
+  ComplexRendererFns selectSHXADDOp(MachineOperand &Root) const {
+    return selectSHXADDOp(Root, ShAmt);
+  }
+
   // Custom renderers for tablegen
   void renderNegImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
                     int OpIdx) const;
   void renderImmPlus1(MachineInstrBuilder &MIB, const MachineInstr &MI,
                       int OpIdx) const;
+  void renderImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
+                 int OpIdx) const;
 
   const RISCVSubtarget &STI;
   const RISCVInstrInfo &TII;
@@ -123,6 +129,108 @@ RISCVInstructionSelector::selectShiftMask(MachineOperand &Root) const {
   return {{[=](MachineInstrBuilder &MIB) { MIB.add(Root); }}};
 }
 
+InstructionSelector::ComplexRendererFns
+RISCVInstructionSelector::selectSHXADDOp(MachineOperand &Root,
+                                         unsigned ShAmt) const {
+  using namespace llvm::MIPatternMatch;
+  MachineFunction &MF = *Root.getParent()->getParent()->getParent();
+  MachineRegisterInfo &MRI = MF.getRegInfo();
+
+  if (!Root.isReg())
+    return std::nullopt;
+  Register RootReg = Root.getReg();
+
+  const unsigned XLen = STI.getXLen();
+  APInt Mask, C2;
+  Register RegY;
+  std::optional<bool> LeftShift;
+  // (and (shl y, c2), mask)
+  if (mi_match(RootReg, MRI,
+               m_GAnd(m_GShl(m_Reg(RegY), m_ICst(C2)), m_ICst(Mask))))
+    LeftShift = true;
+  // (and (lshr y, c2), mask)
+  else if (mi_match(RootReg, MRI,
+                    m_GAnd(m_GLShr(m_Reg(RegY), m_ICst(C2)), m_ICst(Mask))))
+    LeftShift = false;
+
+  if (LeftShift.has_value()) {
+    if (*LeftShift)
+      Mask &= maskTrailingZeros<uint64_t>(C2.getLimitedValue());
+    else
+      Mask &= maskTrailingOnes<uint64_t>(XLen - C2.getLimitedValue());
+
+    if (Mask.isShiftedMask()) {
+      unsigned Leading = XLen - Mask.getActiveBits();
+      unsigned Trailing = Mask.countr_zero();
+      // Given (and (shl y, c2), mask) in which mask has no leading zeros and
+      // c3 trailing zeros. We can use an SRLI by c3 - c2 followed by a SHXADD.
+      if (*LeftShift && Leading == 0 && C2.ult(Trailing) && Trailing == ShAmt) {
+        Register DstReg =
+            MRI.createGenericVirtualRegister(MRI.getType(RootReg));
+        return {{[=](MachineInstrBuilder &MIB) {
+          MachineIRBuilder(*MIB.getInstr())
+              .buildInstr(RISCV::SRLI, {DstReg}, {RegY})
+              .addImm(Trailing - C2.getLimitedValue());
+          MIB.addReg(DstReg);
+        }}};
+      }
+
+      // Given (and (lshr y, c2), mask) in which mask has c2 leading zeros and
+      // c3 trailing zeros. We can use an SRLI by c2 + c3 followed by a SHXADD.
+      if (!*LeftShift && Leading == C2 && Trailing == ShAmt) {
+        Register DstReg =
+            MRI.createGenericVirtualRegister(MRI.getType(RootReg));
+        return {{[=](MachineInstrBuilder &MIB) {
+          MachineIRBuilder(*MIB.getInstr())
+              .buildInstr(RISCV::SRLI, {DstReg}, {RegY})
+              .addImm(Leading + Trailing);
+          MIB.addReg(DstReg);
+        }}};
+      }
+    }
+  }
+
+  LeftShift.reset();
+
+  // (shl (and y, mask), c2)
+  if (mi_match(RootReg, MRI,
+               m_GShl(m_OneNonDBGUse(m_GAnd(m_Reg(RegY), m_ICst(Mask))),
+                      m_ICst(C2))))
+    LeftShift = true;
+  // (lshr (and y, mask), c2)
+  else if (mi_match(RootReg, MRI,
+                    m_GLShr(m_OneNonDBGUse(m_GAnd(m_Reg(RegY), m_ICst(Mask))),
+                            m_ICst(C2))))
+    LeftShift = false;
+
+  if (LeftShift.has_value() && Mask.isShiftedMask()) {
+    unsigned Leading = XLen - Mask.getActiveBits();
+    unsigned Trailing = Mask.countr_zero();
+
+    // Given (shl (and y, mask), c2) in which mask has 32 leading zeros and
+    // c3 trailing zeros. If c1 + c3 == ShAmt, we can emit SRLIW + SHXADD.
+    bool Cond = *LeftShift && Leading == 32 && Trailing > 0 &&
+                (Trailing + C2.getLimitedValue()) == ShAmt;
+    if (!Cond)
+      // Given (lshr (and y, mask), c2) in which mask has 32 leading zeros and
+      // c3 trailing zeros. If c3 - c1 == ShAmt, we can emit SRLIW + SHXADD.
+      Cond = !*LeftShift && Leading == 32 && C2.ult(Trailing) &&
+             (Trailing - C2.getLimitedValue()) == ShAmt;
+
+    if (Cond) {
+      Register DstReg = MRI.createGenericVirtualRegister(MRI.getType(RootReg));
+      return {{[=](MachineInstrBuilder &MIB) {
+        MachineIRBuilder(*MIB.getInstr())
+            .buildInstr(RISCV::SRLIW, {DstReg}, {RegY})
+            .addImm(Trailing);
+        MIB.addReg(DstReg);
+      }}};
+    }
+  }
+
+  return std::nullopt;
+}
+
 InstructionSelector::ComplexRendererFns
 RISCVInstructionSelector::selectAddrRegImm(MachineOperand &Root) const {
   // TODO: Need to get the immediate from a G_PTR_ADD. Should this be done in
@@ -131,30 +239,6 @@ RISCVInstructionSelector::selectAddrRegImm(MachineOperand &Root) const {
            [=](MachineInstrBuilder &MIB) { MIB.addImm(0); }}};
 }
 
-// Tablegen doesn't allow us to write SRLIW/SRAIW/SLLIW patterns because the
-// immediate Operand has type XLenVT. GlobalISel wants it to be i32.
-bool RISCVInstructionSelector::earlySelectShift(
-    unsigned Opc, MachineInstr &I, MachineIRBuilder &MIB,
-    const MachineRegisterInfo &MRI) {
-  if (!Subtarget->is64Bit())
-    return false;
-
-  LLT Ty = MRI.getType(I.getOperand(0).getReg());
-  if (!Ty.isScalar() || Ty.getSizeInBits() != 32)
-    return false;
-
-  std::optional<int64_t> CstVal =
-      getIConstantVRegSExtVal(I.getOperand(2).getReg(), MRI);
-  if (!CstVal || !isUInt<5>(*CstVal))
-    return false;
-
-  auto NewI = MIB.buildInstr(Opc, {I.getOperand(0).getReg()},
-                             {I.getOperand(1).getReg()})
-                  .addImm(*CstVal);
-  I.eraseFromParent();
-  return constrainSelectedInstRegOperands(*NewI, TII, TRI, RBI);
-}
-
 bool RISCVInstructionSelector::select(MachineInstr &MI) {
   MachineBasicBlock &MBB = *MI.getParent();
   MachineFunction &MF = *MBB.getParent();
@@ -199,61 +283,13 @@ bool RISCVInstructionSelector::select(MachineInstr &MI) {
     return true;
   }
 
-  switch (Opc) {
-  case TargetOpcode::G_ADD: {
-    // Tablegen doesn't pick up the ADDIW pattern because i32 isn't a legal
-    // type for RV64 in SelectionDAG. Manually select it here.
-    LLT Ty = MRI.getType(MI.getOperand(0).getReg());
-    if (Subtarget->is64Bit() && Ty.isScalar() && Ty.getSizeInBits() == 32) {
-      std::optional<int64_t> CstVal =
-          getIConstantVRegSExtVal(MI.getOperand(2).getReg(), MRI);
-      if (CstVal && isInt<12>(*CstVal)) {
-        auto NewI = MIB.buildInstr(RISCV::ADDIW, {MI.getOperand(0).getReg()},
-                                   {MI.getOperand(1).getReg()})
-                        .addImm(*CstVal);
-        MI.eraseFromParent();
-        return constrainSelectedInstRegOperands(*NewI, TII, TRI, RBI);
-      }
-    }
-    break;
-  }
-  case TargetOpcode::G_SUB: {
-    // Tablegen doesn't pick up the ADDIW pattern because i32 isn't a legal
-    // type for RV64 in SelectionDAG. Manually select it here.
-    LLT Ty = MRI.getType(MI.getOperand(0).getReg());
-    if (Subtarget->is64Bit() && Ty.isScalar() && Ty.getSizeInBits() == 32) {
-      std::optional<int64_t> CstVal =
-          getIConstantVRegSExtVal(MI.getOperand(2).getReg(), MRI);
-      if (CstVal && ((isInt<12>(*CstVal) && *CstVal != -2048) || *CstVal == 2048)) {
-        auto NewI = MIB.buildInstr(RISCV::ADDIW, {MI.getOperand(0).getReg()},
-                                   {MI.getOperand(1).getReg()})
-                        .addImm(-*CstVal);
-        MI.eraseFromParent();
-        return constrainSelectedInstRegOperands(*NewI, TII, TRI, RBI);
-      }
-    }
-    break;
-  }
-  case TargetOpcode::G_ASHR:
-    if (earlySelectShift(RISCV::SRAIW, MI, MIB, MRI))
-      return true;
-    break;
-  case TargetOpcode::G_LSHR:
-    if (earlySelectShift(RISCV::SRLIW, MI, MIB, MRI))
-      return true;
-    break;
-  case TargetOpcode::G_SHL:
-    if (earlySelectShift(RISCV::SLLIW, MI, MIB, MRI))
-      return true;
-    break;
-  }
-
   if (selectImpl(MI, *CoverageInfo))
     return true;
 
   switch (Opc) {
   case TargetOpcode::G_ANYEXT:
   case TargetOpcode::G_PTRTOINT:
+  case TargetOpcode::G_INTTOPTR:
   case TargetOpcode::G_TRUNC:
     return selectCopy(MI, MRI);
   case TargetOpcode::G_CONSTANT:
@@ -268,6 +304,14 @@ bool RISCVInstructionSelector::select(MachineInstr &MI) {
   }
   case TargetOpcode::G_SEXT_INREG:
     return selectSExtInreg(MI, MIB);
+  case TargetOpcode::G_FRAME_INDEX: {
+    // TODO: We may want to replace this code with the SelectionDAG patterns,
+    // which fail to get imported because it uses FrameAddrRegImm, which is a
+    // ComplexPattern
+    MI.setDesc(TII.get(RISCV::ADDI));
+    MI.addOperand(MachineOperand::CreateImm(0));
+    return constrainSelectedInstRegOperands(MI, TII, TRI, RBI);
+  }
   case TargetOpcode::G_SELECT:
     return selectSelect(MI, MIB, MRI);
   default:
@@ -284,7 +328,6 @@ bool RISCVInstructionSelector::replacePtrWithInt(MachineOperand &Op,
   const LLT XLenLLT = LLT::scalar(STI.getXLen());
   auto PtrToInt = MIB.buildPtrToInt(XLenLLT, PtrReg);
   MRI.setRegBank(PtrToInt.getReg(0), RBI.getRegBank(RISCV::GPRRegBankID));
-  MRI.setType(PtrReg, XLenLLT);
   Op.setReg(PtrToInt.getReg(0));
   return select(*PtrToInt);
 }
@@ -323,6 +366,15 @@ void RISCVInstructionSelector::renderImmPlus1(MachineInstrBuilder &MIB,
   MIB.addImm(CstVal + 1);
 }
 
+void RISCVInstructionSelector::renderImm(MachineInstrBuilder &MIB,
+                                         const MachineInstr &MI,
+                                         int OpIdx) const {
+  assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
+         "Expected G_CONSTANT");
+  int64_t CstVal = MI.getOperand(1).getCImm()->getSExtValue();
+  MIB.addImm(CstVal);
+}
+
 const TargetRegisterClass *RISCVInstructionSelector::getRegClassForTypeOnBank(
     LLT Ty, const RegisterBank &RB) const {
   if (RB.getID() == RISCV::GPRRegBankID) {
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
index 4479bccfd4..3aae38a7d1 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
@@ -122,6 +122,14 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST) {
   getActionDefinitionsBuilder(G_PTR_ADD)
       .legalFor({{p0, XLenLLT}});
 
+  getActionDefinitionsBuilder(G_PTRTOINT)
+      .legalFor({{XLenLLT, p0}})
+      .clampScalar(0, XLenLLT, XLenLLT);
+
+  getActionDefinitionsBuilder(G_INTTOPTR)
+      .legalFor({{p0, XLenLLT}})
+      .clampScalar(1, XLenLLT, XLenLLT);
+
   getActionDefinitionsBuilder(G_BRCOND)
       .legalFor({XLenLLT})
       .minScalar(0, XLenLLT);
@@ -145,6 +153,10 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST) {
         .legalFor({XLenLLT})
         .lower();
     // clang-format on
+
+    getActionDefinitionsBuilder({G_SMULO, G_UMULO})
+        .minScalar(0, XLenLLT)
+        .lower();
   } else {
     getActionDefinitionsBuilder(G_MUL)
         .libcallFor({XLenLLT, DoubleXLenLLT})
@@ -152,6 +164,20 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST) {
         .clampScalar(0, XLenLLT, DoubleXLenLLT);
 
     getActionDefinitionsBuilder({G_SMULH, G_UMULH}).lowerFor({XLenLLT});
+
+    getActionDefinitionsBuilder({G_SMULO, G_UMULO})
+        .minScalar(0, XLenLLT)
+        // Widen XLenLLT to DoubleXLenLLT so we can use a single libcall to get
+        // the low bits for the mul result and high bits to do the overflow
+        // check.
+        .widenScalarIf(
+            [=](const LegalityQuery &Query) {
+              return Query.Types[0] == XLenLLT;
+            },
+            [=](const LegalityQuery &Query) {
+              return std::make_pair(0, DoubleXLenLLT);
+            })
+        .lower();
   }
 
   if (ST.hasStdExtM()) {
@@ -168,6 +194,7 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST) {
   }
 
   getActionDefinitionsBuilder(G_ABS).lower();
+  getActionDefinitionsBuilder({G_UMAX, G_UMIN, G_SMAX, G_SMIN}).lower();
 
   getActionDefinitionsBuilder(G_FRAME_INDEX).legalFor({p0});
 
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp
index e2cf461a1e..4ede55fc8c 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp
@@ -126,6 +126,8 @@ RISCVRegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
   case TargetOpcode::G_UREM:
   case TargetOpcode::G_UMULH:
   case TargetOpcode::G_PTR_ADD:
+  case TargetOpcode::G_PTRTOINT:
+  case TargetOpcode::G_INTTOPTR:
   case TargetOpcode::G_TRUNC:
   case TargetOpcode::G_ANYEXT:
   case TargetOpcode::G_SEXT:
@@ -135,6 +137,7 @@ RISCVRegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
   case TargetOpcode::G_STORE:
     break;
   case TargetOpcode::G_CONSTANT:
+  case TargetOpcode::G_FRAME_INDEX:
   case TargetOpcode::G_GLOBAL_VALUE:
   case TargetOpcode::G_BRCOND:
     OperandsMapping = getOperandsMapping({GPRValueMapping, nullptr});
diff --git a/llvm/lib/Target/RISCV/MCA/RISCVCustomBehaviour.cpp b/llvm/lib/Target/RISCV/MCA/RISCVCustomBehaviour.cpp
index 16f9c113e2..1bb1297026 100644
--- a/llvm/lib/Target/RISCV/MCA/RISCVCustomBehaviour.cpp
+++ b/llvm/lib/Target/RISCV/MCA/RISCVCustomBehaviour.cpp
@@ -185,13 +185,45 @@ RISCVInstrumentManager::createInstruments(const MCInst &Inst) {
   return SmallVector<UniqueInstrument>();
 }
 
+static std::pair<uint8_t, uint8_t>
+getEEWAndEMULForUnitStrideLoadStore(unsigned Opcode, RISCVII::VLMUL LMUL,
+                                    uint8_t SEW) {
+  uint8_t EEW;
+  switch (Opcode) {
+  case RISCV::VLM_V:
+  case RISCV::VSM_V:
+  case RISCV::VLE8_V:
+  case RISCV::VSE8_V:
+    EEW = 8;
+    break;
+  case RISCV::VLE16_V:
+  case RISCV::VSE16_V:
+    EEW = 16;
+    break;
+  case RISCV::VLE32_V:
+  case RISCV::VSE32_V:
+    EEW = 32;
+    break;
+  case RISCV::VLE64_V:
+  case RISCV::VSE64_V:
+    EEW = 64;
+    break;
+  default:
+    llvm_unreachable("Opcode is not a vector unit stride load nor store");
+  }
+
+  uint8_t EMUL =
+      static_cast<uint8_t>(RISCVVType::getSameRatioLMUL(SEW, LMUL, EEW));
+  return std::make_pair(EEW, EMUL);
+}
+
 unsigned RISCVInstrumentManager::getSchedClassID(
     const MCInstrInfo &MCII, const MCInst &MCI,
     const llvm::SmallVector<Instrument *> &IVec) const {
   unsigned short Opcode = MCI.getOpcode();
   unsigned SchedClassID = MCII.get(Opcode).getSchedClass();
 
-  // Unpack all possible RISCV instruments from IVec.
+  // Unpack all possible RISC-V instruments from IVec.
   RISCVLMULInstrument *LI = nullptr;
   RISCVSEWInstrument *SI = nullptr;
   for (auto &I : IVec) {
@@ -214,12 +246,23 @@ unsigned RISCVInstrumentManager::getSchedClassID(
   // or (Opcode, LMUL, SEW) if SEW instrument is active, and depends on LMUL
   // and SEW, or (Opcode, LMUL, 0) if does not depend on SEW.
   uint8_t SEW = SI ? SI->getSEW() : 0;
-  // Check if it depends on LMUL and SEW
-  const RISCVVInversePseudosTable::PseudoInfo *RVV =
-      RISCVVInversePseudosTable::getBaseInfo(Opcode, LMUL, SEW);
-  // Check if it depends only on LMUL
-  if (!RVV)
-    RVV = RISCVVInversePseudosTable::getBaseInfo(Opcode, LMUL, 0);
+
+  const RISCVVInversePseudosTable::PseudoInfo *RVV = nullptr;
+  if (Opcode == RISCV::VLM_V || Opcode == RISCV::VSM_V ||
+      Opcode == RISCV::VLE8_V || Opcode == RISCV::VSE8_V ||
+      Opcode == RISCV::VLE16_V || Opcode == RISCV::VSE16_V ||
+      Opcode == RISCV::VLE32_V || Opcode == RISCV::VSE32_V ||
+      Opcode == RISCV::VLE64_V || Opcode == RISCV::VSE64_V) {
+    RISCVII::VLMUL VLMUL = static_cast<RISCVII::VLMUL>(LMUL);
+    auto [EEW, EMUL] = getEEWAndEMULForUnitStrideLoadStore(Opcode, VLMUL, SEW);
+    RVV = RISCVVInversePseudosTable::getBaseInfo(Opcode, EMUL, EEW);
+  } else {
+    // Check if it depends on LMUL and SEW
+    RVV = RISCVVInversePseudosTable::getBaseInfo(Opcode, LMUL, SEW);
+    // Check if it depends only on LMUL
+    if (!RVV)
+      RVV = RISCVVInversePseudosTable::getBaseInfo(Opcode, LMUL, 0);
+  }
 
   // Not a RVV instr
   if (!RVV) {
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
index ca5aeb943c..731c644198 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
@@ -1,4 +1,4 @@
-//===-- RISCVAsmBackend.cpp - RISCV Assembler Backend ---------------------===//
+//===-- RISCVAsmBackend.cpp - RISC-V Assembler Backend --------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -238,7 +238,7 @@ bool RISCVAsmBackend::relaxDwarfLineAddr(MCDwarfLineAddrFragment &DF,
     OS << uint8_t(dwarf::DW_LNS_fixed_advance_pc);
     Offset = OS.tell();
     Fixup = RISCV::getRelocPairForSize(2);
-    support::endian::write<uint16_t>(OS, 0, support::little);
+    support::endian::write<uint16_t>(OS, 0, llvm::endianness::little);
   }
 
   const MCBinaryExpr &MBE = cast<MCBinaryExpr>(AddrDelta);
@@ -303,15 +303,15 @@ bool RISCVAsmBackend::relaxDwarfCFA(MCDwarfCallFrameFragment &DF,
     AddFixups(0, {ELF::R_RISCV_SET6, ELF::R_RISCV_SUB6});
   } else if (isUInt<8>(Value)) {
     OS << uint8_t(dwarf::DW_CFA_advance_loc1);
-    support::endian::write<uint8_t>(OS, 0, support::little);
+    support::endian::write<uint8_t>(OS, 0, llvm::endianness::little);
     AddFixups(1, {ELF::R_RISCV_SET8, ELF::R_RISCV_SUB8});
   } else if (isUInt<16>(Value)) {
     OS << uint8_t(dwarf::DW_CFA_advance_loc2);
-    support::endian::write<uint16_t>(OS, 0, support::little);
+    support::endian::write<uint16_t>(OS, 0, llvm::endianness::little);
     AddFixups(1, {ELF::R_RISCV_SET16, ELF::R_RISCV_SUB16});
   } else if (isUInt<32>(Value)) {
     OS << uint8_t(dwarf::DW_CFA_advance_loc4);
-    support::endian::write<uint32_t>(OS, 0, support::little);
+    support::endian::write<uint32_t>(OS, 0, llvm::endianness::little);
     AddFixups(1, {ELF::R_RISCV_SET32, ELF::R_RISCV_SUB32});
   } else {
     llvm_unreachable("unsupported CFA encoding");
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h
index 0ea1f32e82..95596ad594 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h
@@ -31,8 +31,8 @@ class RISCVAsmBackend : public MCAsmBackend {
 public:
   RISCVAsmBackend(const MCSubtargetInfo &STI, uint8_t OSABI, bool Is64Bit,
                   const MCTargetOptions &Options)
-      : MCAsmBackend(support::little, RISCV::fixup_riscv_relax), STI(STI),
-        OSABI(OSABI), Is64Bit(Is64Bit), TargetOptions(Options) {
+      : MCAsmBackend(llvm::endianness::little, RISCV::fixup_riscv_relax),
+        STI(STI), OSABI(OSABI), Is64Bit(Is64Bit), TargetOptions(Options) {
     RISCVFeatures::validate(STI.getTargetTriple(), STI.getFeatureBits());
   }
   ~RISCVAsmBackend() override = default;
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp
index d71efc11e6..7919189d19 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp
@@ -206,6 +206,15 @@ unsigned RISCVVType::getSEWLMULRatio(unsigned SEW, RISCVII::VLMUL VLMul) {
   return (SEW * 8) / LMul;
 }
 
+RISCVII::VLMUL RISCVVType::getSameRatioLMUL(unsigned SEW, RISCVII::VLMUL VLMUL,
+                                            unsigned EEW) {
+  unsigned Ratio = RISCVVType::getSEWLMULRatio(SEW, VLMUL);
+  unsigned EMULFixedPoint = (EEW * 8) / Ratio;
+  bool Fractional = EMULFixedPoint < 8;
+  unsigned EMUL = Fractional ? 8 / EMULFixedPoint : EMULFixedPoint / 8;
+  return RISCVVType::encodeLMUL(EMUL, Fractional);
+}
+
 // Include the auto-generated portion of the compress emitter.
 #define GEN_UNCOMPRESS_INSTR
 #define GEN_COMPRESS_INSTR
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
index 20ff26a39d..e7181eadd4 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
@@ -535,6 +535,8 @@ void printVType(unsigned VType, raw_ostream &OS);
 
 unsigned getSEWLMULRatio(unsigned SEW, RISCVII::VLMUL VLMul);
 
+RISCVII::VLMUL getSameRatioLMUL(unsigned SEW, RISCVII::VLMUL VLMUL,
+                                unsigned EEW);
 } // namespace RISCVVType
 
 namespace RISCVRVC {
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVFixupKinds.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVFixupKinds.h
index f3d0841eb6..74bd9398a9 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVFixupKinds.h
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVFixupKinds.h
@@ -1,4 +1,4 @@
-//===-- RISCVFixupKinds.h - RISCV Specific Fixup Entries --------*- C++ -*-===//
+//===-- RISCVFixupKinds.h - RISC-V Specific Fixup Entries -------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
index 716c3ac14d..c5f2d92e9e 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
@@ -137,7 +137,7 @@ void RISCVMCCodeEmitter::expandFunctionCall(const MCInst &MI,
   // Emit AUIPC Ra, Func with R_RISCV_CALL relocation type.
   TmpInst = MCInstBuilder(RISCV::AUIPC).addReg(Ra).addExpr(CallExpr);
   Binary = getBinaryCodeForInstr(TmpInst, Fixups, STI);
-  support::endian::write(CB, Binary, support::little);
+  support::endian::write(CB, Binary, llvm::endianness::little);
 
   if (MI.getOpcode() == RISCV::PseudoTAIL ||
       MI.getOpcode() == RISCV::PseudoJump)
@@ -147,7 +147,7 @@ void RISCVMCCodeEmitter::expandFunctionCall(const MCInst &MI,
     // Emit JALR Ra, Ra, 0
     TmpInst = MCInstBuilder(RISCV::JALR).addReg(Ra).addReg(Ra).addImm(0);
   Binary = getBinaryCodeForInstr(TmpInst, Fixups, STI);
-  support::endian::write(CB, Binary, support::little);
+  support::endian::write(CB, Binary, llvm::endianness::little);
 }
 
 // Expand PseudoAddTPRel to a simple ADD with the correct relocation.
@@ -186,7 +186,7 @@ void RISCVMCCodeEmitter::expandAddTPRel(const MCInst &MI,
                        .addOperand(SrcReg)
                        .addOperand(TPReg);
   uint32_t Binary = getBinaryCodeForInstr(TmpInst, Fixups, STI);
-  support::endian::write(CB, Binary, support::little);
+  support::endian::write(CB, Binary, llvm::endianness::little);
 }
 
 static unsigned getInvertedBranchOp(unsigned BrOp) {
@@ -240,14 +240,14 @@ void RISCVMCCodeEmitter::expandLongCondBr(const MCInst &MI,
         Opcode == RISCV::PseudoLongBNE ? RISCV::C_BEQZ : RISCV::C_BNEZ;
     MCInst TmpInst = MCInstBuilder(InvOpc).addReg(SrcReg1).addImm(6);
     uint16_t Binary = getBinaryCodeForInstr(TmpInst, Fixups, STI);
-    support::endian::write<uint16_t>(CB, Binary, support::little);
+    support::endian::write<uint16_t>(CB, Binary, llvm::endianness::little);
     Offset = 2;
   } else {
     unsigned InvOpc = getInvertedBranchOp(Opcode);
     MCInst TmpInst =
         MCInstBuilder(InvOpc).addReg(SrcReg1).addReg(SrcReg2).addImm(8);
     uint32_t Binary = getBinaryCodeForInstr(TmpInst, Fixups, STI);
-    support::endian::write(CB, Binary, support::little);
+    support::endian::write(CB, Binary, llvm::endianness::little);
     Offset = 4;
   }
 
@@ -255,7 +255,7 @@ void RISCVMCCodeEmitter::expandLongCondBr(const MCInst &MI,
   MCInst TmpInst =
       MCInstBuilder(RISCV::JAL).addReg(RISCV::X0).addOperand(SrcSymbol);
   uint32_t Binary = getBinaryCodeForInstr(TmpInst, Fixups, STI);
-  support::endian::write(CB, Binary, support::little);
+  support::endian::write(CB, Binary, llvm::endianness::little);
 
   Fixups.clear();
   if (SrcSymbol.isExpr()) {
@@ -306,12 +306,12 @@ void RISCVMCCodeEmitter::encodeInstruction(const MCInst &MI,
     llvm_unreachable("Unhandled encodeInstruction length!");
   case 2: {
     uint16_t Bits = getBinaryCodeForInstr(MI, Fixups, STI);
-    support::endian::write<uint16_t>(CB, Bits, support::little);
+    support::endian::write<uint16_t>(CB, Bits, llvm::endianness::little);
     break;
   }
   case 4: {
     uint32_t Bits = getBinaryCodeForInstr(MI, Fixups, STI);
-    support::endian::write(CB, Bits, support::little);
+    support::endian::write(CB, Bits, llvm::endianness::little);
     break;
   }
   }
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCTargetDesc.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCTargetDesc.cpp
index 75af5c2de0..79e56a7a6d 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCTargetDesc.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCTargetDesc.cpp
@@ -31,6 +31,7 @@
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/TargetRegistry.h"
 #include "llvm/Support/ErrorHandling.h"
+#include <bitset>
 
 #define GET_INSTRINFO_MC_DESC
 #define ENABLE_INSTR_PREDICATE_VERIFIER
@@ -114,10 +115,79 @@ static MCTargetStreamer *createRISCVNullTargetStreamer(MCStreamer &S) {
 namespace {
 
 class RISCVMCInstrAnalysis : public MCInstrAnalysis {
+  int64_t GPRState[31] = {};
+  std::bitset<31> GPRValidMask;
+
+  static bool isGPR(unsigned Reg) {
+    return Reg >= RISCV::X0 && Reg <= RISCV::X31;
+  }
+
+  static unsigned getRegIndex(unsigned Reg) {
+    assert(isGPR(Reg) && Reg != RISCV::X0 && "Invalid GPR reg");
+    return Reg - RISCV::X1;
+  }
+
+  void setGPRState(unsigned Reg, std::optional<int64_t> Value) {
+    if (Reg == RISCV::X0)
+      return;
+
+    auto Index = getRegIndex(Reg);
+
+    if (Value) {
+      GPRState[Index] = *Value;
+      GPRValidMask.set(Index);
+    } else {
+      GPRValidMask.reset(Index);
+    }
+  }
+
+  std::optional<int64_t> getGPRState(unsigned Reg) const {
+    if (Reg == RISCV::X0)
+      return 0;
+
+    auto Index = getRegIndex(Reg);
+
+    if (GPRValidMask.test(Index))
+      return GPRState[Index];
+    return std::nullopt;
+  }
+
 public:
   explicit RISCVMCInstrAnalysis(const MCInstrInfo *Info)
       : MCInstrAnalysis(Info) {}
 
+  void resetState() override { GPRValidMask.reset(); }
+
+  void updateState(const MCInst &Inst, uint64_t Addr) override {
+    // Terminators mark the end of a basic block which means the sequentially
+    // next instruction will be the first of another basic block and the current
+    // state will typically not be valid anymore. For calls, we assume all
+    // registers may be clobbered by the callee (TODO: should we take the
+    // calling convention into account?).
+    if (isTerminator(Inst) || isCall(Inst)) {
+      resetState();
+      return;
+    }
+
+    switch (Inst.getOpcode()) {
+    default: {
+      // Clear the state of all defined registers for instructions that we don't
+      // explicitly support.
+      auto NumDefs = Info->get(Inst.getOpcode()).getNumDefs();
+      for (unsigned I = 0; I < NumDefs; ++I) {
+        auto DefReg = Inst.getOperand(I).getReg();
+        if (isGPR(DefReg))
+          setGPRState(DefReg, std::nullopt);
+      }
+      break;
+    }
+    case RISCV::AUIPC:
+      setGPRState(Inst.getOperand(0).getReg(),
+                  Addr + (Inst.getOperand(1).getImm() << 12));
+      break;
+    }
+  }
+
   bool evaluateBranch(const MCInst &Inst, uint64_t Addr, uint64_t Size,
                       uint64_t &Target) const override {
     if (isConditionalBranch(Inst)) {
@@ -140,6 +210,15 @@ class RISCVMCInstrAnalysis : public MCInstrAnalysis {
       return true;
     }
 
+    if (Inst.getOpcode() == RISCV::JALR) {
+      if (auto TargetRegState = getGPRState(Inst.getOperand(1).getReg())) {
+        Target = *TargetRegState + Inst.getOperand(2).getImm();
+        return true;
+      }
+
+      return false;
+    }
+
     return false;
   }
 
diff --git a/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp b/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
index 526520e6c5..0fd514fa87 100644
--- a/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
+++ b/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
@@ -62,6 +62,15 @@ class RISCVAsmPrinter : public AsmPrinter {
 
   StringRef getPassName() const override { return "RISC-V Assembly Printer"; }
 
+  void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
+                     const MachineInstr &MI);
+
+  void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
+                       const MachineInstr &MI);
+
+  void LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
+                       const MachineInstr &MI);
+
   bool runOnMachineFunction(MachineFunction &MF) override;
 
   void emitInstruction(const MachineInstr *MI) override;
@@ -99,6 +108,78 @@ class RISCVAsmPrinter : public AsmPrinter {
 };
 }
 
+void RISCVAsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
+                                    const MachineInstr &MI) {
+  unsigned NOPBytes = STI->getFeatureBits()[RISCV::FeatureStdExtC] ? 2 : 4;
+  unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes();
+
+  auto &Ctx = OutStreamer.getContext();
+  MCSymbol *MILabel = Ctx.createTempSymbol();
+  OutStreamer.emitLabel(MILabel);
+
+  SM.recordStackMap(*MILabel, MI);
+  assert(NumNOPBytes % NOPBytes == 0 &&
+         "Invalid number of NOP bytes requested!");
+
+  // Scan ahead to trim the shadow.
+  const MachineBasicBlock &MBB = *MI.getParent();
+  MachineBasicBlock::const_iterator MII(MI);
+  ++MII;
+  while (NumNOPBytes > 0) {
+    if (MII == MBB.end() || MII->isCall() ||
+        MII->getOpcode() == RISCV::DBG_VALUE ||
+        MII->getOpcode() == TargetOpcode::PATCHPOINT ||
+        MII->getOpcode() == TargetOpcode::STACKMAP)
+      break;
+    ++MII;
+    NumNOPBytes -= 4;
+  }
+
+  // Emit nops.
+  emitNops(NumNOPBytes / NOPBytes);
+}
+
+// Lower a patchpoint of the form:
+// [<def>], <id>, <numBytes>, <target>, <numArgs>
+void RISCVAsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
+                                      const MachineInstr &MI) {
+  unsigned NOPBytes = STI->getFeatureBits()[RISCV::FeatureStdExtC] ? 2 : 4;
+
+  auto &Ctx = OutStreamer.getContext();
+  MCSymbol *MILabel = Ctx.createTempSymbol();
+  OutStreamer.emitLabel(MILabel);
+  SM.recordPatchPoint(*MILabel, MI);
+
+  PatchPointOpers Opers(&MI);
+
+  unsigned EncodedBytes = 0;
+
+  // Emit padding.
+  unsigned NumBytes = Opers.getNumPatchBytes();
+  assert(NumBytes >= EncodedBytes &&
+         "Patchpoint can't request size less than the length of a call.");
+  assert((NumBytes - EncodedBytes) % NOPBytes == 0 &&
+         "Invalid number of NOP bytes requested!");
+  emitNops((NumBytes - EncodedBytes) / NOPBytes);
+}
+
+void RISCVAsmPrinter::LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
+                                      const MachineInstr &MI) {
+  unsigned NOPBytes = STI->getFeatureBits()[RISCV::FeatureStdExtC] ? 2 : 4;
+
+  StatepointOpers SOpers(&MI);
+  if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
+    assert(PatchBytes % NOPBytes == 0 &&
+           "Invalid number of NOP bytes requested!");
+    emitNops(PatchBytes / NOPBytes);
+  }
+
+  auto &Ctx = OutStreamer.getContext();
+  MCSymbol *MILabel = Ctx.createTempSymbol();
+  OutStreamer.emitLabel(MILabel);
+  SM.recordStatepoint(*MILabel, MI);
+}
+
 void RISCVAsmPrinter::EmitToStreamer(MCStreamer &S, const MCInst &Inst) {
   MCInst CInst;
   bool Res = RISCVRVC::compress(CInst, Inst, *STI);
@@ -166,6 +247,12 @@ void RISCVAsmPrinter::emitInstruction(const MachineInstr *MI) {
   case RISCV::PseudoRVVInitUndefM4:
   case RISCV::PseudoRVVInitUndefM8:
     return;
+  case TargetOpcode::STACKMAP:
+    return LowerSTACKMAP(*OutStreamer, SM, *MI);
+  case TargetOpcode::PATCHPOINT:
+    return LowerPATCHPOINT(*OutStreamer, SM, *MI);
+  case TargetOpcode::STATEPOINT:
+    return LowerSTATEPOINT(*OutStreamer, SM, *MI);
   }
 
   MCInst OutInst;
diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td
index 3d3486b7fa..979bc0ea8c 100644
--- a/llvm/lib/Target/RISCV/RISCVFeatures.td
+++ b/llvm/lib/Target/RISCV/RISCVFeatures.td
@@ -813,6 +813,22 @@ def HasVendorXSfcie : Predicate<"Subtarget->hasVendorXSfcie()">,
                         AssemblerPredicate<(all_of FeatureVendorXSfcie),
                         "'XSfcie' (SiFive Custom Instruction Extension SCIE.)">;
 
+def FeatureVendorXSfvqmaccdod
+    : SubtargetFeature<"xsfvqmaccdod", "HasVendorXSfvqmaccdod", "true",
+                       "'XSfvqmaccdod' (SiFive Int8 Matrix Multiplication Instructions (2-by-8 and 8-by-2))",
+                       [FeatureStdExtZve32x]>;
+def HasVendorXSfvqmaccdod : Predicate<"Subtarget->hasVendorXSfvqmaccdod()">,
+                         AssemblerPredicate<(all_of FeatureVendorXSfvqmaccdod),
+                         "'XSfvqmaccdod' (SiFive Int8 Matrix Multiplication Instructions (2-by-8 and 8-by-2))">;
+
+def FeatureVendorXSfvqmaccqoq
+    : SubtargetFeature<"xsfvqmaccqoq", "HasVendorXSfvqmaccqoq", "true",
+                       "'XSfvqmaccqoq' (SiFive Int8 Matrix Multiplication Instructions (4-by-8 and 8-by-4))",
+                       [FeatureStdExtZve32x]>;
+def HasVendorXSfvqmaccqoq : Predicate<"Subtarget->hasVendorXSfvqmaccqoq()">,
+                         AssemblerPredicate<(all_of FeatureVendorXSfvqmaccqoq),
+                         "'XSfvqmaccqoq' (SiFive Int8 Matrix Multiplication Instructions (4-by-8 and 8-by-4))">;
+
 def FeatureVendorXCVbitmanip
     : SubtargetFeature<"xcvbitmanip", "HasVendorXCVbitmanip", "true",
                        "'XCVbitmanip' (CORE-V Bit Manipulation)">;
@@ -902,6 +918,9 @@ def FeatureUnalignedVectorMem
                       "true", "Has reasonably performant unaligned vector "
                       "loads and stores">;
 
+def FeaturePostRAScheduler : SubtargetFeature<"use-postra-scheduler",
+    "UsePostRAScheduler", "true", "Schedule again after register allocation">;
+
 def TuneNoOptimizedZeroStrideLoad
    : SubtargetFeature<"no-optimized-zero-stride-load", "HasOptimizedZeroStrideLoad",
                       "false", "Hasn't optimized (perform fewer memory operations)"
diff --git a/llvm/lib/Target/RISCV/RISCVGISel.td b/llvm/lib/Target/RISCV/RISCVGISel.td
index b20c27517b..8d0d088c11 100644
--- a/llvm/lib/Target/RISCV/RISCVGISel.td
+++ b/llvm/lib/Target/RISCV/RISCVGISel.td
@@ -1,4 +1,4 @@
-//===-- RISCVGIsel.td - RISCV GlobalISel Patterns ----------*- tablegen -*-===//
+//===-- RISCVGIsel.td - RISC-V GlobalISel Patterns ---------*- tablegen -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -18,6 +18,12 @@ include "RISCVCombine.td"
 
 def simm12Plus1 : ImmLeaf<XLenVT, [{
     return (isInt<12>(Imm) && Imm != -2048) || Imm == 2048;}]>;
+def simm12Plus1i32 : ImmLeaf<i32, [{
+    return (isInt<12>(Imm) && Imm != -2048) || Imm == 2048;}]>;
+
+def simm12i32 : ImmLeaf<i32, [{return isInt<12>(Imm);}]>;
+
+def uimm5i32 : ImmLeaf<i32, [{return isUInt<5>(Imm);}]>;
 
 // FIXME: This doesn't check that the G_CONSTANT we're deriving the immediate
 // from is only used once
@@ -43,6 +49,14 @@ def GIAddrRegImm :
   GIComplexOperandMatcher<s32, "selectAddrRegImm">,
   GIComplexPatternEquiv<AddrRegImm>;
 
+// Convert from i32 immediate to i64 target immediate to make SelectionDAG type
+// checking happy so we can use ADDIW which expects an XLen immediate.
+def as_i64imm : SDNodeXForm<imm, [{
+  return CurDAG->getTargetConstant(N->getSExtValue(), SDLoc(N), MVT::i64);
+}]>;
+def gi_as_i64imm : GICustomOperandRenderer<"renderImm">,
+  GISDNodeXFormEquiv<as_i64imm>;
+
 // FIXME: This is labelled as handling 's32', however the ComplexPattern it
 // refers to handles both i32 and i64 based on the HwMode. Currently this LLT
 // parameter appears to be ignored so this pattern works for both, however we
@@ -52,6 +66,13 @@ def ShiftMaskGI :
     GIComplexOperandMatcher<s32, "selectShiftMask">,
     GIComplexPatternEquiv<shiftMaskXLen>;
 
+def gi_sh1add_op : GIComplexOperandMatcher<s32, "selectSHXADDOp<1>">,
+                   GIComplexPatternEquiv<sh1add_op>;
+def gi_sh2add_op : GIComplexOperandMatcher<s32, "selectSHXADDOp<2>">,
+                   GIComplexPatternEquiv<sh2add_op>;
+def gi_sh3add_op : GIComplexOperandMatcher<s32, "selectSHXADDOp<3>">,
+                   GIComplexPatternEquiv<sh3add_op>;
+
 // FIXME: Canonicalize (sub X, C) -> (add X, -C) earlier.
 def : Pat<(XLenVT (sub GPR:$rs1, simm12Plus1:$imm)),
           (ADDI GPR:$rs1, (NegImm simm12Plus1:$imm))>;
@@ -60,11 +81,23 @@ let Predicates = [IsRV64] in {
 def : Pat<(i32 (add GPR:$rs1, GPR:$rs2)), (ADDW GPR:$rs1, GPR:$rs2)>;
 def : Pat<(i32 (sub GPR:$rs1, GPR:$rs2)), (SUBW GPR:$rs1, GPR:$rs2)>;
 
+def : Pat<(i32 (add GPR:$rs1, simm12i32:$imm)),
+          (ADDIW GPR:$rs1, (i64 (as_i64imm $imm)))>;
+def : Pat<(i32 (sub GPR:$rs1, simm12Plus1i32:$imm)),
+          (ADDIW GPR:$rs1, (i64 (NegImm $imm)))>;
+
 def : Pat<(i32 (shl GPR:$rs1, (i32 GPR:$rs2))), (SLLW GPR:$rs1, GPR:$rs2)>;
 def : Pat<(i32 (sra GPR:$rs1, (i32 GPR:$rs2))), (SRAW GPR:$rs1, GPR:$rs2)>;
 def : Pat<(i32 (srl GPR:$rs1, (i32 GPR:$rs2))), (SRLW GPR:$rs1, GPR:$rs2)>;
 
-def: Pat<(i64 (sext i32:$rs)), (ADDIW GPR:$rs, 0)>;
+def : Pat<(i32 (shl GPR:$rs1, uimm5i32:$imm)),
+          (SLLIW GPR:$rs1, (i64 (as_i64imm $imm)))>;
+def : Pat<(i32 (sra GPR:$rs1, uimm5i32:$imm)),
+          (SRAIW GPR:$rs1, (i64 (as_i64imm $imm)))>;
+def : Pat<(i32 (srl GPR:$rs1, uimm5i32:$imm)),
+          (SRLIW GPR:$rs1, (i64 (as_i64imm $imm)))>;
+
+def : Pat<(i64 (sext i32:$rs)), (ADDIW GPR:$rs, 0)>;
 }
 
 let Predicates = [HasStdExtMOrZmmul, IsRV64] in {
diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
index 9bf1e12584..1b067557b2 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
@@ -1013,12 +1013,12 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
     unsigned TrailingOnes = llvm::countr_one(Mask);
     if (ShAmt >= TrailingOnes)
       break;
-    // If the mask has 32 trailing ones, use SRLIW.
+    // If the mask has 32 trailing ones, use SRLI on RV32 or SRLIW on RV64.
     if (TrailingOnes == 32) {
-      SDNode *SRLIW =
-          CurDAG->getMachineNode(RISCV::SRLIW, DL, VT, N0->getOperand(0),
-                                 CurDAG->getTargetConstant(ShAmt, DL, VT));
-      ReplaceNode(Node, SRLIW);
+      SDNode *SRLI = CurDAG->getMachineNode(
+          Subtarget->is64Bit() ? RISCV::SRLIW : RISCV::SRLI, DL, VT,
+          N0->getOperand(0), CurDAG->getTargetConstant(ShAmt, DL, VT));
+      ReplaceNode(Node, SRLI);
       return;
     }
 
@@ -2229,7 +2229,8 @@ bool RISCVDAGToDAGISel::SelectFrameAddrRegImm(SDValue Addr, SDValue &Base,
 // Fold constant addresses.
 static bool selectConstantAddr(SelectionDAG *CurDAG, const SDLoc &DL,
                                const MVT VT, const RISCVSubtarget *Subtarget,
-                               SDValue Addr, SDValue &Base, SDValue &Offset) {
+                               SDValue Addr, SDValue &Base, SDValue &Offset,
+                               bool IsPrefetch = false) {
   if (!isa<ConstantSDNode>(Addr))
     return false;
 
@@ -2241,6 +2242,9 @@ static bool selectConstantAddr(SelectionDAG *CurDAG, const SDLoc &DL,
   int64_t Lo12 = SignExtend64<12>(CVal);
   int64_t Hi = (uint64_t)CVal - (uint64_t)Lo12;
   if (!Subtarget->is64Bit() || isInt<32>(Hi)) {
+    if (IsPrefetch && (Lo12 & 0b11111) != 0)
+      return false;
+
     if (Hi) {
       int64_t Hi20 = (Hi >> 12) & 0xfffff;
       Base = SDValue(
@@ -2263,6 +2267,8 @@ static bool selectConstantAddr(SelectionDAG *CurDAG, const SDLoc &DL,
   if (Seq.back().getOpcode() != RISCV::ADDI)
     return false;
   Lo12 = Seq.back().getImm();
+  if (IsPrefetch && (Lo12 & 0b11111) != 0)
+    return false;
 
   // Drop the last instruction.
   Seq.pop_back();
@@ -2443,6 +2449,72 @@ bool RISCVDAGToDAGISel::SelectAddrRegImm(SDValue Addr, SDValue &Base,
   return true;
 }
 
+/// Similar to SelectAddrRegImm, except that the least significant 5 bits of
+/// Offset shoule be all zeros.
+bool RISCVDAGToDAGISel::SelectAddrRegImmLsb00000(SDValue Addr, SDValue &Base,
+                                                 SDValue &Offset) {
+  if (SelectAddrFrameIndex(Addr, Base, Offset))
+    return true;
+
+  SDLoc DL(Addr);
+  MVT VT = Addr.getSimpleValueType();
+
+  if (CurDAG->isBaseWithConstantOffset(Addr)) {
+    int64_t CVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue();
+    if (isInt<12>(CVal)) {
+      Base = Addr.getOperand(0);
+
+      // Early-out if not a valid offset.
+      if ((CVal & 0b11111) != 0) {
+        Base = Addr;
+        Offset = CurDAG->getTargetConstant(0, DL, VT);
+        return true;
+      }
+
+      if (auto *FIN = dyn_cast<FrameIndexSDNode>(Base))
+        Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), VT);
+      Offset = CurDAG->getTargetConstant(CVal, DL, VT);
+      return true;
+    }
+  }
+
+  // Handle ADD with large immediates.
+  if (Addr.getOpcode() == ISD::ADD && isa<ConstantSDNode>(Addr.getOperand(1))) {
+    int64_t CVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue();
+    assert(!(isInt<12>(CVal) && isInt<12>(CVal)) &&
+           "simm12 not already handled?");
+
+    // Handle immediates in the range [-4096,-2049] or [2017, 4065]. We can save
+    // one instruction by folding adjustment (-2048 or 2016) into the address.
+    if ((-2049 >= CVal && CVal >= -4096) || (4065 >= CVal && CVal >= 2017)) {
+      int64_t Adj = CVal < 0 ? -2048 : 2016;
+      int64_t AdjustedOffset = CVal - Adj;
+      Base = SDValue(CurDAG->getMachineNode(
+                         RISCV::ADDI, DL, VT, Addr.getOperand(0),
+                         CurDAG->getTargetConstant(AdjustedOffset, DL, VT)),
+                     0);
+      Offset = CurDAG->getTargetConstant(Adj, DL, VT);
+      return true;
+    }
+
+    if (selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr.getOperand(1), Base,
+                           Offset, true)) {
+      // Insert an ADD instruction with the materialized Hi52 bits.
+      Base = SDValue(
+          CurDAG->getMachineNode(RISCV::ADD, DL, VT, Addr.getOperand(0), Base),
+          0);
+      return true;
+    }
+  }
+
+  if (selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr, Base, Offset, true))
+    return true;
+
+  Base = Addr;
+  Offset = CurDAG->getTargetConstant(0, DL, VT);
+  return true;
+}
+
 bool RISCVDAGToDAGISel::selectShiftMask(SDValue N, unsigned ShiftWidth,
                                         SDValue &ShAmt) {
   ShAmt = N;
@@ -2806,6 +2878,11 @@ bool RISCVDAGToDAGISel::hasAllNBitUsers(SDNode *Node, unsigned Bits,
   if (Depth >= SelectionDAG::MaxRecursionDepth)
     return false;
 
+  // The PatFrags that call this may run before RISCVGenDAGISel.inc has checked
+  // the VT. Ensure the type is scalar to avoid wasting time on vectors.
+  if (!Node->getValueType(0).isScalarInteger())
+    return false;
+
   for (auto UI = Node->use_begin(), UE = Node->use_end(); UI != UE; ++UI) {
     SDNode *User = *UI;
     // Users of this node should have already been instruction selected
diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
index c220b2d57c..675ab4e74c 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
@@ -54,6 +54,7 @@ class RISCVDAGToDAGISel : public SelectionDAGISel {
   bool SelectAddrRegImmINX(SDValue Addr, SDValue &Base, SDValue &Offset) {
     return SelectAddrRegImm(Addr, Base, Offset, true);
   }
+  bool SelectAddrRegImmLsb00000(SDValue Addr, SDValue &Base, SDValue &Offset);
 
   bool SelectAddrRegRegScale(SDValue Addr, unsigned MaxShiftAmount,
                              SDValue &Base, SDValue &Index, SDValue &Scale);
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index e0ca9913f9..af52e01c27 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -27,6 +27,7 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineJumpTableInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
 #include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/IR/DiagnosticInfo.h"
@@ -730,7 +731,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
                          VT, Custom);
       setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,
                          Custom);
-
+      setOperationAction({ISD::LRINT, ISD::LLRINT}, VT, Custom);
       setOperationAction(
           {ISD::SADDSAT, ISD::UADDSAT, ISD::SSUBSAT, ISD::USUBSAT}, VT, Legal);
 
@@ -941,6 +942,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
         if (!isTypeLegal(VT))
           continue;
         setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom);
+        setOperationAction({ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT,
+                           Custom);
         setOperationAction({ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND}, VT, Custom);
         setOperationAction({ISD::VP_MERGE, ISD::VP_SELECT, ISD::SELECT}, VT,
                            Custom);
@@ -1153,6 +1156,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
         if (VT.getVectorElementType() == MVT::f16 &&
             !Subtarget.hasVInstructionsF16()) {
           setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom);
+          setOperationAction({ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT,
+                             Custom);
           setOperationAction({ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND}, VT, Custom);
           setOperationAction(
               {ISD::VP_MERGE, ISD::VP_SELECT, ISD::VSELECT, ISD::SELECT}, VT,
@@ -2945,6 +2950,31 @@ lowerFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
                      DAG.getTargetConstant(FRM, DL, Subtarget.getXLenVT()));
 }
 
+// Expand vector LRINT and LLRINT by converting to the integer domain.
+static SDValue lowerVectorXRINT(SDValue Op, SelectionDAG &DAG,
+                                const RISCVSubtarget &Subtarget) {
+  MVT VT = Op.getSimpleValueType();
+  assert(VT.isVector() && "Unexpected type");
+
+  SDLoc DL(Op);
+  SDValue Src = Op.getOperand(0);
+  MVT ContainerVT = VT;
+
+  if (VT.isFixedLengthVector()) {
+    ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
+    Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
+  }
+
+  auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
+  SDValue Truncated =
+      DAG.getNode(RISCVISD::VFCVT_X_F_VL, DL, ContainerVT, Src, Mask, VL);
+
+  if (!VT.isFixedLengthVector())
+    return Truncated;
+
+  return convertFromScalableVector(VT, Truncated, DAG, Subtarget);
+}
+
 static SDValue
 getVSlidedown(SelectionDAG &DAG, const RISCVSubtarget &Subtarget,
               const SDLoc &DL, EVT VT, SDValue Merge, SDValue Op,
@@ -5202,7 +5232,7 @@ static SDValue lowerFMAXIMUM_FMINIMUM(SDValue Op, SelectionDAG &DAG,
   return Res;
 }
 
-/// Get a RISCV target specified VL op for a given SDNode.
+/// Get a RISC-V target specified VL op for a given SDNode.
 static unsigned getRISCVVLOp(SDValue Op) {
 #define OP_CASE(NODE)                                                          \
   case ISD::NODE:                                                              \
@@ -5973,6 +6003,9 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
   case ISD::FROUND:
   case ISD::FROUNDEVEN:
     return lowerFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget);
+  case ISD::LRINT:
+  case ISD::LLRINT:
+    return lowerVectorXRINT(Op, DAG, Subtarget);
   case ISD::VECREDUCE_ADD:
   case ISD::VECREDUCE_UMAX:
   case ISD::VECREDUCE_SMAX:
@@ -11579,8 +11612,23 @@ static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp,
   if (VT.isVector())
     return SDValue();
 
-  if (!Subtarget.hasShortForwardBranchOpt() ||
-      (Slct.getOpcode() != ISD::SELECT &&
+  if (!Subtarget.hasShortForwardBranchOpt()) {
+    // (select cond, x, (and x, c)) has custom lowering with Zicond.
+    if ((!Subtarget.hasStdExtZicond() &&
+         !Subtarget.hasVendorXVentanaCondOps()) ||
+        N->getOpcode() != ISD::AND)
+      return SDValue();
+
+    // Maybe harmful when condition code has multiple use.
+    if (Slct.getOpcode() == ISD::SELECT && !Slct.getOperand(0).hasOneUse())
+      return SDValue();
+
+    // Maybe harmful when VT is wider than XLen.
+    if (VT.getSizeInBits() > Subtarget.getXLen())
+      return SDValue();
+  }
+
+  if ((Slct.getOpcode() != ISD::SELECT &&
        Slct.getOpcode() != RISCVISD::SELECT_CC) ||
       !Slct.hasOneUse())
     return SDValue();
@@ -11999,7 +12047,7 @@ static SDValue performXORCombine(SDNode *N, SelectionDAG &DAG,
   }
 
   // Fold (xor (setcc constant, y, setlt), 1) -> (setcc y, constant + 1, setlt)
-  if (N0.hasOneUse() && N0.getOpcode() == ISD::SETCC && isOneConstant(N1)) {
+  if (N0.getOpcode() == ISD::SETCC && isOneConstant(N1) && N0.hasOneUse()) {
     auto *ConstN00 = dyn_cast<ConstantSDNode>(N0.getOperand(0));
     ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
     if (ConstN00 && CC == ISD::SETLT) {
@@ -13712,7 +13760,7 @@ static SDValue performSELECTCombine(SDNode *N, SelectionDAG &DAG,
   return tryFoldSelectIntoOp(N, DAG, FalseVal, TrueVal, /*Swapped*/true);
 }
 
-/// IF we have a build_vector where each lane is binop X, C, where C
+/// If we have a build_vector where each lane is binop X, C, where C
 /// is a constant (but not necessarily the same constant on all lanes),
 /// form binop (build_vector x1, x2, ...), (build_vector c1, c2, c3, ..).
 /// We assume that materializing a constant build vector will be no more
@@ -13739,6 +13787,10 @@ static SDValue performBUILD_VECTORCombine(SDNode *N, SelectionDAG &DAG,
   SmallVector<SDValue> RHSOps;
   for (SDValue Op : N->ops()) {
     if (Op.isUndef()) {
+      // We can't form a divide or remainder from undef.
+      if (!DAG.isSafeToSpeculativelyExecute(Opcode))
+        return SDValue();
+
       LHSOps.push_back(Op);
       RHSOps.push_back(Op);
       continue;
@@ -13754,6 +13806,10 @@ static SDValue performBUILD_VECTORCombine(SDNode *N, SelectionDAG &DAG,
     if (!isa<ConstantSDNode>(Op.getOperand(1)) &&
         !isa<ConstantFPSDNode>(Op.getOperand(1)))
       return SDValue();
+    // FIXME: Return failure if the RHS type doesn't match the LHS. Shifts may
+    // have different LHS and RHS types.
+    if (Op.getOperand(0).getValueType() != Op.getOperand(1).getValueType())
+      return SDValue();
     RHSOps.push_back(Op.getOperand(1));
   }
 
@@ -13785,11 +13841,10 @@ static SDValue performCONCAT_VECTORSCombine(SDNode *N, SelectionDAG &DAG,
     return SDValue();
 
   EVT BaseLdVT = BaseLd->getValueType(0);
-  SDValue BasePtr = BaseLd->getBasePtr();
 
   // Go through the loads and check that they're strided
-  SmallVector<SDValue> Ptrs;
-  Ptrs.push_back(BasePtr);
+  SmallVector<LoadSDNode *> Lds;
+  Lds.push_back(BaseLd);
   Align Align = BaseLd->getAlign();
   for (SDValue Op : N->ops().drop_front()) {
     auto *Ld = dyn_cast<LoadSDNode>(Op);
@@ -13798,60 +13853,46 @@ static SDValue performCONCAT_VECTORSCombine(SDNode *N, SelectionDAG &DAG,
         Ld->getValueType(0) != BaseLdVT)
       return SDValue();
 
-    Ptrs.push_back(Ld->getBasePtr());
+    Lds.push_back(Ld);
 
     // The common alignment is the most restrictive (smallest) of all the loads
     Align = std::min(Align, Ld->getAlign());
   }
 
-  auto matchForwardStrided = [](ArrayRef<SDValue> Ptrs) {
-    SDValue Stride;
-    for (auto Idx : enumerate(Ptrs)) {
-      if (Idx.index() == 0)
-        continue;
-      SDValue Ptr = Idx.value();
-      // Check that each load's pointer is (add LastPtr, Stride)
-      if (Ptr.getOpcode() != ISD::ADD ||
-          Ptr.getOperand(0) != Ptrs[Idx.index()-1])
-        return SDValue();
-      SDValue Offset = Ptr.getOperand(1);
-      if (!Stride)
-        Stride = Offset;
-      else if (Offset != Stride)
-        return SDValue();
-    }
-    return Stride;
-  };
-  auto matchReverseStrided = [](ArrayRef<SDValue> Ptrs) {
-    SDValue Stride;
-    for (auto Idx : enumerate(Ptrs)) {
-      if (Idx.index() == Ptrs.size() - 1)
-        continue;
-      SDValue Ptr = Idx.value();
-      // Check that each load's pointer is (add NextPtr, Stride)
-      if (Ptr.getOpcode() != ISD::ADD ||
-          Ptr.getOperand(0) != Ptrs[Idx.index()+1])
-        return SDValue();
-      SDValue Offset = Ptr.getOperand(1);
-      if (!Stride)
-        Stride = Offset;
-      else if (Offset != Stride)
-        return SDValue();
-    }
-    return Stride;
+  using PtrDiff = std::pair<std::variant<int64_t, SDValue>, bool>;
+  auto GetPtrDiff = [&DAG](LoadSDNode *Ld1,
+                           LoadSDNode *Ld2) -> std::optional<PtrDiff> {
+    // If the load ptrs can be decomposed into a common (Base + Index) with a
+    // common constant stride, then return the constant stride.
+    BaseIndexOffset BIO1 = BaseIndexOffset::match(Ld1, DAG);
+    BaseIndexOffset BIO2 = BaseIndexOffset::match(Ld2, DAG);
+    if (BIO1.equalBaseIndex(BIO2, DAG))
+      return {{BIO2.getOffset() - BIO1.getOffset(), false}};
+
+    // Otherwise try to match (add LastPtr, Stride) or (add NextPtr, Stride)
+    SDValue P1 = Ld1->getBasePtr();
+    SDValue P2 = Ld2->getBasePtr();
+    if (P2.getOpcode() == ISD::ADD && P2.getOperand(0) == P1)
+      return {{P2.getOperand(1), false}};
+    if (P1.getOpcode() == ISD::ADD && P1.getOperand(0) == P2)
+      return {{P1.getOperand(1), true}};
+
+    return std::nullopt;
   };
 
-  bool Reversed = false;
-  SDValue Stride = matchForwardStrided(Ptrs);
-  if (!Stride) {
-    Stride = matchReverseStrided(Ptrs);
-    Reversed = true;
-    // TODO: At this point, we've successfully matched a generalized gather
-    // load.  Maybe we should emit that, and then move the specialized
-    // matchers above and below into a DAG combine?
-    if (!Stride)
+  // Get the distance between the first and second loads
+  auto BaseDiff = GetPtrDiff(Lds[0], Lds[1]);
+  if (!BaseDiff)
+    return SDValue();
+
+  // Check all the loads are the same distance apart
+  for (auto *It = Lds.begin() + 1; It != Lds.end() - 1; It++)
+    if (GetPtrDiff(*It, *std::next(It)) != BaseDiff)
       return SDValue();
-  }
+
+  // TODO: At this point, we've successfully matched a generalized gather
+  // load.  Maybe we should emit that, and then move the specialized
+  // matchers above and below into a DAG combine?
 
   // Get the widened scalar type, e.g. v4i8 -> i64
   unsigned WideScalarBitWidth =
@@ -13867,26 +13908,29 @@ static SDValue performCONCAT_VECTORSCombine(SDNode *N, SelectionDAG &DAG,
   if (!TLI.isLegalStridedLoadStore(WideVecVT, Align))
     return SDValue();
 
+  auto [StrideVariant, MustNegateStride] = *BaseDiff;
+  SDValue Stride = std::holds_alternative<SDValue>(StrideVariant)
+                       ? std::get<SDValue>(StrideVariant)
+                       : DAG.getConstant(std::get<int64_t>(StrideVariant), DL,
+                                         Lds[0]->getOffset().getValueType());
+  if (MustNegateStride)
+    Stride = DAG.getNegative(Stride, DL, Stride.getValueType());
+
   SDVTList VTs = DAG.getVTList({WideVecVT, MVT::Other});
   SDValue IntID =
     DAG.getTargetConstant(Intrinsic::riscv_masked_strided_load, DL,
                           Subtarget.getXLenVT());
-  if (Reversed)
-    Stride = DAG.getNegative(Stride, DL, Stride->getValueType(0));
+
   SDValue AllOneMask =
     DAG.getSplat(WideVecVT.changeVectorElementType(MVT::i1), DL,
                  DAG.getConstant(1, DL, MVT::i1));
 
-  SDValue Ops[] = {BaseLd->getChain(),
-                   IntID,
-                   DAG.getUNDEF(WideVecVT),
-                   BasePtr,
-                   Stride,
-                   AllOneMask};
+  SDValue Ops[] = {BaseLd->getChain(),   IntID,  DAG.getUNDEF(WideVecVT),
+                   BaseLd->getBasePtr(), Stride, AllOneMask};
 
   uint64_t MemSize;
   if (auto *ConstStride = dyn_cast<ConstantSDNode>(Stride);
-      ConstStride && !Reversed && ConstStride->getSExtValue() >= 0)
+      ConstStride && ConstStride->getSExtValue() >= 0)
     // total size = (elsize * n) + (stride - elsize) * (n-1)
     //            = elsize + stride * (n-1)
     MemSize = WideScalarVT.getSizeInBits() +
@@ -14898,11 +14942,14 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
 
     // Use M1 or smaller to avoid over constraining register allocation
     const MVT M1VT = getLMUL1VT(VT);
-    if (M1VT.bitsLT(VT) && Passthru.isUndef()) {
+    if (M1VT.bitsLT(VT)) {
+      SDValue M1Passthru =
+          DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, M1VT, Passthru,
+                      DAG.getVectorIdxConstant(0, DL));
       SDValue Result =
-        DAG.getNode(N->getOpcode(), DL, M1VT, Passthru, Scalar, VL);
-      Result = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT),
-                           Result, DAG.getConstant(0, DL, XLenVT));
+          DAG.getNode(N->getOpcode(), DL, M1VT, M1Passthru, Scalar, VL);
+      Result = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, Passthru, Result,
+                           DAG.getConstant(0, DL, XLenVT));
       return Result;
     }
 
@@ -16211,6 +16258,13 @@ RISCVTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
   case RISCV::PseudoFROUND_D_INX:
   case RISCV::PseudoFROUND_D_IN32X:
     return emitFROUND(MI, BB, Subtarget);
+  case TargetOpcode::STATEPOINT:
+  case TargetOpcode::STACKMAP:
+  case TargetOpcode::PATCHPOINT:
+    if (!Subtarget.is64Bit())
+      report_fatal_error("STACKMAP, PATCHPOINT and STATEPOINT are only "
+                         "supported on 64-bit targets");
+    return emitPatchPoint(MI, BB);
   }
 }
 
@@ -16445,16 +16499,23 @@ bool RISCV::CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo,
     // stack. LowerCall/LowerFormalArguments/LowerReturn must recognise these
     // cases.
     Register Reg = State.AllocateReg(ArgGPRs);
-    LocVT = MVT::i32;
     if (!Reg) {
       unsigned StackOffset = State.AllocateStack(8, Align(8));
       State.addLoc(
           CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
       return false;
     }
-    if (!State.AllocateReg(ArgGPRs))
-      State.AllocateStack(4, Align(4));
-    State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
+    LocVT = MVT::i32;
+    State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));
+    Register HiReg = State.AllocateReg(ArgGPRs);
+    if (HiReg) {
+      State.addLoc(
+          CCValAssign::getCustomReg(ValNo, ValVT, HiReg, LocVT, LocInfo));
+    } else {
+      unsigned StackOffset = State.AllocateStack(4, Align(4));
+      State.addLoc(
+          CCValAssign::getCustomMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
+    }
     return false;
   }
 
@@ -16763,38 +16824,32 @@ static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain,
 }
 
 static SDValue unpackF64OnRV32DSoftABI(SelectionDAG &DAG, SDValue Chain,
-                                       const CCValAssign &VA, const SDLoc &DL) {
+                                       const CCValAssign &VA,
+                                       const CCValAssign &HiVA,
+                                       const SDLoc &DL) {
   assert(VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64 &&
          "Unexpected VA");
   MachineFunction &MF = DAG.getMachineFunction();
   MachineFrameInfo &MFI = MF.getFrameInfo();
   MachineRegisterInfo &RegInfo = MF.getRegInfo();
 
-  if (VA.isMemLoc()) {
-    // f64 is passed on the stack.
-    int FI =
-        MFI.CreateFixedObject(8, VA.getLocMemOffset(), /*IsImmutable=*/true);
-    SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
-    return DAG.getLoad(MVT::f64, DL, Chain, FIN,
-                       MachinePointerInfo::getFixedStack(MF, FI));
-  }
-
   assert(VA.isRegLoc() && "Expected register VA assignment");
 
   Register LoVReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
   RegInfo.addLiveIn(VA.getLocReg(), LoVReg);
   SDValue Lo = DAG.getCopyFromReg(Chain, DL, LoVReg, MVT::i32);
   SDValue Hi;
-  if (VA.getLocReg() == RISCV::X17) {
+  if (HiVA.isMemLoc()) {
     // Second half of f64 is passed on the stack.
-    int FI = MFI.CreateFixedObject(4, 0, /*IsImmutable=*/true);
+    int FI = MFI.CreateFixedObject(4, HiVA.getLocMemOffset(),
+                                   /*IsImmutable=*/true);
     SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
     Hi = DAG.getLoad(MVT::i32, DL, Chain, FIN,
                      MachinePointerInfo::getFixedStack(MF, FI));
   } else {
     // Second half of f64 is passed in another GPR.
     Register HiVReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
-    RegInfo.addLiveIn(VA.getLocReg() + 1, HiVReg);
+    RegInfo.addLiveIn(HiVA.getLocReg(), HiVReg);
     Hi = DAG.getCopyFromReg(Chain, DL, HiVReg, MVT::i32);
   }
   return DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, Lo, Hi);
@@ -16999,6 +17054,7 @@ SDValue RISCVTargetLowering::LowerFormalArguments(
     report_fatal_error("Unsupported calling convention");
   case CallingConv::C:
   case CallingConv::Fast:
+  case CallingConv::SPIR_KERNEL:
     break;
   case CallingConv::GHC:
     if (!Subtarget.hasStdExtFOrZfinx() || !Subtarget.hasStdExtDOrZdinx())
@@ -17037,15 +17093,16 @@ SDValue RISCVTargetLowering::LowerFormalArguments(
                      CallConv == CallingConv::Fast ? RISCV::CC_RISCV_FastCC
                                                    : RISCV::CC_RISCV);
 
-  for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
+  for (unsigned i = 0, e = ArgLocs.size(), InsIdx = 0; i != e; ++i, ++InsIdx) {
     CCValAssign &VA = ArgLocs[i];
     SDValue ArgValue;
     // Passing f64 on RV32D with a soft float ABI must be handled as a special
     // case.
-    if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64)
-      ArgValue = unpackF64OnRV32DSoftABI(DAG, Chain, VA, DL);
-    else if (VA.isRegLoc())
-      ArgValue = unpackFromRegLoc(DAG, Chain, VA, DL, Ins[i], *this);
+    if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
+      assert(VA.needsCustom());
+      ArgValue = unpackF64OnRV32DSoftABI(DAG, Chain, VA, ArgLocs[++i], DL);
+    } else if (VA.isRegLoc())
+      ArgValue = unpackFromRegLoc(DAG, Chain, VA, DL, Ins[InsIdx], *this);
     else
       ArgValue = unpackFromMemLoc(DAG, Chain, VA, DL);
 
@@ -17057,12 +17114,12 @@ SDValue RISCVTargetLowering::LowerFormalArguments(
       // stores are relative to that.
       InVals.push_back(DAG.getLoad(VA.getValVT(), DL, Chain, ArgValue,
                                    MachinePointerInfo()));
-      unsigned ArgIndex = Ins[i].OrigArgIndex;
-      unsigned ArgPartOffset = Ins[i].PartOffset;
+      unsigned ArgIndex = Ins[InsIdx].OrigArgIndex;
+      unsigned ArgPartOffset = Ins[InsIdx].PartOffset;
       assert(VA.getValVT().isVector() || ArgPartOffset == 0);
-      while (i + 1 != e && Ins[i + 1].OrigArgIndex == ArgIndex) {
+      while (i + 1 != e && Ins[InsIdx + 1].OrigArgIndex == ArgIndex) {
         CCValAssign &PartVA = ArgLocs[i + 1];
-        unsigned PartOffset = Ins[i + 1].PartOffset - ArgPartOffset;
+        unsigned PartOffset = Ins[InsIdx + 1].PartOffset - ArgPartOffset;
         SDValue Offset = DAG.getIntPtrConstant(PartOffset, DL);
         if (PartVA.getValVT().isScalableVector())
           Offset = DAG.getNode(ISD::VSCALE, DL, XLenVT, Offset);
@@ -17070,6 +17127,7 @@ SDValue RISCVTargetLowering::LowerFormalArguments(
         InVals.push_back(DAG.getLoad(PartVA.getValVT(), DL, Chain, Address,
                                      MachinePointerInfo()));
         ++i;
+        ++InsIdx;
       }
       continue;
     }
@@ -17285,15 +17343,16 @@ SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI,
   SmallVector<std::pair<Register, SDValue>, 8> RegsToPass;
   SmallVector<SDValue, 8> MemOpChains;
   SDValue StackPtr;
-  for (unsigned i = 0, j = 0, e = ArgLocs.size(); i != e; ++i) {
+  for (unsigned i = 0, j = 0, e = ArgLocs.size(), OutIdx = 0; i != e;
+       ++i, ++OutIdx) {
     CCValAssign &VA = ArgLocs[i];
-    SDValue ArgValue = OutVals[i];
-    ISD::ArgFlagsTy Flags = Outs[i].Flags;
+    SDValue ArgValue = OutVals[OutIdx];
+    ISD::ArgFlagsTy Flags = Outs[OutIdx].Flags;
 
     // Handle passing f64 on RV32D with a soft float ABI as a special case.
-    bool IsF64OnRV32DSoftABI =
-        VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64;
-    if (IsF64OnRV32DSoftABI && VA.isRegLoc()) {
+    if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
+      assert(VA.isRegLoc() && "Expected register VA assignment");
+      assert(VA.needsCustom());
       SDValue SplitF64 = DAG.getNode(
           RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32), ArgValue);
       SDValue Lo = SplitF64.getValue(0);
@@ -17302,32 +17361,33 @@ SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI,
       Register RegLo = VA.getLocReg();
       RegsToPass.push_back(std::make_pair(RegLo, Lo));
 
-      if (RegLo == RISCV::X17) {
+      // Get the CCValAssign for the Hi part.
+      CCValAssign &HiVA = ArgLocs[++i];
+
+      if (HiVA.isMemLoc()) {
         // Second half of f64 is passed on the stack.
-        // Work out the address of the stack slot.
         if (!StackPtr.getNode())
           StackPtr = DAG.getCopyFromReg(Chain, DL, RISCV::X2, PtrVT);
+        SDValue Address =
+            DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr,
+                        DAG.getIntPtrConstant(HiVA.getLocMemOffset(), DL));
         // Emit the store.
         MemOpChains.push_back(
-            DAG.getStore(Chain, DL, Hi, StackPtr, MachinePointerInfo()));
+            DAG.getStore(Chain, DL, Hi, Address, MachinePointerInfo()));
       } else {
         // Second half of f64 is passed in another GPR.
-        assert(RegLo < RISCV::X31 && "Invalid register pair");
-        Register RegHigh = RegLo + 1;
+        Register RegHigh = HiVA.getLocReg();
         RegsToPass.push_back(std::make_pair(RegHigh, Hi));
       }
       continue;
     }
 
-    // IsF64OnRV32DSoftABI && VA.isMemLoc() is handled below in the same way
-    // as any other MemLoc.
-
     // Promote the value if needed.
     // For now, only handle fully promoted and indirect arguments.
     if (VA.getLocInfo() == CCValAssign::Indirect) {
       // Store the argument in a stack slot and pass its address.
       Align StackAlign =
-          std::max(getPrefTypeAlign(Outs[i].ArgVT, DAG),
+          std::max(getPrefTypeAlign(Outs[OutIdx].ArgVT, DAG),
                    getPrefTypeAlign(ArgValue.getValueType(), DAG));
       TypeSize StoredSize = ArgValue.getValueType().getStoreSize();
       // If the original argument was split (e.g. i128), we need
@@ -17335,16 +17395,16 @@ SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI,
       // Vectors may be partly split to registers and partly to the stack, in
       // which case the base address is partly offset and subsequent stores are
       // relative to that.
-      unsigned ArgIndex = Outs[i].OrigArgIndex;
-      unsigned ArgPartOffset = Outs[i].PartOffset;
+      unsigned ArgIndex = Outs[OutIdx].OrigArgIndex;
+      unsigned ArgPartOffset = Outs[OutIdx].PartOffset;
       assert(VA.getValVT().isVector() || ArgPartOffset == 0);
       // Calculate the total size to store. We don't have access to what we're
       // actually storing other than performing the loop and collecting the
       // info.
       SmallVector<std::pair<SDValue, SDValue>> Parts;
-      while (i + 1 != e && Outs[i + 1].OrigArgIndex == ArgIndex) {
-        SDValue PartValue = OutVals[i + 1];
-        unsigned PartOffset = Outs[i + 1].PartOffset - ArgPartOffset;
+      while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == ArgIndex) {
+        SDValue PartValue = OutVals[OutIdx + 1];
+        unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset;
         SDValue Offset = DAG.getIntPtrConstant(PartOffset, DL);
         EVT PartVT = PartValue.getValueType();
         if (PartVT.isScalableVector())
@@ -17353,6 +17413,7 @@ SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI,
         StackAlign = std::max(StackAlign, getPrefTypeAlign(PartVT, DAG));
         Parts.push_back(std::make_pair(PartValue, Offset));
         ++i;
+        ++OutIdx;
       }
       SDValue SpillSlot = DAG.CreateStackTemporary(StoredSize, StackAlign);
       int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
@@ -17494,7 +17555,8 @@ SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI,
   analyzeInputArgs(MF, RetCCInfo, Ins, /*IsRet=*/true, RISCV::CC_RISCV);
 
   // Copy all of the result registers out of their specified physreg.
-  for (auto &VA : RVLocs) {
+  for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) {
+    auto &VA = RVLocs[i];
     // Copy the value out
     SDValue RetValue =
         DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), Glue);
@@ -17503,9 +17565,9 @@ SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI,
     Glue = RetValue.getValue(2);
 
     if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
-      assert(VA.getLocReg() == ArgGPRs[0] && "Unexpected reg assignment");
-      SDValue RetValue2 =
-          DAG.getCopyFromReg(Chain, DL, ArgGPRs[1], MVT::i32, Glue);
+      assert(VA.needsCustom());
+      SDValue RetValue2 = DAG.getCopyFromReg(Chain, DL, RVLocs[++i].getLocReg(),
+                                             MVT::i32, Glue);
       Chain = RetValue2.getValue(1);
       Glue = RetValue2.getValue(2);
       RetValue = DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, RetValue,
@@ -17568,21 +17630,21 @@ RISCVTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
   SmallVector<SDValue, 4> RetOps(1, Chain);
 
   // Copy the result values into the output registers.
-  for (unsigned i = 0, e = RVLocs.size(); i < e; ++i) {
-    SDValue Val = OutVals[i];
+  for (unsigned i = 0, e = RVLocs.size(), OutIdx = 0; i < e; ++i, ++OutIdx) {
+    SDValue Val = OutVals[OutIdx];
     CCValAssign &VA = RVLocs[i];
     assert(VA.isRegLoc() && "Can only return in registers!");
 
     if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
       // Handle returning f64 on RV32D with a soft float ABI.
       assert(VA.isRegLoc() && "Expected return via registers");
+      assert(VA.needsCustom());
       SDValue SplitF64 = DAG.getNode(RISCVISD::SplitF64, DL,
                                      DAG.getVTList(MVT::i32, MVT::i32), Val);
       SDValue Lo = SplitF64.getValue(0);
       SDValue Hi = SplitF64.getValue(1);
       Register RegLo = VA.getLocReg();
-      assert(RegLo < RISCV::X31 && "Invalid register pair");
-      Register RegHi = RegLo + 1;
+      Register RegHi = RVLocs[++i].getLocReg();
 
       if (STI.isRegisterReservedByUser(RegLo) ||
           STI.isRegisterReservedByUser(RegHi))
diff --git a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
index 5584fa8d50..4c99da1244 100644
--- a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
@@ -746,6 +746,7 @@ class RISCVInsertVSETVLI : public MachineFunctionPass {
                    const VSETVLIInfo &CurInfo) const;
   bool needVSETVLIPHI(const VSETVLIInfo &Require,
                       const MachineBasicBlock &MBB) const;
+  bool mayChangeVL(const VSETVLIInfo &Info, const VSETVLIInfo &PrevInfo) const;
   void insertVSETVLI(MachineBasicBlock &MBB, MachineInstr &MI,
                      const VSETVLIInfo &Info, const VSETVLIInfo &PrevInfo);
   void insertVSETVLI(MachineBasicBlock &MBB,
@@ -859,41 +860,47 @@ static VSETVLIInfo getInfoForVSETVLI(const MachineInstr &MI) {
   return NewInfo;
 }
 
+/// Return true if a vsetvli instruction to change from PrevInfo
+/// to Info might change the VL register.  If this returns false,
+/// the vsetvli can use the X0, X0 form.
+bool RISCVInsertVSETVLI::mayChangeVL(const VSETVLIInfo &Info,
+                                     const VSETVLIInfo &PrevInfo) const {
+  if (!PrevInfo.isValid() || PrevInfo.isUnknown())
+    return true;
+
+  // If the AVL is the same and the SEW+LMUL gives the same VLMAX, VL
+  // can not change.
+  if (Info.hasSameAVL(PrevInfo) && Info.hasSameVLMAX(PrevInfo))
+    return false;
+
+  // If our AVL is a virtual register, it might be defined by a VSET(I)VLI. If
+  // it has the same VLMAX we want and the last VL/VTYPE we observed is the
+  // same, then VL can not change.
+  if (Info.hasSameVLMAX(PrevInfo) && Info.hasAVLReg() &&
+      Info.getAVLReg().isVirtual()) {
+    if (MachineInstr *DefMI = MRI->getVRegDef(Info.getAVLReg());
+        DefMI && isVectorConfigInstr(*DefMI)) {
+      VSETVLIInfo DefInfo = getInfoForVSETVLI(*DefMI);
+      if (DefInfo.hasSameAVL(PrevInfo) && DefInfo.hasSameVLMAX(PrevInfo))
+        return false;
+    }
+  }
+  return true;
+}
+
 void RISCVInsertVSETVLI::insertVSETVLI(MachineBasicBlock &MBB,
                      MachineBasicBlock::iterator InsertPt, DebugLoc DL,
                      const VSETVLIInfo &Info, const VSETVLIInfo &PrevInfo) {
 
-  if (PrevInfo.isValid() && !PrevInfo.isUnknown()) {
-    // Use X0, X0 form if the AVL is the same and the SEW+LMUL gives the same
-    // VLMAX.
-    if (Info.hasSameAVL(PrevInfo) && Info.hasSameVLMAX(PrevInfo)) {
-      BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETVLIX0))
-          .addReg(RISCV::X0, RegState::Define | RegState::Dead)
-          .addReg(RISCV::X0, RegState::Kill)
-          .addImm(Info.encodeVTYPE())
-          .addReg(RISCV::VL, RegState::Implicit);
-      return;
-    }
-
-    // If our AVL is a virtual register, it might be defined by a VSET(I)VLI. If
-    // it has the same VLMAX we want and the last VL/VTYPE we observed is the
-    // same, we can use the X0, X0 form.
-    if (Info.hasSameVLMAX(PrevInfo) && Info.hasAVLReg() &&
-        Info.getAVLReg().isVirtual()) {
-      if (MachineInstr *DefMI = MRI->getVRegDef(Info.getAVLReg())) {
-        if (isVectorConfigInstr(*DefMI)) {
-          VSETVLIInfo DefInfo = getInfoForVSETVLI(*DefMI);
-          if (DefInfo.hasSameAVL(PrevInfo) && DefInfo.hasSameVLMAX(PrevInfo)) {
-            BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETVLIX0))
-                .addReg(RISCV::X0, RegState::Define | RegState::Dead)
-                .addReg(RISCV::X0, RegState::Kill)
-                .addImm(Info.encodeVTYPE())
-                .addReg(RISCV::VL, RegState::Implicit);
-            return;
-          }
-        }
-      }
-    }
+  // Use X0, X0 form if VL can't change.  Changing only VTYPE is generally
+  // cheaper than changing VL.
+  if (!mayChangeVL(Info, PrevInfo)) {
+    BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETVLIX0))
+        .addReg(RISCV::X0, RegState::Define | RegState::Dead)
+        .addReg(RISCV::X0, RegState::Kill)
+        .addImm(Info.encodeVTYPE())
+        .addReg(RISCV::VL, RegState::Implicit);
+    return;
   }
 
   if (Info.hasAVLImm()) {
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index 11d99fa510..36ca775c21 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -27,6 +27,7 @@
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/MachineTraceMetrics.h"
 #include "llvm/CodeGen/RegisterScavenging.h"
+#include "llvm/CodeGen/StackMaps.h"
 #include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/MC/MCInstBuilder.h"
 #include "llvm/MC/TargetRegistry.h"
@@ -1301,7 +1302,20 @@ unsigned RISCVInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
     if (isCompressibleInst(MI, STI))
       return 2;
   }
-  return get(Opcode).getSize();
+
+  switch (Opcode) {
+  case TargetOpcode::STACKMAP:
+    // The upper bound for a stackmap intrinsic is the full length of its shadow
+    return StackMapOpers(&MI).getNumPatchBytes();
+  case TargetOpcode::PATCHPOINT:
+    // The size of the patchpoint intrinsic is the number of bytes requested
+    return PatchPointOpers(&MI).getNumPatchBytes();
+  case TargetOpcode::STATEPOINT:
+    // The size of the statepoint intrinsic is the number of bytes requested
+    return StatepointOpers(&MI).getNumPatchBytes();
+  default:
+    return get(Opcode).getSize();
+  }
 }
 
 unsigned RISCVInstrInfo::getInstBundleLength(const MachineInstr &MI) const {
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
index 54efe67f60..94de559b1e 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
@@ -169,7 +169,7 @@ class RISCVSImmOp<int bitsNum> : RISCVOp {
   let OperandType = "OPERAND_SIMM" # bitsNum;
 }
 
-class RISCVSImmLeafOp<int bitsNum > :
+class RISCVSImmLeafOp<int bitsNum> :
   RISCVSImmOp<bitsNum>, ImmLeaf<XLenVT, "return isInt<" # bitsNum # ">(Imm);">;
 
 def FenceArg : AsmOperandClass {
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoC.td b/llvm/lib/Target/RISCV/RISCVInstrInfoC.td
index aff6e77e0c..07137031d9 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoC.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoC.td
@@ -956,8 +956,14 @@ def : CompressPat<(JAL X0, simm12_lsb0:$offset),
                   (C_J simm12_lsb0:$offset)>;
 def : CompressPat<(BEQ GPRC:$rs1, X0, simm9_lsb0:$imm),
                   (C_BEQZ GPRC:$rs1, simm9_lsb0:$imm)>;
+let isCompressOnly = true in
+def : CompressPat<(BEQ X0, GPRC:$rs1, simm9_lsb0:$imm),
+                  (C_BEQZ GPRC:$rs1, simm9_lsb0:$imm)>;
 def : CompressPat<(BNE GPRC:$rs1, X0, simm9_lsb0:$imm),
                   (C_BNEZ GPRC:$rs1, simm9_lsb0:$imm)>;
+let isCompressOnly = true in
+def : CompressPat<(BNE X0, GPRC:$rs1, simm9_lsb0:$imm),
+                  (C_BNEZ GPRC:$rs1, simm9_lsb0:$imm)>;
 } // Predicates = [HasStdExtCOrZca]
 
 // Quadrant 2
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td
index fa6a1af6a0..781a34ba85 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td
@@ -167,6 +167,15 @@ multiclass CustomSiFiveVCIX<string suffix, VCIXType type,
                                              InTyRs1, 1>;
 }
 
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in {
+class CustomSiFiveVMACC<bits<6> funct6, RISCVVFormat opv, string opcodestr>
+    : RVInstVCCustom2<funct6{5-2}, opv.Value, (outs VR:$rd), (ins VR:$rs1, VR:$rs2),
+                      opcodestr, "$rd, $rs1, $rs2"> {
+  let vm = 1;
+  let funct6_lo2 = funct6{1-0};
+}
+}
+
 let Predicates = [HasVendorXSfvcp], mayLoad = 0, mayStore = 0,
     hasSideEffects = 1, hasNoSchedulingInfo = 1, DecoderNamespace = "XSfvcp" in {
   defm X   : CustomSiFiveVCIX<"x",   VCIX_X,   uimm5, uimm5, GPR>,   Sched<[]>;
@@ -185,6 +194,20 @@ let Predicates = [HasVendorXSfvcp], mayLoad = 0, mayStore = 0,
   defm FVW : CustomSiFiveVCIX<"fvw", VCIX_XVW, VR,    VR,    FPR32>, Sched<[]>;
 }
 
+let Predicates = [HasVendorXSfvqmaccdod], DecoderNamespace = "XSfvqmaccdod" in {
+  def VQMACCU_2x8x2  : CustomSiFiveVMACC<0b101100, OPMVV, "sf.vqmaccu.2x8x2">;
+  def VQMACC_2x8x2   : CustomSiFiveVMACC<0b101101, OPMVV, "sf.vqmacc.2x8x2">;
+  def VQMACCUS_2x8x2 : CustomSiFiveVMACC<0b101110, OPMVV, "sf.vqmaccus.2x8x2">;
+  def VQMACCSU_2x8x2 : CustomSiFiveVMACC<0b101111, OPMVV, "sf.vqmaccsu.2x8x2">;
+}
+
+let Predicates = [HasVendorXSfvqmaccqoq], DecoderNamespace = "XSfvqmaccqoq" in {
+  def VQMACCU_4x8x4  : CustomSiFiveVMACC<0b111100, OPMVV, "sf.vqmaccu.4x8x4">;
+  def VQMACC_4x8x4   : CustomSiFiveVMACC<0b111101, OPMVV, "sf.vqmacc.4x8x4">;
+  def VQMACCUS_4x8x4 : CustomSiFiveVMACC<0b111110, OPMVV, "sf.vqmaccus.4x8x4">;
+  def VQMACCSU_4x8x4 : CustomSiFiveVMACC<0b111111, OPMVV, "sf.vqmaccsu.4x8x4">;
+}
+
 class VPseudoVC_X<Operand OpClass, DAGOperand RS1Class,
                   bit HasSideEffect = 1> :
       Pseudo<(outs),
@@ -307,6 +330,17 @@ multiclass VPseudoVC_XVW<LMULInfo m, DAGOperand RS1Class,
   }
 }
 
+multiclass VPseudoSiFiveVMACC<string mx, VReg vd_type, VReg vs2_type,
+                              string Constraint = ""> {
+  def "Pseudo" # NAME # "_" # mx
+      : VPseudoTernaryNoMaskWithPolicy<vd_type, V_M1.vrclass, vs2_type, Constraint>;
+}
+
+multiclass VPseudoSiFiveVQMACC<string Constraint = ""> {
+  foreach m = MxListVF8 in
+    defm NAME : VPseudoSiFiveVMACC<m.MX, m.vrclass, m.vrclass, Constraint>;
+}
+
 let Predicates = [HasVendorXSfvcp] in {
   foreach m = MxList in {
     defm X : VPseudoVC_X<m, GPR>;
@@ -335,6 +369,20 @@ let Predicates = [HasVendorXSfvcp] in {
   }
 }
 
+let Predicates = [HasVendorXSfvqmaccdod], DecoderNamespace = "XSfvqmaccdod" in {
+  defm VQMACCU_2x8x2  : VPseudoSiFiveVQMACC;
+  defm VQMACC_2x8x2   : VPseudoSiFiveVQMACC;
+  defm VQMACCUS_2x8x2 : VPseudoSiFiveVQMACC;
+  defm VQMACCSU_2x8x2 : VPseudoSiFiveVQMACC;
+}
+
+let Predicates = [HasVendorXSfvqmaccqoq], DecoderNamespace = "XSfvqmaccqoq" in {
+  defm VQMACCU_4x8x4  : VPseudoSiFiveVQMACC;
+  defm VQMACC_4x8x4   : VPseudoSiFiveVQMACC;
+  defm VQMACCUS_4x8x4 : VPseudoSiFiveVQMACC;
+  defm VQMACCSU_4x8x4 : VPseudoSiFiveVQMACC;
+}
+
 class VPatVC_OP4<string intrinsic_name,
                  string inst,
                  ValueType op2_type,
@@ -465,6 +513,30 @@ class GetFTypeInfo<int Sew> {
                               !eq(Scalar, f64) : "FPR64");
 }
 
+multiclass VPatVMACC<string intrinsic, string instruction, string kind,
+                     list<VTypeInfoToWide> info_pairs, ValueType vec_m1> {
+  foreach pair = info_pairs in {
+    defvar VdInfo = pair.Wti;
+    defvar Vs2Info = pair.Vti;
+    let Predicates = [HasVInstructions] in
+    def : VPatTernaryNoMaskWithPolicy<"int_riscv_sf_" # intrinsic,
+                                      "Pseudo" # instruction, kind, VdInfo.Vector,
+                                      vec_m1, Vs2Info.Vector,
+                                      Vs2Info.Log2SEW, Vs2Info.LMul,
+                                      VdInfo.RegClass, VR, Vs2Info.RegClass>;
+  }
+}
+
+defset list<VTypeInfoToWide> VQMACCInfoPairs = {
+  def : VTypeInfoToWide<VI8M1, VI32M1>;
+  def : VTypeInfoToWide<VI8M2, VI32M2>;
+  def : VTypeInfoToWide<VI8M4, VI32M4>;
+  def : VTypeInfoToWide<VI8M8, VI32M8>;
+}
+
+multiclass VPatVQMACC<string intrinsic, string instruction, string kind>
+    : VPatVMACC<intrinsic, instruction, kind, VQMACCInfoPairs, vint8m1_t>;
+
 let Predicates = [HasVendorXSfvcp] in {
   foreach vti = AllVectors in {
     defm : VPatVC_X<"x", "X", vti, XLenVT, GPR>;
@@ -500,6 +572,20 @@ let Predicates = [HasVendorXSfvcp] in {
   }
 }
 
+let Predicates = [HasVendorXSfvqmaccdod] in {
+  defm : VPatVQMACC<"vqmaccu_2x8x2", "VQMACCU", "2x8x2">;
+  defm : VPatVQMACC<"vqmacc_2x8x2", "VQMACC", "2x8x2">;
+  defm : VPatVQMACC<"vqmaccus_2x8x2", "VQMACCUS", "2x8x2">;
+  defm : VPatVQMACC<"vqmaccsu_2x8x2", "VQMACCSU", "2x8x2">;
+}
+
+let Predicates = [HasVendorXSfvqmaccqoq] in {
+  defm : VPatVQMACC<"vqmaccu_4x8x4", "VQMACCU", "4x8x4">;
+  defm : VPatVQMACC<"vqmacc_4x8x4", "VQMACC", "4x8x4">;
+  defm : VPatVQMACC<"vqmaccus_4x8x4", "VQMACCUS", "4x8x4">;
+  defm : VPatVQMACC<"vqmaccsu_4x8x4", "VQMACCSU", "4x8x4">;
+}
+
 let Predicates = [HasVendorXSfcie] in {
 let hasSideEffects = 1, mayLoad = 0, mayStore = 0, DecoderNamespace = "XSfcie" in {
 def SF_CFLUSH_D_L1 : RVInstI<0b000, OPC_SYSTEM, (outs), (ins GPR:$rs1), "cflush.d.l1","$rs1">,
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
index cc92edc9b2..a7572e908b 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
@@ -231,11 +231,39 @@ def SimmShiftRightBy3XForm : SDNodeXForm<imm, [{
 }]>;
 
 // Pattern to exclude simm12 immediates from matching.
+// Note: this will be removed once the GISel complex patterns for
+// SHXADD_UW is landed.
 def non_imm12 : PatLeaf<(XLenVT GPR:$a), [{
   auto *C = dyn_cast<ConstantSDNode>(N);
   return !C || !isInt<12>(C->getSExtValue());
 }]>;
 
+// GISel currently doesn't support PatFrag for leaf nodes, so `non_imm12`
+// cannot be directly supported in GISel. To reuse patterns between the two
+// ISels, we instead create PatFrag on operators that use `non_imm12`.
+class binop_with_non_imm12<SDPatternOperator binop>
+  : PatFrag<(ops node:$x, node:$y), (binop node:$x, node:$y), [{
+  auto *C = dyn_cast<ConstantSDNode>(Operands[1]);
+  return !C || !isInt<12>(C->getSExtValue());
+}]> {
+  let PredicateCodeUsesOperands = 1;
+  let GISelPredicateCode = [{
+    const MachineOperand &ImmOp = *Operands[1];
+    const MachineFunction &MF = *MI.getParent()->getParent();
+    const MachineRegisterInfo &MRI = MF.getRegInfo();
+
+    if (ImmOp.isReg() && ImmOp.getReg())
+      if (auto Val = getIConstantVRegValWithLookThrough(ImmOp.getReg(), MRI)) {
+        // We do NOT want immediates that fit in 12 bits.
+        return !isInt<12>(Val->Value.getSExtValue());
+      }
+
+    return true;
+  }];
+}
+def add_non_imm12       : binop_with_non_imm12<add>;
+def or_is_add_non_imm12 : binop_with_non_imm12<or_is_add>;
+
 def Shifted32OnesMask : PatLeaf<(imm), [{
   uint64_t Imm = N->getZExtValue();
   if (!isShiftedMask_64(Imm))
@@ -647,20 +675,17 @@ let Predicates = [HasStdExtZbb, IsRV64] in
 def : Pat<(i64 (and GPR:$rs, 0xFFFF)), (ZEXT_H_RV64 GPR:$rs)>;
 
 let Predicates = [HasStdExtZba] in {
-def : Pat<(add (shl GPR:$rs1, (XLenVT 1)), non_imm12:$rs2),
-          (SH1ADD GPR:$rs1, GPR:$rs2)>;
-def : Pat<(add (shl GPR:$rs1, (XLenVT 2)), non_imm12:$rs2),
-          (SH2ADD GPR:$rs1, GPR:$rs2)>;
-def : Pat<(add (shl GPR:$rs1, (XLenVT 3)), non_imm12:$rs2),
-          (SH3ADD GPR:$rs1, GPR:$rs2)>;
 
-// More complex cases use a ComplexPattern.
-def : Pat<(add sh1add_op:$rs1, non_imm12:$rs2),
-          (SH1ADD sh1add_op:$rs1, GPR:$rs2)>;
-def : Pat<(add sh2add_op:$rs1, non_imm12:$rs2),
-          (SH2ADD sh2add_op:$rs1, GPR:$rs2)>;
-def : Pat<(add sh3add_op:$rs1, non_imm12:$rs2),
-          (SH3ADD sh3add_op:$rs1, GPR:$rs2)>;
+foreach i = {1,2,3} in {
+  defvar shxadd = !cast<Instruction>("SH"#i#"ADD");
+  def : Pat<(XLenVT (add_non_imm12 (shl GPR:$rs1, (XLenVT i)), GPR:$rs2)),
+            (shxadd GPR:$rs1, GPR:$rs2)>;
+
+  defvar pat = !cast<ComplexPattern>("sh"#i#"add_op");
+  // More complex cases use a ComplexPattern.
+  def : Pat<(XLenVT (add_non_imm12 pat:$rs1, GPR:$rs2)),
+            (shxadd pat:$rs1, GPR:$rs2)>;
+}
 
 def : Pat<(add (mul_oneuse GPR:$rs1, (XLenVT 6)), GPR:$rs2),
           (SH1ADD (SH1ADD GPR:$rs1, GPR:$rs1), GPR:$rs2)>;
@@ -730,26 +755,24 @@ def : Pat<(i64 (shl (and GPR:$rs1, 0xFFFFFFFF), uimm5:$shamt)),
 def : Pat<(i64 (and GPR:$rs1, Shifted32OnesMask:$mask)),
           (SLLI_UW (SRLI GPR:$rs1, Shifted32OnesMask:$mask),
                    Shifted32OnesMask:$mask)>;
-
-def : Pat<(i64 (add (and GPR:$rs1, 0xFFFFFFFF), non_imm12:$rs2)),
+def : Pat<(i64 (add_non_imm12 (and GPR:$rs1, 0xFFFFFFFF), GPR:$rs2)),
           (ADD_UW GPR:$rs1, GPR:$rs2)>;
 def : Pat<(i64 (and GPR:$rs, 0xFFFFFFFF)), (ADD_UW GPR:$rs, (XLenVT X0))>;
 
-def : Pat<(i64 (or_is_add (and GPR:$rs1, 0xFFFFFFFF), non_imm12:$rs2)),
+def : Pat<(i64 (or_is_add_non_imm12 (and GPR:$rs1, 0xFFFFFFFF), GPR:$rs2)),
           (ADD_UW GPR:$rs1, GPR:$rs2)>;
 
-def : Pat<(i64 (add (shl (and GPR:$rs1, 0xFFFFFFFF), (i64 1)), non_imm12:$rs2)),
-          (SH1ADD_UW GPR:$rs1, GPR:$rs2)>;
-def : Pat<(i64 (add (shl (and GPR:$rs1, 0xFFFFFFFF), (i64 2)), non_imm12:$rs2)),
-          (SH2ADD_UW GPR:$rs1, GPR:$rs2)>;
-def : Pat<(i64 (add (shl (and GPR:$rs1, 0xFFFFFFFF), (i64 3)), non_imm12:$rs2)),
-          (SH3ADD_UW GPR:$rs1, GPR:$rs2)>;
+foreach i = {1,2,3} in {
+  defvar shxadd_uw = !cast<Instruction>("SH"#i#"ADD_UW");
+  def : Pat<(i64 (add_non_imm12 (shl (and GPR:$rs1, 0xFFFFFFFF), (i64 i)), (XLenVT GPR:$rs2))),
+            (shxadd_uw GPR:$rs1, GPR:$rs2)>;
+}
 
-def : Pat<(i64 (add (and (shl GPR:$rs1, (i64 1)), 0x1FFFFFFFF), non_imm12:$rs2)),
+def : Pat<(i64 (add_non_imm12 (and (shl GPR:$rs1, (i64 1)), 0x1FFFFFFFF), (XLenVT GPR:$rs2))),
           (SH1ADD_UW GPR:$rs1, GPR:$rs2)>;
-def : Pat<(i64 (add (and (shl GPR:$rs1, (i64 2)), 0x3FFFFFFFF), non_imm12:$rs2)),
+def : Pat<(i64 (add_non_imm12 (and (shl GPR:$rs1, (i64 2)), 0x3FFFFFFFF), (XLenVT GPR:$rs2))),
           (SH2ADD_UW GPR:$rs1, GPR:$rs2)>;
-def : Pat<(i64 (add (and (shl GPR:$rs1, (i64 3)), 0x7FFFFFFFF), non_imm12:$rs2)),
+def : Pat<(i64 (add_non_imm12 (and (shl GPR:$rs1, (i64 3)), 0x7FFFFFFFF), (XLenVT GPR:$rs2))),
           (SH3ADD_UW GPR:$rs1, GPR:$rs2)>;
 
 // More complex cases use a ComplexPattern.
@@ -760,19 +783,19 @@ def : Pat<(i64 (add sh2add_uw_op:$rs1, non_imm12:$rs2)),
 def : Pat<(i64 (add sh3add_uw_op:$rs1, non_imm12:$rs2)),
           (SH3ADD_UW sh3add_uw_op:$rs1, GPR:$rs2)>;
 
-def : Pat<(i64 (add (and GPR:$rs1, 0xFFFFFFFE), non_imm12:$rs2)),
+def : Pat<(i64 (add_non_imm12 (and GPR:$rs1, 0xFFFFFFFE), (XLenVT GPR:$rs2))),
           (SH1ADD (SRLIW GPR:$rs1, 1), GPR:$rs2)>;
-def : Pat<(i64 (add (and GPR:$rs1, 0xFFFFFFFC), non_imm12:$rs2)),
+def : Pat<(i64 (add_non_imm12 (and GPR:$rs1, 0xFFFFFFFC), (XLenVT GPR:$rs2))),
           (SH2ADD (SRLIW GPR:$rs1, 2), GPR:$rs2)>;
-def : Pat<(i64 (add (and GPR:$rs1, 0xFFFFFFF8), non_imm12:$rs2)),
+def : Pat<(i64 (add_non_imm12 (and GPR:$rs1, 0xFFFFFFF8), (XLenVT GPR:$rs2))),
           (SH3ADD (SRLIW GPR:$rs1, 3), GPR:$rs2)>;
 
 // Use SRLI to clear the LSBs and SHXADD_UW to mask and shift.
-def : Pat<(i64 (add (and GPR:$rs1, 0x1FFFFFFFE), non_imm12:$rs2)),
+def : Pat<(i64 (add_non_imm12 (and GPR:$rs1, 0x1FFFFFFFE), (XLenVT GPR:$rs2))),
           (SH1ADD_UW (SRLI GPR:$rs1, 1), GPR:$rs2)>;
-def : Pat<(i64 (add (and GPR:$rs1, 0x3FFFFFFFC), non_imm12:$rs2)),
+def : Pat<(i64 (add_non_imm12 (and GPR:$rs1, 0x3FFFFFFFC), (XLenVT GPR:$rs2))),
           (SH2ADD_UW (SRLI GPR:$rs1, 2), GPR:$rs2)>;
-def : Pat<(i64 (add (and GPR:$rs1, 0x7FFFFFFF8), non_imm12:$rs2)),
+def : Pat<(i64 (add_non_imm12 (and GPR:$rs1, 0x7FFFFFFF8), (XLenVT GPR:$rs2))),
           (SH3ADD_UW (SRLI GPR:$rs1, 3), GPR:$rs2)>;
 
 def : Pat<(i64 (mul (and_oneuse GPR:$r, 0xFFFFFFFF), C3LeftShiftUW:$i)),
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZfa.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZfa.td
index 5d6e8821b8..6f88ff7f7a 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZfa.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZfa.td
@@ -112,6 +112,7 @@ def FROUND_D : FPUnaryOp_r_frm<0b0100001, 0b00100, FPR64, FPR64, "fround.d">,
 def FROUNDNX_D : FPUnaryOp_r_frm<0b0100001, 0b00101, FPR64, FPR64, "froundnx.d">,
                  Sched<[WriteFRoundF64, ReadFRoundF64]>;
 
+let IsSignExtendingOpW = 1 in
 def FCVTMOD_W_D
     : FPUnaryOp_r_rtz<0b1100001, 0b01000, GPR, FPR64, "fcvtmod.w.d">,
       Sched<[WriteFCvtF64ToI32, ReadFCvtF64ToI32]>;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZicbo.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZicbo.td
index 537f8c0326..56b68e324d 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZicbo.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZicbo.td
@@ -73,12 +73,16 @@ def PREFETCH_W : Prefetch_ri<0b00011, "prefetch.w">, Sched<[]>;
 // Patterns
 //===----------------------------------------------------------------------===//
 
+def AddrRegImmLsb00000 : ComplexPattern<iPTR, 2, "SelectAddrRegImmLsb00000">;
+
 let Predicates = [HasStdExtZicbop] in {
-  // FIXME: Match address with offset
-  def : Pat<(prefetch GPR:$rs1, timm, timm, (i32 0)),
-            (PREFETCH_I GPR:$rs1, 0)>;
-  def : Pat<(prefetch GPR:$rs1, (i32 0), timm, (i32 1)),
-            (PREFETCH_R GPR:$rs1, 0)>;
-  def : Pat<(prefetch GPR:$rs1, (i32 1), timm, (i32 1)),
-            (PREFETCH_W GPR:$rs1, 0)>;
+  def : Pat<(prefetch (AddrRegImmLsb00000 (XLenVT GPR:$rs1), simm12_lsb00000:$imm12),
+                      timm, timm, (i32 0)),
+            (PREFETCH_I GPR:$rs1, simm12_lsb00000:$imm12)>;
+  def : Pat<(prefetch (AddrRegImmLsb00000 (XLenVT GPR:$rs1), simm12_lsb00000:$imm12),
+                      (i32 0), timm, (i32 1)),
+            (PREFETCH_R GPR:$rs1, simm12_lsb00000:$imm12)>;
+  def : Pat<(prefetch (AddrRegImmLsb00000 (XLenVT GPR:$rs1), simm12_lsb00000:$imm12),
+                      (i32 1), timm, (i32 1)),
+            (PREFETCH_W GPR:$rs1, simm12_lsb00000:$imm12)>;
 }
diff --git a/llvm/lib/Target/RISCV/RISCVMacroFusion.cpp b/llvm/lib/Target/RISCV/RISCVMacroFusion.cpp
index da10465768..02a8d5c18f 100644
--- a/llvm/lib/Target/RISCV/RISCVMacroFusion.cpp
+++ b/llvm/lib/Target/RISCV/RISCVMacroFusion.cpp
@@ -35,10 +35,6 @@ static bool isLUIADDI(const MachineInstr *FirstMI,
   if (FirstMI->getOpcode() != RISCV::LUI)
     return false;
 
-  // The first operand of ADDI might be a frame index.
-  if (!SecondMI.getOperand(1).isReg())
-    return false;
-
   Register FirstDest = FirstMI->getOperand(0).getReg();
 
   // Destination of LUI should be the ADDI(W) source register.
diff --git a/llvm/lib/Target/RISCV/RISCVMoveMerger.cpp b/llvm/lib/Target/RISCV/RISCVMoveMerger.cpp
index 6c1b0cf5ca..934a2a0f8e 100644
--- a/llvm/lib/Target/RISCV/RISCVMoveMerger.cpp
+++ b/llvm/lib/Target/RISCV/RISCVMoveMerger.cpp
@@ -1,4 +1,4 @@
-//===-- RISCVMoveMerger.cpp - RISCV move merge pass -----------------------===//
+//===-- RISCVMoveMerger.cpp - RISC-V move merge pass ----------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp b/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp
index a33ad8a194..63be3f88d4 100644
--- a/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp
+++ b/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp
@@ -602,25 +602,23 @@ bool RISCVOptWInstrs::removeSExtWInstrs(MachineFunction &MF,
 
   bool MadeChange = false;
   for (MachineBasicBlock &MBB : MF) {
-    for (auto I = MBB.begin(), IE = MBB.end(); I != IE;) {
-      MachineInstr *MI = &*I++;
-
+    for (MachineInstr &MI : llvm::make_early_inc_range(MBB)) {
       // We're looking for the sext.w pattern ADDIW rd, rs1, 0.
-      if (!RISCV::isSEXT_W(*MI))
+      if (!RISCV::isSEXT_W(MI))
         continue;
 
-      Register SrcReg = MI->getOperand(1).getReg();
+      Register SrcReg = MI.getOperand(1).getReg();
 
       SmallPtrSet<MachineInstr *, 4> FixableDefs;
 
       // If all users only use the lower bits, this sext.w is redundant.
       // Or if all definitions reaching MI sign-extend their output,
       // then sext.w is redundant.
-      if (!hasAllWUsers(*MI, ST, MRI) &&
+      if (!hasAllWUsers(MI, ST, MRI) &&
           !isSignExtendedW(SrcReg, ST, MRI, FixableDefs))
         continue;
 
-      Register DstReg = MI->getOperand(0).getReg();
+      Register DstReg = MI.getOperand(0).getReg();
       if (!MRI.constrainRegClass(SrcReg, MRI.getRegClass(DstReg)))
         continue;
 
@@ -638,7 +636,7 @@ bool RISCVOptWInstrs::removeSExtWInstrs(MachineFunction &MF,
       LLVM_DEBUG(dbgs() << "Removing redundant sign-extension\n");
       MRI.replaceRegWith(DstReg, SrcReg);
       MRI.clearKillFlags(SrcReg);
-      MI->eraseFromParent();
+      MI.eraseFromParent();
       ++NumRemovedSExtW;
       MadeChange = true;
     }
@@ -656,9 +654,7 @@ bool RISCVOptWInstrs::stripWSuffixes(MachineFunction &MF,
 
   bool MadeChange = false;
   for (MachineBasicBlock &MBB : MF) {
-    for (auto I = MBB.begin(), IE = MBB.end(); I != IE; ++I) {
-      MachineInstr &MI = *I;
-
+    for (MachineInstr &MI : MBB) {
       unsigned Opc;
       switch (MI.getOpcode()) {
       default:
diff --git a/llvm/lib/Target/RISCV/RISCVPushPopOptimizer.cpp b/llvm/lib/Target/RISCV/RISCVPushPopOptimizer.cpp
index 5e8a4a351b..c2c795ec94 100644
--- a/llvm/lib/Target/RISCV/RISCVPushPopOptimizer.cpp
+++ b/llvm/lib/Target/RISCV/RISCVPushPopOptimizer.cpp
@@ -1,4 +1,4 @@
-//===------- RISCVPushPopOptimizer.cpp - RISCV Push/Pop opt. pass ---------===//
+//===------- RISCVPushPopOptimizer.cpp - RISC-V Push/Pop opt. pass --------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
index 3e44cf4781..fcfc5c7821 100644
--- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
@@ -436,9 +436,16 @@ bool RISCVRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
       // offset can by construction, at worst, a LUI and a ADD.
       int64_t Val = Offset.getFixed();
       int64_t Lo12 = SignExtend64<12>(Val);
-      MI.getOperand(FIOperandNum + 1).ChangeToImmediate(Lo12);
-      Offset = StackOffset::get((uint64_t)Val - (uint64_t)Lo12,
-                                Offset.getScalable());
+      if ((MI.getOpcode() == RISCV::PREFETCH_I ||
+           MI.getOpcode() == RISCV::PREFETCH_R ||
+           MI.getOpcode() == RISCV::PREFETCH_W) &&
+          (Lo12 & 0b11111) != 0)
+        MI.getOperand(FIOperandNum + 1).ChangeToImmediate(0);
+      else {
+        MI.getOperand(FIOperandNum + 1).ChangeToImmediate(Lo12);
+        Offset = StackOffset::get((uint64_t)Val - (uint64_t)Lo12,
+                                  Offset.getScalable());
+      }
     }
   }
 
diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.td b/llvm/lib/Target/RISCV/RISCVRegisterInfo.td
index ab0d354967..c59c9b294d 100644
--- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.td
@@ -351,7 +351,7 @@ class NFList<int lmul> {
 // Generate [start, end) SubRegIndex list.
 class SubRegSet<int nf, int lmul> {
   list<SubRegIndex> L = !foldl([]<SubRegIndex>,
-                               [0, 1, 2, 3, 4, 5, 6, 7],
+                               !range(0, 8),
                                AccList, i,
                                !listconcat(AccList,
                                  !if(!lt(i, nf),
@@ -379,15 +379,9 @@ class IndexSet<int tuple_index, int nf, int lmul, bit isV0 = false> {
     !foldl([]<int>,
               !if(isV0, [0],
                 !cond(
-                  !eq(lmul, 1):
-                  [8, 9, 10, 11, 12, 13, 14, 15,
-                   16, 17, 18, 19, 20, 21, 22, 23,
-                   24, 25, 26, 27, 28, 29, 30, 31,
-                   1, 2, 3, 4, 5, 6, 7],
-                  !eq(lmul, 2):
-                  [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 1, 2, 3],
-                  !eq(lmul, 4):
-                  [2, 3, 4, 5, 6, 7, 1])),
+                  !eq(lmul, 1): !listconcat(!range(8, 32), !range(1, 8)),
+                  !eq(lmul, 2): !listconcat(!range(4, 16), !range(1, 4)),
+                  !eq(lmul, 4): !listconcat(!range(2, 8), !range(1, 2)))),
               L, i,
               !listconcat(L,
                           !if(!le(!mul(!add(i, tuple_index), lmul),
@@ -417,12 +411,11 @@ class VRegList<list<dag> LIn, int start, int nf, int lmul, bit isV0> {
 }
 
 // Vector registers
-foreach Index = 0-31 in {
+foreach Index = !range(0, 32, 1) in {
   def V#Index : RISCVReg<Index, "v"#Index>, DwarfRegNum<[!add(Index, 96)]>;
 }
 
-foreach Index = [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22,
-                 24, 26, 28, 30] in {
+foreach Index = !range(0, 32, 2) in {
   def V#Index#M2 : RISCVRegWithSubRegs<Index, "v"#Index,
                      [!cast<Register>("V"#Index),
                       !cast<Register>("V"#!add(Index, 1))]>,
@@ -431,7 +424,7 @@ foreach Index = [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22,
   }
 }
 
-foreach Index = [0, 4, 8, 12, 16, 20, 24, 28] in {
+foreach Index = !range(0, 32, 4) in {
   def V#Index#M4 : RISCVRegWithSubRegs<Index, "v"#Index,
                      [!cast<Register>("V"#Index#"M2"),
                       !cast<Register>("V"#!add(Index, 2)#"M2")]>,
@@ -440,7 +433,7 @@ foreach Index = [0, 4, 8, 12, 16, 20, 24, 28] in {
   }
 }
 
-foreach Index = [0, 8, 16, 24] in {
+foreach Index = !range(0, 32, 8) in {
   def V#Index#M8 : RISCVRegWithSubRegs<Index, "v"#Index,
                      [!cast<Register>("V"#Index#"M4"),
                       !cast<Register>("V"#!add(Index, 4)#"M4")]>,
diff --git a/llvm/lib/Target/RISCV/RISCVSubtarget.h b/llvm/lib/Target/RISCV/RISCVSubtarget.h
index 027d32d541..6b915e61c1 100644
--- a/llvm/lib/Target/RISCV/RISCVSubtarget.h
+++ b/llvm/lib/Target/RISCV/RISCVSubtarget.h
@@ -107,6 +107,10 @@ class RISCVSubtarget : public RISCVGenSubtargetInfo {
   }
   bool enableMachineScheduler() const override { return true; }
 
+  bool enablePostRAScheduler() const override {
+    return getSchedModel().PostRAScheduler || UsePostRAScheduler;
+  }
+
   Align getPrefFunctionAlignment() const {
     return Align(TuneInfo->PrefFunctionAlignment);
   }
diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
index 1281528ea5..651d24bae5 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
@@ -248,6 +248,8 @@ class RISCVPassConfig : public TargetPassConfig {
 public:
   RISCVPassConfig(RISCVTargetMachine &TM, PassManagerBase &PM)
       : TargetPassConfig(TM, PM) {
+    if (TM.getOptLevel() != CodeGenOptLevel::None)
+      substitutePass(&PostRASchedulerID, &PostMachineSchedulerID);
     setEnableSinkAndFold(EnableSinkFold);
   }
 
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index e947d4d1e8..09f9b6035a 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -668,6 +668,31 @@ static const CostTblEntry VectorIntrinsicCostTable[]{
     {Intrinsic::rint, MVT::nxv2f64, 7},
     {Intrinsic::rint, MVT::nxv4f64, 7},
     {Intrinsic::rint, MVT::nxv8f64, 7},
+    {Intrinsic::lrint, MVT::v2i32, 1},
+    {Intrinsic::lrint, MVT::v4i32, 1},
+    {Intrinsic::lrint, MVT::v8i32, 1},
+    {Intrinsic::lrint, MVT::v16i32, 1},
+    {Intrinsic::lrint, MVT::nxv1i32, 1},
+    {Intrinsic::lrint, MVT::nxv2i32, 1},
+    {Intrinsic::lrint, MVT::nxv4i32, 1},
+    {Intrinsic::lrint, MVT::nxv8i32, 1},
+    {Intrinsic::lrint, MVT::nxv16i32, 1},
+    {Intrinsic::lrint, MVT::v2i64, 1},
+    {Intrinsic::lrint, MVT::v4i64, 1},
+    {Intrinsic::lrint, MVT::v8i64, 1},
+    {Intrinsic::lrint, MVT::v16i64, 1},
+    {Intrinsic::lrint, MVT::nxv1i64, 1},
+    {Intrinsic::lrint, MVT::nxv2i64, 1},
+    {Intrinsic::lrint, MVT::nxv4i64, 1},
+    {Intrinsic::lrint, MVT::nxv8i64, 1},
+    {Intrinsic::llrint, MVT::v2i64, 1},
+    {Intrinsic::llrint, MVT::v4i64, 1},
+    {Intrinsic::llrint, MVT::v8i64, 1},
+    {Intrinsic::llrint, MVT::v16i64, 1},
+    {Intrinsic::llrint, MVT::nxv1i64, 1},
+    {Intrinsic::llrint, MVT::nxv2i64, 1},
+    {Intrinsic::llrint, MVT::nxv4i64, 1},
+    {Intrinsic::llrint, MVT::nxv8i64, 1},
     {Intrinsic::nearbyint, MVT::v2f32, 9},
     {Intrinsic::nearbyint, MVT::v4f32, 9},
     {Intrinsic::nearbyint, MVT::v8f32, 9},
@@ -1051,6 +1076,8 @@ RISCVTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
   case Intrinsic::floor:
   case Intrinsic::trunc:
   case Intrinsic::rint:
+  case Intrinsic::lrint:
+  case Intrinsic::llrint:
   case Intrinsic::round:
   case Intrinsic::roundeven: {
     // These all use the same code.
diff --git a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVAsmBackend.cpp b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVAsmBackend.cpp
index 4156a00264..1b80e4b927 100644
--- a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVAsmBackend.cpp
+++ b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVAsmBackend.cpp
@@ -18,7 +18,7 @@ namespace {
 
 class SPIRVAsmBackend : public MCAsmBackend {
 public:
-  SPIRVAsmBackend(support::endianness Endian) : MCAsmBackend(Endian) {}
+  SPIRVAsmBackend(llvm::endianness Endian) : MCAsmBackend(Endian) {}
 
   void applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
                   const MCValue &Target, MutableArrayRef<char> Data,
@@ -59,5 +59,5 @@ MCAsmBackend *llvm::createSPIRVAsmBackend(const Target &T,
                                           const MCSubtargetInfo &STI,
                                           const MCRegisterInfo &MRI,
                                           const MCTargetOptions &) {
-  return new SPIRVAsmBackend(support::little);
+  return new SPIRVAsmBackend(llvm::endianness::little);
 }
diff --git a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVMCCodeEmitter.cpp b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVMCCodeEmitter.cpp
index 60b68dea93..8aea26d996 100644
--- a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVMCCodeEmitter.cpp
+++ b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVMCCodeEmitter.cpp
@@ -78,10 +78,10 @@ static void emitOperand(const MCOperand &Op, SmallVectorImpl<char> &CB) {
   if (Op.isReg()) {
     // Emit the id index starting at 1 (0 is an invalid index).
     support::endian::write<uint32_t>(
-        CB, Register::virtReg2Index(Op.getReg()) + 1, support::little);
+        CB, Register::virtReg2Index(Op.getReg()) + 1, llvm::endianness::little);
   } else if (Op.isImm()) {
     support::endian::write(CB, static_cast<uint32_t>(Op.getImm()),
-                           support::little);
+                           llvm::endianness::little);
   } else {
     llvm_unreachable("Unexpected operand type in VReg");
   }
@@ -113,7 +113,7 @@ void SPIRVMCCodeEmitter::encodeInstruction(const MCInst &MI,
   const uint64_t OpCode = getBinaryCodeForInstr(MI, Fixups, STI);
   const uint32_t NumWords = MI.getNumOperands() + 1;
   const uint32_t FirstWord = (NumWords << 16) | OpCode;
-  support::endian::write(CB, FirstWord, support::little);
+  support::endian::write(CB, FirstWord, llvm::endianness::little);
 
   // Emit the instruction arguments (emitting the output type first if present).
   if (hasType(MI, MCII))
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
index b4452d0f46..d6354876b5 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
@@ -2010,60 +2010,6 @@ static Type *parseTypeString(const StringRef Name, LLVMContext &Context) {
   llvm_unreachable("Unable to recognize type!");
 }
 
-static const TargetExtType *parseToTargetExtType(const Type *OpaqueType,
-                                                 MachineIRBuilder &MIRBuilder) {
-  assert(isSpecialOpaqueType(OpaqueType) &&
-         "Not a SPIR-V/OpenCL special opaque type!");
-  assert(!OpaqueType->isTargetExtTy() &&
-         "This already is SPIR-V/OpenCL TargetExtType!");
-
-  StringRef NameWithParameters = OpaqueType->getStructName();
-
-  // Pointers-to-opaque-structs representing OpenCL types are first translated
-  // to equivalent SPIR-V types. OpenCL builtin type names should have the
-  // following format: e.g. %opencl.event_t
-  if (NameWithParameters.startswith("opencl.")) {
-    const SPIRV::OpenCLType *OCLTypeRecord =
-        SPIRV::lookupOpenCLType(NameWithParameters);
-    if (!OCLTypeRecord)
-      report_fatal_error("Missing TableGen record for OpenCL type: " +
-                         NameWithParameters);
-    NameWithParameters = OCLTypeRecord->SpirvTypeLiteral;
-    // Continue with the SPIR-V builtin type...
-  }
-
-  // Names of the opaque structs representing a SPIR-V builtins without
-  // parameters should have the following format: e.g. %spirv.Event
-  assert(NameWithParameters.startswith("spirv.") &&
-         "Unknown builtin opaque type!");
-
-  // Parameterized SPIR-V builtins names follow this format:
-  // e.g. %spirv.Image._void_1_0_0_0_0_0_0, %spirv.Pipe._0
-  if (NameWithParameters.find('_') == std::string::npos)
-    return TargetExtType::get(OpaqueType->getContext(), NameWithParameters);
-
-  SmallVector<StringRef> Parameters;
-  unsigned BaseNameLength = NameWithParameters.find('_') - 1;
-  SplitString(NameWithParameters.substr(BaseNameLength + 1), Parameters, "_");
-
-  SmallVector<Type *, 1> TypeParameters;
-  bool HasTypeParameter = !isDigit(Parameters[0][0]);
-  if (HasTypeParameter)
-    TypeParameters.push_back(parseTypeString(
-        Parameters[0], MIRBuilder.getMF().getFunction().getContext()));
-  SmallVector<unsigned> IntParameters;
-  for (unsigned i = HasTypeParameter ? 1 : 0; i < Parameters.size(); i++) {
-    unsigned IntParameter = 0;
-    bool ValidLiteral = !Parameters[i].getAsInteger(10, IntParameter);
-    assert(ValidLiteral &&
-           "Invalid format of SPIR-V builtin parameter literal!");
-    IntParameters.push_back(IntParameter);
-  }
-  return TargetExtType::get(OpaqueType->getContext(),
-                            NameWithParameters.substr(0, BaseNameLength),
-                            TypeParameters, IntParameters);
-}
-
 //===----------------------------------------------------------------------===//
 // Implementation functions for builtin types.
 //===----------------------------------------------------------------------===//
@@ -2127,6 +2073,56 @@ static SPIRVType *getSampledImageType(const TargetExtType *OpaqueType,
 }
 
 namespace SPIRV {
+const TargetExtType *
+parseBuiltinTypeNameToTargetExtType(std::string TypeName,
+                                    MachineIRBuilder &MIRBuilder) {
+  StringRef NameWithParameters = TypeName;
+
+  // Pointers-to-opaque-structs representing OpenCL types are first translated
+  // to equivalent SPIR-V types. OpenCL builtin type names should have the
+  // following format: e.g. %opencl.event_t
+  if (NameWithParameters.startswith("opencl.")) {
+    const SPIRV::OpenCLType *OCLTypeRecord =
+        SPIRV::lookupOpenCLType(NameWithParameters);
+    if (!OCLTypeRecord)
+      report_fatal_error("Missing TableGen record for OpenCL type: " +
+                         NameWithParameters);
+    NameWithParameters = OCLTypeRecord->SpirvTypeLiteral;
+    // Continue with the SPIR-V builtin type...
+  }
+
+  // Names of the opaque structs representing a SPIR-V builtins without
+  // parameters should have the following format: e.g. %spirv.Event
+  assert(NameWithParameters.startswith("spirv.") &&
+         "Unknown builtin opaque type!");
+
+  // Parameterized SPIR-V builtins names follow this format:
+  // e.g. %spirv.Image._void_1_0_0_0_0_0_0, %spirv.Pipe._0
+  if (NameWithParameters.find('_') == std::string::npos)
+    return TargetExtType::get(MIRBuilder.getContext(), NameWithParameters);
+
+  SmallVector<StringRef> Parameters;
+  unsigned BaseNameLength = NameWithParameters.find('_') - 1;
+  SplitString(NameWithParameters.substr(BaseNameLength + 1), Parameters, "_");
+
+  SmallVector<Type *, 1> TypeParameters;
+  bool HasTypeParameter = !isDigit(Parameters[0][0]);
+  if (HasTypeParameter)
+    TypeParameters.push_back(parseTypeString(
+        Parameters[0], MIRBuilder.getMF().getFunction().getContext()));
+  SmallVector<unsigned> IntParameters;
+  for (unsigned i = HasTypeParameter ? 1 : 0; i < Parameters.size(); i++) {
+    unsigned IntParameter = 0;
+    bool ValidLiteral = !Parameters[i].getAsInteger(10, IntParameter);
+    assert(ValidLiteral &&
+           "Invalid format of SPIR-V builtin parameter literal!");
+    IntParameters.push_back(IntParameter);
+  }
+  return TargetExtType::get(MIRBuilder.getContext(),
+                            NameWithParameters.substr(0, BaseNameLength),
+                            TypeParameters, IntParameters);
+}
+
 SPIRVType *lowerBuiltinType(const Type *OpaqueType,
                             SPIRV::AccessQualifier::AccessQualifier AccessQual,
                             MachineIRBuilder &MIRBuilder,
@@ -2141,7 +2137,8 @@ SPIRVType *lowerBuiltinType(const Type *OpaqueType,
   // will be removed in the future release of LLVM.
   const TargetExtType *BuiltinType = dyn_cast<TargetExtType>(OpaqueType);
   if (!BuiltinType)
-    BuiltinType = parseToTargetExtType(OpaqueType, MIRBuilder);
+    BuiltinType = parseBuiltinTypeNameToTargetExtType(
+        OpaqueType->getStructName().str(), MIRBuilder);
 
   unsigned NumStartingVRegs = MIRBuilder.getMRI()->getNumVirtRegs();
 
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.h b/llvm/lib/Target/SPIRV/SPIRVBuiltins.h
index 7ee5c49dc5..6f95729546 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.h
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.h
@@ -37,6 +37,18 @@ std::optional<bool> lowerBuiltin(const StringRef DemangledCall,
                                  const Register OrigRet, const Type *OrigRetTy,
                                  const SmallVectorImpl<Register> &Args,
                                  SPIRVGlobalRegistry *GR);
+
+/// Translates a string representing a SPIR-V or OpenCL builtin type to a
+/// TargetExtType that can be further lowered with lowerBuiltinType().
+///
+/// \return A TargetExtType representing the builtin SPIR-V type.
+///
+/// \p TypeName is the full string representation of the SPIR-V or OpenCL
+/// builtin type.
+const TargetExtType *
+parseBuiltinTypeNameToTargetExtType(std::string TypeName,
+                                    MachineIRBuilder &MIRBuilder);
+
 /// Handles the translation of the provided special opaque/builtin type \p Type
 /// to SPIR-V type. Generates the corresponding machine instructions for the
 /// target type or gets the already existing OpType<...> register from the
diff --git a/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
index cae7c0e9ac..629db8e2eb 100644
--- a/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
@@ -194,23 +194,38 @@ getKernelArgTypeQual(const Function &KernelFunction, unsigned ArgIdx) {
   return {};
 }
 
-static Type *getArgType(const Function &F, unsigned ArgIdx) {
+static SPIRVType *getArgSPIRVType(const Function &F, unsigned ArgIdx,
+                                  SPIRVGlobalRegistry *GR,
+                                  MachineIRBuilder &MIRBuilder) {
+  // Read argument's access qualifier from metadata or default.
+  SPIRV::AccessQualifier::AccessQualifier ArgAccessQual =
+      getArgAccessQual(F, ArgIdx);
+
   Type *OriginalArgType = getOriginalFunctionType(F)->getParamType(ArgIdx);
+
+  // In case of non-kernel SPIR-V function or already TargetExtType, use the
+  // original IR type.
   if (F.getCallingConv() != CallingConv::SPIR_KERNEL ||
       isSpecialOpaqueType(OriginalArgType))
-    return OriginalArgType;
+    return GR->getOrCreateSPIRVType(OriginalArgType, MIRBuilder, ArgAccessQual);
 
   MDString *MDKernelArgType =
       getKernelArgAttribute(F, ArgIdx, "kernel_arg_type");
-  if (!MDKernelArgType || !MDKernelArgType->getString().endswith("_t"))
-    return OriginalArgType;
-
-  std::string KernelArgTypeStr = "opencl." + MDKernelArgType->getString().str();
-  Type *ExistingOpaqueType =
-      StructType::getTypeByName(F.getContext(), KernelArgTypeStr);
-  return ExistingOpaqueType
-             ? ExistingOpaqueType
-             : StructType::create(F.getContext(), KernelArgTypeStr);
+  if (!MDKernelArgType || (MDKernelArgType->getString().ends_with("*") &&
+                           MDKernelArgType->getString().ends_with("_t")))
+    return GR->getOrCreateSPIRVType(OriginalArgType, MIRBuilder, ArgAccessQual);
+
+  if (MDKernelArgType->getString().ends_with("*"))
+    return GR->getOrCreateSPIRVTypeByName(
+        MDKernelArgType->getString(), MIRBuilder,
+        addressSpaceToStorageClass(OriginalArgType->getPointerAddressSpace()));
+
+  if (MDKernelArgType->getString().ends_with("_t"))
+    return GR->getOrCreateSPIRVTypeByName(
+        "opencl." + MDKernelArgType->getString().str(), MIRBuilder,
+        SPIRV::StorageClass::Function, ArgAccessQual);
+
+  llvm_unreachable("Unable to recognize argument type name.");
 }
 
 static bool isEntryPoint(const Function &F) {
@@ -262,10 +277,8 @@ bool SPIRVCallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder,
       // TODO: handle the case of multiple registers.
       if (VRegs[i].size() > 1)
         return false;
-      SPIRV::AccessQualifier::AccessQualifier ArgAccessQual =
-          getArgAccessQual(F, i);
-      auto *SpirvTy = GR->assignTypeToVReg(getArgType(F, i), VRegs[i][0],
-                                           MIRBuilder, ArgAccessQual);
+      auto *SpirvTy = getArgSPIRVType(F, i, GR, MIRBuilder);
+      GR->assignSPIRVTypeToVReg(SpirvTy, VRegs[i][0], MIRBuilder.getMF());
       ArgTypeVRegs.push_back(SpirvTy);
 
       if (Arg.hasName())
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
index dbfad5b157..0e3fcf4701 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
@@ -594,12 +594,6 @@ SPIRVType *SPIRVGlobalRegistry::getOpTypeStruct(const StructType *Ty,
 SPIRVType *SPIRVGlobalRegistry::getOrCreateSpecialType(
     const Type *Ty, MachineIRBuilder &MIRBuilder,
     SPIRV::AccessQualifier::AccessQualifier AccQual) {
-  // Some OpenCL and SPIRV builtins like image2d_t are passed in as
-  // pointers, but should be treated as custom types like OpTypeImage.
-  if (auto PType = dyn_cast<PointerType>(Ty)) {
-    assert(!PType->isOpaque());
-    Ty = PType->getNonOpaquePointerElementType();
-  }
   assert(isSpecialOpaqueType(Ty) && "Not a special opaque builtin type");
   return SPIRV::lowerBuiltinType(Ty, AccQual, MIRBuilder, this);
 }
@@ -755,13 +749,10 @@ SPIRVType *SPIRVGlobalRegistry::restOfCreateSPIRVType(
       !isSpecialOpaqueType(Ty)) {
     if (!Ty->isPointerTy())
       DT.add(Ty, &MIRBuilder.getMF(), getSPIRVTypeID(SpirvType));
-    else if (Ty->isOpaquePointerTy())
+    else
       DT.add(Type::getInt8Ty(MIRBuilder.getMF().getFunction().getContext()),
              Ty->getPointerAddressSpace(), &MIRBuilder.getMF(),
              getSPIRVTypeID(SpirvType));
-    else
-      DT.add(Ty->getNonOpaquePointerElementType(), Ty->getPointerAddressSpace(),
-             &MIRBuilder.getMF(), getSPIRVTypeID(SpirvType));
   }
 
   return SpirvType;
@@ -956,40 +947,82 @@ SPIRVGlobalRegistry::checkSpecialInstr(const SPIRV::SpecialTypeDescriptor &TD,
 }
 
 // TODO: maybe use tablegen to implement this.
-SPIRVType *
-SPIRVGlobalRegistry::getOrCreateSPIRVTypeByName(StringRef TypeStr,
-                                                MachineIRBuilder &MIRBuilder) {
+SPIRVType *SPIRVGlobalRegistry::getOrCreateSPIRVTypeByName(
+    StringRef TypeStr, MachineIRBuilder &MIRBuilder,
+    SPIRV::StorageClass::StorageClass SC,
+    SPIRV::AccessQualifier::AccessQualifier AQ) {
   unsigned VecElts = 0;
   auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
 
+  // Parse strings representing either a SPIR-V or OpenCL builtin type.
+  if (hasBuiltinTypePrefix(TypeStr))
+    return getOrCreateSPIRVType(
+        SPIRV::parseBuiltinTypeNameToTargetExtType(TypeStr.str(), MIRBuilder),
+        MIRBuilder, AQ);
+
   // Parse type name in either "typeN" or "type vector[N]" format, where
   // N is the number of elements of the vector.
-  Type *Type;
+  Type *Ty;
+
+  if (TypeStr.starts_with("atomic_"))
+    TypeStr = TypeStr.substr(strlen("atomic_"));
+
   if (TypeStr.startswith("void")) {
-    Type = Type::getVoidTy(Ctx);
+    Ty = Type::getVoidTy(Ctx);
     TypeStr = TypeStr.substr(strlen("void"));
+  } else if (TypeStr.startswith("bool")) {
+    Ty = Type::getIntNTy(Ctx, 1);
+    TypeStr = TypeStr.substr(strlen("bool"));
+  } else if (TypeStr.startswith("char") || TypeStr.startswith("uchar")) {
+    Ty = Type::getInt8Ty(Ctx);
+    TypeStr = TypeStr.startswith("char") ? TypeStr.substr(strlen("char"))
+                                         : TypeStr.substr(strlen("uchar"));
+  } else if (TypeStr.startswith("short") || TypeStr.startswith("ushort")) {
+    Ty = Type::getInt16Ty(Ctx);
+    TypeStr = TypeStr.startswith("short") ? TypeStr.substr(strlen("short"))
+                                          : TypeStr.substr(strlen("ushort"));
   } else if (TypeStr.startswith("int") || TypeStr.startswith("uint")) {
-    Type = Type::getInt32Ty(Ctx);
+    Ty = Type::getInt32Ty(Ctx);
     TypeStr = TypeStr.startswith("int") ? TypeStr.substr(strlen("int"))
                                         : TypeStr.substr(strlen("uint"));
-  } else if (TypeStr.startswith("float")) {
-    Type = Type::getFloatTy(Ctx);
-    TypeStr = TypeStr.substr(strlen("float"));
+  } else if (TypeStr.starts_with("long") || TypeStr.starts_with("ulong")) {
+    Ty = Type::getInt64Ty(Ctx);
+    TypeStr = TypeStr.startswith("long") ? TypeStr.substr(strlen("long"))
+                                         : TypeStr.substr(strlen("ulong"));
   } else if (TypeStr.startswith("half")) {
-    Type = Type::getHalfTy(Ctx);
+    Ty = Type::getHalfTy(Ctx);
     TypeStr = TypeStr.substr(strlen("half"));
-  } else if (TypeStr.startswith("opencl.sampler_t")) {
-    Type = StructType::create(Ctx, "opencl.sampler_t");
+  } else if (TypeStr.startswith("float")) {
+    Ty = Type::getFloatTy(Ctx);
+    TypeStr = TypeStr.substr(strlen("float"));
+  } else if (TypeStr.startswith("double")) {
+    Ty = Type::getDoubleTy(Ctx);
+    TypeStr = TypeStr.substr(strlen("double"));
   } else
     llvm_unreachable("Unable to recognize SPIRV type name.");
+
+  auto SpirvTy = getOrCreateSPIRVType(Ty, MIRBuilder, AQ);
+
+  // Handle "type*" or  "type* vector[N]".
+  if (TypeStr.starts_with("*")) {
+    SpirvTy = getOrCreateSPIRVPointerType(SpirvTy, MIRBuilder, SC);
+    TypeStr = TypeStr.substr(strlen("*"));
+  }
+
+  // Handle "typeN*" or  "type vector[N]*".
+  bool IsPtrToVec = TypeStr.consume_back("*");
+
   if (TypeStr.startswith(" vector[")) {
     TypeStr = TypeStr.substr(strlen(" vector["));
     TypeStr = TypeStr.substr(0, TypeStr.find(']'));
   }
   TypeStr.getAsInteger(10, VecElts);
-  auto SpirvTy = getOrCreateSPIRVType(Type, MIRBuilder);
   if (VecElts > 0)
     SpirvTy = getOrCreateSPIRVVectorType(SpirvTy, VecElts, MIRBuilder);
+
+  if (IsPtrToVec)
+    SpirvTy = getOrCreateSPIRVPointerType(SpirvTy, MIRBuilder, SC);
+
   return SpirvTy;
 }
 
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
index 88769f84b3..60967bfb68 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
@@ -138,8 +138,11 @@ class SPIRVGlobalRegistry {
 
   // Either generate a new OpTypeXXX instruction or return an existing one
   // corresponding to the given string containing the name of the builtin type.
-  SPIRVType *getOrCreateSPIRVTypeByName(StringRef TypeStr,
-                                        MachineIRBuilder &MIRBuilder);
+  SPIRVType *getOrCreateSPIRVTypeByName(
+      StringRef TypeStr, MachineIRBuilder &MIRBuilder,
+      SPIRV::StorageClass::StorageClass SC = SPIRV::StorageClass::Function,
+      SPIRV::AccessQualifier::AccessQualifier AQ =
+          SPIRV::AccessQualifier::ReadWrite);
 
   // Return the SPIR-V type instruction corresponding to the given VReg, or
   // nullptr if no such type instruction exists.
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
index e5094e8a4d..6cb51d7829 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
@@ -332,7 +332,7 @@ const Type *getTypedPtrEltType(const Type *Ty) {
   return PType->getNonOpaquePointerElementType();
 }
 
-static bool hasBuiltinTypePrefix(StringRef Name) {
+bool hasBuiltinTypePrefix(StringRef Name) {
   if (Name.starts_with("opencl.") || Name.starts_with("spirv."))
     return true;
   return false;
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.h b/llvm/lib/Target/SPIRV/SPIRVUtils.h
index 7c193611a8..30fae6c7de 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.h
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.h
@@ -92,6 +92,9 @@ std::string getOclOrSpirvBuiltinDemangledName(StringRef Name);
 // element type, otherwise return Type.
 const Type *getTypedPtrEltType(const Type *Type);
 
+// Check if a string contains a builtin prefix.
+bool hasBuiltinTypePrefix(StringRef Name);
+
 // Check if given LLVM type is a special opaque builtin type.
 bool isSpecialOpaqueType(const Type *Ty);
 } // namespace llvm
diff --git a/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp b/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp
index 2c0696e804..9e14f96b6c 100644
--- a/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp
+++ b/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp
@@ -136,8 +136,9 @@ namespace {
 
   public:
     SparcAsmBackend(const Target &T)
-        : MCAsmBackend(StringRef(T.getName()) == "sparcel" ? support::little
-                                                           : support::big),
+        : MCAsmBackend(StringRef(T.getName()) == "sparcel"
+                           ? llvm::endianness::little
+                           : llvm::endianness::big),
           TheTarget(T), Is64Bit(StringRef(TheTarget.getName()) == "sparcv9") {}
 
     unsigned getNumFixupKinds() const override {
@@ -264,7 +265,7 @@ namespace {
 
       assert(unsigned(Kind - FirstTargetFixupKind) < getNumFixupKinds() &&
              "Invalid kind!");
-      if (Endian == support::little)
+      if (Endian == llvm::endianness::little)
         return InfosLE[Kind - FirstTargetFixupKind];
 
       return InfosBE[Kind - FirstTargetFixupKind];
@@ -355,7 +356,8 @@ namespace {
       // from the fixup value. The Value has been "split up" into the
       // appropriate bitfields above.
       for (unsigned i = 0; i != NumBytes; ++i) {
-        unsigned Idx = Endian == support::little ? i : (NumBytes - 1) - i;
+        unsigned Idx =
+            Endian == llvm::endianness::little ? i : (NumBytes - 1) - i;
         Data[Offset + Idx] |= uint8_t((Value >> (i * 8)) & 0xff);
       }
     }
diff --git a/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCCodeEmitter.cpp b/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCCodeEmitter.cpp
index 9b3282e073..42357e3b1a 100644
--- a/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCCodeEmitter.cpp
+++ b/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCCodeEmitter.cpp
@@ -93,8 +93,9 @@ void SparcMCCodeEmitter::encodeInstruction(const MCInst &MI,
                                            const MCSubtargetInfo &STI) const {
   unsigned Bits = getBinaryCodeForInstr(MI, Fixups, STI);
   support::endian::write(CB, Bits,
-                         Ctx.getAsmInfo()->isLittleEndian() ? support::little
-                                                            : support::big);
+                         Ctx.getAsmInfo()->isLittleEndian()
+                             ? llvm::endianness::little
+                             : llvm::endianness::big);
 
   // Some instructions have phantom operands that only contribute a fixup entry.
   unsigned SymOpNo = 0;
diff --git a/llvm/lib/Target/Sparc/SparcISelLowering.cpp b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
index b6afb8d5a6..4f08014792 100644
--- a/llvm/lib/Target/Sparc/SparcISelLowering.cpp
+++ b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
@@ -2604,9 +2604,8 @@ static SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG,
   if (LHS.getValueType().isInteger()) {
     // On V9 processors running in 64-bit mode, if CC compares two `i64`s
     // and the RHS is zero we might be able to use a specialized branch.
-    const ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS);
-    if (is64Bit && isV9 && LHS.getValueType() == MVT::i64 && RHSC &&
-        RHSC->isZero() && !ISD::isUnsignedIntSetCC(CC))
+    if (is64Bit && isV9 && LHS.getValueType() == MVT::i64 &&
+        isNullConstant(RHS) && !ISD::isUnsignedIntSetCC(CC))
       return DAG.getNode(SPISD::BR_REG, dl, MVT::Other, Chain, Dest,
                          DAG.getConstant(intCondCCodeToRcond(CC), dl, MVT::i32),
                          LHS);
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp
index 20dcf74cb8..eafe411248 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp
@@ -107,8 +107,7 @@ static uint64_t extractBitsForFixup(MCFixupKind Kind, uint64_t Value,
 namespace {
 class SystemZMCAsmBackend : public MCAsmBackend {
 public:
-  SystemZMCAsmBackend()
-      : MCAsmBackend(support::big) {}
+  SystemZMCAsmBackend() : MCAsmBackend(llvm::endianness::big) {}
 
   // Override MCAsmBackend
   unsigned getNumFixupKinds() const override {
diff --git a/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp b/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp
index 75a37060a8..bfd31709eb 100644
--- a/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp
@@ -1275,6 +1275,30 @@ void SystemZXPLINKFrameLowering::emitPrologue(MachineFunction &MF,
     for (MachineBasicBlock &B : llvm::drop_begin(MF))
       B.addLiveIn(Regs.getFramePointerRegister());
   }
+
+  // Save GPRs used for varargs, if any.
+  const TargetInstrInfo *TII = Subtarget.getInstrInfo();
+  bool IsVarArg = MF.getFunction().isVarArg();
+
+  if (IsVarArg) {
+    // FixedRegs is the number of used registers, accounting for shadow
+    // registers.
+    unsigned FixedRegs = ZFI->getVarArgsFirstGPR() + ZFI->getVarArgsFirstFPR();
+    auto &GPRs = SystemZ::XPLINK64ArgGPRs;
+    for (unsigned I = FixedRegs; I < SystemZ::XPLINK64NumArgGPRs; I++) {
+      uint64_t StartOffset = MFFrame.getOffsetAdjustment() +
+                             MFFrame.getStackSize() + Regs.getCallFrameSize() +
+                             getOffsetOfLocalArea() + I * 8;
+      unsigned Reg = GPRs[I];
+      BuildMI(MBB, MBBI, DL, TII->get(SystemZ::STG))
+          .addReg(Reg)
+          .addReg(Regs.getStackPointerRegister())
+          .addImm(StartOffset)
+          .addReg(0);
+      if (!MBB.isLiveIn(Reg))
+        MBB.addLiveIn(Reg);
+    }
+  }
 }
 
 void SystemZXPLINKFrameLowering::emitEpilogue(MachineFunction &MF,
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index 5c5cb964bd..3db777f904 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -1613,7 +1613,23 @@ SDValue SystemZTargetLowering::LowerFormalArguments(
       InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, ArgValue));
   }
 
-  // FIXME: Add support for lowering varargs for XPLINK64 in a later patch.
+  if (IsVarArg && Subtarget.isTargetXPLINK64()) {
+    // Save the number of non-varargs registers for later use by va_start, etc.
+    FuncInfo->setVarArgsFirstGPR(NumFixedGPRs);
+    FuncInfo->setVarArgsFirstFPR(NumFixedFPRs);
+
+    auto *Regs = static_cast<SystemZXPLINK64Registers *>(
+        Subtarget.getSpecialRegisters());
+
+    // Likewise the address (in the form of a frame index) of where the
+    // first stack vararg would be.  The 1-byte size here is arbitrary.
+    // FIXME: Pre-include call frame size in the offset, should not
+    // need to manually add it here.
+    int64_t VarArgOffset = CCInfo.getStackSize() + Regs->getCallFrameSize();
+    int FI = MFI.CreateFixedObject(1, VarArgOffset, true);
+    FuncInfo->setVarArgsFrameIndex(FI);
+  }
+
   if (IsVarArg && Subtarget.isTargetELF()) {
     // Save the number of non-varargs registers for later use by va_start, etc.
     FuncInfo->setVarArgsFirstGPR(NumFixedGPRs);
@@ -3604,9 +3620,17 @@ SDValue SystemZTargetLowering::lowerFRAMEADDR(SDValue Op,
   int BackChainIdx = TFL->getOrCreateFramePointerSaveIndex(MF);
   SDValue BackChain = DAG.getFrameIndex(BackChainIdx, PtrVT);
 
-  // FIXME The frontend should detect this case.
   if (Depth > 0) {
-    report_fatal_error("Unsupported stack frame traversal count");
+    // FIXME The frontend should detect this case.
+    if (!MF.getFunction().hasFnAttribute("backchain"))
+      report_fatal_error("Unsupported stack frame traversal count");
+
+    SDValue Offset = DAG.getConstant(TFL->getBackchainOffset(MF), DL, PtrVT);
+    while (Depth--) {
+      BackChain = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), BackChain,
+                              MachinePointerInfo());
+      BackChain = DAG.getNode(ISD::ADD, DL, PtrVT, BackChain, Offset);
+    }
   }
 
   return BackChain;
@@ -3625,9 +3649,19 @@ SDValue SystemZTargetLowering::lowerRETURNADDR(SDValue Op,
   unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
   EVT PtrVT = getPointerTy(DAG.getDataLayout());
 
-  // FIXME The frontend should detect this case.
   if (Depth > 0) {
-    report_fatal_error("Unsupported stack frame traversal count");
+    // FIXME The frontend should detect this case.
+    if (!MF.getFunction().hasFnAttribute("backchain"))
+      report_fatal_error("Unsupported stack frame traversal count");
+
+    SDValue FrameAddr = lowerFRAMEADDR(Op, DAG);
+    auto *TFL = Subtarget.getFrameLowering<SystemZELFFrameLowering>();
+    int Offset = (TFL->usePackedStack(MF) ? -2 : 14) *
+                 getTargetMachine().getPointerSize(0);
+    SDValue Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, FrameAddr,
+                              DAG.getConstant(Offset, DL, PtrVT));
+    return DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Ptr,
+                       MachinePointerInfo());
   }
 
   // Return R14D, which has the return address. Mark it an implicit live-in.
diff --git a/llvm/lib/Target/VE/MCTargetDesc/VEAsmBackend.cpp b/llvm/lib/Target/VE/MCTargetDesc/VEAsmBackend.cpp
index 38d163b370..2773a7aaba 100644
--- a/llvm/lib/Target/VE/MCTargetDesc/VEAsmBackend.cpp
+++ b/llvm/lib/Target/VE/MCTargetDesc/VEAsmBackend.cpp
@@ -97,7 +97,8 @@ class VEAsmBackend : public MCAsmBackend {
   const Target &TheTarget;
 
 public:
-  VEAsmBackend(const Target &T) : MCAsmBackend(support::little), TheTarget(T) {}
+  VEAsmBackend(const Target &T)
+      : MCAsmBackend(llvm::endianness::little), TheTarget(T) {}
 
   unsigned getNumFixupKinds() const override { return VE::NumTargetFixupKinds; }
 
@@ -174,7 +175,7 @@ class VEAsmBackend : public MCAsmBackend {
 
     for (uint64_t i = 0; i < Count; i += 8)
       support::endian::write<uint64_t>(OS, 0x7900000000000000ULL,
-                                       support::little);
+                                       llvm::endianness::little);
 
     return true;
   }
@@ -207,7 +208,8 @@ class ELFVEAsmBackend : public VEAsmBackend {
     // from the fixup value. The Value has been "split up" into the
     // appropriate bitfields above.
     for (unsigned i = 0; i != NumBytes; ++i) {
-      unsigned Idx = Endian == support::little ? i : (NumBytes - 1) - i;
+      unsigned Idx =
+          Endian == llvm::endianness::little ? i : (NumBytes - 1) - i;
       Data[Offset + Idx] |= static_cast<uint8_t>((Value >> (i * 8)) & 0xff);
     }
   }
diff --git a/llvm/lib/Target/VE/MCTargetDesc/VEMCCodeEmitter.cpp b/llvm/lib/Target/VE/MCTargetDesc/VEMCCodeEmitter.cpp
index bb643d23e6..31a07fab04 100644
--- a/llvm/lib/Target/VE/MCTargetDesc/VEMCCodeEmitter.cpp
+++ b/llvm/lib/Target/VE/MCTargetDesc/VEMCCodeEmitter.cpp
@@ -82,7 +82,7 @@ void VEMCCodeEmitter::encodeInstruction(const MCInst &MI,
                                         SmallVectorImpl<MCFixup> &Fixups,
                                         const MCSubtargetInfo &STI) const {
   uint64_t Bits = getBinaryCodeForInstr(MI, Fixups, STI);
-  support::endian::write<uint64_t>(CB, Bits, support::little);
+  support::endian::write<uint64_t>(CB, Bits, llvm::endianness::little);
 
   ++MCNumEmitted; // Keep track of the # of mi's emitted.
 }
diff --git a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.cpp b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.cpp
index bc0cb2d10c..69466667e4 100644
--- a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.cpp
+++ b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.cpp
@@ -32,7 +32,6 @@
 #include "llvm/MC/MCSymbolWasm.h"
 #include "llvm/MC/TargetRegistry.h"
 #include "llvm/Support/Compiler.h"
-#include "llvm/Support/Endian.h"
 #include "llvm/Support/SourceMgr.h"
 
 using namespace llvm;
diff --git a/llvm/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp b/llvm/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp
index 3ef7db94fc..ed7757be66 100644
--- a/llvm/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp
+++ b/llvm/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp
@@ -109,8 +109,8 @@ template <typename T>
 bool parseImmediate(MCInst &MI, uint64_t &Size, ArrayRef<uint8_t> Bytes) {
   if (Size + sizeof(T) > Bytes.size())
     return false;
-  T Val = support::endian::read<T, support::endianness::little, 1>(
-      Bytes.data() + Size);
+  T Val =
+      support::endian::read<T, llvm::endianness::little>(Bytes.data() + Size);
   Size += sizeof(T);
   if (std::is_floating_point<T>::value) {
     MI.addOperand(
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp
index 85bb52c03e..ffab67f8ab 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp
@@ -35,7 +35,7 @@ class WebAssemblyAsmBackend final : public MCAsmBackend {
 
 public:
   explicit WebAssemblyAsmBackend(bool Is64Bit, bool IsEmscripten)
-      : MCAsmBackend(support::little), Is64Bit(Is64Bit),
+      : MCAsmBackend(llvm::endianness::little), Is64Bit(Is64Bit),
         IsEmscripten(IsEmscripten) {}
 
   unsigned getNumFixupKinds() const override {
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCCodeEmitter.cpp b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCCodeEmitter.cpp
index 634ed10d4d..aaca213c4a 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCCodeEmitter.cpp
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCCodeEmitter.cpp
@@ -112,16 +112,20 @@ void WebAssemblyMCCodeEmitter::encodeInstruction(
           break;
         case WebAssembly::OPERAND_SIGNATURE:
         case WebAssembly::OPERAND_VEC_I8IMM:
-          support::endian::write<uint8_t>(OS, MO.getImm(), support::little);
+          support::endian::write<uint8_t>(OS, MO.getImm(),
+                                          llvm::endianness::little);
           break;
         case WebAssembly::OPERAND_VEC_I16IMM:
-          support::endian::write<uint16_t>(OS, MO.getImm(), support::little);
+          support::endian::write<uint16_t>(OS, MO.getImm(),
+                                           llvm::endianness::little);
           break;
         case WebAssembly::OPERAND_VEC_I32IMM:
-          support::endian::write<uint32_t>(OS, MO.getImm(), support::little);
+          support::endian::write<uint32_t>(OS, MO.getImm(),
+                                           llvm::endianness::little);
           break;
         case WebAssembly::OPERAND_VEC_I64IMM:
-          support::endian::write<uint64_t>(OS, MO.getImm(), support::little);
+          support::endian::write<uint64_t>(OS, MO.getImm(),
+                                           llvm::endianness::little);
           break;
         case WebAssembly::OPERAND_GLOBAL:
           Ctx.reportError(
@@ -137,10 +141,10 @@ void WebAssemblyMCCodeEmitter::encodeInstruction(
 
     } else if (MO.isSFPImm()) {
       uint32_t F = MO.getSFPImm();
-      support::endian::write<uint32_t>(OS, F, support::little);
+      support::endian::write<uint32_t>(OS, F, llvm::endianness::little);
     } else if (MO.isDFPImm()) {
       uint64_t D = MO.getDFPImm();
-      support::endian::write<uint64_t>(OS, D, support::little);
+      support::endian::write<uint64_t>(OS, D, llvm::endianness::little);
     } else if (MO.isExpr()) {
       const MCOperandInfo &Info = Desc.operands()[I];
       llvm::MCFixupKind FixupKind;
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
index 61cfcdc914..70629b2a50 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
@@ -2576,6 +2576,8 @@ performVectorTruncZeroCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
     APInt SplatValue, SplatUndef;
     unsigned SplatBitSize;
     bool HasAnyUndefs;
+    // Endianness doesn't matter in this context because we are looking for
+    // an all-zero value.
     return Splat &&
            Splat->isConstantSplat(SplatValue, SplatUndef, SplatBitSize,
                                   HasAnyUndefs) &&
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp
index 2995b8816d..3e2e029695 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp
@@ -191,6 +191,9 @@ struct RuntimeLibcallSignatureTable {
     Table[RTLIB::EXP2_F32] = f32_func_f32;
     Table[RTLIB::EXP2_F64] = f64_func_f64;
     Table[RTLIB::EXP2_F128] = i64_i64_func_i64_i64;
+    Table[RTLIB::EXP10_F32] = f32_func_f32;
+    Table[RTLIB::EXP10_F64] = f64_func_f64;
+    Table[RTLIB::EXP10_F128] = i64_i64_func_i64_i64;
     Table[RTLIB::SIN_F32] = f32_func_f32;
     Table[RTLIB::SIN_F64] = f64_func_f64;
     Table[RTLIB::SIN_F128] = i64_i64_func_i64_i64;
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp
index 88c727796e..2db1b6493c 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp
@@ -48,6 +48,12 @@ static cl::opt<bool> WasmDisableExplicitLocals(
              " instruction output for test purposes only."),
     cl::init(false));
 
+static cl::opt<bool> WasmDisableFixIrreducibleControlFlowPass(
+    "wasm-disable-fix-irreducible-control-flow-pass", cl::Hidden,
+    cl::desc("webassembly: disables the fix "
+             " irreducible control flow optimization pass"),
+    cl::init(false));
+
 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeWebAssemblyTarget() {
   // Register the target.
   RegisterTargetMachine<WebAssemblyTargetMachine> X(
@@ -538,7 +544,8 @@ void WebAssemblyPassConfig::addPreEmitPass() {
   addPass(createWebAssemblyNullifyDebugValueLists());
 
   // Eliminate multiple-entry loops.
-  addPass(createWebAssemblyFixIrreducibleControlFlow());
+  if (!WasmDisableFixIrreducibleControlFlowPass)
+    addPass(createWebAssemblyFixIrreducibleControlFlow());
 
   // Do various transformations for exception handling.
   // Every CFG-changing optimizations should come before this.
diff --git a/llvm/lib/Target/X86/CMakeLists.txt b/llvm/lib/Target/X86/CMakeLists.txt
index c387d59ea9..0b7a98ad63 100644
--- a/llvm/lib/Target/X86/CMakeLists.txt
+++ b/llvm/lib/Target/X86/CMakeLists.txt
@@ -33,7 +33,6 @@ set(sources
   X86DiscriminateMemOps.cpp
   X86LowerTileCopy.cpp
   X86LowerAMXType.cpp
-  X86PreAMXConfig.cpp
   X86LowerAMXIntrinsics.cpp
   X86TileConfig.cpp
   X86FastPreTileConfig.cpp
diff --git a/llvm/lib/Target/X86/Disassembler/X86Disassembler.cpp b/llvm/lib/Target/X86/Disassembler/X86Disassembler.cpp
index 49651da63e..2ec7a57093 100644
--- a/llvm/lib/Target/X86/Disassembler/X86Disassembler.cpp
+++ b/llvm/lib/Target/X86/Disassembler/X86Disassembler.cpp
@@ -156,6 +156,9 @@ static InstrUID decode(OpcodeType type, InstructionContext insnContext,
   case MAP6:
     dec = &MAP6_SYM.opcodeDecisions[insnContext].modRMDecisions[opcode];
     break;
+  case MAP7:
+    dec = &MAP7_SYM.opcodeDecisions[insnContext].modRMDecisions[opcode];
+    break;
   }
 
   switch (dec->modrm_type) {
@@ -194,7 +197,7 @@ template <typename T> static bool consume(InternalInstruction *insn, T &ptr) {
   uint64_t offset = insn->readerCursor - insn->startLocation;
   if (offset + sizeof(T) > r.size())
     return true;
-  ptr = support::endian::read<T>(&r[offset], support::little);
+  ptr = support::endian::read<T>(&r[offset], llvm::endianness::little);
   insn->readerCursor += sizeof(T);
   return false;
 }
@@ -918,6 +921,9 @@ static bool readOpcode(struct InternalInstruction *insn) {
     case VEX_LOB_MAP6:
       insn->opcodeType = MAP6;
       return consume(insn, insn->opcode);
+    case VEX_LOB_MAP7:
+      insn->opcodeType = MAP7;
+      return consume(insn, insn->opcode);
     }
   } else if (insn->vectorExtensionType == TYPE_VEX_2B) {
     insn->opcodeType = TWOBYTE;
@@ -1059,6 +1065,9 @@ static int getInstructionIDWithAttrMask(uint16_t *instructionID,
   case MAP6:
     decision = &MAP6_SYM;
     break;
+  case MAP7:
+    decision = &MAP7_SYM;
+    break;
   }
 
   if (decision->opcodeDecisions[insnCtx]
diff --git a/llvm/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h b/llvm/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h
index 95d3c8ede3..2d728143d3 100644
--- a/llvm/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h
+++ b/llvm/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h
@@ -484,7 +484,8 @@ enum VEXLeadingOpcodeByte {
   VEX_LOB_0F38 = 0x2,
   VEX_LOB_0F3A = 0x3,
   VEX_LOB_MAP5 = 0x5,
-  VEX_LOB_MAP6 = 0x6
+  VEX_LOB_MAP6 = 0x6,
+  VEX_LOB_MAP7 = 0x7
 };
 
 enum XOPMapSelect {
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
index fbb2fc138d..e01ce4f431 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
@@ -137,7 +137,7 @@ class X86AsmBackend : public MCAsmBackend {
 
 public:
   X86AsmBackend(const Target &T, const MCSubtargetInfo &STI)
-      : MCAsmBackend(support::little), STI(STI),
+      : MCAsmBackend(llvm::endianness::little), STI(STI),
         MCII(T.createMCInstrInfo()) {
     if (X86AlignBranchWithin32BBoundaries) {
       // At the moment, this defaults to aligning fused branches, unconditional
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h b/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h
index e2293fe305..1e5a3606f3 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h
@@ -829,9 +829,10 @@ namespace X86II {
     /// this flag to indicate that the encoder should do the wacky 3DNow! thing.
     ThreeDNow = 7 << OpMapShift,
 
-    // MAP5, MAP6 - Prefix after the 0x0F prefix.
+    // MAP5, MAP6, MAP7 - Prefix after the 0x0F prefix.
     T_MAP5 = 8 << OpMapShift,
     T_MAP6 = 9 << OpMapShift,
+    T_MAP7 = 10 << OpMapShift,
 
     //===------------------------------------------------------------------===//
     // REX_W - REX prefixes are instruction prefixes used in 64-bit mode.
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
index 59a04f3167..b85404be30 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
@@ -89,6 +89,7 @@ class X86OpcodePrefixHelper {
   //  0b00100: Reserved for future use
   //  0b00101: VEX MAP5
   //  0b00110: VEX MAP6
+  //  0b00111: VEX MAP7
   //  0b00111-0b11111: Reserved for future use
   //  0b01000: XOP map select - 08h instructions with imm byte
   //  0b01001: XOP map select - 09h instructions with no imm byte
@@ -917,6 +918,9 @@ X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI,
   case X86II::T_MAP6:
     Prefix.set5M(0x6);
     break;
+  case X86II::T_MAP7:
+    Prefix.set5M(0x7);
+    break;
   }
 
   Prefix.setL(TSFlags & X86II::VEX_L);
diff --git a/llvm/lib/Target/X86/X86.h b/llvm/lib/Target/X86/X86.h
index 3c5ca07884..485afbc1df 100644
--- a/llvm/lib/Target/X86/X86.h
+++ b/llvm/lib/Target/X86/X86.h
@@ -194,7 +194,6 @@ void initializeX86LowerAMXTypeLegacyPassPass(PassRegistry &);
 void initializeX86LowerTileCopyPass(PassRegistry &);
 void initializeX86OptimizeLEAPassPass(PassRegistry &);
 void initializeX86PartialReductionPass(PassRegistry &);
-void initializeX86PreAMXConfigPassPass(PassRegistry &);
 void initializeX86PreTileConfigPass(PassRegistry &);
 void initializeX86ReturnThunksPass(PassRegistry &);
 void initializeX86SpeculativeExecutionSideEffectSuppressionPass(PassRegistry &);
diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td
index 64f91ae90e..e2935a687f 100644
--- a/llvm/lib/Target/X86/X86.td
+++ b/llvm/lib/Target/X86/X86.td
@@ -325,12 +325,22 @@ def FeatureTSXLDTRK : SubtargetFeature<"tsxldtrk", "HasTSXLDTRK", "true",
                                        "Support TSXLDTRK instructions">;
 def FeatureUINTR : SubtargetFeature<"uintr", "HasUINTR", "true",
                                     "Has UINTR Instructions">;
+def FeatureUSERMSR : SubtargetFeature<"usermsr", "HasUSERMSR", "true",
+                                      "Support USERMSR instructions">;
 def FeaturePCONFIG : SubtargetFeature<"pconfig", "HasPCONFIG", "true",
                                       "platform configuration instruction">;
 def FeatureMOVDIRI  : SubtargetFeature<"movdiri", "HasMOVDIRI", "true",
                                        "Support movdiri instruction (direct store integer)">;
 def FeatureMOVDIR64B : SubtargetFeature<"movdir64b", "HasMOVDIR64B", "true",
                                         "Support movdir64b instruction (direct store 64 bytes)">;
+def FeatureAVX10_1 : SubtargetFeature<"avx10.1-256", "HasAVX10_1", "true",
+                                      "Support AVX10.1 up to 256-bit instruction",
+                                      [FeatureCDI, FeatureVBMI, FeatureIFMA, FeatureVNNI,
+                                       FeatureBF16, FeatureVPOPCNTDQ, FeatureVBMI2, FeatureBITALG,
+                                       FeatureVAES, FeatureVPCLMULQDQ, FeatureFP16]>;
+def FeatureAVX10_1_512 : SubtargetFeature<"avx10.1-512", "HasAVX10_1_512", "true",
+                                          "Support AVX10.1 up to 512-bit instruction",
+                                          [FeatureAVX10_1, FeatureEVEX512]>;
 
 // Ivy Bridge and newer processors have enhanced REP MOVSB and STOSB (aka
 // "string operations"). See "REP String Enhancement" in the Intel Software
@@ -1247,6 +1257,18 @@ def ProcessorFeatures {
   list<SubtargetFeature> ARLSFeatures =
     !listconcat(SRFFeatures, ARLSAdditionalFeatures);
 
+  // Pantherlake
+  list<SubtargetFeature> PTLAdditionalFeatures = [FeaturePREFETCHI];
+  list<SubtargetFeature> PTLFeatures =
+    !listconcat(ARLSFeatures, PTLAdditionalFeatures);
+
+
+  // Clearwaterforest
+  list<SubtargetFeature> CWFAdditionalFeatures = [FeaturePREFETCHI,
+                                                  FeatureUSERMSR];
+  list<SubtargetFeature> CWFFeatures =
+    !listconcat(ARLSFeatures, CWFAdditionalFeatures);
+
   // Knights Landing
   list<SubtargetFeature> KNLFeatures = [FeatureX87,
                                         FeatureCX8,
@@ -1760,6 +1782,10 @@ foreach P = ["arrowlake-s", "arrowlake_s", "lunarlake"] in {
 def : ProcModel<P, AlderlakePModel,
                 ProcessorFeatures.ARLSFeatures, ProcessorFeatures.ADLTuning>;
 }
+def : ProcModel<"pantherlake", AlderlakePModel,
+                ProcessorFeatures.PTLFeatures, ProcessorFeatures.ADLTuning>;
+def : ProcModel<"clearwaterforest", AlderlakePModel,
+                ProcessorFeatures.CWFFeatures, ProcessorFeatures.ADLTuning>;
 def : ProcModel<"graniterapids", SapphireRapidsModel,
                 ProcessorFeatures.GNRFeatures, ProcessorFeatures.SPRTuning>;
 def : ProcModel<"emeraldrapids", SapphireRapidsModel,
diff --git a/llvm/lib/Target/X86/X86FastPreTileConfig.cpp b/llvm/lib/Target/X86/X86FastPreTileConfig.cpp
index 69b2adcd5f..ea942445a1 100644
--- a/llvm/lib/Target/X86/X86FastPreTileConfig.cpp
+++ b/llvm/lib/Target/X86/X86FastPreTileConfig.cpp
@@ -20,7 +20,6 @@
 #include "X86MachineFunctionInfo.h"
 #include "X86RegisterInfo.h"
 #include "X86Subtarget.h"
-#include "llvm/ADT/DepthFirstIterator.h"
 #include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index 694a2df195..93e184eca9 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -1699,10 +1699,28 @@ bool X86DAGToDAGISel::foldOffsetIntoAddress(uint64_t Offset,
     if (AM.BaseType == X86ISelAddressMode::FrameIndexBase &&
         !isDispSafeForFrameIndex(Val))
       return true;
+    // In ILP32 (x32) mode, pointers are 32 bits and need to be zero-extended to
+    // 64 bits. Instructions with 32-bit register addresses perform this zero
+    // extension for us and we can safely ignore the high bits of Offset.
+    // Instructions with only a 32-bit immediate address do not, though: they
+    // sign extend instead. This means only address the low 2GB of address space
+    // is directly addressable, we need indirect addressing for the high 2GB of
+    // address space.
+    // TODO: Some of the earlier checks may be relaxed for ILP32 mode as the
+    // implicit zero extension of instructions would cover up any problem.
+    // However, we have asserts elsewhere that get triggered if we do, so keep
+    // the checks for now.
+    // TODO: We would actually be able to accept these, as well as the same
+    // addresses in LP64 mode, by adding the EIZ pseudo-register as an operand
+    // to get an address size override to be emitted. However, this
+    // pseudo-register is not part of any register class and therefore causes
+    // MIR verification to fail.
+    if (Subtarget->isTarget64BitILP32() && !isUInt<31>(Val) &&
+        !AM.hasBaseOrIndexReg())
+      return true;
   }
   AM.Disp = Val;
   return false;
-
 }
 
 bool X86DAGToDAGISel::matchLoadInAddress(LoadSDNode *N, X86ISelAddressMode &AM,
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index f2716b08c4..20e0210bce 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -3257,6 +3257,72 @@ bool X86TargetLowering::
   return NewShiftOpcode == ISD::SHL;
 }
 
+unsigned X86TargetLowering::preferedOpcodeForCmpEqPiecesOfOperand(
+    EVT VT, unsigned ShiftOpc, bool MayTransformRotate,
+    const APInt &ShiftOrRotateAmt, const std::optional<APInt> &AndMask) const {
+  if (!VT.isInteger())
+    return ShiftOpc;
+
+  bool PreferRotate = false;
+  if (VT.isVector()) {
+    // For vectors, if we have rotate instruction support, then its definetly
+    // best. Otherwise its not clear what the best so just don't make changed.
+    PreferRotate = Subtarget.hasAVX512() && (VT.getScalarType() == MVT::i32 ||
+                                             VT.getScalarType() == MVT::i64);
+  } else {
+    // For scalar, if we have bmi prefer rotate for rorx. Otherwise prefer
+    // rotate unless we have a zext mask+shr.
+    PreferRotate = Subtarget.hasBMI2();
+    if (!PreferRotate) {
+      unsigned MaskBits =
+          VT.getScalarSizeInBits() - ShiftOrRotateAmt.getZExtValue();
+      PreferRotate = (MaskBits != 8) && (MaskBits != 16) && (MaskBits != 32);
+    }
+  }
+
+  if (ShiftOpc == ISD::SHL || ShiftOpc == ISD::SRL) {
+    assert(AndMask.has_value() && "Null andmask when querying about shift+and");
+
+    if (PreferRotate && MayTransformRotate)
+      return ISD::ROTL;
+
+    // If vector we don't really get much benefit swapping around constants.
+    // Maybe we could check if the DAG has the flipped node already in the
+    // future.
+    if (VT.isVector())
+      return ShiftOpc;
+
+    // See if the beneficial to swap shift type.
+    if (ShiftOpc == ISD::SHL) {
+      // If the current setup has imm64 mask, then inverse will have
+      // at least imm32 mask (or be zext i32 -> i64).
+      if (VT == MVT::i64)
+        return AndMask->getSignificantBits() > 32 ? ISD::SRL : ShiftOpc;
+
+      // We can only benefit if req at least 7-bit for the mask. We
+      // don't want to replace shl of 1,2,3 as they can be implemented
+      // with lea/add.
+      return ShiftOrRotateAmt.uge(7) ? ISD::SRL : ShiftOpc;
+    }
+
+    if (VT == MVT::i64)
+      // Keep exactly 32-bit imm64, this is zext i32 -> i64 which is
+      // extremely efficient.
+      return AndMask->getSignificantBits() > 33 ? ISD::SHL : ShiftOpc;
+
+    // Keep small shifts as shl so we can generate add/lea.
+    return ShiftOrRotateAmt.ult(7) ? ISD::SHL : ShiftOpc;
+  }
+
+  // We prefer rotate for vectors of if we won't get a zext mask with SRL
+  // (PreferRotate will be set in the latter case).
+  if (PreferRotate || VT.isVector())
+    return ShiftOpc;
+
+  // Non-vector type and we have a zext mask with SRL.
+  return ISD::SRL;
+}
+
 bool X86TargetLowering::preferScalarizeSplat(SDNode *N) const {
   return N->getOpcode() != ISD::FP_EXTEND;
 }
@@ -40685,7 +40751,7 @@ static SDValue combineShuffle(SDNode *N, SelectionDAG &DAG,
   SDLoc dl(N);
   EVT VT = N->getValueType(0);
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  if (TLI.isTypeLegal(VT))
+  if (TLI.isTypeLegal(VT) && !isSoftF16(VT, Subtarget))
     if (SDValue AddSub = combineShuffleToAddSubOrFMAddSub(N, Subtarget, DAG))
       return AddSub;
 
@@ -49835,9 +49901,9 @@ static SDValue combineLoad(SDNode *N, SelectionDAG &DAG,
     if (PtrVT != Ld->getBasePtr().getSimpleValueType()) {
       SDValue Cast =
           DAG.getAddrSpaceCast(dl, PtrVT, Ld->getBasePtr(), AddrSpace, 0);
-      return DAG.getLoad(RegVT, dl, Ld->getChain(), Cast, Ld->getPointerInfo(),
-                         Ld->getOriginalAlign(),
-                         Ld->getMemOperand()->getFlags());
+      return DAG.getExtLoad(Ext, dl, RegVT, Ld->getChain(), Cast,
+                            Ld->getPointerInfo(), MemVT, Ld->getOriginalAlign(),
+                            Ld->getMemOperand()->getFlags());
     }
   }
 
@@ -50339,9 +50405,10 @@ static SDValue combineStore(SDNode *N, SelectionDAG &DAG,
     if (PtrVT != St->getBasePtr().getSimpleValueType()) {
       SDValue Cast =
           DAG.getAddrSpaceCast(dl, PtrVT, St->getBasePtr(), AddrSpace, 0);
-      return DAG.getStore(St->getChain(), dl, StoredVal, Cast,
-                          St->getPointerInfo(), St->getOriginalAlign(),
-                          St->getMemOperand()->getFlags(), St->getAAInfo());
+      return DAG.getTruncStore(
+          St->getChain(), dl, StoredVal, Cast, St->getPointerInfo(), StVT,
+          St->getOriginalAlign(), St->getMemOperand()->getFlags(),
+          St->getAAInfo());
     }
   }
 
@@ -56918,7 +56985,15 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
       case MVT::v8f16:
         if (!Subtarget.hasFP16())
           break;
-        [[fallthrough]];
+        if (VConstraint)
+          return std::make_pair(0U, &X86::VR128XRegClass);
+        return std::make_pair(0U, &X86::VR128RegClass);
+      case MVT::v8bf16:
+        if (!Subtarget.hasBF16() || !Subtarget.hasVLX())
+          break;
+        if (VConstraint)
+          return std::make_pair(0U, &X86::VR128XRegClass);
+        return std::make_pair(0U, &X86::VR128RegClass);
       case MVT::f128:
       case MVT::v16i8:
       case MVT::v8i16:
@@ -56933,7 +57008,15 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
       case MVT::v16f16:
         if (!Subtarget.hasFP16())
           break;
-        [[fallthrough]];
+        if (VConstraint)
+          return std::make_pair(0U, &X86::VR256XRegClass);
+        return std::make_pair(0U, &X86::VR256RegClass);
+      case MVT::v16bf16:
+        if (!Subtarget.hasBF16() || !Subtarget.hasVLX())
+          break;
+        if (VConstraint)
+          return std::make_pair(0U, &X86::VR256XRegClass);
+        return std::make_pair(0U, &X86::VR256RegClass);
       case MVT::v32i8:
       case MVT::v16i16:
       case MVT::v8i32:
@@ -56948,7 +57031,15 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
       case MVT::v32f16:
         if (!Subtarget.hasFP16())
           break;
-        [[fallthrough]];
+        if (VConstraint)
+          return std::make_pair(0U, &X86::VR512RegClass);
+        return std::make_pair(0U, &X86::VR512_0_15RegClass);
+      case MVT::v32bf16:
+        if (!Subtarget.hasBF16())
+          break;
+        if (VConstraint)
+          return std::make_pair(0U, &X86::VR512RegClass);
+        return std::make_pair(0U, &X86::VR512_0_15RegClass);
       case MVT::v64i8:
       case MVT::v32i16:
       case MVT::v8f64:
@@ -56991,7 +57082,11 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
       case MVT::v8f16:
         if (!Subtarget.hasFP16())
           break;
-        [[fallthrough]];
+        return std::make_pair(X86::XMM0, &X86::VR128RegClass);
+      case MVT::v8bf16:
+        if (!Subtarget.hasBF16() || !Subtarget.hasVLX())
+          break;
+        return std::make_pair(X86::XMM0, &X86::VR128RegClass);
       case MVT::f128:
       case MVT::v16i8:
       case MVT::v8i16:
@@ -57004,7 +57099,11 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
       case MVT::v16f16:
         if (!Subtarget.hasFP16())
           break;
-        [[fallthrough]];
+        return std::make_pair(X86::YMM0, &X86::VR256RegClass);
+      case MVT::v16bf16:
+        if (!Subtarget.hasBF16() || !Subtarget.hasVLX())
+          break;
+        return std::make_pair(X86::YMM0, &X86::VR256RegClass);
       case MVT::v32i8:
       case MVT::v16i16:
       case MVT::v8i32:
@@ -57017,7 +57116,11 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
       case MVT::v32f16:
         if (!Subtarget.hasFP16())
           break;
-        [[fallthrough]];
+        return std::make_pair(X86::ZMM0, &X86::VR512_0_15RegClass);
+      case MVT::v32bf16:
+        if (!Subtarget.hasBF16())
+          break;
+        return std::make_pair(X86::ZMM0, &X86::VR512_0_15RegClass);
       case MVT::v64i8:
       case MVT::v32i16:
       case MVT::v8f64:
diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h
index 8046f42736..3b1b2603fd 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.h
+++ b/llvm/lib/Target/X86/X86ISelLowering.h
@@ -1138,6 +1138,11 @@ namespace llvm {
         unsigned OldShiftOpcode, unsigned NewShiftOpcode,
         SelectionDAG &DAG) const override;
 
+    unsigned preferedOpcodeForCmpEqPiecesOfOperand(
+        EVT VT, unsigned ShiftOpc, bool MayTransformRotate,
+        const APInt &ShiftOrRotateAmt,
+        const std::optional<APInt> &AndMask) const override;
+
     bool preferScalarizeSplat(SDNode *N) const override;
 
     bool shouldFoldConstantShiftPairToMask(const SDNode *N,
diff --git a/llvm/lib/Target/X86/X86InsertPrefetch.cpp b/llvm/lib/Target/X86/X86InsertPrefetch.cpp
index 29ae05bf0c..3c9738be54 100644
--- a/llvm/lib/Target/X86/X86InsertPrefetch.cpp
+++ b/llvm/lib/Target/X86/X86InsertPrefetch.cpp
@@ -110,6 +110,11 @@ bool X86InsertPrefetch::findPrefetchInfo(const FunctionSamples *TopSamples,
                                          Prefetches &Prefetches) const {
   assert(Prefetches.empty() &&
          "Expected caller passed empty PrefetchInfo vector.");
+
+  // There is no point to match prefetch hints if the profile is using MD5.
+  if (FunctionSamples::UseMD5)
+    return false;
+
   static constexpr std::pair<StringLiteral, unsigned> HintTypes[] = {
       {"_nta_", X86::PREFETCHNTA},
       {"_t0_", X86::PREFETCHT0},
@@ -125,7 +130,7 @@ bool X86InsertPrefetch::findPrefetchInfo(const FunctionSamples *TopSamples,
   // Convert serialized prefetch hints into PrefetchInfo objects, and populate
   // the Prefetches vector.
   for (const auto &S_V : *T) {
-    StringRef Name = S_V.getKey();
+    StringRef Name = S_V.first.stringRef();
     if (Name.consume_front(SerializedPrefetchPrefix)) {
       int64_t D = static_cast<int64_t>(S_V.second);
       unsigned IID = 0;
diff --git a/llvm/lib/Target/X86/X86InstrFormats.td b/llvm/lib/Target/X86/X86InstrFormats.td
index f45869e152..70ffd4175e 100644
--- a/llvm/lib/Target/X86/X86InstrFormats.td
+++ b/llvm/lib/Target/X86/X86InstrFormats.td
@@ -163,6 +163,7 @@ def XOPA      : Map<6>;
 def ThreeDNow : Map<7>;
 def T_MAP5    : Map<8>;
 def T_MAP6    : Map<9>;
+def T_MAP7    : Map<10>;
 
 // Class specifying the encoding
 class Encoding<bits<2> val> {
@@ -217,6 +218,9 @@ class T_MAP6PS : T_MAP6 { Prefix OpPrefix = PS; }
 class T_MAP6PD : T_MAP6 { Prefix OpPrefix = PD; }
 class T_MAP6XS : T_MAP6 { Prefix OpPrefix = XS; }
 class T_MAP6XD : T_MAP6 { Prefix OpPrefix = XD; }
+class T_MAP7     { Map OpMap = T_MAP7; }
+class T_MAP7XS : T_MAP7 { Prefix OpPrefix = XS; } // 0xF3
+class T_MAP7XD : T_MAP7 { Prefix OpPrefix = XD; } // 0xF2
 class OBXS   { Prefix OpPrefix = XS; }
 class PS   : TB { Prefix OpPrefix = PS; }
 class PD   : TB { Prefix OpPrefix = PD; }
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 4320a0e94b..4c6854da0a 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -3867,12 +3867,42 @@ bool X86InstrInfo::verifyInstruction(const MachineInstr &MI,
 bool X86InstrInfo::getConstValDefinedInReg(const MachineInstr &MI,
                                            const Register Reg,
                                            int64_t &ImmVal) const {
-  if (MI.getOpcode() != X86::MOV32ri && MI.getOpcode() != X86::MOV64ri)
+  Register MovReg = Reg;
+  const MachineInstr *MovMI = &MI;
+
+  // Follow use-def for SUBREG_TO_REG to find the real move immediate
+  // instruction. It is quite common for x86-64.
+  if (MI.isSubregToReg()) {
+    // We use following pattern to setup 64b immediate.
+    //      %8:gr32 = MOV32r0 implicit-def dead $eflags
+    //      %6:gr64 = SUBREG_TO_REG 0, killed %8:gr32, %subreg.sub_32bit
+    if (!MI.getOperand(1).isImm())
+      return false;
+    unsigned FillBits = MI.getOperand(1).getImm();
+    unsigned SubIdx = MI.getOperand(3).getImm();
+    MovReg = MI.getOperand(2).getReg();
+    if (SubIdx != X86::sub_32bit || FillBits != 0)
+      return false;
+    const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
+    MovMI = MRI.getUniqueVRegDef(MovReg);
+    if (!MovMI)
+      return false;
+  }
+
+  if (MovMI->getOpcode() == X86::MOV32r0 &&
+      MovMI->getOperand(0).getReg() == MovReg) {
+    ImmVal = 0;
+    return true;
+  }
+
+  if (MovMI->getOpcode() != X86::MOV32ri &&
+      MovMI->getOpcode() != X86::MOV64ri &&
+      MovMI->getOpcode() != X86::MOV32ri64 && MovMI->getOpcode() != X86::MOV8ri)
     return false;
   // Mov Src can be a global address.
-  if (!MI.getOperand(1).isImm() || MI.getOperand(0).getReg() != Reg)
+  if (!MovMI->getOperand(1).isImm() || MovMI->getOperand(0).getReg() != MovReg)
     return false;
-  ImmVal = MI.getOperand(1).getImm();
+  ImmVal = MovMI->getOperand(1).getImm();
   return true;
 }
 
@@ -4769,6 +4799,310 @@ MachineInstr *X86InstrInfo::optimizeLoadInstr(MachineInstr &MI,
   return nullptr;
 }
 
+/// Convert an ALUrr opcode to corresponding ALUri opcode. Such as
+///     ADD32rr  ==>  ADD32ri
+/// ShiftRotate will be set to true if the Opcode is shift or rotate.
+/// If the ALUri can be further changed to COPY when the immediate is 0, set
+/// CanConvert2Copy to true.
+static unsigned ConvertALUrr2ALUri(unsigned Opcode, bool &CanConvert2Copy,
+                                   bool &ShiftRotate) {
+  CanConvert2Copy = false;
+  ShiftRotate = false;
+  unsigned NewOpcode = 0;
+  switch (Opcode) {
+    case X86::ADD64rr:
+      NewOpcode = X86::ADD64ri32;
+      CanConvert2Copy = true;
+      break;
+    case X86::ADC64rr:
+      NewOpcode = X86::ADC64ri32;
+      break;
+    case X86::SUB64rr:
+      NewOpcode = X86::SUB64ri32;
+      CanConvert2Copy = true;
+      break;
+    case X86::SBB64rr:
+      NewOpcode = X86::SBB64ri32;
+      break;
+    case X86::AND64rr:
+      NewOpcode = X86::AND64ri32;
+      break;
+    case X86::OR64rr:
+      NewOpcode = X86::OR64ri32;
+      CanConvert2Copy = true;
+      break;
+    case X86::XOR64rr:
+      NewOpcode = X86::XOR64ri32;
+      CanConvert2Copy = true;
+      break;
+    case X86::TEST64rr:
+      NewOpcode = X86::TEST64ri32;
+      break;
+    case X86::CMP64rr:
+      NewOpcode = X86::CMP64ri32;
+      break;
+    case X86::SHR64rCL:
+      NewOpcode = X86::SHR64ri;
+      ShiftRotate = true;
+      break;
+    case X86::SHL64rCL:
+      NewOpcode = X86::SHL64ri;
+      ShiftRotate = true;
+      break;
+    case X86::SAR64rCL:
+      NewOpcode = X86::SAR64ri;
+      ShiftRotate = true;
+      break;
+    case X86::ROL64rCL:
+      NewOpcode = X86::ROL64ri;
+      ShiftRotate = true;
+      break;
+    case X86::ROR64rCL:
+      NewOpcode = X86::ROR64ri;
+      ShiftRotate = true;
+      break;
+    case X86::RCL64rCL:
+      NewOpcode = X86::RCL64ri;
+      ShiftRotate = true;
+      break;
+    case X86::RCR64rCL:
+      NewOpcode = X86::RCR64ri;
+      ShiftRotate = true;
+      break;
+    case X86::ADD32rr:
+      NewOpcode = X86::ADD32ri;
+      CanConvert2Copy = true;
+      break;
+    case X86::ADC32rr:
+      NewOpcode = X86::ADC32ri;
+      break;
+    case X86::SUB32rr:
+      NewOpcode = X86::SUB32ri;
+      CanConvert2Copy = true;
+      break;
+    case X86::SBB32rr:
+      NewOpcode = X86::SBB32ri;
+      break;
+    case X86::AND32rr:
+      NewOpcode = X86::AND32ri;
+      break;
+    case X86::OR32rr:
+      NewOpcode = X86::OR32ri;
+      CanConvert2Copy = true;
+      break;
+    case X86::XOR32rr:
+      NewOpcode = X86::XOR32ri;
+      CanConvert2Copy = true;
+      break;
+    case X86::TEST32rr:
+      NewOpcode = X86::TEST32ri;
+      break;
+    case X86::CMP32rr:
+      NewOpcode = X86::CMP32ri;
+      break;
+    case X86::SHR32rCL:
+      NewOpcode = X86::SHR32ri;
+      ShiftRotate = true;
+      break;
+    case X86::SHL32rCL:
+      NewOpcode = X86::SHL32ri;
+      ShiftRotate = true;
+      break;
+    case X86::SAR32rCL:
+      NewOpcode = X86::SAR32ri;
+      ShiftRotate = true;
+      break;
+    case X86::ROL32rCL:
+      NewOpcode = X86::ROL32ri;
+      ShiftRotate = true;
+      break;
+    case X86::ROR32rCL:
+      NewOpcode = X86::ROR32ri;
+      ShiftRotate = true;
+      break;
+    case X86::RCL32rCL:
+      NewOpcode = X86::RCL32ri;
+      ShiftRotate = true;
+      break;
+    case X86::RCR32rCL:
+      NewOpcode = X86::RCR32ri;
+      ShiftRotate = true;
+      break;
+  }
+  return NewOpcode;
+}
+
+/// Real implementation of FoldImmediate.
+/// Reg is assigned ImmVal in DefMI, and is used in UseMI.
+/// If MakeChange is true, this function tries to replace Reg by ImmVal in
+/// UseMI. If MakeChange is false, just check if folding is possible.
+/// Return true if folding is successful or possible.
+bool X86InstrInfo::FoldImmediateImpl(MachineInstr &UseMI, MachineInstr *DefMI,
+                                     Register Reg, int64_t ImmVal,
+                                     MachineRegisterInfo *MRI,
+                                     bool MakeChange) const {
+  bool Modified = false;
+  bool ShiftRotate = false;
+  // When ImmVal is 0, some instructions can be changed to COPY.
+  bool CanChangeToCopy = false;
+  unsigned Opc = UseMI.getOpcode();
+
+  // 64 bit operations accept sign extended 32 bit immediates.
+  // 32 bit operations accept all 32 bit immediates, so we don't need to check
+  // them.
+  const TargetRegisterClass *RC = nullptr;
+  if (Reg.isVirtual())
+    RC = MRI->getRegClass(Reg);
+  if ((Reg.isPhysical() && X86::GR64RegClass.contains(Reg)) ||
+      (Reg.isVirtual() && X86::GR64RegClass.hasSubClassEq(RC))) {
+    if (!isInt<32>(ImmVal))
+      return false;
+  }
+
+  if (UseMI.findRegisterUseOperand(Reg)->getSubReg())
+    return false;
+  // Immediate has larger code size than register. So avoid folding the
+  // immediate if it has more than 1 use and we are optimizing for size.
+  if (UseMI.getMF()->getFunction().hasOptSize() && Reg.isVirtual() &&
+      !MRI->hasOneNonDBGUse(Reg))
+    return false;
+
+  unsigned NewOpc;
+  if (Opc == TargetOpcode::COPY) {
+    Register ToReg = UseMI.getOperand(0).getReg();
+    const TargetRegisterClass *RC = nullptr;
+    if (ToReg.isVirtual())
+      RC = MRI->getRegClass(ToReg);
+    bool GR32Reg = (ToReg.isVirtual() && X86::GR32RegClass.hasSubClassEq(RC)) ||
+                   (ToReg.isPhysical() && X86::GR32RegClass.contains(ToReg));
+    bool GR64Reg = (ToReg.isVirtual() && X86::GR64RegClass.hasSubClassEq(RC)) ||
+                   (ToReg.isPhysical() && X86::GR64RegClass.contains(ToReg));
+    bool GR8Reg = (ToReg.isVirtual() && X86::GR8RegClass.hasSubClassEq(RC)) ||
+                  (ToReg.isPhysical() && X86::GR8RegClass.contains(ToReg));
+
+    if (ImmVal == 0) {
+      // We have MOV32r0 only.
+      if (!GR32Reg)
+        return false;
+    }
+
+    if (GR64Reg) {
+      if (isUInt<32>(ImmVal))
+        NewOpc = X86::MOV32ri64;
+      else
+        NewOpc = X86::MOV64ri;
+    } else if (GR32Reg) {
+      NewOpc = X86::MOV32ri;
+      if (ImmVal == 0) {
+        // MOV32r0 clobbers EFLAGS.
+        const TargetRegisterInfo *TRI = &getRegisterInfo();
+        if (UseMI.getParent()->computeRegisterLiveness(TRI, X86::EFLAGS, UseMI)
+            != MachineBasicBlock::LQR_Dead)
+          return false;
+
+        // MOV32r0 is different than other cases because it doesn't encode the
+        // immediate in the instruction. So we directly modify it here.
+        if (!MakeChange)
+          return true;
+        UseMI.setDesc(get(X86::MOV32r0));
+        UseMI.removeOperand(UseMI.findRegisterUseOperandIdx(Reg));
+        UseMI.addOperand(MachineOperand::CreateReg(X86::EFLAGS, /*isDef=*/ true,
+                                                   /*isImp=*/ true,
+                                                   /*isKill=*/ false,
+                                                   /*isDead=*/ true));
+        Modified = true;
+      }
+    } else if (GR8Reg)
+      NewOpc = X86::MOV8ri;
+    else
+      return false;
+  } else
+    NewOpc = ConvertALUrr2ALUri(Opc, CanChangeToCopy, ShiftRotate);
+
+  if (!NewOpc)
+    return false;
+
+  // For SUB instructions the immediate can only be the second source operand.
+  if ((NewOpc == X86::SUB64ri32 || NewOpc == X86::SUB32ri ||
+       NewOpc == X86::SBB64ri32 || NewOpc == X86::SBB32ri) &&
+      UseMI.findRegisterUseOperandIdx(Reg) != 2)
+    return false;
+  // For CMP instructions the immediate can only be at index 1.
+  if ((NewOpc == X86::CMP64ri32 || NewOpc == X86::CMP32ri) &&
+      UseMI.findRegisterUseOperandIdx(Reg) != 1)
+    return false;
+
+  if (ShiftRotate) {
+    unsigned RegIdx = UseMI.findRegisterUseOperandIdx(Reg);
+    if (RegIdx < 2)
+      return false;
+    if (!isInt<8>(ImmVal))
+      return false;
+    assert(Reg == X86::CL);
+
+    if (!MakeChange)
+      return true;
+    UseMI.setDesc(get(NewOpc));
+    UseMI.removeOperand(RegIdx);
+    UseMI.addOperand(MachineOperand::CreateImm(ImmVal));
+    // Reg is physical register $cl, so we don't know if DefMI is dead through
+    // MRI. Let the caller handle it, or pass dead-mi-elimination can delete
+    // the dead physical register define instruction.
+    return true;
+  }
+
+  if (!MakeChange)
+    return true;
+
+  if (!Modified) {
+    // Modify the instruction.
+    if (ImmVal == 0 && CanChangeToCopy &&
+        UseMI.registerDefIsDead(X86::EFLAGS)) {
+      //          %100 = add %101, 0
+      //    ==>
+      //          %100 = COPY %101
+      UseMI.setDesc(get(TargetOpcode::COPY));
+      UseMI.removeOperand(UseMI.findRegisterUseOperandIdx(Reg));
+      UseMI.removeOperand(UseMI.findRegisterDefOperandIdx(X86::EFLAGS));
+      UseMI.untieRegOperand(0);
+      UseMI.clearFlag(MachineInstr::MIFlag::NoSWrap);
+      UseMI.clearFlag(MachineInstr::MIFlag::NoUWrap);
+    } else {
+      unsigned Op1 = 1, Op2 = CommuteAnyOperandIndex;
+      unsigned ImmOpNum = 2;
+      if (!UseMI.getOperand(0).isDef()) {
+        Op1 = 0;                                      // TEST, CMP
+        ImmOpNum = 1;
+      }
+      if (Opc == TargetOpcode::COPY)
+        ImmOpNum = 1;
+      if (findCommutedOpIndices(UseMI, Op1, Op2) &&
+          UseMI.getOperand(Op1).getReg() == Reg)
+        commuteInstruction(UseMI);
+
+      assert(UseMI.getOperand(ImmOpNum).getReg() == Reg);
+      UseMI.setDesc(get(NewOpc));
+      UseMI.getOperand(ImmOpNum).ChangeToImmediate(ImmVal);
+    }
+  }
+
+  if (Reg.isVirtual() && MRI->use_nodbg_empty(Reg))
+    DefMI->eraseFromBundle();
+
+  return true;
+}
+
+/// FoldImmediate - 'Reg' is known to be defined by a move immediate
+/// instruction, try to fold the immediate into the use instruction.
+bool X86InstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
+                                 Register Reg, MachineRegisterInfo *MRI) const {
+  int64_t ImmVal;
+  if (!getConstValDefinedInReg(DefMI, Reg, ImmVal))
+    return false;
+
+  return FoldImmediateImpl(UseMI, &DefMI, Reg, ImmVal, MRI, true);
+}
+
 /// Expand a single-def pseudo instruction to a two-addr
 /// instruction with two undef reads of the register being defined.
 /// This is used for mapping:
@@ -9796,27 +10130,36 @@ X86InstrInfo::insertOutlinedCall(Module &M, MachineBasicBlock &MBB,
   return It;
 }
 
-void X86InstrInfo::buildClearRegister(Register Reg,
-                                      MachineBasicBlock &MBB,
+void X86InstrInfo::buildClearRegister(Register Reg, MachineBasicBlock &MBB,
                                       MachineBasicBlock::iterator Iter,
-                                      DebugLoc &DL) const {
+                                      DebugLoc &DL,
+                                      bool AllowSideEffects) const {
   const MachineFunction &MF = *MBB.getParent();
   const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
   const TargetRegisterInfo &TRI = getRegisterInfo();
 
   if (ST.hasMMX() && X86::VR64RegClass.contains(Reg))
-    // FIXME: Ignore MMX registers?
+    // FIXME: Should we ignore MMX registers?
     return;
 
   if (TRI.isGeneralPurposeRegister(MF, Reg)) {
-    BuildMI(MBB, Iter, DL, get(X86::XOR32rr), Reg)
-      .addReg(Reg, RegState::Undef)
-      .addReg(Reg, RegState::Undef);
+    // Convert register to the 32-bit version. Both 'movl' and 'xorl' clear the
+    // upper bits of a 64-bit register automagically.
+    Reg = getX86SubSuperRegister(Reg, 32);
+
+    if (!AllowSideEffects)
+      // XOR affects flags, so use a MOV instead.
+      BuildMI(MBB, Iter, DL, get(X86::MOV32ri), Reg).addImm(0);
+    else
+      BuildMI(MBB, Iter, DL, get(X86::XOR32rr), Reg)
+          .addReg(Reg, RegState::Undef)
+          .addReg(Reg, RegState::Undef);
   } else if (X86::VR128RegClass.contains(Reg)) {
     // XMM#
     if (!ST.hasSSE1())
       return;
 
+    // PXOR is safe to use because it doesn't affect flags.
     BuildMI(MBB, Iter, DL, get(X86::PXORrr), Reg)
       .addReg(Reg, RegState::Undef)
       .addReg(Reg, RegState::Undef);
@@ -9825,6 +10168,7 @@ void X86InstrInfo::buildClearRegister(Register Reg,
     if (!ST.hasAVX())
       return;
 
+    // VPXOR is safe to use because it doesn't affect flags.
     BuildMI(MBB, Iter, DL, get(X86::VPXORrr), Reg)
       .addReg(Reg, RegState::Undef)
       .addReg(Reg, RegState::Undef);
@@ -9833,6 +10177,7 @@ void X86InstrInfo::buildClearRegister(Register Reg,
     if (!ST.hasAVX512())
       return;
 
+    // VPXORY is safe to use because it doesn't affect flags.
     BuildMI(MBB, Iter, DL, get(X86::VPXORYrr), Reg)
       .addReg(Reg, RegState::Undef)
       .addReg(Reg, RegState::Undef);
@@ -9844,9 +10189,11 @@ void X86InstrInfo::buildClearRegister(Register Reg,
     if (!ST.hasVLX())
       return;
 
-    BuildMI(MBB, Iter, DL, get(ST.hasBWI() ? X86::KXORQrr : X86::KXORWrr), Reg)
-      .addReg(Reg, RegState::Undef)
-      .addReg(Reg, RegState::Undef);
+    // KXOR is safe to use because it doesn't affect flags.
+    unsigned Op = ST.hasBWI() ? X86::KXORQrr : X86::KXORWrr;
+    BuildMI(MBB, Iter, DL, get(Op), Reg)
+        .addReg(Reg, RegState::Undef)
+        .addReg(Reg, RegState::Undef);
   }
 }
 
diff --git a/llvm/lib/Target/X86/X86InstrInfo.h b/llvm/lib/Target/X86/X86InstrInfo.h
index 8119302f73..e1199e20c3 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.h
+++ b/llvm/lib/Target/X86/X86InstrInfo.h
@@ -550,6 +550,15 @@ class X86InstrInfo final : public X86GenInstrInfo {
                                   Register &FoldAsLoadDefReg,
                                   MachineInstr *&DefMI) const override;
 
+  bool FoldImmediateImpl(MachineInstr &UseMI, MachineInstr *DefMI, Register Reg,
+                         int64_t ImmVal, MachineRegisterInfo *MRI,
+                         bool MakeChange) const;
+
+  /// Reg is known to be defined by a move immediate instruction, try to fold
+  /// the immediate into the use instruction.
+  bool FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, Register Reg,
+                     MachineRegisterInfo *MRI) const override;
+
   std::pair<unsigned, unsigned>
   decomposeMachineOperandsTargetFlags(unsigned TF) const override;
 
@@ -574,8 +583,8 @@ class X86InstrInfo final : public X86GenInstrInfo {
                      outliner::Candidate &C) const override;
 
   void buildClearRegister(Register Reg, MachineBasicBlock &MBB,
-                          MachineBasicBlock::iterator Iter,
-                          DebugLoc &DL) const override;
+                          MachineBasicBlock::iterator Iter, DebugLoc &DL,
+                          bool AllowSideEffects = true) const override;
 
   bool verifyInstruction(const MachineInstr &MI,
                          StringRef &ErrInfo) const override;
diff --git a/llvm/lib/Target/X86/X86InstrInfo.td b/llvm/lib/Target/X86/X86InstrInfo.td
index a20fa6a0c3..9046b6af46 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.td
+++ b/llvm/lib/Target/X86/X86InstrInfo.td
@@ -904,6 +904,8 @@ def HasAVX       : Predicate<"Subtarget->hasAVX()">;
 def HasAVX2      : Predicate<"Subtarget->hasAVX2()">;
 def HasAVX1Only  : Predicate<"Subtarget->hasAVX() && !Subtarget->hasAVX2()">;
 def HasEVEX512   : Predicate<"Subtarget->hasEVEX512()">;
+def HasAVX10_1   : Predicate<"Subtarget->hasAVX10_1()">;
+def HasAVX10_1_512 : Predicate<"Subtarget->hasAVX10_1_512()">;
 def HasAVX512    : Predicate<"Subtarget->hasAVX512()">;
 def UseAVX       : Predicate<"Subtarget->hasAVX() && !Subtarget->hasAVX512()">;
 def UseAVX2      : Predicate<"Subtarget->hasAVX2() && !Subtarget->hasAVX512()">;
@@ -1017,6 +1019,7 @@ def HasAMXBF16   : Predicate<"Subtarget->hasAMXBF16()">;
 def HasAMXINT8   : Predicate<"Subtarget->hasAMXINT8()">;
 def HasAMXCOMPLEX : Predicate<"Subtarget->hasAMXCOMPLEX()">;
 def HasUINTR     : Predicate<"Subtarget->hasUINTR()">;
+def HasUSERMSR   : Predicate<"Subtarget->hasUSERMSR()">;
 def HasCRC32     : Predicate<"Subtarget->hasCRC32()">;
 
 def HasX86_64    : Predicate<"Subtarget->hasX86_64()">;
diff --git a/llvm/lib/Target/X86/X86InstrSystem.td b/llvm/lib/Target/X86/X86InstrSystem.td
index 0272f7de0f..b55956169f 100644
--- a/llvm/lib/Target/X86/X86InstrSystem.td
+++ b/llvm/lib/Target/X86/X86InstrSystem.td
@@ -436,6 +436,22 @@ def WRMSRLIST : I<0x01, MRM_C6, (outs), (ins), "wrmsrlist", []>, XS;
 def RDMSRLIST : I<0x01, MRM_C6, (outs), (ins), "rdmsrlist", []>, XD;
 }
 
+let Predicates = [HasUSERMSR], mayLoad = 1 in {
+  def URDMSRrr : I<0xf8, MRMSrcReg, (outs GR64:$dst), (ins GR64:$src),
+                "urdmsr\t{$src, $dst|$dst, $src}",
+                [(set GR64:$dst, (int_x86_urdmsr GR64:$src))]>, T8XD;
+  def URDMSRri : Ii32<0xf8, MRM0r, (outs GR64:$dst), (ins i64i32imm:$imm),
+                "urdmsr\t{$imm, $dst|$dst, $imm}",
+                [(set GR64:$dst, (int_x86_urdmsr i64immSExt32_su:$imm))]>, T_MAP7XD, VEX;
+}
+let Predicates = [HasUSERMSR], mayStore = 1 in {
+  def UWRMSRrr : I<0xf8, MRMSrcReg, (outs), (ins GR64:$src1, GR64:$src2),
+                "uwrmsr\t{$src1, $src2|$src2, $src1}",
+                [(int_x86_uwrmsr GR64:$src1, GR64:$src2)]>, T8XS;
+  def UWRMSRir : Ii32<0xf8, MRM0r, (outs), (ins GR64:$src, i64i32imm:$imm),
+                "uwrmsr\t{$src, $imm|$imm, $src}",
+                [(int_x86_uwrmsr GR64:$src, i64immSExt32_su:$imm)]>, T_MAP7XS, VEX;
+}
 let Defs = [RAX, RDX], Uses = [ECX] in
 def RDPMC : I<0x33, RawFrm, (outs), (ins), "rdpmc", []>, TB;
 
diff --git a/llvm/lib/Target/X86/X86MCInstLower.cpp b/llvm/lib/Target/X86/X86MCInstLower.cpp
index aed26bfcee..bd64729e7b 100644
--- a/llvm/lib/Target/X86/X86MCInstLower.cpp
+++ b/llvm/lib/Target/X86/X86MCInstLower.cpp
@@ -24,7 +24,6 @@
 #include "X86Subtarget.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringExtras.h"
-#include "llvm/ADT/iterator_range.h"
 #include "llvm/CodeGen/MachineConstantPool.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineModuleInfoImpls.h"
diff --git a/llvm/lib/Target/X86/X86PreAMXConfig.cpp b/llvm/lib/Target/X86/X86PreAMXConfig.cpp
deleted file mode 100644
index 7872a64061..0000000000
--- a/llvm/lib/Target/X86/X86PreAMXConfig.cpp
+++ /dev/null
@@ -1,415 +0,0 @@
-//===- Target/X86/X86PreAMXConfig.cpp - ------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-/// Insert tilecfg for each area of key AMX intrinsic.
-/// All the key AMX intrinsic's tile operand must come from tileload. And the
-/// def tile of key AMX intrinsic must be tilestored.
-/// take tdpbssd for example:
-/// --------------------------------------------------------------------------
-/// %t1 = call x86_amx @llvm.x86.tileloadd64.internal(...)                key
-/// %t2 = call x86_amx @llvm.x86.tileloadd64.internal(...)                 |
-/// %t3 = call x86_amx @llvm.x86.tileloadd64.internal(...)                amx
-/// %td = tail call x86_amx @llvm.x86.tdpbssd.internal(t1, t2, t3)         |
-/// call void @llvm.x86.tilestored64.internal(... td)                     area
-/// --------------------------------------------------------------------------
-/// This pass will insert tilecfg before every key-amx-area, some like:
-/// --------------------------------------------------------------------------
-/// %cfgmem = alloca <16 x i32>, align 4                        * allocate mem
-/// store <16 x i32> zeroinitializer, <16 x i32>* %cfgmem       * zero init
-/// ...
-/// ... pre-config shape of %t1                                 *
-/// store volatile i8 %m, i8* %amx.tmm.0.shape.row, align 1     *
-/// store volatile i16 %k, i16* %amx.tmm.0.shape.col, align 2   * pre-config
-/// ...                                                         *
-/// ... pre-config shape of %t2                                 * shapes
-/// store volatile i8 %k, i8* %amx.tmm.1.shape.row, align 1     *
-/// store volatile i16 %n, i16* %amx.tmm.1.shape.col, align 2   *
-/// ...
-/// call void @llvm.x86.ldtilecfg(i8* %cfgmem)                  * tile config
-//
-//===----------------------------------------------------------------------===//
-//
-#include "X86.h"
-#include "llvm/ADT/SmallSet.h"
-#include "llvm/ADT/StringExtras.h"
-#include "llvm/Analysis/TargetTransformInfo.h"
-#include "llvm/CodeGen/Passes.h"
-#include "llvm/CodeGen/TargetPassConfig.h"
-#include "llvm/CodeGen/ValueTypes.h"
-#include "llvm/IR/DataLayout.h"
-#include "llvm/IR/Function.h"
-#include "llvm/IR/IRBuilder.h"
-#include "llvm/IR/Instructions.h"
-#include "llvm/IR/IntrinsicInst.h"
-#include "llvm/IR/IntrinsicsX86.h"
-#include "llvm/IR/PatternMatch.h"
-#include "llvm/InitializePasses.h"
-#include "llvm/Pass.h"
-#include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetMachine.h"
-
-using namespace llvm;
-using namespace PatternMatch;
-
-#define DEBUG_TYPE "pre-amx-config"
-
-static bool isAMXIntrinsic(IntrinsicInst *II) {
-  for (Value *Operand : II->operands())
-    if (Operand->getType()->isX86_AMXTy())
-      return true;
-  return II->getType()->isX86_AMXTy();
-}
-
-static bool isTileLoad(IntrinsicInst *II) {
-  return II->getIntrinsicID() == Intrinsic::x86_tileloadd64_internal ||
-         II->getIntrinsicID() == Intrinsic::x86_tileloaddt164_internal;
-}
-
-static bool isTileStore(IntrinsicInst *II) {
-  return II->getIntrinsicID() == Intrinsic::x86_tilestored64_internal;
-}
-
-#ifndef NDEBUG
-static bool onlyTileDef(IntrinsicInst *II) {
-  for (Value *Operand : II->operands())
-    if (Operand->getType()->isX86_AMXTy())
-      return false;
-  return II->getType()->isX86_AMXTy();
-}
-
-static bool brokenVolatile(Instruction *I) {
-  // Todo: it is weak to identify a normal call here.
-  if ((isa<CallInst>(I) && !isa<IntrinsicInst>(I)) || I->isTerminator())
-    return true;
-  return false;
-}
-#endif
-
-namespace {
-class X86PreAMXConfig {
-  using PosAndShapesMap = MapVector<Instruction *, SmallVector<Value *, 8>>;
-
-  Function &F;
-
-public:
-  X86PreAMXConfig(Function &Func) : F(Func) {}
-  bool preTileConfig();
-  void addTileConfig(Instruction *ModelStart, SmallVector<Value *, 8> &Shapes);
-  bool findConfigShapes(PosAndShapesMap &PosAndShapes);
-  bool getKeyAMXShapes(IntrinsicInst *KeyAMX, SmallVector<Value *, 8> &Shapes);
-  void preWriteTileCfg(Value *I8Ptr, IRBuilderBase &Builder,
-                       SmallVector<Value *, 8> &Shapes);
-  BasicBlock::iterator
-  getShapesAndConfigPosEnd(BasicBlock::iterator Iter,
-                           SmallVector<Value *, 8> &Shapes);
-  bool checkVolatileModel(SmallSet<Value *, 4> &Loads, IntrinsicInst *Store,
-                          IntrinsicInst *KeyAMX);
-};
-
-// Orderly write the shapes in tilecfg's mem. This maybe not right.
-// Because the first shape may not corresponding to the first tmm register,
-// so we need to handle at at X86FastTileConfig::materializeTileCfg()
-// after register allocation.
-// For example:
-// --------------------------------------------------------------------------
-// zeroinitialize tilecfg's mem (of ldtilecfg)
-// --------------------------------------------------------------------------
-// ... pre-config shape of %t1                                 *
-// %amx.tmm.0.shape.row = getelementptr i8, i8* %mem, i64 48   *
-// %amx.tmm.0.shape.col = getelementptr i16, i16* %mem, i64 16 *
-// store volatile i8 %m, i8* %amx.tmm.0.shape.row, align 1     *
-// store volatile i16 %k, i16* %amx.tmm.0.shape.col, align 2   * pre-config
-// ...                                                         *
-// ... pre-config shape of %t2                                 *
-// %amx.tmm.1.shape.row = getelementptr i8, i8* %mem, i64 49   *
-// %amx.tmm.1.shape.col = getelementptr i16, i16* %mem, i64 18 *
-// store volatile i8 %k, i8* %amx.tmm.1.shape.row, align 1     * shapes
-// store volatile i16 %n, i16* %amx.tmm.1.shape.col, align 2   *
-// ...                                                         *
-// ... pre-config shape of %t3                                 * of
-// %amx.tmm.2.shape.row = getelementptr i8, i8* %mem, i64 50   *
-// %amx.tmm.2.shape.col = getelementptr i16, i16* %mem, i64 20 *
-// store volatile i8 %m, i8* %amx.tmm.2.shape.row, align 1     *
-// store volatile i16 %n, i16* %amx.tmm.2.shape.col, align 2   *
-// ...                                                         * tiles
-// ... pre-config shape of %td                                 *
-// %amx.tmm.3.shape.row = getelementptr i8, i8* %mem, i64 51   *
-// %amx.tmm.3.shape.col = getelementptr i16, i16* %mem, i64 22 *
-// store volatile i8 %m, i8* %amx.tmm.3.shape.row, align 1     *
-// store volatile i16 %n, i16* %amx.tmm.3.shape.col, align 2   *
-// --------------------------------------------------------------------------
-// call void @llvm.x86.ldtilecfg(i8* %mem)                     * tile config
-// --------------------------------------------------------------------------
-// %t1 = call x86_amx @llvm.x86.tileloadd64.internal(m, k, ...)          key
-// %t2 = call x86_amx @llvm.x86.tileloadd64.internal(k, n, ...)
-// %t3 = call x86_amx @llvm.x86.tileloadd64.internal(m, n, ...)          amx
-// %td = tail call x86_amx @llvm.x86.tdpbssd.internal(m, n, k, t1, t2, t3)
-// call void @llvm.x86.tilestored64.internal(... td)                     area
-// --------------------------------------------------------------------------
-void X86PreAMXConfig::preWriteTileCfg(Value *I8Ptr, IRBuilderBase &Builder,
-                                      SmallVector<Value *, 8> &Shapes) {
-  LLVMContext &Ctx = Builder.getContext();
-  Type *I8Ty = Type::getInt8Ty(Ctx);
-  Type *I16Ty = Type::getInt16Ty(Ctx);
-
-  // TODO: Currently we defaultly set Palette = 1, it may be assigned to
-  // other value in the future.
-  Value *PaletteOffset = ConstantInt::get(Type::getInt64Ty(Ctx), 0);
-  Value *PaletteValue = ConstantInt::get(Type::getInt8Ty(Ctx), 1);
-  Value *PalettePos = Builder.CreateGEP(I8Ty, I8Ptr, PaletteOffset);
-  Builder.CreateStore(PaletteValue, PalettePos);
-
-  for (int I = 0, E = Shapes.size() / 2; I < E; I++) {
-    Value *RowOffset = ConstantInt::get(Type::getInt64Ty(Ctx), 48 + I);
-    Value *ColOffset = ConstantInt::get(Type::getInt64Ty(Ctx), 16 + I * 2);
-    const std::string ShapeName = "amx.tmm." + itostr(I);
-    Value *RowPos = Builder.CreateGEP(I8Ty, I8Ptr, RowOffset,
-                                      ShapeName + ".shape.row");
-    Value *ColPos = Builder.CreateGEP(I8Ty, I8Ptr, ColOffset);
-    ColPos = Builder.CreateBitCast(ColPos, PointerType::get(I16Ty, 0),
-                                   ShapeName + ".shape.col");
-    Value *Row = Shapes[I * 2];
-    Value *Col = Shapes[I * 2 + 1];
-    Row = Builder.CreateTrunc(Row, I8Ty);
-    Builder.CreateStore(Row, RowPos);
-    Builder.CreateStore(Col, ColPos);
-  }
-}
-
-void X86PreAMXConfig::addTileConfig(Instruction *ModelStart,
-                                    SmallVector<Value *, 8> &Shapes) {
-  Module *M = F.getParent();
-  IRBuilder<> Builder(ModelStart);
-  const DataLayout &DL = M->getDataLayout();
-  unsigned AddrSpace = DL.getAllocaAddrSpace();
-  LLVMContext &Ctx = Builder.getContext();
-  Type *V512Ty = VectorType::get(Builder.getInt32Ty(), 16, false);
-  Align Alignment = DL.getPrefTypeAlign(Type::getInt32Ty(Ctx));
-
-  AllocaInst *Addr =
-      new AllocaInst(V512Ty, AddrSpace, "", &F.getEntryBlock().front());
-  Addr->setAlignment(Alignment);
-  Value *I8Ptr = Builder.CreateBitCast(Addr, Builder.getInt8PtrTy());
-
-  Builder.CreateAlignedStore(Constant::getNullValue(V512Ty), Addr, Alignment);
-
-  preWriteTileCfg(I8Ptr, Builder, Shapes);
-
-  Builder.CreateIntrinsic(Intrinsic::x86_ldtilecfg_internal, std::nullopt,
-                          {I8Ptr});
-}
-
-// Todo: We may need to handle "more than one store" case in the future.
-bool X86PreAMXConfig::checkVolatileModel(SmallSet<Value *, 4> &Loads,
-                                         IntrinsicInst *Store,
-                                         IntrinsicInst *KeyAMX) {
-  Value *ST = Store->getOperand(4);
-
-  // Only has tileload and tilestore.
-  if (!KeyAMX)
-    return (Loads.size() == 1) && Loads.contains(ST);
-
-  // All Loads should be operands of KeyAMX.
-  // All tile operands of KeyAMX should come from Loads.
-  for (Value *Op : KeyAMX->operands()) {
-    if (Op->getType()->isX86_AMXTy())
-      if (!Loads.erase(Op))
-        return false;
-  }
-
-  // The def of KeyAMX should be stored into mem.
-  // Todo: is it key amx can be no def?
-  return Loads.empty() && (ST == cast<Value>(KeyAMX));
-}
-
-bool X86PreAMXConfig::getKeyAMXShapes(IntrinsicInst *KeyAMX,
-                                      SmallVector<Value *, 8> &Shapes) {
-  for (unsigned I = 0; I < KeyAMX->getNumOperands(); I++) {
-    Value *Op = KeyAMX->getOperand(I);
-    if (!Op->getType()->isX86_AMXTy())
-      continue;
-    IntrinsicInst *TileDef = dyn_cast<IntrinsicInst>(Op);
-    assert((TileDef && isTileLoad(TileDef)) &&
-           "All KeyAMX's tile definiation should comes from TileLoad!");
-    Shapes.push_back(TileDef->getOperand(0));
-    Shapes.push_back(TileDef->getOperand(1));
-  }
-  if (!isTileStore(KeyAMX)) {
-    Shapes.push_back(KeyAMX->getOperand(0));
-    Shapes.push_back(KeyAMX->getOperand(1));
-  }
-  return Shapes.size() != 0;
-}
-
-// Collect the shapes and skip the area of current key amx intrinsic.
-//
-// For example:
-// ...
-// --------------------------------------------------------------------------
-// %t1 = call x86_amx @llvm.x86.tileloadd64.internal(m, k, ...)  record (m,k)
-// %t2 = call x86_amx @llvm.x86.tileloadd64.internal(k, n, ...)  record (m,k)
-// %t3 = call x86_amx @llvm.x86.tileloadd64.internal(m, n, ...)  record (m,k)
-// %td = call x86_amx @llvm.x86.tdpbssd.internal(...t1, t2, t3)
-// call void @llvm.x86.tilestored64.internal(m, n,... td) <--PosEnd record (m,k)
-// --------------------------------------------------------------------------
-BasicBlock::iterator
-X86PreAMXConfig::getShapesAndConfigPosEnd(BasicBlock::iterator Iter,
-                                          SmallVector<Value *, 8> &Shapes) {
-  IntrinsicInst *KeyAMX = nullptr;
-  BasicBlock *BB = Iter->getParent();
-  BasicBlock::iterator PosEnd = BB->end();
-  SmallSet<Value *, 4> Loads;
-
-  // See TileStore as "Config Position End" and check volatile model.
-  for (auto I = Iter, E = BB->end(); I != E; ++I) {
-    assert(!brokenVolatile(&*I) && "Not reach tile store!");
-    IntrinsicInst *II = dyn_cast<IntrinsicInst>(&*I);
-    if (!II || !isAMXIntrinsic(II))
-      continue;
-
-    if (isTileLoad(II)) {
-      Loads.insert(II);
-    } else if (isTileStore(II)) {
-      if (!checkVolatileModel(Loads, II, KeyAMX))
-        report_fatal_error("Not Volatile AMX Model!");
-      PosEnd = I;
-      break;
-    } else {
-      assert(!KeyAMX && "Too many key amx intrinsic!");
-      KeyAMX = II;
-    }
-  }
-  assert(PosEnd != BB->end() && "Not find TileStore!");
-
-  // See KeyAMX as TileStore if only TileLoad and TileStore.
-  if (!KeyAMX)
-    KeyAMX = dyn_cast<IntrinsicInst>(&*PosEnd);
-
-  // Get Shapes in order.
-  assert(Shapes.empty() && "Shapes should be clean.");
-  getKeyAMXShapes(KeyAMX, Shapes);
-
-  return PosEnd;
-}
-
-// Record a key amx area's shapes with its position.
-// Use the first tileload as its position.
-// For example:
-// ...
-// --------------------------------------------------------------------------
-// %t1 = call x86_amx @llvm.x86.tileloadd64.internal(m, k, ...)   <--  pos
-// %t2 = call x86_amx @llvm.x86.tileloadd64.internal(k, n, ...)        /
-// %t3 = call x86_amx @llvm.x86.tileloadd64.internal(m, n, ...)     shapes:
-// %td = call x86_amx @llvm.x86.tdpbssd.internal(...t1, t2, t3)    (m,k)(k,n)
-// call void @llvm.x86.tilestored64.internal(m, n,... td)          (m,n)(m,n)
-// --------------------------------------------------------------------------
-bool X86PreAMXConfig::findConfigShapes(PosAndShapesMap &PosAndShapes) {
-  bool Find = false;
-  for (BasicBlock &BB : F) {
-    for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E; ++I) {
-      IntrinsicInst *II = dyn_cast<IntrinsicInst>(&*I);
-      if (!II)
-        continue;
-      if (!isAMXIntrinsic(II))
-        continue;
-      assert(onlyTileDef(II) && "Not volatile model for AMX at O0!");
-
-      I = getShapesAndConfigPosEnd(I, PosAndShapes[&*I]);
-      Find = true;
-    }
-  }
-  return Find;
-}
-
-// Insert ldtilecfg and preconfig the shapes for each area of key AMX intrinsic.
-// e.g. (key amx = tdpbssd)
-// --------------------------------------------------------------------------
-// %cfgmem = alloca <16 x i32>, align 4                        * allocate mem
-// store <16 x i32> zeroinitializer, <16 x i32>* %cfgmem       * zero init
-// ...
-// ... pre-config shape of %t1                                 *
-// store volatile i8 %m, i8* %amx.tmm.0.shape.row, align 1     *
-// store volatile i16 %k, i16* %amx.tmm.0.shape.col, align 2   * pre-config
-// ...                                                         *
-// ... pre-config shape of %t2                                 *
-// store volatile i8 %k, i8* %amx.tmm.1.shape.row, align 1     * shapes
-// store volatile i16 %n, i16* %amx.tmm.1.shape.col, align 2   *
-// ...                                                         *
-// ... pre-config shape of %t3                                 * of
-// store volatile i8 %m, i8* %amx.tmm.2.shape.row, align 1     *
-// store volatile i16 %n, i16* %amx.tmm.2.shape.col, align 2   *
-// ...                                                         * tiles
-// ... pre-config shape of %td                                 *
-// store volatile i8 %m, i8* %amx.tmm.3.shape.row, align 1     *
-// store volatile i16 %n, i16* %amx.tmm.3.shape.col, align 2   *
-//
-// call void @llvm.x86.ldtilecfg(i8* %cfgmem)                  * pre-config
-// --------------------------------------------------------------------------
-// %t1 = call x86_amx @llvm.x86.tileloadd64.internal(m, k, ...)          key
-// %t2 = call x86_amx @llvm.x86.tileloadd64.internal(k, n, ...)
-// %t3 = call x86_amx @llvm.x86.tileloadd64.internal(m, n, ...)          amx
-// %td = tail call x86_amx @llvm.x86.tdpbssd.internal(m, n, k, t1, t2, t3)
-// call void @llvm.x86.tilestored64.internal(... td)                     area
-// --------------------------------------------------------------------------
-bool X86PreAMXConfig::preTileConfig() {
-  PosAndShapesMap PosAndShapes;
-  bool NeedCfg = findConfigShapes(PosAndShapes);
-  if (!NeedCfg)
-    return false;
-  for (auto &IPAndShapes : PosAndShapes)
-    addTileConfig(IPAndShapes.first, IPAndShapes.second);
-
-  return true;
-}
-} // anonymous namespace
-
-namespace {
-
-class X86PreAMXConfigPass : public FunctionPass {
-public:
-  static char ID;
-
-  X86PreAMXConfigPass() : FunctionPass(ID) {
-    initializeX86PreAMXConfigPassPass(*PassRegistry::getPassRegistry());
-  }
-
-  bool runOnFunction(Function &F) override {
-    TargetMachine *TM = &getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
-    bool C = false;
-
-    // Prepare for fast register allocation at O0.
-    if (TM->getOptLevel() == CodeGenOptLevel::None) {
-
-      // We pre-config each key AMX intrinsic at O0.
-      // In theory, one tile config can cover several AMX intrinsics, but
-      // it is very diffcult to classify the tile shapes at O0. So here we
-      // let thing be easy, pre-config every key AMX intrinsic.
-      X86PreAMXConfig PCFG(F);
-      C = PCFG.preTileConfig();
-    }
-
-    return C;
-  }
-
-  void getAnalysisUsage(AnalysisUsage &AU) const override {
-    AU.setPreservesCFG();
-    AU.addRequired<TargetPassConfig>();
-  }
-};
-
-} // anonymous namespace
-
-static const char PassName[] = "Pre AMX Tile Config";
-char X86PreAMXConfigPass::ID = 0;
-INITIALIZE_PASS_BEGIN(X86PreAMXConfigPass, DEBUG_TYPE, PassName, false, false)
-INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
-INITIALIZE_PASS_END(X86PreAMXConfigPass, DEBUG_TYPE, PassName, false, false)
-
-FunctionPass *llvm::createX86PreAMXConfigPass() {
-  return new X86PreAMXConfigPass();
-}
diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp
index c0d3b8aa93..5668b514d6 100644
--- a/llvm/lib/Target/X86/X86TargetMachine.cpp
+++ b/llvm/lib/Target/X86/X86TargetMachine.cpp
@@ -71,7 +71,6 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeX86Target() {
   PassRegistry &PR = *PassRegistry::getPassRegistry();
   initializeX86LowerAMXIntrinsicsLegacyPassPass(PR);
   initializeX86LowerAMXTypeLegacyPassPass(PR);
-  initializeX86PreAMXConfigPassPass(PR);
   initializeX86PreTileConfigPass(PR);
   initializeGlobalISel(PR);
   initializeWinEHStatePassPass(PR);
@@ -130,12 +129,14 @@ static std::string computeDataLayout(const Triple &TT) {
   Ret += "-p270:32:32-p271:32:32-p272:64:64";
 
   // Some ABIs align 64 bit integers and doubles to 64 bits, others to 32.
+  // 128 bit integers are not specified in the 32-bit ABIs but are used
+  // internally for lowering f128, so we match the alignment to that.
   if (TT.isArch64Bit() || TT.isOSWindows() || TT.isOSNaCl())
-    Ret += "-i64:64";
+    Ret += "-i64:64-i128:128";
   else if (TT.isOSIAMCU())
     Ret += "-i64:32-f64:32";
   else
-    Ret += "-f64:32:64";
+    Ret += "-i128:128-f64:32:64";
 
   // Some ABIs align long double to 128 bits, others to 32.
   if (TT.isOSNaCl() || TT.isOSIAMCU())
diff --git a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaAsmBackend.cpp b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaAsmBackend.cpp
index 61417a2f24..db4484bb57 100644
--- a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaAsmBackend.cpp
+++ b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaAsmBackend.cpp
@@ -30,7 +30,8 @@ class XtensaMCAsmBackend : public MCAsmBackend {
 
 public:
   XtensaMCAsmBackend(uint8_t osABI, bool isLE)
-      : MCAsmBackend(support::little), OSABI(osABI), IsLittleEndian(isLE) {}
+      : MCAsmBackend(llvm::endianness::little), OSABI(osABI),
+        IsLittleEndian(isLE) {}
 
   unsigned getNumFixupKinds() const override {
     return Xtensa::NumTargetFixupKinds;
diff --git a/llvm/lib/TargetParser/ARMTargetParser.cpp b/llvm/lib/TargetParser/ARMTargetParser.cpp
index 20225232b3..d099924419 100644
--- a/llvm/lib/TargetParser/ARMTargetParser.cpp
+++ b/llvm/lib/TargetParser/ARMTargetParser.cpp
@@ -366,26 +366,51 @@ StringRef ARM::getArchExtFeature(StringRef ArchExt) {
 }
 
 static ARM::FPUKind findDoublePrecisionFPU(ARM::FPUKind InputFPUKind) {
+  if (InputFPUKind == ARM::FK_INVALID || InputFPUKind == ARM::FK_NONE)
+    return ARM::FK_INVALID;
+
   const ARM::FPUName &InputFPU = ARM::FPUNames[InputFPUKind];
 
   // If the input FPU already supports double-precision, then there
   // isn't any different FPU we can return here.
-  //
-  // The current available FPURestriction values are None (no
-  // restriction), D16 (only 16 d-regs) and SP_D16 (16 d-regs
-  // and single precision only); there's no value representing
-  // SP restriction without D16. So this test just means 'is it
-  // SP only?'.
-  if (InputFPU.Restriction != ARM::FPURestriction::SP_D16)
+  if (ARM::isDoublePrecision(InputFPU.Restriction))
+    return InputFPUKind;
+
+  // Otherwise, look for an FPU entry with all the same fields, except
+  // that it supports double precision.
+  for (const ARM::FPUName &CandidateFPU : ARM::FPUNames) {
+    if (CandidateFPU.FPUVer == InputFPU.FPUVer &&
+        CandidateFPU.NeonSupport == InputFPU.NeonSupport &&
+        ARM::has32Regs(CandidateFPU.Restriction) ==
+            ARM::has32Regs(InputFPU.Restriction) &&
+        ARM::isDoublePrecision(CandidateFPU.Restriction)) {
+      return CandidateFPU.ID;
+    }
+  }
+
+  // nothing found
+  return ARM::FK_INVALID;
+}
+
+static ARM::FPUKind findSinglePrecisionFPU(ARM::FPUKind InputFPUKind) {
+  if (InputFPUKind == ARM::FK_INVALID || InputFPUKind == ARM::FK_NONE)
     return ARM::FK_INVALID;
 
+  const ARM::FPUName &InputFPU = ARM::FPUNames[InputFPUKind];
+
+  // If the input FPU already is single-precision only, then there
+  // isn't any different FPU we can return here.
+  if (!ARM::isDoublePrecision(InputFPU.Restriction))
+    return InputFPUKind;
+
   // Otherwise, look for an FPU entry with all the same fields, except
-  // that SP_D16 has been replaced with just D16, representing adding
-  // double precision and not changing anything else.
+  // that it does not support double precision.
   for (const ARM::FPUName &CandidateFPU : ARM::FPUNames) {
     if (CandidateFPU.FPUVer == InputFPU.FPUVer &&
         CandidateFPU.NeonSupport == InputFPU.NeonSupport &&
-        CandidateFPU.Restriction == ARM::FPURestriction::D16) {
+        ARM::has32Regs(CandidateFPU.Restriction) ==
+            ARM::has32Regs(InputFPU.Restriction) &&
+        !ARM::isDoublePrecision(CandidateFPU.Restriction)) {
       return CandidateFPU.ID;
     }
   }
@@ -420,20 +445,35 @@ bool ARM::appendArchExtFeatures(StringRef CPU, ARM::ArchKind AK,
     CPU = "generic";
 
   if (ArchExt == "fp" || ArchExt == "fp.dp") {
+    const ARM::FPUKind DefaultFPU = getDefaultFPU(CPU, AK);
     ARM::FPUKind FPUKind;
     if (ArchExt == "fp.dp") {
+      const bool IsDP = ArgFPUKind != ARM::FK_INVALID &&
+                        ArgFPUKind != ARM::FK_NONE &&
+                        isDoublePrecision(getFPURestriction(ArgFPUKind));
       if (Negated) {
-        Features.push_back("-fp64");
-        return true;
+        /* If there is no FPU selected yet, we still need to set ArgFPUKind, as
+         * leaving it as FK_INVALID, would cause default FPU to be selected
+         * later and that could be double precision one. */
+        if (ArgFPUKind != ARM::FK_INVALID && !IsDP)
+          return true;
+        FPUKind = findSinglePrecisionFPU(DefaultFPU);
+        if (FPUKind == ARM::FK_INVALID)
+          FPUKind = ARM::FK_NONE;
+      } else {
+        if (IsDP)
+          return true;
+        FPUKind = findDoublePrecisionFPU(DefaultFPU);
+        if (FPUKind == ARM::FK_INVALID)
+          return false;
       }
-      FPUKind = findDoublePrecisionFPU(getDefaultFPU(CPU, AK));
     } else if (Negated) {
       FPUKind = ARM::FK_NONE;
     } else {
-      FPUKind = getDefaultFPU(CPU, AK);
+      FPUKind = DefaultFPU;
     }
     ArgFPUKind = FPUKind;
-    return ARM::getFPUFeatures(FPUKind, Features);
+    return true;
   }
   return StartingNumFeatures != Features.size();
 }
@@ -519,6 +559,7 @@ StringRef ARM::computeDefaultTargetABI(const Triple &TT, StringRef CPU) {
   case Triple::GNUEABIHF:
   case Triple::MuslEABI:
   case Triple::MuslEABIHF:
+  case Triple::OpenHOS:
     return "aapcs-linux";
   case Triple::EABIHF:
   case Triple::EABI:
diff --git a/llvm/lib/TargetParser/Host.cpp b/llvm/lib/TargetParser/Host.cpp
index 436a5eb04c..aba2ebf317 100644
--- a/llvm/lib/TargetParser/Host.cpp
+++ b/llvm/lib/TargetParser/Host.cpp
@@ -852,6 +852,13 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
       *Subtype = X86::INTEL_COREI7_ARROWLAKE_S;
       break;
 
+    // Pantherlake:
+    case 0xcc:
+      CPU = "pantherlake";
+      *Type = X86::INTEL_COREI7;
+      *Subtype = X86::INTEL_COREI7_PANTHERLAKE;
+      break;
+
     // Graniterapids:
     case 0xad:
       CPU = "graniterapids";
@@ -932,6 +939,12 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
       *Type = X86::INTEL_GRANDRIDGE;
       break;
 
+    // Clearwaterforest:
+    case 0xdd:
+      CPU = "clearwaterforest";
+      *Type = X86::INTEL_CLEARWATERFOREST;
+      break;
+
     // Xeon Phi (Knights Landing + Knights Mill):
     case 0x57:
       CPU = "knl";
@@ -1796,6 +1809,8 @@ bool sys::getHostCPUFeatures(StringMap<bool> &Features) {
   Features["amx-complex"] = HasLeaf7Subleaf1 && ((EDX >> 8) & 1) && HasAMXSave;
   Features["avxvnniint16"] = HasLeaf7Subleaf1 && ((EDX >> 10) & 1) && HasAVXSave;
   Features["prefetchi"]  = HasLeaf7Subleaf1 && ((EDX >> 14) & 1);
+  Features["usermsr"]  = HasLeaf7Subleaf1 && ((EDX >> 15) & 1);
+  Features["avx10.1-256"] = HasLeaf7Subleaf1 && ((EDX >> 19) & 1);
 
   bool HasLeafD = MaxLevel >= 0xd &&
                   !getX86CpuIDAndInfoEx(0xd, 0x1, &EAX, &EBX, &ECX, &EDX);
@@ -1814,6 +1829,11 @@ bool sys::getHostCPUFeatures(StringMap<bool> &Features) {
       MaxLevel >= 0x19 && !getX86CpuIDAndInfo(0x19, &EAX, &EBX, &ECX, &EDX);
   Features["widekl"] = HasLeaf7 && HasLeaf19 && ((EBX >> 2) & 1);
 
+  bool HasLeaf24 =
+      MaxLevel >= 0x24 && !getX86CpuIDAndInfo(0x24, &EAX, &EBX, &ECX, &EDX);
+  Features["avx10.1-512"] =
+      Features["avx10.1-256"] && HasLeaf24 && ((EBX >> 18) & 1);
+
   return true;
 }
 #elif defined(__linux__) && (defined(__arm__) || defined(__aarch64__))
diff --git a/llvm/lib/TargetParser/TargetParser.cpp b/llvm/lib/TargetParser/TargetParser.cpp
index fb7ede1b37..8ab48825d1 100644
--- a/llvm/lib/TargetParser/TargetParser.cpp
+++ b/llvm/lib/TargetParser/TargetParser.cpp
@@ -133,7 +133,7 @@ const GPUInfo *getArchEntry(AMDGPU::GPUKind AK, ArrayRef<GPUInfo> Table) {
         return A.Kind < B.Kind;
       });
 
-  if (I == Table.end())
+  if (I == Table.end() || I->Kind != Search.Kind)
     return nullptr;
   return I;
 }
diff --git a/llvm/lib/TargetParser/X86TargetParser.cpp b/llvm/lib/TargetParser/X86TargetParser.cpp
index b9908dd262..cde36021ee 100644
--- a/llvm/lib/TargetParser/X86TargetParser.cpp
+++ b/llvm/lib/TargetParser/X86TargetParser.cpp
@@ -166,6 +166,10 @@ constexpr FeatureBitset FeaturesGrandridge =
     FeaturesSierraforest | FeatureRAOINT;
 constexpr FeatureBitset FeaturesArrowlakeS = FeaturesSierraforest |
     FeatureAVXVNNIINT16 | FeatureSHA512 | FeatureSM3 | FeatureSM4;
+constexpr FeatureBitset FeaturesPantherlake =
+    FeaturesArrowlakeS | FeaturePREFETCHI;
+constexpr FeatureBitset FeaturesClearwaterforest =
+    FeaturesArrowlakeS | FeatureUSERMSR | FeaturePREFETCHI;
 
 // Geode Processor.
 constexpr FeatureBitset FeaturesGeode =
@@ -360,6 +364,8 @@ constexpr ProcInfo Processors[] = {
   { {"lunarlake"}, CK_Lunarlake, FEATURE_AVX2, FeaturesArrowlakeS, 'p', false },
   // Gracemont microarchitecture based processors.
   { {"gracemont"}, CK_Gracemont, FEATURE_AVX2, FeaturesAlderlake, 'p', false },
+  // Pantherlake microarchitecture based processors.
+  { {"pantherlake"}, CK_Lunarlake, FEATURE_AVX2, FeaturesPantherlake, 'p', false },
   // Sierraforest microarchitecture based processors.
   { {"sierraforest"}, CK_Sierraforest, FEATURE_AVX2, FeaturesSierraforest, 'p', false },
   // Grandridge microarchitecture based processors.
@@ -371,6 +377,8 @@ constexpr ProcInfo Processors[] = {
   { {"graniterapids_d"}, CK_GraniterapidsD, FEATURE_AVX512BF16, FeaturesGraniteRapids | FeatureAMX_COMPLEX, 'n', true },
   // Emerald Rapids microarchitecture based processors.
   { {"emeraldrapids"}, CK_Emeraldrapids, FEATURE_AVX512BF16, FeaturesSapphireRapids, 'n', false },
+  // Clearwaterforest microarchitecture based processors.
+  { {"clearwaterforest"}, CK_Lunarlake, FEATURE_AVX2, FeaturesClearwaterforest, 'p', false },
   // Knights Landing processor.
   { {"knl"}, CK_KNL, FEATURE_AVX512F, FeaturesKNL, 'Z', false },
   { {"mic_avx512"}, CK_KNL, FEATURE_AVX512F, FeaturesKNL, 'Z', true },
@@ -509,6 +517,7 @@ constexpr FeatureBitset ImpliedFeaturesSHSTK = {};
 constexpr FeatureBitset ImpliedFeaturesTBM = {};
 constexpr FeatureBitset ImpliedFeaturesTSXLDTRK = {};
 constexpr FeatureBitset ImpliedFeaturesUINTR = {};
+constexpr FeatureBitset ImpliedFeaturesUSERMSR = {};
 constexpr FeatureBitset ImpliedFeaturesWAITPKG = {};
 constexpr FeatureBitset ImpliedFeaturesWBNOINVD = {};
 constexpr FeatureBitset ImpliedFeaturesVZEROUPPER = {};
@@ -610,6 +619,15 @@ constexpr FeatureBitset ImpliedFeaturesWIDEKL = FeatureKL;
 // AVXVNNI Features
 constexpr FeatureBitset ImpliedFeaturesAVXVNNI = FeatureAVX2;
 
+// AVX10 Features
+constexpr FeatureBitset ImpliedFeaturesAVX10_1 =
+    FeatureAVX512CD | FeatureAVX512VBMI | FeatureAVX512IFMA |
+    FeatureAVX512VNNI | FeatureAVX512BF16 | FeatureAVX512VPOPCNTDQ |
+    FeatureAVX512VBMI2 | FeatureAVX512BITALG | FeatureVAES | FeatureVPCLMULQDQ |
+    FeatureAVX512FP16;
+constexpr FeatureBitset ImpliedFeaturesAVX10_1_512 =
+    FeatureAVX10_1 | FeatureEVEX512;
+
 constexpr FeatureInfo FeatureInfos[X86::CPU_FEATURE_MAX] = {
 #define X86_FEATURE(ENUM, STR) {{"+" STR}, ImpliedFeatures##ENUM},
 #include "llvm/TargetParser/X86TargetParser.def"
diff --git a/llvm/lib/TextAPI/Platform.cpp b/llvm/lib/TextAPI/Platform.cpp
index d0575847a8..9d08469a41 100644
--- a/llvm/lib/TextAPI/Platform.cpp
+++ b/llvm/lib/TextAPI/Platform.cpp
@@ -62,28 +62,11 @@ PlatformSet mapToPlatformSet(ArrayRef<Triple> Targets) {
 
 StringRef getPlatformName(PlatformType Platform) {
   switch (Platform) {
-  case PLATFORM_UNKNOWN:
-    return "unknown";
-  case PLATFORM_MACOS:
-    return "macOS";
-  case PLATFORM_IOS:
-    return "iOS";
-  case PLATFORM_TVOS:
-    return "tvOS";
-  case PLATFORM_WATCHOS:
-    return "watchOS";
-  case PLATFORM_BRIDGEOS:
-    return "bridgeOS";
-  case PLATFORM_MACCATALYST:
-    return "macCatalyst";
-  case PLATFORM_IOSSIMULATOR:
-    return "iOS Simulator";
-  case PLATFORM_TVOSSIMULATOR:
-    return "tvOS Simulator";
-  case PLATFORM_WATCHOSSIMULATOR:
-    return "watchOS Simulator";
-  case PLATFORM_DRIVERKIT:
-    return "DriverKit";
+#define PLATFORM(platform, id, name, build_name, target, tapi_target,          \
+                 marketing)                                                    \
+  case PLATFORM_##platform:                                                    \
+    return #marketing;
+#include "llvm/BinaryFormat/MachO.def"
   }
   llvm_unreachable("Unknown llvm::MachO::PlatformType enum");
 }
@@ -91,16 +74,10 @@ StringRef getPlatformName(PlatformType Platform) {
 PlatformType getPlatformFromName(StringRef Name) {
   return StringSwitch<PlatformType>(Name)
       .Case("osx", PLATFORM_MACOS)
-      .Case("macos", PLATFORM_MACOS)
-      .Case("ios", PLATFORM_IOS)
-      .Case("tvos", PLATFORM_TVOS)
-      .Case("watchos", PLATFORM_WATCHOS)
-      .Case("bridgeos", PLATFORM_BRIDGEOS)
-      .Case("ios-macabi", PLATFORM_MACCATALYST)
-      .Case("ios-simulator", PLATFORM_IOSSIMULATOR)
-      .Case("tvos-simulator", PLATFORM_TVOSSIMULATOR)
-      .Case("watchos-simulator", PLATFORM_WATCHOSSIMULATOR)
-      .Case("driverkit", PLATFORM_DRIVERKIT)
+#define PLATFORM(platform, id, name, build_name, target, tapi_target,          \
+                 marketing)                                                    \
+  .Case(#tapi_target, PLATFORM_##platform)
+#include "llvm/BinaryFormat/MachO.def"
       .Default(PLATFORM_UNKNOWN);
 }
 
diff --git a/llvm/lib/TextAPI/Target.cpp b/llvm/lib/TextAPI/Target.cpp
index e208424983..7f45519735 100644
--- a/llvm/lib/TextAPI/Target.cpp
+++ b/llvm/lib/TextAPI/Target.cpp
@@ -21,16 +21,10 @@ Expected<Target> Target::create(StringRef TargetValue) {
   auto PlatformStr = Result.second;
   PlatformType Platform;
   Platform = StringSwitch<PlatformType>(PlatformStr)
-                 .Case("macos", PLATFORM_MACOS)
-                 .Case("ios", PLATFORM_IOS)
-                 .Case("tvos", PLATFORM_TVOS)
-                 .Case("watchos", PLATFORM_WATCHOS)
-                 .Case("bridgeos", PLATFORM_BRIDGEOS)
-                 .Case("maccatalyst", PLATFORM_MACCATALYST)
-                 .Case("ios-simulator", PLATFORM_IOSSIMULATOR)
-                 .Case("tvos-simulator", PLATFORM_TVOSSIMULATOR)
-                 .Case("watchos-simulator", PLATFORM_WATCHOSSIMULATOR)
-                 .Case("driverkit", PLATFORM_DRIVERKIT)
+#define PLATFORM(platform, id, name, build_name, target, tapi_target,          \
+                 marketing)                                                    \
+  .Case(#tapi_target, PLATFORM_##platform)
+#include "llvm/BinaryFormat/MachO.def"
                  .Default(PLATFORM_UNKNOWN);
 
   if (Platform == PLATFORM_UNKNOWN) {
diff --git a/llvm/lib/TextAPI/TextStub.cpp b/llvm/lib/TextAPI/TextStub.cpp
index 3b94f084b5..cbb185fd51 100644
--- a/llvm/lib/TextAPI/TextStub.cpp
+++ b/llvm/lib/TextAPI/TextStub.cpp
@@ -367,39 +367,12 @@ template <> struct ScalarTraits<Target> {
   static void output(const Target &Value, void *, raw_ostream &OS) {
     OS << Value.Arch << "-";
     switch (Value.Platform) {
-    default:
-      OS << "unknown";
-      break;
-    case PLATFORM_MACOS:
-      OS << "macos";
-      break;
-    case PLATFORM_IOS:
-      OS << "ios";
-      break;
-    case PLATFORM_TVOS:
-      OS << "tvos";
-      break;
-    case PLATFORM_WATCHOS:
-      OS << "watchos";
-      break;
-    case PLATFORM_BRIDGEOS:
-      OS << "bridgeos";
-      break;
-    case PLATFORM_MACCATALYST:
-      OS << "maccatalyst";
-      break;
-    case PLATFORM_IOSSIMULATOR:
-      OS << "ios-simulator";
-      break;
-    case PLATFORM_TVOSSIMULATOR:
-      OS << "tvos-simulator";
-      break;
-    case PLATFORM_WATCHOSSIMULATOR:
-      OS << "watchos-simulator";
-      break;
-    case PLATFORM_DRIVERKIT:
-      OS << "driverkit";
-      break;
+#define PLATFORM(platform, id, name, build_name, target, tapi_target,          \
+                 marketing)                                                    \
+  case PLATFORM_##platform:                                                    \
+    OS << #tapi_target;                                                        \
+    break;
+#include "llvm/BinaryFormat/MachO.def"
     }
   }
 
diff --git a/llvm/lib/Transforms/CMakeLists.txt b/llvm/lib/Transforms/CMakeLists.txt
index dda5f6de11..84a7e34147 100644
--- a/llvm/lib/Transforms/CMakeLists.txt
+++ b/llvm/lib/Transforms/CMakeLists.txt
@@ -9,3 +9,4 @@ add_subdirectory(Hello)
 add_subdirectory(ObjCARC)
 add_subdirectory(Coroutines)
 add_subdirectory(CFGuard)
+add_subdirectory(HipStdPar)
diff --git a/llvm/lib/Transforms/HipStdPar/CMakeLists.txt b/llvm/lib/Transforms/HipStdPar/CMakeLists.txt
new file mode 100644
index 0000000000..bd397f44c3
--- /dev/null
+++ b/llvm/lib/Transforms/HipStdPar/CMakeLists.txt
@@ -0,0 +1,18 @@
+add_llvm_component_library(LLVMHipStdPar
+  HipStdPar.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${LLVM_MAIN_INCLUDE_DIR}/llvm/Transforms/HipStdPar
+
+  DEPENDS
+  intrinsics_gen
+  LLVMAnalysis
+
+  COMPONENT_NAME
+  HipStdPar
+
+  LINK_COMPONENTS
+  Analysis
+  Core
+  Support
+  TransformUtils)
diff --git a/llvm/lib/Transforms/HipStdPar/HipStdPar.cpp b/llvm/lib/Transforms/HipStdPar/HipStdPar.cpp
new file mode 100644
index 0000000000..fb7cba9edb
--- /dev/null
+++ b/llvm/lib/Transforms/HipStdPar/HipStdPar.cpp
@@ -0,0 +1,312 @@
+//===----- HipStdPar.cpp - HIP C++ Standard Parallelism Support Passes ----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// This file implements two passes that enable HIP C++ Standard Parallelism
+// Support:
+//
+// 1. AcceleratorCodeSelection (required): Given that only algorithms are
+//    accelerated, and that the accelerated implementation exists in the form of
+//    a compute kernel, we assume that only the kernel, and all functions
+//    reachable from it, constitute code that the user expects the accelerator
+//    to execute. Thus, we identify the set of all functions reachable from
+//    kernels, and then remove all unreachable ones. This last part is necessary
+//    because it is possible for code that the user did not expect to execute on
+//    an accelerator to contain constructs that cannot be handled by the target
+//    BE, which cannot be provably demonstrated to be dead code in general, and
+//    thus can lead to mis-compilation. The degenerate case of this is when a
+//    Module contains no kernels (the parent TU had no algorithm invocations fit
+//    for acceleration), which we handle by completely emptying said module.
+//    **NOTE**: The above does not handle indirectly reachable functions i.e.
+//              it is possible to obtain a case where the target of an indirect
+//              call is otherwise unreachable and thus is removed; this
+//              restriction is aligned with the current `-hipstdpar` limitations
+//              and will be relaxed in the future.
+//
+// 2. AllocationInterposition (required only when on-demand paging is
+//    unsupported): Some accelerators or operating systems might not support
+//    transparent on-demand paging. Thus, they would only be able to access
+//    memory that is allocated by an accelerator-aware mechanism. For such cases
+//    the user can opt into enabling allocation / deallocation interposition,
+//    whereby we replace calls to known allocation / deallocation functions with
+//    calls to runtime implemented equivalents that forward the requests to
+//    accelerator-aware interfaces. We also support freeing system allocated
+//    memory that ends up in one of the runtime equivalents, since this can
+//    happen if e.g. a library that was compiled without interposition returns
+//    an allocation that can be validly passed to `free`.
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/HipStdPar/HipStdPar.h"
+
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Analysis/CallGraph.h"
+#include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/DebugInfoMetadata.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Transforms/Utils/ModuleUtils.h"
+
+#include <cassert>
+#include <string>
+#include <utility>
+
+using namespace llvm;
+
+template<typename T>
+static inline void eraseFromModule(T &ToErase) {
+  ToErase.replaceAllUsesWith(PoisonValue::get(ToErase.getType()));
+  ToErase.eraseFromParent();
+}
+
+static inline bool checkIfSupported(GlobalVariable &G) {
+  if (!G.isThreadLocal())
+    return true;
+
+  G.dropDroppableUses();
+
+  if (!G.isConstantUsed())
+    return true;
+
+  std::string W;
+  raw_string_ostream OS(W);
+
+  OS << "Accelerator does not support the thread_local variable "
+    << G.getName();
+
+  Instruction *I = nullptr;
+  SmallVector<User *> Tmp(G.user_begin(), G.user_end());
+  SmallPtrSet<User *, 5> Visited;
+  do {
+    auto U = std::move(Tmp.back());
+    Tmp.pop_back();
+
+    if (Visited.contains(U))
+      continue;
+
+    if (isa<Instruction>(U))
+      I = cast<Instruction>(U);
+    else
+      Tmp.insert(Tmp.end(), U->user_begin(), U->user_end());
+
+    Visited.insert(U);
+  } while (!I && !Tmp.empty());
+
+  assert(I && "thread_local global should have at least one non-constant use.");
+
+  G.getContext().diagnose(
+    DiagnosticInfoUnsupported(*I->getParent()->getParent(), W,
+                              I->getDebugLoc(), DS_Error));
+
+  return false;
+}
+
+static inline void clearModule(Module &M) { // TODO: simplify.
+  while (!M.functions().empty())
+    eraseFromModule(*M.begin());
+  while (!M.globals().empty())
+    eraseFromModule(*M.globals().begin());
+  while (!M.aliases().empty())
+    eraseFromModule(*M.aliases().begin());
+  while (!M.ifuncs().empty())
+    eraseFromModule(*M.ifuncs().begin());
+}
+
+static inline void maybeHandleGlobals(Module &M) {
+  unsigned GlobAS = M.getDataLayout().getDefaultGlobalsAddressSpace();
+  for (auto &&G : M.globals()) { // TODO: should we handle these in the FE?
+    if (!checkIfSupported(G))
+      return clearModule(M);
+
+    if (G.isThreadLocal())
+      continue;
+    if (G.isConstant())
+      continue;
+    if (G.getAddressSpace() != GlobAS)
+      continue;
+    if (G.getLinkage() != GlobalVariable::ExternalLinkage)
+      continue;
+
+    G.setLinkage(GlobalVariable::ExternalWeakLinkage);
+    G.setExternallyInitialized(true);
+  }
+}
+
+template<unsigned N>
+static inline void removeUnreachableFunctions(
+  const SmallPtrSet<const Function *, N>& Reachable, Module &M) {
+  removeFromUsedLists(M, [&](Constant *C) {
+    if (auto F = dyn_cast<Function>(C))
+      return !Reachable.contains(F);
+
+    return false;
+  });
+
+  SmallVector<std::reference_wrapper<Function>> ToRemove;
+  copy_if(M, std::back_inserter(ToRemove), [&](auto &&F) {
+    return !F.isIntrinsic() && !Reachable.contains(&F);
+  });
+
+  for_each(ToRemove, eraseFromModule<Function>);
+}
+
+static inline bool isAcceleratorExecutionRoot(const Function *F) {
+    if (!F)
+      return false;
+
+    return F->getCallingConv() == CallingConv::AMDGPU_KERNEL;
+}
+
+static inline bool checkIfSupported(const Function *F, const CallBase *CB) {
+  const auto Dx = F->getName().rfind("__hipstdpar_unsupported");
+
+  if (Dx == StringRef::npos)
+    return true;
+
+  const auto N = F->getName().substr(0, Dx);
+
+  std::string W;
+  raw_string_ostream OS(W);
+
+  if (N == "__ASM")
+    OS << "Accelerator does not support the ASM block:\n"
+      << cast<ConstantDataArray>(CB->getArgOperand(0))->getAsCString();
+  else
+    OS << "Accelerator does not support the " << N << " function.";
+
+  auto Caller = CB->getParent()->getParent();
+
+  Caller->getContext().diagnose(
+    DiagnosticInfoUnsupported(*Caller, W, CB->getDebugLoc(), DS_Error));
+
+  return false;
+}
+
+PreservedAnalyses
+  HipStdParAcceleratorCodeSelectionPass::run(Module &M,
+                                             ModuleAnalysisManager &MAM) {
+  auto &CGA = MAM.getResult<CallGraphAnalysis>(M);
+
+  SmallPtrSet<const Function *, 32> Reachable;
+  for (auto &&CGN : CGA) {
+    if (!isAcceleratorExecutionRoot(CGN.first))
+      continue;
+
+    Reachable.insert(CGN.first);
+
+    SmallVector<const Function *> Tmp({CGN.first});
+    do {
+      auto F = std::move(Tmp.back());
+      Tmp.pop_back();
+
+      for (auto &&N : *CGA[F]) {
+        if (!N.second)
+          continue;
+        if (!N.second->getFunction())
+          continue;
+        if (Reachable.contains(N.second->getFunction()))
+          continue;
+
+        if (!checkIfSupported(N.second->getFunction(),
+                              dyn_cast<CallBase>(*N.first)))
+          return PreservedAnalyses::none();
+
+        Reachable.insert(N.second->getFunction());
+        Tmp.push_back(N.second->getFunction());
+      }
+    } while (!std::empty(Tmp));
+  }
+
+  if (std::empty(Reachable))
+    clearModule(M);
+  else
+    removeUnreachableFunctions(Reachable, M);
+
+  maybeHandleGlobals(M);
+
+  return PreservedAnalyses::none();
+}
+
+static constexpr std::pair<StringLiteral, StringLiteral> ReplaceMap[]{
+  {"aligned_alloc",             "__hipstdpar_aligned_alloc"},
+  {"calloc",                    "__hipstdpar_calloc"},
+  {"free",                      "__hipstdpar_free"},
+  {"malloc",                    "__hipstdpar_malloc"},
+  {"memalign",                  "__hipstdpar_aligned_alloc"},
+  {"posix_memalign",            "__hipstdpar_posix_aligned_alloc"},
+  {"realloc",                   "__hipstdpar_realloc"},
+  {"reallocarray",              "__hipstdpar_realloc_array"},
+  {"_ZdaPv",                    "__hipstdpar_operator_delete"},
+  {"_ZdaPvm",                   "__hipstdpar_operator_delete_sized"},
+  {"_ZdaPvSt11align_val_t",     "__hipstdpar_operator_delete_aligned"},
+  {"_ZdaPvmSt11align_val_t",    "__hipstdpar_operator_delete_aligned_sized"},
+  {"_ZdlPv",                    "__hipstdpar_operator_delete"},
+  {"_ZdlPvm",                   "__hipstdpar_operator_delete_sized"},
+  {"_ZdlPvSt11align_val_t",     "__hipstdpar_operator_delete_aligned"},
+  {"_ZdlPvmSt11align_val_t",    "__hipstdpar_operator_delete_aligned_sized"},
+  {"_Znam",                     "__hipstdpar_operator_new"},
+  {"_ZnamRKSt9nothrow_t",       "__hipstdpar_operator_new_nothrow"},
+  {"_ZnamSt11align_val_t",      "__hipstdpar_operator_new_aligned"},
+  {"_ZnamSt11align_val_tRKSt9nothrow_t",
+                                "__hipstdpar_operator_new_aligned_nothrow"},
+
+  {"_Znwm",                     "__hipstdpar_operator_new"},
+  {"_ZnwmRKSt9nothrow_t",       "__hipstdpar_operator_new_nothrow"},
+  {"_ZnwmSt11align_val_t",      "__hipstdpar_operator_new_aligned"},
+  {"_ZnwmSt11align_val_tRKSt9nothrow_t",
+                                "__hipstdpar_operator_new_aligned_nothrow"},
+  {"__builtin_calloc",          "__hipstdpar_calloc"},
+  {"__builtin_free",            "__hipstdpar_free"},
+  {"__builtin_malloc",          "__hipstdpar_malloc"},
+  {"__builtin_operator_delete", "__hipstdpar_operator_delete"},
+  {"__builtin_operator_new",    "__hipstdpar_operator_new"},
+  {"__builtin_realloc",         "__hipstdpar_realloc"},
+  {"__libc_calloc",             "__hipstdpar_calloc"},
+  {"__libc_free",               "__hipstdpar_free"},
+  {"__libc_malloc",             "__hipstdpar_malloc"},
+  {"__libc_memalign",           "__hipstdpar_aligned_alloc"},
+  {"__libc_realloc",            "__hipstdpar_realloc"}
+};
+
+PreservedAnalyses
+HipStdParAllocationInterpositionPass::run(Module &M, ModuleAnalysisManager&) {
+  SmallDenseMap<StringRef, StringRef> AllocReplacements(std::cbegin(ReplaceMap),
+                                                        std::cend(ReplaceMap));
+
+  for (auto &&F : M) {
+    if (!F.hasName())
+      continue;
+    if (!AllocReplacements.contains(F.getName()))
+      continue;
+
+    if (auto R = M.getFunction(AllocReplacements[F.getName()])) {
+      F.replaceAllUsesWith(R);
+    } else {
+      std::string W;
+      raw_string_ostream OS(W);
+
+      OS << "cannot be interposed, missing: " << AllocReplacements[F.getName()]
+        << ". Tried to run the allocation interposition pass without the "
+        << "replacement functions available.";
+
+      F.getContext().diagnose(DiagnosticInfoUnsupported(F, W,
+                                                        F.getSubprogram(),
+                                                        DS_Warning));
+    }
+  }
+
+  if (auto F = M.getFunction("__hipstdpar_hidden_free")) {
+    auto LibcFree = M.getOrInsertFunction("__libc_free", F->getFunctionType(),
+                                          F->getAttributes());
+    F->replaceAllUsesWith(LibcFree.getCallee());
+
+    eraseFromModule(*F);
+  }
+
+  return PreservedAnalyses::none();
+}
diff --git a/llvm/lib/Transforms/IPO/Attributor.cpp b/llvm/lib/Transforms/IPO/Attributor.cpp
index 1ffafc65ba..49ced893d5 100644
--- a/llvm/lib/Transforms/IPO/Attributor.cpp
+++ b/llvm/lib/Transforms/IPO/Attributor.cpp
@@ -260,7 +260,8 @@ AA::getInitialValueForObj(Attributor &A, const AbstractAttribute &QueryingAA,
     if (!Initializer)
       return nullptr;
   } else {
-    if (!GV->hasLocalLinkage() && !(GV->isConstant() && GV->hasInitializer()))
+    if (!GV->hasLocalLinkage() &&
+        (GV->isInterposable() || !(GV->isConstant() && GV->hasInitializer())))
       return nullptr;
     if (!GV->hasInitializer())
       return UndefValue::get(&Ty);
diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
index 3808eef7ab..e062acd807 100644
--- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -1200,6 +1200,11 @@ struct AAPointerInfoImpl
         A, this, IRPosition::function(Scope), DepClassTy::OPTIONAL,
         IsKnownNoRecurse);
 
+    // TODO: Use reaching kernels from AAKernelInfo (or move it to
+    // AAExecutionDomain) such that we allow scopes other than kernels as long
+    // as the reaching kernels are disjoint.
+    bool InstInKernel = Scope.hasFnAttribute("kernel");
+    bool ObjHasKernelLifetime = false;
     const bool UseDominanceReasoning =
         FindInterferingWrites && IsKnownNoRecurse;
     const DominatorTree *DT =
@@ -1232,6 +1237,7 @@ struct AAPointerInfoImpl
       // If the alloca containing function is not recursive the alloca
       // must be dead in the callee.
       const Function *AIFn = AI->getFunction();
+      ObjHasKernelLifetime = AIFn->hasFnAttribute("kernel");
       bool IsKnownNoRecurse;
       if (AA::hasAssumedIRAttr<Attribute::NoRecurse>(
               A, this, IRPosition::function(*AIFn), DepClassTy::OPTIONAL,
@@ -1241,7 +1247,8 @@ struct AAPointerInfoImpl
     } else if (auto *GV = dyn_cast<GlobalValue>(&getAssociatedValue())) {
       // If the global has kernel lifetime we can stop if we reach a kernel
       // as it is "dead" in the (unknown) callees.
-      if (HasKernelLifetime(GV, *GV->getParent()))
+      ObjHasKernelLifetime = HasKernelLifetime(GV, *GV->getParent());
+      if (ObjHasKernelLifetime)
         IsLiveInCalleeCB = [](const Function &Fn) {
           return !Fn.hasFnAttribute("kernel");
         };
@@ -1252,6 +1259,15 @@ struct AAPointerInfoImpl
     AA::InstExclusionSetTy ExclusionSet;
 
     auto AccessCB = [&](const Access &Acc, bool Exact) {
+      Function *AccScope = Acc.getRemoteInst()->getFunction();
+      bool AccInSameScope = AccScope == &Scope;
+
+      // If the object has kernel lifetime we can ignore accesses only reachable
+      // by other kernels. For now we only skip accesses *in* other kernels.
+      if (InstInKernel && ObjHasKernelLifetime && !AccInSameScope &&
+          AccScope->hasFnAttribute("kernel"))
+        return true;
+
       if (Exact && Acc.isMustAccess() && Acc.getRemoteInst() != &I) {
         if (Acc.isWrite() || (isa<LoadInst>(I) && Acc.isWriteOrAssumption()))
           ExclusionSet.insert(Acc.getRemoteInst());
@@ -1262,8 +1278,7 @@ struct AAPointerInfoImpl
         return true;
 
       bool Dominates = FindInterferingWrites && DT && Exact &&
-                       Acc.isMustAccess() &&
-                       (Acc.getRemoteInst()->getFunction() == &Scope) &&
+                       Acc.isMustAccess() && AccInSameScope &&
                        DT->dominates(Acc.getRemoteInst(), &I);
       if (Dominates)
         DominatingWrites.insert(&Acc);
diff --git a/llvm/lib/Transforms/IPO/FunctionAttrs.cpp b/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
index 95b3204d02..c6891f72ad 100644
--- a/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
+++ b/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
@@ -668,7 +668,8 @@ determinePointerAccessAttrs(Argument *A,
               Worklist.push_back(&UU);
       }
 
-      if (CB.doesNotAccessMemory())
+      ModRefInfo ArgMR = CB.getMemoryEffects().getModRef(IRMemLocation::ArgMem);
+      if (isNoModRef(ArgMR))
         continue;
 
       if (Function *F = CB.getCalledFunction())
@@ -683,9 +684,9 @@ determinePointerAccessAttrs(Argument *A,
       // invokes with operand bundles.
       if (CB.doesNotAccessMemory(UseIndex)) {
         /* nop */
-      } else if (CB.onlyReadsMemory() || CB.onlyReadsMemory(UseIndex)) {
+      } else if (!isModSet(ArgMR) || CB.onlyReadsMemory(UseIndex)) {
         IsRead = true;
-      } else if (CB.hasFnAttr(Attribute::WriteOnly) ||
+      } else if (!isRefSet(ArgMR) ||
                  CB.dataOperandHasImpliedAttr(UseIndex, Attribute::WriteOnly)) {
         IsWrite = true;
       } else {
diff --git a/llvm/lib/Transforms/IPO/FunctionImport.cpp b/llvm/lib/Transforms/IPO/FunctionImport.cpp
index 80c360b8dd..3c07101d87 100644
--- a/llvm/lib/Transforms/IPO/FunctionImport.cpp
+++ b/llvm/lib/Transforms/IPO/FunctionImport.cpp
@@ -16,7 +16,6 @@
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
-#include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Bitcode/BitcodeReader.h"
 #include "llvm/IR/AutoUpgrade.h"
diff --git a/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp b/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
index f835fb26fc..e06c1552e1 100644
--- a/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
+++ b/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
@@ -104,11 +104,13 @@ static cl::opt<std::string> MemProfImportSummary(
     cl::desc("Import summary to use for testing the ThinLTO backend via opt"),
     cl::Hidden);
 
+namespace llvm {
 // Indicate we are linking with an allocator that supports hot/cold operator
 // new interfaces.
 cl::opt<bool> SupportsHotColdNew(
     "supports-hot-cold-new", cl::init(false), cl::Hidden,
     cl::desc("Linking with hot/cold operator new interfaces"));
+} // namespace llvm
 
 namespace {
 /// CRTP base for graphs built from either IR or ThinLTO summary index.
@@ -791,11 +793,10 @@ CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::
 template <typename DerivedCCG, typename FuncTy, typename CallTy>
 void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::
     eraseCalleeEdge(const ContextEdge *Edge) {
-  auto EI =
-      std::find_if(CalleeEdges.begin(), CalleeEdges.end(),
-                   [Edge](const std::shared_ptr<ContextEdge> &CalleeEdge) {
-                     return CalleeEdge.get() == Edge;
-                   });
+  auto EI = llvm::find_if(
+      CalleeEdges, [Edge](const std::shared_ptr<ContextEdge> &CalleeEdge) {
+        return CalleeEdge.get() == Edge;
+      });
   assert(EI != CalleeEdges.end());
   CalleeEdges.erase(EI);
 }
@@ -803,11 +804,10 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::
 template <typename DerivedCCG, typename FuncTy, typename CallTy>
 void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::
     eraseCallerEdge(const ContextEdge *Edge) {
-  auto EI =
-      std::find_if(CallerEdges.begin(), CallerEdges.end(),
-                   [Edge](const std::shared_ptr<ContextEdge> &CallerEdge) {
-                     return CallerEdge.get() == Edge;
-                   });
+  auto EI = llvm::find_if(
+      CallerEdges, [Edge](const std::shared_ptr<ContextEdge> &CallerEdge) {
+        return CallerEdge.get() == Edge;
+      });
   assert(EI != CallerEdges.end());
   CallerEdges.erase(EI);
 }
@@ -2093,8 +2093,7 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::identifyClones(
     for (auto &Edge : CallerEdges) {
       // Skip any that have been removed by an earlier recursive call.
       if (Edge->Callee == nullptr && Edge->Caller == nullptr) {
-        assert(!std::count(Node->CallerEdges.begin(), Node->CallerEdges.end(),
-                           Edge));
+        assert(!llvm::count(Node->CallerEdges, Edge));
         continue;
       }
       // Ignore any caller we previously visited via another edge.
diff --git a/llvm/lib/Transforms/IPO/MergeFunctions.cpp b/llvm/lib/Transforms/IPO/MergeFunctions.cpp
index 89ddd7b6ad..c8c011d94e 100644
--- a/llvm/lib/Transforms/IPO/MergeFunctions.cpp
+++ b/llvm/lib/Transforms/IPO/MergeFunctions.cpp
@@ -653,7 +653,7 @@ static bool canCreateThunkFor(Function *F) {
   // Don't merge tiny functions using a thunk, since it can just end up
   // making the function larger.
   if (F->size() == 1) {
-    if (F->front().size() <= 2) {
+    if (F->front().sizeWithoutDebug() < 2) {
       LLVM_DEBUG(dbgs() << "canCreateThunkFor: " << F->getName()
                         << " is too small to bother creating a thunk for\n");
       return false;
@@ -662,6 +662,13 @@ static bool canCreateThunkFor(Function *F) {
   return true;
 }
 
+/// Copy metadata from one function to another.
+static void copyMetadataIfPresent(Function *From, Function *To, StringRef Key) {
+  if (MDNode *MD = From->getMetadata(Key)) {
+    To->setMetadata(Key, MD);
+  }
+}
+
 // Replace G with a simple tail call to bitcast(F). Also (unless
 // MergeFunctionsPDI holds) replace direct uses of G with bitcast(F),
 // delete G. Under MergeFunctionsPDI, we use G itself for creating
@@ -740,6 +747,9 @@ void MergeFunctions::writeThunk(Function *F, Function *G) {
   } else {
     NewG->copyAttributesFrom(G);
     NewG->takeName(G);
+    // Ensure CFI type metadata is propagated to the new function.
+    copyMetadataIfPresent(G, NewG, "type");
+    copyMetadataIfPresent(G, NewG, "kcfi_type");
     removeUsers(G);
     G->replaceAllUsesWith(NewG);
     G->eraseFromParent();
@@ -815,6 +825,9 @@ void MergeFunctions::mergeTwoFunctions(Function *F, Function *G) {
                                       F->getAddressSpace(), "", F->getParent());
     NewF->copyAttributesFrom(F);
     NewF->takeName(F);
+    // Ensure CFI type metadata is propagated to the new function.
+    copyMetadataIfPresent(F, NewF, "type");
+    copyMetadataIfPresent(F, NewF, "kcfi_type");
     removeUsers(F);
     F->replaceAllUsesWith(NewF);
 
diff --git a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
index 473b56b80b..e50e74ea6c 100644
--- a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
+++ b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
@@ -733,6 +733,7 @@ struct KernelInfoState : AbstractState {
     SPMDCompatibilityTracker.indicatePessimisticFixpoint();
     ReachedKnownParallelRegions.indicatePessimisticFixpoint();
     ReachedUnknownParallelRegions.indicatePessimisticFixpoint();
+    NestedParallelism = true;
     return ChangeStatus::CHANGED;
   }
 
@@ -762,6 +763,8 @@ struct KernelInfoState : AbstractState {
       return false;
     if (ParallelLevels != RHS.ParallelLevels)
       return false;
+    if (NestedParallelism != RHS.NestedParallelism)
+      return false;
     return true;
   }
 
@@ -3580,7 +3583,8 @@ struct AAKernelInfo : public StateWrapper<KernelInfoState, AbstractAttribute> {
            ", #ParLevels: " +
            (ParallelLevels.isValidState()
                 ? std::to_string(ParallelLevels.size())
-                : "<invalid>");
+                : "<invalid>") +
+           ", NestedPar: " + (NestedParallelism ? "yes" : "no");
   }
 
   /// Create an abstract attribute biew for the position \p IRP.
diff --git a/llvm/lib/Transforms/IPO/SampleContextTracker.cpp b/llvm/lib/Transforms/IPO/SampleContextTracker.cpp
index 79bee8362e..f7a54d428f 100644
--- a/llvm/lib/Transforms/IPO/SampleContextTracker.cpp
+++ b/llvm/lib/Transforms/IPO/SampleContextTracker.cpp
@@ -11,7 +11,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Transforms/IPO/SampleContextTracker.h"
-#include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/IR/InstrTypes.h"
@@ -29,7 +28,7 @@ using namespace sampleprof;
 namespace llvm {
 
 ContextTrieNode *ContextTrieNode::getChildContext(const LineLocation &CallSite,
-                                                  StringRef CalleeName) {
+                                                  FunctionId CalleeName) {
   if (CalleeName.empty())
     return getHottestChildContext(CallSite);
 
@@ -104,7 +103,7 @@ SampleContextTracker::moveContextSamples(ContextTrieNode &ToNodeParent,
 }
 
 void ContextTrieNode::removeChildContext(const LineLocation &CallSite,
-                                         StringRef CalleeName) {
+                                         FunctionId CalleeName) {
   uint64_t Hash = FunctionSamples::getCallSiteHash(CalleeName, CallSite);
   // Note this essentially calls dtor and destroys that child context
   AllChildContext.erase(Hash);
@@ -114,7 +113,7 @@ std::map<uint64_t, ContextTrieNode> &ContextTrieNode::getAllChildContext() {
   return AllChildContext;
 }
 
-StringRef ContextTrieNode::getFuncName() const { return FuncName; }
+FunctionId ContextTrieNode::getFuncName() const { return FuncName; }
 
 FunctionSamples *ContextTrieNode::getFunctionSamples() const {
   return FuncSamples;
@@ -178,7 +177,7 @@ void ContextTrieNode::dumpTree() {
 }
 
 ContextTrieNode *ContextTrieNode::getOrCreateChildContext(
-    const LineLocation &CallSite, StringRef CalleeName, bool AllowCreate) {
+    const LineLocation &CallSite, FunctionId CalleeName, bool AllowCreate) {
   uint64_t Hash = FunctionSamples::getCallSiteHash(CalleeName, CallSite);
   auto It = AllChildContext.find(Hash);
   if (It != AllChildContext.end()) {
@@ -232,14 +231,12 @@ SampleContextTracker::getCalleeContextSamplesFor(const CallBase &Inst,
     return nullptr;
 
   CalleeName = FunctionSamples::getCanonicalFnName(CalleeName);
-  // Convert real function names to MD5 names, if the input profile is
-  // MD5-based.
-  std::string FGUID;
-  CalleeName = getRepInFormat(CalleeName, FunctionSamples::UseMD5, FGUID);
+  
+  FunctionId FName = getRepInFormat(CalleeName);
 
   // For indirect call, CalleeName will be empty, in which case the context
   // profile for callee with largest total samples will be returned.
-  ContextTrieNode *CalleeContext = getCalleeContextFor(DIL, CalleeName);
+  ContextTrieNode *CalleeContext = getCalleeContextFor(DIL, FName);
   if (CalleeContext) {
     FunctionSamples *FSamples = CalleeContext->getFunctionSamples();
     LLVM_DEBUG(if (FSamples) {
@@ -305,27 +302,23 @@ SampleContextTracker::getContextSamplesFor(const SampleContext &Context) {
 SampleContextTracker::ContextSamplesTy &
 SampleContextTracker::getAllContextSamplesFor(const Function &Func) {
   StringRef CanonName = FunctionSamples::getCanonicalFnName(Func);
-  return FuncToCtxtProfiles[CanonName];
+  return FuncToCtxtProfiles[getRepInFormat(CanonName)];
 }
 
 SampleContextTracker::ContextSamplesTy &
 SampleContextTracker::getAllContextSamplesFor(StringRef Name) {
-  return FuncToCtxtProfiles[Name];
+  return FuncToCtxtProfiles[getRepInFormat(Name)];
 }
 
 FunctionSamples *SampleContextTracker::getBaseSamplesFor(const Function &Func,
                                                          bool MergeContext) {
   StringRef CanonName = FunctionSamples::getCanonicalFnName(Func);
-  return getBaseSamplesFor(CanonName, MergeContext);
+  return getBaseSamplesFor(getRepInFormat(CanonName), MergeContext);
 }
 
-FunctionSamples *SampleContextTracker::getBaseSamplesFor(StringRef Name,
+FunctionSamples *SampleContextTracker::getBaseSamplesFor(FunctionId Name,
                                                          bool MergeContext) {
   LLVM_DEBUG(dbgs() << "Getting base profile for function: " << Name << "\n");
-  // Convert real function names to MD5 names, if the input profile is
-  // MD5-based.
-  std::string FGUID;
-  Name = getRepInFormat(Name, FunctionSamples::UseMD5, FGUID);
 
   // Base profile is top-level node (child of root node), so try to retrieve
   // existing top-level node for given function first. If it exists, it could be
@@ -373,7 +366,7 @@ void SampleContextTracker::markContextSamplesInlined(
 ContextTrieNode &SampleContextTracker::getRootContext() { return RootContext; }
 
 void SampleContextTracker::promoteMergeContextSamplesTree(
-    const Instruction &Inst, StringRef CalleeName) {
+    const Instruction &Inst, FunctionId CalleeName) {
   LLVM_DEBUG(dbgs() << "Promoting and merging context tree for instr: \n"
                     << Inst << "\n");
   // Get the caller context for the call instruction, we don't use callee
@@ -458,9 +451,9 @@ void SampleContextTracker::dump() { RootContext.dumpTree(); }
 
 StringRef SampleContextTracker::getFuncNameFor(ContextTrieNode *Node) const {
   if (!FunctionSamples::UseMD5)
-    return Node->getFuncName();
+    return Node->getFuncName().stringRef();
   assert(GUIDToFuncNameMap && "GUIDToFuncNameMap needs to be populated first");
-  return GUIDToFuncNameMap->lookup(std::stoull(Node->getFuncName().data()));
+  return GUIDToFuncNameMap->lookup(Node->getFuncName().getHashCode());
 }
 
 ContextTrieNode *
@@ -470,7 +463,7 @@ SampleContextTracker::getContextFor(const SampleContext &Context) {
 
 ContextTrieNode *
 SampleContextTracker::getCalleeContextFor(const DILocation *DIL,
-                                          StringRef CalleeName) {
+                                          FunctionId CalleeName) {
   assert(DIL && "Expect non-null location");
 
   ContextTrieNode *CallContext = getContextFor(DIL);
@@ -485,7 +478,7 @@ SampleContextTracker::getCalleeContextFor(const DILocation *DIL,
 
 ContextTrieNode *SampleContextTracker::getContextFor(const DILocation *DIL) {
   assert(DIL && "Expect non-null location");
-  SmallVector<std::pair<LineLocation, StringRef>, 10> S;
+  SmallVector<std::pair<LineLocation, FunctionId>, 10> S;
 
   // Use C++ linkage name if possible.
   const DILocation *PrevDIL = DIL;
@@ -494,7 +487,8 @@ ContextTrieNode *SampleContextTracker::getContextFor(const DILocation *DIL) {
     if (Name.empty())
       Name = PrevDIL->getScope()->getSubprogram()->getName();
     S.push_back(
-        std::make_pair(FunctionSamples::getCallSiteIdentifier(DIL), Name));
+        std::make_pair(FunctionSamples::getCallSiteIdentifier(DIL),
+                       getRepInFormat(Name)));
     PrevDIL = DIL;
   }
 
@@ -503,24 +497,14 @@ ContextTrieNode *SampleContextTracker::getContextFor(const DILocation *DIL) {
   StringRef RootName = PrevDIL->getScope()->getSubprogram()->getLinkageName();
   if (RootName.empty())
     RootName = PrevDIL->getScope()->getSubprogram()->getName();
-  S.push_back(std::make_pair(LineLocation(0, 0), RootName));
-
-  // Convert real function names to MD5 names, if the input profile is
-  // MD5-based.
-  std::list<std::string> MD5Names;
-  if (FunctionSamples::UseMD5) {
-    for (auto &Location : S) {
-      MD5Names.emplace_back();
-      getRepInFormat(Location.second, FunctionSamples::UseMD5, MD5Names.back());
-      Location.second = MD5Names.back();
-    }
-  }
+  S.push_back(std::make_pair(LineLocation(0, 0),
+                             getRepInFormat(RootName)));
 
   ContextTrieNode *ContextNode = &RootContext;
   int I = S.size();
   while (--I >= 0 && ContextNode) {
     LineLocation &CallSite = S[I].first;
-    StringRef CalleeName = S[I].second;
+    FunctionId CalleeName = S[I].second;
     ContextNode = ContextNode->getChildContext(CallSite, CalleeName);
   }
 
@@ -540,10 +524,10 @@ SampleContextTracker::getOrCreateContextPath(const SampleContext &Context,
     // Create child node at parent line/disc location
     if (AllowCreate) {
       ContextNode =
-          ContextNode->getOrCreateChildContext(CallSiteLoc, Callsite.FuncName);
+          ContextNode->getOrCreateChildContext(CallSiteLoc, Callsite.Func);
     } else {
       ContextNode =
-          ContextNode->getChildContext(CallSiteLoc, Callsite.FuncName);
+          ContextNode->getChildContext(CallSiteLoc, Callsite.Func);
     }
     CallSiteLoc = Callsite.Location;
   }
@@ -553,12 +537,14 @@ SampleContextTracker::getOrCreateContextPath(const SampleContext &Context,
   return ContextNode;
 }
 
-ContextTrieNode *SampleContextTracker::getTopLevelContextNode(StringRef FName) {
+ContextTrieNode *
+SampleContextTracker::getTopLevelContextNode(FunctionId FName) {
   assert(!FName.empty() && "Top level node query must provide valid name");
   return RootContext.getChildContext(LineLocation(0, 0), FName);
 }
 
-ContextTrieNode &SampleContextTracker::addTopLevelContextNode(StringRef FName) {
+ContextTrieNode &
+SampleContextTracker::addTopLevelContextNode(FunctionId FName) {
   assert(!getTopLevelContextNode(FName) && "Node to add must not exist");
   return *RootContext.getOrCreateChildContext(LineLocation(0, 0), FName);
 }
diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp
index 67a2e167b6..a7773737ef 100644
--- a/llvm/lib/Transforms/IPO/SampleProfile.cpp
+++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp
@@ -424,7 +424,7 @@ struct CandidateComparer {
       return LCS->getBodySamples().size() > RCS->getBodySamples().size();
 
     // Tie breaker using GUID so we have stable/deterministic inlining order
-    return LCS->getGUID(LCS->getName()) < RCS->getGUID(RCS->getName());
+    return LCS->getGUID() < RCS->getGUID();
   }
 };
 
@@ -467,7 +467,7 @@ class SampleProfileMatcher {
 private:
   FunctionSamples *getFlattenedSamplesFor(const Function &F) {
     StringRef CanonFName = FunctionSamples::getCanonicalFnName(F);
-    auto It = FlattenedProfiles.find(CanonFName);
+    auto It = FlattenedProfiles.find(FunctionId(CanonFName));
     if (It != FlattenedProfiles.end())
       return &It->second;
     return nullptr;
@@ -475,18 +475,21 @@ class SampleProfileMatcher {
   void runOnFunction(const Function &F);
   void findIRAnchors(const Function &F,
                      std::map<LineLocation, StringRef> &IRAnchors);
-  void findProfileAnchors(const FunctionSamples &FS,
-                          std::map<LineLocation, StringSet<>> &ProfileAnchors);
+  void findProfileAnchors(
+      const FunctionSamples &FS,
+      std::map<LineLocation, std::unordered_set<FunctionId>>
+          &ProfileAnchors);
   void countMismatchedSamples(const FunctionSamples &FS);
   void countProfileMismatches(
       const Function &F, const FunctionSamples &FS,
       const std::map<LineLocation, StringRef> &IRAnchors,
-      const std::map<LineLocation, StringSet<>> &ProfileAnchors);
+      const std::map<LineLocation, std::unordered_set<FunctionId>>
+          &ProfileAnchors);
   void countProfileCallsiteMismatches(
       const FunctionSamples &FS,
       const std::map<LineLocation, StringRef> &IRAnchors,
-      const std::map<LineLocation, StringSet<>> &ProfileAnchors,
-
+      const std::map<LineLocation, std::unordered_set<FunctionId>>
+          &ProfileAnchors,
       uint64_t &FuncMismatchedCallsites, uint64_t &FuncProfiledCallsites);
   LocToLocMap &getIRToProfileLocationMap(const Function &F) {
     auto Ret = FuncMappings.try_emplace(
@@ -497,7 +500,8 @@ class SampleProfileMatcher {
   void distributeIRToProfileLocationMap(FunctionSamples &FS);
   void runStaleProfileMatching(
       const Function &F, const std::map<LineLocation, StringRef> &IRAnchors,
-      const std::map<LineLocation, StringSet<>> &ProfileAnchors,
+      const std::map<LineLocation, std::unordered_set<FunctionId>>
+          &ProfileAnchors,
       LocToLocMap &IRToProfileLocationMap);
 };
 
@@ -539,7 +543,6 @@ class SampleProfileLoader final : public SampleProfileLoaderBaseImpl<Function> {
   findIndirectCallFunctionSamples(const Instruction &I, uint64_t &Sum) const;
   void findExternalInlineCandidate(CallBase *CB, const FunctionSamples *Samples,
                                    DenseSet<GlobalValue::GUID> &InlinedGUIDs,
-                                   const StringMap<Function *> &SymbolMap,
                                    uint64_t Threshold);
   // Attempt to promote indirect call and also inline the promoted call
   bool tryPromoteAndInlineCandidate(
@@ -574,7 +577,7 @@ class SampleProfileLoader final : public SampleProfileLoaderBaseImpl<Function> {
   /// the function name. If the function name contains suffix, additional
   /// entry is added to map from the stripped name to the function if there
   /// is one-to-one mapping.
-  StringMap<Function *> SymbolMap;
+  HashKeyMap<std::unordered_map, FunctionId, Function *> SymbolMap;
 
   std::function<AssumptionCache &(Function &)> GetAC;
   std::function<TargetTransformInfo &(Function &)> GetTTI;
@@ -616,6 +619,11 @@ class SampleProfileLoader final : public SampleProfileLoaderBaseImpl<Function> {
   // All the Names used in FunctionSamples including outline function
   // names, inline instance names and call target names.
   StringSet<> NamesInProfile;
+  // MD5 version of NamesInProfile. Either NamesInProfile or GUIDsInProfile is
+  // populated, depends on whether the profile uses MD5. Because the name table
+  // generally contains several magnitude more entries than the number of
+  // functions, we do not want to convert all names from one form to another.
+  llvm::DenseSet<uint64_t> GUIDsInProfile;
 
   // For symbol in profile symbol list, whether to regard their profiles
   // to be accurate. It is mainly decided by existance of profile symbol
@@ -760,8 +768,7 @@ SampleProfileLoader::findIndirectCallFunctionSamples(
     assert(L && R && "Expect non-null FunctionSamples");
     if (L->getHeadSamplesEstimate() != R->getHeadSamplesEstimate())
       return L->getHeadSamplesEstimate() > R->getHeadSamplesEstimate();
-    return FunctionSamples::getGUID(L->getName()) <
-           FunctionSamples::getGUID(R->getName());
+    return L->getGUID() < R->getGUID();
   };
 
   if (FunctionSamples::ProfileIsCS) {
@@ -971,13 +978,13 @@ bool SampleProfileLoader::tryPromoteAndInlineCandidate(
   // This prevents allocating an array of zero length in callees below.
   if (MaxNumPromotions == 0)
     return false;
-  auto CalleeFunctionName = Candidate.CalleeSamples->getFuncName();
+  auto CalleeFunctionName = Candidate.CalleeSamples->getFunction();
   auto R = SymbolMap.find(CalleeFunctionName);
-  if (R == SymbolMap.end() || !R->getValue())
+  if (R == SymbolMap.end() || !R->second)
     return false;
 
   auto &CI = *Candidate.CallInstr;
-  if (!doesHistoryAllowICP(CI, R->getValue()->getName()))
+  if (!doesHistoryAllowICP(CI, R->second->getName()))
     return false;
 
   const char *Reason = "Callee function not available";
@@ -987,17 +994,17 @@ bool SampleProfileLoader::tryPromoteAndInlineCandidate(
   // clone the caller first, and inline the cloned caller if it is
   // recursive. As llvm does not inline recursive calls, we will
   // simply ignore it instead of handling it explicitly.
-  if (!R->getValue()->isDeclaration() && R->getValue()->getSubprogram() &&
-      R->getValue()->hasFnAttribute("use-sample-profile") &&
-      R->getValue() != &F && isLegalToPromote(CI, R->getValue(), &Reason)) {
+  if (!R->second->isDeclaration() && R->second->getSubprogram() &&
+      R->second->hasFnAttribute("use-sample-profile") &&
+      R->second != &F && isLegalToPromote(CI, R->second, &Reason)) {
     // For promoted target, set its value with NOMORE_ICP_MAGICNUM count
     // in the value profile metadata so the target won't be promoted again.
     SmallVector<InstrProfValueData, 1> SortedCallTargets = {InstrProfValueData{
-        Function::getGUID(R->getValue()->getName()), NOMORE_ICP_MAGICNUM}};
+        Function::getGUID(R->second->getName()), NOMORE_ICP_MAGICNUM}};
     updateIDTMetaData(CI, SortedCallTargets, 0);
 
     auto *DI = &pgo::promoteIndirectCall(
-        CI, R->getValue(), Candidate.CallsiteCount, Sum, false, ORE);
+        CI, R->second, Candidate.CallsiteCount, Sum, false, ORE);
     if (DI) {
       Sum -= Candidate.CallsiteCount;
       // Do not prorate the indirect callsite distribution since the original
@@ -1026,7 +1033,8 @@ bool SampleProfileLoader::tryPromoteAndInlineCandidate(
     }
   } else {
     LLVM_DEBUG(dbgs() << "\nFailed to promote indirect call to "
-                      << Candidate.CalleeSamples->getFuncName() << " because "
+                      << FunctionSamples::getCanonicalFnName(
+                             Candidate.CallInstr->getName())<< " because "
                       << Reason << "\n");
   }
   return false;
@@ -1071,8 +1079,7 @@ void SampleProfileLoader::emitOptimizationRemarksForInlineCandidates(
 
 void SampleProfileLoader::findExternalInlineCandidate(
     CallBase *CB, const FunctionSamples *Samples,
-    DenseSet<GlobalValue::GUID> &InlinedGUIDs,
-    const StringMap<Function *> &SymbolMap, uint64_t Threshold) {
+    DenseSet<GlobalValue::GUID> &InlinedGUIDs, uint64_t Threshold) {
 
   // If ExternalInlineAdvisor(ReplayInlineAdvisor) wants to inline an external
   // function make sure it's imported
@@ -1081,7 +1088,7 @@ void SampleProfileLoader::findExternalInlineCandidate(
     // just add the direct GUID and move on
     if (!Samples) {
       InlinedGUIDs.insert(
-          FunctionSamples::getGUID(CB->getCalledFunction()->getName()));
+          Function::getGUID(CB->getCalledFunction()->getName()));
       return;
     }
     // Otherwise, drop the threshold to import everything that we can
@@ -1122,22 +1129,20 @@ void SampleProfileLoader::findExternalInlineCandidate(
         CalleeSample->getContext().hasAttribute(ContextShouldBeInlined);
     if (!PreInline && CalleeSample->getHeadSamplesEstimate() < Threshold)
       continue;
-
-    StringRef Name = CalleeSample->getFuncName();
-    Function *Func = SymbolMap.lookup(Name);
+    
+    Function *Func = SymbolMap.lookup(CalleeSample->getFunction());
     // Add to the import list only when it's defined out of module.
     if (!Func || Func->isDeclaration())
-      InlinedGUIDs.insert(FunctionSamples::getGUID(CalleeSample->getName()));
+      InlinedGUIDs.insert(CalleeSample->getGUID());
 
     // Import hot CallTargets, which may not be available in IR because full
     // profile annotation cannot be done until backend compilation in ThinLTO.
     for (const auto &BS : CalleeSample->getBodySamples())
       for (const auto &TS : BS.second.getCallTargets())
-        if (TS.getValue() > Threshold) {
-          StringRef CalleeName = CalleeSample->getFuncName(TS.getKey());
-          const Function *Callee = SymbolMap.lookup(CalleeName);
+        if (TS.second > Threshold) {
+          const Function *Callee = SymbolMap.lookup(TS.first);
           if (!Callee || Callee->isDeclaration())
-            InlinedGUIDs.insert(FunctionSamples::getGUID(TS.getKey()));
+            InlinedGUIDs.insert(TS.first.getHashCode());
         }
 
     // Import hot child context profile associted with callees. Note that this
@@ -1235,7 +1240,7 @@ bool SampleProfileLoader::inlineHotFunctions(
         for (const auto *FS : findIndirectCallFunctionSamples(*I, Sum)) {
           uint64_t SumOrigin = Sum;
           if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) {
-            findExternalInlineCandidate(I, FS, InlinedGUIDs, SymbolMap,
+            findExternalInlineCandidate(I, FS, InlinedGUIDs,
                                         PSI->getOrCompHotCountThreshold());
             continue;
           }
@@ -1256,7 +1261,7 @@ bool SampleProfileLoader::inlineHotFunctions(
         }
       } else if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) {
         findExternalInlineCandidate(I, findCalleeFunctionSamples(*I),
-                                    InlinedGUIDs, SymbolMap,
+                                    InlinedGUIDs,
                                     PSI->getOrCompHotCountThreshold());
       }
     }
@@ -1505,7 +1510,7 @@ bool SampleProfileLoader::inlineHotFunctionsWithPriority(
       for (const auto *FS : CalleeSamples) {
         // TODO: Consider disable pre-lTO ICP for MonoLTO as well
         if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) {
-          findExternalInlineCandidate(I, FS, InlinedGUIDs, SymbolMap,
+          findExternalInlineCandidate(I, FS, InlinedGUIDs,
                                       PSI->getOrCompHotCountThreshold());
           continue;
         }
@@ -1558,7 +1563,7 @@ bool SampleProfileLoader::inlineHotFunctionsWithPriority(
       }
     } else if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) {
       findExternalInlineCandidate(I, findCalleeFunctionSamples(*I),
-                                  InlinedGUIDs, SymbolMap,
+                                  InlinedGUIDs,
                                   PSI->getOrCompHotCountThreshold());
     }
   }
@@ -1644,7 +1649,7 @@ GetSortedValueDataFromCallTargets(const SampleRecord::CallTargetMap &M) {
   SmallVector<InstrProfValueData, 2> R;
   for (const auto &I : SampleRecord::SortCallTargets(M)) {
     R.emplace_back(
-        InstrProfValueData{FunctionSamples::getGUID(I.first), I.second});
+        InstrProfValueData{I.first.getHashCode(), I.second});
   }
   return R;
 }
@@ -1871,7 +1876,8 @@ SampleProfileLoader::buildProfiledCallGraph(Module &M) {
   for (Function &F : M) {
     if (F.isDeclaration() || !F.hasFnAttribute("use-sample-profile"))
       continue;
-    ProfiledCG->addProfiledFunction(FunctionSamples::getCanonicalFnName(F));
+    ProfiledCG->addProfiledFunction(
+          getRepInFormat(FunctionSamples::getCanonicalFnName(F)));
   }
 
   return ProfiledCG;
@@ -2022,8 +2028,16 @@ bool SampleProfileLoader::doInitialization(Module &M,
       ProfileAccurateForSymsInList && PSL && !ProfileSampleAccurate;
   if (ProfAccForSymsInList) {
     NamesInProfile.clear();
-    if (auto NameTable = Reader->getNameTable())
-      NamesInProfile.insert(NameTable->begin(), NameTable->end());
+    GUIDsInProfile.clear();
+    if (auto NameTable = Reader->getNameTable()) {
+      if (FunctionSamples::UseMD5) {
+        for (auto Name : *NameTable)
+          GUIDsInProfile.insert(Name.getHashCode());
+      } else {
+        for (auto Name : *NameTable)
+          NamesInProfile.insert(Name.stringRef());
+      }
+    }
     CoverageTracker.setProfAccForSymsInList(true);
   }
 
@@ -2177,7 +2191,7 @@ void SampleProfileMatcher::findIRAnchors(
 }
 
 void SampleProfileMatcher::countMismatchedSamples(const FunctionSamples &FS) {
-  const auto *FuncDesc = ProbeManager->getDesc(FS.getName());
+  const auto *FuncDesc = ProbeManager->getDesc(FS.getGUID());
   // Skip the function that is external or renamed.
   if (!FuncDesc)
     return;
@@ -2194,7 +2208,8 @@ void SampleProfileMatcher::countMismatchedSamples(const FunctionSamples &FS) {
 void SampleProfileMatcher::countProfileMismatches(
     const Function &F, const FunctionSamples &FS,
     const std::map<LineLocation, StringRef> &IRAnchors,
-    const std::map<LineLocation, StringSet<>> &ProfileAnchors) {
+    const std::map<LineLocation, std::unordered_set<FunctionId>>
+        &ProfileAnchors) {
   [[maybe_unused]] bool IsFuncHashMismatch = false;
   if (FunctionSamples::ProfileIsProbeBased) {
     TotalFuncHashSamples += FS.getTotalSamples();
@@ -2228,7 +2243,8 @@ void SampleProfileMatcher::countProfileMismatches(
 void SampleProfileMatcher::countProfileCallsiteMismatches(
     const FunctionSamples &FS,
     const std::map<LineLocation, StringRef> &IRAnchors,
-    const std::map<LineLocation, StringSet<>> &ProfileAnchors,
+    const std::map<LineLocation, std::unordered_set<FunctionId>>
+        &ProfileAnchors,
     uint64_t &FuncMismatchedCallsites, uint64_t &FuncProfiledCallsites) {
 
   // Check if there are any callsites in the profile that does not match to any
@@ -2263,7 +2279,7 @@ void SampleProfileMatcher::countProfileCallsiteMismatches(
     // reported as mismatching.
     if (IRCalleeName == UnknownIndirectCallee)
       CallsiteIsMatched = true;
-    else if (Callees.size() == 1 && Callees.count(IRCalleeName))
+    else if (Callees.size() == 1 && Callees.count(getRepInFormat(IRCalleeName)))
       CallsiteIsMatched = true;
 
     FuncProfiledCallsites++;
@@ -2275,9 +2291,8 @@ void SampleProfileMatcher::countProfileCallsiteMismatches(
   }
 }
 
-void SampleProfileMatcher::findProfileAnchors(
-    const FunctionSamples &FS,
-    std::map<LineLocation, StringSet<>> &ProfileAnchors) {
+void SampleProfileMatcher::findProfileAnchors(const FunctionSamples &FS,
+                                              std::map<LineLocation, std::unordered_set<FunctionId>> &ProfileAnchors) {
   auto isInvalidLineOffset = [](uint32_t LineOffset) {
     return LineOffset & 0x8000;
   };
@@ -2287,8 +2302,9 @@ void SampleProfileMatcher::findProfileAnchors(
     if (isInvalidLineOffset(Loc.LineOffset))
       continue;
     for (const auto &I : I.second.getCallTargets()) {
-      auto Ret = ProfileAnchors.try_emplace(Loc, StringSet<>());
-      Ret.first->second.insert(I.first());
+      auto Ret = ProfileAnchors.try_emplace(Loc,
+                                            std::unordered_set<FunctionId>());
+      Ret.first->second.insert(I.first);
     }
   }
 
@@ -2298,7 +2314,8 @@ void SampleProfileMatcher::findProfileAnchors(
       continue;
     const auto &CalleeMap = I.second;
     for (const auto &I : CalleeMap) {
-      auto Ret = ProfileAnchors.try_emplace(Loc, StringSet<>());
+      auto Ret = ProfileAnchors.try_emplace(Loc,
+                                            std::unordered_set<FunctionId>());
       Ret.first->second.insert(I.first);
     }
   }
@@ -2322,21 +2339,24 @@ void SampleProfileMatcher::findProfileAnchors(
 //   [1, 2, 3(foo), 4,  7,  8(bar), 9]
 // The output mapping: [2->3, 3->4, 5->7, 6->8, 7->9].
 void SampleProfileMatcher::runStaleProfileMatching(
-    const Function &F, const std::map<LineLocation, StringRef> &IRAnchors,
-    const std::map<LineLocation, StringSet<>> &ProfileAnchors,
+    const Function &F,
+    const std::map<LineLocation, StringRef> &IRAnchors,
+    const std::map<LineLocation, std::unordered_set<FunctionId>>
+        &ProfileAnchors,
     LocToLocMap &IRToProfileLocationMap) {
   LLVM_DEBUG(dbgs() << "Run stale profile matching for " << F.getName()
                     << "\n");
   assert(IRToProfileLocationMap.empty() &&
          "Run stale profile matching only once per function");
 
-  StringMap<std::set<LineLocation>> CalleeToCallsitesMap;
+  std::unordered_map<FunctionId, std::set<LineLocation>>
+      CalleeToCallsitesMap;
   for (const auto &I : ProfileAnchors) {
     const auto &Loc = I.first;
     const auto &Callees = I.second;
     // Filter out possible indirect calls, use direct callee name as anchor.
     if (Callees.size() == 1) {
-      StringRef CalleeName = Callees.begin()->first();
+      FunctionId CalleeName = *Callees.begin();
       const auto &Candidates = CalleeToCallsitesMap.try_emplace(
           CalleeName, std::set<LineLocation>());
       Candidates.first->second.insert(Loc);
@@ -2355,11 +2375,12 @@ void SampleProfileMatcher::runStaleProfileMatching(
 
   for (const auto &IR : IRAnchors) {
     const auto &Loc = IR.first;
-    StringRef CalleeName = IR.second;
+    auto CalleeName = IR.second;
     bool IsMatchedAnchor = false;
     // Match the anchor location in lexical order.
     if (!CalleeName.empty()) {
-      auto CandidateAnchors = CalleeToCallsitesMap.find(CalleeName);
+      auto CandidateAnchors = CalleeToCallsitesMap.find(
+          getRepInFormat(CalleeName));
       if (CandidateAnchors != CalleeToCallsitesMap.end() &&
           !CandidateAnchors->second.empty()) {
         auto CI = CandidateAnchors->second.begin();
@@ -2420,7 +2441,7 @@ void SampleProfileMatcher::runOnFunction(const Function &F) {
   findIRAnchors(F, IRAnchors);
   // Anchors for profile. It's a map from callsite location to a set of callee
   // name.
-  std::map<LineLocation, StringSet<>> ProfileAnchors;
+  std::map<LineLocation, std::unordered_set<FunctionId>> ProfileAnchors;
   findProfileAnchors(*FSFlattened, ProfileAnchors);
 
   // Detect profile mismatch for profile staleness metrics report.
@@ -2499,7 +2520,7 @@ void SampleProfileMatcher::runOnModule() {
 
 void SampleProfileMatcher::distributeIRToProfileLocationMap(
     FunctionSamples &FS) {
-  const auto ProfileMappings = FuncMappings.find(FS.getName());
+  const auto ProfileMappings = FuncMappings.find(FS.getFuncName());
   if (ProfileMappings != FuncMappings.end()) {
     FS.setIRToProfileLocationMap(&(ProfileMappings->second));
   }
@@ -2541,10 +2562,10 @@ bool SampleProfileLoader::runOnModule(Module &M, ModuleAnalysisManager *AM,
     Function *F = dyn_cast<Function>(N_F.getValue());
     if (F == nullptr || OrigName.empty())
       continue;
-    SymbolMap[OrigName] = F;
+    SymbolMap[FunctionId(OrigName)] = F;
     StringRef NewName = FunctionSamples::getCanonicalFnName(*F);
     if (OrigName != NewName && !NewName.empty()) {
-      auto r = SymbolMap.insert(std::make_pair(NewName, F));
+      auto r = SymbolMap.emplace(FunctionId(NewName), F);
       // Failiing to insert means there is already an entry in SymbolMap,
       // thus there are multiple functions that are mapped to the same
       // stripped name. In this case of name conflicting, set the value
@@ -2557,11 +2578,11 @@ bool SampleProfileLoader::runOnModule(Module &M, ModuleAnalysisManager *AM,
     if (Remapper) {
       if (auto MapName = Remapper->lookUpNameInProfile(OrigName)) {
         if (*MapName != OrigName && !MapName->empty())
-          SymbolMap.insert(std::make_pair(*MapName, F));
+          SymbolMap.emplace(FunctionId(*MapName), F);
       }
     }
   }
-  assert(SymbolMap.count(StringRef()) == 0 &&
+  assert(SymbolMap.count(FunctionId()) == 0 &&
          "No empty StringRef should be added in SymbolMap");
 
   if (ReportProfileStaleness || PersistProfileStaleness ||
@@ -2625,7 +2646,9 @@ bool SampleProfileLoader::runOnFunction(Function &F, ModuleAnalysisManager *AM)
     // but not cold accumulatively...), so the outline function showing up as
     // cold in sampled binary will actually not be cold after current build.
     StringRef CanonName = FunctionSamples::getCanonicalFnName(F);
-    if (NamesInProfile.count(CanonName))
+    if ((FunctionSamples::UseMD5 &&
+         GUIDsInProfile.count(Function::getGUID(CanonName))) ||
+        (!FunctionSamples::UseMD5 && NamesInProfile.count(CanonName)))
       initialEntryCount = -1;
   }
 
diff --git a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
index ae3ec7c044..df5d14b56b 100644
--- a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
+++ b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
@@ -58,7 +58,6 @@
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
-#include "llvm/ADT/iterator_range.h"
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/BasicAliasAnalysis.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
index 44f6e37cb3..87181650e7 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
@@ -1566,7 +1566,8 @@ Instruction *InstCombinerImpl::visitAdd(BinaryOperator &I) {
     return replaceInstUsesWith(I, Constant::getNullValue(I.getType()));
 
   // A+B --> A|B iff A and B have no bits set in common.
-  if (haveNoCommonBitsSet(LHS, RHS, SQ.getWithInstruction(&I)))
+  WithCache<const Value *> LHSCache(LHS), RHSCache(RHS);
+  if (haveNoCommonBitsSet(LHSCache, RHSCache, SQ.getWithInstruction(&I)))
     return BinaryOperator::CreateOr(LHS, RHS);
 
   if (Instruction *Ext = narrowMathIfNoOverflow(I))
@@ -1661,11 +1662,12 @@ Instruction *InstCombinerImpl::visitAdd(BinaryOperator &I) {
   // willNotOverflowUnsignedAdd to reduce the number of invocations of
   // computeKnownBits.
   bool Changed = false;
-  if (!I.hasNoSignedWrap() && willNotOverflowSignedAdd(LHS, RHS, I)) {
+  if (!I.hasNoSignedWrap() && willNotOverflowSignedAdd(LHSCache, RHSCache, I)) {
     Changed = true;
     I.setHasNoSignedWrap(true);
   }
-  if (!I.hasNoUnsignedWrap() && willNotOverflowUnsignedAdd(LHS, RHS, I)) {
+  if (!I.hasNoUnsignedWrap() &&
+      willNotOverflowUnsignedAdd(LHSCache, RHSCache, I)) {
     Changed = true;
     I.setHasNoUnsignedWrap(true);
   }
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
index cbdab3e9c5..3e0218d9b7 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
@@ -2250,6 +2250,14 @@ Instruction *InstCombinerImpl::visitAnd(BinaryOperator &I) {
     return SelectInst::Create(Cmp, ConstantInt::getNullValue(Ty), Y);
   }
 
+  // Canonicalize:
+  // (X +/- Y) & Y --> ~X & Y when Y is a power of 2.
+  if (match(&I, m_c_And(m_Value(Y), m_OneUse(m_CombineOr(
+                                        m_c_Add(m_Value(X), m_Deferred(Y)),
+                                        m_Sub(m_Value(X), m_Deferred(Y)))))) &&
+      isKnownToBeAPowerOfTwo(Y, /*OrZero*/ true, /*Depth*/ 0, &I))
+    return BinaryOperator::CreateAnd(Builder.CreateNot(X), Y);
+
   const APInt *C;
   if (match(Op1, m_APInt(C))) {
     const APInt *XorC;
@@ -2732,100 +2740,114 @@ static Instruction *matchFunnelShift(Instruction &Or, InstCombinerImpl &IC) {
   // rotate matching code under visitSelect and visitTrunc?
   unsigned Width = Or.getType()->getScalarSizeInBits();
 
-  // First, find an or'd pair of opposite shifts:
-  // or (lshr ShVal0, ShAmt0), (shl ShVal1, ShAmt1)
-  BinaryOperator *Or0, *Or1;
-  if (!match(Or.getOperand(0), m_BinOp(Or0)) ||
-      !match(Or.getOperand(1), m_BinOp(Or1)))
+  Instruction *Or0, *Or1;
+  if (!match(Or.getOperand(0), m_Instruction(Or0)) ||
+      !match(Or.getOperand(1), m_Instruction(Or1)))
     return nullptr;
 
-  Value *ShVal0, *ShVal1, *ShAmt0, *ShAmt1;
-  if (!match(Or0, m_OneUse(m_LogicalShift(m_Value(ShVal0), m_Value(ShAmt0)))) ||
-      !match(Or1, m_OneUse(m_LogicalShift(m_Value(ShVal1), m_Value(ShAmt1)))) ||
-      Or0->getOpcode() == Or1->getOpcode())
-    return nullptr;
-
-  // Canonicalize to or(shl(ShVal0, ShAmt0), lshr(ShVal1, ShAmt1)).
-  if (Or0->getOpcode() == BinaryOperator::LShr) {
-    std::swap(Or0, Or1);
-    std::swap(ShVal0, ShVal1);
-    std::swap(ShAmt0, ShAmt1);
-  }
-  assert(Or0->getOpcode() == BinaryOperator::Shl &&
-         Or1->getOpcode() == BinaryOperator::LShr &&
-         "Illegal or(shift,shift) pair");
-
-  // Match the shift amount operands for a funnel shift pattern. This always
-  // matches a subtraction on the R operand.
-  auto matchShiftAmount = [&](Value *L, Value *R, unsigned Width) -> Value * {
-    // Check for constant shift amounts that sum to the bitwidth.
-    const APInt *LI, *RI;
-    if (match(L, m_APIntAllowUndef(LI)) && match(R, m_APIntAllowUndef(RI)))
-      if (LI->ult(Width) && RI->ult(Width) && (*LI + *RI) == Width)
-        return ConstantInt::get(L->getType(), *LI);
-
-    Constant *LC, *RC;
-    if (match(L, m_Constant(LC)) && match(R, m_Constant(RC)) &&
-        match(L, m_SpecificInt_ICMP(ICmpInst::ICMP_ULT, APInt(Width, Width))) &&
-        match(R, m_SpecificInt_ICMP(ICmpInst::ICMP_ULT, APInt(Width, Width))) &&
-        match(ConstantExpr::getAdd(LC, RC), m_SpecificIntAllowUndef(Width)))
-      return ConstantExpr::mergeUndefsWith(LC, RC);
-
-    // (shl ShVal, X) | (lshr ShVal, (Width - x)) iff X < Width.
-    // We limit this to X < Width in case the backend re-expands the intrinsic,
-    // and has to reintroduce a shift modulo operation (InstCombine might remove
-    // it after this fold). This still doesn't guarantee that the final codegen
-    // will match this original pattern.
-    if (match(R, m_OneUse(m_Sub(m_SpecificInt(Width), m_Specific(L))))) {
-      KnownBits KnownL = IC.computeKnownBits(L, /*Depth*/ 0, &Or);
-      return KnownL.getMaxValue().ult(Width) ? L : nullptr;
-    }
+  bool IsFshl = true; // Sub on LSHR.
+  SmallVector<Value *, 3> FShiftArgs;
 
-    // For non-constant cases, the following patterns currently only work for
-    // rotation patterns.
-    // TODO: Add general funnel-shift compatible patterns.
-    if (ShVal0 != ShVal1)
+  // First, find an or'd pair of opposite shifts:
+  // or (lshr ShVal0, ShAmt0), (shl ShVal1, ShAmt1)
+  if (isa<BinaryOperator>(Or0) && isa<BinaryOperator>(Or1)) {
+    Value *ShVal0, *ShVal1, *ShAmt0, *ShAmt1;
+    if (!match(Or0,
+               m_OneUse(m_LogicalShift(m_Value(ShVal0), m_Value(ShAmt0)))) ||
+        !match(Or1,
+               m_OneUse(m_LogicalShift(m_Value(ShVal1), m_Value(ShAmt1)))) ||
+        Or0->getOpcode() == Or1->getOpcode())
       return nullptr;
 
-    // For non-constant cases we don't support non-pow2 shift masks.
-    // TODO: Is it worth matching urem as well?
-    if (!isPowerOf2_32(Width))
-      return nullptr;
+    // Canonicalize to or(shl(ShVal0, ShAmt0), lshr(ShVal1, ShAmt1)).
+    if (Or0->getOpcode() == BinaryOperator::LShr) {
+      std::swap(Or0, Or1);
+      std::swap(ShVal0, ShVal1);
+      std::swap(ShAmt0, ShAmt1);
+    }
+    assert(Or0->getOpcode() == BinaryOperator::Shl &&
+           Or1->getOpcode() == BinaryOperator::LShr &&
+           "Illegal or(shift,shift) pair");
+
+    // Match the shift amount operands for a funnel shift pattern. This always
+    // matches a subtraction on the R operand.
+    auto matchShiftAmount = [&](Value *L, Value *R, unsigned Width) -> Value * {
+      // Check for constant shift amounts that sum to the bitwidth.
+      const APInt *LI, *RI;
+      if (match(L, m_APIntAllowUndef(LI)) && match(R, m_APIntAllowUndef(RI)))
+        if (LI->ult(Width) && RI->ult(Width) && (*LI + *RI) == Width)
+          return ConstantInt::get(L->getType(), *LI);
+
+      Constant *LC, *RC;
+      if (match(L, m_Constant(LC)) && match(R, m_Constant(RC)) &&
+          match(L,
+                m_SpecificInt_ICMP(ICmpInst::ICMP_ULT, APInt(Width, Width))) &&
+          match(R,
+                m_SpecificInt_ICMP(ICmpInst::ICMP_ULT, APInt(Width, Width))) &&
+          match(ConstantExpr::getAdd(LC, RC), m_SpecificIntAllowUndef(Width)))
+        return ConstantExpr::mergeUndefsWith(LC, RC);
+
+      // (shl ShVal, X) | (lshr ShVal, (Width - x)) iff X < Width.
+      // We limit this to X < Width in case the backend re-expands the
+      // intrinsic, and has to reintroduce a shift modulo operation (InstCombine
+      // might remove it after this fold). This still doesn't guarantee that the
+      // final codegen will match this original pattern.
+      if (match(R, m_OneUse(m_Sub(m_SpecificInt(Width), m_Specific(L))))) {
+        KnownBits KnownL = IC.computeKnownBits(L, /*Depth*/ 0, &Or);
+        return KnownL.getMaxValue().ult(Width) ? L : nullptr;
+      }
 
-    // The shift amount may be masked with negation:
-    // (shl ShVal, (X & (Width - 1))) | (lshr ShVal, ((-X) & (Width - 1)))
-    Value *X;
-    unsigned Mask = Width - 1;
-    if (match(L, m_And(m_Value(X), m_SpecificInt(Mask))) &&
-        match(R, m_And(m_Neg(m_Specific(X)), m_SpecificInt(Mask))))
-      return X;
+      // For non-constant cases, the following patterns currently only work for
+      // rotation patterns.
+      // TODO: Add general funnel-shift compatible patterns.
+      if (ShVal0 != ShVal1)
+        return nullptr;
 
-    // Similar to above, but the shift amount may be extended after masking,
-    // so return the extended value as the parameter for the intrinsic.
-    if (match(L, m_ZExt(m_And(m_Value(X), m_SpecificInt(Mask)))) &&
-        match(R, m_And(m_Neg(m_ZExt(m_And(m_Specific(X), m_SpecificInt(Mask)))),
-                       m_SpecificInt(Mask))))
-      return L;
+      // For non-constant cases we don't support non-pow2 shift masks.
+      // TODO: Is it worth matching urem as well?
+      if (!isPowerOf2_32(Width))
+        return nullptr;
 
-    if (match(L, m_ZExt(m_And(m_Value(X), m_SpecificInt(Mask)))) &&
-        match(R, m_ZExt(m_And(m_Neg(m_Specific(X)), m_SpecificInt(Mask)))))
-      return L;
+      // The shift amount may be masked with negation:
+      // (shl ShVal, (X & (Width - 1))) | (lshr ShVal, ((-X) & (Width - 1)))
+      Value *X;
+      unsigned Mask = Width - 1;
+      if (match(L, m_And(m_Value(X), m_SpecificInt(Mask))) &&
+          match(R, m_And(m_Neg(m_Specific(X)), m_SpecificInt(Mask))))
+        return X;
+
+      // Similar to above, but the shift amount may be extended after masking,
+      // so return the extended value as the parameter for the intrinsic.
+      if (match(L, m_ZExt(m_And(m_Value(X), m_SpecificInt(Mask)))) &&
+          match(R,
+                m_And(m_Neg(m_ZExt(m_And(m_Specific(X), m_SpecificInt(Mask)))),
+                      m_SpecificInt(Mask))))
+        return L;
+
+      if (match(L, m_ZExt(m_And(m_Value(X), m_SpecificInt(Mask)))) &&
+          match(R, m_ZExt(m_And(m_Neg(m_Specific(X)), m_SpecificInt(Mask)))))
+        return L;
 
-    return nullptr;
-  };
+      return nullptr;
+    };
 
-  Value *ShAmt = matchShiftAmount(ShAmt0, ShAmt1, Width);
-  bool IsFshl = true; // Sub on LSHR.
-  if (!ShAmt) {
-    ShAmt = matchShiftAmount(ShAmt1, ShAmt0, Width);
-    IsFshl = false; // Sub on SHL.
+    Value *ShAmt = matchShiftAmount(ShAmt0, ShAmt1, Width);
+    if (!ShAmt) {
+      ShAmt = matchShiftAmount(ShAmt1, ShAmt0, Width);
+      IsFshl = false; // Sub on SHL.
+    }
+    if (!ShAmt)
+      return nullptr;
+
+    FShiftArgs = {ShVal0, ShVal1, ShAmt};
   }
-  if (!ShAmt)
+
+  if (FShiftArgs.empty())
     return nullptr;
 
   Intrinsic::ID IID = IsFshl ? Intrinsic::fshl : Intrinsic::fshr;
   Function *F = Intrinsic::getDeclaration(Or.getModule(), IID, Or.getType());
-  return CallInst::Create(F, {ShVal0, ShVal1, ShAmt});
+  return CallInst::Create(F, FShiftArgs);
 }
 
 /// Attempt to combine or(zext(x),shl(zext(y),bw/2) concat packing patterns.
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
index 95b506f0e3..66e2b6c72c 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -2899,9 +2899,17 @@ Instruction *InstCombinerImpl::foldICmpSubConstant(ICmpInst &Cmp,
 static Value *createLogicFromTable(const std::bitset<4> &Table, Value *Op0,
                                    Value *Op1, IRBuilderBase &Builder,
                                    bool HasOneUse) {
+  auto FoldConstant = [&](bool Val) {
+    Constant *Res = Val ? Builder.getTrue() : Builder.getFalse();
+    if (Op0->getType()->isVectorTy())
+      Res = ConstantVector::getSplat(
+          cast<VectorType>(Op0->getType())->getElementCount(), Res);
+    return Res;
+  };
+
   switch (Table.to_ulong()) {
   case 0: // 0 0 0 0
-    return Builder.getFalse();
+    return FoldConstant(false);
   case 1: // 0 0 0 1
     return HasOneUse ? Builder.CreateNot(Builder.CreateOr(Op0, Op1)) : nullptr;
   case 2: // 0 0 1 0
@@ -2931,7 +2939,7 @@ static Value *createLogicFromTable(const std::bitset<4> &Table, Value *Op0,
   case 14: // 1 1 1 0
     return Builder.CreateOr(Op0, Op1);
   case 15: // 1 1 1 1
-    return Builder.getTrue();
+    return FoldConstant(true);
   default:
     llvm_unreachable("Invalid Operation");
   }
@@ -5382,35 +5390,6 @@ Instruction *InstCombinerImpl::foldICmpEquality(ICmpInst &I) {
       return new ICmpInst(Pred, A, Builder.CreateTrunc(B, A->getType()));
   }
 
-  // Test if 2 values have different or same signbits:
-  // (X u>> BitWidth - 1) == zext (Y s> -1) --> (X ^ Y) < 0
-  // (X u>> BitWidth - 1) != zext (Y s> -1) --> (X ^ Y) > -1
-  // (X s>> BitWidth - 1) == sext (Y s> -1) --> (X ^ Y) < 0
-  // (X s>> BitWidth - 1) != sext (Y s> -1) --> (X ^ Y) > -1
-  Instruction *ExtI;
-  if (match(Op1, m_CombineAnd(m_Instruction(ExtI), m_ZExtOrSExt(m_Value(A)))) &&
-      (Op0->hasOneUse() || Op1->hasOneUse())) {
-    unsigned OpWidth = Op0->getType()->getScalarSizeInBits();
-    Instruction *ShiftI;
-    Value *X, *Y;
-    ICmpInst::Predicate Pred2;
-    if (match(Op0, m_CombineAnd(m_Instruction(ShiftI),
-                                m_Shr(m_Value(X),
-                                      m_SpecificIntAllowUndef(OpWidth - 1)))) &&
-        match(A, m_ICmp(Pred2, m_Value(Y), m_AllOnes())) &&
-        Pred2 == ICmpInst::ICMP_SGT && X->getType() == Y->getType()) {
-      unsigned ExtOpc = ExtI->getOpcode();
-      unsigned ShiftOpc = ShiftI->getOpcode();
-      if ((ExtOpc == Instruction::ZExt && ShiftOpc == Instruction::LShr) ||
-          (ExtOpc == Instruction::SExt && ShiftOpc == Instruction::AShr)) {
-        Value *Xor = Builder.CreateXor(X, Y, "xor.signbits");
-        Value *R = (Pred == ICmpInst::ICMP_EQ) ? Builder.CreateIsNeg(Xor)
-                                               : Builder.CreateIsNotNeg(Xor);
-        return replaceInstUsesWith(I, R);
-      }
-    }
-  }
-
   // (A >> C) == (B >> C) --> (A^B) u< (1 << C)
   // For lshr and ashr pairs.
   const APInt *AP1, *AP2;
@@ -7186,6 +7165,33 @@ Instruction *InstCombinerImpl::visitICmpInst(ICmpInst &I) {
       if (Instruction *R = processUMulZExtIdiom(I, Op1, Op0, *this))
         return R;
     }
+
+    Value *X, *Y;
+    // Signbit test folds
+    // Fold (X u>> BitWidth - 1 Pred ZExt(i1))  -->  X s< 0 Pred i1
+    // Fold (X s>> BitWidth - 1 Pred SExt(i1))  -->  X s< 0 Pred i1
+    Instruction *ExtI;
+    if ((I.isUnsigned() || I.isEquality()) &&
+        match(Op1,
+              m_CombineAnd(m_Instruction(ExtI), m_ZExtOrSExt(m_Value(Y)))) &&
+        Y->getType()->getScalarSizeInBits() == 1 &&
+        (Op0->hasOneUse() || Op1->hasOneUse())) {
+      unsigned OpWidth = Op0->getType()->getScalarSizeInBits();
+      Instruction *ShiftI;
+      if (match(Op0, m_CombineAnd(m_Instruction(ShiftI),
+                                  m_Shr(m_Value(X), m_SpecificIntAllowUndef(
+                                                        OpWidth - 1))))) {
+        unsigned ExtOpc = ExtI->getOpcode();
+        unsigned ShiftOpc = ShiftI->getOpcode();
+        if ((ExtOpc == Instruction::ZExt && ShiftOpc == Instruction::LShr) ||
+            (ExtOpc == Instruction::SExt && ShiftOpc == Instruction::AShr)) {
+          Value *SLTZero =
+              Builder.CreateICmpSLT(X, Constant::getNullValue(X->getType()));
+          Value *Cmp = Builder.CreateICmp(Pred, SLTZero, Y, I.getName());
+          return replaceInstUsesWith(I, Cmp);
+        }
+      }
+    }
   }
 
   if (Instruction *Res = foldICmpEquality(I))
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
index 6d72d3ee38..a53d67b289 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
+++ b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
@@ -295,13 +295,15 @@ class LLVM_LIBRARY_VISIBILITY InstCombinerImpl final
 
   Instruction *transformSExtICmp(ICmpInst *Cmp, SExtInst &Sext);
 
-  bool willNotOverflowSignedAdd(const Value *LHS, const Value *RHS,
+  bool willNotOverflowSignedAdd(const WithCache<const Value *> &LHS,
+                                const WithCache<const Value *> &RHS,
                                 const Instruction &CxtI) const {
     return computeOverflowForSignedAdd(LHS, RHS, &CxtI) ==
            OverflowResult::NeverOverflows;
   }
 
-  bool willNotOverflowUnsignedAdd(const Value *LHS, const Value *RHS,
+  bool willNotOverflowUnsignedAdd(const WithCache<const Value *> &LHS,
+                                  const WithCache<const Value *> &RHS,
                                   const Instruction &CxtI) const {
     return computeOverflowForUnsignedAdd(LHS, RHS, &CxtI) ==
            OverflowResult::NeverOverflows;
@@ -548,15 +550,6 @@ class LLVM_LIBRARY_VISIBILITY InstCombinerImpl final
                                     APInt &UndefElts, unsigned Depth = 0,
                                     bool AllowMultipleUsers = false) override;
 
-  /// Attempts to replace V with a simpler value based on the demanded
-  /// floating-point classes
-  Value *SimplifyDemandedUseFPClass(Value *V, FPClassTest DemandedMask,
-                                    KnownFPClass &Known, unsigned Depth,
-                                    Instruction *CxtI);
-  bool SimplifyDemandedFPClass(Instruction *I, unsigned Op,
-                               FPClassTest DemandedMask, KnownFPClass &Known,
-                               unsigned Depth = 0);
-
   /// Canonicalize the position of binops relative to shufflevector.
   Instruction *foldVectorBinop(BinaryOperator &Inst);
   Instruction *foldVectorSelect(SelectInst &Sel);
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
index ec4523d1a7..5b5760665f 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
@@ -36,10 +36,12 @@ static cl::opt<unsigned> MaxCopiedFromConstantUsers(
     cl::desc("Maximum users to visit in copy from constant transform"),
     cl::Hidden);
 
+namespace llvm {
 cl::opt<bool> EnableInferAlignmentPass(
     "enable-infer-alignment-pass", cl::init(true), cl::Hidden, cl::ZeroOrMore,
     cl::desc("Enable the InferAlignment pass, disabling alignment inference in "
              "InstCombine"));
+}
 
 /// isOnlyCopiedFromConstantMemory - Recursively walk the uses of a (derived)
 /// pointer to an alloca.  Ignore any reads of the pointer, return false if we
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
index 560c87b6ef..bc784390c2 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
@@ -923,8 +923,7 @@ static bool isMultiple(const APInt &C1, const APInt &C2, APInt &Quotient,
   return Remainder.isMinValue();
 }
 
-static Instruction *foldIDivShl(BinaryOperator &I,
-                                InstCombiner::BuilderTy &Builder) {
+static Value *foldIDivShl(BinaryOperator &I, InstCombiner::BuilderTy &Builder) {
   assert((I.getOpcode() == Instruction::SDiv ||
           I.getOpcode() == Instruction::UDiv) &&
          "Expected integer divide");
@@ -933,7 +932,6 @@ static Instruction *foldIDivShl(BinaryOperator &I,
   Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
   Type *Ty = I.getType();
 
-  Instruction *Ret = nullptr;
   Value *X, *Y, *Z;
 
   // With appropriate no-wrap constraints, remove a common factor in the
@@ -948,12 +946,12 @@ static Instruction *foldIDivShl(BinaryOperator &I,
 
     // (X * Y) u/ (X << Z) --> Y u>> Z
     if (!IsSigned && HasNUW)
-      Ret = BinaryOperator::CreateLShr(Y, Z);
+      return Builder.CreateLShr(Y, Z, "", I.isExact());
 
     // (X * Y) s/ (X << Z) --> Y s/ (1 << Z)
     if (IsSigned && HasNSW && (Op0->hasOneUse() || Op1->hasOneUse())) {
       Value *Shl = Builder.CreateShl(ConstantInt::get(Ty, 1), Z);
-      Ret = BinaryOperator::CreateSDiv(Y, Shl);
+      return Builder.CreateSDiv(Y, Shl, "", I.isExact());
     }
   }
 
@@ -971,20 +969,38 @@ static Instruction *foldIDivShl(BinaryOperator &I,
         ((Shl0->hasNoUnsignedWrap() && Shl1->hasNoUnsignedWrap()) ||
          (Shl0->hasNoUnsignedWrap() && Shl0->hasNoSignedWrap() &&
           Shl1->hasNoSignedWrap())))
-      Ret = BinaryOperator::CreateUDiv(X, Y);
+      return Builder.CreateUDiv(X, Y, "", I.isExact());
 
     // For signed div, we need 'nsw' on both shifts + 'nuw' on the divisor.
     // (X << Z) / (Y << Z) --> X / Y
     if (IsSigned && Shl0->hasNoSignedWrap() && Shl1->hasNoSignedWrap() &&
         Shl1->hasNoUnsignedWrap())
-      Ret = BinaryOperator::CreateSDiv(X, Y);
+      return Builder.CreateSDiv(X, Y, "", I.isExact());
   }
 
-  if (!Ret)
-    return nullptr;
+  // If X << Y and X << Z does not overflow, then:
+  // (X << Y) / (X << Z) -> (1 << Y) / (1 << Z) -> 1 << Y >> Z
+  if (match(Op0, m_Shl(m_Value(X), m_Value(Y))) &&
+      match(Op1, m_Shl(m_Specific(X), m_Value(Z)))) {
+    auto *Shl0 = cast<OverflowingBinaryOperator>(Op0);
+    auto *Shl1 = cast<OverflowingBinaryOperator>(Op1);
+
+    if (IsSigned ? (Shl0->hasNoSignedWrap() && Shl1->hasNoSignedWrap())
+                 : (Shl0->hasNoUnsignedWrap() && Shl1->hasNoUnsignedWrap())) {
+      Constant *One = ConstantInt::get(X->getType(), 1);
+      // Only preserve the nsw flag if dividend has nsw
+      // or divisor has nsw and operator is sdiv.
+      Value *Dividend = Builder.CreateShl(
+          One, Y, "shl.dividend",
+          /*HasNUW*/ true,
+          /*HasNSW*/
+          IsSigned ? (Shl0->hasNoUnsignedWrap() || Shl1->hasNoUnsignedWrap())
+                   : Shl0->hasNoSignedWrap());
+      return Builder.CreateLShr(Dividend, Z, "", I.isExact());
+    }
+  }
 
-  Ret->setIsExact(I.isExact());
-  return Ret;
+  return nullptr;
 }
 
 /// This function implements the transforms common to both integer division
@@ -1161,8 +1177,8 @@ Instruction *InstCombinerImpl::commonIDivTransforms(BinaryOperator &I) {
       return NewDiv;
     }
 
-  if (Instruction *R = foldIDivShl(I, Builder))
-    return R;
+  if (Value *R = foldIDivShl(I, Builder))
+    return replaceInstUsesWith(I, R);
 
   // With the appropriate no-wrap constraint, remove a multiply by the divisor
   // after peeking through another divide:
@@ -1764,6 +1780,21 @@ Instruction *InstCombinerImpl::visitFDiv(BinaryOperator &I) {
     return replaceInstUsesWith(I, Pow);
   }
 
+  // powi(X, Y) / X --> powi(X, Y-1)
+  // This is legal when (Y - 1) can't wraparound, in which case reassoc and nnan
+  // are required.
+  // TODO: Multi-use may be also better off creating Powi(x,y-1)
+  if (I.hasAllowReassoc() && I.hasNoNaNs() &&
+      match(Op0, m_OneUse(m_Intrinsic<Intrinsic::powi>(m_Specific(Op1),
+                                                       m_Value(Y)))) &&
+      willNotOverflowSignedSub(Y, ConstantInt::get(Y->getType(), 1), I)) {
+    Constant *NegOne = ConstantInt::getAllOnesValue(Y->getType());
+    Value *Y1 = Builder.CreateAdd(Y, NegOne);
+    Type *Types[] = {Op1->getType(), Y1->getType()};
+    Value *Pow = Builder.CreateIntrinsic(Intrinsic::powi, Types, {Op1, Y1}, &I);
+    return replaceInstUsesWith(I, Pow);
+  }
+
   return nullptr;
 }
 
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineNegator.cpp b/llvm/lib/Transforms/InstCombine/InstCombineNegator.cpp
index 6efa69b368..53b34b03ca 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineNegator.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineNegator.cpp
@@ -20,7 +20,6 @@
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Twine.h"
-#include "llvm/ADT/iterator_range.h"
 #include "llvm/Analysis/TargetFolder.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/Constant.h"
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp b/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
index 83defd5816..e178f9536b 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
@@ -941,6 +941,60 @@ Instruction *InstCombinerImpl::foldLShrOverflowBit(BinaryOperator &I) {
   return new ZExtInst(Overflow, Ty);
 }
 
+// Try to set nuw/nsw flags on shl or exact flag on lshr/ashr using knownbits.
+static bool setShiftFlags(BinaryOperator &I, const SimplifyQuery &Q) {
+  assert(I.isShift() && "Expected a shift as input");
+  // We already have all the flags.
+  if (I.getOpcode() == Instruction::Shl) {
+    if (I.hasNoUnsignedWrap() && I.hasNoSignedWrap())
+      return false;
+  } else {
+    if (I.isExact())
+      return false;
+  }
+
+  // Compute what we know about shift count.
+  KnownBits KnownCnt =
+      computeKnownBits(I.getOperand(1), Q.DL, /*Depth*/ 0, Q.AC, Q.CxtI, Q.DT);
+  // If we know nothing about shift count or its a poison shift, we won't be
+  // able to prove anything so return before computing shift amount.
+  if (KnownCnt.isUnknown())
+    return false;
+  unsigned BitWidth = KnownCnt.getBitWidth();
+  APInt MaxCnt = KnownCnt.getMaxValue();
+  if (MaxCnt.uge(BitWidth))
+    return false;
+
+  KnownBits KnownAmt =
+      computeKnownBits(I.getOperand(0), Q.DL, /*Depth*/ 0, Q.AC, Q.CxtI, Q.DT);
+  bool Changed = false;
+
+  if (I.getOpcode() == Instruction::Shl) {
+    // If we have as many leading zeros than maximum shift cnt we have nuw.
+    if (!I.hasNoUnsignedWrap() && MaxCnt.ule(KnownAmt.countMinLeadingZeros())) {
+      I.setHasNoUnsignedWrap();
+      Changed = true;
+    }
+    // If we have more sign bits than maximum shift cnt we have nsw.
+    if (!I.hasNoSignedWrap()) {
+      if (MaxCnt.ult(KnownAmt.countMinSignBits()) ||
+          MaxCnt.ult(ComputeNumSignBits(I.getOperand(0), Q.DL, /*Depth*/ 0,
+                                        Q.AC, Q.CxtI, Q.DT))) {
+        I.setHasNoSignedWrap();
+        Changed = true;
+      }
+    }
+    return Changed;
+  }
+
+  // If we have at least as many trailing zeros as maximum count then we have
+  // exact.
+  Changed = MaxCnt.ule(KnownAmt.countMinTrailingZeros());
+  I.setIsExact(Changed);
+
+  return Changed;
+}
+
 Instruction *InstCombinerImpl::visitShl(BinaryOperator &I) {
   const SimplifyQuery Q = SQ.getWithInstruction(&I);
 
@@ -1121,22 +1175,11 @@ Instruction *InstCombinerImpl::visitShl(BinaryOperator &I) {
       Value *NewShift = Builder.CreateShl(X, Op1);
       return BinaryOperator::CreateSub(NewLHS, NewShift);
     }
-
-    // If the shifted-out value is known-zero, then this is a NUW shift.
-    if (!I.hasNoUnsignedWrap() &&
-        MaskedValueIsZero(Op0, APInt::getHighBitsSet(BitWidth, ShAmtC), 0,
-                          &I)) {
-      I.setHasNoUnsignedWrap();
-      return &I;
-    }
-
-    // If the shifted-out value is all signbits, then this is a NSW shift.
-    if (!I.hasNoSignedWrap() && ComputeNumSignBits(Op0, 0, &I) > ShAmtC) {
-      I.setHasNoSignedWrap();
-      return &I;
-    }
   }
 
+  if (setShiftFlags(I, Q))
+    return &I;
+
   // Transform  (x >> y) << y  to  x & (-1 << y)
   // Valid for any type of right-shift.
   Value *X;
@@ -1427,15 +1470,12 @@ Instruction *InstCombinerImpl::visitLShr(BinaryOperator &I) {
       Value *And = Builder.CreateAnd(BoolX, BoolY);
       return new ZExtInst(And, Ty);
     }
-
-    // If the shifted-out value is known-zero, then this is an exact shift.
-    if (!I.isExact() &&
-        MaskedValueIsZero(Op0, APInt::getLowBitsSet(BitWidth, ShAmtC), 0, &I)) {
-      I.setIsExact();
-      return &I;
-    }
   }
 
+  const SimplifyQuery Q = SQ.getWithInstruction(&I);
+  if (setShiftFlags(I, Q))
+    return &I;
+
   // Transform  (x << y) >> y  to  x & (-1 >> y)
   if (match(Op0, m_OneUse(m_Shl(m_Value(X), m_Specific(Op1))))) {
     Constant *AllOnes = ConstantInt::getAllOnesValue(Ty);
@@ -1594,15 +1634,12 @@ Instruction *InstCombinerImpl::visitAShr(BinaryOperator &I) {
       if (match(Op0, m_OneUse(m_NSWSub(m_Value(X), m_Value(Y)))))
         return new SExtInst(Builder.CreateICmpSLT(X, Y), Ty);
     }
-
-    // If the shifted-out value is known-zero, then this is an exact shift.
-    if (!I.isExact() &&
-        MaskedValueIsZero(Op0, APInt::getLowBitsSet(BitWidth, ShAmt), 0, &I)) {
-      I.setIsExact();
-      return &I;
-    }
   }
 
+  const SimplifyQuery Q = SQ.getWithInstruction(&I);
+  if (setShiftFlags(I, Q))
+    return &I;
+
   // Prefer `-(x & 1)` over `(x << (bitwidth(x)-1)) a>> (bitwidth(x)-1)`
   // as the pattern to splat the lowest bit.
   // FIXME: iff X is already masked, we don't need the one-use check.
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
index 5b5d2da041..be005e61a8 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
@@ -461,8 +461,7 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Value *V, APInt DemandedMask,
     if (InputKnown.isNonNegative() ||
         DemandedMask.getActiveBits() <= SrcBitWidth) {
       // Convert to ZExt cast.
-      CastInst *NewCast = new ZExtInst(I->getOperand(0), VTy);
-      NewCast->takeName(I);
+      CastInst *NewCast = new ZExtInst(I->getOperand(0), VTy, I->getName());
       return InsertNewInstWith(NewCast, I->getIterator());
      }
 
@@ -771,7 +770,6 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Value *V, APInt DemandedMask,
         BinaryOperator *LShr = BinaryOperator::CreateLShr(I->getOperand(0),
                                                           I->getOperand(1));
         LShr->setIsExact(cast<BinaryOperator>(I)->isExact());
-        LShr->takeName(I);
         return InsertNewInstWith(LShr, I->getIterator());
       } else if (Known.One[BitWidth-ShiftAmt-1]) { // New bits are known one.
         Known.One |= HighBits;
@@ -1783,139 +1781,3 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V,
 
   return MadeChange ? I : nullptr;
 }
-
-/// For floating-point classes that resolve to a single bit pattern, return that
-/// value.
-static Constant *getFPClassConstant(Type *Ty, FPClassTest Mask) {
-  switch (Mask) {
-  case fcPosZero:
-    return ConstantFP::getZero(Ty);
-  case fcNegZero:
-    return ConstantFP::getZero(Ty, true);
-  case fcPosInf:
-    return ConstantFP::getInfinity(Ty);
-  case fcNegInf:
-    return ConstantFP::getInfinity(Ty, true);
-  case fcNone:
-    return PoisonValue::get(Ty);
-  default:
-    return nullptr;
-  }
-}
-
-Value *InstCombinerImpl::SimplifyDemandedUseFPClass(
-    Value *V, const FPClassTest DemandedMask, KnownFPClass &Known,
-    unsigned Depth, Instruction *CxtI) {
-  assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
-  Type *VTy = V->getType();
-
-  assert(Known == KnownFPClass() && "expected uninitialized state");
-
-  if (DemandedMask == fcNone)
-    return isa<UndefValue>(V) ? nullptr : PoisonValue::get(VTy);
-
-  if (Depth == MaxAnalysisRecursionDepth)
-    return nullptr;
-
-  Instruction *I = dyn_cast<Instruction>(V);
-  if (!I) {
-    // Handle constants and arguments
-    Known = computeKnownFPClass(V, fcAllFlags, CxtI, Depth + 1);
-    Value *FoldedToConst =
-        getFPClassConstant(VTy, DemandedMask & Known.KnownFPClasses);
-    return FoldedToConst == V ? nullptr : FoldedToConst;
-  }
-
-  if (!I->hasOneUse())
-    return nullptr;
-
-  // TODO: Should account for nofpclass/FastMathFlags on current instruction
-  switch (I->getOpcode()) {
-  case Instruction::FNeg: {
-    if (SimplifyDemandedFPClass(I, 0, llvm::fneg(DemandedMask), Known,
-                                Depth + 1))
-      return I;
-    Known.fneg();
-    break;
-  }
-  case Instruction::Call: {
-    CallInst *CI = cast<CallInst>(I);
-    switch (CI->getIntrinsicID()) {
-    case Intrinsic::fabs:
-      if (SimplifyDemandedFPClass(I, 0, llvm::inverse_fabs(DemandedMask), Known,
-                                  Depth + 1))
-        return I;
-      Known.fabs();
-      break;
-    case Intrinsic::arithmetic_fence:
-      if (SimplifyDemandedFPClass(I, 0, DemandedMask, Known, Depth + 1))
-        return I;
-      break;
-    case Intrinsic::copysign: {
-      // Flip on more potentially demanded classes
-      const FPClassTest DemandedMaskAnySign = llvm::unknown_sign(DemandedMask);
-      if (SimplifyDemandedFPClass(I, 0, DemandedMaskAnySign, Known, Depth + 1))
-        return I;
-
-      if ((DemandedMask & fcPositive) == fcNone) {
-        // Roundabout way of replacing with fneg(fabs)
-        I->setOperand(1, ConstantFP::get(VTy, -1.0));
-        return I;
-      }
-
-      if ((DemandedMask & fcNegative) == fcNone) {
-        // Roundabout way of replacing with fabs
-        I->setOperand(1, ConstantFP::getZero(VTy));
-        return I;
-      }
-
-      KnownFPClass KnownSign =
-          computeKnownFPClass(I->getOperand(1), fcAllFlags, CxtI, Depth + 1);
-      Known.copysign(KnownSign);
-      break;
-    }
-    default:
-      Known = computeKnownFPClass(I, ~DemandedMask, CxtI, Depth + 1);
-      break;
-    }
-
-    break;
-  }
-  case Instruction::Select: {
-    KnownFPClass KnownLHS, KnownRHS;
-    if (SimplifyDemandedFPClass(I, 2, DemandedMask, KnownRHS, Depth + 1) ||
-        SimplifyDemandedFPClass(I, 1, DemandedMask, KnownLHS, Depth + 1))
-      return I;
-
-    if (KnownLHS.isKnownNever(DemandedMask))
-      return I->getOperand(2);
-    if (KnownRHS.isKnownNever(DemandedMask))
-      return I->getOperand(1);
-
-    // TODO: Recognize clamping patterns
-    Known = KnownLHS | KnownRHS;
-    break;
-  }
-  default:
-    Known = computeKnownFPClass(I, ~DemandedMask, CxtI, Depth + 1);
-    break;
-  }
-
-  return getFPClassConstant(VTy, DemandedMask & Known.KnownFPClasses);
-}
-
-bool InstCombinerImpl::SimplifyDemandedFPClass(Instruction *I, unsigned OpNo,
-                                               FPClassTest DemandedMask,
-                                               KnownFPClass &Known,
-                                               unsigned Depth) {
-  Use &U = I->getOperandUse(OpNo);
-  Value *NewVal =
-      SimplifyDemandedUseFPClass(U.get(), DemandedMask, Known, Depth, I);
-  if (!NewVal)
-    return false;
-  if (Instruction *OpInst = dyn_cast<Instruction>(U))
-    salvageDebugInfo(*OpInst);
-
-  replaceUse(U, NewVal);
-  return true;
-}
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index 4736df4095..559eb2ef47 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -2430,6 +2430,26 @@ static bool isAllocSiteRemovable(Instruction *AI,
         unsigned OtherIndex = (ICI->getOperand(0) == PI) ? 1 : 0;
         if (!isNeverEqualToUnescapedAlloc(ICI->getOperand(OtherIndex), TLI, AI))
           return false;
+
+        // Do not fold compares to aligned_alloc calls, as they may have to
+        // return null in case the required alignment cannot be satisfied,
+        // unless we can prove that both alignment and size are valid.
+        auto AlignmentAndSizeKnownValid = [](CallBase *CB) {
+          // Check if alignment and size of a call to aligned_alloc is valid,
+          // that is alignment is a power-of-2 and the size is a multiple of the
+          // alignment.
+          const APInt *Alignment;
+          const APInt *Size;
+          return match(CB->getArgOperand(0), m_APInt(Alignment)) &&
+                 match(CB->getArgOperand(1), m_APInt(Size)) &&
+                 Alignment->isPowerOf2() && Size->urem(*Alignment).isZero();
+        };
+        auto *CB = dyn_cast<CallBase>(AI);
+        LibFunc TheLibFunc;
+        if (CB && TLI.getLibFunc(*CB->getCalledFunction(), TheLibFunc) &&
+            TLI.has(TheLibFunc) && TheLibFunc == LibFunc_aligned_alloc &&
+            !AlignmentAndSizeKnownValid(CB))
+          return false;
         Users.emplace_back(I);
         continue;
       }
@@ -2748,22 +2768,8 @@ Instruction *InstCombinerImpl::visitFree(CallInst &FI, Value *Op) {
 }
 
 Instruction *InstCombinerImpl::visitReturnInst(ReturnInst &RI) {
-  Value *RetVal = RI.getReturnValue();
-  if (!RetVal || !AttributeFuncs::isNoFPClassCompatibleType(RetVal->getType()))
-    return nullptr;
-
-  Function *F = RI.getFunction();
-  FPClassTest ReturnClass = F->getAttributes().getRetNoFPClass();
-  if (ReturnClass == fcNone)
-    return nullptr;
-
-  KnownFPClass KnownClass;
-  Value *Simplified =
-      SimplifyDemandedUseFPClass(RetVal, ~ReturnClass, KnownClass, 0, &RI);
-  if (!Simplified)
-    return nullptr;
-
-  return ReturnInst::Create(RI.getContext(), Simplified);
+  // Nothing for now.
+  return nullptr;
 }
 
 // WARNING: keep in sync with SimplifyCFGOpt::simplifyUnreachable()!
diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index 9e9b192d44..e80ee1953d 100644
--- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -819,11 +819,11 @@ class ModuleAddressSanitizer {
 private:
   void initializeCallbacks(Module &M);
 
-  bool InstrumentGlobals(IRBuilder<> &IRB, Module &M, bool *CtorComdat);
+  void instrumentGlobals(IRBuilder<> &IRB, Module &M, bool *CtorComdat);
   void InstrumentGlobalsCOFF(IRBuilder<> &IRB, Module &M,
                              ArrayRef<GlobalVariable *> ExtendedGlobals,
                              ArrayRef<Constant *> MetadataInitializers);
-  void InstrumentGlobalsELF(IRBuilder<> &IRB, Module &M,
+  void instrumentGlobalsELF(IRBuilder<> &IRB, Module &M,
                             ArrayRef<GlobalVariable *> ExtendedGlobals,
                             ArrayRef<Constant *> MetadataInitializers,
                             const std::string &UniqueModuleId);
@@ -2177,7 +2177,7 @@ void ModuleAddressSanitizer::InstrumentGlobalsCOFF(
     appendToCompilerUsed(M, MetadataGlobals);
 }
 
-void ModuleAddressSanitizer::InstrumentGlobalsELF(
+void ModuleAddressSanitizer::instrumentGlobalsELF(
     IRBuilder<> &IRB, Module &M, ArrayRef<GlobalVariable *> ExtendedGlobals,
     ArrayRef<Constant *> MetadataInitializers,
     const std::string &UniqueModuleId) {
@@ -2187,7 +2187,7 @@ void ModuleAddressSanitizer::InstrumentGlobalsELF(
   // false negative odr violations at link time. If odr indicators are used, we
   // keep the comdat sections, as link time odr violations will be dectected on
   // the odr indicator symbols.
-  bool UseComdatForGlobalsGC = UseOdrIndicator;
+  bool UseComdatForGlobalsGC = UseOdrIndicator && !UniqueModuleId.empty();
 
   SmallVector<GlobalValue *, 16> MetadataGlobals(ExtendedGlobals.size());
   for (size_t i = 0; i < ExtendedGlobals.size(); i++) {
@@ -2237,7 +2237,7 @@ void ModuleAddressSanitizer::InstrumentGlobalsELF(
 
   // We also need to unregister globals at the end, e.g., when a shared library
   // gets closed.
-  if (DestructorKind != AsanDtorKind::None) {
+  if (DestructorKind != AsanDtorKind::None && !MetadataGlobals.empty()) {
     IRBuilder<> IrbDtor(CreateAsanModuleDtor(M));
     IrbDtor.CreateCall(AsanUnregisterElfGlobals,
                        {IRB.CreatePointerCast(RegisteredFlag, IntptrTy),
@@ -2343,10 +2343,8 @@ void ModuleAddressSanitizer::InstrumentGlobalsWithMetadataArray(
 // redzones and inserts this function into llvm.global_ctors.
 // Sets *CtorComdat to true if the global registration code emitted into the
 // asan constructor is comdat-compatible.
-bool ModuleAddressSanitizer::InstrumentGlobals(IRBuilder<> &IRB, Module &M,
+void ModuleAddressSanitizer::instrumentGlobals(IRBuilder<> &IRB, Module &M,
                                                bool *CtorComdat) {
-  *CtorComdat = false;
-
   // Build set of globals that are aliased by some GA, where
   // getExcludedAliasedGlobal(GA) returns the relevant GlobalVariable.
   SmallPtrSet<const GlobalVariable *, 16> AliasedGlobalExclusions;
@@ -2364,11 +2362,6 @@ bool ModuleAddressSanitizer::InstrumentGlobals(IRBuilder<> &IRB, Module &M,
   }
 
   size_t n = GlobalsToChange.size();
-  if (n == 0) {
-    *CtorComdat = true;
-    return false;
-  }
-
   auto &DL = M.getDataLayout();
 
   // A global is described by a structure
@@ -2391,8 +2384,11 @@ bool ModuleAddressSanitizer::InstrumentGlobals(IRBuilder<> &IRB, Module &M,
 
   // We shouldn't merge same module names, as this string serves as unique
   // module ID in runtime.
-  GlobalVariable *ModuleName = createPrivateGlobalForString(
-      M, M.getModuleIdentifier(), /*AllowMerging*/ false, kAsanGenPrefix);
+  GlobalVariable *ModuleName =
+      n != 0
+          ? createPrivateGlobalForString(M, M.getModuleIdentifier(),
+                                         /*AllowMerging*/ false, kAsanGenPrefix)
+          : nullptr;
 
   for (size_t i = 0; i < n; i++) {
     GlobalVariable *G = GlobalsToChange[i];
@@ -2517,19 +2513,27 @@ bool ModuleAddressSanitizer::InstrumentGlobals(IRBuilder<> &IRB, Module &M,
   }
   appendToCompilerUsed(M, ArrayRef<GlobalValue *>(GlobalsToAddToUsedList));
 
-  std::string ELFUniqueModuleId =
-      (UseGlobalsGC && TargetTriple.isOSBinFormatELF()) ? getUniqueModuleId(&M)
-                                                        : "";
-
-  if (!ELFUniqueModuleId.empty()) {
-    InstrumentGlobalsELF(IRB, M, NewGlobals, Initializers, ELFUniqueModuleId);
+  if (UseGlobalsGC && TargetTriple.isOSBinFormatELF()) {
+    // Use COMDAT and register globals even if n == 0 to ensure that (a) the
+    // linkage unit will only have one module constructor, and (b) the register
+    // function will be called. The module destructor is not created when n ==
+    // 0.
     *CtorComdat = true;
-  } else if (UseGlobalsGC && TargetTriple.isOSBinFormatCOFF()) {
-    InstrumentGlobalsCOFF(IRB, M, NewGlobals, Initializers);
-  } else if (UseGlobalsGC && ShouldUseMachOGlobalsSection()) {
-    InstrumentGlobalsMachO(IRB, M, NewGlobals, Initializers);
+    instrumentGlobalsELF(IRB, M, NewGlobals, Initializers,
+                         getUniqueModuleId(&M));
+  } else if (n == 0) {
+    // When UseGlobalsGC is false, COMDAT can still be used if n == 0, because
+    // all compile units will have identical module constructor/destructor.
+    *CtorComdat = TargetTriple.isOSBinFormatELF();
   } else {
-    InstrumentGlobalsWithMetadataArray(IRB, M, NewGlobals, Initializers);
+    *CtorComdat = false;
+    if (UseGlobalsGC && TargetTriple.isOSBinFormatCOFF()) {
+      InstrumentGlobalsCOFF(IRB, M, NewGlobals, Initializers);
+    } else if (UseGlobalsGC && ShouldUseMachOGlobalsSection()) {
+      InstrumentGlobalsMachO(IRB, M, NewGlobals, Initializers);
+    } else {
+      InstrumentGlobalsWithMetadataArray(IRB, M, NewGlobals, Initializers);
+    }
   }
 
   // Create calls for poisoning before initializers run and unpoisoning after.
@@ -2537,7 +2541,6 @@ bool ModuleAddressSanitizer::InstrumentGlobals(IRBuilder<> &IRB, Module &M,
     createInitializerPoisonCalls(M, ModuleName);
 
   LLVM_DEBUG(dbgs() << M);
-  return true;
 }
 
 uint64_t
@@ -2601,10 +2604,10 @@ bool ModuleAddressSanitizer::instrumentModule(Module &M) {
     assert(AsanCtorFunction || ConstructorKind == AsanCtorKind::None);
     if (AsanCtorFunction) {
       IRBuilder<> IRB(AsanCtorFunction->getEntryBlock().getTerminator());
-      InstrumentGlobals(IRB, M, &CtorComdat);
+      instrumentGlobals(IRB, M, &CtorComdat);
     } else {
       IRBuilder<> IRB(*C);
-      InstrumentGlobals(IRB, M, &CtorComdat);
+      instrumentGlobals(IRB, M, &CtorComdat);
     }
   }
 
diff --git a/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp b/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp
index 74e9a387ea..5d66895d0e 100644
--- a/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp
@@ -148,7 +148,7 @@ class GCOVProfiler {
   std::string mangleName(const DICompileUnit *CU, GCovFileType FileType);
 
   GCOVOptions Options;
-  support::endianness Endian;
+  llvm::endianness Endian;
   raw_ostream *os;
 
   // Checksum, produced by hash of EdgeDestinations
@@ -788,8 +788,8 @@ bool GCOVProfiler::emitProfileNotes(
     std::vector<uint8_t> EdgeDestinations;
     SmallVector<std::pair<GlobalVariable *, MDNode *>, 8> CountersBySP;
 
-    Endian = M->getDataLayout().isLittleEndian() ? support::endianness::little
-                                                 : support::endianness::big;
+    Endian = M->getDataLayout().isLittleEndian() ? llvm::endianness::little
+                                                 : llvm::endianness::big;
     unsigned FunctionIdent = 0;
     for (auto &F : M->functions()) {
       DISubprogram *SP = F.getSubprogram();
@@ -957,7 +957,7 @@ bool GCOVProfiler::emitProfileNotes(
         continue;
       }
       os = &out;
-      if (Endian == support::endianness::big) {
+      if (Endian == llvm::endianness::big) {
         out.write("gcno", 4);
         out.write(Options.Version, 4);
       } else {
diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
index fd7c641ccf..e194b96475 100644
--- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
@@ -21,6 +21,7 @@
 #include "llvm/Analysis/GlobalsModRef.h"
 #include "llvm/Analysis/PostDominators.h"
 #include "llvm/Analysis/StackSafetyAnalysis.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/BinaryFormat/ELF.h"
@@ -52,6 +53,7 @@
 #include "llvm/TargetParser/Triple.h"
 #include "llvm/Transforms/Instrumentation/AddressSanitizerCommon.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/Local.h"
 #include "llvm/Transforms/Utils/MemoryTaggingSupport.h"
 #include "llvm/Transforms/Utils/ModuleUtils.h"
 #include "llvm/Transforms/Utils/PromoteMemToReg.h"
@@ -320,7 +322,8 @@ class HWAddressSanitizer {
                            LoopInfo *LI);
   bool ignoreAccess(Instruction *Inst, Value *Ptr);
   void getInterestingMemoryOperands(
-      Instruction *I, SmallVectorImpl<InterestingMemoryOperand> &Interesting);
+      Instruction *I, const TargetLibraryInfo &TLI,
+      SmallVectorImpl<InterestingMemoryOperand> &Interesting);
 
   void tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag, size_t Size);
   Value *tagPointer(IRBuilder<> &IRB, Type *Ty, Value *PtrLong, Value *Tag);
@@ -779,7 +782,8 @@ bool HWAddressSanitizer::ignoreAccess(Instruction *Inst, Value *Ptr) {
 }
 
 void HWAddressSanitizer::getInterestingMemoryOperands(
-    Instruction *I, SmallVectorImpl<InterestingMemoryOperand> &Interesting) {
+    Instruction *I, const TargetLibraryInfo &TLI,
+    SmallVectorImpl<InterestingMemoryOperand> &Interesting) {
   // Skip memory accesses inserted by another instrumentation.
   if (I->hasMetadata(LLVMContext::MD_nosanitize))
     return;
@@ -817,6 +821,7 @@ void HWAddressSanitizer::getInterestingMemoryOperands(
       Type *Ty = CI->getParamByValType(ArgNo);
       Interesting.emplace_back(I, ArgNo, false, Ty, Align(1));
     }
+    maybeMarkSanitizerLibraryCallNoBuiltin(CI, &TLI);
   }
 }
 
@@ -1493,6 +1498,7 @@ void HWAddressSanitizer::sanitizeFunction(Function &F,
   SmallVector<InterestingMemoryOperand, 16> OperandsToInstrument;
   SmallVector<MemIntrinsic *, 16> IntrinToInstrument;
   SmallVector<Instruction *, 8> LandingPadVec;
+  const TargetLibraryInfo &TLI = FAM.getResult<TargetLibraryAnalysis>(F);
 
   memtag::StackInfoBuilder SIB(SSI);
   for (auto &Inst : instructions(F)) {
@@ -1503,7 +1509,7 @@ void HWAddressSanitizer::sanitizeFunction(Function &F,
     if (InstrumentLandingPads && isa<LandingPadInst>(Inst))
       LandingPadVec.push_back(&Inst);
 
-    getInterestingMemoryOperands(&Inst, OperandsToInstrument);
+    getInterestingMemoryOperands(&Inst, TLI, OperandsToInstrument);
 
     if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(&Inst))
       if (!ignoreMemIntrinsic(MI))
diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index 9b476a4498..e72db2d9d7 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -152,7 +152,6 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DepthFirstIterator.h"
 #include "llvm/ADT/SetVector.h"
-#include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
diff --git a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
index 1eb7e481d4..37f720ec40 100644
--- a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
@@ -451,6 +451,19 @@ static Decomposition decompose(Value *V,
     return ResA;
   };
 
+  Type *Ty = V->getType()->getScalarType();
+  if (Ty->isPointerTy() && !IsSigned) {
+    if (auto *GEP = dyn_cast<GEPOperator>(V))
+      return decomposeGEP(*GEP, Preconditions, IsSigned, DL);
+    return V;
+  }
+
+  // Don't handle integers > 64 bit. Our coefficients are 64-bit large, so
+  // coefficient add/mul may wrap, while the operation in the full bit width
+  // would not.
+  if (!Ty->isIntegerTy() || Ty->getIntegerBitWidth() > 64)
+    return V;
+
   // Decompose \p V used with a signed predicate.
   if (IsSigned) {
     if (auto *CI = dyn_cast<ConstantInt>(V)) {
@@ -478,9 +491,6 @@ static Decomposition decompose(Value *V,
     return int64_t(CI->getZExtValue());
   }
 
-  if (auto *GEP = dyn_cast<GEPOperator>(V))
-    return decomposeGEP(*GEP, Preconditions, IsSigned, DL);
-
   Value *Op0;
   bool IsKnownNonNegative = false;
   if (match(V, m_ZExt(m_Value(Op0)))) {
diff --git a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
index 48b27a1ea0..523196e5e6 100644
--- a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
+++ b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
@@ -470,17 +470,17 @@ static bool processBinOp(BinaryOperator *BinOp, LazyValueInfo *LVI);
 // because it is negation-invariant.
 static bool processAbsIntrinsic(IntrinsicInst *II, LazyValueInfo *LVI) {
   Value *X = II->getArgOperand(0);
-  bool IsIntMinPoison = cast<ConstantInt>(II->getArgOperand(1))->isOne();
-
   Type *Ty = X->getType();
-  Constant *IntMin =
-      ConstantInt::get(Ty, APInt::getSignedMinValue(Ty->getScalarSizeInBits()));
-  LazyValueInfo::Tristate Result;
+  if (!Ty->isIntegerTy())
+    return false;
+
+  bool IsIntMinPoison = cast<ConstantInt>(II->getArgOperand(1))->isOne();
+  APInt IntMin = APInt::getSignedMinValue(Ty->getScalarSizeInBits());
+  ConstantRange Range = LVI->getConstantRangeAtUse(
+      II->getOperandUse(0), /*UndefAllowed*/ IsIntMinPoison);
 
   // Is X in [0, IntMin]?  NOTE: INT_MIN is fine!
-  Result = LVI->getPredicateAt(CmpInst::Predicate::ICMP_ULE, X, IntMin, II,
-                               /*UseBlockValue=*/true);
-  if (Result == LazyValueInfo::True) {
+  if (Range.icmp(CmpInst::ICMP_ULE, IntMin)) {
     ++NumAbs;
     II->replaceAllUsesWith(X);
     II->eraseFromParent();
@@ -488,40 +488,30 @@ static bool processAbsIntrinsic(IntrinsicInst *II, LazyValueInfo *LVI) {
   }
 
   // Is X in [IntMin, 0]?  NOTE: INT_MIN is fine!
-  Constant *Zero = ConstantInt::getNullValue(Ty);
-  Result = LVI->getPredicateAt(CmpInst::Predicate::ICMP_SLE, X, Zero, II,
-                               /*UseBlockValue=*/true);
-  assert(Result != LazyValueInfo::False && "Should have been handled already.");
-
-  if (Result == LazyValueInfo::Unknown) {
-    // Argument's range crosses zero.
-    bool Changed = false;
-    if (!IsIntMinPoison) {
-      // Can we at least tell that the argument is never INT_MIN?
-      Result = LVI->getPredicateAt(CmpInst::Predicate::ICMP_NE, X, IntMin, II,
-                                   /*UseBlockValue=*/true);
-      if (Result == LazyValueInfo::True) {
-        ++NumNSW;
-        ++NumSubNSW;
-        II->setArgOperand(1, ConstantInt::getTrue(II->getContext()));
-        Changed = true;
-      }
-    }
-    return Changed;
-  }
+  if (Range.getSignedMax().isNonPositive()) {
+    IRBuilder<> B(II);
+    Value *NegX = B.CreateNeg(X, II->getName(), /*HasNUW=*/false,
+                              /*HasNSW=*/IsIntMinPoison);
+    ++NumAbs;
+    II->replaceAllUsesWith(NegX);
+    II->eraseFromParent();
 
-  IRBuilder<> B(II);
-  Value *NegX = B.CreateNeg(X, II->getName(), /*HasNUW=*/false,
-                            /*HasNSW=*/IsIntMinPoison);
-  ++NumAbs;
-  II->replaceAllUsesWith(NegX);
-  II->eraseFromParent();
+    // See if we can infer some no-wrap flags.
+    if (auto *BO = dyn_cast<BinaryOperator>(NegX))
+      processBinOp(BO, LVI);
 
-  // See if we can infer some no-wrap flags.
-  if (auto *BO = dyn_cast<BinaryOperator>(NegX))
-    processBinOp(BO, LVI);
+    return true;
+  }
 
-  return true;
+  // Argument's range crosses zero.
+  // Can we at least tell that the argument is never INT_MIN?
+  if (!IsIntMinPoison && !Range.contains(IntMin)) {
+    ++NumNSW;
+    ++NumSubNSW;
+    II->setArgOperand(1, ConstantInt::getTrue(II->getContext()));
+    return true;
+  }
+  return false;
 }
 
 // See if this min/max intrinsic always picks it's one specific operand.
diff --git a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
index 0586ff191d..026758e320 100644
--- a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
@@ -865,7 +865,7 @@ struct DSEState {
   DSEState(Function &F, AliasAnalysis &AA, MemorySSA &MSSA, DominatorTree &DT,
            PostDominatorTree &PDT, AssumptionCache &AC,
            const TargetLibraryInfo &TLI, const LoopInfo &LI)
-      : F(F), AA(AA), EI(DT, LI, EphValues), BatchAA(AA, &EI), MSSA(MSSA),
+      : F(F), AA(AA), EI(DT, &LI, &EphValues), BatchAA(AA, &EI), MSSA(MSSA),
         DT(DT), PDT(PDT), TLI(TLI), DL(F.getParent()->getDataLayout()), LI(LI) {
     // Collect blocks with throwing instructions not modeled in MemorySSA and
     // alloc-like objects.
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index 4c5b0ff3af..5e58af0edc 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -760,7 +760,7 @@ PreservedAnalyses GVNPass::run(Function &F, FunctionAnalysisManager &AM) {
   auto &AA = AM.getResult<AAManager>(F);
   auto *MemDep =
       isMemDepEnabled() ? &AM.getResult<MemoryDependenceAnalysis>(F) : nullptr;
-  auto *LI = AM.getCachedResult<LoopAnalysis>(F);
+  auto &LI = AM.getResult<LoopAnalysis>(F);
   auto *MSSA = AM.getCachedResult<MemorySSAAnalysis>(F);
   auto &ORE = AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
   bool Changed = runImpl(F, AC, DT, TLI, AA, MemDep, LI, &ORE,
@@ -772,8 +772,7 @@ PreservedAnalyses GVNPass::run(Function &F, FunctionAnalysisManager &AM) {
   PA.preserve<TargetLibraryAnalysis>();
   if (MSSA)
     PA.preserve<MemorySSAAnalysis>();
-  if (LI)
-    PA.preserve<LoopAnalysis>();
+  PA.preserve<LoopAnalysis>();
   return PA;
 }
 
@@ -946,9 +945,14 @@ static void replaceValuesPerBlockEntry(
     SmallVectorImpl<AvailableValueInBlock> &ValuesPerBlock, Value *OldValue,
     Value *NewValue) {
   for (AvailableValueInBlock &V : ValuesPerBlock) {
-    if ((V.AV.isSimpleValue() && V.AV.getSimpleValue() == OldValue) ||
-       (V.AV.isCoercedLoadValue() && V.AV.getCoercedLoadValue() == OldValue))
-      V = AvailableValueInBlock::get(V.BB, NewValue);
+    if (V.AV.Val == OldValue)
+      V.AV.Val = NewValue;
+    if (V.AV.isSelectValue()) {
+      if (V.AV.V1 == OldValue)
+        V.AV.V1 = NewValue;
+      if (V.AV.V2 == OldValue)
+        V.AV.V2 = NewValue;
+    }
   }
 }
 
@@ -1434,8 +1438,7 @@ void GVNPass::eliminatePartiallyRedundantLoad(
     if (auto *RangeMD = Load->getMetadata(LLVMContext::MD_range))
       NewLoad->setMetadata(LLVMContext::MD_range, RangeMD);
     if (auto *AccessMD = Load->getMetadata(LLVMContext::MD_access_group))
-      if (LI &&
-          LI->getLoopFor(Load->getParent()) == LI->getLoopFor(UnavailableBlock))
+      if (LI->getLoopFor(Load->getParent()) == LI->getLoopFor(UnavailableBlock))
         NewLoad->setMetadata(LLVMContext::MD_access_group, AccessMD);
 
     // We do not propagate the old load's debug location, because the new
@@ -1743,9 +1746,6 @@ bool GVNPass::PerformLoadPRE(LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
 bool GVNPass::performLoopLoadPRE(LoadInst *Load,
                                  AvailValInBlkVect &ValuesPerBlock,
                                  UnavailBlkVect &UnavailableBlocks) {
-  if (!LI)
-    return false;
-
   const Loop *L = LI->getLoopFor(Load->getParent());
   // TODO: Generalize to other loop blocks that dominate the latch.
   if (!L || L->getHeader() != Load->getParent())
@@ -1914,7 +1914,7 @@ bool GVNPass::processNonLocalLoad(LoadInst *Load) {
   // Step 4: Eliminate partial redundancy.
   if (!isPREEnabled() || !isLoadPREEnabled())
     return Changed;
-  if (!isLoadInLoopPREEnabled() && LI && LI->getLoopFor(Load->getParent()))
+  if (!isLoadInLoopPREEnabled() && LI->getLoopFor(Load->getParent()))
     return Changed;
 
   if (performLoopLoadPRE(Load, ValuesPerBlock, UnavailableBlocks) ||
@@ -2686,7 +2686,7 @@ bool GVNPass::processInstruction(Instruction *I) {
 /// runOnFunction - This is the main transformation entry point for a function.
 bool GVNPass::runImpl(Function &F, AssumptionCache &RunAC, DominatorTree &RunDT,
                       const TargetLibraryInfo &RunTLI, AAResults &RunAA,
-                      MemoryDependenceResults *RunMD, LoopInfo *LI,
+                      MemoryDependenceResults *RunMD, LoopInfo &LI,
                       OptimizationRemarkEmitter *RunORE, MemorySSA *MSSA) {
   AC = &RunAC;
   DT = &RunDT;
@@ -2696,7 +2696,7 @@ bool GVNPass::runImpl(Function &F, AssumptionCache &RunAC, DominatorTree &RunDT,
   MD = RunMD;
   ImplicitControlFlowTracking ImplicitCFT;
   ICF = &ImplicitCFT;
-  this->LI = LI;
+  this->LI = &LI;
   VN.setMemDep(MD);
   ORE = RunORE;
   InvalidBlockRPONumbers = true;
@@ -2710,7 +2710,7 @@ bool GVNPass::runImpl(Function &F, AssumptionCache &RunAC, DominatorTree &RunDT,
   // Merge unconditional branches, allowing PRE to catch more
   // optimization opportunities.
   for (BasicBlock &BB : llvm::make_early_inc_range(F)) {
-    bool removedBlock = MergeBlockIntoPredecessor(&BB, &DTU, LI, MSSAU, MD);
+    bool removedBlock = MergeBlockIntoPredecessor(&BB, &DTU, &LI, MSSAU, MD);
     if (removedBlock)
       ++NumGVNBlocks;
 
@@ -3286,8 +3286,6 @@ class llvm::gvn::GVNLegacyPass : public FunctionPass {
     if (skipFunction(F))
       return false;
 
-    auto *LIWP = getAnalysisIfAvailable<LoopInfoWrapperPass>();
-
     auto *MSSAWP = getAnalysisIfAvailable<MemorySSAWrapperPass>();
     return Impl.runImpl(
         F, getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F),
@@ -3297,7 +3295,7 @@ class llvm::gvn::GVNLegacyPass : public FunctionPass {
         Impl.isMemDepEnabled()
             ? &getAnalysis<MemoryDependenceWrapperPass>().getMemDep()
             : nullptr,
-        LIWP ? &LIWP->getLoopInfo() : nullptr,
+        getAnalysis<LoopInfoWrapperPass>().getLoopInfo(),
         &getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE(),
         MSSAWP ? &MSSAWP->getMSSA() : nullptr);
   }
diff --git a/llvm/lib/Transforms/Scalar/JumpThreading.cpp b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
index 8c3ff39962..f2b9d784ea 100644
--- a/llvm/lib/Transforms/Scalar/JumpThreading.cpp
+++ b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
@@ -1977,11 +1977,9 @@ void JumpThreadingPass::updateSSA(
 
     // Find debug values outside of the block
     findDbgValues(DbgValues, &I);
-    DbgValues.erase(remove_if(DbgValues,
-                              [&](const DbgValueInst *DbgVal) {
-                                return DbgVal->getParent() == BB;
-                              }),
-                    DbgValues.end());
+    llvm::erase_if(DbgValues, [&](const DbgValueInst *DbgVal) {
+      return DbgVal->getParent() == BB;
+    });
 
     // If there are no uses outside the block, we're done with this instruction.
     if (UsesToRename.empty() && DbgValues.empty())
diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp
index 224db19caa..aab6b05385 100644
--- a/llvm/lib/Transforms/Scalar/LICM.cpp
+++ b/llvm/lib/Transforms/Scalar/LICM.cpp
@@ -129,7 +129,7 @@ static cl::opt<uint32_t> MaxNumUsesTraversed(
     cl::desc("Max num uses visited for identifying load "
              "invariance in loop using invariant start (default = 8)"));
 
-cl::opt<unsigned> FPAssociationUpperLimit(
+static cl::opt<unsigned> FPAssociationUpperLimit(
     "licm-max-num-fp-reassociations", cl::init(5U), cl::Hidden,
     cl::desc(
         "Set upper limit for the number of transformations performed "
diff --git a/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp b/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp
index 454aa56be5..b7d40b0cc1 100644
--- a/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp
+++ b/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp
@@ -13,7 +13,6 @@
 #include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
-#include "llvm/ADT/iterator_range.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/Function.h"
@@ -416,29 +415,3 @@ PreservedAnalyses LowerExpectIntrinsicPass::run(Function &F,
 
   return PreservedAnalyses::all();
 }
-
-namespace {
-/// Legacy pass for lowering expect intrinsics out of the IR.
-///
-/// When this pass is run over a function it uses expect intrinsics which feed
-/// branches and switches to provide branch weight metadata for those
-/// terminators. It then removes the expect intrinsics from the IR so the rest
-/// of the optimizer can ignore them.
-class LowerExpectIntrinsic : public FunctionPass {
-public:
-  static char ID;
-  LowerExpectIntrinsic() : FunctionPass(ID) {
-    initializeLowerExpectIntrinsicPass(*PassRegistry::getPassRegistry());
-  }
-
-  bool runOnFunction(Function &F) override { return lowerExpectIntrinsic(F); }
-};
-} // namespace
-
-char LowerExpectIntrinsic::ID = 0;
-INITIALIZE_PASS(LowerExpectIntrinsic, "lower-expect",
-                "Lower 'expect' Intrinsics", false, false)
-
-FunctionPass *llvm::createLowerExpectIntrinsicPass() {
-  return new LowerExpectIntrinsic();
-}
diff --git a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
index 688bcfa571..b1c60d12aa 100644
--- a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
+++ b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
@@ -1068,29 +1068,19 @@ bool MemCpyOptPass::performCallSlotOptzn(Instruction *cpyLoad,
 
   // We can't create address space casts here because we don't know if they're
   // safe for the target.
-  if (cpySrc->getType()->getPointerAddressSpace() !=
-      cpyDest->getType()->getPointerAddressSpace())
+  if (cpySrc->getType() != cpyDest->getType())
     return false;
   for (unsigned ArgI = 0; ArgI < C->arg_size(); ++ArgI)
     if (C->getArgOperand(ArgI)->stripPointerCasts() == cpySrc &&
-        cpySrc->getType()->getPointerAddressSpace() !=
-            C->getArgOperand(ArgI)->getType()->getPointerAddressSpace())
+        cpySrc->getType() != C->getArgOperand(ArgI)->getType())
       return false;
 
   // All the checks have passed, so do the transformation.
   bool changedArgument = false;
   for (unsigned ArgI = 0; ArgI < C->arg_size(); ++ArgI)
     if (C->getArgOperand(ArgI)->stripPointerCasts() == cpySrc) {
-      Value *Dest = cpySrc->getType() == cpyDest->getType() ?  cpyDest
-        : CastInst::CreatePointerCast(cpyDest, cpySrc->getType(),
-                                      cpyDest->getName(), C);
       changedArgument = true;
-      if (C->getArgOperand(ArgI)->getType() == Dest->getType())
-        C->setArgOperand(ArgI, Dest);
-      else
-        C->setArgOperand(ArgI, CastInst::CreatePointerCast(
-                                   Dest, C->getArgOperand(ArgI)->getType(),
-                                   Dest->getName(), C));
+      C->setArgOperand(ArgI, cpyDest);
     }
 
   if (!changedArgument)
@@ -1855,9 +1845,8 @@ bool MemCpyOptPass::processByValArgument(CallBase &CB, unsigned ArgNo) {
                                  DT) < *ByValAlign)
     return false;
 
-  // The address space of the memcpy source must match the byval argument
-  if (MDep->getSource()->getType()->getPointerAddressSpace() !=
-      ByValArg->getType()->getPointerAddressSpace())
+  // The type of the memcpy source must match the byval argument
+  if (MDep->getSource()->getType() != ByValArg->getType())
     return false;
 
   // Verify that the copied-from memory doesn't change in between the memcpy and
@@ -1931,9 +1920,8 @@ bool MemCpyOptPass::processImmutArgument(CallBase &CB, unsigned ArgNo) {
   if (!MDep || MDep->isVolatile() || AI != MDep->getDest())
     return false;
 
-  // The address space of the memcpy source must match the immut argument
-  if (MDep->getSource()->getType()->getPointerAddressSpace() !=
-      ImmutArg->getType()->getPointerAddressSpace())
+  // The type of the memcpy source must match the immut argument
+  if (MDep->getSource()->getType() != ImmutArg->getType())
     return false;
 
   // 2-1. The length of the memcpy must be equal to the size of the alloca.
diff --git a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
index d2984cd829..06c81f53de 100644
--- a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
+++ b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
@@ -18,6 +18,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/Sequence.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
@@ -1422,14 +1423,15 @@ static constexpr Attribute::AttrKind FnAttrsToStrip[] =
   {Attribute::Memory, Attribute::NoSync, Attribute::NoFree};
 
 // Create new attribute set containing only attributes which can be transferred
-// from original call to the safepoint.
-static AttributeList legalizeCallAttributes(LLVMContext &Ctx,
-                                            AttributeList OrigAL,
+// from the original call to the safepoint.
+static AttributeList legalizeCallAttributes(CallBase *Call, bool IsMemIntrinsic,
                                             AttributeList StatepointAL) {
+  AttributeList OrigAL = Call->getAttributes();
   if (OrigAL.isEmpty())
     return StatepointAL;
 
   // Remove the readonly, readnone, and statepoint function attributes.
+  LLVMContext &Ctx = Call->getContext();
   AttrBuilder FnAttrs(Ctx, OrigAL.getFnAttrs());
   for (auto Attr : FnAttrsToStrip)
     FnAttrs.removeAttribute(Attr);
@@ -1439,8 +1441,24 @@ static AttributeList legalizeCallAttributes(LLVMContext &Ctx,
       FnAttrs.removeAttribute(A);
   }
 
-  // Just skip parameter and return attributes for now
-  return StatepointAL.addFnAttributes(Ctx, FnAttrs);
+  StatepointAL = StatepointAL.addFnAttributes(Ctx, FnAttrs);
+
+  // The memory intrinsics do not have a 1:1 correspondence of the original
+  // call arguments to the produced statepoint. Do not transfer the argument
+  // attributes to avoid putting them on incorrect arguments.
+  if (IsMemIntrinsic)
+    return StatepointAL;
+
+  // Attach the argument attributes from the original call at the corresponding
+  // arguments in the statepoint. Note that any argument attributes that are
+  // invalid after lowering are stripped in stripNonValidDataFromBody.
+  for (unsigned I : llvm::seq(Call->arg_size()))
+    StatepointAL = StatepointAL.addParamAttributes(
+        Ctx, GCStatepointInst::CallArgsBeginPos + I,
+        AttrBuilder(Ctx, OrigAL.getParamAttrs(I)));
+
+  // Return attributes are later attached to the gc.result intrinsic.
+  return StatepointAL;
 }
 
 /// Helper function to place all gc relocates necessary for the given
@@ -1630,6 +1648,7 @@ makeStatepointExplicitImpl(CallBase *Call, /* to replace */
   // with a return value, we lower then as never returning calls to
   // __llvm_deoptimize that are followed by unreachable to get better codegen.
   bool IsDeoptimize = false;
+  bool IsMemIntrinsic = false;
 
   StatepointDirectives SD =
       parseStatepointDirectivesFromAttrs(Call->getAttributes());
@@ -1670,6 +1689,8 @@ makeStatepointExplicitImpl(CallBase *Call, /* to replace */
       IsDeoptimize = true;
     } else if (IID == Intrinsic::memcpy_element_unordered_atomic ||
                IID == Intrinsic::memmove_element_unordered_atomic) {
+      IsMemIntrinsic = true;
+
       // Unordered atomic memcpy and memmove intrinsics which are not explicitly
       // marked as "gc-leaf-function" should be lowered in a GC parseable way.
       // Specifically, these calls should be lowered to the
@@ -1785,12 +1806,10 @@ makeStatepointExplicitImpl(CallBase *Call, /* to replace */
     SPCall->setTailCallKind(CI->getTailCallKind());
     SPCall->setCallingConv(CI->getCallingConv());
 
-    // Currently we will fail on parameter attributes and on certain
-    // function attributes.  In case if we can handle this set of attributes -
-    // set up function attrs directly on statepoint and return attrs later for
+    // Set up function attrs directly on statepoint and return attrs later for
     // gc_result intrinsic.
-    SPCall->setAttributes(legalizeCallAttributes(
-        CI->getContext(), CI->getAttributes(), SPCall->getAttributes()));
+    SPCall->setAttributes(
+        legalizeCallAttributes(CI, IsMemIntrinsic, SPCall->getAttributes()));
 
     Token = cast<GCStatepointInst>(SPCall);
 
@@ -1812,12 +1831,10 @@ makeStatepointExplicitImpl(CallBase *Call, /* to replace */
 
     SPInvoke->setCallingConv(II->getCallingConv());
 
-    // Currently we will fail on parameter attributes and on certain
-    // function attributes.  In case if we can handle this set of attributes -
-    // set up function attrs directly on statepoint and return attrs later for
+    // Set up function attrs directly on statepoint and return attrs later for
     // gc_result intrinsic.
-    SPInvoke->setAttributes(legalizeCallAttributes(
-        II->getContext(), II->getAttributes(), SPInvoke->getAttributes()));
+    SPInvoke->setAttributes(
+        legalizeCallAttributes(II, IsMemIntrinsic, SPInvoke->getAttributes()));
 
     Token = cast<GCStatepointInst>(SPInvoke);
 
diff --git a/llvm/lib/Transforms/Scalar/Scalar.cpp b/llvm/lib/Transforms/Scalar/Scalar.cpp
index 4ec249e749..11bb8e936b 100644
--- a/llvm/lib/Transforms/Scalar/Scalar.cpp
+++ b/llvm/lib/Transforms/Scalar/Scalar.cpp
@@ -41,7 +41,6 @@ void llvm::initializeScalarOpts(PassRegistry &Registry) {
   initializeLoopUnrollPass(Registry);
   initializeLowerAtomicLegacyPassPass(Registry);
   initializeLowerConstantIntrinsicsPass(Registry);
-  initializeLowerExpectIntrinsicPass(Registry);
   initializeLowerWidenableConditionLegacyPassPass(Registry);
   initializeMergeICmpsLegacyPassPass(Registry);
   initializeMergedLoadStoreMotionLegacyPassPass(Registry);
diff --git a/llvm/lib/Transforms/Scalar/Scalarizer.cpp b/llvm/lib/Transforms/Scalar/Scalarizer.cpp
index 14df394e2b..f5e480ae36 100644
--- a/llvm/lib/Transforms/Scalar/Scalarizer.cpp
+++ b/llvm/lib/Transforms/Scalar/Scalarizer.cpp
@@ -282,12 +282,10 @@ T getWithDefaultOverride(const cl::opt<T> &ClOption,
 
 class ScalarizerVisitor : public InstVisitor<ScalarizerVisitor, bool> {
 public:
-  ScalarizerVisitor(unsigned ParallelLoopAccessMDKind, DominatorTree *DT,
-                    ScalarizerPassOptions Options)
-      : ParallelLoopAccessMDKind(ParallelLoopAccessMDKind), DT(DT),
-        ScalarizeVariableInsertExtract(
-            getWithDefaultOverride(ClScalarizeVariableInsertExtract,
-                                   Options.ScalarizeVariableInsertExtract)),
+  ScalarizerVisitor(DominatorTree *DT, ScalarizerPassOptions Options)
+      : DT(DT), ScalarizeVariableInsertExtract(getWithDefaultOverride(
+                    ClScalarizeVariableInsertExtract,
+                    Options.ScalarizeVariableInsertExtract)),
         ScalarizeLoadStore(getWithDefaultOverride(ClScalarizeLoadStore,
                                                   Options.ScalarizeLoadStore)),
         ScalarizeMinBits(getWithDefaultOverride(ClScalarizeMinBits,
@@ -337,8 +335,6 @@ class ScalarizerVisitor : public InstVisitor<ScalarizerVisitor, bool> {
 
   SmallVector<WeakTrackingVH, 32> PotentiallyDeadInstrs;
 
-  unsigned ParallelLoopAccessMDKind;
-
   DominatorTree *DT;
 
   const bool ScalarizeVariableInsertExtract;
@@ -447,11 +443,8 @@ bool ScalarizerLegacyPass::runOnFunction(Function &F) {
   if (skipFunction(F))
     return false;
 
-  Module &M = *F.getParent();
-  unsigned ParallelLoopAccessMDKind =
-      M.getContext().getMDKindID("llvm.mem.parallel_loop_access");
   DominatorTree *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
-  ScalarizerVisitor Impl(ParallelLoopAccessMDKind, DT, ScalarizerPassOptions());
+  ScalarizerVisitor Impl(DT, ScalarizerPassOptions());
   return Impl.visit(F);
 }
 
@@ -558,7 +551,7 @@ bool ScalarizerVisitor::canTransferMetadata(unsigned Tag) {
           || Tag == LLVMContext::MD_invariant_load
           || Tag == LLVMContext::MD_alias_scope
           || Tag == LLVMContext::MD_noalias
-          || Tag == ParallelLoopAccessMDKind
+          || Tag == LLVMContext::MD_mem_parallel_loop_access
           || Tag == LLVMContext::MD_access_group);
 }
 
@@ -1254,11 +1247,8 @@ bool ScalarizerVisitor::finish() {
 }
 
 PreservedAnalyses ScalarizerPass::run(Function &F, FunctionAnalysisManager &AM) {
-  Module &M = *F.getParent();
-  unsigned ParallelLoopAccessMDKind =
-      M.getContext().getMDKindID("llvm.mem.parallel_loop_access");
   DominatorTree *DT = &AM.getResult<DominatorTreeAnalysis>(F);
-  ScalarizerVisitor Impl(ParallelLoopAccessMDKind, DT, Options);
+  ScalarizerVisitor Impl(DT, Options);
   bool Changed = Impl.visit(F);
   PreservedAnalyses PA;
   PA.preserve<DominatorTreeAnalysis>();
diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
index 08b2b01b2e..b251a85cf8 100644
--- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp
+++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
@@ -245,12 +245,13 @@ CodeExtractor::CodeExtractor(ArrayRef<BasicBlock *> BBs, DominatorTree *DT,
                              bool AggregateArgs, BlockFrequencyInfo *BFI,
                              BranchProbabilityInfo *BPI, AssumptionCache *AC,
                              bool AllowVarArgs, bool AllowAlloca,
-                             BasicBlock *AllocationBlock, std::string Suffix)
+                             BasicBlock *AllocationBlock, std::string Suffix,
+                             bool ArgsInZeroAddressSpace)
     : DT(DT), AggregateArgs(AggregateArgs || AggregateArgsOpt), BFI(BFI),
       BPI(BPI), AC(AC), AllocationBlock(AllocationBlock),
       AllowVarArgs(AllowVarArgs),
       Blocks(buildExtractionBlockSet(BBs, DT, AllowVarArgs, AllowAlloca)),
-      Suffix(Suffix) {}
+      Suffix(Suffix), ArgsInZeroAddressSpace(ArgsInZeroAddressSpace) {}
 
 CodeExtractor::CodeExtractor(DominatorTree &DT, Loop &L, bool AggregateArgs,
                              BlockFrequencyInfo *BFI,
@@ -866,7 +867,8 @@ Function *CodeExtractor::constructFunction(const ValueSet &inputs,
   StructType *StructTy = nullptr;
   if (AggregateArgs && !AggParamTy.empty()) {
     StructTy = StructType::get(M->getContext(), AggParamTy);
-    ParamTy.push_back(PointerType::get(StructTy, DL.getAllocaAddrSpace()));
+    ParamTy.push_back(PointerType::get(
+        StructTy, ArgsInZeroAddressSpace ? 0 : DL.getAllocaAddrSpace()));
   }
 
   LLVM_DEBUG({
@@ -941,6 +943,7 @@ Function *CodeExtractor::constructFunction(const ValueSet &inputs,
       case Attribute::NoSanitizeBounds:
       case Attribute::NoSanitizeCoverage:
       case Attribute::NullPointerIsValid:
+      case Attribute::OptimizeForDebugging:
       case Attribute::OptForFuzzing:
       case Attribute::OptimizeNone:
       case Attribute::OptimizeForSize:
@@ -1186,8 +1189,15 @@ CallInst *CodeExtractor::emitCallAndSwitchStatement(Function *newFunction,
         StructArgTy, DL.getAllocaAddrSpace(), nullptr, "structArg",
         AllocationBlock ? &*AllocationBlock->getFirstInsertionPt()
                         : &codeReplacer->getParent()->front().front());
-    params.push_back(Struct);
 
+    if (ArgsInZeroAddressSpace && DL.getAllocaAddrSpace() != 0) {
+      auto *StructSpaceCast = new AddrSpaceCastInst(
+          Struct, PointerType ::get(Context, 0), "structArg.ascast");
+      StructSpaceCast->insertAfter(Struct);
+      params.push_back(StructSpaceCast);
+    } else {
+      params.push_back(Struct);
+    }
     // Store aggregated inputs in the struct.
     for (unsigned i = 0, e = StructValues.size(); i != e; ++i) {
       if (inputs.contains(StructValues[i])) {
diff --git a/llvm/lib/Transforms/Utils/CodeLayout.cpp b/llvm/lib/Transforms/Utils/CodeLayout.cpp
index 057a5e86c0..a6c9d2ac6c 100644
--- a/llvm/lib/Transforms/Utils/CodeLayout.cpp
+++ b/llvm/lib/Transforms/Utils/CodeLayout.cpp
@@ -99,7 +99,7 @@ static cl::opt<unsigned> BackwardDistance(
     cl::desc("The maximum distance (in bytes) of a backward jump for ExtTSP"));
 
 // The maximum size of a chain created by the algorithm. The size is bounded
-// so that the algorithm can efficiently process extremely large instance.
+// so that the algorithm can efficiently process extremely large instances.
 static cl::opt<unsigned>
     MaxChainSize("ext-tsp-max-chain-size", cl::ReallyHidden, cl::init(4096),
                  cl::desc("The maximum size of a chain to create."));
@@ -123,6 +123,10 @@ static cl::opt<unsigned> CacheEntries("cds-cache-entries", cl::ReallyHidden,
 static cl::opt<unsigned> CacheSize("cds-cache-size", cl::ReallyHidden,
                                    cl::desc("The size of a line in the cache"));
 
+static cl::opt<unsigned>
+    CDMaxChainSize("cdsort-max-chain-size", cl::ReallyHidden,
+                   cl::desc("The maximum size of a chain to create"));
+
 static cl::opt<double> DistancePower(
     "cds-distance-power", cl::ReallyHidden,
     cl::desc("The power exponent for the distance-based locality"));
@@ -217,8 +221,8 @@ struct NodeT {
   NodeT &operator=(const NodeT &) = delete;
   NodeT &operator=(NodeT &&) = default;
 
-  explicit NodeT(size_t Index, uint64_t Size, uint64_t EC)
-      : Index(Index), Size(Size), ExecutionCount(EC) {}
+  explicit NodeT(size_t Index, uint64_t Size, uint64_t Count)
+      : Index(Index), Size(Size), ExecutionCount(Count) {}
 
   bool isEntry() const { return Index == 0; }
 
@@ -476,13 +480,16 @@ void ChainT::mergeEdges(ChainT *Other) {
 }
 
 using NodeIter = std::vector<NodeT *>::const_iterator;
-
-/// A wrapper around three chains of nodes; it is used to avoid extra
-/// instantiation of the vectors.
-struct MergedChain {
-  MergedChain(NodeIter Begin1, NodeIter End1, NodeIter Begin2 = NodeIter(),
-              NodeIter End2 = NodeIter(), NodeIter Begin3 = NodeIter(),
-              NodeIter End3 = NodeIter())
+static std::vector<NodeT *> EmptyList;
+
+/// A wrapper around three concatenated vectors (chains) of nodes; it is used
+/// to avoid extra instantiation of the vectors.
+struct MergedNodesT {
+  MergedNodesT(NodeIter Begin1, NodeIter End1,
+               NodeIter Begin2 = EmptyList.begin(),
+               NodeIter End2 = EmptyList.end(),
+               NodeIter Begin3 = EmptyList.begin(),
+               NodeIter End3 = EmptyList.end())
       : Begin1(Begin1), End1(End1), Begin2(Begin2), End2(End2), Begin3(Begin3),
         End3(End3) {}
 
@@ -507,6 +514,8 @@ struct MergedChain {
 
   const NodeT *getFirstNode() const { return *Begin1; }
 
+  bool empty() const { return Begin1 == End1; }
+
 private:
   NodeIter Begin1;
   NodeIter End1;
@@ -516,14 +525,34 @@ struct MergedChain {
   NodeIter End3;
 };
 
+/// A wrapper around two concatenated vectors (chains) of jumps.
+struct MergedJumpsT {
+  MergedJumpsT(const std::vector<JumpT *> *Jumps1,
+               const std::vector<JumpT *> *Jumps2 = nullptr) {
+    assert(!Jumps1->empty() && "cannot merge empty jump list");
+    JumpArray[0] = Jumps1;
+    JumpArray[1] = Jumps2;
+  }
+
+  template <typename F> void forEach(const F &Func) const {
+    for (auto Jumps : JumpArray)
+      if (Jumps != nullptr)
+        for (JumpT *Jump : *Jumps)
+          Func(Jump);
+  }
+
+private:
+  std::array<const std::vector<JumpT *> *, 2> JumpArray{nullptr, nullptr};
+};
+
 /// Merge two chains of nodes respecting a given 'type' and 'offset'.
 ///
 /// If MergeType == 0, then the result is a concatenation of two chains.
 /// Otherwise, the first chain is cut into two sub-chains at the offset,
 /// and merged using all possible ways of concatenating three chains.
-MergedChain mergeNodes(const std::vector<NodeT *> &X,
-                       const std::vector<NodeT *> &Y, size_t MergeOffset,
-                       MergeTypeT MergeType) {
+MergedNodesT mergeNodes(const std::vector<NodeT *> &X,
+                        const std::vector<NodeT *> &Y, size_t MergeOffset,
+                        MergeTypeT MergeType) {
   // Split the first chain, X, into X1 and X2.
   NodeIter BeginX1 = X.begin();
   NodeIter EndX1 = X.begin() + MergeOffset;
@@ -535,15 +564,15 @@ MergedChain mergeNodes(const std::vector<NodeT *> &X,
   // Construct a new chain from the three existing ones.
   switch (MergeType) {
   case MergeTypeT::X_Y:
-    return MergedChain(BeginX1, EndX2, BeginY, EndY);
+    return MergedNodesT(BeginX1, EndX2, BeginY, EndY);
   case MergeTypeT::Y_X:
-    return MergedChain(BeginY, EndY, BeginX1, EndX2);
+    return MergedNodesT(BeginY, EndY, BeginX1, EndX2);
   case MergeTypeT::X1_Y_X2:
-    return MergedChain(BeginX1, EndX1, BeginY, EndY, BeginX2, EndX2);
+    return MergedNodesT(BeginX1, EndX1, BeginY, EndY, BeginX2, EndX2);
   case MergeTypeT::Y_X2_X1:
-    return MergedChain(BeginY, EndY, BeginX2, EndX2, BeginX1, EndX1);
+    return MergedNodesT(BeginY, EndY, BeginX2, EndX2, BeginX1, EndX1);
   case MergeTypeT::X2_X1_Y:
-    return MergedChain(BeginX2, EndX2, BeginX1, EndX1, BeginY, EndY);
+    return MergedNodesT(BeginX2, EndX2, BeginX1, EndX1, BeginY, EndY);
   }
   llvm_unreachable("unexpected chain merge type");
 }
@@ -618,6 +647,7 @@ class ExtTSPImpl {
     AllChains.reserve(NumNodes);
     HotChains.reserve(NumNodes);
     for (NodeT &Node : AllNodes) {
+      // Create a chain.
       AllChains.emplace_back(Node.Index, &Node);
       Node.CurChain = &AllChains.back();
       if (Node.ExecutionCount > 0)
@@ -630,13 +660,13 @@ class ExtTSPImpl {
       for (JumpT *Jump : PredNode.OutJumps) {
         NodeT *SuccNode = Jump->Target;
         ChainEdge *CurEdge = PredNode.CurChain->getEdge(SuccNode->CurChain);
-        // this edge is already present in the graph.
+        // This edge is already present in the graph.
         if (CurEdge != nullptr) {
           assert(SuccNode->CurChain->getEdge(PredNode.CurChain) != nullptr);
           CurEdge->appendJump(Jump);
           continue;
         }
-        // this is a new edge.
+        // This is a new edge.
         AllEdges.emplace_back(Jump);
         PredNode.CurChain->addEdge(SuccNode->CurChain, &AllEdges.back());
         SuccNode->CurChain->addEdge(PredNode.CurChain, &AllEdges.back());
@@ -649,7 +679,7 @@ class ExtTSPImpl {
   /// to B are from A. Such nodes should be adjacent in the optimal ordering;
   /// the method finds and merges such pairs of nodes.
   void mergeForcedPairs() {
-    // Find fallthroughs based on edge weights.
+    // Find forced pairs of blocks.
     for (NodeT &Node : AllNodes) {
       if (SuccNodes[Node.Index].size() == 1 &&
           PredNodes[SuccNodes[Node.Index][0]].size() == 1 &&
@@ -699,9 +729,7 @@ class ExtTSPImpl {
     /// Deterministically compare pairs of chains.
     auto compareChainPairs = [](const ChainT *A1, const ChainT *B1,
                                 const ChainT *A2, const ChainT *B2) {
-      if (A1 != A2)
-        return A1->Id < A2->Id;
-      return B1->Id < B2->Id;
+      return std::make_tuple(A1->Id, B1->Id) < std::make_tuple(A2->Id, B2->Id);
     };
 
     while (HotChains.size() > 1) {
@@ -769,24 +797,22 @@ class ExtTSPImpl {
   }
 
   /// Compute the Ext-TSP score for a given node order and a list of jumps.
-  double extTSPScore(const MergedChain &MergedBlocks,
-                     const std::vector<JumpT *> &Jumps) const {
-    if (Jumps.empty())
-      return 0.0;
+  double extTSPScore(const MergedNodesT &Nodes,
+                     const MergedJumpsT &Jumps) const {
     uint64_t CurAddr = 0;
-    MergedBlocks.forEach([&](const NodeT *Node) {
+    Nodes.forEach([&](const NodeT *Node) {
       Node->EstimatedAddr = CurAddr;
       CurAddr += Node->Size;
     });
 
     double Score = 0;
-    for (JumpT *Jump : Jumps) {
+    Jumps.forEach([&](const JumpT *Jump) {
       const NodeT *SrcBlock = Jump->Source;
       const NodeT *DstBlock = Jump->Target;
       Score += ::extTSPScore(SrcBlock->EstimatedAddr, SrcBlock->Size,
                              DstBlock->EstimatedAddr, Jump->ExecutionCount,
                              Jump->IsConditional);
-    }
+    });
     return Score;
   }
 
@@ -798,17 +824,13 @@ class ExtTSPImpl {
   /// element being the corresponding merging type.
   MergeGainT getBestMergeGain(ChainT *ChainPred, ChainT *ChainSucc,
                               ChainEdge *Edge) const {
-    if (Edge->hasCachedMergeGain(ChainPred, ChainSucc)) {
+    if (Edge->hasCachedMergeGain(ChainPred, ChainSucc))
       return Edge->getCachedMergeGain(ChainPred, ChainSucc);
-    }
 
+    assert(!Edge->jumps().empty() && "trying to merge chains w/o jumps");
     // Precompute jumps between ChainPred and ChainSucc.
-    auto Jumps = Edge->jumps();
     ChainEdge *EdgePP = ChainPred->getEdge(ChainPred);
-    if (EdgePP != nullptr) {
-      Jumps.insert(Jumps.end(), EdgePP->jumps().begin(), EdgePP->jumps().end());
-    }
-    assert(!Jumps.empty() && "trying to merge chains w/o jumps");
+    MergedJumpsT Jumps(&Edge->jumps(), EdgePP ? &EdgePP->jumps() : nullptr);
 
     // This object holds the best chosen gain of merging two chains.
     MergeGainT Gain = MergeGainT();
@@ -875,19 +897,20 @@ class ExtTSPImpl {
   ///
   /// The two chains are not modified in the method.
   MergeGainT computeMergeGain(const ChainT *ChainPred, const ChainT *ChainSucc,
-                              const std::vector<JumpT *> &Jumps,
-                              size_t MergeOffset, MergeTypeT MergeType) const {
-    auto MergedBlocks =
+                              const MergedJumpsT &Jumps, size_t MergeOffset,
+                              MergeTypeT MergeType) const {
+    MergedNodesT MergedNodes =
         mergeNodes(ChainPred->Nodes, ChainSucc->Nodes, MergeOffset, MergeType);
 
     // Do not allow a merge that does not preserve the original entry point.
     if ((ChainPred->isEntry() || ChainSucc->isEntry()) &&
-        !MergedBlocks.getFirstNode()->isEntry())
+        !MergedNodes.getFirstNode()->isEntry())
       return MergeGainT();
 
     // The gain for the new chain.
-    auto NewGainScore = extTSPScore(MergedBlocks, Jumps) - ChainPred->Score;
-    return MergeGainT(NewGainScore, MergeOffset, MergeType);
+    double NewScore = extTSPScore(MergedNodes, Jumps);
+    double CurScore = ChainPred->Score;
+    return MergeGainT(NewScore - CurScore, MergeOffset, MergeType);
   }
 
   /// Merge chain From into chain Into, update the list of active chains,
@@ -897,7 +920,7 @@ class ExtTSPImpl {
     assert(Into != From && "a chain cannot be merged with itself");
 
     // Merge the nodes.
-    MergedChain MergedNodes =
+    MergedNodesT MergedNodes =
         mergeNodes(Into->Nodes, From->Nodes, MergeOffset, MergeType);
     Into->merge(From, MergedNodes.getNodes());
 
@@ -908,8 +931,9 @@ class ExtTSPImpl {
     // Update cached ext-tsp score for the new chain.
     ChainEdge *SelfEdge = Into->getEdge(Into);
     if (SelfEdge != nullptr) {
-      MergedNodes = MergedChain(Into->Nodes.begin(), Into->Nodes.end());
-      Into->Score = extTSPScore(MergedNodes, SelfEdge->jumps());
+      MergedNodes = MergedNodesT(Into->Nodes.begin(), Into->Nodes.end());
+      MergedJumpsT MergedJumps(&SelfEdge->jumps());
+      Into->Score = extTSPScore(MergedNodes, MergedJumps);
     }
 
     // Remove the chain from the list of active chains.
@@ -943,7 +967,7 @@ class ExtTSPImpl {
     // Sorting chains by density in the decreasing order.
     std::sort(SortedChains.begin(), SortedChains.end(),
               [&](const ChainT *L, const ChainT *R) {
-                // Place the entry point is at the beginning of the order.
+                // Place the entry point at the beginning of the order.
                 if (L->isEntry() != R->isEntry())
                   return L->isEntry();
 
@@ -1005,10 +1029,6 @@ class CDSortImpl {
     // Merge pairs of chains while improving the objective.
     mergeChainPairs();
 
-    LLVM_DEBUG(dbgs() << "Cache-directed function sorting reduced the number"
-                      << " of chains from " << NumNodes << " to "
-                      << HotChains.size() << "\n");
-
     // Collect nodes from all the chains.
     return concatChains();
   }
@@ -1054,7 +1074,6 @@ class CDSortImpl {
 
     // Initialize chains.
     AllChains.reserve(NumNodes);
-    HotChains.reserve(NumNodes);
     for (NodeT &Node : AllNodes) {
       // Adjust execution counts.
       Node.ExecutionCount = std::max(Node.ExecutionCount, Node.inCount());
@@ -1062,8 +1081,6 @@ class CDSortImpl {
       // Create chain.
       AllChains.emplace_back(Node.Index, &Node);
       Node.CurChain = &AllChains.back();
-      if (Node.ExecutionCount > 0)
-        HotChains.push_back(&AllChains.back());
     }
 
     // Initialize chain edges.
@@ -1096,8 +1113,12 @@ class CDSortImpl {
     std::set<ChainEdge *, decltype(GainComparator)> Queue(GainComparator);
 
     // Insert the edges into the queue.
-    for (ChainT *ChainPred : HotChains) {
-      for (const auto &[_, Edge] : ChainPred->Edges) {
+    [[maybe_unused]] size_t NumActiveChains = 0;
+    for (NodeT &Node : AllNodes) {
+      if (Node.ExecutionCount == 0)
+        continue;
+      ++NumActiveChains;
+      for (const auto &[_, Edge] : Node.CurChain->Edges) {
         // Ignore self-edges.
         if (Edge->isSelfEdge())
           continue;
@@ -1119,13 +1140,6 @@ class CDSortImpl {
       // Extract the best (top) edge for merging.
       ChainEdge *BestEdge = *Queue.begin();
       Queue.erase(Queue.begin());
-      // Ignore self-edges.
-      if (BestEdge->isSelfEdge())
-        continue;
-      // Ignore edges with non-positive gains.
-      if (BestEdge->gain() <= EPS)
-        continue;
-
       ChainT *BestSrcChain = BestEdge->srcChain();
       ChainT *BestDstChain = BestEdge->dstChain();
 
@@ -1139,12 +1153,16 @@ class CDSortImpl {
       MergeGainT BestGain = BestEdge->getMergeGain();
       mergeChains(BestSrcChain, BestDstChain, BestGain.mergeOffset(),
                   BestGain.mergeType());
+      --NumActiveChains;
 
       // Insert newly created edges into the queue.
       for (const auto &[_, Edge] : BestSrcChain->Edges) {
         // Ignore loop edges.
         if (Edge->isSelfEdge())
           continue;
+        if (Edge->srcChain()->numBlocks() + Edge->dstChain()->numBlocks() >
+            Config.MaxChainSize)
+          continue;
 
         // Compute the gain of merging the two chains.
         MergeGainT Gain = getBestMergeGain(Edge);
@@ -1154,6 +1172,10 @@ class CDSortImpl {
           Queue.insert(Edge);
       }
     }
+
+    LLVM_DEBUG(dbgs() << "Cache-directed function sorting reduced the number"
+                      << " of chains from " << NumNodes << " to "
+                      << NumActiveChains << "\n");
   }
 
   /// Compute the gain of merging two chains.
@@ -1163,9 +1185,9 @@ class CDSortImpl {
   /// result is a pair with the first element being the gain and the second
   /// element being the corresponding merging type.
   MergeGainT getBestMergeGain(ChainEdge *Edge) const {
+    assert(!Edge->jumps().empty() && "trying to merge chains w/o jumps");
     // Precompute jumps between ChainPred and ChainSucc.
-    auto Jumps = Edge->jumps();
-    assert(!Jumps.empty() && "trying to merge chains w/o jumps");
+    MergedJumpsT Jumps(&Edge->jumps());
     ChainT *SrcChain = Edge->srcChain();
     ChainT *DstChain = Edge->dstChain();
 
@@ -1204,7 +1226,7 @@ class CDSortImpl {
   ///
   /// The two chains are not modified in the method.
   MergeGainT computeMergeGain(ChainT *ChainPred, ChainT *ChainSucc,
-                              const std::vector<JumpT *> &Jumps,
+                              const MergedJumpsT &Jumps,
                               MergeTypeT MergeType) const {
     // This doesn't depend on the ordering of the nodes
     double FreqGain = freqBasedLocalityGain(ChainPred, ChainSucc);
@@ -1255,24 +1277,22 @@ class CDSortImpl {
   }
 
   /// Compute the change of the distance locality after merging the chains.
-  double distBasedLocalityGain(const MergedChain &MergedBlocks,
-                               const std::vector<JumpT *> &Jumps) const {
-    if (Jumps.empty())
-      return 0.0;
+  double distBasedLocalityGain(const MergedNodesT &Nodes,
+                               const MergedJumpsT &Jumps) const {
     uint64_t CurAddr = 0;
-    MergedBlocks.forEach([&](const NodeT *Node) {
+    Nodes.forEach([&](const NodeT *Node) {
       Node->EstimatedAddr = CurAddr;
       CurAddr += Node->Size;
     });
 
     double CurScore = 0;
     double NewScore = 0;
-    for (const JumpT *Arc : Jumps) {
-      uint64_t SrcAddr = Arc->Source->EstimatedAddr + Arc->Offset;
-      uint64_t DstAddr = Arc->Target->EstimatedAddr;
-      NewScore += distScore(SrcAddr, DstAddr, Arc->ExecutionCount);
-      CurScore += distScore(0, TotalSize, Arc->ExecutionCount);
-    }
+    Jumps.forEach([&](const JumpT *Jump) {
+      uint64_t SrcAddr = Jump->Source->EstimatedAddr + Jump->Offset;
+      uint64_t DstAddr = Jump->Target->EstimatedAddr;
+      NewScore += distScore(SrcAddr, DstAddr, Jump->ExecutionCount);
+      CurScore += distScore(0, TotalSize, Jump->ExecutionCount);
+    });
     return NewScore - CurScore;
   }
 
@@ -1283,16 +1303,13 @@ class CDSortImpl {
     assert(Into != From && "a chain cannot be merged with itself");
 
     // Merge the nodes.
-    MergedChain MergedNodes =
+    MergedNodesT MergedNodes =
         mergeNodes(Into->Nodes, From->Nodes, MergeOffset, MergeType);
     Into->merge(From, MergedNodes.getNodes());
 
     // Merge the edges.
     Into->mergeEdges(From);
     From->clear();
-
-    // Remove the chain from the list of active chains.
-    llvm::erase_value(HotChains, From);
   }
 
   /// Concatenate all chains into the final order.
@@ -1359,9 +1376,6 @@ class CDSortImpl {
   /// All edges between the chains.
   std::vector<ChainEdge> AllEdges;
 
-  /// Active chains. The vector gets updated at runtime when chains are merged.
-  std::vector<ChainT *> HotChains;
-
   /// The total number of samples in the graph.
   uint64_t TotalSamples{0};
 
@@ -1445,6 +1459,8 @@ std::vector<uint64_t> codelayout::computeCacheDirectedLayout(
     Config.CacheEntries = CacheEntries;
   if (CacheSize.getNumOccurrences() > 0)
     Config.CacheSize = CacheSize;
+  if (CDMaxChainSize.getNumOccurrences() > 0)
+    Config.MaxChainSize = CDMaxChainSize;
   if (DistancePower.getNumOccurrences() > 0)
     Config.DistancePower = DistancePower;
   if (FrequencyScale.getNumOccurrences() > 0)
diff --git a/llvm/lib/Transforms/Utils/FunctionComparator.cpp b/llvm/lib/Transforms/Utils/FunctionComparator.cpp
index b1d74f6737..79ca99d156 100644
--- a/llvm/lib/Transforms/Utils/FunctionComparator.cpp
+++ b/llvm/lib/Transforms/Utils/FunctionComparator.cpp
@@ -160,10 +160,23 @@ int FunctionComparator::cmpAttrs(const AttributeList L,
 int FunctionComparator::cmpMetadata(const Metadata *L,
                                     const Metadata *R) const {
   // TODO: the following routine coerce the metadata contents into constants
-  // before comparison.
+  // or MDStrings before comparison.
   // It ignores any other cases, so that the metadata nodes are considered
   // equal even though this is not correct.
   // We should structurally compare the metadata nodes to be perfect here.
+
+  auto *MDStringL = dyn_cast<MDString>(L);
+  auto *MDStringR = dyn_cast<MDString>(R);
+  if (MDStringL && MDStringR) {
+    if (MDStringL == MDStringR)
+      return 0;
+    return MDStringL->getString().compare(MDStringR->getString());
+  }
+  if (MDStringR)
+    return -1;
+  if (MDStringL)
+    return 1;
+
   auto *CL = dyn_cast<ConstantAsMetadata>(L);
   auto *CR = dyn_cast<ConstantAsMetadata>(R);
   if (CL == CR)
@@ -820,6 +833,21 @@ int FunctionComparator::cmpValues(const Value *L, const Value *R) const {
   if (ConstR)
     return -1;
 
+  const MetadataAsValue *MetadataValueL = dyn_cast<MetadataAsValue>(L);
+  const MetadataAsValue *MetadataValueR = dyn_cast<MetadataAsValue>(R);
+  if (MetadataValueL && MetadataValueR) {
+    if (MetadataValueL == MetadataValueR)
+      return 0;
+
+    return cmpMetadata(MetadataValueL->getMetadata(),
+                       MetadataValueR->getMetadata());
+  }
+
+  if (MetadataValueL)
+    return 1;
+  if (MetadataValueR)
+    return -1;
+
   const InlineAsm *InlineAsmL = dyn_cast<InlineAsm>(L);
   const InlineAsm *InlineAsmR = dyn_cast<InlineAsm>(R);
 
diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp
index ddb47e693a..e0467a319c 100644
--- a/llvm/lib/Transforms/Utils/Local.cpp
+++ b/llvm/lib/Transforms/Utils/Local.cpp
@@ -443,9 +443,16 @@ bool llvm::wouldInstructionBeTriviallyDead(const Instruction *I,
     if (!II)
       return false;
 
+    switch (II->getIntrinsicID()) {
+    case Intrinsic::experimental_guard: {
+      // Guards on true are operationally no-ops.  In the future we can
+      // consider more sophisticated tradeoffs for guards considering potential
+      // for check widening, but for now we keep things simple.
+      auto *Cond = dyn_cast<ConstantInt>(II->getArgOperand(0));
+      return Cond && Cond->isOne();
+    }
     // TODO: These intrinsics are not safe to remove, because this may remove
     // a well-defined trap.
-    switch (II->getIntrinsicID()) {
     case Intrinsic::wasm_trunc_signed:
     case Intrinsic::wasm_trunc_unsigned:
     case Intrinsic::ptrauth_auth:
@@ -484,13 +491,9 @@ bool llvm::wouldInstructionBeTriviallyDead(const Instruction *I,
       return false;
     }
 
-    // Assumptions are dead if their condition is trivially true.  Guards on
-    // true are operationally no-ops.  In the future we can consider more
-    // sophisticated tradeoffs for guards considering potential for check
-    // widening, but for now we keep things simple.
-    if ((II->getIntrinsicID() == Intrinsic::assume &&
-         isAssumeWithEmptyBundle(cast<AssumeInst>(*II))) ||
-        II->getIntrinsicID() == Intrinsic::experimental_guard) {
+    // Assumptions are dead if their condition is trivially true.
+    if (II->getIntrinsicID() == Intrinsic::assume &&
+        isAssumeWithEmptyBundle(cast<AssumeInst>(*II))) {
       if (ConstantInt *Cond = dyn_cast<ConstantInt>(II->getArgOperand(0)))
         return !Cond->isZero();
 
diff --git a/llvm/lib/Transforms/Utils/LoopUnroll.cpp b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
index 511dd61308..d3521e9893 100644
--- a/llvm/lib/Transforms/Utils/LoopUnroll.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
@@ -24,7 +24,6 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/ADT/ilist_iterator.h"
-#include "llvm/ADT/iterator_range.h"
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/InstructionSimplify.h"
diff --git a/llvm/lib/Transforms/Utils/LoopUnrollAndJam.cpp b/llvm/lib/Transforms/Utils/LoopUnrollAndJam.cpp
index 31b8cd34eb..3c06a6e47a 100644
--- a/llvm/lib/Transforms/Utils/LoopUnrollAndJam.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnrollAndJam.cpp
@@ -19,7 +19,6 @@
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Twine.h"
-#include "llvm/ADT/iterator_range.h"
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/DependenceAnalysis.h"
 #include "llvm/Analysis/DomTreeUpdater.h"
diff --git a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
index 1caf708bcc..a23ac41aca 100644
--- a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
@@ -659,12 +659,12 @@ bool SimplifyIndvar::replaceFloatIVWithIntegerIV(Instruction *UseInst) {
   Instruction *IVOperand = cast<Instruction>(UseInst->getOperand(0));
   // Get the symbolic expression for this instruction.
   const SCEV *IV = SE->getSCEV(IVOperand);
-  unsigned MaskBits;
+  int MaskBits;
   if (UseInst->getOpcode() == CastInst::SIToFP)
-    MaskBits = SE->getSignedRange(IV).getMinSignedBits();
+    MaskBits = (int)SE->getSignedRange(IV).getMinSignedBits();
   else
-    MaskBits = SE->getUnsignedRange(IV).getActiveBits();
-  unsigned DestNumSigBits = UseInst->getType()->getFPMantissaWidth();
+    MaskBits = (int)SE->getUnsignedRange(IV).getActiveBits();
+  int DestNumSigBits = UseInst->getType()->getFPMantissaWidth();
   if (MaskBits <= DestNumSigBits) {
     for (User *U : UseInst->users()) {
       // Match for fptosi/fptoui of sitofp and with same type.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 9691e1cd4f..577ce8000d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -31,6 +31,7 @@
 namespace llvm {
 
 class LoopInfo;
+class DominatorTree;
 class LoopVectorizationLegality;
 class LoopVectorizationCostModel;
 class PredicatedScalarEvolution;
@@ -287,6 +288,9 @@ class LoopVectorizationPlanner {
   /// Loop Info analysis.
   LoopInfo *LI;
 
+  /// The dominator tree.
+  DominatorTree *DT;
+
   /// Target Library Info.
   const TargetLibraryInfo *TLI;
 
@@ -317,16 +321,14 @@ class LoopVectorizationPlanner {
   VPBuilder Builder;
 
 public:
-  LoopVectorizationPlanner(Loop *L, LoopInfo *LI, const TargetLibraryInfo *TLI,
-                           const TargetTransformInfo &TTI,
-                           LoopVectorizationLegality *Legal,
-                           LoopVectorizationCostModel &CM,
-                           InterleavedAccessInfo &IAI,
-                           PredicatedScalarEvolution &PSE,
-                           const LoopVectorizeHints &Hints,
-                           OptimizationRemarkEmitter *ORE)
-      : OrigLoop(L), LI(LI), TLI(TLI), TTI(TTI), Legal(Legal), CM(CM), IAI(IAI),
-        PSE(PSE), Hints(Hints), ORE(ORE) {}
+  LoopVectorizationPlanner(
+      Loop *L, LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI,
+      const TargetTransformInfo &TTI, LoopVectorizationLegality *Legal,
+      LoopVectorizationCostModel &CM, InterleavedAccessInfo &IAI,
+      PredicatedScalarEvolution &PSE, const LoopVectorizeHints &Hints,
+      OptimizationRemarkEmitter *ORE)
+      : OrigLoop(L), LI(LI), DT(DT), TLI(TLI), TTI(TTI), Legal(Legal), CM(CM),
+        IAI(IAI), PSE(PSE), Hints(Hints), ORE(ORE) {}
 
   /// Plan how to best vectorize, return the best VF and its cost, or
   /// std::nullopt if vectorization and interleaving should be avoided up front.
@@ -352,7 +354,7 @@ class LoopVectorizationPlanner {
   executePlan(ElementCount VF, unsigned UF, VPlan &BestPlan,
               InnerLoopVectorizer &LB, DominatorTree *DT,
               bool IsEpilogueVectorization,
-              DenseMap<const SCEV *, Value *> *ExpandedSCEVs = nullptr);
+              const DenseMap<const SCEV *, Value *> *ExpandedSCEVs = nullptr);
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   void printPlans(raw_ostream &O);
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 88f064b6d5..0807d2a7e5 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3543,6 +3543,7 @@ void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State,
 
   // Forget the original basic block.
   PSE.getSE()->forgetLoop(OrigLoop);
+  PSE.getSE()->forgetBlockAndLoopDispositions();
 
   // After vectorization, the exit blocks of the original loop will have
   // additional predecessors. Invalidate SCEVs for the exit phis in case SE
@@ -3616,40 +3617,10 @@ void InnerLoopVectorizer::fixCrossIterationPHIs(VPTransformState &State) {
   VPBasicBlock *Header =
       State.Plan->getVectorLoopRegion()->getEntryBasicBlock();
 
-  // Gather all VPReductionPHIRecipe and sort them so that Intermediate stores
-  // sank outside of the loop would keep the same order as they had in the
-  // original loop.
-  SmallVector<VPReductionPHIRecipe *> ReductionPHIList;
   for (VPRecipeBase &R : Header->phis()) {
     if (auto *ReductionPhi = dyn_cast<VPReductionPHIRecipe>(&R))
-      ReductionPHIList.emplace_back(ReductionPhi);
+      fixReduction(ReductionPhi, State);
   }
-  stable_sort(ReductionPHIList, [this](const VPReductionPHIRecipe *R1,
-                                       const VPReductionPHIRecipe *R2) {
-    auto *IS1 = R1->getRecurrenceDescriptor().IntermediateStore;
-    auto *IS2 = R2->getRecurrenceDescriptor().IntermediateStore;
-
-    // If neither of the recipes has an intermediate store, keep the order the
-    // same.
-    if (!IS1 && !IS2)
-      return false;
-
-    // If only one of the recipes has an intermediate store, then move it
-    // towards the beginning of the list.
-    if (IS1 && !IS2)
-      return true;
-
-    if (!IS1 && IS2)
-      return false;
-
-    // If both recipes have an intermediate store, then the recipe with the
-    // later store should be processed earlier. So it should go to the beginning
-    // of the list.
-    return DT->dominates(IS2, IS1);
-  });
-
-  for (VPReductionPHIRecipe *ReductionPhi : ReductionPHIList)
-    fixReduction(ReductionPhi, State);
 
   for (VPRecipeBase &R : Header->phis()) {
     if (auto *FOR = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(&R))
@@ -3791,8 +3762,6 @@ void InnerLoopVectorizer::fixReduction(VPReductionPHIRecipe *PhiR,
     State.setDebugLocFrom(I->getDebugLoc());
 
   VPValue *LoopExitInstDef = PhiR->getBackedgeValue();
-  // This is the vector-clone of the value that leaves the loop.
-  Type *VecTy = State.get(LoopExitInstDef, 0)->getType();
 
   // Before each round, move the insertion point right between
   // the PHIs and the values we are going to write.
@@ -3804,10 +3773,6 @@ void InnerLoopVectorizer::fixReduction(VPReductionPHIRecipe *PhiR,
   State.setDebugLocFrom(LoopExitInst->getDebugLoc());
 
   Type *PhiTy = OrigPhi->getType();
-
-  VPBasicBlock *LatchVPBB =
-      PhiR->getParent()->getEnclosingLoopRegion()->getExitingBasicBlock();
-  BasicBlock *VectorLoopLatch = State.CFG.VPBB2IRBB[LatchVPBB];
   // If tail is folded by masking, the vector value to leave the loop should be
   // a Select choosing between the vectorized LoopExitInst and vectorized Phi,
   // instead of the former. For an inloop reduction the reduction will already
@@ -3833,23 +3798,12 @@ void InnerLoopVectorizer::fixReduction(VPReductionPHIRecipe *PhiR,
   // then extend the loop exit value to enable InstCombine to evaluate the
   // entire expression in the smaller type.
   if (VF.isVector() && PhiTy != RdxDesc.getRecurrenceType()) {
-    assert(!PhiR->isInLoop() && "Unexpected truncated inloop reduction!");
-    Type *RdxVecTy = VectorType::get(RdxDesc.getRecurrenceType(), VF);
-    Builder.SetInsertPoint(VectorLoopLatch->getTerminator());
-    for (unsigned Part = 0; Part < UF; ++Part) {
-      Value *Trunc = Builder.CreateTrunc(RdxParts[Part], RdxVecTy);
-      Value *Extnd = RdxDesc.isSigned() ? Builder.CreateSExt(Trunc, VecTy)
-                                        : Builder.CreateZExt(Trunc, VecTy);
-      for (User *U : llvm::make_early_inc_range(RdxParts[Part]->users()))
-        if (U != Trunc) {
-          U->replaceUsesOfWith(RdxParts[Part], Extnd);
-          RdxParts[Part] = Extnd;
-        }
-    }
     Builder.SetInsertPoint(LoopMiddleBlock,
                            LoopMiddleBlock->getFirstInsertionPt());
-    for (unsigned Part = 0; Part < UF; ++Part)
+    Type *RdxVecTy = VectorType::get(RdxDesc.getRecurrenceType(), VF);
+    for (unsigned Part = 0; Part < UF; ++Part) {
       RdxParts[Part] = Builder.CreateTrunc(RdxParts[Part], RdxVecTy);
+    }
   }
 
   // Reduce all of the unrolled parts into a single vector.
@@ -7479,21 +7433,30 @@ VPValue *VPBuilder::createICmp(CmpInst::Predicate Pred, VPValue *A, VPValue *B,
       new VPInstruction(Instruction::ICmp, Pred, A, B, DL, Name));
 }
 
+// This function will select a scalable VF if the target supports scalable
+// vectors and a fixed one otherwise.
 // TODO: we could return a pair of values that specify the max VF and
 // min VF, to be used in `buildVPlans(MinVF, MaxVF)` instead of
 // `buildVPlans(VF, VF)`. We cannot do it because VPLAN at the moment
 // doesn't have a cost model that can choose which plan to execute if
 // more than one is generated.
-static unsigned determineVPlanVF(const unsigned WidestVectorRegBits,
-                                 LoopVectorizationCostModel &CM) {
+static ElementCount determineVPlanVF(const TargetTransformInfo &TTI,
+                                     LoopVectorizationCostModel &CM) {
   unsigned WidestType;
   std::tie(std::ignore, WidestType) = CM.getSmallestAndWidestTypes();
-  return WidestVectorRegBits / WidestType;
+
+  TargetTransformInfo::RegisterKind RegKind =
+      TTI.enableScalableVectorization()
+          ? TargetTransformInfo::RGK_ScalableVector
+          : TargetTransformInfo::RGK_FixedWidthVector;
+
+  TypeSize RegSize = TTI.getRegisterBitWidth(RegKind);
+  unsigned N = RegSize.getKnownMinValue() / WidestType;
+  return ElementCount::get(N, RegSize.isScalable());
 }
 
 VectorizationFactor
 LoopVectorizationPlanner::planInVPlanNativePath(ElementCount UserVF) {
-  assert(!UserVF.isScalable() && "scalable vectors not yet supported");
   ElementCount VF = UserVF;
   // Outer loop handling: They may require CFG and instruction level
   // transformations before even evaluating whether vectorization is profitable.
@@ -7503,10 +7466,7 @@ LoopVectorizationPlanner::planInVPlanNativePath(ElementCount UserVF) {
     // If the user doesn't provide a vectorization factor, determine a
     // reasonable one.
     if (UserVF.isZero()) {
-      VF = ElementCount::getFixed(determineVPlanVF(
-          TTI.getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector)
-              .getFixedValue(),
-          CM));
+      VF = determineVPlanVF(TTI, CM);
       LLVM_DEBUG(dbgs() << "LV: VPlan computed VF " << VF << ".\n");
 
       // Make sure we have a VF > 1 for stress testing.
@@ -7515,6 +7475,17 @@ LoopVectorizationPlanner::planInVPlanNativePath(ElementCount UserVF) {
                           << "overriding computed VF.\n");
         VF = ElementCount::getFixed(4);
       }
+    } else if (UserVF.isScalable() && !TTI.supportsScalableVectors() &&
+               !ForceTargetSupportsScalableVectors) {
+      LLVM_DEBUG(dbgs() << "LV: Not vectorizing. Scalable VF requested, but "
+                        << "not supported by the target.\n");
+      reportVectorizationFailure(
+          "Scalable vectorization requested but not supported by the target",
+          "the scalable user-specified vectorization width for outer-loop "
+          "vectorization cannot be used because the target does not support "
+          "scalable vectors.",
+          "ScalableVFUnfeasible", ORE, OrigLoop);
+      return VectorizationFactor::Disabled();
     }
     assert(EnableVPlanNativePath && "VPlan-native path is not enabled.");
     assert(isPowerOf2_32(VF.getKnownMinValue()) &&
@@ -7673,7 +7644,7 @@ static void AddRuntimeUnrollDisableMetaData(Loop *L) {
 SCEV2ValueTy LoopVectorizationPlanner::executePlan(
     ElementCount BestVF, unsigned BestUF, VPlan &BestVPlan,
     InnerLoopVectorizer &ILV, DominatorTree *DT, bool IsEpilogueVectorization,
-    DenseMap<const SCEV *, Value *> *ExpandedSCEVs) {
+    const DenseMap<const SCEV *, Value *> *ExpandedSCEVs) {
   assert(BestVPlan.hasVF(BestVF) &&
          "Trying to execute plan with unsupported VF");
   assert(BestVPlan.hasUF(BestUF) &&
@@ -8918,8 +8889,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
   // ---------------------------------------------------------------------------
 
   // Adjust the recipes for any inloop reductions.
-  adjustRecipesForReductions(cast<VPBasicBlock>(TopRegion->getExiting()), Plan,
-                             RecipeBuilder, Range.Start);
+  adjustRecipesForReductions(LatchVPBB, Plan, RecipeBuilder, Range.Start);
 
   // Interleave memory: for each Interleave Group we marked earlier as relevant
   // for this VPlan, replace the Recipes widening its memory instructions with a
@@ -9041,9 +9011,48 @@ VPlanPtr LoopVectorizationPlanner::buildVPlan(VFRange &Range) {
 void LoopVectorizationPlanner::adjustRecipesForReductions(
     VPBasicBlock *LatchVPBB, VPlanPtr &Plan, VPRecipeBuilder &RecipeBuilder,
     ElementCount MinVF) {
+  VPBasicBlock *Header = Plan->getVectorLoopRegion()->getEntryBasicBlock();
+  // Gather all VPReductionPHIRecipe and sort them so that Intermediate stores
+  // sank outside of the loop would keep the same order as they had in the
+  // original loop.
+  SmallVector<VPReductionPHIRecipe *> ReductionPHIList;
+  for (VPRecipeBase &R : Header->phis()) {
+    if (auto *ReductionPhi = dyn_cast<VPReductionPHIRecipe>(&R))
+      ReductionPHIList.emplace_back(ReductionPhi);
+  }
+  bool HasIntermediateStore = false;
+  stable_sort(ReductionPHIList,
+              [this, &HasIntermediateStore](const VPReductionPHIRecipe *R1,
+                                            const VPReductionPHIRecipe *R2) {
+                auto *IS1 = R1->getRecurrenceDescriptor().IntermediateStore;
+                auto *IS2 = R2->getRecurrenceDescriptor().IntermediateStore;
+                HasIntermediateStore |= IS1 || IS2;
+
+                // If neither of the recipes has an intermediate store, keep the
+                // order the same.
+                if (!IS1 && !IS2)
+                  return false;
+
+                // If only one of the recipes has an intermediate store, then
+                // move it towards the beginning of the list.
+                if (IS1 && !IS2)
+                  return true;
+
+                if (!IS1 && IS2)
+                  return false;
+
+                // If both recipes have an intermediate store, then the recipe
+                // with the later store should be processed earlier. So it
+                // should go to the beginning of the list.
+                return DT->dominates(IS2, IS1);
+              });
+
+  if (HasIntermediateStore && ReductionPHIList.size() > 1)
+    for (VPRecipeBase *R : ReductionPHIList)
+      R->moveBefore(*Header, Header->getFirstNonPhi());
+
   SmallVector<VPReductionPHIRecipe *> InLoopReductionPhis;
-  for (VPRecipeBase &R :
-       Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
+  for (VPRecipeBase &R : Header->phis()) {
     auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
     if (!PhiR || !PhiR->isInLoop() || (MinVF.isScalar() && !PhiR->isOrdered()))
       continue;
@@ -9155,18 +9164,19 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
       PreviousLink = RedRecipe;
     }
   }
-
-  // If tail is folded by masking, introduce selects between the phi
-  // and the live-out instruction of each reduction, at the beginning of the
-  // dedicated latch block.
-  if (CM.foldTailByMasking()) {
     Builder.setInsertPoint(&*LatchVPBB->begin());
     for (VPRecipeBase &R :
          Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
-      VPReductionPHIRecipe *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
-      if (!PhiR || PhiR->isInLoop())
-        continue;
-      const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor();
+    VPReductionPHIRecipe *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
+    if (!PhiR || PhiR->isInLoop())
+      continue;
+
+    const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor();
+    auto *Result = PhiR->getBackedgeValue()->getDefiningRecipe();
+    // If tail is folded by masking, introduce selects between the phi
+    // and the live-out instruction of each reduction, at the beginning of the
+    // dedicated latch block.
+    if (CM.foldTailByMasking()) {
       VPValue *Cond =
           RecipeBuilder.createBlockInMask(OrigLoop->getHeader(), *Plan);
       VPValue *Red = PhiR->getBackedgeValue();
@@ -9174,16 +9184,35 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
              "reduction recipe must be defined before latch");
       FastMathFlags FMFs = RdxDesc.getFastMathFlags();
       Type *PhiTy = PhiR->getOperand(0)->getLiveInIRValue()->getType();
-      auto *Select =
+      Result =
           PhiTy->isFloatingPointTy()
               ? new VPInstruction(Instruction::Select, {Cond, Red, PhiR}, FMFs)
               : new VPInstruction(Instruction::Select, {Cond, Red, PhiR});
-      Select->insertBefore(&*Builder.getInsertPoint());
+      Result->insertBefore(&*Builder.getInsertPoint());
       if (PreferPredicatedReductionSelect ||
           TTI.preferPredicatedReductionSelect(
               PhiR->getRecurrenceDescriptor().getOpcode(), PhiTy,
               TargetTransformInfo::ReductionFlags()))
-        PhiR->setOperand(1, Select);
+        PhiR->setOperand(1, Result->getVPSingleValue());
+    }
+    // If the vector reduction can be performed in a smaller type, we truncate
+    // then extend the loop exit value to enable InstCombine to evaluate the
+    // entire expression in the smaller type.
+    Type *PhiTy = PhiR->getStartValue()->getLiveInIRValue()->getType();
+    if (MinVF.isVector() && PhiTy != RdxDesc.getRecurrenceType()) {
+      assert(!PhiR->isInLoop() && "Unexpected truncated inloop reduction!");
+      Type *RdxTy = RdxDesc.getRecurrenceType();
+      auto *Trunc = new VPWidenCastRecipe(Instruction::Trunc,
+                                          Result->getVPSingleValue(), RdxTy);
+      auto *Extnd =
+          RdxDesc.isSigned()
+              ? new VPWidenCastRecipe(Instruction::SExt, Trunc, PhiTy)
+              : new VPWidenCastRecipe(Instruction::ZExt, Trunc, PhiTy);
+
+      Trunc->insertAfter(Result);
+      Extnd->insertAfter(Trunc);
+      Result->getVPSingleValue()->replaceAllUsesWith(Extnd);
+      Trunc->setOperand(0, Result->getVPSingleValue());
     }
   }
 
@@ -9662,7 +9691,8 @@ static bool processLoopInVPlanNativePath(
   // Use the planner for outer loop vectorization.
   // TODO: CM is not used at this point inside the planner. Turn CM into an
   // optional argument if we don't need it in the future.
-  LoopVectorizationPlanner LVP(L, LI, TLI, *TTI, LVL, CM, IAI, PSE, Hints, ORE);
+  LoopVectorizationPlanner LVP(L, LI, DT, TLI, *TTI, LVL, CM, IAI, PSE, Hints,
+                               ORE);
 
   // Get user vectorization factor.
   ElementCount UserVF = Hints.getWidth();
@@ -10004,7 +10034,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
   LoopVectorizationCostModel CM(SEL, L, PSE, LI, &LVL, *TTI, TLI, DB, AC, ORE,
                                 F, &Hints, IAI);
   // Use the planner for vectorization.
-  LoopVectorizationPlanner LVP(L, LI, TLI, *TTI, &LVL, CM, IAI, PSE, Hints,
+  LoopVectorizationPlanner LVP(L, LI, DT, TLI, *TTI, &LVL, CM, IAI, PSE, Hints,
                                ORE);
 
   // Get user vectorization factor and interleave count.
@@ -10340,8 +10370,14 @@ LoopVectorizeResult LoopVectorizePass::runImpl(
 
     Changed |= CFGChanged |= processLoop(L);
 
-    if (Changed)
+    if (Changed) {
       LAIs->clear();
+
+#ifndef NDEBUG
+      if (VerifySCEV)
+        SE->verify();
+#endif
+    }
   }
 
   // Process each loop nest in the function.
@@ -10389,10 +10425,6 @@ PreservedAnalyses LoopVectorizePass::run(Function &F,
       PA.preserve<LoopAnalysis>();
       PA.preserve<DominatorTreeAnalysis>();
       PA.preserve<ScalarEvolutionAnalysis>();
-
-#ifdef EXPENSIVE_CHECKS
-      SE.verify();
-#endif
     }
 
     if (Result.MadeCFGChange) {
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index a010ffb3c9..f694ab88cc 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -108,8 +108,9 @@ using namespace slpvectorizer;
 
 STATISTIC(NumVectorInstructions, "Number of vector instructions generated");
 
-cl::opt<bool> RunSLPVectorization("vectorize-slp", cl::init(true), cl::Hidden,
-                                  cl::desc("Run the SLP vectorization passes"));
+static cl::opt<bool>
+    RunSLPVectorization("vectorize-slp", cl::init(true), cl::Hidden,
+                        cl::desc("Run the SLP vectorization passes"));
 
 static cl::opt<int>
     SLPCostThreshold("slp-threshold", cl::init(0), cl::Hidden,
@@ -552,8 +553,8 @@ static std::optional<unsigned> getExtractIndex(Instruction *E) {
 /// successful, the matched scalars are replaced by poison values in \p VL for
 /// future analysis.
 static std::optional<TTI::ShuffleKind>
-tryToGatherExtractElements(SmallVectorImpl<Value *> &VL,
-                           SmallVectorImpl<int> &Mask) {
+tryToGatherSingleRegisterExtractElements(MutableArrayRef<Value *> VL,
+                                         SmallVectorImpl<int> &Mask) {
   // Scan list of gathered scalars for extractelements that can be represented
   // as shuffles.
   MapVector<Value *, SmallVector<int>> VectorOpToIdx;
@@ -640,7 +641,7 @@ tryToGatherExtractElements(SmallVectorImpl<Value *> &VL,
   if (!Res) {
     // TODO: try to check other subsets if possible.
     // Restore the original VL if attempt was not successful.
-    VL.swap(SavedVL);
+    copy(SavedVL, VL.begin());
     return std::nullopt;
   }
   // Restore unused scalars from mask, if some of the extractelements were not
@@ -2476,11 +2477,15 @@ class BoUpSLP {
                        bool ResizeAllowed = false) const;
 
   /// Vectorize a single entry in the tree.
-  Value *vectorizeTree(TreeEntry *E);
+  /// \param PostponedPHIs true, if need to postpone emission of phi nodes to
+  /// avoid issues with def-use order.
+  Value *vectorizeTree(TreeEntry *E, bool PostponedPHIs);
 
   /// Vectorize a single entry in the tree, the \p Idx-th operand of the entry
   /// \p E.
-  Value *vectorizeOperand(TreeEntry *E, unsigned NodeIdx);
+  /// \param PostponedPHIs true, if need to postpone emission of phi nodes to
+  /// avoid issues with def-use order.
+  Value *vectorizeOperand(TreeEntry *E, unsigned NodeIdx, bool PostponedPHIs);
 
   /// Create a new vector from a list of scalar values.  Produces a sequence
   /// which exploits values reused across lanes, and arranges the inserts
@@ -2643,6 +2648,9 @@ class BoUpSLP {
     /// The Scalars are vectorized into this value. It is initialized to Null.
     WeakTrackingVH VectorizedValue = nullptr;
 
+    /// New vector phi instructions emitted for the vectorized phi nodes.
+    PHINode *PHI = nullptr;
+
     /// Do we need to gather this sequence or vectorize it
     /// (either with vector instruction or with scatter/gather
     /// intrinsics for store/load)?
@@ -7608,7 +7616,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
     std::optional<TargetTransformInfo::ShuffleKind> GatherShuffle;
     SmallVector<const TreeEntry *> Entries;
     // Check for gathered extracts.
-    ExtractShuffle = tryToGatherExtractElements(GatheredScalars, ExtractMask);
+    ExtractShuffle = tryToGatherSingleRegisterExtractElements(GatheredScalars, ExtractMask);
     SmallVector<Value *> IgnoredVals;
     if (UserIgnoreList)
       IgnoredVals.assign(UserIgnoreList->begin(), UserIgnoreList->end());
@@ -9028,41 +9036,45 @@ BoUpSLP::isGatherShuffledEntry(const TreeEntry *TE, ArrayRef<Value *> VL,
          "Expected only single user of the gather node.");
   // TODO: currently checking only for Scalars in the tree entry, need to count
   // reused elements too for better cost estimation.
-  Instruction &UserInst =
-      getLastInstructionInBundle(TE->UserTreeIndices.front().UserTE);
-  BasicBlock *ParentBB = nullptr;
+  const EdgeInfo &TEUseEI = TE->UserTreeIndices.front();
+  const Instruction *TEInsertPt = &getLastInstructionInBundle(TEUseEI.UserTE);
+  const BasicBlock *TEInsertBlock = nullptr;
   // Main node of PHI entries keeps the correct order of operands/incoming
   // blocks.
-  if (auto *PHI =
-          dyn_cast<PHINode>(TE->UserTreeIndices.front().UserTE->getMainOp())) {
-    ParentBB = PHI->getIncomingBlock(TE->UserTreeIndices.front().EdgeIdx);
+  if (auto *PHI = dyn_cast<PHINode>(TEUseEI.UserTE->getMainOp())) {
+    TEInsertBlock = PHI->getIncomingBlock(TEUseEI.EdgeIdx);
   } else {
-    ParentBB = UserInst.getParent();
+    TEInsertBlock = TEInsertPt->getParent();
   }
-  auto *NodeUI = DT->getNode(ParentBB);
+  auto *NodeUI = DT->getNode(TEInsertBlock);
   assert(NodeUI && "Should only process reachable instructions");
   SmallPtrSet<Value *, 4> GatheredScalars(VL.begin(), VL.end());
-  auto CheckOrdering = [&](Instruction *LastEI) {
-    // Check if the user node of the TE comes after user node of EntryPtr,
-    // otherwise EntryPtr depends on TE.
-    // Gather nodes usually are not scheduled and inserted before their first
-    // user node. So, instead of checking dependency between the gather nodes
-    // themselves, we check the dependency between their user nodes.
-    // If one user node comes before the second one, we cannot use the second
-    // gather node as the source vector for the first gather node, because in
-    // the list of instructions it will be emitted later.
-    auto *EntryParent = LastEI->getParent();
-    auto *NodeEUI = DT->getNode(EntryParent);
+  auto CheckOrdering = [&](const Instruction *InsertPt) {
+    // Argument InsertPt is an instruction where vector code for some other
+    // tree entry (one that shares one or more scalars with TE) is going to be
+    // generated. This lambda returns true if insertion point of vector code
+    // for the TE dominates that point (otherwise dependency is the other way
+    // around). The other node is not limited to be of a gather kind. Gather
+    // nodes are not scheduled and their vector code is inserted before their
+    // first user. If user is PHI, that is supposed to be at the end of a
+    // predecessor block. Otherwise it is the last instruction among scalars of
+    // the user node. So, instead of checking dependency between instructions
+    // themselves, we check dependency between their insertion points for vector
+    // code (since each scalar instruction ends up as a lane of a vector
+    // instruction).
+    const BasicBlock *InsertBlock = InsertPt->getParent();
+    auto *NodeEUI = DT->getNode(InsertBlock);
     if (!NodeEUI)
       return false;
     assert((NodeUI == NodeEUI) ==
                (NodeUI->getDFSNumIn() == NodeEUI->getDFSNumIn()) &&
            "Different nodes should have different DFS numbers");
     // Check the order of the gather nodes users.
-    if (UserInst.getParent() != EntryParent &&
+    if (TEInsertPt->getParent() != InsertBlock &&
         (DT->dominates(NodeUI, NodeEUI) || !DT->dominates(NodeEUI, NodeUI)))
       return false;
-    if (UserInst.getParent() == EntryParent && UserInst.comesBefore(LastEI))
+    if (TEInsertPt->getParent() == InsertBlock &&
+        TEInsertPt->comesBefore(InsertPt))
       return false;
     return true;
   };
@@ -9087,49 +9099,35 @@ BoUpSLP::isGatherShuffledEntry(const TreeEntry *TE, ArrayRef<Value *> VL,
                     [&](Value *V) { return GatheredScalars.contains(V); }) &&
              "Must contain at least single gathered value.");
       assert(TEPtr->UserTreeIndices.size() == 1 &&
-             "Expected only single user of the gather node.");
-      PHINode *EntryPHI =
-          dyn_cast<PHINode>(TEPtr->UserTreeIndices.front().UserTE->getMainOp());
-      Instruction *EntryUserInst =
-          EntryPHI ? nullptr
-                   : &getLastInstructionInBundle(
-                         TEPtr->UserTreeIndices.front().UserTE);
-      if (&UserInst == EntryUserInst) {
-        assert(!EntryPHI && "Unexpected phi node entry.");
-        // If 2 gathers are operands of the same entry, compare operands
-        // indices, use the earlier one as the base.
-        if (TE->UserTreeIndices.front().UserTE ==
-                TEPtr->UserTreeIndices.front().UserTE &&
-            TE->UserTreeIndices.front().EdgeIdx <
-                TEPtr->UserTreeIndices.front().EdgeIdx)
+             "Expected only single user of a gather node.");
+      const EdgeInfo &UseEI = TEPtr->UserTreeIndices.front();
+
+      PHINode *UserPHI = dyn_cast<PHINode>(UseEI.UserTE->getMainOp());
+      const Instruction *InsertPt =
+          UserPHI ? UserPHI->getIncomingBlock(UseEI.EdgeIdx)->getTerminator()
+                  : &getLastInstructionInBundle(UseEI.UserTE);
+      if (!UserPHI && TEInsertPt == InsertPt) {
+        // If 2 gathers are operands of the same non-PHI entry,
+        // compare operands indices, use the earlier one as the base.
+        if (TEUseEI.UserTE == UseEI.UserTE && TEUseEI.EdgeIdx < UseEI.EdgeIdx)
           continue;
         // If the user instruction is used for some reason in different
         // vectorized nodes - make it depend on index.
-        if (TE->UserTreeIndices.front().UserTE !=
-                TEPtr->UserTreeIndices.front().UserTE &&
-            TE->Idx < TEPtr->Idx)
+        if (TEUseEI.UserTE != UseEI.UserTE && TE->Idx < TEPtr->Idx)
           continue;
       }
-      // Check if the user node of the TE comes after user node of EntryPtr,
-      // otherwise EntryPtr depends on TE.
-      auto *EntryI =
-          EntryPHI
-              ? EntryPHI
-                    ->getIncomingBlock(TEPtr->UserTreeIndices.front().EdgeIdx)
-                    ->getTerminator()
-              : EntryUserInst;
-      if ((ParentBB != EntryI->getParent() ||
-           TE->UserTreeIndices.front().EdgeIdx <
-               TEPtr->UserTreeIndices.front().EdgeIdx ||
-           TE->UserTreeIndices.front().UserTE !=
-               TEPtr->UserTreeIndices.front().UserTE) &&
-          !CheckOrdering(EntryI))
+
+      // Check if the user node of the TE comes after user node of TEPtr,
+      // otherwise TEPtr depends on TE.
+      if ((TEInsertBlock != InsertPt->getParent() ||
+           TEUseEI.EdgeIdx < UseEI.EdgeIdx || TEUseEI.UserTE != UseEI.UserTE) &&
+          !CheckOrdering(InsertPt))
         continue;
       VToTEs.insert(TEPtr);
     }
     if (const TreeEntry *VTE = getTreeEntry(V)) {
-      Instruction &EntryUserInst = getLastInstructionInBundle(VTE);
-      if (&EntryUserInst == &UserInst || !CheckOrdering(&EntryUserInst))
+      Instruction &LastBundleInst = getLastInstructionInBundle(VTE);
+      if (&LastBundleInst == TEInsertPt || !CheckOrdering(&LastBundleInst))
         continue;
       VToTEs.insert(VTE);
     }
@@ -9990,7 +9988,8 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
   }
 };
 
-Value *BoUpSLP::vectorizeOperand(TreeEntry *E, unsigned NodeIdx) {
+Value *BoUpSLP::vectorizeOperand(TreeEntry *E, unsigned NodeIdx,
+                                 bool PostponedPHIs) {
   ValueList &VL = E->getOperand(NodeIdx);
   if (E->State == TreeEntry::PossibleStridedVectorize &&
       !E->ReorderIndices.empty()) {
@@ -10039,7 +10038,7 @@ Value *BoUpSLP::vectorizeOperand(TreeEntry *E, unsigned NodeIdx) {
         ShuffleBuilder.add(V, Mask);
         return ShuffleBuilder.finalize(std::nullopt);
       };
-      Value *V = vectorizeTree(VE);
+      Value *V = vectorizeTree(VE, PostponedPHIs);
       if (VF != cast<FixedVectorType>(V->getType())->getNumElements()) {
         if (!VE->ReuseShuffleIndices.empty()) {
           // Reshuffle to get only unique values.
@@ -10112,14 +10111,7 @@ Value *BoUpSLP::vectorizeOperand(TreeEntry *E, unsigned NodeIdx) {
   assert(I->get()->UserTreeIndices.size() == 1 &&
          "Expected only single user for the gather node.");
   assert(I->get()->isSame(VL) && "Expected same list of scalars.");
-  IRBuilder<>::InsertPointGuard Guard(Builder);
-  if (E->getOpcode() != Instruction::InsertElement &&
-      E->getOpcode() != Instruction::PHI) {
-    Instruction *LastInst = &getLastInstructionInBundle(E);
-    assert(LastInst && "Failed to find last instruction in bundle");
-    Builder.SetInsertPoint(LastInst->getParent(), LastInst->getIterator());
-  }
-  return vectorizeTree(I->get());
+  return vectorizeTree(I->get(), PostponedPHIs);
 }
 
 template <typename BVTy, typename ResTy, typename... Args>
@@ -10137,7 +10129,7 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Args &...Params) {
   inversePermutation(E->ReorderIndices, ReorderMask);
   if (!ReorderMask.empty())
     reorderScalars(GatheredScalars, ReorderMask);
-  auto FindReusedSplat = [&](SmallVectorImpl<int> &Mask) {
+  auto FindReusedSplat = [&](MutableArrayRef<int> Mask) {
     if (!isSplat(E->Scalars) || none_of(E->Scalars, [](Value *V) {
           return isa<UndefValue>(V) && !isa<PoisonValue>(V);
         }))
@@ -10172,7 +10164,8 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Args &...Params) {
   Type *ScalarTy = GatheredScalars.front()->getType();
   if (!all_of(GatheredScalars, UndefValue::classof)) {
     // Check for gathered extracts.
-    ExtractShuffle = tryToGatherExtractElements(GatheredScalars, ExtractMask);
+    ExtractShuffle =
+        tryToGatherSingleRegisterExtractElements(GatheredScalars, ExtractMask);
     SmallVector<Value *> IgnoredVals;
     if (UserIgnoreList)
       IgnoredVals.assign(UserIgnoreList->begin(), UserIgnoreList->end());
@@ -10479,10 +10472,12 @@ Value *BoUpSLP::createBuildVector(const TreeEntry *E) {
                                                                 *this);
 }
 
-Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
+Value *BoUpSLP::vectorizeTree(TreeEntry *E, bool PostponedPHIs) {
   IRBuilder<>::InsertPointGuard Guard(Builder);
 
-  if (E->VectorizedValue) {
+  if (E->VectorizedValue &&
+      (E->State != TreeEntry::Vectorize || E->getOpcode() != Instruction::PHI ||
+       E->isAltShuffle())) {
     LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *E->Scalars[0] << ".\n");
     return E->VectorizedValue;
   }
@@ -10529,21 +10524,32 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
               E != VectorizableTree.front().get() ||
               !E->UserTreeIndices.empty()) &&
              "PHI reordering is free.");
+      if (PostponedPHIs && E->VectorizedValue)
+        return E->VectorizedValue;
       auto *PH = cast<PHINode>(VL0);
       Builder.SetInsertPoint(PH->getParent(),
                              PH->getParent()->getFirstNonPHIIt());
       Builder.SetCurrentDebugLocation(PH->getDebugLoc());
-      PHINode *NewPhi = Builder.CreatePHI(VecTy, PH->getNumIncomingValues());
-      Value *V = NewPhi;
-
-      // Adjust insertion point once all PHI's have been generated.
-      Builder.SetInsertPoint(PH->getParent(),
-                             PH->getParent()->getFirstInsertionPt());
-      Builder.SetCurrentDebugLocation(PH->getDebugLoc());
+      if (PostponedPHIs || !E->VectorizedValue) {
+        PHINode *NewPhi = Builder.CreatePHI(VecTy, PH->getNumIncomingValues());
+        E->PHI = NewPhi;
+        Value *V = NewPhi;
+
+        // Adjust insertion point once all PHI's have been generated.
+        Builder.SetInsertPoint(PH->getParent(),
+                               PH->getParent()->getFirstInsertionPt());
+        Builder.SetCurrentDebugLocation(PH->getDebugLoc());
 
-      V = FinalShuffle(V, E);
+        V = FinalShuffle(V, E);
 
-      E->VectorizedValue = V;
+        E->VectorizedValue = V;
+        if (PostponedPHIs)
+          return V;
+      }
+      PHINode *NewPhi = cast<PHINode>(E->PHI);
+      // If phi node is fully emitted - exit.
+      if (NewPhi->getNumIncomingValues() != 0)
+        return NewPhi;
 
       // PHINodes may have multiple entries from the same block. We want to
       // visit every block once.
@@ -10556,7 +10562,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
         // Stop emission if all incoming values are generated.
         if (NewPhi->getNumIncomingValues() == PH->getNumIncomingValues()) {
           LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
-          return V;
+          return NewPhi;
         }
 
         if (!VisitedBBs.insert(IBB).second) {
@@ -10566,13 +10572,13 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
 
         Builder.SetInsertPoint(IBB->getTerminator());
         Builder.SetCurrentDebugLocation(PH->getDebugLoc());
-        Value *Vec = vectorizeOperand(E, i);
+        Value *Vec = vectorizeOperand(E, i, /*PostponedPHIs=*/true);
         NewPhi->addIncoming(Vec, IBB);
       }
 
       assert(NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() &&
              "Invalid number of incoming values");
-      return V;
+      return NewPhi;
     }
 
     case Instruction::ExtractElement: {
@@ -10595,7 +10601,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
     case Instruction::InsertElement: {
       assert(E->ReuseShuffleIndices.empty() && "All inserts should be unique");
       Builder.SetInsertPoint(cast<Instruction>(E->Scalars.back()));
-      Value *V = vectorizeOperand(E, 1);
+      Value *V = vectorizeOperand(E, 1, PostponedPHIs);
 
       // Create InsertVector shuffle if necessary
       auto *FirstInsert = cast<Instruction>(*find_if(E->Scalars, [E](Value *V) {
@@ -10753,7 +10759,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
     case Instruction::BitCast: {
       setInsertPointAfterBundle(E);
 
-      Value *InVec = vectorizeOperand(E, 0);
+      Value *InVec = vectorizeOperand(E, 0, PostponedPHIs);
       if (E->VectorizedValue) {
         LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
         return E->VectorizedValue;
@@ -10771,12 +10777,12 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
     case Instruction::ICmp: {
       setInsertPointAfterBundle(E);
 
-      Value *L = vectorizeOperand(E, 0);
+      Value *L = vectorizeOperand(E, 0, PostponedPHIs);
       if (E->VectorizedValue) {
         LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
         return E->VectorizedValue;
       }
-      Value *R = vectorizeOperand(E, 1);
+      Value *R = vectorizeOperand(E, 1, PostponedPHIs);
       if (E->VectorizedValue) {
         LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
         return E->VectorizedValue;
@@ -10794,17 +10800,17 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
     case Instruction::Select: {
       setInsertPointAfterBundle(E);
 
-      Value *Cond = vectorizeOperand(E, 0);
+      Value *Cond = vectorizeOperand(E, 0, PostponedPHIs);
       if (E->VectorizedValue) {
         LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
         return E->VectorizedValue;
       }
-      Value *True = vectorizeOperand(E, 1);
+      Value *True = vectorizeOperand(E, 1, PostponedPHIs);
       if (E->VectorizedValue) {
         LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
         return E->VectorizedValue;
       }
-      Value *False = vectorizeOperand(E, 2);
+      Value *False = vectorizeOperand(E, 2, PostponedPHIs);
       if (E->VectorizedValue) {
         LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
         return E->VectorizedValue;
@@ -10820,7 +10826,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
     case Instruction::FNeg: {
       setInsertPointAfterBundle(E);
 
-      Value *Op = vectorizeOperand(E, 0);
+      Value *Op = vectorizeOperand(E, 0, PostponedPHIs);
 
       if (E->VectorizedValue) {
         LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
@@ -10860,12 +10866,12 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
     case Instruction::Xor: {
       setInsertPointAfterBundle(E);
 
-      Value *LHS = vectorizeOperand(E, 0);
+      Value *LHS = vectorizeOperand(E, 0, PostponedPHIs);
       if (E->VectorizedValue) {
         LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
         return E->VectorizedValue;
       }
-      Value *RHS = vectorizeOperand(E, 1);
+      Value *RHS = vectorizeOperand(E, 1, PostponedPHIs);
       if (E->VectorizedValue) {
         LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
         return E->VectorizedValue;
@@ -10910,7 +10916,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
         assert((E->State == TreeEntry::ScatterVectorize ||
                 E->State == TreeEntry::PossibleStridedVectorize) &&
                "Unhandled state");
-        Value *VecPtr = vectorizeOperand(E, 0);
+        Value *VecPtr = vectorizeOperand(E, 0, PostponedPHIs);
         if (E->VectorizedValue) {
           LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
           return E->VectorizedValue;
@@ -10934,7 +10940,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
 
       setInsertPointAfterBundle(E);
 
-      Value *VecValue = vectorizeOperand(E, 0);
+      Value *VecValue = vectorizeOperand(E, 0, PostponedPHIs);
       VecValue = FinalShuffle(VecValue, E);
 
       Value *Ptr = SI->getPointerOperand();
@@ -10962,7 +10968,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
       auto *GEP0 = cast<GetElementPtrInst>(VL0);
       setInsertPointAfterBundle(E);
 
-      Value *Op0 = vectorizeOperand(E, 0);
+      Value *Op0 = vectorizeOperand(E, 0, PostponedPHIs);
       if (E->VectorizedValue) {
         LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
         return E->VectorizedValue;
@@ -10970,7 +10976,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
 
       SmallVector<Value *> OpVecs;
       for (int J = 1, N = GEP0->getNumOperands(); J < N; ++J) {
-        Value *OpVec = vectorizeOperand(E, J);
+        Value *OpVec = vectorizeOperand(E, J, PostponedPHIs);
         if (E->VectorizedValue) {
           LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
           return E->VectorizedValue;
@@ -11029,7 +11035,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
           continue;
         }
 
-        Value *OpVec = vectorizeOperand(E, j);
+        Value *OpVec = vectorizeOperand(E, j, PostponedPHIs);
         if (E->VectorizedValue) {
           LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
           return E->VectorizedValue;
@@ -11086,15 +11092,15 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
       Value *LHS = nullptr, *RHS = nullptr;
       if (Instruction::isBinaryOp(E->getOpcode()) || isa<CmpInst>(VL0)) {
         setInsertPointAfterBundle(E);
-        LHS = vectorizeOperand(E, 0);
+        LHS = vectorizeOperand(E, 0, PostponedPHIs);
         if (E->VectorizedValue) {
           LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
           return E->VectorizedValue;
         }
-        RHS = vectorizeOperand(E, 1);
+        RHS = vectorizeOperand(E, 1, PostponedPHIs);
       } else {
         setInsertPointAfterBundle(E);
-        LHS = vectorizeOperand(E, 0);
+        LHS = vectorizeOperand(E, 0, PostponedPHIs);
       }
       if (E->VectorizedValue) {
         LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
@@ -11196,7 +11202,14 @@ Value *BoUpSLP::vectorizeTree(
   else
     Builder.SetInsertPoint(&F->getEntryBlock(), F->getEntryBlock().begin());
 
-  auto *VectorRoot = vectorizeTree(VectorizableTree[0].get());
+  // Postpone emission of PHIs operands to avoid cyclic dependencies issues.
+  auto *VectorRoot =
+      vectorizeTree(VectorizableTree[0].get(), /*PostponedPHIs=*/true);
+  for (const std::unique_ptr<TreeEntry> &TE : VectorizableTree)
+    if (TE->State == TreeEntry::Vectorize &&
+        TE->getOpcode() == Instruction::PHI && !TE->isAltShuffle() &&
+        TE->VectorizedValue)
+      (void)vectorizeTree(TE.get(), /*PostponedPHIs=*/false);
   // Run through the list of postponed gathers and emit them, replacing the temp
   // emitted allocas with actual vector instructions.
   ArrayRef<const TreeEntry *> PostponedNodes = PostponedGathers.getArrayRef();
@@ -11215,7 +11228,7 @@ Value *BoUpSLP::vectorizeTree(
         cast<Instruction>(TE->UserTreeIndices.front().UserTE->VectorizedValue);
     Builder.SetInsertPoint(PrevVec);
     Builder.SetCurrentDebugLocation(UserI->getDebugLoc());
-    Value *Vec = vectorizeTree(TE);
+    Value *Vec = vectorizeTree(TE, /*PostponedPHIs=*/false);
     PrevVec->replaceAllUsesWith(Vec);
     PostponedValues.try_emplace(Vec).first->second.push_back(TE);
     // Replace the stub vector node, if it was used before for one of the
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index b4d464ad7a..583f652eed 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -19,7 +19,6 @@
 #include "VPlan.h"
 #include "VPlanCFG.h"
 #include "VPlanDominatorTree.h"
-#include "llvm/ADT/DepthFirstIterator.h"
 #include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 2a1213a980..efc95c1cd0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1418,7 +1418,8 @@ void VPCanonicalIVPHIRecipe::print(raw_ostream &O, const Twine &Indent,
                                    VPSlotTracker &SlotTracker) const {
   O << Indent << "EMIT ";
   printAsOperand(O, SlotTracker);
-  O << " = CANONICAL-INDUCTION";
+  O << " = CANONICAL-INDUCTION ";
+  printOperands(O, SlotTracker);
 }
 #endif
 
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 5cade16850..f309ca5f90 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -812,6 +812,8 @@ static bool isConstantOne(VPValue *V) {
 static unsigned getOpcodeForRecipe(VPRecipeBase &R) {
   if (auto *WidenR = dyn_cast<VPWidenRecipe>(&R))
     return WidenR->getUnderlyingInstr()->getOpcode();
+  if (auto *WidenC = dyn_cast<VPWidenCastRecipe>(&R))
+    return WidenC->getOpcode();
   if (auto *RepR = dyn_cast<VPReplicateRecipe>(&R))
     return RepR->getUnderlyingInstr()->getOpcode();
   if (auto *VPI = dyn_cast<VPInstruction>(&R))
@@ -819,16 +821,39 @@ static unsigned getOpcodeForRecipe(VPRecipeBase &R) {
   return 0;
 }
 
+/// Return the scalar size in bits for \p VPV if possible.
+static Type *getTypeForVPValue(VPValue *VPV) {
+  // TODO: Replace with VPlan type inference once ready.
+  if (auto *VPC = dyn_cast<VPWidenCastRecipe>(VPV))
+    return VPC->getResultType();
+  auto *UV = VPV->getUnderlyingValue();
+  return UV->getType();
+}
+
 /// Try to simplify recipe \p R.
 static void simplifyRecipe(VPRecipeBase &R) {
-  unsigned Opcode = getOpcodeForRecipe(R);
-  if (Opcode == Instruction::Mul) {
+  switch (getOpcodeForRecipe(R)) {
+  case Instruction::Mul: {
     VPValue *A = R.getOperand(0);
     VPValue *B = R.getOperand(1);
     if (isConstantOne(A))
       return R.getVPSingleValue()->replaceAllUsesWith(B);
     if (isConstantOne(B))
       return R.getVPSingleValue()->replaceAllUsesWith(A);
+    break;
+  }
+  case Instruction::Trunc: {
+    VPRecipeBase *Zext = R.getOperand(0)->getDefiningRecipe();
+    if (!Zext || getOpcodeForRecipe(*Zext) != Instruction::ZExt)
+      break;
+    VPValue *A = Zext->getOperand(0);
+    VPValue *Trunc = R.getVPSingleValue();
+    if (getTypeForVPValue(Trunc) == getTypeForVPValue(A))
+      Trunc->replaceAllUsesWith(A);
+    break;
+  }
+  default:
+    break;
   }
 }
 
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index 64a515270f..5cbcb017f9 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -825,23 +825,32 @@ bool VectorCombine::scalarizeVPIntrinsic(Instruction &I) {
   ElementCount EC = cast<VectorType>(Op0->getType())->getElementCount();
   Value *EVL = VPI.getArgOperand(3);
   const DataLayout &DL = VPI.getModule()->getDataLayout();
-  bool MustHaveNonZeroVL =
-      IntrID == Intrinsic::vp_sdiv || IntrID == Intrinsic::vp_udiv ||
-      IntrID == Intrinsic::vp_srem || IntrID == Intrinsic::vp_urem;
-
-  if (!MustHaveNonZeroVL || isKnownNonZero(EVL, DL, 0, &AC, &VPI, &DT)) {
-    Value *ScalarOp0 = getSplatValue(Op0);
-    Value *ScalarOp1 = getSplatValue(Op1);
-    Value *ScalarVal =
-        ScalarIntrID
-            ? Builder.CreateIntrinsic(VecTy->getScalarType(), *ScalarIntrID,
-                                      {ScalarOp0, ScalarOp1})
-            : Builder.CreateBinOp((Instruction::BinaryOps)(*FunctionalOpcode),
-                                  ScalarOp0, ScalarOp1);
-    replaceValue(VPI, *Builder.CreateVectorSplat(EC, ScalarVal));
-    return true;
-  }
-  return false;
+
+  // If the VP op might introduce UB or poison, we can scalarize it provided
+  // that we know the EVL > 0: If the EVL is zero, then the original VP op
+  // becomes a no-op and thus won't be UB, so make sure we don't introduce UB by
+  // scalarizing it.
+  bool SafeToSpeculate;
+  if (ScalarIntrID)
+    SafeToSpeculate = Intrinsic::getAttributes(I.getContext(), *ScalarIntrID)
+                          .hasFnAttr(Attribute::AttrKind::Speculatable);
+  else
+    SafeToSpeculate = isSafeToSpeculativelyExecuteWithOpcode(
+        *FunctionalOpcode, &VPI, nullptr, &AC, &DT);
+  if (!SafeToSpeculate && !isKnownNonZero(EVL, DL, 0, &AC, &VPI, &DT))
+    return false;
+
+  Value *ScalarOp0 = getSplatValue(Op0);
+  Value *ScalarOp1 = getSplatValue(Op1);
+  Value *ScalarVal =
+      ScalarIntrID
+          ? Builder.CreateIntrinsic(VecTy->getScalarType(), *ScalarIntrID,
+                                    {ScalarOp0, ScalarOp1})
+          : Builder.CreateBinOp((Instruction::BinaryOps)(*FunctionalOpcode),
+                                ScalarOp0, ScalarOp1);
+
+  replaceValue(VPI, *Builder.CreateVectorSplat(EC, ScalarVal));
+  return true;
 }
 
 /// Match a vector binop or compare instruction with at least one inserted
diff --git a/llvm/lib/XRay/FDRTraceWriter.cpp b/llvm/lib/XRay/FDRTraceWriter.cpp
index 2b80740ed4..8e67a8f984 100644
--- a/llvm/lib/XRay/FDRTraceWriter.cpp
+++ b/llvm/lib/XRay/FDRTraceWriter.cpp
@@ -57,7 +57,7 @@ Error writeMetadata(support::endian::Writer &OS, Values &&... Ds) {
 } // namespace
 
 FDRTraceWriter::FDRTraceWriter(raw_ostream &O, const XRayFileHeader &H)
-    : OS(O, support::endianness::native) {
+    : OS(O, llvm::endianness::native) {
   // We need to re-construct a header, by writing the fields we care about for
   // traces, in the format that the runtime would have written.
   uint32_t BitField =
diff --git a/llvm/test/Analysis/BasicAA/pr69096.ll b/llvm/test/Analysis/BasicAA/pr69096.ll
new file mode 100644
index 0000000000..fe8504a030
--- /dev/null
+++ b/llvm/test/Analysis/BasicAA/pr69096.ll
@@ -0,0 +1,31 @@
+; RUN: opt %s -aa-pipeline=basic-aa -passes=aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+
+target datalayout = "p:64:64:64"
+
+; CHECK-LABEL: Function: pr69096
+; %p == %scevgep.i when %a == -1.
+; CHECK: MayAlias:     i8* %p, i16* %scevgep.i
+
+define i32 @pr69096(i16 %a, ptr %p) {
+entry:
+  %0 = load i8, ptr %p, align 2
+  %dec.i = add i8 %0, -1
+  %cmp636.i = icmp eq i16 %a, -1
+  br i1 %cmp636.i, label %for.cond2.for.inc29_crit_edge.i, label %n.exit
+
+for.cond2.for.inc29_crit_edge.i:
+  %conv3.i = zext i16 %a to i64
+  %sub.i.i = shl i64 %conv3.i, 56
+  %sub21.i = shl nuw nsw i64 %conv3.i, 2
+  %1 = getelementptr i8, ptr %p, i64 %sub21.i
+  %2 = getelementptr i8, ptr %1, i64 -262140
+  %3 = getelementptr i8, ptr %2, i64 %sub.i.i
+  %scevgep.i = getelementptr i8, ptr %3, i64 72057594037927936
+  store i16 1285, ptr %scevgep.i, align 2
+  br label %n.exit
+
+n.exit:
+  %4 = load i8, ptr %p, align 2
+  %conv = sext i8 %4 to i32
+  ret i32 %conv
+}
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/fdiv.ll b/llvm/test/Analysis/CostModel/AMDGPU/fdiv.ll
index 11ce416b7f..2830bfcdae 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/fdiv.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/fdiv.ll
@@ -43,6 +43,37 @@ define amdgpu_kernel void @fdiv_f32_ieee() #0 {
   ret void
 }
 
+define amdgpu_kernel void @fdiv_f32_afn_ieee() #0 {
+; ALL-LABEL: 'fdiv_f32_afn_ieee'
+; ALL-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %f32 = fdiv afn float undef, undef
+; ALL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v2f32 = fdiv afn <2 x float> undef, undef
+; ALL-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v3f32 = fdiv afn <3 x float> undef, undef
+; ALL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v4f32 = fdiv afn <4 x float> undef, undef
+; ALL-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v5f32 = fdiv afn <5 x float> undef, undef
+; ALL-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v8f32 = fdiv afn <8 x float> undef, undef
+; ALL-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %v9f32 = fdiv afn <9 x float> undef, undef
+; ALL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'fdiv_f32_afn_ieee'
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %f32 = fdiv afn float undef, undef
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v2f32 = fdiv afn <2 x float> undef, undef
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v3f32 = fdiv afn <3 x float> undef, undef
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v4f32 = fdiv afn <4 x float> undef, undef
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v5f32 = fdiv afn <5 x float> undef, undef
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %v8f32 = fdiv afn <8 x float> undef, undef
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 81 for instruction: %v9f32 = fdiv afn <9 x float> undef, undef
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %f32 = fdiv afn float undef, undef
+  %v2f32 = fdiv afn <2 x float> undef, undef
+  %v3f32 = fdiv afn <3 x float> undef, undef
+  %v4f32 = fdiv afn <4 x float> undef, undef
+  %v5f32 = fdiv afn <5 x float> undef, undef
+  %v8f32 = fdiv afn <8 x float> undef, undef
+  %v9f32 = fdiv afn <9 x float> undef, undef
+  ret void
+}
+
 define amdgpu_kernel void @fdiv_f32_ftzdaz() #1 {
 ; ALL-LABEL: 'fdiv_f32_ftzdaz'
 ; ALL-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %f32 = fdiv float undef, undef
@@ -74,6 +105,37 @@ define amdgpu_kernel void @fdiv_f32_ftzdaz() #1 {
   ret void
 }
 
+define amdgpu_kernel void @fdiv_f32_afn_ftzdaz() #1 {
+; ALL-LABEL: 'fdiv_f32_afn_ftzdaz'
+; ALL-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %f32 = fdiv afn float undef, undef
+; ALL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v2f32 = fdiv afn <2 x float> undef, undef
+; ALL-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v3f32 = fdiv afn <3 x float> undef, undef
+; ALL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v4f32 = fdiv afn <4 x float> undef, undef
+; ALL-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v5f32 = fdiv afn <5 x float> undef, undef
+; ALL-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v8f32 = fdiv afn <8 x float> undef, undef
+; ALL-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %v9f32 = fdiv afn <9 x float> undef, undef
+; ALL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; ALL-SIZE-LABEL: 'fdiv_f32_afn_ftzdaz'
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %f32 = fdiv afn float undef, undef
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v2f32 = fdiv afn <2 x float> undef, undef
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v3f32 = fdiv afn <3 x float> undef, undef
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v4f32 = fdiv afn <4 x float> undef, undef
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v5f32 = fdiv afn <5 x float> undef, undef
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %v8f32 = fdiv afn <8 x float> undef, undef
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 81 for instruction: %v9f32 = fdiv afn <9 x float> undef, undef
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %f32 = fdiv afn float undef, undef
+  %v2f32 = fdiv afn <2 x float> undef, undef
+  %v3f32 = fdiv afn <3 x float> undef, undef
+  %v4f32 = fdiv afn <4 x float> undef, undef
+  %v5f32 = fdiv afn <5 x float> undef, undef
+  %v8f32 = fdiv afn <8 x float> undef, undef
+  %v9f32 = fdiv afn <9 x float> undef, undef
+  ret void
+}
+
 define amdgpu_kernel void @fdiv_f64() #0 {
 ; CIFASTF64-LABEL: 'fdiv_f64'
 ; CIFASTF64-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %f64 = fdiv double undef, undef
diff --git a/llvm/test/Analysis/CostModel/RISCV/fround.ll b/llvm/test/Analysis/CostModel/RISCV/fround.ll
index 84fc4c454b..71dd64d765 100644
--- a/llvm/test/Analysis/CostModel/RISCV/fround.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/fround.ll
@@ -181,6 +181,96 @@ define void @rint() {
   ret void
 }
 
+define void @lrint() {
+; CHECK-LABEL: 'lrint'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = call i64 @llvm.lrint.i64.f32(float undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = call <2 x i64> @llvm.lrint.v2i64.v2f32(<2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = call <4 x i64> @llvm.lrint.v4i64.v4f32(<4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = call <8 x i64> @llvm.lrint.v8i64.v8f32(<8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = call <16 x i64> @llvm.lrint.v16i64.v16f32(<16 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 1 x i64> @llvm.lrint.nxv1i64.nxv1f32(<vscale x 1 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 2 x i64> @llvm.lrint.nxv2i64.nxv2f32(<vscale x 2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 4 x i64> @llvm.lrint.nxv4i64.nxv4f32(<vscale x 4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 8 x i64> @llvm.lrint.nxv8i64.nxv8f32(<vscale x 8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %10 = call <vscale x 16 x i64> @llvm.lrint.nxv16i64.nxv16f32(<vscale x 16 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = call i64 @llvm.lrint.i64.f64(double undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = call <2 x i64> @llvm.lrint.v2i64.v2f64(<2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = call <4 x i64> @llvm.lrint.v4i64.v4f64(<4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = call <8 x i64> @llvm.lrint.v8i64.v8f64(<8 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %15 = call <16 x i64> @llvm.lrint.v16i64.v16f64(<16 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %16 = call <vscale x 1 x i64> @llvm.lrint.nxv1i64.nxv1f64(<vscale x 1 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %17 = call <vscale x 2 x i64> @llvm.lrint.nxv2i64.nxv2f64(<vscale x 2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %18 = call <vscale x 4 x i64> @llvm.lrint.nxv4i64.nxv4f64(<vscale x 4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %19 = call <vscale x 8 x i64> @llvm.lrint.nxv8i64.nxv8f64(<vscale x 8 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  call i64 @llvm.lrint.i64.f32(float undef)
+  call <2 x i64> @llvm.lrint.v2i64.v2f32(<2 x float> undef)
+  call <4 x i64> @llvm.lrint.v4i64.v4f32(<4 x float> undef)
+  call <8 x i64> @llvm.lrint.v8i64.v8f32(<8 x float> undef)
+  call <16 x i64> @llvm.lrint.v16i64.v16f32(<16 x float> undef)
+  call <vscale x 1 x i64> @llvm.lrint.nvx1i64.nvx1f32(<vscale x 1 x float> undef)
+  call <vscale x 2 x i64> @llvm.lrint.nvx2i64.nvx2f32(<vscale x 2 x float> undef)
+  call <vscale x 4 x i64> @llvm.lrint.nvx4i64.nvx4f32(<vscale x 4 x float> undef)
+  call <vscale x 8 x i64> @llvm.lrint.nvx8i64.nvx8f32(<vscale x 8 x float> undef)
+  call <vscale x 16 x i64> @llvm.lrint.nvx16i64.nvx16f32(<vscale x 16 x float> undef)
+  call i64 @llvm.lrint.i64.f64(double undef)
+  call <2 x i64> @llvm.lrint.v2i64.v2f64(<2 x double> undef)
+  call <4 x i64> @llvm.lrint.v4i64.v4f64(<4 x double> undef)
+  call <8 x i64> @llvm.lrint.v8i64.v8f64(<8 x double> undef)
+  call <16 x i64> @llvm.lrint.v16i64.v16f64(<16 x double> undef)
+  call <vscale x 1 x i64> @llvm.lrint.nvx1i64.nvx1f64(<vscale x 1 x double> undef)
+  call <vscale x 2 x i64> @llvm.lrint.nvx2i64.nvx2f64(<vscale x 2 x double> undef)
+  call <vscale x 4 x i64> @llvm.lrint.nvx4i64.nvx4f64(<vscale x 4 x double> undef)
+  call <vscale x 8 x i64> @llvm.lrint.nvx8i64.nvx8f64(<vscale x 8 x double> undef)
+  ret void
+}
+
+define void @llrint() {
+; CHECK-LABEL: 'llrint'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = call i64 @llvm.llrint.i64.f32(float undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = call <2 x i64> @llvm.llrint.v2i64.v2f32(<2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = call <4 x i64> @llvm.llrint.v4i64.v4f32(<4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = call <8 x i64> @llvm.llrint.v8i64.v8f32(<8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = call <16 x i64> @llvm.llrint.v16i64.v16f32(<16 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = call <vscale x 1 x i64> @llvm.llrint.nxv1i64.nxv1f32(<vscale x 1 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = call <vscale x 2 x i64> @llvm.llrint.nxv2i64.nxv2f32(<vscale x 2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = call <vscale x 4 x i64> @llvm.llrint.nxv4i64.nxv4f32(<vscale x 4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = call <vscale x 8 x i64> @llvm.llrint.nxv8i64.nxv8f32(<vscale x 8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %10 = call <vscale x 16 x i64> @llvm.llrint.nxv16i64.nxv16f32(<vscale x 16 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = call i64 @llvm.llrint.i64.f64(double undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = call <2 x i64> @llvm.llrint.v2i64.v2f64(<2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = call <4 x i64> @llvm.llrint.v4i64.v4f64(<4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = call <8 x i64> @llvm.llrint.v8i64.v8f64(<8 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %15 = call <16 x i64> @llvm.llrint.v16i64.v16f64(<16 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %16 = call <vscale x 1 x i64> @llvm.llrint.nxv1i64.nxv1f64(<vscale x 1 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %17 = call <vscale x 2 x i64> @llvm.llrint.nxv2i64.nxv2f64(<vscale x 2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %18 = call <vscale x 4 x i64> @llvm.llrint.nxv4i64.nxv4f64(<vscale x 4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %19 = call <vscale x 8 x i64> @llvm.llrint.nxv8i64.nxv8f64(<vscale x 8 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  call i64 @llvm.llrint.i64.f32(float undef)
+  call <2 x i64> @llvm.llrint.v2i64.v2f32(<2 x float> undef)
+  call <4 x i64> @llvm.llrint.v4i64.v4f32(<4 x float> undef)
+  call <8 x i64> @llvm.llrint.v8i64.v8f32(<8 x float> undef)
+  call <16 x i64> @llvm.llrint.v16i64.v16f32(<16 x float> undef)
+  call <vscale x 1 x i64> @llvm.llrint.nvx1i64.nvx1f32(<vscale x 1 x float> undef)
+  call <vscale x 2 x i64> @llvm.llrint.nvx2i64.nvx2f32(<vscale x 2 x float> undef)
+  call <vscale x 4 x i64> @llvm.llrint.nvx4i64.nvx4f32(<vscale x 4 x float> undef)
+  call <vscale x 8 x i64> @llvm.llrint.nvx8i64.nvx8f32(<vscale x 8 x float> undef)
+  call <vscale x 16 x i64> @llvm.llrint.nvx16i64.nvx16f32(<vscale x 16 x float> undef)
+  call i64 @llvm.llrint.i64.f64(double undef)
+  call <2 x i64> @llvm.llrint.v2i64.v2f64(<2 x double> undef)
+  call <4 x i64> @llvm.llrint.v4i64.v4f64(<4 x double> undef)
+  call <8 x i64> @llvm.llrint.v8i64.v8f64(<8 x double> undef)
+  call <16 x i64> @llvm.llrint.v16i64.v16f64(<16 x double> undef)
+  call <vscale x 1 x i64> @llvm.llrint.nvx1i64.nvx1f64(<vscale x 1 x double> undef)
+  call <vscale x 2 x i64> @llvm.llrint.nvx2i64.nvx2f64(<vscale x 2 x double> undef)
+  call <vscale x 4 x i64> @llvm.llrint.nvx4i64.nvx4f64(<vscale x 4 x double> undef)
+  call <vscale x 8 x i64> @llvm.llrint.nvx8i64.nvx8f64(<vscale x 8 x double> undef)
+  ret void
+}
+
 define void @nearbyint() {
 ; CHECK-LABEL: 'nearbyint'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %1 = call float @llvm.nearbyint.f32(float undef)
@@ -683,6 +773,46 @@ declare <vscale x 2 x double> @llvm.rint.nvx2f64(<vscale x 2 x double>)
 declare <vscale x 4 x double> @llvm.rint.nvx4f64(<vscale x 4 x double>)
 declare <vscale x 8 x double> @llvm.rint.nvx8f64(<vscale x 8 x double>)
 
+declare i64 @llvm.lrint.i64.f32(float)
+declare <2 x i64> @llvm.lrint.v2i64.v2f32(<2 x float>)
+declare <4 x i64> @llvm.lrint.v4i64.v4f32(<4 x float>)
+declare <8 x i64> @llvm.lrint.v8i64.v8f32(<8 x float>)
+declare <16 x i64> @llvm.lrint.v16i64.v16f32(<16 x float>)
+declare <vscale x 1 x i64> @llvm.lrint.nvx1i64.nvx1f32(<vscale x 1 x float>)
+declare <vscale x 2 x i64> @llvm.lrint.nvx2i64.nvx2f32(<vscale x 2 x float>)
+declare <vscale x 4 x i64> @llvm.lrint.nvx4i64.nvx4f32(<vscale x 4 x float>)
+declare <vscale x 8 x i64> @llvm.lrint.nvx8i64.nvx8f32(<vscale x 8 x float>)
+declare <vscale x 16 x i64> @llvm.lrint.nvx16i64.nvx16f32(<vscale x 16 x float>)
+declare i64 @llvm.lrint.i64.f64(double)
+declare <2 x i64> @llvm.lrint.v2i64.v2f64(<2 x double>)
+declare <4 x i64> @llvm.lrint.v4i64.v4f64(<4 x double>)
+declare <8 x i64> @llvm.lrint.v8i64.v8f64(<8 x double>)
+declare <16 x i64> @llvm.lrint.v16i64.v16f64(<16 x double>)
+declare <vscale x 1 x i64> @llvm.lrint.nvx1i64.nvx1f64(<vscale x 1 x double>)
+declare <vscale x 2 x i64> @llvm.lrint.nvx2i64.nvx2f64(<vscale x 2 x double>)
+declare <vscale x 4 x i64> @llvm.lrint.nvx4i64.nvx4f64(<vscale x 4 x double>)
+declare <vscale x 8 x i64> @llvm.lrint.nvx8i64.nvx8f64(<vscale x 8 x double>)
+
+declare i64 @llvm.llrint.i64.f32(float)
+declare <2 x i64> @llvm.llrint.v2i64.v2f32(<2 x float>)
+declare <4 x i64> @llvm.llrint.v4i64.v4f32(<4 x float>)
+declare <8 x i64> @llvm.llrint.v8i64.v8f32(<8 x float>)
+declare <16 x i64> @llvm.llrint.v16i64.v16f32(<16 x float>)
+declare <vscale x 1 x i64> @llvm.llrint.nvx1i64.nvx1f32(<vscale x 1 x float>)
+declare <vscale x 2 x i64> @llvm.llrint.nvx2i64.nvx2f32(<vscale x 2 x float>)
+declare <vscale x 4 x i64> @llvm.llrint.nvx4i64.nvx4f32(<vscale x 4 x float>)
+declare <vscale x 8 x i64> @llvm.llrint.nvx8i64.nvx8f32(<vscale x 8 x float>)
+declare <vscale x 16 x i64> @llvm.llrint.nvx16i64.nvx16f32(<vscale x 16 x float>)
+declare i64 @llvm.llrint.i64.f64(double)
+declare <2 x i64> @llvm.llrint.v2i64.v2f64(<2 x double>)
+declare <4 x i64> @llvm.llrint.v4i64.v4f64(<4 x double>)
+declare <8 x i64> @llvm.llrint.v8i64.v8f64(<8 x double>)
+declare <16 x i64> @llvm.llrint.v16i64.v16f64(<16 x double>)
+declare <vscale x 1 x i64> @llvm.llrint.nvx1i64.nvx1f64(<vscale x 1 x double>)
+declare <vscale x 2 x i64> @llvm.llrint.nvx2i64.nvx2f64(<vscale x 2 x double>)
+declare <vscale x 4 x i64> @llvm.llrint.nvx4i64.nvx4f64(<vscale x 4 x double>)
+declare <vscale x 8 x i64> @llvm.llrint.nvx8i64.nvx8f64(<vscale x 8 x double>)
+
 declare float @llvm.nearbyint.f32(float)
 declare <2 x float> @llvm.nearbyint.v2f32(<2 x float>)
 declare <4 x float> @llvm.nearbyint.v4f32(<4 x float>)
diff --git a/llvm/test/Analysis/CostModel/RISCV/gep.ll b/llvm/test/Analysis/CostModel/RISCV/gep.ll
index be518faf7e..4fadf34c19 100644
--- a/llvm/test/Analysis/CostModel/RISCV/gep.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/gep.ll
@@ -270,7 +270,7 @@ define void @non_foldable_vector_uses(ptr %base, <2 x ptr> %base.vec) {
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = getelementptr i8, ptr %base, i32 42
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %x4 = call <2 x i8> @llvm.masked.expandload.v2i8(ptr %4, <2 x i1> undef, <2 x i8> undef)
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = getelementptr i8, ptr %base, i32 42
-; RVI-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %x5 = call <2 x i8> @llvm.vp.load.v2i8.p0(ptr %5, <2 x i1> undef, i32 undef)
+; RVI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %x5 = call <2 x i8> @llvm.vp.load.v2i8.p0(ptr %5, <2 x i1> undef, i32 undef)
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = getelementptr i8, ptr %base, i32 42
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %x6 = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr %6, i64 undef, <2 x i1> undef, i32 undef)
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = getelementptr i8, ptr %base, i32 42
@@ -282,7 +282,7 @@ define void @non_foldable_vector_uses(ptr %base, <2 x ptr> %base.vec) {
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = getelementptr i8, ptr %base, i32 42
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.compressstore.v2i8(<2 x i8> undef, ptr %10, <2 x i1> undef)
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = getelementptr i8, ptr %base, i32 42
-; RVI-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.vp.store.v2i8.p0(<2 x i8> undef, ptr %11, <2 x i1> undef, i32 undef)
+; RVI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.vp.store.v2i8.p0(<2 x i8> undef, ptr %11, <2 x i1> undef, i32 undef)
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = getelementptr i8, ptr %base, i32 42
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.experimental.vp.strided.store.v2i8.p0.i64(<2 x i8> undef, ptr %12, i64 undef, <2 x i1> undef, i32 undef)
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
@@ -340,7 +340,7 @@ define void @foldable_vector_uses(ptr %base, <2 x ptr> %base.vec) {
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %4 = getelementptr i8, ptr %base, i32 0
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %x4 = call <2 x i8> @llvm.masked.expandload.v2i8(ptr %4, <2 x i1> undef, <2 x i8> undef)
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %5 = getelementptr i8, ptr %base, i32 0
-; RVI-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %x5 = call <2 x i8> @llvm.vp.load.v2i8.p0(ptr %5, <2 x i1> undef, i32 undef)
+; RVI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %x5 = call <2 x i8> @llvm.vp.load.v2i8.p0(ptr %5, <2 x i1> undef, i32 undef)
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %6 = getelementptr i8, ptr %base, i32 0
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %x6 = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr %6, i64 undef, <2 x i1> undef, i32 undef)
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %7 = getelementptr i8, ptr %base, i32 0
@@ -352,7 +352,7 @@ define void @foldable_vector_uses(ptr %base, <2 x ptr> %base.vec) {
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %10 = getelementptr i8, ptr %base, i32 0
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.masked.compressstore.v2i8(<2 x i8> undef, ptr %10, <2 x i1> undef)
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %11 = getelementptr i8, ptr %base, i32 0
-; RVI-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.vp.store.v2i8.p0(<2 x i8> undef, ptr %11, <2 x i1> undef, i32 undef)
+; RVI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.vp.store.v2i8.p0(<2 x i8> undef, ptr %11, <2 x i1> undef, i32 undef)
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %12 = getelementptr i8, ptr %base, i32 0
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.experimental.vp.strided.store.v2i8.p0.i64(<2 x i8> undef, ptr %12, i64 undef, <2 x i1> undef, i32 undef)
 ; RVI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
diff --git a/llvm/test/Analysis/CostModel/RISCV/rvv-intrinsics.ll b/llvm/test/Analysis/CostModel/RISCV/rvv-intrinsics.ll
index 93de623cf1..43a03404e8 100644
--- a/llvm/test/Analysis/CostModel/RISCV/rvv-intrinsics.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/rvv-intrinsics.ll
@@ -206,10 +206,588 @@ define void @vp_fshl() {
   ret void
 }
 
+define void @add() {
+; CHECK-LABEL: 'add'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t0 = call <2 x i8> @llvm.vp.add.v2i8(<2 x i8> undef, <2 x i8> undef, <2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t1 = add <2 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t2 = call <4 x i8> @llvm.vp.add.v4i8(<4 x i8> undef, <4 x i8> undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t3 = add <4 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t4 = call <8 x i8> @llvm.vp.add.v8i8(<8 x i8> undef, <8 x i8> undef, <8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t5 = add <8 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t6 = call <16 x i8> @llvm.vp.add.v16i8(<16 x i8> undef, <16 x i8> undef, <16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t7 = add <16 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t8 = call <2 x i64> @llvm.vp.add.v2i64(<2 x i64> undef, <2 x i64> undef, <2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t9 = add <2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t10 = call <4 x i64> @llvm.vp.add.v4i64(<4 x i64> undef, <4 x i64> undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t12 = add <4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t13 = call <8 x i64> @llvm.vp.add.v8i64(<8 x i64> undef, <8 x i64> undef, <8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t14 = add <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t15 = call <16 x i64> @llvm.vp.add.v16i64(<16 x i64> undef, <16 x i64> undef, <16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t16 = add <16 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t17 = call <vscale x 2 x i8> @llvm.vp.add.nxv2i8(<vscale x 2 x i8> undef, <vscale x 2 x i8> undef, <vscale x 2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t18 = add <vscale x 2 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t19 = call <vscale x 4 x i8> @llvm.vp.add.nxv4i8(<vscale x 4 x i8> undef, <vscale x 4 x i8> undef, <vscale x 4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t20 = add <vscale x 4 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t21 = call <vscale x 8 x i8> @llvm.vp.add.nxv8i8(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef, <vscale x 8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t22 = add <vscale x 8 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t23 = call <vscale x 16 x i8> @llvm.vp.add.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <vscale x 16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t24 = add <vscale x 16 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t25 = call <vscale x 2 x i64> @llvm.vp.add.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> undef, <vscale x 2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t26 = add <vscale x 2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t27 = call <vscale x 4 x i64> @llvm.vp.add.nxv4i64(<vscale x 4 x i64> undef, <vscale x 4 x i64> undef, <vscale x 4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t28 = add <vscale x 4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t29 = call <vscale x 8 x i64> @llvm.vp.add.nxv8i64(<vscale x 8 x i64> undef, <vscale x 8 x i64> undef, <vscale x 8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t30 = add <vscale x 8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t31 = call <vscale x 16 x i64> @llvm.vp.add.nxv16i64(<vscale x 16 x i64> undef, <vscale x 16 x i64> undef, <vscale x 16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t32 = add <vscale x 16 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %t0 = call <2 x i8> @llvm.vp.add.v2i8(<2 x i8> undef, <2 x i8> undef, <2 x i1> undef, i32 undef)
+  %t1 = add <2 x i8> undef, undef
+  %t2 = call <4 x i8> @llvm.vp.add.v4i8(<4 x i8> undef, <4 x i8> undef, <4 x i1> undef, i32 undef)
+  %t3 = add <4 x i8> undef, undef
+  %t4 = call <8 x i8> @llvm.vp.add.v8i8(<8 x i8> undef, <8 x i8> undef, <8 x i1> undef, i32 undef)
+  %t5 = add <8 x i8> undef, undef
+  %t6 = call <16 x i8> @llvm.vp.add.v16i8(<16 x i8> undef, <16 x i8> undef, <16 x i1> undef, i32 undef)
+  %t7 = add <16 x i8> undef, undef
+  %t8 = call <2 x i64> @llvm.vp.add.v2i64(<2 x i64> undef, <2 x i64> undef, <2 x i1> undef, i32 undef)
+  %t9 = add <2 x i64> undef, undef
+  %t10 = call <4 x i64> @llvm.vp.add.v4i64(<4 x i64> undef, <4 x i64> undef, <4 x i1> undef, i32 undef)
+  %t12 = add <4 x i64> undef, undef
+  %t13 = call <8 x i64> @llvm.vp.add.v8i64(<8 x i64> undef, <8 x i64> undef, <8 x i1> undef, i32 undef)
+  %t14 = add <8 x i64> undef, undef
+  %t15 = call <16 x i64> @llvm.vp.add.v16i64(<16 x i64> undef, <16 x i64> undef, <16 x i1> undef, i32 undef)
+  %t16 = add <16 x i64> undef, undef
+  %t17 = call <vscale x 2 x i8> @llvm.vp.add.nxv2i8(<vscale x 2 x i8> undef, <vscale x 2 x i8> undef, <vscale x 2 x i1> undef, i32 undef)
+  %t18 = add <vscale x 2 x i8> undef, undef
+  %t19 = call <vscale x 4 x i8> @llvm.vp.add.nxv4i8(<vscale x 4 x i8> undef, <vscale x 4 x i8> undef, <vscale x 4 x i1> undef, i32 undef)
+  %t20 = add <vscale x 4 x i8> undef, undef
+  %t21 = call <vscale x 8 x i8> @llvm.vp.add.nxv8i8(<vscale x 8 x i8> undef, <vscale x 8 x i8> undef, <vscale x 8 x i1> undef, i32 undef)
+  %t22 = add <vscale x 8 x i8> undef, undef
+  %t23 = call <vscale x 16 x i8> @llvm.vp.add.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef, <vscale x 16 x i1> undef, i32 undef)
+  %t24 = add <vscale x 16 x i8> undef, undef
+  %t25 = call <vscale x 2 x i64> @llvm.vp.add.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> undef, <vscale x 2 x i1> undef, i32 undef)
+  %t26 = add <vscale x 2 x i64> undef, undef
+  %t27 = call <vscale x 4 x i64> @llvm.vp.add.nxv4i64(<vscale x 4 x i64> undef, <vscale x 4 x i64> undef, <vscale x 4 x i1> undef, i32 undef)
+  %t28 = add <vscale x 4 x i64> undef, undef
+  %t29 = call <vscale x 8 x i64> @llvm.vp.add.nxv8i64(<vscale x 8 x i64> undef, <vscale x 8 x i64> undef, <vscale x 8 x i1> undef, i32 undef)
+  %t30 = add <vscale x 8 x i64> undef, undef
+  %t31 = call <vscale x 16 x i64> @llvm.vp.add.nxv16i64(<vscale x 16 x i64> undef, <vscale x 16 x i64> undef, <vscale x 16 x i1> undef, i32 undef)
+  %t32 = add <vscale x 16 x i64> undef, undef
+  ret void
+}
+
+define void @abs() {
+; CHECK-LABEL: 'abs'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = call <2 x i8> @llvm.vp.abs.v2i8(<2 x i8> undef, i1 false, <2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = call <2 x i8> @llvm.abs.v2i8(<2 x i8> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %3 = call <4 x i8> @llvm.vp.abs.v4i8(<4 x i8> undef, i1 false, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %4 = call <4 x i8> @llvm.abs.v4i8(<4 x i8> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %5 = call <8 x i8> @llvm.vp.abs.v8i8(<8 x i8> undef, i1 false, <8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %6 = call <8 x i8> @llvm.abs.v8i8(<8 x i8> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %7 = call <16 x i8> @llvm.vp.abs.v16i8(<16 x i8> undef, i1 false, <16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %8 = call <16 x i8> @llvm.abs.v16i8(<16 x i8> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %9 = call <2 x i64> @llvm.vp.abs.v2i64(<2 x i64> undef, i1 false, <2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %10 = call <2 x i64> @llvm.abs.v2i64(<2 x i64> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %11 = call <4 x i64> @llvm.vp.abs.v4i64(<4 x i64> undef, i1 false, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %12 = call <4 x i64> @llvm.abs.v4i64(<4 x i64> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %13 = call <8 x i64> @llvm.vp.abs.v8i64(<8 x i64> undef, i1 false, <8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %14 = call <8 x i64> @llvm.abs.v8i64(<8 x i64> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %15 = call <16 x i64> @llvm.vp.abs.v16i64(<16 x i64> undef, i1 false, <16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %16 = call <16 x i64> @llvm.abs.v16i64(<16 x i64> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %17 = call <vscale x 2 x i8> @llvm.vp.abs.nxv2i8(<vscale x 2 x i8> undef, i1 false, <vscale x 2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %18 = call <vscale x 2 x i8> @llvm.abs.nxv2i8(<vscale x 2 x i8> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %19 = call <vscale x 4 x i8> @llvm.vp.abs.nxv4i8(<vscale x 4 x i8> undef, i1 false, <vscale x 4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %20 = call <vscale x 4 x i8> @llvm.abs.nxv4i8(<vscale x 4 x i8> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %21 = call <vscale x 8 x i8> @llvm.vp.abs.nxv8i8(<vscale x 8 x i8> undef, i1 false, <vscale x 8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %22 = call <vscale x 8 x i8> @llvm.abs.nxv8i8(<vscale x 8 x i8> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %23 = call <vscale x 16 x i8> @llvm.vp.abs.nxv16i8(<vscale x 16 x i8> undef, i1 false, <vscale x 16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %24 = call <vscale x 16 x i8> @llvm.abs.nxv16i8(<vscale x 16 x i8> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %25 = call <vscale x 2 x i64> @llvm.vp.abs.nxv2i64(<vscale x 2 x i64> undef, i1 false, <vscale x 2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %26 = call <vscale x 2 x i64> @llvm.abs.nxv2i64(<vscale x 2 x i64> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %27 = call <vscale x 4 x i64> @llvm.vp.abs.nxv4i64(<vscale x 4 x i64> undef, i1 false, <vscale x 4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %28 = call <vscale x 4 x i64> @llvm.abs.nxv4i64(<vscale x 4 x i64> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %29 = call <vscale x 8 x i64> @llvm.vp.abs.nxv8i64(<vscale x 8 x i64> undef, i1 false, <vscale x 8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %30 = call <vscale x 8 x i64> @llvm.abs.nxv8i64(<vscale x 8 x i64> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %31 = call <vscale x 16 x i64> @llvm.vp.abs.nxv16i64(<vscale x 16 x i64> undef, i1 false, <vscale x 16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %32 = call <vscale x 16 x i64> @llvm.abs.nxv16i64(<vscale x 16 x i64> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  call <2 x i8> @llvm.vp.abs.v2i8(<2 x i8> undef, i1 0, <2 x i1> undef, i32 undef)
+  call <2 x i8> @llvm.abs.v2i8(<2 x i8> undef, i1 0)
+  call <4 x i8> @llvm.vp.abs.v4i8(<4 x i8> undef, i1 0, <4 x i1> undef, i32 undef)
+  call <4 x i8> @llvm.abs.v4i8(<4 x i8> undef, i1 0)
+  call <8 x i8> @llvm.vp.abs.v8i8(<8 x i8> undef, i1 0, <8 x i1> undef, i32 undef)
+  call <8 x i8> @llvm.abs.v8i8(<8 x i8> undef, i1 0)
+  call <16 x i8> @llvm.vp.abs.v16i8(<16 x i8> undef, i1 0, <16 x i1> undef, i32 undef)
+  call <16 x i8> @llvm.abs.v16i8(<16 x i8> undef, i1 0)
+  call <2 x i64> @llvm.vp.abs.v2i64(<2 x i64> undef, i1 0, <2 x i1> undef, i32 undef)
+  call <2 x i64> @llvm.abs.v2i64(<2 x i64> undef, i1 0)
+  call <4 x i64> @llvm.vp.abs.v4i64(<4 x i64> undef, i1 0, <4 x i1> undef, i32 undef)
+  call <4 x i64> @llvm.abs.v4i64(<4 x i64> undef, i1 0)
+  call <8 x i64> @llvm.vp.abs.v8i64(<8 x i64> undef, i1 0, <8 x i1> undef, i32 undef)
+  call <8 x i64> @llvm.abs.v8i64(<8 x i64> undef, i1 0)
+  call <16 x i64> @llvm.vp.abs.v16i64(<16 x i64> undef, i1 0, <16 x i1> undef, i32 undef)
+  call <16 x i64> @llvm.abs.v16i64(<16 x i64> undef, i1 0)
+  call <vscale x 2 x i8> @llvm.vp.abs.nxv2i8(<vscale x 2 x i8> undef, i1 0, <vscale x 2 x i1> undef, i32 undef)
+  call <vscale x 2 x i8> @llvm.abs.nxv2i8(<vscale x 2 x i8> undef, i1 0)
+  call <vscale x 4 x i8> @llvm.vp.abs.nxv4i8(<vscale x 4 x i8> undef, i1 0, <vscale x 4 x i1> undef, i32 undef)
+  call <vscale x 4 x i8> @llvm.abs.nxv4i8(<vscale x 4 x i8> undef, i1 0)
+  call <vscale x 8 x i8> @llvm.vp.abs.nxv8i8(<vscale x 8 x i8> undef, i1 0, <vscale x 8 x i1> undef, i32 undef)
+  call <vscale x 8 x i8> @llvm.abs.nxv8i8(<vscale x 8 x i8> undef, i1 0)
+  call <vscale x 16 x i8> @llvm.vp.abs.nxv16i8(<vscale x 16 x i8> undef, i1 0, <vscale x 16 x i1> undef, i32 undef)
+  call <vscale x 16 x i8> @llvm.abs.nxv16i8(<vscale x 16 x i8> undef, i1 0)
+  call <vscale x 2 x i64> @llvm.vp.abs.nxv2i64(<vscale x 2 x i64> undef, i1 0, <vscale x 2 x i1> undef, i32 undef)
+  call <vscale x 2 x i64> @llvm.abs.nxv2i64(<vscale x 2 x i64> undef, i1 0)
+  call <vscale x 4 x i64> @llvm.vp.abs.nxv4i64(<vscale x 4 x i64> undef, i1 0, <vscale x 4 x i1> undef, i32 undef)
+  call <vscale x 4 x i64> @llvm.abs.nxv4i64(<vscale x 4 x i64> undef, i1 0)
+  call <vscale x 8 x i64> @llvm.vp.abs.nxv8i64(<vscale x 8 x i64> undef, i1 0, <vscale x 8 x i1> undef, i32 undef)
+  call <vscale x 8 x i64> @llvm.abs.nxv8i64(<vscale x 8 x i64> undef, i1 0)
+  call <vscale x 16 x i64> @llvm.vp.abs.nxv16i64(<vscale x 16 x i64> undef, i1 0, <vscale x 16 x i1> undef, i32 undef)
+  call <vscale x 16 x i64> @llvm.abs.nxv16i64(<vscale x 16 x i64> undef, i1 0)
+  ret void
+}
+
+define void @load() {
+; CHECK-LABEL: 'load'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t0 = call <2 x i8> @llvm.vp.load.v2i8.p0(ptr undef, <2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t1 = load <2 x i8>, ptr undef, align 2
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t2 = call <4 x i8> @llvm.vp.load.v4i8.p0(ptr undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t3 = load <4 x i8>, ptr undef, align 4
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t4 = call <8 x i8> @llvm.vp.load.v8i8.p0(ptr undef, <8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t5 = load <8 x i8>, ptr undef, align 8
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t6 = call <16 x i8> @llvm.vp.load.v16i8.p0(ptr undef, <16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t7 = load <16 x i8>, ptr undef, align 16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t8 = call <2 x i64> @llvm.vp.load.v2i64.p0(ptr undef, <2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t9 = load <2 x i64>, ptr undef, align 16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t10 = call <4 x i64> @llvm.vp.load.v4i64.p0(ptr undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t12 = load <4 x i64>, ptr undef, align 32
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t13 = call <8 x i64> @llvm.vp.load.v8i64.p0(ptr undef, <8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t14 = load <8 x i64>, ptr undef, align 64
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t15 = call <16 x i64> @llvm.vp.load.v16i64.p0(ptr undef, <16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t16 = load <16 x i64>, ptr undef, align 128
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t17 = call <vscale x 2 x i8> @llvm.vp.load.nxv2i8.p0(ptr undef, <vscale x 2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t18 = load <vscale x 2 x i8>, ptr undef, align 2
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t19 = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr undef, <vscale x 4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t20 = load <vscale x 4 x i8>, ptr undef, align 4
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t21 = call <vscale x 8 x i8> @llvm.vp.load.nxv8i8.p0(ptr undef, <vscale x 8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %t22 = load <vscale x 8 x i8>, ptr undef, align 8
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t23 = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr undef, <vscale x 16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t24 = load <vscale x 16 x i8>, ptr undef, align 16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t25 = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr undef, <vscale x 2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t26 = load <vscale x 2 x i64>, ptr undef, align 16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t27 = call <vscale x 4 x i64> @llvm.vp.load.nxv4i64.p0(ptr undef, <vscale x 4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t28 = load <vscale x 4 x i64>, ptr undef, align 32
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t29 = call <vscale x 8 x i64> @llvm.vp.load.nxv8i64.p0(ptr undef, <vscale x 8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %t30 = load <vscale x 8 x i64>, ptr undef, align 64
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t31 = call <vscale x 16 x i64> @llvm.vp.load.nxv16i64.p0(ptr undef, <vscale x 16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %t32 = load <vscale x 16 x i64>, ptr undef, align 128
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %t0 = call <2 x i8> @llvm.vp.load.v2i8(ptr undef, <2 x i1> undef, i32 undef)
+  %t1 = load <2 x i8>, ptr undef
+  %t2 = call <4 x i8> @llvm.vp.load.v4i8(ptr undef, <4 x i1> undef, i32 undef)
+  %t3 = load <4 x i8>, ptr undef
+  %t4 = call <8 x i8> @llvm.vp.load.v8i8(ptr undef, <8 x i1> undef, i32 undef)
+  %t5 = load <8 x i8>, ptr undef
+  %t6 = call <16 x i8> @llvm.vp.load.v16i8(ptr undef, <16 x i1> undef, i32 undef)
+  %t7 = load <16 x i8>, ptr undef
+  %t8 = call <2 x i64> @llvm.vp.load.v2i64(ptr undef, <2 x i1> undef, i32 undef)
+  %t9 = load <2 x i64>, ptr undef
+  %t10 = call <4 x i64> @llvm.vp.load.v4i64(ptr undef, <4 x i1> undef, i32 undef)
+  %t12 = load <4 x i64>, ptr undef
+  %t13 = call <8 x i64> @llvm.vp.load.v8i64(ptr undef, <8 x i1> undef, i32 undef)
+  %t14 = load <8 x i64>, ptr undef
+  %t15 = call <16 x i64> @llvm.vp.load.v16i64(ptr undef, <16 x i1> undef, i32 undef)
+  %t16 = load <16 x i64>, ptr undef
+  %t17 = call <vscale x 2 x i8> @llvm.vp.load.nxv2i8(ptr undef, <vscale x 2 x i1> undef, i32 undef)
+  %t18 = load <vscale x 2 x i8>, ptr undef
+  %t19 = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8(ptr undef, <vscale x 4 x i1> undef, i32 undef)
+  %t20 = load <vscale x 4 x i8>, ptr undef
+  %t21 = call <vscale x 8 x i8> @llvm.vp.load.nxv8i8(ptr undef, <vscale x 8 x i1> undef, i32 undef)
+  %t22 = load <vscale x 8 x i8>, ptr undef
+  %t23 = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8(ptr undef, <vscale x 16 x i1> undef, i32 undef)
+  %t24 = load <vscale x 16 x i8>, ptr undef
+  %t25 = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64(ptr undef, <vscale x 2 x i1> undef, i32 undef)
+  %t26 = load <vscale x 2 x i64>, ptr undef
+  %t27 = call <vscale x 4 x i64> @llvm.vp.load.nxv4i64(ptr undef, <vscale x 4 x i1> undef, i32 undef)
+  %t28 = load <vscale x 4 x i64>, ptr undef
+  %t29 = call <vscale x 8 x i64> @llvm.vp.load.nxv8i64(ptr undef, <vscale x 8 x i1> undef, i32 undef)
+  %t30 = load <vscale x 8 x i64>, ptr undef
+  %t31 = call <vscale x 16 x i64> @llvm.vp.load.nxv16i64(ptr undef, <vscale x 16 x i1> undef, i32 undef)
+  %t32 = load <vscale x 16 x i64>, ptr undef
+  ret void
+}
+
+define void @store() {
+; CHECK-LABEL: 'store'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.vp.store.v2i8.p0(<2 x i8> undef, ptr undef, <2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> undef, ptr undef, align 2
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.vp.store.v4i8.p0(<4 x i8> undef, ptr undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> undef, ptr undef, align 4
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.vp.store.v8i8.p0(<8 x i8> undef, ptr undef, <8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> undef, ptr undef, align 8
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.vp.store.v16i8.p0(<16 x i8> undef, ptr undef, <16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> undef, ptr undef, align 16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.vp.store.v2i64.p0(<2 x i64> undef, ptr undef, <2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i64> undef, ptr undef, align 16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.vp.store.v4i64.p0(<4 x i64> undef, ptr undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x i64> undef, ptr undef, align 32
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.vp.store.v8i64.p0(<8 x i64> undef, ptr undef, <8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <8 x i64> undef, ptr undef, align 64
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.vp.store.v16i64.p0(<16 x i64> undef, ptr undef, <16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <16 x i64> undef, ptr undef, align 128
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.vp.store.nxv2i8.p0(<vscale x 2 x i8> undef, ptr undef, <vscale x 2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <vscale x 2 x i8> undef, ptr undef, align 2
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.vp.store.nxv4i8.p0(<vscale x 4 x i8> undef, ptr undef, <vscale x 4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <vscale x 4 x i8> undef, ptr undef, align 4
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.vp.store.nxv8i8.p0(<vscale x 8 x i8> undef, ptr undef, <vscale x 8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <vscale x 8 x i8> undef, ptr undef, align 8
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> undef, ptr undef, <vscale x 16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <vscale x 16 x i8> undef, ptr undef, align 16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> undef, ptr undef, <vscale x 2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <vscale x 2 x i64> undef, ptr undef, align 16
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.vp.store.nxv4i64.p0(<vscale x 4 x i64> undef, ptr undef, <vscale x 4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <vscale x 4 x i64> undef, ptr undef, align 32
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.vp.store.nxv8i64.p0(<vscale x 8 x i64> undef, ptr undef, <vscale x 8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <vscale x 8 x i64> undef, ptr undef, align 64
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.vp.store.nxv16i64.p0(<vscale x 16 x i64> undef, ptr undef, <vscale x 16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: store <vscale x 16 x i64> undef, ptr undef, align 128
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  call void @llvm.vp.store.v2i8(<2 x i8> undef, ptr undef, <2 x i1> undef, i32 undef)
+  store <2 x i8> undef, ptr undef
+  call void @llvm.vp.store.v4i8(<4 x i8> undef, ptr undef, <4 x i1> undef, i32 undef)
+  store <4 x i8> undef, ptr undef
+  call void @llvm.vp.store.v8i8(<8 x i8> undef, ptr undef, <8 x i1> undef, i32 undef)
+  store <8 x i8> undef, ptr undef
+  call void @llvm.vp.store.v16i8(<16 x i8> undef, ptr undef, <16 x i1> undef, i32 undef)
+  store <16 x i8> undef, ptr undef
+  call void @llvm.vp.store.v2i64(<2 x i64> undef, ptr undef, <2 x i1> undef, i32 undef)
+  store <2 x i64> undef, ptr undef
+  call void @llvm.vp.store.v4i64(<4 x i64> undef, ptr undef, <4 x i1> undef, i32 undef)
+  store <4 x i64> undef, ptr undef
+  call void @llvm.vp.store.v8i64(<8 x i64> undef, ptr undef, <8 x i1> undef, i32 undef)
+  store <8 x i64> undef, ptr undef
+  call void @llvm.vp.store.v16i64(<16 x i64> undef, ptr undef, <16 x i1> undef, i32 undef)
+  store <16 x i64> undef, ptr undef
+  call void @llvm.vp.store.nxv2i8(<vscale x 2 x i8> undef, ptr undef, <vscale x 2 x i1> undef, i32 undef)
+  store <vscale x 2 x i8> undef, ptr undef
+  call void @llvm.vp.store.nxv4i8(<vscale x 4 x i8> undef, ptr undef, <vscale x 4 x i1> undef, i32 undef)
+  store <vscale x 4 x i8> undef, ptr undef
+  call void @llvm.vp.store.nxv8i8(<vscale x 8 x i8> undef, ptr undef, <vscale x 8 x i1> undef, i32 undef)
+  store <vscale x 8 x i8> undef, ptr undef
+  call void @llvm.vp.store.nxv16i8(<vscale x 16 x i8> undef, ptr undef, <vscale x 16 x i1> undef, i32 undef)
+  store <vscale x 16 x i8> undef, ptr undef
+  call void @llvm.vp.store.nxv2i64(<vscale x 2 x i64> undef, ptr undef, <vscale x 2 x i1> undef, i32 undef)
+  store <vscale x 2 x i64> undef, ptr undef
+  call void @llvm.vp.store.nxv4i64(<vscale x 4 x i64> undef, ptr undef, <vscale x 4 x i1> undef, i32 undef)
+  store <vscale x 4 x i64> undef, ptr undef
+  call void @llvm.vp.store.nxv8i64(<vscale x 8 x i64> undef, ptr undef, <vscale x 8 x i1> undef, i32 undef)
+  store <vscale x 8 x i64> undef, ptr undef
+  call void @llvm.vp.store.nxv16i64(<vscale x 16 x i64> undef, ptr undef, <vscale x 16 x i1> undef, i32 undef)
+  store <vscale x 16 x i64> undef, ptr undef
+  ret void
+}
+
+define void @reduce_add() {
+; CHECK-LABEL: 'reduce_add'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %1 = call i8 @llvm.vp.reduce.add.v2i8(i8 undef, <2 x i8> undef, <2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %2 = call i8 @llvm.vector.reduce.add.v2i8(<2 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %3 = call i8 @llvm.vp.reduce.add.v4i8(i8 undef, <4 x i8> undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %4 = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %5 = call i8 @llvm.vp.reduce.add.v8i8(i8 undef, <8 x i8> undef, <8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %6 = call i8 @llvm.vector.reduce.add.v8i8(<8 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %7 = call i8 @llvm.vp.reduce.add.v16i8(i8 undef, <16 x i8> undef, <16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %8 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %9 = call i64 @llvm.vp.reduce.add.v2i64(i64 undef, <2 x i64> undef, <2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %10 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %11 = call i64 @llvm.vp.reduce.add.v4i64(i64 undef, <4 x i64> undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %12 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %13 = call i64 @llvm.vp.reduce.add.v8i64(i64 undef, <8 x i64> undef, <8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %14 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %15 = call i64 @llvm.vp.reduce.add.v16i64(i64 undef, <16 x i64> undef, <16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %16 = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %17 = call i8 @llvm.vp.reduce.add.nxv8i8(i8 undef, <vscale x 8 x i8> undef, <vscale x 8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %18 = call i8 @llvm.vector.reduce.add.nxv2i8(<vscale x 2 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %19 = call i8 @llvm.vp.reduce.add.nxv4i8(i8 undef, <vscale x 4 x i8> undef, <vscale x 4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %20 = call i8 @llvm.vector.reduce.add.nxv4i8(<vscale x 4 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %21 = call i8 @llvm.vp.reduce.add.nxv8i8(i8 undef, <vscale x 8 x i8> undef, <vscale x 8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %22 = call i8 @llvm.vector.reduce.add.nxv8i8(<vscale x 8 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %23 = call i8 @llvm.vp.reduce.add.nxv16i8(i8 undef, <vscale x 16 x i8> undef, <vscale x 16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %24 = call i8 @llvm.vector.reduce.add.nxv16i8(<vscale x 16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %25 = call i64 @llvm.vp.reduce.add.nxv2i64(i64 undef, <vscale x 2 x i64> undef, <vscale x 2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %26 = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %27 = call i64 @llvm.vp.reduce.add.nxv4i64(i64 undef, <vscale x 4 x i64> undef, <vscale x 4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %28 = call i64 @llvm.vector.reduce.add.nxv4i64(<vscale x 4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %29 = call i64 @llvm.vp.reduce.add.nxv8i64(i64 undef, <vscale x 8 x i64> undef, <vscale x 8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %30 = call i64 @llvm.vector.reduce.add.nxv8i64(<vscale x 8 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %31 = call i64 @llvm.vp.reduce.add.nxv16i64(i64 undef, <vscale x 16 x i64> undef, <vscale x 16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %32 = call i64 @llvm.vector.reduce.add.nxv16i64(<vscale x 16 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  call i8 @llvm.vp.reduce.add.v2i8(i8 undef, <2 x i8> undef, <2 x i1> undef, i32 undef)
+  call i8 @llvm.vector.reduce.add.v2i8(<2 x i8> undef)
+  call i8 @llvm.vp.reduce.add.v4i8(i8 undef, <4 x i8> undef, <4 x i1> undef, i32 undef)
+  call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> undef)
+  call i8 @llvm.vp.reduce.add.v8i8(i8 undef, <8 x i8> undef, <8 x i1> undef, i32 undef)
+  call i8 @llvm.vector.reduce.add.v8i8(<8 x i8> undef)
+  call i8 @llvm.vp.reduce.add.v16i8(i8 undef, <16 x i8> undef, <16 x i1> undef, i32 undef)
+  call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> undef)
+  call i64 @llvm.vp.reduce.add.v2i64(i64 undef, <2 x i64> undef, <2 x i1> undef, i32 undef)
+  call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> undef)
+  call i64 @llvm.vp.reduce.add.v4i64(i64 undef, <4 x i64> undef, <4 x i1> undef, i32 undef)
+  call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> undef)
+  call i64 @llvm.vp.reduce.add.v8i64(i64 undef, <8 x i64> undef, <8 x i1> undef, i32 undef)
+  call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> undef)
+  call i64 @llvm.vp.reduce.add.v16i64(i64 undef, <16 x i64> undef, <16 x i1> undef, i32 undef)
+  call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> undef)
+  call i8 @llvm.vp.reduce.add.nxv8i8(i8  undef, <vscale x 8 x i8> undef, <vscale x 8 x i1> undef, i32 undef)
+  call i8 @llvm.vector.reduce.add.nxv2i8(<vscale x 2 x i8> undef)
+  call i8 @llvm.vp.reduce.add.nxv4i8(i8  undef, <vscale x 4 x i8> undef, <vscale x 4 x i1> undef, i32 undef)
+  call i8 @llvm.vector.reduce.add.nxv4i8(<vscale x 4 x i8> undef)
+  call i8 @llvm.vp.reduce.add.nxv8i8(i8  undef, <vscale x 8 x i8> undef, <vscale x 8 x i1> undef, i32 undef)
+  call i8 @llvm.vector.reduce.add.nxv8i8(<vscale x 8 x i8> undef)
+  call i8 @llvm.vp.reduce.add.nxv16i8(i8  undef, <vscale x 16 x i8> undef, <vscale x 16 x i1> undef, i32 undef)
+  call i8 @llvm.vector.reduce.add.nxv16i8(<vscale x 16 x i8> undef)
+  call i64 @llvm.vp.reduce.add.nxv2i64(i64  undef, <vscale x 2 x i64> undef, <vscale x 2 x i1> undef, i32 undef)
+  call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> undef)
+  call i64 @llvm.vp.reduce.add.nxv4i64(i64  undef, <vscale x 4 x i64> undef, <vscale x 4 x i1> undef, i32 undef)
+  call i64 @llvm.vector.reduce.add.nxv4i64(<vscale x 4 x i64> undef)
+  call i64 @llvm.vp.reduce.add.nxv8i64(i64 undef, <vscale x 8 x i64> undef, <vscale x 8 x i1> undef, i32 undef)
+  call i64 @llvm.vector.reduce.add.nxv8i64(<vscale x 8 x i64> undef)
+  call i64 @llvm.vp.reduce.add.nxv16i64(i64 undef, <vscale x 16 x i64> undef, <vscale x 16 x i1> undef, i32 undef)
+  call i64 @llvm.vector.reduce.add.nxv16i64(<vscale x 16 x i64> undef)
+  ret void
+}
+
+define void @reduce_fadd() {
+; CHECK-LABEL: 'reduce_fadd'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %1 = call float @llvm.vp.reduce.fadd.v2f32(float undef, <2 x float> undef, <2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %2 = call float @llvm.vector.reduce.fadd.v2f32(float undef, <2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %3 = call float @llvm.vp.reduce.fadd.v4f32(float undef, <4 x float> undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %4 = call float @llvm.vector.reduce.fadd.v4f32(float undef, <4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %5 = call float @llvm.vp.reduce.fadd.v8f32(float undef, <8 x float> undef, <8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %6 = call float @llvm.vector.reduce.fadd.v8f32(float undef, <8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %7 = call float @llvm.vp.reduce.fadd.v16f32(float undef, <16 x float> undef, <16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %8 = call float @llvm.vector.reduce.fadd.v16f32(float undef, <16 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %9 = call double @llvm.vp.reduce.fadd.v2f64(double undef, <2 x double> undef, <2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %10 = call double @llvm.vector.reduce.fadd.v2f64(double undef, <2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %11 = call double @llvm.vp.reduce.fadd.v4f64(double undef, <4 x double> undef, <4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %12 = call double @llvm.vector.reduce.fadd.v4f64(double undef, <4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %13 = call double @llvm.vp.reduce.fadd.v8f64(double undef, <8 x double> undef, <8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %14 = call double @llvm.vector.reduce.fadd.v8f64(double undef, <8 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %15 = call double @llvm.vp.reduce.fadd.v16f64(double undef, <16 x double> undef, <16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %16 = call double @llvm.vector.reduce.fadd.v16f64(double undef, <16 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %17 = call float @llvm.vp.reduce.fadd.nxv2f32(float undef, <vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %18 = call float @llvm.vector.reduce.fadd.nxv2f32(float undef, <vscale x 2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %19 = call float @llvm.vp.reduce.fadd.nxv4f32(float undef, <vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %20 = call float @llvm.vector.reduce.fadd.nxv4f32(float undef, <vscale x 4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %21 = call float @llvm.vp.reduce.fadd.nxv8f32(float undef, <vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %22 = call float @llvm.vector.reduce.fadd.nxv8f32(float undef, <vscale x 8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %23 = call float @llvm.vp.reduce.fadd.nxv16f32(float undef, <vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %24 = call float @llvm.vector.reduce.fadd.nxv16f32(float undef, <vscale x 16 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %25 = call double @llvm.vp.reduce.fadd.nxv2f64(double undef, <vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %26 = call double @llvm.vector.reduce.fadd.nxv2f64(double undef, <vscale x 2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %27 = call double @llvm.vp.reduce.fadd.nxv4f64(double undef, <vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %28 = call double @llvm.vector.reduce.fadd.nxv4f64(double undef, <vscale x 4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %29 = call double @llvm.vp.reduce.fadd.nxv8f64(double undef, <vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %30 = call double @llvm.vector.reduce.fadd.nxv8f64(double undef, <vscale x 8 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %31 = call double @llvm.vp.reduce.fadd.nxv16f64(double undef, <vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %32 = call double @llvm.vector.reduce.fadd.nxv16f64(double undef, <vscale x 16 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  call float @llvm.vp.reduce.fadd.v2f32(float undef, <2 x float> undef, <2 x i1> undef, i32 undef)
+  call float @llvm.vector.reduce.fadd.v2f32(float undef, <2 x float> undef)
+  call float @llvm.vp.reduce.fadd.v4f32(float undef, <4 x float> undef, <4 x i1> undef, i32 undef)
+  call float @llvm.vector.reduce.fadd.v4f32(float undef, <4 x float> undef)
+  call float @llvm.vp.reduce.fadd.v8f32(float undef, <8 x float> undef, <8 x i1> undef, i32 undef)
+  call float @llvm.vector.reduce.fadd.v8f32(float undef, <8 x float> undef)
+  call float @llvm.vp.reduce.fadd.v16f32(float undef, <16 x float> undef, <16 x i1> undef, i32 undef)
+  call float @llvm.vector.reduce.fadd.v16f32(float undef, <16 x float> undef)
+  call double @llvm.vp.reduce.fadd.v2f64(double undef, <2 x double> undef, <2 x i1> undef, i32 undef)
+  call double @llvm.vector.reduce.fadd.v2f64(double undef, <2 x double> undef)
+  call double @llvm.vp.reduce.fadd.v4f64(double undef, <4 x double> undef, <4 x i1> undef, i32 undef)
+  call double @llvm.vector.reduce.fadd.v4f64(double undef, <4 x double> undef)
+  call double @llvm.vp.reduce.fadd.v8f64(double undef, <8 x double> undef, <8 x i1> undef, i32 undef)
+  call double @llvm.vector.reduce.fadd.v8f64(double undef, <8 x double> undef)
+  call double @llvm.vp.reduce.fadd.v16f64(double undef, <16 x double> undef, <16 x i1> undef, i32 undef)
+  call double @llvm.vector.reduce.fadd.v16f64(double undef, <16 x double> undef)
+  call float @llvm.vp.reduce.fadd.nxv2f32(float undef, <vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
+  call float @llvm.vector.reduce.fadd.nxv2f32(float undef, <vscale x 2 x float> undef)
+  call float @llvm.vp.reduce.fadd.nxv4f32(float undef, <vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+  call float @llvm.vector.reduce.fadd.nxv4f32(float undef, <vscale x 4 x float> undef)
+  call float @llvm.vp.reduce.fadd.nxv8f32(float undef, <vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
+  call float @llvm.vector.reduce.fadd.nxv8f32(float undef, <vscale x 8 x float> undef)
+  call float @llvm.vp.reduce.fadd.nxv16f32(float undef, <vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
+  call float @llvm.vector.reduce.fadd.nxv16f32(float undef, <vscale x 16 x float> undef)
+  call double @llvm.vp.reduce.fadd.nxv2f64(double undef, <vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
+  call double @llvm.vector.reduce.fadd.nxv2f64(double undef, <vscale x 2 x double> undef)
+  call double @llvm.vp.reduce.fadd.nxv4f64(double undef, <vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
+  call double @llvm.vector.reduce.fadd.nxv4f64(double undef, <vscale x 4 x double> undef)
+  call double @llvm.vp.reduce.fadd.nxv8f64(double undef, <vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
+  call double @llvm.vector.reduce.fadd.nxv8f64(double undef, <vscale x 8 x double> undef)
+  call double @llvm.vp.reduce.fadd.nxv16f64(double undef, <vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
+  call double @llvm.vector.reduce.fadd.nxv16f64(double undef, <vscale x 16 x double> undef)
+  ret void
+}
+
+declare <2 x i8> @llvm.vp.add.v2i8(<2 x i8>, <2 x i8>, <2 x i1>, i32)
+declare <4 x i8> @llvm.vp.add.v4i8(<4 x i8>, <4 x i8>, <4 x i1>, i32)
+declare <8 x i8> @llvm.vp.add.v8i8(<8 x i8>, <8 x i8>, <8 x i1>, i32)
+declare <16 x i8> @llvm.vp.add.v16i8(<16 x i8>, <16 x i8>, <16 x i1>, i32)
+declare <2 x i64> @llvm.vp.add.v2i64(<2 x i64>, <2 x i64>, <2 x i1>, i32)
+declare <4 x i64> @llvm.vp.add.v4i64(<4 x i64>, <4 x i64>, <4 x i1>, i32)
+declare <8 x i64> @llvm.vp.add.v8i64(<8 x i64>, <8 x i64>, <8 x i1>, i32)
+declare <16 x i64> @llvm.vp.add.v16i64(<16 x i64>, <16 x i64>, <16 x i1>, i32)
+declare <vscale x 2 x i8> @llvm.vp.add.nxv2i8(<vscale x 2 x i8>, <vscale x 2 x i8>, <vscale x 2 x i1>, i32)
+declare <vscale x 4 x i8> @llvm.vp.add.nxv4i8(<vscale x 4 x i8>, <vscale x 4 x i8>, <vscale x 4 x i1>, i32)
+declare <vscale x 8 x i8> @llvm.vp.add.nxv8i8(<vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i1>, i32)
+declare <vscale x 16 x i8> @llvm.vp.add.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i1>, i32)
+declare <vscale x 2 x i64> @llvm.vp.add.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i1>, i32)
+declare <vscale x 4 x i64> @llvm.vp.add.nxv4i64(<vscale x 4 x i64>, <vscale x 4 x i64>, <vscale x 4 x i1>, i32)
+declare <vscale x 8 x i64> @llvm.vp.add.nxv8i64(<vscale x 8 x i64>, <vscale x 8 x i64>, <vscale x 8 x i1>, i32)
+declare <vscale x 16 x i64> @llvm.vp.add.nxv16i64(<vscale x 16 x i64>, <vscale x 16 x i64>, <vscale x 16 x i1>, i32)
+
+declare <2 x i8> @llvm.vp.abs.v2i8(<2 x i8>, i1, <2 x i1>, i32)
+declare <4 x i8> @llvm.vp.abs.v4i8(<4 x i8>, i1, <4 x i1>, i32)
+declare <8 x i8> @llvm.vp.abs.v8i8(<8 x i8>, i1, <8 x i1>, i32)
+declare <16 x i8> @llvm.vp.abs.v16i8(<16 x i8>, i1, <16 x i1>, i32)
+declare <2 x i64> @llvm.vp.abs.v2i64(<2 x i64>, i1, <2 x i1>, i32)
+declare <4 x i64> @llvm.vp.abs.v4i64(<4 x i64>, i1, <4 x i1>, i32)
+declare <8 x i64> @llvm.vp.abs.v8i64(<8 x i64>, i1, <8 x i1>, i32)
+declare <16 x i64> @llvm.vp.abs.v16i64(<16 x i64>, i1, <16 x i1>, i32)
+declare <vscale x 2 x i8> @llvm.vp.abs.nxv2i8(<vscale x 2 x i8>, i1, <vscale x 2 x i1>, i32)
+declare <vscale x 4 x i8> @llvm.vp.abs.nxv4i8(<vscale x 4 x i8>, i1, <vscale x 4 x i1>, i32)
+declare <vscale x 8 x i8> @llvm.vp.abs.nxv8i8(<vscale x 8 x i8>, i1, <vscale x 8 x i1>, i32)
+declare <vscale x 16 x i8> @llvm.vp.abs.nxv16i8(<vscale x 16 x i8>, i1, <vscale x 16 x i1>, i32)
+declare <vscale x 2 x i64> @llvm.vp.abs.nxv2i64(<vscale x 2 x i64>, i1, <vscale x 2 x i1>, i32)
+declare <vscale x 4 x i64> @llvm.vp.abs.nxv4i64(<vscale x 4 x i64>, i1, <vscale x 4 x i1>, i32)
+declare <vscale x 8 x i64> @llvm.vp.abs.nxv8i64(<vscale x 8 x i64>, i1, <vscale x 8 x i1>, i32)
+declare <vscale x 16 x i64> @llvm.vp.abs.nxv16i64(<vscale x 16 x i64>, i1, <vscale x 16 x i1>, i32)
+
+declare <2 x i8> @llvm.abs.v2i8(<2 x i8>, i1)
+declare <4 x i8> @llvm.abs.v4i8(<4 x i8>, i1)
+declare <8 x i8> @llvm.abs.v8i8(<8 x i8>, i1)
+declare <16 x i8> @llvm.abs.v16i8(<16 x i8>, i1)
+declare <2 x i64> @llvm.abs.v2i64(<2 x i64>, i1)
+declare <4 x i64> @llvm.abs.v4i64(<4 x i64>, i1)
+declare <8 x i64> @llvm.abs.v8i64(<8 x i64>, i1)
+declare <16 x i64> @llvm.abs.v16i64(<16 x i64>, i1)
+declare <vscale x 2 x i8> @llvm.abs.nxv2i8(<vscale x 2 x i8>, i1)
+declare <vscale x 4 x i8> @llvm.abs.nxv4i8(<vscale x 4 x i8>, i1)
+declare <vscale x 8 x i8> @llvm.abs.nxv8i8(<vscale x 8 x i8>, i1)
+declare <vscale x 16 x i8> @llvm.abs.nxv16i8(<vscale x 16 x i8>, i1)
+declare <vscale x 2 x i64> @llvm.abs.nxv2i64(<vscale x 2 x i64>, i1)
+declare <vscale x 4 x i64> @llvm.abs.nxv4i64(<vscale x 4 x i64>, i1)
+declare <vscale x 8 x i64> @llvm.abs.nxv8i64(<vscale x 8 x i64>, i1)
+declare <vscale x 16 x i64> @llvm.abs.nxv16i64(<vscale x 16 x i64>, i1)
+
+declare <2 x i8> @llvm.vp.load.v2i8(ptr, <2 x i1>, i32)
+declare <4 x i8> @llvm.vp.load.v4i8(ptr, <4 x i1>, i32)
+declare <8 x i8> @llvm.vp.load.v8i8(ptr, <8 x i1>, i32)
+declare <16 x i8> @llvm.vp.load.v16i8(ptr, <16 x i1>, i32)
+declare <2 x i64> @llvm.vp.load.v2i64(ptr, <2 x i1>, i32)
+declare <4 x i64> @llvm.vp.load.v4i64(ptr, <4 x i1>, i32)
+declare <8 x i64> @llvm.vp.load.v8i64(ptr, <8 x i1>, i32)
+declare <16 x i64> @llvm.vp.load.v16i64(ptr, <16 x i1>, i32)
+declare <vscale x 2 x i8> @llvm.vp.load.nxv2i8(ptr, <vscale x 2 x i1>, i32)
+declare <vscale x 4 x i8> @llvm.vp.load.nxv4i8(ptr, <vscale x 4 x i1>, i32)
+declare <vscale x 8 x i8> @llvm.vp.load.nxv8i8(ptr, <vscale x 8 x i1>, i32)
+declare <vscale x 16 x i8> @llvm.vp.load.nxv16i8(ptr, <vscale x 16 x i1>, i32)
+declare <vscale x 2 x i64> @llvm.vp.load.nxv2i64(ptr, <vscale x 2 x i1>, i32)
+declare <vscale x 4 x i64> @llvm.vp.load.nxv4i64(ptr, <vscale x 4 x i1>, i32)
+declare <vscale x 8 x i64> @llvm.vp.load.nxv8i64(ptr, <vscale x 8 x i1>, i32)
+declare <vscale x 16 x i64> @llvm.vp.load.nxv16i64(ptr, <vscale x 16 x i1>, i32)
+
+declare void @llvm.vp.store.v2i8(<2 x i8>, ptr, <2 x i1>, i32)
+declare void @llvm.vp.store.v4i8(<4 x i8>, ptr, <4 x i1>, i32)
+declare void @llvm.vp.store.v8i8(<8 x i8>, ptr, <8 x i1>, i32)
+declare void @llvm.vp.store.v16i8(<16 x i8>, ptr, <16 x i1>, i32)
+declare void @llvm.vp.store.v2i64(<2 x i64>, ptr, <2 x i1>, i32)
+declare void @llvm.vp.store.v4i64(<4 x i64>, ptr, <4 x i1>, i32)
+declare void @llvm.vp.store.v8i64(<8 x i64>, ptr, <8 x i1>, i32)
+declare void @llvm.vp.store.v16i64(<16 x i64>, ptr, <16 x i1>, i32)
+declare void @llvm.vp.store.nxv2i8(<vscale x 2 x i8>, ptr, <vscale x 2 x i1>, i32)
+declare void @llvm.vp.store.nxv4i8(<vscale x 4 x i8>, ptr, <vscale x 4 x i1>, i32)
+declare void @llvm.vp.store.nxv8i8(<vscale x 8 x i8>, ptr, <vscale x 8 x i1>, i32)
+declare void @llvm.vp.store.nxv16i8(<vscale x 16 x i8>, ptr, <vscale x 16 x i1>, i32)
+declare void @llvm.vp.store.nxv2i64(<vscale x 2 x i64>, ptr, <vscale x 2 x i1>, i32)
+declare void @llvm.vp.store.nxv4i64(<vscale x 4 x i64>, ptr, <vscale x 4 x i1>, i32)
+declare void @llvm.vp.store.nxv8i64(<vscale x 8 x i64>, ptr, <vscale x 8 x i1>, i32)
+declare void @llvm.vp.store.nxv16i64(<vscale x 16 x i64>, ptr, <vscale x 16 x i1>, i32)
+
+declare i8 @llvm.vector.reduce.add.v2i8(<2 x i8>)
+declare i8 @llvm.vector.reduce.add.v4i8(<4 x i8>)
+declare i8 @llvm.vector.reduce.add.v8i8(<8 x i8>)
+declare i8 @llvm.vector.reduce.add.v16i8(<16 x i8>)
+declare i64 @llvm.vector.reduce.add.v2i64(<2 x i64>)
+declare i64 @llvm.vector.reduce.add.v4i64(<4 x i64>)
+declare i64 @llvm.vector.reduce.add.v8i64(<8 x i64>)
+declare i64 @llvm.vector.reduce.add.v16i64(<16 x i64>)
+declare i8 @llvm.vector.reduce.add.nxv2i8(<vscale x 2 x i8>)
+declare i8 @llvm.vector.reduce.add.nxv4i8(<vscale x 4 x i8>)
+declare i8 @llvm.vector.reduce.add.nxv8i8(<vscale x 8 x i8>)
+declare i8 @llvm.vector.reduce.add.nxv16i8(<vscale x 16 x i8>)
+declare i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64>)
+declare i64 @llvm.vector.reduce.add.nxv4i64(<vscale x 4 x i64>)
+declare i64 @llvm.vector.reduce.add.nxv8i64(<vscale x 8 x i64>)
+declare i64 @llvm.vector.reduce.add.nxv16i64(<vscale x 16 x i64>)
+
+declare i8 @llvm.vp.reduce.add.v2i8(i8, <2 x i8>, <2 x i1>, i32)
+declare i8 @llvm.vp.reduce.add.v4i8(i8, <4 x i8>, <4 x i1>, i32)
+declare i8 @llvm.vp.reduce.add.v8i8(i8, <8 x i8>, <8 x i1>, i32)
+declare i8 @llvm.vp.reduce.add.v16i8(i8, <16 x i8>, <16 x i1>, i32)
+declare i64 @llvm.vp.reduce.add.v2i64(i64, <2 x i64>, <2 x i1>, i32)
+declare i64 @llvm.vp.reduce.add.v4i64(i64, <4 x i64>, <4 x i1>, i32)
+declare i64 @llvm.vp.reduce.add.v8i64(i64, <8 x i64>, <8 x i1>, i32)
+declare i64 @llvm.vp.reduce.add.v16i64(i64, <16 x i64>, <16 x i1>, i32)
+declare i8 @llvm.vp.reduce.add.nxv2i8(i8, <vscale x 2 x i8>, <vscale x 2 x i1>, i32)
+declare i8 @llvm.vp.reduce.add.nxv4i8(i8, <vscale x 4 x i8>, <vscale x 4 x i1>, i32)
+declare i8 @llvm.vp.reduce.add.nxv8i8(i8, <vscale x 8 x i8>, <vscale x 8 x i1>, i32)
+declare i8 @llvm.vp.reduce.add.nxv16i8(i8, <vscale x 16 x i8>, <vscale x 16 x i1>, i32)
+declare i64 @llvm.vp.reduce.add.nxv2i64(i64, <vscale x 2 x i64>, <vscale x 2 x i1>, i32)
+declare i64 @llvm.vp.reduce.add.nxv4i64(i64, <vscale x 4 x i64>, <vscale x 4 x i1>, i32)
+declare i64 @llvm.vp.reduce.add.nxv8i64(i64, <vscale x 8 x i64>, <vscale x 8 x i1>, i32)
+declare i64 @llvm.vp.reduce.add.nxv16i64(i64, <vscale x 16 x i64>, <vscale x 16 x i1>, i32)
+
+declare float @llvm.vector.reduce.fadd.v2f32(float, <2 x float>)
+declare float @llvm.vector.reduce.fadd.v4f32(float, <4 x float>)
+declare float @llvm.vector.reduce.fadd.v8f32(float, <8 x float>)
+declare float @llvm.vector.reduce.fadd.v16f32(float, <16 x float>)
+declare double @llvm.vector.reduce.fadd.v2f64(double, <2 x double>)
+declare double @llvm.vector.reduce.fadd.v4f64(double, <4 x double>)
+declare double @llvm.vector.reduce.fadd.v8f64(double, <8 x double>)
+declare double @llvm.vector.reduce.fadd.v16f64(double, <16 x double>)
+declare float @llvm.vector.reduce.fadd.nxv2f32(float, <vscale x 2 x float>)
+declare float @llvm.vector.reduce.fadd.nxv4f32(float, <vscale x 4 x float>)
+declare float @llvm.vector.reduce.fadd.nxv8f32(float, <vscale x 8 x float>)
+declare float @llvm.vector.reduce.fadd.nxv16f32(float, <vscale x 16 x float>)
+declare double @llvm.vector.reduce.fadd.nxv2f64(double, <vscale x 2 x double>)
+declare double @llvm.vector.reduce.fadd.nxv4f64(double, <vscale x 4 x double>)
+declare double @llvm.vector.reduce.fadd.nxv8f64(double, <vscale x 8 x double>)
+declare double @llvm.vector.reduce.fadd.nxv16f64(double, <vscale x 16 x double>)
+
+declare float @llvm.vp.reduce.fadd.v2f32(float, <2 x float>, <2 x i1>, i32)
+declare float @llvm.vp.reduce.fadd.v4f32(float, <4 x float>, <4 x i1>, i32)
+declare float @llvm.vp.reduce.fadd.v8f32(float, <8 x float>, <8 x i1>, i32)
+declare float @llvm.vp.reduce.fadd.v16f32(float, <16 x float>, <16 x i1>, i32)
+declare double @llvm.vp.reduce.fadd.v2f64(double, <2 x double>, <2 x i1>, i32)
+declare double @llvm.vp.reduce.fadd.v4f64(double, <4 x double>, <4 x i1>, i32)
+declare double @llvm.vp.reduce.fadd.v8f64(double, <8 x double>, <8 x i1>, i32)
+declare double @llvm.vp.reduce.fadd.v16f64(double, <16 x double>, <16 x i1>, i32)
+declare float @llvm.vp.reduce.fadd.nxv2f32(float, <vscale x 2 x float>, <vscale x 2 x i1>, i32)
+declare float @llvm.vp.reduce.fadd.nxv4f32(float, <vscale x 4 x float>, <vscale x 4 x i1>, i32)
+declare float @llvm.vp.reduce.fadd.nxv8f32(float, <vscale x 8 x float>, <vscale x 8 x i1>, i32)
+declare float @llvm.vp.reduce.fadd.nxv16f32(float, <vscale x 16 x float>, <vscale x 16 x i1>, i32)
+declare double @llvm.vp.reduce.fadd.nxv2f64(double, <vscale x 2 x double>, <vscale x 2 x i1>, i32)
+declare double @llvm.vp.reduce.fadd.nxv4f64(double, <vscale x 4 x double>, <vscale x 4 x i1>, i32)
+declare double @llvm.vp.reduce.fadd.nxv8f64(double, <vscale x 8 x double>, <vscale x 8 x i1>, i32)
+declare double @llvm.vp.reduce.fadd.nxv16f64(double, <vscale x 16 x double>, <vscale x 16 x i1>, i32)
+
 declare <vscale x 1 x i32> @llvm.fshr.nxv4i32(<vscale x 1 x i32> %a, <vscale x 1 x i32> %b, <vscale x 1 x i32> %c)
 declare <vscale x 1 x i32> @llvm.fshl.nxv4i32(<vscale x 1 x i32> %a, <vscale x 1 x i32> %b, <vscale x 1 x i32> %c)
 
-
 declare <vscale x 4 x float> @llvm.pow.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>)
 declare <vscale x 4 x float> @llvm.powi.nxv4f32.i32(<vscale x 4 x float>, i32)
 declare <vscale x 4 x float> @llvm.nearbyint.nxv4f32(<vscale x 4 x float>)
diff --git a/llvm/test/Analysis/ValueTracking/constant-ranges.ll b/llvm/test/Analysis/ValueTracking/constant-ranges.ll
new file mode 100644
index 0000000000..26e01efedd
--- /dev/null
+++ b/llvm/test/Analysis/ValueTracking/constant-ranges.ll
@@ -0,0 +1,132 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes=instsimplify < %s -S | FileCheck %s
+
+define i1 @shl_C_X_ugt(i8 %x) {
+; CHECK-LABEL: @shl_C_X_ugt(
+; CHECK-NEXT:    ret i1 false
+;
+  %shl = shl i8 7, %x
+  %r = icmp ugt i8 %shl, 224
+  ret i1 %r
+}
+
+define i1 @shl_C_X_ugt2(i8 %x) {
+; CHECK-LABEL: @shl_C_X_ugt2(
+; CHECK-NEXT:    ret i1 false
+;
+  %shl = shl i8 5, %x
+  %r = icmp ugt i8 %shl, 192
+  ret i1 %r
+}
+
+define i1 @shl_C_X_ugt_fail(i8 %x) {
+; CHECK-LABEL: @shl_C_X_ugt_fail(
+; CHECK-NEXT:    [[SHL:%.*]] = shl i8 1, [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[SHL]], 127
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %shl = shl i8 1, %x
+  %r = icmp ugt i8 %shl, 127
+  ret i1 %r
+}
+
+define i1 @shl_C_X_ugt_fail2(i8 %x) {
+; CHECK-LABEL: @shl_C_X_ugt_fail2(
+; CHECK-NEXT:    [[SHL:%.*]] = shl i8 3, [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[SHL]], -66
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %shl = shl i8 3, %x
+  %r = icmp ugt i8 %shl, 190
+  ret i1 %r
+}
+
+define i1 @shl_C_X_ugt_fail3(i8 %x) {
+; CHECK-LABEL: @shl_C_X_ugt_fail3(
+; CHECK-NEXT:    [[SHL:%.*]] = shl i8 -1, [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[SHL]], -2
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %shl = shl i8 -1, %x
+  %r = icmp ugt i8 %shl, 254
+  ret i1 %r
+}
+
+define i1 @shl_C_X_ugt_todo(i8 %x) {
+; CHECK-LABEL: @shl_C_X_ugt_todo(
+; CHECK-NEXT:    [[SHL:%.*]] = shl i8 -127, [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[SHL]], -116
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %shl = shl i8 129, %x
+  %r = icmp ugt i8 %shl, 140
+  ret i1 %r
+}
+
+define i1 @shl_X_C_ugt(i8 %x) {
+; CHECK-LABEL: @shl_X_C_ugt(
+; CHECK-NEXT:    ret i1 false
+;
+  %shl = shl i8 %x, 6
+  %r = icmp ugt i8 %shl, 192
+  ret i1 %r
+}
+
+define i1 @shl_X_C_ugt_fail(i8 %x) {
+; CHECK-LABEL: @shl_X_C_ugt_fail(
+; CHECK-NEXT:    [[SHL:%.*]] = shl i8 [[X:%.*]], 6
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[SHL]], -65
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %shl = shl i8 %x, 6
+  %r = icmp ugt i8 %shl, 191
+  ret i1 %r
+}
+
+define i1 @shl_X_C_ugt_fail2(i8 %x) {
+; CHECK-LABEL: @shl_X_C_ugt_fail2(
+; CHECK-NEXT:    [[SHL:%.*]] = shl i8 [[X:%.*]], 5
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[SHL]], -64
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %shl = shl i8 %x, 5
+  %r = icmp ugt i8 %shl, 192
+  ret i1 %r
+}
+
+define i1 @and_ugt(i8 %xx) {
+; CHECK-LABEL: @and_ugt(
+; CHECK-NEXT:    ret i1 false
+;
+  %x = mul i8 %xx, %xx  ; thwart complexity-based canonicalization
+  %negx = sub i8 0, %x
+  %x_p2 = and i8 %negx, %x
+  %r = icmp ugt i8 %x_p2, 128
+  ret i1 %r
+}
+
+define i1 @and_ugt2(i8 %xx) {
+; CHECK-LABEL: @and_ugt2(
+; CHECK-NEXT:    ret i1 false
+;
+  %x = mul i8 %xx, %xx  ; thwart complexity-based canonicalization
+  %negx = sub i8 0, %x
+  %x_p2 = and i8 %x, %negx
+  %r = icmp ugt i8 %x_p2, 128
+  ret i1 %r
+}
+
+define i1 @and_ugt_fail(i8 %xx) {
+; CHECK-LABEL: @and_ugt_fail(
+; CHECK-NEXT:    [[X:%.*]] = mul i8 [[XX:%.*]], [[XX]]
+; CHECK-NEXT:    [[NEGX:%.*]] = sub i8 0, [[X]]
+; CHECK-NEXT:    [[X_P2:%.*]] = and i8 [[X]], [[NEGX]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[X_P2]], 127
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %x = mul i8 %xx, %xx  ; thwart complexity-based canonicalization
+  %negx = sub i8 0, %x
+  %x_p2 = and i8 %x, %negx
+  %r = icmp ugt i8 %x_p2, 127
+  ret i1 %r
+}
diff --git a/llvm/test/Analysis/ValueTracking/known-non-zero.ll b/llvm/test/Analysis/ValueTracking/known-non-zero.ll
index 6dce6e5281..dbec47ea0a 100644
--- a/llvm/test/Analysis/ValueTracking/known-non-zero.ll
+++ b/llvm/test/Analysis/ValueTracking/known-non-zero.ll
@@ -1160,3 +1160,61 @@ define i1 @sdiv_known_non_zero_fail(i8 %x, i8 %y) {
   %nz = icmp ne i8 %xy, 0
   ret i1 %nz
 }
+
+define <2 x i1> @cmp_excludes_zero_with_nonsplat_vec(<2 x i8> %a, <2 x i8> %b) {
+; CHECK-LABEL: @cmp_excludes_zero_with_nonsplat_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %c = icmp sge <2 x i8> %a, <i8 1, i8 4>
+  %s = select <2 x i1> %c, <2 x i8> %a, <2 x i8> <i8 4, i8 5>
+  %and = or <2 x i8> %s, %b
+  %r = icmp eq <2 x i8> %and, zeroinitializer
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @cmp_excludes_zero_with_nonsplat_vec_wundef(<2 x i8> %a, <2 x i8> %b) {
+; CHECK-LABEL: @cmp_excludes_zero_with_nonsplat_vec_wundef(
+; CHECK-NEXT:    [[C:%.*]] = icmp sge <2 x i8> [[A:%.*]], <i8 1, i8 undef>
+; CHECK-NEXT:    [[S:%.*]] = select <2 x i1> [[C]], <2 x i8> [[A]], <2 x i8> <i8 4, i8 5>
+; CHECK-NEXT:    [[AND:%.*]] = or <2 x i8> [[S]], [[B:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq <2 x i8> [[AND]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %c = icmp sge <2 x i8> %a, <i8 1, i8 undef>
+  %s = select <2 x i1> %c, <2 x i8> %a, <2 x i8> <i8 4, i8 5>
+  %and = or <2 x i8> %s, %b
+  %r = icmp eq <2 x i8> %and, zeroinitializer
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @cmp_excludes_zero_with_nonsplat_vec_wpoison(<2 x i8> %a, <2 x i8> %b) {
+; CHECK-LABEL: @cmp_excludes_zero_with_nonsplat_vec_wpoison(
+; CHECK-NEXT:    [[C:%.*]] = icmp sge <2 x i8> [[A:%.*]], <i8 1, i8 poison>
+; CHECK-NEXT:    [[S:%.*]] = select <2 x i1> [[C]], <2 x i8> [[A]], <2 x i8> <i8 4, i8 5>
+; CHECK-NEXT:    [[AND:%.*]] = or <2 x i8> [[S]], [[B:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq <2 x i8> [[AND]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %c = icmp sge <2 x i8> %a, <i8 1, i8 poison>
+  %s = select <2 x i1> %c, <2 x i8> %a, <2 x i8> <i8 4, i8 5>
+  %and = or <2 x i8> %s, %b
+  %r = icmp eq <2 x i8> %and, zeroinitializer
+  ret <2 x i1> %r
+}
+
+
+define <2 x i1> @cmp_excludes_zero_with_nonsplat_vec_fail(<2 x i8> %a, <2 x i8> %b) {
+; CHECK-LABEL: @cmp_excludes_zero_with_nonsplat_vec_fail(
+; CHECK-NEXT:    [[C:%.*]] = icmp sge <2 x i8> [[A:%.*]], <i8 0, i8 4>
+; CHECK-NEXT:    [[S:%.*]] = select <2 x i1> [[C]], <2 x i8> [[A]], <2 x i8> <i8 4, i8 5>
+; CHECK-NEXT:    [[AND:%.*]] = or <2 x i8> [[S]], [[B:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq <2 x i8> [[AND]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %c = icmp sge <2 x i8> %a, <i8 0, i8 4>
+  %s = select <2 x i1> %c, <2 x i8> %a, <2 x i8> <i8 4, i8 5>
+  %and = or <2 x i8> %s, %b
+  %r = icmp eq <2 x i8> %and, zeroinitializer
+  ret <2 x i1> %r
+}
+
diff --git a/llvm/test/Analysis/ValueTracking/known-power-of-two.ll b/llvm/test/Analysis/ValueTracking/known-power-of-two.ll
index b86cf59fa2..7bcf96065a 100644
--- a/llvm/test/Analysis/ValueTracking/known-power-of-two.ll
+++ b/llvm/test/Analysis/ValueTracking/known-power-of-two.ll
@@ -413,11 +413,11 @@ define i1 @mul_is_pow2(i16 %x, i16 %y, i16 %z) {
 ; CHECK-SAME: (i16 [[X:%.*]], i16 [[Y:%.*]], i16 [[Z:%.*]]) {
 ; CHECK-NEXT:    [[XSMALL:%.*]] = and i16 [[X]], 3
 ; CHECK-NEXT:    [[ZSMALL:%.*]] = and i16 [[Z]], 3
-; CHECK-NEXT:    [[XP2:%.*]] = shl i16 4, [[XSMALL]]
-; CHECK-NEXT:    [[ZP2:%.*]] = shl i16 2, [[ZSMALL]]
-; CHECK-NEXT:    [[XX:%.*]] = mul nuw nsw i16 [[XP2]], [[ZP2]]
+; CHECK-NEXT:    [[ZP2:%.*]] = shl nuw nsw i16 2, [[ZSMALL]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add nuw nsw i16 [[XSMALL]], 2
+; CHECK-NEXT:    [[XX:%.*]] = shl nuw nsw i16 [[ZP2]], [[TMP1]]
 ; CHECK-NEXT:    [[AND:%.*]] = and i16 [[XX]], [[Y]]
-; CHECK-NEXT:    [[R:%.*]] = icmp eq i16 [[AND]], [[XX]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i16 [[AND]], 0
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %xsmall = and i16 %x, 3
@@ -436,9 +436,9 @@ define i1 @mul_is_pow2_fail(i16 %x, i16 %y, i16 %z) {
 ; CHECK-SAME: (i16 [[X:%.*]], i16 [[Y:%.*]], i16 [[Z:%.*]]) {
 ; CHECK-NEXT:    [[XSMALL:%.*]] = and i16 [[X]], 7
 ; CHECK-NEXT:    [[ZSMALL:%.*]] = and i16 [[Z]], 7
-; CHECK-NEXT:    [[XP2:%.*]] = shl i16 4, [[XSMALL]]
-; CHECK-NEXT:    [[ZP2:%.*]] = shl i16 2, [[ZSMALL]]
-; CHECK-NEXT:    [[XX:%.*]] = mul i16 [[XP2]], [[ZP2]]
+; CHECK-NEXT:    [[ZP2:%.*]] = shl nuw nsw i16 2, [[ZSMALL]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add nuw nsw i16 [[XSMALL]], 2
+; CHECK-NEXT:    [[XX:%.*]] = shl i16 [[ZP2]], [[TMP1]]
 ; CHECK-NEXT:    [[AND:%.*]] = and i16 [[XX]], [[Y]]
 ; CHECK-NEXT:    [[R:%.*]] = icmp eq i16 [[AND]], [[XX]]
 ; CHECK-NEXT:    ret i1 [[R]]
@@ -459,9 +459,9 @@ define i1 @mul_is_pow2_fail2(i16 %x, i16 %y, i16 %z) {
 ; CHECK-SAME: (i16 [[X:%.*]], i16 [[Y:%.*]], i16 [[Z:%.*]]) {
 ; CHECK-NEXT:    [[XSMALL:%.*]] = and i16 [[X]], 3
 ; CHECK-NEXT:    [[ZSMALL:%.*]] = and i16 [[Z]], 3
-; CHECK-NEXT:    [[XP2:%.*]] = shl i16 3, [[XSMALL]]
-; CHECK-NEXT:    [[ZP2:%.*]] = shl i16 2, [[ZSMALL]]
-; CHECK-NEXT:    [[XX:%.*]] = mul nuw nsw i16 [[XP2]], [[ZP2]]
+; CHECK-NEXT:    [[XP2:%.*]] = shl nuw nsw i16 3, [[XSMALL]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add nuw nsw i16 [[ZSMALL]], 1
+; CHECK-NEXT:    [[XX:%.*]] = shl nuw nsw i16 [[XP2]], [[TMP1]]
 ; CHECK-NEXT:    [[AND:%.*]] = and i16 [[XX]], [[Y]]
 ; CHECK-NEXT:    [[R:%.*]] = icmp eq i16 [[AND]], [[XX]]
 ; CHECK-NEXT:    ret i1 [[R]]
@@ -481,9 +481,9 @@ define i1 @shl_is_pow2(i16 %x, i16 %y) {
 ; CHECK-LABEL: define i1 @shl_is_pow2
 ; CHECK-SAME: (i16 [[X:%.*]], i16 [[Y:%.*]]) {
 ; CHECK-NEXT:    [[XSMALL:%.*]] = and i16 [[X]], 7
-; CHECK-NEXT:    [[XX:%.*]] = shl i16 4, [[XSMALL]]
+; CHECK-NEXT:    [[XX:%.*]] = shl nuw nsw i16 4, [[XSMALL]]
 ; CHECK-NEXT:    [[AND:%.*]] = and i16 [[XX]], [[Y]]
-; CHECK-NEXT:    [[R:%.*]] = icmp eq i16 [[AND]], [[XX]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i16 [[AND]], 0
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %xsmall = and i16 %x, 7
@@ -515,7 +515,7 @@ define i1 @shl_is_pow2_fail2(i16 %x, i16 %y) {
 ; CHECK-LABEL: define i1 @shl_is_pow2_fail2
 ; CHECK-SAME: (i16 [[X:%.*]], i16 [[Y:%.*]]) {
 ; CHECK-NEXT:    [[XSMALL:%.*]] = and i16 [[X]], 7
-; CHECK-NEXT:    [[XX:%.*]] = shl i16 5, [[XSMALL]]
+; CHECK-NEXT:    [[XX:%.*]] = shl nuw nsw i16 5, [[XSMALL]]
 ; CHECK-NEXT:    [[AND:%.*]] = and i16 [[XX]], [[Y]]
 ; CHECK-NEXT:    [[R:%.*]] = icmp eq i16 [[AND]], [[XX]]
 ; CHECK-NEXT:    ret i1 [[R]]
@@ -532,9 +532,9 @@ define i1 @lshr_is_pow2(i16 %x, i16 %y) {
 ; CHECK-LABEL: define i1 @lshr_is_pow2
 ; CHECK-SAME: (i16 [[X:%.*]], i16 [[Y:%.*]]) {
 ; CHECK-NEXT:    [[XSMALL:%.*]] = and i16 [[X]], 7
-; CHECK-NEXT:    [[XX:%.*]] = lshr i16 512, [[XSMALL]]
+; CHECK-NEXT:    [[XX:%.*]] = lshr exact i16 512, [[XSMALL]]
 ; CHECK-NEXT:    [[AND:%.*]] = and i16 [[XX]], [[Y]]
-; CHECK-NEXT:    [[R:%.*]] = icmp eq i16 [[AND]], [[XX]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i16 [[AND]], 0
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %xsmall = and i16 %x, 7
@@ -584,9 +584,9 @@ define i1 @and_is_pow2(i16 %x, i16 %y) {
 ; CHECK-SAME: (i16 [[X:%.*]], i16 [[Y:%.*]]) {
 ; CHECK-NEXT:    [[XNZ:%.*]] = or i16 [[X]], 4
 ; CHECK-NEXT:    [[X_NEG:%.*]] = sub nsw i16 0, [[XNZ]]
-; CHECK-NEXT:    [[XX:%.*]] = and i16 [[XNZ]], [[X_NEG]]
-; CHECK-NEXT:    [[AND:%.*]] = and i16 [[XX]], [[Y]]
-; CHECK-NEXT:    [[R:%.*]] = icmp eq i16 [[AND]], [[XX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i16 [[X_NEG]], [[Y]]
+; CHECK-NEXT:    [[AND:%.*]] = and i16 [[TMP1]], [[XNZ]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i16 [[AND]], 0
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %xnz = or i16 %x, 4
diff --git a/llvm/test/Analysis/ValueTracking/select-known-non-zero.ll b/llvm/test/Analysis/ValueTracking/select-known-non-zero.ll
index 8b1d2fd018..1dc8841204 100644
--- a/llvm/test/Analysis/ValueTracking/select-known-non-zero.ll
+++ b/llvm/test/Analysis/ValueTracking/select-known-non-zero.ll
@@ -393,3 +393,57 @@ define i1 @inv_select_v_sle_nonneg(i8 %v, i8 %C, i8 %y) {
   %r = icmp eq i8 %s, 0
   ret i1 %r
 }
+
+; Check udiv/sdiv occuring before icmp.
+define i64 @incorrect_safe_div_1(i64 %n, i64 %d) {
+; CHECK-LABEL: @incorrect_safe_div_1(
+; CHECK-NEXT:    [[TMP1:%.*]] = udiv i64 [[N:%.*]], [[D:%.*]]
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %1 = udiv i64 %n, %d
+  %2 = icmp eq i64 %d, 0
+  %3 = select i1 %2, i64 -1, i64 %1
+  ret i64 %3
+}
+
+; Check icmp occuring before udiv/sdiv.
+define i64 @incorrect_safe_div_2(i64 %n, i64 %d) {
+; CHECK-LABEL: @incorrect_safe_div_2(
+; CHECK-NEXT:    [[TMP1:%.*]] = sdiv i64 [[N:%.*]], [[D:%.*]]
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %1 = icmp eq i64 %d, 0
+  %2 = sdiv i64 %n, %d
+  %3 = select i1 %1, i64 -1, i64 %2
+  ret i64 %3
+}
+
+define i64 @incorrect_safe_div_call_1(i64 %n, i64 %d) {
+; CHECK-LABEL: @incorrect_safe_div_call_1(
+; CHECK-NEXT:    [[TMP1:%.*]] = sdiv i64 [[N:%.*]], [[D:%.*]]
+; CHECK-NEXT:    tail call void @use(i64 [[D]])
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %1 = sdiv i64 %n, %d
+  tail call void @use(i64 %d)
+  %2 = icmp eq i64 %d, 0
+  %3 = select i1 %2, i64 -1, i64 %1
+  ret i64 %3
+}
+
+define i64 @incorrect_safe_div_call_2(i64 %n, i64 %d) {
+; CHECK-LABEL: @incorrect_safe_div_call_2(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i64 [[D:%.*]], 0
+; CHECK-NEXT:    tail call void @use(i64 [[D]])
+; CHECK-NEXT:    [[TMP2:%.*]] = udiv i64 [[N:%.*]], [[D]]
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i64 -1, i64 [[TMP2]]
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+  %1 = icmp eq i64 %d, 0
+  tail call void @use(i64 %d)
+  %2 = udiv i64 %n, %d
+  %3 = select i1 %1, i64 -1, i64 %2
+  ret i64 %3
+}
+
+declare void @use(i64)
diff --git a/llvm/test/Bindings/OCaml/core.ml b/llvm/test/Bindings/OCaml/core.ml
index 3a9c3c8423..1f6ece9532 100644
--- a/llvm/test/Bindings/OCaml/core.ml
+++ b/llvm/test/Bindings/OCaml/core.ml
@@ -71,11 +71,11 @@ let test_target () =
   end;
 
   begin group "layout";
-    let layout = "e" in
+    let layout = "e-m:o-p:32:32-p270:32:32-p271:32:32-p272:64:64-i128:128-f64:32:64-f80:128-n8:16:32-S128" in
     set_data_layout layout m;
     insist (layout = data_layout m)
   end
-  (* CHECK: target datalayout = "e"
+  (* CHECK: target datalayout = "e-m:o-p:32:32-p270:32:32-p271:32:32-p272:64:64-i128:128-f64:32:64-f80:128-n8:16:32-S128"
    * CHECK: target triple = "i686-apple-darwin8"
    *)
 
diff --git a/llvm/test/Bitcode/attributes.ll b/llvm/test/Bitcode/attributes.ll
index 9af648fe26..eaf670575f 100644
--- a/llvm/test/Bitcode/attributes.ll
+++ b/llvm/test/Bitcode/attributes.ll
@@ -511,6 +511,12 @@ define void @f87() fn_ret_thunk_extern { ret void }
 ; CHECK: define void @f88() [[SKIPPROFILE:#[0-9]+]]
 define void @f88() skipprofile { ret void }
 
+define void @f89() optdebug
+; CHECK: define void @f89() [[OPTDEBUG:#[0-9]+]]
+{
+        ret void;
+}
+
 ; CHECK: attributes #0 = { noreturn }
 ; CHECK: attributes #1 = { nounwind }
 ; CHECK: attributes #2 = { memory(none) }
@@ -566,4 +572,5 @@ define void @f88() skipprofile { ret void }
 ; CHECK: attributes #52 = { nosanitize_bounds }
 ; CHECK: attributes [[FNRETTHUNKEXTERN]] = { fn_ret_thunk_extern }
 ; CHECK: attributes [[SKIPPROFILE]] = { skipprofile }
+; CHECK: attributes [[OPTDEBUG]] = { optdebug }
 ; CHECK: attributes #[[NOBUILTIN]] = { nobuiltin }
diff --git a/llvm/test/Bitcode/upgrade-datalayout.ll b/llvm/test/Bitcode/upgrade-datalayout.ll
index 8c00294f2b..dc2c459cd1 100644
--- a/llvm/test/Bitcode/upgrade-datalayout.ll
+++ b/llvm/test/Bitcode/upgrade-datalayout.ll
@@ -5,5 +5,5 @@
 target datalayout = "e-m:e-p:32:32-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
-; CHECK: target datalayout = "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+; CHECK: target datalayout = "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Bitcode/upgrade-datalayout2.ll b/llvm/test/Bitcode/upgrade-datalayout2.ll
index 21de5b8a67..b516c08fec 100644
--- a/llvm/test/Bitcode/upgrade-datalayout2.ll
+++ b/llvm/test/Bitcode/upgrade-datalayout2.ll
@@ -2,6 +2,12 @@
 ; match a possible x86 datalayout.
 ;
 ; RUN: llvm-as %s -o - | llvm-dis - | FileCheck %s
+;
+; XFAIL: *
+; No implementation of the data layout upgrade ever checked whether the data
+; layout was a possible x86 data layout, so the logic that this test aims to
+; check was never implemented. We always upgraded data layouts that were not
+; possible x86 data layouts, we merely did not previously upgrade this one.
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Bitcode/upgrade-datalayout3.ll b/llvm/test/Bitcode/upgrade-datalayout3.ll
index 6d95f2407a..f59097b57b 100644
--- a/llvm/test/Bitcode/upgrade-datalayout3.ll
+++ b/llvm/test/Bitcode/upgrade-datalayout3.ll
@@ -5,4 +5,4 @@
 target datalayout = "e-m:w-p:32:32-i64:64-f80:32-n8:16:32-S32"
 target triple = "i686-pc-windows-msvc"
 
-; CHECK: target datalayout = "e-m:w-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32-S32"
+; CHECK: target datalayout = "e-m:w-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32-S32"
diff --git a/llvm/test/Bitcode/upgrade-datalayout4.ll b/llvm/test/Bitcode/upgrade-datalayout4.ll
index ee0e5fe3bf..026247bcce 100644
--- a/llvm/test/Bitcode/upgrade-datalayout4.ll
+++ b/llvm/test/Bitcode/upgrade-datalayout4.ll
@@ -5,4 +5,4 @@
 target datalayout = "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:32-n8:16:32-a:0:32-S32"
 target triple = "i686-pc-windows-msvc"
 
-; CHECK: target datalayout = "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32-a:0:32-S32"
+; CHECK: target datalayout = "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32-a:0:32-S32"
diff --git a/llvm/test/Bitcode/upgrade-datalayout5.ll b/llvm/test/Bitcode/upgrade-datalayout5.ll
new file mode 100644
index 0000000000..2b1d5d1467
--- /dev/null
+++ b/llvm/test/Bitcode/upgrade-datalayout5.ll
@@ -0,0 +1,8 @@
+; Test to make sure datalayout is automatically upgraded.
+;
+; RUN: llvm-as %s -o - | llvm-dis - | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128"
+target triple = "i386-pc-linux-gnu"
+
+; CHECK: target datalayout = "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-i128:128-f64:32:64-f80:32-n8:16:32-S128"
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-fma.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-fma.mir
index 3388ab97dc..d344511010 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-fma.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-fma.mir
@@ -13,43 +13,27 @@ body:             |
 
     ; NO-FP16-LABEL: name: test_v4f16.fma
     ; NO-FP16: liveins: $d0, $d1, $d2
-    ; NO-FP16: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $d0
-    ; NO-FP16: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $d1
-    ; NO-FP16: [[COPY2:%[0-9]+]]:_(<4 x s16>) = COPY $d2
-    ; NO-FP16: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
-    ; NO-FP16: [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
-    ; NO-FP16: [[UV8:%[0-9]+]]:_(s16), [[UV9:%[0-9]+]]:_(s16), [[UV10:%[0-9]+]]:_(s16), [[UV11:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY2]](<4 x s16>)
-    ; NO-FP16: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[UV]](s16)
-    ; NO-FP16: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[UV4]](s16)
-    ; NO-FP16: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[UV8]](s16)
-    ; NO-FP16: [[FMA:%[0-9]+]]:_(s32) = G_FMA [[FPEXT]], [[FPEXT1]], [[FPEXT2]]
-    ; NO-FP16: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMA]](s32)
-    ; NO-FP16: [[FPEXT3:%[0-9]+]]:_(s32) = G_FPEXT [[UV1]](s16)
-    ; NO-FP16: [[FPEXT4:%[0-9]+]]:_(s32) = G_FPEXT [[UV5]](s16)
-    ; NO-FP16: [[FPEXT5:%[0-9]+]]:_(s32) = G_FPEXT [[UV9]](s16)
-    ; NO-FP16: [[FMA1:%[0-9]+]]:_(s32) = G_FMA [[FPEXT3]], [[FPEXT4]], [[FPEXT5]]
-    ; NO-FP16: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMA1]](s32)
-    ; NO-FP16: [[FPEXT6:%[0-9]+]]:_(s32) = G_FPEXT [[UV2]](s16)
-    ; NO-FP16: [[FPEXT7:%[0-9]+]]:_(s32) = G_FPEXT [[UV6]](s16)
-    ; NO-FP16: [[FPEXT8:%[0-9]+]]:_(s32) = G_FPEXT [[UV10]](s16)
-    ; NO-FP16: [[FMA2:%[0-9]+]]:_(s32) = G_FMA [[FPEXT6]], [[FPEXT7]], [[FPEXT8]]
-    ; NO-FP16: [[FPTRUNC2:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMA2]](s32)
-    ; NO-FP16: [[FPEXT9:%[0-9]+]]:_(s32) = G_FPEXT [[UV3]](s16)
-    ; NO-FP16: [[FPEXT10:%[0-9]+]]:_(s32) = G_FPEXT [[UV7]](s16)
-    ; NO-FP16: [[FPEXT11:%[0-9]+]]:_(s32) = G_FPEXT [[UV11]](s16)
-    ; NO-FP16: [[FMA3:%[0-9]+]]:_(s32) = G_FMA [[FPEXT9]], [[FPEXT10]], [[FPEXT11]]
-    ; NO-FP16: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMA3]](s32)
-    ; NO-FP16: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[FPTRUNC]](s16), [[FPTRUNC1]](s16), [[FPTRUNC2]](s16), [[FPTRUNC3]](s16)
-    ; NO-FP16: $d0 = COPY [[BUILD_VECTOR]](<4 x s16>)
-    ; NO-FP16: RET_ReallyLR implicit $d0
+    ; NO-FP16-NEXT: {{  $}}
+    ; NO-FP16-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $d0
+    ; NO-FP16-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $d1
+    ; NO-FP16-NEXT: [[COPY2:%[0-9]+]]:_(<4 x s16>) = COPY $d2
+    ; NO-FP16-NEXT: [[FPEXT:%[0-9]+]]:_(<4 x s32>) = G_FPEXT [[COPY]](<4 x s16>)
+    ; NO-FP16-NEXT: [[FPEXT1:%[0-9]+]]:_(<4 x s32>) = G_FPEXT [[COPY1]](<4 x s16>)
+    ; NO-FP16-NEXT: [[FPEXT2:%[0-9]+]]:_(<4 x s32>) = G_FPEXT [[COPY2]](<4 x s16>)
+    ; NO-FP16-NEXT: [[FMA:%[0-9]+]]:_(<4 x s32>) = G_FMA [[FPEXT]], [[FPEXT1]], [[FPEXT2]]
+    ; NO-FP16-NEXT: [[FPTRUNC:%[0-9]+]]:_(<4 x s16>) = G_FPTRUNC [[FMA]](<4 x s32>)
+    ; NO-FP16-NEXT: $d0 = COPY [[FPTRUNC]](<4 x s16>)
+    ; NO-FP16-NEXT: RET_ReallyLR implicit $d0
+    ;
     ; FP16-LABEL: name: test_v4f16.fma
     ; FP16: liveins: $d0, $d1, $d2
-    ; FP16: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $d0
-    ; FP16: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $d1
-    ; FP16: [[COPY2:%[0-9]+]]:_(<4 x s16>) = COPY $d2
-    ; FP16: [[FMA:%[0-9]+]]:_(<4 x s16>) = G_FMA [[COPY]], [[COPY1]], [[COPY2]]
-    ; FP16: $d0 = COPY [[FMA]](<4 x s16>)
-    ; FP16: RET_ReallyLR implicit $d0
+    ; FP16-NEXT: {{  $}}
+    ; FP16-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $d0
+    ; FP16-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $d1
+    ; FP16-NEXT: [[COPY2:%[0-9]+]]:_(<4 x s16>) = COPY $d2
+    ; FP16-NEXT: [[FMA:%[0-9]+]]:_(<4 x s16>) = G_FMA [[COPY]], [[COPY1]], [[COPY2]]
+    ; FP16-NEXT: $d0 = COPY [[FMA]](<4 x s16>)
+    ; FP16-NEXT: RET_ReallyLR implicit $d0
     %0:_(<4 x s16>) = COPY $d0
     %1:_(<4 x s16>) = COPY $d1
     %2:_(<4 x s16>) = COPY $d2
@@ -69,63 +53,36 @@ body:             |
 
     ; NO-FP16-LABEL: name: test_v8f16.fma
     ; NO-FP16: liveins: $q0, $q1, $q2
-    ; NO-FP16: [[COPY:%[0-9]+]]:_(<8 x s16>) = COPY $q0
-    ; NO-FP16: [[COPY1:%[0-9]+]]:_(<8 x s16>) = COPY $q1
-    ; NO-FP16: [[COPY2:%[0-9]+]]:_(<8 x s16>) = COPY $q2
-    ; NO-FP16: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<8 x s16>)
-    ; NO-FP16: [[UV8:%[0-9]+]]:_(s16), [[UV9:%[0-9]+]]:_(s16), [[UV10:%[0-9]+]]:_(s16), [[UV11:%[0-9]+]]:_(s16), [[UV12:%[0-9]+]]:_(s16), [[UV13:%[0-9]+]]:_(s16), [[UV14:%[0-9]+]]:_(s16), [[UV15:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<8 x s16>)
-    ; NO-FP16: [[UV16:%[0-9]+]]:_(s16), [[UV17:%[0-9]+]]:_(s16), [[UV18:%[0-9]+]]:_(s16), [[UV19:%[0-9]+]]:_(s16), [[UV20:%[0-9]+]]:_(s16), [[UV21:%[0-9]+]]:_(s16), [[UV22:%[0-9]+]]:_(s16), [[UV23:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY2]](<8 x s16>)
-    ; NO-FP16: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[UV]](s16)
-    ; NO-FP16: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[UV8]](s16)
-    ; NO-FP16: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[UV16]](s16)
-    ; NO-FP16: [[FMA:%[0-9]+]]:_(s32) = G_FMA [[FPEXT]], [[FPEXT1]], [[FPEXT2]]
-    ; NO-FP16: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMA]](s32)
-    ; NO-FP16: [[FPEXT3:%[0-9]+]]:_(s32) = G_FPEXT [[UV1]](s16)
-    ; NO-FP16: [[FPEXT4:%[0-9]+]]:_(s32) = G_FPEXT [[UV9]](s16)
-    ; NO-FP16: [[FPEXT5:%[0-9]+]]:_(s32) = G_FPEXT [[UV17]](s16)
-    ; NO-FP16: [[FMA1:%[0-9]+]]:_(s32) = G_FMA [[FPEXT3]], [[FPEXT4]], [[FPEXT5]]
-    ; NO-FP16: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMA1]](s32)
-    ; NO-FP16: [[FPEXT6:%[0-9]+]]:_(s32) = G_FPEXT [[UV2]](s16)
-    ; NO-FP16: [[FPEXT7:%[0-9]+]]:_(s32) = G_FPEXT [[UV10]](s16)
-    ; NO-FP16: [[FPEXT8:%[0-9]+]]:_(s32) = G_FPEXT [[UV18]](s16)
-    ; NO-FP16: [[FMA2:%[0-9]+]]:_(s32) = G_FMA [[FPEXT6]], [[FPEXT7]], [[FPEXT8]]
-    ; NO-FP16: [[FPTRUNC2:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMA2]](s32)
-    ; NO-FP16: [[FPEXT9:%[0-9]+]]:_(s32) = G_FPEXT [[UV3]](s16)
-    ; NO-FP16: [[FPEXT10:%[0-9]+]]:_(s32) = G_FPEXT [[UV11]](s16)
-    ; NO-FP16: [[FPEXT11:%[0-9]+]]:_(s32) = G_FPEXT [[UV19]](s16)
-    ; NO-FP16: [[FMA3:%[0-9]+]]:_(s32) = G_FMA [[FPEXT9]], [[FPEXT10]], [[FPEXT11]]
-    ; NO-FP16: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMA3]](s32)
-    ; NO-FP16: [[FPEXT12:%[0-9]+]]:_(s32) = G_FPEXT [[UV4]](s16)
-    ; NO-FP16: [[FPEXT13:%[0-9]+]]:_(s32) = G_FPEXT [[UV12]](s16)
-    ; NO-FP16: [[FPEXT14:%[0-9]+]]:_(s32) = G_FPEXT [[UV20]](s16)
-    ; NO-FP16: [[FMA4:%[0-9]+]]:_(s32) = G_FMA [[FPEXT12]], [[FPEXT13]], [[FPEXT14]]
-    ; NO-FP16: [[FPTRUNC4:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMA4]](s32)
-    ; NO-FP16: [[FPEXT15:%[0-9]+]]:_(s32) = G_FPEXT [[UV5]](s16)
-    ; NO-FP16: [[FPEXT16:%[0-9]+]]:_(s32) = G_FPEXT [[UV13]](s16)
-    ; NO-FP16: [[FPEXT17:%[0-9]+]]:_(s32) = G_FPEXT [[UV21]](s16)
-    ; NO-FP16: [[FMA5:%[0-9]+]]:_(s32) = G_FMA [[FPEXT15]], [[FPEXT16]], [[FPEXT17]]
-    ; NO-FP16: [[FPTRUNC5:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMA5]](s32)
-    ; NO-FP16: [[FPEXT18:%[0-9]+]]:_(s32) = G_FPEXT [[UV6]](s16)
-    ; NO-FP16: [[FPEXT19:%[0-9]+]]:_(s32) = G_FPEXT [[UV14]](s16)
-    ; NO-FP16: [[FPEXT20:%[0-9]+]]:_(s32) = G_FPEXT [[UV22]](s16)
-    ; NO-FP16: [[FMA6:%[0-9]+]]:_(s32) = G_FMA [[FPEXT18]], [[FPEXT19]], [[FPEXT20]]
-    ; NO-FP16: [[FPTRUNC6:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMA6]](s32)
-    ; NO-FP16: [[FPEXT21:%[0-9]+]]:_(s32) = G_FPEXT [[UV7]](s16)
-    ; NO-FP16: [[FPEXT22:%[0-9]+]]:_(s32) = G_FPEXT [[UV15]](s16)
-    ; NO-FP16: [[FPEXT23:%[0-9]+]]:_(s32) = G_FPEXT [[UV23]](s16)
-    ; NO-FP16: [[FMA7:%[0-9]+]]:_(s32) = G_FMA [[FPEXT21]], [[FPEXT22]], [[FPEXT23]]
-    ; NO-FP16: [[FPTRUNC7:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMA7]](s32)
-    ; NO-FP16: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[FPTRUNC]](s16), [[FPTRUNC1]](s16), [[FPTRUNC2]](s16), [[FPTRUNC3]](s16), [[FPTRUNC4]](s16), [[FPTRUNC5]](s16), [[FPTRUNC6]](s16), [[FPTRUNC7]](s16)
-    ; NO-FP16: $q0 = COPY [[BUILD_VECTOR]](<8 x s16>)
-    ; NO-FP16: RET_ReallyLR implicit $q0
+    ; NO-FP16-NEXT: {{  $}}
+    ; NO-FP16-NEXT: [[COPY:%[0-9]+]]:_(<8 x s16>) = COPY $q0
+    ; NO-FP16-NEXT: [[COPY1:%[0-9]+]]:_(<8 x s16>) = COPY $q1
+    ; NO-FP16-NEXT: [[COPY2:%[0-9]+]]:_(<8 x s16>) = COPY $q2
+    ; NO-FP16-NEXT: [[UV:%[0-9]+]]:_(<4 x s16>), [[UV1:%[0-9]+]]:_(<4 x s16>) = G_UNMERGE_VALUES [[COPY]](<8 x s16>)
+    ; NO-FP16-NEXT: [[FPEXT:%[0-9]+]]:_(<4 x s32>) = G_FPEXT [[UV]](<4 x s16>)
+    ; NO-FP16-NEXT: [[FPEXT1:%[0-9]+]]:_(<4 x s32>) = G_FPEXT [[UV1]](<4 x s16>)
+    ; NO-FP16-NEXT: [[UV2:%[0-9]+]]:_(<4 x s16>), [[UV3:%[0-9]+]]:_(<4 x s16>) = G_UNMERGE_VALUES [[COPY1]](<8 x s16>)
+    ; NO-FP16-NEXT: [[FPEXT2:%[0-9]+]]:_(<4 x s32>) = G_FPEXT [[UV2]](<4 x s16>)
+    ; NO-FP16-NEXT: [[FPEXT3:%[0-9]+]]:_(<4 x s32>) = G_FPEXT [[UV3]](<4 x s16>)
+    ; NO-FP16-NEXT: [[UV4:%[0-9]+]]:_(<4 x s16>), [[UV5:%[0-9]+]]:_(<4 x s16>) = G_UNMERGE_VALUES [[COPY2]](<8 x s16>)
+    ; NO-FP16-NEXT: [[FPEXT4:%[0-9]+]]:_(<4 x s32>) = G_FPEXT [[UV4]](<4 x s16>)
+    ; NO-FP16-NEXT: [[FPEXT5:%[0-9]+]]:_(<4 x s32>) = G_FPEXT [[UV5]](<4 x s16>)
+    ; NO-FP16-NEXT: [[FMA:%[0-9]+]]:_(<4 x s32>) = G_FMA [[FPEXT]], [[FPEXT2]], [[FPEXT4]]
+    ; NO-FP16-NEXT: [[FMA1:%[0-9]+]]:_(<4 x s32>) = G_FMA [[FPEXT1]], [[FPEXT3]], [[FPEXT5]]
+    ; NO-FP16-NEXT: [[FPTRUNC:%[0-9]+]]:_(<4 x s16>) = G_FPTRUNC [[FMA]](<4 x s32>)
+    ; NO-FP16-NEXT: [[FPTRUNC1:%[0-9]+]]:_(<4 x s16>) = G_FPTRUNC [[FMA1]](<4 x s32>)
+    ; NO-FP16-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x s16>) = G_CONCAT_VECTORS [[FPTRUNC]](<4 x s16>), [[FPTRUNC1]](<4 x s16>)
+    ; NO-FP16-NEXT: $q0 = COPY [[CONCAT_VECTORS]](<8 x s16>)
+    ; NO-FP16-NEXT: RET_ReallyLR implicit $q0
+    ;
     ; FP16-LABEL: name: test_v8f16.fma
     ; FP16: liveins: $q0, $q1, $q2
-    ; FP16: [[COPY:%[0-9]+]]:_(<8 x s16>) = COPY $q0
-    ; FP16: [[COPY1:%[0-9]+]]:_(<8 x s16>) = COPY $q1
-    ; FP16: [[COPY2:%[0-9]+]]:_(<8 x s16>) = COPY $q2
-    ; FP16: [[FMA:%[0-9]+]]:_(<8 x s16>) = G_FMA [[COPY]], [[COPY1]], [[COPY2]]
-    ; FP16: $q0 = COPY [[FMA]](<8 x s16>)
-    ; FP16: RET_ReallyLR implicit $q0
+    ; FP16-NEXT: {{  $}}
+    ; FP16-NEXT: [[COPY:%[0-9]+]]:_(<8 x s16>) = COPY $q0
+    ; FP16-NEXT: [[COPY1:%[0-9]+]]:_(<8 x s16>) = COPY $q1
+    ; FP16-NEXT: [[COPY2:%[0-9]+]]:_(<8 x s16>) = COPY $q2
+    ; FP16-NEXT: [[FMA:%[0-9]+]]:_(<8 x s16>) = G_FMA [[COPY]], [[COPY1]], [[COPY2]]
+    ; FP16-NEXT: $q0 = COPY [[FMA]](<8 x s16>)
+    ; FP16-NEXT: RET_ReallyLR implicit $q0
     %0:_(<8 x s16>) = COPY $q0
     %1:_(<8 x s16>) = COPY $q1
     %2:_(<8 x s16>) = COPY $q2
@@ -145,20 +102,23 @@ body:             |
 
     ; NO-FP16-LABEL: name: test_v2f32.fma
     ; NO-FP16: liveins: $d0, $d1, $d2
-    ; NO-FP16: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0
-    ; NO-FP16: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $d1
-    ; NO-FP16: [[COPY2:%[0-9]+]]:_(<2 x s32>) = COPY $d2
-    ; NO-FP16: [[FMA:%[0-9]+]]:_(<2 x s32>) = G_FMA [[COPY]], [[COPY1]], [[COPY2]]
-    ; NO-FP16: $d0 = COPY [[FMA]](<2 x s32>)
-    ; NO-FP16: RET_ReallyLR implicit $d0
+    ; NO-FP16-NEXT: {{  $}}
+    ; NO-FP16-NEXT: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0
+    ; NO-FP16-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $d1
+    ; NO-FP16-NEXT: [[COPY2:%[0-9]+]]:_(<2 x s32>) = COPY $d2
+    ; NO-FP16-NEXT: [[FMA:%[0-9]+]]:_(<2 x s32>) = G_FMA [[COPY]], [[COPY1]], [[COPY2]]
+    ; NO-FP16-NEXT: $d0 = COPY [[FMA]](<2 x s32>)
+    ; NO-FP16-NEXT: RET_ReallyLR implicit $d0
+    ;
     ; FP16-LABEL: name: test_v2f32.fma
     ; FP16: liveins: $d0, $d1, $d2
-    ; FP16: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0
-    ; FP16: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $d1
-    ; FP16: [[COPY2:%[0-9]+]]:_(<2 x s32>) = COPY $d2
-    ; FP16: [[FMA:%[0-9]+]]:_(<2 x s32>) = G_FMA [[COPY]], [[COPY1]], [[COPY2]]
-    ; FP16: $d0 = COPY [[FMA]](<2 x s32>)
-    ; FP16: RET_ReallyLR implicit $d0
+    ; FP16-NEXT: {{  $}}
+    ; FP16-NEXT: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0
+    ; FP16-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $d1
+    ; FP16-NEXT: [[COPY2:%[0-9]+]]:_(<2 x s32>) = COPY $d2
+    ; FP16-NEXT: [[FMA:%[0-9]+]]:_(<2 x s32>) = G_FMA [[COPY]], [[COPY1]], [[COPY2]]
+    ; FP16-NEXT: $d0 = COPY [[FMA]](<2 x s32>)
+    ; FP16-NEXT: RET_ReallyLR implicit $d0
     %0:_(<2 x s32>) = COPY $d0
     %1:_(<2 x s32>) = COPY $d1
     %2:_(<2 x s32>) = COPY $d2
@@ -178,20 +138,23 @@ body:             |
 
     ; NO-FP16-LABEL: name: test_v4f32.fma
     ; NO-FP16: liveins: $q0, $q1, $q2
-    ; NO-FP16: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
-    ; NO-FP16: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $q1
-    ; NO-FP16: [[COPY2:%[0-9]+]]:_(<4 x s32>) = COPY $q2
-    ; NO-FP16: [[FMA:%[0-9]+]]:_(<4 x s32>) = G_FMA [[COPY]], [[COPY1]], [[COPY2]]
-    ; NO-FP16: $q0 = COPY [[FMA]](<4 x s32>)
-    ; NO-FP16: RET_ReallyLR implicit $q0
+    ; NO-FP16-NEXT: {{  $}}
+    ; NO-FP16-NEXT: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
+    ; NO-FP16-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $q1
+    ; NO-FP16-NEXT: [[COPY2:%[0-9]+]]:_(<4 x s32>) = COPY $q2
+    ; NO-FP16-NEXT: [[FMA:%[0-9]+]]:_(<4 x s32>) = G_FMA [[COPY]], [[COPY1]], [[COPY2]]
+    ; NO-FP16-NEXT: $q0 = COPY [[FMA]](<4 x s32>)
+    ; NO-FP16-NEXT: RET_ReallyLR implicit $q0
+    ;
     ; FP16-LABEL: name: test_v4f32.fma
     ; FP16: liveins: $q0, $q1, $q2
-    ; FP16: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
-    ; FP16: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $q1
-    ; FP16: [[COPY2:%[0-9]+]]:_(<4 x s32>) = COPY $q2
-    ; FP16: [[FMA:%[0-9]+]]:_(<4 x s32>) = G_FMA [[COPY]], [[COPY1]], [[COPY2]]
-    ; FP16: $q0 = COPY [[FMA]](<4 x s32>)
-    ; FP16: RET_ReallyLR implicit $q0
+    ; FP16-NEXT: {{  $}}
+    ; FP16-NEXT: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
+    ; FP16-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $q1
+    ; FP16-NEXT: [[COPY2:%[0-9]+]]:_(<4 x s32>) = COPY $q2
+    ; FP16-NEXT: [[FMA:%[0-9]+]]:_(<4 x s32>) = G_FMA [[COPY]], [[COPY1]], [[COPY2]]
+    ; FP16-NEXT: $q0 = COPY [[FMA]](<4 x s32>)
+    ; FP16-NEXT: RET_ReallyLR implicit $q0
     %0:_(<4 x s32>) = COPY $q0
     %1:_(<4 x s32>) = COPY $q1
     %2:_(<4 x s32>) = COPY $q2
@@ -211,20 +174,23 @@ body:             |
 
     ; NO-FP16-LABEL: name: test_v2f64.fma
     ; NO-FP16: liveins: $q0, $q1, $q2
-    ; NO-FP16: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $q0
-    ; NO-FP16: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $q1
-    ; NO-FP16: [[COPY2:%[0-9]+]]:_(<2 x s64>) = COPY $q2
-    ; NO-FP16: [[FMA:%[0-9]+]]:_(<2 x s64>) = G_FMA [[COPY]], [[COPY1]], [[COPY2]]
-    ; NO-FP16: $q0 = COPY [[FMA]](<2 x s64>)
-    ; NO-FP16: RET_ReallyLR implicit $q0
+    ; NO-FP16-NEXT: {{  $}}
+    ; NO-FP16-NEXT: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $q0
+    ; NO-FP16-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $q1
+    ; NO-FP16-NEXT: [[COPY2:%[0-9]+]]:_(<2 x s64>) = COPY $q2
+    ; NO-FP16-NEXT: [[FMA:%[0-9]+]]:_(<2 x s64>) = G_FMA [[COPY]], [[COPY1]], [[COPY2]]
+    ; NO-FP16-NEXT: $q0 = COPY [[FMA]](<2 x s64>)
+    ; NO-FP16-NEXT: RET_ReallyLR implicit $q0
+    ;
     ; FP16-LABEL: name: test_v2f64.fma
     ; FP16: liveins: $q0, $q1, $q2
-    ; FP16: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $q0
-    ; FP16: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $q1
-    ; FP16: [[COPY2:%[0-9]+]]:_(<2 x s64>) = COPY $q2
-    ; FP16: [[FMA:%[0-9]+]]:_(<2 x s64>) = G_FMA [[COPY]], [[COPY1]], [[COPY2]]
-    ; FP16: $q0 = COPY [[FMA]](<2 x s64>)
-    ; FP16: RET_ReallyLR implicit $q0
+    ; FP16-NEXT: {{  $}}
+    ; FP16-NEXT: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $q0
+    ; FP16-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $q1
+    ; FP16-NEXT: [[COPY2:%[0-9]+]]:_(<2 x s64>) = COPY $q2
+    ; FP16-NEXT: [[FMA:%[0-9]+]]:_(<2 x s64>) = G_FMA [[COPY]], [[COPY1]], [[COPY2]]
+    ; FP16-NEXT: $q0 = COPY [[FMA]](<2 x s64>)
+    ; FP16-NEXT: RET_ReallyLR implicit $q0
     %0:_(<2 x s64>) = COPY $q0
     %1:_(<2 x s64>) = COPY $q1
     %2:_(<2 x s64>) = COPY $q2
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-xtn.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-xtn.mir
new file mode 100644
index 0000000000..16b780a839
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-xtn.mir
@@ -0,0 +1,531 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 3
+# RUN: llc -O0 -mtriple=arm64-unknown-unknown -global-isel -run-pass=legalizer -global-isel-abort=2 %s -o - | FileCheck %s
+
+---
+name:            xtn_v2i64_v2i8
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; CHECK-LABEL: name: xtn_v2i64_v2i8
+    ; CHECK: liveins: $q0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $q0
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(<2 x s32>) = G_TRUNC [[COPY]](<2 x s64>)
+    ; CHECK-NEXT: $d0 = COPY [[TRUNC]](<2 x s32>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $d0
+    %0:_(<2 x s64>) = COPY $q0
+    %1:_(<2 x s8>) = G_TRUNC %0(<2 x s64>)
+    %2:_(<2 x s32>) = G_ANYEXT %1(<2 x s8>)
+    $d0 = COPY %2(<2 x s32>)
+    RET_ReallyLR implicit $d0
+
+...
+
+---
+name:            xtn_v2i128_v2i8
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0, $x1, $x2, $x3
+
+    ; CHECK-LABEL: name: xtn_v2i128_v2i8
+    ; CHECK: liveins: $x0, $x1, $x2, $x3
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x2
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[COPY]](s64), [[COPY1]](s64)
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(<2 x s32>) = G_TRUNC [[BUILD_VECTOR]](<2 x s64>)
+    ; CHECK-NEXT: $d0 = COPY [[TRUNC]](<2 x s32>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $d0
+    %1:_(s64) = COPY $x0
+    %2:_(s64) = COPY $x1
+    %3:_(s64) = COPY $x2
+    %4:_(s64) = COPY $x3
+    %5:_(s128) = G_MERGE_VALUES %1(s64), %2(s64)
+    %6:_(s128) = G_MERGE_VALUES %3(s64), %4(s64)
+    %0:_(<2 x s128>) = G_BUILD_VECTOR %5(s128), %6(s128)
+    %7:_(<2 x s8>) = G_TRUNC %0(<2 x s128>)
+    %8:_(<2 x s32>) = G_ANYEXT %7(<2 x s8>)
+    $d0 = COPY %8(<2 x s32>)
+    RET_ReallyLR implicit $d0
+
+...
+
+---
+name:            xtn_v2i64_v2i16
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; CHECK-LABEL: name: xtn_v2i64_v2i16
+    ; CHECK: liveins: $q0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $q0
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(<2 x s32>) = G_TRUNC [[COPY]](<2 x s64>)
+    ; CHECK-NEXT: $d0 = COPY [[TRUNC]](<2 x s32>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $d0
+    %0:_(<2 x s64>) = COPY $q0
+    %1:_(<2 x s16>) = G_TRUNC %0(<2 x s64>)
+    %2:_(<2 x s32>) = G_ANYEXT %1(<2 x s16>)
+    $d0 = COPY %2(<2 x s32>)
+    RET_ReallyLR implicit $d0
+
+...
+
+---
+name:            xtn_v2i128_v2i16
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0, $x1, $x2, $x3
+
+    ; CHECK-LABEL: name: xtn_v2i128_v2i16
+    ; CHECK: liveins: $x0, $x1, $x2, $x3
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x2
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[COPY]](s64), [[COPY1]](s64)
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(<2 x s32>) = G_TRUNC [[BUILD_VECTOR]](<2 x s64>)
+    ; CHECK-NEXT: $d0 = COPY [[TRUNC]](<2 x s32>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $d0
+    %1:_(s64) = COPY $x0
+    %2:_(s64) = COPY $x1
+    %3:_(s64) = COPY $x2
+    %4:_(s64) = COPY $x3
+    %5:_(s128) = G_MERGE_VALUES %1(s64), %2(s64)
+    %6:_(s128) = G_MERGE_VALUES %3(s64), %4(s64)
+    %0:_(<2 x s128>) = G_BUILD_VECTOR %5(s128), %6(s128)
+    %7:_(<2 x s16>) = G_TRUNC %0(<2 x s128>)
+    %8:_(<2 x s32>) = G_ANYEXT %7(<2 x s16>)
+    $d0 = COPY %8(<2 x s32>)
+    RET_ReallyLR implicit $d0
+
+...
+
+---
+name:            xtn_v2i128_v2i32
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0, $x1, $x2, $x3
+
+    ; CHECK-LABEL: name: xtn_v2i128_v2i32
+    ; CHECK: liveins: $x0, $x1, $x2, $x3
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x2
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[COPY]](s64), [[COPY1]](s64)
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(<2 x s32>) = G_TRUNC [[BUILD_VECTOR]](<2 x s64>)
+    ; CHECK-NEXT: $d0 = COPY [[TRUNC]](<2 x s32>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $d0
+    %1:_(s64) = COPY $x0
+    %2:_(s64) = COPY $x1
+    %3:_(s64) = COPY $x2
+    %4:_(s64) = COPY $x3
+    %5:_(s128) = G_MERGE_VALUES %1(s64), %2(s64)
+    %6:_(s128) = G_MERGE_VALUES %3(s64), %4(s64)
+    %0:_(<2 x s128>) = G_BUILD_VECTOR %5(s128), %6(s128)
+    %7:_(<2 x s32>) = G_TRUNC %0(<2 x s128>)
+    $d0 = COPY %7(<2 x s32>)
+    RET_ReallyLR implicit $d0
+
+...
+
+---
+name:            xtn_v2i128_v2i64
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0, $x1, $x2, $x3
+
+    ; CHECK-LABEL: name: xtn_v2i128_v2i64
+    ; CHECK: liveins: $x0, $x1, $x2, $x3
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x2
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[COPY]](s64), [[COPY1]](s64)
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(<2 x s64>) = COPY [[BUILD_VECTOR]](<2 x s64>)
+    ; CHECK-NEXT: $q0 = COPY [[COPY2]](<2 x s64>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
+    %1:_(s64) = COPY $x0
+    %2:_(s64) = COPY $x1
+    %3:_(s64) = COPY $x2
+    %4:_(s64) = COPY $x3
+    %5:_(s128) = G_MERGE_VALUES %1(s64), %2(s64)
+    %6:_(s128) = G_MERGE_VALUES %3(s64), %4(s64)
+    %0:_(<2 x s128>) = G_BUILD_VECTOR %5(s128), %6(s128)
+    %7:_(<2 x s64>) = G_TRUNC %0(<2 x s128>)
+    $q0 = COPY %7(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+
+---
+name:            xtn_v3i16_v3i8
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $d0
+
+    ; CHECK-LABEL: name: xtn_v3i16_v3i8
+    ; CHECK: liveins: $d0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $d0
+    ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s16)
+    ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s16)
+    ; CHECK-NEXT: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[UV2]](s16)
+    ; CHECK-NEXT: $w0 = COPY [[ANYEXT]](s32)
+    ; CHECK-NEXT: $w1 = COPY [[ANYEXT1]](s32)
+    ; CHECK-NEXT: $w2 = COPY [[ANYEXT2]](s32)
+    ; CHECK-NEXT: RET_ReallyLR implicit $w0, implicit $w1, implicit $w2
+    %1:_(<4 x s16>) = COPY $d0
+    %2:_(s16), %3:_(s16), %4:_(s16), %5:_(s16) = G_UNMERGE_VALUES %1(<4 x s16>)
+    %0:_(<3 x s16>) = G_BUILD_VECTOR %2(s16), %3(s16), %4(s16)
+    %6:_(<3 x s8>) = G_TRUNC %0(<3 x s16>)
+    %10:_(s8), %11:_(s8), %12:_(s8) = G_UNMERGE_VALUES %6(<3 x s8>)
+    %7:_(s32) = G_ANYEXT %10(s8)
+    %8:_(s32) = G_ANYEXT %11(s8)
+    %9:_(s32) = G_ANYEXT %12(s8)
+    $w0 = COPY %7(s32)
+    $w1 = COPY %8(s32)
+    $w2 = COPY %9(s32)
+    RET_ReallyLR implicit $w0, implicit $w1, implicit $w2
+
+...
+
+---
+name:            xtn_v3i32_v3i8
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; CHECK-LABEL: name: xtn_v3i32_v3i8
+    ; CHECK: liveins: $q0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
+    ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<4 x s32>)
+    ; CHECK-NEXT: $w0 = COPY [[UV]](s32)
+    ; CHECK-NEXT: $w1 = COPY [[UV1]](s32)
+    ; CHECK-NEXT: $w2 = COPY [[UV2]](s32)
+    ; CHECK-NEXT: RET_ReallyLR implicit $w0, implicit $w1, implicit $w2
+    %1:_(<4 x s32>) = COPY $q0
+    %2:_(s32), %3:_(s32), %4:_(s32), %5:_(s32) = G_UNMERGE_VALUES %1(<4 x s32>)
+    %0:_(<3 x s32>) = G_BUILD_VECTOR %2(s32), %3(s32), %4(s32)
+    %6:_(<3 x s8>) = G_TRUNC %0(<3 x s32>)
+    %10:_(s8), %11:_(s8), %12:_(s8) = G_UNMERGE_VALUES %6(<3 x s8>)
+    %7:_(s32) = G_ANYEXT %10(s8)
+    %8:_(s32) = G_ANYEXT %11(s8)
+    %9:_(s32) = G_ANYEXT %12(s8)
+    $w0 = COPY %7(s32)
+    $w1 = COPY %8(s32)
+    $w2 = COPY %9(s32)
+    RET_ReallyLR implicit $w0, implicit $w1, implicit $w2
+
+...
+
+---
+name:            xtn_v3i64_v3i8
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $d0, $d1, $d2
+
+    ; CHECK-LABEL: name: xtn_v3i64_v3i8
+    ; CHECK: liveins: $d0, $d1, $d2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $d0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $d1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $d2
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s32) = G_TRUNC [[COPY2]](s64)
+    ; CHECK-NEXT: $w0 = COPY [[TRUNC]](s32)
+    ; CHECK-NEXT: $w1 = COPY [[TRUNC1]](s32)
+    ; CHECK-NEXT: $w2 = COPY [[TRUNC2]](s32)
+    ; CHECK-NEXT: RET_ReallyLR implicit $w0, implicit $w1, implicit $w2
+    %1:_(s64) = COPY $d0
+    %2:_(s64) = COPY $d1
+    %3:_(s64) = COPY $d2
+    %0:_(<3 x s64>) = G_BUILD_VECTOR %1(s64), %2(s64), %3(s64)
+    %4:_(<3 x s8>) = G_TRUNC %0(<3 x s64>)
+    %8:_(s8), %9:_(s8), %10:_(s8) = G_UNMERGE_VALUES %4(<3 x s8>)
+    %5:_(s32) = G_ANYEXT %8(s8)
+    %6:_(s32) = G_ANYEXT %9(s8)
+    %7:_(s32) = G_ANYEXT %10(s8)
+    $w0 = COPY %5(s32)
+    $w1 = COPY %6(s32)
+    $w2 = COPY %7(s32)
+    RET_ReallyLR implicit $w0, implicit $w1, implicit $w2
+
+...
+
+---
+name:            xtn_v3i64_v3i16
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $d0, $d1, $d2
+
+    ; CHECK-LABEL: name: xtn_v3i64_v3i16
+    ; CHECK: liveins: $d0, $d1, $d2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $d0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $d1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $d2
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s64)
+    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s64)
+    ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY2]](s64)
+    ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16), [[TRUNC2]](s16), [[DEF]](s16)
+    ; CHECK-NEXT: $d0 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $d0
+    %1:_(s64) = COPY $d0
+    %2:_(s64) = COPY $d1
+    %3:_(s64) = COPY $d2
+    %0:_(<3 x s64>) = G_BUILD_VECTOR %1(s64), %2(s64), %3(s64)
+    %4:_(<3 x s16>) = G_TRUNC %0(<3 x s64>)
+    %5:_(s16), %6:_(s16), %7:_(s16) = G_UNMERGE_VALUES %4(<3 x s16>)
+    %8:_(s16) = G_IMPLICIT_DEF
+    %9:_(<4 x s16>) = G_BUILD_VECTOR %5(s16), %6(s16), %7(s16), %8(s16)
+    $d0 = COPY %9(<4 x s16>)
+    RET_ReallyLR implicit $d0
+
+...
+
+---
+name:            xtn_v3i64_v3i32
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $d0, $d1, $d2
+
+    ; CHECK-LABEL: name: xtn_v3i64_v3i32
+    ; CHECK: liveins: $d0, $d1, $d2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $d0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $d1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $d2
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s32) = G_TRUNC [[COPY2]](s64)
+    ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[TRUNC]](s32), [[TRUNC1]](s32), [[TRUNC2]](s32), [[DEF]](s32)
+    ; CHECK-NEXT: $q0 = COPY [[BUILD_VECTOR]](<4 x s32>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
+    %1:_(s64) = COPY $d0
+    %2:_(s64) = COPY $d1
+    %3:_(s64) = COPY $d2
+    %0:_(<3 x s64>) = G_BUILD_VECTOR %1(s64), %2(s64), %3(s64)
+    %4:_(<3 x s32>) = G_TRUNC %0(<3 x s64>)
+    %5:_(s32), %6:_(s32), %7:_(s32) = G_UNMERGE_VALUES %4(<3 x s32>)
+    %8:_(s32) = G_IMPLICIT_DEF
+    %9:_(<4 x s32>) = G_BUILD_VECTOR %5(s32), %6(s32), %7(s32), %8(s32)
+    $q0 = COPY %9(<4 x s32>)
+    RET_ReallyLR implicit $q0
+
+...
+
+---
+name:            xtn_v4i32_v4i8
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; CHECK-LABEL: name: xtn_v4i32_v4i8
+    ; CHECK: liveins: $q0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(<4 x s16>) = G_TRUNC [[COPY]](<4 x s32>)
+    ; CHECK-NEXT: $d0 = COPY [[TRUNC]](<4 x s16>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $d0
+    %0:_(<4 x s32>) = COPY $q0
+    %1:_(<4 x s8>) = G_TRUNC %0(<4 x s32>)
+    %2:_(<4 x s16>) = G_ANYEXT %1(<4 x s8>)
+    $d0 = COPY %2(<4 x s16>)
+    RET_ReallyLR implicit $d0
+
+...
+
+---
+name:            xtn_v4i64_v4i8
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; CHECK-LABEL: name: xtn_v4i64_v4i8
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $q0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $q1
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(<2 x s32>) = G_TRUNC [[COPY]](<2 x s64>)
+    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(<2 x s32>) = G_TRUNC [[COPY1]](<2 x s64>)
+    ; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s32>) = G_CONCAT_VECTORS [[TRUNC]](<2 x s32>), [[TRUNC1]](<2 x s32>)
+    ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(<4 x s16>) = G_TRUNC [[CONCAT_VECTORS]](<4 x s32>)
+    ; CHECK-NEXT: $d0 = COPY [[TRUNC2]](<4 x s16>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $d0
+    %1:_(<2 x s64>) = COPY $q0
+    %2:_(<2 x s64>) = COPY $q1
+    %0:_(<4 x s64>) = G_CONCAT_VECTORS %1(<2 x s64>), %2(<2 x s64>)
+    %3:_(<4 x s8>) = G_TRUNC %0(<4 x s64>)
+    %4:_(<4 x s16>) = G_ANYEXT %3(<4 x s8>)
+    $d0 = COPY %4(<4 x s16>)
+    RET_ReallyLR implicit $d0
+
+...
+
+---
+name:            xtn_v4i64_v4i16
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; CHECK-LABEL: name: xtn_v4i64_v4i16
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $q0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $q1
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(<2 x s32>) = G_TRUNC [[COPY]](<2 x s64>)
+    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(<2 x s32>) = G_TRUNC [[COPY1]](<2 x s64>)
+    ; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s32>) = G_CONCAT_VECTORS [[TRUNC]](<2 x s32>), [[TRUNC1]](<2 x s32>)
+    ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(<4 x s16>) = G_TRUNC [[CONCAT_VECTORS]](<4 x s32>)
+    ; CHECK-NEXT: $d0 = COPY [[TRUNC2]](<4 x s16>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $d0
+    %1:_(<2 x s64>) = COPY $q0
+    %2:_(<2 x s64>) = COPY $q1
+    %0:_(<4 x s64>) = G_CONCAT_VECTORS %1(<2 x s64>), %2(<2 x s64>)
+    %3:_(<4 x s16>) = G_TRUNC %0(<4 x s64>)
+    $d0 = COPY %3(<4 x s16>)
+    RET_ReallyLR implicit $d0
+
+...
+
+---
+name:            xtn_v4i64_v4i32
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; CHECK-LABEL: name: xtn_v4i64_v4i32
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $q0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $q1
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(<2 x s32>) = G_TRUNC [[COPY]](<2 x s64>)
+    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(<2 x s32>) = G_TRUNC [[COPY1]](<2 x s64>)
+    ; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s32>) = G_CONCAT_VECTORS [[TRUNC]](<2 x s32>), [[TRUNC1]](<2 x s32>)
+    ; CHECK-NEXT: $q0 = COPY [[CONCAT_VECTORS]](<4 x s32>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
+    %1:_(<2 x s64>) = COPY $q0
+    %2:_(<2 x s64>) = COPY $q1
+    %0:_(<4 x s64>) = G_CONCAT_VECTORS %1(<2 x s64>), %2(<2 x s64>)
+    %3:_(<4 x s32>) = G_TRUNC %0(<4 x s64>)
+    $q0 = COPY %3(<4 x s32>)
+    RET_ReallyLR implicit $q0
+
+...
+
+---
+name:            xtn_v8i32_v8i8
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; CHECK-LABEL: name: xtn_v8i32_v8i8
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $q1
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(<4 x s16>) = G_TRUNC [[COPY]](<4 x s32>)
+    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(<4 x s16>) = G_TRUNC [[COPY1]](<4 x s32>)
+    ; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x s16>) = G_CONCAT_VECTORS [[TRUNC]](<4 x s16>), [[TRUNC1]](<4 x s16>)
+    ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(<8 x s8>) = G_TRUNC [[CONCAT_VECTORS]](<8 x s16>)
+    ; CHECK-NEXT: $d0 = COPY [[TRUNC2]](<8 x s8>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $d0
+    %1:_(<4 x s32>) = COPY $q0
+    %2:_(<4 x s32>) = COPY $q1
+    %0:_(<8 x s32>) = G_CONCAT_VECTORS %1(<4 x s32>), %2(<4 x s32>)
+    %3:_(<8 x s8>) = G_TRUNC %0(<8 x s32>)
+    $d0 = COPY %3(<8 x s8>)
+    RET_ReallyLR implicit $d0
+
+...
+
+---
+name:            xtn_v8i32_v8i16
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; CHECK-LABEL: name: xtn_v8i32_v8i16
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $q1
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(<4 x s16>) = G_TRUNC [[COPY]](<4 x s32>)
+    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(<4 x s16>) = G_TRUNC [[COPY1]](<4 x s32>)
+    ; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x s16>) = G_CONCAT_VECTORS [[TRUNC]](<4 x s16>), [[TRUNC1]](<4 x s16>)
+    ; CHECK-NEXT: $q0 = COPY [[CONCAT_VECTORS]](<8 x s16>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
+    %1:_(<4 x s32>) = COPY $q0
+    %2:_(<4 x s32>) = COPY $q1
+    %0:_(<8 x s32>) = G_CONCAT_VECTORS %1(<4 x s32>), %2(<4 x s32>)
+    %3:_(<8 x s16>) = G_TRUNC %0(<8 x s32>)
+    $q0 = COPY %3(<8 x s16>)
+    RET_ReallyLR implicit $q0
+
+...
+
+---
+name:            xtn_v16i16_v16i8
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; CHECK-LABEL: name: xtn_v16i16_v16i8
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<8 x s16>) = COPY $q0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<8 x s16>) = COPY $q1
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(<8 x s8>) = G_TRUNC [[COPY]](<8 x s16>)
+    ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(<8 x s8>) = G_TRUNC [[COPY1]](<8 x s16>)
+    ; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<16 x s8>) = G_CONCAT_VECTORS [[TRUNC]](<8 x s8>), [[TRUNC1]](<8 x s8>)
+    ; CHECK-NEXT: $q0 = COPY [[CONCAT_VECTORS]](<16 x s8>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
+    %1:_(<8 x s16>) = COPY $q0
+    %2:_(<8 x s16>) = COPY $q1
+    %0:_(<16 x s16>) = G_CONCAT_VECTORS %1(<8 x s16>), %2(<8 x s16>)
+    %3:_(<16 x s8>) = G_TRUNC %0(<16 x s16>)
+    $q0 = COPY %3(<16 x s8>)
+    RET_ReallyLR implicit $q0
+
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index bb915153c5..549f36b2af 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -154,7 +154,6 @@
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: G_INTRINSIC_LRINT (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
-# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: G_INTRINSIC_ROUNDEVEN (opcode {{[0-9]+}}): 1 type index, 0 imm indices
@@ -442,6 +441,7 @@
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: G_FMA (opcode {{[0-9]+}}): 1 type index, 0 imm indices
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: G_FMAD (opcode {{[0-9]+}}): 1 type index, 0 imm indices
@@ -754,8 +754,8 @@
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: G_VECREDUCE_MUL (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: G_VECREDUCE_AND (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
 # DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-sadde.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-sadde.mir
index 85625ced4b..e98ab4af57 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/select-sadde.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-sadde.mir
@@ -175,3 +175,34 @@ body:             |
     $x2 = COPY %9(s64)
     RET_ReallyLR implicit $x0, implicit $x1, implicit $x2
 ...
+...
+---
+name:            sadde_opt_prev_dead
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $x0, $x1, $x2, $x3
+    ; CHECK-LABEL: name: sadde_opt_prev_dead
+    ; CHECK: liveins: $x0, $x1, $x2, $x3
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64 = COPY $x2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr64 = COPY $x3
+    ; CHECK-NEXT: [[ADDSXrr:%[0-9]+]]:gpr64 = ADDSXrr [[COPY]], [[COPY2]], implicit-def $nzcv
+    ; CHECK-NEXT: [[ADCSXr:%[0-9]+]]:gpr64 = ADCSXr [[COPY1]], [[COPY3]], implicit-def $nzcv, implicit $nzcv
+    ; CHECK-NEXT: [[CSINCWr:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 7, implicit $nzcv
+    ; CHECK-NEXT: $w0 = COPY [[CSINCWr]]
+    ; CHECK-NEXT: RET_ReallyLR implicit $w0
+    %0:gpr(s64) = COPY $x0
+    %1:gpr(s64) = COPY $x1
+    %2:gpr(s64) = COPY $x2
+    %3:gpr(s64) = COPY $x3
+    %4:gpr(s64), %5:gpr(s32) = G_UADDO %0, %2
+    %6:gpr(s64), %7:gpr(s32) = G_SADDE %1, %3, %5
+    $w0 = COPY %7(s32)
+    RET_ReallyLR implicit $w0
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-ssube.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-ssube.mir
index 00bd26cc02..a9da51781e 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/select-ssube.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-ssube.mir
@@ -175,3 +175,34 @@ body:             |
     $x2 = COPY %9(s64)
     RET_ReallyLR implicit $x0, implicit $x1, implicit $x2
 ...
+...
+---
+name:            ssube_opt_prev_dead
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $x0, $x1, $x2, $x3
+    ; CHECK-LABEL: name: ssube_opt_prev_dead
+    ; CHECK: liveins: $x0, $x1, $x2, $x3
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64 = COPY $x2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr64 = COPY $x3
+    ; CHECK-NEXT: [[SUBSXrr:%[0-9]+]]:gpr64 = SUBSXrr [[COPY]], [[COPY2]], implicit-def $nzcv
+    ; CHECK-NEXT: [[SBCSXr:%[0-9]+]]:gpr64 = SBCSXr [[COPY1]], [[COPY3]], implicit-def $nzcv, implicit $nzcv
+    ; CHECK-NEXT: [[CSINCWr:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 7, implicit $nzcv
+    ; CHECK-NEXT: $w0 = COPY [[CSINCWr]]
+    ; CHECK-NEXT: RET_ReallyLR implicit $w0
+    %0:gpr(s64) = COPY $x0
+    %1:gpr(s64) = COPY $x1
+    %2:gpr(s64) = COPY $x2
+    %3:gpr(s64) = COPY $x3
+    %4:gpr(s64), %5:gpr(s32) = G_USUBO %0, %2
+    %6:gpr(s64), %7:gpr(s32) = G_SSUBE %1, %3, %5
+    $w0 = COPY %7(s32)
+    RET_ReallyLR implicit $w0
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-uadde.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-uadde.mir
index dc80d0c9ab..f6f9964e6b 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/select-uadde.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-uadde.mir
@@ -175,3 +175,34 @@ body:             |
     $x2 = COPY %9(s64)
     RET_ReallyLR implicit $x0, implicit $x1, implicit $x2
 ...
+...
+---
+name:            uadde_opt_prev_dead
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $x0, $x1, $x2, $x3
+    ; CHECK-LABEL: name: uadde_opt_prev_dead
+    ; CHECK: liveins: $x0, $x1, $x2, $x3
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64 = COPY $x2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr64 = COPY $x3
+    ; CHECK-NEXT: [[ADDSXrr:%[0-9]+]]:gpr64 = ADDSXrr [[COPY]], [[COPY2]], implicit-def $nzcv
+    ; CHECK-NEXT: [[ADCSXr:%[0-9]+]]:gpr64 = ADCSXr [[COPY1]], [[COPY3]], implicit-def $nzcv, implicit $nzcv
+    ; CHECK-NEXT: [[CSINCWr:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 3, implicit $nzcv
+    ; CHECK-NEXT: $w0 = COPY [[CSINCWr]]
+    ; CHECK-NEXT: RET_ReallyLR implicit $w0
+    %0:gpr(s64) = COPY $x0
+    %1:gpr(s64) = COPY $x1
+    %2:gpr(s64) = COPY $x2
+    %3:gpr(s64) = COPY $x3
+    %4:gpr(s64), %5:gpr(s32) = G_UADDO %0, %2
+    %6:gpr(s64), %7:gpr(s32) = G_UADDE %1, %3, %5
+    $w0 = COPY %7(s32)
+    RET_ReallyLR implicit $w0
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-usube.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-usube.mir
index c532474fc6..fa8799653a 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/select-usube.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-usube.mir
@@ -175,3 +175,34 @@ body:             |
     $x2 = COPY %9(s64)
     RET_ReallyLR implicit $x0, implicit $x1, implicit $x2
 ...
+...
+---
+name:            usube_opt_prev_dead
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $x0, $x1, $x2, $x3
+    ; CHECK-LABEL: name: usube_opt_prev_dead
+    ; CHECK: liveins: $x0, $x1, $x2, $x3
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64 = COPY $x2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr64 = COPY $x3
+    ; CHECK-NEXT: [[SUBSXrr:%[0-9]+]]:gpr64 = SUBSXrr [[COPY]], [[COPY2]], implicit-def $nzcv
+    ; CHECK-NEXT: [[SBCSXr:%[0-9]+]]:gpr64 = SBCSXr [[COPY1]], [[COPY3]], implicit-def $nzcv, implicit $nzcv
+    ; CHECK-NEXT: [[CSINCWr:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 2, implicit $nzcv
+    ; CHECK-NEXT: $w0 = COPY [[CSINCWr]]
+    ; CHECK-NEXT: RET_ReallyLR implicit $w0
+    %0:gpr(s64) = COPY $x0
+    %1:gpr(s64) = COPY $x1
+    %2:gpr(s64) = COPY $x2
+    %3:gpr(s64) = COPY $x3
+    %4:gpr(s64), %5:gpr(s32) = G_USUBO %0, %2
+    %6:gpr(s64), %7:gpr(s32) = G_USUBE %1, %3, %5
+    $w0 = COPY %7(s32)
+    RET_ReallyLR implicit $w0
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/sink-and-fold-illegal-shift.ll b/llvm/test/CodeGen/AArch64/GlobalISel/sink-and-fold-illegal-shift.ll
new file mode 100644
index 0000000000..b9892fc31b
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/sink-and-fold-illegal-shift.ll
@@ -0,0 +1,17 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
+; RUN: llc -global-isel --aarch64-enable-sink-fold=true < %s | FileCheck %s
+
+target triple = "aarch64-linux"
+
+; Test a non-LSL shift cannot be folded into the addressing mode.
+define void @f(ptr %p, i64 %i) optsize {
+; CHECK-LABEL: f:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    add x8, x0, x1, asr #32
+; CHECK-NEXT:    strb wzr, [x8]
+; CHECK-NEXT:    ret
+	%d = ashr i64 %i, 32
+	%a = getelementptr i8, ptr %p, i64 %d
+	store i8 0, ptr %a
+	ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/aarch64-mulv.ll b/llvm/test/CodeGen/AArch64/aarch64-mulv.ll
new file mode 100644
index 0000000000..819bd4f4c4
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/aarch64-mulv.ll
@@ -0,0 +1,596 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
+; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc -mtriple=aarch64 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+
+; CHECK_GI:        warning: Instruction selection used fallback path for mulv_v3i64
+
+declare i8 @llvm.vector.reduce.mul.v2i8(<2 x i8>)
+declare i8 @llvm.vector.reduce.mul.v3i8(<3 x i8>)
+declare i8 @llvm.vector.reduce.mul.v4i8(<4 x i8>)
+declare i8 @llvm.vector.reduce.mul.v8i8(<8 x i8>)
+declare i8 @llvm.vector.reduce.mul.v16i8(<16 x i8>)
+declare i8 @llvm.vector.reduce.mul.v32i8(<32 x i8>)
+declare i16 @llvm.vector.reduce.mul.v2i16(<2 x i16>)
+declare i16 @llvm.vector.reduce.mul.v3i16(<3 x i16>)
+declare i16 @llvm.vector.reduce.mul.v4i16(<4 x i16>)
+declare i16 @llvm.vector.reduce.mul.v8i16(<8 x i16>)
+declare i16 @llvm.vector.reduce.mul.v16i16(<16 x i16>)
+declare i32 @llvm.vector.reduce.mul.v2i32(<2 x i32>)
+declare i32 @llvm.vector.reduce.mul.v3i32(<3 x i32>)
+declare i32 @llvm.vector.reduce.mul.v4i32(<4 x i32>)
+declare i32 @llvm.vector.reduce.mul.v8i32(<8 x i32>)
+declare i64 @llvm.vector.reduce.mul.v2i64(<2 x i64>)
+declare i64 @llvm.vector.reduce.mul.v3i64(<3 x i64>)
+declare i64 @llvm.vector.reduce.mul.v4i64(<4 x i64>)
+declare i128 @llvm.vector.reduce.mul.v2i128(<2 x i128>)
+
+define i8 @mulv_v2i8(<2 x i8> %a) {
+; CHECK-SD-LABEL: mulv_v2i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    mov w8, v0.s[1]
+; CHECK-SD-NEXT:    fmov w9, s0
+; CHECK-SD-NEXT:    mul w0, w9, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulv_v2i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    mul w0, w8, w9
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = call i8 @llvm.vector.reduce.mul.v2i8(<2 x i8> %a)
+  ret i8 %arg1
+}
+
+define i8 @mulv_v3i8(<3 x i8> %a) {
+; CHECK-LABEL: mulv_v3i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mul w8, w0, w1
+; CHECK-NEXT:    mul w0, w8, w2
+; CHECK-NEXT:    ret
+entry:
+  %arg1 = call i8 @llvm.vector.reduce.mul.v3i8(<3 x i8> %a)
+  ret i8 %arg1
+}
+
+define i8 @mulv_v4i8(<4 x i8> %a) {
+; CHECK-SD-LABEL: mulv_v4i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    umov w8, v0.h[1]
+; CHECK-SD-NEXT:    umov w9, v0.h[0]
+; CHECK-SD-NEXT:    umov w10, v0.h[2]
+; CHECK-SD-NEXT:    mul w8, w9, w8
+; CHECK-SD-NEXT:    umov w9, v0.h[3]
+; CHECK-SD-NEXT:    mul w8, w8, w10
+; CHECK-SD-NEXT:    mul w0, w8, w9
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulv_v4i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-NEXT:    mov h2, v0.h[2]
+; CHECK-GI-NEXT:    mov h3, v0.h[3]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w10, s2
+; CHECK-GI-NEXT:    fmov w11, s3
+; CHECK-GI-NEXT:    mul w8, w8, w9
+; CHECK-GI-NEXT:    mul w9, w10, w11
+; CHECK-GI-NEXT:    mul w0, w8, w9
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = call i8 @llvm.vector.reduce.mul.v4i8(<4 x i8> %a)
+  ret i8 %arg1
+}
+
+define i8 @mulv_v8i8(<8 x i8> %a) {
+; CHECK-SD-LABEL: mulv_v8i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    umov w8, v0.b[1]
+; CHECK-SD-NEXT:    umov w9, v0.b[0]
+; CHECK-SD-NEXT:    umov w10, v0.b[2]
+; CHECK-SD-NEXT:    mul w8, w9, w8
+; CHECK-SD-NEXT:    umov w9, v0.b[3]
+; CHECK-SD-NEXT:    mul w8, w8, w10
+; CHECK-SD-NEXT:    umov w10, v0.b[4]
+; CHECK-SD-NEXT:    mul w8, w8, w9
+; CHECK-SD-NEXT:    umov w9, v0.b[5]
+; CHECK-SD-NEXT:    mul w8, w8, w10
+; CHECK-SD-NEXT:    umov w10, v0.b[6]
+; CHECK-SD-NEXT:    mul w8, w8, w9
+; CHECK-SD-NEXT:    umov w9, v0.b[7]
+; CHECK-SD-NEXT:    mul w8, w8, w10
+; CHECK-SD-NEXT:    mul w0, w8, w9
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulv_v8i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    mov b1, v0.b[1]
+; CHECK-GI-NEXT:    mov b2, v0.b[2]
+; CHECK-GI-NEXT:    mov b3, v0.b[3]
+; CHECK-GI-NEXT:    mov b4, v0.b[4]
+; CHECK-GI-NEXT:    mov b5, v0.b[5]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    mov b6, v0.b[6]
+; CHECK-GI-NEXT:    mov b7, v0.b[7]
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w10, s2
+; CHECK-GI-NEXT:    fmov w11, s3
+; CHECK-GI-NEXT:    fmov w12, s5
+; CHECK-GI-NEXT:    mul w8, w8, w9
+; CHECK-GI-NEXT:    fmov w9, s4
+; CHECK-GI-NEXT:    mul w10, w10, w11
+; CHECK-GI-NEXT:    fmov w11, s6
+; CHECK-GI-NEXT:    mul w9, w9, w12
+; CHECK-GI-NEXT:    fmov w12, s7
+; CHECK-GI-NEXT:    mul w8, w8, w10
+; CHECK-GI-NEXT:    mul w11, w11, w12
+; CHECK-GI-NEXT:    mul w9, w9, w11
+; CHECK-GI-NEXT:    mul w0, w8, w9
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = call i8 @llvm.vector.reduce.mul.v8i8(<8 x i8> %a)
+  ret i8 %arg1
+}
+
+define i8 @mulv_v16i8(<16 x i8> %a) {
+; CHECK-SD-LABEL: mulv_v16i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    mul v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    umov w8, v0.b[1]
+; CHECK-SD-NEXT:    umov w9, v0.b[0]
+; CHECK-SD-NEXT:    umov w10, v0.b[2]
+; CHECK-SD-NEXT:    mul w8, w9, w8
+; CHECK-SD-NEXT:    umov w9, v0.b[3]
+; CHECK-SD-NEXT:    mul w8, w8, w10
+; CHECK-SD-NEXT:    umov w10, v0.b[4]
+; CHECK-SD-NEXT:    mul w8, w8, w9
+; CHECK-SD-NEXT:    umov w9, v0.b[5]
+; CHECK-SD-NEXT:    mul w8, w8, w10
+; CHECK-SD-NEXT:    umov w10, v0.b[6]
+; CHECK-SD-NEXT:    mul w8, w8, w9
+; CHECK-SD-NEXT:    umov w9, v0.b[7]
+; CHECK-SD-NEXT:    mul w8, w8, w10
+; CHECK-SD-NEXT:    mul w0, w8, w9
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulv_v16i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    mul v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    mov b1, v0.b[1]
+; CHECK-GI-NEXT:    mov b2, v0.b[2]
+; CHECK-GI-NEXT:    mov b3, v0.b[3]
+; CHECK-GI-NEXT:    mov b4, v0.b[4]
+; CHECK-GI-NEXT:    mov b5, v0.b[5]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    mov b6, v0.b[6]
+; CHECK-GI-NEXT:    mov b7, v0.b[7]
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w10, s2
+; CHECK-GI-NEXT:    fmov w11, s3
+; CHECK-GI-NEXT:    fmov w12, s5
+; CHECK-GI-NEXT:    mul w8, w8, w9
+; CHECK-GI-NEXT:    fmov w9, s4
+; CHECK-GI-NEXT:    mul w10, w10, w11
+; CHECK-GI-NEXT:    fmov w11, s6
+; CHECK-GI-NEXT:    mul w9, w9, w12
+; CHECK-GI-NEXT:    fmov w12, s7
+; CHECK-GI-NEXT:    mul w8, w8, w10
+; CHECK-GI-NEXT:    mul w11, w11, w12
+; CHECK-GI-NEXT:    mul w9, w9, w11
+; CHECK-GI-NEXT:    mul w0, w8, w9
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = call i8 @llvm.vector.reduce.mul.v16i8(<16 x i8> %a)
+  ret i8 %arg1
+}
+
+define i8 @mulv_v32i8(<32 x i8> %a) {
+; CHECK-SD-LABEL: mulv_v32i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mul v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    mul v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    umov w8, v0.b[1]
+; CHECK-SD-NEXT:    umov w9, v0.b[0]
+; CHECK-SD-NEXT:    umov w10, v0.b[2]
+; CHECK-SD-NEXT:    mul w8, w9, w8
+; CHECK-SD-NEXT:    umov w9, v0.b[3]
+; CHECK-SD-NEXT:    mul w8, w8, w10
+; CHECK-SD-NEXT:    umov w10, v0.b[4]
+; CHECK-SD-NEXT:    mul w8, w8, w9
+; CHECK-SD-NEXT:    umov w9, v0.b[5]
+; CHECK-SD-NEXT:    mul w8, w8, w10
+; CHECK-SD-NEXT:    umov w10, v0.b[6]
+; CHECK-SD-NEXT:    mul w8, w8, w9
+; CHECK-SD-NEXT:    umov w9, v0.b[7]
+; CHECK-SD-NEXT:    mul w8, w8, w10
+; CHECK-SD-NEXT:    mul w0, w8, w9
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulv_v32i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d2, v0.d[1]
+; CHECK-GI-NEXT:    mov d3, v1.d[1]
+; CHECK-GI-NEXT:    mul v0.8b, v0.8b, v2.8b
+; CHECK-GI-NEXT:    mul v1.8b, v1.8b, v3.8b
+; CHECK-GI-NEXT:    mul v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    mov b1, v0.b[1]
+; CHECK-GI-NEXT:    mov b2, v0.b[2]
+; CHECK-GI-NEXT:    mov b3, v0.b[3]
+; CHECK-GI-NEXT:    mov b4, v0.b[4]
+; CHECK-GI-NEXT:    mov b5, v0.b[5]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    mov b6, v0.b[6]
+; CHECK-GI-NEXT:    mov b7, v0.b[7]
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w10, s2
+; CHECK-GI-NEXT:    fmov w11, s3
+; CHECK-GI-NEXT:    fmov w12, s5
+; CHECK-GI-NEXT:    mul w8, w8, w9
+; CHECK-GI-NEXT:    fmov w9, s4
+; CHECK-GI-NEXT:    mul w10, w10, w11
+; CHECK-GI-NEXT:    fmov w11, s6
+; CHECK-GI-NEXT:    mul w9, w9, w12
+; CHECK-GI-NEXT:    fmov w12, s7
+; CHECK-GI-NEXT:    mul w8, w8, w10
+; CHECK-GI-NEXT:    mul w11, w11, w12
+; CHECK-GI-NEXT:    mul w9, w9, w11
+; CHECK-GI-NEXT:    mul w0, w8, w9
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = call i8 @llvm.vector.reduce.mul.v32i8(<32 x i8> %a)
+  ret i8 %arg1
+}
+
+define i16 @mulv_v2i16(<2 x i16> %a) {
+; CHECK-SD-LABEL: mulv_v2i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    mov w8, v0.s[1]
+; CHECK-SD-NEXT:    fmov w9, s0
+; CHECK-SD-NEXT:    mul w0, w9, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulv_v2i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    mul w0, w8, w9
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = call i16 @llvm.vector.reduce.mul.v2i16(<2 x i16> %a)
+  ret i16 %arg1
+}
+
+define i16 @mulv_v3i16(<3 x i16> %a) {
+; CHECK-SD-LABEL: mulv_v3i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    umov w8, v0.h[1]
+; CHECK-SD-NEXT:    umov w9, v0.h[0]
+; CHECK-SD-NEXT:    umov w10, v0.h[2]
+; CHECK-SD-NEXT:    mul w8, w9, w8
+; CHECK-SD-NEXT:    mul w0, w8, w10
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulv_v3i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-NEXT:    mov h2, v0.h[2]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    mul w8, w8, w9
+; CHECK-GI-NEXT:    fmov w9, s2
+; CHECK-GI-NEXT:    mul w0, w8, w9
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = call i16 @llvm.vector.reduce.mul.v3i16(<3 x i16> %a)
+  ret i16 %arg1
+}
+
+define i16 @mulv_v4i16(<4 x i16> %a) {
+; CHECK-SD-LABEL: mulv_v4i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    umov w8, v0.h[1]
+; CHECK-SD-NEXT:    umov w9, v0.h[0]
+; CHECK-SD-NEXT:    umov w10, v0.h[2]
+; CHECK-SD-NEXT:    mul w8, w9, w8
+; CHECK-SD-NEXT:    umov w9, v0.h[3]
+; CHECK-SD-NEXT:    mul w8, w8, w10
+; CHECK-SD-NEXT:    mul w0, w8, w9
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulv_v4i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-NEXT:    mov h2, v0.h[2]
+; CHECK-GI-NEXT:    mov h3, v0.h[3]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w10, s2
+; CHECK-GI-NEXT:    fmov w11, s3
+; CHECK-GI-NEXT:    mul w8, w8, w9
+; CHECK-GI-NEXT:    mul w9, w10, w11
+; CHECK-GI-NEXT:    mul w0, w8, w9
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = call i16 @llvm.vector.reduce.mul.v4i16(<4 x i16> %a)
+  ret i16 %arg1
+}
+
+define i16 @mulv_v8i16(<8 x i16> %a) {
+; CHECK-SD-LABEL: mulv_v8i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    mul v0.4h, v0.4h, v1.4h
+; CHECK-SD-NEXT:    umov w8, v0.h[1]
+; CHECK-SD-NEXT:    umov w9, v0.h[0]
+; CHECK-SD-NEXT:    umov w10, v0.h[2]
+; CHECK-SD-NEXT:    mul w8, w9, w8
+; CHECK-SD-NEXT:    umov w9, v0.h[3]
+; CHECK-SD-NEXT:    mul w8, w8, w10
+; CHECK-SD-NEXT:    mul w0, w8, w9
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulv_v8i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    mul v0.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-NEXT:    mov h2, v0.h[2]
+; CHECK-GI-NEXT:    mov h3, v0.h[3]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w10, s2
+; CHECK-GI-NEXT:    fmov w11, s3
+; CHECK-GI-NEXT:    mul w8, w8, w9
+; CHECK-GI-NEXT:    mul w9, w10, w11
+; CHECK-GI-NEXT:    mul w0, w8, w9
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = call i16 @llvm.vector.reduce.mul.v8i16(<8 x i16> %a)
+  ret i16 %arg1
+}
+
+define i16 @mulv_v16i16(<16 x i16> %a) {
+; CHECK-SD-LABEL: mulv_v16i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mul v0.8h, v0.8h, v1.8h
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    mul v0.4h, v0.4h, v1.4h
+; CHECK-SD-NEXT:    umov w8, v0.h[1]
+; CHECK-SD-NEXT:    umov w9, v0.h[0]
+; CHECK-SD-NEXT:    umov w10, v0.h[2]
+; CHECK-SD-NEXT:    mul w8, w9, w8
+; CHECK-SD-NEXT:    umov w9, v0.h[3]
+; CHECK-SD-NEXT:    mul w8, w8, w10
+; CHECK-SD-NEXT:    mul w0, w8, w9
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulv_v16i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d2, v0.d[1]
+; CHECK-GI-NEXT:    mov d3, v1.d[1]
+; CHECK-GI-NEXT:    mul v0.4h, v0.4h, v2.4h
+; CHECK-GI-NEXT:    mul v1.4h, v1.4h, v3.4h
+; CHECK-GI-NEXT:    mul v0.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-NEXT:    mov h2, v0.h[2]
+; CHECK-GI-NEXT:    mov h3, v0.h[3]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w10, s2
+; CHECK-GI-NEXT:    mul w8, w8, w9
+; CHECK-GI-NEXT:    fmov w9, s3
+; CHECK-GI-NEXT:    mul w9, w10, w9
+; CHECK-GI-NEXT:    mul w0, w8, w9
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = call i16 @llvm.vector.reduce.mul.v16i16(<16 x i16> %a)
+  ret i16 %arg1
+}
+
+define i32 @mulv_v2i32(<2 x i32> %a) {
+; CHECK-SD-LABEL: mulv_v2i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    mov w8, v0.s[1]
+; CHECK-SD-NEXT:    fmov w9, s0
+; CHECK-SD-NEXT:    mul w0, w9, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulv_v2i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    mul w0, w8, w9
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = call i32 @llvm.vector.reduce.mul.v2i32(<2 x i32> %a)
+  ret i32 %arg1
+}
+
+define i32 @mulv_v3i32(<3 x i32> %a) {
+; CHECK-LABEL: mulv_v3i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    mov w8, #1 // =0x1
+; CHECK-NEXT:    mov v1.s[3], w8
+; CHECK-NEXT:    ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-NEXT:    mul v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    mov w8, v0.s[1]
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    mul w0, w9, w8
+; CHECK-NEXT:    ret
+entry:
+  %arg1 = call i32 @llvm.vector.reduce.mul.v3i32(<3 x i32> %a)
+  ret i32 %arg1
+}
+
+define i32 @mulv_v4i32(<4 x i32> %a) {
+; CHECK-SD-LABEL: mulv_v4i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    mul v0.2s, v0.2s, v1.2s
+; CHECK-SD-NEXT:    mov w8, v0.s[1]
+; CHECK-SD-NEXT:    fmov w9, s0
+; CHECK-SD-NEXT:    mul w0, w9, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulv_v4i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    mul v0.2s, v0.2s, v1.2s
+; CHECK-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    mul w0, w8, w9
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> %a)
+  ret i32 %arg1
+}
+
+define i32 @mulv_v8i32(<8 x i32> %a) {
+; CHECK-SD-LABEL: mulv_v8i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mul v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    mul v0.2s, v0.2s, v1.2s
+; CHECK-SD-NEXT:    mov w8, v0.s[1]
+; CHECK-SD-NEXT:    fmov w9, s0
+; CHECK-SD-NEXT:    mul w0, w9, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulv_v8i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d2, v0.d[1]
+; CHECK-GI-NEXT:    mov d3, v1.d[1]
+; CHECK-GI-NEXT:    mul v0.2s, v0.2s, v2.2s
+; CHECK-GI-NEXT:    mul v1.2s, v1.2s, v3.2s
+; CHECK-GI-NEXT:    mul v0.2s, v0.2s, v1.2s
+; CHECK-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    mul w0, w8, w9
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = call i32 @llvm.vector.reduce.mul.v8i32(<8 x i32> %a)
+  ret i32 %arg1
+}
+
+define i64 @mulv_v2i64(<2 x i64> %a) {
+; CHECK-SD-LABEL: mulv_v2i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov x8, v0.d[1]
+; CHECK-SD-NEXT:    fmov x9, d0
+; CHECK-SD-NEXT:    mul x0, x9, x8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulv_v2i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    fmov x8, d0
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    mul x0, x8, x9
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = call i64 @llvm.vector.reduce.mul.v2i64(<2 x i64> %a)
+  ret i64 %arg1
+}
+
+define i64 @mulv_v3i64(<3 x i64> %a) {
+; CHECK-SD-LABEL: mulv_v3i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    fmov x8, d2
+; CHECK-SD-NEXT:    fmov x9, d0
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-SD-NEXT:    mul x8, x9, x8
+; CHECK-SD-NEXT:    fmov x9, d1
+; CHECK-SD-NEXT:    mul x0, x9, x8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulv_v3i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov x8, d0
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    mul x8, x8, x9
+; CHECK-GI-NEXT:    fmov x9, d2
+; CHECK-GI-NEXT:    mul x0, x8, x9
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = call i64 @llvm.vector.reduce.mul.v3i64(<3 x i64> %a)
+  ret i64 %arg1
+}
+
+define i64 @mulv_v4i64(<4 x i64> %a) {
+; CHECK-SD-LABEL: mulv_v4i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov x8, v1.d[1]
+; CHECK-SD-NEXT:    mov x9, v0.d[1]
+; CHECK-SD-NEXT:    fmov x10, d0
+; CHECK-SD-NEXT:    mul x8, x9, x8
+; CHECK-SD-NEXT:    fmov x9, d1
+; CHECK-SD-NEXT:    mul x9, x10, x9
+; CHECK-SD-NEXT:    mul x0, x9, x8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulv_v4i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d2, v0.d[1]
+; CHECK-GI-NEXT:    mov d3, v1.d[1]
+; CHECK-GI-NEXT:    fmov x8, d0
+; CHECK-GI-NEXT:    fmov x9, d2
+; CHECK-GI-NEXT:    fmov x10, d3
+; CHECK-GI-NEXT:    mul x8, x8, x9
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    mul x9, x9, x10
+; CHECK-GI-NEXT:    mul x0, x8, x9
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = call i64 @llvm.vector.reduce.mul.v4i64(<4 x i64> %a)
+  ret i64 %arg1
+}
+
+define i128 @mulv_v2i128(<2 x i128> %a) {
+; CHECK-SD-LABEL: mulv_v2i128:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    umulh x8, x0, x2
+; CHECK-SD-NEXT:    madd x8, x0, x3, x8
+; CHECK-SD-NEXT:    mul x0, x0, x2
+; CHECK-SD-NEXT:    madd x1, x1, x2, x8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulv_v2i128:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mul x9, x0, x3
+; CHECK-GI-NEXT:    mul x8, x0, x2
+; CHECK-GI-NEXT:    umulh x10, x0, x2
+; CHECK-GI-NEXT:    madd x9, x1, x2, x9
+; CHECK-GI-NEXT:    mov x0, x8
+; CHECK-GI-NEXT:    add x1, x9, x10
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = call i128 @llvm.vector.reduce.mul.v2i128(<2 x i128> %a)
+  ret i128 %arg1
+}
diff --git a/llvm/test/CodeGen/AArch64/arm64-homogeneous-prolog-epilog-odd-csrs.ll b/llvm/test/CodeGen/AArch64/arm64-homogeneous-prolog-epilog-odd-csrs.ll
new file mode 100644
index 0000000000..3b90163e6d
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/arm64-homogeneous-prolog-epilog-odd-csrs.ll
@@ -0,0 +1,31 @@
+; RUN: llc < %s -mtriple=arm64-apple-ios7.0 -homogeneous-prolog-epilog | FileCheck %s
+; RUN: llc < %s -mtriple=aarch64-unknown-linux-gnu  -homogeneous-prolog-epilog | FileCheck %s --check-prefixes=CHECK-LINUX
+
+declare void @bar(i32 %i)
+
+define void @odd_num_callee_saved_registers(ptr swifterror %error, i32 %i) nounwind minsize {
+  call void asm sideeffect "mov x0, #42", "~{x0},~{x19},~{x20},~{x22},~{x23},~{x24},~{x25},~{x26},~{x27},~{x28}"() nounwind
+  call void @bar(i32 %i)
+  ret void
+}
+
+define void @odd_num_callee_saved_registers_with_fpr(ptr swifterror %error, i32 %i) nounwind minsize {
+  call void asm sideeffect "mov x0, #42", "~{x0},~{x19},~{x20},~{x22},~{x23},~{x24},~{x25},~{x26},~{x27},~{x28},~{d8},~{d9}"() nounwind
+  call void @bar(i32 %i)
+  ret void
+}
+
+; CHECK-LABEL: _OUTLINED_FUNCTION_PROLOG_x30x29x19x20x22x23x24x25x26x27x28:
+; CHECK:	str	x28, [sp, #-80]!
+; CHECK-LABEL: _OUTLINED_FUNCTION_EPILOG_TAIL_x30x29x19x20x22x23x24x25x26x27x28:
+; CHECK:	ldr	x28, [sp], #96
+
+; CHECK-LABEL: _OUTLINED_FUNCTION_PROLOG_x30x29x19x20x22x23x24x25x26x27x28d8d9:
+; CHECK:	stp	d9, d8, [sp, #-96]!
+; CHECK:	str	x28, [sp, #16]
+; CHECK-LABEL: _OUTLINED_FUNCTION_EPILOG_TAIL_x30x29x19x20x22x23x24x25x26x27x28d8d9
+; CHECK:	ldr	x28, [sp, #16]
+; CHECK:	ldp	d9, d8, [sp], #112
+
+; CHECK-LINUX-NOT: OUTLINED_FUNCTION_PROLOG
+; CHECK-LINUX-NOT: OUTLINED_FUNCTION_EPILOG
diff --git a/llvm/test/CodeGen/AArch64/arm64-indexed-memory.ll b/llvm/test/CodeGen/AArch64/arm64-indexed-memory.ll
index d1747e7ca1..2765e22617 100644
--- a/llvm/test/CodeGen/AArch64/arm64-indexed-memory.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-indexed-memory.ll
@@ -1,12 +1,25 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=arm64-apple-ios -aarch64-redzone | FileCheck %s --check-prefixes=CHECK,CHECK64
+; RUN: llc < %s -mtriple=arm64-apple-ios -aarch64-redzone -global-isel -global-isel-abort=1 | FileCheck %s --check-prefixes=CHECK,GISEL
 ; RUN: llc < %s -mtriple=arm64_32-apple-ios -aarch64-redzone | FileCheck %s --check-prefixes=CHECK,CHECK32
 
 define ptr @store64(ptr %ptr, i64 %index, i64 %spacing) {
-; CHECK-LABEL: store64:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    str x2, [x0], #8
-; CHECK-NEXT:    ret
+; CHECK64-LABEL: store64:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    str x2, [x0], #8
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: store64:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    mov x8, x0
+; GISEL-NEXT:    add x0, x0, #8
+; GISEL-NEXT:    str x2, [x8]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: store64:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    str x2, [x0], #8
+; CHECK32-NEXT:    ret
   %incdec.ptr = getelementptr inbounds i64, ptr %ptr, i64 1
   store i64 %spacing, ptr %ptr, align 4
   ret ptr %incdec.ptr
@@ -25,20 +38,44 @@ define ptr @store64idxpos256(ptr %ptr, i64 %index, i64 %spacing) {
 }
 
 define ptr @store64idxneg256(ptr %ptr, i64 %index, i64 %spacing) {
-; CHECK-LABEL: store64idxneg256:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    str x2, [x0], #-256
-; CHECK-NEXT:    ret
+; CHECK64-LABEL: store64idxneg256:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    str x2, [x0], #-256
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: store64idxneg256:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    mov x8, x0
+; GISEL-NEXT:    sub x0, x0, #256
+; GISEL-NEXT:    str x2, [x8]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: store64idxneg256:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    str x2, [x0], #-256
+; CHECK32-NEXT:    ret
   %incdec.ptr = getelementptr inbounds i64, ptr %ptr, i64 -32
   store i64 %spacing, ptr %ptr, align 4
   ret ptr %incdec.ptr
 }
 
 define ptr @store32(ptr %ptr, i32 %index, i32 %spacing) {
-; CHECK-LABEL: store32:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    str w2, [x0], #4
-; CHECK-NEXT:    ret
+; CHECK64-LABEL: store32:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    str w2, [x0], #4
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: store32:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    mov x8, x0
+; GISEL-NEXT:    add x0, x0, #4
+; GISEL-NEXT:    str w2, [x8]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: store32:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    str w2, [x0], #4
+; CHECK32-NEXT:    ret
   %incdec.ptr = getelementptr inbounds i32, ptr %ptr, i64 1
   store i32 %spacing, ptr %ptr, align 4
   ret ptr %incdec.ptr
@@ -57,20 +94,44 @@ define ptr @store32idxpos256(ptr %ptr, i32 %index, i32 %spacing) {
 }
 
 define ptr @store32idxneg256(ptr %ptr, i32 %index, i32 %spacing) {
-; CHECK-LABEL: store32idxneg256:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    str w2, [x0], #-256
-; CHECK-NEXT:    ret
+; CHECK64-LABEL: store32idxneg256:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    str w2, [x0], #-256
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: store32idxneg256:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    mov x8, x0
+; GISEL-NEXT:    sub x0, x0, #256
+; GISEL-NEXT:    str w2, [x8]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: store32idxneg256:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    str w2, [x0], #-256
+; CHECK32-NEXT:    ret
   %incdec.ptr = getelementptr inbounds i32, ptr %ptr, i64 -64
   store i32 %spacing, ptr %ptr, align 4
   ret ptr %incdec.ptr
 }
 
 define ptr @store16(ptr %ptr, i16 %index, i16 %spacing) {
-; CHECK-LABEL: store16:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    strh w2, [x0], #2
-; CHECK-NEXT:    ret
+; CHECK64-LABEL: store16:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    strh w2, [x0], #2
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: store16:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    mov x8, x0
+; GISEL-NEXT:    add x0, x0, #2
+; GISEL-NEXT:    strh w2, [x8]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: store16:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    strh w2, [x0], #2
+; CHECK32-NEXT:    ret
   %incdec.ptr = getelementptr inbounds i16, ptr %ptr, i64 1
   store i16 %spacing, ptr %ptr, align 4
   ret ptr %incdec.ptr
@@ -89,20 +150,44 @@ define ptr @store16idxpos256(ptr %ptr, i16 %index, i16 %spacing) {
 }
 
 define ptr @store16idxneg256(ptr %ptr, i16 %index, i16 %spacing) {
-; CHECK-LABEL: store16idxneg256:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    strh w2, [x0], #-256
-; CHECK-NEXT:    ret
+; CHECK64-LABEL: store16idxneg256:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    strh w2, [x0], #-256
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: store16idxneg256:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    mov x8, x0
+; GISEL-NEXT:    sub x0, x0, #256
+; GISEL-NEXT:    strh w2, [x8]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: store16idxneg256:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    strh w2, [x0], #-256
+; CHECK32-NEXT:    ret
   %incdec.ptr = getelementptr inbounds i16, ptr %ptr, i64 -128
   store i16 %spacing, ptr %ptr, align 4
   ret ptr %incdec.ptr
 }
 
 define ptr @store8(ptr %ptr, i8 %index, i8 %spacing) {
-; CHECK-LABEL: store8:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    strb w2, [x0], #1
-; CHECK-NEXT:    ret
+; CHECK64-LABEL: store8:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    strb w2, [x0], #1
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: store8:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    mov x8, x0
+; GISEL-NEXT:    add x0, x0, #1
+; GISEL-NEXT:    strb w2, [x8]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: store8:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    strb w2, [x0], #1
+; CHECK32-NEXT:    ret
   %incdec.ptr = getelementptr inbounds i8, ptr %ptr, i64 1
   store i8 %spacing, ptr %ptr, align 4
   ret ptr %incdec.ptr
@@ -121,20 +206,44 @@ define ptr @store8idxpos256(ptr %ptr, i8 %index, i8 %spacing) {
 }
 
 define ptr @store8idxneg256(ptr %ptr, i8 %index, i8 %spacing) {
-; CHECK-LABEL: store8idxneg256:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    strb w2, [x0], #-256
-; CHECK-NEXT:    ret
+; CHECK64-LABEL: store8idxneg256:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    strb w2, [x0], #-256
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: store8idxneg256:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    mov x8, x0
+; GISEL-NEXT:    sub x0, x0, #256
+; GISEL-NEXT:    strb w2, [x8]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: store8idxneg256:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    strb w2, [x0], #-256
+; CHECK32-NEXT:    ret
   %incdec.ptr = getelementptr inbounds i8, ptr %ptr, i64 -256
   store i8 %spacing, ptr %ptr, align 4
   ret ptr %incdec.ptr
 }
 
 define ptr @truncst64to32(ptr %ptr, i32 %index, i64 %spacing) {
-; CHECK-LABEL: truncst64to32:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    str w2, [x0], #4
-; CHECK-NEXT:    ret
+; CHECK64-LABEL: truncst64to32:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    str w2, [x0], #4
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: truncst64to32:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    mov x8, x0
+; GISEL-NEXT:    add x0, x0, #4
+; GISEL-NEXT:    str w2, [x8]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: truncst64to32:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    str w2, [x0], #4
+; CHECK32-NEXT:    ret
   %incdec.ptr = getelementptr inbounds i32, ptr %ptr, i64 1
   %trunc = trunc i64 %spacing to i32
   store i32 %trunc, ptr %ptr, align 4
@@ -142,10 +251,22 @@ define ptr @truncst64to32(ptr %ptr, i32 %index, i64 %spacing) {
 }
 
 define ptr @truncst64to16(ptr %ptr, i16 %index, i64 %spacing) {
-; CHECK-LABEL: truncst64to16:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    strh w2, [x0], #2
-; CHECK-NEXT:    ret
+; CHECK64-LABEL: truncst64to16:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    strh w2, [x0], #2
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: truncst64to16:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    mov x8, x0
+; GISEL-NEXT:    add x0, x0, #2
+; GISEL-NEXT:    strh w2, [x8]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: truncst64to16:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    strh w2, [x0], #2
+; CHECK32-NEXT:    ret
   %incdec.ptr = getelementptr inbounds i16, ptr %ptr, i64 1
   %trunc = trunc i64 %spacing to i16
   store i16 %trunc, ptr %ptr, align 4
@@ -153,10 +274,22 @@ define ptr @truncst64to16(ptr %ptr, i16 %index, i64 %spacing) {
 }
 
 define ptr @truncst64to8(ptr %ptr, i8 %index, i64 %spacing) {
-; CHECK-LABEL: truncst64to8:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    strb w2, [x0], #1
-; CHECK-NEXT:    ret
+; CHECK64-LABEL: truncst64to8:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    strb w2, [x0], #1
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: truncst64to8:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    mov x8, x0
+; GISEL-NEXT:    add x0, x0, #1
+; GISEL-NEXT:    strb w2, [x8]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: truncst64to8:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    strb w2, [x0], #1
+; CHECK32-NEXT:    ret
   %incdec.ptr = getelementptr inbounds i8, ptr %ptr, i64 1
   %trunc = trunc i64 %spacing to i8
   store i8 %trunc, ptr %ptr, align 4
@@ -165,30 +298,66 @@ define ptr @truncst64to8(ptr %ptr, i8 %index, i64 %spacing) {
 
 
 define ptr @storef16(ptr %ptr, half %index, half %spacing) nounwind {
-; CHECK-LABEL: storef16:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    str h1, [x0], #2
-; CHECK-NEXT:    ret
+; CHECK64-LABEL: storef16:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    str h1, [x0], #2
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: storef16:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    mov x8, x0
+; GISEL-NEXT:    add x0, x0, #2
+; GISEL-NEXT:    str h1, [x8]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: storef16:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    str h1, [x0], #2
+; CHECK32-NEXT:    ret
   %incdec.ptr = getelementptr inbounds half, ptr %ptr, i64 1
   store half %spacing, ptr %ptr, align 2
   ret ptr %incdec.ptr
 }
 
 define ptr @storef32(ptr %ptr, float %index, float %spacing) {
-; CHECK-LABEL: storef32:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    str s1, [x0], #4
-; CHECK-NEXT:    ret
+; CHECK64-LABEL: storef32:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    str s1, [x0], #4
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: storef32:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    mov x8, x0
+; GISEL-NEXT:    add x0, x0, #4
+; GISEL-NEXT:    str s1, [x8]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: storef32:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    str s1, [x0], #4
+; CHECK32-NEXT:    ret
   %incdec.ptr = getelementptr inbounds float, ptr %ptr, i64 1
   store float %spacing, ptr %ptr, align 4
   ret ptr %incdec.ptr
 }
 
 define ptr @storef64(ptr %ptr, double %index, double %spacing) {
-; CHECK-LABEL: storef64:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    str d1, [x0], #8
-; CHECK-NEXT:    ret
+; CHECK64-LABEL: storef64:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    str d1, [x0], #8
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: storef64:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    mov x8, x0
+; GISEL-NEXT:    add x0, x0, #8
+; GISEL-NEXT:    str d1, [x8]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: storef64:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    str d1, [x0], #8
+; CHECK32-NEXT:    ret
   %incdec.ptr = getelementptr inbounds double, ptr %ptr, i64 1
   store double %spacing, ptr %ptr, align 4
   ret ptr %incdec.ptr
@@ -196,40 +365,88 @@ define ptr @storef64(ptr %ptr, double %index, double %spacing) {
 
 
 define ptr @pref64(ptr %ptr, double %spacing) {
-; CHECK-LABEL: pref64:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    str d0, [x0, #32]!
-; CHECK-NEXT:    ret
+; CHECK64-LABEL: pref64:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    str d0, [x0, #32]!
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: pref64:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    mov x8, x0
+; GISEL-NEXT:    add x0, x0, #32
+; GISEL-NEXT:    str d0, [x8, #32]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: pref64:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    str d0, [x0, #32]!
+; CHECK32-NEXT:    ret
   %incdec.ptr = getelementptr inbounds double, ptr %ptr, i64 4
   store double %spacing, ptr %incdec.ptr, align 4
   ret ptr %incdec.ptr
 }
 
 define ptr @pref32(ptr %ptr, float %spacing) {
-; CHECK-LABEL: pref32:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    str s0, [x0, #12]!
-; CHECK-NEXT:    ret
+; CHECK64-LABEL: pref32:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    str s0, [x0, #12]!
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: pref32:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    mov x8, x0
+; GISEL-NEXT:    add x0, x0, #12
+; GISEL-NEXT:    str s0, [x8, #12]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: pref32:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    str s0, [x0, #12]!
+; CHECK32-NEXT:    ret
   %incdec.ptr = getelementptr inbounds float, ptr %ptr, i64 3
   store float %spacing, ptr %incdec.ptr, align 4
   ret ptr %incdec.ptr
 }
 
 define ptr @pref16(ptr %ptr, half %spacing) nounwind {
-; CHECK-LABEL: pref16:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    str h0, [x0, #6]!
-; CHECK-NEXT:    ret
+; CHECK64-LABEL: pref16:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    str h0, [x0, #6]!
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: pref16:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    mov x8, x0
+; GISEL-NEXT:    add x0, x0, #6
+; GISEL-NEXT:    str h0, [x8, #6]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: pref16:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    str h0, [x0, #6]!
+; CHECK32-NEXT:    ret
   %incdec.ptr = getelementptr inbounds half, ptr %ptr, i64 3
   store half %spacing, ptr %incdec.ptr, align 2
   ret ptr %incdec.ptr
 }
 
 define ptr @pre64(ptr %ptr, i64 %spacing) {
-; CHECK-LABEL: pre64:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    str x1, [x0, #16]!
-; CHECK-NEXT:    ret
+; CHECK64-LABEL: pre64:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    str x1, [x0, #16]!
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: pre64:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    mov x8, x0
+; GISEL-NEXT:    add x0, x0, #16
+; GISEL-NEXT:    str x1, [x8, #16]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: pre64:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    str x1, [x0, #16]!
+; CHECK32-NEXT:    ret
   %incdec.ptr = getelementptr inbounds i64, ptr %ptr, i64 2
   store i64 %spacing, ptr %incdec.ptr, align 4
   ret ptr %incdec.ptr
@@ -248,20 +465,44 @@ define ptr @pre64idxpos256(ptr %ptr, i64 %spacing) {
 }
 
 define ptr @pre64idxneg256(ptr %ptr, i64 %spacing) {
-; CHECK-LABEL: pre64idxneg256:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    str x1, [x0, #-256]!
-; CHECK-NEXT:    ret
+; CHECK64-LABEL: pre64idxneg256:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    str x1, [x0, #-256]!
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: pre64idxneg256:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    mov x8, x0
+; GISEL-NEXT:    sub x0, x0, #256
+; GISEL-NEXT:    stur x1, [x8, #-256]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: pre64idxneg256:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    str x1, [x0, #-256]!
+; CHECK32-NEXT:    ret
   %incdec.ptr = getelementptr inbounds i64, ptr %ptr, i64 -32
   store i64 %spacing, ptr %incdec.ptr, align 4
   ret ptr %incdec.ptr
 }
 
 define ptr @pre32(ptr %ptr, i32 %spacing) {
-; CHECK-LABEL: pre32:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    str w1, [x0, #8]!
-; CHECK-NEXT:    ret
+; CHECK64-LABEL: pre32:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    str w1, [x0, #8]!
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: pre32:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    mov x8, x0
+; GISEL-NEXT:    add x0, x0, #8
+; GISEL-NEXT:    str w1, [x8, #8]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: pre32:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    str w1, [x0, #8]!
+; CHECK32-NEXT:    ret
   %incdec.ptr = getelementptr inbounds i32, ptr %ptr, i64 2
   store i32 %spacing, ptr %incdec.ptr, align 4
   ret ptr %incdec.ptr
@@ -280,20 +521,44 @@ define ptr @pre32idxpos256(ptr %ptr, i32 %spacing) {
 }
 
 define ptr @pre32idxneg256(ptr %ptr, i32 %spacing) {
-; CHECK-LABEL: pre32idxneg256:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    str w1, [x0, #-256]!
-; CHECK-NEXT:    ret
+; CHECK64-LABEL: pre32idxneg256:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    str w1, [x0, #-256]!
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: pre32idxneg256:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    mov x8, x0
+; GISEL-NEXT:    sub x0, x0, #256
+; GISEL-NEXT:    stur w1, [x8, #-256]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: pre32idxneg256:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    str w1, [x0, #-256]!
+; CHECK32-NEXT:    ret
   %incdec.ptr = getelementptr inbounds i32, ptr %ptr, i64 -64
   store i32 %spacing, ptr %incdec.ptr, align 4
   ret ptr %incdec.ptr
 }
 
 define ptr @pre16(ptr %ptr, i16 %spacing) {
-; CHECK-LABEL: pre16:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    strh w1, [x0, #4]!
-; CHECK-NEXT:    ret
+; CHECK64-LABEL: pre16:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    strh w1, [x0, #4]!
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: pre16:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    mov x8, x0
+; GISEL-NEXT:    add x0, x0, #4
+; GISEL-NEXT:    strh w1, [x8, #4]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: pre16:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    strh w1, [x0, #4]!
+; CHECK32-NEXT:    ret
   %incdec.ptr = getelementptr inbounds i16, ptr %ptr, i64 2
   store i16 %spacing, ptr %incdec.ptr, align 4
   ret ptr %incdec.ptr
@@ -312,20 +577,44 @@ define ptr @pre16idxpos256(ptr %ptr, i16 %spacing) {
 }
 
 define ptr @pre16idxneg256(ptr %ptr, i16 %spacing) {
-; CHECK-LABEL: pre16idxneg256:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    strh w1, [x0, #-256]!
-; CHECK-NEXT:    ret
+; CHECK64-LABEL: pre16idxneg256:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    strh w1, [x0, #-256]!
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: pre16idxneg256:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    mov x8, x0
+; GISEL-NEXT:    sub x0, x0, #256
+; GISEL-NEXT:    sturh w1, [x8, #-256]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: pre16idxneg256:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    strh w1, [x0, #-256]!
+; CHECK32-NEXT:    ret
   %incdec.ptr = getelementptr inbounds i16, ptr %ptr, i64 -128
   store i16 %spacing, ptr %incdec.ptr, align 4
   ret ptr %incdec.ptr
 }
 
 define ptr @pre8(ptr %ptr, i8 %spacing) {
-; CHECK-LABEL: pre8:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    strb w1, [x0, #2]!
-; CHECK-NEXT:    ret
+; CHECK64-LABEL: pre8:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    strb w1, [x0, #2]!
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: pre8:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    mov x8, x0
+; GISEL-NEXT:    add x0, x0, #2
+; GISEL-NEXT:    strb w1, [x8, #2]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: pre8:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    strb w1, [x0, #2]!
+; CHECK32-NEXT:    ret
   %incdec.ptr = getelementptr inbounds i8, ptr %ptr, i64 2
   store i8 %spacing, ptr %incdec.ptr, align 4
   ret ptr %incdec.ptr
@@ -344,20 +633,44 @@ define ptr @pre8idxpos256(ptr %ptr, i8 %spacing) {
 }
 
 define ptr @pre8idxneg256(ptr %ptr, i8 %spacing) {
-; CHECK-LABEL: pre8idxneg256:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    strb w1, [x0, #-256]!
-; CHECK-NEXT:    ret
+; CHECK64-LABEL: pre8idxneg256:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    strb w1, [x0, #-256]!
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: pre8idxneg256:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    mov x8, x0
+; GISEL-NEXT:    sub x0, x0, #256
+; GISEL-NEXT:    sturb w1, [x8, #-256]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: pre8idxneg256:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    strb w1, [x0, #-256]!
+; CHECK32-NEXT:    ret
   %incdec.ptr = getelementptr inbounds i8, ptr %ptr, i64 -256
   store i8 %spacing, ptr %incdec.ptr, align 4
   ret ptr %incdec.ptr
 }
 
 define ptr @pretrunc64to32(ptr %ptr, i64 %spacing) {
-; CHECK-LABEL: pretrunc64to32:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    str w1, [x0, #8]!
-; CHECK-NEXT:    ret
+; CHECK64-LABEL: pretrunc64to32:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    str w1, [x0, #8]!
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: pretrunc64to32:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    mov x8, x0
+; GISEL-NEXT:    add x0, x0, #8
+; GISEL-NEXT:    str w1, [x8, #8]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: pretrunc64to32:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    str w1, [x0, #8]!
+; CHECK32-NEXT:    ret
   %incdec.ptr = getelementptr inbounds i32, ptr %ptr, i64 2
   %trunc = trunc i64 %spacing to i32
   store i32 %trunc, ptr %incdec.ptr, align 4
@@ -365,10 +678,22 @@ define ptr @pretrunc64to32(ptr %ptr, i64 %spacing) {
 }
 
 define ptr @pretrunc64to16(ptr %ptr, i64 %spacing) {
-; CHECK-LABEL: pretrunc64to16:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    strh w1, [x0, #4]!
-; CHECK-NEXT:    ret
+; CHECK64-LABEL: pretrunc64to16:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    strh w1, [x0, #4]!
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: pretrunc64to16:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    mov x8, x0
+; GISEL-NEXT:    add x0, x0, #4
+; GISEL-NEXT:    strh w1, [x8, #4]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: pretrunc64to16:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    strh w1, [x0, #4]!
+; CHECK32-NEXT:    ret
   %incdec.ptr = getelementptr inbounds i16, ptr %ptr, i64 2
   %trunc = trunc i64 %spacing to i16
   store i16 %trunc, ptr %incdec.ptr, align 4
@@ -376,10 +701,22 @@ define ptr @pretrunc64to16(ptr %ptr, i64 %spacing) {
 }
 
 define ptr @pretrunc64to8(ptr %ptr, i64 %spacing) {
-; CHECK-LABEL: pretrunc64to8:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    strb w1, [x0, #2]!
-; CHECK-NEXT:    ret
+; CHECK64-LABEL: pretrunc64to8:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    strb w1, [x0, #2]!
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: pretrunc64to8:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    mov x8, x0
+; GISEL-NEXT:    add x0, x0, #2
+; GISEL-NEXT:    strb w1, [x8, #2]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: pretrunc64to8:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    strb w1, [x0, #2]!
+; CHECK32-NEXT:    ret
   %incdec.ptr = getelementptr inbounds i8, ptr %ptr, i64 2
   %trunc = trunc i64 %spacing to i8
   store i8 %trunc, ptr %incdec.ptr, align 4
@@ -414,11 +751,24 @@ define ptr @preidxf32(ptr %src, ptr %out) {
 }
 
 define ptr @preidxf16(ptr %src, ptr %out) {
-; CHECK-LABEL: preidxf16:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ldr h0, [x0, #2]!
-; CHECK-NEXT:    str h0, [x1]
-; CHECK-NEXT:    ret
+; CHECK64-LABEL: preidxf16:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    ldr h0, [x0, #2]!
+; CHECK64-NEXT:    str h0, [x1]
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: preidxf16:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    ldr h0, [x0, #2]
+; GISEL-NEXT:    add x0, x0, #2
+; GISEL-NEXT:    str h0, [x1]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: preidxf16:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    ldr h0, [x0, #2]!
+; CHECK32-NEXT:    str h0, [x1]
+; CHECK32-NEXT:    ret
   %ptr = getelementptr inbounds half, ptr %src, i64 1
   %tmp = load half, ptr %ptr, align 2
   store half %tmp, ptr %out, align 2
@@ -502,11 +852,24 @@ define ptr @preidx8zext64(ptr %src, ptr %out) {
 }
 
 define ptr @preidx32sext64(ptr %src, ptr %out) {
-; CHECK-LABEL: preidx32sext64:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ldrsw x8, [x0, #4]!
-; CHECK-NEXT:    str x8, [x1]
-; CHECK-NEXT:    ret
+; CHECK64-LABEL: preidx32sext64:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    ldrsw x8, [x0, #4]!
+; CHECK64-NEXT:    str x8, [x1]
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: preidx32sext64:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    ldrsw x8, [x0, #4]
+; GISEL-NEXT:    add x0, x0, #4
+; GISEL-NEXT:    str x8, [x1]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: preidx32sext64:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    ldrsw x8, [x0, #4]!
+; CHECK32-NEXT:    str x8, [x1]
+; CHECK32-NEXT:    ret
   %ptr = getelementptr inbounds i32, ptr %src, i64 1
   %tmp = load i32, ptr %ptr, align 4
   %ext = sext i32 %tmp to i64
@@ -515,11 +878,24 @@ define ptr @preidx32sext64(ptr %src, ptr %out) {
 }
 
 define ptr @preidx16sext32(ptr %src, ptr %out) {
-; CHECK-LABEL: preidx16sext32:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ldrsh w8, [x0, #2]!
-; CHECK-NEXT:    str w8, [x1]
-; CHECK-NEXT:    ret
+; CHECK64-LABEL: preidx16sext32:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    ldrsh w8, [x0, #2]!
+; CHECK64-NEXT:    str w8, [x1]
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: preidx16sext32:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    ldrsh w8, [x0, #2]
+; GISEL-NEXT:    add x0, x0, #2
+; GISEL-NEXT:    str w8, [x1]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: preidx16sext32:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    ldrsh w8, [x0, #2]!
+; CHECK32-NEXT:    str w8, [x1]
+; CHECK32-NEXT:    ret
   %ptr = getelementptr inbounds i16, ptr %src, i64 1
   %tmp = load i16, ptr %ptr, align 4
   %ext = sext i16 %tmp to i32
@@ -528,11 +904,24 @@ define ptr @preidx16sext32(ptr %src, ptr %out) {
 }
 
 define ptr @preidx16sext64(ptr %src, ptr %out) {
-; CHECK-LABEL: preidx16sext64:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ldrsh x8, [x0, #2]!
-; CHECK-NEXT:    str x8, [x1]
-; CHECK-NEXT:    ret
+; CHECK64-LABEL: preidx16sext64:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    ldrsh x8, [x0, #2]!
+; CHECK64-NEXT:    str x8, [x1]
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: preidx16sext64:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    ldrsh x8, [x0, #2]
+; GISEL-NEXT:    add x0, x0, #2
+; GISEL-NEXT:    str x8, [x1]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: preidx16sext64:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    ldrsh x8, [x0, #2]!
+; CHECK32-NEXT:    str x8, [x1]
+; CHECK32-NEXT:    ret
   %ptr = getelementptr inbounds i16, ptr %src, i64 1
   %tmp = load i16, ptr %ptr, align 4
   %ext = sext i16 %tmp to i64
@@ -541,11 +930,24 @@ define ptr @preidx16sext64(ptr %src, ptr %out) {
 }
 
 define ptr @preidx8sext32(ptr %src, ptr %out) {
-; CHECK-LABEL: preidx8sext32:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ldrsb w8, [x0, #1]!
-; CHECK-NEXT:    str w8, [x1]
-; CHECK-NEXT:    ret
+; CHECK64-LABEL: preidx8sext32:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    ldrsb w8, [x0, #1]!
+; CHECK64-NEXT:    str w8, [x1]
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: preidx8sext32:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    ldrsb w8, [x0, #1]
+; GISEL-NEXT:    add x0, x0, #1
+; GISEL-NEXT:    str w8, [x1]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: preidx8sext32:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    ldrsb w8, [x0, #1]!
+; CHECK32-NEXT:    str w8, [x1]
+; CHECK32-NEXT:    ret
   %ptr = getelementptr inbounds i8, ptr %src, i64 1
   %tmp = load i8, ptr %ptr, align 4
   %ext = sext i8 %tmp to i32
@@ -554,11 +956,24 @@ define ptr @preidx8sext32(ptr %src, ptr %out) {
 }
 
 define ptr @preidx8sext64(ptr %src, ptr %out) {
-; CHECK-LABEL: preidx8sext64:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ldrsb x8, [x0, #1]!
-; CHECK-NEXT:    str x8, [x1]
-; CHECK-NEXT:    ret
+; CHECK64-LABEL: preidx8sext64:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    ldrsb x8, [x0, #1]!
+; CHECK64-NEXT:    str x8, [x1]
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: preidx8sext64:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    ldrsb x8, [x0, #1]
+; GISEL-NEXT:    add x0, x0, #1
+; GISEL-NEXT:    str x8, [x1]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: preidx8sext64:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    ldrsb x8, [x0, #1]!
+; CHECK32-NEXT:    str x8, [x1]
+; CHECK32-NEXT:    ret
   %ptr = getelementptr inbounds i8, ptr %src, i64 1
   %tmp = load i8, ptr %ptr, align 4
   %ext = sext i8 %tmp to i64
@@ -576,6 +991,13 @@ define ptr @postidx_clobber(ptr %addr) nounwind noinline ssp {
 ; CHECK64-NEXT:    mov x0, x8
 ; CHECK64-NEXT:    ret
 ;
+; GISEL-LABEL: postidx_clobber:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    mov x8, x0
+; GISEL-NEXT:    add x0, x0, #8
+; GISEL-NEXT:    str x8, [x8]
+; GISEL-NEXT:    ret
+;
 ; CHECK32-LABEL: postidx_clobber:
 ; CHECK32:       ; %bb.0:
 ; CHECK32-NEXT:    mov x8, x0
@@ -587,3 +1009,263 @@ define ptr @postidx_clobber(ptr %addr) nounwind noinline ssp {
  %newaddr = getelementptr i64, ptr %addr, i32 1
  ret ptr %newaddr
 }
+
+define ptr @preidx32_sb(ptr %src, ptr %out) {
+; CHECK64-LABEL: preidx32_sb:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    ldrsb w8, [x0, #1]!
+; CHECK64-NEXT:    str w8, [x1]
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: preidx32_sb:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    ldrsb w8, [x0, #1]
+; GISEL-NEXT:    add x0, x0, #1
+; GISEL-NEXT:    str w8, [x1]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: preidx32_sb:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    ldrsb w8, [x0, #1]!
+; CHECK32-NEXT:    str w8, [x1]
+; CHECK32-NEXT:    ret
+  %ptr = getelementptr inbounds i8, ptr %src, i64 1
+  %tmp = load i8, ptr %ptr, align 1
+  %sext = sext i8 %tmp to i32
+  store i32 %sext, ptr %out, align 4
+  ret ptr %ptr
+}
+
+define ptr @preidx32_sh(ptr %src, ptr %out) {
+; CHECK64-LABEL: preidx32_sh:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    ldrsh w8, [x0, #2]!
+; CHECK64-NEXT:    str w8, [x1]
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: preidx32_sh:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    ldrsh w8, [x0, #2]
+; GISEL-NEXT:    add x0, x0, #2
+; GISEL-NEXT:    str w8, [x1]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: preidx32_sh:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    ldrsh w8, [x0, #2]!
+; CHECK32-NEXT:    str w8, [x1]
+; CHECK32-NEXT:    ret
+  %ptr = getelementptr inbounds i16, ptr %src, i64 1
+  %tmp = load i16, ptr %ptr, align 2
+  %sext = sext i16 %tmp to i32
+  store i32 %sext, ptr %out, align 4
+  ret ptr %ptr
+}
+
+define ptr @preidx64_sb(ptr %src, ptr %out) {
+; CHECK64-LABEL: preidx64_sb:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    ldrsb x8, [x0, #1]!
+; CHECK64-NEXT:    str x8, [x1]
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: preidx64_sb:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    ldrsb x8, [x0, #1]
+; GISEL-NEXT:    add x0, x0, #1
+; GISEL-NEXT:    str x8, [x1]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: preidx64_sb:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    ldrsb x8, [x0, #1]!
+; CHECK32-NEXT:    str x8, [x1]
+; CHECK32-NEXT:    ret
+  %ptr = getelementptr inbounds i8, ptr %src, i64 1
+  %tmp = load i8, ptr %ptr, align 1
+  %sext = sext i8 %tmp to i64
+  store i64 %sext, ptr %out, align 8
+  ret ptr %ptr
+}
+
+define ptr @preidx64_sh(ptr %src, ptr %out) {
+; CHECK64-LABEL: preidx64_sh:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    ldrsh x8, [x0, #2]!
+; CHECK64-NEXT:    str x8, [x1]
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: preidx64_sh:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    ldrsh x8, [x0, #2]
+; GISEL-NEXT:    add x0, x0, #2
+; GISEL-NEXT:    str x8, [x1]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: preidx64_sh:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    ldrsh x8, [x0, #2]!
+; CHECK32-NEXT:    str x8, [x1]
+; CHECK32-NEXT:    ret
+  %ptr = getelementptr inbounds i16, ptr %src, i64 1
+  %tmp = load i16, ptr %ptr, align 2
+  %sext = sext i16 %tmp to i64
+  store i64 %sext, ptr %out, align 8
+  ret ptr %ptr
+}
+
+define ptr @preidx64_sw(ptr %src, ptr %out) {
+; CHECK64-LABEL: preidx64_sw:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    ldrsw x8, [x0, #4]!
+; CHECK64-NEXT:    str x8, [x1]
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: preidx64_sw:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    ldrsw x8, [x0, #4]
+; GISEL-NEXT:    add x0, x0, #4
+; GISEL-NEXT:    str x8, [x1]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: preidx64_sw:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    ldrsw x8, [x0, #4]!
+; CHECK32-NEXT:    str x8, [x1]
+; CHECK32-NEXT:    ret
+  %ptr = getelementptr inbounds i32, ptr %src, i64 1
+  %tmp = load i32, ptr %ptr, align 2
+  %sext = sext i32 %tmp to i64
+  store i64 %sext, ptr %out, align 8
+  ret ptr %ptr
+}
+
+define ptr @postidx32_sb(ptr %src, ptr %out) {
+; CHECK64-LABEL: postidx32_sb:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    ldrsb w8, [x0], #1
+; CHECK64-NEXT:    str w8, [x1]
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: postidx32_sb:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    ldrsb w8, [x0]
+; GISEL-NEXT:    add x0, x0, #1
+; GISEL-NEXT:    str w8, [x1]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: postidx32_sb:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    ldrsb w8, [x0], #1
+; CHECK32-NEXT:    str w8, [x1]
+; CHECK32-NEXT:    ret
+  %tmp = load i8, ptr %src, align 1
+  %ptr = getelementptr inbounds i8, ptr %src, i64 1
+  %sext = sext i8 %tmp to i32
+  store i32 %sext, ptr %out, align 4
+  ret ptr %ptr
+}
+
+define ptr @postidx32_sh(ptr %src, ptr %out) {
+; CHECK64-LABEL: postidx32_sh:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    ldrsh w8, [x0], #2
+; CHECK64-NEXT:    str w8, [x1]
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: postidx32_sh:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    ldrsh w8, [x0]
+; GISEL-NEXT:    add x0, x0, #2
+; GISEL-NEXT:    str w8, [x1]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: postidx32_sh:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    ldrsh w8, [x0], #2
+; CHECK32-NEXT:    str w8, [x1]
+; CHECK32-NEXT:    ret
+  %tmp = load i16, ptr %src, align 2
+  %ptr = getelementptr inbounds i16, ptr %src, i64 1
+  %sext = sext i16 %tmp to i32
+  store i32 %sext, ptr %out, align 4
+  ret ptr %ptr
+}
+
+define ptr @postidx64_sb(ptr %src, ptr %out) {
+; CHECK64-LABEL: postidx64_sb:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    ldrsb x8, [x0], #1
+; CHECK64-NEXT:    str x8, [x1]
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: postidx64_sb:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    ldrsb x8, [x0]
+; GISEL-NEXT:    add x0, x0, #1
+; GISEL-NEXT:    str x8, [x1]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: postidx64_sb:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    ldrsb x8, [x0], #1
+; CHECK32-NEXT:    str x8, [x1]
+; CHECK32-NEXT:    ret
+  %tmp = load i8, ptr %src, align 1
+  %ptr = getelementptr inbounds i8, ptr %src, i64 1
+  %sext = sext i8 %tmp to i64
+  store i64 %sext, ptr %out, align 8
+  ret ptr %ptr
+}
+
+define ptr @postidx64_sh(ptr %src, ptr %out) {
+; CHECK64-LABEL: postidx64_sh:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    ldrsh x8, [x0], #2
+; CHECK64-NEXT:    str x8, [x1]
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: postidx64_sh:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    ldrsh x8, [x0]
+; GISEL-NEXT:    add x0, x0, #2
+; GISEL-NEXT:    str x8, [x1]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: postidx64_sh:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    ldrsh x8, [x0], #2
+; CHECK32-NEXT:    str x8, [x1]
+; CHECK32-NEXT:    ret
+  %tmp = load i16, ptr %src, align 2
+  %ptr = getelementptr inbounds i16, ptr %src, i64 1
+  %sext = sext i16 %tmp to i64
+  store i64 %sext, ptr %out, align 8
+  ret ptr %ptr
+}
+
+define ptr @postidx64_sw(ptr %src, ptr %out) {
+; CHECK64-LABEL: postidx64_sw:
+; CHECK64:       ; %bb.0:
+; CHECK64-NEXT:    ldrsw x8, [x0], #4
+; CHECK64-NEXT:    str x8, [x1]
+; CHECK64-NEXT:    ret
+;
+; GISEL-LABEL: postidx64_sw:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    ldrsw x8, [x0]
+; GISEL-NEXT:    add x0, x0, #4
+; GISEL-NEXT:    str x8, [x1]
+; GISEL-NEXT:    ret
+;
+; CHECK32-LABEL: postidx64_sw:
+; CHECK32:       ; %bb.0:
+; CHECK32-NEXT:    ldrsw x8, [x0], #4
+; CHECK32-NEXT:    str x8, [x1]
+; CHECK32-NEXT:    ret
+  %tmp = load i32, ptr %src, align 4
+  %ptr = getelementptr inbounds i32, ptr %src, i64 1
+  %sext = sext i32 %tmp to i64
+  store i64 %sext, ptr %out, align 8
+  ret ptr %ptr
+}
diff --git a/llvm/test/CodeGen/AArch64/arm64-indexed-vector-ldst.ll b/llvm/test/CodeGen/AArch64/arm64-indexed-vector-ldst.ll
index 7d73e1c6c1..2cab4932de 100644
--- a/llvm/test/CodeGen/AArch64/arm64-indexed-vector-ldst.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-indexed-vector-ldst.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=arm64-apple-ios7.0 -o - %s | FileCheck %s --check-prefix=CHECK
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=arm64-apple-ios7.0 -o - %s | FileCheck %s --check-prefix=CHECK-GISEL
+; RUN: llc -mtriple=arm64-apple-ios7.0 -o - %s | FileCheck %s --check-prefixes=CHECK,SDAG
+; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=arm64-apple-ios7.0 -o - %s 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GISEL
 
 ; CHECK-GISEL-NOT: warning: Instruction selection used fallback path for test_v8i8_pre_load
 ; CHECK-GISEL-NOT: warning: Instruction selection used fallback path for test_v8i8_post_load
@@ -620,21 +620,18 @@
 ; CHECK-GISEL-NOT: warning: Instruction selection used fallback path for load_single_extract_variable_index_i8
 ; CHECK-GISEL-NOT: warning: Instruction selection used fallback path for load_single_extract_variable_index_i16
 ; CHECK-GISEL-NOT: warning: Instruction selection used fallback path for load_single_extract_variable_index_i32
-; CHECK-GISEL-NOT: warning: Instruction selection used fallback path for load_single_extract_variable_index_v3i32_small_align
-; CHECK-GISEL-NOT: warning: Instruction selection used fallback path for load_single_extract_variable_index_v3i32_default_align
-; CHECK-GISEL-NOT: warning: Instruction selection used fallback path for load_single_extract_valid_const_index_v3i32
 ; CHECK-GISEL-NOT: warning: Instruction selection used fallback path for load_single_extract_variable_index_masked_i32
 ; CHECK-GISEL-NOT: warning: Instruction selection used fallback path for load_single_extract_variable_index_masked2_i32
 
 @ptr = global ptr null
 
 define <8 x i8> @test_v8i8_pre_load(ptr %addr) {
-; CHECK-LABEL: test_v8i8_pre_load:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ldr d0, [x0, #40]!
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_pre_load:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ldr d0, [x0, #40]!
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_pre_load:
 ; CHECK-GISEL:       ; %bb.0:
@@ -650,12 +647,12 @@ define <8 x i8> @test_v8i8_pre_load(ptr %addr) {
 }
 
 define <8 x i8> @test_v8i8_post_load(ptr %addr) {
-; CHECK-LABEL: test_v8i8_post_load:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ldr d0, [x0], #40
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_load:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ldr d0, [x0], #40
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_load:
 ; CHECK-GISEL:       ; %bb.0:
@@ -671,12 +668,12 @@ define <8 x i8> @test_v8i8_post_load(ptr %addr) {
 }
 
 define void @test_v8i8_pre_store(<8 x i8> %in, ptr %addr) {
-; CHECK-LABEL: test_v8i8_pre_store:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str d0, [x0, #40]!
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_pre_store:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str d0, [x0, #40]!
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_pre_store:
 ; CHECK-GISEL:       ; %bb.0:
@@ -692,12 +689,12 @@ define void @test_v8i8_pre_store(<8 x i8> %in, ptr %addr) {
 }
 
 define void @test_v8i8_post_store(<8 x i8> %in, ptr %addr) {
-; CHECK-LABEL: test_v8i8_post_store:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str d0, [x0], #40
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_store:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str d0, [x0], #40
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_store:
 ; CHECK-GISEL:       ; %bb.0:
@@ -713,12 +710,12 @@ define void @test_v8i8_post_store(<8 x i8> %in, ptr %addr) {
 }
 
 define <4 x i16> @test_v4i16_pre_load(ptr %addr) {
-; CHECK-LABEL: test_v4i16_pre_load:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ldr d0, [x0, #40]!
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_pre_load:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ldr d0, [x0, #40]!
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_pre_load:
 ; CHECK-GISEL:       ; %bb.0:
@@ -734,12 +731,12 @@ define <4 x i16> @test_v4i16_pre_load(ptr %addr) {
 }
 
 define <4 x i16> @test_v4i16_post_load(ptr %addr) {
-; CHECK-LABEL: test_v4i16_post_load:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ldr d0, [x0], #40
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_load:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ldr d0, [x0], #40
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_load:
 ; CHECK-GISEL:       ; %bb.0:
@@ -755,12 +752,12 @@ define <4 x i16> @test_v4i16_post_load(ptr %addr) {
 }
 
 define void @test_v4i16_pre_store(<4 x i16> %in, ptr %addr) {
-; CHECK-LABEL: test_v4i16_pre_store:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str d0, [x0, #40]!
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_pre_store:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str d0, [x0, #40]!
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_pre_store:
 ; CHECK-GISEL:       ; %bb.0:
@@ -776,12 +773,12 @@ define void @test_v4i16_pre_store(<4 x i16> %in, ptr %addr) {
 }
 
 define void @test_v4i16_post_store(<4 x i16> %in, ptr %addr) {
-; CHECK-LABEL: test_v4i16_post_store:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str d0, [x0], #40
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_store:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str d0, [x0], #40
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_store:
 ; CHECK-GISEL:       ; %bb.0:
@@ -797,12 +794,12 @@ define void @test_v4i16_post_store(<4 x i16> %in, ptr %addr) {
 }
 
 define <2 x i32> @test_v2i32_pre_load(ptr %addr) {
-; CHECK-LABEL: test_v2i32_pre_load:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ldr d0, [x0, #40]!
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_pre_load:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ldr d0, [x0, #40]!
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_pre_load:
 ; CHECK-GISEL:       ; %bb.0:
@@ -818,12 +815,12 @@ define <2 x i32> @test_v2i32_pre_load(ptr %addr) {
 }
 
 define <2 x i32> @test_v2i32_post_load(ptr %addr) {
-; CHECK-LABEL: test_v2i32_post_load:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ldr d0, [x0], #40
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_load:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ldr d0, [x0], #40
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_load:
 ; CHECK-GISEL:       ; %bb.0:
@@ -839,12 +836,12 @@ define <2 x i32> @test_v2i32_post_load(ptr %addr) {
 }
 
 define void @test_v2i32_pre_store(<2 x i32> %in, ptr %addr) {
-; CHECK-LABEL: test_v2i32_pre_store:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str d0, [x0, #40]!
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_pre_store:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str d0, [x0, #40]!
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_pre_store:
 ; CHECK-GISEL:       ; %bb.0:
@@ -860,12 +857,12 @@ define void @test_v2i32_pre_store(<2 x i32> %in, ptr %addr) {
 }
 
 define void @test_v2i32_post_store(<2 x i32> %in, ptr %addr) {
-; CHECK-LABEL: test_v2i32_post_store:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str d0, [x0], #40
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_store:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str d0, [x0], #40
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_store:
 ; CHECK-GISEL:       ; %bb.0:
@@ -881,12 +878,12 @@ define void @test_v2i32_post_store(<2 x i32> %in, ptr %addr) {
 }
 
 define <2 x float> @test_v2f32_pre_load(ptr %addr) {
-; CHECK-LABEL: test_v2f32_pre_load:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ldr d0, [x0, #40]!
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_pre_load:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ldr d0, [x0, #40]!
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_pre_load:
 ; CHECK-GISEL:       ; %bb.0:
@@ -902,12 +899,12 @@ define <2 x float> @test_v2f32_pre_load(ptr %addr) {
 }
 
 define <2 x float> @test_v2f32_post_load(ptr %addr) {
-; CHECK-LABEL: test_v2f32_post_load:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ldr d0, [x0], #40
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_load:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ldr d0, [x0], #40
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_load:
 ; CHECK-GISEL:       ; %bb.0:
@@ -923,12 +920,12 @@ define <2 x float> @test_v2f32_post_load(ptr %addr) {
 }
 
 define void @test_v2f32_pre_store(<2 x float> %in, ptr %addr) {
-; CHECK-LABEL: test_v2f32_pre_store:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str d0, [x0, #40]!
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_pre_store:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str d0, [x0, #40]!
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_pre_store:
 ; CHECK-GISEL:       ; %bb.0:
@@ -944,12 +941,12 @@ define void @test_v2f32_pre_store(<2 x float> %in, ptr %addr) {
 }
 
 define void @test_v2f32_post_store(<2 x float> %in, ptr %addr) {
-; CHECK-LABEL: test_v2f32_post_store:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str d0, [x0], #40
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_store:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str d0, [x0], #40
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_store:
 ; CHECK-GISEL:       ; %bb.0:
@@ -965,12 +962,12 @@ define void @test_v2f32_post_store(<2 x float> %in, ptr %addr) {
 }
 
 define <1 x i64> @test_v1i64_pre_load(ptr %addr) {
-; CHECK-LABEL: test_v1i64_pre_load:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ldr d0, [x0, #40]!
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_pre_load:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ldr d0, [x0, #40]!
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_pre_load:
 ; CHECK-GISEL:       ; %bb.0:
@@ -986,12 +983,12 @@ define <1 x i64> @test_v1i64_pre_load(ptr %addr) {
 }
 
 define <1 x i64> @test_v1i64_post_load(ptr %addr) {
-; CHECK-LABEL: test_v1i64_post_load:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ldr d0, [x0], #40
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_load:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ldr d0, [x0], #40
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_load:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1007,12 +1004,12 @@ define <1 x i64> @test_v1i64_post_load(ptr %addr) {
 }
 
 define void @test_v1i64_pre_store(<1 x i64> %in, ptr %addr) {
-; CHECK-LABEL: test_v1i64_pre_store:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str d0, [x0, #40]!
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_pre_store:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str d0, [x0, #40]!
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_pre_store:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1028,12 +1025,12 @@ define void @test_v1i64_pre_store(<1 x i64> %in, ptr %addr) {
 }
 
 define void @test_v1i64_post_store(<1 x i64> %in, ptr %addr) {
-; CHECK-LABEL: test_v1i64_post_store:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str d0, [x0], #40
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_store:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str d0, [x0], #40
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_store:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1049,12 +1046,12 @@ define void @test_v1i64_post_store(<1 x i64> %in, ptr %addr) {
 }
 
 define <16 x i8> @test_v16i8_pre_load(ptr %addr) {
-; CHECK-LABEL: test_v16i8_pre_load:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ldr q0, [x0, #80]!
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_pre_load:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ldr q0, [x0, #80]!
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_pre_load:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1070,12 +1067,12 @@ define <16 x i8> @test_v16i8_pre_load(ptr %addr) {
 }
 
 define <16 x i8> @test_v16i8_post_load(ptr %addr) {
-; CHECK-LABEL: test_v16i8_post_load:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ldr q0, [x0], #80
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_load:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ldr q0, [x0], #80
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_load:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1091,12 +1088,12 @@ define <16 x i8> @test_v16i8_post_load(ptr %addr) {
 }
 
 define void @test_v16i8_pre_store(<16 x i8> %in, ptr %addr) {
-; CHECK-LABEL: test_v16i8_pre_store:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str q0, [x0, #80]!
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_pre_store:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str q0, [x0, #80]!
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_pre_store:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1112,12 +1109,12 @@ define void @test_v16i8_pre_store(<16 x i8> %in, ptr %addr) {
 }
 
 define void @test_v16i8_post_store(<16 x i8> %in, ptr %addr) {
-; CHECK-LABEL: test_v16i8_post_store:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str q0, [x0], #80
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_store:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str q0, [x0], #80
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_store:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1133,12 +1130,12 @@ define void @test_v16i8_post_store(<16 x i8> %in, ptr %addr) {
 }
 
 define <8 x i16> @test_v8i16_pre_load(ptr %addr) {
-; CHECK-LABEL: test_v8i16_pre_load:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ldr q0, [x0, #80]!
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_pre_load:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ldr q0, [x0, #80]!
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_pre_load:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1154,12 +1151,12 @@ define <8 x i16> @test_v8i16_pre_load(ptr %addr) {
 }
 
 define <8 x i16> @test_v8i16_post_load(ptr %addr) {
-; CHECK-LABEL: test_v8i16_post_load:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ldr q0, [x0], #80
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_load:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ldr q0, [x0], #80
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_load:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1175,12 +1172,12 @@ define <8 x i16> @test_v8i16_post_load(ptr %addr) {
 }
 
 define void @test_v8i16_pre_store(<8 x i16> %in, ptr %addr) {
-; CHECK-LABEL: test_v8i16_pre_store:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str q0, [x0, #80]!
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_pre_store:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str q0, [x0, #80]!
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_pre_store:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1196,12 +1193,12 @@ define void @test_v8i16_pre_store(<8 x i16> %in, ptr %addr) {
 }
 
 define void @test_v8i16_post_store(<8 x i16> %in, ptr %addr) {
-; CHECK-LABEL: test_v8i16_post_store:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str q0, [x0], #80
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_store:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str q0, [x0], #80
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_store:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1217,12 +1214,12 @@ define void @test_v8i16_post_store(<8 x i16> %in, ptr %addr) {
 }
 
 define <4 x i32> @test_v4i32_pre_load(ptr %addr) {
-; CHECK-LABEL: test_v4i32_pre_load:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ldr q0, [x0, #80]!
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_pre_load:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ldr q0, [x0, #80]!
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_pre_load:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1238,12 +1235,12 @@ define <4 x i32> @test_v4i32_pre_load(ptr %addr) {
 }
 
 define <4 x i32> @test_v4i32_post_load(ptr %addr) {
-; CHECK-LABEL: test_v4i32_post_load:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ldr q0, [x0], #80
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_load:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ldr q0, [x0], #80
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_load:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1259,12 +1256,12 @@ define <4 x i32> @test_v4i32_post_load(ptr %addr) {
 }
 
 define void @test_v4i32_pre_store(<4 x i32> %in, ptr %addr) {
-; CHECK-LABEL: test_v4i32_pre_store:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str q0, [x0, #80]!
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_pre_store:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str q0, [x0, #80]!
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_pre_store:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1280,12 +1277,12 @@ define void @test_v4i32_pre_store(<4 x i32> %in, ptr %addr) {
 }
 
 define void @test_v4i32_post_store(<4 x i32> %in, ptr %addr) {
-; CHECK-LABEL: test_v4i32_post_store:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str q0, [x0], #80
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_store:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str q0, [x0], #80
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_store:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1302,12 +1299,12 @@ define void @test_v4i32_post_store(<4 x i32> %in, ptr %addr) {
 
 
 define <4 x float> @test_v4f32_pre_load(ptr %addr) {
-; CHECK-LABEL: test_v4f32_pre_load:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ldr q0, [x0, #80]!
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_pre_load:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ldr q0, [x0, #80]!
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_pre_load:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1323,12 +1320,12 @@ define <4 x float> @test_v4f32_pre_load(ptr %addr) {
 }
 
 define <4 x float> @test_v4f32_post_load(ptr %addr) {
-; CHECK-LABEL: test_v4f32_post_load:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ldr q0, [x0], #80
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_load:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ldr q0, [x0], #80
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_load:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1344,12 +1341,12 @@ define <4 x float> @test_v4f32_post_load(ptr %addr) {
 }
 
 define void @test_v4f32_pre_store(<4 x float> %in, ptr %addr) {
-; CHECK-LABEL: test_v4f32_pre_store:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str q0, [x0, #80]!
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_pre_store:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str q0, [x0, #80]!
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_pre_store:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1365,12 +1362,12 @@ define void @test_v4f32_pre_store(<4 x float> %in, ptr %addr) {
 }
 
 define void @test_v4f32_post_store(<4 x float> %in, ptr %addr) {
-; CHECK-LABEL: test_v4f32_post_store:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str q0, [x0], #80
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_store:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str q0, [x0], #80
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_store:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1387,12 +1384,12 @@ define void @test_v4f32_post_store(<4 x float> %in, ptr %addr) {
 
 
 define <2 x i64> @test_v2i64_pre_load(ptr %addr) {
-; CHECK-LABEL: test_v2i64_pre_load:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ldr q0, [x0, #80]!
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_pre_load:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ldr q0, [x0, #80]!
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_pre_load:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1408,12 +1405,12 @@ define <2 x i64> @test_v2i64_pre_load(ptr %addr) {
 }
 
 define <2 x i64> @test_v2i64_post_load(ptr %addr) {
-; CHECK-LABEL: test_v2i64_post_load:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ldr q0, [x0], #80
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_load:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ldr q0, [x0], #80
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_load:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1429,12 +1426,12 @@ define <2 x i64> @test_v2i64_post_load(ptr %addr) {
 }
 
 define void @test_v2i64_pre_store(<2 x i64> %in, ptr %addr) {
-; CHECK-LABEL: test_v2i64_pre_store:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str q0, [x0, #80]!
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_pre_store:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str q0, [x0, #80]!
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_pre_store:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1450,12 +1447,12 @@ define void @test_v2i64_pre_store(<2 x i64> %in, ptr %addr) {
 }
 
 define void @test_v2i64_post_store(<2 x i64> %in, ptr %addr) {
-; CHECK-LABEL: test_v2i64_post_store:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str q0, [x0], #80
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_store:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str q0, [x0], #80
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_store:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1472,12 +1469,12 @@ define void @test_v2i64_post_store(<2 x i64> %in, ptr %addr) {
 
 
 define <2 x double> @test_v2f64_pre_load(ptr %addr) {
-; CHECK-LABEL: test_v2f64_pre_load:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ldr q0, [x0, #80]!
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_pre_load:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ldr q0, [x0, #80]!
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_pre_load:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1493,12 +1490,12 @@ define <2 x double> @test_v2f64_pre_load(ptr %addr) {
 }
 
 define <2 x double> @test_v2f64_post_load(ptr %addr) {
-; CHECK-LABEL: test_v2f64_post_load:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ldr q0, [x0], #80
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_load:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ldr q0, [x0], #80
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_load:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1514,12 +1511,12 @@ define <2 x double> @test_v2f64_post_load(ptr %addr) {
 }
 
 define void @test_v2f64_pre_store(<2 x double> %in, ptr %addr) {
-; CHECK-LABEL: test_v2f64_pre_store:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str q0, [x0, #80]!
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_pre_store:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str q0, [x0, #80]!
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_pre_store:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1535,12 +1532,12 @@ define void @test_v2f64_pre_store(<2 x double> %in, ptr %addr) {
 }
 
 define void @test_v2f64_post_store(<2 x double> %in, ptr %addr) {
-; CHECK-LABEL: test_v2f64_post_store:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    adrp x8, _ptr@PAGE
-; CHECK-NEXT:    str q0, [x0], #80
-; CHECK-NEXT:    str x0, [x8, _ptr@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_store:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    adrp x8, _ptr@PAGE
+; SDAG-NEXT:    str q0, [x0], #80
+; SDAG-NEXT:    str x0, [x8, _ptr@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_store:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1556,10 +1553,10 @@ define void @test_v2f64_post_store(<2 x double> %in, ptr %addr) {
 }
 
 define ptr @test_v16i8_post_imm_st1_lane(<16 x i8> %in, ptr %addr) {
-; CHECK-LABEL: test_v16i8_post_imm_st1_lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    st1.b { v0 }[3], [x0], #1
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_imm_st1_lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    st1.b { v0 }[3], [x0], #1
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_imm_st1_lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1575,11 +1572,11 @@ define ptr @test_v16i8_post_imm_st1_lane(<16 x i8> %in, ptr %addr) {
 }
 
 define ptr @test_v16i8_post_reg_st1_lane(<16 x i8> %in, ptr %addr) {
-; CHECK-LABEL: test_v16i8_post_reg_st1_lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    mov w8, #2 ; =0x2
-; CHECK-NEXT:    st1.b { v0 }[3], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_reg_st1_lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    mov w8, #2 ; =0x2
+; SDAG-NEXT:    st1.b { v0 }[3], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_reg_st1_lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1596,10 +1593,10 @@ define ptr @test_v16i8_post_reg_st1_lane(<16 x i8> %in, ptr %addr) {
 
 
 define ptr @test_v8i16_post_imm_st1_lane(<8 x i16> %in, ptr %addr) {
-; CHECK-LABEL: test_v8i16_post_imm_st1_lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    st1.h { v0 }[3], [x0], #2
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_imm_st1_lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    st1.h { v0 }[3], [x0], #2
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_imm_st1_lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1615,11 +1612,11 @@ define ptr @test_v8i16_post_imm_st1_lane(<8 x i16> %in, ptr %addr) {
 }
 
 define ptr @test_v8i16_post_reg_st1_lane(<8 x i16> %in, ptr %addr) {
-; CHECK-LABEL: test_v8i16_post_reg_st1_lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    mov w8, #4 ; =0x4
-; CHECK-NEXT:    st1.h { v0 }[3], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_reg_st1_lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    mov w8, #4 ; =0x4
+; SDAG-NEXT:    st1.h { v0 }[3], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_reg_st1_lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1635,10 +1632,10 @@ define ptr @test_v8i16_post_reg_st1_lane(<8 x i16> %in, ptr %addr) {
 }
 
 define ptr @test_v4i32_post_imm_st1_lane(<4 x i32> %in, ptr %addr) {
-; CHECK-LABEL: test_v4i32_post_imm_st1_lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    st1.s { v0 }[3], [x0], #4
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_imm_st1_lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    st1.s { v0 }[3], [x0], #4
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_imm_st1_lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1654,11 +1651,11 @@ define ptr @test_v4i32_post_imm_st1_lane(<4 x i32> %in, ptr %addr) {
 }
 
 define ptr @test_v4i32_post_reg_st1_lane(<4 x i32> %in, ptr %addr) {
-; CHECK-LABEL: test_v4i32_post_reg_st1_lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    mov w8, #8 ; =0x8
-; CHECK-NEXT:    st1.s { v0 }[3], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_reg_st1_lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    mov w8, #8 ; =0x8
+; SDAG-NEXT:    st1.s { v0 }[3], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_reg_st1_lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1674,10 +1671,10 @@ define ptr @test_v4i32_post_reg_st1_lane(<4 x i32> %in, ptr %addr) {
 }
 
 define ptr @test_v4f32_post_imm_st1_lane(<4 x float> %in, ptr %addr) {
-; CHECK-LABEL: test_v4f32_post_imm_st1_lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    st1.s { v0 }[3], [x0], #4
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_imm_st1_lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    st1.s { v0 }[3], [x0], #4
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_imm_st1_lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1693,11 +1690,11 @@ define ptr @test_v4f32_post_imm_st1_lane(<4 x float> %in, ptr %addr) {
 }
 
 define ptr @test_v4f32_post_reg_st1_lane(<4 x float> %in, ptr %addr) {
-; CHECK-LABEL: test_v4f32_post_reg_st1_lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    mov w8, #8 ; =0x8
-; CHECK-NEXT:    st1.s { v0 }[3], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_reg_st1_lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    mov w8, #8 ; =0x8
+; SDAG-NEXT:    st1.s { v0 }[3], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_reg_st1_lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1713,10 +1710,10 @@ define ptr @test_v4f32_post_reg_st1_lane(<4 x float> %in, ptr %addr) {
 }
 
 define ptr @test_v2i64_post_imm_st1_lane(<2 x i64> %in, ptr %addr) {
-; CHECK-LABEL: test_v2i64_post_imm_st1_lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    st1.d { v0 }[1], [x0], #8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_imm_st1_lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    st1.d { v0 }[1], [x0], #8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_imm_st1_lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1732,11 +1729,11 @@ define ptr @test_v2i64_post_imm_st1_lane(<2 x i64> %in, ptr %addr) {
 }
 
 define ptr @test_v2i64_post_reg_st1_lane(<2 x i64> %in, ptr %addr) {
-; CHECK-LABEL: test_v2i64_post_reg_st1_lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    mov w8, #16 ; =0x10
-; CHECK-NEXT:    st1.d { v0 }[1], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_reg_st1_lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    mov w8, #16 ; =0x10
+; SDAG-NEXT:    st1.d { v0 }[1], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_reg_st1_lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1752,10 +1749,10 @@ define ptr @test_v2i64_post_reg_st1_lane(<2 x i64> %in, ptr %addr) {
 }
 
 define ptr @test_v2f64_post_imm_st1_lane(<2 x double> %in, ptr %addr) {
-; CHECK-LABEL: test_v2f64_post_imm_st1_lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    st1.d { v0 }[1], [x0], #8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_imm_st1_lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    st1.d { v0 }[1], [x0], #8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_imm_st1_lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1771,11 +1768,11 @@ define ptr @test_v2f64_post_imm_st1_lane(<2 x double> %in, ptr %addr) {
 }
 
 define ptr @test_v2f64_post_reg_st1_lane(<2 x double> %in, ptr %addr) {
-; CHECK-LABEL: test_v2f64_post_reg_st1_lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    mov w8, #16 ; =0x10
-; CHECK-NEXT:    st1.d { v0 }[1], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_reg_st1_lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    mov w8, #16 ; =0x10
+; SDAG-NEXT:    st1.d { v0 }[1], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_reg_st1_lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1791,11 +1788,11 @@ define ptr @test_v2f64_post_reg_st1_lane(<2 x double> %in, ptr %addr) {
 }
 
 define ptr @test_v8i8_post_imm_st1_lane(<8 x i8> %in, ptr %addr) {
-; CHECK-LABEL: test_v8i8_post_imm_st1_lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    st1.b { v0 }[3], [x0], #1
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_imm_st1_lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 def $q0
+; SDAG-NEXT:    st1.b { v0 }[3], [x0], #1
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_imm_st1_lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1812,12 +1809,12 @@ define ptr @test_v8i8_post_imm_st1_lane(<8 x i8> %in, ptr %addr) {
 }
 
 define ptr @test_v8i8_post_reg_st1_lane(<8 x i8> %in, ptr %addr) {
-; CHECK-LABEL: test_v8i8_post_reg_st1_lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    mov w8, #2 ; =0x2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    st1.b { v0 }[3], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_reg_st1_lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    mov w8, #2 ; =0x2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 def $q0
+; SDAG-NEXT:    st1.b { v0 }[3], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_reg_st1_lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1834,11 +1831,11 @@ define ptr @test_v8i8_post_reg_st1_lane(<8 x i8> %in, ptr %addr) {
 }
 
 define ptr @test_v4i16_post_imm_st1_lane(<4 x i16> %in, ptr %addr) {
-; CHECK-LABEL: test_v4i16_post_imm_st1_lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    st1.h { v0 }[3], [x0], #2
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_imm_st1_lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 def $q0
+; SDAG-NEXT:    st1.h { v0 }[3], [x0], #2
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_imm_st1_lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1855,12 +1852,12 @@ define ptr @test_v4i16_post_imm_st1_lane(<4 x i16> %in, ptr %addr) {
 }
 
 define ptr @test_v4i16_post_reg_st1_lane(<4 x i16> %in, ptr %addr) {
-; CHECK-LABEL: test_v4i16_post_reg_st1_lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    mov w8, #4 ; =0x4
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    st1.h { v0 }[3], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_reg_st1_lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    mov w8, #4 ; =0x4
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 def $q0
+; SDAG-NEXT:    st1.h { v0 }[3], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_reg_st1_lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1877,11 +1874,11 @@ define ptr @test_v4i16_post_reg_st1_lane(<4 x i16> %in, ptr %addr) {
 }
 
 define ptr @test_v2i32_post_imm_st1_lane(<2 x i32> %in, ptr %addr) {
-; CHECK-LABEL: test_v2i32_post_imm_st1_lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    st1.s { v0 }[1], [x0], #4
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_imm_st1_lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 def $q0
+; SDAG-NEXT:    st1.s { v0 }[1], [x0], #4
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_imm_st1_lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1898,12 +1895,12 @@ define ptr @test_v2i32_post_imm_st1_lane(<2 x i32> %in, ptr %addr) {
 }
 
 define ptr @test_v2i32_post_reg_st1_lane(<2 x i32> %in, ptr %addr) {
-; CHECK-LABEL: test_v2i32_post_reg_st1_lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    mov w8, #8 ; =0x8
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    st1.s { v0 }[1], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_reg_st1_lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    mov w8, #8 ; =0x8
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 def $q0
+; SDAG-NEXT:    st1.s { v0 }[1], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_reg_st1_lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1920,11 +1917,11 @@ define ptr @test_v2i32_post_reg_st1_lane(<2 x i32> %in, ptr %addr) {
 }
 
 define ptr @test_v2f32_post_imm_st1_lane(<2 x float> %in, ptr %addr) {
-; CHECK-LABEL: test_v2f32_post_imm_st1_lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    st1.s { v0 }[1], [x0], #4
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_imm_st1_lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 def $q0
+; SDAG-NEXT:    st1.s { v0 }[1], [x0], #4
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_imm_st1_lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1941,12 +1938,12 @@ define ptr @test_v2f32_post_imm_st1_lane(<2 x float> %in, ptr %addr) {
 }
 
 define ptr @test_v2f32_post_reg_st1_lane(<2 x float> %in, ptr %addr) {
-; CHECK-LABEL: test_v2f32_post_reg_st1_lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    mov w8, #8 ; =0x8
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    st1.s { v0 }[1], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_reg_st1_lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    mov w8, #8 ; =0x8
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 def $q0
+; SDAG-NEXT:    st1.s { v0 }[1], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_reg_st1_lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1963,11 +1960,11 @@ define ptr @test_v2f32_post_reg_st1_lane(<2 x float> %in, ptr %addr) {
 }
 
 define { <16 x i8>, <16 x i8> } @test_v16i8_post_imm_ld2(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v16i8_post_imm_ld2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld2.16b { v0, v1 }, [x0], #32
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_imm_ld2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld2.16b { v0, v1 }, [x0], #32
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_imm_ld2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -1982,11 +1979,11 @@ define { <16 x i8>, <16 x i8> } @test_v16i8_post_imm_ld2(ptr %A, ptr %ptr) {
 }
 
 define { <16 x i8>, <16 x i8> } @test_v16i8_post_reg_ld2(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v16i8_post_reg_ld2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld2.16b { v0, v1 }, [x0], x2
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_reg_ld2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld2.16b { v0, v1 }, [x0], x2
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_reg_ld2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2004,11 +2001,11 @@ declare { <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld2.v16i8.p0(ptr)
 
 
 define { <8 x i8>, <8 x i8> } @test_v8i8_post_imm_ld2(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v8i8_post_imm_ld2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld2.8b { v0, v1 }, [x0], #16
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_imm_ld2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld2.8b { v0, v1 }, [x0], #16
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_imm_ld2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2023,11 +2020,11 @@ define { <8 x i8>, <8 x i8> } @test_v8i8_post_imm_ld2(ptr %A, ptr %ptr) {
 }
 
 define { <8 x i8>, <8 x i8> } @test_v8i8_post_reg_ld2(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v8i8_post_reg_ld2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld2.8b { v0, v1 }, [x0], x2
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_reg_ld2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld2.8b { v0, v1 }, [x0], x2
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_reg_ld2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2045,11 +2042,11 @@ declare { <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld2.v8i8.p0(ptr)
 
 
 define { <8 x i16>, <8 x i16> } @test_v8i16_post_imm_ld2(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v8i16_post_imm_ld2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld2.8h { v0, v1 }, [x0], #32
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_imm_ld2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld2.8h { v0, v1 }, [x0], #32
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_imm_ld2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2064,12 +2061,12 @@ define { <8 x i16>, <8 x i16> } @test_v8i16_post_imm_ld2(ptr %A, ptr %ptr) {
 }
 
 define { <8 x i16>, <8 x i16> } @test_v8i16_post_reg_ld2(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v8i16_post_reg_ld2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ld2.8h { v0, v1 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_reg_ld2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ld2.8h { v0, v1 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_reg_ld2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2087,11 +2084,11 @@ declare { <8 x i16>, <8 x i16> } @llvm.aarch64.neon.ld2.v8i16.p0(ptr)
 
 
 define { <4 x i16>, <4 x i16> } @test_v4i16_post_imm_ld2(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v4i16_post_imm_ld2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld2.4h { v0, v1 }, [x0], #16
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_imm_ld2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld2.4h { v0, v1 }, [x0], #16
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_imm_ld2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2106,12 +2103,12 @@ define { <4 x i16>, <4 x i16> } @test_v4i16_post_imm_ld2(ptr %A, ptr %ptr) {
 }
 
 define { <4 x i16>, <4 x i16> } @test_v4i16_post_reg_ld2(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v4i16_post_reg_ld2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ld2.4h { v0, v1 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_reg_ld2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ld2.4h { v0, v1 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_reg_ld2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2129,11 +2126,11 @@ declare { <4 x i16>, <4 x i16> } @llvm.aarch64.neon.ld2.v4i16.p0(ptr)
 
 
 define { <4 x i32>, <4 x i32> } @test_v4i32_post_imm_ld2(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v4i32_post_imm_ld2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld2.4s { v0, v1 }, [x0], #32
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_imm_ld2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld2.4s { v0, v1 }, [x0], #32
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_imm_ld2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2148,12 +2145,12 @@ define { <4 x i32>, <4 x i32> } @test_v4i32_post_imm_ld2(ptr %A, ptr %ptr) {
 }
 
 define { <4 x i32>, <4 x i32> } @test_v4i32_post_reg_ld2(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v4i32_post_reg_ld2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld2.4s { v0, v1 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_reg_ld2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld2.4s { v0, v1 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_reg_ld2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2171,11 +2168,11 @@ declare { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld2.v4i32.p0(ptr)
 
 
 define { <2 x i32>, <2 x i32> } @test_v2i32_post_imm_ld2(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v2i32_post_imm_ld2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld2.2s { v0, v1 }, [x0], #16
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_imm_ld2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld2.2s { v0, v1 }, [x0], #16
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_imm_ld2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2190,12 +2187,12 @@ define { <2 x i32>, <2 x i32> } @test_v2i32_post_imm_ld2(ptr %A, ptr %ptr) {
 }
 
 define { <2 x i32>, <2 x i32> } @test_v2i32_post_reg_ld2(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v2i32_post_reg_ld2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld2.2s { v0, v1 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_reg_ld2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld2.2s { v0, v1 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_reg_ld2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2213,11 +2210,11 @@ declare { <2 x i32>, <2 x i32> } @llvm.aarch64.neon.ld2.v2i32.p0(ptr)
 
 
 define { <2 x i64>, <2 x i64> } @test_v2i64_post_imm_ld2(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v2i64_post_imm_ld2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld2.2d { v0, v1 }, [x0], #32
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_imm_ld2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld2.2d { v0, v1 }, [x0], #32
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_imm_ld2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2232,12 +2229,12 @@ define { <2 x i64>, <2 x i64> } @test_v2i64_post_imm_ld2(ptr %A, ptr %ptr) {
 }
 
 define { <2 x i64>, <2 x i64> } @test_v2i64_post_reg_ld2(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v2i64_post_reg_ld2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld2.2d { v0, v1 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_reg_ld2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld2.2d { v0, v1 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_reg_ld2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2255,11 +2252,11 @@ declare { <2 x i64>, <2 x i64> } @llvm.aarch64.neon.ld2.v2i64.p0(ptr)
 
 
 define { <1 x i64>, <1 x i64> } @test_v1i64_post_imm_ld2(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v1i64_post_imm_ld2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.1d { v0, v1 }, [x0], #16
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_imm_ld2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.1d { v0, v1 }, [x0], #16
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_imm_ld2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2274,12 +2271,12 @@ define { <1 x i64>, <1 x i64> } @test_v1i64_post_imm_ld2(ptr %A, ptr %ptr) {
 }
 
 define { <1 x i64>, <1 x i64> } @test_v1i64_post_reg_ld2(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v1i64_post_reg_ld2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld1.1d { v0, v1 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_reg_ld2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld1.1d { v0, v1 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_reg_ld2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2297,11 +2294,11 @@ declare { <1 x i64>, <1 x i64> } @llvm.aarch64.neon.ld2.v1i64.p0(ptr)
 
 
 define { <4 x float>, <4 x float> } @test_v4f32_post_imm_ld2(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v4f32_post_imm_ld2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld2.4s { v0, v1 }, [x0], #32
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_imm_ld2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld2.4s { v0, v1 }, [x0], #32
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_imm_ld2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2316,12 +2313,12 @@ define { <4 x float>, <4 x float> } @test_v4f32_post_imm_ld2(ptr %A, ptr %ptr) {
 }
 
 define { <4 x float>, <4 x float> } @test_v4f32_post_reg_ld2(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v4f32_post_reg_ld2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld2.4s { v0, v1 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_reg_ld2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld2.4s { v0, v1 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_reg_ld2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2339,11 +2336,11 @@ declare { <4 x float>, <4 x float> } @llvm.aarch64.neon.ld2.v4f32.p0(ptr)
 
 
 define { <2 x float>, <2 x float> } @test_v2f32_post_imm_ld2(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v2f32_post_imm_ld2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld2.2s { v0, v1 }, [x0], #16
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_imm_ld2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld2.2s { v0, v1 }, [x0], #16
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_imm_ld2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2358,12 +2355,12 @@ define { <2 x float>, <2 x float> } @test_v2f32_post_imm_ld2(ptr %A, ptr %ptr) {
 }
 
 define { <2 x float>, <2 x float> } @test_v2f32_post_reg_ld2(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v2f32_post_reg_ld2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld2.2s { v0, v1 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_reg_ld2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld2.2s { v0, v1 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_reg_ld2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2381,11 +2378,11 @@ declare { <2 x float>, <2 x float> } @llvm.aarch64.neon.ld2.v2f32.p0(ptr)
 
 
 define { <2 x double>, <2 x double> } @test_v2f64_post_imm_ld2(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v2f64_post_imm_ld2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld2.2d { v0, v1 }, [x0], #32
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_imm_ld2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld2.2d { v0, v1 }, [x0], #32
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_imm_ld2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2400,12 +2397,12 @@ define { <2 x double>, <2 x double> } @test_v2f64_post_imm_ld2(ptr %A, ptr %ptr)
 }
 
 define { <2 x double>, <2 x double> } @test_v2f64_post_reg_ld2(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v2f64_post_reg_ld2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld2.2d { v0, v1 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_reg_ld2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld2.2d { v0, v1 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_reg_ld2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2423,11 +2420,11 @@ declare { <2 x double>, <2 x double> } @llvm.aarch64.neon.ld2.v2f64.p0(ptr)
 
 
 define { <1 x double>, <1 x double> } @test_v1f64_post_imm_ld2(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v1f64_post_imm_ld2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.1d { v0, v1 }, [x0], #16
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_imm_ld2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.1d { v0, v1 }, [x0], #16
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_imm_ld2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2442,12 +2439,12 @@ define { <1 x double>, <1 x double> } @test_v1f64_post_imm_ld2(ptr %A, ptr %ptr)
 }
 
 define { <1 x double>, <1 x double> } @test_v1f64_post_reg_ld2(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v1f64_post_reg_ld2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld1.1d { v0, v1 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_reg_ld2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld1.1d { v0, v1 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_reg_ld2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2465,11 +2462,11 @@ declare { <1 x double>, <1 x double> } @llvm.aarch64.neon.ld2.v1f64.p0(ptr)
 
 
 define { <16 x i8>, <16 x i8>, <16 x i8> } @test_v16i8_post_imm_ld3(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v16i8_post_imm_ld3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld3.16b { v0, v1, v2 }, [x0], #48
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_imm_ld3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld3.16b { v0, v1, v2 }, [x0], #48
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_imm_ld3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2484,11 +2481,11 @@ define { <16 x i8>, <16 x i8>, <16 x i8> } @test_v16i8_post_imm_ld3(ptr %A, ptr
 }
 
 define { <16 x i8>, <16 x i8>, <16 x i8> } @test_v16i8_post_reg_ld3(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v16i8_post_reg_ld3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld3.16b { v0, v1, v2 }, [x0], x2
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_reg_ld3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld3.16b { v0, v1, v2 }, [x0], x2
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_reg_ld3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2506,11 +2503,11 @@ declare { <16 x i8>, <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld3.v16i8.p0(ptr)
 
 
 define { <8 x i8>, <8 x i8>, <8 x i8> } @test_v8i8_post_imm_ld3(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v8i8_post_imm_ld3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld3.8b { v0, v1, v2 }, [x0], #24
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_imm_ld3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld3.8b { v0, v1, v2 }, [x0], #24
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_imm_ld3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2525,11 +2522,11 @@ define { <8 x i8>, <8 x i8>, <8 x i8> } @test_v8i8_post_imm_ld3(ptr %A, ptr %ptr
 }
 
 define { <8 x i8>, <8 x i8>, <8 x i8> } @test_v8i8_post_reg_ld3(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v8i8_post_reg_ld3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld3.8b { v0, v1, v2 }, [x0], x2
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_reg_ld3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld3.8b { v0, v1, v2 }, [x0], x2
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_reg_ld3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2547,11 +2544,11 @@ declare { <8 x i8>, <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld3.v8i8.p0(ptr)
 
 
 define { <8 x i16>, <8 x i16>, <8 x i16> } @test_v8i16_post_imm_ld3(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v8i16_post_imm_ld3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld3.8h { v0, v1, v2 }, [x0], #48
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_imm_ld3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld3.8h { v0, v1, v2 }, [x0], #48
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_imm_ld3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2566,12 +2563,12 @@ define { <8 x i16>, <8 x i16>, <8 x i16> } @test_v8i16_post_imm_ld3(ptr %A, ptr
 }
 
 define { <8 x i16>, <8 x i16>, <8 x i16> } @test_v8i16_post_reg_ld3(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v8i16_post_reg_ld3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ld3.8h { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_reg_ld3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ld3.8h { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_reg_ld3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2589,11 +2586,11 @@ declare { <8 x i16>, <8 x i16>, <8 x i16> } @llvm.aarch64.neon.ld3.v8i16.p0(ptr)
 
 
 define { <4 x i16>, <4 x i16>, <4 x i16> } @test_v4i16_post_imm_ld3(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v4i16_post_imm_ld3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld3.4h { v0, v1, v2 }, [x0], #24
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_imm_ld3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld3.4h { v0, v1, v2 }, [x0], #24
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_imm_ld3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2608,12 +2605,12 @@ define { <4 x i16>, <4 x i16>, <4 x i16> } @test_v4i16_post_imm_ld3(ptr %A, ptr
 }
 
 define { <4 x i16>, <4 x i16>, <4 x i16> } @test_v4i16_post_reg_ld3(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v4i16_post_reg_ld3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ld3.4h { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_reg_ld3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ld3.4h { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_reg_ld3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2631,11 +2628,11 @@ declare { <4 x i16>, <4 x i16>, <4 x i16> } @llvm.aarch64.neon.ld3.v4i16.p0(ptr)
 
 
 define { <4 x i32>, <4 x i32>, <4 x i32> } @test_v4i32_post_imm_ld3(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v4i32_post_imm_ld3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld3.4s { v0, v1, v2 }, [x0], #48
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_imm_ld3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld3.4s { v0, v1, v2 }, [x0], #48
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_imm_ld3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2650,12 +2647,12 @@ define { <4 x i32>, <4 x i32>, <4 x i32> } @test_v4i32_post_imm_ld3(ptr %A, ptr
 }
 
 define { <4 x i32>, <4 x i32>, <4 x i32> } @test_v4i32_post_reg_ld3(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v4i32_post_reg_ld3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld3.4s { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_reg_ld3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld3.4s { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_reg_ld3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2673,11 +2670,11 @@ declare { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld3.v4i32.p0(ptr)
 
 
 define { <2 x i32>, <2 x i32>, <2 x i32> } @test_v2i32_post_imm_ld3(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v2i32_post_imm_ld3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld3.2s { v0, v1, v2 }, [x0], #24
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_imm_ld3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld3.2s { v0, v1, v2 }, [x0], #24
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_imm_ld3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2692,12 +2689,12 @@ define { <2 x i32>, <2 x i32>, <2 x i32> } @test_v2i32_post_imm_ld3(ptr %A, ptr
 }
 
 define { <2 x i32>, <2 x i32>, <2 x i32> } @test_v2i32_post_reg_ld3(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v2i32_post_reg_ld3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld3.2s { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_reg_ld3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld3.2s { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_reg_ld3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2715,11 +2712,11 @@ declare { <2 x i32>, <2 x i32>, <2 x i32> } @llvm.aarch64.neon.ld3.v2i32.p0(ptr)
 
 
 define { <2 x i64>, <2 x i64>, <2 x i64> } @test_v2i64_post_imm_ld3(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v2i64_post_imm_ld3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld3.2d { v0, v1, v2 }, [x0], #48
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_imm_ld3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld3.2d { v0, v1, v2 }, [x0], #48
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_imm_ld3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2734,12 +2731,12 @@ define { <2 x i64>, <2 x i64>, <2 x i64> } @test_v2i64_post_imm_ld3(ptr %A, ptr
 }
 
 define { <2 x i64>, <2 x i64>, <2 x i64> } @test_v2i64_post_reg_ld3(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v2i64_post_reg_ld3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld3.2d { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_reg_ld3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld3.2d { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_reg_ld3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2757,11 +2754,11 @@ declare { <2 x i64>, <2 x i64>, <2 x i64> } @llvm.aarch64.neon.ld3.v2i64.p0(ptr)
 
 
 define { <1 x i64>, <1 x i64>, <1 x i64> } @test_v1i64_post_imm_ld3(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v1i64_post_imm_ld3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.1d { v0, v1, v2 }, [x0], #24
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_imm_ld3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.1d { v0, v1, v2 }, [x0], #24
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_imm_ld3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2776,12 +2773,12 @@ define { <1 x i64>, <1 x i64>, <1 x i64> } @test_v1i64_post_imm_ld3(ptr %A, ptr
 }
 
 define { <1 x i64>, <1 x i64>, <1 x i64> } @test_v1i64_post_reg_ld3(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v1i64_post_reg_ld3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld1.1d { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_reg_ld3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld1.1d { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_reg_ld3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2799,11 +2796,11 @@ declare { <1 x i64>, <1 x i64>, <1 x i64> } @llvm.aarch64.neon.ld3.v1i64.p0(ptr)
 
 
 define { <4 x float>, <4 x float>, <4 x float> } @test_v4f32_post_imm_ld3(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v4f32_post_imm_ld3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld3.4s { v0, v1, v2 }, [x0], #48
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_imm_ld3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld3.4s { v0, v1, v2 }, [x0], #48
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_imm_ld3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2818,12 +2815,12 @@ define { <4 x float>, <4 x float>, <4 x float> } @test_v4f32_post_imm_ld3(ptr %A
 }
 
 define { <4 x float>, <4 x float>, <4 x float> } @test_v4f32_post_reg_ld3(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v4f32_post_reg_ld3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld3.4s { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_reg_ld3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld3.4s { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_reg_ld3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2841,11 +2838,11 @@ declare { <4 x float>, <4 x float>, <4 x float> } @llvm.aarch64.neon.ld3.v4f32.p
 
 
 define { <2 x float>, <2 x float>, <2 x float> } @test_v2f32_post_imm_ld3(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v2f32_post_imm_ld3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld3.2s { v0, v1, v2 }, [x0], #24
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_imm_ld3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld3.2s { v0, v1, v2 }, [x0], #24
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_imm_ld3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2860,12 +2857,12 @@ define { <2 x float>, <2 x float>, <2 x float> } @test_v2f32_post_imm_ld3(ptr %A
 }
 
 define { <2 x float>, <2 x float>, <2 x float> } @test_v2f32_post_reg_ld3(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v2f32_post_reg_ld3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld3.2s { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_reg_ld3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld3.2s { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_reg_ld3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2883,11 +2880,11 @@ declare { <2 x float>, <2 x float>, <2 x float> } @llvm.aarch64.neon.ld3.v2f32.p
 
 
 define { <2 x double>, <2 x double>, <2 x double> } @test_v2f64_post_imm_ld3(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v2f64_post_imm_ld3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld3.2d { v0, v1, v2 }, [x0], #48
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_imm_ld3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld3.2d { v0, v1, v2 }, [x0], #48
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_imm_ld3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2902,12 +2899,12 @@ define { <2 x double>, <2 x double>, <2 x double> } @test_v2f64_post_imm_ld3(ptr
 }
 
 define { <2 x double>, <2 x double>, <2 x double> } @test_v2f64_post_reg_ld3(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v2f64_post_reg_ld3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld3.2d { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_reg_ld3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld3.2d { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_reg_ld3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2925,11 +2922,11 @@ declare { <2 x double>, <2 x double>, <2 x double> } @llvm.aarch64.neon.ld3.v2f6
 
 
 define { <1 x double>, <1 x double>, <1 x double> } @test_v1f64_post_imm_ld3(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v1f64_post_imm_ld3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.1d { v0, v1, v2 }, [x0], #24
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_imm_ld3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.1d { v0, v1, v2 }, [x0], #24
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_imm_ld3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2944,12 +2941,12 @@ define { <1 x double>, <1 x double>, <1 x double> } @test_v1f64_post_imm_ld3(ptr
 }
 
 define { <1 x double>, <1 x double>, <1 x double> } @test_v1f64_post_reg_ld3(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v1f64_post_reg_ld3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld1.1d { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_reg_ld3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld1.1d { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_reg_ld3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2967,11 +2964,11 @@ declare { <1 x double>, <1 x double>, <1 x double> } @llvm.aarch64.neon.ld3.v1f6
 
 
 define { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @test_v16i8_post_imm_ld4(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v16i8_post_imm_ld4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld4.16b { v0, v1, v2, v3 }, [x0], #64
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_imm_ld4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld4.16b { v0, v1, v2, v3 }, [x0], #64
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_imm_ld4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -2986,11 +2983,11 @@ define { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @test_v16i8_post_imm_ld4(p
 }
 
 define { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @test_v16i8_post_reg_ld4(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v16i8_post_reg_ld4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld4.16b { v0, v1, v2, v3 }, [x0], x2
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_reg_ld4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld4.16b { v0, v1, v2, v3 }, [x0], x2
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_reg_ld4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3008,11 +3005,11 @@ declare { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld4.v1
 
 
 define { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } @test_v8i8_post_imm_ld4(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v8i8_post_imm_ld4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld4.8b { v0, v1, v2, v3 }, [x0], #32
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_imm_ld4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld4.8b { v0, v1, v2, v3 }, [x0], #32
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_imm_ld4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3027,11 +3024,11 @@ define { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } @test_v8i8_post_imm_ld4(ptr %A
 }
 
 define { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } @test_v8i8_post_reg_ld4(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v8i8_post_reg_ld4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld4.8b { v0, v1, v2, v3 }, [x0], x2
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_reg_ld4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld4.8b { v0, v1, v2, v3 }, [x0], x2
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_reg_ld4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3049,11 +3046,11 @@ declare { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld4.v8i8.p
 
 
 define { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } @test_v8i16_post_imm_ld4(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v8i16_post_imm_ld4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld4.8h { v0, v1, v2, v3 }, [x0], #64
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_imm_ld4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld4.8h { v0, v1, v2, v3 }, [x0], #64
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_imm_ld4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3068,12 +3065,12 @@ define { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } @test_v8i16_post_imm_ld4(p
 }
 
 define { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } @test_v8i16_post_reg_ld4(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v8i16_post_reg_ld4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ld4.8h { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_reg_ld4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ld4.8h { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_reg_ld4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3091,11 +3088,11 @@ declare { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } @llvm.aarch64.neon.ld4.v8
 
 
 define { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } @test_v4i16_post_imm_ld4(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v4i16_post_imm_ld4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld4.4h { v0, v1, v2, v3 }, [x0], #32
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_imm_ld4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld4.4h { v0, v1, v2, v3 }, [x0], #32
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_imm_ld4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3110,12 +3107,12 @@ define { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } @test_v4i16_post_imm_ld4(p
 }
 
 define { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } @test_v4i16_post_reg_ld4(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v4i16_post_reg_ld4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ld4.4h { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_reg_ld4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ld4.4h { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_reg_ld4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3133,11 +3130,11 @@ declare { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } @llvm.aarch64.neon.ld4.v4
 
 
 define { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @test_v4i32_post_imm_ld4(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v4i32_post_imm_ld4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld4.4s { v0, v1, v2, v3 }, [x0], #64
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_imm_ld4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld4.4s { v0, v1, v2, v3 }, [x0], #64
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_imm_ld4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3152,12 +3149,12 @@ define { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @test_v4i32_post_imm_ld4(p
 }
 
 define { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @test_v4i32_post_reg_ld4(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v4i32_post_reg_ld4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld4.4s { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_reg_ld4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld4.4s { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_reg_ld4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3175,11 +3172,11 @@ declare { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld4.v4
 
 
 define { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } @test_v2i32_post_imm_ld4(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v2i32_post_imm_ld4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld4.2s { v0, v1, v2, v3 }, [x0], #32
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_imm_ld4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld4.2s { v0, v1, v2, v3 }, [x0], #32
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_imm_ld4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3194,12 +3191,12 @@ define { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } @test_v2i32_post_imm_ld4(p
 }
 
 define { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } @test_v2i32_post_reg_ld4(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v2i32_post_reg_ld4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld4.2s { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_reg_ld4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld4.2s { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_reg_ld4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3217,11 +3214,11 @@ declare { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } @llvm.aarch64.neon.ld4.v2
 
 
 define { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @test_v2i64_post_imm_ld4(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v2i64_post_imm_ld4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld4.2d { v0, v1, v2, v3 }, [x0], #64
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_imm_ld4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld4.2d { v0, v1, v2, v3 }, [x0], #64
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_imm_ld4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3236,12 +3233,12 @@ define { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @test_v2i64_post_imm_ld4(p
 }
 
 define { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @test_v2i64_post_reg_ld4(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v2i64_post_reg_ld4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld4.2d { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_reg_ld4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld4.2d { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_reg_ld4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3259,11 +3256,11 @@ declare { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @llvm.aarch64.neon.ld4.v2
 
 
 define { <1 x i64>, <1 x i64>, <1 x i64>, <1 x i64> } @test_v1i64_post_imm_ld4(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v1i64_post_imm_ld4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.1d { v0, v1, v2, v3 }, [x0], #32
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_imm_ld4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.1d { v0, v1, v2, v3 }, [x0], #32
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_imm_ld4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3278,12 +3275,12 @@ define { <1 x i64>, <1 x i64>, <1 x i64>, <1 x i64> } @test_v1i64_post_imm_ld4(p
 }
 
 define { <1 x i64>, <1 x i64>, <1 x i64>, <1 x i64> } @test_v1i64_post_reg_ld4(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v1i64_post_reg_ld4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld1.1d { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_reg_ld4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld1.1d { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_reg_ld4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3301,11 +3298,11 @@ declare { <1 x i64>, <1 x i64>, <1 x i64>, <1 x i64> } @llvm.aarch64.neon.ld4.v1
 
 
 define { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @test_v4f32_post_imm_ld4(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v4f32_post_imm_ld4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld4.4s { v0, v1, v2, v3 }, [x0], #64
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_imm_ld4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld4.4s { v0, v1, v2, v3 }, [x0], #64
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_imm_ld4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3320,12 +3317,12 @@ define { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @test_v4f32_post_i
 }
 
 define { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @test_v4f32_post_reg_ld4(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v4f32_post_reg_ld4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld4.4s { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_reg_ld4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld4.4s { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_reg_ld4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3343,11 +3340,11 @@ declare { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @llvm.aarch64.neo
 
 
 define { <2 x float>, <2 x float>, <2 x float>, <2 x float> } @test_v2f32_post_imm_ld4(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v2f32_post_imm_ld4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld4.2s { v0, v1, v2, v3 }, [x0], #32
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_imm_ld4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld4.2s { v0, v1, v2, v3 }, [x0], #32
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_imm_ld4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3362,12 +3359,12 @@ define { <2 x float>, <2 x float>, <2 x float>, <2 x float> } @test_v2f32_post_i
 }
 
 define { <2 x float>, <2 x float>, <2 x float>, <2 x float> } @test_v2f32_post_reg_ld4(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v2f32_post_reg_ld4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld4.2s { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_reg_ld4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld4.2s { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_reg_ld4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3385,11 +3382,11 @@ declare { <2 x float>, <2 x float>, <2 x float>, <2 x float> } @llvm.aarch64.neo
 
 
 define { <2 x double>, <2 x double>, <2 x double>, <2 x double> } @test_v2f64_post_imm_ld4(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v2f64_post_imm_ld4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld4.2d { v0, v1, v2, v3 }, [x0], #64
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_imm_ld4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld4.2d { v0, v1, v2, v3 }, [x0], #64
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_imm_ld4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3404,12 +3401,12 @@ define { <2 x double>, <2 x double>, <2 x double>, <2 x double> } @test_v2f64_po
 }
 
 define { <2 x double>, <2 x double>, <2 x double>, <2 x double> } @test_v2f64_post_reg_ld4(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v2f64_post_reg_ld4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld4.2d { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_reg_ld4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld4.2d { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_reg_ld4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3427,11 +3424,11 @@ declare { <2 x double>, <2 x double>, <2 x double>, <2 x double> } @llvm.aarch64
 
 
 define { <1 x double>, <1 x double>, <1 x double>, <1 x double> } @test_v1f64_post_imm_ld4(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v1f64_post_imm_ld4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.1d { v0, v1, v2, v3 }, [x0], #32
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_imm_ld4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.1d { v0, v1, v2, v3 }, [x0], #32
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_imm_ld4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3446,12 +3443,12 @@ define { <1 x double>, <1 x double>, <1 x double>, <1 x double> } @test_v1f64_po
 }
 
 define { <1 x double>, <1 x double>, <1 x double>, <1 x double> } @test_v1f64_post_reg_ld4(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v1f64_post_reg_ld4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld1.1d { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_reg_ld4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld1.1d { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_reg_ld4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3468,11 +3465,11 @@ define { <1 x double>, <1 x double>, <1 x double>, <1 x double> } @test_v1f64_po
 declare { <1 x double>, <1 x double>, <1 x double>, <1 x double> } @llvm.aarch64.neon.ld4.v1f64.p0(ptr)
 
 define { <16 x i8>, <16 x i8> } @test_v16i8_post_imm_ld1x2(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v16i8_post_imm_ld1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.16b { v0, v1 }, [x0], #32
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_imm_ld1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.16b { v0, v1 }, [x0], #32
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_imm_ld1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3487,11 +3484,11 @@ define { <16 x i8>, <16 x i8> } @test_v16i8_post_imm_ld1x2(ptr %A, ptr %ptr) {
 }
 
 define { <16 x i8>, <16 x i8> } @test_v16i8_post_reg_ld1x2(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v16i8_post_reg_ld1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.16b { v0, v1 }, [x0], x2
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_reg_ld1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.16b { v0, v1 }, [x0], x2
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_reg_ld1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3509,11 +3506,11 @@ declare { <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld1x2.v16i8.p0(ptr)
 
 
 define { <8 x i8>, <8 x i8> } @test_v8i8_post_imm_ld1x2(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v8i8_post_imm_ld1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.8b { v0, v1 }, [x0], #16
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_imm_ld1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.8b { v0, v1 }, [x0], #16
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_imm_ld1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3528,11 +3525,11 @@ define { <8 x i8>, <8 x i8> } @test_v8i8_post_imm_ld1x2(ptr %A, ptr %ptr) {
 }
 
 define { <8 x i8>, <8 x i8> } @test_v8i8_post_reg_ld1x2(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v8i8_post_reg_ld1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.8b { v0, v1 }, [x0], x2
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_reg_ld1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.8b { v0, v1 }, [x0], x2
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_reg_ld1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3550,11 +3547,11 @@ declare { <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld1x2.v8i8.p0(ptr)
 
 
 define { <8 x i16>, <8 x i16> } @test_v8i16_post_imm_ld1x2(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v8i16_post_imm_ld1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.8h { v0, v1 }, [x0], #32
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_imm_ld1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.8h { v0, v1 }, [x0], #32
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_imm_ld1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3569,12 +3566,12 @@ define { <8 x i16>, <8 x i16> } @test_v8i16_post_imm_ld1x2(ptr %A, ptr %ptr) {
 }
 
 define { <8 x i16>, <8 x i16> } @test_v8i16_post_reg_ld1x2(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v8i16_post_reg_ld1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ld1.8h { v0, v1 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_reg_ld1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ld1.8h { v0, v1 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_reg_ld1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3592,11 +3589,11 @@ declare { <8 x i16>, <8 x i16> } @llvm.aarch64.neon.ld1x2.v8i16.p0(ptr)
 
 
 define { <4 x i16>, <4 x i16> } @test_v4i16_post_imm_ld1x2(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v4i16_post_imm_ld1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.4h { v0, v1 }, [x0], #16
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_imm_ld1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.4h { v0, v1 }, [x0], #16
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_imm_ld1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3611,12 +3608,12 @@ define { <4 x i16>, <4 x i16> } @test_v4i16_post_imm_ld1x2(ptr %A, ptr %ptr) {
 }
 
 define { <4 x i16>, <4 x i16> } @test_v4i16_post_reg_ld1x2(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v4i16_post_reg_ld1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ld1.4h { v0, v1 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_reg_ld1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ld1.4h { v0, v1 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_reg_ld1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3634,11 +3631,11 @@ declare { <4 x i16>, <4 x i16> } @llvm.aarch64.neon.ld1x2.v4i16.p0(ptr)
 
 
 define { <4 x i32>, <4 x i32> } @test_v4i32_post_imm_ld1x2(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v4i32_post_imm_ld1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.4s { v0, v1 }, [x0], #32
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_imm_ld1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.4s { v0, v1 }, [x0], #32
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_imm_ld1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3653,12 +3650,12 @@ define { <4 x i32>, <4 x i32> } @test_v4i32_post_imm_ld1x2(ptr %A, ptr %ptr) {
 }
 
 define { <4 x i32>, <4 x i32> } @test_v4i32_post_reg_ld1x2(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v4i32_post_reg_ld1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld1.4s { v0, v1 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_reg_ld1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld1.4s { v0, v1 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_reg_ld1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3676,11 +3673,11 @@ declare { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld1x2.v4i32.p0(ptr)
 
 
 define { <2 x i32>, <2 x i32> } @test_v2i32_post_imm_ld1x2(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v2i32_post_imm_ld1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.2s { v0, v1 }, [x0], #16
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_imm_ld1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.2s { v0, v1 }, [x0], #16
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_imm_ld1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3695,12 +3692,12 @@ define { <2 x i32>, <2 x i32> } @test_v2i32_post_imm_ld1x2(ptr %A, ptr %ptr) {
 }
 
 define { <2 x i32>, <2 x i32> } @test_v2i32_post_reg_ld1x2(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v2i32_post_reg_ld1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld1.2s { v0, v1 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_reg_ld1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld1.2s { v0, v1 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_reg_ld1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3718,11 +3715,11 @@ declare { <2 x i32>, <2 x i32> } @llvm.aarch64.neon.ld1x2.v2i32.p0(ptr)
 
 
 define { <2 x i64>, <2 x i64> } @test_v2i64_post_imm_ld1x2(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v2i64_post_imm_ld1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.2d { v0, v1 }, [x0], #32
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_imm_ld1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.2d { v0, v1 }, [x0], #32
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_imm_ld1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3737,12 +3734,12 @@ define { <2 x i64>, <2 x i64> } @test_v2i64_post_imm_ld1x2(ptr %A, ptr %ptr) {
 }
 
 define { <2 x i64>, <2 x i64> } @test_v2i64_post_reg_ld1x2(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v2i64_post_reg_ld1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld1.2d { v0, v1 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_reg_ld1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld1.2d { v0, v1 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_reg_ld1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3760,11 +3757,11 @@ declare { <2 x i64>, <2 x i64> } @llvm.aarch64.neon.ld1x2.v2i64.p0(ptr)
 
 
 define { <1 x i64>, <1 x i64> } @test_v1i64_post_imm_ld1x2(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v1i64_post_imm_ld1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.1d { v0, v1 }, [x0], #16
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_imm_ld1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.1d { v0, v1 }, [x0], #16
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_imm_ld1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3779,12 +3776,12 @@ define { <1 x i64>, <1 x i64> } @test_v1i64_post_imm_ld1x2(ptr %A, ptr %ptr) {
 }
 
 define { <1 x i64>, <1 x i64> } @test_v1i64_post_reg_ld1x2(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v1i64_post_reg_ld1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld1.1d { v0, v1 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_reg_ld1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld1.1d { v0, v1 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_reg_ld1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3802,11 +3799,11 @@ declare { <1 x i64>, <1 x i64> } @llvm.aarch64.neon.ld1x2.v1i64.p0(ptr)
 
 
 define { <4 x float>, <4 x float> } @test_v4f32_post_imm_ld1x2(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v4f32_post_imm_ld1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.4s { v0, v1 }, [x0], #32
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_imm_ld1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.4s { v0, v1 }, [x0], #32
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_imm_ld1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3821,12 +3818,12 @@ define { <4 x float>, <4 x float> } @test_v4f32_post_imm_ld1x2(ptr %A, ptr %ptr)
 }
 
 define { <4 x float>, <4 x float> } @test_v4f32_post_reg_ld1x2(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v4f32_post_reg_ld1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld1.4s { v0, v1 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_reg_ld1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld1.4s { v0, v1 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_reg_ld1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3844,11 +3841,11 @@ declare { <4 x float>, <4 x float> } @llvm.aarch64.neon.ld1x2.v4f32.p0(ptr)
 
 
 define { <2 x float>, <2 x float> } @test_v2f32_post_imm_ld1x2(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v2f32_post_imm_ld1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.2s { v0, v1 }, [x0], #16
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_imm_ld1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.2s { v0, v1 }, [x0], #16
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_imm_ld1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3863,12 +3860,12 @@ define { <2 x float>, <2 x float> } @test_v2f32_post_imm_ld1x2(ptr %A, ptr %ptr)
 }
 
 define { <2 x float>, <2 x float> } @test_v2f32_post_reg_ld1x2(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v2f32_post_reg_ld1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld1.2s { v0, v1 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_reg_ld1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld1.2s { v0, v1 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_reg_ld1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3886,11 +3883,11 @@ declare { <2 x float>, <2 x float> } @llvm.aarch64.neon.ld1x2.v2f32.p0(ptr)
 
 
 define { <2 x double>, <2 x double> } @test_v2f64_post_imm_ld1x2(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v2f64_post_imm_ld1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.2d { v0, v1 }, [x0], #32
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_imm_ld1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.2d { v0, v1 }, [x0], #32
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_imm_ld1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3905,12 +3902,12 @@ define { <2 x double>, <2 x double> } @test_v2f64_post_imm_ld1x2(ptr %A, ptr %pt
 }
 
 define { <2 x double>, <2 x double> } @test_v2f64_post_reg_ld1x2(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v2f64_post_reg_ld1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld1.2d { v0, v1 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_reg_ld1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld1.2d { v0, v1 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_reg_ld1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3928,11 +3925,11 @@ declare { <2 x double>, <2 x double> } @llvm.aarch64.neon.ld1x2.v2f64.p0(ptr)
 
 
 define { <1 x double>, <1 x double> } @test_v1f64_post_imm_ld1x2(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v1f64_post_imm_ld1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.1d { v0, v1 }, [x0], #16
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_imm_ld1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.1d { v0, v1 }, [x0], #16
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_imm_ld1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3947,12 +3944,12 @@ define { <1 x double>, <1 x double> } @test_v1f64_post_imm_ld1x2(ptr %A, ptr %pt
 }
 
 define { <1 x double>, <1 x double> } @test_v1f64_post_reg_ld1x2(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v1f64_post_reg_ld1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld1.1d { v0, v1 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_reg_ld1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld1.1d { v0, v1 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_reg_ld1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3970,11 +3967,11 @@ declare { <1 x double>, <1 x double> } @llvm.aarch64.neon.ld1x2.v1f64.p0(ptr)
 
 
 define { <16 x i8>, <16 x i8>, <16 x i8> } @test_v16i8_post_imm_ld1x3(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v16i8_post_imm_ld1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.16b { v0, v1, v2 }, [x0], #48
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_imm_ld1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.16b { v0, v1, v2 }, [x0], #48
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_imm_ld1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -3989,11 +3986,11 @@ define { <16 x i8>, <16 x i8>, <16 x i8> } @test_v16i8_post_imm_ld1x3(ptr %A, pt
 }
 
 define { <16 x i8>, <16 x i8>, <16 x i8> } @test_v16i8_post_reg_ld1x3(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v16i8_post_reg_ld1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.16b { v0, v1, v2 }, [x0], x2
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_reg_ld1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.16b { v0, v1, v2 }, [x0], x2
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_reg_ld1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4011,11 +4008,11 @@ declare { <16 x i8>, <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld1x3.v16i8.p0(pt
 
 
 define { <8 x i8>, <8 x i8>, <8 x i8> } @test_v8i8_post_imm_ld1x3(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v8i8_post_imm_ld1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.8b { v0, v1, v2 }, [x0], #24
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_imm_ld1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.8b { v0, v1, v2 }, [x0], #24
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_imm_ld1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4030,11 +4027,11 @@ define { <8 x i8>, <8 x i8>, <8 x i8> } @test_v8i8_post_imm_ld1x3(ptr %A, ptr %p
 }
 
 define { <8 x i8>, <8 x i8>, <8 x i8> } @test_v8i8_post_reg_ld1x3(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v8i8_post_reg_ld1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.8b { v0, v1, v2 }, [x0], x2
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_reg_ld1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.8b { v0, v1, v2 }, [x0], x2
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_reg_ld1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4052,11 +4049,11 @@ declare { <8 x i8>, <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld1x3.v8i8.p0(ptr)
 
 
 define { <8 x i16>, <8 x i16>, <8 x i16> } @test_v8i16_post_imm_ld1x3(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v8i16_post_imm_ld1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.8h { v0, v1, v2 }, [x0], #48
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_imm_ld1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.8h { v0, v1, v2 }, [x0], #48
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_imm_ld1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4071,12 +4068,12 @@ define { <8 x i16>, <8 x i16>, <8 x i16> } @test_v8i16_post_imm_ld1x3(ptr %A, pt
 }
 
 define { <8 x i16>, <8 x i16>, <8 x i16> } @test_v8i16_post_reg_ld1x3(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v8i16_post_reg_ld1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ld1.8h { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_reg_ld1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ld1.8h { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_reg_ld1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4094,11 +4091,11 @@ declare { <8 x i16>, <8 x i16>, <8 x i16> } @llvm.aarch64.neon.ld1x3.v8i16.p0(pt
 
 
 define { <4 x i16>, <4 x i16>, <4 x i16> } @test_v4i16_post_imm_ld1x3(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v4i16_post_imm_ld1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.4h { v0, v1, v2 }, [x0], #24
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_imm_ld1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.4h { v0, v1, v2 }, [x0], #24
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_imm_ld1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4113,12 +4110,12 @@ define { <4 x i16>, <4 x i16>, <4 x i16> } @test_v4i16_post_imm_ld1x3(ptr %A, pt
 }
 
 define { <4 x i16>, <4 x i16>, <4 x i16> } @test_v4i16_post_reg_ld1x3(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v4i16_post_reg_ld1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ld1.4h { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_reg_ld1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ld1.4h { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_reg_ld1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4136,11 +4133,11 @@ declare { <4 x i16>, <4 x i16>, <4 x i16> } @llvm.aarch64.neon.ld1x3.v4i16.p0(pt
 
 
 define { <4 x i32>, <4 x i32>, <4 x i32> } @test_v4i32_post_imm_ld1x3(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v4i32_post_imm_ld1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.4s { v0, v1, v2 }, [x0], #48
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_imm_ld1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.4s { v0, v1, v2 }, [x0], #48
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_imm_ld1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4155,12 +4152,12 @@ define { <4 x i32>, <4 x i32>, <4 x i32> } @test_v4i32_post_imm_ld1x3(ptr %A, pt
 }
 
 define { <4 x i32>, <4 x i32>, <4 x i32> } @test_v4i32_post_reg_ld1x3(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v4i32_post_reg_ld1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld1.4s { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_reg_ld1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld1.4s { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_reg_ld1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4178,11 +4175,11 @@ declare { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld1x3.v4i32.p0(pt
 
 
 define { <2 x i32>, <2 x i32>, <2 x i32> } @test_v2i32_post_imm_ld1x3(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v2i32_post_imm_ld1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.2s { v0, v1, v2 }, [x0], #24
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_imm_ld1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.2s { v0, v1, v2 }, [x0], #24
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_imm_ld1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4197,12 +4194,12 @@ define { <2 x i32>, <2 x i32>, <2 x i32> } @test_v2i32_post_imm_ld1x3(ptr %A, pt
 }
 
 define { <2 x i32>, <2 x i32>, <2 x i32> } @test_v2i32_post_reg_ld1x3(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v2i32_post_reg_ld1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld1.2s { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_reg_ld1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld1.2s { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_reg_ld1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4220,11 +4217,11 @@ declare { <2 x i32>, <2 x i32>, <2 x i32> } @llvm.aarch64.neon.ld1x3.v2i32.p0(pt
 
 
 define { <2 x i64>, <2 x i64>, <2 x i64> } @test_v2i64_post_imm_ld1x3(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v2i64_post_imm_ld1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.2d { v0, v1, v2 }, [x0], #48
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_imm_ld1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.2d { v0, v1, v2 }, [x0], #48
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_imm_ld1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4239,12 +4236,12 @@ define { <2 x i64>, <2 x i64>, <2 x i64> } @test_v2i64_post_imm_ld1x3(ptr %A, pt
 }
 
 define { <2 x i64>, <2 x i64>, <2 x i64> } @test_v2i64_post_reg_ld1x3(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v2i64_post_reg_ld1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld1.2d { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_reg_ld1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld1.2d { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_reg_ld1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4262,11 +4259,11 @@ declare { <2 x i64>, <2 x i64>, <2 x i64> } @llvm.aarch64.neon.ld1x3.v2i64.p0(pt
 
 
 define { <1 x i64>, <1 x i64>, <1 x i64> } @test_v1i64_post_imm_ld1x3(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v1i64_post_imm_ld1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.1d { v0, v1, v2 }, [x0], #24
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_imm_ld1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.1d { v0, v1, v2 }, [x0], #24
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_imm_ld1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4281,12 +4278,12 @@ define { <1 x i64>, <1 x i64>, <1 x i64> } @test_v1i64_post_imm_ld1x3(ptr %A, pt
 }
 
 define { <1 x i64>, <1 x i64>, <1 x i64> } @test_v1i64_post_reg_ld1x3(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v1i64_post_reg_ld1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld1.1d { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_reg_ld1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld1.1d { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_reg_ld1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4304,11 +4301,11 @@ declare { <1 x i64>, <1 x i64>, <1 x i64> } @llvm.aarch64.neon.ld1x3.v1i64.p0(pt
 
 
 define { <4 x float>, <4 x float>, <4 x float> } @test_v4f32_post_imm_ld1x3(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v4f32_post_imm_ld1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.4s { v0, v1, v2 }, [x0], #48
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_imm_ld1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.4s { v0, v1, v2 }, [x0], #48
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_imm_ld1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4323,12 +4320,12 @@ define { <4 x float>, <4 x float>, <4 x float> } @test_v4f32_post_imm_ld1x3(ptr
 }
 
 define { <4 x float>, <4 x float>, <4 x float> } @test_v4f32_post_reg_ld1x3(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v4f32_post_reg_ld1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld1.4s { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_reg_ld1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld1.4s { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_reg_ld1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4346,11 +4343,11 @@ declare { <4 x float>, <4 x float>, <4 x float> } @llvm.aarch64.neon.ld1x3.v4f32
 
 
 define { <2 x float>, <2 x float>, <2 x float> } @test_v2f32_post_imm_ld1x3(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v2f32_post_imm_ld1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.2s { v0, v1, v2 }, [x0], #24
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_imm_ld1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.2s { v0, v1, v2 }, [x0], #24
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_imm_ld1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4365,12 +4362,12 @@ define { <2 x float>, <2 x float>, <2 x float> } @test_v2f32_post_imm_ld1x3(ptr
 }
 
 define { <2 x float>, <2 x float>, <2 x float> } @test_v2f32_post_reg_ld1x3(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v2f32_post_reg_ld1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld1.2s { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_reg_ld1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld1.2s { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_reg_ld1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4388,11 +4385,11 @@ declare { <2 x float>, <2 x float>, <2 x float> } @llvm.aarch64.neon.ld1x3.v2f32
 
 
 define { <2 x double>, <2 x double>, <2 x double> } @test_v2f64_post_imm_ld1x3(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v2f64_post_imm_ld1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.2d { v0, v1, v2 }, [x0], #48
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_imm_ld1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.2d { v0, v1, v2 }, [x0], #48
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_imm_ld1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4407,12 +4404,12 @@ define { <2 x double>, <2 x double>, <2 x double> } @test_v2f64_post_imm_ld1x3(p
 }
 
 define { <2 x double>, <2 x double>, <2 x double> } @test_v2f64_post_reg_ld1x3(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v2f64_post_reg_ld1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld1.2d { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_reg_ld1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld1.2d { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_reg_ld1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4430,11 +4427,11 @@ declare { <2 x double>, <2 x double>, <2 x double> } @llvm.aarch64.neon.ld1x3.v2
 
 
 define { <1 x double>, <1 x double>, <1 x double> } @test_v1f64_post_imm_ld1x3(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v1f64_post_imm_ld1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.1d { v0, v1, v2 }, [x0], #24
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_imm_ld1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.1d { v0, v1, v2 }, [x0], #24
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_imm_ld1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4449,12 +4446,12 @@ define { <1 x double>, <1 x double>, <1 x double> } @test_v1f64_post_imm_ld1x3(p
 }
 
 define { <1 x double>, <1 x double>, <1 x double> } @test_v1f64_post_reg_ld1x3(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v1f64_post_reg_ld1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld1.1d { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_reg_ld1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld1.1d { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_reg_ld1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4472,11 +4469,11 @@ declare { <1 x double>, <1 x double>, <1 x double> } @llvm.aarch64.neon.ld1x3.v1
 
 
 define { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @test_v16i8_post_imm_ld1x4(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v16i8_post_imm_ld1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.16b { v0, v1, v2, v3 }, [x0], #64
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_imm_ld1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.16b { v0, v1, v2, v3 }, [x0], #64
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_imm_ld1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4491,11 +4488,11 @@ define { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @test_v16i8_post_imm_ld1x4
 }
 
 define { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @test_v16i8_post_reg_ld1x4(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v16i8_post_reg_ld1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.16b { v0, v1, v2, v3 }, [x0], x2
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_reg_ld1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.16b { v0, v1, v2, v3 }, [x0], x2
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_reg_ld1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4513,11 +4510,11 @@ declare { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld1x4.
 
 
 define { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } @test_v8i8_post_imm_ld1x4(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v8i8_post_imm_ld1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.8b { v0, v1, v2, v3 }, [x0], #32
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_imm_ld1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.8b { v0, v1, v2, v3 }, [x0], #32
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_imm_ld1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4532,11 +4529,11 @@ define { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } @test_v8i8_post_imm_ld1x4(ptr
 }
 
 define { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } @test_v8i8_post_reg_ld1x4(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v8i8_post_reg_ld1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.8b { v0, v1, v2, v3 }, [x0], x2
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_reg_ld1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.8b { v0, v1, v2, v3 }, [x0], x2
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_reg_ld1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4554,11 +4551,11 @@ declare { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld1x4.v8i8
 
 
 define { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } @test_v8i16_post_imm_ld1x4(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v8i16_post_imm_ld1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.8h { v0, v1, v2, v3 }, [x0], #64
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_imm_ld1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.8h { v0, v1, v2, v3 }, [x0], #64
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_imm_ld1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4573,12 +4570,12 @@ define { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } @test_v8i16_post_imm_ld1x4
 }
 
 define { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } @test_v8i16_post_reg_ld1x4(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v8i16_post_reg_ld1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ld1.8h { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_reg_ld1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ld1.8h { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_reg_ld1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4596,11 +4593,11 @@ declare { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } @llvm.aarch64.neon.ld1x4.
 
 
 define { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } @test_v4i16_post_imm_ld1x4(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v4i16_post_imm_ld1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.4h { v0, v1, v2, v3 }, [x0], #32
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_imm_ld1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.4h { v0, v1, v2, v3 }, [x0], #32
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_imm_ld1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4615,12 +4612,12 @@ define { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } @test_v4i16_post_imm_ld1x4
 }
 
 define { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } @test_v4i16_post_reg_ld1x4(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v4i16_post_reg_ld1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ld1.4h { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_reg_ld1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ld1.4h { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_reg_ld1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4638,11 +4635,11 @@ declare { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } @llvm.aarch64.neon.ld1x4.
 
 
 define { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @test_v4i32_post_imm_ld1x4(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v4i32_post_imm_ld1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.4s { v0, v1, v2, v3 }, [x0], #64
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_imm_ld1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.4s { v0, v1, v2, v3 }, [x0], #64
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_imm_ld1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4657,12 +4654,12 @@ define { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @test_v4i32_post_imm_ld1x4
 }
 
 define { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @test_v4i32_post_reg_ld1x4(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v4i32_post_reg_ld1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld1.4s { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_reg_ld1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld1.4s { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_reg_ld1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4680,11 +4677,11 @@ declare { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld1x4.
 
 
 define { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } @test_v2i32_post_imm_ld1x4(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v2i32_post_imm_ld1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.2s { v0, v1, v2, v3 }, [x0], #32
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_imm_ld1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.2s { v0, v1, v2, v3 }, [x0], #32
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_imm_ld1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4699,12 +4696,12 @@ define { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } @test_v2i32_post_imm_ld1x4
 }
 
 define { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } @test_v2i32_post_reg_ld1x4(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v2i32_post_reg_ld1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld1.2s { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_reg_ld1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld1.2s { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_reg_ld1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4722,11 +4719,11 @@ declare { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } @llvm.aarch64.neon.ld1x4.
 
 
 define { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @test_v2i64_post_imm_ld1x4(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v2i64_post_imm_ld1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.2d { v0, v1, v2, v3 }, [x0], #64
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_imm_ld1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.2d { v0, v1, v2, v3 }, [x0], #64
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_imm_ld1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4741,12 +4738,12 @@ define { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @test_v2i64_post_imm_ld1x4
 }
 
 define { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @test_v2i64_post_reg_ld1x4(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v2i64_post_reg_ld1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld1.2d { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_reg_ld1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld1.2d { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_reg_ld1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4764,11 +4761,11 @@ declare { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @llvm.aarch64.neon.ld1x4.
 
 
 define { <1 x i64>, <1 x i64>, <1 x i64>, <1 x i64> } @test_v1i64_post_imm_ld1x4(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v1i64_post_imm_ld1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.1d { v0, v1, v2, v3 }, [x0], #32
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_imm_ld1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.1d { v0, v1, v2, v3 }, [x0], #32
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_imm_ld1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4783,12 +4780,12 @@ define { <1 x i64>, <1 x i64>, <1 x i64>, <1 x i64> } @test_v1i64_post_imm_ld1x4
 }
 
 define { <1 x i64>, <1 x i64>, <1 x i64>, <1 x i64> } @test_v1i64_post_reg_ld1x4(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v1i64_post_reg_ld1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld1.1d { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_reg_ld1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld1.1d { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_reg_ld1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4806,11 +4803,11 @@ declare { <1 x i64>, <1 x i64>, <1 x i64>, <1 x i64> } @llvm.aarch64.neon.ld1x4.
 
 
 define { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @test_v4f32_post_imm_ld1x4(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v4f32_post_imm_ld1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.4s { v0, v1, v2, v3 }, [x0], #64
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_imm_ld1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.4s { v0, v1, v2, v3 }, [x0], #64
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_imm_ld1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4825,12 +4822,12 @@ define { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @test_v4f32_post_i
 }
 
 define { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @test_v4f32_post_reg_ld1x4(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v4f32_post_reg_ld1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld1.4s { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_reg_ld1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld1.4s { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_reg_ld1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4848,11 +4845,11 @@ declare { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @llvm.aarch64.neo
 
 
 define { <2 x float>, <2 x float>, <2 x float>, <2 x float> } @test_v2f32_post_imm_ld1x4(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v2f32_post_imm_ld1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.2s { v0, v1, v2, v3 }, [x0], #32
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_imm_ld1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.2s { v0, v1, v2, v3 }, [x0], #32
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_imm_ld1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4867,12 +4864,12 @@ define { <2 x float>, <2 x float>, <2 x float>, <2 x float> } @test_v2f32_post_i
 }
 
 define { <2 x float>, <2 x float>, <2 x float>, <2 x float> } @test_v2f32_post_reg_ld1x4(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v2f32_post_reg_ld1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld1.2s { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_reg_ld1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld1.2s { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_reg_ld1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4890,11 +4887,11 @@ declare { <2 x float>, <2 x float>, <2 x float>, <2 x float> } @llvm.aarch64.neo
 
 
 define { <2 x double>, <2 x double>, <2 x double>, <2 x double> } @test_v2f64_post_imm_ld1x4(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v2f64_post_imm_ld1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.2d { v0, v1, v2, v3 }, [x0], #64
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_imm_ld1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.2d { v0, v1, v2, v3 }, [x0], #64
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_imm_ld1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4909,12 +4906,12 @@ define { <2 x double>, <2 x double>, <2 x double>, <2 x double> } @test_v2f64_po
 }
 
 define { <2 x double>, <2 x double>, <2 x double>, <2 x double> } @test_v2f64_post_reg_ld1x4(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v2f64_post_reg_ld1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld1.2d { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_reg_ld1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld1.2d { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_reg_ld1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4932,11 +4929,11 @@ declare { <2 x double>, <2 x double>, <2 x double>, <2 x double> } @llvm.aarch64
 
 
 define { <1 x double>, <1 x double>, <1 x double>, <1 x double> } @test_v1f64_post_imm_ld1x4(ptr %A, ptr %ptr) {
-; CHECK-LABEL: test_v1f64_post_imm_ld1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.1d { v0, v1, v2, v3 }, [x0], #32
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_imm_ld1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.1d { v0, v1, v2, v3 }, [x0], #32
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_imm_ld1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4951,12 +4948,12 @@ define { <1 x double>, <1 x double>, <1 x double>, <1 x double> } @test_v1f64_po
 }
 
 define { <1 x double>, <1 x double>, <1 x double>, <1 x double> } @test_v1f64_post_reg_ld1x4(ptr %A, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v1f64_post_reg_ld1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld1.1d { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_reg_ld1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld1.1d { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_reg_ld1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4974,11 +4971,11 @@ declare { <1 x double>, <1 x double>, <1 x double>, <1 x double> } @llvm.aarch64
 
 
 define { <16 x i8>, <16 x i8> } @test_v16i8_post_imm_ld2r(ptr %A, ptr %ptr) nounwind {
-; CHECK-LABEL: test_v16i8_post_imm_ld2r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld2r.16b { v0, v1 }, [x0], #2
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_imm_ld2r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld2r.16b { v0, v1 }, [x0], #2
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_imm_ld2r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -4993,11 +4990,11 @@ define { <16 x i8>, <16 x i8> } @test_v16i8_post_imm_ld2r(ptr %A, ptr %ptr) noun
 }
 
 define { <16 x i8>, <16 x i8> } @test_v16i8_post_reg_ld2r(ptr %A, ptr %ptr, i64 %inc) nounwind {
-; CHECK-LABEL: test_v16i8_post_reg_ld2r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld2r.16b { v0, v1 }, [x0], x2
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_reg_ld2r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld2r.16b { v0, v1 }, [x0], x2
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_reg_ld2r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5015,11 +5012,11 @@ declare { <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld2r.v16i8.p0(ptr) nounwind
 
 
 define { <8 x i8>, <8 x i8> } @test_v8i8_post_imm_ld2r(ptr %A, ptr %ptr) nounwind {
-; CHECK-LABEL: test_v8i8_post_imm_ld2r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld2r.8b { v0, v1 }, [x0], #2
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_imm_ld2r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld2r.8b { v0, v1 }, [x0], #2
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_imm_ld2r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5034,11 +5031,11 @@ define { <8 x i8>, <8 x i8> } @test_v8i8_post_imm_ld2r(ptr %A, ptr %ptr) nounwin
 }
 
 define { <8 x i8>, <8 x i8> } @test_v8i8_post_reg_ld2r(ptr %A, ptr %ptr, i64 %inc) nounwind {
-; CHECK-LABEL: test_v8i8_post_reg_ld2r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld2r.8b { v0, v1 }, [x0], x2
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_reg_ld2r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld2r.8b { v0, v1 }, [x0], x2
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_reg_ld2r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5056,11 +5053,11 @@ declare { <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld2r.v8i8.p0(ptr) nounwind rea
 
 
 define { <8 x i16>, <8 x i16> } @test_v8i16_post_imm_ld2r(ptr %A, ptr %ptr) nounwind {
-; CHECK-LABEL: test_v8i16_post_imm_ld2r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld2r.8h { v0, v1 }, [x0], #4
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_imm_ld2r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld2r.8h { v0, v1 }, [x0], #4
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_imm_ld2r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5075,12 +5072,12 @@ define { <8 x i16>, <8 x i16> } @test_v8i16_post_imm_ld2r(ptr %A, ptr %ptr) noun
 }
 
 define { <8 x i16>, <8 x i16> } @test_v8i16_post_reg_ld2r(ptr %A, ptr %ptr, i64 %inc) nounwind {
-; CHECK-LABEL: test_v8i16_post_reg_ld2r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ld2r.8h { v0, v1 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_reg_ld2r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ld2r.8h { v0, v1 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_reg_ld2r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5098,11 +5095,11 @@ declare { <8 x i16>, <8 x i16> } @llvm.aarch64.neon.ld2r.v8i16.p0(ptr) nounwind
 
 
 define { <4 x i16>, <4 x i16> } @test_v4i16_post_imm_ld2r(ptr %A, ptr %ptr) nounwind {
-; CHECK-LABEL: test_v4i16_post_imm_ld2r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld2r.4h { v0, v1 }, [x0], #4
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_imm_ld2r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld2r.4h { v0, v1 }, [x0], #4
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_imm_ld2r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5117,12 +5114,12 @@ define { <4 x i16>, <4 x i16> } @test_v4i16_post_imm_ld2r(ptr %A, ptr %ptr) noun
 }
 
 define { <4 x i16>, <4 x i16> } @test_v4i16_post_reg_ld2r(ptr %A, ptr %ptr, i64 %inc) nounwind {
-; CHECK-LABEL: test_v4i16_post_reg_ld2r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ld2r.4h { v0, v1 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_reg_ld2r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ld2r.4h { v0, v1 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_reg_ld2r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5140,11 +5137,11 @@ declare { <4 x i16>, <4 x i16> } @llvm.aarch64.neon.ld2r.v4i16.p0(ptr) nounwind
 
 
 define { <4 x i32>, <4 x i32> } @test_v4i32_post_imm_ld2r(ptr %A, ptr %ptr) nounwind {
-; CHECK-LABEL: test_v4i32_post_imm_ld2r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld2r.4s { v0, v1 }, [x0], #8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_imm_ld2r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld2r.4s { v0, v1 }, [x0], #8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_imm_ld2r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5159,12 +5156,12 @@ define { <4 x i32>, <4 x i32> } @test_v4i32_post_imm_ld2r(ptr %A, ptr %ptr) noun
 }
 
 define { <4 x i32>, <4 x i32> } @test_v4i32_post_reg_ld2r(ptr %A, ptr %ptr, i64 %inc) nounwind {
-; CHECK-LABEL: test_v4i32_post_reg_ld2r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld2r.4s { v0, v1 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_reg_ld2r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld2r.4s { v0, v1 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_reg_ld2r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5181,11 +5178,11 @@ define { <4 x i32>, <4 x i32> } @test_v4i32_post_reg_ld2r(ptr %A, ptr %ptr, i64
 declare { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld2r.v4i32.p0(ptr) nounwind readonly
 
 define { <2 x i32>, <2 x i32> } @test_v2i32_post_imm_ld2r(ptr %A, ptr %ptr) nounwind {
-; CHECK-LABEL: test_v2i32_post_imm_ld2r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld2r.2s { v0, v1 }, [x0], #8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_imm_ld2r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld2r.2s { v0, v1 }, [x0], #8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_imm_ld2r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5200,12 +5197,12 @@ define { <2 x i32>, <2 x i32> } @test_v2i32_post_imm_ld2r(ptr %A, ptr %ptr) noun
 }
 
 define { <2 x i32>, <2 x i32> } @test_v2i32_post_reg_ld2r(ptr %A, ptr %ptr, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2i32_post_reg_ld2r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld2r.2s { v0, v1 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_reg_ld2r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld2r.2s { v0, v1 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_reg_ld2r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5223,11 +5220,11 @@ declare { <2 x i32>, <2 x i32> } @llvm.aarch64.neon.ld2r.v2i32.p0(ptr) nounwind
 
 
 define { <2 x i64>, <2 x i64> } @test_v2i64_post_imm_ld2r(ptr %A, ptr %ptr) nounwind {
-; CHECK-LABEL: test_v2i64_post_imm_ld2r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld2r.2d { v0, v1 }, [x0], #16
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_imm_ld2r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld2r.2d { v0, v1 }, [x0], #16
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_imm_ld2r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5242,12 +5239,12 @@ define { <2 x i64>, <2 x i64> } @test_v2i64_post_imm_ld2r(ptr %A, ptr %ptr) noun
 }
 
 define { <2 x i64>, <2 x i64> } @test_v2i64_post_reg_ld2r(ptr %A, ptr %ptr, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2i64_post_reg_ld2r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld2r.2d { v0, v1 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_reg_ld2r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld2r.2d { v0, v1 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_reg_ld2r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5264,11 +5261,11 @@ define { <2 x i64>, <2 x i64> } @test_v2i64_post_reg_ld2r(ptr %A, ptr %ptr, i64
 declare { <2 x i64>, <2 x i64> } @llvm.aarch64.neon.ld2r.v2i64.p0(ptr) nounwind readonly
 
 define { <1 x i64>, <1 x i64> } @test_v1i64_post_imm_ld2r(ptr %A, ptr %ptr) nounwind {
-; CHECK-LABEL: test_v1i64_post_imm_ld2r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld2r.1d { v0, v1 }, [x0], #16
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_imm_ld2r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld2r.1d { v0, v1 }, [x0], #16
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_imm_ld2r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5283,12 +5280,12 @@ define { <1 x i64>, <1 x i64> } @test_v1i64_post_imm_ld2r(ptr %A, ptr %ptr) noun
 }
 
 define { <1 x i64>, <1 x i64> } @test_v1i64_post_reg_ld2r(ptr %A, ptr %ptr, i64 %inc) nounwind {
-; CHECK-LABEL: test_v1i64_post_reg_ld2r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld2r.1d { v0, v1 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_reg_ld2r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld2r.1d { v0, v1 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_reg_ld2r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5306,11 +5303,11 @@ declare { <1 x i64>, <1 x i64> } @llvm.aarch64.neon.ld2r.v1i64.p0(ptr) nounwind
 
 
 define { <4 x float>, <4 x float> } @test_v4f32_post_imm_ld2r(ptr %A, ptr %ptr) nounwind {
-; CHECK-LABEL: test_v4f32_post_imm_ld2r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld2r.4s { v0, v1 }, [x0], #8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_imm_ld2r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld2r.4s { v0, v1 }, [x0], #8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_imm_ld2r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5325,12 +5322,12 @@ define { <4 x float>, <4 x float> } @test_v4f32_post_imm_ld2r(ptr %A, ptr %ptr)
 }
 
 define { <4 x float>, <4 x float> } @test_v4f32_post_reg_ld2r(ptr %A, ptr %ptr, i64 %inc) nounwind {
-; CHECK-LABEL: test_v4f32_post_reg_ld2r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld2r.4s { v0, v1 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_reg_ld2r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld2r.4s { v0, v1 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_reg_ld2r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5347,11 +5344,11 @@ define { <4 x float>, <4 x float> } @test_v4f32_post_reg_ld2r(ptr %A, ptr %ptr,
 declare { <4 x float>, <4 x float> } @llvm.aarch64.neon.ld2r.v4f32.p0(ptr) nounwind readonly
 
 define { <2 x float>, <2 x float> } @test_v2f32_post_imm_ld2r(ptr %A, ptr %ptr) nounwind {
-; CHECK-LABEL: test_v2f32_post_imm_ld2r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld2r.2s { v0, v1 }, [x0], #8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_imm_ld2r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld2r.2s { v0, v1 }, [x0], #8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_imm_ld2r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5366,12 +5363,12 @@ define { <2 x float>, <2 x float> } @test_v2f32_post_imm_ld2r(ptr %A, ptr %ptr)
 }
 
 define { <2 x float>, <2 x float> } @test_v2f32_post_reg_ld2r(ptr %A, ptr %ptr, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2f32_post_reg_ld2r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld2r.2s { v0, v1 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_reg_ld2r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld2r.2s { v0, v1 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_reg_ld2r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5389,11 +5386,11 @@ declare { <2 x float>, <2 x float> } @llvm.aarch64.neon.ld2r.v2f32.p0(ptr) nounw
 
 
 define { <2 x double>, <2 x double> } @test_v2f64_post_imm_ld2r(ptr %A, ptr %ptr) nounwind {
-; CHECK-LABEL: test_v2f64_post_imm_ld2r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld2r.2d { v0, v1 }, [x0], #16
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_imm_ld2r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld2r.2d { v0, v1 }, [x0], #16
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_imm_ld2r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5408,12 +5405,12 @@ define { <2 x double>, <2 x double> } @test_v2f64_post_imm_ld2r(ptr %A, ptr %ptr
 }
 
 define { <2 x double>, <2 x double> } @test_v2f64_post_reg_ld2r(ptr %A, ptr %ptr, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2f64_post_reg_ld2r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld2r.2d { v0, v1 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_reg_ld2r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld2r.2d { v0, v1 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_reg_ld2r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5430,11 +5427,11 @@ define { <2 x double>, <2 x double> } @test_v2f64_post_reg_ld2r(ptr %A, ptr %ptr
 declare { <2 x double>, <2 x double> } @llvm.aarch64.neon.ld2r.v2f64.p0(ptr) nounwind readonly
 
 define { <1 x double>, <1 x double> } @test_v1f64_post_imm_ld2r(ptr %A, ptr %ptr) nounwind {
-; CHECK-LABEL: test_v1f64_post_imm_ld2r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld2r.1d { v0, v1 }, [x0], #16
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_imm_ld2r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld2r.1d { v0, v1 }, [x0], #16
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_imm_ld2r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5449,12 +5446,12 @@ define { <1 x double>, <1 x double> } @test_v1f64_post_imm_ld2r(ptr %A, ptr %ptr
 }
 
 define { <1 x double>, <1 x double> } @test_v1f64_post_reg_ld2r(ptr %A, ptr %ptr, i64 %inc) nounwind {
-; CHECK-LABEL: test_v1f64_post_reg_ld2r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld2r.1d { v0, v1 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_reg_ld2r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld2r.1d { v0, v1 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_reg_ld2r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5472,11 +5469,11 @@ declare { <1 x double>, <1 x double> } @llvm.aarch64.neon.ld2r.v1f64.p0(ptr) nou
 
 
 define { <16 x i8>, <16 x i8>, <16 x i8> } @test_v16i8_post_imm_ld3r(ptr %A, ptr %ptr) nounwind {
-; CHECK-LABEL: test_v16i8_post_imm_ld3r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld3r.16b { v0, v1, v2 }, [x0], #3
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_imm_ld3r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld3r.16b { v0, v1, v2 }, [x0], #3
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_imm_ld3r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5491,11 +5488,11 @@ define { <16 x i8>, <16 x i8>, <16 x i8> } @test_v16i8_post_imm_ld3r(ptr %A, ptr
 }
 
 define { <16 x i8>, <16 x i8>, <16 x i8> } @test_v16i8_post_reg_ld3r(ptr %A, ptr %ptr, i64 %inc) nounwind {
-; CHECK-LABEL: test_v16i8_post_reg_ld3r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld3r.16b { v0, v1, v2 }, [x0], x2
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_reg_ld3r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld3r.16b { v0, v1, v2 }, [x0], x2
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_reg_ld3r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5513,11 +5510,11 @@ declare { <16 x i8>, <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld3r.v16i8.p0(ptr
 
 
 define { <8 x i8>, <8 x i8>, <8 x i8> } @test_v8i8_post_imm_ld3r(ptr %A, ptr %ptr) nounwind {
-; CHECK-LABEL: test_v8i8_post_imm_ld3r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld3r.8b { v0, v1, v2 }, [x0], #3
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_imm_ld3r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld3r.8b { v0, v1, v2 }, [x0], #3
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_imm_ld3r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5532,11 +5529,11 @@ define { <8 x i8>, <8 x i8>, <8 x i8> } @test_v8i8_post_imm_ld3r(ptr %A, ptr %pt
 }
 
 define { <8 x i8>, <8 x i8>, <8 x i8> } @test_v8i8_post_reg_ld3r(ptr %A, ptr %ptr, i64 %inc) nounwind {
-; CHECK-LABEL: test_v8i8_post_reg_ld3r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld3r.8b { v0, v1, v2 }, [x0], x2
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_reg_ld3r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld3r.8b { v0, v1, v2 }, [x0], x2
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_reg_ld3r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5554,11 +5551,11 @@ declare { <8 x i8>, <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld3r.v8i8.p0(ptr) no
 
 
 define { <8 x i16>, <8 x i16>, <8 x i16> } @test_v8i16_post_imm_ld3r(ptr %A, ptr %ptr) nounwind {
-; CHECK-LABEL: test_v8i16_post_imm_ld3r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld3r.8h { v0, v1, v2 }, [x0], #6
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_imm_ld3r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld3r.8h { v0, v1, v2 }, [x0], #6
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_imm_ld3r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5573,12 +5570,12 @@ define { <8 x i16>, <8 x i16>, <8 x i16> } @test_v8i16_post_imm_ld3r(ptr %A, ptr
 }
 
 define { <8 x i16>, <8 x i16>, <8 x i16> } @test_v8i16_post_reg_ld3r(ptr %A, ptr %ptr, i64 %inc) nounwind {
-; CHECK-LABEL: test_v8i16_post_reg_ld3r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ld3r.8h { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_reg_ld3r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ld3r.8h { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_reg_ld3r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5596,11 +5593,11 @@ declare { <8 x i16>, <8 x i16>, <8 x i16> } @llvm.aarch64.neon.ld3r.v8i16.p0(ptr
 
 
 define { <4 x i16>, <4 x i16>, <4 x i16> } @test_v4i16_post_imm_ld3r(ptr %A, ptr %ptr) nounwind {
-; CHECK-LABEL: test_v4i16_post_imm_ld3r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld3r.4h { v0, v1, v2 }, [x0], #6
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_imm_ld3r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld3r.4h { v0, v1, v2 }, [x0], #6
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_imm_ld3r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5615,12 +5612,12 @@ define { <4 x i16>, <4 x i16>, <4 x i16> } @test_v4i16_post_imm_ld3r(ptr %A, ptr
 }
 
 define { <4 x i16>, <4 x i16>, <4 x i16> } @test_v4i16_post_reg_ld3r(ptr %A, ptr %ptr, i64 %inc) nounwind {
-; CHECK-LABEL: test_v4i16_post_reg_ld3r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ld3r.4h { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_reg_ld3r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ld3r.4h { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_reg_ld3r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5638,11 +5635,11 @@ declare { <4 x i16>, <4 x i16>, <4 x i16> } @llvm.aarch64.neon.ld3r.v4i16.p0(ptr
 
 
 define { <4 x i32>, <4 x i32>, <4 x i32> } @test_v4i32_post_imm_ld3r(ptr %A, ptr %ptr) nounwind {
-; CHECK-LABEL: test_v4i32_post_imm_ld3r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld3r.4s { v0, v1, v2 }, [x0], #12
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_imm_ld3r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld3r.4s { v0, v1, v2 }, [x0], #12
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_imm_ld3r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5657,12 +5654,12 @@ define { <4 x i32>, <4 x i32>, <4 x i32> } @test_v4i32_post_imm_ld3r(ptr %A, ptr
 }
 
 define { <4 x i32>, <4 x i32>, <4 x i32> } @test_v4i32_post_reg_ld3r(ptr %A, ptr %ptr, i64 %inc) nounwind {
-; CHECK-LABEL: test_v4i32_post_reg_ld3r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld3r.4s { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_reg_ld3r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld3r.4s { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_reg_ld3r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5679,11 +5676,11 @@ define { <4 x i32>, <4 x i32>, <4 x i32> } @test_v4i32_post_reg_ld3r(ptr %A, ptr
 declare { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld3r.v4i32.p0(ptr) nounwind readonly
 
 define { <2 x i32>, <2 x i32>, <2 x i32> } @test_v2i32_post_imm_ld3r(ptr %A, ptr %ptr) nounwind {
-; CHECK-LABEL: test_v2i32_post_imm_ld3r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld3r.2s { v0, v1, v2 }, [x0], #12
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_imm_ld3r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld3r.2s { v0, v1, v2 }, [x0], #12
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_imm_ld3r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5698,12 +5695,12 @@ define { <2 x i32>, <2 x i32>, <2 x i32> } @test_v2i32_post_imm_ld3r(ptr %A, ptr
 }
 
 define { <2 x i32>, <2 x i32>, <2 x i32> } @test_v2i32_post_reg_ld3r(ptr %A, ptr %ptr, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2i32_post_reg_ld3r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld3r.2s { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_reg_ld3r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld3r.2s { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_reg_ld3r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5721,11 +5718,11 @@ declare { <2 x i32>, <2 x i32>, <2 x i32> } @llvm.aarch64.neon.ld3r.v2i32.p0(ptr
 
 
 define { <2 x i64>, <2 x i64>, <2 x i64> } @test_v2i64_post_imm_ld3r(ptr %A, ptr %ptr) nounwind {
-; CHECK-LABEL: test_v2i64_post_imm_ld3r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld3r.2d { v0, v1, v2 }, [x0], #24
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_imm_ld3r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld3r.2d { v0, v1, v2 }, [x0], #24
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_imm_ld3r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5740,12 +5737,12 @@ define { <2 x i64>, <2 x i64>, <2 x i64> } @test_v2i64_post_imm_ld3r(ptr %A, ptr
 }
 
 define { <2 x i64>, <2 x i64>, <2 x i64> } @test_v2i64_post_reg_ld3r(ptr %A, ptr %ptr, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2i64_post_reg_ld3r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld3r.2d { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_reg_ld3r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld3r.2d { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_reg_ld3r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5762,11 +5759,11 @@ define { <2 x i64>, <2 x i64>, <2 x i64> } @test_v2i64_post_reg_ld3r(ptr %A, ptr
 declare { <2 x i64>, <2 x i64>, <2 x i64> } @llvm.aarch64.neon.ld3r.v2i64.p0(ptr) nounwind readonly
 
 define { <1 x i64>, <1 x i64>, <1 x i64> } @test_v1i64_post_imm_ld3r(ptr %A, ptr %ptr) nounwind {
-; CHECK-LABEL: test_v1i64_post_imm_ld3r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld3r.1d { v0, v1, v2 }, [x0], #24
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_imm_ld3r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld3r.1d { v0, v1, v2 }, [x0], #24
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_imm_ld3r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5781,12 +5778,12 @@ define { <1 x i64>, <1 x i64>, <1 x i64> } @test_v1i64_post_imm_ld3r(ptr %A, ptr
 }
 
 define { <1 x i64>, <1 x i64>, <1 x i64> } @test_v1i64_post_reg_ld3r(ptr %A, ptr %ptr, i64 %inc) nounwind {
-; CHECK-LABEL: test_v1i64_post_reg_ld3r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld3r.1d { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_reg_ld3r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld3r.1d { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_reg_ld3r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5804,11 +5801,11 @@ declare { <1 x i64>, <1 x i64>, <1 x i64> } @llvm.aarch64.neon.ld3r.v1i64.p0(ptr
 
 
 define { <4 x float>, <4 x float>, <4 x float> } @test_v4f32_post_imm_ld3r(ptr %A, ptr %ptr) nounwind {
-; CHECK-LABEL: test_v4f32_post_imm_ld3r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld3r.4s { v0, v1, v2 }, [x0], #12
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_imm_ld3r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld3r.4s { v0, v1, v2 }, [x0], #12
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_imm_ld3r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5823,12 +5820,12 @@ define { <4 x float>, <4 x float>, <4 x float> } @test_v4f32_post_imm_ld3r(ptr %
 }
 
 define { <4 x float>, <4 x float>, <4 x float> } @test_v4f32_post_reg_ld3r(ptr %A, ptr %ptr, i64 %inc) nounwind {
-; CHECK-LABEL: test_v4f32_post_reg_ld3r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld3r.4s { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_reg_ld3r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld3r.4s { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_reg_ld3r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5845,11 +5842,11 @@ define { <4 x float>, <4 x float>, <4 x float> } @test_v4f32_post_reg_ld3r(ptr %
 declare { <4 x float>, <4 x float>, <4 x float> } @llvm.aarch64.neon.ld3r.v4f32.p0(ptr) nounwind readonly
 
 define { <2 x float>, <2 x float>, <2 x float> } @test_v2f32_post_imm_ld3r(ptr %A, ptr %ptr) nounwind {
-; CHECK-LABEL: test_v2f32_post_imm_ld3r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld3r.2s { v0, v1, v2 }, [x0], #12
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_imm_ld3r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld3r.2s { v0, v1, v2 }, [x0], #12
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_imm_ld3r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5864,12 +5861,12 @@ define { <2 x float>, <2 x float>, <2 x float> } @test_v2f32_post_imm_ld3r(ptr %
 }
 
 define { <2 x float>, <2 x float>, <2 x float> } @test_v2f32_post_reg_ld3r(ptr %A, ptr %ptr, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2f32_post_reg_ld3r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld3r.2s { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_reg_ld3r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld3r.2s { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_reg_ld3r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5887,11 +5884,11 @@ declare { <2 x float>, <2 x float>, <2 x float> } @llvm.aarch64.neon.ld3r.v2f32.
 
 
 define { <2 x double>, <2 x double>, <2 x double> } @test_v2f64_post_imm_ld3r(ptr %A, ptr %ptr) nounwind {
-; CHECK-LABEL: test_v2f64_post_imm_ld3r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld3r.2d { v0, v1, v2 }, [x0], #24
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_imm_ld3r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld3r.2d { v0, v1, v2 }, [x0], #24
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_imm_ld3r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5906,12 +5903,12 @@ define { <2 x double>, <2 x double>, <2 x double> } @test_v2f64_post_imm_ld3r(pt
 }
 
 define { <2 x double>, <2 x double>, <2 x double> } @test_v2f64_post_reg_ld3r(ptr %A, ptr %ptr, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2f64_post_reg_ld3r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld3r.2d { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_reg_ld3r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld3r.2d { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_reg_ld3r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5928,11 +5925,11 @@ define { <2 x double>, <2 x double>, <2 x double> } @test_v2f64_post_reg_ld3r(pt
 declare { <2 x double>, <2 x double>, <2 x double> } @llvm.aarch64.neon.ld3r.v2f64.p0(ptr) nounwind readonly
 
 define { <1 x double>, <1 x double>, <1 x double> } @test_v1f64_post_imm_ld3r(ptr %A, ptr %ptr) nounwind {
-; CHECK-LABEL: test_v1f64_post_imm_ld3r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld3r.1d { v0, v1, v2 }, [x0], #24
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_imm_ld3r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld3r.1d { v0, v1, v2 }, [x0], #24
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_imm_ld3r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5947,12 +5944,12 @@ define { <1 x double>, <1 x double>, <1 x double> } @test_v1f64_post_imm_ld3r(pt
 }
 
 define { <1 x double>, <1 x double>, <1 x double> } @test_v1f64_post_reg_ld3r(ptr %A, ptr %ptr, i64 %inc) nounwind {
-; CHECK-LABEL: test_v1f64_post_reg_ld3r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld3r.1d { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_reg_ld3r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld3r.1d { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_reg_ld3r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5970,11 +5967,11 @@ declare { <1 x double>, <1 x double>, <1 x double> } @llvm.aarch64.neon.ld3r.v1f
 
 
 define { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @test_v16i8_post_imm_ld4r(ptr %A, ptr %ptr) nounwind {
-; CHECK-LABEL: test_v16i8_post_imm_ld4r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld4r.16b { v0, v1, v2, v3 }, [x0], #4
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_imm_ld4r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld4r.16b { v0, v1, v2, v3 }, [x0], #4
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_imm_ld4r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -5989,11 +5986,11 @@ define { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @test_v16i8_post_imm_ld4r(
 }
 
 define { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @test_v16i8_post_reg_ld4r(ptr %A, ptr %ptr, i64 %inc) nounwind {
-; CHECK-LABEL: test_v16i8_post_reg_ld4r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld4r.16b { v0, v1, v2, v3 }, [x0], x2
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_reg_ld4r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld4r.16b { v0, v1, v2, v3 }, [x0], x2
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_reg_ld4r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6011,11 +6008,11 @@ declare { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld4r.v
 
 
 define { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } @test_v8i8_post_imm_ld4r(ptr %A, ptr %ptr) nounwind {
-; CHECK-LABEL: test_v8i8_post_imm_ld4r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld4r.8b { v0, v1, v2, v3 }, [x0], #4
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_imm_ld4r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld4r.8b { v0, v1, v2, v3 }, [x0], #4
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_imm_ld4r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6030,11 +6027,11 @@ define { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } @test_v8i8_post_imm_ld4r(ptr %
 }
 
 define { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } @test_v8i8_post_reg_ld4r(ptr %A, ptr %ptr, i64 %inc) nounwind {
-; CHECK-LABEL: test_v8i8_post_reg_ld4r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld4r.8b { v0, v1, v2, v3 }, [x0], x2
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_reg_ld4r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld4r.8b { v0, v1, v2, v3 }, [x0], x2
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_reg_ld4r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6052,11 +6049,11 @@ declare { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld4r.v8i8.
 
 
 define { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } @test_v8i16_post_imm_ld4r(ptr %A, ptr %ptr) nounwind {
-; CHECK-LABEL: test_v8i16_post_imm_ld4r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld4r.8h { v0, v1, v2, v3 }, [x0], #8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_imm_ld4r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld4r.8h { v0, v1, v2, v3 }, [x0], #8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_imm_ld4r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6071,12 +6068,12 @@ define { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } @test_v8i16_post_imm_ld4r(
 }
 
 define { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } @test_v8i16_post_reg_ld4r(ptr %A, ptr %ptr, i64 %inc) nounwind {
-; CHECK-LABEL: test_v8i16_post_reg_ld4r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ld4r.8h { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_reg_ld4r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ld4r.8h { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_reg_ld4r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6094,11 +6091,11 @@ declare { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } @llvm.aarch64.neon.ld4r.v
 
 
 define { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } @test_v4i16_post_imm_ld4r(ptr %A, ptr %ptr) nounwind {
-; CHECK-LABEL: test_v4i16_post_imm_ld4r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld4r.4h { v0, v1, v2, v3 }, [x0], #8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_imm_ld4r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld4r.4h { v0, v1, v2, v3 }, [x0], #8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_imm_ld4r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6113,12 +6110,12 @@ define { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } @test_v4i16_post_imm_ld4r(
 }
 
 define { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } @test_v4i16_post_reg_ld4r(ptr %A, ptr %ptr, i64 %inc) nounwind {
-; CHECK-LABEL: test_v4i16_post_reg_ld4r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ld4r.4h { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_reg_ld4r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ld4r.4h { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_reg_ld4r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6136,11 +6133,11 @@ declare { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } @llvm.aarch64.neon.ld4r.v
 
 
 define { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @test_v4i32_post_imm_ld4r(ptr %A, ptr %ptr) nounwind {
-; CHECK-LABEL: test_v4i32_post_imm_ld4r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld4r.4s { v0, v1, v2, v3 }, [x0], #16
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_imm_ld4r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld4r.4s { v0, v1, v2, v3 }, [x0], #16
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_imm_ld4r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6155,12 +6152,12 @@ define { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @test_v4i32_post_imm_ld4r(
 }
 
 define { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @test_v4i32_post_reg_ld4r(ptr %A, ptr %ptr, i64 %inc) nounwind {
-; CHECK-LABEL: test_v4i32_post_reg_ld4r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld4r.4s { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_reg_ld4r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld4r.4s { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_reg_ld4r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6177,11 +6174,11 @@ define { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @test_v4i32_post_reg_ld4r(
 declare { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld4r.v4i32.p0(ptr) nounwind readonly
 
 define { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } @test_v2i32_post_imm_ld4r(ptr %A, ptr %ptr) nounwind {
-; CHECK-LABEL: test_v2i32_post_imm_ld4r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld4r.2s { v0, v1, v2, v3 }, [x0], #16
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_imm_ld4r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld4r.2s { v0, v1, v2, v3 }, [x0], #16
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_imm_ld4r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6196,12 +6193,12 @@ define { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } @test_v2i32_post_imm_ld4r(
 }
 
 define { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } @test_v2i32_post_reg_ld4r(ptr %A, ptr %ptr, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2i32_post_reg_ld4r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld4r.2s { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_reg_ld4r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld4r.2s { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_reg_ld4r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6219,11 +6216,11 @@ declare { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } @llvm.aarch64.neon.ld4r.v
 
 
 define { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @test_v2i64_post_imm_ld4r(ptr %A, ptr %ptr) nounwind {
-; CHECK-LABEL: test_v2i64_post_imm_ld4r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld4r.2d { v0, v1, v2, v3 }, [x0], #32
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_imm_ld4r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld4r.2d { v0, v1, v2, v3 }, [x0], #32
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_imm_ld4r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6238,12 +6235,12 @@ define { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @test_v2i64_post_imm_ld4r(
 }
 
 define { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @test_v2i64_post_reg_ld4r(ptr %A, ptr %ptr, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2i64_post_reg_ld4r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld4r.2d { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_reg_ld4r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld4r.2d { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_reg_ld4r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6260,11 +6257,11 @@ define { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @test_v2i64_post_reg_ld4r(
 declare { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @llvm.aarch64.neon.ld4r.v2i64.p0(ptr) nounwind readonly
 
 define { <1 x i64>, <1 x i64>, <1 x i64>, <1 x i64> } @test_v1i64_post_imm_ld4r(ptr %A, ptr %ptr) nounwind {
-; CHECK-LABEL: test_v1i64_post_imm_ld4r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld4r.1d { v0, v1, v2, v3 }, [x0], #32
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_imm_ld4r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld4r.1d { v0, v1, v2, v3 }, [x0], #32
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_imm_ld4r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6279,12 +6276,12 @@ define { <1 x i64>, <1 x i64>, <1 x i64>, <1 x i64> } @test_v1i64_post_imm_ld4r(
 }
 
 define { <1 x i64>, <1 x i64>, <1 x i64>, <1 x i64> } @test_v1i64_post_reg_ld4r(ptr %A, ptr %ptr, i64 %inc) nounwind {
-; CHECK-LABEL: test_v1i64_post_reg_ld4r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld4r.1d { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_reg_ld4r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld4r.1d { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_reg_ld4r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6302,11 +6299,11 @@ declare { <1 x i64>, <1 x i64>, <1 x i64>, <1 x i64> } @llvm.aarch64.neon.ld4r.v
 
 
 define { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @test_v4f32_post_imm_ld4r(ptr %A, ptr %ptr) nounwind {
-; CHECK-LABEL: test_v4f32_post_imm_ld4r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld4r.4s { v0, v1, v2, v3 }, [x0], #16
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_imm_ld4r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld4r.4s { v0, v1, v2, v3 }, [x0], #16
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_imm_ld4r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6321,12 +6318,12 @@ define { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @test_v4f32_post_i
 }
 
 define { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @test_v4f32_post_reg_ld4r(ptr %A, ptr %ptr, i64 %inc) nounwind {
-; CHECK-LABEL: test_v4f32_post_reg_ld4r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld4r.4s { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_reg_ld4r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld4r.4s { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_reg_ld4r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6343,11 +6340,11 @@ define { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @test_v4f32_post_r
 declare { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @llvm.aarch64.neon.ld4r.v4f32.p0(ptr) nounwind readonly
 
 define { <2 x float>, <2 x float>, <2 x float>, <2 x float> } @test_v2f32_post_imm_ld4r(ptr %A, ptr %ptr) nounwind {
-; CHECK-LABEL: test_v2f32_post_imm_ld4r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld4r.2s { v0, v1, v2, v3 }, [x0], #16
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_imm_ld4r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld4r.2s { v0, v1, v2, v3 }, [x0], #16
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_imm_ld4r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6362,12 +6359,12 @@ define { <2 x float>, <2 x float>, <2 x float>, <2 x float> } @test_v2f32_post_i
 }
 
 define { <2 x float>, <2 x float>, <2 x float>, <2 x float> } @test_v2f32_post_reg_ld4r(ptr %A, ptr %ptr, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2f32_post_reg_ld4r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld4r.2s { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_reg_ld4r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld4r.2s { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_reg_ld4r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6385,11 +6382,11 @@ declare { <2 x float>, <2 x float>, <2 x float>, <2 x float> } @llvm.aarch64.neo
 
 
 define { <2 x double>, <2 x double>, <2 x double>, <2 x double> } @test_v2f64_post_imm_ld4r(ptr %A, ptr %ptr) nounwind {
-; CHECK-LABEL: test_v2f64_post_imm_ld4r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld4r.2d { v0, v1, v2, v3 }, [x0], #32
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_imm_ld4r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld4r.2d { v0, v1, v2, v3 }, [x0], #32
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_imm_ld4r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6404,12 +6401,12 @@ define { <2 x double>, <2 x double>, <2 x double>, <2 x double> } @test_v2f64_po
 }
 
 define { <2 x double>, <2 x double>, <2 x double>, <2 x double> } @test_v2f64_post_reg_ld4r(ptr %A, ptr %ptr, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2f64_post_reg_ld4r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld4r.2d { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_reg_ld4r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld4r.2d { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_reg_ld4r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6426,11 +6423,11 @@ define { <2 x double>, <2 x double>, <2 x double>, <2 x double> } @test_v2f64_po
 declare { <2 x double>, <2 x double>, <2 x double>, <2 x double> } @llvm.aarch64.neon.ld4r.v2f64.p0(ptr) nounwind readonly
 
 define { <1 x double>, <1 x double>, <1 x double>, <1 x double> } @test_v1f64_post_imm_ld4r(ptr %A, ptr %ptr) nounwind {
-; CHECK-LABEL: test_v1f64_post_imm_ld4r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld4r.1d { v0, v1, v2, v3 }, [x0], #32
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_imm_ld4r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld4r.1d { v0, v1, v2, v3 }, [x0], #32
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_imm_ld4r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6445,12 +6442,12 @@ define { <1 x double>, <1 x double>, <1 x double>, <1 x double> } @test_v1f64_po
 }
 
 define { <1 x double>, <1 x double>, <1 x double>, <1 x double> } @test_v1f64_post_reg_ld4r(ptr %A, ptr %ptr, i64 %inc) nounwind {
-; CHECK-LABEL: test_v1f64_post_reg_ld4r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld4r.1d { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_reg_ld4r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld4r.1d { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_reg_ld4r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6468,13 +6465,13 @@ declare { <1 x double>, <1 x double>, <1 x double>, <1 x double> } @llvm.aarch64
 
 
 define { <16 x i8>, <16 x i8> } @test_v16i8_post_imm_ld2lane(ptr %A, ptr %ptr, <16 x i8> %B, <16 x i8> %C) nounwind {
-; CHECK-LABEL: test_v16i8_post_imm_ld2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ld2.b { v0, v1 }[0], [x0], #2
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_imm_ld2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ld2.b { v0, v1 }[0], [x0], #2
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_imm_ld2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6491,13 +6488,13 @@ define { <16 x i8>, <16 x i8> } @test_v16i8_post_imm_ld2lane(ptr %A, ptr %ptr, <
 }
 
 define { <16 x i8>, <16 x i8> } @test_v16i8_post_reg_ld2lane(ptr %A, ptr %ptr, i64 %inc, <16 x i8> %B, <16 x i8> %C) nounwind {
-; CHECK-LABEL: test_v16i8_post_reg_ld2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ld2.b { v0, v1 }[0], [x0], x2
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_reg_ld2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ld2.b { v0, v1 }[0], [x0], x2
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_reg_ld2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6517,13 +6514,13 @@ declare { <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld2lane.v16i8.p0(<16 x i8>,
 
 
 define { <8 x i8>, <8 x i8> } @test_v8i8_post_imm_ld2lane(ptr %A, ptr %ptr, <8 x i8> %B, <8 x i8> %C) nounwind {
-; CHECK-LABEL: test_v8i8_post_imm_ld2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ld2.b { v0, v1 }[0], [x0], #2
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_imm_ld2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ld2.b { v0, v1 }[0], [x0], #2
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_imm_ld2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6540,13 +6537,13 @@ define { <8 x i8>, <8 x i8> } @test_v8i8_post_imm_ld2lane(ptr %A, ptr %ptr, <8 x
 }
 
 define { <8 x i8>, <8 x i8> } @test_v8i8_post_reg_ld2lane(ptr %A, ptr %ptr, i64 %inc, <8 x i8> %B, <8 x i8> %C) nounwind {
-; CHECK-LABEL: test_v8i8_post_reg_ld2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ld2.b { v0, v1 }[0], [x0], x2
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_reg_ld2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ld2.b { v0, v1 }[0], [x0], x2
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_reg_ld2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6566,13 +6563,13 @@ declare { <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld2lane.v8i8.p0(<8 x i8>, <8 x
 
 
 define { <8 x i16>, <8 x i16> } @test_v8i16_post_imm_ld2lane(ptr %A, ptr %ptr, <8 x i16> %B, <8 x i16> %C) nounwind {
-; CHECK-LABEL: test_v8i16_post_imm_ld2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ld2.h { v0, v1 }[0], [x0], #4
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_imm_ld2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ld2.h { v0, v1 }[0], [x0], #4
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_imm_ld2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6589,14 +6586,14 @@ define { <8 x i16>, <8 x i16> } @test_v8i16_post_imm_ld2lane(ptr %A, ptr %ptr, <
 }
 
 define { <8 x i16>, <8 x i16> } @test_v8i16_post_reg_ld2lane(ptr %A, ptr %ptr, i64 %inc, <8 x i16> %B, <8 x i16> %C) nounwind {
-; CHECK-LABEL: test_v8i16_post_reg_ld2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ld2.h { v0, v1 }[0], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_reg_ld2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ld2.h { v0, v1 }[0], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_reg_ld2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6616,13 +6613,13 @@ declare { <8 x i16>, <8 x i16> } @llvm.aarch64.neon.ld2lane.v8i16.p0(<8 x i16>,
 
 
 define { <4 x i16>, <4 x i16> } @test_v4i16_post_imm_ld2lane(ptr %A, ptr %ptr, <4 x i16> %B, <4 x i16> %C) nounwind {
-; CHECK-LABEL: test_v4i16_post_imm_ld2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ld2.h { v0, v1 }[0], [x0], #4
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_imm_ld2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ld2.h { v0, v1 }[0], [x0], #4
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_imm_ld2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6639,14 +6636,14 @@ define { <4 x i16>, <4 x i16> } @test_v4i16_post_imm_ld2lane(ptr %A, ptr %ptr, <
 }
 
 define { <4 x i16>, <4 x i16> } @test_v4i16_post_reg_ld2lane(ptr %A, ptr %ptr, i64 %inc, <4 x i16> %B, <4 x i16> %C) nounwind {
-; CHECK-LABEL: test_v4i16_post_reg_ld2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ld2.h { v0, v1 }[0], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_reg_ld2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ld2.h { v0, v1 }[0], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_reg_ld2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6666,13 +6663,13 @@ declare { <4 x i16>, <4 x i16> } @llvm.aarch64.neon.ld2lane.v4i16.p0(<4 x i16>,
 
 
 define { <4 x i32>, <4 x i32> } @test_v4i32_post_imm_ld2lane(ptr %A, ptr %ptr, <4 x i32> %B, <4 x i32> %C) nounwind {
-; CHECK-LABEL: test_v4i32_post_imm_ld2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ld2.s { v0, v1 }[0], [x0], #8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_imm_ld2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ld2.s { v0, v1 }[0], [x0], #8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_imm_ld2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6689,14 +6686,14 @@ define { <4 x i32>, <4 x i32> } @test_v4i32_post_imm_ld2lane(ptr %A, ptr %ptr, <
 }
 
 define { <4 x i32>, <4 x i32> } @test_v4i32_post_reg_ld2lane(ptr %A, ptr %ptr, i64 %inc, <4 x i32> %B, <4 x i32> %C) nounwind {
-; CHECK-LABEL: test_v4i32_post_reg_ld2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ld2.s { v0, v1 }[0], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_reg_ld2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ld2.s { v0, v1 }[0], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_reg_ld2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6716,13 +6713,13 @@ declare { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld2lane.v4i32.p0(<4 x i32>,
 
 
 define { <2 x i32>, <2 x i32> } @test_v2i32_post_imm_ld2lane(ptr %A, ptr %ptr, <2 x i32> %B, <2 x i32> %C) nounwind {
-; CHECK-LABEL: test_v2i32_post_imm_ld2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ld2.s { v0, v1 }[0], [x0], #8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_imm_ld2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ld2.s { v0, v1 }[0], [x0], #8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_imm_ld2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6739,14 +6736,14 @@ define { <2 x i32>, <2 x i32> } @test_v2i32_post_imm_ld2lane(ptr %A, ptr %ptr, <
 }
 
 define { <2 x i32>, <2 x i32> } @test_v2i32_post_reg_ld2lane(ptr %A, ptr %ptr, i64 %inc, <2 x i32> %B, <2 x i32> %C) nounwind {
-; CHECK-LABEL: test_v2i32_post_reg_ld2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ld2.s { v0, v1 }[0], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_reg_ld2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ld2.s { v0, v1 }[0], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_reg_ld2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6766,13 +6763,13 @@ declare { <2 x i32>, <2 x i32> } @llvm.aarch64.neon.ld2lane.v2i32.p0(<2 x i32>,
 
 
 define { <2 x i64>, <2 x i64> } @test_v2i64_post_imm_ld2lane(ptr %A, ptr %ptr, <2 x i64> %B, <2 x i64> %C) nounwind {
-; CHECK-LABEL: test_v2i64_post_imm_ld2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ld2.d { v0, v1 }[0], [x0], #16
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_imm_ld2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ld2.d { v0, v1 }[0], [x0], #16
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_imm_ld2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6789,14 +6786,14 @@ define { <2 x i64>, <2 x i64> } @test_v2i64_post_imm_ld2lane(ptr %A, ptr %ptr, <
 }
 
 define { <2 x i64>, <2 x i64> } @test_v2i64_post_reg_ld2lane(ptr %A, ptr %ptr, i64 %inc, <2 x i64> %B, <2 x i64> %C) nounwind {
-; CHECK-LABEL: test_v2i64_post_reg_ld2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ld2.d { v0, v1 }[0], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_reg_ld2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ld2.d { v0, v1 }[0], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_reg_ld2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6816,13 +6813,13 @@ declare { <2 x i64>, <2 x i64> } @llvm.aarch64.neon.ld2lane.v2i64.p0(<2 x i64>,
 
 
 define { <1 x i64>, <1 x i64> } @test_v1i64_post_imm_ld2lane(ptr %A, ptr %ptr, <1 x i64> %B, <1 x i64> %C) nounwind {
-; CHECK-LABEL: test_v1i64_post_imm_ld2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ld2.d { v0, v1 }[0], [x0], #16
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_imm_ld2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ld2.d { v0, v1 }[0], [x0], #16
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_imm_ld2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6839,14 +6836,14 @@ define { <1 x i64>, <1 x i64> } @test_v1i64_post_imm_ld2lane(ptr %A, ptr %ptr, <
 }
 
 define { <1 x i64>, <1 x i64> } @test_v1i64_post_reg_ld2lane(ptr %A, ptr %ptr, i64 %inc, <1 x i64> %B, <1 x i64> %C) nounwind {
-; CHECK-LABEL: test_v1i64_post_reg_ld2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ld2.d { v0, v1 }[0], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_reg_ld2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ld2.d { v0, v1 }[0], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_reg_ld2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6866,13 +6863,13 @@ declare { <1 x i64>, <1 x i64> } @llvm.aarch64.neon.ld2lane.v1i64.p0(<1 x i64>,
 
 
 define { <4 x float>, <4 x float> } @test_v4f32_post_imm_ld2lane(ptr %A, ptr %ptr, <4 x float> %B, <4 x float> %C) nounwind {
-; CHECK-LABEL: test_v4f32_post_imm_ld2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ld2.s { v0, v1 }[0], [x0], #8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_imm_ld2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ld2.s { v0, v1 }[0], [x0], #8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_imm_ld2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6889,14 +6886,14 @@ define { <4 x float>, <4 x float> } @test_v4f32_post_imm_ld2lane(ptr %A, ptr %pt
 }
 
 define { <4 x float>, <4 x float> } @test_v4f32_post_reg_ld2lane(ptr %A, ptr %ptr, i64 %inc, <4 x float> %B, <4 x float> %C) nounwind {
-; CHECK-LABEL: test_v4f32_post_reg_ld2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ld2.s { v0, v1 }[0], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_reg_ld2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ld2.s { v0, v1 }[0], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_reg_ld2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6916,13 +6913,13 @@ declare { <4 x float>, <4 x float> } @llvm.aarch64.neon.ld2lane.v4f32.p0(<4 x fl
 
 
 define { <2 x float>, <2 x float> } @test_v2f32_post_imm_ld2lane(ptr %A, ptr %ptr, <2 x float> %B, <2 x float> %C) nounwind {
-; CHECK-LABEL: test_v2f32_post_imm_ld2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ld2.s { v0, v1 }[0], [x0], #8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_imm_ld2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ld2.s { v0, v1 }[0], [x0], #8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_imm_ld2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6939,14 +6936,14 @@ define { <2 x float>, <2 x float> } @test_v2f32_post_imm_ld2lane(ptr %A, ptr %pt
 }
 
 define { <2 x float>, <2 x float> } @test_v2f32_post_reg_ld2lane(ptr %A, ptr %ptr, i64 %inc, <2 x float> %B, <2 x float> %C) nounwind {
-; CHECK-LABEL: test_v2f32_post_reg_ld2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ld2.s { v0, v1 }[0], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_reg_ld2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ld2.s { v0, v1 }[0], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_reg_ld2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6966,13 +6963,13 @@ declare { <2 x float>, <2 x float> } @llvm.aarch64.neon.ld2lane.v2f32.p0(<2 x fl
 
 
 define { <2 x double>, <2 x double> } @test_v2f64_post_imm_ld2lane(ptr %A, ptr %ptr, <2 x double> %B, <2 x double> %C) nounwind {
-; CHECK-LABEL: test_v2f64_post_imm_ld2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ld2.d { v0, v1 }[0], [x0], #16
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_imm_ld2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ld2.d { v0, v1 }[0], [x0], #16
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_imm_ld2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -6989,14 +6986,14 @@ define { <2 x double>, <2 x double> } @test_v2f64_post_imm_ld2lane(ptr %A, ptr %
 }
 
 define { <2 x double>, <2 x double> } @test_v2f64_post_reg_ld2lane(ptr %A, ptr %ptr, i64 %inc, <2 x double> %B, <2 x double> %C) nounwind {
-; CHECK-LABEL: test_v2f64_post_reg_ld2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ld2.d { v0, v1 }[0], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_reg_ld2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ld2.d { v0, v1 }[0], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_reg_ld2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -7016,13 +7013,13 @@ declare { <2 x double>, <2 x double> } @llvm.aarch64.neon.ld2lane.v2f64.p0(<2 x
 
 
 define { <1 x double>, <1 x double> } @test_v1f64_post_imm_ld2lane(ptr %A, ptr %ptr, <1 x double> %B, <1 x double> %C) nounwind {
-; CHECK-LABEL: test_v1f64_post_imm_ld2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ld2.d { v0, v1 }[0], [x0], #16
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_imm_ld2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ld2.d { v0, v1 }[0], [x0], #16
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_imm_ld2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -7039,14 +7036,14 @@ define { <1 x double>, <1 x double> } @test_v1f64_post_imm_ld2lane(ptr %A, ptr %
 }
 
 define { <1 x double>, <1 x double> } @test_v1f64_post_reg_ld2lane(ptr %A, ptr %ptr, i64 %inc, <1 x double> %B, <1 x double> %C) nounwind {
-; CHECK-LABEL: test_v1f64_post_reg_ld2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ld2.d { v0, v1 }[0], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_reg_ld2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ld2.d { v0, v1 }[0], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_reg_ld2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -7066,14 +7063,14 @@ declare { <1 x double>, <1 x double> } @llvm.aarch64.neon.ld2lane.v1f64.p0(<1 x
 
 
 define { <16 x i8>, <16 x i8>, <16 x i8> } @test_v16i8_post_imm_ld3lane(ptr %A, ptr %ptr, <16 x i8> %B, <16 x i8> %C, <16 x i8> %D) nounwind {
-; CHECK-LABEL: test_v16i8_post_imm_ld3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ld3.b { v0, v1, v2 }[0], [x0], #3
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_imm_ld3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ld3.b { v0, v1, v2 }[0], [x0], #3
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_imm_ld3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -7091,14 +7088,14 @@ define { <16 x i8>, <16 x i8>, <16 x i8> } @test_v16i8_post_imm_ld3lane(ptr %A,
 }
 
 define { <16 x i8>, <16 x i8>, <16 x i8> } @test_v16i8_post_reg_ld3lane(ptr %A, ptr %ptr, i64 %inc, <16 x i8> %B, <16 x i8> %C, <16 x i8> %D) nounwind {
-; CHECK-LABEL: test_v16i8_post_reg_ld3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ld3.b { v0, v1, v2 }[0], [x0], x2
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_reg_ld3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ld3.b { v0, v1, v2 }[0], [x0], x2
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_reg_ld3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -7119,14 +7116,14 @@ declare { <16 x i8>, <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld3lane.v16i8.p0(
 
 
 define { <8 x i8>, <8 x i8>, <8 x i8> } @test_v8i8_post_imm_ld3lane(ptr %A, ptr %ptr, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D) nounwind {
-; CHECK-LABEL: test_v8i8_post_imm_ld3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ld3.b { v0, v1, v2 }[0], [x0], #3
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_imm_ld3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ld3.b { v0, v1, v2 }[0], [x0], #3
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_imm_ld3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -7144,14 +7141,14 @@ define { <8 x i8>, <8 x i8>, <8 x i8> } @test_v8i8_post_imm_ld3lane(ptr %A, ptr
 }
 
 define { <8 x i8>, <8 x i8>, <8 x i8> } @test_v8i8_post_reg_ld3lane(ptr %A, ptr %ptr, i64 %inc, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D) nounwind {
-; CHECK-LABEL: test_v8i8_post_reg_ld3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ld3.b { v0, v1, v2 }[0], [x0], x2
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_reg_ld3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ld3.b { v0, v1, v2 }[0], [x0], x2
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_reg_ld3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -7172,14 +7169,14 @@ declare { <8 x i8>, <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld3lane.v8i8.p0(<8 x
 
 
 define { <8 x i16>, <8 x i16>, <8 x i16> } @test_v8i16_post_imm_ld3lane(ptr %A, ptr %ptr, <8 x i16> %B, <8 x i16> %C, <8 x i16> %D) nounwind {
-; CHECK-LABEL: test_v8i16_post_imm_ld3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ld3.h { v0, v1, v2 }[0], [x0], #6
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_imm_ld3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ld3.h { v0, v1, v2 }[0], [x0], #6
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_imm_ld3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -7197,15 +7194,15 @@ define { <8 x i16>, <8 x i16>, <8 x i16> } @test_v8i16_post_imm_ld3lane(ptr %A,
 }
 
 define { <8 x i16>, <8 x i16>, <8 x i16> } @test_v8i16_post_reg_ld3lane(ptr %A, ptr %ptr, i64 %inc, <8 x i16> %B, <8 x i16> %C, <8 x i16> %D) nounwind {
-; CHECK-LABEL: test_v8i16_post_reg_ld3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ld3.h { v0, v1, v2 }[0], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_reg_ld3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ld3.h { v0, v1, v2 }[0], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_reg_ld3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -7226,14 +7223,14 @@ declare { <8 x i16>, <8 x i16>, <8 x i16> } @llvm.aarch64.neon.ld3lane.v8i16.p0(
 
 
 define { <4 x i16>, <4 x i16>, <4 x i16> } @test_v4i16_post_imm_ld3lane(ptr %A, ptr %ptr, <4 x i16> %B, <4 x i16> %C, <4 x i16> %D) nounwind {
-; CHECK-LABEL: test_v4i16_post_imm_ld3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ld3.h { v0, v1, v2 }[0], [x0], #6
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_imm_ld3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ld3.h { v0, v1, v2 }[0], [x0], #6
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_imm_ld3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -7251,15 +7248,15 @@ define { <4 x i16>, <4 x i16>, <4 x i16> } @test_v4i16_post_imm_ld3lane(ptr %A,
 }
 
 define { <4 x i16>, <4 x i16>, <4 x i16> } @test_v4i16_post_reg_ld3lane(ptr %A, ptr %ptr, i64 %inc, <4 x i16> %B, <4 x i16> %C, <4 x i16> %D) nounwind {
-; CHECK-LABEL: test_v4i16_post_reg_ld3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ld3.h { v0, v1, v2 }[0], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_reg_ld3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ld3.h { v0, v1, v2 }[0], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_reg_ld3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -7280,14 +7277,14 @@ declare { <4 x i16>, <4 x i16>, <4 x i16> } @llvm.aarch64.neon.ld3lane.v4i16.p0(
 
 
 define { <4 x i32>, <4 x i32>, <4 x i32> } @test_v4i32_post_imm_ld3lane(ptr %A, ptr %ptr, <4 x i32> %B, <4 x i32> %C, <4 x i32> %D) nounwind {
-; CHECK-LABEL: test_v4i32_post_imm_ld3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ld3.s { v0, v1, v2 }[0], [x0], #12
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_imm_ld3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ld3.s { v0, v1, v2 }[0], [x0], #12
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_imm_ld3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -7305,15 +7302,15 @@ define { <4 x i32>, <4 x i32>, <4 x i32> } @test_v4i32_post_imm_ld3lane(ptr %A,
 }
 
 define { <4 x i32>, <4 x i32>, <4 x i32> } @test_v4i32_post_reg_ld3lane(ptr %A, ptr %ptr, i64 %inc, <4 x i32> %B, <4 x i32> %C, <4 x i32> %D) nounwind {
-; CHECK-LABEL: test_v4i32_post_reg_ld3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ld3.s { v0, v1, v2 }[0], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_reg_ld3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ld3.s { v0, v1, v2 }[0], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_reg_ld3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -7334,14 +7331,14 @@ declare { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld3lane.v4i32.p0(
 
 
 define { <2 x i32>, <2 x i32>, <2 x i32> } @test_v2i32_post_imm_ld3lane(ptr %A, ptr %ptr, <2 x i32> %B, <2 x i32> %C, <2 x i32> %D) nounwind {
-; CHECK-LABEL: test_v2i32_post_imm_ld3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ld3.s { v0, v1, v2 }[0], [x0], #12
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_imm_ld3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ld3.s { v0, v1, v2 }[0], [x0], #12
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_imm_ld3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -7359,15 +7356,15 @@ define { <2 x i32>, <2 x i32>, <2 x i32> } @test_v2i32_post_imm_ld3lane(ptr %A,
 }
 
 define { <2 x i32>, <2 x i32>, <2 x i32> } @test_v2i32_post_reg_ld3lane(ptr %A, ptr %ptr, i64 %inc, <2 x i32> %B, <2 x i32> %C, <2 x i32> %D) nounwind {
-; CHECK-LABEL: test_v2i32_post_reg_ld3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ld3.s { v0, v1, v2 }[0], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_reg_ld3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ld3.s { v0, v1, v2 }[0], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_reg_ld3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -7388,14 +7385,14 @@ declare { <2 x i32>, <2 x i32>, <2 x i32> } @llvm.aarch64.neon.ld3lane.v2i32.p0(
 
 
 define { <2 x i64>, <2 x i64>, <2 x i64> } @test_v2i64_post_imm_ld3lane(ptr %A, ptr %ptr, <2 x i64> %B, <2 x i64> %C, <2 x i64> %D) nounwind {
-; CHECK-LABEL: test_v2i64_post_imm_ld3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ld3.d { v0, v1, v2 }[0], [x0], #24
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_imm_ld3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ld3.d { v0, v1, v2 }[0], [x0], #24
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_imm_ld3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -7413,15 +7410,15 @@ define { <2 x i64>, <2 x i64>, <2 x i64> } @test_v2i64_post_imm_ld3lane(ptr %A,
 }
 
 define { <2 x i64>, <2 x i64>, <2 x i64> } @test_v2i64_post_reg_ld3lane(ptr %A, ptr %ptr, i64 %inc, <2 x i64> %B, <2 x i64> %C, <2 x i64> %D) nounwind {
-; CHECK-LABEL: test_v2i64_post_reg_ld3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ld3.d { v0, v1, v2 }[0], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_reg_ld3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ld3.d { v0, v1, v2 }[0], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_reg_ld3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -7442,14 +7439,14 @@ declare { <2 x i64>, <2 x i64>, <2 x i64> } @llvm.aarch64.neon.ld3lane.v2i64.p0(
 
 
 define { <1 x i64>, <1 x i64>, <1 x i64> } @test_v1i64_post_imm_ld3lane(ptr %A, ptr %ptr, <1 x i64> %B, <1 x i64> %C, <1 x i64> %D) nounwind {
-; CHECK-LABEL: test_v1i64_post_imm_ld3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ld3.d { v0, v1, v2 }[0], [x0], #24
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_imm_ld3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ld3.d { v0, v1, v2 }[0], [x0], #24
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_imm_ld3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -7467,15 +7464,15 @@ define { <1 x i64>, <1 x i64>, <1 x i64> } @test_v1i64_post_imm_ld3lane(ptr %A,
 }
 
 define { <1 x i64>, <1 x i64>, <1 x i64> } @test_v1i64_post_reg_ld3lane(ptr %A, ptr %ptr, i64 %inc, <1 x i64> %B, <1 x i64> %C, <1 x i64> %D) nounwind {
-; CHECK-LABEL: test_v1i64_post_reg_ld3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ld3.d { v0, v1, v2 }[0], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_reg_ld3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ld3.d { v0, v1, v2 }[0], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_reg_ld3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -7496,14 +7493,14 @@ declare { <1 x i64>, <1 x i64>, <1 x i64> } @llvm.aarch64.neon.ld3lane.v1i64.p0(
 
 
 define { <4 x float>, <4 x float>, <4 x float> } @test_v4f32_post_imm_ld3lane(ptr %A, ptr %ptr, <4 x float> %B, <4 x float> %C, <4 x float> %D) nounwind {
-; CHECK-LABEL: test_v4f32_post_imm_ld3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ld3.s { v0, v1, v2 }[0], [x0], #12
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_imm_ld3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ld3.s { v0, v1, v2 }[0], [x0], #12
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_imm_ld3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -7521,15 +7518,15 @@ define { <4 x float>, <4 x float>, <4 x float> } @test_v4f32_post_imm_ld3lane(pt
 }
 
 define { <4 x float>, <4 x float>, <4 x float> } @test_v4f32_post_reg_ld3lane(ptr %A, ptr %ptr, i64 %inc, <4 x float> %B, <4 x float> %C, <4 x float> %D) nounwind {
-; CHECK-LABEL: test_v4f32_post_reg_ld3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ld3.s { v0, v1, v2 }[0], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_reg_ld3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ld3.s { v0, v1, v2 }[0], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_reg_ld3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -7550,14 +7547,14 @@ declare { <4 x float>, <4 x float>, <4 x float> } @llvm.aarch64.neon.ld3lane.v4f
 
 
 define { <2 x float>, <2 x float>, <2 x float> } @test_v2f32_post_imm_ld3lane(ptr %A, ptr %ptr, <2 x float> %B, <2 x float> %C, <2 x float> %D) nounwind {
-; CHECK-LABEL: test_v2f32_post_imm_ld3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ld3.s { v0, v1, v2 }[0], [x0], #12
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_imm_ld3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ld3.s { v0, v1, v2 }[0], [x0], #12
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_imm_ld3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -7575,15 +7572,15 @@ define { <2 x float>, <2 x float>, <2 x float> } @test_v2f32_post_imm_ld3lane(pt
 }
 
 define { <2 x float>, <2 x float>, <2 x float> } @test_v2f32_post_reg_ld3lane(ptr %A, ptr %ptr, i64 %inc, <2 x float> %B, <2 x float> %C, <2 x float> %D) nounwind {
-; CHECK-LABEL: test_v2f32_post_reg_ld3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ld3.s { v0, v1, v2 }[0], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_reg_ld3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ld3.s { v0, v1, v2 }[0], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_reg_ld3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -7604,14 +7601,14 @@ declare { <2 x float>, <2 x float>, <2 x float> } @llvm.aarch64.neon.ld3lane.v2f
 
 
 define { <2 x double>, <2 x double>, <2 x double> } @test_v2f64_post_imm_ld3lane(ptr %A, ptr %ptr, <2 x double> %B, <2 x double> %C, <2 x double> %D) nounwind {
-; CHECK-LABEL: test_v2f64_post_imm_ld3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ld3.d { v0, v1, v2 }[0], [x0], #24
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_imm_ld3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ld3.d { v0, v1, v2 }[0], [x0], #24
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_imm_ld3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -7629,15 +7626,15 @@ define { <2 x double>, <2 x double>, <2 x double> } @test_v2f64_post_imm_ld3lane
 }
 
 define { <2 x double>, <2 x double>, <2 x double> } @test_v2f64_post_reg_ld3lane(ptr %A, ptr %ptr, i64 %inc, <2 x double> %B, <2 x double> %C, <2 x double> %D) nounwind {
-; CHECK-LABEL: test_v2f64_post_reg_ld3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ld3.d { v0, v1, v2 }[0], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_reg_ld3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ld3.d { v0, v1, v2 }[0], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_reg_ld3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -7658,14 +7655,14 @@ declare { <2 x double>, <2 x double>, <2 x double> } @llvm.aarch64.neon.ld3lane.
 
 
 define { <1 x double>, <1 x double>, <1 x double> } @test_v1f64_post_imm_ld3lane(ptr %A, ptr %ptr, <1 x double> %B, <1 x double> %C, <1 x double> %D) nounwind {
-; CHECK-LABEL: test_v1f64_post_imm_ld3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ld3.d { v0, v1, v2 }[0], [x0], #24
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_imm_ld3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ld3.d { v0, v1, v2 }[0], [x0], #24
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_imm_ld3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -7683,15 +7680,15 @@ define { <1 x double>, <1 x double>, <1 x double> } @test_v1f64_post_imm_ld3lane
 }
 
 define { <1 x double>, <1 x double>, <1 x double> } @test_v1f64_post_reg_ld3lane(ptr %A, ptr %ptr, i64 %inc, <1 x double> %B, <1 x double> %C, <1 x double> %D) nounwind {
-; CHECK-LABEL: test_v1f64_post_reg_ld3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ld3.d { v0, v1, v2 }[0], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_reg_ld3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ld3.d { v0, v1, v2 }[0], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_reg_ld3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -7712,15 +7709,15 @@ declare { <1 x double>, <1 x double>, <1 x double> } @llvm.aarch64.neon.ld3lane.
 
 
 define { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @test_v16i8_post_imm_ld4lane(ptr %A, ptr %ptr, <16 x i8> %B, <16 x i8> %C, <16 x i8> %D, <16 x i8> %E) nounwind {
-; CHECK-LABEL: test_v16i8_post_imm_ld4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ld4.b { v0, v1, v2, v3 }[0], [x0], #4
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_imm_ld4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ld4.b { v0, v1, v2, v3 }[0], [x0], #4
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_imm_ld4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -7739,15 +7736,15 @@ define { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @test_v16i8_post_imm_ld4la
 }
 
 define { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @test_v16i8_post_reg_ld4lane(ptr %A, ptr %ptr, i64 %inc, <16 x i8> %B, <16 x i8> %C, <16 x i8> %D, <16 x i8> %E) nounwind {
-; CHECK-LABEL: test_v16i8_post_reg_ld4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ld4.b { v0, v1, v2, v3 }[0], [x0], x2
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_reg_ld4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ld4.b { v0, v1, v2, v3 }[0], [x0], x2
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_reg_ld4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -7769,15 +7766,15 @@ declare { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld4lan
 
 
 define { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } @test_v8i8_post_imm_ld4lane(ptr %A, ptr %ptr, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D, <8 x i8> %E) nounwind {
-; CHECK-LABEL: test_v8i8_post_imm_ld4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ld4.b { v0, v1, v2, v3 }[0], [x0], #4
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_imm_ld4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ld4.b { v0, v1, v2, v3 }[0], [x0], #4
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_imm_ld4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -7796,15 +7793,15 @@ define { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } @test_v8i8_post_imm_ld4lane(pt
 }
 
 define { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } @test_v8i8_post_reg_ld4lane(ptr %A, ptr %ptr, i64 %inc, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D, <8 x i8> %E) nounwind {
-; CHECK-LABEL: test_v8i8_post_reg_ld4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ld4.b { v0, v1, v2, v3 }[0], [x0], x2
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_reg_ld4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ld4.b { v0, v1, v2, v3 }[0], [x0], x2
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_reg_ld4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -7826,15 +7823,15 @@ declare { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld4lane.v8
 
 
 define { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } @test_v8i16_post_imm_ld4lane(ptr %A, ptr %ptr, <8 x i16> %B, <8 x i16> %C, <8 x i16> %D, <8 x i16> %E) nounwind {
-; CHECK-LABEL: test_v8i16_post_imm_ld4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ld4.h { v0, v1, v2, v3 }[0], [x0], #8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_imm_ld4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ld4.h { v0, v1, v2, v3 }[0], [x0], #8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_imm_ld4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -7853,16 +7850,16 @@ define { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } @test_v8i16_post_imm_ld4la
 }
 
 define { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } @test_v8i16_post_reg_ld4lane(ptr %A, ptr %ptr, i64 %inc, <8 x i16> %B, <8 x i16> %C, <8 x i16> %D, <8 x i16> %E) nounwind {
-; CHECK-LABEL: test_v8i16_post_reg_ld4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ld4.h { v0, v1, v2, v3 }[0], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_reg_ld4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ld4.h { v0, v1, v2, v3 }[0], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_reg_ld4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -7884,15 +7881,15 @@ declare { <8 x i16>, <8 x i16>, <8 x i16>, <8 x i16> } @llvm.aarch64.neon.ld4lan
 
 
 define { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } @test_v4i16_post_imm_ld4lane(ptr %A, ptr %ptr, <4 x i16> %B, <4 x i16> %C, <4 x i16> %D, <4 x i16> %E) nounwind {
-; CHECK-LABEL: test_v4i16_post_imm_ld4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ld4.h { v0, v1, v2, v3 }[0], [x0], #8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_imm_ld4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ld4.h { v0, v1, v2, v3 }[0], [x0], #8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_imm_ld4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -7911,16 +7908,16 @@ define { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } @test_v4i16_post_imm_ld4la
 }
 
 define { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } @test_v4i16_post_reg_ld4lane(ptr %A, ptr %ptr, i64 %inc, <4 x i16> %B, <4 x i16> %C, <4 x i16> %D, <4 x i16> %E) nounwind {
-; CHECK-LABEL: test_v4i16_post_reg_ld4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ld4.h { v0, v1, v2, v3 }[0], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_reg_ld4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ld4.h { v0, v1, v2, v3 }[0], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_reg_ld4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -7942,15 +7939,15 @@ declare { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } @llvm.aarch64.neon.ld4lan
 
 
 define { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @test_v4i32_post_imm_ld4lane(ptr %A, ptr %ptr, <4 x i32> %B, <4 x i32> %C, <4 x i32> %D, <4 x i32> %E) nounwind {
-; CHECK-LABEL: test_v4i32_post_imm_ld4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ld4.s { v0, v1, v2, v3 }[0], [x0], #16
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_imm_ld4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ld4.s { v0, v1, v2, v3 }[0], [x0], #16
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_imm_ld4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -7969,16 +7966,16 @@ define { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @test_v4i32_post_imm_ld4la
 }
 
 define { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @test_v4i32_post_reg_ld4lane(ptr %A, ptr %ptr, i64 %inc, <4 x i32> %B, <4 x i32> %C, <4 x i32> %D, <4 x i32> %E) nounwind {
-; CHECK-LABEL: test_v4i32_post_reg_ld4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ld4.s { v0, v1, v2, v3 }[0], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_reg_ld4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ld4.s { v0, v1, v2, v3 }[0], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_reg_ld4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8000,15 +7997,15 @@ declare { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld4lan
 
 
 define { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } @test_v2i32_post_imm_ld4lane(ptr %A, ptr %ptr, <2 x i32> %B, <2 x i32> %C, <2 x i32> %D, <2 x i32> %E) nounwind {
-; CHECK-LABEL: test_v2i32_post_imm_ld4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ld4.s { v0, v1, v2, v3 }[0], [x0], #16
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_imm_ld4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ld4.s { v0, v1, v2, v3 }[0], [x0], #16
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_imm_ld4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8027,16 +8024,16 @@ define { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } @test_v2i32_post_imm_ld4la
 }
 
 define { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } @test_v2i32_post_reg_ld4lane(ptr %A, ptr %ptr, i64 %inc, <2 x i32> %B, <2 x i32> %C, <2 x i32> %D, <2 x i32> %E) nounwind {
-; CHECK-LABEL: test_v2i32_post_reg_ld4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ld4.s { v0, v1, v2, v3 }[0], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_reg_ld4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ld4.s { v0, v1, v2, v3 }[0], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_reg_ld4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8058,15 +8055,15 @@ declare { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } @llvm.aarch64.neon.ld4lan
 
 
 define { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @test_v2i64_post_imm_ld4lane(ptr %A, ptr %ptr, <2 x i64> %B, <2 x i64> %C, <2 x i64> %D, <2 x i64> %E) nounwind {
-; CHECK-LABEL: test_v2i64_post_imm_ld4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ld4.d { v0, v1, v2, v3 }[0], [x0], #32
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_imm_ld4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ld4.d { v0, v1, v2, v3 }[0], [x0], #32
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_imm_ld4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8085,16 +8082,16 @@ define { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @test_v2i64_post_imm_ld4la
 }
 
 define { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @test_v2i64_post_reg_ld4lane(ptr %A, ptr %ptr, i64 %inc, <2 x i64> %B, <2 x i64> %C, <2 x i64> %D, <2 x i64> %E) nounwind {
-; CHECK-LABEL: test_v2i64_post_reg_ld4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ld4.d { v0, v1, v2, v3 }[0], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_reg_ld4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ld4.d { v0, v1, v2, v3 }[0], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_reg_ld4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8116,15 +8113,15 @@ declare { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @llvm.aarch64.neon.ld4lan
 
 
 define { <1 x i64>, <1 x i64>, <1 x i64>, <1 x i64> } @test_v1i64_post_imm_ld4lane(ptr %A, ptr %ptr, <1 x i64> %B, <1 x i64> %C, <1 x i64> %D, <1 x i64> %E) nounwind {
-; CHECK-LABEL: test_v1i64_post_imm_ld4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ld4.d { v0, v1, v2, v3 }[0], [x0], #32
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_imm_ld4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ld4.d { v0, v1, v2, v3 }[0], [x0], #32
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_imm_ld4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8143,16 +8140,16 @@ define { <1 x i64>, <1 x i64>, <1 x i64>, <1 x i64> } @test_v1i64_post_imm_ld4la
 }
 
 define { <1 x i64>, <1 x i64>, <1 x i64>, <1 x i64> } @test_v1i64_post_reg_ld4lane(ptr %A, ptr %ptr, i64 %inc, <1 x i64> %B, <1 x i64> %C, <1 x i64> %D, <1 x i64> %E) nounwind {
-; CHECK-LABEL: test_v1i64_post_reg_ld4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ld4.d { v0, v1, v2, v3 }[0], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_reg_ld4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ld4.d { v0, v1, v2, v3 }[0], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_reg_ld4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8174,15 +8171,15 @@ declare { <1 x i64>, <1 x i64>, <1 x i64>, <1 x i64> } @llvm.aarch64.neon.ld4lan
 
 
 define { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @test_v4f32_post_imm_ld4lane(ptr %A, ptr %ptr, <4 x float> %B, <4 x float> %C, <4 x float> %D, <4 x float> %E) nounwind {
-; CHECK-LABEL: test_v4f32_post_imm_ld4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ld4.s { v0, v1, v2, v3 }[0], [x0], #16
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_imm_ld4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ld4.s { v0, v1, v2, v3 }[0], [x0], #16
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_imm_ld4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8201,16 +8198,16 @@ define { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @test_v4f32_post_i
 }
 
 define { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @test_v4f32_post_reg_ld4lane(ptr %A, ptr %ptr, i64 %inc, <4 x float> %B, <4 x float> %C, <4 x float> %D, <4 x float> %E) nounwind {
-; CHECK-LABEL: test_v4f32_post_reg_ld4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ld4.s { v0, v1, v2, v3 }[0], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_reg_ld4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ld4.s { v0, v1, v2, v3 }[0], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_reg_ld4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8232,15 +8229,15 @@ declare { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @llvm.aarch64.neo
 
 
 define { <2 x float>, <2 x float>, <2 x float>, <2 x float> } @test_v2f32_post_imm_ld4lane(ptr %A, ptr %ptr, <2 x float> %B, <2 x float> %C, <2 x float> %D, <2 x float> %E) nounwind {
-; CHECK-LABEL: test_v2f32_post_imm_ld4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ld4.s { v0, v1, v2, v3 }[0], [x0], #16
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_imm_ld4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ld4.s { v0, v1, v2, v3 }[0], [x0], #16
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_imm_ld4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8259,16 +8256,16 @@ define { <2 x float>, <2 x float>, <2 x float>, <2 x float> } @test_v2f32_post_i
 }
 
 define { <2 x float>, <2 x float>, <2 x float>, <2 x float> } @test_v2f32_post_reg_ld4lane(ptr %A, ptr %ptr, i64 %inc, <2 x float> %B, <2 x float> %C, <2 x float> %D, <2 x float> %E) nounwind {
-; CHECK-LABEL: test_v2f32_post_reg_ld4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ld4.s { v0, v1, v2, v3 }[0], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_reg_ld4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ld4.s { v0, v1, v2, v3 }[0], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_reg_ld4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8290,15 +8287,15 @@ declare { <2 x float>, <2 x float>, <2 x float>, <2 x float> } @llvm.aarch64.neo
 
 
 define { <2 x double>, <2 x double>, <2 x double>, <2 x double> } @test_v2f64_post_imm_ld4lane(ptr %A, ptr %ptr, <2 x double> %B, <2 x double> %C, <2 x double> %D, <2 x double> %E) nounwind {
-; CHECK-LABEL: test_v2f64_post_imm_ld4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ld4.d { v0, v1, v2, v3 }[0], [x0], #32
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_imm_ld4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ld4.d { v0, v1, v2, v3 }[0], [x0], #32
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_imm_ld4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8317,16 +8314,16 @@ define { <2 x double>, <2 x double>, <2 x double>, <2 x double> } @test_v2f64_po
 }
 
 define { <2 x double>, <2 x double>, <2 x double>, <2 x double> } @test_v2f64_post_reg_ld4lane(ptr %A, ptr %ptr, i64 %inc, <2 x double> %B, <2 x double> %C, <2 x double> %D, <2 x double> %E) nounwind {
-; CHECK-LABEL: test_v2f64_post_reg_ld4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ld4.d { v0, v1, v2, v3 }[0], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_reg_ld4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ld4.d { v0, v1, v2, v3 }[0], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_reg_ld4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8348,15 +8345,15 @@ declare { <2 x double>, <2 x double>, <2 x double>, <2 x double> } @llvm.aarch64
 
 
 define { <1 x double>, <1 x double>, <1 x double>, <1 x double> } @test_v1f64_post_imm_ld4lane(ptr %A, ptr %ptr, <1 x double> %B, <1 x double> %C, <1 x double> %D, <1 x double> %E) nounwind {
-; CHECK-LABEL: test_v1f64_post_imm_ld4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ld4.d { v0, v1, v2, v3 }[0], [x0], #32
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_imm_ld4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ld4.d { v0, v1, v2, v3 }[0], [x0], #32
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_imm_ld4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8375,16 +8372,16 @@ define { <1 x double>, <1 x double>, <1 x double>, <1 x double> } @test_v1f64_po
 }
 
 define { <1 x double>, <1 x double>, <1 x double>, <1 x double> } @test_v1f64_post_reg_ld4lane(ptr %A, ptr %ptr, i64 %inc, <1 x double> %B, <1 x double> %C, <1 x double> %D, <1 x double> %E) nounwind {
-; CHECK-LABEL: test_v1f64_post_reg_ld4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ld4.d { v0, v1, v2, v3 }[0], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_reg_ld4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ld4.d { v0, v1, v2, v3 }[0], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_reg_ld4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8406,12 +8403,12 @@ declare { <1 x double>, <1 x double>, <1 x double>, <1 x double> } @llvm.aarch64
 
 
 define ptr @test_v16i8_post_imm_st2(ptr %A, ptr %ptr, <16 x i8> %B, <16 x i8> %C) nounwind {
-; CHECK-LABEL: test_v16i8_post_imm_st2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st2.16b { v0, v1 }, [x0], #32
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_imm_st2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st2.16b { v0, v1 }, [x0], #32
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_imm_st2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8427,12 +8424,12 @@ define ptr @test_v16i8_post_imm_st2(ptr %A, ptr %ptr, <16 x i8> %B, <16 x i8> %C
 }
 
 define ptr @test_v16i8_post_reg_st2(ptr %A, ptr %ptr, <16 x i8> %B, <16 x i8> %C, i64 %inc) nounwind {
-; CHECK-LABEL: test_v16i8_post_reg_st2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st2.16b { v0, v1 }, [x0], x2
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_reg_st2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st2.16b { v0, v1 }, [x0], x2
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_reg_st2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8451,12 +8448,12 @@ declare void @llvm.aarch64.neon.st2.v16i8.p0(<16 x i8>, <16 x i8>, ptr)
 
 
 define ptr @test_v8i8_post_imm_st2(ptr %A, ptr %ptr, <8 x i8> %B, <8 x i8> %C) nounwind {
-; CHECK-LABEL: test_v8i8_post_imm_st2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    st2.8b { v0, v1 }, [x0], #16
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_imm_st2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    st2.8b { v0, v1 }, [x0], #16
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_imm_st2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8472,12 +8469,12 @@ define ptr @test_v8i8_post_imm_st2(ptr %A, ptr %ptr, <8 x i8> %B, <8 x i8> %C) n
 }
 
 define ptr @test_v8i8_post_reg_st2(ptr %A, ptr %ptr, <8 x i8> %B, <8 x i8> %C, i64 %inc) nounwind {
-; CHECK-LABEL: test_v8i8_post_reg_st2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    st2.8b { v0, v1 }, [x0], x2
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_reg_st2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    st2.8b { v0, v1 }, [x0], x2
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_reg_st2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8496,12 +8493,12 @@ declare void @llvm.aarch64.neon.st2.v8i8.p0(<8 x i8>, <8 x i8>, ptr)
 
 
 define ptr @test_v8i16_post_imm_st2(ptr %A, ptr %ptr, <8 x i16> %B, <8 x i16> %C) nounwind {
-; CHECK-LABEL: test_v8i16_post_imm_st2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st2.8h { v0, v1 }, [x0], #32
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_imm_st2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st2.8h { v0, v1 }, [x0], #32
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_imm_st2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8517,13 +8514,13 @@ define ptr @test_v8i16_post_imm_st2(ptr %A, ptr %ptr, <8 x i16> %B, <8 x i16> %C
 }
 
 define ptr @test_v8i16_post_reg_st2(ptr %A, ptr %ptr, <8 x i16> %B, <8 x i16> %C, i64 %inc) nounwind {
-; CHECK-LABEL: test_v8i16_post_reg_st2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st2.8h { v0, v1 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_reg_st2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st2.8h { v0, v1 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_reg_st2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8542,12 +8539,12 @@ declare void @llvm.aarch64.neon.st2.v8i16.p0(<8 x i16>, <8 x i16>, ptr)
 
 
 define ptr @test_v4i16_post_imm_st2(ptr %A, ptr %ptr, <4 x i16> %B, <4 x i16> %C) nounwind {
-; CHECK-LABEL: test_v4i16_post_imm_st2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    st2.4h { v0, v1 }, [x0], #16
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_imm_st2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    st2.4h { v0, v1 }, [x0], #16
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_imm_st2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8563,13 +8560,13 @@ define ptr @test_v4i16_post_imm_st2(ptr %A, ptr %ptr, <4 x i16> %B, <4 x i16> %C
 }
 
 define ptr @test_v4i16_post_reg_st2(ptr %A, ptr %ptr, <4 x i16> %B, <4 x i16> %C, i64 %inc) nounwind {
-; CHECK-LABEL: test_v4i16_post_reg_st2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    st2.4h { v0, v1 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_reg_st2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    st2.4h { v0, v1 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_reg_st2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8588,12 +8585,12 @@ declare void @llvm.aarch64.neon.st2.v4i16.p0(<4 x i16>, <4 x i16>, ptr)
 
 
 define ptr @test_v4i32_post_imm_st2(ptr %A, ptr %ptr, <4 x i32> %B, <4 x i32> %C) nounwind {
-; CHECK-LABEL: test_v4i32_post_imm_st2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st2.4s { v0, v1 }, [x0], #32
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_imm_st2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st2.4s { v0, v1 }, [x0], #32
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_imm_st2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8609,13 +8606,13 @@ define ptr @test_v4i32_post_imm_st2(ptr %A, ptr %ptr, <4 x i32> %B, <4 x i32> %C
 }
 
 define ptr @test_v4i32_post_reg_st2(ptr %A, ptr %ptr, <4 x i32> %B, <4 x i32> %C, i64 %inc) nounwind {
-; CHECK-LABEL: test_v4i32_post_reg_st2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st2.4s { v0, v1 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_reg_st2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st2.4s { v0, v1 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_reg_st2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8634,12 +8631,12 @@ declare void @llvm.aarch64.neon.st2.v4i32.p0(<4 x i32>, <4 x i32>, ptr)
 
 
 define ptr @test_v2i32_post_imm_st2(ptr %A, ptr %ptr, <2 x i32> %B, <2 x i32> %C) nounwind {
-; CHECK-LABEL: test_v2i32_post_imm_st2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    st2.2s { v0, v1 }, [x0], #16
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_imm_st2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    st2.2s { v0, v1 }, [x0], #16
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_imm_st2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8655,13 +8652,13 @@ define ptr @test_v2i32_post_imm_st2(ptr %A, ptr %ptr, <2 x i32> %B, <2 x i32> %C
 }
 
 define ptr @test_v2i32_post_reg_st2(ptr %A, ptr %ptr, <2 x i32> %B, <2 x i32> %C, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2i32_post_reg_st2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    st2.2s { v0, v1 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_reg_st2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    st2.2s { v0, v1 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_reg_st2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8680,12 +8677,12 @@ declare void @llvm.aarch64.neon.st2.v2i32.p0(<2 x i32>, <2 x i32>, ptr)
 
 
 define ptr @test_v2i64_post_imm_st2(ptr %A, ptr %ptr, <2 x i64> %B, <2 x i64> %C) nounwind {
-; CHECK-LABEL: test_v2i64_post_imm_st2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st2.2d { v0, v1 }, [x0], #32
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_imm_st2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st2.2d { v0, v1 }, [x0], #32
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_imm_st2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8701,13 +8698,13 @@ define ptr @test_v2i64_post_imm_st2(ptr %A, ptr %ptr, <2 x i64> %B, <2 x i64> %C
 }
 
 define ptr @test_v2i64_post_reg_st2(ptr %A, ptr %ptr, <2 x i64> %B, <2 x i64> %C, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2i64_post_reg_st2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st2.2d { v0, v1 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_reg_st2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st2.2d { v0, v1 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_reg_st2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8726,12 +8723,12 @@ declare void @llvm.aarch64.neon.st2.v2i64.p0(<2 x i64>, <2 x i64>, ptr)
 
 
 define ptr @test_v1i64_post_imm_st2(ptr %A, ptr %ptr, <1 x i64> %B, <1 x i64> %C) nounwind {
-; CHECK-LABEL: test_v1i64_post_imm_st2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    st1.1d { v0, v1 }, [x0], #16
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_imm_st2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    st1.1d { v0, v1 }, [x0], #16
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_imm_st2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8747,13 +8744,13 @@ define ptr @test_v1i64_post_imm_st2(ptr %A, ptr %ptr, <1 x i64> %B, <1 x i64> %C
 }
 
 define ptr @test_v1i64_post_reg_st2(ptr %A, ptr %ptr, <1 x i64> %B, <1 x i64> %C, i64 %inc) nounwind {
-; CHECK-LABEL: test_v1i64_post_reg_st2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    st1.1d { v0, v1 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_reg_st2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    st1.1d { v0, v1 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_reg_st2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8772,12 +8769,12 @@ declare void @llvm.aarch64.neon.st2.v1i64.p0(<1 x i64>, <1 x i64>, ptr)
 
 
 define ptr @test_v4f32_post_imm_st2(ptr %A, ptr %ptr, <4 x float> %B, <4 x float> %C) nounwind {
-; CHECK-LABEL: test_v4f32_post_imm_st2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st2.4s { v0, v1 }, [x0], #32
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_imm_st2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st2.4s { v0, v1 }, [x0], #32
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_imm_st2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8793,13 +8790,13 @@ define ptr @test_v4f32_post_imm_st2(ptr %A, ptr %ptr, <4 x float> %B, <4 x float
 }
 
 define ptr @test_v4f32_post_reg_st2(ptr %A, ptr %ptr, <4 x float> %B, <4 x float> %C, i64 %inc) nounwind {
-; CHECK-LABEL: test_v4f32_post_reg_st2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st2.4s { v0, v1 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_reg_st2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st2.4s { v0, v1 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_reg_st2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8818,12 +8815,12 @@ declare void @llvm.aarch64.neon.st2.v4f32.p0(<4 x float>, <4 x float>, ptr)
 
 
 define ptr @test_v2f32_post_imm_st2(ptr %A, ptr %ptr, <2 x float> %B, <2 x float> %C) nounwind {
-; CHECK-LABEL: test_v2f32_post_imm_st2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    st2.2s { v0, v1 }, [x0], #16
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_imm_st2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    st2.2s { v0, v1 }, [x0], #16
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_imm_st2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8839,13 +8836,13 @@ define ptr @test_v2f32_post_imm_st2(ptr %A, ptr %ptr, <2 x float> %B, <2 x float
 }
 
 define ptr @test_v2f32_post_reg_st2(ptr %A, ptr %ptr, <2 x float> %B, <2 x float> %C, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2f32_post_reg_st2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    st2.2s { v0, v1 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_reg_st2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    st2.2s { v0, v1 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_reg_st2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8864,12 +8861,12 @@ declare void @llvm.aarch64.neon.st2.v2f32.p0(<2 x float>, <2 x float>, ptr)
 
 
 define ptr @test_v2f64_post_imm_st2(ptr %A, ptr %ptr, <2 x double> %B, <2 x double> %C) nounwind {
-; CHECK-LABEL: test_v2f64_post_imm_st2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st2.2d { v0, v1 }, [x0], #32
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_imm_st2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st2.2d { v0, v1 }, [x0], #32
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_imm_st2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8885,13 +8882,13 @@ define ptr @test_v2f64_post_imm_st2(ptr %A, ptr %ptr, <2 x double> %B, <2 x doub
 }
 
 define ptr @test_v2f64_post_reg_st2(ptr %A, ptr %ptr, <2 x double> %B, <2 x double> %C, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2f64_post_reg_st2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st2.2d { v0, v1 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_reg_st2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st2.2d { v0, v1 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_reg_st2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8910,12 +8907,12 @@ declare void @llvm.aarch64.neon.st2.v2f64.p0(<2 x double>, <2 x double>, ptr)
 
 
 define ptr @test_v1f64_post_imm_st2(ptr %A, ptr %ptr, <1 x double> %B, <1 x double> %C) nounwind {
-; CHECK-LABEL: test_v1f64_post_imm_st2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    st1.1d { v0, v1 }, [x0], #16
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_imm_st2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    st1.1d { v0, v1 }, [x0], #16
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_imm_st2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8931,13 +8928,13 @@ define ptr @test_v1f64_post_imm_st2(ptr %A, ptr %ptr, <1 x double> %B, <1 x doub
 }
 
 define ptr @test_v1f64_post_reg_st2(ptr %A, ptr %ptr, <1 x double> %B, <1 x double> %C, i64 %inc) nounwind {
-; CHECK-LABEL: test_v1f64_post_reg_st2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    st1.1d { v0, v1 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_reg_st2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    st1.1d { v0, v1 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_reg_st2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8956,13 +8953,13 @@ declare void @llvm.aarch64.neon.st2.v1f64.p0(<1 x double>, <1 x double>, ptr)
 
 
 define ptr @test_v16i8_post_imm_st3(ptr %A, ptr %ptr, <16 x i8> %B, <16 x i8> %C, <16 x i8> %D) nounwind {
-; CHECK-LABEL: test_v16i8_post_imm_st3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st3.16b { v0, v1, v2 }, [x0], #48
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_imm_st3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st3.16b { v0, v1, v2 }, [x0], #48
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_imm_st3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -8979,13 +8976,13 @@ define ptr @test_v16i8_post_imm_st3(ptr %A, ptr %ptr, <16 x i8> %B, <16 x i8> %C
 }
 
 define ptr @test_v16i8_post_reg_st3(ptr %A, ptr %ptr, <16 x i8> %B, <16 x i8> %C, <16 x i8> %D, i64 %inc) nounwind {
-; CHECK-LABEL: test_v16i8_post_reg_st3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st3.16b { v0, v1, v2 }, [x0], x2
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_reg_st3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st3.16b { v0, v1, v2 }, [x0], x2
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_reg_st3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9005,13 +9002,13 @@ declare void @llvm.aarch64.neon.st3.v16i8.p0(<16 x i8>, <16 x i8>, <16 x i8>, pt
 
 
 define ptr @test_v8i8_post_imm_st3(ptr %A, ptr %ptr, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D) nounwind {
-; CHECK-LABEL: test_v8i8_post_imm_st3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    st3.8b { v0, v1, v2 }, [x0], #24
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_imm_st3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    st3.8b { v0, v1, v2 }, [x0], #24
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_imm_st3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9028,13 +9025,13 @@ define ptr @test_v8i8_post_imm_st3(ptr %A, ptr %ptr, <8 x i8> %B, <8 x i8> %C, <
 }
 
 define ptr @test_v8i8_post_reg_st3(ptr %A, ptr %ptr, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D, i64 %inc) nounwind {
-; CHECK-LABEL: test_v8i8_post_reg_st3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    st3.8b { v0, v1, v2 }, [x0], x2
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_reg_st3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    st3.8b { v0, v1, v2 }, [x0], x2
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_reg_st3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9054,13 +9051,13 @@ declare void @llvm.aarch64.neon.st3.v8i8.p0(<8 x i8>, <8 x i8>, <8 x i8>, ptr)
 
 
 define ptr @test_v8i16_post_imm_st3(ptr %A, ptr %ptr, <8 x i16> %B, <8 x i16> %C, <8 x i16> %D) nounwind {
-; CHECK-LABEL: test_v8i16_post_imm_st3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st3.8h { v0, v1, v2 }, [x0], #48
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_imm_st3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st3.8h { v0, v1, v2 }, [x0], #48
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_imm_st3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9077,14 +9074,14 @@ define ptr @test_v8i16_post_imm_st3(ptr %A, ptr %ptr, <8 x i16> %B, <8 x i16> %C
 }
 
 define ptr @test_v8i16_post_reg_st3(ptr %A, ptr %ptr, <8 x i16> %B, <8 x i16> %C, <8 x i16> %D, i64 %inc) nounwind {
-; CHECK-LABEL: test_v8i16_post_reg_st3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st3.8h { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_reg_st3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st3.8h { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_reg_st3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9104,13 +9101,13 @@ declare void @llvm.aarch64.neon.st3.v8i16.p0(<8 x i16>, <8 x i16>, <8 x i16>, pt
 
 
 define ptr @test_v4i16_post_imm_st3(ptr %A, ptr %ptr, <4 x i16> %B, <4 x i16> %C, <4 x i16> %D) nounwind {
-; CHECK-LABEL: test_v4i16_post_imm_st3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    st3.4h { v0, v1, v2 }, [x0], #24
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_imm_st3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    st3.4h { v0, v1, v2 }, [x0], #24
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_imm_st3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9127,14 +9124,14 @@ define ptr @test_v4i16_post_imm_st3(ptr %A, ptr %ptr, <4 x i16> %B, <4 x i16> %C
 }
 
 define ptr @test_v4i16_post_reg_st3(ptr %A, ptr %ptr, <4 x i16> %B, <4 x i16> %C, <4 x i16> %D, i64 %inc) nounwind {
-; CHECK-LABEL: test_v4i16_post_reg_st3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    st3.4h { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_reg_st3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    st3.4h { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_reg_st3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9154,13 +9151,13 @@ declare void @llvm.aarch64.neon.st3.v4i16.p0(<4 x i16>, <4 x i16>, <4 x i16>, pt
 
 
 define ptr @test_v4i32_post_imm_st3(ptr %A, ptr %ptr, <4 x i32> %B, <4 x i32> %C, <4 x i32> %D) nounwind {
-; CHECK-LABEL: test_v4i32_post_imm_st3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st3.4s { v0, v1, v2 }, [x0], #48
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_imm_st3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st3.4s { v0, v1, v2 }, [x0], #48
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_imm_st3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9177,14 +9174,14 @@ define ptr @test_v4i32_post_imm_st3(ptr %A, ptr %ptr, <4 x i32> %B, <4 x i32> %C
 }
 
 define ptr @test_v4i32_post_reg_st3(ptr %A, ptr %ptr, <4 x i32> %B, <4 x i32> %C, <4 x i32> %D, i64 %inc) nounwind {
-; CHECK-LABEL: test_v4i32_post_reg_st3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st3.4s { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_reg_st3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st3.4s { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_reg_st3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9204,13 +9201,13 @@ declare void @llvm.aarch64.neon.st3.v4i32.p0(<4 x i32>, <4 x i32>, <4 x i32>, pt
 
 
 define ptr @test_v2i32_post_imm_st3(ptr %A, ptr %ptr, <2 x i32> %B, <2 x i32> %C, <2 x i32> %D) nounwind {
-; CHECK-LABEL: test_v2i32_post_imm_st3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    st3.2s { v0, v1, v2 }, [x0], #24
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_imm_st3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    st3.2s { v0, v1, v2 }, [x0], #24
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_imm_st3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9227,14 +9224,14 @@ define ptr @test_v2i32_post_imm_st3(ptr %A, ptr %ptr, <2 x i32> %B, <2 x i32> %C
 }
 
 define ptr @test_v2i32_post_reg_st3(ptr %A, ptr %ptr, <2 x i32> %B, <2 x i32> %C, <2 x i32> %D, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2i32_post_reg_st3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    st3.2s { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_reg_st3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    st3.2s { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_reg_st3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9254,13 +9251,13 @@ declare void @llvm.aarch64.neon.st3.v2i32.p0(<2 x i32>, <2 x i32>, <2 x i32>, pt
 
 
 define ptr @test_v2i64_post_imm_st3(ptr %A, ptr %ptr, <2 x i64> %B, <2 x i64> %C, <2 x i64> %D) nounwind {
-; CHECK-LABEL: test_v2i64_post_imm_st3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st3.2d { v0, v1, v2 }, [x0], #48
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_imm_st3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st3.2d { v0, v1, v2 }, [x0], #48
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_imm_st3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9277,14 +9274,14 @@ define ptr @test_v2i64_post_imm_st3(ptr %A, ptr %ptr, <2 x i64> %B, <2 x i64> %C
 }
 
 define ptr @test_v2i64_post_reg_st3(ptr %A, ptr %ptr, <2 x i64> %B, <2 x i64> %C, <2 x i64> %D, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2i64_post_reg_st3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st3.2d { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_reg_st3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st3.2d { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_reg_st3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9304,13 +9301,13 @@ declare void @llvm.aarch64.neon.st3.v2i64.p0(<2 x i64>, <2 x i64>, <2 x i64>, pt
 
 
 define ptr @test_v1i64_post_imm_st3(ptr %A, ptr %ptr, <1 x i64> %B, <1 x i64> %C, <1 x i64> %D) nounwind {
-; CHECK-LABEL: test_v1i64_post_imm_st3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    st1.1d { v0, v1, v2 }, [x0], #24
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_imm_st3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    st1.1d { v0, v1, v2 }, [x0], #24
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_imm_st3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9327,14 +9324,14 @@ define ptr @test_v1i64_post_imm_st3(ptr %A, ptr %ptr, <1 x i64> %B, <1 x i64> %C
 }
 
 define ptr @test_v1i64_post_reg_st3(ptr %A, ptr %ptr, <1 x i64> %B, <1 x i64> %C, <1 x i64> %D, i64 %inc) nounwind {
-; CHECK-LABEL: test_v1i64_post_reg_st3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    st1.1d { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_reg_st3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    st1.1d { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_reg_st3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9354,13 +9351,13 @@ declare void @llvm.aarch64.neon.st3.v1i64.p0(<1 x i64>, <1 x i64>, <1 x i64>, pt
 
 
 define ptr @test_v4f32_post_imm_st3(ptr %A, ptr %ptr, <4 x float> %B, <4 x float> %C, <4 x float> %D) nounwind {
-; CHECK-LABEL: test_v4f32_post_imm_st3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st3.4s { v0, v1, v2 }, [x0], #48
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_imm_st3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st3.4s { v0, v1, v2 }, [x0], #48
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_imm_st3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9377,14 +9374,14 @@ define ptr @test_v4f32_post_imm_st3(ptr %A, ptr %ptr, <4 x float> %B, <4 x float
 }
 
 define ptr @test_v4f32_post_reg_st3(ptr %A, ptr %ptr, <4 x float> %B, <4 x float> %C, <4 x float> %D, i64 %inc) nounwind {
-; CHECK-LABEL: test_v4f32_post_reg_st3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st3.4s { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_reg_st3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st3.4s { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_reg_st3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9404,13 +9401,13 @@ declare void @llvm.aarch64.neon.st3.v4f32.p0(<4 x float>, <4 x float>, <4 x floa
 
 
 define ptr @test_v2f32_post_imm_st3(ptr %A, ptr %ptr, <2 x float> %B, <2 x float> %C, <2 x float> %D) nounwind {
-; CHECK-LABEL: test_v2f32_post_imm_st3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    st3.2s { v0, v1, v2 }, [x0], #24
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_imm_st3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    st3.2s { v0, v1, v2 }, [x0], #24
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_imm_st3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9427,14 +9424,14 @@ define ptr @test_v2f32_post_imm_st3(ptr %A, ptr %ptr, <2 x float> %B, <2 x float
 }
 
 define ptr @test_v2f32_post_reg_st3(ptr %A, ptr %ptr, <2 x float> %B, <2 x float> %C, <2 x float> %D, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2f32_post_reg_st3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    st3.2s { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_reg_st3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    st3.2s { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_reg_st3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9454,13 +9451,13 @@ declare void @llvm.aarch64.neon.st3.v2f32.p0(<2 x float>, <2 x float>, <2 x floa
 
 
 define ptr @test_v2f64_post_imm_st3(ptr %A, ptr %ptr, <2 x double> %B, <2 x double> %C, <2 x double> %D) nounwind {
-; CHECK-LABEL: test_v2f64_post_imm_st3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st3.2d { v0, v1, v2 }, [x0], #48
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_imm_st3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st3.2d { v0, v1, v2 }, [x0], #48
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_imm_st3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9477,14 +9474,14 @@ define ptr @test_v2f64_post_imm_st3(ptr %A, ptr %ptr, <2 x double> %B, <2 x doub
 }
 
 define ptr @test_v2f64_post_reg_st3(ptr %A, ptr %ptr, <2 x double> %B, <2 x double> %C, <2 x double> %D, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2f64_post_reg_st3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st3.2d { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_reg_st3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st3.2d { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_reg_st3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9504,13 +9501,13 @@ declare void @llvm.aarch64.neon.st3.v2f64.p0(<2 x double>, <2 x double>, <2 x do
 
 
 define ptr @test_v1f64_post_imm_st3(ptr %A, ptr %ptr, <1 x double> %B, <1 x double> %C, <1 x double> %D) nounwind {
-; CHECK-LABEL: test_v1f64_post_imm_st3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    st1.1d { v0, v1, v2 }, [x0], #24
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_imm_st3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    st1.1d { v0, v1, v2 }, [x0], #24
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_imm_st3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9527,14 +9524,14 @@ define ptr @test_v1f64_post_imm_st3(ptr %A, ptr %ptr, <1 x double> %B, <1 x doub
 }
 
 define ptr @test_v1f64_post_reg_st3(ptr %A, ptr %ptr, <1 x double> %B, <1 x double> %C, <1 x double> %D, i64 %inc) nounwind {
-; CHECK-LABEL: test_v1f64_post_reg_st3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    st1.1d { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_reg_st3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    st1.1d { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_reg_st3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9554,14 +9551,14 @@ declare void @llvm.aarch64.neon.st3.v1f64.p0(<1 x double>, <1 x double>, <1 x do
 
 
 define ptr @test_v16i8_post_imm_st4(ptr %A, ptr %ptr, <16 x i8> %B, <16 x i8> %C, <16 x i8> %D, <16 x i8> %E) nounwind {
-; CHECK-LABEL: test_v16i8_post_imm_st4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st4.16b { v0, v1, v2, v3 }, [x0], #64
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_imm_st4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st4.16b { v0, v1, v2, v3 }, [x0], #64
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_imm_st4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9579,14 +9576,14 @@ define ptr @test_v16i8_post_imm_st4(ptr %A, ptr %ptr, <16 x i8> %B, <16 x i8> %C
 }
 
 define ptr @test_v16i8_post_reg_st4(ptr %A, ptr %ptr, <16 x i8> %B, <16 x i8> %C, <16 x i8> %D, <16 x i8> %E, i64 %inc) nounwind {
-; CHECK-LABEL: test_v16i8_post_reg_st4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st4.16b { v0, v1, v2, v3 }, [x0], x2
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_reg_st4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st4.16b { v0, v1, v2, v3 }, [x0], x2
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_reg_st4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9607,14 +9604,14 @@ declare void @llvm.aarch64.neon.st4.v16i8.p0(<16 x i8>, <16 x i8>, <16 x i8>, <1
 
 
 define ptr @test_v8i8_post_imm_st4(ptr %A, ptr %ptr, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D, <8 x i8> %E) nounwind {
-; CHECK-LABEL: test_v8i8_post_imm_st4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    st4.8b { v0, v1, v2, v3 }, [x0], #32
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_imm_st4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    st4.8b { v0, v1, v2, v3 }, [x0], #32
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_imm_st4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9632,14 +9629,14 @@ define ptr @test_v8i8_post_imm_st4(ptr %A, ptr %ptr, <8 x i8> %B, <8 x i8> %C, <
 }
 
 define ptr @test_v8i8_post_reg_st4(ptr %A, ptr %ptr, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D, <8 x i8> %E, i64 %inc) nounwind {
-; CHECK-LABEL: test_v8i8_post_reg_st4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    st4.8b { v0, v1, v2, v3 }, [x0], x2
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_reg_st4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    st4.8b { v0, v1, v2, v3 }, [x0], x2
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_reg_st4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9660,14 +9657,14 @@ declare void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8>, <8 x i8>, <8 x i8>, <8 x i
 
 
 define ptr @test_v8i16_post_imm_st4(ptr %A, ptr %ptr, <8 x i16> %B, <8 x i16> %C, <8 x i16> %D, <8 x i16> %E) nounwind {
-; CHECK-LABEL: test_v8i16_post_imm_st4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st4.8h { v0, v1, v2, v3 }, [x0], #64
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_imm_st4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st4.8h { v0, v1, v2, v3 }, [x0], #64
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_imm_st4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9685,15 +9682,15 @@ define ptr @test_v8i16_post_imm_st4(ptr %A, ptr %ptr, <8 x i16> %B, <8 x i16> %C
 }
 
 define ptr @test_v8i16_post_reg_st4(ptr %A, ptr %ptr, <8 x i16> %B, <8 x i16> %C, <8 x i16> %D, <8 x i16> %E, i64 %inc) nounwind {
-; CHECK-LABEL: test_v8i16_post_reg_st4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st4.8h { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_reg_st4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st4.8h { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_reg_st4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9714,14 +9711,14 @@ declare void @llvm.aarch64.neon.st4.v8i16.p0(<8 x i16>, <8 x i16>, <8 x i16>, <8
 
 
 define ptr @test_v4i16_post_imm_st4(ptr %A, ptr %ptr, <4 x i16> %B, <4 x i16> %C, <4 x i16> %D, <4 x i16> %E) nounwind {
-; CHECK-LABEL: test_v4i16_post_imm_st4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    st4.4h { v0, v1, v2, v3 }, [x0], #32
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_imm_st4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    st4.4h { v0, v1, v2, v3 }, [x0], #32
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_imm_st4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9739,15 +9736,15 @@ define ptr @test_v4i16_post_imm_st4(ptr %A, ptr %ptr, <4 x i16> %B, <4 x i16> %C
 }
 
 define ptr @test_v4i16_post_reg_st4(ptr %A, ptr %ptr, <4 x i16> %B, <4 x i16> %C, <4 x i16> %D, <4 x i16> %E, i64 %inc) nounwind {
-; CHECK-LABEL: test_v4i16_post_reg_st4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    st4.4h { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_reg_st4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    st4.4h { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_reg_st4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9768,14 +9765,14 @@ declare void @llvm.aarch64.neon.st4.v4i16.p0(<4 x i16>, <4 x i16>, <4 x i16>,<4
 
 
 define ptr @test_v4i32_post_imm_st4(ptr %A, ptr %ptr, <4 x i32> %B, <4 x i32> %C, <4 x i32> %D, <4 x i32> %E) nounwind {
-; CHECK-LABEL: test_v4i32_post_imm_st4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st4.4s { v0, v1, v2, v3 }, [x0], #64
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_imm_st4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st4.4s { v0, v1, v2, v3 }, [x0], #64
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_imm_st4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9793,15 +9790,15 @@ define ptr @test_v4i32_post_imm_st4(ptr %A, ptr %ptr, <4 x i32> %B, <4 x i32> %C
 }
 
 define ptr @test_v4i32_post_reg_st4(ptr %A, ptr %ptr, <4 x i32> %B, <4 x i32> %C, <4 x i32> %D, <4 x i32> %E, i64 %inc) nounwind {
-; CHECK-LABEL: test_v4i32_post_reg_st4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st4.4s { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_reg_st4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st4.4s { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_reg_st4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9822,14 +9819,14 @@ declare void @llvm.aarch64.neon.st4.v4i32.p0(<4 x i32>, <4 x i32>, <4 x i32>,<4
 
 
 define ptr @test_v2i32_post_imm_st4(ptr %A, ptr %ptr, <2 x i32> %B, <2 x i32> %C, <2 x i32> %D, <2 x i32> %E) nounwind {
-; CHECK-LABEL: test_v2i32_post_imm_st4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    st4.2s { v0, v1, v2, v3 }, [x0], #32
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_imm_st4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    st4.2s { v0, v1, v2, v3 }, [x0], #32
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_imm_st4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9847,15 +9844,15 @@ define ptr @test_v2i32_post_imm_st4(ptr %A, ptr %ptr, <2 x i32> %B, <2 x i32> %C
 }
 
 define ptr @test_v2i32_post_reg_st4(ptr %A, ptr %ptr, <2 x i32> %B, <2 x i32> %C, <2 x i32> %D, <2 x i32> %E, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2i32_post_reg_st4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    st4.2s { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_reg_st4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    st4.2s { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_reg_st4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9876,14 +9873,14 @@ declare void @llvm.aarch64.neon.st4.v2i32.p0(<2 x i32>, <2 x i32>, <2 x i32>, <2
 
 
 define ptr @test_v2i64_post_imm_st4(ptr %A, ptr %ptr, <2 x i64> %B, <2 x i64> %C, <2 x i64> %D, <2 x i64> %E) nounwind {
-; CHECK-LABEL: test_v2i64_post_imm_st4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st4.2d { v0, v1, v2, v3 }, [x0], #64
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_imm_st4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st4.2d { v0, v1, v2, v3 }, [x0], #64
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_imm_st4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9901,15 +9898,15 @@ define ptr @test_v2i64_post_imm_st4(ptr %A, ptr %ptr, <2 x i64> %B, <2 x i64> %C
 }
 
 define ptr @test_v2i64_post_reg_st4(ptr %A, ptr %ptr, <2 x i64> %B, <2 x i64> %C, <2 x i64> %D, <2 x i64> %E, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2i64_post_reg_st4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st4.2d { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_reg_st4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st4.2d { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_reg_st4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9930,14 +9927,14 @@ declare void @llvm.aarch64.neon.st4.v2i64.p0(<2 x i64>, <2 x i64>, <2 x i64>,<2
 
 
 define ptr @test_v1i64_post_imm_st4(ptr %A, ptr %ptr, <1 x i64> %B, <1 x i64> %C, <1 x i64> %D, <1 x i64> %E) nounwind {
-; CHECK-LABEL: test_v1i64_post_imm_st4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    st1.1d { v0, v1, v2, v3 }, [x0], #32
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_imm_st4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    st1.1d { v0, v1, v2, v3 }, [x0], #32
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_imm_st4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9955,15 +9952,15 @@ define ptr @test_v1i64_post_imm_st4(ptr %A, ptr %ptr, <1 x i64> %B, <1 x i64> %C
 }
 
 define ptr @test_v1i64_post_reg_st4(ptr %A, ptr %ptr, <1 x i64> %B, <1 x i64> %C, <1 x i64> %D, <1 x i64> %E, i64 %inc) nounwind {
-; CHECK-LABEL: test_v1i64_post_reg_st4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    st1.1d { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_reg_st4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    st1.1d { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_reg_st4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -9984,14 +9981,14 @@ declare void @llvm.aarch64.neon.st4.v1i64.p0(<1 x i64>, <1 x i64>, <1 x i64>,<1
 
 
 define ptr @test_v4f32_post_imm_st4(ptr %A, ptr %ptr, <4 x float> %B, <4 x float> %C, <4 x float> %D, <4 x float> %E) nounwind {
-; CHECK-LABEL: test_v4f32_post_imm_st4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st4.4s { v0, v1, v2, v3 }, [x0], #64
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_imm_st4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st4.4s { v0, v1, v2, v3 }, [x0], #64
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_imm_st4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10009,15 +10006,15 @@ define ptr @test_v4f32_post_imm_st4(ptr %A, ptr %ptr, <4 x float> %B, <4 x float
 }
 
 define ptr @test_v4f32_post_reg_st4(ptr %A, ptr %ptr, <4 x float> %B, <4 x float> %C, <4 x float> %D, <4 x float> %E, i64 %inc) nounwind {
-; CHECK-LABEL: test_v4f32_post_reg_st4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st4.4s { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_reg_st4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st4.4s { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_reg_st4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10038,14 +10035,14 @@ declare void @llvm.aarch64.neon.st4.v4f32.p0(<4 x float>, <4 x float>, <4 x floa
 
 
 define ptr @test_v2f32_post_imm_st4(ptr %A, ptr %ptr, <2 x float> %B, <2 x float> %C, <2 x float> %D, <2 x float> %E) nounwind {
-; CHECK-LABEL: test_v2f32_post_imm_st4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    st4.2s { v0, v1, v2, v3 }, [x0], #32
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_imm_st4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    st4.2s { v0, v1, v2, v3 }, [x0], #32
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_imm_st4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10063,15 +10060,15 @@ define ptr @test_v2f32_post_imm_st4(ptr %A, ptr %ptr, <2 x float> %B, <2 x float
 }
 
 define ptr @test_v2f32_post_reg_st4(ptr %A, ptr %ptr, <2 x float> %B, <2 x float> %C, <2 x float> %D, <2 x float> %E, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2f32_post_reg_st4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    st4.2s { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_reg_st4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    st4.2s { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_reg_st4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10092,14 +10089,14 @@ declare void @llvm.aarch64.neon.st4.v2f32.p0(<2 x float>, <2 x float>, <2 x floa
 
 
 define ptr @test_v2f64_post_imm_st4(ptr %A, ptr %ptr, <2 x double> %B, <2 x double> %C, <2 x double> %D, <2 x double> %E) nounwind {
-; CHECK-LABEL: test_v2f64_post_imm_st4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st4.2d { v0, v1, v2, v3 }, [x0], #64
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_imm_st4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st4.2d { v0, v1, v2, v3 }, [x0], #64
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_imm_st4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10117,15 +10114,15 @@ define ptr @test_v2f64_post_imm_st4(ptr %A, ptr %ptr, <2 x double> %B, <2 x doub
 }
 
 define ptr @test_v2f64_post_reg_st4(ptr %A, ptr %ptr, <2 x double> %B, <2 x double> %C, <2 x double> %D, <2 x double> %E, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2f64_post_reg_st4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st4.2d { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_reg_st4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st4.2d { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_reg_st4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10146,14 +10143,14 @@ declare void @llvm.aarch64.neon.st4.v2f64.p0(<2 x double>, <2 x double>, <2 x do
 
 
 define ptr @test_v1f64_post_imm_st4(ptr %A, ptr %ptr, <1 x double> %B, <1 x double> %C, <1 x double> %D, <1 x double> %E) nounwind {
-; CHECK-LABEL: test_v1f64_post_imm_st4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    st1.1d { v0, v1, v2, v3 }, [x0], #32
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_imm_st4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    st1.1d { v0, v1, v2, v3 }, [x0], #32
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_imm_st4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10171,15 +10168,15 @@ define ptr @test_v1f64_post_imm_st4(ptr %A, ptr %ptr, <1 x double> %B, <1 x doub
 }
 
 define ptr @test_v1f64_post_reg_st4(ptr %A, ptr %ptr, <1 x double> %B, <1 x double> %C, <1 x double> %D, <1 x double> %E, i64 %inc) nounwind {
-; CHECK-LABEL: test_v1f64_post_reg_st4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    st1.1d { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_reg_st4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    st1.1d { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_reg_st4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10200,12 +10197,12 @@ declare void @llvm.aarch64.neon.st4.v1f64.p0(<1 x double>, <1 x double>, <1 x do
 
 
 define ptr @test_v16i8_post_imm_st1x2(ptr %A, ptr %ptr, <16 x i8> %B, <16 x i8> %C) nounwind {
-; CHECK-LABEL: test_v16i8_post_imm_st1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st1.16b { v0, v1 }, [x0], #32
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_imm_st1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st1.16b { v0, v1 }, [x0], #32
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_imm_st1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10221,12 +10218,12 @@ define ptr @test_v16i8_post_imm_st1x2(ptr %A, ptr %ptr, <16 x i8> %B, <16 x i8>
 }
 
 define ptr @test_v16i8_post_reg_st1x2(ptr %A, ptr %ptr, <16 x i8> %B, <16 x i8> %C, i64 %inc) nounwind {
-; CHECK-LABEL: test_v16i8_post_reg_st1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st1.16b { v0, v1 }, [x0], x2
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_reg_st1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st1.16b { v0, v1 }, [x0], x2
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_reg_st1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10245,12 +10242,12 @@ declare void @llvm.aarch64.neon.st1x2.v16i8.p0(<16 x i8>, <16 x i8>, ptr)
 
 
 define ptr @test_v8i8_post_imm_st1x2(ptr %A, ptr %ptr, <8 x i8> %B, <8 x i8> %C) nounwind {
-; CHECK-LABEL: test_v8i8_post_imm_st1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    st1.8b { v0, v1 }, [x0], #16
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_imm_st1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    st1.8b { v0, v1 }, [x0], #16
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_imm_st1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10266,12 +10263,12 @@ define ptr @test_v8i8_post_imm_st1x2(ptr %A, ptr %ptr, <8 x i8> %B, <8 x i8> %C)
 }
 
 define ptr @test_v8i8_post_reg_st1x2(ptr %A, ptr %ptr, <8 x i8> %B, <8 x i8> %C, i64 %inc) nounwind {
-; CHECK-LABEL: test_v8i8_post_reg_st1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    st1.8b { v0, v1 }, [x0], x2
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_reg_st1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    st1.8b { v0, v1 }, [x0], x2
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_reg_st1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10290,12 +10287,12 @@ declare void @llvm.aarch64.neon.st1x2.v8i8.p0(<8 x i8>, <8 x i8>, ptr)
 
 
 define ptr @test_v8i16_post_imm_st1x2(ptr %A, ptr %ptr, <8 x i16> %B, <8 x i16> %C) nounwind {
-; CHECK-LABEL: test_v8i16_post_imm_st1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st1.8h { v0, v1 }, [x0], #32
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_imm_st1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st1.8h { v0, v1 }, [x0], #32
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_imm_st1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10311,13 +10308,13 @@ define ptr @test_v8i16_post_imm_st1x2(ptr %A, ptr %ptr, <8 x i16> %B, <8 x i16>
 }
 
 define ptr @test_v8i16_post_reg_st1x2(ptr %A, ptr %ptr, <8 x i16> %B, <8 x i16> %C, i64 %inc) nounwind {
-; CHECK-LABEL: test_v8i16_post_reg_st1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st1.8h { v0, v1 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_reg_st1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st1.8h { v0, v1 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_reg_st1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10336,12 +10333,12 @@ declare void @llvm.aarch64.neon.st1x2.v8i16.p0(<8 x i16>, <8 x i16>, ptr)
 
 
 define ptr @test_v4i16_post_imm_st1x2(ptr %A, ptr %ptr, <4 x i16> %B, <4 x i16> %C) nounwind {
-; CHECK-LABEL: test_v4i16_post_imm_st1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    st1.4h { v0, v1 }, [x0], #16
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_imm_st1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    st1.4h { v0, v1 }, [x0], #16
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_imm_st1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10357,13 +10354,13 @@ define ptr @test_v4i16_post_imm_st1x2(ptr %A, ptr %ptr, <4 x i16> %B, <4 x i16>
 }
 
 define ptr @test_v4i16_post_reg_st1x2(ptr %A, ptr %ptr, <4 x i16> %B, <4 x i16> %C, i64 %inc) nounwind {
-; CHECK-LABEL: test_v4i16_post_reg_st1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    st1.4h { v0, v1 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_reg_st1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    st1.4h { v0, v1 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_reg_st1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10382,12 +10379,12 @@ declare void @llvm.aarch64.neon.st1x2.v4i16.p0(<4 x i16>, <4 x i16>, ptr)
 
 
 define ptr @test_v4i32_post_imm_st1x2(ptr %A, ptr %ptr, <4 x i32> %B, <4 x i32> %C) nounwind {
-; CHECK-LABEL: test_v4i32_post_imm_st1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st1.4s { v0, v1 }, [x0], #32
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_imm_st1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st1.4s { v0, v1 }, [x0], #32
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_imm_st1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10403,13 +10400,13 @@ define ptr @test_v4i32_post_imm_st1x2(ptr %A, ptr %ptr, <4 x i32> %B, <4 x i32>
 }
 
 define ptr @test_v4i32_post_reg_st1x2(ptr %A, ptr %ptr, <4 x i32> %B, <4 x i32> %C, i64 %inc) nounwind {
-; CHECK-LABEL: test_v4i32_post_reg_st1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st1.4s { v0, v1 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_reg_st1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st1.4s { v0, v1 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_reg_st1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10428,12 +10425,12 @@ declare void @llvm.aarch64.neon.st1x2.v4i32.p0(<4 x i32>, <4 x i32>, ptr)
 
 
 define ptr @test_v2i32_post_imm_st1x2(ptr %A, ptr %ptr, <2 x i32> %B, <2 x i32> %C) nounwind {
-; CHECK-LABEL: test_v2i32_post_imm_st1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    st1.2s { v0, v1 }, [x0], #16
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_imm_st1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    st1.2s { v0, v1 }, [x0], #16
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_imm_st1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10449,13 +10446,13 @@ define ptr @test_v2i32_post_imm_st1x2(ptr %A, ptr %ptr, <2 x i32> %B, <2 x i32>
 }
 
 define ptr @test_v2i32_post_reg_st1x2(ptr %A, ptr %ptr, <2 x i32> %B, <2 x i32> %C, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2i32_post_reg_st1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    st1.2s { v0, v1 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_reg_st1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    st1.2s { v0, v1 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_reg_st1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10474,12 +10471,12 @@ declare void @llvm.aarch64.neon.st1x2.v2i32.p0(<2 x i32>, <2 x i32>, ptr)
 
 
 define ptr @test_v2i64_post_imm_st1x2(ptr %A, ptr %ptr, <2 x i64> %B, <2 x i64> %C) nounwind {
-; CHECK-LABEL: test_v2i64_post_imm_st1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st1.2d { v0, v1 }, [x0], #32
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_imm_st1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st1.2d { v0, v1 }, [x0], #32
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_imm_st1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10495,13 +10492,13 @@ define ptr @test_v2i64_post_imm_st1x2(ptr %A, ptr %ptr, <2 x i64> %B, <2 x i64>
 }
 
 define ptr @test_v2i64_post_reg_st1x2(ptr %A, ptr %ptr, <2 x i64> %B, <2 x i64> %C, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2i64_post_reg_st1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st1.2d { v0, v1 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_reg_st1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st1.2d { v0, v1 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_reg_st1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10520,12 +10517,12 @@ declare void @llvm.aarch64.neon.st1x2.v2i64.p0(<2 x i64>, <2 x i64>, ptr)
 
 
 define ptr @test_v1i64_post_imm_st1x2(ptr %A, ptr %ptr, <1 x i64> %B, <1 x i64> %C) nounwind {
-; CHECK-LABEL: test_v1i64_post_imm_st1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    st1.1d { v0, v1 }, [x0], #16
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_imm_st1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    st1.1d { v0, v1 }, [x0], #16
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_imm_st1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10541,13 +10538,13 @@ define ptr @test_v1i64_post_imm_st1x2(ptr %A, ptr %ptr, <1 x i64> %B, <1 x i64>
 }
 
 define ptr @test_v1i64_post_reg_st1x2(ptr %A, ptr %ptr, <1 x i64> %B, <1 x i64> %C, i64 %inc) nounwind {
-; CHECK-LABEL: test_v1i64_post_reg_st1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    st1.1d { v0, v1 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_reg_st1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    st1.1d { v0, v1 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_reg_st1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10566,12 +10563,12 @@ declare void @llvm.aarch64.neon.st1x2.v1i64.p0(<1 x i64>, <1 x i64>, ptr)
 
 
 define ptr @test_v4f32_post_imm_st1x2(ptr %A, ptr %ptr, <4 x float> %B, <4 x float> %C) nounwind {
-; CHECK-LABEL: test_v4f32_post_imm_st1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st1.4s { v0, v1 }, [x0], #32
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_imm_st1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st1.4s { v0, v1 }, [x0], #32
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_imm_st1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10587,13 +10584,13 @@ define ptr @test_v4f32_post_imm_st1x2(ptr %A, ptr %ptr, <4 x float> %B, <4 x flo
 }
 
 define ptr @test_v4f32_post_reg_st1x2(ptr %A, ptr %ptr, <4 x float> %B, <4 x float> %C, i64 %inc) nounwind {
-; CHECK-LABEL: test_v4f32_post_reg_st1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st1.4s { v0, v1 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_reg_st1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st1.4s { v0, v1 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_reg_st1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10612,12 +10609,12 @@ declare void @llvm.aarch64.neon.st1x2.v4f32.p0(<4 x float>, <4 x float>, ptr)
 
 
 define ptr @test_v2f32_post_imm_st1x2(ptr %A, ptr %ptr, <2 x float> %B, <2 x float> %C) nounwind {
-; CHECK-LABEL: test_v2f32_post_imm_st1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    st1.2s { v0, v1 }, [x0], #16
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_imm_st1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    st1.2s { v0, v1 }, [x0], #16
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_imm_st1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10633,13 +10630,13 @@ define ptr @test_v2f32_post_imm_st1x2(ptr %A, ptr %ptr, <2 x float> %B, <2 x flo
 }
 
 define ptr @test_v2f32_post_reg_st1x2(ptr %A, ptr %ptr, <2 x float> %B, <2 x float> %C, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2f32_post_reg_st1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    st1.2s { v0, v1 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_reg_st1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    st1.2s { v0, v1 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_reg_st1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10658,12 +10655,12 @@ declare void @llvm.aarch64.neon.st1x2.v2f32.p0(<2 x float>, <2 x float>, ptr)
 
 
 define ptr @test_v2f64_post_imm_st1x2(ptr %A, ptr %ptr, <2 x double> %B, <2 x double> %C) nounwind {
-; CHECK-LABEL: test_v2f64_post_imm_st1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st1.2d { v0, v1 }, [x0], #32
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_imm_st1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st1.2d { v0, v1 }, [x0], #32
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_imm_st1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10679,13 +10676,13 @@ define ptr @test_v2f64_post_imm_st1x2(ptr %A, ptr %ptr, <2 x double> %B, <2 x do
 }
 
 define ptr @test_v2f64_post_reg_st1x2(ptr %A, ptr %ptr, <2 x double> %B, <2 x double> %C, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2f64_post_reg_st1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st1.2d { v0, v1 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_reg_st1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st1.2d { v0, v1 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_reg_st1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10704,12 +10701,12 @@ declare void @llvm.aarch64.neon.st1x2.v2f64.p0(<2 x double>, <2 x double>, ptr)
 
 
 define ptr @test_v1f64_post_imm_st1x2(ptr %A, ptr %ptr, <1 x double> %B, <1 x double> %C) nounwind {
-; CHECK-LABEL: test_v1f64_post_imm_st1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    st1.1d { v0, v1 }, [x0], #16
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_imm_st1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    st1.1d { v0, v1 }, [x0], #16
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_imm_st1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10725,13 +10722,13 @@ define ptr @test_v1f64_post_imm_st1x2(ptr %A, ptr %ptr, <1 x double> %B, <1 x do
 }
 
 define ptr @test_v1f64_post_reg_st1x2(ptr %A, ptr %ptr, <1 x double> %B, <1 x double> %C, i64 %inc) nounwind {
-; CHECK-LABEL: test_v1f64_post_reg_st1x2:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
-; CHECK-NEXT:    st1.1d { v0, v1 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_reg_st1x2:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1 def $d0_d1
+; SDAG-NEXT:    st1.1d { v0, v1 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_reg_st1x2:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10750,13 +10747,13 @@ declare void @llvm.aarch64.neon.st1x2.v1f64.p0(<1 x double>, <1 x double>, ptr)
 
 
 define ptr @test_v16i8_post_imm_st1x3(ptr %A, ptr %ptr, <16 x i8> %B, <16 x i8> %C, <16 x i8> %D) nounwind {
-; CHECK-LABEL: test_v16i8_post_imm_st1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st1.16b { v0, v1, v2 }, [x0], #48
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_imm_st1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st1.16b { v0, v1, v2 }, [x0], #48
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_imm_st1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10773,13 +10770,13 @@ define ptr @test_v16i8_post_imm_st1x3(ptr %A, ptr %ptr, <16 x i8> %B, <16 x i8>
 }
 
 define ptr @test_v16i8_post_reg_st1x3(ptr %A, ptr %ptr, <16 x i8> %B, <16 x i8> %C, <16 x i8> %D, i64 %inc) nounwind {
-; CHECK-LABEL: test_v16i8_post_reg_st1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st1.16b { v0, v1, v2 }, [x0], x2
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_reg_st1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st1.16b { v0, v1, v2 }, [x0], x2
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_reg_st1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10799,13 +10796,13 @@ declare void @llvm.aarch64.neon.st1x3.v16i8.p0(<16 x i8>, <16 x i8>, <16 x i8>,
 
 
 define ptr @test_v8i8_post_imm_st1x3(ptr %A, ptr %ptr, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D) nounwind {
-; CHECK-LABEL: test_v8i8_post_imm_st1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    st1.8b { v0, v1, v2 }, [x0], #24
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_imm_st1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    st1.8b { v0, v1, v2 }, [x0], #24
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_imm_st1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10822,13 +10819,13 @@ define ptr @test_v8i8_post_imm_st1x3(ptr %A, ptr %ptr, <8 x i8> %B, <8 x i8> %C,
 }
 
 define ptr @test_v8i8_post_reg_st1x3(ptr %A, ptr %ptr, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D, i64 %inc) nounwind {
-; CHECK-LABEL: test_v8i8_post_reg_st1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    st1.8b { v0, v1, v2 }, [x0], x2
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_reg_st1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    st1.8b { v0, v1, v2 }, [x0], x2
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_reg_st1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10848,13 +10845,13 @@ declare void @llvm.aarch64.neon.st1x3.v8i8.p0(<8 x i8>, <8 x i8>, <8 x i8>, ptr)
 
 
 define ptr @test_v8i16_post_imm_st1x3(ptr %A, ptr %ptr, <8 x i16> %B, <8 x i16> %C, <8 x i16> %D) nounwind {
-; CHECK-LABEL: test_v8i16_post_imm_st1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st1.8h { v0, v1, v2 }, [x0], #48
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_imm_st1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st1.8h { v0, v1, v2 }, [x0], #48
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_imm_st1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10871,14 +10868,14 @@ define ptr @test_v8i16_post_imm_st1x3(ptr %A, ptr %ptr, <8 x i16> %B, <8 x i16>
 }
 
 define ptr @test_v8i16_post_reg_st1x3(ptr %A, ptr %ptr, <8 x i16> %B, <8 x i16> %C, <8 x i16> %D, i64 %inc) nounwind {
-; CHECK-LABEL: test_v8i16_post_reg_st1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st1.8h { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_reg_st1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st1.8h { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_reg_st1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10898,13 +10895,13 @@ declare void @llvm.aarch64.neon.st1x3.v8i16.p0(<8 x i16>, <8 x i16>, <8 x i16>,
 
 
 define ptr @test_v4i16_post_imm_st1x3(ptr %A, ptr %ptr, <4 x i16> %B, <4 x i16> %C, <4 x i16> %D) nounwind {
-; CHECK-LABEL: test_v4i16_post_imm_st1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    st1.4h { v0, v1, v2 }, [x0], #24
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_imm_st1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    st1.4h { v0, v1, v2 }, [x0], #24
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_imm_st1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10921,14 +10918,14 @@ define ptr @test_v4i16_post_imm_st1x3(ptr %A, ptr %ptr, <4 x i16> %B, <4 x i16>
 }
 
 define ptr @test_v4i16_post_reg_st1x3(ptr %A, ptr %ptr, <4 x i16> %B, <4 x i16> %C, <4 x i16> %D, i64 %inc) nounwind {
-; CHECK-LABEL: test_v4i16_post_reg_st1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    st1.4h { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_reg_st1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    st1.4h { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_reg_st1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10948,13 +10945,13 @@ declare void @llvm.aarch64.neon.st1x3.v4i16.p0(<4 x i16>, <4 x i16>, <4 x i16>,
 
 
 define ptr @test_v4i32_post_imm_st1x3(ptr %A, ptr %ptr, <4 x i32> %B, <4 x i32> %C, <4 x i32> %D) nounwind {
-; CHECK-LABEL: test_v4i32_post_imm_st1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st1.4s { v0, v1, v2 }, [x0], #48
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_imm_st1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st1.4s { v0, v1, v2 }, [x0], #48
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_imm_st1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10971,14 +10968,14 @@ define ptr @test_v4i32_post_imm_st1x3(ptr %A, ptr %ptr, <4 x i32> %B, <4 x i32>
 }
 
 define ptr @test_v4i32_post_reg_st1x3(ptr %A, ptr %ptr, <4 x i32> %B, <4 x i32> %C, <4 x i32> %D, i64 %inc) nounwind {
-; CHECK-LABEL: test_v4i32_post_reg_st1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st1.4s { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_reg_st1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st1.4s { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_reg_st1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -10998,13 +10995,13 @@ declare void @llvm.aarch64.neon.st1x3.v4i32.p0(<4 x i32>, <4 x i32>, <4 x i32>,
 
 
 define ptr @test_v2i32_post_imm_st1x3(ptr %A, ptr %ptr, <2 x i32> %B, <2 x i32> %C, <2 x i32> %D) nounwind {
-; CHECK-LABEL: test_v2i32_post_imm_st1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    st1.2s { v0, v1, v2 }, [x0], #24
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_imm_st1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    st1.2s { v0, v1, v2 }, [x0], #24
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_imm_st1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11021,14 +11018,14 @@ define ptr @test_v2i32_post_imm_st1x3(ptr %A, ptr %ptr, <2 x i32> %B, <2 x i32>
 }
 
 define ptr @test_v2i32_post_reg_st1x3(ptr %A, ptr %ptr, <2 x i32> %B, <2 x i32> %C, <2 x i32> %D, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2i32_post_reg_st1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    st1.2s { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_reg_st1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    st1.2s { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_reg_st1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11048,13 +11045,13 @@ declare void @llvm.aarch64.neon.st1x3.v2i32.p0(<2 x i32>, <2 x i32>, <2 x i32>,
 
 
 define ptr @test_v2i64_post_imm_st1x3(ptr %A, ptr %ptr, <2 x i64> %B, <2 x i64> %C, <2 x i64> %D) nounwind {
-; CHECK-LABEL: test_v2i64_post_imm_st1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st1.2d { v0, v1, v2 }, [x0], #48
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_imm_st1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st1.2d { v0, v1, v2 }, [x0], #48
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_imm_st1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11071,14 +11068,14 @@ define ptr @test_v2i64_post_imm_st1x3(ptr %A, ptr %ptr, <2 x i64> %B, <2 x i64>
 }
 
 define ptr @test_v2i64_post_reg_st1x3(ptr %A, ptr %ptr, <2 x i64> %B, <2 x i64> %C, <2 x i64> %D, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2i64_post_reg_st1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st1.2d { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_reg_st1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st1.2d { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_reg_st1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11098,13 +11095,13 @@ declare void @llvm.aarch64.neon.st1x3.v2i64.p0(<2 x i64>, <2 x i64>, <2 x i64>,
 
 
 define ptr @test_v1i64_post_imm_st1x3(ptr %A, ptr %ptr, <1 x i64> %B, <1 x i64> %C, <1 x i64> %D) nounwind {
-; CHECK-LABEL: test_v1i64_post_imm_st1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    st1.1d { v0, v1, v2 }, [x0], #24
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_imm_st1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    st1.1d { v0, v1, v2 }, [x0], #24
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_imm_st1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11121,14 +11118,14 @@ define ptr @test_v1i64_post_imm_st1x3(ptr %A, ptr %ptr, <1 x i64> %B, <1 x i64>
 }
 
 define ptr @test_v1i64_post_reg_st1x3(ptr %A, ptr %ptr, <1 x i64> %B, <1 x i64> %C, <1 x i64> %D, i64 %inc) nounwind {
-; CHECK-LABEL: test_v1i64_post_reg_st1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    st1.1d { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_reg_st1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    st1.1d { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_reg_st1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11148,13 +11145,13 @@ declare void @llvm.aarch64.neon.st1x3.v1i64.p0(<1 x i64>, <1 x i64>, <1 x i64>,
 
 
 define ptr @test_v4f32_post_imm_st1x3(ptr %A, ptr %ptr, <4 x float> %B, <4 x float> %C, <4 x float> %D) nounwind {
-; CHECK-LABEL: test_v4f32_post_imm_st1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st1.4s { v0, v1, v2 }, [x0], #48
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_imm_st1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st1.4s { v0, v1, v2 }, [x0], #48
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_imm_st1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11171,14 +11168,14 @@ define ptr @test_v4f32_post_imm_st1x3(ptr %A, ptr %ptr, <4 x float> %B, <4 x flo
 }
 
 define ptr @test_v4f32_post_reg_st1x3(ptr %A, ptr %ptr, <4 x float> %B, <4 x float> %C, <4 x float> %D, i64 %inc) nounwind {
-; CHECK-LABEL: test_v4f32_post_reg_st1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st1.4s { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_reg_st1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st1.4s { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_reg_st1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11198,13 +11195,13 @@ declare void @llvm.aarch64.neon.st1x3.v4f32.p0(<4 x float>, <4 x float>, <4 x fl
 
 
 define ptr @test_v2f32_post_imm_st1x3(ptr %A, ptr %ptr, <2 x float> %B, <2 x float> %C, <2 x float> %D) nounwind {
-; CHECK-LABEL: test_v2f32_post_imm_st1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    st1.2s { v0, v1, v2 }, [x0], #24
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_imm_st1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    st1.2s { v0, v1, v2 }, [x0], #24
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_imm_st1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11221,14 +11218,14 @@ define ptr @test_v2f32_post_imm_st1x3(ptr %A, ptr %ptr, <2 x float> %B, <2 x flo
 }
 
 define ptr @test_v2f32_post_reg_st1x3(ptr %A, ptr %ptr, <2 x float> %B, <2 x float> %C, <2 x float> %D, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2f32_post_reg_st1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    st1.2s { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_reg_st1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    st1.2s { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_reg_st1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11248,13 +11245,13 @@ declare void @llvm.aarch64.neon.st1x3.v2f32.p0(<2 x float>, <2 x float>, <2 x fl
 
 
 define ptr @test_v2f64_post_imm_st1x3(ptr %A, ptr %ptr, <2 x double> %B, <2 x double> %C, <2 x double> %D) nounwind {
-; CHECK-LABEL: test_v2f64_post_imm_st1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st1.2d { v0, v1, v2 }, [x0], #48
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_imm_st1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st1.2d { v0, v1, v2 }, [x0], #48
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_imm_st1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11271,14 +11268,14 @@ define ptr @test_v2f64_post_imm_st1x3(ptr %A, ptr %ptr, <2 x double> %B, <2 x do
 }
 
 define ptr @test_v2f64_post_reg_st1x3(ptr %A, ptr %ptr, <2 x double> %B, <2 x double> %C, <2 x double> %D, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2f64_post_reg_st1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st1.2d { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_reg_st1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st1.2d { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_reg_st1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11298,13 +11295,13 @@ declare void @llvm.aarch64.neon.st1x3.v2f64.p0(<2 x double>, <2 x double>, <2 x
 
 
 define ptr @test_v1f64_post_imm_st1x3(ptr %A, ptr %ptr, <1 x double> %B, <1 x double> %C, <1 x double> %D) nounwind {
-; CHECK-LABEL: test_v1f64_post_imm_st1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    st1.1d { v0, v1, v2 }, [x0], #24
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_imm_st1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    st1.1d { v0, v1, v2 }, [x0], #24
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_imm_st1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11321,14 +11318,14 @@ define ptr @test_v1f64_post_imm_st1x3(ptr %A, ptr %ptr, <1 x double> %B, <1 x do
 }
 
 define ptr @test_v1f64_post_reg_st1x3(ptr %A, ptr %ptr, <1 x double> %B, <1 x double> %C, <1 x double> %D, i64 %inc) nounwind {
-; CHECK-LABEL: test_v1f64_post_reg_st1x3:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT:    st1.1d { v0, v1, v2 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_reg_st1x3:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
+; SDAG-NEXT:    st1.1d { v0, v1, v2 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_reg_st1x3:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11348,14 +11345,14 @@ declare void @llvm.aarch64.neon.st1x3.v1f64.p0(<1 x double>, <1 x double>, <1 x
 
 
 define ptr @test_v16i8_post_imm_st1x4(ptr %A, ptr %ptr, <16 x i8> %B, <16 x i8> %C, <16 x i8> %D, <16 x i8> %E) nounwind {
-; CHECK-LABEL: test_v16i8_post_imm_st1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st1.16b { v0, v1, v2, v3 }, [x0], #64
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_imm_st1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st1.16b { v0, v1, v2, v3 }, [x0], #64
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_imm_st1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11373,14 +11370,14 @@ define ptr @test_v16i8_post_imm_st1x4(ptr %A, ptr %ptr, <16 x i8> %B, <16 x i8>
 }
 
 define ptr @test_v16i8_post_reg_st1x4(ptr %A, ptr %ptr, <16 x i8> %B, <16 x i8> %C, <16 x i8> %D, <16 x i8> %E, i64 %inc) nounwind {
-; CHECK-LABEL: test_v16i8_post_reg_st1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st1.16b { v0, v1, v2, v3 }, [x0], x2
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_reg_st1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st1.16b { v0, v1, v2, v3 }, [x0], x2
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_reg_st1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11401,14 +11398,14 @@ declare void @llvm.aarch64.neon.st1x4.v16i8.p0(<16 x i8>, <16 x i8>, <16 x i8>,
 
 
 define ptr @test_v8i8_post_imm_st1x4(ptr %A, ptr %ptr, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D, <8 x i8> %E) nounwind {
-; CHECK-LABEL: test_v8i8_post_imm_st1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    st1.8b { v0, v1, v2, v3 }, [x0], #32
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_imm_st1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    st1.8b { v0, v1, v2, v3 }, [x0], #32
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_imm_st1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11426,14 +11423,14 @@ define ptr @test_v8i8_post_imm_st1x4(ptr %A, ptr %ptr, <8 x i8> %B, <8 x i8> %C,
 }
 
 define ptr @test_v8i8_post_reg_st1x4(ptr %A, ptr %ptr, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D, <8 x i8> %E, i64 %inc) nounwind {
-; CHECK-LABEL: test_v8i8_post_reg_st1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    st1.8b { v0, v1, v2, v3 }, [x0], x2
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_reg_st1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    st1.8b { v0, v1, v2, v3 }, [x0], x2
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_reg_st1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11454,14 +11451,14 @@ declare void @llvm.aarch64.neon.st1x4.v8i8.p0(<8 x i8>, <8 x i8>, <8 x i8>, <8 x
 
 
 define ptr @test_v8i16_post_imm_st1x4(ptr %A, ptr %ptr, <8 x i16> %B, <8 x i16> %C, <8 x i16> %D, <8 x i16> %E) nounwind {
-; CHECK-LABEL: test_v8i16_post_imm_st1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st1.8h { v0, v1, v2, v3 }, [x0], #64
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_imm_st1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st1.8h { v0, v1, v2, v3 }, [x0], #64
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_imm_st1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11479,15 +11476,15 @@ define ptr @test_v8i16_post_imm_st1x4(ptr %A, ptr %ptr, <8 x i16> %B, <8 x i16>
 }
 
 define ptr @test_v8i16_post_reg_st1x4(ptr %A, ptr %ptr, <8 x i16> %B, <8 x i16> %C, <8 x i16> %D, <8 x i16> %E, i64 %inc) nounwind {
-; CHECK-LABEL: test_v8i16_post_reg_st1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st1.8h { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_reg_st1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st1.8h { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_reg_st1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11508,14 +11505,14 @@ declare void @llvm.aarch64.neon.st1x4.v8i16.p0(<8 x i16>, <8 x i16>, <8 x i16>,
 
 
 define ptr @test_v4i16_post_imm_st1x4(ptr %A, ptr %ptr, <4 x i16> %B, <4 x i16> %C, <4 x i16> %D, <4 x i16> %E) nounwind {
-; CHECK-LABEL: test_v4i16_post_imm_st1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    st1.4h { v0, v1, v2, v3 }, [x0], #32
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_imm_st1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    st1.4h { v0, v1, v2, v3 }, [x0], #32
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_imm_st1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11533,15 +11530,15 @@ define ptr @test_v4i16_post_imm_st1x4(ptr %A, ptr %ptr, <4 x i16> %B, <4 x i16>
 }
 
 define ptr @test_v4i16_post_reg_st1x4(ptr %A, ptr %ptr, <4 x i16> %B, <4 x i16> %C, <4 x i16> %D, <4 x i16> %E, i64 %inc) nounwind {
-; CHECK-LABEL: test_v4i16_post_reg_st1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    st1.4h { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_reg_st1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    st1.4h { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_reg_st1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11562,14 +11559,14 @@ declare void @llvm.aarch64.neon.st1x4.v4i16.p0(<4 x i16>, <4 x i16>, <4 x i16>,<
 
 
 define ptr @test_v4i32_post_imm_st1x4(ptr %A, ptr %ptr, <4 x i32> %B, <4 x i32> %C, <4 x i32> %D, <4 x i32> %E) nounwind {
-; CHECK-LABEL: test_v4i32_post_imm_st1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st1.4s { v0, v1, v2, v3 }, [x0], #64
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_imm_st1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st1.4s { v0, v1, v2, v3 }, [x0], #64
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_imm_st1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11587,15 +11584,15 @@ define ptr @test_v4i32_post_imm_st1x4(ptr %A, ptr %ptr, <4 x i32> %B, <4 x i32>
 }
 
 define ptr @test_v4i32_post_reg_st1x4(ptr %A, ptr %ptr, <4 x i32> %B, <4 x i32> %C, <4 x i32> %D, <4 x i32> %E, i64 %inc) nounwind {
-; CHECK-LABEL: test_v4i32_post_reg_st1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st1.4s { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_reg_st1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st1.4s { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_reg_st1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11616,14 +11613,14 @@ declare void @llvm.aarch64.neon.st1x4.v4i32.p0(<4 x i32>, <4 x i32>, <4 x i32>,<
 
 
 define ptr @test_v2i32_post_imm_st1x4(ptr %A, ptr %ptr, <2 x i32> %B, <2 x i32> %C, <2 x i32> %D, <2 x i32> %E) nounwind {
-; CHECK-LABEL: test_v2i32_post_imm_st1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    st1.2s { v0, v1, v2, v3 }, [x0], #32
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_imm_st1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    st1.2s { v0, v1, v2, v3 }, [x0], #32
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_imm_st1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11641,15 +11638,15 @@ define ptr @test_v2i32_post_imm_st1x4(ptr %A, ptr %ptr, <2 x i32> %B, <2 x i32>
 }
 
 define ptr @test_v2i32_post_reg_st1x4(ptr %A, ptr %ptr, <2 x i32> %B, <2 x i32> %C, <2 x i32> %D, <2 x i32> %E, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2i32_post_reg_st1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    st1.2s { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_reg_st1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    st1.2s { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_reg_st1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11670,14 +11667,14 @@ declare void @llvm.aarch64.neon.st1x4.v2i32.p0(<2 x i32>, <2 x i32>, <2 x i32>,
 
 
 define ptr @test_v2i64_post_imm_st1x4(ptr %A, ptr %ptr, <2 x i64> %B, <2 x i64> %C, <2 x i64> %D, <2 x i64> %E) nounwind {
-; CHECK-LABEL: test_v2i64_post_imm_st1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st1.2d { v0, v1, v2, v3 }, [x0], #64
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_imm_st1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st1.2d { v0, v1, v2, v3 }, [x0], #64
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_imm_st1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11695,15 +11692,15 @@ define ptr @test_v2i64_post_imm_st1x4(ptr %A, ptr %ptr, <2 x i64> %B, <2 x i64>
 }
 
 define ptr @test_v2i64_post_reg_st1x4(ptr %A, ptr %ptr, <2 x i64> %B, <2 x i64> %C, <2 x i64> %D, <2 x i64> %E, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2i64_post_reg_st1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st1.2d { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_reg_st1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st1.2d { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_reg_st1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11724,14 +11721,14 @@ declare void @llvm.aarch64.neon.st1x4.v2i64.p0(<2 x i64>, <2 x i64>, <2 x i64>,<
 
 
 define ptr @test_v1i64_post_imm_st1x4(ptr %A, ptr %ptr, <1 x i64> %B, <1 x i64> %C, <1 x i64> %D, <1 x i64> %E) nounwind {
-; CHECK-LABEL: test_v1i64_post_imm_st1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    st1.1d { v0, v1, v2, v3 }, [x0], #32
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_imm_st1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    st1.1d { v0, v1, v2, v3 }, [x0], #32
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_imm_st1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11749,15 +11746,15 @@ define ptr @test_v1i64_post_imm_st1x4(ptr %A, ptr %ptr, <1 x i64> %B, <1 x i64>
 }
 
 define ptr @test_v1i64_post_reg_st1x4(ptr %A, ptr %ptr, <1 x i64> %B, <1 x i64> %C, <1 x i64> %D, <1 x i64> %E, i64 %inc) nounwind {
-; CHECK-LABEL: test_v1i64_post_reg_st1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    st1.1d { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_reg_st1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    st1.1d { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_reg_st1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11778,14 +11775,14 @@ declare void @llvm.aarch64.neon.st1x4.v1i64.p0(<1 x i64>, <1 x i64>, <1 x i64>,<
 
 
 define ptr @test_v4f32_post_imm_st1x4(ptr %A, ptr %ptr, <4 x float> %B, <4 x float> %C, <4 x float> %D, <4 x float> %E) nounwind {
-; CHECK-LABEL: test_v4f32_post_imm_st1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st1.4s { v0, v1, v2, v3 }, [x0], #64
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_imm_st1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st1.4s { v0, v1, v2, v3 }, [x0], #64
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_imm_st1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11803,15 +11800,15 @@ define ptr @test_v4f32_post_imm_st1x4(ptr %A, ptr %ptr, <4 x float> %B, <4 x flo
 }
 
 define ptr @test_v4f32_post_reg_st1x4(ptr %A, ptr %ptr, <4 x float> %B, <4 x float> %C, <4 x float> %D, <4 x float> %E, i64 %inc) nounwind {
-; CHECK-LABEL: test_v4f32_post_reg_st1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st1.4s { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_reg_st1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st1.4s { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_reg_st1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11832,14 +11829,14 @@ declare void @llvm.aarch64.neon.st1x4.v4f32.p0(<4 x float>, <4 x float>, <4 x fl
 
 
 define ptr @test_v2f32_post_imm_st1x4(ptr %A, ptr %ptr, <2 x float> %B, <2 x float> %C, <2 x float> %D, <2 x float> %E) nounwind {
-; CHECK-LABEL: test_v2f32_post_imm_st1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    st1.2s { v0, v1, v2, v3 }, [x0], #32
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_imm_st1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    st1.2s { v0, v1, v2, v3 }, [x0], #32
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_imm_st1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11857,15 +11854,15 @@ define ptr @test_v2f32_post_imm_st1x4(ptr %A, ptr %ptr, <2 x float> %B, <2 x flo
 }
 
 define ptr @test_v2f32_post_reg_st1x4(ptr %A, ptr %ptr, <2 x float> %B, <2 x float> %C, <2 x float> %D, <2 x float> %E, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2f32_post_reg_st1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    st1.2s { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_reg_st1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    st1.2s { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_reg_st1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11886,14 +11883,14 @@ declare void @llvm.aarch64.neon.st1x4.v2f32.p0(<2 x float>, <2 x float>, <2 x fl
 
 
 define ptr @test_v2f64_post_imm_st1x4(ptr %A, ptr %ptr, <2 x double> %B, <2 x double> %C, <2 x double> %D, <2 x double> %E) nounwind {
-; CHECK-LABEL: test_v2f64_post_imm_st1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st1.2d { v0, v1, v2, v3 }, [x0], #64
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_imm_st1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st1.2d { v0, v1, v2, v3 }, [x0], #64
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_imm_st1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11911,15 +11908,15 @@ define ptr @test_v2f64_post_imm_st1x4(ptr %A, ptr %ptr, <2 x double> %B, <2 x do
 }
 
 define ptr @test_v2f64_post_reg_st1x4(ptr %A, ptr %ptr, <2 x double> %B, <2 x double> %C, <2 x double> %D, <2 x double> %E, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2f64_post_reg_st1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st1.2d { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_reg_st1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st1.2d { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_reg_st1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11940,14 +11937,14 @@ declare void @llvm.aarch64.neon.st1x4.v2f64.p0(<2 x double>, <2 x double>, <2 x
 
 
 define ptr @test_v1f64_post_imm_st1x4(ptr %A, ptr %ptr, <1 x double> %B, <1 x double> %C, <1 x double> %D, <1 x double> %E) nounwind {
-; CHECK-LABEL: test_v1f64_post_imm_st1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    st1.1d { v0, v1, v2, v3 }, [x0], #32
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_imm_st1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    st1.1d { v0, v1, v2, v3 }, [x0], #32
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_imm_st1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11965,15 +11962,15 @@ define ptr @test_v1f64_post_imm_st1x4(ptr %A, ptr %ptr, <1 x double> %B, <1 x do
 }
 
 define ptr @test_v1f64_post_reg_st1x4(ptr %A, ptr %ptr, <1 x double> %B, <1 x double> %C, <1 x double> %D, <1 x double> %E, i64 %inc) nounwind {
-; CHECK-LABEL: test_v1f64_post_reg_st1x4:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
-; CHECK-NEXT:    st1.1d { v0, v1, v2, v3 }, [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_reg_st1x4:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $d0_d1_d2_d3 def $d0_d1_d2_d3
+; SDAG-NEXT:    st1.1d { v0, v1, v2, v3 }, [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_reg_st1x4:
 ; CHECK-GISEL:       ; %bb.0:
@@ -11993,12 +11990,12 @@ define ptr @test_v1f64_post_reg_st1x4(ptr %A, ptr %ptr, <1 x double> %B, <1 x do
 declare void @llvm.aarch64.neon.st1x4.v1f64.p0(<1 x double>, <1 x double>, <1 x double>, <1 x double>, ptr)
 
 define ptr @test_v16i8_post_imm_st2lane(ptr %A, ptr %ptr, <16 x i8> %B, <16 x i8> %C) nounwind {
-; CHECK-LABEL: test_v16i8_post_imm_st2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st2.b { v0, v1 }[0], [x0], #2
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_imm_st2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st2.b { v0, v1 }[0], [x0], #2
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_imm_st2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12014,12 +12011,12 @@ define ptr @test_v16i8_post_imm_st2lane(ptr %A, ptr %ptr, <16 x i8> %B, <16 x i8
 }
 
 define ptr @test_v16i8_post_reg_st2lane(ptr %A, ptr %ptr, <16 x i8> %B, <16 x i8> %C, i64 %inc) nounwind {
-; CHECK-LABEL: test_v16i8_post_reg_st2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st2.b { v0, v1 }[0], [x0], x2
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_reg_st2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st2.b { v0, v1 }[0], [x0], x2
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_reg_st2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12038,12 +12035,12 @@ declare void @llvm.aarch64.neon.st2lane.v16i8.p0(<16 x i8>, <16 x i8>, i64, ptr)
 
 
 define ptr @test_v8i8_post_imm_st2lane(ptr %A, ptr %ptr, <8 x i8> %B, <8 x i8> %C) nounwind {
-; CHECK-LABEL: test_v8i8_post_imm_st2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st2.b { v0, v1 }[0], [x0], #2
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_imm_st2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st2.b { v0, v1 }[0], [x0], #2
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_imm_st2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12059,12 +12056,12 @@ define ptr @test_v8i8_post_imm_st2lane(ptr %A, ptr %ptr, <8 x i8> %B, <8 x i8> %
 }
 
 define ptr @test_v8i8_post_reg_st2lane(ptr %A, ptr %ptr, <8 x i8> %B, <8 x i8> %C, i64 %inc) nounwind {
-; CHECK-LABEL: test_v8i8_post_reg_st2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st2.b { v0, v1 }[0], [x0], x2
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_reg_st2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st2.b { v0, v1 }[0], [x0], x2
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_reg_st2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12083,12 +12080,12 @@ declare void @llvm.aarch64.neon.st2lane.v8i8.p0(<8 x i8>, <8 x i8>, i64, ptr)
 
 
 define ptr @test_v8i16_post_imm_st2lane(ptr %A, ptr %ptr, <8 x i16> %B, <8 x i16> %C) nounwind {
-; CHECK-LABEL: test_v8i16_post_imm_st2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st2.h { v0, v1 }[0], [x0], #4
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_imm_st2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st2.h { v0, v1 }[0], [x0], #4
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_imm_st2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12104,13 +12101,13 @@ define ptr @test_v8i16_post_imm_st2lane(ptr %A, ptr %ptr, <8 x i16> %B, <8 x i16
 }
 
 define ptr @test_v8i16_post_reg_st2lane(ptr %A, ptr %ptr, <8 x i16> %B, <8 x i16> %C, i64 %inc) nounwind {
-; CHECK-LABEL: test_v8i16_post_reg_st2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st2.h { v0, v1 }[0], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_reg_st2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st2.h { v0, v1 }[0], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_reg_st2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12129,12 +12126,12 @@ declare void @llvm.aarch64.neon.st2lane.v8i16.p0(<8 x i16>, <8 x i16>, i64, ptr)
 
 
 define ptr @test_v4i16_post_imm_st2lane(ptr %A, ptr %ptr, <4 x i16> %B, <4 x i16> %C) nounwind {
-; CHECK-LABEL: test_v4i16_post_imm_st2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st2.h { v0, v1 }[0], [x0], #4
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_imm_st2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st2.h { v0, v1 }[0], [x0], #4
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_imm_st2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12150,13 +12147,13 @@ define ptr @test_v4i16_post_imm_st2lane(ptr %A, ptr %ptr, <4 x i16> %B, <4 x i16
 }
 
 define ptr @test_v4i16_post_reg_st2lane(ptr %A, ptr %ptr, <4 x i16> %B, <4 x i16> %C, i64 %inc) nounwind {
-; CHECK-LABEL: test_v4i16_post_reg_st2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st2.h { v0, v1 }[0], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_reg_st2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st2.h { v0, v1 }[0], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_reg_st2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12175,12 +12172,12 @@ declare void @llvm.aarch64.neon.st2lane.v4i16.p0(<4 x i16>, <4 x i16>, i64, ptr)
 
 
 define ptr @test_v4i32_post_imm_st2lane(ptr %A, ptr %ptr, <4 x i32> %B, <4 x i32> %C) nounwind {
-; CHECK-LABEL: test_v4i32_post_imm_st2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st2.s { v0, v1 }[0], [x0], #8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_imm_st2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st2.s { v0, v1 }[0], [x0], #8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_imm_st2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12196,13 +12193,13 @@ define ptr @test_v4i32_post_imm_st2lane(ptr %A, ptr %ptr, <4 x i32> %B, <4 x i32
 }
 
 define ptr @test_v4i32_post_reg_st2lane(ptr %A, ptr %ptr, <4 x i32> %B, <4 x i32> %C, i64 %inc) nounwind {
-; CHECK-LABEL: test_v4i32_post_reg_st2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st2.s { v0, v1 }[0], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_reg_st2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st2.s { v0, v1 }[0], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_reg_st2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12221,12 +12218,12 @@ declare void @llvm.aarch64.neon.st2lane.v4i32.p0(<4 x i32>, <4 x i32>, i64, ptr)
 
 
 define ptr @test_v2i32_post_imm_st2lane(ptr %A, ptr %ptr, <2 x i32> %B, <2 x i32> %C) nounwind {
-; CHECK-LABEL: test_v2i32_post_imm_st2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st2.s { v0, v1 }[0], [x0], #8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_imm_st2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st2.s { v0, v1 }[0], [x0], #8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_imm_st2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12242,13 +12239,13 @@ define ptr @test_v2i32_post_imm_st2lane(ptr %A, ptr %ptr, <2 x i32> %B, <2 x i32
 }
 
 define ptr @test_v2i32_post_reg_st2lane(ptr %A, ptr %ptr, <2 x i32> %B, <2 x i32> %C, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2i32_post_reg_st2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st2.s { v0, v1 }[0], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_reg_st2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st2.s { v0, v1 }[0], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_reg_st2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12267,12 +12264,12 @@ declare void @llvm.aarch64.neon.st2lane.v2i32.p0(<2 x i32>, <2 x i32>, i64, ptr)
 
 
 define ptr @test_v2i64_post_imm_st2lane(ptr %A, ptr %ptr, <2 x i64> %B, <2 x i64> %C) nounwind {
-; CHECK-LABEL: test_v2i64_post_imm_st2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st2.d { v0, v1 }[0], [x0], #16
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_imm_st2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st2.d { v0, v1 }[0], [x0], #16
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_imm_st2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12288,13 +12285,13 @@ define ptr @test_v2i64_post_imm_st2lane(ptr %A, ptr %ptr, <2 x i64> %B, <2 x i64
 }
 
 define ptr @test_v2i64_post_reg_st2lane(ptr %A, ptr %ptr, <2 x i64> %B, <2 x i64> %C, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2i64_post_reg_st2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st2.d { v0, v1 }[0], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_reg_st2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st2.d { v0, v1 }[0], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_reg_st2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12313,12 +12310,12 @@ declare void @llvm.aarch64.neon.st2lane.v2i64.p0(<2 x i64>, <2 x i64>, i64, ptr)
 
 
 define ptr @test_v1i64_post_imm_st2lane(ptr %A, ptr %ptr, <1 x i64> %B, <1 x i64> %C) nounwind {
-; CHECK-LABEL: test_v1i64_post_imm_st2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st2.d { v0, v1 }[0], [x0], #16
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_imm_st2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st2.d { v0, v1 }[0], [x0], #16
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_imm_st2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12334,13 +12331,13 @@ define ptr @test_v1i64_post_imm_st2lane(ptr %A, ptr %ptr, <1 x i64> %B, <1 x i64
 }
 
 define ptr @test_v1i64_post_reg_st2lane(ptr %A, ptr %ptr, <1 x i64> %B, <1 x i64> %C, i64 %inc) nounwind {
-; CHECK-LABEL: test_v1i64_post_reg_st2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st2.d { v0, v1 }[0], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_reg_st2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st2.d { v0, v1 }[0], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_reg_st2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12359,12 +12356,12 @@ declare void @llvm.aarch64.neon.st2lane.v1i64.p0(<1 x i64>, <1 x i64>, i64, ptr)
 
 
 define ptr @test_v4f32_post_imm_st2lane(ptr %A, ptr %ptr, <4 x float> %B, <4 x float> %C) nounwind {
-; CHECK-LABEL: test_v4f32_post_imm_st2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st2.s { v0, v1 }[0], [x0], #8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_imm_st2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st2.s { v0, v1 }[0], [x0], #8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_imm_st2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12380,13 +12377,13 @@ define ptr @test_v4f32_post_imm_st2lane(ptr %A, ptr %ptr, <4 x float> %B, <4 x f
 }
 
 define ptr @test_v4f32_post_reg_st2lane(ptr %A, ptr %ptr, <4 x float> %B, <4 x float> %C, i64 %inc) nounwind {
-; CHECK-LABEL: test_v4f32_post_reg_st2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st2.s { v0, v1 }[0], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_reg_st2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st2.s { v0, v1 }[0], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_reg_st2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12405,12 +12402,12 @@ declare void @llvm.aarch64.neon.st2lane.v4f32.p0(<4 x float>, <4 x float>, i64,
 
 
 define ptr @test_v2f32_post_imm_st2lane(ptr %A, ptr %ptr, <2 x float> %B, <2 x float> %C) nounwind {
-; CHECK-LABEL: test_v2f32_post_imm_st2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st2.s { v0, v1 }[0], [x0], #8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_imm_st2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st2.s { v0, v1 }[0], [x0], #8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_imm_st2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12426,13 +12423,13 @@ define ptr @test_v2f32_post_imm_st2lane(ptr %A, ptr %ptr, <2 x float> %B, <2 x f
 }
 
 define ptr @test_v2f32_post_reg_st2lane(ptr %A, ptr %ptr, <2 x float> %B, <2 x float> %C, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2f32_post_reg_st2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st2.s { v0, v1 }[0], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_reg_st2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st2.s { v0, v1 }[0], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_reg_st2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12451,12 +12448,12 @@ declare void @llvm.aarch64.neon.st2lane.v2f32.p0(<2 x float>, <2 x float>, i64,
 
 
 define ptr @test_v2f64_post_imm_st2lane(ptr %A, ptr %ptr, <2 x double> %B, <2 x double> %C) nounwind {
-; CHECK-LABEL: test_v2f64_post_imm_st2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st2.d { v0, v1 }[0], [x0], #16
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_imm_st2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st2.d { v0, v1 }[0], [x0], #16
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_imm_st2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12472,13 +12469,13 @@ define ptr @test_v2f64_post_imm_st2lane(ptr %A, ptr %ptr, <2 x double> %B, <2 x
 }
 
 define ptr @test_v2f64_post_reg_st2lane(ptr %A, ptr %ptr, <2 x double> %B, <2 x double> %C, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2f64_post_reg_st2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st2.d { v0, v1 }[0], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_reg_st2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st2.d { v0, v1 }[0], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_reg_st2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12497,12 +12494,12 @@ declare void @llvm.aarch64.neon.st2lane.v2f64.p0(<2 x double>, <2 x double>, i64
 
 
 define ptr @test_v1f64_post_imm_st2lane(ptr %A, ptr %ptr, <1 x double> %B, <1 x double> %C) nounwind {
-; CHECK-LABEL: test_v1f64_post_imm_st2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st2.d { v0, v1 }[0], [x0], #16
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_imm_st2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st2.d { v0, v1 }[0], [x0], #16
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_imm_st2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12518,13 +12515,13 @@ define ptr @test_v1f64_post_imm_st2lane(ptr %A, ptr %ptr, <1 x double> %B, <1 x
 }
 
 define ptr @test_v1f64_post_reg_st2lane(ptr %A, ptr %ptr, <1 x double> %B, <1 x double> %C, i64 %inc) nounwind {
-; CHECK-LABEL: test_v1f64_post_reg_st2lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
-; CHECK-NEXT:    st2.d { v0, v1 }[0], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_reg_st2lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1
+; SDAG-NEXT:    st2.d { v0, v1 }[0], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_reg_st2lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12543,13 +12540,13 @@ declare void @llvm.aarch64.neon.st2lane.v1f64.p0(<1 x double>, <1 x double>, i64
 
 
 define ptr @test_v16i8_post_imm_st3lane(ptr %A, ptr %ptr, <16 x i8> %B, <16 x i8> %C, <16 x i8> %D) nounwind {
-; CHECK-LABEL: test_v16i8_post_imm_st3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st3.b { v0, v1, v2 }[0], [x0], #3
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_imm_st3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st3.b { v0, v1, v2 }[0], [x0], #3
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_imm_st3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12566,13 +12563,13 @@ define ptr @test_v16i8_post_imm_st3lane(ptr %A, ptr %ptr, <16 x i8> %B, <16 x i8
 }
 
 define ptr @test_v16i8_post_reg_st3lane(ptr %A, ptr %ptr, <16 x i8> %B, <16 x i8> %C, <16 x i8> %D, i64 %inc) nounwind {
-; CHECK-LABEL: test_v16i8_post_reg_st3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st3.b { v0, v1, v2 }[0], [x0], x2
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_reg_st3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st3.b { v0, v1, v2 }[0], [x0], x2
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_reg_st3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12592,13 +12589,13 @@ declare void @llvm.aarch64.neon.st3lane.v16i8.p0(<16 x i8>, <16 x i8>, <16 x i8>
 
 
 define ptr @test_v8i8_post_imm_st3lane(ptr %A, ptr %ptr, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D) nounwind {
-; CHECK-LABEL: test_v8i8_post_imm_st3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st3.b { v0, v1, v2 }[0], [x0], #3
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_imm_st3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st3.b { v0, v1, v2 }[0], [x0], #3
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_imm_st3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12615,13 +12612,13 @@ define ptr @test_v8i8_post_imm_st3lane(ptr %A, ptr %ptr, <8 x i8> %B, <8 x i8> %
 }
 
 define ptr @test_v8i8_post_reg_st3lane(ptr %A, ptr %ptr, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D, i64 %inc) nounwind {
-; CHECK-LABEL: test_v8i8_post_reg_st3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st3.b { v0, v1, v2 }[0], [x0], x2
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_reg_st3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st3.b { v0, v1, v2 }[0], [x0], x2
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_reg_st3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12641,13 +12638,13 @@ declare void @llvm.aarch64.neon.st3lane.v8i8.p0(<8 x i8>, <8 x i8>, <8 x i8>, i6
 
 
 define ptr @test_v8i16_post_imm_st3lane(ptr %A, ptr %ptr, <8 x i16> %B, <8 x i16> %C, <8 x i16> %D) nounwind {
-; CHECK-LABEL: test_v8i16_post_imm_st3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st3.h { v0, v1, v2 }[0], [x0], #6
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_imm_st3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st3.h { v0, v1, v2 }[0], [x0], #6
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_imm_st3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12664,14 +12661,14 @@ define ptr @test_v8i16_post_imm_st3lane(ptr %A, ptr %ptr, <8 x i16> %B, <8 x i16
 }
 
 define ptr @test_v8i16_post_reg_st3lane(ptr %A, ptr %ptr, <8 x i16> %B, <8 x i16> %C, <8 x i16> %D, i64 %inc) nounwind {
-; CHECK-LABEL: test_v8i16_post_reg_st3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st3.h { v0, v1, v2 }[0], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_reg_st3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st3.h { v0, v1, v2 }[0], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_reg_st3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12691,13 +12688,13 @@ declare void @llvm.aarch64.neon.st3lane.v8i16.p0(<8 x i16>, <8 x i16>, <8 x i16>
 
 
 define ptr @test_v4i16_post_imm_st3lane(ptr %A, ptr %ptr, <4 x i16> %B, <4 x i16> %C, <4 x i16> %D) nounwind {
-; CHECK-LABEL: test_v4i16_post_imm_st3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st3.h { v0, v1, v2 }[0], [x0], #6
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_imm_st3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st3.h { v0, v1, v2 }[0], [x0], #6
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_imm_st3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12714,14 +12711,14 @@ define ptr @test_v4i16_post_imm_st3lane(ptr %A, ptr %ptr, <4 x i16> %B, <4 x i16
 }
 
 define ptr @test_v4i16_post_reg_st3lane(ptr %A, ptr %ptr, <4 x i16> %B, <4 x i16> %C, <4 x i16> %D, i64 %inc) nounwind {
-; CHECK-LABEL: test_v4i16_post_reg_st3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st3.h { v0, v1, v2 }[0], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_reg_st3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st3.h { v0, v1, v2 }[0], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_reg_st3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12741,13 +12738,13 @@ declare void @llvm.aarch64.neon.st3lane.v4i16.p0(<4 x i16>, <4 x i16>, <4 x i16>
 
 
 define ptr @test_v4i32_post_imm_st3lane(ptr %A, ptr %ptr, <4 x i32> %B, <4 x i32> %C, <4 x i32> %D) nounwind {
-; CHECK-LABEL: test_v4i32_post_imm_st3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st3.s { v0, v1, v2 }[0], [x0], #12
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_imm_st3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st3.s { v0, v1, v2 }[0], [x0], #12
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_imm_st3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12764,14 +12761,14 @@ define ptr @test_v4i32_post_imm_st3lane(ptr %A, ptr %ptr, <4 x i32> %B, <4 x i32
 }
 
 define ptr @test_v4i32_post_reg_st3lane(ptr %A, ptr %ptr, <4 x i32> %B, <4 x i32> %C, <4 x i32> %D, i64 %inc) nounwind {
-; CHECK-LABEL: test_v4i32_post_reg_st3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st3.s { v0, v1, v2 }[0], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_reg_st3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st3.s { v0, v1, v2 }[0], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_reg_st3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12791,13 +12788,13 @@ declare void @llvm.aarch64.neon.st3lane.v4i32.p0(<4 x i32>, <4 x i32>, <4 x i32>
 
 
 define ptr @test_v2i32_post_imm_st3lane(ptr %A, ptr %ptr, <2 x i32> %B, <2 x i32> %C, <2 x i32> %D) nounwind {
-; CHECK-LABEL: test_v2i32_post_imm_st3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st3.s { v0, v1, v2 }[0], [x0], #12
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_imm_st3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st3.s { v0, v1, v2 }[0], [x0], #12
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_imm_st3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12814,14 +12811,14 @@ define ptr @test_v2i32_post_imm_st3lane(ptr %A, ptr %ptr, <2 x i32> %B, <2 x i32
 }
 
 define ptr @test_v2i32_post_reg_st3lane(ptr %A, ptr %ptr, <2 x i32> %B, <2 x i32> %C, <2 x i32> %D, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2i32_post_reg_st3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st3.s { v0, v1, v2 }[0], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_reg_st3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st3.s { v0, v1, v2 }[0], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_reg_st3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12841,13 +12838,13 @@ declare void @llvm.aarch64.neon.st3lane.v2i32.p0(<2 x i32>, <2 x i32>, <2 x i32>
 
 
 define ptr @test_v2i64_post_imm_st3lane(ptr %A, ptr %ptr, <2 x i64> %B, <2 x i64> %C, <2 x i64> %D) nounwind {
-; CHECK-LABEL: test_v2i64_post_imm_st3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st3.d { v0, v1, v2 }[0], [x0], #24
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_imm_st3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st3.d { v0, v1, v2 }[0], [x0], #24
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_imm_st3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12864,14 +12861,14 @@ define ptr @test_v2i64_post_imm_st3lane(ptr %A, ptr %ptr, <2 x i64> %B, <2 x i64
 }
 
 define ptr @test_v2i64_post_reg_st3lane(ptr %A, ptr %ptr, <2 x i64> %B, <2 x i64> %C, <2 x i64> %D, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2i64_post_reg_st3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st3.d { v0, v1, v2 }[0], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_reg_st3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st3.d { v0, v1, v2 }[0], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_reg_st3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12891,13 +12888,13 @@ declare void @llvm.aarch64.neon.st3lane.v2i64.p0(<2 x i64>, <2 x i64>, <2 x i64>
 
 
 define ptr @test_v1i64_post_imm_st3lane(ptr %A, ptr %ptr, <1 x i64> %B, <1 x i64> %C, <1 x i64> %D) nounwind {
-; CHECK-LABEL: test_v1i64_post_imm_st3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st3.d { v0, v1, v2 }[0], [x0], #24
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_imm_st3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st3.d { v0, v1, v2 }[0], [x0], #24
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_imm_st3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12914,14 +12911,14 @@ define ptr @test_v1i64_post_imm_st3lane(ptr %A, ptr %ptr, <1 x i64> %B, <1 x i64
 }
 
 define ptr @test_v1i64_post_reg_st3lane(ptr %A, ptr %ptr, <1 x i64> %B, <1 x i64> %C, <1 x i64> %D, i64 %inc) nounwind {
-; CHECK-LABEL: test_v1i64_post_reg_st3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st3.d { v0, v1, v2 }[0], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_reg_st3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st3.d { v0, v1, v2 }[0], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_reg_st3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12941,13 +12938,13 @@ declare void @llvm.aarch64.neon.st3lane.v1i64.p0(<1 x i64>, <1 x i64>, <1 x i64>
 
 
 define ptr @test_v4f32_post_imm_st3lane(ptr %A, ptr %ptr, <4 x float> %B, <4 x float> %C, <4 x float> %D) nounwind {
-; CHECK-LABEL: test_v4f32_post_imm_st3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st3.s { v0, v1, v2 }[0], [x0], #12
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_imm_st3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st3.s { v0, v1, v2 }[0], [x0], #12
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_imm_st3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12964,14 +12961,14 @@ define ptr @test_v4f32_post_imm_st3lane(ptr %A, ptr %ptr, <4 x float> %B, <4 x f
 }
 
 define ptr @test_v4f32_post_reg_st3lane(ptr %A, ptr %ptr, <4 x float> %B, <4 x float> %C, <4 x float> %D, i64 %inc) nounwind {
-; CHECK-LABEL: test_v4f32_post_reg_st3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st3.s { v0, v1, v2 }[0], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_reg_st3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st3.s { v0, v1, v2 }[0], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_reg_st3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -12991,13 +12988,13 @@ declare void @llvm.aarch64.neon.st3lane.v4f32.p0(<4 x float>, <4 x float>, <4 x
 
 
 define ptr @test_v2f32_post_imm_st3lane(ptr %A, ptr %ptr, <2 x float> %B, <2 x float> %C, <2 x float> %D) nounwind {
-; CHECK-LABEL: test_v2f32_post_imm_st3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st3.s { v0, v1, v2 }[0], [x0], #12
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_imm_st3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st3.s { v0, v1, v2 }[0], [x0], #12
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_imm_st3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -13014,14 +13011,14 @@ define ptr @test_v2f32_post_imm_st3lane(ptr %A, ptr %ptr, <2 x float> %B, <2 x f
 }
 
 define ptr @test_v2f32_post_reg_st3lane(ptr %A, ptr %ptr, <2 x float> %B, <2 x float> %C, <2 x float> %D, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2f32_post_reg_st3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st3.s { v0, v1, v2 }[0], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_reg_st3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st3.s { v0, v1, v2 }[0], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_reg_st3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -13041,13 +13038,13 @@ declare void @llvm.aarch64.neon.st3lane.v2f32.p0(<2 x float>, <2 x float>, <2 x
 
 
 define ptr @test_v2f64_post_imm_st3lane(ptr %A, ptr %ptr, <2 x double> %B, <2 x double> %C, <2 x double> %D) nounwind {
-; CHECK-LABEL: test_v2f64_post_imm_st3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st3.d { v0, v1, v2 }[0], [x0], #24
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_imm_st3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st3.d { v0, v1, v2 }[0], [x0], #24
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_imm_st3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -13064,14 +13061,14 @@ define ptr @test_v2f64_post_imm_st3lane(ptr %A, ptr %ptr, <2 x double> %B, <2 x
 }
 
 define ptr @test_v2f64_post_reg_st3lane(ptr %A, ptr %ptr, <2 x double> %B, <2 x double> %C, <2 x double> %D, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2f64_post_reg_st3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st3.d { v0, v1, v2 }[0], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_reg_st3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st3.d { v0, v1, v2 }[0], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_reg_st3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -13091,13 +13088,13 @@ declare void @llvm.aarch64.neon.st3lane.v2f64.p0(<2 x double>, <2 x double>, <2
 
 
 define ptr @test_v1f64_post_imm_st3lane(ptr %A, ptr %ptr, <1 x double> %B, <1 x double> %C, <1 x double> %D) nounwind {
-; CHECK-LABEL: test_v1f64_post_imm_st3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st3.d { v0, v1, v2 }[0], [x0], #24
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_imm_st3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st3.d { v0, v1, v2 }[0], [x0], #24
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_imm_st3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -13114,14 +13111,14 @@ define ptr @test_v1f64_post_imm_st3lane(ptr %A, ptr %ptr, <1 x double> %B, <1 x
 }
 
 define ptr @test_v1f64_post_reg_st3lane(ptr %A, ptr %ptr, <1 x double> %B, <1 x double> %C, <1 x double> %D, i64 %inc) nounwind {
-; CHECK-LABEL: test_v1f64_post_reg_st3lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT:    st3.d { v0, v1, v2 }[0], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_reg_st3lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2 def $q0_q1_q2
+; SDAG-NEXT:    st3.d { v0, v1, v2 }[0], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_reg_st3lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -13141,14 +13138,14 @@ declare void @llvm.aarch64.neon.st3lane.v1f64.p0(<1 x double>, <1 x double>, <1
 
 
 define ptr @test_v16i8_post_imm_st4lane(ptr %A, ptr %ptr, <16 x i8> %B, <16 x i8> %C, <16 x i8> %D, <16 x i8> %E) nounwind {
-; CHECK-LABEL: test_v16i8_post_imm_st4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st4.b { v0, v1, v2, v3 }[0], [x0], #4
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_imm_st4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st4.b { v0, v1, v2, v3 }[0], [x0], #4
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_imm_st4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -13166,14 +13163,14 @@ define ptr @test_v16i8_post_imm_st4lane(ptr %A, ptr %ptr, <16 x i8> %B, <16 x i8
 }
 
 define ptr @test_v16i8_post_reg_st4lane(ptr %A, ptr %ptr, <16 x i8> %B, <16 x i8> %C, <16 x i8> %D, <16 x i8> %E, i64 %inc) nounwind {
-; CHECK-LABEL: test_v16i8_post_reg_st4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st4.b { v0, v1, v2, v3 }[0], [x0], x2
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_reg_st4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st4.b { v0, v1, v2, v3 }[0], [x0], x2
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_reg_st4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -13194,14 +13191,14 @@ declare void @llvm.aarch64.neon.st4lane.v16i8.p0(<16 x i8>, <16 x i8>, <16 x i8>
 
 
 define ptr @test_v8i8_post_imm_st4lane(ptr %A, ptr %ptr, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D, <8 x i8> %E) nounwind {
-; CHECK-LABEL: test_v8i8_post_imm_st4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st4.b { v0, v1, v2, v3 }[0], [x0], #4
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_imm_st4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st4.b { v0, v1, v2, v3 }[0], [x0], #4
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_imm_st4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -13219,14 +13216,14 @@ define ptr @test_v8i8_post_imm_st4lane(ptr %A, ptr %ptr, <8 x i8> %B, <8 x i8> %
 }
 
 define ptr @test_v8i8_post_reg_st4lane(ptr %A, ptr %ptr, <8 x i8> %B, <8 x i8> %C, <8 x i8> %D, <8 x i8> %E, i64 %inc) nounwind {
-; CHECK-LABEL: test_v8i8_post_reg_st4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st4.b { v0, v1, v2, v3 }[0], [x0], x2
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_reg_st4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st4.b { v0, v1, v2, v3 }[0], [x0], x2
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_reg_st4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -13247,14 +13244,14 @@ declare void @llvm.aarch64.neon.st4lane.v8i8.p0(<8 x i8>, <8 x i8>, <8 x i8>, <8
 
 
 define ptr @test_v8i16_post_imm_st4lane(ptr %A, ptr %ptr, <8 x i16> %B, <8 x i16> %C, <8 x i16> %D, <8 x i16> %E) nounwind {
-; CHECK-LABEL: test_v8i16_post_imm_st4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st4.h { v0, v1, v2, v3 }[0], [x0], #8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_imm_st4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st4.h { v0, v1, v2, v3 }[0], [x0], #8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_imm_st4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -13272,15 +13269,15 @@ define ptr @test_v8i16_post_imm_st4lane(ptr %A, ptr %ptr, <8 x i16> %B, <8 x i16
 }
 
 define ptr @test_v8i16_post_reg_st4lane(ptr %A, ptr %ptr, <8 x i16> %B, <8 x i16> %C, <8 x i16> %D, <8 x i16> %E, i64 %inc) nounwind {
-; CHECK-LABEL: test_v8i16_post_reg_st4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st4.h { v0, v1, v2, v3 }[0], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_reg_st4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st4.h { v0, v1, v2, v3 }[0], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_reg_st4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -13301,14 +13298,14 @@ declare void @llvm.aarch64.neon.st4lane.v8i16.p0(<8 x i16>, <8 x i16>, <8 x i16>
 
 
 define ptr @test_v4i16_post_imm_st4lane(ptr %A, ptr %ptr, <4 x i16> %B, <4 x i16> %C, <4 x i16> %D, <4 x i16> %E) nounwind {
-; CHECK-LABEL: test_v4i16_post_imm_st4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st4.h { v0, v1, v2, v3 }[0], [x0], #8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_imm_st4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st4.h { v0, v1, v2, v3 }[0], [x0], #8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_imm_st4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -13326,15 +13323,15 @@ define ptr @test_v4i16_post_imm_st4lane(ptr %A, ptr %ptr, <4 x i16> %B, <4 x i16
 }
 
 define ptr @test_v4i16_post_reg_st4lane(ptr %A, ptr %ptr, <4 x i16> %B, <4 x i16> %C, <4 x i16> %D, <4 x i16> %E, i64 %inc) nounwind {
-; CHECK-LABEL: test_v4i16_post_reg_st4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st4.h { v0, v1, v2, v3 }[0], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_reg_st4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st4.h { v0, v1, v2, v3 }[0], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_reg_st4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -13355,14 +13352,14 @@ declare void @llvm.aarch64.neon.st4lane.v4i16.p0(<4 x i16>, <4 x i16>, <4 x i16>
 
 
 define ptr @test_v4i32_post_imm_st4lane(ptr %A, ptr %ptr, <4 x i32> %B, <4 x i32> %C, <4 x i32> %D, <4 x i32> %E) nounwind {
-; CHECK-LABEL: test_v4i32_post_imm_st4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st4.s { v0, v1, v2, v3 }[0], [x0], #16
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_imm_st4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st4.s { v0, v1, v2, v3 }[0], [x0], #16
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_imm_st4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -13380,15 +13377,15 @@ define ptr @test_v4i32_post_imm_st4lane(ptr %A, ptr %ptr, <4 x i32> %B, <4 x i32
 }
 
 define ptr @test_v4i32_post_reg_st4lane(ptr %A, ptr %ptr, <4 x i32> %B, <4 x i32> %C, <4 x i32> %D, <4 x i32> %E, i64 %inc) nounwind {
-; CHECK-LABEL: test_v4i32_post_reg_st4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st4.s { v0, v1, v2, v3 }[0], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_reg_st4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st4.s { v0, v1, v2, v3 }[0], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_reg_st4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -13409,14 +13406,14 @@ declare void @llvm.aarch64.neon.st4lane.v4i32.p0(<4 x i32>, <4 x i32>, <4 x i32>
 
 
 define ptr @test_v2i32_post_imm_st4lane(ptr %A, ptr %ptr, <2 x i32> %B, <2 x i32> %C, <2 x i32> %D, <2 x i32> %E) nounwind {
-; CHECK-LABEL: test_v2i32_post_imm_st4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st4.s { v0, v1, v2, v3 }[0], [x0], #16
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_imm_st4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st4.s { v0, v1, v2, v3 }[0], [x0], #16
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_imm_st4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -13434,15 +13431,15 @@ define ptr @test_v2i32_post_imm_st4lane(ptr %A, ptr %ptr, <2 x i32> %B, <2 x i32
 }
 
 define ptr @test_v2i32_post_reg_st4lane(ptr %A, ptr %ptr, <2 x i32> %B, <2 x i32> %C, <2 x i32> %D, <2 x i32> %E, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2i32_post_reg_st4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st4.s { v0, v1, v2, v3 }[0], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_reg_st4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st4.s { v0, v1, v2, v3 }[0], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_reg_st4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -13463,14 +13460,14 @@ declare void @llvm.aarch64.neon.st4lane.v2i32.p0(<2 x i32>, <2 x i32>, <2 x i32>
 
 
 define ptr @test_v2i64_post_imm_st4lane(ptr %A, ptr %ptr, <2 x i64> %B, <2 x i64> %C, <2 x i64> %D, <2 x i64> %E) nounwind {
-; CHECK-LABEL: test_v2i64_post_imm_st4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st4.d { v0, v1, v2, v3 }[0], [x0], #32
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_imm_st4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st4.d { v0, v1, v2, v3 }[0], [x0], #32
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_imm_st4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -13488,15 +13485,15 @@ define ptr @test_v2i64_post_imm_st4lane(ptr %A, ptr %ptr, <2 x i64> %B, <2 x i64
 }
 
 define ptr @test_v2i64_post_reg_st4lane(ptr %A, ptr %ptr, <2 x i64> %B, <2 x i64> %C, <2 x i64> %D, <2 x i64> %E, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2i64_post_reg_st4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st4.d { v0, v1, v2, v3 }[0], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_reg_st4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st4.d { v0, v1, v2, v3 }[0], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_reg_st4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -13517,14 +13514,14 @@ declare void @llvm.aarch64.neon.st4lane.v2i64.p0(<2 x i64>, <2 x i64>, <2 x i64>
 
 
 define ptr @test_v1i64_post_imm_st4lane(ptr %A, ptr %ptr, <1 x i64> %B, <1 x i64> %C, <1 x i64> %D, <1 x i64> %E) nounwind {
-; CHECK-LABEL: test_v1i64_post_imm_st4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st4.d { v0, v1, v2, v3 }[0], [x0], #32
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_imm_st4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st4.d { v0, v1, v2, v3 }[0], [x0], #32
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_imm_st4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -13542,15 +13539,15 @@ define ptr @test_v1i64_post_imm_st4lane(ptr %A, ptr %ptr, <1 x i64> %B, <1 x i64
 }
 
 define ptr @test_v1i64_post_reg_st4lane(ptr %A, ptr %ptr, <1 x i64> %B, <1 x i64> %C, <1 x i64> %D, <1 x i64> %E, i64 %inc) nounwind {
-; CHECK-LABEL: test_v1i64_post_reg_st4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st4.d { v0, v1, v2, v3 }[0], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1i64_post_reg_st4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st4.d { v0, v1, v2, v3 }[0], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1i64_post_reg_st4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -13571,14 +13568,14 @@ declare void @llvm.aarch64.neon.st4lane.v1i64.p0(<1 x i64>, <1 x i64>, <1 x i64>
 
 
 define ptr @test_v4f32_post_imm_st4lane(ptr %A, ptr %ptr, <4 x float> %B, <4 x float> %C, <4 x float> %D, <4 x float> %E) nounwind {
-; CHECK-LABEL: test_v4f32_post_imm_st4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st4.s { v0, v1, v2, v3 }[0], [x0], #16
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_imm_st4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st4.s { v0, v1, v2, v3 }[0], [x0], #16
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_imm_st4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -13596,15 +13593,15 @@ define ptr @test_v4f32_post_imm_st4lane(ptr %A, ptr %ptr, <4 x float> %B, <4 x f
 }
 
 define ptr @test_v4f32_post_reg_st4lane(ptr %A, ptr %ptr, <4 x float> %B, <4 x float> %C, <4 x float> %D, <4 x float> %E, i64 %inc) nounwind {
-; CHECK-LABEL: test_v4f32_post_reg_st4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st4.s { v0, v1, v2, v3 }[0], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_reg_st4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st4.s { v0, v1, v2, v3 }[0], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_reg_st4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -13625,14 +13622,14 @@ declare void @llvm.aarch64.neon.st4lane.v4f32.p0(<4 x float>, <4 x float>, <4 x
 
 
 define ptr @test_v2f32_post_imm_st4lane(ptr %A, ptr %ptr, <2 x float> %B, <2 x float> %C, <2 x float> %D, <2 x float> %E) nounwind {
-; CHECK-LABEL: test_v2f32_post_imm_st4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st4.s { v0, v1, v2, v3 }[0], [x0], #16
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_imm_st4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st4.s { v0, v1, v2, v3 }[0], [x0], #16
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_imm_st4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -13650,15 +13647,15 @@ define ptr @test_v2f32_post_imm_st4lane(ptr %A, ptr %ptr, <2 x float> %B, <2 x f
 }
 
 define ptr @test_v2f32_post_reg_st4lane(ptr %A, ptr %ptr, <2 x float> %B, <2 x float> %C, <2 x float> %D, <2 x float> %E, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2f32_post_reg_st4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st4.s { v0, v1, v2, v3 }[0], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_reg_st4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st4.s { v0, v1, v2, v3 }[0], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_reg_st4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -13679,14 +13676,14 @@ declare void @llvm.aarch64.neon.st4lane.v2f32.p0(<2 x float>, <2 x float>, <2 x
 
 
 define ptr @test_v2f64_post_imm_st4lane(ptr %A, ptr %ptr, <2 x double> %B, <2 x double> %C, <2 x double> %D, <2 x double> %E) nounwind {
-; CHECK-LABEL: test_v2f64_post_imm_st4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st4.d { v0, v1, v2, v3 }[0], [x0], #32
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_imm_st4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st4.d { v0, v1, v2, v3 }[0], [x0], #32
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_imm_st4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -13704,15 +13701,15 @@ define ptr @test_v2f64_post_imm_st4lane(ptr %A, ptr %ptr, <2 x double> %B, <2 x
 }
 
 define ptr @test_v2f64_post_reg_st4lane(ptr %A, ptr %ptr, <2 x double> %B, <2 x double> %C, <2 x double> %D, <2 x double> %E, i64 %inc) nounwind {
-; CHECK-LABEL: test_v2f64_post_reg_st4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st4.d { v0, v1, v2, v3 }[0], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_reg_st4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st4.d { v0, v1, v2, v3 }[0], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_reg_st4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -13733,14 +13730,14 @@ declare void @llvm.aarch64.neon.st4lane.v2f64.p0(<2 x double>, <2 x double>, <2
 
 
 define ptr @test_v1f64_post_imm_st4lane(ptr %A, ptr %ptr, <1 x double> %B, <1 x double> %C, <1 x double> %D, <1 x double> %E) nounwind {
-; CHECK-LABEL: test_v1f64_post_imm_st4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st4.d { v0, v1, v2, v3 }[0], [x0], #32
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_imm_st4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st4.d { v0, v1, v2, v3 }[0], [x0], #32
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_imm_st4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -13758,15 +13755,15 @@ define ptr @test_v1f64_post_imm_st4lane(ptr %A, ptr %ptr, <1 x double> %B, <1 x
 }
 
 define ptr @test_v1f64_post_reg_st4lane(ptr %A, ptr %ptr, <1 x double> %B, <1 x double> %C, <1 x double> %D, <1 x double> %E, i64 %inc) nounwind {
-; CHECK-LABEL: test_v1f64_post_reg_st4lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; CHECK-NEXT:    st4.d { v0, v1, v2, v3 }[0], [x0], x8
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v1f64_post_reg_st4lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d3 killed $d3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ; kill: def $d2 killed $d2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d1 killed $d1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; SDAG-NEXT:    st4.d { v0, v1, v2, v3 }[0], [x0], x8
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v1f64_post_reg_st4lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -13786,16 +13783,17 @@ define ptr @test_v1f64_post_reg_st4lane(ptr %A, ptr %ptr, <1 x double> %B, <1 x
 declare void @llvm.aarch64.neon.st4lane.v1f64.p0(<1 x double>, <1 x double>, <1 x double>, <1 x double>, i64, ptr)
 
 define <16 x i8> @test_v16i8_post_imm_ld1r(ptr %bar, ptr %ptr) {
-; CHECK-LABEL: test_v16i8_post_imm_ld1r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1r.16b { v0 }, [x0], #1
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_imm_ld1r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1r.16b { v0 }, [x0], #1
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_imm_ld1r:
 ; CHECK-GISEL:       ; %bb.0:
-; CHECK-GISEL-NEXT:    ld1r.16b { v0 }, [x0], #1
-; CHECK-GISEL-NEXT:    str x0, [x1]
+; CHECK-GISEL-NEXT:    ld1r.16b { v0 }, [x0]
+; CHECK-GISEL-NEXT:    add x8, x0, #1
+; CHECK-GISEL-NEXT:    str x8, [x1]
 ; CHECK-GISEL-NEXT:    ret
   %tmp1 = load i8, ptr %bar
   %tmp2 = insertelement <16 x i8> <i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, i8 %tmp1, i32 0
@@ -13820,16 +13818,17 @@ define <16 x i8> @test_v16i8_post_imm_ld1r(ptr %bar, ptr %ptr) {
 }
 
 define <16 x i8> @test_v16i8_post_reg_ld1r(ptr %bar, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v16i8_post_reg_ld1r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1r.16b { v0 }, [x0], x2
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_reg_ld1r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1r.16b { v0 }, [x0], x2
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_reg_ld1r:
 ; CHECK-GISEL:       ; %bb.0:
-; CHECK-GISEL-NEXT:    ld1r.16b { v0 }, [x0], x2
-; CHECK-GISEL-NEXT:    str x0, [x1]
+; CHECK-GISEL-NEXT:    ld1r.16b { v0 }, [x0]
+; CHECK-GISEL-NEXT:    add x8, x0, x2
+; CHECK-GISEL-NEXT:    str x8, [x1]
 ; CHECK-GISEL-NEXT:    ret
   %tmp1 = load i8, ptr %bar
   %tmp2 = insertelement <16 x i8> <i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, i8 %tmp1, i32 0
@@ -13854,16 +13853,17 @@ define <16 x i8> @test_v16i8_post_reg_ld1r(ptr %bar, ptr %ptr, i64 %inc) {
 }
 
 define <8 x i8> @test_v8i8_post_imm_ld1r(ptr %bar, ptr %ptr) {
-; CHECK-LABEL: test_v8i8_post_imm_ld1r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1r.8b { v0 }, [x0], #1
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_imm_ld1r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1r.8b { v0 }, [x0], #1
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_imm_ld1r:
 ; CHECK-GISEL:       ; %bb.0:
-; CHECK-GISEL-NEXT:    ld1r.8b { v0 }, [x0], #1
-; CHECK-GISEL-NEXT:    str x0, [x1]
+; CHECK-GISEL-NEXT:    ld1r.8b { v0 }, [x0]
+; CHECK-GISEL-NEXT:    add x8, x0, #1
+; CHECK-GISEL-NEXT:    str x8, [x1]
 ; CHECK-GISEL-NEXT:    ret
   %tmp1 = load i8, ptr %bar
   %tmp2 = insertelement <8 x i8> <i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, i8 %tmp1, i32 0
@@ -13880,16 +13880,17 @@ define <8 x i8> @test_v8i8_post_imm_ld1r(ptr %bar, ptr %ptr) {
 }
 
 define <8 x i8> @test_v8i8_post_reg_ld1r(ptr %bar, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v8i8_post_reg_ld1r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1r.8b { v0 }, [x0], x2
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_reg_ld1r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1r.8b { v0 }, [x0], x2
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_reg_ld1r:
 ; CHECK-GISEL:       ; %bb.0:
-; CHECK-GISEL-NEXT:    ld1r.8b { v0 }, [x0], x2
-; CHECK-GISEL-NEXT:    str x0, [x1]
+; CHECK-GISEL-NEXT:    ld1r.8b { v0 }, [x0]
+; CHECK-GISEL-NEXT:    add x8, x0, x2
+; CHECK-GISEL-NEXT:    str x8, [x1]
 ; CHECK-GISEL-NEXT:    ret
   %tmp1 = load i8, ptr %bar
   %tmp2 = insertelement <8 x i8> <i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, i8 %tmp1, i32 0
@@ -13906,11 +13907,11 @@ define <8 x i8> @test_v8i8_post_reg_ld1r(ptr %bar, ptr %ptr, i64 %inc) {
 }
 
 define <8 x i16> @test_v8i16_post_imm_ld1r(ptr %bar, ptr %ptr) {
-; CHECK-LABEL: test_v8i16_post_imm_ld1r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1r.8h { v0 }, [x0], #2
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_imm_ld1r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1r.8h { v0 }, [x0], #2
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_imm_ld1r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -13933,12 +13934,12 @@ define <8 x i16> @test_v8i16_post_imm_ld1r(ptr %bar, ptr %ptr) {
 }
 
 define <8 x i16> @test_v8i16_post_reg_ld1r(ptr %bar, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v8i16_post_reg_ld1r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ld1r.8h { v0 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_reg_ld1r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ld1r.8h { v0 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_reg_ld1r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -13961,11 +13962,11 @@ define <8 x i16> @test_v8i16_post_reg_ld1r(ptr %bar, ptr %ptr, i64 %inc) {
 }
 
 define <4 x i16> @test_v4i16_post_imm_ld1r(ptr %bar, ptr %ptr) {
-; CHECK-LABEL: test_v4i16_post_imm_ld1r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1r.4h { v0 }, [x0], #2
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_imm_ld1r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1r.4h { v0 }, [x0], #2
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_imm_ld1r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -13984,12 +13985,12 @@ define <4 x i16> @test_v4i16_post_imm_ld1r(ptr %bar, ptr %ptr) {
 }
 
 define <4 x i16> @test_v4i16_post_reg_ld1r(ptr %bar, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v4i16_post_reg_ld1r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ld1r.4h { v0 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_reg_ld1r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ld1r.4h { v0 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_reg_ld1r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14008,11 +14009,11 @@ define <4 x i16> @test_v4i16_post_reg_ld1r(ptr %bar, ptr %ptr, i64 %inc) {
 }
 
 define <4 x i32> @test_v4i32_post_imm_ld1r(ptr %bar, ptr %ptr) {
-; CHECK-LABEL: test_v4i32_post_imm_ld1r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1r.4s { v0 }, [x0], #4
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_imm_ld1r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1r.4s { v0 }, [x0], #4
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_imm_ld1r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14031,12 +14032,12 @@ define <4 x i32> @test_v4i32_post_imm_ld1r(ptr %bar, ptr %ptr) {
 }
 
 define <4 x i32> @test_v4i32_post_reg_ld1r(ptr %bar, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v4i32_post_reg_ld1r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld1r.4s { v0 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_reg_ld1r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld1r.4s { v0 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_reg_ld1r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14055,11 +14056,11 @@ define <4 x i32> @test_v4i32_post_reg_ld1r(ptr %bar, ptr %ptr, i64 %inc) {
 }
 
 define <2 x i32> @test_v2i32_post_imm_ld1r(ptr %bar, ptr %ptr) {
-; CHECK-LABEL: test_v2i32_post_imm_ld1r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1r.2s { v0 }, [x0], #4
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_imm_ld1r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1r.2s { v0 }, [x0], #4
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_imm_ld1r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14076,12 +14077,12 @@ define <2 x i32> @test_v2i32_post_imm_ld1r(ptr %bar, ptr %ptr) {
 }
 
 define <2 x i32> @test_v2i32_post_reg_ld1r(ptr %bar, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v2i32_post_reg_ld1r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld1r.2s { v0 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_reg_ld1r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld1r.2s { v0 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_reg_ld1r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14098,11 +14099,11 @@ define <2 x i32> @test_v2i32_post_reg_ld1r(ptr %bar, ptr %ptr, i64 %inc) {
 }
 
 define <2 x i64> @test_v2i64_post_imm_ld1r(ptr %bar, ptr %ptr) {
-; CHECK-LABEL: test_v2i64_post_imm_ld1r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1r.2d { v0 }, [x0], #8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_imm_ld1r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1r.2d { v0 }, [x0], #8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_imm_ld1r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14119,12 +14120,12 @@ define <2 x i64> @test_v2i64_post_imm_ld1r(ptr %bar, ptr %ptr) {
 }
 
 define <2 x i64> @test_v2i64_post_reg_ld1r(ptr %bar, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v2i64_post_reg_ld1r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld1r.2d { v0 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_reg_ld1r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld1r.2d { v0 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_reg_ld1r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14141,11 +14142,11 @@ define <2 x i64> @test_v2i64_post_reg_ld1r(ptr %bar, ptr %ptr, i64 %inc) {
 }
 
 define <4 x float> @test_v4f32_post_imm_ld1r(ptr %bar, ptr %ptr) {
-; CHECK-LABEL: test_v4f32_post_imm_ld1r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1r.4s { v0 }, [x0], #4
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_imm_ld1r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1r.4s { v0 }, [x0], #4
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_imm_ld1r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14164,12 +14165,12 @@ define <4 x float> @test_v4f32_post_imm_ld1r(ptr %bar, ptr %ptr) {
 }
 
 define <4 x float> @test_v4f32_post_reg_ld1r(ptr %bar, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v4f32_post_reg_ld1r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld1r.4s { v0 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_reg_ld1r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld1r.4s { v0 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_reg_ld1r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14188,11 +14189,11 @@ define <4 x float> @test_v4f32_post_reg_ld1r(ptr %bar, ptr %ptr, i64 %inc) {
 }
 
 define <2 x float> @test_v2f32_post_imm_ld1r(ptr %bar, ptr %ptr) {
-; CHECK-LABEL: test_v2f32_post_imm_ld1r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1r.2s { v0 }, [x0], #4
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_imm_ld1r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1r.2s { v0 }, [x0], #4
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_imm_ld1r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14209,12 +14210,12 @@ define <2 x float> @test_v2f32_post_imm_ld1r(ptr %bar, ptr %ptr) {
 }
 
 define <2 x float> @test_v2f32_post_reg_ld1r(ptr %bar, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v2f32_post_reg_ld1r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld1r.2s { v0 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_reg_ld1r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld1r.2s { v0 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_reg_ld1r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14231,11 +14232,11 @@ define <2 x float> @test_v2f32_post_reg_ld1r(ptr %bar, ptr %ptr, i64 %inc) {
 }
 
 define <2 x double> @test_v2f64_post_imm_ld1r(ptr %bar, ptr %ptr) {
-; CHECK-LABEL: test_v2f64_post_imm_ld1r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1r.2d { v0 }, [x0], #8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_imm_ld1r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1r.2d { v0 }, [x0], #8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_imm_ld1r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14252,12 +14253,12 @@ define <2 x double> @test_v2f64_post_imm_ld1r(ptr %bar, ptr %ptr) {
 }
 
 define <2 x double> @test_v2f64_post_reg_ld1r(ptr %bar, ptr %ptr, i64 %inc) {
-; CHECK-LABEL: test_v2f64_post_reg_ld1r:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld1r.2d { v0 }, [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_reg_ld1r:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld1r.2d { v0 }, [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_reg_ld1r:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14274,11 +14275,11 @@ define <2 x double> @test_v2f64_post_reg_ld1r(ptr %bar, ptr %ptr, i64 %inc) {
 }
 
 define <16 x i8> @test_v16i8_post_imm_ld1lane(ptr %bar, ptr %ptr, <16 x i8> %A) {
-; CHECK-LABEL: test_v16i8_post_imm_ld1lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.b { v0 }[1], [x0], #1
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_imm_ld1lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.b { v0 }[1], [x0], #1
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_imm_ld1lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14295,11 +14296,11 @@ define <16 x i8> @test_v16i8_post_imm_ld1lane(ptr %bar, ptr %ptr, <16 x i8> %A)
 }
 
 define <16 x i8> @test_v16i8_post_reg_ld1lane(ptr %bar, ptr %ptr, i64 %inc, <16 x i8> %A) {
-; CHECK-LABEL: test_v16i8_post_reg_ld1lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.b { v0 }[1], [x0], x2
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v16i8_post_reg_ld1lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.b { v0 }[1], [x0], x2
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v16i8_post_reg_ld1lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14316,13 +14317,13 @@ define <16 x i8> @test_v16i8_post_reg_ld1lane(ptr %bar, ptr %ptr, i64 %inc, <16
 }
 
 define <8 x i8> @test_v8i8_post_imm_ld1lane(ptr %bar, ptr %ptr, <8 x i8> %A) {
-; CHECK-LABEL: test_v8i8_post_imm_ld1lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    ld1.b { v0 }[1], [x0], #1
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_imm_ld1lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 def $q0
+; SDAG-NEXT:    ld1.b { v0 }[1], [x0], #1
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_imm_ld1lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14341,13 +14342,13 @@ define <8 x i8> @test_v8i8_post_imm_ld1lane(ptr %bar, ptr %ptr, <8 x i8> %A) {
 }
 
 define <8 x i8> @test_v8i8_post_reg_ld1lane(ptr %bar, ptr %ptr, i64 %inc, <8 x i8> %A) {
-; CHECK-LABEL: test_v8i8_post_reg_ld1lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    ld1.b { v0 }[1], [x0], x2
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i8_post_reg_ld1lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 def $q0
+; SDAG-NEXT:    ld1.b { v0 }[1], [x0], x2
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i8_post_reg_ld1lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14366,11 +14367,11 @@ define <8 x i8> @test_v8i8_post_reg_ld1lane(ptr %bar, ptr %ptr, i64 %inc, <8 x i
 }
 
 define <8 x i16> @test_v8i16_post_imm_ld1lane(ptr %bar, ptr %ptr, <8 x i16> %A) {
-; CHECK-LABEL: test_v8i16_post_imm_ld1lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.h { v0 }[1], [x0], #2
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_imm_ld1lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.h { v0 }[1], [x0], #2
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_imm_ld1lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14387,12 +14388,12 @@ define <8 x i16> @test_v8i16_post_imm_ld1lane(ptr %bar, ptr %ptr, <8 x i16> %A)
 }
 
 define <8 x i16> @test_v8i16_post_reg_ld1lane(ptr %bar, ptr %ptr, i64 %inc, <8 x i16> %A) {
-; CHECK-LABEL: test_v8i16_post_reg_ld1lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ld1.h { v0 }[1], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v8i16_post_reg_ld1lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ld1.h { v0 }[1], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v8i16_post_reg_ld1lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14409,13 +14410,13 @@ define <8 x i16> @test_v8i16_post_reg_ld1lane(ptr %bar, ptr %ptr, i64 %inc, <8 x
 }
 
 define <4 x i16> @test_v4i16_post_imm_ld1lane(ptr %bar, ptr %ptr, <4 x i16> %A) {
-; CHECK-LABEL: test_v4i16_post_imm_ld1lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    ld1.h { v0 }[1], [x0], #2
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_imm_ld1lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 def $q0
+; SDAG-NEXT:    ld1.h { v0 }[1], [x0], #2
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_imm_ld1lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14434,14 +14435,14 @@ define <4 x i16> @test_v4i16_post_imm_ld1lane(ptr %bar, ptr %ptr, <4 x i16> %A)
 }
 
 define <4 x i16> @test_v4i16_post_reg_ld1lane(ptr %bar, ptr %ptr, i64 %inc, <4 x i16> %A) {
-; CHECK-LABEL: test_v4i16_post_reg_ld1lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    ld1.h { v0 }[1], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_reg_ld1lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 def $q0
+; SDAG-NEXT:    ld1.h { v0 }[1], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_reg_ld1lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14460,11 +14461,11 @@ define <4 x i16> @test_v4i16_post_reg_ld1lane(ptr %bar, ptr %ptr, i64 %inc, <4 x
 }
 
 define <4 x i32> @test_v4i32_post_imm_ld1lane(ptr %bar, ptr %ptr, <4 x i32> %A) {
-; CHECK-LABEL: test_v4i32_post_imm_ld1lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.s { v0 }[1], [x0], #4
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_imm_ld1lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.s { v0 }[1], [x0], #4
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_imm_ld1lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14481,12 +14482,12 @@ define <4 x i32> @test_v4i32_post_imm_ld1lane(ptr %bar, ptr %ptr, <4 x i32> %A)
 }
 
 define <4 x i32> @test_v4i32_post_reg_ld1lane(ptr %bar, ptr %ptr, i64 %inc, <4 x i32> %A) {
-; CHECK-LABEL: test_v4i32_post_reg_ld1lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld1.s { v0 }[1], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i32_post_reg_ld1lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld1.s { v0 }[1], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i32_post_reg_ld1lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14503,13 +14504,13 @@ define <4 x i32> @test_v4i32_post_reg_ld1lane(ptr %bar, ptr %ptr, i64 %inc, <4 x
 }
 
 define <2 x i32> @test_v2i32_post_imm_ld1lane(ptr %bar, ptr %ptr, <2 x i32> %A) {
-; CHECK-LABEL: test_v2i32_post_imm_ld1lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    ld1.s { v0 }[1], [x0], #4
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_imm_ld1lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 def $q0
+; SDAG-NEXT:    ld1.s { v0 }[1], [x0], #4
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_imm_ld1lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14528,14 +14529,14 @@ define <2 x i32> @test_v2i32_post_imm_ld1lane(ptr %bar, ptr %ptr, <2 x i32> %A)
 }
 
 define <2 x i32> @test_v2i32_post_reg_ld1lane(ptr %bar, ptr %ptr, i64 %inc, <2 x i32> %A) {
-; CHECK-LABEL: test_v2i32_post_reg_ld1lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    ld1.s { v0 }[1], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i32_post_reg_ld1lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 def $q0
+; SDAG-NEXT:    ld1.s { v0 }[1], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i32_post_reg_ld1lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14554,11 +14555,11 @@ define <2 x i32> @test_v2i32_post_reg_ld1lane(ptr %bar, ptr %ptr, i64 %inc, <2 x
 }
 
 define <2 x i64> @test_v2i64_post_imm_ld1lane(ptr %bar, ptr %ptr, <2 x i64> %A) {
-; CHECK-LABEL: test_v2i64_post_imm_ld1lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.d { v0 }[1], [x0], #8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_imm_ld1lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.d { v0 }[1], [x0], #8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_imm_ld1lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14575,12 +14576,12 @@ define <2 x i64> @test_v2i64_post_imm_ld1lane(ptr %bar, ptr %ptr, <2 x i64> %A)
 }
 
 define <2 x i64> @test_v2i64_post_reg_ld1lane(ptr %bar, ptr %ptr, i64 %inc, <2 x i64> %A) {
-; CHECK-LABEL: test_v2i64_post_reg_ld1lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld1.d { v0 }[1], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2i64_post_reg_ld1lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld1.d { v0 }[1], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2i64_post_reg_ld1lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14597,11 +14598,11 @@ define <2 x i64> @test_v2i64_post_reg_ld1lane(ptr %bar, ptr %ptr, i64 %inc, <2 x
 }
 
 define <4 x float> @test_v4f32_post_imm_ld1lane(ptr %bar, ptr %ptr, <4 x float> %A) {
-; CHECK-LABEL: test_v4f32_post_imm_ld1lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.s { v0 }[1], [x0], #4
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_imm_ld1lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.s { v0 }[1], [x0], #4
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_imm_ld1lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14618,12 +14619,12 @@ define <4 x float> @test_v4f32_post_imm_ld1lane(ptr %bar, ptr %ptr, <4 x float>
 }
 
 define <4 x float> @test_v4f32_post_reg_ld1lane(ptr %bar, ptr %ptr, i64 %inc, <4 x float> %A) {
-; CHECK-LABEL: test_v4f32_post_reg_ld1lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ld1.s { v0 }[1], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4f32_post_reg_ld1lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ld1.s { v0 }[1], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4f32_post_reg_ld1lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14640,13 +14641,13 @@ define <4 x float> @test_v4f32_post_reg_ld1lane(ptr %bar, ptr %ptr, i64 %inc, <4
 }
 
 define <2 x float> @test_v2f32_post_imm_ld1lane(ptr %bar, ptr %ptr, <2 x float> %A) {
-; CHECK-LABEL: test_v2f32_post_imm_ld1lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    ld1.s { v0 }[1], [x0], #4
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_imm_ld1lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 def $q0
+; SDAG-NEXT:    ld1.s { v0 }[1], [x0], #4
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_imm_ld1lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14665,14 +14666,14 @@ define <2 x float> @test_v2f32_post_imm_ld1lane(ptr %bar, ptr %ptr, <2 x float>
 }
 
 define <2 x float> @test_v2f32_post_reg_ld1lane(ptr %bar, ptr %ptr, i64 %inc, <2 x float> %A) {
-; CHECK-LABEL: test_v2f32_post_reg_ld1lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #2
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    ld1.s { v0 }[1], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f32_post_reg_ld1lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #2
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 def $q0
+; SDAG-NEXT:    ld1.s { v0 }[1], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f32_post_reg_ld1lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14691,11 +14692,11 @@ define <2 x float> @test_v2f32_post_reg_ld1lane(ptr %bar, ptr %ptr, i64 %inc, <2
 }
 
 define <2 x double> @test_v2f64_post_imm_ld1lane(ptr %bar, ptr %ptr, <2 x double> %A) {
-; CHECK-LABEL: test_v2f64_post_imm_ld1lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.d { v0 }[1], [x0], #8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_imm_ld1lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.d { v0 }[1], [x0], #8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_imm_ld1lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14712,12 +14713,12 @@ define <2 x double> @test_v2f64_post_imm_ld1lane(ptr %bar, ptr %ptr, <2 x double
 }
 
 define <2 x double> @test_v2f64_post_reg_ld1lane(ptr %bar, ptr %ptr, i64 %inc, <2 x double> %A) {
-; CHECK-LABEL: test_v2f64_post_reg_ld1lane:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #3
-; CHECK-NEXT:    ld1.d { v0 }[1], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v2f64_post_reg_ld1lane:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #3
+; SDAG-NEXT:    ld1.d { v0 }[1], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v2f64_post_reg_ld1lane:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14744,16 +14745,6 @@ define <4 x float> @test_v4f32_post_reg_ld1lane_dep_vec_on_load(ptr %bar, ptr %p
 ; CHECK-NEXT:    str x8, [x1]
 ; CHECK-NEXT:    mov.s v0[1], v1[0]
 ; CHECK-NEXT:    ret
-;
-; CHECK-GISEL-LABEL: test_v4f32_post_reg_ld1lane_dep_vec_on_load:
-; CHECK-GISEL:       ; %bb.0:
-; CHECK-GISEL-NEXT:    ldr s1, [x0]
-; CHECK-GISEL-NEXT:    str q0, [x3]
-; CHECK-GISEL-NEXT:    add x8, x0, x2, lsl #2
-; CHECK-GISEL-NEXT:    ldr q0, [x4]
-; CHECK-GISEL-NEXT:    str x8, [x1]
-; CHECK-GISEL-NEXT:    mov.s v0[1], v1[0]
-; CHECK-GISEL-NEXT:    ret
   %tmp1 = load float, ptr %bar
   store <4 x float> %vec, ptr %dep_ptr_1, align 16
   %A = load <4 x float>, ptr %dep_ptr_2, align 16
@@ -14771,19 +14762,19 @@ define <4 x float> @test_v4f32_post_reg_ld1lane_dep_vec_on_load(ptr %bar, ptr %p
 ; legalizer to run.  We achieve that using the ctpop.
 ; PR23265
 define <4 x i16> @test_v4i16_post_reg_ld1lane_forced_narrow(ptr %bar, ptr %ptr, i64 %inc, <4 x i16> %A, ptr %d) {
-; CHECK-LABEL: test_v4i16_post_reg_ld1lane_forced_narrow:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    lsl x8, x2, #1
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    ld1.h { v0 }[1], [x0], x8
-; CHECK-NEXT:    str x0, [x1]
-; CHECK-NEXT:    ldr d1, [x3]
-; CHECK-NEXT:    ; kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    cnt.8b v1, v1
-; CHECK-NEXT:    uaddlp.4h v1, v1
-; CHECK-NEXT:    uaddlp.2s v1, v1
-; CHECK-NEXT:    str d1, [x3]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_v4i16_post_reg_ld1lane_forced_narrow:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    lsl x8, x2, #1
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 def $q0
+; SDAG-NEXT:    ld1.h { v0 }[1], [x0], x8
+; SDAG-NEXT:    str x0, [x1]
+; SDAG-NEXT:    ldr d1, [x3]
+; SDAG-NEXT:    ; kill: def $d0 killed $d0 killed $q0
+; SDAG-NEXT:    cnt.8b v1, v1
+; SDAG-NEXT:    uaddlp.4h v1, v1
+; SDAG-NEXT:    uaddlp.2s v1, v1
+; SDAG-NEXT:    str d1, [x3]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_v4i16_post_reg_ld1lane_forced_narrow:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14812,15 +14803,15 @@ define <4 x i16> @test_v4i16_post_reg_ld1lane_forced_narrow(ptr %bar, ptr %ptr,
 declare <2 x i32> @llvm.ctpop.v2i32(<2 x i32>)
 
 define void @test_ld1lane_build(ptr %ptr0, ptr %ptr1, ptr %ptr2, ptr %ptr3, ptr %out) {
-; CHECK-LABEL: test_ld1lane_build:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ldr s0, [x2]
-; CHECK-NEXT:    ldr s1, [x0]
-; CHECK-NEXT:    ld1.s { v0 }[1], [x3]
-; CHECK-NEXT:    ld1.s { v1 }[1], [x1]
-; CHECK-NEXT:    sub.2s v0, v1, v0
-; CHECK-NEXT:    str d0, [x4]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_ld1lane_build:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ldr s0, [x2]
+; SDAG-NEXT:    ldr s1, [x0]
+; SDAG-NEXT:    ld1.s { v0 }[1], [x3]
+; SDAG-NEXT:    ld1.s { v1 }[1], [x1]
+; SDAG-NEXT:    sub.2s v0, v1, v0
+; SDAG-NEXT:    str d0, [x4]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_ld1lane_build:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14849,15 +14840,15 @@ define void @test_ld1lane_build(ptr %ptr0, ptr %ptr1, ptr %ptr2, ptr %ptr3, ptr
 }
 
 define void  @test_ld1lane_build_i16(ptr %a, ptr %b, ptr %c, ptr %d, <4 x i16> %e, ptr %p) {
-; CHECK-LABEL: test_ld1lane_build_i16:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ldr h1, [x0]
-; CHECK-NEXT:    ld1.h { v1 }[1], [x1]
-; CHECK-NEXT:    ld1.h { v1 }[2], [x2]
-; CHECK-NEXT:    ld1.h { v1 }[3], [x3]
-; CHECK-NEXT:    sub.4h v0, v1, v0
-; CHECK-NEXT:    str d0, [x4]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_ld1lane_build_i16:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ldr h1, [x0]
+; SDAG-NEXT:    ld1.h { v1 }[1], [x1]
+; SDAG-NEXT:    ld1.h { v1 }[2], [x2]
+; SDAG-NEXT:    ld1.h { v1 }[3], [x3]
+; SDAG-NEXT:    sub.4h v0, v1, v0
+; SDAG-NEXT:    str d0, [x4]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_ld1lane_build_i16:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14885,18 +14876,18 @@ define void  @test_ld1lane_build_i16(ptr %a, ptr %b, ptr %c, ptr %d, <4 x i16> %
 }
 
 define void  @test_ld1lane_build_half(ptr %a, ptr %b, ptr %c, ptr %d, <4 x half> %e, ptr %p) {
-; CHECK-LABEL: test_ld1lane_build_half:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ldr h1, [x0]
-; CHECK-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-NEXT:    ld1.h { v1 }[1], [x1]
-; CHECK-NEXT:    ld1.h { v1 }[2], [x2]
-; CHECK-NEXT:    ld1.h { v1 }[3], [x3]
-; CHECK-NEXT:    fcvtl v1.4s, v1.4h
-; CHECK-NEXT:    fsub.4s v0, v1, v0
-; CHECK-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-NEXT:    str d0, [x4]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_ld1lane_build_half:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ldr h1, [x0]
+; SDAG-NEXT:    fcvtl v0.4s, v0.4h
+; SDAG-NEXT:    ld1.h { v1 }[1], [x1]
+; SDAG-NEXT:    ld1.h { v1 }[2], [x2]
+; SDAG-NEXT:    ld1.h { v1 }[3], [x3]
+; SDAG-NEXT:    fcvtl v1.4s, v1.4h
+; SDAG-NEXT:    fsub.4s v0, v1, v0
+; SDAG-NEXT:    fcvtn v0.4h, v0.4s
+; SDAG-NEXT:    str d0, [x4]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_ld1lane_build_half:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14927,20 +14918,20 @@ define void  @test_ld1lane_build_half(ptr %a, ptr %b, ptr %c, ptr %d, <4 x half>
 }
 
 define void  @test_ld1lane_build_i8(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e, ptr %f, ptr %g, ptr %h, <8 x i8> %v, ptr %p) {
-; CHECK-LABEL: test_ld1lane_build_i8:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ldr b1, [x0]
-; CHECK-NEXT:    ldr x8, [sp]
-; CHECK-NEXT:    ld1.b { v1 }[1], [x1]
-; CHECK-NEXT:    ld1.b { v1 }[2], [x2]
-; CHECK-NEXT:    ld1.b { v1 }[3], [x3]
-; CHECK-NEXT:    ld1.b { v1 }[4], [x4]
-; CHECK-NEXT:    ld1.b { v1 }[5], [x5]
-; CHECK-NEXT:    ld1.b { v1 }[6], [x6]
-; CHECK-NEXT:    ld1.b { v1 }[7], [x7]
-; CHECK-NEXT:    sub.8b v0, v1, v0
-; CHECK-NEXT:    str d0, [x8]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_ld1lane_build_i8:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ldr b1, [x0]
+; SDAG-NEXT:    ldr x8, [sp]
+; SDAG-NEXT:    ld1.b { v1 }[1], [x1]
+; SDAG-NEXT:    ld1.b { v1 }[2], [x2]
+; SDAG-NEXT:    ld1.b { v1 }[3], [x3]
+; SDAG-NEXT:    ld1.b { v1 }[4], [x4]
+; SDAG-NEXT:    ld1.b { v1 }[5], [x5]
+; SDAG-NEXT:    ld1.b { v1 }[6], [x6]
+; SDAG-NEXT:    ld1.b { v1 }[7], [x7]
+; SDAG-NEXT:    sub.8b v0, v1, v0
+; SDAG-NEXT:    str d0, [x8]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_ld1lane_build_i8:
 ; CHECK-GISEL:       ; %bb.0:
@@ -14985,14 +14976,14 @@ define void  @test_ld1lane_build_i8(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e, ptr
 }
 
 define <4 x i32> @test_inc_cycle(<4 x i32> %vec, ptr %in) {
-; CHECK-LABEL: test_inc_cycle:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ld1.s { v0 }[0], [x0]
-; CHECK-NEXT:    adrp x9, _var@PAGE
-; CHECK-NEXT:    fmov x8, d0
-; CHECK-NEXT:    add x8, x0, x8, lsl #2
-; CHECK-NEXT:    str x8, [x9, _var@PAGEOFF]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: test_inc_cycle:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ld1.s { v0 }[0], [x0]
+; SDAG-NEXT:    adrp x9, _var@PAGE
+; SDAG-NEXT:    fmov x8, d0
+; SDAG-NEXT:    add x8, x0, x8, lsl #2
+; SDAG-NEXT:    str x8, [x9, _var@PAGEOFF]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: test_inc_cycle:
 ; CHECK-GISEL:       ; %bb.0:
@@ -15019,18 +15010,18 @@ define <4 x i32> @test_inc_cycle(<4 x i32> %vec, ptr %in) {
 @var = global ptr null
 
 define i8 @load_single_extract_variable_index_i8(ptr %A, i32 %idx) {
-; CHECK-LABEL: load_single_extract_variable_index_i8:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    sub sp, sp, #16
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    mov x8, sp
-; CHECK-NEXT:    ldr q0, [x0]
-; CHECK-NEXT:    ; kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    bfxil x8, x1, #0, #4
-; CHECK-NEXT:    str q0, [sp]
-; CHECK-NEXT:    ldrb w0, [x8]
-; CHECK-NEXT:    add sp, sp, #16
-; CHECK-NEXT:    ret
+; SDAG-LABEL: load_single_extract_variable_index_i8:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    sub sp, sp, #16
+; SDAG-NEXT:    .cfi_def_cfa_offset 16
+; SDAG-NEXT:    mov x8, sp
+; SDAG-NEXT:    ldr q0, [x0]
+; SDAG-NEXT:    ; kill: def $w1 killed $w1 def $x1
+; SDAG-NEXT:    bfxil x8, x1, #0, #4
+; SDAG-NEXT:    str q0, [sp]
+; SDAG-NEXT:    ldrb w0, [x8]
+; SDAG-NEXT:    add sp, sp, #16
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: load_single_extract_variable_index_i8:
 ; CHECK-GISEL:       ; %bb.0:
@@ -15052,18 +15043,18 @@ define i8 @load_single_extract_variable_index_i8(ptr %A, i32 %idx) {
 }
 
 define i16 @load_single_extract_variable_index_i16(ptr %A, i32 %idx) {
-; CHECK-LABEL: load_single_extract_variable_index_i16:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    sub sp, sp, #16
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    mov x8, sp
-; CHECK-NEXT:    ldr q0, [x0]
-; CHECK-NEXT:    ; kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    bfi x8, x1, #1, #3
-; CHECK-NEXT:    str q0, [sp]
-; CHECK-NEXT:    ldrh w0, [x8]
-; CHECK-NEXT:    add sp, sp, #16
-; CHECK-NEXT:    ret
+; SDAG-LABEL: load_single_extract_variable_index_i16:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    sub sp, sp, #16
+; SDAG-NEXT:    .cfi_def_cfa_offset 16
+; SDAG-NEXT:    mov x8, sp
+; SDAG-NEXT:    ldr q0, [x0]
+; SDAG-NEXT:    ; kill: def $w1 killed $w1 def $x1
+; SDAG-NEXT:    bfi x8, x1, #1, #3
+; SDAG-NEXT:    str q0, [sp]
+; SDAG-NEXT:    ldrh w0, [x8]
+; SDAG-NEXT:    add sp, sp, #16
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: load_single_extract_variable_index_i16:
 ; CHECK-GISEL:       ; %bb.0:
@@ -15083,12 +15074,12 @@ define i16 @load_single_extract_variable_index_i16(ptr %A, i32 %idx) {
 }
 
 define i32 @load_single_extract_variable_index_i32(ptr %A, i32 %idx) {
-; CHECK-LABEL: load_single_extract_variable_index_i32:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    ; kill: def $w1 killed $w1 def $x1
-; CHECK-NEXT:    and x8, x1, #0x3
-; CHECK-NEXT:    ldr w0, [x0, x8, lsl #2]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: load_single_extract_variable_index_i32:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    ; kill: def $w1 killed $w1 def $x1
+; SDAG-NEXT:    and x8, x1, #0x3
+; SDAG-NEXT:    ldr w0, [x0, x8, lsl #2]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: load_single_extract_variable_index_i32:
 ; CHECK-GISEL:       ; %bb.0:
@@ -15116,15 +15107,6 @@ define i32 @load_single_extract_variable_index_v3i32_small_align(ptr %A, i32 %id
 ; CHECK-NEXT:    csel x8, x9, x8, lo
 ; CHECK-NEXT:    ldr w0, [x0, x8, lsl #2]
 ; CHECK-NEXT:    ret
-;
-; CHECK-GISEL-LABEL: load_single_extract_variable_index_v3i32_small_align:
-; CHECK-GISEL:       ; %bb.0:
-; CHECK-GISEL-NEXT:    mov w9, w1
-; CHECK-GISEL-NEXT:    mov w8, #2 ; =0x2
-; CHECK-GISEL-NEXT:    cmp x9, #2
-; CHECK-GISEL-NEXT:    csel x8, x9, x8, lo
-; CHECK-GISEL-NEXT:    ldr w0, [x0, x8, lsl #2]
-; CHECK-GISEL-NEXT:    ret
   %lv = load <3 x i32>, ptr %A, align 2
   %e = extractelement <3 x i32> %lv, i32 %idx
   ret i32 %e
@@ -15139,15 +15121,6 @@ define i32 @load_single_extract_variable_index_v3i32_default_align(ptr %A, i32 %
 ; CHECK-NEXT:    csel x8, x9, x8, lo
 ; CHECK-NEXT:    ldr w0, [x0, x8, lsl #2]
 ; CHECK-NEXT:    ret
-;
-; CHECK-GISEL-LABEL: load_single_extract_variable_index_v3i32_default_align:
-; CHECK-GISEL:       ; %bb.0:
-; CHECK-GISEL-NEXT:    mov w9, w1
-; CHECK-GISEL-NEXT:    mov w8, #2 ; =0x2
-; CHECK-GISEL-NEXT:    cmp x9, #2
-; CHECK-GISEL-NEXT:    csel x8, x9, x8, lo
-; CHECK-GISEL-NEXT:    ldr w0, [x0, x8, lsl #2]
-; CHECK-GISEL-NEXT:    ret
   %lv = load <3 x i32>, ptr %A
   %e = extractelement <3 x i32> %lv, i32 %idx
   ret i32 %e
@@ -15158,22 +15131,17 @@ define i32 @load_single_extract_valid_const_index_v3i32(ptr %A, i32 %idx) {
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    ldr w0, [x0, #8]
 ; CHECK-NEXT:    ret
-;
-; CHECK-GISEL-LABEL: load_single_extract_valid_const_index_v3i32:
-; CHECK-GISEL:       ; %bb.0:
-; CHECK-GISEL-NEXT:    ldr w0, [x0, #8]
-; CHECK-GISEL-NEXT:    ret
   %lv = load <3 x i32>, ptr %A
   %e = extractelement <3 x i32> %lv, i32 2
   ret i32 %e
 }
 
 define i32 @load_single_extract_variable_index_masked_i32(ptr %A, i32 %idx) {
-; CHECK-LABEL: load_single_extract_variable_index_masked_i32:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    and w8, w1, #0x3
-; CHECK-NEXT:    ldr w0, [x0, w8, uxtw #2]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: load_single_extract_variable_index_masked_i32:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    and w8, w1, #0x3
+; SDAG-NEXT:    ldr w0, [x0, w8, uxtw #2]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: load_single_extract_variable_index_masked_i32:
 ; CHECK-GISEL:       ; %bb.0:
@@ -15193,11 +15161,11 @@ define i32 @load_single_extract_variable_index_masked_i32(ptr %A, i32 %idx) {
 }
 
 define i32 @load_single_extract_variable_index_masked2_i32(ptr %A, i32 %idx) {
-; CHECK-LABEL: load_single_extract_variable_index_masked2_i32:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    and w8, w1, #0x1
-; CHECK-NEXT:    ldr w0, [x0, w8, uxtw #2]
-; CHECK-NEXT:    ret
+; SDAG-LABEL: load_single_extract_variable_index_masked2_i32:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    and w8, w1, #0x1
+; SDAG-NEXT:    ldr w0, [x0, w8, uxtw #2]
+; SDAG-NEXT:    ret
 ;
 ; CHECK-GISEL-LABEL: load_single_extract_variable_index_masked2_i32:
 ; CHECK-GISEL:       ; %bb.0:
diff --git a/llvm/test/CodeGen/AArch64/arm64-ld1.ll b/llvm/test/CodeGen/AArch64/arm64-ld1.ll
index 96468b2cfa..54b96520dc 100644
--- a/llvm/test/CodeGen/AArch64/arm64-ld1.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-ld1.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
 ; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK,CHECK-SD
-; RUN: llc < %s -global-isel=1 -global-isel-abort=2 -mtriple=arm64-eabi -aarch64-neon-syntax=apple -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+; RUN: llc < %s -global-isel=1 -global-isel-abort=1 -mtriple=arm64-eabi -aarch64-neon-syntax=apple -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK,CHECK-GI
 
 %struct.__neon_int8x8x2_t = type { <8 x i8>,  <8 x i8> }
 %struct.__neon_int8x8x3_t = type { <8 x i8>,  <8 x i8>,  <8 x i8> }
@@ -1712,3 +1712,30 @@ define %struct.__neon_float64x2x4_t @ld1_x4_v2f64(ptr %addr) {
   %val = call %struct.__neon_float64x2x4_t @llvm.aarch64.neon.ld1x4.v2f64.p0(ptr %addr)
   ret %struct.__neon_float64x2x4_t %val
 }
+
+define <8 x i8> @dup_ld1_from_stack(ptr %__ret) {
+; CHECK-SD-LABEL: dup_ld1_from_stack:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sub sp, sp, #16
+; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-SD-NEXT:    add x8, sp, #15
+; CHECK-SD-NEXT:    ld1r.8b { v0 }, [x8]
+; CHECK-SD-NEXT:    add sp, sp, #16
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: dup_ld1_from_stack:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-GI-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-GI-NEXT:    .cfi_offset w29, -16
+; CHECK-GI-NEXT:    add x8, sp, #15
+; CHECK-GI-NEXT:    ld1r.8b { v0 }, [x8]
+; CHECK-GI-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-GI-NEXT:    ret
+entry:
+  %item = alloca i8, align 1
+  %0 = load i8, ptr %item, align 1
+  %1 = insertelement <8 x i8> poison, i8 %0, i32 0
+  %lane = shufflevector <8 x i8> %1, <8 x i8> %1, <8 x i32> zeroinitializer
+  ret <8 x i8> %lane
+}
diff --git a/llvm/test/CodeGen/AArch64/arm64-neon-mul-div.ll b/llvm/test/CodeGen/AArch64/arm64-neon-mul-div.ll
index 7039cccdf9..ecf3f69825 100644
--- a/llvm/test/CodeGen/AArch64/arm64-neon-mul-div.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-neon-mul-div.ll
@@ -1486,7 +1486,8 @@ define <2 x float> @frem2f32(<2 x float> %A, <2 x float> %B) {
 ; CHECK-NEXT:    mov s0, v0.s[1]
 ; CHECK-NEXT:    mov s1, v1.s[1]
 ; CHECK-NEXT:    bl fmodf
-; CHECK-NEXT:    str d0, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    ldp q0, q1, [sp] // 32-byte Folded Reload
 ; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-NEXT:    // kill: def $s1 killed $s1 killed $q1
@@ -1513,7 +1514,8 @@ define <4 x float> @frem4f32(<4 x float> %A, <4 x float> %B) {
 ; CHECK-NEXT:    mov s0, v0.s[1]
 ; CHECK-NEXT:    mov s1, v1.s[1]
 ; CHECK-NEXT:    bl fmodf
-; CHECK-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    ldp q0, q1, [sp, #16] // 32-byte Folded Reload
 ; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-NEXT:    // kill: def $s1 killed $s1 killed $q1
@@ -1569,6 +1571,7 @@ define <2 x double> @frem2d64(<2 x double> %A, <2 x double> %B) {
 ; CHECK-NEXT:    mov d0, v0.d[1]
 ; CHECK-NEXT:    mov d1, v1.d[1]
 ; CHECK-NEXT:    bl fmod
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    ldp q0, q1, [sp] // 32-byte Folded Reload
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
diff --git a/llvm/test/CodeGen/AArch64/arm64-st1.ll b/llvm/test/CodeGen/AArch64/arm64-st1.ll
index 121ca69bee..6f87c66c87 100644
--- a/llvm/test/CodeGen/AArch64/arm64-st1.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-st1.ll
@@ -1,5 +1,5 @@
 ; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -verify-machineinstrs | FileCheck %s
-; RUN: llc < %s -global-isel -mtriple=arm64-eabi -aarch64-neon-syntax=apple -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -global-isel -global-isel-abort=1 -mtriple=arm64-eabi -aarch64-neon-syntax=apple -verify-machineinstrs | FileCheck %s
 ; The instruction latencies of Exynos-M3 trigger the transform we see under the Exynos check.
 ; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -verify-machineinstrs -mcpu=exynos-m3 | FileCheck --check-prefix=EXYNOS %s
 
diff --git a/llvm/test/CodeGen/AArch64/arm64-vfloatintrinsics.ll b/llvm/test/CodeGen/AArch64/arm64-vfloatintrinsics.ll
deleted file mode 100644
index 0278128b25..0000000000
--- a/llvm/test/CodeGen/AArch64/arm64-vfloatintrinsics.ll
+++ /dev/null
@@ -1,514 +0,0 @@
-; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -mattr=-fullfp16 \
-; RUN:     | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-NOFP16
-; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -mattr=+fullfp16 \
-; RUN:     | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-FP16
-
-; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -mattr=-fullfp16 \
-; RUN:     -global-isel -global-isel-abort=2 -pass-remarks-missed=gisel* \
-; RUN:     2>&1 | FileCheck %s --check-prefixes=GISEL,GISEL-NOFP16,FALLBACK
-; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -mattr=+fullfp16 \
-; RUN:     -global-isel -global-isel-abort=2 -pass-remarks-missed=gisel* \
-; RUN:     2>&1 | FileCheck %s --check-prefixes=GISEL,GISEL-FP16,FALLBACK
-
-;;; Half vectors
-
-%v4f16 = type <4 x half>
-
-define %v4f16 @test_v4f16.powi(%v4f16 %a, i32 %b) {
-  ; This operation is expanded, whether with or without +fullfp16.
-  ; CHECK-LABEL:   test_v4f16.powi:
-  ; CHECK-COUNT-4: bl __powi
-  %1 = call %v4f16 @llvm.powi.v4f16.i32(%v4f16 %a, i32 %b)
-  ret %v4f16 %1
-}
-
-; FALLBACK-NOT: remark{{.*}}test_v4f16.sin
-define %v4f16 @test_v4f16.sin(%v4f16 %a) {
-  ; This operation is expanded, whether with or without +fullfp16.
-  ; CHECK-LABEL:   test_v4f16.sin:
-  ; CHECK-COUNT-4: bl sinf
-  ; GISEL-LABEL:   test_v4f16.sin:
-  ; GISEL-COUNT-4: bl sinf
-  %1 = call %v4f16 @llvm.sin.v4f16(%v4f16 %a)
-  ret %v4f16 %1
-}
-
-; FALLBACK-NOT: remark{{.*}}test_v4f16.cos
-define %v4f16 @test_v4f16.cos(%v4f16 %a) {
-  ; This operation is expanded, whether with or without +fullfp16.
-  ; CHECK-LABEL:   test_v4f16.cos:
-  ; CHECK-COUNT-4: bl cosf
-  ; GISEL-LABEL:   test_v4f16.cos:
-  ; GISEL-COUNT-4: bl cosf
-  %1 = call %v4f16 @llvm.cos.v4f16(%v4f16 %a)
-  ret %v4f16 %1
-}
-
-; FALLBACK-NOT: remark{{.*}}test_v4f16.exp
-define %v4f16 @test_v4f16.exp(%v4f16 %a) {
-  ; This operation is expanded, whether with or without +fullfp16.
-  ; CHECK-LABEL:   test_v4f16.exp:
-  ; CHECK-COUNT-4: bl exp
-  ; GISEL-LABEL:   test_v4f16.exp:
-  ; GISEL-COUNT-4: bl exp
-  %1 = call %v4f16 @llvm.exp.v4f16(%v4f16 %a)
-  ret %v4f16 %1
-}
-define %v4f16 @test_v4f16.exp2(%v4f16 %a) {
-  ; This operation is expanded, whether with or without +fullfp16.
-  ; CHECK-LABEL:   test_v4f16.exp2:
-  ; CHECK-COUNT-4: bl exp2
-  %1 = call %v4f16 @llvm.exp2.v4f16(%v4f16 %a)
-  ret %v4f16 %1
-}
-
-; FALLBACK-NOT: remark{{.*}}test_v4f16.log
-define %v4f16 @test_v4f16.log(%v4f16 %a) {
-  ; This operation is expanded, whether with or without +fullfp16.
-  ; CHECK-LABEL:   test_v4f16.log:
-  ; CHECK-COUNT-4: bl log
-  ; GISEL-LABEL:   test_v4f16.log:
-  ; GISEL-COUNT-4: bl log
-  %1 = call %v4f16 @llvm.log.v4f16(%v4f16 %a)
-  ret %v4f16 %1
-}
-
-; FALLBACK-NOT: remark{{.*}}test_v4f16.log10
-define %v4f16 @test_v4f16.log10(%v4f16 %a) {
-  ; This operation is expanded, whether with or without +fullfp16.
-  ; CHECK-LABEL:   test_v4f16.log10:
-  ; CHECK-COUNT-4: bl log10
-  ; GISEL-LABEL:   test_v4f16.log10:
-  ; GISEL-COUNT-4: bl log10
-  %1 = call %v4f16 @llvm.log10.v4f16(%v4f16 %a)
-  ret %v4f16 %1
-}
-
-; FALLBACK-NOT: remark{{.*}}test_v4f16.log2
-define %v4f16 @test_v4f16.log2(%v4f16 %a) {
-  ; This operation is expanded, whether with or without +fullfp16.
-  ; CHECK-LABEL:   test_v4f16.log2:
-  ; CHECK-COUNT-4: bl log2
-  ; GISEL-LABEL:   test_v4f16.log2:
-  ; GISEL-COUNT-4: bl log2
-  %1 = call %v4f16 @llvm.log2.v4f16(%v4f16 %a)
-  ret %v4f16 %1
-}
-
-; FALLBACK-NOT: remark{{.*}}test_v4f16.fma
-define %v4f16 @test_v4f16.fma(%v4f16 %a, %v4f16 %b, %v4f16 %c) {
-  ; CHECK-LABEL:          test_v4f16.fma:
-  ; CHECK-NOFP16-COUNT-4: fmadd s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
-  ; CHECK-FP16-NOT:       fcvt
-  ; CHECK-FP16:           fmla.4h
-  ; GISEL-LABEL:          test_v4f16.fma:
-  ; GISEL-NOFP16-COUNT-4: fmadd s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
-  ; GISEL-FP16-NOT:       fcvt
-  ; GISEL-FP16:           fmla.4h
-  %1 = call %v4f16 @llvm.fma.v4f16(%v4f16 %a, %v4f16 %b, %v4f16 %c)
-  ret %v4f16 %1
-}
-
-declare %v4f16 @llvm.powi.v4f16.i32(%v4f16, i32) #0
-declare %v4f16 @llvm.sin.v4f16(%v4f16) #0
-declare %v4f16 @llvm.cos.v4f16(%v4f16) #0
-declare %v4f16 @llvm.exp.v4f16(%v4f16) #0
-declare %v4f16 @llvm.exp2.v4f16(%v4f16) #0
-declare %v4f16 @llvm.log.v4f16(%v4f16) #0
-declare %v4f16 @llvm.log10.v4f16(%v4f16) #0
-declare %v4f16 @llvm.log2.v4f16(%v4f16) #0
-declare %v4f16 @llvm.fma.v4f16(%v4f16, %v4f16, %v4f16) #0
-
-;;;
-
-%v8f16 = type <8 x half>
-
-define %v8f16 @test_v8f16.powi(%v8f16 %a, i32 %b) {
-  ; This operation is expanded, whether with or without +fullfp16.
-  ; CHECK-LABEL:   test_v8f16.powi:
-  ; CHECK-COUNT-8: bl __powi
-  ; GISEL-LABEL:   test_v8f16.powi:
-  ; GISEL-COUNT-8: bl __powi
-  %1 = call %v8f16 @llvm.powi.v8f16.i32(%v8f16 %a, i32 %b)
-  ret %v8f16 %1
-}
-
-; FALLBACK-NOT: remark{{.*}}test_v8f16.sin
-define %v8f16 @test_v8f16.sin(%v8f16 %a) {
-  ; This operation is expanded, whether with or without +fullfp16.
-  ; CHECK-LABEL:   test_v8f16.sin:
-  ; CHECK-COUNT-8: bl sinf
-  ; GISEL-LABEL:   test_v8f16.sin:
-  ; GISEL-COUNT-8: bl sinf
-  %1 = call %v8f16 @llvm.sin.v8f16(%v8f16 %a)
-  ret %v8f16 %1
-}
-
-; FALLBACK-NOT: remark{{.*}}test_v8f16.cos
-define %v8f16 @test_v8f16.cos(%v8f16 %a) {
-  ; This operation is expanded, whether with or without +fullfp16.
-  ; CHECK-LABEL:   test_v8f16.cos:
-  ; CHECK-COUNT-8: bl cosf
-  ; GISEL-LABEL:   test_v8f16.cos:
-  ; GISEL-COUNT-8: bl cosf
-  %1 = call %v8f16 @llvm.cos.v8f16(%v8f16 %a)
-  ret %v8f16 %1
-}
-
-; FALLBACK-NOT: remark{{.*}}test_v8f16.exp
-define %v8f16 @test_v8f16.exp(%v8f16 %a) {
-  ; This operation is expanded, whether with or without +fullfp16.
-  ; CHECK-LABEL:   test_v8f16.exp:
-  ; CHECK-COUNT-8: bl exp
-  ; GISEL-LABEL:   test_v8f16.exp:
-  ; GISEL-COUNT-8: bl exp
-  %1 = call %v8f16 @llvm.exp.v8f16(%v8f16 %a)
-  ret %v8f16 %1
-}
-define %v8f16 @test_v8f16.exp2(%v8f16 %a) {
-  ; This operation is expanded, whether with or without +fullfp16.
-  ; CHECK-LABEL:   test_v8f16.exp2:
-  ; CHECK-COUNT-8: bl exp2
-  %1 = call %v8f16 @llvm.exp2.v8f16(%v8f16 %a)
-  ret %v8f16 %1
-}
-
-; FALLBACK-NOT: remark{{.*}}test_v8f16.log
-define %v8f16 @test_v8f16.log(%v8f16 %a) {
-  ; This operation is expanded, whether with or without +fullfp16.
-  ; CHECK-LABEL:   test_v8f16.log:
-  ; CHECK-COUNT-8: bl log
-  ; GISEL-LABEL:   test_v8f16.log:
-  ; GISEL-COUNT-8: bl log
-  %1 = call %v8f16 @llvm.log.v8f16(%v8f16 %a)
-  ret %v8f16 %1
-}
-
-; FALLBACK-NOT: remark{{.*}}test_v8f16.log10
-define %v8f16 @test_v8f16.log10(%v8f16 %a) {
-  ; This operation is expanded, whether with or without +fullfp16.
-  ; CHECK-LABEL:   test_v8f16.log10:
-  ; CHECK-COUNT-8: bl log10
-  ; GISEL-LABEL:   test_v8f16.log10:
-  ; GISEL-COUNT-8: bl log10
-  %1 = call %v8f16 @llvm.log10.v8f16(%v8f16 %a)
-  ret %v8f16 %1
-}
-
-; FALLBACK-NOT: remark{{.*}}test_v8f16.log2
-define %v8f16 @test_v8f16.log2(%v8f16 %a) {
-  ; This operation is expanded, whether with or without +fullfp16.
-  ; CHECK-LABEL:   test_v8f16.log2:
-  ; CHECK-COUNT-8: bl log2
-  ; GISEL-LABEL:   test_v8f16.log2:
-  ; GISEL-COUNT-8: bl log2
-  %1 = call %v8f16 @llvm.log2.v8f16(%v8f16 %a)
-  ret %v8f16 %1
-}
-
-; FALLBACK-NOT: remark{{.*}}test_v8f16.fma
-define %v8f16 @test_v8f16.fma(%v8f16 %a, %v8f16 %b, %v8f16 %c) {
-  ; CHECK-LABEL:          test_v8f16.fma:
-  ; CHECK-NOFP16-COUNT-8: fmadd s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
-  ; CHECK-FP16-NOT:       fcvt
-  ; CHECK-FP16:           fmla.8h
-  ; GISEL-LABEL:          test_v8f16.fma:
-  ; GISEL-NOFP16-COUNT-8: fmadd s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
-  ; GISEL-FP16-NOT:       fcvt
-  ; GISEL-FP16:           fmla.8h
-  %1 = call %v8f16 @llvm.fma.v8f16(%v8f16 %a, %v8f16 %b, %v8f16 %c)
-  ret %v8f16 %1
-}
-
-declare %v8f16 @llvm.powi.v8f16.i32(%v8f16, i32) #0
-declare %v8f16 @llvm.sin.v8f16(%v8f16) #0
-declare %v8f16 @llvm.cos.v8f16(%v8f16) #0
-declare %v8f16 @llvm.exp.v8f16(%v8f16) #0
-declare %v8f16 @llvm.exp2.v8f16(%v8f16) #0
-declare %v8f16 @llvm.log.v8f16(%v8f16) #0
-declare %v8f16 @llvm.log10.v8f16(%v8f16) #0
-declare %v8f16 @llvm.log2.v8f16(%v8f16) #0
-declare %v8f16 @llvm.fma.v8f16(%v8f16, %v8f16, %v8f16) #0
-
-;;; Float vectors
-
-%v2f32 = type <2 x float>
-
-; CHECK: test_v2f32.powi:
-define %v2f32 @test_v2f32.powi(%v2f32 %a, i32 %b) {
-  ; CHECK: pow
-  %1 = call %v2f32 @llvm.powi.v2f32.i32(%v2f32 %a, i32 %b)
-  ret %v2f32 %1
-}
-
-; FALLBACK-NOT: remark{{.*}}test_v2f32.sin
-; CHECK: test_v2f32.sin:
-define %v2f32 @test_v2f32.sin(%v2f32 %a) {
-  ; CHECK: sin
-  ; GISEL: sin
-  %1 = call %v2f32 @llvm.sin.v2f32(%v2f32 %a)
-  ret %v2f32 %1
-}
-
-; FALLBACK-NOT: remark{{.*}}test_v2f32.cos
-; CHECK: test_v2f32.cos:
-define %v2f32 @test_v2f32.cos(%v2f32 %a) {
-  ; CHECK: cos
-  ; GISEL: cos
-  %1 = call %v2f32 @llvm.cos.v2f32(%v2f32 %a)
-  ret %v2f32 %1
-}
-
-; FALLBACK-NOT: remark{{.*}}test_v2f32.exp
-; CHECK: test_v2f32.exp:
-; GISEL: test_v2f32.exp:
-define %v2f32 @test_v2f32.exp(%v2f32 %a) {
-  ; CHECK: exp
-  ; GISEL: exp
-  %1 = call %v2f32 @llvm.exp.v2f32(%v2f32 %a)
-  ret %v2f32 %1
-}
-; CHECK: test_v2f32.exp2:
-define %v2f32 @test_v2f32.exp2(%v2f32 %a) {
-  ; CHECK: exp
-  %1 = call %v2f32 @llvm.exp2.v2f32(%v2f32 %a)
-  ret %v2f32 %1
-}
-
-; FALLBACK-NOT: remark{{.*}}test_v2f32.log
-; CHECK: test_v2f32.log:
-define %v2f32 @test_v2f32.log(%v2f32 %a) {
-  ; CHECK: log
-  ; GISEL: log
-  %1 = call %v2f32 @llvm.log.v2f32(%v2f32 %a)
-  ret %v2f32 %1
-}
-
-; FALLBACK-NOT: remark{{.*}}test_v2f32.log10
-; CHECK: test_v2f32.log10:
-; GISEL: test_v2f32.log10:
-define %v2f32 @test_v2f32.log10(%v2f32 %a) {
-  ; CHECK: log
-  ; GISEL: log
-  %1 = call %v2f32 @llvm.log10.v2f32(%v2f32 %a)
-  ret %v2f32 %1
-}
-
-; FALLBACK-NOT: remark{{.*}}test_v2f32.log2
-; CHECK: test_v2f32.log2:
-; GISEL: test_v2f32.log2:
-define %v2f32 @test_v2f32.log2(%v2f32 %a) {
-  ; CHECK: log
-  ; GISEL: log
-  %1 = call %v2f32 @llvm.log2.v2f32(%v2f32 %a)
-  ret %v2f32 %1
-}
-
-; FALLBACK-NOT: remark{{.*}}test_v2f32.fma
-; CHECK-LABEL: test_v2f32.fma:
-; GISEL-LABEL: test_v2f32.fma:
-define %v2f32 @test_v2f32.fma(%v2f32 %a, %v2f32 %b, %v2f32 %c) {
-  ; CHECK: fmla.2s
-  ; GISEL: fmla.2s
-  %1 = call %v2f32 @llvm.fma.v2f32(%v2f32 %a, %v2f32 %b, %v2f32 %c)
-  ret %v2f32 %1
-}
-
-declare %v2f32 @llvm.powi.v2f32.i32(%v2f32, i32) #0
-declare %v2f32 @llvm.sin.v2f32(%v2f32) #0
-declare %v2f32 @llvm.cos.v2f32(%v2f32) #0
-declare %v2f32 @llvm.exp.v2f32(%v2f32) #0
-declare %v2f32 @llvm.exp2.v2f32(%v2f32) #0
-declare %v2f32 @llvm.log.v2f32(%v2f32) #0
-declare %v2f32 @llvm.log10.v2f32(%v2f32) #0
-declare %v2f32 @llvm.log2.v2f32(%v2f32) #0
-declare %v2f32 @llvm.fma.v2f32(%v2f32, %v2f32, %v2f32) #0
-
-;;;
-
-%v4f32 = type <4 x float>
-
-; CHECK: test_v4f32.powi:
-define %v4f32 @test_v4f32.powi(%v4f32 %a, i32 %b) {
-  ; CHECK: pow
-  %1 = call %v4f32 @llvm.powi.v4f32.i32(%v4f32 %a, i32 %b)
-  ret %v4f32 %1
-}
-
-; FALLBACK-NOT: remark{{.*}}test_v4f32.sin
-; CHECK: test_v4f32.sin:
-define %v4f32 @test_v4f32.sin(%v4f32 %a) {
-  ; CHECK: sin
-  ; GISEL: sin
-  %1 = call %v4f32 @llvm.sin.v4f32(%v4f32 %a)
-  ret %v4f32 %1
-}
-
-; FALLBACK-NOT: remark{{.*}}test_v4f32.cos
-; CHECK: test_v4f32.cos:
-define %v4f32 @test_v4f32.cos(%v4f32 %a) {
-  ; CHECK: cos
-  ; GISEL: cos
-  %1 = call %v4f32 @llvm.cos.v4f32(%v4f32 %a)
-  ret %v4f32 %1
-}
-
-; FALLBACK-NOT: remark{{.*}}test_v4f32.exp
-; CHECK: test_v4f32.exp:
-; GISEL: test_v4f32.exp:
-define %v4f32 @test_v4f32.exp(%v4f32 %a) {
-  ; CHECK: exp
-  ; GISEL: exp
-  %1 = call %v4f32 @llvm.exp.v4f32(%v4f32 %a)
-  ret %v4f32 %1
-}
-; CHECK: test_v4f32.exp2:
-define %v4f32 @test_v4f32.exp2(%v4f32 %a) {
-  ; CHECK: exp
-  %1 = call %v4f32 @llvm.exp2.v4f32(%v4f32 %a)
-  ret %v4f32 %1
-}
-
-; FALLBACK-NOT: remark{{.*}}test_v4f32.log
-; CHECK: test_v4f32.log:
-define %v4f32 @test_v4f32.log(%v4f32 %a) {
-  ; CHECK: log
-  ; GISEL: log
-  %1 = call %v4f32 @llvm.log.v4f32(%v4f32 %a)
-  ret %v4f32 %1
-}
-
-; FALLBACK-NOT: remark{{.*}}test_v4f32.log10
-; CHECK: test_v4f32.log10:
-define %v4f32 @test_v4f32.log10(%v4f32 %a) {
-  ; CHECK: log
-  ; GISEL: log
-  %1 = call %v4f32 @llvm.log10.v4f32(%v4f32 %a)
-  ret %v4f32 %1
-}
-
-; FALLBACK-NOT: remark{{.*}}test_v4f32.log2
-; CHECK: test_v4f32.log2:
-; GISEL: test_v4f32.log2:
-define %v4f32 @test_v4f32.log2(%v4f32 %a) {
-  ; CHECK: log
-  ; GISEL: log
-  %1 = call %v4f32 @llvm.log2.v4f32(%v4f32 %a)
-  ret %v4f32 %1
-}
-
-; FALLBACK-NOT: remark{{.*}}test_v4f32.fma
-; CHECK: test_v4f32.fma:
-; GISEL: test_v4f32.fma:
-define %v4f32 @test_v4f32.fma(%v4f32 %a, %v4f32 %b, %v4f32 %c) {
-  ; CHECK: fma
-  ; GISEL: fma
-  %1 = call %v4f32 @llvm.fma.v4f32(%v4f32 %a, %v4f32 %b, %v4f32 %c)
-  ret %v4f32 %1
-}
-
-declare %v4f32 @llvm.powi.v4f32.i32(%v4f32, i32) #0
-declare %v4f32 @llvm.sin.v4f32(%v4f32) #0
-declare %v4f32 @llvm.cos.v4f32(%v4f32) #0
-declare %v4f32 @llvm.exp.v4f32(%v4f32) #0
-declare %v4f32 @llvm.exp2.v4f32(%v4f32) #0
-declare %v4f32 @llvm.log.v4f32(%v4f32) #0
-declare %v4f32 @llvm.log10.v4f32(%v4f32) #0
-declare %v4f32 @llvm.log2.v4f32(%v4f32) #0
-declare %v4f32 @llvm.fma.v4f32(%v4f32, %v4f32, %v4f32) #0
-
-;;; Double vector
-
-%v2f64 = type <2 x double>
-
-; CHECK: test_v2f64.powi:
-define %v2f64 @test_v2f64.powi(%v2f64 %a, i32 %b) {
-  ; CHECK: pow
-  %1 = call %v2f64 @llvm.powi.v2f64.i32(%v2f64 %a, i32 %b)
-  ret %v2f64 %1
-}
-
-; FALLBACK-NOT: remark{{.*}}test_v2f64.sin
-; CHECK: test_v2f64.sin:
-define %v2f64 @test_v2f64.sin(%v2f64 %a) {
-  ; CHECK: sin
-  ; GISEL: sin
-  %1 = call %v2f64 @llvm.sin.v2f64(%v2f64 %a)
-  ret %v2f64 %1
-}
-
-; FALLBACK-NOT: remark{{.*}}test_v2f64.cos
-; CHECK: test_v2f64.cos:
-define %v2f64 @test_v2f64.cos(%v2f64 %a) {
-  ; CHECK: cos
-  ; GISEL: cos
-  %1 = call %v2f64 @llvm.cos.v2f64(%v2f64 %a)
-  ret %v2f64 %1
-}
-
-; FALLBACK-NOT: remark{{.*}}test_v2f64.exp
-; CHECK: test_v2f64.exp:
-; GISEL: test_v2f64.exp:
-define %v2f64 @test_v2f64.exp(%v2f64 %a) {
-  ; CHECK: exp
-  ; GISEL: exp
-  %1 = call %v2f64 @llvm.exp.v2f64(%v2f64 %a)
-  ret %v2f64 %1
-}
-; CHECK: test_v2f64.exp2:
-define %v2f64 @test_v2f64.exp2(%v2f64 %a) {
-  ; CHECK: exp
-  %1 = call %v2f64 @llvm.exp2.v2f64(%v2f64 %a)
-  ret %v2f64 %1
-}
-
-; FALLBACK-NOT: remark{{.*}}test_v2f64.log
-; CHECK: test_v2f64.log:
-define %v2f64 @test_v2f64.log(%v2f64 %a) {
-  ; CHECK: log
-  ; GISEL: log
-  %1 = call %v2f64 @llvm.log.v2f64(%v2f64 %a)
-  ret %v2f64 %1
-}
-
-; FALLBACK-NOT: remark{{.*}}test_v2f64.log10
-; CHECK: test_v2f64.log10:
-; GISEL: test_v2f64.log10:
-define %v2f64 @test_v2f64.log10(%v2f64 %a) {
-  ; CHECK: log
-  ; GISEL: log
-  %1 = call %v2f64 @llvm.log10.v2f64(%v2f64 %a)
-  ret %v2f64 %1
-}
-
-; FALLBACK-NOT: remark{{.*}}test_v2f64.log2
-; CHECK: test_v2f64.log2:
-; GISEL: test_v2f64.log2:
-define %v2f64 @test_v2f64.log2(%v2f64 %a) {
-  ; CHECK: log
-  ; GISEL: log
-  %1 = call %v2f64 @llvm.log2.v2f64(%v2f64 %a)
-  ret %v2f64 %1
-}
-
-; FALLBACK-NOT: remark{{.*}}test_v2f64.fma
-; CHECK: test_v2f64.fma:
-; GISEL: test_v2f64.fma:
-define %v2f64 @test_v2f64.fma(%v2f64 %a, %v2f64 %b, %v2f64 %c) {
-  ; CHECK: fma
-  ; GISEL: fma
-  %1 = call %v2f64 @llvm.fma.v2f64(%v2f64 %a, %v2f64 %b, %v2f64 %c)
-  ret %v2f64 %1
-}
-
-declare %v2f64 @llvm.powi.v2f64.i32(%v2f64, i32) #0
-declare %v2f64 @llvm.sin.v2f64(%v2f64) #0
-declare %v2f64 @llvm.cos.v2f64(%v2f64) #0
-declare %v2f64 @llvm.exp.v2f64(%v2f64) #0
-declare %v2f64 @llvm.exp2.v2f64(%v2f64) #0
-declare %v2f64 @llvm.log.v2f64(%v2f64) #0
-declare %v2f64 @llvm.log10.v2f64(%v2f64) #0
-declare %v2f64 @llvm.log2.v2f64(%v2f64) #0
-declare %v2f64 @llvm.fma.v2f64(%v2f64, %v2f64, %v2f64) #0
-
-attributes #0 = { nounwind readonly }
diff --git a/llvm/test/CodeGen/AArch64/arm64-vshift.ll b/llvm/test/CodeGen/AArch64/arm64-vshift.ll
index 367c3be242..1dfd977186 100644
--- a/llvm/test/CodeGen/AArch64/arm64-vshift.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-vshift.ll
@@ -2130,9 +2130,8 @@ define <4 x i32> @neon.ushll4s_neg_constant_shift(ptr %A) nounwind {
 ; CHECK-LABEL: neon.ushll4s_neg_constant_shift:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr d0, [x0]
-; CHECK-NEXT:    movi.2d v1, #0xffffffffffffffff
 ; CHECK-NEXT:    ushll.4s v0, v0, #0
-; CHECK-NEXT:    ushl.4s v0, v0, v1
+; CHECK-NEXT:    ushr.4s v0, v0, #1
 ; CHECK-NEXT:    ret
   %tmp1 = load <4 x i16>, ptr %A
   %tmp2 = zext <4 x i16> %tmp1 to <4 x i32>
@@ -2250,9 +2249,8 @@ define <16 x i8> @neon.sshl16b_non_splat_constant_shift(ptr %A) nounwind {
 define <16 x i8> @neon.sshl16b_neg_constant_shift(ptr %A) nounwind {
 ; CHECK-LABEL: neon.sshl16b_neg_constant_shift:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi.16b v1, #254
 ; CHECK-NEXT:    ldr q0, [x0]
-; CHECK-NEXT:    sshl.16b v0, v0, v1
+; CHECK-NEXT:    sshr.16b v0, v0, #2
 ; CHECK-NEXT:    ret
   %tmp1 = load <16 x i8>, ptr %A
   %tmp2 = call <16 x i8> @llvm.aarch64.neon.sshl.v16i8(<16 x i8> %tmp1, <16 x i8> <i8 -2, i8 -2, i8 -2, i8 -2, i8 -2, i8 -2, i8 -2, i8 -2, i8 -2, i8 -2, i8 -2, i8 -2, i8 -2, i8 -2, i8 -2, i8 -2>)
@@ -2300,9 +2298,8 @@ define <4 x i32> @neon.sshll4s_neg_constant_shift(ptr %A) nounwind {
 ; CHECK-LABEL: neon.sshll4s_neg_constant_shift:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr d0, [x0]
-; CHECK-NEXT:    movi.2d v1, #0xffffffffffffffff
 ; CHECK-NEXT:    sshll.4s v0, v0, #0
-; CHECK-NEXT:    sshl.4s v0, v0, v1
+; CHECK-NEXT:    sshr.4s v0, v0, #1
 ; CHECK-NEXT:    ret
   %tmp1 = load <4 x i16>, ptr %A
   %tmp2 = sext <4 x i16> %tmp1 to <4 x i32>
@@ -2377,10 +2374,8 @@ define i64 @neon.sshll_scalar_constant_shift_m1(ptr %A) nounwind {
 ; CHECK-LABEL: neon.sshll_scalar_constant_shift_m1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr w8, [x0]
-; CHECK-NEXT:    mov x9, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    fmov d1, x9
 ; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    sshl d0, d0, d1
+; CHECK-NEXT:    sshr d0, d0, #1
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    ret
   %tmp1 = load i32, ptr %A
diff --git a/llvm/test/CodeGen/AArch64/fexplog.ll b/llvm/test/CodeGen/AArch64/fexplog.ll
index c2d6a1f5de..be6040faa2 100644
--- a/llvm/test/CodeGen/AArch64/fexplog.ll
+++ b/llvm/test/CodeGen/AArch64/fexplog.ll
@@ -64,6 +64,7 @@ define <2 x double> @exp_v2f64(<2 x double> %a) {
 ; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    mov d0, v0.d[1]
 ; CHECK-SD-NEXT:    bl exp
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -86,6 +87,7 @@ define <2 x double> @exp_v2f64(<2 x double> %a) {
 ; CHECK-GI-NEXT:    mov d8, v0.d[1]
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GI-NEXT:    bl exp
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov d0, d8
 ; CHECK-GI-NEXT:    bl exp
@@ -172,6 +174,7 @@ define <4 x double> @exp_v4f64(<4 x double> %a) {
 ; CHECK-SD-NEXT:    mov d0, v0.d[1]
 ; CHECK-SD-NEXT:    str q1, [sp, #32] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    bl exp
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -183,6 +186,7 @@ define <4 x double> @exp_v4f64(<4 x double> %a) {
 ; CHECK-SD-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    mov d0, v0.d[1]
 ; CHECK-SD-NEXT:    bl exp
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -208,13 +212,16 @@ define <4 x double> @exp_v4f64(<4 x double> %a) {
 ; CHECK-GI-NEXT:    mov d9, v1.d[1]
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GI-NEXT:    bl exp
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov d0, d8
 ; CHECK-GI-NEXT:    bl exp
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GI-NEXT:    bl exp
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov d0, d9
 ; CHECK-GI-NEXT:    bl exp
@@ -244,7 +251,8 @@ define <2 x float> @exp_v2f32(<2 x float> %a) {
 ; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    bl expf
-; CHECK-SD-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    bl expf
@@ -268,7 +276,8 @@ define <2 x float> @exp_v2f32(<2 x float> %a) {
 ; CHECK-GI-NEXT:    mov s8, v0.s[1]
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-GI-NEXT:    bl expf
-; CHECK-GI-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s8
 ; CHECK-GI-NEXT:    bl expf
 ; CHECK-GI-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
@@ -294,7 +303,8 @@ define <3 x float> @exp_v3f32(<3 x float> %a) {
 ; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    bl expf
-; CHECK-SD-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    bl expf
@@ -326,10 +336,12 @@ define <3 x float> @exp_v3f32(<3 x float> %a) {
 ; CHECK-GI-NEXT:    mov s9, v0.s[2]
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-GI-NEXT:    bl expf
-; CHECK-GI-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s8
 ; CHECK-GI-NEXT:    bl expf
-; CHECK-GI-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s9
 ; CHECK-GI-NEXT:    bl expf
 ; CHECK-GI-NEXT:    ldp q2, q1, [sp] // 32-byte Folded Reload
@@ -357,7 +369,8 @@ define <4 x float> @exp_v4f32(<4 x float> %a) {
 ; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    bl expf
-; CHECK-SD-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    bl expf
@@ -399,13 +412,16 @@ define <4 x float> @exp_v4f32(<4 x float> %a) {
 ; CHECK-GI-NEXT:    mov s10, v0.s[3]
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-GI-NEXT:    bl expf
-; CHECK-GI-NEXT:    str d0, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s8
 ; CHECK-GI-NEXT:    bl expf
-; CHECK-GI-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s9
 ; CHECK-GI-NEXT:    bl expf
-; CHECK-GI-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s10
 ; CHECK-GI-NEXT:    bl expf
 ; CHECK-GI-NEXT:    ldp q2, q1, [sp, #16] // 32-byte Folded Reload
@@ -435,7 +451,8 @@ define <8 x float> @exp_v8f32(<8 x float> %a) {
 ; CHECK-SD-NEXT:    stp q0, q1, [sp] // 32-byte Folded Spill
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    bl expf
-; CHECK-SD-NEXT:    str d0, [sp, #32] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    bl expf
@@ -460,7 +477,8 @@ define <8 x float> @exp_v8f32(<8 x float> %a) {
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    str q1, [sp, #32] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    bl expf
-; CHECK-SD-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    bl expf
@@ -510,26 +528,33 @@ define <8 x float> @exp_v8f32(<8 x float> %a) {
 ; CHECK-GI-NEXT:    mov s12, v1.s[2]
 ; CHECK-GI-NEXT:    mov s13, v1.s[3]
 ; CHECK-GI-NEXT:    bl expf
-; CHECK-GI-NEXT:    str d0, [sp, #64] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #64] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s8
 ; CHECK-GI-NEXT:    bl expf
-; CHECK-GI-NEXT:    str d0, [sp, #48] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #48] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s9
 ; CHECK-GI-NEXT:    bl expf
-; CHECK-GI-NEXT:    str d0, [sp, #96] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #96] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s10
 ; CHECK-GI-NEXT:    bl expf
-; CHECK-GI-NEXT:    str d0, [sp, #80] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #80] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-GI-NEXT:    bl expf
-; CHECK-GI-NEXT:    str d0, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s11
 ; CHECK-GI-NEXT:    bl expf
-; CHECK-GI-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s12
 ; CHECK-GI-NEXT:    bl expf
-; CHECK-GI-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s13
 ; CHECK-GI-NEXT:    bl expf
 ; CHECK-GI-NEXT:    ldp q2, q1, [sp, #48] // 32-byte Folded Reload
@@ -1320,6 +1345,7 @@ define <2 x double> @exp2_v2f64(<2 x double> %a) {
 ; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    mov d0, v0.d[1]
 ; CHECK-SD-NEXT:    bl exp2
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -1342,6 +1368,7 @@ define <2 x double> @exp2_v2f64(<2 x double> %a) {
 ; CHECK-GI-NEXT:    mov d8, v0.d[1]
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GI-NEXT:    bl exp2
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov d0, d8
 ; CHECK-GI-NEXT:    bl exp2
@@ -1428,6 +1455,7 @@ define <4 x double> @exp2_v4f64(<4 x double> %a) {
 ; CHECK-SD-NEXT:    mov d0, v0.d[1]
 ; CHECK-SD-NEXT:    str q1, [sp, #32] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    bl exp2
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -1439,6 +1467,7 @@ define <4 x double> @exp2_v4f64(<4 x double> %a) {
 ; CHECK-SD-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    mov d0, v0.d[1]
 ; CHECK-SD-NEXT:    bl exp2
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -1464,13 +1493,16 @@ define <4 x double> @exp2_v4f64(<4 x double> %a) {
 ; CHECK-GI-NEXT:    mov d9, v1.d[1]
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GI-NEXT:    bl exp2
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov d0, d8
 ; CHECK-GI-NEXT:    bl exp2
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GI-NEXT:    bl exp2
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov d0, d9
 ; CHECK-GI-NEXT:    bl exp2
@@ -1500,7 +1532,8 @@ define <2 x float> @exp2_v2f32(<2 x float> %a) {
 ; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    bl exp2f
-; CHECK-SD-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    bl exp2f
@@ -1524,7 +1557,8 @@ define <2 x float> @exp2_v2f32(<2 x float> %a) {
 ; CHECK-GI-NEXT:    mov s8, v0.s[1]
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-GI-NEXT:    bl exp2f
-; CHECK-GI-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s8
 ; CHECK-GI-NEXT:    bl exp2f
 ; CHECK-GI-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
@@ -1550,7 +1584,8 @@ define <3 x float> @exp2_v3f32(<3 x float> %a) {
 ; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    bl exp2f
-; CHECK-SD-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    bl exp2f
@@ -1582,10 +1617,12 @@ define <3 x float> @exp2_v3f32(<3 x float> %a) {
 ; CHECK-GI-NEXT:    mov s9, v0.s[2]
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-GI-NEXT:    bl exp2f
-; CHECK-GI-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s8
 ; CHECK-GI-NEXT:    bl exp2f
-; CHECK-GI-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s9
 ; CHECK-GI-NEXT:    bl exp2f
 ; CHECK-GI-NEXT:    ldp q2, q1, [sp] // 32-byte Folded Reload
@@ -1613,7 +1650,8 @@ define <4 x float> @exp2_v4f32(<4 x float> %a) {
 ; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    bl exp2f
-; CHECK-SD-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    bl exp2f
@@ -1655,13 +1693,16 @@ define <4 x float> @exp2_v4f32(<4 x float> %a) {
 ; CHECK-GI-NEXT:    mov s10, v0.s[3]
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-GI-NEXT:    bl exp2f
-; CHECK-GI-NEXT:    str d0, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s8
 ; CHECK-GI-NEXT:    bl exp2f
-; CHECK-GI-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s9
 ; CHECK-GI-NEXT:    bl exp2f
-; CHECK-GI-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s10
 ; CHECK-GI-NEXT:    bl exp2f
 ; CHECK-GI-NEXT:    ldp q2, q1, [sp, #16] // 32-byte Folded Reload
@@ -1691,7 +1732,8 @@ define <8 x float> @exp2_v8f32(<8 x float> %a) {
 ; CHECK-SD-NEXT:    stp q0, q1, [sp] // 32-byte Folded Spill
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    bl exp2f
-; CHECK-SD-NEXT:    str d0, [sp, #32] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    bl exp2f
@@ -1716,7 +1758,8 @@ define <8 x float> @exp2_v8f32(<8 x float> %a) {
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    str q1, [sp, #32] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    bl exp2f
-; CHECK-SD-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    bl exp2f
@@ -1766,26 +1809,33 @@ define <8 x float> @exp2_v8f32(<8 x float> %a) {
 ; CHECK-GI-NEXT:    mov s12, v1.s[2]
 ; CHECK-GI-NEXT:    mov s13, v1.s[3]
 ; CHECK-GI-NEXT:    bl exp2f
-; CHECK-GI-NEXT:    str d0, [sp, #64] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #64] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s8
 ; CHECK-GI-NEXT:    bl exp2f
-; CHECK-GI-NEXT:    str d0, [sp, #48] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #48] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s9
 ; CHECK-GI-NEXT:    bl exp2f
-; CHECK-GI-NEXT:    str d0, [sp, #96] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #96] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s10
 ; CHECK-GI-NEXT:    bl exp2f
-; CHECK-GI-NEXT:    str d0, [sp, #80] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #80] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-GI-NEXT:    bl exp2f
-; CHECK-GI-NEXT:    str d0, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s11
 ; CHECK-GI-NEXT:    bl exp2f
-; CHECK-GI-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s12
 ; CHECK-GI-NEXT:    bl exp2f
-; CHECK-GI-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s13
 ; CHECK-GI-NEXT:    bl exp2f
 ; CHECK-GI-NEXT:    ldp q2, q1, [sp, #48] // 32-byte Folded Reload
@@ -2576,6 +2626,7 @@ define <2 x double> @log_v2f64(<2 x double> %a) {
 ; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    mov d0, v0.d[1]
 ; CHECK-SD-NEXT:    bl log
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -2598,6 +2649,7 @@ define <2 x double> @log_v2f64(<2 x double> %a) {
 ; CHECK-GI-NEXT:    mov d8, v0.d[1]
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GI-NEXT:    bl log
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov d0, d8
 ; CHECK-GI-NEXT:    bl log
@@ -2684,6 +2736,7 @@ define <4 x double> @log_v4f64(<4 x double> %a) {
 ; CHECK-SD-NEXT:    mov d0, v0.d[1]
 ; CHECK-SD-NEXT:    str q1, [sp, #32] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    bl log
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -2695,6 +2748,7 @@ define <4 x double> @log_v4f64(<4 x double> %a) {
 ; CHECK-SD-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    mov d0, v0.d[1]
 ; CHECK-SD-NEXT:    bl log
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -2720,13 +2774,16 @@ define <4 x double> @log_v4f64(<4 x double> %a) {
 ; CHECK-GI-NEXT:    mov d9, v1.d[1]
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GI-NEXT:    bl log
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov d0, d8
 ; CHECK-GI-NEXT:    bl log
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GI-NEXT:    bl log
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov d0, d9
 ; CHECK-GI-NEXT:    bl log
@@ -2756,7 +2813,8 @@ define <2 x float> @log_v2f32(<2 x float> %a) {
 ; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    bl logf
-; CHECK-SD-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    bl logf
@@ -2780,7 +2838,8 @@ define <2 x float> @log_v2f32(<2 x float> %a) {
 ; CHECK-GI-NEXT:    mov s8, v0.s[1]
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-GI-NEXT:    bl logf
-; CHECK-GI-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s8
 ; CHECK-GI-NEXT:    bl logf
 ; CHECK-GI-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
@@ -2806,7 +2865,8 @@ define <3 x float> @log_v3f32(<3 x float> %a) {
 ; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    bl logf
-; CHECK-SD-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    bl logf
@@ -2838,10 +2898,12 @@ define <3 x float> @log_v3f32(<3 x float> %a) {
 ; CHECK-GI-NEXT:    mov s9, v0.s[2]
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-GI-NEXT:    bl logf
-; CHECK-GI-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s8
 ; CHECK-GI-NEXT:    bl logf
-; CHECK-GI-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s9
 ; CHECK-GI-NEXT:    bl logf
 ; CHECK-GI-NEXT:    ldp q2, q1, [sp] // 32-byte Folded Reload
@@ -2869,7 +2931,8 @@ define <4 x float> @log_v4f32(<4 x float> %a) {
 ; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    bl logf
-; CHECK-SD-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    bl logf
@@ -2911,13 +2974,16 @@ define <4 x float> @log_v4f32(<4 x float> %a) {
 ; CHECK-GI-NEXT:    mov s10, v0.s[3]
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-GI-NEXT:    bl logf
-; CHECK-GI-NEXT:    str d0, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s8
 ; CHECK-GI-NEXT:    bl logf
-; CHECK-GI-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s9
 ; CHECK-GI-NEXT:    bl logf
-; CHECK-GI-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s10
 ; CHECK-GI-NEXT:    bl logf
 ; CHECK-GI-NEXT:    ldp q2, q1, [sp, #16] // 32-byte Folded Reload
@@ -2947,7 +3013,8 @@ define <8 x float> @log_v8f32(<8 x float> %a) {
 ; CHECK-SD-NEXT:    stp q0, q1, [sp] // 32-byte Folded Spill
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    bl logf
-; CHECK-SD-NEXT:    str d0, [sp, #32] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    bl logf
@@ -2972,7 +3039,8 @@ define <8 x float> @log_v8f32(<8 x float> %a) {
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    str q1, [sp, #32] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    bl logf
-; CHECK-SD-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    bl logf
@@ -3022,26 +3090,33 @@ define <8 x float> @log_v8f32(<8 x float> %a) {
 ; CHECK-GI-NEXT:    mov s12, v1.s[2]
 ; CHECK-GI-NEXT:    mov s13, v1.s[3]
 ; CHECK-GI-NEXT:    bl logf
-; CHECK-GI-NEXT:    str d0, [sp, #64] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #64] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s8
 ; CHECK-GI-NEXT:    bl logf
-; CHECK-GI-NEXT:    str d0, [sp, #48] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #48] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s9
 ; CHECK-GI-NEXT:    bl logf
-; CHECK-GI-NEXT:    str d0, [sp, #96] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #96] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s10
 ; CHECK-GI-NEXT:    bl logf
-; CHECK-GI-NEXT:    str d0, [sp, #80] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #80] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-GI-NEXT:    bl logf
-; CHECK-GI-NEXT:    str d0, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s11
 ; CHECK-GI-NEXT:    bl logf
-; CHECK-GI-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s12
 ; CHECK-GI-NEXT:    bl logf
-; CHECK-GI-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s13
 ; CHECK-GI-NEXT:    bl logf
 ; CHECK-GI-NEXT:    ldp q2, q1, [sp, #48] // 32-byte Folded Reload
@@ -3832,6 +3907,7 @@ define <2 x double> @log2_v2f64(<2 x double> %a) {
 ; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    mov d0, v0.d[1]
 ; CHECK-SD-NEXT:    bl log2
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -3854,6 +3930,7 @@ define <2 x double> @log2_v2f64(<2 x double> %a) {
 ; CHECK-GI-NEXT:    mov d8, v0.d[1]
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GI-NEXT:    bl log2
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov d0, d8
 ; CHECK-GI-NEXT:    bl log2
@@ -3940,6 +4017,7 @@ define <4 x double> @log2_v4f64(<4 x double> %a) {
 ; CHECK-SD-NEXT:    mov d0, v0.d[1]
 ; CHECK-SD-NEXT:    str q1, [sp, #32] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    bl log2
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -3951,6 +4029,7 @@ define <4 x double> @log2_v4f64(<4 x double> %a) {
 ; CHECK-SD-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    mov d0, v0.d[1]
 ; CHECK-SD-NEXT:    bl log2
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -3976,13 +4055,16 @@ define <4 x double> @log2_v4f64(<4 x double> %a) {
 ; CHECK-GI-NEXT:    mov d9, v1.d[1]
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GI-NEXT:    bl log2
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov d0, d8
 ; CHECK-GI-NEXT:    bl log2
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GI-NEXT:    bl log2
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov d0, d9
 ; CHECK-GI-NEXT:    bl log2
@@ -4012,7 +4094,8 @@ define <2 x float> @log2_v2f32(<2 x float> %a) {
 ; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    bl log2f
-; CHECK-SD-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    bl log2f
@@ -4036,7 +4119,8 @@ define <2 x float> @log2_v2f32(<2 x float> %a) {
 ; CHECK-GI-NEXT:    mov s8, v0.s[1]
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-GI-NEXT:    bl log2f
-; CHECK-GI-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s8
 ; CHECK-GI-NEXT:    bl log2f
 ; CHECK-GI-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
@@ -4062,7 +4146,8 @@ define <3 x float> @log2_v3f32(<3 x float> %a) {
 ; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    bl log2f
-; CHECK-SD-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    bl log2f
@@ -4094,10 +4179,12 @@ define <3 x float> @log2_v3f32(<3 x float> %a) {
 ; CHECK-GI-NEXT:    mov s9, v0.s[2]
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-GI-NEXT:    bl log2f
-; CHECK-GI-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s8
 ; CHECK-GI-NEXT:    bl log2f
-; CHECK-GI-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s9
 ; CHECK-GI-NEXT:    bl log2f
 ; CHECK-GI-NEXT:    ldp q2, q1, [sp] // 32-byte Folded Reload
@@ -4125,7 +4212,8 @@ define <4 x float> @log2_v4f32(<4 x float> %a) {
 ; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    bl log2f
-; CHECK-SD-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    bl log2f
@@ -4167,13 +4255,16 @@ define <4 x float> @log2_v4f32(<4 x float> %a) {
 ; CHECK-GI-NEXT:    mov s10, v0.s[3]
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-GI-NEXT:    bl log2f
-; CHECK-GI-NEXT:    str d0, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s8
 ; CHECK-GI-NEXT:    bl log2f
-; CHECK-GI-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s9
 ; CHECK-GI-NEXT:    bl log2f
-; CHECK-GI-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s10
 ; CHECK-GI-NEXT:    bl log2f
 ; CHECK-GI-NEXT:    ldp q2, q1, [sp, #16] // 32-byte Folded Reload
@@ -4203,7 +4294,8 @@ define <8 x float> @log2_v8f32(<8 x float> %a) {
 ; CHECK-SD-NEXT:    stp q0, q1, [sp] // 32-byte Folded Spill
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    bl log2f
-; CHECK-SD-NEXT:    str d0, [sp, #32] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    bl log2f
@@ -4228,7 +4320,8 @@ define <8 x float> @log2_v8f32(<8 x float> %a) {
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    str q1, [sp, #32] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    bl log2f
-; CHECK-SD-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    bl log2f
@@ -4278,26 +4371,33 @@ define <8 x float> @log2_v8f32(<8 x float> %a) {
 ; CHECK-GI-NEXT:    mov s12, v1.s[2]
 ; CHECK-GI-NEXT:    mov s13, v1.s[3]
 ; CHECK-GI-NEXT:    bl log2f
-; CHECK-GI-NEXT:    str d0, [sp, #64] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #64] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s8
 ; CHECK-GI-NEXT:    bl log2f
-; CHECK-GI-NEXT:    str d0, [sp, #48] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #48] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s9
 ; CHECK-GI-NEXT:    bl log2f
-; CHECK-GI-NEXT:    str d0, [sp, #96] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #96] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s10
 ; CHECK-GI-NEXT:    bl log2f
-; CHECK-GI-NEXT:    str d0, [sp, #80] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #80] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-GI-NEXT:    bl log2f
-; CHECK-GI-NEXT:    str d0, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s11
 ; CHECK-GI-NEXT:    bl log2f
-; CHECK-GI-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s12
 ; CHECK-GI-NEXT:    bl log2f
-; CHECK-GI-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s13
 ; CHECK-GI-NEXT:    bl log2f
 ; CHECK-GI-NEXT:    ldp q2, q1, [sp, #48] // 32-byte Folded Reload
@@ -5088,6 +5188,7 @@ define <2 x double> @log10_v2f64(<2 x double> %a) {
 ; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    mov d0, v0.d[1]
 ; CHECK-SD-NEXT:    bl log10
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -5110,6 +5211,7 @@ define <2 x double> @log10_v2f64(<2 x double> %a) {
 ; CHECK-GI-NEXT:    mov d8, v0.d[1]
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GI-NEXT:    bl log10
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov d0, d8
 ; CHECK-GI-NEXT:    bl log10
@@ -5196,6 +5298,7 @@ define <4 x double> @log10_v4f64(<4 x double> %a) {
 ; CHECK-SD-NEXT:    mov d0, v0.d[1]
 ; CHECK-SD-NEXT:    str q1, [sp, #32] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    bl log10
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -5207,6 +5310,7 @@ define <4 x double> @log10_v4f64(<4 x double> %a) {
 ; CHECK-SD-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    mov d0, v0.d[1]
 ; CHECK-SD-NEXT:    bl log10
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -5232,13 +5336,16 @@ define <4 x double> @log10_v4f64(<4 x double> %a) {
 ; CHECK-GI-NEXT:    mov d9, v1.d[1]
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GI-NEXT:    bl log10
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov d0, d8
 ; CHECK-GI-NEXT:    bl log10
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GI-NEXT:    bl log10
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov d0, d9
 ; CHECK-GI-NEXT:    bl log10
@@ -5268,7 +5375,8 @@ define <2 x float> @log10_v2f32(<2 x float> %a) {
 ; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    bl log10f
-; CHECK-SD-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    bl log10f
@@ -5292,7 +5400,8 @@ define <2 x float> @log10_v2f32(<2 x float> %a) {
 ; CHECK-GI-NEXT:    mov s8, v0.s[1]
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-GI-NEXT:    bl log10f
-; CHECK-GI-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s8
 ; CHECK-GI-NEXT:    bl log10f
 ; CHECK-GI-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
@@ -5318,7 +5427,8 @@ define <3 x float> @log10_v3f32(<3 x float> %a) {
 ; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    bl log10f
-; CHECK-SD-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    bl log10f
@@ -5350,10 +5460,12 @@ define <3 x float> @log10_v3f32(<3 x float> %a) {
 ; CHECK-GI-NEXT:    mov s9, v0.s[2]
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-GI-NEXT:    bl log10f
-; CHECK-GI-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s8
 ; CHECK-GI-NEXT:    bl log10f
-; CHECK-GI-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s9
 ; CHECK-GI-NEXT:    bl log10f
 ; CHECK-GI-NEXT:    ldp q2, q1, [sp] // 32-byte Folded Reload
@@ -5381,7 +5493,8 @@ define <4 x float> @log10_v4f32(<4 x float> %a) {
 ; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    bl log10f
-; CHECK-SD-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    bl log10f
@@ -5423,13 +5536,16 @@ define <4 x float> @log10_v4f32(<4 x float> %a) {
 ; CHECK-GI-NEXT:    mov s10, v0.s[3]
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-GI-NEXT:    bl log10f
-; CHECK-GI-NEXT:    str d0, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s8
 ; CHECK-GI-NEXT:    bl log10f
-; CHECK-GI-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s9
 ; CHECK-GI-NEXT:    bl log10f
-; CHECK-GI-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s10
 ; CHECK-GI-NEXT:    bl log10f
 ; CHECK-GI-NEXT:    ldp q2, q1, [sp, #16] // 32-byte Folded Reload
@@ -5459,7 +5575,8 @@ define <8 x float> @log10_v8f32(<8 x float> %a) {
 ; CHECK-SD-NEXT:    stp q0, q1, [sp] // 32-byte Folded Spill
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    bl log10f
-; CHECK-SD-NEXT:    str d0, [sp, #32] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    bl log10f
@@ -5484,7 +5601,8 @@ define <8 x float> @log10_v8f32(<8 x float> %a) {
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    str q1, [sp, #32] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    bl log10f
-; CHECK-SD-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    bl log10f
@@ -5534,26 +5652,33 @@ define <8 x float> @log10_v8f32(<8 x float> %a) {
 ; CHECK-GI-NEXT:    mov s12, v1.s[2]
 ; CHECK-GI-NEXT:    mov s13, v1.s[3]
 ; CHECK-GI-NEXT:    bl log10f
-; CHECK-GI-NEXT:    str d0, [sp, #64] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #64] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s8
 ; CHECK-GI-NEXT:    bl log10f
-; CHECK-GI-NEXT:    str d0, [sp, #48] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #48] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s9
 ; CHECK-GI-NEXT:    bl log10f
-; CHECK-GI-NEXT:    str d0, [sp, #96] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #96] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s10
 ; CHECK-GI-NEXT:    bl log10f
-; CHECK-GI-NEXT:    str d0, [sp, #80] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #80] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-GI-NEXT:    bl log10f
-; CHECK-GI-NEXT:    str d0, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s11
 ; CHECK-GI-NEXT:    bl log10f
-; CHECK-GI-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s12
 ; CHECK-GI-NEXT:    bl log10f
-; CHECK-GI-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s13
 ; CHECK-GI-NEXT:    bl log10f
 ; CHECK-GI-NEXT:    ldp q2, q1, [sp, #48] // 32-byte Folded Reload
diff --git a/llvm/test/CodeGen/AArch64/fmla.ll b/llvm/test/CodeGen/AArch64/fmla.ll
new file mode 100644
index 0000000000..3ae2158a18
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/fmla.ll
@@ -0,0 +1,2274 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16
+; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16
+; RUN: llc -mtriple=aarch64-none-eabi -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16
+; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16
+
+define double @fma_f64(double %a, double %b, double %c) {
+; CHECK-LABEL: fma_f64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmadd d0, d0, d1, d2
+; CHECK-NEXT:    ret
+entry:
+  %d = call double @llvm.fma.f64(double %a, double %b, double %c)
+  ret double %d
+}
+
+define float @fma_f32(float %a, float %b, float %c) {
+; CHECK-LABEL: fma_f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmadd s0, s0, s1, s2
+; CHECK-NEXT:    ret
+entry:
+  %d = call float @llvm.fma.f32(float %a, float %b, float %c)
+  ret float %d
+}
+
+define half @fma_f16(half %a, half %b, half %c) {
+; CHECK-SD-NOFP16-LABEL: fma_f16:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fmadd s0, s0, s1, s2
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fma_f16:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fmadd h0, h0, h1, h2
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fma_f16:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-GI-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-GI-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-GI-NOFP16-NEXT:    fmadd s0, s0, s1, s2
+; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fma_f16:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    fmadd h0, h0, h1, h2
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %d = call half @llvm.fma.f16(half %a, half %b, half %c)
+  ret half %d
+}
+
+define <2 x double> @fma_v2f64(<2 x double> %a, <2 x double> %b, <2 x double> %c) {
+; CHECK-LABEL: fma_v2f64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmla v2.2d, v1.2d, v0.2d
+; CHECK-NEXT:    mov v0.16b, v2.16b
+; CHECK-NEXT:    ret
+entry:
+  %d = call <2 x double> @llvm.fma.v2f64(<2 x double> %a, <2 x double> %b, <2 x double> %c)
+  ret <2 x double> %d
+}
+
+define <3 x double> @fma_v3f64(<3 x double> %a, <3 x double> %b, <3 x double> %c) {
+; CHECK-SD-LABEL: fma_v3f64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d6 killed $d6 def $q6
+; CHECK-SD-NEXT:    // kill: def $d3 killed $d3 def $q3
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    // kill: def $d7 killed $d7 def $q7
+; CHECK-SD-NEXT:    // kill: def $d4 killed $d4 def $q4
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-SD-NEXT:    // kill: def $d5 killed $d5 def $q5
+; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-SD-NEXT:    mov v3.d[1], v4.d[0]
+; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-SD-NEXT:    mov v6.d[1], v7.d[0]
+; CHECK-SD-NEXT:    fmla v6.2d, v3.2d, v0.2d
+; CHECK-SD-NEXT:    ldr d3, [sp]
+; CHECK-SD-NEXT:    fmla v3.2d, v5.2d, v2.2d
+; CHECK-SD-NEXT:    fmov d0, d6
+; CHECK-SD-NEXT:    ext v1.16b, v6.16b, v6.16b, #8
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-SD-NEXT:    fmov d2, d3
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fma_v3f64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    // kill: def $d3 killed $d3 def $q3
+; CHECK-GI-NEXT:    // kill: def $d6 killed $d6 def $q6
+; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-GI-NEXT:    // kill: def $d4 killed $d4 def $q4
+; CHECK-GI-NEXT:    // kill: def $d7 killed $d7 def $q7
+; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-GI-NEXT:    mov v3.d[1], v4.d[0]
+; CHECK-GI-NEXT:    mov v6.d[1], v7.d[0]
+; CHECK-GI-NEXT:    fmla v6.2d, v3.2d, v0.2d
+; CHECK-GI-NEXT:    ldr d0, [sp]
+; CHECK-GI-NEXT:    fmadd d2, d2, d5, d0
+; CHECK-GI-NEXT:    mov d1, v6.d[1]
+; CHECK-GI-NEXT:    fmov d0, d6
+; CHECK-GI-NEXT:    ret
+entry:
+  %d = call <3 x double> @llvm.fma.v3f64(<3 x double> %a, <3 x double> %b, <3 x double> %c)
+  ret <3 x double> %d
+}
+
+define <4 x double> @fma_v4f64(<4 x double> %a, <4 x double> %b, <4 x double> %c) {
+; CHECK-LABEL: fma_v4f64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmla v4.2d, v2.2d, v0.2d
+; CHECK-NEXT:    fmla v5.2d, v3.2d, v1.2d
+; CHECK-NEXT:    mov v0.16b, v4.16b
+; CHECK-NEXT:    mov v1.16b, v5.16b
+; CHECK-NEXT:    ret
+entry:
+  %d = call <4 x double> @llvm.fma.v4f64(<4 x double> %a, <4 x double> %b, <4 x double> %c)
+  ret <4 x double> %d
+}
+
+define <2 x float> @fma_v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
+; CHECK-LABEL: fma_v2f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmla v2.2s, v1.2s, v0.2s
+; CHECK-NEXT:    fmov d0, d2
+; CHECK-NEXT:    ret
+entry:
+  %d = call <2 x float> @llvm.fma.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c)
+  ret <2 x float> %d
+}
+
+define <3 x float> @fma_v3f32(<3 x float> %a, <3 x float> %b, <3 x float> %c) {
+; CHECK-LABEL: fma_v3f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmla v2.4s, v1.4s, v0.4s
+; CHECK-NEXT:    mov v0.16b, v2.16b
+; CHECK-NEXT:    ret
+entry:
+  %d = call <3 x float> @llvm.fma.v3f32(<3 x float> %a, <3 x float> %b, <3 x float> %c)
+  ret <3 x float> %d
+}
+
+define <4 x float> @fma_v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
+; CHECK-LABEL: fma_v4f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmla v2.4s, v1.4s, v0.4s
+; CHECK-NEXT:    mov v0.16b, v2.16b
+; CHECK-NEXT:    ret
+entry:
+  %d = call <4 x float> @llvm.fma.v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c)
+  ret <4 x float> %d
+}
+
+define <8 x float> @fma_v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c) {
+; CHECK-LABEL: fma_v8f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmla v4.4s, v2.4s, v0.4s
+; CHECK-NEXT:    fmla v5.4s, v3.4s, v1.4s
+; CHECK-NEXT:    mov v0.16b, v4.16b
+; CHECK-NEXT:    mov v1.16b, v5.16b
+; CHECK-NEXT:    ret
+entry:
+  %d = call <8 x float> @llvm.fma.v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c)
+  ret <8 x float> %d
+}
+
+define <7 x half> @fma_v7f16(<7 x half> %a, <7 x half> %b, <7 x half> %c) {
+; CHECK-SD-NOFP16-LABEL: fma_v7f16:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    mov h3, v2.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvt s6, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt s7, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s16, h0
+; CHECK-SD-NOFP16-NEXT:    mov h17, v2.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[2]
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    fmadd s6, s16, s7, s6
+; CHECK-SD-NOFP16-NEXT:    fcvt s7, h17
+; CHECK-SD-NOFP16-NEXT:    fcvt s16, h18
+; CHECK-SD-NOFP16-NEXT:    fcvt s17, h19
+; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[3]
+; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[3]
+; CHECK-SD-NOFP16-NEXT:    fmadd s4, s5, s4, s3
+; CHECK-SD-NOFP16-NEXT:    mov h5, v2.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt h3, s6
+; CHECK-SD-NOFP16-NEXT:    fmadd s6, s17, s16, s7
+; CHECK-SD-NOFP16-NEXT:    mov h17, v2.h[4]
+; CHECK-SD-NOFP16-NEXT:    fcvt s7, h18
+; CHECK-SD-NOFP16-NEXT:    fcvt s16, h19
+; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[4]
+; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[4]
+; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
+; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
+; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
+; CHECK-SD-NOFP16-NEXT:    mov v3.h[1], v4.h[0]
+; CHECK-SD-NOFP16-NEXT:    mov h4, v2.h[5]
+; CHECK-SD-NOFP16-NEXT:    fmadd s5, s16, s7, s5
+; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[5]
+; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[5]
+; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
+; CHECK-SD-NOFP16-NEXT:    mov v3.h[2], v6.h[0]
+; CHECK-SD-NOFP16-NEXT:    mov h6, v2.h[6]
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
+; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
+; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
+; CHECK-SD-NOFP16-NEXT:    fmadd s17, s19, s18, s17
+; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[6]
+; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[6]
+; CHECK-SD-NOFP16-NEXT:    mov h2, v2.h[7]
+; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
+; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
+; CHECK-SD-NOFP16-NEXT:    fmadd s4, s16, s7, s4
+; CHECK-SD-NOFP16-NEXT:    mov v3.h[3], v5.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
+; CHECK-SD-NOFP16-NEXT:    fcvt s6, h18
+; CHECK-SD-NOFP16-NEXT:    fcvt s7, h19
+; CHECK-SD-NOFP16-NEXT:    fcvt h16, s17
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
+; CHECK-SD-NOFP16-NEXT:    fmadd s5, s7, s6, s5
+; CHECK-SD-NOFP16-NEXT:    mov v3.h[4], v16.h[0]
+; CHECK-SD-NOFP16-NEXT:    fmadd s0, s0, s1, s2
+; CHECK-SD-NOFP16-NEXT:    mov v3.h[5], v4.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-SD-NOFP16-NEXT:    mov v3.h[6], v4.h[0]
+; CHECK-SD-NOFP16-NEXT:    mov v3.h[7], v0.h[0]
+; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v3.16b
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fma_v7f16:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fmla v2.8h, v1.8h, v0.8h
+; CHECK-SD-FP16-NEXT:    mov v0.16b, v2.16b
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fma_v7f16:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    mov h3, v0.h[4]
+; CHECK-GI-NOFP16-NEXT:    mov h6, v0.h[5]
+; CHECK-GI-NOFP16-NEXT:    mov h4, v1.h[4]
+; CHECK-GI-NOFP16-NEXT:    mov h7, v1.h[5]
+; CHECK-GI-NOFP16-NEXT:    mov h5, v2.h[4]
+; CHECK-GI-NOFP16-NEXT:    mov h16, v2.h[5]
+; CHECK-GI-NOFP16-NEXT:    fcvtl v17.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v18.4s, v1.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v19.4s, v2.4h
+; CHECK-GI-NOFP16-NEXT:    mov h0, v0.h[6]
+; CHECK-GI-NOFP16-NEXT:    mov h1, v1.h[6]
+; CHECK-GI-NOFP16-NEXT:    mov h2, v2.h[6]
+; CHECK-GI-NOFP16-NEXT:    mov v3.h[1], v6.h[0]
+; CHECK-GI-NOFP16-NEXT:    mov v4.h[1], v7.h[0]
+; CHECK-GI-NOFP16-NEXT:    mov v5.h[1], v16.h[0]
+; CHECK-GI-NOFP16-NEXT:    fmla v19.4s, v18.4s, v17.4s
+; CHECK-GI-NOFP16-NEXT:    mov v3.h[2], v0.h[0]
+; CHECK-GI-NOFP16-NEXT:    mov v4.h[2], v1.h[0]
+; CHECK-GI-NOFP16-NEXT:    mov v5.h[2], v2.h[0]
+; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v19.4s
+; CHECK-GI-NOFP16-NEXT:    mov v3.h[3], v0.h[0]
+; CHECK-GI-NOFP16-NEXT:    mov v4.h[3], v0.h[0]
+; CHECK-GI-NOFP16-NEXT:    mov v5.h[3], v0.h[0]
+; CHECK-GI-NOFP16-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-NOFP16-NEXT:    mov h6, v0.h[3]
+; CHECK-GI-NOFP16-NEXT:    fcvtl v2.4s, v3.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v4.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v4.4s, v5.4h
+; CHECK-GI-NOFP16-NEXT:    mov h5, v0.h[2]
+; CHECK-GI-NOFP16-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-GI-NOFP16-NEXT:    fmla v4.4s, v3.4s, v2.4s
+; CHECK-GI-NOFP16-NEXT:    mov v0.h[2], v5.h[0]
+; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v4.4s
+; CHECK-GI-NOFP16-NEXT:    mov v0.h[3], v6.h[0]
+; CHECK-GI-NOFP16-NEXT:    mov h2, v1.h[1]
+; CHECK-GI-NOFP16-NEXT:    mov v0.h[4], v1.h[0]
+; CHECK-GI-NOFP16-NEXT:    mov h1, v1.h[2]
+; CHECK-GI-NOFP16-NEXT:    mov v0.h[5], v2.h[0]
+; CHECK-GI-NOFP16-NEXT:    mov v0.h[6], v1.h[0]
+; CHECK-GI-NOFP16-NEXT:    mov v0.h[7], v0.h[0]
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fma_v7f16:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    fmla v2.8h, v1.8h, v0.8h
+; CHECK-GI-FP16-NEXT:    mov v0.16b, v2.16b
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %d = call <7 x half> @llvm.fma.v7f16(<7 x half> %a, <7 x half> %b, <7 x half> %c)
+  ret <7 x half> %d
+}
+
+define <4 x half> @fma_v4f16(<4 x half> %a, <4 x half> %b, <4 x half> %c) {
+; CHECK-SD-NOFP16-LABEL: fma_v4f16:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-SD-NOFP16-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NOFP16-NEXT:    mov h3, v2.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvt s6, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt s7, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s16, h0
+; CHECK-SD-NOFP16-NEXT:    mov h17, v2.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h2, v2.h[3]
+; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    fmadd s6, s16, s7, s6
+; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt s7, h19
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fmadd s3, s5, s4, s3
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h17
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h18
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s6
+; CHECK-SD-NOFP16-NEXT:    fmadd s4, s7, s5, s4
+; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h16
+; CHECK-SD-NOFP16-NEXT:    mov v0.h[1], v3.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt h3, s4
+; CHECK-SD-NOFP16-NEXT:    fmadd s1, s5, s1, s2
+; CHECK-SD-NOFP16-NEXT:    mov v0.h[2], v3.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
+; CHECK-SD-NOFP16-NEXT:    mov v0.h[3], v1.h[0]
+; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fma_v4f16:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fmla v2.4h, v1.4h, v0.4h
+; CHECK-SD-FP16-NEXT:    fmov d0, d2
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fma_v4f16:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v2.4s, v2.4h
+; CHECK-GI-NOFP16-NEXT:    fmla v2.4s, v1.4s, v0.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fma_v4f16:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    fmla v2.4h, v1.4h, v0.4h
+; CHECK-GI-FP16-NEXT:    fmov d0, d2
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %d = call <4 x half> @llvm.fma.v4f16(<4 x half> %a, <4 x half> %b, <4 x half> %c)
+  ret <4 x half> %d
+}
+
+define <8 x half> @fma_v8f16(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
+; CHECK-SD-NOFP16-LABEL: fma_v8f16:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    mov h3, v2.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvt s6, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt s7, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s16, h0
+; CHECK-SD-NOFP16-NEXT:    mov h17, v2.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[2]
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    fmadd s6, s16, s7, s6
+; CHECK-SD-NOFP16-NEXT:    fcvt s7, h17
+; CHECK-SD-NOFP16-NEXT:    fcvt s16, h18
+; CHECK-SD-NOFP16-NEXT:    fcvt s17, h19
+; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[3]
+; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[3]
+; CHECK-SD-NOFP16-NEXT:    fmadd s4, s5, s4, s3
+; CHECK-SD-NOFP16-NEXT:    mov h5, v2.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt h3, s6
+; CHECK-SD-NOFP16-NEXT:    fmadd s6, s17, s16, s7
+; CHECK-SD-NOFP16-NEXT:    mov h17, v2.h[4]
+; CHECK-SD-NOFP16-NEXT:    fcvt s7, h18
+; CHECK-SD-NOFP16-NEXT:    fcvt s16, h19
+; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[4]
+; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[4]
+; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
+; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
+; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
+; CHECK-SD-NOFP16-NEXT:    mov v3.h[1], v4.h[0]
+; CHECK-SD-NOFP16-NEXT:    mov h4, v2.h[5]
+; CHECK-SD-NOFP16-NEXT:    fmadd s5, s16, s7, s5
+; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[5]
+; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[5]
+; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
+; CHECK-SD-NOFP16-NEXT:    mov v3.h[2], v6.h[0]
+; CHECK-SD-NOFP16-NEXT:    mov h6, v2.h[6]
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
+; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
+; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
+; CHECK-SD-NOFP16-NEXT:    fmadd s17, s19, s18, s17
+; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[6]
+; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[6]
+; CHECK-SD-NOFP16-NEXT:    mov h2, v2.h[7]
+; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
+; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
+; CHECK-SD-NOFP16-NEXT:    fmadd s4, s16, s7, s4
+; CHECK-SD-NOFP16-NEXT:    mov v3.h[3], v5.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
+; CHECK-SD-NOFP16-NEXT:    fcvt s6, h18
+; CHECK-SD-NOFP16-NEXT:    fcvt s7, h19
+; CHECK-SD-NOFP16-NEXT:    fcvt h16, s17
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
+; CHECK-SD-NOFP16-NEXT:    fmadd s5, s7, s6, s5
+; CHECK-SD-NOFP16-NEXT:    mov v3.h[4], v16.h[0]
+; CHECK-SD-NOFP16-NEXT:    fmadd s0, s0, s1, s2
+; CHECK-SD-NOFP16-NEXT:    mov v3.h[5], v4.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-SD-NOFP16-NEXT:    mov v3.h[6], v4.h[0]
+; CHECK-SD-NOFP16-NEXT:    mov v3.h[7], v0.h[0]
+; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v3.16b
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fma_v8f16:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fmla v2.8h, v1.8h, v0.8h
+; CHECK-SD-FP16-NEXT:    mov v0.16b, v2.16b
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fma_v8f16:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v4.4s, v1.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v5.4s, v2.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
+; CHECK-GI-NOFP16-NEXT:    fmla v5.4s, v4.4s, v3.4s
+; CHECK-GI-NOFP16-NEXT:    fmla v2.4s, v1.4s, v0.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v5.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fma_v8f16:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    fmla v2.8h, v1.8h, v0.8h
+; CHECK-GI-FP16-NEXT:    mov v0.16b, v2.16b
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %d = call <8 x half> @llvm.fma.v8f16(<8 x half> %a, <8 x half> %b, <8 x half> %c)
+  ret <8 x half> %d
+}
+
+define <16 x half> @fma_v16f16(<16 x half> %a, <16 x half> %b, <16 x half> %c) {
+; CHECK-SD-NOFP16-LABEL: fma_v16f16:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    mov h6, v4.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h7, v2.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvt s17, h4
+; CHECK-SD-NOFP16-NEXT:    fcvt s18, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt s19, h0
+; CHECK-SD-NOFP16-NEXT:    mov h20, v4.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h21, v2.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h22, v0.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h23, v4.h[3]
+; CHECK-SD-NOFP16-NEXT:    mov h24, v2.h[3]
+; CHECK-SD-NOFP16-NEXT:    mov h25, v0.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
+; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
+; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
+; CHECK-SD-NOFP16-NEXT:    fmadd s17, s19, s18, s17
+; CHECK-SD-NOFP16-NEXT:    mov h26, v1.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvt s27, h5
+; CHECK-SD-NOFP16-NEXT:    fcvt s18, h20
+; CHECK-SD-NOFP16-NEXT:    fcvt s19, h21
+; CHECK-SD-NOFP16-NEXT:    fcvt s20, h22
+; CHECK-SD-NOFP16-NEXT:    fcvt s21, h23
+; CHECK-SD-NOFP16-NEXT:    fcvt s22, h24
+; CHECK-SD-NOFP16-NEXT:    fcvt s23, h25
+; CHECK-SD-NOFP16-NEXT:    fmadd s7, s16, s7, s6
+; CHECK-SD-NOFP16-NEXT:    mov h24, v5.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h25, v3.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvt h6, s17
+; CHECK-SD-NOFP16-NEXT:    fcvt s28, h3
+; CHECK-SD-NOFP16-NEXT:    fcvt s29, h1
+; CHECK-SD-NOFP16-NEXT:    fmadd s19, s20, s19, s18
+; CHECK-SD-NOFP16-NEXT:    fcvt s26, h26
+; CHECK-SD-NOFP16-NEXT:    mov h16, v4.h[4]
+; CHECK-SD-NOFP16-NEXT:    fmadd s21, s23, s22, s21
+; CHECK-SD-NOFP16-NEXT:    mov h22, v3.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h23, v1.h[2]
+; CHECK-SD-NOFP16-NEXT:    fcvt h20, s7
+; CHECK-SD-NOFP16-NEXT:    fcvt s24, h24
+; CHECK-SD-NOFP16-NEXT:    fcvt s25, h25
+; CHECK-SD-NOFP16-NEXT:    mov h17, v2.h[4]
+; CHECK-SD-NOFP16-NEXT:    mov h18, v0.h[4]
+; CHECK-SD-NOFP16-NEXT:    mov h7, v4.h[5]
+; CHECK-SD-NOFP16-NEXT:    fcvt h19, s19
+; CHECK-SD-NOFP16-NEXT:    mov h30, v2.h[5]
+; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
+; CHECK-SD-NOFP16-NEXT:    fcvt h21, s21
+; CHECK-SD-NOFP16-NEXT:    mov h31, v1.h[4]
+; CHECK-SD-NOFP16-NEXT:    fmadd s24, s26, s25, s24
+; CHECK-SD-NOFP16-NEXT:    fmadd s25, s29, s28, s27
+; CHECK-SD-NOFP16-NEXT:    mov v6.h[1], v20.h[0]
+; CHECK-SD-NOFP16-NEXT:    mov h20, v5.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h26, v5.h[3]
+; CHECK-SD-NOFP16-NEXT:    mov h27, v3.h[3]
+; CHECK-SD-NOFP16-NEXT:    mov h28, v1.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
+; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
+; CHECK-SD-NOFP16-NEXT:    fcvt s29, h7
+; CHECK-SD-NOFP16-NEXT:    fcvt s30, h30
+; CHECK-SD-NOFP16-NEXT:    mov v6.h[2], v19.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt h24, s24
+; CHECK-SD-NOFP16-NEXT:    fcvt h7, s25
+; CHECK-SD-NOFP16-NEXT:    fcvt s19, h20
+; CHECK-SD-NOFP16-NEXT:    fcvt s20, h22
+; CHECK-SD-NOFP16-NEXT:    fcvt s22, h23
+; CHECK-SD-NOFP16-NEXT:    fmadd s16, s18, s17, s16
+; CHECK-SD-NOFP16-NEXT:    mov h23, v0.h[5]
+; CHECK-SD-NOFP16-NEXT:    fcvt s25, h26
+; CHECK-SD-NOFP16-NEXT:    fcvt s26, h27
+; CHECK-SD-NOFP16-NEXT:    fcvt s27, h28
+; CHECK-SD-NOFP16-NEXT:    mov h18, v4.h[6]
+; CHECK-SD-NOFP16-NEXT:    mov v6.h[3], v21.h[0]
+; CHECK-SD-NOFP16-NEXT:    mov v7.h[1], v24.h[0]
+; CHECK-SD-NOFP16-NEXT:    mov h24, v5.h[5]
+; CHECK-SD-NOFP16-NEXT:    fmadd s19, s22, s20, s19
+; CHECK-SD-NOFP16-NEXT:    mov h20, v5.h[4]
+; CHECK-SD-NOFP16-NEXT:    mov h22, v3.h[4]
+; CHECK-SD-NOFP16-NEXT:    fcvt s23, h23
+; CHECK-SD-NOFP16-NEXT:    mov h28, v0.h[6]
+; CHECK-SD-NOFP16-NEXT:    fcvt h16, s16
+; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
+; CHECK-SD-NOFP16-NEXT:    mov h4, v4.h[7]
+; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
+; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
+; CHECK-SD-NOFP16-NEXT:    fcvt s21, h22
+; CHECK-SD-NOFP16-NEXT:    fcvt s22, h31
+; CHECK-SD-NOFP16-NEXT:    fmadd s17, s23, s30, s29
+; CHECK-SD-NOFP16-NEXT:    fmadd s23, s27, s26, s25
+; CHECK-SD-NOFP16-NEXT:    fcvt h19, s19
+; CHECK-SD-NOFP16-NEXT:    mov h25, v3.h[5]
+; CHECK-SD-NOFP16-NEXT:    mov h26, v1.h[5]
+; CHECK-SD-NOFP16-NEXT:    mov h27, v2.h[6]
+; CHECK-SD-NOFP16-NEXT:    mov h29, v1.h[6]
+; CHECK-SD-NOFP16-NEXT:    mov h2, v2.h[7]
+; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
+; CHECK-SD-NOFP16-NEXT:    fmadd s20, s22, s21, s20
+; CHECK-SD-NOFP16-NEXT:    mov h21, v5.h[6]
+; CHECK-SD-NOFP16-NEXT:    mov h22, v3.h[6]
+; CHECK-SD-NOFP16-NEXT:    mov v7.h[2], v19.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt h19, s23
+; CHECK-SD-NOFP16-NEXT:    fcvt s23, h24
+; CHECK-SD-NOFP16-NEXT:    fcvt s24, h25
+; CHECK-SD-NOFP16-NEXT:    fcvt s25, h26
+; CHECK-SD-NOFP16-NEXT:    fcvt s26, h27
+; CHECK-SD-NOFP16-NEXT:    fcvt s27, h28
+; CHECK-SD-NOFP16-NEXT:    fcvt s28, h29
+; CHECK-SD-NOFP16-NEXT:    mov h5, v5.h[7]
+; CHECK-SD-NOFP16-NEXT:    fcvt s21, h21
+; CHECK-SD-NOFP16-NEXT:    fcvt s22, h22
+; CHECK-SD-NOFP16-NEXT:    mov h3, v3.h[7]
+; CHECK-SD-NOFP16-NEXT:    mov v7.h[3], v19.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt h19, s20
+; CHECK-SD-NOFP16-NEXT:    mov v6.h[4], v16.h[0]
+; CHECK-SD-NOFP16-NEXT:    fmadd s20, s25, s24, s23
+; CHECK-SD-NOFP16-NEXT:    fcvt h16, s17
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fmadd s18, s27, s26, s18
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fmadd s21, s28, s22, s21
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    mov v7.h[4], v19.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt h17, s20
+; CHECK-SD-NOFP16-NEXT:    mov v6.h[5], v16.h[0]
+; CHECK-SD-NOFP16-NEXT:    fmadd s0, s0, s2, s4
+; CHECK-SD-NOFP16-NEXT:    fcvt h2, s18
+; CHECK-SD-NOFP16-NEXT:    fcvt h4, s21
+; CHECK-SD-NOFP16-NEXT:    fmadd s1, s1, s3, s5
+; CHECK-SD-NOFP16-NEXT:    mov v7.h[5], v17.h[0]
+; CHECK-SD-NOFP16-NEXT:    mov v6.h[6], v2.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
+; CHECK-SD-NOFP16-NEXT:    mov v7.h[6], v4.h[0]
+; CHECK-SD-NOFP16-NEXT:    mov v6.h[7], v0.h[0]
+; CHECK-SD-NOFP16-NEXT:    mov v7.h[7], v1.h[0]
+; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v6.16b
+; CHECK-SD-NOFP16-NEXT:    mov v1.16b, v7.16b
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fma_v16f16:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fmla v4.8h, v2.8h, v0.8h
+; CHECK-SD-FP16-NEXT:    fmla v5.8h, v3.8h, v1.8h
+; CHECK-SD-FP16-NEXT:    mov v0.16b, v4.16b
+; CHECK-SD-FP16-NEXT:    mov v1.16b, v5.16b
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fma_v16f16:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    fcvtl v6.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v7.4s, v1.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v16.4s, v2.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v17.4s, v3.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v18.4s, v4.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v19.4s, v5.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v3.4s, v3.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v4.4s, v4.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v5.4s, v5.8h
+; CHECK-GI-NOFP16-NEXT:    fmla v18.4s, v16.4s, v6.4s
+; CHECK-GI-NOFP16-NEXT:    fmla v19.4s, v17.4s, v7.4s
+; CHECK-GI-NOFP16-NEXT:    fmla v4.4s, v2.4s, v0.4s
+; CHECK-GI-NOFP16-NEXT:    fmla v5.4s, v3.4s, v1.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v18.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v19.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v4.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtn2 v1.8h, v5.4s
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fma_v16f16:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    fmla v4.8h, v2.8h, v0.8h
+; CHECK-GI-FP16-NEXT:    fmla v5.8h, v3.8h, v1.8h
+; CHECK-GI-FP16-NEXT:    mov v0.16b, v4.16b
+; CHECK-GI-FP16-NEXT:    mov v1.16b, v5.16b
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %d = call <16 x half> @llvm.fma.v16f16(<16 x half> %a, <16 x half> %b, <16 x half> %c)
+  ret <16 x half> %d
+}
+
+define double @fmuladd_f64(double %a, double %b, double %c) {
+; CHECK-LABEL: fmuladd_f64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmadd d0, d0, d1, d2
+; CHECK-NEXT:    ret
+entry:
+  %d = call double @llvm.fmuladd.f64(double %a, double %b, double %c)
+  ret double %d
+}
+
+define float @fmuladd_f32(float %a, float %b, float %c) {
+; CHECK-LABEL: fmuladd_f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmadd s0, s0, s1, s2
+; CHECK-NEXT:    ret
+entry:
+  %d = call float @llvm.fmuladd.f32(float %a, float %b, float %c)
+  ret float %d
+}
+
+define half @fmuladd_f16(half %a, half %b, half %c) {
+; CHECK-SD-NOFP16-LABEL: fmuladd_f16:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s1
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s1
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fmuladd_f16:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fmadd h0, h0, h1, h2
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fmuladd_f16:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-GI-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-GI-NOFP16-NEXT:    fmul s0, s0, s1
+; CHECK-GI-NOFP16-NEXT:    fcvt s1, h2
+; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-GI-NOFP16-NEXT:    fadd s0, s0, s1
+; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fmuladd_f16:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    fmadd h0, h0, h1, h2
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %d = call half @llvm.fmuladd.f16(half %a, half %b, half %c)
+  ret half %d
+}
+
+define <2 x double> @fmuladd_v2f64(<2 x double> %a, <2 x double> %b, <2 x double> %c) {
+; CHECK-LABEL: fmuladd_v2f64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmla v2.2d, v1.2d, v0.2d
+; CHECK-NEXT:    mov v0.16b, v2.16b
+; CHECK-NEXT:    ret
+entry:
+  %d = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> %a, <2 x double> %b, <2 x double> %c)
+  ret <2 x double> %d
+}
+
+define <3 x double> @fmuladd_v3f64(<3 x double> %a, <3 x double> %b, <3 x double> %c) {
+; CHECK-SD-LABEL: fmuladd_v3f64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d6 killed $d6 def $q6
+; CHECK-SD-NEXT:    // kill: def $d3 killed $d3 def $q3
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    // kill: def $d7 killed $d7 def $q7
+; CHECK-SD-NEXT:    // kill: def $d4 killed $d4 def $q4
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-SD-NEXT:    // kill: def $d5 killed $d5 def $q5
+; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-SD-NEXT:    mov v3.d[1], v4.d[0]
+; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-SD-NEXT:    mov v6.d[1], v7.d[0]
+; CHECK-SD-NEXT:    fmla v6.2d, v3.2d, v0.2d
+; CHECK-SD-NEXT:    ldr d3, [sp]
+; CHECK-SD-NEXT:    fmla v3.2d, v5.2d, v2.2d
+; CHECK-SD-NEXT:    fmov d0, d6
+; CHECK-SD-NEXT:    ext v1.16b, v6.16b, v6.16b, #8
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-SD-NEXT:    fmov d2, d3
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fmuladd_v3f64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    // kill: def $d3 killed $d3 def $q3
+; CHECK-GI-NEXT:    // kill: def $d6 killed $d6 def $q6
+; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-GI-NEXT:    // kill: def $d4 killed $d4 def $q4
+; CHECK-GI-NEXT:    // kill: def $d7 killed $d7 def $q7
+; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-GI-NEXT:    mov v3.d[1], v4.d[0]
+; CHECK-GI-NEXT:    mov v6.d[1], v7.d[0]
+; CHECK-GI-NEXT:    fmla v6.2d, v3.2d, v0.2d
+; CHECK-GI-NEXT:    ldr d0, [sp]
+; CHECK-GI-NEXT:    fmadd d2, d2, d5, d0
+; CHECK-GI-NEXT:    mov d1, v6.d[1]
+; CHECK-GI-NEXT:    fmov d0, d6
+; CHECK-GI-NEXT:    ret
+entry:
+  %d = call <3 x double> @llvm.fmuladd.v3f64(<3 x double> %a, <3 x double> %b, <3 x double> %c)
+  ret <3 x double> %d
+}
+
+define <4 x double> @fmuladd_v4f64(<4 x double> %a, <4 x double> %b, <4 x double> %c) {
+; CHECK-LABEL: fmuladd_v4f64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmla v4.2d, v2.2d, v0.2d
+; CHECK-NEXT:    fmla v5.2d, v3.2d, v1.2d
+; CHECK-NEXT:    mov v0.16b, v4.16b
+; CHECK-NEXT:    mov v1.16b, v5.16b
+; CHECK-NEXT:    ret
+entry:
+  %d = call <4 x double> @llvm.fmuladd.v4f64(<4 x double> %a, <4 x double> %b, <4 x double> %c)
+  ret <4 x double> %d
+}
+
+define <2 x float> @fmuladd_v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
+; CHECK-LABEL: fmuladd_v2f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmla v2.2s, v1.2s, v0.2s
+; CHECK-NEXT:    fmov d0, d2
+; CHECK-NEXT:    ret
+entry:
+  %d = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c)
+  ret <2 x float> %d
+}
+
+define <3 x float> @fmuladd_v3f32(<3 x float> %a, <3 x float> %b, <3 x float> %c) {
+; CHECK-LABEL: fmuladd_v3f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmla v2.4s, v1.4s, v0.4s
+; CHECK-NEXT:    mov v0.16b, v2.16b
+; CHECK-NEXT:    ret
+entry:
+  %d = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> %a, <3 x float> %b, <3 x float> %c)
+  ret <3 x float> %d
+}
+
+define <4 x float> @fmuladd_v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
+; CHECK-LABEL: fmuladd_v4f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmla v2.4s, v1.4s, v0.4s
+; CHECK-NEXT:    mov v0.16b, v2.16b
+; CHECK-NEXT:    ret
+entry:
+  %d = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c)
+  ret <4 x float> %d
+}
+
+define <8 x float> @fmuladd_v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c) {
+; CHECK-LABEL: fmuladd_v8f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmla v4.4s, v2.4s, v0.4s
+; CHECK-NEXT:    fmla v5.4s, v3.4s, v1.4s
+; CHECK-NEXT:    mov v0.16b, v4.16b
+; CHECK-NEXT:    mov v1.16b, v5.16b
+; CHECK-NEXT:    ret
+entry:
+  %d = call <8 x float> @llvm.fmuladd.v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c)
+  ret <8 x float> %d
+}
+
+define <7 x half> @fmuladd_v7f16(<7 x half> %a, <7 x half> %b, <7 x half> %c) {
+; CHECK-SD-NOFP16-LABEL: fmuladd_v7f16:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s6, h0
+; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h17, v0.h[3]
+; CHECK-SD-NOFP16-NEXT:    mov h18, v0.h[4]
+; CHECK-SD-NOFP16-NEXT:    fcvt s19, h2
+; CHECK-SD-NOFP16-NEXT:    mov h20, v2.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h21, v1.h[5]
+; CHECK-SD-NOFP16-NEXT:    mov h22, v0.h[6]
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fmul s5, s6, s5
+; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
+; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
+; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
+; CHECK-SD-NOFP16-NEXT:    fcvt s21, h21
+; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
+; CHECK-SD-NOFP16-NEXT:    fmul s3, s4, s3
+; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
+; CHECK-SD-NOFP16-NEXT:    fcvt s6, h17
+; CHECK-SD-NOFP16-NEXT:    fmul s7, s16, s7
+; CHECK-SD-NOFP16-NEXT:    mov h17, v2.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h16, v1.h[4]
+; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fmul s5, s6, s5
+; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[5]
+; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
+; CHECK-SD-NOFP16-NEXT:    fcvt h7, s7
+; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
+; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fadd s4, s4, s19
+; CHECK-SD-NOFP16-NEXT:    mov h19, v2.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
+; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
+; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
+; CHECK-SD-NOFP16-NEXT:    fmul s16, s18, s16
+; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[6]
+; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fadd s17, s3, s17
+; CHECK-SD-NOFP16-NEXT:    fmul s6, s6, s21
+; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
+; CHECK-SD-NOFP16-NEXT:    fcvt h3, s4
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    fadd s4, s7, s20
+; CHECK-SD-NOFP16-NEXT:    fcvt h16, s16
+; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
+; CHECK-SD-NOFP16-NEXT:    fcvt s20, h22
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt h7, s17
+; CHECK-SD-NOFP16-NEXT:    mov h17, v2.h[4]
+; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
+; CHECK-SD-NOFP16-NEXT:    fadd s5, s5, s19
+; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
+; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
+; CHECK-SD-NOFP16-NEXT:    fmul s18, s20, s18
+; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s1
+; CHECK-SD-NOFP16-NEXT:    mov h1, v2.h[6]
+; CHECK-SD-NOFP16-NEXT:    mov v3.h[1], v7.h[0]
+; CHECK-SD-NOFP16-NEXT:    mov h7, v2.h[5]
+; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
+; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
+; CHECK-SD-NOFP16-NEXT:    mov h2, v2.h[7]
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
+; CHECK-SD-NOFP16-NEXT:    fadd s16, s16, s17
+; CHECK-SD-NOFP16-NEXT:    mov v3.h[2], v4.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
+; CHECK-SD-NOFP16-NEXT:    fcvt h5, s18
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fadd s6, s6, s7
+; CHECK-SD-NOFP16-NEXT:    mov v3.h[3], v4.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt h4, s16
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    mov v3.h[4], v4.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt h4, s6
+; CHECK-SD-NOFP16-NEXT:    fadd s1, s5, s1
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-SD-NOFP16-NEXT:    mov v3.h[5], v4.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
+; CHECK-SD-NOFP16-NEXT:    mov v3.h[6], v1.h[0]
+; CHECK-SD-NOFP16-NEXT:    mov v3.h[7], v0.h[0]
+; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v3.16b
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fmuladd_v7f16:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fmla v2.8h, v1.8h, v0.8h
+; CHECK-SD-FP16-NEXT:    mov v0.16b, v2.16b
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fmuladd_v7f16:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    mov h3, v0.h[4]
+; CHECK-GI-NOFP16-NEXT:    mov h4, v0.h[5]
+; CHECK-GI-NOFP16-NEXT:    mov h5, v1.h[4]
+; CHECK-GI-NOFP16-NEXT:    mov h6, v1.h[5]
+; CHECK-GI-NOFP16-NEXT:    fcvtl v7.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v16.4s, v1.4h
+; CHECK-GI-NOFP16-NEXT:    mov h0, v0.h[6]
+; CHECK-GI-NOFP16-NEXT:    mov h1, v1.h[6]
+; CHECK-GI-NOFP16-NEXT:    mov v3.h[1], v4.h[0]
+; CHECK-GI-NOFP16-NEXT:    mov v5.h[1], v6.h[0]
+; CHECK-GI-NOFP16-NEXT:    fmul v4.4s, v7.4s, v16.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtl v6.4s, v2.4h
+; CHECK-GI-NOFP16-NEXT:    mov v3.h[2], v0.h[0]
+; CHECK-GI-NOFP16-NEXT:    mov v5.h[2], v1.h[0]
+; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v4.4s
+; CHECK-GI-NOFP16-NEXT:    mov h1, v2.h[4]
+; CHECK-GI-NOFP16-NEXT:    mov h4, v2.h[5]
+; CHECK-GI-NOFP16-NEXT:    mov h2, v2.h[6]
+; CHECK-GI-NOFP16-NEXT:    mov v3.h[3], v0.h[0]
+; CHECK-GI-NOFP16-NEXT:    mov v5.h[3], v0.h[0]
+; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    mov v1.h[1], v4.h[0]
+; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v3.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v4.4s, v5.4h
+; CHECK-GI-NOFP16-NEXT:    fadd v0.4s, v0.4s, v6.4s
+; CHECK-GI-NOFP16-NEXT:    mov v1.h[2], v2.h[0]
+; CHECK-GI-NOFP16-NEXT:    fmul v2.4s, v3.4s, v4.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-GI-NOFP16-NEXT:    mov v1.h[3], v0.h[0]
+; CHECK-GI-NOFP16-NEXT:    fcvtn v2.4h, v2.4s
+; CHECK-GI-NOFP16-NEXT:    mov h3, v0.h[1]
+; CHECK-GI-NOFP16-NEXT:    mov h4, v0.h[2]
+; CHECK-GI-NOFP16-NEXT:    mov h5, v0.h[3]
+; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v2.4s, v2.4h
+; CHECK-GI-NOFP16-NEXT:    mov v0.h[1], v3.h[0]
+; CHECK-GI-NOFP16-NEXT:    fadd v1.4s, v2.4s, v1.4s
+; CHECK-GI-NOFP16-NEXT:    mov v0.h[2], v4.h[0]
+; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v1.4s
+; CHECK-GI-NOFP16-NEXT:    mov v0.h[3], v5.h[0]
+; CHECK-GI-NOFP16-NEXT:    mov h2, v1.h[1]
+; CHECK-GI-NOFP16-NEXT:    mov v0.h[4], v1.h[0]
+; CHECK-GI-NOFP16-NEXT:    mov h1, v1.h[2]
+; CHECK-GI-NOFP16-NEXT:    mov v0.h[5], v2.h[0]
+; CHECK-GI-NOFP16-NEXT:    mov v0.h[6], v1.h[0]
+; CHECK-GI-NOFP16-NEXT:    mov v0.h[7], v0.h[0]
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fmuladd_v7f16:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    fmla v2.8h, v1.8h, v0.8h
+; CHECK-GI-FP16-NEXT:    mov v0.16b, v2.16b
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %d = call <7 x half> @llvm.fmuladd.v7f16(<7 x half> %a, <7 x half> %b, <7 x half> %c)
+  ret <7 x half> %d
+}
+
+define <4 x half> @fmuladd_v4f16(<4 x half> %a, <4 x half> %b, <4 x half> %c) {
+; CHECK-SD-NOFP16-LABEL: fmuladd_v4f16:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fmul v0.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v2.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fadd v0.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fmuladd_v4f16:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fmla v2.4h, v1.4h, v0.4h
+; CHECK-SD-FP16-NEXT:    fmov d0, d2
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fmuladd_v4f16:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-GI-NOFP16-NEXT:    fmul v0.4s, v0.4s, v1.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v2.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fadd v0.4s, v0.4s, v1.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fmuladd_v4f16:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    fmla v2.4h, v1.4h, v0.4h
+; CHECK-GI-FP16-NEXT:    fmov d0, d2
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %d = call <4 x half> @llvm.fmuladd.v4f16(<4 x half> %a, <4 x half> %b, <4 x half> %c)
+  ret <4 x half> %d
+}
+
+define <8 x half> @fmuladd_v8f16(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
+; CHECK-SD-NOFP16-LABEL: fmuladd_v8f16:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s6, h0
+; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h17, v0.h[3]
+; CHECK-SD-NOFP16-NEXT:    mov h18, v0.h[4]
+; CHECK-SD-NOFP16-NEXT:    fcvt s19, h2
+; CHECK-SD-NOFP16-NEXT:    mov h20, v2.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h21, v1.h[5]
+; CHECK-SD-NOFP16-NEXT:    mov h22, v0.h[6]
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fmul s5, s6, s5
+; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
+; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
+; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
+; CHECK-SD-NOFP16-NEXT:    fcvt s21, h21
+; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
+; CHECK-SD-NOFP16-NEXT:    fmul s3, s4, s3
+; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
+; CHECK-SD-NOFP16-NEXT:    fcvt s6, h17
+; CHECK-SD-NOFP16-NEXT:    fmul s7, s16, s7
+; CHECK-SD-NOFP16-NEXT:    mov h17, v2.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h16, v1.h[4]
+; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fmul s5, s6, s5
+; CHECK-SD-NOFP16-NEXT:    mov h6, v0.h[5]
+; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
+; CHECK-SD-NOFP16-NEXT:    fcvt h7, s7
+; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
+; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fadd s4, s4, s19
+; CHECK-SD-NOFP16-NEXT:    mov h19, v2.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
+; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
+; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
+; CHECK-SD-NOFP16-NEXT:    fmul s16, s18, s16
+; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[6]
+; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fadd s17, s3, s17
+; CHECK-SD-NOFP16-NEXT:    fmul s6, s6, s21
+; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
+; CHECK-SD-NOFP16-NEXT:    fcvt h3, s4
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    fadd s4, s7, s20
+; CHECK-SD-NOFP16-NEXT:    fcvt h16, s16
+; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
+; CHECK-SD-NOFP16-NEXT:    fcvt s20, h22
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt h7, s17
+; CHECK-SD-NOFP16-NEXT:    mov h17, v2.h[4]
+; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
+; CHECK-SD-NOFP16-NEXT:    fadd s5, s5, s19
+; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
+; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
+; CHECK-SD-NOFP16-NEXT:    fmul s18, s20, s18
+; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s1
+; CHECK-SD-NOFP16-NEXT:    mov h1, v2.h[6]
+; CHECK-SD-NOFP16-NEXT:    mov v3.h[1], v7.h[0]
+; CHECK-SD-NOFP16-NEXT:    mov h7, v2.h[5]
+; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
+; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
+; CHECK-SD-NOFP16-NEXT:    mov h2, v2.h[7]
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
+; CHECK-SD-NOFP16-NEXT:    fadd s16, s16, s17
+; CHECK-SD-NOFP16-NEXT:    mov v3.h[2], v4.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
+; CHECK-SD-NOFP16-NEXT:    fcvt h5, s18
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fadd s6, s6, s7
+; CHECK-SD-NOFP16-NEXT:    mov v3.h[3], v4.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt h4, s16
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    mov v3.h[4], v4.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt h4, s6
+; CHECK-SD-NOFP16-NEXT:    fadd s1, s5, s1
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-SD-NOFP16-NEXT:    mov v3.h[5], v4.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
+; CHECK-SD-NOFP16-NEXT:    mov v3.h[6], v1.h[0]
+; CHECK-SD-NOFP16-NEXT:    mov v3.h[7], v0.h[0]
+; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v3.16b
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fmuladd_v8f16:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fmla v2.8h, v1.8h, v0.8h
+; CHECK-SD-FP16-NEXT:    mov v0.16b, v2.16b
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fmuladd_v8f16:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v4.4s, v1.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-GI-NOFP16-NEXT:    fmul v3.4s, v3.4s, v4.4s
+; CHECK-GI-NOFP16-NEXT:    fmul v0.4s, v0.4s, v1.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v3.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v2.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fadd v1.4s, v1.4s, v3.4s
+; CHECK-GI-NOFP16-NEXT:    fadd v2.4s, v0.4s, v2.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fmuladd_v8f16:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    fmla v2.8h, v1.8h, v0.8h
+; CHECK-GI-FP16-NEXT:    mov v0.16b, v2.16b
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %d = call <8 x half> @llvm.fmuladd.v8f16(<8 x half> %a, <8 x half> %b, <8 x half> %c)
+  ret <8 x half> %d
+}
+
+define <16 x half> @fmuladd_v16f16(<16 x half> %a, <16 x half> %b, <16 x half> %c) {
+; CHECK-SD-NOFP16-LABEL: fmuladd_v16f16:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    mov h6, v2.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvt s16, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt s17, h0
+; CHECK-SD-NOFP16-NEXT:    mov h18, v2.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h20, v2.h[3]
+; CHECK-SD-NOFP16-NEXT:    mov h21, v0.h[3]
+; CHECK-SD-NOFP16-NEXT:    mov h24, v3.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h25, v1.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h26, v1.h[2]
+; CHECK-SD-NOFP16-NEXT:    fcvt s27, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
+; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
+; CHECK-SD-NOFP16-NEXT:    mov h29, v1.h[4]
+; CHECK-SD-NOFP16-NEXT:    fmul s16, s17, s16
+; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
+; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
+; CHECK-SD-NOFP16-NEXT:    mov h17, v4.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
+; CHECK-SD-NOFP16-NEXT:    fcvt s21, h21
+; CHECK-SD-NOFP16-NEXT:    fcvt s24, h24
+; CHECK-SD-NOFP16-NEXT:    fcvt s25, h25
+; CHECK-SD-NOFP16-NEXT:    mov h30, v1.h[6]
+; CHECK-SD-NOFP16-NEXT:    fmul s6, s7, s6
+; CHECK-SD-NOFP16-NEXT:    fcvt s7, h4
+; CHECK-SD-NOFP16-NEXT:    fmul s23, s19, s18
+; CHECK-SD-NOFP16-NEXT:    fcvt h22, s16
+; CHECK-SD-NOFP16-NEXT:    mov h18, v2.h[4]
+; CHECK-SD-NOFP16-NEXT:    fmul s20, s21, s20
+; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[4]
+; CHECK-SD-NOFP16-NEXT:    mov h16, v4.h[2]
+; CHECK-SD-NOFP16-NEXT:    fmul s24, s25, s24
+; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
+; CHECK-SD-NOFP16-NEXT:    mov h25, v3.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
+; CHECK-SD-NOFP16-NEXT:    fcvt s21, h22
+; CHECK-SD-NOFP16-NEXT:    fcvt h22, s23
+; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
+; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
+; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
+; CHECK-SD-NOFP16-NEXT:    fcvt h20, s20
+; CHECK-SD-NOFP16-NEXT:    fcvt s23, h6
+; CHECK-SD-NOFP16-NEXT:    mov h6, v4.h[3]
+; CHECK-SD-NOFP16-NEXT:    fadd s7, s21, s7
+; CHECK-SD-NOFP16-NEXT:    fcvt s21, h22
+; CHECK-SD-NOFP16-NEXT:    fcvt s22, h3
+; CHECK-SD-NOFP16-NEXT:    fmul s18, s19, s18
+; CHECK-SD-NOFP16-NEXT:    fcvt s19, h20
+; CHECK-SD-NOFP16-NEXT:    fadd s17, s23, s17
+; CHECK-SD-NOFP16-NEXT:    mov h23, v3.h[2]
+; CHECK-SD-NOFP16-NEXT:    fcvt s28, h6
+; CHECK-SD-NOFP16-NEXT:    fmul s22, s27, s22
+; CHECK-SD-NOFP16-NEXT:    fcvt h6, s7
+; CHECK-SD-NOFP16-NEXT:    fadd s7, s21, s16
+; CHECK-SD-NOFP16-NEXT:    fcvt s21, h26
+; CHECK-SD-NOFP16-NEXT:    mov h26, v1.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt h18, s18
+; CHECK-SD-NOFP16-NEXT:    mov h27, v0.h[5]
+; CHECK-SD-NOFP16-NEXT:    fcvt h16, s17
+; CHECK-SD-NOFP16-NEXT:    mov h17, v4.h[4]
+; CHECK-SD-NOFP16-NEXT:    fcvt s20, h23
+; CHECK-SD-NOFP16-NEXT:    fadd s19, s19, s28
+; CHECK-SD-NOFP16-NEXT:    fcvt h7, s7
+; CHECK-SD-NOFP16-NEXT:    mov h23, v2.h[5]
+; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
+; CHECK-SD-NOFP16-NEXT:    fmul s20, s21, s20
+; CHECK-SD-NOFP16-NEXT:    mov v6.h[1], v16.h[0]
+; CHECK-SD-NOFP16-NEXT:    mov h16, v5.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
+; CHECK-SD-NOFP16-NEXT:    fcvt h21, s22
+; CHECK-SD-NOFP16-NEXT:    fcvt h22, s24
+; CHECK-SD-NOFP16-NEXT:    fcvt s24, h25
+; CHECK-SD-NOFP16-NEXT:    fcvt s25, h26
+; CHECK-SD-NOFP16-NEXT:    mov h26, v5.h[2]
+; CHECK-SD-NOFP16-NEXT:    fcvt h19, s19
+; CHECK-SD-NOFP16-NEXT:    mov v6.h[2], v7.h[0]
+; CHECK-SD-NOFP16-NEXT:    mov h7, v3.h[4]
+; CHECK-SD-NOFP16-NEXT:    fcvt s28, h16
+; CHECK-SD-NOFP16-NEXT:    fcvt h20, s20
+; CHECK-SD-NOFP16-NEXT:    fcvt s21, h21
+; CHECK-SD-NOFP16-NEXT:    fcvt s22, h22
+; CHECK-SD-NOFP16-NEXT:    fadd s16, s18, s17
+; CHECK-SD-NOFP16-NEXT:    fmul s18, s25, s24
+; CHECK-SD-NOFP16-NEXT:    fcvt s17, h5
+; CHECK-SD-NOFP16-NEXT:    mov h24, v0.h[6]
+; CHECK-SD-NOFP16-NEXT:    fcvt s25, h26
+; CHECK-SD-NOFP16-NEXT:    fcvt s26, h27
+; CHECK-SD-NOFP16-NEXT:    mov v6.h[3], v19.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt s19, h23
+; CHECK-SD-NOFP16-NEXT:    mov h23, v2.h[6]
+; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
+; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
+; CHECK-SD-NOFP16-NEXT:    fcvt s27, h29
+; CHECK-SD-NOFP16-NEXT:    fadd s22, s22, s28
+; CHECK-SD-NOFP16-NEXT:    fadd s17, s21, s17
+; CHECK-SD-NOFP16-NEXT:    fcvt h18, s18
+; CHECK-SD-NOFP16-NEXT:    mov h21, v5.h[3]
+; CHECK-SD-NOFP16-NEXT:    mov h28, v3.h[5]
+; CHECK-SD-NOFP16-NEXT:    mov h29, v1.h[5]
+; CHECK-SD-NOFP16-NEXT:    fmul s19, s26, s19
+; CHECK-SD-NOFP16-NEXT:    fcvt s23, h23
+; CHECK-SD-NOFP16-NEXT:    fcvt s24, h24
+; CHECK-SD-NOFP16-NEXT:    fadd s20, s20, s25
+; CHECK-SD-NOFP16-NEXT:    fmul s25, s27, s7
+; CHECK-SD-NOFP16-NEXT:    mov h27, v3.h[6]
+; CHECK-SD-NOFP16-NEXT:    fcvt h22, s22
+; CHECK-SD-NOFP16-NEXT:    fcvt h7, s17
+; CHECK-SD-NOFP16-NEXT:    fcvt s17, h18
+; CHECK-SD-NOFP16-NEXT:    fcvt s18, h21
+; CHECK-SD-NOFP16-NEXT:    fcvt s21, h28
+; CHECK-SD-NOFP16-NEXT:    fcvt s28, h29
+; CHECK-SD-NOFP16-NEXT:    fmul s23, s24, s23
+; CHECK-SD-NOFP16-NEXT:    mov h2, v2.h[7]
+; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
+; CHECK-SD-NOFP16-NEXT:    fcvt s24, h27
+; CHECK-SD-NOFP16-NEXT:    fcvt s26, h30
+; CHECK-SD-NOFP16-NEXT:    fcvt h20, s20
+; CHECK-SD-NOFP16-NEXT:    mov v7.h[1], v22.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt h22, s25
+; CHECK-SD-NOFP16-NEXT:    mov h25, v5.h[4]
+; CHECK-SD-NOFP16-NEXT:    fadd s17, s17, s18
+; CHECK-SD-NOFP16-NEXT:    fmul s18, s28, s21
+; CHECK-SD-NOFP16-NEXT:    mov h3, v3.h[7]
+; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
+; CHECK-SD-NOFP16-NEXT:    fcvt h19, s19
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fmul s21, s26, s24
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    mov h24, v4.h[5]
+; CHECK-SD-NOFP16-NEXT:    fcvt s22, h22
+; CHECK-SD-NOFP16-NEXT:    fcvt s25, h25
+; CHECK-SD-NOFP16-NEXT:    mov v7.h[2], v20.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt h17, s17
+; CHECK-SD-NOFP16-NEXT:    fcvt h18, s18
+; CHECK-SD-NOFP16-NEXT:    mov h20, v5.h[5]
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt h23, s23
+; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    mov h2, v4.h[6]
+; CHECK-SD-NOFP16-NEXT:    fcvt h21, s21
+; CHECK-SD-NOFP16-NEXT:    fadd s22, s22, s25
+; CHECK-SD-NOFP16-NEXT:    mov h25, v5.h[6]
+; CHECK-SD-NOFP16-NEXT:    fcvt h16, s16
+; CHECK-SD-NOFP16-NEXT:    mov v7.h[3], v17.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt s17, h19
+; CHECK-SD-NOFP16-NEXT:    fcvt s19, h24
+; CHECK-SD-NOFP16-NEXT:    fmul s1, s1, s3
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h18
+; CHECK-SD-NOFP16-NEXT:    fcvt s18, h20
+; CHECK-SD-NOFP16-NEXT:    fcvt s20, h23
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt s21, h21
+; CHECK-SD-NOFP16-NEXT:    fcvt h22, s22
+; CHECK-SD-NOFP16-NEXT:    fcvt s23, h25
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-SD-NOFP16-NEXT:    fadd s17, s17, s19
+; CHECK-SD-NOFP16-NEXT:    mov h4, v4.h[7]
+; CHECK-SD-NOFP16-NEXT:    mov h5, v5.h[7]
+; CHECK-SD-NOFP16-NEXT:    fadd s3, s3, s18
+; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
+; CHECK-SD-NOFP16-NEXT:    mov v6.h[4], v16.h[0]
+; CHECK-SD-NOFP16-NEXT:    fadd s2, s20, s2
+; CHECK-SD-NOFP16-NEXT:    mov v7.h[4], v22.h[0]
+; CHECK-SD-NOFP16-NEXT:    fadd s16, s21, s23
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fcvt h17, s17
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
+; CHECK-SD-NOFP16-NEXT:    mov v6.h[5], v17.h[0]
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s4
+; CHECK-SD-NOFP16-NEXT:    mov v7.h[5], v3.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt h3, s16
+; CHECK-SD-NOFP16-NEXT:    fadd s1, s1, s5
+; CHECK-SD-NOFP16-NEXT:    mov v6.h[6], v2.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-SD-NOFP16-NEXT:    mov v7.h[6], v3.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
+; CHECK-SD-NOFP16-NEXT:    mov v6.h[7], v0.h[0]
+; CHECK-SD-NOFP16-NEXT:    mov v7.h[7], v1.h[0]
+; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v6.16b
+; CHECK-SD-NOFP16-NEXT:    mov v1.16b, v7.16b
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fmuladd_v16f16:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fmla v4.8h, v2.8h, v0.8h
+; CHECK-SD-FP16-NEXT:    fmla v5.8h, v3.8h, v1.8h
+; CHECK-SD-FP16-NEXT:    mov v0.16b, v4.16b
+; CHECK-SD-FP16-NEXT:    mov v1.16b, v5.16b
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fmuladd_v16f16:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    fcvtl v6.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v7.4s, v2.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v16.4s, v1.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v17.4s, v3.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v3.4s, v3.8h
+; CHECK-GI-NOFP16-NEXT:    fmul v6.4s, v6.4s, v7.4s
+; CHECK-GI-NOFP16-NEXT:    fmul v7.4s, v16.4s, v17.4s
+; CHECK-GI-NOFP16-NEXT:    fmul v0.4s, v0.4s, v2.4s
+; CHECK-GI-NOFP16-NEXT:    fmul v1.4s, v1.4s, v3.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtn v2.4h, v6.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtl v6.4s, v4.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v4.4s, v4.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtn v3.4h, v7.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtl v7.4s, v5.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v5.4s, v5.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v1.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtl v2.4s, v2.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v3.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-GI-NOFP16-NEXT:    fadd v2.4s, v2.4s, v6.4s
+; CHECK-GI-NOFP16-NEXT:    fadd v3.4s, v3.4s, v7.4s
+; CHECK-GI-NOFP16-NEXT:    fadd v4.4s, v0.4s, v4.4s
+; CHECK-GI-NOFP16-NEXT:    fadd v5.4s, v1.4s, v5.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v3.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v4.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtn2 v1.8h, v5.4s
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fmuladd_v16f16:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    fmla v4.8h, v2.8h, v0.8h
+; CHECK-GI-FP16-NEXT:    fmla v5.8h, v3.8h, v1.8h
+; CHECK-GI-FP16-NEXT:    mov v0.16b, v4.16b
+; CHECK-GI-FP16-NEXT:    mov v1.16b, v5.16b
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %d = call <16 x half> @llvm.fmuladd.v16f16(<16 x half> %a, <16 x half> %b, <16 x half> %c)
+  ret <16 x half> %d
+}
+
+define double @fmul_f64(double %a, double %b, double %c) {
+; CHECK-LABEL: fmul_f64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmadd d0, d0, d1, d2
+; CHECK-NEXT:    ret
+entry:
+  %d = fmul fast double %a, %b
+  %e = fadd fast double %d, %c
+  ret double %e
+}
+
+define float @fmul_f32(float %a, float %b, float %c) {
+; CHECK-LABEL: fmul_f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmadd s0, s0, s1, s2
+; CHECK-NEXT:    ret
+entry:
+  %d = fmul fast float %a, %b
+  %e = fadd fast float %d, %c
+  ret float %e
+}
+
+define half @fmul_f16(half %a, half %b, half %c) {
+; CHECK-SD-NOFP16-LABEL: fmul_f16:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s1
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s1
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fmul_f16:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fmadd h0, h0, h1, h2
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fmul_f16:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-GI-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-GI-NOFP16-NEXT:    fmul s0, s0, s1
+; CHECK-GI-NOFP16-NEXT:    fcvt s1, h2
+; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-GI-NOFP16-NEXT:    fadd s0, s0, s1
+; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fmul_f16:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    fmadd h0, h0, h1, h2
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %d = fmul fast half %a, %b
+  %e = fadd fast half %d, %c
+  ret half %e
+}
+
+define <2 x double> @fmul_v2f64(<2 x double> %a, <2 x double> %b, <2 x double> %c) {
+; CHECK-SD-LABEL: fmul_v2f64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmla v2.2d, v1.2d, v0.2d
+; CHECK-SD-NEXT:    mov v0.16b, v2.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fmul_v2f64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmla v2.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT:    mov v0.16b, v2.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %d = fmul fast <2 x double> %a, %b
+  %e = fadd fast <2 x double> %d, %c
+  ret <2 x double> %e
+}
+
+define <3 x double> @fmul_v3f64(<3 x double> %a, <3 x double> %b, <3 x double> %c) {
+; CHECK-SD-LABEL: fmul_v3f64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d6 killed $d6 def $q6
+; CHECK-SD-NEXT:    // kill: def $d3 killed $d3 def $q3
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    // kill: def $d7 killed $d7 def $q7
+; CHECK-SD-NEXT:    // kill: def $d4 killed $d4 def $q4
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-SD-NEXT:    // kill: def $d5 killed $d5 def $q5
+; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-SD-NEXT:    mov v3.d[1], v4.d[0]
+; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-SD-NEXT:    mov v6.d[1], v7.d[0]
+; CHECK-SD-NEXT:    fmla v6.2d, v3.2d, v0.2d
+; CHECK-SD-NEXT:    ldr d3, [sp]
+; CHECK-SD-NEXT:    fmla v3.2d, v5.2d, v2.2d
+; CHECK-SD-NEXT:    fmov d0, d6
+; CHECK-SD-NEXT:    ext v1.16b, v6.16b, v6.16b, #8
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-SD-NEXT:    fmov d2, d3
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fmul_v3f64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    // kill: def $d3 killed $d3 def $q3
+; CHECK-GI-NEXT:    // kill: def $d6 killed $d6 def $q6
+; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-GI-NEXT:    // kill: def $d4 killed $d4 def $q4
+; CHECK-GI-NEXT:    // kill: def $d7 killed $d7 def $q7
+; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-GI-NEXT:    mov v3.d[1], v4.d[0]
+; CHECK-GI-NEXT:    mov v6.d[1], v7.d[0]
+; CHECK-GI-NEXT:    fmla v6.2d, v0.2d, v3.2d
+; CHECK-GI-NEXT:    ldr d0, [sp]
+; CHECK-GI-NEXT:    fmadd d2, d2, d5, d0
+; CHECK-GI-NEXT:    mov d1, v6.d[1]
+; CHECK-GI-NEXT:    fmov d0, d6
+; CHECK-GI-NEXT:    ret
+entry:
+  %d = fmul fast <3 x double> %a, %b
+  %e = fadd fast <3 x double> %d, %c
+  ret <3 x double> %e
+}
+
+define <4 x double> @fmul_v4f64(<4 x double> %a, <4 x double> %b, <4 x double> %c) {
+; CHECK-SD-LABEL: fmul_v4f64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmla v4.2d, v2.2d, v0.2d
+; CHECK-SD-NEXT:    fmla v5.2d, v3.2d, v1.2d
+; CHECK-SD-NEXT:    mov v0.16b, v4.16b
+; CHECK-SD-NEXT:    mov v1.16b, v5.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fmul_v4f64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmla v4.2d, v0.2d, v2.2d
+; CHECK-GI-NEXT:    fmla v5.2d, v1.2d, v3.2d
+; CHECK-GI-NEXT:    mov v0.16b, v4.16b
+; CHECK-GI-NEXT:    mov v1.16b, v5.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %d = fmul fast <4 x double> %a, %b
+  %e = fadd fast <4 x double> %d, %c
+  ret <4 x double> %e
+}
+
+define <2 x float> @fmul_v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
+; CHECK-SD-LABEL: fmul_v2f32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmla v2.2s, v1.2s, v0.2s
+; CHECK-SD-NEXT:    fmov d0, d2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fmul_v2f32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmla v2.2s, v0.2s, v1.2s
+; CHECK-GI-NEXT:    fmov d0, d2
+; CHECK-GI-NEXT:    ret
+entry:
+  %d = fmul fast <2 x float> %a, %b
+  %e = fadd fast <2 x float> %d, %c
+  ret <2 x float> %e
+}
+
+define <3 x float> @fmul_v3f32(<3 x float> %a, <3 x float> %b, <3 x float> %c) {
+; CHECK-SD-LABEL: fmul_v3f32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmla v2.4s, v1.4s, v0.4s
+; CHECK-SD-NEXT:    mov v0.16b, v2.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fmul_v3f32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmla v2.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    mov v0.16b, v2.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %d = fmul fast <3 x float> %a, %b
+  %e = fadd fast <3 x float> %d, %c
+  ret <3 x float> %e
+}
+
+define <4 x float> @fmul_v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
+; CHECK-SD-LABEL: fmul_v4f32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmla v2.4s, v1.4s, v0.4s
+; CHECK-SD-NEXT:    mov v0.16b, v2.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fmul_v4f32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmla v2.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    mov v0.16b, v2.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %d = fmul fast <4 x float> %a, %b
+  %e = fadd fast <4 x float> %d, %c
+  ret <4 x float> %e
+}
+
+define <8 x float> @fmul_v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c) {
+; CHECK-SD-LABEL: fmul_v8f32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmla v4.4s, v2.4s, v0.4s
+; CHECK-SD-NEXT:    fmla v5.4s, v3.4s, v1.4s
+; CHECK-SD-NEXT:    mov v0.16b, v4.16b
+; CHECK-SD-NEXT:    mov v1.16b, v5.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fmul_v8f32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmla v4.4s, v0.4s, v2.4s
+; CHECK-GI-NEXT:    fmla v5.4s, v1.4s, v3.4s
+; CHECK-GI-NEXT:    mov v0.16b, v4.16b
+; CHECK-GI-NEXT:    mov v1.16b, v5.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %d = fmul fast <8 x float> %a, %b
+  %e = fadd fast <8 x float> %d, %c
+  ret <8 x float> %e
+}
+
+define <7 x half> @fmul_v7f16(<7 x half> %a, <7 x half> %b, <7 x half> %c) {
+; CHECK-SD-NOFP16-LABEL: fmul_v7f16:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s6, h0
+; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[3]
+; CHECK-SD-NOFP16-NEXT:    mov h18, v0.h[3]
+; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[4]
+; CHECK-SD-NOFP16-NEXT:    mov h20, v0.h[5]
+; CHECK-SD-NOFP16-NEXT:    fcvt s21, h2
+; CHECK-SD-NOFP16-NEXT:    mov h22, v2.h[2]
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fmul s5, s6, s5
+; CHECK-SD-NOFP16-NEXT:    fcvt s6, h16
+; CHECK-SD-NOFP16-NEXT:    fcvt s16, h18
+; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[4]
+; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
+; CHECK-SD-NOFP16-NEXT:    fmul s3, s4, s3
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h7
+; CHECK-SD-NOFP16-NEXT:    fcvt s7, h17
+; CHECK-SD-NOFP16-NEXT:    mov h17, v2.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
+; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
+; CHECK-SD-NOFP16-NEXT:    fmul s4, s6, s4
+; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[5]
+; CHECK-SD-NOFP16-NEXT:    fmul s7, s16, s7
+; CHECK-SD-NOFP16-NEXT:    fcvt s16, h17
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    fcvt s17, h18
+; CHECK-SD-NOFP16-NEXT:    fcvt s18, h19
+; CHECK-SD-NOFP16-NEXT:    mov h19, v2.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
+; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
+; CHECK-SD-NOFP16-NEXT:    fcvt h7, s7
+; CHECK-SD-NOFP16-NEXT:    fadd s5, s5, s21
+; CHECK-SD-NOFP16-NEXT:    mov h21, v0.h[6]
+; CHECK-SD-NOFP16-NEXT:    fmul s17, s18, s17
+; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[6]
+; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
+; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
+; CHECK-SD-NOFP16-NEXT:    fadd s3, s3, s16
+; CHECK-SD-NOFP16-NEXT:    fcvt s16, h22
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fmul s6, s20, s6
+; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
+; CHECK-SD-NOFP16-NEXT:    mov h20, v1.h[7]
+; CHECK-SD-NOFP16-NEXT:    fcvt h1, s5
+; CHECK-SD-NOFP16-NEXT:    mov h5, v2.h[4]
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
+; CHECK-SD-NOFP16-NEXT:    fadd s4, s4, s16
+; CHECK-SD-NOFP16-NEXT:    fcvt h16, s17
+; CHECK-SD-NOFP16-NEXT:    fcvt s17, h18
+; CHECK-SD-NOFP16-NEXT:    fcvt s18, h21
+; CHECK-SD-NOFP16-NEXT:    fadd s7, s7, s19
+; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v3.h[0]
+; CHECK-SD-NOFP16-NEXT:    mov h3, v2.h[5]
+; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
+; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
+; CHECK-SD-NOFP16-NEXT:    fmul s17, s18, s17
+; CHECK-SD-NOFP16-NEXT:    fcvt s18, h20
+; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
+; CHECK-SD-NOFP16-NEXT:    fcvt h7, s7
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fadd s5, s16, s5
+; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v4.h[0]
+; CHECK-SD-NOFP16-NEXT:    mov h4, v2.h[6]
+; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s18
+; CHECK-SD-NOFP16-NEXT:    fcvt h16, s17
+; CHECK-SD-NOFP16-NEXT:    mov h2, v2.h[7]
+; CHECK-SD-NOFP16-NEXT:    fadd s3, s6, s3
+; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
+; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], v7.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-SD-NOFP16-NEXT:    fcvt s6, h16
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
+; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], v5.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fadd s4, s6, s4
+; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], v3.h[0]
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    fcvt h3, s4
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], v3.h[0]
+; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], v0.h[0]
+; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fmul_v7f16:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fmla v2.8h, v1.8h, v0.8h
+; CHECK-SD-FP16-NEXT:    mov v0.16b, v2.16b
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fmul_v7f16:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    mov h3, v0.h[4]
+; CHECK-GI-NOFP16-NEXT:    mov h4, v0.h[5]
+; CHECK-GI-NOFP16-NEXT:    mov h5, v1.h[4]
+; CHECK-GI-NOFP16-NEXT:    mov h6, v1.h[5]
+; CHECK-GI-NOFP16-NEXT:    fcvtl v7.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v16.4s, v1.4h
+; CHECK-GI-NOFP16-NEXT:    mov h0, v0.h[6]
+; CHECK-GI-NOFP16-NEXT:    mov h1, v1.h[6]
+; CHECK-GI-NOFP16-NEXT:    mov v3.h[1], v4.h[0]
+; CHECK-GI-NOFP16-NEXT:    mov v5.h[1], v6.h[0]
+; CHECK-GI-NOFP16-NEXT:    fmul v4.4s, v7.4s, v16.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtl v6.4s, v2.4h
+; CHECK-GI-NOFP16-NEXT:    mov v3.h[2], v0.h[0]
+; CHECK-GI-NOFP16-NEXT:    mov v5.h[2], v1.h[0]
+; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v4.4s
+; CHECK-GI-NOFP16-NEXT:    mov h1, v2.h[4]
+; CHECK-GI-NOFP16-NEXT:    mov h4, v2.h[5]
+; CHECK-GI-NOFP16-NEXT:    mov h2, v2.h[6]
+; CHECK-GI-NOFP16-NEXT:    mov v3.h[3], v0.h[0]
+; CHECK-GI-NOFP16-NEXT:    mov v5.h[3], v0.h[0]
+; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    mov v1.h[1], v4.h[0]
+; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v3.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v4.4s, v5.4h
+; CHECK-GI-NOFP16-NEXT:    fadd v0.4s, v0.4s, v6.4s
+; CHECK-GI-NOFP16-NEXT:    mov v1.h[2], v2.h[0]
+; CHECK-GI-NOFP16-NEXT:    fmul v2.4s, v3.4s, v4.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-GI-NOFP16-NEXT:    mov v1.h[3], v0.h[0]
+; CHECK-GI-NOFP16-NEXT:    fcvtn v2.4h, v2.4s
+; CHECK-GI-NOFP16-NEXT:    mov h3, v0.h[1]
+; CHECK-GI-NOFP16-NEXT:    mov h4, v0.h[2]
+; CHECK-GI-NOFP16-NEXT:    mov h5, v0.h[3]
+; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v2.4s, v2.4h
+; CHECK-GI-NOFP16-NEXT:    mov v0.h[1], v3.h[0]
+; CHECK-GI-NOFP16-NEXT:    fadd v1.4s, v2.4s, v1.4s
+; CHECK-GI-NOFP16-NEXT:    mov v0.h[2], v4.h[0]
+; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v1.4s
+; CHECK-GI-NOFP16-NEXT:    mov v0.h[3], v5.h[0]
+; CHECK-GI-NOFP16-NEXT:    mov h2, v1.h[1]
+; CHECK-GI-NOFP16-NEXT:    mov v0.h[4], v1.h[0]
+; CHECK-GI-NOFP16-NEXT:    mov h1, v1.h[2]
+; CHECK-GI-NOFP16-NEXT:    mov v0.h[5], v2.h[0]
+; CHECK-GI-NOFP16-NEXT:    mov v0.h[6], v1.h[0]
+; CHECK-GI-NOFP16-NEXT:    mov v0.h[7], v0.h[0]
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fmul_v7f16:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    fmla v2.8h, v0.8h, v1.8h
+; CHECK-GI-FP16-NEXT:    mov v0.16b, v2.16b
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %d = fmul fast <7 x half> %a, %b
+  %e = fadd fast <7 x half> %d, %c
+  ret <7 x half> %e
+}
+
+define <4 x half> @fmul_v4f16(<4 x half> %a, <4 x half> %b, <4 x half> %c) {
+; CHECK-SD-NOFP16-LABEL: fmul_v4f16:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fmul v0.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v2.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fadd v0.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fmul_v4f16:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fmla v2.4h, v1.4h, v0.4h
+; CHECK-SD-FP16-NEXT:    fmov d0, d2
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fmul_v4f16:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-GI-NOFP16-NEXT:    fmul v0.4s, v0.4s, v1.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v2.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fadd v0.4s, v0.4s, v1.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fmul_v4f16:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    fmla v2.4h, v0.4h, v1.4h
+; CHECK-GI-FP16-NEXT:    fmov d0, d2
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %d = fmul fast <4 x half> %a, %b
+  %e = fadd fast <4 x half> %d, %c
+  ret <4 x half> %e
+}
+
+define <8 x half> @fmul_v8f16(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
+; CHECK-SD-NOFP16-LABEL: fmul_v8f16:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s6, h0
+; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h17, v1.h[3]
+; CHECK-SD-NOFP16-NEXT:    mov h18, v0.h[3]
+; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[4]
+; CHECK-SD-NOFP16-NEXT:    mov h20, v0.h[5]
+; CHECK-SD-NOFP16-NEXT:    fcvt s21, h2
+; CHECK-SD-NOFP16-NEXT:    mov h22, v2.h[2]
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fmul s5, s6, s5
+; CHECK-SD-NOFP16-NEXT:    fcvt s6, h16
+; CHECK-SD-NOFP16-NEXT:    fcvt s16, h18
+; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[4]
+; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
+; CHECK-SD-NOFP16-NEXT:    fmul s3, s4, s3
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h7
+; CHECK-SD-NOFP16-NEXT:    fcvt s7, h17
+; CHECK-SD-NOFP16-NEXT:    mov h17, v2.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
+; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
+; CHECK-SD-NOFP16-NEXT:    fmul s4, s6, s4
+; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[5]
+; CHECK-SD-NOFP16-NEXT:    fmul s7, s16, s7
+; CHECK-SD-NOFP16-NEXT:    fcvt s16, h17
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    fcvt s17, h18
+; CHECK-SD-NOFP16-NEXT:    fcvt s18, h19
+; CHECK-SD-NOFP16-NEXT:    mov h19, v2.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
+; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
+; CHECK-SD-NOFP16-NEXT:    fcvt h7, s7
+; CHECK-SD-NOFP16-NEXT:    fadd s5, s5, s21
+; CHECK-SD-NOFP16-NEXT:    mov h21, v0.h[6]
+; CHECK-SD-NOFP16-NEXT:    fmul s17, s18, s17
+; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[6]
+; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
+; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
+; CHECK-SD-NOFP16-NEXT:    fadd s3, s3, s16
+; CHECK-SD-NOFP16-NEXT:    fcvt s16, h22
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fmul s6, s20, s6
+; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
+; CHECK-SD-NOFP16-NEXT:    mov h20, v1.h[7]
+; CHECK-SD-NOFP16-NEXT:    fcvt h1, s5
+; CHECK-SD-NOFP16-NEXT:    mov h5, v2.h[4]
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
+; CHECK-SD-NOFP16-NEXT:    fadd s4, s4, s16
+; CHECK-SD-NOFP16-NEXT:    fcvt h16, s17
+; CHECK-SD-NOFP16-NEXT:    fcvt s17, h18
+; CHECK-SD-NOFP16-NEXT:    fcvt s18, h21
+; CHECK-SD-NOFP16-NEXT:    fadd s7, s7, s19
+; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v3.h[0]
+; CHECK-SD-NOFP16-NEXT:    mov h3, v2.h[5]
+; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
+; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
+; CHECK-SD-NOFP16-NEXT:    fmul s17, s18, s17
+; CHECK-SD-NOFP16-NEXT:    fcvt s18, h20
+; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
+; CHECK-SD-NOFP16-NEXT:    fcvt h7, s7
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fadd s5, s16, s5
+; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v4.h[0]
+; CHECK-SD-NOFP16-NEXT:    mov h4, v2.h[6]
+; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s18
+; CHECK-SD-NOFP16-NEXT:    fcvt h16, s17
+; CHECK-SD-NOFP16-NEXT:    mov h2, v2.h[7]
+; CHECK-SD-NOFP16-NEXT:    fadd s3, s6, s3
+; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
+; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], v7.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-SD-NOFP16-NEXT:    fcvt s6, h16
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
+; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], v5.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fadd s4, s6, s4
+; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], v3.h[0]
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s2
+; CHECK-SD-NOFP16-NEXT:    fcvt h3, s4
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], v3.h[0]
+; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], v0.h[0]
+; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fmul_v8f16:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fmla v2.8h, v1.8h, v0.8h
+; CHECK-SD-FP16-NEXT:    mov v0.16b, v2.16b
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fmul_v8f16:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v4.4s, v1.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-GI-NOFP16-NEXT:    fmul v3.4s, v3.4s, v4.4s
+; CHECK-GI-NOFP16-NEXT:    fmul v0.4s, v0.4s, v1.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v3.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v2.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fadd v1.4s, v1.4s, v3.4s
+; CHECK-GI-NOFP16-NEXT:    fadd v2.4s, v0.4s, v2.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fmul_v8f16:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    fmla v2.8h, v0.8h, v1.8h
+; CHECK-GI-FP16-NEXT:    mov v0.16b, v2.16b
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %d = fmul fast <8 x half> %a, %b
+  %e = fadd fast <8 x half> %d, %c
+  ret <8 x half> %e
+}
+
+define <16 x half> @fmul_v16f16(<16 x half> %a, <16 x half> %b, <16 x half> %c) {
+; CHECK-SD-NOFP16-LABEL: fmul_v16f16:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    stp d11, d10, [sp, #-32]! // 16-byte Folded Spill
+; CHECK-SD-NOFP16-NEXT:    stp d9, d8, [sp, #16] // 16-byte Folded Spill
+; CHECK-SD-NOFP16-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-SD-NOFP16-NEXT:    .cfi_offset b8, -8
+; CHECK-SD-NOFP16-NEXT:    .cfi_offset b9, -16
+; CHECK-SD-NOFP16-NEXT:    .cfi_offset b10, -24
+; CHECK-SD-NOFP16-NEXT:    .cfi_offset b11, -32
+; CHECK-SD-NOFP16-NEXT:    mov h6, v3.h[7]
+; CHECK-SD-NOFP16-NEXT:    mov h16, v1.h[7]
+; CHECK-SD-NOFP16-NEXT:    mov h20, v3.h[6]
+; CHECK-SD-NOFP16-NEXT:    mov h22, v1.h[6]
+; CHECK-SD-NOFP16-NEXT:    mov h23, v3.h[5]
+; CHECK-SD-NOFP16-NEXT:    mov h24, v3.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h26, v1.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h17, v3.h[3]
+; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[3]
+; CHECK-SD-NOFP16-NEXT:    mov h27, v3.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h28, v1.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[5]
+; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
+; CHECK-SD-NOFP16-NEXT:    fcvt s25, h16
+; CHECK-SD-NOFP16-NEXT:    fcvt s21, h20
+; CHECK-SD-NOFP16-NEXT:    fcvt s22, h22
+; CHECK-SD-NOFP16-NEXT:    fcvt s20, h23
+; CHECK-SD-NOFP16-NEXT:    fcvt s23, h24
+; CHECK-SD-NOFP16-NEXT:    fcvt s24, h26
+; CHECK-SD-NOFP16-NEXT:    mov h26, v0.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h19, v3.h[4]
+; CHECK-SD-NOFP16-NEXT:    mov h16, v1.h[4]
+; CHECK-SD-NOFP16-NEXT:    fcvt s29, h3
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fmul s6, s25, s6
+; CHECK-SD-NOFP16-NEXT:    mov h25, v2.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvt s27, h27
+; CHECK-SD-NOFP16-NEXT:    fmul s3, s22, s21
+; CHECK-SD-NOFP16-NEXT:    fcvt s28, h28
+; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
+; CHECK-SD-NOFP16-NEXT:    fmul s23, s24, s23
+; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
+; CHECK-SD-NOFP16-NEXT:    fcvt s21, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt s22, h0
+; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
+; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
+; CHECK-SD-NOFP16-NEXT:    fcvt s24, h25
+; CHECK-SD-NOFP16-NEXT:    fcvt s25, h26
+; CHECK-SD-NOFP16-NEXT:    fmul s26, s1, s29
+; CHECK-SD-NOFP16-NEXT:    fmul s27, s28, s27
+; CHECK-SD-NOFP16-NEXT:    mov h28, v2.h[7]
+; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
+; CHECK-SD-NOFP16-NEXT:    fmul s1, s18, s17
+; CHECK-SD-NOFP16-NEXT:    fcvt h17, s23
+; CHECK-SD-NOFP16-NEXT:    mov h29, v4.h[1]
+; CHECK-SD-NOFP16-NEXT:    fmul s21, s22, s21
+; CHECK-SD-NOFP16-NEXT:    fmul s16, s16, s19
+; CHECK-SD-NOFP16-NEXT:    mov h8, v2.h[6]
+; CHECK-SD-NOFP16-NEXT:    fmul s23, s25, s24
+; CHECK-SD-NOFP16-NEXT:    mov h24, v2.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h25, v0.h[2]
+; CHECK-SD-NOFP16-NEXT:    fmul s7, s7, s20
+; CHECK-SD-NOFP16-NEXT:    fcvt h18, s26
+; CHECK-SD-NOFP16-NEXT:    fcvt h19, s27
+; CHECK-SD-NOFP16-NEXT:    fcvt s22, h28
+; CHECK-SD-NOFP16-NEXT:    mov h26, v2.h[3]
+; CHECK-SD-NOFP16-NEXT:    mov h27, v0.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt h21, s21
+; CHECK-SD-NOFP16-NEXT:    mov h20, v0.h[7]
+; CHECK-SD-NOFP16-NEXT:    mov h11, v2.h[4]
+; CHECK-SD-NOFP16-NEXT:    fcvt h28, s23
+; CHECK-SD-NOFP16-NEXT:    fcvt s30, h24
+; CHECK-SD-NOFP16-NEXT:    fcvt s31, h25
+; CHECK-SD-NOFP16-NEXT:    mov h24, v0.h[4]
+; CHECK-SD-NOFP16-NEXT:    fcvt s29, h29
+; CHECK-SD-NOFP16-NEXT:    mov h9, v0.h[6]
+; CHECK-SD-NOFP16-NEXT:    fcvt s25, h26
+; CHECK-SD-NOFP16-NEXT:    fcvt s26, h27
+; CHECK-SD-NOFP16-NEXT:    mov h10, v2.h[5]
+; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
+; CHECK-SD-NOFP16-NEXT:    mov h23, v0.h[5]
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h11
+; CHECK-SD-NOFP16-NEXT:    fmul s27, s31, s30
+; CHECK-SD-NOFP16-NEXT:    fcvt s28, h28
+; CHECK-SD-NOFP16-NEXT:    fcvt s30, h21
+; CHECK-SD-NOFP16-NEXT:    fcvt s31, h4
+; CHECK-SD-NOFP16-NEXT:    fcvt s21, h8
+; CHECK-SD-NOFP16-NEXT:    mov h8, v5.h[1]
+; CHECK-SD-NOFP16-NEXT:    fmul s25, s26, s25
+; CHECK-SD-NOFP16-NEXT:    fcvt s24, h24
+; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
+; CHECK-SD-NOFP16-NEXT:    fmul s2, s20, s22
+; CHECK-SD-NOFP16-NEXT:    fcvt s22, h9
+; CHECK-SD-NOFP16-NEXT:    fcvt s20, h10
+; CHECK-SD-NOFP16-NEXT:    fadd s26, s28, s29
+; CHECK-SD-NOFP16-NEXT:    fcvt s23, h23
+; CHECK-SD-NOFP16-NEXT:    fcvt h27, s27
+; CHECK-SD-NOFP16-NEXT:    fadd s28, s30, s31
+; CHECK-SD-NOFP16-NEXT:    mov h29, v4.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h30, v5.h[2]
+; CHECK-SD-NOFP16-NEXT:    fmul s24, s24, s0
+; CHECK-SD-NOFP16-NEXT:    fcvt s31, h8
+; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
+; CHECK-SD-NOFP16-NEXT:    fmul s21, s22, s21
+; CHECK-SD-NOFP16-NEXT:    fcvt s8, h5
+; CHECK-SD-NOFP16-NEXT:    fcvt h25, s25
+; CHECK-SD-NOFP16-NEXT:    fmul s20, s23, s20
+; CHECK-SD-NOFP16-NEXT:    fcvt h26, s26
+; CHECK-SD-NOFP16-NEXT:    fcvt s27, h27
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s28
+; CHECK-SD-NOFP16-NEXT:    mov h28, v4.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt s29, h29
+; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
+; CHECK-SD-NOFP16-NEXT:    fcvt s30, h30
+; CHECK-SD-NOFP16-NEXT:    fadd s19, s19, s31
+; CHECK-SD-NOFP16-NEXT:    fadd s18, s18, s8
+; CHECK-SD-NOFP16-NEXT:    ldp d9, d8, [sp, #16] // 16-byte Folded Reload
+; CHECK-SD-NOFP16-NEXT:    fcvt h22, s1
+; CHECK-SD-NOFP16-NEXT:    mov h23, v5.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt s25, h25
+; CHECK-SD-NOFP16-NEXT:    mov v0.h[1], v26.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt s26, h28
+; CHECK-SD-NOFP16-NEXT:    fadd s27, s27, s29
+; CHECK-SD-NOFP16-NEXT:    fcvt h24, s24
+; CHECK-SD-NOFP16-NEXT:    fadd s17, s17, s30
+; CHECK-SD-NOFP16-NEXT:    mov h28, v4.h[4]
+; CHECK-SD-NOFP16-NEXT:    fcvt h19, s19
+; CHECK-SD-NOFP16-NEXT:    fcvt h1, s18
+; CHECK-SD-NOFP16-NEXT:    fcvt h16, s16
+; CHECK-SD-NOFP16-NEXT:    fcvt s18, h22
+; CHECK-SD-NOFP16-NEXT:    fcvt s22, h23
+; CHECK-SD-NOFP16-NEXT:    fcvt h20, s20
+; CHECK-SD-NOFP16-NEXT:    fadd s23, s25, s26
+; CHECK-SD-NOFP16-NEXT:    mov h25, v5.h[4]
+; CHECK-SD-NOFP16-NEXT:    fcvt h26, s27
+; CHECK-SD-NOFP16-NEXT:    fcvt s24, h24
+; CHECK-SD-NOFP16-NEXT:    fcvt s27, h28
+; CHECK-SD-NOFP16-NEXT:    fcvt h7, s7
+; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v19.h[0]
+; CHECK-SD-NOFP16-NEXT:    mov h19, v4.h[5]
+; CHECK-SD-NOFP16-NEXT:    fcvt h17, s17
+; CHECK-SD-NOFP16-NEXT:    fadd s18, s18, s22
+; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
+; CHECK-SD-NOFP16-NEXT:    fcvt h21, s21
+; CHECK-SD-NOFP16-NEXT:    fcvt s22, h25
+; CHECK-SD-NOFP16-NEXT:    mov h25, v5.h[5]
+; CHECK-SD-NOFP16-NEXT:    mov v0.h[2], v26.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt h23, s23
+; CHECK-SD-NOFP16-NEXT:    fadd s24, s24, s27
+; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
+; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
+; CHECK-SD-NOFP16-NEXT:    mov h26, v4.h[6]
+; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v17.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt h17, s18
+; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
+; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
+; CHECK-SD-NOFP16-NEXT:    fadd s16, s16, s22
+; CHECK-SD-NOFP16-NEXT:    fcvt s18, h25
+; CHECK-SD-NOFP16-NEXT:    mov h22, v5.h[6]
+; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
+; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
+; CHECK-SD-NOFP16-NEXT:    mov v0.h[3], v23.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt h23, s24
+; CHECK-SD-NOFP16-NEXT:    fadd s19, s20, s19
+; CHECK-SD-NOFP16-NEXT:    fcvt s20, h21
+; CHECK-SD-NOFP16-NEXT:    fcvt s21, h26
+; CHECK-SD-NOFP16-NEXT:    mov h4, v4.h[7]
+; CHECK-SD-NOFP16-NEXT:    mov v1.h[3], v17.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt h16, s16
+; CHECK-SD-NOFP16-NEXT:    fadd s7, s7, s18
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fcvt s17, h22
+; CHECK-SD-NOFP16-NEXT:    mov h5, v5.h[7]
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    mov v0.h[4], v23.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt h18, s19
+; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
+; CHECK-SD-NOFP16-NEXT:    fadd s19, s20, s21
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    mov v1.h[4], v16.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt h7, s7
+; CHECK-SD-NOFP16-NEXT:    fadd s3, s3, s17
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    mov v0.h[5], v18.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt h16, s19
+; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s4
+; CHECK-SD-NOFP16-NEXT:    mov v1.h[5], v7.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
+; CHECK-SD-NOFP16-NEXT:    fadd s4, s6, s5
+; CHECK-SD-NOFP16-NEXT:    mov v0.h[6], v16.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
+; CHECK-SD-NOFP16-NEXT:    mov v1.h[6], v3.h[0]
+; CHECK-SD-NOFP16-NEXT:    fcvt h3, s4
+; CHECK-SD-NOFP16-NEXT:    mov v0.h[7], v2.h[0]
+; CHECK-SD-NOFP16-NEXT:    mov v1.h[7], v3.h[0]
+; CHECK-SD-NOFP16-NEXT:    ldp d11, d10, [sp], #32 // 16-byte Folded Reload
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fmul_v16f16:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fmla v4.8h, v2.8h, v0.8h
+; CHECK-SD-FP16-NEXT:    fmla v5.8h, v3.8h, v1.8h
+; CHECK-SD-FP16-NEXT:    mov v0.16b, v4.16b
+; CHECK-SD-FP16-NEXT:    mov v1.16b, v5.16b
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fmul_v16f16:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    fcvtl v6.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v7.4s, v2.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v16.4s, v1.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v17.4s, v3.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v3.4s, v3.8h
+; CHECK-GI-NOFP16-NEXT:    fmul v6.4s, v6.4s, v7.4s
+; CHECK-GI-NOFP16-NEXT:    fmul v7.4s, v16.4s, v17.4s
+; CHECK-GI-NOFP16-NEXT:    fmul v0.4s, v0.4s, v2.4s
+; CHECK-GI-NOFP16-NEXT:    fmul v1.4s, v1.4s, v3.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtn v2.4h, v6.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtl v6.4s, v4.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v4.4s, v4.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtn v3.4h, v7.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtl v7.4s, v5.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v5.4s, v5.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v1.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtl v2.4s, v2.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v3.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-GI-NOFP16-NEXT:    fadd v2.4s, v2.4s, v6.4s
+; CHECK-GI-NOFP16-NEXT:    fadd v3.4s, v3.4s, v7.4s
+; CHECK-GI-NOFP16-NEXT:    fadd v4.4s, v0.4s, v4.4s
+; CHECK-GI-NOFP16-NEXT:    fadd v5.4s, v1.4s, v5.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v3.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v4.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtn2 v1.8h, v5.4s
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fmul_v16f16:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    fmla v4.8h, v0.8h, v2.8h
+; CHECK-GI-FP16-NEXT:    fmla v5.8h, v1.8h, v3.8h
+; CHECK-GI-FP16-NEXT:    mov v0.16b, v4.16b
+; CHECK-GI-FP16-NEXT:    mov v1.16b, v5.16b
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %d = fmul fast <16 x half> %a, %b
+  %e = fadd fast <16 x half> %d, %c
+  ret <16 x half> %e
+}
+
+declare <16 x half> @llvm.fma.v16f16(<16 x half>, <16 x half>, <16 x half>)
+declare <16 x half> @llvm.fmuladd.v16f16(<16 x half>, <16 x half>, <16 x half>)
+declare <2 x double> @llvm.fma.v2f64(<2 x double>, <2 x double>, <2 x double>)
+declare <2 x double> @llvm.fmuladd.v2f64(<2 x double>, <2 x double>, <2 x double>)
+declare <2 x float> @llvm.fma.v2f32(<2 x float>, <2 x float>, <2 x float>)
+declare <2 x float> @llvm.fmuladd.v2f32(<2 x float>, <2 x float>, <2 x float>)
+declare <3 x double> @llvm.fma.v3f64(<3 x double>, <3 x double>, <3 x double>)
+declare <3 x double> @llvm.fmuladd.v3f64(<3 x double>, <3 x double>, <3 x double>)
+declare <3 x float> @llvm.fma.v3f32(<3 x float>, <3 x float>, <3 x float>)
+declare <3 x float> @llvm.fmuladd.v3f32(<3 x float>, <3 x float>, <3 x float>)
+declare <4 x double> @llvm.fma.v4f64(<4 x double>, <4 x double>, <4 x double>)
+declare <4 x double> @llvm.fmuladd.v4f64(<4 x double>, <4 x double>, <4 x double>)
+declare <4 x float> @llvm.fma.v4f32(<4 x float>, <4 x float>, <4 x float>)
+declare <4 x float> @llvm.fmuladd.v4f32(<4 x float>, <4 x float>, <4 x float>)
+declare <4 x half> @llvm.fma.v4f16(<4 x half>, <4 x half>, <4 x half>)
+declare <4 x half> @llvm.fmuladd.v4f16(<4 x half>, <4 x half>, <4 x half>)
+declare <7 x half> @llvm.fma.v7f16(<7 x half>, <7 x half>, <7 x half>)
+declare <7 x half> @llvm.fmuladd.v7f16(<7 x half>, <7 x half>, <7 x half>)
+declare <8 x float> @llvm.fma.v8f32(<8 x float>, <8 x float>, <8 x float>)
+declare <8 x float> @llvm.fmuladd.v8f32(<8 x float>, <8 x float>, <8 x float>)
+declare <8 x half> @llvm.fma.v8f16(<8 x half>, <8 x half>, <8 x half>)
+declare <8 x half> @llvm.fmuladd.v8f16(<8 x half>, <8 x half>, <8 x half>)
+declare double @llvm.fma.f64(double, double, double)
+declare double @llvm.fmuladd.f64(double, double, double)
+declare float @llvm.fma.f32(float, float, float)
+declare float @llvm.fmuladd.f32(float, float, float)
+declare half @llvm.fma.f16(half, half, half)
+declare half @llvm.fmuladd.f16(half, half, half)
diff --git a/llvm/test/CodeGen/AArch64/fold-int-pow2-with-fmul-or-fdiv.ll b/llvm/test/CodeGen/AArch64/fold-int-pow2-with-fmul-or-fdiv.ll
index c30c53835c..8f3100c827 100644
--- a/llvm/test/CodeGen/AArch64/fold-int-pow2-with-fmul-or-fdiv.ll
+++ b/llvm/test/CodeGen/AArch64/fold-int-pow2-with-fmul-or-fdiv.ll
@@ -52,7 +52,8 @@ define <4 x float> @fmul_pow2_ldexp_4xfloat(<4 x i32> %i) {
 ; CHECK-NEON-NEXT:    fmov s0, #9.00000000
 ; CHECK-NEON-NEXT:    bl ldexpf
 ; CHECK-NEON-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEON-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-NEON-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEON-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEON-NEXT:    fmov s0, #9.00000000
 ; CHECK-NEON-NEXT:    fmov w0, s1
 ; CHECK-NEON-NEXT:    bl ldexpf
diff --git a/llvm/test/CodeGen/AArch64/fpow.ll b/llvm/test/CodeGen/AArch64/fpow.ll
index 79c99c48ce..1b3ae7a227 100644
--- a/llvm/test/CodeGen/AArch64/fpow.ll
+++ b/llvm/test/CodeGen/AArch64/fpow.ll
@@ -66,6 +66,7 @@ define <2 x double> @pow_v2f64(<2 x double> %a, <2 x double> %b) {
 ; CHECK-SD-NEXT:    mov d0, v0.d[1]
 ; CHECK-SD-NEXT:    mov d1, v1.d[1]
 ; CHECK-SD-NEXT:    bl pow
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-SD-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldp q0, q1, [sp] // 32-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -92,9 +93,10 @@ define <2 x double> @pow_v2f64(<2 x double> %a, <2 x double> %b) {
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 killed $q1
 ; CHECK-GI-NEXT:    bl pow
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    fmov d0, d8
 ; CHECK-GI-NEXT:    fmov d1, d9
+; CHECK-GI-NEXT:    fmov d0, d8
 ; CHECK-GI-NEXT:    bl pow
 ; CHECK-GI-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
@@ -198,6 +200,7 @@ define <4 x double> @pow_v4f64(<4 x double> %a, <4 x double> %b) {
 ; CHECK-SD-NEXT:    stp q1, q3, [sp, #48] // 32-byte Folded Spill
 ; CHECK-SD-NEXT:    mov d1, v2.d[1]
 ; CHECK-SD-NEXT:    bl pow
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-SD-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldp q0, q1, [sp] // 32-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -211,6 +214,7 @@ define <4 x double> @pow_v4f64(<4 x double> %a, <4 x double> %b) {
 ; CHECK-SD-NEXT:    mov d0, v0.d[1]
 ; CHECK-SD-NEXT:    mov d1, v1.d[1]
 ; CHECK-SD-NEXT:    bl pow
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldp q0, q1, [sp, #48] // 32-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -246,19 +250,22 @@ define <4 x double> @pow_v4f64(<4 x double> %a, <4 x double> %b) {
 ; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 killed $q1
 ; CHECK-GI-NEXT:    mov d9, v4.d[1]
 ; CHECK-GI-NEXT:    bl pow
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    str q0, [sp, #48] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    fmov d0, d8
 ; CHECK-GI-NEXT:    fmov d1, d10
+; CHECK-GI-NEXT:    fmov d0, d8
 ; CHECK-GI-NEXT:    bl pow
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GI-NEXT:    bl pow
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    fmov d0, d9
 ; CHECK-GI-NEXT:    fmov d1, d11
+; CHECK-GI-NEXT:    fmov d0, d9
 ; CHECK-GI-NEXT:    bl pow
 ; CHECK-GI-NEXT:    ldr q2, [sp, #48] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
@@ -290,7 +297,8 @@ define <2 x float> @pow_v2f32(<2 x float> %a, <2 x float> %b) {
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    mov s1, v1.s[1]
 ; CHECK-SD-NEXT:    bl powf
-; CHECK-SD-NEXT:    str d0, [sp, #32] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldp q0, q1, [sp] // 32-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    // kill: def $s1 killed $s1 killed $q1
@@ -319,9 +327,10 @@ define <2 x float> @pow_v2f32(<2 x float> %a, <2 x float> %b) {
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-GI-NEXT:    // kill: def $s1 killed $s1 killed $q1
 ; CHECK-GI-NEXT:    bl powf
-; CHECK-GI-NEXT:    str d0, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    fmov s0, s8
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s1, s9
+; CHECK-GI-NEXT:    fmov s0, s8
 ; CHECK-GI-NEXT:    bl powf
 ; CHECK-GI-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
@@ -347,7 +356,8 @@ define <3 x float> @pow_v3f32(<3 x float> %a, <3 x float> %b) {
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    mov s1, v1.s[1]
 ; CHECK-SD-NEXT:    bl powf
-; CHECK-SD-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldp q0, q1, [sp, #16] // 32-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    // kill: def $s1 killed $s1 killed $q1
@@ -387,13 +397,15 @@ define <3 x float> @pow_v3f32(<3 x float> %a, <3 x float> %b) {
 ; CHECK-GI-NEXT:    mov s11, v1.s[2]
 ; CHECK-GI-NEXT:    // kill: def $s1 killed $s1 killed $q1
 ; CHECK-GI-NEXT:    bl powf
-; CHECK-GI-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    fmov s0, s8
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s1, s10
+; CHECK-GI-NEXT:    fmov s0, s8
 ; CHECK-GI-NEXT:    bl powf
-; CHECK-GI-NEXT:    str d0, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    fmov s0, s9
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s1, s11
+; CHECK-GI-NEXT:    fmov s0, s9
 ; CHECK-GI-NEXT:    bl powf
 ; CHECK-GI-NEXT:    ldp q2, q1, [sp] // 32-byte Folded Reload
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
@@ -422,7 +434,8 @@ define <4 x float> @pow_v4f32(<4 x float> %a, <4 x float> %b) {
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    mov s1, v1.s[1]
 ; CHECK-SD-NEXT:    bl powf
-; CHECK-SD-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldp q0, q1, [sp, #16] // 32-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    // kill: def $s1 killed $s1 killed $q1
@@ -475,17 +488,20 @@ define <4 x float> @pow_v4f32(<4 x float> %a, <4 x float> %b) {
 ; CHECK-GI-NEXT:    mov s13, v1.s[3]
 ; CHECK-GI-NEXT:    // kill: def $s1 killed $s1 killed $q1
 ; CHECK-GI-NEXT:    bl powf
-; CHECK-GI-NEXT:    str d0, [sp, #32] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    fmov s0, s8
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s1, s11
+; CHECK-GI-NEXT:    fmov s0, s8
 ; CHECK-GI-NEXT:    bl powf
-; CHECK-GI-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    fmov s0, s9
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s1, s12
+; CHECK-GI-NEXT:    fmov s0, s9
 ; CHECK-GI-NEXT:    bl powf
-; CHECK-GI-NEXT:    str d0, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    fmov s0, s10
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s1, s13
+; CHECK-GI-NEXT:    fmov s0, s10
 ; CHECK-GI-NEXT:    bl powf
 ; CHECK-GI-NEXT:    ldp q2, q1, [sp, #16] // 32-byte Folded Reload
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
@@ -517,7 +533,8 @@ define <8 x float> @pow_v8f32(<8 x float> %a, <8 x float> %b) {
 ; CHECK-SD-NEXT:    stp q1, q3, [sp, #32] // 32-byte Folded Spill
 ; CHECK-SD-NEXT:    mov s1, v2.s[1]
 ; CHECK-SD-NEXT:    bl powf
-; CHECK-SD-NEXT:    str d0, [sp, #64] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp, #64] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldp q0, q1, [sp] // 32-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    // kill: def $s1 killed $s1 killed $q1
@@ -546,7 +563,8 @@ define <8 x float> @pow_v8f32(<8 x float> %a, <8 x float> %b) {
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    mov s1, v1.s[1]
 ; CHECK-SD-NEXT:    bl powf
-; CHECK-SD-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldp q0, q1, [sp, #32] // 32-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    // kill: def $s1 killed $s1 killed $q1
@@ -614,32 +632,39 @@ define <8 x float> @pow_v8f32(<8 x float> %a, <8 x float> %b) {
 ; CHECK-GI-NEXT:    mov s2, v3.s[3]
 ; CHECK-GI-NEXT:    stp s2, s5, [sp, #200] // 8-byte Folded Spill
 ; CHECK-GI-NEXT:    bl powf
-; CHECK-GI-NEXT:    str d0, [sp, #96] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    fmov s0, s8
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #96] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s1, s14
+; CHECK-GI-NEXT:    fmov s0, s8
 ; CHECK-GI-NEXT:    bl powf
-; CHECK-GI-NEXT:    str d0, [sp, #32] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    fmov s0, s9
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s1, s15
+; CHECK-GI-NEXT:    fmov s0, s9
 ; CHECK-GI-NEXT:    bl powf
-; CHECK-GI-NEXT:    str d0, [sp, #80] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    fmov s0, s10
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #80] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s1, s13
+; CHECK-GI-NEXT:    fmov s0, s10
 ; CHECK-GI-NEXT:    bl powf
-; CHECK-GI-NEXT:    str d0, [sp, #64] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #64] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    ldp q1, q0, [sp] // 32-byte Folded Reload
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-GI-NEXT:    // kill: def $s1 killed $s1 killed $q1
 ; CHECK-GI-NEXT:    bl powf
 ; CHECK-GI-NEXT:    fmov s1, s12
-; CHECK-GI-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    ldr s0, [sp, #48] // 4-byte Folded Reload
 ; CHECK-GI-NEXT:    bl powf
 ; CHECK-GI-NEXT:    fmov s1, s11
-; CHECK-GI-NEXT:    str d0, [sp, #48] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #48] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    ldr s0, [sp, #112] // 4-byte Folded Reload
 ; CHECK-GI-NEXT:    bl powf
-; CHECK-GI-NEXT:    str d0, [sp, #112] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #112] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    ldp s1, s0, [sp, #200] // 8-byte Folded Reload
 ; CHECK-GI-NEXT:    bl powf
 ; CHECK-GI-NEXT:    ldp q3, q2, [sp, #16] // 32-byte Folded Reload
diff --git a/llvm/test/CodeGen/AArch64/fptoi.ll b/llvm/test/CodeGen/AArch64/fptoi.ll
new file mode 100644
index 0000000000..8fbb074136
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/fptoi.ll
@@ -0,0 +1,5518 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16
+; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16
+; RUN: llc -mtriple=aarch64-none-eabi -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16
+; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16
+
+; CHECK-GI:       warning: Instruction selection used fallback path for fptos_v3f64_v3i64
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v3f64_v3i64
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v4f64_v4i64
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v4f64_v4i64
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v8f64_v8i64
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v8f64_v8i64
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v16f64_v16i64
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v16f64_v16i64
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v32f64_v32i64
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v32f64_v32i64
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v3f64_v3i32
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v3f64_v3i32
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v4f64_v4i32
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v4f64_v4i32
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v8f64_v8i32
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v8f64_v8i32
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v16f64_v16i32
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v16f64_v16i32
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v32f64_v32i32
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v32f64_v32i32
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v2f64_v2i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v2f64_v2i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v3f64_v3i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v3f64_v3i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v4f64_v4i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v4f64_v4i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v8f64_v8i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v8f64_v8i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v16f64_v16i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v16f64_v16i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v32f64_v32i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v32f64_v32i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v2f64_v2i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v2f64_v2i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v3f64_v3i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v3f64_v3i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v4f64_v4i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v4f64_v4i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v8f64_v8i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v8f64_v8i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v16f64_v16i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v16f64_v16i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v32f64_v32i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v32f64_v32i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v3f32_v3i64
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v3f32_v3i64
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v4f32_v4i64
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v4f32_v4i64
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v8f32_v8i64
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v8f32_v8i64
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v16f32_v16i64
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v16f32_v16i64
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v32f32_v32i64
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v32f32_v32i64
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v3f32_v3i32
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v3f32_v3i32
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v8f32_v8i32
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v8f32_v8i32
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v16f32_v16i32
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v16f32_v16i32
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v32f32_v32i32
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v32f32_v32i32
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v2f32_v2i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v2f32_v2i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v3f32_v3i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v3f32_v3i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v4f32_v4i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v4f32_v4i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v8f32_v8i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v8f32_v8i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v16f32_v16i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v16f32_v16i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v32f32_v32i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v32f32_v32i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v2f32_v2i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v2f32_v2i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v3f32_v3i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v3f32_v3i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v4f32_v4i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v4f32_v4i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v8f32_v8i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v8f32_v8i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v16f32_v16i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v16f32_v16i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v32f32_v32i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v32f32_v32i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v2f16_v2i64
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v2f16_v2i64
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v3f16_v3i64
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v3f16_v3i64
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v4f16_v4i64
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v4f16_v4i64
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v8f16_v8i64
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v8f16_v8i64
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v16f16_v16i64
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v16f16_v16i64
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v32f16_v32i64
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v32f16_v32i64
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v2f16_v2i32
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v2f16_v2i32
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v3f16_v3i32
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v3f16_v3i32
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v4f16_v4i32
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v4f16_v4i32
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v8f16_v8i32
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v8f16_v8i32
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v16f16_v16i32
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v16f16_v16i32
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v32f16_v32i32
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v32f16_v32i32
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v2f16_v2i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v2f16_v2i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v3f16_v3i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v3f16_v3i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v4f16_v4i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v4f16_v4i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v8f16_v8i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v8f16_v8i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v16f16_v16i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v16f16_v16i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v32f16_v32i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v32f16_v32i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v2f16_v2i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v2f16_v2i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v3f16_v3i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v3f16_v3i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v4f16_v4i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v4f16_v4i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v8f16_v8i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v8f16_v8i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v16f16_v16i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v16f16_v16i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptos_v32f16_v32i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fptou_v32f16_v32i8
+
+define i64 @fptos_f64_i64(double %a) {
+; CHECK-LABEL: fptos_f64_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs x0, d0
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi double %a to i64
+  ret i64 %c
+}
+
+define i64 @fptou_f64_i64(double %a) {
+; CHECK-LABEL: fptou_f64_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu x0, d0
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui double %a to i64
+  ret i64 %c
+}
+
+define i32 @fptos_f64_i32(double %a) {
+; CHECK-LABEL: fptos_f64_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs w0, d0
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi double %a to i32
+  ret i32 %c
+}
+
+define i32 @fptou_f64_i32(double %a) {
+; CHECK-LABEL: fptou_f64_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu w0, d0
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui double %a to i32
+  ret i32 %c
+}
+
+define i16 @fptos_f64_i16(double %a) {
+; CHECK-LABEL: fptos_f64_i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs w0, d0
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi double %a to i16
+  ret i16 %c
+}
+
+define i16 @fptou_f64_i16(double %a) {
+; CHECK-SD-LABEL: fptou_f64_i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fcvtzs w0, d0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fptou_f64_i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fcvtzu w0, d0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = fptoui double %a to i16
+  ret i16 %c
+}
+
+define i8 @fptos_f64_i8(double %a) {
+; CHECK-LABEL: fptos_f64_i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs w0, d0
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi double %a to i8
+  ret i8 %c
+}
+
+define i8 @fptou_f64_i8(double %a) {
+; CHECK-SD-LABEL: fptou_f64_i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fcvtzs w0, d0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fptou_f64_i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fcvtzu w0, d0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = fptoui double %a to i8
+  ret i8 %c
+}
+
+define i64 @fptos_f32_i64(float %a) {
+; CHECK-LABEL: fptos_f32_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs x0, s0
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi float %a to i64
+  ret i64 %c
+}
+
+define i64 @fptou_f32_i64(float %a) {
+; CHECK-LABEL: fptou_f32_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu x0, s0
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui float %a to i64
+  ret i64 %c
+}
+
+define i32 @fptos_f32_i32(float %a) {
+; CHECK-LABEL: fptos_f32_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs w0, s0
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi float %a to i32
+  ret i32 %c
+}
+
+define i32 @fptou_f32_i32(float %a) {
+; CHECK-LABEL: fptou_f32_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu w0, s0
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui float %a to i32
+  ret i32 %c
+}
+
+define i16 @fptos_f32_i16(float %a) {
+; CHECK-LABEL: fptos_f32_i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs w0, s0
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi float %a to i16
+  ret i16 %c
+}
+
+define i16 @fptou_f32_i16(float %a) {
+; CHECK-SD-LABEL: fptou_f32_i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fcvtzs w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fptou_f32_i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fcvtzu w0, s0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = fptoui float %a to i16
+  ret i16 %c
+}
+
+define i8 @fptos_f32_i8(float %a) {
+; CHECK-LABEL: fptos_f32_i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs w0, s0
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi float %a to i8
+  ret i8 %c
+}
+
+define i8 @fptou_f32_i8(float %a) {
+; CHECK-SD-LABEL: fptou_f32_i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fcvtzs w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fptou_f32_i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fcvtzu w0, s0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = fptoui float %a to i8
+  ret i8 %c
+}
+
+define i64 @fptos_f16_i64(half %a) {
+; CHECK-SD-NOFP16-LABEL: fptos_f16_i64:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x0, s0
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptos_f16_i64:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fcvtzs x0, h0
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-LABEL: fptos_f16_i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fcvt s0, h0
+; CHECK-GI-NEXT:    fcvtzs x0, s0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = fptosi half %a to i64
+  ret i64 %c
+}
+
+define i64 @fptou_f16_i64(half %a) {
+; CHECK-SD-NOFP16-LABEL: fptou_f16_i64:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x0, s0
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptou_f16_i64:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fcvtzu x0, h0
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-LABEL: fptou_f16_i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fcvt s0, h0
+; CHECK-GI-NEXT:    fcvtzu x0, s0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = fptoui half %a to i64
+  ret i64 %c
+}
+
+define i32 @fptos_f16_i32(half %a) {
+; CHECK-SD-NOFP16-LABEL: fptos_f16_i32:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fcvtzs w0, s0
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptos_f16_i32:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fcvtzs w0, h0
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-LABEL: fptos_f16_i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fcvt s0, h0
+; CHECK-GI-NEXT:    fcvtzs w0, s0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = fptosi half %a to i32
+  ret i32 %c
+}
+
+define i32 @fptou_f16_i32(half %a) {
+; CHECK-SD-NOFP16-LABEL: fptou_f16_i32:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fcvtzu w0, s0
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptou_f16_i32:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fcvtzu w0, h0
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-LABEL: fptou_f16_i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fcvt s0, h0
+; CHECK-GI-NEXT:    fcvtzu w0, s0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = fptoui half %a to i32
+  ret i32 %c
+}
+
+define i16 @fptos_f16_i16(half %a) {
+; CHECK-SD-NOFP16-LABEL: fptos_f16_i16:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fcvtzs w0, s0
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptos_f16_i16:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fcvtzs w0, h0
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-LABEL: fptos_f16_i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fcvt s0, h0
+; CHECK-GI-NEXT:    fcvtzs w0, s0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = fptosi half %a to i16
+  ret i16 %c
+}
+
+define i16 @fptou_f16_i16(half %a) {
+; CHECK-SD-NOFP16-LABEL: fptou_f16_i16:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fcvtzs w0, s0
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptou_f16_i16:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fcvtzs w0, h0
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-LABEL: fptou_f16_i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fcvt s0, h0
+; CHECK-GI-NEXT:    fcvtzu w0, s0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = fptoui half %a to i16
+  ret i16 %c
+}
+
+define i8 @fptos_f16_i8(half %a) {
+; CHECK-SD-NOFP16-LABEL: fptos_f16_i8:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fcvtzs w0, s0
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptos_f16_i8:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fcvtzs w0, h0
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-LABEL: fptos_f16_i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fcvt s0, h0
+; CHECK-GI-NEXT:    fcvtzs w0, s0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = fptosi half %a to i8
+  ret i8 %c
+}
+
+define i8 @fptou_f16_i8(half %a) {
+; CHECK-SD-NOFP16-LABEL: fptou_f16_i8:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fcvtzs w0, s0
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptou_f16_i8:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fcvtzs w0, h0
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-LABEL: fptou_f16_i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fcvt s0, h0
+; CHECK-GI-NEXT:    fcvtzu w0, s0
+; CHECK-GI-NEXT:    ret
+entry:
+  %c = fptoui half %a to i8
+  ret i8 %c
+}
+
+define <2 x i64> @fptos_v2f64_v2i64(<2 x double> %a) {
+; CHECK-LABEL: fptos_v2f64_v2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <2 x double> %a to <2 x i64>
+  ret <2 x i64> %c
+}
+
+define <2 x i64> @fptou_v2f64_v2i64(<2 x double> %a) {
+; CHECK-LABEL: fptou_v2f64_v2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <2 x double> %a to <2 x i64>
+  ret <2 x i64> %c
+}
+
+define <3 x i64> @fptos_v3f64_v3i64(<3 x double> %a) {
+; CHECK-LABEL: fptos_v3f64_v3i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <3 x double> %a to <3 x i64>
+  ret <3 x i64> %c
+}
+
+define <3 x i64> @fptou_v3f64_v3i64(<3 x double> %a) {
+; CHECK-LABEL: fptou_v3f64_v3i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    fcvtzu v2.2d, v2.2d
+; CHECK-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <3 x double> %a to <3 x i64>
+  ret <3 x i64> %c
+}
+
+define <4 x i64> @fptos_v4f64_v4i64(<4 x double> %a) {
+; CHECK-LABEL: fptos_v4f64_v4i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <4 x double> %a to <4 x i64>
+  ret <4 x i64> %c
+}
+
+define <4 x i64> @fptou_v4f64_v4i64(<4 x double> %a) {
+; CHECK-LABEL: fptou_v4f64_v4i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <4 x double> %a to <4 x i64>
+  ret <4 x i64> %c
+}
+
+define <8 x i64> @fptos_v8f64_v8i64(<8 x double> %a) {
+; CHECK-LABEL: fptos_v8f64_v8i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <8 x double> %a to <8 x i64>
+  ret <8 x i64> %c
+}
+
+define <8 x i64> @fptou_v8f64_v8i64(<8 x double> %a) {
+; CHECK-LABEL: fptou_v8f64_v8i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-NEXT:    fcvtzu v2.2d, v2.2d
+; CHECK-NEXT:    fcvtzu v3.2d, v3.2d
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <8 x double> %a to <8 x i64>
+  ret <8 x i64> %c
+}
+
+define <16 x i64> @fptos_v16f64_v16i64(<16 x double> %a) {
+; CHECK-LABEL: fptos_v16f64_v16i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NEXT:    fcvtzs v4.2d, v4.2d
+; CHECK-NEXT:    fcvtzs v5.2d, v5.2d
+; CHECK-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <16 x double> %a to <16 x i64>
+  ret <16 x i64> %c
+}
+
+define <16 x i64> @fptou_v16f64_v16i64(<16 x double> %a) {
+; CHECK-LABEL: fptou_v16f64_v16i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-NEXT:    fcvtzu v2.2d, v2.2d
+; CHECK-NEXT:    fcvtzu v3.2d, v3.2d
+; CHECK-NEXT:    fcvtzu v4.2d, v4.2d
+; CHECK-NEXT:    fcvtzu v5.2d, v5.2d
+; CHECK-NEXT:    fcvtzu v6.2d, v6.2d
+; CHECK-NEXT:    fcvtzu v7.2d, v7.2d
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <16 x double> %a to <16 x i64>
+  ret <16 x i64> %c
+}
+
+define <32 x i64> @fptos_v32f64_v32i64(<32 x double> %a) {
+; CHECK-LABEL: fptos_v32f64_v32i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldp q17, q16, [sp, #96]
+; CHECK-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-NEXT:    ldp q19, q18, [sp, #64]
+; CHECK-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-NEXT:    ldp q21, q20, [sp, #32]
+; CHECK-NEXT:    fcvtzs v5.2d, v5.2d
+; CHECK-NEXT:    fcvtzs v16.2d, v16.2d
+; CHECK-NEXT:    fcvtzs v17.2d, v17.2d
+; CHECK-NEXT:    fcvtzs v4.2d, v4.2d
+; CHECK-NEXT:    fcvtzs v18.2d, v18.2d
+; CHECK-NEXT:    fcvtzs v19.2d, v19.2d
+; CHECK-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NEXT:    fcvtzs v20.2d, v20.2d
+; CHECK-NEXT:    fcvtzs v21.2d, v21.2d
+; CHECK-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    stp q5, q6, [x8, #80]
+; CHECK-NEXT:    str q16, [x8, #240]
+; CHECK-NEXT:    ldp q22, q16, [sp]
+; CHECK-NEXT:    stp q3, q4, [x8, #48]
+; CHECK-NEXT:    stp q20, q19, [x8, #176]
+; CHECK-NEXT:    fcvtzs v16.2d, v16.2d
+; CHECK-NEXT:    stp q1, q2, [x8, #16]
+; CHECK-NEXT:    stp q18, q17, [x8, #208]
+; CHECK-NEXT:    fcvtzs v17.2d, v22.2d
+; CHECK-NEXT:    str q0, [x8]
+; CHECK-NEXT:    stp q16, q21, [x8, #144]
+; CHECK-NEXT:    stp q7, q17, [x8, #112]
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <32 x double> %a to <32 x i64>
+  ret <32 x i64> %c
+}
+
+define <32 x i64> @fptou_v32f64_v32i64(<32 x double> %a) {
+; CHECK-LABEL: fptou_v32f64_v32i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldp q17, q16, [sp, #96]
+; CHECK-NEXT:    fcvtzu v7.2d, v7.2d
+; CHECK-NEXT:    ldp q19, q18, [sp, #64]
+; CHECK-NEXT:    fcvtzu v6.2d, v6.2d
+; CHECK-NEXT:    ldp q21, q20, [sp, #32]
+; CHECK-NEXT:    fcvtzu v5.2d, v5.2d
+; CHECK-NEXT:    fcvtzu v16.2d, v16.2d
+; CHECK-NEXT:    fcvtzu v17.2d, v17.2d
+; CHECK-NEXT:    fcvtzu v4.2d, v4.2d
+; CHECK-NEXT:    fcvtzu v18.2d, v18.2d
+; CHECK-NEXT:    fcvtzu v19.2d, v19.2d
+; CHECK-NEXT:    fcvtzu v3.2d, v3.2d
+; CHECK-NEXT:    fcvtzu v20.2d, v20.2d
+; CHECK-NEXT:    fcvtzu v21.2d, v21.2d
+; CHECK-NEXT:    fcvtzu v2.2d, v2.2d
+; CHECK-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-NEXT:    stp q5, q6, [x8, #80]
+; CHECK-NEXT:    str q16, [x8, #240]
+; CHECK-NEXT:    ldp q22, q16, [sp]
+; CHECK-NEXT:    stp q3, q4, [x8, #48]
+; CHECK-NEXT:    stp q20, q19, [x8, #176]
+; CHECK-NEXT:    fcvtzu v16.2d, v16.2d
+; CHECK-NEXT:    stp q1, q2, [x8, #16]
+; CHECK-NEXT:    stp q18, q17, [x8, #208]
+; CHECK-NEXT:    fcvtzu v17.2d, v22.2d
+; CHECK-NEXT:    str q0, [x8]
+; CHECK-NEXT:    stp q16, q21, [x8, #144]
+; CHECK-NEXT:    stp q7, q17, [x8, #112]
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <32 x double> %a to <32 x i64>
+  ret <32 x i64> %c
+}
+
+define <2 x i32> @fptos_v2f64_v2i32(<2 x double> %a) {
+; CHECK-LABEL: fptos_v2f64_v2i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <2 x double> %a to <2 x i32>
+  ret <2 x i32> %c
+}
+
+define <2 x i32> @fptou_v2f64_v2i32(<2 x double> %a) {
+; CHECK-LABEL: fptou_v2f64_v2i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <2 x double> %a to <2 x i32>
+  ret <2 x i32> %c
+}
+
+define <3 x i32> @fptos_v3f64_v3i32(<3 x double> %a) {
+; CHECK-LABEL: fptos_v3f64_v3i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    fcvtzs v1.2d, v2.2d
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <3 x double> %a to <3 x i32>
+  ret <3 x i32> %c
+}
+
+define <3 x i32> @fptou_v3f64_v3i32(<3 x double> %a) {
+; CHECK-LABEL: fptou_v3f64_v3i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    fcvtzu v1.2d, v2.2d
+; CHECK-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <3 x double> %a to <3 x i32>
+  ret <3 x i32> %c
+}
+
+define <4 x i32> @fptos_v4f64_v4i32(<4 x double> %a) {
+; CHECK-LABEL: fptos_v4f64_v4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <4 x double> %a to <4 x i32>
+  ret <4 x i32> %c
+}
+
+define <4 x i32> @fptou_v4f64_v4i32(<4 x double> %a) {
+; CHECK-LABEL: fptou_v4f64_v4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <4 x double> %a to <4 x i32>
+  ret <4 x i32> %c
+}
+
+define <8 x i32> @fptos_v8f64_v8i32(<8 x double> %a) {
+; CHECK-LABEL: fptos_v8f64_v8i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <8 x double> %a to <8 x i32>
+  ret <8 x i32> %c
+}
+
+define <8 x i32> @fptou_v8f64_v8i32(<8 x double> %a) {
+; CHECK-LABEL: fptou_v8f64_v8i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-NEXT:    fcvtzu v3.2d, v3.2d
+; CHECK-NEXT:    fcvtzu v2.2d, v2.2d
+; CHECK-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <8 x double> %a to <8 x i32>
+  ret <8 x i32> %c
+}
+
+define <16 x i32> @fptos_v16f64_v16i32(<16 x double> %a) {
+; CHECK-LABEL: fptos_v16f64_v16i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NEXT:    fcvtzs v5.2d, v5.2d
+; CHECK-NEXT:    fcvtzs v4.2d, v4.2d
+; CHECK-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
+; CHECK-NEXT:    uzp1 v2.4s, v4.4s, v5.4s
+; CHECK-NEXT:    uzp1 v3.4s, v6.4s, v7.4s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <16 x double> %a to <16 x i32>
+  ret <16 x i32> %c
+}
+
+define <16 x i32> @fptou_v16f64_v16i32(<16 x double> %a) {
+; CHECK-LABEL: fptou_v16f64_v16i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu v3.2d, v3.2d
+; CHECK-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-NEXT:    fcvtzu v2.2d, v2.2d
+; CHECK-NEXT:    fcvtzu v5.2d, v5.2d
+; CHECK-NEXT:    fcvtzu v4.2d, v4.2d
+; CHECK-NEXT:    fcvtzu v7.2d, v7.2d
+; CHECK-NEXT:    fcvtzu v6.2d, v6.2d
+; CHECK-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
+; CHECK-NEXT:    uzp1 v2.4s, v4.4s, v5.4s
+; CHECK-NEXT:    uzp1 v3.4s, v6.4s, v7.4s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <16 x double> %a to <16 x i32>
+  ret <16 x i32> %c
+}
+
+define <32 x i32> @fptos_v32f64_v32i32(<32 x double> %a) {
+; CHECK-LABEL: fptos_v32f64_v32i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldp q16, q17, [sp, #96]
+; CHECK-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NEXT:    ldp q18, q19, [sp, #64]
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    ldp q20, q21, [sp, #32]
+; CHECK-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NEXT:    ldp q22, q23, [sp]
+; CHECK-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NEXT:    fcvtzs v5.2d, v5.2d
+; CHECK-NEXT:    fcvtzs v4.2d, v4.2d
+; CHECK-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-NEXT:    fcvtzs v21.2d, v21.2d
+; CHECK-NEXT:    fcvtzs v20.2d, v20.2d
+; CHECK-NEXT:    fcvtzs v23.2d, v23.2d
+; CHECK-NEXT:    fcvtzs v22.2d, v22.2d
+; CHECK-NEXT:    fcvtzs v19.2d, v19.2d
+; CHECK-NEXT:    fcvtzs v18.2d, v18.2d
+; CHECK-NEXT:    fcvtzs v17.2d, v17.2d
+; CHECK-NEXT:    fcvtzs v16.2d, v16.2d
+; CHECK-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
+; CHECK-NEXT:    uzp1 v2.4s, v4.4s, v5.4s
+; CHECK-NEXT:    uzp1 v3.4s, v6.4s, v7.4s
+; CHECK-NEXT:    uzp1 v5.4s, v20.4s, v21.4s
+; CHECK-NEXT:    uzp1 v4.4s, v22.4s, v23.4s
+; CHECK-NEXT:    uzp1 v6.4s, v18.4s, v19.4s
+; CHECK-NEXT:    uzp1 v7.4s, v16.4s, v17.4s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <32 x double> %a to <32 x i32>
+  ret <32 x i32> %c
+}
+
+define <32 x i32> @fptou_v32f64_v32i32(<32 x double> %a) {
+; CHECK-LABEL: fptou_v32f64_v32i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldp q16, q17, [sp, #96]
+; CHECK-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-NEXT:    ldp q18, q19, [sp, #64]
+; CHECK-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-NEXT:    ldp q20, q21, [sp, #32]
+; CHECK-NEXT:    fcvtzu v3.2d, v3.2d
+; CHECK-NEXT:    ldp q22, q23, [sp]
+; CHECK-NEXT:    fcvtzu v2.2d, v2.2d
+; CHECK-NEXT:    fcvtzu v5.2d, v5.2d
+; CHECK-NEXT:    fcvtzu v4.2d, v4.2d
+; CHECK-NEXT:    fcvtzu v7.2d, v7.2d
+; CHECK-NEXT:    fcvtzu v6.2d, v6.2d
+; CHECK-NEXT:    fcvtzu v21.2d, v21.2d
+; CHECK-NEXT:    fcvtzu v20.2d, v20.2d
+; CHECK-NEXT:    fcvtzu v23.2d, v23.2d
+; CHECK-NEXT:    fcvtzu v22.2d, v22.2d
+; CHECK-NEXT:    fcvtzu v19.2d, v19.2d
+; CHECK-NEXT:    fcvtzu v18.2d, v18.2d
+; CHECK-NEXT:    fcvtzu v17.2d, v17.2d
+; CHECK-NEXT:    fcvtzu v16.2d, v16.2d
+; CHECK-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
+; CHECK-NEXT:    uzp1 v2.4s, v4.4s, v5.4s
+; CHECK-NEXT:    uzp1 v3.4s, v6.4s, v7.4s
+; CHECK-NEXT:    uzp1 v5.4s, v20.4s, v21.4s
+; CHECK-NEXT:    uzp1 v4.4s, v22.4s, v23.4s
+; CHECK-NEXT:    uzp1 v6.4s, v18.4s, v19.4s
+; CHECK-NEXT:    uzp1 v7.4s, v16.4s, v17.4s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <32 x double> %a to <32 x i32>
+  ret <32 x i32> %c
+}
+
+define <2 x i16> @fptos_v2f64_v2i16(<2 x double> %a) {
+; CHECK-LABEL: fptos_v2f64_v2i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <2 x double> %a to <2 x i16>
+  ret <2 x i16> %c
+}
+
+define <2 x i16> @fptou_v2f64_v2i16(<2 x double> %a) {
+; CHECK-LABEL: fptou_v2f64_v2i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <2 x double> %a to <2 x i16>
+  ret <2 x i16> %c
+}
+
+define <3 x i16> @fptos_v3f64_v3i16(<3 x double> %a) {
+; CHECK-LABEL: fptos_v3f64_v3i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    fcvtzs v1.2d, v2.2d
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    xtn v1.2s, v1.2d
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    uzp1 v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <3 x double> %a to <3 x i16>
+  ret <3 x i16> %c
+}
+
+define <3 x i16> @fptou_v3f64_v3i16(<3 x double> %a) {
+; CHECK-LABEL: fptou_v3f64_v3i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    fcvtzs v1.2d, v2.2d
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    xtn v1.2s, v1.2d
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    uzp1 v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <3 x double> %a to <3 x i16>
+  ret <3 x i16> %c
+}
+
+define <4 x i16> @fptos_v4f64_v4i16(<4 x double> %a) {
+; CHECK-LABEL: fptos_v4f64_v4i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    xtn v1.2s, v1.2d
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    uzp1 v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <4 x double> %a to <4 x i16>
+  ret <4 x i16> %c
+}
+
+define <4 x i16> @fptou_v4f64_v4i16(<4 x double> %a) {
+; CHECK-LABEL: fptou_v4f64_v4i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    xtn v1.2s, v1.2d
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    uzp1 v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <4 x double> %a to <4 x i16>
+  ret <4 x i16> %c
+}
+
+define <8 x i16> @fptos_v8f64_v8i16(<8 x double> %a) {
+; CHECK-LABEL: fptos_v8f64_v8i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NEXT:    adrp x8, .LCPI54_0
+; CHECK-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    xtn v6.2s, v3.2d
+; CHECK-NEXT:    xtn v5.2s, v2.2d
+; CHECK-NEXT:    xtn v4.2s, v1.2d
+; CHECK-NEXT:    xtn v3.2s, v0.2d
+; CHECK-NEXT:    ldr q0, [x8, :lo12:.LCPI54_0]
+; CHECK-NEXT:    tbl v0.16b, { v3.16b, v4.16b, v5.16b, v6.16b }, v0.16b
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <8 x double> %a to <8 x i16>
+  ret <8 x i16> %c
+}
+
+define <8 x i16> @fptou_v8f64_v8i16(<8 x double> %a) {
+; CHECK-LABEL: fptou_v8f64_v8i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NEXT:    adrp x8, .LCPI55_0
+; CHECK-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    xtn v6.2s, v3.2d
+; CHECK-NEXT:    xtn v5.2s, v2.2d
+; CHECK-NEXT:    xtn v4.2s, v1.2d
+; CHECK-NEXT:    xtn v3.2s, v0.2d
+; CHECK-NEXT:    ldr q0, [x8, :lo12:.LCPI55_0]
+; CHECK-NEXT:    tbl v0.16b, { v3.16b, v4.16b, v5.16b, v6.16b }, v0.16b
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <8 x double> %a to <8 x i16>
+  ret <8 x i16> %c
+}
+
+define <16 x i16> @fptos_v16f64_v16i16(<16 x double> %a) {
+; CHECK-LABEL: fptos_v16f64_v16i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-NEXT:    adrp x8, .LCPI56_0
+; CHECK-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NEXT:    fcvtzs v5.2d, v5.2d
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    fcvtzs v4.2d, v4.2d
+; CHECK-NEXT:    xtn v19.2s, v3.2d
+; CHECK-NEXT:    xtn v23.2s, v7.2d
+; CHECK-NEXT:    xtn v18.2s, v2.2d
+; CHECK-NEXT:    xtn v22.2s, v6.2d
+; CHECK-NEXT:    xtn v17.2s, v1.2d
+; CHECK-NEXT:    xtn v21.2s, v5.2d
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI56_0]
+; CHECK-NEXT:    xtn v16.2s, v0.2d
+; CHECK-NEXT:    xtn v20.2s, v4.2d
+; CHECK-NEXT:    tbl v0.16b, { v16.16b, v17.16b, v18.16b, v19.16b }, v1.16b
+; CHECK-NEXT:    tbl v1.16b, { v20.16b, v21.16b, v22.16b, v23.16b }, v1.16b
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <16 x double> %a to <16 x i16>
+  ret <16 x i16> %c
+}
+
+define <16 x i16> @fptou_v16f64_v16i16(<16 x double> %a) {
+; CHECK-LABEL: fptou_v16f64_v16i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-NEXT:    adrp x8, .LCPI57_0
+; CHECK-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NEXT:    fcvtzs v5.2d, v5.2d
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    fcvtzs v4.2d, v4.2d
+; CHECK-NEXT:    xtn v19.2s, v3.2d
+; CHECK-NEXT:    xtn v23.2s, v7.2d
+; CHECK-NEXT:    xtn v18.2s, v2.2d
+; CHECK-NEXT:    xtn v22.2s, v6.2d
+; CHECK-NEXT:    xtn v17.2s, v1.2d
+; CHECK-NEXT:    xtn v21.2s, v5.2d
+; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI57_0]
+; CHECK-NEXT:    xtn v16.2s, v0.2d
+; CHECK-NEXT:    xtn v20.2s, v4.2d
+; CHECK-NEXT:    tbl v0.16b, { v16.16b, v17.16b, v18.16b, v19.16b }, v1.16b
+; CHECK-NEXT:    tbl v1.16b, { v20.16b, v21.16b, v22.16b, v23.16b }, v1.16b
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <16 x double> %a to <16 x i16>
+  ret <16 x i16> %c
+}
+
+define <32 x i16> @fptos_v32f64_v32i16(<32 x double> %a) {
+; CHECK-LABEL: fptos_v32f64_v32i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    stp d15, d14, [sp, #-64]! // 16-byte Folded Spill
+; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    .cfi_offset b8, -8
+; CHECK-NEXT:    .cfi_offset b9, -16
+; CHECK-NEXT:    .cfi_offset b10, -24
+; CHECK-NEXT:    .cfi_offset b11, -32
+; CHECK-NEXT:    .cfi_offset b12, -40
+; CHECK-NEXT:    .cfi_offset b13, -48
+; CHECK-NEXT:    .cfi_offset b14, -56
+; CHECK-NEXT:    .cfi_offset b15, -64
+; CHECK-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NEXT:    fcvtzs v18.2d, v2.2d
+; CHECK-NEXT:    adrp x8, .LCPI58_0
+; CHECK-NEXT:    fcvtzs v19.2d, v1.2d
+; CHECK-NEXT:    ldp q20, q21, [sp, #160]
+; CHECK-NEXT:    fcvtzs v22.2d, v0.2d
+; CHECK-NEXT:    ldp q23, q24, [sp, #96]
+; CHECK-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-NEXT:    ldp q16, q17, [sp, #128]
+; CHECK-NEXT:    xtn v3.2s, v3.2d
+; CHECK-NEXT:    fcvtzs v21.2d, v21.2d
+; CHECK-NEXT:    fcvtzs v20.2d, v20.2d
+; CHECK-NEXT:    xtn v2.2s, v18.2d
+; CHECK-NEXT:    ldp q18, q25, [sp, #64]
+; CHECK-NEXT:    xtn v1.2s, v19.2d
+; CHECK-NEXT:    fcvtzs v19.2d, v24.2d
+; CHECK-NEXT:    fcvtzs v17.2d, v17.2d
+; CHECK-NEXT:    xtn v0.2s, v22.2d
+; CHECK-NEXT:    fcvtzs v22.2d, v23.2d
+; CHECK-NEXT:    xtn v29.2s, v7.2d
+; CHECK-NEXT:    fcvtzs v7.2d, v25.2d
+; CHECK-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-NEXT:    fcvtzs v18.2d, v18.2d
+; CHECK-NEXT:    fcvtzs v16.2d, v16.2d
+; CHECK-NEXT:    fcvtzs v5.2d, v5.2d
+; CHECK-NEXT:    xtn v15.2s, v21.2d
+; CHECK-NEXT:    xtn v11.2s, v19.2d
+; CHECK-NEXT:    fcvtzs v4.2d, v4.2d
+; CHECK-NEXT:    xtn v14.2s, v20.2d
+; CHECK-NEXT:    xtn v10.2s, v22.2d
+; CHECK-NEXT:    xtn v13.2s, v17.2d
+; CHECK-NEXT:    xtn v9.2s, v7.2d
+; CHECK-NEXT:    xtn v28.2s, v6.2d
+; CHECK-NEXT:    xtn v8.2s, v18.2d
+; CHECK-NEXT:    xtn v12.2s, v16.2d
+; CHECK-NEXT:    xtn v27.2s, v5.2d
+; CHECK-NEXT:    xtn v26.2s, v4.2d
+; CHECK-NEXT:    ldr q4, [x8, :lo12:.LCPI58_0]
+; CHECK-NEXT:    tbl v0.16b, { v0.16b, v1.16b, v2.16b, v3.16b }, v4.16b
+; CHECK-NEXT:    tbl v2.16b, { v8.16b, v9.16b, v10.16b, v11.16b }, v4.16b
+; CHECK-NEXT:    tbl v3.16b, { v12.16b, v13.16b, v14.16b, v15.16b }, v4.16b
+; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    tbl v1.16b, { v26.16b, v27.16b, v28.16b, v29.16b }, v4.16b
+; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp], #64 // 16-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <32 x double> %a to <32 x i16>
+  ret <32 x i16> %c
+}
+
+define <32 x i16> @fptou_v32f64_v32i16(<32 x double> %a) {
+; CHECK-LABEL: fptou_v32f64_v32i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    stp d15, d14, [sp, #-64]! // 16-byte Folded Spill
+; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    .cfi_offset b8, -8
+; CHECK-NEXT:    .cfi_offset b9, -16
+; CHECK-NEXT:    .cfi_offset b10, -24
+; CHECK-NEXT:    .cfi_offset b11, -32
+; CHECK-NEXT:    .cfi_offset b12, -40
+; CHECK-NEXT:    .cfi_offset b13, -48
+; CHECK-NEXT:    .cfi_offset b14, -56
+; CHECK-NEXT:    .cfi_offset b15, -64
+; CHECK-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NEXT:    fcvtzs v18.2d, v2.2d
+; CHECK-NEXT:    adrp x8, .LCPI59_0
+; CHECK-NEXT:    fcvtzs v19.2d, v1.2d
+; CHECK-NEXT:    ldp q20, q21, [sp, #160]
+; CHECK-NEXT:    fcvtzs v22.2d, v0.2d
+; CHECK-NEXT:    ldp q23, q24, [sp, #96]
+; CHECK-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-NEXT:    ldp q16, q17, [sp, #128]
+; CHECK-NEXT:    xtn v3.2s, v3.2d
+; CHECK-NEXT:    fcvtzs v21.2d, v21.2d
+; CHECK-NEXT:    fcvtzs v20.2d, v20.2d
+; CHECK-NEXT:    xtn v2.2s, v18.2d
+; CHECK-NEXT:    ldp q18, q25, [sp, #64]
+; CHECK-NEXT:    xtn v1.2s, v19.2d
+; CHECK-NEXT:    fcvtzs v19.2d, v24.2d
+; CHECK-NEXT:    fcvtzs v17.2d, v17.2d
+; CHECK-NEXT:    xtn v0.2s, v22.2d
+; CHECK-NEXT:    fcvtzs v22.2d, v23.2d
+; CHECK-NEXT:    xtn v29.2s, v7.2d
+; CHECK-NEXT:    fcvtzs v7.2d, v25.2d
+; CHECK-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-NEXT:    fcvtzs v18.2d, v18.2d
+; CHECK-NEXT:    fcvtzs v16.2d, v16.2d
+; CHECK-NEXT:    fcvtzs v5.2d, v5.2d
+; CHECK-NEXT:    xtn v15.2s, v21.2d
+; CHECK-NEXT:    xtn v11.2s, v19.2d
+; CHECK-NEXT:    fcvtzs v4.2d, v4.2d
+; CHECK-NEXT:    xtn v14.2s, v20.2d
+; CHECK-NEXT:    xtn v10.2s, v22.2d
+; CHECK-NEXT:    xtn v13.2s, v17.2d
+; CHECK-NEXT:    xtn v9.2s, v7.2d
+; CHECK-NEXT:    xtn v28.2s, v6.2d
+; CHECK-NEXT:    xtn v8.2s, v18.2d
+; CHECK-NEXT:    xtn v12.2s, v16.2d
+; CHECK-NEXT:    xtn v27.2s, v5.2d
+; CHECK-NEXT:    xtn v26.2s, v4.2d
+; CHECK-NEXT:    ldr q4, [x8, :lo12:.LCPI59_0]
+; CHECK-NEXT:    tbl v0.16b, { v0.16b, v1.16b, v2.16b, v3.16b }, v4.16b
+; CHECK-NEXT:    tbl v2.16b, { v8.16b, v9.16b, v10.16b, v11.16b }, v4.16b
+; CHECK-NEXT:    tbl v3.16b, { v12.16b, v13.16b, v14.16b, v15.16b }, v4.16b
+; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    tbl v1.16b, { v26.16b, v27.16b, v28.16b, v29.16b }, v4.16b
+; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp], #64 // 16-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <32 x double> %a to <32 x i16>
+  ret <32 x i16> %c
+}
+
+define <2 x i8> @fptos_v2f64_v2i8(<2 x double> %a) {
+; CHECK-LABEL: fptos_v2f64_v2i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <2 x double> %a to <2 x i8>
+  ret <2 x i8> %c
+}
+
+define <2 x i8> @fptou_v2f64_v2i8(<2 x double> %a) {
+; CHECK-LABEL: fptou_v2f64_v2i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <2 x double> %a to <2 x i8>
+  ret <2 x i8> %c
+}
+
+define <3 x i8> @fptos_v3f64_v3i8(<3 x double> %a) {
+; CHECK-LABEL: fptos_v3f64_v3i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    fcvtzs v1.2d, v2.2d
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    xtn v1.2s, v1.2d
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    uzp1 v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    umov w0, v0.h[0]
+; CHECK-NEXT:    umov w1, v0.h[1]
+; CHECK-NEXT:    umov w2, v0.h[2]
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <3 x double> %a to <3 x i8>
+  ret <3 x i8> %c
+}
+
+define <3 x i8> @fptou_v3f64_v3i8(<3 x double> %a) {
+; CHECK-LABEL: fptou_v3f64_v3i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    fcvtzs v1.2d, v2.2d
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    xtn v1.2s, v1.2d
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    uzp1 v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    umov w0, v0.h[0]
+; CHECK-NEXT:    umov w1, v0.h[1]
+; CHECK-NEXT:    umov w2, v0.h[2]
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <3 x double> %a to <3 x i8>
+  ret <3 x i8> %c
+}
+
+define <4 x i8> @fptos_v4f64_v4i8(<4 x double> %a) {
+; CHECK-LABEL: fptos_v4f64_v4i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    xtn v1.2s, v1.2d
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    uzp1 v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <4 x double> %a to <4 x i8>
+  ret <4 x i8> %c
+}
+
+define <4 x i8> @fptou_v4f64_v4i8(<4 x double> %a) {
+; CHECK-LABEL: fptou_v4f64_v4i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    xtn v1.2s, v1.2d
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    uzp1 v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <4 x double> %a to <4 x i8>
+  ret <4 x i8> %c
+}
+
+define <8 x i8> @fptos_v8f64_v8i8(<8 x double> %a) {
+; CHECK-LABEL: fptos_v8f64_v8i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    xtn v3.2s, v3.2d
+; CHECK-NEXT:    xtn v2.2s, v2.2d
+; CHECK-NEXT:    xtn v1.2s, v1.2d
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    uzp1 v2.4h, v2.4h, v3.4h
+; CHECK-NEXT:    uzp1 v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    uzp1 v0.8b, v0.8b, v2.8b
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <8 x double> %a to <8 x i8>
+  ret <8 x i8> %c
+}
+
+define <8 x i8> @fptou_v8f64_v8i8(<8 x double> %a) {
+; CHECK-LABEL: fptou_v8f64_v8i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    xtn v3.2s, v3.2d
+; CHECK-NEXT:    xtn v2.2s, v2.2d
+; CHECK-NEXT:    xtn v1.2s, v1.2d
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    uzp1 v2.4h, v2.4h, v3.4h
+; CHECK-NEXT:    uzp1 v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    uzp1 v0.8b, v0.8b, v2.8b
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <8 x double> %a to <8 x i8>
+  ret <8 x i8> %c
+}
+
+define <16 x i8> @fptos_v16f64_v16i8(<16 x double> %a) {
+; CHECK-LABEL: fptos_v16f64_v16i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-NEXT:    fcvtzs v5.2d, v5.2d
+; CHECK-NEXT:    fcvtzs v4.2d, v4.2d
+; CHECK-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    xtn v7.2s, v7.2d
+; CHECK-NEXT:    xtn v6.2s, v6.2d
+; CHECK-NEXT:    xtn v5.2s, v5.2d
+; CHECK-NEXT:    xtn v4.2s, v4.2d
+; CHECK-NEXT:    xtn v3.2s, v3.2d
+; CHECK-NEXT:    xtn v2.2s, v2.2d
+; CHECK-NEXT:    xtn v1.2s, v1.2d
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    uzp1 v6.4h, v6.4h, v7.4h
+; CHECK-NEXT:    uzp1 v4.4h, v4.4h, v5.4h
+; CHECK-NEXT:    uzp1 v2.4h, v2.4h, v3.4h
+; CHECK-NEXT:    uzp1 v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    mov v4.d[1], v6.d[0]
+; CHECK-NEXT:    mov v0.d[1], v2.d[0]
+; CHECK-NEXT:    uzp1 v0.16b, v0.16b, v4.16b
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <16 x double> %a to <16 x i8>
+  ret <16 x i8> %c
+}
+
+define <16 x i8> @fptou_v16f64_v16i8(<16 x double> %a) {
+; CHECK-LABEL: fptou_v16f64_v16i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-NEXT:    fcvtzs v5.2d, v5.2d
+; CHECK-NEXT:    fcvtzs v4.2d, v4.2d
+; CHECK-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    xtn v7.2s, v7.2d
+; CHECK-NEXT:    xtn v6.2s, v6.2d
+; CHECK-NEXT:    xtn v5.2s, v5.2d
+; CHECK-NEXT:    xtn v4.2s, v4.2d
+; CHECK-NEXT:    xtn v3.2s, v3.2d
+; CHECK-NEXT:    xtn v2.2s, v2.2d
+; CHECK-NEXT:    xtn v1.2s, v1.2d
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    uzp1 v6.4h, v6.4h, v7.4h
+; CHECK-NEXT:    uzp1 v4.4h, v4.4h, v5.4h
+; CHECK-NEXT:    uzp1 v2.4h, v2.4h, v3.4h
+; CHECK-NEXT:    uzp1 v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    mov v4.d[1], v6.d[0]
+; CHECK-NEXT:    mov v0.d[1], v2.d[0]
+; CHECK-NEXT:    uzp1 v0.16b, v0.16b, v4.16b
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <16 x double> %a to <16 x i8>
+  ret <16 x i8> %c
+}
+
+define <32 x i8> @fptos_v32f64_v32i8(<32 x double> %a) {
+; CHECK-LABEL: fptos_v32f64_v32i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldp q16, q17, [sp]
+; CHECK-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-NEXT:    ldp q18, q19, [sp, #32]
+; CHECK-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-NEXT:    ldp q20, q21, [sp, #64]
+; CHECK-NEXT:    fcvtzs v5.2d, v5.2d
+; CHECK-NEXT:    ldp q22, q23, [sp, #96]
+; CHECK-NEXT:    fcvtzs v4.2d, v4.2d
+; CHECK-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    fcvtzs v21.2d, v21.2d
+; CHECK-NEXT:    fcvtzs v20.2d, v20.2d
+; CHECK-NEXT:    fcvtzs v23.2d, v23.2d
+; CHECK-NEXT:    fcvtzs v22.2d, v22.2d
+; CHECK-NEXT:    fcvtzs v19.2d, v19.2d
+; CHECK-NEXT:    fcvtzs v18.2d, v18.2d
+; CHECK-NEXT:    fcvtzs v17.2d, v17.2d
+; CHECK-NEXT:    fcvtzs v16.2d, v16.2d
+; CHECK-NEXT:    xtn v7.2s, v7.2d
+; CHECK-NEXT:    xtn v6.2s, v6.2d
+; CHECK-NEXT:    xtn v5.2s, v5.2d
+; CHECK-NEXT:    xtn v4.2s, v4.2d
+; CHECK-NEXT:    xtn v3.2s, v3.2d
+; CHECK-NEXT:    xtn v2.2s, v2.2d
+; CHECK-NEXT:    xtn v1.2s, v1.2d
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    xtn v23.2s, v23.2d
+; CHECK-NEXT:    xtn v22.2s, v22.2d
+; CHECK-NEXT:    xtn v21.2s, v21.2d
+; CHECK-NEXT:    xtn v20.2s, v20.2d
+; CHECK-NEXT:    xtn v19.2s, v19.2d
+; CHECK-NEXT:    xtn v18.2s, v18.2d
+; CHECK-NEXT:    xtn v17.2s, v17.2d
+; CHECK-NEXT:    xtn v16.2s, v16.2d
+; CHECK-NEXT:    uzp1 v6.4h, v6.4h, v7.4h
+; CHECK-NEXT:    uzp1 v4.4h, v4.4h, v5.4h
+; CHECK-NEXT:    uzp1 v2.4h, v2.4h, v3.4h
+; CHECK-NEXT:    uzp1 v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    uzp1 v1.4h, v22.4h, v23.4h
+; CHECK-NEXT:    uzp1 v3.4h, v20.4h, v21.4h
+; CHECK-NEXT:    uzp1 v5.4h, v18.4h, v19.4h
+; CHECK-NEXT:    uzp1 v7.4h, v16.4h, v17.4h
+; CHECK-NEXT:    mov v4.d[1], v6.d[0]
+; CHECK-NEXT:    mov v0.d[1], v2.d[0]
+; CHECK-NEXT:    mov v3.d[1], v1.d[0]
+; CHECK-NEXT:    mov v7.d[1], v5.d[0]
+; CHECK-NEXT:    uzp1 v0.16b, v0.16b, v4.16b
+; CHECK-NEXT:    uzp1 v1.16b, v7.16b, v3.16b
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <32 x double> %a to <32 x i8>
+  ret <32 x i8> %c
+}
+
+define <32 x i8> @fptou_v32f64_v32i8(<32 x double> %a) {
+; CHECK-LABEL: fptou_v32f64_v32i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldp q16, q17, [sp]
+; CHECK-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-NEXT:    ldp q18, q19, [sp, #32]
+; CHECK-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-NEXT:    ldp q20, q21, [sp, #64]
+; CHECK-NEXT:    fcvtzs v5.2d, v5.2d
+; CHECK-NEXT:    ldp q22, q23, [sp, #96]
+; CHECK-NEXT:    fcvtzs v4.2d, v4.2d
+; CHECK-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    fcvtzs v21.2d, v21.2d
+; CHECK-NEXT:    fcvtzs v20.2d, v20.2d
+; CHECK-NEXT:    fcvtzs v23.2d, v23.2d
+; CHECK-NEXT:    fcvtzs v22.2d, v22.2d
+; CHECK-NEXT:    fcvtzs v19.2d, v19.2d
+; CHECK-NEXT:    fcvtzs v18.2d, v18.2d
+; CHECK-NEXT:    fcvtzs v17.2d, v17.2d
+; CHECK-NEXT:    fcvtzs v16.2d, v16.2d
+; CHECK-NEXT:    xtn v7.2s, v7.2d
+; CHECK-NEXT:    xtn v6.2s, v6.2d
+; CHECK-NEXT:    xtn v5.2s, v5.2d
+; CHECK-NEXT:    xtn v4.2s, v4.2d
+; CHECK-NEXT:    xtn v3.2s, v3.2d
+; CHECK-NEXT:    xtn v2.2s, v2.2d
+; CHECK-NEXT:    xtn v1.2s, v1.2d
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    xtn v23.2s, v23.2d
+; CHECK-NEXT:    xtn v22.2s, v22.2d
+; CHECK-NEXT:    xtn v21.2s, v21.2d
+; CHECK-NEXT:    xtn v20.2s, v20.2d
+; CHECK-NEXT:    xtn v19.2s, v19.2d
+; CHECK-NEXT:    xtn v18.2s, v18.2d
+; CHECK-NEXT:    xtn v17.2s, v17.2d
+; CHECK-NEXT:    xtn v16.2s, v16.2d
+; CHECK-NEXT:    uzp1 v6.4h, v6.4h, v7.4h
+; CHECK-NEXT:    uzp1 v4.4h, v4.4h, v5.4h
+; CHECK-NEXT:    uzp1 v2.4h, v2.4h, v3.4h
+; CHECK-NEXT:    uzp1 v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    uzp1 v1.4h, v22.4h, v23.4h
+; CHECK-NEXT:    uzp1 v3.4h, v20.4h, v21.4h
+; CHECK-NEXT:    uzp1 v5.4h, v18.4h, v19.4h
+; CHECK-NEXT:    uzp1 v7.4h, v16.4h, v17.4h
+; CHECK-NEXT:    mov v4.d[1], v6.d[0]
+; CHECK-NEXT:    mov v0.d[1], v2.d[0]
+; CHECK-NEXT:    mov v3.d[1], v1.d[0]
+; CHECK-NEXT:    mov v7.d[1], v5.d[0]
+; CHECK-NEXT:    uzp1 v0.16b, v0.16b, v4.16b
+; CHECK-NEXT:    uzp1 v1.16b, v7.16b, v3.16b
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <32 x double> %a to <32 x i8>
+  ret <32 x i8> %c
+}
+
+define <2 x i64> @fptos_v2f32_v2i64(<2 x float> %a) {
+; CHECK-LABEL: fptos_v2f32_v2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <2 x float> %a to <2 x i64>
+  ret <2 x i64> %c
+}
+
+define <2 x i64> @fptou_v2f32_v2i64(<2 x float> %a) {
+; CHECK-LABEL: fptou_v2f32_v2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <2 x float> %a to <2 x i64>
+  ret <2 x i64> %c
+}
+
+define <3 x i64> @fptos_v3f32_v3i64(<3 x float> %a) {
+; CHECK-LABEL: fptos_v3f32_v3i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtl v1.2d, v0.2s
+; CHECK-NEXT:    fcvtl2 v0.2d, v0.4s
+; CHECK-NEXT:    fcvtzs v3.2d, v1.2d
+; CHECK-NEXT:    fcvtzs v2.2d, v0.2d
+; CHECK-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-NEXT:    fmov d0, d3
+; CHECK-NEXT:    ext v1.16b, v3.16b, v3.16b, #8
+; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <3 x float> %a to <3 x i64>
+  ret <3 x i64> %c
+}
+
+define <3 x i64> @fptou_v3f32_v3i64(<3 x float> %a) {
+; CHECK-LABEL: fptou_v3f32_v3i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtl v1.2d, v0.2s
+; CHECK-NEXT:    fcvtl2 v0.2d, v0.4s
+; CHECK-NEXT:    fcvtzu v3.2d, v1.2d
+; CHECK-NEXT:    fcvtzu v2.2d, v0.2d
+; CHECK-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-NEXT:    fmov d0, d3
+; CHECK-NEXT:    ext v1.16b, v3.16b, v3.16b, #8
+; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <3 x float> %a to <3 x i64>
+  ret <3 x i64> %c
+}
+
+define <4 x i64> @fptos_v4f32_v4i64(<4 x float> %a) {
+; CHECK-LABEL: fptos_v4f32_v4i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtl2 v1.2d, v0.4s
+; CHECK-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <4 x float> %a to <4 x i64>
+  ret <4 x i64> %c
+}
+
+define <4 x i64> @fptou_v4f32_v4i64(<4 x float> %a) {
+; CHECK-LABEL: fptou_v4f32_v4i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtl2 v1.2d, v0.4s
+; CHECK-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <4 x float> %a to <4 x i64>
+  ret <4 x i64> %c
+}
+
+define <8 x i64> @fptos_v8f32_v8i64(<8 x float> %a) {
+; CHECK-LABEL: fptos_v8f32_v8i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtl v2.2d, v0.2s
+; CHECK-NEXT:    fcvtl2 v3.2d, v0.4s
+; CHECK-NEXT:    fcvtl2 v4.2d, v1.4s
+; CHECK-NEXT:    fcvtl v5.2d, v1.2s
+; CHECK-NEXT:    fcvtzs v0.2d, v2.2d
+; CHECK-NEXT:    fcvtzs v1.2d, v3.2d
+; CHECK-NEXT:    fcvtzs v3.2d, v4.2d
+; CHECK-NEXT:    fcvtzs v2.2d, v5.2d
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <8 x float> %a to <8 x i64>
+  ret <8 x i64> %c
+}
+
+define <8 x i64> @fptou_v8f32_v8i64(<8 x float> %a) {
+; CHECK-LABEL: fptou_v8f32_v8i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtl v2.2d, v0.2s
+; CHECK-NEXT:    fcvtl2 v3.2d, v0.4s
+; CHECK-NEXT:    fcvtl2 v4.2d, v1.4s
+; CHECK-NEXT:    fcvtl v5.2d, v1.2s
+; CHECK-NEXT:    fcvtzu v0.2d, v2.2d
+; CHECK-NEXT:    fcvtzu v1.2d, v3.2d
+; CHECK-NEXT:    fcvtzu v3.2d, v4.2d
+; CHECK-NEXT:    fcvtzu v2.2d, v5.2d
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <8 x float> %a to <8 x i64>
+  ret <8 x i64> %c
+}
+
+define <16 x i64> @fptos_v16f32_v16i64(<16 x float> %a) {
+; CHECK-LABEL: fptos_v16f32_v16i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtl2 v4.2d, v0.4s
+; CHECK-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-NEXT:    fcvtl2 v5.2d, v1.4s
+; CHECK-NEXT:    fcvtl v6.2d, v1.2s
+; CHECK-NEXT:    fcvtl v7.2d, v2.2s
+; CHECK-NEXT:    fcvtl2 v16.2d, v2.4s
+; CHECK-NEXT:    fcvtl2 v17.2d, v3.4s
+; CHECK-NEXT:    fcvtl v18.2d, v3.2s
+; CHECK-NEXT:    fcvtzs v1.2d, v4.2d
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    fcvtzs v3.2d, v5.2d
+; CHECK-NEXT:    fcvtzs v2.2d, v6.2d
+; CHECK-NEXT:    fcvtzs v4.2d, v7.2d
+; CHECK-NEXT:    fcvtzs v5.2d, v16.2d
+; CHECK-NEXT:    fcvtzs v7.2d, v17.2d
+; CHECK-NEXT:    fcvtzs v6.2d, v18.2d
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <16 x float> %a to <16 x i64>
+  ret <16 x i64> %c
+}
+
+define <16 x i64> @fptou_v16f32_v16i64(<16 x float> %a) {
+; CHECK-LABEL: fptou_v16f32_v16i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtl2 v4.2d, v0.4s
+; CHECK-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-NEXT:    fcvtl2 v5.2d, v1.4s
+; CHECK-NEXT:    fcvtl v6.2d, v1.2s
+; CHECK-NEXT:    fcvtl v7.2d, v2.2s
+; CHECK-NEXT:    fcvtl2 v16.2d, v2.4s
+; CHECK-NEXT:    fcvtl2 v17.2d, v3.4s
+; CHECK-NEXT:    fcvtl v18.2d, v3.2s
+; CHECK-NEXT:    fcvtzu v1.2d, v4.2d
+; CHECK-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-NEXT:    fcvtzu v3.2d, v5.2d
+; CHECK-NEXT:    fcvtzu v2.2d, v6.2d
+; CHECK-NEXT:    fcvtzu v4.2d, v7.2d
+; CHECK-NEXT:    fcvtzu v5.2d, v16.2d
+; CHECK-NEXT:    fcvtzu v7.2d, v17.2d
+; CHECK-NEXT:    fcvtzu v6.2d, v18.2d
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <16 x float> %a to <16 x i64>
+  ret <16 x i64> %c
+}
+
+define <32 x i64> @fptos_v32f32_v32i64(<32 x float> %a) {
+; CHECK-LABEL: fptos_v32f32_v32i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtl2 v16.2d, v7.4s
+; CHECK-NEXT:    fcvtl v7.2d, v7.2s
+; CHECK-NEXT:    fcvtl2 v17.2d, v6.4s
+; CHECK-NEXT:    fcvtl v6.2d, v6.2s
+; CHECK-NEXT:    fcvtl2 v18.2d, v5.4s
+; CHECK-NEXT:    fcvtl v5.2d, v5.2s
+; CHECK-NEXT:    fcvtl2 v19.2d, v4.4s
+; CHECK-NEXT:    fcvtl v4.2d, v4.2s
+; CHECK-NEXT:    fcvtl2 v20.2d, v3.4s
+; CHECK-NEXT:    fcvtl v3.2d, v3.2s
+; CHECK-NEXT:    fcvtzs v16.2d, v16.2d
+; CHECK-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-NEXT:    fcvtzs v17.2d, v17.2d
+; CHECK-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-NEXT:    fcvtzs v18.2d, v18.2d
+; CHECK-NEXT:    fcvtzs v5.2d, v5.2d
+; CHECK-NEXT:    fcvtzs v4.2d, v4.2d
+; CHECK-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NEXT:    stp q7, q16, [x8, #224]
+; CHECK-NEXT:    fcvtl2 v7.2d, v2.4s
+; CHECK-NEXT:    fcvtzs v16.2d, v19.2d
+; CHECK-NEXT:    stp q5, q18, [x8, #160]
+; CHECK-NEXT:    fcvtl v2.2d, v2.2s
+; CHECK-NEXT:    fcvtl2 v5.2d, v0.4s
+; CHECK-NEXT:    stp q6, q17, [x8, #192]
+; CHECK-NEXT:    fcvtl2 v6.2d, v1.4s
+; CHECK-NEXT:    fcvtzs v17.2d, v20.2d
+; CHECK-NEXT:    fcvtl v1.2d, v1.2s
+; CHECK-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-NEXT:    stp q4, q16, [x8, #128]
+; CHECK-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NEXT:    fcvtzs v4.2d, v6.2d
+; CHECK-NEXT:    stp q3, q17, [x8, #96]
+; CHECK-NEXT:    fcvtzs v3.2d, v5.2d
+; CHECK-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NEXT:    stp q2, q7, [x8, #64]
+; CHECK-NEXT:    stp q0, q3, [x8]
+; CHECK-NEXT:    stp q1, q4, [x8, #32]
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <32 x float> %a to <32 x i64>
+  ret <32 x i64> %c
+}
+
+define <32 x i64> @fptou_v32f32_v32i64(<32 x float> %a) {
+; CHECK-LABEL: fptou_v32f32_v32i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtl2 v16.2d, v7.4s
+; CHECK-NEXT:    fcvtl v7.2d, v7.2s
+; CHECK-NEXT:    fcvtl2 v17.2d, v6.4s
+; CHECK-NEXT:    fcvtl v6.2d, v6.2s
+; CHECK-NEXT:    fcvtl2 v18.2d, v5.4s
+; CHECK-NEXT:    fcvtl v5.2d, v5.2s
+; CHECK-NEXT:    fcvtl2 v19.2d, v4.4s
+; CHECK-NEXT:    fcvtl v4.2d, v4.2s
+; CHECK-NEXT:    fcvtl2 v20.2d, v3.4s
+; CHECK-NEXT:    fcvtl v3.2d, v3.2s
+; CHECK-NEXT:    fcvtzu v16.2d, v16.2d
+; CHECK-NEXT:    fcvtzu v7.2d, v7.2d
+; CHECK-NEXT:    fcvtzu v17.2d, v17.2d
+; CHECK-NEXT:    fcvtzu v6.2d, v6.2d
+; CHECK-NEXT:    fcvtzu v18.2d, v18.2d
+; CHECK-NEXT:    fcvtzu v5.2d, v5.2d
+; CHECK-NEXT:    fcvtzu v4.2d, v4.2d
+; CHECK-NEXT:    fcvtzu v3.2d, v3.2d
+; CHECK-NEXT:    stp q7, q16, [x8, #224]
+; CHECK-NEXT:    fcvtl2 v7.2d, v2.4s
+; CHECK-NEXT:    fcvtzu v16.2d, v19.2d
+; CHECK-NEXT:    stp q5, q18, [x8, #160]
+; CHECK-NEXT:    fcvtl v2.2d, v2.2s
+; CHECK-NEXT:    fcvtl2 v5.2d, v0.4s
+; CHECK-NEXT:    stp q6, q17, [x8, #192]
+; CHECK-NEXT:    fcvtl2 v6.2d, v1.4s
+; CHECK-NEXT:    fcvtzu v17.2d, v20.2d
+; CHECK-NEXT:    fcvtl v1.2d, v1.2s
+; CHECK-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-NEXT:    stp q4, q16, [x8, #128]
+; CHECK-NEXT:    fcvtzu v7.2d, v7.2d
+; CHECK-NEXT:    fcvtzu v2.2d, v2.2d
+; CHECK-NEXT:    fcvtzu v4.2d, v6.2d
+; CHECK-NEXT:    stp q3, q17, [x8, #96]
+; CHECK-NEXT:    fcvtzu v3.2d, v5.2d
+; CHECK-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-NEXT:    stp q2, q7, [x8, #64]
+; CHECK-NEXT:    stp q0, q3, [x8]
+; CHECK-NEXT:    stp q1, q4, [x8, #32]
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <32 x float> %a to <32 x i64>
+  ret <32 x i64> %c
+}
+
+define <2 x i32> @fptos_v2f32_v2i32(<2 x float> %a) {
+; CHECK-LABEL: fptos_v2f32_v2i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v0.2s, v0.2s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <2 x float> %a to <2 x i32>
+  ret <2 x i32> %c
+}
+
+define <2 x i32> @fptou_v2f32_v2i32(<2 x float> %a) {
+; CHECK-LABEL: fptou_v2f32_v2i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu v0.2s, v0.2s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <2 x float> %a to <2 x i32>
+  ret <2 x i32> %c
+}
+
+define <3 x i32> @fptos_v3f32_v3i32(<3 x float> %a) {
+; CHECK-LABEL: fptos_v3f32_v3i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <3 x float> %a to <3 x i32>
+  ret <3 x i32> %c
+}
+
+define <3 x i32> @fptou_v3f32_v3i32(<3 x float> %a) {
+; CHECK-LABEL: fptou_v3f32_v3i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <3 x float> %a to <3 x i32>
+  ret <3 x i32> %c
+}
+
+define <4 x i32> @fptos_v4f32_v4i32(<4 x float> %a) {
+; CHECK-LABEL: fptos_v4f32_v4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <4 x float> %a to <4 x i32>
+  ret <4 x i32> %c
+}
+
+define <4 x i32> @fptou_v4f32_v4i32(<4 x float> %a) {
+; CHECK-LABEL: fptou_v4f32_v4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <4 x float> %a to <4 x i32>
+  ret <4 x i32> %c
+}
+
+define <8 x i32> @fptos_v8f32_v8i32(<8 x float> %a) {
+; CHECK-LABEL: fptos_v8f32_v8i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <8 x float> %a to <8 x i32>
+  ret <8 x i32> %c
+}
+
+define <8 x i32> @fptou_v8f32_v8i32(<8 x float> %a) {
+; CHECK-LABEL: fptou_v8f32_v8i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <8 x float> %a to <8 x i32>
+  ret <8 x i32> %c
+}
+
+define <16 x i32> @fptos_v16f32_v16i32(<16 x float> %a) {
+; CHECK-LABEL: fptos_v16f32_v16i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <16 x float> %a to <16 x i32>
+  ret <16 x i32> %c
+}
+
+define <16 x i32> @fptou_v16f32_v16i32(<16 x float> %a) {
+; CHECK-LABEL: fptou_v16f32_v16i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-NEXT:    fcvtzu v2.4s, v2.4s
+; CHECK-NEXT:    fcvtzu v3.4s, v3.4s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <16 x float> %a to <16 x i32>
+  ret <16 x i32> %c
+}
+
+define <32 x i32> @fptos_v32f32_v32i32(<32 x float> %a) {
+; CHECK-LABEL: fptos_v32f32_v32i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-NEXT:    fcvtzs v4.4s, v4.4s
+; CHECK-NEXT:    fcvtzs v5.4s, v5.4s
+; CHECK-NEXT:    fcvtzs v6.4s, v6.4s
+; CHECK-NEXT:    fcvtzs v7.4s, v7.4s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <32 x float> %a to <32 x i32>
+  ret <32 x i32> %c
+}
+
+define <32 x i32> @fptou_v32f32_v32i32(<32 x float> %a) {
+; CHECK-LABEL: fptou_v32f32_v32i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-NEXT:    fcvtzu v2.4s, v2.4s
+; CHECK-NEXT:    fcvtzu v3.4s, v3.4s
+; CHECK-NEXT:    fcvtzu v4.4s, v4.4s
+; CHECK-NEXT:    fcvtzu v5.4s, v5.4s
+; CHECK-NEXT:    fcvtzu v6.4s, v6.4s
+; CHECK-NEXT:    fcvtzu v7.4s, v7.4s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <32 x float> %a to <32 x i32>
+  ret <32 x i32> %c
+}
+
+define <2 x i16> @fptos_v2f32_v2i16(<2 x float> %a) {
+; CHECK-LABEL: fptos_v2f32_v2i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v0.2s, v0.2s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <2 x float> %a to <2 x i16>
+  ret <2 x i16> %c
+}
+
+define <2 x i16> @fptou_v2f32_v2i16(<2 x float> %a) {
+; CHECK-LABEL: fptou_v2f32_v2i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v0.2s, v0.2s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <2 x float> %a to <2 x i16>
+  ret <2 x i16> %c
+}
+
+define <3 x i16> @fptos_v3f32_v3i16(<3 x float> %a) {
+; CHECK-LABEL: fptos_v3f32_v3i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <3 x float> %a to <3 x i16>
+  ret <3 x i16> %c
+}
+
+define <3 x i16> @fptou_v3f32_v3i16(<3 x float> %a) {
+; CHECK-LABEL: fptou_v3f32_v3i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <3 x float> %a to <3 x i16>
+  ret <3 x i16> %c
+}
+
+define <4 x i16> @fptos_v4f32_v4i16(<4 x float> %a) {
+; CHECK-LABEL: fptos_v4f32_v4i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <4 x float> %a to <4 x i16>
+  ret <4 x i16> %c
+}
+
+define <4 x i16> @fptou_v4f32_v4i16(<4 x float> %a) {
+; CHECK-LABEL: fptou_v4f32_v4i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <4 x float> %a to <4 x i16>
+  ret <4 x i16> %c
+}
+
+define <8 x i16> @fptos_v8f32_v8i16(<8 x float> %a) {
+; CHECK-LABEL: fptos_v8f32_v8i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <8 x float> %a to <8 x i16>
+  ret <8 x i16> %c
+}
+
+define <8 x i16> @fptou_v8f32_v8i16(<8 x float> %a) {
+; CHECK-LABEL: fptou_v8f32_v8i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <8 x float> %a to <8 x i16>
+  ret <8 x i16> %c
+}
+
+define <16 x i16> @fptos_v16f32_v16i16(<16 x float> %a) {
+; CHECK-LABEL: fptos_v16f32_v16i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <16 x float> %a to <16 x i16>
+  ret <16 x i16> %c
+}
+
+define <16 x i16> @fptou_v16f32_v16i16(<16 x float> %a) {
+; CHECK-LABEL: fptou_v16f32_v16i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NEXT:    fcvtzu v3.4s, v3.4s
+; CHECK-NEXT:    fcvtzu v2.4s, v2.4s
+; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <16 x float> %a to <16 x i16>
+  ret <16 x i16> %c
+}
+
+define <32 x i16> @fptos_v32f32_v32i16(<32 x float> %a) {
+; CHECK-LABEL: fptos_v32f32_v32i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-NEXT:    fcvtzs v5.4s, v5.4s
+; CHECK-NEXT:    fcvtzs v4.4s, v4.4s
+; CHECK-NEXT:    fcvtzs v7.4s, v7.4s
+; CHECK-NEXT:    fcvtzs v6.4s, v6.4s
+; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
+; CHECK-NEXT:    uzp1 v2.8h, v4.8h, v5.8h
+; CHECK-NEXT:    uzp1 v3.8h, v6.8h, v7.8h
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <32 x float> %a to <32 x i16>
+  ret <32 x i16> %c
+}
+
+define <32 x i16> @fptou_v32f32_v32i16(<32 x float> %a) {
+; CHECK-LABEL: fptou_v32f32_v32i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu v3.4s, v3.4s
+; CHECK-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NEXT:    fcvtzu v2.4s, v2.4s
+; CHECK-NEXT:    fcvtzu v5.4s, v5.4s
+; CHECK-NEXT:    fcvtzu v4.4s, v4.4s
+; CHECK-NEXT:    fcvtzu v7.4s, v7.4s
+; CHECK-NEXT:    fcvtzu v6.4s, v6.4s
+; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
+; CHECK-NEXT:    uzp1 v2.8h, v4.8h, v5.8h
+; CHECK-NEXT:    uzp1 v3.8h, v6.8h, v7.8h
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <32 x float> %a to <32 x i16>
+  ret <32 x i16> %c
+}
+
+define <2 x i8> @fptos_v2f32_v2i8(<2 x float> %a) {
+; CHECK-LABEL: fptos_v2f32_v2i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v0.2s, v0.2s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <2 x float> %a to <2 x i8>
+  ret <2 x i8> %c
+}
+
+define <2 x i8> @fptou_v2f32_v2i8(<2 x float> %a) {
+; CHECK-LABEL: fptou_v2f32_v2i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v0.2s, v0.2s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <2 x float> %a to <2 x i8>
+  ret <2 x i8> %c
+}
+
+define <3 x i8> @fptos_v3f32_v3i8(<3 x float> %a) {
+; CHECK-LABEL: fptos_v3f32_v3i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    umov w0, v0.h[0]
+; CHECK-NEXT:    umov w1, v0.h[1]
+; CHECK-NEXT:    umov w2, v0.h[2]
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <3 x float> %a to <3 x i8>
+  ret <3 x i8> %c
+}
+
+define <3 x i8> @fptou_v3f32_v3i8(<3 x float> %a) {
+; CHECK-LABEL: fptou_v3f32_v3i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    umov w0, v0.h[0]
+; CHECK-NEXT:    umov w1, v0.h[1]
+; CHECK-NEXT:    umov w2, v0.h[2]
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <3 x float> %a to <3 x i8>
+  ret <3 x i8> %c
+}
+
+define <4 x i8> @fptos_v4f32_v4i8(<4 x float> %a) {
+; CHECK-LABEL: fptos_v4f32_v4i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <4 x float> %a to <4 x i8>
+  ret <4 x i8> %c
+}
+
+define <4 x i8> @fptou_v4f32_v4i8(<4 x float> %a) {
+; CHECK-LABEL: fptou_v4f32_v4i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <4 x float> %a to <4 x i8>
+  ret <4 x i8> %c
+}
+
+define <8 x i8> @fptos_v8f32_v8i8(<8 x float> %a) {
+; CHECK-LABEL: fptos_v8f32_v8i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NEXT:    xtn v1.4h, v1.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    uzp1 v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <8 x float> %a to <8 x i8>
+  ret <8 x i8> %c
+}
+
+define <8 x i8> @fptou_v8f32_v8i8(<8 x float> %a) {
+; CHECK-LABEL: fptou_v8f32_v8i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NEXT:    xtn v1.4h, v1.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    uzp1 v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <8 x float> %a to <8 x i8>
+  ret <8 x i8> %c
+}
+
+define <16 x i8> @fptos_v16f32_v16i8(<16 x float> %a) {
+; CHECK-LABEL: fptos_v16f32_v16i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NEXT:    xtn v3.4h, v3.4s
+; CHECK-NEXT:    xtn v2.4h, v2.4s
+; CHECK-NEXT:    xtn v1.4h, v1.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    mov v2.d[1], v3.d[0]
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    uzp1 v0.16b, v0.16b, v2.16b
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <16 x float> %a to <16 x i8>
+  ret <16 x i8> %c
+}
+
+define <16 x i8> @fptou_v16f32_v16i8(<16 x float> %a) {
+; CHECK-LABEL: fptou_v16f32_v16i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NEXT:    xtn v3.4h, v3.4s
+; CHECK-NEXT:    xtn v2.4h, v2.4s
+; CHECK-NEXT:    xtn v1.4h, v1.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    mov v2.d[1], v3.d[0]
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    uzp1 v0.16b, v0.16b, v2.16b
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <16 x float> %a to <16 x i8>
+  ret <16 x i8> %c
+}
+
+define <32 x i8> @fptos_v32f32_v32i8(<32 x float> %a) {
+; CHECK-LABEL: fptos_v32f32_v32i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NEXT:    fcvtzs v7.4s, v7.4s
+; CHECK-NEXT:    fcvtzs v6.4s, v6.4s
+; CHECK-NEXT:    fcvtzs v5.4s, v5.4s
+; CHECK-NEXT:    fcvtzs v4.4s, v4.4s
+; CHECK-NEXT:    xtn v3.4h, v3.4s
+; CHECK-NEXT:    xtn v2.4h, v2.4s
+; CHECK-NEXT:    xtn v1.4h, v1.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    xtn v7.4h, v7.4s
+; CHECK-NEXT:    xtn v6.4h, v6.4s
+; CHECK-NEXT:    xtn v5.4h, v5.4s
+; CHECK-NEXT:    xtn v4.4h, v4.4s
+; CHECK-NEXT:    mov v2.d[1], v3.d[0]
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    mov v6.d[1], v7.d[0]
+; CHECK-NEXT:    mov v4.d[1], v5.d[0]
+; CHECK-NEXT:    uzp1 v0.16b, v0.16b, v2.16b
+; CHECK-NEXT:    uzp1 v1.16b, v4.16b, v6.16b
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <32 x float> %a to <32 x i8>
+  ret <32 x i8> %c
+}
+
+define <32 x i8> @fptou_v32f32_v32i8(<32 x float> %a) {
+; CHECK-LABEL: fptou_v32f32_v32i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NEXT:    fcvtzs v7.4s, v7.4s
+; CHECK-NEXT:    fcvtzs v6.4s, v6.4s
+; CHECK-NEXT:    fcvtzs v5.4s, v5.4s
+; CHECK-NEXT:    fcvtzs v4.4s, v4.4s
+; CHECK-NEXT:    xtn v3.4h, v3.4s
+; CHECK-NEXT:    xtn v2.4h, v2.4s
+; CHECK-NEXT:    xtn v1.4h, v1.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    xtn v7.4h, v7.4s
+; CHECK-NEXT:    xtn v6.4h, v6.4s
+; CHECK-NEXT:    xtn v5.4h, v5.4s
+; CHECK-NEXT:    xtn v4.4h, v4.4s
+; CHECK-NEXT:    mov v2.d[1], v3.d[0]
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    mov v6.d[1], v7.d[0]
+; CHECK-NEXT:    mov v4.d[1], v5.d[0]
+; CHECK-NEXT:    uzp1 v0.16b, v0.16b, v2.16b
+; CHECK-NEXT:    uzp1 v1.16b, v4.16b, v6.16b
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <32 x float> %a to <32 x i8>
+  ret <32 x i8> %c
+}
+
+define <2 x i64> @fptos_v2f16_v2i64(<2 x half> %a) {
+; CHECK-SD-NOFP16-LABEL: fptos_v2f16_v2i64:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x8, s0
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x9, s1
+; CHECK-SD-NOFP16-NEXT:    fmov d0, x8
+; CHECK-SD-NOFP16-NEXT:    mov v0.d[1], x9
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptos_v2f16_v2i64:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-FP16-NEXT:    mov h1, v0.h[1]
+; CHECK-SD-FP16-NEXT:    fcvtzs x8, h0
+; CHECK-SD-FP16-NEXT:    fcvtzs x9, h1
+; CHECK-SD-FP16-NEXT:    fmov d0, x8
+; CHECK-SD-FP16-NEXT:    mov v0.d[1], x9
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fptos_v2f16_v2i64:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NOFP16-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-GI-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x8, s0
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x9, s1
+; CHECK-GI-NOFP16-NEXT:    fmov d0, x8
+; CHECK-GI-NOFP16-NEXT:    mov v0.d[1], x9
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fptos_v2f16_v2i64:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-FP16-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-FP16-NEXT:    fcvtzs x8, h0
+; CHECK-GI-FP16-NEXT:    fcvtzs x9, h1
+; CHECK-GI-FP16-NEXT:    fmov d0, x8
+; CHECK-GI-FP16-NEXT:    mov v0.d[1], x9
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %c = fptosi <2 x half> %a to <2 x i64>
+  ret <2 x i64> %c
+}
+
+define <2 x i64> @fptou_v2f16_v2i64(<2 x half> %a) {
+; CHECK-SD-NOFP16-LABEL: fptou_v2f16_v2i64:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x8, s0
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x9, s1
+; CHECK-SD-NOFP16-NEXT:    fmov d0, x8
+; CHECK-SD-NOFP16-NEXT:    mov v0.d[1], x9
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptou_v2f16_v2i64:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-FP16-NEXT:    mov h1, v0.h[1]
+; CHECK-SD-FP16-NEXT:    fcvtzu x8, h0
+; CHECK-SD-FP16-NEXT:    fcvtzu x9, h1
+; CHECK-SD-FP16-NEXT:    fmov d0, x8
+; CHECK-SD-FP16-NEXT:    mov v0.d[1], x9
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fptou_v2f16_v2i64:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NOFP16-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-GI-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x8, s0
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x9, s1
+; CHECK-GI-NOFP16-NEXT:    fmov d0, x8
+; CHECK-GI-NOFP16-NEXT:    mov v0.d[1], x9
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fptou_v2f16_v2i64:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-FP16-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-FP16-NEXT:    fcvtzu x8, h0
+; CHECK-GI-FP16-NEXT:    fcvtzu x9, h1
+; CHECK-GI-FP16-NEXT:    fmov d0, x8
+; CHECK-GI-FP16-NEXT:    mov v0.d[1], x9
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %c = fptoui <2 x half> %a to <2 x i64>
+  ret <2 x i64> %c
+}
+
+define <3 x i64> @fptos_v3f16_v3i64(<3 x half> %a) {
+; CHECK-SD-NOFP16-LABEL: fptos_v3f16_v3i64:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[2]
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x8, s0
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x9, s1
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x10, s2
+; CHECK-SD-NOFP16-NEXT:    fmov d0, x8
+; CHECK-SD-NOFP16-NEXT:    fmov d1, x9
+; CHECK-SD-NOFP16-NEXT:    fmov d2, x10
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptos_v3f16_v3i64:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-FP16-NEXT:    mov h1, v0.h[1]
+; CHECK-SD-FP16-NEXT:    mov h2, v0.h[2]
+; CHECK-SD-FP16-NEXT:    fcvtzs x8, h0
+; CHECK-SD-FP16-NEXT:    fcvtzs x9, h1
+; CHECK-SD-FP16-NEXT:    fcvtzs x10, h2
+; CHECK-SD-FP16-NEXT:    fmov d0, x8
+; CHECK-SD-FP16-NEXT:    fmov d1, x9
+; CHECK-SD-FP16-NEXT:    fmov d2, x10
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fptos_v3f16_v3i64:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NOFP16-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-NOFP16-NEXT:    mov h2, v0.h[2]
+; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-GI-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-GI-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x8, s0
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x9, s1
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x10, s2
+; CHECK-GI-NOFP16-NEXT:    fmov d0, x8
+; CHECK-GI-NOFP16-NEXT:    fmov d1, x9
+; CHECK-GI-NOFP16-NEXT:    fmov d2, x10
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fptos_v3f16_v3i64:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-FP16-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-FP16-NEXT:    mov h2, v0.h[2]
+; CHECK-GI-FP16-NEXT:    fcvtzs x8, h0
+; CHECK-GI-FP16-NEXT:    fcvtzs x9, h1
+; CHECK-GI-FP16-NEXT:    fcvtzs x10, h2
+; CHECK-GI-FP16-NEXT:    fmov d0, x8
+; CHECK-GI-FP16-NEXT:    fmov d1, x9
+; CHECK-GI-FP16-NEXT:    fmov d2, x10
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %c = fptosi <3 x half> %a to <3 x i64>
+  ret <3 x i64> %c
+}
+
+define <3 x i64> @fptou_v3f16_v3i64(<3 x half> %a) {
+; CHECK-SD-NOFP16-LABEL: fptou_v3f16_v3i64:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[2]
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x8, s0
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x9, s1
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x10, s2
+; CHECK-SD-NOFP16-NEXT:    fmov d0, x8
+; CHECK-SD-NOFP16-NEXT:    fmov d1, x9
+; CHECK-SD-NOFP16-NEXT:    fmov d2, x10
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptou_v3f16_v3i64:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-FP16-NEXT:    mov h1, v0.h[1]
+; CHECK-SD-FP16-NEXT:    mov h2, v0.h[2]
+; CHECK-SD-FP16-NEXT:    fcvtzu x8, h0
+; CHECK-SD-FP16-NEXT:    fcvtzu x9, h1
+; CHECK-SD-FP16-NEXT:    fcvtzu x10, h2
+; CHECK-SD-FP16-NEXT:    fmov d0, x8
+; CHECK-SD-FP16-NEXT:    fmov d1, x9
+; CHECK-SD-FP16-NEXT:    fmov d2, x10
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fptou_v3f16_v3i64:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NOFP16-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-NOFP16-NEXT:    mov h2, v0.h[2]
+; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-GI-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-GI-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x8, s0
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x9, s1
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x10, s2
+; CHECK-GI-NOFP16-NEXT:    fmov d0, x8
+; CHECK-GI-NOFP16-NEXT:    fmov d1, x9
+; CHECK-GI-NOFP16-NEXT:    fmov d2, x10
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fptou_v3f16_v3i64:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-FP16-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-FP16-NEXT:    mov h2, v0.h[2]
+; CHECK-GI-FP16-NEXT:    fcvtzu x8, h0
+; CHECK-GI-FP16-NEXT:    fcvtzu x9, h1
+; CHECK-GI-FP16-NEXT:    fcvtzu x10, h2
+; CHECK-GI-FP16-NEXT:    fmov d0, x8
+; CHECK-GI-FP16-NEXT:    fmov d1, x9
+; CHECK-GI-FP16-NEXT:    fmov d2, x10
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %c = fptoui <3 x half> %a to <3 x i64>
+  ret <3 x i64> %c
+}
+
+define <4 x i64> @fptos_v4f16_v4i64(<4 x half> %a) {
+; CHECK-SD-NOFP16-LABEL: fptos_v4f16_v4i64:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x8, s0
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x9, s1
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x10, s2
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x11, s3
+; CHECK-SD-NOFP16-NEXT:    fmov d0, x8
+; CHECK-SD-NOFP16-NEXT:    fmov d1, x9
+; CHECK-SD-NOFP16-NEXT:    mov v0.d[1], x10
+; CHECK-SD-NOFP16-NEXT:    mov v1.d[1], x11
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptos_v4f16_v4i64:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-FP16-NEXT:    mov h1, v0.h[2]
+; CHECK-SD-FP16-NEXT:    mov h2, v0.h[1]
+; CHECK-SD-FP16-NEXT:    mov h3, v0.h[3]
+; CHECK-SD-FP16-NEXT:    fcvtzs x8, h0
+; CHECK-SD-FP16-NEXT:    fcvtzs x9, h1
+; CHECK-SD-FP16-NEXT:    fcvtzs x10, h2
+; CHECK-SD-FP16-NEXT:    fcvtzs x11, h3
+; CHECK-SD-FP16-NEXT:    fmov d0, x8
+; CHECK-SD-FP16-NEXT:    fmov d1, x9
+; CHECK-SD-FP16-NEXT:    mov v0.d[1], x10
+; CHECK-SD-FP16-NEXT:    mov v1.d[1], x11
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fptos_v4f16_v4i64:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NOFP16-NEXT:    mov h1, v0.h[2]
+; CHECK-GI-NOFP16-NEXT:    mov h2, v0.h[1]
+; CHECK-GI-NOFP16-NEXT:    mov h3, v0.h[3]
+; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-GI-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-GI-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-GI-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x8, s0
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x9, s1
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x10, s2
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x11, s3
+; CHECK-GI-NOFP16-NEXT:    fmov d0, x8
+; CHECK-GI-NOFP16-NEXT:    fmov d1, x9
+; CHECK-GI-NOFP16-NEXT:    mov v0.d[1], x10
+; CHECK-GI-NOFP16-NEXT:    mov v1.d[1], x11
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fptos_v4f16_v4i64:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-FP16-NEXT:    mov h1, v0.h[2]
+; CHECK-GI-FP16-NEXT:    mov h2, v0.h[1]
+; CHECK-GI-FP16-NEXT:    mov h3, v0.h[3]
+; CHECK-GI-FP16-NEXT:    fcvtzs x8, h0
+; CHECK-GI-FP16-NEXT:    fcvtzs x9, h1
+; CHECK-GI-FP16-NEXT:    fcvtzs x10, h2
+; CHECK-GI-FP16-NEXT:    fcvtzs x11, h3
+; CHECK-GI-FP16-NEXT:    fmov d0, x8
+; CHECK-GI-FP16-NEXT:    fmov d1, x9
+; CHECK-GI-FP16-NEXT:    mov v0.d[1], x10
+; CHECK-GI-FP16-NEXT:    mov v1.d[1], x11
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %c = fptosi <4 x half> %a to <4 x i64>
+  ret <4 x i64> %c
+}
+
+define <4 x i64> @fptou_v4f16_v4i64(<4 x half> %a) {
+; CHECK-SD-NOFP16-LABEL: fptou_v4f16_v4i64:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x8, s0
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x9, s1
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x10, s2
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x11, s3
+; CHECK-SD-NOFP16-NEXT:    fmov d0, x8
+; CHECK-SD-NOFP16-NEXT:    fmov d1, x9
+; CHECK-SD-NOFP16-NEXT:    mov v0.d[1], x10
+; CHECK-SD-NOFP16-NEXT:    mov v1.d[1], x11
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptou_v4f16_v4i64:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-FP16-NEXT:    mov h1, v0.h[2]
+; CHECK-SD-FP16-NEXT:    mov h2, v0.h[1]
+; CHECK-SD-FP16-NEXT:    mov h3, v0.h[3]
+; CHECK-SD-FP16-NEXT:    fcvtzu x8, h0
+; CHECK-SD-FP16-NEXT:    fcvtzu x9, h1
+; CHECK-SD-FP16-NEXT:    fcvtzu x10, h2
+; CHECK-SD-FP16-NEXT:    fcvtzu x11, h3
+; CHECK-SD-FP16-NEXT:    fmov d0, x8
+; CHECK-SD-FP16-NEXT:    fmov d1, x9
+; CHECK-SD-FP16-NEXT:    mov v0.d[1], x10
+; CHECK-SD-FP16-NEXT:    mov v1.d[1], x11
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fptou_v4f16_v4i64:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NOFP16-NEXT:    mov h1, v0.h[2]
+; CHECK-GI-NOFP16-NEXT:    mov h2, v0.h[1]
+; CHECK-GI-NOFP16-NEXT:    mov h3, v0.h[3]
+; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-GI-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-GI-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-GI-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x8, s0
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x9, s1
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x10, s2
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x11, s3
+; CHECK-GI-NOFP16-NEXT:    fmov d0, x8
+; CHECK-GI-NOFP16-NEXT:    fmov d1, x9
+; CHECK-GI-NOFP16-NEXT:    mov v0.d[1], x10
+; CHECK-GI-NOFP16-NEXT:    mov v1.d[1], x11
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fptou_v4f16_v4i64:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-FP16-NEXT:    mov h1, v0.h[2]
+; CHECK-GI-FP16-NEXT:    mov h2, v0.h[1]
+; CHECK-GI-FP16-NEXT:    mov h3, v0.h[3]
+; CHECK-GI-FP16-NEXT:    fcvtzu x8, h0
+; CHECK-GI-FP16-NEXT:    fcvtzu x9, h1
+; CHECK-GI-FP16-NEXT:    fcvtzu x10, h2
+; CHECK-GI-FP16-NEXT:    fcvtzu x11, h3
+; CHECK-GI-FP16-NEXT:    fmov d0, x8
+; CHECK-GI-FP16-NEXT:    fmov d1, x9
+; CHECK-GI-FP16-NEXT:    mov v0.d[1], x10
+; CHECK-GI-FP16-NEXT:    mov v1.d[1], x11
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %c = fptoui <4 x half> %a to <4 x i64>
+  ret <4 x i64> %c
+}
+
+define <8 x i64> @fptos_v8f16_v8i64(<8 x half> %a) {
+; CHECK-SD-NOFP16-LABEL: fptos_v8f16_v8i64:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x9, s0
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x8, s1
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x12, s4
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x11, s3
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x15, s7
+; CHECK-SD-NOFP16-NEXT:    fmov d0, x9
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x10, s2
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x13, s5
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x14, s6
+; CHECK-SD-NOFP16-NEXT:    fmov d2, x8
+; CHECK-SD-NOFP16-NEXT:    fmov d1, x12
+; CHECK-SD-NOFP16-NEXT:    mov v0.d[1], x11
+; CHECK-SD-NOFP16-NEXT:    fmov d3, x10
+; CHECK-SD-NOFP16-NEXT:    mov v2.d[1], x13
+; CHECK-SD-NOFP16-NEXT:    mov v1.d[1], x15
+; CHECK-SD-NOFP16-NEXT:    mov v3.d[1], x14
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptos_v8f16_v8i64:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-FP16-NEXT:    mov h4, v0.h[2]
+; CHECK-SD-FP16-NEXT:    mov h3, v0.h[1]
+; CHECK-SD-FP16-NEXT:    mov h7, v0.h[3]
+; CHECK-SD-FP16-NEXT:    fcvtzs x9, h0
+; CHECK-SD-FP16-NEXT:    mov h2, v1.h[2]
+; CHECK-SD-FP16-NEXT:    mov h5, v1.h[1]
+; CHECK-SD-FP16-NEXT:    mov h6, v1.h[3]
+; CHECK-SD-FP16-NEXT:    fcvtzs x8, h1
+; CHECK-SD-FP16-NEXT:    fcvtzs x12, h4
+; CHECK-SD-FP16-NEXT:    fcvtzs x11, h3
+; CHECK-SD-FP16-NEXT:    fcvtzs x15, h7
+; CHECK-SD-FP16-NEXT:    fmov d0, x9
+; CHECK-SD-FP16-NEXT:    fcvtzs x10, h2
+; CHECK-SD-FP16-NEXT:    fcvtzs x13, h5
+; CHECK-SD-FP16-NEXT:    fcvtzs x14, h6
+; CHECK-SD-FP16-NEXT:    fmov d2, x8
+; CHECK-SD-FP16-NEXT:    fmov d1, x12
+; CHECK-SD-FP16-NEXT:    mov v0.d[1], x11
+; CHECK-SD-FP16-NEXT:    fmov d3, x10
+; CHECK-SD-FP16-NEXT:    mov v2.d[1], x13
+; CHECK-SD-FP16-NEXT:    mov v1.d[1], x15
+; CHECK-SD-FP16-NEXT:    mov v3.d[1], x14
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fptos_v8f16_v8i64:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-GI-NOFP16-NEXT:    mov h4, v0.h[2]
+; CHECK-GI-NOFP16-NEXT:    mov h3, v0.h[1]
+; CHECK-GI-NOFP16-NEXT:    mov h7, v0.h[3]
+; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-GI-NOFP16-NEXT:    mov h2, v1.h[2]
+; CHECK-GI-NOFP16-NEXT:    mov h5, v1.h[1]
+; CHECK-GI-NOFP16-NEXT:    mov h6, v1.h[3]
+; CHECK-GI-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-GI-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-GI-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-GI-NOFP16-NEXT:    fcvt s7, h7
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x9, s0
+; CHECK-GI-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-GI-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-GI-NOFP16-NEXT:    fcvt s6, h6
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x8, s1
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x12, s4
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x11, s3
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x15, s7
+; CHECK-GI-NOFP16-NEXT:    fmov d0, x9
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x10, s2
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x13, s5
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x14, s6
+; CHECK-GI-NOFP16-NEXT:    fmov d2, x8
+; CHECK-GI-NOFP16-NEXT:    fmov d1, x12
+; CHECK-GI-NOFP16-NEXT:    mov v0.d[1], x11
+; CHECK-GI-NOFP16-NEXT:    fmov d3, x10
+; CHECK-GI-NOFP16-NEXT:    mov v2.d[1], x13
+; CHECK-GI-NOFP16-NEXT:    mov v1.d[1], x15
+; CHECK-GI-NOFP16-NEXT:    mov v3.d[1], x14
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fptos_v8f16_v8i64:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-GI-FP16-NEXT:    mov h4, v0.h[2]
+; CHECK-GI-FP16-NEXT:    mov h3, v0.h[1]
+; CHECK-GI-FP16-NEXT:    mov h7, v0.h[3]
+; CHECK-GI-FP16-NEXT:    fcvtzs x9, h0
+; CHECK-GI-FP16-NEXT:    mov h2, v1.h[2]
+; CHECK-GI-FP16-NEXT:    mov h5, v1.h[1]
+; CHECK-GI-FP16-NEXT:    mov h6, v1.h[3]
+; CHECK-GI-FP16-NEXT:    fcvtzs x8, h1
+; CHECK-GI-FP16-NEXT:    fcvtzs x12, h4
+; CHECK-GI-FP16-NEXT:    fcvtzs x11, h3
+; CHECK-GI-FP16-NEXT:    fcvtzs x15, h7
+; CHECK-GI-FP16-NEXT:    fmov d0, x9
+; CHECK-GI-FP16-NEXT:    fcvtzs x10, h2
+; CHECK-GI-FP16-NEXT:    fcvtzs x13, h5
+; CHECK-GI-FP16-NEXT:    fcvtzs x14, h6
+; CHECK-GI-FP16-NEXT:    fmov d2, x8
+; CHECK-GI-FP16-NEXT:    fmov d1, x12
+; CHECK-GI-FP16-NEXT:    mov v0.d[1], x11
+; CHECK-GI-FP16-NEXT:    fmov d3, x10
+; CHECK-GI-FP16-NEXT:    mov v2.d[1], x13
+; CHECK-GI-FP16-NEXT:    mov v1.d[1], x15
+; CHECK-GI-FP16-NEXT:    mov v3.d[1], x14
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %c = fptosi <8 x half> %a to <8 x i64>
+  ret <8 x i64> %c
+}
+
+define <8 x i64> @fptou_v8f16_v8i64(<8 x half> %a) {
+; CHECK-SD-NOFP16-LABEL: fptou_v8f16_v8i64:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x9, s0
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x8, s1
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x12, s4
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x11, s3
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x15, s7
+; CHECK-SD-NOFP16-NEXT:    fmov d0, x9
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x10, s2
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x13, s5
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x14, s6
+; CHECK-SD-NOFP16-NEXT:    fmov d2, x8
+; CHECK-SD-NOFP16-NEXT:    fmov d1, x12
+; CHECK-SD-NOFP16-NEXT:    mov v0.d[1], x11
+; CHECK-SD-NOFP16-NEXT:    fmov d3, x10
+; CHECK-SD-NOFP16-NEXT:    mov v2.d[1], x13
+; CHECK-SD-NOFP16-NEXT:    mov v1.d[1], x15
+; CHECK-SD-NOFP16-NEXT:    mov v3.d[1], x14
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptou_v8f16_v8i64:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-FP16-NEXT:    mov h4, v0.h[2]
+; CHECK-SD-FP16-NEXT:    mov h3, v0.h[1]
+; CHECK-SD-FP16-NEXT:    mov h7, v0.h[3]
+; CHECK-SD-FP16-NEXT:    fcvtzu x9, h0
+; CHECK-SD-FP16-NEXT:    mov h2, v1.h[2]
+; CHECK-SD-FP16-NEXT:    mov h5, v1.h[1]
+; CHECK-SD-FP16-NEXT:    mov h6, v1.h[3]
+; CHECK-SD-FP16-NEXT:    fcvtzu x8, h1
+; CHECK-SD-FP16-NEXT:    fcvtzu x12, h4
+; CHECK-SD-FP16-NEXT:    fcvtzu x11, h3
+; CHECK-SD-FP16-NEXT:    fcvtzu x15, h7
+; CHECK-SD-FP16-NEXT:    fmov d0, x9
+; CHECK-SD-FP16-NEXT:    fcvtzu x10, h2
+; CHECK-SD-FP16-NEXT:    fcvtzu x13, h5
+; CHECK-SD-FP16-NEXT:    fcvtzu x14, h6
+; CHECK-SD-FP16-NEXT:    fmov d2, x8
+; CHECK-SD-FP16-NEXT:    fmov d1, x12
+; CHECK-SD-FP16-NEXT:    mov v0.d[1], x11
+; CHECK-SD-FP16-NEXT:    fmov d3, x10
+; CHECK-SD-FP16-NEXT:    mov v2.d[1], x13
+; CHECK-SD-FP16-NEXT:    mov v1.d[1], x15
+; CHECK-SD-FP16-NEXT:    mov v3.d[1], x14
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fptou_v8f16_v8i64:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-GI-NOFP16-NEXT:    mov h4, v0.h[2]
+; CHECK-GI-NOFP16-NEXT:    mov h3, v0.h[1]
+; CHECK-GI-NOFP16-NEXT:    mov h7, v0.h[3]
+; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-GI-NOFP16-NEXT:    mov h2, v1.h[2]
+; CHECK-GI-NOFP16-NEXT:    mov h5, v1.h[1]
+; CHECK-GI-NOFP16-NEXT:    mov h6, v1.h[3]
+; CHECK-GI-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-GI-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-GI-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-GI-NOFP16-NEXT:    fcvt s7, h7
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x9, s0
+; CHECK-GI-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-GI-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-GI-NOFP16-NEXT:    fcvt s6, h6
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x8, s1
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x12, s4
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x11, s3
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x15, s7
+; CHECK-GI-NOFP16-NEXT:    fmov d0, x9
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x10, s2
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x13, s5
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x14, s6
+; CHECK-GI-NOFP16-NEXT:    fmov d2, x8
+; CHECK-GI-NOFP16-NEXT:    fmov d1, x12
+; CHECK-GI-NOFP16-NEXT:    mov v0.d[1], x11
+; CHECK-GI-NOFP16-NEXT:    fmov d3, x10
+; CHECK-GI-NOFP16-NEXT:    mov v2.d[1], x13
+; CHECK-GI-NOFP16-NEXT:    mov v1.d[1], x15
+; CHECK-GI-NOFP16-NEXT:    mov v3.d[1], x14
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fptou_v8f16_v8i64:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-GI-FP16-NEXT:    mov h4, v0.h[2]
+; CHECK-GI-FP16-NEXT:    mov h3, v0.h[1]
+; CHECK-GI-FP16-NEXT:    mov h7, v0.h[3]
+; CHECK-GI-FP16-NEXT:    fcvtzu x9, h0
+; CHECK-GI-FP16-NEXT:    mov h2, v1.h[2]
+; CHECK-GI-FP16-NEXT:    mov h5, v1.h[1]
+; CHECK-GI-FP16-NEXT:    mov h6, v1.h[3]
+; CHECK-GI-FP16-NEXT:    fcvtzu x8, h1
+; CHECK-GI-FP16-NEXT:    fcvtzu x12, h4
+; CHECK-GI-FP16-NEXT:    fcvtzu x11, h3
+; CHECK-GI-FP16-NEXT:    fcvtzu x15, h7
+; CHECK-GI-FP16-NEXT:    fmov d0, x9
+; CHECK-GI-FP16-NEXT:    fcvtzu x10, h2
+; CHECK-GI-FP16-NEXT:    fcvtzu x13, h5
+; CHECK-GI-FP16-NEXT:    fcvtzu x14, h6
+; CHECK-GI-FP16-NEXT:    fmov d2, x8
+; CHECK-GI-FP16-NEXT:    fmov d1, x12
+; CHECK-GI-FP16-NEXT:    mov v0.d[1], x11
+; CHECK-GI-FP16-NEXT:    fmov d3, x10
+; CHECK-GI-FP16-NEXT:    mov v2.d[1], x13
+; CHECK-GI-FP16-NEXT:    mov v1.d[1], x15
+; CHECK-GI-FP16-NEXT:    mov v3.d[1], x14
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %c = fptoui <8 x half> %a to <8 x i64>
+  ret <8 x i64> %c
+}
+
+define <16 x i64> @fptos_v16f16_v16i64(<16 x half> %a) {
+; CHECK-SD-NOFP16-LABEL: fptos_v16f16_v16i64:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NOFP16-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
+; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h0
+; CHECK-SD-NOFP16-NEXT:    mov h18, v0.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt s6, h2
+; CHECK-SD-NOFP16-NEXT:    mov h7, v2.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h16, v2.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h17, v3.h[2]
+; CHECK-SD-NOFP16-NEXT:    fcvt s19, h3
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x8, s5
+; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h2, v2.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x9, s6
+; CHECK-SD-NOFP16-NEXT:    fcvt s6, h7
+; CHECK-SD-NOFP16-NEXT:    fcvt s7, h16
+; CHECK-SD-NOFP16-NEXT:    mov h16, v1.h[2]
+; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x10, s19
+; CHECK-SD-NOFP16-NEXT:    mov h19, v3.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x11, s4
+; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[3]
+; CHECK-SD-NOFP16-NEXT:    mov h3, v3.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x13, s7
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x12, s6
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x15, s18
+; CHECK-SD-NOFP16-NEXT:    fcvt s7, h16
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x14, s17
+; CHECK-SD-NOFP16-NEXT:    fcvt s16, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt s17, h19
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fmov d2, x9
+; CHECK-SD-NOFP16-NEXT:    fcvt s19, h3
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x9, s1
+; CHECK-SD-NOFP16-NEXT:    fmov d6, x10
+; CHECK-SD-NOFP16-NEXT:    fmov d3, x13
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x13, s0
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x16, s5
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x10, s7
+; CHECK-SD-NOFP16-NEXT:    fmov d7, x14
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x14, s16
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x17, s17
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x0, s4
+; CHECK-SD-NOFP16-NEXT:    fmov d0, x8
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x18, s19
+; CHECK-SD-NOFP16-NEXT:    fmov d1, x15
+; CHECK-SD-NOFP16-NEXT:    fmov d4, x9
+; CHECK-SD-NOFP16-NEXT:    mov v2.d[1], x12
+; CHECK-SD-NOFP16-NEXT:    fmov d5, x10
+; CHECK-SD-NOFP16-NEXT:    mov v0.d[1], x11
+; CHECK-SD-NOFP16-NEXT:    mov v3.d[1], x14
+; CHECK-SD-NOFP16-NEXT:    mov v1.d[1], x13
+; CHECK-SD-NOFP16-NEXT:    mov v4.d[1], x16
+; CHECK-SD-NOFP16-NEXT:    mov v6.d[1], x17
+; CHECK-SD-NOFP16-NEXT:    mov v7.d[1], x18
+; CHECK-SD-NOFP16-NEXT:    mov v5.d[1], x0
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptos_v16f16_v16i64:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-SD-FP16-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
+; CHECK-SD-FP16-NEXT:    mov h4, v0.h[1]
+; CHECK-SD-FP16-NEXT:    mov h5, v0.h[2]
+; CHECK-SD-FP16-NEXT:    fcvtzs x8, h0
+; CHECK-SD-FP16-NEXT:    mov h0, v0.h[3]
+; CHECK-SD-FP16-NEXT:    fcvtzs x9, h1
+; CHECK-SD-FP16-NEXT:    mov h7, v1.h[1]
+; CHECK-SD-FP16-NEXT:    mov h6, v2.h[2]
+; CHECK-SD-FP16-NEXT:    mov h16, v3.h[2]
+; CHECK-SD-FP16-NEXT:    fcvtzs x10, h4
+; CHECK-SD-FP16-NEXT:    mov h4, v1.h[2]
+; CHECK-SD-FP16-NEXT:    fcvtzs x11, h2
+; CHECK-SD-FP16-NEXT:    fcvtzs x12, h5
+; CHECK-SD-FP16-NEXT:    mov h5, v2.h[1]
+; CHECK-SD-FP16-NEXT:    mov h17, v2.h[3]
+; CHECK-SD-FP16-NEXT:    fcvtzs x13, h3
+; CHECK-SD-FP16-NEXT:    mov h18, v3.h[1]
+; CHECK-SD-FP16-NEXT:    mov h1, v1.h[3]
+; CHECK-SD-FP16-NEXT:    mov h19, v3.h[3]
+; CHECK-SD-FP16-NEXT:    fcvtzs x14, h6
+; CHECK-SD-FP16-NEXT:    fcvtzs x15, h16
+; CHECK-SD-FP16-NEXT:    fcvtzs x16, h0
+; CHECK-SD-FP16-NEXT:    fcvtzs x0, h4
+; CHECK-SD-FP16-NEXT:    fcvtzs x17, h7
+; CHECK-SD-FP16-NEXT:    fmov d2, x11
+; CHECK-SD-FP16-NEXT:    fcvtzs x11, h5
+; CHECK-SD-FP16-NEXT:    fcvtzs x18, h17
+; CHECK-SD-FP16-NEXT:    fmov d6, x13
+; CHECK-SD-FP16-NEXT:    fcvtzs x13, h18
+; CHECK-SD-FP16-NEXT:    fmov d0, x8
+; CHECK-SD-FP16-NEXT:    fmov d4, x9
+; CHECK-SD-FP16-NEXT:    fmov d3, x14
+; CHECK-SD-FP16-NEXT:    fmov d7, x15
+; CHECK-SD-FP16-NEXT:    fcvtzs x14, h19
+; CHECK-SD-FP16-NEXT:    fcvtzs x15, h1
+; CHECK-SD-FP16-NEXT:    fmov d1, x12
+; CHECK-SD-FP16-NEXT:    fmov d5, x0
+; CHECK-SD-FP16-NEXT:    mov v0.d[1], x10
+; CHECK-SD-FP16-NEXT:    mov v4.d[1], x17
+; CHECK-SD-FP16-NEXT:    mov v2.d[1], x11
+; CHECK-SD-FP16-NEXT:    mov v3.d[1], x18
+; CHECK-SD-FP16-NEXT:    mov v6.d[1], x13
+; CHECK-SD-FP16-NEXT:    mov v1.d[1], x16
+; CHECK-SD-FP16-NEXT:    mov v7.d[1], x14
+; CHECK-SD-FP16-NEXT:    mov v5.d[1], x15
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fptos_v16f16_v16i64:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-GI-NOFP16-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
+; CHECK-GI-NOFP16-NEXT:    mov h4, v0.h[1]
+; CHECK-GI-NOFP16-NEXT:    fcvt s5, h0
+; CHECK-GI-NOFP16-NEXT:    mov h18, v0.h[2]
+; CHECK-GI-NOFP16-NEXT:    mov h0, v0.h[3]
+; CHECK-GI-NOFP16-NEXT:    fcvt s6, h2
+; CHECK-GI-NOFP16-NEXT:    mov h7, v2.h[1]
+; CHECK-GI-NOFP16-NEXT:    mov h16, v2.h[2]
+; CHECK-GI-NOFP16-NEXT:    mov h17, v3.h[2]
+; CHECK-GI-NOFP16-NEXT:    fcvt s19, h3
+; CHECK-GI-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x8, s5
+; CHECK-GI-NOFP16-NEXT:    mov h5, v1.h[1]
+; CHECK-GI-NOFP16-NEXT:    mov h2, v2.h[3]
+; CHECK-GI-NOFP16-NEXT:    fcvt s18, h18
+; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x9, s6
+; CHECK-GI-NOFP16-NEXT:    fcvt s6, h7
+; CHECK-GI-NOFP16-NEXT:    fcvt s7, h16
+; CHECK-GI-NOFP16-NEXT:    mov h16, v1.h[2]
+; CHECK-GI-NOFP16-NEXT:    fcvt s17, h17
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x10, s19
+; CHECK-GI-NOFP16-NEXT:    mov h19, v3.h[1]
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x11, s4
+; CHECK-GI-NOFP16-NEXT:    mov h4, v1.h[3]
+; CHECK-GI-NOFP16-NEXT:    mov h3, v3.h[3]
+; CHECK-GI-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-GI-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x13, s7
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x12, s6
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x15, s18
+; CHECK-GI-NOFP16-NEXT:    fcvt s7, h16
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x14, s17
+; CHECK-GI-NOFP16-NEXT:    fcvt s16, h2
+; CHECK-GI-NOFP16-NEXT:    fcvt s17, h19
+; CHECK-GI-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-GI-NOFP16-NEXT:    fmov d2, x9
+; CHECK-GI-NOFP16-NEXT:    fcvt s19, h3
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x9, s1
+; CHECK-GI-NOFP16-NEXT:    fmov d6, x10
+; CHECK-GI-NOFP16-NEXT:    fmov d3, x13
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x13, s0
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x16, s5
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x10, s7
+; CHECK-GI-NOFP16-NEXT:    fmov d7, x14
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x14, s16
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x17, s17
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x0, s4
+; CHECK-GI-NOFP16-NEXT:    fmov d0, x8
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x18, s19
+; CHECK-GI-NOFP16-NEXT:    fmov d1, x15
+; CHECK-GI-NOFP16-NEXT:    fmov d4, x9
+; CHECK-GI-NOFP16-NEXT:    mov v2.d[1], x12
+; CHECK-GI-NOFP16-NEXT:    fmov d5, x10
+; CHECK-GI-NOFP16-NEXT:    mov v0.d[1], x11
+; CHECK-GI-NOFP16-NEXT:    mov v3.d[1], x14
+; CHECK-GI-NOFP16-NEXT:    mov v1.d[1], x13
+; CHECK-GI-NOFP16-NEXT:    mov v4.d[1], x16
+; CHECK-GI-NOFP16-NEXT:    mov v6.d[1], x17
+; CHECK-GI-NOFP16-NEXT:    mov v7.d[1], x18
+; CHECK-GI-NOFP16-NEXT:    mov v5.d[1], x0
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fptos_v16f16_v16i64:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-GI-FP16-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
+; CHECK-GI-FP16-NEXT:    mov h4, v0.h[1]
+; CHECK-GI-FP16-NEXT:    mov h5, v0.h[2]
+; CHECK-GI-FP16-NEXT:    fcvtzs x8, h0
+; CHECK-GI-FP16-NEXT:    mov h0, v0.h[3]
+; CHECK-GI-FP16-NEXT:    fcvtzs x9, h1
+; CHECK-GI-FP16-NEXT:    mov h7, v1.h[1]
+; CHECK-GI-FP16-NEXT:    mov h6, v2.h[2]
+; CHECK-GI-FP16-NEXT:    mov h16, v3.h[2]
+; CHECK-GI-FP16-NEXT:    fcvtzs x10, h4
+; CHECK-GI-FP16-NEXT:    mov h4, v1.h[2]
+; CHECK-GI-FP16-NEXT:    fcvtzs x11, h2
+; CHECK-GI-FP16-NEXT:    fcvtzs x12, h5
+; CHECK-GI-FP16-NEXT:    mov h5, v2.h[1]
+; CHECK-GI-FP16-NEXT:    mov h17, v2.h[3]
+; CHECK-GI-FP16-NEXT:    fcvtzs x13, h3
+; CHECK-GI-FP16-NEXT:    mov h18, v3.h[1]
+; CHECK-GI-FP16-NEXT:    mov h1, v1.h[3]
+; CHECK-GI-FP16-NEXT:    mov h19, v3.h[3]
+; CHECK-GI-FP16-NEXT:    fcvtzs x14, h6
+; CHECK-GI-FP16-NEXT:    fcvtzs x15, h16
+; CHECK-GI-FP16-NEXT:    fcvtzs x16, h0
+; CHECK-GI-FP16-NEXT:    fcvtzs x0, h4
+; CHECK-GI-FP16-NEXT:    fcvtzs x17, h7
+; CHECK-GI-FP16-NEXT:    fmov d2, x11
+; CHECK-GI-FP16-NEXT:    fcvtzs x11, h5
+; CHECK-GI-FP16-NEXT:    fcvtzs x18, h17
+; CHECK-GI-FP16-NEXT:    fmov d6, x13
+; CHECK-GI-FP16-NEXT:    fcvtzs x13, h18
+; CHECK-GI-FP16-NEXT:    fmov d0, x8
+; CHECK-GI-FP16-NEXT:    fmov d4, x9
+; CHECK-GI-FP16-NEXT:    fmov d3, x14
+; CHECK-GI-FP16-NEXT:    fmov d7, x15
+; CHECK-GI-FP16-NEXT:    fcvtzs x14, h19
+; CHECK-GI-FP16-NEXT:    fcvtzs x15, h1
+; CHECK-GI-FP16-NEXT:    fmov d1, x12
+; CHECK-GI-FP16-NEXT:    fmov d5, x0
+; CHECK-GI-FP16-NEXT:    mov v0.d[1], x10
+; CHECK-GI-FP16-NEXT:    mov v4.d[1], x17
+; CHECK-GI-FP16-NEXT:    mov v2.d[1], x11
+; CHECK-GI-FP16-NEXT:    mov v3.d[1], x18
+; CHECK-GI-FP16-NEXT:    mov v6.d[1], x13
+; CHECK-GI-FP16-NEXT:    mov v1.d[1], x16
+; CHECK-GI-FP16-NEXT:    mov v7.d[1], x14
+; CHECK-GI-FP16-NEXT:    mov v5.d[1], x15
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %c = fptosi <16 x half> %a to <16 x i64>
+  ret <16 x i64> %c
+}
+
+define <16 x i64> @fptou_v16f16_v16i64(<16 x half> %a) {
+; CHECK-SD-NOFP16-LABEL: fptou_v16f16_v16i64:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NOFP16-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
+; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h0
+; CHECK-SD-NOFP16-NEXT:    mov h18, v0.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt s6, h2
+; CHECK-SD-NOFP16-NEXT:    mov h7, v2.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h16, v2.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h17, v3.h[2]
+; CHECK-SD-NOFP16-NEXT:    fcvt s19, h3
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x8, s5
+; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h2, v2.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x9, s6
+; CHECK-SD-NOFP16-NEXT:    fcvt s6, h7
+; CHECK-SD-NOFP16-NEXT:    fcvt s7, h16
+; CHECK-SD-NOFP16-NEXT:    mov h16, v1.h[2]
+; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x10, s19
+; CHECK-SD-NOFP16-NEXT:    mov h19, v3.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x11, s4
+; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[3]
+; CHECK-SD-NOFP16-NEXT:    mov h3, v3.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x13, s7
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x12, s6
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x15, s18
+; CHECK-SD-NOFP16-NEXT:    fcvt s7, h16
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x14, s17
+; CHECK-SD-NOFP16-NEXT:    fcvt s16, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt s17, h19
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fmov d2, x9
+; CHECK-SD-NOFP16-NEXT:    fcvt s19, h3
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x9, s1
+; CHECK-SD-NOFP16-NEXT:    fmov d6, x10
+; CHECK-SD-NOFP16-NEXT:    fmov d3, x13
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x13, s0
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x16, s5
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x10, s7
+; CHECK-SD-NOFP16-NEXT:    fmov d7, x14
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x14, s16
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x17, s17
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x0, s4
+; CHECK-SD-NOFP16-NEXT:    fmov d0, x8
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x18, s19
+; CHECK-SD-NOFP16-NEXT:    fmov d1, x15
+; CHECK-SD-NOFP16-NEXT:    fmov d4, x9
+; CHECK-SD-NOFP16-NEXT:    mov v2.d[1], x12
+; CHECK-SD-NOFP16-NEXT:    fmov d5, x10
+; CHECK-SD-NOFP16-NEXT:    mov v0.d[1], x11
+; CHECK-SD-NOFP16-NEXT:    mov v3.d[1], x14
+; CHECK-SD-NOFP16-NEXT:    mov v1.d[1], x13
+; CHECK-SD-NOFP16-NEXT:    mov v4.d[1], x16
+; CHECK-SD-NOFP16-NEXT:    mov v6.d[1], x17
+; CHECK-SD-NOFP16-NEXT:    mov v7.d[1], x18
+; CHECK-SD-NOFP16-NEXT:    mov v5.d[1], x0
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptou_v16f16_v16i64:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-SD-FP16-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
+; CHECK-SD-FP16-NEXT:    mov h4, v0.h[1]
+; CHECK-SD-FP16-NEXT:    mov h5, v0.h[2]
+; CHECK-SD-FP16-NEXT:    fcvtzu x8, h0
+; CHECK-SD-FP16-NEXT:    mov h0, v0.h[3]
+; CHECK-SD-FP16-NEXT:    fcvtzu x9, h1
+; CHECK-SD-FP16-NEXT:    mov h7, v1.h[1]
+; CHECK-SD-FP16-NEXT:    mov h6, v2.h[2]
+; CHECK-SD-FP16-NEXT:    mov h16, v3.h[2]
+; CHECK-SD-FP16-NEXT:    fcvtzu x10, h4
+; CHECK-SD-FP16-NEXT:    mov h4, v1.h[2]
+; CHECK-SD-FP16-NEXT:    fcvtzu x11, h2
+; CHECK-SD-FP16-NEXT:    fcvtzu x12, h5
+; CHECK-SD-FP16-NEXT:    mov h5, v2.h[1]
+; CHECK-SD-FP16-NEXT:    mov h17, v2.h[3]
+; CHECK-SD-FP16-NEXT:    fcvtzu x13, h3
+; CHECK-SD-FP16-NEXT:    mov h18, v3.h[1]
+; CHECK-SD-FP16-NEXT:    mov h1, v1.h[3]
+; CHECK-SD-FP16-NEXT:    mov h19, v3.h[3]
+; CHECK-SD-FP16-NEXT:    fcvtzu x14, h6
+; CHECK-SD-FP16-NEXT:    fcvtzu x15, h16
+; CHECK-SD-FP16-NEXT:    fcvtzu x16, h0
+; CHECK-SD-FP16-NEXT:    fcvtzu x0, h4
+; CHECK-SD-FP16-NEXT:    fcvtzu x17, h7
+; CHECK-SD-FP16-NEXT:    fmov d2, x11
+; CHECK-SD-FP16-NEXT:    fcvtzu x11, h5
+; CHECK-SD-FP16-NEXT:    fcvtzu x18, h17
+; CHECK-SD-FP16-NEXT:    fmov d6, x13
+; CHECK-SD-FP16-NEXT:    fcvtzu x13, h18
+; CHECK-SD-FP16-NEXT:    fmov d0, x8
+; CHECK-SD-FP16-NEXT:    fmov d4, x9
+; CHECK-SD-FP16-NEXT:    fmov d3, x14
+; CHECK-SD-FP16-NEXT:    fmov d7, x15
+; CHECK-SD-FP16-NEXT:    fcvtzu x14, h19
+; CHECK-SD-FP16-NEXT:    fcvtzu x15, h1
+; CHECK-SD-FP16-NEXT:    fmov d1, x12
+; CHECK-SD-FP16-NEXT:    fmov d5, x0
+; CHECK-SD-FP16-NEXT:    mov v0.d[1], x10
+; CHECK-SD-FP16-NEXT:    mov v4.d[1], x17
+; CHECK-SD-FP16-NEXT:    mov v2.d[1], x11
+; CHECK-SD-FP16-NEXT:    mov v3.d[1], x18
+; CHECK-SD-FP16-NEXT:    mov v6.d[1], x13
+; CHECK-SD-FP16-NEXT:    mov v1.d[1], x16
+; CHECK-SD-FP16-NEXT:    mov v7.d[1], x14
+; CHECK-SD-FP16-NEXT:    mov v5.d[1], x15
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fptou_v16f16_v16i64:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-GI-NOFP16-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
+; CHECK-GI-NOFP16-NEXT:    mov h4, v0.h[1]
+; CHECK-GI-NOFP16-NEXT:    fcvt s5, h0
+; CHECK-GI-NOFP16-NEXT:    mov h18, v0.h[2]
+; CHECK-GI-NOFP16-NEXT:    mov h0, v0.h[3]
+; CHECK-GI-NOFP16-NEXT:    fcvt s6, h2
+; CHECK-GI-NOFP16-NEXT:    mov h7, v2.h[1]
+; CHECK-GI-NOFP16-NEXT:    mov h16, v2.h[2]
+; CHECK-GI-NOFP16-NEXT:    mov h17, v3.h[2]
+; CHECK-GI-NOFP16-NEXT:    fcvt s19, h3
+; CHECK-GI-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x8, s5
+; CHECK-GI-NOFP16-NEXT:    mov h5, v1.h[1]
+; CHECK-GI-NOFP16-NEXT:    mov h2, v2.h[3]
+; CHECK-GI-NOFP16-NEXT:    fcvt s18, h18
+; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x9, s6
+; CHECK-GI-NOFP16-NEXT:    fcvt s6, h7
+; CHECK-GI-NOFP16-NEXT:    fcvt s7, h16
+; CHECK-GI-NOFP16-NEXT:    mov h16, v1.h[2]
+; CHECK-GI-NOFP16-NEXT:    fcvt s17, h17
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x10, s19
+; CHECK-GI-NOFP16-NEXT:    mov h19, v3.h[1]
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x11, s4
+; CHECK-GI-NOFP16-NEXT:    mov h4, v1.h[3]
+; CHECK-GI-NOFP16-NEXT:    mov h3, v3.h[3]
+; CHECK-GI-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-GI-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x13, s7
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x12, s6
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x15, s18
+; CHECK-GI-NOFP16-NEXT:    fcvt s7, h16
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x14, s17
+; CHECK-GI-NOFP16-NEXT:    fcvt s16, h2
+; CHECK-GI-NOFP16-NEXT:    fcvt s17, h19
+; CHECK-GI-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-GI-NOFP16-NEXT:    fmov d2, x9
+; CHECK-GI-NOFP16-NEXT:    fcvt s19, h3
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x9, s1
+; CHECK-GI-NOFP16-NEXT:    fmov d6, x10
+; CHECK-GI-NOFP16-NEXT:    fmov d3, x13
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x13, s0
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x16, s5
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x10, s7
+; CHECK-GI-NOFP16-NEXT:    fmov d7, x14
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x14, s16
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x17, s17
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x0, s4
+; CHECK-GI-NOFP16-NEXT:    fmov d0, x8
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x18, s19
+; CHECK-GI-NOFP16-NEXT:    fmov d1, x15
+; CHECK-GI-NOFP16-NEXT:    fmov d4, x9
+; CHECK-GI-NOFP16-NEXT:    mov v2.d[1], x12
+; CHECK-GI-NOFP16-NEXT:    fmov d5, x10
+; CHECK-GI-NOFP16-NEXT:    mov v0.d[1], x11
+; CHECK-GI-NOFP16-NEXT:    mov v3.d[1], x14
+; CHECK-GI-NOFP16-NEXT:    mov v1.d[1], x13
+; CHECK-GI-NOFP16-NEXT:    mov v4.d[1], x16
+; CHECK-GI-NOFP16-NEXT:    mov v6.d[1], x17
+; CHECK-GI-NOFP16-NEXT:    mov v7.d[1], x18
+; CHECK-GI-NOFP16-NEXT:    mov v5.d[1], x0
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fptou_v16f16_v16i64:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-GI-FP16-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
+; CHECK-GI-FP16-NEXT:    mov h4, v0.h[1]
+; CHECK-GI-FP16-NEXT:    mov h5, v0.h[2]
+; CHECK-GI-FP16-NEXT:    fcvtzu x8, h0
+; CHECK-GI-FP16-NEXT:    mov h0, v0.h[3]
+; CHECK-GI-FP16-NEXT:    fcvtzu x9, h1
+; CHECK-GI-FP16-NEXT:    mov h7, v1.h[1]
+; CHECK-GI-FP16-NEXT:    mov h6, v2.h[2]
+; CHECK-GI-FP16-NEXT:    mov h16, v3.h[2]
+; CHECK-GI-FP16-NEXT:    fcvtzu x10, h4
+; CHECK-GI-FP16-NEXT:    mov h4, v1.h[2]
+; CHECK-GI-FP16-NEXT:    fcvtzu x11, h2
+; CHECK-GI-FP16-NEXT:    fcvtzu x12, h5
+; CHECK-GI-FP16-NEXT:    mov h5, v2.h[1]
+; CHECK-GI-FP16-NEXT:    mov h17, v2.h[3]
+; CHECK-GI-FP16-NEXT:    fcvtzu x13, h3
+; CHECK-GI-FP16-NEXT:    mov h18, v3.h[1]
+; CHECK-GI-FP16-NEXT:    mov h1, v1.h[3]
+; CHECK-GI-FP16-NEXT:    mov h19, v3.h[3]
+; CHECK-GI-FP16-NEXT:    fcvtzu x14, h6
+; CHECK-GI-FP16-NEXT:    fcvtzu x15, h16
+; CHECK-GI-FP16-NEXT:    fcvtzu x16, h0
+; CHECK-GI-FP16-NEXT:    fcvtzu x0, h4
+; CHECK-GI-FP16-NEXT:    fcvtzu x17, h7
+; CHECK-GI-FP16-NEXT:    fmov d2, x11
+; CHECK-GI-FP16-NEXT:    fcvtzu x11, h5
+; CHECK-GI-FP16-NEXT:    fcvtzu x18, h17
+; CHECK-GI-FP16-NEXT:    fmov d6, x13
+; CHECK-GI-FP16-NEXT:    fcvtzu x13, h18
+; CHECK-GI-FP16-NEXT:    fmov d0, x8
+; CHECK-GI-FP16-NEXT:    fmov d4, x9
+; CHECK-GI-FP16-NEXT:    fmov d3, x14
+; CHECK-GI-FP16-NEXT:    fmov d7, x15
+; CHECK-GI-FP16-NEXT:    fcvtzu x14, h19
+; CHECK-GI-FP16-NEXT:    fcvtzu x15, h1
+; CHECK-GI-FP16-NEXT:    fmov d1, x12
+; CHECK-GI-FP16-NEXT:    fmov d5, x0
+; CHECK-GI-FP16-NEXT:    mov v0.d[1], x10
+; CHECK-GI-FP16-NEXT:    mov v4.d[1], x17
+; CHECK-GI-FP16-NEXT:    mov v2.d[1], x11
+; CHECK-GI-FP16-NEXT:    mov v3.d[1], x18
+; CHECK-GI-FP16-NEXT:    mov v6.d[1], x13
+; CHECK-GI-FP16-NEXT:    mov v1.d[1], x16
+; CHECK-GI-FP16-NEXT:    mov v7.d[1], x14
+; CHECK-GI-FP16-NEXT:    mov v5.d[1], x15
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %c = fptoui <16 x half> %a to <16 x i64>
+  ret <16 x i64> %c
+}
+
+define <32 x i64> @fptos_v32f16_v32i64(<32 x half> %a) {
+; CHECK-SD-NOFP16-LABEL: fptos_v32f16_v32i64:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    ext v4.16b, v1.16b, v1.16b, #8
+; CHECK-SD-NOFP16-NEXT:    ext v5.16b, v2.16b, v2.16b, #8
+; CHECK-SD-NOFP16-NEXT:    ext v6.16b, v3.16b, v3.16b, #8
+; CHECK-SD-NOFP16-NEXT:    ext v7.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NOFP16-NEXT:    fcvt s21, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s22, h2
+; CHECK-SD-NOFP16-NEXT:    mov h26, v2.h[2]
+; CHECK-SD-NOFP16-NEXT:    fcvt s19, h0
+; CHECK-SD-NOFP16-NEXT:    mov h27, v3.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h20, v2.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h16, v4.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h17, v5.h[2]
+; CHECK-SD-NOFP16-NEXT:    fcvt s23, h5
+; CHECK-SD-NOFP16-NEXT:    fcvt s24, h6
+; CHECK-SD-NOFP16-NEXT:    mov h25, v6.h[2]
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x9, s21
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x11, s22
+; CHECK-SD-NOFP16-NEXT:    fcvt s22, h7
+; CHECK-SD-NOFP16-NEXT:    mov h21, v3.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x10, s19
+; CHECK-SD-NOFP16-NEXT:    fcvt s27, h27
+; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
+; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
+; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x12, s23
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x13, s24
+; CHECK-SD-NOFP16-NEXT:    fcvt s23, h25
+; CHECK-SD-NOFP16-NEXT:    fcvt s25, h26
+; CHECK-SD-NOFP16-NEXT:    mov h26, v3.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h24, v2.h[3]
+; CHECK-SD-NOFP16-NEXT:    fmov d19, x9
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x9, s22
+; CHECK-SD-NOFP16-NEXT:    fcvt s22, h3
+; CHECK-SD-NOFP16-NEXT:    fcvt s21, h21
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x14, s16
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x15, s17
+; CHECK-SD-NOFP16-NEXT:    fmov d2, x12
+; CHECK-SD-NOFP16-NEXT:    fmov d16, x13
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x12, s23
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x13, s25
+; CHECK-SD-NOFP16-NEXT:    mov h23, v1.h[2]
+; CHECK-SD-NOFP16-NEXT:    fcvt s25, h26
+; CHECK-SD-NOFP16-NEXT:    fcvt s24, h24
+; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[3]
+; CHECK-SD-NOFP16-NEXT:    fmov d26, x11
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x11, s21
+; CHECK-SD-NOFP16-NEXT:    fmov d3, x14
+; CHECK-SD-NOFP16-NEXT:    fmov d17, x15
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x14, s22
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x15, s27
+; CHECK-SD-NOFP16-NEXT:    mov h22, v0.h[2]
+; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
+; CHECK-SD-NOFP16-NEXT:    fcvt s21, h23
+; CHECK-SD-NOFP16-NEXT:    fmov d23, x13
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x13, s25
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fmov d25, x14
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x14, s24
+; CHECK-SD-NOFP16-NEXT:    fmov d24, x15
+; CHECK-SD-NOFP16-NEXT:    fcvt s22, h22
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x15, s18
+; CHECK-SD-NOFP16-NEXT:    mov h18, v7.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov v25.d[1], x13
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x13, s21
+; CHECK-SD-NOFP16-NEXT:    mov h21, v7.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov v24.d[1], x11
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x11, s20
+; CHECK-SD-NOFP16-NEXT:    mov h20, v0.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[3]
+; CHECK-SD-NOFP16-NEXT:    mov v23.d[1], x14
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x14, s1
+; CHECK-SD-NOFP16-NEXT:    mov h1, v6.h[3]
+; CHECK-SD-NOFP16-NEXT:    mov h6, v6.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov v19.d[1], x15
+; CHECK-SD-NOFP16-NEXT:    mov h7, v7.h[3]
+; CHECK-SD-NOFP16-NEXT:    stp q25, q24, [x8, #192]
+; CHECK-SD-NOFP16-NEXT:    fmov d24, x13
+; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
+; CHECK-SD-NOFP16-NEXT:    mov v26.d[1], x11
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x11, s22
+; CHECK-SD-NOFP16-NEXT:    mov h22, v5.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h5, v5.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    mov v24.d[1], x14
+; CHECK-SD-NOFP16-NEXT:    mov h25, v4.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
+; CHECK-SD-NOFP16-NEXT:    stp q26, q23, [x8, #128]
+; CHECK-SD-NOFP16-NEXT:    fmov d23, x12
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x12, s20
+; CHECK-SD-NOFP16-NEXT:    mov h20, v4.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x13, s0
+; CHECK-SD-NOFP16-NEXT:    stp q19, q24, [x8, #64]
+; CHECK-SD-NOFP16-NEXT:    fcvt s22, h22
+; CHECK-SD-NOFP16-NEXT:    fmov d0, x10
+; CHECK-SD-NOFP16-NEXT:    fmov d19, x11
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x10, s1
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h21
+; CHECK-SD-NOFP16-NEXT:    fcvt s24, h25
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x11, s6
+; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
+; CHECK-SD-NOFP16-NEXT:    fcvt s6, h7
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x14, s5
+; CHECK-SD-NOFP16-NEXT:    mov v19.d[1], x13
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h18
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x13, s22
+; CHECK-SD-NOFP16-NEXT:    mov v0.d[1], x12
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x12, s4
+; CHECK-SD-NOFP16-NEXT:    mov v23.d[1], x10
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x10, s1
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x15, s24
+; CHECK-SD-NOFP16-NEXT:    mov v16.d[1], x11
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x11, s20
+; CHECK-SD-NOFP16-NEXT:    mov v17.d[1], x14
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x14, s6
+; CHECK-SD-NOFP16-NEXT:    mov v2.d[1], x13
+; CHECK-SD-NOFP16-NEXT:    fcvtzs x13, s5
+; CHECK-SD-NOFP16-NEXT:    fmov d4, x9
+; CHECK-SD-NOFP16-NEXT:    stp q0, q19, [x8]
+; CHECK-SD-NOFP16-NEXT:    fmov d0, x12
+; CHECK-SD-NOFP16-NEXT:    stp q16, q23, [x8, #224]
+; CHECK-SD-NOFP16-NEXT:    fmov d1, x10
+; CHECK-SD-NOFP16-NEXT:    mov v3.d[1], x15
+; CHECK-SD-NOFP16-NEXT:    stp q2, q17, [x8, #160]
+; CHECK-SD-NOFP16-NEXT:    mov v0.d[1], x11
+; CHECK-SD-NOFP16-NEXT:    mov v4.d[1], x13
+; CHECK-SD-NOFP16-NEXT:    mov v1.d[1], x14
+; CHECK-SD-NOFP16-NEXT:    stp q0, q3, [x8, #96]
+; CHECK-SD-NOFP16-NEXT:    stp q4, q1, [x8, #32]
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptos_v32f16_v32i64:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    ext v4.16b, v1.16b, v1.16b, #8
+; CHECK-SD-FP16-NEXT:    ext v5.16b, v2.16b, v2.16b, #8
+; CHECK-SD-FP16-NEXT:    ext v6.16b, v3.16b, v3.16b, #8
+; CHECK-SD-FP16-NEXT:    mov h16, v3.h[2]
+; CHECK-SD-FP16-NEXT:    fcvtzs x9, h0
+; CHECK-SD-FP16-NEXT:    mov h23, v3.h[3]
+; CHECK-SD-FP16-NEXT:    mov h25, v3.h[1]
+; CHECK-SD-FP16-NEXT:    fcvtzs x15, h3
+; CHECK-SD-FP16-NEXT:    mov h24, v2.h[2]
+; CHECK-SD-FP16-NEXT:    mov h19, v1.h[2]
+; CHECK-SD-FP16-NEXT:    mov h21, v2.h[1]
+; CHECK-SD-FP16-NEXT:    mov h26, v2.h[3]
+; CHECK-SD-FP16-NEXT:    mov h17, v4.h[2]
+; CHECK-SD-FP16-NEXT:    mov h18, v5.h[2]
+; CHECK-SD-FP16-NEXT:    mov h22, v6.h[2]
+; CHECK-SD-FP16-NEXT:    fcvtzs x10, h5
+; CHECK-SD-FP16-NEXT:    fcvtzs x12, h16
+; CHECK-SD-FP16-NEXT:    fcvtzs x11, h6
+; CHECK-SD-FP16-NEXT:    mov h7, v1.h[1]
+; CHECK-SD-FP16-NEXT:    mov h20, v1.h[3]
+; CHECK-SD-FP16-NEXT:    fcvtzs x13, h17
+; CHECK-SD-FP16-NEXT:    fcvtzs x14, h18
+; CHECK-SD-FP16-NEXT:    fmov d18, x9
+; CHECK-SD-FP16-NEXT:    fcvtzs x9, h22
+; CHECK-SD-FP16-NEXT:    fmov d3, x10
+; CHECK-SD-FP16-NEXT:    fcvtzs x10, h23
+; CHECK-SD-FP16-NEXT:    fmov d22, x12
+; CHECK-SD-FP16-NEXT:    fcvtzs x12, h25
+; CHECK-SD-FP16-NEXT:    fmov d23, x15
+; CHECK-SD-FP16-NEXT:    fmov d16, x11
+; CHECK-SD-FP16-NEXT:    fcvtzs x11, h2
+; CHECK-SD-FP16-NEXT:    fcvtzs x15, h21
+; CHECK-SD-FP16-NEXT:    fmov d2, x13
+; CHECK-SD-FP16-NEXT:    fcvtzs x13, h24
+; CHECK-SD-FP16-NEXT:    fmov d17, x14
+; CHECK-SD-FP16-NEXT:    fcvtzs x14, h19
+; CHECK-SD-FP16-NEXT:    mov v22.d[1], x10
+; CHECK-SD-FP16-NEXT:    fcvtzs x10, h1
+; CHECK-SD-FP16-NEXT:    mov v23.d[1], x12
+; CHECK-SD-FP16-NEXT:    fmov d19, x9
+; CHECK-SD-FP16-NEXT:    fcvtzs x9, h26
+; CHECK-SD-FP16-NEXT:    fcvtzs x12, h20
+; CHECK-SD-FP16-NEXT:    mov h20, v0.h[2]
+; CHECK-SD-FP16-NEXT:    fmov d21, x11
+; CHECK-SD-FP16-NEXT:    fmov d1, x13
+; CHECK-SD-FP16-NEXT:    fcvtzs x13, h7
+; CHECK-SD-FP16-NEXT:    mov h24, v0.h[3]
+; CHECK-SD-FP16-NEXT:    fmov d7, x14
+; CHECK-SD-FP16-NEXT:    stp q23, q22, [x8, #192]
+; CHECK-SD-FP16-NEXT:    fmov d22, x10
+; CHECK-SD-FP16-NEXT:    mov v21.d[1], x15
+; CHECK-SD-FP16-NEXT:    mov v1.d[1], x9
+; CHECK-SD-FP16-NEXT:    mov h23, v0.h[1]
+; CHECK-SD-FP16-NEXT:    fcvtzs x9, h20
+; CHECK-SD-FP16-NEXT:    mov v7.d[1], x12
+; CHECK-SD-FP16-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-SD-FP16-NEXT:    mov h20, v6.h[3]
+; CHECK-SD-FP16-NEXT:    mov v22.d[1], x13
+; CHECK-SD-FP16-NEXT:    mov h6, v6.h[1]
+; CHECK-SD-FP16-NEXT:    fcvtzs x10, h24
+; CHECK-SD-FP16-NEXT:    stp q21, q1, [x8, #128]
+; CHECK-SD-FP16-NEXT:    mov h1, v5.h[1]
+; CHECK-SD-FP16-NEXT:    mov h5, v5.h[3]
+; CHECK-SD-FP16-NEXT:    fcvtzs x12, h20
+; CHECK-SD-FP16-NEXT:    mov h20, v0.h[2]
+; CHECK-SD-FP16-NEXT:    fcvtzs x11, h0
+; CHECK-SD-FP16-NEXT:    stp q22, q7, [x8, #64]
+; CHECK-SD-FP16-NEXT:    fmov d7, x9
+; CHECK-SD-FP16-NEXT:    fcvtzs x9, h23
+; CHECK-SD-FP16-NEXT:    mov h21, v4.h[3]
+; CHECK-SD-FP16-NEXT:    mov h22, v4.h[1]
+; CHECK-SD-FP16-NEXT:    fcvtzs x13, h6
+; CHECK-SD-FP16-NEXT:    mov h6, v0.h[3]
+; CHECK-SD-FP16-NEXT:    fcvtzs x14, h5
+; CHECK-SD-FP16-NEXT:    mov h0, v0.h[1]
+; CHECK-SD-FP16-NEXT:    mov v7.d[1], x10
+; CHECK-SD-FP16-NEXT:    fcvtzs x10, h1
+; CHECK-SD-FP16-NEXT:    mov v19.d[1], x12
+; CHECK-SD-FP16-NEXT:    mov v18.d[1], x9
+; CHECK-SD-FP16-NEXT:    fcvtzs x9, h4
+; CHECK-SD-FP16-NEXT:    fcvtzs x12, h20
+; CHECK-SD-FP16-NEXT:    fcvtzs x15, h21
+; CHECK-SD-FP16-NEXT:    mov v16.d[1], x13
+; CHECK-SD-FP16-NEXT:    fcvtzs x13, h22
+; CHECK-SD-FP16-NEXT:    mov v17.d[1], x14
+; CHECK-SD-FP16-NEXT:    fcvtzs x14, h6
+; CHECK-SD-FP16-NEXT:    fmov d4, x11
+; CHECK-SD-FP16-NEXT:    mov v3.d[1], x10
+; CHECK-SD-FP16-NEXT:    fcvtzs x10, h0
+; CHECK-SD-FP16-NEXT:    stp q18, q7, [x8]
+; CHECK-SD-FP16-NEXT:    fmov d0, x9
+; CHECK-SD-FP16-NEXT:    fmov d1, x12
+; CHECK-SD-FP16-NEXT:    stp q16, q19, [x8, #224]
+; CHECK-SD-FP16-NEXT:    mov v2.d[1], x15
+; CHECK-SD-FP16-NEXT:    stp q3, q17, [x8, #160]
+; CHECK-SD-FP16-NEXT:    mov v0.d[1], x13
+; CHECK-SD-FP16-NEXT:    mov v1.d[1], x14
+; CHECK-SD-FP16-NEXT:    mov v4.d[1], x10
+; CHECK-SD-FP16-NEXT:    stp q0, q2, [x8, #96]
+; CHECK-SD-FP16-NEXT:    stp q4, q1, [x8, #32]
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fptos_v32f16_v32i64:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    ext v4.16b, v1.16b, v1.16b, #8
+; CHECK-GI-NOFP16-NEXT:    ext v5.16b, v2.16b, v2.16b, #8
+; CHECK-GI-NOFP16-NEXT:    ext v6.16b, v3.16b, v3.16b, #8
+; CHECK-GI-NOFP16-NEXT:    ext v7.16b, v0.16b, v0.16b, #8
+; CHECK-GI-NOFP16-NEXT:    fcvt s21, h1
+; CHECK-GI-NOFP16-NEXT:    fcvt s22, h2
+; CHECK-GI-NOFP16-NEXT:    mov h26, v2.h[2]
+; CHECK-GI-NOFP16-NEXT:    fcvt s19, h0
+; CHECK-GI-NOFP16-NEXT:    mov h27, v3.h[2]
+; CHECK-GI-NOFP16-NEXT:    mov h20, v2.h[1]
+; CHECK-GI-NOFP16-NEXT:    mov h18, v1.h[1]
+; CHECK-GI-NOFP16-NEXT:    mov h16, v4.h[2]
+; CHECK-GI-NOFP16-NEXT:    mov h17, v5.h[2]
+; CHECK-GI-NOFP16-NEXT:    fcvt s23, h5
+; CHECK-GI-NOFP16-NEXT:    fcvt s24, h6
+; CHECK-GI-NOFP16-NEXT:    mov h25, v6.h[2]
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x9, s21
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x11, s22
+; CHECK-GI-NOFP16-NEXT:    fcvt s22, h7
+; CHECK-GI-NOFP16-NEXT:    mov h21, v3.h[3]
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x10, s19
+; CHECK-GI-NOFP16-NEXT:    fcvt s27, h27
+; CHECK-GI-NOFP16-NEXT:    fcvt s20, h20
+; CHECK-GI-NOFP16-NEXT:    fcvt s16, h16
+; CHECK-GI-NOFP16-NEXT:    fcvt s17, h17
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x12, s23
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x13, s24
+; CHECK-GI-NOFP16-NEXT:    fcvt s23, h25
+; CHECK-GI-NOFP16-NEXT:    fcvt s25, h26
+; CHECK-GI-NOFP16-NEXT:    mov h26, v3.h[1]
+; CHECK-GI-NOFP16-NEXT:    mov h24, v2.h[3]
+; CHECK-GI-NOFP16-NEXT:    fmov d19, x9
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x9, s22
+; CHECK-GI-NOFP16-NEXT:    fcvt s22, h3
+; CHECK-GI-NOFP16-NEXT:    fcvt s21, h21
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x14, s16
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x15, s17
+; CHECK-GI-NOFP16-NEXT:    fmov d2, x12
+; CHECK-GI-NOFP16-NEXT:    fmov d16, x13
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x12, s23
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x13, s25
+; CHECK-GI-NOFP16-NEXT:    mov h23, v1.h[2]
+; CHECK-GI-NOFP16-NEXT:    fcvt s25, h26
+; CHECK-GI-NOFP16-NEXT:    fcvt s24, h24
+; CHECK-GI-NOFP16-NEXT:    mov h1, v1.h[3]
+; CHECK-GI-NOFP16-NEXT:    fmov d26, x11
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x11, s21
+; CHECK-GI-NOFP16-NEXT:    fmov d3, x14
+; CHECK-GI-NOFP16-NEXT:    fmov d17, x15
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x14, s22
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x15, s27
+; CHECK-GI-NOFP16-NEXT:    mov h22, v0.h[2]
+; CHECK-GI-NOFP16-NEXT:    fcvt s18, h18
+; CHECK-GI-NOFP16-NEXT:    fcvt s21, h23
+; CHECK-GI-NOFP16-NEXT:    fmov d23, x13
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x13, s25
+; CHECK-GI-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-GI-NOFP16-NEXT:    fmov d25, x14
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x14, s24
+; CHECK-GI-NOFP16-NEXT:    fmov d24, x15
+; CHECK-GI-NOFP16-NEXT:    fcvt s22, h22
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x15, s18
+; CHECK-GI-NOFP16-NEXT:    mov h18, v7.h[1]
+; CHECK-GI-NOFP16-NEXT:    mov v25.d[1], x13
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x13, s21
+; CHECK-GI-NOFP16-NEXT:    mov h21, v7.h[2]
+; CHECK-GI-NOFP16-NEXT:    mov v24.d[1], x11
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x11, s20
+; CHECK-GI-NOFP16-NEXT:    mov h20, v0.h[1]
+; CHECK-GI-NOFP16-NEXT:    mov h0, v0.h[3]
+; CHECK-GI-NOFP16-NEXT:    mov v23.d[1], x14
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x14, s1
+; CHECK-GI-NOFP16-NEXT:    mov h1, v6.h[3]
+; CHECK-GI-NOFP16-NEXT:    mov h6, v6.h[1]
+; CHECK-GI-NOFP16-NEXT:    mov v19.d[1], x15
+; CHECK-GI-NOFP16-NEXT:    mov h7, v7.h[3]
+; CHECK-GI-NOFP16-NEXT:    stp q25, q24, [x8, #192]
+; CHECK-GI-NOFP16-NEXT:    fmov d24, x13
+; CHECK-GI-NOFP16-NEXT:    fcvt s20, h20
+; CHECK-GI-NOFP16-NEXT:    mov v26.d[1], x11
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x11, s22
+; CHECK-GI-NOFP16-NEXT:    mov h22, v5.h[1]
+; CHECK-GI-NOFP16-NEXT:    mov h5, v5.h[3]
+; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-GI-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-GI-NOFP16-NEXT:    mov v24.d[1], x14
+; CHECK-GI-NOFP16-NEXT:    mov h25, v4.h[3]
+; CHECK-GI-NOFP16-NEXT:    fcvt s6, h6
+; CHECK-GI-NOFP16-NEXT:    stp q26, q23, [x8, #128]
+; CHECK-GI-NOFP16-NEXT:    fmov d23, x12
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x12, s20
+; CHECK-GI-NOFP16-NEXT:    mov h20, v4.h[1]
+; CHECK-GI-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x13, s0
+; CHECK-GI-NOFP16-NEXT:    stp q19, q24, [x8, #64]
+; CHECK-GI-NOFP16-NEXT:    fcvt s22, h22
+; CHECK-GI-NOFP16-NEXT:    fmov d0, x10
+; CHECK-GI-NOFP16-NEXT:    fmov d19, x11
+; CHECK-GI-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x10, s1
+; CHECK-GI-NOFP16-NEXT:    fcvt s1, h21
+; CHECK-GI-NOFP16-NEXT:    fcvt s24, h25
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x11, s6
+; CHECK-GI-NOFP16-NEXT:    fcvt s20, h20
+; CHECK-GI-NOFP16-NEXT:    fcvt s6, h7
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x14, s5
+; CHECK-GI-NOFP16-NEXT:    mov v19.d[1], x13
+; CHECK-GI-NOFP16-NEXT:    fcvt s5, h18
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x13, s22
+; CHECK-GI-NOFP16-NEXT:    mov v0.d[1], x12
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x12, s4
+; CHECK-GI-NOFP16-NEXT:    mov v23.d[1], x10
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x10, s1
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x15, s24
+; CHECK-GI-NOFP16-NEXT:    mov v16.d[1], x11
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x11, s20
+; CHECK-GI-NOFP16-NEXT:    mov v17.d[1], x14
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x14, s6
+; CHECK-GI-NOFP16-NEXT:    mov v2.d[1], x13
+; CHECK-GI-NOFP16-NEXT:    fcvtzs x13, s5
+; CHECK-GI-NOFP16-NEXT:    fmov d4, x9
+; CHECK-GI-NOFP16-NEXT:    stp q0, q19, [x8]
+; CHECK-GI-NOFP16-NEXT:    fmov d0, x12
+; CHECK-GI-NOFP16-NEXT:    stp q16, q23, [x8, #224]
+; CHECK-GI-NOFP16-NEXT:    fmov d1, x10
+; CHECK-GI-NOFP16-NEXT:    mov v3.d[1], x15
+; CHECK-GI-NOFP16-NEXT:    stp q2, q17, [x8, #160]
+; CHECK-GI-NOFP16-NEXT:    mov v0.d[1], x11
+; CHECK-GI-NOFP16-NEXT:    mov v4.d[1], x13
+; CHECK-GI-NOFP16-NEXT:    mov v1.d[1], x14
+; CHECK-GI-NOFP16-NEXT:    stp q0, q3, [x8, #96]
+; CHECK-GI-NOFP16-NEXT:    stp q4, q1, [x8, #32]
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fptos_v32f16_v32i64:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    ext v4.16b, v1.16b, v1.16b, #8
+; CHECK-GI-FP16-NEXT:    ext v5.16b, v2.16b, v2.16b, #8
+; CHECK-GI-FP16-NEXT:    ext v6.16b, v3.16b, v3.16b, #8
+; CHECK-GI-FP16-NEXT:    mov h16, v3.h[2]
+; CHECK-GI-FP16-NEXT:    fcvtzs x9, h0
+; CHECK-GI-FP16-NEXT:    mov h23, v3.h[3]
+; CHECK-GI-FP16-NEXT:    mov h25, v3.h[1]
+; CHECK-GI-FP16-NEXT:    fcvtzs x15, h3
+; CHECK-GI-FP16-NEXT:    mov h24, v2.h[2]
+; CHECK-GI-FP16-NEXT:    mov h19, v1.h[2]
+; CHECK-GI-FP16-NEXT:    mov h21, v2.h[1]
+; CHECK-GI-FP16-NEXT:    mov h26, v2.h[3]
+; CHECK-GI-FP16-NEXT:    mov h17, v4.h[2]
+; CHECK-GI-FP16-NEXT:    mov h18, v5.h[2]
+; CHECK-GI-FP16-NEXT:    mov h22, v6.h[2]
+; CHECK-GI-FP16-NEXT:    fcvtzs x10, h5
+; CHECK-GI-FP16-NEXT:    fcvtzs x12, h16
+; CHECK-GI-FP16-NEXT:    fcvtzs x11, h6
+; CHECK-GI-FP16-NEXT:    mov h7, v1.h[1]
+; CHECK-GI-FP16-NEXT:    mov h20, v1.h[3]
+; CHECK-GI-FP16-NEXT:    fcvtzs x13, h17
+; CHECK-GI-FP16-NEXT:    fcvtzs x14, h18
+; CHECK-GI-FP16-NEXT:    fmov d18, x9
+; CHECK-GI-FP16-NEXT:    fcvtzs x9, h22
+; CHECK-GI-FP16-NEXT:    fmov d3, x10
+; CHECK-GI-FP16-NEXT:    fcvtzs x10, h23
+; CHECK-GI-FP16-NEXT:    fmov d22, x12
+; CHECK-GI-FP16-NEXT:    fcvtzs x12, h25
+; CHECK-GI-FP16-NEXT:    fmov d23, x15
+; CHECK-GI-FP16-NEXT:    fmov d16, x11
+; CHECK-GI-FP16-NEXT:    fcvtzs x11, h2
+; CHECK-GI-FP16-NEXT:    fcvtzs x15, h21
+; CHECK-GI-FP16-NEXT:    fmov d2, x13
+; CHECK-GI-FP16-NEXT:    fcvtzs x13, h24
+; CHECK-GI-FP16-NEXT:    fmov d17, x14
+; CHECK-GI-FP16-NEXT:    fcvtzs x14, h19
+; CHECK-GI-FP16-NEXT:    mov v22.d[1], x10
+; CHECK-GI-FP16-NEXT:    fcvtzs x10, h1
+; CHECK-GI-FP16-NEXT:    mov v23.d[1], x12
+; CHECK-GI-FP16-NEXT:    fmov d19, x9
+; CHECK-GI-FP16-NEXT:    fcvtzs x9, h26
+; CHECK-GI-FP16-NEXT:    fcvtzs x12, h20
+; CHECK-GI-FP16-NEXT:    mov h20, v0.h[2]
+; CHECK-GI-FP16-NEXT:    fmov d21, x11
+; CHECK-GI-FP16-NEXT:    fmov d1, x13
+; CHECK-GI-FP16-NEXT:    fcvtzs x13, h7
+; CHECK-GI-FP16-NEXT:    mov h24, v0.h[3]
+; CHECK-GI-FP16-NEXT:    fmov d7, x14
+; CHECK-GI-FP16-NEXT:    stp q23, q22, [x8, #192]
+; CHECK-GI-FP16-NEXT:    fmov d22, x10
+; CHECK-GI-FP16-NEXT:    mov v21.d[1], x15
+; CHECK-GI-FP16-NEXT:    mov v1.d[1], x9
+; CHECK-GI-FP16-NEXT:    mov h23, v0.h[1]
+; CHECK-GI-FP16-NEXT:    fcvtzs x9, h20
+; CHECK-GI-FP16-NEXT:    mov v7.d[1], x12
+; CHECK-GI-FP16-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-GI-FP16-NEXT:    mov h20, v6.h[3]
+; CHECK-GI-FP16-NEXT:    mov v22.d[1], x13
+; CHECK-GI-FP16-NEXT:    mov h6, v6.h[1]
+; CHECK-GI-FP16-NEXT:    fcvtzs x10, h24
+; CHECK-GI-FP16-NEXT:    stp q21, q1, [x8, #128]
+; CHECK-GI-FP16-NEXT:    mov h1, v5.h[1]
+; CHECK-GI-FP16-NEXT:    mov h5, v5.h[3]
+; CHECK-GI-FP16-NEXT:    fcvtzs x12, h20
+; CHECK-GI-FP16-NEXT:    mov h20, v0.h[2]
+; CHECK-GI-FP16-NEXT:    fcvtzs x11, h0
+; CHECK-GI-FP16-NEXT:    stp q22, q7, [x8, #64]
+; CHECK-GI-FP16-NEXT:    fmov d7, x9
+; CHECK-GI-FP16-NEXT:    fcvtzs x9, h23
+; CHECK-GI-FP16-NEXT:    mov h21, v4.h[3]
+; CHECK-GI-FP16-NEXT:    mov h22, v4.h[1]
+; CHECK-GI-FP16-NEXT:    fcvtzs x13, h6
+; CHECK-GI-FP16-NEXT:    mov h6, v0.h[3]
+; CHECK-GI-FP16-NEXT:    fcvtzs x14, h5
+; CHECK-GI-FP16-NEXT:    mov h0, v0.h[1]
+; CHECK-GI-FP16-NEXT:    mov v7.d[1], x10
+; CHECK-GI-FP16-NEXT:    fcvtzs x10, h1
+; CHECK-GI-FP16-NEXT:    mov v19.d[1], x12
+; CHECK-GI-FP16-NEXT:    mov v18.d[1], x9
+; CHECK-GI-FP16-NEXT:    fcvtzs x9, h4
+; CHECK-GI-FP16-NEXT:    fcvtzs x12, h20
+; CHECK-GI-FP16-NEXT:    fcvtzs x15, h21
+; CHECK-GI-FP16-NEXT:    mov v16.d[1], x13
+; CHECK-GI-FP16-NEXT:    fcvtzs x13, h22
+; CHECK-GI-FP16-NEXT:    mov v17.d[1], x14
+; CHECK-GI-FP16-NEXT:    fcvtzs x14, h6
+; CHECK-GI-FP16-NEXT:    fmov d4, x11
+; CHECK-GI-FP16-NEXT:    mov v3.d[1], x10
+; CHECK-GI-FP16-NEXT:    fcvtzs x10, h0
+; CHECK-GI-FP16-NEXT:    stp q18, q7, [x8]
+; CHECK-GI-FP16-NEXT:    fmov d0, x9
+; CHECK-GI-FP16-NEXT:    fmov d1, x12
+; CHECK-GI-FP16-NEXT:    stp q16, q19, [x8, #224]
+; CHECK-GI-FP16-NEXT:    mov v2.d[1], x15
+; CHECK-GI-FP16-NEXT:    stp q3, q17, [x8, #160]
+; CHECK-GI-FP16-NEXT:    mov v0.d[1], x13
+; CHECK-GI-FP16-NEXT:    mov v1.d[1], x14
+; CHECK-GI-FP16-NEXT:    mov v4.d[1], x10
+; CHECK-GI-FP16-NEXT:    stp q0, q2, [x8, #96]
+; CHECK-GI-FP16-NEXT:    stp q4, q1, [x8, #32]
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %c = fptosi <32 x half> %a to <32 x i64>
+  ret <32 x i64> %c
+}
+
+define <32 x i64> @fptou_v32f16_v32i64(<32 x half> %a) {
+; CHECK-SD-NOFP16-LABEL: fptou_v32f16_v32i64:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    ext v4.16b, v1.16b, v1.16b, #8
+; CHECK-SD-NOFP16-NEXT:    ext v5.16b, v2.16b, v2.16b, #8
+; CHECK-SD-NOFP16-NEXT:    ext v6.16b, v3.16b, v3.16b, #8
+; CHECK-SD-NOFP16-NEXT:    ext v7.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NOFP16-NEXT:    fcvt s21, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s22, h2
+; CHECK-SD-NOFP16-NEXT:    mov h26, v2.h[2]
+; CHECK-SD-NOFP16-NEXT:    fcvt s19, h0
+; CHECK-SD-NOFP16-NEXT:    mov h27, v3.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h20, v2.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h16, v4.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h17, v5.h[2]
+; CHECK-SD-NOFP16-NEXT:    fcvt s23, h5
+; CHECK-SD-NOFP16-NEXT:    fcvt s24, h6
+; CHECK-SD-NOFP16-NEXT:    mov h25, v6.h[2]
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x9, s21
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x11, s22
+; CHECK-SD-NOFP16-NEXT:    fcvt s22, h7
+; CHECK-SD-NOFP16-NEXT:    mov h21, v3.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x10, s19
+; CHECK-SD-NOFP16-NEXT:    fcvt s27, h27
+; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
+; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
+; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x12, s23
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x13, s24
+; CHECK-SD-NOFP16-NEXT:    fcvt s23, h25
+; CHECK-SD-NOFP16-NEXT:    fcvt s25, h26
+; CHECK-SD-NOFP16-NEXT:    mov h26, v3.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h24, v2.h[3]
+; CHECK-SD-NOFP16-NEXT:    fmov d19, x9
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x9, s22
+; CHECK-SD-NOFP16-NEXT:    fcvt s22, h3
+; CHECK-SD-NOFP16-NEXT:    fcvt s21, h21
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x14, s16
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x15, s17
+; CHECK-SD-NOFP16-NEXT:    fmov d2, x12
+; CHECK-SD-NOFP16-NEXT:    fmov d16, x13
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x12, s23
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x13, s25
+; CHECK-SD-NOFP16-NEXT:    mov h23, v1.h[2]
+; CHECK-SD-NOFP16-NEXT:    fcvt s25, h26
+; CHECK-SD-NOFP16-NEXT:    fcvt s24, h24
+; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[3]
+; CHECK-SD-NOFP16-NEXT:    fmov d26, x11
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x11, s21
+; CHECK-SD-NOFP16-NEXT:    fmov d3, x14
+; CHECK-SD-NOFP16-NEXT:    fmov d17, x15
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x14, s22
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x15, s27
+; CHECK-SD-NOFP16-NEXT:    mov h22, v0.h[2]
+; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
+; CHECK-SD-NOFP16-NEXT:    fcvt s21, h23
+; CHECK-SD-NOFP16-NEXT:    fmov d23, x13
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x13, s25
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fmov d25, x14
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x14, s24
+; CHECK-SD-NOFP16-NEXT:    fmov d24, x15
+; CHECK-SD-NOFP16-NEXT:    fcvt s22, h22
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x15, s18
+; CHECK-SD-NOFP16-NEXT:    mov h18, v7.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov v25.d[1], x13
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x13, s21
+; CHECK-SD-NOFP16-NEXT:    mov h21, v7.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov v24.d[1], x11
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x11, s20
+; CHECK-SD-NOFP16-NEXT:    mov h20, v0.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[3]
+; CHECK-SD-NOFP16-NEXT:    mov v23.d[1], x14
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x14, s1
+; CHECK-SD-NOFP16-NEXT:    mov h1, v6.h[3]
+; CHECK-SD-NOFP16-NEXT:    mov h6, v6.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov v19.d[1], x15
+; CHECK-SD-NOFP16-NEXT:    mov h7, v7.h[3]
+; CHECK-SD-NOFP16-NEXT:    stp q25, q24, [x8, #192]
+; CHECK-SD-NOFP16-NEXT:    fmov d24, x13
+; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
+; CHECK-SD-NOFP16-NEXT:    mov v26.d[1], x11
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x11, s22
+; CHECK-SD-NOFP16-NEXT:    mov h22, v5.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h5, v5.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    mov v24.d[1], x14
+; CHECK-SD-NOFP16-NEXT:    mov h25, v4.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
+; CHECK-SD-NOFP16-NEXT:    stp q26, q23, [x8, #128]
+; CHECK-SD-NOFP16-NEXT:    fmov d23, x12
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x12, s20
+; CHECK-SD-NOFP16-NEXT:    mov h20, v4.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x13, s0
+; CHECK-SD-NOFP16-NEXT:    stp q19, q24, [x8, #64]
+; CHECK-SD-NOFP16-NEXT:    fcvt s22, h22
+; CHECK-SD-NOFP16-NEXT:    fmov d0, x10
+; CHECK-SD-NOFP16-NEXT:    fmov d19, x11
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x10, s1
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h21
+; CHECK-SD-NOFP16-NEXT:    fcvt s24, h25
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x11, s6
+; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
+; CHECK-SD-NOFP16-NEXT:    fcvt s6, h7
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x14, s5
+; CHECK-SD-NOFP16-NEXT:    mov v19.d[1], x13
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h18
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x13, s22
+; CHECK-SD-NOFP16-NEXT:    mov v0.d[1], x12
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x12, s4
+; CHECK-SD-NOFP16-NEXT:    mov v23.d[1], x10
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x10, s1
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x15, s24
+; CHECK-SD-NOFP16-NEXT:    mov v16.d[1], x11
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x11, s20
+; CHECK-SD-NOFP16-NEXT:    mov v17.d[1], x14
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x14, s6
+; CHECK-SD-NOFP16-NEXT:    mov v2.d[1], x13
+; CHECK-SD-NOFP16-NEXT:    fcvtzu x13, s5
+; CHECK-SD-NOFP16-NEXT:    fmov d4, x9
+; CHECK-SD-NOFP16-NEXT:    stp q0, q19, [x8]
+; CHECK-SD-NOFP16-NEXT:    fmov d0, x12
+; CHECK-SD-NOFP16-NEXT:    stp q16, q23, [x8, #224]
+; CHECK-SD-NOFP16-NEXT:    fmov d1, x10
+; CHECK-SD-NOFP16-NEXT:    mov v3.d[1], x15
+; CHECK-SD-NOFP16-NEXT:    stp q2, q17, [x8, #160]
+; CHECK-SD-NOFP16-NEXT:    mov v0.d[1], x11
+; CHECK-SD-NOFP16-NEXT:    mov v4.d[1], x13
+; CHECK-SD-NOFP16-NEXT:    mov v1.d[1], x14
+; CHECK-SD-NOFP16-NEXT:    stp q0, q3, [x8, #96]
+; CHECK-SD-NOFP16-NEXT:    stp q4, q1, [x8, #32]
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptou_v32f16_v32i64:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    ext v4.16b, v1.16b, v1.16b, #8
+; CHECK-SD-FP16-NEXT:    ext v5.16b, v2.16b, v2.16b, #8
+; CHECK-SD-FP16-NEXT:    ext v6.16b, v3.16b, v3.16b, #8
+; CHECK-SD-FP16-NEXT:    mov h16, v3.h[2]
+; CHECK-SD-FP16-NEXT:    fcvtzu x9, h0
+; CHECK-SD-FP16-NEXT:    mov h23, v3.h[3]
+; CHECK-SD-FP16-NEXT:    mov h25, v3.h[1]
+; CHECK-SD-FP16-NEXT:    fcvtzu x15, h3
+; CHECK-SD-FP16-NEXT:    mov h24, v2.h[2]
+; CHECK-SD-FP16-NEXT:    mov h19, v1.h[2]
+; CHECK-SD-FP16-NEXT:    mov h21, v2.h[1]
+; CHECK-SD-FP16-NEXT:    mov h26, v2.h[3]
+; CHECK-SD-FP16-NEXT:    mov h17, v4.h[2]
+; CHECK-SD-FP16-NEXT:    mov h18, v5.h[2]
+; CHECK-SD-FP16-NEXT:    mov h22, v6.h[2]
+; CHECK-SD-FP16-NEXT:    fcvtzu x10, h5
+; CHECK-SD-FP16-NEXT:    fcvtzu x12, h16
+; CHECK-SD-FP16-NEXT:    fcvtzu x11, h6
+; CHECK-SD-FP16-NEXT:    mov h7, v1.h[1]
+; CHECK-SD-FP16-NEXT:    mov h20, v1.h[3]
+; CHECK-SD-FP16-NEXT:    fcvtzu x13, h17
+; CHECK-SD-FP16-NEXT:    fcvtzu x14, h18
+; CHECK-SD-FP16-NEXT:    fmov d18, x9
+; CHECK-SD-FP16-NEXT:    fcvtzu x9, h22
+; CHECK-SD-FP16-NEXT:    fmov d3, x10
+; CHECK-SD-FP16-NEXT:    fcvtzu x10, h23
+; CHECK-SD-FP16-NEXT:    fmov d22, x12
+; CHECK-SD-FP16-NEXT:    fcvtzu x12, h25
+; CHECK-SD-FP16-NEXT:    fmov d23, x15
+; CHECK-SD-FP16-NEXT:    fmov d16, x11
+; CHECK-SD-FP16-NEXT:    fcvtzu x11, h2
+; CHECK-SD-FP16-NEXT:    fcvtzu x15, h21
+; CHECK-SD-FP16-NEXT:    fmov d2, x13
+; CHECK-SD-FP16-NEXT:    fcvtzu x13, h24
+; CHECK-SD-FP16-NEXT:    fmov d17, x14
+; CHECK-SD-FP16-NEXT:    fcvtzu x14, h19
+; CHECK-SD-FP16-NEXT:    mov v22.d[1], x10
+; CHECK-SD-FP16-NEXT:    fcvtzu x10, h1
+; CHECK-SD-FP16-NEXT:    mov v23.d[1], x12
+; CHECK-SD-FP16-NEXT:    fmov d19, x9
+; CHECK-SD-FP16-NEXT:    fcvtzu x9, h26
+; CHECK-SD-FP16-NEXT:    fcvtzu x12, h20
+; CHECK-SD-FP16-NEXT:    mov h20, v0.h[2]
+; CHECK-SD-FP16-NEXT:    fmov d21, x11
+; CHECK-SD-FP16-NEXT:    fmov d1, x13
+; CHECK-SD-FP16-NEXT:    fcvtzu x13, h7
+; CHECK-SD-FP16-NEXT:    mov h24, v0.h[3]
+; CHECK-SD-FP16-NEXT:    fmov d7, x14
+; CHECK-SD-FP16-NEXT:    stp q23, q22, [x8, #192]
+; CHECK-SD-FP16-NEXT:    fmov d22, x10
+; CHECK-SD-FP16-NEXT:    mov v21.d[1], x15
+; CHECK-SD-FP16-NEXT:    mov v1.d[1], x9
+; CHECK-SD-FP16-NEXT:    mov h23, v0.h[1]
+; CHECK-SD-FP16-NEXT:    fcvtzu x9, h20
+; CHECK-SD-FP16-NEXT:    mov v7.d[1], x12
+; CHECK-SD-FP16-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-SD-FP16-NEXT:    mov h20, v6.h[3]
+; CHECK-SD-FP16-NEXT:    mov v22.d[1], x13
+; CHECK-SD-FP16-NEXT:    mov h6, v6.h[1]
+; CHECK-SD-FP16-NEXT:    fcvtzu x10, h24
+; CHECK-SD-FP16-NEXT:    stp q21, q1, [x8, #128]
+; CHECK-SD-FP16-NEXT:    mov h1, v5.h[1]
+; CHECK-SD-FP16-NEXT:    mov h5, v5.h[3]
+; CHECK-SD-FP16-NEXT:    fcvtzu x12, h20
+; CHECK-SD-FP16-NEXT:    mov h20, v0.h[2]
+; CHECK-SD-FP16-NEXT:    fcvtzu x11, h0
+; CHECK-SD-FP16-NEXT:    stp q22, q7, [x8, #64]
+; CHECK-SD-FP16-NEXT:    fmov d7, x9
+; CHECK-SD-FP16-NEXT:    fcvtzu x9, h23
+; CHECK-SD-FP16-NEXT:    mov h21, v4.h[3]
+; CHECK-SD-FP16-NEXT:    mov h22, v4.h[1]
+; CHECK-SD-FP16-NEXT:    fcvtzu x13, h6
+; CHECK-SD-FP16-NEXT:    mov h6, v0.h[3]
+; CHECK-SD-FP16-NEXT:    fcvtzu x14, h5
+; CHECK-SD-FP16-NEXT:    mov h0, v0.h[1]
+; CHECK-SD-FP16-NEXT:    mov v7.d[1], x10
+; CHECK-SD-FP16-NEXT:    fcvtzu x10, h1
+; CHECK-SD-FP16-NEXT:    mov v19.d[1], x12
+; CHECK-SD-FP16-NEXT:    mov v18.d[1], x9
+; CHECK-SD-FP16-NEXT:    fcvtzu x9, h4
+; CHECK-SD-FP16-NEXT:    fcvtzu x12, h20
+; CHECK-SD-FP16-NEXT:    fcvtzu x15, h21
+; CHECK-SD-FP16-NEXT:    mov v16.d[1], x13
+; CHECK-SD-FP16-NEXT:    fcvtzu x13, h22
+; CHECK-SD-FP16-NEXT:    mov v17.d[1], x14
+; CHECK-SD-FP16-NEXT:    fcvtzu x14, h6
+; CHECK-SD-FP16-NEXT:    fmov d4, x11
+; CHECK-SD-FP16-NEXT:    mov v3.d[1], x10
+; CHECK-SD-FP16-NEXT:    fcvtzu x10, h0
+; CHECK-SD-FP16-NEXT:    stp q18, q7, [x8]
+; CHECK-SD-FP16-NEXT:    fmov d0, x9
+; CHECK-SD-FP16-NEXT:    fmov d1, x12
+; CHECK-SD-FP16-NEXT:    stp q16, q19, [x8, #224]
+; CHECK-SD-FP16-NEXT:    mov v2.d[1], x15
+; CHECK-SD-FP16-NEXT:    stp q3, q17, [x8, #160]
+; CHECK-SD-FP16-NEXT:    mov v0.d[1], x13
+; CHECK-SD-FP16-NEXT:    mov v1.d[1], x14
+; CHECK-SD-FP16-NEXT:    mov v4.d[1], x10
+; CHECK-SD-FP16-NEXT:    stp q0, q2, [x8, #96]
+; CHECK-SD-FP16-NEXT:    stp q4, q1, [x8, #32]
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fptou_v32f16_v32i64:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    ext v4.16b, v1.16b, v1.16b, #8
+; CHECK-GI-NOFP16-NEXT:    ext v5.16b, v2.16b, v2.16b, #8
+; CHECK-GI-NOFP16-NEXT:    ext v6.16b, v3.16b, v3.16b, #8
+; CHECK-GI-NOFP16-NEXT:    ext v7.16b, v0.16b, v0.16b, #8
+; CHECK-GI-NOFP16-NEXT:    fcvt s21, h1
+; CHECK-GI-NOFP16-NEXT:    fcvt s22, h2
+; CHECK-GI-NOFP16-NEXT:    mov h26, v2.h[2]
+; CHECK-GI-NOFP16-NEXT:    fcvt s19, h0
+; CHECK-GI-NOFP16-NEXT:    mov h27, v3.h[2]
+; CHECK-GI-NOFP16-NEXT:    mov h20, v2.h[1]
+; CHECK-GI-NOFP16-NEXT:    mov h18, v1.h[1]
+; CHECK-GI-NOFP16-NEXT:    mov h16, v4.h[2]
+; CHECK-GI-NOFP16-NEXT:    mov h17, v5.h[2]
+; CHECK-GI-NOFP16-NEXT:    fcvt s23, h5
+; CHECK-GI-NOFP16-NEXT:    fcvt s24, h6
+; CHECK-GI-NOFP16-NEXT:    mov h25, v6.h[2]
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x9, s21
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x11, s22
+; CHECK-GI-NOFP16-NEXT:    fcvt s22, h7
+; CHECK-GI-NOFP16-NEXT:    mov h21, v3.h[3]
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x10, s19
+; CHECK-GI-NOFP16-NEXT:    fcvt s27, h27
+; CHECK-GI-NOFP16-NEXT:    fcvt s20, h20
+; CHECK-GI-NOFP16-NEXT:    fcvt s16, h16
+; CHECK-GI-NOFP16-NEXT:    fcvt s17, h17
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x12, s23
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x13, s24
+; CHECK-GI-NOFP16-NEXT:    fcvt s23, h25
+; CHECK-GI-NOFP16-NEXT:    fcvt s25, h26
+; CHECK-GI-NOFP16-NEXT:    mov h26, v3.h[1]
+; CHECK-GI-NOFP16-NEXT:    mov h24, v2.h[3]
+; CHECK-GI-NOFP16-NEXT:    fmov d19, x9
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x9, s22
+; CHECK-GI-NOFP16-NEXT:    fcvt s22, h3
+; CHECK-GI-NOFP16-NEXT:    fcvt s21, h21
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x14, s16
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x15, s17
+; CHECK-GI-NOFP16-NEXT:    fmov d2, x12
+; CHECK-GI-NOFP16-NEXT:    fmov d16, x13
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x12, s23
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x13, s25
+; CHECK-GI-NOFP16-NEXT:    mov h23, v1.h[2]
+; CHECK-GI-NOFP16-NEXT:    fcvt s25, h26
+; CHECK-GI-NOFP16-NEXT:    fcvt s24, h24
+; CHECK-GI-NOFP16-NEXT:    mov h1, v1.h[3]
+; CHECK-GI-NOFP16-NEXT:    fmov d26, x11
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x11, s21
+; CHECK-GI-NOFP16-NEXT:    fmov d3, x14
+; CHECK-GI-NOFP16-NEXT:    fmov d17, x15
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x14, s22
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x15, s27
+; CHECK-GI-NOFP16-NEXT:    mov h22, v0.h[2]
+; CHECK-GI-NOFP16-NEXT:    fcvt s18, h18
+; CHECK-GI-NOFP16-NEXT:    fcvt s21, h23
+; CHECK-GI-NOFP16-NEXT:    fmov d23, x13
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x13, s25
+; CHECK-GI-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-GI-NOFP16-NEXT:    fmov d25, x14
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x14, s24
+; CHECK-GI-NOFP16-NEXT:    fmov d24, x15
+; CHECK-GI-NOFP16-NEXT:    fcvt s22, h22
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x15, s18
+; CHECK-GI-NOFP16-NEXT:    mov h18, v7.h[1]
+; CHECK-GI-NOFP16-NEXT:    mov v25.d[1], x13
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x13, s21
+; CHECK-GI-NOFP16-NEXT:    mov h21, v7.h[2]
+; CHECK-GI-NOFP16-NEXT:    mov v24.d[1], x11
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x11, s20
+; CHECK-GI-NOFP16-NEXT:    mov h20, v0.h[1]
+; CHECK-GI-NOFP16-NEXT:    mov h0, v0.h[3]
+; CHECK-GI-NOFP16-NEXT:    mov v23.d[1], x14
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x14, s1
+; CHECK-GI-NOFP16-NEXT:    mov h1, v6.h[3]
+; CHECK-GI-NOFP16-NEXT:    mov h6, v6.h[1]
+; CHECK-GI-NOFP16-NEXT:    mov v19.d[1], x15
+; CHECK-GI-NOFP16-NEXT:    mov h7, v7.h[3]
+; CHECK-GI-NOFP16-NEXT:    stp q25, q24, [x8, #192]
+; CHECK-GI-NOFP16-NEXT:    fmov d24, x13
+; CHECK-GI-NOFP16-NEXT:    fcvt s20, h20
+; CHECK-GI-NOFP16-NEXT:    mov v26.d[1], x11
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x11, s22
+; CHECK-GI-NOFP16-NEXT:    mov h22, v5.h[1]
+; CHECK-GI-NOFP16-NEXT:    mov h5, v5.h[3]
+; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-GI-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-GI-NOFP16-NEXT:    mov v24.d[1], x14
+; CHECK-GI-NOFP16-NEXT:    mov h25, v4.h[3]
+; CHECK-GI-NOFP16-NEXT:    fcvt s6, h6
+; CHECK-GI-NOFP16-NEXT:    stp q26, q23, [x8, #128]
+; CHECK-GI-NOFP16-NEXT:    fmov d23, x12
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x12, s20
+; CHECK-GI-NOFP16-NEXT:    mov h20, v4.h[1]
+; CHECK-GI-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x13, s0
+; CHECK-GI-NOFP16-NEXT:    stp q19, q24, [x8, #64]
+; CHECK-GI-NOFP16-NEXT:    fcvt s22, h22
+; CHECK-GI-NOFP16-NEXT:    fmov d0, x10
+; CHECK-GI-NOFP16-NEXT:    fmov d19, x11
+; CHECK-GI-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x10, s1
+; CHECK-GI-NOFP16-NEXT:    fcvt s1, h21
+; CHECK-GI-NOFP16-NEXT:    fcvt s24, h25
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x11, s6
+; CHECK-GI-NOFP16-NEXT:    fcvt s20, h20
+; CHECK-GI-NOFP16-NEXT:    fcvt s6, h7
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x14, s5
+; CHECK-GI-NOFP16-NEXT:    mov v19.d[1], x13
+; CHECK-GI-NOFP16-NEXT:    fcvt s5, h18
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x13, s22
+; CHECK-GI-NOFP16-NEXT:    mov v0.d[1], x12
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x12, s4
+; CHECK-GI-NOFP16-NEXT:    mov v23.d[1], x10
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x10, s1
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x15, s24
+; CHECK-GI-NOFP16-NEXT:    mov v16.d[1], x11
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x11, s20
+; CHECK-GI-NOFP16-NEXT:    mov v17.d[1], x14
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x14, s6
+; CHECK-GI-NOFP16-NEXT:    mov v2.d[1], x13
+; CHECK-GI-NOFP16-NEXT:    fcvtzu x13, s5
+; CHECK-GI-NOFP16-NEXT:    fmov d4, x9
+; CHECK-GI-NOFP16-NEXT:    stp q0, q19, [x8]
+; CHECK-GI-NOFP16-NEXT:    fmov d0, x12
+; CHECK-GI-NOFP16-NEXT:    stp q16, q23, [x8, #224]
+; CHECK-GI-NOFP16-NEXT:    fmov d1, x10
+; CHECK-GI-NOFP16-NEXT:    mov v3.d[1], x15
+; CHECK-GI-NOFP16-NEXT:    stp q2, q17, [x8, #160]
+; CHECK-GI-NOFP16-NEXT:    mov v0.d[1], x11
+; CHECK-GI-NOFP16-NEXT:    mov v4.d[1], x13
+; CHECK-GI-NOFP16-NEXT:    mov v1.d[1], x14
+; CHECK-GI-NOFP16-NEXT:    stp q0, q3, [x8, #96]
+; CHECK-GI-NOFP16-NEXT:    stp q4, q1, [x8, #32]
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fptou_v32f16_v32i64:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    ext v4.16b, v1.16b, v1.16b, #8
+; CHECK-GI-FP16-NEXT:    ext v5.16b, v2.16b, v2.16b, #8
+; CHECK-GI-FP16-NEXT:    ext v6.16b, v3.16b, v3.16b, #8
+; CHECK-GI-FP16-NEXT:    mov h16, v3.h[2]
+; CHECK-GI-FP16-NEXT:    fcvtzu x9, h0
+; CHECK-GI-FP16-NEXT:    mov h23, v3.h[3]
+; CHECK-GI-FP16-NEXT:    mov h25, v3.h[1]
+; CHECK-GI-FP16-NEXT:    fcvtzu x15, h3
+; CHECK-GI-FP16-NEXT:    mov h24, v2.h[2]
+; CHECK-GI-FP16-NEXT:    mov h19, v1.h[2]
+; CHECK-GI-FP16-NEXT:    mov h21, v2.h[1]
+; CHECK-GI-FP16-NEXT:    mov h26, v2.h[3]
+; CHECK-GI-FP16-NEXT:    mov h17, v4.h[2]
+; CHECK-GI-FP16-NEXT:    mov h18, v5.h[2]
+; CHECK-GI-FP16-NEXT:    mov h22, v6.h[2]
+; CHECK-GI-FP16-NEXT:    fcvtzu x10, h5
+; CHECK-GI-FP16-NEXT:    fcvtzu x12, h16
+; CHECK-GI-FP16-NEXT:    fcvtzu x11, h6
+; CHECK-GI-FP16-NEXT:    mov h7, v1.h[1]
+; CHECK-GI-FP16-NEXT:    mov h20, v1.h[3]
+; CHECK-GI-FP16-NEXT:    fcvtzu x13, h17
+; CHECK-GI-FP16-NEXT:    fcvtzu x14, h18
+; CHECK-GI-FP16-NEXT:    fmov d18, x9
+; CHECK-GI-FP16-NEXT:    fcvtzu x9, h22
+; CHECK-GI-FP16-NEXT:    fmov d3, x10
+; CHECK-GI-FP16-NEXT:    fcvtzu x10, h23
+; CHECK-GI-FP16-NEXT:    fmov d22, x12
+; CHECK-GI-FP16-NEXT:    fcvtzu x12, h25
+; CHECK-GI-FP16-NEXT:    fmov d23, x15
+; CHECK-GI-FP16-NEXT:    fmov d16, x11
+; CHECK-GI-FP16-NEXT:    fcvtzu x11, h2
+; CHECK-GI-FP16-NEXT:    fcvtzu x15, h21
+; CHECK-GI-FP16-NEXT:    fmov d2, x13
+; CHECK-GI-FP16-NEXT:    fcvtzu x13, h24
+; CHECK-GI-FP16-NEXT:    fmov d17, x14
+; CHECK-GI-FP16-NEXT:    fcvtzu x14, h19
+; CHECK-GI-FP16-NEXT:    mov v22.d[1], x10
+; CHECK-GI-FP16-NEXT:    fcvtzu x10, h1
+; CHECK-GI-FP16-NEXT:    mov v23.d[1], x12
+; CHECK-GI-FP16-NEXT:    fmov d19, x9
+; CHECK-GI-FP16-NEXT:    fcvtzu x9, h26
+; CHECK-GI-FP16-NEXT:    fcvtzu x12, h20
+; CHECK-GI-FP16-NEXT:    mov h20, v0.h[2]
+; CHECK-GI-FP16-NEXT:    fmov d21, x11
+; CHECK-GI-FP16-NEXT:    fmov d1, x13
+; CHECK-GI-FP16-NEXT:    fcvtzu x13, h7
+; CHECK-GI-FP16-NEXT:    mov h24, v0.h[3]
+; CHECK-GI-FP16-NEXT:    fmov d7, x14
+; CHECK-GI-FP16-NEXT:    stp q23, q22, [x8, #192]
+; CHECK-GI-FP16-NEXT:    fmov d22, x10
+; CHECK-GI-FP16-NEXT:    mov v21.d[1], x15
+; CHECK-GI-FP16-NEXT:    mov v1.d[1], x9
+; CHECK-GI-FP16-NEXT:    mov h23, v0.h[1]
+; CHECK-GI-FP16-NEXT:    fcvtzu x9, h20
+; CHECK-GI-FP16-NEXT:    mov v7.d[1], x12
+; CHECK-GI-FP16-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-GI-FP16-NEXT:    mov h20, v6.h[3]
+; CHECK-GI-FP16-NEXT:    mov v22.d[1], x13
+; CHECK-GI-FP16-NEXT:    mov h6, v6.h[1]
+; CHECK-GI-FP16-NEXT:    fcvtzu x10, h24
+; CHECK-GI-FP16-NEXT:    stp q21, q1, [x8, #128]
+; CHECK-GI-FP16-NEXT:    mov h1, v5.h[1]
+; CHECK-GI-FP16-NEXT:    mov h5, v5.h[3]
+; CHECK-GI-FP16-NEXT:    fcvtzu x12, h20
+; CHECK-GI-FP16-NEXT:    mov h20, v0.h[2]
+; CHECK-GI-FP16-NEXT:    fcvtzu x11, h0
+; CHECK-GI-FP16-NEXT:    stp q22, q7, [x8, #64]
+; CHECK-GI-FP16-NEXT:    fmov d7, x9
+; CHECK-GI-FP16-NEXT:    fcvtzu x9, h23
+; CHECK-GI-FP16-NEXT:    mov h21, v4.h[3]
+; CHECK-GI-FP16-NEXT:    mov h22, v4.h[1]
+; CHECK-GI-FP16-NEXT:    fcvtzu x13, h6
+; CHECK-GI-FP16-NEXT:    mov h6, v0.h[3]
+; CHECK-GI-FP16-NEXT:    fcvtzu x14, h5
+; CHECK-GI-FP16-NEXT:    mov h0, v0.h[1]
+; CHECK-GI-FP16-NEXT:    mov v7.d[1], x10
+; CHECK-GI-FP16-NEXT:    fcvtzu x10, h1
+; CHECK-GI-FP16-NEXT:    mov v19.d[1], x12
+; CHECK-GI-FP16-NEXT:    mov v18.d[1], x9
+; CHECK-GI-FP16-NEXT:    fcvtzu x9, h4
+; CHECK-GI-FP16-NEXT:    fcvtzu x12, h20
+; CHECK-GI-FP16-NEXT:    fcvtzu x15, h21
+; CHECK-GI-FP16-NEXT:    mov v16.d[1], x13
+; CHECK-GI-FP16-NEXT:    fcvtzu x13, h22
+; CHECK-GI-FP16-NEXT:    mov v17.d[1], x14
+; CHECK-GI-FP16-NEXT:    fcvtzu x14, h6
+; CHECK-GI-FP16-NEXT:    fmov d4, x11
+; CHECK-GI-FP16-NEXT:    mov v3.d[1], x10
+; CHECK-GI-FP16-NEXT:    fcvtzu x10, h0
+; CHECK-GI-FP16-NEXT:    stp q18, q7, [x8]
+; CHECK-GI-FP16-NEXT:    fmov d0, x9
+; CHECK-GI-FP16-NEXT:    fmov d1, x12
+; CHECK-GI-FP16-NEXT:    stp q16, q19, [x8, #224]
+; CHECK-GI-FP16-NEXT:    mov v2.d[1], x15
+; CHECK-GI-FP16-NEXT:    stp q3, q17, [x8, #160]
+; CHECK-GI-FP16-NEXT:    mov v0.d[1], x13
+; CHECK-GI-FP16-NEXT:    mov v1.d[1], x14
+; CHECK-GI-FP16-NEXT:    mov v4.d[1], x10
+; CHECK-GI-FP16-NEXT:    stp q0, q2, [x8, #96]
+; CHECK-GI-FP16-NEXT:    stp q4, q1, [x8, #32]
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %c = fptoui <32 x half> %a to <32 x i64>
+  ret <32 x i64> %c
+}
+
+define <2 x i32> @fptos_v2f16_v2i32(<2 x half> %a) {
+; CHECK-LABEL: fptos_v2f16_v2i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <2 x half> %a to <2 x i32>
+  ret <2 x i32> %c
+}
+
+define <2 x i32> @fptou_v2f16_v2i32(<2 x half> %a) {
+; CHECK-LABEL: fptou_v2f16_v2i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <2 x half> %a to <2 x i32>
+  ret <2 x i32> %c
+}
+
+define <3 x i32> @fptos_v3f16_v3i32(<3 x half> %a) {
+; CHECK-LABEL: fptos_v3f16_v3i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <3 x half> %a to <3 x i32>
+  ret <3 x i32> %c
+}
+
+define <3 x i32> @fptou_v3f16_v3i32(<3 x half> %a) {
+; CHECK-LABEL: fptou_v3f16_v3i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <3 x half> %a to <3 x i32>
+  ret <3 x i32> %c
+}
+
+define <4 x i32> @fptos_v4f16_v4i32(<4 x half> %a) {
+; CHECK-LABEL: fptos_v4f16_v4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <4 x half> %a to <4 x i32>
+  ret <4 x i32> %c
+}
+
+define <4 x i32> @fptou_v4f16_v4i32(<4 x half> %a) {
+; CHECK-LABEL: fptou_v4f16_v4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <4 x half> %a to <4 x i32>
+  ret <4 x i32> %c
+}
+
+define <8 x i32> @fptos_v8f16_v8i32(<8 x half> %a) {
+; CHECK-LABEL: fptos_v8f16_v8i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtl2 v1.4s, v0.8h
+; CHECK-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <8 x half> %a to <8 x i32>
+  ret <8 x i32> %c
+}
+
+define <8 x i32> @fptou_v8f16_v8i32(<8 x half> %a) {
+; CHECK-LABEL: fptou_v8f16_v8i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtl2 v1.4s, v0.8h
+; CHECK-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <8 x half> %a to <8 x i32>
+  ret <8 x i32> %c
+}
+
+define <16 x i32> @fptos_v16f16_v16i32(<16 x half> %a) {
+; CHECK-LABEL: fptos_v16f16_v16i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-NEXT:    fcvtl2 v3.4s, v0.8h
+; CHECK-NEXT:    fcvtl2 v4.4s, v1.8h
+; CHECK-NEXT:    fcvtl v5.4s, v1.4h
+; CHECK-NEXT:    fcvtzs v0.4s, v2.4s
+; CHECK-NEXT:    fcvtzs v1.4s, v3.4s
+; CHECK-NEXT:    fcvtzs v3.4s, v4.4s
+; CHECK-NEXT:    fcvtzs v2.4s, v5.4s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <16 x half> %a to <16 x i32>
+  ret <16 x i32> %c
+}
+
+define <16 x i32> @fptou_v16f16_v16i32(<16 x half> %a) {
+; CHECK-LABEL: fptou_v16f16_v16i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-NEXT:    fcvtl2 v3.4s, v0.8h
+; CHECK-NEXT:    fcvtl2 v4.4s, v1.8h
+; CHECK-NEXT:    fcvtl v5.4s, v1.4h
+; CHECK-NEXT:    fcvtzu v0.4s, v2.4s
+; CHECK-NEXT:    fcvtzu v1.4s, v3.4s
+; CHECK-NEXT:    fcvtzu v3.4s, v4.4s
+; CHECK-NEXT:    fcvtzu v2.4s, v5.4s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <16 x half> %a to <16 x i32>
+  ret <16 x i32> %c
+}
+
+define <32 x i32> @fptos_v32f16_v32i32(<32 x half> %a) {
+; CHECK-LABEL: fptos_v32f16_v32i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtl2 v4.4s, v0.8h
+; CHECK-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NEXT:    fcvtl2 v5.4s, v1.8h
+; CHECK-NEXT:    fcvtl v6.4s, v1.4h
+; CHECK-NEXT:    fcvtl v7.4s, v2.4h
+; CHECK-NEXT:    fcvtl2 v16.4s, v2.8h
+; CHECK-NEXT:    fcvtl2 v17.4s, v3.8h
+; CHECK-NEXT:    fcvtl v18.4s, v3.4h
+; CHECK-NEXT:    fcvtzs v1.4s, v4.4s
+; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NEXT:    fcvtzs v3.4s, v5.4s
+; CHECK-NEXT:    fcvtzs v2.4s, v6.4s
+; CHECK-NEXT:    fcvtzs v4.4s, v7.4s
+; CHECK-NEXT:    fcvtzs v5.4s, v16.4s
+; CHECK-NEXT:    fcvtzs v7.4s, v17.4s
+; CHECK-NEXT:    fcvtzs v6.4s, v18.4s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <32 x half> %a to <32 x i32>
+  ret <32 x i32> %c
+}
+
+define <32 x i32> @fptou_v32f16_v32i32(<32 x half> %a) {
+; CHECK-LABEL: fptou_v32f16_v32i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtl2 v4.4s, v0.8h
+; CHECK-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NEXT:    fcvtl2 v5.4s, v1.8h
+; CHECK-NEXT:    fcvtl v6.4s, v1.4h
+; CHECK-NEXT:    fcvtl v7.4s, v2.4h
+; CHECK-NEXT:    fcvtl2 v16.4s, v2.8h
+; CHECK-NEXT:    fcvtl2 v17.4s, v3.8h
+; CHECK-NEXT:    fcvtl v18.4s, v3.4h
+; CHECK-NEXT:    fcvtzu v1.4s, v4.4s
+; CHECK-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NEXT:    fcvtzu v3.4s, v5.4s
+; CHECK-NEXT:    fcvtzu v2.4s, v6.4s
+; CHECK-NEXT:    fcvtzu v4.4s, v7.4s
+; CHECK-NEXT:    fcvtzu v5.4s, v16.4s
+; CHECK-NEXT:    fcvtzu v7.4s, v17.4s
+; CHECK-NEXT:    fcvtzu v6.4s, v18.4s
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <32 x half> %a to <32 x i32>
+  ret <32 x i32> %c
+}
+
+define <2 x i16> @fptos_v2f16_v2i16(<2 x half> %a) {
+; CHECK-LABEL: fptos_v2f16_v2i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <2 x half> %a to <2 x i16>
+  ret <2 x i16> %c
+}
+
+define <2 x i16> @fptou_v2f16_v2i16(<2 x half> %a) {
+; CHECK-LABEL: fptou_v2f16_v2i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <2 x half> %a to <2 x i16>
+  ret <2 x i16> %c
+}
+
+define <3 x i16> @fptos_v3f16_v3i16(<3 x half> %a) {
+; CHECK-SD-NOFP16-LABEL: fptos_v3f16_v3i16:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    xtn v0.4h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptos_v3f16_v3i16:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fcvtzs v0.4h, v0.4h
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fptos_v3f16_v3i16:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-GI-NOFP16-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fptos_v3f16_v3i16:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    fcvtzs v0.4h, v0.4h
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %c = fptosi <3 x half> %a to <3 x i16>
+  ret <3 x i16> %c
+}
+
+define <3 x i16> @fptou_v3f16_v3i16(<3 x half> %a) {
+; CHECK-SD-NOFP16-LABEL: fptou_v3f16_v3i16:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    xtn v0.4h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptou_v3f16_v3i16:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fcvtzu v0.4h, v0.4h
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fptou_v3f16_v3i16:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-GI-NOFP16-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fptou_v3f16_v3i16:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    fcvtzu v0.4h, v0.4h
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %c = fptoui <3 x half> %a to <3 x i16>
+  ret <3 x i16> %c
+}
+
+define <4 x i16> @fptos_v4f16_v4i16(<4 x half> %a) {
+; CHECK-SD-NOFP16-LABEL: fptos_v4f16_v4i16:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    xtn v0.4h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptos_v4f16_v4i16:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fcvtzs v0.4h, v0.4h
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fptos_v4f16_v4i16:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-GI-NOFP16-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fptos_v4f16_v4i16:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    fcvtzs v0.4h, v0.4h
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %c = fptosi <4 x half> %a to <4 x i16>
+  ret <4 x i16> %c
+}
+
+define <4 x i16> @fptou_v4f16_v4i16(<4 x half> %a) {
+; CHECK-SD-NOFP16-LABEL: fptou_v4f16_v4i16:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    xtn v0.4h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptou_v4f16_v4i16:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fcvtzu v0.4h, v0.4h
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fptou_v4f16_v4i16:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-GI-NOFP16-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fptou_v4f16_v4i16:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    fcvtzu v0.4h, v0.4h
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %c = fptoui <4 x half> %a to <4 x i16>
+  ret <4 x i16> %c
+}
+
+define <8 x i16> @fptos_v8f16_v8i16(<8 x half> %a) {
+; CHECK-SD-NOFP16-LABEL: fptos_v8f16_v8i16:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptos_v8f16_v8i16:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fcvtzs v0.8h, v0.8h
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fptos_v8f16_v8i16:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.4s, v0.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-GI-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fptos_v8f16_v8i16:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    fcvtzs v0.8h, v0.8h
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %c = fptosi <8 x half> %a to <8 x i16>
+  ret <8 x i16> %c
+}
+
+define <8 x i16> @fptou_v8f16_v8i16(<8 x half> %a) {
+; CHECK-SD-NOFP16-LABEL: fptou_v8f16_v8i16:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptou_v8f16_v8i16:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fcvtzu v0.8h, v0.8h
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fptou_v8f16_v8i16:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.4s, v0.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-GI-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fptou_v8f16_v8i16:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    fcvtzu v0.8h, v0.8h
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %c = fptoui <8 x half> %a to <8 x i16>
+  ret <8 x i16> %c
+}
+
+define <16 x i16> @fptos_v16f16_v16i16(<16 x half> %a) {
+; CHECK-SD-NOFP16-LABEL: fptos_v16f16_v16i16:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v3.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-SD-NOFP16-NEXT:    uzp1 v1.8h, v1.8h, v3.8h
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptos_v16f16_v16i16:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fcvtzs v0.8h, v0.8h
+; CHECK-SD-FP16-NEXT:    fcvtzs v1.8h, v1.8h
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fptos_v16f16_v16i16:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v3.4s, v1.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-GI-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-GI-NOFP16-NEXT:    uzp1 v1.8h, v1.8h, v3.8h
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fptos_v16f16_v16i16:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    fcvtzs v0.8h, v0.8h
+; CHECK-GI-FP16-NEXT:    fcvtzs v1.8h, v1.8h
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %c = fptosi <16 x half> %a to <16 x i16>
+  ret <16 x i16> %c
+}
+
+define <16 x i16> @fptou_v16f16_v16i16(<16 x half> %a) {
+; CHECK-SD-NOFP16-LABEL: fptou_v16f16_v16i16:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v3.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtzu v2.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzu v3.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-SD-NOFP16-NEXT:    uzp1 v1.8h, v1.8h, v3.8h
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptou_v16f16_v16i16:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fcvtzu v0.8h, v0.8h
+; CHECK-SD-FP16-NEXT:    fcvtzu v1.8h, v1.8h
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fptou_v16f16_v16i16:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v3.4s, v1.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtzu v2.4s, v2.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzu v3.4s, v3.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-GI-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-GI-NOFP16-NEXT:    uzp1 v1.8h, v1.8h, v3.8h
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fptou_v16f16_v16i16:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    fcvtzu v0.8h, v0.8h
+; CHECK-GI-FP16-NEXT:    fcvtzu v1.8h, v1.8h
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %c = fptoui <16 x half> %a to <16 x i16>
+  ret <16 x i16> %c
+}
+
+define <32 x i16> @fptos_v32f16_v32i16(<32 x half> %a) {
+; CHECK-SD-NOFP16-LABEL: fptos_v32f16_v32i16:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v4.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v5.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v6.4s, v2.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v2.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v7.4s, v3.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v3.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtzs v4.4s, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzs v5.4s, v5.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzs v6.4s, v6.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzs v7.4s, v7.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v4.8h
+; CHECK-SD-NOFP16-NEXT:    uzp1 v1.8h, v1.8h, v5.8h
+; CHECK-SD-NOFP16-NEXT:    uzp1 v2.8h, v2.8h, v6.8h
+; CHECK-SD-NOFP16-NEXT:    uzp1 v3.8h, v3.8h, v7.8h
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptos_v32f16_v32i16:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fcvtzs v0.8h, v0.8h
+; CHECK-SD-FP16-NEXT:    fcvtzs v1.8h, v1.8h
+; CHECK-SD-FP16-NEXT:    fcvtzs v2.8h, v2.8h
+; CHECK-SD-FP16-NEXT:    fcvtzs v3.8h, v3.8h
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fptos_v32f16_v32i16:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v4.4s, v0.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v5.4s, v1.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v6.4s, v2.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v2.4s, v2.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v7.4s, v3.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v3.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtzs v4.4s, v4.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzs v5.4s, v5.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzs v6.4s, v6.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzs v7.4s, v7.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-GI-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v4.8h
+; CHECK-GI-NOFP16-NEXT:    uzp1 v1.8h, v1.8h, v5.8h
+; CHECK-GI-NOFP16-NEXT:    uzp1 v2.8h, v2.8h, v6.8h
+; CHECK-GI-NOFP16-NEXT:    uzp1 v3.8h, v3.8h, v7.8h
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fptos_v32f16_v32i16:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    fcvtzs v0.8h, v0.8h
+; CHECK-GI-FP16-NEXT:    fcvtzs v1.8h, v1.8h
+; CHECK-GI-FP16-NEXT:    fcvtzs v2.8h, v2.8h
+; CHECK-GI-FP16-NEXT:    fcvtzs v3.8h, v3.8h
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %c = fptosi <32 x half> %a to <32 x i16>
+  ret <32 x i16> %c
+}
+
+define <32 x i16> @fptou_v32f16_v32i16(<32 x half> %a) {
+; CHECK-SD-NOFP16-LABEL: fptou_v32f16_v32i16:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v4.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v5.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v6.4s, v2.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v2.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v7.4s, v3.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v3.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtzu v4.4s, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzu v5.4s, v5.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzu v6.4s, v6.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzu v2.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzu v7.4s, v7.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzu v3.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v4.8h
+; CHECK-SD-NOFP16-NEXT:    uzp1 v1.8h, v1.8h, v5.8h
+; CHECK-SD-NOFP16-NEXT:    uzp1 v2.8h, v2.8h, v6.8h
+; CHECK-SD-NOFP16-NEXT:    uzp1 v3.8h, v3.8h, v7.8h
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptou_v32f16_v32i16:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fcvtzu v0.8h, v0.8h
+; CHECK-SD-FP16-NEXT:    fcvtzu v1.8h, v1.8h
+; CHECK-SD-FP16-NEXT:    fcvtzu v2.8h, v2.8h
+; CHECK-SD-FP16-NEXT:    fcvtzu v3.8h, v3.8h
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fptou_v32f16_v32i16:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v4.4s, v0.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v5.4s, v1.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v6.4s, v2.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v2.4s, v2.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v7.4s, v3.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v3.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtzu v4.4s, v4.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzu v5.4s, v5.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzu v6.4s, v6.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzu v2.4s, v2.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzu v7.4s, v7.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzu v3.4s, v3.4s
+; CHECK-GI-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v4.8h
+; CHECK-GI-NOFP16-NEXT:    uzp1 v1.8h, v1.8h, v5.8h
+; CHECK-GI-NOFP16-NEXT:    uzp1 v2.8h, v2.8h, v6.8h
+; CHECK-GI-NOFP16-NEXT:    uzp1 v3.8h, v3.8h, v7.8h
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fptou_v32f16_v32i16:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    fcvtzu v0.8h, v0.8h
+; CHECK-GI-FP16-NEXT:    fcvtzu v1.8h, v1.8h
+; CHECK-GI-FP16-NEXT:    fcvtzu v2.8h, v2.8h
+; CHECK-GI-FP16-NEXT:    fcvtzu v3.8h, v3.8h
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %c = fptoui <32 x half> %a to <32 x i16>
+  ret <32 x i16> %c
+}
+
+define <2 x i8> @fptos_v2f16_v2i8(<2 x half> %a) {
+; CHECK-LABEL: fptos_v2f16_v2i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi <2 x half> %a to <2 x i8>
+  ret <2 x i8> %c
+}
+
+define <2 x i8> @fptou_v2f16_v2i8(<2 x half> %a) {
+; CHECK-LABEL: fptou_v2f16_v2i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui <2 x half> %a to <2 x i8>
+  ret <2 x i8> %c
+}
+
+define <3 x i8> @fptos_v3f16_v3i8(<3 x half> %a) {
+; CHECK-SD-NOFP16-LABEL: fptos_v3f16_v3i8:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    xtn v0.4h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    umov w0, v0.h[0]
+; CHECK-SD-NOFP16-NEXT:    umov w1, v0.h[1]
+; CHECK-SD-NOFP16-NEXT:    umov w2, v0.h[2]
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptos_v3f16_v3i8:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fcvtzs v0.4h, v0.4h
+; CHECK-SD-FP16-NEXT:    umov w0, v0.h[0]
+; CHECK-SD-FP16-NEXT:    umov w1, v0.h[1]
+; CHECK-SD-FP16-NEXT:    umov w2, v0.h[2]
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fptos_v3f16_v3i8:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-GI-NOFP16-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NOFP16-NEXT:    umov w0, v0.h[0]
+; CHECK-GI-NOFP16-NEXT:    umov w1, v0.h[1]
+; CHECK-GI-NOFP16-NEXT:    umov w2, v0.h[2]
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fptos_v3f16_v3i8:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    fcvtzs v0.4h, v0.4h
+; CHECK-GI-FP16-NEXT:    umov w0, v0.h[0]
+; CHECK-GI-FP16-NEXT:    umov w1, v0.h[1]
+; CHECK-GI-FP16-NEXT:    umov w2, v0.h[2]
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %c = fptosi <3 x half> %a to <3 x i8>
+  ret <3 x i8> %c
+}
+
+define <3 x i8> @fptou_v3f16_v3i8(<3 x half> %a) {
+; CHECK-SD-NOFP16-LABEL: fptou_v3f16_v3i8:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    xtn v0.4h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    umov w0, v0.h[0]
+; CHECK-SD-NOFP16-NEXT:    umov w1, v0.h[1]
+; CHECK-SD-NOFP16-NEXT:    umov w2, v0.h[2]
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptou_v3f16_v3i8:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fcvtzs v0.4h, v0.4h
+; CHECK-SD-FP16-NEXT:    umov w0, v0.h[0]
+; CHECK-SD-FP16-NEXT:    umov w1, v0.h[1]
+; CHECK-SD-FP16-NEXT:    umov w2, v0.h[2]
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fptou_v3f16_v3i8:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-GI-NOFP16-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NOFP16-NEXT:    umov w0, v0.h[0]
+; CHECK-GI-NOFP16-NEXT:    umov w1, v0.h[1]
+; CHECK-GI-NOFP16-NEXT:    umov w2, v0.h[2]
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fptou_v3f16_v3i8:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    fcvtzs v0.4h, v0.4h
+; CHECK-GI-FP16-NEXT:    umov w0, v0.h[0]
+; CHECK-GI-FP16-NEXT:    umov w1, v0.h[1]
+; CHECK-GI-FP16-NEXT:    umov w2, v0.h[2]
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %c = fptoui <3 x half> %a to <3 x i8>
+  ret <3 x i8> %c
+}
+
+define <4 x i8> @fptos_v4f16_v4i8(<4 x half> %a) {
+; CHECK-SD-NOFP16-LABEL: fptos_v4f16_v4i8:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    xtn v0.4h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptos_v4f16_v4i8:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fcvtzs v0.4h, v0.4h
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fptos_v4f16_v4i8:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-GI-NOFP16-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fptos_v4f16_v4i8:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    fcvtzs v0.4h, v0.4h
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %c = fptosi <4 x half> %a to <4 x i8>
+  ret <4 x i8> %c
+}
+
+define <4 x i8> @fptou_v4f16_v4i8(<4 x half> %a) {
+; CHECK-SD-NOFP16-LABEL: fptou_v4f16_v4i8:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    xtn v0.4h, v0.4s
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptou_v4f16_v4i8:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fcvtzs v0.4h, v0.4h
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fptou_v4f16_v4i8:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-GI-NOFP16-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fptou_v4f16_v4i8:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    fcvtzs v0.4h, v0.4h
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %c = fptoui <4 x half> %a to <4 x i8>
+  ret <4 x i8> %c
+}
+
+define <8 x i8> @fptos_v8f16_v8i8(<8 x half> %a) {
+; CHECK-SD-NOFP16-LABEL: fptos_v8f16_v8i8:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-SD-NOFP16-NEXT:    xtn v0.8b, v0.8h
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptos_v8f16_v8i8:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fcvtzs v0.8h, v0.8h
+; CHECK-SD-FP16-NEXT:    xtn v0.8b, v0.8h
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fptos_v8f16_v8i8:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.4s, v0.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-GI-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-GI-NOFP16-NEXT:    xtn v0.8b, v0.8h
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fptos_v8f16_v8i8:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    fcvtzs v0.8h, v0.8h
+; CHECK-GI-FP16-NEXT:    xtn v0.8b, v0.8h
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %c = fptosi <8 x half> %a to <8 x i8>
+  ret <8 x i8> %c
+}
+
+define <8 x i8> @fptou_v8f16_v8i8(<8 x half> %a) {
+; CHECK-SD-NOFP16-LABEL: fptou_v8f16_v8i8:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-SD-NOFP16-NEXT:    xtn v0.8b, v0.8h
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptou_v8f16_v8i8:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fcvtzu v0.8h, v0.8h
+; CHECK-SD-FP16-NEXT:    xtn v0.8b, v0.8h
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fptou_v8f16_v8i8:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.4s, v0.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-GI-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-GI-NOFP16-NEXT:    xtn v0.8b, v0.8h
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fptou_v8f16_v8i8:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    fcvtzu v0.8h, v0.8h
+; CHECK-GI-FP16-NEXT:    xtn v0.8b, v0.8h
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %c = fptoui <8 x half> %a to <8 x i8>
+  ret <8 x i8> %c
+}
+
+define <16 x i8> @fptos_v16f16_v16i8(<16 x half> %a) {
+; CHECK-SD-NOFP16-LABEL: fptos_v16f16_v16i8:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v3.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    uzp1 v1.8h, v1.8h, v2.8h
+; CHECK-SD-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v3.8h
+; CHECK-SD-NOFP16-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptos_v16f16_v16i8:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fcvtzs v1.8h, v1.8h
+; CHECK-SD-FP16-NEXT:    fcvtzs v0.8h, v0.8h
+; CHECK-SD-FP16-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fptos_v16f16_v16i8:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v2.4s, v1.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v3.4s, v0.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-GI-NOFP16-NEXT:    uzp1 v1.8h, v1.8h, v2.8h
+; CHECK-GI-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v3.8h
+; CHECK-GI-NOFP16-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fptos_v16f16_v16i8:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    fcvtzs v1.8h, v1.8h
+; CHECK-GI-FP16-NEXT:    fcvtzs v0.8h, v0.8h
+; CHECK-GI-FP16-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %c = fptosi <16 x half> %a to <16 x i8>
+  ret <16 x i8> %c
+}
+
+define <16 x i8> @fptou_v16f16_v16i8(<16 x half> %a) {
+; CHECK-SD-NOFP16-LABEL: fptou_v16f16_v16i8:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v3.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtzu v2.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzu v3.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    uzp1 v1.8h, v1.8h, v2.8h
+; CHECK-SD-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v3.8h
+; CHECK-SD-NOFP16-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptou_v16f16_v16i8:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fcvtzu v1.8h, v1.8h
+; CHECK-SD-FP16-NEXT:    fcvtzu v0.8h, v0.8h
+; CHECK-SD-FP16-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fptou_v16f16_v16i8:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v2.4s, v1.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v3.4s, v0.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtzu v2.4s, v2.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzu v3.4s, v3.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-GI-NOFP16-NEXT:    uzp1 v1.8h, v1.8h, v2.8h
+; CHECK-GI-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v3.8h
+; CHECK-GI-NOFP16-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fptou_v16f16_v16i8:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    fcvtzu v1.8h, v1.8h
+; CHECK-GI-FP16-NEXT:    fcvtzu v0.8h, v0.8h
+; CHECK-GI-FP16-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %c = fptoui <16 x half> %a to <16 x i8>
+  ret <16 x i8> %c
+}
+
+define <32 x i8> @fptos_v32f16_v32i8(<32 x half> %a) {
+; CHECK-SD-NOFP16-LABEL: fptos_v32f16_v32i8:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v4.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v5.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v6.4s, v3.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v3.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v7.4s, v2.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v2.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtzs v4.4s, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzs v5.4s, v5.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzs v6.4s, v6.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzs v7.4s, v7.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    uzp1 v1.8h, v1.8h, v4.8h
+; CHECK-SD-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v5.8h
+; CHECK-SD-NOFP16-NEXT:    uzp1 v3.8h, v3.8h, v6.8h
+; CHECK-SD-NOFP16-NEXT:    uzp1 v2.8h, v2.8h, v7.8h
+; CHECK-SD-NOFP16-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    uzp1 v1.16b, v2.16b, v3.16b
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptos_v32f16_v32i8:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fcvtzs v1.8h, v1.8h
+; CHECK-SD-FP16-NEXT:    fcvtzs v0.8h, v0.8h
+; CHECK-SD-FP16-NEXT:    fcvtzs v3.8h, v3.8h
+; CHECK-SD-FP16-NEXT:    fcvtzs v2.8h, v2.8h
+; CHECK-SD-FP16-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-SD-FP16-NEXT:    uzp1 v1.16b, v2.16b, v3.16b
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fptos_v32f16_v32i8:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v4.4s, v1.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v5.4s, v0.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v6.4s, v3.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v3.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v7.4s, v2.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v2.4s, v2.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtzs v4.4s, v4.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzs v5.4s, v5.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzs v6.4s, v6.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzs v7.4s, v7.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-GI-NOFP16-NEXT:    uzp1 v1.8h, v1.8h, v4.8h
+; CHECK-GI-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v5.8h
+; CHECK-GI-NOFP16-NEXT:    uzp1 v3.8h, v3.8h, v6.8h
+; CHECK-GI-NOFP16-NEXT:    uzp1 v2.8h, v2.8h, v7.8h
+; CHECK-GI-NOFP16-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-GI-NOFP16-NEXT:    uzp1 v1.16b, v2.16b, v3.16b
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fptos_v32f16_v32i8:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    fcvtzs v1.8h, v1.8h
+; CHECK-GI-FP16-NEXT:    fcvtzs v0.8h, v0.8h
+; CHECK-GI-FP16-NEXT:    fcvtzs v3.8h, v3.8h
+; CHECK-GI-FP16-NEXT:    fcvtzs v2.8h, v2.8h
+; CHECK-GI-FP16-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-GI-FP16-NEXT:    uzp1 v1.16b, v2.16b, v3.16b
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %c = fptosi <32 x half> %a to <32 x i8>
+  ret <32 x i8> %c
+}
+
+define <32 x i8> @fptou_v32f16_v32i8(<32 x half> %a) {
+; CHECK-SD-NOFP16-LABEL: fptou_v32f16_v32i8:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v4.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v5.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v6.4s, v3.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v3.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtl2 v7.4s, v2.8h
+; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v2.4h
+; CHECK-SD-NOFP16-NEXT:    fcvtzu v4.4s, v4.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzu v5.4s, v5.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzu v6.4s, v6.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzu v3.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzu v7.4s, v7.4s
+; CHECK-SD-NOFP16-NEXT:    fcvtzu v2.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT:    uzp1 v1.8h, v1.8h, v4.8h
+; CHECK-SD-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v5.8h
+; CHECK-SD-NOFP16-NEXT:    uzp1 v3.8h, v3.8h, v6.8h
+; CHECK-SD-NOFP16-NEXT:    uzp1 v2.8h, v2.8h, v7.8h
+; CHECK-SD-NOFP16-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-SD-NOFP16-NEXT:    uzp1 v1.16b, v2.16b, v3.16b
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fptou_v32f16_v32i8:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    fcvtzu v1.8h, v1.8h
+; CHECK-SD-FP16-NEXT:    fcvtzu v0.8h, v0.8h
+; CHECK-SD-FP16-NEXT:    fcvtzu v3.8h, v3.8h
+; CHECK-SD-FP16-NEXT:    fcvtzu v2.8h, v2.8h
+; CHECK-SD-FP16-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-SD-FP16-NEXT:    uzp1 v1.16b, v2.16b, v3.16b
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fptou_v32f16_v32i8:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v4.4s, v1.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v5.4s, v0.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v6.4s, v3.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v3.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v7.4s, v2.8h
+; CHECK-GI-NOFP16-NEXT:    fcvtl v2.4s, v2.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtzu v4.4s, v4.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzu v5.4s, v5.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzu v6.4s, v6.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzu v3.4s, v3.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzu v7.4s, v7.4s
+; CHECK-GI-NOFP16-NEXT:    fcvtzu v2.4s, v2.4s
+; CHECK-GI-NOFP16-NEXT:    uzp1 v1.8h, v1.8h, v4.8h
+; CHECK-GI-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v5.8h
+; CHECK-GI-NOFP16-NEXT:    uzp1 v3.8h, v3.8h, v6.8h
+; CHECK-GI-NOFP16-NEXT:    uzp1 v2.8h, v2.8h, v7.8h
+; CHECK-GI-NOFP16-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-GI-NOFP16-NEXT:    uzp1 v1.16b, v2.16b, v3.16b
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fptou_v32f16_v32i8:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    fcvtzu v1.8h, v1.8h
+; CHECK-GI-FP16-NEXT:    fcvtzu v0.8h, v0.8h
+; CHECK-GI-FP16-NEXT:    fcvtzu v3.8h, v3.8h
+; CHECK-GI-FP16-NEXT:    fcvtzu v2.8h, v2.8h
+; CHECK-GI-FP16-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-GI-FP16-NEXT:    uzp1 v1.16b, v2.16b, v3.16b
+; CHECK-GI-FP16-NEXT:    ret
+entry:
+  %c = fptoui <32 x half> %a to <32 x i8>
+  ret <32 x i8> %c
+}
diff --git a/llvm/test/CodeGen/AArch64/frem.ll b/llvm/test/CodeGen/AArch64/frem.ll
index 16a6ba3f8c..e8077b94f2 100644
--- a/llvm/test/CodeGen/AArch64/frem.ll
+++ b/llvm/test/CodeGen/AArch64/frem.ll
@@ -68,6 +68,7 @@ define <2 x double> @frem_v2f64(<2 x double> %a, <2 x double> %b) {
 ; CHECK-SD-NEXT:    mov d0, v0.d[1]
 ; CHECK-SD-NEXT:    mov d1, v1.d[1]
 ; CHECK-SD-NEXT:    bl fmod
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-SD-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldp q0, q1, [sp] // 32-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -94,9 +95,10 @@ define <2 x double> @frem_v2f64(<2 x double> %a, <2 x double> %b) {
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 killed $q1
 ; CHECK-GI-NEXT:    bl fmod
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    fmov d0, d8
 ; CHECK-GI-NEXT:    fmov d1, d9
+; CHECK-GI-NEXT:    fmov d0, d8
 ; CHECK-GI-NEXT:    bl fmod
 ; CHECK-GI-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
@@ -200,6 +202,7 @@ define <4 x double> @frem_v4f64(<4 x double> %a, <4 x double> %b) {
 ; CHECK-SD-NEXT:    stp q1, q3, [sp, #48] // 32-byte Folded Spill
 ; CHECK-SD-NEXT:    mov d1, v2.d[1]
 ; CHECK-SD-NEXT:    bl fmod
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-SD-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldp q0, q1, [sp] // 32-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -213,6 +216,7 @@ define <4 x double> @frem_v4f64(<4 x double> %a, <4 x double> %b) {
 ; CHECK-SD-NEXT:    mov d0, v0.d[1]
 ; CHECK-SD-NEXT:    mov d1, v1.d[1]
 ; CHECK-SD-NEXT:    bl fmod
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldp q0, q1, [sp, #48] // 32-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -248,19 +252,22 @@ define <4 x double> @frem_v4f64(<4 x double> %a, <4 x double> %b) {
 ; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 killed $q1
 ; CHECK-GI-NEXT:    mov d9, v4.d[1]
 ; CHECK-GI-NEXT:    bl fmod
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    str q0, [sp, #48] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    fmov d0, d8
 ; CHECK-GI-NEXT:    fmov d1, d10
+; CHECK-GI-NEXT:    fmov d0, d8
 ; CHECK-GI-NEXT:    bl fmod
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GI-NEXT:    bl fmod
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    fmov d0, d9
 ; CHECK-GI-NEXT:    fmov d1, d11
+; CHECK-GI-NEXT:    fmov d0, d9
 ; CHECK-GI-NEXT:    bl fmod
 ; CHECK-GI-NEXT:    ldr q2, [sp, #48] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
@@ -292,7 +299,8 @@ define <2 x float> @frem_v2f32(<2 x float> %a, <2 x float> %b) {
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    mov s1, v1.s[1]
 ; CHECK-SD-NEXT:    bl fmodf
-; CHECK-SD-NEXT:    str d0, [sp, #32] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldp q0, q1, [sp] // 32-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    // kill: def $s1 killed $s1 killed $q1
@@ -321,9 +329,10 @@ define <2 x float> @frem_v2f32(<2 x float> %a, <2 x float> %b) {
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-GI-NEXT:    // kill: def $s1 killed $s1 killed $q1
 ; CHECK-GI-NEXT:    bl fmodf
-; CHECK-GI-NEXT:    str d0, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    fmov s0, s8
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s1, s9
+; CHECK-GI-NEXT:    fmov s0, s8
 ; CHECK-GI-NEXT:    bl fmodf
 ; CHECK-GI-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
@@ -349,7 +358,8 @@ define <3 x float> @frem_v3f32(<3 x float> %a, <3 x float> %b) {
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    mov s1, v1.s[1]
 ; CHECK-SD-NEXT:    bl fmodf
-; CHECK-SD-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldp q0, q1, [sp, #16] // 32-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    // kill: def $s1 killed $s1 killed $q1
@@ -389,13 +399,15 @@ define <3 x float> @frem_v3f32(<3 x float> %a, <3 x float> %b) {
 ; CHECK-GI-NEXT:    mov s11, v1.s[2]
 ; CHECK-GI-NEXT:    // kill: def $s1 killed $s1 killed $q1
 ; CHECK-GI-NEXT:    bl fmodf
-; CHECK-GI-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    fmov s0, s8
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s1, s10
+; CHECK-GI-NEXT:    fmov s0, s8
 ; CHECK-GI-NEXT:    bl fmodf
-; CHECK-GI-NEXT:    str d0, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    fmov s0, s9
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s1, s11
+; CHECK-GI-NEXT:    fmov s0, s9
 ; CHECK-GI-NEXT:    bl fmodf
 ; CHECK-GI-NEXT:    ldp q2, q1, [sp] // 32-byte Folded Reload
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
@@ -424,7 +436,8 @@ define <4 x float> @frem_v4f32(<4 x float> %a, <4 x float> %b) {
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    mov s1, v1.s[1]
 ; CHECK-SD-NEXT:    bl fmodf
-; CHECK-SD-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldp q0, q1, [sp, #16] // 32-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    // kill: def $s1 killed $s1 killed $q1
@@ -477,17 +490,20 @@ define <4 x float> @frem_v4f32(<4 x float> %a, <4 x float> %b) {
 ; CHECK-GI-NEXT:    mov s13, v1.s[3]
 ; CHECK-GI-NEXT:    // kill: def $s1 killed $s1 killed $q1
 ; CHECK-GI-NEXT:    bl fmodf
-; CHECK-GI-NEXT:    str d0, [sp, #32] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    fmov s0, s8
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s1, s11
+; CHECK-GI-NEXT:    fmov s0, s8
 ; CHECK-GI-NEXT:    bl fmodf
-; CHECK-GI-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    fmov s0, s9
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s1, s12
+; CHECK-GI-NEXT:    fmov s0, s9
 ; CHECK-GI-NEXT:    bl fmodf
-; CHECK-GI-NEXT:    str d0, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    fmov s0, s10
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s1, s13
+; CHECK-GI-NEXT:    fmov s0, s10
 ; CHECK-GI-NEXT:    bl fmodf
 ; CHECK-GI-NEXT:    ldp q2, q1, [sp, #16] // 32-byte Folded Reload
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
@@ -519,7 +535,8 @@ define <8 x float> @frem_v8f32(<8 x float> %a, <8 x float> %b) {
 ; CHECK-SD-NEXT:    stp q1, q3, [sp, #32] // 32-byte Folded Spill
 ; CHECK-SD-NEXT:    mov s1, v2.s[1]
 ; CHECK-SD-NEXT:    bl fmodf
-; CHECK-SD-NEXT:    str d0, [sp, #64] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp, #64] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldp q0, q1, [sp] // 32-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    // kill: def $s1 killed $s1 killed $q1
@@ -548,7 +565,8 @@ define <8 x float> @frem_v8f32(<8 x float> %a, <8 x float> %b) {
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    mov s1, v1.s[1]
 ; CHECK-SD-NEXT:    bl fmodf
-; CHECK-SD-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldp q0, q1, [sp, #32] // 32-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    // kill: def $s1 killed $s1 killed $q1
@@ -616,32 +634,39 @@ define <8 x float> @frem_v8f32(<8 x float> %a, <8 x float> %b) {
 ; CHECK-GI-NEXT:    mov s2, v3.s[3]
 ; CHECK-GI-NEXT:    stp s2, s5, [sp, #200] // 8-byte Folded Spill
 ; CHECK-GI-NEXT:    bl fmodf
-; CHECK-GI-NEXT:    str d0, [sp, #96] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    fmov s0, s8
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #96] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s1, s14
+; CHECK-GI-NEXT:    fmov s0, s8
 ; CHECK-GI-NEXT:    bl fmodf
-; CHECK-GI-NEXT:    str d0, [sp, #32] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    fmov s0, s9
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s1, s15
+; CHECK-GI-NEXT:    fmov s0, s9
 ; CHECK-GI-NEXT:    bl fmodf
-; CHECK-GI-NEXT:    str d0, [sp, #80] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    fmov s0, s10
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #80] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s1, s13
+; CHECK-GI-NEXT:    fmov s0, s10
 ; CHECK-GI-NEXT:    bl fmodf
-; CHECK-GI-NEXT:    str d0, [sp, #64] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #64] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    ldp q1, q0, [sp] // 32-byte Folded Reload
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-GI-NEXT:    // kill: def $s1 killed $s1 killed $q1
 ; CHECK-GI-NEXT:    bl fmodf
 ; CHECK-GI-NEXT:    fmov s1, s12
-; CHECK-GI-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    ldr s0, [sp, #48] // 4-byte Folded Reload
 ; CHECK-GI-NEXT:    bl fmodf
 ; CHECK-GI-NEXT:    fmov s1, s11
-; CHECK-GI-NEXT:    str d0, [sp, #48] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #48] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    ldr s0, [sp, #112] // 4-byte Folded Reload
 ; CHECK-GI-NEXT:    bl fmodf
-; CHECK-GI-NEXT:    str d0, [sp, #112] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #112] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    ldp s1, s0, [sp, #200] // 8-byte Folded Reload
 ; CHECK-GI-NEXT:    bl fmodf
 ; CHECK-GI-NEXT:    ldp q3, q2, [sp, #16] // 32-byte Folded Reload
diff --git a/llvm/test/CodeGen/AArch64/fsincos.ll b/llvm/test/CodeGen/AArch64/fsincos.ll
index 361a1e996f..ec60fedeb4 100644
--- a/llvm/test/CodeGen/AArch64/fsincos.ll
+++ b/llvm/test/CodeGen/AArch64/fsincos.ll
@@ -64,6 +64,7 @@ define <2 x double> @sin_v2f64(<2 x double> %a) {
 ; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    mov d0, v0.d[1]
 ; CHECK-SD-NEXT:    bl sin
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -86,6 +87,7 @@ define <2 x double> @sin_v2f64(<2 x double> %a) {
 ; CHECK-GI-NEXT:    mov d8, v0.d[1]
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GI-NEXT:    bl sin
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov d0, d8
 ; CHECK-GI-NEXT:    bl sin
@@ -172,6 +174,7 @@ define <4 x double> @sin_v4f64(<4 x double> %a) {
 ; CHECK-SD-NEXT:    mov d0, v0.d[1]
 ; CHECK-SD-NEXT:    str q1, [sp, #32] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    bl sin
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -183,6 +186,7 @@ define <4 x double> @sin_v4f64(<4 x double> %a) {
 ; CHECK-SD-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    mov d0, v0.d[1]
 ; CHECK-SD-NEXT:    bl sin
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -208,13 +212,16 @@ define <4 x double> @sin_v4f64(<4 x double> %a) {
 ; CHECK-GI-NEXT:    mov d9, v1.d[1]
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GI-NEXT:    bl sin
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov d0, d8
 ; CHECK-GI-NEXT:    bl sin
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GI-NEXT:    bl sin
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov d0, d9
 ; CHECK-GI-NEXT:    bl sin
@@ -244,7 +251,8 @@ define <2 x float> @sin_v2f32(<2 x float> %a) {
 ; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    bl sinf
-; CHECK-SD-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    bl sinf
@@ -268,7 +276,8 @@ define <2 x float> @sin_v2f32(<2 x float> %a) {
 ; CHECK-GI-NEXT:    mov s8, v0.s[1]
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-GI-NEXT:    bl sinf
-; CHECK-GI-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s8
 ; CHECK-GI-NEXT:    bl sinf
 ; CHECK-GI-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
@@ -294,7 +303,8 @@ define <3 x float> @sin_v3f32(<3 x float> %a) {
 ; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    bl sinf
-; CHECK-SD-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    bl sinf
@@ -326,10 +336,12 @@ define <3 x float> @sin_v3f32(<3 x float> %a) {
 ; CHECK-GI-NEXT:    mov s9, v0.s[2]
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-GI-NEXT:    bl sinf
-; CHECK-GI-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s8
 ; CHECK-GI-NEXT:    bl sinf
-; CHECK-GI-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s9
 ; CHECK-GI-NEXT:    bl sinf
 ; CHECK-GI-NEXT:    ldp q2, q1, [sp] // 32-byte Folded Reload
@@ -357,7 +369,8 @@ define <4 x float> @sin_v4f32(<4 x float> %a) {
 ; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    bl sinf
-; CHECK-SD-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    bl sinf
@@ -399,13 +412,16 @@ define <4 x float> @sin_v4f32(<4 x float> %a) {
 ; CHECK-GI-NEXT:    mov s10, v0.s[3]
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-GI-NEXT:    bl sinf
-; CHECK-GI-NEXT:    str d0, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s8
 ; CHECK-GI-NEXT:    bl sinf
-; CHECK-GI-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s9
 ; CHECK-GI-NEXT:    bl sinf
-; CHECK-GI-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s10
 ; CHECK-GI-NEXT:    bl sinf
 ; CHECK-GI-NEXT:    ldp q2, q1, [sp, #16] // 32-byte Folded Reload
@@ -435,7 +451,8 @@ define <8 x float> @sin_v8f32(<8 x float> %a) {
 ; CHECK-SD-NEXT:    stp q0, q1, [sp] // 32-byte Folded Spill
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    bl sinf
-; CHECK-SD-NEXT:    str d0, [sp, #32] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    bl sinf
@@ -460,7 +477,8 @@ define <8 x float> @sin_v8f32(<8 x float> %a) {
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    str q1, [sp, #32] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    bl sinf
-; CHECK-SD-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    bl sinf
@@ -510,26 +528,33 @@ define <8 x float> @sin_v8f32(<8 x float> %a) {
 ; CHECK-GI-NEXT:    mov s12, v1.s[2]
 ; CHECK-GI-NEXT:    mov s13, v1.s[3]
 ; CHECK-GI-NEXT:    bl sinf
-; CHECK-GI-NEXT:    str d0, [sp, #64] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #64] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s8
 ; CHECK-GI-NEXT:    bl sinf
-; CHECK-GI-NEXT:    str d0, [sp, #48] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #48] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s9
 ; CHECK-GI-NEXT:    bl sinf
-; CHECK-GI-NEXT:    str d0, [sp, #96] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #96] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s10
 ; CHECK-GI-NEXT:    bl sinf
-; CHECK-GI-NEXT:    str d0, [sp, #80] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #80] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-GI-NEXT:    bl sinf
-; CHECK-GI-NEXT:    str d0, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s11
 ; CHECK-GI-NEXT:    bl sinf
-; CHECK-GI-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s12
 ; CHECK-GI-NEXT:    bl sinf
-; CHECK-GI-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s13
 ; CHECK-GI-NEXT:    bl sinf
 ; CHECK-GI-NEXT:    ldp q2, q1, [sp, #48] // 32-byte Folded Reload
@@ -1320,6 +1345,7 @@ define <2 x double> @cos_v2f64(<2 x double> %a) {
 ; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    mov d0, v0.d[1]
 ; CHECK-SD-NEXT:    bl cos
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -1342,6 +1368,7 @@ define <2 x double> @cos_v2f64(<2 x double> %a) {
 ; CHECK-GI-NEXT:    mov d8, v0.d[1]
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GI-NEXT:    bl cos
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov d0, d8
 ; CHECK-GI-NEXT:    bl cos
@@ -1428,6 +1455,7 @@ define <4 x double> @cos_v4f64(<4 x double> %a) {
 ; CHECK-SD-NEXT:    mov d0, v0.d[1]
 ; CHECK-SD-NEXT:    str q1, [sp, #32] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    bl cos
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -1439,6 +1467,7 @@ define <4 x double> @cos_v4f64(<4 x double> %a) {
 ; CHECK-SD-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    mov d0, v0.d[1]
 ; CHECK-SD-NEXT:    bl cos
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -1464,13 +1493,16 @@ define <4 x double> @cos_v4f64(<4 x double> %a) {
 ; CHECK-GI-NEXT:    mov d9, v1.d[1]
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GI-NEXT:    bl cos
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov d0, d8
 ; CHECK-GI-NEXT:    bl cos
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-GI-NEXT:    bl cos
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov d0, d9
 ; CHECK-GI-NEXT:    bl cos
@@ -1500,7 +1532,8 @@ define <2 x float> @cos_v2f32(<2 x float> %a) {
 ; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    bl cosf
-; CHECK-SD-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    bl cosf
@@ -1524,7 +1557,8 @@ define <2 x float> @cos_v2f32(<2 x float> %a) {
 ; CHECK-GI-NEXT:    mov s8, v0.s[1]
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-GI-NEXT:    bl cosf
-; CHECK-GI-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s8
 ; CHECK-GI-NEXT:    bl cosf
 ; CHECK-GI-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
@@ -1550,7 +1584,8 @@ define <3 x float> @cos_v3f32(<3 x float> %a) {
 ; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    bl cosf
-; CHECK-SD-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    bl cosf
@@ -1582,10 +1617,12 @@ define <3 x float> @cos_v3f32(<3 x float> %a) {
 ; CHECK-GI-NEXT:    mov s9, v0.s[2]
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-GI-NEXT:    bl cosf
-; CHECK-GI-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s8
 ; CHECK-GI-NEXT:    bl cosf
-; CHECK-GI-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s9
 ; CHECK-GI-NEXT:    bl cosf
 ; CHECK-GI-NEXT:    ldp q2, q1, [sp] // 32-byte Folded Reload
@@ -1613,7 +1650,8 @@ define <4 x float> @cos_v4f32(<4 x float> %a) {
 ; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    bl cosf
-; CHECK-SD-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    bl cosf
@@ -1655,13 +1693,16 @@ define <4 x float> @cos_v4f32(<4 x float> %a) {
 ; CHECK-GI-NEXT:    mov s10, v0.s[3]
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-GI-NEXT:    bl cosf
-; CHECK-GI-NEXT:    str d0, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s8
 ; CHECK-GI-NEXT:    bl cosf
-; CHECK-GI-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s9
 ; CHECK-GI-NEXT:    bl cosf
-; CHECK-GI-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s10
 ; CHECK-GI-NEXT:    bl cosf
 ; CHECK-GI-NEXT:    ldp q2, q1, [sp, #16] // 32-byte Folded Reload
@@ -1691,7 +1732,8 @@ define <8 x float> @cos_v8f32(<8 x float> %a) {
 ; CHECK-SD-NEXT:    stp q0, q1, [sp] // 32-byte Folded Spill
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    bl cosf
-; CHECK-SD-NEXT:    str d0, [sp, #32] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    bl cosf
@@ -1716,7 +1758,8 @@ define <8 x float> @cos_v8f32(<8 x float> %a) {
 ; CHECK-SD-NEXT:    mov s0, v0.s[1]
 ; CHECK-SD-NEXT:    str q1, [sp, #32] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    bl cosf
-; CHECK-SD-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-SD-NEXT:    bl cosf
@@ -1766,26 +1809,33 @@ define <8 x float> @cos_v8f32(<8 x float> %a) {
 ; CHECK-GI-NEXT:    mov s12, v1.s[2]
 ; CHECK-GI-NEXT:    mov s13, v1.s[3]
 ; CHECK-GI-NEXT:    bl cosf
-; CHECK-GI-NEXT:    str d0, [sp, #64] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #64] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s8
 ; CHECK-GI-NEXT:    bl cosf
-; CHECK-GI-NEXT:    str d0, [sp, #48] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #48] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s9
 ; CHECK-GI-NEXT:    bl cosf
-; CHECK-GI-NEXT:    str d0, [sp, #96] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #96] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s10
 ; CHECK-GI-NEXT:    bl cosf
-; CHECK-GI-NEXT:    str d0, [sp, #80] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #80] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-GI-NEXT:    bl cosf
-; CHECK-GI-NEXT:    str d0, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s11
 ; CHECK-GI-NEXT:    bl cosf
-; CHECK-GI-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s12
 ; CHECK-GI-NEXT:    bl cosf
-; CHECK-GI-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-GI-NEXT:    fmov s0, s13
 ; CHECK-GI-NEXT:    bl cosf
 ; CHECK-GI-NEXT:    ldp q2, q1, [sp, #48] // 32-byte Folded Reload
diff --git a/llvm/test/CodeGen/AArch64/implicit-def-with-impdef-greedy-assert.mir b/llvm/test/CodeGen/AArch64/implicit-def-with-impdef-greedy-assert.mir
new file mode 100644
index 0000000000..c79c951fdc
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/implicit-def-with-impdef-greedy-assert.mir
@@ -0,0 +1,181 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 3
+# RUN: llc -mtriple=arm64-apple-ios -run-pass=greedy -o - %s | FileCheck %s
+
+---
+name:            widget
+tracksRegLiveness: true
+jumpTable:
+  kind:            label-difference32
+  entries:
+    - id:              0
+      blocks:          [ '%bb.9', '%bb.5', '%bb.2', '%bb.2', '%bb.2' ]
+body:             |
+  ; CHECK-LABEL: name: widget
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT:   liveins: $w0, $w1, $x2, $x3, $x4, $w5, $w6
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:gpr64common = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[DEF1:%[0-9]+]]:gpr64 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[DEF2:%[0-9]+]]:gpr32common = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[DEF3:%[0-9]+]]:gpr32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[DEF4:%[0-9]+]]:gpr32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[DEF5:%[0-9]+]]:gpr64common = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[DEF6:%[0-9]+]]:gpr32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   undef [[DEF7:%[0-9]+]].sub_32:gpr64 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[DEF8:%[0-9]+]]:gpr64common = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[DEF9:%[0-9]+]]:gpr32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   dead [[DEF10:%[0-9]+]]:gpr64 = IMPLICIT_DEF
+  ; CHECK-NEXT:   undef [[DEF11:%[0-9]+]].sub_32:gpr64 = IMPLICIT_DEF implicit-def dead %11
+  ; CHECK-NEXT:   STRXui [[DEF11]], %stack.0, 0 :: (store (s64) into %stack.0)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   successors: %bb.3(0x0fbefbf0), %bb.4(0x70410410)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   Bcc 8, %bb.3, implicit killed undef $nzcv
+  ; CHECK-NEXT:   B %bb.4
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.3:
+  ; CHECK-NEXT:   successors: %bb.11(0x00000000), %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   dead $wzr = SUBSWri [[DEF2]], 64, 0, implicit-def $nzcv
+  ; CHECK-NEXT:   Bcc 0, %bb.11, implicit killed undef $nzcv
+  ; CHECK-NEXT:   B %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.4:
+  ; CHECK-NEXT:   successors: %bb.9(0x01288b01), %bb.5(0x01288b01), %bb.2(0x11f46a91), %bb.6(0x23e8d524), %bb.7(0x47d1aa49)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   dead early-clobber %12:gpr64, dead early-clobber %13:gpr64sp = JumpTableDest32 [[DEF8]], [[DEF7]], %jump-table.0
+  ; CHECK-NEXT:   BR undef %18:gpr64
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.5:
+  ; CHECK-NEXT:   successors: %bb.8(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   B %bb.8
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.6:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   STRWui [[DEF9]], [[DEF5]], 0 :: (store (s32))
+  ; CHECK-NEXT:   B %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.7:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   STRWui $wzr, [[DEF]], 0 :: (store (s32))
+  ; CHECK-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-NEXT:   $w0 = COPY [[DEF4]]
+  ; CHECK-NEXT:   $x1 = COPY [[DEF1]]
+  ; CHECK-NEXT:   BL 0, csr_darwin_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def dead $w0
+  ; CHECK-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-NEXT:   $w0 = COPY [[DEF6]]
+  ; CHECK-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-NEXT:   B %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.8:
+  ; CHECK-NEXT:   successors: %bb.8(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   B %bb.8
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.9:
+  ; CHECK-NEXT:   successors: %bb.10(0x40000000), %bb.1(0x40000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   undef [[DEF12:%[0-9]+]].sub_32:gpr64 = IMPLICIT_DEF
+  ; CHECK-NEXT:   STRXui [[DEF12]], %stack.0, 0 :: (store (s64) into %stack.0)
+  ; CHECK-NEXT:   TBZW [[DEF3]], 0, %bb.1
+  ; CHECK-NEXT:   B %bb.10
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.10:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   ADJCALLSTACKDOWN 32, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-NEXT:   ADJCALLSTACKUP 32, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-NEXT:   B %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.11:
+  ; CHECK-NEXT:   [[LDRXui:%[0-9]+]]:gpr64 = LDRXui %stack.0, 0 :: (load (s64) from %stack.0)
+  ; CHECK-NEXT:   dead undef [[COPY:%[0-9]+]].sub_32:gpr64 = COPY [[LDRXui]].sub_32
+  ; CHECK-NEXT:   ADJCALLSTACKDOWN 8, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-NEXT:   ADJCALLSTACKUP 8, 0, implicit-def dead $sp, implicit $sp
+  bb.0:
+    liveins: $w0, $w1, $x2, $x3, $x4, $w5, $w6
+
+    %0:gpr64common = IMPLICIT_DEF
+    %1:gpr64 = IMPLICIT_DEF
+    %2:gpr32common = IMPLICIT_DEF
+    %3:gpr32 = IMPLICIT_DEF
+    %4:gpr32 = IMPLICIT_DEF
+    %5:gpr64common = IMPLICIT_DEF
+    %6:gpr32 = IMPLICIT_DEF
+    undef %7.sub_32:gpr64 = IMPLICIT_DEF
+    %8:gpr64common = IMPLICIT_DEF
+    %9:gpr32 = IMPLICIT_DEF
+    %10:gpr64 = IMPLICIT_DEF
+    undef %10.sub_32:gpr64 = IMPLICIT_DEF implicit-def %11:gpr64
+
+  bb.1:
+
+  bb.2:
+    successors: %bb.3(0x0fbefbf0), %bb.4(0x70410410)
+
+    Bcc 8, %bb.3, implicit killed undef $nzcv
+    B %bb.4
+
+  bb.3:
+    successors: %bb.11(0x00000000), %bb.2(0x80000000)
+
+    dead $wzr = SUBSWri %2, 64, 0, implicit-def $nzcv
+    Bcc 0, %bb.11, implicit killed undef $nzcv
+    B %bb.2
+
+  bb.4:
+    successors: %bb.9(0x01288b01), %bb.5(0x01288b01), %bb.2(0x11f46a91), %bb.6(0x23e8d524), %bb.7(0x47d1aa49)
+
+    early-clobber %12:gpr64, dead early-clobber %13:gpr64sp = JumpTableDest32 %8, %7, %jump-table.0
+    BR undef %10
+
+  bb.5:
+    B %bb.8
+
+  bb.6:
+    STRWui %9, %5, 0 :: (store (s32))
+    B %bb.2
+
+  bb.7:
+    STRWui $wzr, %0, 0 :: (store (s32))
+    ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
+    $w0 = COPY %4
+    $x1 = COPY %1
+    BL 0, csr_darwin_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def dead $w0
+    ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
+    ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
+    $w0 = COPY %6
+    ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
+    B %bb.2
+
+  bb.8:
+    B %bb.8
+
+  bb.9:
+    successors: %bb.10, %bb.1
+
+    undef %10.sub_32:gpr64 = IMPLICIT_DEF
+    TBZW %3, 0, %bb.1
+    B %bb.10
+
+  bb.10:
+    ADJCALLSTACKDOWN 32, 0, implicit-def dead $sp, implicit $sp
+    ADJCALLSTACKUP 32, 0, implicit-def dead $sp, implicit $sp
+    B %bb.1
+
+  bb.11:
+    undef %14.sub_32:gpr64 = COPY %10.sub_32
+    ADJCALLSTACKDOWN 8, 0, implicit-def dead $sp, implicit $sp
+    ADJCALLSTACKUP 8, 0, implicit-def dead $sp, implicit $sp
+
+...
diff --git a/llvm/test/CodeGen/AArch64/llvm.exp10.ll b/llvm/test/CodeGen/AArch64/llvm.exp10.ll
index ed0d0d5194..c3f14b0dcf 100644
--- a/llvm/test/CodeGen/AArch64/llvm.exp10.ll
+++ b/llvm/test/CodeGen/AArch64/llvm.exp10.ll
@@ -353,7 +353,8 @@ define <2 x float> @exp10_v2f32(<2 x float> %x) {
 ; SDAG-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; SDAG-NEXT:    mov s0, v0.s[1]
 ; SDAG-NEXT:    bl exp10f
-; SDAG-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
+; SDAG-NEXT:    // kill: def $s0 killed $s0 def $q0
+; SDAG-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; SDAG-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; SDAG-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; SDAG-NEXT:    bl exp10f
@@ -377,7 +378,8 @@ define <2 x float> @exp10_v2f32(<2 x float> %x) {
 ; GISEL-NEXT:    mov s8, v0.s[1]
 ; GISEL-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; GISEL-NEXT:    bl exp10f
-; GISEL-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; GISEL-NEXT:    // kill: def $s0 killed $s0 def $q0
+; GISEL-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; GISEL-NEXT:    fmov s0, s8
 ; GISEL-NEXT:    bl exp10f
 ; GISEL-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
@@ -402,7 +404,8 @@ define <3 x float> @exp10_v3f32(<3 x float> %x) {
 ; SDAG-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; SDAG-NEXT:    mov s0, v0.s[1]
 ; SDAG-NEXT:    bl exp10f
-; SDAG-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; SDAG-NEXT:    // kill: def $s0 killed $s0 def $q0
+; SDAG-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; SDAG-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; SDAG-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; SDAG-NEXT:    bl exp10f
@@ -434,10 +437,12 @@ define <3 x float> @exp10_v3f32(<3 x float> %x) {
 ; GISEL-NEXT:    mov s9, v0.s[2]
 ; GISEL-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; GISEL-NEXT:    bl exp10f
-; GISEL-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
+; GISEL-NEXT:    // kill: def $s0 killed $s0 def $q0
+; GISEL-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; GISEL-NEXT:    fmov s0, s8
 ; GISEL-NEXT:    bl exp10f
-; GISEL-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; GISEL-NEXT:    // kill: def $s0 killed $s0 def $q0
+; GISEL-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; GISEL-NEXT:    fmov s0, s9
 ; GISEL-NEXT:    bl exp10f
 ; GISEL-NEXT:    ldp q2, q1, [sp] // 32-byte Folded Reload
@@ -464,7 +469,8 @@ define <4 x float> @exp10_v4f32(<4 x float> %x) {
 ; SDAG-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; SDAG-NEXT:    mov s0, v0.s[1]
 ; SDAG-NEXT:    bl exp10f
-; SDAG-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; SDAG-NEXT:    // kill: def $s0 killed $s0 def $q0
+; SDAG-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; SDAG-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; SDAG-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; SDAG-NEXT:    bl exp10f
@@ -506,13 +512,16 @@ define <4 x float> @exp10_v4f32(<4 x float> %x) {
 ; GISEL-NEXT:    mov s10, v0.s[3]
 ; GISEL-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; GISEL-NEXT:    bl exp10f
-; GISEL-NEXT:    str d0, [sp, #32] // 16-byte Folded Spill
+; GISEL-NEXT:    // kill: def $s0 killed $s0 def $q0
+; GISEL-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; GISEL-NEXT:    fmov s0, s8
 ; GISEL-NEXT:    bl exp10f
-; GISEL-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
+; GISEL-NEXT:    // kill: def $s0 killed $s0 def $q0
+; GISEL-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; GISEL-NEXT:    fmov s0, s9
 ; GISEL-NEXT:    bl exp10f
-; GISEL-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; GISEL-NEXT:    // kill: def $s0 killed $s0 def $q0
+; GISEL-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; GISEL-NEXT:    fmov s0, s10
 ; GISEL-NEXT:    bl exp10f
 ; GISEL-NEXT:    ldp q2, q1, [sp, #16] // 32-byte Folded Reload
@@ -564,6 +573,7 @@ define <2 x double> @exp10_v2f64(<2 x double> %x) {
 ; SDAG-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; SDAG-NEXT:    mov d0, v0.d[1]
 ; SDAG-NEXT:    bl exp10
+; SDAG-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; SDAG-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; SDAG-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; SDAG-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -586,6 +596,7 @@ define <2 x double> @exp10_v2f64(<2 x double> %x) {
 ; GISEL-NEXT:    mov d8, v0.d[1]
 ; GISEL-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; GISEL-NEXT:    bl exp10
+; GISEL-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; GISEL-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; GISEL-NEXT:    fmov d0, d8
 ; GISEL-NEXT:    bl exp10
@@ -670,6 +681,7 @@ define <4 x double> @exp10_v4f64(<4 x double> %x) {
 ; SDAG-NEXT:    mov d0, v0.d[1]
 ; SDAG-NEXT:    str q1, [sp, #32] // 16-byte Folded Spill
 ; SDAG-NEXT:    bl exp10
+; SDAG-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; SDAG-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; SDAG-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; SDAG-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -681,6 +693,7 @@ define <4 x double> @exp10_v4f64(<4 x double> %x) {
 ; SDAG-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
 ; SDAG-NEXT:    mov d0, v0.d[1]
 ; SDAG-NEXT:    bl exp10
+; SDAG-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; SDAG-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; SDAG-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
 ; SDAG-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -706,13 +719,16 @@ define <4 x double> @exp10_v4f64(<4 x double> %x) {
 ; GISEL-NEXT:    mov d9, v1.d[1]
 ; GISEL-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; GISEL-NEXT:    bl exp10
+; GISEL-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; GISEL-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; GISEL-NEXT:    fmov d0, d8
 ; GISEL-NEXT:    bl exp10
+; GISEL-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; GISEL-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; GISEL-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; GISEL-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; GISEL-NEXT:    bl exp10
+; GISEL-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; GISEL-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; GISEL-NEXT:    fmov d0, d9
 ; GISEL-NEXT:    bl exp10
diff --git a/llvm/test/CodeGen/AArch64/machine-sink-cache-invalidation.ll b/llvm/test/CodeGen/AArch64/machine-sink-cache-invalidation.ll
index ad6fdb6f1f..ce000021fb 100644
--- a/llvm/test/CodeGen/AArch64/machine-sink-cache-invalidation.ll
+++ b/llvm/test/CodeGen/AArch64/machine-sink-cache-invalidation.ll
@@ -22,8 +22,7 @@ define i32 @nsis_BZ2_bzDecompress(ptr %pos.i, i1 %cmp661.not3117.i, i1 %exitcond
 ; CHECK-NEXT:    // in Loop: Header=BB0_2 Depth=1
 ; CHECK-NEXT:    mov x9, xzr
 ; CHECK-NEXT:    ldrb w9, [x9]
-; CHECK-NEXT:    add x9, x0, x9
-; CHECK-NEXT:    strb wzr, [x9]
+; CHECK-NEXT:    strb wzr, [x0, x9]
 ; CHECK-NEXT:    b .LBB0_1
 ; CHECK-NEXT:  .LBB0_4: // %for.end677.i
 ; CHECK-NEXT:    mov w0, wzr
diff --git a/llvm/test/CodeGen/AArch64/pow.ll b/llvm/test/CodeGen/AArch64/pow.ll
index 5141b21c79..495541e9fd 100644
--- a/llvm/test/CodeGen/AArch64/pow.ll
+++ b/llvm/test/CodeGen/AArch64/pow.ll
@@ -75,7 +75,8 @@ define <4 x float> @pow_v4f32_one_fourth_not_enough_fmf(<4 x float> %x) nounwind
 ; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
 ; CHECK-NEXT:    bl powf
 ; CHECK-NEXT:    fmov s1, #0.25000000
-; CHECK-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-NEXT:    bl powf
@@ -116,6 +117,7 @@ define <2 x double> @pow_v2f64_one_fourth_not_enough_fmf(<2 x double> %x) nounwi
 ; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
 ; CHECK-NEXT:    bl pow
 ; CHECK-NEXT:    fmov d1, #0.25000000
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
diff --git a/llvm/test/CodeGen/AArch64/pr66603.ll b/llvm/test/CodeGen/AArch64/pr66603.ll
new file mode 100644
index 0000000000..2373b722fa
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/pr66603.ll
@@ -0,0 +1,15 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
+; RUN: llc < %s -mtriple=aarch64-- -verify-machineinstrs | FileCheck %s
+
+; Don't fold freeze(assertsext(x)) -> assertsext(freeze(x))
+define i32 @PR66603(double %x) nounwind {
+; CHECK-LABEL: PR66603:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtzs w8, d0
+; CHECK-NEXT:    sxtb w0, w8
+; CHECK-NEXT:    ret
+  %as_i8 = fptosi double %x to i8
+  %frozen_i8 = freeze i8 %as_i8
+  %ext = sext i8 %frozen_i8 to i32
+  ret i32 %ext
+}
diff --git a/llvm/test/CodeGen/AArch64/prologue-epilogue-remarks.mir b/llvm/test/CodeGen/AArch64/prologue-epilogue-remarks.mir
index 4bc1b968d1..223719d44d 100644
--- a/llvm/test/CodeGen/AArch64/prologue-epilogue-remarks.mir
+++ b/llvm/test/CodeGen/AArch64/prologue-epilogue-remarks.mir
@@ -11,7 +11,9 @@ stack:
 # CHECK:      Function:        fun0
 # CHECK-NEXT: Args:
 # CHECK-NEXT:   - NumStackBytes:   '16'
-# CHECK-NEXT:   - String:          ' stack bytes in function'
+# CHECK-NEXT:   - String:          ' stack bytes in function '''
+# CHECK-NEXT:   - Function: fun0
+# CHECK-NEXT:   - String: ''''
 # CHECK-NEXT: ...
 constants:
 body:             |
@@ -29,7 +31,9 @@ stack:
 # CHECK:      Function:        fun1
 # CHECK-NEXT: Args:
 # CHECK-NEXT:   - NumStackBytes:   '32'
-# CHECK-NEXT:   - String:          ' stack bytes in function'
+# CHECK-NEXT:   - String:          ' stack bytes in function '''
+# CHECK-NEXT:   - Function: fun1
+# CHECK-NEXT:   - String: ''''
 # CHECK-NEXT: ...
 constants:
 body:             |
@@ -47,7 +51,9 @@ stack:
 # CHECK:      Function:        fun2
 # CHECK-NEXT: Args:
 # CHECK-NEXT:   - NumStackBytes:   '1040'
-# CHECK-NEXT:   - String:          ' stack bytes in function'
+# CHECK-NEXT:   - String:          ' stack bytes in function '''
+# CHECK-NEXT:   - Function: fun2
+# CHECK-NEXT:   - String: ''''
 # CHECK-NEXT: ...
 constants:
 body:             |
diff --git a/llvm/test/CodeGen/AArch64/sign-return-address-tailcall.ll b/llvm/test/CodeGen/AArch64/sign-return-address-tailcall.ll
new file mode 100644
index 0000000000..ec04e553ca
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sign-return-address-tailcall.ll
@@ -0,0 +1,121 @@
+; RUN: llc -mtriple=aarch64 -asm-verbose=0 < %s | FileCheck -DAUTIASP="hint #29" --check-prefixes=COMMON %s
+; RUN: llc -mtriple=aarch64 -asm-verbose=0 -aarch64-authenticated-lr-check-method=load                   < %s | FileCheck -DAUTIASP="hint #29" --check-prefixes=COMMON,LDR %s
+; RUN: llc -mtriple=aarch64 -asm-verbose=0 -aarch64-authenticated-lr-check-method=high-bits-notbi        < %s | FileCheck -DAUTIASP="hint #29" --check-prefixes=COMMON,BITS-NOTBI,BRK %s
+; RUN: llc -mtriple=aarch64 -asm-verbose=0 -aarch64-authenticated-lr-check-method=xpac-hint              < %s | FileCheck -DAUTIASP="hint #29" -DXPACLRI="hint #7" --check-prefixes=COMMON,XPAC,BRK %s
+; RUN: llc -mtriple=aarch64 -asm-verbose=0 -aarch64-authenticated-lr-check-method=xpac-hint -mattr=v8.3a < %s | FileCheck -DAUTIASP="autiasp"  -DXPACLRI="xpaclri" --check-prefixes=COMMON,XPAC,BRK %s
+
+define i32 @tailcall_direct() "sign-return-address"="non-leaf" {
+; COMMON-LABEL: tailcall_direct:
+; COMMON:         str x30, [sp, #-16]!
+; COMMON:         ldr x30, [sp], #16
+;
+; COMMON-NEXT:    [[AUTIASP]]
+;
+; LDR-NEXT:       ldr w16, [x30]
+;
+; BITS-NOTBI-NEXT: eor x16, x30, x30, lsl #1
+; BITS-NOTBI-NEXT: tbnz x16, #62, .[[FAIL:LBB[_0-9]+]]
+;
+; XPAC-NEXT:      mov x16, x30
+; XPAC-NEXT:      [[XPACLRI]]
+; XPAC-NEXT:      cmp x16, x30
+; XPAC-NEXT:      b.ne .[[FAIL:LBB[_0-9]+]]
+;
+; COMMON-NEXT:    b callee
+; BRK-NEXT:     .[[FAIL]]:
+; BRK-NEXT:       brk #0xc471
+  tail call void asm sideeffect "", "~{lr}"()
+  %call = tail call i32 @callee()
+  ret i32 %call
+}
+
+define i32 @tailcall_indirect(ptr %fptr) "sign-return-address"="non-leaf" {
+; COMMON-LABEL: tailcall_indirect:
+; COMMON:         str x30, [sp, #-16]!
+; COMMON:         ldr x30, [sp], #16
+;
+; COMMON-NEXT:    [[AUTIASP]]
+;
+; LDR-NEXT:       ldr w16, [x30]
+;
+; BITS-NOTBI-NEXT: eor x16, x30, x30, lsl #1
+; BITS-NOTBI-NEXT: tbnz x16, #62, .[[FAIL:LBB[_0-9]+]]
+;
+; XPAC-NEXT:      mov x16, x30
+; XPAC-NEXT:      [[XPACLRI]]
+; XPAC-NEXT:      cmp x16, x30
+; XPAC-NEXT:      b.ne .[[FAIL:LBB[_0-9]+]]
+;
+; COMMON-NEXT:    br x0
+; BRK-NEXT:     .[[FAIL]]:
+; BRK-NEXT:       brk #0xc471
+  tail call void asm sideeffect "", "~{lr}"()
+  %call = tail call i32 %fptr()
+  ret i32 %call
+}
+
+define i32 @tailcall_direct_noframe() "sign-return-address"="non-leaf" {
+; COMMON-LABEL: tailcall_direct_noframe:
+; COMMON-NEXT:    .cfi_startproc
+; COMMON-NEXT:    b callee
+  %call = tail call i32 @callee()
+  ret i32 %call
+}
+
+define i32 @tailcall_indirect_noframe(ptr %fptr) "sign-return-address"="non-leaf" {
+; COMMON-LABEL: tailcall_indirect_noframe:
+; COMMON-NEXT:    .cfi_startproc
+; COMMON-NEXT:    br x0
+  %call = tail call i32 %fptr()
+  ret i32 %call
+}
+
+define i32 @tailcall_direct_noframe_sign_all() "sign-return-address"="all" {
+; COMMON-LABEL: tailcall_direct_noframe_sign_all:
+; COMMON-NOT:     str{{.*}}x30
+; COMMON-NOT:     ldr{{.*}}x30
+;
+; COMMON:         [[AUTIASP]]
+;
+; LDR-NEXT:       ldr w16, [x30]
+;
+; BITS-NOTBI-NEXT: eor x16, x30, x30, lsl #1
+; BITS-NOTBI-NEXT: tbnz x16, #62, .[[FAIL:LBB[_0-9]+]]
+;
+; XPAC-NEXT:      mov x16, x30
+; XPAC-NEXT:      [[XPACLRI]]
+; XPAC-NEXT:      cmp x16, x30
+; XPAC-NEXT:      b.ne .[[FAIL:LBB[_0-9]+]]
+;
+; COMMON-NEXT:    b callee
+; BRK-NEXT:     .[[FAIL]]:
+; BRK-NEXT:       brk #0xc471
+  %call = tail call i32 @callee()
+  ret i32 %call
+}
+
+define i32 @tailcall_indirect_noframe_sign_all(ptr %fptr) "sign-return-address"="all" {
+; COMMON-LABEL: tailcall_indirect_noframe_sign_all:
+; COMMON-NOT:     str{{.*}}x30
+; COMMON-NOT:     ldr{{.*}}x30
+;
+; COMMON:         [[AUTIASP]]
+;
+; LDR-NEXT:       ldr w16, [x30]
+;
+; BITS-NOTBI-NEXT: eor x16, x30, x30, lsl #1
+; BITS-NOTBI-NEXT: tbnz x16, #62, .[[FAIL:LBB[_0-9]+]]
+;
+; XPAC-NEXT:      mov x16, x30
+; XPAC-NEXT:      [[XPACLRI]]
+; XPAC-NEXT:      cmp x16, x30
+; XPAC-NEXT:      b.ne .[[FAIL:LBB[_0-9]+]]
+;
+; COMMON-NEXT:    br x0
+; BRK-NEXT:     .[[FAIL]]:
+; BRK-NEXT:       brk #0xc471
+  %call = tail call i32 %fptr()
+  ret i32 %call
+}
+
+declare i32 @callee()
diff --git a/llvm/test/CodeGen/AArch64/sink-and-fold-illegal-shift.mir b/llvm/test/CodeGen/AArch64/sink-and-fold-illegal-shift.mir
new file mode 100644
index 0000000000..d2f6a3ab1a
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sink-and-fold-illegal-shift.mir
@@ -0,0 +1,95 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 3
+# RUN: llc --run-pass=machine-sink --aarch64-enable-sink-fold=true %s -o - | FileCheck %s
+--- |
+  source_filename = "../llvm/test/CodeGen/AArch64/GlobalISel/sink-and-fold-illegal-shift.ll"
+  target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+  target triple = "aarch64-linux"
+
+  define void @f(ptr %p, i64 %i) #0 {
+    %d = ashr i64 %i, 32
+    %a = getelementptr i8, ptr %p, i64 %d
+    store i8 0, ptr %a, align 1
+    ret void
+  }
+
+  attributes #0 = { optsize }
+
+...
+---
+name:            f
+alignment:       4
+exposesReturnsTwice: false
+legalized:       true
+regBankSelected: true
+selected:        true
+failedISel:      false
+tracksRegLiveness: true
+hasWinCFI:       false
+callsEHReturn:   false
+callsUnwindInit: false
+hasEHCatchret:   false
+hasEHScopes:     false
+hasEHFunclets:   false
+isOutlined:      false
+debugInstrRef:   false
+failsVerification: false
+tracksDebugUserValues: false
+registers:
+  - { id: 0, class: gpr64, preferred-register: '' }
+  - { id: 1, class: gpr64, preferred-register: '' }
+  - { id: 2, class: gpr, preferred-register: '' }
+  - { id: 3, class: gpr, preferred-register: '' }
+  - { id: 4, class: gpr64common, preferred-register: '' }
+  - { id: 5, class: _, preferred-register: '' }
+  - { id: 6, class: gpr, preferred-register: '' }
+  - { id: 7, class: gpr64, preferred-register: '' }
+liveins:
+  - { reg: '$x0', virtual-reg: '' }
+  - { reg: '$x1', virtual-reg: '' }
+frameInfo:
+  isFrameAddressTaken: false
+  isReturnAddressTaken: false
+  hasStackMap:     false
+  hasPatchPoint:   false
+  stackSize:       0
+  offsetAdjustment: 0
+  maxAlignment:    1
+  adjustsStack:    false
+  hasCalls:        false
+  stackProtector:  ''
+  functionContext: ''
+  maxCallFrameSize: 0
+  cvBytesOfCalleeSavedRegisters: 0
+  hasOpaqueSPAdjustment: false
+  hasVAStart:      false
+  hasMustTailInVarArgFunc: false
+  hasTailCall:     false
+  localFrameSize:  0
+  savePoint:       ''
+  restorePoint:    ''
+fixedStack:      []
+stack:           []
+entry_values:    []
+callSites:       []
+debugValueSubstitutions: []
+constants:       []
+machineFunctionInfo: {}
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: $x0, $x1
+
+    ; CHECK-LABEL: name: f
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1
+    ; CHECK-NEXT: [[ADDXrs:%[0-9]+]]:gpr64common = ADDXrs [[COPY]], [[COPY1]], 160
+    ; CHECK-NEXT: STRBBui $wzr, [[ADDXrs]], 0 :: (store (s8) into %ir.a)
+    ; CHECK-NEXT: RET_ReallyLR
+    %0:gpr64 = COPY $x0
+    %1:gpr64 = COPY $x1
+    %4:gpr64common = ADDXrs %0, %1, 160
+    STRBBui $wzr, %4, 0 :: (store (s8) into %ir.a)
+    RET_ReallyLR
+
+...
diff --git a/llvm/test/CodeGen/AArch64/sink-and-fold.ll b/llvm/test/CodeGen/AArch64/sink-and-fold.ll
index 632fdb3910..52007221e1 100644
--- a/llvm/test/CodeGen/AArch64/sink-and-fold.ll
+++ b/llvm/test/CodeGen/AArch64/sink-and-fold.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -aarch64-enable-sink-fold=true < %s | FileCheck %s
+; RUN: llc < %s | FileCheck %s
 target triple = "aarch64-linux"
 
 declare i32 @use(...)
diff --git a/llvm/test/CodeGen/AArch64/spill-fold.mir b/llvm/test/CodeGen/AArch64/spill-fold.mir
index 02d409d04d..b1e7ebe3a7 100644
--- a/llvm/test/CodeGen/AArch64/spill-fold.mir
+++ b/llvm/test/CodeGen/AArch64/spill-fold.mir
@@ -1,10 +1,13 @@
 # RUN: llc -mtriple=aarch64-none-linux-gnu -run-pass greedy -verify-machineinstrs  -o - %s | FileCheck %s
+# RUN: llc -mtriple=aarch64_be-none-linux-gnu -run-pass greedy -verify-machineinstrs  -o - %s | FileCheck %s
 --- |
   define i64 @test_subreg_spill_fold() { ret i64 0 }
   define i64 @test_subreg_spill_fold2() { ret i64 0 }
   define i64 @test_subreg_spill_fold3() { ret i64 0 }
+  define <4 x float> @test_subreg_spill_fold4() { ret <4 x float> undef }
   define i64 @test_subreg_fill_fold() { ret i64 0 }
   define double @test_subreg_fill_fold2() { ret double 0.0 }
+  define <4 x float> @test_subreg_fill_fold3() { ret <4 x float> undef }
   define i64 @test_nzcv_spill_fold() { ret i64 0 }
 ...
 ---
@@ -50,6 +53,21 @@ body:             |
     RET_ReallyLR implicit $x0
 ...
 ---
+# CHECK-LABEL: name: test_subreg_spill_fold4
+# A spilled write to a 128-bit register needs a 128-bit store
+name:            test_subreg_spill_fold4
+registers:
+  - { id: 0, class: fpr128 }
+body:             |
+  bb.0:
+    ; CHECK:      undef %1.ssub:fpr128 = COPY $wzr
+    ; CHECK-NEXT: STRQui %1, %stack.0, 0 :: (store (s128) into %stack.0)
+    undef %0.ssub:fpr128 = COPY $wzr
+    INLINEASM &nop, 1, 12, implicit-def dead $d0, 12, implicit-def dead $d1, 12, implicit-def dead $d2, 12, implicit-def dead $d3, 12, implicit-def dead $d4, 12, implicit-def dead $d5, 12, implicit-def dead $d6, 12, implicit-def dead $d7, 12, implicit-def dead $d8, 12, implicit-def dead $d9, 12, implicit-def dead $d10, 12, implicit-def dead $d11, 12, implicit-def dead $d12, 12, implicit-def dead $d13, 12, implicit-def dead $d14, 12, implicit-def dead $d15, 12, implicit-def dead $d16, 12, implicit-def dead $d17, 12, implicit-def dead $d18, 12, implicit-def dead $d19, 12, implicit-def dead $d20, 12, implicit-def dead $d21, 12, implicit-def dead $d22, 12, implicit-def dead $d23, 12, implicit-def dead $d24, 12, implicit-def dead $d25, 12, implicit-def dead $d26, 12, implicit-def dead $d27, 12, implicit-def dead $d28, 12, implicit-def dead $d29, 12, implicit-def dead $d30, 12, implicit-def $d31
+    $q0 = COPY %0
+    RET_ReallyLR implicit $q0
+...
+---
 # CHECK-LABEL: name: test_subreg_fill_fold
 # Ensure that the filled COPY is eliminated and folded into the fill load.
 name:            test_subreg_fill_fold
@@ -82,6 +100,25 @@ body:             |
     RET_ReallyLR implicit $d0
 ...
 ---
+# CHECK-LABEL: name: test_subreg_fill_fold3
+# A filled copy from a sub-register should load the whole register
+name:              test_subreg_fill_fold3
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr128 }
+liveins:
+  - { reg: '$q0' }
+body:             |
+  bb.0:
+    liveins: $q0
+    %0 = COPY $q0
+    INLINEASM &nop, 1, 12, implicit-def dead $d0, 12, implicit-def dead $d1, 12, implicit-def dead $d2, 12, implicit-def dead $d3, 12, implicit-def dead $d4, 12, implicit-def dead $d5, 12, implicit-def dead $d6, 12, implicit-def dead $d7, 12, implicit-def dead $d8, 12, implicit-def dead $d9, 12, implicit-def dead $d10, 12, implicit-def dead $d11, 12, implicit-def dead $d12, 12, implicit-def dead $d13, 12, implicit-def dead $d14, 12, implicit-def dead $d15, 12, implicit-def dead $d16, 12, implicit-def dead $d17, 12, implicit-def dead $d18, 12, implicit-def dead $d19, 12, implicit-def dead $d20, 12, implicit-def dead $d21, 12, implicit-def dead $d22, 12, implicit-def dead $d23, 12, implicit-def dead $d24, 12, implicit-def dead $d25, 12, implicit-def dead $d26, 12, implicit-def dead $d27, 12, implicit-def dead $d28, 12, implicit-def dead $d29, 12, implicit-def dead $d30, 12, implicit-def $d31
+    ; CHECK:      %1:fpr128 = LDRQui %stack.0, 0 :: (load (s128) from %stack.0)
+    ; CHECK-NEXT: $s0 = COPY %1.ssub
+    $s0 = COPY %0.ssub
+    RET_ReallyLR implicit $s0
+...
+---
 # CHECK-LABEL: name: test_nzcv_spill_fold
 # Ensure that nzcv COPY cannot be folded.
 name:            test_nzcv_spill_fold
diff --git a/llvm/test/CodeGen/AArch64/statepoint-call-lowering.ll b/llvm/test/CodeGen/AArch64/statepoint-call-lowering.ll
index 6326d3db9a..9619895c45 100644
--- a/llvm/test/CodeGen/AArch64/statepoint-call-lowering.ll
+++ b/llvm/test/CodeGen/AArch64/statepoint-call-lowering.ll
@@ -23,7 +23,6 @@ define i1 @test_i1_return() gc "statepoint-example" {
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    bl return_i1
 ; CHECK-NEXT:  .Ltmp0:
-; CHECK-NEXT:    and w0, w0, #0x1
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
 ; This is just checking that a i1 gets lowered normally when there's no extra
@@ -106,7 +105,6 @@ define i1 @test_relocate(ptr addrspace(1) %a) gc "statepoint-example" {
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    bl return_i1
 ; CHECK-NEXT:  .Ltmp5:
-; CHECK-NEXT:    and w0, w0, #0x1
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
 ; Check that an ununsed relocate has no code-generation impact
@@ -145,7 +143,6 @@ define i1 @test_i1_return_patchable() gc "statepoint-example" {
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:  .Ltmp7:
-; CHECK-NEXT:    and w0, w0, #0x1
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
 ; A patchable variant of test_i1_return
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp128.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp128.ll
index 9b0d86556f..7ac1b58c62 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp128.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp128.ll
@@ -75,6 +75,7 @@ define void @fcvt_v4f128_v4f64(ptr %a, ptr %b) vscale_range(2,0) #0 {
 ; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    ldr q0, [x0, #48]
 ; CHECK-NEXT:    bl __trunctfdf2
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    bl __trunctfdf2
@@ -85,6 +86,7 @@ define void @fcvt_v4f128_v4f64(ptr %a, ptr %b) vscale_range(2,0) #0 {
 ; CHECK-NEXT:    str z0, [x8, #1, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    bl __trunctfdf2
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    bl __trunctfdf2
@@ -98,6 +100,7 @@ define void @fcvt_v4f128_v4f64(ptr %a, ptr %b) vscale_range(2,0) #0 {
 ; CHECK-NEXT:    str z0, [x8, #1, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    ldr q0, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    bl __trunctfdf2
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-NEXT:    str q0, [sp, #64] // 16-byte Folded Spill
 ; CHECK-NEXT:    ldr q0, [sp, #80] // 16-byte Folded Reload
 ; CHECK-NEXT:    bl __trunctfdf2
@@ -108,6 +111,7 @@ define void @fcvt_v4f128_v4f64(ptr %a, ptr %b) vscale_range(2,0) #0 {
 ; CHECK-NEXT:    str z0, [x8] // 16-byte Folded Spill
 ; CHECK-NEXT:    ldr q0, [sp, #96] // 16-byte Folded Reload
 ; CHECK-NEXT:    bl __trunctfdf2
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-NEXT:    str q0, [sp, #96] // 16-byte Folded Spill
 ; CHECK-NEXT:    ldr q0, [sp, #112] // 16-byte Folded Reload
 ; CHECK-NEXT:    bl __trunctfdf2
diff --git a/llvm/test/CodeGen/AArch64/sve-zeroinit.ll b/llvm/test/CodeGen/AArch64/sve-zeroinit.ll
index c436bb7f82..eab39d0ef4 100644
--- a/llvm/test/CodeGen/AArch64/sve-zeroinit.ll
+++ b/llvm/test/CodeGen/AArch64/sve-zeroinit.ll
@@ -86,3 +86,10 @@ define <vscale x 16 x i1> @test_zeroinit_16xi1() {
 ; CHECK-NEXT:  ret
   ret <vscale x 16 x i1> zeroinitializer
 }
+
+define target("aarch64.svcount") @test_zeroinit_svcount() "target-features"="+sme2" {
+; CHECK-LABEL: test_zeroinit_svcount
+; CHECK:       pfalse p0.b
+; CHECK-NEXT:  ret
+  ret target("aarch64.svcount") zeroinitializer
+}
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-bfmls.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-bfmls.ll
new file mode 100644
index 0000000000..04ad000f20
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-bfmls.ll
@@ -0,0 +1,43 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2p1 -mattr=+b16b16 -verify-machineinstrs < %s | FileCheck %s
+
+define <vscale x 4 x float> @bfmlslb_f32(<vscale x 4 x float> %zda, <vscale x 8 x bfloat> %zn, <vscale x 8 x bfloat> %zm) {
+; CHECK-LABEL: bfmlslb_f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    bfmlslb z0.s, z1.h, z2.h
+; CHECK-NEXT:    ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.bfmlslb(<vscale x 4 x float> %zda, <vscale x 8 x bfloat> %zn, <vscale x 8 x bfloat> %zm)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 4 x float> @bfmlslt_f32(<vscale x 4 x float> %zda, <vscale x 8 x bfloat> %zn, <vscale x 8 x bfloat> %zm) {
+; CHECK-LABEL: bfmlslt_f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    bfmlslt z0.s, z1.h, z2.h
+; CHECK-NEXT:    ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.bfmlslt(<vscale x 4 x float> %zda, <vscale x 8 x bfloat> %zn, <vscale x 8 x bfloat> %zm)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 4 x float> @bfmlslb_lane_f32(<vscale x 4 x float> %zda, <vscale x 8 x bfloat> %zn, <vscale x 8 x bfloat> %zm) {
+; CHECK-LABEL: bfmlslb_lane_f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    bfmlslb z0.s, z1.h, z2.h[7]
+; CHECK-NEXT:    ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.bfmlslb.lane(<vscale x 4 x float> %zda, <vscale x 8 x bfloat> %zn, <vscale x 8 x bfloat> %zm, i32 7)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 4 x float> @bfmlslt_lane_f32(<vscale x 4 x float> %zda, <vscale x 8 x bfloat> %zn, <vscale x 8 x bfloat> %zm) {
+; CHECK-LABEL: bfmlslt_lane_f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    bfmlslt z0.s, z1.h, z2.h[7]
+; CHECK-NEXT:    ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.bfmlslt.lane(<vscale x 4 x float> %zda, <vscale x 8 x bfloat> %zn, <vscale x 8 x bfloat> %zm, i32 7)
+  ret <vscale x 4 x float> %out
+}
+
+declare <vscale x 4 x float> @llvm.aarch64.sve.bfmlslb(<vscale x 4 x float>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.bfmlslt(<vscale x 4 x float>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.bfmlslb.lane(<vscale x 4 x float>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, i32)
+declare <vscale x 4 x float> @llvm.aarch64.sve.bfmlslt.lane(<vscale x 4 x float>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, i32)
diff --git a/llvm/test/CodeGen/AArch64/vec-libcalls.ll b/llvm/test/CodeGen/AArch64/vec-libcalls.ll
index e1b4967ed0..aa2cca40a9 100644
--- a/llvm/test/CodeGen/AArch64/vec-libcalls.ll
+++ b/llvm/test/CodeGen/AArch64/vec-libcalls.ll
@@ -56,7 +56,8 @@ define <2 x float> @sin_v2f32(<2 x float> %x) nounwind {
 ; CHECK-NEXT:    mov s0, v0.s[1]
 ; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
 ; CHECK-NEXT:    bl sinf
-; CHECK-NEXT:    str d0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-NEXT:    bl sinf
@@ -79,7 +80,8 @@ define <3 x float> @sin_v3f32(<3 x float> %x) nounwind {
 ; CHECK-NEXT:    mov s0, v0.s[1]
 ; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
 ; CHECK-NEXT:    bl sinf
-; CHECK-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-NEXT:    bl sinf
@@ -109,7 +111,8 @@ define <4 x float> @sin_v4f32(<4 x float> %x) nounwind {
 ; CHECK-NEXT:    mov s0, v0.s[1]
 ; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
 ; CHECK-NEXT:    bl sinf
-; CHECK-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-NEXT:    bl sinf
@@ -271,7 +274,8 @@ define <3 x float> @cos_v3f32(<3 x float> %x) nounwind {
 ; CHECK-NEXT:    mov s0, v0.s[1]
 ; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
 ; CHECK-NEXT:    bl cosf
-; CHECK-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-NEXT:    bl cosf
@@ -301,7 +305,8 @@ define <3 x float> @exp_v3f32(<3 x float> %x) nounwind {
 ; CHECK-NEXT:    mov s0, v0.s[1]
 ; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
 ; CHECK-NEXT:    bl expf
-; CHECK-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-NEXT:    bl expf
@@ -331,7 +336,8 @@ define <3 x float> @exp2_v3f32(<3 x float> %x) nounwind {
 ; CHECK-NEXT:    mov s0, v0.s[1]
 ; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
 ; CHECK-NEXT:    bl exp2f
-; CHECK-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-NEXT:    bl exp2f
@@ -370,7 +376,8 @@ define <3 x float> @log_v3f32(<3 x float> %x) nounwind {
 ; CHECK-NEXT:    mov s0, v0.s[1]
 ; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
 ; CHECK-NEXT:    bl logf
-; CHECK-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-NEXT:    bl logf
@@ -400,7 +407,8 @@ define <3 x float> @log10_v3f32(<3 x float> %x) nounwind {
 ; CHECK-NEXT:    mov s0, v0.s[1]
 ; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
 ; CHECK-NEXT:    bl log10f
-; CHECK-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-NEXT:    bl log10f
@@ -430,7 +438,8 @@ define <3 x float> @log2_v3f32(<3 x float> %x) nounwind {
 ; CHECK-NEXT:    mov s0, v0.s[1]
 ; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
 ; CHECK-NEXT:    bl log2f
-; CHECK-NEXT:    str d0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
 ; CHECK-NEXT:    bl log2f
diff --git a/llvm/test/CodeGen/AArch64/vec_uaddo.ll b/llvm/test/CodeGen/AArch64/vec_uaddo.ll
index 6ad880020c..00609b0df9 100644
--- a/llvm/test/CodeGen/AArch64/vec_uaddo.ll
+++ b/llvm/test/CodeGen/AArch64/vec_uaddo.ll
@@ -245,21 +245,17 @@ define <4 x i32> @uaddo_v4i24(<4 x i24> %a0, <4 x i24> %a1, ptr %p2) nounwind {
 define <4 x i32> @uaddo_v4i1(<4 x i1> %a0, <4 x i1> %a1, ptr %p2) nounwind {
 ; CHECK-LABEL: uaddo_v4i1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v2.4h, #1
+; CHECK-NEXT:    eor v2.8b, v0.8b, v1.8b
+; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    adrp x8, .LCPI10_0
+; CHECK-NEXT:    shl v2.4h, v2.4h, #15
+; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NEXT:    cmlt v1.4h, v2.4h, #0
+; CHECK-NEXT:    ldr d2, [x8, :lo12:.LCPI10_0]
+; CHECK-NEXT:    shl v0.4s, v0.4s, #31
 ; CHECK-NEXT:    and v1.8b, v1.8b, v2.8b
-; CHECK-NEXT:    and v0.8b, v0.8b, v2.8b
-; CHECK-NEXT:    add v0.4h, v0.4h, v1.4h
-; CHECK-NEXT:    fmov d1, d0
-; CHECK-NEXT:    shl v2.4h, v0.4h, #15
-; CHECK-NEXT:    cmlt v2.4h, v2.4h, #0
-; CHECK-NEXT:    bic v1.4h, #2
-; CHECK-NEXT:    cmeq v0.4h, v1.4h, v0.4h
-; CHECK-NEXT:    ldr d1, [x8, :lo12:.LCPI10_0]
-; CHECK-NEXT:    and v1.8b, v2.8b, v1.8b
-; CHECK-NEXT:    mvn v0.8b, v0.8b
+; CHECK-NEXT:    cmlt v0.4s, v0.4s, #0
 ; CHECK-NEXT:    addv h1, v1.4h
-; CHECK-NEXT:    sshll v0.4s, v0.4h, #0
 ; CHECK-NEXT:    fmov w8, s1
 ; CHECK-NEXT:    strb w8, [x0]
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/vector-llrint.ll b/llvm/test/CodeGen/AArch64/vector-llrint.ll
new file mode 100644
index 0000000000..beb2b6a134
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/vector-llrint.ll
@@ -0,0 +1,621 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64 -mattr=+neon | FileCheck %s
+
+define <1 x i64> @llrint_v1i64_v1f16(<1 x half> %x) {
+; CHECK-LABEL: llrint_v1i64_v1f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    frintx s0, s0
+; CHECK-NEXT:    fcvtzs x8, s0
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    ret
+  %a = call <1 x i64> @llvm.llrint.v1i64.v1f16(<1 x half> %x)
+  ret <1 x i64> %a
+}
+declare <1 x i64> @llvm.llrint.v1i64.v1f16(<1 x half>)
+
+define <2 x i64> @llrint_v1i64_v2f16(<2 x half> %x) {
+; CHECK-LABEL: llrint_v1i64_v2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    mov h1, v0.h[1]
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    fcvt s1, h1
+; CHECK-NEXT:    frintx s0, s0
+; CHECK-NEXT:    frintx s1, s1
+; CHECK-NEXT:    fcvtzs x8, s0
+; CHECK-NEXT:    fcvtzs x9, s1
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    mov v0.d[1], x9
+; CHECK-NEXT:    ret
+  %a = call <2 x i64> @llvm.llrint.v2i64.v2f16(<2 x half> %x)
+  ret <2 x i64> %a
+}
+declare <2 x i64> @llvm.llrint.v2i64.v2f16(<2 x half>)
+
+define <4 x i64> @llrint_v4i64_v4f16(<4 x half> %x) {
+; CHECK-LABEL: llrint_v4i64_v4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    mov h1, v0.h[2]
+; CHECK-NEXT:    mov h2, v0.h[1]
+; CHECK-NEXT:    mov h3, v0.h[3]
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    fcvt s1, h1
+; CHECK-NEXT:    fcvt s2, h2
+; CHECK-NEXT:    fcvt s3, h3
+; CHECK-NEXT:    frintx s0, s0
+; CHECK-NEXT:    frintx s1, s1
+; CHECK-NEXT:    frintx s2, s2
+; CHECK-NEXT:    frintx s3, s3
+; CHECK-NEXT:    fcvtzs x8, s0
+; CHECK-NEXT:    fcvtzs x9, s1
+; CHECK-NEXT:    fcvtzs x10, s2
+; CHECK-NEXT:    fcvtzs x11, s3
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    fmov d1, x9
+; CHECK-NEXT:    mov v0.d[1], x10
+; CHECK-NEXT:    mov v1.d[1], x11
+; CHECK-NEXT:    ret
+  %a = call <4 x i64> @llvm.llrint.v4i64.v4f16(<4 x half> %x)
+  ret <4 x i64> %a
+}
+declare <4 x i64> @llvm.llrint.v4i64.v4f16(<4 x half>)
+
+define <8 x i64> @llrint_v8i64_v8f16(<8 x half> %x) {
+; CHECK-LABEL: llrint_v8i64_v8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    mov h4, v0.h[2]
+; CHECK-NEXT:    mov h3, v0.h[1]
+; CHECK-NEXT:    mov h7, v0.h[3]
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    mov h2, v1.h[2]
+; CHECK-NEXT:    mov h5, v1.h[1]
+; CHECK-NEXT:    mov h6, v1.h[3]
+; CHECK-NEXT:    fcvt s1, h1
+; CHECK-NEXT:    fcvt s4, h4
+; CHECK-NEXT:    fcvt s3, h3
+; CHECK-NEXT:    fcvt s7, h7
+; CHECK-NEXT:    frintx s0, s0
+; CHECK-NEXT:    fcvt s2, h2
+; CHECK-NEXT:    fcvt s5, h5
+; CHECK-NEXT:    fcvt s6, h6
+; CHECK-NEXT:    frintx s1, s1
+; CHECK-NEXT:    frintx s4, s4
+; CHECK-NEXT:    frintx s3, s3
+; CHECK-NEXT:    frintx s7, s7
+; CHECK-NEXT:    fcvtzs x9, s0
+; CHECK-NEXT:    frintx s2, s2
+; CHECK-NEXT:    frintx s5, s5
+; CHECK-NEXT:    frintx s6, s6
+; CHECK-NEXT:    fcvtzs x8, s1
+; CHECK-NEXT:    fcvtzs x12, s4
+; CHECK-NEXT:    fcvtzs x11, s3
+; CHECK-NEXT:    fcvtzs x15, s7
+; CHECK-NEXT:    fmov d0, x9
+; CHECK-NEXT:    fcvtzs x10, s2
+; CHECK-NEXT:    fcvtzs x13, s5
+; CHECK-NEXT:    fcvtzs x14, s6
+; CHECK-NEXT:    fmov d2, x8
+; CHECK-NEXT:    fmov d1, x12
+; CHECK-NEXT:    mov v0.d[1], x11
+; CHECK-NEXT:    fmov d3, x10
+; CHECK-NEXT:    mov v2.d[1], x13
+; CHECK-NEXT:    mov v1.d[1], x15
+; CHECK-NEXT:    mov v3.d[1], x14
+; CHECK-NEXT:    ret
+  %a = call <8 x i64> @llvm.llrint.v8i64.v8f16(<8 x half> %x)
+  ret <8 x i64> %a
+}
+declare <8 x i64> @llvm.llrint.v8i64.v8f16(<8 x half>)
+
+define <16 x i64> @llrint_v16i64_v16f16(<16 x half> %x) {
+; CHECK-LABEL: llrint_v16i64_v16f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
+; CHECK-NEXT:    mov h17, v0.h[1]
+; CHECK-NEXT:    mov h19, v0.h[2]
+; CHECK-NEXT:    fcvt s18, h0
+; CHECK-NEXT:    mov h0, v0.h[3]
+; CHECK-NEXT:    mov h4, v2.h[1]
+; CHECK-NEXT:    mov h5, v2.h[2]
+; CHECK-NEXT:    fcvt s7, h3
+; CHECK-NEXT:    fcvt s6, h2
+; CHECK-NEXT:    mov h16, v3.h[2]
+; CHECK-NEXT:    mov h2, v2.h[3]
+; CHECK-NEXT:    fcvt s17, h17
+; CHECK-NEXT:    fcvt s19, h19
+; CHECK-NEXT:    frintx s18, s18
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    fcvt s4, h4
+; CHECK-NEXT:    fcvt s5, h5
+; CHECK-NEXT:    frintx s7, s7
+; CHECK-NEXT:    frintx s6, s6
+; CHECK-NEXT:    fcvt s16, h16
+; CHECK-NEXT:    fcvt s2, h2
+; CHECK-NEXT:    frintx s17, s17
+; CHECK-NEXT:    frintx s19, s19
+; CHECK-NEXT:    fcvtzs x13, s18
+; CHECK-NEXT:    frintx s0, s0
+; CHECK-NEXT:    frintx s4, s4
+; CHECK-NEXT:    frintx s5, s5
+; CHECK-NEXT:    fcvtzs x9, s7
+; CHECK-NEXT:    mov h7, v1.h[2]
+; CHECK-NEXT:    fcvtzs x8, s6
+; CHECK-NEXT:    mov h6, v1.h[1]
+; CHECK-NEXT:    frintx s16, s16
+; CHECK-NEXT:    fcvtzs x14, s17
+; CHECK-NEXT:    fcvtzs x15, s19
+; CHECK-NEXT:    fcvtzs x10, s4
+; CHECK-NEXT:    mov h4, v3.h[1]
+; CHECK-NEXT:    fcvtzs x11, s5
+; CHECK-NEXT:    mov h5, v1.h[3]
+; CHECK-NEXT:    mov h3, v3.h[3]
+; CHECK-NEXT:    fcvt s1, h1
+; CHECK-NEXT:    fcvt s7, h7
+; CHECK-NEXT:    fcvt s6, h6
+; CHECK-NEXT:    fcvtzs x12, s16
+; CHECK-NEXT:    frintx s16, s2
+; CHECK-NEXT:    fmov d2, x8
+; CHECK-NEXT:    fcvt s4, h4
+; CHECK-NEXT:    fcvt s3, h3
+; CHECK-NEXT:    fcvt s5, h5
+; CHECK-NEXT:    frintx s1, s1
+; CHECK-NEXT:    frintx s7, s7
+; CHECK-NEXT:    frintx s17, s6
+; CHECK-NEXT:    fmov d6, x9
+; CHECK-NEXT:    mov v2.d[1], x10
+; CHECK-NEXT:    frintx s4, s4
+; CHECK-NEXT:    frintx s18, s3
+; CHECK-NEXT:    frintx s5, s5
+; CHECK-NEXT:    fcvtzs x8, s1
+; CHECK-NEXT:    fcvtzs x9, s7
+; CHECK-NEXT:    fmov d3, x11
+; CHECK-NEXT:    fcvtzs x11, s0
+; CHECK-NEXT:    fmov d7, x12
+; CHECK-NEXT:    fcvtzs x12, s16
+; CHECK-NEXT:    fcvtzs x16, s17
+; CHECK-NEXT:    fcvtzs x17, s4
+; CHECK-NEXT:    fmov d0, x13
+; CHECK-NEXT:    fmov d1, x15
+; CHECK-NEXT:    fcvtzs x18, s18
+; CHECK-NEXT:    fcvtzs x0, s5
+; CHECK-NEXT:    fmov d4, x8
+; CHECK-NEXT:    fmov d5, x9
+; CHECK-NEXT:    mov v0.d[1], x14
+; CHECK-NEXT:    mov v1.d[1], x11
+; CHECK-NEXT:    mov v3.d[1], x12
+; CHECK-NEXT:    mov v4.d[1], x16
+; CHECK-NEXT:    mov v6.d[1], x17
+; CHECK-NEXT:    mov v7.d[1], x18
+; CHECK-NEXT:    mov v5.d[1], x0
+; CHECK-NEXT:    ret
+  %a = call <16 x i64> @llvm.llrint.v16i64.v16f16(<16 x half> %x)
+  ret <16 x i64> %a
+}
+declare <16 x i64> @llvm.llrint.v16i64.v16f16(<16 x half>)
+
+define <32 x i64> @llrint_v32i64_v32f16(<32 x half> %x) {
+; CHECK-LABEL: llrint_v32i64_v32f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ext v4.16b, v1.16b, v1.16b, #8
+; CHECK-NEXT:    ext v5.16b, v2.16b, v2.16b, #8
+; CHECK-NEXT:    ext v6.16b, v3.16b, v3.16b, #8
+; CHECK-NEXT:    ext v7.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    mov h19, v0.h[1]
+; CHECK-NEXT:    fcvt s21, h0
+; CHECK-NEXT:    mov h23, v1.h[2]
+; CHECK-NEXT:    fcvt s22, h1
+; CHECK-NEXT:    fcvt s26, h2
+; CHECK-NEXT:    mov h27, v2.h[1]
+; CHECK-NEXT:    mov h28, v2.h[2]
+; CHECK-NEXT:    mov h16, v4.h[2]
+; CHECK-NEXT:    fcvt s17, h5
+; CHECK-NEXT:    mov h18, v5.h[2]
+; CHECK-NEXT:    mov h20, v6.h[2]
+; CHECK-NEXT:    fcvt s24, h7
+; CHECK-NEXT:    fcvt s25, h6
+; CHECK-NEXT:    fcvt s19, h19
+; CHECK-NEXT:    frintx s22, s22
+; CHECK-NEXT:    fcvt s16, h16
+; CHECK-NEXT:    frintx s17, s17
+; CHECK-NEXT:    fcvt s18, h18
+; CHECK-NEXT:    fcvt s20, h20
+; CHECK-NEXT:    frintx s16, s16
+; CHECK-NEXT:    fcvtzs x12, s17
+; CHECK-NEXT:    frintx s17, s18
+; CHECK-NEXT:    frintx s18, s21
+; CHECK-NEXT:    fcvt s21, h23
+; CHECK-NEXT:    frintx s23, s24
+; CHECK-NEXT:    frintx s24, s25
+; CHECK-NEXT:    frintx s25, s19
+; CHECK-NEXT:    mov h19, v7.h[1]
+; CHECK-NEXT:    fcvtzs x13, s16
+; CHECK-NEXT:    frintx s16, s20
+; CHECK-NEXT:    frintx s20, s26
+; CHECK-NEXT:    fcvtzs x9, s23
+; CHECK-NEXT:    mov h23, v3.h[2]
+; CHECK-NEXT:    fcvt s26, h27
+; CHECK-NEXT:    fcvtzs x15, s24
+; CHECK-NEXT:    fcvtzs x10, s25
+; CHECK-NEXT:    fcvt s24, h28
+; CHECK-NEXT:    mov h25, v3.h[3]
+; CHECK-NEXT:    fcvtzs x14, s17
+; CHECK-NEXT:    frintx s21, s21
+; CHECK-NEXT:    fmov d17, x12
+; CHECK-NEXT:    fcvtzs x12, s16
+; CHECK-NEXT:    fmov d16, x13
+; CHECK-NEXT:    fcvtzs x13, s22
+; CHECK-NEXT:    fcvt s22, h3
+; CHECK-NEXT:    mov h3, v3.h[1]
+; CHECK-NEXT:    mov h27, v0.h[2]
+; CHECK-NEXT:    mov h28, v2.h[3]
+; CHECK-NEXT:    fcvt s23, h23
+; CHECK-NEXT:    frintx s26, s26
+; CHECK-NEXT:    fcvtzs x16, s20
+; CHECK-NEXT:    frintx s20, s24
+; CHECK-NEXT:    fcvt s24, h25
+; CHECK-NEXT:    fcvtzs x11, s18
+; CHECK-NEXT:    fmov d18, x14
+; CHECK-NEXT:    fcvtzs x14, s21
+; CHECK-NEXT:    frintx s22, s22
+; CHECK-NEXT:    fcvt s3, h3
+; CHECK-NEXT:    fcvt s25, h27
+; CHECK-NEXT:    fcvt s27, h28
+; CHECK-NEXT:    frintx s23, s23
+; CHECK-NEXT:    mov h21, v1.h[3]
+; CHECK-NEXT:    fmov d2, x15
+; CHECK-NEXT:    fcvtzs x15, s26
+; CHECK-NEXT:    fmov d26, x13
+; CHECK-NEXT:    mov h1, v1.h[1]
+; CHECK-NEXT:    fcvtzs x13, s20
+; CHECK-NEXT:    frintx s20, s24
+; CHECK-NEXT:    fmov d24, x14
+; CHECK-NEXT:    fcvtzs x14, s22
+; CHECK-NEXT:    frintx s3, s3
+; CHECK-NEXT:    fmov d22, x16
+; CHECK-NEXT:    frintx s27, s27
+; CHECK-NEXT:    fcvtzs x16, s23
+; CHECK-NEXT:    fcvt s21, h21
+; CHECK-NEXT:    frintx s25, s25
+; CHECK-NEXT:    fcvt s1, h1
+; CHECK-NEXT:    mov h0, v0.h[3]
+; CHECK-NEXT:    mov h23, v7.h[2]
+; CHECK-NEXT:    mov v22.d[1], x15
+; CHECK-NEXT:    fcvtzs x15, s20
+; CHECK-NEXT:    fmov d20, x13
+; CHECK-NEXT:    fcvtzs x13, s3
+; CHECK-NEXT:    fmov d3, x14
+; CHECK-NEXT:    fcvtzs x14, s27
+; CHECK-NEXT:    fmov d27, x16
+; CHECK-NEXT:    frintx s21, s21
+; CHECK-NEXT:    mov h7, v7.h[3]
+; CHECK-NEXT:    frintx s1, s1
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    fcvt s23, h23
+; CHECK-NEXT:    fcvt s19, h19
+; CHECK-NEXT:    mov v27.d[1], x15
+; CHECK-NEXT:    fcvtzs x15, s25
+; CHECK-NEXT:    mov h25, v6.h[3]
+; CHECK-NEXT:    mov h6, v6.h[1]
+; CHECK-NEXT:    mov v3.d[1], x13
+; CHECK-NEXT:    fcvtzs x13, s21
+; CHECK-NEXT:    mov h21, v5.h[1]
+; CHECK-NEXT:    mov h5, v5.h[3]
+; CHECK-NEXT:    mov v20.d[1], x14
+; CHECK-NEXT:    fcvtzs x14, s1
+; CHECK-NEXT:    mov h1, v4.h[1]
+; CHECK-NEXT:    frintx s0, s0
+; CHECK-NEXT:    fcvt s25, h25
+; CHECK-NEXT:    fcvt s7, h7
+; CHECK-NEXT:    stp q3, q27, [x8, #192]
+; CHECK-NEXT:    fcvt s6, h6
+; CHECK-NEXT:    mov h3, v4.h[3]
+; CHECK-NEXT:    stp q22, q20, [x8, #128]
+; CHECK-NEXT:    fcvt s21, h21
+; CHECK-NEXT:    fcvt s5, h5
+; CHECK-NEXT:    mov v24.d[1], x13
+; CHECK-NEXT:    mov v26.d[1], x14
+; CHECK-NEXT:    fcvt s4, h4
+; CHECK-NEXT:    frintx s22, s25
+; CHECK-NEXT:    fmov d20, x12
+; CHECK-NEXT:    fcvt s1, h1
+; CHECK-NEXT:    frintx s6, s6
+; CHECK-NEXT:    fcvt s3, h3
+; CHECK-NEXT:    fcvtzs x12, s0
+; CHECK-NEXT:    frintx s5, s5
+; CHECK-NEXT:    frintx s21, s21
+; CHECK-NEXT:    fmov d0, x11
+; CHECK-NEXT:    stp q26, q24, [x8, #64]
+; CHECK-NEXT:    fmov d24, x15
+; CHECK-NEXT:    frintx s4, s4
+; CHECK-NEXT:    fcvtzs x11, s22
+; CHECK-NEXT:    frintx s22, s23
+; CHECK-NEXT:    frintx s1, s1
+; CHECK-NEXT:    fcvtzs x13, s6
+; CHECK-NEXT:    frintx s3, s3
+; CHECK-NEXT:    frintx s6, s7
+; CHECK-NEXT:    fcvtzs x14, s5
+; CHECK-NEXT:    mov v24.d[1], x12
+; CHECK-NEXT:    frintx s5, s19
+; CHECK-NEXT:    fcvtzs x12, s21
+; CHECK-NEXT:    mov v0.d[1], x10
+; CHECK-NEXT:    fcvtzs x10, s4
+; CHECK-NEXT:    mov v20.d[1], x11
+; CHECK-NEXT:    fcvtzs x11, s22
+; CHECK-NEXT:    mov v2.d[1], x13
+; CHECK-NEXT:    fcvtzs x15, s3
+; CHECK-NEXT:    fcvtzs x13, s1
+; CHECK-NEXT:    mov v18.d[1], x14
+; CHECK-NEXT:    fcvtzs x14, s6
+; CHECK-NEXT:    stp q0, q24, [x8]
+; CHECK-NEXT:    mov v17.d[1], x12
+; CHECK-NEXT:    fcvtzs x12, s5
+; CHECK-NEXT:    fmov d0, x10
+; CHECK-NEXT:    fmov d1, x11
+; CHECK-NEXT:    stp q2, q20, [x8, #224]
+; CHECK-NEXT:    fmov d2, x9
+; CHECK-NEXT:    mov v16.d[1], x15
+; CHECK-NEXT:    stp q17, q18, [x8, #160]
+; CHECK-NEXT:    mov v0.d[1], x13
+; CHECK-NEXT:    mov v1.d[1], x14
+; CHECK-NEXT:    mov v2.d[1], x12
+; CHECK-NEXT:    stp q0, q16, [x8, #96]
+; CHECK-NEXT:    stp q2, q1, [x8, #32]
+; CHECK-NEXT:    ret
+  %a = call <32 x i64> @llvm.llrint.v32i64.v32f16(<32 x half> %x)
+  ret <32 x i64> %a
+}
+declare <32 x i64> @llvm.llrint.v32i64.v32f16(<32 x half>)
+
+define <1 x i64> @llrint_v1i64_v1f32(<1 x float> %x) {
+; CHECK-LABEL: llrint_v1i64_v1f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    frintx s0, s0
+; CHECK-NEXT:    fcvtzs x8, s0
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    ret
+  %a = call <1 x i64> @llvm.llrint.v1i64.v1f32(<1 x float> %x)
+  ret <1 x i64> %a
+}
+declare <1 x i64> @llvm.llrint.v1i64.v1f32(<1 x float>)
+
+define <2 x i64> @llrint_v2i64_v2f32(<2 x float> %x) {
+; CHECK-LABEL: llrint_v2i64_v2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    mov s1, v0.s[1]
+; CHECK-NEXT:    frintx s0, s0
+; CHECK-NEXT:    frintx s1, s1
+; CHECK-NEXT:    fcvtzs x8, s0
+; CHECK-NEXT:    fcvtzs x9, s1
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    mov v0.d[1], x9
+; CHECK-NEXT:    ret
+  %a = call <2 x i64> @llvm.llrint.v2i64.v2f32(<2 x float> %x)
+  ret <2 x i64> %a
+}
+declare <2 x i64> @llvm.llrint.v2i64.v2f32(<2 x float>)
+
+define <4 x i64> @llrint_v4i64_v4f32(<4 x float> %x) {
+; CHECK-LABEL: llrint_v4i64_v4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    mov s3, v0.s[1]
+; CHECK-NEXT:    frintx s0, s0
+; CHECK-NEXT:    mov s2, v1.s[1]
+; CHECK-NEXT:    frintx s1, s1
+; CHECK-NEXT:    frintx s3, s3
+; CHECK-NEXT:    fcvtzs x9, s0
+; CHECK-NEXT:    frintx s2, s2
+; CHECK-NEXT:    fcvtzs x8, s1
+; CHECK-NEXT:    fcvtzs x11, s3
+; CHECK-NEXT:    fmov d0, x9
+; CHECK-NEXT:    fcvtzs x10, s2
+; CHECK-NEXT:    fmov d1, x8
+; CHECK-NEXT:    mov v0.d[1], x11
+; CHECK-NEXT:    mov v1.d[1], x10
+; CHECK-NEXT:    ret
+  %a = call <4 x i64> @llvm.llrint.v4i64.v4f32(<4 x float> %x)
+  ret <4 x i64> %a
+}
+declare <4 x i64> @llvm.llrint.v4i64.v4f32(<4 x float>)
+
+define <8 x i64> @llrint_v8i64_v8f32(<8 x float> %x) {
+; CHECK-LABEL: llrint_v8i64_v8f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
+; CHECK-NEXT:    mov s4, v0.s[1]
+; CHECK-NEXT:    mov s7, v1.s[1]
+; CHECK-NEXT:    frintx s0, s0
+; CHECK-NEXT:    frintx s1, s1
+; CHECK-NEXT:    mov s5, v2.s[1]
+; CHECK-NEXT:    mov s6, v3.s[1]
+; CHECK-NEXT:    frintx s2, s2
+; CHECK-NEXT:    frintx s3, s3
+; CHECK-NEXT:    frintx s4, s4
+; CHECK-NEXT:    frintx s7, s7
+; CHECK-NEXT:    fcvtzs x9, s0
+; CHECK-NEXT:    fcvtzs x12, s1
+; CHECK-NEXT:    frintx s5, s5
+; CHECK-NEXT:    frintx s6, s6
+; CHECK-NEXT:    fcvtzs x8, s2
+; CHECK-NEXT:    fcvtzs x10, s3
+; CHECK-NEXT:    fcvtzs x11, s4
+; CHECK-NEXT:    fcvtzs x15, s7
+; CHECK-NEXT:    fmov d0, x9
+; CHECK-NEXT:    fmov d2, x12
+; CHECK-NEXT:    fcvtzs x13, s5
+; CHECK-NEXT:    fcvtzs x14, s6
+; CHECK-NEXT:    fmov d1, x8
+; CHECK-NEXT:    fmov d3, x10
+; CHECK-NEXT:    mov v0.d[1], x11
+; CHECK-NEXT:    mov v2.d[1], x15
+; CHECK-NEXT:    mov v1.d[1], x13
+; CHECK-NEXT:    mov v3.d[1], x14
+; CHECK-NEXT:    ret
+  %a = call <8 x i64> @llvm.llrint.v8i64.v8f32(<8 x float> %x)
+  ret <8 x i64> %a
+}
+declare <8 x i64> @llvm.llrint.v8i64.v8f32(<8 x float>)
+
+define <16 x i64> @llrint_v16i64_v16f32(<16 x float> %x) {
+; CHECK-LABEL: llrint_v16i64_v16f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ext v4.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    ext v5.16b, v1.16b, v1.16b, #8
+; CHECK-NEXT:    ext v6.16b, v2.16b, v2.16b, #8
+; CHECK-NEXT:    frintx s7, s0
+; CHECK-NEXT:    ext v16.16b, v3.16b, v3.16b, #8
+; CHECK-NEXT:    mov s0, v0.s[1]
+; CHECK-NEXT:    frintx s17, s4
+; CHECK-NEXT:    mov s4, v4.s[1]
+; CHECK-NEXT:    mov s18, v5.s[1]
+; CHECK-NEXT:    frintx s5, s5
+; CHECK-NEXT:    frintx s19, s6
+; CHECK-NEXT:    fcvtzs x8, s7
+; CHECK-NEXT:    frintx s7, s16
+; CHECK-NEXT:    mov s6, v6.s[1]
+; CHECK-NEXT:    mov s16, v16.s[1]
+; CHECK-NEXT:    frintx s0, s0
+; CHECK-NEXT:    frintx s4, s4
+; CHECK-NEXT:    fcvtzs x9, s17
+; CHECK-NEXT:    frintx s17, s1
+; CHECK-NEXT:    mov s1, v1.s[1]
+; CHECK-NEXT:    frintx s18, s18
+; CHECK-NEXT:    fcvtzs x10, s5
+; CHECK-NEXT:    mov s5, v2.s[1]
+; CHECK-NEXT:    fcvtzs x11, s19
+; CHECK-NEXT:    mov s19, v3.s[1]
+; CHECK-NEXT:    frintx s2, s2
+; CHECK-NEXT:    fcvtzs x12, s7
+; CHECK-NEXT:    frintx s6, s6
+; CHECK-NEXT:    fcvtzs x13, s4
+; CHECK-NEXT:    frintx s4, s3
+; CHECK-NEXT:    frintx s16, s16
+; CHECK-NEXT:    fcvtzs x14, s18
+; CHECK-NEXT:    frintx s18, s1
+; CHECK-NEXT:    fcvtzs x15, s17
+; CHECK-NEXT:    frintx s20, s5
+; CHECK-NEXT:    frintx s17, s19
+; CHECK-NEXT:    fmov d1, x9
+; CHECK-NEXT:    fcvtzs x9, s2
+; CHECK-NEXT:    fmov d5, x11
+; CHECK-NEXT:    fmov d3, x10
+; CHECK-NEXT:    fcvtzs x11, s4
+; CHECK-NEXT:    fcvtzs x10, s0
+; CHECK-NEXT:    fmov d7, x12
+; CHECK-NEXT:    fcvtzs x12, s18
+; CHECK-NEXT:    fcvtzs x17, s6
+; CHECK-NEXT:    fcvtzs x18, s16
+; CHECK-NEXT:    fcvtzs x16, s20
+; CHECK-NEXT:    fcvtzs x0, s17
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    fmov d2, x15
+; CHECK-NEXT:    fmov d4, x9
+; CHECK-NEXT:    mov v1.d[1], x13
+; CHECK-NEXT:    fmov d6, x11
+; CHECK-NEXT:    mov v3.d[1], x14
+; CHECK-NEXT:    mov v0.d[1], x10
+; CHECK-NEXT:    mov v5.d[1], x17
+; CHECK-NEXT:    mov v7.d[1], x18
+; CHECK-NEXT:    mov v2.d[1], x12
+; CHECK-NEXT:    mov v4.d[1], x16
+; CHECK-NEXT:    mov v6.d[1], x0
+; CHECK-NEXT:    ret
+  %a = call <16 x i64> @llvm.llrint.v16i64.v16f32(<16 x float> %x)
+  ret <16 x i64> %a
+}
+declare <16 x i64> @llvm.llrint.v16i64.v16f32(<16 x float>)
+
+define <1 x i64> @llrint_v1i64_v1f64(<1 x double> %x) {
+; CHECK-LABEL: llrint_v1i64_v1f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintx d0, d0
+; CHECK-NEXT:    fcvtzs x8, d0
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    ret
+  %a = call <1 x i64> @llvm.llrint.v1i64.v1f64(<1 x double> %x)
+  ret <1 x i64> %a
+}
+declare <1 x i64> @llvm.llrint.v1i64.v1f64(<1 x double>)
+
+define <2 x i64> @llrint_v2i64_v2f64(<2 x double> %x) {
+; CHECK-LABEL: llrint_v2i64_v2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov d1, v0.d[1]
+; CHECK-NEXT:    frintx d0, d0
+; CHECK-NEXT:    frintx d1, d1
+; CHECK-NEXT:    fcvtzs x8, d0
+; CHECK-NEXT:    fcvtzs x9, d1
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    mov v0.d[1], x9
+; CHECK-NEXT:    ret
+  %a = call <2 x i64> @llvm.llrint.v2i64.v2f64(<2 x double> %x)
+  ret <2 x i64> %a
+}
+declare <2 x i64> @llvm.llrint.v2i64.v2f64(<2 x double>)
+
+define <4 x i64> @llrint_v4i64_v4f64(<4 x double> %x) {
+; CHECK-LABEL: llrint_v4i64_v4f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov d2, v0.d[1]
+; CHECK-NEXT:    mov d3, v1.d[1]
+; CHECK-NEXT:    frintx d0, d0
+; CHECK-NEXT:    frintx d1, d1
+; CHECK-NEXT:    frintx d2, d2
+; CHECK-NEXT:    frintx d3, d3
+; CHECK-NEXT:    fcvtzs x8, d0
+; CHECK-NEXT:    fcvtzs x9, d1
+; CHECK-NEXT:    fcvtzs x10, d2
+; CHECK-NEXT:    fcvtzs x11, d3
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    fmov d1, x9
+; CHECK-NEXT:    mov v0.d[1], x10
+; CHECK-NEXT:    mov v1.d[1], x11
+; CHECK-NEXT:    ret
+  %a = call <4 x i64> @llvm.llrint.v4i64.v4f64(<4 x double> %x)
+  ret <4 x i64> %a
+}
+declare <4 x i64> @llvm.llrint.v4i64.v4f64(<4 x double>)
+
+define <8 x i64> @llrint_v8i64_v8f64(<8 x double> %x) {
+; CHECK-LABEL: llrint_v8i64_v8f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov d4, v0.d[1]
+; CHECK-NEXT:    mov d5, v1.d[1]
+; CHECK-NEXT:    mov d6, v2.d[1]
+; CHECK-NEXT:    mov d7, v3.d[1]
+; CHECK-NEXT:    frintx d0, d0
+; CHECK-NEXT:    frintx d1, d1
+; CHECK-NEXT:    frintx d2, d2
+; CHECK-NEXT:    frintx d3, d3
+; CHECK-NEXT:    frintx d4, d4
+; CHECK-NEXT:    frintx d5, d5
+; CHECK-NEXT:    frintx d6, d6
+; CHECK-NEXT:    frintx d7, d7
+; CHECK-NEXT:    fcvtzs x8, d0
+; CHECK-NEXT:    fcvtzs x9, d1
+; CHECK-NEXT:    fcvtzs x10, d2
+; CHECK-NEXT:    fcvtzs x11, d3
+; CHECK-NEXT:    fcvtzs x12, d4
+; CHECK-NEXT:    fcvtzs x13, d5
+; CHECK-NEXT:    fcvtzs x14, d6
+; CHECK-NEXT:    fcvtzs x15, d7
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    fmov d1, x9
+; CHECK-NEXT:    fmov d2, x10
+; CHECK-NEXT:    fmov d3, x11
+; CHECK-NEXT:    mov v0.d[1], x12
+; CHECK-NEXT:    mov v1.d[1], x13
+; CHECK-NEXT:    mov v2.d[1], x14
+; CHECK-NEXT:    mov v3.d[1], x15
+; CHECK-NEXT:    ret
+  %a = call <8 x i64> @llvm.llrint.v8i64.v8f64(<8 x double> %x)
+  ret <8 x i64> %a
+}
+declare <8 x i64> @llvm.llrint.v8i64.v8f64(<8 x double>)
diff --git a/llvm/test/CodeGen/AArch64/vector-lrint.ll b/llvm/test/CodeGen/AArch64/vector-lrint.ll
new file mode 100644
index 0000000000..9c46cf69cb
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/vector-lrint.ll
@@ -0,0 +1,622 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=aarch64 -mattr=+neon | FileCheck %s
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=aarch64 -mattr=+neon | FileCheck %s
+
+define <1 x i64> @lrint_v1f16(<1 x half> %x) {
+; CHECK-LABEL: lrint_v1f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    frintx s0, s0
+; CHECK-NEXT:    fcvtzs x8, s0
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    ret
+  %a = call <1 x i64> @llvm.lrint.v1i64.v1f16(<1 x half> %x)
+  ret <1 x i64> %a
+}
+declare <1 x i64> @llvm.lrint.v1i64.v1f16(<1 x half>)
+
+define <2 x i64> @lrint_v2f16(<2 x half> %x) {
+; CHECK-LABEL: lrint_v2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    mov h1, v0.h[1]
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    fcvt s1, h1
+; CHECK-NEXT:    frintx s0, s0
+; CHECK-NEXT:    frintx s1, s1
+; CHECK-NEXT:    fcvtzs x8, s0
+; CHECK-NEXT:    fcvtzs x9, s1
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    mov v0.d[1], x9
+; CHECK-NEXT:    ret
+  %a = call <2 x i64> @llvm.lrint.v2i64.v2f16(<2 x half> %x)
+  ret <2 x i64> %a
+}
+declare <2 x i64> @llvm.lrint.v2i64.v2f16(<2 x half>)
+
+define <4 x i64> @lrint_v4f16(<4 x half> %x) {
+; CHECK-LABEL: lrint_v4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    mov h1, v0.h[2]
+; CHECK-NEXT:    mov h2, v0.h[1]
+; CHECK-NEXT:    mov h3, v0.h[3]
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    fcvt s1, h1
+; CHECK-NEXT:    fcvt s2, h2
+; CHECK-NEXT:    fcvt s3, h3
+; CHECK-NEXT:    frintx s0, s0
+; CHECK-NEXT:    frintx s1, s1
+; CHECK-NEXT:    frintx s2, s2
+; CHECK-NEXT:    frintx s3, s3
+; CHECK-NEXT:    fcvtzs x8, s0
+; CHECK-NEXT:    fcvtzs x9, s1
+; CHECK-NEXT:    fcvtzs x10, s2
+; CHECK-NEXT:    fcvtzs x11, s3
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    fmov d1, x9
+; CHECK-NEXT:    mov v0.d[1], x10
+; CHECK-NEXT:    mov v1.d[1], x11
+; CHECK-NEXT:    ret
+  %a = call <4 x i64> @llvm.lrint.v4i64.v4f16(<4 x half> %x)
+  ret <4 x i64> %a
+}
+declare <4 x i64> @llvm.lrint.v4i64.v4f16(<4 x half>)
+
+define <8 x i64> @lrint_v8f16(<8 x half> %x) {
+; CHECK-LABEL: lrint_v8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    mov h4, v0.h[2]
+; CHECK-NEXT:    mov h3, v0.h[1]
+; CHECK-NEXT:    mov h7, v0.h[3]
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    mov h2, v1.h[2]
+; CHECK-NEXT:    mov h5, v1.h[1]
+; CHECK-NEXT:    mov h6, v1.h[3]
+; CHECK-NEXT:    fcvt s1, h1
+; CHECK-NEXT:    fcvt s4, h4
+; CHECK-NEXT:    fcvt s3, h3
+; CHECK-NEXT:    fcvt s7, h7
+; CHECK-NEXT:    frintx s0, s0
+; CHECK-NEXT:    fcvt s2, h2
+; CHECK-NEXT:    fcvt s5, h5
+; CHECK-NEXT:    fcvt s6, h6
+; CHECK-NEXT:    frintx s1, s1
+; CHECK-NEXT:    frintx s4, s4
+; CHECK-NEXT:    frintx s3, s3
+; CHECK-NEXT:    frintx s7, s7
+; CHECK-NEXT:    fcvtzs x9, s0
+; CHECK-NEXT:    frintx s2, s2
+; CHECK-NEXT:    frintx s5, s5
+; CHECK-NEXT:    frintx s6, s6
+; CHECK-NEXT:    fcvtzs x8, s1
+; CHECK-NEXT:    fcvtzs x12, s4
+; CHECK-NEXT:    fcvtzs x11, s3
+; CHECK-NEXT:    fcvtzs x15, s7
+; CHECK-NEXT:    fmov d0, x9
+; CHECK-NEXT:    fcvtzs x10, s2
+; CHECK-NEXT:    fcvtzs x13, s5
+; CHECK-NEXT:    fcvtzs x14, s6
+; CHECK-NEXT:    fmov d2, x8
+; CHECK-NEXT:    fmov d1, x12
+; CHECK-NEXT:    mov v0.d[1], x11
+; CHECK-NEXT:    fmov d3, x10
+; CHECK-NEXT:    mov v2.d[1], x13
+; CHECK-NEXT:    mov v1.d[1], x15
+; CHECK-NEXT:    mov v3.d[1], x14
+; CHECK-NEXT:    ret
+  %a = call <8 x i64> @llvm.lrint.v8i64.v8f16(<8 x half> %x)
+  ret <8 x i64> %a
+}
+declare <8 x i64> @llvm.lrint.v8i64.v8f16(<8 x half>)
+
+define <16 x i64> @lrint_v16i64_v16f16(<16 x half> %x) {
+; CHECK-LABEL: lrint_v16i64_v16f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
+; CHECK-NEXT:    mov h17, v0.h[1]
+; CHECK-NEXT:    mov h19, v0.h[2]
+; CHECK-NEXT:    fcvt s18, h0
+; CHECK-NEXT:    mov h0, v0.h[3]
+; CHECK-NEXT:    mov h4, v2.h[1]
+; CHECK-NEXT:    mov h5, v2.h[2]
+; CHECK-NEXT:    fcvt s7, h3
+; CHECK-NEXT:    fcvt s6, h2
+; CHECK-NEXT:    mov h16, v3.h[2]
+; CHECK-NEXT:    mov h2, v2.h[3]
+; CHECK-NEXT:    fcvt s17, h17
+; CHECK-NEXT:    fcvt s19, h19
+; CHECK-NEXT:    frintx s18, s18
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    fcvt s4, h4
+; CHECK-NEXT:    fcvt s5, h5
+; CHECK-NEXT:    frintx s7, s7
+; CHECK-NEXT:    frintx s6, s6
+; CHECK-NEXT:    fcvt s16, h16
+; CHECK-NEXT:    fcvt s2, h2
+; CHECK-NEXT:    frintx s17, s17
+; CHECK-NEXT:    frintx s19, s19
+; CHECK-NEXT:    fcvtzs x13, s18
+; CHECK-NEXT:    frintx s0, s0
+; CHECK-NEXT:    frintx s4, s4
+; CHECK-NEXT:    frintx s5, s5
+; CHECK-NEXT:    fcvtzs x9, s7
+; CHECK-NEXT:    mov h7, v1.h[2]
+; CHECK-NEXT:    fcvtzs x8, s6
+; CHECK-NEXT:    mov h6, v1.h[1]
+; CHECK-NEXT:    frintx s16, s16
+; CHECK-NEXT:    fcvtzs x14, s17
+; CHECK-NEXT:    fcvtzs x15, s19
+; CHECK-NEXT:    fcvtzs x10, s4
+; CHECK-NEXT:    mov h4, v3.h[1]
+; CHECK-NEXT:    fcvtzs x11, s5
+; CHECK-NEXT:    mov h5, v1.h[3]
+; CHECK-NEXT:    mov h3, v3.h[3]
+; CHECK-NEXT:    fcvt s1, h1
+; CHECK-NEXT:    fcvt s7, h7
+; CHECK-NEXT:    fcvt s6, h6
+; CHECK-NEXT:    fcvtzs x12, s16
+; CHECK-NEXT:    frintx s16, s2
+; CHECK-NEXT:    fmov d2, x8
+; CHECK-NEXT:    fcvt s4, h4
+; CHECK-NEXT:    fcvt s3, h3
+; CHECK-NEXT:    fcvt s5, h5
+; CHECK-NEXT:    frintx s1, s1
+; CHECK-NEXT:    frintx s7, s7
+; CHECK-NEXT:    frintx s17, s6
+; CHECK-NEXT:    fmov d6, x9
+; CHECK-NEXT:    mov v2.d[1], x10
+; CHECK-NEXT:    frintx s4, s4
+; CHECK-NEXT:    frintx s18, s3
+; CHECK-NEXT:    frintx s5, s5
+; CHECK-NEXT:    fcvtzs x8, s1
+; CHECK-NEXT:    fcvtzs x9, s7
+; CHECK-NEXT:    fmov d3, x11
+; CHECK-NEXT:    fcvtzs x11, s0
+; CHECK-NEXT:    fmov d7, x12
+; CHECK-NEXT:    fcvtzs x12, s16
+; CHECK-NEXT:    fcvtzs x16, s17
+; CHECK-NEXT:    fcvtzs x17, s4
+; CHECK-NEXT:    fmov d0, x13
+; CHECK-NEXT:    fmov d1, x15
+; CHECK-NEXT:    fcvtzs x18, s18
+; CHECK-NEXT:    fcvtzs x0, s5
+; CHECK-NEXT:    fmov d4, x8
+; CHECK-NEXT:    fmov d5, x9
+; CHECK-NEXT:    mov v0.d[1], x14
+; CHECK-NEXT:    mov v1.d[1], x11
+; CHECK-NEXT:    mov v3.d[1], x12
+; CHECK-NEXT:    mov v4.d[1], x16
+; CHECK-NEXT:    mov v6.d[1], x17
+; CHECK-NEXT:    mov v7.d[1], x18
+; CHECK-NEXT:    mov v5.d[1], x0
+; CHECK-NEXT:    ret
+  %a = call <16 x i64> @llvm.lrint.v16i64.v16f16(<16 x half> %x)
+  ret <16 x i64> %a
+}
+declare <16 x i64> @llvm.lrint.v16i64.v16f16(<16 x half>)
+
+define <32 x i64> @lrint_v32i64_v32f16(<32 x half> %x) {
+; CHECK-LABEL: lrint_v32i64_v32f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ext v4.16b, v1.16b, v1.16b, #8
+; CHECK-NEXT:    ext v5.16b, v2.16b, v2.16b, #8
+; CHECK-NEXT:    ext v6.16b, v3.16b, v3.16b, #8
+; CHECK-NEXT:    ext v7.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    mov h19, v0.h[1]
+; CHECK-NEXT:    fcvt s21, h0
+; CHECK-NEXT:    mov h23, v1.h[2]
+; CHECK-NEXT:    fcvt s22, h1
+; CHECK-NEXT:    fcvt s26, h2
+; CHECK-NEXT:    mov h27, v2.h[1]
+; CHECK-NEXT:    mov h28, v2.h[2]
+; CHECK-NEXT:    mov h16, v4.h[2]
+; CHECK-NEXT:    fcvt s17, h5
+; CHECK-NEXT:    mov h18, v5.h[2]
+; CHECK-NEXT:    mov h20, v6.h[2]
+; CHECK-NEXT:    fcvt s24, h7
+; CHECK-NEXT:    fcvt s25, h6
+; CHECK-NEXT:    fcvt s19, h19
+; CHECK-NEXT:    frintx s22, s22
+; CHECK-NEXT:    fcvt s16, h16
+; CHECK-NEXT:    frintx s17, s17
+; CHECK-NEXT:    fcvt s18, h18
+; CHECK-NEXT:    fcvt s20, h20
+; CHECK-NEXT:    frintx s16, s16
+; CHECK-NEXT:    fcvtzs x12, s17
+; CHECK-NEXT:    frintx s17, s18
+; CHECK-NEXT:    frintx s18, s21
+; CHECK-NEXT:    fcvt s21, h23
+; CHECK-NEXT:    frintx s23, s24
+; CHECK-NEXT:    frintx s24, s25
+; CHECK-NEXT:    frintx s25, s19
+; CHECK-NEXT:    mov h19, v7.h[1]
+; CHECK-NEXT:    fcvtzs x13, s16
+; CHECK-NEXT:    frintx s16, s20
+; CHECK-NEXT:    frintx s20, s26
+; CHECK-NEXT:    fcvtzs x9, s23
+; CHECK-NEXT:    mov h23, v3.h[2]
+; CHECK-NEXT:    fcvt s26, h27
+; CHECK-NEXT:    fcvtzs x15, s24
+; CHECK-NEXT:    fcvtzs x10, s25
+; CHECK-NEXT:    fcvt s24, h28
+; CHECK-NEXT:    mov h25, v3.h[3]
+; CHECK-NEXT:    fcvtzs x14, s17
+; CHECK-NEXT:    frintx s21, s21
+; CHECK-NEXT:    fmov d17, x12
+; CHECK-NEXT:    fcvtzs x12, s16
+; CHECK-NEXT:    fmov d16, x13
+; CHECK-NEXT:    fcvtzs x13, s22
+; CHECK-NEXT:    fcvt s22, h3
+; CHECK-NEXT:    mov h3, v3.h[1]
+; CHECK-NEXT:    mov h27, v0.h[2]
+; CHECK-NEXT:    mov h28, v2.h[3]
+; CHECK-NEXT:    fcvt s23, h23
+; CHECK-NEXT:    frintx s26, s26
+; CHECK-NEXT:    fcvtzs x16, s20
+; CHECK-NEXT:    frintx s20, s24
+; CHECK-NEXT:    fcvt s24, h25
+; CHECK-NEXT:    fcvtzs x11, s18
+; CHECK-NEXT:    fmov d18, x14
+; CHECK-NEXT:    fcvtzs x14, s21
+; CHECK-NEXT:    frintx s22, s22
+; CHECK-NEXT:    fcvt s3, h3
+; CHECK-NEXT:    fcvt s25, h27
+; CHECK-NEXT:    fcvt s27, h28
+; CHECK-NEXT:    frintx s23, s23
+; CHECK-NEXT:    mov h21, v1.h[3]
+; CHECK-NEXT:    fmov d2, x15
+; CHECK-NEXT:    fcvtzs x15, s26
+; CHECK-NEXT:    fmov d26, x13
+; CHECK-NEXT:    mov h1, v1.h[1]
+; CHECK-NEXT:    fcvtzs x13, s20
+; CHECK-NEXT:    frintx s20, s24
+; CHECK-NEXT:    fmov d24, x14
+; CHECK-NEXT:    fcvtzs x14, s22
+; CHECK-NEXT:    frintx s3, s3
+; CHECK-NEXT:    fmov d22, x16
+; CHECK-NEXT:    frintx s27, s27
+; CHECK-NEXT:    fcvtzs x16, s23
+; CHECK-NEXT:    fcvt s21, h21
+; CHECK-NEXT:    frintx s25, s25
+; CHECK-NEXT:    fcvt s1, h1
+; CHECK-NEXT:    mov h0, v0.h[3]
+; CHECK-NEXT:    mov h23, v7.h[2]
+; CHECK-NEXT:    mov v22.d[1], x15
+; CHECK-NEXT:    fcvtzs x15, s20
+; CHECK-NEXT:    fmov d20, x13
+; CHECK-NEXT:    fcvtzs x13, s3
+; CHECK-NEXT:    fmov d3, x14
+; CHECK-NEXT:    fcvtzs x14, s27
+; CHECK-NEXT:    fmov d27, x16
+; CHECK-NEXT:    frintx s21, s21
+; CHECK-NEXT:    mov h7, v7.h[3]
+; CHECK-NEXT:    frintx s1, s1
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    fcvt s23, h23
+; CHECK-NEXT:    fcvt s19, h19
+; CHECK-NEXT:    mov v27.d[1], x15
+; CHECK-NEXT:    fcvtzs x15, s25
+; CHECK-NEXT:    mov h25, v6.h[3]
+; CHECK-NEXT:    mov h6, v6.h[1]
+; CHECK-NEXT:    mov v3.d[1], x13
+; CHECK-NEXT:    fcvtzs x13, s21
+; CHECK-NEXT:    mov h21, v5.h[1]
+; CHECK-NEXT:    mov h5, v5.h[3]
+; CHECK-NEXT:    mov v20.d[1], x14
+; CHECK-NEXT:    fcvtzs x14, s1
+; CHECK-NEXT:    mov h1, v4.h[1]
+; CHECK-NEXT:    frintx s0, s0
+; CHECK-NEXT:    fcvt s25, h25
+; CHECK-NEXT:    fcvt s7, h7
+; CHECK-NEXT:    stp q3, q27, [x8, #192]
+; CHECK-NEXT:    fcvt s6, h6
+; CHECK-NEXT:    mov h3, v4.h[3]
+; CHECK-NEXT:    stp q22, q20, [x8, #128]
+; CHECK-NEXT:    fcvt s21, h21
+; CHECK-NEXT:    fcvt s5, h5
+; CHECK-NEXT:    mov v24.d[1], x13
+; CHECK-NEXT:    mov v26.d[1], x14
+; CHECK-NEXT:    fcvt s4, h4
+; CHECK-NEXT:    frintx s22, s25
+; CHECK-NEXT:    fmov d20, x12
+; CHECK-NEXT:    fcvt s1, h1
+; CHECK-NEXT:    frintx s6, s6
+; CHECK-NEXT:    fcvt s3, h3
+; CHECK-NEXT:    fcvtzs x12, s0
+; CHECK-NEXT:    frintx s5, s5
+; CHECK-NEXT:    frintx s21, s21
+; CHECK-NEXT:    fmov d0, x11
+; CHECK-NEXT:    stp q26, q24, [x8, #64]
+; CHECK-NEXT:    fmov d24, x15
+; CHECK-NEXT:    frintx s4, s4
+; CHECK-NEXT:    fcvtzs x11, s22
+; CHECK-NEXT:    frintx s22, s23
+; CHECK-NEXT:    frintx s1, s1
+; CHECK-NEXT:    fcvtzs x13, s6
+; CHECK-NEXT:    frintx s3, s3
+; CHECK-NEXT:    frintx s6, s7
+; CHECK-NEXT:    fcvtzs x14, s5
+; CHECK-NEXT:    mov v24.d[1], x12
+; CHECK-NEXT:    frintx s5, s19
+; CHECK-NEXT:    fcvtzs x12, s21
+; CHECK-NEXT:    mov v0.d[1], x10
+; CHECK-NEXT:    fcvtzs x10, s4
+; CHECK-NEXT:    mov v20.d[1], x11
+; CHECK-NEXT:    fcvtzs x11, s22
+; CHECK-NEXT:    mov v2.d[1], x13
+; CHECK-NEXT:    fcvtzs x15, s3
+; CHECK-NEXT:    fcvtzs x13, s1
+; CHECK-NEXT:    mov v18.d[1], x14
+; CHECK-NEXT:    fcvtzs x14, s6
+; CHECK-NEXT:    stp q0, q24, [x8]
+; CHECK-NEXT:    mov v17.d[1], x12
+; CHECK-NEXT:    fcvtzs x12, s5
+; CHECK-NEXT:    fmov d0, x10
+; CHECK-NEXT:    fmov d1, x11
+; CHECK-NEXT:    stp q2, q20, [x8, #224]
+; CHECK-NEXT:    fmov d2, x9
+; CHECK-NEXT:    mov v16.d[1], x15
+; CHECK-NEXT:    stp q17, q18, [x8, #160]
+; CHECK-NEXT:    mov v0.d[1], x13
+; CHECK-NEXT:    mov v1.d[1], x14
+; CHECK-NEXT:    mov v2.d[1], x12
+; CHECK-NEXT:    stp q0, q16, [x8, #96]
+; CHECK-NEXT:    stp q2, q1, [x8, #32]
+; CHECK-NEXT:    ret
+  %a = call <32 x i64> @llvm.lrint.v32i64.v32f16(<32 x half> %x)
+  ret <32 x i64> %a
+}
+declare <32 x i64> @llvm.lrint.v32i64.v32f16(<32 x half>)
+
+define <1 x i64> @lrint_v1f32(<1 x float> %x) {
+; CHECK-LABEL: lrint_v1f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    frintx s0, s0
+; CHECK-NEXT:    fcvtzs x8, s0
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    ret
+  %a = call <1 x i64> @llvm.lrint.v1i64.v1f32(<1 x float> %x)
+  ret <1 x i64> %a
+}
+declare <1 x i64> @llvm.lrint.v1i64.v1f32(<1 x float>)
+
+define <2 x i64> @lrint_v2f32(<2 x float> %x) {
+; CHECK-LABEL: lrint_v2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    mov s1, v0.s[1]
+; CHECK-NEXT:    frintx s0, s0
+; CHECK-NEXT:    frintx s1, s1
+; CHECK-NEXT:    fcvtzs x8, s0
+; CHECK-NEXT:    fcvtzs x9, s1
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    mov v0.d[1], x9
+; CHECK-NEXT:    ret
+  %a = call <2 x i64> @llvm.lrint.v2i64.v2f32(<2 x float> %x)
+  ret <2 x i64> %a
+}
+declare <2 x i64> @llvm.lrint.v2i64.v2f32(<2 x float>)
+
+define <4 x i64> @lrint_v4f32(<4 x float> %x) {
+; CHECK-LABEL: lrint_v4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    mov s3, v0.s[1]
+; CHECK-NEXT:    frintx s0, s0
+; CHECK-NEXT:    mov s2, v1.s[1]
+; CHECK-NEXT:    frintx s1, s1
+; CHECK-NEXT:    frintx s3, s3
+; CHECK-NEXT:    fcvtzs x9, s0
+; CHECK-NEXT:    frintx s2, s2
+; CHECK-NEXT:    fcvtzs x8, s1
+; CHECK-NEXT:    fcvtzs x11, s3
+; CHECK-NEXT:    fmov d0, x9
+; CHECK-NEXT:    fcvtzs x10, s2
+; CHECK-NEXT:    fmov d1, x8
+; CHECK-NEXT:    mov v0.d[1], x11
+; CHECK-NEXT:    mov v1.d[1], x10
+; CHECK-NEXT:    ret
+  %a = call <4 x i64> @llvm.lrint.v4i64.v4f32(<4 x float> %x)
+  ret <4 x i64> %a
+}
+declare <4 x i64> @llvm.lrint.v4i64.v4f32(<4 x float>)
+
+define <8 x i64> @lrint_v8f32(<8 x float> %x) {
+; CHECK-LABEL: lrint_v8f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
+; CHECK-NEXT:    mov s4, v0.s[1]
+; CHECK-NEXT:    mov s7, v1.s[1]
+; CHECK-NEXT:    frintx s0, s0
+; CHECK-NEXT:    frintx s1, s1
+; CHECK-NEXT:    mov s5, v2.s[1]
+; CHECK-NEXT:    mov s6, v3.s[1]
+; CHECK-NEXT:    frintx s2, s2
+; CHECK-NEXT:    frintx s3, s3
+; CHECK-NEXT:    frintx s4, s4
+; CHECK-NEXT:    frintx s7, s7
+; CHECK-NEXT:    fcvtzs x9, s0
+; CHECK-NEXT:    fcvtzs x12, s1
+; CHECK-NEXT:    frintx s5, s5
+; CHECK-NEXT:    frintx s6, s6
+; CHECK-NEXT:    fcvtzs x8, s2
+; CHECK-NEXT:    fcvtzs x10, s3
+; CHECK-NEXT:    fcvtzs x11, s4
+; CHECK-NEXT:    fcvtzs x15, s7
+; CHECK-NEXT:    fmov d0, x9
+; CHECK-NEXT:    fmov d2, x12
+; CHECK-NEXT:    fcvtzs x13, s5
+; CHECK-NEXT:    fcvtzs x14, s6
+; CHECK-NEXT:    fmov d1, x8
+; CHECK-NEXT:    fmov d3, x10
+; CHECK-NEXT:    mov v0.d[1], x11
+; CHECK-NEXT:    mov v2.d[1], x15
+; CHECK-NEXT:    mov v1.d[1], x13
+; CHECK-NEXT:    mov v3.d[1], x14
+; CHECK-NEXT:    ret
+  %a = call <8 x i64> @llvm.lrint.v8i64.v8f32(<8 x float> %x)
+  ret <8 x i64> %a
+}
+declare <8 x i64> @llvm.lrint.v8i64.v8f32(<8 x float>)
+
+define <16 x i64> @lrint_v16i64_v16f32(<16 x float> %x) {
+; CHECK-LABEL: lrint_v16i64_v16f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ext v4.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    ext v5.16b, v1.16b, v1.16b, #8
+; CHECK-NEXT:    ext v6.16b, v2.16b, v2.16b, #8
+; CHECK-NEXT:    frintx s7, s0
+; CHECK-NEXT:    ext v16.16b, v3.16b, v3.16b, #8
+; CHECK-NEXT:    mov s0, v0.s[1]
+; CHECK-NEXT:    frintx s17, s4
+; CHECK-NEXT:    mov s4, v4.s[1]
+; CHECK-NEXT:    mov s18, v5.s[1]
+; CHECK-NEXT:    frintx s5, s5
+; CHECK-NEXT:    frintx s19, s6
+; CHECK-NEXT:    fcvtzs x8, s7
+; CHECK-NEXT:    frintx s7, s16
+; CHECK-NEXT:    mov s6, v6.s[1]
+; CHECK-NEXT:    mov s16, v16.s[1]
+; CHECK-NEXT:    frintx s0, s0
+; CHECK-NEXT:    frintx s4, s4
+; CHECK-NEXT:    fcvtzs x9, s17
+; CHECK-NEXT:    frintx s17, s1
+; CHECK-NEXT:    mov s1, v1.s[1]
+; CHECK-NEXT:    frintx s18, s18
+; CHECK-NEXT:    fcvtzs x10, s5
+; CHECK-NEXT:    mov s5, v2.s[1]
+; CHECK-NEXT:    fcvtzs x11, s19
+; CHECK-NEXT:    mov s19, v3.s[1]
+; CHECK-NEXT:    frintx s2, s2
+; CHECK-NEXT:    fcvtzs x12, s7
+; CHECK-NEXT:    frintx s6, s6
+; CHECK-NEXT:    fcvtzs x13, s4
+; CHECK-NEXT:    frintx s4, s3
+; CHECK-NEXT:    frintx s16, s16
+; CHECK-NEXT:    fcvtzs x14, s18
+; CHECK-NEXT:    frintx s18, s1
+; CHECK-NEXT:    fcvtzs x15, s17
+; CHECK-NEXT:    frintx s20, s5
+; CHECK-NEXT:    frintx s17, s19
+; CHECK-NEXT:    fmov d1, x9
+; CHECK-NEXT:    fcvtzs x9, s2
+; CHECK-NEXT:    fmov d5, x11
+; CHECK-NEXT:    fmov d3, x10
+; CHECK-NEXT:    fcvtzs x11, s4
+; CHECK-NEXT:    fcvtzs x10, s0
+; CHECK-NEXT:    fmov d7, x12
+; CHECK-NEXT:    fcvtzs x12, s18
+; CHECK-NEXT:    fcvtzs x17, s6
+; CHECK-NEXT:    fcvtzs x18, s16
+; CHECK-NEXT:    fcvtzs x16, s20
+; CHECK-NEXT:    fcvtzs x0, s17
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    fmov d2, x15
+; CHECK-NEXT:    fmov d4, x9
+; CHECK-NEXT:    mov v1.d[1], x13
+; CHECK-NEXT:    fmov d6, x11
+; CHECK-NEXT:    mov v3.d[1], x14
+; CHECK-NEXT:    mov v0.d[1], x10
+; CHECK-NEXT:    mov v5.d[1], x17
+; CHECK-NEXT:    mov v7.d[1], x18
+; CHECK-NEXT:    mov v2.d[1], x12
+; CHECK-NEXT:    mov v4.d[1], x16
+; CHECK-NEXT:    mov v6.d[1], x0
+; CHECK-NEXT:    ret
+  %a = call <16 x i64> @llvm.lrint.v16i64.v16f32(<16 x float> %x)
+  ret <16 x i64> %a
+}
+declare <16 x i64> @llvm.lrint.v16i64.v16f32(<16 x float>)
+
+define <1 x i64> @lrint_v1f64(<1 x double> %x) {
+; CHECK-LABEL: lrint_v1f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    frintx d0, d0
+; CHECK-NEXT:    fcvtzs x8, d0
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    ret
+  %a = call <1 x i64> @llvm.lrint.v1i64.v1f64(<1 x double> %x)
+  ret <1 x i64> %a
+}
+declare <1 x i64> @llvm.lrint.v1i64.v1f64(<1 x double>)
+
+define <2 x i64> @lrint_v2f64(<2 x double> %x) {
+; CHECK-LABEL: lrint_v2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov d1, v0.d[1]
+; CHECK-NEXT:    frintx d0, d0
+; CHECK-NEXT:    frintx d1, d1
+; CHECK-NEXT:    fcvtzs x8, d0
+; CHECK-NEXT:    fcvtzs x9, d1
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    mov v0.d[1], x9
+; CHECK-NEXT:    ret
+  %a = call <2 x i64> @llvm.lrint.v2i64.v2f64(<2 x double> %x)
+  ret <2 x i64> %a
+}
+declare <2 x i64> @llvm.lrint.v2i64.v2f64(<2 x double>)
+
+define <4 x i64> @lrint_v4f64(<4 x double> %x) {
+; CHECK-LABEL: lrint_v4f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov d2, v0.d[1]
+; CHECK-NEXT:    mov d3, v1.d[1]
+; CHECK-NEXT:    frintx d0, d0
+; CHECK-NEXT:    frintx d1, d1
+; CHECK-NEXT:    frintx d2, d2
+; CHECK-NEXT:    frintx d3, d3
+; CHECK-NEXT:    fcvtzs x8, d0
+; CHECK-NEXT:    fcvtzs x9, d1
+; CHECK-NEXT:    fcvtzs x10, d2
+; CHECK-NEXT:    fcvtzs x11, d3
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    fmov d1, x9
+; CHECK-NEXT:    mov v0.d[1], x10
+; CHECK-NEXT:    mov v1.d[1], x11
+; CHECK-NEXT:    ret
+  %a = call <4 x i64> @llvm.lrint.v4i64.v4f64(<4 x double> %x)
+  ret <4 x i64> %a
+}
+declare <4 x i64> @llvm.lrint.v4i64.v4f64(<4 x double>)
+
+define <8 x i64> @lrint_v8f64(<8 x double> %x) {
+; CHECK-LABEL: lrint_v8f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov d4, v0.d[1]
+; CHECK-NEXT:    mov d5, v1.d[1]
+; CHECK-NEXT:    mov d6, v2.d[1]
+; CHECK-NEXT:    mov d7, v3.d[1]
+; CHECK-NEXT:    frintx d0, d0
+; CHECK-NEXT:    frintx d1, d1
+; CHECK-NEXT:    frintx d2, d2
+; CHECK-NEXT:    frintx d3, d3
+; CHECK-NEXT:    frintx d4, d4
+; CHECK-NEXT:    frintx d5, d5
+; CHECK-NEXT:    frintx d6, d6
+; CHECK-NEXT:    frintx d7, d7
+; CHECK-NEXT:    fcvtzs x8, d0
+; CHECK-NEXT:    fcvtzs x9, d1
+; CHECK-NEXT:    fcvtzs x10, d2
+; CHECK-NEXT:    fcvtzs x11, d3
+; CHECK-NEXT:    fcvtzs x12, d4
+; CHECK-NEXT:    fcvtzs x13, d5
+; CHECK-NEXT:    fcvtzs x14, d6
+; CHECK-NEXT:    fcvtzs x15, d7
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    fmov d1, x9
+; CHECK-NEXT:    fmov d2, x10
+; CHECK-NEXT:    fmov d3, x11
+; CHECK-NEXT:    mov v0.d[1], x12
+; CHECK-NEXT:    mov v1.d[1], x13
+; CHECK-NEXT:    mov v2.d[1], x14
+; CHECK-NEXT:    mov v3.d[1], x15
+; CHECK-NEXT:    ret
+  %a = call <8 x i64> @llvm.lrint.v8i64.v8f64(<8 x double> %x)
+  ret <8 x i64> %a
+}
+declare <8 x i64> @llvm.lrint.v8i64.v8f64(<8 x double>)
diff --git a/llvm/test/CodeGen/AArch64/xtn.ll b/llvm/test/CodeGen/AArch64/xtn.ll
new file mode 100644
index 0000000000..0dd4e3644b
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/xtn.ll
@@ -0,0 +1,473 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
+; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc -mtriple=aarch64 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+
+define i8 @xtn_i16_to_i8(i16 %a) {
+; CHECK-LABEL: xtn_i16_to_i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ret
+entry:
+  %arg1 = trunc i16 %a to i8
+  ret i8 %arg1
+}
+
+define i8 @xtn_i32_to_i8(i32 %a) {
+; CHECK-LABEL: xtn_i32_to_i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ret
+entry:
+  %arg1 = trunc i32 %a to i8
+  ret i8 %arg1
+}
+
+define i8 @xtn_i64_to_i8(i64 %a) {
+; CHECK-LABEL: xtn_i64_to_i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-NEXT:    ret
+entry:
+  %arg1 = trunc i64 %a to i8
+  ret i8 %arg1
+}
+
+define i8 @xtn_i128_to_i8(i128 %a) {
+; CHECK-LABEL: xtn_i128_to_i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-NEXT:    ret
+entry:
+  %arg1 = trunc i128 %a to i8
+  ret i8 %arg1
+}
+
+define i16 @xtn_i32_to_i16(i32 %a) {
+; CHECK-LABEL: xtn_i32_to_i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ret
+entry:
+  %arg1 = trunc i32 %a to i16
+  ret i16 %arg1
+}
+
+define i16 @xtn_i64_to_i16(i64 %a) {
+; CHECK-LABEL: xtn_i64_to_i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-NEXT:    ret
+entry:
+  %arg1 = trunc i64 %a to i16
+  ret i16 %arg1
+}
+
+define i16 @xtn_i128_to_i16(i128 %a) {
+; CHECK-LABEL: xtn_i128_to_i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-NEXT:    ret
+entry:
+  %arg1 = trunc i128 %a to i16
+  ret i16 %arg1
+}
+
+define i32 @xtn_i64_to_i32(i64 %a) {
+; CHECK-LABEL: xtn_i64_to_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-NEXT:    ret
+entry:
+  %arg1 = trunc i64 %a to i32
+  ret i32 %arg1
+}
+
+define i32 @xtn_i128_to_i32(i128 %a) {
+; CHECK-LABEL: xtn_i128_to_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-NEXT:    ret
+entry:
+  %arg1 = trunc i128 %a to i32
+  ret i32 %arg1
+}
+
+define i64 @xtn_i128_to_i64(i128 %a) {
+; CHECK-LABEL: xtn_i128_to_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ret
+entry:
+  %arg1 = trunc i128 %a to i64
+  ret i64 %arg1
+}
+
+define <2 x i8> @xtn_v2i16_v2i8(<2 x i16> %a) {
+; CHECK-LABEL: xtn_v2i16_v2i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ret
+entry:
+  %arg1 = trunc <2 x i16> %a to <2 x i8>
+  ret <2 x i8> %arg1
+}
+
+define <2 x i8> @xtn_v2i32_v2i8(<2 x i32> %a) {
+; CHECK-LABEL: xtn_v2i32_v2i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ret
+entry:
+  %arg1 = trunc <2 x i32> %a to <2 x i8>
+  ret <2 x i8> %arg1
+}
+
+define <2 x i8> @xtn_v2i64_v2i8(<2 x i64> %a) {
+; CHECK-LABEL: xtn_v2i64_v2i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    ret
+entry:
+  %arg1 = trunc <2 x i64> %a to <2 x i8>
+  ret <2 x i8> %arg1
+}
+
+define <2 x i8> @xtn_v2i128_v2i8(<2 x i128> %a) {
+; CHECK-SD-LABEL: xtn_v2i128_v2i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov s0, w0
+; CHECK-SD-NEXT:    mov v0.s[1], w2
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: xtn_v2i128_v2i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov d0, x0
+; CHECK-GI-NEXT:    mov v0.d[1], x2
+; CHECK-GI-NEXT:    xtn v0.2s, v0.2d
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = trunc <2 x i128> %a to <2 x i8>
+  ret <2 x i8> %arg1
+}
+
+define <2 x i16> @xtn_v2i32_v2i16(<2 x i32> %a) {
+; CHECK-LABEL: xtn_v2i32_v2i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ret
+entry:
+  %arg1 = trunc <2 x i32> %a to <2 x i16>
+  ret <2 x i16> %arg1
+}
+
+define <2 x i16> @xtn_v2i64_v2i16(<2 x i64> %a) {
+; CHECK-LABEL: xtn_v2i64_v2i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    ret
+entry:
+  %arg1 = trunc <2 x i64> %a to <2 x i16>
+  ret <2 x i16> %arg1
+}
+
+define <2 x i16> @xtn_v2i128_v2i16(<2 x i128> %a) {
+; CHECK-SD-LABEL: xtn_v2i128_v2i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov s0, w0
+; CHECK-SD-NEXT:    mov v0.s[1], w2
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: xtn_v2i128_v2i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov d0, x0
+; CHECK-GI-NEXT:    mov v0.d[1], x2
+; CHECK-GI-NEXT:    xtn v0.2s, v0.2d
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = trunc <2 x i128> %a to <2 x i16>
+  ret <2 x i16> %arg1
+}
+
+define <2 x i32> @xtn_v2i64_v2i32(<2 x i64> %a) {
+; CHECK-LABEL: xtn_v2i64_v2i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    ret
+entry:
+  %arg1 = trunc <2 x i64> %a to <2 x i32>
+  ret <2 x i32> %arg1
+}
+
+define <2 x i32> @xtn_v2i128_v2i32(<2 x i128> %a) {
+; CHECK-SD-LABEL: xtn_v2i128_v2i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov s0, w0
+; CHECK-SD-NEXT:    mov v0.s[1], w2
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: xtn_v2i128_v2i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov d0, x0
+; CHECK-GI-NEXT:    mov v0.d[1], x2
+; CHECK-GI-NEXT:    xtn v0.2s, v0.2d
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = trunc <2 x i128> %a to <2 x i32>
+  ret <2 x i32> %arg1
+}
+
+define <2 x i64> @xtn_v2i128_v2i64(<2 x i128> %a) {
+; CHECK-LABEL: xtn_v2i128_v2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    mov v0.d[1], x2
+; CHECK-NEXT:    ret
+entry:
+  %arg1 = trunc <2 x i128> %a to <2 x i64>
+  ret <2 x i64> %arg1
+}
+
+define <3 x i8> @xtn_v3i16_v3i8(<3 x i16> %a) {
+; CHECK-SD-LABEL: xtn_v3i16_v3i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    umov w0, v0.h[0]
+; CHECK-SD-NEXT:    umov w1, v0.h[1]
+; CHECK-SD-NEXT:    umov w2, v0.h[2]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: xtn_v3i16_v3i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-NEXT:    mov h2, v0.h[2]
+; CHECK-GI-NEXT:    fmov w0, s0
+; CHECK-GI-NEXT:    fmov w1, s1
+; CHECK-GI-NEXT:    fmov w2, s2
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = trunc <3 x i16> %a to <3 x i8>
+  ret <3 x i8> %arg1
+}
+
+define <3 x i8> @xtn_v3i32_v3i8(<3 x i32> %a) {
+; CHECK-SD-LABEL: xtn_v3i32_v3i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-SD-NEXT:    umov w0, v0.h[0]
+; CHECK-SD-NEXT:    umov w1, v0.h[1]
+; CHECK-SD-NEXT:    umov w2, v0.h[2]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: xtn_v3i32_v3i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-NEXT:    mov s2, v0.s[2]
+; CHECK-GI-NEXT:    fmov w0, s0
+; CHECK-GI-NEXT:    fmov w1, s1
+; CHECK-GI-NEXT:    fmov w2, s2
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = trunc <3 x i32> %a to <3 x i8>
+  ret <3 x i8> %arg1
+}
+
+define <3 x i8> @xtn_v3i64_v3i8(<3 x i64> %a) {
+; CHECK-SD-LABEL: xtn_v3i64_v3i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-SD-NEXT:    xtn v1.2s, v2.2d
+; CHECK-SD-NEXT:    xtn v0.2s, v0.2d
+; CHECK-SD-NEXT:    fmov w2, s1
+; CHECK-SD-NEXT:    mov w1, v0.s[1]
+; CHECK-SD-NEXT:    fmov w0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: xtn_v3i64_v3i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov x0, d0
+; CHECK-GI-NEXT:    fmov x1, d1
+; CHECK-GI-NEXT:    fmov x2, d2
+; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 killed $x1
+; CHECK-GI-NEXT:    // kill: def $w2 killed $w2 killed $x2
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = trunc <3 x i64> %a to <3 x i8>
+  ret <3 x i8> %arg1
+}
+
+define <3 x i16> @xtn_v3i32_v3i16(<3 x i32> %a) {
+; CHECK-SD-LABEL: xtn_v3i32_v3i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: xtn_v3i32_v3i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-NEXT:    mov s2, v0.s[2]
+; CHECK-GI-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-GI-NEXT:    mov v0.h[2], v2.h[0]
+; CHECK-GI-NEXT:    mov v0.h[3], v0.h[0]
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = trunc <3 x i32> %a to <3 x i16>
+  ret <3 x i16> %arg1
+}
+
+define <3 x i16> @xtn_v3i64_v3i16(<3 x i64> %a) {
+; CHECK-SD-LABEL: xtn_v3i64_v3i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-SD-NEXT:    uzp1 v0.4s, v0.4s, v2.4s
+; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: xtn_v3i64_v3i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov x8, d0
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    fmov s1, w9
+; CHECK-GI-NEXT:    fmov x8, d2
+; CHECK-GI-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    mov v0.h[2], v1.h[0]
+; CHECK-GI-NEXT:    mov v0.h[3], v0.h[0]
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = trunc <3 x i64> %a to <3 x i16>
+  ret <3 x i16> %arg1
+}
+
+define <3 x i32> @xtn_v3i64_v3i32(<3 x i64> %a) {
+; CHECK-SD-LABEL: xtn_v3i64_v3i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-SD-NEXT:    uzp1 v0.4s, v0.4s, v2.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: xtn_v3i64_v3i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov x8, d0
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    fmov x8, d2
+; CHECK-GI-NEXT:    mov v0.s[1], w9
+; CHECK-GI-NEXT:    mov v0.s[2], w8
+; CHECK-GI-NEXT:    mov v0.s[3], w8
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = trunc <3 x i64> %a to <3 x i32>
+  ret <3 x i32> %arg1
+}
+
+define <4 x i8> @xtn_v4i16_v4i8(<4 x i16> %a) {
+; CHECK-LABEL: xtn_v4i16_v4i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ret
+entry:
+  %arg1 = trunc <4 x i16> %a to <4 x i8>
+  ret <4 x i8> %arg1
+}
+
+define <4 x i8> @xtn_v4i32_v4i8(<4 x i32> %a) {
+; CHECK-LABEL: xtn_v4i32_v4i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+entry:
+  %arg1 = trunc <4 x i32> %a to <4 x i8>
+  ret <4 x i8> %arg1
+}
+
+define <4 x i8> @xtn_v4i64_v4i8(<4 x i64> %a) {
+; CHECK-LABEL: xtn_v4i64_v4i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+entry:
+  %arg1 = trunc <4 x i64> %a to <4 x i8>
+  ret <4 x i8> %arg1
+}
+
+define <4 x i16> @xtn_v4i32_v4i16(<4 x i32> %a) {
+; CHECK-LABEL: xtn_v4i32_v4i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+entry:
+  %arg1 = trunc <4 x i32> %a to <4 x i16>
+  ret <4 x i16> %arg1
+}
+
+define <4 x i16> @xtn_v4i64_v4i16(<4 x i64> %a) {
+; CHECK-LABEL: xtn_v4i64_v4i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+entry:
+  %arg1 = trunc <4 x i64> %a to <4 x i16>
+  ret <4 x i16> %arg1
+}
+
+define <4 x i32> @xtn_v4i64_v4i32(<4 x i64> %a) {
+; CHECK-LABEL: xtn_v4i64_v4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    ret
+entry:
+  %arg1 = trunc <4 x i64> %a to <4 x i32>
+  ret <4 x i32> %arg1
+}
+
+define <8 x i8> @xtn_v8i16_v8i8(<8 x i16> %a) {
+; CHECK-LABEL: xtn_v8i16_v8i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NEXT:    ret
+entry:
+  %arg1 = trunc <8 x i16> %a to <8 x i8>
+  ret <8 x i8> %arg1
+}
+
+define <8 x i8> @xtn_v8i32_v8i8(<8 x i32> %a) {
+; CHECK-LABEL: xtn_v8i32_v8i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NEXT:    ret
+entry:
+  %arg1 = trunc <8 x i32> %a to <8 x i8>
+  ret <8 x i8> %arg1
+}
+
+define <8 x i16> @xtn_v8i32_v8i16(<8 x i32> %a) {
+; CHECK-LABEL: xtn_v8i32_v8i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    ret
+entry:
+  %arg1 = trunc <8 x i32> %a to <8 x i16>
+  ret <8 x i16> %arg1
+}
+
+define <16 x i8> @xtn_v16i16_v16i8(<16 x i16> %a) {
+; CHECK-LABEL: xtn_v16i16_v16i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+entry:
+  %arg1 = trunc <16 x i16> %a to <16 x i8>
+  ret <16 x i8> %arg1
+}
diff --git a/llvm/test/CodeGen/AArch64/zext.ll b/llvm/test/CodeGen/AArch64/zext.ll
index 8b30ee257a..8573a8a2d2 100644
--- a/llvm/test/CodeGen/AArch64/zext.ll
+++ b/llvm/test/CodeGen/AArch64/zext.ll
@@ -2,8 +2,6 @@
 ; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-SD
 ; RUN: llc -mtriple=aarch64 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
 
-; CHECK-GI:       warning: Instruction selection used fallback path for zext_v16i10_v16i16
-
 define i16 @zext_i8_to_i16(i8 %a) {
 ; CHECK-LABEL: zext_i8_to_i16:
 ; CHECK:       // %bb.0: // %entry
@@ -242,19 +240,18 @@ define <3 x i16> @zext_v3i8_v3i16(<3 x i8> %a) {
 ;
 ; CHECK-GI-LABEL: zext_v3i8_v3i16:
 ; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov s0, w0
 ; CHECK-GI-NEXT:    mov w8, #255 // =0xff
-; CHECK-GI-NEXT:    fmov s1, w0
-; CHECK-GI-NEXT:    fmov s2, w1
-; CHECK-GI-NEXT:    fmov s0, w8
-; CHECK-GI-NEXT:    mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT:    fmov s2, w2
-; CHECK-GI-NEXT:    mov v3.16b, v0.16b
-; CHECK-GI-NEXT:    mov v3.h[1], v0.h[0]
-; CHECK-GI-NEXT:    mov v1.h[2], v2.h[0]
-; CHECK-GI-NEXT:    mov v3.h[2], v0.h[0]
-; CHECK-GI-NEXT:    mov v1.h[3], v0.h[0]
-; CHECK-GI-NEXT:    mov v3.h[3], v0.h[0]
-; CHECK-GI-NEXT:    and v0.8b, v1.8b, v3.8b
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    mov v0.s[1], w1
+; CHECK-GI-NEXT:    mov v2.16b, v1.16b
+; CHECK-GI-NEXT:    mov v0.s[2], w2
+; CHECK-GI-NEXT:    mov v2.h[1], v1.h[0]
+; CHECK-GI-NEXT:    mov v0.s[3], w8
+; CHECK-GI-NEXT:    mov v2.h[2], v1.h[0]
+; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    mov v2.h[3], v0.h[0]
+; CHECK-GI-NEXT:    and v0.8b, v0.8b, v2.8b
 ; CHECK-GI-NEXT:    ret
 entry:
   %c = zext <3 x i8> %a to <3 x i16>
@@ -425,19 +422,18 @@ define <3 x i16> @zext_v3i10_v3i16(<3 x i10> %a) {
 ;
 ; CHECK-GI-LABEL: zext_v3i10_v3i16:
 ; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov s0, w0
 ; CHECK-GI-NEXT:    mov w8, #1023 // =0x3ff
-; CHECK-GI-NEXT:    fmov s1, w0
-; CHECK-GI-NEXT:    fmov s2, w1
-; CHECK-GI-NEXT:    fmov s0, w8
-; CHECK-GI-NEXT:    mov v1.h[1], v2.h[0]
-; CHECK-GI-NEXT:    fmov s2, w2
-; CHECK-GI-NEXT:    mov v3.16b, v0.16b
-; CHECK-GI-NEXT:    mov v3.h[1], v0.h[0]
-; CHECK-GI-NEXT:    mov v1.h[2], v2.h[0]
-; CHECK-GI-NEXT:    mov v3.h[2], v0.h[0]
-; CHECK-GI-NEXT:    mov v1.h[3], v0.h[0]
-; CHECK-GI-NEXT:    mov v3.h[3], v0.h[0]
-; CHECK-GI-NEXT:    and v0.8b, v1.8b, v3.8b
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    mov v0.s[1], w1
+; CHECK-GI-NEXT:    mov v2.16b, v1.16b
+; CHECK-GI-NEXT:    mov v0.s[2], w2
+; CHECK-GI-NEXT:    mov v2.h[1], v1.h[0]
+; CHECK-GI-NEXT:    mov v0.s[3], w8
+; CHECK-GI-NEXT:    mov v2.h[2], v1.h[0]
+; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    mov v2.h[3], v0.h[0]
+; CHECK-GI-NEXT:    and v0.8b, v0.8b, v2.8b
 ; CHECK-GI-NEXT:    ret
 entry:
   %c = zext <3 x i10> %a to <3 x i16>
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
index c793f9ee68..7eb1cb926c 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
@@ -7139,70 +7139,42 @@ define amdgpu_kernel void @udiv_i64_oddk_denom(ptr addrspace(1) %out, i64 %x) {
 ;
 ; GFX6-LABEL: udiv_i64_oddk_denom:
 ; GFX6:       ; %bb.0:
-; GFX6-NEXT:    v_mov_b32_e32 v0, 0x4f176a73
-; GFX6-NEXT:    v_mov_b32_e32 v1, 0x4f800000
-; GFX6-NEXT:    v_madmk_f32 v0, v1, 0x438f8000, v0
-; GFX6-NEXT:    v_rcp_f32_e32 v0, v0
-; GFX6-NEXT:    s_movk_i32 s4, 0xfee0
-; GFX6-NEXT:    s_mov_b32 s5, 0x68958c89
+; GFX6-NEXT:    s_add_u32 s4, 3, 0
+; GFX6-NEXT:    v_mov_b32_e32 v0, 0xe3e0f6
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s4, v0
+; GFX6-NEXT:    s_addc_u32 s5, 0, 0
+; GFX6-NEXT:    s_or_b32 s4, vcc_lo, vcc_hi
+; GFX6-NEXT:    s_cmp_lg_u32 s4, 0
+; GFX6-NEXT:    s_mov_b32 s4, 0x68958c89
+; GFX6-NEXT:    s_movk_i32 s6, 0xfee0
+; GFX6-NEXT:    v_mul_lo_u32 v1, v0, s6
+; GFX6-NEXT:    v_mul_hi_u32 v2, v0, s4
+; GFX6-NEXT:    s_addc_u32 s5, s5, 0
+; GFX6-NEXT:    s_mul_i32 s6, s5, 0x68958c89
 ; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
-; GFX6-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
-; GFX6-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
-; GFX6-NEXT:    v_trunc_f32_e32 v1, v1
-; GFX6-NEXT:    v_madmk_f32 v0, v1, 0xcf800000, v0
-; GFX6-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX6-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GFX6-NEXT:    s_movk_i32 s8, 0x11f
-; GFX6-NEXT:    s_mov_b32 s9, 0x976a7377
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
 ; GFX6-NEXT:    v_mul_lo_u32 v2, v0, s4
-; GFX6-NEXT:    v_mul_hi_u32 v3, v0, s5
-; GFX6-NEXT:    v_mul_lo_u32 v4, v1, s5
-; GFX6-NEXT:    v_mul_lo_u32 v5, v0, s5
-; GFX6-NEXT:    s_mov_b32 s7, 0xf000
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
-; GFX6-NEXT:    v_mul_lo_u32 v3, v0, v2
-; GFX6-NEXT:    v_mul_hi_u32 v4, v0, v5
-; GFX6-NEXT:    v_mul_hi_u32 v6, v0, v2
-; GFX6-NEXT:    v_mul_hi_u32 v7, v1, v2
-; GFX6-NEXT:    v_mul_lo_u32 v2, v1, v2
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, s6, v1
+; GFX6-NEXT:    v_mul_lo_u32 v3, v0, v1
+; GFX6-NEXT:    v_mul_hi_u32 v4, v0, v2
+; GFX6-NEXT:    v_mul_hi_u32 v5, v0, v1
+; GFX6-NEXT:    v_mul_hi_u32 v6, s5, v1
+; GFX6-NEXT:    v_mul_lo_u32 v1, s5, v1
 ; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
-; GFX6-NEXT:    v_addc_u32_e32 v4, vcc, 0, v6, vcc
-; GFX6-NEXT:    v_mul_lo_u32 v6, v1, v5
-; GFX6-NEXT:    v_mul_hi_u32 v5, v1, v5
-; GFX6-NEXT:    s_mov_b32 s6, -1
-; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v3, v6
-; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, v4, v5, vcc
-; GFX6-NEXT:    v_addc_u32_e32 v4, vcc, 0, v7, vcc
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc
-; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
-; GFX6-NEXT:    v_mul_lo_u32 v2, v0, s4
-; GFX6-NEXT:    v_mul_hi_u32 v3, v0, s5
-; GFX6-NEXT:    v_mul_lo_u32 v4, v1, s5
+; GFX6-NEXT:    v_addc_u32_e32 v4, vcc, 0, v5, vcc
+; GFX6-NEXT:    v_mul_lo_u32 v5, s5, v2
+; GFX6-NEXT:    v_mul_hi_u32 v2, s5, v2
+; GFX6-NEXT:    s_movk_i32 s8, 0x11f
+; GFX6-NEXT:    s_mov_b32 s9, 0x976a7377
+; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GFX6-NEXT:    v_addc_u32_e32 v2, vcc, v4, v2, vcc
+; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, 0, v6, vcc
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
+; GFX6-NEXT:    v_addc_u32_e32 v2, vcc, 0, v3, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v3, s5
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v3, v2, vcc
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    s_mov_b32 s4, s0
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
-; GFX6-NEXT:    v_mul_lo_u32 v3, v0, s5
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v4, v2
-; GFX6-NEXT:    v_mul_lo_u32 v4, v0, v2
-; GFX6-NEXT:    v_mul_hi_u32 v5, v0, v3
-; GFX6-NEXT:    v_mul_hi_u32 v6, v0, v2
-; GFX6-NEXT:    v_mul_hi_u32 v7, v1, v2
-; GFX6-NEXT:    v_mul_lo_u32 v2, v1, v2
-; GFX6-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; GFX6-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc
-; GFX6-NEXT:    v_mul_lo_u32 v6, v1, v3
-; GFX6-NEXT:    v_mul_hi_u32 v3, v1, v3
-; GFX6-NEXT:    s_mov_b32 s5, s1
-; GFX6-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
-; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, v5, v3, vcc
-; GFX6-NEXT:    v_addc_u32_e32 v4, vcc, 0, v7, vcc
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc
-; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
 ; GFX6-NEXT:    v_mul_lo_u32 v2, s2, v1
 ; GFX6-NEXT:    v_mul_hi_u32 v3, s2, v0
 ; GFX6-NEXT:    v_mul_hi_u32 v4, s2, v1
@@ -7212,6 +7184,8 @@ define amdgpu_kernel void @udiv_i64_oddk_denom(ptr addrspace(1) %out, i64 %x) {
 ; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc
 ; GFX6-NEXT:    v_mul_lo_u32 v4, s3, v0
 ; GFX6-NEXT:    v_mul_hi_u32 v0, s3, v0
+; GFX6-NEXT:    s_mov_b32 s4, s0
+; GFX6-NEXT:    s_mov_b32 s5, s1
 ; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
 ; GFX6-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
 ; GFX6-NEXT:    v_addc_u32_e32 v2, vcc, 0, v5, vcc
@@ -7221,6 +7195,7 @@ define amdgpu_kernel void @udiv_i64_oddk_denom(ptr addrspace(1) %out, i64 %x) {
 ; GFX6-NEXT:    v_mul_hi_u32 v3, v0, s9
 ; GFX6-NEXT:    v_mul_lo_u32 v4, v1, s9
 ; GFX6-NEXT:    v_mov_b32_e32 v5, 0x11f
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
 ; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
 ; GFX6-NEXT:    v_mul_lo_u32 v3, v0, s9
 ; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v4, v2
@@ -7253,6 +7228,7 @@ define amdgpu_kernel void @udiv_i64_oddk_denom(ptr addrspace(1) %out, i64 %x) {
 ; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, s8, v2
 ; GFX6-NEXT:    v_cndmask_b32_e32 v2, v6, v3, vcc
 ; GFX6-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; GFX6-NEXT:    s_mov_b32 s6, -1
 ; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
 ; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
 ; GFX6-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
@@ -7260,44 +7236,14 @@ define amdgpu_kernel void @udiv_i64_oddk_denom(ptr addrspace(1) %out, i64 %x) {
 ;
 ; GFX9-LABEL: udiv_i64_oddk_denom:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0x4f176a73
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0x4f800000
-; GFX9-NEXT:    v_madmk_f32 v0, v1, 0x438f8000, v0
-; GFX9-NEXT:    v_rcp_f32_e32 v0, v0
 ; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
-; GFX9-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
-; GFX9-NEXT:    v_trunc_f32_e32 v1, v1
-; GFX9-NEXT:    v_madmk_f32 v0, v1, 0xcf800000, v0
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
-; GFX9-NEXT:    s_mul_i32 s1, s0, 0xfffffee0
-; GFX9-NEXT:    s_mul_hi_u32 s2, s0, 0x68958c89
-; GFX9-NEXT:    s_add_i32 s1, s2, s1
-; GFX9-NEXT:    v_readfirstlane_b32 s2, v1
-; GFX9-NEXT:    s_mul_i32 s3, s2, 0x68958c89
-; GFX9-NEXT:    s_add_i32 s1, s1, s3
-; GFX9-NEXT:    s_mul_i32 s9, s0, 0x68958c89
-; GFX9-NEXT:    s_mul_hi_u32 s3, s0, s1
-; GFX9-NEXT:    s_mul_i32 s8, s0, s1
-; GFX9-NEXT:    s_mul_hi_u32 s0, s0, s9
-; GFX9-NEXT:    s_add_u32 s0, s0, s8
-; GFX9-NEXT:    s_addc_u32 s3, 0, s3
-; GFX9-NEXT:    s_mul_hi_u32 s10, s2, s9
-; GFX9-NEXT:    s_mul_i32 s9, s2, s9
-; GFX9-NEXT:    s_add_u32 s0, s0, s9
-; GFX9-NEXT:    s_mul_hi_u32 s8, s2, s1
-; GFX9-NEXT:    s_addc_u32 s0, s3, s10
-; GFX9-NEXT:    s_addc_u32 s3, s8, 0
-; GFX9-NEXT:    s_mul_i32 s1, s2, s1
-; GFX9-NEXT:    s_add_u32 s0, s0, s1
-; GFX9-NEXT:    s_addc_u32 s1, 0, s3
+; GFX9-NEXT:    s_add_u32 s0, 3, 0
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0xe3e0f6
+; GFX9-NEXT:    s_addc_u32 s1, 0, 0
 ; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v0
 ; GFX9-NEXT:    s_cmp_lg_u64 vcc, 0
-; GFX9-NEXT:    s_addc_u32 s0, s2, s1
 ; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX9-NEXT:    s_addc_u32 s0, s1, 0
 ; GFX9-NEXT:    s_mul_i32 s3, s2, 0xfffffee0
 ; GFX9-NEXT:    s_mul_hi_u32 s8, s2, 0x68958c89
 ; GFX9-NEXT:    s_mul_i32 s1, s0, 0x68958c89
@@ -7385,6 +7331,7 @@ define amdgpu_kernel void @udiv_i64_oddk_denom(ptr addrspace(1) %out, i64 %x) {
 ; GFX9-NEXT:    v_mov_b32_e32 v4, s2
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s3
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
 ; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[4:5]
@@ -7529,74 +7476,53 @@ define amdgpu_kernel void @udiv_v2i64_mixed_pow2k_denom(ptr addrspace(1) %out, <
 ;
 ; GFX6-LABEL: udiv_v2i64_mixed_pow2k_denom:
 ; GFX6:       ; %bb.0:
-; GFX6-NEXT:    v_mov_b32_e32 v0, 0x4f800000
-; GFX6-NEXT:    v_madak_f32 v0, 0, v0, 0x457ff000
-; GFX6-NEXT:    v_rcp_f32_e32 v0, v0
-; GFX6-NEXT:    s_movk_i32 s6, 0xf001
-; GFX6-NEXT:    s_movk_i32 s8, 0xfff
-; GFX6-NEXT:    s_mov_b32 s7, 0xf000
-; GFX6-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
-; GFX6-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
-; GFX6-NEXT:    v_trunc_f32_e32 v1, v1
-; GFX6-NEXT:    v_madmk_f32 v0, v1, 0xcf800000, v0
-; GFX6-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX6-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
-; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0xd
-; GFX6-NEXT:    v_mul_hi_u32 v2, v0, s6
-; GFX6-NEXT:    v_mul_lo_u32 v4, v1, s6
-; GFX6-NEXT:    v_mul_lo_u32 v3, v0, s6
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0xd
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s2, 0x2ff2fc01
+; GFX6-NEXT:    v_bfrev_b32_e32 v0, 7
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    s_lshr_b64 s[0:1], s[0:1], 12
-; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v0
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
-; GFX6-NEXT:    v_mul_hi_u32 v5, v0, v3
-; GFX6-NEXT:    v_mul_lo_u32 v4, v0, v2
-; GFX6-NEXT:    v_mul_hi_u32 v6, v0, v2
-; GFX6-NEXT:    v_mul_hi_u32 v7, v1, v2
-; GFX6-NEXT:    v_mul_lo_u32 v2, v1, v2
-; GFX6-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; GFX6-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc
-; GFX6-NEXT:    v_mul_lo_u32 v6, v1, v3
-; GFX6-NEXT:    v_mul_hi_u32 v3, v1, v3
-; GFX6-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
-; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, v5, v3, vcc
-; GFX6-NEXT:    v_addc_u32_e32 v4, vcc, 0, v7, vcc
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc
-; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
-; GFX6-NEXT:    v_mul_hi_u32 v2, v0, s6
-; GFX6-NEXT:    v_mul_lo_u32 v3, v1, s6
-; GFX6-NEXT:    v_mul_lo_u32 v4, v0, s6
-; GFX6-NEXT:    s_mov_b32 s6, -1
-; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v0
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GFX6-NEXT:    v_mul_lo_u32 v3, v0, v2
-; GFX6-NEXT:    v_mul_hi_u32 v5, v0, v4
-; GFX6-NEXT:    v_mul_hi_u32 v6, v0, v2
-; GFX6-NEXT:    v_mul_hi_u32 v7, v1, v2
-; GFX6-NEXT:    v_mul_lo_u32 v2, v1, v2
-; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v5, v3
-; GFX6-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc
-; GFX6-NEXT:    v_mul_lo_u32 v6, v1, v4
-; GFX6-NEXT:    v_mul_hi_u32 v4, v1, v4
-; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v3, v6
-; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, v5, v4, vcc
-; GFX6-NEXT:    v_addc_u32_e32 v4, vcc, 0, v7, vcc
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc
-; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
-; GFX6-NEXT:    v_mul_lo_u32 v2, s2, v1
-; GFX6-NEXT:    v_mul_hi_u32 v3, s2, v0
-; GFX6-NEXT:    v_mul_hi_u32 v4, s2, v1
-; GFX6-NEXT:    v_mul_hi_u32 v5, s3, v1
-; GFX6-NEXT:    v_mul_lo_u32 v1, s3, v1
+; GFX6-NEXT:    s_lshr_b64 s[4:5], s[4:5], 12
+; GFX6-NEXT:    s_add_u32 s2, 0xe037f, s2
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s2, v0
+; GFX6-NEXT:    s_addc_u32 s3, 0, 0
+; GFX6-NEXT:    s_or_b32 s2, vcc_lo, vcc_hi
+; GFX6-NEXT:    s_cmp_lg_u32 s2, 0
+; GFX6-NEXT:    s_movk_i32 s2, 0xf001
+; GFX6-NEXT:    v_mul_hi_u32 v1, v0, s2
+; GFX6-NEXT:    v_mul_lo_u32 v2, v0, s2
+; GFX6-NEXT:    s_addc_u32 s8, s3, 0x1000ff
+; GFX6-NEXT:    s_mul_i32 s3, s8, 0xfffff001
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v0
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, s3, v1
+; GFX6-NEXT:    v_mul_lo_u32 v3, v0, v1
+; GFX6-NEXT:    v_mul_hi_u32 v4, v0, v2
+; GFX6-NEXT:    v_mul_hi_u32 v5, v0, v1
+; GFX6-NEXT:    v_mul_hi_u32 v6, s8, v1
+; GFX6-NEXT:    v_mul_lo_u32 v1, s8, v1
+; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; GFX6-NEXT:    v_addc_u32_e32 v4, vcc, 0, v5, vcc
+; GFX6-NEXT:    v_mul_lo_u32 v5, s8, v2
+; GFX6-NEXT:    v_mul_hi_u32 v2, s8, v2
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GFX6-NEXT:    v_addc_u32_e32 v2, vcc, v4, v2, vcc
+; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, 0, v6, vcc
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
+; GFX6-NEXT:    v_addc_u32_e32 v2, vcc, 0, v3, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v3, s8
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v3, v2, vcc
+; GFX6-NEXT:    v_mul_lo_u32 v2, s6, v1
+; GFX6-NEXT:    v_mul_hi_u32 v3, s6, v0
+; GFX6-NEXT:    v_mul_hi_u32 v4, s6, v1
+; GFX6-NEXT:    v_mul_hi_u32 v5, s7, v1
+; GFX6-NEXT:    v_mul_lo_u32 v1, s7, v1
 ; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
 ; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc
-; GFX6-NEXT:    v_mul_lo_u32 v4, s3, v0
-; GFX6-NEXT:    v_mul_hi_u32 v0, s3, v0
+; GFX6-NEXT:    v_mul_lo_u32 v4, s7, v0
+; GFX6-NEXT:    v_mul_hi_u32 v0, s7, v0
+; GFX6-NEXT:    s_movk_i32 s8, 0xfff
 ; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
 ; GFX6-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
 ; GFX6-NEXT:    v_addc_u32_e32 v2, vcc, 0, v5, vcc
@@ -7610,73 +7536,46 @@ define amdgpu_kernel void @udiv_v2i64_mixed_pow2k_denom(ptr addrspace(1) %out, <
 ; GFX6-NEXT:    v_add_i32_e32 v6, vcc, 2, v0
 ; GFX6-NEXT:    v_addc_u32_e32 v7, vcc, 0, v1, vcc
 ; GFX6-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
-; GFX6-NEXT:    v_mov_b32_e32 v5, s3
-; GFX6-NEXT:    v_sub_i32_e32 v8, vcc, s2, v8
+; GFX6-NEXT:    v_mov_b32_e32 v5, s7
+; GFX6-NEXT:    v_sub_i32_e32 v8, vcc, s6, v8
 ; GFX6-NEXT:    v_subb_u32_e32 v4, vcc, v5, v4, vcc
 ; GFX6-NEXT:    v_subrev_i32_e32 v5, vcc, s8, v8
 ; GFX6-NEXT:    v_subbrev_u32_e32 v9, vcc, 0, v4, vcc
-; GFX6-NEXT:    s_movk_i32 s2, 0xffe
-; GFX6-NEXT:    v_cmp_lt_u32_e32 vcc, s2, v5
+; GFX6-NEXT:    s_movk_i32 s6, 0xffe
+; GFX6-NEXT:    v_cmp_lt_u32_e32 vcc, s6, v5
 ; GFX6-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
 ; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v9
 ; GFX6-NEXT:    v_cndmask_b32_e32 v5, -1, v5, vcc
 ; GFX6-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
 ; GFX6-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
 ; GFX6-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
-; GFX6-NEXT:    v_cmp_lt_u32_e32 vcc, s2, v8
+; GFX6-NEXT:    v_cmp_lt_u32_e32 vcc, s6, v8
 ; GFX6-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
 ; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
 ; GFX6-NEXT:    v_cndmask_b32_e32 v4, -1, v5, vcc
 ; GFX6-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
 ; GFX6-NEXT:    v_cndmask_b32_e32 v3, v1, v3, vcc
 ; GFX6-NEXT:    v_cndmask_b32_e32 v2, v0, v2, vcc
-; GFX6-NEXT:    v_mov_b32_e32 v0, s0
-; GFX6-NEXT:    v_mov_b32_e32 v1, s1
-; GFX6-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    v_mov_b32_e32 v1, s5
+; GFX6-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
 ; GFX6-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: udiv_v2i64_mixed_pow2k_denom:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0x457ff000
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0x4f800000
-; GFX9-NEXT:    v_mac_f32_e32 v0, 0, v1
-; GFX9-NEXT:    v_rcp_f32_e32 v0, v0
-; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
 ; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x34
+; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; GFX9-NEXT:    s_mov_b32 s8, 0x2ff2fc01
+; GFX9-NEXT:    v_bfrev_b32_e32 v0, 7
 ; GFX9-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
-; GFX9-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
-; GFX9-NEXT:    v_trunc_f32_e32 v1, v1
-; GFX9-NEXT:    v_madmk_f32 v0, v1, 0xcf800000, v0
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v1, v1
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_lshr_b64 s[0:1], s[4:5], 12
-; GFX9-NEXT:    v_readfirstlane_b32 s4, v0
-; GFX9-NEXT:    s_mul_hi_u32 s5, s4, 0xfffff001
-; GFX9-NEXT:    v_readfirstlane_b32 s8, v1
-; GFX9-NEXT:    s_sub_i32 s5, s5, s4
-; GFX9-NEXT:    s_mul_i32 s9, s8, 0xfffff001
-; GFX9-NEXT:    s_add_i32 s5, s5, s9
-; GFX9-NEXT:    s_mul_i32 s11, s4, 0xfffff001
-; GFX9-NEXT:    s_mul_hi_u32 s9, s4, s5
-; GFX9-NEXT:    s_mul_i32 s10, s4, s5
-; GFX9-NEXT:    s_mul_hi_u32 s4, s4, s11
-; GFX9-NEXT:    s_add_u32 s4, s4, s10
-; GFX9-NEXT:    s_addc_u32 s9, 0, s9
-; GFX9-NEXT:    s_mul_hi_u32 s12, s8, s11
-; GFX9-NEXT:    s_mul_i32 s11, s8, s11
-; GFX9-NEXT:    s_add_u32 s4, s4, s11
-; GFX9-NEXT:    s_mul_hi_u32 s10, s8, s5
-; GFX9-NEXT:    s_addc_u32 s4, s9, s12
-; GFX9-NEXT:    s_addc_u32 s9, s10, 0
-; GFX9-NEXT:    s_mul_i32 s5, s8, s5
-; GFX9-NEXT:    s_add_u32 s4, s4, s5
-; GFX9-NEXT:    s_addc_u32 s5, 0, s9
+; GFX9-NEXT:    s_add_u32 s4, 0xe037f, s8
+; GFX9-NEXT:    s_addc_u32 s5, 0, 0
 ; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s4, v0
 ; GFX9-NEXT:    s_cmp_lg_u64 vcc, 0
-; GFX9-NEXT:    s_addc_u32 s4, s8, s5
 ; GFX9-NEXT:    v_readfirstlane_b32 s8, v0
+; GFX9-NEXT:    s_addc_u32 s4, s5, 0x1000ff
 ; GFX9-NEXT:    s_mul_hi_u32 s9, s8, 0xfffff001
 ; GFX9-NEXT:    s_mul_i32 s5, s4, 0xfffff001
 ; GFX9-NEXT:    s_sub_i32 s9, s9, s8
@@ -7824,165 +7723,110 @@ define amdgpu_kernel void @urem_i64_oddk_denom(ptr addrspace(1) %out, i64 %x) {
 ;
 ; GFX6-LABEL: urem_i64_oddk_denom:
 ; GFX6:       ; %bb.0:
-; GFX6-NEXT:    v_mov_b32_e32 v0, 0x4f1761f8
-; GFX6-NEXT:    v_mov_b32_e32 v1, 0x4f800000
-; GFX6-NEXT:    v_madmk_f32 v0, v1, 0x438f8000, v0
-; GFX6-NEXT:    v_rcp_f32_e32 v0, v0
+; GFX6-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
+; GFX6-NEXT:    s_add_u32 s0, 4, 0
+; GFX6-NEXT:    v_mov_b32_e32 v0, 0xe3e0fc
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
+; GFX6-NEXT:    s_addc_u32 s1, 0, 0
+; GFX6-NEXT:    s_or_b32 s0, vcc_lo, vcc_hi
+; GFX6-NEXT:    s_cmp_lg_u32 s0, 0
+; GFX6-NEXT:    s_mov_b32 s0, 0x689e0837
 ; GFX6-NEXT:    s_movk_i32 s2, 0xfee0
-; GFX6-NEXT:    s_mov_b32 s3, 0x689e0837
-; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
-; GFX6-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
-; GFX6-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
-; GFX6-NEXT:    v_trunc_f32_e32 v1, v1
-; GFX6-NEXT:    v_madmk_f32 v0, v1, 0xcf800000, v0
-; GFX6-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX6-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GFX6-NEXT:    v_mul_lo_u32 v1, v0, s2
+; GFX6-NEXT:    v_mul_hi_u32 v2, v0, s0
+; GFX6-NEXT:    s_addc_u32 s1, s1, 0
+; GFX6-NEXT:    s_mul_i32 s2, s1, 0x689e0837
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    s_mov_b32 s8, s4
-; GFX6-NEXT:    s_movk_i32 s4, 0x11f
-; GFX6-NEXT:    v_mul_lo_u32 v2, v0, s2
-; GFX6-NEXT:    v_mul_hi_u32 v3, v0, s3
-; GFX6-NEXT:    v_mul_lo_u32 v4, v1, s3
-; GFX6-NEXT:    v_mul_lo_u32 v5, v0, s3
-; GFX6-NEXT:    s_mov_b32 s12, 0x9761f7c9
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
-; GFX6-NEXT:    v_mul_lo_u32 v3, v0, v2
-; GFX6-NEXT:    v_mul_hi_u32 v4, v0, v5
-; GFX6-NEXT:    v_mul_hi_u32 v6, v0, v2
-; GFX6-NEXT:    v_mul_hi_u32 v7, v1, v2
-; GFX6-NEXT:    v_mul_lo_u32 v2, v1, v2
+; GFX6-NEXT:    s_mov_b32 s4, s8
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
+; GFX6-NEXT:    v_mul_lo_u32 v2, v0, s0
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, s2, v1
+; GFX6-NEXT:    v_mul_lo_u32 v3, v0, v1
+; GFX6-NEXT:    v_mul_hi_u32 v4, v0, v2
+; GFX6-NEXT:    v_mul_hi_u32 v5, v0, v1
+; GFX6-NEXT:    v_mul_hi_u32 v6, s1, v1
+; GFX6-NEXT:    v_mul_lo_u32 v1, s1, v1
 ; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
-; GFX6-NEXT:    v_addc_u32_e32 v4, vcc, 0, v6, vcc
-; GFX6-NEXT:    v_mul_lo_u32 v6, v1, v5
-; GFX6-NEXT:    v_mul_hi_u32 v5, v1, v5
-; GFX6-NEXT:    s_mov_b32 s9, s5
-; GFX6-NEXT:    s_movk_i32 s5, 0x11e
-; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v3, v6
-; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, v4, v5, vcc
-; GFX6-NEXT:    v_addc_u32_e32 v4, vcc, 0, v7, vcc
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc
-; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
-; GFX6-NEXT:    v_mul_lo_u32 v2, v0, s2
-; GFX6-NEXT:    v_mul_hi_u32 v3, v0, s3
-; GFX6-NEXT:    v_mul_lo_u32 v4, v1, s3
-; GFX6-NEXT:    s_mov_b32 s11, 0xf000
-; GFX6-NEXT:    s_mov_b32 s10, -1
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
-; GFX6-NEXT:    v_mul_lo_u32 v3, v0, s3
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v4, v2
-; GFX6-NEXT:    v_mul_lo_u32 v4, v0, v2
-; GFX6-NEXT:    v_mul_hi_u32 v5, v0, v3
-; GFX6-NEXT:    v_mul_hi_u32 v6, v0, v2
-; GFX6-NEXT:    v_mul_hi_u32 v7, v1, v2
-; GFX6-NEXT:    v_mul_lo_u32 v2, v1, v2
-; GFX6-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; GFX6-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc
-; GFX6-NEXT:    v_mul_lo_u32 v6, v1, v3
-; GFX6-NEXT:    v_mul_hi_u32 v3, v1, v3
-; GFX6-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
-; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, v5, v3, vcc
-; GFX6-NEXT:    v_addc_u32_e32 v4, vcc, 0, v7, vcc
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc
-; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
-; GFX6-NEXT:    v_mul_lo_u32 v2, s6, v1
-; GFX6-NEXT:    v_mul_hi_u32 v3, s6, v0
-; GFX6-NEXT:    v_mul_hi_u32 v4, s6, v1
-; GFX6-NEXT:    v_mul_hi_u32 v5, s7, v1
-; GFX6-NEXT:    v_mul_lo_u32 v1, s7, v1
+; GFX6-NEXT:    v_addc_u32_e32 v4, vcc, 0, v5, vcc
+; GFX6-NEXT:    v_mul_lo_u32 v5, s1, v2
+; GFX6-NEXT:    v_mul_hi_u32 v2, s1, v2
+; GFX6-NEXT:    s_movk_i32 s8, 0x11f
+; GFX6-NEXT:    s_mov_b32 s12, 0x9761f7c9
+; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GFX6-NEXT:    v_addc_u32_e32 v2, vcc, v4, v2, vcc
+; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, 0, v6, vcc
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
+; GFX6-NEXT:    v_addc_u32_e32 v2, vcc, 0, v3, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v3, s1
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v3, v2, vcc
+; GFX6-NEXT:    v_mul_lo_u32 v2, s10, v1
+; GFX6-NEXT:    v_mul_hi_u32 v3, s10, v0
+; GFX6-NEXT:    v_mul_hi_u32 v4, s10, v1
+; GFX6-NEXT:    v_mul_hi_u32 v5, s11, v1
+; GFX6-NEXT:    v_mul_lo_u32 v1, s11, v1
 ; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
 ; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc
-; GFX6-NEXT:    v_mul_lo_u32 v4, s7, v0
-; GFX6-NEXT:    v_mul_hi_u32 v0, s7, v0
+; GFX6-NEXT:    v_mul_lo_u32 v4, s11, v0
+; GFX6-NEXT:    v_mul_hi_u32 v0, s11, v0
+; GFX6-NEXT:    s_mov_b32 s5, s9
+; GFX6-NEXT:    s_movk_i32 s9, 0x11e
 ; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
 ; GFX6-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
 ; GFX6-NEXT:    v_addc_u32_e32 v2, vcc, 0, v5, vcc
 ; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
 ; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, 0, v2, vcc
-; GFX6-NEXT:    v_mul_lo_u32 v2, v0, s4
+; GFX6-NEXT:    v_mul_lo_u32 v2, v0, s8
 ; GFX6-NEXT:    v_mul_hi_u32 v3, v0, s12
 ; GFX6-NEXT:    v_mul_lo_u32 v1, v1, s12
 ; GFX6-NEXT:    v_mul_lo_u32 v0, v0, s12
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
 ; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
 ; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
-; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, s7, v1
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, s11, v1
 ; GFX6-NEXT:    v_mov_b32_e32 v3, 0x11f
-; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, s10, v0
 ; GFX6-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, vcc
 ; GFX6-NEXT:    v_subrev_i32_e64 v4, s[0:1], s12, v0
 ; GFX6-NEXT:    v_subbrev_u32_e64 v5, s[2:3], 0, v2, s[0:1]
-; GFX6-NEXT:    v_cmp_lt_u32_e64 s[2:3], s5, v5
-; GFX6-NEXT:    s_mov_b32 s6, 0x9761f7c8
+; GFX6-NEXT:    v_cmp_lt_u32_e64 s[2:3], s9, v5
+; GFX6-NEXT:    s_mov_b32 s10, 0x9761f7c8
 ; GFX6-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[2:3]
-; GFX6-NEXT:    v_cmp_lt_u32_e64 s[2:3], s6, v4
+; GFX6-NEXT:    v_cmp_lt_u32_e64 s[2:3], s10, v4
 ; GFX6-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, s[0:1]
 ; GFX6-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[2:3]
-; GFX6-NEXT:    v_cmp_eq_u32_e64 s[2:3], s4, v5
+; GFX6-NEXT:    v_cmp_eq_u32_e64 s[2:3], s8, v5
 ; GFX6-NEXT:    v_subrev_i32_e64 v3, s[0:1], s12, v4
 ; GFX6-NEXT:    v_cndmask_b32_e64 v6, v6, v7, s[2:3]
 ; GFX6-NEXT:    v_subbrev_u32_e64 v2, s[0:1], 0, v2, s[0:1]
 ; GFX6-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v6
 ; GFX6-NEXT:    v_cndmask_b32_e64 v3, v4, v3, s[0:1]
-; GFX6-NEXT:    v_mov_b32_e32 v4, s7
+; GFX6-NEXT:    v_mov_b32_e32 v4, s11
 ; GFX6-NEXT:    v_subb_u32_e32 v1, vcc, v4, v1, vcc
-; GFX6-NEXT:    v_cmp_lt_u32_e32 vcc, s5, v1
+; GFX6-NEXT:    v_cmp_lt_u32_e32 vcc, s9, v1
 ; GFX6-NEXT:    v_cndmask_b32_e64 v4, 0, -1, vcc
-; GFX6-NEXT:    v_cmp_lt_u32_e32 vcc, s6, v0
+; GFX6-NEXT:    v_cmp_lt_u32_e32 vcc, s10, v0
 ; GFX6-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[0:1]
 ; GFX6-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
-; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, s4, v1
+; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, s8, v1
 ; GFX6-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
 ; GFX6-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
+; GFX6-NEXT:    s_mov_b32 s6, -1
 ; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
 ; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX6-NEXT:    buffer_store_dwordx2 v[0:1], off, s[8:11], 0
+; GFX6-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; GFX6-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: urem_i64_oddk_denom:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0x4f1761f8
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0x4f800000
-; GFX9-NEXT:    v_madmk_f32 v0, v1, 0x438f8000, v0
-; GFX9-NEXT:    v_rcp_f32_e32 v0, v0
 ; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX9-NEXT:    s_mov_b32 s12, 0x9761f7c8
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
-; GFX9-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
-; GFX9-NEXT:    v_trunc_f32_e32 v1, v1
-; GFX9-NEXT:    v_madmk_f32 v0, v1, 0xcf800000, v0
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GFX9-NEXT:    v_readfirstlane_b32 s0, v0
-; GFX9-NEXT:    s_mul_i32 s1, s0, 0xfffffee0
-; GFX9-NEXT:    s_mul_hi_u32 s2, s0, 0x689e0837
-; GFX9-NEXT:    s_add_i32 s1, s2, s1
-; GFX9-NEXT:    v_readfirstlane_b32 s2, v1
-; GFX9-NEXT:    s_mul_i32 s3, s2, 0x689e0837
-; GFX9-NEXT:    s_add_i32 s1, s1, s3
-; GFX9-NEXT:    s_mul_i32 s9, s0, 0x689e0837
-; GFX9-NEXT:    s_mul_hi_u32 s3, s0, s1
-; GFX9-NEXT:    s_mul_i32 s8, s0, s1
-; GFX9-NEXT:    s_mul_hi_u32 s0, s0, s9
-; GFX9-NEXT:    s_add_u32 s0, s0, s8
-; GFX9-NEXT:    s_addc_u32 s3, 0, s3
-; GFX9-NEXT:    s_mul_hi_u32 s10, s2, s9
-; GFX9-NEXT:    s_mul_i32 s9, s2, s9
-; GFX9-NEXT:    s_add_u32 s0, s0, s9
-; GFX9-NEXT:    s_mul_hi_u32 s8, s2, s1
-; GFX9-NEXT:    s_addc_u32 s0, s3, s10
-; GFX9-NEXT:    s_addc_u32 s3, s8, 0
-; GFX9-NEXT:    s_mul_i32 s1, s2, s1
-; GFX9-NEXT:    s_add_u32 s0, s0, s1
-; GFX9-NEXT:    s_addc_u32 s1, 0, s3
+; GFX9-NEXT:    s_add_u32 s0, 4, 0
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0xe3e0fc
+; GFX9-NEXT:    s_addc_u32 s1, 0, 0
 ; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v0
 ; GFX9-NEXT:    s_cmp_lg_u64 vcc, 0
-; GFX9-NEXT:    s_addc_u32 s0, s2, s1
 ; GFX9-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX9-NEXT:    s_addc_u32 s0, s1, 0
 ; GFX9-NEXT:    s_mul_i32 s3, s2, 0xfffffee0
 ; GFX9-NEXT:    s_mul_hi_u32 s8, s2, 0x689e0837
 ; GFX9-NEXT:    s_mul_i32 s1, s0, 0x689e0837
@@ -8038,6 +7882,7 @@ define amdgpu_kernel void @urem_i64_oddk_denom(ptr addrspace(1) %out, i64 %x) {
 ; GFX9-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GFX9-NEXT:    s_subb_u32 s10, s6, 0
 ; GFX9-NEXT:    s_cmpk_gt_u32 s10, 0x11e
+; GFX9-NEXT:    s_mov_b32 s12, 0x9761f7c8
 ; GFX9-NEXT:    s_cselect_b32 s11, -1, 0
 ; GFX9-NEXT:    v_cmp_lt_u32_e64 s[2:3], s12, v1
 ; GFX9-NEXT:    s_cmpk_eq_i32 s10, 0x11f
@@ -8067,6 +7912,7 @@ define amdgpu_kernel void @urem_i64_oddk_denom(ptr addrspace(1) %out, i64 %x) {
 ; GFX9-NEXT:    v_cndmask_b32_e32 v4, v5, v4, vcc
 ; GFX9-NEXT:    v_mov_b32_e32 v5, s0
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
 ; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[4:5]
@@ -8274,69 +8120,44 @@ define amdgpu_kernel void @sdiv_i64_oddk_denom(ptr addrspace(1) %out, i64 %x) {
 ;
 ; GFX6-LABEL: sdiv_i64_oddk_denom:
 ; GFX6:       ; %bb.0:
-; GFX6-NEXT:    v_mov_b32_e32 v0, 0x4f800000
-; GFX6-NEXT:    v_madak_f32 v0, 0, v0, 0x4996c7d8
-; GFX6-NEXT:    v_rcp_f32_e32 v0, v0
-; GFX6-NEXT:    s_mov_b32 s8, 0xffed2705
 ; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
-; GFX6-NEXT:    s_mov_b32 s3, 0xf000
-; GFX6-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
-; GFX6-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
-; GFX6-NEXT:    v_trunc_f32_e32 v1, v1
-; GFX6-NEXT:    v_madmk_f32 v0, v1, 0xcf800000, v0
-; GFX6-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GFX6-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX6-NEXT:    s_mov_b32 s0, 0x33fe64
+; GFX6-NEXT:    s_add_u32 s1, 0x396, s0
+; GFX6-NEXT:    v_mov_b32_e32 v0, 0x28100000
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s1, v0
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX6-NEXT:    s_mov_b32 s0, s4
-; GFX6-NEXT:    s_mov_b32 s4, 0x12d8fb
-; GFX6-NEXT:    v_mul_lo_u32 v2, v1, s8
-; GFX6-NEXT:    v_mul_hi_u32 v3, v0, s8
-; GFX6-NEXT:    v_mul_lo_u32 v4, v0, s8
-; GFX6-NEXT:    s_mov_b32 s2, -1
-; GFX6-NEXT:    s_mov_b32 s1, s5
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
-; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v0
-; GFX6-NEXT:    v_mul_hi_u32 v3, v0, v4
-; GFX6-NEXT:    v_mul_lo_u32 v5, v0, v2
+; GFX6-NEXT:    s_addc_u32 s4, 0, 0
+; GFX6-NEXT:    s_or_b32 s1, vcc_lo, vcc_hi
+; GFX6-NEXT:    s_cmp_lg_u32 s1, 0
+; GFX6-NEXT:    s_mov_b32 s1, 0xffed2705
+; GFX6-NEXT:    v_mul_hi_u32 v1, v0, s1
+; GFX6-NEXT:    s_addc_u32 s4, s4, 0xd95
+; GFX6-NEXT:    v_mul_lo_u32 v2, v0, s1
+; GFX6-NEXT:    s_mul_i32 s8, s4, 0xffed2705
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, s8, v1
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v0
+; GFX6-NEXT:    v_mul_lo_u32 v5, v0, v1
 ; GFX6-NEXT:    v_mul_hi_u32 v6, v0, v2
-; GFX6-NEXT:    v_mul_hi_u32 v7, v1, v2
-; GFX6-NEXT:    v_mul_lo_u32 v2, v1, v2
-; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
-; GFX6-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc
-; GFX6-NEXT:    v_mul_lo_u32 v6, v1, v4
-; GFX6-NEXT:    v_mul_hi_u32 v4, v1, v4
-; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v3, v6
-; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, v5, v4, vcc
-; GFX6-NEXT:    v_addc_u32_e32 v4, vcc, 0, v7, vcc
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc
-; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
-; GFX6-NEXT:    v_mul_lo_u32 v2, v1, s8
-; GFX6-NEXT:    v_mul_hi_u32 v3, v0, s8
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
-; GFX6-NEXT:    v_mul_lo_u32 v3, v0, s8
-; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v0
-; GFX6-NEXT:    v_mul_lo_u32 v6, v0, v2
-; GFX6-NEXT:    v_mul_hi_u32 v7, v0, v3
-; GFX6-NEXT:    v_mul_hi_u32 v8, v0, v2
-; GFX6-NEXT:    v_mul_hi_u32 v5, v1, v3
-; GFX6-NEXT:    v_mul_lo_u32 v3, v1, v3
-; GFX6-NEXT:    v_mul_hi_u32 v4, v1, v2
-; GFX6-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
-; GFX6-NEXT:    v_addc_u32_e32 v7, vcc, 0, v8, vcc
-; GFX6-NEXT:    v_mul_lo_u32 v2, v1, v2
-; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v6, v3
-; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, v7, v5, vcc
-; GFX6-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GFX6-NEXT:    v_mul_hi_u32 v7, v0, v1
+; GFX6-NEXT:    v_mul_hi_u32 v4, s4, v2
+; GFX6-NEXT:    v_mul_lo_u32 v2, s4, v2
+; GFX6-NEXT:    v_mul_hi_u32 v3, s4, v1
+; GFX6-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; GFX6-NEXT:    v_addc_u32_e32 v6, vcc, 0, v7, vcc
+; GFX6-NEXT:    v_mul_lo_u32 v1, s4, v1
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v5, v2
+; GFX6-NEXT:    v_addc_u32_e32 v2, vcc, v6, v4, vcc
+; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; GFX6-NEXT:    s_ashr_i32 s8, s7, 31
-; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc
+; GFX6-NEXT:    v_addc_u32_e32 v2, vcc, 0, v3, vcc
 ; GFX6-NEXT:    s_add_u32 s6, s6, s8
-; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GFX6-NEXT:    v_mov_b32_e32 v3, s4
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
 ; GFX6-NEXT:    s_mov_b32 s9, s8
 ; GFX6-NEXT:    s_addc_u32 s7, s7, s8
-; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
+; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v3, v2, vcc
 ; GFX6-NEXT:    s_xor_b64 s[6:7], s[6:7], s[8:9]
 ; GFX6-NEXT:    v_mul_lo_u32 v2, s6, v1
 ; GFX6-NEXT:    v_mul_hi_u32 v3, s6, v0
@@ -8347,6 +8168,8 @@ define amdgpu_kernel void @sdiv_i64_oddk_denom(ptr addrspace(1) %out, i64 %x) {
 ; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc
 ; GFX6-NEXT:    v_mul_lo_u32 v4, s7, v0
 ; GFX6-NEXT:    v_mul_hi_u32 v0, s7, v0
+; GFX6-NEXT:    s_mov_b32 s4, 0x12d8fb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
 ; GFX6-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
 ; GFX6-NEXT:    v_addc_u32_e32 v2, vcc, 0, v5, vcc
@@ -8377,55 +8200,28 @@ define amdgpu_kernel void @sdiv_i64_oddk_denom(ptr addrspace(1) %out, i64 %x) {
 ; GFX6-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
 ; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
 ; GFX6-NEXT:    v_cndmask_b32_e32 v4, -1, v5, vcc
-; GFX6-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
-; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX6-NEXT:    v_xor_b32_e32 v0, s8, v0
-; GFX6-NEXT:    v_xor_b32_e32 v1, s8, v1
-; GFX6-NEXT:    v_mov_b32_e32 v2, s8
-; GFX6-NEXT:    v_subrev_i32_e32 v0, vcc, s8, v0
-; GFX6-NEXT:    v_subb_u32_e32 v1, vcc, v1, v2, vcc
-; GFX6-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
-; GFX6-NEXT:    s_endpgm
-;
-; GFX9-LABEL: sdiv_i64_oddk_denom:
-; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0x4996c7d8
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0x4f800000
-; GFX9-NEXT:    v_mac_f32_e32 v0, 0, v1
-; GFX9-NEXT:    v_rcp_f32_e32 v0, v0
-; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
-; GFX9-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
-; GFX9-NEXT:    v_trunc_f32_e32 v1, v1
-; GFX9-NEXT:    v_madmk_f32 v0, v1, 0xcf800000, v0
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT:    v_readfirstlane_b32 s4, v1
-; GFX9-NEXT:    v_readfirstlane_b32 s5, v0
-; GFX9-NEXT:    s_mul_hi_u32 s6, s5, 0xffed2705
-; GFX9-NEXT:    s_mul_i32 s7, s4, 0xffed2705
-; GFX9-NEXT:    s_add_i32 s6, s6, s7
-; GFX9-NEXT:    s_sub_i32 s6, s6, s5
-; GFX9-NEXT:    s_mul_i32 s9, s5, 0xffed2705
-; GFX9-NEXT:    s_mul_hi_u32 s7, s5, s6
-; GFX9-NEXT:    s_mul_i32 s8, s5, s6
-; GFX9-NEXT:    s_mul_hi_u32 s5, s5, s9
-; GFX9-NEXT:    s_add_u32 s5, s5, s8
-; GFX9-NEXT:    s_addc_u32 s7, 0, s7
-; GFX9-NEXT:    s_mul_hi_u32 s10, s4, s9
-; GFX9-NEXT:    s_mul_i32 s9, s4, s9
-; GFX9-NEXT:    s_add_u32 s5, s5, s9
-; GFX9-NEXT:    s_mul_hi_u32 s8, s4, s6
-; GFX9-NEXT:    s_addc_u32 s5, s7, s10
-; GFX9-NEXT:    s_addc_u32 s7, s8, 0
-; GFX9-NEXT:    s_mul_i32 s6, s4, s6
-; GFX9-NEXT:    s_add_u32 s5, s5, s6
-; GFX9-NEXT:    s_addc_u32 s6, 0, s7
-; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s5, v0
+; GFX6-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
+; GFX6-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX6-NEXT:    v_xor_b32_e32 v0, s8, v0
+; GFX6-NEXT:    v_xor_b32_e32 v1, s8, v1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s8
+; GFX6-NEXT:    v_subrev_i32_e32 v0, vcc, s8, v0
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    v_subb_u32_e32 v1, vcc, v1, v2, vcc
+; GFX6-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX9-LABEL: sdiv_i64_oddk_denom:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_mov_b32 s4, 0x33fe64
+; GFX9-NEXT:    s_add_u32 s4, 0x396, s4
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0x28100000
+; GFX9-NEXT:    s_addc_u32 s5, 0, 0
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s4, v0
 ; GFX9-NEXT:    s_cmp_lg_u64 vcc, 0
-; GFX9-NEXT:    s_addc_u32 s4, s4, s6
+; GFX9-NEXT:    s_addc_u32 s4, s5, 0xd95
 ; GFX9-NEXT:    v_readfirstlane_b32 s6, v0
 ; GFX9-NEXT:    s_mul_i32 s5, s4, 0xffed2705
 ; GFX9-NEXT:    s_mul_hi_u32 s7, s6, 0xffed2705
@@ -8440,6 +8236,7 @@ define amdgpu_kernel void @sdiv_i64_oddk_denom(ptr addrspace(1) %out, i64 %x) {
 ; GFX9-NEXT:    s_mul_i32 s10, s4, s8
 ; GFX9-NEXT:    s_addc_u32 s8, 0, s11
 ; GFX9-NEXT:    s_add_u32 s6, s6, s10
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX9-NEXT:    s_mul_hi_u32 s7, s4, s5
 ; GFX9-NEXT:    s_addc_u32 s6, s8, s9
 ; GFX9-NEXT:    s_addc_u32 s7, s7, 0
@@ -8455,65 +8252,59 @@ define amdgpu_kernel void @sdiv_i64_oddk_denom(ptr addrspace(1) %out, i64 %x) {
 ; GFX9-NEXT:    s_mov_b32 s5, s4
 ; GFX9-NEXT:    s_addc_u32 s3, s3, s4
 ; GFX9-NEXT:    s_xor_b64 s[2:3], s[2:3], s[4:5]
-; GFX9-NEXT:    v_readfirstlane_b32 s8, v0
-; GFX9-NEXT:    s_mul_i32 s7, s2, s6
-; GFX9-NEXT:    s_mul_hi_u32 s9, s2, s8
-; GFX9-NEXT:    s_mul_hi_u32 s5, s2, s6
-; GFX9-NEXT:    s_add_u32 s7, s9, s7
-; GFX9-NEXT:    s_addc_u32 s5, 0, s5
-; GFX9-NEXT:    s_mul_hi_u32 s10, s3, s8
-; GFX9-NEXT:    s_mul_i32 s8, s3, s8
-; GFX9-NEXT:    s_add_u32 s7, s7, s8
-; GFX9-NEXT:    s_mul_hi_u32 s9, s3, s6
-; GFX9-NEXT:    s_addc_u32 s5, s5, s10
-; GFX9-NEXT:    s_addc_u32 s7, s9, 0
+; GFX9-NEXT:    v_readfirstlane_b32 s9, v0
+; GFX9-NEXT:    s_mul_i32 s8, s2, s6
+; GFX9-NEXT:    s_mul_hi_u32 s10, s2, s9
+; GFX9-NEXT:    s_mul_hi_u32 s7, s2, s6
+; GFX9-NEXT:    s_add_u32 s8, s10, s8
+; GFX9-NEXT:    s_addc_u32 s7, 0, s7
+; GFX9-NEXT:    s_mul_hi_u32 s11, s3, s9
+; GFX9-NEXT:    s_mul_i32 s9, s3, s9
+; GFX9-NEXT:    s_add_u32 s8, s8, s9
+; GFX9-NEXT:    s_mul_hi_u32 s10, s3, s6
+; GFX9-NEXT:    s_addc_u32 s7, s7, s11
+; GFX9-NEXT:    s_addc_u32 s8, s10, 0
 ; GFX9-NEXT:    s_mul_i32 s6, s3, s6
-; GFX9-NEXT:    s_add_u32 s5, s5, s6
-; GFX9-NEXT:    s_addc_u32 s6, 0, s7
-; GFX9-NEXT:    s_add_u32 s7, s5, 1
-; GFX9-NEXT:    s_addc_u32 s8, s6, 0
-; GFX9-NEXT:    s_add_u32 s9, s5, 2
-; GFX9-NEXT:    s_mul_i32 s12, s6, 0x12d8fb
-; GFX9-NEXT:    s_mul_hi_u32 s13, s5, 0x12d8fb
-; GFX9-NEXT:    s_addc_u32 s10, s6, 0
-; GFX9-NEXT:    s_add_i32 s13, s13, s12
-; GFX9-NEXT:    s_mul_i32 s12, s5, 0x12d8fb
-; GFX9-NEXT:    v_mov_b32_e32 v0, s12
+; GFX9-NEXT:    s_add_u32 s6, s7, s6
+; GFX9-NEXT:    s_addc_u32 s7, 0, s8
+; GFX9-NEXT:    s_add_u32 s8, s6, 1
+; GFX9-NEXT:    s_addc_u32 s9, s7, 0
+; GFX9-NEXT:    s_add_u32 s10, s6, 2
+; GFX9-NEXT:    s_mul_i32 s13, s7, 0x12d8fb
+; GFX9-NEXT:    s_mul_hi_u32 s14, s6, 0x12d8fb
+; GFX9-NEXT:    s_addc_u32 s11, s7, 0
+; GFX9-NEXT:    s_add_i32 s14, s14, s13
+; GFX9-NEXT:    s_mul_i32 s13, s6, 0x12d8fb
+; GFX9-NEXT:    v_mov_b32_e32 v0, s13
 ; GFX9-NEXT:    v_sub_co_u32_e32 v0, vcc, s2, v0
-; GFX9-NEXT:    s_mov_b32 s11, 0x12d8fb
+; GFX9-NEXT:    s_mov_b32 s12, 0x12d8fb
 ; GFX9-NEXT:    s_cmp_lg_u64 vcc, 0
-; GFX9-NEXT:    s_subb_u32 s2, s3, s13
-; GFX9-NEXT:    v_subrev_co_u32_e32 v1, vcc, s11, v0
+; GFX9-NEXT:    s_subb_u32 s2, s3, s14
+; GFX9-NEXT:    v_subrev_co_u32_e32 v1, vcc, s12, v0
 ; GFX9-NEXT:    s_cmp_lg_u64 vcc, 0
 ; GFX9-NEXT:    s_subb_u32 s3, s2, 0
-; GFX9-NEXT:    s_mov_b32 s11, 0x12d8fa
-; GFX9-NEXT:    v_cmp_lt_u32_e32 vcc, s11, v1
+; GFX9-NEXT:    v_readfirstlane_b32 s12, v1
+; GFX9-NEXT:    s_cmp_gt_u32 s12, 0x12d8fa
+; GFX9-NEXT:    s_cselect_b32 s12, -1, 0
 ; GFX9-NEXT:    s_cmp_eq_u32 s3, 0
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc
-; GFX9-NEXT:    s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, -1, v1, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v3, s7
-; GFX9-NEXT:    v_mov_b32_e32 v4, s9
-; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v3, s8
-; GFX9-NEXT:    v_mov_b32_e32 v4, s10
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
-; GFX9-NEXT:    v_cmp_lt_u32_e32 vcc, s11, v0
+; GFX9-NEXT:    s_cselect_b32 s3, s12, -1
+; GFX9-NEXT:    s_cmp_lg_u32 s3, 0
+; GFX9-NEXT:    s_cselect_b32 s3, s11, s9
+; GFX9-NEXT:    v_readfirstlane_b32 s9, v0
+; GFX9-NEXT:    s_cselect_b32 s8, s10, s8
+; GFX9-NEXT:    s_cmp_gt_u32 s9, 0x12d8fa
+; GFX9-NEXT:    s_cselect_b32 s9, -1, 0
 ; GFX9-NEXT:    s_cmp_eq_u32 s2, 0
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
-; GFX9-NEXT:    s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v4, s6
-; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v3, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX9-NEXT:    v_xor_b32_e32 v1, s4, v1
-; GFX9-NEXT:    v_xor_b32_e32 v3, s4, v0
-; GFX9-NEXT:    v_mov_b32_e32 v4, s4
-; GFX9-NEXT:    v_subrev_co_u32_e32 v0, vcc, s4, v1
-; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v3, v4, vcc
+; GFX9-NEXT:    s_cselect_b32 s2, s9, -1
+; GFX9-NEXT:    s_cmp_lg_u32 s2, 0
+; GFX9-NEXT:    s_cselect_b32 s3, s3, s7
+; GFX9-NEXT:    s_cselect_b32 s2, s8, s6
+; GFX9-NEXT:    s_xor_b64 s[2:3], s[2:3], s[4:5]
+; GFX9-NEXT:    s_sub_u32 s2, s2, s4
+; GFX9-NEXT:    s_subb_u32 s3, s3, s4
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
 ; GFX9-NEXT:    s_endpgm
   %r = sdiv i64 %x, 1235195
@@ -8940,84 +8731,62 @@ define amdgpu_kernel void @ssdiv_v2i64_mixed_pow2k_denom(ptr addrspace(1) %out,
 ;
 ; GFX6-LABEL: ssdiv_v2i64_mixed_pow2k_denom:
 ; GFX6:       ; %bb.0:
-; GFX6-NEXT:    v_mov_b32_e32 v0, 0x457ff000
-; GFX6-NEXT:    v_mov_b32_e32 v1, 0x4f800000
-; GFX6-NEXT:    v_mac_f32_e32 v0, 0, v1
-; GFX6-NEXT:    v_rcp_f32_e32 v0, v0
-; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
-; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0xd
-; GFX6-NEXT:    s_movk_i32 s6, 0xf001
-; GFX6-NEXT:    s_mov_b32 s7, 0xf000
-; GFX6-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
-; GFX6-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
-; GFX6-NEXT:    v_trunc_f32_e32 v1, v1
-; GFX6-NEXT:    v_madmk_f32 v0, v1, 0xcf800000, v0
-; GFX6-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX6-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0xd
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s2, 0x2ff2fc01
+; GFX6-NEXT:    v_bfrev_b32_e32 v0, 7
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    s_ashr_i32 s8, s1, 31
+; GFX6-NEXT:    s_ashr_i32 s8, s5, 31
 ; GFX6-NEXT:    s_lshr_b32 s8, s8, 20
-; GFX6-NEXT:    v_mul_hi_u32 v2, v0, s6
-; GFX6-NEXT:    v_mul_lo_u32 v3, v1, s6
-; GFX6-NEXT:    s_add_u32 s0, s0, s8
-; GFX6-NEXT:    s_addc_u32 s1, s1, 0
-; GFX6-NEXT:    s_ashr_i64 s[0:1], s[0:1], 12
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GFX6-NEXT:    v_mul_lo_u32 v3, v0, s6
-; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v0
-; GFX6-NEXT:    v_mul_lo_u32 v4, v0, v2
-; GFX6-NEXT:    v_mul_hi_u32 v5, v0, v3
+; GFX6-NEXT:    s_add_u32 s4, s4, s8
+; GFX6-NEXT:    s_addc_u32 s5, s5, 0
+; GFX6-NEXT:    s_ashr_i64 s[4:5], s[4:5], 12
+; GFX6-NEXT:    s_add_u32 s2, 0xe037f, s2
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s2, v0
+; GFX6-NEXT:    s_addc_u32 s8, 0, 0
+; GFX6-NEXT:    s_or_b32 s2, vcc_lo, vcc_hi
+; GFX6-NEXT:    s_cmp_lg_u32 s2, 0
+; GFX6-NEXT:    s_movk_i32 s2, 0xf001
+; GFX6-NEXT:    v_mul_hi_u32 v1, v0, s2
+; GFX6-NEXT:    s_addc_u32 s8, s8, 0x1000ff
+; GFX6-NEXT:    v_mul_lo_u32 v2, v0, s2
+; GFX6-NEXT:    s_mul_i32 s9, s8, 0xfffff001
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, s9, v1
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v0
+; GFX6-NEXT:    v_mul_lo_u32 v5, v0, v1
 ; GFX6-NEXT:    v_mul_hi_u32 v6, v0, v2
-; GFX6-NEXT:    v_mul_hi_u32 v7, v1, v2
-; GFX6-NEXT:    v_mul_lo_u32 v2, v1, v2
-; GFX6-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; GFX6-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc
-; GFX6-NEXT:    v_mul_lo_u32 v6, v1, v3
-; GFX6-NEXT:    v_mul_hi_u32 v3, v1, v3
-; GFX6-NEXT:    s_ashr_i32 s8, s3, 31
-; GFX6-NEXT:    s_add_u32 s2, s2, s8
-; GFX6-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
-; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, v5, v3, vcc
-; GFX6-NEXT:    v_addc_u32_e32 v4, vcc, 0, v7, vcc
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc
-; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
-; GFX6-NEXT:    v_mul_lo_u32 v2, v1, s6
-; GFX6-NEXT:    v_mul_hi_u32 v3, v0, s6
+; GFX6-NEXT:    v_mul_hi_u32 v7, v0, v1
+; GFX6-NEXT:    v_mul_hi_u32 v4, s8, v2
+; GFX6-NEXT:    v_mul_lo_u32 v2, s8, v2
+; GFX6-NEXT:    v_mul_hi_u32 v3, s8, v1
+; GFX6-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; GFX6-NEXT:    v_addc_u32_e32 v6, vcc, 0, v7, vcc
+; GFX6-NEXT:    v_mul_lo_u32 v1, s8, v1
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v5, v2
+; GFX6-NEXT:    v_addc_u32_e32 v2, vcc, v6, v4, vcc
+; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
+; GFX6-NEXT:    v_addc_u32_e32 v2, vcc, 0, v3, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v3, s8
+; GFX6-NEXT:    s_ashr_i32 s8, s7, 31
+; GFX6-NEXT:    s_add_u32 s6, s6, s8
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
 ; GFX6-NEXT:    s_mov_b32 s9, s8
-; GFX6-NEXT:    s_addc_u32 s3, s3, s8
-; GFX6-NEXT:    s_xor_b64 s[2:3], s[2:3], s[8:9]
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
-; GFX6-NEXT:    v_mul_lo_u32 v3, v0, s6
-; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v0
-; GFX6-NEXT:    v_mul_lo_u32 v6, v0, v2
-; GFX6-NEXT:    v_mul_hi_u32 v7, v0, v3
-; GFX6-NEXT:    v_mul_hi_u32 v8, v0, v2
-; GFX6-NEXT:    v_mul_hi_u32 v5, v1, v3
-; GFX6-NEXT:    v_mul_lo_u32 v3, v1, v3
-; GFX6-NEXT:    v_mul_hi_u32 v4, v1, v2
-; GFX6-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
-; GFX6-NEXT:    v_addc_u32_e32 v7, vcc, 0, v8, vcc
-; GFX6-NEXT:    v_mul_lo_u32 v2, v1, v2
-; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v6, v3
-; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, v7, v5, vcc
-; GFX6-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc
-; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
-; GFX6-NEXT:    v_mul_lo_u32 v2, s2, v1
-; GFX6-NEXT:    v_mul_hi_u32 v3, s2, v0
-; GFX6-NEXT:    v_mul_hi_u32 v4, s2, v1
-; GFX6-NEXT:    v_mul_hi_u32 v5, s3, v1
-; GFX6-NEXT:    v_mul_lo_u32 v1, s3, v1
+; GFX6-NEXT:    s_addc_u32 s7, s7, s8
+; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v3, v2, vcc
+; GFX6-NEXT:    s_xor_b64 s[6:7], s[6:7], s[8:9]
+; GFX6-NEXT:    v_mul_lo_u32 v2, s6, v1
+; GFX6-NEXT:    v_mul_hi_u32 v3, s6, v0
+; GFX6-NEXT:    v_mul_hi_u32 v4, s6, v1
+; GFX6-NEXT:    v_mul_hi_u32 v5, s7, v1
+; GFX6-NEXT:    v_mul_lo_u32 v1, s7, v1
 ; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
 ; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc
-; GFX6-NEXT:    v_mul_lo_u32 v4, s3, v0
-; GFX6-NEXT:    v_mul_hi_u32 v0, s3, v0
+; GFX6-NEXT:    v_mul_lo_u32 v4, s7, v0
+; GFX6-NEXT:    v_mul_hi_u32 v0, s7, v0
 ; GFX6-NEXT:    s_movk_i32 s9, 0xfff
-; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_mov_b32 s2, -1
 ; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
 ; GFX6-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
 ; GFX6-NEXT:    v_addc_u32_e32 v2, vcc, 0, v5, vcc
@@ -9031,20 +8800,20 @@ define amdgpu_kernel void @ssdiv_v2i64_mixed_pow2k_denom(ptr addrspace(1) %out,
 ; GFX6-NEXT:    v_add_i32_e32 v6, vcc, 2, v0
 ; GFX6-NEXT:    v_addc_u32_e32 v7, vcc, 0, v1, vcc
 ; GFX6-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
-; GFX6-NEXT:    v_mov_b32_e32 v5, s3
-; GFX6-NEXT:    v_sub_i32_e32 v8, vcc, s2, v8
+; GFX6-NEXT:    v_mov_b32_e32 v5, s7
+; GFX6-NEXT:    v_sub_i32_e32 v8, vcc, s6, v8
 ; GFX6-NEXT:    v_subb_u32_e32 v4, vcc, v5, v4, vcc
 ; GFX6-NEXT:    v_subrev_i32_e32 v5, vcc, s9, v8
 ; GFX6-NEXT:    v_subbrev_u32_e32 v9, vcc, 0, v4, vcc
-; GFX6-NEXT:    s_movk_i32 s2, 0xffe
-; GFX6-NEXT:    v_cmp_lt_u32_e32 vcc, s2, v5
+; GFX6-NEXT:    s_movk_i32 s6, 0xffe
+; GFX6-NEXT:    v_cmp_lt_u32_e32 vcc, s6, v5
 ; GFX6-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
 ; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v9
 ; GFX6-NEXT:    v_cndmask_b32_e32 v5, -1, v5, vcc
 ; GFX6-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
 ; GFX6-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
 ; GFX6-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
-; GFX6-NEXT:    v_cmp_lt_u32_e32 vcc, s2, v8
+; GFX6-NEXT:    v_cmp_lt_u32_e32 vcc, s6, v8
 ; GFX6-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
 ; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
 ; GFX6-NEXT:    v_cndmask_b32_e32 v4, -1, v5, vcc
@@ -9056,57 +8825,30 @@ define amdgpu_kernel void @ssdiv_v2i64_mixed_pow2k_denom(ptr addrspace(1) %out,
 ; GFX6-NEXT:    v_mov_b32_e32 v3, s8
 ; GFX6-NEXT:    v_subrev_i32_e32 v2, vcc, s8, v0
 ; GFX6-NEXT:    v_subb_u32_e32 v3, vcc, v1, v3, vcc
-; GFX6-NEXT:    v_mov_b32_e32 v0, s0
-; GFX6-NEXT:    v_mov_b32_e32 v1, s1
-; GFX6-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    v_mov_b32_e32 v1, s5
+; GFX6-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
 ; GFX6-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: ssdiv_v2i64_mixed_pow2k_denom:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0x457ff000
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GFX9-NEXT:    v_mac_f32_e32 v1, 0, v2
-; GFX9-NEXT:    v_rcp_f32_e32 v1, v1
-; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
 ; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x34
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    v_mul_f32_e32 v1, 0x5f7ffffc, v1
-; GFX9-NEXT:    v_mul_f32_e32 v2, 0x2f800000, v1
-; GFX9-NEXT:    v_trunc_f32_e32 v2, v2
-; GFX9-NEXT:    v_madmk_f32 v1, v2, 0xcf800000, v1
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; GFX9-NEXT:    s_mov_b32 s8, 0x2ff2fc01
+; GFX9-NEXT:    v_bfrev_b32_e32 v0, 7
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_ashr_i32 s0, s5, 31
 ; GFX9-NEXT:    s_lshr_b32 s0, s0, 20
 ; GFX9-NEXT:    s_add_u32 s0, s4, s0
-; GFX9-NEXT:    v_readfirstlane_b32 s4, v1
-; GFX9-NEXT:    v_readfirstlane_b32 s8, v2
 ; GFX9-NEXT:    s_addc_u32 s1, s5, 0
-; GFX9-NEXT:    s_mul_hi_u32 s5, s4, 0xfffff001
-; GFX9-NEXT:    s_mul_i32 s9, s8, 0xfffff001
-; GFX9-NEXT:    s_add_i32 s5, s5, s9
-; GFX9-NEXT:    s_sub_i32 s5, s5, s4
-; GFX9-NEXT:    s_mul_i32 s11, s4, 0xfffff001
 ; GFX9-NEXT:    s_ashr_i64 s[0:1], s[0:1], 12
-; GFX9-NEXT:    s_mul_hi_u32 s9, s4, s5
-; GFX9-NEXT:    s_mul_i32 s10, s4, s5
-; GFX9-NEXT:    s_mul_hi_u32 s4, s4, s11
-; GFX9-NEXT:    s_add_u32 s4, s4, s10
-; GFX9-NEXT:    s_addc_u32 s9, 0, s9
-; GFX9-NEXT:    s_mul_hi_u32 s12, s8, s11
-; GFX9-NEXT:    s_mul_i32 s11, s8, s11
-; GFX9-NEXT:    s_add_u32 s4, s4, s11
-; GFX9-NEXT:    s_mul_hi_u32 s10, s8, s5
-; GFX9-NEXT:    s_addc_u32 s4, s9, s12
-; GFX9-NEXT:    s_addc_u32 s9, s10, 0
-; GFX9-NEXT:    s_mul_i32 s5, s8, s5
-; GFX9-NEXT:    s_add_u32 s4, s4, s5
-; GFX9-NEXT:    s_addc_u32 s5, 0, s9
-; GFX9-NEXT:    v_add_co_u32_e32 v1, vcc, s4, v1
+; GFX9-NEXT:    s_add_u32 s4, 0xe037f, s8
+; GFX9-NEXT:    s_addc_u32 s5, 0, 0
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s4, v0
 ; GFX9-NEXT:    s_cmp_lg_u64 vcc, 0
-; GFX9-NEXT:    s_addc_u32 s4, s8, s5
-; GFX9-NEXT:    v_readfirstlane_b32 s8, v1
+; GFX9-NEXT:    s_addc_u32 s4, s5, 0x1000ff
+; GFX9-NEXT:    v_readfirstlane_b32 s8, v0
 ; GFX9-NEXT:    s_mul_i32 s5, s4, 0xfffff001
 ; GFX9-NEXT:    s_mul_hi_u32 s9, s8, 0xfffff001
 ; GFX9-NEXT:    s_add_i32 s9, s9, s5
@@ -9126,7 +8868,7 @@ define amdgpu_kernel void @ssdiv_v2i64_mixed_pow2k_denom(ptr addrspace(1) %out,
 ; GFX9-NEXT:    s_mul_i32 s5, s4, s5
 ; GFX9-NEXT:    s_add_u32 s5, s8, s5
 ; GFX9-NEXT:    s_addc_u32 s8, 0, s9
-; GFX9-NEXT:    v_add_co_u32_e32 v1, vcc, s5, v1
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s5, v0
 ; GFX9-NEXT:    s_cmp_lg_u64 vcc, 0
 ; GFX9-NEXT:    s_addc_u32 s8, s4, s8
 ; GFX9-NEXT:    s_ashr_i32 s4, s7, 31
@@ -9134,68 +8876,61 @@ define amdgpu_kernel void @ssdiv_v2i64_mixed_pow2k_denom(ptr addrspace(1) %out,
 ; GFX9-NEXT:    s_mov_b32 s5, s4
 ; GFX9-NEXT:    s_addc_u32 s7, s7, s4
 ; GFX9-NEXT:    s_xor_b64 s[6:7], s[6:7], s[4:5]
-; GFX9-NEXT:    v_readfirstlane_b32 s10, v1
-; GFX9-NEXT:    s_mul_i32 s9, s6, s8
-; GFX9-NEXT:    s_mul_hi_u32 s11, s6, s10
-; GFX9-NEXT:    s_mul_hi_u32 s5, s6, s8
-; GFX9-NEXT:    s_add_u32 s9, s11, s9
-; GFX9-NEXT:    s_addc_u32 s5, 0, s5
-; GFX9-NEXT:    s_mul_hi_u32 s12, s7, s10
-; GFX9-NEXT:    s_mul_i32 s10, s7, s10
-; GFX9-NEXT:    s_add_u32 s9, s9, s10
-; GFX9-NEXT:    s_mul_hi_u32 s11, s7, s8
-; GFX9-NEXT:    s_addc_u32 s5, s5, s12
-; GFX9-NEXT:    s_addc_u32 s9, s11, 0
+; GFX9-NEXT:    v_readfirstlane_b32 s11, v0
+; GFX9-NEXT:    s_mul_i32 s10, s6, s8
+; GFX9-NEXT:    s_mul_hi_u32 s12, s6, s11
+; GFX9-NEXT:    s_mul_hi_u32 s9, s6, s8
+; GFX9-NEXT:    s_add_u32 s10, s12, s10
+; GFX9-NEXT:    s_addc_u32 s9, 0, s9
+; GFX9-NEXT:    s_mul_hi_u32 s13, s7, s11
+; GFX9-NEXT:    s_mul_i32 s11, s7, s11
+; GFX9-NEXT:    s_add_u32 s10, s10, s11
+; GFX9-NEXT:    s_mul_hi_u32 s12, s7, s8
+; GFX9-NEXT:    s_addc_u32 s9, s9, s13
+; GFX9-NEXT:    s_addc_u32 s10, s12, 0
 ; GFX9-NEXT:    s_mul_i32 s8, s7, s8
-; GFX9-NEXT:    s_add_u32 s5, s5, s8
-; GFX9-NEXT:    s_addc_u32 s8, 0, s9
-; GFX9-NEXT:    s_add_u32 s9, s5, 1
-; GFX9-NEXT:    s_addc_u32 s10, s8, 0
-; GFX9-NEXT:    s_add_u32 s11, s5, 2
-; GFX9-NEXT:    s_mul_i32 s14, s8, 0xfff
-; GFX9-NEXT:    s_mul_hi_u32 s15, s5, 0xfff
-; GFX9-NEXT:    s_addc_u32 s12, s8, 0
-; GFX9-NEXT:    s_add_i32 s15, s15, s14
-; GFX9-NEXT:    s_mul_i32 s14, s5, 0xfff
-; GFX9-NEXT:    v_mov_b32_e32 v1, s14
-; GFX9-NEXT:    v_sub_co_u32_e32 v1, vcc, s6, v1
-; GFX9-NEXT:    s_movk_i32 s13, 0xfff
+; GFX9-NEXT:    s_add_u32 s8, s9, s8
+; GFX9-NEXT:    s_addc_u32 s9, 0, s10
+; GFX9-NEXT:    s_add_u32 s10, s8, 1
+; GFX9-NEXT:    s_addc_u32 s11, s9, 0
+; GFX9-NEXT:    s_add_u32 s12, s8, 2
+; GFX9-NEXT:    s_mul_i32 s15, s9, 0xfff
+; GFX9-NEXT:    s_mul_hi_u32 s16, s8, 0xfff
+; GFX9-NEXT:    s_addc_u32 s13, s9, 0
+; GFX9-NEXT:    s_add_i32 s16, s16, s15
+; GFX9-NEXT:    s_mul_i32 s15, s8, 0xfff
+; GFX9-NEXT:    v_mov_b32_e32 v0, s15
+; GFX9-NEXT:    v_sub_co_u32_e32 v0, vcc, s6, v0
+; GFX9-NEXT:    s_movk_i32 s14, 0xfff
 ; GFX9-NEXT:    s_cmp_lg_u64 vcc, 0
-; GFX9-NEXT:    s_subb_u32 s6, s7, s15
-; GFX9-NEXT:    v_subrev_co_u32_e32 v2, vcc, s13, v1
+; GFX9-NEXT:    s_subb_u32 s6, s7, s16
+; GFX9-NEXT:    v_subrev_co_u32_e32 v1, vcc, s14, v0
 ; GFX9-NEXT:    s_cmp_lg_u64 vcc, 0
 ; GFX9-NEXT:    s_subb_u32 s7, s6, 0
-; GFX9-NEXT:    s_movk_i32 s13, 0xffe
-; GFX9-NEXT:    v_cmp_lt_u32_e32 vcc, s13, v2
+; GFX9-NEXT:    v_readfirstlane_b32 s14, v1
+; GFX9-NEXT:    s_cmpk_gt_u32 s14, 0xffe
+; GFX9-NEXT:    s_cselect_b32 s14, -1, 0
 ; GFX9-NEXT:    s_cmp_eq_u32 s7, 0
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
-; GFX9-NEXT:    s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, -1, v2, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v3, s9
-; GFX9-NEXT:    v_mov_b32_e32 v4, s11
-; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v3, s10
-; GFX9-NEXT:    v_mov_b32_e32 v4, s12
-; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v4, vcc
-; GFX9-NEXT:    v_cmp_lt_u32_e32 vcc, s13, v1
+; GFX9-NEXT:    s_cselect_b32 s7, s14, -1
+; GFX9-NEXT:    s_cmp_lg_u32 s7, 0
+; GFX9-NEXT:    s_cselect_b32 s7, s13, s11
+; GFX9-NEXT:    v_readfirstlane_b32 s11, v0
+; GFX9-NEXT:    s_cselect_b32 s10, s12, s10
+; GFX9-NEXT:    s_cmpk_gt_u32 s11, 0xffe
+; GFX9-NEXT:    s_cselect_b32 s11, -1, 0
 ; GFX9-NEXT:    s_cmp_eq_u32 s6, 0
-; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc
-; GFX9-NEXT:    s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, -1, v1, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v4, s8
-; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v4, v3, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX9-NEXT:    v_xor_b32_e32 v2, s4, v2
-; GFX9-NEXT:    v_xor_b32_e32 v1, s4, v1
-; GFX9-NEXT:    v_mov_b32_e32 v4, s4
-; GFX9-NEXT:    v_subrev_co_u32_e32 v3, vcc, s4, v2
-; GFX9-NEXT:    v_subb_co_u32_e32 v4, vcc, v1, v4, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v1, s0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-NEXT:    global_store_dwordx4 v0, v[1:4], s[2:3]
+; GFX9-NEXT:    s_cselect_b32 s6, s11, -1
+; GFX9-NEXT:    s_cmp_lg_u32 s6, 0
+; GFX9-NEXT:    s_cselect_b32 s7, s7, s9
+; GFX9-NEXT:    s_cselect_b32 s6, s10, s8
+; GFX9-NEXT:    s_xor_b64 s[6:7], s[6:7], s[4:5]
+; GFX9-NEXT:    s_sub_u32 s5, s6, s4
+; GFX9-NEXT:    s_subb_u32 s4, s7, s4
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-NEXT:    v_mov_b32_e32 v3, s4
+; GFX9-NEXT:    global_store_dwordx4 v4, v[0:3], s[2:3]
 ; GFX9-NEXT:    s_endpgm
   %r = sdiv <2 x i64> %x, <i64 4096, i64 4095>
   store <2 x i64> %r, ptr addrspace(1) %out
@@ -9780,66 +9515,43 @@ define amdgpu_kernel void @srem_i64_oddk_denom(ptr addrspace(1) %out, i64 %x) {
 ;
 ; GFX6-LABEL: srem_i64_oddk_denom:
 ; GFX6:       ; %bb.0:
-; GFX6-NEXT:    v_mov_b32_e32 v0, 0x4f800000
-; GFX6-NEXT:    v_madak_f32 v0, 0, v0, 0x4996c7d8
-; GFX6-NEXT:    v_rcp_f32_e32 v0, v0
-; GFX6-NEXT:    s_mov_b32 s8, 0xffed2705
 ; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
-; GFX6-NEXT:    s_mov_b32 s3, 0xf000
-; GFX6-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
-; GFX6-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
-; GFX6-NEXT:    v_trunc_f32_e32 v1, v1
-; GFX6-NEXT:    v_madmk_f32 v0, v1, 0xcf800000, v0
-; GFX6-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GFX6-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX6-NEXT:    s_mov_b32 s2, -1
-; GFX6-NEXT:    v_mul_lo_u32 v2, v1, s8
-; GFX6-NEXT:    v_mul_hi_u32 v3, v0, s8
-; GFX6-NEXT:    v_mul_lo_u32 v4, v0, s8
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
-; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v0
-; GFX6-NEXT:    v_mul_hi_u32 v3, v0, v4
-; GFX6-NEXT:    v_mul_lo_u32 v5, v0, v2
+; GFX6-NEXT:    s_mov_b32 s0, 0x33fe64
+; GFX6-NEXT:    s_add_u32 s0, 0x396, s0
+; GFX6-NEXT:    v_mov_b32_e32 v0, 0x28100000
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
+; GFX6-NEXT:    s_addc_u32 s1, 0, 0
+; GFX6-NEXT:    s_or_b32 s0, vcc_lo, vcc_hi
+; GFX6-NEXT:    s_cmp_lg_u32 s0, 0
+; GFX6-NEXT:    s_mov_b32 s0, 0xffed2705
+; GFX6-NEXT:    v_mul_hi_u32 v1, v0, s0
+; GFX6-NEXT:    s_addc_u32 s1, s1, 0xd95
+; GFX6-NEXT:    v_mul_lo_u32 v2, v0, s0
+; GFX6-NEXT:    s_mul_i32 s8, s1, 0xffed2705
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, s8, v1
+; GFX6-NEXT:    v_sub_i32_e32 v1, vcc, v1, v0
+; GFX6-NEXT:    v_mul_lo_u32 v5, v0, v1
 ; GFX6-NEXT:    v_mul_hi_u32 v6, v0, v2
-; GFX6-NEXT:    v_mul_hi_u32 v7, v1, v2
-; GFX6-NEXT:    v_mul_lo_u32 v2, v1, v2
-; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
-; GFX6-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc
-; GFX6-NEXT:    v_mul_lo_u32 v6, v1, v4
-; GFX6-NEXT:    v_mul_hi_u32 v4, v1, v4
-; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v3, v6
-; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, v5, v4, vcc
-; GFX6-NEXT:    v_addc_u32_e32 v4, vcc, 0, v7, vcc
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc
-; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
-; GFX6-NEXT:    v_mul_lo_u32 v2, v1, s8
-; GFX6-NEXT:    v_mul_hi_u32 v3, v0, s8
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
-; GFX6-NEXT:    v_mul_lo_u32 v3, v0, s8
-; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, v2, v0
-; GFX6-NEXT:    v_mul_lo_u32 v6, v0, v2
-; GFX6-NEXT:    v_mul_hi_u32 v7, v0, v3
-; GFX6-NEXT:    v_mul_hi_u32 v8, v0, v2
-; GFX6-NEXT:    v_mul_hi_u32 v5, v1, v3
-; GFX6-NEXT:    v_mul_lo_u32 v3, v1, v3
-; GFX6-NEXT:    v_mul_hi_u32 v4, v1, v2
-; GFX6-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
-; GFX6-NEXT:    v_addc_u32_e32 v7, vcc, 0, v8, vcc
-; GFX6-NEXT:    v_mul_lo_u32 v2, v1, v2
-; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v6, v3
-; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, v7, v5, vcc
-; GFX6-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
-; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GFX6-NEXT:    v_mul_hi_u32 v7, v0, v1
+; GFX6-NEXT:    v_mul_hi_u32 v4, s1, v2
+; GFX6-NEXT:    v_mul_lo_u32 v2, s1, v2
+; GFX6-NEXT:    v_mul_hi_u32 v3, s1, v1
+; GFX6-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; GFX6-NEXT:    v_addc_u32_e32 v6, vcc, 0, v7, vcc
+; GFX6-NEXT:    v_mul_lo_u32 v1, s1, v1
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v5, v2
+; GFX6-NEXT:    v_addc_u32_e32 v2, vcc, v6, v4, vcc
+; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX6-NEXT:    s_ashr_i32 s8, s7, 31
-; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc
+; GFX6-NEXT:    v_addc_u32_e32 v2, vcc, 0, v3, vcc
 ; GFX6-NEXT:    s_add_u32 s0, s6, s8
-; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GFX6-NEXT:    v_mov_b32_e32 v3, s1
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
 ; GFX6-NEXT:    s_mov_b32 s9, s8
 ; GFX6-NEXT:    s_addc_u32 s1, s7, s8
-; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
+; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v3, v2, vcc
 ; GFX6-NEXT:    s_xor_b64 s[6:7], s[0:1], s[8:9]
 ; GFX6-NEXT:    v_mul_lo_u32 v2, s6, v1
 ; GFX6-NEXT:    v_mul_hi_u32 v3, s6, v0
@@ -9860,7 +9572,8 @@ define amdgpu_kernel void @srem_i64_oddk_denom(ptr addrspace(1) %out, i64 %x) {
 ; GFX6-NEXT:    v_mul_lo_u32 v1, v1, s4
 ; GFX6-NEXT:    v_mul_hi_u32 v2, v0, s4
 ; GFX6-NEXT:    v_mul_lo_u32 v0, v0, s4
-; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
 ; GFX6-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
 ; GFX6-NEXT:    v_mov_b32_e32 v2, s7
 ; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
@@ -9888,48 +9601,20 @@ define amdgpu_kernel void @srem_i64_oddk_denom(ptr addrspace(1) %out, i64 %x) {
 ; GFX6-NEXT:    v_xor_b32_e32 v1, s8, v1
 ; GFX6-NEXT:    v_mov_b32_e32 v2, s8
 ; GFX6-NEXT:    v_subrev_i32_e32 v0, vcc, s8, v0
+; GFX6-NEXT:    s_mov_b32 s1, s5
 ; GFX6-NEXT:    v_subb_u32_e32 v1, vcc, v1, v2, vcc
 ; GFX6-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GFX6-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: srem_i64_oddk_denom:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0x4996c7d8
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0x4f800000
-; GFX9-NEXT:    v_mac_f32_e32 v0, 0, v1
-; GFX9-NEXT:    v_rcp_f32_e32 v0, v0
-; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
-; GFX9-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
-; GFX9-NEXT:    v_trunc_f32_e32 v1, v1
-; GFX9-NEXT:    v_madmk_f32 v0, v1, 0xcf800000, v0
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT:    v_readfirstlane_b32 s4, v1
-; GFX9-NEXT:    v_readfirstlane_b32 s5, v0
-; GFX9-NEXT:    s_mul_hi_u32 s6, s5, 0xffed2705
-; GFX9-NEXT:    s_mul_i32 s7, s4, 0xffed2705
-; GFX9-NEXT:    s_add_i32 s6, s6, s7
-; GFX9-NEXT:    s_sub_i32 s6, s6, s5
-; GFX9-NEXT:    s_mul_i32 s9, s5, 0xffed2705
-; GFX9-NEXT:    s_mul_hi_u32 s7, s5, s6
-; GFX9-NEXT:    s_mul_i32 s8, s5, s6
-; GFX9-NEXT:    s_mul_hi_u32 s5, s5, s9
-; GFX9-NEXT:    s_add_u32 s5, s5, s8
-; GFX9-NEXT:    s_addc_u32 s7, 0, s7
-; GFX9-NEXT:    s_mul_hi_u32 s10, s4, s9
-; GFX9-NEXT:    s_mul_i32 s9, s4, s9
-; GFX9-NEXT:    s_add_u32 s5, s5, s9
-; GFX9-NEXT:    s_mul_hi_u32 s8, s4, s6
-; GFX9-NEXT:    s_addc_u32 s5, s7, s10
-; GFX9-NEXT:    s_addc_u32 s7, s8, 0
-; GFX9-NEXT:    s_mul_i32 s6, s4, s6
-; GFX9-NEXT:    s_add_u32 s5, s5, s6
-; GFX9-NEXT:    s_addc_u32 s6, 0, s7
-; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s5, v0
+; GFX9-NEXT:    s_mov_b32 s4, 0x33fe64
+; GFX9-NEXT:    s_add_u32 s4, 0x396, s4
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0x28100000
+; GFX9-NEXT:    s_addc_u32 s5, 0, 0
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, s4, v0
 ; GFX9-NEXT:    s_cmp_lg_u64 vcc, 0
-; GFX9-NEXT:    s_addc_u32 s4, s4, s6
+; GFX9-NEXT:    s_addc_u32 s4, s5, 0xd95
 ; GFX9-NEXT:    v_readfirstlane_b32 s6, v0
 ; GFX9-NEXT:    s_mul_i32 s5, s4, 0xffed2705
 ; GFX9-NEXT:    s_mul_hi_u32 s7, s6, 0xffed2705
@@ -9944,6 +9629,7 @@ define amdgpu_kernel void @srem_i64_oddk_denom(ptr addrspace(1) %out, i64 %x) {
 ; GFX9-NEXT:    s_mul_i32 s10, s4, s8
 ; GFX9-NEXT:    s_addc_u32 s8, 0, s11
 ; GFX9-NEXT:    s_add_u32 s6, s6, s10
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX9-NEXT:    s_mul_hi_u32 s7, s4, s5
 ; GFX9-NEXT:    s_addc_u32 s6, s8, s9
 ; GFX9-NEXT:    s_addc_u32 s7, s7, 0
@@ -10013,6 +9699,7 @@ define amdgpu_kernel void @srem_i64_oddk_denom(ptr addrspace(1) %out, i64 %x) {
 ; GFX9-NEXT:    v_xor_b32_e32 v1, s4, v3
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s4
 ; GFX9-NEXT:    v_subrev_co_u32_e32 v0, vcc, s4, v0
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v3, vcc
 ; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
 ; GFX9-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-f16.mir b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-f16.mir
new file mode 100644
index 0000000000..5eafe8b261
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-f16.mir
@@ -0,0 +1,58 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 3
+# RUN: llc -march=amdgcn -mcpu=gfx1100 -run-pass=si-fix-sgpr-copies -verify-machineinstrs -o - %s | FileCheck --check-prefix=GCN %s
+
+---
+name:            cmp_f16
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: cmp_f16
+    ; GCN: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[V_CVT_F16_U16_e64_:%[0-9]+]]:vgpr_32 = V_CVT_F16_U16_e64 [[DEF]], 0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: [[DEF2:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[V_CMP_LT_F16_t16_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_LT_F16_t16_e64 0, [[V_CVT_F16_U16_e64_]], 0, [[DEF1]], 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, -1, killed [[V_CMP_LT_F16_t16_e64_]], implicit $exec
+    %0:vgpr_32 = IMPLICIT_DEF
+    %1:sreg_32 = IMPLICIT_DEF
+    %2:vgpr_32 = V_CVT_F16_U16_e64 %0:vgpr_32, 0, 0, implicit $mode, implicit $exec
+    %3:sreg_32 = COPY %2:vgpr_32
+    nofpexcept S_CMP_LT_F16 killed %3:sreg_32, %1:sreg_32, implicit-def $scc, implicit $mode
+    %4:sreg_32_xm0_xexec = COPY $scc
+    %5:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, -1, killed %4, implicit $exec
+...
+
+# Needs extra shift instruction to select hi 16 bits
+---
+name:            cvt_hi_f32_f16
+body:             |
+  bb.0:
+    ; GCN-LABEL: name: cvt_hi_f32_f16
+    ; GCN: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[V_CVT_F16_U16_e64_:%[0-9]+]]:vgpr_32 = V_CVT_F16_U16_e64 [[DEF]], 0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 16, [[V_CVT_F16_U16_e64_]], implicit $exec
+    ; GCN-NEXT: [[V_CVT_F32_F16_t16_e64_:%[0-9]+]]:vgpr_32 = V_CVT_F32_F16_t16_e64 0, [[V_LSHRREV_B32_e64_]], 0, 0, implicit $mode, implicit $exec
+    %0:vgpr_32 = IMPLICIT_DEF
+    %1:vgpr_32 = V_CVT_F16_U16_e64 %0:vgpr_32, 0, 0, implicit $mode, implicit $exec
+    %2:sreg_32 = COPY %1:vgpr_32
+    %3:sreg_32 = S_CVT_HI_F32_F16 %2:sreg_32, implicit $mode
+...
+
+---
+name:            fmac_f16
+body:             |
+  bb.0:
+    ; GCN-LABEL: name: fmac_f16
+    ; GCN: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF2:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[V_CVT_F32_U32_e64_:%[0-9]+]]:vgpr_32 = V_CVT_F32_U32_e64 [[DEF]], 0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: [[DEF3:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[V_FMAC_F16_t16_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_FMAC_F16_t16_e64 0, killed [[DEF1]], 0, [[DEF2]], 0, [[V_CVT_F32_U32_e64_]], 0, 0, 0, implicit $mode, implicit $exec
+    %0:vgpr_32 = IMPLICIT_DEF
+    %1:sreg_32 = IMPLICIT_DEF
+    %2:sreg_32 = IMPLICIT_DEF
+    %3:vgpr_32 = V_CVT_F32_U32_e64 %0:vgpr_32, 0, 0, implicit $mode, implicit $exec
+    %4:sreg_32 = COPY %3:vgpr_32
+    %5:sreg_32 = nofpexcept S_FMAC_F16 killed %1:sreg_32, %2:sreg_32, %4:sreg_32, implicit $mode
+...
diff --git a/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies.mir b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies.mir
index ff02c6ba69..11cabedd27 100644
--- a/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies.mir
+++ b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies.mir
@@ -202,44 +202,6 @@ body:             |
     %5:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, -1, killed %4, implicit $exec
 ...
 
----
-name:            cmp_f16
-body:             |
-  bb.0.entry:
-    ; GCN-LABEL: name: cmp_f16
-    ; GCN: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
-    ; GCN-NEXT: [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
-    ; GCN-NEXT: [[V_CVT_F16_U16_e64_:%[0-9]+]]:vgpr_32 = V_CVT_F16_U16_e64 [[DEF]], 0, 0, implicit $mode, implicit $exec
-    ; GCN-NEXT: [[DEF2:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
-    ; GCN-NEXT: [[V_CMP_LT_F16_t16_e64_:%[0-9]+]]:sreg_64_xexec = nofpexcept V_CMP_LT_F16_t16_e64 0, [[V_CVT_F16_U16_e64_]], 0, [[DEF1]], 0, implicit $mode, implicit $exec
-    ; GCN-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, -1, killed [[V_CMP_LT_F16_t16_e64_]], implicit $exec
-    %0:vgpr_32 = IMPLICIT_DEF
-    %1:sreg_32 = IMPLICIT_DEF
-    %2:vgpr_32 = V_CVT_F16_U16_e64 %0:vgpr_32, 0, 0, implicit $mode, implicit $exec
-    %3:sreg_32 = COPY %2:vgpr_32
-    nofpexcept S_CMP_LT_F16 killed %3:sreg_32, %1:sreg_32, implicit-def $scc, implicit $mode
-    %4:sreg_64_xexec = COPY $scc
-    %5:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, -1, killed %4, implicit $exec
-...
-
-# Needs extra shift instruction to select hi 16 bits
----
-name:            cvt_hi_f32_f16
-body:             |
-  bb.0:
-
-    ; GCN-LABEL: name: cvt_hi_f32_f16
-    ; GCN: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
-    ; GCN-NEXT: [[V_CVT_F16_U16_e64_:%[0-9]+]]:vgpr_32 = V_CVT_F16_U16_e64 [[DEF]], 0, 0, implicit $mode, implicit $exec
-    ; GCN-NEXT: [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
-    ; GCN-NEXT: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 16, [[V_CVT_F16_U16_e64_]], implicit $exec
-    ; GCN-NEXT: [[V_CVT_F32_F16_e32_:%[0-9]+]]:vgpr_32 = V_CVT_F32_F16_t16_e64 0, [[V_LSHRREV_B32_e64_]], 0, 0, implicit $mode, implicit $exec
-    %0:vgpr_32 = IMPLICIT_DEF
-    %1:vgpr_32 = V_CVT_F16_U16_e64 %0:vgpr_32, 0, 0, implicit $mode, implicit $exec
-    %2:sreg_32 = COPY %1:vgpr_32
-    %3:sreg_32 = S_CVT_HI_F32_F16 %2:sreg_32, implicit $mode
-...
-
 # Test to ensure that src2 of fmac is moved to vgpr
 ---
 name:            fmac_f32
@@ -260,23 +222,3 @@ body:             |
     %4:sreg_32 = COPY %3:vgpr_32
     %5:sreg_32 = nofpexcept S_FMAC_F32 killed %4:sreg_32, %1:sreg_32, %2:sreg_32, implicit $mode
 ...
-
----
-name:            fmac_f16
-body:             |
-  bb.0:
-    ; GCN-LABEL: name: fmac_f16
-    ; GCN: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
-    ; GCN-NEXT: [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
-    ; GCN-NEXT: [[DEF2:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
-    ; GCN-NEXT: [[V_CVT_F32_U32_e64_:%[0-9]+]]:vgpr_32 = V_CVT_F32_U32_e64 [[DEF]], 0, 0, implicit $mode, implicit $exec
-    ; GCN-NEXT: [[DEF3:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
-    ; GCN-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY [[DEF2]]
-    ; GCN-NEXT: %6:vgpr_32 = nofpexcept V_FMAC_F16_t16_e64 0, killed [[DEF1]], 0, [[COPY]], 0, [[V_CVT_F32_U32_e64_]], 0, 0, 0, implicit $mode, implicit $exec
-    %0:vgpr_32 = IMPLICIT_DEF
-    %1:sreg_32 = IMPLICIT_DEF
-    %2:sreg_32 = IMPLICIT_DEF
-    %3:vgpr_32 = V_CVT_F32_U32_e64 %0:vgpr_32, 0, 0, implicit $mode, implicit $exec
-    %4:sreg_32 = COPY %3:vgpr_32
-    %5:sreg_32 = nofpexcept S_FMAC_F16 killed %1:sreg_32, %2:sreg_32, %4:sreg_32, implicit $mode
-...
diff --git a/llvm/test/CodeGen/AMDGPU/fold-short-64-bit-literals.mir b/llvm/test/CodeGen/AMDGPU/fold-short-64-bit-literals.mir
new file mode 100644
index 0000000000..6e975c8a53
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/fold-short-64-bit-literals.mir
@@ -0,0 +1,134 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 3
+# RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -run-pass=si-fold-operands -o - %s | FileCheck --check-prefix=GCN %s
+
+---
+name:            no_fold_fp_64bit_literal_sgpr
+tracksRegLiveness: true
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: no_fold_fp_64bit_literal_sgpr
+    ; GCN: [[DEF:%[0-9]+]]:vreg_64 = IMPLICIT_DEF
+    ; GCN-NEXT: [[S_MOV_B:%[0-9]+]]:sreg_64 = S_MOV_B64_IMM_PSEUDO 1311768467750121200
+    ; GCN-NEXT: [[V_ADD_F64_e64_:%[0-9]+]]:vreg_64 = V_ADD_F64_e64 0, [[S_MOV_B]], 0, [[DEF]], 0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: SI_RETURN_TO_EPILOG [[V_ADD_F64_e64_]]
+    %0:vreg_64 = IMPLICIT_DEF
+    %1:sreg_64 = S_MOV_B64_IMM_PSEUDO 1311768467750121200
+    %2:vreg_64 = V_ADD_F64_e64 0, %1, 0, %0, 0, 0, implicit $mode, implicit $exec
+    SI_RETURN_TO_EPILOG %2
+...
+
+---
+name:            no_fold_fp_64bit_literal_vgpr
+tracksRegLiveness: true
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: no_fold_fp_64bit_literal_vgpr
+    ; GCN: [[DEF:%[0-9]+]]:vreg_64 = IMPLICIT_DEF
+    ; GCN-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 1311768467750121200, implicit $exec
+    ; GCN-NEXT: [[V_ADD_F64_e64_:%[0-9]+]]:vreg_64 = V_ADD_F64_e64 0, [[V_MOV_B]], 0, [[DEF]], 0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: SI_RETURN_TO_EPILOG [[V_ADD_F64_e64_]]
+    %0:vreg_64 = IMPLICIT_DEF
+    %1:vreg_64 = V_MOV_B64_PSEUDO 1311768467750121200, implicit $exec
+    %2:vreg_64 = V_ADD_F64_e64 0, %1, 0, %0, 0, 0, implicit $mode, implicit $exec
+    SI_RETURN_TO_EPILOG %2
+...
+
+---
+name:            fold_fp_32bit_literal_sgpr
+tracksRegLiveness: true
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: fold_fp_32bit_literal_sgpr
+    ; GCN: [[DEF:%[0-9]+]]:vreg_64 = IMPLICIT_DEF
+    ; GCN-NEXT: [[V_ADD_F64_e64_:%[0-9]+]]:vreg_64 = V_ADD_F64_e64 0, 4636737291354636288, 0, [[DEF]], 0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: SI_RETURN_TO_EPILOG [[V_ADD_F64_e64_]]
+    %0:vreg_64 = IMPLICIT_DEF
+    %1:sreg_64 = S_MOV_B64_IMM_PSEUDO 4636737291354636288
+    %2:vreg_64 = V_ADD_F64_e64 0, %1, 0, %0, 0, 0, implicit $mode, implicit $exec
+    SI_RETURN_TO_EPILOG %2
+...
+
+---
+name:            no_fold_int_64bit_literal_sgpr
+tracksRegLiveness: true
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: no_fold_int_64bit_literal_sgpr
+    ; GCN: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+    ; GCN-NEXT: [[S_MOV_B:%[0-9]+]]:sreg_64 = S_MOV_B64_IMM_PSEUDO 1311768467750121200
+    ; GCN-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64 = S_AND_B64 [[DEF]], [[S_MOV_B]], implicit-def $scc
+    ; GCN-NEXT: SI_RETURN_TO_EPILOG [[S_AND_B64_]]
+    %0:sreg_64 = IMPLICIT_DEF
+    %1:sreg_64 = S_MOV_B64_IMM_PSEUDO 1311768467750121200
+    %2:sreg_64 = S_AND_B64 %0, %1, implicit-def $scc
+    SI_RETURN_TO_EPILOG %2
+...
+
+---
+name:            fold_int_32bit_literal_sgpr
+tracksRegLiveness: true
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: fold_int_32bit_literal_sgpr
+    ; GCN: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+    ; GCN-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64 = S_AND_B64 [[DEF]], 2147483647, implicit-def $scc
+    ; GCN-NEXT: SI_RETURN_TO_EPILOG [[S_AND_B64_]]
+    %0:sreg_64 = IMPLICIT_DEF
+    %1:sreg_64 = S_MOV_B64 2147483647
+    %2:sreg_64 = S_AND_B64 %0, %1, implicit-def $scc
+    SI_RETURN_TO_EPILOG %2
+...
+
+---
+name:            fold_uint_32bit_literal_sgpr
+tracksRegLiveness: true
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: fold_uint_32bit_literal_sgpr
+    ; GCN: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+    ; GCN-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64 = S_AND_B64 [[DEF]], 4294967295, implicit-def $scc
+    ; GCN-NEXT: SI_RETURN_TO_EPILOG [[S_AND_B64_]]
+    %0:sreg_64 = IMPLICIT_DEF
+    %1:sreg_64 = S_MOV_B64 4294967295
+    %2:sreg_64 = S_AND_B64 %0, %1, implicit-def $scc
+    SI_RETURN_TO_EPILOG %2
+...
+
+---
+name:            no_fold_v2fp_64bit_literal_sgpr
+tracksRegLiveness: true
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: no_fold_v2fp_64bit_literal_sgpr
+    ; GCN: [[DEF:%[0-9]+]]:vreg_64 = IMPLICIT_DEF
+    ; GCN-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 4629700418019000320, implicit $exec
+    ; GCN-NEXT: [[V_PK_ADD_F32_:%[0-9]+]]:vreg_64 = V_PK_ADD_F32 0, [[DEF]], 0, [[V_MOV_B]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: SI_RETURN_TO_EPILOG [[V_PK_ADD_F32_]]
+    %0:vreg_64 = IMPLICIT_DEF
+    %1:vreg_64 = V_MOV_B64_PSEUDO 4629700418019000320, implicit $exec
+    %2:vreg_64 = V_PK_ADD_F32 0, %0, 0, %1, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+    SI_RETURN_TO_EPILOG %2
+...
+
+---
+name:            fold_v2fp_32bit_literal_sgpr
+tracksRegLiveness: true
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: fold_v2fp_32bit_literal_sgpr
+    ; GCN: [[DEF:%[0-9]+]]:vreg_64 = IMPLICIT_DEF
+    ; GCN-NEXT: [[V_PK_ADD_F32_:%[0-9]+]]:vreg_64 = V_PK_ADD_F32 0, [[DEF]], 0, 1065353216, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: SI_RETURN_TO_EPILOG [[V_PK_ADD_F32_]]
+    %0:vreg_64 = IMPLICIT_DEF
+    %1:vreg_64 = V_MOV_B64_PSEUDO 1065353216, implicit $exec
+    %2:vreg_64 = V_PK_ADD_F32 0, %0, 0, %1, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+    SI_RETURN_TO_EPILOG %2
+...
diff --git a/llvm/test/CodeGen/AMDGPU/ipra-return-address-save-restore.ll b/llvm/test/CodeGen/AMDGPU/ipra-return-address-save-restore.ll
index ef3c95b175..741164bc04 100644
--- a/llvm/test/CodeGen/AMDGPU/ipra-return-address-save-restore.ll
+++ b/llvm/test/CodeGen/AMDGPU/ipra-return-address-save-restore.ll
@@ -28,12 +28,10 @@ declare void @llvm.lifetime.end.p5(i64 immarg, ptr addrspace(5) nocapture) #1
 ; Function Attrs: norecurse
 define internal fastcc void @svm_node_closure_bsdf(ptr addrspace(1) %sd, ptr %stack, <4 x i32> %node, ptr %offset, i32 %0, i8 %trunc, float %1, float %2, float %mul80, i1 %cmp412.old, <4 x i32> %3, float %4, i32 %5, i1 %cmp440, i1 %cmp442, i1 %or.cond1306, float %.op, ptr addrspace(1) %arrayidx.i.i2202, ptr addrspace(1) %retval.0.i.i22089, ptr addrspace(1) %retval.1.i221310, i1 %cmp575, ptr addrspace(1) %num_closure_left.i2215, i32 %6, i1 %cmp.i2216, i32 %7, i64 %idx.ext.i2223, i32 %sub5.i2221) #2 {
 ; GCN-LABEL: {{^}}svm_node_closure_bsdf:
-; GCN-DAG: v_writelane_b32 [[CSR_VGPR:v[0-9]+]], s30,
-; GCN-DAG: v_writelane_b32 [[CSR_VGPR]], s31,
-; GCN: s_movk_i32 s30, 0x60
+; GCN-NOT: v_writelane_b32
+; GCN: s_movk_i32 s28, 0x60
 ; GCN-NOT: s31
-; GCN-DAG: v_readlane_b32 s31, [[CSR_VGPR]],
-; GCN-DAG: v_readlane_b32 s30, [[CSR_VGPR]],
+; GCN-NOT: v_readlane_b32
 ; GCN: s_waitcnt vmcnt(0)
 ; GCN: s_setpc_b64 s[30:31]
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.csub.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.csub.ll
index e96a3545cb..a046179636 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.csub.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.csub.ll
@@ -4,35 +4,70 @@
 declare i32 @llvm.amdgcn.buffer.atomic.csub(i32, <4 x i32>, i32, i32, i1)
 declare i32 @llvm.amdgcn.global.atomic.csub(ptr addrspace(1), i32)
 
-; GCN-LABEL: {{^}}buffer_atomic_csub:
+; GCN-LABEL: {{^}}buffer_atomic_csub_rtn:
 ; GCN: buffer_atomic_csub v0, v1, s[0:3], 0 idxen glc
-define amdgpu_ps void @buffer_atomic_csub(<4 x i32> inreg %rsrc, i32 %data, i32 %vindex) {
+define amdgpu_ps void @buffer_atomic_csub_rtn(<4 x i32> inreg %rsrc, i32 %data, i32 %vindex) {
 main_body:
   %ret = call i32 @llvm.amdgcn.buffer.atomic.csub(i32 %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i1 0)
   ret void
 }
 
-; GCN-LABEL: {{^}}buffer_atomic_csub_off4_slc:
+; GCN-LABEL: {{^}}buffer_atomic_csub_no_rtn:
+; GCN: buffer_atomic_csub v0, v1, s[0:3], 0 idxen
+define amdgpu_ps void @buffer_atomic_csub_no_rtn(<4 x i32> inreg %rsrc, i32 %data, i32 %vindex) #0 {
+main_body:
+  %ret = call i32 @llvm.amdgcn.buffer.atomic.csub(i32 %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i1 0)
+  ret void
+}
+
+; GCN-LABEL: {{^}}buffer_atomic_csub_off4_slc_rtn:
 ; GCN: buffer_atomic_csub v0, v1, s[0:3], 0 idxen offset:4 glc slc
-define amdgpu_ps void @buffer_atomic_csub_off4_slc(<4 x i32> inreg %rsrc, i32 %data, i32 %vindex) {
+define amdgpu_ps void @buffer_atomic_csub_off4_slc_rtn(<4 x i32> inreg %rsrc, i32 %data, i32 %vindex) {
 main_body:
   %ret = call i32 @llvm.amdgcn.buffer.atomic.csub(i32 %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i1 1)
   ret void
 }
 
-; GCN-LABEL: {{^}}global_atomic_csub:
+; GCN-LABEL: {{^}}buffer_atomic_csub_off4_slc_no_rtn:
+; GCN: buffer_atomic_csub v0, v1, s[0:3], 0 idxen offset:4 slc
+define amdgpu_ps void @buffer_atomic_csub_off4_slc_no_rtn(<4 x i32> inreg %rsrc, i32 %data, i32 %vindex) #0 {
+main_body:
+  %ret = call i32 @llvm.amdgcn.buffer.atomic.csub(i32 %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i1 1)
+  ret void
+}
+
+; GCN-LABEL: {{^}}global_atomic_csub_rtn:
 ; GCN: global_atomic_csub v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9:]+}}, s{{\[[0-9]+:[0-9]+\]}} glc
-define amdgpu_kernel void @global_atomic_csub(ptr addrspace(1) %ptr, i32 %data) {
+define amdgpu_kernel void @global_atomic_csub_rtn(ptr addrspace(1) %ptr, i32 %data) {
+main_body:
+  %ret = call i32 @llvm.amdgcn.global.atomic.csub(ptr addrspace(1) %ptr, i32 %data)
+  ret void
+}
+
+; GCN-LABEL: {{^}}global_atomic_csub_no_rtn:
+; GCN: global_atomic_csub v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}
+define amdgpu_kernel void @global_atomic_csub_no_rtn(ptr addrspace(1) %ptr, i32 %data) #0 {
 main_body:
   %ret = call i32 @llvm.amdgcn.global.atomic.csub(ptr addrspace(1) %ptr, i32 %data)
   ret void
 }
 
-; GCN-LABEL: {{^}}global_atomic_csub_off4:
+; GCN-LABEL: {{^}}global_atomic_csub_off4_rtn:
 ; GCN: global_atomic_csub v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}} offset:4 glc
-define amdgpu_kernel void @global_atomic_csub_off4(ptr addrspace(1) %ptr, i32 %data) {
+define amdgpu_kernel void @global_atomic_csub_off4_rtn(ptr addrspace(1) %ptr, i32 %data) {
 main_body:
   %p = getelementptr i32, ptr addrspace(1) %ptr, i64 1
   %ret = call i32 @llvm.amdgcn.global.atomic.csub(ptr addrspace(1) %p, i32 %data)
   ret void
 }
+
+; GCN-LABEL: {{^}}global_atomic_csub_off4_no_rtn:
+; GCN: global_atomic_csub v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}} offset:4
+define amdgpu_kernel void @global_atomic_csub_off4_no_rtn(ptr addrspace(1) %ptr, i32 %data) #0 {
+main_body:
+  %p = getelementptr i32, ptr addrspace(1) %ptr, i64 1
+  %ret = call i32 @llvm.amdgcn.global.atomic.csub(ptr addrspace(1) %p, i32 %data)
+  ret void
+}
+
+attributes #0 = { "target-features"="+atomic-csub-no-rtn-insts" }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w32.ll
index 007b52fa3a..5a950d803e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w32.ll
@@ -494,6 +494,121 @@ define amdgpu_kernel void @v_fcmp_f32_ole(ptr addrspace(1) %out, float %src) {
   ret void
 }
 
+define amdgpu_kernel void @v_fcmp_f32_o(ptr addrspace(1) %out, float %src) {
+; SDAG-GFX11-LABEL: v_fcmp_f32_o:
+; SDAG-GFX11:       ; %bb.0:
+; SDAG-GFX11-NEXT:    s_clause 0x1
+; SDAG-GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; SDAG-GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; SDAG-GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; SDAG-GFX11-NEXT:    v_cmp_o_f32_e64 s2, 0x42c80000, s2
+; SDAG-GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; SDAG-GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; SDAG-GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
+; SDAG-GFX11-NEXT:    s_nop 0
+; SDAG-GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; SDAG-GFX11-NEXT:    s_endpgm
+;
+; SDAG-GFX10-LABEL: v_fcmp_f32_o:
+; SDAG-GFX10:       ; %bb.0:
+; SDAG-GFX10-NEXT:    s_clause 0x1
+; SDAG-GFX10-NEXT:    s_load_dword s4, s[0:1], 0x2c
+; SDAG-GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; SDAG-GFX10-NEXT:    v_mov_b32_e32 v0, 0
+; SDAG-GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; SDAG-GFX10-NEXT:    v_cmp_o_f32_e64 s0, 0x42c80000, s4
+; SDAG-GFX10-NEXT:    v_mov_b32_e32 v1, s0
+; SDAG-GFX10-NEXT:    global_store_dword v0, v1, s[2:3]
+; SDAG-GFX10-NEXT:    s_endpgm
+;
+; GISEL-GFX11-LABEL: v_fcmp_f32_o:
+; GISEL-GFX11:       ; %bb.0:
+; GISEL-GFX11-NEXT:    s_clause 0x1
+; GISEL-GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; GISEL-GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GISEL-GFX11-NEXT:    v_mov_b32_e32 v1, 0
+; GISEL-GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT:    v_cmp_o_f32_e64 s2, 0x42c80000, s2
+; GISEL-GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GISEL-GFX11-NEXT:    v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GISEL-GFX11-NEXT:    s_nop 0
+; GISEL-GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GISEL-GFX11-NEXT:    s_endpgm
+;
+; GISEL-GFX10-LABEL: v_fcmp_f32_o:
+; GISEL-GFX10:       ; %bb.0:
+; GISEL-GFX10-NEXT:    s_clause 0x1
+; GISEL-GFX10-NEXT:    s_load_dword s4, s[0:1], 0x2c
+; GISEL-GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; GISEL-GFX10-NEXT:    v_mov_b32_e32 v1, 0
+; GISEL-GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GISEL-GFX10-NEXT:    v_cmp_o_f32_e64 s0, 0x42c80000, s4
+; GISEL-GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GISEL-GFX10-NEXT:    global_store_dword v1, v0, s[2:3]
+; GISEL-GFX10-NEXT:    s_endpgm
+  %result = call i32 @llvm.amdgcn.fcmp.f32(float %src, float 100.00, i32 7)
+  store i32 %result, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @v_fcmp_f32_uo(ptr addrspace(1) %out, float %src) {
+; SDAG-GFX11-LABEL: v_fcmp_f32_uo:
+; SDAG-GFX11:       ; %bb.0:
+; SDAG-GFX11-NEXT:    s_clause 0x1
+; SDAG-GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; SDAG-GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; SDAG-GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; SDAG-GFX11-NEXT:    v_cmp_u_f32_e64 s2, 0x42c80000, s2
+; SDAG-GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; SDAG-GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; SDAG-GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
+; SDAG-GFX11-NEXT:    s_nop 0
+; SDAG-GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; SDAG-GFX11-NEXT:    s_endpgm
+;
+; SDAG-GFX10-LABEL: v_fcmp_f32_uo:
+; SDAG-GFX10:       ; %bb.0:
+; SDAG-GFX10-NEXT:    s_clause 0x1
+; SDAG-GFX10-NEXT:    s_load_dword s4, s[0:1], 0x2c
+; SDAG-GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; SDAG-GFX10-NEXT:    v_mov_b32_e32 v0, 0
+; SDAG-GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; SDAG-GFX10-NEXT:    v_cmp_u_f32_e64 s0, 0x42c80000, s4
+; SDAG-GFX10-NEXT:    v_mov_b32_e32 v1, s0
+; SDAG-GFX10-NEXT:    global_store_dword v0, v1, s[2:3]
+; SDAG-GFX10-NEXT:    s_endpgm
+;
+; GISEL-GFX11-LABEL: v_fcmp_f32_uo:
+; GISEL-GFX11:       ; %bb.0:
+; GISEL-GFX11-NEXT:    s_clause 0x1
+; GISEL-GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; GISEL-GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GISEL-GFX11-NEXT:    v_mov_b32_e32 v1, 0
+; GISEL-GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT:    v_cmp_u_f32_e64 s2, 0x42c80000, s2
+; GISEL-GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GISEL-GFX11-NEXT:    v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GISEL-GFX11-NEXT:    s_nop 0
+; GISEL-GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GISEL-GFX11-NEXT:    s_endpgm
+;
+; GISEL-GFX10-LABEL: v_fcmp_f32_uo:
+; GISEL-GFX10:       ; %bb.0:
+; GISEL-GFX10-NEXT:    s_clause 0x1
+; GISEL-GFX10-NEXT:    s_load_dword s4, s[0:1], 0x2c
+; GISEL-GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; GISEL-GFX10-NEXT:    v_mov_b32_e32 v1, 0
+; GISEL-GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GISEL-GFX10-NEXT:    v_cmp_u_f32_e64 s0, 0x42c80000, s4
+; GISEL-GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GISEL-GFX10-NEXT:    global_store_dword v1, v0, s[2:3]
+; GISEL-GFX10-NEXT:    s_endpgm
+  %result = call i32 @llvm.amdgcn.fcmp.f32(float %src, float 100.00, i32 8)
+  store i32 %result, ptr addrspace(1) %out
+  ret void
+}
 
 define amdgpu_kernel void @v_fcmp_f32_ueq(ptr addrspace(1) %out, float %src) {
 ; SDAG-GFX11-LABEL: v_fcmp_f32_ueq:
@@ -1249,6 +1364,122 @@ define amdgpu_kernel void @v_fcmp_f64_ueq(ptr addrspace(1) %out, double %src) {
   ret void
 }
 
+define amdgpu_kernel void @v_fcmp_f64_o(ptr addrspace(1) %out, double %src) {
+; SDAG-GFX11-LABEL: v_fcmp_f64_o:
+; SDAG-GFX11:       ; %bb.0:
+; SDAG-GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; SDAG-GFX11-NEXT:    s_mov_b32 s4, 0
+; SDAG-GFX11-NEXT:    s_mov_b32 s5, 0x40590000
+; SDAG-GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; SDAG-GFX11-NEXT:    v_cmp_o_f64_e64 s2, s[2:3], s[4:5]
+; SDAG-GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; SDAG-GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; SDAG-GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
+; SDAG-GFX11-NEXT:    s_nop 0
+; SDAG-GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; SDAG-GFX11-NEXT:    s_endpgm
+;
+; SDAG-GFX10-LABEL: v_fcmp_f64_o:
+; SDAG-GFX10:       ; %bb.0:
+; SDAG-GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; SDAG-GFX10-NEXT:    s_mov_b32 s4, 0
+; SDAG-GFX10-NEXT:    s_mov_b32 s5, 0x40590000
+; SDAG-GFX10-NEXT:    v_mov_b32_e32 v0, 0
+; SDAG-GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; SDAG-GFX10-NEXT:    v_cmp_o_f64_e64 s2, s[2:3], s[4:5]
+; SDAG-GFX10-NEXT:    v_mov_b32_e32 v1, s2
+; SDAG-GFX10-NEXT:    global_store_dword v0, v1, s[0:1]
+; SDAG-GFX10-NEXT:    s_endpgm
+;
+; GISEL-GFX11-LABEL: v_fcmp_f64_o:
+; GISEL-GFX11:       ; %bb.0:
+; GISEL-GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GISEL-GFX11-NEXT:    s_mov_b32 s4, 0
+; GISEL-GFX11-NEXT:    s_mov_b32 s5, 0x40590000
+; GISEL-GFX11-NEXT:    v_mov_b32_e32 v1, 0
+; GISEL-GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT:    v_cmp_o_f64_e64 s2, s[2:3], s[4:5]
+; GISEL-GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GISEL-GFX11-NEXT:    v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GISEL-GFX11-NEXT:    s_nop 0
+; GISEL-GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GISEL-GFX11-NEXT:    s_endpgm
+;
+; GISEL-GFX10-LABEL: v_fcmp_f64_o:
+; GISEL-GFX10:       ; %bb.0:
+; GISEL-GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GISEL-GFX10-NEXT:    s_mov_b32 s4, 0
+; GISEL-GFX10-NEXT:    s_mov_b32 s5, 0x40590000
+; GISEL-GFX10-NEXT:    v_mov_b32_e32 v1, 0
+; GISEL-GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GISEL-GFX10-NEXT:    v_cmp_o_f64_e64 s2, s[2:3], s[4:5]
+; GISEL-GFX10-NEXT:    v_mov_b32_e32 v0, s2
+; GISEL-GFX10-NEXT:    global_store_dword v1, v0, s[0:1]
+; GISEL-GFX10-NEXT:    s_endpgm
+  %result = call i32 @llvm.amdgcn.fcmp.f64(double %src, double 100.00, i32 7)
+  store i32 %result, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @v_fcmp_f64_uo(ptr addrspace(1) %out, double %src) {
+; SDAG-GFX11-LABEL: v_fcmp_f64_uo:
+; SDAG-GFX11:       ; %bb.0:
+; SDAG-GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; SDAG-GFX11-NEXT:    s_mov_b32 s4, 0
+; SDAG-GFX11-NEXT:    s_mov_b32 s5, 0x40590000
+; SDAG-GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; SDAG-GFX11-NEXT:    v_cmp_u_f64_e64 s2, s[2:3], s[4:5]
+; SDAG-GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; SDAG-GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; SDAG-GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
+; SDAG-GFX11-NEXT:    s_nop 0
+; SDAG-GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; SDAG-GFX11-NEXT:    s_endpgm
+;
+; SDAG-GFX10-LABEL: v_fcmp_f64_uo:
+; SDAG-GFX10:       ; %bb.0:
+; SDAG-GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; SDAG-GFX10-NEXT:    s_mov_b32 s4, 0
+; SDAG-GFX10-NEXT:    s_mov_b32 s5, 0x40590000
+; SDAG-GFX10-NEXT:    v_mov_b32_e32 v0, 0
+; SDAG-GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; SDAG-GFX10-NEXT:    v_cmp_u_f64_e64 s2, s[2:3], s[4:5]
+; SDAG-GFX10-NEXT:    v_mov_b32_e32 v1, s2
+; SDAG-GFX10-NEXT:    global_store_dword v0, v1, s[0:1]
+; SDAG-GFX10-NEXT:    s_endpgm
+;
+; GISEL-GFX11-LABEL: v_fcmp_f64_uo:
+; GISEL-GFX11:       ; %bb.0:
+; GISEL-GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GISEL-GFX11-NEXT:    s_mov_b32 s4, 0
+; GISEL-GFX11-NEXT:    s_mov_b32 s5, 0x40590000
+; GISEL-GFX11-NEXT:    v_mov_b32_e32 v1, 0
+; GISEL-GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT:    v_cmp_u_f64_e64 s2, s[2:3], s[4:5]
+; GISEL-GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GISEL-GFX11-NEXT:    v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GISEL-GFX11-NEXT:    s_nop 0
+; GISEL-GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GISEL-GFX11-NEXT:    s_endpgm
+;
+; GISEL-GFX10-LABEL: v_fcmp_f64_uo:
+; GISEL-GFX10:       ; %bb.0:
+; GISEL-GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GISEL-GFX10-NEXT:    s_mov_b32 s4, 0
+; GISEL-GFX10-NEXT:    s_mov_b32 s5, 0x40590000
+; GISEL-GFX10-NEXT:    v_mov_b32_e32 v1, 0
+; GISEL-GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GISEL-GFX10-NEXT:    v_cmp_u_f64_e64 s2, s[2:3], s[4:5]
+; GISEL-GFX10-NEXT:    v_mov_b32_e32 v0, s2
+; GISEL-GFX10-NEXT:    global_store_dword v1, v0, s[0:1]
+; GISEL-GFX10-NEXT:    s_endpgm
+  %result = call i32 @llvm.amdgcn.fcmp.f64(double %src, double 100.00, i32 8)
+  store i32 %result, ptr addrspace(1) %out
+  ret void
+}
+
 define amdgpu_kernel void @v_fcmp_f64_une(ptr addrspace(1) %out, double %src) {
 ; SDAG-GFX11-LABEL: v_fcmp_f64_une:
 ; SDAG-GFX11:       ; %bb.0:
@@ -2348,6 +2579,121 @@ define amdgpu_kernel void @v_fcmp_f16_ult(ptr addrspace(1) %out, half %src) {
   ret void
 }
 
+define amdgpu_kernel void @v_fcmp_f16_o(ptr addrspace(1) %out, half %src) {
+; SDAG-GFX11-LABEL: v_fcmp_f16_o:
+; SDAG-GFX11:       ; %bb.0:
+; SDAG-GFX11-NEXT:    s_clause 0x1
+; SDAG-GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; SDAG-GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; SDAG-GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; SDAG-GFX11-NEXT:    v_cmp_o_f16_e64 s2, 0x5640, s2
+; SDAG-GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; SDAG-GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; SDAG-GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
+; SDAG-GFX11-NEXT:    s_nop 0
+; SDAG-GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; SDAG-GFX11-NEXT:    s_endpgm
+;
+; SDAG-GFX10-LABEL: v_fcmp_f16_o:
+; SDAG-GFX10:       ; %bb.0:
+; SDAG-GFX10-NEXT:    s_clause 0x1
+; SDAG-GFX10-NEXT:    s_load_dword s4, s[0:1], 0x2c
+; SDAG-GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; SDAG-GFX10-NEXT:    v_mov_b32_e32 v0, 0
+; SDAG-GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; SDAG-GFX10-NEXT:    v_cmp_o_f16_e64 s0, 0x5640, s4
+; SDAG-GFX10-NEXT:    v_mov_b32_e32 v1, s0
+; SDAG-GFX10-NEXT:    global_store_dword v0, v1, s[2:3]
+; SDAG-GFX10-NEXT:    s_endpgm
+;
+; GISEL-GFX11-LABEL: v_fcmp_f16_o:
+; GISEL-GFX11:       ; %bb.0:
+; GISEL-GFX11-NEXT:    s_clause 0x1
+; GISEL-GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; GISEL-GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GISEL-GFX11-NEXT:    v_mov_b32_e32 v1, 0
+; GISEL-GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT:    v_cmp_o_f16_e64 s2, 0x5640, s2
+; GISEL-GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GISEL-GFX11-NEXT:    v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GISEL-GFX11-NEXT:    s_nop 0
+; GISEL-GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GISEL-GFX11-NEXT:    s_endpgm
+;
+; GISEL-GFX10-LABEL: v_fcmp_f16_o:
+; GISEL-GFX10:       ; %bb.0:
+; GISEL-GFX10-NEXT:    s_clause 0x1
+; GISEL-GFX10-NEXT:    s_load_dword s4, s[0:1], 0x2c
+; GISEL-GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; GISEL-GFX10-NEXT:    v_mov_b32_e32 v1, 0
+; GISEL-GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GISEL-GFX10-NEXT:    v_cmp_o_f16_e64 s0, 0x5640, s4
+; GISEL-GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GISEL-GFX10-NEXT:    global_store_dword v1, v0, s[2:3]
+; GISEL-GFX10-NEXT:    s_endpgm
+  %result = call i32 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 7)
+  store i32 %result, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @v_fcmp_f16_uo(ptr addrspace(1) %out, half %src) {
+; SDAG-GFX11-LABEL: v_fcmp_f16_uo:
+; SDAG-GFX11:       ; %bb.0:
+; SDAG-GFX11-NEXT:    s_clause 0x1
+; SDAG-GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; SDAG-GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; SDAG-GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; SDAG-GFX11-NEXT:    v_cmp_u_f16_e64 s2, 0x5640, s2
+; SDAG-GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; SDAG-GFX11-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; SDAG-GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
+; SDAG-GFX11-NEXT:    s_nop 0
+; SDAG-GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; SDAG-GFX11-NEXT:    s_endpgm
+;
+; SDAG-GFX10-LABEL: v_fcmp_f16_uo:
+; SDAG-GFX10:       ; %bb.0:
+; SDAG-GFX10-NEXT:    s_clause 0x1
+; SDAG-GFX10-NEXT:    s_load_dword s4, s[0:1], 0x2c
+; SDAG-GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; SDAG-GFX10-NEXT:    v_mov_b32_e32 v0, 0
+; SDAG-GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; SDAG-GFX10-NEXT:    v_cmp_u_f16_e64 s0, 0x5640, s4
+; SDAG-GFX10-NEXT:    v_mov_b32_e32 v1, s0
+; SDAG-GFX10-NEXT:    global_store_dword v0, v1, s[2:3]
+; SDAG-GFX10-NEXT:    s_endpgm
+;
+; GISEL-GFX11-LABEL: v_fcmp_f16_uo:
+; GISEL-GFX11:       ; %bb.0:
+; GISEL-GFX11-NEXT:    s_clause 0x1
+; GISEL-GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; GISEL-GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GISEL-GFX11-NEXT:    v_mov_b32_e32 v1, 0
+; GISEL-GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT:    v_cmp_u_f16_e64 s2, 0x5640, s2
+; GISEL-GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GISEL-GFX11-NEXT:    v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GISEL-GFX11-NEXT:    s_nop 0
+; GISEL-GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GISEL-GFX11-NEXT:    s_endpgm
+;
+; GISEL-GFX10-LABEL: v_fcmp_f16_uo:
+; GISEL-GFX10:       ; %bb.0:
+; GISEL-GFX10-NEXT:    s_clause 0x1
+; GISEL-GFX10-NEXT:    s_load_dword s4, s[0:1], 0x2c
+; GISEL-GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; GISEL-GFX10-NEXT:    v_mov_b32_e32 v1, 0
+; GISEL-GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GISEL-GFX10-NEXT:    v_cmp_u_f16_e64 s0, 0x5640, s4
+; GISEL-GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GISEL-GFX10-NEXT:    global_store_dword v1, v0, s[2:3]
+; GISEL-GFX10-NEXT:    s_endpgm
+  %result = call i32 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 8)
+  store i32 %result, ptr addrspace(1) %out
+  ret void
+}
 
 define amdgpu_kernel void @v_fcmp_f16_ule(ptr addrspace(1) %out, half %src) {
 ; SDAG-GFX11-LABEL: v_fcmp_f16_ule:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll
index eeff0c57bb..e2bdcfa6bb 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll
@@ -546,6 +546,129 @@ define amdgpu_kernel void @v_fcmp_f32_ole(ptr addrspace(1) %out, float %src) {
   ret void
 }
 
+define amdgpu_kernel void @v_fcmp_f32_o(ptr addrspace(1) %out, float %src) {
+; GFX11-LABEL: v_fcmp_f32_o:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_o_f32_e64 s[2:3], 0x42c80000, s2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_mov_b32_e32 v0, s2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s3
+; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-NEXT:    s_nop 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+;
+; GFX9-LABEL: v_fcmp_f32_o:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dword s4, s[0:1], 0x2c
+; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0x42c80000
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_o_f32_e64 s[0:1], s4, v0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX9-NEXT:    s_endpgm
+;
+; VI-SDAG-LABEL: v_fcmp_f32_o:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x42c80000
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_o_f32_e64 s[2:3], s2, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f32_o:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, 0x42c80000
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_o_f32_e64 s[2:3], s2, v0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
+  %result = call i64 @llvm.amdgcn.fcmp.f32(float %src, float 100.00, i32 7)
+  store i64 %result, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @v_fcmp_f32_uo(ptr addrspace(1) %out, float %src) {
+; GFX11-LABEL: v_fcmp_f32_uo:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_u_f32_e64 s[2:3], 0x42c80000, s2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_mov_b32_e32 v0, s2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s3
+; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-NEXT:    s_nop 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+;
+; GFX9-LABEL: v_fcmp_f32_uo:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dword s4, s[0:1], 0x2c
+; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0x42c80000
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_u_f32_e64 s[0:1], s4, v0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX9-NEXT:    s_endpgm
+;
+; VI-SDAG-LABEL: v_fcmp_f32_uo:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x42c80000
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_u_f32_e64 s[2:3], s2, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f32_uo:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, 0x42c80000
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_u_f32_e64 s[2:3], s2, v0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
+  %result = call i64 @llvm.amdgcn.fcmp.f32(float %src, float 100.00, i32 8)
+  store i64 %result, ptr addrspace(1) %out
+  ret void
+}
 
 define amdgpu_kernel void @v_fcmp_f32_ueq(ptr addrspace(1) %out, float %src) {
 ; GFX11-LABEL: v_fcmp_f32_ueq:
@@ -1465,6 +1588,162 @@ define amdgpu_kernel void @v_fcmp_f64_ueq(ptr addrspace(1) %out, double %src) {
   ret void
 }
 
+define amdgpu_kernel void @v_fcmp_f64_o(ptr addrspace(1) %out, double %src) {
+; GFX11-LABEL: v_fcmp_f64_o:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:    s_mov_b32 s5, 0x40590000
+; GFX11-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_o_f64_e64 s[2:3], s[2:3], s[4:5]
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_mov_b32_e32 v0, s2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s3
+; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-NEXT:    s_nop 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+;
+; GFX9-SDAG-LABEL: v_fcmp_f64_o:
+; GFX9-SDAG:       ; %bb.0:
+; GFX9-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v1, 0x40590000
+; GFX9-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT:    v_cmp_o_f64_e64 s[2:3], s[2:3], v[0:1]
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v2, s3
+; GFX9-SDAG-NEXT:    global_store_dwordx2 v0, v[1:2], s[0:1]
+; GFX9-SDAG-NEXT:    s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f64_o:
+; GFX9-GISEL:       ; %bb.0:
+; GFX9-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-GISEL-NEXT:    s_mov_b32 s4, 0
+; GFX9-GISEL-NEXT:    s_mov_b32 s5, 0x40590000
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT:    v_cmp_o_f64_e64 s[2:3], s[2:3], v[0:1]
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT:    s_endpgm
+;
+; VI-SDAG-LABEL: v_fcmp_f64_o:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x40590000
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_o_f64_e64 s[2:3], s[2:3], v[0:1]
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f64_o:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-GISEL-NEXT:    s_mov_b32 s4, 0
+; VI-GISEL-NEXT:    s_mov_b32 s5, 0x40590000
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s4
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s5
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_o_f64_e64 s[2:3], s[2:3], v[0:1]
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
+  %result = call i64 @llvm.amdgcn.fcmp.f64(double %src, double 100.00, i32 7)
+  store i64 %result, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @v_fcmp_f64_uo(ptr addrspace(1) %out, double %src) {
+; GFX11-LABEL: v_fcmp_f64_uo:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:    s_mov_b32 s5, 0x40590000
+; GFX11-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_u_f64_e64 s[2:3], s[2:3], s[4:5]
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_mov_b32_e32 v0, s2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s3
+; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-NEXT:    s_nop 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+;
+; GFX9-SDAG-LABEL: v_fcmp_f64_uo:
+; GFX9-SDAG:       ; %bb.0:
+; GFX9-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v1, 0x40590000
+; GFX9-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT:    v_cmp_u_f64_e64 s[2:3], s[2:3], v[0:1]
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v2, s3
+; GFX9-SDAG-NEXT:    global_store_dwordx2 v0, v[1:2], s[0:1]
+; GFX9-SDAG-NEXT:    s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f64_uo:
+; GFX9-GISEL:       ; %bb.0:
+; GFX9-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-GISEL-NEXT:    s_mov_b32 s4, 0
+; GFX9-GISEL-NEXT:    s_mov_b32 s5, 0x40590000
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT:    v_cmp_u_f64_e64 s[2:3], s[2:3], v[0:1]
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT:    s_endpgm
+;
+; VI-SDAG-LABEL: v_fcmp_f64_uo:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x40590000
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_u_f64_e64 s[2:3], s[2:3], v[0:1]
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f64_uo:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-GISEL-NEXT:    s_mov_b32 s4, 0
+; VI-GISEL-NEXT:    s_mov_b32 s5, 0x40590000
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s4
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s5
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_u_f64_e64 s[2:3], s[2:3], v[0:1]
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
+  %result = call i64 @llvm.amdgcn.fcmp.f64(double %src, double 100.00, i32 8)
+  store i64 %result, ptr addrspace(1) %out
+  ret void
+}
+
 define amdgpu_kernel void @v_fcmp_f64_une(ptr addrspace(1) %out, double %src) {
 ; GFX11-LABEL: v_fcmp_f64_une:
 ; GFX11:       ; %bb.0:
@@ -2731,6 +3010,129 @@ define amdgpu_kernel void @v_fcmp_f16_ult(ptr addrspace(1) %out, half %src) {
   ret void
 }
 
+define amdgpu_kernel void @v_fcmp_f16_o(ptr addrspace(1) %out, half %src) {
+; GFX11-LABEL: v_fcmp_f16_o:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_o_f16_e64 s[2:3], 0x5640, s2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_mov_b32_e32 v0, s2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s3
+; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-NEXT:    s_nop 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+;
+; GFX9-LABEL: v_fcmp_f16_o:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dword s4, s[0:1], 0x2c
+; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0x5640
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_o_f16_e64 s[0:1], s4, v0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX9-NEXT:    s_endpgm
+;
+; VI-SDAG-LABEL: v_fcmp_f16_o:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x5640
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_o_f16_e64 s[2:3], s2, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f16_o:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, 0x5640
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_o_f16_e64 s[2:3], s2, v0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
+  %result = call i64 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 7)
+  store i64 %result, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @v_fcmp_f16_uo(ptr addrspace(1) %out, half %src) {
+; GFX11-LABEL: v_fcmp_f16_uo:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_u_f16_e64 s[2:3], 0x5640, s2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_mov_b32_e32 v0, s2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s3
+; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-NEXT:    s_nop 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+;
+; GFX9-LABEL: v_fcmp_f16_uo:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dword s4, s[0:1], 0x2c
+; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0x5640
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_u_f16_e64 s[0:1], s4, v0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX9-NEXT:    s_endpgm
+;
+; VI-SDAG-LABEL: v_fcmp_f16_uo:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x5640
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_u_f16_e64 s[2:3], s2, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f16_uo:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, 0x5640
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_u_f16_e64 s[2:3], s2, v0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
+  %result = call i64 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 8)
+  store i64 %result, ptr addrspace(1) %out
+  ret void
+}
 
 define amdgpu_kernel void @v_fcmp_f16_ule(ptr addrspace(1) %out, half %src) {
 ; GFX11-LABEL: v_fcmp_f16_ule:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.load.2dmsaa.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.load.2dmsaa.ll
index 853ca53767..5ffdbb0f8c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.load.2dmsaa.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.load.2dmsaa.ll
@@ -1184,6 +1184,47 @@ merge:
   ret [4 x float] %i25
 }
 
+define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask1_different_rsrc(<8 x i32> inreg %rsrc1, <8 x i32> inreg %rsrc2, i32 %s, i32 %t) {
+; NO-MSAA-LABEL: define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask1_different_rsrc(
+; NO-MSAA-SAME: <8 x i32> inreg [[RSRC1:%.*]], <8 x i32> inreg [[RSRC2:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
+; NO-MSAA-NEXT:  main_body:
+; NO-MSAA-NEXT:    [[I:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC1]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC1]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC2]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I3:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC2]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
+; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x float] [[I4]], float [[I1]], 1
+; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x float] [[I5]], float [[I2]], 2
+; NO-MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x float] [[I6]], float [[I3]], 3
+; NO-MSAA-NEXT:    ret [4 x float] [[I7]]
+;
+; MSAA-LABEL: define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask1_different_rsrc(
+; MSAA-SAME: <8 x i32> inreg [[RSRC1:%.*]], <8 x i32> inreg [[RSRC2:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
+; MSAA-NEXT:  main_body:
+; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC1]], i32 0, i32 0)
+; MSAA-NEXT:    [[I:%.*]] = extractelement <4 x float> [[TMP0]], i64 0
+; MSAA-NEXT:    [[I1:%.*]] = extractelement <4 x float> [[TMP0]], i64 1
+; MSAA-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC2]], i32 0, i32 0)
+; MSAA-NEXT:    [[I2:%.*]] = extractelement <4 x float> [[TMP1]], i64 0
+; MSAA-NEXT:    [[I3:%.*]] = extractelement <4 x float> [[TMP1]], i64 1
+; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
+; MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x float] [[I4]], float [[I1]], 1
+; MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x float] [[I5]], float [[I2]], 2
+; MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x float] [[I6]], float [[I3]], 3
+; MSAA-NEXT:    ret [4 x float] [[I7]]
+;
+main_body:
+  %i = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc1, i32 0, i32 0)
+  %i1 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc1, i32 0, i32 0)
+  %i2 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc2, i32 0, i32 0)
+  %i3 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc2, i32 0, i32 0)
+  %i4 = insertvalue [4 x float] undef, float %i, 0
+  %i5 = insertvalue [4 x float] %i4, float %i1, 1
+  %i6 = insertvalue [4 x float] %i5, float %i2, 2
+  %i7 = insertvalue [4 x float] %i6, float %i3, 3
+  ret [4 x float] %i7
+}
+
 declare float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #0
 declare <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #0
 declare <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #0
diff --git a/llvm/test/CodeGen/AMDGPU/mul.ll b/llvm/test/CodeGen/AMDGPU/mul.ll
index b4e9376d82..da7645d501 100644
--- a/llvm/test/CodeGen/AMDGPU/mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/mul.ll
@@ -1059,6 +1059,255 @@ entry:
   ret void
 }
 
+define amdgpu_kernel void @s_mul_i1(ptr addrspace(1) %out, [8 x i32], i1 %a, [8 x i32], i1 %b) nounwind {
+; SI-LABEL: s_mul_i1:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dword s4, s[0:1], 0x13
+; SI-NEXT:    s_load_dword s5, s[0:1], 0x1c
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mul_i32 s4, s4, s5
+; SI-NEXT:    s_and_b32 s4, s4, 1
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: s_mul_i1:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dword s4, s[0:1], 0x70
+; VI-NEXT:    s_load_dword s5, s[0:1], 0x4c
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    s_mov_b32 s3, 0xf000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_mul_lo_u16_e32 v0, s5, v0
+; VI-NEXT:    v_and_b32_e32 v0, 1, v0
+; VI-NEXT:    buffer_store_byte v0, off, s[0:3], 0
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: s_mul_i1:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x70
+; GFX9-NEXT:    s_load_dword s3, s[0:1], 0x4c
+; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
+; GFX9-NEXT:    s_mov_b32 s7, 0xf000
+; GFX9-NEXT:    s_mov_b32 s6, -1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    v_mul_lo_u16_e32 v0, s3, v0
+; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: s_mul_i1:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_clause 0x2
+; GFX10-NEXT:    s_load_dword s2, s[0:1], 0x4c
+; GFX10-NEXT:    s_load_dword s3, s[0:1], 0x70
+; GFX10-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
+; GFX10-NEXT:    s_mov_b32 s7, 0x31016000
+; GFX10-NEXT:    s_mov_b32 s6, -1
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_mul_lo_u16 v0, s2, s3
+; GFX10-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX10-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: s_mul_i1:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x4c
+; GFX11-NEXT:    s_load_b32 s3, s[0:1], 0x70
+; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_mul_lo_u16 v0, s2, s3
+; GFX11-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX11-NEXT:    s_mov_b32 s2, -1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX11-NEXT:    buffer_store_b8 v0, off, s[0:3], 0
+; GFX11-NEXT:    s_nop 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+;
+; EG-LABEL: s_mul_i1:
+; EG:       ; %bb.0: ; %entry
+; EG-NEXT:    ALU 0, @10, KC0[], KC1[]
+; EG-NEXT:    TEX 1 @6
+; EG-NEXT:    ALU 12, @11, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_8 T1.X, T0.X, 72, #3
+; EG-NEXT:     VTX_READ_8 T0.X, T0.X, 108, #3
+; EG-NEXT:    ALU clause starting at 10:
+; EG-NEXT:     MOV * T0.X, 0.0,
+; EG-NEXT:    ALU clause starting at 11:
+; EG-NEXT:     AND_INT T0.W, KC0[2].Y, literal.x,
+; EG-NEXT:     MULLO_INT * T0.X, T1.X, T0.X,
+; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T1.W, PS, 1,
+; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
+; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT:     LSHL T0.X, PV.W, PS,
+; EG-NEXT:     LSHL * T0.W, literal.x, PS,
+; EG-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT:     MOV T0.Y, 0.0,
+; EG-NEXT:     MOV * T0.Z, 0.0,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+entry:
+  %mul = mul i1 %a, %b
+  store i1 %mul, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @v_mul_i1(ptr addrspace(1) %out, ptr addrspace(1) %in) {
+; SI-LABEL: v_mul_i1:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s10, s6
+; SI-NEXT:    s_mov_b32 s11, s7
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s8, s2
+; SI-NEXT:    s_mov_b32 s9, s3
+; SI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; SI-NEXT:    buffer_load_ubyte v1, off, s[8:11], 0 offset:4
+; SI-NEXT:    s_mov_b32 s4, s0
+; SI-NEXT:    s_mov_b32 s5, s1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_mul_lo_u32 v0, v0, v1
+; SI-NEXT:    v_and_b32_e32 v0, 1, v0
+; SI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: v_mul_i1:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT:    s_mov_b32 s7, 0xf000
+; VI-NEXT:    s_mov_b32 s6, -1
+; VI-NEXT:    s_mov_b32 s10, s6
+; VI-NEXT:    s_mov_b32 s11, s7
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_mov_b32 s8, s2
+; VI-NEXT:    s_mov_b32 s9, s3
+; VI-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; VI-NEXT:    buffer_load_ubyte v1, off, s[8:11], 0 offset:4
+; VI-NEXT:    s_mov_b32 s4, s0
+; VI-NEXT:    s_mov_b32 s5, s1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
+; VI-NEXT:    v_and_b32_e32 v0, 1, v0
+; VI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: v_mul_i1:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-NEXT:    s_mov_b32 s7, 0xf000
+; GFX9-NEXT:    s_mov_b32 s6, -1
+; GFX9-NEXT:    s_mov_b32 s10, s6
+; GFX9-NEXT:    s_mov_b32 s11, s7
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b32 s8, s2
+; GFX9-NEXT:    s_mov_b32 s9, s3
+; GFX9-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; GFX9-NEXT:    buffer_load_ubyte v1, off, s[8:11], 0 offset:4
+; GFX9-NEXT:    s_mov_b32 s4, s0
+; GFX9-NEXT:    s_mov_b32 s5, s1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mul_lo_u16_e32 v0, v0, v1
+; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: v_mul_i1:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX10-NEXT:    s_mov_b32 s6, -1
+; GFX10-NEXT:    s_mov_b32 s7, 0x31016000
+; GFX10-NEXT:    s_mov_b32 s10, s6
+; GFX10-NEXT:    s_mov_b32 s11, s7
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_mov_b32 s8, s2
+; GFX10-NEXT:    s_mov_b32 s9, s3
+; GFX10-NEXT:    s_clause 0x1
+; GFX10-NEXT:    buffer_load_ubyte v0, off, s[8:11], 0
+; GFX10-NEXT:    buffer_load_ubyte v1, off, s[8:11], 0 offset:4
+; GFX10-NEXT:    s_mov_b32 s4, s0
+; GFX10-NEXT:    s_mov_b32 s5, s1
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_mul_lo_u16 v0, v0, v1
+; GFX10-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX10-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: v_mul_i1:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX11-NEXT:    s_mov_b32 s6, -1
+; GFX11-NEXT:    s_mov_b32 s7, 0x31016000
+; GFX11-NEXT:    s_mov_b32 s10, s6
+; GFX11-NEXT:    s_mov_b32 s11, s7
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_mov_b32 s8, s2
+; GFX11-NEXT:    s_mov_b32 s9, s3
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    buffer_load_u8 v0, off, s[8:11], 0
+; GFX11-NEXT:    buffer_load_u8 v1, off, s[8:11], 0 offset:4
+; GFX11-NEXT:    s_mov_b32 s4, s0
+; GFX11-NEXT:    s_mov_b32 s5, s1
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_mul_lo_u16 v0, v0, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX11-NEXT:    buffer_store_b8 v0, off, s[4:7], 0
+; GFX11-NEXT:    s_nop 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+;
+; EG-LABEL: v_mul_i1:
+; EG:       ; %bb.0: ; %entry
+; EG-NEXT:    ALU 0, @10, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    TEX 1 @6
+; EG-NEXT:    ALU 12, @11, KC0[CB0:0-32], KC1[]
+; EG-NEXT:    MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT:    CF_END
+; EG-NEXT:    PAD
+; EG-NEXT:    Fetch clause starting at 6:
+; EG-NEXT:     VTX_READ_8 T1.X, T0.X, 4, #1
+; EG-NEXT:     VTX_READ_8 T0.X, T0.X, 0, #1
+; EG-NEXT:    ALU clause starting at 10:
+; EG-NEXT:     MOV * T0.X, KC0[2].Z,
+; EG-NEXT:    ALU clause starting at 11:
+; EG-NEXT:     AND_INT T0.W, KC0[2].Y, literal.x,
+; EG-NEXT:     MULLO_INT * T0.X, T0.X, T1.X,
+; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT:     AND_INT T1.W, PS, 1,
+; EG-NEXT:     LSHL * T0.W, PV.W, literal.x,
+; EG-NEXT:    3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT:     LSHL T0.X, PV.W, PS,
+; EG-NEXT:     LSHL * T0.W, literal.x, PS,
+; EG-NEXT:    255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT:     MOV T0.Y, 0.0,
+; EG-NEXT:     MOV * T0.Z, 0.0,
+; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
+entry:
+  %b_ptr = getelementptr i32, ptr addrspace(1) %in, i32 1
+  %a = load i1, ptr addrspace(1) %in
+  %b = load i1, ptr addrspace(1) %b_ptr
+  %result = mul i1 %a, %b
+  store i1 %result, ptr addrspace(1) %out
+  ret void
+}
+
 ; A standard 64-bit multiply.  The expansion should be around 6 instructions.
 ; It would be difficult to match the expansion correctly without writing
 ; a really complicated list of FileCheck expressions.  I don't want
@@ -1213,7 +1462,7 @@ define amdgpu_kernel void @v_mul_i64(ptr addrspace(1) %out, ptr addrspace(1) %ap
 ; SI-NEXT:    v_mul_hi_u32 v4, v2, v0
 ; SI-NEXT:    v_mul_lo_u32 v3, v3, v0
 ; SI-NEXT:    v_mul_lo_u32 v0, v2, v0
-; SI-NEXT:    v_add_i32_e32 v1, vcc, v4, v1
+; SI-NEXT:    v_add_i32_e32 v1, vcc, v1, v4
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
 ; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
@@ -1367,30 +1616,30 @@ define amdgpu_kernel void @mul32_in_branch(ptr addrspace(1) %out, ptr addrspace(
 ; SI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xd
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_cmp_lg_u32 s2, 0
-; SI-NEXT:    s_cbranch_scc0 .LBB11_2
+; SI-NEXT:    s_cbranch_scc0 .LBB13_2
 ; SI-NEXT:  ; %bb.1: ; %else
 ; SI-NEXT:    s_mul_i32 s6, s2, s3
 ; SI-NEXT:    s_mov_b64 s[4:5], 0
-; SI-NEXT:    s_branch .LBB11_3
-; SI-NEXT:  .LBB11_2:
+; SI-NEXT:    s_branch .LBB13_3
+; SI-NEXT:  .LBB13_2:
 ; SI-NEXT:    s_mov_b64 s[4:5], -1
 ; SI-NEXT:    ; implicit-def: $sgpr6
-; SI-NEXT:  .LBB11_3: ; %Flow
+; SI-NEXT:  .LBB13_3: ; %Flow
 ; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
 ; SI-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_mov_b64 vcc, vcc
-; SI-NEXT:    s_cbranch_vccnz .LBB11_5
+; SI-NEXT:    s_cbranch_vccnz .LBB13_5
 ; SI-NEXT:  ; %bb.4: ; %if
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    s_mov_b32 s4, s2
 ; SI-NEXT:    s_mov_b32 s5, s3
 ; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0
-; SI-NEXT:    s_branch .LBB11_6
-; SI-NEXT:  .LBB11_5:
+; SI-NEXT:    s_branch .LBB13_6
+; SI-NEXT:  .LBB13_5:
 ; SI-NEXT:    v_mov_b32_e32 v0, s6
-; SI-NEXT:  .LBB11_6: ; %endif
+; SI-NEXT:  .LBB13_6: ; %endif
 ; SI-NEXT:    s_mov_b32 s3, 0xf000
 ; SI-NEXT:    s_mov_b32 s2, -1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
@@ -1402,18 +1651,18 @@ define amdgpu_kernel void @mul32_in_branch(ptr addrspace(1) %out, ptr addrspace(
 ; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_cmp_lg_u32 s2, 0
-; VI-NEXT:    s_cbranch_scc0 .LBB11_2
+; VI-NEXT:    s_cbranch_scc0 .LBB13_2
 ; VI-NEXT:  ; %bb.1: ; %else
 ; VI-NEXT:    s_mul_i32 s6, s2, s3
 ; VI-NEXT:    s_mov_b64 s[4:5], 0
-; VI-NEXT:    s_branch .LBB11_3
-; VI-NEXT:  .LBB11_2:
+; VI-NEXT:    s_branch .LBB13_3
+; VI-NEXT:  .LBB13_2:
 ; VI-NEXT:    s_mov_b64 s[4:5], -1
 ; VI-NEXT:    ; implicit-def: $sgpr6
-; VI-NEXT:  .LBB11_3: ; %Flow
+; VI-NEXT:  .LBB13_3: ; %Flow
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; VI-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
-; VI-NEXT:    s_cbranch_vccnz .LBB11_5
+; VI-NEXT:    s_cbranch_vccnz .LBB13_5
 ; VI-NEXT:  ; %bb.4: ; %if
 ; VI-NEXT:    s_mov_b32 s7, 0xf000
 ; VI-NEXT:    s_mov_b32 s6, -1
@@ -1421,10 +1670,10 @@ define amdgpu_kernel void @mul32_in_branch(ptr addrspace(1) %out, ptr addrspace(
 ; VI-NEXT:    s_mov_b32 s4, s2
 ; VI-NEXT:    s_mov_b32 s5, s3
 ; VI-NEXT:    buffer_load_dword v0, off, s[4:7], 0
-; VI-NEXT:    s_branch .LBB11_6
-; VI-NEXT:  .LBB11_5:
+; VI-NEXT:    s_branch .LBB13_6
+; VI-NEXT:  .LBB13_5:
 ; VI-NEXT:    v_mov_b32_e32 v0, s6
-; VI-NEXT:  .LBB11_6: ; %endif
+; VI-NEXT:  .LBB13_6: ; %endif
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_mov_b32 s3, 0xf000
 ; VI-NEXT:    s_mov_b32 s2, -1
@@ -1437,18 +1686,18 @@ define amdgpu_kernel void @mul32_in_branch(ptr addrspace(1) %out, ptr addrspace(
 ; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_cmp_lg_u32 s2, 0
-; GFX9-NEXT:    s_cbranch_scc0 .LBB11_2
+; GFX9-NEXT:    s_cbranch_scc0 .LBB13_2
 ; GFX9-NEXT:  ; %bb.1: ; %else
 ; GFX9-NEXT:    s_mul_i32 s6, s2, s3
 ; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:    s_branch .LBB11_3
-; GFX9-NEXT:  .LBB11_2:
+; GFX9-NEXT:    s_branch .LBB13_3
+; GFX9-NEXT:  .LBB13_2:
 ; GFX9-NEXT:    s_mov_b64 s[4:5], -1
 ; GFX9-NEXT:    ; implicit-def: $sgpr6
-; GFX9-NEXT:  .LBB11_3: ; %Flow
+; GFX9-NEXT:  .LBB13_3: ; %Flow
 ; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX9-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_vccnz .LBB11_5
+; GFX9-NEXT:    s_cbranch_vccnz .LBB13_5
 ; GFX9-NEXT:  ; %bb.4: ; %if
 ; GFX9-NEXT:    s_mov_b32 s7, 0xf000
 ; GFX9-NEXT:    s_mov_b32 s6, -1
@@ -1456,10 +1705,10 @@ define amdgpu_kernel void @mul32_in_branch(ptr addrspace(1) %out, ptr addrspace(
 ; GFX9-NEXT:    s_mov_b32 s4, s2
 ; GFX9-NEXT:    s_mov_b32 s5, s3
 ; GFX9-NEXT:    buffer_load_dword v0, off, s[4:7], 0
-; GFX9-NEXT:    s_branch .LBB11_6
-; GFX9-NEXT:  .LBB11_5:
+; GFX9-NEXT:    s_branch .LBB13_6
+; GFX9-NEXT:  .LBB13_5:
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:  .LBB11_6: ; %endif
+; GFX9-NEXT:  .LBB13_6: ; %endif
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX9-NEXT:    s_mov_b32 s2, -1
@@ -1473,17 +1722,17 @@ define amdgpu_kernel void @mul32_in_branch(ptr addrspace(1) %out, ptr addrspace(
 ; GFX10-NEXT:    s_mov_b32 s4, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_cmp_lg_u32 s2, 0
-; GFX10-NEXT:    s_cbranch_scc0 .LBB11_2
+; GFX10-NEXT:    s_cbranch_scc0 .LBB13_2
 ; GFX10-NEXT:  ; %bb.1: ; %else
 ; GFX10-NEXT:    s_mul_i32 s5, s2, s3
-; GFX10-NEXT:    s_branch .LBB11_3
-; GFX10-NEXT:  .LBB11_2:
+; GFX10-NEXT:    s_branch .LBB13_3
+; GFX10-NEXT:  .LBB13_2:
 ; GFX10-NEXT:    s_mov_b32 s4, -1
 ; GFX10-NEXT:    ; implicit-def: $sgpr5
-; GFX10-NEXT:  .LBB11_3: ; %Flow
+; GFX10-NEXT:  .LBB13_3: ; %Flow
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    s_andn2_b32 vcc_lo, exec_lo, s4
-; GFX10-NEXT:    s_cbranch_vccnz .LBB11_5
+; GFX10-NEXT:    s_cbranch_vccnz .LBB13_5
 ; GFX10-NEXT:  ; %bb.4: ; %if
 ; GFX10-NEXT:    s_mov_b32 s7, 0x31016000
 ; GFX10-NEXT:    s_mov_b32 s6, -1
@@ -1491,10 +1740,10 @@ define amdgpu_kernel void @mul32_in_branch(ptr addrspace(1) %out, ptr addrspace(
 ; GFX10-NEXT:    s_mov_b32 s4, s2
 ; GFX10-NEXT:    s_mov_b32 s5, s3
 ; GFX10-NEXT:    buffer_load_dword v0, off, s[4:7], 0
-; GFX10-NEXT:    s_branch .LBB11_6
-; GFX10-NEXT:  .LBB11_5:
+; GFX10-NEXT:    s_branch .LBB13_6
+; GFX10-NEXT:  .LBB13_5:
 ; GFX10-NEXT:    v_mov_b32_e32 v0, s5
-; GFX10-NEXT:  .LBB11_6: ; %endif
+; GFX10-NEXT:  .LBB13_6: ; %endif
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX10-NEXT:    s_mov_b32 s2, -1
@@ -1508,17 +1757,17 @@ define amdgpu_kernel void @mul32_in_branch(ptr addrspace(1) %out, ptr addrspace(
 ; GFX11-NEXT:    s_mov_b32 s4, 0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    s_cmp_lg_u32 s2, 0
-; GFX11-NEXT:    s_cbranch_scc0 .LBB11_2
+; GFX11-NEXT:    s_cbranch_scc0 .LBB13_2
 ; GFX11-NEXT:  ; %bb.1: ; %else
 ; GFX11-NEXT:    s_mul_i32 s5, s2, s3
-; GFX11-NEXT:    s_branch .LBB11_3
-; GFX11-NEXT:  .LBB11_2:
+; GFX11-NEXT:    s_branch .LBB13_3
+; GFX11-NEXT:  .LBB13_2:
 ; GFX11-NEXT:    s_mov_b32 s4, -1
 ; GFX11-NEXT:    ; implicit-def: $sgpr5
-; GFX11-NEXT:  .LBB11_3: ; %Flow
+; GFX11-NEXT:  .LBB13_3: ; %Flow
 ; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
 ; GFX11-NEXT:    s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX11-NEXT:    s_cbranch_vccnz .LBB11_5
+; GFX11-NEXT:    s_cbranch_vccnz .LBB13_5
 ; GFX11-NEXT:  ; %bb.4: ; %if
 ; GFX11-NEXT:    s_mov_b32 s7, 0x31016000
 ; GFX11-NEXT:    s_mov_b32 s6, -1
@@ -1526,10 +1775,10 @@ define amdgpu_kernel void @mul32_in_branch(ptr addrspace(1) %out, ptr addrspace(
 ; GFX11-NEXT:    s_mov_b32 s4, s2
 ; GFX11-NEXT:    s_mov_b32 s5, s3
 ; GFX11-NEXT:    buffer_load_b32 v0, off, s[4:7], 0
-; GFX11-NEXT:    s_branch .LBB11_6
-; GFX11-NEXT:  .LBB11_5:
+; GFX11-NEXT:    s_branch .LBB13_6
+; GFX11-NEXT:  .LBB13_5:
 ; GFX11-NEXT:    v_mov_b32_e32 v0, s5
-; GFX11-NEXT:  .LBB11_6: ; %endif
+; GFX11-NEXT:  .LBB13_6: ; %endif
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX11-NEXT:    s_mov_b32 s2, -1
@@ -1601,7 +1850,7 @@ define amdgpu_kernel void @mul64_in_branch(ptr addrspace(1) %out, ptr addrspace(
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    v_cmp_ne_u64_e64 s[10:11], s[4:5], 0
 ; SI-NEXT:    s_and_b64 vcc, exec, s[10:11]
-; SI-NEXT:    s_cbranch_vccz .LBB12_4
+; SI-NEXT:    s_cbranch_vccz .LBB14_4
 ; SI-NEXT:  ; %bb.1: ; %else
 ; SI-NEXT:    v_mov_b32_e32 v0, s6
 ; SI-NEXT:    v_mul_hi_u32 v0, s4, v0
@@ -1612,22 +1861,22 @@ define amdgpu_kernel void @mul64_in_branch(ptr addrspace(1) %out, ptr addrspace(
 ; SI-NEXT:    v_add_i32_e32 v1, vcc, s5, v0
 ; SI-NEXT:    v_mov_b32_e32 v0, s4
 ; SI-NEXT:    s_andn2_b64 vcc, exec, s[8:9]
-; SI-NEXT:    s_cbranch_vccnz .LBB12_3
-; SI-NEXT:  .LBB12_2: ; %if
+; SI-NEXT:    s_cbranch_vccnz .LBB14_3
+; SI-NEXT:  .LBB14_2: ; %if
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    s_mov_b32 s4, s2
 ; SI-NEXT:    s_mov_b32 s5, s3
 ; SI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[4:7], 0
-; SI-NEXT:  .LBB12_3: ; %endif
+; SI-NEXT:  .LBB14_3: ; %endif
 ; SI-NEXT:    s_mov_b32 s3, 0xf000
 ; SI-NEXT:    s_mov_b32 s2, -1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
-; SI-NEXT:  .LBB12_4:
+; SI-NEXT:  .LBB14_4:
 ; SI-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; SI-NEXT:    s_branch .LBB12_2
+; SI-NEXT:    s_branch .LBB14_2
 ;
 ; VI-LABEL: mul64_in_branch:
 ; VI:       ; %bb.0: ; %entry
@@ -1635,7 +1884,7 @@ define amdgpu_kernel void @mul64_in_branch(ptr addrspace(1) %out, ptr addrspace(
 ; VI-NEXT:    s_mov_b64 s[8:9], 0
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_cmp_lg_u64 s[4:5], 0
-; VI-NEXT:    s_cbranch_scc0 .LBB12_4
+; VI-NEXT:    s_cbranch_scc0 .LBB14_4
 ; VI-NEXT:  ; %bb.1: ; %else
 ; VI-NEXT:    v_mov_b32_e32 v0, s6
 ; VI-NEXT:    v_mad_u64_u32 v[0:1], s[10:11], s4, v0, 0
@@ -1644,22 +1893,22 @@ define amdgpu_kernel void @mul64_in_branch(ptr addrspace(1) %out, ptr addrspace(
 ; VI-NEXT:    s_mul_i32 s4, s5, s6
 ; VI-NEXT:    v_add_u32_e32 v1, vcc, s4, v1
 ; VI-NEXT:    s_andn2_b64 vcc, exec, s[8:9]
-; VI-NEXT:    s_cbranch_vccnz .LBB12_3
-; VI-NEXT:  .LBB12_2: ; %if
+; VI-NEXT:    s_cbranch_vccnz .LBB14_3
+; VI-NEXT:  .LBB14_2: ; %if
 ; VI-NEXT:    s_mov_b32 s7, 0xf000
 ; VI-NEXT:    s_mov_b32 s6, -1
 ; VI-NEXT:    s_mov_b32 s4, s2
 ; VI-NEXT:    s_mov_b32 s5, s3
 ; VI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[4:7], 0
-; VI-NEXT:  .LBB12_3: ; %endif
+; VI-NEXT:  .LBB14_3: ; %endif
 ; VI-NEXT:    s_mov_b32 s3, 0xf000
 ; VI-NEXT:    s_mov_b32 s2, -1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
-; VI-NEXT:  .LBB12_4:
+; VI-NEXT:  .LBB14_4:
 ; VI-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; VI-NEXT:    s_branch .LBB12_2
+; VI-NEXT:    s_branch .LBB14_2
 ;
 ; GFX9-LABEL: mul64_in_branch:
 ; GFX9:       ; %bb.0: ; %entry
@@ -1667,7 +1916,7 @@ define amdgpu_kernel void @mul64_in_branch(ptr addrspace(1) %out, ptr addrspace(
 ; GFX9-NEXT:    s_mov_b64 s[8:9], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_cmp_lg_u64 s[4:5], 0
-; GFX9-NEXT:    s_cbranch_scc0 .LBB12_3
+; GFX9-NEXT:    s_cbranch_scc0 .LBB14_3
 ; GFX9-NEXT:  ; %bb.1: ; %else
 ; GFX9-NEXT:    s_mul_i32 s7, s4, s7
 ; GFX9-NEXT:    s_mul_hi_u32 s10, s4, s6
@@ -1676,21 +1925,21 @@ define amdgpu_kernel void @mul64_in_branch(ptr addrspace(1) %out, ptr addrspace(
 ; GFX9-NEXT:    s_add_i32 s5, s7, s5
 ; GFX9-NEXT:    s_mul_i32 s4, s4, s6
 ; GFX9-NEXT:    s_andn2_b64 vcc, exec, s[8:9]
-; GFX9-NEXT:    s_cbranch_vccnz .LBB12_4
-; GFX9-NEXT:  .LBB12_2: ; %if
+; GFX9-NEXT:    s_cbranch_vccnz .LBB14_4
+; GFX9-NEXT:  .LBB14_2: ; %if
 ; GFX9-NEXT:    s_mov_b32 s7, 0xf000
 ; GFX9-NEXT:    s_mov_b32 s6, -1
 ; GFX9-NEXT:    s_mov_b32 s4, s2
 ; GFX9-NEXT:    s_mov_b32 s5, s3
 ; GFX9-NEXT:    buffer_load_dwordx2 v[0:1], off, s[4:7], 0
-; GFX9-NEXT:    s_branch .LBB12_5
-; GFX9-NEXT:  .LBB12_3:
+; GFX9-NEXT:    s_branch .LBB14_5
+; GFX9-NEXT:  .LBB14_3:
 ; GFX9-NEXT:    ; implicit-def: $sgpr4_sgpr5
-; GFX9-NEXT:    s_branch .LBB12_2
-; GFX9-NEXT:  .LBB12_4:
+; GFX9-NEXT:    s_branch .LBB14_2
+; GFX9-NEXT:  .LBB14_4:
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s5
-; GFX9-NEXT:  .LBB12_5: ; %endif
+; GFX9-NEXT:  .LBB14_5: ; %endif
 ; GFX9-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX9-NEXT:    s_mov_b32 s2, -1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -1702,7 +1951,7 @@ define amdgpu_kernel void @mul64_in_branch(ptr addrspace(1) %out, ptr addrspace(
 ; GFX10-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_cmp_lg_u64 s[4:5], 0
-; GFX10-NEXT:    s_cbranch_scc0 .LBB12_3
+; GFX10-NEXT:    s_cbranch_scc0 .LBB14_3
 ; GFX10-NEXT:  ; %bb.1: ; %else
 ; GFX10-NEXT:    s_mul_i32 s7, s4, s7
 ; GFX10-NEXT:    s_mul_hi_u32 s8, s4, s6
@@ -1711,22 +1960,22 @@ define amdgpu_kernel void @mul64_in_branch(ptr addrspace(1) %out, ptr addrspace(
 ; GFX10-NEXT:    s_mul_i32 s4, s4, s6
 ; GFX10-NEXT:    s_add_i32 s5, s7, s5
 ; GFX10-NEXT:    s_mov_b32 s6, 0
-; GFX10-NEXT:    s_cbranch_execnz .LBB12_4
-; GFX10-NEXT:  .LBB12_2: ; %if
+; GFX10-NEXT:    s_cbranch_execnz .LBB14_4
+; GFX10-NEXT:  .LBB14_2: ; %if
 ; GFX10-NEXT:    s_mov_b32 s7, 0x31016000
 ; GFX10-NEXT:    s_mov_b32 s6, -1
 ; GFX10-NEXT:    s_mov_b32 s4, s2
 ; GFX10-NEXT:    s_mov_b32 s5, s3
 ; GFX10-NEXT:    buffer_load_dwordx2 v[0:1], off, s[4:7], 0
-; GFX10-NEXT:    s_branch .LBB12_5
-; GFX10-NEXT:  .LBB12_3:
+; GFX10-NEXT:    s_branch .LBB14_5
+; GFX10-NEXT:  .LBB14_3:
 ; GFX10-NEXT:    s_mov_b32 s6, -1
 ; GFX10-NEXT:    ; implicit-def: $sgpr4_sgpr5
-; GFX10-NEXT:    s_branch .LBB12_2
-; GFX10-NEXT:  .LBB12_4:
+; GFX10-NEXT:    s_branch .LBB14_2
+; GFX10-NEXT:  .LBB14_4:
 ; GFX10-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX10-NEXT:    v_mov_b32_e32 v1, s5
-; GFX10-NEXT:  .LBB12_5: ; %endif
+; GFX10-NEXT:  .LBB14_5: ; %endif
 ; GFX10-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX10-NEXT:    s_mov_b32 s2, -1
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -1738,7 +1987,7 @@ define amdgpu_kernel void @mul64_in_branch(ptr addrspace(1) %out, ptr addrspace(
 ; GFX11-NEXT:    s_load_b256 s[0:7], s[0:1], 0x24
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    s_cmp_lg_u64 s[4:5], 0
-; GFX11-NEXT:    s_cbranch_scc0 .LBB12_3
+; GFX11-NEXT:    s_cbranch_scc0 .LBB14_3
 ; GFX11-NEXT:  ; %bb.1: ; %else
 ; GFX11-NEXT:    s_mul_i32 s7, s4, s7
 ; GFX11-NEXT:    s_mul_hi_u32 s8, s4, s6
@@ -1747,21 +1996,21 @@ define amdgpu_kernel void @mul64_in_branch(ptr addrspace(1) %out, ptr addrspace(
 ; GFX11-NEXT:    s_mul_i32 s4, s4, s6
 ; GFX11-NEXT:    s_add_i32 s5, s7, s5
 ; GFX11-NEXT:    s_mov_b32 s6, 0
-; GFX11-NEXT:    s_cbranch_execnz .LBB12_4
-; GFX11-NEXT:  .LBB12_2: ; %if
+; GFX11-NEXT:    s_cbranch_execnz .LBB14_4
+; GFX11-NEXT:  .LBB14_2: ; %if
 ; GFX11-NEXT:    s_mov_b32 s7, 0x31016000
 ; GFX11-NEXT:    s_mov_b32 s6, -1
 ; GFX11-NEXT:    s_mov_b32 s4, s2
 ; GFX11-NEXT:    s_mov_b32 s5, s3
 ; GFX11-NEXT:    buffer_load_b64 v[0:1], off, s[4:7], 0
-; GFX11-NEXT:    s_branch .LBB12_5
-; GFX11-NEXT:  .LBB12_3:
+; GFX11-NEXT:    s_branch .LBB14_5
+; GFX11-NEXT:  .LBB14_3:
 ; GFX11-NEXT:    s_mov_b32 s6, -1
 ; GFX11-NEXT:    ; implicit-def: $sgpr4_sgpr5
-; GFX11-NEXT:    s_branch .LBB12_2
-; GFX11-NEXT:  .LBB12_4:
+; GFX11-NEXT:    s_branch .LBB14_2
+; GFX11-NEXT:  .LBB14_4:
 ; GFX11-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
-; GFX11-NEXT:  .LBB12_5: ; %endif
+; GFX11-NEXT:  .LBB14_5: ; %endif
 ; GFX11-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX11-NEXT:    s_mov_b32 s2, -1
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/peephole-fold-imm.mir b/llvm/test/CodeGen/AMDGPU/peephole-fold-imm.mir
index 099aaa449b..ade192bde4 100644
--- a/llvm/test/CodeGen/AMDGPU/peephole-fold-imm.mir
+++ b/llvm/test/CodeGen/AMDGPU/peephole-fold-imm.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx908 -verify-machineinstrs -run-pass peephole-opt -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx940 -verify-machineinstrs -run-pass peephole-opt -o - %s | FileCheck -check-prefix=GCN %s
 
 ---
 name:            fold_simm_virtual
@@ -8,7 +8,6 @@ body:             |
 
     ; GCN-LABEL: name: fold_simm_virtual
     ; GCN: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
-    ; GCN-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0
     ; GCN-NEXT: SI_RETURN_TO_EPILOG
     %0:sreg_32 = S_MOV_B32 0
     %1:sreg_32 = COPY killed %0
@@ -120,3 +119,228 @@ body:             |
     SI_RETURN_TO_EPILOG $vgpr0_lo16
 
 ...
+
+---
+name:            fold_sreg_64_sub0_to_vgpr_32
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: fold_sreg_64_sub0_to_vgpr_32
+    ; GCN: [[S_MOV_B:%[0-9]+]]:sreg_64 = S_MOV_B64_IMM_PSEUDO 1311768467750121200
+    ; GCN-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -1412567312, implicit $exec
+    ; GCN-NEXT: SI_RETURN_TO_EPILOG [[V_MOV_B32_e32_]]
+    %0:sreg_64 = S_MOV_B64_IMM_PSEUDO 1311768467750121200
+    %1:vgpr_32 = COPY killed %0.sub0
+    SI_RETURN_TO_EPILOG %1
+
+...
+
+---
+name:            fold_sreg_64_sub1_to_vgpr_32
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: fold_sreg_64_sub1_to_vgpr_32
+    ; GCN: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 1311768467750121200
+    ; GCN-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 305419896, implicit $exec
+    ; GCN-NEXT: SI_RETURN_TO_EPILOG [[V_MOV_B32_e32_]]
+    %0:sreg_64 = S_MOV_B64 1311768467750121200
+    %1:vgpr_32 = COPY killed %0.sub1
+    SI_RETURN_TO_EPILOG %1
+
+...
+
+---
+name:            fold_vreg_64_sub1_to_vgpr_32
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: fold_vreg_64_sub1_to_vgpr_32
+    ; GCN: [[V_MOV_B:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO 1311768467750121200, implicit $exec
+    ; GCN-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 305419896, implicit $exec
+    ; GCN-NEXT: SI_RETURN_TO_EPILOG [[V_MOV_B32_e32_]]
+    %0:vreg_64_align2 = V_MOV_B64_PSEUDO 1311768467750121200, implicit $exec
+    %1:vgpr_32 = COPY killed %0.sub1
+    SI_RETURN_TO_EPILOG %1
+
+...
+
+---
+name:            fold_sreg_64_to_vreg_64
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: fold_sreg_64_to_vreg_64
+    ; GCN: [[S_MOV_B:%[0-9]+]]:sreg_64 = S_MOV_B64_IMM_PSEUDO 1311768467750121200
+    ; GCN-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO 1311768467750121200, implicit $exec
+    ; GCN-NEXT: SI_RETURN_TO_EPILOG [[V_MOV_B]]
+    %0:sreg_64 = S_MOV_B64_IMM_PSEUDO 1311768467750121200
+    %1:vreg_64_align2 = COPY killed %0
+    SI_RETURN_TO_EPILOG %1
+
+...
+
+---
+name:            fold_sreg_64_to_sreg_64
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: fold_sreg_64_to_sreg_64
+    ; GCN: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 1311768467750121200
+    ; GCN-NEXT: [[S_MOV_B:%[0-9]+]]:sreg_64 = S_MOV_B64_IMM_PSEUDO 1311768467750121200
+    ; GCN-NEXT: SI_RETURN_TO_EPILOG [[S_MOV_B]]
+    %0:sreg_64 = S_MOV_B64 1311768467750121200
+    %1:sreg_64 = COPY killed %0
+    SI_RETURN_TO_EPILOG %1
+
+...
+
+---
+name:            fold_sreg_64_lo16_to_sgpr_lo16
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: fold_sreg_64_lo16_to_sgpr_lo16
+    ; GCN: [[S_MOV_B:%[0-9]+]]:sreg_64 = S_MOV_B64_IMM_PSEUDO 1125912791875585
+    ; GCN-NEXT: $sgpr0 = S_MOV_B32 1
+    ; GCN-NEXT: SI_RETURN_TO_EPILOG $sgpr0_lo16
+    %0:sreg_64 = S_MOV_B64_IMM_PSEUDO 1125912791875585
+    $sgpr0_lo16 = COPY killed %0.lo16
+    SI_RETURN_TO_EPILOG $sgpr0_lo16
+
+...
+
+---
+name:            fold_sreg_64_hi16_to_sgpr_lo16
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: fold_sreg_64_hi16_to_sgpr_lo16
+    ; GCN: [[S_MOV_B:%[0-9]+]]:sreg_64 = S_MOV_B64_IMM_PSEUDO 1125912791875585
+    ; GCN-NEXT: $sgpr0 = S_MOV_B32 2
+    ; GCN-NEXT: SI_RETURN_TO_EPILOG $sgpr0_lo16
+    %0:sreg_64 = S_MOV_B64_IMM_PSEUDO 1125912791875585
+    $sgpr0_lo16 = COPY killed %0.hi16
+    SI_RETURN_TO_EPILOG $sgpr0_lo16
+
+...
+
+---
+name:            fold_sreg_64_sub1_lo16_to_sgpr_lo16
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: fold_sreg_64_sub1_lo16_to_sgpr_lo16
+    ; GCN: [[S_MOV_B:%[0-9]+]]:sreg_64 = S_MOV_B64_IMM_PSEUDO 1125912791875585
+    ; GCN-NEXT: $sgpr0 = S_MOV_B32 3
+    ; GCN-NEXT: SI_RETURN_TO_EPILOG $sgpr0_lo16
+    %0:sreg_64 = S_MOV_B64_IMM_PSEUDO 1125912791875585
+    $sgpr0_lo16 = COPY killed %0.sub1_lo16
+    SI_RETURN_TO_EPILOG $sgpr0_lo16
+
+...
+
+---
+name:            fold_sreg_64_sub1_hi16_to_sgpr_lo16
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: fold_sreg_64_sub1_hi16_to_sgpr_lo16
+    ; GCN: [[S_MOV_B:%[0-9]+]]:sreg_64 = S_MOV_B64_IMM_PSEUDO 1125912791875585
+    ; GCN-NEXT: $sgpr0 = S_MOV_B32 4
+    ; GCN-NEXT: SI_RETURN_TO_EPILOG $sgpr0_lo16
+    %0:sreg_64 = S_MOV_B64_IMM_PSEUDO 1125912791875585
+    $sgpr0_lo16 = COPY killed %0.sub1_hi16
+    SI_RETURN_TO_EPILOG $sgpr0_lo16
+
+...
+
+---
+name:            fmac_sreg_64_sub0_src0_to_fmamk
+tracksRegLiveness: true
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: fmac_sreg_64_sub0_src0_to_fmamk
+    ; GCN: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[V_FMAMK_F32_:%[0-9]+]]:vgpr_32 = V_FMAMK_F32 [[DEF]], 2882399984, [[DEF1]], implicit $mode, implicit $exec
+    ; GCN-NEXT: SI_RETURN_TO_EPILOG [[V_FMAMK_F32_]]
+    %0:vgpr_32 = IMPLICIT_DEF
+    %1:vgpr_32 = IMPLICIT_DEF
+    %2:sreg_64 = S_MOV_B64_IMM_PSEUDO 1311768467750121200
+    %3:vgpr_32 = V_FMAC_F32_e64 0, %2.sub0, 0, %0, 0, %1, 0, 0, implicit $mode, implicit $exec
+    SI_RETURN_TO_EPILOG %3
+...
+
+---
+name:            fmac_sreg_64_sub1_src0_to_fmamk
+tracksRegLiveness: true
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: fmac_sreg_64_sub1_src0_to_fmamk
+    ; GCN: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[V_FMAMK_F32_:%[0-9]+]]:vgpr_32 = V_FMAMK_F32 [[DEF]], 305419896, [[DEF1]], implicit $mode, implicit $exec
+    ; GCN-NEXT: SI_RETURN_TO_EPILOG [[V_FMAMK_F32_]]
+    %0:vgpr_32 = IMPLICIT_DEF
+    %1:vgpr_32 = IMPLICIT_DEF
+    %2:sreg_64 = S_MOV_B64_IMM_PSEUDO 1311768467750121200
+    %3:vgpr_32 = V_FMAC_F32_e64 0, %2.sub1, 0, %0, 0, %1, 0, 0, implicit $mode, implicit $exec
+    SI_RETURN_TO_EPILOG %3
+...
+
+---
+name:            fmac_sreg_64_sub1_src1_to_fmaak
+tracksRegLiveness: true
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: fmac_sreg_64_sub1_src1_to_fmaak
+    ; GCN: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[V_FMAMK_F32_:%[0-9]+]]:vgpr_32 = V_FMAMK_F32 [[DEF]], 305419896, [[DEF1]], implicit $mode, implicit $exec
+    ; GCN-NEXT: SI_RETURN_TO_EPILOG [[V_FMAMK_F32_]]
+    %0:vgpr_32 = IMPLICIT_DEF
+    %1:vgpr_32 = IMPLICIT_DEF
+    %2:sreg_64 = S_MOV_B64_IMM_PSEUDO 1311768467750121200
+    %3:vgpr_32 = V_FMAC_F32_e64 0, %0, 0, %2.sub1, 0, %1, 0, 0, implicit $mode, implicit $exec
+    SI_RETURN_TO_EPILOG %3
+...
+
+---
+name:            fma_sreg_64_sub0_to_fmaak
+tracksRegLiveness: true
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: fma_sreg_64_sub0_to_fmaak
+    ; GCN: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[V_FMAAK_F32_:%[0-9]+]]:vgpr_32 = V_FMAAK_F32 [[DEF]], [[DEF1]], 2882399984, implicit $mode, implicit $exec
+    ; GCN-NEXT: SI_RETURN_TO_EPILOG [[V_FMAAK_F32_]]
+    %0:vgpr_32 = IMPLICIT_DEF
+    %1:vgpr_32 = IMPLICIT_DEF
+    %2:sreg_64 = S_MOV_B64_IMM_PSEUDO 1311768467750121200
+    %3:vgpr_32 = V_FMA_F32_e64 0, %0, 0, %1, 0, %2.sub0, 0, 0, implicit $mode, implicit $exec
+    SI_RETURN_TO_EPILOG %3
+...
+
+---
+name:            fma_sreg_64_sub1_to_fmaak
+tracksRegLiveness: true
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: fma_sreg_64_sub1_to_fmaak
+    ; GCN: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[V_FMAAK_F32_:%[0-9]+]]:vgpr_32 = V_FMAAK_F32 [[DEF]], [[DEF1]], 305419896, implicit $mode, implicit $exec
+    ; GCN-NEXT: SI_RETURN_TO_EPILOG [[V_FMAAK_F32_]]
+    %0:vgpr_32 = IMPLICIT_DEF
+    %1:vgpr_32 = IMPLICIT_DEF
+    %2:sreg_64 = S_MOV_B64_IMM_PSEUDO 1311768467750121200
+    %3:vgpr_32 = V_FMA_F32_e64 0, %0, 0, %1, 0, %2.sub1, 0, 0, implicit $mode, implicit $exec
+    SI_RETURN_TO_EPILOG %3
+...
diff --git a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
index a462c19ce6..17b387be79 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
@@ -573,8 +573,7 @@ define hidden amdgpu_kernel void @clmem_read(ptr addrspace(1)  %buffer) {
 ; GFX900-NEXT:    v_mov_b32_e32 v3, s35
 ; GFX900-NEXT:    v_add_co_u32_e32 v1, vcc, s34, v1
 ; GFX900-NEXT:    v_addc_co_u32_e32 v2, vcc, v2, v3, vcc
-; GFX900-NEXT:    s_movk_i32 s0, 0x5000
-; GFX900-NEXT:    v_add_co_u32_e32 v1, vcc, s0, v1
+; GFX900-NEXT:    v_add_co_u32_e32 v1, vcc, 0x5000, v1
 ; GFX900-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX900-NEXT:    v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
 ; GFX900-NEXT:    s_movk_i32 s2, 0x7f
@@ -805,8 +804,7 @@ define hidden amdgpu_kernel void @clmem_read(ptr addrspace(1)  %buffer) {
 ; GFX90A-NEXT:    v_mov_b32_e32 v2, s35
 ; GFX90A-NEXT:    v_add_co_u32_e32 v1, vcc, s34, v1
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v3, vcc, v3, v2, vcc
-; GFX90A-NEXT:    s_movk_i32 s0, 0x5000
-; GFX90A-NEXT:    v_add_co_u32_e32 v2, vcc, s0, v1
+; GFX90A-NEXT:    v_add_co_u32_e32 v2, vcc, 0x5000, v1
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
 ; GFX90A-NEXT:    s_movk_i32 s2, 0x7f
 ; GFX90A-NEXT:    v_pk_mov_b32 v[4:5], 0, 0
diff --git a/llvm/test/CodeGen/AMDGPU/remat-smrd.mir b/llvm/test/CodeGen/AMDGPU/remat-smrd.mir
index fb443df164..753cb135fd 100644
--- a/llvm/test/CodeGen/AMDGPU/remat-smrd.mir
+++ b/llvm/test/CodeGen/AMDGPU/remat-smrd.mir
@@ -165,6 +165,128 @@ body:             |
     S_ENDPGM 0, implicit %0
 ...
 
+---
+name:            test_remat_s_load_dword_immx16_subreg
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $sgpr8_sgpr9, $vgpr0_vgpr1
+    ; GCN-LABEL: name: test_remat_s_load_dword_immx16_subreg
+    ; GCN: liveins: $sgpr8_sgpr9, $vgpr0_vgpr1
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: renamable $sgpr0_sgpr1 = COPY $sgpr8_sgpr9
+    ; GCN-NEXT: renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19 = S_LOAD_DWORDX16_IMM renamable $sgpr0_sgpr1, 0, 0 :: (dereferenceable invariant load (s512), align 4, addrspace 4)
+    ; GCN-NEXT: SI_SPILL_S512_SAVE killed renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, %stack.0, implicit $exec, implicit $sp_reg :: (store (s512) into %stack.0, align 4, addrspace 5)
+    ; GCN-NEXT: renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19 = S_LOAD_DWORDX16_IMM renamable $sgpr0_sgpr1, 128, 0 :: (dereferenceable invariant load (s512), align 4, addrspace 4)
+    ; GCN-NEXT: renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19 = SI_SPILL_S512_RESTORE %stack.0, implicit $exec, implicit $sp_reg :: (load (s512) from %stack.0, align 4, addrspace 5)
+    ; GCN-NEXT: S_NOP 0, implicit killed renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19
+    ; GCN-NEXT: renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19 = S_LOAD_DWORDX16_IMM renamable $sgpr0_sgpr1, 128, 0 :: (dereferenceable invariant load (s512), align 4, addrspace 4)
+    ; GCN-NEXT: renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19 = SI_SPILL_S512_RESTORE %stack.0, implicit $exec, implicit $sp_reg :: (load (s512) from %stack.0, align 4, addrspace 5)
+    ; GCN-NEXT: S_NOP 0, implicit killed renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11
+    ; GCN-NEXT: S_ENDPGM 0, implicit killed renamable $sgpr0_sgpr1
+    %0:sreg_64_xexec = COPY $sgpr8_sgpr9
+    %1:sgpr_512 = S_LOAD_DWORDX16_IMM %0, 0, 0 :: (invariant dereferenceable load (s512), align 4, addrspace 4)
+    %2:sgpr_512 = S_LOAD_DWORDX16_IMM %0, 128, 0 :: (invariant dereferenceable load (s512), align 4, addrspace 4)
+    %2.sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15:sgpr_512 = COPY %1.sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15:sgpr_512
+    S_NOP 0, implicit %2.sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15
+    %3:sgpr_512 = S_LOAD_DWORDX16_IMM %0, 128, 0 :: (invariant dereferenceable load (s512), align 4, addrspace 4)
+    %3.sub0_sub1_sub2_sub3_sub4_sub5_sub6_sub7:sgpr_512 = COPY %1.sub0_sub1_sub2_sub3_sub4_sub5_sub6_sub7:sgpr_512
+    S_NOP 0, implicit %3.sub0_sub1_sub2_sub3_sub4_sub5_sub6_sub7:sgpr_512
+
+    S_ENDPGM 0, implicit %0
+...
+
+---
+name:            test_remat_s_load_dword_immx16_subreg_no_shrinking_double_use
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $sgpr8_sgpr9, $vgpr0_vgpr1
+    ; GCN-LABEL: name: test_remat_s_load_dword_immx16_subreg_no_shrinking_double_use
+    ; GCN: liveins: $sgpr8_sgpr9, $vgpr0_vgpr1
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: renamable $sgpr0_sgpr1 = COPY $sgpr8_sgpr9
+    ; GCN-NEXT: renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19 = S_LOAD_DWORDX16_IMM renamable $sgpr0_sgpr1, 0, 0 :: (dereferenceable invariant load (s512), align 4, addrspace 4)
+    ; GCN-NEXT: SI_SPILL_S512_SAVE killed renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, %stack.0, implicit $exec, implicit $sp_reg :: (store (s512) into %stack.0, align 4, addrspace 5)
+    ; GCN-NEXT: renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19 = S_LOAD_DWORDX16_IMM renamable $sgpr0_sgpr1, 128, 0 :: (dereferenceable invariant load (s512), align 4, addrspace 4)
+    ; GCN-NEXT: renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19 = SI_SPILL_S512_RESTORE %stack.0, implicit $exec, implicit $sp_reg :: (load (s512) from %stack.0, align 4, addrspace 5)
+    ; GCN-NEXT: renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19 = KILL killed renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, implicit renamable $sgpr4_sgpr5_sgpr6_sgpr7
+    ; GCN-NEXT: S_NOP 0, implicit killed renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19
+    ; GCN-NEXT: renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19 = S_LOAD_DWORDX16_IMM renamable $sgpr0_sgpr1, 128, 0 :: (dereferenceable invariant load (s512), align 4, addrspace 4)
+    ; GCN-NEXT: renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19 = SI_SPILL_S512_RESTORE %stack.0, implicit $exec, implicit $sp_reg :: (load (s512) from %stack.0, align 4, addrspace 5)
+    ; GCN-NEXT: renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11 = KILL killed renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, implicit renamable $sgpr0_sgpr1
+    ; GCN-NEXT: S_NOP 0, implicit killed renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11
+    ; GCN-NEXT: S_ENDPGM 0, implicit killed renamable $sgpr0_sgpr1
+    %0:sreg_64_xexec = COPY $sgpr8_sgpr9
+    %1:sgpr_512 = S_LOAD_DWORDX16_IMM %0, 0, 0 :: (invariant dereferenceable load (s512), align 4, addrspace 4)
+    %2:sgpr_512 = S_LOAD_DWORDX16_IMM %0, 128, 0 :: (invariant dereferenceable load (s512), align 4, addrspace 4)
+    %2.sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15:sgpr_512 = COPY %1.sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15:sgpr_512, implicit %1.sub0_sub1_sub2_sub3:sgpr_512
+    S_NOP 0, implicit %2.sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15
+    %3:sgpr_512 = S_LOAD_DWORDX16_IMM %0, 128, 0 :: (invariant dereferenceable load (s512), align 4, addrspace 4)
+    %3.sub0_sub1_sub2_sub3_sub4_sub5_sub6_sub7:sgpr_512 = COPY %1.sub0_sub1_sub2_sub3_sub4_sub5_sub6_sub7:sgpr_512, implicit %0
+    S_NOP 0, implicit %3.sub0_sub1_sub2_sub3_sub4_sub5_sub6_sub7:sgpr_512
+    S_ENDPGM 0, implicit %0
+...
+
+---
+name:            test_remat_s_load_dword_immx16_subreg_no_shrinking_bundle
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $sgpr8_sgpr9, $vgpr0_vgpr1
+    ; GCN-LABEL: name: test_remat_s_load_dword_immx16_subreg_no_shrinking_bundle
+    ; GCN: liveins: $sgpr8_sgpr9, $vgpr0_vgpr1
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: renamable $sgpr0_sgpr1 = COPY $sgpr8_sgpr9
+    ; GCN-NEXT: renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19 = S_LOAD_DWORDX16_IMM renamable $sgpr0_sgpr1, 0, 0 :: (dereferenceable invariant load (s512), align 4, addrspace 4)
+    ; GCN-NEXT: SI_SPILL_S512_SAVE killed renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, %stack.0, implicit $exec, implicit $sp_reg :: (store (s512) into %stack.0, align 4, addrspace 5)
+    ; GCN-NEXT: renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19 = S_LOAD_DWORDX16_IMM renamable $sgpr0_sgpr1, 128, 0 :: (dereferenceable invariant load (s512), align 4, addrspace 4)
+    ; GCN-NEXT: renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19 = SI_SPILL_S512_RESTORE %stack.0, implicit $exec, implicit $sp_reg :: (load (s512) from %stack.0, align 4, addrspace 5)
+    ; GCN-NEXT: S_NOP 0, implicit killed renamable $sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19
+    ; GCN-NEXT: S_ENDPGM 0, implicit killed renamable $sgpr0_sgpr1
+    %0:sreg_64_xexec = COPY $sgpr8_sgpr9
+    %1:sgpr_512 = S_LOAD_DWORDX16_IMM %0, 0, 0 :: (invariant dereferenceable load (s512), align 4, addrspace 4)
+    %2:sgpr_512 = S_LOAD_DWORDX16_IMM %0, 128, 0 :: (invariant dereferenceable load (s512), align 4, addrspace 4)
+    %2.sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15:sgpr_512 = COPY %1.sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15:sgpr_512 {
+      internal %2.sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15:sgpr_512 = COPY %1.sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15:sgpr_512
+    }
+    S_NOP 0, implicit %2.sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15
+
+    S_ENDPGM 0, implicit %0
+...
+
+---
+name:            test_remat_s_load_dword_immx8_subreg
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $sgpr8_sgpr9, $vgpr0_vgpr1
+    ; GCN-LABEL: name: test_remat_s_load_dword_immx8_subreg
+    ; GCN: liveins: $sgpr8_sgpr9, $vgpr0_vgpr1
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: renamable $sgpr0_sgpr1 = COPY $sgpr8_sgpr9
+    ; GCN-NEXT: renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11 = S_LOAD_DWORDX8_IMM renamable $sgpr0_sgpr1, 0, 0 :: (dereferenceable invariant load (s256), align 4, addrspace 4)
+    ; GCN-NEXT: SI_SPILL_S256_SAVE killed renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11, %stack.0, implicit $exec, implicit $sp_reg :: (store (s256) into %stack.0, align 4, addrspace 5)
+    ; GCN-NEXT: renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11 = S_LOAD_DWORDX8_IMM renamable $sgpr0_sgpr1, 128, 0 :: (dereferenceable invariant load (s256), align 4, addrspace 4)
+    ; GCN-NEXT: renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11 = SI_SPILL_S256_RESTORE %stack.0, implicit $exec, implicit $sp_reg :: (load (s256) from %stack.0, align 4, addrspace 5)
+    ; GCN-NEXT: S_NOP 0, implicit killed renamable $sgpr8_sgpr9_sgpr10_sgpr11
+    ; GCN-NEXT: renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11 = S_LOAD_DWORDX8_IMM renamable $sgpr0_sgpr1, 128, 0 :: (dereferenceable invariant load (s256), align 4, addrspace 4)
+    ; GCN-NEXT: renamable $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11 = SI_SPILL_S256_RESTORE %stack.0, implicit $exec, implicit $sp_reg :: (load (s256) from %stack.0, align 4, addrspace 5)
+    ; GCN-NEXT: S_NOP 0, implicit killed renamable $sgpr4_sgpr5_sgpr6_sgpr7
+    ; GCN-NEXT: S_ENDPGM 0, implicit killed renamable $sgpr0_sgpr1
+    %0:sreg_64_xexec = COPY $sgpr8_sgpr9
+    %1:sgpr_256 = S_LOAD_DWORDX8_IMM %0, 0, 0 :: (invariant dereferenceable load (s256), align 4, addrspace 4)
+    %2:sgpr_256 = S_LOAD_DWORDX8_IMM %0, 128, 0 :: (invariant dereferenceable load (s256), align 4, addrspace 4)
+    %2.sub4_sub5_sub6_sub7:sgpr_256 = COPY %1.sub4_sub5_sub6_sub7:sgpr_256
+    S_NOP 0, implicit %2.sub4_sub5_sub6_sub7
+    %3:sgpr_256 = S_LOAD_DWORDX8_IMM %0, 128, 0 :: (invariant dereferenceable load (s256), align 4, addrspace 4)
+    %3.sub0_sub1_sub2_sub3:sgpr_256 = COPY %1.sub0_sub1_sub2_sub3:sgpr_256
+    S_NOP 0, implicit %3.sub0_sub1_sub2_sub3:sgpr_256
+
+    S_ENDPGM 0, implicit %0
+...
+
+
 ---
 name:            test_remat_s_get_waveid_in_workgroup
 tracksRegLiveness: true
@@ -360,24 +482,20 @@ body:             |
     S_ENDPGM 0, implicit %0
 ...
 
-#---------------------------------------------------------------------
-# Negative tests
-#---------------------------------------------------------------------
-
 ---
-name:            test_no_remat_s_load_dword_not_invariant
+name:            test_remat_s_load_dword_invariant_not_dereferenceable
 tracksRegLiveness: true
 body:             |
   bb.0:
     liveins: $sgpr8_sgpr9
-    ; GCN-LABEL: name: test_no_remat_s_load_dword_not_invariant
+    ; GCN-LABEL: name: test_remat_s_load_dword_invariant_not_dereferenceable
     ; GCN: liveins: $sgpr8_sgpr9
     ; GCN-NEXT: {{  $}}
     ; GCN-NEXT: renamable $sgpr2_sgpr3 = COPY $sgpr8_sgpr9
-    ; GCN-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM renamable $sgpr2_sgpr3, 0, 0 :: (dereferenceable load (s32), addrspace 4)
+    ; GCN-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM renamable $sgpr2_sgpr3, 0, 0 :: (invariant load (s32), addrspace 4)
     ; GCN-NEXT: SI_SPILL_S32_SAVE killed renamable $sgpr0, %stack.0, implicit $exec, implicit $sp_reg :: (store (s32) into %stack.0, addrspace 5)
-    ; GCN-NEXT: renamable $sgpr1 = S_LOAD_DWORD_IMM renamable $sgpr2_sgpr3, 4, 0 :: (dereferenceable load (s32), addrspace 4)
-    ; GCN-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM renamable $sgpr2_sgpr3, 8, 0 :: (dereferenceable load (s32), addrspace 4)
+    ; GCN-NEXT: renamable $sgpr1 = S_LOAD_DWORD_IMM renamable $sgpr2_sgpr3, 4, 0 :: (invariant load (s32), addrspace 4)
+    ; GCN-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM renamable $sgpr2_sgpr3, 8, 0 :: (invariant load (s32), addrspace 4)
     ; GCN-NEXT: SI_SPILL_S32_SAVE killed renamable $sgpr0, %stack.1, implicit $exec, implicit $sp_reg :: (store (s32) into %stack.1, addrspace 5)
     ; GCN-NEXT: renamable $sgpr0 = SI_SPILL_S32_RESTORE %stack.0, implicit $exec, implicit $sp_reg :: (load (s32) from %stack.0, addrspace 5)
     ; GCN-NEXT: S_NOP 0, implicit killed renamable $sgpr0
@@ -386,29 +504,34 @@ body:             |
     ; GCN-NEXT: S_NOP 0, implicit killed renamable $sgpr0
     ; GCN-NEXT: S_ENDPGM 0, implicit killed renamable $sgpr2_sgpr3
     %0:sreg_64_xexec = COPY $sgpr8_sgpr9
-    %1:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0, 0, 0 :: (dereferenceable load (s32), addrspace 4)
-    %2:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0, 4, 0 :: (dereferenceable load (s32), addrspace 4)
-    %3:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0, 8, 0 :: (dereferenceable load (s32), addrspace 4)
+    %1:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0, 0, 0 :: (invariant load (s32), addrspace 4)
+    %2:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0, 4, 0 :: (invariant load (s32), addrspace 4)
+    %3:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0, 8, 0 :: (invariant load (s32), addrspace 4)
     S_NOP 0, implicit %1
     S_NOP 0, implicit %2
     S_NOP 0, implicit %3
     S_ENDPGM 0, implicit %0
 ...
 
+
+#---------------------------------------------------------------------
+# Negative tests
+#---------------------------------------------------------------------
+
 ---
-name:            test_no_remat_s_load_dword_not_dereferenceable
+name:            test_no_remat_s_load_dword_not_invariant
 tracksRegLiveness: true
 body:             |
   bb.0:
     liveins: $sgpr8_sgpr9
-    ; GCN-LABEL: name: test_no_remat_s_load_dword_not_dereferenceable
+    ; GCN-LABEL: name: test_no_remat_s_load_dword_not_invariant
     ; GCN: liveins: $sgpr8_sgpr9
     ; GCN-NEXT: {{  $}}
     ; GCN-NEXT: renamable $sgpr2_sgpr3 = COPY $sgpr8_sgpr9
-    ; GCN-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM renamable $sgpr2_sgpr3, 0, 0 :: (invariant load (s32), addrspace 4)
+    ; GCN-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM renamable $sgpr2_sgpr3, 0, 0 :: (dereferenceable load (s32), addrspace 4)
     ; GCN-NEXT: SI_SPILL_S32_SAVE killed renamable $sgpr0, %stack.0, implicit $exec, implicit $sp_reg :: (store (s32) into %stack.0, addrspace 5)
-    ; GCN-NEXT: renamable $sgpr1 = S_LOAD_DWORD_IMM renamable $sgpr2_sgpr3, 4, 0 :: (invariant load (s32), addrspace 4)
-    ; GCN-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM renamable $sgpr2_sgpr3, 8, 0 :: (invariant load (s32), addrspace 4)
+    ; GCN-NEXT: renamable $sgpr1 = S_LOAD_DWORD_IMM renamable $sgpr2_sgpr3, 4, 0 :: (dereferenceable load (s32), addrspace 4)
+    ; GCN-NEXT: renamable $sgpr0 = S_LOAD_DWORD_IMM renamable $sgpr2_sgpr3, 8, 0 :: (dereferenceable load (s32), addrspace 4)
     ; GCN-NEXT: SI_SPILL_S32_SAVE killed renamable $sgpr0, %stack.1, implicit $exec, implicit $sp_reg :: (store (s32) into %stack.1, addrspace 5)
     ; GCN-NEXT: renamable $sgpr0 = SI_SPILL_S32_RESTORE %stack.0, implicit $exec, implicit $sp_reg :: (load (s32) from %stack.0, addrspace 5)
     ; GCN-NEXT: S_NOP 0, implicit killed renamable $sgpr0
@@ -417,9 +540,9 @@ body:             |
     ; GCN-NEXT: S_NOP 0, implicit killed renamable $sgpr0
     ; GCN-NEXT: S_ENDPGM 0, implicit killed renamable $sgpr2_sgpr3
     %0:sreg_64_xexec = COPY $sgpr8_sgpr9
-    %1:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0, 0, 0 :: (invariant load (s32), addrspace 4)
-    %2:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0, 4, 0 :: (invariant load (s32), addrspace 4)
-    %3:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0, 8, 0 :: (invariant load (s32), addrspace 4)
+    %1:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0, 0, 0 :: (dereferenceable load (s32), addrspace 4)
+    %2:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0, 4, 0 :: (dereferenceable load (s32), addrspace 4)
+    %3:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0, 8, 0 :: (dereferenceable load (s32), addrspace 4)
     S_NOP 0, implicit %1
     S_NOP 0, implicit %2
     S_NOP 0, implicit %3
@@ -489,3 +612,32 @@ body:             |
     S_NOP 0, implicit %3
     S_ENDPGM 0, implicit %0
 ...
+
+---
+name:            test_no_remat_s_load_dword_immx2_tied
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $sgpr8_sgpr9
+    ; GCN-LABEL: name: test_no_remat_s_load_dword_immx2_tied
+    ; GCN: liveins: $sgpr8_sgpr9
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: renamable $sgpr0_sgpr1 = COPY $sgpr8_sgpr9
+    ; GCN-NEXT: renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM renamable $sgpr0_sgpr1, 0, 0 :: (dereferenceable invariant load (s64), addrspace 4)
+    ; GCN-NEXT: SI_SPILL_S64_SAVE killed renamable $sgpr2_sgpr3, %stack.1, implicit $exec, implicit $sp_reg :: (store (s64) into %stack.1, align 4, addrspace 5)
+    ; GCN-NEXT: renamable $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM renamable $sgpr0_sgpr1, 4, 0 :: (dereferenceable invariant load (s64), addrspace 4)
+    ; GCN-NEXT: SI_SPILL_S64_SAVE killed renamable $sgpr2_sgpr3, %stack.0, implicit $exec, implicit $sp_reg :: (store (s64) into %stack.0, align 4, addrspace 5)
+    ; GCN-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM renamable $sgpr0_sgpr1, 8, 0 :: (dereferenceable invariant load (s64), addrspace 4)
+    ; GCN-NEXT: renamable $sgpr2_sgpr3 = SI_SPILL_S64_RESTORE %stack.1, implicit $exec, implicit $sp_reg :: (load (s64) from %stack.1, align 4, addrspace 5)
+    ; GCN-NEXT: S_NOP 0, implicit killed renamable $sgpr2_sgpr3
+    ; GCN-NEXT: renamable $sgpr2_sgpr3 = SI_SPILL_S64_RESTORE %stack.0, implicit $exec, implicit $sp_reg :: (load (s64) from %stack.0, align 4, addrspace 5)
+    ; GCN-NEXT: S_NOP 0, implicit killed renamable $sgpr2_sgpr3
+    ; GCN-NEXT: S_NOP 0, implicit killed renamable $sgpr0_sgpr1
+    undef %0.sub0_sub1:sgpr_256 = COPY $sgpr8_sgpr9
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0.sub0_sub1:sgpr_256, 0, 0 :: (invariant dereferenceable load (s64), addrspace 4)
+    %2:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0.sub0_sub1:sgpr_256, 4, 0 :: (invariant dereferenceable load (s64), addrspace 4)
+    %0.sub0_sub1:sgpr_256 = S_LOAD_DWORDX2_IMM %0.sub0_sub1:sgpr_256, 8, 0 :: (invariant dereferenceable load (s64), addrspace 4)
+    S_NOP 0, implicit %1
+    S_NOP 0, implicit %2
+    S_NOP 0, implicit %0.sub0_sub1:sgpr_256
+...
diff --git a/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll b/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
index 79e96b45a9..086fab3f52 100644
--- a/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
+++ b/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
@@ -2098,6 +2098,113 @@ bb11:                                             ; preds = %bb10, %bb2
   br label %bb1
 }
 
+define void @crash_lshlrevb16_not_reg_op() {
+; NOSDWA-LABEL: crash_lshlrevb16_not_reg_op:
+; NOSDWA:       ; %bb.0: ; %bb0
+; NOSDWA-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; NOSDWA-NEXT:    s_mov_b64 s[4:5], 0
+; NOSDWA-NEXT:    s_and_b64 vcc, exec, -1
+; NOSDWA-NEXT:    v_lshlrev_b16_e64 v3, 8, 1
+; NOSDWA-NEXT:  .LBB22_1: ; %bb1
+; NOSDWA-NEXT:    ; =>This Inner Loop Header: Depth=1
+; NOSDWA-NEXT:    v_mov_b32_e32 v0, s4
+; NOSDWA-NEXT:    v_mov_b32_e32 v2, 0xff
+; NOSDWA-NEXT:    s_lshl_b32 s6, s4, 3
+; NOSDWA-NEXT:    v_mov_b32_e32 v1, s5
+; NOSDWA-NEXT:    s_mov_b64 s[4:5], 1
+; NOSDWA-NEXT:    v_and_b32_e32 v2, s4, v2
+; NOSDWA-NEXT:    v_or_b32_e32 v2, v2, v3
+; NOSDWA-NEXT:    v_lshrrev_b16_e32 v2, s6, v2
+; NOSDWA-NEXT:    flat_store_byte v[0:1], v2
+; NOSDWA-NEXT:    s_mov_b64 vcc, vcc
+; NOSDWA-NEXT:    s_cbranch_vccnz .LBB22_1
+; NOSDWA-NEXT:  ; %bb.2: ; %DummyReturnBlock
+; NOSDWA-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; NOSDWA-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX89-LABEL: crash_lshlrevb16_not_reg_op:
+; GFX89:       ; %bb.0: ; %bb0
+; GFX89-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX89-NEXT:    s_mov_b64 s[4:5], 0
+; GFX89-NEXT:    s_and_b64 vcc, exec, -1
+; GFX89-NEXT:    v_lshlrev_b16_e64 v0, 8, 1
+; GFX89-NEXT:  .LBB22_1: ; %bb1
+; GFX89-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX89-NEXT:    v_mov_b32_e32 v3, s4
+; GFX89-NEXT:    s_lshl_b32 s6, s4, 3
+; GFX89-NEXT:    v_mov_b32_e32 v1, s4
+; GFX89-NEXT:    v_or_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX89-NEXT:    v_mov_b32_e32 v2, s5
+; GFX89-NEXT:    s_mov_b64 s[4:5], 1
+; GFX89-NEXT:    v_lshrrev_b16_e32 v3, s6, v3
+; GFX89-NEXT:    flat_store_byte v[1:2], v3
+; GFX89-NEXT:    s_mov_b64 vcc, vcc
+; GFX89-NEXT:    s_cbranch_vccnz .LBB22_1
+; GFX89-NEXT:  ; %bb.2: ; %DummyReturnBlock
+; GFX89-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX89-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: crash_lshlrevb16_not_reg_op:
+; GFX9:       ; %bb.0: ; %bb0
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:    v_lshlrev_b16_e64 v0, 8, 1
+; GFX9-NEXT:    s_and_b64 vcc, exec, -1
+; GFX9-NEXT:    v_or_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:  .LBB22_1: ; %bb1
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_lshl_b32 s6, s4, 3
+; GFX9-NEXT:    v_mov_b32_e32 v1, s4
+; GFX9-NEXT:    v_mov_b32_e32 v2, s5
+; GFX9-NEXT:    s_mov_b64 s[4:5], 1
+; GFX9-NEXT:    v_lshrrev_b16_e32 v3, s6, v0
+; GFX9-NEXT:    flat_store_byte v[1:2], v3
+; GFX9-NEXT:    s_mov_b64 vcc, vcc
+; GFX9-NEXT:    s_cbranch_vccnz .LBB22_1
+; GFX9-NEXT:  ; %bb.2: ; %DummyReturnBlock
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: crash_lshlrevb16_not_reg_op:
+; GFX10:       ; %bb.0: ; %bb0
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_lshlrev_b16 v0, 8, 1
+; GFX10-NEXT:    s_mov_b32 vcc_lo, exec_lo
+; GFX10-NEXT:    v_or_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT:    s_mov_b64 s[4:5], 0
+; GFX10-NEXT:  .LBB22_1: ; %bb1
+; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT:    s_lshl_b32 s6, s4, 3
+; GFX10-NEXT:    v_mov_b32_e32 v1, s4
+; GFX10-NEXT:    v_mov_b32_e32 v2, s5
+; GFX10-NEXT:    v_lshrrev_b16 v3, s6, v0
+; GFX10-NEXT:    s_mov_b64 s[4:5], 1
+; GFX10-NEXT:    flat_store_byte v[1:2], v3
+; GFX10-NEXT:    s_cbranch_vccnz .LBB22_1
+; GFX10-NEXT:  ; %bb.2: ; %DummyReturnBlock
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+  %1 = alloca [2 x i8], align 1, addrspace(5)
+  %2 = getelementptr [2 x i8], ptr addrspace(5) %1, i32 0, i32 1
+  br label %bb0
+
+bb0:
+  store i8 1, ptr addrspace(5) %2, align 1
+  br label %bb1
+
+bb1:
+  %3 = phi i64 [ 1, %bb1 ], [ 0, %bb0 ]
+  %4 = trunc i64 %3 to i32
+  %5 = getelementptr i8, ptr addrspace(5) %1, i32 %4
+  %6 = load i8, ptr addrspace(5) %5, align 1
+  %7 = getelementptr i8, ptr null, i64 %3
+  store i8 %6, ptr %7, align 1
+  br i1 false, label %bb2, label %bb1
+
+bb2:
+  br label %bb0
+}
+
 declare i32 @llvm.amdgcn.workitem.id.x()
 
 attributes #0 = { "denormal-fp-math"="preserve-sign,preserve-sign" }
diff --git a/llvm/test/CodeGen/AMDGPU/udiv.ll b/llvm/test/CodeGen/AMDGPU/udiv.ll
index 012b3f9767..5d1db03a1a 100644
--- a/llvm/test/CodeGen/AMDGPU/udiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/udiv.ll
@@ -2520,59 +2520,38 @@ define i64 @v_test_udiv64_mulhi_fold(i64 %arg) {
 ; SI-LABEL: v_test_udiv64_mulhi_fold:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; SI-NEXT:    v_madak_f32 v2, 0, v2, 0x47c35000
-; SI-NEXT:    v_rcp_f32_e32 v2, v2
+; SI-NEXT:    s_mov_b32 s4, 0x346d900
+; SI-NEXT:    s_add_u32 s4, 0x4237, s4
+; SI-NEXT:    v_mov_b32_e32 v2, 0xa9000000
+; SI-NEXT:    v_add_i32_e32 v2, vcc, s4, v2
+; SI-NEXT:    s_addc_u32 s5, 0, 0
+; SI-NEXT:    s_or_b32 s4, vcc_lo, vcc_hi
+; SI-NEXT:    s_cmp_lg_u32 s4, 0
 ; SI-NEXT:    s_mov_b32 s4, 0xfffe7960
-; SI-NEXT:    v_mul_f32_e32 v2, 0x5f7ffffc, v2
-; SI-NEXT:    v_mul_f32_e32 v3, 0x2f800000, v2
-; SI-NEXT:    v_trunc_f32_e32 v3, v3
-; SI-NEXT:    v_madmk_f32 v2, v3, 0xcf800000, v2
-; SI-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; SI-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; SI-NEXT:    v_mul_hi_u32 v4, v2, s4
-; SI-NEXT:    v_mul_lo_u32 v6, v3, s4
-; SI-NEXT:    v_mul_lo_u32 v5, v2, s4
-; SI-NEXT:    v_sub_i32_e32 v4, vcc, v4, v2
-; SI-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
-; SI-NEXT:    v_mul_hi_u32 v7, v2, v5
-; SI-NEXT:    v_mul_lo_u32 v6, v2, v4
-; SI-NEXT:    v_mul_hi_u32 v8, v2, v4
-; SI-NEXT:    v_mul_hi_u32 v9, v3, v4
-; SI-NEXT:    v_mul_lo_u32 v4, v3, v4
-; SI-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
-; SI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v8, vcc
-; SI-NEXT:    v_mul_lo_u32 v8, v3, v5
-; SI-NEXT:    v_mul_hi_u32 v5, v3, v5
-; SI-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
-; SI-NEXT:    v_addc_u32_e32 v5, vcc, v7, v5, vcc
-; SI-NEXT:    v_addc_u32_e32 v6, vcc, 0, v9, vcc
-; SI-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; SI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc
-; SI-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
-; SI-NEXT:    v_addc_u32_e32 v3, vcc, v3, v5, vcc
-; SI-NEXT:    v_mul_hi_u32 v4, v2, s4
-; SI-NEXT:    v_mul_lo_u32 v5, v3, s4
-; SI-NEXT:    v_mul_lo_u32 v6, v2, s4
+; SI-NEXT:    v_mul_hi_u32 v3, v2, s4
+; SI-NEXT:    v_mul_lo_u32 v4, v2, s4
+; SI-NEXT:    s_addc_u32 s5, s5, 0xa7c5
+; SI-NEXT:    s_mul_i32 s6, s5, 0xfffe7960
+; SI-NEXT:    v_sub_i32_e32 v3, vcc, v3, v2
+; SI-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; SI-NEXT:    v_mul_lo_u32 v5, v2, v3
+; SI-NEXT:    v_mul_hi_u32 v6, v2, v4
+; SI-NEXT:    v_mul_hi_u32 v7, v2, v3
+; SI-NEXT:    v_mul_hi_u32 v8, s5, v3
+; SI-NEXT:    v_mul_lo_u32 v3, s5, v3
+; SI-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; SI-NEXT:    v_addc_u32_e32 v6, vcc, 0, v7, vcc
+; SI-NEXT:    v_mul_lo_u32 v7, s5, v4
+; SI-NEXT:    v_mul_hi_u32 v4, s5, v4
 ; SI-NEXT:    s_mov_b32 s4, 0x186a0
-; SI-NEXT:    v_sub_i32_e32 v4, vcc, v4, v2
-; SI-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; SI-NEXT:    v_mul_lo_u32 v5, v2, v4
-; SI-NEXT:    v_mul_hi_u32 v7, v2, v6
-; SI-NEXT:    v_mul_hi_u32 v8, v2, v4
-; SI-NEXT:    v_mul_hi_u32 v9, v3, v4
-; SI-NEXT:    v_mul_lo_u32 v4, v3, v4
-; SI-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
-; SI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v8, vcc
-; SI-NEXT:    v_mul_lo_u32 v8, v3, v6
-; SI-NEXT:    v_mul_hi_u32 v6, v3, v6
-; SI-NEXT:    v_add_i32_e32 v5, vcc, v5, v8
-; SI-NEXT:    v_addc_u32_e32 v5, vcc, v7, v6, vcc
-; SI-NEXT:    v_addc_u32_e32 v6, vcc, 0, v9, vcc
-; SI-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; SI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc
-; SI-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
-; SI-NEXT:    v_addc_u32_e32 v3, vcc, v3, v5, vcc
+; SI-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
+; SI-NEXT:    v_addc_u32_e32 v4, vcc, v6, v4, vcc
+; SI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v8, vcc
+; SI-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; SI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v5, vcc
+; SI-NEXT:    v_mov_b32_e32 v5, s5
+; SI-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; SI-NEXT:    v_addc_u32_e32 v3, vcc, v5, v4, vcc
 ; SI-NEXT:    v_mul_lo_u32 v4, v0, v3
 ; SI-NEXT:    v_mul_hi_u32 v5, v0, v2
 ; SI-NEXT:    v_mul_hi_u32 v6, v0, v3
@@ -2619,39 +2598,20 @@ define i64 @v_test_udiv64_mulhi_fold(i64 %arg) {
 ; VI-LABEL: v_test_udiv64_mulhi_fold:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; VI-NEXT:    v_madak_f32 v2, 0, v2, 0x47c35000
-; VI-NEXT:    v_rcp_f32_e32 v2, v2
+; VI-NEXT:    v_mov_b32_e32 v4, 0xa7c5
+; VI-NEXT:    v_mul_u32_u24_e32 v3, 0x500, v4
+; VI-NEXT:    v_mul_hi_u32_u24_e32 v2, 0x500, v4
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 0x4237, v3
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v2, vcc
+; VI-NEXT:    v_add_u32_e32 v6, vcc, 0xa9000000, v3
 ; VI-NEXT:    s_mov_b32 s6, 0xfffe7960
-; VI-NEXT:    v_mul_f32_e32 v2, 0x5f7ffffc, v2
-; VI-NEXT:    v_mul_f32_e32 v3, 0x2f800000, v2
-; VI-NEXT:    v_trunc_f32_e32 v3, v3
-; VI-NEXT:    v_madmk_f32 v2, v3, 0xcf800000, v2
-; VI-NEXT:    v_cvt_u32_f32_e32 v6, v2
-; VI-NEXT:    v_cvt_u32_f32_e32 v7, v3
-; VI-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v6, s6, 0
-; VI-NEXT:    v_mul_lo_u32 v4, v7, s6
-; VI-NEXT:    v_sub_u32_e32 v3, vcc, v3, v6
-; VI-NEXT:    v_add_u32_e32 v8, vcc, v3, v4
-; VI-NEXT:    v_mul_hi_u32 v5, v6, v2
-; VI-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], v6, v8, 0
-; VI-NEXT:    v_add_u32_e32 v9, vcc, v5, v3
-; VI-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v7, v2, 0
-; VI-NEXT:    v_addc_u32_e32 v10, vcc, 0, v4, vcc
-; VI-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v7, v8, 0
-; VI-NEXT:    v_add_u32_e32 v2, vcc, v9, v2
-; VI-NEXT:    v_addc_u32_e32 v2, vcc, v10, v3, vcc
-; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
-; VI-NEXT:    v_add_u32_e32 v2, vcc, v2, v4
-; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; VI-NEXT:    v_add_u32_e32 v6, vcc, v6, v2
-; VI-NEXT:    v_addc_u32_e32 v7, vcc, v7, v3, vcc
 ; VI-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v6, s6, 0
+; VI-NEXT:    v_addc_u32_e32 v7, vcc, v5, v4, vcc
 ; VI-NEXT:    v_mul_lo_u32 v4, v7, s6
 ; VI-NEXT:    v_sub_u32_e32 v3, vcc, v3, v6
+; VI-NEXT:    v_mul_hi_u32 v8, v6, v2
 ; VI-NEXT:    v_add_u32_e32 v5, vcc, v4, v3
 ; VI-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], v6, v5, 0
-; VI-NEXT:    v_mul_hi_u32 v8, v6, v2
 ; VI-NEXT:    v_add_u32_e32 v8, vcc, v8, v3
 ; VI-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v7, v2, 0
 ; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v4, vcc
@@ -2706,39 +2666,20 @@ define i64 @v_test_udiv64_mulhi_fold(i64 %arg) {
 ; GCN-LABEL: v_test_udiv64_mulhi_fold:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GCN-NEXT:    v_madak_f32 v2, 0, v2, 0x47c35000
-; GCN-NEXT:    v_rcp_f32_e32 v2, v2
+; GCN-NEXT:    v_mov_b32_e32 v4, 0xa7c5
+; GCN-NEXT:    v_mul_u32_u24_e32 v3, 0x500, v4
+; GCN-NEXT:    v_mul_hi_u32_u24_e32 v2, 0x500, v4
+; GCN-NEXT:    v_add_u32_e32 v3, vcc, 0x4237, v3
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v2, vcc
+; GCN-NEXT:    v_add_u32_e32 v6, vcc, 0xa9000000, v3
 ; GCN-NEXT:    s_mov_b32 s6, 0xfffe7960
-; GCN-NEXT:    v_mul_f32_e32 v2, 0x5f7ffffc, v2
-; GCN-NEXT:    v_mul_f32_e32 v3, 0x2f800000, v2
-; GCN-NEXT:    v_trunc_f32_e32 v3, v3
-; GCN-NEXT:    v_madmk_f32 v2, v3, 0xcf800000, v2
-; GCN-NEXT:    v_cvt_u32_f32_e32 v6, v2
-; GCN-NEXT:    v_cvt_u32_f32_e32 v7, v3
-; GCN-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v6, s6, 0
-; GCN-NEXT:    v_mul_lo_u32 v4, v7, s6
-; GCN-NEXT:    v_sub_u32_e32 v3, vcc, v3, v6
-; GCN-NEXT:    v_add_u32_e32 v8, vcc, v3, v4
-; GCN-NEXT:    v_mul_hi_u32 v5, v6, v2
-; GCN-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], v6, v8, 0
-; GCN-NEXT:    v_add_u32_e32 v9, vcc, v5, v3
-; GCN-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v7, v2, 0
-; GCN-NEXT:    v_addc_u32_e32 v10, vcc, 0, v4, vcc
-; GCN-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v7, v8, 0
-; GCN-NEXT:    v_add_u32_e32 v2, vcc, v9, v2
-; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v10, v3, vcc
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
-; GCN-NEXT:    v_add_u32_e32 v2, vcc, v2, v4
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GCN-NEXT:    v_add_u32_e32 v6, vcc, v6, v2
-; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v7, v3, vcc
 ; GCN-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v6, s6, 0
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v5, v4, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v4, v7, s6
 ; GCN-NEXT:    v_sub_u32_e32 v3, vcc, v3, v6
+; GCN-NEXT:    v_mul_hi_u32 v8, v6, v2
 ; GCN-NEXT:    v_add_u32_e32 v5, vcc, v4, v3
 ; GCN-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], v6, v5, 0
-; GCN-NEXT:    v_mul_hi_u32 v8, v6, v2
 ; GCN-NEXT:    v_add_u32_e32 v8, vcc, v8, v3
 ; GCN-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v7, v2, 0
 ; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v4, vcc
diff --git a/llvm/test/CodeGen/AMDGPU/udiv64.ll b/llvm/test/CodeGen/AMDGPU/udiv64.ll
index e23f3cfad8..7aa36a8b37 100644
--- a/llvm/test/CodeGen/AMDGPU/udiv64.ll
+++ b/llvm/test/CodeGen/AMDGPU/udiv64.ll
@@ -1326,64 +1326,37 @@ define i64 @v_test_udiv_pow2_k_den_i64(i64 %x) {
 define amdgpu_kernel void @s_test_udiv_k_den_i64(ptr addrspace(1) %out, i64 %x) {
 ; GCN-LABEL: s_test_udiv_k_den_i64:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    v_mov_b32_e32 v0, 0x4f800000
-; GCN-NEXT:    v_madak_f32 v0, 0, v0, 0x41c00000
-; GCN-NEXT:    v_rcp_f32_e32 v0, v0
-; GCN-NEXT:    s_movk_i32 s8, 0xffe8
 ; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
-; GCN-NEXT:    s_mov_b32 s3, 0xf000
-; GCN-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
-; GCN-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
-; GCN-NEXT:    v_trunc_f32_e32 v1, v1
-; GCN-NEXT:    v_madmk_f32 v0, v1, 0xcf800000, v0
-; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    s_add_u32 s1, 0, 0xaaaa0000
+; GCN-NEXT:    v_mov_b32_e32 v0, 0xffffffe8
+; GCN-NEXT:    v_mul_hi_u32 v0, s1, v0
+; GCN-NEXT:    s_addc_u32 s8, 0, 42
+; GCN-NEXT:    s_add_i32 s8, s8, 0xaaaaa80
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_mov_b32 s0, s4
-; GCN-NEXT:    v_mul_hi_u32 v2, v0, s8
-; GCN-NEXT:    v_mul_lo_u32 v4, v1, s8
-; GCN-NEXT:    v_mul_lo_u32 v3, v0, s8
-; GCN-NEXT:    s_mov_b32 s1, s5
-; GCN-NEXT:    v_sub_i32_e32 v2, vcc, v2, v0
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
-; GCN-NEXT:    v_mul_hi_u32 v5, v0, v3
-; GCN-NEXT:    v_mul_lo_u32 v4, v0, v2
-; GCN-NEXT:    v_mul_hi_u32 v6, v0, v2
-; GCN-NEXT:    v_mul_hi_u32 v7, v1, v2
-; GCN-NEXT:    v_mul_lo_u32 v2, v1, v2
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc
-; GCN-NEXT:    v_mul_lo_u32 v6, v1, v3
-; GCN-NEXT:    v_mul_hi_u32 v3, v1, v3
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v5, v3, vcc
-; GCN-NEXT:    v_addc_u32_e32 v4, vcc, 0, v7, vcc
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
-; GCN-NEXT:    v_mul_hi_u32 v2, v0, s8
-; GCN-NEXT:    v_mul_lo_u32 v3, v1, s8
-; GCN-NEXT:    v_mul_lo_u32 v4, v0, s8
-; GCN-NEXT:    v_sub_i32_e32 v2, vcc, v2, v0
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GCN-NEXT:    v_mul_lo_u32 v3, v0, v2
-; GCN-NEXT:    v_mul_hi_u32 v5, v0, v4
-; GCN-NEXT:    v_mul_hi_u32 v6, v0, v2
-; GCN-NEXT:    v_mul_hi_u32 v7, v1, v2
-; GCN-NEXT:    v_mul_lo_u32 v2, v1, v2
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v5, v3
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc
-; GCN-NEXT:    v_mul_lo_u32 v6, v1, v4
-; GCN-NEXT:    v_mul_hi_u32 v4, v1, v4
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v6
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v5, v4, vcc
-; GCN-NEXT:    v_addc_u32_e32 v4, vcc, 0, v7, vcc
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    s_mul_i32 s4, s1, 0xffffffe8
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s1, v0
+; GCN-NEXT:    s_mul_i32 s9, s8, 0xffffffe8
+; GCN-NEXT:    v_mov_b32_e32 v1, s4
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s9, v0
+; GCN-NEXT:    v_mul_hi_u32 v2, s8, v1
+; GCN-NEXT:    v_mul_lo_u32 v3, s1, v0
+; GCN-NEXT:    v_mul_hi_u32 v1, s1, v1
+; GCN-NEXT:    v_mul_hi_u32 v4, s1, v0
+; GCN-NEXT:    s_mul_i32 s4, s8, s4
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
+; GCN-NEXT:    v_mul_hi_u32 v4, s8, v0
+; GCN-NEXT:    v_mul_lo_u32 v0, s8, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s4, v1
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v3, v2, vcc
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v2, vcc
+; GCN-NEXT:    v_mov_b32_e32 v2, s8
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s1, v0
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v2, v1, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v2, s6, v1
 ; GCN-NEXT:    v_mul_hi_u32 v3, s6, v0
 ; GCN-NEXT:    v_mul_hi_u32 v4, s6, v1
@@ -1393,6 +1366,8 @@ define amdgpu_kernel void @s_test_udiv_k_den_i64(ptr addrspace(1) %out, i64 %x)
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v4, s7, v0
 ; GCN-NEXT:    v_mul_hi_u32 v0, s7, v0
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    s_mov_b32 s1, s5
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
 ; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v5, vcc
@@ -1502,58 +1477,33 @@ define i64 @v_test_udiv_k_den_i64(i64 %x) {
 ; GCN-LABEL: v_test_udiv_k_den_i64:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    v_mov_b32_e32 v2, 0x4f800000
-; GCN-NEXT:    v_madak_f32 v2, 0, v2, 0x41c00000
-; GCN-NEXT:    v_rcp_f32_e32 v2, v2
-; GCN-NEXT:    s_movk_i32 s4, 0xffe8
-; GCN-NEXT:    v_mul_f32_e32 v2, 0x5f7ffffc, v2
-; GCN-NEXT:    v_mul_f32_e32 v3, 0x2f800000, v2
-; GCN-NEXT:    v_trunc_f32_e32 v3, v3
-; GCN-NEXT:    v_madmk_f32 v2, v3, 0xcf800000, v2
-; GCN-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; GCN-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; GCN-NEXT:    v_mul_hi_u32 v4, v2, s4
-; GCN-NEXT:    v_mul_lo_u32 v6, v3, s4
-; GCN-NEXT:    v_mul_lo_u32 v5, v2, s4
-; GCN-NEXT:    v_sub_i32_e32 v4, vcc, v4, v2
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
-; GCN-NEXT:    v_mul_hi_u32 v7, v2, v5
-; GCN-NEXT:    v_mul_lo_u32 v6, v2, v4
-; GCN-NEXT:    v_mul_hi_u32 v8, v2, v4
-; GCN-NEXT:    v_mul_hi_u32 v9, v3, v4
-; GCN-NEXT:    v_mul_lo_u32 v4, v3, v4
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
-; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v8, vcc
-; GCN-NEXT:    v_mul_lo_u32 v8, v3, v5
-; GCN-NEXT:    v_mul_hi_u32 v5, v3, v5
-; GCN-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v5, vcc
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v9, vcc
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v3, v5, vcc
-; GCN-NEXT:    v_mul_hi_u32 v4, v2, s4
-; GCN-NEXT:    v_mul_lo_u32 v5, v3, s4
-; GCN-NEXT:    v_mul_lo_u32 v6, v2, s4
-; GCN-NEXT:    v_sub_i32_e32 v4, vcc, v4, v2
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_mul_lo_u32 v5, v2, v4
-; GCN-NEXT:    v_mul_hi_u32 v7, v2, v6
-; GCN-NEXT:    v_mul_hi_u32 v8, v2, v4
-; GCN-NEXT:    v_mul_hi_u32 v9, v3, v4
-; GCN-NEXT:    v_mul_lo_u32 v4, v3, v4
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
-; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v8, vcc
-; GCN-NEXT:    v_mul_lo_u32 v8, v3, v6
-; GCN-NEXT:    v_mul_hi_u32 v6, v3, v6
-; GCN-NEXT:    v_add_i32_e32 v5, vcc, v5, v8
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v6, vcc
-; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v9, vcc
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
+; GCN-NEXT:    s_add_u32 s4, 0, 0xaaaa0000
+; GCN-NEXT:    v_mov_b32_e32 v2, 0xffffffe8
+; GCN-NEXT:    v_mul_hi_u32 v2, s4, v2
+; GCN-NEXT:    s_addc_u32 s5, 0, 42
+; GCN-NEXT:    s_add_i32 s5, s5, 0xaaaaa80
+; GCN-NEXT:    s_mul_i32 s6, s4, 0xffffffe8
+; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, s4, v2
+; GCN-NEXT:    s_mul_i32 s7, s5, 0xffffffe8
+; GCN-NEXT:    v_mov_b32_e32 v3, s6
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s7, v2
+; GCN-NEXT:    v_mul_hi_u32 v4, s5, v3
+; GCN-NEXT:    v_mul_lo_u32 v5, s4, v2
+; GCN-NEXT:    v_mul_hi_u32 v3, s4, v3
+; GCN-NEXT:    v_mul_hi_u32 v6, s4, v2
+; GCN-NEXT:    s_mul_i32 s6, s5, s6
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
 ; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v3, v5, vcc
+; GCN-NEXT:    v_mul_hi_u32 v6, s5, v2
+; GCN-NEXT:    v_mul_lo_u32 v2, s5, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v5, v4, vcc
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, 0, v6, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc
+; GCN-NEXT:    v_mov_b32_e32 v4, s5
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s4, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v4, v3, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v4, v0, v3
 ; GCN-NEXT:    v_mul_hi_u32 v5, v0, v2
 ; GCN-NEXT:    v_mul_hi_u32 v6, v0, v3
diff --git a/llvm/test/CodeGen/AMDGPU/urem64.ll b/llvm/test/CodeGen/AMDGPU/urem64.ll
index f68d14a32b..91d09c0163 100644
--- a/llvm/test/CodeGen/AMDGPU/urem64.ll
+++ b/llvm/test/CodeGen/AMDGPU/urem64.ll
@@ -894,64 +894,36 @@ define amdgpu_kernel void @s_test_urem_k_num_i64(ptr addrspace(1) %out, i64 %x)
 define amdgpu_kernel void @s_test_urem_k_den_i64(ptr addrspace(1) %out, i64 %x) {
 ; GCN-LABEL: s_test_urem_k_den_i64:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    v_mov_b32_e32 v0, 0x4f800000
-; GCN-NEXT:    v_madak_f32 v0, 0, v0, 0x41c00000
-; GCN-NEXT:    v_rcp_f32_e32 v0, v0
-; GCN-NEXT:    s_movk_i32 s2, 0xffe8
 ; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-NEXT:    s_add_u32 s0, 0, 0xaaaa0000
+; GCN-NEXT:    v_mov_b32_e32 v0, 0xffffffe8
+; GCN-NEXT:    v_mul_hi_u32 v0, s0, v0
+; GCN-NEXT:    s_addc_u32 s1, 0, 42
+; GCN-NEXT:    s_add_i32 s1, s1, 0xaaaaa80
+; GCN-NEXT:    s_mul_i32 s8, s0, 0xffffffe8
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s0, v0
+; GCN-NEXT:    s_mul_i32 s9, s1, 0xffffffe8
+; GCN-NEXT:    v_mov_b32_e32 v1, s8
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s9, v0
+; GCN-NEXT:    v_mul_hi_u32 v2, s1, v1
+; GCN-NEXT:    v_mul_lo_u32 v3, s0, v0
+; GCN-NEXT:    v_mul_hi_u32 v1, s0, v1
+; GCN-NEXT:    v_mul_hi_u32 v4, s0, v0
+; GCN-NEXT:    s_mul_i32 s8, s1, s8
 ; GCN-NEXT:    s_mov_b32 s3, 0xf000
-; GCN-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
-; GCN-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
-; GCN-NEXT:    v_trunc_f32_e32 v1, v1
-; GCN-NEXT:    v_madmk_f32 v0, v1, 0xcf800000, v0
-; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_mov_b32 s0, s4
-; GCN-NEXT:    s_mov_b32 s1, s5
-; GCN-NEXT:    v_mul_hi_u32 v2, v0, s2
-; GCN-NEXT:    v_mul_lo_u32 v4, v1, s2
-; GCN-NEXT:    v_mul_lo_u32 v3, v0, s2
-; GCN-NEXT:    v_sub_i32_e32 v2, vcc, v2, v0
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
-; GCN-NEXT:    v_mul_hi_u32 v5, v0, v3
-; GCN-NEXT:    v_mul_lo_u32 v4, v0, v2
-; GCN-NEXT:    v_mul_hi_u32 v6, v0, v2
-; GCN-NEXT:    v_mul_hi_u32 v7, v1, v2
-; GCN-NEXT:    v_mul_lo_u32 v2, v1, v2
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc
-; GCN-NEXT:    v_mul_lo_u32 v6, v1, v3
-; GCN-NEXT:    v_mul_hi_u32 v3, v1, v3
-; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v5, v3, vcc
-; GCN-NEXT:    v_addc_u32_e32 v4, vcc, 0, v7, vcc
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
-; GCN-NEXT:    v_mul_hi_u32 v2, v0, s2
-; GCN-NEXT:    v_mul_lo_u32 v3, v1, s2
-; GCN-NEXT:    v_mul_lo_u32 v4, v0, s2
-; GCN-NEXT:    s_mov_b32 s2, -1
-; GCN-NEXT:    v_sub_i32_e32 v2, vcc, v2, v0
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GCN-NEXT:    v_mul_lo_u32 v3, v0, v2
-; GCN-NEXT:    v_mul_hi_u32 v5, v0, v4
-; GCN-NEXT:    v_mul_hi_u32 v6, v0, v2
-; GCN-NEXT:    v_mul_hi_u32 v7, v1, v2
-; GCN-NEXT:    v_mul_lo_u32 v2, v1, v2
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v5, v3
-; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc
-; GCN-NEXT:    v_mul_lo_u32 v6, v1, v4
-; GCN-NEXT:    v_mul_hi_u32 v4, v1, v4
-; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v6
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v5, v4, vcc
-; GCN-NEXT:    v_addc_u32_e32 v4, vcc, 0, v7, vcc
-; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc
-; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
+; GCN-NEXT:    v_mul_hi_u32 v4, s1, v0
+; GCN-NEXT:    v_mul_lo_u32 v0, s1, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, s8, v1
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v3, v2, vcc
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v2, vcc
+; GCN-NEXT:    v_mov_b32_e32 v2, s1
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v2, v1, vcc
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_mul_lo_u32 v2, s6, v1
 ; GCN-NEXT:    v_mul_hi_u32 v3, s6, v0
 ; GCN-NEXT:    v_mul_hi_u32 v4, s6, v1
@@ -961,6 +933,8 @@ define amdgpu_kernel void @s_test_urem_k_den_i64(ptr addrspace(1) %out, i64 %x)
 ; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc
 ; GCN-NEXT:    v_mul_lo_u32 v4, s7, v0
 ; GCN-NEXT:    v_mul_hi_u32 v0, s7, v0
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    s_mov_b32 s0, s4
 ; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
 ; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
 ; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v5, vcc
@@ -969,6 +943,7 @@ define amdgpu_kernel void @s_test_urem_k_den_i64(ptr addrspace(1) %out, i64 %x)
 ; GCN-NEXT:    v_mul_lo_u32 v1, v1, 24
 ; GCN-NEXT:    v_mul_hi_u32 v2, v0, 24
 ; GCN-NEXT:    v_mul_lo_u32 v0, v0, 24
+; GCN-NEXT:    s_mov_b32 s1, s5
 ; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
 ; GCN-NEXT:    v_mov_b32_e32 v2, s7
 ; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
diff --git a/llvm/test/CodeGen/AMDGPU/waterfall_kills_scc.ll b/llvm/test/CodeGen/AMDGPU/waterfall_kills_scc.ll
new file mode 100644
index 0000000000..8673ce765a
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/waterfall_kills_scc.ll
@@ -0,0 +1,83 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
+; RUN:  llc -march=amdgcn -mcpu=gfx906 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX906 %s
+declare float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32>, i32, i32, i32 immarg) #0
+declare void @llvm.amdgcn.raw.buffer.store.f32(float, <4 x i32>, i32, i32, i32 immarg) #1
+
+; Check that the compiler doesn't crash with a "undefined physical register" error;
+; bb.0 sets SCC bit in s_cmp_eq_u32 s0, 1
+; bb.1 overrides it
+; bb.2 uses the value from bb.0
+; Preserve SCC across bb.1 with s_cselect_b32 s5, 1, 0 -> s_cmp_lg_u32 s5, 0
+; Otherwise, we will see the following error.
+;*** Bad machine code: Using an undefined physical register ***
+;- function:    foo
+;- basic block: %bb.3  (0x53198c0)
+;- instruction: %33.sub1:sgpr_128 = S_CSELECT_B32 1072693248, 0, implicit $scc
+;- operand 3:   implicit $scc
+
+define amdgpu_kernel void  @foo(i1 %cmp1) {
+; GFX906-LABEL: foo:
+; GFX906:       ; %bb.0: ; %entry
+; GFX906-NEXT:    s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX906-NEXT:    s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX906-NEXT:    s_mov_b32 s10, -1
+; GFX906-NEXT:    s_mov_b32 s11, 0xe00000
+; GFX906-NEXT:    s_add_u32 s8, s8, s3
+; GFX906-NEXT:    s_addc_u32 s9, s9, 0
+; GFX906-NEXT:    buffer_load_dword v3, off, s[8:11], 0
+; GFX906-NEXT:    buffer_load_dword v4, off, s[8:11], 0 offset:4
+; GFX906-NEXT:    buffer_load_dword v5, off, s[8:11], 0 offset:8
+; GFX906-NEXT:    buffer_load_dword v6, off, s[8:11], 0 offset:12
+; GFX906-NEXT:    s_load_dword s4, s[0:1], 0x24
+; GFX906-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x1c
+; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX906-NEXT:    s_bitcmp1_b32 s4, 0
+; GFX906-NEXT:    s_mul_i32 s0, s2, s3
+; GFX906-NEXT:    v_mul_u32_u24_e32 v1, s3, v1
+; GFX906-NEXT:    v_mad_u32_u24 v0, s0, v0, v1
+; GFX906-NEXT:    v_add_lshl_u32 v2, v0, v2, 4
+; GFX906-NEXT:    v_mov_b32_e32 v0, 0
+; GFX906-NEXT:    s_mov_b32 s4, 0
+; GFX906-NEXT:    v_mov_b32_e32 v1, v0
+; GFX906-NEXT:    s_cselect_b32 s5, 1, 0
+; GFX906-NEXT:    s_mov_b64 s[2:3], exec
+; GFX906-NEXT:    ds_write_b64 v2, v[0:1]
+; GFX906-NEXT:  .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; GFX906-NEXT:    s_waitcnt vmcnt(3)
+; GFX906-NEXT:    v_readfirstlane_b32 s0, v3
+; GFX906-NEXT:    s_waitcnt vmcnt(2)
+; GFX906-NEXT:    v_readfirstlane_b32 s1, v4
+; GFX906-NEXT:    v_cmp_eq_u64_e32 vcc, s[0:1], v[3:4]
+; GFX906-NEXT:    s_waitcnt vmcnt(1)
+; GFX906-NEXT:    v_readfirstlane_b32 s0, v5
+; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    v_readfirstlane_b32 s1, v6
+; GFX906-NEXT:    v_cmp_eq_u64_e64 s[0:1], s[0:1], v[5:6]
+; GFX906-NEXT:    s_and_b64 s[0:1], vcc, s[0:1]
+; GFX906-NEXT:    s_and_saveexec_b64 s[0:1], s[0:1]
+; GFX906-NEXT:    ; implicit-def: $vgpr3_vgpr4_vgpr5_vgpr6
+; GFX906-NEXT:    s_xor_b64 exec, exec, s[0:1]
+; GFX906-NEXT:    s_cbranch_execnz .LBB0_1
+; GFX906-NEXT:  ; %bb.2:
+; GFX906-NEXT:    s_cmp_lg_u32 s5, 0
+; GFX906-NEXT:    s_mov_b64 exec, s[2:3]
+; GFX906-NEXT:    s_cselect_b32 s5, 0x3ff00000, 0
+; GFX906-NEXT:    v_cvt_f32_f64_e32 v0, s[4:5]
+; GFX906-NEXT:    s_mov_b32 s5, s4
+; GFX906-NEXT:    s_mov_b32 s6, s4
+; GFX906-NEXT:    s_mov_b32 s7, s4
+; GFX906-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX906-NEXT:    s_endpgm
+entry:
+  %wbr = alloca <4 x i32>, align 16, addrspace(5)
+  store ptr null, ptr addrspace(5) %wbr, align 16
+  %wbr_1 = load <4 x i32>, ptr addrspace(5) null, align 16
+  %call1 = tail call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %wbr_1, i32 0, i32 0, i32 0)
+  %0 = fpext float %call1 to double
+  %sel1 = select i1 %cmp1, double 1.000000e+00, double 0.000000e+00
+  %sel2 = select i1 %cmp1, double %0, double 0.000000e+00
+  %mul = fmul double %sel2, 0.000000e+00
+  %fptruncate = fptrunc double %sel1 to float
+  tail call void @llvm.amdgcn.raw.buffer.store.f32(float %fptruncate, <4 x i32> zeroinitializer, i32 0, i32 0, i32 0)
+  ret void
+}
diff --git a/llvm/test/CodeGen/ARM/fadd-select-fneg-combine.ll b/llvm/test/CodeGen/ARM/fadd-select-fneg-combine.ll
index 656bce616e..3f2b404609 100644
--- a/llvm/test/CodeGen/ARM/fadd-select-fneg-combine.ll
+++ b/llvm/test/CodeGen/ARM/fadd-select-fneg-combine.ll
@@ -284,40 +284,33 @@ define half @fadd_select_fneg_posk_f16(i32 %arg0, half %x, half %y) {
 define <8 x half> @fadd_vselect_fneg_posk_v8f16(<8 x i32> %arg0, <8 x half> %x, <8 x half> %y) {
 ; CHECK-LABEL: fadd_vselect_fneg_posk_v8f16:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    push {r4, r5, r6, lr}
+; CHECK-NEXT:    sub sp, sp, #16
 ; CHECK-NEXT:    vmov d0, r0, r1
-; CHECK-NEXT:    vmov.i8 q1, #0xff
-; CHECK-NEXT:    vmov d1, r2, r3
 ; CHECK-NEXT:    add r0, sp, #16
+; CHECK-NEXT:    vmov d1, r2, r3
+; CHECK-NEXT:    vldrw.u32 q3, [r0]
 ; CHECK-NEXT:    vcmp.i32 eq, q0, zr
 ; CHECK-NEXT:    vmov.i8 q0, #0x0
+; CHECK-NEXT:    vmov.i8 q1, #0xff
+; CHECK-NEXT:    mov r0, sp
 ; CHECK-NEXT:    vpsel q2, q1, q0
-; CHECK-NEXT:    vldrw.u32 q3, [r0]
-; CHECK-NEXT:    vmov r2, r1, d4
-; CHECK-NEXT:    add r12, sp, #32
-; CHECK-NEXT:    vmov r4, r5, d5
-; CHECK-NEXT:    vmov.16 q2[0], r2
-; CHECK-NEXT:    vmov.16 q2[1], r1
 ; CHECK-NEXT:    vcmp.i32 eq, q3, zr
-; CHECK-NEXT:    vpsel q1, q1, q0
-; CHECK-NEXT:    vmov.16 q2[2], r4
-; CHECK-NEXT:    vmov r3, r0, d2
-; CHECK-NEXT:    vmov.16 q2[3], r5
-; CHECK-NEXT:    vmov.16 q2[4], r3
-; CHECK-NEXT:    vmov r6, lr, d3
-; CHECK-NEXT:    vmov.16 q2[5], r0
-; CHECK-NEXT:    vldrw.u32 q1, [r12]
-; CHECK-NEXT:    vmov.16 q2[6], r6
-; CHECK-NEXT:    vmov.i16 q0, #0xc400
-; CHECK-NEXT:    vmov.16 q2[7], lr
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vstrh.32 q2, [r0]
+; CHECK-NEXT:    vstrh.32 q0, [r0, #8]
+; CHECK-NEXT:    add r1, sp, #32
+; CHECK-NEXT:    vldrw.u32 q2, [r0]
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vmov.i16 q1, #0xc400
 ; CHECK-NEXT:    add r0, sp, #48
 ; CHECK-NEXT:    vcmp.i16 ne, q2, zr
-; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    vldrw.u32 q1, [r0]
 ; CHECK-NEXT:    vsub.f16 q0, q1, q0
 ; CHECK-NEXT:    vmov r0, r1, d0
 ; CHECK-NEXT:    vmov r2, r3, d1
-; CHECK-NEXT:    pop {r4, r5, r6, pc}
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    bx lr
   %cmp = icmp eq <8 x i32> %arg0, zeroinitializer
   %neg.x = fneg <8 x half> %x
   %select = select <8 x i1> %cmp, <8 x half> %neg.x, <8 x half> <half 4.0, half 4.0, half 4.0, half 4.0, half 4.0, half 4.0, half 4.0, half 4.0>
diff --git a/llvm/test/CodeGen/ARM/stack-frame-layout-remarks.ll b/llvm/test/CodeGen/ARM/stack-frame-layout-remarks.ll
index 53e09216ab..c76dc24bae 100644
--- a/llvm/test/CodeGen/ARM/stack-frame-layout-remarks.ll
+++ b/llvm/test/CodeGen/ARM/stack-frame-layout-remarks.ll
@@ -236,19 +236,19 @@ attributes #2 = { ssp "stack-protector-buffer-size"="5" "frame-pointer"="all" }
 !2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, retainedTypes: !4, splitDebugInlining: false, nameTableKind: None)
 !3 = !DIFile(filename: "dot.c", directory: "")
 !4 = !{!5, !6, !10, !13}
-!5 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: null, size: 32)
-!6 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !7, size: 32)
-!7 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Array", file: !3, line: 3, size: 64, elements: !8)
+!5 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: null, size: 64)
+!6 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !7, size: 64)
+!7 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Array", file: !3, line: 3, size: 128, elements: !8)
 !8 = !{!9, !12}
-!9 = !DIDerivedType(tag: DW_TAG_member, name: "data", scope: !7, file: !3, line: 4, baseType: !10, size: 32)
-!10 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !11, size: 32)
+!9 = !DIDerivedType(tag: DW_TAG_member, name: "data", scope: !7, file: !3, line: 4, baseType: !10, size: 64)
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !11, size: 64)
 !11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
-!12 = !DIDerivedType(tag: DW_TAG_member, name: "size", scope: !7, file: !3, line: 5, baseType: !11, size: 32, offset: 32)
-!13 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !14, size: 32)
-!14 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Result", file: !3, line: 8, size: 64, elements: !15)
+!12 = !DIDerivedType(tag: DW_TAG_member, name: "size", scope: !7, file: !3, line: 5, baseType: !11, size: 32, offset: 64)
+!13 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !14, size: 64)
+!14 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Result", file: !3, line: 8, size: 128, elements: !15)
 !15 = !{!16, !17}
-!16 = !DIDerivedType(tag: DW_TAG_member, name: "data", scope: !14, file: !3, line: 9, baseType: !6, size: 32)
-!17 = !DIDerivedType(tag: DW_TAG_member, name: "sum", scope: !14, file: !3, line: 10, baseType: !11, size: 32, offset: 32)
+!16 = !DIDerivedType(tag: DW_TAG_member, name: "data", scope: !14, file: !3, line: 9, baseType: !6, size: 64)
+!17 = !DIDerivedType(tag: DW_TAG_member, name: "sum", scope: !14, file: !3, line: 10, baseType: !11, size: 32, offset: 64)
 !18 = !{i32 7, !"Dwarf Version", i32 5}
 !19 = !{i32 2, !"Debug Info Version", i32 3}
 !20 = !{i32 1, !"wchar_size", i32 4}
diff --git a/llvm/test/CodeGen/ARM/thumb2-size-opt.ll b/llvm/test/CodeGen/ARM/thumb2-size-opt.ll
index 8cf7a702e8..f9f29fc064 100644
--- a/llvm/test/CodeGen/ARM/thumb2-size-opt.ll
+++ b/llvm/test/CodeGen/ARM/thumb2-size-opt.ll
@@ -85,8 +85,8 @@ entry:
 
 define i32 @bundled_instruction(ptr %addr, ptr %addr2, i1 %tst) minsize {
 ; CHECK-LABEL: bundled_instruction:
-; CHECK: iteee ne
-; CHECK: ldmeq r2!, {{{r[0-9]+}}}
+; CHECK: itee ne
+; CHECK: ldmeq r3!, {{{r[0-9]+}}}
   br i1 %tst, label %true, label %false
 
 true:
diff --git a/llvm/test/CodeGen/BPF/warn-stack.ll b/llvm/test/CodeGen/BPF/warn-stack.ll
index 14d0f16f0f..807e196b92 100644
--- a/llvm/test/CodeGen/BPF/warn-stack.ll
+++ b/llvm/test/CodeGen/BPF/warn-stack.ll
@@ -2,11 +2,11 @@
 
 ;; CHECK-NOT: nowarn
 define void @nowarn() local_unnamed_addr #0 !dbg !6 {
-  %1 = alloca [511 x i8], align 1
-  call void @llvm.lifetime.start.p0(i64 511, ptr nonnull %1) #4, !dbg !15
+  %1 = alloca [504 x i8], align 1
+  call void @llvm.lifetime.start.p0(i64 504, ptr nonnull %1) #4, !dbg !15
   tail call void @llvm.dbg.declare(metadata ptr %1, metadata !10, metadata !16), !dbg !17
   call void @doit(ptr nonnull %1) #4, !dbg !18
-  call void @llvm.lifetime.end.p0(i64 511, ptr nonnull %1) #4, !dbg !19
+  call void @llvm.lifetime.end.p0(i64 504, ptr nonnull %1) #4, !dbg !19
   ret void, !dbg !19
 }
 
diff --git a/llvm/test/CodeGen/LoongArch/O0-pipeline.ll b/llvm/test/CodeGen/LoongArch/O0-pipeline.ll
index 327e461eb6..84d235d78e 100644
--- a/llvm/test/CodeGen/LoongArch/O0-pipeline.ll
+++ b/llvm/test/CodeGen/LoongArch/O0-pipeline.ll
@@ -69,6 +69,7 @@
 ; CHECK-NEXT:       Lazy Machine Block Frequency Analysis
 ; CHECK-NEXT:       Machine Optimization Remark Emitter
 ; CHECK-NEXT:       Stack Frame Layout Analysis
+; CHECK-NEXT:       LoongArch pseudo instruction expansion pass
 ; CHECK-NEXT:       LoongArch atomic pseudo instruction expansion pass
 ; CHECK-NEXT:       Lazy Machine Block Frequency Analysis
 ; CHECK-NEXT:       Machine Optimization Remark Emitter
diff --git a/llvm/test/CodeGen/LoongArch/atomicrmw-uinc-udec-wrap.ll b/llvm/test/CodeGen/LoongArch/atomicrmw-uinc-udec-wrap.ll
index fba340bed4..b0f29ee790 100644
--- a/llvm/test/CodeGen/LoongArch/atomicrmw-uinc-udec-wrap.ll
+++ b/llvm/test/CodeGen/LoongArch/atomicrmw-uinc-udec-wrap.ll
@@ -33,14 +33,13 @@ define i8 @atomicrmw_uinc_wrap_i8(ptr %ptr, i8 %val) {
 ; LA64-NEXT:    bne $a5, $a2, .LBB0_5
 ; LA64-NEXT:  # %bb.4: # %atomicrmw.start
 ; LA64-NEXT:    # in Loop: Header=BB0_3 Depth=2
-; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    move $a7, $a6
 ; LA64-NEXT:    sc.w $a7, $a0, 0
 ; LA64-NEXT:    beqz $a7, .LBB0_3
 ; LA64-NEXT:    b .LBB0_6
 ; LA64-NEXT:  .LBB0_5: # %atomicrmw.start
 ; LA64-NEXT:    # in Loop: Header=BB0_1 Depth=1
-; LA64-NEXT:    dbar 1792
+; LA64-NEXT:    dbar 20
 ; LA64-NEXT:  .LBB0_6: # %atomicrmw.start
 ; LA64-NEXT:    # in Loop: Header=BB0_1 Depth=1
 ; LA64-NEXT:    addi.w $a6, $a2, 0
@@ -86,14 +85,13 @@ define i16 @atomicrmw_uinc_wrap_i16(ptr %ptr, i16 %val) {
 ; LA64-NEXT:    bne $a5, $a2, .LBB1_5
 ; LA64-NEXT:  # %bb.4: # %atomicrmw.start
 ; LA64-NEXT:    # in Loop: Header=BB1_3 Depth=2
-; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    move $a7, $a6
 ; LA64-NEXT:    sc.w $a7, $a0, 0
 ; LA64-NEXT:    beqz $a7, .LBB1_3
 ; LA64-NEXT:    b .LBB1_6
 ; LA64-NEXT:  .LBB1_5: # %atomicrmw.start
 ; LA64-NEXT:    # in Loop: Header=BB1_1 Depth=1
-; LA64-NEXT:    dbar 1792
+; LA64-NEXT:    dbar 20
 ; LA64-NEXT:  .LBB1_6: # %atomicrmw.start
 ; LA64-NEXT:    # in Loop: Header=BB1_1 Depth=1
 ; LA64-NEXT:    addi.w $a6, $a2, 0
@@ -127,14 +125,13 @@ define i32 @atomicrmw_uinc_wrap_i32(ptr %ptr, i32 %val) {
 ; LA64-NEXT:    bne $a1, $a3, .LBB2_5
 ; LA64-NEXT:  # %bb.4: # %atomicrmw.start
 ; LA64-NEXT:    # in Loop: Header=BB2_3 Depth=2
-; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    move $a6, $a5
 ; LA64-NEXT:    sc.w $a6, $a0, 0
 ; LA64-NEXT:    beqz $a6, .LBB2_3
 ; LA64-NEXT:    b .LBB2_6
 ; LA64-NEXT:  .LBB2_5: # %atomicrmw.start
 ; LA64-NEXT:    # in Loop: Header=BB2_1 Depth=1
-; LA64-NEXT:    dbar 1792
+; LA64-NEXT:    dbar 20
 ; LA64-NEXT:  .LBB2_6: # %atomicrmw.start
 ; LA64-NEXT:    # in Loop: Header=BB2_1 Depth=1
 ; LA64-NEXT:    move $a3, $a1
@@ -166,14 +163,13 @@ define i64 @atomicrmw_uinc_wrap_i64(ptr %ptr, i64 %val) {
 ; LA64-NEXT:    bne $a2, $a3, .LBB3_5
 ; LA64-NEXT:  # %bb.4: # %atomicrmw.start
 ; LA64-NEXT:    # in Loop: Header=BB3_3 Depth=2
-; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    move $a5, $a4
 ; LA64-NEXT:    sc.d $a5, $a0, 0
 ; LA64-NEXT:    beqz $a5, .LBB3_3
 ; LA64-NEXT:    b .LBB3_6
 ; LA64-NEXT:  .LBB3_5: # %atomicrmw.start
 ; LA64-NEXT:    # in Loop: Header=BB3_1 Depth=1
-; LA64-NEXT:    dbar 1792
+; LA64-NEXT:    dbar 20
 ; LA64-NEXT:  .LBB3_6: # %atomicrmw.start
 ; LA64-NEXT:    # in Loop: Header=BB3_1 Depth=1
 ; LA64-NEXT:    bne $a2, $a3, .LBB3_1
@@ -221,14 +217,13 @@ define i8 @atomicrmw_udec_wrap_i8(ptr %ptr, i8 %val) {
 ; LA64-NEXT:    bne $a6, $a2, .LBB4_5
 ; LA64-NEXT:  # %bb.4: # %atomicrmw.start
 ; LA64-NEXT:    # in Loop: Header=BB4_3 Depth=2
-; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    move $t0, $a7
 ; LA64-NEXT:    sc.w $t0, $a0, 0
 ; LA64-NEXT:    beqz $t0, .LBB4_3
 ; LA64-NEXT:    b .LBB4_6
 ; LA64-NEXT:  .LBB4_5: # %atomicrmw.start
 ; LA64-NEXT:    # in Loop: Header=BB4_1 Depth=1
-; LA64-NEXT:    dbar 1792
+; LA64-NEXT:    dbar 20
 ; LA64-NEXT:  .LBB4_6: # %atomicrmw.start
 ; LA64-NEXT:    # in Loop: Header=BB4_1 Depth=1
 ; LA64-NEXT:    addi.w $a7, $a2, 0
@@ -279,14 +274,13 @@ define i16 @atomicrmw_udec_wrap_i16(ptr %ptr, i16 %val) {
 ; LA64-NEXT:    bne $a6, $a2, .LBB5_5
 ; LA64-NEXT:  # %bb.4: # %atomicrmw.start
 ; LA64-NEXT:    # in Loop: Header=BB5_3 Depth=2
-; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    move $t0, $a7
 ; LA64-NEXT:    sc.w $t0, $a0, 0
 ; LA64-NEXT:    beqz $t0, .LBB5_3
 ; LA64-NEXT:    b .LBB5_6
 ; LA64-NEXT:  .LBB5_5: # %atomicrmw.start
 ; LA64-NEXT:    # in Loop: Header=BB5_1 Depth=1
-; LA64-NEXT:    dbar 1792
+; LA64-NEXT:    dbar 20
 ; LA64-NEXT:  .LBB5_6: # %atomicrmw.start
 ; LA64-NEXT:    # in Loop: Header=BB5_1 Depth=1
 ; LA64-NEXT:    addi.w $a7, $a2, 0
@@ -325,14 +319,13 @@ define i32 @atomicrmw_udec_wrap_i32(ptr %ptr, i32 %val) {
 ; LA64-NEXT:    bne $a2, $a4, .LBB6_5
 ; LA64-NEXT:  # %bb.4: # %atomicrmw.start
 ; LA64-NEXT:    # in Loop: Header=BB6_3 Depth=2
-; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    move $a7, $a6
 ; LA64-NEXT:    sc.w $a7, $a0, 0
 ; LA64-NEXT:    beqz $a7, .LBB6_3
 ; LA64-NEXT:    b .LBB6_6
 ; LA64-NEXT:  .LBB6_5: # %atomicrmw.start
 ; LA64-NEXT:    # in Loop: Header=BB6_1 Depth=1
-; LA64-NEXT:    dbar 1792
+; LA64-NEXT:    dbar 20
 ; LA64-NEXT:  .LBB6_6: # %atomicrmw.start
 ; LA64-NEXT:    # in Loop: Header=BB6_1 Depth=1
 ; LA64-NEXT:    move $a4, $a2
@@ -369,14 +362,13 @@ define i64 @atomicrmw_udec_wrap_i64(ptr %ptr, i64 %val) {
 ; LA64-NEXT:    bne $a2, $a3, .LBB7_5
 ; LA64-NEXT:  # %bb.4: # %atomicrmw.start
 ; LA64-NEXT:    # in Loop: Header=BB7_3 Depth=2
-; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    move $a5, $a4
 ; LA64-NEXT:    sc.d $a5, $a0, 0
 ; LA64-NEXT:    beqz $a5, .LBB7_3
 ; LA64-NEXT:    b .LBB7_6
 ; LA64-NEXT:  .LBB7_5: # %atomicrmw.start
 ; LA64-NEXT:    # in Loop: Header=BB7_1 Depth=1
-; LA64-NEXT:    dbar 1792
+; LA64-NEXT:    dbar 20
 ; LA64-NEXT:  .LBB7_6: # %atomicrmw.start
 ; LA64-NEXT:    # in Loop: Header=BB7_1 Depth=1
 ; LA64-NEXT:    bne $a2, $a3, .LBB7_1
diff --git a/llvm/test/CodeGen/LoongArch/cfr-copy.mir b/llvm/test/CodeGen/LoongArch/cfr-copy.mir
new file mode 100644
index 0000000000..4224c99081
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/cfr-copy.mir
@@ -0,0 +1,34 @@
+# NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
+# RUN: llc --mtriple=loongarch64 --mattr=+d %s -o - | FileCheck %s
+
+## Check the PseudoCopyCFR instruction expand.
+
+--- |
+  target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128"
+  target triple = "loongarch64"
+
+  define void @test() {
+  ; CHECK-LABEL: test:
+  ; CHECK:       # %bb.0:
+  ; CHECK-NEXT:    fcmp.caf.s $fcc1, $fa0, $fa0
+  ; CHECK-NEXT:    bceqz $fcc0, .LBB0_2
+  ; CHECK-NEXT:  # %bb.1:
+  ; CHECK-NEXT:    fcmp.cueq.s $fcc1, $fa0, $fa0
+  ; CHECK-NEXT:  .LBB0_2:
+  ; CHECK-NEXT:    movcf2gr $a0, $fcc1
+  ; CHECK-NEXT:    ret
+    ret void
+  }
+...
+---
+name: test
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $fcc0
+
+    $fcc1 = COPY $fcc0
+    $r4 = COPY $fcc1
+    PseudoRET implicit killed $r4
+
+...
diff --git a/llvm/test/CodeGen/LoongArch/cfr-pseudo-copy.mir b/llvm/test/CodeGen/LoongArch/cfr-pseudo-copy.mir
new file mode 100644
index 0000000000..c5a6da7238
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/cfr-pseudo-copy.mir
@@ -0,0 +1,26 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc --mtriple=loongarch64 --mattr=+d --stop-after=postrapseudos %s \
+# RUN:     -o - | FileCheck %s
+
+## Check the COPY instruction between CFRs.
+## A pseudo (PseudoCopyCFR) is generated after postrapseudos pass.
+
+...
+---
+name: test
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $fcc0
+
+    ; CHECK-LABEL: name: test
+    ; CHECK: liveins: $fcc0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $fcc1 = PseudoCopyCFR $fcc0
+    ; CHECK-NEXT: $r4 = MOVCF2GR killed $fcc1
+    ; CHECK-NEXT: PseudoRET implicit killed $r4
+    $fcc1 = COPY $fcc0
+    $r4 = COPY $fcc1
+    PseudoRET implicit killed $r4
+
+...
diff --git a/llvm/test/CodeGen/LoongArch/inline-asm-clobbers-fcc.mir b/llvm/test/CodeGen/LoongArch/inline-asm-clobbers-fcc.mir
index fa5fccb1a5..18dbc5ca2e 100644
--- a/llvm/test/CodeGen/LoongArch/inline-asm-clobbers-fcc.mir
+++ b/llvm/test/CodeGen/LoongArch/inline-asm-clobbers-fcc.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc --mtriple=loongarch64 --mattr=+d --run-pass=greedy %s -o - | FileCheck %s
+# RUN: llc --mtriple=loongarch64 --mattr=+d --regalloc=fast \
+# RUN:     --stop-before=postra-machine-sink %s -o - | FileCheck %s
 
 ## Check that fcc register clobbered by inlineasm is correctly saved by examing
 ## a pair of pseudos (PseudoST_CFR and PseudoLD_CFR) are generated before and
@@ -15,13 +16,11 @@ body:             |
     ; CHECK-LABEL: name: test
     ; CHECK: liveins: $f0_64, $f1_64
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:fpr64 = COPY $f1_64
-    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:fpr64 = COPY $f0_64
-    ; CHECK-NEXT: [[FCMP_CLT_D:%[0-9]+]]:cfr = FCMP_CLT_D [[COPY]], [[COPY1]]
-    ; CHECK-NEXT: PseudoST_CFR [[FCMP_CLT_D]], %stack.0, 0 :: (store (s64) into %stack.0)
+    ; CHECK-NEXT: renamable $fcc0 = FCMP_CLT_D renamable $f1_64, renamable $f0_64
+    ; CHECK-NEXT: PseudoST_CFR $fcc0, %stack.0, 0 :: (store (s64) into %stack.0)
     ; CHECK-NEXT: INLINEASM &nop, 1 /* sideeffect attdialect */, 12 /* clobber */, implicit-def dead early-clobber $fcc0
-    ; CHECK-NEXT: [[PseudoLD_CFR:%[0-9]+]]:cfr = PseudoLD_CFR %stack.0, 0 :: (load (s64) from %stack.0)
-    ; CHECK-NEXT: $r4 = COPY [[PseudoLD_CFR]]
+    ; CHECK-NEXT: $fcc0 = PseudoLD_CFR %stack.0, 0 :: (load (s64) from %stack.0)
+    ; CHECK-NEXT: $r4 = COPY killed renamable $fcc0
     ; CHECK-NEXT: PseudoRET implicit killed $r4
     %1:fpr64 = COPY $f1_64
     %0:fpr64 = COPY $f0_64
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/atomic-cmpxchg.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/atomic-cmpxchg.ll
index 4e458e989c..417c865f63 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/atomic-cmpxchg.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/atomic-cmpxchg.ll
@@ -20,14 +20,13 @@ define void @cmpxchg_i8_acquire_acquire(ptr %ptr, i8 %cmp, i8 %val) nounwind {
 ; LA64-NEXT:    and $a5, $a4, $a3
 ; LA64-NEXT:    bne $a5, $a1, .LBB0_3
 ; LA64-NEXT:  # %bb.2: # in Loop: Header=BB0_1 Depth=1
-; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    andn $a5, $a4, $a3
 ; LA64-NEXT:    or $a5, $a5, $a2
 ; LA64-NEXT:    sc.w $a5, $a0, 0
 ; LA64-NEXT:    beqz $a5, .LBB0_1
 ; LA64-NEXT:    b .LBB0_4
 ; LA64-NEXT:  .LBB0_3:
-; LA64-NEXT:    dbar 1792
+; LA64-NEXT:    dbar 20
 ; LA64-NEXT:  .LBB0_4:
 ; LA64-NEXT:    ret
   %res = cmpxchg ptr %ptr, i8 %cmp, i8 %val acquire acquire
@@ -54,14 +53,13 @@ define void @cmpxchg_i16_acquire_acquire(ptr %ptr, i16 %cmp, i16 %val) nounwind
 ; LA64-NEXT:    and $a5, $a4, $a3
 ; LA64-NEXT:    bne $a5, $a1, .LBB1_3
 ; LA64-NEXT:  # %bb.2: # in Loop: Header=BB1_1 Depth=1
-; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    andn $a5, $a4, $a3
 ; LA64-NEXT:    or $a5, $a5, $a2
 ; LA64-NEXT:    sc.w $a5, $a0, 0
 ; LA64-NEXT:    beqz $a5, .LBB1_1
 ; LA64-NEXT:    b .LBB1_4
 ; LA64-NEXT:  .LBB1_3:
-; LA64-NEXT:    dbar 1792
+; LA64-NEXT:    dbar 20
 ; LA64-NEXT:  .LBB1_4:
 ; LA64-NEXT:    ret
   %res = cmpxchg ptr %ptr, i16 %cmp, i16 %val acquire acquire
@@ -75,13 +73,12 @@ define void @cmpxchg_i32_acquire_acquire(ptr %ptr, i32 %cmp, i32 %val) nounwind
 ; LA64-NEXT:    ll.w $a3, $a0, 0
 ; LA64-NEXT:    bne $a3, $a1, .LBB2_3
 ; LA64-NEXT:  # %bb.2: # in Loop: Header=BB2_1 Depth=1
-; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    move $a4, $a2
 ; LA64-NEXT:    sc.w $a4, $a0, 0
 ; LA64-NEXT:    beqz $a4, .LBB2_1
 ; LA64-NEXT:    b .LBB2_4
 ; LA64-NEXT:  .LBB2_3:
-; LA64-NEXT:    dbar 1792
+; LA64-NEXT:    dbar 20
 ; LA64-NEXT:  .LBB2_4:
 ; LA64-NEXT:    ret
   %res = cmpxchg ptr %ptr, i32 %cmp, i32 %val acquire acquire
@@ -95,19 +92,121 @@ define void @cmpxchg_i64_acquire_acquire(ptr %ptr, i64 %cmp, i64 %val) nounwind
 ; LA64-NEXT:    ll.d $a3, $a0, 0
 ; LA64-NEXT:    bne $a3, $a1, .LBB3_3
 ; LA64-NEXT:  # %bb.2: # in Loop: Header=BB3_1 Depth=1
-; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    move $a4, $a2
 ; LA64-NEXT:    sc.d $a4, $a0, 0
 ; LA64-NEXT:    beqz $a4, .LBB3_1
 ; LA64-NEXT:    b .LBB3_4
 ; LA64-NEXT:  .LBB3_3:
-; LA64-NEXT:    dbar 1792
+; LA64-NEXT:    dbar 20
 ; LA64-NEXT:  .LBB3_4:
 ; LA64-NEXT:    ret
   %res = cmpxchg ptr %ptr, i64 %cmp, i64 %val acquire acquire
   ret void
 }
 
+define void @cmpxchg_i8_acquire_monotonic(ptr %ptr, i8 %cmp, i8 %val) nounwind {
+; LA64-LABEL: cmpxchg_i8_acquire_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    slli.d $a3, $a0, 3
+; LA64-NEXT:    sll.w $a1, $a1, $a3
+; LA64-NEXT:    andi $a2, $a2, 255
+; LA64-NEXT:    sll.w $a2, $a2, $a3
+; LA64-NEXT:    ori $a4, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a4, $a3
+; LA64-NEXT:    addi.w $a3, $a3, 0
+; LA64-NEXT:    addi.w $a2, $a2, 0
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB4_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a5, $a4, $a3
+; LA64-NEXT:    bne $a5, $a1, .LBB4_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB4_1 Depth=1
+; LA64-NEXT:    andn $a5, $a4, $a3
+; LA64-NEXT:    or $a5, $a5, $a2
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB4_1
+; LA64-NEXT:    b .LBB4_4
+; LA64-NEXT:  .LBB4_3:
+; LA64-NEXT:    dbar 1792
+; LA64-NEXT:  .LBB4_4:
+; LA64-NEXT:    ret
+  %res = cmpxchg ptr %ptr, i8 %cmp, i8 %val acquire monotonic
+  ret void
+}
+
+define void @cmpxchg_i16_acquire_monotonic(ptr %ptr, i16 %cmp, i16 %val) nounwind {
+; LA64-LABEL: cmpxchg_i16_acquire_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    slli.d $a3, $a0, 3
+; LA64-NEXT:    sll.w $a1, $a1, $a3
+; LA64-NEXT:    bstrpick.d $a2, $a2, 15, 0
+; LA64-NEXT:    sll.w $a2, $a2, $a3
+; LA64-NEXT:    lu12i.w $a4, 15
+; LA64-NEXT:    ori $a4, $a4, 4095
+; LA64-NEXT:    sll.w $a3, $a4, $a3
+; LA64-NEXT:    addi.w $a3, $a3, 0
+; LA64-NEXT:    addi.w $a2, $a2, 0
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB5_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a5, $a4, $a3
+; LA64-NEXT:    bne $a5, $a1, .LBB5_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB5_1 Depth=1
+; LA64-NEXT:    andn $a5, $a4, $a3
+; LA64-NEXT:    or $a5, $a5, $a2
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB5_1
+; LA64-NEXT:    b .LBB5_4
+; LA64-NEXT:  .LBB5_3:
+; LA64-NEXT:    dbar 1792
+; LA64-NEXT:  .LBB5_4:
+; LA64-NEXT:    ret
+  %res = cmpxchg ptr %ptr, i16 %cmp, i16 %val acquire monotonic
+  ret void
+}
+
+define void @cmpxchg_i32_acquire_monotonic(ptr %ptr, i32 %cmp, i32 %val) nounwind {
+; LA64-LABEL: cmpxchg_i32_acquire_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:  .LBB6_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a3, $a0, 0
+; LA64-NEXT:    bne $a3, $a1, .LBB6_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB6_1 Depth=1
+; LA64-NEXT:    move $a4, $a2
+; LA64-NEXT:    sc.w $a4, $a0, 0
+; LA64-NEXT:    beqz $a4, .LBB6_1
+; LA64-NEXT:    b .LBB6_4
+; LA64-NEXT:  .LBB6_3:
+; LA64-NEXT:    dbar 1792
+; LA64-NEXT:  .LBB6_4:
+; LA64-NEXT:    ret
+  %res = cmpxchg ptr %ptr, i32 %cmp, i32 %val acquire monotonic
+  ret void
+}
+
+define void @cmpxchg_i64_acquire_monotonic(ptr %ptr, i64 %cmp, i64 %val) nounwind {
+; LA64-LABEL: cmpxchg_i64_acquire_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:  .LBB7_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.d $a3, $a0, 0
+; LA64-NEXT:    bne $a3, $a1, .LBB7_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB7_1 Depth=1
+; LA64-NEXT:    move $a4, $a2
+; LA64-NEXT:    sc.d $a4, $a0, 0
+; LA64-NEXT:    beqz $a4, .LBB7_1
+; LA64-NEXT:    b .LBB7_4
+; LA64-NEXT:  .LBB7_3:
+; LA64-NEXT:    dbar 1792
+; LA64-NEXT:  .LBB7_4:
+; LA64-NEXT:    ret
+  %res = cmpxchg ptr %ptr, i64 %cmp, i64 %val acquire monotonic
+  ret void
+}
+
 define i8 @cmpxchg_i8_acquire_acquire_reti8(ptr %ptr, i8 %cmp, i8 %val) nounwind {
 ; LA64-LABEL: cmpxchg_i8_acquire_acquire_reti8:
 ; LA64:       # %bb.0:
@@ -122,20 +221,19 @@ define i8 @cmpxchg_i8_acquire_acquire_reti8(ptr %ptr, i8 %cmp, i8 %val) nounwind
 ; LA64-NEXT:    sll.w $a1, $a1, $a3
 ; LA64-NEXT:    addi.w $a1, $a1, 0
 ; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
-; LA64-NEXT:  .LBB4_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:  .LBB8_1: # =>This Inner Loop Header: Depth=1
 ; LA64-NEXT:    ll.w $a5, $a0, 0
 ; LA64-NEXT:    and $a6, $a5, $a4
-; LA64-NEXT:    bne $a6, $a1, .LBB4_3
-; LA64-NEXT:  # %bb.2: # in Loop: Header=BB4_1 Depth=1
-; LA64-NEXT:    dbar 0
+; LA64-NEXT:    bne $a6, $a1, .LBB8_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB8_1 Depth=1
 ; LA64-NEXT:    andn $a6, $a5, $a4
 ; LA64-NEXT:    or $a6, $a6, $a2
 ; LA64-NEXT:    sc.w $a6, $a0, 0
-; LA64-NEXT:    beqz $a6, .LBB4_1
-; LA64-NEXT:    b .LBB4_4
-; LA64-NEXT:  .LBB4_3:
-; LA64-NEXT:    dbar 1792
-; LA64-NEXT:  .LBB4_4:
+; LA64-NEXT:    beqz $a6, .LBB8_1
+; LA64-NEXT:    b .LBB8_4
+; LA64-NEXT:  .LBB8_3:
+; LA64-NEXT:    dbar 20
+; LA64-NEXT:  .LBB8_4:
 ; LA64-NEXT:    srl.w $a0, $a5, $a3
 ; LA64-NEXT:    ret
   %tmp = cmpxchg ptr %ptr, i8 %cmp, i8 %val acquire acquire
@@ -158,20 +256,19 @@ define i16 @cmpxchg_i16_acquire_acquire_reti16(ptr %ptr, i16 %cmp, i16 %val) nou
 ; LA64-NEXT:    sll.w $a1, $a1, $a4
 ; LA64-NEXT:    addi.w $a1, $a1, 0
 ; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
-; LA64-NEXT:  .LBB5_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:  .LBB9_1: # =>This Inner Loop Header: Depth=1
 ; LA64-NEXT:    ll.w $a5, $a0, 0
 ; LA64-NEXT:    and $a6, $a5, $a3
-; LA64-NEXT:    bne $a6, $a1, .LBB5_3
-; LA64-NEXT:  # %bb.2: # in Loop: Header=BB5_1 Depth=1
-; LA64-NEXT:    dbar 0
+; LA64-NEXT:    bne $a6, $a1, .LBB9_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB9_1 Depth=1
 ; LA64-NEXT:    andn $a6, $a5, $a3
 ; LA64-NEXT:    or $a6, $a6, $a2
 ; LA64-NEXT:    sc.w $a6, $a0, 0
-; LA64-NEXT:    beqz $a6, .LBB5_1
-; LA64-NEXT:    b .LBB5_4
-; LA64-NEXT:  .LBB5_3:
-; LA64-NEXT:    dbar 1792
-; LA64-NEXT:  .LBB5_4:
+; LA64-NEXT:    beqz $a6, .LBB9_1
+; LA64-NEXT:    b .LBB9_4
+; LA64-NEXT:  .LBB9_3:
+; LA64-NEXT:    dbar 20
+; LA64-NEXT:  .LBB9_4:
 ; LA64-NEXT:    srl.w $a0, $a5, $a4
 ; LA64-NEXT:    ret
   %tmp = cmpxchg ptr %ptr, i16 %cmp, i16 %val acquire acquire
@@ -182,18 +279,17 @@ define i16 @cmpxchg_i16_acquire_acquire_reti16(ptr %ptr, i16 %cmp, i16 %val) nou
 define i32 @cmpxchg_i32_acquire_acquire_reti32(ptr %ptr, i32 %cmp, i32 %val) nounwind {
 ; LA64-LABEL: cmpxchg_i32_acquire_acquire_reti32:
 ; LA64:       # %bb.0:
-; LA64-NEXT:  .LBB6_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:  .LBB10_1: # =>This Inner Loop Header: Depth=1
 ; LA64-NEXT:    ll.w $a3, $a0, 0
-; LA64-NEXT:    bne $a3, $a1, .LBB6_3
-; LA64-NEXT:  # %bb.2: # in Loop: Header=BB6_1 Depth=1
-; LA64-NEXT:    dbar 0
+; LA64-NEXT:    bne $a3, $a1, .LBB10_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB10_1 Depth=1
 ; LA64-NEXT:    move $a4, $a2
 ; LA64-NEXT:    sc.w $a4, $a0, 0
-; LA64-NEXT:    beqz $a4, .LBB6_1
-; LA64-NEXT:    b .LBB6_4
-; LA64-NEXT:  .LBB6_3:
-; LA64-NEXT:    dbar 1792
-; LA64-NEXT:  .LBB6_4:
+; LA64-NEXT:    beqz $a4, .LBB10_1
+; LA64-NEXT:    b .LBB10_4
+; LA64-NEXT:  .LBB10_3:
+; LA64-NEXT:    dbar 20
+; LA64-NEXT:  .LBB10_4:
 ; LA64-NEXT:    move $a0, $a3
 ; LA64-NEXT:    ret
   %tmp = cmpxchg ptr %ptr, i32 %cmp, i32 %val acquire acquire
@@ -204,18 +300,17 @@ define i32 @cmpxchg_i32_acquire_acquire_reti32(ptr %ptr, i32 %cmp, i32 %val) nou
 define i64 @cmpxchg_i64_acquire_acquire_reti64(ptr %ptr, i64 %cmp, i64 %val) nounwind {
 ; LA64-LABEL: cmpxchg_i64_acquire_acquire_reti64:
 ; LA64:       # %bb.0:
-; LA64-NEXT:  .LBB7_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:  .LBB11_1: # =>This Inner Loop Header: Depth=1
 ; LA64-NEXT:    ll.d $a3, $a0, 0
-; LA64-NEXT:    bne $a3, $a1, .LBB7_3
-; LA64-NEXT:  # %bb.2: # in Loop: Header=BB7_1 Depth=1
-; LA64-NEXT:    dbar 0
+; LA64-NEXT:    bne $a3, $a1, .LBB11_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB11_1 Depth=1
 ; LA64-NEXT:    move $a4, $a2
 ; LA64-NEXT:    sc.d $a4, $a0, 0
-; LA64-NEXT:    beqz $a4, .LBB7_1
-; LA64-NEXT:    b .LBB7_4
-; LA64-NEXT:  .LBB7_3:
-; LA64-NEXT:    dbar 1792
-; LA64-NEXT:  .LBB7_4:
+; LA64-NEXT:    beqz $a4, .LBB11_1
+; LA64-NEXT:    b .LBB11_4
+; LA64-NEXT:  .LBB11_3:
+; LA64-NEXT:    dbar 20
+; LA64-NEXT:  .LBB11_4:
 ; LA64-NEXT:    move $a0, $a3
 ; LA64-NEXT:    ret
   %tmp = cmpxchg ptr %ptr, i64 %cmp, i64 %val acquire acquire
@@ -237,20 +332,19 @@ define i1 @cmpxchg_i8_acquire_acquire_reti1(ptr %ptr, i8 %cmp, i8 %val) nounwind
 ; LA64-NEXT:    addi.w $a1, $a1, 0
 ; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
 ; LA64-NEXT:    addi.w $a3, $a4, 0
-; LA64-NEXT:  .LBB8_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:  .LBB12_1: # =>This Inner Loop Header: Depth=1
 ; LA64-NEXT:    ll.w $a5, $a0, 0
 ; LA64-NEXT:    and $a6, $a5, $a3
-; LA64-NEXT:    bne $a6, $a1, .LBB8_3
-; LA64-NEXT:  # %bb.2: # in Loop: Header=BB8_1 Depth=1
-; LA64-NEXT:    dbar 0
+; LA64-NEXT:    bne $a6, $a1, .LBB12_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB12_1 Depth=1
 ; LA64-NEXT:    andn $a6, $a5, $a3
 ; LA64-NEXT:    or $a6, $a6, $a2
 ; LA64-NEXT:    sc.w $a6, $a0, 0
-; LA64-NEXT:    beqz $a6, .LBB8_1
-; LA64-NEXT:    b .LBB8_4
-; LA64-NEXT:  .LBB8_3:
-; LA64-NEXT:    dbar 1792
-; LA64-NEXT:  .LBB8_4:
+; LA64-NEXT:    beqz $a6, .LBB12_1
+; LA64-NEXT:    b .LBB12_4
+; LA64-NEXT:  .LBB12_3:
+; LA64-NEXT:    dbar 20
+; LA64-NEXT:  .LBB12_4:
 ; LA64-NEXT:    and $a0, $a5, $a4
 ; LA64-NEXT:    addi.w $a0, $a0, 0
 ; LA64-NEXT:    xor $a0, $a1, $a0
@@ -276,20 +370,19 @@ define i1 @cmpxchg_i16_acquire_acquire_reti1(ptr %ptr, i16 %cmp, i16 %val) nounw
 ; LA64-NEXT:    addi.w $a1, $a1, 0
 ; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
 ; LA64-NEXT:    addi.w $a4, $a3, 0
-; LA64-NEXT:  .LBB9_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:  .LBB13_1: # =>This Inner Loop Header: Depth=1
 ; LA64-NEXT:    ll.w $a5, $a0, 0
 ; LA64-NEXT:    and $a6, $a5, $a4
-; LA64-NEXT:    bne $a6, $a1, .LBB9_3
-; LA64-NEXT:  # %bb.2: # in Loop: Header=BB9_1 Depth=1
-; LA64-NEXT:    dbar 0
+; LA64-NEXT:    bne $a6, $a1, .LBB13_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB13_1 Depth=1
 ; LA64-NEXT:    andn $a6, $a5, $a4
 ; LA64-NEXT:    or $a6, $a6, $a2
 ; LA64-NEXT:    sc.w $a6, $a0, 0
-; LA64-NEXT:    beqz $a6, .LBB9_1
-; LA64-NEXT:    b .LBB9_4
-; LA64-NEXT:  .LBB9_3:
-; LA64-NEXT:    dbar 1792
-; LA64-NEXT:  .LBB9_4:
+; LA64-NEXT:    beqz $a6, .LBB13_1
+; LA64-NEXT:    b .LBB13_4
+; LA64-NEXT:  .LBB13_3:
+; LA64-NEXT:    dbar 20
+; LA64-NEXT:  .LBB13_4:
 ; LA64-NEXT:    and $a0, $a5, $a3
 ; LA64-NEXT:    addi.w $a0, $a0, 0
 ; LA64-NEXT:    xor $a0, $a1, $a0
@@ -303,18 +396,17 @@ define i1 @cmpxchg_i16_acquire_acquire_reti1(ptr %ptr, i16 %cmp, i16 %val) nounw
 define i1 @cmpxchg_i32_acquire_acquire_reti1(ptr %ptr, i32 %cmp, i32 %val) nounwind {
 ; LA64-LABEL: cmpxchg_i32_acquire_acquire_reti1:
 ; LA64:       # %bb.0:
-; LA64-NEXT:  .LBB10_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:  .LBB14_1: # =>This Inner Loop Header: Depth=1
 ; LA64-NEXT:    ll.w $a3, $a0, 0
-; LA64-NEXT:    bne $a3, $a1, .LBB10_3
-; LA64-NEXT:  # %bb.2: # in Loop: Header=BB10_1 Depth=1
-; LA64-NEXT:    dbar 0
+; LA64-NEXT:    bne $a3, $a1, .LBB14_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB14_1 Depth=1
 ; LA64-NEXT:    move $a4, $a2
 ; LA64-NEXT:    sc.w $a4, $a0, 0
-; LA64-NEXT:    beqz $a4, .LBB10_1
-; LA64-NEXT:    b .LBB10_4
-; LA64-NEXT:  .LBB10_3:
-; LA64-NEXT:    dbar 1792
-; LA64-NEXT:  .LBB10_4:
+; LA64-NEXT:    beqz $a4, .LBB14_1
+; LA64-NEXT:    b .LBB14_4
+; LA64-NEXT:  .LBB14_3:
+; LA64-NEXT:    dbar 20
+; LA64-NEXT:  .LBB14_4:
 ; LA64-NEXT:    addi.w $a0, $a1, 0
 ; LA64-NEXT:    xor $a0, $a3, $a0
 ; LA64-NEXT:    sltui $a0, $a0, 1
@@ -327,18 +419,17 @@ define i1 @cmpxchg_i32_acquire_acquire_reti1(ptr %ptr, i32 %cmp, i32 %val) nounw
 define i1 @cmpxchg_i64_acquire_acquire_reti1(ptr %ptr, i64 %cmp, i64 %val) nounwind {
 ; LA64-LABEL: cmpxchg_i64_acquire_acquire_reti1:
 ; LA64:       # %bb.0:
-; LA64-NEXT:  .LBB11_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:  .LBB15_1: # =>This Inner Loop Header: Depth=1
 ; LA64-NEXT:    ll.d $a3, $a0, 0
-; LA64-NEXT:    bne $a3, $a1, .LBB11_3
-; LA64-NEXT:  # %bb.2: # in Loop: Header=BB11_1 Depth=1
-; LA64-NEXT:    dbar 0
+; LA64-NEXT:    bne $a3, $a1, .LBB15_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB15_1 Depth=1
 ; LA64-NEXT:    move $a4, $a2
 ; LA64-NEXT:    sc.d $a4, $a0, 0
-; LA64-NEXT:    beqz $a4, .LBB11_1
-; LA64-NEXT:    b .LBB11_4
-; LA64-NEXT:  .LBB11_3:
-; LA64-NEXT:    dbar 1792
-; LA64-NEXT:  .LBB11_4:
+; LA64-NEXT:    beqz $a4, .LBB15_1
+; LA64-NEXT:    b .LBB15_4
+; LA64-NEXT:  .LBB15_3:
+; LA64-NEXT:    dbar 20
+; LA64-NEXT:  .LBB15_4:
 ; LA64-NEXT:    xor $a0, $a3, $a1
 ; LA64-NEXT:    sltui $a0, $a0, 1
 ; LA64-NEXT:    ret
@@ -346,3 +437,337 @@ define i1 @cmpxchg_i64_acquire_acquire_reti1(ptr %ptr, i64 %cmp, i64 %val) nounw
   %res = extractvalue { i64, i1 } %tmp, 1
   ret i1 %res
 }
+
+define void @cmpxchg_i8_monotonic_monotonic(ptr %ptr, i8 %cmp, i8 %val) nounwind {
+; LA64-LABEL: cmpxchg_i8_monotonic_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    slli.d $a3, $a0, 3
+; LA64-NEXT:    sll.w $a1, $a1, $a3
+; LA64-NEXT:    andi $a2, $a2, 255
+; LA64-NEXT:    sll.w $a2, $a2, $a3
+; LA64-NEXT:    ori $a4, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a4, $a3
+; LA64-NEXT:    addi.w $a3, $a3, 0
+; LA64-NEXT:    addi.w $a2, $a2, 0
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB16_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a5, $a4, $a3
+; LA64-NEXT:    bne $a5, $a1, .LBB16_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB16_1 Depth=1
+; LA64-NEXT:    andn $a5, $a4, $a3
+; LA64-NEXT:    or $a5, $a5, $a2
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB16_1
+; LA64-NEXT:    b .LBB16_4
+; LA64-NEXT:  .LBB16_3:
+; LA64-NEXT:    dbar 1792
+; LA64-NEXT:  .LBB16_4:
+; LA64-NEXT:    ret
+  %res = cmpxchg ptr %ptr, i8 %cmp, i8 %val monotonic monotonic
+  ret void
+}
+
+define void @cmpxchg_i16_monotonic_monotonic(ptr %ptr, i16 %cmp, i16 %val) nounwind {
+; LA64-LABEL: cmpxchg_i16_monotonic_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    slli.d $a3, $a0, 3
+; LA64-NEXT:    sll.w $a1, $a1, $a3
+; LA64-NEXT:    bstrpick.d $a2, $a2, 15, 0
+; LA64-NEXT:    sll.w $a2, $a2, $a3
+; LA64-NEXT:    lu12i.w $a4, 15
+; LA64-NEXT:    ori $a4, $a4, 4095
+; LA64-NEXT:    sll.w $a3, $a4, $a3
+; LA64-NEXT:    addi.w $a3, $a3, 0
+; LA64-NEXT:    addi.w $a2, $a2, 0
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB17_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a5, $a4, $a3
+; LA64-NEXT:    bne $a5, $a1, .LBB17_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB17_1 Depth=1
+; LA64-NEXT:    andn $a5, $a4, $a3
+; LA64-NEXT:    or $a5, $a5, $a2
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB17_1
+; LA64-NEXT:    b .LBB17_4
+; LA64-NEXT:  .LBB17_3:
+; LA64-NEXT:    dbar 1792
+; LA64-NEXT:  .LBB17_4:
+; LA64-NEXT:    ret
+  %res = cmpxchg ptr %ptr, i16 %cmp, i16 %val monotonic monotonic
+  ret void
+}
+
+define void @cmpxchg_i32_monotonic_monotonic(ptr %ptr, i32 %cmp, i32 %val) nounwind {
+; LA64-LABEL: cmpxchg_i32_monotonic_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:  .LBB18_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a3, $a0, 0
+; LA64-NEXT:    bne $a3, $a1, .LBB18_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB18_1 Depth=1
+; LA64-NEXT:    move $a4, $a2
+; LA64-NEXT:    sc.w $a4, $a0, 0
+; LA64-NEXT:    beqz $a4, .LBB18_1
+; LA64-NEXT:    b .LBB18_4
+; LA64-NEXT:  .LBB18_3:
+; LA64-NEXT:    dbar 1792
+; LA64-NEXT:  .LBB18_4:
+; LA64-NEXT:    ret
+  %res = cmpxchg ptr %ptr, i32 %cmp, i32 %val monotonic monotonic
+  ret void
+}
+
+define void @cmpxchg_i64_monotonic_monotonic(ptr %ptr, i64 %cmp, i64 %val) nounwind {
+; LA64-LABEL: cmpxchg_i64_monotonic_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:  .LBB19_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.d $a3, $a0, 0
+; LA64-NEXT:    bne $a3, $a1, .LBB19_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB19_1 Depth=1
+; LA64-NEXT:    move $a4, $a2
+; LA64-NEXT:    sc.d $a4, $a0, 0
+; LA64-NEXT:    beqz $a4, .LBB19_1
+; LA64-NEXT:    b .LBB19_4
+; LA64-NEXT:  .LBB19_3:
+; LA64-NEXT:    dbar 1792
+; LA64-NEXT:  .LBB19_4:
+; LA64-NEXT:    ret
+  %res = cmpxchg ptr %ptr, i64 %cmp, i64 %val monotonic monotonic
+  ret void
+}
+
+define i8 @cmpxchg_i8_monotonic_monotonic_reti8(ptr %ptr, i8 %cmp, i8 %val) nounwind {
+; LA64-LABEL: cmpxchg_i8_monotonic_monotonic_reti8:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a3, $a0, 3
+; LA64-NEXT:    ori $a4, $zero, 255
+; LA64-NEXT:    sll.w $a4, $a4, $a3
+; LA64-NEXT:    andi $a2, $a2, 255
+; LA64-NEXT:    addi.w $a4, $a4, 0
+; LA64-NEXT:    sll.w $a2, $a2, $a3
+; LA64-NEXT:    addi.w $a2, $a2, 0
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a3
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB20_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a6, $a5, $a4
+; LA64-NEXT:    bne $a6, $a1, .LBB20_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB20_1 Depth=1
+; LA64-NEXT:    andn $a6, $a5, $a4
+; LA64-NEXT:    or $a6, $a6, $a2
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB20_1
+; LA64-NEXT:    b .LBB20_4
+; LA64-NEXT:  .LBB20_3:
+; LA64-NEXT:    dbar 1792
+; LA64-NEXT:  .LBB20_4:
+; LA64-NEXT:    srl.w $a0, $a5, $a3
+; LA64-NEXT:    ret
+  %tmp = cmpxchg ptr %ptr, i8 %cmp, i8 %val monotonic monotonic
+  %res = extractvalue { i8, i1 } %tmp, 0
+  ret i8 %res
+}
+
+define i16 @cmpxchg_i16_monotonic_monotonic_reti16(ptr %ptr, i16 %cmp, i16 %val) nounwind {
+; LA64-LABEL: cmpxchg_i16_monotonic_monotonic_reti16:
+; LA64:       # %bb.0:
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    slli.d $a4, $a0, 3
+; LA64-NEXT:    sll.w $a3, $a3, $a4
+; LA64-NEXT:    bstrpick.d $a2, $a2, 15, 0
+; LA64-NEXT:    addi.w $a3, $a3, 0
+; LA64-NEXT:    sll.w $a2, $a2, $a4
+; LA64-NEXT:    addi.w $a2, $a2, 0
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a4
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB21_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a6, $a5, $a3
+; LA64-NEXT:    bne $a6, $a1, .LBB21_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB21_1 Depth=1
+; LA64-NEXT:    andn $a6, $a5, $a3
+; LA64-NEXT:    or $a6, $a6, $a2
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB21_1
+; LA64-NEXT:    b .LBB21_4
+; LA64-NEXT:  .LBB21_3:
+; LA64-NEXT:    dbar 1792
+; LA64-NEXT:  .LBB21_4:
+; LA64-NEXT:    srl.w $a0, $a5, $a4
+; LA64-NEXT:    ret
+  %tmp = cmpxchg ptr %ptr, i16 %cmp, i16 %val monotonic monotonic
+  %res = extractvalue { i16, i1 } %tmp, 0
+  ret i16 %res
+}
+
+define i32 @cmpxchg_i32_monotonic_monotonic_reti32(ptr %ptr, i32 %cmp, i32 %val) nounwind {
+; LA64-LABEL: cmpxchg_i32_monotonic_monotonic_reti32:
+; LA64:       # %bb.0:
+; LA64-NEXT:  .LBB22_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a3, $a0, 0
+; LA64-NEXT:    bne $a3, $a1, .LBB22_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB22_1 Depth=1
+; LA64-NEXT:    move $a4, $a2
+; LA64-NEXT:    sc.w $a4, $a0, 0
+; LA64-NEXT:    beqz $a4, .LBB22_1
+; LA64-NEXT:    b .LBB22_4
+; LA64-NEXT:  .LBB22_3:
+; LA64-NEXT:    dbar 1792
+; LA64-NEXT:  .LBB22_4:
+; LA64-NEXT:    move $a0, $a3
+; LA64-NEXT:    ret
+  %tmp = cmpxchg ptr %ptr, i32 %cmp, i32 %val monotonic monotonic
+  %res = extractvalue { i32, i1 } %tmp, 0
+  ret i32 %res
+}
+
+define i64 @cmpxchg_i64_monotonic_monotonic_reti64(ptr %ptr, i64 %cmp, i64 %val) nounwind {
+; LA64-LABEL: cmpxchg_i64_monotonic_monotonic_reti64:
+; LA64:       # %bb.0:
+; LA64-NEXT:  .LBB23_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.d $a3, $a0, 0
+; LA64-NEXT:    bne $a3, $a1, .LBB23_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB23_1 Depth=1
+; LA64-NEXT:    move $a4, $a2
+; LA64-NEXT:    sc.d $a4, $a0, 0
+; LA64-NEXT:    beqz $a4, .LBB23_1
+; LA64-NEXT:    b .LBB23_4
+; LA64-NEXT:  .LBB23_3:
+; LA64-NEXT:    dbar 1792
+; LA64-NEXT:  .LBB23_4:
+; LA64-NEXT:    move $a0, $a3
+; LA64-NEXT:    ret
+  %tmp = cmpxchg ptr %ptr, i64 %cmp, i64 %val monotonic monotonic
+  %res = extractvalue { i64, i1 } %tmp, 0
+  ret i64 %res
+}
+
+define i1 @cmpxchg_i8_monotonic_monotonic_reti1(ptr %ptr, i8 %cmp, i8 %val) nounwind {
+; LA64-LABEL: cmpxchg_i8_monotonic_monotonic_reti1:
+; LA64:       # %bb.0:
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    slli.d $a3, $a0, 3
+; LA64-NEXT:    ori $a4, $zero, 255
+; LA64-NEXT:    sll.w $a4, $a4, $a3
+; LA64-NEXT:    sll.w $a1, $a1, $a3
+; LA64-NEXT:    andi $a2, $a2, 255
+; LA64-NEXT:    sll.w $a2, $a2, $a3
+; LA64-NEXT:    addi.w $a2, $a2, 0
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    addi.w $a3, $a4, 0
+; LA64-NEXT:  .LBB24_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a6, $a5, $a3
+; LA64-NEXT:    bne $a6, $a1, .LBB24_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB24_1 Depth=1
+; LA64-NEXT:    andn $a6, $a5, $a3
+; LA64-NEXT:    or $a6, $a6, $a2
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB24_1
+; LA64-NEXT:    b .LBB24_4
+; LA64-NEXT:  .LBB24_3:
+; LA64-NEXT:    dbar 1792
+; LA64-NEXT:  .LBB24_4:
+; LA64-NEXT:    and $a0, $a5, $a4
+; LA64-NEXT:    addi.w $a0, $a0, 0
+; LA64-NEXT:    xor $a0, $a1, $a0
+; LA64-NEXT:    sltui $a0, $a0, 1
+; LA64-NEXT:    ret
+  %tmp = cmpxchg ptr %ptr, i8 %cmp, i8 %val monotonic monotonic
+  %res = extractvalue { i8, i1 } %tmp, 1
+  ret i1 %res
+}
+
+define i1 @cmpxchg_i16_monotonic_monotonic_reti1(ptr %ptr, i16 %cmp, i16 %val) nounwind {
+; LA64-LABEL: cmpxchg_i16_monotonic_monotonic_reti1:
+; LA64:       # %bb.0:
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    slli.d $a4, $a0, 3
+; LA64-NEXT:    sll.w $a3, $a3, $a4
+; LA64-NEXT:    sll.w $a1, $a1, $a4
+; LA64-NEXT:    bstrpick.d $a2, $a2, 15, 0
+; LA64-NEXT:    sll.w $a2, $a2, $a4
+; LA64-NEXT:    addi.w $a2, $a2, 0
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    addi.w $a4, $a3, 0
+; LA64-NEXT:  .LBB25_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a6, $a5, $a4
+; LA64-NEXT:    bne $a6, $a1, .LBB25_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB25_1 Depth=1
+; LA64-NEXT:    andn $a6, $a5, $a4
+; LA64-NEXT:    or $a6, $a6, $a2
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB25_1
+; LA64-NEXT:    b .LBB25_4
+; LA64-NEXT:  .LBB25_3:
+; LA64-NEXT:    dbar 1792
+; LA64-NEXT:  .LBB25_4:
+; LA64-NEXT:    and $a0, $a5, $a3
+; LA64-NEXT:    addi.w $a0, $a0, 0
+; LA64-NEXT:    xor $a0, $a1, $a0
+; LA64-NEXT:    sltui $a0, $a0, 1
+; LA64-NEXT:    ret
+  %tmp = cmpxchg ptr %ptr, i16 %cmp, i16 %val monotonic monotonic
+  %res = extractvalue { i16, i1 } %tmp, 1
+  ret i1 %res
+}
+
+define i1 @cmpxchg_i32_monotonic_monotonic_reti1(ptr %ptr, i32 %cmp, i32 %val) nounwind {
+; LA64-LABEL: cmpxchg_i32_monotonic_monotonic_reti1:
+; LA64:       # %bb.0:
+; LA64-NEXT:  .LBB26_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a3, $a0, 0
+; LA64-NEXT:    bne $a3, $a1, .LBB26_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB26_1 Depth=1
+; LA64-NEXT:    move $a4, $a2
+; LA64-NEXT:    sc.w $a4, $a0, 0
+; LA64-NEXT:    beqz $a4, .LBB26_1
+; LA64-NEXT:    b .LBB26_4
+; LA64-NEXT:  .LBB26_3:
+; LA64-NEXT:    dbar 1792
+; LA64-NEXT:  .LBB26_4:
+; LA64-NEXT:    addi.w $a0, $a1, 0
+; LA64-NEXT:    xor $a0, $a3, $a0
+; LA64-NEXT:    sltui $a0, $a0, 1
+; LA64-NEXT:    ret
+  %tmp = cmpxchg ptr %ptr, i32 %cmp, i32 %val monotonic monotonic
+  %res = extractvalue { i32, i1 } %tmp, 1
+  ret i1 %res
+}
+
+define i1 @cmpxchg_i64_monotonic_monotonic_reti1(ptr %ptr, i64 %cmp, i64 %val) nounwind {
+; LA64-LABEL: cmpxchg_i64_monotonic_monotonic_reti1:
+; LA64:       # %bb.0:
+; LA64-NEXT:  .LBB27_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.d $a3, $a0, 0
+; LA64-NEXT:    bne $a3, $a1, .LBB27_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB27_1 Depth=1
+; LA64-NEXT:    move $a4, $a2
+; LA64-NEXT:    sc.d $a4, $a0, 0
+; LA64-NEXT:    beqz $a4, .LBB27_1
+; LA64-NEXT:    b .LBB27_4
+; LA64-NEXT:  .LBB27_3:
+; LA64-NEXT:    dbar 1792
+; LA64-NEXT:  .LBB27_4:
+; LA64-NEXT:    xor $a0, $a3, $a1
+; LA64-NEXT:    sltui $a0, $a0, 1
+; LA64-NEXT:    ret
+  %tmp = cmpxchg ptr %ptr, i64 %cmp, i64 %val monotonic monotonic
+  %res = extractvalue { i64, i1 } %tmp, 1
+  ret i1 %res
+}
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw-fp.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw-fp.ll
index 9767717395..589360823b 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw-fp.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw-fp.ll
@@ -2,8 +2,6 @@
 ; RUN: llc --mtriple=loongarch64 --mattr=+f,-d < %s | FileCheck %s --check-prefix=LA64F
 ; RUN: llc --mtriple=loongarch64 --mattr=+d < %s | FileCheck %s --check-prefix=LA64D
 
-;; Add more test cases after supporting different AtomicOrdering.
-
 define float @float_fadd_acquire(ptr %p) nounwind {
 ; LA64F-LABEL: float_fadd_acquire:
 ; LA64F:       # %bb.0:
@@ -25,14 +23,13 @@ define float @float_fadd_acquire(ptr %p) nounwind {
 ; LA64F-NEXT:    bne $a3, $a2, .LBB0_5
 ; LA64F-NEXT:  # %bb.4: # %atomicrmw.start
 ; LA64F-NEXT:    # in Loop: Header=BB0_3 Depth=2
-; LA64F-NEXT:    dbar 0
 ; LA64F-NEXT:    move $a4, $a1
 ; LA64F-NEXT:    sc.w $a4, $a0, 0
 ; LA64F-NEXT:    beqz $a4, .LBB0_3
 ; LA64F-NEXT:    b .LBB0_6
 ; LA64F-NEXT:  .LBB0_5: # %atomicrmw.start
 ; LA64F-NEXT:    # in Loop: Header=BB0_1 Depth=1
-; LA64F-NEXT:    dbar 1792
+; LA64F-NEXT:    dbar 20
 ; LA64F-NEXT:  .LBB0_6: # %atomicrmw.start
 ; LA64F-NEXT:    # in Loop: Header=BB0_1 Depth=1
 ; LA64F-NEXT:    movgr2fr.w $fa0, $a3
@@ -61,14 +58,13 @@ define float @float_fadd_acquire(ptr %p) nounwind {
 ; LA64D-NEXT:    bne $a3, $a2, .LBB0_5
 ; LA64D-NEXT:  # %bb.4: # %atomicrmw.start
 ; LA64D-NEXT:    # in Loop: Header=BB0_3 Depth=2
-; LA64D-NEXT:    dbar 0
 ; LA64D-NEXT:    move $a4, $a1
 ; LA64D-NEXT:    sc.w $a4, $a0, 0
 ; LA64D-NEXT:    beqz $a4, .LBB0_3
 ; LA64D-NEXT:    b .LBB0_6
 ; LA64D-NEXT:  .LBB0_5: # %atomicrmw.start
 ; LA64D-NEXT:    # in Loop: Header=BB0_1 Depth=1
-; LA64D-NEXT:    dbar 1792
+; LA64D-NEXT:    dbar 20
 ; LA64D-NEXT:  .LBB0_6: # %atomicrmw.start
 ; LA64D-NEXT:    # in Loop: Header=BB0_1 Depth=1
 ; LA64D-NEXT:    movgr2fr.w $fa0, $a3
@@ -101,14 +97,13 @@ define float @float_fsub_acquire(ptr %p) nounwind {
 ; LA64F-NEXT:    bne $a3, $a2, .LBB1_5
 ; LA64F-NEXT:  # %bb.4: # %atomicrmw.start
 ; LA64F-NEXT:    # in Loop: Header=BB1_3 Depth=2
-; LA64F-NEXT:    dbar 0
 ; LA64F-NEXT:    move $a4, $a1
 ; LA64F-NEXT:    sc.w $a4, $a0, 0
 ; LA64F-NEXT:    beqz $a4, .LBB1_3
 ; LA64F-NEXT:    b .LBB1_6
 ; LA64F-NEXT:  .LBB1_5: # %atomicrmw.start
 ; LA64F-NEXT:    # in Loop: Header=BB1_1 Depth=1
-; LA64F-NEXT:    dbar 1792
+; LA64F-NEXT:    dbar 20
 ; LA64F-NEXT:  .LBB1_6: # %atomicrmw.start
 ; LA64F-NEXT:    # in Loop: Header=BB1_1 Depth=1
 ; LA64F-NEXT:    movgr2fr.w $fa0, $a3
@@ -137,14 +132,13 @@ define float @float_fsub_acquire(ptr %p) nounwind {
 ; LA64D-NEXT:    bne $a3, $a2, .LBB1_5
 ; LA64D-NEXT:  # %bb.4: # %atomicrmw.start
 ; LA64D-NEXT:    # in Loop: Header=BB1_3 Depth=2
-; LA64D-NEXT:    dbar 0
 ; LA64D-NEXT:    move $a4, $a1
 ; LA64D-NEXT:    sc.w $a4, $a0, 0
 ; LA64D-NEXT:    beqz $a4, .LBB1_3
 ; LA64D-NEXT:    b .LBB1_6
 ; LA64D-NEXT:  .LBB1_5: # %atomicrmw.start
 ; LA64D-NEXT:    # in Loop: Header=BB1_1 Depth=1
-; LA64D-NEXT:    dbar 1792
+; LA64D-NEXT:    dbar 20
 ; LA64D-NEXT:  .LBB1_6: # %atomicrmw.start
 ; LA64D-NEXT:    # in Loop: Header=BB1_1 Depth=1
 ; LA64D-NEXT:    movgr2fr.w $fa0, $a3
@@ -178,14 +172,13 @@ define float @float_fmin_acquire(ptr %p) nounwind {
 ; LA64F-NEXT:    bne $a3, $a2, .LBB2_5
 ; LA64F-NEXT:  # %bb.4: # %atomicrmw.start
 ; LA64F-NEXT:    # in Loop: Header=BB2_3 Depth=2
-; LA64F-NEXT:    dbar 0
 ; LA64F-NEXT:    move $a4, $a1
 ; LA64F-NEXT:    sc.w $a4, $a0, 0
 ; LA64F-NEXT:    beqz $a4, .LBB2_3
 ; LA64F-NEXT:    b .LBB2_6
 ; LA64F-NEXT:  .LBB2_5: # %atomicrmw.start
 ; LA64F-NEXT:    # in Loop: Header=BB2_1 Depth=1
-; LA64F-NEXT:    dbar 1792
+; LA64F-NEXT:    dbar 20
 ; LA64F-NEXT:  .LBB2_6: # %atomicrmw.start
 ; LA64F-NEXT:    # in Loop: Header=BB2_1 Depth=1
 ; LA64F-NEXT:    movgr2fr.w $fa0, $a3
@@ -215,14 +208,13 @@ define float @float_fmin_acquire(ptr %p) nounwind {
 ; LA64D-NEXT:    bne $a3, $a2, .LBB2_5
 ; LA64D-NEXT:  # %bb.4: # %atomicrmw.start
 ; LA64D-NEXT:    # in Loop: Header=BB2_3 Depth=2
-; LA64D-NEXT:    dbar 0
 ; LA64D-NEXT:    move $a4, $a1
 ; LA64D-NEXT:    sc.w $a4, $a0, 0
 ; LA64D-NEXT:    beqz $a4, .LBB2_3
 ; LA64D-NEXT:    b .LBB2_6
 ; LA64D-NEXT:  .LBB2_5: # %atomicrmw.start
 ; LA64D-NEXT:    # in Loop: Header=BB2_1 Depth=1
-; LA64D-NEXT:    dbar 1792
+; LA64D-NEXT:    dbar 20
 ; LA64D-NEXT:  .LBB2_6: # %atomicrmw.start
 ; LA64D-NEXT:    # in Loop: Header=BB2_1 Depth=1
 ; LA64D-NEXT:    movgr2fr.w $fa0, $a3
@@ -256,14 +248,13 @@ define float @float_fmax_acquire(ptr %p) nounwind {
 ; LA64F-NEXT:    bne $a3, $a2, .LBB3_5
 ; LA64F-NEXT:  # %bb.4: # %atomicrmw.start
 ; LA64F-NEXT:    # in Loop: Header=BB3_3 Depth=2
-; LA64F-NEXT:    dbar 0
 ; LA64F-NEXT:    move $a4, $a1
 ; LA64F-NEXT:    sc.w $a4, $a0, 0
 ; LA64F-NEXT:    beqz $a4, .LBB3_3
 ; LA64F-NEXT:    b .LBB3_6
 ; LA64F-NEXT:  .LBB3_5: # %atomicrmw.start
 ; LA64F-NEXT:    # in Loop: Header=BB3_1 Depth=1
-; LA64F-NEXT:    dbar 1792
+; LA64F-NEXT:    dbar 20
 ; LA64F-NEXT:  .LBB3_6: # %atomicrmw.start
 ; LA64F-NEXT:    # in Loop: Header=BB3_1 Depth=1
 ; LA64F-NEXT:    movgr2fr.w $fa0, $a3
@@ -293,14 +284,13 @@ define float @float_fmax_acquire(ptr %p) nounwind {
 ; LA64D-NEXT:    bne $a3, $a2, .LBB3_5
 ; LA64D-NEXT:  # %bb.4: # %atomicrmw.start
 ; LA64D-NEXT:    # in Loop: Header=BB3_3 Depth=2
-; LA64D-NEXT:    dbar 0
 ; LA64D-NEXT:    move $a4, $a1
 ; LA64D-NEXT:    sc.w $a4, $a0, 0
 ; LA64D-NEXT:    beqz $a4, .LBB3_3
 ; LA64D-NEXT:    b .LBB3_6
 ; LA64D-NEXT:  .LBB3_5: # %atomicrmw.start
 ; LA64D-NEXT:    # in Loop: Header=BB3_1 Depth=1
-; LA64D-NEXT:    dbar 1792
+; LA64D-NEXT:    dbar 20
 ; LA64D-NEXT:  .LBB3_6: # %atomicrmw.start
 ; LA64D-NEXT:    # in Loop: Header=BB3_1 Depth=1
 ; LA64D-NEXT:    movgr2fr.w $fa0, $a3
@@ -689,3 +679,2715 @@ define double @double_fmax_acquire(ptr %p) nounwind {
   %v = atomicrmw fmax ptr %p, double 1.0 acquire, align 4
   ret double %v
 }
+
+define float @float_fadd_release(ptr %p) nounwind {
+; LA64F-LABEL: float_fadd_release:
+; LA64F:       # %bb.0:
+; LA64F-NEXT:    fld.s $fa0, $a0, 0
+; LA64F-NEXT:    addi.w $a1, $zero, 1
+; LA64F-NEXT:    movgr2fr.w $fa1, $a1
+; LA64F-NEXT:    ffint.s.w $fa1, $fa1
+; LA64F-NEXT:    .p2align 4, , 16
+; LA64F-NEXT:  .LBB8_1: # %atomicrmw.start
+; LA64F-NEXT:    # =>This Loop Header: Depth=1
+; LA64F-NEXT:    # Child Loop BB8_3 Depth 2
+; LA64F-NEXT:    fadd.s $fa2, $fa0, $fa1
+; LA64F-NEXT:    movfr2gr.s $a1, $fa2
+; LA64F-NEXT:    movfr2gr.s $a2, $fa0
+; LA64F-NEXT:  .LBB8_3: # %atomicrmw.start
+; LA64F-NEXT:    # Parent Loop BB8_1 Depth=1
+; LA64F-NEXT:    # => This Inner Loop Header: Depth=2
+; LA64F-NEXT:    ll.w $a3, $a0, 0
+; LA64F-NEXT:    bne $a3, $a2, .LBB8_5
+; LA64F-NEXT:  # %bb.4: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB8_3 Depth=2
+; LA64F-NEXT:    move $a4, $a1
+; LA64F-NEXT:    sc.w $a4, $a0, 0
+; LA64F-NEXT:    beqz $a4, .LBB8_3
+; LA64F-NEXT:    b .LBB8_6
+; LA64F-NEXT:  .LBB8_5: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB8_1 Depth=1
+; LA64F-NEXT:    dbar 1792
+; LA64F-NEXT:  .LBB8_6: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB8_1 Depth=1
+; LA64F-NEXT:    movgr2fr.w $fa0, $a3
+; LA64F-NEXT:    addi.w $a1, $a2, 0
+; LA64F-NEXT:    bne $a3, $a1, .LBB8_1
+; LA64F-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64F-NEXT:    ret
+;
+; LA64D-LABEL: float_fadd_release:
+; LA64D:       # %bb.0:
+; LA64D-NEXT:    fld.s $fa0, $a0, 0
+; LA64D-NEXT:    addi.w $a1, $zero, 1
+; LA64D-NEXT:    movgr2fr.w $fa1, $a1
+; LA64D-NEXT:    ffint.s.w $fa1, $fa1
+; LA64D-NEXT:    .p2align 4, , 16
+; LA64D-NEXT:  .LBB8_1: # %atomicrmw.start
+; LA64D-NEXT:    # =>This Loop Header: Depth=1
+; LA64D-NEXT:    # Child Loop BB8_3 Depth 2
+; LA64D-NEXT:    fadd.s $fa2, $fa0, $fa1
+; LA64D-NEXT:    movfr2gr.s $a1, $fa2
+; LA64D-NEXT:    movfr2gr.s $a2, $fa0
+; LA64D-NEXT:  .LBB8_3: # %atomicrmw.start
+; LA64D-NEXT:    # Parent Loop BB8_1 Depth=1
+; LA64D-NEXT:    # => This Inner Loop Header: Depth=2
+; LA64D-NEXT:    ll.w $a3, $a0, 0
+; LA64D-NEXT:    bne $a3, $a2, .LBB8_5
+; LA64D-NEXT:  # %bb.4: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB8_3 Depth=2
+; LA64D-NEXT:    move $a4, $a1
+; LA64D-NEXT:    sc.w $a4, $a0, 0
+; LA64D-NEXT:    beqz $a4, .LBB8_3
+; LA64D-NEXT:    b .LBB8_6
+; LA64D-NEXT:  .LBB8_5: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB8_1 Depth=1
+; LA64D-NEXT:    dbar 1792
+; LA64D-NEXT:  .LBB8_6: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB8_1 Depth=1
+; LA64D-NEXT:    movgr2fr.w $fa0, $a3
+; LA64D-NEXT:    addi.w $a1, $a2, 0
+; LA64D-NEXT:    bne $a3, $a1, .LBB8_1
+; LA64D-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64D-NEXT:    ret
+  %v = atomicrmw fadd ptr %p, float 1.0 release, align 4
+  ret float %v
+}
+
+define float @float_fsub_release(ptr %p) nounwind {
+; LA64F-LABEL: float_fsub_release:
+; LA64F:       # %bb.0:
+; LA64F-NEXT:    fld.s $fa0, $a0, 0
+; LA64F-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI9_0)
+; LA64F-NEXT:    addi.d $a1, $a1, %pc_lo12(.LCPI9_0)
+; LA64F-NEXT:    fld.s $fa1, $a1, 0
+; LA64F-NEXT:    .p2align 4, , 16
+; LA64F-NEXT:  .LBB9_1: # %atomicrmw.start
+; LA64F-NEXT:    # =>This Loop Header: Depth=1
+; LA64F-NEXT:    # Child Loop BB9_3 Depth 2
+; LA64F-NEXT:    fadd.s $fa2, $fa0, $fa1
+; LA64F-NEXT:    movfr2gr.s $a1, $fa2
+; LA64F-NEXT:    movfr2gr.s $a2, $fa0
+; LA64F-NEXT:  .LBB9_3: # %atomicrmw.start
+; LA64F-NEXT:    # Parent Loop BB9_1 Depth=1
+; LA64F-NEXT:    # => This Inner Loop Header: Depth=2
+; LA64F-NEXT:    ll.w $a3, $a0, 0
+; LA64F-NEXT:    bne $a3, $a2, .LBB9_5
+; LA64F-NEXT:  # %bb.4: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB9_3 Depth=2
+; LA64F-NEXT:    move $a4, $a1
+; LA64F-NEXT:    sc.w $a4, $a0, 0
+; LA64F-NEXT:    beqz $a4, .LBB9_3
+; LA64F-NEXT:    b .LBB9_6
+; LA64F-NEXT:  .LBB9_5: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB9_1 Depth=1
+; LA64F-NEXT:    dbar 1792
+; LA64F-NEXT:  .LBB9_6: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB9_1 Depth=1
+; LA64F-NEXT:    movgr2fr.w $fa0, $a3
+; LA64F-NEXT:    addi.w $a1, $a2, 0
+; LA64F-NEXT:    bne $a3, $a1, .LBB9_1
+; LA64F-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64F-NEXT:    ret
+;
+; LA64D-LABEL: float_fsub_release:
+; LA64D:       # %bb.0:
+; LA64D-NEXT:    fld.s $fa0, $a0, 0
+; LA64D-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI9_0)
+; LA64D-NEXT:    addi.d $a1, $a1, %pc_lo12(.LCPI9_0)
+; LA64D-NEXT:    fld.s $fa1, $a1, 0
+; LA64D-NEXT:    .p2align 4, , 16
+; LA64D-NEXT:  .LBB9_1: # %atomicrmw.start
+; LA64D-NEXT:    # =>This Loop Header: Depth=1
+; LA64D-NEXT:    # Child Loop BB9_3 Depth 2
+; LA64D-NEXT:    fadd.s $fa2, $fa0, $fa1
+; LA64D-NEXT:    movfr2gr.s $a1, $fa2
+; LA64D-NEXT:    movfr2gr.s $a2, $fa0
+; LA64D-NEXT:  .LBB9_3: # %atomicrmw.start
+; LA64D-NEXT:    # Parent Loop BB9_1 Depth=1
+; LA64D-NEXT:    # => This Inner Loop Header: Depth=2
+; LA64D-NEXT:    ll.w $a3, $a0, 0
+; LA64D-NEXT:    bne $a3, $a2, .LBB9_5
+; LA64D-NEXT:  # %bb.4: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB9_3 Depth=2
+; LA64D-NEXT:    move $a4, $a1
+; LA64D-NEXT:    sc.w $a4, $a0, 0
+; LA64D-NEXT:    beqz $a4, .LBB9_3
+; LA64D-NEXT:    b .LBB9_6
+; LA64D-NEXT:  .LBB9_5: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB9_1 Depth=1
+; LA64D-NEXT:    dbar 1792
+; LA64D-NEXT:  .LBB9_6: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB9_1 Depth=1
+; LA64D-NEXT:    movgr2fr.w $fa0, $a3
+; LA64D-NEXT:    addi.w $a1, $a2, 0
+; LA64D-NEXT:    bne $a3, $a1, .LBB9_1
+; LA64D-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64D-NEXT:    ret
+  %v = atomicrmw fsub ptr %p, float 1.0 release, align 4
+  ret float %v
+}
+
+define float @float_fmin_release(ptr %p) nounwind {
+; LA64F-LABEL: float_fmin_release:
+; LA64F:       # %bb.0:
+; LA64F-NEXT:    fld.s $fa0, $a0, 0
+; LA64F-NEXT:    addi.w $a1, $zero, 1
+; LA64F-NEXT:    movgr2fr.w $fa1, $a1
+; LA64F-NEXT:    ffint.s.w $fa1, $fa1
+; LA64F-NEXT:    .p2align 4, , 16
+; LA64F-NEXT:  .LBB10_1: # %atomicrmw.start
+; LA64F-NEXT:    # =>This Loop Header: Depth=1
+; LA64F-NEXT:    # Child Loop BB10_3 Depth 2
+; LA64F-NEXT:    fmax.s $fa2, $fa0, $fa0
+; LA64F-NEXT:    fmin.s $fa2, $fa2, $fa1
+; LA64F-NEXT:    movfr2gr.s $a1, $fa2
+; LA64F-NEXT:    movfr2gr.s $a2, $fa0
+; LA64F-NEXT:  .LBB10_3: # %atomicrmw.start
+; LA64F-NEXT:    # Parent Loop BB10_1 Depth=1
+; LA64F-NEXT:    # => This Inner Loop Header: Depth=2
+; LA64F-NEXT:    ll.w $a3, $a0, 0
+; LA64F-NEXT:    bne $a3, $a2, .LBB10_5
+; LA64F-NEXT:  # %bb.4: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB10_3 Depth=2
+; LA64F-NEXT:    move $a4, $a1
+; LA64F-NEXT:    sc.w $a4, $a0, 0
+; LA64F-NEXT:    beqz $a4, .LBB10_3
+; LA64F-NEXT:    b .LBB10_6
+; LA64F-NEXT:  .LBB10_5: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB10_1 Depth=1
+; LA64F-NEXT:    dbar 1792
+; LA64F-NEXT:  .LBB10_6: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB10_1 Depth=1
+; LA64F-NEXT:    movgr2fr.w $fa0, $a3
+; LA64F-NEXT:    addi.w $a1, $a2, 0
+; LA64F-NEXT:    bne $a3, $a1, .LBB10_1
+; LA64F-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64F-NEXT:    ret
+;
+; LA64D-LABEL: float_fmin_release:
+; LA64D:       # %bb.0:
+; LA64D-NEXT:    fld.s $fa0, $a0, 0
+; LA64D-NEXT:    addi.w $a1, $zero, 1
+; LA64D-NEXT:    movgr2fr.w $fa1, $a1
+; LA64D-NEXT:    ffint.s.w $fa1, $fa1
+; LA64D-NEXT:    .p2align 4, , 16
+; LA64D-NEXT:  .LBB10_1: # %atomicrmw.start
+; LA64D-NEXT:    # =>This Loop Header: Depth=1
+; LA64D-NEXT:    # Child Loop BB10_3 Depth 2
+; LA64D-NEXT:    fmax.s $fa2, $fa0, $fa0
+; LA64D-NEXT:    fmin.s $fa2, $fa2, $fa1
+; LA64D-NEXT:    movfr2gr.s $a1, $fa2
+; LA64D-NEXT:    movfr2gr.s $a2, $fa0
+; LA64D-NEXT:  .LBB10_3: # %atomicrmw.start
+; LA64D-NEXT:    # Parent Loop BB10_1 Depth=1
+; LA64D-NEXT:    # => This Inner Loop Header: Depth=2
+; LA64D-NEXT:    ll.w $a3, $a0, 0
+; LA64D-NEXT:    bne $a3, $a2, .LBB10_5
+; LA64D-NEXT:  # %bb.4: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB10_3 Depth=2
+; LA64D-NEXT:    move $a4, $a1
+; LA64D-NEXT:    sc.w $a4, $a0, 0
+; LA64D-NEXT:    beqz $a4, .LBB10_3
+; LA64D-NEXT:    b .LBB10_6
+; LA64D-NEXT:  .LBB10_5: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB10_1 Depth=1
+; LA64D-NEXT:    dbar 1792
+; LA64D-NEXT:  .LBB10_6: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB10_1 Depth=1
+; LA64D-NEXT:    movgr2fr.w $fa0, $a3
+; LA64D-NEXT:    addi.w $a1, $a2, 0
+; LA64D-NEXT:    bne $a3, $a1, .LBB10_1
+; LA64D-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64D-NEXT:    ret
+  %v = atomicrmw fmin ptr %p, float 1.0 release, align 4
+  ret float %v
+}
+
+define float @float_fmax_release(ptr %p) nounwind {
+; LA64F-LABEL: float_fmax_release:
+; LA64F:       # %bb.0:
+; LA64F-NEXT:    fld.s $fa0, $a0, 0
+; LA64F-NEXT:    addi.w $a1, $zero, 1
+; LA64F-NEXT:    movgr2fr.w $fa1, $a1
+; LA64F-NEXT:    ffint.s.w $fa1, $fa1
+; LA64F-NEXT:    .p2align 4, , 16
+; LA64F-NEXT:  .LBB11_1: # %atomicrmw.start
+; LA64F-NEXT:    # =>This Loop Header: Depth=1
+; LA64F-NEXT:    # Child Loop BB11_3 Depth 2
+; LA64F-NEXT:    fmax.s $fa2, $fa0, $fa0
+; LA64F-NEXT:    fmax.s $fa2, $fa2, $fa1
+; LA64F-NEXT:    movfr2gr.s $a1, $fa2
+; LA64F-NEXT:    movfr2gr.s $a2, $fa0
+; LA64F-NEXT:  .LBB11_3: # %atomicrmw.start
+; LA64F-NEXT:    # Parent Loop BB11_1 Depth=1
+; LA64F-NEXT:    # => This Inner Loop Header: Depth=2
+; LA64F-NEXT:    ll.w $a3, $a0, 0
+; LA64F-NEXT:    bne $a3, $a2, .LBB11_5
+; LA64F-NEXT:  # %bb.4: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB11_3 Depth=2
+; LA64F-NEXT:    move $a4, $a1
+; LA64F-NEXT:    sc.w $a4, $a0, 0
+; LA64F-NEXT:    beqz $a4, .LBB11_3
+; LA64F-NEXT:    b .LBB11_6
+; LA64F-NEXT:  .LBB11_5: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB11_1 Depth=1
+; LA64F-NEXT:    dbar 1792
+; LA64F-NEXT:  .LBB11_6: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB11_1 Depth=1
+; LA64F-NEXT:    movgr2fr.w $fa0, $a3
+; LA64F-NEXT:    addi.w $a1, $a2, 0
+; LA64F-NEXT:    bne $a3, $a1, .LBB11_1
+; LA64F-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64F-NEXT:    ret
+;
+; LA64D-LABEL: float_fmax_release:
+; LA64D:       # %bb.0:
+; LA64D-NEXT:    fld.s $fa0, $a0, 0
+; LA64D-NEXT:    addi.w $a1, $zero, 1
+; LA64D-NEXT:    movgr2fr.w $fa1, $a1
+; LA64D-NEXT:    ffint.s.w $fa1, $fa1
+; LA64D-NEXT:    .p2align 4, , 16
+; LA64D-NEXT:  .LBB11_1: # %atomicrmw.start
+; LA64D-NEXT:    # =>This Loop Header: Depth=1
+; LA64D-NEXT:    # Child Loop BB11_3 Depth 2
+; LA64D-NEXT:    fmax.s $fa2, $fa0, $fa0
+; LA64D-NEXT:    fmax.s $fa2, $fa2, $fa1
+; LA64D-NEXT:    movfr2gr.s $a1, $fa2
+; LA64D-NEXT:    movfr2gr.s $a2, $fa0
+; LA64D-NEXT:  .LBB11_3: # %atomicrmw.start
+; LA64D-NEXT:    # Parent Loop BB11_1 Depth=1
+; LA64D-NEXT:    # => This Inner Loop Header: Depth=2
+; LA64D-NEXT:    ll.w $a3, $a0, 0
+; LA64D-NEXT:    bne $a3, $a2, .LBB11_5
+; LA64D-NEXT:  # %bb.4: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB11_3 Depth=2
+; LA64D-NEXT:    move $a4, $a1
+; LA64D-NEXT:    sc.w $a4, $a0, 0
+; LA64D-NEXT:    beqz $a4, .LBB11_3
+; LA64D-NEXT:    b .LBB11_6
+; LA64D-NEXT:  .LBB11_5: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB11_1 Depth=1
+; LA64D-NEXT:    dbar 1792
+; LA64D-NEXT:  .LBB11_6: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB11_1 Depth=1
+; LA64D-NEXT:    movgr2fr.w $fa0, $a3
+; LA64D-NEXT:    addi.w $a1, $a2, 0
+; LA64D-NEXT:    bne $a3, $a1, .LBB11_1
+; LA64D-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64D-NEXT:    ret
+  %v = atomicrmw fmax ptr %p, float 1.0 release, align 4
+  ret float %v
+}
+
+define double @double_fadd_release(ptr %p) nounwind {
+; LA64F-LABEL: double_fadd_release:
+; LA64F:       # %bb.0:
+; LA64F-NEXT:    addi.d $sp, $sp, -80
+; LA64F-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $fp, $sp, 64 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s0, $sp, 56 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s1, $sp, 48 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s2, $sp, 40 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s3, $sp, 32 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s4, $sp, 24 # 8-byte Folded Spill
+; LA64F-NEXT:    move $fp, $a0
+; LA64F-NEXT:    ld.d $a0, $a0, 0
+; LA64F-NEXT:    lu52i.d $s0, $zero, 1023
+; LA64F-NEXT:    ori $s1, $zero, 8
+; LA64F-NEXT:    addi.d $s2, $sp, 16
+; LA64F-NEXT:    addi.d $s3, $sp, 8
+; LA64F-NEXT:    ori $s4, $zero, 3
+; LA64F-NEXT:    .p2align 4, , 16
+; LA64F-NEXT:  .LBB12_1: # %atomicrmw.start
+; LA64F-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64F-NEXT:    st.d $a0, $sp, 16
+; LA64F-NEXT:    move $a1, $s0
+; LA64F-NEXT:    bl %plt(__adddf3)
+; LA64F-NEXT:    st.d $a0, $sp, 8
+; LA64F-NEXT:    move $a0, $s1
+; LA64F-NEXT:    move $a1, $fp
+; LA64F-NEXT:    move $a2, $s2
+; LA64F-NEXT:    move $a3, $s3
+; LA64F-NEXT:    move $a4, $s4
+; LA64F-NEXT:    move $a5, $zero
+; LA64F-NEXT:    bl %plt(__atomic_compare_exchange)
+; LA64F-NEXT:    move $a1, $a0
+; LA64F-NEXT:    ld.d $a0, $sp, 16
+; LA64F-NEXT:    beqz $a1, .LBB12_1
+; LA64F-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64F-NEXT:    ld.d $s4, $sp, 24 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s3, $sp, 32 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s2, $sp, 40 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s1, $sp, 48 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s0, $sp, 56 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $fp, $sp, 64 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
+; LA64F-NEXT:    addi.d $sp, $sp, 80
+; LA64F-NEXT:    ret
+;
+; LA64D-LABEL: double_fadd_release:
+; LA64D:       # %bb.0:
+; LA64D-NEXT:    addi.d $sp, $sp, -80
+; LA64D-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $fp, $sp, 64 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s0, $sp, 56 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s1, $sp, 48 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s2, $sp, 40 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s3, $sp, 32 # 8-byte Folded Spill
+; LA64D-NEXT:    fst.d $fs0, $sp, 24 # 8-byte Folded Spill
+; LA64D-NEXT:    move $fp, $a0
+; LA64D-NEXT:    fld.d $fa0, $a0, 0
+; LA64D-NEXT:    addi.d $a0, $zero, 1
+; LA64D-NEXT:    movgr2fr.d $fa1, $a0
+; LA64D-NEXT:    ffint.d.l $fs0, $fa1
+; LA64D-NEXT:    ori $s0, $zero, 8
+; LA64D-NEXT:    addi.d $s1, $sp, 16
+; LA64D-NEXT:    addi.d $s2, $sp, 8
+; LA64D-NEXT:    ori $s3, $zero, 3
+; LA64D-NEXT:    .p2align 4, , 16
+; LA64D-NEXT:  .LBB12_1: # %atomicrmw.start
+; LA64D-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64D-NEXT:    fst.d $fa0, $sp, 16
+; LA64D-NEXT:    fadd.d $fa0, $fa0, $fs0
+; LA64D-NEXT:    fst.d $fa0, $sp, 8
+; LA64D-NEXT:    move $a0, $s0
+; LA64D-NEXT:    move $a1, $fp
+; LA64D-NEXT:    move $a2, $s1
+; LA64D-NEXT:    move $a3, $s2
+; LA64D-NEXT:    move $a4, $s3
+; LA64D-NEXT:    move $a5, $zero
+; LA64D-NEXT:    bl %plt(__atomic_compare_exchange)
+; LA64D-NEXT:    fld.d $fa0, $sp, 16
+; LA64D-NEXT:    beqz $a0, .LBB12_1
+; LA64D-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64D-NEXT:    fld.d $fs0, $sp, 24 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s3, $sp, 32 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s2, $sp, 40 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s1, $sp, 48 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s0, $sp, 56 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $fp, $sp, 64 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
+; LA64D-NEXT:    addi.d $sp, $sp, 80
+; LA64D-NEXT:    ret
+  %v = atomicrmw fadd ptr %p, double 1.0 release, align 4
+  ret double %v
+}
+
+define double @double_fsub_release(ptr %p) nounwind {
+; LA64F-LABEL: double_fsub_release:
+; LA64F:       # %bb.0:
+; LA64F-NEXT:    addi.d $sp, $sp, -80
+; LA64F-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $fp, $sp, 64 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s0, $sp, 56 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s1, $sp, 48 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s2, $sp, 40 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s3, $sp, 32 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s4, $sp, 24 # 8-byte Folded Spill
+; LA64F-NEXT:    move $fp, $a0
+; LA64F-NEXT:    ld.d $a0, $a0, 0
+; LA64F-NEXT:    lu52i.d $s0, $zero, -1025
+; LA64F-NEXT:    ori $s1, $zero, 8
+; LA64F-NEXT:    addi.d $s2, $sp, 16
+; LA64F-NEXT:    addi.d $s3, $sp, 8
+; LA64F-NEXT:    ori $s4, $zero, 3
+; LA64F-NEXT:    .p2align 4, , 16
+; LA64F-NEXT:  .LBB13_1: # %atomicrmw.start
+; LA64F-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64F-NEXT:    st.d $a0, $sp, 16
+; LA64F-NEXT:    move $a1, $s0
+; LA64F-NEXT:    bl %plt(__adddf3)
+; LA64F-NEXT:    st.d $a0, $sp, 8
+; LA64F-NEXT:    move $a0, $s1
+; LA64F-NEXT:    move $a1, $fp
+; LA64F-NEXT:    move $a2, $s2
+; LA64F-NEXT:    move $a3, $s3
+; LA64F-NEXT:    move $a4, $s4
+; LA64F-NEXT:    move $a5, $zero
+; LA64F-NEXT:    bl %plt(__atomic_compare_exchange)
+; LA64F-NEXT:    move $a1, $a0
+; LA64F-NEXT:    ld.d $a0, $sp, 16
+; LA64F-NEXT:    beqz $a1, .LBB13_1
+; LA64F-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64F-NEXT:    ld.d $s4, $sp, 24 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s3, $sp, 32 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s2, $sp, 40 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s1, $sp, 48 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s0, $sp, 56 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $fp, $sp, 64 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
+; LA64F-NEXT:    addi.d $sp, $sp, 80
+; LA64F-NEXT:    ret
+;
+; LA64D-LABEL: double_fsub_release:
+; LA64D:       # %bb.0:
+; LA64D-NEXT:    addi.d $sp, $sp, -80
+; LA64D-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $fp, $sp, 64 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s0, $sp, 56 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s1, $sp, 48 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s2, $sp, 40 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s3, $sp, 32 # 8-byte Folded Spill
+; LA64D-NEXT:    fst.d $fs0, $sp, 24 # 8-byte Folded Spill
+; LA64D-NEXT:    move $fp, $a0
+; LA64D-NEXT:    fld.d $fa0, $a0, 0
+; LA64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI13_0)
+; LA64D-NEXT:    addi.d $a0, $a0, %pc_lo12(.LCPI13_0)
+; LA64D-NEXT:    fld.d $fs0, $a0, 0
+; LA64D-NEXT:    ori $s0, $zero, 8
+; LA64D-NEXT:    addi.d $s1, $sp, 16
+; LA64D-NEXT:    addi.d $s2, $sp, 8
+; LA64D-NEXT:    ori $s3, $zero, 3
+; LA64D-NEXT:    .p2align 4, , 16
+; LA64D-NEXT:  .LBB13_1: # %atomicrmw.start
+; LA64D-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64D-NEXT:    fst.d $fa0, $sp, 16
+; LA64D-NEXT:    fadd.d $fa0, $fa0, $fs0
+; LA64D-NEXT:    fst.d $fa0, $sp, 8
+; LA64D-NEXT:    move $a0, $s0
+; LA64D-NEXT:    move $a1, $fp
+; LA64D-NEXT:    move $a2, $s1
+; LA64D-NEXT:    move $a3, $s2
+; LA64D-NEXT:    move $a4, $s3
+; LA64D-NEXT:    move $a5, $zero
+; LA64D-NEXT:    bl %plt(__atomic_compare_exchange)
+; LA64D-NEXT:    fld.d $fa0, $sp, 16
+; LA64D-NEXT:    beqz $a0, .LBB13_1
+; LA64D-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64D-NEXT:    fld.d $fs0, $sp, 24 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s3, $sp, 32 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s2, $sp, 40 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s1, $sp, 48 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s0, $sp, 56 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $fp, $sp, 64 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
+; LA64D-NEXT:    addi.d $sp, $sp, 80
+; LA64D-NEXT:    ret
+  %v = atomicrmw fsub ptr %p, double 1.0 release, align 4
+  ret double %v
+}
+
+define double @double_fmin_release(ptr %p) nounwind {
+; LA64F-LABEL: double_fmin_release:
+; LA64F:       # %bb.0:
+; LA64F-NEXT:    addi.d $sp, $sp, -80
+; LA64F-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $fp, $sp, 64 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s0, $sp, 56 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s1, $sp, 48 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s2, $sp, 40 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s3, $sp, 32 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s4, $sp, 24 # 8-byte Folded Spill
+; LA64F-NEXT:    move $fp, $a0
+; LA64F-NEXT:    ld.d $a0, $a0, 0
+; LA64F-NEXT:    lu52i.d $s0, $zero, 1023
+; LA64F-NEXT:    ori $s1, $zero, 8
+; LA64F-NEXT:    addi.d $s2, $sp, 16
+; LA64F-NEXT:    addi.d $s3, $sp, 8
+; LA64F-NEXT:    ori $s4, $zero, 3
+; LA64F-NEXT:    .p2align 4, , 16
+; LA64F-NEXT:  .LBB14_1: # %atomicrmw.start
+; LA64F-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64F-NEXT:    st.d $a0, $sp, 16
+; LA64F-NEXT:    move $a1, $s0
+; LA64F-NEXT:    bl %plt(fmin)
+; LA64F-NEXT:    st.d $a0, $sp, 8
+; LA64F-NEXT:    move $a0, $s1
+; LA64F-NEXT:    move $a1, $fp
+; LA64F-NEXT:    move $a2, $s2
+; LA64F-NEXT:    move $a3, $s3
+; LA64F-NEXT:    move $a4, $s4
+; LA64F-NEXT:    move $a5, $zero
+; LA64F-NEXT:    bl %plt(__atomic_compare_exchange)
+; LA64F-NEXT:    move $a1, $a0
+; LA64F-NEXT:    ld.d $a0, $sp, 16
+; LA64F-NEXT:    beqz $a1, .LBB14_1
+; LA64F-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64F-NEXT:    ld.d $s4, $sp, 24 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s3, $sp, 32 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s2, $sp, 40 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s1, $sp, 48 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s0, $sp, 56 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $fp, $sp, 64 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
+; LA64F-NEXT:    addi.d $sp, $sp, 80
+; LA64F-NEXT:    ret
+;
+; LA64D-LABEL: double_fmin_release:
+; LA64D:       # %bb.0:
+; LA64D-NEXT:    addi.d $sp, $sp, -80
+; LA64D-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $fp, $sp, 64 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s0, $sp, 56 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s1, $sp, 48 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s2, $sp, 40 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s3, $sp, 32 # 8-byte Folded Spill
+; LA64D-NEXT:    fst.d $fs0, $sp, 24 # 8-byte Folded Spill
+; LA64D-NEXT:    move $fp, $a0
+; LA64D-NEXT:    fld.d $fa0, $a0, 0
+; LA64D-NEXT:    addi.d $a0, $zero, 1
+; LA64D-NEXT:    movgr2fr.d $fa1, $a0
+; LA64D-NEXT:    ffint.d.l $fs0, $fa1
+; LA64D-NEXT:    ori $s0, $zero, 8
+; LA64D-NEXT:    addi.d $s1, $sp, 16
+; LA64D-NEXT:    addi.d $s2, $sp, 8
+; LA64D-NEXT:    ori $s3, $zero, 3
+; LA64D-NEXT:    .p2align 4, , 16
+; LA64D-NEXT:  .LBB14_1: # %atomicrmw.start
+; LA64D-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64D-NEXT:    fst.d $fa0, $sp, 16
+; LA64D-NEXT:    fmax.d $fa0, $fa0, $fa0
+; LA64D-NEXT:    fmin.d $fa0, $fa0, $fs0
+; LA64D-NEXT:    fst.d $fa0, $sp, 8
+; LA64D-NEXT:    move $a0, $s0
+; LA64D-NEXT:    move $a1, $fp
+; LA64D-NEXT:    move $a2, $s1
+; LA64D-NEXT:    move $a3, $s2
+; LA64D-NEXT:    move $a4, $s3
+; LA64D-NEXT:    move $a5, $zero
+; LA64D-NEXT:    bl %plt(__atomic_compare_exchange)
+; LA64D-NEXT:    fld.d $fa0, $sp, 16
+; LA64D-NEXT:    beqz $a0, .LBB14_1
+; LA64D-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64D-NEXT:    fld.d $fs0, $sp, 24 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s3, $sp, 32 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s2, $sp, 40 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s1, $sp, 48 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s0, $sp, 56 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $fp, $sp, 64 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
+; LA64D-NEXT:    addi.d $sp, $sp, 80
+; LA64D-NEXT:    ret
+  %v = atomicrmw fmin ptr %p, double 1.0 release, align 4
+  ret double %v
+}
+
+define double @double_fmax_release(ptr %p) nounwind {
+; LA64F-LABEL: double_fmax_release:
+; LA64F:       # %bb.0:
+; LA64F-NEXT:    addi.d $sp, $sp, -80
+; LA64F-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $fp, $sp, 64 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s0, $sp, 56 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s1, $sp, 48 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s2, $sp, 40 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s3, $sp, 32 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s4, $sp, 24 # 8-byte Folded Spill
+; LA64F-NEXT:    move $fp, $a0
+; LA64F-NEXT:    ld.d $a0, $a0, 0
+; LA64F-NEXT:    lu52i.d $s0, $zero, 1023
+; LA64F-NEXT:    ori $s1, $zero, 8
+; LA64F-NEXT:    addi.d $s2, $sp, 16
+; LA64F-NEXT:    addi.d $s3, $sp, 8
+; LA64F-NEXT:    ori $s4, $zero, 3
+; LA64F-NEXT:    .p2align 4, , 16
+; LA64F-NEXT:  .LBB15_1: # %atomicrmw.start
+; LA64F-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64F-NEXT:    st.d $a0, $sp, 16
+; LA64F-NEXT:    move $a1, $s0
+; LA64F-NEXT:    bl %plt(fmax)
+; LA64F-NEXT:    st.d $a0, $sp, 8
+; LA64F-NEXT:    move $a0, $s1
+; LA64F-NEXT:    move $a1, $fp
+; LA64F-NEXT:    move $a2, $s2
+; LA64F-NEXT:    move $a3, $s3
+; LA64F-NEXT:    move $a4, $s4
+; LA64F-NEXT:    move $a5, $zero
+; LA64F-NEXT:    bl %plt(__atomic_compare_exchange)
+; LA64F-NEXT:    move $a1, $a0
+; LA64F-NEXT:    ld.d $a0, $sp, 16
+; LA64F-NEXT:    beqz $a1, .LBB15_1
+; LA64F-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64F-NEXT:    ld.d $s4, $sp, 24 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s3, $sp, 32 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s2, $sp, 40 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s1, $sp, 48 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s0, $sp, 56 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $fp, $sp, 64 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
+; LA64F-NEXT:    addi.d $sp, $sp, 80
+; LA64F-NEXT:    ret
+;
+; LA64D-LABEL: double_fmax_release:
+; LA64D:       # %bb.0:
+; LA64D-NEXT:    addi.d $sp, $sp, -80
+; LA64D-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $fp, $sp, 64 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s0, $sp, 56 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s1, $sp, 48 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s2, $sp, 40 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s3, $sp, 32 # 8-byte Folded Spill
+; LA64D-NEXT:    fst.d $fs0, $sp, 24 # 8-byte Folded Spill
+; LA64D-NEXT:    move $fp, $a0
+; LA64D-NEXT:    fld.d $fa0, $a0, 0
+; LA64D-NEXT:    addi.d $a0, $zero, 1
+; LA64D-NEXT:    movgr2fr.d $fa1, $a0
+; LA64D-NEXT:    ffint.d.l $fs0, $fa1
+; LA64D-NEXT:    ori $s0, $zero, 8
+; LA64D-NEXT:    addi.d $s1, $sp, 16
+; LA64D-NEXT:    addi.d $s2, $sp, 8
+; LA64D-NEXT:    ori $s3, $zero, 3
+; LA64D-NEXT:    .p2align 4, , 16
+; LA64D-NEXT:  .LBB15_1: # %atomicrmw.start
+; LA64D-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64D-NEXT:    fst.d $fa0, $sp, 16
+; LA64D-NEXT:    fmax.d $fa0, $fa0, $fa0
+; LA64D-NEXT:    fmax.d $fa0, $fa0, $fs0
+; LA64D-NEXT:    fst.d $fa0, $sp, 8
+; LA64D-NEXT:    move $a0, $s0
+; LA64D-NEXT:    move $a1, $fp
+; LA64D-NEXT:    move $a2, $s1
+; LA64D-NEXT:    move $a3, $s2
+; LA64D-NEXT:    move $a4, $s3
+; LA64D-NEXT:    move $a5, $zero
+; LA64D-NEXT:    bl %plt(__atomic_compare_exchange)
+; LA64D-NEXT:    fld.d $fa0, $sp, 16
+; LA64D-NEXT:    beqz $a0, .LBB15_1
+; LA64D-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64D-NEXT:    fld.d $fs0, $sp, 24 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s3, $sp, 32 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s2, $sp, 40 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s1, $sp, 48 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s0, $sp, 56 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $fp, $sp, 64 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
+; LA64D-NEXT:    addi.d $sp, $sp, 80
+; LA64D-NEXT:    ret
+  %v = atomicrmw fmax ptr %p, double 1.0 release, align 4
+  ret double %v
+}
+
+define float @float_fadd_acq_rel(ptr %p) nounwind {
+; LA64F-LABEL: float_fadd_acq_rel:
+; LA64F:       # %bb.0:
+; LA64F-NEXT:    fld.s $fa0, $a0, 0
+; LA64F-NEXT:    addi.w $a1, $zero, 1
+; LA64F-NEXT:    movgr2fr.w $fa1, $a1
+; LA64F-NEXT:    ffint.s.w $fa1, $fa1
+; LA64F-NEXT:    .p2align 4, , 16
+; LA64F-NEXT:  .LBB16_1: # %atomicrmw.start
+; LA64F-NEXT:    # =>This Loop Header: Depth=1
+; LA64F-NEXT:    # Child Loop BB16_3 Depth 2
+; LA64F-NEXT:    fadd.s $fa2, $fa0, $fa1
+; LA64F-NEXT:    movfr2gr.s $a1, $fa2
+; LA64F-NEXT:    movfr2gr.s $a2, $fa0
+; LA64F-NEXT:  .LBB16_3: # %atomicrmw.start
+; LA64F-NEXT:    # Parent Loop BB16_1 Depth=1
+; LA64F-NEXT:    # => This Inner Loop Header: Depth=2
+; LA64F-NEXT:    ll.w $a3, $a0, 0
+; LA64F-NEXT:    bne $a3, $a2, .LBB16_5
+; LA64F-NEXT:  # %bb.4: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB16_3 Depth=2
+; LA64F-NEXT:    move $a4, $a1
+; LA64F-NEXT:    sc.w $a4, $a0, 0
+; LA64F-NEXT:    beqz $a4, .LBB16_3
+; LA64F-NEXT:    b .LBB16_6
+; LA64F-NEXT:  .LBB16_5: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB16_1 Depth=1
+; LA64F-NEXT:    dbar 20
+; LA64F-NEXT:  .LBB16_6: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB16_1 Depth=1
+; LA64F-NEXT:    movgr2fr.w $fa0, $a3
+; LA64F-NEXT:    addi.w $a1, $a2, 0
+; LA64F-NEXT:    bne $a3, $a1, .LBB16_1
+; LA64F-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64F-NEXT:    ret
+;
+; LA64D-LABEL: float_fadd_acq_rel:
+; LA64D:       # %bb.0:
+; LA64D-NEXT:    fld.s $fa0, $a0, 0
+; LA64D-NEXT:    addi.w $a1, $zero, 1
+; LA64D-NEXT:    movgr2fr.w $fa1, $a1
+; LA64D-NEXT:    ffint.s.w $fa1, $fa1
+; LA64D-NEXT:    .p2align 4, , 16
+; LA64D-NEXT:  .LBB16_1: # %atomicrmw.start
+; LA64D-NEXT:    # =>This Loop Header: Depth=1
+; LA64D-NEXT:    # Child Loop BB16_3 Depth 2
+; LA64D-NEXT:    fadd.s $fa2, $fa0, $fa1
+; LA64D-NEXT:    movfr2gr.s $a1, $fa2
+; LA64D-NEXT:    movfr2gr.s $a2, $fa0
+; LA64D-NEXT:  .LBB16_3: # %atomicrmw.start
+; LA64D-NEXT:    # Parent Loop BB16_1 Depth=1
+; LA64D-NEXT:    # => This Inner Loop Header: Depth=2
+; LA64D-NEXT:    ll.w $a3, $a0, 0
+; LA64D-NEXT:    bne $a3, $a2, .LBB16_5
+; LA64D-NEXT:  # %bb.4: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB16_3 Depth=2
+; LA64D-NEXT:    move $a4, $a1
+; LA64D-NEXT:    sc.w $a4, $a0, 0
+; LA64D-NEXT:    beqz $a4, .LBB16_3
+; LA64D-NEXT:    b .LBB16_6
+; LA64D-NEXT:  .LBB16_5: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB16_1 Depth=1
+; LA64D-NEXT:    dbar 20
+; LA64D-NEXT:  .LBB16_6: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB16_1 Depth=1
+; LA64D-NEXT:    movgr2fr.w $fa0, $a3
+; LA64D-NEXT:    addi.w $a1, $a2, 0
+; LA64D-NEXT:    bne $a3, $a1, .LBB16_1
+; LA64D-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64D-NEXT:    ret
+  %v = atomicrmw fadd ptr %p, float 1.0 acq_rel, align 4
+  ret float %v
+}
+
+define float @float_fsub_acq_rel(ptr %p) nounwind {
+; LA64F-LABEL: float_fsub_acq_rel:
+; LA64F:       # %bb.0:
+; LA64F-NEXT:    fld.s $fa0, $a0, 0
+; LA64F-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI17_0)
+; LA64F-NEXT:    addi.d $a1, $a1, %pc_lo12(.LCPI17_0)
+; LA64F-NEXT:    fld.s $fa1, $a1, 0
+; LA64F-NEXT:    .p2align 4, , 16
+; LA64F-NEXT:  .LBB17_1: # %atomicrmw.start
+; LA64F-NEXT:    # =>This Loop Header: Depth=1
+; LA64F-NEXT:    # Child Loop BB17_3 Depth 2
+; LA64F-NEXT:    fadd.s $fa2, $fa0, $fa1
+; LA64F-NEXT:    movfr2gr.s $a1, $fa2
+; LA64F-NEXT:    movfr2gr.s $a2, $fa0
+; LA64F-NEXT:  .LBB17_3: # %atomicrmw.start
+; LA64F-NEXT:    # Parent Loop BB17_1 Depth=1
+; LA64F-NEXT:    # => This Inner Loop Header: Depth=2
+; LA64F-NEXT:    ll.w $a3, $a0, 0
+; LA64F-NEXT:    bne $a3, $a2, .LBB17_5
+; LA64F-NEXT:  # %bb.4: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB17_3 Depth=2
+; LA64F-NEXT:    move $a4, $a1
+; LA64F-NEXT:    sc.w $a4, $a0, 0
+; LA64F-NEXT:    beqz $a4, .LBB17_3
+; LA64F-NEXT:    b .LBB17_6
+; LA64F-NEXT:  .LBB17_5: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB17_1 Depth=1
+; LA64F-NEXT:    dbar 20
+; LA64F-NEXT:  .LBB17_6: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB17_1 Depth=1
+; LA64F-NEXT:    movgr2fr.w $fa0, $a3
+; LA64F-NEXT:    addi.w $a1, $a2, 0
+; LA64F-NEXT:    bne $a3, $a1, .LBB17_1
+; LA64F-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64F-NEXT:    ret
+;
+; LA64D-LABEL: float_fsub_acq_rel:
+; LA64D:       # %bb.0:
+; LA64D-NEXT:    fld.s $fa0, $a0, 0
+; LA64D-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI17_0)
+; LA64D-NEXT:    addi.d $a1, $a1, %pc_lo12(.LCPI17_0)
+; LA64D-NEXT:    fld.s $fa1, $a1, 0
+; LA64D-NEXT:    .p2align 4, , 16
+; LA64D-NEXT:  .LBB17_1: # %atomicrmw.start
+; LA64D-NEXT:    # =>This Loop Header: Depth=1
+; LA64D-NEXT:    # Child Loop BB17_3 Depth 2
+; LA64D-NEXT:    fadd.s $fa2, $fa0, $fa1
+; LA64D-NEXT:    movfr2gr.s $a1, $fa2
+; LA64D-NEXT:    movfr2gr.s $a2, $fa0
+; LA64D-NEXT:  .LBB17_3: # %atomicrmw.start
+; LA64D-NEXT:    # Parent Loop BB17_1 Depth=1
+; LA64D-NEXT:    # => This Inner Loop Header: Depth=2
+; LA64D-NEXT:    ll.w $a3, $a0, 0
+; LA64D-NEXT:    bne $a3, $a2, .LBB17_5
+; LA64D-NEXT:  # %bb.4: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB17_3 Depth=2
+; LA64D-NEXT:    move $a4, $a1
+; LA64D-NEXT:    sc.w $a4, $a0, 0
+; LA64D-NEXT:    beqz $a4, .LBB17_3
+; LA64D-NEXT:    b .LBB17_6
+; LA64D-NEXT:  .LBB17_5: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB17_1 Depth=1
+; LA64D-NEXT:    dbar 20
+; LA64D-NEXT:  .LBB17_6: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB17_1 Depth=1
+; LA64D-NEXT:    movgr2fr.w $fa0, $a3
+; LA64D-NEXT:    addi.w $a1, $a2, 0
+; LA64D-NEXT:    bne $a3, $a1, .LBB17_1
+; LA64D-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64D-NEXT:    ret
+  %v = atomicrmw fsub ptr %p, float 1.0 acq_rel, align 4
+  ret float %v
+}
+
+define float @float_fmin_acq_rel(ptr %p) nounwind {
+; LA64F-LABEL: float_fmin_acq_rel:
+; LA64F:       # %bb.0:
+; LA64F-NEXT:    fld.s $fa0, $a0, 0
+; LA64F-NEXT:    addi.w $a1, $zero, 1
+; LA64F-NEXT:    movgr2fr.w $fa1, $a1
+; LA64F-NEXT:    ffint.s.w $fa1, $fa1
+; LA64F-NEXT:    .p2align 4, , 16
+; LA64F-NEXT:  .LBB18_1: # %atomicrmw.start
+; LA64F-NEXT:    # =>This Loop Header: Depth=1
+; LA64F-NEXT:    # Child Loop BB18_3 Depth 2
+; LA64F-NEXT:    fmax.s $fa2, $fa0, $fa0
+; LA64F-NEXT:    fmin.s $fa2, $fa2, $fa1
+; LA64F-NEXT:    movfr2gr.s $a1, $fa2
+; LA64F-NEXT:    movfr2gr.s $a2, $fa0
+; LA64F-NEXT:  .LBB18_3: # %atomicrmw.start
+; LA64F-NEXT:    # Parent Loop BB18_1 Depth=1
+; LA64F-NEXT:    # => This Inner Loop Header: Depth=2
+; LA64F-NEXT:    ll.w $a3, $a0, 0
+; LA64F-NEXT:    bne $a3, $a2, .LBB18_5
+; LA64F-NEXT:  # %bb.4: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB18_3 Depth=2
+; LA64F-NEXT:    move $a4, $a1
+; LA64F-NEXT:    sc.w $a4, $a0, 0
+; LA64F-NEXT:    beqz $a4, .LBB18_3
+; LA64F-NEXT:    b .LBB18_6
+; LA64F-NEXT:  .LBB18_5: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB18_1 Depth=1
+; LA64F-NEXT:    dbar 20
+; LA64F-NEXT:  .LBB18_6: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB18_1 Depth=1
+; LA64F-NEXT:    movgr2fr.w $fa0, $a3
+; LA64F-NEXT:    addi.w $a1, $a2, 0
+; LA64F-NEXT:    bne $a3, $a1, .LBB18_1
+; LA64F-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64F-NEXT:    ret
+;
+; LA64D-LABEL: float_fmin_acq_rel:
+; LA64D:       # %bb.0:
+; LA64D-NEXT:    fld.s $fa0, $a0, 0
+; LA64D-NEXT:    addi.w $a1, $zero, 1
+; LA64D-NEXT:    movgr2fr.w $fa1, $a1
+; LA64D-NEXT:    ffint.s.w $fa1, $fa1
+; LA64D-NEXT:    .p2align 4, , 16
+; LA64D-NEXT:  .LBB18_1: # %atomicrmw.start
+; LA64D-NEXT:    # =>This Loop Header: Depth=1
+; LA64D-NEXT:    # Child Loop BB18_3 Depth 2
+; LA64D-NEXT:    fmax.s $fa2, $fa0, $fa0
+; LA64D-NEXT:    fmin.s $fa2, $fa2, $fa1
+; LA64D-NEXT:    movfr2gr.s $a1, $fa2
+; LA64D-NEXT:    movfr2gr.s $a2, $fa0
+; LA64D-NEXT:  .LBB18_3: # %atomicrmw.start
+; LA64D-NEXT:    # Parent Loop BB18_1 Depth=1
+; LA64D-NEXT:    # => This Inner Loop Header: Depth=2
+; LA64D-NEXT:    ll.w $a3, $a0, 0
+; LA64D-NEXT:    bne $a3, $a2, .LBB18_5
+; LA64D-NEXT:  # %bb.4: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB18_3 Depth=2
+; LA64D-NEXT:    move $a4, $a1
+; LA64D-NEXT:    sc.w $a4, $a0, 0
+; LA64D-NEXT:    beqz $a4, .LBB18_3
+; LA64D-NEXT:    b .LBB18_6
+; LA64D-NEXT:  .LBB18_5: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB18_1 Depth=1
+; LA64D-NEXT:    dbar 20
+; LA64D-NEXT:  .LBB18_6: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB18_1 Depth=1
+; LA64D-NEXT:    movgr2fr.w $fa0, $a3
+; LA64D-NEXT:    addi.w $a1, $a2, 0
+; LA64D-NEXT:    bne $a3, $a1, .LBB18_1
+; LA64D-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64D-NEXT:    ret
+  %v = atomicrmw fmin ptr %p, float 1.0 acq_rel, align 4
+  ret float %v
+}
+
+define float @float_fmax_acq_rel(ptr %p) nounwind {
+; LA64F-LABEL: float_fmax_acq_rel:
+; LA64F:       # %bb.0:
+; LA64F-NEXT:    fld.s $fa0, $a0, 0
+; LA64F-NEXT:    addi.w $a1, $zero, 1
+; LA64F-NEXT:    movgr2fr.w $fa1, $a1
+; LA64F-NEXT:    ffint.s.w $fa1, $fa1
+; LA64F-NEXT:    .p2align 4, , 16
+; LA64F-NEXT:  .LBB19_1: # %atomicrmw.start
+; LA64F-NEXT:    # =>This Loop Header: Depth=1
+; LA64F-NEXT:    # Child Loop BB19_3 Depth 2
+; LA64F-NEXT:    fmax.s $fa2, $fa0, $fa0
+; LA64F-NEXT:    fmax.s $fa2, $fa2, $fa1
+; LA64F-NEXT:    movfr2gr.s $a1, $fa2
+; LA64F-NEXT:    movfr2gr.s $a2, $fa0
+; LA64F-NEXT:  .LBB19_3: # %atomicrmw.start
+; LA64F-NEXT:    # Parent Loop BB19_1 Depth=1
+; LA64F-NEXT:    # => This Inner Loop Header: Depth=2
+; LA64F-NEXT:    ll.w $a3, $a0, 0
+; LA64F-NEXT:    bne $a3, $a2, .LBB19_5
+; LA64F-NEXT:  # %bb.4: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB19_3 Depth=2
+; LA64F-NEXT:    move $a4, $a1
+; LA64F-NEXT:    sc.w $a4, $a0, 0
+; LA64F-NEXT:    beqz $a4, .LBB19_3
+; LA64F-NEXT:    b .LBB19_6
+; LA64F-NEXT:  .LBB19_5: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB19_1 Depth=1
+; LA64F-NEXT:    dbar 20
+; LA64F-NEXT:  .LBB19_6: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB19_1 Depth=1
+; LA64F-NEXT:    movgr2fr.w $fa0, $a3
+; LA64F-NEXT:    addi.w $a1, $a2, 0
+; LA64F-NEXT:    bne $a3, $a1, .LBB19_1
+; LA64F-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64F-NEXT:    ret
+;
+; LA64D-LABEL: float_fmax_acq_rel:
+; LA64D:       # %bb.0:
+; LA64D-NEXT:    fld.s $fa0, $a0, 0
+; LA64D-NEXT:    addi.w $a1, $zero, 1
+; LA64D-NEXT:    movgr2fr.w $fa1, $a1
+; LA64D-NEXT:    ffint.s.w $fa1, $fa1
+; LA64D-NEXT:    .p2align 4, , 16
+; LA64D-NEXT:  .LBB19_1: # %atomicrmw.start
+; LA64D-NEXT:    # =>This Loop Header: Depth=1
+; LA64D-NEXT:    # Child Loop BB19_3 Depth 2
+; LA64D-NEXT:    fmax.s $fa2, $fa0, $fa0
+; LA64D-NEXT:    fmax.s $fa2, $fa2, $fa1
+; LA64D-NEXT:    movfr2gr.s $a1, $fa2
+; LA64D-NEXT:    movfr2gr.s $a2, $fa0
+; LA64D-NEXT:  .LBB19_3: # %atomicrmw.start
+; LA64D-NEXT:    # Parent Loop BB19_1 Depth=1
+; LA64D-NEXT:    # => This Inner Loop Header: Depth=2
+; LA64D-NEXT:    ll.w $a3, $a0, 0
+; LA64D-NEXT:    bne $a3, $a2, .LBB19_5
+; LA64D-NEXT:  # %bb.4: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB19_3 Depth=2
+; LA64D-NEXT:    move $a4, $a1
+; LA64D-NEXT:    sc.w $a4, $a0, 0
+; LA64D-NEXT:    beqz $a4, .LBB19_3
+; LA64D-NEXT:    b .LBB19_6
+; LA64D-NEXT:  .LBB19_5: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB19_1 Depth=1
+; LA64D-NEXT:    dbar 20
+; LA64D-NEXT:  .LBB19_6: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB19_1 Depth=1
+; LA64D-NEXT:    movgr2fr.w $fa0, $a3
+; LA64D-NEXT:    addi.w $a1, $a2, 0
+; LA64D-NEXT:    bne $a3, $a1, .LBB19_1
+; LA64D-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64D-NEXT:    ret
+  %v = atomicrmw fmax ptr %p, float 1.0 acq_rel, align 4
+  ret float %v
+}
+
+define double @double_fadd_acq_rel(ptr %p) nounwind {
+; LA64F-LABEL: double_fadd_acq_rel:
+; LA64F:       # %bb.0:
+; LA64F-NEXT:    addi.d $sp, $sp, -80
+; LA64F-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $fp, $sp, 64 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s0, $sp, 56 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s1, $sp, 48 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s2, $sp, 40 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s3, $sp, 32 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s4, $sp, 24 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s5, $sp, 16 # 8-byte Folded Spill
+; LA64F-NEXT:    move $fp, $a0
+; LA64F-NEXT:    ld.d $a0, $a0, 0
+; LA64F-NEXT:    lu52i.d $s0, $zero, 1023
+; LA64F-NEXT:    ori $s1, $zero, 8
+; LA64F-NEXT:    addi.d $s2, $sp, 8
+; LA64F-NEXT:    addi.d $s3, $sp, 0
+; LA64F-NEXT:    ori $s4, $zero, 4
+; LA64F-NEXT:    ori $s5, $zero, 2
+; LA64F-NEXT:    .p2align 4, , 16
+; LA64F-NEXT:  .LBB20_1: # %atomicrmw.start
+; LA64F-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64F-NEXT:    st.d $a0, $sp, 8
+; LA64F-NEXT:    move $a1, $s0
+; LA64F-NEXT:    bl %plt(__adddf3)
+; LA64F-NEXT:    st.d $a0, $sp, 0
+; LA64F-NEXT:    move $a0, $s1
+; LA64F-NEXT:    move $a1, $fp
+; LA64F-NEXT:    move $a2, $s2
+; LA64F-NEXT:    move $a3, $s3
+; LA64F-NEXT:    move $a4, $s4
+; LA64F-NEXT:    move $a5, $s5
+; LA64F-NEXT:    bl %plt(__atomic_compare_exchange)
+; LA64F-NEXT:    move $a1, $a0
+; LA64F-NEXT:    ld.d $a0, $sp, 8
+; LA64F-NEXT:    beqz $a1, .LBB20_1
+; LA64F-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64F-NEXT:    ld.d $s5, $sp, 16 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s4, $sp, 24 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s3, $sp, 32 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s2, $sp, 40 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s1, $sp, 48 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s0, $sp, 56 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $fp, $sp, 64 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
+; LA64F-NEXT:    addi.d $sp, $sp, 80
+; LA64F-NEXT:    ret
+;
+; LA64D-LABEL: double_fadd_acq_rel:
+; LA64D:       # %bb.0:
+; LA64D-NEXT:    addi.d $sp, $sp, -80
+; LA64D-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $fp, $sp, 64 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s0, $sp, 56 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s1, $sp, 48 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s2, $sp, 40 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s3, $sp, 32 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s4, $sp, 24 # 8-byte Folded Spill
+; LA64D-NEXT:    fst.d $fs0, $sp, 16 # 8-byte Folded Spill
+; LA64D-NEXT:    move $fp, $a0
+; LA64D-NEXT:    fld.d $fa0, $a0, 0
+; LA64D-NEXT:    addi.d $a0, $zero, 1
+; LA64D-NEXT:    movgr2fr.d $fa1, $a0
+; LA64D-NEXT:    ffint.d.l $fs0, $fa1
+; LA64D-NEXT:    ori $s0, $zero, 8
+; LA64D-NEXT:    addi.d $s1, $sp, 8
+; LA64D-NEXT:    addi.d $s2, $sp, 0
+; LA64D-NEXT:    ori $s3, $zero, 4
+; LA64D-NEXT:    ori $s4, $zero, 2
+; LA64D-NEXT:    .p2align 4, , 16
+; LA64D-NEXT:  .LBB20_1: # %atomicrmw.start
+; LA64D-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64D-NEXT:    fst.d $fa0, $sp, 8
+; LA64D-NEXT:    fadd.d $fa0, $fa0, $fs0
+; LA64D-NEXT:    fst.d $fa0, $sp, 0
+; LA64D-NEXT:    move $a0, $s0
+; LA64D-NEXT:    move $a1, $fp
+; LA64D-NEXT:    move $a2, $s1
+; LA64D-NEXT:    move $a3, $s2
+; LA64D-NEXT:    move $a4, $s3
+; LA64D-NEXT:    move $a5, $s4
+; LA64D-NEXT:    bl %plt(__atomic_compare_exchange)
+; LA64D-NEXT:    fld.d $fa0, $sp, 8
+; LA64D-NEXT:    beqz $a0, .LBB20_1
+; LA64D-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64D-NEXT:    fld.d $fs0, $sp, 16 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s4, $sp, 24 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s3, $sp, 32 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s2, $sp, 40 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s1, $sp, 48 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s0, $sp, 56 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $fp, $sp, 64 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
+; LA64D-NEXT:    addi.d $sp, $sp, 80
+; LA64D-NEXT:    ret
+  %v = atomicrmw fadd ptr %p, double 1.0 acq_rel, align 4
+  ret double %v
+}
+
+define double @double_fsub_acq_rel(ptr %p) nounwind {
+; LA64F-LABEL: double_fsub_acq_rel:
+; LA64F:       # %bb.0:
+; LA64F-NEXT:    addi.d $sp, $sp, -80
+; LA64F-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $fp, $sp, 64 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s0, $sp, 56 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s1, $sp, 48 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s2, $sp, 40 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s3, $sp, 32 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s4, $sp, 24 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s5, $sp, 16 # 8-byte Folded Spill
+; LA64F-NEXT:    move $fp, $a0
+; LA64F-NEXT:    ld.d $a0, $a0, 0
+; LA64F-NEXT:    lu52i.d $s0, $zero, -1025
+; LA64F-NEXT:    ori $s1, $zero, 8
+; LA64F-NEXT:    addi.d $s2, $sp, 8
+; LA64F-NEXT:    addi.d $s3, $sp, 0
+; LA64F-NEXT:    ori $s4, $zero, 4
+; LA64F-NEXT:    ori $s5, $zero, 2
+; LA64F-NEXT:    .p2align 4, , 16
+; LA64F-NEXT:  .LBB21_1: # %atomicrmw.start
+; LA64F-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64F-NEXT:    st.d $a0, $sp, 8
+; LA64F-NEXT:    move $a1, $s0
+; LA64F-NEXT:    bl %plt(__adddf3)
+; LA64F-NEXT:    st.d $a0, $sp, 0
+; LA64F-NEXT:    move $a0, $s1
+; LA64F-NEXT:    move $a1, $fp
+; LA64F-NEXT:    move $a2, $s2
+; LA64F-NEXT:    move $a3, $s3
+; LA64F-NEXT:    move $a4, $s4
+; LA64F-NEXT:    move $a5, $s5
+; LA64F-NEXT:    bl %plt(__atomic_compare_exchange)
+; LA64F-NEXT:    move $a1, $a0
+; LA64F-NEXT:    ld.d $a0, $sp, 8
+; LA64F-NEXT:    beqz $a1, .LBB21_1
+; LA64F-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64F-NEXT:    ld.d $s5, $sp, 16 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s4, $sp, 24 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s3, $sp, 32 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s2, $sp, 40 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s1, $sp, 48 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s0, $sp, 56 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $fp, $sp, 64 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
+; LA64F-NEXT:    addi.d $sp, $sp, 80
+; LA64F-NEXT:    ret
+;
+; LA64D-LABEL: double_fsub_acq_rel:
+; LA64D:       # %bb.0:
+; LA64D-NEXT:    addi.d $sp, $sp, -80
+; LA64D-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $fp, $sp, 64 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s0, $sp, 56 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s1, $sp, 48 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s2, $sp, 40 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s3, $sp, 32 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s4, $sp, 24 # 8-byte Folded Spill
+; LA64D-NEXT:    fst.d $fs0, $sp, 16 # 8-byte Folded Spill
+; LA64D-NEXT:    move $fp, $a0
+; LA64D-NEXT:    fld.d $fa0, $a0, 0
+; LA64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI21_0)
+; LA64D-NEXT:    addi.d $a0, $a0, %pc_lo12(.LCPI21_0)
+; LA64D-NEXT:    fld.d $fs0, $a0, 0
+; LA64D-NEXT:    ori $s0, $zero, 8
+; LA64D-NEXT:    addi.d $s1, $sp, 8
+; LA64D-NEXT:    addi.d $s2, $sp, 0
+; LA64D-NEXT:    ori $s3, $zero, 4
+; LA64D-NEXT:    ori $s4, $zero, 2
+; LA64D-NEXT:    .p2align 4, , 16
+; LA64D-NEXT:  .LBB21_1: # %atomicrmw.start
+; LA64D-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64D-NEXT:    fst.d $fa0, $sp, 8
+; LA64D-NEXT:    fadd.d $fa0, $fa0, $fs0
+; LA64D-NEXT:    fst.d $fa0, $sp, 0
+; LA64D-NEXT:    move $a0, $s0
+; LA64D-NEXT:    move $a1, $fp
+; LA64D-NEXT:    move $a2, $s1
+; LA64D-NEXT:    move $a3, $s2
+; LA64D-NEXT:    move $a4, $s3
+; LA64D-NEXT:    move $a5, $s4
+; LA64D-NEXT:    bl %plt(__atomic_compare_exchange)
+; LA64D-NEXT:    fld.d $fa0, $sp, 8
+; LA64D-NEXT:    beqz $a0, .LBB21_1
+; LA64D-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64D-NEXT:    fld.d $fs0, $sp, 16 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s4, $sp, 24 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s3, $sp, 32 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s2, $sp, 40 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s1, $sp, 48 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s0, $sp, 56 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $fp, $sp, 64 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
+; LA64D-NEXT:    addi.d $sp, $sp, 80
+; LA64D-NEXT:    ret
+  %v = atomicrmw fsub ptr %p, double 1.0 acq_rel, align 4
+  ret double %v
+}
+
+define double @double_fmin_acq_rel(ptr %p) nounwind {
+; LA64F-LABEL: double_fmin_acq_rel:
+; LA64F:       # %bb.0:
+; LA64F-NEXT:    addi.d $sp, $sp, -80
+; LA64F-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $fp, $sp, 64 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s0, $sp, 56 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s1, $sp, 48 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s2, $sp, 40 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s3, $sp, 32 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s4, $sp, 24 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s5, $sp, 16 # 8-byte Folded Spill
+; LA64F-NEXT:    move $fp, $a0
+; LA64F-NEXT:    ld.d $a0, $a0, 0
+; LA64F-NEXT:    lu52i.d $s0, $zero, 1023
+; LA64F-NEXT:    ori $s1, $zero, 8
+; LA64F-NEXT:    addi.d $s2, $sp, 8
+; LA64F-NEXT:    addi.d $s3, $sp, 0
+; LA64F-NEXT:    ori $s4, $zero, 4
+; LA64F-NEXT:    ori $s5, $zero, 2
+; LA64F-NEXT:    .p2align 4, , 16
+; LA64F-NEXT:  .LBB22_1: # %atomicrmw.start
+; LA64F-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64F-NEXT:    st.d $a0, $sp, 8
+; LA64F-NEXT:    move $a1, $s0
+; LA64F-NEXT:    bl %plt(fmin)
+; LA64F-NEXT:    st.d $a0, $sp, 0
+; LA64F-NEXT:    move $a0, $s1
+; LA64F-NEXT:    move $a1, $fp
+; LA64F-NEXT:    move $a2, $s2
+; LA64F-NEXT:    move $a3, $s3
+; LA64F-NEXT:    move $a4, $s4
+; LA64F-NEXT:    move $a5, $s5
+; LA64F-NEXT:    bl %plt(__atomic_compare_exchange)
+; LA64F-NEXT:    move $a1, $a0
+; LA64F-NEXT:    ld.d $a0, $sp, 8
+; LA64F-NEXT:    beqz $a1, .LBB22_1
+; LA64F-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64F-NEXT:    ld.d $s5, $sp, 16 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s4, $sp, 24 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s3, $sp, 32 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s2, $sp, 40 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s1, $sp, 48 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s0, $sp, 56 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $fp, $sp, 64 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
+; LA64F-NEXT:    addi.d $sp, $sp, 80
+; LA64F-NEXT:    ret
+;
+; LA64D-LABEL: double_fmin_acq_rel:
+; LA64D:       # %bb.0:
+; LA64D-NEXT:    addi.d $sp, $sp, -80
+; LA64D-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $fp, $sp, 64 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s0, $sp, 56 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s1, $sp, 48 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s2, $sp, 40 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s3, $sp, 32 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s4, $sp, 24 # 8-byte Folded Spill
+; LA64D-NEXT:    fst.d $fs0, $sp, 16 # 8-byte Folded Spill
+; LA64D-NEXT:    move $fp, $a0
+; LA64D-NEXT:    fld.d $fa0, $a0, 0
+; LA64D-NEXT:    addi.d $a0, $zero, 1
+; LA64D-NEXT:    movgr2fr.d $fa1, $a0
+; LA64D-NEXT:    ffint.d.l $fs0, $fa1
+; LA64D-NEXT:    ori $s0, $zero, 8
+; LA64D-NEXT:    addi.d $s1, $sp, 8
+; LA64D-NEXT:    addi.d $s2, $sp, 0
+; LA64D-NEXT:    ori $s3, $zero, 4
+; LA64D-NEXT:    ori $s4, $zero, 2
+; LA64D-NEXT:    .p2align 4, , 16
+; LA64D-NEXT:  .LBB22_1: # %atomicrmw.start
+; LA64D-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64D-NEXT:    fst.d $fa0, $sp, 8
+; LA64D-NEXT:    fmax.d $fa0, $fa0, $fa0
+; LA64D-NEXT:    fmin.d $fa0, $fa0, $fs0
+; LA64D-NEXT:    fst.d $fa0, $sp, 0
+; LA64D-NEXT:    move $a0, $s0
+; LA64D-NEXT:    move $a1, $fp
+; LA64D-NEXT:    move $a2, $s1
+; LA64D-NEXT:    move $a3, $s2
+; LA64D-NEXT:    move $a4, $s3
+; LA64D-NEXT:    move $a5, $s4
+; LA64D-NEXT:    bl %plt(__atomic_compare_exchange)
+; LA64D-NEXT:    fld.d $fa0, $sp, 8
+; LA64D-NEXT:    beqz $a0, .LBB22_1
+; LA64D-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64D-NEXT:    fld.d $fs0, $sp, 16 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s4, $sp, 24 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s3, $sp, 32 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s2, $sp, 40 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s1, $sp, 48 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s0, $sp, 56 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $fp, $sp, 64 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
+; LA64D-NEXT:    addi.d $sp, $sp, 80
+; LA64D-NEXT:    ret
+  %v = atomicrmw fmin ptr %p, double 1.0 acq_rel, align 4
+  ret double %v
+}
+
+define double @double_fmax_acq_rel(ptr %p) nounwind {
+; LA64F-LABEL: double_fmax_acq_rel:
+; LA64F:       # %bb.0:
+; LA64F-NEXT:    addi.d $sp, $sp, -80
+; LA64F-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $fp, $sp, 64 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s0, $sp, 56 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s1, $sp, 48 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s2, $sp, 40 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s3, $sp, 32 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s4, $sp, 24 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s5, $sp, 16 # 8-byte Folded Spill
+; LA64F-NEXT:    move $fp, $a0
+; LA64F-NEXT:    ld.d $a0, $a0, 0
+; LA64F-NEXT:    lu52i.d $s0, $zero, 1023
+; LA64F-NEXT:    ori $s1, $zero, 8
+; LA64F-NEXT:    addi.d $s2, $sp, 8
+; LA64F-NEXT:    addi.d $s3, $sp, 0
+; LA64F-NEXT:    ori $s4, $zero, 4
+; LA64F-NEXT:    ori $s5, $zero, 2
+; LA64F-NEXT:    .p2align 4, , 16
+; LA64F-NEXT:  .LBB23_1: # %atomicrmw.start
+; LA64F-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64F-NEXT:    st.d $a0, $sp, 8
+; LA64F-NEXT:    move $a1, $s0
+; LA64F-NEXT:    bl %plt(fmax)
+; LA64F-NEXT:    st.d $a0, $sp, 0
+; LA64F-NEXT:    move $a0, $s1
+; LA64F-NEXT:    move $a1, $fp
+; LA64F-NEXT:    move $a2, $s2
+; LA64F-NEXT:    move $a3, $s3
+; LA64F-NEXT:    move $a4, $s4
+; LA64F-NEXT:    move $a5, $s5
+; LA64F-NEXT:    bl %plt(__atomic_compare_exchange)
+; LA64F-NEXT:    move $a1, $a0
+; LA64F-NEXT:    ld.d $a0, $sp, 8
+; LA64F-NEXT:    beqz $a1, .LBB23_1
+; LA64F-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64F-NEXT:    ld.d $s5, $sp, 16 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s4, $sp, 24 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s3, $sp, 32 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s2, $sp, 40 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s1, $sp, 48 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s0, $sp, 56 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $fp, $sp, 64 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
+; LA64F-NEXT:    addi.d $sp, $sp, 80
+; LA64F-NEXT:    ret
+;
+; LA64D-LABEL: double_fmax_acq_rel:
+; LA64D:       # %bb.0:
+; LA64D-NEXT:    addi.d $sp, $sp, -80
+; LA64D-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $fp, $sp, 64 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s0, $sp, 56 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s1, $sp, 48 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s2, $sp, 40 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s3, $sp, 32 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s4, $sp, 24 # 8-byte Folded Spill
+; LA64D-NEXT:    fst.d $fs0, $sp, 16 # 8-byte Folded Spill
+; LA64D-NEXT:    move $fp, $a0
+; LA64D-NEXT:    fld.d $fa0, $a0, 0
+; LA64D-NEXT:    addi.d $a0, $zero, 1
+; LA64D-NEXT:    movgr2fr.d $fa1, $a0
+; LA64D-NEXT:    ffint.d.l $fs0, $fa1
+; LA64D-NEXT:    ori $s0, $zero, 8
+; LA64D-NEXT:    addi.d $s1, $sp, 8
+; LA64D-NEXT:    addi.d $s2, $sp, 0
+; LA64D-NEXT:    ori $s3, $zero, 4
+; LA64D-NEXT:    ori $s4, $zero, 2
+; LA64D-NEXT:    .p2align 4, , 16
+; LA64D-NEXT:  .LBB23_1: # %atomicrmw.start
+; LA64D-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64D-NEXT:    fst.d $fa0, $sp, 8
+; LA64D-NEXT:    fmax.d $fa0, $fa0, $fa0
+; LA64D-NEXT:    fmax.d $fa0, $fa0, $fs0
+; LA64D-NEXT:    fst.d $fa0, $sp, 0
+; LA64D-NEXT:    move $a0, $s0
+; LA64D-NEXT:    move $a1, $fp
+; LA64D-NEXT:    move $a2, $s1
+; LA64D-NEXT:    move $a3, $s2
+; LA64D-NEXT:    move $a4, $s3
+; LA64D-NEXT:    move $a5, $s4
+; LA64D-NEXT:    bl %plt(__atomic_compare_exchange)
+; LA64D-NEXT:    fld.d $fa0, $sp, 8
+; LA64D-NEXT:    beqz $a0, .LBB23_1
+; LA64D-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64D-NEXT:    fld.d $fs0, $sp, 16 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s4, $sp, 24 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s3, $sp, 32 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s2, $sp, 40 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s1, $sp, 48 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s0, $sp, 56 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $fp, $sp, 64 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
+; LA64D-NEXT:    addi.d $sp, $sp, 80
+; LA64D-NEXT:    ret
+  %v = atomicrmw fmax ptr %p, double 1.0 acq_rel, align 4
+  ret double %v
+}
+
+define float @float_fadd_seq_cst(ptr %p) nounwind {
+; LA64F-LABEL: float_fadd_seq_cst:
+; LA64F:       # %bb.0:
+; LA64F-NEXT:    fld.s $fa0, $a0, 0
+; LA64F-NEXT:    addi.w $a1, $zero, 1
+; LA64F-NEXT:    movgr2fr.w $fa1, $a1
+; LA64F-NEXT:    ffint.s.w $fa1, $fa1
+; LA64F-NEXT:    .p2align 4, , 16
+; LA64F-NEXT:  .LBB24_1: # %atomicrmw.start
+; LA64F-NEXT:    # =>This Loop Header: Depth=1
+; LA64F-NEXT:    # Child Loop BB24_3 Depth 2
+; LA64F-NEXT:    fadd.s $fa2, $fa0, $fa1
+; LA64F-NEXT:    movfr2gr.s $a1, $fa2
+; LA64F-NEXT:    movfr2gr.s $a2, $fa0
+; LA64F-NEXT:  .LBB24_3: # %atomicrmw.start
+; LA64F-NEXT:    # Parent Loop BB24_1 Depth=1
+; LA64F-NEXT:    # => This Inner Loop Header: Depth=2
+; LA64F-NEXT:    ll.w $a3, $a0, 0
+; LA64F-NEXT:    bne $a3, $a2, .LBB24_5
+; LA64F-NEXT:  # %bb.4: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB24_3 Depth=2
+; LA64F-NEXT:    move $a4, $a1
+; LA64F-NEXT:    sc.w $a4, $a0, 0
+; LA64F-NEXT:    beqz $a4, .LBB24_3
+; LA64F-NEXT:    b .LBB24_6
+; LA64F-NEXT:  .LBB24_5: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB24_1 Depth=1
+; LA64F-NEXT:    dbar 20
+; LA64F-NEXT:  .LBB24_6: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB24_1 Depth=1
+; LA64F-NEXT:    movgr2fr.w $fa0, $a3
+; LA64F-NEXT:    addi.w $a1, $a2, 0
+; LA64F-NEXT:    bne $a3, $a1, .LBB24_1
+; LA64F-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64F-NEXT:    ret
+;
+; LA64D-LABEL: float_fadd_seq_cst:
+; LA64D:       # %bb.0:
+; LA64D-NEXT:    fld.s $fa0, $a0, 0
+; LA64D-NEXT:    addi.w $a1, $zero, 1
+; LA64D-NEXT:    movgr2fr.w $fa1, $a1
+; LA64D-NEXT:    ffint.s.w $fa1, $fa1
+; LA64D-NEXT:    .p2align 4, , 16
+; LA64D-NEXT:  .LBB24_1: # %atomicrmw.start
+; LA64D-NEXT:    # =>This Loop Header: Depth=1
+; LA64D-NEXT:    # Child Loop BB24_3 Depth 2
+; LA64D-NEXT:    fadd.s $fa2, $fa0, $fa1
+; LA64D-NEXT:    movfr2gr.s $a1, $fa2
+; LA64D-NEXT:    movfr2gr.s $a2, $fa0
+; LA64D-NEXT:  .LBB24_3: # %atomicrmw.start
+; LA64D-NEXT:    # Parent Loop BB24_1 Depth=1
+; LA64D-NEXT:    # => This Inner Loop Header: Depth=2
+; LA64D-NEXT:    ll.w $a3, $a0, 0
+; LA64D-NEXT:    bne $a3, $a2, .LBB24_5
+; LA64D-NEXT:  # %bb.4: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB24_3 Depth=2
+; LA64D-NEXT:    move $a4, $a1
+; LA64D-NEXT:    sc.w $a4, $a0, 0
+; LA64D-NEXT:    beqz $a4, .LBB24_3
+; LA64D-NEXT:    b .LBB24_6
+; LA64D-NEXT:  .LBB24_5: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB24_1 Depth=1
+; LA64D-NEXT:    dbar 20
+; LA64D-NEXT:  .LBB24_6: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB24_1 Depth=1
+; LA64D-NEXT:    movgr2fr.w $fa0, $a3
+; LA64D-NEXT:    addi.w $a1, $a2, 0
+; LA64D-NEXT:    bne $a3, $a1, .LBB24_1
+; LA64D-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64D-NEXT:    ret
+  %v = atomicrmw fadd ptr %p, float 1.0 seq_cst, align 4
+  ret float %v
+}
+
+define float @float_fsub_seq_cst(ptr %p) nounwind {
+; LA64F-LABEL: float_fsub_seq_cst:
+; LA64F:       # %bb.0:
+; LA64F-NEXT:    fld.s $fa0, $a0, 0
+; LA64F-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI25_0)
+; LA64F-NEXT:    addi.d $a1, $a1, %pc_lo12(.LCPI25_0)
+; LA64F-NEXT:    fld.s $fa1, $a1, 0
+; LA64F-NEXT:    .p2align 4, , 16
+; LA64F-NEXT:  .LBB25_1: # %atomicrmw.start
+; LA64F-NEXT:    # =>This Loop Header: Depth=1
+; LA64F-NEXT:    # Child Loop BB25_3 Depth 2
+; LA64F-NEXT:    fadd.s $fa2, $fa0, $fa1
+; LA64F-NEXT:    movfr2gr.s $a1, $fa2
+; LA64F-NEXT:    movfr2gr.s $a2, $fa0
+; LA64F-NEXT:  .LBB25_3: # %atomicrmw.start
+; LA64F-NEXT:    # Parent Loop BB25_1 Depth=1
+; LA64F-NEXT:    # => This Inner Loop Header: Depth=2
+; LA64F-NEXT:    ll.w $a3, $a0, 0
+; LA64F-NEXT:    bne $a3, $a2, .LBB25_5
+; LA64F-NEXT:  # %bb.4: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB25_3 Depth=2
+; LA64F-NEXT:    move $a4, $a1
+; LA64F-NEXT:    sc.w $a4, $a0, 0
+; LA64F-NEXT:    beqz $a4, .LBB25_3
+; LA64F-NEXT:    b .LBB25_6
+; LA64F-NEXT:  .LBB25_5: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB25_1 Depth=1
+; LA64F-NEXT:    dbar 20
+; LA64F-NEXT:  .LBB25_6: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB25_1 Depth=1
+; LA64F-NEXT:    movgr2fr.w $fa0, $a3
+; LA64F-NEXT:    addi.w $a1, $a2, 0
+; LA64F-NEXT:    bne $a3, $a1, .LBB25_1
+; LA64F-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64F-NEXT:    ret
+;
+; LA64D-LABEL: float_fsub_seq_cst:
+; LA64D:       # %bb.0:
+; LA64D-NEXT:    fld.s $fa0, $a0, 0
+; LA64D-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI25_0)
+; LA64D-NEXT:    addi.d $a1, $a1, %pc_lo12(.LCPI25_0)
+; LA64D-NEXT:    fld.s $fa1, $a1, 0
+; LA64D-NEXT:    .p2align 4, , 16
+; LA64D-NEXT:  .LBB25_1: # %atomicrmw.start
+; LA64D-NEXT:    # =>This Loop Header: Depth=1
+; LA64D-NEXT:    # Child Loop BB25_3 Depth 2
+; LA64D-NEXT:    fadd.s $fa2, $fa0, $fa1
+; LA64D-NEXT:    movfr2gr.s $a1, $fa2
+; LA64D-NEXT:    movfr2gr.s $a2, $fa0
+; LA64D-NEXT:  .LBB25_3: # %atomicrmw.start
+; LA64D-NEXT:    # Parent Loop BB25_1 Depth=1
+; LA64D-NEXT:    # => This Inner Loop Header: Depth=2
+; LA64D-NEXT:    ll.w $a3, $a0, 0
+; LA64D-NEXT:    bne $a3, $a2, .LBB25_5
+; LA64D-NEXT:  # %bb.4: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB25_3 Depth=2
+; LA64D-NEXT:    move $a4, $a1
+; LA64D-NEXT:    sc.w $a4, $a0, 0
+; LA64D-NEXT:    beqz $a4, .LBB25_3
+; LA64D-NEXT:    b .LBB25_6
+; LA64D-NEXT:  .LBB25_5: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB25_1 Depth=1
+; LA64D-NEXT:    dbar 20
+; LA64D-NEXT:  .LBB25_6: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB25_1 Depth=1
+; LA64D-NEXT:    movgr2fr.w $fa0, $a3
+; LA64D-NEXT:    addi.w $a1, $a2, 0
+; LA64D-NEXT:    bne $a3, $a1, .LBB25_1
+; LA64D-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64D-NEXT:    ret
+  %v = atomicrmw fsub ptr %p, float 1.0 seq_cst, align 4
+  ret float %v
+}
+
+define float @float_fmin_seq_cst(ptr %p) nounwind {
+; LA64F-LABEL: float_fmin_seq_cst:
+; LA64F:       # %bb.0:
+; LA64F-NEXT:    fld.s $fa0, $a0, 0
+; LA64F-NEXT:    addi.w $a1, $zero, 1
+; LA64F-NEXT:    movgr2fr.w $fa1, $a1
+; LA64F-NEXT:    ffint.s.w $fa1, $fa1
+; LA64F-NEXT:    .p2align 4, , 16
+; LA64F-NEXT:  .LBB26_1: # %atomicrmw.start
+; LA64F-NEXT:    # =>This Loop Header: Depth=1
+; LA64F-NEXT:    # Child Loop BB26_3 Depth 2
+; LA64F-NEXT:    fmax.s $fa2, $fa0, $fa0
+; LA64F-NEXT:    fmin.s $fa2, $fa2, $fa1
+; LA64F-NEXT:    movfr2gr.s $a1, $fa2
+; LA64F-NEXT:    movfr2gr.s $a2, $fa0
+; LA64F-NEXT:  .LBB26_3: # %atomicrmw.start
+; LA64F-NEXT:    # Parent Loop BB26_1 Depth=1
+; LA64F-NEXT:    # => This Inner Loop Header: Depth=2
+; LA64F-NEXT:    ll.w $a3, $a0, 0
+; LA64F-NEXT:    bne $a3, $a2, .LBB26_5
+; LA64F-NEXT:  # %bb.4: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB26_3 Depth=2
+; LA64F-NEXT:    move $a4, $a1
+; LA64F-NEXT:    sc.w $a4, $a0, 0
+; LA64F-NEXT:    beqz $a4, .LBB26_3
+; LA64F-NEXT:    b .LBB26_6
+; LA64F-NEXT:  .LBB26_5: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB26_1 Depth=1
+; LA64F-NEXT:    dbar 20
+; LA64F-NEXT:  .LBB26_6: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB26_1 Depth=1
+; LA64F-NEXT:    movgr2fr.w $fa0, $a3
+; LA64F-NEXT:    addi.w $a1, $a2, 0
+; LA64F-NEXT:    bne $a3, $a1, .LBB26_1
+; LA64F-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64F-NEXT:    ret
+;
+; LA64D-LABEL: float_fmin_seq_cst:
+; LA64D:       # %bb.0:
+; LA64D-NEXT:    fld.s $fa0, $a0, 0
+; LA64D-NEXT:    addi.w $a1, $zero, 1
+; LA64D-NEXT:    movgr2fr.w $fa1, $a1
+; LA64D-NEXT:    ffint.s.w $fa1, $fa1
+; LA64D-NEXT:    .p2align 4, , 16
+; LA64D-NEXT:  .LBB26_1: # %atomicrmw.start
+; LA64D-NEXT:    # =>This Loop Header: Depth=1
+; LA64D-NEXT:    # Child Loop BB26_3 Depth 2
+; LA64D-NEXT:    fmax.s $fa2, $fa0, $fa0
+; LA64D-NEXT:    fmin.s $fa2, $fa2, $fa1
+; LA64D-NEXT:    movfr2gr.s $a1, $fa2
+; LA64D-NEXT:    movfr2gr.s $a2, $fa0
+; LA64D-NEXT:  .LBB26_3: # %atomicrmw.start
+; LA64D-NEXT:    # Parent Loop BB26_1 Depth=1
+; LA64D-NEXT:    # => This Inner Loop Header: Depth=2
+; LA64D-NEXT:    ll.w $a3, $a0, 0
+; LA64D-NEXT:    bne $a3, $a2, .LBB26_5
+; LA64D-NEXT:  # %bb.4: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB26_3 Depth=2
+; LA64D-NEXT:    move $a4, $a1
+; LA64D-NEXT:    sc.w $a4, $a0, 0
+; LA64D-NEXT:    beqz $a4, .LBB26_3
+; LA64D-NEXT:    b .LBB26_6
+; LA64D-NEXT:  .LBB26_5: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB26_1 Depth=1
+; LA64D-NEXT:    dbar 20
+; LA64D-NEXT:  .LBB26_6: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB26_1 Depth=1
+; LA64D-NEXT:    movgr2fr.w $fa0, $a3
+; LA64D-NEXT:    addi.w $a1, $a2, 0
+; LA64D-NEXT:    bne $a3, $a1, .LBB26_1
+; LA64D-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64D-NEXT:    ret
+  %v = atomicrmw fmin ptr %p, float 1.0 seq_cst, align 4
+  ret float %v
+}
+
+define float @float_fmax_seq_cst(ptr %p) nounwind {
+; LA64F-LABEL: float_fmax_seq_cst:
+; LA64F:       # %bb.0:
+; LA64F-NEXT:    fld.s $fa0, $a0, 0
+; LA64F-NEXT:    addi.w $a1, $zero, 1
+; LA64F-NEXT:    movgr2fr.w $fa1, $a1
+; LA64F-NEXT:    ffint.s.w $fa1, $fa1
+; LA64F-NEXT:    .p2align 4, , 16
+; LA64F-NEXT:  .LBB27_1: # %atomicrmw.start
+; LA64F-NEXT:    # =>This Loop Header: Depth=1
+; LA64F-NEXT:    # Child Loop BB27_3 Depth 2
+; LA64F-NEXT:    fmax.s $fa2, $fa0, $fa0
+; LA64F-NEXT:    fmax.s $fa2, $fa2, $fa1
+; LA64F-NEXT:    movfr2gr.s $a1, $fa2
+; LA64F-NEXT:    movfr2gr.s $a2, $fa0
+; LA64F-NEXT:  .LBB27_3: # %atomicrmw.start
+; LA64F-NEXT:    # Parent Loop BB27_1 Depth=1
+; LA64F-NEXT:    # => This Inner Loop Header: Depth=2
+; LA64F-NEXT:    ll.w $a3, $a0, 0
+; LA64F-NEXT:    bne $a3, $a2, .LBB27_5
+; LA64F-NEXT:  # %bb.4: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB27_3 Depth=2
+; LA64F-NEXT:    move $a4, $a1
+; LA64F-NEXT:    sc.w $a4, $a0, 0
+; LA64F-NEXT:    beqz $a4, .LBB27_3
+; LA64F-NEXT:    b .LBB27_6
+; LA64F-NEXT:  .LBB27_5: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB27_1 Depth=1
+; LA64F-NEXT:    dbar 20
+; LA64F-NEXT:  .LBB27_6: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB27_1 Depth=1
+; LA64F-NEXT:    movgr2fr.w $fa0, $a3
+; LA64F-NEXT:    addi.w $a1, $a2, 0
+; LA64F-NEXT:    bne $a3, $a1, .LBB27_1
+; LA64F-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64F-NEXT:    ret
+;
+; LA64D-LABEL: float_fmax_seq_cst:
+; LA64D:       # %bb.0:
+; LA64D-NEXT:    fld.s $fa0, $a0, 0
+; LA64D-NEXT:    addi.w $a1, $zero, 1
+; LA64D-NEXT:    movgr2fr.w $fa1, $a1
+; LA64D-NEXT:    ffint.s.w $fa1, $fa1
+; LA64D-NEXT:    .p2align 4, , 16
+; LA64D-NEXT:  .LBB27_1: # %atomicrmw.start
+; LA64D-NEXT:    # =>This Loop Header: Depth=1
+; LA64D-NEXT:    # Child Loop BB27_3 Depth 2
+; LA64D-NEXT:    fmax.s $fa2, $fa0, $fa0
+; LA64D-NEXT:    fmax.s $fa2, $fa2, $fa1
+; LA64D-NEXT:    movfr2gr.s $a1, $fa2
+; LA64D-NEXT:    movfr2gr.s $a2, $fa0
+; LA64D-NEXT:  .LBB27_3: # %atomicrmw.start
+; LA64D-NEXT:    # Parent Loop BB27_1 Depth=1
+; LA64D-NEXT:    # => This Inner Loop Header: Depth=2
+; LA64D-NEXT:    ll.w $a3, $a0, 0
+; LA64D-NEXT:    bne $a3, $a2, .LBB27_5
+; LA64D-NEXT:  # %bb.4: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB27_3 Depth=2
+; LA64D-NEXT:    move $a4, $a1
+; LA64D-NEXT:    sc.w $a4, $a0, 0
+; LA64D-NEXT:    beqz $a4, .LBB27_3
+; LA64D-NEXT:    b .LBB27_6
+; LA64D-NEXT:  .LBB27_5: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB27_1 Depth=1
+; LA64D-NEXT:    dbar 20
+; LA64D-NEXT:  .LBB27_6: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB27_1 Depth=1
+; LA64D-NEXT:    movgr2fr.w $fa0, $a3
+; LA64D-NEXT:    addi.w $a1, $a2, 0
+; LA64D-NEXT:    bne $a3, $a1, .LBB27_1
+; LA64D-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64D-NEXT:    ret
+  %v = atomicrmw fmax ptr %p, float 1.0 seq_cst, align 4
+  ret float %v
+}
+
+define double @double_fadd_seq_cst(ptr %p) nounwind {
+; LA64F-LABEL: double_fadd_seq_cst:
+; LA64F:       # %bb.0:
+; LA64F-NEXT:    addi.d $sp, $sp, -80
+; LA64F-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $fp, $sp, 64 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s0, $sp, 56 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s1, $sp, 48 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s2, $sp, 40 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s3, $sp, 32 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s4, $sp, 24 # 8-byte Folded Spill
+; LA64F-NEXT:    move $fp, $a0
+; LA64F-NEXT:    ld.d $a0, $a0, 0
+; LA64F-NEXT:    lu52i.d $s0, $zero, 1023
+; LA64F-NEXT:    ori $s1, $zero, 8
+; LA64F-NEXT:    addi.d $s2, $sp, 16
+; LA64F-NEXT:    addi.d $s3, $sp, 8
+; LA64F-NEXT:    ori $s4, $zero, 5
+; LA64F-NEXT:    .p2align 4, , 16
+; LA64F-NEXT:  .LBB28_1: # %atomicrmw.start
+; LA64F-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64F-NEXT:    st.d $a0, $sp, 16
+; LA64F-NEXT:    move $a1, $s0
+; LA64F-NEXT:    bl %plt(__adddf3)
+; LA64F-NEXT:    st.d $a0, $sp, 8
+; LA64F-NEXT:    move $a0, $s1
+; LA64F-NEXT:    move $a1, $fp
+; LA64F-NEXT:    move $a2, $s2
+; LA64F-NEXT:    move $a3, $s3
+; LA64F-NEXT:    move $a4, $s4
+; LA64F-NEXT:    move $a5, $s4
+; LA64F-NEXT:    bl %plt(__atomic_compare_exchange)
+; LA64F-NEXT:    move $a1, $a0
+; LA64F-NEXT:    ld.d $a0, $sp, 16
+; LA64F-NEXT:    beqz $a1, .LBB28_1
+; LA64F-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64F-NEXT:    ld.d $s4, $sp, 24 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s3, $sp, 32 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s2, $sp, 40 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s1, $sp, 48 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s0, $sp, 56 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $fp, $sp, 64 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
+; LA64F-NEXT:    addi.d $sp, $sp, 80
+; LA64F-NEXT:    ret
+;
+; LA64D-LABEL: double_fadd_seq_cst:
+; LA64D:       # %bb.0:
+; LA64D-NEXT:    addi.d $sp, $sp, -80
+; LA64D-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $fp, $sp, 64 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s0, $sp, 56 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s1, $sp, 48 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s2, $sp, 40 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s3, $sp, 32 # 8-byte Folded Spill
+; LA64D-NEXT:    fst.d $fs0, $sp, 24 # 8-byte Folded Spill
+; LA64D-NEXT:    move $fp, $a0
+; LA64D-NEXT:    fld.d $fa0, $a0, 0
+; LA64D-NEXT:    addi.d $a0, $zero, 1
+; LA64D-NEXT:    movgr2fr.d $fa1, $a0
+; LA64D-NEXT:    ffint.d.l $fs0, $fa1
+; LA64D-NEXT:    ori $s0, $zero, 8
+; LA64D-NEXT:    addi.d $s1, $sp, 16
+; LA64D-NEXT:    addi.d $s2, $sp, 8
+; LA64D-NEXT:    ori $s3, $zero, 5
+; LA64D-NEXT:    .p2align 4, , 16
+; LA64D-NEXT:  .LBB28_1: # %atomicrmw.start
+; LA64D-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64D-NEXT:    fst.d $fa0, $sp, 16
+; LA64D-NEXT:    fadd.d $fa0, $fa0, $fs0
+; LA64D-NEXT:    fst.d $fa0, $sp, 8
+; LA64D-NEXT:    move $a0, $s0
+; LA64D-NEXT:    move $a1, $fp
+; LA64D-NEXT:    move $a2, $s1
+; LA64D-NEXT:    move $a3, $s2
+; LA64D-NEXT:    move $a4, $s3
+; LA64D-NEXT:    move $a5, $s3
+; LA64D-NEXT:    bl %plt(__atomic_compare_exchange)
+; LA64D-NEXT:    fld.d $fa0, $sp, 16
+; LA64D-NEXT:    beqz $a0, .LBB28_1
+; LA64D-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64D-NEXT:    fld.d $fs0, $sp, 24 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s3, $sp, 32 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s2, $sp, 40 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s1, $sp, 48 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s0, $sp, 56 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $fp, $sp, 64 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
+; LA64D-NEXT:    addi.d $sp, $sp, 80
+; LA64D-NEXT:    ret
+  %v = atomicrmw fadd ptr %p, double 1.0 seq_cst, align 4
+  ret double %v
+}
+
+define double @double_fsub_seq_cst(ptr %p) nounwind {
+; LA64F-LABEL: double_fsub_seq_cst:
+; LA64F:       # %bb.0:
+; LA64F-NEXT:    addi.d $sp, $sp, -80
+; LA64F-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $fp, $sp, 64 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s0, $sp, 56 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s1, $sp, 48 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s2, $sp, 40 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s3, $sp, 32 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s4, $sp, 24 # 8-byte Folded Spill
+; LA64F-NEXT:    move $fp, $a0
+; LA64F-NEXT:    ld.d $a0, $a0, 0
+; LA64F-NEXT:    lu52i.d $s0, $zero, -1025
+; LA64F-NEXT:    ori $s1, $zero, 8
+; LA64F-NEXT:    addi.d $s2, $sp, 16
+; LA64F-NEXT:    addi.d $s3, $sp, 8
+; LA64F-NEXT:    ori $s4, $zero, 5
+; LA64F-NEXT:    .p2align 4, , 16
+; LA64F-NEXT:  .LBB29_1: # %atomicrmw.start
+; LA64F-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64F-NEXT:    st.d $a0, $sp, 16
+; LA64F-NEXT:    move $a1, $s0
+; LA64F-NEXT:    bl %plt(__adddf3)
+; LA64F-NEXT:    st.d $a0, $sp, 8
+; LA64F-NEXT:    move $a0, $s1
+; LA64F-NEXT:    move $a1, $fp
+; LA64F-NEXT:    move $a2, $s2
+; LA64F-NEXT:    move $a3, $s3
+; LA64F-NEXT:    move $a4, $s4
+; LA64F-NEXT:    move $a5, $s4
+; LA64F-NEXT:    bl %plt(__atomic_compare_exchange)
+; LA64F-NEXT:    move $a1, $a0
+; LA64F-NEXT:    ld.d $a0, $sp, 16
+; LA64F-NEXT:    beqz $a1, .LBB29_1
+; LA64F-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64F-NEXT:    ld.d $s4, $sp, 24 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s3, $sp, 32 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s2, $sp, 40 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s1, $sp, 48 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s0, $sp, 56 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $fp, $sp, 64 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
+; LA64F-NEXT:    addi.d $sp, $sp, 80
+; LA64F-NEXT:    ret
+;
+; LA64D-LABEL: double_fsub_seq_cst:
+; LA64D:       # %bb.0:
+; LA64D-NEXT:    addi.d $sp, $sp, -80
+; LA64D-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $fp, $sp, 64 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s0, $sp, 56 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s1, $sp, 48 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s2, $sp, 40 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s3, $sp, 32 # 8-byte Folded Spill
+; LA64D-NEXT:    fst.d $fs0, $sp, 24 # 8-byte Folded Spill
+; LA64D-NEXT:    move $fp, $a0
+; LA64D-NEXT:    fld.d $fa0, $a0, 0
+; LA64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI29_0)
+; LA64D-NEXT:    addi.d $a0, $a0, %pc_lo12(.LCPI29_0)
+; LA64D-NEXT:    fld.d $fs0, $a0, 0
+; LA64D-NEXT:    ori $s0, $zero, 8
+; LA64D-NEXT:    addi.d $s1, $sp, 16
+; LA64D-NEXT:    addi.d $s2, $sp, 8
+; LA64D-NEXT:    ori $s3, $zero, 5
+; LA64D-NEXT:    .p2align 4, , 16
+; LA64D-NEXT:  .LBB29_1: # %atomicrmw.start
+; LA64D-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64D-NEXT:    fst.d $fa0, $sp, 16
+; LA64D-NEXT:    fadd.d $fa0, $fa0, $fs0
+; LA64D-NEXT:    fst.d $fa0, $sp, 8
+; LA64D-NEXT:    move $a0, $s0
+; LA64D-NEXT:    move $a1, $fp
+; LA64D-NEXT:    move $a2, $s1
+; LA64D-NEXT:    move $a3, $s2
+; LA64D-NEXT:    move $a4, $s3
+; LA64D-NEXT:    move $a5, $s3
+; LA64D-NEXT:    bl %plt(__atomic_compare_exchange)
+; LA64D-NEXT:    fld.d $fa0, $sp, 16
+; LA64D-NEXT:    beqz $a0, .LBB29_1
+; LA64D-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64D-NEXT:    fld.d $fs0, $sp, 24 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s3, $sp, 32 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s2, $sp, 40 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s1, $sp, 48 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s0, $sp, 56 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $fp, $sp, 64 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
+; LA64D-NEXT:    addi.d $sp, $sp, 80
+; LA64D-NEXT:    ret
+  %v = atomicrmw fsub ptr %p, double 1.0 seq_cst, align 4
+  ret double %v
+}
+
+define double @double_fmin_seq_cst(ptr %p) nounwind {
+; LA64F-LABEL: double_fmin_seq_cst:
+; LA64F:       # %bb.0:
+; LA64F-NEXT:    addi.d $sp, $sp, -80
+; LA64F-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $fp, $sp, 64 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s0, $sp, 56 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s1, $sp, 48 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s2, $sp, 40 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s3, $sp, 32 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s4, $sp, 24 # 8-byte Folded Spill
+; LA64F-NEXT:    move $fp, $a0
+; LA64F-NEXT:    ld.d $a0, $a0, 0
+; LA64F-NEXT:    lu52i.d $s0, $zero, 1023
+; LA64F-NEXT:    ori $s1, $zero, 8
+; LA64F-NEXT:    addi.d $s2, $sp, 16
+; LA64F-NEXT:    addi.d $s3, $sp, 8
+; LA64F-NEXT:    ori $s4, $zero, 5
+; LA64F-NEXT:    .p2align 4, , 16
+; LA64F-NEXT:  .LBB30_1: # %atomicrmw.start
+; LA64F-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64F-NEXT:    st.d $a0, $sp, 16
+; LA64F-NEXT:    move $a1, $s0
+; LA64F-NEXT:    bl %plt(fmin)
+; LA64F-NEXT:    st.d $a0, $sp, 8
+; LA64F-NEXT:    move $a0, $s1
+; LA64F-NEXT:    move $a1, $fp
+; LA64F-NEXT:    move $a2, $s2
+; LA64F-NEXT:    move $a3, $s3
+; LA64F-NEXT:    move $a4, $s4
+; LA64F-NEXT:    move $a5, $s4
+; LA64F-NEXT:    bl %plt(__atomic_compare_exchange)
+; LA64F-NEXT:    move $a1, $a0
+; LA64F-NEXT:    ld.d $a0, $sp, 16
+; LA64F-NEXT:    beqz $a1, .LBB30_1
+; LA64F-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64F-NEXT:    ld.d $s4, $sp, 24 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s3, $sp, 32 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s2, $sp, 40 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s1, $sp, 48 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s0, $sp, 56 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $fp, $sp, 64 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
+; LA64F-NEXT:    addi.d $sp, $sp, 80
+; LA64F-NEXT:    ret
+;
+; LA64D-LABEL: double_fmin_seq_cst:
+; LA64D:       # %bb.0:
+; LA64D-NEXT:    addi.d $sp, $sp, -80
+; LA64D-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $fp, $sp, 64 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s0, $sp, 56 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s1, $sp, 48 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s2, $sp, 40 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s3, $sp, 32 # 8-byte Folded Spill
+; LA64D-NEXT:    fst.d $fs0, $sp, 24 # 8-byte Folded Spill
+; LA64D-NEXT:    move $fp, $a0
+; LA64D-NEXT:    fld.d $fa0, $a0, 0
+; LA64D-NEXT:    addi.d $a0, $zero, 1
+; LA64D-NEXT:    movgr2fr.d $fa1, $a0
+; LA64D-NEXT:    ffint.d.l $fs0, $fa1
+; LA64D-NEXT:    ori $s0, $zero, 8
+; LA64D-NEXT:    addi.d $s1, $sp, 16
+; LA64D-NEXT:    addi.d $s2, $sp, 8
+; LA64D-NEXT:    ori $s3, $zero, 5
+; LA64D-NEXT:    .p2align 4, , 16
+; LA64D-NEXT:  .LBB30_1: # %atomicrmw.start
+; LA64D-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64D-NEXT:    fst.d $fa0, $sp, 16
+; LA64D-NEXT:    fmax.d $fa0, $fa0, $fa0
+; LA64D-NEXT:    fmin.d $fa0, $fa0, $fs0
+; LA64D-NEXT:    fst.d $fa0, $sp, 8
+; LA64D-NEXT:    move $a0, $s0
+; LA64D-NEXT:    move $a1, $fp
+; LA64D-NEXT:    move $a2, $s1
+; LA64D-NEXT:    move $a3, $s2
+; LA64D-NEXT:    move $a4, $s3
+; LA64D-NEXT:    move $a5, $s3
+; LA64D-NEXT:    bl %plt(__atomic_compare_exchange)
+; LA64D-NEXT:    fld.d $fa0, $sp, 16
+; LA64D-NEXT:    beqz $a0, .LBB30_1
+; LA64D-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64D-NEXT:    fld.d $fs0, $sp, 24 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s3, $sp, 32 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s2, $sp, 40 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s1, $sp, 48 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s0, $sp, 56 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $fp, $sp, 64 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
+; LA64D-NEXT:    addi.d $sp, $sp, 80
+; LA64D-NEXT:    ret
+  %v = atomicrmw fmin ptr %p, double 1.0 seq_cst, align 4
+  ret double %v
+}
+
+define double @double_fmax_seq_cst(ptr %p) nounwind {
+; LA64F-LABEL: double_fmax_seq_cst:
+; LA64F:       # %bb.0:
+; LA64F-NEXT:    addi.d $sp, $sp, -80
+; LA64F-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $fp, $sp, 64 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s0, $sp, 56 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s1, $sp, 48 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s2, $sp, 40 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s3, $sp, 32 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s4, $sp, 24 # 8-byte Folded Spill
+; LA64F-NEXT:    move $fp, $a0
+; LA64F-NEXT:    ld.d $a0, $a0, 0
+; LA64F-NEXT:    lu52i.d $s0, $zero, 1023
+; LA64F-NEXT:    ori $s1, $zero, 8
+; LA64F-NEXT:    addi.d $s2, $sp, 16
+; LA64F-NEXT:    addi.d $s3, $sp, 8
+; LA64F-NEXT:    ori $s4, $zero, 5
+; LA64F-NEXT:    .p2align 4, , 16
+; LA64F-NEXT:  .LBB31_1: # %atomicrmw.start
+; LA64F-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64F-NEXT:    st.d $a0, $sp, 16
+; LA64F-NEXT:    move $a1, $s0
+; LA64F-NEXT:    bl %plt(fmax)
+; LA64F-NEXT:    st.d $a0, $sp, 8
+; LA64F-NEXT:    move $a0, $s1
+; LA64F-NEXT:    move $a1, $fp
+; LA64F-NEXT:    move $a2, $s2
+; LA64F-NEXT:    move $a3, $s3
+; LA64F-NEXT:    move $a4, $s4
+; LA64F-NEXT:    move $a5, $s4
+; LA64F-NEXT:    bl %plt(__atomic_compare_exchange)
+; LA64F-NEXT:    move $a1, $a0
+; LA64F-NEXT:    ld.d $a0, $sp, 16
+; LA64F-NEXT:    beqz $a1, .LBB31_1
+; LA64F-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64F-NEXT:    ld.d $s4, $sp, 24 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s3, $sp, 32 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s2, $sp, 40 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s1, $sp, 48 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s0, $sp, 56 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $fp, $sp, 64 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
+; LA64F-NEXT:    addi.d $sp, $sp, 80
+; LA64F-NEXT:    ret
+;
+; LA64D-LABEL: double_fmax_seq_cst:
+; LA64D:       # %bb.0:
+; LA64D-NEXT:    addi.d $sp, $sp, -80
+; LA64D-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $fp, $sp, 64 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s0, $sp, 56 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s1, $sp, 48 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s2, $sp, 40 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s3, $sp, 32 # 8-byte Folded Spill
+; LA64D-NEXT:    fst.d $fs0, $sp, 24 # 8-byte Folded Spill
+; LA64D-NEXT:    move $fp, $a0
+; LA64D-NEXT:    fld.d $fa0, $a0, 0
+; LA64D-NEXT:    addi.d $a0, $zero, 1
+; LA64D-NEXT:    movgr2fr.d $fa1, $a0
+; LA64D-NEXT:    ffint.d.l $fs0, $fa1
+; LA64D-NEXT:    ori $s0, $zero, 8
+; LA64D-NEXT:    addi.d $s1, $sp, 16
+; LA64D-NEXT:    addi.d $s2, $sp, 8
+; LA64D-NEXT:    ori $s3, $zero, 5
+; LA64D-NEXT:    .p2align 4, , 16
+; LA64D-NEXT:  .LBB31_1: # %atomicrmw.start
+; LA64D-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64D-NEXT:    fst.d $fa0, $sp, 16
+; LA64D-NEXT:    fmax.d $fa0, $fa0, $fa0
+; LA64D-NEXT:    fmax.d $fa0, $fa0, $fs0
+; LA64D-NEXT:    fst.d $fa0, $sp, 8
+; LA64D-NEXT:    move $a0, $s0
+; LA64D-NEXT:    move $a1, $fp
+; LA64D-NEXT:    move $a2, $s1
+; LA64D-NEXT:    move $a3, $s2
+; LA64D-NEXT:    move $a4, $s3
+; LA64D-NEXT:    move $a5, $s3
+; LA64D-NEXT:    bl %plt(__atomic_compare_exchange)
+; LA64D-NEXT:    fld.d $fa0, $sp, 16
+; LA64D-NEXT:    beqz $a0, .LBB31_1
+; LA64D-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64D-NEXT:    fld.d $fs0, $sp, 24 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s3, $sp, 32 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s2, $sp, 40 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s1, $sp, 48 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s0, $sp, 56 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $fp, $sp, 64 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
+; LA64D-NEXT:    addi.d $sp, $sp, 80
+; LA64D-NEXT:    ret
+  %v = atomicrmw fmax ptr %p, double 1.0 seq_cst, align 4
+  ret double %v
+}
+
+define float @float_fadd_monotonic(ptr %p) nounwind {
+; LA64F-LABEL: float_fadd_monotonic:
+; LA64F:       # %bb.0:
+; LA64F-NEXT:    fld.s $fa0, $a0, 0
+; LA64F-NEXT:    addi.w $a1, $zero, 1
+; LA64F-NEXT:    movgr2fr.w $fa1, $a1
+; LA64F-NEXT:    ffint.s.w $fa1, $fa1
+; LA64F-NEXT:    .p2align 4, , 16
+; LA64F-NEXT:  .LBB32_1: # %atomicrmw.start
+; LA64F-NEXT:    # =>This Loop Header: Depth=1
+; LA64F-NEXT:    # Child Loop BB32_3 Depth 2
+; LA64F-NEXT:    fadd.s $fa2, $fa0, $fa1
+; LA64F-NEXT:    movfr2gr.s $a1, $fa2
+; LA64F-NEXT:    movfr2gr.s $a2, $fa0
+; LA64F-NEXT:  .LBB32_3: # %atomicrmw.start
+; LA64F-NEXT:    # Parent Loop BB32_1 Depth=1
+; LA64F-NEXT:    # => This Inner Loop Header: Depth=2
+; LA64F-NEXT:    ll.w $a3, $a0, 0
+; LA64F-NEXT:    bne $a3, $a2, .LBB32_5
+; LA64F-NEXT:  # %bb.4: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB32_3 Depth=2
+; LA64F-NEXT:    move $a4, $a1
+; LA64F-NEXT:    sc.w $a4, $a0, 0
+; LA64F-NEXT:    beqz $a4, .LBB32_3
+; LA64F-NEXT:    b .LBB32_6
+; LA64F-NEXT:  .LBB32_5: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB32_1 Depth=1
+; LA64F-NEXT:    dbar 1792
+; LA64F-NEXT:  .LBB32_6: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB32_1 Depth=1
+; LA64F-NEXT:    movgr2fr.w $fa0, $a3
+; LA64F-NEXT:    addi.w $a1, $a2, 0
+; LA64F-NEXT:    bne $a3, $a1, .LBB32_1
+; LA64F-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64F-NEXT:    ret
+;
+; LA64D-LABEL: float_fadd_monotonic:
+; LA64D:       # %bb.0:
+; LA64D-NEXT:    fld.s $fa0, $a0, 0
+; LA64D-NEXT:    addi.w $a1, $zero, 1
+; LA64D-NEXT:    movgr2fr.w $fa1, $a1
+; LA64D-NEXT:    ffint.s.w $fa1, $fa1
+; LA64D-NEXT:    .p2align 4, , 16
+; LA64D-NEXT:  .LBB32_1: # %atomicrmw.start
+; LA64D-NEXT:    # =>This Loop Header: Depth=1
+; LA64D-NEXT:    # Child Loop BB32_3 Depth 2
+; LA64D-NEXT:    fadd.s $fa2, $fa0, $fa1
+; LA64D-NEXT:    movfr2gr.s $a1, $fa2
+; LA64D-NEXT:    movfr2gr.s $a2, $fa0
+; LA64D-NEXT:  .LBB32_3: # %atomicrmw.start
+; LA64D-NEXT:    # Parent Loop BB32_1 Depth=1
+; LA64D-NEXT:    # => This Inner Loop Header: Depth=2
+; LA64D-NEXT:    ll.w $a3, $a0, 0
+; LA64D-NEXT:    bne $a3, $a2, .LBB32_5
+; LA64D-NEXT:  # %bb.4: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB32_3 Depth=2
+; LA64D-NEXT:    move $a4, $a1
+; LA64D-NEXT:    sc.w $a4, $a0, 0
+; LA64D-NEXT:    beqz $a4, .LBB32_3
+; LA64D-NEXT:    b .LBB32_6
+; LA64D-NEXT:  .LBB32_5: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB32_1 Depth=1
+; LA64D-NEXT:    dbar 1792
+; LA64D-NEXT:  .LBB32_6: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB32_1 Depth=1
+; LA64D-NEXT:    movgr2fr.w $fa0, $a3
+; LA64D-NEXT:    addi.w $a1, $a2, 0
+; LA64D-NEXT:    bne $a3, $a1, .LBB32_1
+; LA64D-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64D-NEXT:    ret
+  %v = atomicrmw fadd ptr %p, float 1.0 monotonic, align 4
+  ret float %v
+}
+
+define float @float_fsub_monotonic(ptr %p) nounwind {
+; LA64F-LABEL: float_fsub_monotonic:
+; LA64F:       # %bb.0:
+; LA64F-NEXT:    fld.s $fa0, $a0, 0
+; LA64F-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI33_0)
+; LA64F-NEXT:    addi.d $a1, $a1, %pc_lo12(.LCPI33_0)
+; LA64F-NEXT:    fld.s $fa1, $a1, 0
+; LA64F-NEXT:    .p2align 4, , 16
+; LA64F-NEXT:  .LBB33_1: # %atomicrmw.start
+; LA64F-NEXT:    # =>This Loop Header: Depth=1
+; LA64F-NEXT:    # Child Loop BB33_3 Depth 2
+; LA64F-NEXT:    fadd.s $fa2, $fa0, $fa1
+; LA64F-NEXT:    movfr2gr.s $a1, $fa2
+; LA64F-NEXT:    movfr2gr.s $a2, $fa0
+; LA64F-NEXT:  .LBB33_3: # %atomicrmw.start
+; LA64F-NEXT:    # Parent Loop BB33_1 Depth=1
+; LA64F-NEXT:    # => This Inner Loop Header: Depth=2
+; LA64F-NEXT:    ll.w $a3, $a0, 0
+; LA64F-NEXT:    bne $a3, $a2, .LBB33_5
+; LA64F-NEXT:  # %bb.4: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB33_3 Depth=2
+; LA64F-NEXT:    move $a4, $a1
+; LA64F-NEXT:    sc.w $a4, $a0, 0
+; LA64F-NEXT:    beqz $a4, .LBB33_3
+; LA64F-NEXT:    b .LBB33_6
+; LA64F-NEXT:  .LBB33_5: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB33_1 Depth=1
+; LA64F-NEXT:    dbar 1792
+; LA64F-NEXT:  .LBB33_6: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB33_1 Depth=1
+; LA64F-NEXT:    movgr2fr.w $fa0, $a3
+; LA64F-NEXT:    addi.w $a1, $a2, 0
+; LA64F-NEXT:    bne $a3, $a1, .LBB33_1
+; LA64F-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64F-NEXT:    ret
+;
+; LA64D-LABEL: float_fsub_monotonic:
+; LA64D:       # %bb.0:
+; LA64D-NEXT:    fld.s $fa0, $a0, 0
+; LA64D-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI33_0)
+; LA64D-NEXT:    addi.d $a1, $a1, %pc_lo12(.LCPI33_0)
+; LA64D-NEXT:    fld.s $fa1, $a1, 0
+; LA64D-NEXT:    .p2align 4, , 16
+; LA64D-NEXT:  .LBB33_1: # %atomicrmw.start
+; LA64D-NEXT:    # =>This Loop Header: Depth=1
+; LA64D-NEXT:    # Child Loop BB33_3 Depth 2
+; LA64D-NEXT:    fadd.s $fa2, $fa0, $fa1
+; LA64D-NEXT:    movfr2gr.s $a1, $fa2
+; LA64D-NEXT:    movfr2gr.s $a2, $fa0
+; LA64D-NEXT:  .LBB33_3: # %atomicrmw.start
+; LA64D-NEXT:    # Parent Loop BB33_1 Depth=1
+; LA64D-NEXT:    # => This Inner Loop Header: Depth=2
+; LA64D-NEXT:    ll.w $a3, $a0, 0
+; LA64D-NEXT:    bne $a3, $a2, .LBB33_5
+; LA64D-NEXT:  # %bb.4: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB33_3 Depth=2
+; LA64D-NEXT:    move $a4, $a1
+; LA64D-NEXT:    sc.w $a4, $a0, 0
+; LA64D-NEXT:    beqz $a4, .LBB33_3
+; LA64D-NEXT:    b .LBB33_6
+; LA64D-NEXT:  .LBB33_5: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB33_1 Depth=1
+; LA64D-NEXT:    dbar 1792
+; LA64D-NEXT:  .LBB33_6: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB33_1 Depth=1
+; LA64D-NEXT:    movgr2fr.w $fa0, $a3
+; LA64D-NEXT:    addi.w $a1, $a2, 0
+; LA64D-NEXT:    bne $a3, $a1, .LBB33_1
+; LA64D-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64D-NEXT:    ret
+  %v = atomicrmw fsub ptr %p, float 1.0 monotonic, align 4
+  ret float %v
+}
+
+define float @float_fmin_monotonic(ptr %p) nounwind {
+; LA64F-LABEL: float_fmin_monotonic:
+; LA64F:       # %bb.0:
+; LA64F-NEXT:    fld.s $fa0, $a0, 0
+; LA64F-NEXT:    addi.w $a1, $zero, 1
+; LA64F-NEXT:    movgr2fr.w $fa1, $a1
+; LA64F-NEXT:    ffint.s.w $fa1, $fa1
+; LA64F-NEXT:    .p2align 4, , 16
+; LA64F-NEXT:  .LBB34_1: # %atomicrmw.start
+; LA64F-NEXT:    # =>This Loop Header: Depth=1
+; LA64F-NEXT:    # Child Loop BB34_3 Depth 2
+; LA64F-NEXT:    fmax.s $fa2, $fa0, $fa0
+; LA64F-NEXT:    fmin.s $fa2, $fa2, $fa1
+; LA64F-NEXT:    movfr2gr.s $a1, $fa2
+; LA64F-NEXT:    movfr2gr.s $a2, $fa0
+; LA64F-NEXT:  .LBB34_3: # %atomicrmw.start
+; LA64F-NEXT:    # Parent Loop BB34_1 Depth=1
+; LA64F-NEXT:    # => This Inner Loop Header: Depth=2
+; LA64F-NEXT:    ll.w $a3, $a0, 0
+; LA64F-NEXT:    bne $a3, $a2, .LBB34_5
+; LA64F-NEXT:  # %bb.4: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB34_3 Depth=2
+; LA64F-NEXT:    move $a4, $a1
+; LA64F-NEXT:    sc.w $a4, $a0, 0
+; LA64F-NEXT:    beqz $a4, .LBB34_3
+; LA64F-NEXT:    b .LBB34_6
+; LA64F-NEXT:  .LBB34_5: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB34_1 Depth=1
+; LA64F-NEXT:    dbar 1792
+; LA64F-NEXT:  .LBB34_6: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB34_1 Depth=1
+; LA64F-NEXT:    movgr2fr.w $fa0, $a3
+; LA64F-NEXT:    addi.w $a1, $a2, 0
+; LA64F-NEXT:    bne $a3, $a1, .LBB34_1
+; LA64F-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64F-NEXT:    ret
+;
+; LA64D-LABEL: float_fmin_monotonic:
+; LA64D:       # %bb.0:
+; LA64D-NEXT:    fld.s $fa0, $a0, 0
+; LA64D-NEXT:    addi.w $a1, $zero, 1
+; LA64D-NEXT:    movgr2fr.w $fa1, $a1
+; LA64D-NEXT:    ffint.s.w $fa1, $fa1
+; LA64D-NEXT:    .p2align 4, , 16
+; LA64D-NEXT:  .LBB34_1: # %atomicrmw.start
+; LA64D-NEXT:    # =>This Loop Header: Depth=1
+; LA64D-NEXT:    # Child Loop BB34_3 Depth 2
+; LA64D-NEXT:    fmax.s $fa2, $fa0, $fa0
+; LA64D-NEXT:    fmin.s $fa2, $fa2, $fa1
+; LA64D-NEXT:    movfr2gr.s $a1, $fa2
+; LA64D-NEXT:    movfr2gr.s $a2, $fa0
+; LA64D-NEXT:  .LBB34_3: # %atomicrmw.start
+; LA64D-NEXT:    # Parent Loop BB34_1 Depth=1
+; LA64D-NEXT:    # => This Inner Loop Header: Depth=2
+; LA64D-NEXT:    ll.w $a3, $a0, 0
+; LA64D-NEXT:    bne $a3, $a2, .LBB34_5
+; LA64D-NEXT:  # %bb.4: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB34_3 Depth=2
+; LA64D-NEXT:    move $a4, $a1
+; LA64D-NEXT:    sc.w $a4, $a0, 0
+; LA64D-NEXT:    beqz $a4, .LBB34_3
+; LA64D-NEXT:    b .LBB34_6
+; LA64D-NEXT:  .LBB34_5: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB34_1 Depth=1
+; LA64D-NEXT:    dbar 1792
+; LA64D-NEXT:  .LBB34_6: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB34_1 Depth=1
+; LA64D-NEXT:    movgr2fr.w $fa0, $a3
+; LA64D-NEXT:    addi.w $a1, $a2, 0
+; LA64D-NEXT:    bne $a3, $a1, .LBB34_1
+; LA64D-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64D-NEXT:    ret
+  %v = atomicrmw fmin ptr %p, float 1.0 monotonic, align 4
+  ret float %v
+}
+
+define float @float_fmax_monotonic(ptr %p) nounwind {
+; LA64F-LABEL: float_fmax_monotonic:
+; LA64F:       # %bb.0:
+; LA64F-NEXT:    fld.s $fa0, $a0, 0
+; LA64F-NEXT:    addi.w $a1, $zero, 1
+; LA64F-NEXT:    movgr2fr.w $fa1, $a1
+; LA64F-NEXT:    ffint.s.w $fa1, $fa1
+; LA64F-NEXT:    .p2align 4, , 16
+; LA64F-NEXT:  .LBB35_1: # %atomicrmw.start
+; LA64F-NEXT:    # =>This Loop Header: Depth=1
+; LA64F-NEXT:    # Child Loop BB35_3 Depth 2
+; LA64F-NEXT:    fmax.s $fa2, $fa0, $fa0
+; LA64F-NEXT:    fmax.s $fa2, $fa2, $fa1
+; LA64F-NEXT:    movfr2gr.s $a1, $fa2
+; LA64F-NEXT:    movfr2gr.s $a2, $fa0
+; LA64F-NEXT:  .LBB35_3: # %atomicrmw.start
+; LA64F-NEXT:    # Parent Loop BB35_1 Depth=1
+; LA64F-NEXT:    # => This Inner Loop Header: Depth=2
+; LA64F-NEXT:    ll.w $a3, $a0, 0
+; LA64F-NEXT:    bne $a3, $a2, .LBB35_5
+; LA64F-NEXT:  # %bb.4: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB35_3 Depth=2
+; LA64F-NEXT:    move $a4, $a1
+; LA64F-NEXT:    sc.w $a4, $a0, 0
+; LA64F-NEXT:    beqz $a4, .LBB35_3
+; LA64F-NEXT:    b .LBB35_6
+; LA64F-NEXT:  .LBB35_5: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB35_1 Depth=1
+; LA64F-NEXT:    dbar 1792
+; LA64F-NEXT:  .LBB35_6: # %atomicrmw.start
+; LA64F-NEXT:    # in Loop: Header=BB35_1 Depth=1
+; LA64F-NEXT:    movgr2fr.w $fa0, $a3
+; LA64F-NEXT:    addi.w $a1, $a2, 0
+; LA64F-NEXT:    bne $a3, $a1, .LBB35_1
+; LA64F-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64F-NEXT:    ret
+;
+; LA64D-LABEL: float_fmax_monotonic:
+; LA64D:       # %bb.0:
+; LA64D-NEXT:    fld.s $fa0, $a0, 0
+; LA64D-NEXT:    addi.w $a1, $zero, 1
+; LA64D-NEXT:    movgr2fr.w $fa1, $a1
+; LA64D-NEXT:    ffint.s.w $fa1, $fa1
+; LA64D-NEXT:    .p2align 4, , 16
+; LA64D-NEXT:  .LBB35_1: # %atomicrmw.start
+; LA64D-NEXT:    # =>This Loop Header: Depth=1
+; LA64D-NEXT:    # Child Loop BB35_3 Depth 2
+; LA64D-NEXT:    fmax.s $fa2, $fa0, $fa0
+; LA64D-NEXT:    fmax.s $fa2, $fa2, $fa1
+; LA64D-NEXT:    movfr2gr.s $a1, $fa2
+; LA64D-NEXT:    movfr2gr.s $a2, $fa0
+; LA64D-NEXT:  .LBB35_3: # %atomicrmw.start
+; LA64D-NEXT:    # Parent Loop BB35_1 Depth=1
+; LA64D-NEXT:    # => This Inner Loop Header: Depth=2
+; LA64D-NEXT:    ll.w $a3, $a0, 0
+; LA64D-NEXT:    bne $a3, $a2, .LBB35_5
+; LA64D-NEXT:  # %bb.4: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB35_3 Depth=2
+; LA64D-NEXT:    move $a4, $a1
+; LA64D-NEXT:    sc.w $a4, $a0, 0
+; LA64D-NEXT:    beqz $a4, .LBB35_3
+; LA64D-NEXT:    b .LBB35_6
+; LA64D-NEXT:  .LBB35_5: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB35_1 Depth=1
+; LA64D-NEXT:    dbar 1792
+; LA64D-NEXT:  .LBB35_6: # %atomicrmw.start
+; LA64D-NEXT:    # in Loop: Header=BB35_1 Depth=1
+; LA64D-NEXT:    movgr2fr.w $fa0, $a3
+; LA64D-NEXT:    addi.w $a1, $a2, 0
+; LA64D-NEXT:    bne $a3, $a1, .LBB35_1
+; LA64D-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64D-NEXT:    ret
+  %v = atomicrmw fmax ptr %p, float 1.0 monotonic, align 4
+  ret float %v
+}
+
+define double @double_fadd_monotonic(ptr %p) nounwind {
+; LA64F-LABEL: double_fadd_monotonic:
+; LA64F:       # %bb.0:
+; LA64F-NEXT:    addi.d $sp, $sp, -64
+; LA64F-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s0, $sp, 40 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s1, $sp, 32 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s2, $sp, 24 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s3, $sp, 16 # 8-byte Folded Spill
+; LA64F-NEXT:    move $fp, $a0
+; LA64F-NEXT:    ld.d $a0, $a0, 0
+; LA64F-NEXT:    lu52i.d $s0, $zero, 1023
+; LA64F-NEXT:    ori $s1, $zero, 8
+; LA64F-NEXT:    addi.d $s2, $sp, 8
+; LA64F-NEXT:    addi.d $s3, $sp, 0
+; LA64F-NEXT:    .p2align 4, , 16
+; LA64F-NEXT:  .LBB36_1: # %atomicrmw.start
+; LA64F-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64F-NEXT:    st.d $a0, $sp, 8
+; LA64F-NEXT:    move $a1, $s0
+; LA64F-NEXT:    bl %plt(__adddf3)
+; LA64F-NEXT:    st.d $a0, $sp, 0
+; LA64F-NEXT:    move $a0, $s1
+; LA64F-NEXT:    move $a1, $fp
+; LA64F-NEXT:    move $a2, $s2
+; LA64F-NEXT:    move $a3, $s3
+; LA64F-NEXT:    move $a4, $zero
+; LA64F-NEXT:    move $a5, $zero
+; LA64F-NEXT:    bl %plt(__atomic_compare_exchange)
+; LA64F-NEXT:    move $a1, $a0
+; LA64F-NEXT:    ld.d $a0, $sp, 8
+; LA64F-NEXT:    beqz $a1, .LBB36_1
+; LA64F-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64F-NEXT:    ld.d $s3, $sp, 16 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s2, $sp, 24 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s1, $sp, 32 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s0, $sp, 40 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
+; LA64F-NEXT:    addi.d $sp, $sp, 64
+; LA64F-NEXT:    ret
+;
+; LA64D-LABEL: double_fadd_monotonic:
+; LA64D:       # %bb.0:
+; LA64D-NEXT:    addi.d $sp, $sp, -64
+; LA64D-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s0, $sp, 40 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s1, $sp, 32 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s2, $sp, 24 # 8-byte Folded Spill
+; LA64D-NEXT:    fst.d $fs0, $sp, 16 # 8-byte Folded Spill
+; LA64D-NEXT:    move $fp, $a0
+; LA64D-NEXT:    fld.d $fa0, $a0, 0
+; LA64D-NEXT:    addi.d $a0, $zero, 1
+; LA64D-NEXT:    movgr2fr.d $fa1, $a0
+; LA64D-NEXT:    ffint.d.l $fs0, $fa1
+; LA64D-NEXT:    ori $s0, $zero, 8
+; LA64D-NEXT:    addi.d $s1, $sp, 8
+; LA64D-NEXT:    addi.d $s2, $sp, 0
+; LA64D-NEXT:    .p2align 4, , 16
+; LA64D-NEXT:  .LBB36_1: # %atomicrmw.start
+; LA64D-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64D-NEXT:    fst.d $fa0, $sp, 8
+; LA64D-NEXT:    fadd.d $fa0, $fa0, $fs0
+; LA64D-NEXT:    fst.d $fa0, $sp, 0
+; LA64D-NEXT:    move $a0, $s0
+; LA64D-NEXT:    move $a1, $fp
+; LA64D-NEXT:    move $a2, $s1
+; LA64D-NEXT:    move $a3, $s2
+; LA64D-NEXT:    move $a4, $zero
+; LA64D-NEXT:    move $a5, $zero
+; LA64D-NEXT:    bl %plt(__atomic_compare_exchange)
+; LA64D-NEXT:    fld.d $fa0, $sp, 8
+; LA64D-NEXT:    beqz $a0, .LBB36_1
+; LA64D-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64D-NEXT:    fld.d $fs0, $sp, 16 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s2, $sp, 24 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s1, $sp, 32 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s0, $sp, 40 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
+; LA64D-NEXT:    addi.d $sp, $sp, 64
+; LA64D-NEXT:    ret
+  %v = atomicrmw fadd ptr %p, double 1.0 monotonic, align 4
+  ret double %v
+}
+
+define double @double_fsub_monotonic(ptr %p) nounwind {
+; LA64F-LABEL: double_fsub_monotonic:
+; LA64F:       # %bb.0:
+; LA64F-NEXT:    addi.d $sp, $sp, -64
+; LA64F-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s0, $sp, 40 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s1, $sp, 32 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s2, $sp, 24 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s3, $sp, 16 # 8-byte Folded Spill
+; LA64F-NEXT:    move $fp, $a0
+; LA64F-NEXT:    ld.d $a0, $a0, 0
+; LA64F-NEXT:    lu52i.d $s0, $zero, -1025
+; LA64F-NEXT:    ori $s1, $zero, 8
+; LA64F-NEXT:    addi.d $s2, $sp, 8
+; LA64F-NEXT:    addi.d $s3, $sp, 0
+; LA64F-NEXT:    .p2align 4, , 16
+; LA64F-NEXT:  .LBB37_1: # %atomicrmw.start
+; LA64F-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64F-NEXT:    st.d $a0, $sp, 8
+; LA64F-NEXT:    move $a1, $s0
+; LA64F-NEXT:    bl %plt(__adddf3)
+; LA64F-NEXT:    st.d $a0, $sp, 0
+; LA64F-NEXT:    move $a0, $s1
+; LA64F-NEXT:    move $a1, $fp
+; LA64F-NEXT:    move $a2, $s2
+; LA64F-NEXT:    move $a3, $s3
+; LA64F-NEXT:    move $a4, $zero
+; LA64F-NEXT:    move $a5, $zero
+; LA64F-NEXT:    bl %plt(__atomic_compare_exchange)
+; LA64F-NEXT:    move $a1, $a0
+; LA64F-NEXT:    ld.d $a0, $sp, 8
+; LA64F-NEXT:    beqz $a1, .LBB37_1
+; LA64F-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64F-NEXT:    ld.d $s3, $sp, 16 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s2, $sp, 24 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s1, $sp, 32 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s0, $sp, 40 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
+; LA64F-NEXT:    addi.d $sp, $sp, 64
+; LA64F-NEXT:    ret
+;
+; LA64D-LABEL: double_fsub_monotonic:
+; LA64D:       # %bb.0:
+; LA64D-NEXT:    addi.d $sp, $sp, -64
+; LA64D-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s0, $sp, 40 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s1, $sp, 32 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s2, $sp, 24 # 8-byte Folded Spill
+; LA64D-NEXT:    fst.d $fs0, $sp, 16 # 8-byte Folded Spill
+; LA64D-NEXT:    move $fp, $a0
+; LA64D-NEXT:    fld.d $fa0, $a0, 0
+; LA64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI37_0)
+; LA64D-NEXT:    addi.d $a0, $a0, %pc_lo12(.LCPI37_0)
+; LA64D-NEXT:    fld.d $fs0, $a0, 0
+; LA64D-NEXT:    ori $s0, $zero, 8
+; LA64D-NEXT:    addi.d $s1, $sp, 8
+; LA64D-NEXT:    addi.d $s2, $sp, 0
+; LA64D-NEXT:    .p2align 4, , 16
+; LA64D-NEXT:  .LBB37_1: # %atomicrmw.start
+; LA64D-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64D-NEXT:    fst.d $fa0, $sp, 8
+; LA64D-NEXT:    fadd.d $fa0, $fa0, $fs0
+; LA64D-NEXT:    fst.d $fa0, $sp, 0
+; LA64D-NEXT:    move $a0, $s0
+; LA64D-NEXT:    move $a1, $fp
+; LA64D-NEXT:    move $a2, $s1
+; LA64D-NEXT:    move $a3, $s2
+; LA64D-NEXT:    move $a4, $zero
+; LA64D-NEXT:    move $a5, $zero
+; LA64D-NEXT:    bl %plt(__atomic_compare_exchange)
+; LA64D-NEXT:    fld.d $fa0, $sp, 8
+; LA64D-NEXT:    beqz $a0, .LBB37_1
+; LA64D-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64D-NEXT:    fld.d $fs0, $sp, 16 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s2, $sp, 24 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s1, $sp, 32 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s0, $sp, 40 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
+; LA64D-NEXT:    addi.d $sp, $sp, 64
+; LA64D-NEXT:    ret
+  %v = atomicrmw fsub ptr %p, double 1.0 monotonic, align 4
+  ret double %v
+}
+
+define double @double_fmin_monotonic(ptr %p) nounwind {
+; LA64F-LABEL: double_fmin_monotonic:
+; LA64F:       # %bb.0:
+; LA64F-NEXT:    addi.d $sp, $sp, -64
+; LA64F-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s0, $sp, 40 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s1, $sp, 32 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s2, $sp, 24 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s3, $sp, 16 # 8-byte Folded Spill
+; LA64F-NEXT:    move $fp, $a0
+; LA64F-NEXT:    ld.d $a0, $a0, 0
+; LA64F-NEXT:    lu52i.d $s0, $zero, 1023
+; LA64F-NEXT:    ori $s1, $zero, 8
+; LA64F-NEXT:    addi.d $s2, $sp, 8
+; LA64F-NEXT:    addi.d $s3, $sp, 0
+; LA64F-NEXT:    .p2align 4, , 16
+; LA64F-NEXT:  .LBB38_1: # %atomicrmw.start
+; LA64F-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64F-NEXT:    st.d $a0, $sp, 8
+; LA64F-NEXT:    move $a1, $s0
+; LA64F-NEXT:    bl %plt(fmin)
+; LA64F-NEXT:    st.d $a0, $sp, 0
+; LA64F-NEXT:    move $a0, $s1
+; LA64F-NEXT:    move $a1, $fp
+; LA64F-NEXT:    move $a2, $s2
+; LA64F-NEXT:    move $a3, $s3
+; LA64F-NEXT:    move $a4, $zero
+; LA64F-NEXT:    move $a5, $zero
+; LA64F-NEXT:    bl %plt(__atomic_compare_exchange)
+; LA64F-NEXT:    move $a1, $a0
+; LA64F-NEXT:    ld.d $a0, $sp, 8
+; LA64F-NEXT:    beqz $a1, .LBB38_1
+; LA64F-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64F-NEXT:    ld.d $s3, $sp, 16 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s2, $sp, 24 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s1, $sp, 32 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s0, $sp, 40 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
+; LA64F-NEXT:    addi.d $sp, $sp, 64
+; LA64F-NEXT:    ret
+;
+; LA64D-LABEL: double_fmin_monotonic:
+; LA64D:       # %bb.0:
+; LA64D-NEXT:    addi.d $sp, $sp, -64
+; LA64D-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s0, $sp, 40 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s1, $sp, 32 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s2, $sp, 24 # 8-byte Folded Spill
+; LA64D-NEXT:    fst.d $fs0, $sp, 16 # 8-byte Folded Spill
+; LA64D-NEXT:    move $fp, $a0
+; LA64D-NEXT:    fld.d $fa0, $a0, 0
+; LA64D-NEXT:    addi.d $a0, $zero, 1
+; LA64D-NEXT:    movgr2fr.d $fa1, $a0
+; LA64D-NEXT:    ffint.d.l $fs0, $fa1
+; LA64D-NEXT:    ori $s0, $zero, 8
+; LA64D-NEXT:    addi.d $s1, $sp, 8
+; LA64D-NEXT:    addi.d $s2, $sp, 0
+; LA64D-NEXT:    .p2align 4, , 16
+; LA64D-NEXT:  .LBB38_1: # %atomicrmw.start
+; LA64D-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64D-NEXT:    fst.d $fa0, $sp, 8
+; LA64D-NEXT:    fmax.d $fa0, $fa0, $fa0
+; LA64D-NEXT:    fmin.d $fa0, $fa0, $fs0
+; LA64D-NEXT:    fst.d $fa0, $sp, 0
+; LA64D-NEXT:    move $a0, $s0
+; LA64D-NEXT:    move $a1, $fp
+; LA64D-NEXT:    move $a2, $s1
+; LA64D-NEXT:    move $a3, $s2
+; LA64D-NEXT:    move $a4, $zero
+; LA64D-NEXT:    move $a5, $zero
+; LA64D-NEXT:    bl %plt(__atomic_compare_exchange)
+; LA64D-NEXT:    fld.d $fa0, $sp, 8
+; LA64D-NEXT:    beqz $a0, .LBB38_1
+; LA64D-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64D-NEXT:    fld.d $fs0, $sp, 16 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s2, $sp, 24 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s1, $sp, 32 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s0, $sp, 40 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
+; LA64D-NEXT:    addi.d $sp, $sp, 64
+; LA64D-NEXT:    ret
+  %v = atomicrmw fmin ptr %p, double 1.0 monotonic, align 4
+  ret double %v
+}
+
+define double @double_fmax_monotonic(ptr %p) nounwind {
+; LA64F-LABEL: double_fmax_monotonic:
+; LA64F:       # %bb.0:
+; LA64F-NEXT:    addi.d $sp, $sp, -64
+; LA64F-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s0, $sp, 40 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s1, $sp, 32 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s2, $sp, 24 # 8-byte Folded Spill
+; LA64F-NEXT:    st.d $s3, $sp, 16 # 8-byte Folded Spill
+; LA64F-NEXT:    move $fp, $a0
+; LA64F-NEXT:    ld.d $a0, $a0, 0
+; LA64F-NEXT:    lu52i.d $s0, $zero, 1023
+; LA64F-NEXT:    ori $s1, $zero, 8
+; LA64F-NEXT:    addi.d $s2, $sp, 8
+; LA64F-NEXT:    addi.d $s3, $sp, 0
+; LA64F-NEXT:    .p2align 4, , 16
+; LA64F-NEXT:  .LBB39_1: # %atomicrmw.start
+; LA64F-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64F-NEXT:    st.d $a0, $sp, 8
+; LA64F-NEXT:    move $a1, $s0
+; LA64F-NEXT:    bl %plt(fmax)
+; LA64F-NEXT:    st.d $a0, $sp, 0
+; LA64F-NEXT:    move $a0, $s1
+; LA64F-NEXT:    move $a1, $fp
+; LA64F-NEXT:    move $a2, $s2
+; LA64F-NEXT:    move $a3, $s3
+; LA64F-NEXT:    move $a4, $zero
+; LA64F-NEXT:    move $a5, $zero
+; LA64F-NEXT:    bl %plt(__atomic_compare_exchange)
+; LA64F-NEXT:    move $a1, $a0
+; LA64F-NEXT:    ld.d $a0, $sp, 8
+; LA64F-NEXT:    beqz $a1, .LBB39_1
+; LA64F-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64F-NEXT:    ld.d $s3, $sp, 16 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s2, $sp, 24 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s1, $sp, 32 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $s0, $sp, 40 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
+; LA64F-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
+; LA64F-NEXT:    addi.d $sp, $sp, 64
+; LA64F-NEXT:    ret
+;
+; LA64D-LABEL: double_fmax_monotonic:
+; LA64D:       # %bb.0:
+; LA64D-NEXT:    addi.d $sp, $sp, -64
+; LA64D-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s0, $sp, 40 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s1, $sp, 32 # 8-byte Folded Spill
+; LA64D-NEXT:    st.d $s2, $sp, 24 # 8-byte Folded Spill
+; LA64D-NEXT:    fst.d $fs0, $sp, 16 # 8-byte Folded Spill
+; LA64D-NEXT:    move $fp, $a0
+; LA64D-NEXT:    fld.d $fa0, $a0, 0
+; LA64D-NEXT:    addi.d $a0, $zero, 1
+; LA64D-NEXT:    movgr2fr.d $fa1, $a0
+; LA64D-NEXT:    ffint.d.l $fs0, $fa1
+; LA64D-NEXT:    ori $s0, $zero, 8
+; LA64D-NEXT:    addi.d $s1, $sp, 8
+; LA64D-NEXT:    addi.d $s2, $sp, 0
+; LA64D-NEXT:    .p2align 4, , 16
+; LA64D-NEXT:  .LBB39_1: # %atomicrmw.start
+; LA64D-NEXT:    # =>This Inner Loop Header: Depth=1
+; LA64D-NEXT:    fst.d $fa0, $sp, 8
+; LA64D-NEXT:    fmax.d $fa0, $fa0, $fa0
+; LA64D-NEXT:    fmax.d $fa0, $fa0, $fs0
+; LA64D-NEXT:    fst.d $fa0, $sp, 0
+; LA64D-NEXT:    move $a0, $s0
+; LA64D-NEXT:    move $a1, $fp
+; LA64D-NEXT:    move $a2, $s1
+; LA64D-NEXT:    move $a3, $s2
+; LA64D-NEXT:    move $a4, $zero
+; LA64D-NEXT:    move $a5, $zero
+; LA64D-NEXT:    bl %plt(__atomic_compare_exchange)
+; LA64D-NEXT:    fld.d $fa0, $sp, 8
+; LA64D-NEXT:    beqz $a0, .LBB39_1
+; LA64D-NEXT:  # %bb.2: # %atomicrmw.end
+; LA64D-NEXT:    fld.d $fs0, $sp, 16 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s2, $sp, 24 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s1, $sp, 32 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $s0, $sp, 40 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
+; LA64D-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
+; LA64D-NEXT:    addi.d $sp, $sp, 64
+; LA64D-NEXT:    ret
+  %v = atomicrmw fmax ptr %p, double 1.0 monotonic, align 4
+  ret double %v
+}
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw-minmax.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw-minmax.ll
index d1fb9ede9a..464c9ce97c 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw-minmax.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw-minmax.ll
@@ -16,7 +16,6 @@ define i8 @atomicrmw_umax_i8_acquire(ptr %a, i8 %b) nounwind {
 ; LA64-NEXT:    addi.w $a1, $a1, 0
 ; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
 ; LA64-NEXT:  .LBB0_1: # =>This Inner Loop Header: Depth=1
-; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    ll.w $a4, $a0, 0
 ; LA64-NEXT:    and $a6, $a4, $a3
 ; LA64-NEXT:    move $a5, $a4
@@ -29,8 +28,6 @@ define i8 @atomicrmw_umax_i8_acquire(ptr %a, i8 %b) nounwind {
 ; LA64-NEXT:    sc.w $a5, $a0, 0
 ; LA64-NEXT:    beqz $a5, .LBB0_1
 ; LA64-NEXT:  # %bb.4:
-; LA64-NEXT:    dbar 1792
-; LA64-NEXT:  # %bb.5:
 ; LA64-NEXT:    srl.w $a0, $a4, $a2
 ; LA64-NEXT:    ret
   %1 = atomicrmw umax ptr %a, i8 %b acquire
@@ -50,7 +47,6 @@ define i16 @atomicrmw_umax_i16_acquire(ptr %a, i16 %b) nounwind {
 ; LA64-NEXT:    addi.w $a1, $a1, 0
 ; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
 ; LA64-NEXT:  .LBB1_1: # =>This Inner Loop Header: Depth=1
-; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    ll.w $a4, $a0, 0
 ; LA64-NEXT:    and $a6, $a4, $a2
 ; LA64-NEXT:    move $a5, $a4
@@ -63,8 +59,6 @@ define i16 @atomicrmw_umax_i16_acquire(ptr %a, i16 %b) nounwind {
 ; LA64-NEXT:    sc.w $a5, $a0, 0
 ; LA64-NEXT:    beqz $a5, .LBB1_1
 ; LA64-NEXT:  # %bb.4:
-; LA64-NEXT:    dbar 1792
-; LA64-NEXT:  # %bb.5:
 ; LA64-NEXT:    srl.w $a0, $a4, $a3
 ; LA64-NEXT:    ret
   %1 = atomicrmw umax ptr %a, i16 %b acquire
@@ -103,7 +97,6 @@ define i8 @atomicrmw_umin_i8_acquire(ptr %a, i8 %b) nounwind {
 ; LA64-NEXT:    addi.w $a1, $a1, 0
 ; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
 ; LA64-NEXT:  .LBB4_1: # =>This Inner Loop Header: Depth=1
-; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    ll.w $a4, $a0, 0
 ; LA64-NEXT:    and $a6, $a4, $a3
 ; LA64-NEXT:    move $a5, $a4
@@ -116,8 +109,6 @@ define i8 @atomicrmw_umin_i8_acquire(ptr %a, i8 %b) nounwind {
 ; LA64-NEXT:    sc.w $a5, $a0, 0
 ; LA64-NEXT:    beqz $a5, .LBB4_1
 ; LA64-NEXT:  # %bb.4:
-; LA64-NEXT:    dbar 1792
-; LA64-NEXT:  # %bb.5:
 ; LA64-NEXT:    srl.w $a0, $a4, $a2
 ; LA64-NEXT:    ret
   %1 = atomicrmw umin ptr %a, i8 %b acquire
@@ -137,7 +128,6 @@ define i16 @atomicrmw_umin_i16_acquire(ptr %a, i16 %b) nounwind {
 ; LA64-NEXT:    addi.w $a1, $a1, 0
 ; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
 ; LA64-NEXT:  .LBB5_1: # =>This Inner Loop Header: Depth=1
-; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    ll.w $a4, $a0, 0
 ; LA64-NEXT:    and $a6, $a4, $a2
 ; LA64-NEXT:    move $a5, $a4
@@ -150,8 +140,6 @@ define i16 @atomicrmw_umin_i16_acquire(ptr %a, i16 %b) nounwind {
 ; LA64-NEXT:    sc.w $a5, $a0, 0
 ; LA64-NEXT:    beqz $a5, .LBB5_1
 ; LA64-NEXT:  # %bb.4:
-; LA64-NEXT:    dbar 1792
-; LA64-NEXT:  # %bb.5:
 ; LA64-NEXT:    srl.w $a0, $a4, $a3
 ; LA64-NEXT:    ret
   %1 = atomicrmw umin ptr %a, i16 %b acquire
@@ -192,7 +180,6 @@ define i8 @atomicrmw_max_i8_acquire(ptr %a, i8 %b) nounwind {
 ; LA64-NEXT:    andi $a4, $a2, 24
 ; LA64-NEXT:    xori $a4, $a4, 56
 ; LA64-NEXT:  .LBB8_1: # =>This Inner Loop Header: Depth=1
-; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    ll.w $a5, $a0, 0
 ; LA64-NEXT:    and $a7, $a5, $a3
 ; LA64-NEXT:    move $a6, $a5
@@ -207,8 +194,6 @@ define i8 @atomicrmw_max_i8_acquire(ptr %a, i8 %b) nounwind {
 ; LA64-NEXT:    sc.w $a6, $a0, 0
 ; LA64-NEXT:    beqz $a6, .LBB8_1
 ; LA64-NEXT:  # %bb.4:
-; LA64-NEXT:    dbar 1792
-; LA64-NEXT:  # %bb.5:
 ; LA64-NEXT:    srl.w $a0, $a5, $a2
 ; LA64-NEXT:    ret
   %1 = atomicrmw max ptr %a, i8 %b acquire
@@ -231,7 +216,6 @@ define i16 @atomicrmw_max_i16_acquire(ptr %a, i16 %b) nounwind {
 ; LA64-NEXT:    addi.w $a1, $a1, 0
 ; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
 ; LA64-NEXT:  .LBB9_1: # =>This Inner Loop Header: Depth=1
-; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    ll.w $a5, $a0, 0
 ; LA64-NEXT:    and $a7, $a5, $a4
 ; LA64-NEXT:    move $a6, $a5
@@ -246,8 +230,6 @@ define i16 @atomicrmw_max_i16_acquire(ptr %a, i16 %b) nounwind {
 ; LA64-NEXT:    sc.w $a6, $a0, 0
 ; LA64-NEXT:    beqz $a6, .LBB9_1
 ; LA64-NEXT:  # %bb.4:
-; LA64-NEXT:    dbar 1792
-; LA64-NEXT:  # %bb.5:
 ; LA64-NEXT:    srl.w $a0, $a5, $a2
 ; LA64-NEXT:    ret
   %1 = atomicrmw max ptr %a, i16 %b acquire
@@ -288,7 +270,6 @@ define i8 @atomicrmw_min_i8_acquire(ptr %a, i8 %b) nounwind {
 ; LA64-NEXT:    andi $a4, $a2, 24
 ; LA64-NEXT:    xori $a4, $a4, 56
 ; LA64-NEXT:  .LBB12_1: # =>This Inner Loop Header: Depth=1
-; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    ll.w $a5, $a0, 0
 ; LA64-NEXT:    and $a7, $a5, $a3
 ; LA64-NEXT:    move $a6, $a5
@@ -303,8 +284,6 @@ define i8 @atomicrmw_min_i8_acquire(ptr %a, i8 %b) nounwind {
 ; LA64-NEXT:    sc.w $a6, $a0, 0
 ; LA64-NEXT:    beqz $a6, .LBB12_1
 ; LA64-NEXT:  # %bb.4:
-; LA64-NEXT:    dbar 1792
-; LA64-NEXT:  # %bb.5:
 ; LA64-NEXT:    srl.w $a0, $a5, $a2
 ; LA64-NEXT:    ret
   %1 = atomicrmw min ptr %a, i8 %b acquire
@@ -327,7 +306,6 @@ define i16 @atomicrmw_min_i16_acquire(ptr %a, i16 %b) nounwind {
 ; LA64-NEXT:    addi.w $a1, $a1, 0
 ; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
 ; LA64-NEXT:  .LBB13_1: # =>This Inner Loop Header: Depth=1
-; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    ll.w $a5, $a0, 0
 ; LA64-NEXT:    and $a7, $a5, $a4
 ; LA64-NEXT:    move $a6, $a5
@@ -342,8 +320,6 @@ define i16 @atomicrmw_min_i16_acquire(ptr %a, i16 %b) nounwind {
 ; LA64-NEXT:    sc.w $a6, $a0, 0
 ; LA64-NEXT:    beqz $a6, .LBB13_1
 ; LA64-NEXT:  # %bb.4:
-; LA64-NEXT:    dbar 1792
-; LA64-NEXT:  # %bb.5:
 ; LA64-NEXT:    srl.w $a0, $a5, $a2
 ; LA64-NEXT:    ret
   %1 = atomicrmw min ptr %a, i16 %b acquire
@@ -369,3 +345,1371 @@ define i64 @atomicrmw_min_i64_acquire(ptr %a, i64 %b) nounwind {
   %1 = atomicrmw min ptr %a, i64 %b acquire
   ret i64 %1
 }
+
+define i8 @atomicrmw_umax_i8_release(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_umax_i8_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    addi.w $a3, $a3, 0
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB16_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a6, $a4, $a3
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a6, $a1, .LBB16_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB16_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB16_3: # in Loop: Header=BB16_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB16_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw umax ptr %a, i8 %b release
+  ret i8 %1
+}
+
+define i16 @atomicrmw_umax_i16_release(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_umax_i16_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    lu12i.w $a2, 15
+; LA64-NEXT:    ori $a2, $a2, 4095
+; LA64-NEXT:    slli.d $a3, $a0, 3
+; LA64-NEXT:    sll.w $a2, $a2, $a3
+; LA64-NEXT:    addi.w $a2, $a2, 0
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a3
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB17_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a6, $a4, $a2
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a6, $a1, .LBB17_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB17_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a2
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB17_3: # in Loop: Header=BB17_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB17_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a3
+; LA64-NEXT:    ret
+  %1 = atomicrmw umax ptr %a, i16 %b release
+  ret i16 %1
+}
+
+define i32 @atomicrmw_umax_i32_release(ptr %a, i32 %b) nounwind {
+; LA64-LABEL: atomicrmw_umax_i32_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ammax_db.wu $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw umax ptr %a, i32 %b release
+  ret i32 %1
+}
+
+define i64 @atomicrmw_umax_i64_release(ptr %a, i64 %b) nounwind {
+; LA64-LABEL: atomicrmw_umax_i64_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ammax_db.du $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw umax ptr %a, i64 %b release
+  ret i64 %1
+}
+
+define i8 @atomicrmw_umin_i8_release(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_umin_i8_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    addi.w $a3, $a3, 0
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB20_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a6, $a4, $a3
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a1, $a6, .LBB20_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB20_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB20_3: # in Loop: Header=BB20_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB20_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw umin ptr %a, i8 %b release
+  ret i8 %1
+}
+
+define i16 @atomicrmw_umin_i16_release(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_umin_i16_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    lu12i.w $a2, 15
+; LA64-NEXT:    ori $a2, $a2, 4095
+; LA64-NEXT:    slli.d $a3, $a0, 3
+; LA64-NEXT:    sll.w $a2, $a2, $a3
+; LA64-NEXT:    addi.w $a2, $a2, 0
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a3
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB21_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a6, $a4, $a2
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a1, $a6, .LBB21_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB21_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a2
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB21_3: # in Loop: Header=BB21_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB21_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a3
+; LA64-NEXT:    ret
+  %1 = atomicrmw umin ptr %a, i16 %b release
+  ret i16 %1
+}
+
+define i32 @atomicrmw_umin_i32_release(ptr %a, i32 %b) nounwind {
+; LA64-LABEL: atomicrmw_umin_i32_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ammin_db.wu $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw umin ptr %a, i32 %b release
+  ret i32 %1
+}
+
+define i64 @atomicrmw_umin_i64_release(ptr %a, i64 %b) nounwind {
+; LA64-LABEL: atomicrmw_umin_i64_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ammin_db.du $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw umin ptr %a, i64 %b release
+  ret i64 %1
+}
+
+define i8 @atomicrmw_max_i8_release(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_max_i8_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    addi.w $a3, $a3, 0
+; LA64-NEXT:    ext.w.b $a1, $a1
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a4, $a2, 24
+; LA64-NEXT:    xori $a4, $a4, 56
+; LA64-NEXT:  .LBB24_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a7, $a5, $a3
+; LA64-NEXT:    move $a6, $a5
+; LA64-NEXT:    sll.w $a7, $a7, $a4
+; LA64-NEXT:    sra.w $a7, $a7, $a4
+; LA64-NEXT:    bge $a7, $a1, .LBB24_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB24_1 Depth=1
+; LA64-NEXT:    xor $a6, $a5, $a1
+; LA64-NEXT:    and $a6, $a6, $a3
+; LA64-NEXT:    xor $a6, $a5, $a6
+; LA64-NEXT:  .LBB24_3: # in Loop: Header=BB24_1 Depth=1
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB24_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a5, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw max ptr %a, i8 %b release
+  ret i8 %1
+}
+
+define i16 @atomicrmw_max_i16_release(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_max_i16_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    andi $a3, $a2, 24
+; LA64-NEXT:    ori $a4, $zero, 48
+; LA64-NEXT:    sub.d $a3, $a4, $a3
+; LA64-NEXT:    lu12i.w $a4, 15
+; LA64-NEXT:    ori $a4, $a4, 4095
+; LA64-NEXT:    sll.w $a4, $a4, $a2
+; LA64-NEXT:    addi.w $a4, $a4, 0
+; LA64-NEXT:    ext.w.h $a1, $a1
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB25_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a7, $a5, $a4
+; LA64-NEXT:    move $a6, $a5
+; LA64-NEXT:    sll.w $a7, $a7, $a3
+; LA64-NEXT:    sra.w $a7, $a7, $a3
+; LA64-NEXT:    bge $a7, $a1, .LBB25_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB25_1 Depth=1
+; LA64-NEXT:    xor $a6, $a5, $a1
+; LA64-NEXT:    and $a6, $a6, $a4
+; LA64-NEXT:    xor $a6, $a5, $a6
+; LA64-NEXT:  .LBB25_3: # in Loop: Header=BB25_1 Depth=1
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB25_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a5, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw max ptr %a, i16 %b release
+  ret i16 %1
+}
+
+define i32 @atomicrmw_max_i32_release(ptr %a, i32 %b) nounwind {
+; LA64-LABEL: atomicrmw_max_i32_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ammax_db.w $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw max ptr %a, i32 %b release
+  ret i32 %1
+}
+
+define i64 @atomicrmw_max_i64_release(ptr %a, i64 %b) nounwind {
+; LA64-LABEL: atomicrmw_max_i64_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ammax_db.d $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw max ptr %a, i64 %b release
+  ret i64 %1
+}
+
+define i8 @atomicrmw_min_i8_release(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_min_i8_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    addi.w $a3, $a3, 0
+; LA64-NEXT:    ext.w.b $a1, $a1
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a4, $a2, 24
+; LA64-NEXT:    xori $a4, $a4, 56
+; LA64-NEXT:  .LBB28_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a7, $a5, $a3
+; LA64-NEXT:    move $a6, $a5
+; LA64-NEXT:    sll.w $a7, $a7, $a4
+; LA64-NEXT:    sra.w $a7, $a7, $a4
+; LA64-NEXT:    bge $a1, $a7, .LBB28_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB28_1 Depth=1
+; LA64-NEXT:    xor $a6, $a5, $a1
+; LA64-NEXT:    and $a6, $a6, $a3
+; LA64-NEXT:    xor $a6, $a5, $a6
+; LA64-NEXT:  .LBB28_3: # in Loop: Header=BB28_1 Depth=1
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB28_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a5, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw min ptr %a, i8 %b release
+  ret i8 %1
+}
+
+define i16 @atomicrmw_min_i16_release(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_min_i16_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    andi $a3, $a2, 24
+; LA64-NEXT:    ori $a4, $zero, 48
+; LA64-NEXT:    sub.d $a3, $a4, $a3
+; LA64-NEXT:    lu12i.w $a4, 15
+; LA64-NEXT:    ori $a4, $a4, 4095
+; LA64-NEXT:    sll.w $a4, $a4, $a2
+; LA64-NEXT:    addi.w $a4, $a4, 0
+; LA64-NEXT:    ext.w.h $a1, $a1
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB29_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a7, $a5, $a4
+; LA64-NEXT:    move $a6, $a5
+; LA64-NEXT:    sll.w $a7, $a7, $a3
+; LA64-NEXT:    sra.w $a7, $a7, $a3
+; LA64-NEXT:    bge $a1, $a7, .LBB29_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB29_1 Depth=1
+; LA64-NEXT:    xor $a6, $a5, $a1
+; LA64-NEXT:    and $a6, $a6, $a4
+; LA64-NEXT:    xor $a6, $a5, $a6
+; LA64-NEXT:  .LBB29_3: # in Loop: Header=BB29_1 Depth=1
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB29_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a5, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw min ptr %a, i16 %b release
+  ret i16 %1
+}
+
+define i32 @atomicrmw_min_i32_release(ptr %a, i32 %b) nounwind {
+; LA64-LABEL: atomicrmw_min_i32_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ammin_db.w $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw min ptr %a, i32 %b release
+  ret i32 %1
+}
+
+define i64 @atomicrmw_min_i64_release(ptr %a, i64 %b) nounwind {
+; LA64-LABEL: atomicrmw_min_i64_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ammin_db.d $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw min ptr %a, i64 %b release
+  ret i64 %1
+}
+
+define i8 @atomicrmw_umax_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_umax_i8_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    addi.w $a3, $a3, 0
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB32_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a6, $a4, $a3
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a6, $a1, .LBB32_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB32_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB32_3: # in Loop: Header=BB32_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB32_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw umax ptr %a, i8 %b acq_rel
+  ret i8 %1
+}
+
+define i16 @atomicrmw_umax_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_umax_i16_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    lu12i.w $a2, 15
+; LA64-NEXT:    ori $a2, $a2, 4095
+; LA64-NEXT:    slli.d $a3, $a0, 3
+; LA64-NEXT:    sll.w $a2, $a2, $a3
+; LA64-NEXT:    addi.w $a2, $a2, 0
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a3
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB33_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a6, $a4, $a2
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a6, $a1, .LBB33_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB33_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a2
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB33_3: # in Loop: Header=BB33_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB33_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a3
+; LA64-NEXT:    ret
+  %1 = atomicrmw umax ptr %a, i16 %b acq_rel
+  ret i16 %1
+}
+
+define i32 @atomicrmw_umax_i32_acq_rel(ptr %a, i32 %b) nounwind {
+; LA64-LABEL: atomicrmw_umax_i32_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ammax_db.wu $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw umax ptr %a, i32 %b acq_rel
+  ret i32 %1
+}
+
+define i64 @atomicrmw_umax_i64_acq_rel(ptr %a, i64 %b) nounwind {
+; LA64-LABEL: atomicrmw_umax_i64_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ammax_db.du $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw umax ptr %a, i64 %b acq_rel
+  ret i64 %1
+}
+
+define i8 @atomicrmw_umin_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_umin_i8_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    addi.w $a3, $a3, 0
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB36_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a6, $a4, $a3
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a1, $a6, .LBB36_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB36_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB36_3: # in Loop: Header=BB36_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB36_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw umin ptr %a, i8 %b acq_rel
+  ret i8 %1
+}
+
+define i16 @atomicrmw_umin_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_umin_i16_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    lu12i.w $a2, 15
+; LA64-NEXT:    ori $a2, $a2, 4095
+; LA64-NEXT:    slli.d $a3, $a0, 3
+; LA64-NEXT:    sll.w $a2, $a2, $a3
+; LA64-NEXT:    addi.w $a2, $a2, 0
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a3
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB37_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a6, $a4, $a2
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a1, $a6, .LBB37_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB37_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a2
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB37_3: # in Loop: Header=BB37_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB37_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a3
+; LA64-NEXT:    ret
+  %1 = atomicrmw umin ptr %a, i16 %b acq_rel
+  ret i16 %1
+}
+
+define i32 @atomicrmw_umin_i32_acq_rel(ptr %a, i32 %b) nounwind {
+; LA64-LABEL: atomicrmw_umin_i32_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ammin_db.wu $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw umin ptr %a, i32 %b acq_rel
+  ret i32 %1
+}
+
+define i64 @atomicrmw_umin_i64_acq_rel(ptr %a, i64 %b) nounwind {
+; LA64-LABEL: atomicrmw_umin_i64_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ammin_db.du $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw umin ptr %a, i64 %b acq_rel
+  ret i64 %1
+}
+
+define i8 @atomicrmw_max_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_max_i8_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    addi.w $a3, $a3, 0
+; LA64-NEXT:    ext.w.b $a1, $a1
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a4, $a2, 24
+; LA64-NEXT:    xori $a4, $a4, 56
+; LA64-NEXT:  .LBB40_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a7, $a5, $a3
+; LA64-NEXT:    move $a6, $a5
+; LA64-NEXT:    sll.w $a7, $a7, $a4
+; LA64-NEXT:    sra.w $a7, $a7, $a4
+; LA64-NEXT:    bge $a7, $a1, .LBB40_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB40_1 Depth=1
+; LA64-NEXT:    xor $a6, $a5, $a1
+; LA64-NEXT:    and $a6, $a6, $a3
+; LA64-NEXT:    xor $a6, $a5, $a6
+; LA64-NEXT:  .LBB40_3: # in Loop: Header=BB40_1 Depth=1
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB40_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a5, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw max ptr %a, i8 %b acq_rel
+  ret i8 %1
+}
+
+define i16 @atomicrmw_max_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_max_i16_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    andi $a3, $a2, 24
+; LA64-NEXT:    ori $a4, $zero, 48
+; LA64-NEXT:    sub.d $a3, $a4, $a3
+; LA64-NEXT:    lu12i.w $a4, 15
+; LA64-NEXT:    ori $a4, $a4, 4095
+; LA64-NEXT:    sll.w $a4, $a4, $a2
+; LA64-NEXT:    addi.w $a4, $a4, 0
+; LA64-NEXT:    ext.w.h $a1, $a1
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB41_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a7, $a5, $a4
+; LA64-NEXT:    move $a6, $a5
+; LA64-NEXT:    sll.w $a7, $a7, $a3
+; LA64-NEXT:    sra.w $a7, $a7, $a3
+; LA64-NEXT:    bge $a7, $a1, .LBB41_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB41_1 Depth=1
+; LA64-NEXT:    xor $a6, $a5, $a1
+; LA64-NEXT:    and $a6, $a6, $a4
+; LA64-NEXT:    xor $a6, $a5, $a6
+; LA64-NEXT:  .LBB41_3: # in Loop: Header=BB41_1 Depth=1
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB41_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a5, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw max ptr %a, i16 %b acq_rel
+  ret i16 %1
+}
+
+define i32 @atomicrmw_max_i32_acq_rel(ptr %a, i32 %b) nounwind {
+; LA64-LABEL: atomicrmw_max_i32_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ammax_db.w $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw max ptr %a, i32 %b acq_rel
+  ret i32 %1
+}
+
+define i64 @atomicrmw_max_i64_acq_rel(ptr %a, i64 %b) nounwind {
+; LA64-LABEL: atomicrmw_max_i64_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ammax_db.d $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw max ptr %a, i64 %b acq_rel
+  ret i64 %1
+}
+
+define i8 @atomicrmw_min_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_min_i8_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    addi.w $a3, $a3, 0
+; LA64-NEXT:    ext.w.b $a1, $a1
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a4, $a2, 24
+; LA64-NEXT:    xori $a4, $a4, 56
+; LA64-NEXT:  .LBB44_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a7, $a5, $a3
+; LA64-NEXT:    move $a6, $a5
+; LA64-NEXT:    sll.w $a7, $a7, $a4
+; LA64-NEXT:    sra.w $a7, $a7, $a4
+; LA64-NEXT:    bge $a1, $a7, .LBB44_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB44_1 Depth=1
+; LA64-NEXT:    xor $a6, $a5, $a1
+; LA64-NEXT:    and $a6, $a6, $a3
+; LA64-NEXT:    xor $a6, $a5, $a6
+; LA64-NEXT:  .LBB44_3: # in Loop: Header=BB44_1 Depth=1
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB44_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a5, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw min ptr %a, i8 %b acq_rel
+  ret i8 %1
+}
+
+define i16 @atomicrmw_min_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_min_i16_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    andi $a3, $a2, 24
+; LA64-NEXT:    ori $a4, $zero, 48
+; LA64-NEXT:    sub.d $a3, $a4, $a3
+; LA64-NEXT:    lu12i.w $a4, 15
+; LA64-NEXT:    ori $a4, $a4, 4095
+; LA64-NEXT:    sll.w $a4, $a4, $a2
+; LA64-NEXT:    addi.w $a4, $a4, 0
+; LA64-NEXT:    ext.w.h $a1, $a1
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB45_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a7, $a5, $a4
+; LA64-NEXT:    move $a6, $a5
+; LA64-NEXT:    sll.w $a7, $a7, $a3
+; LA64-NEXT:    sra.w $a7, $a7, $a3
+; LA64-NEXT:    bge $a1, $a7, .LBB45_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB45_1 Depth=1
+; LA64-NEXT:    xor $a6, $a5, $a1
+; LA64-NEXT:    and $a6, $a6, $a4
+; LA64-NEXT:    xor $a6, $a5, $a6
+; LA64-NEXT:  .LBB45_3: # in Loop: Header=BB45_1 Depth=1
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB45_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a5, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw min ptr %a, i16 %b acq_rel
+  ret i16 %1
+}
+
+define i32 @atomicrmw_min_i32_acq_rel(ptr %a, i32 %b) nounwind {
+; LA64-LABEL: atomicrmw_min_i32_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ammin_db.w $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw min ptr %a, i32 %b acq_rel
+  ret i32 %1
+}
+
+define i64 @atomicrmw_min_i64_acq_rel(ptr %a, i64 %b) nounwind {
+; LA64-LABEL: atomicrmw_min_i64_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ammin_db.d $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw min ptr %a, i64 %b acq_rel
+  ret i64 %1
+}
+
+define i8 @atomicrmw_umax_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_umax_i8_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    addi.w $a3, $a3, 0
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB48_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a6, $a4, $a3
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a6, $a1, .LBB48_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB48_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB48_3: # in Loop: Header=BB48_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB48_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw umax ptr %a, i8 %b seq_cst
+  ret i8 %1
+}
+
+define i16 @atomicrmw_umax_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_umax_i16_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    lu12i.w $a2, 15
+; LA64-NEXT:    ori $a2, $a2, 4095
+; LA64-NEXT:    slli.d $a3, $a0, 3
+; LA64-NEXT:    sll.w $a2, $a2, $a3
+; LA64-NEXT:    addi.w $a2, $a2, 0
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a3
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB49_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a6, $a4, $a2
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a6, $a1, .LBB49_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB49_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a2
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB49_3: # in Loop: Header=BB49_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB49_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a3
+; LA64-NEXT:    ret
+  %1 = atomicrmw umax ptr %a, i16 %b seq_cst
+  ret i16 %1
+}
+
+define i32 @atomicrmw_umax_i32_seq_cst(ptr %a, i32 %b) nounwind {
+; LA64-LABEL: atomicrmw_umax_i32_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ammax_db.wu $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw umax ptr %a, i32 %b seq_cst
+  ret i32 %1
+}
+
+define i64 @atomicrmw_umax_i64_seq_cst(ptr %a, i64 %b) nounwind {
+; LA64-LABEL: atomicrmw_umax_i64_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ammax_db.du $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw umax ptr %a, i64 %b seq_cst
+  ret i64 %1
+}
+
+define i8 @atomicrmw_umin_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_umin_i8_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    addi.w $a3, $a3, 0
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB52_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a6, $a4, $a3
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a1, $a6, .LBB52_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB52_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB52_3: # in Loop: Header=BB52_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB52_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw umin ptr %a, i8 %b seq_cst
+  ret i8 %1
+}
+
+define i16 @atomicrmw_umin_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_umin_i16_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    lu12i.w $a2, 15
+; LA64-NEXT:    ori $a2, $a2, 4095
+; LA64-NEXT:    slli.d $a3, $a0, 3
+; LA64-NEXT:    sll.w $a2, $a2, $a3
+; LA64-NEXT:    addi.w $a2, $a2, 0
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a3
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB53_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a6, $a4, $a2
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a1, $a6, .LBB53_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB53_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a2
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB53_3: # in Loop: Header=BB53_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB53_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a3
+; LA64-NEXT:    ret
+  %1 = atomicrmw umin ptr %a, i16 %b seq_cst
+  ret i16 %1
+}
+
+define i32 @atomicrmw_umin_i32_seq_cst(ptr %a, i32 %b) nounwind {
+; LA64-LABEL: atomicrmw_umin_i32_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ammin_db.wu $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw umin ptr %a, i32 %b seq_cst
+  ret i32 %1
+}
+
+define i64 @atomicrmw_umin_i64_seq_cst(ptr %a, i64 %b) nounwind {
+; LA64-LABEL: atomicrmw_umin_i64_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ammin_db.du $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw umin ptr %a, i64 %b seq_cst
+  ret i64 %1
+}
+
+define i8 @atomicrmw_max_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_max_i8_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    addi.w $a3, $a3, 0
+; LA64-NEXT:    ext.w.b $a1, $a1
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a4, $a2, 24
+; LA64-NEXT:    xori $a4, $a4, 56
+; LA64-NEXT:  .LBB56_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a7, $a5, $a3
+; LA64-NEXT:    move $a6, $a5
+; LA64-NEXT:    sll.w $a7, $a7, $a4
+; LA64-NEXT:    sra.w $a7, $a7, $a4
+; LA64-NEXT:    bge $a7, $a1, .LBB56_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB56_1 Depth=1
+; LA64-NEXT:    xor $a6, $a5, $a1
+; LA64-NEXT:    and $a6, $a6, $a3
+; LA64-NEXT:    xor $a6, $a5, $a6
+; LA64-NEXT:  .LBB56_3: # in Loop: Header=BB56_1 Depth=1
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB56_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a5, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw max ptr %a, i8 %b seq_cst
+  ret i8 %1
+}
+
+define i16 @atomicrmw_max_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_max_i16_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    andi $a3, $a2, 24
+; LA64-NEXT:    ori $a4, $zero, 48
+; LA64-NEXT:    sub.d $a3, $a4, $a3
+; LA64-NEXT:    lu12i.w $a4, 15
+; LA64-NEXT:    ori $a4, $a4, 4095
+; LA64-NEXT:    sll.w $a4, $a4, $a2
+; LA64-NEXT:    addi.w $a4, $a4, 0
+; LA64-NEXT:    ext.w.h $a1, $a1
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB57_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a7, $a5, $a4
+; LA64-NEXT:    move $a6, $a5
+; LA64-NEXT:    sll.w $a7, $a7, $a3
+; LA64-NEXT:    sra.w $a7, $a7, $a3
+; LA64-NEXT:    bge $a7, $a1, .LBB57_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB57_1 Depth=1
+; LA64-NEXT:    xor $a6, $a5, $a1
+; LA64-NEXT:    and $a6, $a6, $a4
+; LA64-NEXT:    xor $a6, $a5, $a6
+; LA64-NEXT:  .LBB57_3: # in Loop: Header=BB57_1 Depth=1
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB57_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a5, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw max ptr %a, i16 %b seq_cst
+  ret i16 %1
+}
+
+define i32 @atomicrmw_max_i32_seq_cst(ptr %a, i32 %b) nounwind {
+; LA64-LABEL: atomicrmw_max_i32_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ammax_db.w $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw max ptr %a, i32 %b seq_cst
+  ret i32 %1
+}
+
+define i64 @atomicrmw_max_i64_seq_cst(ptr %a, i64 %b) nounwind {
+; LA64-LABEL: atomicrmw_max_i64_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ammax_db.d $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw max ptr %a, i64 %b seq_cst
+  ret i64 %1
+}
+
+define i8 @atomicrmw_min_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_min_i8_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    addi.w $a3, $a3, 0
+; LA64-NEXT:    ext.w.b $a1, $a1
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a4, $a2, 24
+; LA64-NEXT:    xori $a4, $a4, 56
+; LA64-NEXT:  .LBB60_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a7, $a5, $a3
+; LA64-NEXT:    move $a6, $a5
+; LA64-NEXT:    sll.w $a7, $a7, $a4
+; LA64-NEXT:    sra.w $a7, $a7, $a4
+; LA64-NEXT:    bge $a1, $a7, .LBB60_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB60_1 Depth=1
+; LA64-NEXT:    xor $a6, $a5, $a1
+; LA64-NEXT:    and $a6, $a6, $a3
+; LA64-NEXT:    xor $a6, $a5, $a6
+; LA64-NEXT:  .LBB60_3: # in Loop: Header=BB60_1 Depth=1
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB60_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a5, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw min ptr %a, i8 %b seq_cst
+  ret i8 %1
+}
+
+define i16 @atomicrmw_min_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_min_i16_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    andi $a3, $a2, 24
+; LA64-NEXT:    ori $a4, $zero, 48
+; LA64-NEXT:    sub.d $a3, $a4, $a3
+; LA64-NEXT:    lu12i.w $a4, 15
+; LA64-NEXT:    ori $a4, $a4, 4095
+; LA64-NEXT:    sll.w $a4, $a4, $a2
+; LA64-NEXT:    addi.w $a4, $a4, 0
+; LA64-NEXT:    ext.w.h $a1, $a1
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB61_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a7, $a5, $a4
+; LA64-NEXT:    move $a6, $a5
+; LA64-NEXT:    sll.w $a7, $a7, $a3
+; LA64-NEXT:    sra.w $a7, $a7, $a3
+; LA64-NEXT:    bge $a1, $a7, .LBB61_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB61_1 Depth=1
+; LA64-NEXT:    xor $a6, $a5, $a1
+; LA64-NEXT:    and $a6, $a6, $a4
+; LA64-NEXT:    xor $a6, $a5, $a6
+; LA64-NEXT:  .LBB61_3: # in Loop: Header=BB61_1 Depth=1
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB61_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a5, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw min ptr %a, i16 %b seq_cst
+  ret i16 %1
+}
+
+define i32 @atomicrmw_min_i32_seq_cst(ptr %a, i32 %b) nounwind {
+; LA64-LABEL: atomicrmw_min_i32_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ammin_db.w $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw min ptr %a, i32 %b seq_cst
+  ret i32 %1
+}
+
+define i64 @atomicrmw_min_i64_seq_cst(ptr %a, i64 %b) nounwind {
+; LA64-LABEL: atomicrmw_min_i64_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ammin_db.d $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw min ptr %a, i64 %b seq_cst
+  ret i64 %1
+}
+
+define i8 @atomicrmw_umax_i8_monotonic(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_umax_i8_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    addi.w $a3, $a3, 0
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB64_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a6, $a4, $a3
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a6, $a1, .LBB64_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB64_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB64_3: # in Loop: Header=BB64_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB64_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw umax ptr %a, i8 %b monotonic
+  ret i8 %1
+}
+
+define i16 @atomicrmw_umax_i16_monotonic(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_umax_i16_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    lu12i.w $a2, 15
+; LA64-NEXT:    ori $a2, $a2, 4095
+; LA64-NEXT:    slli.d $a3, $a0, 3
+; LA64-NEXT:    sll.w $a2, $a2, $a3
+; LA64-NEXT:    addi.w $a2, $a2, 0
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a3
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB65_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a6, $a4, $a2
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a6, $a1, .LBB65_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB65_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a2
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB65_3: # in Loop: Header=BB65_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB65_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a3
+; LA64-NEXT:    ret
+  %1 = atomicrmw umax ptr %a, i16 %b monotonic
+  ret i16 %1
+}
+
+define i32 @atomicrmw_umax_i32_monotonic(ptr %a, i32 %b) nounwind {
+; LA64-LABEL: atomicrmw_umax_i32_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ammax_db.wu $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw umax ptr %a, i32 %b monotonic
+  ret i32 %1
+}
+
+define i64 @atomicrmw_umax_i64_monotonic(ptr %a, i64 %b) nounwind {
+; LA64-LABEL: atomicrmw_umax_i64_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ammax_db.du $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw umax ptr %a, i64 %b monotonic
+  ret i64 %1
+}
+
+define i8 @atomicrmw_umin_i8_monotonic(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_umin_i8_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    addi.w $a3, $a3, 0
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB68_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a6, $a4, $a3
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a1, $a6, .LBB68_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB68_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB68_3: # in Loop: Header=BB68_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB68_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw umin ptr %a, i8 %b monotonic
+  ret i8 %1
+}
+
+define i16 @atomicrmw_umin_i16_monotonic(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_umin_i16_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    lu12i.w $a2, 15
+; LA64-NEXT:    ori $a2, $a2, 4095
+; LA64-NEXT:    slli.d $a3, $a0, 3
+; LA64-NEXT:    sll.w $a2, $a2, $a3
+; LA64-NEXT:    addi.w $a2, $a2, 0
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a3
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB69_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a6, $a4, $a2
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a1, $a6, .LBB69_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB69_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a2
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB69_3: # in Loop: Header=BB69_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB69_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a3
+; LA64-NEXT:    ret
+  %1 = atomicrmw umin ptr %a, i16 %b monotonic
+  ret i16 %1
+}
+
+define i32 @atomicrmw_umin_i32_monotonic(ptr %a, i32 %b) nounwind {
+; LA64-LABEL: atomicrmw_umin_i32_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ammin_db.wu $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw umin ptr %a, i32 %b monotonic
+  ret i32 %1
+}
+
+define i64 @atomicrmw_umin_i64_monotonic(ptr %a, i64 %b) nounwind {
+; LA64-LABEL: atomicrmw_umin_i64_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ammin_db.du $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw umin ptr %a, i64 %b monotonic
+  ret i64 %1
+}
+
+define i8 @atomicrmw_max_i8_monotonic(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_max_i8_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    addi.w $a3, $a3, 0
+; LA64-NEXT:    ext.w.b $a1, $a1
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a4, $a2, 24
+; LA64-NEXT:    xori $a4, $a4, 56
+; LA64-NEXT:  .LBB72_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a7, $a5, $a3
+; LA64-NEXT:    move $a6, $a5
+; LA64-NEXT:    sll.w $a7, $a7, $a4
+; LA64-NEXT:    sra.w $a7, $a7, $a4
+; LA64-NEXT:    bge $a7, $a1, .LBB72_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB72_1 Depth=1
+; LA64-NEXT:    xor $a6, $a5, $a1
+; LA64-NEXT:    and $a6, $a6, $a3
+; LA64-NEXT:    xor $a6, $a5, $a6
+; LA64-NEXT:  .LBB72_3: # in Loop: Header=BB72_1 Depth=1
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB72_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a5, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw max ptr %a, i8 %b monotonic
+  ret i8 %1
+}
+
+define i16 @atomicrmw_max_i16_monotonic(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_max_i16_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    andi $a3, $a2, 24
+; LA64-NEXT:    ori $a4, $zero, 48
+; LA64-NEXT:    sub.d $a3, $a4, $a3
+; LA64-NEXT:    lu12i.w $a4, 15
+; LA64-NEXT:    ori $a4, $a4, 4095
+; LA64-NEXT:    sll.w $a4, $a4, $a2
+; LA64-NEXT:    addi.w $a4, $a4, 0
+; LA64-NEXT:    ext.w.h $a1, $a1
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB73_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a7, $a5, $a4
+; LA64-NEXT:    move $a6, $a5
+; LA64-NEXT:    sll.w $a7, $a7, $a3
+; LA64-NEXT:    sra.w $a7, $a7, $a3
+; LA64-NEXT:    bge $a7, $a1, .LBB73_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB73_1 Depth=1
+; LA64-NEXT:    xor $a6, $a5, $a1
+; LA64-NEXT:    and $a6, $a6, $a4
+; LA64-NEXT:    xor $a6, $a5, $a6
+; LA64-NEXT:  .LBB73_3: # in Loop: Header=BB73_1 Depth=1
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB73_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a5, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw max ptr %a, i16 %b monotonic
+  ret i16 %1
+}
+
+define i32 @atomicrmw_max_i32_monotonic(ptr %a, i32 %b) nounwind {
+; LA64-LABEL: atomicrmw_max_i32_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ammax_db.w $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw max ptr %a, i32 %b monotonic
+  ret i32 %1
+}
+
+define i64 @atomicrmw_max_i64_monotonic(ptr %a, i64 %b) nounwind {
+; LA64-LABEL: atomicrmw_max_i64_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ammax_db.d $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw max ptr %a, i64 %b monotonic
+  ret i64 %1
+}
+
+define i8 @atomicrmw_min_i8_monotonic(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_min_i8_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    addi.w $a3, $a3, 0
+; LA64-NEXT:    ext.w.b $a1, $a1
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a4, $a2, 24
+; LA64-NEXT:    xori $a4, $a4, 56
+; LA64-NEXT:  .LBB76_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a7, $a5, $a3
+; LA64-NEXT:    move $a6, $a5
+; LA64-NEXT:    sll.w $a7, $a7, $a4
+; LA64-NEXT:    sra.w $a7, $a7, $a4
+; LA64-NEXT:    bge $a1, $a7, .LBB76_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB76_1 Depth=1
+; LA64-NEXT:    xor $a6, $a5, $a1
+; LA64-NEXT:    and $a6, $a6, $a3
+; LA64-NEXT:    xor $a6, $a5, $a6
+; LA64-NEXT:  .LBB76_3: # in Loop: Header=BB76_1 Depth=1
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB76_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a5, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw min ptr %a, i8 %b monotonic
+  ret i8 %1
+}
+
+define i16 @atomicrmw_min_i16_monotonic(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_min_i16_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    andi $a3, $a2, 24
+; LA64-NEXT:    ori $a4, $zero, 48
+; LA64-NEXT:    sub.d $a3, $a4, $a3
+; LA64-NEXT:    lu12i.w $a4, 15
+; LA64-NEXT:    ori $a4, $a4, 4095
+; LA64-NEXT:    sll.w $a4, $a4, $a2
+; LA64-NEXT:    addi.w $a4, $a4, 0
+; LA64-NEXT:    ext.w.h $a1, $a1
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB77_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a5, $a0, 0
+; LA64-NEXT:    and $a7, $a5, $a4
+; LA64-NEXT:    move $a6, $a5
+; LA64-NEXT:    sll.w $a7, $a7, $a3
+; LA64-NEXT:    sra.w $a7, $a7, $a3
+; LA64-NEXT:    bge $a1, $a7, .LBB77_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB77_1 Depth=1
+; LA64-NEXT:    xor $a6, $a5, $a1
+; LA64-NEXT:    and $a6, $a6, $a4
+; LA64-NEXT:    xor $a6, $a5, $a6
+; LA64-NEXT:  .LBB77_3: # in Loop: Header=BB77_1 Depth=1
+; LA64-NEXT:    sc.w $a6, $a0, 0
+; LA64-NEXT:    beqz $a6, .LBB77_1
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a5, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw min ptr %a, i16 %b monotonic
+  ret i16 %1
+}
+
+define i32 @atomicrmw_min_i32_monotonic(ptr %a, i32 %b) nounwind {
+; LA64-LABEL: atomicrmw_min_i32_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ammin_db.w $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw min ptr %a, i32 %b monotonic
+  ret i32 %1
+}
+
+define i64 @atomicrmw_min_i64_monotonic(ptr %a, i64 %b) nounwind {
+; LA64-LABEL: atomicrmw_min_i64_monotonic:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ammin_db.d $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw min ptr %a, i64 %b monotonic
+  ret i64 %1
+}
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw.ll
index 8bab095fba..d4f7ed0171 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw.ll
@@ -12,7 +12,6 @@ define i8 @atomicrmw_xchg_i8_acquire(ptr %a, i8 %b) nounwind {
 ; LA32-NEXT:    sll.w $a1, $a1, $a2
 ; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
 ; LA32-NEXT:  .LBB0_1: # =>This Inner Loop Header: Depth=1
-; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a4, $a0, 0
 ; LA32-NEXT:    addi.w $a5, $a1, 0
 ; LA32-NEXT:    xor $a5, $a4, $a5
@@ -35,7 +34,6 @@ define i8 @atomicrmw_xchg_i8_acquire(ptr %a, i8 %b) nounwind {
 ; LA64-NEXT:    addi.w $a1, $a1, 0
 ; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
 ; LA64-NEXT:  .LBB0_1: # =>This Inner Loop Header: Depth=1
-; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    ll.w $a4, $a0, 0
 ; LA64-NEXT:    addi.w $a5, $a1, 0
 ; LA64-NEXT:    xor $a5, $a4, $a5
@@ -59,7 +57,6 @@ define i8 @atomicrmw_xchg_0_i8_acquire(ptr %a) nounwind {
 ; LA32-NEXT:    nor $a2, $a2, $zero
 ; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
 ; LA32-NEXT:  .LBB1_1: # =>This Inner Loop Header: Depth=1
-; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a3, $a0, 0
 ; LA32-NEXT:    and $a4, $a3, $a2
 ; LA32-NEXT:    sc.w $a4, $a0, 0
@@ -90,7 +87,6 @@ define i8 @atomicrmw_xchg_minus_1_i8_acquire(ptr %a) nounwind {
 ; LA32-NEXT:    sll.w $a2, $a2, $a1
 ; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
 ; LA32-NEXT:  .LBB2_1: # =>This Inner Loop Header: Depth=1
-; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a3, $a0, 0
 ; LA32-NEXT:    or $a4, $a3, $a2
 ; LA32-NEXT:    sc.w $a4, $a0, 0
@@ -123,7 +119,6 @@ define i16 @atomicrmw_xchg_i16_acquire(ptr %a, i16 %b) nounwind {
 ; LA32-NEXT:    sll.w $a1, $a1, $a3
 ; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
 ; LA32-NEXT:  .LBB3_1: # =>This Inner Loop Header: Depth=1
-; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a4, $a0, 0
 ; LA32-NEXT:    addi.w $a5, $a1, 0
 ; LA32-NEXT:    xor $a5, $a4, $a5
@@ -147,7 +142,6 @@ define i16 @atomicrmw_xchg_i16_acquire(ptr %a, i16 %b) nounwind {
 ; LA64-NEXT:    addi.w $a1, $a1, 0
 ; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
 ; LA64-NEXT:  .LBB3_1: # =>This Inner Loop Header: Depth=1
-; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    ll.w $a4, $a0, 0
 ; LA64-NEXT:    addi.w $a5, $a1, 0
 ; LA64-NEXT:    xor $a5, $a4, $a5
@@ -172,7 +166,6 @@ define i16 @atomicrmw_xchg_0_i16_acquire(ptr %a) nounwind {
 ; LA32-NEXT:    nor $a1, $a1, $zero
 ; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
 ; LA32-NEXT:  .LBB4_1: # =>This Inner Loop Header: Depth=1
-; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a3, $a0, 0
 ; LA32-NEXT:    and $a4, $a3, $a1
 ; LA32-NEXT:    sc.w $a4, $a0, 0
@@ -205,7 +198,6 @@ define i16 @atomicrmw_xchg_minus_1_i16_acquire(ptr %a) nounwind {
 ; LA32-NEXT:    sll.w $a1, $a1, $a2
 ; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
 ; LA32-NEXT:  .LBB5_1: # =>This Inner Loop Header: Depth=1
-; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a3, $a0, 0
 ; LA32-NEXT:    or $a4, $a3, $a1
 ; LA32-NEXT:    sc.w $a4, $a0, 0
@@ -232,7 +224,6 @@ define i32 @atomicrmw_xchg_i32_acquire(ptr %a, i32 %b) nounwind {
 ; LA32-LABEL: atomicrmw_xchg_i32_acquire:
 ; LA32:       # %bb.0:
 ; LA32-NEXT:  .LBB6_1: # =>This Inner Loop Header: Depth=1
-; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a2, $a0, 0
 ; LA32-NEXT:    move $a3, $a1
 ; LA32-NEXT:    sc.w $a3, $a0, 0
@@ -280,7 +271,6 @@ define i8 @atomicrmw_add_i8_acquire(ptr %a, i8 %b) nounwind {
 ; LA32-NEXT:    sll.w $a1, $a1, $a2
 ; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
 ; LA32-NEXT:  .LBB8_1: # =>This Inner Loop Header: Depth=1
-; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a4, $a0, 0
 ; LA32-NEXT:    add.w $a5, $a4, $a1
 ; LA32-NEXT:    xor $a5, $a4, $a5
@@ -303,7 +293,6 @@ define i8 @atomicrmw_add_i8_acquire(ptr %a, i8 %b) nounwind {
 ; LA64-NEXT:    addi.w $a1, $a1, 0
 ; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
 ; LA64-NEXT:  .LBB8_1: # =>This Inner Loop Header: Depth=1
-; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    ll.w $a4, $a0, 0
 ; LA64-NEXT:    add.w $a5, $a4, $a1
 ; LA64-NEXT:    xor $a5, $a4, $a5
@@ -329,7 +318,6 @@ define i16 @atomicrmw_add_i16_acquire(ptr %a, i16 %b) nounwind {
 ; LA32-NEXT:    sll.w $a1, $a1, $a3
 ; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
 ; LA32-NEXT:  .LBB9_1: # =>This Inner Loop Header: Depth=1
-; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a4, $a0, 0
 ; LA32-NEXT:    add.w $a5, $a4, $a1
 ; LA32-NEXT:    xor $a5, $a4, $a5
@@ -353,7 +341,6 @@ define i16 @atomicrmw_add_i16_acquire(ptr %a, i16 %b) nounwind {
 ; LA64-NEXT:    addi.w $a1, $a1, 0
 ; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
 ; LA64-NEXT:  .LBB9_1: # =>This Inner Loop Header: Depth=1
-; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    ll.w $a4, $a0, 0
 ; LA64-NEXT:    add.w $a5, $a4, $a1
 ; LA64-NEXT:    xor $a5, $a4, $a5
@@ -372,7 +359,6 @@ define i32 @atomicrmw_add_i32_acquire(ptr %a, i32 %b) nounwind {
 ; LA32-LABEL: atomicrmw_add_i32_acquire:
 ; LA32:       # %bb.0:
 ; LA32-NEXT:  .LBB10_1: # =>This Inner Loop Header: Depth=1
-; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a2, $a0, 0
 ; LA32-NEXT:    add.w $a3, $a2, $a1
 ; LA32-NEXT:    sc.w $a3, $a0, 0
@@ -420,7 +406,6 @@ define i8 @atomicrmw_sub_i8_acquire(ptr %a, i8 %b) nounwind {
 ; LA32-NEXT:    sll.w $a1, $a1, $a2
 ; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
 ; LA32-NEXT:  .LBB12_1: # =>This Inner Loop Header: Depth=1
-; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a4, $a0, 0
 ; LA32-NEXT:    sub.w $a5, $a4, $a1
 ; LA32-NEXT:    xor $a5, $a4, $a5
@@ -443,7 +428,6 @@ define i8 @atomicrmw_sub_i8_acquire(ptr %a, i8 %b) nounwind {
 ; LA64-NEXT:    addi.w $a1, $a1, 0
 ; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
 ; LA64-NEXT:  .LBB12_1: # =>This Inner Loop Header: Depth=1
-; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    ll.w $a4, $a0, 0
 ; LA64-NEXT:    sub.w $a5, $a4, $a1
 ; LA64-NEXT:    xor $a5, $a4, $a5
@@ -469,7 +453,6 @@ define i16 @atomicrmw_sub_i16_acquire(ptr %a, i16 %b) nounwind {
 ; LA32-NEXT:    sll.w $a1, $a1, $a3
 ; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
 ; LA32-NEXT:  .LBB13_1: # =>This Inner Loop Header: Depth=1
-; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a4, $a0, 0
 ; LA32-NEXT:    sub.w $a5, $a4, $a1
 ; LA32-NEXT:    xor $a5, $a4, $a5
@@ -493,7 +476,6 @@ define i16 @atomicrmw_sub_i16_acquire(ptr %a, i16 %b) nounwind {
 ; LA64-NEXT:    addi.w $a1, $a1, 0
 ; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
 ; LA64-NEXT:  .LBB13_1: # =>This Inner Loop Header: Depth=1
-; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    ll.w $a4, $a0, 0
 ; LA64-NEXT:    sub.w $a5, $a4, $a1
 ; LA64-NEXT:    xor $a5, $a4, $a5
@@ -512,7 +494,6 @@ define i32 @atomicrmw_sub_i32_acquire(ptr %a, i32 %b) nounwind {
 ; LA32-LABEL: atomicrmw_sub_i32_acquire:
 ; LA32:       # %bb.0:
 ; LA32-NEXT:  .LBB14_1: # =>This Inner Loop Header: Depth=1
-; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a2, $a0, 0
 ; LA32-NEXT:    sub.w $a3, $a2, $a1
 ; LA32-NEXT:    sc.w $a3, $a0, 0
@@ -562,7 +543,6 @@ define i8 @atomicrmw_nand_i8_acquire(ptr %a, i8 %b) nounwind {
 ; LA32-NEXT:    sll.w $a1, $a1, $a2
 ; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
 ; LA32-NEXT:  .LBB16_1: # =>This Inner Loop Header: Depth=1
-; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a4, $a0, 0
 ; LA32-NEXT:    and $a5, $a4, $a1
 ; LA32-NEXT:    nor $a5, $a5, $zero
@@ -586,7 +566,6 @@ define i8 @atomicrmw_nand_i8_acquire(ptr %a, i8 %b) nounwind {
 ; LA64-NEXT:    addi.w $a1, $a1, 0
 ; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
 ; LA64-NEXT:  .LBB16_1: # =>This Inner Loop Header: Depth=1
-; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    ll.w $a4, $a0, 0
 ; LA64-NEXT:    and $a5, $a4, $a1
 ; LA64-NEXT:    nor $a5, $a5, $zero
@@ -613,7 +592,6 @@ define i16 @atomicrmw_nand_i16_acquire(ptr %a, i16 %b) nounwind {
 ; LA32-NEXT:    sll.w $a1, $a1, $a3
 ; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
 ; LA32-NEXT:  .LBB17_1: # =>This Inner Loop Header: Depth=1
-; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a4, $a0, 0
 ; LA32-NEXT:    and $a5, $a4, $a1
 ; LA32-NEXT:    nor $a5, $a5, $zero
@@ -638,7 +616,6 @@ define i16 @atomicrmw_nand_i16_acquire(ptr %a, i16 %b) nounwind {
 ; LA64-NEXT:    addi.w $a1, $a1, 0
 ; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
 ; LA64-NEXT:  .LBB17_1: # =>This Inner Loop Header: Depth=1
-; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    ll.w $a4, $a0, 0
 ; LA64-NEXT:    and $a5, $a4, $a1
 ; LA64-NEXT:    nor $a5, $a5, $zero
@@ -658,7 +635,6 @@ define i32 @atomicrmw_nand_i32_acquire(ptr %a, i32 %b) nounwind {
 ; LA32-LABEL: atomicrmw_nand_i32_acquire:
 ; LA32:       # %bb.0:
 ; LA32-NEXT:  .LBB18_1: # =>This Inner Loop Header: Depth=1
-; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a2, $a0, 0
 ; LA32-NEXT:    and $a3, $a2, $a1
 ; LA32-NEXT:    nor $a3, $a3, $zero
@@ -671,7 +647,6 @@ define i32 @atomicrmw_nand_i32_acquire(ptr %a, i32 %b) nounwind {
 ; LA64-LABEL: atomicrmw_nand_i32_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:  .LBB18_1: # =>This Inner Loop Header: Depth=1
-; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    ll.w $a2, $a0, 0
 ; LA64-NEXT:    and $a3, $a2, $a1
 ; LA64-NEXT:    nor $a3, $a3, $zero
@@ -698,7 +673,6 @@ define i64 @atomicrmw_nand_i64_acquire(ptr %a, i64 %b) nounwind {
 ; LA64-LABEL: atomicrmw_nand_i64_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:  .LBB19_1: # =>This Inner Loop Header: Depth=1
-; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    ll.d $a2, $a0, 0
 ; LA64-NEXT:    and $a3, $a2, $a1
 ; LA64-NEXT:    nor $a3, $a3, $zero
@@ -722,7 +696,6 @@ define i8 @atomicrmw_and_i8_acquire(ptr %a, i8 %b) nounwind {
 ; LA32-NEXT:    orn $a1, $a1, $a3
 ; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
 ; LA32-NEXT:  .LBB20_1: # =>This Inner Loop Header: Depth=1
-; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a3, $a0, 0
 ; LA32-NEXT:    and $a4, $a3, $a1
 ; LA32-NEXT:    sc.w $a4, $a0, 0
@@ -759,7 +732,6 @@ define i16 @atomicrmw_and_i16_acquire(ptr %a, i16 %b) nounwind {
 ; LA32-NEXT:    orn $a1, $a1, $a2
 ; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
 ; LA32-NEXT:  .LBB21_1: # =>This Inner Loop Header: Depth=1
-; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a2, $a0, 0
 ; LA32-NEXT:    and $a4, $a2, $a1
 ; LA32-NEXT:    sc.w $a4, $a0, 0
@@ -789,7 +761,6 @@ define i32 @atomicrmw_and_i32_acquire(ptr %a, i32 %b) nounwind {
 ; LA32-LABEL: atomicrmw_and_i32_acquire:
 ; LA32:       # %bb.0:
 ; LA32-NEXT:  .LBB22_1: # =>This Inner Loop Header: Depth=1
-; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a2, $a0, 0
 ; LA32-NEXT:    and $a3, $a2, $a1
 ; LA32-NEXT:    sc.w $a3, $a0, 0
@@ -835,7 +806,6 @@ define i8 @atomicrmw_or_i8_acquire(ptr %a, i8 %b) nounwind {
 ; LA32-NEXT:    sll.w $a1, $a1, $a2
 ; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
 ; LA32-NEXT:  .LBB24_1: # =>This Inner Loop Header: Depth=1
-; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a3, $a0, 0
 ; LA32-NEXT:    or $a4, $a3, $a1
 ; LA32-NEXT:    sc.w $a4, $a0, 0
@@ -865,7 +835,6 @@ define i16 @atomicrmw_or_i16_acquire(ptr %a, i16 %b) nounwind {
 ; LA32-NEXT:    sll.w $a1, $a1, $a2
 ; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
 ; LA32-NEXT:  .LBB25_1: # =>This Inner Loop Header: Depth=1
-; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a3, $a0, 0
 ; LA32-NEXT:    or $a4, $a3, $a1
 ; LA32-NEXT:    sc.w $a4, $a0, 0
@@ -891,7 +860,6 @@ define i32 @atomicrmw_or_i32_acquire(ptr %a, i32 %b) nounwind {
 ; LA32-LABEL: atomicrmw_or_i32_acquire:
 ; LA32:       # %bb.0:
 ; LA32-NEXT:  .LBB26_1: # =>This Inner Loop Header: Depth=1
-; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a2, $a0, 0
 ; LA32-NEXT:    or $a3, $a2, $a1
 ; LA32-NEXT:    sc.w $a3, $a0, 0
@@ -937,7 +905,6 @@ define i8 @atomicrmw_xor_i8_acquire(ptr %a, i8 %b) nounwind {
 ; LA32-NEXT:    sll.w $a1, $a1, $a2
 ; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
 ; LA32-NEXT:  .LBB28_1: # =>This Inner Loop Header: Depth=1
-; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a3, $a0, 0
 ; LA32-NEXT:    xor $a4, $a3, $a1
 ; LA32-NEXT:    sc.w $a4, $a0, 0
@@ -967,7 +934,6 @@ define i16 @atomicrmw_xor_i16_acquire(ptr %a, i16 %b) nounwind {
 ; LA32-NEXT:    sll.w $a1, $a1, $a2
 ; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
 ; LA32-NEXT:  .LBB29_1: # =>This Inner Loop Header: Depth=1
-; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a3, $a0, 0
 ; LA32-NEXT:    xor $a4, $a3, $a1
 ; LA32-NEXT:    sc.w $a4, $a0, 0
@@ -993,7 +959,6 @@ define i32 @atomicrmw_xor_i32_acquire(ptr %a, i32 %b) nounwind {
 ; LA32-LABEL: atomicrmw_xor_i32_acquire:
 ; LA32:       # %bb.0:
 ; LA32-NEXT:  .LBB30_1: # =>This Inner Loop Header: Depth=1
-; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a2, $a0, 0
 ; LA32-NEXT:    xor $a3, $a2, $a1
 ; LA32-NEXT:    sc.w $a3, $a0, 0
@@ -1031,6 +996,2988 @@ define i64 @atomicrmw_xor_i64_acquire(ptr %a, i64 %b) nounwind {
   ret i64 %1
 }
 
+define i8 @atomicrmw_xchg_i8_release(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_xchg_i8_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:    slli.w $a2, $a0, 3
+; LA32-NEXT:    ori $a3, $zero, 255
+; LA32-NEXT:    sll.w $a3, $a3, $a2
+; LA32-NEXT:    andi $a1, $a1, 255
+; LA32-NEXT:    sll.w $a1, $a1, $a2
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB32_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a4, $a0, 0
+; LA32-NEXT:    addi.w $a5, $a1, 0
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    and $a5, $a5, $a3
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    sc.w $a5, $a0, 0
+; LA32-NEXT:    beqz $a5, .LBB32_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a4, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_xchg_i8_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    addi.w $a3, $a3, 0
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB32_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    addi.w $a5, $a1, 0
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB32_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i8 %b release
+  ret i8 %1
+}
+
+define i8 @atomicrmw_xchg_0_i8_release(ptr %a) nounwind {
+; LA32-LABEL: atomicrmw_xchg_0_i8_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:    slli.w $a1, $a0, 3
+; LA32-NEXT:    ori $a2, $zero, 255
+; LA32-NEXT:    sll.w $a2, $a2, $a1
+; LA32-NEXT:    nor $a2, $a2, $zero
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB33_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a3, $a0, 0
+; LA32-NEXT:    and $a4, $a3, $a2
+; LA32-NEXT:    sc.w $a4, $a0, 0
+; LA32-NEXT:    beqz $a4, .LBB33_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a3, $a1
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_xchg_0_i8_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a1, $a0, 3
+; LA64-NEXT:    ori $a2, $zero, 255
+; LA64-NEXT:    sll.w $a2, $a2, $a1
+; LA64-NEXT:    nor $a2, $a2, $zero
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    amand_db.w $a3, $a2, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a1
+; LA64-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i8 0 release
+  ret i8 %1
+}
+
+define i8 @atomicrmw_xchg_minus_1_i8_release(ptr %a) nounwind {
+; LA32-LABEL: atomicrmw_xchg_minus_1_i8_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:    slli.w $a1, $a0, 3
+; LA32-NEXT:    ori $a2, $zero, 255
+; LA32-NEXT:    sll.w $a2, $a2, $a1
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB34_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a3, $a0, 0
+; LA32-NEXT:    or $a4, $a3, $a2
+; LA32-NEXT:    sc.w $a4, $a0, 0
+; LA32-NEXT:    beqz $a4, .LBB34_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a3, $a1
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_xchg_minus_1_i8_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a1, $a0, 3
+; LA64-NEXT:    ori $a2, $zero, 255
+; LA64-NEXT:    sll.w $a2, $a2, $a1
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    amor_db.w $a3, $a2, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a1
+; LA64-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i8 -1 release
+  ret i8 %1
+}
+
+define i16 @atomicrmw_xchg_i16_release(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_xchg_i16_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:    lu12i.w $a2, 15
+; LA32-NEXT:    ori $a2, $a2, 4095
+; LA32-NEXT:    slli.w $a3, $a0, 3
+; LA32-NEXT:    sll.w $a2, $a2, $a3
+; LA32-NEXT:    bstrpick.w $a1, $a1, 15, 0
+; LA32-NEXT:    sll.w $a1, $a1, $a3
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB35_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a4, $a0, 0
+; LA32-NEXT:    addi.w $a5, $a1, 0
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    and $a5, $a5, $a2
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    sc.w $a5, $a0, 0
+; LA32-NEXT:    beqz $a5, .LBB35_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a4, $a3
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_xchg_i16_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    lu12i.w $a2, 15
+; LA64-NEXT:    ori $a2, $a2, 4095
+; LA64-NEXT:    slli.d $a3, $a0, 3
+; LA64-NEXT:    sll.w $a2, $a2, $a3
+; LA64-NEXT:    addi.w $a2, $a2, 0
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a3
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB35_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    addi.w $a5, $a1, 0
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a2
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB35_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a3
+; LA64-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i16 %b release
+  ret i16 %1
+}
+
+define i16 @atomicrmw_xchg_0_i16_release(ptr %a) nounwind {
+; LA32-LABEL: atomicrmw_xchg_0_i16_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:    lu12i.w $a1, 15
+; LA32-NEXT:    ori $a1, $a1, 4095
+; LA32-NEXT:    slli.w $a2, $a0, 3
+; LA32-NEXT:    sll.w $a1, $a1, $a2
+; LA32-NEXT:    nor $a1, $a1, $zero
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB36_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a3, $a0, 0
+; LA32-NEXT:    and $a4, $a3, $a1
+; LA32-NEXT:    sc.w $a4, $a0, 0
+; LA32-NEXT:    beqz $a4, .LBB36_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a3, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_xchg_0_i16_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    lu12i.w $a1, 15
+; LA64-NEXT:    ori $a1, $a1, 4095
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    nor $a1, $a1, $zero
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    amand_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i16 0 release
+  ret i16 %1
+}
+
+define i16 @atomicrmw_xchg_minus_1_i16_release(ptr %a) nounwind {
+; LA32-LABEL: atomicrmw_xchg_minus_1_i16_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:    lu12i.w $a1, 15
+; LA32-NEXT:    ori $a1, $a1, 4095
+; LA32-NEXT:    slli.w $a2, $a0, 3
+; LA32-NEXT:    sll.w $a1, $a1, $a2
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB37_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a3, $a0, 0
+; LA32-NEXT:    or $a4, $a3, $a1
+; LA32-NEXT:    sc.w $a4, $a0, 0
+; LA32-NEXT:    beqz $a4, .LBB37_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a3, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_xchg_minus_1_i16_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    lu12i.w $a1, 15
+; LA64-NEXT:    ori $a1, $a1, 4095
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    amor_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i16 -1 release
+  ret i16 %1
+}
+
+define i32 @atomicrmw_xchg_i32_release(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_xchg_i32_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB38_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    move $a3, $a1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beqz $a3, .LBB38_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_xchg_i32_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    amswap_db.w $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i32 %b release
+  ret i32 %1
+}
+
+define i64 @atomicrmw_xchg_i64_release(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_xchg_i64_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    ori $a3, $zero, 3
+; LA32-NEXT:    bl %plt(__atomic_exchange_8)
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_xchg_i64_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    amswap_db.d $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i64 %b release
+  ret i64 %1
+}
+
+define i8 @atomicrmw_add_i8_release(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_add_i8_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:    slli.w $a2, $a0, 3
+; LA32-NEXT:    ori $a3, $zero, 255
+; LA32-NEXT:    sll.w $a3, $a3, $a2
+; LA32-NEXT:    andi $a1, $a1, 255
+; LA32-NEXT:    sll.w $a1, $a1, $a2
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB40_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a4, $a0, 0
+; LA32-NEXT:    add.w $a5, $a4, $a1
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    and $a5, $a5, $a3
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    sc.w $a5, $a0, 0
+; LA32-NEXT:    beqz $a5, .LBB40_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a4, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_add_i8_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    addi.w $a3, $a3, 0
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB40_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    add.w $a5, $a4, $a1
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB40_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw add ptr %a, i8 %b release
+  ret i8 %1
+}
+
+define i16 @atomicrmw_add_i16_release(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_add_i16_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:    lu12i.w $a2, 15
+; LA32-NEXT:    ori $a2, $a2, 4095
+; LA32-NEXT:    slli.w $a3, $a0, 3
+; LA32-NEXT:    sll.w $a2, $a2, $a3
+; LA32-NEXT:    bstrpick.w $a1, $a1, 15, 0
+; LA32-NEXT:    sll.w $a1, $a1, $a3
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB41_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a4, $a0, 0
+; LA32-NEXT:    add.w $a5, $a4, $a1
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    and $a5, $a5, $a2
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    sc.w $a5, $a0, 0
+; LA32-NEXT:    beqz $a5, .LBB41_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a4, $a3
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_add_i16_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    lu12i.w $a2, 15
+; LA64-NEXT:    ori $a2, $a2, 4095
+; LA64-NEXT:    slli.d $a3, $a0, 3
+; LA64-NEXT:    sll.w $a2, $a2, $a3
+; LA64-NEXT:    addi.w $a2, $a2, 0
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a3
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB41_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    add.w $a5, $a4, $a1
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a2
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB41_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a3
+; LA64-NEXT:    ret
+  %1 = atomicrmw add ptr %a, i16 %b release
+  ret i16 %1
+}
+
+define i32 @atomicrmw_add_i32_release(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_add_i32_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB42_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    add.w $a3, $a2, $a1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beqz $a3, .LBB42_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_add_i32_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    amadd_db.w $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw add ptr %a, i32 %b release
+  ret i32 %1
+}
+
+define i64 @atomicrmw_add_i64_release(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_add_i64_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    ori $a3, $zero, 3
+; LA32-NEXT:    bl %plt(__atomic_fetch_add_8)
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_add_i64_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    amadd_db.d $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw add ptr %a, i64 %b release
+  ret i64 %1
+}
+
+define i8 @atomicrmw_sub_i8_release(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_sub_i8_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:    slli.w $a2, $a0, 3
+; LA32-NEXT:    ori $a3, $zero, 255
+; LA32-NEXT:    sll.w $a3, $a3, $a2
+; LA32-NEXT:    andi $a1, $a1, 255
+; LA32-NEXT:    sll.w $a1, $a1, $a2
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB44_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a4, $a0, 0
+; LA32-NEXT:    sub.w $a5, $a4, $a1
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    and $a5, $a5, $a3
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    sc.w $a5, $a0, 0
+; LA32-NEXT:    beqz $a5, .LBB44_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a4, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_sub_i8_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    addi.w $a3, $a3, 0
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB44_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    sub.w $a5, $a4, $a1
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB44_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw sub ptr %a, i8 %b release
+  ret i8 %1
+}
+
+define i16 @atomicrmw_sub_i16_release(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_sub_i16_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:    lu12i.w $a2, 15
+; LA32-NEXT:    ori $a2, $a2, 4095
+; LA32-NEXT:    slli.w $a3, $a0, 3
+; LA32-NEXT:    sll.w $a2, $a2, $a3
+; LA32-NEXT:    bstrpick.w $a1, $a1, 15, 0
+; LA32-NEXT:    sll.w $a1, $a1, $a3
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB45_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a4, $a0, 0
+; LA32-NEXT:    sub.w $a5, $a4, $a1
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    and $a5, $a5, $a2
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    sc.w $a5, $a0, 0
+; LA32-NEXT:    beqz $a5, .LBB45_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a4, $a3
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_sub_i16_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    lu12i.w $a2, 15
+; LA64-NEXT:    ori $a2, $a2, 4095
+; LA64-NEXT:    slli.d $a3, $a0, 3
+; LA64-NEXT:    sll.w $a2, $a2, $a3
+; LA64-NEXT:    addi.w $a2, $a2, 0
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a3
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB45_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    sub.w $a5, $a4, $a1
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a2
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB45_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a3
+; LA64-NEXT:    ret
+  %1 = atomicrmw sub ptr %a, i16 %b release
+  ret i16 %1
+}
+
+define i32 @atomicrmw_sub_i32_release(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_sub_i32_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB46_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    sub.w $a3, $a2, $a1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beqz $a3, .LBB46_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_sub_i32_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    sub.w $a2, $zero, $a1
+; LA64-NEXT:    amadd_db.w $a1, $a2, $a0
+; LA64-NEXT:    move $a0, $a1
+; LA64-NEXT:    ret
+  %1 = atomicrmw sub ptr %a, i32 %b release
+  ret i32 %1
+}
+
+define i64 @atomicrmw_sub_i64_release(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_sub_i64_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    ori $a3, $zero, 3
+; LA32-NEXT:    bl %plt(__atomic_fetch_sub_8)
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_sub_i64_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    sub.d $a2, $zero, $a1
+; LA64-NEXT:    amadd_db.d $a1, $a2, $a0
+; LA64-NEXT:    move $a0, $a1
+; LA64-NEXT:    ret
+  %1 = atomicrmw sub ptr %a, i64 %b release
+  ret i64 %1
+}
+
+define i8 @atomicrmw_nand_i8_release(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_nand_i8_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:    slli.w $a2, $a0, 3
+; LA32-NEXT:    ori $a3, $zero, 255
+; LA32-NEXT:    sll.w $a3, $a3, $a2
+; LA32-NEXT:    andi $a1, $a1, 255
+; LA32-NEXT:    sll.w $a1, $a1, $a2
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB48_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a4, $a0, 0
+; LA32-NEXT:    and $a5, $a4, $a1
+; LA32-NEXT:    nor $a5, $a5, $zero
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    and $a5, $a5, $a3
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    sc.w $a5, $a0, 0
+; LA32-NEXT:    beqz $a5, .LBB48_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a4, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_nand_i8_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    addi.w $a3, $a3, 0
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB48_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a5, $a4, $a1
+; LA64-NEXT:    nor $a5, $a5, $zero
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB48_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw nand ptr %a, i8 %b release
+  ret i8 %1
+}
+
+define i16 @atomicrmw_nand_i16_release(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_nand_i16_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:    lu12i.w $a2, 15
+; LA32-NEXT:    ori $a2, $a2, 4095
+; LA32-NEXT:    slli.w $a3, $a0, 3
+; LA32-NEXT:    sll.w $a2, $a2, $a3
+; LA32-NEXT:    bstrpick.w $a1, $a1, 15, 0
+; LA32-NEXT:    sll.w $a1, $a1, $a3
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB49_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a4, $a0, 0
+; LA32-NEXT:    and $a5, $a4, $a1
+; LA32-NEXT:    nor $a5, $a5, $zero
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    and $a5, $a5, $a2
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    sc.w $a5, $a0, 0
+; LA32-NEXT:    beqz $a5, .LBB49_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a4, $a3
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_nand_i16_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    lu12i.w $a2, 15
+; LA64-NEXT:    ori $a2, $a2, 4095
+; LA64-NEXT:    slli.d $a3, $a0, 3
+; LA64-NEXT:    sll.w $a2, $a2, $a3
+; LA64-NEXT:    addi.w $a2, $a2, 0
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a3
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB49_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a5, $a4, $a1
+; LA64-NEXT:    nor $a5, $a5, $zero
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a2
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB49_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a3
+; LA64-NEXT:    ret
+  %1 = atomicrmw nand ptr %a, i16 %b release
+  ret i16 %1
+}
+
+define i32 @atomicrmw_nand_i32_release(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_nand_i32_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB50_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    and $a3, $a2, $a1
+; LA32-NEXT:    nor $a3, $a3, $zero
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beqz $a3, .LBB50_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_nand_i32_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:  .LBB50_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a2, $a0, 0
+; LA64-NEXT:    and $a3, $a2, $a1
+; LA64-NEXT:    nor $a3, $a3, $zero
+; LA64-NEXT:    sc.w $a3, $a0, 0
+; LA64-NEXT:    beqz $a3, .LBB50_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw nand ptr %a, i32 %b release
+  ret i32 %1
+}
+
+define i64 @atomicrmw_nand_i64_release(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_nand_i64_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    ori $a3, $zero, 3
+; LA32-NEXT:    bl %plt(__atomic_fetch_nand_8)
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_nand_i64_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:  .LBB51_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.d $a2, $a0, 0
+; LA64-NEXT:    and $a3, $a2, $a1
+; LA64-NEXT:    nor $a3, $a3, $zero
+; LA64-NEXT:    sc.d $a3, $a0, 0
+; LA64-NEXT:    beqz $a3, .LBB51_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw nand ptr %a, i64 %b release
+  ret i64 %1
+}
+
+define i8 @atomicrmw_and_i8_release(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_and_i8_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:    slli.w $a2, $a0, 3
+; LA32-NEXT:    ori $a3, $zero, 255
+; LA32-NEXT:    sll.w $a3, $a3, $a2
+; LA32-NEXT:    andi $a1, $a1, 255
+; LA32-NEXT:    sll.w $a1, $a1, $a2
+; LA32-NEXT:    orn $a1, $a1, $a3
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB52_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a3, $a0, 0
+; LA32-NEXT:    and $a4, $a3, $a1
+; LA32-NEXT:    sc.w $a4, $a0, 0
+; LA32-NEXT:    beqz $a4, .LBB52_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a3, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_and_i8_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    orn $a1, $a1, $a3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    amand_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw and ptr %a, i8 %b release
+  ret i8 %1
+}
+
+define i16 @atomicrmw_and_i16_release(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_and_i16_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:    lu12i.w $a2, 15
+; LA32-NEXT:    ori $a2, $a2, 4095
+; LA32-NEXT:    slli.w $a3, $a0, 3
+; LA32-NEXT:    sll.w $a2, $a2, $a3
+; LA32-NEXT:    bstrpick.w $a1, $a1, 15, 0
+; LA32-NEXT:    sll.w $a1, $a1, $a3
+; LA32-NEXT:    orn $a1, $a1, $a2
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB53_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    and $a4, $a2, $a1
+; LA32-NEXT:    sc.w $a4, $a0, 0
+; LA32-NEXT:    beqz $a4, .LBB53_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a2, $a3
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_and_i16_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    lu12i.w $a2, 15
+; LA64-NEXT:    ori $a2, $a2, 4095
+; LA64-NEXT:    slli.d $a3, $a0, 3
+; LA64-NEXT:    sll.w $a2, $a2, $a3
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a3
+; LA64-NEXT:    orn $a1, $a1, $a2
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    amand_db.w $a2, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a2, $a3
+; LA64-NEXT:    ret
+  %1 = atomicrmw and ptr %a, i16 %b release
+  ret i16 %1
+}
+
+define i32 @atomicrmw_and_i32_release(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_and_i32_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB54_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    and $a3, $a2, $a1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beqz $a3, .LBB54_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_and_i32_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    amand_db.w $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw and ptr %a, i32 %b release
+  ret i32 %1
+}
+
+define i64 @atomicrmw_and_i64_release(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_and_i64_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    ori $a3, $zero, 3
+; LA32-NEXT:    bl %plt(__atomic_fetch_and_8)
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_and_i64_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    amand_db.d $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw and ptr %a, i64 %b release
+  ret i64 %1
+}
+
+define i8 @atomicrmw_or_i8_release(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_or_i8_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:    andi $a1, $a1, 255
+; LA32-NEXT:    slli.w $a2, $a0, 3
+; LA32-NEXT:    sll.w $a1, $a1, $a2
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB56_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a3, $a0, 0
+; LA32-NEXT:    or $a4, $a3, $a1
+; LA32-NEXT:    sc.w $a4, $a0, 0
+; LA32-NEXT:    beqz $a4, .LBB56_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a3, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_or_i8_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    amor_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw or ptr %a, i8 %b release
+  ret i8 %1
+}
+
+define i16 @atomicrmw_or_i16_release(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_or_i16_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:    bstrpick.w $a1, $a1, 15, 0
+; LA32-NEXT:    slli.w $a2, $a0, 3
+; LA32-NEXT:    sll.w $a1, $a1, $a2
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB57_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a3, $a0, 0
+; LA32-NEXT:    or $a4, $a3, $a1
+; LA32-NEXT:    sc.w $a4, $a0, 0
+; LA32-NEXT:    beqz $a4, .LBB57_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a3, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_or_i16_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    amor_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw or ptr %a, i16 %b release
+  ret i16 %1
+}
+
+define i32 @atomicrmw_or_i32_release(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_or_i32_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB58_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    or $a3, $a2, $a1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beqz $a3, .LBB58_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_or_i32_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    amor_db.w $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw or ptr %a, i32 %b release
+  ret i32 %1
+}
+
+define i64 @atomicrmw_or_i64_release(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_or_i64_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    ori $a3, $zero, 3
+; LA32-NEXT:    bl %plt(__atomic_fetch_or_8)
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_or_i64_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    amor_db.d $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw or ptr %a, i64 %b release
+  ret i64 %1
+}
+
+define i8 @atomicrmw_xor_i8_release(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_xor_i8_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:    andi $a1, $a1, 255
+; LA32-NEXT:    slli.w $a2, $a0, 3
+; LA32-NEXT:    sll.w $a1, $a1, $a2
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB60_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a3, $a0, 0
+; LA32-NEXT:    xor $a4, $a3, $a1
+; LA32-NEXT:    sc.w $a4, $a0, 0
+; LA32-NEXT:    beqz $a4, .LBB60_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a3, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_xor_i8_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    amxor_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw xor ptr %a, i8 %b release
+  ret i8 %1
+}
+
+define i16 @atomicrmw_xor_i16_release(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_xor_i16_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:    bstrpick.w $a1, $a1, 15, 0
+; LA32-NEXT:    slli.w $a2, $a0, 3
+; LA32-NEXT:    sll.w $a1, $a1, $a2
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB61_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a3, $a0, 0
+; LA32-NEXT:    xor $a4, $a3, $a1
+; LA32-NEXT:    sc.w $a4, $a0, 0
+; LA32-NEXT:    beqz $a4, .LBB61_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a3, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_xor_i16_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    amxor_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw xor ptr %a, i16 %b release
+  ret i16 %1
+}
+
+define i32 @atomicrmw_xor_i32_release(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_xor_i32_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB62_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    xor $a3, $a2, $a1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beqz $a3, .LBB62_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_xor_i32_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    amxor_db.w $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw xor ptr %a, i32 %b release
+  ret i32 %1
+}
+
+define i64 @atomicrmw_xor_i64_release(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_xor_i64_release:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    ori $a3, $zero, 3
+; LA32-NEXT:    bl %plt(__atomic_fetch_xor_8)
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_xor_i64_release:
+; LA64:       # %bb.0:
+; LA64-NEXT:    amxor_db.d $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw xor ptr %a, i64 %b release
+  ret i64 %1
+}
+
+define i8 @atomicrmw_xchg_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_xchg_i8_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:    slli.w $a2, $a0, 3
+; LA32-NEXT:    ori $a3, $zero, 255
+; LA32-NEXT:    sll.w $a3, $a3, $a2
+; LA32-NEXT:    andi $a1, $a1, 255
+; LA32-NEXT:    sll.w $a1, $a1, $a2
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB64_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a4, $a0, 0
+; LA32-NEXT:    addi.w $a5, $a1, 0
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    and $a5, $a5, $a3
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    sc.w $a5, $a0, 0
+; LA32-NEXT:    beqz $a5, .LBB64_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a4, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_xchg_i8_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    addi.w $a3, $a3, 0
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB64_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    addi.w $a5, $a1, 0
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB64_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i8 %b acq_rel
+  ret i8 %1
+}
+
+define i8 @atomicrmw_xchg_0_i8_acq_rel(ptr %a) nounwind {
+; LA32-LABEL: atomicrmw_xchg_0_i8_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:    slli.w $a1, $a0, 3
+; LA32-NEXT:    ori $a2, $zero, 255
+; LA32-NEXT:    sll.w $a2, $a2, $a1
+; LA32-NEXT:    nor $a2, $a2, $zero
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB65_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a3, $a0, 0
+; LA32-NEXT:    and $a4, $a3, $a2
+; LA32-NEXT:    sc.w $a4, $a0, 0
+; LA32-NEXT:    beqz $a4, .LBB65_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a3, $a1
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_xchg_0_i8_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a1, $a0, 3
+; LA64-NEXT:    ori $a2, $zero, 255
+; LA64-NEXT:    sll.w $a2, $a2, $a1
+; LA64-NEXT:    nor $a2, $a2, $zero
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    amand_db.w $a3, $a2, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a1
+; LA64-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i8 0 acq_rel
+  ret i8 %1
+}
+
+define i8 @atomicrmw_xchg_minus_1_i8_acq_rel(ptr %a) nounwind {
+; LA32-LABEL: atomicrmw_xchg_minus_1_i8_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:    slli.w $a1, $a0, 3
+; LA32-NEXT:    ori $a2, $zero, 255
+; LA32-NEXT:    sll.w $a2, $a2, $a1
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB66_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a3, $a0, 0
+; LA32-NEXT:    or $a4, $a3, $a2
+; LA32-NEXT:    sc.w $a4, $a0, 0
+; LA32-NEXT:    beqz $a4, .LBB66_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a3, $a1
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_xchg_minus_1_i8_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a1, $a0, 3
+; LA64-NEXT:    ori $a2, $zero, 255
+; LA64-NEXT:    sll.w $a2, $a2, $a1
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    amor_db.w $a3, $a2, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a1
+; LA64-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i8 -1 acq_rel
+  ret i8 %1
+}
+
+define i16 @atomicrmw_xchg_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_xchg_i16_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:    lu12i.w $a2, 15
+; LA32-NEXT:    ori $a2, $a2, 4095
+; LA32-NEXT:    slli.w $a3, $a0, 3
+; LA32-NEXT:    sll.w $a2, $a2, $a3
+; LA32-NEXT:    bstrpick.w $a1, $a1, 15, 0
+; LA32-NEXT:    sll.w $a1, $a1, $a3
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB67_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a4, $a0, 0
+; LA32-NEXT:    addi.w $a5, $a1, 0
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    and $a5, $a5, $a2
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    sc.w $a5, $a0, 0
+; LA32-NEXT:    beqz $a5, .LBB67_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a4, $a3
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_xchg_i16_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    lu12i.w $a2, 15
+; LA64-NEXT:    ori $a2, $a2, 4095
+; LA64-NEXT:    slli.d $a3, $a0, 3
+; LA64-NEXT:    sll.w $a2, $a2, $a3
+; LA64-NEXT:    addi.w $a2, $a2, 0
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a3
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB67_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    addi.w $a5, $a1, 0
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a2
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB67_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a3
+; LA64-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i16 %b acq_rel
+  ret i16 %1
+}
+
+define i16 @atomicrmw_xchg_0_i16_acq_rel(ptr %a) nounwind {
+; LA32-LABEL: atomicrmw_xchg_0_i16_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:    lu12i.w $a1, 15
+; LA32-NEXT:    ori $a1, $a1, 4095
+; LA32-NEXT:    slli.w $a2, $a0, 3
+; LA32-NEXT:    sll.w $a1, $a1, $a2
+; LA32-NEXT:    nor $a1, $a1, $zero
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB68_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a3, $a0, 0
+; LA32-NEXT:    and $a4, $a3, $a1
+; LA32-NEXT:    sc.w $a4, $a0, 0
+; LA32-NEXT:    beqz $a4, .LBB68_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a3, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_xchg_0_i16_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    lu12i.w $a1, 15
+; LA64-NEXT:    ori $a1, $a1, 4095
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    nor $a1, $a1, $zero
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    amand_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i16 0 acq_rel
+  ret i16 %1
+}
+
+define i16 @atomicrmw_xchg_minus_1_i16_acq_rel(ptr %a) nounwind {
+; LA32-LABEL: atomicrmw_xchg_minus_1_i16_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:    lu12i.w $a1, 15
+; LA32-NEXT:    ori $a1, $a1, 4095
+; LA32-NEXT:    slli.w $a2, $a0, 3
+; LA32-NEXT:    sll.w $a1, $a1, $a2
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB69_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a3, $a0, 0
+; LA32-NEXT:    or $a4, $a3, $a1
+; LA32-NEXT:    sc.w $a4, $a0, 0
+; LA32-NEXT:    beqz $a4, .LBB69_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a3, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_xchg_minus_1_i16_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    lu12i.w $a1, 15
+; LA64-NEXT:    ori $a1, $a1, 4095
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    amor_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i16 -1 acq_rel
+  ret i16 %1
+}
+
+define i32 @atomicrmw_xchg_i32_acq_rel(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_xchg_i32_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB70_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    move $a3, $a1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beqz $a3, .LBB70_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_xchg_i32_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    amswap_db.w $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i32 %b acq_rel
+  ret i32 %1
+}
+
+define i64 @atomicrmw_xchg_i64_acq_rel(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_xchg_i64_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    ori $a3, $zero, 4
+; LA32-NEXT:    bl %plt(__atomic_exchange_8)
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_xchg_i64_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    amswap_db.d $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i64 %b acq_rel
+  ret i64 %1
+}
+
+define i8 @atomicrmw_add_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_add_i8_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:    slli.w $a2, $a0, 3
+; LA32-NEXT:    ori $a3, $zero, 255
+; LA32-NEXT:    sll.w $a3, $a3, $a2
+; LA32-NEXT:    andi $a1, $a1, 255
+; LA32-NEXT:    sll.w $a1, $a1, $a2
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB72_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a4, $a0, 0
+; LA32-NEXT:    add.w $a5, $a4, $a1
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    and $a5, $a5, $a3
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    sc.w $a5, $a0, 0
+; LA32-NEXT:    beqz $a5, .LBB72_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a4, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_add_i8_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    addi.w $a3, $a3, 0
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB72_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    add.w $a5, $a4, $a1
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB72_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw add ptr %a, i8 %b acq_rel
+  ret i8 %1
+}
+
+define i16 @atomicrmw_add_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_add_i16_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:    lu12i.w $a2, 15
+; LA32-NEXT:    ori $a2, $a2, 4095
+; LA32-NEXT:    slli.w $a3, $a0, 3
+; LA32-NEXT:    sll.w $a2, $a2, $a3
+; LA32-NEXT:    bstrpick.w $a1, $a1, 15, 0
+; LA32-NEXT:    sll.w $a1, $a1, $a3
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB73_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a4, $a0, 0
+; LA32-NEXT:    add.w $a5, $a4, $a1
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    and $a5, $a5, $a2
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    sc.w $a5, $a0, 0
+; LA32-NEXT:    beqz $a5, .LBB73_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a4, $a3
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_add_i16_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    lu12i.w $a2, 15
+; LA64-NEXT:    ori $a2, $a2, 4095
+; LA64-NEXT:    slli.d $a3, $a0, 3
+; LA64-NEXT:    sll.w $a2, $a2, $a3
+; LA64-NEXT:    addi.w $a2, $a2, 0
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a3
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB73_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    add.w $a5, $a4, $a1
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a2
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB73_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a3
+; LA64-NEXT:    ret
+  %1 = atomicrmw add ptr %a, i16 %b acq_rel
+  ret i16 %1
+}
+
+define i32 @atomicrmw_add_i32_acq_rel(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_add_i32_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB74_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    add.w $a3, $a2, $a1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beqz $a3, .LBB74_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_add_i32_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    amadd_db.w $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw add ptr %a, i32 %b acq_rel
+  ret i32 %1
+}
+
+define i64 @atomicrmw_add_i64_acq_rel(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_add_i64_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    ori $a3, $zero, 4
+; LA32-NEXT:    bl %plt(__atomic_fetch_add_8)
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_add_i64_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    amadd_db.d $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw add ptr %a, i64 %b acq_rel
+  ret i64 %1
+}
+
+define i8 @atomicrmw_sub_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_sub_i8_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:    slli.w $a2, $a0, 3
+; LA32-NEXT:    ori $a3, $zero, 255
+; LA32-NEXT:    sll.w $a3, $a3, $a2
+; LA32-NEXT:    andi $a1, $a1, 255
+; LA32-NEXT:    sll.w $a1, $a1, $a2
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB76_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a4, $a0, 0
+; LA32-NEXT:    sub.w $a5, $a4, $a1
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    and $a5, $a5, $a3
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    sc.w $a5, $a0, 0
+; LA32-NEXT:    beqz $a5, .LBB76_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a4, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_sub_i8_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    addi.w $a3, $a3, 0
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB76_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    sub.w $a5, $a4, $a1
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB76_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw sub ptr %a, i8 %b acq_rel
+  ret i8 %1
+}
+
+define i16 @atomicrmw_sub_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_sub_i16_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:    lu12i.w $a2, 15
+; LA32-NEXT:    ori $a2, $a2, 4095
+; LA32-NEXT:    slli.w $a3, $a0, 3
+; LA32-NEXT:    sll.w $a2, $a2, $a3
+; LA32-NEXT:    bstrpick.w $a1, $a1, 15, 0
+; LA32-NEXT:    sll.w $a1, $a1, $a3
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB77_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a4, $a0, 0
+; LA32-NEXT:    sub.w $a5, $a4, $a1
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    and $a5, $a5, $a2
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    sc.w $a5, $a0, 0
+; LA32-NEXT:    beqz $a5, .LBB77_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a4, $a3
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_sub_i16_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    lu12i.w $a2, 15
+; LA64-NEXT:    ori $a2, $a2, 4095
+; LA64-NEXT:    slli.d $a3, $a0, 3
+; LA64-NEXT:    sll.w $a2, $a2, $a3
+; LA64-NEXT:    addi.w $a2, $a2, 0
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a3
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB77_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    sub.w $a5, $a4, $a1
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a2
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB77_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a3
+; LA64-NEXT:    ret
+  %1 = atomicrmw sub ptr %a, i16 %b acq_rel
+  ret i16 %1
+}
+
+define i32 @atomicrmw_sub_i32_acq_rel(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_sub_i32_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB78_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    sub.w $a3, $a2, $a1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beqz $a3, .LBB78_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_sub_i32_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    sub.w $a2, $zero, $a1
+; LA64-NEXT:    amadd_db.w $a1, $a2, $a0
+; LA64-NEXT:    move $a0, $a1
+; LA64-NEXT:    ret
+  %1 = atomicrmw sub ptr %a, i32 %b acq_rel
+  ret i32 %1
+}
+
+define i64 @atomicrmw_sub_i64_acq_rel(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_sub_i64_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    ori $a3, $zero, 4
+; LA32-NEXT:    bl %plt(__atomic_fetch_sub_8)
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_sub_i64_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    sub.d $a2, $zero, $a1
+; LA64-NEXT:    amadd_db.d $a1, $a2, $a0
+; LA64-NEXT:    move $a0, $a1
+; LA64-NEXT:    ret
+  %1 = atomicrmw sub ptr %a, i64 %b acq_rel
+  ret i64 %1
+}
+
+define i8 @atomicrmw_nand_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_nand_i8_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:    slli.w $a2, $a0, 3
+; LA32-NEXT:    ori $a3, $zero, 255
+; LA32-NEXT:    sll.w $a3, $a3, $a2
+; LA32-NEXT:    andi $a1, $a1, 255
+; LA32-NEXT:    sll.w $a1, $a1, $a2
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB80_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a4, $a0, 0
+; LA32-NEXT:    and $a5, $a4, $a1
+; LA32-NEXT:    nor $a5, $a5, $zero
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    and $a5, $a5, $a3
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    sc.w $a5, $a0, 0
+; LA32-NEXT:    beqz $a5, .LBB80_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a4, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_nand_i8_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    addi.w $a3, $a3, 0
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB80_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a5, $a4, $a1
+; LA64-NEXT:    nor $a5, $a5, $zero
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB80_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw nand ptr %a, i8 %b acq_rel
+  ret i8 %1
+}
+
+define i16 @atomicrmw_nand_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_nand_i16_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:    lu12i.w $a2, 15
+; LA32-NEXT:    ori $a2, $a2, 4095
+; LA32-NEXT:    slli.w $a3, $a0, 3
+; LA32-NEXT:    sll.w $a2, $a2, $a3
+; LA32-NEXT:    bstrpick.w $a1, $a1, 15, 0
+; LA32-NEXT:    sll.w $a1, $a1, $a3
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB81_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a4, $a0, 0
+; LA32-NEXT:    and $a5, $a4, $a1
+; LA32-NEXT:    nor $a5, $a5, $zero
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    and $a5, $a5, $a2
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    sc.w $a5, $a0, 0
+; LA32-NEXT:    beqz $a5, .LBB81_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a4, $a3
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_nand_i16_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    lu12i.w $a2, 15
+; LA64-NEXT:    ori $a2, $a2, 4095
+; LA64-NEXT:    slli.d $a3, $a0, 3
+; LA64-NEXT:    sll.w $a2, $a2, $a3
+; LA64-NEXT:    addi.w $a2, $a2, 0
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a3
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB81_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a5, $a4, $a1
+; LA64-NEXT:    nor $a5, $a5, $zero
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a2
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB81_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a3
+; LA64-NEXT:    ret
+  %1 = atomicrmw nand ptr %a, i16 %b acq_rel
+  ret i16 %1
+}
+
+define i32 @atomicrmw_nand_i32_acq_rel(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_nand_i32_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB82_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    and $a3, $a2, $a1
+; LA32-NEXT:    nor $a3, $a3, $zero
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beqz $a3, .LBB82_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_nand_i32_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:  .LBB82_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a2, $a0, 0
+; LA64-NEXT:    and $a3, $a2, $a1
+; LA64-NEXT:    nor $a3, $a3, $zero
+; LA64-NEXT:    sc.w $a3, $a0, 0
+; LA64-NEXT:    beqz $a3, .LBB82_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw nand ptr %a, i32 %b acq_rel
+  ret i32 %1
+}
+
+define i64 @atomicrmw_nand_i64_acq_rel(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_nand_i64_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    ori $a3, $zero, 4
+; LA32-NEXT:    bl %plt(__atomic_fetch_nand_8)
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_nand_i64_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:  .LBB83_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.d $a2, $a0, 0
+; LA64-NEXT:    and $a3, $a2, $a1
+; LA64-NEXT:    nor $a3, $a3, $zero
+; LA64-NEXT:    sc.d $a3, $a0, 0
+; LA64-NEXT:    beqz $a3, .LBB83_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw nand ptr %a, i64 %b acq_rel
+  ret i64 %1
+}
+
+define i8 @atomicrmw_and_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_and_i8_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:    slli.w $a2, $a0, 3
+; LA32-NEXT:    ori $a3, $zero, 255
+; LA32-NEXT:    sll.w $a3, $a3, $a2
+; LA32-NEXT:    andi $a1, $a1, 255
+; LA32-NEXT:    sll.w $a1, $a1, $a2
+; LA32-NEXT:    orn $a1, $a1, $a3
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB84_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a3, $a0, 0
+; LA32-NEXT:    and $a4, $a3, $a1
+; LA32-NEXT:    sc.w $a4, $a0, 0
+; LA32-NEXT:    beqz $a4, .LBB84_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a3, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_and_i8_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    orn $a1, $a1, $a3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    amand_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw and ptr %a, i8 %b acq_rel
+  ret i8 %1
+}
+
+define i16 @atomicrmw_and_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_and_i16_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:    lu12i.w $a2, 15
+; LA32-NEXT:    ori $a2, $a2, 4095
+; LA32-NEXT:    slli.w $a3, $a0, 3
+; LA32-NEXT:    sll.w $a2, $a2, $a3
+; LA32-NEXT:    bstrpick.w $a1, $a1, 15, 0
+; LA32-NEXT:    sll.w $a1, $a1, $a3
+; LA32-NEXT:    orn $a1, $a1, $a2
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB85_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    and $a4, $a2, $a1
+; LA32-NEXT:    sc.w $a4, $a0, 0
+; LA32-NEXT:    beqz $a4, .LBB85_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a2, $a3
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_and_i16_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    lu12i.w $a2, 15
+; LA64-NEXT:    ori $a2, $a2, 4095
+; LA64-NEXT:    slli.d $a3, $a0, 3
+; LA64-NEXT:    sll.w $a2, $a2, $a3
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a3
+; LA64-NEXT:    orn $a1, $a1, $a2
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    amand_db.w $a2, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a2, $a3
+; LA64-NEXT:    ret
+  %1 = atomicrmw and ptr %a, i16 %b acq_rel
+  ret i16 %1
+}
+
+define i32 @atomicrmw_and_i32_acq_rel(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_and_i32_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB86_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    and $a3, $a2, $a1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beqz $a3, .LBB86_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_and_i32_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    amand_db.w $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw and ptr %a, i32 %b acq_rel
+  ret i32 %1
+}
+
+define i64 @atomicrmw_and_i64_acq_rel(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_and_i64_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    ori $a3, $zero, 4
+; LA32-NEXT:    bl %plt(__atomic_fetch_and_8)
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_and_i64_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    amand_db.d $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw and ptr %a, i64 %b acq_rel
+  ret i64 %1
+}
+
+define i8 @atomicrmw_or_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_or_i8_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:    andi $a1, $a1, 255
+; LA32-NEXT:    slli.w $a2, $a0, 3
+; LA32-NEXT:    sll.w $a1, $a1, $a2
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB88_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a3, $a0, 0
+; LA32-NEXT:    or $a4, $a3, $a1
+; LA32-NEXT:    sc.w $a4, $a0, 0
+; LA32-NEXT:    beqz $a4, .LBB88_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a3, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_or_i8_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    amor_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw or ptr %a, i8 %b acq_rel
+  ret i8 %1
+}
+
+define i16 @atomicrmw_or_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_or_i16_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:    bstrpick.w $a1, $a1, 15, 0
+; LA32-NEXT:    slli.w $a2, $a0, 3
+; LA32-NEXT:    sll.w $a1, $a1, $a2
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB89_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a3, $a0, 0
+; LA32-NEXT:    or $a4, $a3, $a1
+; LA32-NEXT:    sc.w $a4, $a0, 0
+; LA32-NEXT:    beqz $a4, .LBB89_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a3, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_or_i16_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    amor_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw or ptr %a, i16 %b acq_rel
+  ret i16 %1
+}
+
+define i32 @atomicrmw_or_i32_acq_rel(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_or_i32_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB90_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    or $a3, $a2, $a1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beqz $a3, .LBB90_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_or_i32_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    amor_db.w $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw or ptr %a, i32 %b acq_rel
+  ret i32 %1
+}
+
+define i64 @atomicrmw_or_i64_acq_rel(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_or_i64_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    ori $a3, $zero, 4
+; LA32-NEXT:    bl %plt(__atomic_fetch_or_8)
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_or_i64_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    amor_db.d $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw or ptr %a, i64 %b acq_rel
+  ret i64 %1
+}
+
+define i8 @atomicrmw_xor_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_xor_i8_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:    andi $a1, $a1, 255
+; LA32-NEXT:    slli.w $a2, $a0, 3
+; LA32-NEXT:    sll.w $a1, $a1, $a2
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB92_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a3, $a0, 0
+; LA32-NEXT:    xor $a4, $a3, $a1
+; LA32-NEXT:    sc.w $a4, $a0, 0
+; LA32-NEXT:    beqz $a4, .LBB92_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a3, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_xor_i8_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    amxor_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw xor ptr %a, i8 %b acq_rel
+  ret i8 %1
+}
+
+define i16 @atomicrmw_xor_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_xor_i16_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:    bstrpick.w $a1, $a1, 15, 0
+; LA32-NEXT:    slli.w $a2, $a0, 3
+; LA32-NEXT:    sll.w $a1, $a1, $a2
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB93_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a3, $a0, 0
+; LA32-NEXT:    xor $a4, $a3, $a1
+; LA32-NEXT:    sc.w $a4, $a0, 0
+; LA32-NEXT:    beqz $a4, .LBB93_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a3, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_xor_i16_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    amxor_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw xor ptr %a, i16 %b acq_rel
+  ret i16 %1
+}
+
+define i32 @atomicrmw_xor_i32_acq_rel(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_xor_i32_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB94_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    xor $a3, $a2, $a1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beqz $a3, .LBB94_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_xor_i32_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    amxor_db.w $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw xor ptr %a, i32 %b acq_rel
+  ret i32 %1
+}
+
+define i64 @atomicrmw_xor_i64_acq_rel(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_xor_i64_acq_rel:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    ori $a3, $zero, 4
+; LA32-NEXT:    bl %plt(__atomic_fetch_xor_8)
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_xor_i64_acq_rel:
+; LA64:       # %bb.0:
+; LA64-NEXT:    amxor_db.d $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw xor ptr %a, i64 %b acq_rel
+  ret i64 %1
+}
+
+define i8 @atomicrmw_xchg_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_xchg_i8_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:    slli.w $a2, $a0, 3
+; LA32-NEXT:    ori $a3, $zero, 255
+; LA32-NEXT:    sll.w $a3, $a3, $a2
+; LA32-NEXT:    andi $a1, $a1, 255
+; LA32-NEXT:    sll.w $a1, $a1, $a2
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB96_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a4, $a0, 0
+; LA32-NEXT:    addi.w $a5, $a1, 0
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    and $a5, $a5, $a3
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    sc.w $a5, $a0, 0
+; LA32-NEXT:    beqz $a5, .LBB96_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a4, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_xchg_i8_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    addi.w $a3, $a3, 0
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB96_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    addi.w $a5, $a1, 0
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB96_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i8 %b seq_cst
+  ret i8 %1
+}
+
+define i8 @atomicrmw_xchg_0_i8_seq_cst(ptr %a) nounwind {
+; LA32-LABEL: atomicrmw_xchg_0_i8_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:    slli.w $a1, $a0, 3
+; LA32-NEXT:    ori $a2, $zero, 255
+; LA32-NEXT:    sll.w $a2, $a2, $a1
+; LA32-NEXT:    nor $a2, $a2, $zero
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB97_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a3, $a0, 0
+; LA32-NEXT:    and $a4, $a3, $a2
+; LA32-NEXT:    sc.w $a4, $a0, 0
+; LA32-NEXT:    beqz $a4, .LBB97_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a3, $a1
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_xchg_0_i8_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a1, $a0, 3
+; LA64-NEXT:    ori $a2, $zero, 255
+; LA64-NEXT:    sll.w $a2, $a2, $a1
+; LA64-NEXT:    nor $a2, $a2, $zero
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    amand_db.w $a3, $a2, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a1
+; LA64-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i8 0 seq_cst
+  ret i8 %1
+}
+
+define i8 @atomicrmw_xchg_minus_1_i8_seq_cst(ptr %a) nounwind {
+; LA32-LABEL: atomicrmw_xchg_minus_1_i8_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:    slli.w $a1, $a0, 3
+; LA32-NEXT:    ori $a2, $zero, 255
+; LA32-NEXT:    sll.w $a2, $a2, $a1
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB98_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a3, $a0, 0
+; LA32-NEXT:    or $a4, $a3, $a2
+; LA32-NEXT:    sc.w $a4, $a0, 0
+; LA32-NEXT:    beqz $a4, .LBB98_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a3, $a1
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_xchg_minus_1_i8_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a1, $a0, 3
+; LA64-NEXT:    ori $a2, $zero, 255
+; LA64-NEXT:    sll.w $a2, $a2, $a1
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    amor_db.w $a3, $a2, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a1
+; LA64-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i8 -1 seq_cst
+  ret i8 %1
+}
+
+define i16 @atomicrmw_xchg_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_xchg_i16_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:    lu12i.w $a2, 15
+; LA32-NEXT:    ori $a2, $a2, 4095
+; LA32-NEXT:    slli.w $a3, $a0, 3
+; LA32-NEXT:    sll.w $a2, $a2, $a3
+; LA32-NEXT:    bstrpick.w $a1, $a1, 15, 0
+; LA32-NEXT:    sll.w $a1, $a1, $a3
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB99_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a4, $a0, 0
+; LA32-NEXT:    addi.w $a5, $a1, 0
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    and $a5, $a5, $a2
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    sc.w $a5, $a0, 0
+; LA32-NEXT:    beqz $a5, .LBB99_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a4, $a3
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_xchg_i16_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    lu12i.w $a2, 15
+; LA64-NEXT:    ori $a2, $a2, 4095
+; LA64-NEXT:    slli.d $a3, $a0, 3
+; LA64-NEXT:    sll.w $a2, $a2, $a3
+; LA64-NEXT:    addi.w $a2, $a2, 0
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a3
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB99_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    addi.w $a5, $a1, 0
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a2
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB99_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a3
+; LA64-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i16 %b seq_cst
+  ret i16 %1
+}
+
+define i16 @atomicrmw_xchg_0_i16_seq_cst(ptr %a) nounwind {
+; LA32-LABEL: atomicrmw_xchg_0_i16_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:    lu12i.w $a1, 15
+; LA32-NEXT:    ori $a1, $a1, 4095
+; LA32-NEXT:    slli.w $a2, $a0, 3
+; LA32-NEXT:    sll.w $a1, $a1, $a2
+; LA32-NEXT:    nor $a1, $a1, $zero
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB100_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a3, $a0, 0
+; LA32-NEXT:    and $a4, $a3, $a1
+; LA32-NEXT:    sc.w $a4, $a0, 0
+; LA32-NEXT:    beqz $a4, .LBB100_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a3, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_xchg_0_i16_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    lu12i.w $a1, 15
+; LA64-NEXT:    ori $a1, $a1, 4095
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    nor $a1, $a1, $zero
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    amand_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i16 0 seq_cst
+  ret i16 %1
+}
+
+define i16 @atomicrmw_xchg_minus_1_i16_seq_cst(ptr %a) nounwind {
+; LA32-LABEL: atomicrmw_xchg_minus_1_i16_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:    lu12i.w $a1, 15
+; LA32-NEXT:    ori $a1, $a1, 4095
+; LA32-NEXT:    slli.w $a2, $a0, 3
+; LA32-NEXT:    sll.w $a1, $a1, $a2
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB101_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a3, $a0, 0
+; LA32-NEXT:    or $a4, $a3, $a1
+; LA32-NEXT:    sc.w $a4, $a0, 0
+; LA32-NEXT:    beqz $a4, .LBB101_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a3, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_xchg_minus_1_i16_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    lu12i.w $a1, 15
+; LA64-NEXT:    ori $a1, $a1, 4095
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    amor_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i16 -1 seq_cst
+  ret i16 %1
+}
+
+define i32 @atomicrmw_xchg_i32_seq_cst(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_xchg_i32_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB102_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    move $a3, $a1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beqz $a3, .LBB102_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_xchg_i32_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    amswap_db.w $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i32 %b seq_cst
+  ret i32 %1
+}
+
+define i64 @atomicrmw_xchg_i64_seq_cst(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_xchg_i64_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    ori $a3, $zero, 5
+; LA32-NEXT:    bl %plt(__atomic_exchange_8)
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_xchg_i64_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    amswap_db.d $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw xchg ptr %a, i64 %b seq_cst
+  ret i64 %1
+}
+
+define i8 @atomicrmw_add_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_add_i8_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:    slli.w $a2, $a0, 3
+; LA32-NEXT:    ori $a3, $zero, 255
+; LA32-NEXT:    sll.w $a3, $a3, $a2
+; LA32-NEXT:    andi $a1, $a1, 255
+; LA32-NEXT:    sll.w $a1, $a1, $a2
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB104_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a4, $a0, 0
+; LA32-NEXT:    add.w $a5, $a4, $a1
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    and $a5, $a5, $a3
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    sc.w $a5, $a0, 0
+; LA32-NEXT:    beqz $a5, .LBB104_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a4, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_add_i8_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    addi.w $a3, $a3, 0
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB104_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    add.w $a5, $a4, $a1
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB104_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw add ptr %a, i8 %b seq_cst
+  ret i8 %1
+}
+
+define i16 @atomicrmw_add_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_add_i16_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:    lu12i.w $a2, 15
+; LA32-NEXT:    ori $a2, $a2, 4095
+; LA32-NEXT:    slli.w $a3, $a0, 3
+; LA32-NEXT:    sll.w $a2, $a2, $a3
+; LA32-NEXT:    bstrpick.w $a1, $a1, 15, 0
+; LA32-NEXT:    sll.w $a1, $a1, $a3
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB105_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a4, $a0, 0
+; LA32-NEXT:    add.w $a5, $a4, $a1
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    and $a5, $a5, $a2
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    sc.w $a5, $a0, 0
+; LA32-NEXT:    beqz $a5, .LBB105_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a4, $a3
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_add_i16_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    lu12i.w $a2, 15
+; LA64-NEXT:    ori $a2, $a2, 4095
+; LA64-NEXT:    slli.d $a3, $a0, 3
+; LA64-NEXT:    sll.w $a2, $a2, $a3
+; LA64-NEXT:    addi.w $a2, $a2, 0
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a3
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB105_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    add.w $a5, $a4, $a1
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a2
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB105_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a3
+; LA64-NEXT:    ret
+  %1 = atomicrmw add ptr %a, i16 %b seq_cst
+  ret i16 %1
+}
+
+define i32 @atomicrmw_add_i32_seq_cst(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_add_i32_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB106_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    add.w $a3, $a2, $a1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beqz $a3, .LBB106_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_add_i32_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    amadd_db.w $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw add ptr %a, i32 %b seq_cst
+  ret i32 %1
+}
+
+define i64 @atomicrmw_add_i64_seq_cst(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_add_i64_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    ori $a3, $zero, 5
+; LA32-NEXT:    bl %plt(__atomic_fetch_add_8)
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_add_i64_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    amadd_db.d $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw add ptr %a, i64 %b seq_cst
+  ret i64 %1
+}
+
+define i8 @atomicrmw_sub_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_sub_i8_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:    slli.w $a2, $a0, 3
+; LA32-NEXT:    ori $a3, $zero, 255
+; LA32-NEXT:    sll.w $a3, $a3, $a2
+; LA32-NEXT:    andi $a1, $a1, 255
+; LA32-NEXT:    sll.w $a1, $a1, $a2
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB108_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a4, $a0, 0
+; LA32-NEXT:    sub.w $a5, $a4, $a1
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    and $a5, $a5, $a3
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    sc.w $a5, $a0, 0
+; LA32-NEXT:    beqz $a5, .LBB108_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a4, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_sub_i8_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    addi.w $a3, $a3, 0
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB108_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    sub.w $a5, $a4, $a1
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB108_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw sub ptr %a, i8 %b seq_cst
+  ret i8 %1
+}
+
+define i16 @atomicrmw_sub_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_sub_i16_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:    lu12i.w $a2, 15
+; LA32-NEXT:    ori $a2, $a2, 4095
+; LA32-NEXT:    slli.w $a3, $a0, 3
+; LA32-NEXT:    sll.w $a2, $a2, $a3
+; LA32-NEXT:    bstrpick.w $a1, $a1, 15, 0
+; LA32-NEXT:    sll.w $a1, $a1, $a3
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB109_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a4, $a0, 0
+; LA32-NEXT:    sub.w $a5, $a4, $a1
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    and $a5, $a5, $a2
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    sc.w $a5, $a0, 0
+; LA32-NEXT:    beqz $a5, .LBB109_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a4, $a3
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_sub_i16_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    lu12i.w $a2, 15
+; LA64-NEXT:    ori $a2, $a2, 4095
+; LA64-NEXT:    slli.d $a3, $a0, 3
+; LA64-NEXT:    sll.w $a2, $a2, $a3
+; LA64-NEXT:    addi.w $a2, $a2, 0
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a3
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB109_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    sub.w $a5, $a4, $a1
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a2
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB109_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a3
+; LA64-NEXT:    ret
+  %1 = atomicrmw sub ptr %a, i16 %b seq_cst
+  ret i16 %1
+}
+
+define i32 @atomicrmw_sub_i32_seq_cst(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_sub_i32_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB110_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    sub.w $a3, $a2, $a1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beqz $a3, .LBB110_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_sub_i32_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    sub.w $a2, $zero, $a1
+; LA64-NEXT:    amadd_db.w $a1, $a2, $a0
+; LA64-NEXT:    move $a0, $a1
+; LA64-NEXT:    ret
+  %1 = atomicrmw sub ptr %a, i32 %b seq_cst
+  ret i32 %1
+}
+
+define i64 @atomicrmw_sub_i64_seq_cst(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_sub_i64_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    ori $a3, $zero, 5
+; LA32-NEXT:    bl %plt(__atomic_fetch_sub_8)
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_sub_i64_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    sub.d $a2, $zero, $a1
+; LA64-NEXT:    amadd_db.d $a1, $a2, $a0
+; LA64-NEXT:    move $a0, $a1
+; LA64-NEXT:    ret
+  %1 = atomicrmw sub ptr %a, i64 %b seq_cst
+  ret i64 %1
+}
+
+define i8 @atomicrmw_nand_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_nand_i8_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:    slli.w $a2, $a0, 3
+; LA32-NEXT:    ori $a3, $zero, 255
+; LA32-NEXT:    sll.w $a3, $a3, $a2
+; LA32-NEXT:    andi $a1, $a1, 255
+; LA32-NEXT:    sll.w $a1, $a1, $a2
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB112_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a4, $a0, 0
+; LA32-NEXT:    and $a5, $a4, $a1
+; LA32-NEXT:    nor $a5, $a5, $zero
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    and $a5, $a5, $a3
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    sc.w $a5, $a0, 0
+; LA32-NEXT:    beqz $a5, .LBB112_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a4, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_nand_i8_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    addi.w $a3, $a3, 0
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB112_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a5, $a4, $a1
+; LA64-NEXT:    nor $a5, $a5, $zero
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB112_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw nand ptr %a, i8 %b seq_cst
+  ret i8 %1
+}
+
+define i16 @atomicrmw_nand_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_nand_i16_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:    lu12i.w $a2, 15
+; LA32-NEXT:    ori $a2, $a2, 4095
+; LA32-NEXT:    slli.w $a3, $a0, 3
+; LA32-NEXT:    sll.w $a2, $a2, $a3
+; LA32-NEXT:    bstrpick.w $a1, $a1, 15, 0
+; LA32-NEXT:    sll.w $a1, $a1, $a3
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB113_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a4, $a0, 0
+; LA32-NEXT:    and $a5, $a4, $a1
+; LA32-NEXT:    nor $a5, $a5, $zero
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    and $a5, $a5, $a2
+; LA32-NEXT:    xor $a5, $a4, $a5
+; LA32-NEXT:    sc.w $a5, $a0, 0
+; LA32-NEXT:    beqz $a5, .LBB113_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a4, $a3
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_nand_i16_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    lu12i.w $a2, 15
+; LA64-NEXT:    ori $a2, $a2, 4095
+; LA64-NEXT:    slli.d $a3, $a0, 3
+; LA64-NEXT:    sll.w $a2, $a2, $a3
+; LA64-NEXT:    addi.w $a2, $a2, 0
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a3
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:  .LBB113_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a4, $a0, 0
+; LA64-NEXT:    and $a5, $a4, $a1
+; LA64-NEXT:    nor $a5, $a5, $zero
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    and $a5, $a5, $a2
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:    sc.w $a5, $a0, 0
+; LA64-NEXT:    beqz $a5, .LBB113_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    srl.w $a0, $a4, $a3
+; LA64-NEXT:    ret
+  %1 = atomicrmw nand ptr %a, i16 %b seq_cst
+  ret i16 %1
+}
+
+define i32 @atomicrmw_nand_i32_seq_cst(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_nand_i32_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB114_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    and $a3, $a2, $a1
+; LA32-NEXT:    nor $a3, $a3, $zero
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beqz $a3, .LBB114_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_nand_i32_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:  .LBB114_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.w $a2, $a0, 0
+; LA64-NEXT:    and $a3, $a2, $a1
+; LA64-NEXT:    nor $a3, $a3, $zero
+; LA64-NEXT:    sc.w $a3, $a0, 0
+; LA64-NEXT:    beqz $a3, .LBB114_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw nand ptr %a, i32 %b seq_cst
+  ret i32 %1
+}
+
+define i64 @atomicrmw_nand_i64_seq_cst(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_nand_i64_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    ori $a3, $zero, 5
+; LA32-NEXT:    bl %plt(__atomic_fetch_nand_8)
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_nand_i64_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:  .LBB115_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    ll.d $a2, $a0, 0
+; LA64-NEXT:    and $a3, $a2, $a1
+; LA64-NEXT:    nor $a3, $a3, $zero
+; LA64-NEXT:    sc.d $a3, $a0, 0
+; LA64-NEXT:    beqz $a3, .LBB115_1
+; LA64-NEXT:  # %bb.2:
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw nand ptr %a, i64 %b seq_cst
+  ret i64 %1
+}
+
+define i8 @atomicrmw_and_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_and_i8_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:    slli.w $a2, $a0, 3
+; LA32-NEXT:    ori $a3, $zero, 255
+; LA32-NEXT:    sll.w $a3, $a3, $a2
+; LA32-NEXT:    andi $a1, $a1, 255
+; LA32-NEXT:    sll.w $a1, $a1, $a2
+; LA32-NEXT:    orn $a1, $a1, $a3
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB116_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a3, $a0, 0
+; LA32-NEXT:    and $a4, $a3, $a1
+; LA32-NEXT:    sc.w $a4, $a0, 0
+; LA32-NEXT:    beqz $a4, .LBB116_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a3, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_and_i8_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a2
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    orn $a1, $a1, $a3
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    amand_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw and ptr %a, i8 %b seq_cst
+  ret i8 %1
+}
+
+define i16 @atomicrmw_and_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_and_i16_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:    lu12i.w $a2, 15
+; LA32-NEXT:    ori $a2, $a2, 4095
+; LA32-NEXT:    slli.w $a3, $a0, 3
+; LA32-NEXT:    sll.w $a2, $a2, $a3
+; LA32-NEXT:    bstrpick.w $a1, $a1, 15, 0
+; LA32-NEXT:    sll.w $a1, $a1, $a3
+; LA32-NEXT:    orn $a1, $a1, $a2
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB117_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    and $a4, $a2, $a1
+; LA32-NEXT:    sc.w $a4, $a0, 0
+; LA32-NEXT:    beqz $a4, .LBB117_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a2, $a3
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_and_i16_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    lu12i.w $a2, 15
+; LA64-NEXT:    ori $a2, $a2, 4095
+; LA64-NEXT:    slli.d $a3, $a0, 3
+; LA64-NEXT:    sll.w $a2, $a2, $a3
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a3
+; LA64-NEXT:    orn $a1, $a1, $a2
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    amand_db.w $a2, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a2, $a3
+; LA64-NEXT:    ret
+  %1 = atomicrmw and ptr %a, i16 %b seq_cst
+  ret i16 %1
+}
+
+define i32 @atomicrmw_and_i32_seq_cst(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_and_i32_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB118_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    and $a3, $a2, $a1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beqz $a3, .LBB118_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_and_i32_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    amand_db.w $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw and ptr %a, i32 %b seq_cst
+  ret i32 %1
+}
+
+define i64 @atomicrmw_and_i64_seq_cst(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_and_i64_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    ori $a3, $zero, 5
+; LA32-NEXT:    bl %plt(__atomic_fetch_and_8)
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_and_i64_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    amand_db.d $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw and ptr %a, i64 %b seq_cst
+  ret i64 %1
+}
+
+define i8 @atomicrmw_or_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_or_i8_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:    andi $a1, $a1, 255
+; LA32-NEXT:    slli.w $a2, $a0, 3
+; LA32-NEXT:    sll.w $a1, $a1, $a2
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB120_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a3, $a0, 0
+; LA32-NEXT:    or $a4, $a3, $a1
+; LA32-NEXT:    sc.w $a4, $a0, 0
+; LA32-NEXT:    beqz $a4, .LBB120_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a3, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_or_i8_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    amor_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw or ptr %a, i8 %b seq_cst
+  ret i8 %1
+}
+
+define i16 @atomicrmw_or_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_or_i16_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:    bstrpick.w $a1, $a1, 15, 0
+; LA32-NEXT:    slli.w $a2, $a0, 3
+; LA32-NEXT:    sll.w $a1, $a1, $a2
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB121_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a3, $a0, 0
+; LA32-NEXT:    or $a4, $a3, $a1
+; LA32-NEXT:    sc.w $a4, $a0, 0
+; LA32-NEXT:    beqz $a4, .LBB121_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a3, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_or_i16_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    amor_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw or ptr %a, i16 %b seq_cst
+  ret i16 %1
+}
+
+define i32 @atomicrmw_or_i32_seq_cst(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_or_i32_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB122_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    or $a3, $a2, $a1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beqz $a3, .LBB122_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_or_i32_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    amor_db.w $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw or ptr %a, i32 %b seq_cst
+  ret i32 %1
+}
+
+define i64 @atomicrmw_or_i64_seq_cst(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_or_i64_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    ori $a3, $zero, 5
+; LA32-NEXT:    bl %plt(__atomic_fetch_or_8)
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_or_i64_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    amor_db.d $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw or ptr %a, i64 %b seq_cst
+  ret i64 %1
+}
+
+define i8 @atomicrmw_xor_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; LA32-LABEL: atomicrmw_xor_i8_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:    andi $a1, $a1, 255
+; LA32-NEXT:    slli.w $a2, $a0, 3
+; LA32-NEXT:    sll.w $a1, $a1, $a2
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB124_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a3, $a0, 0
+; LA32-NEXT:    xor $a4, $a3, $a1
+; LA32-NEXT:    sc.w $a4, $a0, 0
+; LA32-NEXT:    beqz $a4, .LBB124_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a3, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_xor_i8_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    amxor_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw xor ptr %a, i8 %b seq_cst
+  ret i8 %1
+}
+
+define i16 @atomicrmw_xor_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; LA32-LABEL: atomicrmw_xor_i16_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:    bstrpick.w $a1, $a1, 15, 0
+; LA32-NEXT:    slli.w $a2, $a0, 3
+; LA32-NEXT:    sll.w $a1, $a1, $a2
+; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
+; LA32-NEXT:  .LBB125_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a3, $a0, 0
+; LA32-NEXT:    xor $a4, $a3, $a1
+; LA32-NEXT:    sc.w $a4, $a0, 0
+; LA32-NEXT:    beqz $a4, .LBB125_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    srl.w $a0, $a3, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_xor_i16_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    sll.w $a1, $a1, $a2
+; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    amxor_db.w $a3, $a1, $a0
+; LA64-NEXT:    srl.w $a0, $a3, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw xor ptr %a, i16 %b seq_cst
+  ret i16 %1
+}
+
+define i32 @atomicrmw_xor_i32_seq_cst(ptr %a, i32 %b) nounwind {
+; LA32-LABEL: atomicrmw_xor_i32_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:  .LBB126_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:    ll.w $a2, $a0, 0
+; LA32-NEXT:    xor $a3, $a2, $a1
+; LA32-NEXT:    sc.w $a3, $a0, 0
+; LA32-NEXT:    beqz $a3, .LBB126_1
+; LA32-NEXT:  # %bb.2:
+; LA32-NEXT:    move $a0, $a2
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_xor_i32_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    amxor_db.w $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw xor ptr %a, i32 %b seq_cst
+  ret i32 %1
+}
+
+define i64 @atomicrmw_xor_i64_seq_cst(ptr %a, i64 %b) nounwind {
+; LA32-LABEL: atomicrmw_xor_i64_seq_cst:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    ori $a3, $zero, 5
+; LA32-NEXT:    bl %plt(__atomic_fetch_xor_8)
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: atomicrmw_xor_i64_seq_cst:
+; LA64:       # %bb.0:
+; LA64-NEXT:    amxor_db.d $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw xor ptr %a, i64 %b seq_cst
+  ret i64 %1
+}
+
 define i8 @atomicrmw_xchg_i8_monotonic(ptr %a, i8 %b) nounwind {
 ; LA32-LABEL: atomicrmw_xchg_i8_monotonic:
 ; LA32:       # %bb.0:
@@ -1040,14 +3987,14 @@ define i8 @atomicrmw_xchg_i8_monotonic(ptr %a, i8 %b) nounwind {
 ; LA32-NEXT:    andi $a1, $a1, 255
 ; LA32-NEXT:    sll.w $a1, $a1, $a2
 ; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
-; LA32-NEXT:  .LBB32_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:  .LBB128_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    ll.w $a4, $a0, 0
 ; LA32-NEXT:    addi.w $a5, $a1, 0
 ; LA32-NEXT:    xor $a5, $a4, $a5
 ; LA32-NEXT:    and $a5, $a5, $a3
 ; LA32-NEXT:    xor $a5, $a4, $a5
 ; LA32-NEXT:    sc.w $a5, $a0, 0
-; LA32-NEXT:    beqz $a5, .LBB32_1
+; LA32-NEXT:    beqz $a5, .LBB128_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    srl.w $a0, $a4, $a2
 ; LA32-NEXT:    ret
@@ -1062,14 +4009,14 @@ define i8 @atomicrmw_xchg_i8_monotonic(ptr %a, i8 %b) nounwind {
 ; LA64-NEXT:    sll.w $a1, $a1, $a2
 ; LA64-NEXT:    addi.w $a1, $a1, 0
 ; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
-; LA64-NEXT:  .LBB32_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:  .LBB128_1: # =>This Inner Loop Header: Depth=1
 ; LA64-NEXT:    ll.w $a4, $a0, 0
 ; LA64-NEXT:    addi.w $a5, $a1, 0
 ; LA64-NEXT:    xor $a5, $a4, $a5
 ; LA64-NEXT:    and $a5, $a5, $a3
 ; LA64-NEXT:    xor $a5, $a4, $a5
 ; LA64-NEXT:    sc.w $a5, $a0, 0
-; LA64-NEXT:    beqz $a5, .LBB32_1
+; LA64-NEXT:    beqz $a5, .LBB128_1
 ; LA64-NEXT:  # %bb.2:
 ; LA64-NEXT:    srl.w $a0, $a4, $a2
 ; LA64-NEXT:    ret
@@ -1085,11 +4032,11 @@ define i8 @atomicrmw_xchg_0_i8_monotonic(ptr %a) nounwind {
 ; LA32-NEXT:    sll.w $a2, $a2, $a1
 ; LA32-NEXT:    nor $a2, $a2, $zero
 ; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
-; LA32-NEXT:  .LBB33_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:  .LBB129_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    ll.w $a3, $a0, 0
 ; LA32-NEXT:    and $a4, $a3, $a2
 ; LA32-NEXT:    sc.w $a4, $a0, 0
-; LA32-NEXT:    beqz $a4, .LBB33_1
+; LA32-NEXT:    beqz $a4, .LBB129_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    srl.w $a0, $a3, $a1
 ; LA32-NEXT:    ret
@@ -1115,11 +4062,11 @@ define i8 @atomicrmw_xchg_minus_1_i8_monotonic(ptr %a) nounwind {
 ; LA32-NEXT:    ori $a2, $zero, 255
 ; LA32-NEXT:    sll.w $a2, $a2, $a1
 ; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
-; LA32-NEXT:  .LBB34_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:  .LBB130_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    ll.w $a3, $a0, 0
 ; LA32-NEXT:    or $a4, $a3, $a2
 ; LA32-NEXT:    sc.w $a4, $a0, 0
-; LA32-NEXT:    beqz $a4, .LBB34_1
+; LA32-NEXT:    beqz $a4, .LBB130_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    srl.w $a0, $a3, $a1
 ; LA32-NEXT:    ret
@@ -1147,14 +4094,14 @@ define i16 @atomicrmw_xchg_i16_monotonic(ptr %a, i16 %b) nounwind {
 ; LA32-NEXT:    bstrpick.w $a1, $a1, 15, 0
 ; LA32-NEXT:    sll.w $a1, $a1, $a3
 ; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
-; LA32-NEXT:  .LBB35_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:  .LBB131_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    ll.w $a4, $a0, 0
 ; LA32-NEXT:    addi.w $a5, $a1, 0
 ; LA32-NEXT:    xor $a5, $a4, $a5
 ; LA32-NEXT:    and $a5, $a5, $a2
 ; LA32-NEXT:    xor $a5, $a4, $a5
 ; LA32-NEXT:    sc.w $a5, $a0, 0
-; LA32-NEXT:    beqz $a5, .LBB35_1
+; LA32-NEXT:    beqz $a5, .LBB131_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    srl.w $a0, $a4, $a3
 ; LA32-NEXT:    ret
@@ -1170,14 +4117,14 @@ define i16 @atomicrmw_xchg_i16_monotonic(ptr %a, i16 %b) nounwind {
 ; LA64-NEXT:    sll.w $a1, $a1, $a3
 ; LA64-NEXT:    addi.w $a1, $a1, 0
 ; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
-; LA64-NEXT:  .LBB35_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:  .LBB131_1: # =>This Inner Loop Header: Depth=1
 ; LA64-NEXT:    ll.w $a4, $a0, 0
 ; LA64-NEXT:    addi.w $a5, $a1, 0
 ; LA64-NEXT:    xor $a5, $a4, $a5
 ; LA64-NEXT:    and $a5, $a5, $a2
 ; LA64-NEXT:    xor $a5, $a4, $a5
 ; LA64-NEXT:    sc.w $a5, $a0, 0
-; LA64-NEXT:    beqz $a5, .LBB35_1
+; LA64-NEXT:    beqz $a5, .LBB131_1
 ; LA64-NEXT:  # %bb.2:
 ; LA64-NEXT:    srl.w $a0, $a4, $a3
 ; LA64-NEXT:    ret
@@ -1194,11 +4141,11 @@ define i16 @atomicrmw_xchg_0_i16_monotonic(ptr %a) nounwind {
 ; LA32-NEXT:    sll.w $a1, $a1, $a2
 ; LA32-NEXT:    nor $a1, $a1, $zero
 ; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
-; LA32-NEXT:  .LBB36_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:  .LBB132_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    ll.w $a3, $a0, 0
 ; LA32-NEXT:    and $a4, $a3, $a1
 ; LA32-NEXT:    sc.w $a4, $a0, 0
-; LA32-NEXT:    beqz $a4, .LBB36_1
+; LA32-NEXT:    beqz $a4, .LBB132_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    srl.w $a0, $a3, $a2
 ; LA32-NEXT:    ret
@@ -1226,11 +4173,11 @@ define i16 @atomicrmw_xchg_minus_1_i16_monotonic(ptr %a) nounwind {
 ; LA32-NEXT:    slli.w $a2, $a0, 3
 ; LA32-NEXT:    sll.w $a1, $a1, $a2
 ; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
-; LA32-NEXT:  .LBB37_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:  .LBB133_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    ll.w $a3, $a0, 0
 ; LA32-NEXT:    or $a4, $a3, $a1
 ; LA32-NEXT:    sc.w $a4, $a0, 0
-; LA32-NEXT:    beqz $a4, .LBB37_1
+; LA32-NEXT:    beqz $a4, .LBB133_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    srl.w $a0, $a3, $a2
 ; LA32-NEXT:    ret
@@ -1252,11 +4199,11 @@ define i16 @atomicrmw_xchg_minus_1_i16_monotonic(ptr %a) nounwind {
 define i32 @atomicrmw_xchg_i32_monotonic(ptr %a, i32 %b) nounwind {
 ; LA32-LABEL: atomicrmw_xchg_i32_monotonic:
 ; LA32:       # %bb.0:
-; LA32-NEXT:  .LBB38_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:  .LBB134_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    ll.w $a2, $a0, 0
 ; LA32-NEXT:    move $a3, $a1
 ; LA32-NEXT:    sc.w $a3, $a0, 0
-; LA32-NEXT:    beqz $a3, .LBB38_1
+; LA32-NEXT:    beqz $a3, .LBB134_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    move $a0, $a2
 ; LA32-NEXT:    ret
@@ -1299,14 +4246,14 @@ define i8 @atomicrmw_add_i8_monotonic(ptr %a, i8 %b) nounwind {
 ; LA32-NEXT:    andi $a1, $a1, 255
 ; LA32-NEXT:    sll.w $a1, $a1, $a2
 ; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
-; LA32-NEXT:  .LBB40_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:  .LBB136_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    ll.w $a4, $a0, 0
 ; LA32-NEXT:    add.w $a5, $a4, $a1
 ; LA32-NEXT:    xor $a5, $a4, $a5
 ; LA32-NEXT:    and $a5, $a5, $a3
 ; LA32-NEXT:    xor $a5, $a4, $a5
 ; LA32-NEXT:    sc.w $a5, $a0, 0
-; LA32-NEXT:    beqz $a5, .LBB40_1
+; LA32-NEXT:    beqz $a5, .LBB136_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    srl.w $a0, $a4, $a2
 ; LA32-NEXT:    ret
@@ -1321,14 +4268,14 @@ define i8 @atomicrmw_add_i8_monotonic(ptr %a, i8 %b) nounwind {
 ; LA64-NEXT:    sll.w $a1, $a1, $a2
 ; LA64-NEXT:    addi.w $a1, $a1, 0
 ; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
-; LA64-NEXT:  .LBB40_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:  .LBB136_1: # =>This Inner Loop Header: Depth=1
 ; LA64-NEXT:    ll.w $a4, $a0, 0
 ; LA64-NEXT:    add.w $a5, $a4, $a1
 ; LA64-NEXT:    xor $a5, $a4, $a5
 ; LA64-NEXT:    and $a5, $a5, $a3
 ; LA64-NEXT:    xor $a5, $a4, $a5
 ; LA64-NEXT:    sc.w $a5, $a0, 0
-; LA64-NEXT:    beqz $a5, .LBB40_1
+; LA64-NEXT:    beqz $a5, .LBB136_1
 ; LA64-NEXT:  # %bb.2:
 ; LA64-NEXT:    srl.w $a0, $a4, $a2
 ; LA64-NEXT:    ret
@@ -1346,14 +4293,14 @@ define i16 @atomicrmw_add_i16_monotonic(ptr %a, i16 %b) nounwind {
 ; LA32-NEXT:    bstrpick.w $a1, $a1, 15, 0
 ; LA32-NEXT:    sll.w $a1, $a1, $a3
 ; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
-; LA32-NEXT:  .LBB41_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:  .LBB137_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    ll.w $a4, $a0, 0
 ; LA32-NEXT:    add.w $a5, $a4, $a1
 ; LA32-NEXT:    xor $a5, $a4, $a5
 ; LA32-NEXT:    and $a5, $a5, $a2
 ; LA32-NEXT:    xor $a5, $a4, $a5
 ; LA32-NEXT:    sc.w $a5, $a0, 0
-; LA32-NEXT:    beqz $a5, .LBB41_1
+; LA32-NEXT:    beqz $a5, .LBB137_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    srl.w $a0, $a4, $a3
 ; LA32-NEXT:    ret
@@ -1369,14 +4316,14 @@ define i16 @atomicrmw_add_i16_monotonic(ptr %a, i16 %b) nounwind {
 ; LA64-NEXT:    sll.w $a1, $a1, $a3
 ; LA64-NEXT:    addi.w $a1, $a1, 0
 ; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
-; LA64-NEXT:  .LBB41_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:  .LBB137_1: # =>This Inner Loop Header: Depth=1
 ; LA64-NEXT:    ll.w $a4, $a0, 0
 ; LA64-NEXT:    add.w $a5, $a4, $a1
 ; LA64-NEXT:    xor $a5, $a4, $a5
 ; LA64-NEXT:    and $a5, $a5, $a2
 ; LA64-NEXT:    xor $a5, $a4, $a5
 ; LA64-NEXT:    sc.w $a5, $a0, 0
-; LA64-NEXT:    beqz $a5, .LBB41_1
+; LA64-NEXT:    beqz $a5, .LBB137_1
 ; LA64-NEXT:  # %bb.2:
 ; LA64-NEXT:    srl.w $a0, $a4, $a3
 ; LA64-NEXT:    ret
@@ -1387,11 +4334,11 @@ define i16 @atomicrmw_add_i16_monotonic(ptr %a, i16 %b) nounwind {
 define i32 @atomicrmw_add_i32_monotonic(ptr %a, i32 %b) nounwind {
 ; LA32-LABEL: atomicrmw_add_i32_monotonic:
 ; LA32:       # %bb.0:
-; LA32-NEXT:  .LBB42_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:  .LBB138_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    ll.w $a2, $a0, 0
 ; LA32-NEXT:    add.w $a3, $a2, $a1
 ; LA32-NEXT:    sc.w $a3, $a0, 0
-; LA32-NEXT:    beqz $a3, .LBB42_1
+; LA32-NEXT:    beqz $a3, .LBB138_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    move $a0, $a2
 ; LA32-NEXT:    ret
@@ -1434,14 +4381,14 @@ define i8 @atomicrmw_sub_i8_monotonic(ptr %a, i8 %b) nounwind {
 ; LA32-NEXT:    andi $a1, $a1, 255
 ; LA32-NEXT:    sll.w $a1, $a1, $a2
 ; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
-; LA32-NEXT:  .LBB44_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:  .LBB140_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    ll.w $a4, $a0, 0
 ; LA32-NEXT:    sub.w $a5, $a4, $a1
 ; LA32-NEXT:    xor $a5, $a4, $a5
 ; LA32-NEXT:    and $a5, $a5, $a3
 ; LA32-NEXT:    xor $a5, $a4, $a5
 ; LA32-NEXT:    sc.w $a5, $a0, 0
-; LA32-NEXT:    beqz $a5, .LBB44_1
+; LA32-NEXT:    beqz $a5, .LBB140_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    srl.w $a0, $a4, $a2
 ; LA32-NEXT:    ret
@@ -1456,14 +4403,14 @@ define i8 @atomicrmw_sub_i8_monotonic(ptr %a, i8 %b) nounwind {
 ; LA64-NEXT:    sll.w $a1, $a1, $a2
 ; LA64-NEXT:    addi.w $a1, $a1, 0
 ; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
-; LA64-NEXT:  .LBB44_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:  .LBB140_1: # =>This Inner Loop Header: Depth=1
 ; LA64-NEXT:    ll.w $a4, $a0, 0
 ; LA64-NEXT:    sub.w $a5, $a4, $a1
 ; LA64-NEXT:    xor $a5, $a4, $a5
 ; LA64-NEXT:    and $a5, $a5, $a3
 ; LA64-NEXT:    xor $a5, $a4, $a5
 ; LA64-NEXT:    sc.w $a5, $a0, 0
-; LA64-NEXT:    beqz $a5, .LBB44_1
+; LA64-NEXT:    beqz $a5, .LBB140_1
 ; LA64-NEXT:  # %bb.2:
 ; LA64-NEXT:    srl.w $a0, $a4, $a2
 ; LA64-NEXT:    ret
@@ -1481,14 +4428,14 @@ define i16 @atomicrmw_sub_i16_monotonic(ptr %a, i16 %b) nounwind {
 ; LA32-NEXT:    bstrpick.w $a1, $a1, 15, 0
 ; LA32-NEXT:    sll.w $a1, $a1, $a3
 ; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
-; LA32-NEXT:  .LBB45_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:  .LBB141_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    ll.w $a4, $a0, 0
 ; LA32-NEXT:    sub.w $a5, $a4, $a1
 ; LA32-NEXT:    xor $a5, $a4, $a5
 ; LA32-NEXT:    and $a5, $a5, $a2
 ; LA32-NEXT:    xor $a5, $a4, $a5
 ; LA32-NEXT:    sc.w $a5, $a0, 0
-; LA32-NEXT:    beqz $a5, .LBB45_1
+; LA32-NEXT:    beqz $a5, .LBB141_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    srl.w $a0, $a4, $a3
 ; LA32-NEXT:    ret
@@ -1504,14 +4451,14 @@ define i16 @atomicrmw_sub_i16_monotonic(ptr %a, i16 %b) nounwind {
 ; LA64-NEXT:    sll.w $a1, $a1, $a3
 ; LA64-NEXT:    addi.w $a1, $a1, 0
 ; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
-; LA64-NEXT:  .LBB45_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:  .LBB141_1: # =>This Inner Loop Header: Depth=1
 ; LA64-NEXT:    ll.w $a4, $a0, 0
 ; LA64-NEXT:    sub.w $a5, $a4, $a1
 ; LA64-NEXT:    xor $a5, $a4, $a5
 ; LA64-NEXT:    and $a5, $a5, $a2
 ; LA64-NEXT:    xor $a5, $a4, $a5
 ; LA64-NEXT:    sc.w $a5, $a0, 0
-; LA64-NEXT:    beqz $a5, .LBB45_1
+; LA64-NEXT:    beqz $a5, .LBB141_1
 ; LA64-NEXT:  # %bb.2:
 ; LA64-NEXT:    srl.w $a0, $a4, $a3
 ; LA64-NEXT:    ret
@@ -1522,11 +4469,11 @@ define i16 @atomicrmw_sub_i16_monotonic(ptr %a, i16 %b) nounwind {
 define i32 @atomicrmw_sub_i32_monotonic(ptr %a, i32 %b) nounwind {
 ; LA32-LABEL: atomicrmw_sub_i32_monotonic:
 ; LA32:       # %bb.0:
-; LA32-NEXT:  .LBB46_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:  .LBB142_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    ll.w $a2, $a0, 0
 ; LA32-NEXT:    sub.w $a3, $a2, $a1
 ; LA32-NEXT:    sc.w $a3, $a0, 0
-; LA32-NEXT:    beqz $a3, .LBB46_1
+; LA32-NEXT:    beqz $a3, .LBB142_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    move $a0, $a2
 ; LA32-NEXT:    ret
@@ -1571,7 +4518,7 @@ define i8 @atomicrmw_nand_i8_monotonic(ptr %a, i8 %b) nounwind {
 ; LA32-NEXT:    andi $a1, $a1, 255
 ; LA32-NEXT:    sll.w $a1, $a1, $a2
 ; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
-; LA32-NEXT:  .LBB48_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:  .LBB144_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    ll.w $a4, $a0, 0
 ; LA32-NEXT:    and $a5, $a4, $a1
 ; LA32-NEXT:    nor $a5, $a5, $zero
@@ -1579,7 +4526,7 @@ define i8 @atomicrmw_nand_i8_monotonic(ptr %a, i8 %b) nounwind {
 ; LA32-NEXT:    and $a5, $a5, $a3
 ; LA32-NEXT:    xor $a5, $a4, $a5
 ; LA32-NEXT:    sc.w $a5, $a0, 0
-; LA32-NEXT:    beqz $a5, .LBB48_1
+; LA32-NEXT:    beqz $a5, .LBB144_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    srl.w $a0, $a4, $a2
 ; LA32-NEXT:    ret
@@ -1594,7 +4541,7 @@ define i8 @atomicrmw_nand_i8_monotonic(ptr %a, i8 %b) nounwind {
 ; LA64-NEXT:    sll.w $a1, $a1, $a2
 ; LA64-NEXT:    addi.w $a1, $a1, 0
 ; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
-; LA64-NEXT:  .LBB48_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:  .LBB144_1: # =>This Inner Loop Header: Depth=1
 ; LA64-NEXT:    ll.w $a4, $a0, 0
 ; LA64-NEXT:    and $a5, $a4, $a1
 ; LA64-NEXT:    nor $a5, $a5, $zero
@@ -1602,7 +4549,7 @@ define i8 @atomicrmw_nand_i8_monotonic(ptr %a, i8 %b) nounwind {
 ; LA64-NEXT:    and $a5, $a5, $a3
 ; LA64-NEXT:    xor $a5, $a4, $a5
 ; LA64-NEXT:    sc.w $a5, $a0, 0
-; LA64-NEXT:    beqz $a5, .LBB48_1
+; LA64-NEXT:    beqz $a5, .LBB144_1
 ; LA64-NEXT:  # %bb.2:
 ; LA64-NEXT:    srl.w $a0, $a4, $a2
 ; LA64-NEXT:    ret
@@ -1620,7 +4567,7 @@ define i16 @atomicrmw_nand_i16_monotonic(ptr %a, i16 %b) nounwind {
 ; LA32-NEXT:    bstrpick.w $a1, $a1, 15, 0
 ; LA32-NEXT:    sll.w $a1, $a1, $a3
 ; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
-; LA32-NEXT:  .LBB49_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:  .LBB145_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    ll.w $a4, $a0, 0
 ; LA32-NEXT:    and $a5, $a4, $a1
 ; LA32-NEXT:    nor $a5, $a5, $zero
@@ -1628,7 +4575,7 @@ define i16 @atomicrmw_nand_i16_monotonic(ptr %a, i16 %b) nounwind {
 ; LA32-NEXT:    and $a5, $a5, $a2
 ; LA32-NEXT:    xor $a5, $a4, $a5
 ; LA32-NEXT:    sc.w $a5, $a0, 0
-; LA32-NEXT:    beqz $a5, .LBB49_1
+; LA32-NEXT:    beqz $a5, .LBB145_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    srl.w $a0, $a4, $a3
 ; LA32-NEXT:    ret
@@ -1644,7 +4591,7 @@ define i16 @atomicrmw_nand_i16_monotonic(ptr %a, i16 %b) nounwind {
 ; LA64-NEXT:    sll.w $a1, $a1, $a3
 ; LA64-NEXT:    addi.w $a1, $a1, 0
 ; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
-; LA64-NEXT:  .LBB49_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:  .LBB145_1: # =>This Inner Loop Header: Depth=1
 ; LA64-NEXT:    ll.w $a4, $a0, 0
 ; LA64-NEXT:    and $a5, $a4, $a1
 ; LA64-NEXT:    nor $a5, $a5, $zero
@@ -1652,7 +4599,7 @@ define i16 @atomicrmw_nand_i16_monotonic(ptr %a, i16 %b) nounwind {
 ; LA64-NEXT:    and $a5, $a5, $a2
 ; LA64-NEXT:    xor $a5, $a4, $a5
 ; LA64-NEXT:    sc.w $a5, $a0, 0
-; LA64-NEXT:    beqz $a5, .LBB49_1
+; LA64-NEXT:    beqz $a5, .LBB145_1
 ; LA64-NEXT:  # %bb.2:
 ; LA64-NEXT:    srl.w $a0, $a4, $a3
 ; LA64-NEXT:    ret
@@ -1663,24 +4610,24 @@ define i16 @atomicrmw_nand_i16_monotonic(ptr %a, i16 %b) nounwind {
 define i32 @atomicrmw_nand_i32_monotonic(ptr %a, i32 %b) nounwind {
 ; LA32-LABEL: atomicrmw_nand_i32_monotonic:
 ; LA32:       # %bb.0:
-; LA32-NEXT:  .LBB50_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:  .LBB146_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    ll.w $a2, $a0, 0
 ; LA32-NEXT:    and $a3, $a2, $a1
 ; LA32-NEXT:    nor $a3, $a3, $zero
 ; LA32-NEXT:    sc.w $a3, $a0, 0
-; LA32-NEXT:    beqz $a3, .LBB50_1
+; LA32-NEXT:    beqz $a3, .LBB146_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    move $a0, $a2
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: atomicrmw_nand_i32_monotonic:
 ; LA64:       # %bb.0:
-; LA64-NEXT:  .LBB50_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:  .LBB146_1: # =>This Inner Loop Header: Depth=1
 ; LA64-NEXT:    ll.w $a2, $a0, 0
 ; LA64-NEXT:    and $a3, $a2, $a1
 ; LA64-NEXT:    nor $a3, $a3, $zero
 ; LA64-NEXT:    sc.w $a3, $a0, 0
-; LA64-NEXT:    beqz $a3, .LBB50_1
+; LA64-NEXT:    beqz $a3, .LBB146_1
 ; LA64-NEXT:  # %bb.2:
 ; LA64-NEXT:    move $a0, $a2
 ; LA64-NEXT:    ret
@@ -1701,12 +4648,12 @@ define i64 @atomicrmw_nand_i64_monotonic(ptr %a, i64 %b) nounwind {
 ;
 ; LA64-LABEL: atomicrmw_nand_i64_monotonic:
 ; LA64:       # %bb.0:
-; LA64-NEXT:  .LBB51_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:  .LBB147_1: # =>This Inner Loop Header: Depth=1
 ; LA64-NEXT:    ll.d $a2, $a0, 0
 ; LA64-NEXT:    and $a3, $a2, $a1
 ; LA64-NEXT:    nor $a3, $a3, $zero
 ; LA64-NEXT:    sc.d $a3, $a0, 0
-; LA64-NEXT:    beqz $a3, .LBB51_1
+; LA64-NEXT:    beqz $a3, .LBB147_1
 ; LA64-NEXT:  # %bb.2:
 ; LA64-NEXT:    move $a0, $a2
 ; LA64-NEXT:    ret
@@ -1724,11 +4671,11 @@ define i8 @atomicrmw_and_i8_monotonic(ptr %a, i8 %b) nounwind {
 ; LA32-NEXT:    sll.w $a1, $a1, $a2
 ; LA32-NEXT:    orn $a1, $a1, $a3
 ; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
-; LA32-NEXT:  .LBB52_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:  .LBB148_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    ll.w $a3, $a0, 0
 ; LA32-NEXT:    and $a4, $a3, $a1
 ; LA32-NEXT:    sc.w $a4, $a0, 0
-; LA32-NEXT:    beqz $a4, .LBB52_1
+; LA32-NEXT:    beqz $a4, .LBB148_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    srl.w $a0, $a3, $a2
 ; LA32-NEXT:    ret
@@ -1760,11 +4707,11 @@ define i16 @atomicrmw_and_i16_monotonic(ptr %a, i16 %b) nounwind {
 ; LA32-NEXT:    sll.w $a1, $a1, $a3
 ; LA32-NEXT:    orn $a1, $a1, $a2
 ; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
-; LA32-NEXT:  .LBB53_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:  .LBB149_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    ll.w $a2, $a0, 0
 ; LA32-NEXT:    and $a4, $a2, $a1
 ; LA32-NEXT:    sc.w $a4, $a0, 0
-; LA32-NEXT:    beqz $a4, .LBB53_1
+; LA32-NEXT:    beqz $a4, .LBB149_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    srl.w $a0, $a2, $a3
 ; LA32-NEXT:    ret
@@ -1789,11 +4736,11 @@ define i16 @atomicrmw_and_i16_monotonic(ptr %a, i16 %b) nounwind {
 define i32 @atomicrmw_and_i32_monotonic(ptr %a, i32 %b) nounwind {
 ; LA32-LABEL: atomicrmw_and_i32_monotonic:
 ; LA32:       # %bb.0:
-; LA32-NEXT:  .LBB54_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:  .LBB150_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    ll.w $a2, $a0, 0
 ; LA32-NEXT:    and $a3, $a2, $a1
 ; LA32-NEXT:    sc.w $a3, $a0, 0
-; LA32-NEXT:    beqz $a3, .LBB54_1
+; LA32-NEXT:    beqz $a3, .LBB150_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    move $a0, $a2
 ; LA32-NEXT:    ret
@@ -1834,11 +4781,11 @@ define i8 @atomicrmw_or_i8_monotonic(ptr %a, i8 %b) nounwind {
 ; LA32-NEXT:    slli.w $a2, $a0, 3
 ; LA32-NEXT:    sll.w $a1, $a1, $a2
 ; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
-; LA32-NEXT:  .LBB56_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:  .LBB152_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    ll.w $a3, $a0, 0
 ; LA32-NEXT:    or $a4, $a3, $a1
 ; LA32-NEXT:    sc.w $a4, $a0, 0
-; LA32-NEXT:    beqz $a4, .LBB56_1
+; LA32-NEXT:    beqz $a4, .LBB152_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    srl.w $a0, $a3, $a2
 ; LA32-NEXT:    ret
@@ -1863,11 +4810,11 @@ define i16 @atomicrmw_or_i16_monotonic(ptr %a, i16 %b) nounwind {
 ; LA32-NEXT:    slli.w $a2, $a0, 3
 ; LA32-NEXT:    sll.w $a1, $a1, $a2
 ; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
-; LA32-NEXT:  .LBB57_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:  .LBB153_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    ll.w $a3, $a0, 0
 ; LA32-NEXT:    or $a4, $a3, $a1
 ; LA32-NEXT:    sc.w $a4, $a0, 0
-; LA32-NEXT:    beqz $a4, .LBB57_1
+; LA32-NEXT:    beqz $a4, .LBB153_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    srl.w $a0, $a3, $a2
 ; LA32-NEXT:    ret
@@ -1888,11 +4835,11 @@ define i16 @atomicrmw_or_i16_monotonic(ptr %a, i16 %b) nounwind {
 define i32 @atomicrmw_or_i32_monotonic(ptr %a, i32 %b) nounwind {
 ; LA32-LABEL: atomicrmw_or_i32_monotonic:
 ; LA32:       # %bb.0:
-; LA32-NEXT:  .LBB58_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:  .LBB154_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    ll.w $a2, $a0, 0
 ; LA32-NEXT:    or $a3, $a2, $a1
 ; LA32-NEXT:    sc.w $a3, $a0, 0
-; LA32-NEXT:    beqz $a3, .LBB58_1
+; LA32-NEXT:    beqz $a3, .LBB154_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    move $a0, $a2
 ; LA32-NEXT:    ret
@@ -1933,11 +4880,11 @@ define i8 @atomicrmw_xor_i8_monotonic(ptr %a, i8 %b) nounwind {
 ; LA32-NEXT:    slli.w $a2, $a0, 3
 ; LA32-NEXT:    sll.w $a1, $a1, $a2
 ; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
-; LA32-NEXT:  .LBB60_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:  .LBB156_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    ll.w $a3, $a0, 0
 ; LA32-NEXT:    xor $a4, $a3, $a1
 ; LA32-NEXT:    sc.w $a4, $a0, 0
-; LA32-NEXT:    beqz $a4, .LBB60_1
+; LA32-NEXT:    beqz $a4, .LBB156_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    srl.w $a0, $a3, $a2
 ; LA32-NEXT:    ret
@@ -1962,11 +4909,11 @@ define i16 @atomicrmw_xor_i16_monotonic(ptr %a, i16 %b) nounwind {
 ; LA32-NEXT:    slli.w $a2, $a0, 3
 ; LA32-NEXT:    sll.w $a1, $a1, $a2
 ; LA32-NEXT:    bstrins.w $a0, $zero, 1, 0
-; LA32-NEXT:  .LBB61_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:  .LBB157_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    ll.w $a3, $a0, 0
 ; LA32-NEXT:    xor $a4, $a3, $a1
 ; LA32-NEXT:    sc.w $a4, $a0, 0
-; LA32-NEXT:    beqz $a4, .LBB61_1
+; LA32-NEXT:    beqz $a4, .LBB157_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    srl.w $a0, $a3, $a2
 ; LA32-NEXT:    ret
@@ -1987,11 +4934,11 @@ define i16 @atomicrmw_xor_i16_monotonic(ptr %a, i16 %b) nounwind {
 define i32 @atomicrmw_xor_i32_monotonic(ptr %a, i32 %b) nounwind {
 ; LA32-LABEL: atomicrmw_xor_i32_monotonic:
 ; LA32:       # %bb.0:
-; LA32-NEXT:  .LBB62_1: # =>This Inner Loop Header: Depth=1
+; LA32-NEXT:  .LBB158_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    ll.w $a2, $a0, 0
 ; LA32-NEXT:    xor $a3, $a2, $a1
 ; LA32-NEXT:    sc.w $a3, $a0, 0
-; LA32-NEXT:    beqz $a3, .LBB62_1
+; LA32-NEXT:    beqz $a3, .LBB158_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    move $a0, $a2
 ; LA32-NEXT:    ret
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/fence-singlethread.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/fence-singlethread.ll
new file mode 100644
index 0000000000..a8b164a4cd
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/fence-singlethread.ll
@@ -0,0 +1,17 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
+; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+
+define void @fence_singlethread() {
+; LA32-LABEL: fence_singlethread:
+; LA32:       # %bb.0:
+; LA32-NEXT:    #MEMBARRIER
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: fence_singlethread:
+; LA64:       # %bb.0:
+; LA64-NEXT:    #MEMBARRIER
+; LA64-NEXT:    ret
+  fence syncscope("singlethread") seq_cst
+  ret void
+}
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/fence.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/fence.ll
index 724639f3c6..c5b2232f9b 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/fence.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/fence.ll
@@ -5,12 +5,12 @@
 define void @fence_acquire() nounwind {
 ; LA32-LABEL: fence_acquire:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    dbar 0
+; LA32-NEXT:    dbar 20
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: fence_acquire:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    dbar 0
+; LA64-NEXT:    dbar 20
 ; LA64-NEXT:    ret
   fence acquire
   ret void
@@ -19,12 +19,12 @@ define void @fence_acquire() nounwind {
 define void @fence_release() nounwind {
 ; LA32-LABEL: fence_release:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    dbar 0
+; LA32-NEXT:    dbar 18
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: fence_release:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    dbar 0
+; LA64-NEXT:    dbar 18
 ; LA64-NEXT:    ret
   fence release
   ret void
@@ -33,12 +33,12 @@ define void @fence_release() nounwind {
 define void @fence_acq_rel() nounwind {
 ; LA32-LABEL: fence_acq_rel:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    dbar 0
+; LA32-NEXT:    dbar 16
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: fence_acq_rel:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    dbar 0
+; LA64-NEXT:    dbar 16
 ; LA64-NEXT:    ret
   fence acq_rel
   ret void
@@ -47,12 +47,12 @@ define void @fence_acq_rel() nounwind {
 define void @fence_seq_cst() nounwind {
 ; LA32-LABEL: fence_seq_cst:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    dbar 0
+; LA32-NEXT:    dbar 16
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: fence_seq_cst:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    dbar 0
+; LA64-NEXT:    dbar 16
 ; LA64-NEXT:    ret
   fence seq_cst
   ret void
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/load-store-atomic.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/load-store-atomic.ll
index e91d0c145e..8b170c479e 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/load-store-atomic.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/load-store-atomic.ll
@@ -6,13 +6,13 @@ define i8 @load_acquire_i8(ptr %ptr) {
 ; LA32-LABEL: load_acquire_i8:
 ; LA32:       # %bb.0:
 ; LA32-NEXT:    ld.b $a0, $a0, 0
-; LA32-NEXT:    dbar 0
+; LA32-NEXT:    dbar 20
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: load_acquire_i8:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ld.b $a0, $a0, 0
-; LA64-NEXT:    dbar 0
+; LA64-NEXT:    dbar 20
 ; LA64-NEXT:    ret
   %val = load atomic i8, ptr %ptr acquire, align 1
   ret i8 %val
@@ -22,13 +22,13 @@ define i16 @load_acquire_i16(ptr %ptr) {
 ; LA32-LABEL: load_acquire_i16:
 ; LA32:       # %bb.0:
 ; LA32-NEXT:    ld.h $a0, $a0, 0
-; LA32-NEXT:    dbar 0
+; LA32-NEXT:    dbar 20
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: load_acquire_i16:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ld.h $a0, $a0, 0
-; LA64-NEXT:    dbar 0
+; LA64-NEXT:    dbar 20
 ; LA64-NEXT:    ret
   %val = load atomic i16, ptr %ptr acquire, align 2
   ret i16 %val
@@ -38,13 +38,13 @@ define i32 @load_acquire_i32(ptr %ptr) {
 ; LA32-LABEL: load_acquire_i32:
 ; LA32:       # %bb.0:
 ; LA32-NEXT:    ld.w $a0, $a0, 0
-; LA32-NEXT:    dbar 0
+; LA32-NEXT:    dbar 20
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: load_acquire_i32:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ld.w $a0, $a0, 0
-; LA64-NEXT:    dbar 0
+; LA64-NEXT:    dbar 20
 ; LA64-NEXT:    ret
   %val = load atomic i32, ptr %ptr acquire, align 4
   ret i32 %val
@@ -66,22 +66,218 @@ define i64 @load_acquire_i64(ptr %ptr) {
 ; LA64-LABEL: load_acquire_i64:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ld.d $a0, $a0, 0
-; LA64-NEXT:    dbar 0
+; LA64-NEXT:    dbar 20
 ; LA64-NEXT:    ret
   %val = load atomic i64, ptr %ptr acquire, align 8
   ret i64 %val
 }
 
+define i8 @load_unordered_i8(ptr %ptr) {
+; LA32-LABEL: load_unordered_i8:
+; LA32:       # %bb.0:
+; LA32-NEXT:    ld.b $a0, $a0, 0
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: load_unordered_i8:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ld.b $a0, $a0, 0
+; LA64-NEXT:    ret
+  %val = load atomic i8, ptr %ptr unordered, align 1
+  ret i8 %val
+}
+
+define i16 @load_unordered_i16(ptr %ptr) {
+; LA32-LABEL: load_unordered_i16:
+; LA32:       # %bb.0:
+; LA32-NEXT:    ld.h $a0, $a0, 0
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: load_unordered_i16:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ld.h $a0, $a0, 0
+; LA64-NEXT:    ret
+  %val = load atomic i16, ptr %ptr unordered, align 2
+  ret i16 %val
+}
+
+define i32 @load_unordered_i32(ptr %ptr) {
+; LA32-LABEL: load_unordered_i32:
+; LA32:       # %bb.0:
+; LA32-NEXT:    ld.w $a0, $a0, 0
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: load_unordered_i32:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ld.w $a0, $a0, 0
+; LA64-NEXT:    ret
+  %val = load atomic i32, ptr %ptr unordered, align 4
+  ret i32 %val
+}
+
+define i64 @load_unordered_i64(ptr %ptr) {
+; LA32-LABEL: load_unordered_i64:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    .cfi_def_cfa_offset 16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    .cfi_offset 1, -4
+; LA32-NEXT:    move $a1, $zero
+; LA32-NEXT:    bl %plt(__atomic_load_8)
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: load_unordered_i64:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ld.d $a0, $a0, 0
+; LA64-NEXT:    ret
+  %val = load atomic i64, ptr %ptr unordered, align 8
+  ret i64 %val
+}
+
+define i8 @load_monotonic_i8(ptr %ptr) {
+; LA32-LABEL: load_monotonic_i8:
+; LA32:       # %bb.0:
+; LA32-NEXT:    ld.b $a0, $a0, 0
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: load_monotonic_i8:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ld.b $a0, $a0, 0
+; LA64-NEXT:    ret
+  %val = load atomic i8, ptr %ptr monotonic, align 1
+  ret i8 %val
+}
+
+define i16 @load_monotonic_i16(ptr %ptr) {
+; LA32-LABEL: load_monotonic_i16:
+; LA32:       # %bb.0:
+; LA32-NEXT:    ld.h $a0, $a0, 0
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: load_monotonic_i16:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ld.h $a0, $a0, 0
+; LA64-NEXT:    ret
+  %val = load atomic i16, ptr %ptr monotonic, align 2
+  ret i16 %val
+}
+
+define i32 @load_monotonic_i32(ptr %ptr) {
+; LA32-LABEL: load_monotonic_i32:
+; LA32:       # %bb.0:
+; LA32-NEXT:    ld.w $a0, $a0, 0
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: load_monotonic_i32:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ld.w $a0, $a0, 0
+; LA64-NEXT:    ret
+  %val = load atomic i32, ptr %ptr monotonic, align 4
+  ret i32 %val
+}
+
+define i64 @load_monotonic_i64(ptr %ptr) {
+; LA32-LABEL: load_monotonic_i64:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    .cfi_def_cfa_offset 16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    .cfi_offset 1, -4
+; LA32-NEXT:    move $a1, $zero
+; LA32-NEXT:    bl %plt(__atomic_load_8)
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: load_monotonic_i64:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ld.d $a0, $a0, 0
+; LA64-NEXT:    ret
+  %val = load atomic i64, ptr %ptr monotonic, align 8
+  ret i64 %val
+}
+
+define i8 @load_seq_cst_i8(ptr %ptr) {
+; LA32-LABEL: load_seq_cst_i8:
+; LA32:       # %bb.0:
+; LA32-NEXT:    ld.b $a0, $a0, 0
+; LA32-NEXT:    dbar 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: load_seq_cst_i8:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ld.b $a0, $a0, 0
+; LA64-NEXT:    dbar 16
+; LA64-NEXT:    ret
+  %val = load atomic i8, ptr %ptr seq_cst, align 1
+  ret i8 %val
+}
+
+define i16 @load_seq_cst_i16(ptr %ptr) {
+; LA32-LABEL: load_seq_cst_i16:
+; LA32:       # %bb.0:
+; LA32-NEXT:    ld.h $a0, $a0, 0
+; LA32-NEXT:    dbar 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: load_seq_cst_i16:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ld.h $a0, $a0, 0
+; LA64-NEXT:    dbar 16
+; LA64-NEXT:    ret
+  %val = load atomic i16, ptr %ptr seq_cst, align 2
+  ret i16 %val
+}
+
+define i32 @load_seq_cst_i32(ptr %ptr) {
+; LA32-LABEL: load_seq_cst_i32:
+; LA32:       # %bb.0:
+; LA32-NEXT:    ld.w $a0, $a0, 0
+; LA32-NEXT:    dbar 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: load_seq_cst_i32:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ld.w $a0, $a0, 0
+; LA64-NEXT:    dbar 16
+; LA64-NEXT:    ret
+  %val = load atomic i32, ptr %ptr seq_cst, align 4
+  ret i32 %val
+}
+
+define i64 @load_seq_cst_i64(ptr %ptr) {
+; LA32-LABEL: load_seq_cst_i64:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    .cfi_def_cfa_offset 16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    .cfi_offset 1, -4
+; LA32-NEXT:    ori $a1, $zero, 5
+; LA32-NEXT:    bl %plt(__atomic_load_8)
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: load_seq_cst_i64:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ld.d $a0, $a0, 0
+; LA64-NEXT:    dbar 16
+; LA64-NEXT:    ret
+  %val = load atomic i64, ptr %ptr seq_cst, align 8
+  ret i64 %val
+}
+
 define void @store_release_i8(ptr %ptr, i8 signext %v) {
 ; LA32-LABEL: store_release_i8:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    dbar 0
+; LA32-NEXT:    dbar 18
 ; LA32-NEXT:    st.b $a1, $a0, 0
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: store_release_i8:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    dbar 0
+; LA64-NEXT:    dbar 18
 ; LA64-NEXT:    st.b $a1, $a0, 0
 ; LA64-NEXT:    ret
   store atomic i8 %v, ptr %ptr release, align 1
@@ -91,13 +287,13 @@ define void @store_release_i8(ptr %ptr, i8 signext %v) {
 define void @store_release_i16(ptr %ptr, i16 signext %v) {
 ; LA32-LABEL: store_release_i16:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    dbar 0
+; LA32-NEXT:    dbar 18
 ; LA32-NEXT:    st.h $a1, $a0, 0
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: store_release_i16:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    dbar 0
+; LA64-NEXT:    dbar 18
 ; LA64-NEXT:    st.h $a1, $a0, 0
 ; LA64-NEXT:    ret
   store atomic i16 %v, ptr %ptr release, align 2
@@ -107,7 +303,7 @@ define void @store_release_i16(ptr %ptr, i16 signext %v) {
 define void @store_release_i32(ptr %ptr, i32 signext %v) {
 ; LA32-LABEL: store_release_i32:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    dbar 0
+; LA32-NEXT:    dbar 18
 ; LA32-NEXT:    st.w $a1, $a0, 0
 ; LA32-NEXT:    ret
 ;
@@ -269,16 +465,16 @@ define void @store_monotonic_i64(ptr %ptr, i64 %v) {
 define void @store_seq_cst_i8(ptr %ptr, i8 signext %v) {
 ; LA32-LABEL: store_seq_cst_i8:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    dbar 0
+; LA32-NEXT:    dbar 16
 ; LA32-NEXT:    st.b $a1, $a0, 0
-; LA32-NEXT:    dbar 0
+; LA32-NEXT:    dbar 16
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: store_seq_cst_i8:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    dbar 0
+; LA64-NEXT:    dbar 16
 ; LA64-NEXT:    st.b $a1, $a0, 0
-; LA64-NEXT:    dbar 0
+; LA64-NEXT:    dbar 16
 ; LA64-NEXT:    ret
   store atomic i8 %v, ptr %ptr seq_cst, align 1
   ret void
@@ -287,16 +483,16 @@ define void @store_seq_cst_i8(ptr %ptr, i8 signext %v) {
 define void @store_seq_cst_i16(ptr %ptr, i16 signext %v) {
 ; LA32-LABEL: store_seq_cst_i16:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    dbar 0
+; LA32-NEXT:    dbar 16
 ; LA32-NEXT:    st.h $a1, $a0, 0
-; LA32-NEXT:    dbar 0
+; LA32-NEXT:    dbar 16
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: store_seq_cst_i16:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    dbar 0
+; LA64-NEXT:    dbar 16
 ; LA64-NEXT:    st.h $a1, $a0, 0
-; LA64-NEXT:    dbar 0
+; LA64-NEXT:    dbar 16
 ; LA64-NEXT:    ret
   store atomic i16 %v, ptr %ptr seq_cst, align 2
   ret void
@@ -305,9 +501,9 @@ define void @store_seq_cst_i16(ptr %ptr, i16 signext %v) {
 define void @store_seq_cst_i32(ptr %ptr, i32 signext %v) {
 ; LA32-LABEL: store_seq_cst_i32:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    dbar 0
+; LA32-NEXT:    dbar 16
 ; LA32-NEXT:    st.w $a1, $a0, 0
-; LA32-NEXT:    dbar 0
+; LA32-NEXT:    dbar 16
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: store_seq_cst_i32:
diff --git a/llvm/test/CodeGen/LoongArch/opt-pipeline.ll b/llvm/test/CodeGen/LoongArch/opt-pipeline.ll
index 8b1d635b60..3134d94054 100644
--- a/llvm/test/CodeGen/LoongArch/opt-pipeline.ll
+++ b/llvm/test/CodeGen/LoongArch/opt-pipeline.ll
@@ -165,6 +165,7 @@
 ; CHECK-NEXT:       Lazy Machine Block Frequency Analysis
 ; CHECK-NEXT:       Machine Optimization Remark Emitter
 ; CHECK-NEXT:       Stack Frame Layout Analysis
+; CHECK-NEXT:       LoongArch pseudo instruction expansion pass
 ; CHECK-NEXT:       LoongArch atomic pseudo instruction expansion pass
 ; CHECK-NEXT:       Lazy Machine Block Frequency Analysis
 ; CHECK-NEXT:       Machine Optimization Remark Emitter
diff --git a/llvm/test/CodeGen/M68k/CConv/rtd-call.ll b/llvm/test/CodeGen/M68k/CConv/rtd-call.ll
new file mode 100644
index 0000000000..56f36efbe0
--- /dev/null
+++ b/llvm/test/CodeGen/M68k/CConv/rtd-call.ll
@@ -0,0 +1,48 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -mtriple=m68k %s -stop-after=finalize-isel -o - | FileCheck %s
+
+; We want to make sure caller doesn't pop the stack for callees using
+; the M68k_RTD CC. However, we've implemented some frame optimization
+; techniques to eliminate as many as frame setup/destroy instructions.
+; Therefore, to make test case small and concise, we check the MIR generated
+; after ISel instead.
+
+declare dso_local m68k_rtdcc void @callee(i32 noundef)
+declare dso_local m68k_rtdcc void @va_callee(i32 noundef, ...)
+
+define dso_local i32 @caller(ptr noundef %y) {
+  ; CHECK-LABEL: name: caller
+  ; CHECK: bb.0.entry:
+  ; CHECK-NEXT:   [[MOV32rp:%[0-9]+]]:ar32 = MOV32rp 0, %fixed-stack.0, implicit-def dead $ccr :: (load (s32) from %fixed-stack.0, align 8)
+  ; CHECK-NEXT:   [[MOV32rj:%[0-9]+]]:xr32 = MOV32rj killed [[MOV32rp]], implicit-def dead $ccr :: (load (s32) from %ir.y)
+  ; CHECK-NEXT:   ADJCALLSTACKDOWN 4, 0, implicit-def dead $sp, implicit-def dead $ccr, implicit $sp
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:ar32 = COPY $sp
+  ; CHECK-NEXT:   MOV32jr [[COPY]], [[MOV32rj]], implicit-def dead $ccr :: (store (s32) into stack, align 2)
+  ; CHECK-NEXT:   CALLb @callee, csr_std, implicit $sp, implicit-def $sp
+  ; CHECK-NEXT:   ADJCALLSTACKUP 4, 4, implicit-def dead $sp, implicit-def dead $ccr, implicit $sp
+  ; CHECK-NEXT:   $d0 = COPY [[MOV32rj]]
+  ; CHECK-NEXT:   RET 0, $d0
+entry:
+  %0 = load i32, ptr %y, align 4
+  call m68k_rtdcc void @callee(i32 noundef %0)
+  ret i32 %0
+}
+
+define dso_local i32 @va_caller(ptr noundef %y) {
+  ; CHECK-LABEL: name: va_caller
+  ; CHECK: bb.0.entry:
+  ; CHECK-NEXT:   [[MOV32rp:%[0-9]+]]:ar32 = MOV32rp 0, %fixed-stack.0, implicit-def dead $ccr :: (load (s32) from %fixed-stack.0, align 8)
+  ; CHECK-NEXT:   [[MOV32rj:%[0-9]+]]:xr32 = MOV32rj killed [[MOV32rp]], implicit-def dead $ccr :: (load (s32) from %ir.y)
+  ; CHECK-NEXT:   ADJCALLSTACKDOWN 4, 0, implicit-def dead $sp, implicit-def dead $ccr, implicit $sp
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:ar32 = COPY $sp
+  ; CHECK-NEXT:   MOV32jr [[COPY]], [[MOV32rj]], implicit-def dead $ccr :: (store (s32) into stack, align 2)
+  ; CHECK-NEXT:   CALLb @va_callee, csr_std, implicit $sp, implicit-def $sp
+  ; CHECK-NEXT:   ADJCALLSTACKUP 4, 0, implicit-def dead $sp, implicit-def dead $ccr, implicit $sp
+  ; CHECK-NEXT:   $d0 = COPY [[MOV32rj]]
+  ; CHECK-NEXT:   RET 0, $d0
+entry:
+  %0 = load i32, ptr %y, align 4
+  call m68k_rtdcc void (i32, ...) @va_callee(i32 noundef %0)
+  ret i32 %0
+}
+
diff --git a/llvm/test/CodeGen/M68k/CConv/rtd-ret.ll b/llvm/test/CodeGen/M68k/CConv/rtd-ret.ll
new file mode 100644
index 0000000000..2dc5f2812f
--- /dev/null
+++ b/llvm/test/CodeGen/M68k/CConv/rtd-ret.ll
@@ -0,0 +1,31 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -mtriple=m68k < %s | FileCheck %s
+
+define dso_local m68k_rtdcc i32 @ret(i32 noundef %a, i32 noundef %b, i32 noundef %c) nounwind {
+; CHECK-LABEL: ret:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    move.l (8,%sp), %d0
+; CHECK-NEXT:    add.l (4,%sp), %d0
+; CHECK-NEXT:    add.l (12,%sp), %d0
+; CHECK-NEXT:    move.l (%sp), %a1
+; CHECK-NEXT:    adda.l #12, %sp
+; CHECK-NEXT:    move.l %a1, (%sp)
+; CHECK-NEXT:    rts
+entry:
+  %add = add nsw i32 %b, %a
+  %add1 = add nsw i32 %add, %c
+  ret i32 %add1
+}
+
+define dso_local m68k_rtdcc i32 @va_ret(i32 noundef %a, i32 noundef %b, i32 noundef %c, ...) nounwind {
+; CHECK-LABEL: va_ret:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    move.l (8,%sp), %d0
+; CHECK-NEXT:    add.l (4,%sp), %d0
+; CHECK-NEXT:    add.l (12,%sp), %d0
+; CHECK-NEXT:    rts
+entry:
+  %add = add nsw i32 %b, %a
+  %add1 = add nsw i32 %add, %c
+  ret i32 %add1
+}
diff --git a/llvm/test/CodeGen/MIR/Generic/first-character-parse-error.mir b/llvm/test/CodeGen/MIR/Generic/first-character-parse-error.mir
index 00a01058dc..869392f3e4 100644
--- a/llvm/test/CodeGen/MIR/Generic/first-character-parse-error.mir
+++ b/llvm/test/CodeGen/MIR/Generic/first-character-parse-error.mir
@@ -1,6 +1,6 @@
-:# RUN: not llc -run-pass=none %s -o - 2>&1 | FileCheck %s
+@# RUN: not llc -run-pass=none %s -o - 2>&1 | FileCheck %s
 
-# The : before the run comment is syntactically invalid. This used to
+# The @ before the run comment is syntactically invalid. This used to
 # crash in the SourceMgr diagnostic printer because it was called
 # before the LLVMContext was initialized.
 
diff --git a/llvm/test/CodeGen/MLRegAlloc/Inputs/reference-log-noml.txt b/llvm/test/CodeGen/MLRegAlloc/Inputs/reference-log-noml.txt
index 0c024ad2b2..a5ccdde751 100644
--- a/llvm/test/CodeGen/MLRegAlloc/Inputs/reference-log-noml.txt
+++ b/llvm/test/CodeGen/MLRegAlloc/Inputs/reference-log-noml.txt
@@ -16,8 +16,8 @@ hint_weights_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.7265065908432007,0.0,
 start_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.3333333432674408,0.3333333432674408,0.3333333432674408,0.3333333432674408,0.1666666716337204,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.3333333432674408
 end_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,2.2724486181679993e-10,0.9760092496871948,0.9760092496871948,0.9760092496871948,2.2724486181679993e-10,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9760092496871948
 hottest_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.23831403255462646,0.07943800836801529,0.07943800836801529,0.07943800836801529,0.9912577867507935,0.07069581001996994,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0
-liverange_size: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.9591121673583984,0.7940031290054321,0.7908878326416016,0.7379283308982849,0.9061526656150818,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.7352024912834167
-use_def_density: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.05104188248515129,0.01772311143577099,0.01417447254061699,0.014231426641345024,1.0,0.0737093985080719,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.4279724359512329
+liverange_size: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.9647942781448364,0.7939082384109497,0.7907436490058899,0.7401107549667358,0.9173259735107422,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.7436708807945251
+use_def_density: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.05123833194375038,0.017619721591472626,0.014218696393072605,0.014276761561632156,1.0,0.07275574654340744,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.4243086874485016
 max_stage: 0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1
 min_stage: 0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1
 progress: 0.7777777910232544
@@ -40,8 +40,8 @@ hint_weights_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0
 start_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.3333333432674408,0.0,0.3333333432674408,0.3333333432674408,0.1666666716337204,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.3333333432674408
 end_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,2.2724486181679993e-10,0.0,0.9760092496871948,0.9760092496871948,2.2724486181679993e-10,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9760092496871948
 hottest_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.2404157966375351,0.08013860136270523,0.0,0.08013860136270523,1.0,0.07131929695606232,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.08013860136270523
-liverange_size: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.9591121673583984,0.0,0.7908878326416016,0.7379283308982849,0.9061526656150818,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.7940031290054321
-use_def_density: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.05104188248515129,0.01772311143577099,0.0,0.014231426641345024,1.0,0.0737093985080719,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.01417447254061699
+liverange_size: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.9647942781448364,0.0,0.7907436490058899,0.7401107549667358,0.9173259735107422,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.7939082384109497
+use_def_density: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.05123833194375038,0.017619721591472626,0.0,0.014276761561632156,1.0,0.07275574654340744,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.014218696393072605
 max_stage: 0,0,0,0,0,0,0,0,0,1,1,0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1
 min_stage: 0,0,0,0,0,0,0,0,0,1,1,0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1
 progress: 0.7777777910232544
@@ -64,8 +64,8 @@ hint_weights_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.2421688437461853,0.0,
 start_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.1666666716337204,0.3333333432674408,0.3333333432674408,0.3333333432674408,0.3333333432674408,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.1666666716337204
 end_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,2.2724486181679993e-10,2.2724486181679993e-10,0.9760092496871948,0.9760092496871948,0.9760092496871948,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9760092496871948
 hottest_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.06705831736326218,0.01989283785223961,0.02235277369618416,0.2813863754272461,0.02235277369618416,0.27892643213272095,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0
-liverange_size: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.9061526656150818,0.9591121673583984,0.7352024912834167,0.7908878326416016,0.7379283308982849,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.6725077629089355
-use_def_density: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.05104188248515129,0.0737093985080719,0.01772311143577099,0.4279724359512329,0.014231426641345024,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.4858442544937134
+liverange_size: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.9173259735107422,0.9647942781448364,0.7436708807945251,0.7907436490058899,0.7401107549667358,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.6831487417221069
+use_def_density: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.05123833194375038,0.07275574654340744,0.017619721591472626,0.4243086874485016,0.014276761561632156,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.47955840826034546
 max_stage: 0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1
 min_stage: 0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1
 progress: 0.7777777910232544
@@ -88,8 +88,8 @@ hint_weights_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.2421688437461853,0.0,
 start_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.3333333432674408,0.3333333432674408,0.0,0.3333333432674408,0.1666666716337204,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.3333333432674408
 end_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,2.2724486181679993e-10,0.9760092496871948,0.0,0.9760092496871948,2.2724486181679993e-10,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9760092496871948
 hottest_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.23831403255462646,0.07943800836801529,1.0,0.0,0.9912577867507935,0.07069581001996994,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.07943800836801529
-liverange_size: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.9591121673583984,0.7352024912834167,0.0,0.7379283308982849,0.9061526656150818,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.7908878326416016
-use_def_density: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.05104188248515129,0.01772311143577099,0.4279724359512329,0.0,1.0,0.0737093985080719,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.014231426641345024
+liverange_size: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.9647942781448364,0.7436708807945251,0.0,0.7401107549667358,0.9173259735107422,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.7907436490058899
+use_def_density: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.05123833194375038,0.017619721591472626,0.4243086874485016,0.0,1.0,0.07275574654340744,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.014276761561632156
 max_stage: 0,0,0,0,0,0,0,0,0,1,1,1,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1
 min_stage: 0,0,0,0,0,0,0,0,0,1,1,1,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1
 progress: 0.7777777910232544
@@ -112,8 +112,8 @@ hint_weights_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.2421688437461853,0.0,
 start_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.3333333432674408,0.3333333432674408,0.1666666716337204,0.3333333432674408,0.1666666716337204,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.1666666716337204
 end_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,2.2724486181679993e-10,0.9760092496871948,0.9760092496871948,0.9760092496871948,2.2724486181679993e-10,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9760092496871948
 hottest_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.06705831736326218,0.02235277369618416,0.2813863754272461,1.0,0.27892643213272095,0.01989283785223961,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.01117638684809208
-liverange_size: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.9591121673583984,0.7352024912834167,0.6725077629089355,0.7379283308982849,0.9061526656150818,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.6693925261497498
-use_def_density: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.05104188248515129,0.01772311143577099,0.4279724359512329,0.4858442544937134,1.0,0.0737093985080719,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.00449750293046236
+liverange_size: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.9647942781448364,0.7436708807945251,0.6831487417221069,0.7401107549667358,0.9173259735107422,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.6799841523170471
+use_def_density: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.05123833194375038,0.017619721591472626,0.4243086874485016,0.47955840826034546,1.0,0.07275574654340744,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.004439314361661673
 max_stage: 0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1
 min_stage: 0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1
 progress: 0.7777777910232544
@@ -136,8 +136,8 @@ hint_weights_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.2421688437461853,0.0,
 start_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.3333333432674408,0.3333333432674408,0.1666666716337204,0.3333333432674408,0.1666666716337204,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.1666666716337204
 end_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,2.2724486181679993e-10,0.9760092496871948,0.9760092496871948,0.9760092496871948,2.2724486181679993e-10,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9760092496871948
 hottest_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.06705831736326218,0.02235277369618416,0.2813863754272461,1.0,0.27892643213272095,0.01989283785223961,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.01822916604578495
-liverange_size: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.9591121673583984,0.7352024912834167,0.6725077629089355,0.7379283308982849,0.9061526656150818,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.6662772297859192
-use_def_density: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.05104188248515129,0.01772311143577099,0.4279724359512329,0.4858442544937134,1.0,0.0737093985080719,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.008109557442367077
+liverange_size: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.9647942781448364,0.7436708807945251,0.6831487417221069,0.7401107549667358,0.9173259735107422,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.6768196225166321
+use_def_density: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.05123833194375038,0.017619721591472626,0.4243086874485016,0.47955840826034546,1.0,0.07275574654340744,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.008004635572433472
 max_stage: 0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1
 min_stage: 0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1
 progress: 0.7777777910232544
@@ -160,8 +160,8 @@ hint_weights_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.2421688586473465,0.0,
 start_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.1666666716337204,0.3333333432674408,0.3333333432674408,0.1666666716337204,0.3333333432674408,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.1666666716337204
 end_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,2.2724486181679993e-10,2.2724486181679993e-10,0.9760092496871948,0.9760092496871948,0.9760092496871948,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9760092496871948
 hottest_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.06705831736326218,0.01989283785223961,0.02235277369618416,0.2813863754272461,1.0,0.27892643213272095,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.2631579041481018
-liverange_size: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.9061526656150818,0.9591121673583984,0.7352024912834167,0.6725077629089355,0.7379283308982849,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.6631619930267334
-use_def_density: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.05104188248515129,0.0737093985080719,0.01772311143577099,0.4279724359512329,0.4858442544937134,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.07601386308670044
+liverange_size: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.9173259735107422,0.9647942781448364,0.7436708807945251,0.6831487417221069,0.7401107549667358,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.673655092716217
+use_def_density: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.05123833194375038,0.07275574654340744,0.017619721591472626,0.4243086874485016,0.47955840826034546,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.07503040134906769
 max_stage: 0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1
 min_stage: 0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1
 progress: 0.7777777910232544
@@ -184,8 +184,8 @@ hint_weights_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.2421688437461853,0.0,
 start_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.3333333432674408,0.1666666716337204,0.3333333432674408,0.1666666716337204,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.3333333432674408
 end_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.9760092496871948,0.9760092496871948,0.9760092496871948,2.2724486181679993e-10,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2.2724486181679993e-10
 hottest_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.06705831736326218,0.0,0.2813863754272461,1.0,0.27892643213272095,0.01989283785223961,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.02235277369618416
-liverange_size: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.7352024912834167,0.6725077629089355,0.7379283308982849,0.9061526656150818,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9591121673583984
-use_def_density: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.05104188248515129,0.0,0.4279724359512329,0.4858442544937134,1.0,0.0737093985080719,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.01772311143577099
+liverange_size: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.7436708807945251,0.6831487417221069,0.7401107549667358,0.9173259735107422,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9647942781448364
+use_def_density: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.05123833194375038,0.0,0.4243086874485016,0.47955840826034546,1.0,0.07275574654340744,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.017619721591472626
 max_stage: 0,0,0,0,0,0,0,0,0,1,0,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1
 min_stage: 0,0,0,0,0,0,0,0,0,1,0,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1
 progress: 0.7777777910232544
@@ -208,8 +208,8 @@ hint_weights_by_max: 1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.2421688437461853,
 start_bb_freq_by_max: 0.3333333432674408,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.1666666716337204,0.1666666716337204,0.3333333432674408,0.1666666716337204,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.1666666716337204
 end_bb_freq_by_max: 0.9760092496871948,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.9760092496871948,0.9760092496871948,0.9760092496871948,2.2724486181679993e-10,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.5110765099525452
 hottest_bb_freq_by_max: 0.2813863754272461,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.06705831736326218,0.2631579041481018,1.0,0.27892643213272095,0.01989283785223961,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.27892643213272095
-liverange_size: 0.7352024912834167,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.6631619930267334,0.6725077629089355,0.7379283308982849,0.9061526656150818,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.46105918288230896
-use_def_density: 0.42606985569000244,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.05081497132778168,0.07567594200372696,0.48368439078330994,0.9955543875694275,0.07338171452283859,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0
+liverange_size: 0.7436708807945251,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.673655092716217,0.6831487417221069,0.7401107549667358,0.9173259735107422,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.4683544337749481
+use_def_density: 0.4243086874485016,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.05123833194375038,0.07503040134906769,0.47955840826034546,1.0,0.07275574654340744,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.99146968126297
 max_stage: 1,0,0,0,0,0,0,0,0,0,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1
 min_stage: 1,0,0,0,0,0,0,0,0,0,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1
 progress: 0.7777777910232544
@@ -232,8 +232,8 @@ hint_weights_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.7265065908432007,
 start_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.1666666716337204,0.1666666716337204,0.3333333432674408,0.1666666716337204,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.3333333432674408
 end_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.9760092496871948,0.9760092496871948,0.9760092496871948,2.2724486181679993e-10,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9760092496871948
 hottest_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.06705831736326218,0.2631579041481018,1.0,0.27892643213272095,0.01989283785223961,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.2813863754272461
-liverange_size: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.6631619930267334,0.6725077629089355,0.7379283308982849,0.9061526656150818,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.7352024912834167
-use_def_density: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.05104188248515129,0.07601386308670044,0.4858442544937134,1.0,0.0737093985080719,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.4279724359512329
+liverange_size: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.673655092716217,0.6831487417221069,0.7401107549667358,0.9173259735107422,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.7436708807945251
+use_def_density: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.05123833194375038,0.07503040134906769,0.47955840826034546,1.0,0.07275574654340744,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.4243086874485016
 max_stage: 0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1
 min_stage: 0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1
 progress: 0.7777777910232544
@@ -256,8 +256,8 @@ hint_weights_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
 start_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.1666666716337204,0.0,0.1666666716337204,0.3333333432674408,0.1666666716337204,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0
 end_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9760092496871948,0.0,0.9760092496871948,0.9760092496871948,2.2724486181679993e-10,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0
 hottest_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.2631579041481018,0.0,1.0,0.27892643213272095,0.01989283785223961,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.06705831736326218
-liverange_size: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.6631619930267334,0.0,0.6725077629089355,0.7379283308982849,0.9061526656150818,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0
-use_def_density: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.07601386308670044,0.0,0.4858442544937134,1.0,0.0737093985080719,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.05104188248515129
+liverange_size: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.673655092716217,0.0,0.6831487417221069,0.7401107549667358,0.9173259735107422,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0
+use_def_density: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.07503040134906769,0.0,0.47955840826034546,1.0,0.07275574654340744,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.05123833194375038
 max_stage: 0,0,0,0,0,0,0,0,0,0,1,0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1
 min_stage: 0,0,0,0,0,0,0,0,0,0,1,0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1
 progress: 0.7777777910232544
@@ -280,8 +280,8 @@ hint_weights_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,1.0,0.0,0.0,0.0
 start_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.7152887582778931,0.35764437913894653,0.35764437913894653,0.35764437913894653,0.7152887582778931,0.35764437913894653,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0
 end_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.6365708112716675,0.6365708112716675,0.3333333432674408,0.6365708112716675,0.6365708112716675,1.482131917196483e-10,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0
 hottest_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.2813863754272461,0.2631579041481018,0.27892643213272095,1.0,0.27892643213272095,0.01989283785223961,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.2631579041481018
-liverange_size: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.811345100402832,0.7318435907363892,0.5088096261024475,0.7421572804450989,0.8143532276153564,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.2819080352783203
-use_def_density: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.3284657895565033,0.058340102434158325,0.7709200978279114,0.37288200855255127,0.7674928903579712,0.05657143518328667,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0
+liverange_size: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.8106942772865295,0.7343682646751404,0.510564923286438,0.744717538356781,0.8068132996559143,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.2828805446624756
+use_def_density: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.3299224376678467,0.058340102434158325,0.7709200978279114,0.37288200855255127,0.7775528430938721,0.05657143518328667,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0
 max_stage: 0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1
 min_stage: 0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1
 progress: 0.7777777910232544
@@ -304,8 +304,8 @@ hint_weights_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0,1.0,0.0,0.0
 start_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.5,0.5,0.5,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.5
 end_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,1.0,0.5236390233039856,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2.3283064365386963e-10
 hottest_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.2813863754272461,0.0,0.2631579041481018,0.27892643213272095,1.0,0.27892643213272095,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.01989283785223961
-liverange_size: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.811345100402832,0.0,0.7318435907363892,0.5088096261024475,0.7421572804450989,0.8143532276153564,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0
-use_def_density: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.42606985569000244,0.0,0.07567594200372696,1.0,0.48368439078330994,0.9955543875694275,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.07338171452283859
+liverange_size: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.8106942772865295,0.0,0.7343682646751404,0.510564923286438,0.744717538356781,0.8068132996559143,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0
+use_def_density: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.4243086874485016,0.0,0.07503040134906769,0.99146968126297,0.47955840826034546,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.07275574654340744
 max_stage: 0,0,0,0,0,0,0,0,0,1,0,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1
 min_stage: 0,0,0,0,0,0,0,0,0,1,0,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1
 progress: 0.7777777910232544
@@ -328,8 +328,8 @@ hint_weights_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.9982500076293
 start_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.7152887582778931,0.35764437913894653,0.35764437913894653,0.35764437913894653,0.7152887582778931,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.7152887582778931
 end_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.6522180438041687,0.6365708112716675,0.3333333432674408,0.6365708112716675,0.6365708112716675,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.015647225081920624
 hottest_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.2813863754272461,0.2631579041481018,0.27892643213272095,1.0,0.27892643213272095,0.2631579041481018,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.02235277369618416
-liverange_size: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.8219112157821655,0.5714285969734192,0.8334941864013672,0.9145752787590027,0.31660231947898865,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9652509689331055
-use_def_density: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.3284657895565033,0.058340102434158325,0.7709200978279114,0.37288200855255127,0.7674928903579712,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.016097404062747955
+liverange_size: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.8283073902130127,0.575875461101532,0.8399805426597595,0.9100194573402405,0.3190661370754242,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9688715934753418
+use_def_density: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.3299224376678467,0.058340102434158325,0.7709200978279114,0.37288200855255127,0.7775528430938721,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.016164302825927734
 max_stage: 0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4
 min_stage: 0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4
 progress: 0.2222222238779068
@@ -352,8 +352,8 @@ hint_weights_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,1.0,0.0,0.0,0.9
 start_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.7152887582778931,0.35764437913894653,0.35764437913894653,0.35764437913894653,0.7152887582778931,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.7152887582778931
 end_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.6365708112716675,0.6365708112716675,0.3333333432674408,0.6365708112716675,0.6365708112716675,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2.778997310048936e-10
 hottest_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.2813863754272461,0.2631579041481018,0.27892643213272095,1.0,0.27892643213272095,0.2631579041481018,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.02235277369618416
-liverange_size: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.8797762989997864,0.7935694456100464,0.5517241358757019,0.8047530055046082,0.8830382227897644,0.30568498373031616,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0
-use_def_density: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.3284657895565033,0.058340102434158325,0.7709200978279114,0.37288200855255127,0.7674928903579712,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.008228360675275326
+liverange_size: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.8826290965080261,0.7995305061340332,0.5558685660362244,0.8107981085777283,0.8784037828445435,0.3079812228679657,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0
+use_def_density: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.3299224376678467,0.058340102434158325,0.7709200978279114,0.37288200855255127,0.7775528430938721,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.00826177466660738
 max_stage: 0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4
 min_stage: 0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4
 progress: 0.1944444477558136
@@ -376,8 +376,8 @@ hint_weights_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.9434669613838196,
 start_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.7152887582778931,1.0,0.35764437913894653,0.35764437913894653,0.35764437913894653,0.7152887582778931,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.35764437913894653
 end_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.6365708112716675,1.0,1.482131917196483e-10,0.3333333432674408,0.6365708112716675,0.6365708112716675,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2.778997310048936e-10
 hottest_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.2813863754272461,0.2631579041481018,0.2631579041481018,0.27892643213272095,1.0,0.27892643213272095,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.015625
-liverange_size: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.8342907428741455,0.2898806929588318,1.0,0.5231993198394775,0.7631462812423706,0.8373839855194092,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9253203868865967
-use_def_density: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.3284657895565033,1.0,0.058340102434158325,0.7709200978279114,0.37288200855255127,0.7674928903579712,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.015127303078770638
+liverange_size: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.8337028622627258,0.290909081697464,1.0,0.5250554084777832,0.7658536434173584,0.8297117352485657,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9250554442405701
+use_def_density: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.3299224376678467,1.0,0.058340102434158325,0.7709200978279114,0.37288200855255127,0.7775528430938721,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.015127303078770638
 max_stage: 0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4
 min_stage: 0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4
 progress: 0.1388888955116272
@@ -400,8 +400,8 @@ hint_weights_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.07419288158416748
 start_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.35764437913894653,0.35764437913894653,0.35764437913894653,0.35764437913894653,0.7152887582778931,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.7152887582778931
 end_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.6365708112716675,0.6365708112716675,0.3333333432674408,0.6365708112716675,0.6365708112716675,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.6365708112716675
 hottest_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.2813863754272461,0.2631579041481018,0.27892643213272095,1.0,0.27892643213272095,0.2631579041481018,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.02235277369618416
-liverange_size: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.9451456069946289,0.5747572779655457,0.8383495211601257,0.9199029207229614,0.3184466063976288,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9839805960655212
-use_def_density: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.3284657895565033,0.07900823652744293,0.7709200978279114,0.37288200855255127,0.7674928903579712,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.006958406884223223
+liverange_size: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.9521695971488953,0.5838264226913452,0.8515779376029968,0.922583818435669,0.32347139716148376,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9837278127670288
+use_def_density: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.3299224376678467,0.07900823652744293,0.7709200978279114,0.37288200855255127,0.7775528430938721,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.007072303909808397
 max_stage: 0,0,0,0,0,0,0,0,0,4,4,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4
 min_stage: 0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4
 progress: 0.1111111119389534
@@ -424,8 +424,8 @@ hint_weights_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.07419288158416748
 start_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.35764437913894653,0.35764437913894653,0.35764437913894653,0.35764437913894653,0.7152887582778931,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.7152887582778931
 end_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.6365708112716675,0.6365708112716675,0.3333333432674408,0.6365708112716675,0.6365708112716675,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.6365708112716675
 hottest_bb_freq_by_max: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.2813863754272461,0.2631579041481018,0.27892643213272095,1.0,0.27892643213272095,0.2631579041481018,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.02235277369618416
-liverange_size: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.9451456069946289,0.5747572779655457,0.8383495211601257,0.9199029207229614,0.3184466063976288,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9849514365196228
-use_def_density: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.3284657895565033,0.07900823652744293,0.7709200978279114,0.37288200855255127,0.7674928903579712,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.006951410323381424
+liverange_size: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.9521695971488953,0.5838264226913452,0.8515779376029968,0.922583818435669,0.32347139716148376,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9847140312194824
+use_def_density: 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.3299224376678467,0.07900823652744293,0.7709200978279114,0.37288200855255127,0.7775528430938721,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.007065076380968094
 max_stage: 0,0,0,0,0,0,0,0,0,4,4,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4
 min_stage: 0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4
 progress: 0.0833333358168602
diff --git a/llvm/test/CodeGen/MLRegAlloc/Inputs/reference-prio-log-noml.txt b/llvm/test/CodeGen/MLRegAlloc/Inputs/reference-prio-log-noml.txt
index beb0c52059..01b4a3835c 100644
--- a/llvm/test/CodeGen/MLRegAlloc/Inputs/reference-prio-log-noml.txt
+++ b/llvm/test/CodeGen/MLRegAlloc/Inputs/reference-prio-log-noml.txt
@@ -171,7 +171,7 @@ observation: 28
 li_size: 0
 stage: 0
 weight: 0.0
-priority: 2147485184.0
+priority: 2147484928.0
 reward: 0.0
 observation: 29
 li_size: 0
@@ -237,7 +237,7 @@ observation: 39
 li_size: 0
 stage: 0
 weight: 0.0
-priority: 3598.0
+priority: 3534.0
 reward: 0.0
 observation: 40
 li_size: 0
@@ -249,7 +249,7 @@ observation: 41
 li_size: 0
 stage: 0
 weight: 0.0
-priority: 3582.0
+priority: 3518.0
 reward: 0.0
 observation: 42
 li_size: 0
@@ -273,7 +273,7 @@ observation: 45
 li_size: 0
 stage: 0
 weight: 0.0
-priority: 4078.0
+priority: 4046.0
 reward: 0.0
 observation: 46
 li_size: 0
@@ -291,7 +291,7 @@ observation: 48
 li_size: 0
 stage: 0
 weight: 0.0
-priority: 4384.0
+priority: 4304.0
 reward: 0.0
 observation: 49
 li_size: 0
@@ -309,7 +309,7 @@ observation: 51
 li_size: 0
 stage: 0
 weight: 0.0
-priority: 2684358144.0
+priority: 2684357888.0
 reward: 0.0
 observation: 52
 li_size: 0
diff --git a/llvm/test/CodeGen/MLRegAlloc/dev-mode-prio-logging.ll b/llvm/test/CodeGen/MLRegAlloc/dev-mode-prio-logging.ll
index 21bb752788..6b013b55df 100644
--- a/llvm/test/CodeGen/MLRegAlloc/dev-mode-prio-logging.ll
+++ b/llvm/test/CodeGen/MLRegAlloc/dev-mode-prio-logging.ll
@@ -24,5 +24,5 @@
 ; CHECK-NOT: nan
 ; CHECK-LABEL: priority:
 ; NOML-SAME: 2684358144.0
-; ML-SAME: 3599
+; ML-SAME: 3535
 ; CHECK-LABEL: reward:
diff --git a/llvm/test/CodeGen/NVPTX/LoadStoreVectorizer.ll b/llvm/test/CodeGen/NVPTX/LoadStoreVectorizer.ll
index 4f13b6d9d1..868a06e2a8 100644
--- a/llvm/test/CodeGen/NVPTX/LoadStoreVectorizer.ll
+++ b/llvm/test/CodeGen/NVPTX/LoadStoreVectorizer.ll
@@ -52,3 +52,126 @@ define float @ff(ptr %p) {
   %sum = fadd float %sum3, %v4
   ret float %sum
 }
+
+define void @combine_v16i8(ptr noundef align 16 %ptr1, ptr noundef align 16 %ptr2) {
+  ; ENABLED-LABEL: combine_v16i8
+  ; ENABLED: ld.v4.u32
+  %val0 = load i8, ptr %ptr1, align 16
+  %ptr1.1 = getelementptr inbounds i8, ptr %ptr1, i64 1
+  %val1 = load i8, ptr %ptr1.1, align 1
+  %ptr1.2 = getelementptr inbounds i8, ptr %ptr1, i64 2
+  %val2 = load i8, ptr %ptr1.2, align 2
+  %ptr1.3 = getelementptr inbounds i8, ptr %ptr1, i64 3
+  %val3 = load i8, ptr %ptr1.3, align 1
+  %ptr1.4 = getelementptr inbounds i8, ptr %ptr1, i64 4
+  %val4 = load i8, ptr %ptr1.4, align 4
+  %ptr1.5 = getelementptr inbounds i8, ptr %ptr1, i64 5
+  %val5 = load i8, ptr %ptr1.5, align 1
+  %ptr1.6 = getelementptr inbounds i8, ptr %ptr1, i64 6
+  %val6 = load i8, ptr %ptr1.6, align 2
+  %ptr1.7 = getelementptr inbounds i8, ptr %ptr1, i64 7
+  %val7 = load i8, ptr %ptr1.7, align 1
+  %ptr1.8 = getelementptr inbounds i8, ptr %ptr1, i64 8
+  %val8 = load i8, ptr %ptr1.8, align 8
+  %ptr1.9 = getelementptr inbounds i8, ptr %ptr1, i64 9
+  %val9 = load i8, ptr %ptr1.9, align 1
+  %ptr1.10 = getelementptr inbounds i8, ptr %ptr1, i64 10
+  %val10 = load i8, ptr %ptr1.10, align 2
+  %ptr1.11 = getelementptr inbounds i8, ptr %ptr1, i64 11
+  %val11 = load i8, ptr %ptr1.11, align 1
+  %ptr1.12 = getelementptr inbounds i8, ptr %ptr1, i64 12
+  %val12 = load i8, ptr %ptr1.12, align 4
+  %ptr1.13 = getelementptr inbounds i8, ptr %ptr1, i64 13
+  %val13 = load i8, ptr %ptr1.13, align 1
+  %ptr1.14 = getelementptr inbounds i8, ptr %ptr1, i64 14
+  %val14 = load i8, ptr %ptr1.14, align 2
+  %ptr1.15 = getelementptr inbounds i8, ptr %ptr1, i64 15
+  %val15 = load i8, ptr %ptr1.15, align 1
+  %lane0 = zext i8 %val0 to i32
+  %lane1 = zext i8 %val1 to i32
+  %lane2 = zext i8 %val2 to i32
+  %lane3 = zext i8 %val3 to i32
+  %lane4 = zext i8 %val4 to i32
+  %lane5 = zext i8 %val5 to i32
+  %lane6 = zext i8 %val6 to i32
+  %lane7 = zext i8 %val7 to i32
+  %lane8 = zext i8 %val8 to i32
+  %lane9 = zext i8 %val9 to i32
+  %lane10 = zext i8 %val10 to i32
+  %lane11 = zext i8 %val11 to i32
+  %lane12 = zext i8 %val12 to i32
+  %lane13 = zext i8 %val13 to i32
+  %lane14 = zext i8 %val14 to i32
+  %lane15 = zext i8 %val15 to i32
+  %red.1 = add i32 %lane0, %lane1
+  %red.2 = add i32 %red.1, %lane2
+  %red.3 = add i32 %red.2, %lane3
+  %red.4 = add i32 %red.3, %lane4
+  %red.5 = add i32 %red.4, %lane5
+  %red.6 = add i32 %red.5, %lane6
+  %red.7 = add i32 %red.6, %lane7
+  %red.8 = add i32 %red.7, %lane8
+  %red.9 = add i32 %red.8, %lane9
+  %red.10 = add i32 %red.9, %lane10
+  %red.11 = add i32 %red.10, %lane11
+  %red.12 = add i32 %red.11, %lane12
+  %red.13 = add i32 %red.12, %lane13
+  %red.14 = add i32 %red.13, %lane14
+  %red = add i32 %red.14, %lane15
+  store i32 %red, ptr %ptr2, align 4
+  ret void
+}
+
+define void @combine_v8i16(ptr noundef align 16 %ptr1, ptr noundef align 16 %ptr2) {
+  ; ENABLED-LABEL: combine_v8i16
+  ; ENABLED: ld.v4.b32
+  %val0 = load i16, ptr %ptr1, align 16
+  %ptr1.1 = getelementptr inbounds i16, ptr %ptr1, i64 1
+  %val1 = load i16, ptr %ptr1.1, align 2
+  %ptr1.2 = getelementptr inbounds i16, ptr %ptr1, i64 2
+  %val2 = load i16, ptr %ptr1.2, align 4
+  %ptr1.3 = getelementptr inbounds i16, ptr %ptr1, i64 3
+  %val3 = load i16, ptr %ptr1.3, align 2
+  %ptr1.4 = getelementptr inbounds i16, ptr %ptr1, i64 4
+  %val4 = load i16, ptr %ptr1.4, align 4
+  %ptr1.5 = getelementptr inbounds i16, ptr %ptr1, i64 5
+  %val5 = load i16, ptr %ptr1.5, align 2
+  %ptr1.6 = getelementptr inbounds i16, ptr %ptr1, i64 6
+  %val6 = load i16, ptr %ptr1.6, align 4
+  %ptr1.7 = getelementptr inbounds i16, ptr %ptr1, i64 7
+  %val7 = load i16, ptr %ptr1.7, align 2
+  %lane0 = zext i16 %val0 to i32
+  %lane1 = zext i16 %val1 to i32
+  %lane2 = zext i16 %val2 to i32
+  %lane3 = zext i16 %val3 to i32
+  %lane4 = zext i16 %val4 to i32
+  %lane5 = zext i16 %val5 to i32
+  %lane6 = zext i16 %val6 to i32
+  %lane7 = zext i16 %val7 to i32
+  %red.1 = add i32 %lane0, %lane1
+  %red.2 = add i32 %red.1, %lane2
+  %red.3 = add i32 %red.2, %lane3
+  %red.4 = add i32 %red.3, %lane4
+  %red.5 = add i32 %red.4, %lane5
+  %red.6 = add i32 %red.5, %lane6
+  %red = add i32 %red.6, %lane7
+  store i32 %red, ptr %ptr2, align 4
+  ret void
+}
+
+define void @combine_v4i32(ptr noundef align 16 %ptr1, ptr noundef align 16 %ptr2) {
+  ; ENABLED-LABEL: combine_v4i32
+  ; ENABLED: ld.v4.u32
+  %val0 = load i32, ptr %ptr1, align 16
+  %ptr1.1 = getelementptr inbounds i32, ptr %ptr1, i64 1
+  %val1 = load i32, ptr %ptr1.1, align 4
+  %ptr1.2 = getelementptr inbounds i32, ptr %ptr1, i64 2
+  %val2 = load i32, ptr %ptr1.2, align 8
+  %ptr1.3 = getelementptr inbounds i32, ptr %ptr1, i64 3
+  %val3 = load i32, ptr %ptr1.3, align 4
+  %red.1 = add i32 %val0, %val1
+  %red.2 = add i32 %red.1, %val2
+  %red = add i32 %red.2, %val3
+  store i32 %red, ptr %ptr2, align 4
+  ret void
+}
diff --git a/llvm/test/CodeGen/NVPTX/f16x2-instructions.ll b/llvm/test/CodeGen/NVPTX/f16x2-instructions.ll
index 18788c776f..464b3a7548 100644
--- a/llvm/test/CodeGen/NVPTX/f16x2-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/f16x2-instructions.ll
@@ -1319,10 +1319,8 @@ define <2 x half> @test_copysign_f64(<2 x half> %a, <2 x double> %b) #0 {
 ; CHECK-DAG:  and.b16         [[BX1:%rs[0-9]+]], [[B1]], -32768;
 ; CHECK-DAG:  or.b16          [[R0:%rs[0-9]+]], [[AX0]], [[BX0]];
 ; CHECK-DAG:  or.b16          [[R1:%rs[0-9]+]], [[AX1]], [[BX1]];
-; CHECK-DAG:  mov.b32         [[R:%r[0-9]+]], {[[R0]], [[R1]]}
-; CHECK:      mov.b32         {[[RX0:%rs[0-9]+]], [[RX1:%rs[0-9]+]]}, [[R]]
-; CHECK-DAG:  cvt.f32.f16     [[XR0:%f[0-9]+]], [[RX0]];
-; CHECK-DAG:  cvt.f32.f16     [[XR1:%f[0-9]+]], [[RX1]];
+; CHECK-DAG:  cvt.f32.f16     [[XR0:%f[0-9]+]], [[R0]];
+; CHECK-DAG:  cvt.f32.f16     [[XR1:%f[0-9]+]], [[R1]];
 ; CHECK:      st.param.v2.f32 [func_retval0+0], {[[XR0]], [[XR1]]};
 ; CHECK:      ret;
 define <2 x float> @test_copysign_extended(<2 x half> %a, <2 x half> %b) #0 {
diff --git a/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll b/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll
index fd48313ad6..ddad374a4d 100644
--- a/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll
@@ -1269,4 +1269,158 @@ define <4 x i8> @test_fptoui_2xhalf_to_2xi8(<4 x half> %a) #0 {
   ret <4 x i8> %r
 }
 
+define void @test_srem_v4i8(ptr %a, ptr %b, ptr %c) {
+; CHECK-LABEL: test_srem_v4i8(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<13>;
+; CHECK-NEXT:    .reg .b32 %r<18>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0: // %entry
+; CHECK-NEXT:    ld.param.u64 %rd3, [test_srem_v4i8_param_2];
+; CHECK-NEXT:    ld.param.u64 %rd2, [test_srem_v4i8_param_1];
+; CHECK-NEXT:    ld.param.u64 %rd1, [test_srem_v4i8_param_0];
+; CHECK-NEXT:    ld.u32 %r1, [%rd1];
+; CHECK-NEXT:    ld.u32 %r2, [%rd2];
+; CHECK-NEXT:    bfe.s32 %r3, %r2, 0, 8;
+; CHECK-NEXT:    cvt.s8.s32 %rs1, %r3;
+; CHECK-NEXT:    bfe.s32 %r4, %r1, 0, 8;
+; CHECK-NEXT:    cvt.s8.s32 %rs2, %r4;
+; CHECK-NEXT:    rem.s16 %rs3, %rs2, %rs1;
+; CHECK-NEXT:    cvt.u32.u16 %r5, %rs3;
+; CHECK-NEXT:    bfe.s32 %r6, %r2, 8, 8;
+; CHECK-NEXT:    cvt.s8.s32 %rs4, %r6;
+; CHECK-NEXT:    bfe.s32 %r7, %r1, 8, 8;
+; CHECK-NEXT:    cvt.s8.s32 %rs5, %r7;
+; CHECK-NEXT:    rem.s16 %rs6, %rs5, %rs4;
+; CHECK-NEXT:    cvt.u32.u16 %r8, %rs6;
+; CHECK-NEXT:    bfi.b32 %r9, %r8, %r5, 8, 8;
+; CHECK-NEXT:    bfe.s32 %r10, %r2, 16, 8;
+; CHECK-NEXT:    cvt.s8.s32 %rs7, %r10;
+; CHECK-NEXT:    bfe.s32 %r11, %r1, 16, 8;
+; CHECK-NEXT:    cvt.s8.s32 %rs8, %r11;
+; CHECK-NEXT:    rem.s16 %rs9, %rs8, %rs7;
+; CHECK-NEXT:    cvt.u32.u16 %r12, %rs9;
+; CHECK-NEXT:    bfi.b32 %r13, %r12, %r9, 16, 8;
+; CHECK-NEXT:    bfe.s32 %r14, %r2, 24, 8;
+; CHECK-NEXT:    cvt.s8.s32 %rs10, %r14;
+; CHECK-NEXT:    bfe.s32 %r15, %r1, 24, 8;
+; CHECK-NEXT:    cvt.s8.s32 %rs11, %r15;
+; CHECK-NEXT:    rem.s16 %rs12, %rs11, %rs10;
+; CHECK-NEXT:    cvt.u32.u16 %r16, %rs12;
+; CHECK-NEXT:    bfi.b32 %r17, %r16, %r13, 24, 8;
+; CHECK-NEXT:    st.u32 [%rd3], %r17;
+; CHECK-NEXT:    ret;
+entry:
+  %t57 = load <4 x i8>, ptr %a, align 4
+  %t59 = load <4 x i8>, ptr %b, align 4
+  %x = srem <4 x i8> %t57, %t59
+  store <4 x i8> %x, ptr %c, align 4
+  ret void
+}
+
+;; v3i8 lowering, especially for unaligned loads is terrible. We end up doing
+;; tons of pointless scalar_to_vector/bitcast/extract_elt on v2i16/v4i8, which
+;; is further complicated by LLVM trying to use i16 as an intermediate type,
+;; because we don't have i8 registers. It's a mess.
+;; Ideally we want to split it into element-wise ops, but legalizer can't handle
+;; odd-sized vectors.  TL;DR; don't use odd-sized vectors of v8.
+define void @test_srem_v3i8(ptr %a, ptr %b, ptr %c) {
+; CHECK-LABEL: test_srem_v3i8(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<20>;
+; CHECK-NEXT:    .reg .b32 %r<16>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0: // %entry
+; CHECK-NEXT:    ld.param.u64 %rd3, [test_srem_v3i8_param_2];
+; CHECK-NEXT:    ld.param.u64 %rd2, [test_srem_v3i8_param_1];
+; CHECK-NEXT:    ld.param.u64 %rd1, [test_srem_v3i8_param_0];
+; CHECK-NEXT:    ld.u8 %rs1, [%rd1];
+; CHECK-NEXT:    ld.u8 %rs2, [%rd1+1];
+; CHECK-NEXT:    shl.b16 %rs3, %rs2, 8;
+; CHECK-NEXT:    or.b16 %rs4, %rs3, %rs1;
+; CHECK-NEXT:    cvt.u32.u16 %r1, %rs4;
+; CHECK-NEXT:    ld.s8 %rs5, [%rd1+2];
+; CHECK-NEXT:    ld.u8 %rs6, [%rd2];
+; CHECK-NEXT:    ld.u8 %rs7, [%rd2+1];
+; CHECK-NEXT:    shl.b16 %rs8, %rs7, 8;
+; CHECK-NEXT:    or.b16 %rs9, %rs8, %rs6;
+; CHECK-NEXT:    cvt.u32.u16 %r3, %rs9;
+; CHECK-NEXT:    ld.s8 %rs10, [%rd2+2];
+; CHECK-NEXT:    bfe.s32 %r5, %r3, 0, 8;
+; CHECK-NEXT:    cvt.s8.s32 %rs11, %r5;
+; CHECK-NEXT:    bfe.s32 %r6, %r1, 0, 8;
+; CHECK-NEXT:    cvt.s8.s32 %rs12, %r6;
+; CHECK-NEXT:    rem.s16 %rs13, %rs12, %rs11;
+; CHECK-NEXT:    cvt.u32.u16 %r7, %rs13;
+; CHECK-NEXT:    bfe.s32 %r8, %r3, 8, 8;
+; CHECK-NEXT:    cvt.s8.s32 %rs14, %r8;
+; CHECK-NEXT:    bfe.s32 %r9, %r1, 8, 8;
+; CHECK-NEXT:    cvt.s8.s32 %rs15, %r9;
+; CHECK-NEXT:    rem.s16 %rs16, %rs15, %rs14;
+; CHECK-NEXT:    cvt.u32.u16 %r10, %rs16;
+; CHECK-NEXT:    bfi.b32 %r11, %r10, %r7, 8, 8;
+; CHECK-NEXT:    // implicit-def: %r13
+; CHECK-NEXT:    bfi.b32 %r12, %r13, %r11, 16, 8;
+; CHECK-NEXT:    // implicit-def: %r15
+; CHECK-NEXT:    bfi.b32 %r14, %r15, %r12, 24, 8;
+; CHECK-NEXT:    rem.s16 %rs17, %rs5, %rs10;
+; CHECK-NEXT:    cvt.u16.u32 %rs18, %r14;
+; CHECK-NEXT:    st.u8 [%rd3], %rs18;
+; CHECK-NEXT:    shr.u16 %rs19, %rs18, 8;
+; CHECK-NEXT:    st.u8 [%rd3+1], %rs19;
+; CHECK-NEXT:    st.u8 [%rd3+2], %rs17;
+; CHECK-NEXT:    ret;
+entry:
+  %t57 = load <3 x i8>, ptr %a, align 1
+  %t59 = load <3 x i8>, ptr %b, align 1
+  %x = srem <3 x i8> %t57, %t59
+  store <3 x i8> %x, ptr %c, align 1
+  ret void
+}
+
+define void @test_sext_v4i1_to_v4i8(ptr %a, ptr %b, ptr %c) {
+; CHECK-LABEL: test_sext_v4i1_to_v4i8(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<5>;
+; CHECK-NEXT:    .reg .b32 %r<18>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0: // %entry
+; CHECK-NEXT:    ld.param.u64 %rd3, [test_sext_v4i1_to_v4i8_param_2];
+; CHECK-NEXT:    ld.param.u64 %rd2, [test_sext_v4i1_to_v4i8_param_1];
+; CHECK-NEXT:    ld.param.u64 %rd1, [test_sext_v4i1_to_v4i8_param_0];
+; CHECK-NEXT:    ld.u32 %r1, [%rd1];
+; CHECK-NEXT:    ld.u32 %r2, [%rd2];
+; CHECK-NEXT:    bfe.s32 %r3, %r2, 24, 8;
+; CHECK-NEXT:    bfe.s32 %r4, %r1, 24, 8;
+; CHECK-NEXT:    setp.hi.u32 %p1, %r4, %r3;
+; CHECK-NEXT:    bfe.s32 %r5, %r2, 16, 8;
+; CHECK-NEXT:    bfe.s32 %r6, %r1, 16, 8;
+; CHECK-NEXT:    setp.hi.u32 %p2, %r6, %r5;
+; CHECK-NEXT:    bfe.s32 %r7, %r2, 8, 8;
+; CHECK-NEXT:    bfe.s32 %r8, %r1, 8, 8;
+; CHECK-NEXT:    setp.hi.u32 %p3, %r8, %r7;
+; CHECK-NEXT:    bfe.s32 %r9, %r2, 0, 8;
+; CHECK-NEXT:    bfe.s32 %r10, %r1, 0, 8;
+; CHECK-NEXT:    setp.hi.u32 %p4, %r10, %r9;
+; CHECK-NEXT:    selp.s32 %r11, -1, 0, %p4;
+; CHECK-NEXT:    selp.s32 %r12, -1, 0, %p3;
+; CHECK-NEXT:    bfi.b32 %r13, %r12, %r11, 8, 8;
+; CHECK-NEXT:    selp.s32 %r14, -1, 0, %p2;
+; CHECK-NEXT:    bfi.b32 %r15, %r14, %r13, 16, 8;
+; CHECK-NEXT:    selp.s32 %r16, -1, 0, %p1;
+; CHECK-NEXT:    bfi.b32 %r17, %r16, %r15, 24, 8;
+; CHECK-NEXT:    st.u32 [%rd3], %r17;
+; CHECK-NEXT:    ret;
+entry:
+  %t1 = load <4 x i8>, ptr %a, align 4
+  %t2 = load <4 x i8>, ptr %b, align 4
+  %t5 = icmp ugt <4 x i8> %t1, %t2
+  %t6 = sext <4 x i1> %t5 to <4 x i8>
+  store <4 x i8> %t6, ptr %c, align 4
+  ret void
+}
+
 attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/NVPTX/param-load-store.ll b/llvm/test/CodeGen/NVPTX/param-load-store.ll
index b4208c691c..c14dc88431 100644
--- a/llvm/test/CodeGen/NVPTX/param-load-store.ll
+++ b/llvm/test/CodeGen/NVPTX/param-load-store.ll
@@ -364,10 +364,6 @@ define <4 x i16> @test_v4i16(<4 x i16> %a) {
 ; CHECK-NEXT: .param .align 16 .b8 test_v5i16_param_0[16]
 ; CHECK-DAG:  ld.param.u16    [[E4:%rs[0-9]+]], [test_v5i16_param_0+8];
 ; CHECK-DAG:  ld.param.v4.u16 {[[E0:%rs[0-9]+]], [[E1:%rs[0-9]+]], [[E2:%rs[0-9]+]], [[E3:%rs[0-9]+]]}, [test_v5i16_param_0]
-; CHECK-DAG:	mov.b32 	[[R0:%r[0-9]+]], {[[E0]], [[E1]]};
-; CHECK-DAG:	mov.b32 	{[[E0:%rs[0-9]+]], [[E1:%rs[0-9]+]]}, [[R0]];
-; CHECK-DAG:	mov.b32 	[[R1:%r[0-9]+]], {[[E2]], [[E3]]};
-; CHECK-DAG:	mov.b32 	{[[E2:%rs[0-9]+]], [[E3:%rs[0-9]+]]}, [[R1]];
 ; CHECK:      .param .align 16 .b8 param0[16];
 ; CHECK-DAG:  st.param.v4.b16 [param0+0], {[[E0]], [[E1]], [[E2]], [[E3]]};
 ; CHECK-DAG:  st.param.b16    [param0+8], [[E4]];
@@ -496,7 +492,6 @@ define <4 x half> @test_v4f16(<4 x half> %a) {
 ; CHECK-LABEL: test_v5f16(
 ; CHECK:      .param .align 16 .b8 test_v5f16_param_0[16]
 ; CHECK-DAG:  ld.param.v4.b16  {[[E0:%rs[0-9]+]], [[E1:%rs[0-9]+]], [[E2:%rs[0-9]+]], [[E3:%rs[0-9]+]]}, [test_v5f16_param_0];
-; CHECK-DAG:  mov.b32         {[[E0:%rs[0-9]+]], [[E1:%rs[0-9]+]]}, [[HH01]];
 ; CHECK-DAG:  ld.param.b16    [[E4:%rs[0-9]+]], [test_v5f16_param_0+8];
 ; CHECK:      .param .align 16 .b8 param0[16];
 ; CHECK-DAG:  st.param.v4.b16 [param0+0],
diff --git a/llvm/test/CodeGen/PowerPC/and-extend-combine.ll b/llvm/test/CodeGen/PowerPC/and-extend-combine.ll
index 301a28ac64..0b30d48912 100644
--- a/llvm/test/CodeGen/PowerPC/and-extend-combine.ll
+++ b/llvm/test/CodeGen/PowerPC/and-extend-combine.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
 ; RUN: llc < %s -mtriple=powerpc64le-unknown-unknown -ppc-asm-full-reg-names \
 ; RUN:   -mcpu=pwr8 -verify-machineinstrs | FileCheck %s
+
 define dso_local ptr @foo(i32 noundef zeroext %arg, ptr nocapture noundef readonly %arg1, ptr noundef writeonly %arg2) local_unnamed_addr {
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0: # %bb
@@ -21,3 +22,18 @@ bb:
   %i8 = getelementptr inbounds i8, ptr %arg2, i64 %i7
   ret ptr %i8
 }
+
+define void @pr68783(i32 %x, ptr %p) {
+; CHECK-LABEL: pr68783:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    rlwinm r3, r3, 31, 24, 31
+; CHECK-NEXT:    li r5, 0
+; CHECK-NEXT:    sth r5, 4(r4)
+; CHECK-NEXT:    stw r3, 0(r4)
+; CHECK-NEXT:    blr
+  %lshr = lshr i32 %x, 1
+  %zext = zext i32 %lshr to i48
+  %and = and i48 %zext, 255
+  store i48 %and, ptr %p
+  ret void
+}
diff --git a/llvm/test/CodeGen/PowerPC/atomics-i128.ll b/llvm/test/CodeGen/PowerPC/atomics-i128.ll
index 66d727caed..f5422a9b7b 100644
--- a/llvm/test/CodeGen/PowerPC/atomics-i128.ll
+++ b/llvm/test/CodeGen/PowerPC/atomics-i128.ll
@@ -986,10 +986,7 @@ define i128 @cas_weak_acquire_acquire(ptr %a, i128 %cmp, i128 %new) {
 ; CHECK-NEXT:    mr r10, r6
 ; CHECK-NEXT:    stqcx. r10, 0, r3
 ; CHECK-NEXT:    bne cr0, .LBB7_1
-; CHECK-NEXT:    b .LBB7_4
 ; CHECK-NEXT:  .LBB7_3: # %entry
-; CHECK-NEXT:    stqcx. r8, 0, r3
-; CHECK-NEXT:  .LBB7_4: # %entry
 ; CHECK-NEXT:    lwsync
 ; CHECK-NEXT:    mr r3, r8
 ; CHECK-NEXT:    mr r4, r9
@@ -1033,10 +1030,7 @@ define i128 @cas_weak_acquire_acquire(ptr %a, i128 %cmp, i128 %new) {
 ; LE-PWR8-NEXT:    mr r10, r7
 ; LE-PWR8-NEXT:    stqcx. r10, 0, r3
 ; LE-PWR8-NEXT:    bne cr0, .LBB7_1
-; LE-PWR8-NEXT:    b .LBB7_4
 ; LE-PWR8-NEXT:  .LBB7_3: # %entry
-; LE-PWR8-NEXT:    stqcx. r8, 0, r3
-; LE-PWR8-NEXT:  .LBB7_4: # %entry
 ; LE-PWR8-NEXT:    lwsync
 ; LE-PWR8-NEXT:    mr r3, r9
 ; LE-PWR8-NEXT:    mr r4, r8
@@ -1057,10 +1051,7 @@ define i128 @cas_weak_acquire_acquire(ptr %a, i128 %cmp, i128 %new) {
 ; AIX64-PWR8-NEXT:    mr r10, r6
 ; AIX64-PWR8-NEXT:    stqcx. r10, 0, r3
 ; AIX64-PWR8-NEXT:    bne cr0, L..BB7_1
-; AIX64-PWR8-NEXT:    b L..BB7_4
 ; AIX64-PWR8-NEXT:  L..BB7_3: # %entry
-; AIX64-PWR8-NEXT:    stqcx. r8, 0, r3
-; AIX64-PWR8-NEXT:  L..BB7_4: # %entry
 ; AIX64-PWR8-NEXT:    lwsync
 ; AIX64-PWR8-NEXT:    mr r3, r8
 ; AIX64-PWR8-NEXT:    mr r4, r9
@@ -1121,10 +1112,7 @@ define i128 @cas_weak_release_monotonic(ptr %a, i128 %cmp, i128 %new) {
 ; CHECK-NEXT:    mr r10, r6
 ; CHECK-NEXT:    stqcx. r10, 0, r3
 ; CHECK-NEXT:    bne cr0, .LBB8_1
-; CHECK-NEXT:    b .LBB8_4
 ; CHECK-NEXT:  .LBB8_3: # %entry
-; CHECK-NEXT:    stqcx. r8, 0, r3
-; CHECK-NEXT:  .LBB8_4: # %entry
 ; CHECK-NEXT:    mr r3, r8
 ; CHECK-NEXT:    mr r4, r9
 ; CHECK-NEXT:    blr
@@ -1168,10 +1156,7 @@ define i128 @cas_weak_release_monotonic(ptr %a, i128 %cmp, i128 %new) {
 ; LE-PWR8-NEXT:    mr r10, r7
 ; LE-PWR8-NEXT:    stqcx. r10, 0, r3
 ; LE-PWR8-NEXT:    bne cr0, .LBB8_1
-; LE-PWR8-NEXT:    b .LBB8_4
 ; LE-PWR8-NEXT:  .LBB8_3: # %entry
-; LE-PWR8-NEXT:    stqcx. r8, 0, r3
-; LE-PWR8-NEXT:  .LBB8_4: # %entry
 ; LE-PWR8-NEXT:    mr r3, r9
 ; LE-PWR8-NEXT:    mr r4, r8
 ; LE-PWR8-NEXT:    blr
@@ -1192,10 +1177,7 @@ define i128 @cas_weak_release_monotonic(ptr %a, i128 %cmp, i128 %new) {
 ; AIX64-PWR8-NEXT:    mr r10, r6
 ; AIX64-PWR8-NEXT:    stqcx. r10, 0, r3
 ; AIX64-PWR8-NEXT:    bne cr0, L..BB8_1
-; AIX64-PWR8-NEXT:    b L..BB8_4
 ; AIX64-PWR8-NEXT:  L..BB8_3: # %entry
-; AIX64-PWR8-NEXT:    stqcx. r8, 0, r3
-; AIX64-PWR8-NEXT:  L..BB8_4: # %entry
 ; AIX64-PWR8-NEXT:    mr r3, r8
 ; AIX64-PWR8-NEXT:    mr r4, r9
 ; AIX64-PWR8-NEXT:    blr
@@ -1255,10 +1237,7 @@ define i128 @cas_sc_sc(ptr %a, i128 %cmp, i128 %new) {
 ; CHECK-NEXT:    mr r10, r6
 ; CHECK-NEXT:    stqcx. r10, 0, r3
 ; CHECK-NEXT:    bne cr0, .LBB9_1
-; CHECK-NEXT:    b .LBB9_4
 ; CHECK-NEXT:  .LBB9_3: # %entry
-; CHECK-NEXT:    stqcx. r8, 0, r3
-; CHECK-NEXT:  .LBB9_4: # %entry
 ; CHECK-NEXT:    lwsync
 ; CHECK-NEXT:    mr r3, r8
 ; CHECK-NEXT:    mr r4, r9
@@ -1303,10 +1282,7 @@ define i128 @cas_sc_sc(ptr %a, i128 %cmp, i128 %new) {
 ; LE-PWR8-NEXT:    mr r10, r7
 ; LE-PWR8-NEXT:    stqcx. r10, 0, r3
 ; LE-PWR8-NEXT:    bne cr0, .LBB9_1
-; LE-PWR8-NEXT:    b .LBB9_4
 ; LE-PWR8-NEXT:  .LBB9_3: # %entry
-; LE-PWR8-NEXT:    stqcx. r8, 0, r3
-; LE-PWR8-NEXT:  .LBB9_4: # %entry
 ; LE-PWR8-NEXT:    lwsync
 ; LE-PWR8-NEXT:    mr r3, r9
 ; LE-PWR8-NEXT:    mr r4, r8
@@ -1328,10 +1304,7 @@ define i128 @cas_sc_sc(ptr %a, i128 %cmp, i128 %new) {
 ; AIX64-PWR8-NEXT:    mr r10, r6
 ; AIX64-PWR8-NEXT:    stqcx. r10, 0, r3
 ; AIX64-PWR8-NEXT:    bne cr0, L..BB9_1
-; AIX64-PWR8-NEXT:    b L..BB9_4
 ; AIX64-PWR8-NEXT:  L..BB9_3: # %entry
-; AIX64-PWR8-NEXT:    stqcx. r8, 0, r3
-; AIX64-PWR8-NEXT:  L..BB9_4: # %entry
 ; AIX64-PWR8-NEXT:    lwsync
 ; AIX64-PWR8-NEXT:    mr r3, r8
 ; AIX64-PWR8-NEXT:    mr r4, r9
@@ -1392,10 +1365,7 @@ define i128 @cas_acqrel_acquire(ptr %a, i128 %cmp, i128 %new) {
 ; CHECK-NEXT:    mr r10, r6
 ; CHECK-NEXT:    stqcx. r10, 0, r3
 ; CHECK-NEXT:    bne cr0, .LBB10_1
-; CHECK-NEXT:    b .LBB10_4
 ; CHECK-NEXT:  .LBB10_3: # %entry
-; CHECK-NEXT:    stqcx. r8, 0, r3
-; CHECK-NEXT:  .LBB10_4: # %entry
 ; CHECK-NEXT:    lwsync
 ; CHECK-NEXT:    mr r3, r8
 ; CHECK-NEXT:    mr r4, r9
@@ -1440,10 +1410,7 @@ define i128 @cas_acqrel_acquire(ptr %a, i128 %cmp, i128 %new) {
 ; LE-PWR8-NEXT:    mr r10, r7
 ; LE-PWR8-NEXT:    stqcx. r10, 0, r3
 ; LE-PWR8-NEXT:    bne cr0, .LBB10_1
-; LE-PWR8-NEXT:    b .LBB10_4
 ; LE-PWR8-NEXT:  .LBB10_3: # %entry
-; LE-PWR8-NEXT:    stqcx. r8, 0, r3
-; LE-PWR8-NEXT:  .LBB10_4: # %entry
 ; LE-PWR8-NEXT:    lwsync
 ; LE-PWR8-NEXT:    mr r3, r9
 ; LE-PWR8-NEXT:    mr r4, r8
@@ -1465,10 +1432,7 @@ define i128 @cas_acqrel_acquire(ptr %a, i128 %cmp, i128 %new) {
 ; AIX64-PWR8-NEXT:    mr r10, r6
 ; AIX64-PWR8-NEXT:    stqcx. r10, 0, r3
 ; AIX64-PWR8-NEXT:    bne cr0, L..BB10_1
-; AIX64-PWR8-NEXT:    b L..BB10_4
 ; AIX64-PWR8-NEXT:  L..BB10_3: # %entry
-; AIX64-PWR8-NEXT:    stqcx. r8, 0, r3
-; AIX64-PWR8-NEXT:  L..BB10_4: # %entry
 ; AIX64-PWR8-NEXT:    lwsync
 ; AIX64-PWR8-NEXT:    mr r3, r8
 ; AIX64-PWR8-NEXT:    mr r4, r9
@@ -1529,10 +1493,7 @@ define i1 @cas_acqrel_acquire_check_succ(ptr %a, i128 %cmp, i128 %new) {
 ; CHECK-NEXT:    mr r10, r6
 ; CHECK-NEXT:    stqcx. r10, 0, r3
 ; CHECK-NEXT:    bne cr0, .LBB11_1
-; CHECK-NEXT:    b .LBB11_4
 ; CHECK-NEXT:  .LBB11_3: # %entry
-; CHECK-NEXT:    stqcx. r8, 0, r3
-; CHECK-NEXT:  .LBB11_4: # %entry
 ; CHECK-NEXT:    lwsync
 ; CHECK-NEXT:    xor r3, r4, r8
 ; CHECK-NEXT:    xor r4, r5, r9
@@ -1578,10 +1539,7 @@ define i1 @cas_acqrel_acquire_check_succ(ptr %a, i128 %cmp, i128 %new) {
 ; LE-PWR8-NEXT:    mr r10, r7
 ; LE-PWR8-NEXT:    stqcx. r10, 0, r3
 ; LE-PWR8-NEXT:    bne cr0, .LBB11_1
-; LE-PWR8-NEXT:    b .LBB11_4
 ; LE-PWR8-NEXT:  .LBB11_3: # %entry
-; LE-PWR8-NEXT:    stqcx. r8, 0, r3
-; LE-PWR8-NEXT:  .LBB11_4: # %entry
 ; LE-PWR8-NEXT:    lwsync
 ; LE-PWR8-NEXT:    xor r3, r5, r8
 ; LE-PWR8-NEXT:    xor r4, r4, r9
@@ -1606,10 +1564,7 @@ define i1 @cas_acqrel_acquire_check_succ(ptr %a, i128 %cmp, i128 %new) {
 ; AIX64-PWR8-NEXT:    mr r10, r6
 ; AIX64-PWR8-NEXT:    stqcx. r10, 0, r3
 ; AIX64-PWR8-NEXT:    bne cr0, L..BB11_1
-; AIX64-PWR8-NEXT:    b L..BB11_4
 ; AIX64-PWR8-NEXT:  L..BB11_3: # %entry
-; AIX64-PWR8-NEXT:    stqcx. r8, 0, r3
-; AIX64-PWR8-NEXT:  L..BB11_4: # %entry
 ; AIX64-PWR8-NEXT:    lwsync
 ; AIX64-PWR8-NEXT:    xor r3, r4, r8
 ; AIX64-PWR8-NEXT:    xor r4, r5, r9
@@ -1651,3 +1606,132 @@ entry:
   %1 = extractvalue { i128, i1 } %0, 1
   ret i1 %1
 }
+
+;; TODO: Optimize CAS at exit block when bool value is returned.
+define i1 @bool_cas_weak_acquire_acquire(ptr %a, i128 %cmp, i128 %new) {
+; CHECK-LABEL: bool_cas_weak_acquire_acquire:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:  .LBB12_1: # %entry
+; CHECK-NEXT:    #
+; CHECK-NEXT:    lqarx r8, 0, r3
+; CHECK-NEXT:    xor r11, r9, r5
+; CHECK-NEXT:    xor r10, r8, r4
+; CHECK-NEXT:    or. r11, r11, r10
+; CHECK-NEXT:    bne cr0, .LBB12_3
+; CHECK-NEXT:  # %bb.2: # %entry
+; CHECK-NEXT:    #
+; CHECK-NEXT:    mr r11, r7
+; CHECK-NEXT:    mr r10, r6
+; CHECK-NEXT:    stqcx. r10, 0, r3
+; CHECK-NEXT:    bne cr0, .LBB12_1
+; CHECK-NEXT:  .LBB12_3: # %entry
+; CHECK-NEXT:    lwsync
+; CHECK-NEXT:    xor r3, r4, r8
+; CHECK-NEXT:    xor r4, r5, r9
+; CHECK-NEXT:    or r3, r4, r3
+; CHECK-NEXT:    cntlzd r3, r3
+; CHECK-NEXT:    rldicl r3, r3, 58, 63
+; CHECK-NEXT:    blr
+;
+; PWR7-LABEL: bool_cas_weak_acquire_acquire:
+; PWR7:       # %bb.0: # %entry
+; PWR7-NEXT:    mflr r0
+; PWR7-NEXT:    stdu r1, -128(r1)
+; PWR7-NEXT:    std r0, 144(r1)
+; PWR7-NEXT:    .cfi_def_cfa_offset 128
+; PWR7-NEXT:    .cfi_offset lr, 16
+; PWR7-NEXT:    std r5, 120(r1)
+; PWR7-NEXT:    std r4, 112(r1)
+; PWR7-NEXT:    addi r4, r1, 112
+; PWR7-NEXT:    mr r5, r6
+; PWR7-NEXT:    mr r6, r7
+; PWR7-NEXT:    li r7, 2
+; PWR7-NEXT:    li r8, 2
+; PWR7-NEXT:    bl __atomic_compare_exchange_16
+; PWR7-NEXT:    nop
+; PWR7-NEXT:    addi r1, r1, 128
+; PWR7-NEXT:    ld r0, 16(r1)
+; PWR7-NEXT:    mtlr r0
+; PWR7-NEXT:    blr
+;
+; LE-PWR8-LABEL: bool_cas_weak_acquire_acquire:
+; LE-PWR8:       # %bb.0: # %entry
+; LE-PWR8-NEXT:  .LBB12_1: # %entry
+; LE-PWR8-NEXT:    #
+; LE-PWR8-NEXT:    lqarx r8, 0, r3
+; LE-PWR8-NEXT:    xor r11, r9, r4
+; LE-PWR8-NEXT:    xor r10, r8, r5
+; LE-PWR8-NEXT:    or. r11, r11, r10
+; LE-PWR8-NEXT:    bne cr0, .LBB12_3
+; LE-PWR8-NEXT:  # %bb.2: # %entry
+; LE-PWR8-NEXT:    #
+; LE-PWR8-NEXT:    mr r11, r6
+; LE-PWR8-NEXT:    mr r10, r7
+; LE-PWR8-NEXT:    stqcx. r10, 0, r3
+; LE-PWR8-NEXT:    bne cr0, .LBB12_1
+; LE-PWR8-NEXT:  .LBB12_3: # %entry
+; LE-PWR8-NEXT:    lwsync
+; LE-PWR8-NEXT:    xor r3, r5, r8
+; LE-PWR8-NEXT:    xor r4, r4, r9
+; LE-PWR8-NEXT:    or r3, r4, r3
+; LE-PWR8-NEXT:    cntlzd r3, r3
+; LE-PWR8-NEXT:    rldicl r3, r3, 58, 63
+; LE-PWR8-NEXT:    blr
+;
+; AIX64-PWR8-LABEL: bool_cas_weak_acquire_acquire:
+; AIX64-PWR8:       # %bb.0: # %entry
+; AIX64-PWR8-NEXT:  L..BB12_1: # %entry
+; AIX64-PWR8-NEXT:    #
+; AIX64-PWR8-NEXT:    lqarx r8, 0, r3
+; AIX64-PWR8-NEXT:    xor r11, r9, r5
+; AIX64-PWR8-NEXT:    xor r10, r8, r4
+; AIX64-PWR8-NEXT:    or. r11, r11, r10
+; AIX64-PWR8-NEXT:    bne cr0, L..BB12_3
+; AIX64-PWR8-NEXT:  # %bb.2: # %entry
+; AIX64-PWR8-NEXT:    #
+; AIX64-PWR8-NEXT:    mr r11, r7
+; AIX64-PWR8-NEXT:    mr r10, r6
+; AIX64-PWR8-NEXT:    stqcx. r10, 0, r3
+; AIX64-PWR8-NEXT:    bne cr0, L..BB12_1
+; AIX64-PWR8-NEXT:  L..BB12_3: # %entry
+; AIX64-PWR8-NEXT:    lwsync
+; AIX64-PWR8-NEXT:    xor r3, r4, r8
+; AIX64-PWR8-NEXT:    xor r4, r5, r9
+; AIX64-PWR8-NEXT:    or r3, r4, r3
+; AIX64-PWR8-NEXT:    cntlzd r3, r3
+; AIX64-PWR8-NEXT:    rldicl r3, r3, 58, 63
+; AIX64-PWR8-NEXT:    blr
+;
+; PPC-PWR8-LABEL: bool_cas_weak_acquire_acquire:
+; PPC-PWR8:       # %bb.0: # %entry
+; PPC-PWR8-NEXT:    mflr r0
+; PPC-PWR8-NEXT:    stwu r1, -48(r1)
+; PPC-PWR8-NEXT:    stw r0, 52(r1)
+; PPC-PWR8-NEXT:    .cfi_def_cfa_offset 48
+; PPC-PWR8-NEXT:    .cfi_offset lr, 4
+; PPC-PWR8-NEXT:    mr r4, r3
+; PPC-PWR8-NEXT:    lwz r3, 60(r1)
+; PPC-PWR8-NEXT:    stw r8, 44(r1)
+; PPC-PWR8-NEXT:    stw r7, 40(r1)
+; PPC-PWR8-NEXT:    stw r6, 36(r1)
+; PPC-PWR8-NEXT:    stw r5, 32(r1)
+; PPC-PWR8-NEXT:    addi r5, r1, 32
+; PPC-PWR8-NEXT:    addi r6, r1, 16
+; PPC-PWR8-NEXT:    li r7, 2
+; PPC-PWR8-NEXT:    li r8, 2
+; PPC-PWR8-NEXT:    stw r10, 20(r1)
+; PPC-PWR8-NEXT:    stw r9, 16(r1)
+; PPC-PWR8-NEXT:    stw r3, 28(r1)
+; PPC-PWR8-NEXT:    lwz r3, 56(r1)
+; PPC-PWR8-NEXT:    stw r3, 24(r1)
+; PPC-PWR8-NEXT:    li r3, 16
+; PPC-PWR8-NEXT:    bl __atomic_compare_exchange
+; PPC-PWR8-NEXT:    lwz r0, 52(r1)
+; PPC-PWR8-NEXT:    addi r1, r1, 48
+; PPC-PWR8-NEXT:    mtlr r0
+; PPC-PWR8-NEXT:    blr
+entry:
+  %0 = cmpxchg weak ptr %a, i128 %cmp, i128 %new acquire acquire
+  %1 = extractvalue { i128, i1 } %0, 1
+  ret i1 %1
+}
diff --git a/llvm/test/CodeGen/PowerPC/crbit-asm.ll b/llvm/test/CodeGen/PowerPC/crbit-asm.ll
index ffd8cb10f8..617d6ec27b 100644
--- a/llvm/test/CodeGen/PowerPC/crbit-asm.ll
+++ b/llvm/test/CodeGen/PowerPC/crbit-asm.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
 ; RUN: llc -verify-machineinstrs -mcpu=pwr7 < %s | FileCheck %s
 ; RUN: llc -verify-machineinstrs -O1 -mcpu=pwr7 < %s | FileCheck %s
 ; RUN: llc -verify-machineinstrs -mcpu=pwr7 -ppc-gen-isel=false  < %s | FileCheck --check-prefix=CHECK-NO-ISEL %s
@@ -7,29 +8,71 @@ target datalayout = "E-m:e-i64:64-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
 
 define zeroext i1 @testi1(i1 zeroext %b1, i1 zeroext %b2) #0 {
+; CHECK-LABEL: testi1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    andi. 3, 3, 1
+; CHECK-NEXT:    crmove 20, 1
+; CHECK-NEXT:    andi. 3, 4, 1
+; CHECK-NEXT:    li 3, 0
+; CHECK-NEXT:    li 4, 1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    crand 20, 20, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    isel 3, 4, 3, 20
+; CHECK-NEXT:    blr
+;
+; CHECK-NO-ISEL-LABEL: testi1:
+; CHECK-NO-ISEL:       # %bb.0: # %entry
+; CHECK-NO-ISEL-NEXT:    andi. 3, 3, 1
+; CHECK-NO-ISEL-NEXT:    crmove 20, 1
+; CHECK-NO-ISEL-NEXT:    andi. 3, 4, 1
+; CHECK-NO-ISEL-NEXT:    li 3, 0
+; CHECK-NO-ISEL-NEXT:    li 4, 1
+; CHECK-NO-ISEL-NEXT:    #APP
+; CHECK-NO-ISEL-NEXT:    crand 20, 20, 1
+; CHECK-NO-ISEL-NEXT:    #NO_APP
+; CHECK-NO-ISEL-NEXT:    bc 12, 20, .LBB0_1
+; CHECK-NO-ISEL-NEXT:    blr
+; CHECK-NO-ISEL-NEXT:  .LBB0_1: # %entry
+; CHECK-NO-ISEL-NEXT:    addi 3, 4, 0
+; CHECK-NO-ISEL-NEXT:    blr
 entry:
   %0 = tail call i8 asm "crand $0, $1, $2", "=^wc,^wc,^wc"(i1 %b1, i1 %b2) #0
   %1 = and i8 %0, 1
   %tobool3 = icmp ne i8 %1, 0
   ret i1 %tobool3
 
-; CHECK-LABEL: @testi1
-; CHECK-DAG: andi. {{[0-9]+}}, 3, 1
-; CHECK-DAG: li [[REG1:[0-9]+]], 0
-; CHECK-DAG: crmove [[REG2:[0-9]+]], 1
-; CHECK-DAG: andi. {{[0-9]+}}, 4, 1
-; CHECK-DAG: crand [[REG3:[0-9]+]], [[REG2]], 1
-; CHECK-DAG: li [[REG4:[0-9]+]], 1
-; CHECK: isel 3, [[REG4]], [[REG1]], [[REG3]]
-; CHECK-NO-ISEL-LABEL: @testi1
-; CHECK-NO-ISEL: bc 12, 20,
-; CHECK-NO-ISEL: blr
-; CHECK-NO-ISEL: addi 3, 4, 0
-; CHECK-NO-ISEL-NEXT: blr
-; CHECK: blr
 }
 
 define signext i32 @testi32(i32 signext %b1, i32 signext %b2) #0 {
+; CHECK-LABEL: testi32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    andi. 3, 3, 1
+; CHECK-NEXT:    crmove 20, 1
+; CHECK-NEXT:    andi. 3, 4, 1
+; CHECK-NEXT:    li 3, 0
+; CHECK-NEXT:    li 4, -1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    crand 20, 20, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    isel 3, 4, 3, 20
+; CHECK-NEXT:    blr
+;
+; CHECK-NO-ISEL-LABEL: testi32:
+; CHECK-NO-ISEL:       # %bb.0: # %entry
+; CHECK-NO-ISEL-NEXT:    andi. 3, 3, 1
+; CHECK-NO-ISEL-NEXT:    crmove 20, 1
+; CHECK-NO-ISEL-NEXT:    andi. 3, 4, 1
+; CHECK-NO-ISEL-NEXT:    li 3, 0
+; CHECK-NO-ISEL-NEXT:    li 4, -1
+; CHECK-NO-ISEL-NEXT:    #APP
+; CHECK-NO-ISEL-NEXT:    crand 20, 20, 1
+; CHECK-NO-ISEL-NEXT:    #NO_APP
+; CHECK-NO-ISEL-NEXT:    bc 12, 20, .LBB1_1
+; CHECK-NO-ISEL-NEXT:    blr
+; CHECK-NO-ISEL-NEXT:  .LBB1_1: # %entry
+; CHECK-NO-ISEL-NEXT:    addi 3, 4, 0
+; CHECK-NO-ISEL-NEXT:    blr
 entry:
   %0 = tail call i32 asm "crand $0, $1, $2", "=^wc,^wc,^wc"(i32 %b1, i32 %b2) #0
   ret i32 %0
@@ -37,31 +80,41 @@ entry:
 ; The ABI sign_extend should combine with the any_extend from the asm result,
 ; and the result will be 0 or -1. This highlights the fact that only the first
 ; bit is meaningful.
-; CHECK-LABEL: @testi32
-; CHECK-DAG: andi. {{[0-9]+}}, 3, 1
-; CHECK-DAG: li [[REG1:[0-9]+]], 0
-; CHECK-DAG: crmove [[REG2:[0-9]+]], 1
-; CHECK-DAG: andi. {{[0-9]+}}, 4, 1
-; CHECK-DAG: crand [[REG3:[0-9]+]], [[REG2]], 1
-; CHECK-DAG: li [[REG4:[0-9]+]], -1
-; CHECK: isel 3, [[REG4]], [[REG1]], [[REG3]]
-; CHECK: blr
 }
 
 define zeroext i8 @testi8(i8 zeroext %b1, i8 zeroext %b2) #0 {
+; CHECK-LABEL: testi8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    andi. 3, 3, 1
+; CHECK-NEXT:    crmove 20, 1
+; CHECK-NEXT:    andi. 3, 4, 1
+; CHECK-NEXT:    li 3, 0
+; CHECK-NEXT:    li 4, 1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    crand 20, 20, 1
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    isel 3, 4, 3, 20
+; CHECK-NEXT:    blr
+;
+; CHECK-NO-ISEL-LABEL: testi8:
+; CHECK-NO-ISEL:       # %bb.0: # %entry
+; CHECK-NO-ISEL-NEXT:    andi. 3, 3, 1
+; CHECK-NO-ISEL-NEXT:    crmove 20, 1
+; CHECK-NO-ISEL-NEXT:    andi. 3, 4, 1
+; CHECK-NO-ISEL-NEXT:    li 3, 0
+; CHECK-NO-ISEL-NEXT:    li 4, 1
+; CHECK-NO-ISEL-NEXT:    #APP
+; CHECK-NO-ISEL-NEXT:    crand 20, 20, 1
+; CHECK-NO-ISEL-NEXT:    #NO_APP
+; CHECK-NO-ISEL-NEXT:    bc 12, 20, .LBB2_1
+; CHECK-NO-ISEL-NEXT:    blr
+; CHECK-NO-ISEL-NEXT:  .LBB2_1: # %entry
+; CHECK-NO-ISEL-NEXT:    addi 3, 4, 0
+; CHECK-NO-ISEL-NEXT:    blr
 entry:
   %0 = tail call i8 asm "crand $0, $1, $2", "=^wc,^wc,^wc"(i8 %b1, i8 %b2) #0
   ret i8 %0
 
-; CHECK-LABEL: @testi8
-; CHECK-DAG: andi. {{[0-9]+}}, 3, 1
-; CHECK-DAG: li [[REG1:[0-9]+]], 0
-; CHECK-DAG: crmove [[REG2:[0-9]+]], 1
-; CHECK-DAG: andi. {{[0-9]+}}, 4, 1
-; CHECK-DAG: crand [[REG3:[0-9]+]], [[REG2]], 1
-; CHECK-DAG: li [[REG4:[0-9]+]], 1
-; CHECK: isel 3, [[REG4]], [[REG1]], [[REG3]]
-; CHECK: blr
 }
 
 attributes #0 = { nounwind "target-features"="+crbits" }
diff --git a/llvm/test/CodeGen/PowerPC/vector-llrint.ll b/llvm/test/CodeGen/PowerPC/vector-llrint.ll
new file mode 100644
index 0000000000..4321b213b6
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/vector-llrint.ll
@@ -0,0 +1,4848 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mcpu=pwr7 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr \
+; RUN:   -mtriple=powerpc64-unknown-unknown -verify-machineinstrs < %s | \
+; RUN:   FileCheck %s --check-prefix=BE
+; RUN: llc -mcpu=pwr8 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr \
+; RUN:   -mtriple=powerpc64le-unknown-unknown -verify-machineinstrs < %s | \
+; RUN:   FileCheck %s
+; RUN: llc -mcpu=pwr8 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr \
+; RUN:   -mtriple=powerpc64le-unknown-unknown -verify-machineinstrs < %s \
+; RUN:   --enable-unsafe-fp-math | FileCheck %s --check-prefix=FAST
+
+define <1 x i64> @llrint_v1i64_v1f16(<1 x half> %x) {
+; BE-LABEL: llrint_v1i64_v1f16:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -112(r1)
+; BE-NEXT:    std r0, 128(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 112
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    addi r1, r1, 112
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: llrint_v1i64_v1f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -32(r1)
+; CHECK-NEXT:    std r0, 48(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addi r1, r1, 32
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: llrint_v1i64_v1f16:
+; FAST:       # %bb.0:
+; FAST-NEXT:    mflr r0
+; FAST-NEXT:    stdu r1, -32(r1)
+; FAST-NEXT:    std r0, 48(r1)
+; FAST-NEXT:    .cfi_def_cfa_offset 32
+; FAST-NEXT:    .cfi_offset lr, 16
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fctid f0, f1
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    addi r1, r1, 32
+; FAST-NEXT:    ld r0, 16(r1)
+; FAST-NEXT:    mtlr r0
+; FAST-NEXT:    blr
+  %a = call <1 x i64> @llvm.llrint.v1i64.v1f16(<1 x half> %x)
+  ret <1 x i64> %a
+}
+declare <1 x i64> @llvm.llrint.v1i64.v1f16(<1 x half>)
+
+define <2 x i64> @llrint_v1i64_v2f16(<2 x half> %x) {
+; BE-LABEL: llrint_v1i64_v2f16:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -160(r1)
+; BE-NEXT:    std r0, 176(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 160
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    .cfi_offset r30, -24
+; BE-NEXT:    .cfi_offset f31, -8
+; BE-NEXT:    stfd f31, 152(r1) # 8-byte Folded Spill
+; BE-NEXT:    fmr f31, f1
+; BE-NEXT:    fmr f1, f2
+; BE-NEXT:    std r30, 136(r1) # 8-byte Folded Spill
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f31
+; BE-NEXT:    mr r30, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r30, 48
+; BE-NEXT:    fmr f31, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f31
+; BE-NEXT:    std r3, 120(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 112(r1)
+; BE-NEXT:    addi r3, r1, 112
+; BE-NEXT:    ld r30, 136(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f31, 152(r1) # 8-byte Folded Reload
+; BE-NEXT:    lxvd2x v2, 0, r3
+; BE-NEXT:    addi r1, r1, 160
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: llrint_v1i64_v2f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -96(r1)
+; CHECK-NEXT:    std r0, 112(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 96
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    .cfi_offset r30, -24
+; CHECK-NEXT:    .cfi_offset f31, -8
+; CHECK-NEXT:    .cfi_offset v31, -48
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    std r30, 72(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stfd f31, 88(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f31, f2
+; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f31
+; CHECK-NEXT:    mr r30, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r30, 48
+; CHECK-NEXT:    fmr f31, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f31
+; CHECK-NEXT:    mtvsrd v31, r3
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    lfd f31, 88(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r30, 72(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    xxmrghd v2, vs0, v31
+; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    addi r1, r1, 96
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: llrint_v1i64_v2f16:
+; FAST:       # %bb.0:
+; FAST-NEXT:    mflr r0
+; FAST-NEXT:    .cfi_def_cfa_offset 48
+; FAST-NEXT:    .cfi_offset lr, 16
+; FAST-NEXT:    .cfi_offset f30, -16
+; FAST-NEXT:    .cfi_offset f31, -8
+; FAST-NEXT:    stfd f30, -16(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f31, -8(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stdu r1, -48(r1)
+; FAST-NEXT:    fmr f31, f1
+; FAST-NEXT:    fmr f1, f2
+; FAST-NEXT:    std r0, 64(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f30, f1
+; FAST-NEXT:    fmr f1, f31
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fctid f0, f1
+; FAST-NEXT:    fctid f1, f30
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    mffprd r3, f1
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    xxmrghd v2, vs1, vs0
+; FAST-NEXT:    addi r1, r1, 48
+; FAST-NEXT:    ld r0, 16(r1)
+; FAST-NEXT:    lfd f31, -8(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f30, -16(r1) # 8-byte Folded Reload
+; FAST-NEXT:    mtlr r0
+; FAST-NEXT:    blr
+  %a = call <2 x i64> @llvm.llrint.v2i64.v2f16(<2 x half> %x)
+  ret <2 x i64> %a
+}
+declare <2 x i64> @llvm.llrint.v2i64.v2f16(<2 x half>)
+
+define <4 x i64> @llrint_v4i64_v4f16(<4 x half> %x) {
+; BE-LABEL: llrint_v4i64_v4f16:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -208(r1)
+; BE-NEXT:    std r0, 224(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 208
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    .cfi_offset r28, -56
+; BE-NEXT:    .cfi_offset r29, -48
+; BE-NEXT:    .cfi_offset r30, -40
+; BE-NEXT:    .cfi_offset f29, -24
+; BE-NEXT:    .cfi_offset f30, -16
+; BE-NEXT:    .cfi_offset f31, -8
+; BE-NEXT:    stfd f29, 184(r1) # 8-byte Folded Spill
+; BE-NEXT:    fmr f29, f1
+; BE-NEXT:    fmr f1, f2
+; BE-NEXT:    std r28, 152(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r29, 160(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r30, 168(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f30, 192(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f31, 200(r1) # 8-byte Folded Spill
+; BE-NEXT:    fmr f31, f4
+; BE-NEXT:    fmr f30, f3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f29
+; BE-NEXT:    mr r30, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f31
+; BE-NEXT:    mr r29, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f30
+; BE-NEXT:    mr r28, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r28, 48
+; BE-NEXT:    fmr f31, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r29, 48
+; BE-NEXT:    fmr f30, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r30, 48
+; BE-NEXT:    fmr f29, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f29
+; BE-NEXT:    std r3, 120(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f30
+; BE-NEXT:    std r3, 112(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f31
+; BE-NEXT:    std r3, 136(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 128(r1)
+; BE-NEXT:    addi r3, r1, 112
+; BE-NEXT:    ld r30, 168(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f31, 200(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f30, 192(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f29, 184(r1) # 8-byte Folded Reload
+; BE-NEXT:    lxvd2x v2, 0, r3
+; BE-NEXT:    addi r3, r1, 128
+; BE-NEXT:    ld r29, 160(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r28, 152(r1) # 8-byte Folded Reload
+; BE-NEXT:    lxvd2x v3, 0, r3
+; BE-NEXT:    addi r1, r1, 208
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: llrint_v4i64_v4f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -144(r1)
+; CHECK-NEXT:    std r0, 160(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 144
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    .cfi_offset r28, -56
+; CHECK-NEXT:    .cfi_offset r29, -48
+; CHECK-NEXT:    .cfi_offset r30, -40
+; CHECK-NEXT:    .cfi_offset f29, -24
+; CHECK-NEXT:    .cfi_offset f30, -16
+; CHECK-NEXT:    .cfi_offset f31, -8
+; CHECK-NEXT:    .cfi_offset v30, -96
+; CHECK-NEXT:    .cfi_offset v31, -80
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    std r28, 88(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r29, 96(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r30, 104(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stfd f29, 120(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f29, f2
+; CHECK-NEXT:    stfd f30, 128(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f30, f3
+; CHECK-NEXT:    stvx v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    stfd f31, 136(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f31, f4
+; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f29
+; CHECK-NEXT:    mr r30, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f30
+; CHECK-NEXT:    mr r29, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f31
+; CHECK-NEXT:    mr r28, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r28, 48
+; CHECK-NEXT:    fmr f31, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r29, 48
+; CHECK-NEXT:    fmr f30, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r30, 48
+; CHECK-NEXT:    fmr f29, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f29
+; CHECK-NEXT:    mtvsrd v31, r3
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f30
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v31, vs0, v31
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f31
+; CHECK-NEXT:    mtvsrd v30, r3
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    vmr v2, v31
+; CHECK-NEXT:    lfd f31, 136(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f30, 128(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f29, 120(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r30, 104(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r29, 96(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    ld r28, 88(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    xxmrghd v3, vs0, v30
+; CHECK-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    addi r1, r1, 144
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: llrint_v4i64_v4f16:
+; FAST:       # %bb.0:
+; FAST-NEXT:    mflr r0
+; FAST-NEXT:    .cfi_def_cfa_offset 64
+; FAST-NEXT:    .cfi_offset lr, 16
+; FAST-NEXT:    .cfi_offset f28, -32
+; FAST-NEXT:    .cfi_offset f29, -24
+; FAST-NEXT:    .cfi_offset f30, -16
+; FAST-NEXT:    .cfi_offset f31, -8
+; FAST-NEXT:    stfd f28, -32(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f29, -24(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f30, -16(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f31, -8(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stdu r1, -64(r1)
+; FAST-NEXT:    fmr f29, f1
+; FAST-NEXT:    fmr f1, f4
+; FAST-NEXT:    std r0, 80(r1)
+; FAST-NEXT:    fmr f31, f3
+; FAST-NEXT:    fmr f30, f2
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f28, f1
+; FAST-NEXT:    fmr f1, f31
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f31, f1
+; FAST-NEXT:    fmr f1, f30
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f30, f1
+; FAST-NEXT:    fmr f1, f29
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fctid f0, f30
+; FAST-NEXT:    fctid f2, f31
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    fctid f1, f1
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    mffprd r3, f2
+; FAST-NEXT:    mtfprd f2, r3
+; FAST-NEXT:    mffprd r3, f1
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    xxmrghd v2, vs0, vs1
+; FAST-NEXT:    fctid f0, f28
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    xxmrghd v3, vs0, vs2
+; FAST-NEXT:    addi r1, r1, 64
+; FAST-NEXT:    ld r0, 16(r1)
+; FAST-NEXT:    lfd f31, -8(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f30, -16(r1) # 8-byte Folded Reload
+; FAST-NEXT:    mtlr r0
+; FAST-NEXT:    lfd f29, -24(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f28, -32(r1) # 8-byte Folded Reload
+; FAST-NEXT:    blr
+  %a = call <4 x i64> @llvm.llrint.v4i64.v4f16(<4 x half> %x)
+  ret <4 x i64> %a
+}
+declare <4 x i64> @llvm.llrint.v4i64.v4f16(<4 x half>)
+
+define <8 x i64> @llrint_v8i64_v8f16(<8 x half> %x) {
+; BE-LABEL: llrint_v8i64_v8f16:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -304(r1)
+; BE-NEXT:    std r0, 320(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 304
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    .cfi_offset r24, -120
+; BE-NEXT:    .cfi_offset r25, -112
+; BE-NEXT:    .cfi_offset r26, -104
+; BE-NEXT:    .cfi_offset r27, -96
+; BE-NEXT:    .cfi_offset r28, -88
+; BE-NEXT:    .cfi_offset r29, -80
+; BE-NEXT:    .cfi_offset r30, -72
+; BE-NEXT:    .cfi_offset f25, -56
+; BE-NEXT:    .cfi_offset f26, -48
+; BE-NEXT:    .cfi_offset f27, -40
+; BE-NEXT:    .cfi_offset f28, -32
+; BE-NEXT:    .cfi_offset f29, -24
+; BE-NEXT:    .cfi_offset f30, -16
+; BE-NEXT:    .cfi_offset f31, -8
+; BE-NEXT:    stfd f25, 248(r1) # 8-byte Folded Spill
+; BE-NEXT:    fmr f25, f1
+; BE-NEXT:    fmr f1, f2
+; BE-NEXT:    std r24, 184(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r25, 192(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r26, 200(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r27, 208(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r28, 216(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r29, 224(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r30, 232(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f26, 256(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f27, 264(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f28, 272(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f29, 280(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f30, 288(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f31, 296(r1) # 8-byte Folded Spill
+; BE-NEXT:    fmr f31, f8
+; BE-NEXT:    fmr f30, f7
+; BE-NEXT:    fmr f29, f6
+; BE-NEXT:    fmr f28, f5
+; BE-NEXT:    fmr f27, f4
+; BE-NEXT:    fmr f26, f3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f25
+; BE-NEXT:    mr r30, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f27
+; BE-NEXT:    mr r29, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f26
+; BE-NEXT:    mr r28, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f29
+; BE-NEXT:    mr r27, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f28
+; BE-NEXT:    mr r26, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f31
+; BE-NEXT:    mr r25, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f30
+; BE-NEXT:    mr r24, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r24, 48
+; BE-NEXT:    fmr f31, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r25, 48
+; BE-NEXT:    fmr f30, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r26, 48
+; BE-NEXT:    fmr f29, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r27, 48
+; BE-NEXT:    fmr f28, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r28, 48
+; BE-NEXT:    fmr f27, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r29, 48
+; BE-NEXT:    fmr f26, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r30, 48
+; BE-NEXT:    fmr f25, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f25
+; BE-NEXT:    std r3, 120(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f26
+; BE-NEXT:    std r3, 112(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f27
+; BE-NEXT:    std r3, 136(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f28
+; BE-NEXT:    std r3, 128(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f29
+; BE-NEXT:    std r3, 152(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f30
+; BE-NEXT:    std r3, 144(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f31
+; BE-NEXT:    std r3, 168(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 160(r1)
+; BE-NEXT:    addi r3, r1, 112
+; BE-NEXT:    ld r30, 232(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f31, 296(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f30, 288(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f29, 280(r1) # 8-byte Folded Reload
+; BE-NEXT:    lxvd2x v2, 0, r3
+; BE-NEXT:    addi r3, r1, 128
+; BE-NEXT:    lfd f28, 272(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f27, 264(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f26, 256(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r29, 224(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r28, 216(r1) # 8-byte Folded Reload
+; BE-NEXT:    lxvd2x v3, 0, r3
+; BE-NEXT:    addi r3, r1, 144
+; BE-NEXT:    lfd f25, 248(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r27, 208(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r26, 200(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r25, 192(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r24, 184(r1) # 8-byte Folded Reload
+; BE-NEXT:    lxvd2x v4, 0, r3
+; BE-NEXT:    addi r3, r1, 160
+; BE-NEXT:    lxvd2x v5, 0, r3
+; BE-NEXT:    addi r1, r1, 304
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: llrint_v8i64_v8f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -240(r1)
+; CHECK-NEXT:    std r0, 256(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 240
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    .cfi_offset r24, -120
+; CHECK-NEXT:    .cfi_offset r25, -112
+; CHECK-NEXT:    .cfi_offset r26, -104
+; CHECK-NEXT:    .cfi_offset r27, -96
+; CHECK-NEXT:    .cfi_offset r28, -88
+; CHECK-NEXT:    .cfi_offset r29, -80
+; CHECK-NEXT:    .cfi_offset r30, -72
+; CHECK-NEXT:    .cfi_offset f25, -56
+; CHECK-NEXT:    .cfi_offset f26, -48
+; CHECK-NEXT:    .cfi_offset f27, -40
+; CHECK-NEXT:    .cfi_offset f28, -32
+; CHECK-NEXT:    .cfi_offset f29, -24
+; CHECK-NEXT:    .cfi_offset f30, -16
+; CHECK-NEXT:    .cfi_offset f31, -8
+; CHECK-NEXT:    .cfi_offset v28, -192
+; CHECK-NEXT:    .cfi_offset v29, -176
+; CHECK-NEXT:    .cfi_offset v30, -160
+; CHECK-NEXT:    .cfi_offset v31, -144
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    std r24, 120(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r25, 128(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r26, 136(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r27, 144(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r28, 152(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r29, 160(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r30, 168(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stvx v28, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    stfd f25, 184(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f25, f2
+; CHECK-NEXT:    stfd f26, 192(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f26, f3
+; CHECK-NEXT:    stfd f27, 200(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f27, f4
+; CHECK-NEXT:    stvx v29, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 80
+; CHECK-NEXT:    stfd f28, 208(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f28, f5
+; CHECK-NEXT:    stfd f29, 216(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f29, f6
+; CHECK-NEXT:    stfd f30, 224(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f30, f7
+; CHECK-NEXT:    stvx v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 96
+; CHECK-NEXT:    stfd f31, 232(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f31, f8
+; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f25
+; CHECK-NEXT:    mr r30, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f26
+; CHECK-NEXT:    mr r29, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f27
+; CHECK-NEXT:    mr r28, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f28
+; CHECK-NEXT:    mr r27, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f29
+; CHECK-NEXT:    mr r26, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f30
+; CHECK-NEXT:    mr r25, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f31
+; CHECK-NEXT:    mr r24, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r24, 48
+; CHECK-NEXT:    fmr f31, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r25, 48
+; CHECK-NEXT:    fmr f30, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r26, 48
+; CHECK-NEXT:    fmr f29, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r27, 48
+; CHECK-NEXT:    fmr f28, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r28, 48
+; CHECK-NEXT:    fmr f27, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r29, 48
+; CHECK-NEXT:    fmr f26, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r30, 48
+; CHECK-NEXT:    fmr f25, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f25
+; CHECK-NEXT:    mtvsrd v31, r3
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f26
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v31, vs0, v31
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f27
+; CHECK-NEXT:    mtvsrd v30, r3
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f28
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v30, vs0, v30
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f29
+; CHECK-NEXT:    mtvsrd v29, r3
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f30
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v29, vs0, v29
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f31
+; CHECK-NEXT:    mtvsrd v28, r3
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 96
+; CHECK-NEXT:    vmr v2, v31
+; CHECK-NEXT:    lfd f31, 232(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    vmr v3, v30
+; CHECK-NEXT:    vmr v4, v29
+; CHECK-NEXT:    lfd f30, 224(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f29, 216(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 80
+; CHECK-NEXT:    lfd f28, 208(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f27, 200(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f26, 192(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f25, 184(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r30, 168(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r29, 160(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    ld r28, 152(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r27, 144(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    xxmrghd v5, vs0, v28
+; CHECK-NEXT:    ld r26, 136(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r25, 128(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r24, 120(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lvx v29, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    lvx v28, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    addi r1, r1, 240
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: llrint_v8i64_v8f16:
+; FAST:       # %bb.0:
+; FAST-NEXT:    mflr r0
+; FAST-NEXT:    .cfi_def_cfa_offset 96
+; FAST-NEXT:    .cfi_offset lr, 16
+; FAST-NEXT:    .cfi_offset f24, -64
+; FAST-NEXT:    .cfi_offset f25, -56
+; FAST-NEXT:    .cfi_offset f26, -48
+; FAST-NEXT:    .cfi_offset f27, -40
+; FAST-NEXT:    .cfi_offset f28, -32
+; FAST-NEXT:    .cfi_offset f29, -24
+; FAST-NEXT:    .cfi_offset f30, -16
+; FAST-NEXT:    .cfi_offset f31, -8
+; FAST-NEXT:    stfd f24, -64(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f25, -56(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f26, -48(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f27, -40(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f28, -32(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f29, -24(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f30, -16(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f31, -8(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stdu r1, -96(r1)
+; FAST-NEXT:    fmr f24, f1
+; FAST-NEXT:    fmr f1, f8
+; FAST-NEXT:    std r0, 112(r1)
+; FAST-NEXT:    fmr f30, f7
+; FAST-NEXT:    fmr f29, f6
+; FAST-NEXT:    fmr f28, f5
+; FAST-NEXT:    fmr f27, f4
+; FAST-NEXT:    fmr f26, f3
+; FAST-NEXT:    fmr f25, f2
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f31, f1
+; FAST-NEXT:    fmr f1, f30
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f30, f1
+; FAST-NEXT:    fmr f1, f29
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f29, f1
+; FAST-NEXT:    fmr f1, f28
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f28, f1
+; FAST-NEXT:    fmr f1, f27
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f27, f1
+; FAST-NEXT:    fmr f1, f26
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f26, f1
+; FAST-NEXT:    fmr f1, f25
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f25, f1
+; FAST-NEXT:    fmr f1, f24
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fctid f0, f25
+; FAST-NEXT:    fctid f2, f26
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    fctid f3, f27
+; FAST-NEXT:    fctid f4, f28
+; FAST-NEXT:    fctid f5, f29
+; FAST-NEXT:    fctid f6, f30
+; FAST-NEXT:    fctid f1, f1
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    mffprd r3, f2
+; FAST-NEXT:    mtfprd f2, r3
+; FAST-NEXT:    mffprd r3, f3
+; FAST-NEXT:    mtfprd f3, r3
+; FAST-NEXT:    mffprd r3, f4
+; FAST-NEXT:    mtfprd f4, r3
+; FAST-NEXT:    mffprd r3, f5
+; FAST-NEXT:    mtfprd f5, r3
+; FAST-NEXT:    mffprd r3, f6
+; FAST-NEXT:    mtfprd f6, r3
+; FAST-NEXT:    mffprd r3, f1
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    xxmrghd v3, vs3, vs2
+; FAST-NEXT:    xxmrghd v4, vs5, vs4
+; FAST-NEXT:    xxmrghd v2, vs0, vs1
+; FAST-NEXT:    fctid f0, f31
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    xxmrghd v5, vs0, vs6
+; FAST-NEXT:    addi r1, r1, 96
+; FAST-NEXT:    ld r0, 16(r1)
+; FAST-NEXT:    lfd f31, -8(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f30, -16(r1) # 8-byte Folded Reload
+; FAST-NEXT:    mtlr r0
+; FAST-NEXT:    lfd f29, -24(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f28, -32(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f27, -40(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f26, -48(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f25, -56(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f24, -64(r1) # 8-byte Folded Reload
+; FAST-NEXT:    blr
+  %a = call <8 x i64> @llvm.llrint.v8i64.v8f16(<8 x half> %x)
+  ret <8 x i64> %a
+}
+declare <8 x i64> @llvm.llrint.v8i64.v8f16(<8 x half>)
+
+define <16 x i64> @llrint_v16i64_v16f16(<16 x half> %x) {
+; BE-LABEL: llrint_v16i64_v16f16:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -496(r1)
+; BE-NEXT:    std r0, 512(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 496
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    .cfi_offset r16, -248
+; BE-NEXT:    .cfi_offset r17, -240
+; BE-NEXT:    .cfi_offset r18, -232
+; BE-NEXT:    .cfi_offset r19, -224
+; BE-NEXT:    .cfi_offset r20, -216
+; BE-NEXT:    .cfi_offset r21, -208
+; BE-NEXT:    .cfi_offset r22, -200
+; BE-NEXT:    .cfi_offset r23, -192
+; BE-NEXT:    .cfi_offset r24, -184
+; BE-NEXT:    .cfi_offset r25, -176
+; BE-NEXT:    .cfi_offset r26, -168
+; BE-NEXT:    .cfi_offset r27, -160
+; BE-NEXT:    .cfi_offset r28, -152
+; BE-NEXT:    .cfi_offset r29, -144
+; BE-NEXT:    .cfi_offset r30, -136
+; BE-NEXT:    .cfi_offset f17, -120
+; BE-NEXT:    .cfi_offset f18, -112
+; BE-NEXT:    .cfi_offset f19, -104
+; BE-NEXT:    .cfi_offset f20, -96
+; BE-NEXT:    .cfi_offset f21, -88
+; BE-NEXT:    .cfi_offset f22, -80
+; BE-NEXT:    .cfi_offset f23, -72
+; BE-NEXT:    .cfi_offset f24, -64
+; BE-NEXT:    .cfi_offset f25, -56
+; BE-NEXT:    .cfi_offset f26, -48
+; BE-NEXT:    .cfi_offset f27, -40
+; BE-NEXT:    .cfi_offset f28, -32
+; BE-NEXT:    .cfi_offset f29, -24
+; BE-NEXT:    .cfi_offset f30, -16
+; BE-NEXT:    .cfi_offset f31, -8
+; BE-NEXT:    stfd f20, 400(r1) # 8-byte Folded Spill
+; BE-NEXT:    fmr f20, f1
+; BE-NEXT:    fmr f1, f2
+; BE-NEXT:    std r16, 248(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r17, 256(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r18, 264(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r19, 272(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r20, 280(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r21, 288(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r22, 296(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r23, 304(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r24, 312(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r25, 320(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r26, 328(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r27, 336(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r28, 344(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r29, 352(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r30, 360(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f17, 376(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f18, 384(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f19, 392(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f21, 408(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f22, 416(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f23, 424(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f24, 432(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f25, 440(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f26, 448(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f27, 456(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f28, 464(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f29, 472(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f30, 480(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f31, 488(r1) # 8-byte Folded Spill
+; BE-NEXT:    fmr f31, f13
+; BE-NEXT:    fmr f29, f12
+; BE-NEXT:    fmr f30, f11
+; BE-NEXT:    fmr f28, f10
+; BE-NEXT:    fmr f27, f9
+; BE-NEXT:    fmr f26, f8
+; BE-NEXT:    fmr f25, f7
+; BE-NEXT:    fmr f24, f6
+; BE-NEXT:    fmr f23, f5
+; BE-NEXT:    fmr f22, f4
+; BE-NEXT:    fmr f21, f3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f20
+; BE-NEXT:    mr r30, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f22
+; BE-NEXT:    mr r29, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f21
+; BE-NEXT:    mr r28, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f24
+; BE-NEXT:    mr r27, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f23
+; BE-NEXT:    mr r26, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f26
+; BE-NEXT:    mr r25, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f25
+; BE-NEXT:    mr r24, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f28
+; BE-NEXT:    mr r23, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f27
+; BE-NEXT:    mr r22, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f29
+; BE-NEXT:    mr r21, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f30
+; BE-NEXT:    mr r20, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 652(r1)
+; BE-NEXT:    mr r19, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f31
+; BE-NEXT:    mr r18, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 668(r1)
+; BE-NEXT:    mr r17, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 660(r1)
+; BE-NEXT:    mr r16, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r16, 48
+; BE-NEXT:    fmr f31, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r17, 48
+; BE-NEXT:    fmr f30, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r18, 48
+; BE-NEXT:    fmr f29, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r19, 48
+; BE-NEXT:    fmr f28, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r20, 48
+; BE-NEXT:    fmr f27, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r21, 48
+; BE-NEXT:    fmr f26, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r22, 48
+; BE-NEXT:    fmr f25, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r23, 48
+; BE-NEXT:    fmr f24, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r24, 48
+; BE-NEXT:    fmr f23, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r25, 48
+; BE-NEXT:    fmr f22, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r26, 48
+; BE-NEXT:    fmr f21, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r27, 48
+; BE-NEXT:    fmr f20, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r28, 48
+; BE-NEXT:    fmr f19, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r29, 48
+; BE-NEXT:    fmr f18, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r30, 48
+; BE-NEXT:    fmr f17, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f17
+; BE-NEXT:    std r3, 120(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f18
+; BE-NEXT:    std r3, 112(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f19
+; BE-NEXT:    std r3, 136(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f20
+; BE-NEXT:    std r3, 128(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f21
+; BE-NEXT:    std r3, 152(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f22
+; BE-NEXT:    std r3, 144(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f23
+; BE-NEXT:    std r3, 168(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f24
+; BE-NEXT:    std r3, 160(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f25
+; BE-NEXT:    std r3, 184(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f26
+; BE-NEXT:    std r3, 176(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f27
+; BE-NEXT:    std r3, 200(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f28
+; BE-NEXT:    std r3, 192(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f29
+; BE-NEXT:    std r3, 216(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f30
+; BE-NEXT:    std r3, 208(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f31
+; BE-NEXT:    std r3, 232(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 224(r1)
+; BE-NEXT:    addi r3, r1, 112
+; BE-NEXT:    ld r30, 360(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f31, 488(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f30, 480(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f29, 472(r1) # 8-byte Folded Reload
+; BE-NEXT:    lxvd2x v2, 0, r3
+; BE-NEXT:    addi r3, r1, 128
+; BE-NEXT:    lfd f28, 464(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f27, 456(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f26, 448(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r29, 352(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r28, 344(r1) # 8-byte Folded Reload
+; BE-NEXT:    lxvd2x v3, 0, r3
+; BE-NEXT:    addi r3, r1, 144
+; BE-NEXT:    lfd f25, 440(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f24, 432(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f23, 424(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r27, 336(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r26, 328(r1) # 8-byte Folded Reload
+; BE-NEXT:    lxvd2x v4, 0, r3
+; BE-NEXT:    addi r3, r1, 160
+; BE-NEXT:    lfd f22, 416(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f21, 408(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f20, 400(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r25, 320(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r24, 312(r1) # 8-byte Folded Reload
+; BE-NEXT:    lxvd2x v5, 0, r3
+; BE-NEXT:    addi r3, r1, 176
+; BE-NEXT:    lfd f19, 392(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f18, 384(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f17, 376(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r23, 304(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r22, 296(r1) # 8-byte Folded Reload
+; BE-NEXT:    lxvd2x v6, 0, r3
+; BE-NEXT:    addi r3, r1, 192
+; BE-NEXT:    ld r21, 288(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r20, 280(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r19, 272(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r18, 264(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r17, 256(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r16, 248(r1) # 8-byte Folded Reload
+; BE-NEXT:    lxvd2x v7, 0, r3
+; BE-NEXT:    addi r3, r1, 208
+; BE-NEXT:    lxvd2x v8, 0, r3
+; BE-NEXT:    addi r3, r1, 224
+; BE-NEXT:    lxvd2x v9, 0, r3
+; BE-NEXT:    addi r1, r1, 496
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: llrint_v16i64_v16f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -432(r1)
+; CHECK-NEXT:    std r0, 448(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 432
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    .cfi_offset r16, -248
+; CHECK-NEXT:    .cfi_offset r17, -240
+; CHECK-NEXT:    .cfi_offset r18, -232
+; CHECK-NEXT:    .cfi_offset r19, -224
+; CHECK-NEXT:    .cfi_offset r20, -216
+; CHECK-NEXT:    .cfi_offset r21, -208
+; CHECK-NEXT:    .cfi_offset r22, -200
+; CHECK-NEXT:    .cfi_offset r23, -192
+; CHECK-NEXT:    .cfi_offset r24, -184
+; CHECK-NEXT:    .cfi_offset r25, -176
+; CHECK-NEXT:    .cfi_offset r26, -168
+; CHECK-NEXT:    .cfi_offset r27, -160
+; CHECK-NEXT:    .cfi_offset r28, -152
+; CHECK-NEXT:    .cfi_offset r29, -144
+; CHECK-NEXT:    .cfi_offset r30, -136
+; CHECK-NEXT:    .cfi_offset f17, -120
+; CHECK-NEXT:    .cfi_offset f18, -112
+; CHECK-NEXT:    .cfi_offset f19, -104
+; CHECK-NEXT:    .cfi_offset f20, -96
+; CHECK-NEXT:    .cfi_offset f21, -88
+; CHECK-NEXT:    .cfi_offset f22, -80
+; CHECK-NEXT:    .cfi_offset f23, -72
+; CHECK-NEXT:    .cfi_offset f24, -64
+; CHECK-NEXT:    .cfi_offset f25, -56
+; CHECK-NEXT:    .cfi_offset f26, -48
+; CHECK-NEXT:    .cfi_offset f27, -40
+; CHECK-NEXT:    .cfi_offset f28, -32
+; CHECK-NEXT:    .cfi_offset f29, -24
+; CHECK-NEXT:    .cfi_offset f30, -16
+; CHECK-NEXT:    .cfi_offset f31, -8
+; CHECK-NEXT:    .cfi_offset v24, -384
+; CHECK-NEXT:    .cfi_offset v25, -368
+; CHECK-NEXT:    .cfi_offset v26, -352
+; CHECK-NEXT:    .cfi_offset v27, -336
+; CHECK-NEXT:    .cfi_offset v28, -320
+; CHECK-NEXT:    .cfi_offset v29, -304
+; CHECK-NEXT:    .cfi_offset v30, -288
+; CHECK-NEXT:    .cfi_offset v31, -272
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    std r16, 184(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r17, 192(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r18, 200(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r19, 208(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r20, 216(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r21, 224(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r22, 232(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stvx v24, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    std r23, 240(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r24, 248(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r25, 256(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r26, 264(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r27, 272(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r28, 280(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stvx v25, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 80
+; CHECK-NEXT:    std r29, 288(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r30, 296(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stfd f17, 312(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stfd f18, 320(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stfd f19, 328(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stfd f20, 336(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f20, f2
+; CHECK-NEXT:    stvx v26, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 96
+; CHECK-NEXT:    stfd f21, 344(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f21, f3
+; CHECK-NEXT:    stfd f22, 352(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f22, f4
+; CHECK-NEXT:    stfd f23, 360(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f23, f5
+; CHECK-NEXT:    stvx v27, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 112
+; CHECK-NEXT:    stfd f24, 368(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f24, f6
+; CHECK-NEXT:    stfd f25, 376(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f25, f7
+; CHECK-NEXT:    stfd f26, 384(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f26, f8
+; CHECK-NEXT:    stvx v28, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 128
+; CHECK-NEXT:    stfd f27, 392(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f27, f9
+; CHECK-NEXT:    stfd f28, 400(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f28, f10
+; CHECK-NEXT:    stfd f29, 408(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f29, f11
+; CHECK-NEXT:    stvx v29, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 144
+; CHECK-NEXT:    stfd f30, 416(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f30, f12
+; CHECK-NEXT:    stfd f31, 424(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f31, f13
+; CHECK-NEXT:    stvx v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 160
+; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f20
+; CHECK-NEXT:    mr r30, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f21
+; CHECK-NEXT:    mr r29, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f22
+; CHECK-NEXT:    mr r28, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f23
+; CHECK-NEXT:    mr r27, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f24
+; CHECK-NEXT:    mr r26, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f25
+; CHECK-NEXT:    mr r25, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f26
+; CHECK-NEXT:    mr r24, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f27
+; CHECK-NEXT:    mr r23, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f28
+; CHECK-NEXT:    mr r22, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f29
+; CHECK-NEXT:    mr r21, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f30
+; CHECK-NEXT:    mr r20, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f31
+; CHECK-NEXT:    mr r19, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 568(r1)
+; CHECK-NEXT:    mr r18, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 576(r1)
+; CHECK-NEXT:    mr r17, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 584(r1)
+; CHECK-NEXT:    mr r16, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r16, 48
+; CHECK-NEXT:    fmr f31, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r17, 48
+; CHECK-NEXT:    fmr f30, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r18, 48
+; CHECK-NEXT:    fmr f29, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r19, 48
+; CHECK-NEXT:    fmr f28, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r20, 48
+; CHECK-NEXT:    fmr f27, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r21, 48
+; CHECK-NEXT:    fmr f26, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r22, 48
+; CHECK-NEXT:    fmr f25, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r23, 48
+; CHECK-NEXT:    fmr f24, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r24, 48
+; CHECK-NEXT:    fmr f23, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r25, 48
+; CHECK-NEXT:    fmr f22, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r26, 48
+; CHECK-NEXT:    fmr f21, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r27, 48
+; CHECK-NEXT:    fmr f20, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r28, 48
+; CHECK-NEXT:    fmr f19, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r29, 48
+; CHECK-NEXT:    fmr f18, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r30, 48
+; CHECK-NEXT:    fmr f17, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f17
+; CHECK-NEXT:    mtvsrd v31, r3
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f18
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v31, vs0, v31
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f19
+; CHECK-NEXT:    mtvsrd v30, r3
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f20
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v30, vs0, v30
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f21
+; CHECK-NEXT:    mtvsrd v29, r3
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f22
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v29, vs0, v29
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f23
+; CHECK-NEXT:    mtvsrd v28, r3
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f24
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v28, vs0, v28
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f25
+; CHECK-NEXT:    mtvsrd v27, r3
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f26
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v27, vs0, v27
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f27
+; CHECK-NEXT:    mtvsrd v26, r3
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f28
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v26, vs0, v26
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f29
+; CHECK-NEXT:    mtvsrd v25, r3
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f30
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v25, vs0, v25
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f31
+; CHECK-NEXT:    mtvsrd v24, r3
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 160
+; CHECK-NEXT:    vmr v2, v31
+; CHECK-NEXT:    lfd f31, 424(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    vmr v3, v30
+; CHECK-NEXT:    vmr v4, v29
+; CHECK-NEXT:    lfd f30, 416(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f29, 408(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 144
+; CHECK-NEXT:    vmr v5, v28
+; CHECK-NEXT:    vmr v6, v27
+; CHECK-NEXT:    vmr v7, v26
+; CHECK-NEXT:    vmr v8, v25
+; CHECK-NEXT:    lfd f28, 400(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f27, 392(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 128
+; CHECK-NEXT:    lfd f26, 384(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f25, 376(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    xxmrghd v9, vs0, v24
+; CHECK-NEXT:    lfd f24, 368(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f23, 360(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f22, 352(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lvx v29, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 112
+; CHECK-NEXT:    lfd f21, 344(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r30, 296(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f20, 336(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f19, 328(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r29, 288(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r28, 280(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lvx v28, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 96
+; CHECK-NEXT:    lfd f18, 320(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r27, 272(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f17, 312(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r26, 264(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r25, 256(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r24, 248(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lvx v27, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 80
+; CHECK-NEXT:    ld r23, 240(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r22, 232(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r21, 224(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r20, 216(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r19, 208(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r18, 200(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lvx v26, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    ld r17, 192(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r16, 184(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lvx v25, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    lvx v24, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    addi r1, r1, 432
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: llrint_v16i64_v16f16:
+; FAST:       # %bb.0:
+; FAST-NEXT:    mflr r0
+; FAST-NEXT:    .cfi_def_cfa_offset 160
+; FAST-NEXT:    .cfi_offset lr, 16
+; FAST-NEXT:    .cfi_offset f16, -128
+; FAST-NEXT:    .cfi_offset f17, -120
+; FAST-NEXT:    .cfi_offset f18, -112
+; FAST-NEXT:    .cfi_offset f19, -104
+; FAST-NEXT:    .cfi_offset f20, -96
+; FAST-NEXT:    .cfi_offset f21, -88
+; FAST-NEXT:    .cfi_offset f22, -80
+; FAST-NEXT:    .cfi_offset f23, -72
+; FAST-NEXT:    .cfi_offset f24, -64
+; FAST-NEXT:    .cfi_offset f25, -56
+; FAST-NEXT:    .cfi_offset f26, -48
+; FAST-NEXT:    .cfi_offset f27, -40
+; FAST-NEXT:    .cfi_offset f28, -32
+; FAST-NEXT:    .cfi_offset f29, -24
+; FAST-NEXT:    .cfi_offset f30, -16
+; FAST-NEXT:    .cfi_offset f31, -8
+; FAST-NEXT:    stfd f16, -128(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f17, -120(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f18, -112(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f19, -104(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f20, -96(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f21, -88(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f22, -80(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f23, -72(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f24, -64(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f25, -56(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f26, -48(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f27, -40(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f28, -32(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f29, -24(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f30, -16(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f31, -8(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stdu r1, -160(r1)
+; FAST-NEXT:    fmr f26, f1
+; FAST-NEXT:    lfs f1, 312(r1)
+; FAST-NEXT:    std r0, 176(r1)
+; FAST-NEXT:    fmr f28, f13
+; FAST-NEXT:    fmr f27, f12
+; FAST-NEXT:    fmr f24, f11
+; FAST-NEXT:    fmr f21, f10
+; FAST-NEXT:    fmr f19, f9
+; FAST-NEXT:    fmr f18, f8
+; FAST-NEXT:    fmr f17, f7
+; FAST-NEXT:    fmr f16, f6
+; FAST-NEXT:    fmr f20, f5
+; FAST-NEXT:    fmr f22, f4
+; FAST-NEXT:    fmr f23, f3
+; FAST-NEXT:    fmr f25, f2
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f31, f1
+; FAST-NEXT:    lfs f1, 304(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f30, f1
+; FAST-NEXT:    lfs f1, 296(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f29, f1
+; FAST-NEXT:    fmr f1, f28
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f28, f1
+; FAST-NEXT:    fmr f1, f27
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f27, f1
+; FAST-NEXT:    fmr f1, f24
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f24, f1
+; FAST-NEXT:    fmr f1, f21
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f21, f1
+; FAST-NEXT:    fmr f1, f19
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f19, f1
+; FAST-NEXT:    fmr f1, f18
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f18, f1
+; FAST-NEXT:    fmr f1, f17
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f17, f1
+; FAST-NEXT:    fmr f1, f16
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f16, f1
+; FAST-NEXT:    fmr f1, f20
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f20, f1
+; FAST-NEXT:    fmr f1, f22
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f22, f1
+; FAST-NEXT:    fmr f1, f23
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f23, f1
+; FAST-NEXT:    fmr f1, f25
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f25, f1
+; FAST-NEXT:    fmr f1, f26
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fctid f0, f25
+; FAST-NEXT:    fctid f2, f23
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    fctid f3, f22
+; FAST-NEXT:    fctid f4, f20
+; FAST-NEXT:    fctid f5, f16
+; FAST-NEXT:    fctid f6, f17
+; FAST-NEXT:    fctid f7, f18
+; FAST-NEXT:    fctid f8, f19
+; FAST-NEXT:    fctid f9, f21
+; FAST-NEXT:    fctid f10, f24
+; FAST-NEXT:    fctid f1, f1
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    mffprd r3, f2
+; FAST-NEXT:    mtfprd f2, r3
+; FAST-NEXT:    mffprd r3, f3
+; FAST-NEXT:    mtfprd f3, r3
+; FAST-NEXT:    mffprd r3, f4
+; FAST-NEXT:    mtfprd f4, r3
+; FAST-NEXT:    mffprd r3, f5
+; FAST-NEXT:    mtfprd f5, r3
+; FAST-NEXT:    mffprd r3, f6
+; FAST-NEXT:    mtfprd f6, r3
+; FAST-NEXT:    mffprd r3, f7
+; FAST-NEXT:    mtfprd f7, r3
+; FAST-NEXT:    mffprd r3, f8
+; FAST-NEXT:    mtfprd f8, r3
+; FAST-NEXT:    mffprd r3, f9
+; FAST-NEXT:    mtfprd f9, r3
+; FAST-NEXT:    mffprd r3, f10
+; FAST-NEXT:    mtfprd f10, r3
+; FAST-NEXT:    mffprd r3, f1
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    xxmrghd v3, vs3, vs2
+; FAST-NEXT:    xxmrghd v4, vs5, vs4
+; FAST-NEXT:    xxmrghd v5, vs7, vs6
+; FAST-NEXT:    xxmrghd v6, vs9, vs8
+; FAST-NEXT:    xxmrghd v2, vs0, vs1
+; FAST-NEXT:    fctid f0, f27
+; FAST-NEXT:    fctid f1, f29
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    xxmrghd v7, vs0, vs10
+; FAST-NEXT:    fctid f0, f28
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    mffprd r3, f1
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    xxmrghd v8, vs1, vs0
+; FAST-NEXT:    fctid f0, f30
+; FAST-NEXT:    fctid f1, f31
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    mffprd r3, f1
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    xxmrghd v9, vs1, vs0
+; FAST-NEXT:    addi r1, r1, 160
+; FAST-NEXT:    ld r0, 16(r1)
+; FAST-NEXT:    lfd f31, -8(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f30, -16(r1) # 8-byte Folded Reload
+; FAST-NEXT:    mtlr r0
+; FAST-NEXT:    lfd f29, -24(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f28, -32(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f27, -40(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f26, -48(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f25, -56(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f24, -64(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f23, -72(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f22, -80(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f21, -88(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f20, -96(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f19, -104(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f18, -112(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f17, -120(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f16, -128(r1) # 8-byte Folded Reload
+; FAST-NEXT:    blr
+  %a = call <16 x i64> @llvm.llrint.v16i64.v16f16(<16 x half> %x)
+  ret <16 x i64> %a
+}
+declare <16 x i64> @llvm.llrint.v16i64.v16f16(<16 x half>)
+
+define <32 x i64> @llrint_v32i64_v32f16(<32 x half> %x) {
+; BE-LABEL: llrint_v32i64_v32f16:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -864(r1)
+; BE-NEXT:    std r0, 880(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 864
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    .cfi_offset r14, -288
+; BE-NEXT:    .cfi_offset r15, -280
+; BE-NEXT:    .cfi_offset r16, -272
+; BE-NEXT:    .cfi_offset r17, -264
+; BE-NEXT:    .cfi_offset r18, -256
+; BE-NEXT:    .cfi_offset r19, -248
+; BE-NEXT:    .cfi_offset r20, -240
+; BE-NEXT:    .cfi_offset r21, -232
+; BE-NEXT:    .cfi_offset r22, -224
+; BE-NEXT:    .cfi_offset r23, -216
+; BE-NEXT:    .cfi_offset r24, -208
+; BE-NEXT:    .cfi_offset r25, -200
+; BE-NEXT:    .cfi_offset r26, -192
+; BE-NEXT:    .cfi_offset r27, -184
+; BE-NEXT:    .cfi_offset r28, -176
+; BE-NEXT:    .cfi_offset r29, -168
+; BE-NEXT:    .cfi_offset r30, -160
+; BE-NEXT:    .cfi_offset r31, -152
+; BE-NEXT:    .cfi_offset f14, -144
+; BE-NEXT:    .cfi_offset f15, -136
+; BE-NEXT:    .cfi_offset f16, -128
+; BE-NEXT:    .cfi_offset f17, -120
+; BE-NEXT:    .cfi_offset f18, -112
+; BE-NEXT:    .cfi_offset f19, -104
+; BE-NEXT:    .cfi_offset f20, -96
+; BE-NEXT:    .cfi_offset f21, -88
+; BE-NEXT:    .cfi_offset f22, -80
+; BE-NEXT:    .cfi_offset f23, -72
+; BE-NEXT:    .cfi_offset f24, -64
+; BE-NEXT:    .cfi_offset f25, -56
+; BE-NEXT:    .cfi_offset f26, -48
+; BE-NEXT:    .cfi_offset f27, -40
+; BE-NEXT:    .cfi_offset f28, -32
+; BE-NEXT:    .cfi_offset f29, -24
+; BE-NEXT:    .cfi_offset f30, -16
+; BE-NEXT:    .cfi_offset f31, -8
+; BE-NEXT:    stfd f20, 768(r1) # 8-byte Folded Spill
+; BE-NEXT:    fmr f20, f1
+; BE-NEXT:    fmr f1, f2
+; BE-NEXT:    std r14, 576(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r15, 584(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r16, 592(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r17, 600(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r18, 608(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r19, 616(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r20, 624(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r21, 632(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r22, 640(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r23, 648(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r24, 656(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r25, 664(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r26, 672(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r27, 680(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r28, 688(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r29, 696(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r30, 704(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r31, 712(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f14, 720(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f15, 728(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f16, 736(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f17, 744(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f18, 752(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f19, 760(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f21, 776(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f22, 784(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f23, 792(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f24, 800(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f25, 808(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f26, 816(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f27, 824(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f28, 832(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f29, 840(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f30, 848(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f31, 856(r1) # 8-byte Folded Spill
+; BE-NEXT:    fmr f31, f13
+; BE-NEXT:    fmr f29, f12
+; BE-NEXT:    fmr f30, f11
+; BE-NEXT:    fmr f28, f10
+; BE-NEXT:    fmr f27, f9
+; BE-NEXT:    fmr f26, f8
+; BE-NEXT:    fmr f25, f7
+; BE-NEXT:    fmr f24, f6
+; BE-NEXT:    fmr f23, f5
+; BE-NEXT:    fmr f22, f4
+; BE-NEXT:    fmr f21, f3
+; BE-NEXT:    mr r30, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f20
+; BE-NEXT:    std r3, 304(r1) # 8-byte Folded Spill
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f22
+; BE-NEXT:    std r3, 296(r1) # 8-byte Folded Spill
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f21
+; BE-NEXT:    std r3, 280(r1) # 8-byte Folded Spill
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f24
+; BE-NEXT:    std r3, 264(r1) # 8-byte Folded Spill
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f23
+; BE-NEXT:    std r3, 248(r1) # 8-byte Folded Spill
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f26
+; BE-NEXT:    std r3, 232(r1) # 8-byte Folded Spill
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f25
+; BE-NEXT:    std r3, 216(r1) # 8-byte Folded Spill
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f28
+; BE-NEXT:    std r3, 200(r1) # 8-byte Folded Spill
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f27
+; BE-NEXT:    std r3, 184(r1) # 8-byte Folded Spill
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f29
+; BE-NEXT:    std r3, 168(r1) # 8-byte Folded Spill
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f30
+; BE-NEXT:    std r3, 152(r1) # 8-byte Folded Spill
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1028(r1)
+; BE-NEXT:    std r3, 136(r1) # 8-byte Folded Spill
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f31
+; BE-NEXT:    std r3, 120(r1) # 8-byte Folded Spill
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1044(r1)
+; BE-NEXT:    std r3, 112(r1) # 8-byte Folded Spill
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1036(r1)
+; BE-NEXT:    mr r15, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1060(r1)
+; BE-NEXT:    mr r14, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1052(r1)
+; BE-NEXT:    mr r31, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1076(r1)
+; BE-NEXT:    mr r29, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1068(r1)
+; BE-NEXT:    mr r28, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1092(r1)
+; BE-NEXT:    mr r27, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1084(r1)
+; BE-NEXT:    mr r26, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1108(r1)
+; BE-NEXT:    mr r25, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1100(r1)
+; BE-NEXT:    mr r24, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1124(r1)
+; BE-NEXT:    mr r23, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1116(r1)
+; BE-NEXT:    mr r22, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1140(r1)
+; BE-NEXT:    mr r21, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1132(r1)
+; BE-NEXT:    mr r20, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1156(r1)
+; BE-NEXT:    mr r19, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1148(r1)
+; BE-NEXT:    mr r18, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1172(r1)
+; BE-NEXT:    mr r17, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1164(r1)
+; BE-NEXT:    mr r16, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r16, 48
+; BE-NEXT:    stfs f1, 316(r1) # 4-byte Folded Spill
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r17, 48
+; BE-NEXT:    stfs f1, 312(r1) # 4-byte Folded Spill
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r18, 48
+; BE-NEXT:    stfs f1, 292(r1) # 4-byte Folded Spill
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r19, 48
+; BE-NEXT:    stfs f1, 276(r1) # 4-byte Folded Spill
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r20, 48
+; BE-NEXT:    stfs f1, 260(r1) # 4-byte Folded Spill
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r21, 48
+; BE-NEXT:    stfs f1, 244(r1) # 4-byte Folded Spill
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r22, 48
+; BE-NEXT:    stfs f1, 228(r1) # 4-byte Folded Spill
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r23, 48
+; BE-NEXT:    stfs f1, 212(r1) # 4-byte Folded Spill
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r24, 48
+; BE-NEXT:    stfs f1, 196(r1) # 4-byte Folded Spill
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r25, 48
+; BE-NEXT:    stfs f1, 180(r1) # 4-byte Folded Spill
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r26, 48
+; BE-NEXT:    stfs f1, 164(r1) # 4-byte Folded Spill
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r27, 48
+; BE-NEXT:    stfs f1, 148(r1) # 4-byte Folded Spill
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r28, 48
+; BE-NEXT:    stfs f1, 132(r1) # 4-byte Folded Spill
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r29, 48
+; BE-NEXT:    fmr f18, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r31, 48
+; BE-NEXT:    fmr f17, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r14, 48
+; BE-NEXT:    fmr f16, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r15, 48
+; BE-NEXT:    fmr f15, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    ld r3, 112(r1) # 8-byte Folded Reload
+; BE-NEXT:    fmr f14, f1
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    ld r3, 120(r1) # 8-byte Folded Reload
+; BE-NEXT:    fmr f31, f1
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    ld r3, 136(r1) # 8-byte Folded Reload
+; BE-NEXT:    fmr f30, f1
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    ld r3, 152(r1) # 8-byte Folded Reload
+; BE-NEXT:    fmr f29, f1
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    ld r3, 168(r1) # 8-byte Folded Reload
+; BE-NEXT:    fmr f28, f1
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    ld r3, 184(r1) # 8-byte Folded Reload
+; BE-NEXT:    fmr f27, f1
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    ld r3, 200(r1) # 8-byte Folded Reload
+; BE-NEXT:    fmr f26, f1
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    ld r3, 216(r1) # 8-byte Folded Reload
+; BE-NEXT:    fmr f25, f1
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    ld r3, 232(r1) # 8-byte Folded Reload
+; BE-NEXT:    fmr f24, f1
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    ld r3, 248(r1) # 8-byte Folded Reload
+; BE-NEXT:    fmr f23, f1
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    ld r3, 264(r1) # 8-byte Folded Reload
+; BE-NEXT:    fmr f22, f1
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    ld r3, 280(r1) # 8-byte Folded Reload
+; BE-NEXT:    fmr f21, f1
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    ld r3, 296(r1) # 8-byte Folded Reload
+; BE-NEXT:    fmr f20, f1
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    ld r3, 304(r1) # 8-byte Folded Reload
+; BE-NEXT:    fmr f19, f1
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f19
+; BE-NEXT:    std r3, 328(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f20
+; BE-NEXT:    std r3, 320(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f21
+; BE-NEXT:    std r3, 344(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f22
+; BE-NEXT:    std r3, 336(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f23
+; BE-NEXT:    std r3, 360(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f24
+; BE-NEXT:    std r3, 352(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f25
+; BE-NEXT:    std r3, 376(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f26
+; BE-NEXT:    std r3, 368(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f27
+; BE-NEXT:    std r3, 392(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f28
+; BE-NEXT:    std r3, 384(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f29
+; BE-NEXT:    std r3, 408(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f30
+; BE-NEXT:    std r3, 400(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f31
+; BE-NEXT:    std r3, 424(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f14
+; BE-NEXT:    std r3, 416(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f15
+; BE-NEXT:    std r3, 440(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f16
+; BE-NEXT:    std r3, 432(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f17
+; BE-NEXT:    std r3, 456(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f18
+; BE-NEXT:    std r3, 448(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 132(r1) # 4-byte Folded Reload
+; BE-NEXT:    std r3, 472(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 148(r1) # 4-byte Folded Reload
+; BE-NEXT:    std r3, 464(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 164(r1) # 4-byte Folded Reload
+; BE-NEXT:    std r3, 488(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 180(r1) # 4-byte Folded Reload
+; BE-NEXT:    std r3, 480(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 196(r1) # 4-byte Folded Reload
+; BE-NEXT:    std r3, 504(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 212(r1) # 4-byte Folded Reload
+; BE-NEXT:    std r3, 496(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 228(r1) # 4-byte Folded Reload
+; BE-NEXT:    std r3, 520(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 244(r1) # 4-byte Folded Reload
+; BE-NEXT:    std r3, 512(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 260(r1) # 4-byte Folded Reload
+; BE-NEXT:    std r3, 536(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 276(r1) # 4-byte Folded Reload
+; BE-NEXT:    std r3, 528(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 292(r1) # 4-byte Folded Reload
+; BE-NEXT:    std r3, 552(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 312(r1) # 4-byte Folded Reload
+; BE-NEXT:    std r3, 544(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 316(r1) # 4-byte Folded Reload
+; BE-NEXT:    std r3, 568(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 560(r1)
+; BE-NEXT:    addi r3, r1, 320
+; BE-NEXT:    lxvd2x vs0, 0, r3
+; BE-NEXT:    addi r3, r1, 336
+; BE-NEXT:    lxvd2x vs1, 0, r3
+; BE-NEXT:    addi r3, r1, 352
+; BE-NEXT:    lxvd2x vs2, 0, r3
+; BE-NEXT:    addi r3, r1, 368
+; BE-NEXT:    lxvd2x vs3, 0, r3
+; BE-NEXT:    addi r3, r1, 384
+; BE-NEXT:    lxvd2x vs4, 0, r3
+; BE-NEXT:    addi r3, r1, 400
+; BE-NEXT:    lxvd2x vs5, 0, r3
+; BE-NEXT:    addi r3, r1, 416
+; BE-NEXT:    lxvd2x vs6, 0, r3
+; BE-NEXT:    addi r3, r1, 432
+; BE-NEXT:    lxvd2x vs7, 0, r3
+; BE-NEXT:    addi r3, r1, 448
+; BE-NEXT:    lxvd2x vs8, 0, r3
+; BE-NEXT:    addi r3, r1, 464
+; BE-NEXT:    lxvd2x vs9, 0, r3
+; BE-NEXT:    addi r3, r1, 480
+; BE-NEXT:    lxvd2x vs10, 0, r3
+; BE-NEXT:    addi r3, r1, 496
+; BE-NEXT:    lxvd2x vs11, 0, r3
+; BE-NEXT:    addi r3, r1, 512
+; BE-NEXT:    lxvd2x vs12, 0, r3
+; BE-NEXT:    addi r3, r1, 528
+; BE-NEXT:    lxvd2x vs13, 0, r3
+; BE-NEXT:    addi r3, r1, 544
+; BE-NEXT:    lxvd2x v2, 0, r3
+; BE-NEXT:    addi r3, r1, 560
+; BE-NEXT:    lxvd2x v3, 0, r3
+; BE-NEXT:    li r3, 240
+; BE-NEXT:    stxvd2x v3, r30, r3
+; BE-NEXT:    li r3, 224
+; BE-NEXT:    stxvd2x v2, r30, r3
+; BE-NEXT:    li r3, 208
+; BE-NEXT:    stxvd2x vs13, r30, r3
+; BE-NEXT:    li r3, 192
+; BE-NEXT:    stxvd2x vs12, r30, r3
+; BE-NEXT:    li r3, 176
+; BE-NEXT:    stxvd2x vs11, r30, r3
+; BE-NEXT:    li r3, 160
+; BE-NEXT:    stxvd2x vs10, r30, r3
+; BE-NEXT:    li r3, 144
+; BE-NEXT:    stxvd2x vs9, r30, r3
+; BE-NEXT:    li r3, 128
+; BE-NEXT:    stxvd2x vs8, r30, r3
+; BE-NEXT:    li r3, 112
+; BE-NEXT:    stxvd2x vs7, r30, r3
+; BE-NEXT:    li r3, 96
+; BE-NEXT:    stxvd2x vs6, r30, r3
+; BE-NEXT:    li r3, 80
+; BE-NEXT:    stxvd2x vs5, r30, r3
+; BE-NEXT:    li r3, 64
+; BE-NEXT:    stxvd2x vs4, r30, r3
+; BE-NEXT:    li r3, 48
+; BE-NEXT:    stxvd2x vs3, r30, r3
+; BE-NEXT:    li r3, 32
+; BE-NEXT:    stxvd2x vs2, r30, r3
+; BE-NEXT:    li r3, 16
+; BE-NEXT:    stxvd2x vs1, r30, r3
+; BE-NEXT:    stxvd2x vs0, 0, r30
+; BE-NEXT:    lfd f31, 856(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f30, 848(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f29, 840(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f28, 832(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f27, 824(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f26, 816(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f25, 808(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f24, 800(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f23, 792(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f22, 784(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f21, 776(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f20, 768(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f19, 760(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f18, 752(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f17, 744(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f16, 736(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f15, 728(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f14, 720(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r31, 712(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r30, 704(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r29, 696(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r28, 688(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r27, 680(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r26, 672(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r25, 664(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r24, 656(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r23, 648(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r22, 640(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r21, 632(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r20, 624(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r19, 616(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r18, 608(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r17, 600(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r16, 592(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r15, 584(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r14, 576(r1) # 8-byte Folded Reload
+; BE-NEXT:    addi r1, r1, 864
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: llrint_v32i64_v32f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -688(r1)
+; CHECK-NEXT:    std r0, 704(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 688
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    .cfi_offset r14, -288
+; CHECK-NEXT:    .cfi_offset r15, -280
+; CHECK-NEXT:    .cfi_offset r16, -272
+; CHECK-NEXT:    .cfi_offset r17, -264
+; CHECK-NEXT:    .cfi_offset r18, -256
+; CHECK-NEXT:    .cfi_offset r19, -248
+; CHECK-NEXT:    .cfi_offset r20, -240
+; CHECK-NEXT:    .cfi_offset r21, -232
+; CHECK-NEXT:    .cfi_offset r22, -224
+; CHECK-NEXT:    .cfi_offset r23, -216
+; CHECK-NEXT:    .cfi_offset r24, -208
+; CHECK-NEXT:    .cfi_offset r25, -200
+; CHECK-NEXT:    .cfi_offset r26, -192
+; CHECK-NEXT:    .cfi_offset r27, -184
+; CHECK-NEXT:    .cfi_offset r28, -176
+; CHECK-NEXT:    .cfi_offset r29, -168
+; CHECK-NEXT:    .cfi_offset r30, -160
+; CHECK-NEXT:    .cfi_offset r31, -152
+; CHECK-NEXT:    .cfi_offset f14, -144
+; CHECK-NEXT:    .cfi_offset f15, -136
+; CHECK-NEXT:    .cfi_offset f16, -128
+; CHECK-NEXT:    .cfi_offset f17, -120
+; CHECK-NEXT:    .cfi_offset f18, -112
+; CHECK-NEXT:    .cfi_offset f19, -104
+; CHECK-NEXT:    .cfi_offset f20, -96
+; CHECK-NEXT:    .cfi_offset f21, -88
+; CHECK-NEXT:    .cfi_offset f22, -80
+; CHECK-NEXT:    .cfi_offset f23, -72
+; CHECK-NEXT:    .cfi_offset f24, -64
+; CHECK-NEXT:    .cfi_offset f25, -56
+; CHECK-NEXT:    .cfi_offset f26, -48
+; CHECK-NEXT:    .cfi_offset f27, -40
+; CHECK-NEXT:    .cfi_offset f28, -32
+; CHECK-NEXT:    .cfi_offset f29, -24
+; CHECK-NEXT:    .cfi_offset f30, -16
+; CHECK-NEXT:    .cfi_offset f31, -8
+; CHECK-NEXT:    .cfi_offset v20, -480
+; CHECK-NEXT:    .cfi_offset v21, -464
+; CHECK-NEXT:    .cfi_offset v22, -448
+; CHECK-NEXT:    .cfi_offset v23, -432
+; CHECK-NEXT:    .cfi_offset v24, -416
+; CHECK-NEXT:    .cfi_offset v25, -400
+; CHECK-NEXT:    .cfi_offset v26, -384
+; CHECK-NEXT:    .cfi_offset v27, -368
+; CHECK-NEXT:    .cfi_offset v28, -352
+; CHECK-NEXT:    .cfi_offset v29, -336
+; CHECK-NEXT:    .cfi_offset v30, -320
+; CHECK-NEXT:    .cfi_offset v31, -304
+; CHECK-NEXT:    li r4, 208
+; CHECK-NEXT:    std r14, 400(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r15, 408(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r16, 416(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r17, 424(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r18, 432(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r19, 440(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r20, 448(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stvx v20, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    li r4, 224
+; CHECK-NEXT:    std r21, 456(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r22, 464(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r23, 472(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r24, 480(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r25, 488(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r26, 496(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stvx v21, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    li r4, 240
+; CHECK-NEXT:    std r27, 504(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r28, 512(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r29, 520(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r30, 528(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    mr r30, r3
+; CHECK-NEXT:    std r31, 536(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stvx v22, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    li r4, 256
+; CHECK-NEXT:    stfd f14, 544(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stfd f15, 552(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stfd f16, 560(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stfd f17, 568(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stfd f18, 576(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stfd f19, 584(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stvx v23, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    li r4, 272
+; CHECK-NEXT:    stfd f20, 592(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f20, f2
+; CHECK-NEXT:    stfd f21, 600(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f21, f3
+; CHECK-NEXT:    stfd f22, 608(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f22, f4
+; CHECK-NEXT:    stvx v24, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    li r4, 288
+; CHECK-NEXT:    stfd f23, 616(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f23, f5
+; CHECK-NEXT:    stfd f24, 624(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f24, f6
+; CHECK-NEXT:    stfd f25, 632(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f25, f7
+; CHECK-NEXT:    stvx v25, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    li r4, 304
+; CHECK-NEXT:    stfd f26, 640(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f26, f8
+; CHECK-NEXT:    stfd f27, 648(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f27, f9
+; CHECK-NEXT:    stfd f28, 656(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f28, f10
+; CHECK-NEXT:    stvx v26, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    li r4, 320
+; CHECK-NEXT:    stfd f29, 664(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f29, f11
+; CHECK-NEXT:    stfd f30, 672(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f30, f12
+; CHECK-NEXT:    stfd f31, 680(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f31, f13
+; CHECK-NEXT:    stvx v27, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    li r4, 336
+; CHECK-NEXT:    stvx v28, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    li r4, 352
+; CHECK-NEXT:    stvx v29, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    li r4, 368
+; CHECK-NEXT:    stvx v30, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    li r4, 384
+; CHECK-NEXT:    stvx v31, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f20
+; CHECK-NEXT:    std r3, 176(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f21
+; CHECK-NEXT:    std r3, 160(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f22
+; CHECK-NEXT:    std r3, 144(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f23
+; CHECK-NEXT:    std r3, 128(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f24
+; CHECK-NEXT:    std r3, 120(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f25
+; CHECK-NEXT:    std r3, 112(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f26
+; CHECK-NEXT:    std r3, 104(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f27
+; CHECK-NEXT:    std r3, 96(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f28
+; CHECK-NEXT:    std r3, 88(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f29
+; CHECK-NEXT:    std r3, 80(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f30
+; CHECK-NEXT:    std r3, 72(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f31
+; CHECK-NEXT:    std r3, 64(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 832(r1)
+; CHECK-NEXT:    std r3, 56(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 840(r1)
+; CHECK-NEXT:    std r3, 48(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 848(r1)
+; CHECK-NEXT:    mr r15, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 856(r1)
+; CHECK-NEXT:    mr r14, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 864(r1)
+; CHECK-NEXT:    mr r31, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 872(r1)
+; CHECK-NEXT:    mr r29, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 880(r1)
+; CHECK-NEXT:    mr r28, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 888(r1)
+; CHECK-NEXT:    mr r27, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 896(r1)
+; CHECK-NEXT:    mr r26, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 904(r1)
+; CHECK-NEXT:    mr r25, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 912(r1)
+; CHECK-NEXT:    mr r24, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 920(r1)
+; CHECK-NEXT:    mr r23, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 928(r1)
+; CHECK-NEXT:    mr r22, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 936(r1)
+; CHECK-NEXT:    mr r21, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 944(r1)
+; CHECK-NEXT:    mr r20, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 952(r1)
+; CHECK-NEXT:    mr r19, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 960(r1)
+; CHECK-NEXT:    mr r18, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 968(r1)
+; CHECK-NEXT:    mr r17, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 976(r1)
+; CHECK-NEXT:    mr r16, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    li r3, 204
+; CHECK-NEXT:    stxsspx f1, r1, r3 # 4-byte Folded Spill
+; CHECK-NEXT:    clrldi r3, r16, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    li r3, 200
+; CHECK-NEXT:    stxsspx f1, r1, r3 # 4-byte Folded Spill
+; CHECK-NEXT:    clrldi r3, r17, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r18, 48
+; CHECK-NEXT:    fmr f29, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r19, 48
+; CHECK-NEXT:    fmr f28, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r20, 48
+; CHECK-NEXT:    fmr f27, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r21, 48
+; CHECK-NEXT:    fmr f26, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r22, 48
+; CHECK-NEXT:    fmr f25, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r23, 48
+; CHECK-NEXT:    fmr f24, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r24, 48
+; CHECK-NEXT:    fmr f23, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r25, 48
+; CHECK-NEXT:    fmr f22, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r26, 48
+; CHECK-NEXT:    fmr f21, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r27, 48
+; CHECK-NEXT:    fmr f20, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r28, 48
+; CHECK-NEXT:    fmr f19, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r29, 48
+; CHECK-NEXT:    fmr f18, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r31, 48
+; CHECK-NEXT:    fmr f17, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r14, 48
+; CHECK-NEXT:    fmr f16, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r15, 48
+; CHECK-NEXT:    fmr f15, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    ld r3, 48(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    fmr f14, f1
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    ld r3, 56(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    fmr f30, f1
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    ld r3, 64(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    xxlor v30, f1, f1
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    ld r3, 72(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    xxlor v29, f1, f1
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    ld r3, 80(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    xxlor v28, f1, f1
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    ld r3, 88(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    xxlor v27, f1, f1
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    ld r3, 96(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    xxlor v26, f1, f1
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    ld r3, 104(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    xxlor v25, f1, f1
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    ld r3, 112(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    xxlor v24, f1, f1
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    ld r3, 120(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    xxlor v23, f1, f1
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    ld r3, 128(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    xxlor v22, f1, f1
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    ld r3, 144(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    xxlor v21, f1, f1
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    ld r3, 160(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    xxlor v20, f1, f1
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    ld r3, 176(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    fmr f31, f1
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f31
+; CHECK-NEXT:    mtvsrd v31, r3
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 176
+; CHECK-NEXT:    xxlor f1, v20, v20
+; CHECK-NEXT:    xxmrghd vs0, vs0, v31
+; CHECK-NEXT:    stxvd2x vs0, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxlor f1, v21, v21
+; CHECK-NEXT:    mtvsrd v31, r3
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 160
+; CHECK-NEXT:    xxlor f1, v22, v22
+; CHECK-NEXT:    xxmrghd vs0, vs0, v31
+; CHECK-NEXT:    stxvd2x vs0, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxlor f1, v23, v23
+; CHECK-NEXT:    mtvsrd v31, r3
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 144
+; CHECK-NEXT:    xxlor f1, v24, v24
+; CHECK-NEXT:    xxmrghd vs0, vs0, v31
+; CHECK-NEXT:    stxvd2x vs0, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxlor f1, v25, v25
+; CHECK-NEXT:    mtvsrd v31, r3
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 128
+; CHECK-NEXT:    xxlor f1, v26, v26
+; CHECK-NEXT:    xxmrghd vs0, vs0, v31
+; CHECK-NEXT:    stxvd2x vs0, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxlor f1, v27, v27
+; CHECK-NEXT:    mtvsrd v31, r3
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxlor f1, v28, v28
+; CHECK-NEXT:    xxmrghd v27, vs0, v31
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxlor f1, v29, v29
+; CHECK-NEXT:    mtvsrd v31, r3
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxlor f1, v30, v30
+; CHECK-NEXT:    xxmrghd v29, vs0, v31
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f30
+; CHECK-NEXT:    mtvsrd v31, r3
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f14
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v31, vs0, v31
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f15
+; CHECK-NEXT:    mtvsrd v30, r3
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f16
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v30, vs0, v30
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f17
+; CHECK-NEXT:    mtvsrd v28, r3
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f18
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v28, vs0, v28
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f19
+; CHECK-NEXT:    mtvsrd v26, r3
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f20
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v26, vs0, v26
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f21
+; CHECK-NEXT:    mtvsrd v24, r3
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f22
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v24, vs0, v24
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f23
+; CHECK-NEXT:    mtvsrd v22, r3
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f24
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v22, vs0, v22
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f25
+; CHECK-NEXT:    mtvsrd v20, r3
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f26
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v20, vs0, v20
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f27
+; CHECK-NEXT:    mtvsrd v21, r3
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f28
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v21, vs0, v21
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f29
+; CHECK-NEXT:    mtvsrd v23, r3
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 200
+; CHECK-NEXT:    lxsspx f1, r1, r3 # 4-byte Folded Reload
+; CHECK-NEXT:    xxmrghd v23, vs0, v23
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtvsrd v25, r3
+; CHECK-NEXT:    li r3, 204
+; CHECK-NEXT:    lxsspx f1, r1, r3 # 4-byte Folded Reload
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 240
+; CHECK-NEXT:    xxswapd vs1, v23
+; CHECK-NEXT:    li r4, 128
+; CHECK-NEXT:    xxswapd vs2, v21
+; CHECK-NEXT:    xxswapd vs3, v31
+; CHECK-NEXT:    xxmrghd v2, vs0, v25
+; CHECK-NEXT:    xxswapd vs0, v2
+; CHECK-NEXT:    stxvd2x vs0, r30, r3
+; CHECK-NEXT:    li r3, 224
+; CHECK-NEXT:    stxvd2x vs1, r30, r3
+; CHECK-NEXT:    li r3, 208
+; CHECK-NEXT:    stxvd2x vs2, r30, r3
+; CHECK-NEXT:    li r3, 192
+; CHECK-NEXT:    xxswapd vs0, v20
+; CHECK-NEXT:    stxvd2x vs0, r30, r3
+; CHECK-NEXT:    li r3, 176
+; CHECK-NEXT:    xxswapd vs1, v22
+; CHECK-NEXT:    stxvd2x vs1, r30, r3
+; CHECK-NEXT:    li r3, 160
+; CHECK-NEXT:    xxswapd vs2, v28
+; CHECK-NEXT:    xxswapd vs0, v24
+; CHECK-NEXT:    stxvd2x vs0, r30, r3
+; CHECK-NEXT:    li r3, 144
+; CHECK-NEXT:    xxswapd vs1, v26
+; CHECK-NEXT:    stxvd2x vs1, r30, r3
+; CHECK-NEXT:    li r3, 128
+; CHECK-NEXT:    stxvd2x vs2, r30, r3
+; CHECK-NEXT:    li r3, 112
+; CHECK-NEXT:    xxswapd vs0, v30
+; CHECK-NEXT:    stxvd2x vs0, r30, r3
+; CHECK-NEXT:    li r3, 96
+; CHECK-NEXT:    stxvd2x vs3, r30, r3
+; CHECK-NEXT:    li r3, 80
+; CHECK-NEXT:    lxvd2x vs2, r1, r4 # 16-byte Folded Reload
+; CHECK-NEXT:    li r4, 144
+; CHECK-NEXT:    xxswapd vs1, v29
+; CHECK-NEXT:    stxvd2x vs1, r30, r3
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    lxvd2x vs1, r1, r4 # 16-byte Folded Reload
+; CHECK-NEXT:    li r4, 160
+; CHECK-NEXT:    lxvd2x vs3, r1, r4 # 16-byte Folded Reload
+; CHECK-NEXT:    li r4, 176
+; CHECK-NEXT:    lxvd2x vs4, r1, r4 # 16-byte Folded Reload
+; CHECK-NEXT:    xxswapd vs0, v27
+; CHECK-NEXT:    stxvd2x vs0, r30, r3
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    xxswapd vs2, vs2
+; CHECK-NEXT:    stxvd2x vs2, r30, r3
+; CHECK-NEXT:    li r3, 32
+; CHECK-NEXT:    xxswapd vs1, vs1
+; CHECK-NEXT:    stxvd2x vs1, r30, r3
+; CHECK-NEXT:    li r3, 16
+; CHECK-NEXT:    xxswapd vs3, vs3
+; CHECK-NEXT:    stxvd2x vs3, r30, r3
+; CHECK-NEXT:    li r3, 384
+; CHECK-NEXT:    xxswapd vs4, vs4
+; CHECK-NEXT:    stxvd2x vs4, 0, r30
+; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 368
+; CHECK-NEXT:    lfd f31, 680(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f30, 672(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f29, 664(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f28, 656(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f27, 648(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f26, 640(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f25, 632(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f24, 624(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f23, 616(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f22, 608(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f21, 600(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f20, 592(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f19, 584(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f18, 576(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f17, 568(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f16, 560(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 352
+; CHECK-NEXT:    lfd f15, 552(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f14, 544(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r31, 536(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r30, 528(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r29, 520(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r28, 512(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lvx v29, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 336
+; CHECK-NEXT:    ld r27, 504(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r26, 496(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r25, 488(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r24, 480(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r23, 472(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r22, 464(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lvx v28, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 320
+; CHECK-NEXT:    ld r21, 456(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r20, 448(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r19, 440(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r18, 432(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r17, 424(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r16, 416(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lvx v27, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 304
+; CHECK-NEXT:    ld r15, 408(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r14, 400(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lvx v26, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 288
+; CHECK-NEXT:    lvx v25, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 272
+; CHECK-NEXT:    lvx v24, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 256
+; CHECK-NEXT:    lvx v23, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 240
+; CHECK-NEXT:    lvx v22, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 224
+; CHECK-NEXT:    lvx v21, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 208
+; CHECK-NEXT:    lvx v20, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    addi r1, r1, 688
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: llrint_v32i64_v32f16:
+; FAST:       # %bb.0:
+; FAST-NEXT:    mflr r0
+; FAST-NEXT:    stdu r1, -480(r1)
+; FAST-NEXT:    std r0, 496(r1)
+; FAST-NEXT:    .cfi_def_cfa_offset 480
+; FAST-NEXT:    .cfi_offset lr, 16
+; FAST-NEXT:    .cfi_offset r30, -160
+; FAST-NEXT:    .cfi_offset f14, -144
+; FAST-NEXT:    .cfi_offset f15, -136
+; FAST-NEXT:    .cfi_offset f16, -128
+; FAST-NEXT:    .cfi_offset f17, -120
+; FAST-NEXT:    .cfi_offset f18, -112
+; FAST-NEXT:    .cfi_offset f19, -104
+; FAST-NEXT:    .cfi_offset f20, -96
+; FAST-NEXT:    .cfi_offset f21, -88
+; FAST-NEXT:    .cfi_offset f22, -80
+; FAST-NEXT:    .cfi_offset f23, -72
+; FAST-NEXT:    .cfi_offset f24, -64
+; FAST-NEXT:    .cfi_offset f25, -56
+; FAST-NEXT:    .cfi_offset f26, -48
+; FAST-NEXT:    .cfi_offset f27, -40
+; FAST-NEXT:    .cfi_offset f28, -32
+; FAST-NEXT:    .cfi_offset f29, -24
+; FAST-NEXT:    .cfi_offset f30, -16
+; FAST-NEXT:    .cfi_offset f31, -8
+; FAST-NEXT:    .cfi_offset v20, -352
+; FAST-NEXT:    .cfi_offset v21, -336
+; FAST-NEXT:    .cfi_offset v22, -320
+; FAST-NEXT:    .cfi_offset v23, -304
+; FAST-NEXT:    .cfi_offset v24, -288
+; FAST-NEXT:    .cfi_offset v25, -272
+; FAST-NEXT:    .cfi_offset v26, -256
+; FAST-NEXT:    .cfi_offset v27, -240
+; FAST-NEXT:    .cfi_offset v28, -224
+; FAST-NEXT:    .cfi_offset v29, -208
+; FAST-NEXT:    .cfi_offset v30, -192
+; FAST-NEXT:    .cfi_offset v31, -176
+; FAST-NEXT:    li r4, 128
+; FAST-NEXT:    std r30, 320(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f14, 336(r1) # 8-byte Folded Spill
+; FAST-NEXT:    fmr f14, f5
+; FAST-NEXT:    stfd f15, 344(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f16, 352(r1) # 8-byte Folded Spill
+; FAST-NEXT:    fmr f16, f4
+; FAST-NEXT:    mr r30, r3
+; FAST-NEXT:    stvx v20, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    li r4, 144
+; FAST-NEXT:    stfd f17, 360(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f18, 368(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f19, 376(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f20, 384(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f21, 392(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f22, 400(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stvx v21, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    li r4, 160
+; FAST-NEXT:    stfd f23, 408(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f24, 416(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f25, 424(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f26, 432(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f27, 440(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f28, 448(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stvx v22, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    li r4, 176
+; FAST-NEXT:    xxlor v22, f3, f3
+; FAST-NEXT:    stfd f29, 456(r1) # 8-byte Folded Spill
+; FAST-NEXT:    fmr f29, f9
+; FAST-NEXT:    stfd f30, 464(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f31, 472(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stvx v23, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    li r4, 192
+; FAST-NEXT:    xxlor v23, f2, f2
+; FAST-NEXT:    stvx v24, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    li r4, 208
+; FAST-NEXT:    stvx v25, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    li r4, 224
+; FAST-NEXT:    xxlor v25, f13, f13
+; FAST-NEXT:    stvx v26, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    li r4, 240
+; FAST-NEXT:    xxlor v26, f12, f12
+; FAST-NEXT:    stvx v27, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    li r4, 256
+; FAST-NEXT:    xxlor v27, f11, f11
+; FAST-NEXT:    stvx v28, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    li r4, 272
+; FAST-NEXT:    xxlor v28, f10, f10
+; FAST-NEXT:    stvx v29, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    li r4, 288
+; FAST-NEXT:    xxlor v29, f8, f8
+; FAST-NEXT:    stvx v30, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    li r4, 304
+; FAST-NEXT:    xxlor v30, f7, f7
+; FAST-NEXT:    stvx v31, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    li r4, 44
+; FAST-NEXT:    xxlor v31, f6, f6
+; FAST-NEXT:    stxsspx f1, r1, r4 # 4-byte Folded Spill
+; FAST-NEXT:    lfs f1, 768(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    li r3, 120
+; FAST-NEXT:    stxsdx f1, r1, r3 # 8-byte Folded Spill
+; FAST-NEXT:    lfs f1, 760(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    li r3, 112
+; FAST-NEXT:    stxsdx f1, r1, r3 # 8-byte Folded Spill
+; FAST-NEXT:    lfs f1, 752(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    li r3, 104
+; FAST-NEXT:    stxsdx f1, r1, r3 # 8-byte Folded Spill
+; FAST-NEXT:    lfs f1, 744(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    li r3, 96
+; FAST-NEXT:    stxsdx f1, r1, r3 # 8-byte Folded Spill
+; FAST-NEXT:    lfs f1, 736(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    li r3, 88
+; FAST-NEXT:    stxsdx f1, r1, r3 # 8-byte Folded Spill
+; FAST-NEXT:    lfs f1, 728(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    li r3, 80
+; FAST-NEXT:    stxsdx f1, r1, r3 # 8-byte Folded Spill
+; FAST-NEXT:    lfs f1, 720(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    li r3, 72
+; FAST-NEXT:    stxsdx f1, r1, r3 # 8-byte Folded Spill
+; FAST-NEXT:    lfs f1, 712(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    li r3, 64
+; FAST-NEXT:    stxsdx f1, r1, r3 # 8-byte Folded Spill
+; FAST-NEXT:    lfs f1, 704(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    li r3, 56
+; FAST-NEXT:    stxsdx f1, r1, r3 # 8-byte Folded Spill
+; FAST-NEXT:    lfs f1, 696(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    li r3, 48
+; FAST-NEXT:    stxsdx f1, r1, r3 # 8-byte Folded Spill
+; FAST-NEXT:    lfs f1, 688(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    xxlor v21, f1, f1
+; FAST-NEXT:    lfs f1, 680(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    xxlor v20, f1, f1
+; FAST-NEXT:    lfs f1, 672(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    xxlor v24, f1, f1
+; FAST-NEXT:    lfs f1, 664(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f31, f1
+; FAST-NEXT:    lfs f1, 656(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f30, f1
+; FAST-NEXT:    lfs f1, 648(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f28, f1
+; FAST-NEXT:    lfs f1, 640(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f27, f1
+; FAST-NEXT:    lfs f1, 632(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f26, f1
+; FAST-NEXT:    lfs f1, 624(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f25, f1
+; FAST-NEXT:    xxlor f1, v25, v25
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f24, f1
+; FAST-NEXT:    xxlor f1, v26, v26
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f23, f1
+; FAST-NEXT:    xxlor f1, v27, v27
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f22, f1
+; FAST-NEXT:    xxlor f1, v28, v28
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f21, f1
+; FAST-NEXT:    fmr f1, f29
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f20, f1
+; FAST-NEXT:    xxlor f1, v29, v29
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f19, f1
+; FAST-NEXT:    xxlor f1, v30, v30
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f18, f1
+; FAST-NEXT:    xxlor f1, v31, v31
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f29, f1
+; FAST-NEXT:    fmr f1, f14
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f14, f1
+; FAST-NEXT:    fmr f1, f16
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f16, f1
+; FAST-NEXT:    xxlor f1, v22, v22
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f17, f1
+; FAST-NEXT:    xxlor f1, v23, v23
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    li r3, 44
+; FAST-NEXT:    fmr f15, f1
+; FAST-NEXT:    lxsspx f1, r1, r3 # 4-byte Folded Reload
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fctid f3, f15
+; FAST-NEXT:    fctid f4, f17
+; FAST-NEXT:    mffprd r3, f3
+; FAST-NEXT:    fctid f5, f16
+; FAST-NEXT:    fctid f6, f14
+; FAST-NEXT:    fctid f7, f18
+; FAST-NEXT:    fctid f8, f19
+; FAST-NEXT:    fctid f13, f1
+; FAST-NEXT:    fctid f9, f20
+; FAST-NEXT:    fctid f10, f22
+; FAST-NEXT:    fctid f11, f24
+; FAST-NEXT:    fctid f12, f25
+; FAST-NEXT:    fctid f2, f23
+; FAST-NEXT:    fctid f0, f21
+; FAST-NEXT:    mtvsrd v2, r3
+; FAST-NEXT:    mffprd r3, f4
+; FAST-NEXT:    mtvsrd v3, r3
+; FAST-NEXT:    mffprd r3, f5
+; FAST-NEXT:    mtfprd f5, r3
+; FAST-NEXT:    mffprd r3, f6
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    mffprd r3, f7
+; FAST-NEXT:    mtfprd f6, r3
+; FAST-NEXT:    mffprd r3, f8
+; FAST-NEXT:    mtfprd f7, r3
+; FAST-NEXT:    mffprd r3, f9
+; FAST-NEXT:    mtfprd f3, r3
+; FAST-NEXT:    mffprd r3, f10
+; FAST-NEXT:    mtfprd f4, r3
+; FAST-NEXT:    mffprd r3, f11
+; FAST-NEXT:    fctid f11, f31
+; FAST-NEXT:    lfd f31, 56(r1) # 8-byte Folded Reload
+; FAST-NEXT:    mtfprd f8, r3
+; FAST-NEXT:    mffprd r3, f12
+; FAST-NEXT:    xxlor f12, v24, v24
+; FAST-NEXT:    fctid f31, f31
+; FAST-NEXT:    fctid f12, f12
+; FAST-NEXT:    mtfprd f9, r3
+; FAST-NEXT:    mffprd r3, f13
+; FAST-NEXT:    lfd f13, 48(r1) # 8-byte Folded Reload
+; FAST-NEXT:    mtfprd f10, r3
+; FAST-NEXT:    fctid f13, f13
+; FAST-NEXT:    xxmrghd v3, vs5, v3
+; FAST-NEXT:    fctid f5, f26
+; FAST-NEXT:    mffprd r3, f5
+; FAST-NEXT:    mtfprd f5, r3
+; FAST-NEXT:    xxmrghd v4, vs7, vs6
+; FAST-NEXT:    fctid f6, f27
+; FAST-NEXT:    fctid f7, f28
+; FAST-NEXT:    mffprd r3, f6
+; FAST-NEXT:    lfd f28, 96(r1) # 8-byte Folded Reload
+; FAST-NEXT:    fctid f28, f28
+; FAST-NEXT:    mtfprd f6, r3
+; FAST-NEXT:    mffprd r3, f7
+; FAST-NEXT:    mtfprd f7, r3
+; FAST-NEXT:    xxmrghd v2, v2, vs10
+; FAST-NEXT:    fctid f10, f30
+; FAST-NEXT:    mffprd r3, f10
+; FAST-NEXT:    lfd f30, 80(r1) # 8-byte Folded Reload
+; FAST-NEXT:    fctid f30, f30
+; FAST-NEXT:    mtfprd f10, r3
+; FAST-NEXT:    mffprd r3, f11
+; FAST-NEXT:    mtfprd f11, r3
+; FAST-NEXT:    mffprd r3, f12
+; FAST-NEXT:    mtfprd f12, r3
+; FAST-NEXT:    xxmrghd v5, vs12, vs11
+; FAST-NEXT:    xxlor f11, v20, v20
+; FAST-NEXT:    xxlor f12, v21, v21
+; FAST-NEXT:    fctid f11, f11
+; FAST-NEXT:    fctid f12, f12
+; FAST-NEXT:    mffprd r3, f11
+; FAST-NEXT:    mtfprd f11, r3
+; FAST-NEXT:    mffprd r3, f12
+; FAST-NEXT:    mtfprd f12, r3
+; FAST-NEXT:    mffprd r3, f13
+; FAST-NEXT:    mtfprd f13, r3
+; FAST-NEXT:    mffprd r3, f31
+; FAST-NEXT:    lfd f31, 64(r1) # 8-byte Folded Reload
+; FAST-NEXT:    fctid f31, f31
+; FAST-NEXT:    mtvsrd v0, r3
+; FAST-NEXT:    mffprd r3, f31
+; FAST-NEXT:    lfd f31, 72(r1) # 8-byte Folded Reload
+; FAST-NEXT:    mtvsrd v1, r3
+; FAST-NEXT:    mffprd r3, f30
+; FAST-NEXT:    lfd f30, 88(r1) # 8-byte Folded Reload
+; FAST-NEXT:    fctid f31, f31
+; FAST-NEXT:    mtvsrd v6, r3
+; FAST-NEXT:    mffprd r3, f28
+; FAST-NEXT:    lfd f28, 104(r1) # 8-byte Folded Reload
+; FAST-NEXT:    fctid f30, f30
+; FAST-NEXT:    fctid f28, f28
+; FAST-NEXT:    mtvsrd v7, r3
+; FAST-NEXT:    mffprd r3, f28
+; FAST-NEXT:    lfd f28, 112(r1) # 8-byte Folded Reload
+; FAST-NEXT:    fctid f28, f28
+; FAST-NEXT:    mtvsrd v8, r3
+; FAST-NEXT:    mffprd r3, f28
+; FAST-NEXT:    lfd f28, 120(r1) # 8-byte Folded Reload
+; FAST-NEXT:    fctid f28, f28
+; FAST-NEXT:    xxmrghd v10, vs12, vs11
+; FAST-NEXT:    xxmrghd v0, v0, vs13
+; FAST-NEXT:    xxswapd vs12, v0
+; FAST-NEXT:    xxmrghd v0, vs9, vs8
+; FAST-NEXT:    xxmrghd v7, v8, v7
+; FAST-NEXT:    mtvsrd v8, r3
+; FAST-NEXT:    mffprd r3, f28
+; FAST-NEXT:    mtvsrd v9, r3
+; FAST-NEXT:    mffprd r3, f30
+; FAST-NEXT:    xxswapd v7, v7
+; FAST-NEXT:    xxmrghd v8, v9, v8
+; FAST-NEXT:    mtvsrd v9, r3
+; FAST-NEXT:    mffprd r3, f31
+; FAST-NEXT:    xxswapd v8, v8
+; FAST-NEXT:    xxmrghd v6, v9, v6
+; FAST-NEXT:    mtvsrd v9, r3
+; FAST-NEXT:    li r3, 240
+; FAST-NEXT:    stxvd2x v8, r30, r3
+; FAST-NEXT:    li r3, 224
+; FAST-NEXT:    stxvd2x v7, r30, r3
+; FAST-NEXT:    li r3, 208
+; FAST-NEXT:    xxswapd vs11, v6
+; FAST-NEXT:    xxmrghd v6, vs10, vs7
+; FAST-NEXT:    stxvd2x vs11, r30, r3
+; FAST-NEXT:    li r3, 192
+; FAST-NEXT:    xxmrghd v1, v9, v1
+; FAST-NEXT:    xxswapd vs11, v1
+; FAST-NEXT:    xxmrghd v1, vs6, vs5
+; FAST-NEXT:    xxswapd vs5, v10
+; FAST-NEXT:    xxswapd vs6, v5
+; FAST-NEXT:    stxvd2x vs11, r30, r3
+; FAST-NEXT:    li r3, 176
+; FAST-NEXT:    stxvd2x vs12, r30, r3
+; FAST-NEXT:    li r3, 160
+; FAST-NEXT:    stxvd2x vs5, r30, r3
+; FAST-NEXT:    li r3, 144
+; FAST-NEXT:    stxvd2x vs6, r30, r3
+; FAST-NEXT:    mffprd r3, f2
+; FAST-NEXT:    mtfprd f7, r3
+; FAST-NEXT:    li r3, 128
+; FAST-NEXT:    xxswapd vs5, v6
+; FAST-NEXT:    stxvd2x vs5, r30, r3
+; FAST-NEXT:    li r3, 112
+; FAST-NEXT:    xxswapd vs2, v1
+; FAST-NEXT:    xxswapd vs6, v0
+; FAST-NEXT:    stxvd2x vs2, r30, r3
+; FAST-NEXT:    li r3, 96
+; FAST-NEXT:    fctid f2, f29
+; FAST-NEXT:    stxvd2x vs6, r30, r3
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    mffprd r3, f2
+; FAST-NEXT:    mtfprd f2, r3
+; FAST-NEXT:    li r3, 80
+; FAST-NEXT:    xxmrghd v5, vs7, vs4
+; FAST-NEXT:    xxswapd vs4, v2
+; FAST-NEXT:    xxmrghd v0, vs0, vs3
+; FAST-NEXT:    xxswapd vs0, v5
+; FAST-NEXT:    xxswapd vs3, v3
+; FAST-NEXT:    stxvd2x vs0, r30, r3
+; FAST-NEXT:    li r3, 64
+; FAST-NEXT:    xxswapd vs0, v0
+; FAST-NEXT:    stxvd2x vs0, r30, r3
+; FAST-NEXT:    li r3, 48
+; FAST-NEXT:    xxmrghd v5, vs2, vs1
+; FAST-NEXT:    xxswapd vs1, v4
+; FAST-NEXT:    stxvd2x vs1, r30, r3
+; FAST-NEXT:    li r3, 32
+; FAST-NEXT:    xxswapd vs2, v5
+; FAST-NEXT:    stxvd2x vs2, r30, r3
+; FAST-NEXT:    li r3, 16
+; FAST-NEXT:    stxvd2x vs3, r30, r3
+; FAST-NEXT:    li r3, 304
+; FAST-NEXT:    stxvd2x vs4, 0, r30
+; FAST-NEXT:    lfd f31, 472(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f30, 464(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f29, 456(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f28, 448(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f27, 440(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f26, 432(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f25, 424(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f24, 416(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f23, 408(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f22, 400(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f21, 392(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f20, 384(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f19, 376(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f18, 368(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f17, 360(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f16, 352(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f15, 344(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f14, 336(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 288
+; FAST-NEXT:    ld r30, 320(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 272
+; FAST-NEXT:    lvx v29, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 256
+; FAST-NEXT:    lvx v28, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 240
+; FAST-NEXT:    lvx v27, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 224
+; FAST-NEXT:    lvx v26, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 208
+; FAST-NEXT:    lvx v25, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 192
+; FAST-NEXT:    lvx v24, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 176
+; FAST-NEXT:    lvx v23, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 160
+; FAST-NEXT:    lvx v22, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 144
+; FAST-NEXT:    lvx v21, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 128
+; FAST-NEXT:    lvx v20, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    addi r1, r1, 480
+; FAST-NEXT:    ld r0, 16(r1)
+; FAST-NEXT:    mtlr r0
+; FAST-NEXT:    blr
+  %a = call <32 x i64> @llvm.llrint.v32i64.v32f16(<32 x half> %x)
+  ret <32 x i64> %a
+}
+declare <32 x i64> @llvm.llrint.v32i64.v32f16(<32 x half>)
+
+define <1 x i64> @llrint_v1i64_v1f32(<1 x float> %x) {
+; BE-LABEL: llrint_v1i64_v1f32:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -112(r1)
+; BE-NEXT:    std r0, 128(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 112
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    addi r1, r1, 112
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: llrint_v1i64_v1f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -32(r1)
+; CHECK-NEXT:    std r0, 48(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addi r1, r1, 32
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: llrint_v1i64_v1f32:
+; FAST:       # %bb.0:
+; FAST-NEXT:    fctid f0, f1
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    blr
+  %a = call <1 x i64> @llvm.llrint.v1i64.v1f32(<1 x float> %x)
+  ret <1 x i64> %a
+}
+declare <1 x i64> @llvm.llrint.v1i64.v1f32(<1 x float>)
+
+define <2 x i64> @llrint_v2i64_v2f32(<2 x float> %x) {
+; BE-LABEL: llrint_v2i64_v2f32:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -144(r1)
+; BE-NEXT:    std r0, 160(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 144
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    addi r3, r1, 112
+; BE-NEXT:    stxvw4x v2, 0, r3
+; BE-NEXT:    lfs f1, 116(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 112(r1)
+; BE-NEXT:    std r3, 136(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 128(r1)
+; BE-NEXT:    addi r3, r1, 128
+; BE-NEXT:    lxvd2x v2, 0, r3
+; BE-NEXT:    addi r1, r1, 144
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: llrint_v2i64_v2f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -64(r1)
+; CHECK-NEXT:    std r0, 80(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    .cfi_offset v31, -16
+; CHECK-NEXT:    xxsldwi vs0, v2, v2, 3
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    vmr v31, v2
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxswapd vs0, v31
+; CHECK-NEXT:    mtvsrd v31, r3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    xxmrghd v2, vs0, v31
+; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    addi r1, r1, 64
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: llrint_v2i64_v2f32:
+; FAST:       # %bb.0:
+; FAST-NEXT:    xxsldwi vs0, v2, v2, 3
+; FAST-NEXT:    xxswapd vs1, v2
+; FAST-NEXT:    xscvspdpn f0, vs0
+; FAST-NEXT:    xscvspdpn f1, vs1
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    fctid f1, f1
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    mffprd r3, f1
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    xxmrghd v2, vs1, vs0
+; FAST-NEXT:    blr
+  %a = call <2 x i64> @llvm.llrint.v2i64.v2f32(<2 x float> %x)
+  ret <2 x i64> %a
+}
+declare <2 x i64> @llvm.llrint.v2i64.v2f32(<2 x float>)
+
+define <4 x i64> @llrint_v4i64_v4f32(<4 x float> %x) {
+; BE-LABEL: llrint_v4i64_v4f32:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -160(r1)
+; BE-NEXT:    std r0, 176(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 160
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    addi r3, r1, 112
+; BE-NEXT:    stxvw4x v2, 0, r3
+; BE-NEXT:    lfs f1, 116(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 112(r1)
+; BE-NEXT:    std r3, 136(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 124(r1)
+; BE-NEXT:    std r3, 128(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 120(r1)
+; BE-NEXT:    std r3, 152(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 144(r1)
+; BE-NEXT:    addi r3, r1, 128
+; BE-NEXT:    lxvd2x v2, 0, r3
+; BE-NEXT:    addi r3, r1, 144
+; BE-NEXT:    lxvd2x v3, 0, r3
+; BE-NEXT:    addi r1, r1, 160
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: llrint_v4i64_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -80(r1)
+; CHECK-NEXT:    std r0, 96(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 80
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    .cfi_offset v30, -32
+; CHECK-NEXT:    .cfi_offset v31, -16
+; CHECK-NEXT:    xxsldwi vs0, v2, v2, 3
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    stvx v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    vmr v31, v2
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxswapd vs0, v31
+; CHECK-NEXT:    mtvsrd v30, r3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xscvspdpn f1, v31
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v30, vs0, v30
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxsldwi vs0, v31, v31, 1
+; CHECK-NEXT:    mtvsrd v31, r3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    vmr v2, v30
+; CHECK-NEXT:    xxmrghd v3, v31, vs0
+; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    addi r1, r1, 80
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: llrint_v4i64_v4f32:
+; FAST:       # %bb.0:
+; FAST-NEXT:    xxsldwi vs0, v2, v2, 3
+; FAST-NEXT:    xxswapd vs1, v2
+; FAST-NEXT:    xscvspdpn f0, vs0
+; FAST-NEXT:    xxsldwi vs2, v2, v2, 1
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    xscvspdpn f0, vs1
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    xxmrghd v4, vs0, vs1
+; FAST-NEXT:    xscvspdpn f0, v2
+; FAST-NEXT:    vmr v2, v4
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    xscvspdpn f0, vs2
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    xxmrghd v3, vs1, vs0
+; FAST-NEXT:    blr
+  %a = call <4 x i64> @llvm.llrint.v4i64.v4f32(<4 x float> %x)
+  ret <4 x i64> %a
+}
+declare <4 x i64> @llvm.llrint.v4i64.v4f32(<4 x float>)
+
+define <8 x i64> @llrint_v8i64_v8f32(<8 x float> %x) {
+; BE-LABEL: llrint_v8i64_v8f32:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -208(r1)
+; BE-NEXT:    std r0, 224(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 208
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    addi r3, r1, 112
+; BE-NEXT:    stxvw4x v2, 0, r3
+; BE-NEXT:    addi r3, r1, 128
+; BE-NEXT:    stxvw4x v3, 0, r3
+; BE-NEXT:    lfs f1, 116(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 112(r1)
+; BE-NEXT:    std r3, 152(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 124(r1)
+; BE-NEXT:    std r3, 144(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 120(r1)
+; BE-NEXT:    std r3, 168(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 132(r1)
+; BE-NEXT:    std r3, 160(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 128(r1)
+; BE-NEXT:    std r3, 184(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 140(r1)
+; BE-NEXT:    std r3, 176(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 136(r1)
+; BE-NEXT:    std r3, 200(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 192(r1)
+; BE-NEXT:    addi r3, r1, 144
+; BE-NEXT:    lxvd2x v2, 0, r3
+; BE-NEXT:    addi r3, r1, 160
+; BE-NEXT:    lxvd2x v3, 0, r3
+; BE-NEXT:    addi r3, r1, 176
+; BE-NEXT:    lxvd2x v4, 0, r3
+; BE-NEXT:    addi r3, r1, 192
+; BE-NEXT:    lxvd2x v5, 0, r3
+; BE-NEXT:    addi r1, r1, 208
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: llrint_v8i64_v8f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -112(r1)
+; CHECK-NEXT:    std r0, 128(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 112
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    .cfi_offset v28, -64
+; CHECK-NEXT:    .cfi_offset v29, -48
+; CHECK-NEXT:    .cfi_offset v30, -32
+; CHECK-NEXT:    .cfi_offset v31, -16
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    xxsldwi vs0, v2, v2, 3
+; CHECK-NEXT:    stvx v28, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    stvx v29, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 80
+; CHECK-NEXT:    stvx v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 96
+; CHECK-NEXT:    vmr v30, v2
+; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    vmr v31, v3
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxswapd vs0, v30
+; CHECK-NEXT:    mtvsrd v29, r3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xscvspdpn f1, v30
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v29, vs0, v29
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxsldwi vs0, v30, v30, 1
+; CHECK-NEXT:    mtvsrd v30, r3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v30, v30, vs0
+; CHECK-NEXT:    xxsldwi vs0, v31, v31, 3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxswapd vs0, v31
+; CHECK-NEXT:    mtvsrd v28, r3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xscvspdpn f1, v31
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v28, vs0, v28
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxsldwi vs0, v31, v31, 1
+; CHECK-NEXT:    mtvsrd v31, r3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 96
+; CHECK-NEXT:    vmr v3, v30
+; CHECK-NEXT:    vmr v2, v29
+; CHECK-NEXT:    vmr v4, v28
+; CHECK-NEXT:    xxmrghd v5, v31, vs0
+; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 80
+; CHECK-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    lvx v29, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    lvx v28, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    addi r1, r1, 112
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: llrint_v8i64_v8f32:
+; FAST:       # %bb.0:
+; FAST-NEXT:    xxsldwi vs0, v2, v2, 3
+; FAST-NEXT:    xxswapd vs1, v2
+; FAST-NEXT:    xscvspdpn f0, vs0
+; FAST-NEXT:    xxsldwi vs2, v2, v2, 1
+; FAST-NEXT:    xxsldwi vs3, v3, v3, 3
+; FAST-NEXT:    xxswapd vs4, v3
+; FAST-NEXT:    xxsldwi vs5, v3, v3, 1
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    xscvspdpn f0, vs1
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    xxmrghd v0, vs0, vs1
+; FAST-NEXT:    xscvspdpn f0, v2
+; FAST-NEXT:    vmr v2, v0
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    xscvspdpn f0, vs2
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    xxmrghd v1, vs1, vs0
+; FAST-NEXT:    xscvspdpn f0, vs3
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    xscvspdpn f0, vs4
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    xxmrghd v4, vs0, vs1
+; FAST-NEXT:    xscvspdpn f0, v3
+; FAST-NEXT:    vmr v3, v1
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    xscvspdpn f0, vs5
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    xxmrghd v5, vs1, vs0
+; FAST-NEXT:    blr
+  %a = call <8 x i64> @llvm.llrint.v8i64.v8f32(<8 x float> %x)
+  ret <8 x i64> %a
+}
+declare <8 x i64> @llvm.llrint.v8i64.v8f32(<8 x float>)
+
+define <16 x i64> @llrint_v16i64_v16f32(<16 x float> %x) {
+; BE-LABEL: llrint_v16i64_v16f32:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -304(r1)
+; BE-NEXT:    std r0, 320(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 304
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    addi r3, r1, 112
+; BE-NEXT:    stxvw4x v2, 0, r3
+; BE-NEXT:    addi r3, r1, 128
+; BE-NEXT:    stxvw4x v3, 0, r3
+; BE-NEXT:    addi r3, r1, 144
+; BE-NEXT:    stxvw4x v4, 0, r3
+; BE-NEXT:    addi r3, r1, 160
+; BE-NEXT:    stxvw4x v5, 0, r3
+; BE-NEXT:    lfs f1, 116(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 112(r1)
+; BE-NEXT:    std r3, 184(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 124(r1)
+; BE-NEXT:    std r3, 176(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 120(r1)
+; BE-NEXT:    std r3, 200(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 132(r1)
+; BE-NEXT:    std r3, 192(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 128(r1)
+; BE-NEXT:    std r3, 216(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 140(r1)
+; BE-NEXT:    std r3, 208(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 136(r1)
+; BE-NEXT:    std r3, 232(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 148(r1)
+; BE-NEXT:    std r3, 224(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 144(r1)
+; BE-NEXT:    std r3, 248(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 156(r1)
+; BE-NEXT:    std r3, 240(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 152(r1)
+; BE-NEXT:    std r3, 264(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 164(r1)
+; BE-NEXT:    std r3, 256(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 160(r1)
+; BE-NEXT:    std r3, 280(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 172(r1)
+; BE-NEXT:    std r3, 272(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 168(r1)
+; BE-NEXT:    std r3, 296(r1)
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 288(r1)
+; BE-NEXT:    addi r3, r1, 176
+; BE-NEXT:    lxvd2x v2, 0, r3
+; BE-NEXT:    addi r3, r1, 192
+; BE-NEXT:    lxvd2x v3, 0, r3
+; BE-NEXT:    addi r3, r1, 208
+; BE-NEXT:    lxvd2x v4, 0, r3
+; BE-NEXT:    addi r3, r1, 224
+; BE-NEXT:    lxvd2x v5, 0, r3
+; BE-NEXT:    addi r3, r1, 240
+; BE-NEXT:    lxvd2x v6, 0, r3
+; BE-NEXT:    addi r3, r1, 256
+; BE-NEXT:    lxvd2x v7, 0, r3
+; BE-NEXT:    addi r3, r1, 272
+; BE-NEXT:    lxvd2x v8, 0, r3
+; BE-NEXT:    addi r3, r1, 288
+; BE-NEXT:    lxvd2x v9, 0, r3
+; BE-NEXT:    addi r1, r1, 304
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: llrint_v16i64_v16f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -176(r1)
+; CHECK-NEXT:    std r0, 192(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 176
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    .cfi_offset v24, -128
+; CHECK-NEXT:    .cfi_offset v25, -112
+; CHECK-NEXT:    .cfi_offset v26, -96
+; CHECK-NEXT:    .cfi_offset v27, -80
+; CHECK-NEXT:    .cfi_offset v28, -64
+; CHECK-NEXT:    .cfi_offset v29, -48
+; CHECK-NEXT:    .cfi_offset v30, -32
+; CHECK-NEXT:    .cfi_offset v31, -16
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    xxsldwi vs0, v2, v2, 3
+; CHECK-NEXT:    stvx v24, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    stvx v25, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 80
+; CHECK-NEXT:    stvx v26, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 96
+; CHECK-NEXT:    vmr v26, v3
+; CHECK-NEXT:    stvx v27, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 112
+; CHECK-NEXT:    stvx v28, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 128
+; CHECK-NEXT:    vmr v28, v4
+; CHECK-NEXT:    stvx v29, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 144
+; CHECK-NEXT:    vmr v29, v2
+; CHECK-NEXT:    stvx v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 160
+; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    vmr v31, v5
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxswapd vs0, v29
+; CHECK-NEXT:    mtvsrd v30, r3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xscvspdpn f1, v29
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v30, vs0, v30
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxsldwi vs0, v29, v29, 1
+; CHECK-NEXT:    mtvsrd v29, r3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v29, v29, vs0
+; CHECK-NEXT:    xxsldwi vs0, v26, v26, 3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxswapd vs0, v26
+; CHECK-NEXT:    mtvsrd v27, r3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xscvspdpn f1, v26
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v27, vs0, v27
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxsldwi vs0, v26, v26, 1
+; CHECK-NEXT:    mtvsrd v26, r3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v26, v26, vs0
+; CHECK-NEXT:    xxsldwi vs0, v28, v28, 3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxswapd vs0, v28
+; CHECK-NEXT:    mtvsrd v25, r3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xscvspdpn f1, v28
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v25, vs0, v25
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxsldwi vs0, v28, v28, 1
+; CHECK-NEXT:    mtvsrd v28, r3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v28, v28, vs0
+; CHECK-NEXT:    xxsldwi vs0, v31, v31, 3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxswapd vs0, v31
+; CHECK-NEXT:    mtvsrd v24, r3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xscvspdpn f1, v31
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v24, vs0, v24
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxsldwi vs0, v31, v31, 1
+; CHECK-NEXT:    mtvsrd v31, r3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl llrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 160
+; CHECK-NEXT:    vmr v2, v30
+; CHECK-NEXT:    vmr v3, v29
+; CHECK-NEXT:    vmr v7, v28
+; CHECK-NEXT:    vmr v4, v27
+; CHECK-NEXT:    vmr v5, v26
+; CHECK-NEXT:    vmr v6, v25
+; CHECK-NEXT:    vmr v8, v24
+; CHECK-NEXT:    xxmrghd v9, v31, vs0
+; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 144
+; CHECK-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 128
+; CHECK-NEXT:    lvx v29, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 112
+; CHECK-NEXT:    lvx v28, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 96
+; CHECK-NEXT:    lvx v27, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 80
+; CHECK-NEXT:    lvx v26, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    lvx v25, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    lvx v24, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    addi r1, r1, 176
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: llrint_v16i64_v16f32:
+; FAST:       # %bb.0:
+; FAST-NEXT:    xxsldwi vs0, v2, v2, 3
+; FAST-NEXT:    xxswapd vs1, v2
+; FAST-NEXT:    xscvspdpn f0, vs0
+; FAST-NEXT:    xxsldwi vs2, v2, v2, 1
+; FAST-NEXT:    xxsldwi vs3, v3, v3, 3
+; FAST-NEXT:    xxswapd vs4, v3
+; FAST-NEXT:    xxsldwi vs5, v3, v3, 1
+; FAST-NEXT:    xxsldwi vs6, v4, v4, 3
+; FAST-NEXT:    xxswapd vs7, v4
+; FAST-NEXT:    xxsldwi vs8, v4, v4, 1
+; FAST-NEXT:    xxsldwi vs9, v5, v5, 3
+; FAST-NEXT:    xxswapd vs10, v5
+; FAST-NEXT:    xxsldwi vs11, v5, v5, 1
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    xscvspdpn f0, vs1
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    xxmrghd v0, vs0, vs1
+; FAST-NEXT:    xscvspdpn f0, v2
+; FAST-NEXT:    vmr v2, v0
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    xscvspdpn f0, vs2
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    xxmrghd v1, vs1, vs0
+; FAST-NEXT:    xscvspdpn f0, vs3
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    xscvspdpn f0, vs4
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    xxmrghd v10, vs0, vs1
+; FAST-NEXT:    xscvspdpn f0, v3
+; FAST-NEXT:    vmr v3, v1
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    xscvspdpn f0, vs5
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    xxmrghd v11, vs1, vs0
+; FAST-NEXT:    xscvspdpn f0, vs6
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    xscvspdpn f0, vs7
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    xxmrghd v6, vs0, vs1
+; FAST-NEXT:    xscvspdpn f0, v4
+; FAST-NEXT:    xscvspdpn f1, vs8
+; FAST-NEXT:    vmr v4, v10
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    fctid f1, f1
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    mffprd r3, f1
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    xxmrghd v7, vs0, vs1
+; FAST-NEXT:    xscvspdpn f0, vs9
+; FAST-NEXT:    xscvspdpn f1, vs10
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    fctid f1, f1
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    mffprd r3, f1
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    xxmrghd v8, vs1, vs0
+; FAST-NEXT:    xscvspdpn f0, v5
+; FAST-NEXT:    xscvspdpn f1, vs11
+; FAST-NEXT:    vmr v5, v11
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    fctid f1, f1
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    mffprd r3, f1
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    xxmrghd v9, vs0, vs1
+; FAST-NEXT:    blr
+  %a = call <16 x i64> @llvm.llrint.v16i64.v16f32(<16 x float> %x)
+  ret <16 x i64> %a
+}
+declare <16 x i64> @llvm.llrint.v16i64.v16f32(<16 x float>)
+
+define <1 x i64> @llrint_v1i64_v1f64(<1 x double> %x) {
+; BE-LABEL: llrint_v1i64_v1f64:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -112(r1)
+; BE-NEXT:    std r0, 128(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 112
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    bl llrint
+; BE-NEXT:    nop
+; BE-NEXT:    addi r1, r1, 112
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: llrint_v1i64_v1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -32(r1)
+; CHECK-NEXT:    std r0, 48(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    bl llrint
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addi r1, r1, 32
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: llrint_v1i64_v1f64:
+; FAST:       # %bb.0:
+; FAST-NEXT:    fctid f0, f1
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    blr
+  %a = call <1 x i64> @llvm.llrint.v1i64.v1f64(<1 x double> %x)
+  ret <1 x i64> %a
+}
+declare <1 x i64> @llvm.llrint.v1i64.v1f64(<1 x double>)
+
+define <2 x i64> @llrint_v2i64_v2f64(<2 x double> %x) {
+; BE-LABEL: llrint_v2i64_v2f64:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -160(r1)
+; BE-NEXT:    std r0, 176(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 160
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    .cfi_offset v31, -16
+; BE-NEXT:    li r3, 144
+; BE-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    vmr v31, v2
+; BE-NEXT:    xxlor f1, v31, v31
+; BE-NEXT:    bl llrint
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 128(r1)
+; BE-NEXT:    xxswapd vs1, v31
+; BE-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; BE-NEXT:    bl llrint
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 136(r1)
+; BE-NEXT:    addi r3, r1, 128
+; BE-NEXT:    lxvd2x v2, 0, r3
+; BE-NEXT:    li r3, 144
+; BE-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    addi r1, r1, 160
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: llrint_v2i64_v2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -64(r1)
+; CHECK-NEXT:    std r0, 80(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    .cfi_offset v31, -16
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    vmr v31, v2
+; CHECK-NEXT:    xxlor f1, v31, v31
+; CHECK-NEXT:    bl llrint
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxswapd vs1, v31
+; CHECK-NEXT:    mtvsrd v31, r3
+; CHECK-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; CHECK-NEXT:    bl llrint
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    xxmrghd v2, v31, vs0
+; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    addi r1, r1, 64
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: llrint_v2i64_v2f64:
+; FAST:       # %bb.0:
+; FAST-NEXT:    xxlor f1, v2, v2
+; FAST-NEXT:    xxswapd vs0, v2
+; FAST-NEXT:    fctid f1, f1
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f1
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    xxmrghd v2, vs1, vs0
+; FAST-NEXT:    blr
+  %a = call <2 x i64> @llvm.llrint.v2i64.v2f64(<2 x double> %x)
+  ret <2 x i64> %a
+}
+declare <2 x i64> @llvm.llrint.v2i64.v2f64(<2 x double>)
+
+define <4 x i64> @llrint_v4i64_v4f64(<4 x double> %x) {
+; BE-LABEL: llrint_v4i64_v4f64:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -192(r1)
+; BE-NEXT:    std r0, 208(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 192
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    .cfi_offset v30, -32
+; BE-NEXT:    .cfi_offset v31, -16
+; BE-NEXT:    li r3, 160
+; BE-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    vmr v30, v2
+; BE-NEXT:    li r3, 176
+; BE-NEXT:    xxlor f1, v30, v30
+; BE-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    vmr v31, v3
+; BE-NEXT:    bl llrint
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 128(r1)
+; BE-NEXT:    xxswapd vs1, v30
+; BE-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; BE-NEXT:    bl llrint
+; BE-NEXT:    nop
+; BE-NEXT:    xxlor f1, v31, v31
+; BE-NEXT:    std r3, 136(r1)
+; BE-NEXT:    bl llrint
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 144(r1)
+; BE-NEXT:    xxswapd vs1, v31
+; BE-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; BE-NEXT:    bl llrint
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 152(r1)
+; BE-NEXT:    addi r3, r1, 128
+; BE-NEXT:    lxvd2x v2, 0, r3
+; BE-NEXT:    addi r3, r1, 144
+; BE-NEXT:    lxvd2x v3, 0, r3
+; BE-NEXT:    li r3, 176
+; BE-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    li r3, 160
+; BE-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    addi r1, r1, 192
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: llrint_v4i64_v4f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -80(r1)
+; CHECK-NEXT:    std r0, 96(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 80
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    .cfi_offset v30, -32
+; CHECK-NEXT:    .cfi_offset v31, -16
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    stvx v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    vmr v30, v2
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    xxlor f1, v30, v30
+; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    vmr v31, v3
+; CHECK-NEXT:    bl llrint
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxswapd vs1, v30
+; CHECK-NEXT:    mtvsrd v30, r3
+; CHECK-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; CHECK-NEXT:    bl llrint
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxlor f1, v31, v31
+; CHECK-NEXT:    xxmrghd v30, v30, vs0
+; CHECK-NEXT:    bl llrint
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxswapd vs1, v31
+; CHECK-NEXT:    mtvsrd v31, r3
+; CHECK-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; CHECK-NEXT:    bl llrint
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    vmr v2, v30
+; CHECK-NEXT:    xxmrghd v3, v31, vs0
+; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    addi r1, r1, 80
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: llrint_v4i64_v4f64:
+; FAST:       # %bb.0:
+; FAST-NEXT:    xxswapd vs0, v2
+; FAST-NEXT:    xxlor f2, v2, v2
+; FAST-NEXT:    xxswapd vs1, v3
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    fctid f2, f2
+; FAST-NEXT:    fctid f1, f1
+; FAST-NEXT:    mffprd r4, f0
+; FAST-NEXT:    xxlor f0, v3, v3
+; FAST-NEXT:    mffprd r3, f2
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mtfprd f2, r4
+; FAST-NEXT:    mffprd r5, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    mffprd r3, f1
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    xxmrghd v2, vs0, vs2
+; FAST-NEXT:    mtfprd f0, r5
+; FAST-NEXT:    xxmrghd v3, vs0, vs1
+; FAST-NEXT:    blr
+  %a = call <4 x i64> @llvm.llrint.v4i64.v4f64(<4 x double> %x)
+  ret <4 x i64> %a
+}
+declare <4 x i64> @llvm.llrint.v4i64.v4f64(<4 x double>)
+
+define <8 x i64> @llrint_v8i64_v8f64(<8 x double> %x) {
+; BE-LABEL: llrint_v8i64_v8f64:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -256(r1)
+; BE-NEXT:    std r0, 272(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 256
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    .cfi_offset v28, -64
+; BE-NEXT:    .cfi_offset v29, -48
+; BE-NEXT:    .cfi_offset v30, -32
+; BE-NEXT:    .cfi_offset v31, -16
+; BE-NEXT:    li r3, 192
+; BE-NEXT:    stxvd2x v28, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    li r3, 208
+; BE-NEXT:    vmr v28, v2
+; BE-NEXT:    xxlor f1, v28, v28
+; BE-NEXT:    stxvd2x v29, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    li r3, 224
+; BE-NEXT:    vmr v29, v3
+; BE-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    li r3, 240
+; BE-NEXT:    vmr v30, v4
+; BE-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    vmr v31, v5
+; BE-NEXT:    bl llrint
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 128(r1)
+; BE-NEXT:    xxswapd vs1, v28
+; BE-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; BE-NEXT:    bl llrint
+; BE-NEXT:    nop
+; BE-NEXT:    xxlor f1, v29, v29
+; BE-NEXT:    std r3, 136(r1)
+; BE-NEXT:    bl llrint
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 144(r1)
+; BE-NEXT:    xxswapd vs1, v29
+; BE-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; BE-NEXT:    bl llrint
+; BE-NEXT:    nop
+; BE-NEXT:    xxlor f1, v30, v30
+; BE-NEXT:    std r3, 152(r1)
+; BE-NEXT:    bl llrint
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 160(r1)
+; BE-NEXT:    xxswapd vs1, v30
+; BE-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; BE-NEXT:    bl llrint
+; BE-NEXT:    nop
+; BE-NEXT:    xxlor f1, v31, v31
+; BE-NEXT:    std r3, 168(r1)
+; BE-NEXT:    bl llrint
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 176(r1)
+; BE-NEXT:    xxswapd vs1, v31
+; BE-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; BE-NEXT:    bl llrint
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 184(r1)
+; BE-NEXT:    addi r3, r1, 128
+; BE-NEXT:    lxvd2x v2, 0, r3
+; BE-NEXT:    addi r3, r1, 144
+; BE-NEXT:    lxvd2x v3, 0, r3
+; BE-NEXT:    addi r3, r1, 160
+; BE-NEXT:    lxvd2x v4, 0, r3
+; BE-NEXT:    addi r3, r1, 176
+; BE-NEXT:    lxvd2x v5, 0, r3
+; BE-NEXT:    li r3, 240
+; BE-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    li r3, 224
+; BE-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    li r3, 208
+; BE-NEXT:    lxvd2x v29, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    li r3, 192
+; BE-NEXT:    lxvd2x v28, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    addi r1, r1, 256
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: llrint_v8i64_v8f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -112(r1)
+; CHECK-NEXT:    std r0, 128(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 112
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    .cfi_offset v28, -64
+; CHECK-NEXT:    .cfi_offset v29, -48
+; CHECK-NEXT:    .cfi_offset v30, -32
+; CHECK-NEXT:    .cfi_offset v31, -16
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    stvx v28, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    vmr v28, v2
+; CHECK-NEXT:    stvx v29, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 80
+; CHECK-NEXT:    xxlor f1, v28, v28
+; CHECK-NEXT:    vmr v29, v3
+; CHECK-NEXT:    stvx v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 96
+; CHECK-NEXT:    vmr v30, v4
+; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    vmr v31, v5
+; CHECK-NEXT:    bl llrint
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxswapd vs1, v28
+; CHECK-NEXT:    mtvsrd v28, r3
+; CHECK-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; CHECK-NEXT:    bl llrint
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxlor f1, v29, v29
+; CHECK-NEXT:    xxmrghd v28, v28, vs0
+; CHECK-NEXT:    bl llrint
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxswapd vs1, v29
+; CHECK-NEXT:    mtvsrd v29, r3
+; CHECK-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; CHECK-NEXT:    bl llrint
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxlor f1, v30, v30
+; CHECK-NEXT:    xxmrghd v29, v29, vs0
+; CHECK-NEXT:    bl llrint
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxswapd vs1, v30
+; CHECK-NEXT:    mtvsrd v30, r3
+; CHECK-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; CHECK-NEXT:    bl llrint
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxlor f1, v31, v31
+; CHECK-NEXT:    xxmrghd v30, v30, vs0
+; CHECK-NEXT:    bl llrint
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxswapd vs1, v31
+; CHECK-NEXT:    mtvsrd v31, r3
+; CHECK-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; CHECK-NEXT:    bl llrint
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 96
+; CHECK-NEXT:    vmr v4, v30
+; CHECK-NEXT:    vmr v3, v29
+; CHECK-NEXT:    vmr v2, v28
+; CHECK-NEXT:    xxmrghd v5, v31, vs0
+; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 80
+; CHECK-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    lvx v29, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    lvx v28, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    addi r1, r1, 112
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: llrint_v8i64_v8f64:
+; FAST:       # %bb.0:
+; FAST-NEXT:    xxswapd vs0, v2
+; FAST-NEXT:    xxswapd vs1, v3
+; FAST-NEXT:    xxlor f4, v2, v2
+; FAST-NEXT:    xxswapd vs2, v4
+; FAST-NEXT:    xxswapd vs3, v5
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    fctid f4, f4
+; FAST-NEXT:    mffprd r4, f0
+; FAST-NEXT:    xxlor f0, v3, v3
+; FAST-NEXT:    mffprd r3, f4
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r5, f0
+; FAST-NEXT:    fctid f0, f1
+; FAST-NEXT:    mtfprd f1, r4
+; FAST-NEXT:    mffprd r6, f0
+; FAST-NEXT:    xxlor f0, v4, v4
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mtfprd f4, r6
+; FAST-NEXT:    mffprd r7, f0
+; FAST-NEXT:    fctid f0, f2
+; FAST-NEXT:    mtfprd f2, r5
+; FAST-NEXT:    mtfprd f5, r7
+; FAST-NEXT:    mffprd r8, f0
+; FAST-NEXT:    xxlor f0, v5, v5
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mtfprd f6, r8
+; FAST-NEXT:    mffprd r9, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    xxmrghd v3, vs2, vs4
+; FAST-NEXT:    xxmrghd v4, vs5, vs6
+; FAST-NEXT:    xxmrghd v2, vs0, vs1
+; FAST-NEXT:    fctid f1, f3
+; FAST-NEXT:    mtfprd f0, r9
+; FAST-NEXT:    mffprd r3, f1
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    xxmrghd v5, vs0, vs1
+; FAST-NEXT:    blr
+  %a = call <8 x i64> @llvm.llrint.v8i64.v8f64(<8 x double> %x)
+  ret <8 x i64> %a
+}
+declare <8 x i64> @llvm.llrint.v8i64.v8f64(<8 x double>)
diff --git a/llvm/test/CodeGen/PowerPC/vector-lrint.ll b/llvm/test/CodeGen/PowerPC/vector-lrint.ll
new file mode 100644
index 0000000000..9667a26120
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/vector-lrint.ll
@@ -0,0 +1,4859 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: sed 's/iXLen/i32/g' %s | llc -mcpu=pwr7 -ppc-asm-full-reg-names \
+; RUN:   -ppc-vsr-nums-as-vr -mtriple=powerpc64-unknown-unknown \
+; RUN:   -verify-machineinstrs | FileCheck %s --check-prefixes=BE
+; RUN: sed 's/iXLen/i32/g' %s | llc -mcpu=pwr8 -ppc-asm-full-reg-names \
+; RUN:   -ppc-vsr-nums-as-vr -mtriple=powerpc64le-unknown-unknown \
+; RUN:   -verify-machineinstrs | FileCheck %s
+; RUN: sed 's/iXLen/i32/g' %s | llc -mcpu=pwr8 -ppc-asm-full-reg-names \
+; RUN:   -ppc-vsr-nums-as-vr -mtriple=powerpc64le-unknown-unknown \
+; RUN:   -verify-machineinstrs --enable-unsafe-fp-math | \
+; RUN:   FileCheck %s --check-prefixes=FAST
+; RUN: sed 's/iXLen/i64/g' %s | llc -mcpu=pwr7 -ppc-asm-full-reg-names \
+; RUN:   -ppc-vsr-nums-as-vr -mtriple=powerpc64-unknown-unknown \
+; RUN:   -verify-machineinstrs | FileCheck %s --check-prefixes=BE
+; RUN: sed 's/iXLen/i64/g' %s | llc -mcpu=pwr8 -ppc-asm-full-reg-names \
+; RUN:   -ppc-vsr-nums-as-vr -mtriple=powerpc64le-unknown-unknown \
+; RUN:   -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK
+; RUN: sed 's/iXLen/i64/g' %s | llc -mcpu=pwr8 -ppc-asm-full-reg-names \
+; RUN:   -ppc-vsr-nums-as-vr -mtriple=powerpc64le-unknown-unknown \
+; RUN:   -verify-machineinstrs --enable-unsafe-fp-math | \
+; RUN:   FileCheck %s --check-prefixes=FAST
+
+define <1 x i64> @lrint_v1f16(<1 x half> %x) {
+; BE-LABEL: lrint_v1f16:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -112(r1)
+; BE-NEXT:    std r0, 128(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 112
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    addi r1, r1, 112
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: lrint_v1f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -32(r1)
+; CHECK-NEXT:    std r0, 48(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addi r1, r1, 32
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: lrint_v1f16:
+; FAST:       # %bb.0:
+; FAST-NEXT:    mflr r0
+; FAST-NEXT:    stdu r1, -32(r1)
+; FAST-NEXT:    std r0, 48(r1)
+; FAST-NEXT:    .cfi_def_cfa_offset 32
+; FAST-NEXT:    .cfi_offset lr, 16
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fctid f0, f1
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    addi r1, r1, 32
+; FAST-NEXT:    ld r0, 16(r1)
+; FAST-NEXT:    mtlr r0
+; FAST-NEXT:    blr
+  %a = call <1 x i64> @llvm.lrint.v1i64.v1f16(<1 x half> %x)
+  ret <1 x i64> %a
+}
+declare <1 x i64> @llvm.lrint.v1i64.v1f16(<1 x half>)
+
+define <2 x i64> @lrint_v2f16(<2 x half> %x) {
+; BE-LABEL: lrint_v2f16:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -160(r1)
+; BE-NEXT:    std r0, 176(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 160
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    .cfi_offset r30, -24
+; BE-NEXT:    .cfi_offset f31, -8
+; BE-NEXT:    stfd f31, 152(r1) # 8-byte Folded Spill
+; BE-NEXT:    fmr f31, f1
+; BE-NEXT:    fmr f1, f2
+; BE-NEXT:    std r30, 136(r1) # 8-byte Folded Spill
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f31
+; BE-NEXT:    mr r30, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r30, 48
+; BE-NEXT:    fmr f31, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f31
+; BE-NEXT:    std r3, 120(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 112(r1)
+; BE-NEXT:    addi r3, r1, 112
+; BE-NEXT:    ld r30, 136(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f31, 152(r1) # 8-byte Folded Reload
+; BE-NEXT:    lxvd2x v2, 0, r3
+; BE-NEXT:    addi r1, r1, 160
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: lrint_v2f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -96(r1)
+; CHECK-NEXT:    std r0, 112(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 96
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    .cfi_offset r30, -24
+; CHECK-NEXT:    .cfi_offset f31, -8
+; CHECK-NEXT:    .cfi_offset v31, -48
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    std r30, 72(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stfd f31, 88(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f31, f2
+; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f31
+; CHECK-NEXT:    mr r30, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r30, 48
+; CHECK-NEXT:    fmr f31, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f31
+; CHECK-NEXT:    mtvsrd v31, r3
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    lfd f31, 88(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r30, 72(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    xxmrghd v2, vs0, v31
+; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    addi r1, r1, 96
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: lrint_v2f16:
+; FAST:       # %bb.0:
+; FAST-NEXT:    mflr r0
+; FAST-NEXT:    .cfi_def_cfa_offset 48
+; FAST-NEXT:    .cfi_offset lr, 16
+; FAST-NEXT:    .cfi_offset f30, -16
+; FAST-NEXT:    .cfi_offset f31, -8
+; FAST-NEXT:    stfd f30, -16(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f31, -8(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stdu r1, -48(r1)
+; FAST-NEXT:    fmr f31, f1
+; FAST-NEXT:    fmr f1, f2
+; FAST-NEXT:    std r0, 64(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f30, f1
+; FAST-NEXT:    fmr f1, f31
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fctid f0, f1
+; FAST-NEXT:    fctid f1, f30
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    mffprd r3, f1
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    xxmrghd v2, vs1, vs0
+; FAST-NEXT:    addi r1, r1, 48
+; FAST-NEXT:    ld r0, 16(r1)
+; FAST-NEXT:    lfd f31, -8(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f30, -16(r1) # 8-byte Folded Reload
+; FAST-NEXT:    mtlr r0
+; FAST-NEXT:    blr
+  %a = call <2 x i64> @llvm.lrint.v2i64.v2f16(<2 x half> %x)
+  ret <2 x i64> %a
+}
+declare <2 x i64> @llvm.lrint.v2i64.v2f16(<2 x half>)
+
+define <4 x i64> @lrint_v4f16(<4 x half> %x) {
+; BE-LABEL: lrint_v4f16:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -208(r1)
+; BE-NEXT:    std r0, 224(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 208
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    .cfi_offset r28, -56
+; BE-NEXT:    .cfi_offset r29, -48
+; BE-NEXT:    .cfi_offset r30, -40
+; BE-NEXT:    .cfi_offset f29, -24
+; BE-NEXT:    .cfi_offset f30, -16
+; BE-NEXT:    .cfi_offset f31, -8
+; BE-NEXT:    stfd f29, 184(r1) # 8-byte Folded Spill
+; BE-NEXT:    fmr f29, f1
+; BE-NEXT:    fmr f1, f2
+; BE-NEXT:    std r28, 152(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r29, 160(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r30, 168(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f30, 192(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f31, 200(r1) # 8-byte Folded Spill
+; BE-NEXT:    fmr f31, f4
+; BE-NEXT:    fmr f30, f3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f29
+; BE-NEXT:    mr r30, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f31
+; BE-NEXT:    mr r29, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f30
+; BE-NEXT:    mr r28, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r28, 48
+; BE-NEXT:    fmr f31, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r29, 48
+; BE-NEXT:    fmr f30, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r30, 48
+; BE-NEXT:    fmr f29, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f29
+; BE-NEXT:    std r3, 120(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f30
+; BE-NEXT:    std r3, 112(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f31
+; BE-NEXT:    std r3, 136(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 128(r1)
+; BE-NEXT:    addi r3, r1, 112
+; BE-NEXT:    ld r30, 168(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f31, 200(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f30, 192(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f29, 184(r1) # 8-byte Folded Reload
+; BE-NEXT:    lxvd2x v2, 0, r3
+; BE-NEXT:    addi r3, r1, 128
+; BE-NEXT:    ld r29, 160(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r28, 152(r1) # 8-byte Folded Reload
+; BE-NEXT:    lxvd2x v3, 0, r3
+; BE-NEXT:    addi r1, r1, 208
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: lrint_v4f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -144(r1)
+; CHECK-NEXT:    std r0, 160(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 144
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    .cfi_offset r28, -56
+; CHECK-NEXT:    .cfi_offset r29, -48
+; CHECK-NEXT:    .cfi_offset r30, -40
+; CHECK-NEXT:    .cfi_offset f29, -24
+; CHECK-NEXT:    .cfi_offset f30, -16
+; CHECK-NEXT:    .cfi_offset f31, -8
+; CHECK-NEXT:    .cfi_offset v30, -96
+; CHECK-NEXT:    .cfi_offset v31, -80
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    std r28, 88(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r29, 96(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r30, 104(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stfd f29, 120(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f29, f2
+; CHECK-NEXT:    stfd f30, 128(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f30, f3
+; CHECK-NEXT:    stvx v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    stfd f31, 136(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f31, f4
+; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f29
+; CHECK-NEXT:    mr r30, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f30
+; CHECK-NEXT:    mr r29, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f31
+; CHECK-NEXT:    mr r28, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r28, 48
+; CHECK-NEXT:    fmr f31, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r29, 48
+; CHECK-NEXT:    fmr f30, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r30, 48
+; CHECK-NEXT:    fmr f29, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f29
+; CHECK-NEXT:    mtvsrd v31, r3
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f30
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v31, vs0, v31
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f31
+; CHECK-NEXT:    mtvsrd v30, r3
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    vmr v2, v31
+; CHECK-NEXT:    lfd f31, 136(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f30, 128(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f29, 120(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r30, 104(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r29, 96(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    ld r28, 88(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    xxmrghd v3, vs0, v30
+; CHECK-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    addi r1, r1, 144
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: lrint_v4f16:
+; FAST:       # %bb.0:
+; FAST-NEXT:    mflr r0
+; FAST-NEXT:    .cfi_def_cfa_offset 64
+; FAST-NEXT:    .cfi_offset lr, 16
+; FAST-NEXT:    .cfi_offset f28, -32
+; FAST-NEXT:    .cfi_offset f29, -24
+; FAST-NEXT:    .cfi_offset f30, -16
+; FAST-NEXT:    .cfi_offset f31, -8
+; FAST-NEXT:    stfd f28, -32(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f29, -24(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f30, -16(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f31, -8(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stdu r1, -64(r1)
+; FAST-NEXT:    fmr f29, f1
+; FAST-NEXT:    fmr f1, f4
+; FAST-NEXT:    std r0, 80(r1)
+; FAST-NEXT:    fmr f31, f3
+; FAST-NEXT:    fmr f30, f2
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f28, f1
+; FAST-NEXT:    fmr f1, f31
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f31, f1
+; FAST-NEXT:    fmr f1, f30
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f30, f1
+; FAST-NEXT:    fmr f1, f29
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fctid f0, f30
+; FAST-NEXT:    fctid f2, f31
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    fctid f1, f1
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    mffprd r3, f2
+; FAST-NEXT:    mtfprd f2, r3
+; FAST-NEXT:    mffprd r3, f1
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    xxmrghd v2, vs0, vs1
+; FAST-NEXT:    fctid f0, f28
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    xxmrghd v3, vs0, vs2
+; FAST-NEXT:    addi r1, r1, 64
+; FAST-NEXT:    ld r0, 16(r1)
+; FAST-NEXT:    lfd f31, -8(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f30, -16(r1) # 8-byte Folded Reload
+; FAST-NEXT:    mtlr r0
+; FAST-NEXT:    lfd f29, -24(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f28, -32(r1) # 8-byte Folded Reload
+; FAST-NEXT:    blr
+  %a = call <4 x i64> @llvm.lrint.v4i64.v4f16(<4 x half> %x)
+  ret <4 x i64> %a
+}
+declare <4 x i64> @llvm.lrint.v4i64.v4f16(<4 x half>)
+
+define <8 x i64> @lrint_v8f16(<8 x half> %x) {
+; BE-LABEL: lrint_v8f16:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -304(r1)
+; BE-NEXT:    std r0, 320(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 304
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    .cfi_offset r24, -120
+; BE-NEXT:    .cfi_offset r25, -112
+; BE-NEXT:    .cfi_offset r26, -104
+; BE-NEXT:    .cfi_offset r27, -96
+; BE-NEXT:    .cfi_offset r28, -88
+; BE-NEXT:    .cfi_offset r29, -80
+; BE-NEXT:    .cfi_offset r30, -72
+; BE-NEXT:    .cfi_offset f25, -56
+; BE-NEXT:    .cfi_offset f26, -48
+; BE-NEXT:    .cfi_offset f27, -40
+; BE-NEXT:    .cfi_offset f28, -32
+; BE-NEXT:    .cfi_offset f29, -24
+; BE-NEXT:    .cfi_offset f30, -16
+; BE-NEXT:    .cfi_offset f31, -8
+; BE-NEXT:    stfd f25, 248(r1) # 8-byte Folded Spill
+; BE-NEXT:    fmr f25, f1
+; BE-NEXT:    fmr f1, f2
+; BE-NEXT:    std r24, 184(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r25, 192(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r26, 200(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r27, 208(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r28, 216(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r29, 224(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r30, 232(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f26, 256(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f27, 264(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f28, 272(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f29, 280(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f30, 288(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f31, 296(r1) # 8-byte Folded Spill
+; BE-NEXT:    fmr f31, f8
+; BE-NEXT:    fmr f30, f7
+; BE-NEXT:    fmr f29, f6
+; BE-NEXT:    fmr f28, f5
+; BE-NEXT:    fmr f27, f4
+; BE-NEXT:    fmr f26, f3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f25
+; BE-NEXT:    mr r30, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f27
+; BE-NEXT:    mr r29, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f26
+; BE-NEXT:    mr r28, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f29
+; BE-NEXT:    mr r27, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f28
+; BE-NEXT:    mr r26, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f31
+; BE-NEXT:    mr r25, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f30
+; BE-NEXT:    mr r24, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r24, 48
+; BE-NEXT:    fmr f31, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r25, 48
+; BE-NEXT:    fmr f30, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r26, 48
+; BE-NEXT:    fmr f29, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r27, 48
+; BE-NEXT:    fmr f28, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r28, 48
+; BE-NEXT:    fmr f27, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r29, 48
+; BE-NEXT:    fmr f26, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r30, 48
+; BE-NEXT:    fmr f25, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f25
+; BE-NEXT:    std r3, 120(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f26
+; BE-NEXT:    std r3, 112(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f27
+; BE-NEXT:    std r3, 136(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f28
+; BE-NEXT:    std r3, 128(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f29
+; BE-NEXT:    std r3, 152(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f30
+; BE-NEXT:    std r3, 144(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f31
+; BE-NEXT:    std r3, 168(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 160(r1)
+; BE-NEXT:    addi r3, r1, 112
+; BE-NEXT:    ld r30, 232(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f31, 296(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f30, 288(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f29, 280(r1) # 8-byte Folded Reload
+; BE-NEXT:    lxvd2x v2, 0, r3
+; BE-NEXT:    addi r3, r1, 128
+; BE-NEXT:    lfd f28, 272(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f27, 264(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f26, 256(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r29, 224(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r28, 216(r1) # 8-byte Folded Reload
+; BE-NEXT:    lxvd2x v3, 0, r3
+; BE-NEXT:    addi r3, r1, 144
+; BE-NEXT:    lfd f25, 248(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r27, 208(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r26, 200(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r25, 192(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r24, 184(r1) # 8-byte Folded Reload
+; BE-NEXT:    lxvd2x v4, 0, r3
+; BE-NEXT:    addi r3, r1, 160
+; BE-NEXT:    lxvd2x v5, 0, r3
+; BE-NEXT:    addi r1, r1, 304
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: lrint_v8f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -240(r1)
+; CHECK-NEXT:    std r0, 256(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 240
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    .cfi_offset r24, -120
+; CHECK-NEXT:    .cfi_offset r25, -112
+; CHECK-NEXT:    .cfi_offset r26, -104
+; CHECK-NEXT:    .cfi_offset r27, -96
+; CHECK-NEXT:    .cfi_offset r28, -88
+; CHECK-NEXT:    .cfi_offset r29, -80
+; CHECK-NEXT:    .cfi_offset r30, -72
+; CHECK-NEXT:    .cfi_offset f25, -56
+; CHECK-NEXT:    .cfi_offset f26, -48
+; CHECK-NEXT:    .cfi_offset f27, -40
+; CHECK-NEXT:    .cfi_offset f28, -32
+; CHECK-NEXT:    .cfi_offset f29, -24
+; CHECK-NEXT:    .cfi_offset f30, -16
+; CHECK-NEXT:    .cfi_offset f31, -8
+; CHECK-NEXT:    .cfi_offset v28, -192
+; CHECK-NEXT:    .cfi_offset v29, -176
+; CHECK-NEXT:    .cfi_offset v30, -160
+; CHECK-NEXT:    .cfi_offset v31, -144
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    std r24, 120(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r25, 128(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r26, 136(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r27, 144(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r28, 152(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r29, 160(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r30, 168(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stvx v28, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    stfd f25, 184(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f25, f2
+; CHECK-NEXT:    stfd f26, 192(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f26, f3
+; CHECK-NEXT:    stfd f27, 200(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f27, f4
+; CHECK-NEXT:    stvx v29, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 80
+; CHECK-NEXT:    stfd f28, 208(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f28, f5
+; CHECK-NEXT:    stfd f29, 216(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f29, f6
+; CHECK-NEXT:    stfd f30, 224(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f30, f7
+; CHECK-NEXT:    stvx v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 96
+; CHECK-NEXT:    stfd f31, 232(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f31, f8
+; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f25
+; CHECK-NEXT:    mr r30, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f26
+; CHECK-NEXT:    mr r29, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f27
+; CHECK-NEXT:    mr r28, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f28
+; CHECK-NEXT:    mr r27, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f29
+; CHECK-NEXT:    mr r26, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f30
+; CHECK-NEXT:    mr r25, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f31
+; CHECK-NEXT:    mr r24, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r24, 48
+; CHECK-NEXT:    fmr f31, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r25, 48
+; CHECK-NEXT:    fmr f30, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r26, 48
+; CHECK-NEXT:    fmr f29, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r27, 48
+; CHECK-NEXT:    fmr f28, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r28, 48
+; CHECK-NEXT:    fmr f27, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r29, 48
+; CHECK-NEXT:    fmr f26, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r30, 48
+; CHECK-NEXT:    fmr f25, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f25
+; CHECK-NEXT:    mtvsrd v31, r3
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f26
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v31, vs0, v31
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f27
+; CHECK-NEXT:    mtvsrd v30, r3
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f28
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v30, vs0, v30
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f29
+; CHECK-NEXT:    mtvsrd v29, r3
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f30
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v29, vs0, v29
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f31
+; CHECK-NEXT:    mtvsrd v28, r3
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 96
+; CHECK-NEXT:    vmr v2, v31
+; CHECK-NEXT:    lfd f31, 232(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    vmr v3, v30
+; CHECK-NEXT:    vmr v4, v29
+; CHECK-NEXT:    lfd f30, 224(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f29, 216(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 80
+; CHECK-NEXT:    lfd f28, 208(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f27, 200(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f26, 192(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f25, 184(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r30, 168(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r29, 160(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    ld r28, 152(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r27, 144(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    xxmrghd v5, vs0, v28
+; CHECK-NEXT:    ld r26, 136(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r25, 128(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r24, 120(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lvx v29, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    lvx v28, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    addi r1, r1, 240
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: lrint_v8f16:
+; FAST:       # %bb.0:
+; FAST-NEXT:    mflr r0
+; FAST-NEXT:    .cfi_def_cfa_offset 96
+; FAST-NEXT:    .cfi_offset lr, 16
+; FAST-NEXT:    .cfi_offset f24, -64
+; FAST-NEXT:    .cfi_offset f25, -56
+; FAST-NEXT:    .cfi_offset f26, -48
+; FAST-NEXT:    .cfi_offset f27, -40
+; FAST-NEXT:    .cfi_offset f28, -32
+; FAST-NEXT:    .cfi_offset f29, -24
+; FAST-NEXT:    .cfi_offset f30, -16
+; FAST-NEXT:    .cfi_offset f31, -8
+; FAST-NEXT:    stfd f24, -64(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f25, -56(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f26, -48(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f27, -40(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f28, -32(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f29, -24(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f30, -16(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f31, -8(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stdu r1, -96(r1)
+; FAST-NEXT:    fmr f24, f1
+; FAST-NEXT:    fmr f1, f8
+; FAST-NEXT:    std r0, 112(r1)
+; FAST-NEXT:    fmr f30, f7
+; FAST-NEXT:    fmr f29, f6
+; FAST-NEXT:    fmr f28, f5
+; FAST-NEXT:    fmr f27, f4
+; FAST-NEXT:    fmr f26, f3
+; FAST-NEXT:    fmr f25, f2
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f31, f1
+; FAST-NEXT:    fmr f1, f30
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f30, f1
+; FAST-NEXT:    fmr f1, f29
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f29, f1
+; FAST-NEXT:    fmr f1, f28
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f28, f1
+; FAST-NEXT:    fmr f1, f27
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f27, f1
+; FAST-NEXT:    fmr f1, f26
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f26, f1
+; FAST-NEXT:    fmr f1, f25
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f25, f1
+; FAST-NEXT:    fmr f1, f24
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fctid f0, f25
+; FAST-NEXT:    fctid f2, f26
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    fctid f3, f27
+; FAST-NEXT:    fctid f4, f28
+; FAST-NEXT:    fctid f5, f29
+; FAST-NEXT:    fctid f6, f30
+; FAST-NEXT:    fctid f1, f1
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    mffprd r3, f2
+; FAST-NEXT:    mtfprd f2, r3
+; FAST-NEXT:    mffprd r3, f3
+; FAST-NEXT:    mtfprd f3, r3
+; FAST-NEXT:    mffprd r3, f4
+; FAST-NEXT:    mtfprd f4, r3
+; FAST-NEXT:    mffprd r3, f5
+; FAST-NEXT:    mtfprd f5, r3
+; FAST-NEXT:    mffprd r3, f6
+; FAST-NEXT:    mtfprd f6, r3
+; FAST-NEXT:    mffprd r3, f1
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    xxmrghd v3, vs3, vs2
+; FAST-NEXT:    xxmrghd v4, vs5, vs4
+; FAST-NEXT:    xxmrghd v2, vs0, vs1
+; FAST-NEXT:    fctid f0, f31
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    xxmrghd v5, vs0, vs6
+; FAST-NEXT:    addi r1, r1, 96
+; FAST-NEXT:    ld r0, 16(r1)
+; FAST-NEXT:    lfd f31, -8(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f30, -16(r1) # 8-byte Folded Reload
+; FAST-NEXT:    mtlr r0
+; FAST-NEXT:    lfd f29, -24(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f28, -32(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f27, -40(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f26, -48(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f25, -56(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f24, -64(r1) # 8-byte Folded Reload
+; FAST-NEXT:    blr
+  %a = call <8 x i64> @llvm.lrint.v8i64.v8f16(<8 x half> %x)
+  ret <8 x i64> %a
+}
+declare <8 x i64> @llvm.lrint.v8i64.v8f16(<8 x half>)
+
+define <16 x i64> @lrint_v16i64_v16f16(<16 x half> %x) {
+; BE-LABEL: lrint_v16i64_v16f16:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -496(r1)
+; BE-NEXT:    std r0, 512(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 496
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    .cfi_offset r16, -248
+; BE-NEXT:    .cfi_offset r17, -240
+; BE-NEXT:    .cfi_offset r18, -232
+; BE-NEXT:    .cfi_offset r19, -224
+; BE-NEXT:    .cfi_offset r20, -216
+; BE-NEXT:    .cfi_offset r21, -208
+; BE-NEXT:    .cfi_offset r22, -200
+; BE-NEXT:    .cfi_offset r23, -192
+; BE-NEXT:    .cfi_offset r24, -184
+; BE-NEXT:    .cfi_offset r25, -176
+; BE-NEXT:    .cfi_offset r26, -168
+; BE-NEXT:    .cfi_offset r27, -160
+; BE-NEXT:    .cfi_offset r28, -152
+; BE-NEXT:    .cfi_offset r29, -144
+; BE-NEXT:    .cfi_offset r30, -136
+; BE-NEXT:    .cfi_offset f17, -120
+; BE-NEXT:    .cfi_offset f18, -112
+; BE-NEXT:    .cfi_offset f19, -104
+; BE-NEXT:    .cfi_offset f20, -96
+; BE-NEXT:    .cfi_offset f21, -88
+; BE-NEXT:    .cfi_offset f22, -80
+; BE-NEXT:    .cfi_offset f23, -72
+; BE-NEXT:    .cfi_offset f24, -64
+; BE-NEXT:    .cfi_offset f25, -56
+; BE-NEXT:    .cfi_offset f26, -48
+; BE-NEXT:    .cfi_offset f27, -40
+; BE-NEXT:    .cfi_offset f28, -32
+; BE-NEXT:    .cfi_offset f29, -24
+; BE-NEXT:    .cfi_offset f30, -16
+; BE-NEXT:    .cfi_offset f31, -8
+; BE-NEXT:    stfd f20, 400(r1) # 8-byte Folded Spill
+; BE-NEXT:    fmr f20, f1
+; BE-NEXT:    fmr f1, f2
+; BE-NEXT:    std r16, 248(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r17, 256(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r18, 264(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r19, 272(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r20, 280(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r21, 288(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r22, 296(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r23, 304(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r24, 312(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r25, 320(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r26, 328(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r27, 336(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r28, 344(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r29, 352(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r30, 360(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f17, 376(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f18, 384(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f19, 392(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f21, 408(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f22, 416(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f23, 424(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f24, 432(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f25, 440(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f26, 448(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f27, 456(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f28, 464(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f29, 472(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f30, 480(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f31, 488(r1) # 8-byte Folded Spill
+; BE-NEXT:    fmr f31, f13
+; BE-NEXT:    fmr f29, f12
+; BE-NEXT:    fmr f30, f11
+; BE-NEXT:    fmr f28, f10
+; BE-NEXT:    fmr f27, f9
+; BE-NEXT:    fmr f26, f8
+; BE-NEXT:    fmr f25, f7
+; BE-NEXT:    fmr f24, f6
+; BE-NEXT:    fmr f23, f5
+; BE-NEXT:    fmr f22, f4
+; BE-NEXT:    fmr f21, f3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f20
+; BE-NEXT:    mr r30, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f22
+; BE-NEXT:    mr r29, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f21
+; BE-NEXT:    mr r28, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f24
+; BE-NEXT:    mr r27, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f23
+; BE-NEXT:    mr r26, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f26
+; BE-NEXT:    mr r25, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f25
+; BE-NEXT:    mr r24, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f28
+; BE-NEXT:    mr r23, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f27
+; BE-NEXT:    mr r22, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f29
+; BE-NEXT:    mr r21, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f30
+; BE-NEXT:    mr r20, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 652(r1)
+; BE-NEXT:    mr r19, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f31
+; BE-NEXT:    mr r18, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 668(r1)
+; BE-NEXT:    mr r17, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 660(r1)
+; BE-NEXT:    mr r16, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r16, 48
+; BE-NEXT:    fmr f31, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r17, 48
+; BE-NEXT:    fmr f30, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r18, 48
+; BE-NEXT:    fmr f29, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r19, 48
+; BE-NEXT:    fmr f28, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r20, 48
+; BE-NEXT:    fmr f27, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r21, 48
+; BE-NEXT:    fmr f26, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r22, 48
+; BE-NEXT:    fmr f25, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r23, 48
+; BE-NEXT:    fmr f24, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r24, 48
+; BE-NEXT:    fmr f23, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r25, 48
+; BE-NEXT:    fmr f22, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r26, 48
+; BE-NEXT:    fmr f21, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r27, 48
+; BE-NEXT:    fmr f20, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r28, 48
+; BE-NEXT:    fmr f19, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r29, 48
+; BE-NEXT:    fmr f18, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r30, 48
+; BE-NEXT:    fmr f17, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f17
+; BE-NEXT:    std r3, 120(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f18
+; BE-NEXT:    std r3, 112(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f19
+; BE-NEXT:    std r3, 136(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f20
+; BE-NEXT:    std r3, 128(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f21
+; BE-NEXT:    std r3, 152(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f22
+; BE-NEXT:    std r3, 144(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f23
+; BE-NEXT:    std r3, 168(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f24
+; BE-NEXT:    std r3, 160(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f25
+; BE-NEXT:    std r3, 184(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f26
+; BE-NEXT:    std r3, 176(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f27
+; BE-NEXT:    std r3, 200(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f28
+; BE-NEXT:    std r3, 192(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f29
+; BE-NEXT:    std r3, 216(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f30
+; BE-NEXT:    std r3, 208(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f31
+; BE-NEXT:    std r3, 232(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 224(r1)
+; BE-NEXT:    addi r3, r1, 112
+; BE-NEXT:    ld r30, 360(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f31, 488(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f30, 480(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f29, 472(r1) # 8-byte Folded Reload
+; BE-NEXT:    lxvd2x v2, 0, r3
+; BE-NEXT:    addi r3, r1, 128
+; BE-NEXT:    lfd f28, 464(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f27, 456(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f26, 448(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r29, 352(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r28, 344(r1) # 8-byte Folded Reload
+; BE-NEXT:    lxvd2x v3, 0, r3
+; BE-NEXT:    addi r3, r1, 144
+; BE-NEXT:    lfd f25, 440(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f24, 432(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f23, 424(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r27, 336(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r26, 328(r1) # 8-byte Folded Reload
+; BE-NEXT:    lxvd2x v4, 0, r3
+; BE-NEXT:    addi r3, r1, 160
+; BE-NEXT:    lfd f22, 416(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f21, 408(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f20, 400(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r25, 320(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r24, 312(r1) # 8-byte Folded Reload
+; BE-NEXT:    lxvd2x v5, 0, r3
+; BE-NEXT:    addi r3, r1, 176
+; BE-NEXT:    lfd f19, 392(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f18, 384(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f17, 376(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r23, 304(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r22, 296(r1) # 8-byte Folded Reload
+; BE-NEXT:    lxvd2x v6, 0, r3
+; BE-NEXT:    addi r3, r1, 192
+; BE-NEXT:    ld r21, 288(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r20, 280(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r19, 272(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r18, 264(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r17, 256(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r16, 248(r1) # 8-byte Folded Reload
+; BE-NEXT:    lxvd2x v7, 0, r3
+; BE-NEXT:    addi r3, r1, 208
+; BE-NEXT:    lxvd2x v8, 0, r3
+; BE-NEXT:    addi r3, r1, 224
+; BE-NEXT:    lxvd2x v9, 0, r3
+; BE-NEXT:    addi r1, r1, 496
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: lrint_v16i64_v16f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -432(r1)
+; CHECK-NEXT:    std r0, 448(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 432
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    .cfi_offset r16, -248
+; CHECK-NEXT:    .cfi_offset r17, -240
+; CHECK-NEXT:    .cfi_offset r18, -232
+; CHECK-NEXT:    .cfi_offset r19, -224
+; CHECK-NEXT:    .cfi_offset r20, -216
+; CHECK-NEXT:    .cfi_offset r21, -208
+; CHECK-NEXT:    .cfi_offset r22, -200
+; CHECK-NEXT:    .cfi_offset r23, -192
+; CHECK-NEXT:    .cfi_offset r24, -184
+; CHECK-NEXT:    .cfi_offset r25, -176
+; CHECK-NEXT:    .cfi_offset r26, -168
+; CHECK-NEXT:    .cfi_offset r27, -160
+; CHECK-NEXT:    .cfi_offset r28, -152
+; CHECK-NEXT:    .cfi_offset r29, -144
+; CHECK-NEXT:    .cfi_offset r30, -136
+; CHECK-NEXT:    .cfi_offset f17, -120
+; CHECK-NEXT:    .cfi_offset f18, -112
+; CHECK-NEXT:    .cfi_offset f19, -104
+; CHECK-NEXT:    .cfi_offset f20, -96
+; CHECK-NEXT:    .cfi_offset f21, -88
+; CHECK-NEXT:    .cfi_offset f22, -80
+; CHECK-NEXT:    .cfi_offset f23, -72
+; CHECK-NEXT:    .cfi_offset f24, -64
+; CHECK-NEXT:    .cfi_offset f25, -56
+; CHECK-NEXT:    .cfi_offset f26, -48
+; CHECK-NEXT:    .cfi_offset f27, -40
+; CHECK-NEXT:    .cfi_offset f28, -32
+; CHECK-NEXT:    .cfi_offset f29, -24
+; CHECK-NEXT:    .cfi_offset f30, -16
+; CHECK-NEXT:    .cfi_offset f31, -8
+; CHECK-NEXT:    .cfi_offset v24, -384
+; CHECK-NEXT:    .cfi_offset v25, -368
+; CHECK-NEXT:    .cfi_offset v26, -352
+; CHECK-NEXT:    .cfi_offset v27, -336
+; CHECK-NEXT:    .cfi_offset v28, -320
+; CHECK-NEXT:    .cfi_offset v29, -304
+; CHECK-NEXT:    .cfi_offset v30, -288
+; CHECK-NEXT:    .cfi_offset v31, -272
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    std r16, 184(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r17, 192(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r18, 200(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r19, 208(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r20, 216(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r21, 224(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r22, 232(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stvx v24, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    std r23, 240(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r24, 248(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r25, 256(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r26, 264(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r27, 272(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r28, 280(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stvx v25, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 80
+; CHECK-NEXT:    std r29, 288(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r30, 296(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stfd f17, 312(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stfd f18, 320(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stfd f19, 328(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stfd f20, 336(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f20, f2
+; CHECK-NEXT:    stvx v26, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 96
+; CHECK-NEXT:    stfd f21, 344(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f21, f3
+; CHECK-NEXT:    stfd f22, 352(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f22, f4
+; CHECK-NEXT:    stfd f23, 360(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f23, f5
+; CHECK-NEXT:    stvx v27, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 112
+; CHECK-NEXT:    stfd f24, 368(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f24, f6
+; CHECK-NEXT:    stfd f25, 376(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f25, f7
+; CHECK-NEXT:    stfd f26, 384(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f26, f8
+; CHECK-NEXT:    stvx v28, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 128
+; CHECK-NEXT:    stfd f27, 392(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f27, f9
+; CHECK-NEXT:    stfd f28, 400(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f28, f10
+; CHECK-NEXT:    stfd f29, 408(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f29, f11
+; CHECK-NEXT:    stvx v29, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 144
+; CHECK-NEXT:    stfd f30, 416(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f30, f12
+; CHECK-NEXT:    stfd f31, 424(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f31, f13
+; CHECK-NEXT:    stvx v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 160
+; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f20
+; CHECK-NEXT:    mr r30, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f21
+; CHECK-NEXT:    mr r29, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f22
+; CHECK-NEXT:    mr r28, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f23
+; CHECK-NEXT:    mr r27, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f24
+; CHECK-NEXT:    mr r26, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f25
+; CHECK-NEXT:    mr r25, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f26
+; CHECK-NEXT:    mr r24, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f27
+; CHECK-NEXT:    mr r23, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f28
+; CHECK-NEXT:    mr r22, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f29
+; CHECK-NEXT:    mr r21, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f30
+; CHECK-NEXT:    mr r20, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f31
+; CHECK-NEXT:    mr r19, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 568(r1)
+; CHECK-NEXT:    mr r18, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 576(r1)
+; CHECK-NEXT:    mr r17, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 584(r1)
+; CHECK-NEXT:    mr r16, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r16, 48
+; CHECK-NEXT:    fmr f31, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r17, 48
+; CHECK-NEXT:    fmr f30, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r18, 48
+; CHECK-NEXT:    fmr f29, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r19, 48
+; CHECK-NEXT:    fmr f28, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r20, 48
+; CHECK-NEXT:    fmr f27, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r21, 48
+; CHECK-NEXT:    fmr f26, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r22, 48
+; CHECK-NEXT:    fmr f25, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r23, 48
+; CHECK-NEXT:    fmr f24, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r24, 48
+; CHECK-NEXT:    fmr f23, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r25, 48
+; CHECK-NEXT:    fmr f22, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r26, 48
+; CHECK-NEXT:    fmr f21, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r27, 48
+; CHECK-NEXT:    fmr f20, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r28, 48
+; CHECK-NEXT:    fmr f19, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r29, 48
+; CHECK-NEXT:    fmr f18, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r30, 48
+; CHECK-NEXT:    fmr f17, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f17
+; CHECK-NEXT:    mtvsrd v31, r3
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f18
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v31, vs0, v31
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f19
+; CHECK-NEXT:    mtvsrd v30, r3
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f20
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v30, vs0, v30
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f21
+; CHECK-NEXT:    mtvsrd v29, r3
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f22
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v29, vs0, v29
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f23
+; CHECK-NEXT:    mtvsrd v28, r3
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f24
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v28, vs0, v28
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f25
+; CHECK-NEXT:    mtvsrd v27, r3
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f26
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v27, vs0, v27
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f27
+; CHECK-NEXT:    mtvsrd v26, r3
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f28
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v26, vs0, v26
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f29
+; CHECK-NEXT:    mtvsrd v25, r3
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f30
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v25, vs0, v25
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f31
+; CHECK-NEXT:    mtvsrd v24, r3
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 160
+; CHECK-NEXT:    vmr v2, v31
+; CHECK-NEXT:    lfd f31, 424(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    vmr v3, v30
+; CHECK-NEXT:    vmr v4, v29
+; CHECK-NEXT:    lfd f30, 416(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f29, 408(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 144
+; CHECK-NEXT:    vmr v5, v28
+; CHECK-NEXT:    vmr v6, v27
+; CHECK-NEXT:    vmr v7, v26
+; CHECK-NEXT:    vmr v8, v25
+; CHECK-NEXT:    lfd f28, 400(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f27, 392(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 128
+; CHECK-NEXT:    lfd f26, 384(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f25, 376(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    xxmrghd v9, vs0, v24
+; CHECK-NEXT:    lfd f24, 368(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f23, 360(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f22, 352(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lvx v29, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 112
+; CHECK-NEXT:    lfd f21, 344(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r30, 296(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f20, 336(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f19, 328(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r29, 288(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r28, 280(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lvx v28, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 96
+; CHECK-NEXT:    lfd f18, 320(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r27, 272(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f17, 312(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r26, 264(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r25, 256(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r24, 248(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lvx v27, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 80
+; CHECK-NEXT:    ld r23, 240(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r22, 232(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r21, 224(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r20, 216(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r19, 208(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r18, 200(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lvx v26, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    ld r17, 192(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r16, 184(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lvx v25, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    lvx v24, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    addi r1, r1, 432
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: lrint_v16i64_v16f16:
+; FAST:       # %bb.0:
+; FAST-NEXT:    mflr r0
+; FAST-NEXT:    .cfi_def_cfa_offset 160
+; FAST-NEXT:    .cfi_offset lr, 16
+; FAST-NEXT:    .cfi_offset f16, -128
+; FAST-NEXT:    .cfi_offset f17, -120
+; FAST-NEXT:    .cfi_offset f18, -112
+; FAST-NEXT:    .cfi_offset f19, -104
+; FAST-NEXT:    .cfi_offset f20, -96
+; FAST-NEXT:    .cfi_offset f21, -88
+; FAST-NEXT:    .cfi_offset f22, -80
+; FAST-NEXT:    .cfi_offset f23, -72
+; FAST-NEXT:    .cfi_offset f24, -64
+; FAST-NEXT:    .cfi_offset f25, -56
+; FAST-NEXT:    .cfi_offset f26, -48
+; FAST-NEXT:    .cfi_offset f27, -40
+; FAST-NEXT:    .cfi_offset f28, -32
+; FAST-NEXT:    .cfi_offset f29, -24
+; FAST-NEXT:    .cfi_offset f30, -16
+; FAST-NEXT:    .cfi_offset f31, -8
+; FAST-NEXT:    stfd f16, -128(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f17, -120(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f18, -112(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f19, -104(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f20, -96(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f21, -88(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f22, -80(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f23, -72(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f24, -64(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f25, -56(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f26, -48(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f27, -40(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f28, -32(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f29, -24(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f30, -16(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f31, -8(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stdu r1, -160(r1)
+; FAST-NEXT:    fmr f26, f1
+; FAST-NEXT:    lfs f1, 312(r1)
+; FAST-NEXT:    std r0, 176(r1)
+; FAST-NEXT:    fmr f28, f13
+; FAST-NEXT:    fmr f27, f12
+; FAST-NEXT:    fmr f24, f11
+; FAST-NEXT:    fmr f21, f10
+; FAST-NEXT:    fmr f19, f9
+; FAST-NEXT:    fmr f18, f8
+; FAST-NEXT:    fmr f17, f7
+; FAST-NEXT:    fmr f16, f6
+; FAST-NEXT:    fmr f20, f5
+; FAST-NEXT:    fmr f22, f4
+; FAST-NEXT:    fmr f23, f3
+; FAST-NEXT:    fmr f25, f2
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f31, f1
+; FAST-NEXT:    lfs f1, 304(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f30, f1
+; FAST-NEXT:    lfs f1, 296(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f29, f1
+; FAST-NEXT:    fmr f1, f28
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f28, f1
+; FAST-NEXT:    fmr f1, f27
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f27, f1
+; FAST-NEXT:    fmr f1, f24
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f24, f1
+; FAST-NEXT:    fmr f1, f21
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f21, f1
+; FAST-NEXT:    fmr f1, f19
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f19, f1
+; FAST-NEXT:    fmr f1, f18
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f18, f1
+; FAST-NEXT:    fmr f1, f17
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f17, f1
+; FAST-NEXT:    fmr f1, f16
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f16, f1
+; FAST-NEXT:    fmr f1, f20
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f20, f1
+; FAST-NEXT:    fmr f1, f22
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f22, f1
+; FAST-NEXT:    fmr f1, f23
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f23, f1
+; FAST-NEXT:    fmr f1, f25
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f25, f1
+; FAST-NEXT:    fmr f1, f26
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fctid f0, f25
+; FAST-NEXT:    fctid f2, f23
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    fctid f3, f22
+; FAST-NEXT:    fctid f4, f20
+; FAST-NEXT:    fctid f5, f16
+; FAST-NEXT:    fctid f6, f17
+; FAST-NEXT:    fctid f7, f18
+; FAST-NEXT:    fctid f8, f19
+; FAST-NEXT:    fctid f9, f21
+; FAST-NEXT:    fctid f10, f24
+; FAST-NEXT:    fctid f1, f1
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    mffprd r3, f2
+; FAST-NEXT:    mtfprd f2, r3
+; FAST-NEXT:    mffprd r3, f3
+; FAST-NEXT:    mtfprd f3, r3
+; FAST-NEXT:    mffprd r3, f4
+; FAST-NEXT:    mtfprd f4, r3
+; FAST-NEXT:    mffprd r3, f5
+; FAST-NEXT:    mtfprd f5, r3
+; FAST-NEXT:    mffprd r3, f6
+; FAST-NEXT:    mtfprd f6, r3
+; FAST-NEXT:    mffprd r3, f7
+; FAST-NEXT:    mtfprd f7, r3
+; FAST-NEXT:    mffprd r3, f8
+; FAST-NEXT:    mtfprd f8, r3
+; FAST-NEXT:    mffprd r3, f9
+; FAST-NEXT:    mtfprd f9, r3
+; FAST-NEXT:    mffprd r3, f10
+; FAST-NEXT:    mtfprd f10, r3
+; FAST-NEXT:    mffprd r3, f1
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    xxmrghd v3, vs3, vs2
+; FAST-NEXT:    xxmrghd v4, vs5, vs4
+; FAST-NEXT:    xxmrghd v5, vs7, vs6
+; FAST-NEXT:    xxmrghd v6, vs9, vs8
+; FAST-NEXT:    xxmrghd v2, vs0, vs1
+; FAST-NEXT:    fctid f0, f27
+; FAST-NEXT:    fctid f1, f29
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    xxmrghd v7, vs0, vs10
+; FAST-NEXT:    fctid f0, f28
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    mffprd r3, f1
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    xxmrghd v8, vs1, vs0
+; FAST-NEXT:    fctid f0, f30
+; FAST-NEXT:    fctid f1, f31
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    mffprd r3, f1
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    xxmrghd v9, vs1, vs0
+; FAST-NEXT:    addi r1, r1, 160
+; FAST-NEXT:    ld r0, 16(r1)
+; FAST-NEXT:    lfd f31, -8(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f30, -16(r1) # 8-byte Folded Reload
+; FAST-NEXT:    mtlr r0
+; FAST-NEXT:    lfd f29, -24(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f28, -32(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f27, -40(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f26, -48(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f25, -56(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f24, -64(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f23, -72(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f22, -80(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f21, -88(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f20, -96(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f19, -104(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f18, -112(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f17, -120(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f16, -128(r1) # 8-byte Folded Reload
+; FAST-NEXT:    blr
+  %a = call <16 x i64> @llvm.lrint.v16i64.v16f16(<16 x half> %x)
+  ret <16 x i64> %a
+}
+declare <16 x i64> @llvm.lrint.v16i64.v16f16(<16 x half>)
+
+define <32 x i64> @lrint_v32i64_v32f16(<32 x half> %x) {
+; BE-LABEL: lrint_v32i64_v32f16:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -864(r1)
+; BE-NEXT:    std r0, 880(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 864
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    .cfi_offset r14, -288
+; BE-NEXT:    .cfi_offset r15, -280
+; BE-NEXT:    .cfi_offset r16, -272
+; BE-NEXT:    .cfi_offset r17, -264
+; BE-NEXT:    .cfi_offset r18, -256
+; BE-NEXT:    .cfi_offset r19, -248
+; BE-NEXT:    .cfi_offset r20, -240
+; BE-NEXT:    .cfi_offset r21, -232
+; BE-NEXT:    .cfi_offset r22, -224
+; BE-NEXT:    .cfi_offset r23, -216
+; BE-NEXT:    .cfi_offset r24, -208
+; BE-NEXT:    .cfi_offset r25, -200
+; BE-NEXT:    .cfi_offset r26, -192
+; BE-NEXT:    .cfi_offset r27, -184
+; BE-NEXT:    .cfi_offset r28, -176
+; BE-NEXT:    .cfi_offset r29, -168
+; BE-NEXT:    .cfi_offset r30, -160
+; BE-NEXT:    .cfi_offset r31, -152
+; BE-NEXT:    .cfi_offset f14, -144
+; BE-NEXT:    .cfi_offset f15, -136
+; BE-NEXT:    .cfi_offset f16, -128
+; BE-NEXT:    .cfi_offset f17, -120
+; BE-NEXT:    .cfi_offset f18, -112
+; BE-NEXT:    .cfi_offset f19, -104
+; BE-NEXT:    .cfi_offset f20, -96
+; BE-NEXT:    .cfi_offset f21, -88
+; BE-NEXT:    .cfi_offset f22, -80
+; BE-NEXT:    .cfi_offset f23, -72
+; BE-NEXT:    .cfi_offset f24, -64
+; BE-NEXT:    .cfi_offset f25, -56
+; BE-NEXT:    .cfi_offset f26, -48
+; BE-NEXT:    .cfi_offset f27, -40
+; BE-NEXT:    .cfi_offset f28, -32
+; BE-NEXT:    .cfi_offset f29, -24
+; BE-NEXT:    .cfi_offset f30, -16
+; BE-NEXT:    .cfi_offset f31, -8
+; BE-NEXT:    stfd f20, 768(r1) # 8-byte Folded Spill
+; BE-NEXT:    fmr f20, f1
+; BE-NEXT:    fmr f1, f2
+; BE-NEXT:    std r14, 576(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r15, 584(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r16, 592(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r17, 600(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r18, 608(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r19, 616(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r20, 624(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r21, 632(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r22, 640(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r23, 648(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r24, 656(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r25, 664(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r26, 672(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r27, 680(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r28, 688(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r29, 696(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r30, 704(r1) # 8-byte Folded Spill
+; BE-NEXT:    std r31, 712(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f14, 720(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f15, 728(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f16, 736(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f17, 744(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f18, 752(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f19, 760(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f21, 776(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f22, 784(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f23, 792(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f24, 800(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f25, 808(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f26, 816(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f27, 824(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f28, 832(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f29, 840(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f30, 848(r1) # 8-byte Folded Spill
+; BE-NEXT:    stfd f31, 856(r1) # 8-byte Folded Spill
+; BE-NEXT:    fmr f31, f13
+; BE-NEXT:    fmr f29, f12
+; BE-NEXT:    fmr f30, f11
+; BE-NEXT:    fmr f28, f10
+; BE-NEXT:    fmr f27, f9
+; BE-NEXT:    fmr f26, f8
+; BE-NEXT:    fmr f25, f7
+; BE-NEXT:    fmr f24, f6
+; BE-NEXT:    fmr f23, f5
+; BE-NEXT:    fmr f22, f4
+; BE-NEXT:    fmr f21, f3
+; BE-NEXT:    mr r30, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f20
+; BE-NEXT:    std r3, 304(r1) # 8-byte Folded Spill
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f22
+; BE-NEXT:    std r3, 296(r1) # 8-byte Folded Spill
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f21
+; BE-NEXT:    std r3, 280(r1) # 8-byte Folded Spill
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f24
+; BE-NEXT:    std r3, 264(r1) # 8-byte Folded Spill
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f23
+; BE-NEXT:    std r3, 248(r1) # 8-byte Folded Spill
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f26
+; BE-NEXT:    std r3, 232(r1) # 8-byte Folded Spill
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f25
+; BE-NEXT:    std r3, 216(r1) # 8-byte Folded Spill
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f28
+; BE-NEXT:    std r3, 200(r1) # 8-byte Folded Spill
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f27
+; BE-NEXT:    std r3, 184(r1) # 8-byte Folded Spill
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f29
+; BE-NEXT:    std r3, 168(r1) # 8-byte Folded Spill
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f30
+; BE-NEXT:    std r3, 152(r1) # 8-byte Folded Spill
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1028(r1)
+; BE-NEXT:    std r3, 136(r1) # 8-byte Folded Spill
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f31
+; BE-NEXT:    std r3, 120(r1) # 8-byte Folded Spill
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1044(r1)
+; BE-NEXT:    std r3, 112(r1) # 8-byte Folded Spill
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1036(r1)
+; BE-NEXT:    mr r15, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1060(r1)
+; BE-NEXT:    mr r14, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1052(r1)
+; BE-NEXT:    mr r31, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1076(r1)
+; BE-NEXT:    mr r29, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1068(r1)
+; BE-NEXT:    mr r28, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1092(r1)
+; BE-NEXT:    mr r27, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1084(r1)
+; BE-NEXT:    mr r26, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1108(r1)
+; BE-NEXT:    mr r25, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1100(r1)
+; BE-NEXT:    mr r24, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1124(r1)
+; BE-NEXT:    mr r23, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1116(r1)
+; BE-NEXT:    mr r22, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1140(r1)
+; BE-NEXT:    mr r21, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1132(r1)
+; BE-NEXT:    mr r20, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1156(r1)
+; BE-NEXT:    mr r19, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1148(r1)
+; BE-NEXT:    mr r18, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1172(r1)
+; BE-NEXT:    mr r17, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 1164(r1)
+; BE-NEXT:    mr r16, r3
+; BE-NEXT:    bl __gnu_f2h_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r16, 48
+; BE-NEXT:    stfs f1, 316(r1) # 4-byte Folded Spill
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r17, 48
+; BE-NEXT:    stfs f1, 312(r1) # 4-byte Folded Spill
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r18, 48
+; BE-NEXT:    stfs f1, 292(r1) # 4-byte Folded Spill
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r19, 48
+; BE-NEXT:    stfs f1, 276(r1) # 4-byte Folded Spill
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r20, 48
+; BE-NEXT:    stfs f1, 260(r1) # 4-byte Folded Spill
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r21, 48
+; BE-NEXT:    stfs f1, 244(r1) # 4-byte Folded Spill
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r22, 48
+; BE-NEXT:    stfs f1, 228(r1) # 4-byte Folded Spill
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r23, 48
+; BE-NEXT:    stfs f1, 212(r1) # 4-byte Folded Spill
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r24, 48
+; BE-NEXT:    stfs f1, 196(r1) # 4-byte Folded Spill
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r25, 48
+; BE-NEXT:    stfs f1, 180(r1) # 4-byte Folded Spill
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r26, 48
+; BE-NEXT:    stfs f1, 164(r1) # 4-byte Folded Spill
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r27, 48
+; BE-NEXT:    stfs f1, 148(r1) # 4-byte Folded Spill
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r28, 48
+; BE-NEXT:    stfs f1, 132(r1) # 4-byte Folded Spill
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r29, 48
+; BE-NEXT:    fmr f18, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r31, 48
+; BE-NEXT:    fmr f17, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r14, 48
+; BE-NEXT:    fmr f16, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    clrldi r3, r15, 48
+; BE-NEXT:    fmr f15, f1
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    ld r3, 112(r1) # 8-byte Folded Reload
+; BE-NEXT:    fmr f14, f1
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    ld r3, 120(r1) # 8-byte Folded Reload
+; BE-NEXT:    fmr f31, f1
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    ld r3, 136(r1) # 8-byte Folded Reload
+; BE-NEXT:    fmr f30, f1
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    ld r3, 152(r1) # 8-byte Folded Reload
+; BE-NEXT:    fmr f29, f1
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    ld r3, 168(r1) # 8-byte Folded Reload
+; BE-NEXT:    fmr f28, f1
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    ld r3, 184(r1) # 8-byte Folded Reload
+; BE-NEXT:    fmr f27, f1
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    ld r3, 200(r1) # 8-byte Folded Reload
+; BE-NEXT:    fmr f26, f1
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    ld r3, 216(r1) # 8-byte Folded Reload
+; BE-NEXT:    fmr f25, f1
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    ld r3, 232(r1) # 8-byte Folded Reload
+; BE-NEXT:    fmr f24, f1
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    ld r3, 248(r1) # 8-byte Folded Reload
+; BE-NEXT:    fmr f23, f1
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    ld r3, 264(r1) # 8-byte Folded Reload
+; BE-NEXT:    fmr f22, f1
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    ld r3, 280(r1) # 8-byte Folded Reload
+; BE-NEXT:    fmr f21, f1
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    ld r3, 296(r1) # 8-byte Folded Reload
+; BE-NEXT:    fmr f20, f1
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    ld r3, 304(r1) # 8-byte Folded Reload
+; BE-NEXT:    fmr f19, f1
+; BE-NEXT:    clrldi r3, r3, 48
+; BE-NEXT:    bl __gnu_h2f_ieee
+; BE-NEXT:    nop
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f19
+; BE-NEXT:    std r3, 328(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f20
+; BE-NEXT:    std r3, 320(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f21
+; BE-NEXT:    std r3, 344(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f22
+; BE-NEXT:    std r3, 336(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f23
+; BE-NEXT:    std r3, 360(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f24
+; BE-NEXT:    std r3, 352(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f25
+; BE-NEXT:    std r3, 376(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f26
+; BE-NEXT:    std r3, 368(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f27
+; BE-NEXT:    std r3, 392(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f28
+; BE-NEXT:    std r3, 384(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f29
+; BE-NEXT:    std r3, 408(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f30
+; BE-NEXT:    std r3, 400(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f31
+; BE-NEXT:    std r3, 424(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f14
+; BE-NEXT:    std r3, 416(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f15
+; BE-NEXT:    std r3, 440(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f16
+; BE-NEXT:    std r3, 432(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f17
+; BE-NEXT:    std r3, 456(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    fmr f1, f18
+; BE-NEXT:    std r3, 448(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 132(r1) # 4-byte Folded Reload
+; BE-NEXT:    std r3, 472(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 148(r1) # 4-byte Folded Reload
+; BE-NEXT:    std r3, 464(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 164(r1) # 4-byte Folded Reload
+; BE-NEXT:    std r3, 488(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 180(r1) # 4-byte Folded Reload
+; BE-NEXT:    std r3, 480(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 196(r1) # 4-byte Folded Reload
+; BE-NEXT:    std r3, 504(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 212(r1) # 4-byte Folded Reload
+; BE-NEXT:    std r3, 496(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 228(r1) # 4-byte Folded Reload
+; BE-NEXT:    std r3, 520(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 244(r1) # 4-byte Folded Reload
+; BE-NEXT:    std r3, 512(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 260(r1) # 4-byte Folded Reload
+; BE-NEXT:    std r3, 536(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 276(r1) # 4-byte Folded Reload
+; BE-NEXT:    std r3, 528(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 292(r1) # 4-byte Folded Reload
+; BE-NEXT:    std r3, 552(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 312(r1) # 4-byte Folded Reload
+; BE-NEXT:    std r3, 544(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 316(r1) # 4-byte Folded Reload
+; BE-NEXT:    std r3, 568(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 560(r1)
+; BE-NEXT:    addi r3, r1, 320
+; BE-NEXT:    lxvd2x vs0, 0, r3
+; BE-NEXT:    addi r3, r1, 336
+; BE-NEXT:    lxvd2x vs1, 0, r3
+; BE-NEXT:    addi r3, r1, 352
+; BE-NEXT:    lxvd2x vs2, 0, r3
+; BE-NEXT:    addi r3, r1, 368
+; BE-NEXT:    lxvd2x vs3, 0, r3
+; BE-NEXT:    addi r3, r1, 384
+; BE-NEXT:    lxvd2x vs4, 0, r3
+; BE-NEXT:    addi r3, r1, 400
+; BE-NEXT:    lxvd2x vs5, 0, r3
+; BE-NEXT:    addi r3, r1, 416
+; BE-NEXT:    lxvd2x vs6, 0, r3
+; BE-NEXT:    addi r3, r1, 432
+; BE-NEXT:    lxvd2x vs7, 0, r3
+; BE-NEXT:    addi r3, r1, 448
+; BE-NEXT:    lxvd2x vs8, 0, r3
+; BE-NEXT:    addi r3, r1, 464
+; BE-NEXT:    lxvd2x vs9, 0, r3
+; BE-NEXT:    addi r3, r1, 480
+; BE-NEXT:    lxvd2x vs10, 0, r3
+; BE-NEXT:    addi r3, r1, 496
+; BE-NEXT:    lxvd2x vs11, 0, r3
+; BE-NEXT:    addi r3, r1, 512
+; BE-NEXT:    lxvd2x vs12, 0, r3
+; BE-NEXT:    addi r3, r1, 528
+; BE-NEXT:    lxvd2x vs13, 0, r3
+; BE-NEXT:    addi r3, r1, 544
+; BE-NEXT:    lxvd2x v2, 0, r3
+; BE-NEXT:    addi r3, r1, 560
+; BE-NEXT:    lxvd2x v3, 0, r3
+; BE-NEXT:    li r3, 240
+; BE-NEXT:    stxvd2x v3, r30, r3
+; BE-NEXT:    li r3, 224
+; BE-NEXT:    stxvd2x v2, r30, r3
+; BE-NEXT:    li r3, 208
+; BE-NEXT:    stxvd2x vs13, r30, r3
+; BE-NEXT:    li r3, 192
+; BE-NEXT:    stxvd2x vs12, r30, r3
+; BE-NEXT:    li r3, 176
+; BE-NEXT:    stxvd2x vs11, r30, r3
+; BE-NEXT:    li r3, 160
+; BE-NEXT:    stxvd2x vs10, r30, r3
+; BE-NEXT:    li r3, 144
+; BE-NEXT:    stxvd2x vs9, r30, r3
+; BE-NEXT:    li r3, 128
+; BE-NEXT:    stxvd2x vs8, r30, r3
+; BE-NEXT:    li r3, 112
+; BE-NEXT:    stxvd2x vs7, r30, r3
+; BE-NEXT:    li r3, 96
+; BE-NEXT:    stxvd2x vs6, r30, r3
+; BE-NEXT:    li r3, 80
+; BE-NEXT:    stxvd2x vs5, r30, r3
+; BE-NEXT:    li r3, 64
+; BE-NEXT:    stxvd2x vs4, r30, r3
+; BE-NEXT:    li r3, 48
+; BE-NEXT:    stxvd2x vs3, r30, r3
+; BE-NEXT:    li r3, 32
+; BE-NEXT:    stxvd2x vs2, r30, r3
+; BE-NEXT:    li r3, 16
+; BE-NEXT:    stxvd2x vs1, r30, r3
+; BE-NEXT:    stxvd2x vs0, 0, r30
+; BE-NEXT:    lfd f31, 856(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f30, 848(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f29, 840(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f28, 832(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f27, 824(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f26, 816(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f25, 808(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f24, 800(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f23, 792(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f22, 784(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f21, 776(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f20, 768(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f19, 760(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f18, 752(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f17, 744(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f16, 736(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f15, 728(r1) # 8-byte Folded Reload
+; BE-NEXT:    lfd f14, 720(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r31, 712(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r30, 704(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r29, 696(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r28, 688(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r27, 680(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r26, 672(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r25, 664(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r24, 656(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r23, 648(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r22, 640(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r21, 632(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r20, 624(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r19, 616(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r18, 608(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r17, 600(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r16, 592(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r15, 584(r1) # 8-byte Folded Reload
+; BE-NEXT:    ld r14, 576(r1) # 8-byte Folded Reload
+; BE-NEXT:    addi r1, r1, 864
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: lrint_v32i64_v32f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -688(r1)
+; CHECK-NEXT:    std r0, 704(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 688
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    .cfi_offset r14, -288
+; CHECK-NEXT:    .cfi_offset r15, -280
+; CHECK-NEXT:    .cfi_offset r16, -272
+; CHECK-NEXT:    .cfi_offset r17, -264
+; CHECK-NEXT:    .cfi_offset r18, -256
+; CHECK-NEXT:    .cfi_offset r19, -248
+; CHECK-NEXT:    .cfi_offset r20, -240
+; CHECK-NEXT:    .cfi_offset r21, -232
+; CHECK-NEXT:    .cfi_offset r22, -224
+; CHECK-NEXT:    .cfi_offset r23, -216
+; CHECK-NEXT:    .cfi_offset r24, -208
+; CHECK-NEXT:    .cfi_offset r25, -200
+; CHECK-NEXT:    .cfi_offset r26, -192
+; CHECK-NEXT:    .cfi_offset r27, -184
+; CHECK-NEXT:    .cfi_offset r28, -176
+; CHECK-NEXT:    .cfi_offset r29, -168
+; CHECK-NEXT:    .cfi_offset r30, -160
+; CHECK-NEXT:    .cfi_offset r31, -152
+; CHECK-NEXT:    .cfi_offset f14, -144
+; CHECK-NEXT:    .cfi_offset f15, -136
+; CHECK-NEXT:    .cfi_offset f16, -128
+; CHECK-NEXT:    .cfi_offset f17, -120
+; CHECK-NEXT:    .cfi_offset f18, -112
+; CHECK-NEXT:    .cfi_offset f19, -104
+; CHECK-NEXT:    .cfi_offset f20, -96
+; CHECK-NEXT:    .cfi_offset f21, -88
+; CHECK-NEXT:    .cfi_offset f22, -80
+; CHECK-NEXT:    .cfi_offset f23, -72
+; CHECK-NEXT:    .cfi_offset f24, -64
+; CHECK-NEXT:    .cfi_offset f25, -56
+; CHECK-NEXT:    .cfi_offset f26, -48
+; CHECK-NEXT:    .cfi_offset f27, -40
+; CHECK-NEXT:    .cfi_offset f28, -32
+; CHECK-NEXT:    .cfi_offset f29, -24
+; CHECK-NEXT:    .cfi_offset f30, -16
+; CHECK-NEXT:    .cfi_offset f31, -8
+; CHECK-NEXT:    .cfi_offset v20, -480
+; CHECK-NEXT:    .cfi_offset v21, -464
+; CHECK-NEXT:    .cfi_offset v22, -448
+; CHECK-NEXT:    .cfi_offset v23, -432
+; CHECK-NEXT:    .cfi_offset v24, -416
+; CHECK-NEXT:    .cfi_offset v25, -400
+; CHECK-NEXT:    .cfi_offset v26, -384
+; CHECK-NEXT:    .cfi_offset v27, -368
+; CHECK-NEXT:    .cfi_offset v28, -352
+; CHECK-NEXT:    .cfi_offset v29, -336
+; CHECK-NEXT:    .cfi_offset v30, -320
+; CHECK-NEXT:    .cfi_offset v31, -304
+; CHECK-NEXT:    li r4, 208
+; CHECK-NEXT:    std r14, 400(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r15, 408(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r16, 416(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r17, 424(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r18, 432(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r19, 440(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r20, 448(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stvx v20, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    li r4, 224
+; CHECK-NEXT:    std r21, 456(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r22, 464(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r23, 472(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r24, 480(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r25, 488(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r26, 496(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stvx v21, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    li r4, 240
+; CHECK-NEXT:    std r27, 504(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r28, 512(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r29, 520(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r30, 528(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    mr r30, r3
+; CHECK-NEXT:    std r31, 536(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stvx v22, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    li r4, 256
+; CHECK-NEXT:    stfd f14, 544(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stfd f15, 552(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stfd f16, 560(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stfd f17, 568(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stfd f18, 576(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stfd f19, 584(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stvx v23, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    li r4, 272
+; CHECK-NEXT:    stfd f20, 592(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f20, f2
+; CHECK-NEXT:    stfd f21, 600(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f21, f3
+; CHECK-NEXT:    stfd f22, 608(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f22, f4
+; CHECK-NEXT:    stvx v24, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    li r4, 288
+; CHECK-NEXT:    stfd f23, 616(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f23, f5
+; CHECK-NEXT:    stfd f24, 624(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f24, f6
+; CHECK-NEXT:    stfd f25, 632(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f25, f7
+; CHECK-NEXT:    stvx v25, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    li r4, 304
+; CHECK-NEXT:    stfd f26, 640(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f26, f8
+; CHECK-NEXT:    stfd f27, 648(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f27, f9
+; CHECK-NEXT:    stfd f28, 656(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f28, f10
+; CHECK-NEXT:    stvx v26, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    li r4, 320
+; CHECK-NEXT:    stfd f29, 664(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f29, f11
+; CHECK-NEXT:    stfd f30, 672(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f30, f12
+; CHECK-NEXT:    stfd f31, 680(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    fmr f31, f13
+; CHECK-NEXT:    stvx v27, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    li r4, 336
+; CHECK-NEXT:    stvx v28, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    li r4, 352
+; CHECK-NEXT:    stvx v29, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    li r4, 368
+; CHECK-NEXT:    stvx v30, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    li r4, 384
+; CHECK-NEXT:    stvx v31, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f20
+; CHECK-NEXT:    std r3, 176(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f21
+; CHECK-NEXT:    std r3, 160(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f22
+; CHECK-NEXT:    std r3, 144(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f23
+; CHECK-NEXT:    std r3, 128(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f24
+; CHECK-NEXT:    std r3, 120(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f25
+; CHECK-NEXT:    std r3, 112(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f26
+; CHECK-NEXT:    std r3, 104(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f27
+; CHECK-NEXT:    std r3, 96(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f28
+; CHECK-NEXT:    std r3, 88(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f29
+; CHECK-NEXT:    std r3, 80(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f30
+; CHECK-NEXT:    std r3, 72(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f31
+; CHECK-NEXT:    std r3, 64(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 832(r1)
+; CHECK-NEXT:    std r3, 56(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 840(r1)
+; CHECK-NEXT:    std r3, 48(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 848(r1)
+; CHECK-NEXT:    mr r15, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 856(r1)
+; CHECK-NEXT:    mr r14, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 864(r1)
+; CHECK-NEXT:    mr r31, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 872(r1)
+; CHECK-NEXT:    mr r29, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 880(r1)
+; CHECK-NEXT:    mr r28, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 888(r1)
+; CHECK-NEXT:    mr r27, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 896(r1)
+; CHECK-NEXT:    mr r26, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 904(r1)
+; CHECK-NEXT:    mr r25, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 912(r1)
+; CHECK-NEXT:    mr r24, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 920(r1)
+; CHECK-NEXT:    mr r23, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 928(r1)
+; CHECK-NEXT:    mr r22, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 936(r1)
+; CHECK-NEXT:    mr r21, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 944(r1)
+; CHECK-NEXT:    mr r20, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 952(r1)
+; CHECK-NEXT:    mr r19, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 960(r1)
+; CHECK-NEXT:    mr r18, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 968(r1)
+; CHECK-NEXT:    mr r17, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lfs f1, 976(r1)
+; CHECK-NEXT:    mr r16, r3
+; CHECK-NEXT:    bl __gnu_f2h_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    li r3, 204
+; CHECK-NEXT:    stxsspx f1, r1, r3 # 4-byte Folded Spill
+; CHECK-NEXT:    clrldi r3, r16, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    li r3, 200
+; CHECK-NEXT:    stxsspx f1, r1, r3 # 4-byte Folded Spill
+; CHECK-NEXT:    clrldi r3, r17, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r18, 48
+; CHECK-NEXT:    fmr f29, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r19, 48
+; CHECK-NEXT:    fmr f28, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r20, 48
+; CHECK-NEXT:    fmr f27, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r21, 48
+; CHECK-NEXT:    fmr f26, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r22, 48
+; CHECK-NEXT:    fmr f25, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r23, 48
+; CHECK-NEXT:    fmr f24, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r24, 48
+; CHECK-NEXT:    fmr f23, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r25, 48
+; CHECK-NEXT:    fmr f22, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r26, 48
+; CHECK-NEXT:    fmr f21, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r27, 48
+; CHECK-NEXT:    fmr f20, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r28, 48
+; CHECK-NEXT:    fmr f19, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r29, 48
+; CHECK-NEXT:    fmr f18, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r31, 48
+; CHECK-NEXT:    fmr f17, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r14, 48
+; CHECK-NEXT:    fmr f16, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    clrldi r3, r15, 48
+; CHECK-NEXT:    fmr f15, f1
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    ld r3, 48(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    fmr f14, f1
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    ld r3, 56(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    fmr f30, f1
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    ld r3, 64(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    xxlor v30, f1, f1
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    ld r3, 72(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    xxlor v29, f1, f1
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    ld r3, 80(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    xxlor v28, f1, f1
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    ld r3, 88(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    xxlor v27, f1, f1
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    ld r3, 96(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    xxlor v26, f1, f1
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    ld r3, 104(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    xxlor v25, f1, f1
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    ld r3, 112(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    xxlor v24, f1, f1
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    ld r3, 120(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    xxlor v23, f1, f1
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    ld r3, 128(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    xxlor v22, f1, f1
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    ld r3, 144(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    xxlor v21, f1, f1
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    ld r3, 160(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    xxlor v20, f1, f1
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    ld r3, 176(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    fmr f31, f1
+; CHECK-NEXT:    clrldi r3, r3, 48
+; CHECK-NEXT:    bl __gnu_h2f_ieee
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f31
+; CHECK-NEXT:    mtvsrd v31, r3
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 176
+; CHECK-NEXT:    xxlor f1, v20, v20
+; CHECK-NEXT:    xxmrghd vs0, vs0, v31
+; CHECK-NEXT:    stxvd2x vs0, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxlor f1, v21, v21
+; CHECK-NEXT:    mtvsrd v31, r3
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 160
+; CHECK-NEXT:    xxlor f1, v22, v22
+; CHECK-NEXT:    xxmrghd vs0, vs0, v31
+; CHECK-NEXT:    stxvd2x vs0, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxlor f1, v23, v23
+; CHECK-NEXT:    mtvsrd v31, r3
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 144
+; CHECK-NEXT:    xxlor f1, v24, v24
+; CHECK-NEXT:    xxmrghd vs0, vs0, v31
+; CHECK-NEXT:    stxvd2x vs0, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxlor f1, v25, v25
+; CHECK-NEXT:    mtvsrd v31, r3
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 128
+; CHECK-NEXT:    xxlor f1, v26, v26
+; CHECK-NEXT:    xxmrghd vs0, vs0, v31
+; CHECK-NEXT:    stxvd2x vs0, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxlor f1, v27, v27
+; CHECK-NEXT:    mtvsrd v31, r3
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxlor f1, v28, v28
+; CHECK-NEXT:    xxmrghd v27, vs0, v31
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxlor f1, v29, v29
+; CHECK-NEXT:    mtvsrd v31, r3
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxlor f1, v30, v30
+; CHECK-NEXT:    xxmrghd v29, vs0, v31
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f30
+; CHECK-NEXT:    mtvsrd v31, r3
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f14
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v31, vs0, v31
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f15
+; CHECK-NEXT:    mtvsrd v30, r3
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f16
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v30, vs0, v30
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f17
+; CHECK-NEXT:    mtvsrd v28, r3
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f18
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v28, vs0, v28
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f19
+; CHECK-NEXT:    mtvsrd v26, r3
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f20
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v26, vs0, v26
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f21
+; CHECK-NEXT:    mtvsrd v24, r3
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f22
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v24, vs0, v24
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f23
+; CHECK-NEXT:    mtvsrd v22, r3
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f24
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v22, vs0, v22
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f25
+; CHECK-NEXT:    mtvsrd v20, r3
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f26
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v20, vs0, v20
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f27
+; CHECK-NEXT:    mtvsrd v21, r3
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f28
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v21, vs0, v21
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    fmr f1, f29
+; CHECK-NEXT:    mtvsrd v23, r3
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 200
+; CHECK-NEXT:    lxsspx f1, r1, r3 # 4-byte Folded Reload
+; CHECK-NEXT:    xxmrghd v23, vs0, v23
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtvsrd v25, r3
+; CHECK-NEXT:    li r3, 204
+; CHECK-NEXT:    lxsspx f1, r1, r3 # 4-byte Folded Reload
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 240
+; CHECK-NEXT:    xxswapd vs1, v23
+; CHECK-NEXT:    li r4, 128
+; CHECK-NEXT:    xxswapd vs2, v21
+; CHECK-NEXT:    xxswapd vs3, v31
+; CHECK-NEXT:    xxmrghd v2, vs0, v25
+; CHECK-NEXT:    xxswapd vs0, v2
+; CHECK-NEXT:    stxvd2x vs0, r30, r3
+; CHECK-NEXT:    li r3, 224
+; CHECK-NEXT:    stxvd2x vs1, r30, r3
+; CHECK-NEXT:    li r3, 208
+; CHECK-NEXT:    stxvd2x vs2, r30, r3
+; CHECK-NEXT:    li r3, 192
+; CHECK-NEXT:    xxswapd vs0, v20
+; CHECK-NEXT:    stxvd2x vs0, r30, r3
+; CHECK-NEXT:    li r3, 176
+; CHECK-NEXT:    xxswapd vs1, v22
+; CHECK-NEXT:    stxvd2x vs1, r30, r3
+; CHECK-NEXT:    li r3, 160
+; CHECK-NEXT:    xxswapd vs2, v28
+; CHECK-NEXT:    xxswapd vs0, v24
+; CHECK-NEXT:    stxvd2x vs0, r30, r3
+; CHECK-NEXT:    li r3, 144
+; CHECK-NEXT:    xxswapd vs1, v26
+; CHECK-NEXT:    stxvd2x vs1, r30, r3
+; CHECK-NEXT:    li r3, 128
+; CHECK-NEXT:    stxvd2x vs2, r30, r3
+; CHECK-NEXT:    li r3, 112
+; CHECK-NEXT:    xxswapd vs0, v30
+; CHECK-NEXT:    stxvd2x vs0, r30, r3
+; CHECK-NEXT:    li r3, 96
+; CHECK-NEXT:    stxvd2x vs3, r30, r3
+; CHECK-NEXT:    li r3, 80
+; CHECK-NEXT:    lxvd2x vs2, r1, r4 # 16-byte Folded Reload
+; CHECK-NEXT:    li r4, 144
+; CHECK-NEXT:    xxswapd vs1, v29
+; CHECK-NEXT:    stxvd2x vs1, r30, r3
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    lxvd2x vs1, r1, r4 # 16-byte Folded Reload
+; CHECK-NEXT:    li r4, 160
+; CHECK-NEXT:    lxvd2x vs3, r1, r4 # 16-byte Folded Reload
+; CHECK-NEXT:    li r4, 176
+; CHECK-NEXT:    lxvd2x vs4, r1, r4 # 16-byte Folded Reload
+; CHECK-NEXT:    xxswapd vs0, v27
+; CHECK-NEXT:    stxvd2x vs0, r30, r3
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    xxswapd vs2, vs2
+; CHECK-NEXT:    stxvd2x vs2, r30, r3
+; CHECK-NEXT:    li r3, 32
+; CHECK-NEXT:    xxswapd vs1, vs1
+; CHECK-NEXT:    stxvd2x vs1, r30, r3
+; CHECK-NEXT:    li r3, 16
+; CHECK-NEXT:    xxswapd vs3, vs3
+; CHECK-NEXT:    stxvd2x vs3, r30, r3
+; CHECK-NEXT:    li r3, 384
+; CHECK-NEXT:    xxswapd vs4, vs4
+; CHECK-NEXT:    stxvd2x vs4, 0, r30
+; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 368
+; CHECK-NEXT:    lfd f31, 680(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f30, 672(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f29, 664(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f28, 656(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f27, 648(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f26, 640(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f25, 632(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f24, 624(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f23, 616(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f22, 608(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f21, 600(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f20, 592(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f19, 584(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f18, 576(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f17, 568(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f16, 560(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 352
+; CHECK-NEXT:    lfd f15, 552(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd f14, 544(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r31, 536(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r30, 528(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r29, 520(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r28, 512(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lvx v29, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 336
+; CHECK-NEXT:    ld r27, 504(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r26, 496(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r25, 488(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r24, 480(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r23, 472(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r22, 464(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lvx v28, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 320
+; CHECK-NEXT:    ld r21, 456(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r20, 448(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r19, 440(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r18, 432(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r17, 424(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r16, 416(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lvx v27, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 304
+; CHECK-NEXT:    ld r15, 408(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld r14, 400(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    lvx v26, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 288
+; CHECK-NEXT:    lvx v25, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 272
+; CHECK-NEXT:    lvx v24, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 256
+; CHECK-NEXT:    lvx v23, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 240
+; CHECK-NEXT:    lvx v22, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 224
+; CHECK-NEXT:    lvx v21, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 208
+; CHECK-NEXT:    lvx v20, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    addi r1, r1, 688
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: lrint_v32i64_v32f16:
+; FAST:       # %bb.0:
+; FAST-NEXT:    mflr r0
+; FAST-NEXT:    stdu r1, -480(r1)
+; FAST-NEXT:    std r0, 496(r1)
+; FAST-NEXT:    .cfi_def_cfa_offset 480
+; FAST-NEXT:    .cfi_offset lr, 16
+; FAST-NEXT:    .cfi_offset r30, -160
+; FAST-NEXT:    .cfi_offset f14, -144
+; FAST-NEXT:    .cfi_offset f15, -136
+; FAST-NEXT:    .cfi_offset f16, -128
+; FAST-NEXT:    .cfi_offset f17, -120
+; FAST-NEXT:    .cfi_offset f18, -112
+; FAST-NEXT:    .cfi_offset f19, -104
+; FAST-NEXT:    .cfi_offset f20, -96
+; FAST-NEXT:    .cfi_offset f21, -88
+; FAST-NEXT:    .cfi_offset f22, -80
+; FAST-NEXT:    .cfi_offset f23, -72
+; FAST-NEXT:    .cfi_offset f24, -64
+; FAST-NEXT:    .cfi_offset f25, -56
+; FAST-NEXT:    .cfi_offset f26, -48
+; FAST-NEXT:    .cfi_offset f27, -40
+; FAST-NEXT:    .cfi_offset f28, -32
+; FAST-NEXT:    .cfi_offset f29, -24
+; FAST-NEXT:    .cfi_offset f30, -16
+; FAST-NEXT:    .cfi_offset f31, -8
+; FAST-NEXT:    .cfi_offset v20, -352
+; FAST-NEXT:    .cfi_offset v21, -336
+; FAST-NEXT:    .cfi_offset v22, -320
+; FAST-NEXT:    .cfi_offset v23, -304
+; FAST-NEXT:    .cfi_offset v24, -288
+; FAST-NEXT:    .cfi_offset v25, -272
+; FAST-NEXT:    .cfi_offset v26, -256
+; FAST-NEXT:    .cfi_offset v27, -240
+; FAST-NEXT:    .cfi_offset v28, -224
+; FAST-NEXT:    .cfi_offset v29, -208
+; FAST-NEXT:    .cfi_offset v30, -192
+; FAST-NEXT:    .cfi_offset v31, -176
+; FAST-NEXT:    li r4, 128
+; FAST-NEXT:    std r30, 320(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f14, 336(r1) # 8-byte Folded Spill
+; FAST-NEXT:    fmr f14, f5
+; FAST-NEXT:    stfd f15, 344(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f16, 352(r1) # 8-byte Folded Spill
+; FAST-NEXT:    fmr f16, f4
+; FAST-NEXT:    mr r30, r3
+; FAST-NEXT:    stvx v20, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    li r4, 144
+; FAST-NEXT:    stfd f17, 360(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f18, 368(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f19, 376(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f20, 384(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f21, 392(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f22, 400(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stvx v21, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    li r4, 160
+; FAST-NEXT:    stfd f23, 408(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f24, 416(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f25, 424(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f26, 432(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f27, 440(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f28, 448(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stvx v22, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    li r4, 176
+; FAST-NEXT:    xxlor v22, f3, f3
+; FAST-NEXT:    stfd f29, 456(r1) # 8-byte Folded Spill
+; FAST-NEXT:    fmr f29, f9
+; FAST-NEXT:    stfd f30, 464(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stfd f31, 472(r1) # 8-byte Folded Spill
+; FAST-NEXT:    stvx v23, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    li r4, 192
+; FAST-NEXT:    xxlor v23, f2, f2
+; FAST-NEXT:    stvx v24, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    li r4, 208
+; FAST-NEXT:    stvx v25, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    li r4, 224
+; FAST-NEXT:    xxlor v25, f13, f13
+; FAST-NEXT:    stvx v26, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    li r4, 240
+; FAST-NEXT:    xxlor v26, f12, f12
+; FAST-NEXT:    stvx v27, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    li r4, 256
+; FAST-NEXT:    xxlor v27, f11, f11
+; FAST-NEXT:    stvx v28, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    li r4, 272
+; FAST-NEXT:    xxlor v28, f10, f10
+; FAST-NEXT:    stvx v29, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    li r4, 288
+; FAST-NEXT:    xxlor v29, f8, f8
+; FAST-NEXT:    stvx v30, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    li r4, 304
+; FAST-NEXT:    xxlor v30, f7, f7
+; FAST-NEXT:    stvx v31, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    li r4, 44
+; FAST-NEXT:    xxlor v31, f6, f6
+; FAST-NEXT:    stxsspx f1, r1, r4 # 4-byte Folded Spill
+; FAST-NEXT:    lfs f1, 768(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    li r3, 120
+; FAST-NEXT:    stxsdx f1, r1, r3 # 8-byte Folded Spill
+; FAST-NEXT:    lfs f1, 760(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    li r3, 112
+; FAST-NEXT:    stxsdx f1, r1, r3 # 8-byte Folded Spill
+; FAST-NEXT:    lfs f1, 752(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    li r3, 104
+; FAST-NEXT:    stxsdx f1, r1, r3 # 8-byte Folded Spill
+; FAST-NEXT:    lfs f1, 744(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    li r3, 96
+; FAST-NEXT:    stxsdx f1, r1, r3 # 8-byte Folded Spill
+; FAST-NEXT:    lfs f1, 736(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    li r3, 88
+; FAST-NEXT:    stxsdx f1, r1, r3 # 8-byte Folded Spill
+; FAST-NEXT:    lfs f1, 728(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    li r3, 80
+; FAST-NEXT:    stxsdx f1, r1, r3 # 8-byte Folded Spill
+; FAST-NEXT:    lfs f1, 720(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    li r3, 72
+; FAST-NEXT:    stxsdx f1, r1, r3 # 8-byte Folded Spill
+; FAST-NEXT:    lfs f1, 712(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    li r3, 64
+; FAST-NEXT:    stxsdx f1, r1, r3 # 8-byte Folded Spill
+; FAST-NEXT:    lfs f1, 704(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    li r3, 56
+; FAST-NEXT:    stxsdx f1, r1, r3 # 8-byte Folded Spill
+; FAST-NEXT:    lfs f1, 696(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    li r3, 48
+; FAST-NEXT:    stxsdx f1, r1, r3 # 8-byte Folded Spill
+; FAST-NEXT:    lfs f1, 688(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    xxlor v21, f1, f1
+; FAST-NEXT:    lfs f1, 680(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    xxlor v20, f1, f1
+; FAST-NEXT:    lfs f1, 672(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    xxlor v24, f1, f1
+; FAST-NEXT:    lfs f1, 664(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f31, f1
+; FAST-NEXT:    lfs f1, 656(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f30, f1
+; FAST-NEXT:    lfs f1, 648(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f28, f1
+; FAST-NEXT:    lfs f1, 640(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f27, f1
+; FAST-NEXT:    lfs f1, 632(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f26, f1
+; FAST-NEXT:    lfs f1, 624(r1)
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f25, f1
+; FAST-NEXT:    xxlor f1, v25, v25
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f24, f1
+; FAST-NEXT:    xxlor f1, v26, v26
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f23, f1
+; FAST-NEXT:    xxlor f1, v27, v27
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f22, f1
+; FAST-NEXT:    xxlor f1, v28, v28
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f21, f1
+; FAST-NEXT:    fmr f1, f29
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f20, f1
+; FAST-NEXT:    xxlor f1, v29, v29
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f19, f1
+; FAST-NEXT:    xxlor f1, v30, v30
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f18, f1
+; FAST-NEXT:    xxlor f1, v31, v31
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f29, f1
+; FAST-NEXT:    fmr f1, f14
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f14, f1
+; FAST-NEXT:    fmr f1, f16
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f16, f1
+; FAST-NEXT:    xxlor f1, v22, v22
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fmr f17, f1
+; FAST-NEXT:    xxlor f1, v23, v23
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    li r3, 44
+; FAST-NEXT:    fmr f15, f1
+; FAST-NEXT:    lxsspx f1, r1, r3 # 4-byte Folded Reload
+; FAST-NEXT:    bl __gnu_f2h_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    clrldi r3, r3, 48
+; FAST-NEXT:    bl __gnu_h2f_ieee
+; FAST-NEXT:    nop
+; FAST-NEXT:    fctid f3, f15
+; FAST-NEXT:    fctid f4, f17
+; FAST-NEXT:    mffprd r3, f3
+; FAST-NEXT:    fctid f5, f16
+; FAST-NEXT:    fctid f6, f14
+; FAST-NEXT:    fctid f7, f18
+; FAST-NEXT:    fctid f8, f19
+; FAST-NEXT:    fctid f13, f1
+; FAST-NEXT:    fctid f9, f20
+; FAST-NEXT:    fctid f10, f22
+; FAST-NEXT:    fctid f11, f24
+; FAST-NEXT:    fctid f12, f25
+; FAST-NEXT:    fctid f2, f23
+; FAST-NEXT:    fctid f0, f21
+; FAST-NEXT:    mtvsrd v2, r3
+; FAST-NEXT:    mffprd r3, f4
+; FAST-NEXT:    mtvsrd v3, r3
+; FAST-NEXT:    mffprd r3, f5
+; FAST-NEXT:    mtfprd f5, r3
+; FAST-NEXT:    mffprd r3, f6
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    mffprd r3, f7
+; FAST-NEXT:    mtfprd f6, r3
+; FAST-NEXT:    mffprd r3, f8
+; FAST-NEXT:    mtfprd f7, r3
+; FAST-NEXT:    mffprd r3, f9
+; FAST-NEXT:    mtfprd f3, r3
+; FAST-NEXT:    mffprd r3, f10
+; FAST-NEXT:    mtfprd f4, r3
+; FAST-NEXT:    mffprd r3, f11
+; FAST-NEXT:    fctid f11, f31
+; FAST-NEXT:    lfd f31, 56(r1) # 8-byte Folded Reload
+; FAST-NEXT:    mtfprd f8, r3
+; FAST-NEXT:    mffprd r3, f12
+; FAST-NEXT:    xxlor f12, v24, v24
+; FAST-NEXT:    fctid f31, f31
+; FAST-NEXT:    fctid f12, f12
+; FAST-NEXT:    mtfprd f9, r3
+; FAST-NEXT:    mffprd r3, f13
+; FAST-NEXT:    lfd f13, 48(r1) # 8-byte Folded Reload
+; FAST-NEXT:    mtfprd f10, r3
+; FAST-NEXT:    fctid f13, f13
+; FAST-NEXT:    xxmrghd v3, vs5, v3
+; FAST-NEXT:    fctid f5, f26
+; FAST-NEXT:    mffprd r3, f5
+; FAST-NEXT:    mtfprd f5, r3
+; FAST-NEXT:    xxmrghd v4, vs7, vs6
+; FAST-NEXT:    fctid f6, f27
+; FAST-NEXT:    fctid f7, f28
+; FAST-NEXT:    mffprd r3, f6
+; FAST-NEXT:    lfd f28, 96(r1) # 8-byte Folded Reload
+; FAST-NEXT:    fctid f28, f28
+; FAST-NEXT:    mtfprd f6, r3
+; FAST-NEXT:    mffprd r3, f7
+; FAST-NEXT:    mtfprd f7, r3
+; FAST-NEXT:    xxmrghd v2, v2, vs10
+; FAST-NEXT:    fctid f10, f30
+; FAST-NEXT:    mffprd r3, f10
+; FAST-NEXT:    lfd f30, 80(r1) # 8-byte Folded Reload
+; FAST-NEXT:    fctid f30, f30
+; FAST-NEXT:    mtfprd f10, r3
+; FAST-NEXT:    mffprd r3, f11
+; FAST-NEXT:    mtfprd f11, r3
+; FAST-NEXT:    mffprd r3, f12
+; FAST-NEXT:    mtfprd f12, r3
+; FAST-NEXT:    xxmrghd v5, vs12, vs11
+; FAST-NEXT:    xxlor f11, v20, v20
+; FAST-NEXT:    xxlor f12, v21, v21
+; FAST-NEXT:    fctid f11, f11
+; FAST-NEXT:    fctid f12, f12
+; FAST-NEXT:    mffprd r3, f11
+; FAST-NEXT:    mtfprd f11, r3
+; FAST-NEXT:    mffprd r3, f12
+; FAST-NEXT:    mtfprd f12, r3
+; FAST-NEXT:    mffprd r3, f13
+; FAST-NEXT:    mtfprd f13, r3
+; FAST-NEXT:    mffprd r3, f31
+; FAST-NEXT:    lfd f31, 64(r1) # 8-byte Folded Reload
+; FAST-NEXT:    fctid f31, f31
+; FAST-NEXT:    mtvsrd v0, r3
+; FAST-NEXT:    mffprd r3, f31
+; FAST-NEXT:    lfd f31, 72(r1) # 8-byte Folded Reload
+; FAST-NEXT:    mtvsrd v1, r3
+; FAST-NEXT:    mffprd r3, f30
+; FAST-NEXT:    lfd f30, 88(r1) # 8-byte Folded Reload
+; FAST-NEXT:    fctid f31, f31
+; FAST-NEXT:    mtvsrd v6, r3
+; FAST-NEXT:    mffprd r3, f28
+; FAST-NEXT:    lfd f28, 104(r1) # 8-byte Folded Reload
+; FAST-NEXT:    fctid f30, f30
+; FAST-NEXT:    fctid f28, f28
+; FAST-NEXT:    mtvsrd v7, r3
+; FAST-NEXT:    mffprd r3, f28
+; FAST-NEXT:    lfd f28, 112(r1) # 8-byte Folded Reload
+; FAST-NEXT:    fctid f28, f28
+; FAST-NEXT:    mtvsrd v8, r3
+; FAST-NEXT:    mffprd r3, f28
+; FAST-NEXT:    lfd f28, 120(r1) # 8-byte Folded Reload
+; FAST-NEXT:    fctid f28, f28
+; FAST-NEXT:    xxmrghd v10, vs12, vs11
+; FAST-NEXT:    xxmrghd v0, v0, vs13
+; FAST-NEXT:    xxswapd vs12, v0
+; FAST-NEXT:    xxmrghd v0, vs9, vs8
+; FAST-NEXT:    xxmrghd v7, v8, v7
+; FAST-NEXT:    mtvsrd v8, r3
+; FAST-NEXT:    mffprd r3, f28
+; FAST-NEXT:    mtvsrd v9, r3
+; FAST-NEXT:    mffprd r3, f30
+; FAST-NEXT:    xxswapd v7, v7
+; FAST-NEXT:    xxmrghd v8, v9, v8
+; FAST-NEXT:    mtvsrd v9, r3
+; FAST-NEXT:    mffprd r3, f31
+; FAST-NEXT:    xxswapd v8, v8
+; FAST-NEXT:    xxmrghd v6, v9, v6
+; FAST-NEXT:    mtvsrd v9, r3
+; FAST-NEXT:    li r3, 240
+; FAST-NEXT:    stxvd2x v8, r30, r3
+; FAST-NEXT:    li r3, 224
+; FAST-NEXT:    stxvd2x v7, r30, r3
+; FAST-NEXT:    li r3, 208
+; FAST-NEXT:    xxswapd vs11, v6
+; FAST-NEXT:    xxmrghd v6, vs10, vs7
+; FAST-NEXT:    stxvd2x vs11, r30, r3
+; FAST-NEXT:    li r3, 192
+; FAST-NEXT:    xxmrghd v1, v9, v1
+; FAST-NEXT:    xxswapd vs11, v1
+; FAST-NEXT:    xxmrghd v1, vs6, vs5
+; FAST-NEXT:    xxswapd vs5, v10
+; FAST-NEXT:    xxswapd vs6, v5
+; FAST-NEXT:    stxvd2x vs11, r30, r3
+; FAST-NEXT:    li r3, 176
+; FAST-NEXT:    stxvd2x vs12, r30, r3
+; FAST-NEXT:    li r3, 160
+; FAST-NEXT:    stxvd2x vs5, r30, r3
+; FAST-NEXT:    li r3, 144
+; FAST-NEXT:    stxvd2x vs6, r30, r3
+; FAST-NEXT:    mffprd r3, f2
+; FAST-NEXT:    mtfprd f7, r3
+; FAST-NEXT:    li r3, 128
+; FAST-NEXT:    xxswapd vs5, v6
+; FAST-NEXT:    stxvd2x vs5, r30, r3
+; FAST-NEXT:    li r3, 112
+; FAST-NEXT:    xxswapd vs2, v1
+; FAST-NEXT:    xxswapd vs6, v0
+; FAST-NEXT:    stxvd2x vs2, r30, r3
+; FAST-NEXT:    li r3, 96
+; FAST-NEXT:    fctid f2, f29
+; FAST-NEXT:    stxvd2x vs6, r30, r3
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    mffprd r3, f2
+; FAST-NEXT:    mtfprd f2, r3
+; FAST-NEXT:    li r3, 80
+; FAST-NEXT:    xxmrghd v5, vs7, vs4
+; FAST-NEXT:    xxswapd vs4, v2
+; FAST-NEXT:    xxmrghd v0, vs0, vs3
+; FAST-NEXT:    xxswapd vs0, v5
+; FAST-NEXT:    xxswapd vs3, v3
+; FAST-NEXT:    stxvd2x vs0, r30, r3
+; FAST-NEXT:    li r3, 64
+; FAST-NEXT:    xxswapd vs0, v0
+; FAST-NEXT:    stxvd2x vs0, r30, r3
+; FAST-NEXT:    li r3, 48
+; FAST-NEXT:    xxmrghd v5, vs2, vs1
+; FAST-NEXT:    xxswapd vs1, v4
+; FAST-NEXT:    stxvd2x vs1, r30, r3
+; FAST-NEXT:    li r3, 32
+; FAST-NEXT:    xxswapd vs2, v5
+; FAST-NEXT:    stxvd2x vs2, r30, r3
+; FAST-NEXT:    li r3, 16
+; FAST-NEXT:    stxvd2x vs3, r30, r3
+; FAST-NEXT:    li r3, 304
+; FAST-NEXT:    stxvd2x vs4, 0, r30
+; FAST-NEXT:    lfd f31, 472(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f30, 464(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f29, 456(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f28, 448(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f27, 440(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f26, 432(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f25, 424(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f24, 416(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f23, 408(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f22, 400(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f21, 392(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f20, 384(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f19, 376(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f18, 368(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f17, 360(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f16, 352(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f15, 344(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lfd f14, 336(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 288
+; FAST-NEXT:    ld r30, 320(r1) # 8-byte Folded Reload
+; FAST-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 272
+; FAST-NEXT:    lvx v29, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 256
+; FAST-NEXT:    lvx v28, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 240
+; FAST-NEXT:    lvx v27, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 224
+; FAST-NEXT:    lvx v26, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 208
+; FAST-NEXT:    lvx v25, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 192
+; FAST-NEXT:    lvx v24, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 176
+; FAST-NEXT:    lvx v23, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 160
+; FAST-NEXT:    lvx v22, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 144
+; FAST-NEXT:    lvx v21, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 128
+; FAST-NEXT:    lvx v20, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    addi r1, r1, 480
+; FAST-NEXT:    ld r0, 16(r1)
+; FAST-NEXT:    mtlr r0
+; FAST-NEXT:    blr
+  %a = call <32 x i64> @llvm.lrint.v32i64.v32f16(<32 x half> %x)
+  ret <32 x i64> %a
+}
+declare <32 x i64> @llvm.lrint.v32i64.v32f16(<32 x half>)
+
+define <1 x i64> @lrint_v1f32(<1 x float> %x) {
+; BE-LABEL: lrint_v1f32:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -112(r1)
+; BE-NEXT:    std r0, 128(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 112
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    addi r1, r1, 112
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: lrint_v1f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -32(r1)
+; CHECK-NEXT:    std r0, 48(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addi r1, r1, 32
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: lrint_v1f32:
+; FAST:       # %bb.0:
+; FAST-NEXT:    fctid f0, f1
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    blr
+  %a = call <1 x i64> @llvm.lrint.v1i64.v1f32(<1 x float> %x)
+  ret <1 x i64> %a
+}
+declare <1 x i64> @llvm.lrint.v1i64.v1f32(<1 x float>)
+
+define <2 x i64> @lrint_v2f32(<2 x float> %x) {
+; BE-LABEL: lrint_v2f32:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -144(r1)
+; BE-NEXT:    std r0, 160(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 144
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    addi r3, r1, 112
+; BE-NEXT:    stxvw4x v2, 0, r3
+; BE-NEXT:    lfs f1, 116(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 112(r1)
+; BE-NEXT:    std r3, 136(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 128(r1)
+; BE-NEXT:    addi r3, r1, 128
+; BE-NEXT:    lxvd2x v2, 0, r3
+; BE-NEXT:    addi r1, r1, 144
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: lrint_v2f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -64(r1)
+; CHECK-NEXT:    std r0, 80(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    .cfi_offset v31, -16
+; CHECK-NEXT:    xxsldwi vs0, v2, v2, 3
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    vmr v31, v2
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxswapd vs0, v31
+; CHECK-NEXT:    mtvsrd v31, r3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    xxmrghd v2, vs0, v31
+; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    addi r1, r1, 64
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: lrint_v2f32:
+; FAST:       # %bb.0:
+; FAST-NEXT:    xxsldwi vs0, v2, v2, 3
+; FAST-NEXT:    xxswapd vs1, v2
+; FAST-NEXT:    xscvspdpn f0, vs0
+; FAST-NEXT:    xscvspdpn f1, vs1
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    fctid f1, f1
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    mffprd r3, f1
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    xxmrghd v2, vs1, vs0
+; FAST-NEXT:    blr
+  %a = call <2 x i64> @llvm.lrint.v2i64.v2f32(<2 x float> %x)
+  ret <2 x i64> %a
+}
+declare <2 x i64> @llvm.lrint.v2i64.v2f32(<2 x float>)
+
+define <4 x i64> @lrint_v4f32(<4 x float> %x) {
+; BE-LABEL: lrint_v4f32:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -160(r1)
+; BE-NEXT:    std r0, 176(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 160
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    addi r3, r1, 112
+; BE-NEXT:    stxvw4x v2, 0, r3
+; BE-NEXT:    lfs f1, 116(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 112(r1)
+; BE-NEXT:    std r3, 136(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 124(r1)
+; BE-NEXT:    std r3, 128(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 120(r1)
+; BE-NEXT:    std r3, 152(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 144(r1)
+; BE-NEXT:    addi r3, r1, 128
+; BE-NEXT:    lxvd2x v2, 0, r3
+; BE-NEXT:    addi r3, r1, 144
+; BE-NEXT:    lxvd2x v3, 0, r3
+; BE-NEXT:    addi r1, r1, 160
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: lrint_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -80(r1)
+; CHECK-NEXT:    std r0, 96(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 80
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    .cfi_offset v30, -32
+; CHECK-NEXT:    .cfi_offset v31, -16
+; CHECK-NEXT:    xxsldwi vs0, v2, v2, 3
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    stvx v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    vmr v31, v2
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxswapd vs0, v31
+; CHECK-NEXT:    mtvsrd v30, r3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xscvspdpn f1, v31
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v30, vs0, v30
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxsldwi vs0, v31, v31, 1
+; CHECK-NEXT:    mtvsrd v31, r3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    vmr v2, v30
+; CHECK-NEXT:    xxmrghd v3, v31, vs0
+; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    addi r1, r1, 80
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: lrint_v4f32:
+; FAST:       # %bb.0:
+; FAST-NEXT:    xxsldwi vs0, v2, v2, 3
+; FAST-NEXT:    xxswapd vs1, v2
+; FAST-NEXT:    xscvspdpn f0, vs0
+; FAST-NEXT:    xxsldwi vs2, v2, v2, 1
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    xscvspdpn f0, vs1
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    xxmrghd v4, vs0, vs1
+; FAST-NEXT:    xscvspdpn f0, v2
+; FAST-NEXT:    vmr v2, v4
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    xscvspdpn f0, vs2
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    xxmrghd v3, vs1, vs0
+; FAST-NEXT:    blr
+  %a = call <4 x i64> @llvm.lrint.v4i64.v4f32(<4 x float> %x)
+  ret <4 x i64> %a
+}
+declare <4 x i64> @llvm.lrint.v4i64.v4f32(<4 x float>)
+
+define <8 x i64> @lrint_v8f32(<8 x float> %x) {
+; BE-LABEL: lrint_v8f32:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -208(r1)
+; BE-NEXT:    std r0, 224(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 208
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    addi r3, r1, 112
+; BE-NEXT:    stxvw4x v2, 0, r3
+; BE-NEXT:    addi r3, r1, 128
+; BE-NEXT:    stxvw4x v3, 0, r3
+; BE-NEXT:    lfs f1, 116(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 112(r1)
+; BE-NEXT:    std r3, 152(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 124(r1)
+; BE-NEXT:    std r3, 144(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 120(r1)
+; BE-NEXT:    std r3, 168(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 132(r1)
+; BE-NEXT:    std r3, 160(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 128(r1)
+; BE-NEXT:    std r3, 184(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 140(r1)
+; BE-NEXT:    std r3, 176(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 136(r1)
+; BE-NEXT:    std r3, 200(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 192(r1)
+; BE-NEXT:    addi r3, r1, 144
+; BE-NEXT:    lxvd2x v2, 0, r3
+; BE-NEXT:    addi r3, r1, 160
+; BE-NEXT:    lxvd2x v3, 0, r3
+; BE-NEXT:    addi r3, r1, 176
+; BE-NEXT:    lxvd2x v4, 0, r3
+; BE-NEXT:    addi r3, r1, 192
+; BE-NEXT:    lxvd2x v5, 0, r3
+; BE-NEXT:    addi r1, r1, 208
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: lrint_v8f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -112(r1)
+; CHECK-NEXT:    std r0, 128(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 112
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    .cfi_offset v28, -64
+; CHECK-NEXT:    .cfi_offset v29, -48
+; CHECK-NEXT:    .cfi_offset v30, -32
+; CHECK-NEXT:    .cfi_offset v31, -16
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    xxsldwi vs0, v2, v2, 3
+; CHECK-NEXT:    stvx v28, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    stvx v29, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 80
+; CHECK-NEXT:    stvx v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 96
+; CHECK-NEXT:    vmr v30, v2
+; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    vmr v31, v3
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxswapd vs0, v30
+; CHECK-NEXT:    mtvsrd v29, r3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xscvspdpn f1, v30
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v29, vs0, v29
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxsldwi vs0, v30, v30, 1
+; CHECK-NEXT:    mtvsrd v30, r3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v30, v30, vs0
+; CHECK-NEXT:    xxsldwi vs0, v31, v31, 3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxswapd vs0, v31
+; CHECK-NEXT:    mtvsrd v28, r3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xscvspdpn f1, v31
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v28, vs0, v28
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxsldwi vs0, v31, v31, 1
+; CHECK-NEXT:    mtvsrd v31, r3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 96
+; CHECK-NEXT:    vmr v3, v30
+; CHECK-NEXT:    vmr v2, v29
+; CHECK-NEXT:    vmr v4, v28
+; CHECK-NEXT:    xxmrghd v5, v31, vs0
+; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 80
+; CHECK-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    lvx v29, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    lvx v28, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    addi r1, r1, 112
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: lrint_v8f32:
+; FAST:       # %bb.0:
+; FAST-NEXT:    xxsldwi vs0, v2, v2, 3
+; FAST-NEXT:    xxswapd vs1, v2
+; FAST-NEXT:    xscvspdpn f0, vs0
+; FAST-NEXT:    xxsldwi vs2, v2, v2, 1
+; FAST-NEXT:    xxsldwi vs3, v3, v3, 3
+; FAST-NEXT:    xxswapd vs4, v3
+; FAST-NEXT:    xxsldwi vs5, v3, v3, 1
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    xscvspdpn f0, vs1
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    xxmrghd v0, vs0, vs1
+; FAST-NEXT:    xscvspdpn f0, v2
+; FAST-NEXT:    vmr v2, v0
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    xscvspdpn f0, vs2
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    xxmrghd v1, vs1, vs0
+; FAST-NEXT:    xscvspdpn f0, vs3
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    xscvspdpn f0, vs4
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    xxmrghd v4, vs0, vs1
+; FAST-NEXT:    xscvspdpn f0, v3
+; FAST-NEXT:    vmr v3, v1
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    xscvspdpn f0, vs5
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    xxmrghd v5, vs1, vs0
+; FAST-NEXT:    blr
+  %a = call <8 x i64> @llvm.lrint.v8i64.v8f32(<8 x float> %x)
+  ret <8 x i64> %a
+}
+declare <8 x i64> @llvm.lrint.v8i64.v8f32(<8 x float>)
+
+define <16 x i64> @lrint_v16i64_v16f32(<16 x float> %x) {
+; BE-LABEL: lrint_v16i64_v16f32:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -304(r1)
+; BE-NEXT:    std r0, 320(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 304
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    addi r3, r1, 112
+; BE-NEXT:    stxvw4x v2, 0, r3
+; BE-NEXT:    addi r3, r1, 128
+; BE-NEXT:    stxvw4x v3, 0, r3
+; BE-NEXT:    addi r3, r1, 144
+; BE-NEXT:    stxvw4x v4, 0, r3
+; BE-NEXT:    addi r3, r1, 160
+; BE-NEXT:    stxvw4x v5, 0, r3
+; BE-NEXT:    lfs f1, 116(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 112(r1)
+; BE-NEXT:    std r3, 184(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 124(r1)
+; BE-NEXT:    std r3, 176(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 120(r1)
+; BE-NEXT:    std r3, 200(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 132(r1)
+; BE-NEXT:    std r3, 192(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 128(r1)
+; BE-NEXT:    std r3, 216(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 140(r1)
+; BE-NEXT:    std r3, 208(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 136(r1)
+; BE-NEXT:    std r3, 232(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 148(r1)
+; BE-NEXT:    std r3, 224(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 144(r1)
+; BE-NEXT:    std r3, 248(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 156(r1)
+; BE-NEXT:    std r3, 240(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 152(r1)
+; BE-NEXT:    std r3, 264(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 164(r1)
+; BE-NEXT:    std r3, 256(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 160(r1)
+; BE-NEXT:    std r3, 280(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 172(r1)
+; BE-NEXT:    std r3, 272(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    lfs f1, 168(r1)
+; BE-NEXT:    std r3, 296(r1)
+; BE-NEXT:    bl lrintf
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 288(r1)
+; BE-NEXT:    addi r3, r1, 176
+; BE-NEXT:    lxvd2x v2, 0, r3
+; BE-NEXT:    addi r3, r1, 192
+; BE-NEXT:    lxvd2x v3, 0, r3
+; BE-NEXT:    addi r3, r1, 208
+; BE-NEXT:    lxvd2x v4, 0, r3
+; BE-NEXT:    addi r3, r1, 224
+; BE-NEXT:    lxvd2x v5, 0, r3
+; BE-NEXT:    addi r3, r1, 240
+; BE-NEXT:    lxvd2x v6, 0, r3
+; BE-NEXT:    addi r3, r1, 256
+; BE-NEXT:    lxvd2x v7, 0, r3
+; BE-NEXT:    addi r3, r1, 272
+; BE-NEXT:    lxvd2x v8, 0, r3
+; BE-NEXT:    addi r3, r1, 288
+; BE-NEXT:    lxvd2x v9, 0, r3
+; BE-NEXT:    addi r1, r1, 304
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: lrint_v16i64_v16f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -176(r1)
+; CHECK-NEXT:    std r0, 192(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 176
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    .cfi_offset v24, -128
+; CHECK-NEXT:    .cfi_offset v25, -112
+; CHECK-NEXT:    .cfi_offset v26, -96
+; CHECK-NEXT:    .cfi_offset v27, -80
+; CHECK-NEXT:    .cfi_offset v28, -64
+; CHECK-NEXT:    .cfi_offset v29, -48
+; CHECK-NEXT:    .cfi_offset v30, -32
+; CHECK-NEXT:    .cfi_offset v31, -16
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    xxsldwi vs0, v2, v2, 3
+; CHECK-NEXT:    stvx v24, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    stvx v25, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 80
+; CHECK-NEXT:    stvx v26, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 96
+; CHECK-NEXT:    vmr v26, v3
+; CHECK-NEXT:    stvx v27, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 112
+; CHECK-NEXT:    stvx v28, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 128
+; CHECK-NEXT:    vmr v28, v4
+; CHECK-NEXT:    stvx v29, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 144
+; CHECK-NEXT:    vmr v29, v2
+; CHECK-NEXT:    stvx v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 160
+; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    vmr v31, v5
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxswapd vs0, v29
+; CHECK-NEXT:    mtvsrd v30, r3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xscvspdpn f1, v29
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v30, vs0, v30
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxsldwi vs0, v29, v29, 1
+; CHECK-NEXT:    mtvsrd v29, r3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v29, v29, vs0
+; CHECK-NEXT:    xxsldwi vs0, v26, v26, 3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxswapd vs0, v26
+; CHECK-NEXT:    mtvsrd v27, r3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xscvspdpn f1, v26
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v27, vs0, v27
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxsldwi vs0, v26, v26, 1
+; CHECK-NEXT:    mtvsrd v26, r3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v26, v26, vs0
+; CHECK-NEXT:    xxsldwi vs0, v28, v28, 3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxswapd vs0, v28
+; CHECK-NEXT:    mtvsrd v25, r3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xscvspdpn f1, v28
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v25, vs0, v25
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxsldwi vs0, v28, v28, 1
+; CHECK-NEXT:    mtvsrd v28, r3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v28, v28, vs0
+; CHECK-NEXT:    xxsldwi vs0, v31, v31, 3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxswapd vs0, v31
+; CHECK-NEXT:    mtvsrd v24, r3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xscvspdpn f1, v31
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxmrghd v24, vs0, v24
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxsldwi vs0, v31, v31, 1
+; CHECK-NEXT:    mtvsrd v31, r3
+; CHECK-NEXT:    xscvspdpn f1, vs0
+; CHECK-NEXT:    bl lrintf
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 160
+; CHECK-NEXT:    vmr v2, v30
+; CHECK-NEXT:    vmr v3, v29
+; CHECK-NEXT:    vmr v7, v28
+; CHECK-NEXT:    vmr v4, v27
+; CHECK-NEXT:    vmr v5, v26
+; CHECK-NEXT:    vmr v6, v25
+; CHECK-NEXT:    vmr v8, v24
+; CHECK-NEXT:    xxmrghd v9, v31, vs0
+; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 144
+; CHECK-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 128
+; CHECK-NEXT:    lvx v29, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 112
+; CHECK-NEXT:    lvx v28, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 96
+; CHECK-NEXT:    lvx v27, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 80
+; CHECK-NEXT:    lvx v26, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    lvx v25, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    lvx v24, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    addi r1, r1, 176
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: lrint_v16i64_v16f32:
+; FAST:       # %bb.0:
+; FAST-NEXT:    xxsldwi vs0, v2, v2, 3
+; FAST-NEXT:    xxswapd vs1, v2
+; FAST-NEXT:    xscvspdpn f0, vs0
+; FAST-NEXT:    xxsldwi vs2, v2, v2, 1
+; FAST-NEXT:    xxsldwi vs3, v3, v3, 3
+; FAST-NEXT:    xxswapd vs4, v3
+; FAST-NEXT:    xxsldwi vs5, v3, v3, 1
+; FAST-NEXT:    xxsldwi vs6, v4, v4, 3
+; FAST-NEXT:    xxswapd vs7, v4
+; FAST-NEXT:    xxsldwi vs8, v4, v4, 1
+; FAST-NEXT:    xxsldwi vs9, v5, v5, 3
+; FAST-NEXT:    xxswapd vs10, v5
+; FAST-NEXT:    xxsldwi vs11, v5, v5, 1
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    xscvspdpn f0, vs1
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    xxmrghd v0, vs0, vs1
+; FAST-NEXT:    xscvspdpn f0, v2
+; FAST-NEXT:    vmr v2, v0
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    xscvspdpn f0, vs2
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    xxmrghd v1, vs1, vs0
+; FAST-NEXT:    xscvspdpn f0, vs3
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    xscvspdpn f0, vs4
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    xxmrghd v10, vs0, vs1
+; FAST-NEXT:    xscvspdpn f0, v3
+; FAST-NEXT:    vmr v3, v1
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    xscvspdpn f0, vs5
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    xxmrghd v11, vs1, vs0
+; FAST-NEXT:    xscvspdpn f0, vs6
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    xscvspdpn f0, vs7
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    xxmrghd v6, vs0, vs1
+; FAST-NEXT:    xscvspdpn f0, v4
+; FAST-NEXT:    xscvspdpn f1, vs8
+; FAST-NEXT:    vmr v4, v10
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    fctid f1, f1
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    mffprd r3, f1
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    xxmrghd v7, vs0, vs1
+; FAST-NEXT:    xscvspdpn f0, vs9
+; FAST-NEXT:    xscvspdpn f1, vs10
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    fctid f1, f1
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    mffprd r3, f1
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    xxmrghd v8, vs1, vs0
+; FAST-NEXT:    xscvspdpn f0, v5
+; FAST-NEXT:    xscvspdpn f1, vs11
+; FAST-NEXT:    vmr v5, v11
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    fctid f1, f1
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    mffprd r3, f1
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    xxmrghd v9, vs0, vs1
+; FAST-NEXT:    blr
+  %a = call <16 x i64> @llvm.lrint.v16i64.v16f32(<16 x float> %x)
+  ret <16 x i64> %a
+}
+declare <16 x i64> @llvm.lrint.v16i64.v16f32(<16 x float>)
+
+define <1 x i64> @lrint_v1f64(<1 x double> %x) {
+; BE-LABEL: lrint_v1f64:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -112(r1)
+; BE-NEXT:    std r0, 128(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 112
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    bl lrint
+; BE-NEXT:    nop
+; BE-NEXT:    addi r1, r1, 112
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: lrint_v1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -32(r1)
+; CHECK-NEXT:    std r0, 48(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    bl lrint
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addi r1, r1, 32
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: lrint_v1f64:
+; FAST:       # %bb.0:
+; FAST-NEXT:    fctid f0, f1
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    blr
+  %a = call <1 x i64> @llvm.lrint.v1i64.v1f64(<1 x double> %x)
+  ret <1 x i64> %a
+}
+declare <1 x i64> @llvm.lrint.v1i64.v1f64(<1 x double>)
+
+define <2 x i64> @lrint_v2f64(<2 x double> %x) {
+; BE-LABEL: lrint_v2f64:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -160(r1)
+; BE-NEXT:    std r0, 176(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 160
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    .cfi_offset v31, -16
+; BE-NEXT:    li r3, 144
+; BE-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    vmr v31, v2
+; BE-NEXT:    xxlor f1, v31, v31
+; BE-NEXT:    bl lrint
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 128(r1)
+; BE-NEXT:    xxswapd vs1, v31
+; BE-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; BE-NEXT:    bl lrint
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 136(r1)
+; BE-NEXT:    addi r3, r1, 128
+; BE-NEXT:    lxvd2x v2, 0, r3
+; BE-NEXT:    li r3, 144
+; BE-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    addi r1, r1, 160
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: lrint_v2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -64(r1)
+; CHECK-NEXT:    std r0, 80(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    .cfi_offset v31, -16
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    vmr v31, v2
+; CHECK-NEXT:    xxlor f1, v31, v31
+; CHECK-NEXT:    bl lrint
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxswapd vs1, v31
+; CHECK-NEXT:    mtvsrd v31, r3
+; CHECK-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; CHECK-NEXT:    bl lrint
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    xxmrghd v2, v31, vs0
+; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    addi r1, r1, 64
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: lrint_v2f64:
+; FAST:       # %bb.0:
+; FAST-NEXT:    xxlor f1, v2, v2
+; FAST-NEXT:    xxswapd vs0, v2
+; FAST-NEXT:    fctid f1, f1
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r3, f1
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    mffprd r3, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    xxmrghd v2, vs1, vs0
+; FAST-NEXT:    blr
+  %a = call <2 x i64> @llvm.lrint.v2i64.v2f64(<2 x double> %x)
+  ret <2 x i64> %a
+}
+declare <2 x i64> @llvm.lrint.v2i64.v2f64(<2 x double>)
+
+define <4 x i64> @lrint_v4f64(<4 x double> %x) {
+; BE-LABEL: lrint_v4f64:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -192(r1)
+; BE-NEXT:    std r0, 208(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 192
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    .cfi_offset v30, -32
+; BE-NEXT:    .cfi_offset v31, -16
+; BE-NEXT:    li r3, 160
+; BE-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    vmr v30, v2
+; BE-NEXT:    li r3, 176
+; BE-NEXT:    xxlor f1, v30, v30
+; BE-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    vmr v31, v3
+; BE-NEXT:    bl lrint
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 128(r1)
+; BE-NEXT:    xxswapd vs1, v30
+; BE-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; BE-NEXT:    bl lrint
+; BE-NEXT:    nop
+; BE-NEXT:    xxlor f1, v31, v31
+; BE-NEXT:    std r3, 136(r1)
+; BE-NEXT:    bl lrint
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 144(r1)
+; BE-NEXT:    xxswapd vs1, v31
+; BE-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; BE-NEXT:    bl lrint
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 152(r1)
+; BE-NEXT:    addi r3, r1, 128
+; BE-NEXT:    lxvd2x v2, 0, r3
+; BE-NEXT:    addi r3, r1, 144
+; BE-NEXT:    lxvd2x v3, 0, r3
+; BE-NEXT:    li r3, 176
+; BE-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    li r3, 160
+; BE-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    addi r1, r1, 192
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: lrint_v4f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -80(r1)
+; CHECK-NEXT:    std r0, 96(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 80
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    .cfi_offset v30, -32
+; CHECK-NEXT:    .cfi_offset v31, -16
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    stvx v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    vmr v30, v2
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    xxlor f1, v30, v30
+; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    vmr v31, v3
+; CHECK-NEXT:    bl lrint
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxswapd vs1, v30
+; CHECK-NEXT:    mtvsrd v30, r3
+; CHECK-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; CHECK-NEXT:    bl lrint
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxlor f1, v31, v31
+; CHECK-NEXT:    xxmrghd v30, v30, vs0
+; CHECK-NEXT:    bl lrint
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxswapd vs1, v31
+; CHECK-NEXT:    mtvsrd v31, r3
+; CHECK-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; CHECK-NEXT:    bl lrint
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    vmr v2, v30
+; CHECK-NEXT:    xxmrghd v3, v31, vs0
+; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    addi r1, r1, 80
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: lrint_v4f64:
+; FAST:       # %bb.0:
+; FAST-NEXT:    xxswapd vs0, v2
+; FAST-NEXT:    xxlor f2, v2, v2
+; FAST-NEXT:    xxswapd vs1, v3
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    fctid f2, f2
+; FAST-NEXT:    fctid f1, f1
+; FAST-NEXT:    mffprd r4, f0
+; FAST-NEXT:    xxlor f0, v3, v3
+; FAST-NEXT:    mffprd r3, f2
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mtfprd f2, r4
+; FAST-NEXT:    mffprd r5, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    mffprd r3, f1
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    xxmrghd v2, vs0, vs2
+; FAST-NEXT:    mtfprd f0, r5
+; FAST-NEXT:    xxmrghd v3, vs0, vs1
+; FAST-NEXT:    blr
+  %a = call <4 x i64> @llvm.lrint.v4i64.v4f64(<4 x double> %x)
+  ret <4 x i64> %a
+}
+declare <4 x i64> @llvm.lrint.v4i64.v4f64(<4 x double>)
+
+define <8 x i64> @lrint_v8f64(<8 x double> %x) {
+; BE-LABEL: lrint_v8f64:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -256(r1)
+; BE-NEXT:    std r0, 272(r1)
+; BE-NEXT:    .cfi_def_cfa_offset 256
+; BE-NEXT:    .cfi_offset lr, 16
+; BE-NEXT:    .cfi_offset v28, -64
+; BE-NEXT:    .cfi_offset v29, -48
+; BE-NEXT:    .cfi_offset v30, -32
+; BE-NEXT:    .cfi_offset v31, -16
+; BE-NEXT:    li r3, 192
+; BE-NEXT:    stxvd2x v28, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    li r3, 208
+; BE-NEXT:    vmr v28, v2
+; BE-NEXT:    xxlor f1, v28, v28
+; BE-NEXT:    stxvd2x v29, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    li r3, 224
+; BE-NEXT:    vmr v29, v3
+; BE-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    li r3, 240
+; BE-NEXT:    vmr v30, v4
+; BE-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    vmr v31, v5
+; BE-NEXT:    bl lrint
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 128(r1)
+; BE-NEXT:    xxswapd vs1, v28
+; BE-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; BE-NEXT:    bl lrint
+; BE-NEXT:    nop
+; BE-NEXT:    xxlor f1, v29, v29
+; BE-NEXT:    std r3, 136(r1)
+; BE-NEXT:    bl lrint
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 144(r1)
+; BE-NEXT:    xxswapd vs1, v29
+; BE-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; BE-NEXT:    bl lrint
+; BE-NEXT:    nop
+; BE-NEXT:    xxlor f1, v30, v30
+; BE-NEXT:    std r3, 152(r1)
+; BE-NEXT:    bl lrint
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 160(r1)
+; BE-NEXT:    xxswapd vs1, v30
+; BE-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; BE-NEXT:    bl lrint
+; BE-NEXT:    nop
+; BE-NEXT:    xxlor f1, v31, v31
+; BE-NEXT:    std r3, 168(r1)
+; BE-NEXT:    bl lrint
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 176(r1)
+; BE-NEXT:    xxswapd vs1, v31
+; BE-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; BE-NEXT:    bl lrint
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 184(r1)
+; BE-NEXT:    addi r3, r1, 128
+; BE-NEXT:    lxvd2x v2, 0, r3
+; BE-NEXT:    addi r3, r1, 144
+; BE-NEXT:    lxvd2x v3, 0, r3
+; BE-NEXT:    addi r3, r1, 160
+; BE-NEXT:    lxvd2x v4, 0, r3
+; BE-NEXT:    addi r3, r1, 176
+; BE-NEXT:    lxvd2x v5, 0, r3
+; BE-NEXT:    li r3, 240
+; BE-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    li r3, 224
+; BE-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    li r3, 208
+; BE-NEXT:    lxvd2x v29, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    li r3, 192
+; BE-NEXT:    lxvd2x v28, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    addi r1, r1, 256
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: lrint_v8f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -112(r1)
+; CHECK-NEXT:    std r0, 128(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 112
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    .cfi_offset v28, -64
+; CHECK-NEXT:    .cfi_offset v29, -48
+; CHECK-NEXT:    .cfi_offset v30, -32
+; CHECK-NEXT:    .cfi_offset v31, -16
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    stvx v28, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    vmr v28, v2
+; CHECK-NEXT:    stvx v29, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 80
+; CHECK-NEXT:    xxlor f1, v28, v28
+; CHECK-NEXT:    vmr v29, v3
+; CHECK-NEXT:    stvx v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 96
+; CHECK-NEXT:    vmr v30, v4
+; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    vmr v31, v5
+; CHECK-NEXT:    bl lrint
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxswapd vs1, v28
+; CHECK-NEXT:    mtvsrd v28, r3
+; CHECK-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; CHECK-NEXT:    bl lrint
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxlor f1, v29, v29
+; CHECK-NEXT:    xxmrghd v28, v28, vs0
+; CHECK-NEXT:    bl lrint
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxswapd vs1, v29
+; CHECK-NEXT:    mtvsrd v29, r3
+; CHECK-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; CHECK-NEXT:    bl lrint
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxlor f1, v30, v30
+; CHECK-NEXT:    xxmrghd v29, v29, vs0
+; CHECK-NEXT:    bl lrint
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxswapd vs1, v30
+; CHECK-NEXT:    mtvsrd v30, r3
+; CHECK-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; CHECK-NEXT:    bl lrint
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    xxlor f1, v31, v31
+; CHECK-NEXT:    xxmrghd v30, v30, vs0
+; CHECK-NEXT:    bl lrint
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    xxswapd vs1, v31
+; CHECK-NEXT:    mtvsrd v31, r3
+; CHECK-NEXT:    # kill: def $f1 killed $f1 killed $vsl1
+; CHECK-NEXT:    bl lrint
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 96
+; CHECK-NEXT:    vmr v4, v30
+; CHECK-NEXT:    vmr v3, v29
+; CHECK-NEXT:    vmr v2, v28
+; CHECK-NEXT:    xxmrghd v5, v31, vs0
+; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 80
+; CHECK-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    lvx v29, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    lvx v28, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    addi r1, r1, 112
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: lrint_v8f64:
+; FAST:       # %bb.0:
+; FAST-NEXT:    xxswapd vs0, v2
+; FAST-NEXT:    xxswapd vs1, v3
+; FAST-NEXT:    xxlor f4, v2, v2
+; FAST-NEXT:    xxswapd vs2, v4
+; FAST-NEXT:    xxswapd vs3, v5
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    fctid f4, f4
+; FAST-NEXT:    mffprd r4, f0
+; FAST-NEXT:    xxlor f0, v3, v3
+; FAST-NEXT:    mffprd r3, f4
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mffprd r5, f0
+; FAST-NEXT:    fctid f0, f1
+; FAST-NEXT:    mtfprd f1, r4
+; FAST-NEXT:    mffprd r6, f0
+; FAST-NEXT:    xxlor f0, v4, v4
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mtfprd f4, r6
+; FAST-NEXT:    mffprd r7, f0
+; FAST-NEXT:    fctid f0, f2
+; FAST-NEXT:    mtfprd f2, r5
+; FAST-NEXT:    mtfprd f5, r7
+; FAST-NEXT:    mffprd r8, f0
+; FAST-NEXT:    xxlor f0, v5, v5
+; FAST-NEXT:    fctid f0, f0
+; FAST-NEXT:    mtfprd f6, r8
+; FAST-NEXT:    mffprd r9, f0
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    xxmrghd v3, vs2, vs4
+; FAST-NEXT:    xxmrghd v4, vs5, vs6
+; FAST-NEXT:    xxmrghd v2, vs0, vs1
+; FAST-NEXT:    fctid f1, f3
+; FAST-NEXT:    mtfprd f0, r9
+; FAST-NEXT:    mffprd r3, f1
+; FAST-NEXT:    mtfprd f1, r3
+; FAST-NEXT:    xxmrghd v5, vs0, vs1
+; FAST-NEXT:    blr
+  %a = call <8 x i64> @llvm.lrint.v8i64.v8f64(<8 x double> %x)
+  ret <8 x i64> %a
+}
+declare <8 x i64> @llvm.lrint.v8i64.v8f64(<8 x double>)
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/frame-index-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/frame-index-rv32.mir
new file mode 100644
index 0000000000..20747bd187
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/frame-index-rv32.mir
@@ -0,0 +1,32 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv32 -run-pass=instruction-select %s -o - \
+# RUN: | FileCheck %s
+--- |
+  define ptr @frame_index() {
+  entry:
+    %x = alloca i32, align 4
+    ret ptr %x
+  }
+
+...
+---
+name:            frame_index
+legalized:       true
+regBankSelected: true
+registers:
+  - { id: 0, class: gprb, preferred-register: '' }
+stack:
+  - { id: 0, name: x, type: default, offset: 0, size: 4, alignment: 4,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+body:             |
+  bb.1.entry:
+    ; CHECK-LABEL: name: frame_index
+    ; CHECK: [[ADDI:%[0-9]+]]:gpr = ADDI %stack.0.x, 0
+    ; CHECK-NEXT: $x10 = COPY [[ADDI]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(p0) = G_FRAME_INDEX %stack.0.x
+    $x10 = COPY %0(p0)
+    PseudoRET implicit $x10
+
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/frame-index-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/frame-index-rv64.mir
new file mode 100644
index 0000000000..dc265bda0a
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/frame-index-rv64.mir
@@ -0,0 +1,32 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv64 -run-pass=instruction-select %s -o - \
+# RUN: | FileCheck %s
+--- |
+  define ptr @frame_index() {
+  entry:
+    %x = alloca i32, align 4
+    ret ptr %x
+  }
+
+...
+---
+name:            frame_index
+legalized:       true
+regBankSelected: true
+registers:
+  - { id: 0, class: gprb, preferred-register: '' }
+stack:
+  - { id: 0, name: x, type: default, offset: 0, size: 4, alignment: 4,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+body:             |
+  bb.1.entry:
+    ; CHECK-LABEL: name: frame_index
+    ; CHECK: [[ADDI:%[0-9]+]]:gpr = ADDI %stack.0.x, 0
+    ; CHECK-NEXT: $x10 = COPY [[ADDI]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(p0) = G_FRAME_INDEX %stack.0.x
+    $x10 = COPY %0(p0)
+    PseudoRET implicit $x10
+
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/icmp-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/icmp-rv32.mir
index c45f10752f..cf0baeb952 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/icmp-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/icmp-rv32.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -march=riscv32 -run-pass=instruction-select -simplify-mir \
+# RUN: llc -mtriple=riscv32 -run-pass=instruction-select -simplify-mir \
 # RUN: -verify-machineinstrs %s -o - | FileCheck %s
 ---
 name:            cmp_ult_i32
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/icmp-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/icmp-rv64.mir
index 1a4232db97..7b6506e738 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/icmp-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/icmp-rv64.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -march=riscv64 -run-pass=instruction-select -simplify-mir \
+# RUN: llc -mtriple=riscv64 -run-pass=instruction-select -simplify-mir \
 # RUN: -verify-machineinstrs %s -o - | FileCheck %s
 ---
 name:            cmp_ult_i64
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/int-ptr-cast-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/int-ptr-cast-rv32.mir
new file mode 100644
index 0000000000..9a56020006
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/int-ptr-cast-rv32.mir
@@ -0,0 +1,46 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=riscv32 -run-pass=instruction-select -simplify-mir \
+# RUN: -verify-machineinstrs %s -o - | FileCheck %s
+
+---
+name:            ptrtoint_s32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: ptrtoint_s32
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(p0) = COPY $x10
+    %1:gprb(s32) = G_PTRTOINT %0(p0)
+    $x10 = COPY %1(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            inttoptr_s32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: inttoptr_s32
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(s32) = COPY $x10
+    %1:gprb(p0) = G_INTTOPTR %0(s32)
+    $x10 = COPY %1(p0)
+    PseudoRET implicit $x10
+
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/int-ptr-cast-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/int-ptr-cast-rv64.mir
new file mode 100644
index 0000000000..4f8259e695
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/int-ptr-cast-rv64.mir
@@ -0,0 +1,46 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=riscv64 -run-pass=instruction-select -simplify-mir \
+# RUN: -verify-machineinstrs %s -o - | FileCheck %s
+
+---
+name:            ptrtoint_s64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: ptrtoint_s64
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(p0) = COPY $x10
+    %1:gprb(s64) = G_PTRTOINT %0(p0)
+    $x10 = COPY %1(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            inttoptr_s64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: inttoptr_s64
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: $x10 = COPY [[COPY]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(s64) = COPY $x10
+    %1:gprb(p0) = G_INTTOPTR %0(s64)
+    $x10 = COPY %1(p0)
+    PseudoRET implicit $x10
+
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/ptradd-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/ptradd-rv32.mir
index bc0395685b..a96f55fa66 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/ptradd-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/ptradd-rv32.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -march=riscv32 -run-pass=instruction-select -simplify-mir \
+# RUN: llc -mtriple=riscv32 -run-pass=instruction-select -simplify-mir \
 # RUN: -verify-machineinstrs %s -o - | FileCheck %s
 ---
 name:            add_i32
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/ptradd-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/ptradd-rv64.mir
index d024a7c659..6346d913fa 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/ptradd-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/ptradd-rv64.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -march=riscv64 -run-pass=instruction-select -simplify-mir \
+# RUN: llc -mtriple=riscv64 -run-pass=instruction-select -simplify-mir \
 # RUN: -verify-machineinstrs %s -o - | FileCheck %s
 ---
 name:            add_i64
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/zba-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/zba-rv32.mir
new file mode 100644
index 0000000000..f90de3ea55
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/zba-rv32.mir
@@ -0,0 +1,152 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 3
+# RUN: llc -mtriple=riscv32 -mattr='+zba' -run-pass=instruction-select -simplify-mir -verify-machineinstrs %s -o - \
+# RUN: | FileCheck %s
+
+---
+name:            sh1add
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: sh1add
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; CHECK-NEXT: [[SH1ADD:%[0-9]+]]:gpr = SH1ADD [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: $x10 = COPY [[SH1ADD]]
+    %0:gprb(s32) = COPY $x10
+    %1:gprb(s32) = COPY $x11
+    %2:gprb(s32) = G_CONSTANT i32 1
+    %3:gprb(s32) = G_SHL %0, %2
+    %4:gprb(s32) = G_ADD %3, %1
+    $x10 = COPY %4(s32)
+...
+---
+name:            sh2add
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: sh2add
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; CHECK-NEXT: [[SH2ADD:%[0-9]+]]:gpr = SH2ADD [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: $x10 = COPY [[SH2ADD]]
+    %0:gprb(s32) = COPY $x10
+    %1:gprb(s32) = COPY $x11
+    %2:gprb(s32) = G_CONSTANT i32 2
+    %3:gprb(s32) = G_SHL %0, %2
+    %4:gprb(s32) = G_ADD %3, %1
+    $x10 = COPY %4(s32)
+...
+---
+name:            sh3add
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: sh3add
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; CHECK-NEXT: [[SH3ADD:%[0-9]+]]:gpr = SH3ADD [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: $x10 = COPY [[SH3ADD]]
+    %0:gprb(s32) = COPY $x10
+    %1:gprb(s32) = COPY $x11
+    %2:gprb(s32) = G_CONSTANT i32 3
+    %3:gprb(s32) = G_SHL %0, %2
+    %4:gprb(s32) = G_ADD %3, %1
+    $x10 = COPY %4(s32)
+...
+---
+name:            no_sh1add
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: no_sh1add
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[SLLI:%[0-9]+]]:gpr = SLLI [[COPY]], 1
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI [[SLLI]], 37
+    ; CHECK-NEXT: $x10 = COPY [[ADDI]]
+    %0:gprb(s32) = COPY $x10
+    %1:gprb(s32) = G_CONSTANT i32 37
+    %2:gprb(s32) = G_CONSTANT i32 1
+    %3:gprb(s32) = G_SHL %0, %2
+    %4:gprb(s32) = G_ADD %3, %1
+    $x10 = COPY %4(s32)
+...
+---
+name:            shXadd_complex_shl_and
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: shXadd_complex_shl_and
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[COPY]], 1
+    ; CHECK-NEXT: [[SH2ADD:%[0-9]+]]:gpr = SH2ADD [[SRLI]], [[COPY1]]
+    ; CHECK-NEXT: $x10 = COPY [[SH2ADD]]
+    %0:gprb(s32) = COPY $x10
+    %1:gprb(s32) = COPY $x11
+
+    %2:gprb(s32) = G_CONSTANT i32 1
+    %3:gprb(s32) = G_SHL %0, %2
+    %4:gprb(s32) = G_CONSTANT i32 4294967292
+    %5:gprb(s32) = G_AND %3, %4
+
+    %6:gprb(s32) = G_ADD %5, %1
+    $x10 = COPY %6(s32)
+...
+---
+name:            shXadd_complex_lshr_and
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: shXadd_complex_lshr_and
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; CHECK-NEXT: [[SRLI:%[0-9]+]]:gpr = SRLI [[COPY]], 29
+    ; CHECK-NEXT: [[SH2ADD:%[0-9]+]]:gpr = SH2ADD [[SRLI]], [[COPY1]]
+    ; CHECK-NEXT: $x10 = COPY [[SH2ADD]]
+    %0:gprb(s32) = COPY $x10
+    %1:gprb(s32) = COPY $x11
+
+    %2:gprb(s32) = G_CONSTANT i32 27
+    %3:gprb(s32) = G_LSHR %0, %2
+    %4:gprb(s32) = G_CONSTANT i32 60
+    %5:gprb(s32) = G_AND %3, %4
+
+    %6:gprb(s32) = G_ADD %5, %1
+    $x10 = COPY %6(s32)
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/zba-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/zba-rv64.mir
new file mode 100644
index 0000000000..092a3305b3
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/zba-rv64.mir
@@ -0,0 +1,152 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 3
+# RUN: llc -mtriple=riscv64 -mattr='+zba' -run-pass=instruction-select -simplify-mir -verify-machineinstrs %s -o - \
+# RUN: | FileCheck %s
+
+---
+name:            sh1add
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: sh1add
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; CHECK-NEXT: [[SH1ADD:%[0-9]+]]:gpr = SH1ADD [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: $x10 = COPY [[SH1ADD]]
+    %0:gprb(s64) = COPY $x10
+    %1:gprb(s64) = COPY $x11
+    %2:gprb(s64) = G_CONSTANT i64 1
+    %3:gprb(s64) = G_SHL %0, %2
+    %4:gprb(s64) = G_ADD %3, %1
+    $x10 = COPY %4(s64)
+...
+---
+name:            sh2add
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: sh2add
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; CHECK-NEXT: [[SH2ADD:%[0-9]+]]:gpr = SH2ADD [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: $x10 = COPY [[SH2ADD]]
+    %0:gprb(s64) = COPY $x10
+    %1:gprb(s64) = COPY $x11
+    %2:gprb(s64) = G_CONSTANT i64 2
+    %3:gprb(s64) = G_SHL %0, %2
+    %4:gprb(s64) = G_ADD %3, %1
+    $x10 = COPY %4(s64)
+...
+---
+name:            sh3add
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: sh3add
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; CHECK-NEXT: [[SH3ADD:%[0-9]+]]:gpr = SH3ADD [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: $x10 = COPY [[SH3ADD]]
+    %0:gprb(s64) = COPY $x10
+    %1:gprb(s64) = COPY $x11
+    %2:gprb(s64) = G_CONSTANT i64 3
+    %3:gprb(s64) = G_SHL %0, %2
+    %4:gprb(s64) = G_ADD %3, %1
+    $x10 = COPY %4(s64)
+...
+---
+name:            no_sh1add
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: no_sh1add
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[SLLI:%[0-9]+]]:gpr = SLLI [[COPY]], 1
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI [[SLLI]], 37
+    ; CHECK-NEXT: $x10 = COPY [[ADDI]]
+    %0:gprb(s64) = COPY $x10
+    %1:gprb(s64) = G_CONSTANT i64 37
+    %2:gprb(s64) = G_CONSTANT i64 1
+    %3:gprb(s64) = G_SHL %0, %2
+    %4:gprb(s64) = G_ADD %3, %1
+    $x10 = COPY %4(s64)
+...
+---
+name:            shXadd_complex_and_shl
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: shXadd_complex_and_shl
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; CHECK-NEXT: [[SRLIW:%[0-9]+]]:gpr = SRLIW [[COPY]], 1
+    ; CHECK-NEXT: [[SH3ADD:%[0-9]+]]:gpr = SH3ADD [[SRLIW]], [[COPY1]]
+    ; CHECK-NEXT: $x10 = COPY [[SH3ADD]]
+    %0:gprb(s64) = COPY $x10
+    %1:gprb(s64) = COPY $x11
+
+    %2:gprb(s64) = G_CONSTANT i64 4294967294
+    %3:gprb(s64) = G_AND %0, %2
+    %4:gprb(s64) = G_CONSTANT i64 2
+    %5:gprb(s64) = G_SHL %3, %4
+
+    %6:gprb(s64) = G_ADD %5, %1
+    $x10 = COPY %6(s64)
+...
+---
+name:            shXadd_complex_and_lshr
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: shXadd_complex_and_lshr
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; CHECK-NEXT: [[SRLIW:%[0-9]+]]:gpr = SRLIW [[COPY]], 2
+    ; CHECK-NEXT: [[SH1ADD:%[0-9]+]]:gpr = SH1ADD [[SRLIW]], [[COPY1]]
+    ; CHECK-NEXT: $x10 = COPY [[SH1ADD]]
+    %0:gprb(s64) = COPY $x10
+    %1:gprb(s64) = COPY $x11
+
+    %2:gprb(s64) = G_CONSTANT i64 4294967292
+    %3:gprb(s64) = G_AND %0, %2
+    %4:gprb(s64) = G_CONSTANT i64 1
+    %5:gprb(s64) = G_LSHR %3, %4
+
+    %6:gprb(s64) = G_ADD %5, %1
+    $x10 = COPY %6(s64)
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/calling-conv-ilp32-ilp32f-ilp32d-common.ll b/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/calling-conv-ilp32-ilp32f-ilp32d-common.ll
new file mode 100644
index 0000000000..e6837b90e3
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/calling-conv-ilp32-ilp32f-ilp32d-common.ll
@@ -0,0 +1,259 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 3
+; RUN: llc -mtriple=riscv32 \
+; RUN:    -global-isel -stop-after=irtranslator -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32I %s
+; RUN: llc -mtriple=riscv32 -mattr=+f -target-abi ilp32f \
+; RUN:    -global-isel -stop-after=irtranslator -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32I %s
+; RUN: llc -mtriple=riscv32 -mattr=+d -target-abi ilp32d \
+; RUN:    -global-isel -stop-after=irtranslator -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32I %s
+
+; This file contains tests that should have identical output for the ilp32,
+; ilp32f, and ilp32d ABIs. i.e. where no arguments are passed according to
+; the floating point ABI.
+
+; Check that on RV32, i64 is passed in a pair of registers. Unlike
+; the convention for varargs, this need not be an aligned pair.
+
+define i32 @callee_i64_in_regs(i32 %a, i64 %b) nounwind {
+  ; RV32I-LABEL: name: callee_i64_in_regs
+  ; RV32I: bb.1 (%ir-block.0):
+  ; RV32I-NEXT:   liveins: $x10, $x11, $x12
+  ; RV32I-NEXT: {{  $}}
+  ; RV32I-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+  ; RV32I-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+  ; RV32I-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $x12
+  ; RV32I-NEXT:   [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
+  ; RV32I-NEXT:   [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[MV]](s64)
+  ; RV32I-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[TRUNC]]
+  ; RV32I-NEXT:   $x10 = COPY [[ADD]](s32)
+  ; RV32I-NEXT:   PseudoRET implicit $x10
+  %b_trunc = trunc i64 %b to i32
+  %1 = add i32 %a, %b_trunc
+  ret i32 %1
+}
+
+define i32 @caller_i64_in_regs() nounwind {
+  ; RV32I-LABEL: name: caller_i64_in_regs
+  ; RV32I: bb.1 (%ir-block.0):
+  ; RV32I-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; RV32I-NEXT:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+  ; RV32I-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C1]](s64)
+  ; RV32I-NEXT:   $x10 = COPY [[C]](s32)
+  ; RV32I-NEXT:   $x11 = COPY [[UV]](s32)
+  ; RV32I-NEXT:   $x12 = COPY [[UV1]](s32)
+  ; RV32I-NEXT:   PseudoCALL target-flags(riscv-call) @callee_i64_in_regs, implicit-def $x1, implicit $x10, implicit $x11, implicit $x12, implicit-def $x10
+  ; RV32I-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+  ; RV32I-NEXT:   $x10 = COPY [[COPY]](s32)
+  ; RV32I-NEXT:   PseudoRET implicit $x10
+  %1 = call i32 @callee_i64_in_regs(i32 1, i64 2)
+  ret i32 %1
+}
+
+; Check that the stack is used once the GPRs are exhausted
+
+define i32 @callee_many_scalars(i8 %a, i16 %b, i32 %c, i64 %d, i32 %e, i32 %f, i64 %g, i32 %h) nounwind {
+  ; RV32I-LABEL: name: callee_many_scalars
+  ; RV32I: bb.1 (%ir-block.0):
+  ; RV32I-NEXT:   liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17
+  ; RV32I-NEXT: {{  $}}
+  ; RV32I-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+  ; RV32I-NEXT:   [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s32)
+  ; RV32I-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+  ; RV32I-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+  ; RV32I-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $x12
+  ; RV32I-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $x13
+  ; RV32I-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY $x14
+  ; RV32I-NEXT:   [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY3]](s32), [[COPY4]](s32)
+  ; RV32I-NEXT:   [[COPY5:%[0-9]+]]:_(s32) = COPY $x15
+  ; RV32I-NEXT:   [[COPY6:%[0-9]+]]:_(s32) = COPY $x16
+  ; RV32I-NEXT:   [[COPY7:%[0-9]+]]:_(s32) = COPY $x17
+  ; RV32I-NEXT:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1
+  ; RV32I-NEXT:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load (s32) from %fixed-stack.1, align 16)
+  ; RV32I-NEXT:   [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY7]](s32), [[LOAD]](s32)
+  ; RV32I-NEXT:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
+  ; RV32I-NEXT:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p0) :: (load (s32) from %fixed-stack.0)
+  ; RV32I-NEXT:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[TRUNC]](s8)
+  ; RV32I-NEXT:   [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[TRUNC1]](s16)
+  ; RV32I-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+  ; RV32I-NEXT:   [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[COPY2]]
+  ; RV32I-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[MV]](s64), [[MV1]]
+  ; RV32I-NEXT:   [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[ICMP]](s1)
+  ; RV32I-NEXT:   [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ADD1]]
+  ; RV32I-NEXT:   [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ADD2]], [[COPY5]]
+  ; RV32I-NEXT:   [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[COPY6]]
+  ; RV32I-NEXT:   [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[ADD4]], [[LOAD1]]
+  ; RV32I-NEXT:   $x10 = COPY [[ADD5]](s32)
+  ; RV32I-NEXT:   PseudoRET implicit $x10
+  %a_ext = zext i8 %a to i32
+  %b_ext = zext i16 %b to i32
+  %1 = add i32 %a_ext, %b_ext
+  %2 = add i32 %1, %c
+  %3 = icmp eq i64 %d, %g
+  %4 = zext i1 %3 to i32
+  %5 = add i32 %4, %2
+  %6 = add i32 %5, %e
+  %7 = add i32 %6, %f
+  %8 = add i32 %7, %h
+  ret i32 %8
+}
+
+define i32 @caller_many_scalars() nounwind {
+  ; RV32I-LABEL: name: caller_many_scalars
+  ; RV32I: bb.1 (%ir-block.0):
+  ; RV32I-NEXT:   [[C:%[0-9]+]]:_(s8) = G_CONSTANT i8 1
+  ; RV32I-NEXT:   [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 2
+  ; RV32I-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+  ; RV32I-NEXT:   [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+  ; RV32I-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
+  ; RV32I-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 6
+  ; RV32I-NEXT:   [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 7
+  ; RV32I-NEXT:   [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+  ; RV32I-NEXT:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[C]](s8)
+  ; RV32I-NEXT:   [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[C1]](s16)
+  ; RV32I-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C3]](s64)
+  ; RV32I-NEXT:   [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C6]](s64)
+  ; RV32I-NEXT:   [[COPY:%[0-9]+]]:_(p0) = COPY $x2
+  ; RV32I-NEXT:   [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; RV32I-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C8]](s32)
+  ; RV32I-NEXT:   G_STORE [[UV3]](s32), [[PTR_ADD]](p0) :: (store (s32) into stack, align 16)
+  ; RV32I-NEXT:   [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; RV32I-NEXT:   [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C9]](s32)
+  ; RV32I-NEXT:   G_STORE [[C7]](s32), [[PTR_ADD1]](p0) :: (store (s32) into stack + 4)
+  ; RV32I-NEXT:   $x10 = COPY [[ANYEXT]](s32)
+  ; RV32I-NEXT:   $x11 = COPY [[ANYEXT1]](s32)
+  ; RV32I-NEXT:   $x12 = COPY [[C2]](s32)
+  ; RV32I-NEXT:   $x13 = COPY [[UV]](s32)
+  ; RV32I-NEXT:   $x14 = COPY [[UV1]](s32)
+  ; RV32I-NEXT:   $x15 = COPY [[C4]](s32)
+  ; RV32I-NEXT:   $x16 = COPY [[C5]](s32)
+  ; RV32I-NEXT:   $x17 = COPY [[UV2]](s32)
+  ; RV32I-NEXT:   PseudoCALL target-flags(riscv-call) @callee_many_scalars, implicit-def $x1, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit $x14, implicit $x15, implicit $x16, implicit $x17, implicit-def $x10
+  ; RV32I-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $x10
+  ; RV32I-NEXT:   $x10 = COPY [[COPY1]](s32)
+  ; RV32I-NEXT:   PseudoRET implicit $x10
+  %1 = call i32 @callee_many_scalars(i8 1, i16 2, i32 3, i64 4, i32 5, i32 6, i64 7, i32 8)
+  ret i32 %1
+}
+
+; Check return of 2x xlen scalars
+
+define i64 @callee_small_scalar_ret() nounwind {
+  ; RV32I-LABEL: name: callee_small_scalar_ret
+  ; RV32I: bb.1 (%ir-block.0):
+  ; RV32I-NEXT:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1234567898765
+  ; RV32I-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C]](s64)
+  ; RV32I-NEXT:   $x10 = COPY [[UV]](s32)
+  ; RV32I-NEXT:   $x11 = COPY [[UV1]](s32)
+  ; RV32I-NEXT:   PseudoRET implicit $x10, implicit $x11
+  ret i64 1234567898765
+}
+
+define i32 @caller_small_scalar_ret() nounwind {
+  ; RV32I-LABEL: name: caller_small_scalar_ret
+  ; RV32I: bb.1 (%ir-block.0):
+  ; RV32I-NEXT:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 987654321234567
+  ; RV32I-NEXT:   PseudoCALL target-flags(riscv-call) @callee_small_scalar_ret, implicit-def $x1, implicit-def $x10, implicit-def $x11
+  ; RV32I-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+  ; RV32I-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+  ; RV32I-NEXT:   [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; RV32I-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[C]](s64), [[MV]]
+  ; RV32I-NEXT:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ICMP]](s1)
+  ; RV32I-NEXT:   $x10 = COPY [[ZEXT]](s32)
+  ; RV32I-NEXT:   PseudoRET implicit $x10
+  %1 = call i64 @callee_small_scalar_ret()
+  %2 = icmp eq i64 987654321234567, %1
+  %3 = zext i1 %2 to i32
+  ret i32 %3
+}
+
+; Check return of 2x xlen structs
+
+%struct.small = type { i32, ptr }
+
+define %struct.small @callee_small_struct_ret() nounwind {
+  ; RV32I-LABEL: name: callee_small_struct_ret
+  ; RV32I: bb.1 (%ir-block.0):
+  ; RV32I-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; RV32I-NEXT:   [[C1:%[0-9]+]]:_(p0) = G_CONSTANT i32 0
+  ; RV32I-NEXT:   $x10 = COPY [[C]](s32)
+  ; RV32I-NEXT:   $x11 = COPY [[C1]](p0)
+  ; RV32I-NEXT:   PseudoRET implicit $x10, implicit $x11
+  ret %struct.small { i32 1, ptr null }
+}
+
+define i32 @caller_small_struct_ret() nounwind {
+  ; RV32I-LABEL: name: caller_small_struct_ret
+  ; RV32I: bb.1 (%ir-block.0):
+  ; RV32I-NEXT:   PseudoCALL target-flags(riscv-call) @callee_small_struct_ret, implicit-def $x1, implicit-def $x10, implicit-def $x11
+  ; RV32I-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+  ; RV32I-NEXT:   [[COPY1:%[0-9]+]]:_(p0) = COPY $x11
+  ; RV32I-NEXT:   [[PTRTOINT:%[0-9]+]]:_(s32) = G_PTRTOINT [[COPY1]](p0)
+  ; RV32I-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[PTRTOINT]]
+  ; RV32I-NEXT:   $x10 = COPY [[ADD]](s32)
+  ; RV32I-NEXT:   PseudoRET implicit $x10
+  %1 = call %struct.small @callee_small_struct_ret()
+  %2 = extractvalue %struct.small %1, 0
+  %3 = extractvalue %struct.small %1, 1
+  %4 = ptrtoint ptr %3 to i32
+  %5 = add i32 %2, %4
+  ret i32 %5
+}
+
+; Check return of >2x xlen structs
+
+%struct.large = type { i32, i32, i32, i32 }
+
+define void @callee_large_struct_ret(ptr noalias sret(%struct.large) %agg.result) nounwind {
+  ; RV32I-LABEL: name: callee_large_struct_ret
+  ; RV32I: bb.1 (%ir-block.0):
+  ; RV32I-NEXT:   liveins: $x10
+  ; RV32I-NEXT: {{  $}}
+  ; RV32I-NEXT:   [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+  ; RV32I-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; RV32I-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; RV32I-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+  ; RV32I-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; RV32I-NEXT:   G_STORE [[C]](s32), [[COPY]](p0) :: (store (s32) into %ir.agg.result)
+  ; RV32I-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; RV32I-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p0) = nuw G_PTR_ADD [[COPY]], [[C4]](s32)
+  ; RV32I-NEXT:   G_STORE [[C1]](s32), [[PTR_ADD]](p0) :: (store (s32) into %ir.b)
+  ; RV32I-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+  ; RV32I-NEXT:   [[PTR_ADD1:%[0-9]+]]:_(p0) = nuw G_PTR_ADD [[COPY]], [[C5]](s32)
+  ; RV32I-NEXT:   G_STORE [[C2]](s32), [[PTR_ADD1]](p0) :: (store (s32) into %ir.c)
+  ; RV32I-NEXT:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+  ; RV32I-NEXT:   [[PTR_ADD2:%[0-9]+]]:_(p0) = nuw G_PTR_ADD [[COPY]], [[C6]](s32)
+  ; RV32I-NEXT:   G_STORE [[C3]](s32), [[PTR_ADD2]](p0) :: (store (s32) into %ir.d)
+  ; RV32I-NEXT:   PseudoRET
+  store i32 1, ptr %agg.result, align 4
+  %b = getelementptr inbounds %struct.large, ptr %agg.result, i32 0, i32 1
+  store i32 2, ptr %b, align 4
+  %c = getelementptr inbounds %struct.large, ptr %agg.result, i32 0, i32 2
+  store i32 3, ptr %c, align 4
+  %d = getelementptr inbounds %struct.large, ptr %agg.result, i32 0, i32 3
+  store i32 4, ptr %d, align 4
+  ret void
+}
+
+define i32 @caller_large_struct_ret() nounwind {
+  ; RV32I-LABEL: name: caller_large_struct_ret
+  ; RV32I: bb.1 (%ir-block.0):
+  ; RV32I-NEXT:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0
+  ; RV32I-NEXT:   $x10 = COPY [[FRAME_INDEX]](p0)
+  ; RV32I-NEXT:   PseudoCALL target-flags(riscv-call) @callee_large_struct_ret, implicit-def $x1, implicit $x10
+  ; RV32I-NEXT:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (dereferenceable load (s32) from %ir.1)
+  ; RV32I-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
+  ; RV32I-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p0) = nuw G_PTR_ADD [[FRAME_INDEX]], [[C]](s32)
+  ; RV32I-NEXT:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (dereferenceable load (s32) from %ir.3)
+  ; RV32I-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[LOAD]], [[LOAD1]]
+  ; RV32I-NEXT:   $x10 = COPY [[ADD]](s32)
+  ; RV32I-NEXT:   PseudoRET implicit $x10
+  %1 = alloca %struct.large
+  call void @callee_large_struct_ret(ptr sret(%struct.large) %1)
+  %2 = load i32, ptr %1
+  %3 = getelementptr inbounds %struct.large, ptr %1, i32 0, i32 3
+  %4 = load i32, ptr %3
+  %5 = add i32 %2, %4
+  ret i32 %5
+}
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/calling-conv-lp64-lp64f-lp64d-common.ll b/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/calling-conv-lp64-lp64f-lp64d-common.ll
new file mode 100644
index 0000000000..8533ed77b5
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/calling-conv-lp64-lp64f-lp64d-common.ll
@@ -0,0 +1,270 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 3
+; RUN: llc -mtriple=riscv64 \
+; RUN:    -global-isel -stop-after=irtranslator -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I %s
+; RUN: llc -mtriple=riscv64 -mattr=+f -target-abi lp64f \
+; RUN:    -global-isel -stop-after=irtranslator -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I %s
+; RUN: llc -mtriple=riscv64 -mattr=+d -target-abi lp64d \
+; RUN:    -global-isel -stop-after=irtranslator -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I %s
+
+; This file contains tests that should have identical output for the lp64,
+; lp64f, and lp64d ABIs. i.e. where no arguments are passed according to
+; the floating point ABI.
+
+; Check that on RV64, i128 is passed in a pair of registers. Unlike
+; the convention for varargs, this need not be an aligned pair.
+
+define i64 @callee_i128_in_regs(i64 %a, i128 %b) nounwind {
+  ; RV64I-LABEL: name: callee_i128_in_regs
+  ; RV64I: bb.1 (%ir-block.0):
+  ; RV64I-NEXT:   liveins: $x10, $x11, $x12
+  ; RV64I-NEXT: {{  $}}
+  ; RV64I-NEXT:   [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+  ; RV64I-NEXT:   [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+  ; RV64I-NEXT:   [[COPY2:%[0-9]+]]:_(s64) = COPY $x12
+  ; RV64I-NEXT:   [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[COPY1]](s64), [[COPY2]](s64)
+  ; RV64I-NEXT:   [[TRUNC:%[0-9]+]]:_(s64) = G_TRUNC [[MV]](s128)
+  ; RV64I-NEXT:   [[ADD:%[0-9]+]]:_(s64) = G_ADD [[COPY]], [[TRUNC]]
+  ; RV64I-NEXT:   $x10 = COPY [[ADD]](s64)
+  ; RV64I-NEXT:   PseudoRET implicit $x10
+  %b_trunc = trunc i128 %b to i64
+  %1 = add i64 %a, %b_trunc
+  ret i64 %1
+}
+
+define i64 @caller_i128_in_regs() nounwind {
+  ; RV64I-LABEL: name: caller_i128_in_regs
+  ; RV64I: bb.1 (%ir-block.0):
+  ; RV64I-NEXT:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+  ; RV64I-NEXT:   [[C1:%[0-9]+]]:_(s128) = G_CONSTANT i128 2
+  ; RV64I-NEXT:   [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[C1]](s128)
+  ; RV64I-NEXT:   $x10 = COPY [[C]](s64)
+  ; RV64I-NEXT:   $x11 = COPY [[UV]](s64)
+  ; RV64I-NEXT:   $x12 = COPY [[UV1]](s64)
+  ; RV64I-NEXT:   PseudoCALL target-flags(riscv-call) @callee_i128_in_regs, implicit-def $x1, implicit $x10, implicit $x11, implicit $x12, implicit-def $x10
+  ; RV64I-NEXT:   [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+  ; RV64I-NEXT:   $x10 = COPY [[COPY]](s64)
+  ; RV64I-NEXT:   PseudoRET implicit $x10
+  %1 = call i64 @callee_i128_in_regs(i64 1, i128 2)
+  ret i64 %1
+}
+
+; Check that the stack is used once the GPRs are exhausted
+
+define i32 @callee_many_scalars(i8 %a, i16 %b, i32 %c, i128 %d, i32 %e, i32 %f, i128 %g, i32 %h) nounwind {
+  ; RV64I-LABEL: name: callee_many_scalars
+  ; RV64I: bb.1 (%ir-block.0):
+  ; RV64I-NEXT:   liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17
+  ; RV64I-NEXT: {{  $}}
+  ; RV64I-NEXT:   [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+  ; RV64I-NEXT:   [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY]](s64)
+  ; RV64I-NEXT:   [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+  ; RV64I-NEXT:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s64)
+  ; RV64I-NEXT:   [[COPY2:%[0-9]+]]:_(s64) = COPY $x12
+  ; RV64I-NEXT:   [[TRUNC2:%[0-9]+]]:_(s32) = G_TRUNC [[COPY2]](s64)
+  ; RV64I-NEXT:   [[COPY3:%[0-9]+]]:_(s64) = COPY $x13
+  ; RV64I-NEXT:   [[COPY4:%[0-9]+]]:_(s64) = COPY $x14
+  ; RV64I-NEXT:   [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[COPY3]](s64), [[COPY4]](s64)
+  ; RV64I-NEXT:   [[COPY5:%[0-9]+]]:_(s64) = COPY $x15
+  ; RV64I-NEXT:   [[TRUNC3:%[0-9]+]]:_(s32) = G_TRUNC [[COPY5]](s64)
+  ; RV64I-NEXT:   [[COPY6:%[0-9]+]]:_(s64) = COPY $x16
+  ; RV64I-NEXT:   [[TRUNC4:%[0-9]+]]:_(s32) = G_TRUNC [[COPY6]](s64)
+  ; RV64I-NEXT:   [[COPY7:%[0-9]+]]:_(s64) = COPY $x17
+  ; RV64I-NEXT:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1
+  ; RV64I-NEXT:   [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[FRAME_INDEX]](p0) :: (load (s64) from %fixed-stack.1, align 16)
+  ; RV64I-NEXT:   [[MV1:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[COPY7]](s64), [[LOAD]](s64)
+  ; RV64I-NEXT:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
+  ; RV64I-NEXT:   [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[FRAME_INDEX1]](p0) :: (load (s64) from %fixed-stack.0)
+  ; RV64I-NEXT:   [[TRUNC5:%[0-9]+]]:_(s32) = G_TRUNC [[LOAD1]](s64)
+  ; RV64I-NEXT:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[TRUNC]](s8)
+  ; RV64I-NEXT:   [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[TRUNC1]](s16)
+  ; RV64I-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[ZEXT]], [[ZEXT1]]
+  ; RV64I-NEXT:   [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[TRUNC2]]
+  ; RV64I-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[MV]](s128), [[MV1]]
+  ; RV64I-NEXT:   [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[ICMP]](s1)
+  ; RV64I-NEXT:   [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ZEXT2]], [[ADD1]]
+  ; RV64I-NEXT:   [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ADD2]], [[TRUNC3]]
+  ; RV64I-NEXT:   [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[ADD3]], [[TRUNC4]]
+  ; RV64I-NEXT:   [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[ADD4]], [[TRUNC5]]
+  ; RV64I-NEXT:   [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[ADD5]](s32)
+  ; RV64I-NEXT:   $x10 = COPY [[ANYEXT]](s64)
+  ; RV64I-NEXT:   PseudoRET implicit $x10
+  %a_ext = zext i8 %a to i32
+  %b_ext = zext i16 %b to i32
+  %1 = add i32 %a_ext, %b_ext
+  %2 = add i32 %1, %c
+  %3 = icmp eq i128 %d, %g
+  %4 = zext i1 %3 to i32
+  %5 = add i32 %4, %2
+  %6 = add i32 %5, %e
+  %7 = add i32 %6, %f
+  %8 = add i32 %7, %h
+  ret i32 %8
+}
+
+define i32 @caller_many_scalars() nounwind {
+  ; RV64I-LABEL: name: caller_many_scalars
+  ; RV64I: bb.1 (%ir-block.0):
+  ; RV64I-NEXT:   [[C:%[0-9]+]]:_(s8) = G_CONSTANT i8 1
+  ; RV64I-NEXT:   [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 2
+  ; RV64I-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+  ; RV64I-NEXT:   [[C3:%[0-9]+]]:_(s128) = G_CONSTANT i128 4
+  ; RV64I-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
+  ; RV64I-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 6
+  ; RV64I-NEXT:   [[C6:%[0-9]+]]:_(s128) = G_CONSTANT i128 7
+  ; RV64I-NEXT:   [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+  ; RV64I-NEXT:   [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[C]](s8)
+  ; RV64I-NEXT:   [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[C1]](s16)
+  ; RV64I-NEXT:   [[ANYEXT2:%[0-9]+]]:_(s64) = G_ANYEXT [[C2]](s32)
+  ; RV64I-NEXT:   [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[C3]](s128)
+  ; RV64I-NEXT:   [[ANYEXT3:%[0-9]+]]:_(s64) = G_ANYEXT [[C4]](s32)
+  ; RV64I-NEXT:   [[ANYEXT4:%[0-9]+]]:_(s64) = G_ANYEXT [[C5]](s32)
+  ; RV64I-NEXT:   [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[C6]](s128)
+  ; RV64I-NEXT:   [[COPY:%[0-9]+]]:_(p0) = COPY $x2
+  ; RV64I-NEXT:   [[C8:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; RV64I-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C8]](s64)
+  ; RV64I-NEXT:   G_STORE [[UV3]](s64), [[PTR_ADD]](p0) :: (store (s64) into stack, align 16)
+  ; RV64I-NEXT:   [[ANYEXT5:%[0-9]+]]:_(s64) = G_ANYEXT [[C7]](s32)
+  ; RV64I-NEXT:   [[C9:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+  ; RV64I-NEXT:   [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[COPY]], [[C9]](s64)
+  ; RV64I-NEXT:   G_STORE [[ANYEXT5]](s64), [[PTR_ADD1]](p0) :: (store (s64) into stack + 8)
+  ; RV64I-NEXT:   $x10 = COPY [[ANYEXT]](s64)
+  ; RV64I-NEXT:   $x11 = COPY [[ANYEXT1]](s64)
+  ; RV64I-NEXT:   $x12 = COPY [[ANYEXT2]](s64)
+  ; RV64I-NEXT:   $x13 = COPY [[UV]](s64)
+  ; RV64I-NEXT:   $x14 = COPY [[UV1]](s64)
+  ; RV64I-NEXT:   $x15 = COPY [[ANYEXT3]](s64)
+  ; RV64I-NEXT:   $x16 = COPY [[ANYEXT4]](s64)
+  ; RV64I-NEXT:   $x17 = COPY [[UV2]](s64)
+  ; RV64I-NEXT:   PseudoCALL target-flags(riscv-call) @callee_many_scalars, implicit-def $x1, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit $x14, implicit $x15, implicit $x16, implicit $x17, implicit-def $x10
+  ; RV64I-NEXT:   [[COPY1:%[0-9]+]]:_(s64) = COPY $x10
+  ; RV64I-NEXT:   [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+  ; RV64I-NEXT:   [[ANYEXT6:%[0-9]+]]:_(s64) = G_ANYEXT [[TRUNC]](s32)
+  ; RV64I-NEXT:   $x10 = COPY [[ANYEXT6]](s64)
+  ; RV64I-NEXT:   PseudoRET implicit $x10
+  %1 = call i32 @callee_many_scalars(i8 1, i16 2, i32 3, i128 4, i32 5, i32 6, i128 7, i32 8)
+  ret i32 %1
+}
+
+; Check return of 2x xlen scalars
+
+define i128 @callee_small_scalar_ret() nounwind {
+  ; RV64I-LABEL: name: callee_small_scalar_ret
+  ; RV64I: bb.1 (%ir-block.0):
+  ; RV64I-NEXT:   [[C:%[0-9]+]]:_(s128) = G_CONSTANT i128 -1
+  ; RV64I-NEXT:   [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[C]](s128)
+  ; RV64I-NEXT:   $x10 = COPY [[UV]](s64)
+  ; RV64I-NEXT:   $x11 = COPY [[UV1]](s64)
+  ; RV64I-NEXT:   PseudoRET implicit $x10, implicit $x11
+  ret i128 -1
+}
+
+define i64 @caller_small_scalar_ret() nounwind {
+  ; RV64I-LABEL: name: caller_small_scalar_ret
+  ; RV64I: bb.1 (%ir-block.0):
+  ; RV64I-NEXT:   [[C:%[0-9]+]]:_(s128) = G_CONSTANT i128 -2
+  ; RV64I-NEXT:   PseudoCALL target-flags(riscv-call) @callee_small_scalar_ret, implicit-def $x1, implicit-def $x10, implicit-def $x11
+  ; RV64I-NEXT:   [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+  ; RV64I-NEXT:   [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+  ; RV64I-NEXT:   [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[COPY]](s64), [[COPY1]](s64)
+  ; RV64I-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[C]](s128), [[MV]]
+  ; RV64I-NEXT:   [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[ICMP]](s1)
+  ; RV64I-NEXT:   $x10 = COPY [[ZEXT]](s64)
+  ; RV64I-NEXT:   PseudoRET implicit $x10
+  %1 = call i128 @callee_small_scalar_ret()
+  %2 = icmp eq i128 -2, %1
+  %3 = zext i1 %2 to i64
+  ret i64 %3
+}
+
+; Check return of 2x xlen structs
+
+%struct.small = type { i64, ptr }
+
+define %struct.small @callee_small_struct_ret() nounwind {
+  ; RV64I-LABEL: name: callee_small_struct_ret
+  ; RV64I: bb.1 (%ir-block.0):
+  ; RV64I-NEXT:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+  ; RV64I-NEXT:   [[C1:%[0-9]+]]:_(p0) = G_CONSTANT i64 0
+  ; RV64I-NEXT:   $x10 = COPY [[C]](s64)
+  ; RV64I-NEXT:   $x11 = COPY [[C1]](p0)
+  ; RV64I-NEXT:   PseudoRET implicit $x10, implicit $x11
+  ret %struct.small { i64 1, ptr null }
+}
+
+define i64 @caller_small_struct_ret() nounwind {
+  ; RV64I-LABEL: name: caller_small_struct_ret
+  ; RV64I: bb.1 (%ir-block.0):
+  ; RV64I-NEXT:   PseudoCALL target-flags(riscv-call) @callee_small_struct_ret, implicit-def $x1, implicit-def $x10, implicit-def $x11
+  ; RV64I-NEXT:   [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+  ; RV64I-NEXT:   [[COPY1:%[0-9]+]]:_(p0) = COPY $x11
+  ; RV64I-NEXT:   [[PTRTOINT:%[0-9]+]]:_(s64) = G_PTRTOINT [[COPY1]](p0)
+  ; RV64I-NEXT:   [[ADD:%[0-9]+]]:_(s64) = G_ADD [[COPY]], [[PTRTOINT]]
+  ; RV64I-NEXT:   $x10 = COPY [[ADD]](s64)
+  ; RV64I-NEXT:   PseudoRET implicit $x10
+  %1 = call %struct.small @callee_small_struct_ret()
+  %2 = extractvalue %struct.small %1, 0
+  %3 = extractvalue %struct.small %1, 1
+  %4 = ptrtoint ptr %3 to i64
+  %5 = add i64 %2, %4
+  ret i64 %5
+}
+
+; Check return of >2x xlen structs
+
+%struct.large = type { i64, i64, i64, i64 }
+
+define void @callee_large_struct_ret(ptr noalias sret(%struct.large) %agg.result) nounwind {
+  ; RV64I-LABEL: name: callee_large_struct_ret
+  ; RV64I: bb.1 (%ir-block.0):
+  ; RV64I-NEXT:   liveins: $x10
+  ; RV64I-NEXT: {{  $}}
+  ; RV64I-NEXT:   [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+  ; RV64I-NEXT:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+  ; RV64I-NEXT:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+  ; RV64I-NEXT:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 3
+  ; RV64I-NEXT:   [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+  ; RV64I-NEXT:   G_STORE [[C]](s64), [[COPY]](p0) :: (store (s64) into %ir.agg.result, align 4)
+  ; RV64I-NEXT:   [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+  ; RV64I-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p0) = nuw G_PTR_ADD [[COPY]], [[C4]](s64)
+  ; RV64I-NEXT:   G_STORE [[C1]](s64), [[PTR_ADD]](p0) :: (store (s64) into %ir.b, align 4)
+  ; RV64I-NEXT:   [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+  ; RV64I-NEXT:   [[PTR_ADD1:%[0-9]+]]:_(p0) = nuw G_PTR_ADD [[COPY]], [[C5]](s64)
+  ; RV64I-NEXT:   G_STORE [[C2]](s64), [[PTR_ADD1]](p0) :: (store (s64) into %ir.c, align 4)
+  ; RV64I-NEXT:   [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
+  ; RV64I-NEXT:   [[PTR_ADD2:%[0-9]+]]:_(p0) = nuw G_PTR_ADD [[COPY]], [[C6]](s64)
+  ; RV64I-NEXT:   G_STORE [[C3]](s64), [[PTR_ADD2]](p0) :: (store (s64) into %ir.d, align 4)
+  ; RV64I-NEXT:   PseudoRET
+  store i64 1, ptr %agg.result, align 4
+  %b = getelementptr inbounds %struct.large, ptr %agg.result, i64 0, i32 1
+  store i64 2, ptr %b, align 4
+  %c = getelementptr inbounds %struct.large, ptr %agg.result, i64 0, i32 2
+  store i64 3, ptr %c, align 4
+  %d = getelementptr inbounds %struct.large, ptr %agg.result, i64 0, i32 3
+  store i64 4, ptr %d, align 4
+  ret void
+}
+
+define i64 @caller_large_struct_ret() nounwind {
+  ; RV64I-LABEL: name: caller_large_struct_ret
+  ; RV64I: bb.1 (%ir-block.0):
+  ; RV64I-NEXT:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0
+  ; RV64I-NEXT:   $x10 = COPY [[FRAME_INDEX]](p0)
+  ; RV64I-NEXT:   PseudoCALL target-flags(riscv-call) @callee_large_struct_ret, implicit-def $x1, implicit $x10
+  ; RV64I-NEXT:   [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[FRAME_INDEX]](p0) :: (dereferenceable load (s64) from %ir.1)
+  ; RV64I-NEXT:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
+  ; RV64I-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p0) = nuw G_PTR_ADD [[FRAME_INDEX]], [[C]](s64)
+  ; RV64I-NEXT:   [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[PTR_ADD]](p0) :: (dereferenceable load (s64) from %ir.3)
+  ; RV64I-NEXT:   [[ADD:%[0-9]+]]:_(s64) = G_ADD [[LOAD]], [[LOAD1]]
+  ; RV64I-NEXT:   $x10 = COPY [[ADD]](s64)
+  ; RV64I-NEXT:   PseudoRET implicit $x10
+  %1 = alloca %struct.large
+  call void @callee_large_struct_ret(ptr sret(%struct.large) %1)
+  %2 = load i64, ptr %1
+  %3 = getelementptr inbounds %struct.large, ptr %1, i64 0, i32 3
+  %4 = load i64, ptr %3
+  %5 = add i64 %2, %4
+  ret i64 %5
+}
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/call-lowering/variadic-call.ll b/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/variadic-call.ll
similarity index 100%
rename from llvm/test/CodeGen/RISCV/GlobalISel/call-lowering/variadic-call.ll
rename to llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/variadic-call.ll
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv32/legalize-inttoptr.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv32/legalize-inttoptr.mir
new file mode 100644
index 0000000000..074c5c2dda
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv32/legalize-inttoptr.mir
@@ -0,0 +1,131 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv32 -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+
+---
+name:            p0_s32
+body:             |
+  bb.0:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: p0_s32
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[COPY]](s32)
+    ; CHECK-NEXT: $x10 = COPY [[INTTOPTR]](p0)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = COPY $x10
+    %1:_(p0) = G_INTTOPTR %0(s32)
+    $x10 = COPY %1(p0)
+    PseudoRET implicit $x10
+
+...
+---
+name:            p0_s16
+body:             |
+  bb.0:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: p0_s16
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[AND]](s32)
+    ; CHECK-NEXT: $x10 = COPY [[INTTOPTR]](p0)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s32) = COPY $x10
+    %0:_(s16) = G_TRUNC %1(s32)
+    %2:_(p0) = G_INTTOPTR %0(s16)
+    $x10 = COPY %2(p0)
+    PseudoRET implicit $x10
+
+...
+---
+name:            p0_s8
+body:             |
+  bb.0:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: p0_s8
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[AND]](s32)
+    ; CHECK-NEXT: $x10 = COPY [[INTTOPTR]](p0)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s32) = COPY $x10
+    %0:_(s8) = G_TRUNC %1(s32)
+    %2:_(p0) = G_INTTOPTR %0(s8)
+    $x10 = COPY %2(p0)
+    PseudoRET implicit $x10
+
+...
+---
+name:            p0_s1
+body:             |
+  bb.0:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: p0_s1
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[AND]](s32)
+    ; CHECK-NEXT: $x10 = COPY [[INTTOPTR]](p0)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s32) = COPY $x10
+    %0:_(s1) = G_TRUNC %1(s32)
+    %2:_(p0) = G_INTTOPTR %0(s1)
+    $x10 = COPY %2(p0)
+    PseudoRET implicit $x10
+
+...
+---
+name:            p0_s64
+body:             |
+  bb.0:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: p0_s64
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[COPY]](s32)
+    ; CHECK-NEXT: $x10 = COPY [[INTTOPTR]](p0)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s32) = COPY $x10
+    %2:_(s32) = COPY $x11
+    %0:_(s64) = G_MERGE_VALUES %1(s32), %2(s32)
+    %3:_(p0) = G_INTTOPTR %0(s64)
+    $x10 = COPY %3(p0)
+    PseudoRET implicit $x10
+
+...
+---
+name:            p0_s44
+body:             |
+  bb.0:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: p0_s44
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[COPY]](s32)
+    ; CHECK-NEXT: $x10 = COPY [[INTTOPTR]](p0)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s32) = COPY $x10
+    %2:_(s32) = COPY $x11
+    %3:_(s64) = G_MERGE_VALUES %1(s32), %2(s32)
+    %0:_(s44) = G_TRUNC %3(s64)
+    %4:_(p0) = G_INTTOPTR %0(s44)
+    $x10 = COPY %4(p0)
+    PseudoRET implicit $x10
+
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv32/legalize-mulo.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv32/legalize-mulo.mir
new file mode 100644
index 0000000000..43fd1f99fd
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv32/legalize-mulo.mir
@@ -0,0 +1,348 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mattr=+m -mtriple=riscv32 -run-pass=legalizer %s -o - \
+# RUN: | FileCheck %s
+# RUN: llc -mattr=+zmmul -mtriple=riscv32 -run-pass=legalizer %s -o - \
+# RUN: | FileCheck %s
+# RUN: llc -mtriple=riscv32 -run-pass=legalizer %s -o - \
+# RUN: | FileCheck %s --check-prefix=LIBCALL
+
+---
+name:            smulo_i8
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: smulo_i8
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[ASHR]], [[ASHR1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[MUL]], [[C2]](s32)
+    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL2]], [[C2]](s32)
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[MUL]](s32), [[ASHR2]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[ICMP]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
+    ;
+    ; LIBCALL-LABEL: name: smulo_i8
+    ; LIBCALL: liveins: $x10, $x11
+    ; LIBCALL-NEXT: {{  $}}
+    ; LIBCALL-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; LIBCALL-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; LIBCALL-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
+    ; LIBCALL-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
+    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; LIBCALL-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
+    ; LIBCALL-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; LIBCALL-NEXT: $x10 = COPY [[ASHR]](s32)
+    ; LIBCALL-NEXT: $x11 = COPY [[ASHR1]](s32)
+    ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x10
+    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; LIBCALL-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C2]](s32)
+    ; LIBCALL-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL2]], [[C2]](s32)
+    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[COPY2]](s32), [[ASHR2]]
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s32)
+    ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s32)
+    ; LIBCALL-NEXT: PseudoRET implicit $x10, implicit $x11
+    %2:_(s32) = COPY $x10
+    %0:_(s8) = G_TRUNC %2(s32)
+    %3:_(s32) = COPY $x11
+    %1:_(s8) = G_TRUNC %3(s32)
+    %4:_(s8), %5:_(s1) = G_SMULO %0, %1
+    %6:_(s32) = G_ANYEXT %4(s8)
+    %7:_(s32) = G_ANYEXT %5(s1)
+    $x10 = COPY %6(s32)
+    $x11 = COPY %7(s32)
+    PseudoRET implicit $x10, implicit $x11
+
+...
+---
+name:            smulo_i16
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: smulo_i16
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[ASHR]], [[ASHR1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[MUL]], [[C2]](s32)
+    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL2]], [[C2]](s32)
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[MUL]](s32), [[ASHR2]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[ICMP]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
+    ;
+    ; LIBCALL-LABEL: name: smulo_i16
+    ; LIBCALL: liveins: $x10, $x11
+    ; LIBCALL-NEXT: {{  $}}
+    ; LIBCALL-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; LIBCALL-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; LIBCALL-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C]](s32)
+    ; LIBCALL-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
+    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; LIBCALL-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C1]](s32)
+    ; LIBCALL-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; LIBCALL-NEXT: $x10 = COPY [[ASHR]](s32)
+    ; LIBCALL-NEXT: $x11 = COPY [[ASHR1]](s32)
+    ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x10
+    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; LIBCALL-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C2]](s32)
+    ; LIBCALL-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL2]], [[C2]](s32)
+    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[COPY2]](s32), [[ASHR2]]
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s32)
+    ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s32)
+    ; LIBCALL-NEXT: PseudoRET implicit $x10, implicit $x11
+    %2:_(s32) = COPY $x10
+    %0:_(s16) = G_TRUNC %2(s32)
+    %3:_(s32) = COPY $x11
+    %1:_(s16) = G_TRUNC %3(s32)
+    %4:_(s16), %5:_(s1) = G_SMULO %0, %1
+    %6:_(s32) = G_ANYEXT %4(s16)
+    %7:_(s32) = G_ANYEXT %5(s1)
+    $x10 = COPY %6(s32)
+    $x11 = COPY %7(s32)
+    PseudoRET implicit $x10, implicit $x11
+
+...
+---
+name:            smulo_i32
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: smulo_i32
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: [[SMULH:%[0-9]+]]:_(s32) = G_SMULH [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[MUL]], [[C]](s32)
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[SMULH]](s32), [[ASHR]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[ICMP]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
+    ;
+    ; LIBCALL-LABEL: name: smulo_i32
+    ; LIBCALL: liveins: $x10, $x11
+    ; LIBCALL-NEXT: {{  $}}
+    ; LIBCALL-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; LIBCALL-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; LIBCALL-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[COPY]], [[C]](s32)
+    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; LIBCALL-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[COPY1]], [[C1]](s32)
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY]](s32)
+    ; LIBCALL-NEXT: $x11 = COPY [[ASHR]](s32)
+    ; LIBCALL-NEXT: $x12 = COPY [[COPY1]](s32)
+    ; LIBCALL-NEXT: $x13 = COPY [[ASHR1]](s32)
+    ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, implicit-def $x1, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit-def $x10, implicit-def $x11
+    ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x10
+    ; LIBCALL-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x11
+    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; LIBCALL-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C2]](s32)
+    ; LIBCALL-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C2]](s32)
+    ; LIBCALL-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; LIBCALL-NEXT: [[ASHR3:%[0-9]+]]:_(s32) = G_ASHR [[ASHR2]], [[C3]](s32)
+    ; LIBCALL-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; LIBCALL-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[COPY2]], [[ASHR2]]
+    ; LIBCALL-NEXT: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[COPY3]], [[ASHR3]]
+    ; LIBCALL-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[XOR]], [[XOR1]]
+    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[OR]](s32), [[C4]]
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s32)
+    ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s32)
+    ; LIBCALL-NEXT: PseudoRET implicit $x10, implicit $x11
+    %0:_(s32) = COPY $x10
+    %1:_(s32) = COPY $x11
+    %2:_(s32), %3:_(s1) = G_SMULO %0, %1
+    %4:_(s32) = G_ANYEXT %3(s1)
+    $x10 = COPY %2(s32)
+    $x11 = COPY %4(s32)
+    PseudoRET implicit $x10, implicit $x11
+
+...
+---
+name:            umulo_i8
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: umulo_i8
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND]], [[AND1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C2]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[MUL]](s32), [[AND2]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[ICMP]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
+    ;
+    ; LIBCALL-LABEL: name: umulo_i8
+    ; LIBCALL: liveins: $x10, $x11
+    ; LIBCALL-NEXT: {{  $}}
+    ; LIBCALL-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; LIBCALL-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; LIBCALL-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
+    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; LIBCALL-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; LIBCALL-NEXT: $x10 = COPY [[AND]](s32)
+    ; LIBCALL-NEXT: $x11 = COPY [[AND1]](s32)
+    ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x10
+    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; LIBCALL-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C2]]
+    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[COPY2]](s32), [[AND2]]
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s32)
+    ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s32)
+    ; LIBCALL-NEXT: PseudoRET implicit $x10, implicit $x11
+    %2:_(s32) = COPY $x10
+    %0:_(s8) = G_TRUNC %2(s32)
+    %3:_(s32) = COPY $x11
+    %1:_(s8) = G_TRUNC %3(s32)
+    %4:_(s8), %5:_(s1) = G_UMULO %0, %1
+    %6:_(s32) = G_ANYEXT %4(s8)
+    %7:_(s32) = G_ANYEXT %5(s1)
+    $x10 = COPY %6(s32)
+    $x11 = COPY %7(s32)
+    PseudoRET implicit $x10, implicit $x11
+
+...
+---
+name:            umulo_i16
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: umulo_i16
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND]], [[AND1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C2]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[MUL]](s32), [[AND2]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[ICMP]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
+    ;
+    ; LIBCALL-LABEL: name: umulo_i16
+    ; LIBCALL: liveins: $x10, $x11
+    ; LIBCALL-NEXT: {{  $}}
+    ; LIBCALL-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; LIBCALL-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; LIBCALL-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
+    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; LIBCALL-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; LIBCALL-NEXT: $x10 = COPY [[AND]](s32)
+    ; LIBCALL-NEXT: $x11 = COPY [[AND1]](s32)
+    ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__mulsi3, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x10
+    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; LIBCALL-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C2]]
+    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[COPY2]](s32), [[AND2]]
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s32)
+    ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s32)
+    ; LIBCALL-NEXT: PseudoRET implicit $x10, implicit $x11
+    %2:_(s32) = COPY $x10
+    %0:_(s16) = G_TRUNC %2(s32)
+    %3:_(s32) = COPY $x11
+    %1:_(s16) = G_TRUNC %3(s32)
+    %4:_(s16), %5:_(s1) = G_UMULO %0, %1
+    %6:_(s32) = G_ANYEXT %4(s16)
+    %7:_(s32) = G_ANYEXT %5(s1)
+    $x10 = COPY %6(s32)
+    $x11 = COPY %7(s32)
+    PseudoRET implicit $x10, implicit $x11
+
+...
+---
+name:            umulo_i32
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: umulo_i32
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[UMULH]](s32), [[C]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[ICMP]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
+    ;
+    ; LIBCALL-LABEL: name: umulo_i32
+    ; LIBCALL: liveins: $x10, $x11
+    ; LIBCALL-NEXT: {{  $}}
+    ; LIBCALL-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; LIBCALL-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY]](s32)
+    ; LIBCALL-NEXT: $x11 = COPY [[C]](s32)
+    ; LIBCALL-NEXT: $x12 = COPY [[COPY1]](s32)
+    ; LIBCALL-NEXT: $x13 = COPY [[C1]](s32)
+    ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, implicit-def $x1, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit-def $x10, implicit-def $x11
+    ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x10
+    ; LIBCALL-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x11
+    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; LIBCALL-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; LIBCALL-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C2]]
+    ; LIBCALL-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C3]]
+    ; LIBCALL-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; LIBCALL-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[COPY2]], [[AND]]
+    ; LIBCALL-NEXT: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[COPY3]], [[AND1]]
+    ; LIBCALL-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[XOR]], [[XOR1]]
+    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[OR]](s32), [[C4]]
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s32)
+    ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s32)
+    ; LIBCALL-NEXT: PseudoRET implicit $x10, implicit $x11
+    %0:_(s32) = COPY $x10
+    %1:_(s32) = COPY $x11
+    %2:_(s32), %3:_(s1) = G_UMULO %0, %1
+    %4:_(s32) = G_ANYEXT %3(s1)
+    $x10 = COPY %2(s32)
+    $x11 = COPY %4(s32)
+    PseudoRET implicit $x10, implicit $x11
+
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv32/legalize-ptrtoint.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv32/legalize-ptrtoint.mir
new file mode 100644
index 0000000000..9fba2dd962
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv32/legalize-ptrtoint.mir
@@ -0,0 +1,125 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv32 -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+
+---
+name:            p0_s32
+body:             |
+  bb.0:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: p0_s32
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[PTRTOINT:%[0-9]+]]:_(s32) = G_PTRTOINT [[COPY]](p0)
+    ; CHECK-NEXT: $x10 = COPY [[PTRTOINT]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(p0) = COPY $x10
+    %1:_(s32) = G_PTRTOINT %0(p0)
+    $x10 = COPY %1(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            p0_s16
+body:             |
+  bb.0:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: p0_s16
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[PTRTOINT:%[0-9]+]]:_(s32) = G_PTRTOINT [[COPY]](p0)
+    ; CHECK-NEXT: $x10 = COPY [[PTRTOINT]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(p0) = COPY $x10
+    %1:_(s16) = G_PTRTOINT %0(p0)
+    %2:_(s32) = G_ANYEXT %1(s16)
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            p0_s8
+body:             |
+  bb.0:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: p0_s8
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[PTRTOINT:%[0-9]+]]:_(s32) = G_PTRTOINT [[COPY]](p0)
+    ; CHECK-NEXT: $x10 = COPY [[PTRTOINT]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(p0) = COPY $x10
+    %1:_(s8) = G_PTRTOINT %0(p0)
+    %2:_(s32) = G_ANYEXT %1(s8)
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            p0_s1
+body:             |
+  bb.0:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: p0_s1
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[PTRTOINT:%[0-9]+]]:_(s32) = G_PTRTOINT [[COPY]](p0)
+    ; CHECK-NEXT: $x10 = COPY [[PTRTOINT]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(p0) = COPY $x10
+    %1:_(s1) = G_PTRTOINT %0(p0)
+    %2:_(s32) = G_ANYEXT %1(s1)
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            p0_s64
+body:             |
+  bb.0:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: p0_s64
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[PTRTOINT:%[0-9]+]]:_(s32) = G_PTRTOINT [[COPY]](p0)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: $x10 = COPY [[PTRTOINT]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[C]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
+    %0:_(p0) = COPY $x10
+    %1:_(s64) = G_PTRTOINT %0(p0)
+    %2:_(s32), %3:_(s32) = G_UNMERGE_VALUES %1(s64)
+    $x10 = COPY %2(s32)
+    $x11 = COPY %3(s32)
+    PseudoRET implicit $x10, implicit $x11
+
+...
+---
+name:            p0_s44
+body:             |
+  bb.0:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: p0_s44
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[PTRTOINT:%[0-9]+]]:_(s32) = G_PTRTOINT [[COPY]](p0)
+    ; CHECK-NEXT: $x10 = COPY [[PTRTOINT]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(p0) = COPY $x10
+    %1:_(s44) = G_PTRTOINT %0(p0)
+    %4:_(s32) = G_TRUNC %1
+    $x10 = COPY %4(s32)
+    PseudoRET implicit $x10
+
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv32/legalize-smax.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv32/legalize-smax.mir
new file mode 100644
index 0000000000..31df394c4f
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv32/legalize-smax.mir
@@ -0,0 +1,114 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv32 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name:            umax_i8
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: umax_i8
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ugt), [[AND]](s32), [[AND1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ICMP]], [[C2]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[AND2]](s32), [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[SELECT]], [[C3]](s32)
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C3]](s32)
+    ; CHECK-NEXT: $x10 = COPY [[ASHR]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = COPY $x10
+    %1:_(s32) = COPY $x11
+    %2:_(s8) = G_TRUNC %0(s32)
+    %3:_(s8) = G_TRUNC %1(s32)
+    %4:_(s8) = G_UMAX %2, %3
+    %5:_(s32) = G_SEXT %4(s8)
+    $x10 = COPY %5(s32)
+    PseudoRET implicit $x10
+...
+
+---
+name:            umax_i16
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: umax_i16
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ugt), [[AND]](s32), [[AND1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ICMP]], [[C2]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[AND2]](s32), [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[SELECT]], [[C3]](s32)
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C3]](s32)
+    ; CHECK-NEXT: $x10 = COPY [[ASHR]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = COPY $x10
+    %1:_(s32) = COPY $x11
+    %2:_(s16) = G_TRUNC %0(s32)
+    %3:_(s16) = G_TRUNC %1(s32)
+    %4:_(s16) = G_UMAX %2, %3
+    %5:_(s32) = G_SEXT %4(s16)
+    $x10 = COPY %5(s32)
+    PseudoRET implicit $x10
+...
+
+---
+name:            umax_i32
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: umax_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ugt), [[COPY]](s32), [[COPY1]]
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ICMP]], [[C]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s32), [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: $x10 = COPY [[SELECT]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = COPY $x10
+    %1:_(s32) = COPY $x11
+    %2:_(s32) = G_UMAX %0, %1
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+...
+
+---
+name:            umax_i64
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: umax_i64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ugt), [[COPY1]](s32), [[COPY3]]
+    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[COPY1]](s32), [[COPY3]]
+    ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(ugt), [[COPY]](s32), [[COPY2]]
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ICMP1]], [[C]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s32), [[ICMP2]], [[ICMP]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SELECT]], [[C1]]
+    ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[AND1]](s32), [[COPY]], [[COPY2]]
+    ; CHECK-NEXT: $x10 = COPY [[SELECT1]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = COPY $x10
+    %1:_(s32) = COPY $x11
+    %2:_(s32) = COPY $x12
+    %3:_(s32) = COPY $x13
+    %4:_(s64) = G_MERGE_VALUES %0(s32), %1(s32)
+    %5:_(s64) = G_MERGE_VALUES %2(s32), %3(s32)
+    %6:_(s64) = G_UMAX %4, %5
+    %7:_(s32) = G_TRUNC %6(s64)
+    $x10 = COPY %7(s32)
+    PseudoRET implicit $x10
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv32/legalize-smin.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv32/legalize-smin.mir
new file mode 100644
index 0000000000..2b589e6bb6
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv32/legalize-smin.mir
@@ -0,0 +1,114 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv32 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name:            umin_i8
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: umin_i8
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[AND]](s32), [[AND1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ICMP]], [[C2]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[AND2]](s32), [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[SELECT]], [[C3]](s32)
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C3]](s32)
+    ; CHECK-NEXT: $x10 = COPY [[ASHR]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = COPY $x10
+    %1:_(s32) = COPY $x11
+    %2:_(s8) = G_TRUNC %0(s32)
+    %3:_(s8) = G_TRUNC %1(s32)
+    %4:_(s8) = G_UMIN %2, %3
+    %5:_(s32) = G_SEXT %4(s8)
+    $x10 = COPY %5(s32)
+    PseudoRET implicit $x10
+...
+
+---
+name:            umin_i16
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: umin_i16
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[AND]](s32), [[AND1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ICMP]], [[C2]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[AND2]](s32), [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[SELECT]], [[C3]](s32)
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C3]](s32)
+    ; CHECK-NEXT: $x10 = COPY [[ASHR]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = COPY $x10
+    %1:_(s32) = COPY $x11
+    %2:_(s16) = G_TRUNC %0(s32)
+    %3:_(s16) = G_TRUNC %1(s32)
+    %4:_(s16) = G_UMIN %2, %3
+    %5:_(s32) = G_SEXT %4(s16)
+    $x10 = COPY %5(s32)
+    PseudoRET implicit $x10
+...
+
+---
+name:            umin_i32
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: umin_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[COPY]](s32), [[COPY1]]
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ICMP]], [[C]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s32), [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: $x10 = COPY [[SELECT]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = COPY $x10
+    %1:_(s32) = COPY $x11
+    %2:_(s32) = G_UMIN %0, %1
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+...
+
+---
+name:            umin_i64
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: umin_i64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[COPY1]](s32), [[COPY3]]
+    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[COPY1]](s32), [[COPY3]]
+    ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[COPY]](s32), [[COPY2]]
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ICMP1]], [[C]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s32), [[ICMP2]], [[ICMP]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SELECT]], [[C1]]
+    ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[AND1]](s32), [[COPY]], [[COPY2]]
+    ; CHECK-NEXT: $x10 = COPY [[SELECT1]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = COPY $x10
+    %1:_(s32) = COPY $x11
+    %2:_(s32) = COPY $x12
+    %3:_(s32) = COPY $x13
+    %4:_(s64) = G_MERGE_VALUES %0(s32), %1(s32)
+    %5:_(s64) = G_MERGE_VALUES %2(s32), %3(s32)
+    %6:_(s64) = G_UMIN %4, %5
+    %7:_(s32) = G_TRUNC %6(s64)
+    $x10 = COPY %7(s32)
+    PseudoRET implicit $x10
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv32/legalize-umax.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv32/legalize-umax.mir
new file mode 100644
index 0000000000..8dea2cb875
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv32/legalize-umax.mir
@@ -0,0 +1,112 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv32 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name:            umax_i8
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: umax_i8
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ugt), [[AND]](s32), [[AND1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ICMP]], [[C2]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[AND2]](s32), [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SELECT]], [[C3]]
+    ; CHECK-NEXT: $x10 = COPY [[AND3]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = COPY $x10
+    %1:_(s32) = COPY $x11
+    %2:_(s8) = G_TRUNC %0(s32)
+    %3:_(s8) = G_TRUNC %1(s32)
+    %4:_(s8) = G_UMAX %2, %3
+    %5:_(s32) = G_ZEXT %4(s8)
+    $x10 = COPY %5(s32)
+    PseudoRET implicit $x10
+...
+
+---
+name:            umax_i16
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: umax_i16
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ugt), [[AND]](s32), [[AND1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ICMP]], [[C2]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[AND2]](s32), [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SELECT]], [[C3]]
+    ; CHECK-NEXT: $x10 = COPY [[AND3]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = COPY $x10
+    %1:_(s32) = COPY $x11
+    %2:_(s16) = G_TRUNC %0(s32)
+    %3:_(s16) = G_TRUNC %1(s32)
+    %4:_(s16) = G_UMAX %2, %3
+    %5:_(s32) = G_ZEXT %4(s16)
+    $x10 = COPY %5(s32)
+    PseudoRET implicit $x10
+...
+
+---
+name:            umax_i32
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: umax_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ugt), [[COPY]](s32), [[COPY1]]
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ICMP]], [[C]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s32), [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: $x10 = COPY [[SELECT]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = COPY $x10
+    %1:_(s32) = COPY $x11
+    %2:_(s32) = G_UMAX %0, %1
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+...
+
+---
+name:            umax_i64
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: umax_i64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ugt), [[COPY1]](s32), [[COPY3]]
+    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[COPY1]](s32), [[COPY3]]
+    ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(ugt), [[COPY]](s32), [[COPY2]]
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ICMP1]], [[C]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s32), [[ICMP2]], [[ICMP]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SELECT]], [[C1]]
+    ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[AND1]](s32), [[COPY]], [[COPY2]]
+    ; CHECK-NEXT: $x10 = COPY [[SELECT1]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = COPY $x10
+    %1:_(s32) = COPY $x11
+    %2:_(s32) = COPY $x12
+    %3:_(s32) = COPY $x13
+    %4:_(s64) = G_MERGE_VALUES %0(s32), %1(s32)
+    %5:_(s64) = G_MERGE_VALUES %2(s32), %3(s32)
+    %6:_(s64) = G_UMAX %4, %5
+    %7:_(s32) = G_TRUNC %6(s64)
+    $x10 = COPY %7(s32)
+    PseudoRET implicit $x10
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv32/legalize-umin.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv32/legalize-umin.mir
new file mode 100644
index 0000000000..cd180a2a5b
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv32/legalize-umin.mir
@@ -0,0 +1,112 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv32 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name:            umin_i8
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: umin_i8
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[AND]](s32), [[AND1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ICMP]], [[C2]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[AND2]](s32), [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SELECT]], [[C3]]
+    ; CHECK-NEXT: $x10 = COPY [[AND3]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = COPY $x10
+    %1:_(s32) = COPY $x11
+    %2:_(s8) = G_TRUNC %0(s32)
+    %3:_(s8) = G_TRUNC %1(s32)
+    %4:_(s8) = G_UMIN %2, %3
+    %5:_(s32) = G_ZEXT %4(s8)
+    $x10 = COPY %5(s32)
+    PseudoRET implicit $x10
+...
+
+---
+name:            umin_i16
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: umin_i16
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[AND]](s32), [[AND1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ICMP]], [[C2]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[AND2]](s32), [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SELECT]], [[C3]]
+    ; CHECK-NEXT: $x10 = COPY [[AND3]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = COPY $x10
+    %1:_(s32) = COPY $x11
+    %2:_(s16) = G_TRUNC %0(s32)
+    %3:_(s16) = G_TRUNC %1(s32)
+    %4:_(s16) = G_UMIN %2, %3
+    %5:_(s32) = G_ZEXT %4(s16)
+    $x10 = COPY %5(s32)
+    PseudoRET implicit $x10
+...
+
+---
+name:            umin_i32
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: umin_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[COPY]](s32), [[COPY1]]
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ICMP]], [[C]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s32), [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: $x10 = COPY [[SELECT]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = COPY $x10
+    %1:_(s32) = COPY $x11
+    %2:_(s32) = G_UMIN %0, %1
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+...
+
+---
+name:            umin_i64
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: umin_i64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[COPY1]](s32), [[COPY3]]
+    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[COPY1]](s32), [[COPY3]]
+    ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[COPY]](s32), [[COPY2]]
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ICMP1]], [[C]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s32), [[ICMP2]], [[ICMP]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SELECT]], [[C1]]
+    ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[AND1]](s32), [[COPY]], [[COPY2]]
+    ; CHECK-NEXT: $x10 = COPY [[SELECT1]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = COPY $x10
+    %1:_(s32) = COPY $x11
+    %2:_(s32) = COPY $x12
+    %3:_(s32) = COPY $x13
+    %4:_(s64) = G_MERGE_VALUES %0(s32), %1(s32)
+    %5:_(s64) = G_MERGE_VALUES %2(s32), %3(s32)
+    %6:_(s64) = G_UMIN %4, %5
+    %7:_(s32) = G_TRUNC %6(s64)
+    $x10 = COPY %7(s32)
+    PseudoRET implicit $x10
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv64/legalize-inttoptr.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv64/legalize-inttoptr.mir
new file mode 100644
index 0000000000..cff8d36a20
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv64/legalize-inttoptr.mir
@@ -0,0 +1,153 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv64 -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+
+---
+name:            p0_s64
+body:             |
+  bb.0:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: p0_s64
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[COPY]](s64)
+    ; CHECK-NEXT: $x10 = COPY [[INTTOPTR]](p0)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = COPY $x10
+    %1:_(p0) = G_INTTOPTR %0(s64)
+    $x10 = COPY %1(p0)
+    PseudoRET implicit $x10
+
+...
+---
+name:            p0_s32
+body:             |
+  bb.0:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: p0_s32
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[AND]](s64)
+    ; CHECK-NEXT: $x10 = COPY [[INTTOPTR]](p0)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s64) = COPY $x10
+    %0:_(s32) = G_TRUNC %1(s64)
+    %2:_(p0) = G_INTTOPTR %0(s32)
+    $x10 = COPY %2(p0)
+    PseudoRET implicit $x10
+
+...
+---
+name:            p0_s16
+body:             |
+  bb.0:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: p0_s16
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[AND]](s64)
+    ; CHECK-NEXT: $x10 = COPY [[INTTOPTR]](p0)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s64) = COPY $x10
+    %0:_(s16) = G_TRUNC %1(s64)
+    %2:_(p0) = G_INTTOPTR %0(s16)
+    $x10 = COPY %2(p0)
+    PseudoRET implicit $x10
+
+...
+---
+name:            p0_s8
+body:             |
+  bb.0:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: p0_s8
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[AND]](s64)
+    ; CHECK-NEXT: $x10 = COPY [[INTTOPTR]](p0)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s64) = COPY $x10
+    %0:_(s8) = G_TRUNC %1(s64)
+    %2:_(p0) = G_INTTOPTR %0(s8)
+    $x10 = COPY %2(p0)
+    PseudoRET implicit $x10
+
+...
+---
+name:            p0_s1
+body:             |
+  bb.0:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: p0_s1
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[AND]](s64)
+    ; CHECK-NEXT: $x10 = COPY [[INTTOPTR]](p0)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s64) = COPY $x10
+    %0:_(s1) = G_TRUNC %1(s64)
+    %2:_(p0) = G_INTTOPTR %0(s1)
+    $x10 = COPY %2(p0)
+    PseudoRET implicit $x10
+
+...
+---
+name:            p0_s128
+body:             |
+  bb.0:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: p0_s128
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[COPY]](s64)
+    ; CHECK-NEXT: $x10 = COPY [[INTTOPTR]](p0)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s64) = COPY $x10
+    %2:_(s64) = COPY $x11
+    %0:_(s128) = G_MERGE_VALUES %1(s64), %2(s64)
+    %3:_(p0) = G_INTTOPTR %0(s128)
+    $x10 = COPY %3(p0)
+    PseudoRET implicit $x10
+
+...
+---
+name:            p0_s88
+body:             |
+  bb.0:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: p0_s88
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[COPY]](s64)
+    ; CHECK-NEXT: $x10 = COPY [[INTTOPTR]](p0)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %1:_(s64) = COPY $x10
+    %2:_(s64) = COPY $x11
+    %3:_(s128) = G_MERGE_VALUES %1(s64), %2(s64)
+    %0:_(s88) = G_TRUNC %3(s128)
+    %4:_(p0) = G_INTTOPTR %0(s88)
+    $x10 = COPY %4(p0)
+    PseudoRET implicit $x10
+
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv64/legalize-mulo.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv64/legalize-mulo.mir
new file mode 100644
index 0000000000..7e1ec1e096
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv64/legalize-mulo.mir
@@ -0,0 +1,450 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mattr=+m -mtriple=riscv64 -run-pass=legalizer %s -o - \
+# RUN: | FileCheck %s
+# RUN: llc -mattr=+zmmul -mtriple=riscv64 -run-pass=legalizer %s -o - \
+# RUN: | FileCheck %s
+# RUN: llc -mtriple=riscv64 -run-pass=legalizer %s -o - \
+# RUN: | FileCheck %s --check-prefix=LIBCALL
+
+---
+name:            smulo_i8
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: smulo_i8
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[ASHR]], [[ASHR1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
+    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[MUL]], [[C2]](s64)
+    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[SHL2]], [[C2]](s64)
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[MUL]](s64), [[ASHR2]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s64)
+    ; CHECK-NEXT: $x11 = COPY [[ICMP]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
+    ;
+    ; LIBCALL-LABEL: name: smulo_i8
+    ; LIBCALL: liveins: $x10, $x11
+    ; LIBCALL-NEXT: {{  $}}
+    ; LIBCALL-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; LIBCALL-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
+    ; LIBCALL-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
+    ; LIBCALL-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
+    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
+    ; LIBCALL-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
+    ; LIBCALL-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; LIBCALL-NEXT: $x10 = COPY [[ASHR]](s64)
+    ; LIBCALL-NEXT: $x11 = COPY [[ASHR1]](s64)
+    ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x10
+    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
+    ; LIBCALL-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[COPY2]], [[C2]](s64)
+    ; LIBCALL-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[SHL2]], [[C2]](s64)
+    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[COPY2]](s64), [[ASHR2]]
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s64)
+    ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s64)
+    ; LIBCALL-NEXT: PseudoRET implicit $x10, implicit $x11
+    %2:_(s64) = COPY $x10
+    %0:_(s8) = G_TRUNC %2(s64)
+    %3:_(s64) = COPY $x11
+    %1:_(s8) = G_TRUNC %3(s64)
+    %4:_(s8), %5:_(s1) = G_SMULO %0, %1
+    %6:_(s64) = G_ANYEXT %4(s8)
+    %7:_(s64) = G_ANYEXT %5(s1)
+    $x10 = COPY %6(s64)
+    $x11 = COPY %7(s64)
+    PseudoRET implicit $x10, implicit $x11
+
+...
+---
+name:            smulo_i16
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: smulo_i16
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
+    ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
+    ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[ASHR]], [[ASHR1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
+    ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[MUL]], [[C2]](s64)
+    ; CHECK-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[SHL2]], [[C2]](s64)
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[MUL]](s64), [[ASHR2]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s64)
+    ; CHECK-NEXT: $x11 = COPY [[ICMP]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
+    ;
+    ; LIBCALL-LABEL: name: smulo_i16
+    ; LIBCALL: liveins: $x10, $x11
+    ; LIBCALL-NEXT: {{  $}}
+    ; LIBCALL-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; LIBCALL-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
+    ; LIBCALL-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[C]](s64)
+    ; LIBCALL-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
+    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
+    ; LIBCALL-NEXT: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[C1]](s64)
+    ; LIBCALL-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[C1]](s64)
+    ; LIBCALL-NEXT: $x10 = COPY [[ASHR]](s64)
+    ; LIBCALL-NEXT: $x11 = COPY [[ASHR1]](s64)
+    ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x10
+    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
+    ; LIBCALL-NEXT: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[COPY2]], [[C2]](s64)
+    ; LIBCALL-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[SHL2]], [[C2]](s64)
+    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[COPY2]](s64), [[ASHR2]]
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s64)
+    ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s64)
+    ; LIBCALL-NEXT: PseudoRET implicit $x10, implicit $x11
+    %2:_(s64) = COPY $x10
+    %0:_(s16) = G_TRUNC %2(s64)
+    %3:_(s64) = COPY $x11
+    %1:_(s16) = G_TRUNC %3(s64)
+    %4:_(s16), %5:_(s1) = G_SMULO %0, %1
+    %6:_(s64) = G_ANYEXT %4(s16)
+    %7:_(s64) = G_ANYEXT %5(s1)
+    $x10 = COPY %6(s64)
+    $x11 = COPY %7(s64)
+    PseudoRET implicit $x10, implicit $x11
+
+...
+---
+name:            smulo_i32
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: smulo_i32
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s64) = G_SEXT_INREG [[COPY]], 32
+    ; CHECK-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s64) = G_SEXT_INREG [[COPY1]], 32
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[SEXT_INREG]], [[SEXT_INREG1]]
+    ; CHECK-NEXT: [[SEXT_INREG2:%[0-9]+]]:_(s64) = G_SEXT_INREG [[MUL]], 32
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[MUL]](s64), [[SEXT_INREG2]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s64)
+    ; CHECK-NEXT: $x11 = COPY [[ICMP]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
+    ;
+    ; LIBCALL-LABEL: name: smulo_i32
+    ; LIBCALL: liveins: $x10, $x11
+    ; LIBCALL-NEXT: {{  $}}
+    ; LIBCALL-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; LIBCALL-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; LIBCALL-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s64) = G_SEXT_INREG [[COPY]], 32
+    ; LIBCALL-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s64) = G_SEXT_INREG [[COPY1]], 32
+    ; LIBCALL-NEXT: $x10 = COPY [[SEXT_INREG]](s64)
+    ; LIBCALL-NEXT: $x11 = COPY [[SEXT_INREG1]](s64)
+    ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x10
+    ; LIBCALL-NEXT: [[SEXT_INREG2:%[0-9]+]]:_(s64) = G_SEXT_INREG [[COPY2]], 32
+    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[COPY2]](s64), [[SEXT_INREG2]]
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s64)
+    ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s64)
+    ; LIBCALL-NEXT: PseudoRET implicit $x10, implicit $x11
+    %2:_(s64) = COPY $x10
+    %0:_(s32) = G_TRUNC %2(s64)
+    %3:_(s64) = COPY $x11
+    %1:_(s32) = G_TRUNC %3(s64)
+    %4:_(s32), %5:_(s1) = G_SMULO %0, %1
+    %6:_(s64) = G_ANYEXT %4(s32)
+    %7:_(s64) = G_ANYEXT %5(s1)
+    $x10 = COPY %6(s64)
+    $x11 = COPY %7(s64)
+    PseudoRET implicit $x10, implicit $x11
+
+...
+---
+name:            smulo_i64
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: smulo_i64
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[SMULH:%[0-9]+]]:_(s64) = G_SMULH [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 63
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[MUL]], [[C]](s64)
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[SMULH]](s64), [[ASHR]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s64)
+    ; CHECK-NEXT: $x11 = COPY [[ICMP]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
+    ;
+    ; LIBCALL-LABEL: name: smulo_i64
+    ; LIBCALL: liveins: $x10, $x11
+    ; LIBCALL-NEXT: {{  $}}
+    ; LIBCALL-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; LIBCALL-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 63
+    ; LIBCALL-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[COPY]], [[C]](s64)
+    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 63
+    ; LIBCALL-NEXT: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[COPY1]], [[C1]](s64)
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY]](s64)
+    ; LIBCALL-NEXT: $x11 = COPY [[ASHR]](s64)
+    ; LIBCALL-NEXT: $x12 = COPY [[COPY1]](s64)
+    ; LIBCALL-NEXT: $x13 = COPY [[ASHR1]](s64)
+    ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__multi3, implicit-def $x1, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit-def $x10, implicit-def $x11
+    ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x10
+    ; LIBCALL-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x11
+    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; LIBCALL-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY2]], [[C2]](s64)
+    ; LIBCALL-NEXT: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C2]](s64)
+    ; LIBCALL-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 63
+    ; LIBCALL-NEXT: [[ASHR3:%[0-9]+]]:_(s64) = G_ASHR [[ASHR2]], [[C3]](s64)
+    ; LIBCALL-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; LIBCALL-NEXT: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[COPY2]], [[ASHR2]]
+    ; LIBCALL-NEXT: [[XOR1:%[0-9]+]]:_(s64) = G_XOR [[COPY3]], [[ASHR3]]
+    ; LIBCALL-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[XOR]], [[XOR1]]
+    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[OR]](s64), [[C4]]
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s64)
+    ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s64)
+    ; LIBCALL-NEXT: PseudoRET implicit $x10, implicit $x11
+    %0:_(s64) = COPY $x10
+    %1:_(s64) = COPY $x11
+    %2:_(s64), %3:_(s1) = G_SMULO %0, %1
+    %4:_(s64) = G_ANYEXT %3(s1)
+    $x10 = COPY %2(s64)
+    $x11 = COPY %4(s64)
+    PseudoRET implicit $x10, implicit $x11
+
+...
+---
+name:            umulo_i8
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: umulo_i8
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[AND]], [[AND1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[MUL]], [[C2]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[MUL]](s64), [[AND2]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s64)
+    ; CHECK-NEXT: $x11 = COPY [[ICMP]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
+    ;
+    ; LIBCALL-LABEL: name: umulo_i8
+    ; LIBCALL: liveins: $x10, $x11
+    ; LIBCALL-NEXT: {{  $}}
+    ; LIBCALL-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; LIBCALL-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
+    ; LIBCALL-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
+    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
+    ; LIBCALL-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; LIBCALL-NEXT: $x10 = COPY [[AND]](s64)
+    ; LIBCALL-NEXT: $x11 = COPY [[AND1]](s64)
+    ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x10
+    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
+    ; LIBCALL-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[COPY2]], [[C2]]
+    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[COPY2]](s64), [[AND2]]
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s64)
+    ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s64)
+    ; LIBCALL-NEXT: PseudoRET implicit $x10, implicit $x11
+    %2:_(s64) = COPY $x10
+    %0:_(s8) = G_TRUNC %2(s64)
+    %3:_(s64) = COPY $x11
+    %1:_(s8) = G_TRUNC %3(s64)
+    %4:_(s8), %5:_(s1) = G_UMULO %0, %1
+    %6:_(s64) = G_ANYEXT %4(s8)
+    %7:_(s64) = G_ANYEXT %5(s1)
+    $x10 = COPY %6(s64)
+    $x11 = COPY %7(s64)
+    PseudoRET implicit $x10, implicit $x11
+
+...
+---
+name:            umulo_i16
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: umulo_i16
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[AND]], [[AND1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[MUL]], [[C2]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[MUL]](s64), [[AND2]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s64)
+    ; CHECK-NEXT: $x11 = COPY [[ICMP]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
+    ;
+    ; LIBCALL-LABEL: name: umulo_i16
+    ; LIBCALL: liveins: $x10, $x11
+    ; LIBCALL-NEXT: {{  $}}
+    ; LIBCALL-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; LIBCALL-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
+    ; LIBCALL-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
+    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
+    ; LIBCALL-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; LIBCALL-NEXT: $x10 = COPY [[AND]](s64)
+    ; LIBCALL-NEXT: $x11 = COPY [[AND1]](s64)
+    ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x10
+    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
+    ; LIBCALL-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[COPY2]], [[C2]]
+    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[COPY2]](s64), [[AND2]]
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s64)
+    ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s64)
+    ; LIBCALL-NEXT: PseudoRET implicit $x10, implicit $x11
+    %2:_(s64) = COPY $x10
+    %0:_(s16) = G_TRUNC %2(s64)
+    %3:_(s64) = COPY $x11
+    %1:_(s16) = G_TRUNC %3(s64)
+    %4:_(s16), %5:_(s1) = G_UMULO %0, %1
+    %6:_(s64) = G_ANYEXT %4(s16)
+    %7:_(s64) = G_ANYEXT %5(s1)
+    $x10 = COPY %6(s64)
+    $x11 = COPY %7(s64)
+    PseudoRET implicit $x10, implicit $x11
+
+...
+---
+name:            umulo_i32
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: umulo_i32
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[AND]], [[AND1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[MUL]], [[C2]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[MUL]](s64), [[AND2]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s64)
+    ; CHECK-NEXT: $x11 = COPY [[ICMP]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
+    ;
+    ; LIBCALL-LABEL: name: umulo_i32
+    ; LIBCALL: liveins: $x10, $x11
+    ; LIBCALL-NEXT: {{  $}}
+    ; LIBCALL-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; LIBCALL-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
+    ; LIBCALL-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
+    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
+    ; LIBCALL-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; LIBCALL-NEXT: $x10 = COPY [[AND]](s64)
+    ; LIBCALL-NEXT: $x11 = COPY [[AND1]](s64)
+    ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__muldi3, implicit-def $x1, implicit $x10, implicit $x11, implicit-def $x10
+    ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x10
+    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
+    ; LIBCALL-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[COPY2]], [[C2]]
+    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[COPY2]](s64), [[AND2]]
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s64)
+    ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s64)
+    ; LIBCALL-NEXT: PseudoRET implicit $x10, implicit $x11
+    %2:_(s64) = COPY $x10
+    %0:_(s32) = G_TRUNC %2(s64)
+    %3:_(s64) = COPY $x11
+    %1:_(s32) = G_TRUNC %3(s64)
+    %4:_(s32), %5:_(s1) = G_UMULO %0, %1
+    %6:_(s64) = G_ANYEXT %4(s32)
+    %7:_(s64) = G_ANYEXT %5(s1)
+    $x10 = COPY %6(s64)
+    $x11 = COPY %7(s64)
+    PseudoRET implicit $x10, implicit $x11
+
+...
+---
+name:            umulo_i64
+body:             |
+  bb.1:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: umulo_i64
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[UMULH:%[0-9]+]]:_(s64) = G_UMULH [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[UMULH]](s64), [[C]]
+    ; CHECK-NEXT: $x10 = COPY [[MUL]](s64)
+    ; CHECK-NEXT: $x11 = COPY [[ICMP]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
+    ;
+    ; LIBCALL-LABEL: name: umulo_i64
+    ; LIBCALL: liveins: $x10, $x11
+    ; LIBCALL-NEXT: {{  $}}
+    ; LIBCALL-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; LIBCALL-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; LIBCALL-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; LIBCALL-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY]](s64)
+    ; LIBCALL-NEXT: $x11 = COPY [[C]](s64)
+    ; LIBCALL-NEXT: $x12 = COPY [[COPY1]](s64)
+    ; LIBCALL-NEXT: $x13 = COPY [[C1]](s64)
+    ; LIBCALL-NEXT: PseudoCALL target-flags(riscv-call) &__multi3, implicit-def $x1, implicit $x10, implicit $x11, implicit $x12, implicit $x13, implicit-def $x10, implicit-def $x11
+    ; LIBCALL-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x10
+    ; LIBCALL-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x11
+    ; LIBCALL-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; LIBCALL-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; LIBCALL-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY2]], [[C2]]
+    ; LIBCALL-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY3]], [[C3]]
+    ; LIBCALL-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; LIBCALL-NEXT: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[COPY2]], [[AND]]
+    ; LIBCALL-NEXT: [[XOR1:%[0-9]+]]:_(s64) = G_XOR [[COPY3]], [[AND1]]
+    ; LIBCALL-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[XOR]], [[XOR1]]
+    ; LIBCALL-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ne), [[OR]](s64), [[C4]]
+    ; LIBCALL-NEXT: $x10 = COPY [[COPY2]](s64)
+    ; LIBCALL-NEXT: $x11 = COPY [[ICMP]](s64)
+    ; LIBCALL-NEXT: PseudoRET implicit $x10, implicit $x11
+    %0:_(s64) = COPY $x10
+    %1:_(s64) = COPY $x11
+    %2:_(s64), %3:_(s1) = G_UMULO %0, %1
+    %4:_(s64) = G_ANYEXT %3(s1)
+    $x10 = COPY %2(s64)
+    $x11 = COPY %4(s64)
+    PseudoRET implicit $x10, implicit $x11
+
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv64/legalize-ptrtoint.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv64/legalize-ptrtoint.mir
new file mode 100644
index 0000000000..7299c59631
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv64/legalize-ptrtoint.mir
@@ -0,0 +1,145 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv64 -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+
+---
+name:            p0_s64
+body:             |
+  bb.0:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: p0_s64
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[PTRTOINT:%[0-9]+]]:_(s64) = G_PTRTOINT [[COPY]](p0)
+    ; CHECK-NEXT: $x10 = COPY [[PTRTOINT]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(p0) = COPY $x10
+    %1:_(s64) = G_PTRTOINT %0(p0)
+    $x10 = COPY %1(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            p0_s32
+body:             |
+  bb.0:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: p0_s32
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[PTRTOINT:%[0-9]+]]:_(s64) = G_PTRTOINT [[COPY]](p0)
+    ; CHECK-NEXT: $x10 = COPY [[PTRTOINT]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(p0) = COPY $x10
+    %1:_(s32) = G_PTRTOINT %0(p0)
+    %2:_(s64) = G_ANYEXT %1(s32)
+    $x10 = COPY %2(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            p0_s16
+body:             |
+  bb.0:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: p0_s16
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[PTRTOINT:%[0-9]+]]:_(s64) = G_PTRTOINT [[COPY]](p0)
+    ; CHECK-NEXT: $x10 = COPY [[PTRTOINT]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(p0) = COPY $x10
+    %1:_(s16) = G_PTRTOINT %0(p0)
+    %2:_(s64) = G_ANYEXT %1(s16)
+    $x10 = COPY %2(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            p0_s8
+body:             |
+  bb.0:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: p0_s8
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[PTRTOINT:%[0-9]+]]:_(s64) = G_PTRTOINT [[COPY]](p0)
+    ; CHECK-NEXT: $x10 = COPY [[PTRTOINT]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(p0) = COPY $x10
+    %1:_(s8) = G_PTRTOINT %0(p0)
+    %2:_(s64) = G_ANYEXT %1(s8)
+    $x10 = COPY %2(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            p0_s1
+body:             |
+  bb.0:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: p0_s1
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[PTRTOINT:%[0-9]+]]:_(s64) = G_PTRTOINT [[COPY]](p0)
+    ; CHECK-NEXT: $x10 = COPY [[PTRTOINT]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(p0) = COPY $x10
+    %1:_(s1) = G_PTRTOINT %0(p0)
+    %2:_(s64) = G_ANYEXT %1(s1)
+    $x10 = COPY %2(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            p0_s128
+body:             |
+  bb.0:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: p0_s128
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[PTRTOINT:%[0-9]+]]:_(s64) = G_PTRTOINT [[COPY]](p0)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: $x10 = COPY [[PTRTOINT]](s64)
+    ; CHECK-NEXT: $x11 = COPY [[C]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
+    %0:_(p0) = COPY $x10
+    %1:_(s128) = G_PTRTOINT %0(p0)
+    %2:_(s64), %3:_(s64) = G_UNMERGE_VALUES %1(s128)
+    $x10 = COPY %2(s64)
+    $x11 = COPY %3(s64)
+    PseudoRET implicit $x10, implicit $x11
+
+...
+---
+name:            p0_s88
+body:             |
+  bb.0:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: p0_s88
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[PTRTOINT:%[0-9]+]]:_(s64) = G_PTRTOINT [[COPY]](p0)
+    ; CHECK-NEXT: $x10 = COPY [[PTRTOINT]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(p0) = COPY $x10
+    %1:_(s88) = G_PTRTOINT %0(p0)
+    %4:_(s64) = G_TRUNC %1
+    $x10 = COPY %4(s64)
+    PseudoRET implicit $x10
+
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv64/legalize-smax.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv64/legalize-smax.mir
new file mode 100644
index 0000000000..43f4309dc5
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv64/legalize-smax.mir
@@ -0,0 +1,110 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv64 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name:            umax_i8
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: umax_i8
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ugt), [[AND]](s64), [[AND1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[ICMP]], [[C2]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[AND2]](s64), [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[SELECT]], [[C3]](s64)
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C3]](s64)
+    ; CHECK-NEXT: $x10 = COPY [[ASHR]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = COPY $x10
+    %1:_(s64) = COPY $x11
+    %2:_(s8) = G_TRUNC %0(s64)
+    %3:_(s8) = G_TRUNC %1(s64)
+    %4:_(s8) = G_UMAX %2, %3
+    %5:_(s64) = G_SEXT %4(s8)
+    $x10 = COPY %5(s64)
+    PseudoRET implicit $x10
+...
+
+---
+name:            umax_i16
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: umax_i16
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ugt), [[AND]](s64), [[AND1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[ICMP]], [[C2]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[AND2]](s64), [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[SELECT]], [[C3]](s64)
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C3]](s64)
+    ; CHECK-NEXT: $x10 = COPY [[ASHR]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = COPY $x10
+    %1:_(s64) = COPY $x11
+    %2:_(s16) = G_TRUNC %0(s64)
+    %3:_(s16) = G_TRUNC %1(s64)
+    %4:_(s16) = G_UMAX %2, %3
+    %5:_(s64) = G_SEXT %4(s16)
+    $x10 = COPY %5(s64)
+    PseudoRET implicit $x10
+...
+
+---
+name:            umax_i32
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: umax_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ugt), [[AND]](s64), [[AND1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[ICMP]], [[C2]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[AND2]](s64), [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s64) = G_SEXT_INREG [[SELECT]], 32
+    ; CHECK-NEXT: $x10 = COPY [[SEXT_INREG]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = COPY $x10
+    %1:_(s64) = COPY $x11
+    %2:_(s32) = G_TRUNC %0(s64)
+    %3:_(s32) = G_TRUNC %1(s64)
+    %4:_(s32) = G_UMAX %2, %3
+    %5:_(s64) = G_SEXT %4(s32)
+    $x10 = COPY %5(s64)
+    PseudoRET implicit $x10
+...
+
+---
+name:            umax_i64
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: umax_i64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ugt), [[COPY]](s64), [[COPY1]]
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[ICMP]], [[C]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[AND]](s64), [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: $x10 = COPY [[SELECT]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = COPY $x10
+    %1:_(s64) = COPY $x11
+    %2:_(s64) = G_UMAX %0, %1
+    $x10 = COPY %2(s64)
+    PseudoRET implicit $x10
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv64/legalize-smin.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv64/legalize-smin.mir
new file mode 100644
index 0000000000..85fea46b4b
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv64/legalize-smin.mir
@@ -0,0 +1,110 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv64 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name:            umin_i8
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: umin_i8
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[AND]](s64), [[AND1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[ICMP]], [[C2]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[AND2]](s64), [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 56
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[SELECT]], [[C3]](s64)
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C3]](s64)
+    ; CHECK-NEXT: $x10 = COPY [[ASHR]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = COPY $x10
+    %1:_(s64) = COPY $x11
+    %2:_(s8) = G_TRUNC %0(s64)
+    %3:_(s8) = G_TRUNC %1(s64)
+    %4:_(s8) = G_UMIN %2, %3
+    %5:_(s64) = G_SEXT %4(s8)
+    $x10 = COPY %5(s64)
+    PseudoRET implicit $x10
+...
+
+---
+name:            umin_i16
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: umin_i16
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[AND]](s64), [[AND1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[ICMP]], [[C2]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[AND2]](s64), [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
+    ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[SELECT]], [[C3]](s64)
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C3]](s64)
+    ; CHECK-NEXT: $x10 = COPY [[ASHR]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = COPY $x10
+    %1:_(s64) = COPY $x11
+    %2:_(s16) = G_TRUNC %0(s64)
+    %3:_(s16) = G_TRUNC %1(s64)
+    %4:_(s16) = G_UMIN %2, %3
+    %5:_(s64) = G_SEXT %4(s16)
+    $x10 = COPY %5(s64)
+    PseudoRET implicit $x10
+...
+
+---
+name:            umin_i32
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: umin_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[AND]](s64), [[AND1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[ICMP]], [[C2]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[AND2]](s64), [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s64) = G_SEXT_INREG [[SELECT]], 32
+    ; CHECK-NEXT: $x10 = COPY [[SEXT_INREG]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = COPY $x10
+    %1:_(s64) = COPY $x11
+    %2:_(s32) = G_TRUNC %0(s64)
+    %3:_(s32) = G_TRUNC %1(s64)
+    %4:_(s32) = G_UMIN %2, %3
+    %5:_(s64) = G_SEXT %4(s32)
+    $x10 = COPY %5(s64)
+    PseudoRET implicit $x10
+...
+
+---
+name:            umin_i64
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: umin_i64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[COPY]](s64), [[COPY1]]
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[ICMP]], [[C]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[AND]](s64), [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: $x10 = COPY [[SELECT]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = COPY $x10
+    %1:_(s64) = COPY $x11
+    %2:_(s64) = G_UMIN %0, %1
+    $x10 = COPY %2(s64)
+    PseudoRET implicit $x10
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv64/legalize-umax.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv64/legalize-umax.mir
new file mode 100644
index 0000000000..d0310e3e21
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv64/legalize-umax.mir
@@ -0,0 +1,109 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv64 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name:            umax_i8
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: umax_i8
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ugt), [[AND]](s64), [[AND1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[ICMP]], [[C2]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[AND2]](s64), [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s64) = G_AND [[SELECT]], [[C3]]
+    ; CHECK-NEXT: $x10 = COPY [[AND3]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = COPY $x10
+    %1:_(s64) = COPY $x11
+    %2:_(s8) = G_TRUNC %0(s64)
+    %3:_(s8) = G_TRUNC %1(s64)
+    %4:_(s8) = G_UMAX %2, %3
+    %5:_(s64) = G_ZEXT %4(s8)
+    $x10 = COPY %5(s64)
+    PseudoRET implicit $x10
+...
+
+---
+name:            umax_i16
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: umax_i16
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ugt), [[AND]](s64), [[AND1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[ICMP]], [[C2]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[AND2]](s64), [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s64) = G_AND [[SELECT]], [[C3]]
+    ; CHECK-NEXT: $x10 = COPY [[AND3]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = COPY $x10
+    %1:_(s64) = COPY $x11
+    %2:_(s16) = G_TRUNC %0(s64)
+    %3:_(s16) = G_TRUNC %1(s64)
+    %4:_(s16) = G_UMAX %2, %3
+    %5:_(s64) = G_ZEXT %4(s16)
+    $x10 = COPY %5(s64)
+    PseudoRET implicit $x10
+...
+
+---
+name:            umax_i32
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: umax_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ugt), [[AND]](s64), [[AND1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[ICMP]], [[C2]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[AND2]](s64), [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s64) = G_AND [[SELECT]], [[C3]]
+    ; CHECK-NEXT: $x10 = COPY [[AND3]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = COPY $x10
+    %1:_(s64) = COPY $x11
+    %2:_(s32) = G_TRUNC %0(s64)
+    %3:_(s32) = G_TRUNC %1(s64)
+    %4:_(s32) = G_UMAX %2, %3
+    %5:_(s64) = G_ZEXT %4(s32)
+    $x10 = COPY %5(s64)
+    PseudoRET implicit $x10
+...
+
+---
+name:            umax_i64
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: umax_i64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ugt), [[COPY]](s64), [[COPY1]]
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[ICMP]], [[C]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[AND]](s64), [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: $x10 = COPY [[SELECT]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = COPY $x10
+    %1:_(s64) = COPY $x11
+    %2:_(s64) = G_UMAX %0, %1
+    $x10 = COPY %2(s64)
+    PseudoRET implicit $x10
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv64/legalize-umin.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv64/legalize-umin.mir
new file mode 100644
index 0000000000..a0eec3298a
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rv64/legalize-umin.mir
@@ -0,0 +1,109 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv64 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name:            umin_i8
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: umin_i8
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[AND]](s64), [[AND1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[ICMP]], [[C2]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[AND2]](s64), [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s64) = G_AND [[SELECT]], [[C3]]
+    ; CHECK-NEXT: $x10 = COPY [[AND3]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = COPY $x10
+    %1:_(s64) = COPY $x11
+    %2:_(s8) = G_TRUNC %0(s64)
+    %3:_(s8) = G_TRUNC %1(s64)
+    %4:_(s8) = G_UMIN %2, %3
+    %5:_(s64) = G_ZEXT %4(s8)
+    $x10 = COPY %5(s64)
+    PseudoRET implicit $x10
+...
+
+---
+name:            umin_i16
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: umin_i16
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[AND]](s64), [[AND1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[ICMP]], [[C2]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[AND2]](s64), [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s64) = G_AND [[SELECT]], [[C3]]
+    ; CHECK-NEXT: $x10 = COPY [[AND3]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = COPY $x10
+    %1:_(s64) = COPY $x11
+    %2:_(s16) = G_TRUNC %0(s64)
+    %3:_(s16) = G_TRUNC %1(s64)
+    %4:_(s16) = G_UMIN %2, %3
+    %5:_(s64) = G_ZEXT %4(s16)
+    $x10 = COPY %5(s64)
+    PseudoRET implicit $x10
+...
+
+---
+name:            umin_i32
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: umin_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C1]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[AND]](s64), [[AND1]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[ICMP]], [[C2]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[AND2]](s64), [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s64) = G_AND [[SELECT]], [[C3]]
+    ; CHECK-NEXT: $x10 = COPY [[AND3]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = COPY $x10
+    %1:_(s64) = COPY $x11
+    %2:_(s32) = G_TRUNC %0(s64)
+    %3:_(s32) = G_TRUNC %1(s64)
+    %4:_(s32) = G_UMIN %2, %3
+    %5:_(s64) = G_ZEXT %4(s32)
+    $x10 = COPY %5(s64)
+    PseudoRET implicit $x10
+...
+
+---
+name:            umin_i64
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: umin_i64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[COPY]](s64), [[COPY1]]
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[ICMP]], [[C]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[AND]](s64), [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: $x10 = COPY [[SELECT]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = COPY $x10
+    %1:_(s64) = COPY $x11
+    %2:_(s64) = G_UMIN %0, %1
+    $x10 = COPY %2(s64)
+    PseudoRET implicit $x10
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/alu-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/alu-rv32.mir
index 6f8d76953d..1677c734eb 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/alu-rv32.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/alu-rv32.mir
@@ -408,14 +408,54 @@ body:             |
     ; RV32I-NEXT: [[COPY:%[0-9]+]]:gprb(p0) = COPY $x10
     ; RV32I-NEXT: [[COPY1:%[0-9]+]]:gprb(s32) = COPY $x11
     ; RV32I-NEXT: [[PTR_ADD:%[0-9]+]]:gprb(p0) = G_PTR_ADD [[COPY]], [[COPY1]](s32)
-    ; RV32I-NEXT: [[COPY2:%[0-9]+]]:gprb(p0) = COPY [[PTR_ADD]](p0)
-    ; RV32I-NEXT: $x10 = COPY [[COPY2]](p0)
+    ; RV32I-NEXT: $x10 = COPY [[PTR_ADD]](p0)
     ; RV32I-NEXT: PseudoRET implicit $x10
     %0:_(p0) = COPY $x10
     %1:_(s32) = COPY $x11
     %2:_(p0) = G_PTR_ADD %0, %1(s32)
-    %3:_(p0) = COPY %2(p0)
-    $x10 = COPY %3(p0)
+    $x10 = COPY %2(p0)
+    PseudoRET implicit $x10
+
+...
+---
+name:            ptrtoint
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+
+    ; RV32I-LABEL: name: ptrtoint
+    ; RV32I: liveins: $x10
+    ; RV32I-NEXT: {{  $}}
+    ; RV32I-NEXT: [[COPY:%[0-9]+]]:gprb(p0) = COPY $x10
+    ; RV32I-NEXT: [[PTRTOINT:%[0-9]+]]:gprb(s32) = G_PTRTOINT [[COPY]](p0)
+    ; RV32I-NEXT: $x10 = COPY [[PTRTOINT]](s32)
+    ; RV32I-NEXT: PseudoRET implicit $x10
+    %0:_(p0) = COPY $x10
+    %1:_(s32) = G_PTRTOINT %0(p0)
+    $x10 = COPY %1(s32)
+    PseudoRET implicit $x10
+
+...
+---
+name:            inttoprt
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+
+    ; RV32I-LABEL: name: inttoprt
+    ; RV32I: liveins: $x10
+    ; RV32I-NEXT: {{  $}}
+    ; RV32I-NEXT: [[COPY:%[0-9]+]]:gprb(s32) = COPY $x10
+    ; RV32I-NEXT: [[INTTOPTR:%[0-9]+]]:gprb(p0) = G_INTTOPTR [[COPY]](s32)
+    ; RV32I-NEXT: $x10 = COPY [[INTTOPTR]](p0)
+    ; RV32I-NEXT: PseudoRET implicit $x10
+    %0:_(s32) = COPY $x10
+    %1:_(p0) = G_INTTOPTR %0(s32)
+    $x10 = COPY %1(p0)
     PseudoRET implicit $x10
 
 ...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/alu-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/alu-rv64.mir
index 26120f7704..701f3d49ed 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/alu-rv64.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/alu-rv64.mir
@@ -429,14 +429,54 @@ body:             |
     ; RV64I-NEXT: [[COPY:%[0-9]+]]:gprb(p0) = COPY $x10
     ; RV64I-NEXT: [[COPY1:%[0-9]+]]:gprb(s64) = COPY $x11
     ; RV64I-NEXT: [[PTR_ADD:%[0-9]+]]:gprb(p0) = G_PTR_ADD [[COPY]], [[COPY1]](s64)
-    ; RV64I-NEXT: [[COPY2:%[0-9]+]]:gprb(p0) = COPY [[PTR_ADD]](p0)
-    ; RV64I-NEXT: $x10 = COPY [[COPY2]](p0)
+    ; RV64I-NEXT: $x10 = COPY [[PTR_ADD]](p0)
     ; RV64I-NEXT: PseudoRET implicit $x10
     %0:_(p0) = COPY $x10
     %1:_(s64) = COPY $x11
     %2:_(p0) = G_PTR_ADD %0, %1(s64)
-    %3:_(p0) = COPY %2(p0)
-    $x10 = COPY %3(p0)
+    $x10 = COPY %2(p0)
+    PseudoRET implicit $x10
+
+...
+---
+name:            ptrtoint
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+
+    ; RV64I-LABEL: name: ptrtoint
+    ; RV64I: liveins: $x10
+    ; RV64I-NEXT: {{  $}}
+    ; RV64I-NEXT: [[COPY:%[0-9]+]]:gprb(p0) = COPY $x10
+    ; RV64I-NEXT: [[PTRTOINT:%[0-9]+]]:gprb(s64) = G_PTRTOINT [[COPY]](p0)
+    ; RV64I-NEXT: $x10 = COPY [[PTRTOINT]](s64)
+    ; RV64I-NEXT: PseudoRET implicit $x10
+    %0:_(p0) = COPY $x10
+    %1:_(s64) = G_PTRTOINT %0(p0)
+    $x10 = COPY %1(s64)
+    PseudoRET implicit $x10
+
+...
+---
+name:            inttoprt
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+
+    ; RV64I-LABEL: name: inttoprt
+    ; RV64I: liveins: $x10
+    ; RV64I-NEXT: {{  $}}
+    ; RV64I-NEXT: [[COPY:%[0-9]+]]:gprb(s64) = COPY $x10
+    ; RV64I-NEXT: [[INTTOPTR:%[0-9]+]]:gprb(p0) = G_INTTOPTR [[COPY]](s64)
+    ; RV64I-NEXT: $x10 = COPY [[INTTOPTR]](p0)
+    ; RV64I-NEXT: PseudoRET implicit $x10
+    %0:_(s64) = COPY $x10
+    %1:_(p0) = G_INTTOPTR %0(s64)
+    $x10 = COPY %1(p0)
     PseudoRET implicit $x10
 
 ...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/frame-index.mir b/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/frame-index.mir
new file mode 100644
index 0000000000..f701ef6169
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/regbankselect/frame-index.mir
@@ -0,0 +1,33 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv32 -run-pass=regbankselect %s -o - \
+# RUN: | FileCheck %s
+# RUN: llc -mtriple=riscv64 -run-pass=regbankselect %s -o - \
+# RUN: | FileCheck %s
+--- |
+  define ptr @frame_index() {
+  entry:
+    %x = alloca i32, align 4
+    ret ptr %x
+  }
+
+...
+---
+name:            frame_index
+legalized:       true
+registers:
+  - { id: 0, class: _, preferred-register: '' }
+stack:
+  - { id: 0, name: x, type: default, offset: 0, size: 4, alignment: 4,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+body:             |
+  bb.1.entry:
+    ; CHECK-LABEL: name: frame_index
+    ; CHECK: [[FRAME_INDEX:%[0-9]+]]:gprb(p0) = G_FRAME_INDEX %stack.0.x
+    ; CHECK-NEXT: $x10 = COPY [[FRAME_INDEX]](p0)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:_(p0) = G_FRAME_INDEX %stack.0.x
+    $x10 = COPY %0(p0)
+    PseudoRET implicit $x10
+
+...
diff --git a/llvm/test/CodeGen/RISCV/O3-pipeline.ll b/llvm/test/CodeGen/RISCV/O3-pipeline.ll
index 277951782c..30b6e1e541 100644
--- a/llvm/test/CodeGen/RISCV/O3-pipeline.ll
+++ b/llvm/test/CodeGen/RISCV/O3-pipeline.ll
@@ -159,7 +159,7 @@
 ; CHECK-NEXT:       Insert KCFI indirect call checks
 ; CHECK-NEXT:       MachineDominator Tree Construction
 ; CHECK-NEXT:       Machine Natural Loop Construction
-; CHECK-NEXT:       Post RA top-down list latency scheduler
+; CHECK-NEXT:       PostRA Machine Instruction Scheduler
 ; CHECK-NEXT:       Analyze Machine Code For Garbage Collection
 ; CHECK-NEXT:       Machine Block Frequency Analysis
 ; CHECK-NEXT:       MachinePostDominator Tree Construction
diff --git a/llvm/test/CodeGen/RISCV/aext.ll b/llvm/test/CodeGen/RISCV/aext.ll
new file mode 100644
index 0000000000..bff654d4bd
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/aext.ll
@@ -0,0 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32I %s
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I %s
+
+define i24 @aext(i32 %0) {
+; RV32I-LABEL: aext:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    srli a0, a0, 8
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: aext:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    srliw a0, a0, 8
+; RV64I-NEXT:    ret
+  %2 = and i32 %0, -256
+  %3 = lshr exact i32 %2, 8
+  %4 = trunc i32 %3 to i24
+  ret i24 %4
+}
diff --git a/llvm/test/CodeGen/RISCV/macro-fusion-lui-addi.ll b/llvm/test/CodeGen/RISCV/macro-fusion-lui-addi.ll
index 7de0fd050d..498e6cf23b 100644
--- a/llvm/test/CodeGen/RISCV/macro-fusion-lui-addi.ll
+++ b/llvm/test/CodeGen/RISCV/macro-fusion-lui-addi.ll
@@ -3,6 +3,8 @@
 ;RUN:   | FileCheck %s --check-prefix=NOFUSION
 ;RUN: llc < %s -mtriple=riscv64 -mattr=+f,+lui-addi-fusion -mcpu=sifive-u74 \
 ;RUN:   -target-abi=lp64f | FileCheck %s --check-prefix=FUSION
+;RUN: llc < %s -mtriple=riscv64 -mattr=+f,+lui-addi-fusion,+use-postra-scheduler -mcpu=sifive-u74 \
+;RUN:   -target-abi=lp64f | FileCheck %s --check-prefixes=FUSION-POSTRA
 
 @.str = private constant [4 x i8] c"%f\0A\00", align 1
 
@@ -20,6 +22,13 @@ define void @foo(i32 signext %0, i32 signext %1) {
 ; FUSION-NEXT:    lui a0, %hi(.L.str)
 ; FUSION-NEXT:    addi a0, a0, %lo(.L.str)
 ; FUSION-NEXT:    tail bar@plt
+;
+; FUSION-POSTRA-LABEL: foo:
+; FUSION-POSTRA:       # %bb.0:
+; FUSION-POSTRA-NEXT:    fcvt.s.w fa0, a1
+; FUSION-POSTRA-NEXT:    lui a0, %hi(.L.str)
+; FUSION-POSTRA-NEXT:    addi a0, a0, %lo(.L.str)
+; FUSION-POSTRA-NEXT:    tail bar@plt
   %3 = sitofp i32 %1 to float
   tail call void @bar(ptr @.str, float %3)
   ret void
@@ -40,5 +49,11 @@ define i32 @test_matint() {
 ; FUSION-NEXT:    lui a0, 1
 ; FUSION-NEXT:    addiw a0, a0, -2048
 ; FUSION-NEXT:    ret
+;
+; FUSION-POSTRA-LABEL: test_matint:
+; FUSION-POSTRA:       # %bb.0:
+; FUSION-POSTRA-NEXT:    lui a0, 1
+; FUSION-POSTRA-NEXT:    addiw a0, a0, -2048
+; FUSION-POSTRA-NEXT:    ret
   ret i32 2048
 }
diff --git a/llvm/test/CodeGen/RISCV/opt-w-instrs.mir b/llvm/test/CodeGen/RISCV/opt-w-instrs.mir
new file mode 100644
index 0000000000..0ecf8fd6be
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/opt-w-instrs.mir
@@ -0,0 +1,30 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 3
+# RUN: llc -mtriple=riscv64 -mattr='+d,+zfa' -verify-machineinstrs -run-pass=riscv-opt-w-instrs %s -o - | FileCheck %s --check-prefix=CHECK-ZFA
+
+---
+name:            fcvtmod_w_d
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+
+    ; CHECK-ZFA-LABEL: name: fcvtmod_w_d
+    ; CHECK-ZFA: liveins: $x10, $x11
+    ; CHECK-ZFA-NEXT: {{  $}}
+    ; CHECK-ZFA-NEXT: [[COPY:%[0-9]+]]:fpr64 = COPY $x10
+    ; CHECK-ZFA-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; CHECK-ZFA-NEXT: [[FCVTMOD_W_D:%[0-9]+]]:gpr = nofpexcept FCVTMOD_W_D [[COPY]], 1
+    ; CHECK-ZFA-NEXT: [[ADD:%[0-9]+]]:gpr = ADD [[COPY1]], [[FCVTMOD_W_D]]
+    ; CHECK-ZFA-NEXT: $x10 = COPY [[ADD]]
+    ; CHECK-ZFA-NEXT: $x11 = COPY [[FCVTMOD_W_D]]
+    ; CHECK-ZFA-NEXT: PseudoRET
+    %0:fpr64 = COPY $x10
+    %1:gpr = COPY $x11
+
+    %2:gpr = nofpexcept FCVTMOD_W_D %0, 1
+    %3:gpr = ADD %1, %2
+    %4:gpr = ADDIW %2, 0
+    $x10 = COPY %3
+    $x11 = COPY %4
+    PseudoRET
+...
diff --git a/llvm/test/CodeGen/RISCV/pr66603.ll b/llvm/test/CodeGen/RISCV/pr66603.ll
new file mode 100644
index 0000000000..cfe8ceed12
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/pr66603.ll
@@ -0,0 +1,24 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
+; RUN: llc < %s -mtriple=riscv32-- -mattr=+d -verify-machineinstrs | FileCheck %s -check-prefix=RV32
+; RUN: llc < %s -mtriple=riscv64-- -mattr=+d -verify-machineinstrs | FileCheck %s -check-prefix=RV64
+
+; Don't fold freeze(assertsext(x)) -> assertsext(freeze(x))
+define i32 @PR66603(double %x) nounwind {
+; RV32-LABEL: PR66603:
+; RV32:       # %bb.0:
+; RV32-NEXT:    fcvt.w.d a0, fa0, rtz
+; RV32-NEXT:    slli a0, a0, 24
+; RV32-NEXT:    srai a0, a0, 24
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: PR66603:
+; RV64:       # %bb.0:
+; RV64-NEXT:    fcvt.l.d a0, fa0, rtz
+; RV64-NEXT:    slli a0, a0, 56
+; RV64-NEXT:    srai a0, a0, 56
+; RV64-NEXT:    ret
+  %as_i8 = fptosi double %x to i8
+  %frozen_i8 = freeze i8 %as_i8
+  %ext = sext i8 %frozen_i8 to i32
+  ret i32 %ext
+}
diff --git a/llvm/test/CodeGen/RISCV/pr68855.ll b/llvm/test/CodeGen/RISCV/pr68855.ll
new file mode 100644
index 0000000000..e9d1f6c2d1
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/pr68855.ll
@@ -0,0 +1,28 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
+; RUN: llc < %s -mtriple=riscv64 | FileCheck %s
+
+define i16 @narrow_load(ptr %p1, ptr %p2) {
+; CHECK-LABEL: narrow_load:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lhu a2, 0(a0)
+; CHECK-NEXT:    lui a3, 2
+; CHECK-NEXT:    addiw a3, a3, -1
+; CHECK-NEXT:    xor a2, a2, a3
+; CHECK-NEXT:    lui a4, 16
+; CHECK-NEXT:    addi a4, a4, -1
+; CHECK-NEXT:    xor a4, a3, a4
+; CHECK-NEXT:    or a2, a2, a4
+; CHECK-NEXT:    sw a2, 0(a1)
+; CHECK-NEXT:    lhu a0, 0(a0)
+; CHECK-NEXT:    and a0, a0, a3
+; CHECK-NEXT:    ret
+entry:
+  %bf.load = load i16, ptr %p1, align 2
+  %bf.clear = and i16 %bf.load, 8191
+  %not = xor i16 %bf.clear, -1
+  %conv1 = zext i16 %not to i32
+  store i32 %conv1, ptr %p2, align 4
+  %bf.load2 = load i16, ptr %p1, align 2
+  %bf.clear3 = and i16 %bf.load2, 8191
+  ret i16 %bf.clear3
+}
diff --git a/llvm/test/CodeGen/RISCV/prefetch.ll b/llvm/test/CodeGen/RISCV/prefetch.ll
index 39732636d2..7ef33f8aa1 100644
--- a/llvm/test/CodeGen/RISCV/prefetch.ll
+++ b/llvm/test/CodeGen/RISCV/prefetch.ll
@@ -5,7 +5,7 @@
 ; RUN:   | FileCheck -check-prefix=RV64I %s
 ; RUN: llc -mtriple=riscv32 -mattr=+zicbop -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32ZICBOP %s
-; RUN: llc -mtriple=riscv64 -mattr=zicbop -verify-machineinstrs < %s \
+; RUN: llc -mtriple=riscv64 -mattr=+zicbop -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV64ZICBOP %s
 ; RUN: llc -mtriple=riscv64 -mattr=+zicbop,+zihintntl -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV64ZICBOPZIHINTNTL %s
@@ -345,3 +345,1215 @@ define void @test_prefetch_instruction_locality_3(ptr %a) nounwind {
   call void @llvm.prefetch(ptr %a, i32 0, i32 3, i32 0)
   ret void
 }
+
+define void @test_prefetch_offsetable_0(ptr %a) nounwind {
+; RV32I-LABEL: test_prefetch_offsetable_0:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: test_prefetch_offsetable_0:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    ret
+;
+; RV32ZICBOP-LABEL: test_prefetch_offsetable_0:
+; RV32ZICBOP:       # %bb.0:
+; RV32ZICBOP-NEXT:    prefetch.r 2016(a0)
+; RV32ZICBOP-NEXT:    ret
+;
+; RV64ZICBOP-LABEL: test_prefetch_offsetable_0:
+; RV64ZICBOP:       # %bb.0:
+; RV64ZICBOP-NEXT:    prefetch.r 2016(a0)
+; RV64ZICBOP-NEXT:    ret
+;
+; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_offsetable_0:
+; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
+; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r 2016(a0)
+; RV64ZICBOPZIHINTNTL-NEXT:    ret
+  %addr = getelementptr i8, ptr %a, i64 2016
+  call void @llvm.prefetch(ptr %addr, i32 0, i32 0, i32 1)
+  ret void
+}
+
+define void @test_prefetch_offsetable_1(ptr %a) nounwind {
+; RV32I-LABEL: test_prefetch_offsetable_1:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: test_prefetch_offsetable_1:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    ret
+;
+; RV32ZICBOP-LABEL: test_prefetch_offsetable_1:
+; RV32ZICBOP:       # %bb.0:
+; RV32ZICBOP-NEXT:    prefetch.r -2048(a0)
+; RV32ZICBOP-NEXT:    ret
+;
+; RV64ZICBOP-LABEL: test_prefetch_offsetable_1:
+; RV64ZICBOP:       # %bb.0:
+; RV64ZICBOP-NEXT:    prefetch.r -2048(a0)
+; RV64ZICBOP-NEXT:    ret
+;
+; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_offsetable_1:
+; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
+; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r -2048(a0)
+; RV64ZICBOPZIHINTNTL-NEXT:    ret
+  %addr = getelementptr i8, ptr %a, i64 -2048
+  call void @llvm.prefetch(ptr %addr, i32 0, i32 0, i32 1)
+  ret void
+}
+
+define void @test_prefetch_offsetable_2(ptr %a) nounwind {
+; RV32I-LABEL: test_prefetch_offsetable_2:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: test_prefetch_offsetable_2:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    ret
+;
+; RV32ZICBOP-LABEL: test_prefetch_offsetable_2:
+; RV32ZICBOP:       # %bb.0:
+; RV32ZICBOP-NEXT:    prefetch.r 32(a0)
+; RV32ZICBOP-NEXT:    ret
+;
+; RV64ZICBOP-LABEL: test_prefetch_offsetable_2:
+; RV64ZICBOP:       # %bb.0:
+; RV64ZICBOP-NEXT:    prefetch.r 32(a0)
+; RV64ZICBOP-NEXT:    ret
+;
+; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_offsetable_2:
+; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
+; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r 32(a0)
+; RV64ZICBOPZIHINTNTL-NEXT:    ret
+  %addr = getelementptr i8, ptr %a, i64 32
+  call void @llvm.prefetch(ptr %addr, i32 0, i32 0, i32 1)
+  ret void
+}
+
+define void @test_prefetch_offsetable_3(ptr %a) nounwind {
+; RV32I-LABEL: test_prefetch_offsetable_3:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: test_prefetch_offsetable_3:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    ret
+;
+; RV32ZICBOP-LABEL: test_prefetch_offsetable_3:
+; RV32ZICBOP:       # %bb.0:
+; RV32ZICBOP-NEXT:    prefetch.r -32(a0)
+; RV32ZICBOP-NEXT:    ret
+;
+; RV64ZICBOP-LABEL: test_prefetch_offsetable_3:
+; RV64ZICBOP:       # %bb.0:
+; RV64ZICBOP-NEXT:    prefetch.r -32(a0)
+; RV64ZICBOP-NEXT:    ret
+;
+; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_offsetable_3:
+; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
+; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r -32(a0)
+; RV64ZICBOPZIHINTNTL-NEXT:    ret
+  %addr = getelementptr i8, ptr %a, i64 -32
+  call void @llvm.prefetch(ptr %addr, i32 0, i32 0, i32 1)
+  ret void
+}
+
+define void @test_prefetch_offsetable_4(ptr %a) nounwind {
+; RV32I-LABEL: test_prefetch_offsetable_4:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: test_prefetch_offsetable_4:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    ret
+;
+; RV32ZICBOP-LABEL: test_prefetch_offsetable_4:
+; RV32ZICBOP:       # %bb.0:
+; RV32ZICBOP-NEXT:    addi a0, a0, 32
+; RV32ZICBOP-NEXT:    prefetch.r 2016(a0)
+; RV32ZICBOP-NEXT:    ret
+;
+; RV64ZICBOP-LABEL: test_prefetch_offsetable_4:
+; RV64ZICBOP:       # %bb.0:
+; RV64ZICBOP-NEXT:    addi a0, a0, 32
+; RV64ZICBOP-NEXT:    prefetch.r 2016(a0)
+; RV64ZICBOP-NEXT:    ret
+;
+; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_offsetable_4:
+; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    addi a0, a0, 32
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
+; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r 2016(a0)
+; RV64ZICBOPZIHINTNTL-NEXT:    ret
+  %addr = getelementptr i8, ptr %a, i64 2048
+  call void @llvm.prefetch(ptr %addr, i32 0, i32 0, i32 1)
+  ret void
+}
+
+define void @test_prefetch_offsetable_5(ptr %a) nounwind {
+; RV32I-LABEL: test_prefetch_offsetable_5:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: test_prefetch_offsetable_5:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    ret
+;
+; RV32ZICBOP-LABEL: test_prefetch_offsetable_5:
+; RV32ZICBOP:       # %bb.0:
+; RV32ZICBOP-NEXT:    addi a0, a0, -1
+; RV32ZICBOP-NEXT:    prefetch.r -2048(a0)
+; RV32ZICBOP-NEXT:    ret
+;
+; RV64ZICBOP-LABEL: test_prefetch_offsetable_5:
+; RV64ZICBOP:       # %bb.0:
+; RV64ZICBOP-NEXT:    addi a0, a0, -1
+; RV64ZICBOP-NEXT:    prefetch.r -2048(a0)
+; RV64ZICBOP-NEXT:    ret
+;
+; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_offsetable_5:
+; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    addi a0, a0, -1
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
+; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r -2048(a0)
+; RV64ZICBOPZIHINTNTL-NEXT:    ret
+  %addr = getelementptr i8, ptr %a, i64 -2049
+  call void @llvm.prefetch(ptr %addr, i32 0, i32 0, i32 1)
+  ret void
+}
+
+define void @test_prefetch_offsetable_6(ptr %a) nounwind {
+; RV32I-LABEL: test_prefetch_offsetable_6:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: test_prefetch_offsetable_6:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    ret
+;
+; RV32ZICBOP-LABEL: test_prefetch_offsetable_6:
+; RV32ZICBOP:       # %bb.0:
+; RV32ZICBOP-NEXT:    addi a0, a0, 16
+; RV32ZICBOP-NEXT:    prefetch.r 0(a0)
+; RV32ZICBOP-NEXT:    ret
+;
+; RV64ZICBOP-LABEL: test_prefetch_offsetable_6:
+; RV64ZICBOP:       # %bb.0:
+; RV64ZICBOP-NEXT:    addi a0, a0, 16
+; RV64ZICBOP-NEXT:    prefetch.r 0(a0)
+; RV64ZICBOP-NEXT:    ret
+;
+; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_offsetable_6:
+; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    addi a0, a0, 16
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
+; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r 0(a0)
+; RV64ZICBOPZIHINTNTL-NEXT:    ret
+  %addr = getelementptr i8, ptr %a, i64 16
+  call void @llvm.prefetch(ptr %addr, i32 0, i32 0, i32 1)
+  ret void
+}
+
+define void @test_prefetch_offsetable_7(ptr %a) nounwind {
+; RV32I-LABEL: test_prefetch_offsetable_7:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: test_prefetch_offsetable_7:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    ret
+;
+; RV32ZICBOP-LABEL: test_prefetch_offsetable_7:
+; RV32ZICBOP:       # %bb.0:
+; RV32ZICBOP-NEXT:    addi a0, a0, -16
+; RV32ZICBOP-NEXT:    prefetch.r 0(a0)
+; RV32ZICBOP-NEXT:    ret
+;
+; RV64ZICBOP-LABEL: test_prefetch_offsetable_7:
+; RV64ZICBOP:       # %bb.0:
+; RV64ZICBOP-NEXT:    addi a0, a0, -16
+; RV64ZICBOP-NEXT:    prefetch.r 0(a0)
+; RV64ZICBOP-NEXT:    ret
+;
+; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_offsetable_7:
+; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    addi a0, a0, -16
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
+; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r 0(a0)
+; RV64ZICBOPZIHINTNTL-NEXT:    ret
+  %addr = getelementptr i8, ptr %a, i64 -16
+  call void @llvm.prefetch(ptr %addr, i32 0, i32 0, i32 1)
+  ret void
+}
+
+define void @test_prefetch_offsetable_9(ptr %a) nounwind {
+; RV32I-LABEL: test_prefetch_offsetable_9:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: test_prefetch_offsetable_9:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    ret
+;
+; RV32ZICBOP-LABEL: test_prefetch_offsetable_9:
+; RV32ZICBOP:       # %bb.0:
+; RV32ZICBOP-NEXT:    lui a1, 1
+; RV32ZICBOP-NEXT:    add a0, a0, a1
+; RV32ZICBOP-NEXT:    prefetch.r 64(a0)
+; RV32ZICBOP-NEXT:    ret
+;
+; RV64ZICBOP-LABEL: test_prefetch_offsetable_9:
+; RV64ZICBOP:       # %bb.0:
+; RV64ZICBOP-NEXT:    lui a1, 1
+; RV64ZICBOP-NEXT:    add a0, a0, a1
+; RV64ZICBOP-NEXT:    prefetch.r 64(a0)
+; RV64ZICBOP-NEXT:    ret
+;
+; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_offsetable_9:
+; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    lui a1, 1
+; RV64ZICBOPZIHINTNTL-NEXT:    add a0, a0, a1
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
+; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r 64(a0)
+; RV64ZICBOPZIHINTNTL-NEXT:    ret
+  %addr = getelementptr i8, ptr %a, i64 4160
+  call void @llvm.prefetch(ptr %addr, i32 0, i32 0, i32 1)
+  ret void
+}
+
+define void @test_prefetch_offsetable_8(ptr %a) nounwind {
+; RV32I-LABEL: test_prefetch_offsetable_8:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: test_prefetch_offsetable_8:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    ret
+;
+; RV32ZICBOP-LABEL: test_prefetch_offsetable_8:
+; RV32ZICBOP:       # %bb.0:
+; RV32ZICBOP-NEXT:    lui a1, 1048575
+; RV32ZICBOP-NEXT:    add a0, a0, a1
+; RV32ZICBOP-NEXT:    prefetch.r -64(a0)
+; RV32ZICBOP-NEXT:    ret
+;
+; RV64ZICBOP-LABEL: test_prefetch_offsetable_8:
+; RV64ZICBOP:       # %bb.0:
+; RV64ZICBOP-NEXT:    lui a1, 1048575
+; RV64ZICBOP-NEXT:    add a0, a0, a1
+; RV64ZICBOP-NEXT:    prefetch.r -64(a0)
+; RV64ZICBOP-NEXT:    ret
+;
+; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_offsetable_8:
+; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    lui a1, 1048575
+; RV64ZICBOPZIHINTNTL-NEXT:    add a0, a0, a1
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
+; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r -64(a0)
+; RV64ZICBOPZIHINTNTL-NEXT:    ret
+  %addr = getelementptr i8, ptr %a, i64 -4160
+  call void @llvm.prefetch(ptr %addr, i32 0, i32 0, i32 1)
+  ret void
+}
+
+define void @test_prefetch_frameindex_0() nounwind {
+; RV32I-LABEL: test_prefetch_frameindex_0:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -512
+; RV32I-NEXT:    addi sp, sp, 512
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: test_prefetch_frameindex_0:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -512
+; RV64I-NEXT:    addi sp, sp, 512
+; RV64I-NEXT:    ret
+;
+; RV32ZICBOP-LABEL: test_prefetch_frameindex_0:
+; RV32ZICBOP:       # %bb.0:
+; RV32ZICBOP-NEXT:    addi sp, sp, -512
+; RV32ZICBOP-NEXT:    prefetch.r 0(sp)
+; RV32ZICBOP-NEXT:    addi sp, sp, 512
+; RV32ZICBOP-NEXT:    ret
+;
+; RV64ZICBOP-LABEL: test_prefetch_frameindex_0:
+; RV64ZICBOP:       # %bb.0:
+; RV64ZICBOP-NEXT:    addi sp, sp, -512
+; RV64ZICBOP-NEXT:    prefetch.r 0(sp)
+; RV64ZICBOP-NEXT:    addi sp, sp, 512
+; RV64ZICBOP-NEXT:    ret
+;
+; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_frameindex_0:
+; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    addi sp, sp, -512
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
+; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r 0(sp)
+; RV64ZICBOPZIHINTNTL-NEXT:    addi sp, sp, 512
+; RV64ZICBOPZIHINTNTL-NEXT:    ret
+  %data = alloca [128 x i32], align 4
+  %base = bitcast [128 x i32]* %data to i8*
+  %ptr = getelementptr [128 x i32], ptr %base, i32 0, i32 0
+  call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+  ret void
+}
+
+define void @test_prefetch_frameindex_1() nounwind {
+; RV32I-LABEL: test_prefetch_frameindex_1:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a0, 1
+; RV32I-NEXT:    addi a0, a0, 16
+; RV32I-NEXT:    sub sp, sp, a0
+; RV32I-NEXT:    lui a0, 1
+; RV32I-NEXT:    addi a0, a0, 16
+; RV32I-NEXT:    add sp, sp, a0
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: test_prefetch_frameindex_1:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a0, 1
+; RV64I-NEXT:    addiw a0, a0, 16
+; RV64I-NEXT:    sub sp, sp, a0
+; RV64I-NEXT:    lui a0, 1
+; RV64I-NEXT:    addiw a0, a0, 16
+; RV64I-NEXT:    add sp, sp, a0
+; RV64I-NEXT:    ret
+;
+; RV32ZICBOP-LABEL: test_prefetch_frameindex_1:
+; RV32ZICBOP:       # %bb.0:
+; RV32ZICBOP-NEXT:    lui a0, 1
+; RV32ZICBOP-NEXT:    addi a0, a0, 16
+; RV32ZICBOP-NEXT:    sub sp, sp, a0
+; RV32ZICBOP-NEXT:    addi a0, sp, 16
+; RV32ZICBOP-NEXT:    prefetch.r 0(a0)
+; RV32ZICBOP-NEXT:    lui a0, 1
+; RV32ZICBOP-NEXT:    addi a0, a0, 16
+; RV32ZICBOP-NEXT:    add sp, sp, a0
+; RV32ZICBOP-NEXT:    ret
+;
+; RV64ZICBOP-LABEL: test_prefetch_frameindex_1:
+; RV64ZICBOP:       # %bb.0:
+; RV64ZICBOP-NEXT:    lui a0, 1
+; RV64ZICBOP-NEXT:    addiw a0, a0, 16
+; RV64ZICBOP-NEXT:    sub sp, sp, a0
+; RV64ZICBOP-NEXT:    addi a0, sp, 16
+; RV64ZICBOP-NEXT:    prefetch.r 0(a0)
+; RV64ZICBOP-NEXT:    lui a0, 1
+; RV64ZICBOP-NEXT:    addiw a0, a0, 16
+; RV64ZICBOP-NEXT:    add sp, sp, a0
+; RV64ZICBOP-NEXT:    ret
+;
+; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_frameindex_1:
+; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    lui a0, 1
+; RV64ZICBOPZIHINTNTL-NEXT:    addiw a0, a0, 16
+; RV64ZICBOPZIHINTNTL-NEXT:    sub sp, sp, a0
+; RV64ZICBOPZIHINTNTL-NEXT:    addi a0, sp, 16
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
+; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r 0(a0)
+; RV64ZICBOPZIHINTNTL-NEXT:    lui a0, 1
+; RV64ZICBOPZIHINTNTL-NEXT:    addiw a0, a0, 16
+; RV64ZICBOPZIHINTNTL-NEXT:    add sp, sp, a0
+; RV64ZICBOPZIHINTNTL-NEXT:    ret
+  %data = alloca [1024 x i32], align 4
+  %ptr = bitcast [1024 x i32]* %data to i8*
+  call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+  ret void
+}
+
+define void @test_prefetch_frameindex_2() nounwind {
+; RV32I-LABEL: test_prefetch_frameindex_2:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -512
+; RV32I-NEXT:    addi sp, sp, 512
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: test_prefetch_frameindex_2:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -512
+; RV64I-NEXT:    addi sp, sp, 512
+; RV64I-NEXT:    ret
+;
+; RV32ZICBOP-LABEL: test_prefetch_frameindex_2:
+; RV32ZICBOP:       # %bb.0:
+; RV32ZICBOP-NEXT:    addi sp, sp, -512
+; RV32ZICBOP-NEXT:    addi a0, sp, 16
+; RV32ZICBOP-NEXT:    prefetch.r 0(a0)
+; RV32ZICBOP-NEXT:    addi sp, sp, 512
+; RV32ZICBOP-NEXT:    ret
+;
+; RV64ZICBOP-LABEL: test_prefetch_frameindex_2:
+; RV64ZICBOP:       # %bb.0:
+; RV64ZICBOP-NEXT:    addi sp, sp, -512
+; RV64ZICBOP-NEXT:    addi a0, sp, 16
+; RV64ZICBOP-NEXT:    prefetch.r 0(a0)
+; RV64ZICBOP-NEXT:    addi sp, sp, 512
+; RV64ZICBOP-NEXT:    ret
+;
+; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_frameindex_2:
+; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    addi sp, sp, -512
+; RV64ZICBOPZIHINTNTL-NEXT:    addi a0, sp, 16
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
+; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r 0(a0)
+; RV64ZICBOPZIHINTNTL-NEXT:    addi sp, sp, 512
+; RV64ZICBOPZIHINTNTL-NEXT:    ret
+  %data = alloca [128 x i32], align 4
+  %base = bitcast [128 x i32]* %data to i8*
+  %ptr = getelementptr [128 x i32], ptr %base, i32 0, i32 4
+  call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+  ret void
+}
+
+define void @test_prefetch_frameindex_3() nounwind {
+; RV32I-LABEL: test_prefetch_frameindex_3:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -512
+; RV32I-NEXT:    addi sp, sp, 512
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: test_prefetch_frameindex_3:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -512
+; RV64I-NEXT:    addi sp, sp, 512
+; RV64I-NEXT:    ret
+;
+; RV32ZICBOP-LABEL: test_prefetch_frameindex_3:
+; RV32ZICBOP:       # %bb.0:
+; RV32ZICBOP-NEXT:    addi sp, sp, -512
+; RV32ZICBOP-NEXT:    addi a0, sp, -16
+; RV32ZICBOP-NEXT:    prefetch.r 0(a0)
+; RV32ZICBOP-NEXT:    addi sp, sp, 512
+; RV32ZICBOP-NEXT:    ret
+;
+; RV64ZICBOP-LABEL: test_prefetch_frameindex_3:
+; RV64ZICBOP:       # %bb.0:
+; RV64ZICBOP-NEXT:    addi sp, sp, -512
+; RV64ZICBOP-NEXT:    addi a0, sp, -16
+; RV64ZICBOP-NEXT:    prefetch.r 0(a0)
+; RV64ZICBOP-NEXT:    addi sp, sp, 512
+; RV64ZICBOP-NEXT:    ret
+;
+; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_frameindex_3:
+; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    addi sp, sp, -512
+; RV64ZICBOPZIHINTNTL-NEXT:    addi a0, sp, -16
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
+; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r 0(a0)
+; RV64ZICBOPZIHINTNTL-NEXT:    addi sp, sp, 512
+; RV64ZICBOPZIHINTNTL-NEXT:    ret
+  %data = alloca [128 x i32], align 4
+  %base = bitcast [128 x i32]* %data to i8*
+  %ptr = getelementptr [128 x i32], ptr %base, i32 0, i32 -4
+  call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+  ret void
+}
+
+define void @test_prefetch_frameindex_4() nounwind {
+; RV32I-LABEL: test_prefetch_frameindex_4:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -512
+; RV32I-NEXT:    addi sp, sp, 512
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: test_prefetch_frameindex_4:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -512
+; RV64I-NEXT:    addi sp, sp, 512
+; RV64I-NEXT:    ret
+;
+; RV32ZICBOP-LABEL: test_prefetch_frameindex_4:
+; RV32ZICBOP:       # %bb.0:
+; RV32ZICBOP-NEXT:    addi sp, sp, -512
+; RV32ZICBOP-NEXT:    prefetch.r 32(sp)
+; RV32ZICBOP-NEXT:    addi sp, sp, 512
+; RV32ZICBOP-NEXT:    ret
+;
+; RV64ZICBOP-LABEL: test_prefetch_frameindex_4:
+; RV64ZICBOP:       # %bb.0:
+; RV64ZICBOP-NEXT:    addi sp, sp, -512
+; RV64ZICBOP-NEXT:    prefetch.r 32(sp)
+; RV64ZICBOP-NEXT:    addi sp, sp, 512
+; RV64ZICBOP-NEXT:    ret
+;
+; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_frameindex_4:
+; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    addi sp, sp, -512
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
+; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r 32(sp)
+; RV64ZICBOPZIHINTNTL-NEXT:    addi sp, sp, 512
+; RV64ZICBOPZIHINTNTL-NEXT:    ret
+  %data = alloca [128 x i32], align 4
+  %base = bitcast [128 x i32]* %data to i8*
+  %ptr = getelementptr [128 x i32], ptr %base, i32 0, i32 8
+  call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+  ret void
+}
+
+define void @test_prefetch_frameindex_5() nounwind {
+; RV32I-LABEL: test_prefetch_frameindex_5:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -512
+; RV32I-NEXT:    addi sp, sp, 512
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: test_prefetch_frameindex_5:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -512
+; RV64I-NEXT:    addi sp, sp, 512
+; RV64I-NEXT:    ret
+;
+; RV32ZICBOP-LABEL: test_prefetch_frameindex_5:
+; RV32ZICBOP:       # %bb.0:
+; RV32ZICBOP-NEXT:    addi sp, sp, -512
+; RV32ZICBOP-NEXT:    prefetch.r -32(sp)
+; RV32ZICBOP-NEXT:    addi sp, sp, 512
+; RV32ZICBOP-NEXT:    ret
+;
+; RV64ZICBOP-LABEL: test_prefetch_frameindex_5:
+; RV64ZICBOP:       # %bb.0:
+; RV64ZICBOP-NEXT:    addi sp, sp, -512
+; RV64ZICBOP-NEXT:    prefetch.r -32(sp)
+; RV64ZICBOP-NEXT:    addi sp, sp, 512
+; RV64ZICBOP-NEXT:    ret
+;
+; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_frameindex_5:
+; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    addi sp, sp, -512
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
+; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r -32(sp)
+; RV64ZICBOPZIHINTNTL-NEXT:    addi sp, sp, 512
+; RV64ZICBOPZIHINTNTL-NEXT:    ret
+  %data = alloca [128 x i32], align 4
+  %base = bitcast [128 x i32]* %data to i8*
+  %ptr = getelementptr [128 x i32], ptr %base, i32 0, i32 -8
+  call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+  ret void
+}
+
+define void @test_prefetch_frameindex_6() nounwind {
+; RV32I-LABEL: test_prefetch_frameindex_6:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -512
+; RV32I-NEXT:    addi sp, sp, 512
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: test_prefetch_frameindex_6:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -512
+; RV64I-NEXT:    addi sp, sp, 512
+; RV64I-NEXT:    ret
+;
+; RV32ZICBOP-LABEL: test_prefetch_frameindex_6:
+; RV32ZICBOP:       # %bb.0:
+; RV32ZICBOP-NEXT:    addi sp, sp, -512
+; RV32ZICBOP-NEXT:    prefetch.r 2016(sp)
+; RV32ZICBOP-NEXT:    addi sp, sp, 512
+; RV32ZICBOP-NEXT:    ret
+;
+; RV64ZICBOP-LABEL: test_prefetch_frameindex_6:
+; RV64ZICBOP:       # %bb.0:
+; RV64ZICBOP-NEXT:    addi sp, sp, -512
+; RV64ZICBOP-NEXT:    prefetch.r 2016(sp)
+; RV64ZICBOP-NEXT:    addi sp, sp, 512
+; RV64ZICBOP-NEXT:    ret
+;
+; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_frameindex_6:
+; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    addi sp, sp, -512
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
+; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r 2016(sp)
+; RV64ZICBOPZIHINTNTL-NEXT:    addi sp, sp, 512
+; RV64ZICBOPZIHINTNTL-NEXT:    ret
+  %data = alloca [128 x i32], align 4
+  %base = bitcast [128 x i32]* %data to i8*
+  %ptr = getelementptr [128 x i32], ptr %base, i32 0, i32 504
+  call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+  ret void
+}
+
+define void @test_prefetch_frameindex_7() nounwind {
+; RV32I-LABEL: test_prefetch_frameindex_7:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -512
+; RV32I-NEXT:    addi sp, sp, 512
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: test_prefetch_frameindex_7:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -512
+; RV64I-NEXT:    addi sp, sp, 512
+; RV64I-NEXT:    ret
+;
+; RV32ZICBOP-LABEL: test_prefetch_frameindex_7:
+; RV32ZICBOP:       # %bb.0:
+; RV32ZICBOP-NEXT:    addi sp, sp, -512
+; RV32ZICBOP-NEXT:    prefetch.r -2048(sp)
+; RV32ZICBOP-NEXT:    addi sp, sp, 512
+; RV32ZICBOP-NEXT:    ret
+;
+; RV64ZICBOP-LABEL: test_prefetch_frameindex_7:
+; RV64ZICBOP:       # %bb.0:
+; RV64ZICBOP-NEXT:    addi sp, sp, -512
+; RV64ZICBOP-NEXT:    prefetch.r -2048(sp)
+; RV64ZICBOP-NEXT:    addi sp, sp, 512
+; RV64ZICBOP-NEXT:    ret
+;
+; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_frameindex_7:
+; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    addi sp, sp, -512
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
+; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r -2048(sp)
+; RV64ZICBOPZIHINTNTL-NEXT:    addi sp, sp, 512
+; RV64ZICBOPZIHINTNTL-NEXT:    ret
+  %data = alloca [128 x i32], align 4
+  %base = bitcast [128 x i32]* %data to i8*
+  %ptr = getelementptr [128 x i32], ptr %base, i32 0, i32 -512
+  call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+  ret void
+}
+
+define void @test_prefetch_frameindex_8() nounwind {
+; RV32I-LABEL: test_prefetch_frameindex_8:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -512
+; RV32I-NEXT:    addi sp, sp, 512
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: test_prefetch_frameindex_8:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -512
+; RV64I-NEXT:    addi sp, sp, 512
+; RV64I-NEXT:    ret
+;
+; RV32ZICBOP-LABEL: test_prefetch_frameindex_8:
+; RV32ZICBOP:       # %bb.0:
+; RV32ZICBOP-NEXT:    addi sp, sp, -512
+; RV32ZICBOP-NEXT:    addi a0, sp, 2020
+; RV32ZICBOP-NEXT:    prefetch.r 0(a0)
+; RV32ZICBOP-NEXT:    addi sp, sp, 512
+; RV32ZICBOP-NEXT:    ret
+;
+; RV64ZICBOP-LABEL: test_prefetch_frameindex_8:
+; RV64ZICBOP:       # %bb.0:
+; RV64ZICBOP-NEXT:    addi sp, sp, -512
+; RV64ZICBOP-NEXT:    addi a0, sp, 2020
+; RV64ZICBOP-NEXT:    prefetch.r 0(a0)
+; RV64ZICBOP-NEXT:    addi sp, sp, 512
+; RV64ZICBOP-NEXT:    ret
+;
+; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_frameindex_8:
+; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    addi sp, sp, -512
+; RV64ZICBOPZIHINTNTL-NEXT:    addi a0, sp, 2020
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
+; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r 0(a0)
+; RV64ZICBOPZIHINTNTL-NEXT:    addi sp, sp, 512
+; RV64ZICBOPZIHINTNTL-NEXT:    ret
+  %data = alloca [128 x i32], align 4
+  %base = bitcast [128 x i32]* %data to i8*
+  %ptr = getelementptr [128 x i32], ptr %base, i32 0, i32 505
+  call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+  ret void
+}
+
+define void @test_prefetch_frameindex_9() nounwind {
+; RV32I-LABEL: test_prefetch_frameindex_9:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -512
+; RV32I-NEXT:    addi sp, sp, 512
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: test_prefetch_frameindex_9:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -512
+; RV64I-NEXT:    addi sp, sp, 512
+; RV64I-NEXT:    ret
+;
+; RV32ZICBOP-LABEL: test_prefetch_frameindex_9:
+; RV32ZICBOP:       # %bb.0:
+; RV32ZICBOP-NEXT:    addi sp, sp, -512
+; RV32ZICBOP-NEXT:    mv a0, sp
+; RV32ZICBOP-NEXT:    addi a0, a0, -4
+; RV32ZICBOP-NEXT:    prefetch.r -2048(a0)
+; RV32ZICBOP-NEXT:    addi sp, sp, 512
+; RV32ZICBOP-NEXT:    ret
+;
+; RV64ZICBOP-LABEL: test_prefetch_frameindex_9:
+; RV64ZICBOP:       # %bb.0:
+; RV64ZICBOP-NEXT:    addi sp, sp, -512
+; RV64ZICBOP-NEXT:    mv a0, sp
+; RV64ZICBOP-NEXT:    addi a0, a0, -4
+; RV64ZICBOP-NEXT:    prefetch.r -2048(a0)
+; RV64ZICBOP-NEXT:    addi sp, sp, 512
+; RV64ZICBOP-NEXT:    ret
+;
+; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_frameindex_9:
+; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    addi sp, sp, -512
+; RV64ZICBOPZIHINTNTL-NEXT:    mv a0, sp
+; RV64ZICBOPZIHINTNTL-NEXT:    addi a0, a0, -4
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
+; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r -2048(a0)
+; RV64ZICBOPZIHINTNTL-NEXT:    addi sp, sp, 512
+; RV64ZICBOPZIHINTNTL-NEXT:    ret
+  %data = alloca [128 x i32], align 4
+  %base = bitcast [128 x i32]* %data to i8*
+  %ptr = getelementptr [128 x i32], ptr %base, i32 0, i32 -513
+  call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+  ret void
+}
+
+define void @test_prefetch_constant_address_0() nounwind {
+; RV32I-LABEL: test_prefetch_constant_address_0:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: test_prefetch_constant_address_0:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    ret
+;
+; RV32ZICBOP-LABEL: test_prefetch_constant_address_0:
+; RV32ZICBOP:       # %bb.0:
+; RV32ZICBOP-NEXT:    lui a0, 1
+; RV32ZICBOP-NEXT:    prefetch.r 32(a0)
+; RV32ZICBOP-NEXT:    ret
+;
+; RV64ZICBOP-LABEL: test_prefetch_constant_address_0:
+; RV64ZICBOP:       # %bb.0:
+; RV64ZICBOP-NEXT:    lui a0, 1
+; RV64ZICBOP-NEXT:    prefetch.r 32(a0)
+; RV64ZICBOP-NEXT:    ret
+;
+; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_constant_address_0:
+; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    lui a0, 1
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
+; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r 32(a0)
+; RV64ZICBOPZIHINTNTL-NEXT:    ret
+  %ptr = inttoptr i64 4128 to i8*
+  call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+  ret void
+}
+
+define void @test_prefetch_constant_address_1() nounwind {
+; RV32I-LABEL: test_prefetch_constant_address_1:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: test_prefetch_constant_address_1:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    ret
+;
+; RV32ZICBOP-LABEL: test_prefetch_constant_address_1:
+; RV32ZICBOP:       # %bb.0:
+; RV32ZICBOP-NEXT:    lui a0, 1
+; RV32ZICBOP-NEXT:    addi a0, a0, 31
+; RV32ZICBOP-NEXT:    prefetch.r 0(a0)
+; RV32ZICBOP-NEXT:    ret
+;
+; RV64ZICBOP-LABEL: test_prefetch_constant_address_1:
+; RV64ZICBOP:       # %bb.0:
+; RV64ZICBOP-NEXT:    lui a0, 1
+; RV64ZICBOP-NEXT:    addiw a0, a0, 31
+; RV64ZICBOP-NEXT:    prefetch.r 0(a0)
+; RV64ZICBOP-NEXT:    ret
+;
+; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_constant_address_1:
+; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    lui a0, 1
+; RV64ZICBOPZIHINTNTL-NEXT:    addiw a0, a0, 31
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
+; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r 0(a0)
+; RV64ZICBOPZIHINTNTL-NEXT:    ret
+  %ptr = inttoptr i64 4127 to i8*
+  call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+  ret void
+}
+
+define void @test_prefetch_constant_address_2() nounwind {
+; RV32I-LABEL: test_prefetch_constant_address_2:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: test_prefetch_constant_address_2:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    ret
+;
+; RV32ZICBOP-LABEL: test_prefetch_constant_address_2:
+; RV32ZICBOP:       # %bb.0:
+; RV32ZICBOP-NEXT:    lui a0, 1048561
+; RV32ZICBOP-NEXT:    prefetch.r 32(a0)
+; RV32ZICBOP-NEXT:    ret
+;
+; RV64ZICBOP-LABEL: test_prefetch_constant_address_2:
+; RV64ZICBOP:       # %bb.0:
+; RV64ZICBOP-NEXT:    lui a0, 1048561
+; RV64ZICBOP-NEXT:    prefetch.r 32(a0)
+; RV64ZICBOP-NEXT:    ret
+;
+; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_constant_address_2:
+; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    lui a0, 1048561
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
+; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r 32(a0)
+; RV64ZICBOPZIHINTNTL-NEXT:    ret
+  %ptr = inttoptr i64 18446744073709490208 to i8*
+  call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+  ret void
+}
+
+define void @test_prefetch_constant_address_3() nounwind {
+; RV32I-LABEL: test_prefetch_constant_address_3:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: test_prefetch_constant_address_3:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    ret
+;
+; RV32ZICBOP-LABEL: test_prefetch_constant_address_3:
+; RV32ZICBOP:       # %bb.0:
+; RV32ZICBOP-NEXT:    lui a0, 1048561
+; RV32ZICBOP-NEXT:    addi a0, a0, 31
+; RV32ZICBOP-NEXT:    prefetch.r 0(a0)
+; RV32ZICBOP-NEXT:    ret
+;
+; RV64ZICBOP-LABEL: test_prefetch_constant_address_3:
+; RV64ZICBOP:       # %bb.0:
+; RV64ZICBOP-NEXT:    lui a0, 1048561
+; RV64ZICBOP-NEXT:    addiw a0, a0, 31
+; RV64ZICBOP-NEXT:    prefetch.r 0(a0)
+; RV64ZICBOP-NEXT:    ret
+;
+; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_constant_address_3:
+; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    lui a0, 1048561
+; RV64ZICBOPZIHINTNTL-NEXT:    addiw a0, a0, 31
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
+; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r 0(a0)
+; RV64ZICBOPZIHINTNTL-NEXT:    ret
+  %ptr = inttoptr i64 18446744073709490207 to i8*
+  call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+  ret void
+}
+
+@g = external global [1024 x i32], align 4
+
+define void @test_prefetch_global_0() nounwind {
+; RV32I-LABEL: test_prefetch_global_0:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: test_prefetch_global_0:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    ret
+;
+; RV32ZICBOP-LABEL: test_prefetch_global_0:
+; RV32ZICBOP:       # %bb.0:
+; RV32ZICBOP-NEXT:    lui a0, %hi(g)
+; RV32ZICBOP-NEXT:    addi a0, a0, %lo(g)
+; RV32ZICBOP-NEXT:    prefetch.r 0(a0)
+; RV32ZICBOP-NEXT:    ret
+;
+; RV64ZICBOP-LABEL: test_prefetch_global_0:
+; RV64ZICBOP:       # %bb.0:
+; RV64ZICBOP-NEXT:    lui a0, %hi(g)
+; RV64ZICBOP-NEXT:    addi a0, a0, %lo(g)
+; RV64ZICBOP-NEXT:    prefetch.r 0(a0)
+; RV64ZICBOP-NEXT:    ret
+;
+; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_global_0:
+; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    lui a0, %hi(g)
+; RV64ZICBOPZIHINTNTL-NEXT:    addi a0, a0, %lo(g)
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
+; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r 0(a0)
+; RV64ZICBOPZIHINTNTL-NEXT:    ret
+  %ptr = getelementptr [1024 x i32], ptr @g, i32 0, i32 0
+  call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+  ret void
+}
+
+define void @test_prefetch_global_1() nounwind {
+; RV32I-LABEL: test_prefetch_global_1:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: test_prefetch_global_1:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    ret
+;
+; RV32ZICBOP-LABEL: test_prefetch_global_1:
+; RV32ZICBOP:       # %bb.0:
+; RV32ZICBOP-NEXT:    lui a0, %hi(g+16)
+; RV32ZICBOP-NEXT:    addi a0, a0, %lo(g+16)
+; RV32ZICBOP-NEXT:    prefetch.r 0(a0)
+; RV32ZICBOP-NEXT:    ret
+;
+; RV64ZICBOP-LABEL: test_prefetch_global_1:
+; RV64ZICBOP:       # %bb.0:
+; RV64ZICBOP-NEXT:    lui a0, %hi(g+16)
+; RV64ZICBOP-NEXT:    addi a0, a0, %lo(g+16)
+; RV64ZICBOP-NEXT:    prefetch.r 0(a0)
+; RV64ZICBOP-NEXT:    ret
+;
+; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_global_1:
+; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    lui a0, %hi(g+16)
+; RV64ZICBOPZIHINTNTL-NEXT:    addi a0, a0, %lo(g+16)
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
+; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r 0(a0)
+; RV64ZICBOPZIHINTNTL-NEXT:    ret
+  %ptr = getelementptr [1024 x i32], ptr @g, i32 0, i32 4
+  call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+  ret void
+}
+
+define void @test_prefetch_global_2() nounwind {
+; RV32I-LABEL: test_prefetch_global_2:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: test_prefetch_global_2:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    ret
+;
+; RV32ZICBOP-LABEL: test_prefetch_global_2:
+; RV32ZICBOP:       # %bb.0:
+; RV32ZICBOP-NEXT:    lui a0, %hi(g-16)
+; RV32ZICBOP-NEXT:    addi a0, a0, %lo(g-16)
+; RV32ZICBOP-NEXT:    prefetch.r 0(a0)
+; RV32ZICBOP-NEXT:    ret
+;
+; RV64ZICBOP-LABEL: test_prefetch_global_2:
+; RV64ZICBOP:       # %bb.0:
+; RV64ZICBOP-NEXT:    lui a0, %hi(g-16)
+; RV64ZICBOP-NEXT:    addi a0, a0, %lo(g-16)
+; RV64ZICBOP-NEXT:    prefetch.r 0(a0)
+; RV64ZICBOP-NEXT:    ret
+;
+; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_global_2:
+; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    lui a0, %hi(g-16)
+; RV64ZICBOPZIHINTNTL-NEXT:    addi a0, a0, %lo(g-16)
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
+; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r 0(a0)
+; RV64ZICBOPZIHINTNTL-NEXT:    ret
+  %ptr = getelementptr [1024 x i32], ptr @g, i32 0, i32 -4
+  call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+  ret void
+}
+
+define void @test_prefetch_global_3() nounwind {
+; RV32I-LABEL: test_prefetch_global_3:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: test_prefetch_global_3:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    ret
+;
+; RV32ZICBOP-LABEL: test_prefetch_global_3:
+; RV32ZICBOP:       # %bb.0:
+; RV32ZICBOP-NEXT:    lui a0, %hi(g)
+; RV32ZICBOP-NEXT:    addi a0, a0, %lo(g)
+; RV32ZICBOP-NEXT:    prefetch.r 32(a0)
+; RV32ZICBOP-NEXT:    ret
+;
+; RV64ZICBOP-LABEL: test_prefetch_global_3:
+; RV64ZICBOP:       # %bb.0:
+; RV64ZICBOP-NEXT:    lui a0, %hi(g)
+; RV64ZICBOP-NEXT:    addi a0, a0, %lo(g)
+; RV64ZICBOP-NEXT:    prefetch.r 32(a0)
+; RV64ZICBOP-NEXT:    ret
+;
+; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_global_3:
+; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    lui a0, %hi(g)
+; RV64ZICBOPZIHINTNTL-NEXT:    addi a0, a0, %lo(g)
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
+; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r 32(a0)
+; RV64ZICBOPZIHINTNTL-NEXT:    ret
+  %ptr = getelementptr [1024 x i32], ptr @g, i32 0, i32 8
+  call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+  ret void
+}
+
+define void @test_prefetch_global_4() nounwind {
+; RV32I-LABEL: test_prefetch_global_4:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: test_prefetch_global_4:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    ret
+;
+; RV32ZICBOP-LABEL: test_prefetch_global_4:
+; RV32ZICBOP:       # %bb.0:
+; RV32ZICBOP-NEXT:    lui a0, %hi(g)
+; RV32ZICBOP-NEXT:    addi a0, a0, %lo(g)
+; RV32ZICBOP-NEXT:    prefetch.r -32(a0)
+; RV32ZICBOP-NEXT:    ret
+;
+; RV64ZICBOP-LABEL: test_prefetch_global_4:
+; RV64ZICBOP:       # %bb.0:
+; RV64ZICBOP-NEXT:    lui a0, %hi(g)
+; RV64ZICBOP-NEXT:    addi a0, a0, %lo(g)
+; RV64ZICBOP-NEXT:    prefetch.r -32(a0)
+; RV64ZICBOP-NEXT:    ret
+;
+; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_global_4:
+; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    lui a0, %hi(g)
+; RV64ZICBOPZIHINTNTL-NEXT:    addi a0, a0, %lo(g)
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
+; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r -32(a0)
+; RV64ZICBOPZIHINTNTL-NEXT:    ret
+  %ptr = getelementptr [1024 x i32], ptr @g, i32 0, i32 -8
+  call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+  ret void
+}
+
+define void @test_prefetch_global_5() nounwind {
+; RV32I-LABEL: test_prefetch_global_5:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: test_prefetch_global_5:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    ret
+;
+; RV32ZICBOP-LABEL: test_prefetch_global_5:
+; RV32ZICBOP:       # %bb.0:
+; RV32ZICBOP-NEXT:    lui a0, %hi(g)
+; RV32ZICBOP-NEXT:    addi a0, a0, %lo(g)
+; RV32ZICBOP-NEXT:    prefetch.r 2016(a0)
+; RV32ZICBOP-NEXT:    ret
+;
+; RV64ZICBOP-LABEL: test_prefetch_global_5:
+; RV64ZICBOP:       # %bb.0:
+; RV64ZICBOP-NEXT:    lui a0, %hi(g)
+; RV64ZICBOP-NEXT:    addi a0, a0, %lo(g)
+; RV64ZICBOP-NEXT:    prefetch.r 2016(a0)
+; RV64ZICBOP-NEXT:    ret
+;
+; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_global_5:
+; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    lui a0, %hi(g)
+; RV64ZICBOPZIHINTNTL-NEXT:    addi a0, a0, %lo(g)
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
+; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r 2016(a0)
+; RV64ZICBOPZIHINTNTL-NEXT:    ret
+  %ptr = getelementptr [1024 x i32], ptr @g, i32 0, i32 504
+  call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+  ret void
+}
+
+define void @test_prefetch_global_6() nounwind {
+; RV32I-LABEL: test_prefetch_global_6:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: test_prefetch_global_6:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    ret
+;
+; RV32ZICBOP-LABEL: test_prefetch_global_6:
+; RV32ZICBOP:       # %bb.0:
+; RV32ZICBOP-NEXT:    lui a0, %hi(g)
+; RV32ZICBOP-NEXT:    addi a0, a0, %lo(g)
+; RV32ZICBOP-NEXT:    prefetch.r -2048(a0)
+; RV32ZICBOP-NEXT:    ret
+;
+; RV64ZICBOP-LABEL: test_prefetch_global_6:
+; RV64ZICBOP:       # %bb.0:
+; RV64ZICBOP-NEXT:    lui a0, %hi(g)
+; RV64ZICBOP-NEXT:    addi a0, a0, %lo(g)
+; RV64ZICBOP-NEXT:    prefetch.r -2048(a0)
+; RV64ZICBOP-NEXT:    ret
+;
+; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_global_6:
+; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    lui a0, %hi(g)
+; RV64ZICBOPZIHINTNTL-NEXT:    addi a0, a0, %lo(g)
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
+; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r -2048(a0)
+; RV64ZICBOPZIHINTNTL-NEXT:    ret
+  %ptr = getelementptr [1024 x i32], ptr @g, i32 0, i32 -512
+  call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+  ret void
+}
+
+define void @test_prefetch_global_7() nounwind {
+; RV32I-LABEL: test_prefetch_global_7:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: test_prefetch_global_7:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    ret
+;
+; RV32ZICBOP-LABEL: test_prefetch_global_7:
+; RV32ZICBOP:       # %bb.0:
+; RV32ZICBOP-NEXT:    lui a0, %hi(g+2020)
+; RV32ZICBOP-NEXT:    addi a0, a0, %lo(g+2020)
+; RV32ZICBOP-NEXT:    prefetch.r 0(a0)
+; RV32ZICBOP-NEXT:    ret
+;
+; RV64ZICBOP-LABEL: test_prefetch_global_7:
+; RV64ZICBOP:       # %bb.0:
+; RV64ZICBOP-NEXT:    lui a0, %hi(g+2020)
+; RV64ZICBOP-NEXT:    addi a0, a0, %lo(g+2020)
+; RV64ZICBOP-NEXT:    prefetch.r 0(a0)
+; RV64ZICBOP-NEXT:    ret
+;
+; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_global_7:
+; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    lui a0, %hi(g+2020)
+; RV64ZICBOPZIHINTNTL-NEXT:    addi a0, a0, %lo(g+2020)
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
+; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r 0(a0)
+; RV64ZICBOPZIHINTNTL-NEXT:    ret
+  %ptr = getelementptr [1024 x i32], ptr @g, i32 0, i32 505
+  call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+  ret void
+}
+
+define void @test_prefetch_global_8() nounwind {
+; RV32I-LABEL: test_prefetch_global_8:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: test_prefetch_global_8:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    ret
+;
+; RV32ZICBOP-LABEL: test_prefetch_global_8:
+; RV32ZICBOP:       # %bb.0:
+; RV32ZICBOP-NEXT:    lui a0, %hi(g-4)
+; RV32ZICBOP-NEXT:    addi a0, a0, %lo(g-4)
+; RV32ZICBOP-NEXT:    prefetch.r -2048(a0)
+; RV32ZICBOP-NEXT:    ret
+;
+; RV64ZICBOP-LABEL: test_prefetch_global_8:
+; RV64ZICBOP:       # %bb.0:
+; RV64ZICBOP-NEXT:    lui a0, %hi(g-4)
+; RV64ZICBOP-NEXT:    addi a0, a0, %lo(g-4)
+; RV64ZICBOP-NEXT:    prefetch.r -2048(a0)
+; RV64ZICBOP-NEXT:    ret
+;
+; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_global_8:
+; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    lui a0, %hi(g-4)
+; RV64ZICBOPZIHINTNTL-NEXT:    addi a0, a0, %lo(g-4)
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
+; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r -2048(a0)
+; RV64ZICBOPZIHINTNTL-NEXT:    ret
+  %ptr = getelementptr [1024 x i32], ptr @g, i32 0, i32 -513
+  call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/riscv-shifted-extend.ll b/llvm/test/CodeGen/RISCV/riscv-shifted-extend.ll
new file mode 100644
index 0000000000..957f44f9f6
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/riscv-shifted-extend.ll
@@ -0,0 +1,124 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64 %s
+
+define void @test(ptr nocapture noundef writeonly %array1, i32 noundef signext %a, i32 noundef signext %b) {
+; RV64-LABEL: test:
+; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    addiw a3, a1, 5
+; RV64-NEXT:    slli a4, a3, 2
+; RV64-NEXT:    add a4, a0, a4
+; RV64-NEXT:    sw a2, 0(a4)
+; RV64-NEXT:    slli a1, a1, 2
+; RV64-NEXT:    add a0, a1, a0
+; RV64-NEXT:    sw a2, 24(a0)
+; RV64-NEXT:    sw a3, 140(a0)
+; RV64-NEXT:    ret
+entry:
+  %add = add nsw i32 %a, 5
+  %idxprom = sext i32 %add to i64
+  %arrayidx = getelementptr inbounds i32, ptr %array1, i64 %idxprom
+  store i32 %b, ptr %arrayidx, align 4
+  %add3 = add nsw i32 %a, 6
+  %idxprom4 = sext i32 %add3 to i64
+  %arrayidx5 = getelementptr inbounds i32, ptr %array1, i64 %idxprom4
+  store i32 %b, ptr %arrayidx5, align 4
+  %add6 = add nsw i32 %a, 35
+  %idxprom7 = sext i32 %add6 to i64
+  %arrayidx8 = getelementptr inbounds i32, ptr %array1, i64 %idxprom7
+  store i32 %add, ptr %arrayidx8, align 4
+  ret void
+}
+
+; test of jumpping, find add's operand has one more use can simplified
+define void @test1(ptr nocapture noundef %array1, i32 noundef signext %a, i32 noundef signext %b, i32 noundef signext %x) {
+; RV64-LABEL: test1:
+; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    addiw a4, a1, 5
+; RV64-NEXT:    slli a5, a4, 2
+; RV64-NEXT:    add a5, a0, a5
+; RV64-NEXT:    mv a6, a4
+; RV64-NEXT:    bgtz a3, .LBB1_2
+; RV64-NEXT:  # %bb.1: # %entry
+; RV64-NEXT:    mv a6, a2
+; RV64-NEXT:  .LBB1_2: # %entry
+; RV64-NEXT:    sw a6, 0(a5)
+; RV64-NEXT:    slli a1, a1, 2
+; RV64-NEXT:    add a0, a1, a0
+; RV64-NEXT:    sw a6, 24(a0)
+; RV64-NEXT:    sw a4, 140(a0)
+; RV64-NEXT:    ret
+entry:
+  %add = add nsw i32 %a, 5
+  %cmp = icmp sgt i32 %x, 0
+  %idxprom = sext i32 %add to i64
+  %arrayidx = getelementptr inbounds i32, ptr %array1, i64 %idxprom
+  %add.b = select i1 %cmp, i32 %add, i32 %b
+  store i32 %add.b, ptr %arrayidx, align 4
+  %add5 = add nsw i32 %a, 6
+  %idxprom6 = sext i32 %add5 to i64
+  %arrayidx7 = getelementptr inbounds i32, ptr %array1, i64 %idxprom6
+  store i32 %add.b, ptr %arrayidx7, align 4
+  %add8 = add nsw i32 %a, 35
+  %idxprom9 = sext i32 %add8 to i64
+  %arrayidx10 = getelementptr inbounds i32, ptr %array1, i64 %idxprom9
+  store i32 %add, ptr %arrayidx10, align 4
+  ret void
+}
+
+define void @test2(ptr nocapture noundef writeonly %array1, i64 noundef %a, i64 noundef %b) local_unnamed_addr #0 {
+; RV64-LABEL: test2:
+; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    addi a3, a1, 5
+; RV64-NEXT:    slli a4, a3, 3
+; RV64-NEXT:    add a4, a0, a4
+; RV64-NEXT:    sd a2, 0(a4)
+; RV64-NEXT:    slli a1, a1, 3
+; RV64-NEXT:    add a0, a1, a0
+; RV64-NEXT:    sd a2, 48(a0)
+; RV64-NEXT:    sd a3, 280(a0)
+; RV64-NEXT:    ret
+entry:
+  %add = add nsw i64 %a, 5
+  %arrayidx = getelementptr inbounds i64, ptr %array1, i64 %add
+  store i64 %b, ptr %arrayidx, align 8
+  %add2 = add nsw i64 %a, 6
+  %arrayidx3 = getelementptr inbounds i64, ptr %array1, i64 %add2
+  store i64 %b, ptr %arrayidx3, align 8
+  %add4 = add nsw i64 %a, 35
+  %arrayidx5 = getelementptr inbounds i64, ptr %array1, i64 %add4
+  store i64 %add, ptr %arrayidx5, align 8
+  ret void
+}
+
+define void @test3(ptr nocapture noundef %array1, i64 noundef %a, i64 noundef %b, i64 noundef %x) {
+; RV64-LABEL: test3:
+; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    addi a4, a1, 5
+; RV64-NEXT:    mv a5, a4
+; RV64-NEXT:    bgtz a3, .LBB3_2
+; RV64-NEXT:  # %bb.1: # %entry
+; RV64-NEXT:    mv a5, a2
+; RV64-NEXT:  .LBB3_2: # %entry
+; RV64-NEXT:    slli a2, a4, 3
+; RV64-NEXT:    add a2, a0, a2
+; RV64-NEXT:    sd a5, 0(a2)
+; RV64-NEXT:    slli a1, a1, 3
+; RV64-NEXT:    add a0, a1, a0
+; RV64-NEXT:    sd a5, 48(a0)
+; RV64-NEXT:    sd a4, 280(a0)
+; RV64-NEXT:    ret
+entry:
+  %add = add nsw i64 %a, 5
+  %cmp = icmp sgt i64 %x, 0
+  %spec.select = select i1 %cmp, i64 %add, i64 %b
+  %0 = getelementptr inbounds i64, ptr %array1, i64 %add
+  store i64 %spec.select, ptr %0, align 8
+  %add3 = add nsw i64 %a, 6
+  %arrayidx4 = getelementptr inbounds i64, ptr %array1, i64 %add3
+  store i64 %spec.select, ptr %arrayidx4, align 8
+  %add5 = add nsw i64 %a, 35
+  %arrayidx6 = getelementptr inbounds i64, ptr %array1, i64 %add5
+  store i64 %add, ptr %arrayidx6, align 8
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/rv64-patchpoint.ll b/llvm/test/CodeGen/RISCV/rv64-patchpoint.ll
new file mode 100644
index 0000000000..3fbd13f1f2
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rv64-patchpoint.ll
@@ -0,0 +1,24 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv64 -debug-entry-values -enable-misched=0 < %s | FileCheck %s
+
+; Test small patchpoints that don't emit calls.
+define void @small_patchpoint_codegen(i64 %p1, i64 %p2, i64 %p3, i64 %p4) {
+; CHECK-LABEL: small_patchpoint_codegen:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:  .Ltmp0:
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    ret
+entry:
+  %result = tail call i64 (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.i64(i64 5, i32 20, i8* null, i32 2, i64 %p1, i64 %p2)
+  ret void
+}
+
+declare void @llvm.experimental.stackmap(i64, i32, ...)
+declare void @llvm.experimental.patchpoint.void(i64, i32, i8*, i32, ...)
+declare i64 @llvm.experimental.patchpoint.i64(i64, i32, i8*, i32, ...)
+
diff --git a/llvm/test/CodeGen/RISCV/rv64-stackmap-frame-setup.ll b/llvm/test/CodeGen/RISCV/rv64-stackmap-frame-setup.ll
new file mode 100644
index 0000000000..e1bed39a50
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rv64-stackmap-frame-setup.ll
@@ -0,0 +1,22 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -o - -verify-machineinstrs -mtriple=riscv64 -stop-after machine-sink %s | FileCheck %s --check-prefix=ISEL
+
+define void @caller_meta_leaf() {
+  ; ISEL-LABEL: name: caller_meta_leaf
+  ; ISEL: bb.0.entry:
+  ; ISEL-NEXT:   [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 13
+  ; ISEL-NEXT:   SD killed [[ADDI]], %stack.0.metadata, 0 :: (store (s64) into %ir.metadata)
+  ; ISEL-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2
+  ; ISEL-NEXT:   STACKMAP 4, 0, 0, %stack.0.metadata, 0 :: (load (s64) from %stack.0.metadata)
+  ; ISEL-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $x2, implicit $x2
+  ; ISEL-NEXT:   PseudoRET
+entry:
+  %metadata = alloca i64, i32 3, align 8
+  store i64 11, i64* %metadata
+  store i64 12, i64* %metadata
+  store i64 13, i64* %metadata
+  call void (i64, i32, ...) @llvm.experimental.stackmap(i64 4, i32 0, i64* %metadata)
+  ret void
+}
+
+declare void @llvm.experimental.stackmap(i64, i32, ...)
diff --git a/llvm/test/CodeGen/RISCV/rv64-stackmap-nops.ll b/llvm/test/CodeGen/RISCV/rv64-stackmap-nops.ll
new file mode 100644
index 0000000000..6edc88de92
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rv64-stackmap-nops.ll
@@ -0,0 +1,18 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=riscv64 | FileCheck %s
+
+define void @test_shadow_optimization() {
+; CHECK-LABEL: test_shadow_optimization:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:  .Ltmp0:
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    ret
+entry:
+  tail call void (i64, i32, ...) @llvm.experimental.stackmap(i64  0, i32  16)
+  ret void
+}
+
+declare void @llvm.experimental.stackmap(i64, i32, ...)
diff --git a/llvm/test/CodeGen/RISCV/rv64-stackmap.ll b/llvm/test/CodeGen/RISCV/rv64-stackmap.ll
new file mode 100644
index 0000000000..e1dde97337
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rv64-stackmap.ll
@@ -0,0 +1,384 @@
+; RUN: llc -mtriple=riscv64 < %s | FileCheck %s
+
+; CHECK-LABEL:  .section	.llvm_stackmaps
+; CHECK-NEXT:  __LLVM_StackMaps:
+; Header
+; CHECK-NEXT:   .byte   3
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .half   0
+; Num Functions
+; CHECK-NEXT:   .word   12
+; Num LargeConstants
+; CHECK-NEXT:   .word   2
+; Num Callsites
+; CHECK-NEXT:   .word   16
+
+; Functions and stack size
+; CHECK-NEXT:   .quad   constantargs
+; CHECK-NEXT:   .quad   0
+; CHECK-NEXT:   .quad   1
+; CHECK-NEXT:   .quad   osrinline
+; CHECK-NEXT:   .quad   32
+; CHECK-NEXT:   .quad   1
+; CHECK-NEXT:   .quad   osrcold
+; CHECK-NEXT:   .quad   0
+; CHECK-NEXT:   .quad   1
+; CHECK-NEXT:   .quad   propertyRead
+; CHECK-NEXT:   .quad   16
+; CHECK-NEXT:   .quad   1
+; CHECK-NEXT:   .quad   propertyWrite
+; CHECK-NEXT:   .quad   0
+; CHECK-NEXT:   .quad   1
+; CHECK-NEXT:   .quad   jsVoidCall
+; CHECK-NEXT:   .quad   0
+; CHECK-NEXT:   .quad   1
+; CHECK-NEXT:   .quad   jsIntCall
+; CHECK-NEXT:   .quad   0
+; CHECK-NEXT:   .quad   1
+; CHECK-NEXT:   .quad   liveConstant
+; CHECK-NEXT:   .quad   0
+; CHECK-NEXT:   .quad   1
+; CHECK-NEXT:   .quad   spilledValue
+; CHECK-NEXT:   .quad   144
+; CHECK-NEXT:   .quad   1
+; CHECK-NEXT:   .quad   directFrameIdx
+; CHECK-NEXT:   .quad   48
+; CHECK-NEXT:   .quad   2
+; CHECK-NEXT:   .quad   longid
+; CHECK-NEXT:   .quad   0
+; CHECK-NEXT:   .quad   4
+; CHECK-NEXT:   .quad   needsStackRealignment
+; CHECK-NEXT:   .quad   -1
+; CHECK-NEXT:   .quad   1
+
+; Num LargeConstants
+; CHECK-NEXT:   .quad   4294967295
+; CHECK-NEXT:   .quad   4294967296
+
+; Constant arguments
+;
+; CHECK-NEXT:   .quad   1
+; CHECK-NEXT:   .word   .L{{.*}}-constantargs
+; CHECK-NEXT:   .half   0
+; CHECK-NEXT:   .half   4
+; SmallConstant
+; CHECK-NEXT:   .byte   4
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .half   8
+; CHECK-NEXT:   .half   0
+; CHECK-NEXT:   .half   0
+; CHECK-NEXT:   .word   65535
+; SmallConstant
+; CHECK-NEXT:   .byte   4
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .half   8
+; CHECK-NEXT:   .half   0
+; CHECK-NEXT:   .half   0
+; CHECK-NEXT:   .word   65536
+; SmallConstant
+; CHECK-NEXT:   .byte   5
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .half   8
+; CHECK-NEXT:   .half   0
+; CHECK-NEXT:   .half   0
+; CHECK-NEXT:   .word   0
+; LargeConstant at index 0
+; CHECK-NEXT:   .byte   5
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .half   8
+; CHECK-NEXT:   .half   0
+; CHECK-NEXT:   .half   0
+; CHECK-NEXT:   .word   1
+
+define void @constantargs() {
+entry:
+  %0 = inttoptr i64 244837814094590 to i8*
+  tail call void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 1, i32 28, i8* %0, i32 0, i64 65535, i64 65536, i64 4294967295, i64 4294967296)
+  ret void
+}
+
+; Inline OSR Exit
+;
+; CHECK:        .word   .L{{.*}}-osrinline
+; CHECK-NEXT:   .half   0
+; CHECK-NEXT:   .half   2
+; CHECK-NEXT:   .byte   1
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .half   8
+; CHECK-NEXT:   .half   {{[0-9]+}}
+; CHECK-NEXT:   .half   0
+; CHECK-NEXT:   .word   0
+; CHECK-NEXT:   .byte   1
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .half   8
+; CHECK-NEXT:   .half   {{[0-9]+}}
+; CHECK-NEXT:   .half   0
+; CHECK-NEXT:   .word   0
+define void @osrinline(i64 %a, i64 %b) {
+entry:
+  ; Runtime void->void call.
+  call void inttoptr (i64 244837814094590 to void ()*)()
+  ; Followed by inline OSR patchpoint with 12-byte shadow and 2 live vars.
+  call void (i64, i32, ...) @llvm.experimental.stackmap(i64 3, i32 12, i64 %a, i64 %b)
+  ret void
+}
+
+; Cold OSR Exit
+;
+; 2 live variables in register.
+;
+; CHECK:        .word   .L{{.*}}-osrcold
+; CHECK-NEXT:   .half   0
+; CHECK-NEXT:   .half   2
+; CHECK-NEXT:   .byte   1
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .half   8
+; CHECK-NEXT:   .half   {{[0-9]+}}
+; CHECK-NEXT:   .half   0
+; CHECK-NEXT:   .word   0
+; CHECK-NEXT:   .byte   1
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .half   8
+; CHECK-NEXT:   .half   {{[0-9]+}}
+; CHECK-NEXT:   .half   0
+; CHECK-NEXT:   .word   0
+define void @osrcold(i64 %a, i64 %b) {
+entry:
+  %test = icmp slt i64 %a, %b
+  br i1 %test, label %ret, label %cold
+cold:
+  ; OSR patchpoint with 28-byte nop-slide and 2 live vars.
+  %thunk = inttoptr i64 244837814094590 to i8*
+  call void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 4, i32 28, i8* %thunk, i32 0, i64 %a, i64 %b)
+  unreachable
+ret:
+  ret void
+}
+
+; Property Read
+; CHECK-LABEL:  .word   .L{{.*}}-propertyRead
+; CHECK-NEXT:   .half   0
+; CHECK-NEXT:   .half   2
+; CHECK-NEXT:   .byte   1
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .half   8
+; CHECK-NEXT:   .half   {{[0-9]+}}
+; CHECK-NEXT:   .half   0
+; CHECK-NEXT:   .word   0
+; CHECK-NEXT:   .byte   1
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .half   8
+; CHECK-NEXT:   .half   {{[0-9]+}}
+; CHECK-NEXT:   .half   0
+; CHECK-NEXT:   .word   0
+define i64 @propertyRead(i64* %obj) {
+entry:
+  %resolveRead = inttoptr i64 244837814094590 to i8*
+  %result = call anyregcc i64 (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.i64(i64 5, i32 28, i8* %resolveRead, i32 1, i64* %obj)
+  %add = add i64 %result, 3
+  ret i64 %add
+}
+
+; Property Write
+; CHECK:        .word   .L{{.*}}-propertyWrite
+; CHECK-NEXT:   .half   0
+; CHECK-NEXT:   .half   2
+; CHECK-NEXT:   .byte   1
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .half   8
+; CHECK-NEXT:   .half   {{[0-9]+}}
+; CHECK-NEXT:   .half   0
+; CHECK-NEXT:   .word   0
+; CHECK-NEXT:   .byte   1
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .half   8
+; CHECK-NEXT:   .half   {{[0-9]+}}
+; CHECK-NEXT:   .half   0
+; CHECK-NEXT:   .word   0
+define void @propertyWrite(i64 %dummy1, i64* %obj, i64 %dummy2, i64 %a) {
+entry:
+  %resolveWrite = inttoptr i64 244837814094590 to i8*
+  call anyregcc void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 6, i32 28, i8* %resolveWrite, i32 2, i64* %obj, i64 %a)
+  ret void
+}
+
+; Void JS Call
+;
+; 2 live variables in registers.
+;
+; CHECK:        .word   .L{{.*}}-jsVoidCall
+; CHECK-NEXT:   .half   0
+; CHECK-NEXT:   .half   2
+; CHECK-NEXT:   .byte   1
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .half   8
+; CHECK-NEXT:   .half   {{[0-9]+}}
+; CHECK-NEXT:   .half   0
+; CHECK-NEXT:   .word   0
+; CHECK-NEXT:   .byte   1
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .half   8
+; CHECK-NEXT:   .half   {{[0-9]+}}
+; CHECK-NEXT:   .half   0
+; CHECK-NEXT:   .word   0
+define void @jsVoidCall(i64 %dummy1, i64* %obj, i64 %arg, i64 %l1, i64 %l2) {
+entry:
+  %resolveCall = inttoptr i64 244837814094590 to i8*
+  call void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 7, i32 28, i8* %resolveCall, i32 2, i64* %obj, i64 %arg, i64 %l1, i64 %l2)
+  ret void
+}
+
+; i64 JS Call
+;
+; 2 live variables in registers.
+;
+; CHECK:        .word   .L{{.*}}-jsIntCall
+; CHECK-NEXT:   .half   0
+; CHECK-NEXT:   .half   2
+; CHECK-NEXT:   .byte   1
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .half   8
+; CHECK-NEXT:   .half   {{[0-9]+}}
+; CHECK-NEXT:   .half   0
+; CHECK-NEXT:   .word   0
+; CHECK-NEXT:   .byte   1
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .half   8
+; CHECK-NEXT:   .half   {{[0-9]+}}
+; CHECK-NEXT:   .half   0
+; CHECK-NEXT:   .word   0
+define i64 @jsIntCall(i64 %dummy1, i64* %obj, i64 %arg, i64 %l1, i64 %l2) {
+entry:
+  %resolveCall = inttoptr i64 244837814094590 to i8*
+  %result = call i64 (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.i64(i64 8, i32 28, i8* %resolveCall, i32 2, i64* %obj, i64 %arg, i64 %l1, i64 %l2)
+  %add = add i64 %result, 3
+  ret i64 %add
+}
+
+; Map a constant value.
+;
+; CHECK:        .word   .L{{.*}}-liveConstant
+; CHECK-NEXT:   .half   0
+; 1 location
+; CHECK-NEXT:   .half   1
+; Loc 0: SmallConstant
+; CHECK-NEXT:   .byte   4
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .half   8
+; CHECK-NEXT:   .half   0
+; CHECK-NEXT:   .half   0
+; CHECK-NEXT:   .word   33
+
+define void @liveConstant() {
+  tail call void (i64, i32, ...) @llvm.experimental.stackmap(i64 15, i32 8, i32 33)
+  ret void
+}
+
+; Spilled stack map values.
+;
+; Verify 28 stack map entries.
+;
+; CHECK-LABEL:  .word   .L{{.*}}-spilledValue
+; CHECK-NEXT:   .half   0
+; CHECK-NEXT:   .half   28
+;
+; Check that at least one is a spilled entry from RBP.
+; Location: Indirect RBP + ...
+; CHECK:        .byte   3
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .half   8
+; CHECK-NEXT:   .half   2
+; CHECK-NEXT:   .half   0
+; CHECK-NEXT:   .word
+define void @spilledValue(i64 %arg0, i64 %arg1, i64 %arg2, i64 %arg3, i64 %arg4, i64 %l0, i64 %l1, i64 %l2, i64 %l3, i64 %l4, i64 %l5, i64 %l6, i64 %l7, i64 %l8, i64 %l9, i64 %l10, i64 %l11, i64 %l12, i64 %l13, i64 %l14, i64 %l15, i64 %l16, i64 %l17, i64 %l18, i64 %l19, i64 %l20, i64 %l21, i64 %l22, i64 %l23, i64 %l24, i64 %l25, i64 %l26, i64 %l27) {
+entry:
+  call void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 11, i32 28, i8* null, i32 5, i64 %arg0, i64 %arg1, i64 %arg2, i64 %arg3, i64 %arg4, i64 %l0, i64 %l1, i64 %l2, i64 %l3, i64 %l4, i64 %l5, i64 %l6, i64 %l7, i64 %l8, i64 %l9, i64 %l10, i64 %l11, i64 %l12, i64 %l13, i64 %l14, i64 %l15, i64 %l16, i64 %l17, i64 %l18, i64 %l19, i64 %l20, i64 %l21, i64 %l22, i64 %l23, i64 %l24, i64 %l25, i64 %l26, i64 %l27)
+  ret void
+}
+
+; Directly map an alloca's address.
+;
+; Callsite 16
+; CHECK-LABEL:  .word .L{{.*}}-directFrameIdx
+; CHECK-NEXT:   .half   0
+; 1 location
+; CHECK-NEXT:   .half   1
+; Loc 0: Direct RBP - ofs
+; CHECK-NEXT:   .byte   2
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .half   8
+; CHECK-NEXT:   .half   2
+; CHECK-NEXT:   .half   0
+; CHECK-NEXT:   .word
+
+; Callsite 17
+; CHECK-LABEL:  .word   .L{{.*}}-directFrameIdx
+; CHECK-NEXT:   .half   0
+; 2 locations
+; CHECK-NEXT:   .half   2
+; Loc 0: Direct RBP - ofs
+; CHECK-NEXT:   .byte   2
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .half   8
+; CHECK-NEXT:   .half   2
+; CHECK-NEXT:   .half   0
+; CHECK-NEXT:   .word
+; Loc 1: Direct RBP - ofs
+; CHECK-NEXT:   .byte   2
+; CHECK-NEXT:   .byte   0
+; CHECK-NEXT:   .half   8
+; CHECK-NEXT:   .half   2
+; CHECK-NEXT:   .half   0
+; CHECK-NEXT:   .word
+define void @directFrameIdx() {
+entry:
+  %metadata1 = alloca i64, i32 3, align 8
+  store i64 11, i64* %metadata1
+  store i64 12, i64* %metadata1
+  store i64 13, i64* %metadata1
+  call void (i64, i32, ...) @llvm.experimental.stackmap(i64 16, i32 0, i64* %metadata1)
+  %metadata2 = alloca i8, i32 4, align 8
+  %metadata3 = alloca i16, i32 4, align 8
+  call void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 17, i32 4, i8* null, i32 0, i8* %metadata2, i16* %metadata3)
+  ret void
+}
+
+; Test a 64-bit ID.
+;
+; CHECK:        .quad   4294967295
+; CHECK-LABEL:  .word   .L{{.*}}-longid
+; CHECK:        .quad   4294967296
+; CHECK-LABEL:  .word   .L{{.*}}-longid
+; CHECK:        .quad   9223372036854775807
+; CHECK-LABEL:  .word   .L{{.*}}-longid
+; CHECK:        .quad   -1
+; CHECK-LABEL:  .word   .L{{.*}}-longid
+define void @longid() {
+entry:
+  tail call void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 4294967295, i32 0, i8* null, i32 0)
+  tail call void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 4294967296, i32 0, i8* null, i32 0)
+  tail call void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 9223372036854775807, i32 0, i8* null, i32 0)
+  tail call void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 -1, i32 0, i8* null, i32 0)
+  ret void
+}
+
+; A stack frame which needs to be realigned at runtime (to meet alignment
+; criteria for values on the stack) does not have a fixed frame size.
+; CHECK-LABEL:  .word   .L{{.*}}-needsStackRealignment
+; CHECK-NEXT:   .half   0
+; 0 locations
+; CHECK-NEXT:   .half   0
+define void @needsStackRealignment() {
+  %val = alloca i64, i32 3, align 128
+  tail call void (...) @escape_values(i64* %val)
+; Note: Adding any non-constant to the stackmap would fail because we
+; expected to be able to address off the frame pointer.  In a realigned
+; frame, we must use the stack pointer instead.  This is a separate bug.
+  tail call void (i64, i32, ...) @llvm.experimental.stackmap(i64 0, i32 0)
+  ret void
+}
+declare void @escape_values(...)
+
+declare void @llvm.experimental.stackmap(i64, i32, ...)
+declare void @llvm.experimental.patchpoint.void(i64, i32, i8*, i32, ...)
+declare i64 @llvm.experimental.patchpoint.i64(i64, i32, i8*, i32, ...)
diff --git a/llvm/test/CodeGen/RISCV/rv64zbb.ll b/llvm/test/CodeGen/RISCV/rv64zbb.ll
index f038af255a..cb1b152b83 100644
--- a/llvm/test/CodeGen/RISCV/rv64zbb.ll
+++ b/llvm/test/CodeGen/RISCV/rv64zbb.ll
@@ -1422,7 +1422,7 @@ define i32 @abs_i32(i32 %x) {
 define signext i32 @abs_i32_sext(i32 signext %x) {
 ; RV64I-LABEL: abs_i32_sext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    srai a1, a0, 31
+; RV64I-NEXT:    sraiw a1, a0, 31
 ; RV64I-NEXT:    xor a0, a0, a1
 ; RV64I-NEXT:    subw a0, a0, a1
 ; RV64I-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/concat-vectors-constant-stride.ll b/llvm/test/CodeGen/RISCV/rvv/concat-vectors-constant-stride.ll
new file mode 100644
index 0000000000..ff35043dbd
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/concat-vectors-constant-stride.ll
@@ -0,0 +1,167 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+v,+unaligned-vector-mem -target-abi=ilp32 \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32
+; RUN: llc -mtriple=riscv64 -mattr=+v,+unaligned-vector-mem -target-abi=lp64 \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64
+
+define void @constant_forward_stride(ptr %s, ptr %d) {
+; CHECK-LABEL: constant_forward_stride:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a2, 16
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vlse16.v v8, (a0), a2
+; CHECK-NEXT:    vse16.v v8, (a1)
+; CHECK-NEXT:    ret
+  %1 = getelementptr inbounds i8, ptr %s, i64 16
+  %2 = getelementptr inbounds i8, ptr %s, i64 32
+  %3 = getelementptr inbounds i8, ptr %s, i64 48
+  %4 = load <2 x i8>, ptr %s, align 1
+  %5 = load <2 x i8>, ptr %1, align 1
+  %6 = load <2 x i8>, ptr %2, align 1
+  %7 = load <2 x i8>, ptr %3, align 1
+  %8 = shufflevector <2 x i8> %4, <2 x i8> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %9 = shufflevector <2 x i8> %6, <2 x i8> %7, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %10 = shufflevector <4 x i8> %8, <4 x i8> %9, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  store <8 x i8> %10, ptr %d, align 1
+  ret void
+}
+
+define void @constant_forward_stride2(ptr %s, ptr %d) {
+; CHECK-LABEL: constant_forward_stride2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a0, a0, -48
+; CHECK-NEXT:    li a2, 16
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vlse16.v v8, (a0), a2
+; CHECK-NEXT:    vse16.v v8, (a1)
+; CHECK-NEXT:    ret
+  %1 = getelementptr inbounds i8, ptr %s, i64 -16
+  %2 = getelementptr inbounds i8, ptr %s, i64 -32
+  %3 = getelementptr inbounds i8, ptr %s, i64 -48
+  %4 = load <2 x i8>, ptr %3, align 1
+  %5 = load <2 x i8>, ptr %2, align 1
+  %6 = load <2 x i8>, ptr %1, align 1
+  %7 = load <2 x i8>, ptr %s, align 1
+  %8 = shufflevector <2 x i8> %4, <2 x i8> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %9 = shufflevector <2 x i8> %6, <2 x i8> %7, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %10 = shufflevector <4 x i8> %8, <4 x i8> %9, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  store <8 x i8> %10, ptr %d, align 1
+  ret void
+}
+
+define void @constant_forward_stride3(ptr %s, ptr %d) {
+; CHECK-LABEL: constant_forward_stride3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a2, 16
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vlse16.v v8, (a0), a2
+; CHECK-NEXT:    vse16.v v8, (a1)
+; CHECK-NEXT:    ret
+  %1 = getelementptr inbounds i8, ptr %s, i64 16
+  %2 = getelementptr inbounds i8, ptr %s, i64 32
+  %3 = getelementptr inbounds i8, ptr %s, i64 48
+  %4 = getelementptr inbounds i8, ptr %1, i64 0
+  %5 = getelementptr inbounds i8, ptr %2, i64 0
+  %6 = getelementptr inbounds i8, ptr %3, i64 0
+  %7 = load <2 x i8>, ptr %s, align 1
+  %8 = load <2 x i8>, ptr %4, align 1
+  %9 = load <2 x i8>, ptr %5, align 1
+  %10 = load <2 x i8>, ptr %6, align 1
+  %11 = shufflevector <2 x i8> %7, <2 x i8> %8, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %12 = shufflevector <2 x i8> %9, <2 x i8> %10, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %13 = shufflevector <4 x i8> %11, <4 x i8> %12, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  store <8 x i8> %13, ptr %d, align 1
+  ret void
+}
+
+define void @constant_back_stride(ptr %s, ptr %d) {
+; CHECK-LABEL: constant_back_stride:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a2, -16
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vlse16.v v8, (a0), a2
+; CHECK-NEXT:    vse16.v v8, (a1)
+; CHECK-NEXT:    ret
+  %1 = getelementptr inbounds i8, ptr %s, i64 -16
+  %2 = getelementptr inbounds i8, ptr %s, i64 -32
+  %3 = getelementptr inbounds i8, ptr %s, i64 -48
+  %4 = load <2 x i8>, ptr %s, align 1
+  %5 = load <2 x i8>, ptr %1, align 1
+  %6 = load <2 x i8>, ptr %2, align 1
+  %7 = load <2 x i8>, ptr %3, align 1
+  %8 = shufflevector <2 x i8> %4, <2 x i8> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %9 = shufflevector <2 x i8> %6, <2 x i8> %7, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %10 = shufflevector <4 x i8> %8, <4 x i8> %9, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  store <8 x i8> %10, ptr %d, align 1
+  ret void
+}
+
+define void @constant_back_stride2(ptr %s, ptr %d) {
+; CHECK-LABEL: constant_back_stride2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a0, a0, 48
+; CHECK-NEXT:    li a2, -16
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vlse16.v v8, (a0), a2
+; CHECK-NEXT:    vse16.v v8, (a1)
+; CHECK-NEXT:    ret
+  %1 = getelementptr inbounds i8, ptr %s, i64 16
+  %2 = getelementptr inbounds i8, ptr %s, i64 32
+  %3 = getelementptr inbounds i8, ptr %s, i64 48
+  %4 = load <2 x i8>, ptr %3, align 1
+  %5 = load <2 x i8>, ptr %2, align 1
+  %6 = load <2 x i8>, ptr %1, align 1
+  %7 = load <2 x i8>, ptr %s, align 1
+  %8 = shufflevector <2 x i8> %4, <2 x i8> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %9 = shufflevector <2 x i8> %6, <2 x i8> %7, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %10 = shufflevector <4 x i8> %8, <4 x i8> %9, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  store <8 x i8> %10, ptr %d, align 1
+  ret void
+}
+
+define void @constant_back_stride3(ptr %s, ptr %d) {
+; CHECK-LABEL: constant_back_stride3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a2, -16
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vlse16.v v8, (a0), a2
+; CHECK-NEXT:    vse16.v v8, (a1)
+; CHECK-NEXT:    ret
+  %1 = getelementptr inbounds i8, ptr %s, i64 -16
+  %2 = getelementptr inbounds i8, ptr %s, i64 -32
+  %3 = getelementptr inbounds i8, ptr %s, i64 -48
+  %4 = getelementptr inbounds i8, ptr %1, i64 0
+  %5 = getelementptr inbounds i8, ptr %2, i64 0
+  %6 = getelementptr inbounds i8, ptr %3, i64 0
+  %7 = load <2 x i8>, ptr %s, align 1
+  %8 = load <2 x i8>, ptr %4, align 1
+  %9 = load <2 x i8>, ptr %5, align 1
+  %10 = load <2 x i8>, ptr %6, align 1
+  %11 = shufflevector <2 x i8> %7, <2 x i8> %8, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %12 = shufflevector <2 x i8> %9, <2 x i8> %10, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %13 = shufflevector <4 x i8> %11, <4 x i8> %12, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  store <8 x i8> %13, ptr %d, align 1
+  ret void
+}
+
+define void @constant_zero_stride(ptr %s, ptr %d) {
+; CHECK-LABEL: constant_zero_stride:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
+; CHECK-NEXT:    vle8.v v8, (a0)
+; CHECK-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; CHECK-NEXT:    vmv1r.v v9, v8
+; CHECK-NEXT:    vslideup.vi v9, v8, 2
+; CHECK-NEXT:    vse8.v v9, (a1)
+; CHECK-NEXT:    ret
+  %1 = getelementptr inbounds i8, ptr %s, i64 0
+  %2 = load <2 x i8>, ptr %s, align 1
+  %3 = load <2 x i8>, ptr %1, align 1
+  %4 = shufflevector <2 x i8> %2, <2 x i8> %3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  store <4 x i8> %4, ptr %d, align 1
+  ret void
+}
+
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; RV32: {{.*}}
+; RV64: {{.*}}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-buildvec-of-binop.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-buildvec-of-binop.ll
index 2bbc04172b..717dfb1bfd 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-buildvec-of-binop.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-buildvec-of-binop.ll
@@ -442,3 +442,15 @@ define <4 x i32> @add_general_splat(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e) {
   %v3 = insertelement <4 x i32> %v2, i32 %e3, i32 3
   ret <4 x i32> %v3
 }
+
+; This test previously failed with an assertion failure because constant shift
+; amounts are type legalized early.
+define void @buggy(i32 %0) #0 {
+entry:
+  %mul.us.us.i.3 = shl i32 %0, 1
+  %1 = insertelement <4 x i32> zeroinitializer, i32 %mul.us.us.i.3, i64 0
+  %2 = or <4 x i32> %1, <i32 1, i32 1, i32 1, i32 1>
+  %3 = shufflevector <4 x i32> %2, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer
+  store <4 x i32> %3, ptr null, align 16
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfptrunc-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfptrunc-constrained-sdnode.ll
index 32a050800b..fd53113741 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfptrunc-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfptrunc-constrained-sdnode.ll
@@ -3,6 +3,10 @@
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 ; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=ilp32d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=lp64d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <2 x float> @llvm.experimental.constrained.fptrunc.v2f32.v2f64(<2 x double>, metadata, metadata)
 define <2 x float> @vfptrunc_v2f64_v2f32(<2 x double> %va) strictfp {
diff --git a/llvm/test/CodeGen/RISCV/rvv/fmaximum-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fmaximum-sdnode.ll
index 6fd0b280a5..c53fa71415 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fmaximum-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fmaximum-sdnode.ll
@@ -1,12 +1,20 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN:   -verify-machineinstrs -early-live-intervals < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN:   -verify-machineinstrs -early-live-intervals < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=ilp32d \
+; RUN:   -verify-machineinstrs -early-live-intervals < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 ; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=lp64d \
+; RUN:   -verify-machineinstrs -early-live-intervals < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 
 declare <vscale x 1 x half> @llvm.maximum.nxv1f16(<vscale x 1 x half>, <vscale x 1 x half>)
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/fminimum-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fminimum-sdnode.ll
index d9d53fb53e..b386792cd3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fminimum-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fminimum-sdnode.ll
@@ -1,12 +1,20 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN:   -verify-machineinstrs -early-live-intervals < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN:   -verify-machineinstrs -early-live-intervals < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=ilp32d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=ilp32d \
+; RUN:   -verify-machineinstrs -early-live-intervals < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 ; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=lp64d \
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v,+m -target-abi=lp64d \
+; RUN:   -verify-machineinstrs -early-live-intervals < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 
 declare <vscale x 1 x half> @llvm.minimum.nxv1f16(<vscale x 1 x half>, <vscale x 1 x half>)
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/llrint-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/llrint-sdnode.ll
new file mode 100644
index 0000000000..9a485a8b58
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/llrint-sdnode.ll
@@ -0,0 +1,108 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv64 -mattr=+v,+f,+d -target-abi=lp64d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s
+
+define <vscale x 1 x i64> @llrint_nxv1i64_nxv1f32(<vscale x 1 x float> %x) {
+; CHECK-LABEL: llrint_nxv1i64_nxv1f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vfwcvt.x.f.v v9, v8
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %a = call <vscale x 1 x i64> @llvm.llrint.nxv1i64.nxv1f32(<vscale x 1 x float> %x)
+  ret <vscale x 1 x i64> %a
+}
+declare <vscale x 1 x i64> @llvm.llrint.nxv1i64.nxv1f32(<vscale x 1 x float>)
+
+define <vscale x 2 x i64> @llrint_nxv2i64_nxv2f32(<vscale x 2 x float> %x) {
+; CHECK-LABEL: llrint_nxv2i64_nxv2f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vfwcvt.x.f.v v10, v8
+; CHECK-NEXT:    vmv2r.v v8, v10
+; CHECK-NEXT:    ret
+  %a = call <vscale x 2 x i64> @llvm.llrint.nxv2i64.nxv2f32(<vscale x 2 x float> %x)
+  ret <vscale x 2 x i64> %a
+}
+declare <vscale x 2 x i64> @llvm.llrint.nxv2i64.nxv2f32(<vscale x 2 x float>)
+
+define <vscale x 4 x i64> @llrint_nxv4i64_nxv4f32(<vscale x 4 x float> %x) {
+; CHECK-LABEL: llrint_nxv4i64_nxv4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vfwcvt.x.f.v v12, v8
+; CHECK-NEXT:    vmv4r.v v8, v12
+; CHECK-NEXT:    ret
+  %a = call <vscale x 4 x i64> @llvm.llrint.nxv4i64.nxv4f32(<vscale x 4 x float> %x)
+  ret <vscale x 4 x i64> %a
+}
+declare <vscale x 4 x i64> @llvm.llrint.nxv4i64.nxv4f32(<vscale x 4 x float>)
+
+define <vscale x 8 x i64> @llrint_nxv8i64_nxv8f32(<vscale x 8 x float> %x) {
+; CHECK-LABEL: llrint_nxv8i64_nxv8f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vfwcvt.x.f.v v16, v8
+; CHECK-NEXT:    vmv8r.v v8, v16
+; CHECK-NEXT:    ret
+  %a = call <vscale x 8 x i64> @llvm.llrint.nxv8i64.nxv8f32(<vscale x 8 x float> %x)
+  ret <vscale x 8 x i64> %a
+}
+declare <vscale x 8 x i64> @llvm.llrint.nxv8i64.nxv8f32(<vscale x 8 x float>)
+
+define <vscale x 16 x i64> @llrint_nxv16i64_nxv16f32(<vscale x 16 x float> %x) {
+; CHECK-LABEL: llrint_nxv16i64_nxv16f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vfwcvt.x.f.v v24, v8
+; CHECK-NEXT:    vfwcvt.x.f.v v16, v12
+; CHECK-NEXT:    vmv8r.v v8, v24
+; CHECK-NEXT:    ret
+  %a = call <vscale x 16 x i64> @llvm.llrint.nxv16i64.nxv16f32(<vscale x 16 x float> %x)
+  ret <vscale x 16 x i64> %a
+}
+declare <vscale x 16 x i64> @llvm.llrint.nxv16i64.nxv16f32(<vscale x 16 x float>)
+
+define <vscale x 1 x i64> @llrint_nxv1i64_nxv1f64(<vscale x 1 x double> %x) {
+; CHECK-LABEL: llrint_nxv1i64_nxv1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
+; CHECK-NEXT:    vfcvt.x.f.v v8, v8
+; CHECK-NEXT:    ret
+  %a = call <vscale x 1 x i64> @llvm.llrint.nxv1i64.nxv1f64(<vscale x 1 x double> %x)
+  ret <vscale x 1 x i64> %a
+}
+declare <vscale x 1 x i64> @llvm.llrint.nxv1i64.nxv1f64(<vscale x 1 x double>)
+
+define <vscale x 2 x i64> @llrint_nxv2i64_nxv2f64(<vscale x 2 x double> %x) {
+; CHECK-LABEL: llrint_nxv2i64_nxv2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vfcvt.x.f.v v8, v8
+; CHECK-NEXT:    ret
+  %a = call <vscale x 2 x i64> @llvm.llrint.nxv2i64.nxv2f64(<vscale x 2 x double> %x)
+  ret <vscale x 2 x i64> %a
+}
+declare <vscale x 2 x i64> @llvm.llrint.nxv2i64.nxv2f64(<vscale x 2 x double>)
+
+define <vscale x 4 x i64> @llrint_nxv4i64_nxv4f64(<vscale x 4 x double> %x) {
+; CHECK-LABEL: llrint_nxv4i64_nxv4f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vfcvt.x.f.v v8, v8
+; CHECK-NEXT:    ret
+  %a = call <vscale x 4 x i64> @llvm.llrint.nxv4i64.nxv4f64(<vscale x 4 x double> %x)
+  ret <vscale x 4 x i64> %a
+}
+declare <vscale x 4 x i64> @llvm.llrint.nxv4i64.nxv4f64(<vscale x 4 x double>)
+
+define <vscale x 8 x i64> @llrint_nxv8i64_nxv8f64(<vscale x 8 x double> %x) {
+; CHECK-LABEL: llrint_nxv8i64_nxv8f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vfcvt.x.f.v v8, v8
+; CHECK-NEXT:    ret
+  %a = call <vscale x 8 x i64> @llvm.llrint.nxv8i64.nxv8f64(<vscale x 8 x double> %x)
+  ret <vscale x 8 x i64> %a
+}
+declare <vscale x 8 x i64> @llvm.llrint.nxv8i64.nxv8f64(<vscale x 8 x double>)
diff --git a/llvm/test/CodeGen/RISCV/rvv/lrint-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/lrint-sdnode.ll
new file mode 100644
index 0000000000..61a5367b7f
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/lrint-sdnode.ll
@@ -0,0 +1,155 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+f,+d \
+; RUN:     -target-abi=ilp32d -verify-machineinstrs | FileCheck %s --check-prefix=RV32
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+f,+d \
+; RUN:     -target-abi=lp64d -verify-machineinstrs | FileCheck %s --check-prefix=RV64
+
+define <vscale x 1 x iXLen> @lrint_nxv1f32(<vscale x 1 x float> %x) {
+; RV32-LABEL: lrint_nxv1f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
+; RV32-NEXT:    vfcvt.x.f.v v8, v8
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: lrint_nxv1f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
+; RV64-NEXT:    vfwcvt.x.f.v v9, v8
+; RV64-NEXT:    vmv1r.v v8, v9
+; RV64-NEXT:    ret
+  %a = call <vscale x 1 x iXLen> @llvm.lrint.nxv1iXLen.nxv1f32(<vscale x 1 x float> %x)
+  ret <vscale x 1 x iXLen> %a
+}
+declare <vscale x 1 x iXLen> @llvm.lrint.nxv1iXLen.nxv1f32(<vscale x 1 x float>)
+
+define <vscale x 2 x iXLen> @lrint_nxv2f32(<vscale x 2 x float> %x) {
+; RV32-LABEL: lrint_nxv2f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
+; RV32-NEXT:    vfcvt.x.f.v v8, v8
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: lrint_nxv2f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
+; RV64-NEXT:    vfwcvt.x.f.v v10, v8
+; RV64-NEXT:    vmv2r.v v8, v10
+; RV64-NEXT:    ret
+  %a = call <vscale x 2 x iXLen> @llvm.lrint.nxv2iXLen.nxv2f32(<vscale x 2 x float> %x)
+  ret <vscale x 2 x iXLen> %a
+}
+declare <vscale x 2 x iXLen> @llvm.lrint.nxv2iXLen.nxv2f32(<vscale x 2 x float>)
+
+define <vscale x 4 x iXLen> @lrint_nxv4f32(<vscale x 4 x float> %x) {
+; RV32-LABEL: lrint_nxv4f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; RV32-NEXT:    vfcvt.x.f.v v8, v8
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: lrint_nxv4f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; RV64-NEXT:    vfwcvt.x.f.v v12, v8
+; RV64-NEXT:    vmv4r.v v8, v12
+; RV64-NEXT:    ret
+  %a = call <vscale x 4 x iXLen> @llvm.lrint.nxv4iXLen.nxv4f32(<vscale x 4 x float> %x)
+  ret <vscale x 4 x iXLen> %a
+}
+declare <vscale x 4 x iXLen> @llvm.lrint.nxv4iXLen.nxv4f32(<vscale x 4 x float>)
+
+define <vscale x 8 x iXLen> @lrint_nxv8f32(<vscale x 8 x float> %x) {
+; RV32-LABEL: lrint_nxv8f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; RV32-NEXT:    vfcvt.x.f.v v8, v8
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: lrint_nxv8f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; RV64-NEXT:    vfwcvt.x.f.v v16, v8
+; RV64-NEXT:    vmv8r.v v8, v16
+; RV64-NEXT:    ret
+  %a = call <vscale x 8 x iXLen> @llvm.lrint.nxv8iXLen.nxv8f32(<vscale x 8 x float> %x)
+  ret <vscale x 8 x iXLen> %a
+}
+declare <vscale x 8 x iXLen> @llvm.lrint.nxv8iXLen.nxv8f32(<vscale x 8 x float>)
+
+define <vscale x 16 x iXLen> @lrint_nxv16iXLen_nxv16f32(<vscale x 16 x float> %x) {
+  %a = call <vscale x 16 x iXLen> @llvm.lrint.nxv16iXLen.nxv16f32(<vscale x 16 x float> %x)
+  ret <vscale x 16 x iXLen> %a
+}
+declare <vscale x 16 x iXLen> @llvm.lrint.nxv16iXLen.nxv16f32(<vscale x 16 x float>)
+
+define <vscale x 1 x iXLen> @lrint_nxv1f64(<vscale x 1 x double> %x) {
+; RV32-LABEL: lrint_nxv1f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
+; RV32-NEXT:    vfncvt.x.f.w v9, v8
+; RV32-NEXT:    vmv1r.v v8, v9
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: lrint_nxv1f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
+; RV64-NEXT:    vfcvt.x.f.v v8, v8
+; RV64-NEXT:    ret
+  %a = call <vscale x 1 x iXLen> @llvm.lrint.nxv1iXLen.nxv1f64(<vscale x 1 x double> %x)
+  ret <vscale x 1 x iXLen> %a
+}
+declare <vscale x 1 x iXLen> @llvm.lrint.nxv1iXLen.nxv1f64(<vscale x 1 x double>)
+
+define <vscale x 2 x iXLen> @lrint_nxv2f64(<vscale x 2 x double> %x) {
+; RV32-LABEL: lrint_nxv2f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
+; RV32-NEXT:    vfncvt.x.f.w v10, v8
+; RV32-NEXT:    vmv.v.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: lrint_nxv2f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e64, m2, ta, ma
+; RV64-NEXT:    vfcvt.x.f.v v8, v8
+; RV64-NEXT:    ret
+  %a = call <vscale x 2 x iXLen> @llvm.lrint.nxv2iXLen.nxv2f64(<vscale x 2 x double> %x)
+  ret <vscale x 2 x iXLen> %a
+}
+declare <vscale x 2 x iXLen> @llvm.lrint.nxv2iXLen.nxv2f64(<vscale x 2 x double>)
+
+define <vscale x 4 x iXLen> @lrint_nxv4f64(<vscale x 4 x double> %x) {
+; RV32-LABEL: lrint_nxv4f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; RV32-NEXT:    vfncvt.x.f.w v12, v8
+; RV32-NEXT:    vmv.v.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: lrint_nxv4f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
+; RV64-NEXT:    vfcvt.x.f.v v8, v8
+; RV64-NEXT:    ret
+  %a = call <vscale x 4 x iXLen> @llvm.lrint.nxv4iXLen.nxv4f64(<vscale x 4 x double> %x)
+  ret <vscale x 4 x iXLen> %a
+}
+declare <vscale x 4 x iXLen> @llvm.lrint.nxv4iXLen.nxv4f64(<vscale x 4 x double>)
+
+define <vscale x 8 x iXLen> @lrint_nxv8f64(<vscale x 8 x double> %x) {
+; RV32-LABEL: lrint_nxv8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e32, m4, ta, ma
+; RV32-NEXT:    vfncvt.x.f.w v16, v8
+; RV32-NEXT:    vmv.v.v v8, v16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: lrint_nxv8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
+; RV64-NEXT:    vfcvt.x.f.v v8, v8
+; RV64-NEXT:    ret
+  %a = call <vscale x 8 x iXLen> @llvm.lrint.nxv8iXLen.nxv8f64(<vscale x 8 x double> %x)
+  ret <vscale x 8 x iXLen> %a
+}
+declare <vscale x 8 x iXLen> @llvm.lrint.nxv8iXLen.nxv8f64(<vscale x 8 x double>)
diff --git a/llvm/test/CodeGen/RISCV/rvv/sf_vqmacc_2x8x2.ll b/llvm/test/CodeGen/RISCV/rvv/sf_vqmacc_2x8x2.ll
new file mode 100644
index 0000000000..0a3623c236
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/sf_vqmacc_2x8x2.ll
@@ -0,0 +1,159 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+xsfvqmaccdod \
+; RUN:   -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+xsfvqmaccdod \
+; RUN:   -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK
+
+declare <vscale x 2 x i32> @llvm.riscv.sf.vqmacc.2x8x2.nxv2i32.nxv8i8.nxv8i8(
+  <vscale x 2 x i32>,
+  <vscale x 8 x i8>,
+  <vscale x 8 x i8>,
+  iXLen, iXLen);
+
+define <vscale x 2 x i32> @intrinsic_vqmacc_2x8x2_tu_i32m1(<vscale x 2 x i32> %0, <vscale x 8 x i8> %1, <vscale x 8 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmacc_2x8x2_tu_i32m1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, tu, ma
+; CHECK-NEXT:    sf.vqmacc.2x8x2 v8, v9, v10
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 2 x i32> @llvm.riscv.sf.vqmacc.2x8x2.nxv2i32.nxv8i8.nxv8i8(
+    <vscale x 2 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 8 x i8> %2,
+    iXLen %3, iXLen 2)
+
+  ret <vscale x 2 x i32> %a
+}
+
+define <vscale x 2 x i32> @intrinsic_vqmacc_2x8x2_ta_i32m1(<vscale x 2 x i32> %0, <vscale x 8 x i8> %1, <vscale x 8 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmacc_2x8x2_ta_i32m1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; CHECK-NEXT:    sf.vqmacc.2x8x2 v8, v9, v10
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 2 x i32> @llvm.riscv.sf.vqmacc.2x8x2.nxv2i32.nxv8i8.nxv8i8(
+    <vscale x 2 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 8 x i8> %2,
+    iXLen %3, iXLen 3)
+
+  ret <vscale x 2 x i32> %a
+}
+
+declare <vscale x 4 x i32> @llvm.riscv.sf.vqmacc.2x8x2.nxv4i32.nxv8i8.nxv16i8(
+  <vscale x 4 x i32>,
+  <vscale x 8 x i8>,
+  <vscale x 16 x i8>,
+  iXLen, iXLen);
+
+define <vscale x 4 x i32> @intrinsic_vqmacc_2x8x2_tu_i32m2(<vscale x 4 x i32> %0, <vscale x 8 x i8> %1, <vscale x 16 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmacc_2x8x2_tu_i32m2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, tu, ma
+; CHECK-NEXT:    sf.vqmacc.2x8x2 v8, v10, v12
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 4 x i32> @llvm.riscv.sf.vqmacc.2x8x2.nxv4i32.nxv8i8.nxv16i8(
+    <vscale x 4 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 16 x i8> %2,
+    iXLen %3, iXLen 2)
+
+  ret <vscale x 4 x i32> %a
+}
+
+define <vscale x 4 x i32> @intrinsic_vqmacc_2x8x2_ta_i32m2(<vscale x 4 x i32> %0, <vscale x 8 x i8> %1, <vscale x 16 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmacc_2x8x2_ta_i32m2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; CHECK-NEXT:    sf.vqmacc.2x8x2 v8, v10, v12
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 4 x i32> @llvm.riscv.sf.vqmacc.2x8x2.nxv4i32.nxv8i8.nxv16i8(
+    <vscale x 4 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 16 x i8> %2,
+    iXLen %3, iXLen 3)
+
+  ret <vscale x 4 x i32> %a
+}
+
+declare <vscale x 8 x i32> @llvm.riscv.sf.vqmacc.2x8x2.nxv8i32.nxv8i8.nxv32i8(
+  <vscale x 8 x i32>,
+  <vscale x 8 x i8>,
+  <vscale x 32 x i8>,
+  iXLen, iXLen);
+
+define <vscale x 8 x i32> @intrinsic_vqmacc_2x8x2_tu_i32m4(<vscale x 8 x i32> %0, <vscale x 8 x i8> %1, <vscale x 32 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmacc_2x8x2_tu_i32m4:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, tu, ma
+; CHECK-NEXT:    sf.vqmacc.2x8x2 v8, v12, v16
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 8 x i32> @llvm.riscv.sf.vqmacc.2x8x2.nxv8i32.nxv8i8.nxv32i8(
+    <vscale x 8 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 32 x i8> %2,
+    iXLen %3, iXLen 2)
+
+  ret <vscale x 8 x i32> %a
+}
+
+define <vscale x 8 x i32> @intrinsic_vqmacc_2x8x2_ta_i32m4(<vscale x 8 x i32> %0, <vscale x 8 x i8> %1, <vscale x 32 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmacc_2x8x2_ta_i32m4:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; CHECK-NEXT:    sf.vqmacc.2x8x2 v8, v12, v16
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 8 x i32> @llvm.riscv.sf.vqmacc.2x8x2.nxv8i32.nxv8i8.nxv32i8(
+    <vscale x 8 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 32 x i8> %2,
+    iXLen %3, iXLen 3)
+
+  ret <vscale x 8 x i32> %a
+}
+
+declare <vscale x 16 x i32> @llvm.riscv.sf.vqmacc.2x8x2.nxv16i32.nxv8i8.nxv64i8(
+  <vscale x 16 x i32>,
+  <vscale x 8 x i8>,
+  <vscale x 64 x i8>,
+  iXLen, iXLen);
+
+define <vscale x 16 x i32> @intrinsic_vqmacc_2x8x2_tu_i32m8(<vscale x 16 x i32> %0, <vscale x 8 x i8> %1, <vscale x 64 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmacc_2x8x2_tu_i32m8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vl8r.v v24, (a0)
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, tu, ma
+; CHECK-NEXT:    sf.vqmacc.2x8x2 v8, v16, v24
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 16 x i32> @llvm.riscv.sf.vqmacc.2x8x2.nxv16i32.nxv8i8.nxv64i8(
+    <vscale x 16 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 64 x i8> %2,
+    iXLen %3, iXLen 2)
+
+  ret <vscale x 16 x i32> %a
+}
+
+define <vscale x 16 x i32> @intrinsic_vqmacc_2x8x2_ta_i32m8(<vscale x 16 x i32> %0, <vscale x 8 x i8> %1, <vscale x 64 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmacc_2x8x2_ta_i32m8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vl8r.v v24, (a0)
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
+; CHECK-NEXT:    sf.vqmacc.2x8x2 v8, v16, v24
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 16 x i32> @llvm.riscv.sf.vqmacc.2x8x2.nxv16i32.nxv8i8.nxv64i8(
+    <vscale x 16 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 64 x i8> %2,
+    iXLen %3, iXLen 3)
+
+  ret <vscale x 16 x i32> %a
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/sf_vqmacc_4x8x4.ll b/llvm/test/CodeGen/RISCV/rvv/sf_vqmacc_4x8x4.ll
new file mode 100644
index 0000000000..843e4bda4d
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/sf_vqmacc_4x8x4.ll
@@ -0,0 +1,159 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+xsfvqmaccqoq \
+; RUN:   -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+xsfvqmaccqoq \
+; RUN:   -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK
+
+declare <vscale x 2 x i32> @llvm.riscv.sf.vqmacc.4x8x4.nxv2i32.nxv8i8.nxv8i8(
+  <vscale x 2 x i32>,
+  <vscale x 8 x i8>,
+  <vscale x 8 x i8>,
+  iXLen, iXLen);
+
+define <vscale x 2 x i32> @intrinsic_vqmacc_4x8x4_tu_i32m1(<vscale x 2 x i32> %0, <vscale x 8 x i8> %1, <vscale x 8 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmacc_4x8x4_tu_i32m1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, tu, ma
+; CHECK-NEXT:    sf.vqmacc.4x8x4 v8, v9, v10
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 2 x i32> @llvm.riscv.sf.vqmacc.4x8x4.nxv2i32.nxv8i8.nxv8i8(
+    <vscale x 2 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 8 x i8> %2,
+    iXLen %3, iXLen 2)
+
+  ret <vscale x 2 x i32> %a
+}
+
+define <vscale x 2 x i32> @intrinsic_vqmacc_4x8x4_ta_i32m1(<vscale x 2 x i32> %0, <vscale x 8 x i8> %1, <vscale x 8 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmacc_4x8x4_ta_i32m1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; CHECK-NEXT:    sf.vqmacc.4x8x4 v8, v9, v10
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 2 x i32> @llvm.riscv.sf.vqmacc.4x8x4.nxv2i32.nxv8i8.nxv8i8(
+    <vscale x 2 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 8 x i8> %2,
+    iXLen %3, iXLen 3)
+
+  ret <vscale x 2 x i32> %a
+}
+
+declare <vscale x 4 x i32> @llvm.riscv.sf.vqmacc.4x8x4.nxv4i32.nxv8i8.nxv16i8(
+  <vscale x 4 x i32>,
+  <vscale x 8 x i8>,
+  <vscale x 16 x i8>,
+  iXLen, iXLen);
+
+define <vscale x 4 x i32> @intrinsic_vqmacc_4x8x4_tu_i32m2(<vscale x 4 x i32> %0, <vscale x 8 x i8> %1, <vscale x 16 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmacc_4x8x4_tu_i32m2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, tu, ma
+; CHECK-NEXT:    sf.vqmacc.4x8x4 v8, v10, v12
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 4 x i32> @llvm.riscv.sf.vqmacc.4x8x4.nxv4i32.nxv8i8.nxv16i8(
+    <vscale x 4 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 16 x i8> %2,
+    iXLen %3, iXLen 2)
+
+  ret <vscale x 4 x i32> %a
+}
+
+define <vscale x 4 x i32> @intrinsic_vqmacc_4x8x4_ta_i32m2(<vscale x 4 x i32> %0, <vscale x 8 x i8> %1, <vscale x 16 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmacc_4x8x4_ta_i32m2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; CHECK-NEXT:    sf.vqmacc.4x8x4 v8, v10, v12
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 4 x i32> @llvm.riscv.sf.vqmacc.4x8x4.nxv4i32.nxv8i8.nxv16i8(
+    <vscale x 4 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 16 x i8> %2,
+    iXLen %3, iXLen 3)
+
+  ret <vscale x 4 x i32> %a
+}
+
+declare <vscale x 8 x i32> @llvm.riscv.sf.vqmacc.4x8x4.nxv8i32.nxv8i8.nxv32i8(
+  <vscale x 8 x i32>,
+  <vscale x 8 x i8>,
+  <vscale x 32 x i8>,
+  iXLen, iXLen);
+
+define <vscale x 8 x i32> @intrinsic_vqmacc_4x8x4_tu_i32m4(<vscale x 8 x i32> %0, <vscale x 8 x i8> %1, <vscale x 32 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmacc_4x8x4_tu_i32m4:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, tu, ma
+; CHECK-NEXT:    sf.vqmacc.4x8x4 v8, v12, v16
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 8 x i32> @llvm.riscv.sf.vqmacc.4x8x4.nxv8i32.nxv8i8.nxv32i8(
+    <vscale x 8 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 32 x i8> %2,
+    iXLen %3, iXLen 2)
+
+  ret <vscale x 8 x i32> %a
+}
+
+define <vscale x 8 x i32> @intrinsic_vqmacc_4x8x4_ta_i32m4(<vscale x 8 x i32> %0, <vscale x 8 x i8> %1, <vscale x 32 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmacc_4x8x4_ta_i32m4:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; CHECK-NEXT:    sf.vqmacc.4x8x4 v8, v12, v16
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 8 x i32> @llvm.riscv.sf.vqmacc.4x8x4.nxv8i32.nxv8i8.nxv32i8(
+    <vscale x 8 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 32 x i8> %2,
+    iXLen %3, iXLen 3)
+
+  ret <vscale x 8 x i32> %a
+}
+
+declare <vscale x 16 x i32> @llvm.riscv.sf.vqmacc.4x8x4.nxv16i32.nxv8i8.nxv64i8(
+  <vscale x 16 x i32>,
+  <vscale x 8 x i8>,
+  <vscale x 64 x i8>,
+  iXLen, iXLen);
+
+define <vscale x 16 x i32> @intrinsic_vqmacc_4x8x4_tu_i32m8(<vscale x 16 x i32> %0, <vscale x 8 x i8> %1, <vscale x 64 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmacc_4x8x4_tu_i32m8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vl8r.v v24, (a0)
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, tu, ma
+; CHECK-NEXT:    sf.vqmacc.4x8x4 v8, v16, v24
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 16 x i32> @llvm.riscv.sf.vqmacc.4x8x4.nxv16i32.nxv8i8.nxv64i8(
+    <vscale x 16 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 64 x i8> %2,
+    iXLen %3, iXLen 2)
+
+  ret <vscale x 16 x i32> %a
+}
+
+define <vscale x 16 x i32> @intrinsic_vqmacc_4x8x4_ta_i32m8(<vscale x 16 x i32> %0, <vscale x 8 x i8> %1, <vscale x 64 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmacc_4x8x4_ta_i32m8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vl8r.v v24, (a0)
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
+; CHECK-NEXT:    sf.vqmacc.4x8x4 v8, v16, v24
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 16 x i32> @llvm.riscv.sf.vqmacc.4x8x4.nxv16i32.nxv8i8.nxv64i8(
+    <vscale x 16 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 64 x i8> %2,
+    iXLen %3, iXLen 3)
+
+  ret <vscale x 16 x i32> %a
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/sf_vqmaccsu_2x8x2.ll b/llvm/test/CodeGen/RISCV/rvv/sf_vqmaccsu_2x8x2.ll
new file mode 100644
index 0000000000..106d318399
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/sf_vqmaccsu_2x8x2.ll
@@ -0,0 +1,159 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+xsfvqmaccdod \
+; RUN:   -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+xsfvqmaccdod \
+; RUN:   -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK
+
+declare <vscale x 2 x i32> @llvm.riscv.sf.vqmaccsu.2x8x2.nxv2i32.nxv8i8.nxv8i8(
+  <vscale x 2 x i32>,
+  <vscale x 8 x i8>,
+  <vscale x 8 x i8>,
+  iXLen, iXLen);
+
+define <vscale x 2 x i32> @intrinsic_vqmaccsu_2x8x2_tu_i32m1(<vscale x 2 x i32> %0, <vscale x 8 x i8> %1, <vscale x 8 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccsu_2x8x2_tu_i32m1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, tu, ma
+; CHECK-NEXT:    sf.vqmaccsu.2x8x2 v8, v9, v10
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 2 x i32> @llvm.riscv.sf.vqmaccsu.2x8x2.nxv2i32.nxv8i8.nxv8i8(
+    <vscale x 2 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 8 x i8> %2,
+    iXLen %3, iXLen 2)
+
+  ret <vscale x 2 x i32> %a
+}
+
+define <vscale x 2 x i32> @intrinsic_vqmaccsu_2x8x2_ta_i32m1(<vscale x 2 x i32> %0, <vscale x 8 x i8> %1, <vscale x 8 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccsu_2x8x2_ta_i32m1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; CHECK-NEXT:    sf.vqmaccsu.2x8x2 v8, v9, v10
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 2 x i32> @llvm.riscv.sf.vqmaccsu.2x8x2.nxv2i32.nxv8i8.nxv8i8(
+    <vscale x 2 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 8 x i8> %2,
+    iXLen %3, iXLen 3)
+
+  ret <vscale x 2 x i32> %a
+}
+
+declare <vscale x 4 x i32> @llvm.riscv.sf.vqmaccsu.2x8x2.nxv4i32.nxv8i8.nxv16i8(
+  <vscale x 4 x i32>,
+  <vscale x 8 x i8>,
+  <vscale x 16 x i8>,
+  iXLen, iXLen);
+
+define <vscale x 4 x i32> @intrinsic_vqmaccsu_2x8x2_tu_i32m2(<vscale x 4 x i32> %0, <vscale x 8 x i8> %1, <vscale x 16 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccsu_2x8x2_tu_i32m2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, tu, ma
+; CHECK-NEXT:    sf.vqmaccsu.2x8x2 v8, v10, v12
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 4 x i32> @llvm.riscv.sf.vqmaccsu.2x8x2.nxv4i32.nxv8i8.nxv16i8(
+    <vscale x 4 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 16 x i8> %2,
+    iXLen %3, iXLen 2)
+
+  ret <vscale x 4 x i32> %a
+}
+
+define <vscale x 4 x i32> @intrinsic_vqmaccsu_2x8x2_ta_i32m2(<vscale x 4 x i32> %0, <vscale x 8 x i8> %1, <vscale x 16 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccsu_2x8x2_ta_i32m2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; CHECK-NEXT:    sf.vqmaccsu.2x8x2 v8, v10, v12
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 4 x i32> @llvm.riscv.sf.vqmaccsu.2x8x2.nxv4i32.nxv8i8.nxv16i8(
+    <vscale x 4 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 16 x i8> %2,
+    iXLen %3, iXLen 3)
+
+  ret <vscale x 4 x i32> %a
+}
+
+declare <vscale x 8 x i32> @llvm.riscv.sf.vqmaccsu.2x8x2.nxv8i32.nxv8i8.nxv32i8(
+  <vscale x 8 x i32>,
+  <vscale x 8 x i8>,
+  <vscale x 32 x i8>,
+  iXLen, iXLen);
+
+define <vscale x 8 x i32> @intrinsic_vqmaccsu_2x8x2_tu_i32m4(<vscale x 8 x i32> %0, <vscale x 8 x i8> %1, <vscale x 32 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccsu_2x8x2_tu_i32m4:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, tu, ma
+; CHECK-NEXT:    sf.vqmaccsu.2x8x2 v8, v12, v16
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 8 x i32> @llvm.riscv.sf.vqmaccsu.2x8x2.nxv8i32.nxv8i8.nxv32i8(
+    <vscale x 8 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 32 x i8> %2,
+    iXLen %3, iXLen 2)
+
+  ret <vscale x 8 x i32> %a
+}
+
+define <vscale x 8 x i32> @intrinsic_vqmaccsu_2x8x2_ta_i32m4(<vscale x 8 x i32> %0, <vscale x 8 x i8> %1, <vscale x 32 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccsu_2x8x2_ta_i32m4:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; CHECK-NEXT:    sf.vqmaccsu.2x8x2 v8, v12, v16
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 8 x i32> @llvm.riscv.sf.vqmaccsu.2x8x2.nxv8i32.nxv8i8.nxv32i8(
+    <vscale x 8 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 32 x i8> %2,
+    iXLen %3, iXLen 3)
+
+  ret <vscale x 8 x i32> %a
+}
+
+declare <vscale x 16 x i32> @llvm.riscv.sf.vqmaccsu.2x8x2.nxv16i32.nxv8i8.nxv64i8(
+  <vscale x 16 x i32>,
+  <vscale x 8 x i8>,
+  <vscale x 64 x i8>,
+  iXLen, iXLen);
+
+define <vscale x 16 x i32> @intrinsic_vqmaccsu_2x8x2_tu_i32m8(<vscale x 16 x i32> %0, <vscale x 8 x i8> %1, <vscale x 64 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccsu_2x8x2_tu_i32m8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vl8r.v v24, (a0)
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, tu, ma
+; CHECK-NEXT:    sf.vqmaccsu.2x8x2 v8, v16, v24
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 16 x i32> @llvm.riscv.sf.vqmaccsu.2x8x2.nxv16i32.nxv8i8.nxv64i8(
+    <vscale x 16 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 64 x i8> %2,
+    iXLen %3, iXLen 2)
+
+  ret <vscale x 16 x i32> %a
+}
+
+define <vscale x 16 x i32> @intrinsic_vqmaccsu_2x8x2_ta_i32m8(<vscale x 16 x i32> %0, <vscale x 8 x i8> %1, <vscale x 64 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccsu_2x8x2_ta_i32m8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vl8r.v v24, (a0)
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
+; CHECK-NEXT:    sf.vqmaccsu.2x8x2 v8, v16, v24
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 16 x i32> @llvm.riscv.sf.vqmaccsu.2x8x2.nxv16i32.nxv8i8.nxv64i8(
+    <vscale x 16 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 64 x i8> %2,
+    iXLen %3, iXLen 3)
+
+  ret <vscale x 16 x i32> %a
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/sf_vqmaccsu_4x8x4.ll b/llvm/test/CodeGen/RISCV/rvv/sf_vqmaccsu_4x8x4.ll
new file mode 100644
index 0000000000..45a3b22d96
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/sf_vqmaccsu_4x8x4.ll
@@ -0,0 +1,159 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+xsfvqmaccqoq \
+; RUN:   -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+xsfvqmaccqoq \
+; RUN:   -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK
+
+declare <vscale x 2 x i32> @llvm.riscv.sf.vqmaccsu.4x8x4.nxv2i32.nxv8i8.nxv8i8(
+  <vscale x 2 x i32>,
+  <vscale x 8 x i8>,
+  <vscale x 8 x i8>,
+  iXLen, iXLen);
+
+define <vscale x 2 x i32> @intrinsic_vqmaccsu_4x8x4_tu_i32m1(<vscale x 2 x i32> %0, <vscale x 8 x i8> %1, <vscale x 8 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccsu_4x8x4_tu_i32m1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, tu, ma
+; CHECK-NEXT:    sf.vqmaccsu.4x8x4 v8, v9, v10
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 2 x i32> @llvm.riscv.sf.vqmaccsu.4x8x4.nxv2i32.nxv8i8.nxv8i8(
+    <vscale x 2 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 8 x i8> %2,
+    iXLen %3, iXLen 2)
+
+  ret <vscale x 2 x i32> %a
+}
+
+define <vscale x 2 x i32> @intrinsic_vqmaccsu_4x8x4_ta_i32m1(<vscale x 2 x i32> %0, <vscale x 8 x i8> %1, <vscale x 8 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccsu_4x8x4_ta_i32m1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; CHECK-NEXT:    sf.vqmaccsu.4x8x4 v8, v9, v10
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 2 x i32> @llvm.riscv.sf.vqmaccsu.4x8x4.nxv2i32.nxv8i8.nxv8i8(
+    <vscale x 2 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 8 x i8> %2,
+    iXLen %3, iXLen 3)
+
+  ret <vscale x 2 x i32> %a
+}
+
+declare <vscale x 4 x i32> @llvm.riscv.sf.vqmaccsu.4x8x4.nxv4i32.nxv8i8.nxv16i8(
+  <vscale x 4 x i32>,
+  <vscale x 8 x i8>,
+  <vscale x 16 x i8>,
+  iXLen, iXLen);
+
+define <vscale x 4 x i32> @intrinsic_vqmaccsu_4x8x4_tu_i32m2(<vscale x 4 x i32> %0, <vscale x 8 x i8> %1, <vscale x 16 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccsu_4x8x4_tu_i32m2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, tu, ma
+; CHECK-NEXT:    sf.vqmaccsu.4x8x4 v8, v10, v12
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 4 x i32> @llvm.riscv.sf.vqmaccsu.4x8x4.nxv4i32.nxv8i8.nxv16i8(
+    <vscale x 4 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 16 x i8> %2,
+    iXLen %3, iXLen 2)
+
+  ret <vscale x 4 x i32> %a
+}
+
+define <vscale x 4 x i32> @intrinsic_vqmaccsu_4x8x4_ta_i32m2(<vscale x 4 x i32> %0, <vscale x 8 x i8> %1, <vscale x 16 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccsu_4x8x4_ta_i32m2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; CHECK-NEXT:    sf.vqmaccsu.4x8x4 v8, v10, v12
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 4 x i32> @llvm.riscv.sf.vqmaccsu.4x8x4.nxv4i32.nxv8i8.nxv16i8(
+    <vscale x 4 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 16 x i8> %2,
+    iXLen %3, iXLen 3)
+
+  ret <vscale x 4 x i32> %a
+}
+
+declare <vscale x 8 x i32> @llvm.riscv.sf.vqmaccsu.4x8x4.nxv8i32.nxv8i8.nxv32i8(
+  <vscale x 8 x i32>,
+  <vscale x 8 x i8>,
+  <vscale x 32 x i8>,
+  iXLen, iXLen);
+
+define <vscale x 8 x i32> @intrinsic_vqmaccsu_4x8x4_tu_i32m4(<vscale x 8 x i32> %0, <vscale x 8 x i8> %1, <vscale x 32 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccsu_4x8x4_tu_i32m4:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, tu, ma
+; CHECK-NEXT:    sf.vqmaccsu.4x8x4 v8, v12, v16
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 8 x i32> @llvm.riscv.sf.vqmaccsu.4x8x4.nxv8i32.nxv8i8.nxv32i8(
+    <vscale x 8 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 32 x i8> %2,
+    iXLen %3, iXLen 2)
+
+  ret <vscale x 8 x i32> %a
+}
+
+define <vscale x 8 x i32> @intrinsic_vqmaccsu_4x8x4_ta_i32m4(<vscale x 8 x i32> %0, <vscale x 8 x i8> %1, <vscale x 32 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccsu_4x8x4_ta_i32m4:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; CHECK-NEXT:    sf.vqmaccsu.4x8x4 v8, v12, v16
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 8 x i32> @llvm.riscv.sf.vqmaccsu.4x8x4.nxv8i32.nxv8i8.nxv32i8(
+    <vscale x 8 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 32 x i8> %2,
+    iXLen %3, iXLen 3)
+
+  ret <vscale x 8 x i32> %a
+}
+
+declare <vscale x 16 x i32> @llvm.riscv.sf.vqmaccsu.4x8x4.nxv16i32.nxv8i8.nxv64i8(
+  <vscale x 16 x i32>,
+  <vscale x 8 x i8>,
+  <vscale x 64 x i8>,
+  iXLen, iXLen);
+
+define <vscale x 16 x i32> @intrinsic_vqmaccsu_4x8x4_tu_i32m8(<vscale x 16 x i32> %0, <vscale x 8 x i8> %1, <vscale x 64 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccsu_4x8x4_tu_i32m8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vl8r.v v24, (a0)
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, tu, ma
+; CHECK-NEXT:    sf.vqmaccsu.4x8x4 v8, v16, v24
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 16 x i32> @llvm.riscv.sf.vqmaccsu.4x8x4.nxv16i32.nxv8i8.nxv64i8(
+    <vscale x 16 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 64 x i8> %2,
+    iXLen %3, iXLen 2)
+
+  ret <vscale x 16 x i32> %a
+}
+
+define <vscale x 16 x i32> @intrinsic_vqmaccsu_4x8x4_ta_i32m8(<vscale x 16 x i32> %0, <vscale x 8 x i8> %1, <vscale x 64 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccsu_4x8x4_ta_i32m8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vl8r.v v24, (a0)
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
+; CHECK-NEXT:    sf.vqmaccsu.4x8x4 v8, v16, v24
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 16 x i32> @llvm.riscv.sf.vqmaccsu.4x8x4.nxv16i32.nxv8i8.nxv64i8(
+    <vscale x 16 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 64 x i8> %2,
+    iXLen %3, iXLen 3)
+
+  ret <vscale x 16 x i32> %a
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/sf_vqmaccu_2x8x2.ll b/llvm/test/CodeGen/RISCV/rvv/sf_vqmaccu_2x8x2.ll
new file mode 100644
index 0000000000..d9ab4559ea
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/sf_vqmaccu_2x8x2.ll
@@ -0,0 +1,159 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+xsfvqmaccdod \
+; RUN:   -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+xsfvqmaccdod \
+; RUN:   -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK
+
+declare <vscale x 2 x i32> @llvm.riscv.sf.vqmaccu.2x8x2.nxv2i32.nxv8i8.nxv8i8(
+  <vscale x 2 x i32>,
+  <vscale x 8 x i8>,
+  <vscale x 8 x i8>,
+  iXLen, iXLen);
+
+define <vscale x 2 x i32> @intrinsic_vqmaccu_2x8x2_tu_i32m1(<vscale x 2 x i32> %0, <vscale x 8 x i8> %1, <vscale x 8 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccu_2x8x2_tu_i32m1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, tu, ma
+; CHECK-NEXT:    sf.vqmaccu.2x8x2 v8, v9, v10
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 2 x i32> @llvm.riscv.sf.vqmaccu.2x8x2.nxv2i32.nxv8i8.nxv8i8(
+    <vscale x 2 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 8 x i8> %2,
+    iXLen %3, iXLen 2)
+
+  ret <vscale x 2 x i32> %a
+}
+
+define <vscale x 2 x i32> @intrinsic_vqmaccu_2x8x2_ta_i32m1(<vscale x 2 x i32> %0, <vscale x 8 x i8> %1, <vscale x 8 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccu_2x8x2_ta_i32m1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; CHECK-NEXT:    sf.vqmaccu.2x8x2 v8, v9, v10
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 2 x i32> @llvm.riscv.sf.vqmaccu.2x8x2.nxv2i32.nxv8i8.nxv8i8(
+    <vscale x 2 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 8 x i8> %2,
+    iXLen %3, iXLen 3)
+
+  ret <vscale x 2 x i32> %a
+}
+
+declare <vscale x 4 x i32> @llvm.riscv.sf.vqmaccu.2x8x2.nxv4i32.nxv8i8.nxv16i8(
+  <vscale x 4 x i32>,
+  <vscale x 8 x i8>,
+  <vscale x 16 x i8>,
+  iXLen, iXLen);
+
+define <vscale x 4 x i32> @intrinsic_vqmaccu_2x8x2_tu_i32m2(<vscale x 4 x i32> %0, <vscale x 8 x i8> %1, <vscale x 16 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccu_2x8x2_tu_i32m2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, tu, ma
+; CHECK-NEXT:    sf.vqmaccu.2x8x2 v8, v10, v12
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 4 x i32> @llvm.riscv.sf.vqmaccu.2x8x2.nxv4i32.nxv8i8.nxv16i8(
+    <vscale x 4 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 16 x i8> %2,
+    iXLen %3, iXLen 2)
+
+  ret <vscale x 4 x i32> %a
+}
+
+define <vscale x 4 x i32> @intrinsic_vqmaccu_2x8x2_ta_i32m2(<vscale x 4 x i32> %0, <vscale x 8 x i8> %1, <vscale x 16 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccu_2x8x2_ta_i32m2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; CHECK-NEXT:    sf.vqmaccu.2x8x2 v8, v10, v12
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 4 x i32> @llvm.riscv.sf.vqmaccu.2x8x2.nxv4i32.nxv8i8.nxv16i8(
+    <vscale x 4 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 16 x i8> %2,
+    iXLen %3, iXLen 3)
+
+  ret <vscale x 4 x i32> %a
+}
+
+declare <vscale x 8 x i32> @llvm.riscv.sf.vqmaccu.2x8x2.nxv8i32.nxv8i8.nxv32i8(
+  <vscale x 8 x i32>,
+  <vscale x 8 x i8>,
+  <vscale x 32 x i8>,
+  iXLen, iXLen);
+
+define <vscale x 8 x i32> @intrinsic_vqmaccu_2x8x2_tu_i32m4(<vscale x 8 x i32> %0, <vscale x 8 x i8> %1, <vscale x 32 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccu_2x8x2_tu_i32m4:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, tu, ma
+; CHECK-NEXT:    sf.vqmaccu.2x8x2 v8, v12, v16
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 8 x i32> @llvm.riscv.sf.vqmaccu.2x8x2.nxv8i32.nxv8i8.nxv32i8(
+    <vscale x 8 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 32 x i8> %2,
+    iXLen %3, iXLen 2)
+
+  ret <vscale x 8 x i32> %a
+}
+
+define <vscale x 8 x i32> @intrinsic_vqmaccu_2x8x2_ta_i32m4(<vscale x 8 x i32> %0, <vscale x 8 x i8> %1, <vscale x 32 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccu_2x8x2_ta_i32m4:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; CHECK-NEXT:    sf.vqmaccu.2x8x2 v8, v12, v16
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 8 x i32> @llvm.riscv.sf.vqmaccu.2x8x2.nxv8i32.nxv8i8.nxv32i8(
+    <vscale x 8 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 32 x i8> %2,
+    iXLen %3, iXLen 3)
+
+  ret <vscale x 8 x i32> %a
+}
+
+declare <vscale x 16 x i32> @llvm.riscv.sf.vqmaccu.2x8x2.nxv16i32.nxv8i8.nxv64i8(
+  <vscale x 16 x i32>,
+  <vscale x 8 x i8>,
+  <vscale x 64 x i8>,
+  iXLen, iXLen);
+
+define <vscale x 16 x i32> @intrinsic_vqmaccu_2x8x2_tu_i32m8(<vscale x 16 x i32> %0, <vscale x 8 x i8> %1, <vscale x 64 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccu_2x8x2_tu_i32m8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vl8r.v v24, (a0)
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, tu, ma
+; CHECK-NEXT:    sf.vqmaccu.2x8x2 v8, v16, v24
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 16 x i32> @llvm.riscv.sf.vqmaccu.2x8x2.nxv16i32.nxv8i8.nxv64i8(
+    <vscale x 16 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 64 x i8> %2,
+    iXLen %3, iXLen 2)
+
+  ret <vscale x 16 x i32> %a
+}
+
+define <vscale x 16 x i32> @intrinsic_vqmaccu_2x8x2_ta_i32m8(<vscale x 16 x i32> %0, <vscale x 8 x i8> %1, <vscale x 64 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccu_2x8x2_ta_i32m8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vl8r.v v24, (a0)
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
+; CHECK-NEXT:    sf.vqmaccu.2x8x2 v8, v16, v24
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 16 x i32> @llvm.riscv.sf.vqmaccu.2x8x2.nxv16i32.nxv8i8.nxv64i8(
+    <vscale x 16 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 64 x i8> %2,
+    iXLen %3, iXLen 3)
+
+  ret <vscale x 16 x i32> %a
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/sf_vqmaccu_4x8x4.ll b/llvm/test/CodeGen/RISCV/rvv/sf_vqmaccu_4x8x4.ll
new file mode 100644
index 0000000000..9d15ab68a0
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/sf_vqmaccu_4x8x4.ll
@@ -0,0 +1,159 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+xsfvqmaccqoq \
+; RUN:   -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+xsfvqmaccqoq \
+; RUN:   -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK
+
+declare <vscale x 2 x i32> @llvm.riscv.sf.vqmaccu.4x8x4.nxv2i32.nxv8i8.nxv8i8(
+  <vscale x 2 x i32>,
+  <vscale x 8 x i8>,
+  <vscale x 8 x i8>,
+  iXLen, iXLen);
+
+define <vscale x 2 x i32> @intrinsic_vqmaccu_4x8x4_tu_i32m1(<vscale x 2 x i32> %0, <vscale x 8 x i8> %1, <vscale x 8 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccu_4x8x4_tu_i32m1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, tu, ma
+; CHECK-NEXT:    sf.vqmaccu.4x8x4 v8, v9, v10
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 2 x i32> @llvm.riscv.sf.vqmaccu.4x8x4.nxv2i32.nxv8i8.nxv8i8(
+    <vscale x 2 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 8 x i8> %2,
+    iXLen %3, iXLen 2)
+
+  ret <vscale x 2 x i32> %a
+}
+
+define <vscale x 2 x i32> @intrinsic_vqmaccu_4x8x4_ta_i32m1(<vscale x 2 x i32> %0, <vscale x 8 x i8> %1, <vscale x 8 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccu_4x8x4_ta_i32m1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; CHECK-NEXT:    sf.vqmaccu.4x8x4 v8, v9, v10
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 2 x i32> @llvm.riscv.sf.vqmaccu.4x8x4.nxv2i32.nxv8i8.nxv8i8(
+    <vscale x 2 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 8 x i8> %2,
+    iXLen %3, iXLen 3)
+
+  ret <vscale x 2 x i32> %a
+}
+
+declare <vscale x 4 x i32> @llvm.riscv.sf.vqmaccu.4x8x4.nxv4i32.nxv8i8.nxv16i8(
+  <vscale x 4 x i32>,
+  <vscale x 8 x i8>,
+  <vscale x 16 x i8>,
+  iXLen, iXLen);
+
+define <vscale x 4 x i32> @intrinsic_vqmaccu_4x8x4_tu_i32m2(<vscale x 4 x i32> %0, <vscale x 8 x i8> %1, <vscale x 16 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccu_4x8x4_tu_i32m2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, tu, ma
+; CHECK-NEXT:    sf.vqmaccu.4x8x4 v8, v10, v12
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 4 x i32> @llvm.riscv.sf.vqmaccu.4x8x4.nxv4i32.nxv8i8.nxv16i8(
+    <vscale x 4 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 16 x i8> %2,
+    iXLen %3, iXLen 2)
+
+  ret <vscale x 4 x i32> %a
+}
+
+define <vscale x 4 x i32> @intrinsic_vqmaccu_4x8x4_ta_i32m2(<vscale x 4 x i32> %0, <vscale x 8 x i8> %1, <vscale x 16 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccu_4x8x4_ta_i32m2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; CHECK-NEXT:    sf.vqmaccu.4x8x4 v8, v10, v12
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 4 x i32> @llvm.riscv.sf.vqmaccu.4x8x4.nxv4i32.nxv8i8.nxv16i8(
+    <vscale x 4 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 16 x i8> %2,
+    iXLen %3, iXLen 3)
+
+  ret <vscale x 4 x i32> %a
+}
+
+declare <vscale x 8 x i32> @llvm.riscv.sf.vqmaccu.4x8x4.nxv8i32.nxv8i8.nxv32i8(
+  <vscale x 8 x i32>,
+  <vscale x 8 x i8>,
+  <vscale x 32 x i8>,
+  iXLen, iXLen);
+
+define <vscale x 8 x i32> @intrinsic_vqmaccu_4x8x4_tu_i32m4(<vscale x 8 x i32> %0, <vscale x 8 x i8> %1, <vscale x 32 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccu_4x8x4_tu_i32m4:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, tu, ma
+; CHECK-NEXT:    sf.vqmaccu.4x8x4 v8, v12, v16
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 8 x i32> @llvm.riscv.sf.vqmaccu.4x8x4.nxv8i32.nxv8i8.nxv32i8(
+    <vscale x 8 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 32 x i8> %2,
+    iXLen %3, iXLen 2)
+
+  ret <vscale x 8 x i32> %a
+}
+
+define <vscale x 8 x i32> @intrinsic_vqmaccu_4x8x4_ta_i32m4(<vscale x 8 x i32> %0, <vscale x 8 x i8> %1, <vscale x 32 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccu_4x8x4_ta_i32m4:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; CHECK-NEXT:    sf.vqmaccu.4x8x4 v8, v12, v16
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 8 x i32> @llvm.riscv.sf.vqmaccu.4x8x4.nxv8i32.nxv8i8.nxv32i8(
+    <vscale x 8 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 32 x i8> %2,
+    iXLen %3, iXLen 3)
+
+  ret <vscale x 8 x i32> %a
+}
+
+declare <vscale x 16 x i32> @llvm.riscv.sf.vqmaccu.4x8x4.nxv16i32.nxv8i8.nxv64i8(
+  <vscale x 16 x i32>,
+  <vscale x 8 x i8>,
+  <vscale x 64 x i8>,
+  iXLen, iXLen);
+
+define <vscale x 16 x i32> @intrinsic_vqmaccu_4x8x4_tu_i32m8(<vscale x 16 x i32> %0, <vscale x 8 x i8> %1, <vscale x 64 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccu_4x8x4_tu_i32m8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vl8r.v v24, (a0)
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, tu, ma
+; CHECK-NEXT:    sf.vqmaccu.4x8x4 v8, v16, v24
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 16 x i32> @llvm.riscv.sf.vqmaccu.4x8x4.nxv16i32.nxv8i8.nxv64i8(
+    <vscale x 16 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 64 x i8> %2,
+    iXLen %3, iXLen 2)
+
+  ret <vscale x 16 x i32> %a
+}
+
+define <vscale x 16 x i32> @intrinsic_vqmaccu_4x8x4_ta_i32m8(<vscale x 16 x i32> %0, <vscale x 8 x i8> %1, <vscale x 64 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccu_4x8x4_ta_i32m8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vl8r.v v24, (a0)
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
+; CHECK-NEXT:    sf.vqmaccu.4x8x4 v8, v16, v24
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 16 x i32> @llvm.riscv.sf.vqmaccu.4x8x4.nxv16i32.nxv8i8.nxv64i8(
+    <vscale x 16 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 64 x i8> %2,
+    iXLen %3, iXLen 3)
+
+  ret <vscale x 16 x i32> %a
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/sf_vqmaccus_2x8x2.ll b/llvm/test/CodeGen/RISCV/rvv/sf_vqmaccus_2x8x2.ll
new file mode 100644
index 0000000000..7ce486993c
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/sf_vqmaccus_2x8x2.ll
@@ -0,0 +1,159 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+xsfvqmaccdod \
+; RUN:   -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+xsfvqmaccdod \
+; RUN:   -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK
+
+declare <vscale x 2 x i32> @llvm.riscv.sf.vqmaccus.2x8x2.nxv2i32.nxv8i8.nxv8i8(
+  <vscale x 2 x i32>,
+  <vscale x 8 x i8>,
+  <vscale x 8 x i8>,
+  iXLen, iXLen);
+
+define <vscale x 2 x i32> @intrinsic_vqmaccus_2x8x2_tu_i32m1(<vscale x 2 x i32> %0, <vscale x 8 x i8> %1, <vscale x 8 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccus_2x8x2_tu_i32m1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, tu, ma
+; CHECK-NEXT:    sf.vqmaccus.2x8x2 v8, v9, v10
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 2 x i32> @llvm.riscv.sf.vqmaccus.2x8x2.nxv2i32.nxv8i8.nxv8i8(
+    <vscale x 2 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 8 x i8> %2,
+    iXLen %3, iXLen 2)
+
+  ret <vscale x 2 x i32> %a
+}
+
+define <vscale x 2 x i32> @intrinsic_vqmaccus_2x8x2_ta_i32m1(<vscale x 2 x i32> %0, <vscale x 8 x i8> %1, <vscale x 8 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccus_2x8x2_ta_i32m1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; CHECK-NEXT:    sf.vqmaccus.2x8x2 v8, v9, v10
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 2 x i32> @llvm.riscv.sf.vqmaccus.2x8x2.nxv2i32.nxv8i8.nxv8i8(
+    <vscale x 2 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 8 x i8> %2,
+    iXLen %3, iXLen 3)
+
+  ret <vscale x 2 x i32> %a
+}
+
+declare <vscale x 4 x i32> @llvm.riscv.sf.vqmaccus.2x8x2.nxv4i32.nxv8i8.nxv16i8(
+  <vscale x 4 x i32>,
+  <vscale x 8 x i8>,
+  <vscale x 16 x i8>,
+  iXLen, iXLen);
+
+define <vscale x 4 x i32> @intrinsic_vqmaccus_2x8x2_tu_i32m2(<vscale x 4 x i32> %0, <vscale x 8 x i8> %1, <vscale x 16 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccus_2x8x2_tu_i32m2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, tu, ma
+; CHECK-NEXT:    sf.vqmaccus.2x8x2 v8, v10, v12
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 4 x i32> @llvm.riscv.sf.vqmaccus.2x8x2.nxv4i32.nxv8i8.nxv16i8(
+    <vscale x 4 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 16 x i8> %2,
+    iXLen %3, iXLen 2)
+
+  ret <vscale x 4 x i32> %a
+}
+
+define <vscale x 4 x i32> @intrinsic_vqmaccus_2x8x2_ta_i32m2(<vscale x 4 x i32> %0, <vscale x 8 x i8> %1, <vscale x 16 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccus_2x8x2_ta_i32m2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; CHECK-NEXT:    sf.vqmaccus.2x8x2 v8, v10, v12
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 4 x i32> @llvm.riscv.sf.vqmaccus.2x8x2.nxv4i32.nxv8i8.nxv16i8(
+    <vscale x 4 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 16 x i8> %2,
+    iXLen %3, iXLen 3)
+
+  ret <vscale x 4 x i32> %a
+}
+
+declare <vscale x 8 x i32> @llvm.riscv.sf.vqmaccus.2x8x2.nxv8i32.nxv8i8.nxv32i8(
+  <vscale x 8 x i32>,
+  <vscale x 8 x i8>,
+  <vscale x 32 x i8>,
+  iXLen, iXLen);
+
+define <vscale x 8 x i32> @intrinsic_vqmaccus_2x8x2_tu_i32m4(<vscale x 8 x i32> %0, <vscale x 8 x i8> %1, <vscale x 32 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccus_2x8x2_tu_i32m4:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, tu, ma
+; CHECK-NEXT:    sf.vqmaccus.2x8x2 v8, v12, v16
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 8 x i32> @llvm.riscv.sf.vqmaccus.2x8x2.nxv8i32.nxv8i8.nxv32i8(
+    <vscale x 8 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 32 x i8> %2,
+    iXLen %3, iXLen 2)
+
+  ret <vscale x 8 x i32> %a
+}
+
+define <vscale x 8 x i32> @intrinsic_vqmaccus_2x8x2_ta_i32m4(<vscale x 8 x i32> %0, <vscale x 8 x i8> %1, <vscale x 32 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccus_2x8x2_ta_i32m4:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; CHECK-NEXT:    sf.vqmaccus.2x8x2 v8, v12, v16
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 8 x i32> @llvm.riscv.sf.vqmaccus.2x8x2.nxv8i32.nxv8i8.nxv32i8(
+    <vscale x 8 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 32 x i8> %2,
+    iXLen %3, iXLen 3)
+
+  ret <vscale x 8 x i32> %a
+}
+
+declare <vscale x 16 x i32> @llvm.riscv.sf.vqmaccus.2x8x2.nxv16i32.nxv8i8.nxv64i8(
+  <vscale x 16 x i32>,
+  <vscale x 8 x i8>,
+  <vscale x 64 x i8>,
+  iXLen, iXLen);
+
+define <vscale x 16 x i32> @intrinsic_vqmaccus_2x8x2_tu_i32m8(<vscale x 16 x i32> %0, <vscale x 8 x i8> %1, <vscale x 64 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccus_2x8x2_tu_i32m8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vl8r.v v24, (a0)
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, tu, ma
+; CHECK-NEXT:    sf.vqmaccus.2x8x2 v8, v16, v24
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 16 x i32> @llvm.riscv.sf.vqmaccus.2x8x2.nxv16i32.nxv8i8.nxv64i8(
+    <vscale x 16 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 64 x i8> %2,
+    iXLen %3, iXLen 2)
+
+  ret <vscale x 16 x i32> %a
+}
+
+define <vscale x 16 x i32> @intrinsic_vqmaccus_2x8x2_ta_i32m8(<vscale x 16 x i32> %0, <vscale x 8 x i8> %1, <vscale x 64 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccus_2x8x2_ta_i32m8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vl8r.v v24, (a0)
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
+; CHECK-NEXT:    sf.vqmaccus.2x8x2 v8, v16, v24
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 16 x i32> @llvm.riscv.sf.vqmaccus.2x8x2.nxv16i32.nxv8i8.nxv64i8(
+    <vscale x 16 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 64 x i8> %2,
+    iXLen %3, iXLen 3)
+
+  ret <vscale x 16 x i32> %a
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/sf_vqmaccus_4x8x4.ll b/llvm/test/CodeGen/RISCV/rvv/sf_vqmaccus_4x8x4.ll
new file mode 100644
index 0000000000..d9b62d8a57
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/sf_vqmaccus_4x8x4.ll
@@ -0,0 +1,159 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+xsfvqmaccqoq \
+; RUN:   -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+xsfvqmaccqoq \
+; RUN:   -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK
+
+declare <vscale x 2 x i32> @llvm.riscv.sf.vqmaccus.4x8x4.nxv2i32.nxv8i8.nxv8i8(
+  <vscale x 2 x i32>,
+  <vscale x 8 x i8>,
+  <vscale x 8 x i8>,
+  iXLen, iXLen);
+
+define <vscale x 2 x i32> @intrinsic_vqmaccus_4x8x4_tu_i32m1(<vscale x 2 x i32> %0, <vscale x 8 x i8> %1, <vscale x 8 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccus_4x8x4_tu_i32m1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, tu, ma
+; CHECK-NEXT:    sf.vqmaccus.4x8x4 v8, v9, v10
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 2 x i32> @llvm.riscv.sf.vqmaccus.4x8x4.nxv2i32.nxv8i8.nxv8i8(
+    <vscale x 2 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 8 x i8> %2,
+    iXLen %3, iXLen 2)
+
+  ret <vscale x 2 x i32> %a
+}
+
+define <vscale x 2 x i32> @intrinsic_vqmaccus_4x8x4_ta_i32m1(<vscale x 2 x i32> %0, <vscale x 8 x i8> %1, <vscale x 8 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccus_4x8x4_ta_i32m1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; CHECK-NEXT:    sf.vqmaccus.4x8x4 v8, v9, v10
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 2 x i32> @llvm.riscv.sf.vqmaccus.4x8x4.nxv2i32.nxv8i8.nxv8i8(
+    <vscale x 2 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 8 x i8> %2,
+    iXLen %3, iXLen 3)
+
+  ret <vscale x 2 x i32> %a
+}
+
+declare <vscale x 4 x i32> @llvm.riscv.sf.vqmaccus.4x8x4.nxv4i32.nxv8i8.nxv16i8(
+  <vscale x 4 x i32>,
+  <vscale x 8 x i8>,
+  <vscale x 16 x i8>,
+  iXLen, iXLen);
+
+define <vscale x 4 x i32> @intrinsic_vqmaccus_4x8x4_tu_i32m2(<vscale x 4 x i32> %0, <vscale x 8 x i8> %1, <vscale x 16 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccus_4x8x4_tu_i32m2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, tu, ma
+; CHECK-NEXT:    sf.vqmaccus.4x8x4 v8, v10, v12
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 4 x i32> @llvm.riscv.sf.vqmaccus.4x8x4.nxv4i32.nxv8i8.nxv16i8(
+    <vscale x 4 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 16 x i8> %2,
+    iXLen %3, iXLen 2)
+
+  ret <vscale x 4 x i32> %a
+}
+
+define <vscale x 4 x i32> @intrinsic_vqmaccus_4x8x4_ta_i32m2(<vscale x 4 x i32> %0, <vscale x 8 x i8> %1, <vscale x 16 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccus_4x8x4_ta_i32m2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; CHECK-NEXT:    sf.vqmaccus.4x8x4 v8, v10, v12
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 4 x i32> @llvm.riscv.sf.vqmaccus.4x8x4.nxv4i32.nxv8i8.nxv16i8(
+    <vscale x 4 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 16 x i8> %2,
+    iXLen %3, iXLen 3)
+
+  ret <vscale x 4 x i32> %a
+}
+
+declare <vscale x 8 x i32> @llvm.riscv.sf.vqmaccus.4x8x4.nxv8i32.nxv8i8.nxv32i8(
+  <vscale x 8 x i32>,
+  <vscale x 8 x i8>,
+  <vscale x 32 x i8>,
+  iXLen, iXLen);
+
+define <vscale x 8 x i32> @intrinsic_vqmaccus_4x8x4_tu_i32m4(<vscale x 8 x i32> %0, <vscale x 8 x i8> %1, <vscale x 32 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccus_4x8x4_tu_i32m4:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, tu, ma
+; CHECK-NEXT:    sf.vqmaccus.4x8x4 v8, v12, v16
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 8 x i32> @llvm.riscv.sf.vqmaccus.4x8x4.nxv8i32.nxv8i8.nxv32i8(
+    <vscale x 8 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 32 x i8> %2,
+    iXLen %3, iXLen 2)
+
+  ret <vscale x 8 x i32> %a
+}
+
+define <vscale x 8 x i32> @intrinsic_vqmaccus_4x8x4_ta_i32m4(<vscale x 8 x i32> %0, <vscale x 8 x i8> %1, <vscale x 32 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccus_4x8x4_ta_i32m4:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; CHECK-NEXT:    sf.vqmaccus.4x8x4 v8, v12, v16
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 8 x i32> @llvm.riscv.sf.vqmaccus.4x8x4.nxv8i32.nxv8i8.nxv32i8(
+    <vscale x 8 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 32 x i8> %2,
+    iXLen %3, iXLen 3)
+
+  ret <vscale x 8 x i32> %a
+}
+
+declare <vscale x 16 x i32> @llvm.riscv.sf.vqmaccus.4x8x4.nxv16i32.nxv8i8.nxv64i8(
+  <vscale x 16 x i32>,
+  <vscale x 8 x i8>,
+  <vscale x 64 x i8>,
+  iXLen, iXLen);
+
+define <vscale x 16 x i32> @intrinsic_vqmaccus_4x8x4_tu_i32m8(<vscale x 16 x i32> %0, <vscale x 8 x i8> %1, <vscale x 64 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccus_4x8x4_tu_i32m8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vl8r.v v24, (a0)
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, tu, ma
+; CHECK-NEXT:    sf.vqmaccus.4x8x4 v8, v16, v24
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 16 x i32> @llvm.riscv.sf.vqmaccus.4x8x4.nxv16i32.nxv8i8.nxv64i8(
+    <vscale x 16 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 64 x i8> %2,
+    iXLen %3, iXLen 2)
+
+  ret <vscale x 16 x i32> %a
+}
+
+define <vscale x 16 x i32> @intrinsic_vqmaccus_4x8x4_ta_i32m8(<vscale x 16 x i32> %0, <vscale x 8 x i8> %1, <vscale x 64 x i8> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vqmaccus_4x8x4_ta_i32m8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vl8r.v v24, (a0)
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
+; CHECK-NEXT:    sf.vqmaccus.4x8x4 v8, v16, v24
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 16 x i32> @llvm.riscv.sf.vqmaccus.4x8x4.nxv16i32.nxv8i8.nxv64i8(
+    <vscale x 16 x i32> %0,
+    <vscale x 8 x i8> %1,
+    <vscale x 64 x i8> %2,
+    iXLen %3, iXLen 3)
+
+  ret <vscale x 16 x i32> %a
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfmsub-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfmsub-sdnode.ll
index 5792171269..433b0d1cbd 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfmsub-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfmsub-sdnode.ll
@@ -1,8 +1,12 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN:     -verify-machineinstrs -early-live-intervals < %s | FileCheck %s
 ; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN:     -verify-machineinstrs -early-live-intervals < %s | FileCheck %s
 
 ; This tests a mix of vfmsac and vfmsub by using different operand orders to
 ; trigger commuting in TwoAddressInstructionPass.
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfptrunc-constrained-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfptrunc-constrained-sdnode.ll
index 72bf2b94e6..4404a27585 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfptrunc-constrained-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfptrunc-constrained-sdnode.ll
@@ -3,6 +3,10 @@
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 ; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=ilp32d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=lp64d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 declare <vscale x 1 x float> @llvm.experimental.constrained.fptrunc.nxv1f32.nxv1f64(<vscale x 1 x double>, metadata, metadata)
 define <vscale x 1 x float> @vfptrunc_nxv1f64_nxv1f32(<vscale x 1 x double> %va) strictfp {
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmv.s.x-rv32.ll b/llvm/test/CodeGen/RISCV/rvv/vmv.s.x-rv32.ll
index 477b12c29b..7c3e94ab71 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmv.s.x-rv32.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmv.s.x-rv32.ll
@@ -58,7 +58,7 @@ declare <vscale x 16 x i8> @llvm.riscv.vmv.s.x.nxv16i8(<vscale x 16 x i8>, i8, i
 define <vscale x 16 x i8> @intrinsic_vmv.s.x_x_nxv16i8(<vscale x 16 x i8> %0, i8 %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vmv.s.x_x_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, tu, ma
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, tu, ma
 ; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -71,7 +71,7 @@ declare <vscale x 32 x i8> @llvm.riscv.vmv.s.x.nxv32i8(<vscale x 32 x i8>, i8, i
 define <vscale x 32 x i8> @intrinsic_vmv.s.x_x_nxv32i8(<vscale x 32 x i8> %0, i8 %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vmv.s.x_x_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, tu, ma
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, tu, ma
 ; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -84,7 +84,7 @@ declare <vscale x 64 x i8> @llvm.riscv.vmv.s.x.nxv64i8(<vscale x 64 x i8>, i8, i
 define <vscale x 64 x i8> @intrinsic_vmv.s.x_x_nxv64i8(<vscale x 64 x i8> %0, i8 %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vmv.s.x_x_nxv64i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, tu, ma
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, tu, ma
 ; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -136,7 +136,7 @@ declare <vscale x 8 x i16> @llvm.riscv.vmv.s.x.nxv8i16(<vscale x 8 x i16>, i16,
 define <vscale x 8 x i16> @intrinsic_vmv.s.x_x_nxv8i16(<vscale x 8 x i16> %0, i16 %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vmv.s.x_x_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, tu, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, tu, ma
 ; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -149,7 +149,7 @@ declare <vscale x 16 x i16> @llvm.riscv.vmv.s.x.nxv16i16(<vscale x 16 x i16>, i1
 define <vscale x 16 x i16> @intrinsic_vmv.s.x_x_nxv16i16(<vscale x 16 x i16> %0, i16 %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vmv.s.x_x_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, tu, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, tu, ma
 ; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -162,7 +162,7 @@ declare <vscale x 32 x i16> @llvm.riscv.vmv.s.x.nxv32i16(<vscale x 32 x i16>, i1
 define <vscale x 32 x i16> @intrinsic_vmv.s.x_x_nxv32i16(<vscale x 32 x i16> %0, i16 %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vmv.s.x_x_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m8, tu, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, tu, ma
 ; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -201,7 +201,7 @@ declare <vscale x 4 x i32> @llvm.riscv.vmv.s.x.nxv4i32(<vscale x 4 x i32>, i32,
 define <vscale x 4 x i32> @intrinsic_vmv.s.x_x_nxv4i32(<vscale x 4 x i32> %0, i32 %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vmv.s.x_x_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, tu, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, tu, ma
 ; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -214,7 +214,7 @@ declare <vscale x 8 x i32> @llvm.riscv.vmv.s.x.nxv8i32(<vscale x 8 x i32>, i32,
 define <vscale x 8 x i32> @intrinsic_vmv.s.x_x_nxv8i32(<vscale x 8 x i32> %0, i32 %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vmv.s.x_x_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, tu, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, tu, ma
 ; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -227,7 +227,7 @@ declare <vscale x 16 x i32> @llvm.riscv.vmv.s.x.nxv16i32(<vscale x 16 x i32>, i3
 define <vscale x 16 x i32> @intrinsic_vmv.s.x_x_nxv16i32(<vscale x 16 x i32> %0, i32 %1, i32 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vmv.s.x_x_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, tu, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, tu, ma
 ; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmv.s.x-rv64.ll b/llvm/test/CodeGen/RISCV/rvv/vmv.s.x-rv64.ll
index 8bba3b841a..79931acbdc 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmv.s.x-rv64.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmv.s.x-rv64.ll
@@ -58,7 +58,7 @@ declare <vscale x 16 x i8> @llvm.riscv.vmv.s.x.nxv16i8(<vscale x 16 x i8>, i8, i
 define <vscale x 16 x i8> @intrinsic_vmv.s.x_x_nxv16i8(<vscale x 16 x i8> %0, i8 %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vmv.s.x_x_nxv16i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m2, tu, ma
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, tu, ma
 ; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -71,7 +71,7 @@ declare <vscale x 32 x i8> @llvm.riscv.vmv.s.x.nxv32i8(<vscale x 32 x i8>, i8, i
 define <vscale x 32 x i8> @intrinsic_vmv.s.x_x_nxv32i8(<vscale x 32 x i8> %0, i8 %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vmv.s.x_x_nxv32i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m4, tu, ma
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, tu, ma
 ; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -84,7 +84,7 @@ declare <vscale x 64 x i8> @llvm.riscv.vmv.s.x.nxv64i8(<vscale x 64 x i8>, i8, i
 define <vscale x 64 x i8> @intrinsic_vmv.s.x_x_nxv64i8(<vscale x 64 x i8> %0, i8 %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vmv.s.x_x_nxv64i8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e8, m8, tu, ma
+; CHECK-NEXT:    vsetvli zero, a1, e8, m1, tu, ma
 ; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -136,7 +136,7 @@ declare <vscale x 8 x i16> @llvm.riscv.vmv.s.x.nxv8i16(<vscale x 8 x i16>, i16,
 define <vscale x 8 x i16> @intrinsic_vmv.s.x_x_nxv8i16(<vscale x 8 x i16> %0, i16 %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vmv.s.x_x_nxv8i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m2, tu, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, tu, ma
 ; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -149,7 +149,7 @@ declare <vscale x 16 x i16> @llvm.riscv.vmv.s.x.nxv16i16(<vscale x 16 x i16>, i1
 define <vscale x 16 x i16> @intrinsic_vmv.s.x_x_nxv16i16(<vscale x 16 x i16> %0, i16 %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vmv.s.x_x_nxv16i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m4, tu, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, tu, ma
 ; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -162,7 +162,7 @@ declare <vscale x 32 x i16> @llvm.riscv.vmv.s.x.nxv32i16(<vscale x 32 x i16>, i1
 define <vscale x 32 x i16> @intrinsic_vmv.s.x_x_nxv32i16(<vscale x 32 x i16> %0, i16 %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vmv.s.x_x_nxv32i16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e16, m8, tu, ma
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, tu, ma
 ; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -201,7 +201,7 @@ declare <vscale x 4 x i32> @llvm.riscv.vmv.s.x.nxv4i32(<vscale x 4 x i32>, i32,
 define <vscale x 4 x i32> @intrinsic_vmv.s.x_x_nxv4i32(<vscale x 4 x i32> %0, i32 %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vmv.s.x_x_nxv4i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m2, tu, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, tu, ma
 ; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -214,7 +214,7 @@ declare <vscale x 8 x i32> @llvm.riscv.vmv.s.x.nxv8i32(<vscale x 8 x i32>, i32,
 define <vscale x 8 x i32> @intrinsic_vmv.s.x_x_nxv8i32(<vscale x 8 x i32> %0, i32 %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vmv.s.x_x_nxv8i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m4, tu, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, tu, ma
 ; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -227,7 +227,7 @@ declare <vscale x 16 x i32> @llvm.riscv.vmv.s.x.nxv16i32(<vscale x 16 x i32>, i3
 define <vscale x 16 x i32> @intrinsic_vmv.s.x_x_nxv16i32(<vscale x 16 x i32> %0, i32 %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vmv.s.x_x_nxv16i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, tu, ma
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, tu, ma
 ; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -253,7 +253,7 @@ declare <vscale x 2 x i64> @llvm.riscv.vmv.s.x.nxv2i64(<vscale x 2 x i64>, i64,
 define <vscale x 2 x i64> @intrinsic_vmv.s.x_x_nxv2i64(<vscale x 2 x i64> %0, i64 %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vmv.s.x_x_nxv2i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m2, tu, ma
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, tu, ma
 ; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -266,7 +266,7 @@ declare <vscale x 4 x i64> @llvm.riscv.vmv.s.x.nxv4i64(<vscale x 4 x i64>, i64,
 define <vscale x 4 x i64> @intrinsic_vmv.s.x_x_nxv4i64(<vscale x 4 x i64> %0, i64 %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vmv.s.x_x_nxv4i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m4, tu, ma
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, tu, ma
 ; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    ret
 entry:
@@ -279,7 +279,7 @@ declare <vscale x 8 x i64> @llvm.riscv.vmv.s.x.nxv8i64(<vscale x 8 x i64>, i64,
 define <vscale x 8 x i64> @intrinsic_vmv.s.x_x_nxv8i64(<vscale x 8 x i64> %0, i64 %1, i64 %2) nounwind {
 ; CHECK-LABEL: intrinsic_vmv.s.x_x_nxv8i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli zero, a1, e64, m8, tu, ma
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, tu, ma
 ; CHECK-NEXT:    vmv.s.x v8, a0
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/select-binop-identity.ll b/llvm/test/CodeGen/RISCV/select-binop-identity.ll
index 61344bc897..f45d67164d 100644
--- a/llvm/test/CodeGen/RISCV/select-binop-identity.ll
+++ b/llvm/test/CodeGen/RISCV/select-binop-identity.ll
@@ -2,15 +2,15 @@
 ; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefixes=RV32,RV32I %s
 ; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
-; RUN:   | FileCheck -check-prefixes=RV64,RV64I %s
+; RUN:   | FileCheck -check-prefixes=RV64I %s
 ; RUN: llc -mtriple=riscv64 -mcpu=sifive-u74 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=SFB64 %s
 ; RUN: llc -mtriple=riscv64 -mattr=+xventanacondops -verify-machineinstrs < %s \
-; RUN:   | FileCheck -check-prefixes=RV64,VTCONDOPS64 %s
+; RUN:   | FileCheck -check-prefixes=VTCONDOPS64 %s
 ; RUN: llc -mtriple=riscv32 -mattr=+experimental-zicond -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefixes=RV32,ZICOND,ZICOND32 %s
 ; RUN: llc -mtriple=riscv64 -mattr=+experimental-zicond -verify-machineinstrs < %s \
-; RUN:   | FileCheck -check-prefixes=RV64,ZICOND,ZICOND64 %s
+; RUN:   | FileCheck -check-prefixes=ZICOND,ZICOND64 %s
 
 ; InstCombine canonicalizes (c ? x | y : x) to (x | (c ? y : 0)) similar for
 ; other binary operations using their identity value as the constant.
@@ -19,19 +19,19 @@
 ; the basic block we create when we expand select.
 
 define signext i32 @and_select_all_ones_i32(i1 zeroext %c, i32 signext %x, i32 signext %y) {
-; RV32-LABEL: and_select_all_ones_i32:
-; RV32:       # %bb.0:
-; RV32-NEXT:    addi a0, a0, -1
-; RV32-NEXT:    or a0, a0, a1
-; RV32-NEXT:    and a0, a0, a2
-; RV32-NEXT:    ret
+; RV32I-LABEL: and_select_all_ones_i32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi a0, a0, -1
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    and a0, a0, a2
+; RV32I-NEXT:    ret
 ;
-; RV64-LABEL: and_select_all_ones_i32:
-; RV64:       # %bb.0:
-; RV64-NEXT:    addi a0, a0, -1
-; RV64-NEXT:    or a0, a0, a1
-; RV64-NEXT:    and a0, a0, a2
-; RV64-NEXT:    ret
+; RV64I-LABEL: and_select_all_ones_i32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi a0, a0, -1
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    and a0, a0, a2
+; RV64I-NEXT:    ret
 ;
 ; SFB64-LABEL: and_select_all_ones_i32:
 ; SFB64:       # %bb.0:
@@ -41,6 +41,118 @@ define signext i32 @and_select_all_ones_i32(i1 zeroext %c, i32 signext %x, i32 s
 ; SFB64-NEXT:  .LBB0_2:
 ; SFB64-NEXT:    mv a0, a2
 ; SFB64-NEXT:    ret
+;
+; VTCONDOPS64-LABEL: and_select_all_ones_i32:
+; VTCONDOPS64:       # %bb.0:
+; VTCONDOPS64-NEXT:    vt.maskcn a0, a2, a0
+; VTCONDOPS64-NEXT:    and a1, a2, a1
+; VTCONDOPS64-NEXT:    or a0, a1, a0
+; VTCONDOPS64-NEXT:    ret
+;
+; ZICOND-LABEL: and_select_all_ones_i32:
+; ZICOND:       # %bb.0:
+; ZICOND-NEXT:    czero.nez a0, a2, a0
+; ZICOND-NEXT:    and a1, a2, a1
+; ZICOND-NEXT:    or a0, a1, a0
+; ZICOND-NEXT:    ret
+  %a = select i1 %c, i32 %x, i32 -1
+  %b = and i32 %a, %y
+  ret i32 %b
+}
+
+define signext i32 @and_select_all_ones_i32_cmp(i32 signext %x, i32 signext %y, i32 signext %z) {
+; RV32I-LABEL: and_select_all_ones_i32_cmp:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi a2, a2, -4
+; RV32I-NEXT:    seqz a2, a2
+; RV32I-NEXT:    addi a2, a2, -1
+; RV32I-NEXT:    or a0, a2, a0
+; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: and_select_all_ones_i32_cmp:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi a2, a2, -4
+; RV64I-NEXT:    seqz a2, a2
+; RV64I-NEXT:    addi a2, a2, -1
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; SFB64-LABEL: and_select_all_ones_i32_cmp:
+; SFB64:       # %bb.0:
+; SFB64-NEXT:    li a3, 4
+; SFB64-NEXT:    bne a2, a3, .LBB1_2
+; SFB64-NEXT:  # %bb.1:
+; SFB64-NEXT:    and a1, a1, a0
+; SFB64-NEXT:  .LBB1_2:
+; SFB64-NEXT:    mv a0, a1
+; SFB64-NEXT:    ret
+;
+; VTCONDOPS64-LABEL: and_select_all_ones_i32_cmp:
+; VTCONDOPS64:       # %bb.0:
+; VTCONDOPS64-NEXT:    addi a2, a2, -4
+; VTCONDOPS64-NEXT:    and a0, a1, a0
+; VTCONDOPS64-NEXT:    vt.maskc a1, a1, a2
+; VTCONDOPS64-NEXT:    or a0, a0, a1
+; VTCONDOPS64-NEXT:    ret
+;
+; ZICOND-LABEL: and_select_all_ones_i32_cmp:
+; ZICOND:       # %bb.0:
+; ZICOND-NEXT:    addi a2, a2, -4
+; ZICOND-NEXT:    and a0, a1, a0
+; ZICOND-NEXT:    czero.eqz a1, a1, a2
+; ZICOND-NEXT:    or a0, a0, a1
+; ZICOND-NEXT:    ret
+  %c = icmp eq i32 %z, 4
+  %a = select i1 %c, i32 %x, i32 -1
+  %b = and i32 %a, %y
+  ret i32 %b
+}
+
+define signext i32 @and_select_all_ones_i32_cmp2(i32 signext %x, i32 signext %y, i32 signext %z) {
+; RV32I-LABEL: and_select_all_ones_i32_cmp2:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slti a2, a2, 4
+; RV32I-NEXT:    addi a2, a2, -1
+; RV32I-NEXT:    or a0, a2, a0
+; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: and_select_all_ones_i32_cmp2:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slti a2, a2, 4
+; RV64I-NEXT:    addi a2, a2, -1
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; SFB64-LABEL: and_select_all_ones_i32_cmp2:
+; SFB64:       # %bb.0:
+; SFB64-NEXT:    li a3, 4
+; SFB64-NEXT:    bge a2, a3, .LBB2_2
+; SFB64-NEXT:  # %bb.1:
+; SFB64-NEXT:    and a1, a1, a0
+; SFB64-NEXT:  .LBB2_2:
+; SFB64-NEXT:    mv a0, a1
+; SFB64-NEXT:    ret
+;
+; VTCONDOPS64-LABEL: and_select_all_ones_i32_cmp2:
+; VTCONDOPS64:       # %bb.0:
+; VTCONDOPS64-NEXT:    slti a2, a2, 4
+; VTCONDOPS64-NEXT:    and a0, a1, a0
+; VTCONDOPS64-NEXT:    vt.maskcn a1, a1, a2
+; VTCONDOPS64-NEXT:    or a0, a0, a1
+; VTCONDOPS64-NEXT:    ret
+;
+; ZICOND-LABEL: and_select_all_ones_i32_cmp2:
+; ZICOND:       # %bb.0:
+; ZICOND-NEXT:    slti a2, a2, 4
+; ZICOND-NEXT:    and a0, a1, a0
+; ZICOND-NEXT:    czero.nez a1, a1, a2
+; ZICOND-NEXT:    or a0, a0, a1
+; ZICOND-NEXT:    ret
+  %c = icmp slt i32 %z, 4
   %a = select i1 %c, i32 %x, i32 -1
   %b = and i32 %a, %y
   ret i32 %b
@@ -56,26 +168,163 @@ define i64 @and_select_all_ones_i64(i1 zeroext %c, i64 %x, i64 %y) {
 ; RV32-NEXT:    and a1, a4, a2
 ; RV32-NEXT:    ret
 ;
-; RV64-LABEL: and_select_all_ones_i64:
-; RV64:       # %bb.0:
-; RV64-NEXT:    neg a0, a0
-; RV64-NEXT:    or a0, a0, a1
-; RV64-NEXT:    and a0, a2, a0
-; RV64-NEXT:    ret
+; RV64I-LABEL: and_select_all_ones_i64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    neg a0, a0
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    and a0, a2, a0
+; RV64I-NEXT:    ret
 ;
 ; SFB64-LABEL: and_select_all_ones_i64:
 ; SFB64:       # %bb.0:
-; SFB64-NEXT:    bnez a0, .LBB1_2
+; SFB64-NEXT:    bnez a0, .LBB3_2
 ; SFB64-NEXT:  # %bb.1:
 ; SFB64-NEXT:    and a2, a2, a1
-; SFB64-NEXT:  .LBB1_2:
+; SFB64-NEXT:  .LBB3_2:
 ; SFB64-NEXT:    mv a0, a2
 ; SFB64-NEXT:    ret
+;
+; VTCONDOPS64-LABEL: and_select_all_ones_i64:
+; VTCONDOPS64:       # %bb.0:
+; VTCONDOPS64-NEXT:    vt.maskc a0, a2, a0
+; VTCONDOPS64-NEXT:    and a1, a2, a1
+; VTCONDOPS64-NEXT:    or a0, a1, a0
+; VTCONDOPS64-NEXT:    ret
+;
+; ZICOND64-LABEL: and_select_all_ones_i64:
+; ZICOND64:       # %bb.0:
+; ZICOND64-NEXT:    czero.eqz a0, a2, a0
+; ZICOND64-NEXT:    and a1, a2, a1
+; ZICOND64-NEXT:    or a0, a1, a0
+; ZICOND64-NEXT:    ret
   %a = select i1 %c, i64 -1, i64 %x
   %b = and i64 %y, %a
   ret i64 %b
 }
 
+define i64 @and_select_all_ones_i64_cmp(i64 %x, i64 %y, i64 %z) {
+; RV32-LABEL: and_select_all_ones_i64_cmp:
+; RV32:       # %bb.0:
+; RV32-NEXT:    xori a4, a4, 4
+; RV32-NEXT:    or a4, a4, a5
+; RV32-NEXT:    seqz a4, a4
+; RV32-NEXT:    addi a4, a4, -1
+; RV32-NEXT:    or a1, a4, a1
+; RV32-NEXT:    or a0, a4, a0
+; RV32-NEXT:    and a0, a0, a2
+; RV32-NEXT:    and a1, a1, a3
+; RV32-NEXT:    ret
+;
+; RV64I-LABEL: and_select_all_ones_i64_cmp:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi a2, a2, -4
+; RV64I-NEXT:    seqz a2, a2
+; RV64I-NEXT:    addi a2, a2, -1
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; SFB64-LABEL: and_select_all_ones_i64_cmp:
+; SFB64:       # %bb.0:
+; SFB64-NEXT:    li a3, 4
+; SFB64-NEXT:    bne a2, a3, .LBB4_2
+; SFB64-NEXT:  # %bb.1:
+; SFB64-NEXT:    and a1, a1, a0
+; SFB64-NEXT:  .LBB4_2:
+; SFB64-NEXT:    mv a0, a1
+; SFB64-NEXT:    ret
+;
+; VTCONDOPS64-LABEL: and_select_all_ones_i64_cmp:
+; VTCONDOPS64:       # %bb.0:
+; VTCONDOPS64-NEXT:    addi a2, a2, -4
+; VTCONDOPS64-NEXT:    and a0, a1, a0
+; VTCONDOPS64-NEXT:    vt.maskc a1, a1, a2
+; VTCONDOPS64-NEXT:    or a0, a0, a1
+; VTCONDOPS64-NEXT:    ret
+;
+; ZICOND64-LABEL: and_select_all_ones_i64_cmp:
+; ZICOND64:       # %bb.0:
+; ZICOND64-NEXT:    addi a2, a2, -4
+; ZICOND64-NEXT:    and a0, a1, a0
+; ZICOND64-NEXT:    czero.eqz a1, a1, a2
+; ZICOND64-NEXT:    or a0, a0, a1
+; ZICOND64-NEXT:    ret
+  %c = icmp eq i64 %z, 4
+  %a = select i1 %c, i64 %x, i64 -1
+  %b = and i64 %a, %y
+  ret i64 %b
+}
+
+define i64 @and_select_all_ones_i64_cmp2(i64 %x, i64 %y, i64 %z) {
+; RV32I-LABEL: and_select_all_ones_i64_cmp2:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    beqz a5, .LBB5_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    slti a4, a5, 0
+; RV32I-NEXT:    j .LBB5_3
+; RV32I-NEXT:  .LBB5_2:
+; RV32I-NEXT:    sltiu a4, a4, 4
+; RV32I-NEXT:  .LBB5_3:
+; RV32I-NEXT:    addi a4, a4, -1
+; RV32I-NEXT:    or a1, a4, a1
+; RV32I-NEXT:    or a0, a4, a0
+; RV32I-NEXT:    and a0, a0, a2
+; RV32I-NEXT:    and a1, a1, a3
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: and_select_all_ones_i64_cmp2:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slti a2, a2, 4
+; RV64I-NEXT:    addi a2, a2, -1
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; SFB64-LABEL: and_select_all_ones_i64_cmp2:
+; SFB64:       # %bb.0:
+; SFB64-NEXT:    li a3, 4
+; SFB64-NEXT:    bge a2, a3, .LBB5_2
+; SFB64-NEXT:  # %bb.1:
+; SFB64-NEXT:    and a1, a1, a0
+; SFB64-NEXT:  .LBB5_2:
+; SFB64-NEXT:    mv a0, a1
+; SFB64-NEXT:    ret
+;
+; VTCONDOPS64-LABEL: and_select_all_ones_i64_cmp2:
+; VTCONDOPS64:       # %bb.0:
+; VTCONDOPS64-NEXT:    slti a2, a2, 4
+; VTCONDOPS64-NEXT:    and a0, a1, a0
+; VTCONDOPS64-NEXT:    vt.maskcn a1, a1, a2
+; VTCONDOPS64-NEXT:    or a0, a0, a1
+; VTCONDOPS64-NEXT:    ret
+;
+; ZICOND32-LABEL: and_select_all_ones_i64_cmp2:
+; ZICOND32:       # %bb.0:
+; ZICOND32-NEXT:    slti a6, a5, 0
+; ZICOND32-NEXT:    czero.eqz a6, a6, a5
+; ZICOND32-NEXT:    sltiu a4, a4, 4
+; ZICOND32-NEXT:    czero.nez a4, a4, a5
+; ZICOND32-NEXT:    or a4, a4, a6
+; ZICOND32-NEXT:    addi a4, a4, -1
+; ZICOND32-NEXT:    or a1, a4, a1
+; ZICOND32-NEXT:    or a0, a4, a0
+; ZICOND32-NEXT:    and a0, a0, a2
+; ZICOND32-NEXT:    and a1, a1, a3
+; ZICOND32-NEXT:    ret
+;
+; ZICOND64-LABEL: and_select_all_ones_i64_cmp2:
+; ZICOND64:       # %bb.0:
+; ZICOND64-NEXT:    slti a2, a2, 4
+; ZICOND64-NEXT:    and a0, a1, a0
+; ZICOND64-NEXT:    czero.nez a1, a1, a2
+; ZICOND64-NEXT:    or a0, a0, a1
+; ZICOND64-NEXT:    ret
+  %c = icmp slt i64 %z, 4
+  %a = select i1 %c, i64 %x, i64 -1
+  %b = and i64 %a, %y
+  ret i64 %b
+}
+
 define signext i32 @or_select_all_zeros_i32(i1 zeroext %c, i32 signext %x, i32 signext %y) {
 ; RV32I-LABEL: or_select_all_zeros_i32:
 ; RV32I:       # %bb.0:
@@ -93,10 +342,10 @@ define signext i32 @or_select_all_zeros_i32(i1 zeroext %c, i32 signext %x, i32 s
 ;
 ; SFB64-LABEL: or_select_all_zeros_i32:
 ; SFB64:       # %bb.0:
-; SFB64-NEXT:    beqz a0, .LBB2_2
+; SFB64-NEXT:    beqz a0, .LBB6_2
 ; SFB64-NEXT:  # %bb.1:
 ; SFB64-NEXT:    or a2, a2, a1
-; SFB64-NEXT:  .LBB2_2:
+; SFB64-NEXT:  .LBB6_2:
 ; SFB64-NEXT:    mv a0, a2
 ; SFB64-NEXT:    ret
 ;
@@ -135,10 +384,10 @@ define i64 @or_select_all_zeros_i64(i1 zeroext %c, i64 %x, i64 %y) {
 ;
 ; SFB64-LABEL: or_select_all_zeros_i64:
 ; SFB64:       # %bb.0:
-; SFB64-NEXT:    bnez a0, .LBB3_2
+; SFB64-NEXT:    bnez a0, .LBB7_2
 ; SFB64-NEXT:  # %bb.1:
 ; SFB64-NEXT:    or a2, a2, a1
-; SFB64-NEXT:  .LBB3_2:
+; SFB64-NEXT:  .LBB7_2:
 ; SFB64-NEXT:    mv a0, a2
 ; SFB64-NEXT:    ret
 ;
@@ -183,10 +432,10 @@ define signext i32 @xor_select_all_zeros_i32(i1 zeroext %c, i32 signext %x, i32
 ;
 ; SFB64-LABEL: xor_select_all_zeros_i32:
 ; SFB64:       # %bb.0:
-; SFB64-NEXT:    bnez a0, .LBB4_2
+; SFB64-NEXT:    bnez a0, .LBB8_2
 ; SFB64-NEXT:  # %bb.1:
 ; SFB64-NEXT:    xor a2, a2, a1
-; SFB64-NEXT:  .LBB4_2:
+; SFB64-NEXT:  .LBB8_2:
 ; SFB64-NEXT:    mv a0, a2
 ; SFB64-NEXT:    ret
 ;
@@ -225,10 +474,10 @@ define i64 @xor_select_all_zeros_i64(i1 zeroext %c, i64 %x, i64 %y) {
 ;
 ; SFB64-LABEL: xor_select_all_zeros_i64:
 ; SFB64:       # %bb.0:
-; SFB64-NEXT:    beqz a0, .LBB5_2
+; SFB64-NEXT:    beqz a0, .LBB9_2
 ; SFB64-NEXT:  # %bb.1:
 ; SFB64-NEXT:    xor a2, a2, a1
-; SFB64-NEXT:  .LBB5_2:
+; SFB64-NEXT:  .LBB9_2:
 ; SFB64-NEXT:    mv a0, a2
 ; SFB64-NEXT:    ret
 ;
@@ -273,10 +522,10 @@ define signext i32 @add_select_all_zeros_i32(i1 zeroext %c, i32 signext %x, i32
 ;
 ; SFB64-LABEL: add_select_all_zeros_i32:
 ; SFB64:       # %bb.0:
-; SFB64-NEXT:    bnez a0, .LBB6_2
+; SFB64-NEXT:    bnez a0, .LBB10_2
 ; SFB64-NEXT:  # %bb.1:
 ; SFB64-NEXT:    addw a2, a2, a1
-; SFB64-NEXT:  .LBB6_2:
+; SFB64-NEXT:  .LBB10_2:
 ; SFB64-NEXT:    mv a0, a2
 ; SFB64-NEXT:    ret
 ;
@@ -323,10 +572,10 @@ define i64 @add_select_all_zeros_i64(i1 zeroext %c, i64 %x, i64 %y) {
 ;
 ; SFB64-LABEL: add_select_all_zeros_i64:
 ; SFB64:       # %bb.0:
-; SFB64-NEXT:    beqz a0, .LBB7_2
+; SFB64-NEXT:    beqz a0, .LBB11_2
 ; SFB64-NEXT:  # %bb.1:
 ; SFB64-NEXT:    add a2, a2, a1
-; SFB64-NEXT:  .LBB7_2:
+; SFB64-NEXT:  .LBB11_2:
 ; SFB64-NEXT:    mv a0, a2
 ; SFB64-NEXT:    ret
 ;
@@ -373,10 +622,10 @@ define signext i32 @sub_select_all_zeros_i32(i1 zeroext %c, i32 signext %x, i32
 ;
 ; SFB64-LABEL: sub_select_all_zeros_i32:
 ; SFB64:       # %bb.0:
-; SFB64-NEXT:    bnez a0, .LBB8_2
+; SFB64-NEXT:    bnez a0, .LBB12_2
 ; SFB64-NEXT:  # %bb.1:
 ; SFB64-NEXT:    subw a2, a2, a1
-; SFB64-NEXT:  .LBB8_2:
+; SFB64-NEXT:  .LBB12_2:
 ; SFB64-NEXT:    mv a0, a2
 ; SFB64-NEXT:    ret
 ;
@@ -423,10 +672,10 @@ define i64 @sub_select_all_zeros_i64(i1 zeroext %c, i64 %x, i64 %y) {
 ;
 ; SFB64-LABEL: sub_select_all_zeros_i64:
 ; SFB64:       # %bb.0:
-; SFB64-NEXT:    beqz a0, .LBB9_2
+; SFB64-NEXT:    beqz a0, .LBB13_2
 ; SFB64-NEXT:  # %bb.1:
 ; SFB64-NEXT:    sub a2, a2, a1
-; SFB64-NEXT:  .LBB9_2:
+; SFB64-NEXT:  .LBB13_2:
 ; SFB64-NEXT:    mv a0, a2
 ; SFB64-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/spir-kernel-cc.ll b/llvm/test/CodeGen/RISCV/spir-kernel-cc.ll
new file mode 100644
index 0000000000..283f397373
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/spir-kernel-cc.ll
@@ -0,0 +1,16 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 < %s | FileCheck %s -check-prefix=RV32
+; RUN: llc -mtriple=riscv64 < %s | FileCheck %s -check-prefix=RV64
+
+; Check the SPIR_KERNEL call convention works.
+
+define dso_local spir_kernel void @foo() {
+; RV32-LABEL: foo:
+; RV32:       # %bb.0:
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: foo:
+; RV64:       # %bb.0:
+; RV64-NEXT:    ret
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/srem-seteq-illegal-types.ll b/llvm/test/CodeGen/RISCV/srem-seteq-illegal-types.ll
index 6ed352b51f..d311311175 100644
--- a/llvm/test/CodeGen/RISCV/srem-seteq-illegal-types.ll
+++ b/llvm/test/CodeGen/RISCV/srem-seteq-illegal-types.ll
@@ -725,24 +725,79 @@ define void @test_srem_vec(ptr %X) nounwind {
 ;
 ; RV64MV-LABEL: test_srem_vec:
 ; RV64MV:       # %bb.0:
+; RV64MV-NEXT:    ld a1, 0(a0)
+; RV64MV-NEXT:    lwu a2, 8(a0)
+; RV64MV-NEXT:    srli a3, a1, 2
+; RV64MV-NEXT:    lbu a4, 12(a0)
+; RV64MV-NEXT:    slli a5, a2, 62
+; RV64MV-NEXT:    or a3, a5, a3
+; RV64MV-NEXT:    srai a3, a3, 31
+; RV64MV-NEXT:    slli a4, a4, 32
+; RV64MV-NEXT:    or a2, a2, a4
+; RV64MV-NEXT:    slli a2, a2, 29
+; RV64MV-NEXT:    lui a4, %hi(.LCPI3_0)
+; RV64MV-NEXT:    ld a4, %lo(.LCPI3_0)(a4)
+; RV64MV-NEXT:    srai a2, a2, 31
+; RV64MV-NEXT:    slli a1, a1, 31
+; RV64MV-NEXT:    srai a1, a1, 31
+; RV64MV-NEXT:    mulh a4, a2, a4
+; RV64MV-NEXT:    srli a5, a4, 63
+; RV64MV-NEXT:    srai a4, a4, 1
+; RV64MV-NEXT:    add a4, a4, a5
+; RV64MV-NEXT:    lui a5, %hi(.LCPI3_1)
+; RV64MV-NEXT:    ld a5, %lo(.LCPI3_1)(a5)
+; RV64MV-NEXT:    add a2, a2, a4
+; RV64MV-NEXT:    slli a4, a4, 2
+; RV64MV-NEXT:    add a2, a2, a4
+; RV64MV-NEXT:    mulh a4, a3, a5
+; RV64MV-NEXT:    srli a5, a4, 63
+; RV64MV-NEXT:    srai a4, a4, 1
+; RV64MV-NEXT:    add a4, a4, a5
+; RV64MV-NEXT:    lui a5, %hi(.LCPI3_2)
+; RV64MV-NEXT:    ld a5, %lo(.LCPI3_2)(a5)
+; RV64MV-NEXT:    add a3, a3, a4
+; RV64MV-NEXT:    slli a4, a4, 3
+; RV64MV-NEXT:    sub a3, a3, a4
+; RV64MV-NEXT:    mulh a4, a1, a5
+; RV64MV-NEXT:    srli a5, a4, 63
+; RV64MV-NEXT:    add a4, a4, a5
+; RV64MV-NEXT:    li a5, 6
+; RV64MV-NEXT:    mul a4, a4, a5
+; RV64MV-NEXT:    sub a1, a1, a4
 ; RV64MV-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
-; RV64MV-NEXT:    vmv.v.i v8, -1
+; RV64MV-NEXT:    vslide1down.vx v8, v8, a1
+; RV64MV-NEXT:    vslide1down.vx v8, v8, a3
+; RV64MV-NEXT:    vslide1down.vx v8, v8, a2
+; RV64MV-NEXT:    vslidedown.vi v8, v8, 1
+; RV64MV-NEXT:    li a1, -1
+; RV64MV-NEXT:    srli a1, a1, 31
+; RV64MV-NEXT:    vand.vx v8, v8, a1
+; RV64MV-NEXT:    lui a2, 32
+; RV64MV-NEXT:    addi a2, a2, 256
+; RV64MV-NEXT:    vmv.s.x v10, a2
+; RV64MV-NEXT:    vsext.vf8 v12, v10
+; RV64MV-NEXT:    vmsne.vv v0, v8, v12
+; RV64MV-NEXT:    vmv.v.i v8, 0
+; RV64MV-NEXT:    vmerge.vim v8, v8, -1, v0
 ; RV64MV-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
 ; RV64MV-NEXT:    vslidedown.vi v10, v8, 2
-; RV64MV-NEXT:    vmv.x.s a1, v10
-; RV64MV-NEXT:    slli a2, a1, 31
-; RV64MV-NEXT:    srli a2, a2, 61
-; RV64MV-NEXT:    sb a2, 12(a0)
+; RV64MV-NEXT:    vmv.x.s a2, v10
+; RV64MV-NEXT:    slli a3, a2, 31
+; RV64MV-NEXT:    srli a3, a3, 61
+; RV64MV-NEXT:    sb a3, 12(a0)
+; RV64MV-NEXT:    vmv.x.s a3, v8
+; RV64MV-NEXT:    and a1, a3, a1
 ; RV64MV-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; RV64MV-NEXT:    vslidedown.vi v8, v8, 1
-; RV64MV-NEXT:    vmv.x.s a2, v8
-; RV64MV-NEXT:    slli a3, a2, 33
-; RV64MV-NEXT:    sd a3, 0(a0)
-; RV64MV-NEXT:    slli a1, a1, 2
-; RV64MV-NEXT:    slli a2, a2, 31
-; RV64MV-NEXT:    srli a2, a2, 62
-; RV64MV-NEXT:    or a1, a2, a1
-; RV64MV-NEXT:    sw a1, 8(a0)
+; RV64MV-NEXT:    vmv.x.s a3, v8
+; RV64MV-NEXT:    slli a4, a3, 33
+; RV64MV-NEXT:    or a1, a1, a4
+; RV64MV-NEXT:    sd a1, 0(a0)
+; RV64MV-NEXT:    slli a2, a2, 2
+; RV64MV-NEXT:    slli a3, a3, 31
+; RV64MV-NEXT:    srli a3, a3, 62
+; RV64MV-NEXT:    or a2, a3, a2
+; RV64MV-NEXT:    sw a2, 8(a0)
 ; RV64MV-NEXT:    ret
   %ld = load <3 x i33>, ptr %X
   %srem = srem <3 x i33> %ld, <i33 6, i33 7, i33 -5>
diff --git a/llvm/test/CodeGen/SPIRV/pointers/getelementptr-base-type.ll b/llvm/test/CodeGen/SPIRV/pointers/getelementptr-base-type.ll
new file mode 100644
index 0000000000..aaf97f8cc8
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/pointers/getelementptr-base-type.ll
@@ -0,0 +1,18 @@
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+
+; CHECK:  %[[#FLOAT32:]] = OpTypeFloat 32
+; CHECK:  %[[#PTR:]] = OpTypePointer CrossWorkgroup %[[#FLOAT32]]
+; CHECK:  %[[#ARG:]] = OpFunctionParameter %[[#PTR]]
+; CHECK:  %[[#GEP:]] = OpInBoundsPtrAccessChain %[[#PTR]] %[[#ARG]] %[[#]]
+; CHECK:  %[[#]] = OpLoad %[[#FLOAT32]] %[[#GEP]] Aligned 4
+
+define spir_kernel void @test1(ptr addrspace(1) %arg1) !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !3 !kernel_arg_type_qual !4 {
+  %a = getelementptr inbounds float, ptr addrspace(1) %arg1, i64 1
+  %b = load float, ptr addrspace(1) %a, align 4
+  ret void
+}
+
+!1 = !{i32 1}
+!2 = !{!"none"}
+!3 = !{!"float*"}
+!4 = !{!""}
diff --git a/llvm/test/CodeGen/SPIRV/pointers/kernel-argument-pointer-addressspace.ll b/llvm/test/CodeGen/SPIRV/pointers/kernel-argument-pointer-addressspace.ll
new file mode 100644
index 0000000000..6d12023281
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/pointers/kernel-argument-pointer-addressspace.ll
@@ -0,0 +1,63 @@
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+
+; CHECK-DAG:  %[[#INT:]] = OpTypeInt 32 0
+; CHECK-DAG:  %[[#PTR1:]] = OpTypePointer Function %[[#INT]]
+; CHECK-DAG:  %[[#ARG:]] = OpFunctionParameter %[[#PTR1]]
+
+define spir_kernel void @test1(ptr addrspace(0) %arg1) !kernel_arg_addr_space !1 !kernel_arg_type !2 {
+  %a = getelementptr inbounds i32, ptr addrspace(0) %arg1, i32 2
+  ret void
+}
+
+!1 = !{i32 0}
+!2 = !{!"int*"}
+
+; CHECK-DAG:  %[[#PTR2:]] = OpTypePointer CrossWorkgroup %[[#INT]]
+; CHECK-DAG:  %[[#ARG:]] = OpFunctionParameter %[[#PTR2]]
+
+define spir_kernel void @test2(ptr addrspace(1) %arg1) !kernel_arg_addr_space !3 !kernel_arg_type !2 {
+  %a = getelementptr inbounds i32, ptr addrspace(1) %arg1, i32 2
+  ret void
+}
+
+!3 = !{i32 1}
+
+; CHECK-DAG:  %[[#PTR3:]] = OpTypePointer UniformConstant %[[#INT]]
+; CHECK-DAG:  %[[#ARG:]] = OpFunctionParameter %[[#PTR3]]
+
+define spir_kernel void @test3(ptr addrspace(2) %arg1) !kernel_arg_addr_space !4 !kernel_arg_type !2 {
+  %a = getelementptr inbounds i32, ptr addrspace(2) %arg1, i32 2
+  ret void
+}
+
+!4 = !{i32 2}
+
+; CHECK-DAG:  %[[#PTR4:]] = OpTypePointer Workgroup %[[#INT]]
+; CHECK-DAG:  %[[#ARG:]] = OpFunctionParameter %[[#PTR4]]
+
+define spir_kernel void @test4(ptr addrspace(3) %arg1) !kernel_arg_addr_space !5 !kernel_arg_type !2 {
+  %a = getelementptr inbounds i32, ptr addrspace(3) %arg1, i32 2
+  ret void
+}
+
+!5 = !{i32 3}
+
+; CHECK-DAG:  %[[#PTR5:]] = OpTypePointer Generic %[[#INT]]
+; CHECK-DAG:  %[[#ARG:]] = OpFunctionParameter %[[#PTR5]]
+
+define spir_kernel void @test5(ptr addrspace(4) %arg1) !kernel_arg_addr_space !6 !kernel_arg_type !2 {
+  %a = getelementptr inbounds i32, ptr addrspace(4) %arg1, i32 2
+  ret void
+}
+
+!6 = !{i32 4}
+
+; CHECK-DAG:  %[[#PTR6:]] = OpTypePointer Input %[[#INT]]
+; CHECK-DAG:  %[[#ARG:]] = OpFunctionParameter %[[#PTR6]]
+
+define spir_kernel void @test6(ptr addrspace(7) %arg1) !kernel_arg_addr_space !7 !kernel_arg_type !2 {
+  %a = getelementptr inbounds i32, ptr addrspace(7) %arg1, i32 2
+  ret void
+}
+
+!7 = !{i32 7}
diff --git a/llvm/test/CodeGen/SPIRV/pointers/kernel-argument-pointer-type.ll b/llvm/test/CodeGen/SPIRV/pointers/kernel-argument-pointer-type.ll
new file mode 100644
index 0000000000..1fcc6d9da9
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/pointers/kernel-argument-pointer-type.ll
@@ -0,0 +1,78 @@
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+
+; CHECK-DAG:  %[[#FLOAT32:]] = OpTypeFloat 32
+; CHECK-DAG:  %[[#PTR1:]] = OpTypePointer Function %[[#FLOAT32]]
+; CHECK-DAG:  %[[#ARG:]] = OpFunctionParameter %[[#PTR1]]
+
+define spir_kernel void @test1(ptr %arg) !kernel_arg_type !1 {
+  %a = getelementptr inbounds float, ptr %arg, i64 1
+  ret void
+}
+
+!1 = !{!"float*"}
+
+; CHECK-DAG:  %[[#CHAR:]] = OpTypeInt 8 0
+; CHECK-DAG:  %[[#PTR2:]] = OpTypePointer Function %[[#CHAR]]
+; CHECK-DAG:  %[[#ARG:]] = OpFunctionParameter %[[#PTR2]]
+
+define spir_kernel void @test2(ptr %arg) !kernel_arg_type !2 {
+  %a = getelementptr inbounds i8, ptr %arg, i64 1
+  ret void
+}
+
+!2 = !{!"char*"}
+
+; CHECK-DAG:  %[[#SHORT:]] = OpTypeInt 16 0
+; CHECK-DAG:  %[[#PTR3:]] = OpTypePointer Function %[[#SHORT]]
+; CHECK-DAG:  %[[#ARG:]] = OpFunctionParameter %[[#PTR3]]
+
+define spir_kernel void @test3(ptr %arg) !kernel_arg_type !3 {
+  %a = getelementptr inbounds i16, ptr %arg, i64 1
+  ret void
+}
+
+!3 = !{!"short*"}
+
+; CHECK-DAG:  %[[#INT:]] = OpTypeInt 32 0
+; CHECK-DAG:  %[[#PTR4:]] = OpTypePointer Function %[[#INT]]
+; CHECK-DAG:  %[[#ARG:]] = OpFunctionParameter %[[#PTR4]]
+
+define spir_kernel void @test4(ptr %arg) !kernel_arg_type !4 {
+  %a = getelementptr inbounds i32, ptr %arg, i64 1
+  ret void
+}
+
+!4 = !{!"int*"}
+
+; CHECK-DAG:  %[[#LONG:]] = OpTypeInt 64 0
+; CHECK-DAG:  %[[#PTR5:]] = OpTypePointer Function %[[#LONG]]
+; CHECK-DAG:  %[[#ARG:]] = OpFunctionParameter %[[#PTR5]]
+
+define spir_kernel void @test5(ptr %arg) !kernel_arg_type !5 {
+  %a = getelementptr inbounds i64, ptr %arg, i64 1
+  ret void
+}
+
+!5 = !{!"long*"}
+
+; CHECK-DAG:  %[[#DOUBLE:]] = OpTypeFloat 64
+; CHECK-DAG:  %[[#PTR6:]] = OpTypePointer Function %[[#DOUBLE]]
+; CHECK-DAG:  %[[#ARG:]] = OpFunctionParameter %[[#PTR6]]
+
+define spir_kernel void @test6(ptr %arg) !kernel_arg_type !6 {
+  %a = getelementptr inbounds double, ptr %arg, i64 1
+  ret void
+}
+
+!6 = !{!"double*"}
+
+; CHECK-DAG:  %[[#HALF:]] = OpTypeFloat 16
+; CHECK-DAG:  %[[#PTR7:]] = OpTypePointer Function %[[#HALF]]
+; CHECK-DAG:  %[[#ARG:]] = OpFunctionParameter %[[#PTR7]]
+
+define spir_kernel void @test7(ptr %arg) !kernel_arg_type !7 {
+  %a = getelementptr inbounds half, ptr %arg, i64 1
+  ret void
+}
+
+!7 = !{!"half*"}
diff --git a/llvm/test/CodeGen/SystemZ/call-zos-vararg.ll b/llvm/test/CodeGen/SystemZ/call-zos-vararg.ll
index ac157e5fa6..bde59a6be7 100644
--- a/llvm/test/CodeGen/SystemZ/call-zos-vararg.ll
+++ b/llvm/test/CodeGen/SystemZ/call-zos-vararg.ll
@@ -1,88 +1,149 @@
 ; Test passing variable argument lists in 64-bit calls on z/OS.
 ; RUN: llc < %s -mtriple=s390x-ibm-zos -mcpu=z10 | FileCheck %s
 ; RUN: llc < %s -mtriple=s390x-ibm-zos -mcpu=z14 | FileCheck %s -check-prefix=ARCH12
-; CHECK-LABEL: call_vararg_double0
-; CHECK:       llihf 3, 1074118262
-; CHECK-NEXT:  oilf  3, 3367254360
-; CHECK:       lghi  1, 1
-; CHECK:       lghi  2, 2
+; CHECK-LABEL: call_vararg_double0:
+; CHECK:         stmg 6, 7, 1872(4)
+; CHECK-NEXT:    aghi 4, -192
+; CHECK-NEXT:    lg 6, 8(5)
+; CHECK-NEXT:    lg 5, 0(5)
+; CHECK-NEXT:    llihf 3, 1074118262
+; CHECK-NEXT:    oilf 3, 3367254360
+; CHECK-NEXT:    lghi 1, 1
+; CHECK-NEXT:    lghi 2, 2
+; CHECK-NEXT:    basr 7, 6
+; CHECK-NEXT:    bcr 0, 0
+; CHECK-NEXT:    lg 7, 2072(4)
+; CHECK-NEXT:    aghi 4, 192
+; CHECK-NEXT:    b 2(7)
 define i64 @call_vararg_double0() {
 entry:
   %retval = call i64 (i64, i64, ...) @pass_vararg0(i64 1, i64 2, double 2.718000e+00)
   ret i64 %retval
 }
 
-; CHECK-LABEL:  call_vararg_double1
-; CHECK:        llihf 0, 1074118262
-; CHECK-NEXT:   oilf  0, 3367254360
-; CHECK:        llihf 3, 1074340036
-; CHECK-NEXT:   oilf  3, 2611340116
-; CHECK:        lghi  1, 1
-; CHECK:        lghi  2, 2
-; CHECK:        stg 0, 2200(4)
+; CHECK-LABEL: call_vararg_double1:
+; CHECK:         stmg 6, 7, 1872(4)
+; CHECK-NEXT:    aghi 4, -192
+; CHECK-NEXT:    llihf 0, 1074118262
+; CHECK-NEXT:    oilf 0, 3367254360
+; CHECK-NEXT:    lg 6, 8(5)
+; CHECK-NEXT:    lg 5, 0(5)
+; CHECK-NEXT:    llihf 3, 1074340036
+; CHECK-NEXT:    oilf 3, 2611340116
+; CHECK-NEXT:    lghi 1, 1
+; CHECK-NEXT:    lghi 2, 2
+; CHECK-NEXT:    stg 0, 2200(4)
+; CHECK-NEXT:    basr 7, 6
+; CHECK-NEXT:    bcr 0, 0
+; CHECK-NEXT:    lg 7, 2072(4)
+; CHECK-NEXT:    aghi 4, 192
+; CHECK-NEXT:    b 2(7)
 define i64 @call_vararg_double1() {
 entry:
   %retval = call i64 (i64, i64, ...) @pass_vararg0(i64 1, i64 2, double 3.141000e+00, double 2.718000e+00)
   ret i64 %retval
 }
 
-; CHECK-LABEL: call_vararg_double2
-; CHECK-NOT:   llihf 0
-; CHECK-NOT:   oilf 0
-; CHECK:       llihf 2, 1074118262
-; CHECK-NEXT:  oilf  2, 3367254360
-; CHECK:       lghi  1, 8200
+; CHECK-LABEL: call_vararg_double2:
+; CHECK:         stmg 6, 7, 1872(4)
+; CHECK-NEXT:    aghi 4, -192
+; CHECK-NEXT:    lg 6, 24(5)
+; CHECK-NEXT:    lg 5, 16(5)
+; CHECK-NEXT:    llihf 2, 1074118262
+; CHECK-NEXT:    oilf 2, 3367254360
+; CHECK-NEXT:    lghi 1, 8200
+; CHECK-NEXT:    basr 7, 6
+; CHECK-NEXT:    bcr 0, 0
+; CHECK-NEXT:    lg 7, 2072(4)
+; CHECK-NEXT:    aghi 4, 192
+; CHECK-NEXT:    b 2(7)
 define i64 @call_vararg_double2() {
 entry:
   %retval = call i64 (i64, ...) @pass_vararg2(i64 8200, double 2.718000e+00)
   ret i64 %retval
 }
 
-; CHECK-LABEL: call_vararg_double3
-; CHECK:       llihf   0, 1072703839
-; CHECK-NEXT:  oilf    0, 2861204133
-; CHECK:       llihf   1, 1074118262
-; CHECK-NEXT:  oilf    1, 3367254360
-; CHECK:       llihf   2, 1074340036
-; CHECK-NEXT:  oilf    2, 2611340116
-; CHECK:       llihf   3, 1073127358
-; CHECK-NEXT:  oilf    3, 1992864825
-; CHECK:       stg     0, 2200(4)
+; CHECK-LABEL: call_vararg_double3:
+; CHECK:         stmg 6, 7, 1872(4)
+; CHECK-NEXT:    aghi 4, -192
+; CHECK-NEXT:    llihf 0, 1072703839
+; CHECK-NEXT:    oilf 0, 2861204133
+; CHECK-NEXT:    lg 6, 40(5)
+; CHECK-NEXT:    lg 5, 32(5)
+; CHECK-NEXT:    llihf 1, 1074118262
+; CHECK-NEXT:    oilf 1, 3367254360
+; CHECK-NEXT:    llihf 2, 1074340036
+; CHECK-NEXT:    oilf 2, 2611340116
+; CHECK-NEXT:    llihf 3, 1073127358
+; CHECK-NEXT:    oilf 3, 1992864825
+; CHECK-NEXT:    stg 0, 2200(4)
+; CHECK-NEXT:    basr 7, 6
+; CHECK-NEXT:    bcr 0, 0
+; CHECK-NEXT:    lg 7, 2072(4)
+; CHECK-NEXT:    aghi 4, 192
+; CHECK-NEXT:    b 2(7)
 define i64 @call_vararg_double3() {
 entry:
   %retval = call i64 (...) @pass_vararg3(double 2.718000e+00, double 3.141000e+00, double 1.414000e+00, double 1.010101e+00)
   ret i64 %retval
 }
 
-; CHECK-LABEL: call_vararg_both0
-; CHECK:       lgr   2, 1
-; CHECK:       lgdr  1, 0
+; CHECK-LABEL: call_vararg_both0:
+; CHECK:         stmg 6, 7, 1872(4)
+; CHECK-NEXT:    aghi 4, -192
+; CHECK-NEXT:    lg 6, 40(5)
+; CHECK-NEXT:    lg 5, 32(5)
+; CHECK-NEXT:    lgr 2, 1
+; CHECK-NEXT:    lgdr 1, 0
+; CHECK-NEXT:    basr 7, 6
+; CHECK-NEXT:    bcr 0, 0
+; CHECK-NEXT:    lg 7, 2072(4)
+; CHECK-NEXT:    aghi 4, 192
+; CHECK-NEXT:    b 2(7)
 define i64 @call_vararg_both0(i64 %arg0, double %arg1) {
   %retval  = call i64(...) @pass_vararg3(double %arg1, i64 %arg0)
   ret i64 %retval
 }
 
-; CHECK-LABEL: call_vararg_long_double0
-; CHECK:       larl  1, @CPI5_0
-; CHECK-NEXT:  ld    0, 0(1)
-; CHECK-NEXT:  ld    2, 8(1)
-; CHECK:       lgdr  3, 0
-; CHECK:       lghi  1, 1
-; CHECK:       lghi  2, 2
-; CHECK:       std   0, 2192(4)
-; CHECK-NEXT:  std   2, 2200(4)
+; CHECK-LABEL: call_vararg_long_double0:
+; CHECK:         stmg 6, 7, 1872(4)
+; CHECK-NEXT:    aghi 4, -192
+; CHECK-NEXT:    larl 1, @CPI5_0
+; CHECK-NEXT:    ld 0, 0(1)
+; CHECK-NEXT:    ld 2, 8(1)
+; CHECK-NEXT:    lg 6, 8(5)
+; CHECK-NEXT:    lg 5, 0(5)
+; CHECK-NEXT:    lgdr 3, 0
+; CHECK-NEXT:    lghi 1, 1
+; CHECK-NEXT:    lghi 2, 2
+; CHECK-NEXT:    std 0, 2192(4)
+; CHECK-NEXT:    std 2, 2200(4)
+; CHECK-NEXT:    basr 7, 6
+; CHECK-NEXT:    bcr 0, 0
+; CHECK-NEXT:    lg 7, 2072(4)
+; CHECK-NEXT:    aghi 4, 192
+; CHECK-NEXT:    b 2(7)
 define i64 @call_vararg_long_double0() {
 entry:
   %retval = call i64 (i64, i64, ...) @pass_vararg0(i64 1, i64 2, fp128 0xLE0FC1518450562CD4000921FB5444261)
   ret i64 %retval
 }
 
-; CHECK-LABEL: call_vararg_long_double1
-; CHECK:       lgdr  3, 0
-; CHECK:       lghi  1, 1
-; CHECK:       lghi  2, 2
-; CHECK:       std   0, 2192(4)
-; CHECK-NEXT:  std   2, 2200(4)
+; CHECK-LABEL: call_vararg_long_double1:
+; CHECK:         stmg 6, 7, 1872(4)
+; CHECK-NEXT:    aghi 4, -192
+; CHECK-NEXT:    lg 6, 8(5)
+; CHECK-NEXT:    lg 5, 0(5)
+; CHECK-NEXT:    lgdr 3, 0
+; CHECK-NEXT:    lghi 1, 1
+; CHECK-NEXT:    lghi 2, 2
+; CHECK-NEXT:    std 0, 2192(4)
+; CHECK-NEXT:    std 2, 2200(4)
+; CHECK-NEXT:    basr 7, 6
+; CHECK-NEXT:    bcr 0, 0
+; CHECK-NEXT:    lg 7, 2072(4)
+; CHECK-NEXT:    aghi 4, 192
+; CHECK-NEXT:    b 2(7)
 define i64 @call_vararg_long_double1(fp128 %arg0) {
 entry:
   %retval = call i64 (i64, i64, ...) @pass_vararg0(i64 1, i64 2, fp128 %arg0)
@@ -90,22 +151,41 @@ entry:
 }
 
 ; CHECK-LABEL: call_vararg_long_double2
-; CHECK:      std   4, 2208(4)
-; CHECK-NEXT: std   6, 2216(4)
-; CHECK:      lgdr  3, 0
-; CHECK:      lghi  1, 1
-; CHECK:      lghi  2, 2
-; CHECK:      std   0, 2192(4)
-; CHECK-NEXT: std   2, 2200(4)
+; CHECK-LABEL: call_vararg_long_double2:
+; CHECK:         stmg 6, 7, 1872(4)
+; CHECK-NEXT:    aghi 4, -192
+; CHECK-NEXT:    std 4, 2208(4)
+; CHECK-NEXT:    std 6, 2216(4)
+; CHECK-NEXT:    lg 6, 8(5)
+; CHECK-NEXT:    lg 5, 0(5)
+; CHECK-NEXT:    lgdr 3, 0
+; CHECK-NEXT:    lghi 1, 1
+; CHECK-NEXT:    lghi 2, 2
+; CHECK-NEXT:    std 0, 2192(4)
+; CHECK-NEXT:    std 2, 2200(4)
+; CHECK-NEXT:    basr 7, 6
+; CHECK-NEXT:    bcr 0, 0
+; CHECK-NEXT:    lg 7, 2072(4)
+; CHECK-NEXT:    aghi 4, 192
+; CHECK-NEXT:    b 2(7)
 define i64 @call_vararg_long_double2(fp128 %arg0, fp128 %arg1) {
 entry:
   %retval = call i64 (i64, i64, ...) @pass_vararg0(i64 1, i64 2, fp128 %arg0, fp128 %arg1)
   ret i64 %retval
 }
 
-; CHECK-LABEL: call_vararg_long_double3
-; CHECK:       lgdr 3, 2
-; CHECK-NEXT:  lgdr 2, 0
+; CHECK-LABEL: call_vararg_long_double3:
+; CHECK:         stmg 6, 7, 1872(4)
+; CHECK-NEXT:    aghi 4, -192
+; CHECK-NEXT:    lg 6, 40(5)
+; CHECK-NEXT:    lg 5, 32(5)
+; CHECK-NEXT:    lgdr 3, 2
+; CHECK-NEXT:    lgdr 2, 0
+; CHECK-NEXT:    basr 7, 6
+; CHECK-NEXT:    bcr 0, 0
+; CHECK-NEXT:    lg 7, 2072(4)
+; CHECK-NEXT:    aghi 4, 192
+; CHECK-NEXT:    b 2(7)
 define i64 @call_vararg_long_double3(fp128 %arg0) {
 entry:
   %retval = call i64 (...) @pass_vararg3(fp128 %arg0)
@@ -173,38 +253,58 @@ define void @call_vec_double_vararg_straddle(<2 x double> %v) {
   ret void
 }
 
-; CHECK-LABEL: call_vararg_integral0
-; Since arguments 0, 1, and 2 are already in the correct
-; registers, we should have no loads of any sort into
-; GPRs 1, 2, and 3.
-; CHECK-NOT: lg  1
-; CHECK-NOT: lgr  1
-; CHECK-NOT: lg  2
-; CHECK-NOT: lgr  2
-; CHECK-NOT: lg  3
-; CHECK-NOT: lgr  3
+; CHECK-LABEL: call_vararg_integral0:
+; CHECK:         stmg 6, 7, 1872(4)
+; CHECK-NEXT:    aghi 4, -192
+; CHECK-NEXT:    lg 0, 2392(4)
+; CHECK-NEXT:    lg 6, 40(5)
+; CHECK-NEXT:    lg 5, 32(5)
+; CHECK-NEXT:    stg 0, 2200(4)
+; CHECK-NEXT:    basr 7, 6
+; CHECK-NEXT:    bcr 0, 0
+; CHECK-NEXT:    lg 7, 2072(4)
+; CHECK-NEXT:    aghi 4, 192
+; CHECK-NEXT:    b 2(7)
 define i64 @call_vararg_integral0(i32 signext %arg0, i16 signext %arg1, i64 signext %arg2, i8 signext %arg3) {
 entry:
   %retval = call i64(...) @pass_vararg3(i32 signext %arg0, i16 signext %arg1, i64 signext %arg2, i8 signext %arg3)
   ret i64 %retval
 }
 
-; CHECK-LABEL: call_vararg_float0
-; CHECK:       lghi  1, 1
-; CHECK:       llihf 2, 1073692672
+; CHECK-LABEL: call_vararg_float0:
+; CHECK:         stmg 6, 7, 1872(4)
+; CHECK-NEXT:    aghi 4, -192
+; CHECK-NEXT:    lg 6, 24(5)
+; CHECK-NEXT:    lg 5, 16(5)
+; CHECK-NEXT:    lghi 1, 1
+; CHECK-NEXT:    llihf 2, 1073692672
+; CHECK-NEXT:    basr 7, 6
+; CHECK-NEXT:    bcr 0, 0
+; CHECK-NEXT:    lg 7, 2072(4)
+; CHECK-NEXT:    aghi 4, 192
+; CHECK-NEXT:    b 2(7)
 define i64 @call_vararg_float0() {
 entry:
   %retval = call i64 (i64, ...) @pass_vararg2(i64 1, float 1.953125)
   ret i64 %retval
 }
 
-; CHECK-LABEL: call_vararg_float1
-; CHECK:       larl  1, @CPI17_0
-; CHECK:       le  0, 0(1)
-; CHECK:       llihf 0, 1073692672
-; CHECK:       llihh 2, 16384
-; CHECK:       llihh 3, 16392
-; CHECK:       stg  0, 2200(4)
+; CHECK-LABEL: call_vararg_float1:
+; CHECK:         stmg 6, 7, 1872(4)
+; CHECK-NEXT:    aghi 4, -192
+; CHECK-NEXT:    lg 6, 72(5)
+; CHECK-NEXT:    lg 5, 64(5)
+; CHECK-NEXT:    larl 1, @CPI17_0
+; CHECK-NEXT:    le 0, 0(1)
+; CHECK-NEXT:    llihf 0, 1073692672
+; CHECK-NEXT:    llihh 2, 16384
+; CHECK-NEXT:    llihh 3, 16392
+; CHECK-NEXT:    stg 0, 2200(4)
+; CHECK-NEXT:    basr 7, 6
+; CHECK-NEXT:    bcr 0, 0
+; CHECK-NEXT:    lg 7, 2072(4)
+; CHECK-NEXT:    aghi 4, 192
+; CHECK-NEXT:    b 2(7)
 define i64 @call_vararg_float1() {
 entry:
   %retval = call i64 (float, ...) @pass_vararg4(float 1.0, float 2.0, float 3.0, float 1.953125)
@@ -224,9 +324,16 @@ entry:
 ; }
 ;
 ; CHECK-LABEL: pass_vararg:
-; CHECK: aghi    4, -160
-; CHECK: la      0, 2208(4)
-; CHECK: stg     0, 2200(4)
+; CHECK:         stmg 6, 7, 1904(4)
+; CHECK-NEXT:    aghi 4, -160
+; CHECK-NEXT:    stg 2, 2344(4)
+; CHECK-NEXT:    stg 3, 2352(4)
+; CHECK-NEXT:    la 0, 2352(4)
+; CHECK-NEXT:    stg 0, 2200(4)
+; CHECK-NEXT:    lg 3, 2344(4)
+; CHECK-NEXT:    lg 7, 2072(4)
+; CHECK-NEXT:    aghi 4, 160
+; CHECK-NEXT:    b 2(7)
 define hidden i64 @pass_vararg(i64 %x, ...) {
 entry:
   %va = alloca ptr, align 8
diff --git a/llvm/test/CodeGen/SystemZ/frameaddr-01.ll b/llvm/test/CodeGen/SystemZ/frameaddr-01.ll
index db3d7b33f9..e8521883f0 100644
--- a/llvm/test/CodeGen/SystemZ/frameaddr-01.ll
+++ b/llvm/test/CodeGen/SystemZ/frameaddr-01.ll
@@ -25,4 +25,25 @@ entry:
   ret ptr %1
 }
 
+; Check the caller's frame address.
+define ptr @fpcaller() nounwind "backchain" {
+entry:
+; CHECK-LABEL: fpcaller:
+; CHECK: lg   %r2, 0(%r15)
+; CHECK: br   %r14
+  %0 = tail call ptr @llvm.frameaddress(i32 1)
+  ret ptr %0
+}
+
+; Check the caller's frame address.
+define ptr @fpcallercaller() nounwind "backchain" {
+entry:
+; CHECK-LABEL: fpcallercaller:
+; CHECK: lg   %r1, 0(%r15)
+; CHECK: lg   %r2, 0(%r1)
+; CHECK: br   %r14
+  %0 = tail call ptr @llvm.frameaddress(i32 2)
+  ret ptr %0
+}
+
 declare ptr @llvm.frameaddress(i32) nounwind readnone
diff --git a/llvm/test/CodeGen/SystemZ/frameaddr-02.ll b/llvm/test/CodeGen/SystemZ/frameaddr-02.ll
index a5e7e701c3..d3977928d4 100644
--- a/llvm/test/CodeGen/SystemZ/frameaddr-02.ll
+++ b/llvm/test/CodeGen/SystemZ/frameaddr-02.ll
@@ -27,6 +27,29 @@ entry:
   ret ptr %1
 }
 
+; Check the caller's frame address.
+define ptr @fpcaller() #0 {
+entry:
+; CHECK-LABEL: fpcaller:
+; CHECK: lghi %r2, 152
+; CHECK: ag   %r2, 152(%r15)
+; CHECK: br   %r14
+  %0 = tail call ptr @llvm.frameaddress(i32 1)
+  ret ptr %0
+}
+
+; Check the caller's caller's frame address.
+define ptr @fpcallercaller() #0 {
+entry:
+; CHECK-LABEL: fpcallercaller:
+; CHECK: lg   %r1, 152(%r15)
+; CHECK: lghi %r2, 152
+; CHECK: ag   %r2, 152(%r1)
+; CHECK: br   %r14
+  %0 = tail call ptr @llvm.frameaddress(i32 2)
+  ret ptr %0
+}
+
 ; Without back chain
 
 attributes #1 = { nounwind "packed-stack" }
diff --git a/llvm/test/CodeGen/SystemZ/ret-addr-01.ll b/llvm/test/CodeGen/SystemZ/ret-addr-01.ll
index 8111d2a724..e3644feb0b 100644
--- a/llvm/test/CodeGen/SystemZ/ret-addr-01.ll
+++ b/llvm/test/CodeGen/SystemZ/ret-addr-01.ll
@@ -12,4 +12,27 @@ entry:
   ret ptr %0
 }
 
+; Check the caller's return address.
+define ptr @rtcaller() nounwind "backchain" {
+entry:
+; CHECK-LABEL: rtcaller:
+; CHECK: lg   %r1, 0(%r15)
+; CHECK  lg   %r2, 112(%r1)
+; CHECK: br   %r14
+  %0 = tail call ptr @llvm.returnaddress(i32 1)
+  ret ptr %0
+}
+
+; Check the caller's caller's return address.
+define ptr @rtcallercaller() nounwind "backchain" {
+entry:
+; CHECK-LABEL: rtcallercaller:
+; CHECK: lg   %r1, 0(%r15)
+; CHECK: lg   %r1, 0(%r1)
+; CHECK  lg   %r2, 112(%r1)
+; CHECK: br   %r14
+  %0 = tail call ptr @llvm.returnaddress(i32 2)
+  ret ptr %0
+}
+
 declare ptr @llvm.returnaddress(i32) nounwind readnone
diff --git a/llvm/test/CodeGen/SystemZ/ret-addr-02.ll b/llvm/test/CodeGen/SystemZ/ret-addr-02.ll
new file mode 100644
index 0000000000..5c1e56d56c
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/ret-addr-02.ll
@@ -0,0 +1,39 @@
+; Test support for the llvm.returnaddress intrinsic with packed-stack.
+
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+; The current function's return address is in the link register.
+attributes #0 = { nounwind "packed-stack" "backchain" "use-soft-float"="true" }
+define ptr @rt0() #0 {
+entry:
+; CHECK-LABEL: rt0:
+; CHECK: lgr  %r2, %r14
+; CHECK: br   %r14
+  %0 = tail call ptr @llvm.returnaddress(i32 0)
+  ret ptr %0
+}
+
+; Check the caller's return address.
+define ptr @rtcaller() #0 {
+entry:
+; CHECK-LABEL: rtcaller:
+; CHECK: lg   %r1, 152(%r15)
+; CHECK  lg   %r2, 136(%r1)
+; CHECK: br   %r14
+  %0 = tail call ptr @llvm.returnaddress(i32 1)
+  ret ptr %0
+}
+
+; Check the caller's caller's return address.
+define ptr @rtcallercaller() #0 {
+entry:
+; CHECK-LABEL: rtcallercaller:
+; CHECK: lg   %r1, 152(%r15)
+; CHECK: lg   %r1, 152(%r1)
+; CHECK  lg   %r2, 136(%r1)
+; CHECK: br   %r14
+  %0 = tail call ptr @llvm.returnaddress(i32 2)
+  ret ptr %0
+}
+
+declare ptr @llvm.returnaddress(i32) nounwind readnone
diff --git a/llvm/test/CodeGen/Thumb2/active_lane_mask.ll b/llvm/test/CodeGen/Thumb2/active_lane_mask.ll
index 3ce79225cd..bcd92f8191 100644
--- a/llvm/test/CodeGen/Thumb2/active_lane_mask.ll
+++ b/llvm/test/CodeGen/Thumb2/active_lane_mask.ll
@@ -146,54 +146,47 @@ define <7 x i32> @v7i32(i32 %index, i32 %TC, <7 x i32> %V1, <7 x i32> %V2) {
 define <8 x i16> @v8i16(i32 %index, i32 %TC, <8 x i16> %V1, <8 x i16> %V2) {
 ; CHECK-LABEL: v8i16:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    sub sp, #16
 ; CHECK-NEXT:    adr.w r12, .LCPI3_0
 ; CHECK-NEXT:    vdup.32 q1, r1
 ; CHECK-NEXT:    vldrw.u32 q0, [r12]
-; CHECK-NEXT:    vmov.i8 q2, #0x0
-; CHECK-NEXT:    vmov.i8 q3, #0xff
+; CHECK-NEXT:    vmov.i8 q2, #0xff
+; CHECK-NEXT:    mov r4, sp
+; CHECK-NEXT:    adr r1, .LCPI3_1
 ; CHECK-NEXT:    vqadd.u32 q0, q0, r0
 ; CHECK-NEXT:    vcmp.u32 hi, q1, q0
-; CHECK-NEXT:    vpsel q4, q3, q2
-; CHECK-NEXT:    vmov r1, r12, d8
-; CHECK-NEXT:    vmov.16 q0[0], r1
-; CHECK-NEXT:    vmov.16 q0[1], r12
-; CHECK-NEXT:    vmov r1, r12, d9
-; CHECK-NEXT:    vmov.16 q0[2], r1
-; CHECK-NEXT:    adr r1, .LCPI3_1
-; CHECK-NEXT:    vldrw.u32 q4, [r1]
-; CHECK-NEXT:    vmov.16 q0[3], r12
-; CHECK-NEXT:    vqadd.u32 q4, q4, r0
-; CHECK-NEXT:    vcmp.u32 hi, q1, q4
-; CHECK-NEXT:    vpsel q1, q3, q2
-; CHECK-NEXT:    vmov r0, r1, d2
-; CHECK-NEXT:    vmov.16 q0[4], r0
-; CHECK-NEXT:    vmov.16 q0[5], r1
-; CHECK-NEXT:    vmov r0, r1, d3
-; CHECK-NEXT:    vmov.16 q0[6], r0
-; CHECK-NEXT:    add r0, sp, #24
-; CHECK-NEXT:    vmov.16 q0[7], r1
+; CHECK-NEXT:    vmov.i8 q0, #0x0
+; CHECK-NEXT:    vpsel q3, q2, q0
+; CHECK-NEXT:    vstrh.32 q3, [r4, #8]
+; CHECK-NEXT:    vldrw.u32 q3, [r1]
+; CHECK-NEXT:    vqadd.u32 q3, q3, r0
+; CHECK-NEXT:    add r0, sp, #32
+; CHECK-NEXT:    vcmp.u32 hi, q1, q3
 ; CHECK-NEXT:    vldrw.u32 q1, [r0]
-; CHECK-NEXT:    vcmp.i16 ne, q0, zr
-; CHECK-NEXT:    vldr d1, [sp, #16]
+; CHECK-NEXT:    vpsel q0, q2, q0
+; CHECK-NEXT:    vstrh.32 q0, [r4]
+; CHECK-NEXT:    vldr d1, [sp, #24]
+; CHECK-NEXT:    vldrw.u32 q2, [r4]
 ; CHECK-NEXT:    vmov d0, r2, r3
+; CHECK-NEXT:    vcmp.i16 ne, q2, zr
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    vmov r0, r1, d0
 ; CHECK-NEXT:    vmov r2, r3, d1
-; CHECK-NEXT:    vpop {d8, d9}
-; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    add sp, #16
+; CHECK-NEXT:    pop {r4, pc}
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  @ %bb.1:
 ; CHECK-NEXT:  .LCPI3_0:
-; CHECK-NEXT:    .long 0 @ 0x0
-; CHECK-NEXT:    .long 1 @ 0x1
-; CHECK-NEXT:    .long 2 @ 0x2
-; CHECK-NEXT:    .long 3 @ 0x3
-; CHECK-NEXT:  .LCPI3_1:
 ; CHECK-NEXT:    .long 4 @ 0x4
 ; CHECK-NEXT:    .long 5 @ 0x5
 ; CHECK-NEXT:    .long 6 @ 0x6
 ; CHECK-NEXT:    .long 7 @ 0x7
+; CHECK-NEXT:  .LCPI3_1:
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 1 @ 0x1
+; CHECK-NEXT:    .long 2 @ 0x2
+; CHECK-NEXT:    .long 3 @ 0x3
   %active.lane.mask = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 %index, i32 %TC)
   %select = select <8 x i1> %active.lane.mask, <8 x i16> %V1, <8 x i16> %V2
   ret <8 x i16> %select
@@ -202,122 +195,79 @@ define <8 x i16> @v8i16(i32 %index, i32 %TC, <8 x i16> %V1, <8 x i16> %V2) {
 define <16 x i8> @v16i8(i32 %index, i32 %TC, <16 x i8> %V1, <16 x i8> %V2) {
 ; CHECK-LABEL: v16i8:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    sub sp, #48
 ; CHECK-NEXT:    adr.w r12, .LCPI4_0
-; CHECK-NEXT:    vdup.32 q3, r1
+; CHECK-NEXT:    vdup.32 q2, r1
 ; CHECK-NEXT:    vldrw.u32 q0, [r12]
 ; CHECK-NEXT:    vmov.i8 q1, #0xff
+; CHECK-NEXT:    add r5, sp, #16
+; CHECK-NEXT:    adr r1, .LCPI4_1
 ; CHECK-NEXT:    vqadd.u32 q0, q0, r0
-; CHECK-NEXT:    vcmp.u32 hi, q3, q0
+; CHECK-NEXT:    adr r4, .LCPI4_3
+; CHECK-NEXT:    vcmp.u32 hi, q2, q0
 ; CHECK-NEXT:    vmov.i8 q0, #0x0
-; CHECK-NEXT:    vpsel q4, q1, q0
-; CHECK-NEXT:    vmov r1, r12, d8
-; CHECK-NEXT:    vmov.16 q2[0], r1
-; CHECK-NEXT:    vmov.16 q2[1], r12
-; CHECK-NEXT:    vmov r1, r12, d9
-; CHECK-NEXT:    vmov.16 q2[2], r1
-; CHECK-NEXT:    adr r1, .LCPI4_1
-; CHECK-NEXT:    vldrw.u32 q4, [r1]
-; CHECK-NEXT:    vmov.16 q2[3], r12
-; CHECK-NEXT:    vqadd.u32 q4, q4, r0
-; CHECK-NEXT:    vcmp.u32 hi, q3, q4
-; CHECK-NEXT:    vpsel q4, q1, q0
-; CHECK-NEXT:    vmov r1, r12, d8
-; CHECK-NEXT:    vmov.16 q2[4], r1
-; CHECK-NEXT:    vmov.16 q2[5], r12
-; CHECK-NEXT:    vmov r1, r12, d9
-; CHECK-NEXT:    vmov.16 q2[6], r1
-; CHECK-NEXT:    vmov.16 q2[7], r12
-; CHECK-NEXT:    vcmp.i16 ne, q2, zr
-; CHECK-NEXT:    vpsel q4, q1, q0
-; CHECK-NEXT:    vmov.u16 r1, q4[0]
-; CHECK-NEXT:    vmov.8 q2[0], r1
-; CHECK-NEXT:    vmov.u16 r1, q4[1]
-; CHECK-NEXT:    vmov.8 q2[1], r1
-; CHECK-NEXT:    vmov.u16 r1, q4[2]
-; CHECK-NEXT:    vmov.8 q2[2], r1
-; CHECK-NEXT:    vmov.u16 r1, q4[3]
-; CHECK-NEXT:    vmov.8 q2[3], r1
-; CHECK-NEXT:    vmov.u16 r1, q4[4]
-; CHECK-NEXT:    vmov.8 q2[4], r1
-; CHECK-NEXT:    vmov.u16 r1, q4[5]
-; CHECK-NEXT:    vmov.8 q2[5], r1
-; CHECK-NEXT:    vmov.u16 r1, q4[6]
-; CHECK-NEXT:    vmov.8 q2[6], r1
-; CHECK-NEXT:    vmov.u16 r1, q4[7]
-; CHECK-NEXT:    vmov.8 q2[7], r1
+; CHECK-NEXT:    vpsel q3, q1, q0
+; CHECK-NEXT:    vstrh.32 q3, [r5, #8]
+; CHECK-NEXT:    vldrw.u32 q3, [r1]
 ; CHECK-NEXT:    adr r1, .LCPI4_2
-; CHECK-NEXT:    vldrw.u32 q4, [r1]
-; CHECK-NEXT:    vqadd.u32 q4, q4, r0
-; CHECK-NEXT:    vcmp.u32 hi, q3, q4
-; CHECK-NEXT:    vpsel q5, q1, q0
-; CHECK-NEXT:    vmov r1, r12, d10
-; CHECK-NEXT:    vmov.16 q4[0], r1
-; CHECK-NEXT:    vmov.16 q4[1], r12
-; CHECK-NEXT:    vmov r1, r12, d11
-; CHECK-NEXT:    vmov.16 q4[2], r1
-; CHECK-NEXT:    adr r1, .LCPI4_3
-; CHECK-NEXT:    vldrw.u32 q5, [r1]
-; CHECK-NEXT:    vmov.16 q4[3], r12
-; CHECK-NEXT:    vqadd.u32 q5, q5, r0
-; CHECK-NEXT:    vcmp.u32 hi, q3, q5
+; CHECK-NEXT:    vqadd.u32 q3, q3, r0
+; CHECK-NEXT:    vcmp.u32 hi, q2, q3
+; CHECK-NEXT:    vpsel q3, q1, q0
+; CHECK-NEXT:    vstrh.32 q3, [r5]
+; CHECK-NEXT:    vldrw.u32 q3, [r1]
+; CHECK-NEXT:    mov r1, sp
+; CHECK-NEXT:    vqadd.u32 q3, q3, r0
+; CHECK-NEXT:    vcmp.u32 hi, q2, q3
 ; CHECK-NEXT:    vpsel q3, q1, q0
-; CHECK-NEXT:    vmov r0, r1, d6
-; CHECK-NEXT:    vmov.16 q4[4], r0
-; CHECK-NEXT:    vmov.16 q4[5], r1
-; CHECK-NEXT:    vmov r0, r1, d7
-; CHECK-NEXT:    vmov.16 q4[6], r0
-; CHECK-NEXT:    vmov.16 q4[7], r1
-; CHECK-NEXT:    vcmp.i16 ne, q4, zr
+; CHECK-NEXT:    vstrh.32 q3, [r1, #8]
+; CHECK-NEXT:    vldrw.u32 q3, [r4]
+; CHECK-NEXT:    vqadd.u32 q3, q3, r0
+; CHECK-NEXT:    add r0, sp, #32
+; CHECK-NEXT:    vcmp.u32 hi, q2, q3
+; CHECK-NEXT:    vpsel q2, q1, q0
+; CHECK-NEXT:    vstrh.32 q2, [r1]
+; CHECK-NEXT:    vldrw.u32 q2, [r5]
+; CHECK-NEXT:    vcmp.i16 ne, q2, zr
+; CHECK-NEXT:    vpsel q2, q1, q0
+; CHECK-NEXT:    vstrb.16 q2, [r0, #8]
+; CHECK-NEXT:    vldrw.u32 q2, [r1]
+; CHECK-NEXT:    add r1, sp, #72
+; CHECK-NEXT:    vcmp.i16 ne, q2, zr
 ; CHECK-NEXT:    vpsel q0, q1, q0
-; CHECK-NEXT:    vmov.u16 r0, q0[0]
-; CHECK-NEXT:    vmov.8 q2[8], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[1]
-; CHECK-NEXT:    vmov.8 q2[9], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[2]
-; CHECK-NEXT:    vmov.8 q2[10], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[3]
-; CHECK-NEXT:    vmov.8 q2[11], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[4]
-; CHECK-NEXT:    vmov.8 q2[12], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[5]
-; CHECK-NEXT:    vmov.8 q2[13], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[6]
-; CHECK-NEXT:    vmov.8 q2[14], r0
-; CHECK-NEXT:    vmov.u16 r0, q0[7]
-; CHECK-NEXT:    vmov.8 q2[15], r0
-; CHECK-NEXT:    add r0, sp, #40
-; CHECK-NEXT:    vldr d1, [sp, #32]
-; CHECK-NEXT:    vldrw.u32 q1, [r0]
-; CHECK-NEXT:    vcmp.i8 ne, q2, zr
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-NEXT:    vstrb.16 q0, [r0]
+; CHECK-NEXT:    vldr d1, [sp, #64]
+; CHECK-NEXT:    vldrw.u32 q2, [r0]
 ; CHECK-NEXT:    vmov d0, r2, r3
+; CHECK-NEXT:    vcmp.i8 ne, q2, zr
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    vmov r0, r1, d0
 ; CHECK-NEXT:    vmov r2, r3, d1
-; CHECK-NEXT:    vpop {d8, d9, d10, d11}
-; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    add sp, #48
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  @ %bb.1:
 ; CHECK-NEXT:  .LCPI4_0:
-; CHECK-NEXT:    .long 0 @ 0x0
-; CHECK-NEXT:    .long 1 @ 0x1
-; CHECK-NEXT:    .long 2 @ 0x2
-; CHECK-NEXT:    .long 3 @ 0x3
+; CHECK-NEXT:    .long 12 @ 0xc
+; CHECK-NEXT:    .long 13 @ 0xd
+; CHECK-NEXT:    .long 14 @ 0xe
+; CHECK-NEXT:    .long 15 @ 0xf
 ; CHECK-NEXT:  .LCPI4_1:
-; CHECK-NEXT:    .long 4 @ 0x4
-; CHECK-NEXT:    .long 5 @ 0x5
-; CHECK-NEXT:    .long 6 @ 0x6
-; CHECK-NEXT:    .long 7 @ 0x7
-; CHECK-NEXT:  .LCPI4_2:
 ; CHECK-NEXT:    .long 8 @ 0x8
 ; CHECK-NEXT:    .long 9 @ 0x9
 ; CHECK-NEXT:    .long 10 @ 0xa
 ; CHECK-NEXT:    .long 11 @ 0xb
+; CHECK-NEXT:  .LCPI4_2:
+; CHECK-NEXT:    .long 4 @ 0x4
+; CHECK-NEXT:    .long 5 @ 0x5
+; CHECK-NEXT:    .long 6 @ 0x6
+; CHECK-NEXT:    .long 7 @ 0x7
 ; CHECK-NEXT:  .LCPI4_3:
-; CHECK-NEXT:    .long 12 @ 0xc
-; CHECK-NEXT:    .long 13 @ 0xd
-; CHECK-NEXT:    .long 14 @ 0xe
-; CHECK-NEXT:    .long 15 @ 0xf
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 1 @ 0x1
+; CHECK-NEXT:    .long 2 @ 0x2
+; CHECK-NEXT:    .long 3 @ 0x3
   %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %index, i32 %TC)
   %select = select <16 x i1> %active.lane.mask, <16 x i8> %V1, <16 x i8> %V2
   ret <16 x i8> %select
diff --git a/llvm/test/CodeGen/Thumb2/mve-concat.ll b/llvm/test/CodeGen/Thumb2/mve-concat.ll
new file mode 100644
index 0000000000..8fa8c6c56e
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-concat.ll
@@ -0,0 +1,148 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc <4 x i32> @concat_v2i1(i32 %a, i32 %b, <4 x i32> %c) {
+; CHECK-LABEL: concat_v2i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vmov.i8 q1, #0x0
+; CHECK-NEXT:    vmov.i8 q2, #0xff
+; CHECK-NEXT:    vpsel q3, q2, q1
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpsel q1, q2, q1
+; CHECK-NEXT:    vmov r1, s12
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov q2[2], q2[0], r0, r1
+; CHECK-NEXT:    vmov r1, s6
+; CHECK-NEXT:    vmov r0, s14
+; CHECK-NEXT:    vmov.i32 q1, #0x0
+; CHECK-NEXT:    vmov q2[3], q2[1], r1, r0
+; CHECK-NEXT:    vcmp.i32 ne, q2, zr
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %ai = call <2 x i1> @llvm.arm.mve.pred.i2v.v2i1(i32 %a)
+  %bi = call <2 x i1> @llvm.arm.mve.pred.i2v.v2i1(i32 %b)
+  %s = shufflevector <2 x i1> %ai, <2 x i1> %bi, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %ci = select <4 x i1> %s, <4 x i32> %c, <4 x i32> zeroinitializer
+  ret <4 x i32> %ci
+}
+
+declare <2 x i1> @llvm.arm.mve.pred.i2v.v2i1(i32)
+
+
+define arm_aapcs_vfpcc <8 x i16> @concat_v4i1(<4 x i32> %a, <4 x i32> %b, <8 x i16> %c) {
+; CHECK-LABEL: concat_v4i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
+; CHECK-NEXT:    vmov.i8 q3, #0x0
+; CHECK-NEXT:    vmov.i8 q4, #0xff
+; CHECK-NEXT:    vcmp.s32 lt, q1, zr
+; CHECK-NEXT:    mov r0, sp
+; CHECK-NEXT:    vpsel q1, q4, q3
+; CHECK-NEXT:    vcmp.s32 lt, q0, zr
+; CHECK-NEXT:    vpsel q0, q4, q3
+; CHECK-NEXT:    vstrh.32 q1, [r0, #8]
+; CHECK-NEXT:    vstrh.32 q0, [r0]
+; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vcmp.i16 ne, q1, zr
+; CHECK-NEXT:    vpsel q0, q2, q0
+; CHECK-NEXT:    add sp, #16
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    bx lr
+entry:
+  %ai = icmp slt <4 x i32> %a, zeroinitializer
+  %bi = icmp slt <4 x i32> %b, zeroinitializer
+  %s = shufflevector <4 x i1> %ai, <4 x i1> %bi, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %ci = select <8 x i1> %s, <8 x i16> %c, <8 x i16> zeroinitializer
+  ret <8 x i16> %ci
+}
+
+define arm_aapcs_vfpcc <16 x i8> @concat_v8i1(<8 x i16> %a, <8 x i16> %b, <16 x i8> %c) {
+; CHECK-LABEL: concat_v8i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
+; CHECK-NEXT:    vmov.i8 q3, #0x0
+; CHECK-NEXT:    vmov.i8 q4, #0xff
+; CHECK-NEXT:    vcmp.s16 lt, q1, zr
+; CHECK-NEXT:    mov r0, sp
+; CHECK-NEXT:    vpsel q1, q4, q3
+; CHECK-NEXT:    vcmp.s16 lt, q0, zr
+; CHECK-NEXT:    vpsel q0, q4, q3
+; CHECK-NEXT:    vstrb.16 q1, [r0, #8]
+; CHECK-NEXT:    vstrb.16 q0, [r0]
+; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vcmp.i8 ne, q1, zr
+; CHECK-NEXT:    vpsel q0, q2, q0
+; CHECK-NEXT:    add sp, #16
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    bx lr
+entry:
+  %ai = icmp slt <8 x i16> %a, zeroinitializer
+  %bi = icmp slt <8 x i16> %b, zeroinitializer
+  %s = shufflevector <8 x i1> %ai, <8 x i1> %bi, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %ci = select <16 x i1> %s, <16 x i8> %c, <16 x i8> zeroinitializer
+  ret <16 x i8> %ci
+}
+
+
+define arm_aapcs_vfpcc <16 x i8> @concat_v48i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %d, <4 x i32> %e, <16 x i8> %c) {
+; CHECK-LABEL: concat_v48i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    .pad #48
+; CHECK-NEXT:    sub sp, #48
+; CHECK-NEXT:    vmov.i8 q4, #0x0
+; CHECK-NEXT:    vmov.i8 q5, #0xff
+; CHECK-NEXT:    vcmp.s32 lt, q3, zr
+; CHECK-NEXT:    add r0, sp, #16
+; CHECK-NEXT:    vpsel q3, q5, q4
+; CHECK-NEXT:    vcmp.s32 lt, q2, zr
+; CHECK-NEXT:    vpsel q2, q5, q4
+; CHECK-NEXT:    vcmp.s32 lt, q1, zr
+; CHECK-NEXT:    vpsel q1, q5, q4
+; CHECK-NEXT:    vcmp.s32 lt, q0, zr
+; CHECK-NEXT:    mov r1, sp
+; CHECK-NEXT:    vpsel q0, q5, q4
+; CHECK-NEXT:    vstrh.32 q3, [r0, #8]
+; CHECK-NEXT:    vstrh.32 q2, [r0]
+; CHECK-NEXT:    vstrh.32 q1, [r1, #8]
+; CHECK-NEXT:    vstrh.32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    add r0, sp, #32
+; CHECK-NEXT:    vcmp.i16 ne, q0, zr
+; CHECK-NEXT:    vpsel q0, q5, q4
+; CHECK-NEXT:    vstrb.16 q0, [r0, #8]
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    add r1, sp, #80
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
+; CHECK-NEXT:    vcmp.i16 ne, q0, zr
+; CHECK-NEXT:    vpsel q0, q5, q4
+; CHECK-NEXT:    vstrb.16 q0, [r0]
+; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    vldrw.u32 q2, [r0]
+; CHECK-NEXT:    vcmp.i8 ne, q2, zr
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    add sp, #48
+; CHECK-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-NEXT:    bx lr
+entry:
+  %ai = icmp slt <4 x i32> %a, zeroinitializer
+  %bi = icmp slt <4 x i32> %b, zeroinitializer
+  %di = icmp slt <4 x i32> %d, zeroinitializer
+  %ei = icmp slt <4 x i32> %e, zeroinitializer
+  %s1 = shufflevector <4 x i1> %ai, <4 x i1> %bi, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %s2 = shufflevector <4 x i1> %di, <4 x i1> %ei, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %s = shufflevector <8 x i1> %s1, <8 x i1> %s2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %ci = select <16 x i1> %s, <16 x i8> %c, <16 x i8> zeroinitializer
+  ret <16 x i8> %ci
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-laneinterleaving-reduct.ll b/llvm/test/CodeGen/Thumb2/mve-laneinterleaving-reduct.ll
index cfdb20d15e..9987ff940b 100644
--- a/llvm/test/CodeGen/Thumb2/mve-laneinterleaving-reduct.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-laneinterleaving-reduct.ll
@@ -62,40 +62,30 @@ entry:
 define arm_aapcs_vfpcc i32 @mlapred_v4i32_v4i64_zext(<8 x i16> %x, <8 x i16> %y, <8 x i16> %a, <8 x i16> %b) {
 ; CHECK-LABEL: mlapred_v4i32_v4i64_zext:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r7, lr}
-; CHECK-NEXT:    push {r4, r5, r7, lr}
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11}
-; CHECK-NEXT:    .pad #16
-; CHECK-NEXT:    sub sp, #16
+; CHECK-NEXT:    .pad #32
+; CHECK-NEXT:    sub sp, #32
 ; CHECK-NEXT:    vorr q2, q2, q3
 ; CHECK-NEXT:    mov r0, sp
 ; CHECK-NEXT:    vstrw.32 q2, [r0]
 ; CHECK-NEXT:    vmov.i8 q3, #0xff
 ; CHECK-NEXT:    vldrh.u32 q2, [r0, #8]
 ; CHECK-NEXT:    vldrh.u32 q5, [r0]
+; CHECK-NEXT:    add r0, sp, #16
 ; CHECK-NEXT:    vcmp.i32 eq, q2, zr
 ; CHECK-NEXT:    vmov.i8 q2, #0x0
 ; CHECK-NEXT:    vpsel q4, q3, q2
 ; CHECK-NEXT:    vcmp.i32 eq, q5, zr
 ; CHECK-NEXT:    vpsel q2, q3, q2
-; CHECK-NEXT:    vmov r2, r3, d8
-; CHECK-NEXT:    vmov r4, r5, d4
-; CHECK-NEXT:    vmov r0, r1, d5
-; CHECK-NEXT:    vmov.16 q2[0], r4
-; CHECK-NEXT:    vmov.16 q2[1], r5
-; CHECK-NEXT:    vmov r12, lr, d9
-; CHECK-NEXT:    vmov.16 q2[2], r0
-; CHECK-NEXT:    vmov.16 q2[3], r1
-; CHECK-NEXT:    vmov.16 q2[4], r2
-; CHECK-NEXT:    vmov.16 q2[5], r3
-; CHECK-NEXT:    vmov.16 q2[6], r12
-; CHECK-NEXT:    vmov.16 q2[7], lr
+; CHECK-NEXT:    vstrh.32 q4, [r0, #8]
+; CHECK-NEXT:    vstrh.32 q2, [r0]
+; CHECK-NEXT:    vldrw.u32 q2, [r0]
 ; CHECK-NEXT:    vpt.i16 ne, q2, zr
 ; CHECK-NEXT:    vmlavt.u16 r0, q0, q1
-; CHECK-NEXT:    add sp, #16
+; CHECK-NEXT:    add sp, #32
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11}
-; CHECK-NEXT:    pop {r4, r5, r7, pc}
+; CHECK-NEXT:    bx lr
 entry:
   %aa = zext <8 x i16> %a to <8 x i32>
   %bb = zext <8 x i16> %b to <8 x i32>
diff --git a/llvm/test/CodeGen/Thumb2/mve-phireg.ll b/llvm/test/CodeGen/Thumb2/mve-phireg.ll
index 72df912b25..d2f79fcd5f 100644
--- a/llvm/test/CodeGen/Thumb2/mve-phireg.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-phireg.ll
@@ -6,52 +6,42 @@
 define arm_aapcs_vfpcc void @k() {
 ; CHECK-LABEL: k:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, lr}
-; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, lr}
-; CHECK-NEXT:    .pad #4
-; CHECK-NEXT:    sub sp, #4
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14}
-; CHECK-NEXT:    .pad #16
-; CHECK-NEXT:    sub sp, #16
-; CHECK-NEXT:    adr.w r8, .LCPI0_0
-; CHECK-NEXT:    adr.w r9, .LCPI0_1
-; CHECK-NEXT:    vldrw.u32 q6, [r8]
-; CHECK-NEXT:    vldrw.u32 q5, [r9]
+; CHECK-NEXT:    .pad #32
+; CHECK-NEXT:    sub sp, #32
+; CHECK-NEXT:    adr r5, .LCPI0_0
+; CHECK-NEXT:    adr r4, .LCPI0_1
+; CHECK-NEXT:    vldrw.u32 q6, [r5]
+; CHECK-NEXT:    vldrw.u32 q5, [r4]
+; CHECK-NEXT:    add r0, sp, #16
 ; CHECK-NEXT:    vmov.i32 q0, #0x1
 ; CHECK-NEXT:    vmov.i8 q1, #0x0
 ; CHECK-NEXT:    vmov.i8 q2, #0xff
 ; CHECK-NEXT:    vmov.i16 q3, #0x6
 ; CHECK-NEXT:    vmov.i16 q4, #0x3
-; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    movs r1, #0
 ; CHECK-NEXT:  .LBB0_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vand q6, q6, q0
 ; CHECK-NEXT:    vand q5, q5, q0
-; CHECK-NEXT:    vcmp.i32 eq, q6, zr
-; CHECK-NEXT:    cmp.w r12, #0
-; CHECK-NEXT:    vpsel q6, q2, q1
+; CHECK-NEXT:    vand q6, q6, q0
 ; CHECK-NEXT:    vcmp.i32 eq, q5, zr
 ; CHECK-NEXT:    vpsel q5, q2, q1
-; CHECK-NEXT:    vmov r4, r0, d12
-; CHECK-NEXT:    vmov r3, r6, d10
-; CHECK-NEXT:    vmov r1, r2, d11
-; CHECK-NEXT:    vmov.16 q5[0], r3
-; CHECK-NEXT:    vmov.16 q5[1], r6
-; CHECK-NEXT:    vmov r5, r7, d13
-; CHECK-NEXT:    vmov.16 q5[2], r1
-; CHECK-NEXT:    vmov.16 q5[3], r2
-; CHECK-NEXT:    vmov.16 q5[4], r4
-; CHECK-NEXT:    vmov.16 q5[5], r0
-; CHECK-NEXT:    vmov.16 q5[6], r5
-; CHECK-NEXT:    vmov.16 q5[7], r7
+; CHECK-NEXT:    vcmp.i32 eq, q6, zr
+; CHECK-NEXT:    vpsel q6, q2, q1
+; CHECK-NEXT:    vstrh.32 q5, [r0]
+; CHECK-NEXT:    vstrh.32 q6, [r0, #8]
+; CHECK-NEXT:    vldrw.u32 q5, [r0]
 ; CHECK-NEXT:    vcmp.i16 ne, q5, zr
 ; CHECK-NEXT:    vmov.i32 q5, #0x0
 ; CHECK-NEXT:    vpsel q6, q4, q3
 ; CHECK-NEXT:    vstrh.16 q6, [r0]
 ; CHECK-NEXT:    vmov q6, q5
-; CHECK-NEXT:    bne .LBB0_1
-; CHECK-NEXT:  @ %bb.2: @ %for.cond4.preheader
+; CHECK-NEXT:    cbz r1, .LBB0_2
+; CHECK-NEXT:    le .LBB0_1
+; CHECK-NEXT:  .LBB0_2: @ %for.cond4.preheader
 ; CHECK-NEXT:    movs r6, #0
 ; CHECK-NEXT:    cbnz r6, .LBB0_5
 ; CHECK-NEXT:  .LBB0_3: @ %for.body10
@@ -61,8 +51,8 @@ define arm_aapcs_vfpcc void @k() {
 ; CHECK-NEXT:  .LBB0_4: @ %for.cond4.loopexit
 ; CHECK-NEXT:    bl l
 ; CHECK-NEXT:  .LBB0_5: @ %vector.body105.preheader
-; CHECK-NEXT:    vldrw.u32 q0, [r8]
-; CHECK-NEXT:    vldrw.u32 q1, [r9]
+; CHECK-NEXT:    vldrw.u32 q0, [r5]
+; CHECK-NEXT:    vldrw.u32 q1, [r4]
 ; CHECK-NEXT:    movs r0, #8
 ; CHECK-NEXT:  .LBB0_6: @ %vector.body105
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
@@ -71,7 +61,7 @@ define arm_aapcs_vfpcc void @k() {
 ; CHECK-NEXT:    cbz r6, .LBB0_7
 ; CHECK-NEXT:    le .LBB0_6
 ; CHECK-NEXT:  .LBB0_7: @ %vector.body115.ph
-; CHECK-NEXT:    vldrw.u32 q0, [r9]
+; CHECK-NEXT:    vldrw.u32 q0, [r4]
 ; CHECK-NEXT:    movs r0, #4
 ; CHECK-NEXT:    vstrw.32 q0, [sp] @ 16-byte Spill
 ; CHECK-NEXT:    @APP
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-shuffle.ll b/llvm/test/CodeGen/Thumb2/mve-pred-shuffle.ll
index cca15165e0..a55a8e4a68 100644
--- a/llvm/test/CodeGen/Thumb2/mve-pred-shuffle.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-shuffle.ll
@@ -640,37 +640,34 @@ entry:
 define <8 x i16> @shuffle6_v4i32(<4 x i32> %src1, <4 x i32> %src2, <8 x i16> %a, <8 x i16> %b) {
 ; CHECK-LABEL: shuffle6_v4i32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov d1, r2, r3
-; CHECK-NEXT:    vmov.i8 q1, #0x0
-; CHECK-NEXT:    vmov d0, r0, r1
-; CHECK-NEXT:    vmov.i8 q2, #0xff
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vpsel q3, q2, q1
-; CHECK-NEXT:    vmov r0, r1, d6
-; CHECK-NEXT:    vmov.16 q0[0], r0
-; CHECK-NEXT:    vmov.16 q0[1], r1
-; CHECK-NEXT:    vmov r0, r1, d7
-; CHECK-NEXT:    vmov.16 q0[2], r0
-; CHECK-NEXT:    mov r0, sp
-; CHECK-NEXT:    vldrw.u32 q3, [r0]
-; CHECK-NEXT:    vmov.16 q0[3], r1
-; CHECK-NEXT:    vcmp.i32 eq, q3, zr
-; CHECK-NEXT:    vpsel q1, q2, q1
-; CHECK-NEXT:    vmov r0, r1, d2
-; CHECK-NEXT:    vmov.16 q0[4], r0
-; CHECK-NEXT:    vmov.16 q0[5], r1
-; CHECK-NEXT:    vmov r0, r1, d3
-; CHECK-NEXT:    vmov.16 q0[6], r0
-; CHECK-NEXT:    add r0, sp, #32
-; CHECK-NEXT:    vmov.16 q0[7], r1
-; CHECK-NEXT:    vcmp.i16 ne, q0, zr
+; CHECK-NEXT:    .save {r4, lr}
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
+; CHECK-NEXT:    add.w r12, sp, #24
+; CHECK-NEXT:    vmov.i8 q0, #0x0
+; CHECK-NEXT:    vldrw.u32 q2, [r12]
+; CHECK-NEXT:    vmov.i8 q1, #0xff
+; CHECK-NEXT:    mov r4, sp
+; CHECK-NEXT:    vcmp.i32 eq, q2, zr
+; CHECK-NEXT:    vpsel q2, q1, q0
+; CHECK-NEXT:    vstrh.32 q2, [r4, #8]
+; CHECK-NEXT:    vmov d4, r0, r1
+; CHECK-NEXT:    vmov d5, r2, r3
+; CHECK-NEXT:    add r0, sp, #56
+; CHECK-NEXT:    vcmp.i32 eq, q2, zr
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vstrh.32 q0, [r4]
 ; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    add r0, sp, #16
+; CHECK-NEXT:    add r0, sp, #40
+; CHECK-NEXT:    vldrw.u32 q2, [r4]
 ; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vcmp.i16 ne, q2, zr
 ; CHECK-NEXT:    vpsel q0, q1, q0
 ; CHECK-NEXT:    vmov r0, r1, d0
 ; CHECK-NEXT:    vmov r2, r3, d1
-; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    add sp, #16
+; CHECK-NEXT:    pop {r4, pc}
 entry:
   %c1 = icmp eq <4 x i32> %src1, zeroinitializer
   %c2 = icmp eq <4 x i32> %src2, zeroinitializer
@@ -682,57 +679,34 @@ entry:
 define <16 x i8> @shuffle6_v8i16(<8 x i16> %src1, <8 x i16> %src2, <16 x i8> %a, <16 x i8> %b) {
 ; CHECK-LABEL: shuffle6_v8i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov d1, r2, r3
-; CHECK-NEXT:    vmov.i8 q1, #0x0
-; CHECK-NEXT:    vmov d0, r0, r1
-; CHECK-NEXT:    vmov.i8 q2, #0xff
-; CHECK-NEXT:    vcmp.i16 eq, q0, zr
-; CHECK-NEXT:    vpsel q3, q2, q1
-; CHECK-NEXT:    vmov.u16 r0, q3[0]
-; CHECK-NEXT:    vmov.8 q0[0], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[1]
-; CHECK-NEXT:    vmov.8 q0[1], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[2]
-; CHECK-NEXT:    vmov.8 q0[2], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[3]
-; CHECK-NEXT:    vmov.8 q0[3], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[4]
-; CHECK-NEXT:    vmov.8 q0[4], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[5]
-; CHECK-NEXT:    vmov.8 q0[5], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[6]
-; CHECK-NEXT:    vmov.8 q0[6], r0
-; CHECK-NEXT:    vmov.u16 r0, q3[7]
-; CHECK-NEXT:    vmov.8 q0[7], r0
-; CHECK-NEXT:    mov r0, sp
-; CHECK-NEXT:    vldrw.u32 q3, [r0]
-; CHECK-NEXT:    vcmp.i16 eq, q3, zr
-; CHECK-NEXT:    vpsel q1, q2, q1
-; CHECK-NEXT:    vmov.u16 r0, q1[0]
-; CHECK-NEXT:    vmov.8 q0[8], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[1]
-; CHECK-NEXT:    vmov.8 q0[9], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[2]
-; CHECK-NEXT:    vmov.8 q0[10], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[3]
-; CHECK-NEXT:    vmov.8 q0[11], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[4]
-; CHECK-NEXT:    vmov.8 q0[12], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[5]
-; CHECK-NEXT:    vmov.8 q0[13], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[6]
-; CHECK-NEXT:    vmov.8 q0[14], r0
-; CHECK-NEXT:    vmov.u16 r0, q1[7]
-; CHECK-NEXT:    vmov.8 q0[15], r0
-; CHECK-NEXT:    add r0, sp, #32
-; CHECK-NEXT:    vcmp.i8 ne, q0, zr
+; CHECK-NEXT:    .save {r4, lr}
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
+; CHECK-NEXT:    add.w r12, sp, #24
+; CHECK-NEXT:    vmov.i8 q0, #0x0
+; CHECK-NEXT:    vldrw.u32 q2, [r12]
+; CHECK-NEXT:    vmov.i8 q1, #0xff
+; CHECK-NEXT:    mov r4, sp
+; CHECK-NEXT:    vcmp.i16 eq, q2, zr
+; CHECK-NEXT:    vpsel q2, q1, q0
+; CHECK-NEXT:    vstrb.16 q2, [r4, #8]
+; CHECK-NEXT:    vmov d4, r0, r1
+; CHECK-NEXT:    vmov d5, r2, r3
+; CHECK-NEXT:    add r0, sp, #56
+; CHECK-NEXT:    vcmp.i16 eq, q2, zr
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vstrb.16 q0, [r4]
 ; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    add r0, sp, #16
+; CHECK-NEXT:    add r0, sp, #40
+; CHECK-NEXT:    vldrw.u32 q2, [r4]
 ; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vcmp.i8 ne, q2, zr
 ; CHECK-NEXT:    vpsel q0, q1, q0
 ; CHECK-NEXT:    vmov r0, r1, d0
 ; CHECK-NEXT:    vmov r2, r3, d1
-; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    add sp, #16
+; CHECK-NEXT:    pop {r4, pc}
 entry:
   %c1 = icmp eq <8 x i16> %src1, zeroinitializer
   %c2 = icmp eq <8 x i16> %src2, zeroinitializer
diff --git a/llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll b/llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll
index b5d981ef34..fef2c39e08 100644
--- a/llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll
@@ -1386,49 +1386,43 @@ for.cond.cleanup:                                 ; preds = %vector.body, %entry
 define arm_aapcs_vfpcc void @ssatmul_8t_q15(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) {
 ; CHECK-LABEL: ssatmul_8t_q15:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, lr}
-; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
 ; CHECK-NEXT:    cmp r3, #0
 ; CHECK-NEXT:    beq .LBB9_3
 ; CHECK-NEXT:  @ %bb.1: @ %vector.ph
-; CHECK-NEXT:    add.w r12, r3, #7
+; CHECK-NEXT:    adds r4, r3, #7
+; CHECK-NEXT:    vmov.i8 q2, #0x0
+; CHECK-NEXT:    bic r4, r4, #7
+; CHECK-NEXT:    vmov.i8 q3, #0xff
+; CHECK-NEXT:    sub.w r12, r4, #8
+; CHECK-NEXT:    movs r4, #1
+; CHECK-NEXT:    mov r5, sp
+; CHECK-NEXT:    add.w lr, r4, r12, lsr #3
 ; CHECK-NEXT:    adr r4, .LCPI9_0
-; CHECK-NEXT:    bic r12, r12, #7
-; CHECK-NEXT:    mov.w lr, #1
-; CHECK-NEXT:    sub.w r12, r12, #8
 ; CHECK-NEXT:    vldrw.u32 q0, [r4]
 ; CHECK-NEXT:    adr r4, .LCPI9_1
-; CHECK-NEXT:    vmov.i8 q2, #0x0
-; CHECK-NEXT:    add.w lr, lr, r12, lsr #3
 ; CHECK-NEXT:    sub.w r12, r3, #1
 ; CHECK-NEXT:    vldrw.u32 q4, [r4]
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    vdup.32 q1, r12
-; CHECK-NEXT:    vmov.i8 q3, #0xff
 ; CHECK-NEXT:  .LBB9_2: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vdup.32 q6, r3
+; CHECK-NEXT:    vdup.32 q5, r3
 ; CHECK-NEXT:    adds r3, #8
-; CHECK-NEXT:    vorr q5, q6, q0
-; CHECK-NEXT:    vorr q6, q6, q4
-; CHECK-NEXT:    vcmp.u32 cs, q1, q5
-; CHECK-NEXT:    vpsel q7, q3, q2
+; CHECK-NEXT:    vorr q6, q5, q0
+; CHECK-NEXT:    vorr q5, q5, q4
 ; CHECK-NEXT:    vcmp.u32 cs, q1, q6
-; CHECK-NEXT:    vmov r4, r12, d14
 ; CHECK-NEXT:    vpsel q6, q3, q2
-; CHECK-NEXT:    vmov.16 q5[0], r4
-; CHECK-NEXT:    vmov.16 q5[1], r12
-; CHECK-NEXT:    vmov r4, r12, d15
-; CHECK-NEXT:    vmov.16 q5[2], r4
-; CHECK-NEXT:    vmov.16 q5[3], r12
-; CHECK-NEXT:    vmov r4, r12, d12
-; CHECK-NEXT:    vmov.16 q5[4], r4
-; CHECK-NEXT:    vmov.16 q5[5], r12
-; CHECK-NEXT:    vmov r4, r12, d13
-; CHECK-NEXT:    vmov.16 q5[6], r4
-; CHECK-NEXT:    vmov.16 q5[7], r12
+; CHECK-NEXT:    vcmp.u32 cs, q1, q5
+; CHECK-NEXT:    vpsel q5, q3, q2
+; CHECK-NEXT:    vstrh.32 q6, [r5, #8]
+; CHECK-NEXT:    vstrh.32 q5, [r5]
+; CHECK-NEXT:    vldrw.u32 q5, [r5]
 ; CHECK-NEXT:    vptt.i16 ne, q5, zr
 ; CHECK-NEXT:    vldrht.u16 q5, [r0], #16
 ; CHECK-NEXT:    vldrht.u16 q6, [r1], #16
@@ -1440,20 +1434,21 @@ define arm_aapcs_vfpcc void @ssatmul_8t_q15(ptr nocapture readonly %pSrcA, ptr n
 ; CHECK-NEXT:    vstrht.16 q5, [r2], #16
 ; CHECK-NEXT:    le lr, .LBB9_2
 ; CHECK-NEXT:  .LBB9_3: @ %for.cond.cleanup
+; CHECK-NEXT:    add sp, #16
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
-; CHECK-NEXT:    pop {r4, pc}
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  @ %bb.4:
 ; CHECK-NEXT:  .LCPI9_0:
-; CHECK-NEXT:    .long 0 @ 0x0
-; CHECK-NEXT:    .long 1 @ 0x1
-; CHECK-NEXT:    .long 2 @ 0x2
-; CHECK-NEXT:    .long 3 @ 0x3
-; CHECK-NEXT:  .LCPI9_1:
 ; CHECK-NEXT:    .long 4 @ 0x4
 ; CHECK-NEXT:    .long 5 @ 0x5
 ; CHECK-NEXT:    .long 6 @ 0x6
 ; CHECK-NEXT:    .long 7 @ 0x7
+; CHECK-NEXT:  .LCPI9_1:
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 1 @ 0x1
+; CHECK-NEXT:    .long 2 @ 0x2
+; CHECK-NEXT:    .long 3 @ 0x3
 entry:
   %cmp8 = icmp eq i32 %N, 0
   br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
@@ -1498,49 +1493,43 @@ for.cond.cleanup:                                 ; preds = %vector.body, %entry
 define arm_aapcs_vfpcc void @ssatmul_8ti_q15(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) {
 ; CHECK-LABEL: ssatmul_8ti_q15:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, lr}
-; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
 ; CHECK-NEXT:    cmp r3, #0
 ; CHECK-NEXT:    beq .LBB10_3
 ; CHECK-NEXT:  @ %bb.1: @ %vector.ph
-; CHECK-NEXT:    add.w r12, r3, #7
+; CHECK-NEXT:    adds r4, r3, #7
+; CHECK-NEXT:    vmov.i8 q2, #0x0
+; CHECK-NEXT:    bic r4, r4, #7
+; CHECK-NEXT:    vmov.i8 q3, #0xff
+; CHECK-NEXT:    sub.w r12, r4, #8
+; CHECK-NEXT:    movs r4, #1
+; CHECK-NEXT:    mov r5, sp
+; CHECK-NEXT:    add.w lr, r4, r12, lsr #3
 ; CHECK-NEXT:    adr r4, .LCPI10_0
-; CHECK-NEXT:    bic r12, r12, #7
-; CHECK-NEXT:    mov.w lr, #1
-; CHECK-NEXT:    sub.w r12, r12, #8
 ; CHECK-NEXT:    vldrw.u32 q0, [r4]
 ; CHECK-NEXT:    adr r4, .LCPI10_1
-; CHECK-NEXT:    vmov.i8 q2, #0x0
-; CHECK-NEXT:    add.w lr, lr, r12, lsr #3
 ; CHECK-NEXT:    sub.w r12, r3, #1
 ; CHECK-NEXT:    vldrw.u32 q4, [r4]
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    vdup.32 q1, r12
-; CHECK-NEXT:    vmov.i8 q3, #0xff
 ; CHECK-NEXT:  .LBB10_2: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vdup.32 q6, r3
+; CHECK-NEXT:    vdup.32 q5, r3
 ; CHECK-NEXT:    adds r3, #8
-; CHECK-NEXT:    vorr q5, q6, q0
-; CHECK-NEXT:    vorr q6, q6, q4
-; CHECK-NEXT:    vcmp.u32 cs, q1, q5
-; CHECK-NEXT:    vpsel q7, q3, q2
+; CHECK-NEXT:    vorr q6, q5, q0
+; CHECK-NEXT:    vorr q5, q5, q4
 ; CHECK-NEXT:    vcmp.u32 cs, q1, q6
-; CHECK-NEXT:    vmov r4, r12, d14
 ; CHECK-NEXT:    vpsel q6, q3, q2
-; CHECK-NEXT:    vmov.16 q5[0], r4
-; CHECK-NEXT:    vmov.16 q5[1], r12
-; CHECK-NEXT:    vmov r4, r12, d15
-; CHECK-NEXT:    vmov.16 q5[2], r4
-; CHECK-NEXT:    vmov.16 q5[3], r12
-; CHECK-NEXT:    vmov r4, r12, d12
-; CHECK-NEXT:    vmov.16 q5[4], r4
-; CHECK-NEXT:    vmov.16 q5[5], r12
-; CHECK-NEXT:    vmov r4, r12, d13
-; CHECK-NEXT:    vmov.16 q5[6], r4
-; CHECK-NEXT:    vmov.16 q5[7], r12
+; CHECK-NEXT:    vcmp.u32 cs, q1, q5
+; CHECK-NEXT:    vpsel q5, q3, q2
+; CHECK-NEXT:    vstrh.32 q6, [r5, #8]
+; CHECK-NEXT:    vstrh.32 q5, [r5]
+; CHECK-NEXT:    vldrw.u32 q5, [r5]
 ; CHECK-NEXT:    vptt.i16 ne, q5, zr
 ; CHECK-NEXT:    vldrht.u16 q5, [r0], #16
 ; CHECK-NEXT:    vldrht.u16 q6, [r1], #16
@@ -1552,20 +1541,21 @@ define arm_aapcs_vfpcc void @ssatmul_8ti_q15(ptr nocapture readonly %pSrcA, ptr
 ; CHECK-NEXT:    vstrht.16 q5, [r2], #16
 ; CHECK-NEXT:    le lr, .LBB10_2
 ; CHECK-NEXT:  .LBB10_3: @ %for.cond.cleanup
+; CHECK-NEXT:    add sp, #16
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
-; CHECK-NEXT:    pop {r4, pc}
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  @ %bb.4:
 ; CHECK-NEXT:  .LCPI10_0:
-; CHECK-NEXT:    .long 0 @ 0x0
-; CHECK-NEXT:    .long 1 @ 0x1
-; CHECK-NEXT:    .long 2 @ 0x2
-; CHECK-NEXT:    .long 3 @ 0x3
-; CHECK-NEXT:  .LCPI10_1:
 ; CHECK-NEXT:    .long 4 @ 0x4
 ; CHECK-NEXT:    .long 5 @ 0x5
 ; CHECK-NEXT:    .long 6 @ 0x6
 ; CHECK-NEXT:    .long 7 @ 0x7
+; CHECK-NEXT:  .LCPI10_1:
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 1 @ 0x1
+; CHECK-NEXT:    .long 2 @ 0x2
+; CHECK-NEXT:    .long 3 @ 0x3
 entry:
   %cmp8 = icmp eq i32 %N, 0
   br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
@@ -2386,49 +2376,43 @@ for.body:                                         ; preds = %for.body, %for.body
 define arm_aapcs_vfpcc void @ssatmul_8t_q7(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) {
 ; CHECK-LABEL: ssatmul_8t_q7:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, lr}
-; CHECK-NEXT:    push {r4, lr}
-; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
-; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    .pad #16
+; CHECK-NEXT:    sub sp, #16
 ; CHECK-NEXT:    cmp r3, #0
 ; CHECK-NEXT:    beq .LBB17_3
 ; CHECK-NEXT:  @ %bb.1: @ %vector.ph
-; CHECK-NEXT:    add.w r12, r3, #7
+; CHECK-NEXT:    adds r4, r3, #7
+; CHECK-NEXT:    vmov.i8 q2, #0x0
+; CHECK-NEXT:    bic r4, r4, #7
+; CHECK-NEXT:    vmov.i8 q3, #0xff
+; CHECK-NEXT:    sub.w r12, r4, #8
+; CHECK-NEXT:    movs r4, #1
+; CHECK-NEXT:    mov r5, sp
+; CHECK-NEXT:    add.w lr, r4, r12, lsr #3
 ; CHECK-NEXT:    adr r4, .LCPI17_0
-; CHECK-NEXT:    bic r12, r12, #7
-; CHECK-NEXT:    mov.w lr, #1
-; CHECK-NEXT:    sub.w r12, r12, #8
 ; CHECK-NEXT:    vldrw.u32 q0, [r4]
 ; CHECK-NEXT:    adr r4, .LCPI17_1
-; CHECK-NEXT:    vmov.i8 q2, #0x0
-; CHECK-NEXT:    add.w lr, lr, r12, lsr #3
 ; CHECK-NEXT:    sub.w r12, r3, #1
 ; CHECK-NEXT:    vldrw.u32 q4, [r4]
 ; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    vdup.32 q1, r12
-; CHECK-NEXT:    vmov.i8 q3, #0xff
 ; CHECK-NEXT:  .LBB17_2: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vdup.32 q6, r3
+; CHECK-NEXT:    vdup.32 q5, r3
 ; CHECK-NEXT:    adds r3, #8
-; CHECK-NEXT:    vorr q5, q6, q0
-; CHECK-NEXT:    vorr q6, q6, q4
-; CHECK-NEXT:    vcmp.u32 cs, q1, q5
-; CHECK-NEXT:    vpsel q7, q3, q2
+; CHECK-NEXT:    vorr q6, q5, q0
+; CHECK-NEXT:    vorr q5, q5, q4
 ; CHECK-NEXT:    vcmp.u32 cs, q1, q6
-; CHECK-NEXT:    vmov r4, r12, d14
 ; CHECK-NEXT:    vpsel q6, q3, q2
-; CHECK-NEXT:    vmov.16 q5[0], r4
-; CHECK-NEXT:    vmov.16 q5[1], r12
-; CHECK-NEXT:    vmov r4, r12, d15
-; CHECK-NEXT:    vmov.16 q5[2], r4
-; CHECK-NEXT:    vmov.16 q5[3], r12
-; CHECK-NEXT:    vmov r4, r12, d12
-; CHECK-NEXT:    vmov.16 q5[4], r4
-; CHECK-NEXT:    vmov.16 q5[5], r12
-; CHECK-NEXT:    vmov r4, r12, d13
-; CHECK-NEXT:    vmov.16 q5[6], r4
-; CHECK-NEXT:    vmov.16 q5[7], r12
+; CHECK-NEXT:    vcmp.u32 cs, q1, q5
+; CHECK-NEXT:    vpsel q5, q3, q2
+; CHECK-NEXT:    vstrh.32 q6, [r5, #8]
+; CHECK-NEXT:    vstrh.32 q5, [r5]
+; CHECK-NEXT:    vldrw.u32 q5, [r5]
 ; CHECK-NEXT:    vptt.i16 ne, q5, zr
 ; CHECK-NEXT:    vldrbt.s16 q5, [r0], #8
 ; CHECK-NEXT:    vldrbt.s16 q6, [r1], #8
@@ -2438,20 +2422,21 @@ define arm_aapcs_vfpcc void @ssatmul_8t_q7(ptr nocapture readonly %pSrcA, ptr no
 ; CHECK-NEXT:    vstrbt.16 q5, [r2], #8
 ; CHECK-NEXT:    le lr, .LBB17_2
 ; CHECK-NEXT:  .LBB17_3: @ %for.cond.cleanup
-; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
-; CHECK-NEXT:    pop {r4, pc}
+; CHECK-NEXT:    add sp, #16
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  @ %bb.4:
 ; CHECK-NEXT:  .LCPI17_0:
-; CHECK-NEXT:    .long 0 @ 0x0
-; CHECK-NEXT:    .long 1 @ 0x1
-; CHECK-NEXT:    .long 2 @ 0x2
-; CHECK-NEXT:    .long 3 @ 0x3
-; CHECK-NEXT:  .LCPI17_1:
 ; CHECK-NEXT:    .long 4 @ 0x4
 ; CHECK-NEXT:    .long 5 @ 0x5
 ; CHECK-NEXT:    .long 6 @ 0x6
 ; CHECK-NEXT:    .long 7 @ 0x7
+; CHECK-NEXT:  .LCPI17_1:
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 1 @ 0x1
+; CHECK-NEXT:    .long 2 @ 0x2
+; CHECK-NEXT:    .long 3 @ 0x3
 entry:
   %cmp10 = icmp eq i32 %N, 0
   br i1 %cmp10, label %for.cond.cleanup, label %vector.ph
@@ -2496,151 +2481,105 @@ for.cond.cleanup:                                 ; preds = %vector.body, %entry
 define arm_aapcs_vfpcc void @ssatmul_16t_q7(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) {
 ; CHECK-LABEL: ssatmul_16t_q7:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r7, lr}
-; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
-; CHECK-NEXT:    .pad #48
-; CHECK-NEXT:    sub sp, #48
+; CHECK-NEXT:    .pad #80
+; CHECK-NEXT:    sub sp, #80
 ; CHECK-NEXT:    cmp r3, #0
-; CHECK-NEXT:    beq.w .LBB18_3
+; CHECK-NEXT:    beq .LBB18_3
 ; CHECK-NEXT:  @ %bb.1: @ %vector.ph
-; CHECK-NEXT:    add.w r12, r3, #15
-; CHECK-NEXT:    adr r4, .LCPI18_0
-; CHECK-NEXT:    bic r12, r12, #15
-; CHECK-NEXT:    vldrw.u32 q0, [r4]
-; CHECK-NEXT:    sub.w r12, r12, #16
-; CHECK-NEXT:    mov.w lr, #1
-; CHECK-NEXT:    adr r4, .LCPI18_1
-; CHECK-NEXT:    movs r5, #0
-; CHECK-NEXT:    add.w lr, lr, r12, lsr #4
-; CHECK-NEXT:    sub.w r12, r3, #1
-; CHECK-NEXT:    vstrw.32 q0, [sp, #32] @ 16-byte Spill
-; CHECK-NEXT:    vldrw.u32 q0, [r4]
-; CHECK-NEXT:    adr r4, .LCPI18_2
-; CHECK-NEXT:    vdup.32 q1, r12
+; CHECK-NEXT:    add.w r6, r3, #15
+; CHECK-NEXT:    movs r5, #1
+; CHECK-NEXT:    bic r6, r6, #15
+; CHECK-NEXT:    add r4, sp, #48
+; CHECK-NEXT:    subs r6, #16
 ; CHECK-NEXT:    vmov.i8 q2, #0x0
-; CHECK-NEXT:    vstrw.32 q0, [sp, #16] @ 16-byte Spill
-; CHECK-NEXT:    vldrw.u32 q0, [r4]
-; CHECK-NEXT:    adr r4, .LCPI18_3
 ; CHECK-NEXT:    vmov.i8 q3, #0xff
-; CHECK-NEXT:    vldrw.u32 q6, [r4]
+; CHECK-NEXT:    add.w lr, r5, r6, lsr #4
+; CHECK-NEXT:    adr r5, .LCPI18_0
+; CHECK-NEXT:    subs r6, r3, #1
+; CHECK-NEXT:    vldrw.u32 q0, [r5]
+; CHECK-NEXT:    vdup.32 q1, r6
+; CHECK-NEXT:    adr r6, .LCPI18_1
+; CHECK-NEXT:    vstrw.32 q0, [sp, #16] @ 16-byte Spill
+; CHECK-NEXT:    vldrw.u32 q0, [r6]
+; CHECK-NEXT:    adr r6, .LCPI18_2
+; CHECK-NEXT:    vldrw.u32 q5, [r6]
+; CHECK-NEXT:    adr r6, .LCPI18_3
+; CHECK-NEXT:    vldrw.u32 q6, [r6]
+; CHECK-NEXT:    add r5, sp, #32
+; CHECK-NEXT:    add r6, sp, #64
+; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    vstrw.32 q0, [sp] @ 16-byte Spill
 ; CHECK-NEXT:  .LBB18_2: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q4, [sp, #32] @ 16-byte Reload
-; CHECK-NEXT:    vdup.32 q0, r5
-; CHECK-NEXT:    adds r5, #16
-; CHECK-NEXT:    vorr q4, q0, q4
-; CHECK-NEXT:    vcmp.u32 cs, q1, q4
-; CHECK-NEXT:    vpsel q4, q3, q2
-; CHECK-NEXT:    vmov r4, r3, d8
-; CHECK-NEXT:    vmov.16 q7[0], r4
-; CHECK-NEXT:    vmov.16 q7[1], r3
-; CHECK-NEXT:    vmov r3, r4, d9
-; CHECK-NEXT:    vldrw.u32 q4, [sp, #16] @ 16-byte Reload
-; CHECK-NEXT:    vmov.16 q7[2], r3
-; CHECK-NEXT:    vmov.16 q7[3], r4
-; CHECK-NEXT:    vorr q4, q0, q4
-; CHECK-NEXT:    vcmp.u32 cs, q1, q4
-; CHECK-NEXT:    vpsel q4, q3, q2
-; CHECK-NEXT:    vmov r3, r4, d8
-; CHECK-NEXT:    vmov.16 q7[4], r3
-; CHECK-NEXT:    vmov.16 q7[5], r4
-; CHECK-NEXT:    vmov r3, r4, d9
-; CHECK-NEXT:    vmov.16 q7[6], r3
-; CHECK-NEXT:    vmov.16 q7[7], r4
-; CHECK-NEXT:    vcmp.i16 ne, q7, zr
-; CHECK-NEXT:    vpsel q4, q3, q2
-; CHECK-NEXT:    vmov.u16 r3, q4[0]
-; CHECK-NEXT:    vmov.8 q7[0], r3
-; CHECK-NEXT:    vmov.u16 r3, q4[1]
-; CHECK-NEXT:    vmov.8 q7[1], r3
-; CHECK-NEXT:    vmov.u16 r3, q4[2]
-; CHECK-NEXT:    vmov.8 q7[2], r3
-; CHECK-NEXT:    vmov.u16 r3, q4[3]
-; CHECK-NEXT:    vmov.8 q7[3], r3
-; CHECK-NEXT:    vmov.u16 r3, q4[4]
-; CHECK-NEXT:    vmov.8 q7[4], r3
-; CHECK-NEXT:    vmov.u16 r3, q4[5]
-; CHECK-NEXT:    vmov.8 q7[5], r3
-; CHECK-NEXT:    vmov.u16 r3, q4[6]
-; CHECK-NEXT:    vmov.8 q7[6], r3
-; CHECK-NEXT:    vmov.u16 r3, q4[7]
-; CHECK-NEXT:    vldrw.u32 q4, [sp] @ 16-byte Reload
-; CHECK-NEXT:    vmov.8 q7[7], r3
-; CHECK-NEXT:    vorr q4, q0, q4
-; CHECK-NEXT:    vorr q0, q0, q6
-; CHECK-NEXT:    vcmp.u32 cs, q1, q4
-; CHECK-NEXT:    vpsel q5, q3, q2
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #16] @ 16-byte Reload
+; CHECK-NEXT:    vdup.32 q7, r3
+; CHECK-NEXT:    adds r3, #16
+; CHECK-NEXT:    vorr q0, q7, q0
 ; CHECK-NEXT:    vcmp.u32 cs, q1, q0
-; CHECK-NEXT:    vmov r3, r4, d10
 ; CHECK-NEXT:    vpsel q0, q3, q2
-; CHECK-NEXT:    vmov.16 q4[0], r3
-; CHECK-NEXT:    vmov.16 q4[1], r4
-; CHECK-NEXT:    vmov r3, r4, d11
-; CHECK-NEXT:    vmov.16 q4[2], r3
-; CHECK-NEXT:    vmov.16 q4[3], r4
-; CHECK-NEXT:    vmov r3, r4, d0
-; CHECK-NEXT:    vmov.16 q4[4], r3
-; CHECK-NEXT:    vmov.16 q4[5], r4
-; CHECK-NEXT:    vmov r3, r4, d1
-; CHECK-NEXT:    vmov.16 q4[6], r3
-; CHECK-NEXT:    vmov.16 q4[7], r4
-; CHECK-NEXT:    vcmp.i16 ne, q4, zr
+; CHECK-NEXT:    vstrh.32 q0, [r4, #8]
+; CHECK-NEXT:    vldrw.u32 q0, [sp] @ 16-byte Reload
+; CHECK-NEXT:    vorr q0, q7, q0
+; CHECK-NEXT:    vcmp.u32 cs, q1, q0
+; CHECK-NEXT:    vpsel q0, q3, q2
+; CHECK-NEXT:    vstrh.32 q0, [r4]
+; CHECK-NEXT:    vorr q0, q7, q5
+; CHECK-NEXT:    vcmp.u32 cs, q1, q0
 ; CHECK-NEXT:    vpsel q0, q3, q2
-; CHECK-NEXT:    vmov.u16 r3, q0[0]
-; CHECK-NEXT:    vmov.8 q7[8], r3
-; CHECK-NEXT:    vmov.u16 r3, q0[1]
-; CHECK-NEXT:    vmov.8 q7[9], r3
-; CHECK-NEXT:    vmov.u16 r3, q0[2]
-; CHECK-NEXT:    vmov.8 q7[10], r3
-; CHECK-NEXT:    vmov.u16 r3, q0[3]
-; CHECK-NEXT:    vmov.8 q7[11], r3
-; CHECK-NEXT:    vmov.u16 r3, q0[4]
-; CHECK-NEXT:    vmov.8 q7[12], r3
-; CHECK-NEXT:    vmov.u16 r3, q0[5]
-; CHECK-NEXT:    vmov.8 q7[13], r3
-; CHECK-NEXT:    vmov.u16 r3, q0[6]
-; CHECK-NEXT:    vmov.8 q7[14], r3
-; CHECK-NEXT:    vmov.u16 r3, q0[7]
-; CHECK-NEXT:    vmov.8 q7[15], r3
-; CHECK-NEXT:    vptt.i8 ne, q7, zr
+; CHECK-NEXT:    vstrh.32 q0, [r5, #8]
+; CHECK-NEXT:    vorr q0, q7, q6
+; CHECK-NEXT:    vcmp.u32 cs, q1, q0
+; CHECK-NEXT:    vpsel q0, q3, q2
+; CHECK-NEXT:    vstrh.32 q0, [r5]
+; CHECK-NEXT:    vldrw.u32 q0, [r4]
+; CHECK-NEXT:    vcmp.i16 ne, q0, zr
+; CHECK-NEXT:    vpsel q0, q3, q2
+; CHECK-NEXT:    vstrb.16 q0, [r6, #8]
+; CHECK-NEXT:    vldrw.u32 q0, [r5]
+; CHECK-NEXT:    vcmp.i16 ne, q0, zr
+; CHECK-NEXT:    vpsel q0, q3, q2
+; CHECK-NEXT:    vstrb.16 q0, [r6]
+; CHECK-NEXT:    vldrw.u32 q0, [r6]
+; CHECK-NEXT:    vptt.i8 ne, q0, zr
 ; CHECK-NEXT:    vldrbt.u8 q0, [r0], #16
-; CHECK-NEXT:    vldrbt.u8 q4, [r1], #16
-; CHECK-NEXT:    vmullt.s8 q5, q4, q0
-; CHECK-NEXT:    vmullb.s8 q0, q4, q0
+; CHECK-NEXT:    vldrbt.u8 q7, [r1], #16
+; CHECK-NEXT:    vmullt.s8 q4, q7, q0
+; CHECK-NEXT:    vmullb.s8 q0, q7, q0
 ; CHECK-NEXT:    vqshrnb.s16 q0, q0, #7
-; CHECK-NEXT:    vqshrnt.s16 q0, q5, #7
+; CHECK-NEXT:    vqshrnt.s16 q0, q4, #7
 ; CHECK-NEXT:    vpst
 ; CHECK-NEXT:    vstrbt.8 q0, [r2], #16
 ; CHECK-NEXT:    le lr, .LBB18_2
 ; CHECK-NEXT:  .LBB18_3: @ %for.cond.cleanup
-; CHECK-NEXT:    add sp, #48
+; CHECK-NEXT:    add sp, #80
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
-; CHECK-NEXT:    pop {r4, r5, r7, pc}
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  @ %bb.4:
 ; CHECK-NEXT:  .LCPI18_0:
-; CHECK-NEXT:    .long 0 @ 0x0
-; CHECK-NEXT:    .long 1 @ 0x1
-; CHECK-NEXT:    .long 2 @ 0x2
-; CHECK-NEXT:    .long 3 @ 0x3
+; CHECK-NEXT:    .long 12 @ 0xc
+; CHECK-NEXT:    .long 13 @ 0xd
+; CHECK-NEXT:    .long 14 @ 0xe
+; CHECK-NEXT:    .long 15 @ 0xf
 ; CHECK-NEXT:  .LCPI18_1:
-; CHECK-NEXT:    .long 4 @ 0x4
-; CHECK-NEXT:    .long 5 @ 0x5
-; CHECK-NEXT:    .long 6 @ 0x6
-; CHECK-NEXT:    .long 7 @ 0x7
-; CHECK-NEXT:  .LCPI18_2:
 ; CHECK-NEXT:    .long 8 @ 0x8
 ; CHECK-NEXT:    .long 9 @ 0x9
 ; CHECK-NEXT:    .long 10 @ 0xa
 ; CHECK-NEXT:    .long 11 @ 0xb
+; CHECK-NEXT:  .LCPI18_2:
+; CHECK-NEXT:    .long 4 @ 0x4
+; CHECK-NEXT:    .long 5 @ 0x5
+; CHECK-NEXT:    .long 6 @ 0x6
+; CHECK-NEXT:    .long 7 @ 0x7
 ; CHECK-NEXT:  .LCPI18_3:
-; CHECK-NEXT:    .long 12 @ 0xc
-; CHECK-NEXT:    .long 13 @ 0xd
-; CHECK-NEXT:    .long 14 @ 0xe
-; CHECK-NEXT:    .long 15 @ 0xf
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 1 @ 0x1
+; CHECK-NEXT:    .long 2 @ 0x2
+; CHECK-NEXT:    .long 3 @ 0x3
 entry:
   %cmp10 = icmp eq i32 %N, 0
   br i1 %cmp10, label %for.cond.cleanup, label %vector.ph
@@ -2685,151 +2624,105 @@ for.cond.cleanup:                                 ; preds = %vector.body, %entry
 define arm_aapcs_vfpcc void @ssatmul_16ti_q7(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) {
 ; CHECK-LABEL: ssatmul_16ti_q7:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r7, lr}
-; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
-; CHECK-NEXT:    .pad #48
-; CHECK-NEXT:    sub sp, #48
+; CHECK-NEXT:    .pad #80
+; CHECK-NEXT:    sub sp, #80
 ; CHECK-NEXT:    cmp r3, #0
-; CHECK-NEXT:    beq.w .LBB19_3
+; CHECK-NEXT:    beq .LBB19_3
 ; CHECK-NEXT:  @ %bb.1: @ %vector.ph
-; CHECK-NEXT:    add.w r12, r3, #15
-; CHECK-NEXT:    adr r4, .LCPI19_0
-; CHECK-NEXT:    bic r12, r12, #15
-; CHECK-NEXT:    vldrw.u32 q0, [r4]
-; CHECK-NEXT:    sub.w r12, r12, #16
-; CHECK-NEXT:    mov.w lr, #1
-; CHECK-NEXT:    adr r4, .LCPI19_1
-; CHECK-NEXT:    movs r5, #0
-; CHECK-NEXT:    add.w lr, lr, r12, lsr #4
-; CHECK-NEXT:    sub.w r12, r3, #1
-; CHECK-NEXT:    vstrw.32 q0, [sp, #32] @ 16-byte Spill
-; CHECK-NEXT:    vldrw.u32 q0, [r4]
-; CHECK-NEXT:    adr r4, .LCPI19_2
-; CHECK-NEXT:    vdup.32 q1, r12
+; CHECK-NEXT:    add.w r6, r3, #15
+; CHECK-NEXT:    movs r5, #1
+; CHECK-NEXT:    bic r6, r6, #15
+; CHECK-NEXT:    add r4, sp, #48
+; CHECK-NEXT:    subs r6, #16
 ; CHECK-NEXT:    vmov.i8 q2, #0x0
-; CHECK-NEXT:    vstrw.32 q0, [sp, #16] @ 16-byte Spill
-; CHECK-NEXT:    vldrw.u32 q0, [r4]
-; CHECK-NEXT:    adr r4, .LCPI19_3
 ; CHECK-NEXT:    vmov.i8 q3, #0xff
-; CHECK-NEXT:    vldrw.u32 q6, [r4]
+; CHECK-NEXT:    add.w lr, r5, r6, lsr #4
+; CHECK-NEXT:    adr r5, .LCPI19_0
+; CHECK-NEXT:    subs r6, r3, #1
+; CHECK-NEXT:    vldrw.u32 q0, [r5]
+; CHECK-NEXT:    vdup.32 q1, r6
+; CHECK-NEXT:    adr r6, .LCPI19_1
+; CHECK-NEXT:    vstrw.32 q0, [sp, #16] @ 16-byte Spill
+; CHECK-NEXT:    vldrw.u32 q0, [r6]
+; CHECK-NEXT:    adr r6, .LCPI19_2
+; CHECK-NEXT:    vldrw.u32 q5, [r6]
+; CHECK-NEXT:    adr r6, .LCPI19_3
+; CHECK-NEXT:    vldrw.u32 q6, [r6]
+; CHECK-NEXT:    add r5, sp, #32
+; CHECK-NEXT:    add r6, sp, #64
+; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    vstrw.32 q0, [sp] @ 16-byte Spill
 ; CHECK-NEXT:  .LBB19_2: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrw.u32 q4, [sp, #32] @ 16-byte Reload
-; CHECK-NEXT:    vdup.32 q0, r5
-; CHECK-NEXT:    adds r5, #16
-; CHECK-NEXT:    vorr q4, q0, q4
-; CHECK-NEXT:    vcmp.u32 cs, q1, q4
-; CHECK-NEXT:    vpsel q4, q3, q2
-; CHECK-NEXT:    vmov r4, r3, d8
-; CHECK-NEXT:    vmov.16 q7[0], r4
-; CHECK-NEXT:    vmov.16 q7[1], r3
-; CHECK-NEXT:    vmov r3, r4, d9
-; CHECK-NEXT:    vldrw.u32 q4, [sp, #16] @ 16-byte Reload
-; CHECK-NEXT:    vmov.16 q7[2], r3
-; CHECK-NEXT:    vmov.16 q7[3], r4
-; CHECK-NEXT:    vorr q4, q0, q4
-; CHECK-NEXT:    vcmp.u32 cs, q1, q4
-; CHECK-NEXT:    vpsel q4, q3, q2
-; CHECK-NEXT:    vmov r3, r4, d8
-; CHECK-NEXT:    vmov.16 q7[4], r3
-; CHECK-NEXT:    vmov.16 q7[5], r4
-; CHECK-NEXT:    vmov r3, r4, d9
-; CHECK-NEXT:    vmov.16 q7[6], r3
-; CHECK-NEXT:    vmov.16 q7[7], r4
-; CHECK-NEXT:    vcmp.i16 ne, q7, zr
-; CHECK-NEXT:    vpsel q4, q3, q2
-; CHECK-NEXT:    vmov.u16 r3, q4[0]
-; CHECK-NEXT:    vmov.8 q7[0], r3
-; CHECK-NEXT:    vmov.u16 r3, q4[1]
-; CHECK-NEXT:    vmov.8 q7[1], r3
-; CHECK-NEXT:    vmov.u16 r3, q4[2]
-; CHECK-NEXT:    vmov.8 q7[2], r3
-; CHECK-NEXT:    vmov.u16 r3, q4[3]
-; CHECK-NEXT:    vmov.8 q7[3], r3
-; CHECK-NEXT:    vmov.u16 r3, q4[4]
-; CHECK-NEXT:    vmov.8 q7[4], r3
-; CHECK-NEXT:    vmov.u16 r3, q4[5]
-; CHECK-NEXT:    vmov.8 q7[5], r3
-; CHECK-NEXT:    vmov.u16 r3, q4[6]
-; CHECK-NEXT:    vmov.8 q7[6], r3
-; CHECK-NEXT:    vmov.u16 r3, q4[7]
-; CHECK-NEXT:    vldrw.u32 q4, [sp] @ 16-byte Reload
-; CHECK-NEXT:    vmov.8 q7[7], r3
-; CHECK-NEXT:    vorr q4, q0, q4
-; CHECK-NEXT:    vorr q0, q0, q6
-; CHECK-NEXT:    vcmp.u32 cs, q1, q4
-; CHECK-NEXT:    vpsel q5, q3, q2
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #16] @ 16-byte Reload
+; CHECK-NEXT:    vdup.32 q7, r3
+; CHECK-NEXT:    adds r3, #16
+; CHECK-NEXT:    vorr q0, q7, q0
 ; CHECK-NEXT:    vcmp.u32 cs, q1, q0
-; CHECK-NEXT:    vmov r3, r4, d10
 ; CHECK-NEXT:    vpsel q0, q3, q2
-; CHECK-NEXT:    vmov.16 q4[0], r3
-; CHECK-NEXT:    vmov.16 q4[1], r4
-; CHECK-NEXT:    vmov r3, r4, d11
-; CHECK-NEXT:    vmov.16 q4[2], r3
-; CHECK-NEXT:    vmov.16 q4[3], r4
-; CHECK-NEXT:    vmov r3, r4, d0
-; CHECK-NEXT:    vmov.16 q4[4], r3
-; CHECK-NEXT:    vmov.16 q4[5], r4
-; CHECK-NEXT:    vmov r3, r4, d1
-; CHECK-NEXT:    vmov.16 q4[6], r3
-; CHECK-NEXT:    vmov.16 q4[7], r4
-; CHECK-NEXT:    vcmp.i16 ne, q4, zr
+; CHECK-NEXT:    vstrh.32 q0, [r4, #8]
+; CHECK-NEXT:    vldrw.u32 q0, [sp] @ 16-byte Reload
+; CHECK-NEXT:    vorr q0, q7, q0
+; CHECK-NEXT:    vcmp.u32 cs, q1, q0
+; CHECK-NEXT:    vpsel q0, q3, q2
+; CHECK-NEXT:    vstrh.32 q0, [r4]
+; CHECK-NEXT:    vorr q0, q7, q5
+; CHECK-NEXT:    vcmp.u32 cs, q1, q0
+; CHECK-NEXT:    vpsel q0, q3, q2
+; CHECK-NEXT:    vstrh.32 q0, [r5, #8]
+; CHECK-NEXT:    vorr q0, q7, q6
+; CHECK-NEXT:    vcmp.u32 cs, q1, q0
 ; CHECK-NEXT:    vpsel q0, q3, q2
-; CHECK-NEXT:    vmov.u16 r3, q0[0]
-; CHECK-NEXT:    vmov.8 q7[8], r3
-; CHECK-NEXT:    vmov.u16 r3, q0[1]
-; CHECK-NEXT:    vmov.8 q7[9], r3
-; CHECK-NEXT:    vmov.u16 r3, q0[2]
-; CHECK-NEXT:    vmov.8 q7[10], r3
-; CHECK-NEXT:    vmov.u16 r3, q0[3]
-; CHECK-NEXT:    vmov.8 q7[11], r3
-; CHECK-NEXT:    vmov.u16 r3, q0[4]
-; CHECK-NEXT:    vmov.8 q7[12], r3
-; CHECK-NEXT:    vmov.u16 r3, q0[5]
-; CHECK-NEXT:    vmov.8 q7[13], r3
-; CHECK-NEXT:    vmov.u16 r3, q0[6]
-; CHECK-NEXT:    vmov.8 q7[14], r3
-; CHECK-NEXT:    vmov.u16 r3, q0[7]
-; CHECK-NEXT:    vmov.8 q7[15], r3
-; CHECK-NEXT:    vptt.i8 ne, q7, zr
+; CHECK-NEXT:    vstrh.32 q0, [r5]
+; CHECK-NEXT:    vldrw.u32 q0, [r4]
+; CHECK-NEXT:    vcmp.i16 ne, q0, zr
+; CHECK-NEXT:    vpsel q0, q3, q2
+; CHECK-NEXT:    vstrb.16 q0, [r6, #8]
+; CHECK-NEXT:    vldrw.u32 q0, [r5]
+; CHECK-NEXT:    vcmp.i16 ne, q0, zr
+; CHECK-NEXT:    vpsel q0, q3, q2
+; CHECK-NEXT:    vstrb.16 q0, [r6]
+; CHECK-NEXT:    vldrw.u32 q0, [r6]
+; CHECK-NEXT:    vptt.i8 ne, q0, zr
 ; CHECK-NEXT:    vldrbt.u8 q0, [r0], #16
-; CHECK-NEXT:    vldrbt.u8 q4, [r1], #16
-; CHECK-NEXT:    vmullt.s8 q5, q4, q0
-; CHECK-NEXT:    vmullb.s8 q0, q4, q0
+; CHECK-NEXT:    vldrbt.u8 q7, [r1], #16
+; CHECK-NEXT:    vmullt.s8 q4, q7, q0
+; CHECK-NEXT:    vmullb.s8 q0, q7, q0
 ; CHECK-NEXT:    vqshrnb.s16 q0, q0, #7
-; CHECK-NEXT:    vqshrnt.s16 q0, q5, #7
+; CHECK-NEXT:    vqshrnt.s16 q0, q4, #7
 ; CHECK-NEXT:    vpst
 ; CHECK-NEXT:    vstrbt.8 q0, [r2], #16
 ; CHECK-NEXT:    le lr, .LBB19_2
 ; CHECK-NEXT:  .LBB19_3: @ %for.cond.cleanup
-; CHECK-NEXT:    add sp, #48
+; CHECK-NEXT:    add sp, #80
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
-; CHECK-NEXT:    pop {r4, r5, r7, pc}
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  @ %bb.4:
 ; CHECK-NEXT:  .LCPI19_0:
-; CHECK-NEXT:    .long 0 @ 0x0
-; CHECK-NEXT:    .long 1 @ 0x1
-; CHECK-NEXT:    .long 2 @ 0x2
-; CHECK-NEXT:    .long 3 @ 0x3
+; CHECK-NEXT:    .long 12 @ 0xc
+; CHECK-NEXT:    .long 13 @ 0xd
+; CHECK-NEXT:    .long 14 @ 0xe
+; CHECK-NEXT:    .long 15 @ 0xf
 ; CHECK-NEXT:  .LCPI19_1:
-; CHECK-NEXT:    .long 4 @ 0x4
-; CHECK-NEXT:    .long 5 @ 0x5
-; CHECK-NEXT:    .long 6 @ 0x6
-; CHECK-NEXT:    .long 7 @ 0x7
-; CHECK-NEXT:  .LCPI19_2:
 ; CHECK-NEXT:    .long 8 @ 0x8
 ; CHECK-NEXT:    .long 9 @ 0x9
 ; CHECK-NEXT:    .long 10 @ 0xa
 ; CHECK-NEXT:    .long 11 @ 0xb
+; CHECK-NEXT:  .LCPI19_2:
+; CHECK-NEXT:    .long 4 @ 0x4
+; CHECK-NEXT:    .long 5 @ 0x5
+; CHECK-NEXT:    .long 6 @ 0x6
+; CHECK-NEXT:    .long 7 @ 0x7
 ; CHECK-NEXT:  .LCPI19_3:
-; CHECK-NEXT:    .long 12 @ 0xc
-; CHECK-NEXT:    .long 13 @ 0xd
-; CHECK-NEXT:    .long 14 @ 0xe
-; CHECK-NEXT:    .long 15 @ 0xf
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 1 @ 0x1
+; CHECK-NEXT:    .long 2 @ 0x2
+; CHECK-NEXT:    .long 3 @ 0x3
 entry:
   %cmp10 = icmp eq i32 %N, 0
   br i1 %cmp10, label %for.cond.cleanup, label %vector.ph
diff --git a/llvm/test/CodeGen/WebAssembly/libcalls.ll b/llvm/test/CodeGen/WebAssembly/libcalls.ll
index 19102ecd18..4f57c347a1 100644
--- a/llvm/test/CodeGen/WebAssembly/libcalls.ll
+++ b/llvm/test/CodeGen/WebAssembly/libcalls.ll
@@ -18,6 +18,7 @@ declare double @llvm.pow.f64(double, double)
 declare double @llvm.powi.f64.i32(double, i32)
 declare double @llvm.log.f64(double)
 declare double @llvm.exp.f64(double)
+declare double @llvm.exp10.f64(double)
 declare double @llvm.ldexp.f64.i32(double, i32)
 declare {double, i32} @llvm.frexp.f64.i32(double)
 declare i32 @llvm.lround(double)
@@ -239,38 +240,39 @@ define double @f64libcalls(double %x, double %y, i32 %z) {
 ; CHECK:         .functype f64libcalls (f64, f64, i32) -> (f64)
 ; CHECK-NEXT:    .local i32
 ; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    global.get $push10=, __stack_pointer
-; CHECK-NEXT:    i32.const $push11=, 16
-; CHECK-NEXT:    i32.sub $push17=, $pop10, $pop11
-; CHECK-NEXT:    local.tee $push16=, 3, $pop17
-; CHECK-NEXT:    global.set __stack_pointer, $pop16
-; CHECK-NEXT:    local.get $push21=, 0
-; CHECK-NEXT:    local.get $push18=, 0
-; CHECK-NEXT:    call $push0=, cos, $pop18
+; CHECK-NEXT:    global.get $push11=, __stack_pointer
+; CHECK-NEXT:    i32.const $push12=, 16
+; CHECK-NEXT:    i32.sub $push18=, $pop11, $pop12
+; CHECK-NEXT:    local.tee $push17=, 3, $pop18
+; CHECK-NEXT:    global.set __stack_pointer, $pop17
+; CHECK-NEXT:    local.get $push22=, 0
+; CHECK-NEXT:    local.get $push19=, 0
+; CHECK-NEXT:    call $push0=, cos, $pop19
 ; CHECK-NEXT:    call $push1=, log10, $pop0
-; CHECK-NEXT:    local.get $push19=, 1
-; CHECK-NEXT:    call $push2=, pow, $pop1, $pop19
-; CHECK-NEXT:    local.get $push20=, 2
-; CHECK-NEXT:    call $push3=, __powidf2, $pop2, $pop20
+; CHECK-NEXT:    local.get $push20=, 1
+; CHECK-NEXT:    call $push2=, pow, $pop1, $pop20
+; CHECK-NEXT:    local.get $push21=, 2
+; CHECK-NEXT:    call $push3=, __powidf2, $pop2, $pop21
 ; CHECK-NEXT:    call $push4=, log, $pop3
 ; CHECK-NEXT:    call $push5=, exp, $pop4
-; CHECK-NEXT:    call $push6=, cbrt, $pop5
-; CHECK-NEXT:    call $push7=, lround, $pop6
-; CHECK-NEXT:    call $push8=, ldexp, $pop21, $pop7
-; CHECK-NEXT:    local.get $push22=, 3
-; CHECK-NEXT:    i32.const $push14=, 12
-; CHECK-NEXT:    i32.add $push15=, $pop22, $pop14
-; CHECK-NEXT:    call $push23=, frexp, $pop8, $pop15
-; CHECK-NEXT:    local.set 0, $pop23
-; CHECK-NEXT:    local.get $push24=, 3
-; CHECK-NEXT:    i32.load $push9=, 12($pop24)
-; CHECK-NEXT:    call escape_value, $pop9
+; CHECK-NEXT:    call $push6=, exp10, $pop5
+; CHECK-NEXT:    call $push7=, cbrt, $pop6
+; CHECK-NEXT:    call $push8=, lround, $pop7
+; CHECK-NEXT:    call $push9=, ldexp, $pop22, $pop8
+; CHECK-NEXT:    local.get $push23=, 3
+; CHECK-NEXT:    i32.const $push15=, 12
+; CHECK-NEXT:    i32.add $push16=, $pop23, $pop15
+; CHECK-NEXT:    call $push24=, frexp, $pop9, $pop16
+; CHECK-NEXT:    local.set 0, $pop24
 ; CHECK-NEXT:    local.get $push25=, 3
-; CHECK-NEXT:    i32.const $push12=, 16
-; CHECK-NEXT:    i32.add $push13=, $pop25, $pop12
-; CHECK-NEXT:    global.set __stack_pointer, $pop13
-; CHECK-NEXT:    local.get $push26=, 0
-; CHECK-NEXT:    return $pop26
+; CHECK-NEXT:    i32.load $push10=, 12($pop25)
+; CHECK-NEXT:    call escape_value, $pop10
+; CHECK-NEXT:    local.get $push26=, 3
+; CHECK-NEXT:    i32.const $push13=, 16
+; CHECK-NEXT:    i32.add $push14=, $pop26, $pop13
+; CHECK-NEXT:    global.set __stack_pointer, $pop14
+; CHECK-NEXT:    local.get $push27=, 0
+; CHECK-NEXT:    return $pop27
 
 
  %a = call double @llvm.cos.f64(double %x)
@@ -279,10 +281,11 @@ define double @f64libcalls(double %x, double %y, i32 %z) {
  %d = call double @llvm.powi.f64.i32(double %c, i32 %z)
  %e = call double @llvm.log.f64(double %d)
  %f = call double @llvm.exp.f64(double %e)
- %g = call fast double @llvm.pow.f64(double %f, double 0x3FD5555555555555)
- %h = call i32 @llvm.lround(double %g)
- %i = call double @llvm.ldexp.f64.i32(double %x, i32 %h);
- %result = call {double, i32} @llvm.frexp.f64.i32(double %i)
+ %g = call double @llvm.exp10.f64(double %f)
+ %h = call fast double @llvm.pow.f64(double %g, double 0x3FD5555555555555)
+ %i = call i32 @llvm.lround(double %h)
+ %j = call double @llvm.ldexp.f64.i32(double %x, i32 %i);
+ %result = call {double, i32} @llvm.frexp.f64.i32(double %j)
  %result.0 = extractvalue { double, i32 } %result, 0
  %result.1 = extractvalue { double, i32 } %result, 1
  call void @escape_value(i32 %result.1)
diff --git a/llvm/test/CodeGen/X86/AMX/amx-config.ll b/llvm/test/CodeGen/X86/AMX/amx-config.ll
index 74282a0afe..275c85389c 100644
--- a/llvm/test/CodeGen/X86/AMX/amx-config.ll
+++ b/llvm/test/CodeGen/X86/AMX/amx-config.ll
@@ -79,10 +79,10 @@ define <4 x i32> @test_api(i32 %0, i16 signext %1, i16 signext %2, <4 x i32> %xm
 ; AVX1-LABEL: test_api:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    movups %xmm1, -{{[0-9]+}}(%rsp)
-; AVX1-NEXT:    movups %xmm1, -{{[0-9]+}}(%rsp)
-; AVX1-NEXT:    movups %xmm1, -{{[0-9]+}}(%rsp)
-; AVX1-NEXT:    movups %xmm1, -{{[0-9]+}}(%rsp)
+; AVX1-NEXT:    vmovups %xmm1, -{{[0-9]+}}(%rsp)
+; AVX1-NEXT:    vmovups %xmm1, -{{[0-9]+}}(%rsp)
+; AVX1-NEXT:    vmovups %xmm1, -{{[0-9]+}}(%rsp)
+; AVX1-NEXT:    vmovups %xmm1, -{{[0-9]+}}(%rsp)
 ; AVX1-NEXT:    movb $1, -{{[0-9]+}}(%rsp)
 ; AVX1-NEXT:    movw %dx, -{{[0-9]+}}(%rsp)
 ; AVX1-NEXT:    movw %dx, -{{[0-9]+}}(%rsp)
diff --git a/llvm/test/CodeGen/X86/AMX/amx-configO2toO0-precfg.ll b/llvm/test/CodeGen/X86/AMX/amx-configO2toO0-precfg.ll
deleted file mode 100644
index 82b9746c41..0000000000
--- a/llvm/test/CodeGen/X86/AMX/amx-configO2toO0-precfg.ll
+++ /dev/null
@@ -1,178 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown-unknown -mattr=+amx-int8 -mattr=+avx512f -pre-amx-config -S | FileCheck %s
-
-@buf = dso_local global [1024 x i8] zeroinitializer, align 16
-@buf2 = dso_local global [1024 x i8] zeroinitializer, align 16
-
-define dso_local void @test_api(i32 %cond, i16 signext %row, i16 signext %col) local_unnamed_addr {
-; CHECK-LABEL: @test_api(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = alloca <16 x i32>, align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = alloca <16 x i32>, align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = alloca <16 x i32>, align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = alloca <16 x i32>, align 4
-; CHECK-NEXT:    [[TMP4:%.*]] = alloca <16 x i32>, align 4
-; CHECK-NEXT:    [[TMP5:%.*]] = alloca <16 x i32>, align 4
-; CHECK-NEXT:    [[TMP6:%.*]] = alloca <16 x i32>, align 4
-; CHECK-NEXT:    [[TMP7:%.*]] = alloca <16 x i32>, align 4
-; CHECK-NEXT:    [[I:%.*]] = alloca <256 x i32>, align 1024
-; CHECK-NEXT:    [[I2:%.*]] = alloca <256 x i32>, align 1024
-; CHECK-NEXT:    [[I4:%.*]] = alloca <256 x i32>, align 1024
-; CHECK-NEXT:    [[I6:%.*]] = alloca <256 x i32>, align 1024
-; CHECK-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i32 [[COND:%.*]], 0
-; CHECK-NEXT:    br i1 [[TOBOOL_NOT]], label [[IF_ELSE:%.*]], label [[IF_THEN:%.*]]
-; CHECK:       if.then:
-; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr [[TMP7]], align 4
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i64 0
-; CHECK-NEXT:    store i8 1, ptr [[TMP8]], align 1
-; CHECK-NEXT:    [[AMX_TMM_0_SHAPE_ROW:%.*]] = getelementptr i8, ptr [[TMP7]], i64 48
-; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[TMP7]], i64 16
-; CHECK-NEXT:    [[TMP10:%.*]] = trunc i16 [[ROW:%.*]] to i8
-; CHECK-NEXT:    store i8 [[TMP10]], ptr [[AMX_TMM_0_SHAPE_ROW]], align 1
-; CHECK-NEXT:    store i16 8, ptr [[TMP9]], align 2
-; CHECK-NEXT:    call void @llvm.x86.ldtilecfg.internal(ptr [[TMP7]])
-; CHECK-NEXT:    [[I8:%.*]] = tail call x86_amx @llvm.x86.tileloadd64.internal(i16 [[ROW]], i16 8, ptr @buf, i64 32)
-; CHECK-NEXT:    call void @llvm.x86.tilestored64.internal(i16 [[ROW]], i16 8, ptr [[I4]], i64 64, x86_amx [[I8]])
-; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr [[TMP6]], align 4
-; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr i8, ptr [[TMP6]], i64 0
-; CHECK-NEXT:    store i8 1, ptr [[TMP11]], align 1
-; CHECK-NEXT:    [[AMX_TMM_0_SHAPE_ROW1:%.*]] = getelementptr i8, ptr [[TMP6]], i64 48
-; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[TMP6]], i64 16
-; CHECK-NEXT:    store i8 8, ptr [[AMX_TMM_0_SHAPE_ROW1]], align 1
-; CHECK-NEXT:    store i16 [[COL:%.*]], ptr [[TMP12]], align 2
-; CHECK-NEXT:    call void @llvm.x86.ldtilecfg.internal(ptr [[TMP6]])
-; CHECK-NEXT:    [[I9:%.*]] = tail call x86_amx @llvm.x86.tileloadd64.internal(i16 8, i16 [[COL]], ptr @buf, i64 32)
-; CHECK-NEXT:    call void @llvm.x86.tilestored64.internal(i16 8, i16 [[COL]], ptr [[I2]], i64 64, x86_amx [[I9]])
-; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr [[TMP5]], align 4
-; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr [[TMP5]], i64 0
-; CHECK-NEXT:    store i8 1, ptr [[TMP13]], align 1
-; CHECK-NEXT:    [[AMX_TMM_0_SHAPE_ROW2:%.*]] = getelementptr i8, ptr [[TMP5]], i64 48
-; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[TMP5]], i64 16
-; CHECK-NEXT:    [[TMP15:%.*]] = trunc i16 [[ROW]] to i8
-; CHECK-NEXT:    store i8 [[TMP15]], ptr [[AMX_TMM_0_SHAPE_ROW2]], align 1
-; CHECK-NEXT:    store i16 [[COL]], ptr [[TMP14]], align 2
-; CHECK-NEXT:    call void @llvm.x86.ldtilecfg.internal(ptr [[TMP5]])
-; CHECK-NEXT:    [[I10:%.*]] = tail call x86_amx @llvm.x86.tileloadd64.internal(i16 [[ROW]], i16 [[COL]], ptr @buf, i64 32)
-; CHECK-NEXT:    call void @llvm.x86.tilestored64.internal(i16 [[ROW]], i16 [[COL]], ptr [[I]], i64 64, x86_amx [[I10]])
-; CHECK-NEXT:    br label [[IF_END:%.*]]
-; CHECK:       if.else:
-; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr [[TMP4]], align 4
-; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr i8, ptr [[TMP4]], i64 0
-; CHECK-NEXT:    store i8 1, ptr [[TMP16]], align 1
-; CHECK-NEXT:    [[AMX_TMM_0_SHAPE_ROW3:%.*]] = getelementptr i8, ptr [[TMP4]], i64 48
-; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[TMP4]], i64 16
-; CHECK-NEXT:    [[TMP18:%.*]] = trunc i16 [[ROW]] to i8
-; CHECK-NEXT:    store i8 [[TMP18]], ptr [[AMX_TMM_0_SHAPE_ROW3]], align 1
-; CHECK-NEXT:    store i16 8, ptr [[TMP17]], align 2
-; CHECK-NEXT:    call void @llvm.x86.ldtilecfg.internal(ptr [[TMP4]])
-; CHECK-NEXT:    [[I11:%.*]] = tail call x86_amx @llvm.x86.tileloadd64.internal(i16 [[ROW]], i16 8, ptr @buf2, i64 32)
-; CHECK-NEXT:    call void @llvm.x86.tilestored64.internal(i16 [[ROW]], i16 8, ptr [[I4]], i64 64, x86_amx [[I11]])
-; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr [[TMP3]], align 4
-; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr i8, ptr [[TMP3]], i64 0
-; CHECK-NEXT:    store i8 1, ptr [[TMP19]], align 1
-; CHECK-NEXT:    [[AMX_TMM_0_SHAPE_ROW4:%.*]] = getelementptr i8, ptr [[TMP3]], i64 48
-; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr i8, ptr [[TMP3]], i64 16
-; CHECK-NEXT:    store i8 8, ptr [[AMX_TMM_0_SHAPE_ROW4]], align 1
-; CHECK-NEXT:    store i16 [[COL]], ptr [[TMP20]], align 2
-; CHECK-NEXT:    call void @llvm.x86.ldtilecfg.internal(ptr [[TMP3]])
-; CHECK-NEXT:    [[I12:%.*]] = tail call x86_amx @llvm.x86.tileloadd64.internal(i16 8, i16 [[COL]], ptr @buf2, i64 32)
-; CHECK-NEXT:    call void @llvm.x86.tilestored64.internal(i16 8, i16 [[COL]], ptr [[I2]], i64 64, x86_amx [[I12]])
-; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr [[TMP2]], align 4
-; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr i8, ptr [[TMP2]], i64 0
-; CHECK-NEXT:    store i8 1, ptr [[TMP21]], align 1
-; CHECK-NEXT:    [[AMX_TMM_0_SHAPE_ROW5:%.*]] = getelementptr i8, ptr [[TMP2]], i64 48
-; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr i8, ptr [[TMP2]], i64 16
-; CHECK-NEXT:    [[TMP23:%.*]] = trunc i16 [[ROW]] to i8
-; CHECK-NEXT:    store i8 [[TMP23]], ptr [[AMX_TMM_0_SHAPE_ROW5]], align 1
-; CHECK-NEXT:    store i16 [[COL]], ptr [[TMP22]], align 2
-; CHECK-NEXT:    call void @llvm.x86.ldtilecfg.internal(ptr [[TMP2]])
-; CHECK-NEXT:    [[I13:%.*]] = tail call x86_amx @llvm.x86.tileloadd64.internal(i16 [[ROW]], i16 [[COL]], ptr @buf2, i64 32)
-; CHECK-NEXT:    call void @llvm.x86.tilestored64.internal(i16 [[ROW]], i16 [[COL]], ptr [[I]], i64 64, x86_amx [[I13]])
-; CHECK-NEXT:    br label [[IF_END]]
-; CHECK:       if.end:
-; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr [[TMP1]], align 4
-; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr i8, ptr [[TMP1]], i64 0
-; CHECK-NEXT:    store i8 1, ptr [[TMP24]], align 1
-; CHECK-NEXT:    [[AMX_TMM_0_SHAPE_ROW6:%.*]] = getelementptr i8, ptr [[TMP1]], i64 48
-; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr i8, ptr [[TMP1]], i64 16
-; CHECK-NEXT:    [[TMP26:%.*]] = trunc i16 [[ROW]] to i8
-; CHECK-NEXT:    store i8 [[TMP26]], ptr [[AMX_TMM_0_SHAPE_ROW6]], align 1
-; CHECK-NEXT:    store i16 [[COL]], ptr [[TMP25]], align 2
-; CHECK-NEXT:    [[AMX_TMM_1_SHAPE_ROW:%.*]] = getelementptr i8, ptr [[TMP1]], i64 49
-; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr i8, ptr [[TMP1]], i64 18
-; CHECK-NEXT:    [[TMP28:%.*]] = trunc i16 [[ROW]] to i8
-; CHECK-NEXT:    store i8 [[TMP28]], ptr [[AMX_TMM_1_SHAPE_ROW]], align 1
-; CHECK-NEXT:    store i16 8, ptr [[TMP27]], align 2
-; CHECK-NEXT:    [[AMX_TMM_2_SHAPE_ROW:%.*]] = getelementptr i8, ptr [[TMP1]], i64 50
-; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr i8, ptr [[TMP1]], i64 20
-; CHECK-NEXT:    store i8 8, ptr [[AMX_TMM_2_SHAPE_ROW]], align 1
-; CHECK-NEXT:    store i16 [[COL]], ptr [[TMP29]], align 2
-; CHECK-NEXT:    [[AMX_TMM_3_SHAPE_ROW:%.*]] = getelementptr i8, ptr [[TMP1]], i64 51
-; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr i8, ptr [[TMP1]], i64 22
-; CHECK-NEXT:    [[TMP31:%.*]] = trunc i16 [[ROW]] to i8
-; CHECK-NEXT:    store i8 [[TMP31]], ptr [[AMX_TMM_3_SHAPE_ROW]], align 1
-; CHECK-NEXT:    store i16 [[COL]], ptr [[TMP30]], align 2
-; CHECK-NEXT:    call void @llvm.x86.ldtilecfg.internal(ptr [[TMP1]])
-; CHECK-NEXT:    [[I14:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[ROW]], i16 8, ptr [[I4]], i64 64)
-; CHECK-NEXT:    [[I15:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 8, i16 [[COL]], ptr [[I2]], i64 64)
-; CHECK-NEXT:    [[I16:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[ROW]], i16 [[COL]], ptr [[I]], i64 64)
-; CHECK-NEXT:    [[I17:%.*]] = tail call x86_amx @llvm.x86.tdpbssd.internal(i16 [[ROW]], i16 [[COL]], i16 8, x86_amx [[I16]], x86_amx [[I14]], x86_amx [[I15]])
-; CHECK-NEXT:    call void @llvm.x86.tilestored64.internal(i16 [[ROW]], i16 [[COL]], ptr [[I6]], i64 64, x86_amx [[I17]])
-; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr [[TMP0]], align 4
-; CHECK-NEXT:    [[TMP32:%.*]] = getelementptr i8, ptr [[TMP0]], i64 0
-; CHECK-NEXT:    store i8 1, ptr [[TMP32]], align 1
-; CHECK-NEXT:    [[AMX_TMM_0_SHAPE_ROW7:%.*]] = getelementptr i8, ptr [[TMP0]], i64 48
-; CHECK-NEXT:    [[TMP33:%.*]] = getelementptr i8, ptr [[TMP0]], i64 16
-; CHECK-NEXT:    [[TMP34:%.*]] = trunc i16 [[ROW]] to i8
-; CHECK-NEXT:    store i8 [[TMP34]], ptr [[AMX_TMM_0_SHAPE_ROW7]], align 1
-; CHECK-NEXT:    store i16 [[COL]], ptr [[TMP33]], align 2
-; CHECK-NEXT:    call void @llvm.x86.ldtilecfg.internal(ptr [[TMP0]])
-; CHECK-NEXT:    [[I18:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[ROW]], i16 [[COL]], ptr [[I6]], i64 64)
-; CHECK-NEXT:    tail call void @llvm.x86.tilestored64.internal(i16 [[ROW]], i16 [[COL]], ptr @buf, i64 32, x86_amx [[I18]])
-; CHECK-NEXT:    ret void
-;
-entry:
-  %i = alloca <256 x i32>, align 1024
-  %i2 = alloca <256 x i32>, align 1024
-  %i4 = alloca <256 x i32>, align 1024
-  %i6 = alloca <256 x i32>, align 1024
-  %tobool.not = icmp eq i32 %cond, 0
-  br i1 %tobool.not, label %if.else, label %if.then
-
-if.then:                                          ; preds = %entry
-  %i8 = tail call x86_amx @llvm.x86.tileloadd64.internal(i16 %row, i16 8, ptr @buf, i64 32)
-  call void @llvm.x86.tilestored64.internal(i16 %row, i16 8, ptr %i4, i64 64, x86_amx %i8)
-  %i9 = tail call x86_amx @llvm.x86.tileloadd64.internal(i16 8, i16 %col, ptr @buf, i64 32)
-  call void @llvm.x86.tilestored64.internal(i16 8, i16 %col, ptr %i2, i64 64, x86_amx %i9)
-  %i10 = tail call x86_amx @llvm.x86.tileloadd64.internal(i16 %row, i16 %col, ptr @buf, i64 32)
-  call void @llvm.x86.tilestored64.internal(i16 %row, i16 %col, ptr %i, i64 64, x86_amx %i10)
-  br label %if.end
-
-if.else:                                          ; preds = %entry
-  %i11 = tail call x86_amx @llvm.x86.tileloadd64.internal(i16 %row, i16 8, ptr @buf2, i64 32)
-  call void @llvm.x86.tilestored64.internal(i16 %row, i16 8, ptr %i4, i64 64, x86_amx %i11)
-  %i12 = tail call x86_amx @llvm.x86.tileloadd64.internal(i16 8, i16 %col, ptr @buf2, i64 32)
-  call void @llvm.x86.tilestored64.internal(i16 8, i16 %col, ptr %i2, i64 64, x86_amx %i12)
-  %i13 = tail call x86_amx @llvm.x86.tileloadd64.internal(i16 %row, i16 %col, ptr @buf2, i64 32)
-  call void @llvm.x86.tilestored64.internal(i16 %row, i16 %col, ptr %i, i64 64, x86_amx %i13)
-  br label %if.end
-
-if.end:                                           ; preds = %if.else, %if.then
-  %i14 = call x86_amx @llvm.x86.tileloadd64.internal(i16 %row, i16 8, ptr %i4, i64 64)
-  %i15 = call x86_amx @llvm.x86.tileloadd64.internal(i16 8, i16 %col, ptr %i2, i64 64)
-  %i16 = call x86_amx @llvm.x86.tileloadd64.internal(i16 %row, i16 %col, ptr %i, i64 64)
-  %i17 = tail call x86_amx @llvm.x86.tdpbssd.internal(i16 %row, i16 %col, i16 8, x86_amx %i16, x86_amx %i14, x86_amx %i15)
-  call void @llvm.x86.tilestored64.internal(i16 %row, i16 %col, ptr %i6, i64 64, x86_amx %i17)
-  %i18 = call x86_amx @llvm.x86.tileloadd64.internal(i16 %row, i16 %col, ptr %i6, i64 64)
-  tail call void @llvm.x86.tilestored64.internal(i16 %row, i16 %col, ptr @buf, i64 32, x86_amx %i18)
-  ret void
-}
-
-; Function Attrs: nounwind
-declare x86_amx @llvm.x86.tileloadd64.internal(i16, i16, ptr, i64)
-
-; Function Attrs: nounwind
-declare x86_amx @llvm.x86.tdpbssd.internal(i16, i16, i16, x86_amx, x86_amx, x86_amx)
-
-; Function Attrs: nounwind
-declare void @llvm.x86.tilestored64.internal(i16, i16, ptr, i64, x86_amx)
diff --git a/llvm/test/CodeGen/X86/GlobalISel/phi.ll b/llvm/test/CodeGen/X86/GlobalISel/phi.ll
index d2ce98d0fb..b29540f002 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/phi.ll
+++ b/llvm/test/CodeGen/X86/GlobalISel/phi.ll
@@ -4,8 +4,7 @@
 define i8 @test_i8(i32 %a, i8 %f, i8 %t) {
 ; ALL-LABEL: test_i8:
 ; ALL:       # %bb.0: # %entry
-; ALL-NEXT:    xorl %ecx, %ecx
-; ALL-NEXT:    cmpl %ecx, %edi
+; ALL-NEXT:    cmpl $0, %edi
 ; ALL-NEXT:    setg %cl
 ; ALL-NEXT:    testb $1, %cl
 ; ALL-NEXT:    je .LBB0_2
@@ -35,8 +34,7 @@ cond.end:                                         ; preds = %cond.false, %cond.t
 define i16 @test_i16(i32 %a, i16 %f, i16 %t) {
 ; ALL-LABEL: test_i16:
 ; ALL:       # %bb.0: # %entry
-; ALL-NEXT:    xorl %ecx, %ecx
-; ALL-NEXT:    cmpl %ecx, %edi
+; ALL-NEXT:    cmpl $0, %edi
 ; ALL-NEXT:    setg %cl
 ; ALL-NEXT:    testb $1, %cl
 ; ALL-NEXT:    je .LBB1_2
@@ -67,8 +65,7 @@ define i32 @test_i32(i32 %a, i32 %f, i32 %t) {
 ; ALL-LABEL: test_i32:
 ; ALL:       # %bb.0: # %entry
 ; ALL-NEXT:    movl %esi, %eax
-; ALL-NEXT:    xorl %ecx, %ecx
-; ALL-NEXT:    cmpl %ecx, %edi
+; ALL-NEXT:    cmpl $0, %edi
 ; ALL-NEXT:    setg %cl
 ; ALL-NEXT:    testb $1, %cl
 ; ALL-NEXT:    je .LBB2_1
@@ -96,8 +93,7 @@ define i64 @test_i64(i32 %a, i64 %f, i64 %t) {
 ; ALL-LABEL: test_i64:
 ; ALL:       # %bb.0: # %entry
 ; ALL-NEXT:    movq %rsi, %rax
-; ALL-NEXT:    xorl %ecx, %ecx
-; ALL-NEXT:    cmpl %ecx, %edi
+; ALL-NEXT:    cmpl $0, %edi
 ; ALL-NEXT:    setg %cl
 ; ALL-NEXT:    testb $1, %cl
 ; ALL-NEXT:    je .LBB3_1
@@ -124,8 +120,7 @@ cond.end:                                         ; preds = %cond.false, %cond.t
 define float @test_float(i32 %a, float %f, float %t) {
 ; ALL-LABEL: test_float:
 ; ALL:       # %bb.0: # %entry
-; ALL-NEXT:    xorl %eax, %eax
-; ALL-NEXT:    cmpl %eax, %edi
+; ALL-NEXT:    cmpl $0, %edi
 ; ALL-NEXT:    setg %al
 ; ALL-NEXT:    testb $1, %al
 ; ALL-NEXT:    je .LBB4_1
@@ -152,8 +147,7 @@ cond.end:                                         ; preds = %cond.false, %cond.t
 define double @test_double(i32 %a, double %f, double %t) {
 ; ALL-LABEL: test_double:
 ; ALL:       # %bb.0: # %entry
-; ALL-NEXT:    xorl %eax, %eax
-; ALL-NEXT:    cmpl %eax, %edi
+; ALL-NEXT:    cmpl $0, %edi
 ; ALL-NEXT:    setg %al
 ; ALL-NEXT:    testb $1, %al
 ; ALL-NEXT:    je .LBB5_1
diff --git a/llvm/test/CodeGen/X86/arg-copy-elide.ll b/llvm/test/CodeGen/X86/arg-copy-elide.ll
index 9d57c9cb2b..0eb2c630e6 100644
--- a/llvm/test/CodeGen/X86/arg-copy-elide.ll
+++ b/llvm/test/CodeGen/X86/arg-copy-elide.ll
@@ -186,8 +186,8 @@ define void @split_i128(ptr %sret, i128 %x) {
 ; CHECK-NEXT:    pushl %ebx
 ; CHECK-NEXT:    pushl %edi
 ; CHECK-NEXT:    pushl %esi
-; CHECK-NEXT:    andl $-8, %esp
-; CHECK-NEXT:    subl $32, %esp
+; CHECK-NEXT:    andl $-16, %esp
+; CHECK-NEXT:    subl $48, %esp
 ; CHECK-NEXT:    movl 12(%ebp), %eax
 ; CHECK-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; CHECK-NEXT:    movl 16(%ebp), %ebx
diff --git a/llvm/test/CodeGen/X86/atomic-idempotent.ll b/llvm/test/CodeGen/X86/atomic-idempotent.ll
index ef9ed79363..3a9648bd1f 100644
--- a/llvm/test/CodeGen/X86/atomic-idempotent.ll
+++ b/llvm/test/CodeGen/X86/atomic-idempotent.ll
@@ -182,12 +182,10 @@ define i128 @or128(ptr %p) {
 ; X86-SSE2-NEXT:    .cfi_offset %ebp, -8
 ; X86-SSE2-NEXT:    movl %esp, %ebp
 ; X86-SSE2-NEXT:    .cfi_def_cfa_register %ebp
-; X86-SSE2-NEXT:    pushl %edi
 ; X86-SSE2-NEXT:    pushl %esi
-; X86-SSE2-NEXT:    andl $-8, %esp
-; X86-SSE2-NEXT:    subl $16, %esp
-; X86-SSE2-NEXT:    .cfi_offset %esi, -16
-; X86-SSE2-NEXT:    .cfi_offset %edi, -12
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $32, %esp
+; X86-SSE2-NEXT:    .cfi_offset %esi, -12
 ; X86-SSE2-NEXT:    movl 8(%ebp), %esi
 ; X86-SSE2-NEXT:    movl %esp, %eax
 ; X86-SSE2-NEXT:    pushl $0
@@ -198,18 +196,11 @@ define i128 @or128(ptr %p) {
 ; X86-SSE2-NEXT:    pushl %eax
 ; X86-SSE2-NEXT:    calll __sync_fetch_and_or_16
 ; X86-SSE2-NEXT:    addl $20, %esp
-; X86-SSE2-NEXT:    movl (%esp), %eax
-; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-SSE2-NEXT:    movl %edi, 8(%esi)
-; X86-SSE2-NEXT:    movl %edx, 12(%esi)
-; X86-SSE2-NEXT:    movl %eax, (%esi)
-; X86-SSE2-NEXT:    movl %ecx, 4(%esi)
+; X86-SSE2-NEXT:    movaps (%esp), %xmm0
+; X86-SSE2-NEXT:    movaps %xmm0, (%esi)
 ; X86-SSE2-NEXT:    movl %esi, %eax
-; X86-SSE2-NEXT:    leal -8(%ebp), %esp
+; X86-SSE2-NEXT:    leal -4(%ebp), %esp
 ; X86-SSE2-NEXT:    popl %esi
-; X86-SSE2-NEXT:    popl %edi
 ; X86-SSE2-NEXT:    popl %ebp
 ; X86-SSE2-NEXT:    .cfi_def_cfa %esp, 4
 ; X86-SSE2-NEXT:    retl $4
@@ -223,7 +214,7 @@ define i128 @or128(ptr %p) {
 ; X86-SLM-NEXT:    .cfi_def_cfa_register %ebp
 ; X86-SLM-NEXT:    pushl %edi
 ; X86-SLM-NEXT:    pushl %esi
-; X86-SLM-NEXT:    andl $-8, %esp
+; X86-SLM-NEXT:    andl $-16, %esp
 ; X86-SLM-NEXT:    subl $16, %esp
 ; X86-SLM-NEXT:    .cfi_offset %esi, -16
 ; X86-SLM-NEXT:    .cfi_offset %edi, -12
@@ -263,7 +254,7 @@ define i128 @or128(ptr %p) {
 ; X86-ATOM-NEXT:    .cfi_def_cfa_register %ebp
 ; X86-ATOM-NEXT:    pushl %edi
 ; X86-ATOM-NEXT:    pushl %esi
-; X86-ATOM-NEXT:    andl $-8, %esp
+; X86-ATOM-NEXT:    andl $-16, %esp
 ; X86-ATOM-NEXT:    leal -{{[0-9]+}}(%esp), %esp
 ; X86-ATOM-NEXT:    .cfi_offset %esi, -16
 ; X86-ATOM-NEXT:    .cfi_offset %edi, -12
@@ -528,8 +519,8 @@ define void @or128_nouse_seq_cst(ptr %p) {
 ; X86-SSE2-NEXT:    .cfi_offset %ebp, -8
 ; X86-SSE2-NEXT:    movl %esp, %ebp
 ; X86-SSE2-NEXT:    .cfi_def_cfa_register %ebp
-; X86-SSE2-NEXT:    andl $-8, %esp
-; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $32, %esp
 ; X86-SSE2-NEXT:    movl %esp, %eax
 ; X86-SSE2-NEXT:    pushl $0
 ; X86-SSE2-NEXT:    pushl $0
@@ -551,8 +542,8 @@ define void @or128_nouse_seq_cst(ptr %p) {
 ; X86-SLM-NEXT:    .cfi_offset %ebp, -8
 ; X86-SLM-NEXT:    movl %esp, %ebp
 ; X86-SLM-NEXT:    .cfi_def_cfa_register %ebp
-; X86-SLM-NEXT:    andl $-8, %esp
-; X86-SLM-NEXT:    subl $16, %esp
+; X86-SLM-NEXT:    andl $-16, %esp
+; X86-SLM-NEXT:    subl $32, %esp
 ; X86-SLM-NEXT:    movl 8(%ebp), %eax
 ; X86-SLM-NEXT:    movl %esp, %ecx
 ; X86-SLM-NEXT:    pushl $0
@@ -575,7 +566,7 @@ define void @or128_nouse_seq_cst(ptr %p) {
 ; X86-ATOM-NEXT:    .cfi_offset %ebp, -8
 ; X86-ATOM-NEXT:    leal (%esp), %ebp
 ; X86-ATOM-NEXT:    .cfi_def_cfa_register %ebp
-; X86-ATOM-NEXT:    andl $-8, %esp
+; X86-ATOM-NEXT:    andl $-16, %esp
 ; X86-ATOM-NEXT:    leal -{{[0-9]+}}(%esp), %esp
 ; X86-ATOM-NEXT:    movl 8(%ebp), %eax
 ; X86-ATOM-NEXT:    movl %esp, %ecx
diff --git a/llvm/test/CodeGen/X86/atomic-non-integer.ll b/llvm/test/CodeGen/X86/atomic-non-integer.ll
index 84f944f2bd..7d2810e57a 100644
--- a/llvm/test/CodeGen/X86/atomic-non-integer.ll
+++ b/llvm/test/CodeGen/X86/atomic-non-integer.ll
@@ -157,8 +157,8 @@ define void @store_fp128(ptr %fptr, fp128 %v) {
 ;
 ; X86-AVX-LABEL: store_fp128:
 ; X86-AVX:       # %bb.0:
-; X86-AVX-NEXT:    subl $44, %esp
-; X86-AVX-NEXT:    .cfi_def_cfa_offset 48
+; X86-AVX-NEXT:    subl $60, %esp
+; X86-AVX-NEXT:    .cfi_def_cfa_offset 64
 ; X86-AVX-NEXT:    vmovaps {{[0-9]+}}(%esp), %xmm0
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-AVX-NEXT:    movl %eax, {{[0-9]+}}(%esp)
@@ -166,7 +166,7 @@ define void @store_fp128(ptr %fptr, fp128 %v) {
 ; X86-AVX-NEXT:    leal {{[0-9]+}}(%esp), %eax
 ; X86-AVX-NEXT:    movl %eax, (%esp)
 ; X86-AVX-NEXT:    calll __sync_lock_test_and_set_16
-; X86-AVX-NEXT:    addl $40, %esp
+; X86-AVX-NEXT:    addl $56, %esp
 ; X86-AVX-NEXT:    .cfi_def_cfa_offset 4
 ; X86-AVX-NEXT:    retl
 ;
@@ -394,67 +394,111 @@ define double @load_double(ptr %fptr) {
 }
 
 define fp128 @load_fp128(ptr %fptr) {
-; X86-SSE-LABEL: load_fp128:
-; X86-SSE:       # %bb.0:
-; X86-SSE-NEXT:    pushl %edi
-; X86-SSE-NEXT:    .cfi_def_cfa_offset 8
-; X86-SSE-NEXT:    pushl %esi
-; X86-SSE-NEXT:    .cfi_def_cfa_offset 12
-; X86-SSE-NEXT:    subl $20, %esp
-; X86-SSE-NEXT:    .cfi_def_cfa_offset 32
-; X86-SSE-NEXT:    .cfi_offset %esi, -12
-; X86-SSE-NEXT:    .cfi_offset %edi, -8
-; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-SSE-NEXT:    subl $8, %esp
-; X86-SSE-NEXT:    .cfi_adjust_cfa_offset 8
-; X86-SSE-NEXT:    leal {{[0-9]+}}(%esp), %eax
-; X86-SSE-NEXT:    pushl $0
-; X86-SSE-NEXT:    .cfi_adjust_cfa_offset 4
-; X86-SSE-NEXT:    pushl $0
-; X86-SSE-NEXT:    .cfi_adjust_cfa_offset 4
-; X86-SSE-NEXT:    pushl $0
-; X86-SSE-NEXT:    .cfi_adjust_cfa_offset 4
-; X86-SSE-NEXT:    pushl $0
-; X86-SSE-NEXT:    .cfi_adjust_cfa_offset 4
-; X86-SSE-NEXT:    pushl $0
-; X86-SSE-NEXT:    .cfi_adjust_cfa_offset 4
-; X86-SSE-NEXT:    pushl $0
-; X86-SSE-NEXT:    .cfi_adjust_cfa_offset 4
-; X86-SSE-NEXT:    pushl $0
-; X86-SSE-NEXT:    .cfi_adjust_cfa_offset 4
-; X86-SSE-NEXT:    pushl $0
-; X86-SSE-NEXT:    .cfi_adjust_cfa_offset 4
-; X86-SSE-NEXT:    pushl {{[0-9]+}}(%esp)
-; X86-SSE-NEXT:    .cfi_adjust_cfa_offset 4
-; X86-SSE-NEXT:    pushl %eax
-; X86-SSE-NEXT:    .cfi_adjust_cfa_offset 4
-; X86-SSE-NEXT:    calll __sync_val_compare_and_swap_16
-; X86-SSE-NEXT:    .cfi_adjust_cfa_offset -4
-; X86-SSE-NEXT:    addl $44, %esp
-; X86-SSE-NEXT:    .cfi_adjust_cfa_offset -44
-; X86-SSE-NEXT:    movl (%esp), %eax
-; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-SSE-NEXT:    movl %edi, 8(%esi)
-; X86-SSE-NEXT:    movl %edx, 12(%esi)
-; X86-SSE-NEXT:    movl %eax, (%esi)
-; X86-SSE-NEXT:    movl %ecx, 4(%esi)
-; X86-SSE-NEXT:    movl %esi, %eax
-; X86-SSE-NEXT:    addl $20, %esp
-; X86-SSE-NEXT:    .cfi_def_cfa_offset 12
-; X86-SSE-NEXT:    popl %esi
-; X86-SSE-NEXT:    .cfi_def_cfa_offset 8
-; X86-SSE-NEXT:    popl %edi
-; X86-SSE-NEXT:    .cfi_def_cfa_offset 4
-; X86-SSE-NEXT:    retl $4
+; X86-SSE1-LABEL: load_fp128:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl %edi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE1-NEXT:    pushl %esi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 12
+; X86-SSE1-NEXT:    subl $20, %esp
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 32
+; X86-SSE1-NEXT:    .cfi_offset %esi, -12
+; X86-SSE1-NEXT:    .cfi_offset %edi, -8
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-SSE1-NEXT:    subl $8, %esp
+; X86-SSE1-NEXT:    .cfi_adjust_cfa_offset 8
+; X86-SSE1-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    pushl $0
+; X86-SSE1-NEXT:    .cfi_adjust_cfa_offset 4
+; X86-SSE1-NEXT:    pushl $0
+; X86-SSE1-NEXT:    .cfi_adjust_cfa_offset 4
+; X86-SSE1-NEXT:    pushl $0
+; X86-SSE1-NEXT:    .cfi_adjust_cfa_offset 4
+; X86-SSE1-NEXT:    pushl $0
+; X86-SSE1-NEXT:    .cfi_adjust_cfa_offset 4
+; X86-SSE1-NEXT:    pushl $0
+; X86-SSE1-NEXT:    .cfi_adjust_cfa_offset 4
+; X86-SSE1-NEXT:    pushl $0
+; X86-SSE1-NEXT:    .cfi_adjust_cfa_offset 4
+; X86-SSE1-NEXT:    pushl $0
+; X86-SSE1-NEXT:    .cfi_adjust_cfa_offset 4
+; X86-SSE1-NEXT:    pushl $0
+; X86-SSE1-NEXT:    .cfi_adjust_cfa_offset 4
+; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    .cfi_adjust_cfa_offset 4
+; X86-SSE1-NEXT:    pushl %eax
+; X86-SSE1-NEXT:    .cfi_adjust_cfa_offset 4
+; X86-SSE1-NEXT:    calll __sync_val_compare_and_swap_16
+; X86-SSE1-NEXT:    .cfi_adjust_cfa_offset -4
+; X86-SSE1-NEXT:    addl $44, %esp
+; X86-SSE1-NEXT:    .cfi_adjust_cfa_offset -44
+; X86-SSE1-NEXT:    movl (%esp), %eax
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-SSE1-NEXT:    movl %edi, 8(%esi)
+; X86-SSE1-NEXT:    movl %edx, 12(%esi)
+; X86-SSE1-NEXT:    movl %eax, (%esi)
+; X86-SSE1-NEXT:    movl %ecx, 4(%esi)
+; X86-SSE1-NEXT:    movl %esi, %eax
+; X86-SSE1-NEXT:    addl $20, %esp
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 12
+; X86-SSE1-NEXT:    popl %esi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE1-NEXT:    popl %edi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE1-NEXT:    retl $4
+;
+; X86-SSE2-LABEL: load_fp128:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %esi
+; X86-SSE2-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE2-NEXT:    subl $24, %esp
+; X86-SSE2-NEXT:    .cfi_def_cfa_offset 32
+; X86-SSE2-NEXT:    .cfi_offset %esi, -8
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-SSE2-NEXT:    subl $8, %esp
+; X86-SSE2-NEXT:    .cfi_adjust_cfa_offset 8
+; X86-SSE2-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    pushl $0
+; X86-SSE2-NEXT:    .cfi_adjust_cfa_offset 4
+; X86-SSE2-NEXT:    pushl $0
+; X86-SSE2-NEXT:    .cfi_adjust_cfa_offset 4
+; X86-SSE2-NEXT:    pushl $0
+; X86-SSE2-NEXT:    .cfi_adjust_cfa_offset 4
+; X86-SSE2-NEXT:    pushl $0
+; X86-SSE2-NEXT:    .cfi_adjust_cfa_offset 4
+; X86-SSE2-NEXT:    pushl $0
+; X86-SSE2-NEXT:    .cfi_adjust_cfa_offset 4
+; X86-SSE2-NEXT:    pushl $0
+; X86-SSE2-NEXT:    .cfi_adjust_cfa_offset 4
+; X86-SSE2-NEXT:    pushl $0
+; X86-SSE2-NEXT:    .cfi_adjust_cfa_offset 4
+; X86-SSE2-NEXT:    pushl $0
+; X86-SSE2-NEXT:    .cfi_adjust_cfa_offset 4
+; X86-SSE2-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    .cfi_adjust_cfa_offset 4
+; X86-SSE2-NEXT:    pushl %eax
+; X86-SSE2-NEXT:    .cfi_adjust_cfa_offset 4
+; X86-SSE2-NEXT:    calll __sync_val_compare_and_swap_16
+; X86-SSE2-NEXT:    .cfi_adjust_cfa_offset -4
+; X86-SSE2-NEXT:    addl $44, %esp
+; X86-SSE2-NEXT:    .cfi_adjust_cfa_offset -44
+; X86-SSE2-NEXT:    movaps (%esp), %xmm0
+; X86-SSE2-NEXT:    movaps %xmm0, (%esi)
+; X86-SSE2-NEXT:    movl %esi, %eax
+; X86-SSE2-NEXT:    addl $24, %esp
+; X86-SSE2-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE2-NEXT:    popl %esi
+; X86-SSE2-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE2-NEXT:    retl $4
 ;
 ; X86-AVX-LABEL: load_fp128:
 ; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    pushl %esi
 ; X86-AVX-NEXT:    .cfi_def_cfa_offset 8
-; X86-AVX-NEXT:    subl $56, %esp
-; X86-AVX-NEXT:    .cfi_def_cfa_offset 64
+; X86-AVX-NEXT:    subl $72, %esp
+; X86-AVX-NEXT:    .cfi_def_cfa_offset 80
 ; X86-AVX-NEXT:    .cfi_offset %esi, -8
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
@@ -466,10 +510,10 @@ define fp128 @load_fp128(ptr %fptr) {
 ; X86-AVX-NEXT:    vzeroupper
 ; X86-AVX-NEXT:    calll __sync_val_compare_and_swap_16
 ; X86-AVX-NEXT:    subl $4, %esp
-; X86-AVX-NEXT:    vmovups {{[0-9]+}}(%esp), %xmm0
+; X86-AVX-NEXT:    vmovaps {{[0-9]+}}(%esp), %xmm0
 ; X86-AVX-NEXT:    vmovaps %xmm0, (%esi)
 ; X86-AVX-NEXT:    movl %esi, %eax
-; X86-AVX-NEXT:    addl $56, %esp
+; X86-AVX-NEXT:    addl $72, %esp
 ; X86-AVX-NEXT:    .cfi_def_cfa_offset 8
 ; X86-AVX-NEXT:    popl %esi
 ; X86-AVX-NEXT:    .cfi_def_cfa_offset 4
diff --git a/llvm/test/CodeGen/X86/atomic-unordered.ll b/llvm/test/CodeGen/X86/atomic-unordered.ll
index 9482d71ebf..b66988c8bd 100644
--- a/llvm/test/CodeGen/X86/atomic-unordered.ll
+++ b/llvm/test/CodeGen/X86/atomic-unordered.ll
@@ -322,12 +322,12 @@ define i256 @load_i256(ptr %ptr) {
 ; CHECK-O0-NEXT:    .cfi_def_cfa_offset 64
 ; CHECK-O0-NEXT:    movq %rdi, %rax
 ; CHECK-O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-O0-NEXT:    movq %rdi, (%rsp) # 8-byte Spill
 ; CHECK-O0-NEXT:    movl $32, %edi
 ; CHECK-O0-NEXT:    leaq {{[0-9]+}}(%rsp), %rdx
 ; CHECK-O0-NEXT:    xorl %ecx, %ecx
 ; CHECK-O0-NEXT:    callq __atomic_load@PLT
-; CHECK-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; CHECK-O0-NEXT:    movq (%rsp), %rdi # 8-byte Reload
 ; CHECK-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
 ; CHECK-O0-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
 ; CHECK-O0-NEXT:    movq {{[0-9]+}}(%rsp), %rdx
@@ -373,12 +373,12 @@ define void @store_i256(ptr %ptr, i256 %v) {
 ; CHECK-O0-NEXT:    .cfi_def_cfa_offset 48
 ; CHECK-O0-NEXT:    movq %rsi, %rax
 ; CHECK-O0-NEXT:    movq %rdi, %rsi
-; CHECK-O0-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
+; CHECK-O0-NEXT:    movq %rax, (%rsp)
 ; CHECK-O0-NEXT:    movq %rdx, {{[0-9]+}}(%rsp)
 ; CHECK-O0-NEXT:    movq %rcx, {{[0-9]+}}(%rsp)
 ; CHECK-O0-NEXT:    movq %r8, {{[0-9]+}}(%rsp)
 ; CHECK-O0-NEXT:    movl $32, %edi
-; CHECK-O0-NEXT:    leaq {{[0-9]+}}(%rsp), %rdx
+; CHECK-O0-NEXT:    movq %rsp, %rdx
 ; CHECK-O0-NEXT:    xorl %ecx, %ecx
 ; CHECK-O0-NEXT:    callq __atomic_store@PLT
 ; CHECK-O0-NEXT:    addq $40, %rsp
@@ -393,8 +393,8 @@ define void @store_i256(ptr %ptr, i256 %v) {
 ; CHECK-O3-NEXT:    movq %r8, {{[0-9]+}}(%rsp)
 ; CHECK-O3-NEXT:    movq %rcx, {{[0-9]+}}(%rsp)
 ; CHECK-O3-NEXT:    movq %rdx, {{[0-9]+}}(%rsp)
-; CHECK-O3-NEXT:    movq %rsi, {{[0-9]+}}(%rsp)
-; CHECK-O3-NEXT:    leaq {{[0-9]+}}(%rsp), %rdx
+; CHECK-O3-NEXT:    movq %rsi, (%rsp)
+; CHECK-O3-NEXT:    movq %rsp, %rdx
 ; CHECK-O3-NEXT:    movl $32, %edi
 ; CHECK-O3-NEXT:    movq %rax, %rsi
 ; CHECK-O3-NEXT:    xorl %ecx, %ecx
diff --git a/llvm/test/CodeGen/X86/atomic-xor.ll b/llvm/test/CodeGen/X86/atomic-xor.ll
index 5f33cb8cd9..97fa908f1b 100644
--- a/llvm/test/CodeGen/X86/atomic-xor.ll
+++ b/llvm/test/CodeGen/X86/atomic-xor.ll
@@ -24,7 +24,7 @@ define i128 @xor128_signbit_used(ptr %p) nounwind {
 ; X86-NEXT:    movl %esp, %ebp
 ; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    andl $-8, %esp
+; X86-NEXT:    andl $-16, %esp
 ; X86-NEXT:    subl $16, %esp
 ; X86-NEXT:    movl 8(%ebp), %esi
 ; X86-NEXT:    movl %esp, %eax
diff --git a/llvm/test/CodeGen/X86/atomic128.ll b/llvm/test/CodeGen/X86/atomic128.ll
index 5e4b885c39..d5600b54a1 100644
--- a/llvm/test/CodeGen/X86/atomic128.ll
+++ b/llvm/test/CodeGen/X86/atomic128.ll
@@ -169,7 +169,7 @@ define void @fetch_and_nand(ptr %p, i128 %bits) {
 ; CHECK32-NEXT:    .cfi_adjust_cfa_offset -4
 ; CHECK32-NEXT:    addl $28, %esp
 ; CHECK32-NEXT:    .cfi_adjust_cfa_offset -28
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK32-NEXT:    movl (%esp), %eax
 ; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %esi
@@ -237,7 +237,7 @@ define void @fetch_and_or(ptr %p, i128 %bits) {
 ; CHECK32-NEXT:    .cfi_adjust_cfa_offset -4
 ; CHECK32-NEXT:    addl $28, %esp
 ; CHECK32-NEXT:    .cfi_adjust_cfa_offset -28
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK32-NEXT:    movl (%esp), %eax
 ; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %esi
@@ -305,7 +305,7 @@ define void @fetch_and_add(ptr %p, i128 %bits) {
 ; CHECK32-NEXT:    .cfi_adjust_cfa_offset -4
 ; CHECK32-NEXT:    addl $28, %esp
 ; CHECK32-NEXT:    .cfi_adjust_cfa_offset -28
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK32-NEXT:    movl (%esp), %eax
 ; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %esi
@@ -373,7 +373,7 @@ define void @fetch_and_sub(ptr %p, i128 %bits) {
 ; CHECK32-NEXT:    .cfi_adjust_cfa_offset -4
 ; CHECK32-NEXT:    addl $28, %esp
 ; CHECK32-NEXT:    .cfi_adjust_cfa_offset -28
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK32-NEXT:    movl (%esp), %eax
 ; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %esi
@@ -444,7 +444,7 @@ define void @fetch_and_min(ptr %p, i128 %bits) {
 ; CHECK32-NEXT:    .cfi_adjust_cfa_offset -4
 ; CHECK32-NEXT:    addl $28, %esp
 ; CHECK32-NEXT:    .cfi_adjust_cfa_offset -28
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK32-NEXT:    movl (%esp), %eax
 ; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %esi
@@ -515,7 +515,7 @@ define void @fetch_and_max(ptr %p, i128 %bits) {
 ; CHECK32-NEXT:    .cfi_adjust_cfa_offset -4
 ; CHECK32-NEXT:    addl $28, %esp
 ; CHECK32-NEXT:    .cfi_adjust_cfa_offset -28
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK32-NEXT:    movl (%esp), %eax
 ; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %esi
@@ -586,7 +586,7 @@ define void @fetch_and_umin(ptr %p, i128 %bits) {
 ; CHECK32-NEXT:    .cfi_adjust_cfa_offset -4
 ; CHECK32-NEXT:    addl $28, %esp
 ; CHECK32-NEXT:    .cfi_adjust_cfa_offset -28
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK32-NEXT:    movl (%esp), %eax
 ; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %esi
@@ -657,7 +657,7 @@ define void @fetch_and_umax(ptr %p, i128 %bits) {
 ; CHECK32-NEXT:    .cfi_adjust_cfa_offset -4
 ; CHECK32-NEXT:    addl $28, %esp
 ; CHECK32-NEXT:    .cfi_adjust_cfa_offset -28
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK32-NEXT:    movl (%esp), %eax
 ; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %esi
diff --git a/llvm/test/CodeGen/X86/avx512fp16-combine-shuffle-fma.ll b/llvm/test/CodeGen/X86/avx512fp16-combine-shuffle-fma.ll
new file mode 100644
index 0000000000..54ccc23840
--- /dev/null
+++ b/llvm/test/CodeGen/X86/avx512fp16-combine-shuffle-fma.ll
@@ -0,0 +1,58 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx2 | FileCheck %s --check-prefix=AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=f16c,fma | FileCheck %s --check-prefix=F16C
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512vl | FileCheck %s --check-prefix=F16C
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512fp16 | FileCheck %s --check-prefix=FP16
+
+define <2 x half> @foo(<2 x half> %0) "unsafe-fp-math"="true" nounwind {
+; AVX2-LABEL: foo:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    subq $40, %rsp
+; AVX2-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm0
+; AVX2-NEXT:    callq __extendhfsf2@PLT
+; AVX2-NEXT:    vmulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    callq __truncsfhf2@PLT
+; AVX2-NEXT:    vmovss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; AVX2-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX2-NEXT:    callq __extendhfsf2@PLT
+; AVX2-NEXT:    vmovss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; AVX2-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
+; AVX2-NEXT:    # xmm0 = mem[0],zero,zero,zero
+; AVX2-NEXT:    callq __extendhfsf2@PLT
+; AVX2-NEXT:    vsubss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 4-byte Folded Reload
+; AVX2-NEXT:    callq __truncsfhf2@PLT
+; AVX2-NEXT:    addq $40, %rsp
+; AVX2-NEXT:    retq
+;
+; F16C-LABEL: foo:
+; F16C:       # %bb.0:
+; F16C-NEXT:    vpsrld $16, %xmm0, %xmm1
+; F16C-NEXT:    vcvtph2ps %xmm1, %ymm1
+; F16C-NEXT:    vmulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; F16C-NEXT:    vcvtps2ph $4, %ymm1, %xmm1
+; F16C-NEXT:    vcvtph2ps %xmm0, %ymm0
+; F16C-NEXT:    vcvtph2ps %xmm1, %ymm1
+; F16C-NEXT:    vsubps %ymm0, %ymm1, %ymm2
+; F16C-NEXT:    vcvtps2ph $4, %ymm2, %xmm2
+; F16C-NEXT:    vaddps %ymm0, %ymm1, %ymm0
+; F16C-NEXT:    vcvtps2ph $4, %ymm0, %xmm0
+; F16C-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0],xmm0[1],xmm2[2,3,4,5,6,7]
+; F16C-NEXT:    vzeroupper
+; F16C-NEXT:    retq
+;
+; FP16-LABEL: foo:
+; FP16:       # %bb.0:
+; FP16-NEXT:    vpsrld $16, %xmm0, %xmm1
+; FP16-NEXT:    vfmaddsub231ph {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm0
+; FP16-NEXT:    retq
+  %2 = shufflevector <2 x half> %0, <2 x half> undef, <2 x i32> <i32 1, i32 2>
+  %3 = fmul fast <2 x half> %2, <half 0xH3D3A, half 0xH3854>
+  %4 = fsub fast <2 x half> %3, %0
+  %5 = fadd fast <2 x half> %3, %0
+  %6 = shufflevector <2 x half> %4, <2 x half> %5, <2 x i32> <i32 0, i32 3>
+  %7 = fadd fast <2 x half> %6, zeroinitializer
+  %8 = shufflevector <2 x half> undef, <2 x half> %7, <2 x i32> <i32 0, i32 3>
+  %9 = fsub fast <2 x half> %8, zeroinitializer
+  ret <2 x half> %9
+}
diff --git a/llvm/test/CodeGen/X86/avx512fp16-cvt.ll b/llvm/test/CodeGen/X86/avx512fp16-cvt.ll
index c3a979f984..e1e0135287 100644
--- a/llvm/test/CodeGen/X86/avx512fp16-cvt.ll
+++ b/llvm/test/CodeGen/X86/avx512fp16-cvt.ll
@@ -804,8 +804,8 @@ define i128 @half_to_s128(half %x) {
 ; X86-NEXT:    movl %esp, %ebp
 ; X86-NEXT:    .cfi_def_cfa_register %ebp
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    andl $-8, %esp
-; X86-NEXT:    subl $32, %esp
+; X86-NEXT:    andl $-16, %esp
+; X86-NEXT:    subl $48, %esp
 ; X86-NEXT:    .cfi_offset %esi, -12
 ; X86-NEXT:    movl 8(%ebp), %esi
 ; X86-NEXT:    vmovsh 12(%ebp), %xmm0
@@ -814,8 +814,8 @@ define i128 @half_to_s128(half %x) {
 ; X86-NEXT:    movl %eax, (%esp)
 ; X86-NEXT:    calll __fixhfti
 ; X86-NEXT:    subl $4, %esp
-; X86-NEXT:    vmovups {{[0-9]+}}(%esp), %xmm0
-; X86-NEXT:    vmovups %xmm0, (%esi)
+; X86-NEXT:    vmovaps {{[0-9]+}}(%esp), %xmm0
+; X86-NEXT:    vmovaps %xmm0, (%esi)
 ; X86-NEXT:    movl %esi, %eax
 ; X86-NEXT:    leal -4(%ebp), %esp
 ; X86-NEXT:    popl %esi
@@ -907,8 +907,8 @@ define i128 @half_to_u128(half %x) {
 ; X86-NEXT:    movl %esp, %ebp
 ; X86-NEXT:    .cfi_def_cfa_register %ebp
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    andl $-8, %esp
-; X86-NEXT:    subl $32, %esp
+; X86-NEXT:    andl $-16, %esp
+; X86-NEXT:    subl $48, %esp
 ; X86-NEXT:    .cfi_offset %esi, -12
 ; X86-NEXT:    movl 8(%ebp), %esi
 ; X86-NEXT:    vmovsh 12(%ebp), %xmm0
@@ -917,8 +917,8 @@ define i128 @half_to_u128(half %x) {
 ; X86-NEXT:    movl %eax, (%esp)
 ; X86-NEXT:    calll __fixunshfti
 ; X86-NEXT:    subl $4, %esp
-; X86-NEXT:    vmovups {{[0-9]+}}(%esp), %xmm0
-; X86-NEXT:    vmovups %xmm0, (%esi)
+; X86-NEXT:    vmovaps {{[0-9]+}}(%esp), %xmm0
+; X86-NEXT:    vmovaps %xmm0, (%esi)
 ; X86-NEXT:    movl %esi, %eax
 ; X86-NEXT:    leal -4(%ebp), %esp
 ; X86-NEXT:    popl %esi
@@ -987,8 +987,8 @@ define fp128 @half_to_f128(half %x) nounwind {
 ; X86-NEXT:    pushl %ebp
 ; X86-NEXT:    movl %esp, %ebp
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    andl $-8, %esp
-; X86-NEXT:    subl $32, %esp
+; X86-NEXT:    andl $-16, %esp
+; X86-NEXT:    subl $48, %esp
 ; X86-NEXT:    movl 8(%ebp), %esi
 ; X86-NEXT:    vmovsh 12(%ebp), %xmm0
 ; X86-NEXT:    vcvtsh2ss %xmm0, %xmm0, %xmm0
@@ -997,7 +997,7 @@ define fp128 @half_to_f128(half %x) nounwind {
 ; X86-NEXT:    movl %eax, (%esp)
 ; X86-NEXT:    calll __extendsftf2
 ; X86-NEXT:    subl $4, %esp
-; X86-NEXT:    vmovups {{[0-9]+}}(%esp), %xmm0
+; X86-NEXT:    vmovaps {{[0-9]+}}(%esp), %xmm0
 ; X86-NEXT:    vmovaps %xmm0, (%esi)
 ; X86-NEXT:    movl %esi, %eax
 ; X86-NEXT:    leal -4(%ebp), %esp
diff --git a/llvm/test/CodeGen/X86/basic-block-sections-clusters-error.ll b/llvm/test/CodeGen/X86/basic-block-sections-clusters-error.ll
index 5577601c02..597d8f6707 100644
--- a/llvm/test/CodeGen/X86/basic-block-sections-clusters-error.ll
+++ b/llvm/test/CodeGen/X86/basic-block-sections-clusters-error.ll
@@ -32,13 +32,35 @@
 ; RUN: echo '!dummy1' >> %t8
 ; RUN: not --crash llc < %s -O0 -mtriple=x86_64 -function-sections -basic-block-sections=%t8 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR8
 ; CHECK-ERROR8: LLVM ERROR: invalid profile {{.*}} at line 2: invalid specifier: '!'
-; RUN: echo 'v1' > %t0
-; RUN: echo 'm dummy1/module1 dummy1/module2'
+; RUN: echo 'v1' > %t9
+; RUN: echo 'm dummy1/module1 dummy1/module2' >> %t9
 ; RUN: echo 'f dummy1' >> %t9
-; RUN: not --crash llc < %s -O0 -mtriple=x86_64 -function-sections -basic-block-sections=%t8 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR8
-; CHECK-ERROR9: LLVM ERROR: invalid profile {{.*}} at line 2: invalid module name value: 'dummy1/module dummy1/module2'
-
-
+; RUN: not --crash llc < %s -O0 -mtriple=x86_64 -function-sections -basic-block-sections=%t9 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR9
+; CHECK-ERROR9: LLVM ERROR: invalid profile {{.*}} at line 2: invalid module name value: 'dummy1/module1 dummy1/module2'
+;;
+;; Error handling for version 1, cloning paths.
+; RUN: echo 'v1' > %t10
+; RUN: echo 'f dummy1' >> %t10
+; RUN: echo 'c 0 1.1.1' >> %t10
+; RUN: not --crash llc < %s -O0 -mtriple=x86_64 -function-sections -basic-block-sections=%t10 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR10
+; CHECK-ERROR10: LLVM ERROR: invalid profile {{.*}} at line 3: unable to parse basic block id: '1.1.1'
+; RUN: echo 'v1' > %t11
+; RUN: echo 'f dummy1' >> %t11
+; RUN: echo 'c 0 1.a' >> %t11
+; RUN: not --crash llc < %s -O0 -mtriple=x86_64 -function-sections -basic-block-sections=%t11 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR11
+; CHECK-ERROR11: LLVM ERROR: invalid profile {{.*}} at line 3: unable to parse clone id: 'a' 
+; RUN: echo 'v1' > %t12
+; RUN: echo 'f dummy1' >> %t12
+; RUN: echo 'c 0 1' >> %t12
+; RUN: echo 'p 1 2.1' >> %t12
+; RUN: not --crash llc < %s -O0 -mtriple=x86_64 -function-sections -basic-block-sections=%t12 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR12
+; CHECK-ERROR12: LLVM ERROR: invalid profile {{.*}} at line 4: unsigned integer expected: '2.1'
+; RUN: echo 'v1' > %t13
+; RUN: echo 'f dummy1' >> %t13
+; RUN: echo 'c 0 1' >> %t13
+; RUN: echo 'p 1 2 3 2' >> %t13
+; RUN: not --crash llc < %s -O0 -mtriple=x86_64 -function-sections -basic-block-sections=%t13 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR13
+; CHECK-ERROR13: LLVM ERROR: invalid profile {{.*}} at line 4: duplicate cloned block in path: '2'
 
 define i32 @dummy1(i32 %x, i32 %y, i32 %z) {
   entry:
@@ -63,4 +85,3 @@ define i32 @dummy2(i32 %x, i32 %y, i32 %z) !dbg !4 {
 !3 = !{i32 2, !"Debug Info Version", i32 3}
 !4 = distinct !DISubprogram(name: "dummy1", scope: !1, unit: !0)
 
-
diff --git a/llvm/test/CodeGen/X86/bitcast-i256.ll b/llvm/test/CodeGen/X86/bitcast-i256.ll
index 0a1953bcbe..57fc26600c 100644
--- a/llvm/test/CodeGen/X86/bitcast-i256.ll
+++ b/llvm/test/CodeGen/X86/bitcast-i256.ll
@@ -14,7 +14,7 @@ define i256 @foo(<8 x i32> %a) {
 ; SLOW:       # %bb.0:
 ; SLOW-NEXT:    movq %rdi, %rax
 ; SLOW-NEXT:    vextractf128 $1, %ymm0, 16(%rdi)
-; SLOW-NEXT:    vmovups %xmm0, (%rdi)
+; SLOW-NEXT:    vmovaps %xmm0, (%rdi)
 ; SLOW-NEXT:    vzeroupper
 ; SLOW-NEXT:    retq
   %r = bitcast <8 x i32> %a to i256
diff --git a/llvm/test/CodeGen/X86/catchpad-dynamic-alloca.ll b/llvm/test/CodeGen/X86/catchpad-dynamic-alloca.ll
index 88530cc0e1..6b63162717 100644
--- a/llvm/test/CodeGen/X86/catchpad-dynamic-alloca.ll
+++ b/llvm/test/CodeGen/X86/catchpad-dynamic-alloca.ll
@@ -62,4 +62,4 @@ catch.switch:
 ; CHECK-LABEL: $handlerMap$0$test2:
 ; CHECK:      .long   0
 ; CHECK-NEXT: .long   0
-; CHECK-NEXT: .long   8
+; CHECK-NEXT: .long   16
diff --git a/llvm/test/CodeGen/X86/cmp-shiftX-maskX.ll b/llvm/test/CodeGen/X86/cmp-shiftX-maskX.ll
new file mode 100644
index 0000000000..115f3863af
--- /dev/null
+++ b/llvm/test/CodeGen/X86/cmp-shiftX-maskX.ll
@@ -0,0 +1,579 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefixes=CHECK,CHECK-NOBMI,CHECK-NOBMI-SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+bmi2 | FileCheck %s --check-prefixes=CHECK,CHECK-BMI2,CHECK-BMI2-SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+bmi2,+avx2 | FileCheck %s --check-prefixes=CHECK,CHECK-BMI2,CHECK-AVX,CHECK-AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+bmi2,+avx512f,+avx512vl | FileCheck %s --check-prefixes=CHECK,CHECK-BMI2,CHECK-AVX,CHECK-AVX512
+declare <4 x i32> @llvm.fshl.v4i32(<4 x i32>, <4 x i32>, <4 x i32>)
+declare <4 x i32> @llvm.fshr.v4i32(<4 x i32>, <4 x i32>, <4 x i32>)
+declare <16 x i16> @llvm.fshl.v16i16(<16 x i16>, <16 x i16>, <16 x i16>)
+declare <16 x i16> @llvm.fshr.v16i16(<16 x i16>, <16 x i16>, <16 x i16>)
+declare i64 @llvm.fshl.i64(i64, i64, i64)
+declare i64 @llvm.fshr.i64(i64, i64, i64)
+declare i32 @llvm.fshl.i32(i32, i32, i32)
+declare i32 @llvm.fshr.i32(i32, i32, i32)
+declare i16 @llvm.fshl.i16(i16, i16, i16)
+declare i16 @llvm.fshr.i16(i16, i16, i16)
+declare i8 @llvm.fshl.i8(i8, i8, i8)
+declare i8 @llvm.fshr.i8(i8, i8, i8)
+
+define i1 @shr_to_shl_eq_i8_s2(i8 %x) {
+; CHECK-LABEL: shr_to_shl_eq_i8_s2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    rolb $2, %al
+; CHECK-NEXT:    cmpb %al, %dil
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    retq
+  %and = and i8 %x, 63
+  %shr = lshr i8 %x, 2
+  %r = icmp eq i8 %and, %shr
+  ret i1 %r
+}
+
+define i1 @shl_to_shr_ne_i8_s7(i8 %x) {
+; CHECK-LABEL: shl_to_shr_ne_i8_s7:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    shrb $7, %al
+; CHECK-NEXT:    andb $1, %dil
+; CHECK-NEXT:    cmpb %al, %dil
+; CHECK-NEXT:    setne %al
+; CHECK-NEXT:    retq
+  %shl = shl i8 %x, 7
+  %and = and i8 %x, 128
+  %r = icmp ne i8 %shl, %and
+  ret i1 %r
+}
+
+define i1 @rorl_to_srl_ne_i8_s5_fail(i8 %x) {
+; CHECK-LABEL: rorl_to_srl_ne_i8_s5_fail:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    rolb $5, %al
+; CHECK-NEXT:    cmpb %dil, %al
+; CHECK-NEXT:    setne %al
+; CHECK-NEXT:    retq
+  %ror = call i8 @llvm.fshl.i8(i8 %x, i8 %x, i8 5)
+  %r = icmp ne i8 %ror, %x
+  ret i1 %r
+}
+
+define i1 @shr_to_shl_eq_i8_s1(i8 %x) {
+; CHECK-LABEL: shr_to_shl_eq_i8_s1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    rolb %al
+; CHECK-NEXT:    cmpb %al, %dil
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    retq
+  %and = and i8 %x, 127
+  %shr = lshr i8 %x, 1
+  %r = icmp eq i8 %and, %shr
+  ret i1 %r
+}
+
+define i1 @shr_to_shl_eq_i32_s3(i32 %x) {
+; CHECK-LABEL: shr_to_shl_eq_i32_s3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT:    leal (,%rdi,8), %eax
+; CHECK-NEXT:    andl $-8, %edi
+; CHECK-NEXT:    cmpl %eax, %edi
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    retq
+  %and = and i32 %x, 536870911
+  %shr = lshr i32 %x, 3
+  %r = icmp eq i32 %and, %shr
+  ret i1 %r
+}
+
+define i1 @shl_to_shr_eq_i32_s3_fail(i32 %x) {
+; CHECK-LABEL: shl_to_shr_eq_i32_s3_fail:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    andl $536870911, %eax # imm = 0x1FFFFFFF
+; CHECK-NEXT:    shll $3, %edi
+; CHECK-NEXT:    cmpl %edi, %eax
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    retq
+  %and = and i32 %x, 536870911
+  %shr = shl i32 %x, 3
+  %r = icmp eq i32 %and, %shr
+  ret i1 %r
+}
+
+define i1 @shl_to_shr_ne_i32_s16(i32 %x) {
+; CHECK-NOBMI-LABEL: shl_to_shr_ne_i32_s16:
+; CHECK-NOBMI:       # %bb.0:
+; CHECK-NOBMI-NEXT:    movzwl %di, %eax
+; CHECK-NOBMI-NEXT:    shrl $16, %edi
+; CHECK-NOBMI-NEXT:    cmpl %edi, %eax
+; CHECK-NOBMI-NEXT:    setne %al
+; CHECK-NOBMI-NEXT:    retq
+;
+; CHECK-BMI2-LABEL: shl_to_shr_ne_i32_s16:
+; CHECK-BMI2:       # %bb.0:
+; CHECK-BMI2-NEXT:    rorxl $16, %edi, %eax
+; CHECK-BMI2-NEXT:    cmpl %eax, %edi
+; CHECK-BMI2-NEXT:    setne %al
+; CHECK-BMI2-NEXT:    retq
+  %shl = shl i32 %x, 16
+  %and = and i32 %x, 4294901760
+  %r = icmp ne i32 %shl, %and
+  ret i1 %r
+}
+
+define i1 @shl_to_shr_ne_i32_s16_fail(i32 %x) {
+; CHECK-LABEL: shl_to_shr_ne_i32_s16_fail:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    shll $16, %eax
+; CHECK-NEXT:    andl $2147450880, %edi # imm = 0x7FFF8000
+; CHECK-NEXT:    cmpl %edi, %eax
+; CHECK-NEXT:    setne %al
+; CHECK-NEXT:    retq
+  %shl = shl i32 %x, 16
+  %and = and i32 %x, 2147450880
+  %r = icmp ne i32 %shl, %and
+  ret i1 %r
+}
+
+define i1 @shr_to_shl_eq_i16_s1(i16 %x) {
+; CHECK-LABEL: shr_to_shl_eq_i16_s1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    rolw %ax
+; CHECK-NEXT:    cmpw %ax, %di
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    retq
+  %and = and i16 %x, 32767
+  %shr = lshr i16 %x, 1
+  %r = icmp eq i16 %and, %shr
+  ret i1 %r
+}
+
+define i1 @shr_to_shl_eq_i16_s1_fail(i16 %x) {
+; CHECK-LABEL: shr_to_shl_eq_i16_s1_fail:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movzwl %di, %eax
+; CHECK-NEXT:    andl $32766, %edi # imm = 0x7FFE
+; CHECK-NEXT:    shrl %eax
+; CHECK-NEXT:    cmpw %ax, %di
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    retq
+  %and = and i16 %x, 32766
+  %shr = lshr i16 %x, 1
+  %r = icmp eq i16 %and, %shr
+  ret i1 %r
+}
+
+define i1 @shl_to_shr_eq_i64_s44(i64 %x) {
+; CHECK-LABEL: shl_to_shr_eq_i64_s44:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    shrq $44, %rax
+; CHECK-NEXT:    andl $1048575, %edi # imm = 0xFFFFF
+; CHECK-NEXT:    cmpq %rax, %rdi
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    retq
+  %shl = shl i64 %x, 44
+  %and = and i64 %x, 18446726481523507200
+  %r = icmp eq i64 %shl, %and
+  ret i1 %r
+}
+
+define i1 @shr_to_shl_ne_i64_s32(i64 %x) {
+; CHECK-NOBMI-LABEL: shr_to_shl_ne_i64_s32:
+; CHECK-NOBMI:       # %bb.0:
+; CHECK-NOBMI-NEXT:    movl %edi, %eax
+; CHECK-NOBMI-NEXT:    shrq $32, %rdi
+; CHECK-NOBMI-NEXT:    cmpq %rdi, %rax
+; CHECK-NOBMI-NEXT:    setne %al
+; CHECK-NOBMI-NEXT:    retq
+;
+; CHECK-BMI2-LABEL: shr_to_shl_ne_i64_s32:
+; CHECK-BMI2:       # %bb.0:
+; CHECK-BMI2-NEXT:    rorxq $32, %rdi, %rax
+; CHECK-BMI2-NEXT:    cmpq %rax, %rdi
+; CHECK-BMI2-NEXT:    setne %al
+; CHECK-BMI2-NEXT:    retq
+  %and = and i64 %x, 4294967295
+  %shr = lshr i64 %x, 32
+  %r = icmp ne i64 %and, %shr
+  ret i1 %r
+}
+
+define i1 @rorl_to_shl_eq_i64_s16(i64 %x) {
+; CHECK-NOBMI-LABEL: rorl_to_shl_eq_i64_s16:
+; CHECK-NOBMI:       # %bb.0:
+; CHECK-NOBMI-NEXT:    movq %rdi, %rax
+; CHECK-NOBMI-NEXT:    rolq $16, %rax
+; CHECK-NOBMI-NEXT:    cmpq %rdi, %rax
+; CHECK-NOBMI-NEXT:    sete %al
+; CHECK-NOBMI-NEXT:    retq
+;
+; CHECK-BMI2-LABEL: rorl_to_shl_eq_i64_s16:
+; CHECK-BMI2:       # %bb.0:
+; CHECK-BMI2-NEXT:    rorxq $48, %rdi, %rax
+; CHECK-BMI2-NEXT:    cmpq %rdi, %rax
+; CHECK-BMI2-NEXT:    sete %al
+; CHECK-BMI2-NEXT:    retq
+  %ror = call i64 @llvm.fshl.i64(i64 %x, i64 %x, i64 16)
+  %r = icmp eq i64 %ror, %x
+  ret i1 %r
+}
+
+define i1 @ashr_to_shl_ne_i64_s32_fail(i64 %x) {
+; CHECK-LABEL: ashr_to_shl_ne_i64_s32_fail:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    sarq $32, %rdi
+; CHECK-NEXT:    cmpq %rdi, %rax
+; CHECK-NEXT:    setne %al
+; CHECK-NEXT:    retq
+  %and = and i64 %x, 4294967295
+  %shr = ashr i64 %x, 32
+  %r = icmp ne i64 %and, %shr
+  ret i1 %r
+}
+
+define i1 @shl_to_shr_eq_i64_s63(i64 %x) {
+; CHECK-LABEL: shl_to_shr_eq_i64_s63:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    shrq $63, %rax
+; CHECK-NEXT:    andl $1, %edi
+; CHECK-NEXT:    cmpq %rax, %rdi
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    retq
+  %shl = shl i64 %x, 63
+  %and = and i64 %x, 9223372036854775808
+  %r = icmp eq i64 %shl, %and
+  ret i1 %r
+}
+
+define i1 @shl_to_shr_eq_i64_s63_fail(i64 %x) {
+; CHECK-LABEL: shl_to_shr_eq_i64_s63_fail:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000
+; CHECK-NEXT:    andq %rdi, %rax
+; CHECK-NEXT:    shlq $63, %rdi
+; CHECK-NEXT:    cmpq %rax, %rdi
+; CHECK-NEXT:    seta %al
+; CHECK-NEXT:    retq
+  %shl = shl i64 %x, 63
+  %and = and i64 %x, 9223372036854775808
+  %r = icmp ugt i64 %shl, %and
+  ret i1 %r
+}
+
+define i1 @shr_to_shl_eq_i64_s7(i64 %x) {
+; CHECK-LABEL: shr_to_shl_eq_i64_s7:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    shlq $7, %rax
+; CHECK-NEXT:    andq $-128, %rdi
+; CHECK-NEXT:    cmpq %rax, %rdi
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    retq
+  %and = and i64 %x, 144115188075855871
+  %shr = lshr i64 %x, 7
+  %r = icmp eq i64 %and, %shr
+  ret i1 %r
+}
+
+define i1 @shl_to_shr_ne_i32_s24(i32 %x) {
+; CHECK-LABEL: shl_to_shr_ne_i32_s24:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    shrl $24, %edi
+; CHECK-NEXT:    cmpl %edi, %eax
+; CHECK-NEXT:    setne %al
+; CHECK-NEXT:    retq
+  %shl = shl i32 %x, 24
+  %and = and i32 %x, 4278190080
+  %r = icmp ne i32 %shl, %and
+  ret i1 %r
+}
+
+define i1 @shr_to_shl_ne_i32_s24_fail(i32 %x) {
+; CHECK-LABEL: shr_to_shl_ne_i32_s24_fail:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    shrl $24, %eax
+; CHECK-NEXT:    andl $-16777216, %edi # imm = 0xFF000000
+; CHECK-NEXT:    cmpl %edi, %eax
+; CHECK-NEXT:    setne %al
+; CHECK-NEXT:    retq
+  %shl = lshr i32 %x, 24
+  %and = and i32 %x, 4278190080
+  %r = icmp ne i32 %shl, %and
+  ret i1 %r
+}
+
+define i1 @shr_to_shl_ne_i32_s8(i32 %x) {
+; CHECK-NOBMI-LABEL: shr_to_shl_ne_i32_s8:
+; CHECK-NOBMI:       # %bb.0:
+; CHECK-NOBMI-NEXT:    movl %edi, %eax
+; CHECK-NOBMI-NEXT:    roll $8, %eax
+; CHECK-NOBMI-NEXT:    cmpl %eax, %edi
+; CHECK-NOBMI-NEXT:    setne %al
+; CHECK-NOBMI-NEXT:    retq
+;
+; CHECK-BMI2-LABEL: shr_to_shl_ne_i32_s8:
+; CHECK-BMI2:       # %bb.0:
+; CHECK-BMI2-NEXT:    rorxl $24, %edi, %eax
+; CHECK-BMI2-NEXT:    cmpl %eax, %edi
+; CHECK-BMI2-NEXT:    setne %al
+; CHECK-BMI2-NEXT:    retq
+  %and = and i32 %x, 16777215
+  %shr = lshr i32 %x, 8
+  %r = icmp ne i32 %and, %shr
+  ret i1 %r
+}
+
+define <4 x i1> @shr_to_ror_eq_4xi32_s4(<4 x i32> %x) {
+; CHECK-NOBMI-LABEL: shr_to_ror_eq_4xi32_s4:
+; CHECK-NOBMI:       # %bb.0:
+; CHECK-NOBMI-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-NOBMI-NEXT:    psrld $4, %xmm1
+; CHECK-NOBMI-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; CHECK-NOBMI-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-NOBMI-NEXT:    pcmpeqd %xmm1, %xmm1
+; CHECK-NOBMI-NEXT:    pxor %xmm1, %xmm0
+; CHECK-NOBMI-NEXT:    retq
+;
+; CHECK-BMI2-SSE2-LABEL: shr_to_ror_eq_4xi32_s4:
+; CHECK-BMI2-SSE2:       # %bb.0:
+; CHECK-BMI2-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-BMI2-SSE2-NEXT:    psrld $4, %xmm1
+; CHECK-BMI2-SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; CHECK-BMI2-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-BMI2-SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
+; CHECK-BMI2-SSE2-NEXT:    pxor %xmm1, %xmm0
+; CHECK-BMI2-SSE2-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: shr_to_ror_eq_4xi32_s4:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vpsrld $4, %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [268435455,268435455,268435455,268435455]
+; CHECK-AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm0, %xmm1, %xmm0
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512-LABEL: shr_to_ror_eq_4xi32_s4:
+; CHECK-AVX512:       # %bb.0:
+; CHECK-AVX512-NEXT:    vprold $4, %xmm0, %xmm1
+; CHECK-AVX512-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
+; CHECK-AVX512-NEXT:    retq
+  %shr = lshr <4 x i32> %x, <i32 4, i32 4, i32 4, i32 4>
+  %and = and <4 x i32> %x, <i32 268435455, i32 268435455, i32 268435455, i32 268435455>
+  %r = icmp ne <4 x i32> %shr, %and
+  ret <4 x i1> %r
+}
+
+define <4 x i1> @shl_to_ror_eq_4xi32_s8(<4 x i32> %x) {
+; CHECK-NOBMI-LABEL: shl_to_ror_eq_4xi32_s8:
+; CHECK-NOBMI:       # %bb.0:
+; CHECK-NOBMI-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-NOBMI-NEXT:    pslld $8, %xmm1
+; CHECK-NOBMI-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; CHECK-NOBMI-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-NOBMI-NEXT:    pcmpeqd %xmm1, %xmm1
+; CHECK-NOBMI-NEXT:    pxor %xmm1, %xmm0
+; CHECK-NOBMI-NEXT:    retq
+;
+; CHECK-BMI2-SSE2-LABEL: shl_to_ror_eq_4xi32_s8:
+; CHECK-BMI2-SSE2:       # %bb.0:
+; CHECK-BMI2-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-BMI2-SSE2-NEXT:    pslld $8, %xmm1
+; CHECK-BMI2-SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; CHECK-BMI2-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-BMI2-SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
+; CHECK-BMI2-SSE2-NEXT:    pxor %xmm1, %xmm0
+; CHECK-BMI2-SSE2-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: shl_to_ror_eq_4xi32_s8:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vpslld $8, %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm0, %xmm1, %xmm0
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512-LABEL: shl_to_ror_eq_4xi32_s8:
+; CHECK-AVX512:       # %bb.0:
+; CHECK-AVX512-NEXT:    vprold $8, %xmm0, %xmm1
+; CHECK-AVX512-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-AVX512-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
+; CHECK-AVX512-NEXT:    retq
+  %shr = shl <4 x i32> %x, <i32 8, i32 8, i32 8, i32 8>
+  %and = and <4 x i32> %x, <i32 4294967040, i32 4294967040, i32 4294967040, i32 4294967040>
+  %r = icmp ne <4 x i32> %shr, %and
+  ret <4 x i1> %r
+}
+
+define <4 x i1> @shl_to_ror_eq_4xi32_s7_fail_no_p2(<4 x i32> %x) {
+; CHECK-NOBMI-LABEL: shl_to_ror_eq_4xi32_s7_fail_no_p2:
+; CHECK-NOBMI:       # %bb.0:
+; CHECK-NOBMI-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-NOBMI-NEXT:    pslld $7, %xmm1
+; CHECK-NOBMI-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; CHECK-NOBMI-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-NOBMI-NEXT:    pcmpeqd %xmm1, %xmm1
+; CHECK-NOBMI-NEXT:    pxor %xmm1, %xmm0
+; CHECK-NOBMI-NEXT:    retq
+;
+; CHECK-BMI2-SSE2-LABEL: shl_to_ror_eq_4xi32_s7_fail_no_p2:
+; CHECK-BMI2-SSE2:       # %bb.0:
+; CHECK-BMI2-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-BMI2-SSE2-NEXT:    pslld $7, %xmm1
+; CHECK-BMI2-SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; CHECK-BMI2-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-BMI2-SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
+; CHECK-BMI2-SSE2-NEXT:    pxor %xmm1, %xmm0
+; CHECK-BMI2-SSE2-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: shl_to_ror_eq_4xi32_s7_fail_no_p2:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vpslld $7, %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [4294967168,4294967168,4294967168,4294967168]
+; CHECK-AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm0, %xmm1, %xmm0
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512-LABEL: shl_to_ror_eq_4xi32_s7_fail_no_p2:
+; CHECK-AVX512:       # %bb.0:
+; CHECK-AVX512-NEXT:    vpslld $7, %xmm0, %xmm1
+; CHECK-AVX512-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512-NEXT:    vpcmpeqd %xmm0, %xmm1, %xmm0
+; CHECK-AVX512-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
+; CHECK-AVX512-NEXT:    retq
+  %shr = shl <4 x i32> %x, <i32 7, i32 7, i32 7, i32 7>
+  %and = and <4 x i32> %x, <i32 4294967168, i32 4294967168, i32 4294967168, i32 4294967168>
+  %r = icmp ne <4 x i32> %shr, %and
+  ret <4 x i1> %r
+}
+
+define <4 x i1> @shr_to_ror_eq_4xi32_s4_fail_no_splat(<4 x i32> %x) {
+; CHECK-NOBMI-LABEL: shr_to_ror_eq_4xi32_s4_fail_no_splat:
+; CHECK-NOBMI:       # %bb.0:
+; CHECK-NOBMI-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-NOBMI-NEXT:    psrld $4, %xmm1
+; CHECK-NOBMI-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-NOBMI-NEXT:    psrld $8, %xmm2
+; CHECK-NOBMI-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,0],xmm1[2,0]
+; CHECK-NOBMI-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,0]
+; CHECK-NOBMI-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; CHECK-NOBMI-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-NOBMI-NEXT:    pcmpeqd %xmm1, %xmm1
+; CHECK-NOBMI-NEXT:    pxor %xmm1, %xmm0
+; CHECK-NOBMI-NEXT:    retq
+;
+; CHECK-BMI2-SSE2-LABEL: shr_to_ror_eq_4xi32_s4_fail_no_splat:
+; CHECK-BMI2-SSE2:       # %bb.0:
+; CHECK-BMI2-SSE2-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-BMI2-SSE2-NEXT:    psrld $4, %xmm1
+; CHECK-BMI2-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-BMI2-SSE2-NEXT:    psrld $8, %xmm2
+; CHECK-BMI2-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,0],xmm1[2,0]
+; CHECK-BMI2-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,0]
+; CHECK-BMI2-SSE2-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; CHECK-BMI2-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; CHECK-BMI2-SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
+; CHECK-BMI2-SSE2-NEXT:    pxor %xmm1, %xmm0
+; CHECK-BMI2-SSE2-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: shr_to_ror_eq_4xi32_s4_fail_no_splat:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vpsrlvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
+; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [268435455,268435455,268435455,268435455]
+; CHECK-AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm0, %xmm1, %xmm0
+; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512-LABEL: shr_to_ror_eq_4xi32_s4_fail_no_splat:
+; CHECK-AVX512:       # %bb.0:
+; CHECK-AVX512-NEXT:    vpsrlvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
+; CHECK-AVX512-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0
+; CHECK-AVX512-NEXT:    vpcmpeqd %xmm0, %xmm1, %xmm0
+; CHECK-AVX512-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
+; CHECK-AVX512-NEXT:    retq
+  %shr = lshr <4 x i32> %x, <i32 4, i32 4, i32 4, i32 8>
+  %and = and <4 x i32> %x, <i32 268435455, i32 268435455, i32 268435455, i32 268435455>
+  %r = icmp ne <4 x i32> %shr, %and
+  ret <4 x i1> %r
+}
+
+define <16 x i1> @shl_to_ror_eq_16xi16_s8_fail_preserve_i16(<16 x i16> %x) {
+; CHECK-NOBMI-LABEL: shl_to_ror_eq_16xi16_s8_fail_preserve_i16:
+; CHECK-NOBMI:       # %bb.0:
+; CHECK-NOBMI-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-NOBMI-NEXT:    psllw $8, %xmm2
+; CHECK-NOBMI-NEXT:    movdqa %xmm1, %xmm3
+; CHECK-NOBMI-NEXT:    psllw $8, %xmm3
+; CHECK-NOBMI-NEXT:    movdqa {{.*#+}} xmm4 = [0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255]
+; CHECK-NOBMI-NEXT:    pand %xmm4, %xmm0
+; CHECK-NOBMI-NEXT:    pcmpeqw %xmm2, %xmm0
+; CHECK-NOBMI-NEXT:    pand %xmm4, %xmm1
+; CHECK-NOBMI-NEXT:    pcmpeqw %xmm3, %xmm1
+; CHECK-NOBMI-NEXT:    packsswb %xmm1, %xmm0
+; CHECK-NOBMI-NEXT:    pcmpeqd %xmm1, %xmm1
+; CHECK-NOBMI-NEXT:    pxor %xmm1, %xmm0
+; CHECK-NOBMI-NEXT:    retq
+;
+; CHECK-BMI2-SSE2-LABEL: shl_to_ror_eq_16xi16_s8_fail_preserve_i16:
+; CHECK-BMI2-SSE2:       # %bb.0:
+; CHECK-BMI2-SSE2-NEXT:    movdqa %xmm0, %xmm2
+; CHECK-BMI2-SSE2-NEXT:    psllw $8, %xmm2
+; CHECK-BMI2-SSE2-NEXT:    movdqa %xmm1, %xmm3
+; CHECK-BMI2-SSE2-NEXT:    psllw $8, %xmm3
+; CHECK-BMI2-SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255]
+; CHECK-BMI2-SSE2-NEXT:    pand %xmm4, %xmm0
+; CHECK-BMI2-SSE2-NEXT:    pcmpeqw %xmm2, %xmm0
+; CHECK-BMI2-SSE2-NEXT:    pand %xmm4, %xmm1
+; CHECK-BMI2-SSE2-NEXT:    pcmpeqw %xmm3, %xmm1
+; CHECK-BMI2-SSE2-NEXT:    packsswb %xmm1, %xmm0
+; CHECK-BMI2-SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
+; CHECK-BMI2-SSE2-NEXT:    pxor %xmm1, %xmm0
+; CHECK-BMI2-SSE2-NEXT:    retq
+;
+; CHECK-AVX2-LABEL: shl_to_ror_eq_16xi16_s8_fail_preserve_i16:
+; CHECK-AVX2:       # %bb.0:
+; CHECK-AVX2-NEXT:    vpsllw $8, %ymm0, %ymm1
+; CHECK-AVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; CHECK-AVX2-NEXT:    vpcmpeqw %ymm0, %ymm1, %ymm0
+; CHECK-AVX2-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1
+; CHECK-AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; CHECK-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; CHECK-AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
+; CHECK-AVX2-NEXT:    vzeroupper
+; CHECK-AVX2-NEXT:    retq
+;
+; CHECK-AVX512-LABEL: shl_to_ror_eq_16xi16_s8_fail_preserve_i16:
+; CHECK-AVX512:       # %bb.0:
+; CHECK-AVX512-NEXT:    vpsllw $8, %ymm0, %ymm1
+; CHECK-AVX512-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
+; CHECK-AVX512-NEXT:    vpcmpeqw %ymm0, %ymm1, %ymm0
+; CHECK-AVX512-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; CHECK-AVX512-NEXT:    vpmovdb %zmm0, %xmm0
+; CHECK-AVX512-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
+; CHECK-AVX512-NEXT:    vzeroupper
+; CHECK-AVX512-NEXT:    retq
+  %shr = shl <16 x i16> %x, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
+  %and = and <16 x i16> %x, <i16 4294967040, i16 4294967040, i16 4294967040, i16 4294967040, i16 4294967040, i16 4294967040, i16 4294967040, i16 4294967040, i16 4294967040, i16 4294967040, i16 4294967040, i16 4294967040, i16 4294967040, i16 4294967040, i16 4294967040, i16 4294967040>
+  %r = icmp ne <16 x i16> %shr, %and
+  ret <16 x i1> %r
+}
+
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK-AVX: {{.*}}
+; CHECK-NOBMI-SSE2: {{.*}}
diff --git a/llvm/test/CodeGen/X86/code-model-elf-sections.ll b/llvm/test/CodeGen/X86/code-model-elf-sections.ll
index 716bf01bb5..fe659fa9a4 100644
--- a/llvm/test/CodeGen/X86/code-model-elf-sections.ll
+++ b/llvm/test/CodeGen/X86/code-model-elf-sections.ll
@@ -17,6 +17,7 @@
 ; RUN: llvm-readelf -S %t | FileCheck %s --check-prefix=SMALL-DS
 
 ; SMALL: .data {{.*}} WA {{.*}}
+; SMALL: foo {{.*}} WA {{.*}}
 ; SMALL: .bss {{.*}} WA {{.*}}
 ; SMALL: .rodata {{.*}} A {{.*}}
 ; SMALL: .data.rel.ro {{.*}} WA {{.*}}
@@ -24,6 +25,7 @@
 ; SMALL: .tdata {{.*}} WAT {{.*}}
 
 ; SMALL-DS: .data.data {{.*}} WA {{.*}}
+; SMALL-DS: foo {{.*}} WA {{.*}}
 ; SMALL-DS: .bss.bss {{.*}} WA {{.*}}
 ; SMALL-DS: .rodata.rodata {{.*}} A {{.*}}
 ; SMALL-DS: .data.rel.ro.relro {{.*}} WA {{.*}}
@@ -31,6 +33,7 @@
 ; SMALL-DS: .tdata.tdata {{.*}} WAT {{.*}}
 
 ; LARGE: .ldata {{.*}} WAl {{.*}}
+; LARGE: foo {{.*}} WAl {{.*}}
 ; LARGE: .lbss {{.*}} WAl {{.*}}
 ; LARGE: .lrodata {{.*}} Al {{.*}}
 ; LARGE: .ldata.rel.ro {{.*}} WAl {{.*}}
@@ -38,6 +41,7 @@
 ; LARGE: .tdata {{.*}} WAT {{.*}}
 
 ; LARGE-DS: .ldata.data {{.*}} WAl {{.*}}
+; LARGE-DS: foo {{.*}} WAl {{.*}}
 ; LARGE-DS: .lbss.bss {{.*}} WAl {{.*}}
 ; LARGE-DS: .lrodata.rodata {{.*}} Al {{.*}}
 ; LARGE-DS: .ldata.rel.ro.relro {{.*}} WAl {{.*}}
@@ -48,6 +52,7 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64--linux"
 
 @data = internal global [10 x i64] [i64 1, i64 2, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0]
+@data_with_explicit_section = internal global [10 x i64] [i64 1, i64 2, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0], section "foo"
 @bss = internal global [10 x i64] zeroinitializer
 @rodata = internal constant [10 x i64] zeroinitializer
 @relro = internal constant [10 x ptr] [ptr @func, ptr @func, ptr @func, ptr @func, ptr @func, ptr @func, ptr @func, ptr @func, ptr @func, ptr @func]
diff --git a/llvm/test/CodeGen/X86/combine-or.ll b/llvm/test/CodeGen/X86/combine-or.ll
index 460251ffa6..3073cf0124 100644
--- a/llvm/test/CodeGen/X86/combine-or.ll
+++ b/llvm/test/CodeGen/X86/combine-or.ll
@@ -249,9 +249,8 @@ define <4 x i32> @test18(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LIS-LABEL: test18:
 ; CHECK-LIS:       # %bb.0:
 ; CHECK-LIS-NEXT:    pxor %xmm2, %xmm2
-; CHECK-LIS-NEXT:    pxor %xmm3, %xmm3
-; CHECK-LIS-NEXT:    pblendw {{.*#+}} xmm3 = xmm0[0,1],xmm3[2,3,4,5,6,7]
-; CHECK-LIS-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,0,1,1]
+; CHECK-LIS-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3,4,5,6,7]
+; CHECK-LIS-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,0,1,1]
 ; CHECK-LIS-NEXT:    pblendw {{.*#+}} xmm2 = xmm1[0,1],xmm2[2,3,4,5,6,7]
 ; CHECK-LIS-NEXT:    por %xmm0, %xmm2
 ; CHECK-LIS-NEXT:    movdqa %xmm2, %xmm0
diff --git a/llvm/test/CodeGen/X86/cpus-intel.ll b/llvm/test/CodeGen/X86/cpus-intel.ll
index 8d4600c638..5e4d09e081 100644
--- a/llvm/test/CodeGen/X86/cpus-intel.ll
+++ b/llvm/test/CodeGen/X86/cpus-intel.ll
@@ -37,6 +37,8 @@
 ; RUN: llc < %s -o /dev/null -mtriple=i686-unknown-unknown -mcpu=arrowlake_s 2>&1 | FileCheck %s --check-prefix=CHECK-NO-ERROR --allow-empty
 ; RUN: llc < %s -o /dev/null -mtriple=i686-unknown-unknown -mcpu=lunarlake 2>&1 | FileCheck %s --check-prefix=CHECK-NO-ERROR --allow-empty
 ; RUN: llc < %s -o /dev/null -mtriple=i686-unknown-unknown -mcpu=gracemont 2>&1 | FileCheck %s --check-prefix=CHECK-NO-ERROR --allow-empty
+; RUN: llc < %s -o /dev/null -mtriple=i686-unknown-unknown -mcpu=pantherlake 2>&1 | FileCheck %s --check-prefix=CHECK-NO-ERROR --allow-empty
+; RUN: llc < %s -o /dev/null -mtriple=i686-unknown-unknown -mcpu=clearwaterforest 2>&1 | FileCheck %s --check-prefix=CHECK-NO-ERROR --allow-empty
 
 ; RUN: llc < %s -o /dev/null -mtriple=x86_64-unknown-unknown -mcpu=nocona 2>&1 | FileCheck %s --check-prefix=CHECK-NO-ERROR --allow-empty
 ; RUN: llc < %s -o /dev/null -mtriple=x86_64-unknown-unknown -mcpu=core2 2>&1 | FileCheck %s --check-prefix=CHECK-NO-ERROR --allow-empty
@@ -100,6 +102,8 @@
 ; RUN: llc < %s -o /dev/null -mtriple=x86_64-unknown-unknown -mcpu=arrowlake_s 2>&1 | FileCheck %s --check-prefix=CHECK-NO-ERROR --allow-empty
 ; RUN: llc < %s -o /dev/null -mtriple=x86_64-unknown-unknown -mcpu=lunarlake 2>&1 | FileCheck %s --check-prefix=CHECK-NO-ERROR --allow-empty
 ; RUN: llc < %s -o /dev/null -mtriple=x86_64-unknown-unknown -mcpu=gracemont 2>&1 | FileCheck %s --check-prefix=CHECK-NO-ERROR --allow-empty
+; RUN: llc < %s -o /dev/null -mtriple=x86_64-unknown-unknown -mcpu=pantherlake 2>&1 | FileCheck %s --check-prefix=CHECK-NO-ERROR --allow-empty
+; RUN: llc < %s -o /dev/null -mtriple=x86_64-unknown-unknown -mcpu=clearwaterforest 2>&1 | FileCheck %s --check-prefix=CHECK-NO-ERROR --allow-empty
 
 define void @foo() {
   ret void
diff --git a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
index d26f4b7044..e12ca56023 100644
--- a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
+++ b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
@@ -178,15 +178,15 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
 ; X86-NEXT:    subl $152, %esp
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl %ecx, %eax
 ; X86-NEXT:    sarl $31, %eax
-; X86-NEXT:    movl %ebp, %edx
-; X86-NEXT:    sarl $31, %edx
+; X86-NEXT:    movl %edx, %edi
+; X86-NEXT:    sarl $31, %edi
 ; X86-NEXT:    movl %eax, %esi
 ; X86-NEXT:    xorl %ecx, %esi
-; X86-NEXT:    movl %esi, %edi
+; X86-NEXT:    movl %esi, %ebp
 ; X86-NEXT:    movl %eax, %ecx
 ; X86-NEXT:    xorl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl %ecx, %ebx
@@ -195,66 +195,67 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    movl %eax, %esi
 ; X86-NEXT:    xorl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    subl %eax, %esi
-; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl %esi, (%esp) # 4-byte Spill
 ; X86-NEXT:    sbbl %eax, %ecx
 ; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    sbbl %eax, %ebx
-; X86-NEXT:    movl %ebx, (%esp) # 4-byte Spill
-; X86-NEXT:    sbbl %eax, %edi
-; X86-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl %edx, %edi
-; X86-NEXT:    xorl %ebp, %edi
-; X86-NEXT:    movl %edx, %ebx
+; X86-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    sbbl %eax, %ebp
+; X86-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl %edi, %esi
+; X86-NEXT:    xorl %edx, %esi
+; X86-NEXT:    movl %edi, %edx
+; X86-NEXT:    xorl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl %edi, %ebx
 ; X86-NEXT:    xorl {{[0-9]+}}(%esp), %ebx
-; X86-NEXT:    movl %edx, %esi
-; X86-NEXT:    xorl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    movl %edx, %ebp
+; X86-NEXT:    movl %edi, %ebp
 ; X86-NEXT:    xorl {{[0-9]+}}(%esp), %ebp
-; X86-NEXT:    subl %edx, %ebp
-; X86-NEXT:    sbbl %edx, %esi
-; X86-NEXT:    sbbl %edx, %ebx
-; X86-NEXT:    sbbl %edx, %edi
-; X86-NEXT:    xorl %eax, %edx
-; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    orl %edi, %eax
+; X86-NEXT:    subl %edi, %ebp
+; X86-NEXT:    sbbl %edi, %ebx
+; X86-NEXT:    sbbl %edi, %edx
+; X86-NEXT:    sbbl %edi, %esi
+; X86-NEXT:    xorl %eax, %edi
+; X86-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl %ebx, %eax
+; X86-NEXT:    orl %esi, %eax
 ; X86-NEXT:    movl %ebp, %ecx
-; X86-NEXT:    orl %ebx, %ecx
+; X86-NEXT:    orl %edx, %ecx
+; X86-NEXT:    movl %edx, %edi
 ; X86-NEXT:    orl %eax, %ecx
 ; X86-NEXT:    sete %cl
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X86-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT:    orl (%esp), %edx # 4-byte Folded Reload
+; X86-NEXT:    movl (%esp), %edx # 4-byte Reload
+; X86-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X86-NEXT:    orl %eax, %edx
 ; X86-NEXT:    sete %al
 ; X86-NEXT:    orb %cl, %al
 ; X86-NEXT:    movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X86-NEXT:    bsrl %edi, %edx
+; X86-NEXT:    bsrl %esi, %edx
 ; X86-NEXT:    xorl $31, %edx
-; X86-NEXT:    bsrl %ebx, %ecx
+; X86-NEXT:    bsrl %edi, %ecx
 ; X86-NEXT:    xorl $31, %ecx
 ; X86-NEXT:    addl $32, %ecx
-; X86-NEXT:    testl %edi, %edi
+; X86-NEXT:    testl %esi, %esi
 ; X86-NEXT:    cmovnel %edx, %ecx
-; X86-NEXT:    bsrl %esi, %edx
+; X86-NEXT:    bsrl %ebx, %edx
 ; X86-NEXT:    xorl $31, %edx
 ; X86-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    bsrl %ebp, %ebp
 ; X86-NEXT:    xorl $31, %ebp
 ; X86-NEXT:    addl $32, %ebp
-; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    testl %esi, %esi
+; X86-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    testl %ebx, %ebx
 ; X86-NEXT:    cmovnel %edx, %ebp
 ; X86-NEXT:    addl $64, %ebp
-; X86-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    orl %edi, %ebx
+; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    orl %esi, %edi
 ; X86-NEXT:    cmovnel %ecx, %ebp
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X86-NEXT:    bsrl %edi, %edx
 ; X86-NEXT:    xorl $31, %edx
-; X86-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X86-NEXT:    bsrl %eax, %ecx
 ; X86-NEXT:    xorl $31, %ecx
 ; X86-NEXT:    addl $32, %ecx
@@ -263,7 +264,7 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
 ; X86-NEXT:    bsrl %ebx, %esi
 ; X86-NEXT:    xorl $31, %esi
-; X86-NEXT:    bsrl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT:    bsrl (%esp), %edx # 4-byte Folded Reload
 ; X86-NEXT:    xorl $31, %edx
 ; X86-NEXT:    addl $32, %edx
 ; X86-NEXT:    testl %ebx, %ebx
@@ -272,53 +273,50 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    movl %eax, %esi
 ; X86-NEXT:    orl %edi, %esi
 ; X86-NEXT:    cmovnel %ecx, %edx
-; X86-NEXT:    xorl %ebx, %ebx
+; X86-NEXT:    xorl %esi, %esi
 ; X86-NEXT:    subl %edx, %ebp
-; X86-NEXT:    movl $0, %eax
-; X86-NEXT:    sbbl %eax, %eax
+; X86-NEXT:    movl $0, %ebx
+; X86-NEXT:    sbbl %ebx, %ebx
 ; X86-NEXT:    movl $0, %edx
 ; X86-NEXT:    sbbl %edx, %edx
-; X86-NEXT:    movl $0, %esi
-; X86-NEXT:    sbbl %esi, %esi
+; X86-NEXT:    movl $0, %eax
+; X86-NEXT:    sbbl %eax, %eax
 ; X86-NEXT:    movl $127, %ecx
 ; X86-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    cmpl %ebp, %ecx
-; X86-NEXT:    movl %esi, %ebp
 ; X86-NEXT:    movl $0, %ecx
-; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    sbbl %eax, %ecx
+; X86-NEXT:    sbbl %ebx, %ecx
 ; X86-NEXT:    movl $0, %ecx
 ; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    sbbl %edx, %ecx
 ; X86-NEXT:    movl $0, %ecx
-; X86-NEXT:    sbbl %esi, %ecx
+; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    sbbl %eax, %ecx
 ; X86-NEXT:    setb %cl
 ; X86-NEXT:    orb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Folded Reload
-; X86-NEXT:    cmovnel %ebx, %edi
-; X86-NEXT:    movl (%esp), %edx # 4-byte Reload
-; X86-NEXT:    cmovnel %ebx, %edx
+; X86-NEXT:    cmovnel %esi, %edi
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT:    cmovnel %esi, %edx
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT:    cmovnel %ebx, %eax
-; X86-NEXT:    cmovel {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X86-NEXT:    movl %ebx, %esi
-; X86-NEXT:    jne .LBB4_1
-; X86-NEXT:  # %bb.8: # %_udiv-special-cases
-; X86-NEXT:    movl %ebp, %ebx
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X86-NEXT:    xorl $127, %ebp
-; X86-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    orl %ebx, %ecx
-; X86-NEXT:    orl %ebp, %ecx
+; X86-NEXT:    cmovnel %esi, %eax
+; X86-NEXT:    cmovel (%esp), %esi # 4-byte Folded Reload
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    jne .LBB4_8
+; X86-NEXT:  # %bb.1: # %_udiv-special-cases
+; X86-NEXT:    movl %ebx, %ecx
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT:    je .LBB4_9
-; X86-NEXT:  # %bb.5: # %udiv-bb1
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    xorl $127, %ebx
+; X86-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT:    orl %ebx, %ecx
+; X86-NEXT:    je .LBB4_8
+; X86-NEXT:  # %bb.2: # %udiv-bb1
+; X86-NEXT:    movl (%esp), %eax # 4-byte Reload
 ; X86-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X86-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT:    movl (%esp), %eax # 4-byte Reload
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X86-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X86-NEXT:    movl %eax, {{[0-9]+}}(%esp)
@@ -326,57 +324,49 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    movl $0, {{[0-9]+}}(%esp)
 ; X86-NEXT:    movl $0, {{[0-9]+}}(%esp)
 ; X86-NEXT:    movl $0, {{[0-9]+}}(%esp)
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT:    movl %ecx, %eax
-; X86-NEXT:    movl %ecx, %ebp
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    # kill: def $al killed $al killed $eax
 ; X86-NEXT:    xorb $127, %al
 ; X86-NEXT:    movb %al, %ch
 ; X86-NEXT:    andb $7, %ch
 ; X86-NEXT:    shrb $3, %al
 ; X86-NEXT:    andb $15, %al
 ; X86-NEXT:    negb %al
-; X86-NEXT:    movsbl %al, %edi
-; X86-NEXT:    movl 144(%esp,%edi), %edx
-; X86-NEXT:    movl 148(%esp,%edi), %esi
+; X86-NEXT:    movsbl %al, %ebx
+; X86-NEXT:    movl 144(%esp,%ebx), %edx
+; X86-NEXT:    movl 148(%esp,%ebx), %edi
 ; X86-NEXT:    movb %ch, %cl
-; X86-NEXT:    shldl %cl, %edx, %esi
-; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    shldl %cl, %edx, %edi
 ; X86-NEXT:    shll %cl, %edx
 ; X86-NEXT:    notb %cl
-; X86-NEXT:    movl 140(%esp,%edi), %eax
+; X86-NEXT:    movl 140(%esp,%ebx), %eax
 ; X86-NEXT:    movl %eax, %esi
 ; X86-NEXT:    shrl %esi
 ; X86-NEXT:    shrl %cl, %esi
 ; X86-NEXT:    orl %edx, %esi
 ; X86-NEXT:    movl %esi, %edx
-; X86-NEXT:    movl 136(%esp,%edi), %esi
+; X86-NEXT:    movl 136(%esp,%ebx), %esi
 ; X86-NEXT:    movb %ch, %cl
 ; X86-NEXT:    shldl %cl, %esi, %eax
 ; X86-NEXT:    shll %cl, %esi
 ; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    addl $1, %ebp
-; X86-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    addl $1, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X86-NEXT:    adcl $0, %ecx
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT:    adcl $0, %edi
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT:    adcl $0, %esi
-; X86-NEXT:    jae .LBB4_2
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT:    adcl $0, %ebx
+; X86-NEXT:    adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT:    jae .LBB4_3
 ; X86-NEXT:  # %bb.6:
-; X86-NEXT:    xorl %ebp, %ebp
-; X86-NEXT:    xorl %ecx, %ecx
+; X86-NEXT:    xorl %ebx, %ebx
+; X86-NEXT:    xorl %esi, %esi
 ; X86-NEXT:    jmp .LBB4_7
-; X86-NEXT:  .LBB4_1:
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT:    jmp .LBB4_9
-; X86-NEXT:  .LBB4_2: # %udiv-preheader
-; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT:  .LBB4_3: # %udiv-preheader
+; X86-NEXT:    movl (%esp), %esi # 4-byte Reload
 ; X86-NEXT:    movl %esi, {{[0-9]+}}(%esp)
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X86-NEXT:    movl %esi, {{[0-9]+}}(%esp)
-; X86-NEXT:    movl (%esp), %esi # 4-byte Reload
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X86-NEXT:    movl %esi, {{[0-9]+}}(%esp)
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X86-NEXT:    movl %esi, {{[0-9]+}}(%esp)
@@ -384,37 +374,36 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    movl $0, {{[0-9]+}}(%esp)
 ; X86-NEXT:    movl $0, {{[0-9]+}}(%esp)
 ; X86-NEXT:    movl $0, {{[0-9]+}}(%esp)
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
 ; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
 ; X86-NEXT:    movb %bl, %ch
 ; X86-NEXT:    andb $7, %ch
 ; X86-NEXT:    movb %bl, %cl
 ; X86-NEXT:    shrb $3, %cl
 ; X86-NEXT:    andb $15, %cl
-; X86-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movzbl %cl, %ebx
-; X86-NEXT:    movl 100(%esp,%ebx), %esi
-; X86-NEXT:    movl %esi, (%esp) # 4-byte Spill
-; X86-NEXT:    movl 96(%esp,%ebx), %edi
+; X86-NEXT:    movzbl %cl, %ebp
+; X86-NEXT:    movl 100(%esp,%ebp), %esi
+; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl 96(%esp,%ebp), %ebx
+; X86-NEXT:    movl %ebp, %eax
 ; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl %edi, %ebp
+; X86-NEXT:    movl %ebx, %edx
 ; X86-NEXT:    movb %ch, %cl
-; X86-NEXT:    shrdl %cl, %esi, %ebp
-; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT:    movl 88(%esp,%ebx), %esi
-; X86-NEXT:    movl 92(%esp,%ebx), %ebx
-; X86-NEXT:    movl %ebx, %eax
+; X86-NEXT:    shrdl %cl, %esi, %edx
+; X86-NEXT:    movl 88(%esp,%ebp), %ebp
+; X86-NEXT:    movl 92(%esp,%eax), %esi
+; X86-NEXT:    movl %esi, %eax
 ; X86-NEXT:    shrl %cl, %eax
 ; X86-NEXT:    notb %cl
-; X86-NEXT:    addl %edi, %edi
-; X86-NEXT:    shll %cl, %edi
-; X86-NEXT:    orl %eax, %edi
-; X86-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    addl %ebx, %ebx
+; X86-NEXT:    shll %cl, %ebx
+; X86-NEXT:    orl %eax, %ebx
+; X86-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movb %ch, %cl
-; X86-NEXT:    shrl %cl, (%esp) # 4-byte Folded Spill
-; X86-NEXT:    shrdl %cl, %ebx, %esi
-; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    shrl %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT:    shrdl %cl, %esi, %ebp
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X86-NEXT:    addl $-1, %eax
 ; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
@@ -424,115 +413,117 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X86-NEXT:    adcl $-1, %eax
 ; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, %eax
 ; X86-NEXT:    adcl $-1, %eax
 ; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    xorl %esi, %esi
 ; X86-NEXT:    movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT:    movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
 ; X86-NEXT:    .p2align 4, 0x90
-; X86-NEXT:  .LBB4_3: # %udiv-do-while
+; X86-NEXT:  .LBB4_4: # %udiv-do-while
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
+; X86-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl %ebp, %edx
-; X86-NEXT:    shldl $1, %ebp, (%esp) # 4-byte Folded Spill
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X86-NEXT:    shldl $1, %ebp, %edx
+; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT:    shldl $1, %ebx, %ebp
-; X86-NEXT:    shldl $1, %esi, %ebx
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT:    shldl $1, %ecx, %esi
+; X86-NEXT:    shldl $1, %edx, %ebx
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT:    shldl $1, %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT:    shldl $1, %ebp, %edx
+; X86-NEXT:    shldl $1, %edi, %ebp
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT:    orl %eax, %esi
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT:    shldl $1, %edi, %ecx
-; X86-NEXT:    orl %eax, %ecx
-; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    shldl $1, %eax, %edi
+; X86-NEXT:    orl %esi, %edi
+; X86-NEXT:    movl %edi, (%esp) # 4-byte Spill
+; X86-NEXT:    movl %ecx, %edi
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT:    shldl $1, %ecx, %edi
-; X86-NEXT:    orl %eax, %edi
-; X86-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    addl %ecx, %ecx
-; X86-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT:    shldl $1, %ecx, %eax
+; X86-NEXT:    orl %esi, %eax
+; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    shldl $1, %eax, %ecx
+; X86-NEXT:    orl %esi, %ecx
 ; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    cmpl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT:    sbbl %ebp, %ecx
+; X86-NEXT:    addl %eax, %eax
+; X86-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    cmpl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X86-NEXT:    sbbl %edx, %ecx
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT:    sbbl (%esp), %ecx # 4-byte Folded Reload
+; X86-NEXT:    sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    sbbl %ebx, %ecx
 ; X86-NEXT:    sarl $31, %ecx
 ; X86-NEXT:    movl %ecx, %eax
 ; X86-NEXT:    andl $1, %eax
 ; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl %ecx, %eax
-; X86-NEXT:    andl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl %ecx, %esi
+; X86-NEXT:    andl %edi, %esi
 ; X86-NEXT:    movl %ecx, %edi
 ; X86-NEXT:    andl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
 ; X86-NEXT:    movl %ecx, %eax
 ; X86-NEXT:    andl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
 ; X86-NEXT:    andl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-NEXT:    subl %ecx, %ebx
-; X86-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    sbbl %eax, %ebp
-; X86-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    sbbl %edi, %edx
-; X86-NEXT:    movl %edx, %ebp
+; X86-NEXT:    subl %ecx, %ebp
+; X86-NEXT:    sbbl %eax, %edx
+; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT:    sbbl %eax, (%esp) # 4-byte Folded Spill
+; X86-NEXT:    sbbl %edi, %edx
+; X86-NEXT:    movl (%esp), %edi # 4-byte Reload
+; X86-NEXT:    sbbl %esi, %ebx
+; X86-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X86-NEXT:    addl $-1, %ecx
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X86-NEXT:    adcl $-1, %eax
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT:    adcl $-1, %edi
-; X86-NEXT:    adcl $-1, %edx
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT:    adcl $-1, %esi
+; X86-NEXT:    adcl $-1, %ebx
 ; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    orl %edx, %eax
+; X86-NEXT:    orl %ebx, %eax
 ; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    orl %edi, %ecx
-; X86-NEXT:    orl %eax, %ecx
-; X86-NEXT:    jne .LBB4_3
-; X86-NEXT:  # %bb.4:
 ; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT:    orl %esi, %ecx
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT:    orl %eax, %ecx
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    jne .LBB4_4
+; X86-NEXT:  # %bb.5:
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
 ; X86-NEXT:  .LBB4_7: # %udiv-loop-exit
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X86-NEXT:    shldl $1, %edx, %edi
-; X86-NEXT:    orl %ecx, %edi
+; X86-NEXT:    orl %esi, %edi
 ; X86-NEXT:    shldl $1, %eax, %edx
-; X86-NEXT:    orl %ecx, %edx
+; X86-NEXT:    orl %esi, %edx
+; X86-NEXT:    movl %esi, %ecx
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X86-NEXT:    shldl $1, %esi, %eax
 ; X86-NEXT:    orl %ecx, %eax
 ; X86-NEXT:    addl %esi, %esi
-; X86-NEXT:    orl %ebp, %esi
-; X86-NEXT:  .LBB4_9: # %udiv-end
-; X86-NEXT:    xorl %ebx, %edi
-; X86-NEXT:    xorl %ebx, %edx
-; X86-NEXT:    xorl %ebx, %eax
-; X86-NEXT:    xorl %ebx, %esi
-; X86-NEXT:    subl %ebx, %esi
+; X86-NEXT:    orl %ebx, %esi
+; X86-NEXT:  .LBB4_8: # %udiv-end
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    xorl %ecx, %edi
+; X86-NEXT:    xorl %ecx, %edx
+; X86-NEXT:    xorl %ecx, %eax
+; X86-NEXT:    xorl %ecx, %esi
+; X86-NEXT:    subl %ecx, %esi
 ; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    sbbl %ebx, %eax
+; X86-NEXT:    sbbl %ecx, %eax
 ; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    sbbl %ebx, %edx
-; X86-NEXT:    sbbl %ebx, %edi
-; X86-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl %esi, (%ecx)
-; X86-NEXT:    movl %eax, 4(%ecx)
-; X86-NEXT:    movl %edx, 8(%ecx)
-; X86-NEXT:    movl %edi, 12(%ecx)
+; X86-NEXT:    sbbl %ecx, %edx
+; X86-NEXT:    sbbl %ecx, %edi
+; X86-NEXT:    movl %edi, (%esp) # 4-byte Spill
+; X86-NEXT:    movl %esi, (%ebp)
+; X86-NEXT:    movl %eax, 4(%ebp)
+; X86-NEXT:    movl %edx, 8(%ebp)
+; X86-NEXT:    movl %edi, 12(%ebp)
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
 ; X86-NEXT:    movl %edx, %ebx
@@ -541,7 +532,7 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movl %esi, %eax
 ; X86-NEXT:    mull %edi
-; X86-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movl %edx, %edi
 ; X86-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
 ; X86-NEXT:    adcl $0, %ecx
@@ -562,10 +553,10 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    adcl %eax, %edx
 ; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl (%esp), %ecx # 4-byte Reload
 ; X86-NEXT:    imull %eax, %ecx
 ; X86-NEXT:    mull %ebx
-; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl %eax, (%esp) # 4-byte Spill
 ; X86-NEXT:    imull {{[0-9]+}}(%esp), %ebx
 ; X86-NEXT:    addl %edx, %ebx
 ; X86-NEXT:    addl %ecx, %ebx
@@ -577,12 +568,12 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    mull %edx
 ; X86-NEXT:    addl %edx, %ebp
 ; X86-NEXT:    addl %ecx, %ebp
-; X86-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NEXT:    addl (%esp), %eax # 4-byte Folded Reload
 ; X86-NEXT:    adcl %ebx, %ebp
 ; X86-NEXT:    addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
 ; X86-NEXT:    adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    subl (%esp), %edx # 4-byte Folded Reload
+; X86-NEXT:    subl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
diff --git a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
index ebb95f16a7..ae57d31167 100644
--- a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
+++ b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
@@ -304,7 +304,7 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    movl 128(%esp,%eax), %esi
 ; X86-NEXT:    movb %ch, %cl
 ; X86-NEXT:    shldl %cl, %edx, %esi
-; X86-NEXT:    movl %esi, (%esp) # 4-byte Spill
+; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    shll %cl, %edx
 ; X86-NEXT:    notb %cl
 ; X86-NEXT:    movl 120(%esp,%eax), %ebp
@@ -319,10 +319,10 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    shll %cl, %ebp
 ; X86-NEXT:    addl $1, %edi
 ; X86-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    adcl $0, %ecx
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X86-NEXT:    adcl $0, %eax
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT:    adcl $0, %edi
 ; X86-NEXT:    adcl $0, %ebx
 ; X86-NEXT:    jae .LBB4_3
 ; X86-NEXT:  # %bb.6:
@@ -331,14 +331,14 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    movl %edx, %ebx
 ; X86-NEXT:    jmp .LBB4_7
 ; X86-NEXT:  .LBB4_3: # %udiv-preheader
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
 ; X86-NEXT:    movl $0, {{[0-9]+}}(%esp)
@@ -348,22 +348,23 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X86-NEXT:    movb %al, %ch
 ; X86-NEXT:    andb $7, %ch
-; X86-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    # kill: def $al killed $al killed $eax
 ; X86-NEXT:    shrb $3, %al
 ; X86-NEXT:    andb $15, %al
 ; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    movl 80(%esp,%eax), %ebp
+; X86-NEXT:    movl 80(%esp,%eax), %edi
+; X86-NEXT:    movl %edi, (%esp) # 4-byte Spill
 ; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movl 76(%esp,%eax), %edi
 ; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movl %edi, %ebx
 ; X86-NEXT:    movb %ch, %cl
-; X86-NEXT:    shrdl %cl, %ebp, %ebx
+; X86-NEXT:    movl (%esp), %edx # 4-byte Reload
+; X86-NEXT:    shrdl %cl, %edx, %ebx
 ; X86-NEXT:    movl 68(%esp,%eax), %esi
-; X86-NEXT:    movl 72(%esp,%eax), %edx
-; X86-NEXT:    movl %edx, %eax
+; X86-NEXT:    movl 72(%esp,%eax), %eax
+; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    shrl %cl, %eax
 ; X86-NEXT:    notb %cl
 ; X86-NEXT:    addl %edi, %edi
@@ -371,8 +372,10 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    orl %eax, %edi
 ; X86-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movb %ch, %cl
-; X86-NEXT:    shrl %cl, %ebp
-; X86-NEXT:    shrdl %cl, %edx, %esi
+; X86-NEXT:    movl (%esp), %edx # 4-byte Reload
+; X86-NEXT:    shrl %cl, %edx
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    shrdl %cl, %eax, %esi
 ; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    addl $-1, %eax
@@ -383,19 +386,20 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    adcl $-1, %ecx
 ; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    adcl $-1, %esi
-; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    adcl $-1, %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X86-NEXT:    movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT:    movl (%esp), %esi # 4-byte Reload
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X86-NEXT:    .p2align 4, 0x90
 ; X86-NEXT:  .LBB4_4: # %udiv-do-while
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
-; X86-NEXT:    movl %ebx, (%esp) # 4-byte Spill
-; X86-NEXT:    shldl $1, %ebx, %ebp
+; X86-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    shldl $1, %ebx, %edx
+; X86-NEXT:    movl %edx, (%esp) # 4-byte Spill
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT:    shldl $1, %ebx, (%esp) # 4-byte Folded Spill
+; X86-NEXT:    shldl $1, %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
 ; X86-NEXT:    shldl $1, %edx, %ebx
 ; X86-NEXT:    shldl $1, %esi, %edx
@@ -407,27 +411,25 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    shldl $1, %ecx, %eax
 ; X86-NEXT:    orl %edi, %eax
 ; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT:    shldl $1, %eax, %ecx
+; X86-NEXT:    shldl $1, %ebp, %ecx
 ; X86-NEXT:    orl %edi, %ecx
 ; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    addl %eax, %eax
-; X86-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    addl %ebp, %ebp
+; X86-NEXT:    orl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X86-NEXT:    movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    cmpl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X86-NEXT:    sbbl %ebx, %ecx
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT:    sbbl (%esp), %ecx # 4-byte Folded Reload
+; X86-NEXT:    sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT:    sbbl %ebp, %ecx
+; X86-NEXT:    sbbl (%esp), %ecx # 4-byte Folded Reload
 ; X86-NEXT:    sarl $31, %ecx
 ; X86-NEXT:    movl %ecx, %eax
 ; X86-NEXT:    andl $1, %eax
 ; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl %ecx, %eax
-; X86-NEXT:    andl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl %ecx, %ebp
+; X86-NEXT:    andl {{[0-9]+}}(%esp), %ebp
 ; X86-NEXT:    movl %ecx, %edi
 ; X86-NEXT:    andl {{[0-9]+}}(%esp), %edi
 ; X86-NEXT:    movl %ecx, %eax
@@ -437,36 +439,35 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    sbbl %eax, %ebx
 ; X86-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl (%esp), %ebx # 4-byte Reload
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
 ; X86-NEXT:    sbbl %edi, %ebx
-; X86-NEXT:    sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X86-NEXT:    movl %ebp, (%esp) # 4-byte Spill
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    sbbl %ebp, (%esp) # 4-byte Folded Spill
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X86-NEXT:    addl $-1, %ecx
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X86-NEXT:    adcl $-1, %eax
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT:    adcl $-1, %edx
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
 ; X86-NEXT:    adcl $-1, %edi
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT:    adcl $-1, %edx
 ; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    orl %edi, %eax
-; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    orl %edx, %ecx
-; X86-NEXT:    movl (%esp), %ebp # 4-byte Reload
+; X86-NEXT:    orl %edx, %eax
+; X86-NEXT:    movl (%esp), %edx # 4-byte Reload
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    orl %edi, %ecx
 ; X86-NEXT:    orl %eax, %ecx
 ; X86-NEXT:    jne .LBB4_4
 ; X86-NEXT:  # %bb.5:
-; X86-NEXT:    movl %esi, (%esp) # 4-byte Spill
+; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X86-NEXT:  .LBB4_7: # %udiv-loop-exit
-; X86-NEXT:    movl (%esp), %edx # 4-byte Reload
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
 ; X86-NEXT:    shldl $1, %esi, %edx
 ; X86-NEXT:    orl %eax, %edx
 ; X86-NEXT:    shldl $1, %ebx, %esi
diff --git a/llvm/test/CodeGen/X86/fast-isel-freeze.ll b/llvm/test/CodeGen/X86/fast-isel-freeze.ll
index 8308a28e00..031bccb018 100644
--- a/llvm/test/CodeGen/X86/fast-isel-freeze.ll
+++ b/llvm/test/CodeGen/X86/fast-isel-freeze.ll
@@ -11,8 +11,8 @@ define i32 @freeze(i32 %t) {
 ;
 ; FAST-LABEL: freeze:
 ; FAST:       # %bb.0:
-; FAST-NEXT:    movl $10, %eax
-; FAST-NEXT:    xorl %edi, %eax
+; FAST-NEXT:    movl %edi, %eax
+; FAST-NEXT:    xorl $10, %eax
 ; FAST-NEXT:    retq
   %1 = freeze i32 %t
   %2 = freeze i32 10
diff --git a/llvm/test/CodeGen/X86/foldimmediate-size.ll b/llvm/test/CodeGen/X86/foldimmediate-size.ll
new file mode 100644
index 0000000000..8d4c0a462d
--- /dev/null
+++ b/llvm/test/CodeGen/X86/foldimmediate-size.ll
@@ -0,0 +1,57 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
+; RUN: llc < %s -mtriple=x86_64-unknown | FileCheck %s
+
+; When optimize for size, the constant $858993459 is moved into a register,
+; and use that register in following two andl instructions.
+
+define i32 @cnt32_optsize(i32 %x) nounwind readnone optsize {
+; CHECK-LABEL: cnt32_optsize:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    shrl %eax
+; CHECK-NEXT:    andl $1431655765, %eax # imm = 0x55555555
+; CHECK-NEXT:    subl %eax, %edi
+; CHECK-NEXT:    movl $858993459, %eax # imm = 0x33333333
+; CHECK-NEXT:    movl %edi, %ecx
+; CHECK-NEXT:    andl %eax, %ecx
+; CHECK-NEXT:    shrl $2, %edi
+; CHECK-NEXT:    andl %eax, %edi
+; CHECK-NEXT:    addl %ecx, %edi
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    shrl $4, %eax
+; CHECK-NEXT:    addl %edi, %eax
+; CHECK-NEXT:    andl $252645135, %eax # imm = 0xF0F0F0F
+; CHECK-NEXT:    imull $16843009, %eax, %eax # imm = 0x1010101
+; CHECK-NEXT:    shrl $24, %eax
+; CHECK-NEXT:    retq
+  %cnt = tail call i32 @llvm.ctpop.i32(i32 %x)
+  ret i32 %cnt
+}
+
+; When optimize for speed, the constant $858993459 can be directly folded into
+; two andl instructions.
+
+define i32 @cnt32_optspeed(i32 %x) nounwind readnone {
+; CHECK-LABEL: cnt32_optspeed:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    shrl %eax
+; CHECK-NEXT:    andl $1431655765, %eax # imm = 0x55555555
+; CHECK-NEXT:    subl %eax, %edi
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    andl $858993459, %eax # imm = 0x33333333
+; CHECK-NEXT:    shrl $2, %edi
+; CHECK-NEXT:    andl $858993459, %edi # imm = 0x33333333
+; CHECK-NEXT:    addl %eax, %edi
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    shrl $4, %eax
+; CHECK-NEXT:    addl %edi, %eax
+; CHECK-NEXT:    andl $252645135, %eax # imm = 0xF0F0F0F
+; CHECK-NEXT:    imull $16843009, %eax, %eax # imm = 0x1010101
+; CHECK-NEXT:    shrl $24, %eax
+; CHECK-NEXT:    retq
+  %cnt = tail call i32 @llvm.ctpop.i32(i32 %x)
+  ret i32 %cnt
+}
+
+declare i32 @llvm.ctpop.i32(i32) nounwind readnone
diff --git a/llvm/test/CodeGen/X86/foldimmediate.mir b/llvm/test/CodeGen/X86/foldimmediate.mir
new file mode 100644
index 0000000000..5fd5ae9c1c
--- /dev/null
+++ b/llvm/test/CodeGen/X86/foldimmediate.mir
@@ -0,0 +1,143 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 3
+# RUN: llc -mtriple=x86_64-- -run-pass=peephole-opt %s -o - | FileCheck %s
+--- |
+  define void @foldImmediate() { ret void }
+...
+---
+# Check that immediates can be folded into ALU instructions.
+name: foldImmediate
+registers:
+  - { id: 0, class: gr32 }
+  - { id: 1, class: gr32 }
+  - { id: 2, class: gr32 }
+  - { id: 3, class: gr32 }
+  - { id: 4, class: gr32 }
+  - { id: 5, class: gr32 }
+  - { id: 6, class: gr32 }
+  - { id: 7, class: gr64 }
+  - { id: 8, class: gr64 }
+  - { id: 9, class: gr64 }
+  - { id: 10, class: gr64 }
+  - { id: 11, class: gr64 }
+  - { id: 12, class: gr64 }
+  - { id: 13, class: gr64 }
+  - { id: 14, class: gr64 }
+  - { id: 15, class: gr64 }
+  - { id: 16, class: gr32 }
+  - { id: 17, class: gr64 }
+  - { id: 18, class: gr32 }
+
+body: |
+  bb.0:
+    liveins: $rdi, $rsi
+
+    ; CHECK-LABEL: name: foldImmediate
+    ; CHECK: liveins: $rdi, $rsi
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[MOV32ri:%[0-9]+]]:gr32 = MOV32ri 81
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $edi
+    ; CHECK-NEXT: [[ADD32ri:%[0-9]+]]:gr32 = ADD32ri [[COPY]], 81, implicit-def $eflags
+    ; CHECK-NEXT: NOOP implicit [[ADD32ri]]
+    ; CHECK-NEXT: [[SUB32ri:%[0-9]+]]:gr32 = SUB32ri [[COPY]], 81, implicit-def $eflags
+    ; CHECK-NEXT: NOOP implicit [[SUB32ri]]
+    ; CHECK-NEXT: [[AND32ri:%[0-9]+]]:gr32 = AND32ri [[COPY]], 81, implicit-def $eflags
+    ; CHECK-NEXT: NOOP implicit [[AND32ri]]
+    ; CHECK-NEXT: [[OR32ri:%[0-9]+]]:gr32 = OR32ri [[COPY]], 81, implicit-def $eflags
+    ; CHECK-NEXT: NOOP implicit [[OR32ri]]
+    ; CHECK-NEXT: [[XOR32ri:%[0-9]+]]:gr32 = XOR32ri [[COPY]], 81, implicit-def $eflags
+    ; CHECK-NEXT: NOOP implicit [[XOR32ri]]
+    ; CHECK-NEXT: TEST32ri [[COPY]], 81, implicit-def $eflags
+    ; CHECK-NEXT: NOOP implicit $eflags
+    ; CHECK-NEXT: CMP32ri [[COPY]], 81, implicit-def $eflags
+    ; CHECK-NEXT: NOOP implicit $eflags
+    ; CHECK-NEXT: [[ADC32ri:%[0-9]+]]:gr32 = ADC32ri [[COPY]], 81, implicit-def $eflags, implicit $eflags
+    ; CHECK-NEXT: NOOP implicit [[ADC32ri]]
+    ; CHECK-NEXT: [[SBB32ri:%[0-9]+]]:gr32 = SBB32ri [[COPY]], 81, implicit-def $eflags, implicit $eflags
+    ; CHECK-NEXT: NOOP implicit [[SBB32ri]]
+    ; CHECK-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:gr64 = SUBREG_TO_REG 0, killed [[MOV32ri]], %subreg.sub_32bit
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rsi
+    ; CHECK-NEXT: [[ADD64ri32_:%[0-9]+]]:gr64 = ADD64ri32 [[COPY1]], 81, implicit-def $eflags
+    ; CHECK-NEXT: NOOP implicit [[ADD64ri32_]]
+    ; CHECK-NEXT: [[SUB64ri32_:%[0-9]+]]:gr64 = SUB64ri32 [[COPY1]], 81, implicit-def $eflags
+    ; CHECK-NEXT: NOOP implicit [[SUB64ri32_]]
+    ; CHECK-NEXT: [[AND64ri32_:%[0-9]+]]:gr64 = AND64ri32 [[COPY1]], 81, implicit-def $eflags
+    ; CHECK-NEXT: NOOP implicit [[AND64ri32_]]
+    ; CHECK-NEXT: [[OR64ri32_:%[0-9]+]]:gr64 = OR64ri32 [[COPY1]], 81, implicit-def $eflags
+    ; CHECK-NEXT: NOOP implicit [[OR64ri32_]]
+    ; CHECK-NEXT: [[XOR64ri32_:%[0-9]+]]:gr64 = XOR64ri32 [[COPY1]], 81, implicit-def $eflags
+    ; CHECK-NEXT: NOOP implicit [[XOR64ri32_]]
+    ; CHECK-NEXT: [[MOV32ri64_:%[0-9]+]]:gr64 = MOV32ri64 81
+    ; CHECK-NEXT: NOOP implicit [[MOV32ri64_]]
+    ; CHECK-NEXT: TEST64ri32 [[COPY1]], 81, implicit-def $eflags
+    ; CHECK-NEXT: NOOP implicit $eflags
+    ; CHECK-NEXT: [[ADC64ri32_:%[0-9]+]]:gr64 = ADC64ri32 [[COPY1]], 81, implicit-def $eflags, implicit $eflags
+    ; CHECK-NEXT: NOOP implicit [[ADC64ri32_]]
+    ; CHECK-NEXT: [[SBB64ri32_:%[0-9]+]]:gr64 = SBB64ri32 [[COPY1]], 81, implicit-def $eflags, implicit $eflags
+    ; CHECK-NEXT: NOOP implicit [[SBB64ri32_]]
+    ; CHECK-NEXT: CMP64ri32 [[COPY1]], 81, implicit-def $eflags
+    ; CHECK-NEXT: NOOP implicit $eflags
+    ; CHECK-NEXT: CMP64rr [[SUBREG_TO_REG]], [[COPY1]], implicit-def $eflags
+    ; CHECK-NEXT: NOOP implicit $eflags
+    %0 = MOV32ri 81
+    %1 = COPY $edi
+    %2 = ADD32rr %0, %1, implicit-def $eflags
+    NOOP implicit %2
+
+    %3 = SUB32rr %1, %0, implicit-def $eflags
+    NOOP implicit %3
+
+    %4 = AND32rr %0, %1, implicit-def $eflags
+    NOOP implicit %4
+
+    %5 = OR32rr %0, %1, implicit-def $eflags
+    NOOP implicit %5
+
+    %6 = XOR32rr %0, %1, implicit-def $eflags
+    NOOP implicit %6
+
+    TEST32rr %0, %1, implicit-def $eflags
+    NOOP implicit $eflags
+
+    CMP32rr %1, %0, implicit-def $eflags
+    NOOP implicit $eflags
+
+    %16 = ADC32rr %0, %1, implicit-def $eflags, implicit $eflags
+    NOOP implicit %16
+
+    %18 = SBB32rr %1, %0, implicit-def $eflags, implicit $eflags
+    NOOP implicit %18
+
+    %7 = SUBREG_TO_REG 0, killed %0:gr32, %subreg.sub_32bit
+    %8 = COPY $rsi
+    %9 = ADD64rr %7, %8, implicit-def $eflags
+    NOOP implicit %9
+
+    %10 = SUB64rr %8, %7, implicit-def $eflags
+    NOOP implicit %10
+
+    %11 = AND64rr %8, %7, implicit-def $eflags
+    NOOP implicit %11
+
+    %12 = OR64rr %8, %7, implicit-def $eflags
+    NOOP implicit %12
+
+    %13 = XOR64rr %8, %7, implicit-def $eflags
+    NOOP implicit %13
+
+    %14 = COPY %7
+    NOOP implicit %14
+
+    TEST64rr %8, %7, implicit-def $eflags
+    NOOP implicit $eflags
+
+    %15 = ADC64rr %8, %7, implicit-def $eflags, implicit $eflags
+    NOOP implicit %15
+
+    %17 = SBB64rr %8, %7, implicit-def $eflags, implicit $eflags
+    NOOP implicit %17
+
+    CMP64rr %8, %7, implicit-def $eflags
+    NOOP implicit $eflags
+    CMP64rr %7, %8, implicit-def $eflags
+    NOOP implicit $eflags
+...
diff --git a/llvm/test/CodeGen/X86/fp-intrinsics.ll b/llvm/test/CodeGen/X86/fp-intrinsics.ll
index f4689b2ab6..32e45adcb9 100644
--- a/llvm/test/CodeGen/X86/fp-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/fp-intrinsics.ll
@@ -1145,9 +1145,8 @@ define i128 @f20s128(double %x) nounwind strictfp {
 ;
 ; X86-SSE-LABEL: f20s128:
 ; X86-SSE:       # %bb.0: # %entry
-; X86-SSE-NEXT:    pushl %edi
 ; X86-SSE-NEXT:    pushl %esi
-; X86-SSE-NEXT:    subl $36, %esp
+; X86-SSE-NEXT:    subl $40, %esp
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; X86-SSE-NEXT:    movsd %xmm0, {{[0-9]+}}(%esp)
@@ -1155,18 +1154,11 @@ define i128 @f20s128(double %x) nounwind strictfp {
 ; X86-SSE-NEXT:    movl %eax, (%esp)
 ; X86-SSE-NEXT:    calll __fixdfti
 ; X86-SSE-NEXT:    subl $4, %esp
-; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-SSE-NEXT:    movl %edi, 8(%esi)
-; X86-SSE-NEXT:    movl %edx, 12(%esi)
-; X86-SSE-NEXT:    movl %eax, (%esi)
-; X86-SSE-NEXT:    movl %ecx, 4(%esi)
+; X86-SSE-NEXT:    movaps {{[0-9]+}}(%esp), %xmm0
+; X86-SSE-NEXT:    movaps %xmm0, (%esi)
 ; X86-SSE-NEXT:    movl %esi, %eax
-; X86-SSE-NEXT:    addl $36, %esp
+; X86-SSE-NEXT:    addl $40, %esp
 ; X86-SSE-NEXT:    popl %esi
-; X86-SSE-NEXT:    popl %edi
 ; X86-SSE-NEXT:    retl $4
 ;
 ; SSE-LABEL: f20s128:
@@ -1490,9 +1482,8 @@ define i128 @f20u128(double %x) nounwind strictfp {
 ;
 ; X86-SSE-LABEL: f20u128:
 ; X86-SSE:       # %bb.0: # %entry
-; X86-SSE-NEXT:    pushl %edi
 ; X86-SSE-NEXT:    pushl %esi
-; X86-SSE-NEXT:    subl $36, %esp
+; X86-SSE-NEXT:    subl $40, %esp
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; X86-SSE-NEXT:    movsd %xmm0, {{[0-9]+}}(%esp)
@@ -1500,18 +1491,11 @@ define i128 @f20u128(double %x) nounwind strictfp {
 ; X86-SSE-NEXT:    movl %eax, (%esp)
 ; X86-SSE-NEXT:    calll __fixunsdfti
 ; X86-SSE-NEXT:    subl $4, %esp
-; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-SSE-NEXT:    movl %edi, 8(%esi)
-; X86-SSE-NEXT:    movl %edx, 12(%esi)
-; X86-SSE-NEXT:    movl %eax, (%esi)
-; X86-SSE-NEXT:    movl %ecx, 4(%esi)
+; X86-SSE-NEXT:    movaps {{[0-9]+}}(%esp), %xmm0
+; X86-SSE-NEXT:    movaps %xmm0, (%esi)
 ; X86-SSE-NEXT:    movl %esi, %eax
-; X86-SSE-NEXT:    addl $36, %esp
+; X86-SSE-NEXT:    addl $40, %esp
 ; X86-SSE-NEXT:    popl %esi
-; X86-SSE-NEXT:    popl %edi
 ; X86-SSE-NEXT:    retl $4
 ;
 ; SSE-LABEL: f20u128:
diff --git a/llvm/test/CodeGen/X86/fp128-cast-strict.ll b/llvm/test/CodeGen/X86/fp128-cast-strict.ll
index e116697adc..f141153d05 100644
--- a/llvm/test/CodeGen/X86/fp128-cast-strict.ll
+++ b/llvm/test/CodeGen/X86/fp128-cast-strict.ll
@@ -37,7 +37,7 @@ define dso_local void @TestFPExtF16_F128() nounwind strictfp {
 ; X86-LABEL: TestFPExtF16_F128:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $24, %esp
+; X86-NEXT:    subl $40, %esp
 ; X86-NEXT:    movzwl vf16, %eax
 ; X86-NEXT:    movl %eax, (%esp)
 ; X86-NEXT:    calll __gnu_h2f_ieee
@@ -55,7 +55,7 @@ define dso_local void @TestFPExtF16_F128() nounwind strictfp {
 ; X86-NEXT:    movl %edx, vf128+8
 ; X86-NEXT:    movl %ecx, vf128+4
 ; X86-NEXT:    movl %eax, vf128
-; X86-NEXT:    addl $24, %esp
+; X86-NEXT:    addl $40, %esp
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
 entry:
@@ -87,7 +87,7 @@ define dso_local void @TestFPExtF32_F128() nounwind strictfp {
 ; X86-LABEL: TestFPExtF32_F128:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $24, %esp
+; X86-NEXT:    subl $40, %esp
 ; X86-NEXT:    flds vf32
 ; X86-NEXT:    fstps {{[0-9]+}}(%esp)
 ; X86-NEXT:    wait
@@ -103,7 +103,7 @@ define dso_local void @TestFPExtF32_F128() nounwind strictfp {
 ; X86-NEXT:    movl %edx, vf128+8
 ; X86-NEXT:    movl %ecx, vf128+4
 ; X86-NEXT:    movl %eax, vf128
-; X86-NEXT:    addl $24, %esp
+; X86-NEXT:    addl $40, %esp
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
 entry:
diff --git a/llvm/test/CodeGen/X86/fp128-cast.ll b/llvm/test/CodeGen/X86/fp128-cast.ll
index 5bb44436ab..b39533c0ee 100644
--- a/llvm/test/CodeGen/X86/fp128-cast.ll
+++ b/llvm/test/CodeGen/X86/fp128-cast.ll
@@ -34,7 +34,7 @@ define dso_local void @TestFPExtF32_F128() nounwind {
 ; X32-LABEL: TestFPExtF32_F128:
 ; X32:       # %bb.0: # %entry
 ; X32-NEXT:    pushl %esi
-; X32-NEXT:    subl $24, %esp
+; X32-NEXT:    subl $40, %esp
 ; X32-NEXT:    flds vf32
 ; X32-NEXT:    fstps {{[0-9]+}}(%esp)
 ; X32-NEXT:    leal {{[0-9]+}}(%esp), %eax
@@ -49,7 +49,7 @@ define dso_local void @TestFPExtF32_F128() nounwind {
 ; X32-NEXT:    movl %edx, vf128+8
 ; X32-NEXT:    movl %ecx, vf128+4
 ; X32-NEXT:    movl %eax, vf128
-; X32-NEXT:    addl $24, %esp
+; X32-NEXT:    addl $40, %esp
 ; X32-NEXT:    popl %esi
 ; X32-NEXT:    retl
 ;
@@ -424,7 +424,7 @@ define dso_local void @TestFPToSIF128_I128() nounwind {
 ; X32-NEXT:    pushl %eax
 ; X32-NEXT:    calll __fixtfti
 ; X32-NEXT:    addl $28, %esp
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl (%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
@@ -475,7 +475,7 @@ define dso_local void @TestFPToUIF128_U128() nounwind {
 ; X32-NEXT:    pushl %eax
 ; X32-NEXT:    calll __fixunstfti
 ; X32-NEXT:    addl $28, %esp
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl (%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
@@ -638,7 +638,7 @@ define dso_local void @TestSIToFPI16_F128() nounwind {
 ; X32-NEXT:    pushl %ecx
 ; X32-NEXT:    calll __floatsitf
 ; X32-NEXT:    addl $12, %esp
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl (%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
@@ -686,7 +686,7 @@ define dso_local void @TestSIToFPU16_F128() nounwind {
 ; X32-NEXT:    pushl %ecx
 ; X32-NEXT:    calll __floatunsitf
 ; X32-NEXT:    addl $12, %esp
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl (%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
@@ -732,7 +732,7 @@ define dso_local void @TestSIToFPI32_F128() nounwind {
 ; X32-NEXT:    pushl %eax
 ; X32-NEXT:    calll __floatsitf
 ; X32-NEXT:    addl $12, %esp
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl (%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
@@ -778,7 +778,7 @@ define dso_local void @TestUIToFPU32_F128() #2 {
 ; X32-NEXT:    pushl %eax
 ; X32-NEXT:    calll __floatunsitf
 ; X32-NEXT:    addl $12, %esp
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl (%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
@@ -825,7 +825,7 @@ define dso_local void @TestSIToFPI64_F128() nounwind {
 ; X32-NEXT:    pushl %eax
 ; X32-NEXT:    calll __floatditf
 ; X32-NEXT:    addl $12, %esp
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl (%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
@@ -872,7 +872,7 @@ define dso_local void @TestUIToFPU64_F128() #2 {
 ; X32-NEXT:    pushl %eax
 ; X32-NEXT:    calll __floatunditf
 ; X32-NEXT:    addl $12, %esp
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl (%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
@@ -922,7 +922,7 @@ define dso_local void @TestSIToFPI128_F128() nounwind {
 ; X32-NEXT:    pushl %eax
 ; X32-NEXT:    calll __floattitf
 ; X32-NEXT:    addl $28, %esp
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl (%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
@@ -973,7 +973,7 @@ define dso_local void @TestUIToFPU128_F128() #2 {
 ; X32-NEXT:    pushl %eax
 ; X32-NEXT:    calll __floatuntitf
 ; X32-NEXT:    addl $28, %esp
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl (%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
@@ -1303,7 +1303,7 @@ define fp128 @TestTruncCopysign(fp128 %x, i32 %n) nounwind {
 ; X32-NEXT:    addl $12, %esp
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl (%esp), %edx
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X32-NEXT:  .LBB26_4: # %cleanup
 ; X32-NEXT:    movl %edx, (%esi)
diff --git a/llvm/test/CodeGen/X86/fp128-libcalls-strict.ll b/llvm/test/CodeGen/X86/fp128-libcalls-strict.ll
index caea147ec8..4722ce62f6 100644
--- a/llvm/test/CodeGen/X86/fp128-libcalls-strict.ll
+++ b/llvm/test/CodeGen/X86/fp128-libcalls-strict.ll
@@ -21,9 +21,8 @@ define fp128 @add(fp128 %x, fp128 %y) nounwind strictfp {
 ;
 ; X86-LABEL: add:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    subl $24, %esp
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    subl $12, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
@@ -38,18 +37,11 @@ define fp128 @add(fp128 %x, fp128 %y) nounwind strictfp {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll __addtf3
 ; X86-NEXT:    addl $44, %esp
-; X86-NEXT:    movl (%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT:    movl %edi, 8(%esi)
-; X86-NEXT:    movl %edx, 12(%esi)
-; X86-NEXT:    movl %eax, (%esi)
-; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, (%esi)
 ; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    addl $24, %esp
 ; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
 ; X86-NEXT:    retl $4
 entry:
   %add = call fp128 @llvm.experimental.constrained.fadd.f128(fp128 %x, fp128 %y,  metadata !"round.dynamic", metadata !"fpexcept.strict") #0
@@ -66,9 +58,8 @@ define fp128 @sub(fp128 %x, fp128 %y) nounwind strictfp {
 ;
 ; X86-LABEL: sub:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    subl $24, %esp
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    subl $12, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
@@ -83,18 +74,11 @@ define fp128 @sub(fp128 %x, fp128 %y) nounwind strictfp {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll __subtf3
 ; X86-NEXT:    addl $44, %esp
-; X86-NEXT:    movl (%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT:    movl %edi, 8(%esi)
-; X86-NEXT:    movl %edx, 12(%esi)
-; X86-NEXT:    movl %eax, (%esi)
-; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, (%esi)
 ; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    addl $24, %esp
 ; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
 ; X86-NEXT:    retl $4
 entry:
   %sub = call fp128 @llvm.experimental.constrained.fsub.f128(fp128 %x, fp128 %y,  metadata !"round.dynamic", metadata !"fpexcept.strict") #0
@@ -111,9 +95,8 @@ define fp128 @mul(fp128 %x, fp128 %y) nounwind strictfp {
 ;
 ; X86-LABEL: mul:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    subl $24, %esp
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    subl $12, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
@@ -128,18 +111,11 @@ define fp128 @mul(fp128 %x, fp128 %y) nounwind strictfp {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll __multf3
 ; X86-NEXT:    addl $44, %esp
-; X86-NEXT:    movl (%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT:    movl %edi, 8(%esi)
-; X86-NEXT:    movl %edx, 12(%esi)
-; X86-NEXT:    movl %eax, (%esi)
-; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, (%esi)
 ; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    addl $24, %esp
 ; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
 ; X86-NEXT:    retl $4
 entry:
   %mul = call fp128 @llvm.experimental.constrained.fmul.f128(fp128 %x, fp128 %y,  metadata !"round.dynamic", metadata !"fpexcept.strict") #0
@@ -156,9 +132,8 @@ define fp128 @div(fp128 %x, fp128 %y) nounwind strictfp {
 ;
 ; X86-LABEL: div:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    subl $24, %esp
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    subl $12, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
@@ -173,18 +148,11 @@ define fp128 @div(fp128 %x, fp128 %y) nounwind strictfp {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll __divtf3
 ; X86-NEXT:    addl $44, %esp
-; X86-NEXT:    movl (%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT:    movl %edi, 8(%esi)
-; X86-NEXT:    movl %edx, 12(%esi)
-; X86-NEXT:    movl %eax, (%esi)
-; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, (%esi)
 ; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    addl $24, %esp
 ; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
 ; X86-NEXT:    retl $4
 entry:
   %div = call fp128 @llvm.experimental.constrained.fdiv.f128(fp128 %x, fp128 %y,  metadata !"round.dynamic", metadata !"fpexcept.strict") #0
@@ -201,9 +169,8 @@ define fp128 @fma(fp128 %x, fp128 %y, fp128 %z) nounwind strictfp {
 ;
 ; X86-LABEL: fma:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    subl $24, %esp
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    subl $12, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
@@ -222,18 +189,11 @@ define fp128 @fma(fp128 %x, fp128 %y, fp128 %z) nounwind strictfp {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll fmal
 ; X86-NEXT:    addl $60, %esp
-; X86-NEXT:    movl (%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT:    movl %edi, 8(%esi)
-; X86-NEXT:    movl %edx, 12(%esi)
-; X86-NEXT:    movl %eax, (%esi)
-; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, (%esi)
 ; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    addl $24, %esp
 ; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
 ; X86-NEXT:    retl $4
 entry:
   %fma = call fp128 @llvm.experimental.constrained.fma.f128(fp128 %x, fp128 %y,  fp128 %z, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
@@ -250,9 +210,8 @@ define fp128 @frem(fp128 %x, fp128 %y) nounwind strictfp {
 ;
 ; X86-LABEL: frem:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    subl $24, %esp
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    subl $12, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
@@ -267,18 +226,11 @@ define fp128 @frem(fp128 %x, fp128 %y) nounwind strictfp {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll fmodl
 ; X86-NEXT:    addl $44, %esp
-; X86-NEXT:    movl (%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT:    movl %edi, 8(%esi)
-; X86-NEXT:    movl %edx, 12(%esi)
-; X86-NEXT:    movl %eax, (%esi)
-; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, (%esi)
 ; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    addl $24, %esp
 ; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
 ; X86-NEXT:    retl $4
 entry:
   %div = call fp128 @llvm.experimental.constrained.frem.f128(fp128 %x, fp128 %y,  metadata !"round.dynamic", metadata !"fpexcept.strict") #0
@@ -295,9 +247,8 @@ define fp128 @ceil(fp128 %x) nounwind strictfp {
 ;
 ; X86-LABEL: ceil:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    subl $24, %esp
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    subl $12, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
@@ -308,18 +259,11 @@ define fp128 @ceil(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll ceill
 ; X86-NEXT:    addl $28, %esp
-; X86-NEXT:    movl (%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT:    movl %edi, 8(%esi)
-; X86-NEXT:    movl %edx, 12(%esi)
-; X86-NEXT:    movl %eax, (%esi)
-; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, (%esi)
 ; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    addl $24, %esp
 ; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
 ; X86-NEXT:    retl $4
 entry:
   %ceil = call fp128 @llvm.experimental.constrained.ceil.f128(fp128 %x, metadata !"fpexcept.strict") #0
@@ -336,9 +280,8 @@ define fp128 @cos(fp128 %x) nounwind strictfp {
 ;
 ; X86-LABEL: cos:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    subl $24, %esp
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    subl $12, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
@@ -349,18 +292,11 @@ define fp128 @cos(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll cosl
 ; X86-NEXT:    addl $28, %esp
-; X86-NEXT:    movl (%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT:    movl %edi, 8(%esi)
-; X86-NEXT:    movl %edx, 12(%esi)
-; X86-NEXT:    movl %eax, (%esi)
-; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, (%esi)
 ; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    addl $24, %esp
 ; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
 ; X86-NEXT:    retl $4
 entry:
   %cos = call fp128 @llvm.experimental.constrained.cos.f128(fp128 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
@@ -377,9 +313,8 @@ define fp128 @exp(fp128 %x) nounwind strictfp {
 ;
 ; X86-LABEL: exp:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    subl $24, %esp
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    subl $12, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
@@ -390,18 +325,11 @@ define fp128 @exp(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll expl
 ; X86-NEXT:    addl $28, %esp
-; X86-NEXT:    movl (%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT:    movl %edi, 8(%esi)
-; X86-NEXT:    movl %edx, 12(%esi)
-; X86-NEXT:    movl %eax, (%esi)
-; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, (%esi)
 ; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    addl $24, %esp
 ; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
 ; X86-NEXT:    retl $4
 entry:
   %exp = call fp128 @llvm.experimental.constrained.exp.f128(fp128 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
@@ -418,9 +346,8 @@ define fp128 @exp2(fp128 %x) nounwind strictfp {
 ;
 ; X86-LABEL: exp2:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    subl $24, %esp
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    subl $12, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
@@ -431,18 +358,11 @@ define fp128 @exp2(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll exp2l
 ; X86-NEXT:    addl $28, %esp
-; X86-NEXT:    movl (%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT:    movl %edi, 8(%esi)
-; X86-NEXT:    movl %edx, 12(%esi)
-; X86-NEXT:    movl %eax, (%esi)
-; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, (%esi)
 ; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    addl $24, %esp
 ; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
 ; X86-NEXT:    retl $4
 entry:
   %exp2 = call fp128 @llvm.experimental.constrained.exp2.f128(fp128 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
@@ -459,9 +379,8 @@ define fp128 @floor(fp128 %x) nounwind strictfp {
 ;
 ; X86-LABEL: floor:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    subl $24, %esp
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    subl $12, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
@@ -472,18 +391,11 @@ define fp128 @floor(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll floorl
 ; X86-NEXT:    addl $28, %esp
-; X86-NEXT:    movl (%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT:    movl %edi, 8(%esi)
-; X86-NEXT:    movl %edx, 12(%esi)
-; X86-NEXT:    movl %eax, (%esi)
-; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, (%esi)
 ; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    addl $24, %esp
 ; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
 ; X86-NEXT:    retl $4
 entry:
   %floor = call fp128 @llvm.experimental.constrained.floor.f128(fp128 %x, metadata !"fpexcept.strict") #0
@@ -500,9 +412,8 @@ define fp128 @log(fp128 %x) nounwind strictfp {
 ;
 ; X86-LABEL: log:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    subl $24, %esp
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    subl $12, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
@@ -513,18 +424,11 @@ define fp128 @log(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll logl
 ; X86-NEXT:    addl $28, %esp
-; X86-NEXT:    movl (%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT:    movl %edi, 8(%esi)
-; X86-NEXT:    movl %edx, 12(%esi)
-; X86-NEXT:    movl %eax, (%esi)
-; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, (%esi)
 ; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    addl $24, %esp
 ; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
 ; X86-NEXT:    retl $4
 entry:
   %log = call fp128 @llvm.experimental.constrained.log.f128(fp128 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
@@ -541,9 +445,8 @@ define fp128 @log10(fp128 %x) nounwind strictfp {
 ;
 ; X86-LABEL: log10:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    subl $24, %esp
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    subl $12, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
@@ -554,18 +457,11 @@ define fp128 @log10(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll log10l
 ; X86-NEXT:    addl $28, %esp
-; X86-NEXT:    movl (%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT:    movl %edi, 8(%esi)
-; X86-NEXT:    movl %edx, 12(%esi)
-; X86-NEXT:    movl %eax, (%esi)
-; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, (%esi)
 ; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    addl $24, %esp
 ; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
 ; X86-NEXT:    retl $4
 entry:
   %log10 = call fp128 @llvm.experimental.constrained.log10.f128(fp128 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
@@ -582,9 +478,8 @@ define fp128 @log2(fp128 %x) nounwind strictfp {
 ;
 ; X86-LABEL: log2:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    subl $24, %esp
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    subl $12, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
@@ -595,18 +490,11 @@ define fp128 @log2(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll log2l
 ; X86-NEXT:    addl $28, %esp
-; X86-NEXT:    movl (%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT:    movl %edi, 8(%esi)
-; X86-NEXT:    movl %edx, 12(%esi)
-; X86-NEXT:    movl %eax, (%esi)
-; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, (%esi)
 ; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    addl $24, %esp
 ; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
 ; X86-NEXT:    retl $4
 entry:
   %log2 = call fp128 @llvm.experimental.constrained.log2.f128(fp128 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
@@ -623,9 +511,8 @@ define fp128 @maxnum(fp128 %x, fp128 %y) nounwind strictfp {
 ;
 ; X86-LABEL: maxnum:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    subl $24, %esp
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    subl $12, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
@@ -640,18 +527,11 @@ define fp128 @maxnum(fp128 %x, fp128 %y) nounwind strictfp {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll fmaxl
 ; X86-NEXT:    addl $44, %esp
-; X86-NEXT:    movl (%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT:    movl %edi, 8(%esi)
-; X86-NEXT:    movl %edx, 12(%esi)
-; X86-NEXT:    movl %eax, (%esi)
-; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, (%esi)
 ; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    addl $24, %esp
 ; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
 ; X86-NEXT:    retl $4
 entry:
   %maxnum = call fp128 @llvm.experimental.constrained.maxnum.f128(fp128 %x, fp128 %y, metadata !"fpexcept.strict") #0
@@ -668,9 +548,8 @@ define fp128 @minnum(fp128 %x, fp128 %y) nounwind strictfp {
 ;
 ; X86-LABEL: minnum:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    subl $24, %esp
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    subl $12, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
@@ -685,18 +564,11 @@ define fp128 @minnum(fp128 %x, fp128 %y) nounwind strictfp {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll fminl
 ; X86-NEXT:    addl $44, %esp
-; X86-NEXT:    movl (%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT:    movl %edi, 8(%esi)
-; X86-NEXT:    movl %edx, 12(%esi)
-; X86-NEXT:    movl %eax, (%esi)
-; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, (%esi)
 ; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    addl $24, %esp
 ; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
 ; X86-NEXT:    retl $4
 entry:
   %minnum = call fp128 @llvm.experimental.constrained.minnum.f128(fp128 %x, fp128 %y, metadata !"fpexcept.strict") #0
@@ -713,9 +585,8 @@ define fp128 @nearbyint(fp128 %x) nounwind strictfp {
 ;
 ; X86-LABEL: nearbyint:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    subl $24, %esp
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    subl $12, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
@@ -726,18 +597,11 @@ define fp128 @nearbyint(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll nearbyintl
 ; X86-NEXT:    addl $28, %esp
-; X86-NEXT:    movl (%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT:    movl %edi, 8(%esi)
-; X86-NEXT:    movl %edx, 12(%esi)
-; X86-NEXT:    movl %eax, (%esi)
-; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, (%esi)
 ; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    addl $24, %esp
 ; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
 ; X86-NEXT:    retl $4
 entry:
   %nearbyint = call fp128 @llvm.experimental.constrained.nearbyint.f128(fp128 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
@@ -754,9 +618,8 @@ define fp128 @pow(fp128 %x, fp128 %y) nounwind strictfp {
 ;
 ; X86-LABEL: pow:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    subl $24, %esp
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    subl $12, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
@@ -771,18 +634,11 @@ define fp128 @pow(fp128 %x, fp128 %y) nounwind strictfp {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll powl
 ; X86-NEXT:    addl $44, %esp
-; X86-NEXT:    movl (%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT:    movl %edi, 8(%esi)
-; X86-NEXT:    movl %edx, 12(%esi)
-; X86-NEXT:    movl %eax, (%esi)
-; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, (%esi)
 ; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    addl $24, %esp
 ; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
 ; X86-NEXT:    retl $4
 entry:
   %pow = call fp128 @llvm.experimental.constrained.pow.f128(fp128 %x, fp128 %y, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
@@ -799,9 +655,8 @@ define fp128 @powi(fp128 %x, i32 %y) nounwind strictfp {
 ;
 ; X86-LABEL: powi:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    subl $24, %esp
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    subl $8, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
@@ -813,18 +668,11 @@ define fp128 @powi(fp128 %x, i32 %y) nounwind strictfp {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll __powitf2
 ; X86-NEXT:    addl $28, %esp
-; X86-NEXT:    movl (%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT:    movl %edi, 8(%esi)
-; X86-NEXT:    movl %edx, 12(%esi)
-; X86-NEXT:    movl %eax, (%esi)
-; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, (%esi)
 ; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    addl $24, %esp
 ; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
 ; X86-NEXT:    retl $4
 entry:
   %powi = call fp128 @llvm.experimental.constrained.powi.f128(fp128 %x, i32 %y, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
@@ -841,9 +689,8 @@ define fp128 @rint(fp128 %x) nounwind strictfp {
 ;
 ; X86-LABEL: rint:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    subl $24, %esp
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    subl $12, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
@@ -854,18 +701,11 @@ define fp128 @rint(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll rintl
 ; X86-NEXT:    addl $28, %esp
-; X86-NEXT:    movl (%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT:    movl %edi, 8(%esi)
-; X86-NEXT:    movl %edx, 12(%esi)
-; X86-NEXT:    movl %eax, (%esi)
-; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, (%esi)
 ; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    addl $24, %esp
 ; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
 ; X86-NEXT:    retl $4
 entry:
   %rint = call fp128 @llvm.experimental.constrained.rint.f128(fp128 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
@@ -882,9 +722,8 @@ define fp128 @round(fp128 %x) nounwind strictfp {
 ;
 ; X86-LABEL: round:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    subl $24, %esp
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    subl $12, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
@@ -895,18 +734,11 @@ define fp128 @round(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll roundl
 ; X86-NEXT:    addl $28, %esp
-; X86-NEXT:    movl (%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT:    movl %edi, 8(%esi)
-; X86-NEXT:    movl %edx, 12(%esi)
-; X86-NEXT:    movl %eax, (%esi)
-; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, (%esi)
 ; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    addl $24, %esp
 ; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
 ; X86-NEXT:    retl $4
 entry:
   %round = call fp128 @llvm.experimental.constrained.round.f128(fp128 %x, metadata !"fpexcept.strict") #0
@@ -923,9 +755,8 @@ define fp128 @roundeven(fp128 %x) nounwind strictfp {
 ;
 ; X86-LABEL: roundeven:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    subl $24, %esp
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    subl $12, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
@@ -936,18 +767,11 @@ define fp128 @roundeven(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll roundevenl
 ; X86-NEXT:    addl $28, %esp
-; X86-NEXT:    movl (%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT:    movl %edi, 8(%esi)
-; X86-NEXT:    movl %edx, 12(%esi)
-; X86-NEXT:    movl %eax, (%esi)
-; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, (%esi)
 ; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    addl $24, %esp
 ; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
 ; X86-NEXT:    retl $4
 entry:
   %roundeven = call fp128 @llvm.experimental.constrained.roundeven.f128(fp128 %x, metadata !"fpexcept.strict") #0
@@ -964,9 +788,8 @@ define fp128 @sin(fp128 %x) nounwind strictfp {
 ;
 ; X86-LABEL: sin:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    subl $24, %esp
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    subl $12, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
@@ -977,18 +800,11 @@ define fp128 @sin(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll sinl
 ; X86-NEXT:    addl $28, %esp
-; X86-NEXT:    movl (%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT:    movl %edi, 8(%esi)
-; X86-NEXT:    movl %edx, 12(%esi)
-; X86-NEXT:    movl %eax, (%esi)
-; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, (%esi)
 ; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    addl $24, %esp
 ; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
 ; X86-NEXT:    retl $4
 entry:
   %sin = call fp128 @llvm.experimental.constrained.sin.f128(fp128 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
@@ -1005,9 +821,8 @@ define fp128 @sqrt(fp128 %x) nounwind strictfp {
 ;
 ; X86-LABEL: sqrt:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    subl $24, %esp
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    subl $12, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
@@ -1018,18 +833,11 @@ define fp128 @sqrt(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll sqrtl
 ; X86-NEXT:    addl $28, %esp
-; X86-NEXT:    movl (%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT:    movl %edi, 8(%esi)
-; X86-NEXT:    movl %edx, 12(%esi)
-; X86-NEXT:    movl %eax, (%esi)
-; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, (%esi)
 ; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    addl $24, %esp
 ; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
 ; X86-NEXT:    retl $4
 entry:
   %sqrt = call fp128 @llvm.experimental.constrained.sqrt.f128(fp128 %x, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
@@ -1046,9 +854,8 @@ define fp128 @trunc(fp128 %x) nounwind strictfp {
 ;
 ; X86-LABEL: trunc:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    subl $24, %esp
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    subl $12, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
@@ -1059,18 +866,11 @@ define fp128 @trunc(fp128 %x) nounwind strictfp {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll truncl
 ; X86-NEXT:    addl $28, %esp
-; X86-NEXT:    movl (%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT:    movl %edi, 8(%esi)
-; X86-NEXT:    movl %edx, 12(%esi)
-; X86-NEXT:    movl %eax, (%esi)
-; X86-NEXT:    movl %ecx, 4(%esi)
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, (%esi)
 ; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    addl $24, %esp
 ; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
 ; X86-NEXT:    retl $4
 entry:
   %trunc = call fp128 @llvm.experimental.constrained.trunc.f128(fp128 %x, metadata !"fpexcept.strict") #0
diff --git a/llvm/test/CodeGen/X86/fp128-libcalls.ll b/llvm/test/CodeGen/X86/fp128-libcalls.ll
index 72cb547240..4e7e6b4d01 100644
--- a/llvm/test/CodeGen/X86/fp128-libcalls.ll
+++ b/llvm/test/CodeGen/X86/fp128-libcalls.ll
@@ -22,8 +22,7 @@ define dso_local void @Test128Add(fp128 %d1, fp128 %d2) nounwind {
 ;
 ; X86-LABEL: Test128Add:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    subl $40, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
@@ -36,16 +35,9 @@ define dso_local void @Test128Add(fp128 %d1, fp128 %d2) nounwind {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll __addtf3
 ; X86-NEXT:    addl $44, %esp
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    movl %esi, vf128+12
-; X86-NEXT:    movl %edx, vf128+8
-; X86-NEXT:    movl %ecx, vf128+4
-; X86-NEXT:    movl %eax, vf128
-; X86-NEXT:    addl $24, %esp
-; X86-NEXT:    popl %esi
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, vf128
+; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    retl
 entry:
   %add = fadd fp128 %d1, %d2
@@ -66,8 +58,7 @@ define dso_local void @Test128_1Add(fp128 %d1) nounwind {
 ;
 ; X86-LABEL: Test128_1Add:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    subl $40, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
@@ -80,16 +71,9 @@ define dso_local void @Test128_1Add(fp128 %d1) nounwind {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll __addtf3
 ; X86-NEXT:    addl $44, %esp
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    movl %esi, vf128+8
-; X86-NEXT:    movl %edx, vf128+12
-; X86-NEXT:    movl %eax, vf128
-; X86-NEXT:    movl %ecx, vf128+4
-; X86-NEXT:    addl $24, %esp
-; X86-NEXT:    popl %esi
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, vf128
+; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    retl
 entry:
   %0 = load fp128, ptr @vf128, align 16
@@ -109,8 +93,7 @@ define dso_local void @Test128Sub(fp128 %d1, fp128 %d2) nounwind {
 ;
 ; X86-LABEL: Test128Sub:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    subl $40, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
@@ -123,16 +106,9 @@ define dso_local void @Test128Sub(fp128 %d1, fp128 %d2) nounwind {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll __subtf3
 ; X86-NEXT:    addl $44, %esp
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    movl %esi, vf128+12
-; X86-NEXT:    movl %edx, vf128+8
-; X86-NEXT:    movl %ecx, vf128+4
-; X86-NEXT:    movl %eax, vf128
-; X86-NEXT:    addl $24, %esp
-; X86-NEXT:    popl %esi
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, vf128
+; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    retl
 entry:
   %sub = fsub fp128 %d1, %d2
@@ -153,8 +129,7 @@ define dso_local void @Test128_1Sub(fp128 %d1) nounwind {
 ;
 ; X86-LABEL: Test128_1Sub:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    subl $40, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
@@ -167,16 +142,9 @@ define dso_local void @Test128_1Sub(fp128 %d1) nounwind {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll __subtf3
 ; X86-NEXT:    addl $44, %esp
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    movl %esi, vf128+8
-; X86-NEXT:    movl %edx, vf128+12
-; X86-NEXT:    movl %eax, vf128
-; X86-NEXT:    movl %ecx, vf128+4
-; X86-NEXT:    addl $24, %esp
-; X86-NEXT:    popl %esi
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, vf128
+; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    retl
 entry:
   %0 = load fp128, ptr @vf128, align 16
@@ -196,8 +164,7 @@ define dso_local void @Test128Mul(fp128 %d1, fp128 %d2) nounwind {
 ;
 ; X86-LABEL: Test128Mul:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    subl $40, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
@@ -210,16 +177,9 @@ define dso_local void @Test128Mul(fp128 %d1, fp128 %d2) nounwind {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll __multf3
 ; X86-NEXT:    addl $44, %esp
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    movl %esi, vf128+12
-; X86-NEXT:    movl %edx, vf128+8
-; X86-NEXT:    movl %ecx, vf128+4
-; X86-NEXT:    movl %eax, vf128
-; X86-NEXT:    addl $24, %esp
-; X86-NEXT:    popl %esi
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, vf128
+; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    retl
 entry:
   %mul = fmul fp128 %d1, %d2
@@ -240,8 +200,7 @@ define dso_local void @Test128_1Mul(fp128 %d1) nounwind {
 ;
 ; X86-LABEL: Test128_1Mul:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    subl $40, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
@@ -254,16 +213,9 @@ define dso_local void @Test128_1Mul(fp128 %d1) nounwind {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll __multf3
 ; X86-NEXT:    addl $44, %esp
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    movl %esi, vf128+8
-; X86-NEXT:    movl %edx, vf128+12
-; X86-NEXT:    movl %eax, vf128
-; X86-NEXT:    movl %ecx, vf128+4
-; X86-NEXT:    addl $24, %esp
-; X86-NEXT:    popl %esi
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, vf128
+; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    retl
 entry:
   %0 = load fp128, ptr @vf128, align 16
@@ -283,8 +235,7 @@ define dso_local void @Test128Div(fp128 %d1, fp128 %d2) nounwind {
 ;
 ; X86-LABEL: Test128Div:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    subl $40, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
@@ -297,16 +248,9 @@ define dso_local void @Test128Div(fp128 %d1, fp128 %d2) nounwind {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll __divtf3
 ; X86-NEXT:    addl $44, %esp
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    movl %esi, vf128+12
-; X86-NEXT:    movl %edx, vf128+8
-; X86-NEXT:    movl %ecx, vf128+4
-; X86-NEXT:    movl %eax, vf128
-; X86-NEXT:    addl $24, %esp
-; X86-NEXT:    popl %esi
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, vf128
+; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    retl
 entry:
   %div = fdiv fp128 %d1, %d2
@@ -327,8 +271,7 @@ define dso_local void @Test128_1Div(fp128 %d1) nounwind {
 ;
 ; X86-LABEL: Test128_1Div:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    subl $40, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
@@ -341,16 +284,9 @@ define dso_local void @Test128_1Div(fp128 %d1) nounwind {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll __divtf3
 ; X86-NEXT:    addl $44, %esp
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    movl %esi, vf128+8
-; X86-NEXT:    movl %edx, vf128+12
-; X86-NEXT:    movl %eax, vf128
-; X86-NEXT:    movl %ecx, vf128+4
-; X86-NEXT:    addl $24, %esp
-; X86-NEXT:    popl %esi
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, vf128
+; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    retl
 entry:
   %0 = load fp128, ptr @vf128, align 16
@@ -370,8 +306,7 @@ define dso_local void @Test128Rem(fp128 %d1, fp128 %d2) nounwind {
 ;
 ; X86-LABEL: Test128Rem:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    subl $40, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
@@ -384,16 +319,9 @@ define dso_local void @Test128Rem(fp128 %d1, fp128 %d2) nounwind {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll fmodl
 ; X86-NEXT:    addl $44, %esp
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    movl %esi, vf128+12
-; X86-NEXT:    movl %edx, vf128+8
-; X86-NEXT:    movl %ecx, vf128+4
-; X86-NEXT:    movl %eax, vf128
-; X86-NEXT:    addl $24, %esp
-; X86-NEXT:    popl %esi
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, vf128
+; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    retl
 entry:
   %div = frem fp128 %d1, %d2
@@ -414,8 +342,7 @@ define dso_local void @Test128_1Rem(fp128 %d1) nounwind {
 ;
 ; X86-LABEL: Test128_1Rem:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    subl $40, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
@@ -428,16 +355,9 @@ define dso_local void @Test128_1Rem(fp128 %d1) nounwind {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll fmodl
 ; X86-NEXT:    addl $44, %esp
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    movl %esi, vf128+8
-; X86-NEXT:    movl %edx, vf128+12
-; X86-NEXT:    movl %eax, vf128
-; X86-NEXT:    movl %ecx, vf128+4
-; X86-NEXT:    addl $24, %esp
-; X86-NEXT:    popl %esi
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, vf128
+; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    retl
 entry:
   %0 = load fp128, ptr @vf128, align 16
@@ -457,8 +377,7 @@ define dso_local void @Test128Sqrt(fp128 %d1) nounwind {
 ;
 ; X86-LABEL: Test128Sqrt:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    subl $40, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
@@ -467,16 +386,9 @@ define dso_local void @Test128Sqrt(fp128 %d1) nounwind {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll sqrtl
 ; X86-NEXT:    addl $28, %esp
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    movl %esi, vf128+12
-; X86-NEXT:    movl %edx, vf128+8
-; X86-NEXT:    movl %ecx, vf128+4
-; X86-NEXT:    movl %eax, vf128
-; X86-NEXT:    addl $24, %esp
-; X86-NEXT:    popl %esi
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, vf128
+; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    retl
 entry:
   %sqrt = call fp128 @llvm.sqrt.f128(fp128 %d1)
@@ -496,8 +408,7 @@ define dso_local void @Test128Sin(fp128 %d1) nounwind {
 ;
 ; X86-LABEL: Test128Sin:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    subl $40, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
@@ -506,16 +417,9 @@ define dso_local void @Test128Sin(fp128 %d1) nounwind {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll sinl
 ; X86-NEXT:    addl $28, %esp
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    movl %esi, vf128+12
-; X86-NEXT:    movl %edx, vf128+8
-; X86-NEXT:    movl %ecx, vf128+4
-; X86-NEXT:    movl %eax, vf128
-; X86-NEXT:    addl $24, %esp
-; X86-NEXT:    popl %esi
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, vf128
+; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    retl
 entry:
   %sqrt = call fp128 @llvm.sin.f128(fp128 %d1)
@@ -535,8 +439,7 @@ define dso_local void @Test128Cos(fp128 %d1) nounwind {
 ;
 ; X86-LABEL: Test128Cos:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    subl $40, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
@@ -545,16 +448,9 @@ define dso_local void @Test128Cos(fp128 %d1) nounwind {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll cosl
 ; X86-NEXT:    addl $28, %esp
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    movl %esi, vf128+12
-; X86-NEXT:    movl %edx, vf128+8
-; X86-NEXT:    movl %ecx, vf128+4
-; X86-NEXT:    movl %eax, vf128
-; X86-NEXT:    addl $24, %esp
-; X86-NEXT:    popl %esi
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, vf128
+; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    retl
 entry:
   %sqrt = call fp128 @llvm.cos.f128(fp128 %d1)
@@ -574,8 +470,7 @@ define dso_local void @Test128Ceil(fp128 %d1) nounwind {
 ;
 ; X86-LABEL: Test128Ceil:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    subl $40, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
@@ -584,16 +479,9 @@ define dso_local void @Test128Ceil(fp128 %d1) nounwind {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll ceill
 ; X86-NEXT:    addl $28, %esp
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    movl %esi, vf128+12
-; X86-NEXT:    movl %edx, vf128+8
-; X86-NEXT:    movl %ecx, vf128+4
-; X86-NEXT:    movl %eax, vf128
-; X86-NEXT:    addl $24, %esp
-; X86-NEXT:    popl %esi
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, vf128
+; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    retl
 entry:
   %sqrt = call fp128 @llvm.ceil.f128(fp128 %d1)
@@ -613,8 +501,7 @@ define dso_local void @Test128Floor(fp128 %d1) nounwind {
 ;
 ; X86-LABEL: Test128Floor:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    subl $40, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
@@ -623,16 +510,9 @@ define dso_local void @Test128Floor(fp128 %d1) nounwind {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll floorl
 ; X86-NEXT:    addl $28, %esp
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    movl %esi, vf128+12
-; X86-NEXT:    movl %edx, vf128+8
-; X86-NEXT:    movl %ecx, vf128+4
-; X86-NEXT:    movl %eax, vf128
-; X86-NEXT:    addl $24, %esp
-; X86-NEXT:    popl %esi
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, vf128
+; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    retl
 entry:
   %sqrt = call fp128 @llvm.floor.f128(fp128 %d1)
@@ -652,8 +532,7 @@ define dso_local void @Test128Trunc(fp128 %d1) nounwind {
 ;
 ; X86-LABEL: Test128Trunc:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    subl $40, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
@@ -662,16 +541,9 @@ define dso_local void @Test128Trunc(fp128 %d1) nounwind {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll truncl
 ; X86-NEXT:    addl $28, %esp
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    movl %esi, vf128+12
-; X86-NEXT:    movl %edx, vf128+8
-; X86-NEXT:    movl %ecx, vf128+4
-; X86-NEXT:    movl %eax, vf128
-; X86-NEXT:    addl $24, %esp
-; X86-NEXT:    popl %esi
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, vf128
+; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    retl
 entry:
   %sqrt = call fp128 @llvm.trunc.f128(fp128 %d1)
@@ -691,8 +563,7 @@ define dso_local void @Test128Nearbyint(fp128 %d1) nounwind {
 ;
 ; X86-LABEL: Test128Nearbyint:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    subl $40, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
@@ -701,16 +572,9 @@ define dso_local void @Test128Nearbyint(fp128 %d1) nounwind {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll nearbyintl
 ; X86-NEXT:    addl $28, %esp
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    movl %esi, vf128+12
-; X86-NEXT:    movl %edx, vf128+8
-; X86-NEXT:    movl %ecx, vf128+4
-; X86-NEXT:    movl %eax, vf128
-; X86-NEXT:    addl $24, %esp
-; X86-NEXT:    popl %esi
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, vf128
+; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    retl
 entry:
   %sqrt = call fp128 @llvm.nearbyint.f128(fp128 %d1)
@@ -730,8 +594,7 @@ define dso_local void @Test128Rint(fp128 %d1) nounwind {
 ;
 ; X86-LABEL: Test128Rint:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    subl $40, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
@@ -740,16 +603,9 @@ define dso_local void @Test128Rint(fp128 %d1) nounwind {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll rintl
 ; X86-NEXT:    addl $28, %esp
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    movl %esi, vf128+12
-; X86-NEXT:    movl %edx, vf128+8
-; X86-NEXT:    movl %ecx, vf128+4
-; X86-NEXT:    movl %eax, vf128
-; X86-NEXT:    addl $24, %esp
-; X86-NEXT:    popl %esi
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, vf128
+; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    retl
 entry:
   %sqrt = call fp128 @llvm.rint.f128(fp128 %d1)
@@ -769,8 +625,7 @@ define dso_local void @Test128Round(fp128 %d1) nounwind {
 ;
 ; X86-LABEL: Test128Round:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $36, %esp
+; X86-NEXT:    subl $40, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
@@ -779,16 +634,9 @@ define dso_local void @Test128Round(fp128 %d1) nounwind {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll roundl
 ; X86-NEXT:    addl $28, %esp
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    movl %esi, vf128+12
-; X86-NEXT:    movl %edx, vf128+8
-; X86-NEXT:    movl %ecx, vf128+4
-; X86-NEXT:    movl %eax, vf128
-; X86-NEXT:    addl $24, %esp
-; X86-NEXT:    popl %esi
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, vf128
+; X86-NEXT:    addl $28, %esp
 ; X86-NEXT:    retl
 entry:
   %sqrt = call fp128 @llvm.round.f128(fp128 %d1)
@@ -804,9 +652,8 @@ define fp128 @Test128FMA(fp128 %a, fp128 %b, fp128 %c) nounwind {
 ;
 ; X86-LABEL: Test128FMA:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    subl $20, %esp
+; X86-NEXT:    subl $24, %esp
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    subl $12, %esp
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
@@ -825,18 +672,11 @@ define fp128 @Test128FMA(fp128 %a, fp128 %b, fp128 %c) nounwind {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    calll fmal
 ; X86-NEXT:    addl $60, %esp
-; X86-NEXT:    movl (%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT:    movl %edi, 12(%esi)
-; X86-NEXT:    movl %edx, 8(%esi)
-; X86-NEXT:    movl %ecx, 4(%esi)
-; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movaps (%esp), %xmm0
+; X86-NEXT:    movaps %xmm0, (%esi)
 ; X86-NEXT:    movl %esi, %eax
-; X86-NEXT:    addl $20, %esp
+; X86-NEXT:    addl $24, %esp
 ; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
 ; X86-NEXT:    retl $4
 entry:
   %call = call fp128 @llvm.fma.f128(fp128 %a, fp128 %b, fp128 %c)
diff --git a/llvm/test/CodeGen/X86/fpenv-combine.ll b/llvm/test/CodeGen/X86/fpenv-combine.ll
index 568369a061..d1be4a5f44 100644
--- a/llvm/test/CodeGen/X86/fpenv-combine.ll
+++ b/llvm/test/CodeGen/X86/fpenv-combine.ll
@@ -15,19 +15,19 @@ define void @get_fpenv_02(ptr %ptr1, ptr %ptr2) #0 {
 ; X64-NEXT:    subq $40, %rsp
 ; X64-NEXT:    movq %rsi, %rbx
 ; X64-NEXT:    movq %rdi, %r14
-; X64-NEXT:    leaq {{[0-9]+}}(%rsp), %rdi
+; X64-NEXT:    movq %rsp, %rdi
 ; X64-NEXT:    callq fegetenv@PLT
 ; X64-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; X64-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
+; X64-NEXT:    movq (%rsp), %rcx
 ; X64-NEXT:    movq {{[0-9]+}}(%rsp), %rdx
 ; X64-NEXT:    movq {{[0-9]+}}(%rsp), %rsi
-; X64-NEXT:    movq %rsi, 24(%r14)
+; X64-NEXT:    movq %rsi, 16(%r14)
 ; X64-NEXT:    movq %rcx, (%r14)
+; X64-NEXT:    movq %rax, 24(%r14)
 ; X64-NEXT:    movq %rdx, 8(%r14)
-; X64-NEXT:    movq %rax, 16(%r14)
-; X64-NEXT:    movq %rax, 16(%rbx)
-; X64-NEXT:    movq %rsi, 24(%rbx)
+; X64-NEXT:    movq %rsi, 16(%rbx)
 ; X64-NEXT:    movq %rcx, (%rbx)
+; X64-NEXT:    movq %rax, 24(%rbx)
 ; X64-NEXT:    movq %rdx, 8(%rbx)
 ; X64-NEXT:    addq $40, %rsp
 ; X64-NEXT:    popq %rbx
@@ -72,9 +72,9 @@ define void @get_fpenv_04(ptr %ptr) #0 {
 ; X64-NEXT:    movq (%rsp), %rax
 ; X64-NEXT:    andl $1, %eax
 ; X64-NEXT:    movq %rax, (%rbx)
-; X64-NEXT:    movq $0, 16(%rbx)
 ; X64-NEXT:    movq $0, 24(%rbx)
 ; X64-NEXT:    movq $0, 8(%rbx)
+; X64-NEXT:    movq $0, 16(%rbx)
 ; X64-NEXT:    addq $32, %rsp
 ; X64-NEXT:    popq %rbx
 ; X64-NEXT:    retq
@@ -94,9 +94,9 @@ define void @get_fpenv_05(ptr %ptr1, ptr %ptr2) #0 {
 ; X64-NEXT:    subq $40, %rsp
 ; X64-NEXT:    movq %rsi, %rbx
 ; X64-NEXT:    movq %rdi, %r14
-; X64-NEXT:    leaq {{[0-9]+}}(%rsp), %rdi
+; X64-NEXT:    movq %rsp, %rdi
 ; X64-NEXT:    callq fegetenv@PLT
-; X64-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; X64-NEXT:    movq (%rsp), %rax
 ; X64-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
 ; X64-NEXT:    movq {{[0-9]+}}(%rsp), %rdx
 ; X64-NEXT:    movq {{[0-9]+}}(%rsp), %rsi
@@ -129,8 +129,8 @@ define void @set_fpenv_02(ptr %ptr1, ptr %ptr2) #0 {
 ; X64-NEXT:    movq %rdi, {{[0-9]+}}(%rsp)
 ; X64-NEXT:    movq %rdx, {{[0-9]+}}(%rsp)
 ; X64-NEXT:    movq %rcx, {{[0-9]+}}(%rsp)
-; X64-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
-; X64-NEXT:    leaq {{[0-9]+}}(%rsp), %rdi
+; X64-NEXT:    movq %rax, (%rsp)
+; X64-NEXT:    movq %rsp, %rdi
 ; X64-NEXT:    callq fesetenv@PLT
 ; X64-NEXT:    addq $40, %rsp
 ; X64-NEXT:    retq
@@ -182,11 +182,11 @@ define void @set_fpenv_04(ptr %ptr) #0 {
 ; X64-NEXT:    subq $40, %rsp
 ; X64-NEXT:    movq (%rdi), %rax
 ; X64-NEXT:    andl $1, %eax
-; X64-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
+; X64-NEXT:    movq %rax, (%rsp)
 ; X64-NEXT:    movq $0, {{[0-9]+}}(%rsp)
 ; X64-NEXT:    movq $0, {{[0-9]+}}(%rsp)
 ; X64-NEXT:    movq $0, {{[0-9]+}}(%rsp)
-; X64-NEXT:    leaq {{[0-9]+}}(%rsp), %rdi
+; X64-NEXT:    movq %rsp, %rdi
 ; X64-NEXT:    callq fesetenv@PLT
 ; X64-NEXT:    addq $40, %rsp
 ; X64-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/fpenv.ll b/llvm/test/CodeGen/X86/fpenv.ll
index 1f01af92ca..c79e19f07c 100644
--- a/llvm/test/CodeGen/X86/fpenv.ll
+++ b/llvm/test/CodeGen/X86/fpenv.ll
@@ -252,20 +252,20 @@ define void @func_05(i32 %x) nounwind {
 define void @get_fpenv_01(ptr %ptr) #0 {
 ; X86-NOSSE-LABEL: get_fpenv_01:
 ; X86-NOSSE:       # %bb.0: # %entry
-; X86-NOSSE-NEXT:    subl $44, %esp
+; X86-NOSSE-NEXT:    subl $60, %esp
 ; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NOSSE-NEXT:    movl %eax, (%esp)
 ; X86-NOSSE-NEXT:    calll fegetenv
-; X86-NOSSE-NEXT:    addl $44, %esp
+; X86-NOSSE-NEXT:    addl $60, %esp
 ; X86-NOSSE-NEXT:    retl
 ;
 ; X86-SSE-LABEL: get_fpenv_01:
 ; X86-SSE:       # %bb.0: # %entry
-; X86-SSE-NEXT:    subl $44, %esp
+; X86-SSE-NEXT:    subl $60, %esp
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE-NEXT:    movl %eax, (%esp)
 ; X86-SSE-NEXT:    calll fegetenv
-; X86-SSE-NEXT:    addl $44, %esp
+; X86-SSE-NEXT:    addl $60, %esp
 ; X86-SSE-NEXT:    retl
 ;
 ; X64-LABEL: get_fpenv_01:
@@ -283,21 +283,21 @@ entry:
 define void @get_fpenv_01_native(ptr %ptr) nounwind {
 ; X86-NOSSE-LABEL: get_fpenv_01_native:
 ; X86-NOSSE:       # %bb.0: # %entry
-; X86-NOSSE-NEXT:    subl $36, %esp
+; X86-NOSSE-NEXT:    subl $44, %esp
 ; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NOSSE-NEXT:    fnstenv (%eax)
 ; X86-NOSSE-NEXT:    fldenv (%eax)
-; X86-NOSSE-NEXT:    addl $36, %esp
+; X86-NOSSE-NEXT:    addl $44, %esp
 ; X86-NOSSE-NEXT:    retl
 ;
 ; X86-SSE-LABEL: get_fpenv_01_native:
 ; X86-SSE:       # %bb.0: # %entry
-; X86-SSE-NEXT:    subl $36, %esp
+; X86-SSE-NEXT:    subl $44, %esp
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE-NEXT:    fnstenv (%eax)
 ; X86-SSE-NEXT:    fldenv (%eax)
 ; X86-SSE-NEXT:    stmxcsr 28(%eax)
-; X86-SSE-NEXT:    addl $36, %esp
+; X86-SSE-NEXT:    addl $44, %esp
 ; X86-SSE-NEXT:    retl
 ;
 ; X64-LABEL: get_fpenv_01_native:
@@ -315,20 +315,20 @@ entry:
 define void @set_fpenv_01(ptr %ptr) #0 {
 ; X86-NOSSE-LABEL: set_fpenv_01:
 ; X86-NOSSE:       # %bb.0: # %entry
-; X86-NOSSE-NEXT:    subl $44, %esp
+; X86-NOSSE-NEXT:    subl $60, %esp
 ; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NOSSE-NEXT:    movl %eax, (%esp)
 ; X86-NOSSE-NEXT:    calll fesetenv
-; X86-NOSSE-NEXT:    addl $44, %esp
+; X86-NOSSE-NEXT:    addl $60, %esp
 ; X86-NOSSE-NEXT:    retl
 ;
 ; X86-SSE-LABEL: set_fpenv_01:
 ; X86-SSE:       # %bb.0: # %entry
-; X86-SSE-NEXT:    subl $44, %esp
+; X86-SSE-NEXT:    subl $60, %esp
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE-NEXT:    movl %eax, (%esp)
 ; X86-SSE-NEXT:    calll fesetenv
-; X86-SSE-NEXT:    addl $44, %esp
+; X86-SSE-NEXT:    addl $60, %esp
 ; X86-SSE-NEXT:    retl
 ;
 ; X64-LABEL: set_fpenv_01:
@@ -346,19 +346,19 @@ entry:
 define void @set_fpenv_01_native(ptr %ptr) nounwind {
 ; X86-NOSSE-LABEL: set_fpenv_01_native:
 ; X86-NOSSE:       # %bb.0: # %entry
-; X86-NOSSE-NEXT:    subl $36, %esp
+; X86-NOSSE-NEXT:    subl $44, %esp
 ; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NOSSE-NEXT:    fldenv (%eax)
-; X86-NOSSE-NEXT:    addl $36, %esp
+; X86-NOSSE-NEXT:    addl $44, %esp
 ; X86-NOSSE-NEXT:    retl
 ;
 ; X86-SSE-LABEL: set_fpenv_01_native:
 ; X86-SSE:       # %bb.0: # %entry
-; X86-SSE-NEXT:    subl $36, %esp
+; X86-SSE-NEXT:    subl $44, %esp
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE-NEXT:    fldenv (%eax)
 ; X86-SSE-NEXT:    ldmxcsr 28(%eax)
-; X86-SSE-NEXT:    addl $36, %esp
+; X86-SSE-NEXT:    addl $44, %esp
 ; X86-SSE-NEXT:    retl
 ;
 ; X64-LABEL: set_fpenv_01_native:
diff --git a/llvm/test/CodeGen/X86/fptosi-sat-scalar.ll b/llvm/test/CodeGen/X86/fptosi-sat-scalar.ll
index 76e15ca0bf..2eb351c8fa 100644
--- a/llvm/test/CodeGen/X86/fptosi-sat-scalar.ll
+++ b/llvm/test/CodeGen/X86/fptosi-sat-scalar.ll
@@ -690,7 +690,7 @@ define i100 @test_signed_i100_f32(float %f) nounwind {
 ; X86-X87-NEXT:    pushl %ebx
 ; X86-X87-NEXT:    pushl %edi
 ; X86-X87-NEXT:    pushl %esi
-; X86-X87-NEXT:    subl $44, %esp
+; X86-X87-NEXT:    subl $60, %esp
 ; X86-X87-NEXT:    flds {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fsts {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
@@ -766,7 +766,7 @@ define i100 @test_signed_i100_f32(float %f) nounwind {
 ; X86-X87-NEXT:    andl $15, %edx
 ; X86-X87-NEXT:    movb %dl, 12(%ecx)
 ; X86-X87-NEXT:    movl %ecx, %eax
-; X86-X87-NEXT:    addl $44, %esp
+; X86-X87-NEXT:    addl $60, %esp
 ; X86-X87-NEXT:    popl %esi
 ; X86-X87-NEXT:    popl %edi
 ; X86-X87-NEXT:    popl %ebx
@@ -779,7 +779,7 @@ define i100 @test_signed_i100_f32(float %f) nounwind {
 ; X86-SSE-NEXT:    pushl %ebx
 ; X86-SSE-NEXT:    pushl %edi
 ; X86-SSE-NEXT:    pushl %esi
-; X86-SSE-NEXT:    subl $28, %esp
+; X86-SSE-NEXT:    subl $44, %esp
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X86-SSE-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
@@ -819,7 +819,7 @@ define i100 @test_signed_i100_f32(float %f) nounwind {
 ; X86-SSE-NEXT:    andl $15, %eax
 ; X86-SSE-NEXT:    movb %al, 12(%esi)
 ; X86-SSE-NEXT:    movl %esi, %eax
-; X86-SSE-NEXT:    addl $28, %esp
+; X86-SSE-NEXT:    addl $44, %esp
 ; X86-SSE-NEXT:    popl %esi
 ; X86-SSE-NEXT:    popl %edi
 ; X86-SSE-NEXT:    popl %ebx
@@ -859,7 +859,7 @@ define i128 @test_signed_i128_f32(float %f) nounwind {
 ; X86-X87-NEXT:    pushl %ebx
 ; X86-X87-NEXT:    pushl %edi
 ; X86-X87-NEXT:    pushl %esi
-; X86-X87-NEXT:    subl $44, %esp
+; X86-X87-NEXT:    subl $60, %esp
 ; X86-X87-NEXT:    flds {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fsts {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
@@ -931,7 +931,7 @@ define i128 @test_signed_i128_f32(float %f) nounwind {
 ; X86-X87-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X86-X87-NEXT:    movl %eax, (%ecx)
 ; X86-X87-NEXT:    movl %ecx, %eax
-; X86-X87-NEXT:    addl $44, %esp
+; X86-X87-NEXT:    addl $60, %esp
 ; X86-X87-NEXT:    popl %esi
 ; X86-X87-NEXT:    popl %edi
 ; X86-X87-NEXT:    popl %ebx
@@ -954,7 +954,7 @@ define i128 @test_signed_i128_f32(float %f) nounwind {
 ; X86-SSE-NEXT:    pushl %ebx
 ; X86-SSE-NEXT:    pushl %edi
 ; X86-SSE-NEXT:    pushl %esi
-; X86-SSE-NEXT:    subl $28, %esp
+; X86-SSE-NEXT:    subl $44, %esp
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X86-SSE-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
@@ -990,7 +990,7 @@ define i128 @test_signed_i128_f32(float %f) nounwind {
 ; X86-SSE-NEXT:    movl %edx, 4(%esi)
 ; X86-SSE-NEXT:    movl %eax, (%esi)
 ; X86-SSE-NEXT:    movl %esi, %eax
-; X86-SSE-NEXT:    addl $28, %esp
+; X86-SSE-NEXT:    addl $44, %esp
 ; X86-SSE-NEXT:    popl %esi
 ; X86-SSE-NEXT:    popl %edi
 ; X86-SSE-NEXT:    popl %ebx
@@ -2882,7 +2882,7 @@ define i100 @test_signed_i100_f16(half %f) nounwind {
 ; X86-X87-NEXT:    pushl %ebx
 ; X86-X87-NEXT:    pushl %edi
 ; X86-X87-NEXT:    pushl %esi
-; X86-X87-NEXT:    subl $44, %esp
+; X86-X87-NEXT:    subl $60, %esp
 ; X86-X87-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; X86-X87-NEXT:    movl %eax, (%esp)
 ; X86-X87-NEXT:    calll __gnu_h2f_ieee
@@ -2960,7 +2960,7 @@ define i100 @test_signed_i100_f16(half %f) nounwind {
 ; X86-X87-NEXT:    andl $15, %edx
 ; X86-X87-NEXT:    movb %dl, 12(%ecx)
 ; X86-X87-NEXT:    movl %ecx, %eax
-; X86-X87-NEXT:    addl $44, %esp
+; X86-X87-NEXT:    addl $60, %esp
 ; X86-X87-NEXT:    popl %esi
 ; X86-X87-NEXT:    popl %edi
 ; X86-X87-NEXT:    popl %ebx
@@ -3061,7 +3061,7 @@ define i128 @test_signed_i128_f16(half %f) nounwind {
 ; X86-X87-NEXT:    pushl %ebx
 ; X86-X87-NEXT:    pushl %edi
 ; X86-X87-NEXT:    pushl %esi
-; X86-X87-NEXT:    subl $44, %esp
+; X86-X87-NEXT:    subl $60, %esp
 ; X86-X87-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; X86-X87-NEXT:    movl %eax, (%esp)
 ; X86-X87-NEXT:    calll __gnu_h2f_ieee
@@ -3135,7 +3135,7 @@ define i128 @test_signed_i128_f16(half %f) nounwind {
 ; X86-X87-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X86-X87-NEXT:    movl %eax, (%ecx)
 ; X86-X87-NEXT:    movl %ecx, %eax
-; X86-X87-NEXT:    addl $44, %esp
+; X86-X87-NEXT:    addl $60, %esp
 ; X86-X87-NEXT:    popl %esi
 ; X86-X87-NEXT:    popl %edi
 ; X86-X87-NEXT:    popl %ebx
diff --git a/llvm/test/CodeGen/X86/fptoui-sat-scalar.ll b/llvm/test/CodeGen/X86/fptoui-sat-scalar.ll
index 01426b1ac9..e334af7139 100644
--- a/llvm/test/CodeGen/X86/fptoui-sat-scalar.ll
+++ b/llvm/test/CodeGen/X86/fptoui-sat-scalar.ll
@@ -807,7 +807,7 @@ define i128 @test_unsigned_i128_f32(float %f) nounwind {
 ; X86-X87-NEXT:    pushl %ebx
 ; X86-X87-NEXT:    pushl %edi
 ; X86-X87-NEXT:    pushl %esi
-; X86-X87-NEXT:    subl $44, %esp
+; X86-X87-NEXT:    subl $60, %esp
 ; X86-X87-NEXT:    flds {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    fsts {{[0-9]+}}(%esp)
 ; X86-X87-NEXT:    leal {{[0-9]+}}(%esp), %eax
@@ -864,7 +864,7 @@ define i128 @test_unsigned_i128_f32(float %f) nounwind {
 ; X86-X87-NEXT:    movl %ebp, 4(%ecx)
 ; X86-X87-NEXT:    movl %eax, (%ecx)
 ; X86-X87-NEXT:    movl %ecx, %eax
-; X86-X87-NEXT:    addl $44, %esp
+; X86-X87-NEXT:    addl $60, %esp
 ; X86-X87-NEXT:    popl %esi
 ; X86-X87-NEXT:    popl %edi
 ; X86-X87-NEXT:    popl %ebx
@@ -2818,7 +2818,7 @@ define i128 @test_unsigned_i128_f16(half %f) nounwind {
 ; X86-X87-NEXT:    pushl %ebx
 ; X86-X87-NEXT:    pushl %edi
 ; X86-X87-NEXT:    pushl %esi
-; X86-X87-NEXT:    subl $44, %esp
+; X86-X87-NEXT:    subl $60, %esp
 ; X86-X87-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; X86-X87-NEXT:    movl %eax, (%esp)
 ; X86-X87-NEXT:    calll __gnu_h2f_ieee
@@ -2877,7 +2877,7 @@ define i128 @test_unsigned_i128_f16(half %f) nounwind {
 ; X86-X87-NEXT:    movl %ebp, 4(%ecx)
 ; X86-X87-NEXT:    movl %eax, (%ecx)
 ; X86-X87-NEXT:    movl %ecx, %eax
-; X86-X87-NEXT:    addl $44, %esp
+; X86-X87-NEXT:    addl $60, %esp
 ; X86-X87-NEXT:    popl %esi
 ; X86-X87-NEXT:    popl %edi
 ; X86-X87-NEXT:    popl %ebx
diff --git a/llvm/test/CodeGen/X86/implicit-null-check.ll b/llvm/test/CodeGen/X86/implicit-null-check.ll
index b02f66fe8c..fc81f703f5 100644
--- a/llvm/test/CodeGen/X86/implicit-null-check.ll
+++ b/llvm/test/CodeGen/X86/implicit-null-check.ll
@@ -128,19 +128,15 @@ define i256 @imp_null_check_load_i256(ptr %x) {
 ; CHECK:       ## %bb.0: ## %entry
 ; CHECK-NEXT:    movq %rdi, %rax
 ; CHECK-NEXT:  Ltmp3:
-; CHECK-NEXT:    movq (%rsi), %rcx ## on-fault: LBB5_1
+; CHECK-NEXT:    movaps (%rsi), %xmm0 ## on-fault: LBB5_1
 ; CHECK-NEXT:  ## %bb.2: ## %not_null
-; CHECK-NEXT:    movq 8(%rsi), %rdx
-; CHECK-NEXT:    movq 16(%rsi), %rdi
-; CHECK-NEXT:    movq 24(%rsi), %rsi
-; CHECK-NEXT:    movq %rsi, 24(%rax)
-; CHECK-NEXT:    movq %rdi, 16(%rax)
-; CHECK-NEXT:    movq %rdx, 8(%rax)
-; CHECK-NEXT:    movq %rcx, (%rax)
+; CHECK-NEXT:    movaps 16(%rsi), %xmm1
+; CHECK-NEXT:    movaps %xmm1, 16(%rax)
+; CHECK-NEXT:    movaps %xmm0, (%rax)
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:  LBB5_1: ## %is_null
-; CHECK-NEXT:    movq $0, 24(%rax)
-; CHECK-NEXT:    movq $0, 16(%rax)
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    movaps %xmm0, 16(%rax)
 ; CHECK-NEXT:    movq $0, 8(%rax)
 ; CHECK-NEXT:    movq $42, (%rax)
 ; CHECK-NEXT:    retq
@@ -622,7 +618,8 @@ define i64 @imp_null_check_load_shift_by_3_addr(ptr %x) {
 define i64 @imp_null_check_load_shift_add_addr(ptr %x) {
 ; CHECK-LABEL: imp_null_check_load_shift_add_addr:
 ; CHECK:       ## %bb.0: ## %entry
-; CHECK:         movq 3526(,%rdi,8), %rax ## on-fault: LBB23_1
+; CHECK-NEXT:  Ltmp19:
+; CHECK-NEXT:    movq 3526(,%rdi,8), %rax ## on-fault: LBB23_1
 ; CHECK-NEXT:  ## %bb.2: ## %not_null
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:  LBB23_1: ## %is_null
diff --git a/llvm/test/CodeGen/X86/indirect-br-gep-unmerge.ll b/llvm/test/CodeGen/X86/indirect-br-gep-unmerge.ll
new file mode 100644
index 0000000000..6b953e3004
--- /dev/null
+++ b/llvm/test/CodeGen/X86/indirect-br-gep-unmerge.ll
@@ -0,0 +1,51 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
+; RUN: opt -S -codegenprepare %s -o - | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.Blub = type { i8, i8, ptr }
+
+@indirectBrPtr = external hidden global ptr
+
+define ptr @testFunc(ptr readonly %array, i1 %skip) {
+; CHECK-LABEL: define ptr @testFunc(
+; CHECK-SAME: ptr readonly [[ARRAY:%.*]], i1 [[SKIP:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[SKIP]], label [[LOOPHEADER:%.*]], label [[ENDBLOCK_CLONE:%.*]]
+; CHECK:       loopHeader:
+; CHECK-NEXT:    [[CURRENTARRAYELEMENT:%.*]] = phi ptr [ [[ARRAY]], [[ENTRY:%.*]] ], [ [[NEXTARRAYELEMENT:%.*]], [[LOOPFOOTER:%.*]] ]
+; CHECK-NEXT:    [[ELEMENTVALUEPTR:%.*]] = getelementptr inbounds i8, ptr [[CURRENTARRAYELEMENT]], i64 1
+; CHECK-NEXT:    [[ELEMENTVALUE:%.*]] = load i8, ptr [[ELEMENTVALUEPTR]], align 1
+; CHECK-NEXT:    indirectbr ptr @indirectBrPtr, [label [[LOOPFOOTER]], label %endBlock]
+; CHECK:       loopFooter:
+; CHECK-NEXT:    [[ISGOODVALUE:%.*]] = icmp eq i8 [[ELEMENTVALUE]], 0
+; CHECK-NEXT:    [[NEXTARRAYELEMENT]] = getelementptr inbounds [[STRUCT_BLUB:%.*]], ptr [[CURRENTARRAYELEMENT]], i64 1
+; CHECK-NEXT:    br i1 [[ISGOODVALUE]], label [[LOOPHEADER]], label [[ENDBLOCK_CLONE]]
+; CHECK:       endBlock:
+; CHECK-NEXT:    br label [[DOTSPLIT:%.*]]
+; CHECK:       .split:
+; CHECK-NEXT:    [[MERGE:%.*]] = phi ptr [ [[ELEMENTVALUEPTR]], [[ENDBLOCK:%.*]] ], [ [[RETVAL_CLONE:%.*]], [[ENDBLOCK_CLONE]] ]
+; CHECK-NEXT:    ret ptr [[MERGE]]
+; CHECK:       endBlock.clone:
+; CHECK-NEXT:    [[RETVAL_CLONE]] = phi ptr [ [[ARRAY]], [[ENTRY]] ], [ [[ELEMENTVALUEPTR]], [[LOOPFOOTER]] ]
+; CHECK-NEXT:    br label [[DOTSPLIT]]
+;
+entry:
+  br i1 %skip, label %loopHeader, label %endBlock
+
+loopHeader:
+  %currentArrayElement = phi ptr [ %array, %entry ], [ %nextArrayElement, %loopFooter ]
+  %elementValuePtr = getelementptr inbounds i8, ptr %currentArrayElement, i64 1
+  %elementValue = load i8, ptr %elementValuePtr, align 1
+  indirectbr ptr @indirectBrPtr, [label %loopFooter, label %endBlock]
+
+loopFooter:
+  %isGoodValue = icmp eq i8 %elementValue, 0
+  %nextArrayElement = getelementptr inbounds %struct.Blub, ptr %currentArrayElement, i64 1
+  br i1 %isGoodValue, label %loopHeader, label %endBlock
+
+endBlock:
+  %retVal = phi ptr [ %array, %entry ], [ %elementValuePtr, %loopFooter ], [ %elementValuePtr, %loopHeader ]
+  ret ptr %retVal
+}
diff --git a/llvm/test/CodeGen/X86/inline-asm-avx512f-x-constraint.ll b/llvm/test/CodeGen/X86/inline-asm-avx512f-x-constraint.ll
index fcea55c47c..e153387d16 100644
--- a/llvm/test/CodeGen/X86/inline-asm-avx512f-x-constraint.ll
+++ b/llvm/test/CodeGen/X86/inline-asm-avx512f-x-constraint.ll
@@ -1,7 +1,7 @@
 ; RUN: not llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512f -stop-after=finalize-isel > %t 2> %t.err
 ; RUN: FileCheck < %t %s
 ; RUN: FileCheck --check-prefix=CHECK-STDERR < %t.err %s
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512fp16 -stop-after=finalize-isel | FileCheck --check-prefixes=CHECK,FP16 %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512bf16,avx512fp16 -stop-after=finalize-isel | FileCheck --check-prefixes=CHECK,FP16 %s
 
 ; CHECK-LABEL: name: mask_Yk_i8
 ; CHECK: %[[REG1:.*]]:vr512_0_15 = COPY %1
@@ -24,3 +24,14 @@ entry:
   %0 = tail call <32 x half> asm "vaddph\09$3, $2, $0 {$1}", "=x,^Yk,x,x,~{dirflag},~{fpsr},~{flags}"(i8 %msk, <32 x half> %x, <32 x half> %y)
   ret <32 x half> %0
 }
+
+; FP16-LABEL: name: mask_Yk_bf16
+; FP16: %[[REG1:.*]]:vr512_0_15 = COPY %1
+; FP16: %[[REG2:.*]]:vr512_0_15 = COPY %2
+; FP16: INLINEASM &"vaddph\09$3, $2, $0 {$1}", 0 /* attdialect */, {{.*}}, def %{{.*}}, {{.*}}, %{{.*}}, {{.*}}, %[[REG1]], {{.*}}, %[[REG2]], 12 /* clobber */, implicit-def early-clobber $df, 12 /* clobber */, implicit-def early-clobber $fpsw, 12 /* clobber */, implicit-def early-clobber $eflags
+; CHECK-STDERR: couldn't allocate output register for constraint 'x'
+define <32 x bfloat> @mask_Yk_bf16(i8 signext %msk, <32 x bfloat> %x, <32 x bfloat> %y) {
+entry:
+  %0 = tail call <32 x bfloat> asm "vaddph\09$3, $2, $0 {$1}", "=x,^Yk,x,x,~{dirflag},~{fpsr},~{flags}"(i8 %msk, <32 x bfloat> %x, <32 x bfloat> %y)
+  ret <32 x bfloat> %0
+}
diff --git a/llvm/test/CodeGen/X86/inline-spiller-impdef-on-implicit-def-regression.ll b/llvm/test/CodeGen/X86/inline-spiller-impdef-on-implicit-def-regression.ll
new file mode 100644
index 0000000000..8b8500ef72
--- /dev/null
+++ b/llvm/test/CodeGen/X86/inline-spiller-impdef-on-implicit-def-regression.ll
@@ -0,0 +1,178 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+; Make sure there's no assert on an implicit-def with implicit operands
+; during register allocation.
+
+%struct.BlockContext = type { [32 x i8], [32 x i8], [2 x [32 x i8]], [32 x i8], [32 x i8], [32 x i8], [32 x i8], [32 x i8], [2 x [32 x i8]], [2 x [32 x i8]], [32 x i8], [32 x i8], [32 x i8], [32 x i8], [16 x i8], [32 x i8], [32 x i8] }
+%struct.CdfModeContext = type { [4 x [16 x i16]], [2 x [13 x [16 x i16]]], [9 x [16 x i16]], [5 x [4 x [16 x i16]]], [6 x [16 x i16]], [2 x [16 x i16]], [16 x i16], [2 x [13 x [8 x i16]]], [3 x [13 x [8 x i16]]], [8 x i16], [8 x [8 x i16]], [8 x i16], [8 x [8 x i16]], [3 x [8 x i16]], [2 x [7 x [8 x i16]]], [2 x [7 x [5 x [8 x i16]]]], [2 x [8 x [4 x i16]]], [4 x [3 x [4 x i16]]], [22 x [4 x i16]], [4 x i16], [5 x [4 x i16]], [4 x [4 x i16]], [4 x i16], [2 x i16], [2 x i16], [7 x [2 x i16]], [7 x [2 x i16]], [4 x [2 x i16]], [22 x [2 x i16]], [6 x [2 x i16]], [2 x [2 x i16]], [6 x [2 x i16]], [3 x [2 x i16]], [4 x [2 x i16]], [5 x [2 x i16]], [5 x [2 x i16]], [6 x [2 x i16]], [6 x [2 x i16]], [9 x [2 x i16]], [6 x [3 x [2 x i16]]], [3 x [3 x [2 x i16]]], [2 x [3 x [2 x i16]]], [3 x [3 x [2 x i16]]], [7 x [3 x [2 x i16]]], [3 x [2 x i16]], [3 x [2 x i16]], [3 x [2 x i16]], [22 x [2 x i16]], [7 x [3 x [2 x i16]]], [2 x [2 x i16]], [2 x i16], [8 x i8] }
+
+define i32 @decode_sb(ptr %t, i32 %bl, i32 %_msprop1966, i32 %sub.i, i64 %idxprom, i1 %cmp54) #0 {
+; CHECK-LABEL: decode_sb:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rsp, %rbp
+; CHECK-NEXT:    .cfi_def_cfa_register %rbp
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    subq $24, %rsp
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    movl %r9d, %ebx
+; CHECK-NEXT:    movabsq $87960930222080, %r15 # imm = 0x500000000000
+; CHECK-NEXT:    movl 0, %r13d
+; CHECK-NEXT:    movl %esi, %r12d
+; CHECK-NEXT:    # implicit-def: $eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    testb $1, %bl
+; CHECK-NEXT:    jne .LBB0_7
+; CHECK-NEXT:  # %bb.1: # %if.else
+; CHECK-NEXT:    movq %r8, %r14
+; CHECK-NEXT:    movl %ecx, %eax
+; CHECK-NEXT:    andl $1, %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movzbl 544(%rax), %eax
+; CHECK-NEXT:    andl $1, %eax
+; CHECK-NEXT:    movl %r15d, %r9d
+; CHECK-NEXT:    andl $1, %r9d
+; CHECK-NEXT:    movl %r14d, %r10d
+; CHECK-NEXT:    andl $1, %r10d
+; CHECK-NEXT:    movl %esi, %r11d
+; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
+; CHECK-NEXT:    shrl %cl, %r11d
+; CHECK-NEXT:    movabsq $17592186044416, %r8 # imm = 0x100000000000
+; CHECK-NEXT:    orq %r10, %r8
+; CHECK-NEXT:    andl $2, %r11d
+; CHECK-NEXT:    testb $1, %bl
+; CHECK-NEXT:    cmoveq %r9, %r8
+; CHECK-NEXT:    movl %edx, %ecx
+; CHECK-NEXT:    orq %rax, %rcx
+; CHECK-NEXT:    movq %r13, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    orq $1, %r13
+; CHECK-NEXT:    orl %esi, %r11d
+; CHECK-NEXT:    movl $1, %edx
+; CHECK-NEXT:    je .LBB0_3
+; CHECK-NEXT:  # %bb.2: # %if.else
+; CHECK-NEXT:    movl (%r8), %edx
+; CHECK-NEXT:  .LBB0_3: # %if.else
+; CHECK-NEXT:    shlq $5, %rcx
+; CHECK-NEXT:    movq %r12, %rsi
+; CHECK-NEXT:    shlq $7, %rsi
+; CHECK-NEXT:    addq %rcx, %rsi
+; CHECK-NEXT:    addq $1248, %rsi # imm = 0x4E0
+; CHECK-NEXT:    movq %r13, 0
+; CHECK-NEXT:    movq %rdi, %r15
+; CHECK-NEXT:    movl %edx, (%rdi)
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    xorl %edi, %edi
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    callq *%rax
+; CHECK-NEXT:    xorq $1, %r14
+; CHECK-NEXT:    cmpl $0, (%r14)
+; CHECK-NEXT:    je .LBB0_6
+; CHECK-NEXT:  # %bb.4: # %if.else
+; CHECK-NEXT:    movb $1, %al
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:    je .LBB0_5
+; CHECK-NEXT:  .LBB0_6: # %bb19
+; CHECK-NEXT:    testb $1, %bl
+; CHECK-NEXT:    movq %r15, %rdi
+; CHECK-NEXT:    movabsq $87960930222080, %r15 # imm = 0x500000000000
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload
+; CHECK-NEXT:    jne .LBB0_8
+; CHECK-NEXT:  .LBB0_7: # %if.end69
+; CHECK-NEXT:    movl %r13d, 0
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    xorl %esi, %esi
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    xorl %ecx, %ecx
+; CHECK-NEXT:    xorl %r8d, %r8d
+; CHECK-NEXT:    callq *%rax
+; CHECK-NEXT:    xorq %r15, %r12
+; CHECK-NEXT:    movslq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 4-byte Folded Reload
+; CHECK-NEXT:    movzbl (%r12), %ecx
+; CHECK-NEXT:    movb %cl, 544(%rax)
+; CHECK-NEXT:  .LBB0_8: # %land.lhs.true56
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    addq $24, %rsp
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa %rsp, 8
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB0_5: # %bb
+entry:
+  %i = load i32, ptr null, align 8
+  br i1 %cmp54, label %if.end69, label %if.else
+
+if.else:                                          ; preds = %entry
+  %shr18 = and i32 %sub.i, 1
+  %idxprom.i = zext i32 %shr18 to i64
+  %arrayidx.i = getelementptr %struct.BlockContext, ptr null, i64 0, i32 14, i64 %idxprom.i
+  %i1 = load i8, ptr %arrayidx.i, align 1
+  %conv.i = zext i8 %i1 to i32
+  %and.i = and i32 %conv.i, 1
+  %i2 = and i64 87960930222080, 1
+  %i3 = inttoptr i64 %i2 to ptr
+  %i4 = load i32, ptr %i3, align 4
+  %i5 = and i64 %idxprom, 1
+  %i6 = or i64 %i5, 17592186044416
+  %i7 = inttoptr i64 %i6 to ptr
+  %i8 = load i32, ptr %i7, align 4
+  %i9 = lshr i32 %bl, %sub.i
+  %i10 = and i32 %i9, 2
+  %i11 = or i32 %bl, %i10
+  %i12 = select i1 %cmp54, i32 %i8, i32 %i4
+  %add.i = or i32 %_msprop1966, %and.i
+  %idxprom4 = zext i32 %bl to i64
+  %idxprom24 = zext i32 %add.i to i64
+  %i13 = or i32 %i, 1
+  %i14 = zext i32 %i13 to i64
+  %.not2329 = icmp eq i32 %i11, 0
+  %i15 = select i1 %.not2329, i32 1, i32 %i12
+  %arrayidx25 = getelementptr %struct.CdfModeContext, ptr null, i64 0, i32 3, i64 %idxprom4, i64 %idxprom24
+  store i64 %i14, ptr null, align 8
+  store i32 %i15, ptr %t, align 4
+  %call53 = tail call i32 null(ptr null, ptr %arrayidx25, i64 0)
+  %i16 = xor i64 %idxprom, 1
+  %i17 = inttoptr i64 %i16 to ptr
+  %_msld1992 = load i32, ptr %i17, align 8
+  %i18 = icmp ne i32 %_msld1992, 0
+  %_msprop_icmp1993 = and i1 %i18, false
+  br i1 %_msprop_icmp1993, label %bb, label %bb19
+
+bb:                                               ; preds = %if.else
+  unreachable
+
+bb19:                                             ; preds = %if.else
+  br i1 %cmp54, label %land.lhs.true56, label %if.end69
+
+land.lhs.true56:                                  ; preds = %bb19
+  ret i32 0
+
+if.end69:                                         ; preds = %bb19, %entry
+  %bx8.011941201 = phi i32 [ %shr18, %bb19 ], [ undef, %entry ]
+  store i32 %i, ptr null, align 8
+  %call79 = tail call fastcc i32 null(ptr %t, i32 0, i32 0, i32 0, i32 0)
+  %idxprom666 = zext i32 %bl to i64
+  %i20 = xor i64 %idxprom666, 87960930222080
+  %idxprom675 = sext i32 %bx8.011941201 to i64
+  %arrayidx676 = getelementptr %struct.BlockContext, ptr null, i64 0, i32 14, i64 %idxprom675
+  %i21 = inttoptr i64 %i20 to ptr
+  %_msld1414 = load i8, ptr %i21, align 1
+  store i8 %_msld1414, ptr %arrayidx676, align 1
+  ret i32 0
+}
+
+attributes #0 = { "frame-pointer"="all" "target-cpu"="x86-64" }
diff --git a/llvm/test/CodeGen/X86/large-constants-x32.ll b/llvm/test/CodeGen/X86/large-constants-x32.ll
index e875920b8b..95a75006e0 100644
--- a/llvm/test/CodeGen/X86/large-constants-x32.ll
+++ b/llvm/test/CodeGen/X86/large-constants-x32.ll
@@ -4,12 +4,16 @@
 define void @constant_expressions() {
 ; CHECK-LABEL: constant_expressions:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    movl -289477652, %eax
-; CHECK-NEXT:    movl -289477644, %ecx
-; CHECK-NEXT:    addl -289477648, %eax
-; CHECK-NEXT:    addl -289477636, %ecx
-; CHECK-NEXT:    addl %eax, %ecx
-; CHECK-NEXT:    movl %ecx, -289477652
+; CHECK-NEXT:    movl $-289477652, %eax # imm = 0xEEBEEBEC
+; CHECK-NEXT:    movl (%eax), %ecx
+; CHECK-NEXT:    movl $-289477644, %edx # imm = 0xEEBEEBF4
+; CHECK-NEXT:    movl (%edx), %edx
+; CHECK-NEXT:    movl $-289477648, %esi # imm = 0xEEBEEBF0
+; CHECK-NEXT:    addl (%esi), %ecx
+; CHECK-NEXT:    movl $-289477636, %esi # imm = 0xEEBEEBFC
+; CHECK-NEXT:    addl (%esi), %edx
+; CHECK-NEXT:    addl %ecx, %edx
+; CHECK-NEXT:    movl %edx, (%eax)
 ; CHECK-NEXT:    retq
 entry:
   %0 = load i32, i32* inttoptr (i32 add (i32 -289477652, i32 0) to i32*)
@@ -27,12 +31,16 @@ entry:
 define void @constant_expressions2() {
 ; CHECK-LABEL: constant_expressions2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    movl -289477652, %eax
-; CHECK-NEXT:    movl -289477644, %ecx
-; CHECK-NEXT:    addl -289477648, %eax
-; CHECK-NEXT:    addl -289477640, %ecx
-; CHECK-NEXT:    addl %eax, %ecx
-; CHECK-NEXT:    movl %ecx, -289477652
+; CHECK-NEXT:    movl $-289477652, %eax # imm = 0xEEBEEBEC
+; CHECK-NEXT:    movl (%eax), %ecx
+; CHECK-NEXT:    movl $-289477644, %edx # imm = 0xEEBEEBF4
+; CHECK-NEXT:    movl (%edx), %edx
+; CHECK-NEXT:    movl $-289477648, %esi # imm = 0xEEBEEBF0
+; CHECK-NEXT:    addl (%esi), %ecx
+; CHECK-NEXT:    movl $-289477640, %esi # imm = 0xEEBEEBF8
+; CHECK-NEXT:    addl (%esi), %edx
+; CHECK-NEXT:    addl %ecx, %edx
+; CHECK-NEXT:    movl %edx, (%eax)
 ; CHECK-NEXT:    retq
 entry:
   %0 = load i32, i32* inttoptr (i32 -289477652 to i32*)
diff --git a/llvm/test/CodeGen/X86/machine-cse.ll b/llvm/test/CodeGen/X86/machine-cse.ll
index d436f10fdc..431031136e 100644
--- a/llvm/test/CodeGen/X86/machine-cse.ll
+++ b/llvm/test/CodeGen/X86/machine-cse.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=x86_64-unknown-unknown < %s | FileCheck %s
+; RUN: llc -mtriple=x86_64-unknown-unknown -early-live-intervals < %s | FileCheck %s
 ; rdar://7610418
 
 %ptr = type { ptr }
diff --git a/llvm/test/CodeGen/X86/mixed-ptr-sizes.ll b/llvm/test/CodeGen/X86/mixed-ptr-sizes.ll
index 2dd6011f36..67539b07f5 100644
--- a/llvm/test/CodeGen/X86/mixed-ptr-sizes.ll
+++ b/llvm/test/CodeGen/X86/mixed-ptr-sizes.ll
@@ -261,7 +261,7 @@ define i64 @test_load_sptr32_zext_i64(ptr addrspace(270) %i) {
 ; CHECK-LABEL: test_load_sptr32_zext_i64:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movslq %ecx, %rax
-; CHECK-NEXT:    movq (%rax), %rax
+; CHECK-NEXT:    movl (%rax), %eax
 ; CHECK-NEXT:    retq
 ;
 ; CHECK-O0-LABEL: test_load_sptr32_zext_i64:
@@ -278,11 +278,20 @@ entry:
 }
 
 define void @test_store_sptr32_trunc_i1(ptr addrspace(270) %s, i32 %i) {
-; ALL-LABEL: test_store_sptr32_trunc_i1:
-; ALL:       # %bb.0: # %entry
-; ALL-NEXT:    movslq %ecx, %rax
-; ALL-NEXT:    movl %edx, (%rax)
-; ALL-NEXT:    retq
+; CHECK-LABEL: test_store_sptr32_trunc_i1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movslq %ecx, %rax
+; CHECK-NEXT:    andl $1, %edx
+; CHECK-NEXT:    movb %dl, (%rax)
+; CHECK-NEXT:    retq
+;
+; CHECK-O0-LABEL: test_store_sptr32_trunc_i1:
+; CHECK-O0:       # %bb.0: # %entry
+; CHECK-O0-NEXT:    movslq %ecx, %rax
+; CHECK-O0-NEXT:    andl $1, %edx
+; CHECK-O0-NEXT:    movb %dl, %cl
+; CHECK-O0-NEXT:    movb %cl, (%rax)
+; CHECK-O0-NEXT:    retq
 entry:
   %0 = trunc i32 %i to i1
   store i1 %0, ptr addrspace(270) %s
diff --git a/llvm/test/CodeGen/X86/no-split-size.ll b/llvm/test/CodeGen/X86/no-split-size.ll
new file mode 100644
index 0000000000..c1f93acd77
--- /dev/null
+++ b/llvm/test/CodeGen/X86/no-split-size.ll
@@ -0,0 +1,90 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s
+
+; @foo is optimized for size. Variables %p2, %p3, %p4, %p5 and %p6 are not split
+; in cold blocks.
+
+define i64 @foo(ptr %ptr, i64 %p2, i64 %p3, i64 %p4, i64 %p5, i64 %p6) optsize {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    .cfi_offset %rbx, -48
+; CHECK-NEXT:    .cfi_offset %r12, -40
+; CHECK-NEXT:    .cfi_offset %r13, -32
+; CHECK-NEXT:    .cfi_offset %r14, -24
+; CHECK-NEXT:    .cfi_offset %r15, -16
+; CHECK-NEXT:    movq %r9, %r14
+; CHECK-NEXT:    movq %r8, %rbx
+; CHECK-NEXT:    movq %rcx, %r12
+; CHECK-NEXT:    movq %rdx, %r15
+; CHECK-NEXT:    movq %rsi, %r13
+; CHECK-NEXT:    testq %rdi, %rdi
+; CHECK-NEXT:    je .LBB0_1
+; CHECK-NEXT:  # %bb.2: # %if.else
+; CHECK-NEXT:    testq %r13, %r13
+; CHECK-NEXT:    movq %r15, %rax
+; CHECK-NEXT:    je .LBB0_3
+; CHECK-NEXT:  .LBB0_4: # %if.end
+; CHECK-NEXT:    addq %r13, %rax
+; CHECK-NEXT:    addq %r12, %r15
+; CHECK-NEXT:    addq %rax, %r15
+; CHECK-NEXT:    addq %r14, %rbx
+; CHECK-NEXT:    addq %r15, %rbx
+; CHECK-NEXT:    movq %rbx, %rax
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB0_1: # %if.then
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    callq bar1@PLT
+; CHECK-NEXT:    jmp .LBB0_4
+; CHECK-NEXT:  .LBB0_3: # %if.then2
+; CHECK-NEXT:    callq bar2@PLT
+; CHECK-NEXT:    jmp .LBB0_4
+entry:
+  %tobool.not = icmp eq ptr %ptr, null
+  br i1 %tobool.not, label %if.then, label %if.else, !prof !5
+
+if.then:                                          ; preds = %entry
+  %call1 = call i64 @bar1()
+  br label %if.end
+
+if.else:
+  %cond = icmp eq i64 %p2, 0
+  br i1 %cond, label %if.then2, label %if.end, !prof !5
+
+if.then2:
+  %call2 = call i64 @bar2()
+  br label %if.end
+
+if.end:
+  %call = phi i64 [ %call1, %if.then ], [%call2, %if.then2], [ %p3, %if.else ]
+  %add1 = add i64 %call, %p2
+  %add2 = add i64 %add1, %p3
+  %add3 = add i64 %add2, %p4
+  %add4 = add i64 %add3, %p5
+  %res = add i64 %add4, %p6
+  ret i64 %res
+}
+
+!5 = !{!"branch_weights", i32 1, i32 2000}
+
+declare i64 @bar1()
+declare i64 @bar2()
diff --git a/llvm/test/CodeGen/X86/osx-private-labels.ll b/llvm/test/CodeGen/X86/osx-private-labels.ll
index 7fed2ad23e..f3343ccb98 100644
--- a/llvm/test/CodeGen/X86/osx-private-labels.ll
+++ b/llvm/test/CodeGen/X86/osx-private-labels.ll
@@ -36,7 +36,7 @@
 
 @private6 = private unnamed_addr constant i128 42
 ; CHECK: .section	__TEXT,__literal16,16byte_literals
-; CHECK-NEXT: .p2align	3
+; CHECK-NEXT: .p2align	4
 ; CHECK-NEXT: L_private6:
 
 %struct._objc_class = type { ptr }
diff --git a/llvm/test/CodeGen/X86/pcsections-atomics.ll b/llvm/test/CodeGen/X86/pcsections-atomics.ll
index e6604c957f..cfc9d50763 100644
--- a/llvm/test/CodeGen/X86/pcsections-atomics.ll
+++ b/llvm/test/CodeGen/X86/pcsections-atomics.ll
@@ -2148,14 +2148,17 @@ define void @atomic8_cas_monotonic(ptr %a) {
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movb $1, %cl
-; O1-NEXT:    movb $42, %al
 ; O1-NEXT:  .Lpcsection65:
-; O1-NEXT:    lock cmpxchgb %cl, (%rdi)
 ; O1-NEXT:    movb $42, %al
 ; O1-NEXT:  .Lpcsection66:
 ; O1-NEXT:    lock cmpxchgb %cl, (%rdi)
-; O1-NEXT:    movb $42, %al
 ; O1-NEXT:  .Lpcsection67:
+; O1-NEXT:    movb $42, %al
+; O1-NEXT:  .Lpcsection68:
+; O1-NEXT:    lock cmpxchgb %cl, (%rdi)
+; O1-NEXT:  .Lpcsection69:
+; O1-NEXT:    movb $42, %al
+; O1-NEXT:  .Lpcsection70:
 ; O1-NEXT:    lock cmpxchgb %cl, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -2164,14 +2167,17 @@ define void @atomic8_cas_monotonic(ptr %a) {
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movb $1, %cl
-; O2-NEXT:    movb $42, %al
 ; O2-NEXT:  .Lpcsection65:
-; O2-NEXT:    lock cmpxchgb %cl, (%rdi)
 ; O2-NEXT:    movb $42, %al
 ; O2-NEXT:  .Lpcsection66:
 ; O2-NEXT:    lock cmpxchgb %cl, (%rdi)
-; O2-NEXT:    movb $42, %al
 ; O2-NEXT:  .Lpcsection67:
+; O2-NEXT:    movb $42, %al
+; O2-NEXT:  .Lpcsection68:
+; O2-NEXT:    lock cmpxchgb %cl, (%rdi)
+; O2-NEXT:  .Lpcsection69:
+; O2-NEXT:    movb $42, %al
+; O2-NEXT:  .Lpcsection70:
 ; O2-NEXT:    lock cmpxchgb %cl, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -2180,14 +2186,17 @@ define void @atomic8_cas_monotonic(ptr %a) {
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movb $1, %cl
-; O3-NEXT:    movb $42, %al
 ; O3-NEXT:  .Lpcsection65:
-; O3-NEXT:    lock cmpxchgb %cl, (%rdi)
 ; O3-NEXT:    movb $42, %al
 ; O3-NEXT:  .Lpcsection66:
 ; O3-NEXT:    lock cmpxchgb %cl, (%rdi)
-; O3-NEXT:    movb $42, %al
 ; O3-NEXT:  .Lpcsection67:
+; O3-NEXT:    movb $42, %al
+; O3-NEXT:  .Lpcsection68:
+; O3-NEXT:    lock cmpxchgb %cl, (%rdi)
+; O3-NEXT:  .Lpcsection69:
+; O3-NEXT:    movb $42, %al
+; O3-NEXT:  .Lpcsection70:
 ; O3-NEXT:    lock cmpxchgb %cl, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -2226,14 +2235,17 @@ define void @atomic8_cas_acquire(ptr %a) {
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movb $1, %cl
+; O1-NEXT:  .Lpcsection71:
 ; O1-NEXT:    movb $42, %al
-; O1-NEXT:  .Lpcsection68:
+; O1-NEXT:  .Lpcsection72:
 ; O1-NEXT:    lock cmpxchgb %cl, (%rdi)
+; O1-NEXT:  .Lpcsection73:
 ; O1-NEXT:    movb $42, %al
-; O1-NEXT:  .Lpcsection69:
+; O1-NEXT:  .Lpcsection74:
 ; O1-NEXT:    lock cmpxchgb %cl, (%rdi)
+; O1-NEXT:  .Lpcsection75:
 ; O1-NEXT:    movb $42, %al
-; O1-NEXT:  .Lpcsection70:
+; O1-NEXT:  .Lpcsection76:
 ; O1-NEXT:    lock cmpxchgb %cl, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -2242,14 +2254,17 @@ define void @atomic8_cas_acquire(ptr %a) {
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movb $1, %cl
+; O2-NEXT:  .Lpcsection71:
 ; O2-NEXT:    movb $42, %al
-; O2-NEXT:  .Lpcsection68:
+; O2-NEXT:  .Lpcsection72:
 ; O2-NEXT:    lock cmpxchgb %cl, (%rdi)
+; O2-NEXT:  .Lpcsection73:
 ; O2-NEXT:    movb $42, %al
-; O2-NEXT:  .Lpcsection69:
+; O2-NEXT:  .Lpcsection74:
 ; O2-NEXT:    lock cmpxchgb %cl, (%rdi)
+; O2-NEXT:  .Lpcsection75:
 ; O2-NEXT:    movb $42, %al
-; O2-NEXT:  .Lpcsection70:
+; O2-NEXT:  .Lpcsection76:
 ; O2-NEXT:    lock cmpxchgb %cl, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -2258,14 +2273,17 @@ define void @atomic8_cas_acquire(ptr %a) {
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movb $1, %cl
+; O3-NEXT:  .Lpcsection71:
 ; O3-NEXT:    movb $42, %al
-; O3-NEXT:  .Lpcsection68:
+; O3-NEXT:  .Lpcsection72:
 ; O3-NEXT:    lock cmpxchgb %cl, (%rdi)
+; O3-NEXT:  .Lpcsection73:
 ; O3-NEXT:    movb $42, %al
-; O3-NEXT:  .Lpcsection69:
+; O3-NEXT:  .Lpcsection74:
 ; O3-NEXT:    lock cmpxchgb %cl, (%rdi)
+; O3-NEXT:  .Lpcsection75:
 ; O3-NEXT:    movb $42, %al
-; O3-NEXT:  .Lpcsection70:
+; O3-NEXT:  .Lpcsection76:
 ; O3-NEXT:    lock cmpxchgb %cl, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -2304,14 +2322,17 @@ define void @atomic8_cas_release(ptr %a) {
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movb $1, %cl
+; O1-NEXT:  .Lpcsection77:
 ; O1-NEXT:    movb $42, %al
-; O1-NEXT:  .Lpcsection71:
+; O1-NEXT:  .Lpcsection78:
 ; O1-NEXT:    lock cmpxchgb %cl, (%rdi)
+; O1-NEXT:  .Lpcsection79:
 ; O1-NEXT:    movb $42, %al
-; O1-NEXT:  .Lpcsection72:
+; O1-NEXT:  .Lpcsection80:
 ; O1-NEXT:    lock cmpxchgb %cl, (%rdi)
+; O1-NEXT:  .Lpcsection81:
 ; O1-NEXT:    movb $42, %al
-; O1-NEXT:  .Lpcsection73:
+; O1-NEXT:  .Lpcsection82:
 ; O1-NEXT:    lock cmpxchgb %cl, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -2320,14 +2341,17 @@ define void @atomic8_cas_release(ptr %a) {
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movb $1, %cl
+; O2-NEXT:  .Lpcsection77:
 ; O2-NEXT:    movb $42, %al
-; O2-NEXT:  .Lpcsection71:
+; O2-NEXT:  .Lpcsection78:
 ; O2-NEXT:    lock cmpxchgb %cl, (%rdi)
+; O2-NEXT:  .Lpcsection79:
 ; O2-NEXT:    movb $42, %al
-; O2-NEXT:  .Lpcsection72:
+; O2-NEXT:  .Lpcsection80:
 ; O2-NEXT:    lock cmpxchgb %cl, (%rdi)
+; O2-NEXT:  .Lpcsection81:
 ; O2-NEXT:    movb $42, %al
-; O2-NEXT:  .Lpcsection73:
+; O2-NEXT:  .Lpcsection82:
 ; O2-NEXT:    lock cmpxchgb %cl, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -2336,14 +2360,17 @@ define void @atomic8_cas_release(ptr %a) {
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movb $1, %cl
+; O3-NEXT:  .Lpcsection77:
 ; O3-NEXT:    movb $42, %al
-; O3-NEXT:  .Lpcsection71:
+; O3-NEXT:  .Lpcsection78:
 ; O3-NEXT:    lock cmpxchgb %cl, (%rdi)
+; O3-NEXT:  .Lpcsection79:
 ; O3-NEXT:    movb $42, %al
-; O3-NEXT:  .Lpcsection72:
+; O3-NEXT:  .Lpcsection80:
 ; O3-NEXT:    lock cmpxchgb %cl, (%rdi)
+; O3-NEXT:  .Lpcsection81:
 ; O3-NEXT:    movb $42, %al
-; O3-NEXT:  .Lpcsection73:
+; O3-NEXT:  .Lpcsection82:
 ; O3-NEXT:    lock cmpxchgb %cl, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -2382,14 +2409,17 @@ define void @atomic8_cas_acq_rel(ptr %a) {
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movb $1, %cl
+; O1-NEXT:  .Lpcsection83:
 ; O1-NEXT:    movb $42, %al
-; O1-NEXT:  .Lpcsection74:
+; O1-NEXT:  .Lpcsection84:
 ; O1-NEXT:    lock cmpxchgb %cl, (%rdi)
+; O1-NEXT:  .Lpcsection85:
 ; O1-NEXT:    movb $42, %al
-; O1-NEXT:  .Lpcsection75:
+; O1-NEXT:  .Lpcsection86:
 ; O1-NEXT:    lock cmpxchgb %cl, (%rdi)
+; O1-NEXT:  .Lpcsection87:
 ; O1-NEXT:    movb $42, %al
-; O1-NEXT:  .Lpcsection76:
+; O1-NEXT:  .Lpcsection88:
 ; O1-NEXT:    lock cmpxchgb %cl, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -2398,14 +2428,17 @@ define void @atomic8_cas_acq_rel(ptr %a) {
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movb $1, %cl
+; O2-NEXT:  .Lpcsection83:
 ; O2-NEXT:    movb $42, %al
-; O2-NEXT:  .Lpcsection74:
+; O2-NEXT:  .Lpcsection84:
 ; O2-NEXT:    lock cmpxchgb %cl, (%rdi)
+; O2-NEXT:  .Lpcsection85:
 ; O2-NEXT:    movb $42, %al
-; O2-NEXT:  .Lpcsection75:
+; O2-NEXT:  .Lpcsection86:
 ; O2-NEXT:    lock cmpxchgb %cl, (%rdi)
+; O2-NEXT:  .Lpcsection87:
 ; O2-NEXT:    movb $42, %al
-; O2-NEXT:  .Lpcsection76:
+; O2-NEXT:  .Lpcsection88:
 ; O2-NEXT:    lock cmpxchgb %cl, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -2414,14 +2447,17 @@ define void @atomic8_cas_acq_rel(ptr %a) {
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movb $1, %cl
+; O3-NEXT:  .Lpcsection83:
 ; O3-NEXT:    movb $42, %al
-; O3-NEXT:  .Lpcsection74:
+; O3-NEXT:  .Lpcsection84:
 ; O3-NEXT:    lock cmpxchgb %cl, (%rdi)
+; O3-NEXT:  .Lpcsection85:
 ; O3-NEXT:    movb $42, %al
-; O3-NEXT:  .Lpcsection75:
+; O3-NEXT:  .Lpcsection86:
 ; O3-NEXT:    lock cmpxchgb %cl, (%rdi)
+; O3-NEXT:  .Lpcsection87:
 ; O3-NEXT:    movb $42, %al
-; O3-NEXT:  .Lpcsection76:
+; O3-NEXT:  .Lpcsection88:
 ; O3-NEXT:    lock cmpxchgb %cl, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -2460,14 +2496,17 @@ define void @atomic8_cas_seq_cst(ptr %a) {
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movb $1, %cl
+; O1-NEXT:  .Lpcsection89:
 ; O1-NEXT:    movb $42, %al
-; O1-NEXT:  .Lpcsection77:
+; O1-NEXT:  .Lpcsection90:
 ; O1-NEXT:    lock cmpxchgb %cl, (%rdi)
+; O1-NEXT:  .Lpcsection91:
 ; O1-NEXT:    movb $42, %al
-; O1-NEXT:  .Lpcsection78:
+; O1-NEXT:  .Lpcsection92:
 ; O1-NEXT:    lock cmpxchgb %cl, (%rdi)
+; O1-NEXT:  .Lpcsection93:
 ; O1-NEXT:    movb $42, %al
-; O1-NEXT:  .Lpcsection79:
+; O1-NEXT:  .Lpcsection94:
 ; O1-NEXT:    lock cmpxchgb %cl, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -2476,14 +2515,17 @@ define void @atomic8_cas_seq_cst(ptr %a) {
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movb $1, %cl
+; O2-NEXT:  .Lpcsection89:
 ; O2-NEXT:    movb $42, %al
-; O2-NEXT:  .Lpcsection77:
+; O2-NEXT:  .Lpcsection90:
 ; O2-NEXT:    lock cmpxchgb %cl, (%rdi)
+; O2-NEXT:  .Lpcsection91:
 ; O2-NEXT:    movb $42, %al
-; O2-NEXT:  .Lpcsection78:
+; O2-NEXT:  .Lpcsection92:
 ; O2-NEXT:    lock cmpxchgb %cl, (%rdi)
+; O2-NEXT:  .Lpcsection93:
 ; O2-NEXT:    movb $42, %al
-; O2-NEXT:  .Lpcsection79:
+; O2-NEXT:  .Lpcsection94:
 ; O2-NEXT:    lock cmpxchgb %cl, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -2492,14 +2534,17 @@ define void @atomic8_cas_seq_cst(ptr %a) {
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movb $1, %cl
+; O3-NEXT:  .Lpcsection89:
 ; O3-NEXT:    movb $42, %al
-; O3-NEXT:  .Lpcsection77:
+; O3-NEXT:  .Lpcsection90:
 ; O3-NEXT:    lock cmpxchgb %cl, (%rdi)
+; O3-NEXT:  .Lpcsection91:
 ; O3-NEXT:    movb $42, %al
-; O3-NEXT:  .Lpcsection78:
+; O3-NEXT:  .Lpcsection92:
 ; O3-NEXT:    lock cmpxchgb %cl, (%rdi)
+; O3-NEXT:  .Lpcsection93:
 ; O3-NEXT:    movb $42, %al
-; O3-NEXT:  .Lpcsection79:
+; O3-NEXT:  .Lpcsection94:
 ; O3-NEXT:    lock cmpxchgb %cl, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -2524,7 +2569,7 @@ define i16 @atomic16_load_unordered(ptr %a) {
 ; O1-LABEL: atomic16_load_unordered:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection80:
+; O1-NEXT:  .Lpcsection95:
 ; O1-NEXT:    movzwl (%rdi), %eax
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -2532,7 +2577,7 @@ define i16 @atomic16_load_unordered(ptr %a) {
 ; O2-LABEL: atomic16_load_unordered:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection80:
+; O2-NEXT:  .Lpcsection95:
 ; O2-NEXT:    movzwl (%rdi), %eax
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -2540,7 +2585,7 @@ define i16 @atomic16_load_unordered(ptr %a) {
 ; O3-LABEL: atomic16_load_unordered:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection80:
+; O3-NEXT:  .Lpcsection95:
 ; O3-NEXT:    movzwl (%rdi), %eax
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -2563,7 +2608,7 @@ define i16 @atomic16_load_monotonic(ptr %a) {
 ; O1-LABEL: atomic16_load_monotonic:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection81:
+; O1-NEXT:  .Lpcsection96:
 ; O1-NEXT:    movzwl (%rdi), %eax
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -2571,7 +2616,7 @@ define i16 @atomic16_load_monotonic(ptr %a) {
 ; O2-LABEL: atomic16_load_monotonic:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection81:
+; O2-NEXT:  .Lpcsection96:
 ; O2-NEXT:    movzwl (%rdi), %eax
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -2579,7 +2624,7 @@ define i16 @atomic16_load_monotonic(ptr %a) {
 ; O3-LABEL: atomic16_load_monotonic:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection81:
+; O3-NEXT:  .Lpcsection96:
 ; O3-NEXT:    movzwl (%rdi), %eax
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -2602,7 +2647,7 @@ define i16 @atomic16_load_acquire(ptr %a) {
 ; O1-LABEL: atomic16_load_acquire:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection82:
+; O1-NEXT:  .Lpcsection97:
 ; O1-NEXT:    movzwl (%rdi), %eax
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -2610,7 +2655,7 @@ define i16 @atomic16_load_acquire(ptr %a) {
 ; O2-LABEL: atomic16_load_acquire:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection82:
+; O2-NEXT:  .Lpcsection97:
 ; O2-NEXT:    movzwl (%rdi), %eax
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -2618,7 +2663,7 @@ define i16 @atomic16_load_acquire(ptr %a) {
 ; O3-LABEL: atomic16_load_acquire:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection82:
+; O3-NEXT:  .Lpcsection97:
 ; O3-NEXT:    movzwl (%rdi), %eax
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -2641,7 +2686,7 @@ define i16 @atomic16_load_seq_cst(ptr %a) {
 ; O1-LABEL: atomic16_load_seq_cst:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection83:
+; O1-NEXT:  .Lpcsection98:
 ; O1-NEXT:    movzwl (%rdi), %eax
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -2649,7 +2694,7 @@ define i16 @atomic16_load_seq_cst(ptr %a) {
 ; O2-LABEL: atomic16_load_seq_cst:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection83:
+; O2-NEXT:  .Lpcsection98:
 ; O2-NEXT:    movzwl (%rdi), %eax
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -2657,7 +2702,7 @@ define i16 @atomic16_load_seq_cst(ptr %a) {
 ; O3-LABEL: atomic16_load_seq_cst:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection83:
+; O3-NEXT:  .Lpcsection98:
 ; O3-NEXT:    movzwl (%rdi), %eax
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -2680,7 +2725,7 @@ define void @atomic16_store_unordered(ptr %a) {
 ; O1-LABEL: atomic16_store_unordered:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection84:
+; O1-NEXT:  .Lpcsection99:
 ; O1-NEXT:    movw $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -2688,7 +2733,7 @@ define void @atomic16_store_unordered(ptr %a) {
 ; O2-LABEL: atomic16_store_unordered:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection84:
+; O2-NEXT:  .Lpcsection99:
 ; O2-NEXT:    movw $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -2696,7 +2741,7 @@ define void @atomic16_store_unordered(ptr %a) {
 ; O3-LABEL: atomic16_store_unordered:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection84:
+; O3-NEXT:  .Lpcsection99:
 ; O3-NEXT:    movw $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -2719,7 +2764,7 @@ define void @atomic16_store_monotonic(ptr %a) {
 ; O1-LABEL: atomic16_store_monotonic:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection85:
+; O1-NEXT:  .Lpcsection100:
 ; O1-NEXT:    movw $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -2727,7 +2772,7 @@ define void @atomic16_store_monotonic(ptr %a) {
 ; O2-LABEL: atomic16_store_monotonic:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection85:
+; O2-NEXT:  .Lpcsection100:
 ; O2-NEXT:    movw $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -2735,7 +2780,7 @@ define void @atomic16_store_monotonic(ptr %a) {
 ; O3-LABEL: atomic16_store_monotonic:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection85:
+; O3-NEXT:  .Lpcsection100:
 ; O3-NEXT:    movw $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -2758,7 +2803,7 @@ define void @atomic16_store_release(ptr %a) {
 ; O1-LABEL: atomic16_store_release:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection86:
+; O1-NEXT:  .Lpcsection101:
 ; O1-NEXT:    movw $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -2766,7 +2811,7 @@ define void @atomic16_store_release(ptr %a) {
 ; O2-LABEL: atomic16_store_release:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection86:
+; O2-NEXT:  .Lpcsection101:
 ; O2-NEXT:    movw $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -2774,7 +2819,7 @@ define void @atomic16_store_release(ptr %a) {
 ; O3-LABEL: atomic16_store_release:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection86:
+; O3-NEXT:  .Lpcsection101:
 ; O3-NEXT:    movw $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -2799,7 +2844,7 @@ define void @atomic16_store_seq_cst(ptr %a) {
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movw $42, %ax
-; O1-NEXT:  .Lpcsection87:
+; O1-NEXT:  .Lpcsection102:
 ; O1-NEXT:    xchgw %ax, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -2808,7 +2853,7 @@ define void @atomic16_store_seq_cst(ptr %a) {
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movw $42, %ax
-; O2-NEXT:  .Lpcsection87:
+; O2-NEXT:  .Lpcsection102:
 ; O2-NEXT:    xchgw %ax, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -2817,7 +2862,7 @@ define void @atomic16_store_seq_cst(ptr %a) {
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movw $42, %ax
-; O3-NEXT:  .Lpcsection87:
+; O3-NEXT:  .Lpcsection102:
 ; O3-NEXT:    xchgw %ax, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -2842,7 +2887,7 @@ define void @atomic16_xchg_monotonic(ptr %a) {
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movw $42, %ax
-; O1-NEXT:  .Lpcsection88:
+; O1-NEXT:  .Lpcsection103:
 ; O1-NEXT:    xchgw %ax, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -2851,7 +2896,7 @@ define void @atomic16_xchg_monotonic(ptr %a) {
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movw $42, %ax
-; O2-NEXT:  .Lpcsection88:
+; O2-NEXT:  .Lpcsection103:
 ; O2-NEXT:    xchgw %ax, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -2860,7 +2905,7 @@ define void @atomic16_xchg_monotonic(ptr %a) {
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movw $42, %ax
-; O3-NEXT:  .Lpcsection88:
+; O3-NEXT:  .Lpcsection103:
 ; O3-NEXT:    xchgw %ax, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -2883,7 +2928,7 @@ define void @atomic16_add_monotonic(ptr %a) {
 ; O1-LABEL: atomic16_add_monotonic:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection89:
+; O1-NEXT:  .Lpcsection104:
 ; O1-NEXT:    lock addw $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -2891,7 +2936,7 @@ define void @atomic16_add_monotonic(ptr %a) {
 ; O2-LABEL: atomic16_add_monotonic:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection89:
+; O2-NEXT:  .Lpcsection104:
 ; O2-NEXT:    lock addw $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -2899,7 +2944,7 @@ define void @atomic16_add_monotonic(ptr %a) {
 ; O3-LABEL: atomic16_add_monotonic:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection89:
+; O3-NEXT:  .Lpcsection104:
 ; O3-NEXT:    lock addw $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -2922,7 +2967,7 @@ define void @atomic16_sub_monotonic(ptr %a) {
 ; O1-LABEL: atomic16_sub_monotonic:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection90:
+; O1-NEXT:  .Lpcsection105:
 ; O1-NEXT:    lock subw $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -2930,7 +2975,7 @@ define void @atomic16_sub_monotonic(ptr %a) {
 ; O2-LABEL: atomic16_sub_monotonic:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection90:
+; O2-NEXT:  .Lpcsection105:
 ; O2-NEXT:    lock subw $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -2938,7 +2983,7 @@ define void @atomic16_sub_monotonic(ptr %a) {
 ; O3-LABEL: atomic16_sub_monotonic:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection90:
+; O3-NEXT:  .Lpcsection105:
 ; O3-NEXT:    lock subw $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -2961,7 +3006,7 @@ define void @atomic16_and_monotonic(ptr %a) {
 ; O1-LABEL: atomic16_and_monotonic:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection91:
+; O1-NEXT:  .Lpcsection106:
 ; O1-NEXT:    lock andw $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -2969,7 +3014,7 @@ define void @atomic16_and_monotonic(ptr %a) {
 ; O2-LABEL: atomic16_and_monotonic:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection91:
+; O2-NEXT:  .Lpcsection106:
 ; O2-NEXT:    lock andw $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -2977,7 +3022,7 @@ define void @atomic16_and_monotonic(ptr %a) {
 ; O3-LABEL: atomic16_and_monotonic:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection91:
+; O3-NEXT:  .Lpcsection106:
 ; O3-NEXT:    lock andw $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -3000,7 +3045,7 @@ define void @atomic16_or_monotonic(ptr %a) {
 ; O1-LABEL: atomic16_or_monotonic:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection92:
+; O1-NEXT:  .Lpcsection107:
 ; O1-NEXT:    lock orw $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -3008,7 +3053,7 @@ define void @atomic16_or_monotonic(ptr %a) {
 ; O2-LABEL: atomic16_or_monotonic:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection92:
+; O2-NEXT:  .Lpcsection107:
 ; O2-NEXT:    lock orw $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -3016,7 +3061,7 @@ define void @atomic16_or_monotonic(ptr %a) {
 ; O3-LABEL: atomic16_or_monotonic:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection92:
+; O3-NEXT:  .Lpcsection107:
 ; O3-NEXT:    lock orw $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -3039,7 +3084,7 @@ define void @atomic16_xor_monotonic(ptr %a) {
 ; O1-LABEL: atomic16_xor_monotonic:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection93:
+; O1-NEXT:  .Lpcsection108:
 ; O1-NEXT:    lock xorw $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -3047,7 +3092,7 @@ define void @atomic16_xor_monotonic(ptr %a) {
 ; O2-LABEL: atomic16_xor_monotonic:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection93:
+; O2-NEXT:  .Lpcsection108:
 ; O2-NEXT:    lock xorw $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -3055,7 +3100,7 @@ define void @atomic16_xor_monotonic(ptr %a) {
 ; O3-LABEL: atomic16_xor_monotonic:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection93:
+; O3-NEXT:  .Lpcsection108:
 ; O3-NEXT:    lock xorw $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -3104,23 +3149,23 @@ define void @atomic16_nand_monotonic(ptr %a) {
 ; O1-LABEL: atomic16_nand_monotonic:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection94:
+; O1-NEXT:  .Lpcsection109:
 ; O1-NEXT:    movzwl (%rdi), %eax
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB64_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ecx
-; O1-NEXT:  .Lpcsection95:
+; O1-NEXT:  .Lpcsection110:
 ; O1-NEXT:    notl %ecx
-; O1-NEXT:  .Lpcsection96:
+; O1-NEXT:  .Lpcsection111:
 ; O1-NEXT:    orl $65493, %ecx # imm = 0xFFD5
-; O1-NEXT:  .Lpcsection97:
+; O1-NEXT:  .Lpcsection112:
 ; O1-NEXT:    # kill: def $ax killed $ax killed $eax
-; O1-NEXT:  .Lpcsection98:
+; O1-NEXT:  .Lpcsection113:
 ; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
-; O1-NEXT:  .Lpcsection99:
+; O1-NEXT:  .Lpcsection114:
 ; O1-NEXT:    # kill: def $ax killed $ax def $eax
-; O1-NEXT:  .Lpcsection100:
+; O1-NEXT:  .Lpcsection115:
 ; O1-NEXT:    jne .LBB64_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -3129,23 +3174,23 @@ define void @atomic16_nand_monotonic(ptr %a) {
 ; O2-LABEL: atomic16_nand_monotonic:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection94:
+; O2-NEXT:  .Lpcsection109:
 ; O2-NEXT:    movzwl (%rdi), %eax
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB64_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ecx
-; O2-NEXT:  .Lpcsection95:
+; O2-NEXT:  .Lpcsection110:
 ; O2-NEXT:    notl %ecx
-; O2-NEXT:  .Lpcsection96:
+; O2-NEXT:  .Lpcsection111:
 ; O2-NEXT:    orl $65493, %ecx # imm = 0xFFD5
-; O2-NEXT:  .Lpcsection97:
+; O2-NEXT:  .Lpcsection112:
 ; O2-NEXT:    # kill: def $ax killed $ax killed $eax
-; O2-NEXT:  .Lpcsection98:
+; O2-NEXT:  .Lpcsection113:
 ; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
-; O2-NEXT:  .Lpcsection99:
+; O2-NEXT:  .Lpcsection114:
 ; O2-NEXT:    # kill: def $ax killed $ax def $eax
-; O2-NEXT:  .Lpcsection100:
+; O2-NEXT:  .Lpcsection115:
 ; O2-NEXT:    jne .LBB64_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -3154,23 +3199,23 @@ define void @atomic16_nand_monotonic(ptr %a) {
 ; O3-LABEL: atomic16_nand_monotonic:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection94:
+; O3-NEXT:  .Lpcsection109:
 ; O3-NEXT:    movzwl (%rdi), %eax
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB64_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ecx
-; O3-NEXT:  .Lpcsection95:
+; O3-NEXT:  .Lpcsection110:
 ; O3-NEXT:    notl %ecx
-; O3-NEXT:  .Lpcsection96:
+; O3-NEXT:  .Lpcsection111:
 ; O3-NEXT:    orl $65493, %ecx # imm = 0xFFD5
-; O3-NEXT:  .Lpcsection97:
+; O3-NEXT:  .Lpcsection112:
 ; O3-NEXT:    # kill: def $ax killed $ax killed $eax
-; O3-NEXT:  .Lpcsection98:
+; O3-NEXT:  .Lpcsection113:
 ; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
-; O3-NEXT:  .Lpcsection99:
+; O3-NEXT:  .Lpcsection114:
 ; O3-NEXT:    # kill: def $ax killed $ax def $eax
-; O3-NEXT:  .Lpcsection100:
+; O3-NEXT:  .Lpcsection115:
 ; O3-NEXT:    jne .LBB64_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -3196,7 +3241,7 @@ define void @atomic16_xchg_acquire(ptr %a) {
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movw $42, %ax
-; O1-NEXT:  .Lpcsection101:
+; O1-NEXT:  .Lpcsection116:
 ; O1-NEXT:    xchgw %ax, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -3205,7 +3250,7 @@ define void @atomic16_xchg_acquire(ptr %a) {
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movw $42, %ax
-; O2-NEXT:  .Lpcsection101:
+; O2-NEXT:  .Lpcsection116:
 ; O2-NEXT:    xchgw %ax, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -3214,7 +3259,7 @@ define void @atomic16_xchg_acquire(ptr %a) {
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movw $42, %ax
-; O3-NEXT:  .Lpcsection101:
+; O3-NEXT:  .Lpcsection116:
 ; O3-NEXT:    xchgw %ax, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -3237,7 +3282,7 @@ define void @atomic16_add_acquire(ptr %a) {
 ; O1-LABEL: atomic16_add_acquire:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection102:
+; O1-NEXT:  .Lpcsection117:
 ; O1-NEXT:    lock addw $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -3245,7 +3290,7 @@ define void @atomic16_add_acquire(ptr %a) {
 ; O2-LABEL: atomic16_add_acquire:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection102:
+; O2-NEXT:  .Lpcsection117:
 ; O2-NEXT:    lock addw $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -3253,7 +3298,7 @@ define void @atomic16_add_acquire(ptr %a) {
 ; O3-LABEL: atomic16_add_acquire:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection102:
+; O3-NEXT:  .Lpcsection117:
 ; O3-NEXT:    lock addw $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -3276,7 +3321,7 @@ define void @atomic16_sub_acquire(ptr %a) {
 ; O1-LABEL: atomic16_sub_acquire:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection103:
+; O1-NEXT:  .Lpcsection118:
 ; O1-NEXT:    lock subw $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -3284,7 +3329,7 @@ define void @atomic16_sub_acquire(ptr %a) {
 ; O2-LABEL: atomic16_sub_acquire:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection103:
+; O2-NEXT:  .Lpcsection118:
 ; O2-NEXT:    lock subw $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -3292,7 +3337,7 @@ define void @atomic16_sub_acquire(ptr %a) {
 ; O3-LABEL: atomic16_sub_acquire:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection103:
+; O3-NEXT:  .Lpcsection118:
 ; O3-NEXT:    lock subw $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -3315,7 +3360,7 @@ define void @atomic16_and_acquire(ptr %a) {
 ; O1-LABEL: atomic16_and_acquire:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection104:
+; O1-NEXT:  .Lpcsection119:
 ; O1-NEXT:    lock andw $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -3323,7 +3368,7 @@ define void @atomic16_and_acquire(ptr %a) {
 ; O2-LABEL: atomic16_and_acquire:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection104:
+; O2-NEXT:  .Lpcsection119:
 ; O2-NEXT:    lock andw $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -3331,7 +3376,7 @@ define void @atomic16_and_acquire(ptr %a) {
 ; O3-LABEL: atomic16_and_acquire:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection104:
+; O3-NEXT:  .Lpcsection119:
 ; O3-NEXT:    lock andw $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -3354,7 +3399,7 @@ define void @atomic16_or_acquire(ptr %a) {
 ; O1-LABEL: atomic16_or_acquire:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection105:
+; O1-NEXT:  .Lpcsection120:
 ; O1-NEXT:    lock orw $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -3362,7 +3407,7 @@ define void @atomic16_or_acquire(ptr %a) {
 ; O2-LABEL: atomic16_or_acquire:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection105:
+; O2-NEXT:  .Lpcsection120:
 ; O2-NEXT:    lock orw $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -3370,7 +3415,7 @@ define void @atomic16_or_acquire(ptr %a) {
 ; O3-LABEL: atomic16_or_acquire:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection105:
+; O3-NEXT:  .Lpcsection120:
 ; O3-NEXT:    lock orw $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -3393,7 +3438,7 @@ define void @atomic16_xor_acquire(ptr %a) {
 ; O1-LABEL: atomic16_xor_acquire:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection106:
+; O1-NEXT:  .Lpcsection121:
 ; O1-NEXT:    lock xorw $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -3401,7 +3446,7 @@ define void @atomic16_xor_acquire(ptr %a) {
 ; O2-LABEL: atomic16_xor_acquire:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection106:
+; O2-NEXT:  .Lpcsection121:
 ; O2-NEXT:    lock xorw $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -3409,7 +3454,7 @@ define void @atomic16_xor_acquire(ptr %a) {
 ; O3-LABEL: atomic16_xor_acquire:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection106:
+; O3-NEXT:  .Lpcsection121:
 ; O3-NEXT:    lock xorw $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -3458,23 +3503,23 @@ define void @atomic16_nand_acquire(ptr %a) {
 ; O1-LABEL: atomic16_nand_acquire:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection107:
+; O1-NEXT:  .Lpcsection122:
 ; O1-NEXT:    movzwl (%rdi), %eax
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB71_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ecx
-; O1-NEXT:  .Lpcsection108:
+; O1-NEXT:  .Lpcsection123:
 ; O1-NEXT:    notl %ecx
-; O1-NEXT:  .Lpcsection109:
+; O1-NEXT:  .Lpcsection124:
 ; O1-NEXT:    orl $65493, %ecx # imm = 0xFFD5
-; O1-NEXT:  .Lpcsection110:
+; O1-NEXT:  .Lpcsection125:
 ; O1-NEXT:    # kill: def $ax killed $ax killed $eax
-; O1-NEXT:  .Lpcsection111:
+; O1-NEXT:  .Lpcsection126:
 ; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
-; O1-NEXT:  .Lpcsection112:
+; O1-NEXT:  .Lpcsection127:
 ; O1-NEXT:    # kill: def $ax killed $ax def $eax
-; O1-NEXT:  .Lpcsection113:
+; O1-NEXT:  .Lpcsection128:
 ; O1-NEXT:    jne .LBB71_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -3483,23 +3528,23 @@ define void @atomic16_nand_acquire(ptr %a) {
 ; O2-LABEL: atomic16_nand_acquire:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection107:
+; O2-NEXT:  .Lpcsection122:
 ; O2-NEXT:    movzwl (%rdi), %eax
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB71_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ecx
-; O2-NEXT:  .Lpcsection108:
+; O2-NEXT:  .Lpcsection123:
 ; O2-NEXT:    notl %ecx
-; O2-NEXT:  .Lpcsection109:
+; O2-NEXT:  .Lpcsection124:
 ; O2-NEXT:    orl $65493, %ecx # imm = 0xFFD5
-; O2-NEXT:  .Lpcsection110:
+; O2-NEXT:  .Lpcsection125:
 ; O2-NEXT:    # kill: def $ax killed $ax killed $eax
-; O2-NEXT:  .Lpcsection111:
+; O2-NEXT:  .Lpcsection126:
 ; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
-; O2-NEXT:  .Lpcsection112:
+; O2-NEXT:  .Lpcsection127:
 ; O2-NEXT:    # kill: def $ax killed $ax def $eax
-; O2-NEXT:  .Lpcsection113:
+; O2-NEXT:  .Lpcsection128:
 ; O2-NEXT:    jne .LBB71_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -3508,23 +3553,23 @@ define void @atomic16_nand_acquire(ptr %a) {
 ; O3-LABEL: atomic16_nand_acquire:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection107:
+; O3-NEXT:  .Lpcsection122:
 ; O3-NEXT:    movzwl (%rdi), %eax
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB71_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ecx
-; O3-NEXT:  .Lpcsection108:
+; O3-NEXT:  .Lpcsection123:
 ; O3-NEXT:    notl %ecx
-; O3-NEXT:  .Lpcsection109:
+; O3-NEXT:  .Lpcsection124:
 ; O3-NEXT:    orl $65493, %ecx # imm = 0xFFD5
-; O3-NEXT:  .Lpcsection110:
+; O3-NEXT:  .Lpcsection125:
 ; O3-NEXT:    # kill: def $ax killed $ax killed $eax
-; O3-NEXT:  .Lpcsection111:
+; O3-NEXT:  .Lpcsection126:
 ; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
-; O3-NEXT:  .Lpcsection112:
+; O3-NEXT:  .Lpcsection127:
 ; O3-NEXT:    # kill: def $ax killed $ax def $eax
-; O3-NEXT:  .Lpcsection113:
+; O3-NEXT:  .Lpcsection128:
 ; O3-NEXT:    jne .LBB71_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -3550,7 +3595,7 @@ define void @atomic16_xchg_release(ptr %a) {
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movw $42, %ax
-; O1-NEXT:  .Lpcsection114:
+; O1-NEXT:  .Lpcsection129:
 ; O1-NEXT:    xchgw %ax, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -3559,7 +3604,7 @@ define void @atomic16_xchg_release(ptr %a) {
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movw $42, %ax
-; O2-NEXT:  .Lpcsection114:
+; O2-NEXT:  .Lpcsection129:
 ; O2-NEXT:    xchgw %ax, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -3568,7 +3613,7 @@ define void @atomic16_xchg_release(ptr %a) {
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movw $42, %ax
-; O3-NEXT:  .Lpcsection114:
+; O3-NEXT:  .Lpcsection129:
 ; O3-NEXT:    xchgw %ax, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -3591,7 +3636,7 @@ define void @atomic16_add_release(ptr %a) {
 ; O1-LABEL: atomic16_add_release:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection115:
+; O1-NEXT:  .Lpcsection130:
 ; O1-NEXT:    lock addw $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -3599,7 +3644,7 @@ define void @atomic16_add_release(ptr %a) {
 ; O2-LABEL: atomic16_add_release:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection115:
+; O2-NEXT:  .Lpcsection130:
 ; O2-NEXT:    lock addw $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -3607,7 +3652,7 @@ define void @atomic16_add_release(ptr %a) {
 ; O3-LABEL: atomic16_add_release:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection115:
+; O3-NEXT:  .Lpcsection130:
 ; O3-NEXT:    lock addw $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -3630,7 +3675,7 @@ define void @atomic16_sub_release(ptr %a) {
 ; O1-LABEL: atomic16_sub_release:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection116:
+; O1-NEXT:  .Lpcsection131:
 ; O1-NEXT:    lock subw $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -3638,7 +3683,7 @@ define void @atomic16_sub_release(ptr %a) {
 ; O2-LABEL: atomic16_sub_release:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection116:
+; O2-NEXT:  .Lpcsection131:
 ; O2-NEXT:    lock subw $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -3646,7 +3691,7 @@ define void @atomic16_sub_release(ptr %a) {
 ; O3-LABEL: atomic16_sub_release:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection116:
+; O3-NEXT:  .Lpcsection131:
 ; O3-NEXT:    lock subw $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -3669,7 +3714,7 @@ define void @atomic16_and_release(ptr %a) {
 ; O1-LABEL: atomic16_and_release:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection117:
+; O1-NEXT:  .Lpcsection132:
 ; O1-NEXT:    lock andw $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -3677,7 +3722,7 @@ define void @atomic16_and_release(ptr %a) {
 ; O2-LABEL: atomic16_and_release:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection117:
+; O2-NEXT:  .Lpcsection132:
 ; O2-NEXT:    lock andw $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -3685,7 +3730,7 @@ define void @atomic16_and_release(ptr %a) {
 ; O3-LABEL: atomic16_and_release:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection117:
+; O3-NEXT:  .Lpcsection132:
 ; O3-NEXT:    lock andw $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -3708,7 +3753,7 @@ define void @atomic16_or_release(ptr %a) {
 ; O1-LABEL: atomic16_or_release:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection118:
+; O1-NEXT:  .Lpcsection133:
 ; O1-NEXT:    lock orw $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -3716,7 +3761,7 @@ define void @atomic16_or_release(ptr %a) {
 ; O2-LABEL: atomic16_or_release:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection118:
+; O2-NEXT:  .Lpcsection133:
 ; O2-NEXT:    lock orw $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -3724,7 +3769,7 @@ define void @atomic16_or_release(ptr %a) {
 ; O3-LABEL: atomic16_or_release:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection118:
+; O3-NEXT:  .Lpcsection133:
 ; O3-NEXT:    lock orw $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -3747,7 +3792,7 @@ define void @atomic16_xor_release(ptr %a) {
 ; O1-LABEL: atomic16_xor_release:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection119:
+; O1-NEXT:  .Lpcsection134:
 ; O1-NEXT:    lock xorw $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -3755,7 +3800,7 @@ define void @atomic16_xor_release(ptr %a) {
 ; O2-LABEL: atomic16_xor_release:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection119:
+; O2-NEXT:  .Lpcsection134:
 ; O2-NEXT:    lock xorw $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -3763,7 +3808,7 @@ define void @atomic16_xor_release(ptr %a) {
 ; O3-LABEL: atomic16_xor_release:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection119:
+; O3-NEXT:  .Lpcsection134:
 ; O3-NEXT:    lock xorw $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -3812,23 +3857,23 @@ define void @atomic16_nand_release(ptr %a) {
 ; O1-LABEL: atomic16_nand_release:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection120:
+; O1-NEXT:  .Lpcsection135:
 ; O1-NEXT:    movzwl (%rdi), %eax
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB78_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ecx
-; O1-NEXT:  .Lpcsection121:
+; O1-NEXT:  .Lpcsection136:
 ; O1-NEXT:    notl %ecx
-; O1-NEXT:  .Lpcsection122:
+; O1-NEXT:  .Lpcsection137:
 ; O1-NEXT:    orl $65493, %ecx # imm = 0xFFD5
-; O1-NEXT:  .Lpcsection123:
+; O1-NEXT:  .Lpcsection138:
 ; O1-NEXT:    # kill: def $ax killed $ax killed $eax
-; O1-NEXT:  .Lpcsection124:
+; O1-NEXT:  .Lpcsection139:
 ; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
-; O1-NEXT:  .Lpcsection125:
+; O1-NEXT:  .Lpcsection140:
 ; O1-NEXT:    # kill: def $ax killed $ax def $eax
-; O1-NEXT:  .Lpcsection126:
+; O1-NEXT:  .Lpcsection141:
 ; O1-NEXT:    jne .LBB78_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -3837,23 +3882,23 @@ define void @atomic16_nand_release(ptr %a) {
 ; O2-LABEL: atomic16_nand_release:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection120:
+; O2-NEXT:  .Lpcsection135:
 ; O2-NEXT:    movzwl (%rdi), %eax
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB78_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ecx
-; O2-NEXT:  .Lpcsection121:
+; O2-NEXT:  .Lpcsection136:
 ; O2-NEXT:    notl %ecx
-; O2-NEXT:  .Lpcsection122:
+; O2-NEXT:  .Lpcsection137:
 ; O2-NEXT:    orl $65493, %ecx # imm = 0xFFD5
-; O2-NEXT:  .Lpcsection123:
+; O2-NEXT:  .Lpcsection138:
 ; O2-NEXT:    # kill: def $ax killed $ax killed $eax
-; O2-NEXT:  .Lpcsection124:
+; O2-NEXT:  .Lpcsection139:
 ; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
-; O2-NEXT:  .Lpcsection125:
+; O2-NEXT:  .Lpcsection140:
 ; O2-NEXT:    # kill: def $ax killed $ax def $eax
-; O2-NEXT:  .Lpcsection126:
+; O2-NEXT:  .Lpcsection141:
 ; O2-NEXT:    jne .LBB78_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -3862,23 +3907,23 @@ define void @atomic16_nand_release(ptr %a) {
 ; O3-LABEL: atomic16_nand_release:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection120:
+; O3-NEXT:  .Lpcsection135:
 ; O3-NEXT:    movzwl (%rdi), %eax
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB78_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ecx
-; O3-NEXT:  .Lpcsection121:
+; O3-NEXT:  .Lpcsection136:
 ; O3-NEXT:    notl %ecx
-; O3-NEXT:  .Lpcsection122:
+; O3-NEXT:  .Lpcsection137:
 ; O3-NEXT:    orl $65493, %ecx # imm = 0xFFD5
-; O3-NEXT:  .Lpcsection123:
+; O3-NEXT:  .Lpcsection138:
 ; O3-NEXT:    # kill: def $ax killed $ax killed $eax
-; O3-NEXT:  .Lpcsection124:
+; O3-NEXT:  .Lpcsection139:
 ; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
-; O3-NEXT:  .Lpcsection125:
+; O3-NEXT:  .Lpcsection140:
 ; O3-NEXT:    # kill: def $ax killed $ax def $eax
-; O3-NEXT:  .Lpcsection126:
+; O3-NEXT:  .Lpcsection141:
 ; O3-NEXT:    jne .LBB78_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -3904,7 +3949,7 @@ define void @atomic16_xchg_acq_rel(ptr %a) {
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movw $42, %ax
-; O1-NEXT:  .Lpcsection127:
+; O1-NEXT:  .Lpcsection142:
 ; O1-NEXT:    xchgw %ax, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -3913,7 +3958,7 @@ define void @atomic16_xchg_acq_rel(ptr %a) {
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movw $42, %ax
-; O2-NEXT:  .Lpcsection127:
+; O2-NEXT:  .Lpcsection142:
 ; O2-NEXT:    xchgw %ax, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -3922,7 +3967,7 @@ define void @atomic16_xchg_acq_rel(ptr %a) {
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movw $42, %ax
-; O3-NEXT:  .Lpcsection127:
+; O3-NEXT:  .Lpcsection142:
 ; O3-NEXT:    xchgw %ax, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -3945,7 +3990,7 @@ define void @atomic16_add_acq_rel(ptr %a) {
 ; O1-LABEL: atomic16_add_acq_rel:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection128:
+; O1-NEXT:  .Lpcsection143:
 ; O1-NEXT:    lock addw $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -3953,7 +3998,7 @@ define void @atomic16_add_acq_rel(ptr %a) {
 ; O2-LABEL: atomic16_add_acq_rel:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection128:
+; O2-NEXT:  .Lpcsection143:
 ; O2-NEXT:    lock addw $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -3961,7 +4006,7 @@ define void @atomic16_add_acq_rel(ptr %a) {
 ; O3-LABEL: atomic16_add_acq_rel:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection128:
+; O3-NEXT:  .Lpcsection143:
 ; O3-NEXT:    lock addw $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -3984,7 +4029,7 @@ define void @atomic16_sub_acq_rel(ptr %a) {
 ; O1-LABEL: atomic16_sub_acq_rel:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection129:
+; O1-NEXT:  .Lpcsection144:
 ; O1-NEXT:    lock subw $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -3992,7 +4037,7 @@ define void @atomic16_sub_acq_rel(ptr %a) {
 ; O2-LABEL: atomic16_sub_acq_rel:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection129:
+; O2-NEXT:  .Lpcsection144:
 ; O2-NEXT:    lock subw $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -4000,7 +4045,7 @@ define void @atomic16_sub_acq_rel(ptr %a) {
 ; O3-LABEL: atomic16_sub_acq_rel:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection129:
+; O3-NEXT:  .Lpcsection144:
 ; O3-NEXT:    lock subw $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -4023,7 +4068,7 @@ define void @atomic16_and_acq_rel(ptr %a) {
 ; O1-LABEL: atomic16_and_acq_rel:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection130:
+; O1-NEXT:  .Lpcsection145:
 ; O1-NEXT:    lock andw $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -4031,7 +4076,7 @@ define void @atomic16_and_acq_rel(ptr %a) {
 ; O2-LABEL: atomic16_and_acq_rel:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection130:
+; O2-NEXT:  .Lpcsection145:
 ; O2-NEXT:    lock andw $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -4039,7 +4084,7 @@ define void @atomic16_and_acq_rel(ptr %a) {
 ; O3-LABEL: atomic16_and_acq_rel:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection130:
+; O3-NEXT:  .Lpcsection145:
 ; O3-NEXT:    lock andw $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -4062,7 +4107,7 @@ define void @atomic16_or_acq_rel(ptr %a) {
 ; O1-LABEL: atomic16_or_acq_rel:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection131:
+; O1-NEXT:  .Lpcsection146:
 ; O1-NEXT:    lock orw $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -4070,7 +4115,7 @@ define void @atomic16_or_acq_rel(ptr %a) {
 ; O2-LABEL: atomic16_or_acq_rel:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection131:
+; O2-NEXT:  .Lpcsection146:
 ; O2-NEXT:    lock orw $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -4078,7 +4123,7 @@ define void @atomic16_or_acq_rel(ptr %a) {
 ; O3-LABEL: atomic16_or_acq_rel:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection131:
+; O3-NEXT:  .Lpcsection146:
 ; O3-NEXT:    lock orw $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -4101,7 +4146,7 @@ define void @atomic16_xor_acq_rel(ptr %a) {
 ; O1-LABEL: atomic16_xor_acq_rel:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection132:
+; O1-NEXT:  .Lpcsection147:
 ; O1-NEXT:    lock xorw $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -4109,7 +4154,7 @@ define void @atomic16_xor_acq_rel(ptr %a) {
 ; O2-LABEL: atomic16_xor_acq_rel:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection132:
+; O2-NEXT:  .Lpcsection147:
 ; O2-NEXT:    lock xorw $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -4117,7 +4162,7 @@ define void @atomic16_xor_acq_rel(ptr %a) {
 ; O3-LABEL: atomic16_xor_acq_rel:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection132:
+; O3-NEXT:  .Lpcsection147:
 ; O3-NEXT:    lock xorw $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -4166,23 +4211,23 @@ define void @atomic16_nand_acq_rel(ptr %a) {
 ; O1-LABEL: atomic16_nand_acq_rel:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection133:
+; O1-NEXT:  .Lpcsection148:
 ; O1-NEXT:    movzwl (%rdi), %eax
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB85_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ecx
-; O1-NEXT:  .Lpcsection134:
+; O1-NEXT:  .Lpcsection149:
 ; O1-NEXT:    notl %ecx
-; O1-NEXT:  .Lpcsection135:
+; O1-NEXT:  .Lpcsection150:
 ; O1-NEXT:    orl $65493, %ecx # imm = 0xFFD5
-; O1-NEXT:  .Lpcsection136:
+; O1-NEXT:  .Lpcsection151:
 ; O1-NEXT:    # kill: def $ax killed $ax killed $eax
-; O1-NEXT:  .Lpcsection137:
+; O1-NEXT:  .Lpcsection152:
 ; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
-; O1-NEXT:  .Lpcsection138:
+; O1-NEXT:  .Lpcsection153:
 ; O1-NEXT:    # kill: def $ax killed $ax def $eax
-; O1-NEXT:  .Lpcsection139:
+; O1-NEXT:  .Lpcsection154:
 ; O1-NEXT:    jne .LBB85_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -4191,23 +4236,23 @@ define void @atomic16_nand_acq_rel(ptr %a) {
 ; O2-LABEL: atomic16_nand_acq_rel:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection133:
+; O2-NEXT:  .Lpcsection148:
 ; O2-NEXT:    movzwl (%rdi), %eax
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB85_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ecx
-; O2-NEXT:  .Lpcsection134:
+; O2-NEXT:  .Lpcsection149:
 ; O2-NEXT:    notl %ecx
-; O2-NEXT:  .Lpcsection135:
+; O2-NEXT:  .Lpcsection150:
 ; O2-NEXT:    orl $65493, %ecx # imm = 0xFFD5
-; O2-NEXT:  .Lpcsection136:
+; O2-NEXT:  .Lpcsection151:
 ; O2-NEXT:    # kill: def $ax killed $ax killed $eax
-; O2-NEXT:  .Lpcsection137:
+; O2-NEXT:  .Lpcsection152:
 ; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
-; O2-NEXT:  .Lpcsection138:
+; O2-NEXT:  .Lpcsection153:
 ; O2-NEXT:    # kill: def $ax killed $ax def $eax
-; O2-NEXT:  .Lpcsection139:
+; O2-NEXT:  .Lpcsection154:
 ; O2-NEXT:    jne .LBB85_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -4216,23 +4261,23 @@ define void @atomic16_nand_acq_rel(ptr %a) {
 ; O3-LABEL: atomic16_nand_acq_rel:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection133:
+; O3-NEXT:  .Lpcsection148:
 ; O3-NEXT:    movzwl (%rdi), %eax
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB85_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ecx
-; O3-NEXT:  .Lpcsection134:
+; O3-NEXT:  .Lpcsection149:
 ; O3-NEXT:    notl %ecx
-; O3-NEXT:  .Lpcsection135:
+; O3-NEXT:  .Lpcsection150:
 ; O3-NEXT:    orl $65493, %ecx # imm = 0xFFD5
-; O3-NEXT:  .Lpcsection136:
+; O3-NEXT:  .Lpcsection151:
 ; O3-NEXT:    # kill: def $ax killed $ax killed $eax
-; O3-NEXT:  .Lpcsection137:
+; O3-NEXT:  .Lpcsection152:
 ; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
-; O3-NEXT:  .Lpcsection138:
+; O3-NEXT:  .Lpcsection153:
 ; O3-NEXT:    # kill: def $ax killed $ax def $eax
-; O3-NEXT:  .Lpcsection139:
+; O3-NEXT:  .Lpcsection154:
 ; O3-NEXT:    jne .LBB85_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -4258,7 +4303,7 @@ define void @atomic16_xchg_seq_cst(ptr %a) {
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movw $42, %ax
-; O1-NEXT:  .Lpcsection140:
+; O1-NEXT:  .Lpcsection155:
 ; O1-NEXT:    xchgw %ax, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -4267,7 +4312,7 @@ define void @atomic16_xchg_seq_cst(ptr %a) {
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movw $42, %ax
-; O2-NEXT:  .Lpcsection140:
+; O2-NEXT:  .Lpcsection155:
 ; O2-NEXT:    xchgw %ax, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -4276,7 +4321,7 @@ define void @atomic16_xchg_seq_cst(ptr %a) {
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movw $42, %ax
-; O3-NEXT:  .Lpcsection140:
+; O3-NEXT:  .Lpcsection155:
 ; O3-NEXT:    xchgw %ax, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -4299,7 +4344,7 @@ define void @atomic16_add_seq_cst(ptr %a) {
 ; O1-LABEL: atomic16_add_seq_cst:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection141:
+; O1-NEXT:  .Lpcsection156:
 ; O1-NEXT:    lock addw $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -4307,7 +4352,7 @@ define void @atomic16_add_seq_cst(ptr %a) {
 ; O2-LABEL: atomic16_add_seq_cst:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection141:
+; O2-NEXT:  .Lpcsection156:
 ; O2-NEXT:    lock addw $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -4315,7 +4360,7 @@ define void @atomic16_add_seq_cst(ptr %a) {
 ; O3-LABEL: atomic16_add_seq_cst:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection141:
+; O3-NEXT:  .Lpcsection156:
 ; O3-NEXT:    lock addw $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -4338,7 +4383,7 @@ define void @atomic16_sub_seq_cst(ptr %a) {
 ; O1-LABEL: atomic16_sub_seq_cst:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection142:
+; O1-NEXT:  .Lpcsection157:
 ; O1-NEXT:    lock subw $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -4346,7 +4391,7 @@ define void @atomic16_sub_seq_cst(ptr %a) {
 ; O2-LABEL: atomic16_sub_seq_cst:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection142:
+; O2-NEXT:  .Lpcsection157:
 ; O2-NEXT:    lock subw $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -4354,7 +4399,7 @@ define void @atomic16_sub_seq_cst(ptr %a) {
 ; O3-LABEL: atomic16_sub_seq_cst:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection142:
+; O3-NEXT:  .Lpcsection157:
 ; O3-NEXT:    lock subw $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -4377,7 +4422,7 @@ define void @atomic16_and_seq_cst(ptr %a) {
 ; O1-LABEL: atomic16_and_seq_cst:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection143:
+; O1-NEXT:  .Lpcsection158:
 ; O1-NEXT:    lock andw $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -4385,7 +4430,7 @@ define void @atomic16_and_seq_cst(ptr %a) {
 ; O2-LABEL: atomic16_and_seq_cst:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection143:
+; O2-NEXT:  .Lpcsection158:
 ; O2-NEXT:    lock andw $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -4393,7 +4438,7 @@ define void @atomic16_and_seq_cst(ptr %a) {
 ; O3-LABEL: atomic16_and_seq_cst:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection143:
+; O3-NEXT:  .Lpcsection158:
 ; O3-NEXT:    lock andw $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -4416,7 +4461,7 @@ define void @atomic16_or_seq_cst(ptr %a) {
 ; O1-LABEL: atomic16_or_seq_cst:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection144:
+; O1-NEXT:  .Lpcsection159:
 ; O1-NEXT:    lock orw $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -4424,7 +4469,7 @@ define void @atomic16_or_seq_cst(ptr %a) {
 ; O2-LABEL: atomic16_or_seq_cst:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection144:
+; O2-NEXT:  .Lpcsection159:
 ; O2-NEXT:    lock orw $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -4432,7 +4477,7 @@ define void @atomic16_or_seq_cst(ptr %a) {
 ; O3-LABEL: atomic16_or_seq_cst:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection144:
+; O3-NEXT:  .Lpcsection159:
 ; O3-NEXT:    lock orw $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -4455,7 +4500,7 @@ define void @atomic16_xor_seq_cst(ptr %a) {
 ; O1-LABEL: atomic16_xor_seq_cst:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection145:
+; O1-NEXT:  .Lpcsection160:
 ; O1-NEXT:    lock xorw $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -4463,7 +4508,7 @@ define void @atomic16_xor_seq_cst(ptr %a) {
 ; O2-LABEL: atomic16_xor_seq_cst:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection145:
+; O2-NEXT:  .Lpcsection160:
 ; O2-NEXT:    lock xorw $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -4471,7 +4516,7 @@ define void @atomic16_xor_seq_cst(ptr %a) {
 ; O3-LABEL: atomic16_xor_seq_cst:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection145:
+; O3-NEXT:  .Lpcsection160:
 ; O3-NEXT:    lock xorw $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -4520,23 +4565,23 @@ define void @atomic16_nand_seq_cst(ptr %a) {
 ; O1-LABEL: atomic16_nand_seq_cst:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection146:
+; O1-NEXT:  .Lpcsection161:
 ; O1-NEXT:    movzwl (%rdi), %eax
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB92_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ecx
-; O1-NEXT:  .Lpcsection147:
+; O1-NEXT:  .Lpcsection162:
 ; O1-NEXT:    notl %ecx
-; O1-NEXT:  .Lpcsection148:
+; O1-NEXT:  .Lpcsection163:
 ; O1-NEXT:    orl $65493, %ecx # imm = 0xFFD5
-; O1-NEXT:  .Lpcsection149:
+; O1-NEXT:  .Lpcsection164:
 ; O1-NEXT:    # kill: def $ax killed $ax killed $eax
-; O1-NEXT:  .Lpcsection150:
+; O1-NEXT:  .Lpcsection165:
 ; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
-; O1-NEXT:  .Lpcsection151:
+; O1-NEXT:  .Lpcsection166:
 ; O1-NEXT:    # kill: def $ax killed $ax def $eax
-; O1-NEXT:  .Lpcsection152:
+; O1-NEXT:  .Lpcsection167:
 ; O1-NEXT:    jne .LBB92_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -4545,23 +4590,23 @@ define void @atomic16_nand_seq_cst(ptr %a) {
 ; O2-LABEL: atomic16_nand_seq_cst:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection146:
+; O2-NEXT:  .Lpcsection161:
 ; O2-NEXT:    movzwl (%rdi), %eax
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB92_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ecx
-; O2-NEXT:  .Lpcsection147:
+; O2-NEXT:  .Lpcsection162:
 ; O2-NEXT:    notl %ecx
-; O2-NEXT:  .Lpcsection148:
+; O2-NEXT:  .Lpcsection163:
 ; O2-NEXT:    orl $65493, %ecx # imm = 0xFFD5
-; O2-NEXT:  .Lpcsection149:
+; O2-NEXT:  .Lpcsection164:
 ; O2-NEXT:    # kill: def $ax killed $ax killed $eax
-; O2-NEXT:  .Lpcsection150:
+; O2-NEXT:  .Lpcsection165:
 ; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
-; O2-NEXT:  .Lpcsection151:
+; O2-NEXT:  .Lpcsection166:
 ; O2-NEXT:    # kill: def $ax killed $ax def $eax
-; O2-NEXT:  .Lpcsection152:
+; O2-NEXT:  .Lpcsection167:
 ; O2-NEXT:    jne .LBB92_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -4570,23 +4615,23 @@ define void @atomic16_nand_seq_cst(ptr %a) {
 ; O3-LABEL: atomic16_nand_seq_cst:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection146:
+; O3-NEXT:  .Lpcsection161:
 ; O3-NEXT:    movzwl (%rdi), %eax
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB92_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ecx
-; O3-NEXT:  .Lpcsection147:
+; O3-NEXT:  .Lpcsection162:
 ; O3-NEXT:    notl %ecx
-; O3-NEXT:  .Lpcsection148:
+; O3-NEXT:  .Lpcsection163:
 ; O3-NEXT:    orl $65493, %ecx # imm = 0xFFD5
-; O3-NEXT:  .Lpcsection149:
+; O3-NEXT:  .Lpcsection164:
 ; O3-NEXT:    # kill: def $ax killed $ax killed $eax
-; O3-NEXT:  .Lpcsection150:
+; O3-NEXT:  .Lpcsection165:
 ; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
-; O3-NEXT:  .Lpcsection151:
+; O3-NEXT:  .Lpcsection166:
 ; O3-NEXT:    # kill: def $ax killed $ax def $eax
-; O3-NEXT:  .Lpcsection152:
+; O3-NEXT:  .Lpcsection167:
 ; O3-NEXT:    jne .LBB92_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -4625,13 +4670,13 @@ define void @atomic16_cas_monotonic(ptr %a) {
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movw $1, %cx
 ; O1-NEXT:    movw $42, %ax
-; O1-NEXT:  .Lpcsection153:
+; O1-NEXT:  .Lpcsection168:
 ; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O1-NEXT:    movw $42, %ax
-; O1-NEXT:  .Lpcsection154:
+; O1-NEXT:  .Lpcsection169:
 ; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O1-NEXT:    movw $42, %ax
-; O1-NEXT:  .Lpcsection155:
+; O1-NEXT:  .Lpcsection170:
 ; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -4641,13 +4686,13 @@ define void @atomic16_cas_monotonic(ptr %a) {
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movw $1, %cx
 ; O2-NEXT:    movw $42, %ax
-; O2-NEXT:  .Lpcsection153:
+; O2-NEXT:  .Lpcsection168:
 ; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O2-NEXT:    movw $42, %ax
-; O2-NEXT:  .Lpcsection154:
+; O2-NEXT:  .Lpcsection169:
 ; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O2-NEXT:    movw $42, %ax
-; O2-NEXT:  .Lpcsection155:
+; O2-NEXT:  .Lpcsection170:
 ; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -4657,13 +4702,13 @@ define void @atomic16_cas_monotonic(ptr %a) {
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movw $1, %cx
 ; O3-NEXT:    movw $42, %ax
-; O3-NEXT:  .Lpcsection153:
+; O3-NEXT:  .Lpcsection168:
 ; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O3-NEXT:    movw $42, %ax
-; O3-NEXT:  .Lpcsection154:
+; O3-NEXT:  .Lpcsection169:
 ; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O3-NEXT:    movw $42, %ax
-; O3-NEXT:  .Lpcsection155:
+; O3-NEXT:  .Lpcsection170:
 ; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -4703,13 +4748,13 @@ define void @atomic16_cas_acquire(ptr %a) {
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movw $1, %cx
 ; O1-NEXT:    movw $42, %ax
-; O1-NEXT:  .Lpcsection156:
+; O1-NEXT:  .Lpcsection171:
 ; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O1-NEXT:    movw $42, %ax
-; O1-NEXT:  .Lpcsection157:
+; O1-NEXT:  .Lpcsection172:
 ; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O1-NEXT:    movw $42, %ax
-; O1-NEXT:  .Lpcsection158:
+; O1-NEXT:  .Lpcsection173:
 ; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -4719,13 +4764,13 @@ define void @atomic16_cas_acquire(ptr %a) {
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movw $1, %cx
 ; O2-NEXT:    movw $42, %ax
-; O2-NEXT:  .Lpcsection156:
+; O2-NEXT:  .Lpcsection171:
 ; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O2-NEXT:    movw $42, %ax
-; O2-NEXT:  .Lpcsection157:
+; O2-NEXT:  .Lpcsection172:
 ; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O2-NEXT:    movw $42, %ax
-; O2-NEXT:  .Lpcsection158:
+; O2-NEXT:  .Lpcsection173:
 ; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -4735,13 +4780,13 @@ define void @atomic16_cas_acquire(ptr %a) {
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movw $1, %cx
 ; O3-NEXT:    movw $42, %ax
-; O3-NEXT:  .Lpcsection156:
+; O3-NEXT:  .Lpcsection171:
 ; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O3-NEXT:    movw $42, %ax
-; O3-NEXT:  .Lpcsection157:
+; O3-NEXT:  .Lpcsection172:
 ; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O3-NEXT:    movw $42, %ax
-; O3-NEXT:  .Lpcsection158:
+; O3-NEXT:  .Lpcsection173:
 ; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -4781,13 +4826,13 @@ define void @atomic16_cas_release(ptr %a) {
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movw $1, %cx
 ; O1-NEXT:    movw $42, %ax
-; O1-NEXT:  .Lpcsection159:
+; O1-NEXT:  .Lpcsection174:
 ; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O1-NEXT:    movw $42, %ax
-; O1-NEXT:  .Lpcsection160:
+; O1-NEXT:  .Lpcsection175:
 ; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O1-NEXT:    movw $42, %ax
-; O1-NEXT:  .Lpcsection161:
+; O1-NEXT:  .Lpcsection176:
 ; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -4797,13 +4842,13 @@ define void @atomic16_cas_release(ptr %a) {
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movw $1, %cx
 ; O2-NEXT:    movw $42, %ax
-; O2-NEXT:  .Lpcsection159:
+; O2-NEXT:  .Lpcsection174:
 ; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O2-NEXT:    movw $42, %ax
-; O2-NEXT:  .Lpcsection160:
+; O2-NEXT:  .Lpcsection175:
 ; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O2-NEXT:    movw $42, %ax
-; O2-NEXT:  .Lpcsection161:
+; O2-NEXT:  .Lpcsection176:
 ; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -4813,13 +4858,13 @@ define void @atomic16_cas_release(ptr %a) {
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movw $1, %cx
 ; O3-NEXT:    movw $42, %ax
-; O3-NEXT:  .Lpcsection159:
+; O3-NEXT:  .Lpcsection174:
 ; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O3-NEXT:    movw $42, %ax
-; O3-NEXT:  .Lpcsection160:
+; O3-NEXT:  .Lpcsection175:
 ; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O3-NEXT:    movw $42, %ax
-; O3-NEXT:  .Lpcsection161:
+; O3-NEXT:  .Lpcsection176:
 ; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -4859,13 +4904,13 @@ define void @atomic16_cas_acq_rel(ptr %a) {
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movw $1, %cx
 ; O1-NEXT:    movw $42, %ax
-; O1-NEXT:  .Lpcsection162:
+; O1-NEXT:  .Lpcsection177:
 ; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O1-NEXT:    movw $42, %ax
-; O1-NEXT:  .Lpcsection163:
+; O1-NEXT:  .Lpcsection178:
 ; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O1-NEXT:    movw $42, %ax
-; O1-NEXT:  .Lpcsection164:
+; O1-NEXT:  .Lpcsection179:
 ; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -4875,13 +4920,13 @@ define void @atomic16_cas_acq_rel(ptr %a) {
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movw $1, %cx
 ; O2-NEXT:    movw $42, %ax
-; O2-NEXT:  .Lpcsection162:
+; O2-NEXT:  .Lpcsection177:
 ; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O2-NEXT:    movw $42, %ax
-; O2-NEXT:  .Lpcsection163:
+; O2-NEXT:  .Lpcsection178:
 ; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O2-NEXT:    movw $42, %ax
-; O2-NEXT:  .Lpcsection164:
+; O2-NEXT:  .Lpcsection179:
 ; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -4891,13 +4936,13 @@ define void @atomic16_cas_acq_rel(ptr %a) {
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movw $1, %cx
 ; O3-NEXT:    movw $42, %ax
-; O3-NEXT:  .Lpcsection162:
+; O3-NEXT:  .Lpcsection177:
 ; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O3-NEXT:    movw $42, %ax
-; O3-NEXT:  .Lpcsection163:
+; O3-NEXT:  .Lpcsection178:
 ; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O3-NEXT:    movw $42, %ax
-; O3-NEXT:  .Lpcsection164:
+; O3-NEXT:  .Lpcsection179:
 ; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -4937,13 +4982,13 @@ define void @atomic16_cas_seq_cst(ptr %a) {
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movw $1, %cx
 ; O1-NEXT:    movw $42, %ax
-; O1-NEXT:  .Lpcsection165:
+; O1-NEXT:  .Lpcsection180:
 ; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O1-NEXT:    movw $42, %ax
-; O1-NEXT:  .Lpcsection166:
+; O1-NEXT:  .Lpcsection181:
 ; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O1-NEXT:    movw $42, %ax
-; O1-NEXT:  .Lpcsection167:
+; O1-NEXT:  .Lpcsection182:
 ; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -4953,13 +4998,13 @@ define void @atomic16_cas_seq_cst(ptr %a) {
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movw $1, %cx
 ; O2-NEXT:    movw $42, %ax
-; O2-NEXT:  .Lpcsection165:
+; O2-NEXT:  .Lpcsection180:
 ; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O2-NEXT:    movw $42, %ax
-; O2-NEXT:  .Lpcsection166:
+; O2-NEXT:  .Lpcsection181:
 ; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O2-NEXT:    movw $42, %ax
-; O2-NEXT:  .Lpcsection167:
+; O2-NEXT:  .Lpcsection182:
 ; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -4969,13 +5014,13 @@ define void @atomic16_cas_seq_cst(ptr %a) {
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movw $1, %cx
 ; O3-NEXT:    movw $42, %ax
-; O3-NEXT:  .Lpcsection165:
+; O3-NEXT:  .Lpcsection180:
 ; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O3-NEXT:    movw $42, %ax
-; O3-NEXT:  .Lpcsection166:
+; O3-NEXT:  .Lpcsection181:
 ; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O3-NEXT:    movw $42, %ax
-; O3-NEXT:  .Lpcsection167:
+; O3-NEXT:  .Lpcsection182:
 ; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -5000,7 +5045,7 @@ define i32 @atomic32_load_unordered(ptr %a) {
 ; O1-LABEL: atomic32_load_unordered:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection168:
+; O1-NEXT:  .Lpcsection183:
 ; O1-NEXT:    movl (%rdi), %eax
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -5008,7 +5053,7 @@ define i32 @atomic32_load_unordered(ptr %a) {
 ; O2-LABEL: atomic32_load_unordered:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection168:
+; O2-NEXT:  .Lpcsection183:
 ; O2-NEXT:    movl (%rdi), %eax
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -5016,7 +5061,7 @@ define i32 @atomic32_load_unordered(ptr %a) {
 ; O3-LABEL: atomic32_load_unordered:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection168:
+; O3-NEXT:  .Lpcsection183:
 ; O3-NEXT:    movl (%rdi), %eax
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -5039,7 +5084,7 @@ define i32 @atomic32_load_monotonic(ptr %a) {
 ; O1-LABEL: atomic32_load_monotonic:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection169:
+; O1-NEXT:  .Lpcsection184:
 ; O1-NEXT:    movl (%rdi), %eax
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -5047,7 +5092,7 @@ define i32 @atomic32_load_monotonic(ptr %a) {
 ; O2-LABEL: atomic32_load_monotonic:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection169:
+; O2-NEXT:  .Lpcsection184:
 ; O2-NEXT:    movl (%rdi), %eax
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -5055,7 +5100,7 @@ define i32 @atomic32_load_monotonic(ptr %a) {
 ; O3-LABEL: atomic32_load_monotonic:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection169:
+; O3-NEXT:  .Lpcsection184:
 ; O3-NEXT:    movl (%rdi), %eax
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -5078,7 +5123,7 @@ define i32 @atomic32_load_acquire(ptr %a) {
 ; O1-LABEL: atomic32_load_acquire:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection170:
+; O1-NEXT:  .Lpcsection185:
 ; O1-NEXT:    movl (%rdi), %eax
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -5086,7 +5131,7 @@ define i32 @atomic32_load_acquire(ptr %a) {
 ; O2-LABEL: atomic32_load_acquire:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection170:
+; O2-NEXT:  .Lpcsection185:
 ; O2-NEXT:    movl (%rdi), %eax
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -5094,7 +5139,7 @@ define i32 @atomic32_load_acquire(ptr %a) {
 ; O3-LABEL: atomic32_load_acquire:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection170:
+; O3-NEXT:  .Lpcsection185:
 ; O3-NEXT:    movl (%rdi), %eax
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -5117,7 +5162,7 @@ define i32 @atomic32_load_seq_cst(ptr %a) {
 ; O1-LABEL: atomic32_load_seq_cst:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection171:
+; O1-NEXT:  .Lpcsection186:
 ; O1-NEXT:    movl (%rdi), %eax
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -5125,7 +5170,7 @@ define i32 @atomic32_load_seq_cst(ptr %a) {
 ; O2-LABEL: atomic32_load_seq_cst:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection171:
+; O2-NEXT:  .Lpcsection186:
 ; O2-NEXT:    movl (%rdi), %eax
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -5133,7 +5178,7 @@ define i32 @atomic32_load_seq_cst(ptr %a) {
 ; O3-LABEL: atomic32_load_seq_cst:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection171:
+; O3-NEXT:  .Lpcsection186:
 ; O3-NEXT:    movl (%rdi), %eax
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -5156,7 +5201,7 @@ define void @atomic32_store_unordered(ptr %a) {
 ; O1-LABEL: atomic32_store_unordered:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection172:
+; O1-NEXT:  .Lpcsection187:
 ; O1-NEXT:    movl $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -5164,7 +5209,7 @@ define void @atomic32_store_unordered(ptr %a) {
 ; O2-LABEL: atomic32_store_unordered:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection172:
+; O2-NEXT:  .Lpcsection187:
 ; O2-NEXT:    movl $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -5172,7 +5217,7 @@ define void @atomic32_store_unordered(ptr %a) {
 ; O3-LABEL: atomic32_store_unordered:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection172:
+; O3-NEXT:  .Lpcsection187:
 ; O3-NEXT:    movl $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -5195,7 +5240,7 @@ define void @atomic32_store_monotonic(ptr %a) {
 ; O1-LABEL: atomic32_store_monotonic:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection173:
+; O1-NEXT:  .Lpcsection188:
 ; O1-NEXT:    movl $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -5203,7 +5248,7 @@ define void @atomic32_store_monotonic(ptr %a) {
 ; O2-LABEL: atomic32_store_monotonic:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection173:
+; O2-NEXT:  .Lpcsection188:
 ; O2-NEXT:    movl $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -5211,7 +5256,7 @@ define void @atomic32_store_monotonic(ptr %a) {
 ; O3-LABEL: atomic32_store_monotonic:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection173:
+; O3-NEXT:  .Lpcsection188:
 ; O3-NEXT:    movl $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -5234,7 +5279,7 @@ define void @atomic32_store_release(ptr %a) {
 ; O1-LABEL: atomic32_store_release:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection174:
+; O1-NEXT:  .Lpcsection189:
 ; O1-NEXT:    movl $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -5242,7 +5287,7 @@ define void @atomic32_store_release(ptr %a) {
 ; O2-LABEL: atomic32_store_release:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection174:
+; O2-NEXT:  .Lpcsection189:
 ; O2-NEXT:    movl $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -5250,7 +5295,7 @@ define void @atomic32_store_release(ptr %a) {
 ; O3-LABEL: atomic32_store_release:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection174:
+; O3-NEXT:  .Lpcsection189:
 ; O3-NEXT:    movl $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -5275,7 +5320,7 @@ define void @atomic32_store_seq_cst(ptr %a) {
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection175:
+; O1-NEXT:  .Lpcsection190:
 ; O1-NEXT:    xchgl %eax, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -5284,7 +5329,7 @@ define void @atomic32_store_seq_cst(ptr %a) {
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection175:
+; O2-NEXT:  .Lpcsection190:
 ; O2-NEXT:    xchgl %eax, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -5293,7 +5338,7 @@ define void @atomic32_store_seq_cst(ptr %a) {
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection175:
+; O3-NEXT:  .Lpcsection190:
 ; O3-NEXT:    xchgl %eax, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -5318,7 +5363,7 @@ define void @atomic32_xchg_monotonic(ptr %a) {
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection176:
+; O1-NEXT:  .Lpcsection191:
 ; O1-NEXT:    xchgl %eax, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -5327,7 +5372,7 @@ define void @atomic32_xchg_monotonic(ptr %a) {
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection176:
+; O2-NEXT:  .Lpcsection191:
 ; O2-NEXT:    xchgl %eax, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -5336,7 +5381,7 @@ define void @atomic32_xchg_monotonic(ptr %a) {
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection176:
+; O3-NEXT:  .Lpcsection191:
 ; O3-NEXT:    xchgl %eax, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -5359,7 +5404,7 @@ define void @atomic32_add_monotonic(ptr %a) {
 ; O1-LABEL: atomic32_add_monotonic:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection177:
+; O1-NEXT:  .Lpcsection192:
 ; O1-NEXT:    lock addl $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -5367,7 +5412,7 @@ define void @atomic32_add_monotonic(ptr %a) {
 ; O2-LABEL: atomic32_add_monotonic:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection177:
+; O2-NEXT:  .Lpcsection192:
 ; O2-NEXT:    lock addl $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -5375,7 +5420,7 @@ define void @atomic32_add_monotonic(ptr %a) {
 ; O3-LABEL: atomic32_add_monotonic:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection177:
+; O3-NEXT:  .Lpcsection192:
 ; O3-NEXT:    lock addl $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -5398,7 +5443,7 @@ define void @atomic32_sub_monotonic(ptr %a) {
 ; O1-LABEL: atomic32_sub_monotonic:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection178:
+; O1-NEXT:  .Lpcsection193:
 ; O1-NEXT:    lock subl $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -5406,7 +5451,7 @@ define void @atomic32_sub_monotonic(ptr %a) {
 ; O2-LABEL: atomic32_sub_monotonic:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection178:
+; O2-NEXT:  .Lpcsection193:
 ; O2-NEXT:    lock subl $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -5414,7 +5459,7 @@ define void @atomic32_sub_monotonic(ptr %a) {
 ; O3-LABEL: atomic32_sub_monotonic:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection178:
+; O3-NEXT:  .Lpcsection193:
 ; O3-NEXT:    lock subl $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -5437,7 +5482,7 @@ define void @atomic32_and_monotonic(ptr %a) {
 ; O1-LABEL: atomic32_and_monotonic:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection179:
+; O1-NEXT:  .Lpcsection194:
 ; O1-NEXT:    lock andl $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -5445,7 +5490,7 @@ define void @atomic32_and_monotonic(ptr %a) {
 ; O2-LABEL: atomic32_and_monotonic:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection179:
+; O2-NEXT:  .Lpcsection194:
 ; O2-NEXT:    lock andl $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -5453,7 +5498,7 @@ define void @atomic32_and_monotonic(ptr %a) {
 ; O3-LABEL: atomic32_and_monotonic:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection179:
+; O3-NEXT:  .Lpcsection194:
 ; O3-NEXT:    lock andl $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -5476,7 +5521,7 @@ define void @atomic32_or_monotonic(ptr %a) {
 ; O1-LABEL: atomic32_or_monotonic:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection180:
+; O1-NEXT:  .Lpcsection195:
 ; O1-NEXT:    lock orl $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -5484,7 +5529,7 @@ define void @atomic32_or_monotonic(ptr %a) {
 ; O2-LABEL: atomic32_or_monotonic:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection180:
+; O2-NEXT:  .Lpcsection195:
 ; O2-NEXT:    lock orl $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -5492,7 +5537,7 @@ define void @atomic32_or_monotonic(ptr %a) {
 ; O3-LABEL: atomic32_or_monotonic:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection180:
+; O3-NEXT:  .Lpcsection195:
 ; O3-NEXT:    lock orl $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -5515,7 +5560,7 @@ define void @atomic32_xor_monotonic(ptr %a) {
 ; O1-LABEL: atomic32_xor_monotonic:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection181:
+; O1-NEXT:  .Lpcsection196:
 ; O1-NEXT:    lock xorl $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -5523,7 +5568,7 @@ define void @atomic32_xor_monotonic(ptr %a) {
 ; O2-LABEL: atomic32_xor_monotonic:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection181:
+; O2-NEXT:  .Lpcsection196:
 ; O2-NEXT:    lock xorl $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -5531,7 +5576,7 @@ define void @atomic32_xor_monotonic(ptr %a) {
 ; O3-LABEL: atomic32_xor_monotonic:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection181:
+; O3-NEXT:  .Lpcsection196:
 ; O3-NEXT:    lock xorl $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -5576,19 +5621,19 @@ define void @atomic32_nand_monotonic(ptr %a) {
 ; O1-LABEL: atomic32_nand_monotonic:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection182:
+; O1-NEXT:  .Lpcsection197:
 ; O1-NEXT:    movl (%rdi), %eax
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB112_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ecx
-; O1-NEXT:  .Lpcsection183:
+; O1-NEXT:  .Lpcsection198:
 ; O1-NEXT:    notl %ecx
-; O1-NEXT:  .Lpcsection184:
+; O1-NEXT:  .Lpcsection199:
 ; O1-NEXT:    orl $-43, %ecx
-; O1-NEXT:  .Lpcsection185:
+; O1-NEXT:  .Lpcsection200:
 ; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
-; O1-NEXT:  .Lpcsection186:
+; O1-NEXT:  .Lpcsection201:
 ; O1-NEXT:    jne .LBB112_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -5597,19 +5642,19 @@ define void @atomic32_nand_monotonic(ptr %a) {
 ; O2-LABEL: atomic32_nand_monotonic:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection182:
+; O2-NEXT:  .Lpcsection197:
 ; O2-NEXT:    movl (%rdi), %eax
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB112_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ecx
-; O2-NEXT:  .Lpcsection183:
+; O2-NEXT:  .Lpcsection198:
 ; O2-NEXT:    notl %ecx
-; O2-NEXT:  .Lpcsection184:
+; O2-NEXT:  .Lpcsection199:
 ; O2-NEXT:    orl $-43, %ecx
-; O2-NEXT:  .Lpcsection185:
+; O2-NEXT:  .Lpcsection200:
 ; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
-; O2-NEXT:  .Lpcsection186:
+; O2-NEXT:  .Lpcsection201:
 ; O2-NEXT:    jne .LBB112_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -5618,19 +5663,19 @@ define void @atomic32_nand_monotonic(ptr %a) {
 ; O3-LABEL: atomic32_nand_monotonic:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection182:
+; O3-NEXT:  .Lpcsection197:
 ; O3-NEXT:    movl (%rdi), %eax
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB112_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ecx
-; O3-NEXT:  .Lpcsection183:
+; O3-NEXT:  .Lpcsection198:
 ; O3-NEXT:    notl %ecx
-; O3-NEXT:  .Lpcsection184:
+; O3-NEXT:  .Lpcsection199:
 ; O3-NEXT:    orl $-43, %ecx
-; O3-NEXT:  .Lpcsection185:
+; O3-NEXT:  .Lpcsection200:
 ; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
-; O3-NEXT:  .Lpcsection186:
+; O3-NEXT:  .Lpcsection201:
 ; O3-NEXT:    jne .LBB112_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -5656,7 +5701,7 @@ define void @atomic32_xchg_acquire(ptr %a) {
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection187:
+; O1-NEXT:  .Lpcsection202:
 ; O1-NEXT:    xchgl %eax, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -5665,7 +5710,7 @@ define void @atomic32_xchg_acquire(ptr %a) {
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection187:
+; O2-NEXT:  .Lpcsection202:
 ; O2-NEXT:    xchgl %eax, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -5674,7 +5719,7 @@ define void @atomic32_xchg_acquire(ptr %a) {
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection187:
+; O3-NEXT:  .Lpcsection202:
 ; O3-NEXT:    xchgl %eax, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -5697,7 +5742,7 @@ define void @atomic32_add_acquire(ptr %a) {
 ; O1-LABEL: atomic32_add_acquire:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection188:
+; O1-NEXT:  .Lpcsection203:
 ; O1-NEXT:    lock addl $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -5705,7 +5750,7 @@ define void @atomic32_add_acquire(ptr %a) {
 ; O2-LABEL: atomic32_add_acquire:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection188:
+; O2-NEXT:  .Lpcsection203:
 ; O2-NEXT:    lock addl $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -5713,7 +5758,7 @@ define void @atomic32_add_acquire(ptr %a) {
 ; O3-LABEL: atomic32_add_acquire:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection188:
+; O3-NEXT:  .Lpcsection203:
 ; O3-NEXT:    lock addl $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -5736,7 +5781,7 @@ define void @atomic32_sub_acquire(ptr %a) {
 ; O1-LABEL: atomic32_sub_acquire:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection189:
+; O1-NEXT:  .Lpcsection204:
 ; O1-NEXT:    lock subl $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -5744,7 +5789,7 @@ define void @atomic32_sub_acquire(ptr %a) {
 ; O2-LABEL: atomic32_sub_acquire:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection189:
+; O2-NEXT:  .Lpcsection204:
 ; O2-NEXT:    lock subl $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -5752,7 +5797,7 @@ define void @atomic32_sub_acquire(ptr %a) {
 ; O3-LABEL: atomic32_sub_acquire:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection189:
+; O3-NEXT:  .Lpcsection204:
 ; O3-NEXT:    lock subl $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -5775,7 +5820,7 @@ define void @atomic32_and_acquire(ptr %a) {
 ; O1-LABEL: atomic32_and_acquire:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection190:
+; O1-NEXT:  .Lpcsection205:
 ; O1-NEXT:    lock andl $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -5783,7 +5828,7 @@ define void @atomic32_and_acquire(ptr %a) {
 ; O2-LABEL: atomic32_and_acquire:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection190:
+; O2-NEXT:  .Lpcsection205:
 ; O2-NEXT:    lock andl $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -5791,7 +5836,7 @@ define void @atomic32_and_acquire(ptr %a) {
 ; O3-LABEL: atomic32_and_acquire:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection190:
+; O3-NEXT:  .Lpcsection205:
 ; O3-NEXT:    lock andl $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -5814,7 +5859,7 @@ define void @atomic32_or_acquire(ptr %a) {
 ; O1-LABEL: atomic32_or_acquire:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection191:
+; O1-NEXT:  .Lpcsection206:
 ; O1-NEXT:    lock orl $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -5822,7 +5867,7 @@ define void @atomic32_or_acquire(ptr %a) {
 ; O2-LABEL: atomic32_or_acquire:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection191:
+; O2-NEXT:  .Lpcsection206:
 ; O2-NEXT:    lock orl $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -5830,7 +5875,7 @@ define void @atomic32_or_acquire(ptr %a) {
 ; O3-LABEL: atomic32_or_acquire:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection191:
+; O3-NEXT:  .Lpcsection206:
 ; O3-NEXT:    lock orl $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -5853,7 +5898,7 @@ define void @atomic32_xor_acquire(ptr %a) {
 ; O1-LABEL: atomic32_xor_acquire:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection192:
+; O1-NEXT:  .Lpcsection207:
 ; O1-NEXT:    lock xorl $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -5861,7 +5906,7 @@ define void @atomic32_xor_acquire(ptr %a) {
 ; O2-LABEL: atomic32_xor_acquire:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection192:
+; O2-NEXT:  .Lpcsection207:
 ; O2-NEXT:    lock xorl $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -5869,7 +5914,7 @@ define void @atomic32_xor_acquire(ptr %a) {
 ; O3-LABEL: atomic32_xor_acquire:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection192:
+; O3-NEXT:  .Lpcsection207:
 ; O3-NEXT:    lock xorl $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -5914,19 +5959,19 @@ define void @atomic32_nand_acquire(ptr %a) {
 ; O1-LABEL: atomic32_nand_acquire:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection193:
+; O1-NEXT:  .Lpcsection208:
 ; O1-NEXT:    movl (%rdi), %eax
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB119_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ecx
-; O1-NEXT:  .Lpcsection194:
+; O1-NEXT:  .Lpcsection209:
 ; O1-NEXT:    notl %ecx
-; O1-NEXT:  .Lpcsection195:
+; O1-NEXT:  .Lpcsection210:
 ; O1-NEXT:    orl $-43, %ecx
-; O1-NEXT:  .Lpcsection196:
+; O1-NEXT:  .Lpcsection211:
 ; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
-; O1-NEXT:  .Lpcsection197:
+; O1-NEXT:  .Lpcsection212:
 ; O1-NEXT:    jne .LBB119_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -5935,19 +5980,19 @@ define void @atomic32_nand_acquire(ptr %a) {
 ; O2-LABEL: atomic32_nand_acquire:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection193:
+; O2-NEXT:  .Lpcsection208:
 ; O2-NEXT:    movl (%rdi), %eax
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB119_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ecx
-; O2-NEXT:  .Lpcsection194:
+; O2-NEXT:  .Lpcsection209:
 ; O2-NEXT:    notl %ecx
-; O2-NEXT:  .Lpcsection195:
+; O2-NEXT:  .Lpcsection210:
 ; O2-NEXT:    orl $-43, %ecx
-; O2-NEXT:  .Lpcsection196:
+; O2-NEXT:  .Lpcsection211:
 ; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
-; O2-NEXT:  .Lpcsection197:
+; O2-NEXT:  .Lpcsection212:
 ; O2-NEXT:    jne .LBB119_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -5956,19 +6001,19 @@ define void @atomic32_nand_acquire(ptr %a) {
 ; O3-LABEL: atomic32_nand_acquire:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection193:
+; O3-NEXT:  .Lpcsection208:
 ; O3-NEXT:    movl (%rdi), %eax
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB119_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ecx
-; O3-NEXT:  .Lpcsection194:
+; O3-NEXT:  .Lpcsection209:
 ; O3-NEXT:    notl %ecx
-; O3-NEXT:  .Lpcsection195:
+; O3-NEXT:  .Lpcsection210:
 ; O3-NEXT:    orl $-43, %ecx
-; O3-NEXT:  .Lpcsection196:
+; O3-NEXT:  .Lpcsection211:
 ; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
-; O3-NEXT:  .Lpcsection197:
+; O3-NEXT:  .Lpcsection212:
 ; O3-NEXT:    jne .LBB119_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -5994,7 +6039,7 @@ define void @atomic32_xchg_release(ptr %a) {
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection198:
+; O1-NEXT:  .Lpcsection213:
 ; O1-NEXT:    xchgl %eax, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -6003,7 +6048,7 @@ define void @atomic32_xchg_release(ptr %a) {
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection198:
+; O2-NEXT:  .Lpcsection213:
 ; O2-NEXT:    xchgl %eax, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -6012,7 +6057,7 @@ define void @atomic32_xchg_release(ptr %a) {
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection198:
+; O3-NEXT:  .Lpcsection213:
 ; O3-NEXT:    xchgl %eax, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -6035,7 +6080,7 @@ define void @atomic32_add_release(ptr %a) {
 ; O1-LABEL: atomic32_add_release:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection199:
+; O1-NEXT:  .Lpcsection214:
 ; O1-NEXT:    lock addl $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -6043,7 +6088,7 @@ define void @atomic32_add_release(ptr %a) {
 ; O2-LABEL: atomic32_add_release:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection199:
+; O2-NEXT:  .Lpcsection214:
 ; O2-NEXT:    lock addl $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -6051,7 +6096,7 @@ define void @atomic32_add_release(ptr %a) {
 ; O3-LABEL: atomic32_add_release:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection199:
+; O3-NEXT:  .Lpcsection214:
 ; O3-NEXT:    lock addl $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -6074,7 +6119,7 @@ define void @atomic32_sub_release(ptr %a) {
 ; O1-LABEL: atomic32_sub_release:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection200:
+; O1-NEXT:  .Lpcsection215:
 ; O1-NEXT:    lock subl $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -6082,7 +6127,7 @@ define void @atomic32_sub_release(ptr %a) {
 ; O2-LABEL: atomic32_sub_release:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection200:
+; O2-NEXT:  .Lpcsection215:
 ; O2-NEXT:    lock subl $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -6090,7 +6135,7 @@ define void @atomic32_sub_release(ptr %a) {
 ; O3-LABEL: atomic32_sub_release:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection200:
+; O3-NEXT:  .Lpcsection215:
 ; O3-NEXT:    lock subl $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -6113,7 +6158,7 @@ define void @atomic32_and_release(ptr %a) {
 ; O1-LABEL: atomic32_and_release:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection201:
+; O1-NEXT:  .Lpcsection216:
 ; O1-NEXT:    lock andl $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -6121,7 +6166,7 @@ define void @atomic32_and_release(ptr %a) {
 ; O2-LABEL: atomic32_and_release:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection201:
+; O2-NEXT:  .Lpcsection216:
 ; O2-NEXT:    lock andl $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -6129,7 +6174,7 @@ define void @atomic32_and_release(ptr %a) {
 ; O3-LABEL: atomic32_and_release:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection201:
+; O3-NEXT:  .Lpcsection216:
 ; O3-NEXT:    lock andl $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -6152,7 +6197,7 @@ define void @atomic32_or_release(ptr %a) {
 ; O1-LABEL: atomic32_or_release:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection202:
+; O1-NEXT:  .Lpcsection217:
 ; O1-NEXT:    lock orl $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -6160,7 +6205,7 @@ define void @atomic32_or_release(ptr %a) {
 ; O2-LABEL: atomic32_or_release:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection202:
+; O2-NEXT:  .Lpcsection217:
 ; O2-NEXT:    lock orl $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -6168,7 +6213,7 @@ define void @atomic32_or_release(ptr %a) {
 ; O3-LABEL: atomic32_or_release:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection202:
+; O3-NEXT:  .Lpcsection217:
 ; O3-NEXT:    lock orl $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -6191,7 +6236,7 @@ define void @atomic32_xor_release(ptr %a) {
 ; O1-LABEL: atomic32_xor_release:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection203:
+; O1-NEXT:  .Lpcsection218:
 ; O1-NEXT:    lock xorl $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -6199,7 +6244,7 @@ define void @atomic32_xor_release(ptr %a) {
 ; O2-LABEL: atomic32_xor_release:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection203:
+; O2-NEXT:  .Lpcsection218:
 ; O2-NEXT:    lock xorl $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -6207,7 +6252,7 @@ define void @atomic32_xor_release(ptr %a) {
 ; O3-LABEL: atomic32_xor_release:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection203:
+; O3-NEXT:  .Lpcsection218:
 ; O3-NEXT:    lock xorl $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -6252,19 +6297,19 @@ define void @atomic32_nand_release(ptr %a) {
 ; O1-LABEL: atomic32_nand_release:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection204:
+; O1-NEXT:  .Lpcsection219:
 ; O1-NEXT:    movl (%rdi), %eax
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB126_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ecx
-; O1-NEXT:  .Lpcsection205:
+; O1-NEXT:  .Lpcsection220:
 ; O1-NEXT:    notl %ecx
-; O1-NEXT:  .Lpcsection206:
+; O1-NEXT:  .Lpcsection221:
 ; O1-NEXT:    orl $-43, %ecx
-; O1-NEXT:  .Lpcsection207:
+; O1-NEXT:  .Lpcsection222:
 ; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
-; O1-NEXT:  .Lpcsection208:
+; O1-NEXT:  .Lpcsection223:
 ; O1-NEXT:    jne .LBB126_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -6273,19 +6318,19 @@ define void @atomic32_nand_release(ptr %a) {
 ; O2-LABEL: atomic32_nand_release:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection204:
+; O2-NEXT:  .Lpcsection219:
 ; O2-NEXT:    movl (%rdi), %eax
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB126_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ecx
-; O2-NEXT:  .Lpcsection205:
+; O2-NEXT:  .Lpcsection220:
 ; O2-NEXT:    notl %ecx
-; O2-NEXT:  .Lpcsection206:
+; O2-NEXT:  .Lpcsection221:
 ; O2-NEXT:    orl $-43, %ecx
-; O2-NEXT:  .Lpcsection207:
+; O2-NEXT:  .Lpcsection222:
 ; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
-; O2-NEXT:  .Lpcsection208:
+; O2-NEXT:  .Lpcsection223:
 ; O2-NEXT:    jne .LBB126_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -6294,19 +6339,19 @@ define void @atomic32_nand_release(ptr %a) {
 ; O3-LABEL: atomic32_nand_release:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection204:
+; O3-NEXT:  .Lpcsection219:
 ; O3-NEXT:    movl (%rdi), %eax
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB126_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ecx
-; O3-NEXT:  .Lpcsection205:
+; O3-NEXT:  .Lpcsection220:
 ; O3-NEXT:    notl %ecx
-; O3-NEXT:  .Lpcsection206:
+; O3-NEXT:  .Lpcsection221:
 ; O3-NEXT:    orl $-43, %ecx
-; O3-NEXT:  .Lpcsection207:
+; O3-NEXT:  .Lpcsection222:
 ; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
-; O3-NEXT:  .Lpcsection208:
+; O3-NEXT:  .Lpcsection223:
 ; O3-NEXT:    jne .LBB126_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -6332,7 +6377,7 @@ define void @atomic32_xchg_acq_rel(ptr %a) {
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection209:
+; O1-NEXT:  .Lpcsection224:
 ; O1-NEXT:    xchgl %eax, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -6341,7 +6386,7 @@ define void @atomic32_xchg_acq_rel(ptr %a) {
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection209:
+; O2-NEXT:  .Lpcsection224:
 ; O2-NEXT:    xchgl %eax, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -6350,7 +6395,7 @@ define void @atomic32_xchg_acq_rel(ptr %a) {
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection209:
+; O3-NEXT:  .Lpcsection224:
 ; O3-NEXT:    xchgl %eax, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -6373,7 +6418,7 @@ define void @atomic32_add_acq_rel(ptr %a) {
 ; O1-LABEL: atomic32_add_acq_rel:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection210:
+; O1-NEXT:  .Lpcsection225:
 ; O1-NEXT:    lock addl $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -6381,7 +6426,7 @@ define void @atomic32_add_acq_rel(ptr %a) {
 ; O2-LABEL: atomic32_add_acq_rel:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection210:
+; O2-NEXT:  .Lpcsection225:
 ; O2-NEXT:    lock addl $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -6389,7 +6434,7 @@ define void @atomic32_add_acq_rel(ptr %a) {
 ; O3-LABEL: atomic32_add_acq_rel:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection210:
+; O3-NEXT:  .Lpcsection225:
 ; O3-NEXT:    lock addl $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -6412,7 +6457,7 @@ define void @atomic32_sub_acq_rel(ptr %a) {
 ; O1-LABEL: atomic32_sub_acq_rel:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection211:
+; O1-NEXT:  .Lpcsection226:
 ; O1-NEXT:    lock subl $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -6420,7 +6465,7 @@ define void @atomic32_sub_acq_rel(ptr %a) {
 ; O2-LABEL: atomic32_sub_acq_rel:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection211:
+; O2-NEXT:  .Lpcsection226:
 ; O2-NEXT:    lock subl $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -6428,7 +6473,7 @@ define void @atomic32_sub_acq_rel(ptr %a) {
 ; O3-LABEL: atomic32_sub_acq_rel:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection211:
+; O3-NEXT:  .Lpcsection226:
 ; O3-NEXT:    lock subl $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -6451,7 +6496,7 @@ define void @atomic32_and_acq_rel(ptr %a) {
 ; O1-LABEL: atomic32_and_acq_rel:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection212:
+; O1-NEXT:  .Lpcsection227:
 ; O1-NEXT:    lock andl $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -6459,7 +6504,7 @@ define void @atomic32_and_acq_rel(ptr %a) {
 ; O2-LABEL: atomic32_and_acq_rel:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection212:
+; O2-NEXT:  .Lpcsection227:
 ; O2-NEXT:    lock andl $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -6467,7 +6512,7 @@ define void @atomic32_and_acq_rel(ptr %a) {
 ; O3-LABEL: atomic32_and_acq_rel:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection212:
+; O3-NEXT:  .Lpcsection227:
 ; O3-NEXT:    lock andl $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -6490,7 +6535,7 @@ define void @atomic32_or_acq_rel(ptr %a) {
 ; O1-LABEL: atomic32_or_acq_rel:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection213:
+; O1-NEXT:  .Lpcsection228:
 ; O1-NEXT:    lock orl $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -6498,7 +6543,7 @@ define void @atomic32_or_acq_rel(ptr %a) {
 ; O2-LABEL: atomic32_or_acq_rel:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection213:
+; O2-NEXT:  .Lpcsection228:
 ; O2-NEXT:    lock orl $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -6506,7 +6551,7 @@ define void @atomic32_or_acq_rel(ptr %a) {
 ; O3-LABEL: atomic32_or_acq_rel:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection213:
+; O3-NEXT:  .Lpcsection228:
 ; O3-NEXT:    lock orl $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -6529,7 +6574,7 @@ define void @atomic32_xor_acq_rel(ptr %a) {
 ; O1-LABEL: atomic32_xor_acq_rel:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection214:
+; O1-NEXT:  .Lpcsection229:
 ; O1-NEXT:    lock xorl $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -6537,7 +6582,7 @@ define void @atomic32_xor_acq_rel(ptr %a) {
 ; O2-LABEL: atomic32_xor_acq_rel:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection214:
+; O2-NEXT:  .Lpcsection229:
 ; O2-NEXT:    lock xorl $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -6545,7 +6590,7 @@ define void @atomic32_xor_acq_rel(ptr %a) {
 ; O3-LABEL: atomic32_xor_acq_rel:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection214:
+; O3-NEXT:  .Lpcsection229:
 ; O3-NEXT:    lock xorl $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -6590,19 +6635,19 @@ define void @atomic32_nand_acq_rel(ptr %a) {
 ; O1-LABEL: atomic32_nand_acq_rel:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection215:
+; O1-NEXT:  .Lpcsection230:
 ; O1-NEXT:    movl (%rdi), %eax
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB133_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ecx
-; O1-NEXT:  .Lpcsection216:
+; O1-NEXT:  .Lpcsection231:
 ; O1-NEXT:    notl %ecx
-; O1-NEXT:  .Lpcsection217:
+; O1-NEXT:  .Lpcsection232:
 ; O1-NEXT:    orl $-43, %ecx
-; O1-NEXT:  .Lpcsection218:
+; O1-NEXT:  .Lpcsection233:
 ; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
-; O1-NEXT:  .Lpcsection219:
+; O1-NEXT:  .Lpcsection234:
 ; O1-NEXT:    jne .LBB133_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -6611,19 +6656,19 @@ define void @atomic32_nand_acq_rel(ptr %a) {
 ; O2-LABEL: atomic32_nand_acq_rel:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection215:
+; O2-NEXT:  .Lpcsection230:
 ; O2-NEXT:    movl (%rdi), %eax
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB133_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ecx
-; O2-NEXT:  .Lpcsection216:
+; O2-NEXT:  .Lpcsection231:
 ; O2-NEXT:    notl %ecx
-; O2-NEXT:  .Lpcsection217:
+; O2-NEXT:  .Lpcsection232:
 ; O2-NEXT:    orl $-43, %ecx
-; O2-NEXT:  .Lpcsection218:
+; O2-NEXT:  .Lpcsection233:
 ; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
-; O2-NEXT:  .Lpcsection219:
+; O2-NEXT:  .Lpcsection234:
 ; O2-NEXT:    jne .LBB133_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -6632,19 +6677,19 @@ define void @atomic32_nand_acq_rel(ptr %a) {
 ; O3-LABEL: atomic32_nand_acq_rel:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection215:
+; O3-NEXT:  .Lpcsection230:
 ; O3-NEXT:    movl (%rdi), %eax
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB133_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ecx
-; O3-NEXT:  .Lpcsection216:
+; O3-NEXT:  .Lpcsection231:
 ; O3-NEXT:    notl %ecx
-; O3-NEXT:  .Lpcsection217:
+; O3-NEXT:  .Lpcsection232:
 ; O3-NEXT:    orl $-43, %ecx
-; O3-NEXT:  .Lpcsection218:
+; O3-NEXT:  .Lpcsection233:
 ; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
-; O3-NEXT:  .Lpcsection219:
+; O3-NEXT:  .Lpcsection234:
 ; O3-NEXT:    jne .LBB133_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -6670,7 +6715,7 @@ define void @atomic32_xchg_seq_cst(ptr %a) {
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection220:
+; O1-NEXT:  .Lpcsection235:
 ; O1-NEXT:    xchgl %eax, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -6679,7 +6724,7 @@ define void @atomic32_xchg_seq_cst(ptr %a) {
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection220:
+; O2-NEXT:  .Lpcsection235:
 ; O2-NEXT:    xchgl %eax, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -6688,7 +6733,7 @@ define void @atomic32_xchg_seq_cst(ptr %a) {
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection220:
+; O3-NEXT:  .Lpcsection235:
 ; O3-NEXT:    xchgl %eax, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -6711,7 +6756,7 @@ define void @atomic32_add_seq_cst(ptr %a) {
 ; O1-LABEL: atomic32_add_seq_cst:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection221:
+; O1-NEXT:  .Lpcsection236:
 ; O1-NEXT:    lock addl $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -6719,7 +6764,7 @@ define void @atomic32_add_seq_cst(ptr %a) {
 ; O2-LABEL: atomic32_add_seq_cst:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection221:
+; O2-NEXT:  .Lpcsection236:
 ; O2-NEXT:    lock addl $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -6727,7 +6772,7 @@ define void @atomic32_add_seq_cst(ptr %a) {
 ; O3-LABEL: atomic32_add_seq_cst:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection221:
+; O3-NEXT:  .Lpcsection236:
 ; O3-NEXT:    lock addl $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -6750,7 +6795,7 @@ define void @atomic32_sub_seq_cst(ptr %a) {
 ; O1-LABEL: atomic32_sub_seq_cst:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection222:
+; O1-NEXT:  .Lpcsection237:
 ; O1-NEXT:    lock subl $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -6758,7 +6803,7 @@ define void @atomic32_sub_seq_cst(ptr %a) {
 ; O2-LABEL: atomic32_sub_seq_cst:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection222:
+; O2-NEXT:  .Lpcsection237:
 ; O2-NEXT:    lock subl $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -6766,7 +6811,7 @@ define void @atomic32_sub_seq_cst(ptr %a) {
 ; O3-LABEL: atomic32_sub_seq_cst:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection222:
+; O3-NEXT:  .Lpcsection237:
 ; O3-NEXT:    lock subl $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -6789,7 +6834,7 @@ define void @atomic32_and_seq_cst(ptr %a) {
 ; O1-LABEL: atomic32_and_seq_cst:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection223:
+; O1-NEXT:  .Lpcsection238:
 ; O1-NEXT:    lock andl $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -6797,7 +6842,7 @@ define void @atomic32_and_seq_cst(ptr %a) {
 ; O2-LABEL: atomic32_and_seq_cst:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection223:
+; O2-NEXT:  .Lpcsection238:
 ; O2-NEXT:    lock andl $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -6805,7 +6850,7 @@ define void @atomic32_and_seq_cst(ptr %a) {
 ; O3-LABEL: atomic32_and_seq_cst:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection223:
+; O3-NEXT:  .Lpcsection238:
 ; O3-NEXT:    lock andl $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -6828,7 +6873,7 @@ define void @atomic32_or_seq_cst(ptr %a) {
 ; O1-LABEL: atomic32_or_seq_cst:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection224:
+; O1-NEXT:  .Lpcsection239:
 ; O1-NEXT:    lock orl $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -6836,7 +6881,7 @@ define void @atomic32_or_seq_cst(ptr %a) {
 ; O2-LABEL: atomic32_or_seq_cst:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection224:
+; O2-NEXT:  .Lpcsection239:
 ; O2-NEXT:    lock orl $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -6844,7 +6889,7 @@ define void @atomic32_or_seq_cst(ptr %a) {
 ; O3-LABEL: atomic32_or_seq_cst:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection224:
+; O3-NEXT:  .Lpcsection239:
 ; O3-NEXT:    lock orl $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -6867,7 +6912,7 @@ define void @atomic32_xor_seq_cst(ptr %a) {
 ; O1-LABEL: atomic32_xor_seq_cst:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection225:
+; O1-NEXT:  .Lpcsection240:
 ; O1-NEXT:    lock xorl $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -6875,7 +6920,7 @@ define void @atomic32_xor_seq_cst(ptr %a) {
 ; O2-LABEL: atomic32_xor_seq_cst:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection225:
+; O2-NEXT:  .Lpcsection240:
 ; O2-NEXT:    lock xorl $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -6883,7 +6928,7 @@ define void @atomic32_xor_seq_cst(ptr %a) {
 ; O3-LABEL: atomic32_xor_seq_cst:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection225:
+; O3-NEXT:  .Lpcsection240:
 ; O3-NEXT:    lock xorl $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -6928,19 +6973,19 @@ define void @atomic32_nand_seq_cst(ptr %a) {
 ; O1-LABEL: atomic32_nand_seq_cst:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection226:
+; O1-NEXT:  .Lpcsection241:
 ; O1-NEXT:    movl (%rdi), %eax
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB140_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ecx
-; O1-NEXT:  .Lpcsection227:
+; O1-NEXT:  .Lpcsection242:
 ; O1-NEXT:    notl %ecx
-; O1-NEXT:  .Lpcsection228:
+; O1-NEXT:  .Lpcsection243:
 ; O1-NEXT:    orl $-43, %ecx
-; O1-NEXT:  .Lpcsection229:
+; O1-NEXT:  .Lpcsection244:
 ; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
-; O1-NEXT:  .Lpcsection230:
+; O1-NEXT:  .Lpcsection245:
 ; O1-NEXT:    jne .LBB140_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -6949,19 +6994,19 @@ define void @atomic32_nand_seq_cst(ptr %a) {
 ; O2-LABEL: atomic32_nand_seq_cst:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection226:
+; O2-NEXT:  .Lpcsection241:
 ; O2-NEXT:    movl (%rdi), %eax
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB140_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ecx
-; O2-NEXT:  .Lpcsection227:
+; O2-NEXT:  .Lpcsection242:
 ; O2-NEXT:    notl %ecx
-; O2-NEXT:  .Lpcsection228:
+; O2-NEXT:  .Lpcsection243:
 ; O2-NEXT:    orl $-43, %ecx
-; O2-NEXT:  .Lpcsection229:
+; O2-NEXT:  .Lpcsection244:
 ; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
-; O2-NEXT:  .Lpcsection230:
+; O2-NEXT:  .Lpcsection245:
 ; O2-NEXT:    jne .LBB140_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -6970,19 +7015,19 @@ define void @atomic32_nand_seq_cst(ptr %a) {
 ; O3-LABEL: atomic32_nand_seq_cst:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection226:
+; O3-NEXT:  .Lpcsection241:
 ; O3-NEXT:    movl (%rdi), %eax
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB140_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ecx
-; O3-NEXT:  .Lpcsection227:
+; O3-NEXT:  .Lpcsection242:
 ; O3-NEXT:    notl %ecx
-; O3-NEXT:  .Lpcsection228:
+; O3-NEXT:  .Lpcsection243:
 ; O3-NEXT:    orl $-43, %ecx
-; O3-NEXT:  .Lpcsection229:
+; O3-NEXT:  .Lpcsection244:
 ; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
-; O3-NEXT:  .Lpcsection230:
+; O3-NEXT:  .Lpcsection245:
 ; O3-NEXT:    jne .LBB140_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -7020,14 +7065,17 @@ define void @atomic32_cas_monotonic(ptr %a) {
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movl $1, %ecx
+; O1-NEXT:  .Lpcsection246:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection231:
+; O1-NEXT:  .Lpcsection247:
 ; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
+; O1-NEXT:  .Lpcsection248:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection232:
+; O1-NEXT:  .Lpcsection249:
 ; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
+; O1-NEXT:  .Lpcsection250:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection233:
+; O1-NEXT:  .Lpcsection251:
 ; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -7036,14 +7084,17 @@ define void @atomic32_cas_monotonic(ptr %a) {
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movl $1, %ecx
+; O2-NEXT:  .Lpcsection246:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection231:
+; O2-NEXT:  .Lpcsection247:
 ; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
+; O2-NEXT:  .Lpcsection248:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection232:
+; O2-NEXT:  .Lpcsection249:
 ; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
+; O2-NEXT:  .Lpcsection250:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection233:
+; O2-NEXT:  .Lpcsection251:
 ; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -7052,14 +7103,17 @@ define void @atomic32_cas_monotonic(ptr %a) {
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movl $1, %ecx
+; O3-NEXT:  .Lpcsection246:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection231:
+; O3-NEXT:  .Lpcsection247:
 ; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
+; O3-NEXT:  .Lpcsection248:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection232:
+; O3-NEXT:  .Lpcsection249:
 ; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
+; O3-NEXT:  .Lpcsection250:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection233:
+; O3-NEXT:  .Lpcsection251:
 ; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -7098,14 +7152,17 @@ define void @atomic32_cas_acquire(ptr %a) {
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movl $1, %ecx
+; O1-NEXT:  .Lpcsection252:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection234:
+; O1-NEXT:  .Lpcsection253:
 ; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
+; O1-NEXT:  .Lpcsection254:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection235:
+; O1-NEXT:  .Lpcsection255:
 ; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
+; O1-NEXT:  .Lpcsection256:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection236:
+; O1-NEXT:  .Lpcsection257:
 ; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -7114,14 +7171,17 @@ define void @atomic32_cas_acquire(ptr %a) {
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movl $1, %ecx
+; O2-NEXT:  .Lpcsection252:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection234:
+; O2-NEXT:  .Lpcsection253:
 ; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
+; O2-NEXT:  .Lpcsection254:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection235:
+; O2-NEXT:  .Lpcsection255:
 ; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
+; O2-NEXT:  .Lpcsection256:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection236:
+; O2-NEXT:  .Lpcsection257:
 ; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -7130,14 +7190,17 @@ define void @atomic32_cas_acquire(ptr %a) {
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movl $1, %ecx
+; O3-NEXT:  .Lpcsection252:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection234:
+; O3-NEXT:  .Lpcsection253:
 ; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
+; O3-NEXT:  .Lpcsection254:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection235:
+; O3-NEXT:  .Lpcsection255:
 ; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
+; O3-NEXT:  .Lpcsection256:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection236:
+; O3-NEXT:  .Lpcsection257:
 ; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -7176,14 +7239,17 @@ define void @atomic32_cas_release(ptr %a) {
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movl $1, %ecx
+; O1-NEXT:  .Lpcsection258:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection237:
+; O1-NEXT:  .Lpcsection259:
 ; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
+; O1-NEXT:  .Lpcsection260:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection238:
+; O1-NEXT:  .Lpcsection261:
 ; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
+; O1-NEXT:  .Lpcsection262:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection239:
+; O1-NEXT:  .Lpcsection263:
 ; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -7192,14 +7258,17 @@ define void @atomic32_cas_release(ptr %a) {
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movl $1, %ecx
+; O2-NEXT:  .Lpcsection258:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection237:
+; O2-NEXT:  .Lpcsection259:
 ; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
+; O2-NEXT:  .Lpcsection260:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection238:
+; O2-NEXT:  .Lpcsection261:
 ; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
+; O2-NEXT:  .Lpcsection262:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection239:
+; O2-NEXT:  .Lpcsection263:
 ; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -7208,14 +7277,17 @@ define void @atomic32_cas_release(ptr %a) {
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movl $1, %ecx
+; O3-NEXT:  .Lpcsection258:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection237:
+; O3-NEXT:  .Lpcsection259:
 ; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
+; O3-NEXT:  .Lpcsection260:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection238:
+; O3-NEXT:  .Lpcsection261:
 ; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
+; O3-NEXT:  .Lpcsection262:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection239:
+; O3-NEXT:  .Lpcsection263:
 ; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -7254,14 +7326,17 @@ define void @atomic32_cas_acq_rel(ptr %a) {
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movl $1, %ecx
+; O1-NEXT:  .Lpcsection264:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection240:
+; O1-NEXT:  .Lpcsection265:
 ; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
+; O1-NEXT:  .Lpcsection266:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection241:
+; O1-NEXT:  .Lpcsection267:
 ; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
+; O1-NEXT:  .Lpcsection268:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection242:
+; O1-NEXT:  .Lpcsection269:
 ; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -7270,14 +7345,17 @@ define void @atomic32_cas_acq_rel(ptr %a) {
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movl $1, %ecx
+; O2-NEXT:  .Lpcsection264:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection240:
+; O2-NEXT:  .Lpcsection265:
 ; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
+; O2-NEXT:  .Lpcsection266:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection241:
+; O2-NEXT:  .Lpcsection267:
 ; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
+; O2-NEXT:  .Lpcsection268:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection242:
+; O2-NEXT:  .Lpcsection269:
 ; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -7286,14 +7364,17 @@ define void @atomic32_cas_acq_rel(ptr %a) {
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movl $1, %ecx
+; O3-NEXT:  .Lpcsection264:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection240:
+; O3-NEXT:  .Lpcsection265:
 ; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
+; O3-NEXT:  .Lpcsection266:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection241:
+; O3-NEXT:  .Lpcsection267:
 ; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
+; O3-NEXT:  .Lpcsection268:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection242:
+; O3-NEXT:  .Lpcsection269:
 ; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -7332,14 +7413,17 @@ define void @atomic32_cas_seq_cst(ptr %a) {
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movl $1, %ecx
+; O1-NEXT:  .Lpcsection270:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection243:
+; O1-NEXT:  .Lpcsection271:
 ; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
+; O1-NEXT:  .Lpcsection272:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection244:
+; O1-NEXT:  .Lpcsection273:
 ; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
+; O1-NEXT:  .Lpcsection274:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection245:
+; O1-NEXT:  .Lpcsection275:
 ; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -7348,14 +7432,17 @@ define void @atomic32_cas_seq_cst(ptr %a) {
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movl $1, %ecx
+; O2-NEXT:  .Lpcsection270:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection243:
+; O2-NEXT:  .Lpcsection271:
 ; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
+; O2-NEXT:  .Lpcsection272:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection244:
+; O2-NEXT:  .Lpcsection273:
 ; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
+; O2-NEXT:  .Lpcsection274:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection245:
+; O2-NEXT:  .Lpcsection275:
 ; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -7364,14 +7451,17 @@ define void @atomic32_cas_seq_cst(ptr %a) {
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movl $1, %ecx
+; O3-NEXT:  .Lpcsection270:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection243:
+; O3-NEXT:  .Lpcsection271:
 ; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
+; O3-NEXT:  .Lpcsection272:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection244:
+; O3-NEXT:  .Lpcsection273:
 ; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
+; O3-NEXT:  .Lpcsection274:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection245:
+; O3-NEXT:  .Lpcsection275:
 ; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -7396,7 +7486,7 @@ define i64 @atomic64_load_unordered(ptr %a) {
 ; O1-LABEL: atomic64_load_unordered:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection246:
+; O1-NEXT:  .Lpcsection276:
 ; O1-NEXT:    movq (%rdi), %rax
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -7404,7 +7494,7 @@ define i64 @atomic64_load_unordered(ptr %a) {
 ; O2-LABEL: atomic64_load_unordered:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection246:
+; O2-NEXT:  .Lpcsection276:
 ; O2-NEXT:    movq (%rdi), %rax
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -7412,7 +7502,7 @@ define i64 @atomic64_load_unordered(ptr %a) {
 ; O3-LABEL: atomic64_load_unordered:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection246:
+; O3-NEXT:  .Lpcsection276:
 ; O3-NEXT:    movq (%rdi), %rax
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -7435,7 +7525,7 @@ define i64 @atomic64_load_monotonic(ptr %a) {
 ; O1-LABEL: atomic64_load_monotonic:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection247:
+; O1-NEXT:  .Lpcsection277:
 ; O1-NEXT:    movq (%rdi), %rax
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -7443,7 +7533,7 @@ define i64 @atomic64_load_monotonic(ptr %a) {
 ; O2-LABEL: atomic64_load_monotonic:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection247:
+; O2-NEXT:  .Lpcsection277:
 ; O2-NEXT:    movq (%rdi), %rax
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -7451,7 +7541,7 @@ define i64 @atomic64_load_monotonic(ptr %a) {
 ; O3-LABEL: atomic64_load_monotonic:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection247:
+; O3-NEXT:  .Lpcsection277:
 ; O3-NEXT:    movq (%rdi), %rax
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -7474,7 +7564,7 @@ define i64 @atomic64_load_acquire(ptr %a) {
 ; O1-LABEL: atomic64_load_acquire:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection248:
+; O1-NEXT:  .Lpcsection278:
 ; O1-NEXT:    movq (%rdi), %rax
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -7482,7 +7572,7 @@ define i64 @atomic64_load_acquire(ptr %a) {
 ; O2-LABEL: atomic64_load_acquire:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection248:
+; O2-NEXT:  .Lpcsection278:
 ; O2-NEXT:    movq (%rdi), %rax
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -7490,7 +7580,7 @@ define i64 @atomic64_load_acquire(ptr %a) {
 ; O3-LABEL: atomic64_load_acquire:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection248:
+; O3-NEXT:  .Lpcsection278:
 ; O3-NEXT:    movq (%rdi), %rax
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -7513,7 +7603,7 @@ define i64 @atomic64_load_seq_cst(ptr %a) {
 ; O1-LABEL: atomic64_load_seq_cst:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection249:
+; O1-NEXT:  .Lpcsection279:
 ; O1-NEXT:    movq (%rdi), %rax
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -7521,7 +7611,7 @@ define i64 @atomic64_load_seq_cst(ptr %a) {
 ; O2-LABEL: atomic64_load_seq_cst:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection249:
+; O2-NEXT:  .Lpcsection279:
 ; O2-NEXT:    movq (%rdi), %rax
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -7529,7 +7619,7 @@ define i64 @atomic64_load_seq_cst(ptr %a) {
 ; O3-LABEL: atomic64_load_seq_cst:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection249:
+; O3-NEXT:  .Lpcsection279:
 ; O3-NEXT:    movq (%rdi), %rax
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -7552,7 +7642,7 @@ define ptr @atomic64_load_seq_cst_ptr_ty(ptr %a) {
 ; O1-LABEL: atomic64_load_seq_cst_ptr_ty:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection250:
+; O1-NEXT:  .Lpcsection280:
 ; O1-NEXT:    movq (%rdi), %rax
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -7560,7 +7650,7 @@ define ptr @atomic64_load_seq_cst_ptr_ty(ptr %a) {
 ; O2-LABEL: atomic64_load_seq_cst_ptr_ty:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection250:
+; O2-NEXT:  .Lpcsection280:
 ; O2-NEXT:    movq (%rdi), %rax
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -7568,7 +7658,7 @@ define ptr @atomic64_load_seq_cst_ptr_ty(ptr %a) {
 ; O3-LABEL: atomic64_load_seq_cst_ptr_ty:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection250:
+; O3-NEXT:  .Lpcsection280:
 ; O3-NEXT:    movq (%rdi), %rax
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -7591,7 +7681,7 @@ define void @atomic64_store_unordered(ptr %a) {
 ; O1-LABEL: atomic64_store_unordered:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection251:
+; O1-NEXT:  .Lpcsection281:
 ; O1-NEXT:    movq $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -7599,7 +7689,7 @@ define void @atomic64_store_unordered(ptr %a) {
 ; O2-LABEL: atomic64_store_unordered:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection251:
+; O2-NEXT:  .Lpcsection281:
 ; O2-NEXT:    movq $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -7607,7 +7697,7 @@ define void @atomic64_store_unordered(ptr %a) {
 ; O3-LABEL: atomic64_store_unordered:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection251:
+; O3-NEXT:  .Lpcsection281:
 ; O3-NEXT:    movq $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -7630,7 +7720,7 @@ define void @atomic64_store_monotonic(ptr %a) {
 ; O1-LABEL: atomic64_store_monotonic:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection252:
+; O1-NEXT:  .Lpcsection282:
 ; O1-NEXT:    movq $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -7638,7 +7728,7 @@ define void @atomic64_store_monotonic(ptr %a) {
 ; O2-LABEL: atomic64_store_monotonic:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection252:
+; O2-NEXT:  .Lpcsection282:
 ; O2-NEXT:    movq $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -7646,7 +7736,7 @@ define void @atomic64_store_monotonic(ptr %a) {
 ; O3-LABEL: atomic64_store_monotonic:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection252:
+; O3-NEXT:  .Lpcsection282:
 ; O3-NEXT:    movq $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -7669,7 +7759,7 @@ define void @atomic64_store_release(ptr %a) {
 ; O1-LABEL: atomic64_store_release:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection253:
+; O1-NEXT:  .Lpcsection283:
 ; O1-NEXT:    movq $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -7677,7 +7767,7 @@ define void @atomic64_store_release(ptr %a) {
 ; O2-LABEL: atomic64_store_release:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection253:
+; O2-NEXT:  .Lpcsection283:
 ; O2-NEXT:    movq $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -7685,7 +7775,7 @@ define void @atomic64_store_release(ptr %a) {
 ; O3-LABEL: atomic64_store_release:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection253:
+; O3-NEXT:  .Lpcsection283:
 ; O3-NEXT:    movq $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -7710,7 +7800,7 @@ define void @atomic64_store_seq_cst(ptr %a) {
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection254:
+; O1-NEXT:  .Lpcsection284:
 ; O1-NEXT:    xchgq %rax, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -7719,7 +7809,7 @@ define void @atomic64_store_seq_cst(ptr %a) {
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection254:
+; O2-NEXT:  .Lpcsection284:
 ; O2-NEXT:    xchgq %rax, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -7728,7 +7818,7 @@ define void @atomic64_store_seq_cst(ptr %a) {
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection254:
+; O3-NEXT:  .Lpcsection284:
 ; O3-NEXT:    xchgq %rax, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -7751,7 +7841,7 @@ define void @atomic64_store_seq_cst_ptr_ty(ptr %a, ptr %v) {
 ; O1-LABEL: atomic64_store_seq_cst_ptr_ty:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection255:
+; O1-NEXT:  .Lpcsection285:
 ; O1-NEXT:    xchgq %rsi, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -7759,7 +7849,7 @@ define void @atomic64_store_seq_cst_ptr_ty(ptr %a, ptr %v) {
 ; O2-LABEL: atomic64_store_seq_cst_ptr_ty:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection255:
+; O2-NEXT:  .Lpcsection285:
 ; O2-NEXT:    xchgq %rsi, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -7767,7 +7857,7 @@ define void @atomic64_store_seq_cst_ptr_ty(ptr %a, ptr %v) {
 ; O3-LABEL: atomic64_store_seq_cst_ptr_ty:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection255:
+; O3-NEXT:  .Lpcsection285:
 ; O3-NEXT:    xchgq %rsi, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -7792,7 +7882,7 @@ define void @atomic64_xchg_monotonic(ptr %a) {
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection256:
+; O1-NEXT:  .Lpcsection286:
 ; O1-NEXT:    xchgq %rax, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -7801,7 +7891,7 @@ define void @atomic64_xchg_monotonic(ptr %a) {
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection256:
+; O2-NEXT:  .Lpcsection286:
 ; O2-NEXT:    xchgq %rax, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -7810,7 +7900,7 @@ define void @atomic64_xchg_monotonic(ptr %a) {
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection256:
+; O3-NEXT:  .Lpcsection286:
 ; O3-NEXT:    xchgq %rax, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -7833,7 +7923,7 @@ define void @atomic64_add_monotonic(ptr %a) {
 ; O1-LABEL: atomic64_add_monotonic:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection257:
+; O1-NEXT:  .Lpcsection287:
 ; O1-NEXT:    lock addq $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -7841,7 +7931,7 @@ define void @atomic64_add_monotonic(ptr %a) {
 ; O2-LABEL: atomic64_add_monotonic:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection257:
+; O2-NEXT:  .Lpcsection287:
 ; O2-NEXT:    lock addq $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -7849,7 +7939,7 @@ define void @atomic64_add_monotonic(ptr %a) {
 ; O3-LABEL: atomic64_add_monotonic:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection257:
+; O3-NEXT:  .Lpcsection287:
 ; O3-NEXT:    lock addq $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -7872,7 +7962,7 @@ define void @atomic64_sub_monotonic(ptr %a) {
 ; O1-LABEL: atomic64_sub_monotonic:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection258:
+; O1-NEXT:  .Lpcsection288:
 ; O1-NEXT:    lock subq $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -7880,7 +7970,7 @@ define void @atomic64_sub_monotonic(ptr %a) {
 ; O2-LABEL: atomic64_sub_monotonic:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection258:
+; O2-NEXT:  .Lpcsection288:
 ; O2-NEXT:    lock subq $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -7888,7 +7978,7 @@ define void @atomic64_sub_monotonic(ptr %a) {
 ; O3-LABEL: atomic64_sub_monotonic:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection258:
+; O3-NEXT:  .Lpcsection288:
 ; O3-NEXT:    lock subq $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -7911,7 +8001,7 @@ define void @atomic64_and_monotonic(ptr %a) {
 ; O1-LABEL: atomic64_and_monotonic:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection259:
+; O1-NEXT:  .Lpcsection289:
 ; O1-NEXT:    lock andq $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -7919,7 +8009,7 @@ define void @atomic64_and_monotonic(ptr %a) {
 ; O2-LABEL: atomic64_and_monotonic:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection259:
+; O2-NEXT:  .Lpcsection289:
 ; O2-NEXT:    lock andq $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -7927,7 +8017,7 @@ define void @atomic64_and_monotonic(ptr %a) {
 ; O3-LABEL: atomic64_and_monotonic:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection259:
+; O3-NEXT:  .Lpcsection289:
 ; O3-NEXT:    lock andq $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -7950,7 +8040,7 @@ define void @atomic64_or_monotonic(ptr %a) {
 ; O1-LABEL: atomic64_or_monotonic:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection260:
+; O1-NEXT:  .Lpcsection290:
 ; O1-NEXT:    lock orq $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -7958,7 +8048,7 @@ define void @atomic64_or_monotonic(ptr %a) {
 ; O2-LABEL: atomic64_or_monotonic:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection260:
+; O2-NEXT:  .Lpcsection290:
 ; O2-NEXT:    lock orq $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -7966,7 +8056,7 @@ define void @atomic64_or_monotonic(ptr %a) {
 ; O3-LABEL: atomic64_or_monotonic:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection260:
+; O3-NEXT:  .Lpcsection290:
 ; O3-NEXT:    lock orq $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -7989,7 +8079,7 @@ define void @atomic64_xor_monotonic(ptr %a) {
 ; O1-LABEL: atomic64_xor_monotonic:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection261:
+; O1-NEXT:  .Lpcsection291:
 ; O1-NEXT:    lock xorq $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -7997,7 +8087,7 @@ define void @atomic64_xor_monotonic(ptr %a) {
 ; O2-LABEL: atomic64_xor_monotonic:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection261:
+; O2-NEXT:  .Lpcsection291:
 ; O2-NEXT:    lock xorq $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -8005,7 +8095,7 @@ define void @atomic64_xor_monotonic(ptr %a) {
 ; O3-LABEL: atomic64_xor_monotonic:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection261:
+; O3-NEXT:  .Lpcsection291:
 ; O3-NEXT:    lock xorq $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -8053,19 +8143,19 @@ define void @atomic64_nand_monotonic(ptr %a) {
 ; O1-LABEL: atomic64_nand_monotonic:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection262:
+; O1-NEXT:  .Lpcsection292:
 ; O1-NEXT:    movq (%rdi), %rax
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB162_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ecx
-; O1-NEXT:  .Lpcsection263:
+; O1-NEXT:  .Lpcsection293:
 ; O1-NEXT:    notl %ecx
-; O1-NEXT:  .Lpcsection264:
+; O1-NEXT:  .Lpcsection294:
 ; O1-NEXT:    orq $-43, %rcx
-; O1-NEXT:  .Lpcsection265:
+; O1-NEXT:  .Lpcsection295:
 ; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
-; O1-NEXT:  .Lpcsection266:
+; O1-NEXT:  .Lpcsection296:
 ; O1-NEXT:    jne .LBB162_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -8074,19 +8164,19 @@ define void @atomic64_nand_monotonic(ptr %a) {
 ; O2-LABEL: atomic64_nand_monotonic:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection262:
+; O2-NEXT:  .Lpcsection292:
 ; O2-NEXT:    movq (%rdi), %rax
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB162_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ecx
-; O2-NEXT:  .Lpcsection263:
+; O2-NEXT:  .Lpcsection293:
 ; O2-NEXT:    notl %ecx
-; O2-NEXT:  .Lpcsection264:
+; O2-NEXT:  .Lpcsection294:
 ; O2-NEXT:    orq $-43, %rcx
-; O2-NEXT:  .Lpcsection265:
+; O2-NEXT:  .Lpcsection295:
 ; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
-; O2-NEXT:  .Lpcsection266:
+; O2-NEXT:  .Lpcsection296:
 ; O2-NEXT:    jne .LBB162_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -8095,19 +8185,19 @@ define void @atomic64_nand_monotonic(ptr %a) {
 ; O3-LABEL: atomic64_nand_monotonic:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection262:
+; O3-NEXT:  .Lpcsection292:
 ; O3-NEXT:    movq (%rdi), %rax
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB162_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ecx
-; O3-NEXT:  .Lpcsection263:
+; O3-NEXT:  .Lpcsection293:
 ; O3-NEXT:    notl %ecx
-; O3-NEXT:  .Lpcsection264:
+; O3-NEXT:  .Lpcsection294:
 ; O3-NEXT:    orq $-43, %rcx
-; O3-NEXT:  .Lpcsection265:
+; O3-NEXT:  .Lpcsection295:
 ; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
-; O3-NEXT:  .Lpcsection266:
+; O3-NEXT:  .Lpcsection296:
 ; O3-NEXT:    jne .LBB162_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -8133,7 +8223,7 @@ define void @atomic64_xchg_acquire(ptr %a) {
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection267:
+; O1-NEXT:  .Lpcsection297:
 ; O1-NEXT:    xchgq %rax, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -8142,7 +8232,7 @@ define void @atomic64_xchg_acquire(ptr %a) {
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection267:
+; O2-NEXT:  .Lpcsection297:
 ; O2-NEXT:    xchgq %rax, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -8151,7 +8241,7 @@ define void @atomic64_xchg_acquire(ptr %a) {
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection267:
+; O3-NEXT:  .Lpcsection297:
 ; O3-NEXT:    xchgq %rax, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -8174,7 +8264,7 @@ define void @atomic64_add_acquire(ptr %a) {
 ; O1-LABEL: atomic64_add_acquire:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection268:
+; O1-NEXT:  .Lpcsection298:
 ; O1-NEXT:    lock addq $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -8182,7 +8272,7 @@ define void @atomic64_add_acquire(ptr %a) {
 ; O2-LABEL: atomic64_add_acquire:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection268:
+; O2-NEXT:  .Lpcsection298:
 ; O2-NEXT:    lock addq $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -8190,7 +8280,7 @@ define void @atomic64_add_acquire(ptr %a) {
 ; O3-LABEL: atomic64_add_acquire:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection268:
+; O3-NEXT:  .Lpcsection298:
 ; O3-NEXT:    lock addq $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -8213,7 +8303,7 @@ define void @atomic64_sub_acquire(ptr %a) {
 ; O1-LABEL: atomic64_sub_acquire:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection269:
+; O1-NEXT:  .Lpcsection299:
 ; O1-NEXT:    lock subq $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -8221,7 +8311,7 @@ define void @atomic64_sub_acquire(ptr %a) {
 ; O2-LABEL: atomic64_sub_acquire:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection269:
+; O2-NEXT:  .Lpcsection299:
 ; O2-NEXT:    lock subq $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -8229,7 +8319,7 @@ define void @atomic64_sub_acquire(ptr %a) {
 ; O3-LABEL: atomic64_sub_acquire:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection269:
+; O3-NEXT:  .Lpcsection299:
 ; O3-NEXT:    lock subq $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -8252,7 +8342,7 @@ define void @atomic64_and_acquire(ptr %a) {
 ; O1-LABEL: atomic64_and_acquire:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection270:
+; O1-NEXT:  .Lpcsection300:
 ; O1-NEXT:    lock andq $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -8260,7 +8350,7 @@ define void @atomic64_and_acquire(ptr %a) {
 ; O2-LABEL: atomic64_and_acquire:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection270:
+; O2-NEXT:  .Lpcsection300:
 ; O2-NEXT:    lock andq $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -8268,7 +8358,7 @@ define void @atomic64_and_acquire(ptr %a) {
 ; O3-LABEL: atomic64_and_acquire:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection270:
+; O3-NEXT:  .Lpcsection300:
 ; O3-NEXT:    lock andq $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -8291,7 +8381,7 @@ define void @atomic64_or_acquire(ptr %a) {
 ; O1-LABEL: atomic64_or_acquire:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection271:
+; O1-NEXT:  .Lpcsection301:
 ; O1-NEXT:    lock orq $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -8299,7 +8389,7 @@ define void @atomic64_or_acquire(ptr %a) {
 ; O2-LABEL: atomic64_or_acquire:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection271:
+; O2-NEXT:  .Lpcsection301:
 ; O2-NEXT:    lock orq $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -8307,7 +8397,7 @@ define void @atomic64_or_acquire(ptr %a) {
 ; O3-LABEL: atomic64_or_acquire:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection271:
+; O3-NEXT:  .Lpcsection301:
 ; O3-NEXT:    lock orq $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -8330,7 +8420,7 @@ define void @atomic64_xor_acquire(ptr %a) {
 ; O1-LABEL: atomic64_xor_acquire:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection272:
+; O1-NEXT:  .Lpcsection302:
 ; O1-NEXT:    lock xorq $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -8338,7 +8428,7 @@ define void @atomic64_xor_acquire(ptr %a) {
 ; O2-LABEL: atomic64_xor_acquire:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection272:
+; O2-NEXT:  .Lpcsection302:
 ; O2-NEXT:    lock xorq $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -8346,7 +8436,7 @@ define void @atomic64_xor_acquire(ptr %a) {
 ; O3-LABEL: atomic64_xor_acquire:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection272:
+; O3-NEXT:  .Lpcsection302:
 ; O3-NEXT:    lock xorq $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -8394,19 +8484,19 @@ define void @atomic64_nand_acquire(ptr %a) {
 ; O1-LABEL: atomic64_nand_acquire:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection273:
+; O1-NEXT:  .Lpcsection303:
 ; O1-NEXT:    movq (%rdi), %rax
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB169_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ecx
-; O1-NEXT:  .Lpcsection274:
+; O1-NEXT:  .Lpcsection304:
 ; O1-NEXT:    notl %ecx
-; O1-NEXT:  .Lpcsection275:
+; O1-NEXT:  .Lpcsection305:
 ; O1-NEXT:    orq $-43, %rcx
-; O1-NEXT:  .Lpcsection276:
+; O1-NEXT:  .Lpcsection306:
 ; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
-; O1-NEXT:  .Lpcsection277:
+; O1-NEXT:  .Lpcsection307:
 ; O1-NEXT:    jne .LBB169_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -8415,19 +8505,19 @@ define void @atomic64_nand_acquire(ptr %a) {
 ; O2-LABEL: atomic64_nand_acquire:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection273:
+; O2-NEXT:  .Lpcsection303:
 ; O2-NEXT:    movq (%rdi), %rax
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB169_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ecx
-; O2-NEXT:  .Lpcsection274:
+; O2-NEXT:  .Lpcsection304:
 ; O2-NEXT:    notl %ecx
-; O2-NEXT:  .Lpcsection275:
+; O2-NEXT:  .Lpcsection305:
 ; O2-NEXT:    orq $-43, %rcx
-; O2-NEXT:  .Lpcsection276:
+; O2-NEXT:  .Lpcsection306:
 ; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
-; O2-NEXT:  .Lpcsection277:
+; O2-NEXT:  .Lpcsection307:
 ; O2-NEXT:    jne .LBB169_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -8436,19 +8526,19 @@ define void @atomic64_nand_acquire(ptr %a) {
 ; O3-LABEL: atomic64_nand_acquire:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection273:
+; O3-NEXT:  .Lpcsection303:
 ; O3-NEXT:    movq (%rdi), %rax
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB169_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ecx
-; O3-NEXT:  .Lpcsection274:
+; O3-NEXT:  .Lpcsection304:
 ; O3-NEXT:    notl %ecx
-; O3-NEXT:  .Lpcsection275:
+; O3-NEXT:  .Lpcsection305:
 ; O3-NEXT:    orq $-43, %rcx
-; O3-NEXT:  .Lpcsection276:
+; O3-NEXT:  .Lpcsection306:
 ; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
-; O3-NEXT:  .Lpcsection277:
+; O3-NEXT:  .Lpcsection307:
 ; O3-NEXT:    jne .LBB169_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -8474,7 +8564,7 @@ define void @atomic64_xchg_release(ptr %a) {
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection278:
+; O1-NEXT:  .Lpcsection308:
 ; O1-NEXT:    xchgq %rax, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -8483,7 +8573,7 @@ define void @atomic64_xchg_release(ptr %a) {
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection278:
+; O2-NEXT:  .Lpcsection308:
 ; O2-NEXT:    xchgq %rax, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -8492,7 +8582,7 @@ define void @atomic64_xchg_release(ptr %a) {
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection278:
+; O3-NEXT:  .Lpcsection308:
 ; O3-NEXT:    xchgq %rax, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -8515,7 +8605,7 @@ define void @atomic64_add_release(ptr %a) {
 ; O1-LABEL: atomic64_add_release:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection279:
+; O1-NEXT:  .Lpcsection309:
 ; O1-NEXT:    lock addq $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -8523,7 +8613,7 @@ define void @atomic64_add_release(ptr %a) {
 ; O2-LABEL: atomic64_add_release:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection279:
+; O2-NEXT:  .Lpcsection309:
 ; O2-NEXT:    lock addq $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -8531,7 +8621,7 @@ define void @atomic64_add_release(ptr %a) {
 ; O3-LABEL: atomic64_add_release:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection279:
+; O3-NEXT:  .Lpcsection309:
 ; O3-NEXT:    lock addq $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -8554,7 +8644,7 @@ define void @atomic64_sub_release(ptr %a) {
 ; O1-LABEL: atomic64_sub_release:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection280:
+; O1-NEXT:  .Lpcsection310:
 ; O1-NEXT:    lock subq $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -8562,7 +8652,7 @@ define void @atomic64_sub_release(ptr %a) {
 ; O2-LABEL: atomic64_sub_release:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection280:
+; O2-NEXT:  .Lpcsection310:
 ; O2-NEXT:    lock subq $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -8570,7 +8660,7 @@ define void @atomic64_sub_release(ptr %a) {
 ; O3-LABEL: atomic64_sub_release:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection280:
+; O3-NEXT:  .Lpcsection310:
 ; O3-NEXT:    lock subq $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -8593,7 +8683,7 @@ define void @atomic64_and_release(ptr %a) {
 ; O1-LABEL: atomic64_and_release:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection281:
+; O1-NEXT:  .Lpcsection311:
 ; O1-NEXT:    lock andq $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -8601,7 +8691,7 @@ define void @atomic64_and_release(ptr %a) {
 ; O2-LABEL: atomic64_and_release:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection281:
+; O2-NEXT:  .Lpcsection311:
 ; O2-NEXT:    lock andq $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -8609,7 +8699,7 @@ define void @atomic64_and_release(ptr %a) {
 ; O3-LABEL: atomic64_and_release:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection281:
+; O3-NEXT:  .Lpcsection311:
 ; O3-NEXT:    lock andq $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -8632,7 +8722,7 @@ define void @atomic64_or_release(ptr %a) {
 ; O1-LABEL: atomic64_or_release:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection282:
+; O1-NEXT:  .Lpcsection312:
 ; O1-NEXT:    lock orq $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -8640,7 +8730,7 @@ define void @atomic64_or_release(ptr %a) {
 ; O2-LABEL: atomic64_or_release:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection282:
+; O2-NEXT:  .Lpcsection312:
 ; O2-NEXT:    lock orq $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -8648,7 +8738,7 @@ define void @atomic64_or_release(ptr %a) {
 ; O3-LABEL: atomic64_or_release:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection282:
+; O3-NEXT:  .Lpcsection312:
 ; O3-NEXT:    lock orq $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -8671,7 +8761,7 @@ define void @atomic64_xor_release(ptr %a) {
 ; O1-LABEL: atomic64_xor_release:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection283:
+; O1-NEXT:  .Lpcsection313:
 ; O1-NEXT:    lock xorq $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -8679,7 +8769,7 @@ define void @atomic64_xor_release(ptr %a) {
 ; O2-LABEL: atomic64_xor_release:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection283:
+; O2-NEXT:  .Lpcsection313:
 ; O2-NEXT:    lock xorq $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -8687,7 +8777,7 @@ define void @atomic64_xor_release(ptr %a) {
 ; O3-LABEL: atomic64_xor_release:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection283:
+; O3-NEXT:  .Lpcsection313:
 ; O3-NEXT:    lock xorq $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -8735,19 +8825,19 @@ define void @atomic64_nand_release(ptr %a) {
 ; O1-LABEL: atomic64_nand_release:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection284:
+; O1-NEXT:  .Lpcsection314:
 ; O1-NEXT:    movq (%rdi), %rax
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB176_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ecx
-; O1-NEXT:  .Lpcsection285:
+; O1-NEXT:  .Lpcsection315:
 ; O1-NEXT:    notl %ecx
-; O1-NEXT:  .Lpcsection286:
+; O1-NEXT:  .Lpcsection316:
 ; O1-NEXT:    orq $-43, %rcx
-; O1-NEXT:  .Lpcsection287:
+; O1-NEXT:  .Lpcsection317:
 ; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
-; O1-NEXT:  .Lpcsection288:
+; O1-NEXT:  .Lpcsection318:
 ; O1-NEXT:    jne .LBB176_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -8756,19 +8846,19 @@ define void @atomic64_nand_release(ptr %a) {
 ; O2-LABEL: atomic64_nand_release:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection284:
+; O2-NEXT:  .Lpcsection314:
 ; O2-NEXT:    movq (%rdi), %rax
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB176_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ecx
-; O2-NEXT:  .Lpcsection285:
+; O2-NEXT:  .Lpcsection315:
 ; O2-NEXT:    notl %ecx
-; O2-NEXT:  .Lpcsection286:
+; O2-NEXT:  .Lpcsection316:
 ; O2-NEXT:    orq $-43, %rcx
-; O2-NEXT:  .Lpcsection287:
+; O2-NEXT:  .Lpcsection317:
 ; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
-; O2-NEXT:  .Lpcsection288:
+; O2-NEXT:  .Lpcsection318:
 ; O2-NEXT:    jne .LBB176_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -8777,19 +8867,19 @@ define void @atomic64_nand_release(ptr %a) {
 ; O3-LABEL: atomic64_nand_release:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection284:
+; O3-NEXT:  .Lpcsection314:
 ; O3-NEXT:    movq (%rdi), %rax
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB176_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ecx
-; O3-NEXT:  .Lpcsection285:
+; O3-NEXT:  .Lpcsection315:
 ; O3-NEXT:    notl %ecx
-; O3-NEXT:  .Lpcsection286:
+; O3-NEXT:  .Lpcsection316:
 ; O3-NEXT:    orq $-43, %rcx
-; O3-NEXT:  .Lpcsection287:
+; O3-NEXT:  .Lpcsection317:
 ; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
-; O3-NEXT:  .Lpcsection288:
+; O3-NEXT:  .Lpcsection318:
 ; O3-NEXT:    jne .LBB176_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -8815,7 +8905,7 @@ define void @atomic64_xchg_acq_rel(ptr %a) {
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection289:
+; O1-NEXT:  .Lpcsection319:
 ; O1-NEXT:    xchgq %rax, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -8824,7 +8914,7 @@ define void @atomic64_xchg_acq_rel(ptr %a) {
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection289:
+; O2-NEXT:  .Lpcsection319:
 ; O2-NEXT:    xchgq %rax, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -8833,7 +8923,7 @@ define void @atomic64_xchg_acq_rel(ptr %a) {
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection289:
+; O3-NEXT:  .Lpcsection319:
 ; O3-NEXT:    xchgq %rax, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -8856,7 +8946,7 @@ define void @atomic64_add_acq_rel(ptr %a) {
 ; O1-LABEL: atomic64_add_acq_rel:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection290:
+; O1-NEXT:  .Lpcsection320:
 ; O1-NEXT:    lock addq $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -8864,7 +8954,7 @@ define void @atomic64_add_acq_rel(ptr %a) {
 ; O2-LABEL: atomic64_add_acq_rel:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection290:
+; O2-NEXT:  .Lpcsection320:
 ; O2-NEXT:    lock addq $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -8872,7 +8962,7 @@ define void @atomic64_add_acq_rel(ptr %a) {
 ; O3-LABEL: atomic64_add_acq_rel:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection290:
+; O3-NEXT:  .Lpcsection320:
 ; O3-NEXT:    lock addq $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -8895,7 +8985,7 @@ define void @atomic64_sub_acq_rel(ptr %a) {
 ; O1-LABEL: atomic64_sub_acq_rel:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection291:
+; O1-NEXT:  .Lpcsection321:
 ; O1-NEXT:    lock subq $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -8903,7 +8993,7 @@ define void @atomic64_sub_acq_rel(ptr %a) {
 ; O2-LABEL: atomic64_sub_acq_rel:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection291:
+; O2-NEXT:  .Lpcsection321:
 ; O2-NEXT:    lock subq $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -8911,7 +9001,7 @@ define void @atomic64_sub_acq_rel(ptr %a) {
 ; O3-LABEL: atomic64_sub_acq_rel:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection291:
+; O3-NEXT:  .Lpcsection321:
 ; O3-NEXT:    lock subq $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -8934,7 +9024,7 @@ define void @atomic64_and_acq_rel(ptr %a) {
 ; O1-LABEL: atomic64_and_acq_rel:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection292:
+; O1-NEXT:  .Lpcsection322:
 ; O1-NEXT:    lock andq $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -8942,7 +9032,7 @@ define void @atomic64_and_acq_rel(ptr %a) {
 ; O2-LABEL: atomic64_and_acq_rel:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection292:
+; O2-NEXT:  .Lpcsection322:
 ; O2-NEXT:    lock andq $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -8950,7 +9040,7 @@ define void @atomic64_and_acq_rel(ptr %a) {
 ; O3-LABEL: atomic64_and_acq_rel:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection292:
+; O3-NEXT:  .Lpcsection322:
 ; O3-NEXT:    lock andq $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -8973,7 +9063,7 @@ define void @atomic64_or_acq_rel(ptr %a) {
 ; O1-LABEL: atomic64_or_acq_rel:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection293:
+; O1-NEXT:  .Lpcsection323:
 ; O1-NEXT:    lock orq $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -8981,7 +9071,7 @@ define void @atomic64_or_acq_rel(ptr %a) {
 ; O2-LABEL: atomic64_or_acq_rel:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection293:
+; O2-NEXT:  .Lpcsection323:
 ; O2-NEXT:    lock orq $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -8989,7 +9079,7 @@ define void @atomic64_or_acq_rel(ptr %a) {
 ; O3-LABEL: atomic64_or_acq_rel:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection293:
+; O3-NEXT:  .Lpcsection323:
 ; O3-NEXT:    lock orq $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -9012,7 +9102,7 @@ define void @atomic64_xor_acq_rel(ptr %a) {
 ; O1-LABEL: atomic64_xor_acq_rel:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection294:
+; O1-NEXT:  .Lpcsection324:
 ; O1-NEXT:    lock xorq $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -9020,7 +9110,7 @@ define void @atomic64_xor_acq_rel(ptr %a) {
 ; O2-LABEL: atomic64_xor_acq_rel:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection294:
+; O2-NEXT:  .Lpcsection324:
 ; O2-NEXT:    lock xorq $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -9028,7 +9118,7 @@ define void @atomic64_xor_acq_rel(ptr %a) {
 ; O3-LABEL: atomic64_xor_acq_rel:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection294:
+; O3-NEXT:  .Lpcsection324:
 ; O3-NEXT:    lock xorq $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -9076,19 +9166,19 @@ define void @atomic64_nand_acq_rel(ptr %a) {
 ; O1-LABEL: atomic64_nand_acq_rel:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection295:
+; O1-NEXT:  .Lpcsection325:
 ; O1-NEXT:    movq (%rdi), %rax
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB183_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ecx
-; O1-NEXT:  .Lpcsection296:
+; O1-NEXT:  .Lpcsection326:
 ; O1-NEXT:    notl %ecx
-; O1-NEXT:  .Lpcsection297:
+; O1-NEXT:  .Lpcsection327:
 ; O1-NEXT:    orq $-43, %rcx
-; O1-NEXT:  .Lpcsection298:
+; O1-NEXT:  .Lpcsection328:
 ; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
-; O1-NEXT:  .Lpcsection299:
+; O1-NEXT:  .Lpcsection329:
 ; O1-NEXT:    jne .LBB183_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -9097,19 +9187,19 @@ define void @atomic64_nand_acq_rel(ptr %a) {
 ; O2-LABEL: atomic64_nand_acq_rel:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection295:
+; O2-NEXT:  .Lpcsection325:
 ; O2-NEXT:    movq (%rdi), %rax
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB183_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ecx
-; O2-NEXT:  .Lpcsection296:
+; O2-NEXT:  .Lpcsection326:
 ; O2-NEXT:    notl %ecx
-; O2-NEXT:  .Lpcsection297:
+; O2-NEXT:  .Lpcsection327:
 ; O2-NEXT:    orq $-43, %rcx
-; O2-NEXT:  .Lpcsection298:
+; O2-NEXT:  .Lpcsection328:
 ; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
-; O2-NEXT:  .Lpcsection299:
+; O2-NEXT:  .Lpcsection329:
 ; O2-NEXT:    jne .LBB183_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -9118,19 +9208,19 @@ define void @atomic64_nand_acq_rel(ptr %a) {
 ; O3-LABEL: atomic64_nand_acq_rel:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection295:
+; O3-NEXT:  .Lpcsection325:
 ; O3-NEXT:    movq (%rdi), %rax
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB183_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ecx
-; O3-NEXT:  .Lpcsection296:
+; O3-NEXT:  .Lpcsection326:
 ; O3-NEXT:    notl %ecx
-; O3-NEXT:  .Lpcsection297:
+; O3-NEXT:  .Lpcsection327:
 ; O3-NEXT:    orq $-43, %rcx
-; O3-NEXT:  .Lpcsection298:
+; O3-NEXT:  .Lpcsection328:
 ; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
-; O3-NEXT:  .Lpcsection299:
+; O3-NEXT:  .Lpcsection329:
 ; O3-NEXT:    jne .LBB183_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -9156,7 +9246,7 @@ define void @atomic64_xchg_seq_cst(ptr %a) {
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection300:
+; O1-NEXT:  .Lpcsection330:
 ; O1-NEXT:    xchgq %rax, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -9165,7 +9255,7 @@ define void @atomic64_xchg_seq_cst(ptr %a) {
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection300:
+; O2-NEXT:  .Lpcsection330:
 ; O2-NEXT:    xchgq %rax, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -9174,7 +9264,7 @@ define void @atomic64_xchg_seq_cst(ptr %a) {
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection300:
+; O3-NEXT:  .Lpcsection330:
 ; O3-NEXT:    xchgq %rax, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -9197,7 +9287,7 @@ define void @atomic64_add_seq_cst(ptr %a) {
 ; O1-LABEL: atomic64_add_seq_cst:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection301:
+; O1-NEXT:  .Lpcsection331:
 ; O1-NEXT:    lock addq $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -9205,7 +9295,7 @@ define void @atomic64_add_seq_cst(ptr %a) {
 ; O2-LABEL: atomic64_add_seq_cst:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection301:
+; O2-NEXT:  .Lpcsection331:
 ; O2-NEXT:    lock addq $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -9213,7 +9303,7 @@ define void @atomic64_add_seq_cst(ptr %a) {
 ; O3-LABEL: atomic64_add_seq_cst:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection301:
+; O3-NEXT:  .Lpcsection331:
 ; O3-NEXT:    lock addq $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -9236,7 +9326,7 @@ define void @atomic64_sub_seq_cst(ptr %a) {
 ; O1-LABEL: atomic64_sub_seq_cst:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection302:
+; O1-NEXT:  .Lpcsection332:
 ; O1-NEXT:    lock subq $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -9244,7 +9334,7 @@ define void @atomic64_sub_seq_cst(ptr %a) {
 ; O2-LABEL: atomic64_sub_seq_cst:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection302:
+; O2-NEXT:  .Lpcsection332:
 ; O2-NEXT:    lock subq $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -9252,7 +9342,7 @@ define void @atomic64_sub_seq_cst(ptr %a) {
 ; O3-LABEL: atomic64_sub_seq_cst:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection302:
+; O3-NEXT:  .Lpcsection332:
 ; O3-NEXT:    lock subq $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -9275,7 +9365,7 @@ define void @atomic64_and_seq_cst(ptr %a) {
 ; O1-LABEL: atomic64_and_seq_cst:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection303:
+; O1-NEXT:  .Lpcsection333:
 ; O1-NEXT:    lock andq $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -9283,7 +9373,7 @@ define void @atomic64_and_seq_cst(ptr %a) {
 ; O2-LABEL: atomic64_and_seq_cst:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection303:
+; O2-NEXT:  .Lpcsection333:
 ; O2-NEXT:    lock andq $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -9291,7 +9381,7 @@ define void @atomic64_and_seq_cst(ptr %a) {
 ; O3-LABEL: atomic64_and_seq_cst:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection303:
+; O3-NEXT:  .Lpcsection333:
 ; O3-NEXT:    lock andq $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -9314,7 +9404,7 @@ define void @atomic64_or_seq_cst(ptr %a) {
 ; O1-LABEL: atomic64_or_seq_cst:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection304:
+; O1-NEXT:  .Lpcsection334:
 ; O1-NEXT:    lock orq $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -9322,7 +9412,7 @@ define void @atomic64_or_seq_cst(ptr %a) {
 ; O2-LABEL: atomic64_or_seq_cst:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection304:
+; O2-NEXT:  .Lpcsection334:
 ; O2-NEXT:    lock orq $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -9330,7 +9420,7 @@ define void @atomic64_or_seq_cst(ptr %a) {
 ; O3-LABEL: atomic64_or_seq_cst:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection304:
+; O3-NEXT:  .Lpcsection334:
 ; O3-NEXT:    lock orq $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -9353,7 +9443,7 @@ define void @atomic64_xor_seq_cst(ptr %a) {
 ; O1-LABEL: atomic64_xor_seq_cst:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection305:
+; O1-NEXT:  .Lpcsection335:
 ; O1-NEXT:    lock xorq $42, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -9361,7 +9451,7 @@ define void @atomic64_xor_seq_cst(ptr %a) {
 ; O2-LABEL: atomic64_xor_seq_cst:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection305:
+; O2-NEXT:  .Lpcsection335:
 ; O2-NEXT:    lock xorq $42, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -9369,7 +9459,7 @@ define void @atomic64_xor_seq_cst(ptr %a) {
 ; O3-LABEL: atomic64_xor_seq_cst:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection305:
+; O3-NEXT:  .Lpcsection335:
 ; O3-NEXT:    lock xorq $42, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -9417,19 +9507,19 @@ define void @atomic64_nand_seq_cst(ptr %a) {
 ; O1-LABEL: atomic64_nand_seq_cst:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection306:
+; O1-NEXT:  .Lpcsection336:
 ; O1-NEXT:    movq (%rdi), %rax
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB190_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ecx
-; O1-NEXT:  .Lpcsection307:
+; O1-NEXT:  .Lpcsection337:
 ; O1-NEXT:    notl %ecx
-; O1-NEXT:  .Lpcsection308:
+; O1-NEXT:  .Lpcsection338:
 ; O1-NEXT:    orq $-43, %rcx
-; O1-NEXT:  .Lpcsection309:
+; O1-NEXT:  .Lpcsection339:
 ; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
-; O1-NEXT:  .Lpcsection310:
+; O1-NEXT:  .Lpcsection340:
 ; O1-NEXT:    jne .LBB190_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -9438,19 +9528,19 @@ define void @atomic64_nand_seq_cst(ptr %a) {
 ; O2-LABEL: atomic64_nand_seq_cst:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection306:
+; O2-NEXT:  .Lpcsection336:
 ; O2-NEXT:    movq (%rdi), %rax
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB190_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ecx
-; O2-NEXT:  .Lpcsection307:
+; O2-NEXT:  .Lpcsection337:
 ; O2-NEXT:    notl %ecx
-; O2-NEXT:  .Lpcsection308:
+; O2-NEXT:  .Lpcsection338:
 ; O2-NEXT:    orq $-43, %rcx
-; O2-NEXT:  .Lpcsection309:
+; O2-NEXT:  .Lpcsection339:
 ; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
-; O2-NEXT:  .Lpcsection310:
+; O2-NEXT:  .Lpcsection340:
 ; O2-NEXT:    jne .LBB190_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -9459,19 +9549,19 @@ define void @atomic64_nand_seq_cst(ptr %a) {
 ; O3-LABEL: atomic64_nand_seq_cst:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection306:
+; O3-NEXT:  .Lpcsection336:
 ; O3-NEXT:    movq (%rdi), %rax
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB190_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ecx
-; O3-NEXT:  .Lpcsection307:
+; O3-NEXT:  .Lpcsection337:
 ; O3-NEXT:    notl %ecx
-; O3-NEXT:  .Lpcsection308:
+; O3-NEXT:  .Lpcsection338:
 ; O3-NEXT:    orq $-43, %rcx
-; O3-NEXT:  .Lpcsection309:
+; O3-NEXT:  .Lpcsection339:
 ; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
-; O3-NEXT:  .Lpcsection310:
+; O3-NEXT:  .Lpcsection340:
 ; O3-NEXT:    jne .LBB190_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -9509,14 +9599,17 @@ define void @atomic64_cas_monotonic(ptr %a) {
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movl $1, %ecx
+; O1-NEXT:  .Lpcsection341:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection311:
+; O1-NEXT:  .Lpcsection342:
 ; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
+; O1-NEXT:  .Lpcsection343:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection312:
+; O1-NEXT:  .Lpcsection344:
 ; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
+; O1-NEXT:  .Lpcsection345:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection313:
+; O1-NEXT:  .Lpcsection346:
 ; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -9525,14 +9618,17 @@ define void @atomic64_cas_monotonic(ptr %a) {
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movl $1, %ecx
+; O2-NEXT:  .Lpcsection341:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection311:
+; O2-NEXT:  .Lpcsection342:
 ; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
+; O2-NEXT:  .Lpcsection343:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection312:
+; O2-NEXT:  .Lpcsection344:
 ; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
+; O2-NEXT:  .Lpcsection345:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection313:
+; O2-NEXT:  .Lpcsection346:
 ; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -9541,14 +9637,17 @@ define void @atomic64_cas_monotonic(ptr %a) {
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movl $1, %ecx
+; O3-NEXT:  .Lpcsection341:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection311:
+; O3-NEXT:  .Lpcsection342:
 ; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
+; O3-NEXT:  .Lpcsection343:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection312:
+; O3-NEXT:  .Lpcsection344:
 ; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
+; O3-NEXT:  .Lpcsection345:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection313:
+; O3-NEXT:  .Lpcsection346:
 ; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -9587,14 +9686,17 @@ define void @atomic64_cas_acquire(ptr %a) {
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movl $1, %ecx
+; O1-NEXT:  .Lpcsection347:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection314:
+; O1-NEXT:  .Lpcsection348:
 ; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
+; O1-NEXT:  .Lpcsection349:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection315:
+; O1-NEXT:  .Lpcsection350:
 ; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
+; O1-NEXT:  .Lpcsection351:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection316:
+; O1-NEXT:  .Lpcsection352:
 ; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -9603,14 +9705,17 @@ define void @atomic64_cas_acquire(ptr %a) {
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movl $1, %ecx
+; O2-NEXT:  .Lpcsection347:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection314:
+; O2-NEXT:  .Lpcsection348:
 ; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
+; O2-NEXT:  .Lpcsection349:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection315:
+; O2-NEXT:  .Lpcsection350:
 ; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
+; O2-NEXT:  .Lpcsection351:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection316:
+; O2-NEXT:  .Lpcsection352:
 ; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -9619,14 +9724,17 @@ define void @atomic64_cas_acquire(ptr %a) {
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movl $1, %ecx
+; O3-NEXT:  .Lpcsection347:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection314:
+; O3-NEXT:  .Lpcsection348:
 ; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
+; O3-NEXT:  .Lpcsection349:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection315:
+; O3-NEXT:  .Lpcsection350:
 ; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
+; O3-NEXT:  .Lpcsection351:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection316:
+; O3-NEXT:  .Lpcsection352:
 ; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -9665,14 +9773,17 @@ define void @atomic64_cas_release(ptr %a) {
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movl $1, %ecx
+; O1-NEXT:  .Lpcsection353:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection317:
+; O1-NEXT:  .Lpcsection354:
 ; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
+; O1-NEXT:  .Lpcsection355:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection318:
+; O1-NEXT:  .Lpcsection356:
 ; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
+; O1-NEXT:  .Lpcsection357:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection319:
+; O1-NEXT:  .Lpcsection358:
 ; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -9681,14 +9792,17 @@ define void @atomic64_cas_release(ptr %a) {
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movl $1, %ecx
+; O2-NEXT:  .Lpcsection353:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection317:
+; O2-NEXT:  .Lpcsection354:
 ; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
+; O2-NEXT:  .Lpcsection355:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection318:
+; O2-NEXT:  .Lpcsection356:
 ; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
+; O2-NEXT:  .Lpcsection357:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection319:
+; O2-NEXT:  .Lpcsection358:
 ; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -9697,14 +9811,17 @@ define void @atomic64_cas_release(ptr %a) {
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movl $1, %ecx
+; O3-NEXT:  .Lpcsection353:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection317:
+; O3-NEXT:  .Lpcsection354:
 ; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
+; O3-NEXT:  .Lpcsection355:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection318:
+; O3-NEXT:  .Lpcsection356:
 ; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
+; O3-NEXT:  .Lpcsection357:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection319:
+; O3-NEXT:  .Lpcsection358:
 ; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -9743,14 +9860,17 @@ define void @atomic64_cas_acq_rel(ptr %a) {
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movl $1, %ecx
+; O1-NEXT:  .Lpcsection359:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection320:
+; O1-NEXT:  .Lpcsection360:
 ; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
+; O1-NEXT:  .Lpcsection361:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection321:
+; O1-NEXT:  .Lpcsection362:
 ; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
+; O1-NEXT:  .Lpcsection363:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection322:
+; O1-NEXT:  .Lpcsection364:
 ; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -9759,14 +9879,17 @@ define void @atomic64_cas_acq_rel(ptr %a) {
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movl $1, %ecx
+; O2-NEXT:  .Lpcsection359:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection320:
+; O2-NEXT:  .Lpcsection360:
 ; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
+; O2-NEXT:  .Lpcsection361:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection321:
+; O2-NEXT:  .Lpcsection362:
 ; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
+; O2-NEXT:  .Lpcsection363:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection322:
+; O2-NEXT:  .Lpcsection364:
 ; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -9775,14 +9898,17 @@ define void @atomic64_cas_acq_rel(ptr %a) {
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movl $1, %ecx
+; O3-NEXT:  .Lpcsection359:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection320:
+; O3-NEXT:  .Lpcsection360:
 ; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
+; O3-NEXT:  .Lpcsection361:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection321:
+; O3-NEXT:  .Lpcsection362:
 ; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
+; O3-NEXT:  .Lpcsection363:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection322:
+; O3-NEXT:  .Lpcsection364:
 ; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -9821,14 +9947,17 @@ define void @atomic64_cas_seq_cst(ptr %a) {
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
 ; O1-NEXT:    movl $1, %ecx
+; O1-NEXT:  .Lpcsection365:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection323:
+; O1-NEXT:  .Lpcsection366:
 ; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
+; O1-NEXT:  .Lpcsection367:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection324:
+; O1-NEXT:  .Lpcsection368:
 ; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
+; O1-NEXT:  .Lpcsection369:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection325:
+; O1-NEXT:  .Lpcsection370:
 ; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
 ; O1-NEXT:    movq $3, foo(%rip)
 ; O1-NEXT:    retq
@@ -9837,14 +9966,17 @@ define void @atomic64_cas_seq_cst(ptr %a) {
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
 ; O2-NEXT:    movl $1, %ecx
+; O2-NEXT:  .Lpcsection365:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection323:
+; O2-NEXT:  .Lpcsection366:
 ; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
+; O2-NEXT:  .Lpcsection367:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection324:
+; O2-NEXT:  .Lpcsection368:
 ; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
+; O2-NEXT:  .Lpcsection369:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection325:
+; O2-NEXT:  .Lpcsection370:
 ; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
 ; O2-NEXT:    movq $3, foo(%rip)
 ; O2-NEXT:    retq
@@ -9853,14 +9985,17 @@ define void @atomic64_cas_seq_cst(ptr %a) {
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
 ; O3-NEXT:    movl $1, %ecx
+; O3-NEXT:  .Lpcsection365:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection323:
+; O3-NEXT:  .Lpcsection366:
 ; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
+; O3-NEXT:  .Lpcsection367:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection324:
+; O3-NEXT:  .Lpcsection368:
 ; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
+; O3-NEXT:  .Lpcsection369:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection325:
+; O3-NEXT:  .Lpcsection370:
 ; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
 ; O3-NEXT:    movq $3, foo(%rip)
 ; O3-NEXT:    retq
@@ -9887,7 +10022,7 @@ define void @atomic64_cas_seq_cst_ptr_ty(ptr %a, ptr %v1, ptr %v2) {
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq %rsi, %rax
 ; O1-NEXT:    movq foo(%rip), %rcx
-; O1-NEXT:  .Lpcsection326:
+; O1-NEXT:  .Lpcsection371:
 ; O1-NEXT:    lock cmpxchgq %rdx, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -9896,7 +10031,7 @@ define void @atomic64_cas_seq_cst_ptr_ty(ptr %a, ptr %v1, ptr %v2) {
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq %rsi, %rax
 ; O2-NEXT:    movq foo(%rip), %rcx
-; O2-NEXT:  .Lpcsection326:
+; O2-NEXT:  .Lpcsection371:
 ; O2-NEXT:    lock cmpxchgq %rdx, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -9905,7 +10040,7 @@ define void @atomic64_cas_seq_cst_ptr_ty(ptr %a, ptr %v1, ptr %v2) {
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq %rsi, %rax
 ; O3-NEXT:    movq foo(%rip), %rcx
-; O3-NEXT:  .Lpcsection326:
+; O3-NEXT:  .Lpcsection371:
 ; O3-NEXT:    lock cmpxchgq %rdx, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -9934,7 +10069,7 @@ define i64 @atomic_use_cond(ptr %a) {
 ;
 ; O1-LABEL: atomic_use_cond:
 ; O1:       # %bb.0: # %entry
-; O1-NEXT:  .Lpcsection327:
+; O1-NEXT:  .Lpcsection372:
 ; O1-NEXT:    lock decq (%rdi)
 ; O1-NEXT:    jne .LBB197_2
 ; O1-NEXT:  # %bb.1: # %then
@@ -9946,7 +10081,7 @@ define i64 @atomic_use_cond(ptr %a) {
 ;
 ; O2-LABEL: atomic_use_cond:
 ; O2:       # %bb.0: # %entry
-; O2-NEXT:  .Lpcsection327:
+; O2-NEXT:  .Lpcsection372:
 ; O2-NEXT:    lock decq (%rdi)
 ; O2-NEXT:    jne .LBB197_2
 ; O2-NEXT:  # %bb.1: # %then
@@ -9958,7 +10093,7 @@ define i64 @atomic_use_cond(ptr %a) {
 ;
 ; O3-LABEL: atomic_use_cond:
 ; O3:       # %bb.0: # %entry
-; O3-NEXT:  .Lpcsection327:
+; O3-NEXT:  .Lpcsection372:
 ; O3-NEXT:    lock decq (%rdi)
 ; O3-NEXT:    jne .LBB197_2
 ; O3-NEXT:  # %bb.1: # %then
@@ -10005,15 +10140,15 @@ define i128 @atomic128_load_unordered(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection328:
+; O1-NEXT:  .Lpcsection373:
 ; O1-NEXT:    xorl %eax, %eax
-; O1-NEXT:  .Lpcsection329:
+; O1-NEXT:  .Lpcsection374:
 ; O1-NEXT:    xorl %edx, %edx
-; O1-NEXT:  .Lpcsection330:
+; O1-NEXT:  .Lpcsection375:
 ; O1-NEXT:    xorl %ecx, %ecx
-; O1-NEXT:  .Lpcsection331:
+; O1-NEXT:  .Lpcsection376:
 ; O1-NEXT:    xorl %ebx, %ebx
-; O1-NEXT:  .Lpcsection332:
+; O1-NEXT:  .Lpcsection377:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    popq %rbx
@@ -10026,15 +10161,15 @@ define i128 @atomic128_load_unordered(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection328:
+; O2-NEXT:  .Lpcsection373:
 ; O2-NEXT:    xorl %eax, %eax
-; O2-NEXT:  .Lpcsection329:
+; O2-NEXT:  .Lpcsection374:
 ; O2-NEXT:    xorl %edx, %edx
-; O2-NEXT:  .Lpcsection330:
+; O2-NEXT:  .Lpcsection375:
 ; O2-NEXT:    xorl %ecx, %ecx
-; O2-NEXT:  .Lpcsection331:
+; O2-NEXT:  .Lpcsection376:
 ; O2-NEXT:    xorl %ebx, %ebx
-; O2-NEXT:  .Lpcsection332:
+; O2-NEXT:  .Lpcsection377:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    popq %rbx
@@ -10047,15 +10182,15 @@ define i128 @atomic128_load_unordered(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection328:
+; O3-NEXT:  .Lpcsection373:
 ; O3-NEXT:    xorl %eax, %eax
-; O3-NEXT:  .Lpcsection329:
+; O3-NEXT:  .Lpcsection374:
 ; O3-NEXT:    xorl %edx, %edx
-; O3-NEXT:  .Lpcsection330:
+; O3-NEXT:  .Lpcsection375:
 ; O3-NEXT:    xorl %ecx, %ecx
-; O3-NEXT:  .Lpcsection331:
+; O3-NEXT:  .Lpcsection376:
 ; O3-NEXT:    xorl %ebx, %ebx
-; O3-NEXT:  .Lpcsection332:
+; O3-NEXT:  .Lpcsection377:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    popq %rbx
@@ -10094,15 +10229,15 @@ define i128 @atomic128_load_monotonic(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection333:
+; O1-NEXT:  .Lpcsection378:
 ; O1-NEXT:    xorl %eax, %eax
-; O1-NEXT:  .Lpcsection334:
+; O1-NEXT:  .Lpcsection379:
 ; O1-NEXT:    xorl %edx, %edx
-; O1-NEXT:  .Lpcsection335:
+; O1-NEXT:  .Lpcsection380:
 ; O1-NEXT:    xorl %ecx, %ecx
-; O1-NEXT:  .Lpcsection336:
+; O1-NEXT:  .Lpcsection381:
 ; O1-NEXT:    xorl %ebx, %ebx
-; O1-NEXT:  .Lpcsection337:
+; O1-NEXT:  .Lpcsection382:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    popq %rbx
@@ -10115,15 +10250,15 @@ define i128 @atomic128_load_monotonic(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection333:
+; O2-NEXT:  .Lpcsection378:
 ; O2-NEXT:    xorl %eax, %eax
-; O2-NEXT:  .Lpcsection334:
+; O2-NEXT:  .Lpcsection379:
 ; O2-NEXT:    xorl %edx, %edx
-; O2-NEXT:  .Lpcsection335:
+; O2-NEXT:  .Lpcsection380:
 ; O2-NEXT:    xorl %ecx, %ecx
-; O2-NEXT:  .Lpcsection336:
+; O2-NEXT:  .Lpcsection381:
 ; O2-NEXT:    xorl %ebx, %ebx
-; O2-NEXT:  .Lpcsection337:
+; O2-NEXT:  .Lpcsection382:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    popq %rbx
@@ -10136,15 +10271,15 @@ define i128 @atomic128_load_monotonic(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection333:
+; O3-NEXT:  .Lpcsection378:
 ; O3-NEXT:    xorl %eax, %eax
-; O3-NEXT:  .Lpcsection334:
+; O3-NEXT:  .Lpcsection379:
 ; O3-NEXT:    xorl %edx, %edx
-; O3-NEXT:  .Lpcsection335:
+; O3-NEXT:  .Lpcsection380:
 ; O3-NEXT:    xorl %ecx, %ecx
-; O3-NEXT:  .Lpcsection336:
+; O3-NEXT:  .Lpcsection381:
 ; O3-NEXT:    xorl %ebx, %ebx
-; O3-NEXT:  .Lpcsection337:
+; O3-NEXT:  .Lpcsection382:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    popq %rbx
@@ -10183,15 +10318,15 @@ define i128 @atomic128_load_acquire(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection338:
+; O1-NEXT:  .Lpcsection383:
 ; O1-NEXT:    xorl %eax, %eax
-; O1-NEXT:  .Lpcsection339:
+; O1-NEXT:  .Lpcsection384:
 ; O1-NEXT:    xorl %edx, %edx
-; O1-NEXT:  .Lpcsection340:
+; O1-NEXT:  .Lpcsection385:
 ; O1-NEXT:    xorl %ecx, %ecx
-; O1-NEXT:  .Lpcsection341:
+; O1-NEXT:  .Lpcsection386:
 ; O1-NEXT:    xorl %ebx, %ebx
-; O1-NEXT:  .Lpcsection342:
+; O1-NEXT:  .Lpcsection387:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    popq %rbx
@@ -10204,15 +10339,15 @@ define i128 @atomic128_load_acquire(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection338:
+; O2-NEXT:  .Lpcsection383:
 ; O2-NEXT:    xorl %eax, %eax
-; O2-NEXT:  .Lpcsection339:
+; O2-NEXT:  .Lpcsection384:
 ; O2-NEXT:    xorl %edx, %edx
-; O2-NEXT:  .Lpcsection340:
+; O2-NEXT:  .Lpcsection385:
 ; O2-NEXT:    xorl %ecx, %ecx
-; O2-NEXT:  .Lpcsection341:
+; O2-NEXT:  .Lpcsection386:
 ; O2-NEXT:    xorl %ebx, %ebx
-; O2-NEXT:  .Lpcsection342:
+; O2-NEXT:  .Lpcsection387:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    popq %rbx
@@ -10225,15 +10360,15 @@ define i128 @atomic128_load_acquire(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection338:
+; O3-NEXT:  .Lpcsection383:
 ; O3-NEXT:    xorl %eax, %eax
-; O3-NEXT:  .Lpcsection339:
+; O3-NEXT:  .Lpcsection384:
 ; O3-NEXT:    xorl %edx, %edx
-; O3-NEXT:  .Lpcsection340:
+; O3-NEXT:  .Lpcsection385:
 ; O3-NEXT:    xorl %ecx, %ecx
-; O3-NEXT:  .Lpcsection341:
+; O3-NEXT:  .Lpcsection386:
 ; O3-NEXT:    xorl %ebx, %ebx
-; O3-NEXT:  .Lpcsection342:
+; O3-NEXT:  .Lpcsection387:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    popq %rbx
@@ -10272,15 +10407,15 @@ define i128 @atomic128_load_seq_cst(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection343:
+; O1-NEXT:  .Lpcsection388:
 ; O1-NEXT:    xorl %eax, %eax
-; O1-NEXT:  .Lpcsection344:
+; O1-NEXT:  .Lpcsection389:
 ; O1-NEXT:    xorl %edx, %edx
-; O1-NEXT:  .Lpcsection345:
+; O1-NEXT:  .Lpcsection390:
 ; O1-NEXT:    xorl %ecx, %ecx
-; O1-NEXT:  .Lpcsection346:
+; O1-NEXT:  .Lpcsection391:
 ; O1-NEXT:    xorl %ebx, %ebx
-; O1-NEXT:  .Lpcsection347:
+; O1-NEXT:  .Lpcsection392:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    popq %rbx
@@ -10293,15 +10428,15 @@ define i128 @atomic128_load_seq_cst(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection343:
+; O2-NEXT:  .Lpcsection388:
 ; O2-NEXT:    xorl %eax, %eax
-; O2-NEXT:  .Lpcsection344:
+; O2-NEXT:  .Lpcsection389:
 ; O2-NEXT:    xorl %edx, %edx
-; O2-NEXT:  .Lpcsection345:
+; O2-NEXT:  .Lpcsection390:
 ; O2-NEXT:    xorl %ecx, %ecx
-; O2-NEXT:  .Lpcsection346:
+; O2-NEXT:  .Lpcsection391:
 ; O2-NEXT:    xorl %ebx, %ebx
-; O2-NEXT:  .Lpcsection347:
+; O2-NEXT:  .Lpcsection392:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    popq %rbx
@@ -10314,15 +10449,15 @@ define i128 @atomic128_load_seq_cst(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection343:
+; O3-NEXT:  .Lpcsection388:
 ; O3-NEXT:    xorl %eax, %eax
-; O3-NEXT:  .Lpcsection344:
+; O3-NEXT:  .Lpcsection389:
 ; O3-NEXT:    xorl %edx, %edx
-; O3-NEXT:  .Lpcsection345:
+; O3-NEXT:  .Lpcsection390:
 ; O3-NEXT:    xorl %ecx, %ecx
-; O3-NEXT:  .Lpcsection346:
+; O3-NEXT:  .Lpcsection391:
 ; O3-NEXT:    xorl %ebx, %ebx
-; O3-NEXT:  .Lpcsection347:
+; O3-NEXT:  .Lpcsection392:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    popq %rbx
@@ -10347,7 +10482,7 @@ define ptr @atomic128_load_seq_cst_ptr_ty(ptr %a) {
 ; O1-LABEL: atomic128_load_seq_cst_ptr_ty:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection348:
+; O1-NEXT:  .Lpcsection393:
 ; O1-NEXT:    movq (%rdi), %rax
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -10355,7 +10490,7 @@ define ptr @atomic128_load_seq_cst_ptr_ty(ptr %a) {
 ; O2-LABEL: atomic128_load_seq_cst_ptr_ty:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection348:
+; O2-NEXT:  .Lpcsection393:
 ; O2-NEXT:    movq (%rdi), %rax
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -10363,7 +10498,7 @@ define ptr @atomic128_load_seq_cst_ptr_ty(ptr %a) {
 ; O3-LABEL: atomic128_load_seq_cst_ptr_ty:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection348:
+; O3-NEXT:  .Lpcsection393:
 ; O3-NEXT:    movq (%rdi), %rax
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -10420,20 +10555,20 @@ define void @atomic128_store_unordered(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection349:
+; O1-NEXT:  .Lpcsection394:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection350:
+; O1-NEXT:  .Lpcsection395:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:  .Lpcsection351:
+; O1-NEXT:  .Lpcsection396:
 ; O1-NEXT:    movl $42, %ebx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB203_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
-; O1-NEXT:  .Lpcsection352:
+; O1-NEXT:  .Lpcsection397:
 ; O1-NEXT:    xorl %ecx, %ecx
-; O1-NEXT:  .Lpcsection353:
+; O1-NEXT:  .Lpcsection398:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection354:
+; O1-NEXT:  .Lpcsection399:
 ; O1-NEXT:    jne .LBB203_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -10447,20 +10582,20 @@ define void @atomic128_store_unordered(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection349:
+; O2-NEXT:  .Lpcsection394:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection350:
+; O2-NEXT:  .Lpcsection395:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:  .Lpcsection351:
+; O2-NEXT:  .Lpcsection396:
 ; O2-NEXT:    movl $42, %ebx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB203_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
-; O2-NEXT:  .Lpcsection352:
+; O2-NEXT:  .Lpcsection397:
 ; O2-NEXT:    xorl %ecx, %ecx
-; O2-NEXT:  .Lpcsection353:
+; O2-NEXT:  .Lpcsection398:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection354:
+; O2-NEXT:  .Lpcsection399:
 ; O2-NEXT:    jne .LBB203_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -10474,20 +10609,20 @@ define void @atomic128_store_unordered(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection349:
+; O3-NEXT:  .Lpcsection394:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection350:
+; O3-NEXT:  .Lpcsection395:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:  .Lpcsection351:
+; O3-NEXT:  .Lpcsection396:
 ; O3-NEXT:    movl $42, %ebx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB203_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
-; O3-NEXT:  .Lpcsection352:
+; O3-NEXT:  .Lpcsection397:
 ; O3-NEXT:    xorl %ecx, %ecx
-; O3-NEXT:  .Lpcsection353:
+; O3-NEXT:  .Lpcsection398:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection354:
+; O3-NEXT:  .Lpcsection399:
 ; O3-NEXT:    jne .LBB203_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -10547,20 +10682,20 @@ define void @atomic128_store_monotonic(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection355:
+; O1-NEXT:  .Lpcsection400:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection356:
+; O1-NEXT:  .Lpcsection401:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:  .Lpcsection357:
+; O1-NEXT:  .Lpcsection402:
 ; O1-NEXT:    movl $42, %ebx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB204_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
-; O1-NEXT:  .Lpcsection358:
+; O1-NEXT:  .Lpcsection403:
 ; O1-NEXT:    xorl %ecx, %ecx
-; O1-NEXT:  .Lpcsection359:
+; O1-NEXT:  .Lpcsection404:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection360:
+; O1-NEXT:  .Lpcsection405:
 ; O1-NEXT:    jne .LBB204_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -10574,20 +10709,20 @@ define void @atomic128_store_monotonic(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection355:
+; O2-NEXT:  .Lpcsection400:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection356:
+; O2-NEXT:  .Lpcsection401:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:  .Lpcsection357:
+; O2-NEXT:  .Lpcsection402:
 ; O2-NEXT:    movl $42, %ebx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB204_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
-; O2-NEXT:  .Lpcsection358:
+; O2-NEXT:  .Lpcsection403:
 ; O2-NEXT:    xorl %ecx, %ecx
-; O2-NEXT:  .Lpcsection359:
+; O2-NEXT:  .Lpcsection404:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection360:
+; O2-NEXT:  .Lpcsection405:
 ; O2-NEXT:    jne .LBB204_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -10601,20 +10736,20 @@ define void @atomic128_store_monotonic(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection355:
+; O3-NEXT:  .Lpcsection400:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection356:
+; O3-NEXT:  .Lpcsection401:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:  .Lpcsection357:
+; O3-NEXT:  .Lpcsection402:
 ; O3-NEXT:    movl $42, %ebx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB204_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
-; O3-NEXT:  .Lpcsection358:
+; O3-NEXT:  .Lpcsection403:
 ; O3-NEXT:    xorl %ecx, %ecx
-; O3-NEXT:  .Lpcsection359:
+; O3-NEXT:  .Lpcsection404:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection360:
+; O3-NEXT:  .Lpcsection405:
 ; O3-NEXT:    jne .LBB204_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -10674,20 +10809,20 @@ define void @atomic128_store_release(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection361:
+; O1-NEXT:  .Lpcsection406:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection362:
+; O1-NEXT:  .Lpcsection407:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:  .Lpcsection363:
+; O1-NEXT:  .Lpcsection408:
 ; O1-NEXT:    movl $42, %ebx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB205_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
-; O1-NEXT:  .Lpcsection364:
+; O1-NEXT:  .Lpcsection409:
 ; O1-NEXT:    xorl %ecx, %ecx
-; O1-NEXT:  .Lpcsection365:
+; O1-NEXT:  .Lpcsection410:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection366:
+; O1-NEXT:  .Lpcsection411:
 ; O1-NEXT:    jne .LBB205_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -10701,20 +10836,20 @@ define void @atomic128_store_release(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection361:
+; O2-NEXT:  .Lpcsection406:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection362:
+; O2-NEXT:  .Lpcsection407:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:  .Lpcsection363:
+; O2-NEXT:  .Lpcsection408:
 ; O2-NEXT:    movl $42, %ebx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB205_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
-; O2-NEXT:  .Lpcsection364:
+; O2-NEXT:  .Lpcsection409:
 ; O2-NEXT:    xorl %ecx, %ecx
-; O2-NEXT:  .Lpcsection365:
+; O2-NEXT:  .Lpcsection410:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection366:
+; O2-NEXT:  .Lpcsection411:
 ; O2-NEXT:    jne .LBB205_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -10728,20 +10863,20 @@ define void @atomic128_store_release(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection361:
+; O3-NEXT:  .Lpcsection406:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection362:
+; O3-NEXT:  .Lpcsection407:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:  .Lpcsection363:
+; O3-NEXT:  .Lpcsection408:
 ; O3-NEXT:    movl $42, %ebx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB205_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
-; O3-NEXT:  .Lpcsection364:
+; O3-NEXT:  .Lpcsection409:
 ; O3-NEXT:    xorl %ecx, %ecx
-; O3-NEXT:  .Lpcsection365:
+; O3-NEXT:  .Lpcsection410:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection366:
+; O3-NEXT:  .Lpcsection411:
 ; O3-NEXT:    jne .LBB205_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -10801,20 +10936,20 @@ define void @atomic128_store_seq_cst(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection367:
+; O1-NEXT:  .Lpcsection412:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection368:
+; O1-NEXT:  .Lpcsection413:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:  .Lpcsection369:
+; O1-NEXT:  .Lpcsection414:
 ; O1-NEXT:    movl $42, %ebx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB206_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
-; O1-NEXT:  .Lpcsection370:
+; O1-NEXT:  .Lpcsection415:
 ; O1-NEXT:    xorl %ecx, %ecx
-; O1-NEXT:  .Lpcsection371:
+; O1-NEXT:  .Lpcsection416:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection372:
+; O1-NEXT:  .Lpcsection417:
 ; O1-NEXT:    jne .LBB206_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -10828,20 +10963,20 @@ define void @atomic128_store_seq_cst(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection367:
+; O2-NEXT:  .Lpcsection412:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection368:
+; O2-NEXT:  .Lpcsection413:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:  .Lpcsection369:
+; O2-NEXT:  .Lpcsection414:
 ; O2-NEXT:    movl $42, %ebx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB206_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
-; O2-NEXT:  .Lpcsection370:
+; O2-NEXT:  .Lpcsection415:
 ; O2-NEXT:    xorl %ecx, %ecx
-; O2-NEXT:  .Lpcsection371:
+; O2-NEXT:  .Lpcsection416:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection372:
+; O2-NEXT:  .Lpcsection417:
 ; O2-NEXT:    jne .LBB206_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -10855,20 +10990,20 @@ define void @atomic128_store_seq_cst(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection367:
+; O3-NEXT:  .Lpcsection412:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection368:
+; O3-NEXT:  .Lpcsection413:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:  .Lpcsection369:
+; O3-NEXT:  .Lpcsection414:
 ; O3-NEXT:    movl $42, %ebx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB206_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
-; O3-NEXT:  .Lpcsection370:
+; O3-NEXT:  .Lpcsection415:
 ; O3-NEXT:    xorl %ecx, %ecx
-; O3-NEXT:  .Lpcsection371:
+; O3-NEXT:  .Lpcsection416:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection372:
+; O3-NEXT:  .Lpcsection417:
 ; O3-NEXT:    jne .LBB206_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -10894,7 +11029,7 @@ define void @atomic128_store_seq_cst_ptr_ty(ptr %a, ptr %v) {
 ; O1-LABEL: atomic128_store_seq_cst_ptr_ty:
 ; O1:       # %bb.0: # %entry
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection373:
+; O1-NEXT:  .Lpcsection418:
 ; O1-NEXT:    xchgq %rsi, (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    retq
@@ -10902,7 +11037,7 @@ define void @atomic128_store_seq_cst_ptr_ty(ptr %a, ptr %v) {
 ; O2-LABEL: atomic128_store_seq_cst_ptr_ty:
 ; O2:       # %bb.0: # %entry
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection373:
+; O2-NEXT:  .Lpcsection418:
 ; O2-NEXT:    xchgq %rsi, (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    retq
@@ -10910,7 +11045,7 @@ define void @atomic128_store_seq_cst_ptr_ty(ptr %a, ptr %v) {
 ; O3-LABEL: atomic128_store_seq_cst_ptr_ty:
 ; O3:       # %bb.0: # %entry
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection373:
+; O3-NEXT:  .Lpcsection418:
 ; O3-NEXT:    xchgq %rsi, (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    retq
@@ -10967,20 +11102,20 @@ define void @atomic128_xchg_monotonic(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection374:
+; O1-NEXT:  .Lpcsection419:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection375:
+; O1-NEXT:  .Lpcsection420:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:  .Lpcsection376:
+; O1-NEXT:  .Lpcsection421:
 ; O1-NEXT:    movl $42, %ebx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB208_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
-; O1-NEXT:  .Lpcsection377:
+; O1-NEXT:  .Lpcsection422:
 ; O1-NEXT:    xorl %ecx, %ecx
-; O1-NEXT:  .Lpcsection378:
+; O1-NEXT:  .Lpcsection423:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection379:
+; O1-NEXT:  .Lpcsection424:
 ; O1-NEXT:    jne .LBB208_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -10994,20 +11129,20 @@ define void @atomic128_xchg_monotonic(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection374:
+; O2-NEXT:  .Lpcsection419:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection375:
+; O2-NEXT:  .Lpcsection420:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:  .Lpcsection376:
+; O2-NEXT:  .Lpcsection421:
 ; O2-NEXT:    movl $42, %ebx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB208_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
-; O2-NEXT:  .Lpcsection377:
+; O2-NEXT:  .Lpcsection422:
 ; O2-NEXT:    xorl %ecx, %ecx
-; O2-NEXT:  .Lpcsection378:
+; O2-NEXT:  .Lpcsection423:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection379:
+; O2-NEXT:  .Lpcsection424:
 ; O2-NEXT:    jne .LBB208_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -11021,20 +11156,20 @@ define void @atomic128_xchg_monotonic(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection374:
+; O3-NEXT:  .Lpcsection419:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection375:
+; O3-NEXT:  .Lpcsection420:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:  .Lpcsection376:
+; O3-NEXT:  .Lpcsection421:
 ; O3-NEXT:    movl $42, %ebx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB208_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
-; O3-NEXT:  .Lpcsection377:
+; O3-NEXT:  .Lpcsection422:
 ; O3-NEXT:    xorl %ecx, %ecx
-; O3-NEXT:  .Lpcsection378:
+; O3-NEXT:  .Lpcsection423:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection379:
+; O3-NEXT:  .Lpcsection424:
 ; O3-NEXT:    jne .LBB208_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -11094,22 +11229,22 @@ define void @atomic128_add_monotonic(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection380:
+; O1-NEXT:  .Lpcsection425:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection381:
+; O1-NEXT:  .Lpcsection426:
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB209_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
-; O1-NEXT:  .Lpcsection382:
+; O1-NEXT:  .Lpcsection427:
 ; O1-NEXT:    addq $42, %rbx
 ; O1-NEXT:    movq %rdx, %rcx
-; O1-NEXT:  .Lpcsection383:
+; O1-NEXT:  .Lpcsection428:
 ; O1-NEXT:    adcq $0, %rcx
-; O1-NEXT:  .Lpcsection384:
+; O1-NEXT:  .Lpcsection429:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection385:
+; O1-NEXT:  .Lpcsection430:
 ; O1-NEXT:    jne .LBB209_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -11123,22 +11258,22 @@ define void @atomic128_add_monotonic(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection380:
+; O2-NEXT:  .Lpcsection425:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection381:
+; O2-NEXT:  .Lpcsection426:
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB209_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
-; O2-NEXT:  .Lpcsection382:
+; O2-NEXT:  .Lpcsection427:
 ; O2-NEXT:    addq $42, %rbx
 ; O2-NEXT:    movq %rdx, %rcx
-; O2-NEXT:  .Lpcsection383:
+; O2-NEXT:  .Lpcsection428:
 ; O2-NEXT:    adcq $0, %rcx
-; O2-NEXT:  .Lpcsection384:
+; O2-NEXT:  .Lpcsection429:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection385:
+; O2-NEXT:  .Lpcsection430:
 ; O2-NEXT:    jne .LBB209_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -11152,22 +11287,22 @@ define void @atomic128_add_monotonic(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection380:
+; O3-NEXT:  .Lpcsection425:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection381:
+; O3-NEXT:  .Lpcsection426:
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB209_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
-; O3-NEXT:  .Lpcsection382:
+; O3-NEXT:  .Lpcsection427:
 ; O3-NEXT:    addq $42, %rbx
 ; O3-NEXT:    movq %rdx, %rcx
-; O3-NEXT:  .Lpcsection383:
+; O3-NEXT:  .Lpcsection428:
 ; O3-NEXT:    adcq $0, %rcx
-; O3-NEXT:  .Lpcsection384:
+; O3-NEXT:  .Lpcsection429:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection385:
+; O3-NEXT:  .Lpcsection430:
 ; O3-NEXT:    jne .LBB209_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -11227,22 +11362,22 @@ define void @atomic128_sub_monotonic(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection386:
+; O1-NEXT:  .Lpcsection431:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection387:
+; O1-NEXT:  .Lpcsection432:
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB210_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
-; O1-NEXT:  .Lpcsection388:
+; O1-NEXT:  .Lpcsection433:
 ; O1-NEXT:    addq $-42, %rbx
 ; O1-NEXT:    movq %rdx, %rcx
-; O1-NEXT:  .Lpcsection389:
+; O1-NEXT:  .Lpcsection434:
 ; O1-NEXT:    adcq $-1, %rcx
-; O1-NEXT:  .Lpcsection390:
+; O1-NEXT:  .Lpcsection435:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection391:
+; O1-NEXT:  .Lpcsection436:
 ; O1-NEXT:    jne .LBB210_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -11256,22 +11391,22 @@ define void @atomic128_sub_monotonic(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection386:
+; O2-NEXT:  .Lpcsection431:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection387:
+; O2-NEXT:  .Lpcsection432:
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB210_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
-; O2-NEXT:  .Lpcsection388:
+; O2-NEXT:  .Lpcsection433:
 ; O2-NEXT:    addq $-42, %rbx
 ; O2-NEXT:    movq %rdx, %rcx
-; O2-NEXT:  .Lpcsection389:
+; O2-NEXT:  .Lpcsection434:
 ; O2-NEXT:    adcq $-1, %rcx
-; O2-NEXT:  .Lpcsection390:
+; O2-NEXT:  .Lpcsection435:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection391:
+; O2-NEXT:  .Lpcsection436:
 ; O2-NEXT:    jne .LBB210_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -11285,22 +11420,22 @@ define void @atomic128_sub_monotonic(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection386:
+; O3-NEXT:  .Lpcsection431:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection387:
+; O3-NEXT:  .Lpcsection432:
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB210_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
-; O3-NEXT:  .Lpcsection388:
+; O3-NEXT:  .Lpcsection433:
 ; O3-NEXT:    addq $-42, %rbx
 ; O3-NEXT:    movq %rdx, %rcx
-; O3-NEXT:  .Lpcsection389:
+; O3-NEXT:  .Lpcsection434:
 ; O3-NEXT:    adcq $-1, %rcx
-; O3-NEXT:  .Lpcsection390:
+; O3-NEXT:  .Lpcsection435:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection391:
+; O3-NEXT:  .Lpcsection436:
 ; O3-NEXT:    jne .LBB210_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -11362,21 +11497,21 @@ define void @atomic128_and_monotonic(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection392:
+; O1-NEXT:  .Lpcsection437:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection393:
+; O1-NEXT:  .Lpcsection438:
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB211_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ebx
-; O1-NEXT:  .Lpcsection394:
+; O1-NEXT:  .Lpcsection439:
 ; O1-NEXT:    andl $42, %ebx
-; O1-NEXT:  .Lpcsection395:
+; O1-NEXT:  .Lpcsection440:
 ; O1-NEXT:    xorl %ecx, %ecx
-; O1-NEXT:  .Lpcsection396:
+; O1-NEXT:  .Lpcsection441:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection397:
+; O1-NEXT:  .Lpcsection442:
 ; O1-NEXT:    jne .LBB211_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -11390,21 +11525,21 @@ define void @atomic128_and_monotonic(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection392:
+; O2-NEXT:  .Lpcsection437:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection393:
+; O2-NEXT:  .Lpcsection438:
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB211_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ebx
-; O2-NEXT:  .Lpcsection394:
+; O2-NEXT:  .Lpcsection439:
 ; O2-NEXT:    andl $42, %ebx
-; O2-NEXT:  .Lpcsection395:
+; O2-NEXT:  .Lpcsection440:
 ; O2-NEXT:    xorl %ecx, %ecx
-; O2-NEXT:  .Lpcsection396:
+; O2-NEXT:  .Lpcsection441:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection397:
+; O2-NEXT:  .Lpcsection442:
 ; O2-NEXT:    jne .LBB211_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -11418,21 +11553,21 @@ define void @atomic128_and_monotonic(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection392:
+; O3-NEXT:  .Lpcsection437:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection393:
+; O3-NEXT:  .Lpcsection438:
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB211_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ebx
-; O3-NEXT:  .Lpcsection394:
+; O3-NEXT:  .Lpcsection439:
 ; O3-NEXT:    andl $42, %ebx
-; O3-NEXT:  .Lpcsection395:
+; O3-NEXT:  .Lpcsection440:
 ; O3-NEXT:    xorl %ecx, %ecx
-; O3-NEXT:  .Lpcsection396:
+; O3-NEXT:  .Lpcsection441:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection397:
+; O3-NEXT:  .Lpcsection442:
 ; O3-NEXT:    jne .LBB211_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -11490,20 +11625,20 @@ define void @atomic128_or_monotonic(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection398:
+; O1-NEXT:  .Lpcsection443:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection399:
+; O1-NEXT:  .Lpcsection444:
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB212_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
-; O1-NEXT:  .Lpcsection400:
+; O1-NEXT:  .Lpcsection445:
 ; O1-NEXT:    orq $42, %rbx
 ; O1-NEXT:    movq %rdx, %rcx
-; O1-NEXT:  .Lpcsection401:
+; O1-NEXT:  .Lpcsection446:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection402:
+; O1-NEXT:  .Lpcsection447:
 ; O1-NEXT:    jne .LBB212_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -11517,20 +11652,20 @@ define void @atomic128_or_monotonic(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection398:
+; O2-NEXT:  .Lpcsection443:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection399:
+; O2-NEXT:  .Lpcsection444:
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB212_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
-; O2-NEXT:  .Lpcsection400:
+; O2-NEXT:  .Lpcsection445:
 ; O2-NEXT:    orq $42, %rbx
 ; O2-NEXT:    movq %rdx, %rcx
-; O2-NEXT:  .Lpcsection401:
+; O2-NEXT:  .Lpcsection446:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection402:
+; O2-NEXT:  .Lpcsection447:
 ; O2-NEXT:    jne .LBB212_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -11544,20 +11679,20 @@ define void @atomic128_or_monotonic(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection398:
+; O3-NEXT:  .Lpcsection443:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection399:
+; O3-NEXT:  .Lpcsection444:
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB212_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
-; O3-NEXT:  .Lpcsection400:
+; O3-NEXT:  .Lpcsection445:
 ; O3-NEXT:    orq $42, %rbx
 ; O3-NEXT:    movq %rdx, %rcx
-; O3-NEXT:  .Lpcsection401:
+; O3-NEXT:  .Lpcsection446:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection402:
+; O3-NEXT:  .Lpcsection447:
 ; O3-NEXT:    jne .LBB212_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -11615,20 +11750,20 @@ define void @atomic128_xor_monotonic(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection403:
+; O1-NEXT:  .Lpcsection448:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection404:
+; O1-NEXT:  .Lpcsection449:
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB213_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
-; O1-NEXT:  .Lpcsection405:
+; O1-NEXT:  .Lpcsection450:
 ; O1-NEXT:    xorq $42, %rbx
 ; O1-NEXT:    movq %rdx, %rcx
-; O1-NEXT:  .Lpcsection406:
+; O1-NEXT:  .Lpcsection451:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection407:
+; O1-NEXT:  .Lpcsection452:
 ; O1-NEXT:    jne .LBB213_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -11642,20 +11777,20 @@ define void @atomic128_xor_monotonic(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection403:
+; O2-NEXT:  .Lpcsection448:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection404:
+; O2-NEXT:  .Lpcsection449:
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB213_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
-; O2-NEXT:  .Lpcsection405:
+; O2-NEXT:  .Lpcsection450:
 ; O2-NEXT:    xorq $42, %rbx
 ; O2-NEXT:    movq %rdx, %rcx
-; O2-NEXT:  .Lpcsection406:
+; O2-NEXT:  .Lpcsection451:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection407:
+; O2-NEXT:  .Lpcsection452:
 ; O2-NEXT:    jne .LBB213_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -11669,20 +11804,20 @@ define void @atomic128_xor_monotonic(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection403:
+; O3-NEXT:  .Lpcsection448:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection404:
+; O3-NEXT:  .Lpcsection449:
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB213_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
-; O3-NEXT:  .Lpcsection405:
+; O3-NEXT:  .Lpcsection450:
 ; O3-NEXT:    xorq $42, %rbx
 ; O3-NEXT:    movq %rdx, %rcx
-; O3-NEXT:  .Lpcsection406:
+; O3-NEXT:  .Lpcsection451:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection407:
+; O3-NEXT:  .Lpcsection452:
 ; O3-NEXT:    jne .LBB213_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -11746,23 +11881,23 @@ define void @atomic128_nand_monotonic(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection408:
+; O1-NEXT:  .Lpcsection453:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection409:
+; O1-NEXT:  .Lpcsection454:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:  .Lpcsection410:
+; O1-NEXT:  .Lpcsection455:
 ; O1-NEXT:    movq $-1, %rcx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB214_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ebx
-; O1-NEXT:  .Lpcsection411:
+; O1-NEXT:  .Lpcsection456:
 ; O1-NEXT:    notl %ebx
-; O1-NEXT:  .Lpcsection412:
+; O1-NEXT:  .Lpcsection457:
 ; O1-NEXT:    orq $-43, %rbx
-; O1-NEXT:  .Lpcsection413:
+; O1-NEXT:  .Lpcsection458:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection414:
+; O1-NEXT:  .Lpcsection459:
 ; O1-NEXT:    jne .LBB214_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -11776,23 +11911,23 @@ define void @atomic128_nand_monotonic(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection408:
+; O2-NEXT:  .Lpcsection453:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection409:
+; O2-NEXT:  .Lpcsection454:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:  .Lpcsection410:
+; O2-NEXT:  .Lpcsection455:
 ; O2-NEXT:    movq $-1, %rcx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB214_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ebx
-; O2-NEXT:  .Lpcsection411:
+; O2-NEXT:  .Lpcsection456:
 ; O2-NEXT:    notl %ebx
-; O2-NEXT:  .Lpcsection412:
+; O2-NEXT:  .Lpcsection457:
 ; O2-NEXT:    orq $-43, %rbx
-; O2-NEXT:  .Lpcsection413:
+; O2-NEXT:  .Lpcsection458:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection414:
+; O2-NEXT:  .Lpcsection459:
 ; O2-NEXT:    jne .LBB214_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -11806,23 +11941,23 @@ define void @atomic128_nand_monotonic(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection408:
+; O3-NEXT:  .Lpcsection453:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection409:
+; O3-NEXT:  .Lpcsection454:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:  .Lpcsection410:
+; O3-NEXT:  .Lpcsection455:
 ; O3-NEXT:    movq $-1, %rcx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB214_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ebx
-; O3-NEXT:  .Lpcsection411:
+; O3-NEXT:  .Lpcsection456:
 ; O3-NEXT:    notl %ebx
-; O3-NEXT:  .Lpcsection412:
+; O3-NEXT:  .Lpcsection457:
 ; O3-NEXT:    orq $-43, %rbx
-; O3-NEXT:  .Lpcsection413:
+; O3-NEXT:  .Lpcsection458:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection414:
+; O3-NEXT:  .Lpcsection459:
 ; O3-NEXT:    jne .LBB214_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -11882,20 +12017,20 @@ define void @atomic128_xchg_acquire(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection415:
+; O1-NEXT:  .Lpcsection460:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection416:
+; O1-NEXT:  .Lpcsection461:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:  .Lpcsection417:
+; O1-NEXT:  .Lpcsection462:
 ; O1-NEXT:    movl $42, %ebx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB215_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
-; O1-NEXT:  .Lpcsection418:
+; O1-NEXT:  .Lpcsection463:
 ; O1-NEXT:    xorl %ecx, %ecx
-; O1-NEXT:  .Lpcsection419:
+; O1-NEXT:  .Lpcsection464:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection420:
+; O1-NEXT:  .Lpcsection465:
 ; O1-NEXT:    jne .LBB215_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -11909,20 +12044,20 @@ define void @atomic128_xchg_acquire(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection415:
+; O2-NEXT:  .Lpcsection460:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection416:
+; O2-NEXT:  .Lpcsection461:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:  .Lpcsection417:
+; O2-NEXT:  .Lpcsection462:
 ; O2-NEXT:    movl $42, %ebx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB215_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
-; O2-NEXT:  .Lpcsection418:
+; O2-NEXT:  .Lpcsection463:
 ; O2-NEXT:    xorl %ecx, %ecx
-; O2-NEXT:  .Lpcsection419:
+; O2-NEXT:  .Lpcsection464:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection420:
+; O2-NEXT:  .Lpcsection465:
 ; O2-NEXT:    jne .LBB215_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -11936,20 +12071,20 @@ define void @atomic128_xchg_acquire(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection415:
+; O3-NEXT:  .Lpcsection460:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection416:
+; O3-NEXT:  .Lpcsection461:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:  .Lpcsection417:
+; O3-NEXT:  .Lpcsection462:
 ; O3-NEXT:    movl $42, %ebx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB215_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
-; O3-NEXT:  .Lpcsection418:
+; O3-NEXT:  .Lpcsection463:
 ; O3-NEXT:    xorl %ecx, %ecx
-; O3-NEXT:  .Lpcsection419:
+; O3-NEXT:  .Lpcsection464:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection420:
+; O3-NEXT:  .Lpcsection465:
 ; O3-NEXT:    jne .LBB215_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -12009,22 +12144,22 @@ define void @atomic128_add_acquire(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection421:
+; O1-NEXT:  .Lpcsection466:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection422:
+; O1-NEXT:  .Lpcsection467:
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB216_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
-; O1-NEXT:  .Lpcsection423:
+; O1-NEXT:  .Lpcsection468:
 ; O1-NEXT:    addq $42, %rbx
 ; O1-NEXT:    movq %rdx, %rcx
-; O1-NEXT:  .Lpcsection424:
+; O1-NEXT:  .Lpcsection469:
 ; O1-NEXT:    adcq $0, %rcx
-; O1-NEXT:  .Lpcsection425:
+; O1-NEXT:  .Lpcsection470:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection426:
+; O1-NEXT:  .Lpcsection471:
 ; O1-NEXT:    jne .LBB216_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -12038,22 +12173,22 @@ define void @atomic128_add_acquire(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection421:
+; O2-NEXT:  .Lpcsection466:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection422:
+; O2-NEXT:  .Lpcsection467:
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB216_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
-; O2-NEXT:  .Lpcsection423:
+; O2-NEXT:  .Lpcsection468:
 ; O2-NEXT:    addq $42, %rbx
 ; O2-NEXT:    movq %rdx, %rcx
-; O2-NEXT:  .Lpcsection424:
+; O2-NEXT:  .Lpcsection469:
 ; O2-NEXT:    adcq $0, %rcx
-; O2-NEXT:  .Lpcsection425:
+; O2-NEXT:  .Lpcsection470:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection426:
+; O2-NEXT:  .Lpcsection471:
 ; O2-NEXT:    jne .LBB216_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -12067,22 +12202,22 @@ define void @atomic128_add_acquire(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection421:
+; O3-NEXT:  .Lpcsection466:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection422:
+; O3-NEXT:  .Lpcsection467:
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB216_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
-; O3-NEXT:  .Lpcsection423:
+; O3-NEXT:  .Lpcsection468:
 ; O3-NEXT:    addq $42, %rbx
 ; O3-NEXT:    movq %rdx, %rcx
-; O3-NEXT:  .Lpcsection424:
+; O3-NEXT:  .Lpcsection469:
 ; O3-NEXT:    adcq $0, %rcx
-; O3-NEXT:  .Lpcsection425:
+; O3-NEXT:  .Lpcsection470:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection426:
+; O3-NEXT:  .Lpcsection471:
 ; O3-NEXT:    jne .LBB216_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -12142,22 +12277,22 @@ define void @atomic128_sub_acquire(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection427:
+; O1-NEXT:  .Lpcsection472:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection428:
+; O1-NEXT:  .Lpcsection473:
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB217_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
-; O1-NEXT:  .Lpcsection429:
+; O1-NEXT:  .Lpcsection474:
 ; O1-NEXT:    addq $-42, %rbx
 ; O1-NEXT:    movq %rdx, %rcx
-; O1-NEXT:  .Lpcsection430:
+; O1-NEXT:  .Lpcsection475:
 ; O1-NEXT:    adcq $-1, %rcx
-; O1-NEXT:  .Lpcsection431:
+; O1-NEXT:  .Lpcsection476:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection432:
+; O1-NEXT:  .Lpcsection477:
 ; O1-NEXT:    jne .LBB217_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -12171,22 +12306,22 @@ define void @atomic128_sub_acquire(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection427:
+; O2-NEXT:  .Lpcsection472:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection428:
+; O2-NEXT:  .Lpcsection473:
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB217_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
-; O2-NEXT:  .Lpcsection429:
+; O2-NEXT:  .Lpcsection474:
 ; O2-NEXT:    addq $-42, %rbx
 ; O2-NEXT:    movq %rdx, %rcx
-; O2-NEXT:  .Lpcsection430:
+; O2-NEXT:  .Lpcsection475:
 ; O2-NEXT:    adcq $-1, %rcx
-; O2-NEXT:  .Lpcsection431:
+; O2-NEXT:  .Lpcsection476:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection432:
+; O2-NEXT:  .Lpcsection477:
 ; O2-NEXT:    jne .LBB217_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -12200,22 +12335,22 @@ define void @atomic128_sub_acquire(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection427:
+; O3-NEXT:  .Lpcsection472:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection428:
+; O3-NEXT:  .Lpcsection473:
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB217_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
-; O3-NEXT:  .Lpcsection429:
+; O3-NEXT:  .Lpcsection474:
 ; O3-NEXT:    addq $-42, %rbx
 ; O3-NEXT:    movq %rdx, %rcx
-; O3-NEXT:  .Lpcsection430:
+; O3-NEXT:  .Lpcsection475:
 ; O3-NEXT:    adcq $-1, %rcx
-; O3-NEXT:  .Lpcsection431:
+; O3-NEXT:  .Lpcsection476:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection432:
+; O3-NEXT:  .Lpcsection477:
 ; O3-NEXT:    jne .LBB217_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -12277,21 +12412,21 @@ define void @atomic128_and_acquire(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection433:
+; O1-NEXT:  .Lpcsection478:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection434:
+; O1-NEXT:  .Lpcsection479:
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB218_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ebx
-; O1-NEXT:  .Lpcsection435:
+; O1-NEXT:  .Lpcsection480:
 ; O1-NEXT:    andl $42, %ebx
-; O1-NEXT:  .Lpcsection436:
+; O1-NEXT:  .Lpcsection481:
 ; O1-NEXT:    xorl %ecx, %ecx
-; O1-NEXT:  .Lpcsection437:
+; O1-NEXT:  .Lpcsection482:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection438:
+; O1-NEXT:  .Lpcsection483:
 ; O1-NEXT:    jne .LBB218_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -12305,21 +12440,21 @@ define void @atomic128_and_acquire(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection433:
+; O2-NEXT:  .Lpcsection478:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection434:
+; O2-NEXT:  .Lpcsection479:
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB218_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ebx
-; O2-NEXT:  .Lpcsection435:
+; O2-NEXT:  .Lpcsection480:
 ; O2-NEXT:    andl $42, %ebx
-; O2-NEXT:  .Lpcsection436:
+; O2-NEXT:  .Lpcsection481:
 ; O2-NEXT:    xorl %ecx, %ecx
-; O2-NEXT:  .Lpcsection437:
+; O2-NEXT:  .Lpcsection482:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection438:
+; O2-NEXT:  .Lpcsection483:
 ; O2-NEXT:    jne .LBB218_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -12333,21 +12468,21 @@ define void @atomic128_and_acquire(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection433:
+; O3-NEXT:  .Lpcsection478:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection434:
+; O3-NEXT:  .Lpcsection479:
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB218_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ebx
-; O3-NEXT:  .Lpcsection435:
+; O3-NEXT:  .Lpcsection480:
 ; O3-NEXT:    andl $42, %ebx
-; O3-NEXT:  .Lpcsection436:
+; O3-NEXT:  .Lpcsection481:
 ; O3-NEXT:    xorl %ecx, %ecx
-; O3-NEXT:  .Lpcsection437:
+; O3-NEXT:  .Lpcsection482:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection438:
+; O3-NEXT:  .Lpcsection483:
 ; O3-NEXT:    jne .LBB218_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -12405,20 +12540,20 @@ define void @atomic128_or_acquire(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection439:
+; O1-NEXT:  .Lpcsection484:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection440:
+; O1-NEXT:  .Lpcsection485:
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB219_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
-; O1-NEXT:  .Lpcsection441:
+; O1-NEXT:  .Lpcsection486:
 ; O1-NEXT:    orq $42, %rbx
 ; O1-NEXT:    movq %rdx, %rcx
-; O1-NEXT:  .Lpcsection442:
+; O1-NEXT:  .Lpcsection487:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection443:
+; O1-NEXT:  .Lpcsection488:
 ; O1-NEXT:    jne .LBB219_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -12432,20 +12567,20 @@ define void @atomic128_or_acquire(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection439:
+; O2-NEXT:  .Lpcsection484:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection440:
+; O2-NEXT:  .Lpcsection485:
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB219_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
-; O2-NEXT:  .Lpcsection441:
+; O2-NEXT:  .Lpcsection486:
 ; O2-NEXT:    orq $42, %rbx
 ; O2-NEXT:    movq %rdx, %rcx
-; O2-NEXT:  .Lpcsection442:
+; O2-NEXT:  .Lpcsection487:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection443:
+; O2-NEXT:  .Lpcsection488:
 ; O2-NEXT:    jne .LBB219_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -12459,20 +12594,20 @@ define void @atomic128_or_acquire(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection439:
+; O3-NEXT:  .Lpcsection484:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection440:
+; O3-NEXT:  .Lpcsection485:
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB219_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
-; O3-NEXT:  .Lpcsection441:
+; O3-NEXT:  .Lpcsection486:
 ; O3-NEXT:    orq $42, %rbx
 ; O3-NEXT:    movq %rdx, %rcx
-; O3-NEXT:  .Lpcsection442:
+; O3-NEXT:  .Lpcsection487:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection443:
+; O3-NEXT:  .Lpcsection488:
 ; O3-NEXT:    jne .LBB219_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -12530,20 +12665,20 @@ define void @atomic128_xor_acquire(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection444:
+; O1-NEXT:  .Lpcsection489:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection445:
+; O1-NEXT:  .Lpcsection490:
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB220_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
-; O1-NEXT:  .Lpcsection446:
+; O1-NEXT:  .Lpcsection491:
 ; O1-NEXT:    xorq $42, %rbx
 ; O1-NEXT:    movq %rdx, %rcx
-; O1-NEXT:  .Lpcsection447:
+; O1-NEXT:  .Lpcsection492:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection448:
+; O1-NEXT:  .Lpcsection493:
 ; O1-NEXT:    jne .LBB220_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -12557,20 +12692,20 @@ define void @atomic128_xor_acquire(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection444:
+; O2-NEXT:  .Lpcsection489:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection445:
+; O2-NEXT:  .Lpcsection490:
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB220_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
-; O2-NEXT:  .Lpcsection446:
+; O2-NEXT:  .Lpcsection491:
 ; O2-NEXT:    xorq $42, %rbx
 ; O2-NEXT:    movq %rdx, %rcx
-; O2-NEXT:  .Lpcsection447:
+; O2-NEXT:  .Lpcsection492:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection448:
+; O2-NEXT:  .Lpcsection493:
 ; O2-NEXT:    jne .LBB220_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -12584,20 +12719,20 @@ define void @atomic128_xor_acquire(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection444:
+; O3-NEXT:  .Lpcsection489:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection445:
+; O3-NEXT:  .Lpcsection490:
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB220_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
-; O3-NEXT:  .Lpcsection446:
+; O3-NEXT:  .Lpcsection491:
 ; O3-NEXT:    xorq $42, %rbx
 ; O3-NEXT:    movq %rdx, %rcx
-; O3-NEXT:  .Lpcsection447:
+; O3-NEXT:  .Lpcsection492:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection448:
+; O3-NEXT:  .Lpcsection493:
 ; O3-NEXT:    jne .LBB220_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -12661,23 +12796,23 @@ define void @atomic128_nand_acquire(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection449:
+; O1-NEXT:  .Lpcsection494:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection450:
+; O1-NEXT:  .Lpcsection495:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:  .Lpcsection451:
+; O1-NEXT:  .Lpcsection496:
 ; O1-NEXT:    movq $-1, %rcx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB221_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ebx
-; O1-NEXT:  .Lpcsection452:
+; O1-NEXT:  .Lpcsection497:
 ; O1-NEXT:    notl %ebx
-; O1-NEXT:  .Lpcsection453:
+; O1-NEXT:  .Lpcsection498:
 ; O1-NEXT:    orq $-43, %rbx
-; O1-NEXT:  .Lpcsection454:
+; O1-NEXT:  .Lpcsection499:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection455:
+; O1-NEXT:  .Lpcsection500:
 ; O1-NEXT:    jne .LBB221_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -12691,23 +12826,23 @@ define void @atomic128_nand_acquire(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection449:
+; O2-NEXT:  .Lpcsection494:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection450:
+; O2-NEXT:  .Lpcsection495:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:  .Lpcsection451:
+; O2-NEXT:  .Lpcsection496:
 ; O2-NEXT:    movq $-1, %rcx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB221_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ebx
-; O2-NEXT:  .Lpcsection452:
+; O2-NEXT:  .Lpcsection497:
 ; O2-NEXT:    notl %ebx
-; O2-NEXT:  .Lpcsection453:
+; O2-NEXT:  .Lpcsection498:
 ; O2-NEXT:    orq $-43, %rbx
-; O2-NEXT:  .Lpcsection454:
+; O2-NEXT:  .Lpcsection499:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection455:
+; O2-NEXT:  .Lpcsection500:
 ; O2-NEXT:    jne .LBB221_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -12721,23 +12856,23 @@ define void @atomic128_nand_acquire(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection449:
+; O3-NEXT:  .Lpcsection494:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection450:
+; O3-NEXT:  .Lpcsection495:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:  .Lpcsection451:
+; O3-NEXT:  .Lpcsection496:
 ; O3-NEXT:    movq $-1, %rcx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB221_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ebx
-; O3-NEXT:  .Lpcsection452:
+; O3-NEXT:  .Lpcsection497:
 ; O3-NEXT:    notl %ebx
-; O3-NEXT:  .Lpcsection453:
+; O3-NEXT:  .Lpcsection498:
 ; O3-NEXT:    orq $-43, %rbx
-; O3-NEXT:  .Lpcsection454:
+; O3-NEXT:  .Lpcsection499:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection455:
+; O3-NEXT:  .Lpcsection500:
 ; O3-NEXT:    jne .LBB221_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -12797,20 +12932,20 @@ define void @atomic128_xchg_release(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection456:
+; O1-NEXT:  .Lpcsection501:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection457:
+; O1-NEXT:  .Lpcsection502:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:  .Lpcsection458:
+; O1-NEXT:  .Lpcsection503:
 ; O1-NEXT:    movl $42, %ebx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB222_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
-; O1-NEXT:  .Lpcsection459:
+; O1-NEXT:  .Lpcsection504:
 ; O1-NEXT:    xorl %ecx, %ecx
-; O1-NEXT:  .Lpcsection460:
+; O1-NEXT:  .Lpcsection505:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection461:
+; O1-NEXT:  .Lpcsection506:
 ; O1-NEXT:    jne .LBB222_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -12824,20 +12959,20 @@ define void @atomic128_xchg_release(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection456:
+; O2-NEXT:  .Lpcsection501:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection457:
+; O2-NEXT:  .Lpcsection502:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:  .Lpcsection458:
+; O2-NEXT:  .Lpcsection503:
 ; O2-NEXT:    movl $42, %ebx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB222_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
-; O2-NEXT:  .Lpcsection459:
+; O2-NEXT:  .Lpcsection504:
 ; O2-NEXT:    xorl %ecx, %ecx
-; O2-NEXT:  .Lpcsection460:
+; O2-NEXT:  .Lpcsection505:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection461:
+; O2-NEXT:  .Lpcsection506:
 ; O2-NEXT:    jne .LBB222_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -12851,20 +12986,20 @@ define void @atomic128_xchg_release(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection456:
+; O3-NEXT:  .Lpcsection501:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection457:
+; O3-NEXT:  .Lpcsection502:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:  .Lpcsection458:
+; O3-NEXT:  .Lpcsection503:
 ; O3-NEXT:    movl $42, %ebx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB222_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
-; O3-NEXT:  .Lpcsection459:
+; O3-NEXT:  .Lpcsection504:
 ; O3-NEXT:    xorl %ecx, %ecx
-; O3-NEXT:  .Lpcsection460:
+; O3-NEXT:  .Lpcsection505:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection461:
+; O3-NEXT:  .Lpcsection506:
 ; O3-NEXT:    jne .LBB222_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -12923,22 +13058,22 @@ define void @atomic128_add_release(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection462:
+; O1-NEXT:  .Lpcsection507:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection463:
+; O1-NEXT:  .Lpcsection508:
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB223_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
-; O1-NEXT:  .Lpcsection464:
+; O1-NEXT:  .Lpcsection509:
 ; O1-NEXT:    addq $42, %rbx
 ; O1-NEXT:    movq %rdx, %rcx
-; O1-NEXT:  .Lpcsection465:
+; O1-NEXT:  .Lpcsection510:
 ; O1-NEXT:    adcq $0, %rcx
-; O1-NEXT:  .Lpcsection466:
+; O1-NEXT:  .Lpcsection511:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection467:
+; O1-NEXT:  .Lpcsection512:
 ; O1-NEXT:    jne .LBB223_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -12952,22 +13087,22 @@ define void @atomic128_add_release(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection462:
+; O2-NEXT:  .Lpcsection507:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection463:
+; O2-NEXT:  .Lpcsection508:
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB223_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
-; O2-NEXT:  .Lpcsection464:
+; O2-NEXT:  .Lpcsection509:
 ; O2-NEXT:    addq $42, %rbx
 ; O2-NEXT:    movq %rdx, %rcx
-; O2-NEXT:  .Lpcsection465:
+; O2-NEXT:  .Lpcsection510:
 ; O2-NEXT:    adcq $0, %rcx
-; O2-NEXT:  .Lpcsection466:
+; O2-NEXT:  .Lpcsection511:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection467:
+; O2-NEXT:  .Lpcsection512:
 ; O2-NEXT:    jne .LBB223_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -12981,22 +13116,22 @@ define void @atomic128_add_release(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection462:
+; O3-NEXT:  .Lpcsection507:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection463:
+; O3-NEXT:  .Lpcsection508:
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB223_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
-; O3-NEXT:  .Lpcsection464:
+; O3-NEXT:  .Lpcsection509:
 ; O3-NEXT:    addq $42, %rbx
 ; O3-NEXT:    movq %rdx, %rcx
-; O3-NEXT:  .Lpcsection465:
+; O3-NEXT:  .Lpcsection510:
 ; O3-NEXT:    adcq $0, %rcx
-; O3-NEXT:  .Lpcsection466:
+; O3-NEXT:  .Lpcsection511:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection467:
+; O3-NEXT:  .Lpcsection512:
 ; O3-NEXT:    jne .LBB223_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -13056,22 +13191,22 @@ define void @atomic128_sub_release(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection468:
+; O1-NEXT:  .Lpcsection513:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection469:
+; O1-NEXT:  .Lpcsection514:
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB224_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
-; O1-NEXT:  .Lpcsection470:
+; O1-NEXT:  .Lpcsection515:
 ; O1-NEXT:    addq $-42, %rbx
 ; O1-NEXT:    movq %rdx, %rcx
-; O1-NEXT:  .Lpcsection471:
+; O1-NEXT:  .Lpcsection516:
 ; O1-NEXT:    adcq $-1, %rcx
-; O1-NEXT:  .Lpcsection472:
+; O1-NEXT:  .Lpcsection517:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection473:
+; O1-NEXT:  .Lpcsection518:
 ; O1-NEXT:    jne .LBB224_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -13085,22 +13220,22 @@ define void @atomic128_sub_release(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection468:
+; O2-NEXT:  .Lpcsection513:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection469:
+; O2-NEXT:  .Lpcsection514:
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB224_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
-; O2-NEXT:  .Lpcsection470:
+; O2-NEXT:  .Lpcsection515:
 ; O2-NEXT:    addq $-42, %rbx
 ; O2-NEXT:    movq %rdx, %rcx
-; O2-NEXT:  .Lpcsection471:
+; O2-NEXT:  .Lpcsection516:
 ; O2-NEXT:    adcq $-1, %rcx
-; O2-NEXT:  .Lpcsection472:
+; O2-NEXT:  .Lpcsection517:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection473:
+; O2-NEXT:  .Lpcsection518:
 ; O2-NEXT:    jne .LBB224_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -13114,22 +13249,22 @@ define void @atomic128_sub_release(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection468:
+; O3-NEXT:  .Lpcsection513:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection469:
+; O3-NEXT:  .Lpcsection514:
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB224_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
-; O3-NEXT:  .Lpcsection470:
+; O3-NEXT:  .Lpcsection515:
 ; O3-NEXT:    addq $-42, %rbx
 ; O3-NEXT:    movq %rdx, %rcx
-; O3-NEXT:  .Lpcsection471:
+; O3-NEXT:  .Lpcsection516:
 ; O3-NEXT:    adcq $-1, %rcx
-; O3-NEXT:  .Lpcsection472:
+; O3-NEXT:  .Lpcsection517:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection473:
+; O3-NEXT:  .Lpcsection518:
 ; O3-NEXT:    jne .LBB224_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -13191,21 +13326,21 @@ define void @atomic128_and_release(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection474:
+; O1-NEXT:  .Lpcsection519:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection475:
+; O1-NEXT:  .Lpcsection520:
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB225_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ebx
-; O1-NEXT:  .Lpcsection476:
+; O1-NEXT:  .Lpcsection521:
 ; O1-NEXT:    andl $42, %ebx
-; O1-NEXT:  .Lpcsection477:
+; O1-NEXT:  .Lpcsection522:
 ; O1-NEXT:    xorl %ecx, %ecx
-; O1-NEXT:  .Lpcsection478:
+; O1-NEXT:  .Lpcsection523:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection479:
+; O1-NEXT:  .Lpcsection524:
 ; O1-NEXT:    jne .LBB225_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -13219,21 +13354,21 @@ define void @atomic128_and_release(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection474:
+; O2-NEXT:  .Lpcsection519:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection475:
+; O2-NEXT:  .Lpcsection520:
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB225_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ebx
-; O2-NEXT:  .Lpcsection476:
+; O2-NEXT:  .Lpcsection521:
 ; O2-NEXT:    andl $42, %ebx
-; O2-NEXT:  .Lpcsection477:
+; O2-NEXT:  .Lpcsection522:
 ; O2-NEXT:    xorl %ecx, %ecx
-; O2-NEXT:  .Lpcsection478:
+; O2-NEXT:  .Lpcsection523:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection479:
+; O2-NEXT:  .Lpcsection524:
 ; O2-NEXT:    jne .LBB225_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -13247,21 +13382,21 @@ define void @atomic128_and_release(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection474:
+; O3-NEXT:  .Lpcsection519:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection475:
+; O3-NEXT:  .Lpcsection520:
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB225_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ebx
-; O3-NEXT:  .Lpcsection476:
+; O3-NEXT:  .Lpcsection521:
 ; O3-NEXT:    andl $42, %ebx
-; O3-NEXT:  .Lpcsection477:
+; O3-NEXT:  .Lpcsection522:
 ; O3-NEXT:    xorl %ecx, %ecx
-; O3-NEXT:  .Lpcsection478:
+; O3-NEXT:  .Lpcsection523:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection479:
+; O3-NEXT:  .Lpcsection524:
 ; O3-NEXT:    jne .LBB225_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -13319,20 +13454,20 @@ define void @atomic128_or_release(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection480:
+; O1-NEXT:  .Lpcsection525:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection481:
+; O1-NEXT:  .Lpcsection526:
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB226_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
-; O1-NEXT:  .Lpcsection482:
+; O1-NEXT:  .Lpcsection527:
 ; O1-NEXT:    orq $42, %rbx
 ; O1-NEXT:    movq %rdx, %rcx
-; O1-NEXT:  .Lpcsection483:
+; O1-NEXT:  .Lpcsection528:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection484:
+; O1-NEXT:  .Lpcsection529:
 ; O1-NEXT:    jne .LBB226_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -13346,20 +13481,20 @@ define void @atomic128_or_release(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection480:
+; O2-NEXT:  .Lpcsection525:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection481:
+; O2-NEXT:  .Lpcsection526:
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB226_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
-; O2-NEXT:  .Lpcsection482:
+; O2-NEXT:  .Lpcsection527:
 ; O2-NEXT:    orq $42, %rbx
 ; O2-NEXT:    movq %rdx, %rcx
-; O2-NEXT:  .Lpcsection483:
+; O2-NEXT:  .Lpcsection528:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection484:
+; O2-NEXT:  .Lpcsection529:
 ; O2-NEXT:    jne .LBB226_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -13373,20 +13508,20 @@ define void @atomic128_or_release(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection480:
+; O3-NEXT:  .Lpcsection525:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection481:
+; O3-NEXT:  .Lpcsection526:
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB226_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
-; O3-NEXT:  .Lpcsection482:
+; O3-NEXT:  .Lpcsection527:
 ; O3-NEXT:    orq $42, %rbx
 ; O3-NEXT:    movq %rdx, %rcx
-; O3-NEXT:  .Lpcsection483:
+; O3-NEXT:  .Lpcsection528:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection484:
+; O3-NEXT:  .Lpcsection529:
 ; O3-NEXT:    jne .LBB226_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -13444,20 +13579,20 @@ define void @atomic128_xor_release(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection485:
+; O1-NEXT:  .Lpcsection530:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection486:
+; O1-NEXT:  .Lpcsection531:
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB227_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
-; O1-NEXT:  .Lpcsection487:
+; O1-NEXT:  .Lpcsection532:
 ; O1-NEXT:    xorq $42, %rbx
 ; O1-NEXT:    movq %rdx, %rcx
-; O1-NEXT:  .Lpcsection488:
+; O1-NEXT:  .Lpcsection533:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection489:
+; O1-NEXT:  .Lpcsection534:
 ; O1-NEXT:    jne .LBB227_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -13471,20 +13606,20 @@ define void @atomic128_xor_release(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection485:
+; O2-NEXT:  .Lpcsection530:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection486:
+; O2-NEXT:  .Lpcsection531:
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB227_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
-; O2-NEXT:  .Lpcsection487:
+; O2-NEXT:  .Lpcsection532:
 ; O2-NEXT:    xorq $42, %rbx
 ; O2-NEXT:    movq %rdx, %rcx
-; O2-NEXT:  .Lpcsection488:
+; O2-NEXT:  .Lpcsection533:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection489:
+; O2-NEXT:  .Lpcsection534:
 ; O2-NEXT:    jne .LBB227_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -13498,20 +13633,20 @@ define void @atomic128_xor_release(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection485:
+; O3-NEXT:  .Lpcsection530:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection486:
+; O3-NEXT:  .Lpcsection531:
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB227_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
-; O3-NEXT:  .Lpcsection487:
+; O3-NEXT:  .Lpcsection532:
 ; O3-NEXT:    xorq $42, %rbx
 ; O3-NEXT:    movq %rdx, %rcx
-; O3-NEXT:  .Lpcsection488:
+; O3-NEXT:  .Lpcsection533:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection489:
+; O3-NEXT:  .Lpcsection534:
 ; O3-NEXT:    jne .LBB227_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -13575,23 +13710,23 @@ define void @atomic128_nand_release(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection490:
+; O1-NEXT:  .Lpcsection535:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection491:
+; O1-NEXT:  .Lpcsection536:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:  .Lpcsection492:
+; O1-NEXT:  .Lpcsection537:
 ; O1-NEXT:    movq $-1, %rcx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB228_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ebx
-; O1-NEXT:  .Lpcsection493:
+; O1-NEXT:  .Lpcsection538:
 ; O1-NEXT:    notl %ebx
-; O1-NEXT:  .Lpcsection494:
+; O1-NEXT:  .Lpcsection539:
 ; O1-NEXT:    orq $-43, %rbx
-; O1-NEXT:  .Lpcsection495:
+; O1-NEXT:  .Lpcsection540:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection496:
+; O1-NEXT:  .Lpcsection541:
 ; O1-NEXT:    jne .LBB228_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -13605,23 +13740,23 @@ define void @atomic128_nand_release(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection490:
+; O2-NEXT:  .Lpcsection535:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection491:
+; O2-NEXT:  .Lpcsection536:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:  .Lpcsection492:
+; O2-NEXT:  .Lpcsection537:
 ; O2-NEXT:    movq $-1, %rcx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB228_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ebx
-; O2-NEXT:  .Lpcsection493:
+; O2-NEXT:  .Lpcsection538:
 ; O2-NEXT:    notl %ebx
-; O2-NEXT:  .Lpcsection494:
+; O2-NEXT:  .Lpcsection539:
 ; O2-NEXT:    orq $-43, %rbx
-; O2-NEXT:  .Lpcsection495:
+; O2-NEXT:  .Lpcsection540:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection496:
+; O2-NEXT:  .Lpcsection541:
 ; O2-NEXT:    jne .LBB228_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -13635,23 +13770,23 @@ define void @atomic128_nand_release(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection490:
+; O3-NEXT:  .Lpcsection535:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection491:
+; O3-NEXT:  .Lpcsection536:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:  .Lpcsection492:
+; O3-NEXT:  .Lpcsection537:
 ; O3-NEXT:    movq $-1, %rcx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB228_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ebx
-; O3-NEXT:  .Lpcsection493:
+; O3-NEXT:  .Lpcsection538:
 ; O3-NEXT:    notl %ebx
-; O3-NEXT:  .Lpcsection494:
+; O3-NEXT:  .Lpcsection539:
 ; O3-NEXT:    orq $-43, %rbx
-; O3-NEXT:  .Lpcsection495:
+; O3-NEXT:  .Lpcsection540:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection496:
+; O3-NEXT:  .Lpcsection541:
 ; O3-NEXT:    jne .LBB228_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -13711,20 +13846,20 @@ define void @atomic128_xchg_acq_rel(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection497:
+; O1-NEXT:  .Lpcsection542:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection498:
+; O1-NEXT:  .Lpcsection543:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:  .Lpcsection499:
+; O1-NEXT:  .Lpcsection544:
 ; O1-NEXT:    movl $42, %ebx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB229_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
-; O1-NEXT:  .Lpcsection500:
+; O1-NEXT:  .Lpcsection545:
 ; O1-NEXT:    xorl %ecx, %ecx
-; O1-NEXT:  .Lpcsection501:
+; O1-NEXT:  .Lpcsection546:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection502:
+; O1-NEXT:  .Lpcsection547:
 ; O1-NEXT:    jne .LBB229_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -13738,20 +13873,20 @@ define void @atomic128_xchg_acq_rel(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection497:
+; O2-NEXT:  .Lpcsection542:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection498:
+; O2-NEXT:  .Lpcsection543:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:  .Lpcsection499:
+; O2-NEXT:  .Lpcsection544:
 ; O2-NEXT:    movl $42, %ebx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB229_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
-; O2-NEXT:  .Lpcsection500:
+; O2-NEXT:  .Lpcsection545:
 ; O2-NEXT:    xorl %ecx, %ecx
-; O2-NEXT:  .Lpcsection501:
+; O2-NEXT:  .Lpcsection546:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection502:
+; O2-NEXT:  .Lpcsection547:
 ; O2-NEXT:    jne .LBB229_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -13765,20 +13900,20 @@ define void @atomic128_xchg_acq_rel(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection497:
+; O3-NEXT:  .Lpcsection542:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection498:
+; O3-NEXT:  .Lpcsection543:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:  .Lpcsection499:
+; O3-NEXT:  .Lpcsection544:
 ; O3-NEXT:    movl $42, %ebx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB229_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
-; O3-NEXT:  .Lpcsection500:
+; O3-NEXT:  .Lpcsection545:
 ; O3-NEXT:    xorl %ecx, %ecx
-; O3-NEXT:  .Lpcsection501:
+; O3-NEXT:  .Lpcsection546:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection502:
+; O3-NEXT:  .Lpcsection547:
 ; O3-NEXT:    jne .LBB229_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -13838,22 +13973,22 @@ define void @atomic128_add_acq_rel(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection503:
+; O1-NEXT:  .Lpcsection548:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection504:
+; O1-NEXT:  .Lpcsection549:
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB230_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
-; O1-NEXT:  .Lpcsection505:
+; O1-NEXT:  .Lpcsection550:
 ; O1-NEXT:    addq $42, %rbx
 ; O1-NEXT:    movq %rdx, %rcx
-; O1-NEXT:  .Lpcsection506:
+; O1-NEXT:  .Lpcsection551:
 ; O1-NEXT:    adcq $0, %rcx
-; O1-NEXT:  .Lpcsection507:
+; O1-NEXT:  .Lpcsection552:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection508:
+; O1-NEXT:  .Lpcsection553:
 ; O1-NEXT:    jne .LBB230_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -13867,22 +14002,22 @@ define void @atomic128_add_acq_rel(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection503:
+; O2-NEXT:  .Lpcsection548:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection504:
+; O2-NEXT:  .Lpcsection549:
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB230_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
-; O2-NEXT:  .Lpcsection505:
+; O2-NEXT:  .Lpcsection550:
 ; O2-NEXT:    addq $42, %rbx
 ; O2-NEXT:    movq %rdx, %rcx
-; O2-NEXT:  .Lpcsection506:
+; O2-NEXT:  .Lpcsection551:
 ; O2-NEXT:    adcq $0, %rcx
-; O2-NEXT:  .Lpcsection507:
+; O2-NEXT:  .Lpcsection552:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection508:
+; O2-NEXT:  .Lpcsection553:
 ; O2-NEXT:    jne .LBB230_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -13896,22 +14031,22 @@ define void @atomic128_add_acq_rel(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection503:
+; O3-NEXT:  .Lpcsection548:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection504:
+; O3-NEXT:  .Lpcsection549:
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB230_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
-; O3-NEXT:  .Lpcsection505:
+; O3-NEXT:  .Lpcsection550:
 ; O3-NEXT:    addq $42, %rbx
 ; O3-NEXT:    movq %rdx, %rcx
-; O3-NEXT:  .Lpcsection506:
+; O3-NEXT:  .Lpcsection551:
 ; O3-NEXT:    adcq $0, %rcx
-; O3-NEXT:  .Lpcsection507:
+; O3-NEXT:  .Lpcsection552:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection508:
+; O3-NEXT:  .Lpcsection553:
 ; O3-NEXT:    jne .LBB230_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -13971,22 +14106,22 @@ define void @atomic128_sub_acq_rel(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection509:
+; O1-NEXT:  .Lpcsection554:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection510:
+; O1-NEXT:  .Lpcsection555:
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB231_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
-; O1-NEXT:  .Lpcsection511:
+; O1-NEXT:  .Lpcsection556:
 ; O1-NEXT:    addq $-42, %rbx
 ; O1-NEXT:    movq %rdx, %rcx
-; O1-NEXT:  .Lpcsection512:
+; O1-NEXT:  .Lpcsection557:
 ; O1-NEXT:    adcq $-1, %rcx
-; O1-NEXT:  .Lpcsection513:
+; O1-NEXT:  .Lpcsection558:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection514:
+; O1-NEXT:  .Lpcsection559:
 ; O1-NEXT:    jne .LBB231_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -14000,22 +14135,22 @@ define void @atomic128_sub_acq_rel(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection509:
+; O2-NEXT:  .Lpcsection554:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection510:
+; O2-NEXT:  .Lpcsection555:
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB231_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
-; O2-NEXT:  .Lpcsection511:
+; O2-NEXT:  .Lpcsection556:
 ; O2-NEXT:    addq $-42, %rbx
 ; O2-NEXT:    movq %rdx, %rcx
-; O2-NEXT:  .Lpcsection512:
+; O2-NEXT:  .Lpcsection557:
 ; O2-NEXT:    adcq $-1, %rcx
-; O2-NEXT:  .Lpcsection513:
+; O2-NEXT:  .Lpcsection558:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection514:
+; O2-NEXT:  .Lpcsection559:
 ; O2-NEXT:    jne .LBB231_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -14029,22 +14164,22 @@ define void @atomic128_sub_acq_rel(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection509:
+; O3-NEXT:  .Lpcsection554:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection510:
+; O3-NEXT:  .Lpcsection555:
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB231_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
-; O3-NEXT:  .Lpcsection511:
+; O3-NEXT:  .Lpcsection556:
 ; O3-NEXT:    addq $-42, %rbx
 ; O3-NEXT:    movq %rdx, %rcx
-; O3-NEXT:  .Lpcsection512:
+; O3-NEXT:  .Lpcsection557:
 ; O3-NEXT:    adcq $-1, %rcx
-; O3-NEXT:  .Lpcsection513:
+; O3-NEXT:  .Lpcsection558:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection514:
+; O3-NEXT:  .Lpcsection559:
 ; O3-NEXT:    jne .LBB231_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -14106,21 +14241,21 @@ define void @atomic128_and_acq_rel(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection515:
+; O1-NEXT:  .Lpcsection560:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection516:
+; O1-NEXT:  .Lpcsection561:
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB232_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ebx
-; O1-NEXT:  .Lpcsection517:
+; O1-NEXT:  .Lpcsection562:
 ; O1-NEXT:    andl $42, %ebx
-; O1-NEXT:  .Lpcsection518:
+; O1-NEXT:  .Lpcsection563:
 ; O1-NEXT:    xorl %ecx, %ecx
-; O1-NEXT:  .Lpcsection519:
+; O1-NEXT:  .Lpcsection564:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection520:
+; O1-NEXT:  .Lpcsection565:
 ; O1-NEXT:    jne .LBB232_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -14134,21 +14269,21 @@ define void @atomic128_and_acq_rel(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection515:
+; O2-NEXT:  .Lpcsection560:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection516:
+; O2-NEXT:  .Lpcsection561:
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB232_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ebx
-; O2-NEXT:  .Lpcsection517:
+; O2-NEXT:  .Lpcsection562:
 ; O2-NEXT:    andl $42, %ebx
-; O2-NEXT:  .Lpcsection518:
+; O2-NEXT:  .Lpcsection563:
 ; O2-NEXT:    xorl %ecx, %ecx
-; O2-NEXT:  .Lpcsection519:
+; O2-NEXT:  .Lpcsection564:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection520:
+; O2-NEXT:  .Lpcsection565:
 ; O2-NEXT:    jne .LBB232_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -14162,21 +14297,21 @@ define void @atomic128_and_acq_rel(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection515:
+; O3-NEXT:  .Lpcsection560:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection516:
+; O3-NEXT:  .Lpcsection561:
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB232_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ebx
-; O3-NEXT:  .Lpcsection517:
+; O3-NEXT:  .Lpcsection562:
 ; O3-NEXT:    andl $42, %ebx
-; O3-NEXT:  .Lpcsection518:
+; O3-NEXT:  .Lpcsection563:
 ; O3-NEXT:    xorl %ecx, %ecx
-; O3-NEXT:  .Lpcsection519:
+; O3-NEXT:  .Lpcsection564:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection520:
+; O3-NEXT:  .Lpcsection565:
 ; O3-NEXT:    jne .LBB232_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -14234,20 +14369,20 @@ define void @atomic128_or_acq_rel(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection521:
+; O1-NEXT:  .Lpcsection566:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection522:
+; O1-NEXT:  .Lpcsection567:
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB233_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
-; O1-NEXT:  .Lpcsection523:
+; O1-NEXT:  .Lpcsection568:
 ; O1-NEXT:    orq $42, %rbx
 ; O1-NEXT:    movq %rdx, %rcx
-; O1-NEXT:  .Lpcsection524:
+; O1-NEXT:  .Lpcsection569:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection525:
+; O1-NEXT:  .Lpcsection570:
 ; O1-NEXT:    jne .LBB233_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -14261,20 +14396,20 @@ define void @atomic128_or_acq_rel(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection521:
+; O2-NEXT:  .Lpcsection566:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection522:
+; O2-NEXT:  .Lpcsection567:
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB233_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
-; O2-NEXT:  .Lpcsection523:
+; O2-NEXT:  .Lpcsection568:
 ; O2-NEXT:    orq $42, %rbx
 ; O2-NEXT:    movq %rdx, %rcx
-; O2-NEXT:  .Lpcsection524:
+; O2-NEXT:  .Lpcsection569:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection525:
+; O2-NEXT:  .Lpcsection570:
 ; O2-NEXT:    jne .LBB233_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -14288,20 +14423,20 @@ define void @atomic128_or_acq_rel(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection521:
+; O3-NEXT:  .Lpcsection566:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection522:
+; O3-NEXT:  .Lpcsection567:
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB233_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
-; O3-NEXT:  .Lpcsection523:
+; O3-NEXT:  .Lpcsection568:
 ; O3-NEXT:    orq $42, %rbx
 ; O3-NEXT:    movq %rdx, %rcx
-; O3-NEXT:  .Lpcsection524:
+; O3-NEXT:  .Lpcsection569:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection525:
+; O3-NEXT:  .Lpcsection570:
 ; O3-NEXT:    jne .LBB233_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -14359,20 +14494,20 @@ define void @atomic128_xor_acq_rel(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection526:
+; O1-NEXT:  .Lpcsection571:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection527:
+; O1-NEXT:  .Lpcsection572:
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB234_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
-; O1-NEXT:  .Lpcsection528:
+; O1-NEXT:  .Lpcsection573:
 ; O1-NEXT:    xorq $42, %rbx
 ; O1-NEXT:    movq %rdx, %rcx
-; O1-NEXT:  .Lpcsection529:
+; O1-NEXT:  .Lpcsection574:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection530:
+; O1-NEXT:  .Lpcsection575:
 ; O1-NEXT:    jne .LBB234_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -14386,20 +14521,20 @@ define void @atomic128_xor_acq_rel(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection526:
+; O2-NEXT:  .Lpcsection571:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection527:
+; O2-NEXT:  .Lpcsection572:
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB234_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
-; O2-NEXT:  .Lpcsection528:
+; O2-NEXT:  .Lpcsection573:
 ; O2-NEXT:    xorq $42, %rbx
 ; O2-NEXT:    movq %rdx, %rcx
-; O2-NEXT:  .Lpcsection529:
+; O2-NEXT:  .Lpcsection574:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection530:
+; O2-NEXT:  .Lpcsection575:
 ; O2-NEXT:    jne .LBB234_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -14413,20 +14548,20 @@ define void @atomic128_xor_acq_rel(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection526:
+; O3-NEXT:  .Lpcsection571:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection527:
+; O3-NEXT:  .Lpcsection572:
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB234_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
-; O3-NEXT:  .Lpcsection528:
+; O3-NEXT:  .Lpcsection573:
 ; O3-NEXT:    xorq $42, %rbx
 ; O3-NEXT:    movq %rdx, %rcx
-; O3-NEXT:  .Lpcsection529:
+; O3-NEXT:  .Lpcsection574:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection530:
+; O3-NEXT:  .Lpcsection575:
 ; O3-NEXT:    jne .LBB234_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -14490,23 +14625,23 @@ define void @atomic128_nand_acq_rel(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection531:
+; O1-NEXT:  .Lpcsection576:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection532:
+; O1-NEXT:  .Lpcsection577:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:  .Lpcsection533:
+; O1-NEXT:  .Lpcsection578:
 ; O1-NEXT:    movq $-1, %rcx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB235_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ebx
-; O1-NEXT:  .Lpcsection534:
+; O1-NEXT:  .Lpcsection579:
 ; O1-NEXT:    notl %ebx
-; O1-NEXT:  .Lpcsection535:
+; O1-NEXT:  .Lpcsection580:
 ; O1-NEXT:    orq $-43, %rbx
-; O1-NEXT:  .Lpcsection536:
+; O1-NEXT:  .Lpcsection581:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection537:
+; O1-NEXT:  .Lpcsection582:
 ; O1-NEXT:    jne .LBB235_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -14520,23 +14655,23 @@ define void @atomic128_nand_acq_rel(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection531:
+; O2-NEXT:  .Lpcsection576:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection532:
+; O2-NEXT:  .Lpcsection577:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:  .Lpcsection533:
+; O2-NEXT:  .Lpcsection578:
 ; O2-NEXT:    movq $-1, %rcx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB235_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ebx
-; O2-NEXT:  .Lpcsection534:
+; O2-NEXT:  .Lpcsection579:
 ; O2-NEXT:    notl %ebx
-; O2-NEXT:  .Lpcsection535:
+; O2-NEXT:  .Lpcsection580:
 ; O2-NEXT:    orq $-43, %rbx
-; O2-NEXT:  .Lpcsection536:
+; O2-NEXT:  .Lpcsection581:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection537:
+; O2-NEXT:  .Lpcsection582:
 ; O2-NEXT:    jne .LBB235_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -14550,23 +14685,23 @@ define void @atomic128_nand_acq_rel(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection531:
+; O3-NEXT:  .Lpcsection576:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection532:
+; O3-NEXT:  .Lpcsection577:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:  .Lpcsection533:
+; O3-NEXT:  .Lpcsection578:
 ; O3-NEXT:    movq $-1, %rcx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB235_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ebx
-; O3-NEXT:  .Lpcsection534:
+; O3-NEXT:  .Lpcsection579:
 ; O3-NEXT:    notl %ebx
-; O3-NEXT:  .Lpcsection535:
+; O3-NEXT:  .Lpcsection580:
 ; O3-NEXT:    orq $-43, %rbx
-; O3-NEXT:  .Lpcsection536:
+; O3-NEXT:  .Lpcsection581:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection537:
+; O3-NEXT:  .Lpcsection582:
 ; O3-NEXT:    jne .LBB235_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -14626,20 +14761,20 @@ define void @atomic128_xchg_seq_cst(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection538:
+; O1-NEXT:  .Lpcsection583:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection539:
+; O1-NEXT:  .Lpcsection584:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:  .Lpcsection540:
+; O1-NEXT:  .Lpcsection585:
 ; O1-NEXT:    movl $42, %ebx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB236_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
-; O1-NEXT:  .Lpcsection541:
+; O1-NEXT:  .Lpcsection586:
 ; O1-NEXT:    xorl %ecx, %ecx
-; O1-NEXT:  .Lpcsection542:
+; O1-NEXT:  .Lpcsection587:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection543:
+; O1-NEXT:  .Lpcsection588:
 ; O1-NEXT:    jne .LBB236_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -14653,20 +14788,20 @@ define void @atomic128_xchg_seq_cst(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection538:
+; O2-NEXT:  .Lpcsection583:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection539:
+; O2-NEXT:  .Lpcsection584:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:  .Lpcsection540:
+; O2-NEXT:  .Lpcsection585:
 ; O2-NEXT:    movl $42, %ebx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB236_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
-; O2-NEXT:  .Lpcsection541:
+; O2-NEXT:  .Lpcsection586:
 ; O2-NEXT:    xorl %ecx, %ecx
-; O2-NEXT:  .Lpcsection542:
+; O2-NEXT:  .Lpcsection587:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection543:
+; O2-NEXT:  .Lpcsection588:
 ; O2-NEXT:    jne .LBB236_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -14680,20 +14815,20 @@ define void @atomic128_xchg_seq_cst(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection538:
+; O3-NEXT:  .Lpcsection583:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection539:
+; O3-NEXT:  .Lpcsection584:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:  .Lpcsection540:
+; O3-NEXT:  .Lpcsection585:
 ; O3-NEXT:    movl $42, %ebx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB236_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
-; O3-NEXT:  .Lpcsection541:
+; O3-NEXT:  .Lpcsection586:
 ; O3-NEXT:    xorl %ecx, %ecx
-; O3-NEXT:  .Lpcsection542:
+; O3-NEXT:  .Lpcsection587:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection543:
+; O3-NEXT:  .Lpcsection588:
 ; O3-NEXT:    jne .LBB236_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -14753,22 +14888,22 @@ define void @atomic128_add_seq_cst(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection544:
+; O1-NEXT:  .Lpcsection589:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection545:
+; O1-NEXT:  .Lpcsection590:
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB237_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
-; O1-NEXT:  .Lpcsection546:
+; O1-NEXT:  .Lpcsection591:
 ; O1-NEXT:    addq $42, %rbx
 ; O1-NEXT:    movq %rdx, %rcx
-; O1-NEXT:  .Lpcsection547:
+; O1-NEXT:  .Lpcsection592:
 ; O1-NEXT:    adcq $0, %rcx
-; O1-NEXT:  .Lpcsection548:
+; O1-NEXT:  .Lpcsection593:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection549:
+; O1-NEXT:  .Lpcsection594:
 ; O1-NEXT:    jne .LBB237_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -14782,22 +14917,22 @@ define void @atomic128_add_seq_cst(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection544:
+; O2-NEXT:  .Lpcsection589:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection545:
+; O2-NEXT:  .Lpcsection590:
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB237_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
-; O2-NEXT:  .Lpcsection546:
+; O2-NEXT:  .Lpcsection591:
 ; O2-NEXT:    addq $42, %rbx
 ; O2-NEXT:    movq %rdx, %rcx
-; O2-NEXT:  .Lpcsection547:
+; O2-NEXT:  .Lpcsection592:
 ; O2-NEXT:    adcq $0, %rcx
-; O2-NEXT:  .Lpcsection548:
+; O2-NEXT:  .Lpcsection593:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection549:
+; O2-NEXT:  .Lpcsection594:
 ; O2-NEXT:    jne .LBB237_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -14811,22 +14946,22 @@ define void @atomic128_add_seq_cst(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection544:
+; O3-NEXT:  .Lpcsection589:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection545:
+; O3-NEXT:  .Lpcsection590:
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB237_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
-; O3-NEXT:  .Lpcsection546:
+; O3-NEXT:  .Lpcsection591:
 ; O3-NEXT:    addq $42, %rbx
 ; O3-NEXT:    movq %rdx, %rcx
-; O3-NEXT:  .Lpcsection547:
+; O3-NEXT:  .Lpcsection592:
 ; O3-NEXT:    adcq $0, %rcx
-; O3-NEXT:  .Lpcsection548:
+; O3-NEXT:  .Lpcsection593:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection549:
+; O3-NEXT:  .Lpcsection594:
 ; O3-NEXT:    jne .LBB237_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -14886,22 +15021,22 @@ define void @atomic128_sub_seq_cst(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection550:
+; O1-NEXT:  .Lpcsection595:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection551:
+; O1-NEXT:  .Lpcsection596:
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB238_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
-; O1-NEXT:  .Lpcsection552:
+; O1-NEXT:  .Lpcsection597:
 ; O1-NEXT:    addq $-42, %rbx
 ; O1-NEXT:    movq %rdx, %rcx
-; O1-NEXT:  .Lpcsection553:
+; O1-NEXT:  .Lpcsection598:
 ; O1-NEXT:    adcq $-1, %rcx
-; O1-NEXT:  .Lpcsection554:
+; O1-NEXT:  .Lpcsection599:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection555:
+; O1-NEXT:  .Lpcsection600:
 ; O1-NEXT:    jne .LBB238_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -14915,22 +15050,22 @@ define void @atomic128_sub_seq_cst(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection550:
+; O2-NEXT:  .Lpcsection595:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection551:
+; O2-NEXT:  .Lpcsection596:
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB238_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
-; O2-NEXT:  .Lpcsection552:
+; O2-NEXT:  .Lpcsection597:
 ; O2-NEXT:    addq $-42, %rbx
 ; O2-NEXT:    movq %rdx, %rcx
-; O2-NEXT:  .Lpcsection553:
+; O2-NEXT:  .Lpcsection598:
 ; O2-NEXT:    adcq $-1, %rcx
-; O2-NEXT:  .Lpcsection554:
+; O2-NEXT:  .Lpcsection599:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection555:
+; O2-NEXT:  .Lpcsection600:
 ; O2-NEXT:    jne .LBB238_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -14944,22 +15079,22 @@ define void @atomic128_sub_seq_cst(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection550:
+; O3-NEXT:  .Lpcsection595:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection551:
+; O3-NEXT:  .Lpcsection596:
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB238_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
-; O3-NEXT:  .Lpcsection552:
+; O3-NEXT:  .Lpcsection597:
 ; O3-NEXT:    addq $-42, %rbx
 ; O3-NEXT:    movq %rdx, %rcx
-; O3-NEXT:  .Lpcsection553:
+; O3-NEXT:  .Lpcsection598:
 ; O3-NEXT:    adcq $-1, %rcx
-; O3-NEXT:  .Lpcsection554:
+; O3-NEXT:  .Lpcsection599:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection555:
+; O3-NEXT:  .Lpcsection600:
 ; O3-NEXT:    jne .LBB238_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -15021,21 +15156,21 @@ define void @atomic128_and_seq_cst(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection556:
+; O1-NEXT:  .Lpcsection601:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection557:
+; O1-NEXT:  .Lpcsection602:
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB239_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ebx
-; O1-NEXT:  .Lpcsection558:
+; O1-NEXT:  .Lpcsection603:
 ; O1-NEXT:    andl $42, %ebx
-; O1-NEXT:  .Lpcsection559:
+; O1-NEXT:  .Lpcsection604:
 ; O1-NEXT:    xorl %ecx, %ecx
-; O1-NEXT:  .Lpcsection560:
+; O1-NEXT:  .Lpcsection605:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection561:
+; O1-NEXT:  .Lpcsection606:
 ; O1-NEXT:    jne .LBB239_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -15049,21 +15184,21 @@ define void @atomic128_and_seq_cst(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection556:
+; O2-NEXT:  .Lpcsection601:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection557:
+; O2-NEXT:  .Lpcsection602:
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB239_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ebx
-; O2-NEXT:  .Lpcsection558:
+; O2-NEXT:  .Lpcsection603:
 ; O2-NEXT:    andl $42, %ebx
-; O2-NEXT:  .Lpcsection559:
+; O2-NEXT:  .Lpcsection604:
 ; O2-NEXT:    xorl %ecx, %ecx
-; O2-NEXT:  .Lpcsection560:
+; O2-NEXT:  .Lpcsection605:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection561:
+; O2-NEXT:  .Lpcsection606:
 ; O2-NEXT:    jne .LBB239_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -15077,21 +15212,21 @@ define void @atomic128_and_seq_cst(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection556:
+; O3-NEXT:  .Lpcsection601:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection557:
+; O3-NEXT:  .Lpcsection602:
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB239_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ebx
-; O3-NEXT:  .Lpcsection558:
+; O3-NEXT:  .Lpcsection603:
 ; O3-NEXT:    andl $42, %ebx
-; O3-NEXT:  .Lpcsection559:
+; O3-NEXT:  .Lpcsection604:
 ; O3-NEXT:    xorl %ecx, %ecx
-; O3-NEXT:  .Lpcsection560:
+; O3-NEXT:  .Lpcsection605:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection561:
+; O3-NEXT:  .Lpcsection606:
 ; O3-NEXT:    jne .LBB239_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -15149,20 +15284,20 @@ define void @atomic128_or_seq_cst(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection562:
+; O1-NEXT:  .Lpcsection607:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection563:
+; O1-NEXT:  .Lpcsection608:
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB240_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
-; O1-NEXT:  .Lpcsection564:
+; O1-NEXT:  .Lpcsection609:
 ; O1-NEXT:    orq $42, %rbx
 ; O1-NEXT:    movq %rdx, %rcx
-; O1-NEXT:  .Lpcsection565:
+; O1-NEXT:  .Lpcsection610:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection566:
+; O1-NEXT:  .Lpcsection611:
 ; O1-NEXT:    jne .LBB240_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -15176,20 +15311,20 @@ define void @atomic128_or_seq_cst(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection562:
+; O2-NEXT:  .Lpcsection607:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection563:
+; O2-NEXT:  .Lpcsection608:
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB240_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
-; O2-NEXT:  .Lpcsection564:
+; O2-NEXT:  .Lpcsection609:
 ; O2-NEXT:    orq $42, %rbx
 ; O2-NEXT:    movq %rdx, %rcx
-; O2-NEXT:  .Lpcsection565:
+; O2-NEXT:  .Lpcsection610:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection566:
+; O2-NEXT:  .Lpcsection611:
 ; O2-NEXT:    jne .LBB240_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -15203,20 +15338,20 @@ define void @atomic128_or_seq_cst(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection562:
+; O3-NEXT:  .Lpcsection607:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection563:
+; O3-NEXT:  .Lpcsection608:
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB240_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
-; O3-NEXT:  .Lpcsection564:
+; O3-NEXT:  .Lpcsection609:
 ; O3-NEXT:    orq $42, %rbx
 ; O3-NEXT:    movq %rdx, %rcx
-; O3-NEXT:  .Lpcsection565:
+; O3-NEXT:  .Lpcsection610:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection566:
+; O3-NEXT:  .Lpcsection611:
 ; O3-NEXT:    jne .LBB240_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -15274,20 +15409,20 @@ define void @atomic128_xor_seq_cst(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection567:
+; O1-NEXT:  .Lpcsection612:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection568:
+; O1-NEXT:  .Lpcsection613:
 ; O1-NEXT:    movq 8(%rdi), %rdx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB241_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movq %rax, %rbx
-; O1-NEXT:  .Lpcsection569:
+; O1-NEXT:  .Lpcsection614:
 ; O1-NEXT:    xorq $42, %rbx
 ; O1-NEXT:    movq %rdx, %rcx
-; O1-NEXT:  .Lpcsection570:
+; O1-NEXT:  .Lpcsection615:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection571:
+; O1-NEXT:  .Lpcsection616:
 ; O1-NEXT:    jne .LBB241_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -15301,20 +15436,20 @@ define void @atomic128_xor_seq_cst(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection567:
+; O2-NEXT:  .Lpcsection612:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection568:
+; O2-NEXT:  .Lpcsection613:
 ; O2-NEXT:    movq 8(%rdi), %rdx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB241_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movq %rax, %rbx
-; O2-NEXT:  .Lpcsection569:
+; O2-NEXT:  .Lpcsection614:
 ; O2-NEXT:    xorq $42, %rbx
 ; O2-NEXT:    movq %rdx, %rcx
-; O2-NEXT:  .Lpcsection570:
+; O2-NEXT:  .Lpcsection615:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection571:
+; O2-NEXT:  .Lpcsection616:
 ; O2-NEXT:    jne .LBB241_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -15328,20 +15463,20 @@ define void @atomic128_xor_seq_cst(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection567:
+; O3-NEXT:  .Lpcsection612:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection568:
+; O3-NEXT:  .Lpcsection613:
 ; O3-NEXT:    movq 8(%rdi), %rdx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB241_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movq %rax, %rbx
-; O3-NEXT:  .Lpcsection569:
+; O3-NEXT:  .Lpcsection614:
 ; O3-NEXT:    xorq $42, %rbx
 ; O3-NEXT:    movq %rdx, %rcx
-; O3-NEXT:  .Lpcsection570:
+; O3-NEXT:  .Lpcsection615:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection571:
+; O3-NEXT:  .Lpcsection616:
 ; O3-NEXT:    jne .LBB241_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -15405,23 +15540,23 @@ define void @atomic128_nand_seq_cst(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection572:
+; O1-NEXT:  .Lpcsection617:
 ; O1-NEXT:    movq (%rdi), %rax
-; O1-NEXT:  .Lpcsection573:
+; O1-NEXT:  .Lpcsection618:
 ; O1-NEXT:    movq 8(%rdi), %rdx
-; O1-NEXT:  .Lpcsection574:
+; O1-NEXT:  .Lpcsection619:
 ; O1-NEXT:    movq $-1, %rcx
 ; O1-NEXT:    .p2align 4, 0x90
 ; O1-NEXT:  .LBB242_1: # %atomicrmw.start
 ; O1-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O1-NEXT:    movl %eax, %ebx
-; O1-NEXT:  .Lpcsection575:
+; O1-NEXT:  .Lpcsection620:
 ; O1-NEXT:    notl %ebx
-; O1-NEXT:  .Lpcsection576:
+; O1-NEXT:  .Lpcsection621:
 ; O1-NEXT:    orq $-43, %rbx
-; O1-NEXT:  .Lpcsection577:
+; O1-NEXT:  .Lpcsection622:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection578:
+; O1-NEXT:  .Lpcsection623:
 ; O1-NEXT:    jne .LBB242_1
 ; O1-NEXT:  # %bb.2: # %atomicrmw.end
 ; O1-NEXT:    movq $1, foo(%rip)
@@ -15435,23 +15570,23 @@ define void @atomic128_nand_seq_cst(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection572:
+; O2-NEXT:  .Lpcsection617:
 ; O2-NEXT:    movq (%rdi), %rax
-; O2-NEXT:  .Lpcsection573:
+; O2-NEXT:  .Lpcsection618:
 ; O2-NEXT:    movq 8(%rdi), %rdx
-; O2-NEXT:  .Lpcsection574:
+; O2-NEXT:  .Lpcsection619:
 ; O2-NEXT:    movq $-1, %rcx
 ; O2-NEXT:    .p2align 4, 0x90
 ; O2-NEXT:  .LBB242_1: # %atomicrmw.start
 ; O2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O2-NEXT:    movl %eax, %ebx
-; O2-NEXT:  .Lpcsection575:
+; O2-NEXT:  .Lpcsection620:
 ; O2-NEXT:    notl %ebx
-; O2-NEXT:  .Lpcsection576:
+; O2-NEXT:  .Lpcsection621:
 ; O2-NEXT:    orq $-43, %rbx
-; O2-NEXT:  .Lpcsection577:
+; O2-NEXT:  .Lpcsection622:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection578:
+; O2-NEXT:  .Lpcsection623:
 ; O2-NEXT:    jne .LBB242_1
 ; O2-NEXT:  # %bb.2: # %atomicrmw.end
 ; O2-NEXT:    movq $1, foo(%rip)
@@ -15465,23 +15600,23 @@ define void @atomic128_nand_seq_cst(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection572:
+; O3-NEXT:  .Lpcsection617:
 ; O3-NEXT:    movq (%rdi), %rax
-; O3-NEXT:  .Lpcsection573:
+; O3-NEXT:  .Lpcsection618:
 ; O3-NEXT:    movq 8(%rdi), %rdx
-; O3-NEXT:  .Lpcsection574:
+; O3-NEXT:  .Lpcsection619:
 ; O3-NEXT:    movq $-1, %rcx
 ; O3-NEXT:    .p2align 4, 0x90
 ; O3-NEXT:  .LBB242_1: # %atomicrmw.start
 ; O3-NEXT:    # =>This Inner Loop Header: Depth=1
 ; O3-NEXT:    movl %eax, %ebx
-; O3-NEXT:  .Lpcsection575:
+; O3-NEXT:  .Lpcsection620:
 ; O3-NEXT:    notl %ebx
-; O3-NEXT:  .Lpcsection576:
+; O3-NEXT:  .Lpcsection621:
 ; O3-NEXT:    orq $-43, %rbx
-; O3-NEXT:  .Lpcsection577:
+; O3-NEXT:  .Lpcsection622:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection578:
+; O3-NEXT:  .Lpcsection623:
 ; O3-NEXT:    jne .LBB242_1
 ; O3-NEXT:  # %bb.2: # %atomicrmw.end
 ; O3-NEXT:    movq $1, foo(%rip)
@@ -15542,31 +15677,31 @@ define void @atomic128_cas_monotonic(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection579:
+; O1-NEXT:  .Lpcsection624:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection580:
+; O1-NEXT:  .Lpcsection625:
 ; O1-NEXT:    movl $1, %ebx
-; O1-NEXT:  .Lpcsection581:
+; O1-NEXT:  .Lpcsection626:
 ; O1-NEXT:    xorl %edx, %edx
-; O1-NEXT:  .Lpcsection582:
+; O1-NEXT:  .Lpcsection627:
 ; O1-NEXT:    xorl %ecx, %ecx
-; O1-NEXT:  .Lpcsection583:
+; O1-NEXT:  .Lpcsection628:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection584:
+; O1-NEXT:  .Lpcsection629:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection585:
+; O1-NEXT:  .Lpcsection630:
 ; O1-NEXT:    xorl %edx, %edx
-; O1-NEXT:  .Lpcsection586:
+; O1-NEXT:  .Lpcsection631:
 ; O1-NEXT:    xorl %ecx, %ecx
-; O1-NEXT:  .Lpcsection587:
+; O1-NEXT:  .Lpcsection632:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection588:
+; O1-NEXT:  .Lpcsection633:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection589:
+; O1-NEXT:  .Lpcsection634:
 ; O1-NEXT:    xorl %edx, %edx
-; O1-NEXT:  .Lpcsection590:
+; O1-NEXT:  .Lpcsection635:
 ; O1-NEXT:    xorl %ecx, %ecx
-; O1-NEXT:  .Lpcsection591:
+; O1-NEXT:  .Lpcsection636:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    popq %rbx
@@ -15579,31 +15714,31 @@ define void @atomic128_cas_monotonic(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection579:
+; O2-NEXT:  .Lpcsection624:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection580:
+; O2-NEXT:  .Lpcsection625:
 ; O2-NEXT:    movl $1, %ebx
-; O2-NEXT:  .Lpcsection581:
+; O2-NEXT:  .Lpcsection626:
 ; O2-NEXT:    xorl %edx, %edx
-; O2-NEXT:  .Lpcsection582:
+; O2-NEXT:  .Lpcsection627:
 ; O2-NEXT:    xorl %ecx, %ecx
-; O2-NEXT:  .Lpcsection583:
+; O2-NEXT:  .Lpcsection628:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection584:
+; O2-NEXT:  .Lpcsection629:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection585:
+; O2-NEXT:  .Lpcsection630:
 ; O2-NEXT:    xorl %edx, %edx
-; O2-NEXT:  .Lpcsection586:
+; O2-NEXT:  .Lpcsection631:
 ; O2-NEXT:    xorl %ecx, %ecx
-; O2-NEXT:  .Lpcsection587:
+; O2-NEXT:  .Lpcsection632:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection588:
+; O2-NEXT:  .Lpcsection633:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection589:
+; O2-NEXT:  .Lpcsection634:
 ; O2-NEXT:    xorl %edx, %edx
-; O2-NEXT:  .Lpcsection590:
+; O2-NEXT:  .Lpcsection635:
 ; O2-NEXT:    xorl %ecx, %ecx
-; O2-NEXT:  .Lpcsection591:
+; O2-NEXT:  .Lpcsection636:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    popq %rbx
@@ -15616,31 +15751,31 @@ define void @atomic128_cas_monotonic(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection579:
+; O3-NEXT:  .Lpcsection624:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection580:
+; O3-NEXT:  .Lpcsection625:
 ; O3-NEXT:    movl $1, %ebx
-; O3-NEXT:  .Lpcsection581:
+; O3-NEXT:  .Lpcsection626:
 ; O3-NEXT:    xorl %edx, %edx
-; O3-NEXT:  .Lpcsection582:
+; O3-NEXT:  .Lpcsection627:
 ; O3-NEXT:    xorl %ecx, %ecx
-; O3-NEXT:  .Lpcsection583:
+; O3-NEXT:  .Lpcsection628:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection584:
+; O3-NEXT:  .Lpcsection629:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection585:
+; O3-NEXT:  .Lpcsection630:
 ; O3-NEXT:    xorl %edx, %edx
-; O3-NEXT:  .Lpcsection586:
+; O3-NEXT:  .Lpcsection631:
 ; O3-NEXT:    xorl %ecx, %ecx
-; O3-NEXT:  .Lpcsection587:
+; O3-NEXT:  .Lpcsection632:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection588:
+; O3-NEXT:  .Lpcsection633:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection589:
+; O3-NEXT:  .Lpcsection634:
 ; O3-NEXT:    xorl %edx, %edx
-; O3-NEXT:  .Lpcsection590:
+; O3-NEXT:  .Lpcsection635:
 ; O3-NEXT:    xorl %ecx, %ecx
-; O3-NEXT:  .Lpcsection591:
+; O3-NEXT:  .Lpcsection636:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    popq %rbx
@@ -15702,31 +15837,31 @@ define void @atomic128_cas_acquire(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection592:
+; O1-NEXT:  .Lpcsection637:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection593:
+; O1-NEXT:  .Lpcsection638:
 ; O1-NEXT:    movl $1, %ebx
-; O1-NEXT:  .Lpcsection594:
+; O1-NEXT:  .Lpcsection639:
 ; O1-NEXT:    xorl %edx, %edx
-; O1-NEXT:  .Lpcsection595:
+; O1-NEXT:  .Lpcsection640:
 ; O1-NEXT:    xorl %ecx, %ecx
-; O1-NEXT:  .Lpcsection596:
+; O1-NEXT:  .Lpcsection641:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection597:
+; O1-NEXT:  .Lpcsection642:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection598:
+; O1-NEXT:  .Lpcsection643:
 ; O1-NEXT:    xorl %edx, %edx
-; O1-NEXT:  .Lpcsection599:
+; O1-NEXT:  .Lpcsection644:
 ; O1-NEXT:    xorl %ecx, %ecx
-; O1-NEXT:  .Lpcsection600:
+; O1-NEXT:  .Lpcsection645:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection601:
+; O1-NEXT:  .Lpcsection646:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection602:
+; O1-NEXT:  .Lpcsection647:
 ; O1-NEXT:    xorl %edx, %edx
-; O1-NEXT:  .Lpcsection603:
+; O1-NEXT:  .Lpcsection648:
 ; O1-NEXT:    xorl %ecx, %ecx
-; O1-NEXT:  .Lpcsection604:
+; O1-NEXT:  .Lpcsection649:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    popq %rbx
@@ -15739,31 +15874,31 @@ define void @atomic128_cas_acquire(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection592:
+; O2-NEXT:  .Lpcsection637:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection593:
+; O2-NEXT:  .Lpcsection638:
 ; O2-NEXT:    movl $1, %ebx
-; O2-NEXT:  .Lpcsection594:
+; O2-NEXT:  .Lpcsection639:
 ; O2-NEXT:    xorl %edx, %edx
-; O2-NEXT:  .Lpcsection595:
+; O2-NEXT:  .Lpcsection640:
 ; O2-NEXT:    xorl %ecx, %ecx
-; O2-NEXT:  .Lpcsection596:
+; O2-NEXT:  .Lpcsection641:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection597:
+; O2-NEXT:  .Lpcsection642:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection598:
+; O2-NEXT:  .Lpcsection643:
 ; O2-NEXT:    xorl %edx, %edx
-; O2-NEXT:  .Lpcsection599:
+; O2-NEXT:  .Lpcsection644:
 ; O2-NEXT:    xorl %ecx, %ecx
-; O2-NEXT:  .Lpcsection600:
+; O2-NEXT:  .Lpcsection645:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection601:
+; O2-NEXT:  .Lpcsection646:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection602:
+; O2-NEXT:  .Lpcsection647:
 ; O2-NEXT:    xorl %edx, %edx
-; O2-NEXT:  .Lpcsection603:
+; O2-NEXT:  .Lpcsection648:
 ; O2-NEXT:    xorl %ecx, %ecx
-; O2-NEXT:  .Lpcsection604:
+; O2-NEXT:  .Lpcsection649:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    popq %rbx
@@ -15776,31 +15911,31 @@ define void @atomic128_cas_acquire(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection592:
+; O3-NEXT:  .Lpcsection637:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection593:
+; O3-NEXT:  .Lpcsection638:
 ; O3-NEXT:    movl $1, %ebx
-; O3-NEXT:  .Lpcsection594:
+; O3-NEXT:  .Lpcsection639:
 ; O3-NEXT:    xorl %edx, %edx
-; O3-NEXT:  .Lpcsection595:
+; O3-NEXT:  .Lpcsection640:
 ; O3-NEXT:    xorl %ecx, %ecx
-; O3-NEXT:  .Lpcsection596:
+; O3-NEXT:  .Lpcsection641:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection597:
+; O3-NEXT:  .Lpcsection642:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection598:
+; O3-NEXT:  .Lpcsection643:
 ; O3-NEXT:    xorl %edx, %edx
-; O3-NEXT:  .Lpcsection599:
+; O3-NEXT:  .Lpcsection644:
 ; O3-NEXT:    xorl %ecx, %ecx
-; O3-NEXT:  .Lpcsection600:
+; O3-NEXT:  .Lpcsection645:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection601:
+; O3-NEXT:  .Lpcsection646:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection602:
+; O3-NEXT:  .Lpcsection647:
 ; O3-NEXT:    xorl %edx, %edx
-; O3-NEXT:  .Lpcsection603:
+; O3-NEXT:  .Lpcsection648:
 ; O3-NEXT:    xorl %ecx, %ecx
-; O3-NEXT:  .Lpcsection604:
+; O3-NEXT:  .Lpcsection649:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    popq %rbx
@@ -15862,31 +15997,31 @@ define void @atomic128_cas_release(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection605:
+; O1-NEXT:  .Lpcsection650:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection606:
+; O1-NEXT:  .Lpcsection651:
 ; O1-NEXT:    movl $1, %ebx
-; O1-NEXT:  .Lpcsection607:
+; O1-NEXT:  .Lpcsection652:
 ; O1-NEXT:    xorl %edx, %edx
-; O1-NEXT:  .Lpcsection608:
+; O1-NEXT:  .Lpcsection653:
 ; O1-NEXT:    xorl %ecx, %ecx
-; O1-NEXT:  .Lpcsection609:
+; O1-NEXT:  .Lpcsection654:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection610:
+; O1-NEXT:  .Lpcsection655:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection611:
+; O1-NEXT:  .Lpcsection656:
 ; O1-NEXT:    xorl %edx, %edx
-; O1-NEXT:  .Lpcsection612:
+; O1-NEXT:  .Lpcsection657:
 ; O1-NEXT:    xorl %ecx, %ecx
-; O1-NEXT:  .Lpcsection613:
+; O1-NEXT:  .Lpcsection658:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection614:
+; O1-NEXT:  .Lpcsection659:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection615:
+; O1-NEXT:  .Lpcsection660:
 ; O1-NEXT:    xorl %edx, %edx
-; O1-NEXT:  .Lpcsection616:
+; O1-NEXT:  .Lpcsection661:
 ; O1-NEXT:    xorl %ecx, %ecx
-; O1-NEXT:  .Lpcsection617:
+; O1-NEXT:  .Lpcsection662:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    popq %rbx
@@ -15899,31 +16034,31 @@ define void @atomic128_cas_release(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection605:
+; O2-NEXT:  .Lpcsection650:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection606:
+; O2-NEXT:  .Lpcsection651:
 ; O2-NEXT:    movl $1, %ebx
-; O2-NEXT:  .Lpcsection607:
+; O2-NEXT:  .Lpcsection652:
 ; O2-NEXT:    xorl %edx, %edx
-; O2-NEXT:  .Lpcsection608:
+; O2-NEXT:  .Lpcsection653:
 ; O2-NEXT:    xorl %ecx, %ecx
-; O2-NEXT:  .Lpcsection609:
+; O2-NEXT:  .Lpcsection654:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection610:
+; O2-NEXT:  .Lpcsection655:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection611:
+; O2-NEXT:  .Lpcsection656:
 ; O2-NEXT:    xorl %edx, %edx
-; O2-NEXT:  .Lpcsection612:
+; O2-NEXT:  .Lpcsection657:
 ; O2-NEXT:    xorl %ecx, %ecx
-; O2-NEXT:  .Lpcsection613:
+; O2-NEXT:  .Lpcsection658:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection614:
+; O2-NEXT:  .Lpcsection659:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection615:
+; O2-NEXT:  .Lpcsection660:
 ; O2-NEXT:    xorl %edx, %edx
-; O2-NEXT:  .Lpcsection616:
+; O2-NEXT:  .Lpcsection661:
 ; O2-NEXT:    xorl %ecx, %ecx
-; O2-NEXT:  .Lpcsection617:
+; O2-NEXT:  .Lpcsection662:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    popq %rbx
@@ -15936,31 +16071,31 @@ define void @atomic128_cas_release(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection605:
+; O3-NEXT:  .Lpcsection650:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection606:
+; O3-NEXT:  .Lpcsection651:
 ; O3-NEXT:    movl $1, %ebx
-; O3-NEXT:  .Lpcsection607:
+; O3-NEXT:  .Lpcsection652:
 ; O3-NEXT:    xorl %edx, %edx
-; O3-NEXT:  .Lpcsection608:
+; O3-NEXT:  .Lpcsection653:
 ; O3-NEXT:    xorl %ecx, %ecx
-; O3-NEXT:  .Lpcsection609:
+; O3-NEXT:  .Lpcsection654:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection610:
+; O3-NEXT:  .Lpcsection655:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection611:
+; O3-NEXT:  .Lpcsection656:
 ; O3-NEXT:    xorl %edx, %edx
-; O3-NEXT:  .Lpcsection612:
+; O3-NEXT:  .Lpcsection657:
 ; O3-NEXT:    xorl %ecx, %ecx
-; O3-NEXT:  .Lpcsection613:
+; O3-NEXT:  .Lpcsection658:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection614:
+; O3-NEXT:  .Lpcsection659:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection615:
+; O3-NEXT:  .Lpcsection660:
 ; O3-NEXT:    xorl %edx, %edx
-; O3-NEXT:  .Lpcsection616:
+; O3-NEXT:  .Lpcsection661:
 ; O3-NEXT:    xorl %ecx, %ecx
-; O3-NEXT:  .Lpcsection617:
+; O3-NEXT:  .Lpcsection662:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    popq %rbx
@@ -16022,31 +16157,31 @@ define void @atomic128_cas_acq_rel(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection618:
+; O1-NEXT:  .Lpcsection663:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection619:
+; O1-NEXT:  .Lpcsection664:
 ; O1-NEXT:    movl $1, %ebx
-; O1-NEXT:  .Lpcsection620:
+; O1-NEXT:  .Lpcsection665:
 ; O1-NEXT:    xorl %edx, %edx
-; O1-NEXT:  .Lpcsection621:
+; O1-NEXT:  .Lpcsection666:
 ; O1-NEXT:    xorl %ecx, %ecx
-; O1-NEXT:  .Lpcsection622:
+; O1-NEXT:  .Lpcsection667:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection623:
+; O1-NEXT:  .Lpcsection668:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection624:
+; O1-NEXT:  .Lpcsection669:
 ; O1-NEXT:    xorl %edx, %edx
-; O1-NEXT:  .Lpcsection625:
+; O1-NEXT:  .Lpcsection670:
 ; O1-NEXT:    xorl %ecx, %ecx
-; O1-NEXT:  .Lpcsection626:
+; O1-NEXT:  .Lpcsection671:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection627:
+; O1-NEXT:  .Lpcsection672:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection628:
+; O1-NEXT:  .Lpcsection673:
 ; O1-NEXT:    xorl %edx, %edx
-; O1-NEXT:  .Lpcsection629:
+; O1-NEXT:  .Lpcsection674:
 ; O1-NEXT:    xorl %ecx, %ecx
-; O1-NEXT:  .Lpcsection630:
+; O1-NEXT:  .Lpcsection675:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
 ; O1-NEXT:    movq $1, foo(%rip)
 ; O1-NEXT:    popq %rbx
@@ -16059,31 +16194,31 @@ define void @atomic128_cas_acq_rel(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection618:
+; O2-NEXT:  .Lpcsection663:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection619:
+; O2-NEXT:  .Lpcsection664:
 ; O2-NEXT:    movl $1, %ebx
-; O2-NEXT:  .Lpcsection620:
+; O2-NEXT:  .Lpcsection665:
 ; O2-NEXT:    xorl %edx, %edx
-; O2-NEXT:  .Lpcsection621:
+; O2-NEXT:  .Lpcsection666:
 ; O2-NEXT:    xorl %ecx, %ecx
-; O2-NEXT:  .Lpcsection622:
+; O2-NEXT:  .Lpcsection667:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection623:
+; O2-NEXT:  .Lpcsection668:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection624:
+; O2-NEXT:  .Lpcsection669:
 ; O2-NEXT:    xorl %edx, %edx
-; O2-NEXT:  .Lpcsection625:
+; O2-NEXT:  .Lpcsection670:
 ; O2-NEXT:    xorl %ecx, %ecx
-; O2-NEXT:  .Lpcsection626:
+; O2-NEXT:  .Lpcsection671:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection627:
+; O2-NEXT:  .Lpcsection672:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection628:
+; O2-NEXT:  .Lpcsection673:
 ; O2-NEXT:    xorl %edx, %edx
-; O2-NEXT:  .Lpcsection629:
+; O2-NEXT:  .Lpcsection674:
 ; O2-NEXT:    xorl %ecx, %ecx
-; O2-NEXT:  .Lpcsection630:
+; O2-NEXT:  .Lpcsection675:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
 ; O2-NEXT:    movq $1, foo(%rip)
 ; O2-NEXT:    popq %rbx
@@ -16096,31 +16231,31 @@ define void @atomic128_cas_acq_rel(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection618:
+; O3-NEXT:  .Lpcsection663:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection619:
+; O3-NEXT:  .Lpcsection664:
 ; O3-NEXT:    movl $1, %ebx
-; O3-NEXT:  .Lpcsection620:
+; O3-NEXT:  .Lpcsection665:
 ; O3-NEXT:    xorl %edx, %edx
-; O3-NEXT:  .Lpcsection621:
+; O3-NEXT:  .Lpcsection666:
 ; O3-NEXT:    xorl %ecx, %ecx
-; O3-NEXT:  .Lpcsection622:
+; O3-NEXT:  .Lpcsection667:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection623:
+; O3-NEXT:  .Lpcsection668:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection624:
+; O3-NEXT:  .Lpcsection669:
 ; O3-NEXT:    xorl %edx, %edx
-; O3-NEXT:  .Lpcsection625:
+; O3-NEXT:  .Lpcsection670:
 ; O3-NEXT:    xorl %ecx, %ecx
-; O3-NEXT:  .Lpcsection626:
+; O3-NEXT:  .Lpcsection671:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection627:
+; O3-NEXT:  .Lpcsection672:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection628:
+; O3-NEXT:  .Lpcsection673:
 ; O3-NEXT:    xorl %edx, %edx
-; O3-NEXT:  .Lpcsection629:
+; O3-NEXT:  .Lpcsection674:
 ; O3-NEXT:    xorl %ecx, %ecx
-; O3-NEXT:  .Lpcsection630:
+; O3-NEXT:  .Lpcsection675:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
 ; O3-NEXT:    movq $1, foo(%rip)
 ; O3-NEXT:    popq %rbx
@@ -16182,31 +16317,31 @@ define void @atomic128_cas_seq_cst(ptr %a) {
 ; O1-NEXT:    .cfi_def_cfa_offset 16
 ; O1-NEXT:    .cfi_offset %rbx, -16
 ; O1-NEXT:    movq foo(%rip), %rax
-; O1-NEXT:  .Lpcsection631:
+; O1-NEXT:  .Lpcsection676:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection632:
+; O1-NEXT:  .Lpcsection677:
 ; O1-NEXT:    movl $1, %ebx
-; O1-NEXT:  .Lpcsection633:
+; O1-NEXT:  .Lpcsection678:
 ; O1-NEXT:    xorl %edx, %edx
-; O1-NEXT:  .Lpcsection634:
+; O1-NEXT:  .Lpcsection679:
 ; O1-NEXT:    xorl %ecx, %ecx
-; O1-NEXT:  .Lpcsection635:
+; O1-NEXT:  .Lpcsection680:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection636:
+; O1-NEXT:  .Lpcsection681:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection637:
+; O1-NEXT:  .Lpcsection682:
 ; O1-NEXT:    xorl %edx, %edx
-; O1-NEXT:  .Lpcsection638:
+; O1-NEXT:  .Lpcsection683:
 ; O1-NEXT:    xorl %ecx, %ecx
-; O1-NEXT:  .Lpcsection639:
+; O1-NEXT:  .Lpcsection684:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
-; O1-NEXT:  .Lpcsection640:
+; O1-NEXT:  .Lpcsection685:
 ; O1-NEXT:    movl $42, %eax
-; O1-NEXT:  .Lpcsection641:
+; O1-NEXT:  .Lpcsection686:
 ; O1-NEXT:    xorl %edx, %edx
-; O1-NEXT:  .Lpcsection642:
+; O1-NEXT:  .Lpcsection687:
 ; O1-NEXT:    xorl %ecx, %ecx
-; O1-NEXT:  .Lpcsection643:
+; O1-NEXT:  .Lpcsection688:
 ; O1-NEXT:    lock cmpxchg16b (%rdi)
 ; O1-NEXT:    movq $3, foo(%rip)
 ; O1-NEXT:    popq %rbx
@@ -16219,31 +16354,31 @@ define void @atomic128_cas_seq_cst(ptr %a) {
 ; O2-NEXT:    .cfi_def_cfa_offset 16
 ; O2-NEXT:    .cfi_offset %rbx, -16
 ; O2-NEXT:    movq foo(%rip), %rax
-; O2-NEXT:  .Lpcsection631:
+; O2-NEXT:  .Lpcsection676:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection632:
+; O2-NEXT:  .Lpcsection677:
 ; O2-NEXT:    movl $1, %ebx
-; O2-NEXT:  .Lpcsection633:
+; O2-NEXT:  .Lpcsection678:
 ; O2-NEXT:    xorl %edx, %edx
-; O2-NEXT:  .Lpcsection634:
+; O2-NEXT:  .Lpcsection679:
 ; O2-NEXT:    xorl %ecx, %ecx
-; O2-NEXT:  .Lpcsection635:
+; O2-NEXT:  .Lpcsection680:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection636:
+; O2-NEXT:  .Lpcsection681:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection637:
+; O2-NEXT:  .Lpcsection682:
 ; O2-NEXT:    xorl %edx, %edx
-; O2-NEXT:  .Lpcsection638:
+; O2-NEXT:  .Lpcsection683:
 ; O2-NEXT:    xorl %ecx, %ecx
-; O2-NEXT:  .Lpcsection639:
+; O2-NEXT:  .Lpcsection684:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
-; O2-NEXT:  .Lpcsection640:
+; O2-NEXT:  .Lpcsection685:
 ; O2-NEXT:    movl $42, %eax
-; O2-NEXT:  .Lpcsection641:
+; O2-NEXT:  .Lpcsection686:
 ; O2-NEXT:    xorl %edx, %edx
-; O2-NEXT:  .Lpcsection642:
+; O2-NEXT:  .Lpcsection687:
 ; O2-NEXT:    xorl %ecx, %ecx
-; O2-NEXT:  .Lpcsection643:
+; O2-NEXT:  .Lpcsection688:
 ; O2-NEXT:    lock cmpxchg16b (%rdi)
 ; O2-NEXT:    movq $3, foo(%rip)
 ; O2-NEXT:    popq %rbx
@@ -16256,31 +16391,31 @@ define void @atomic128_cas_seq_cst(ptr %a) {
 ; O3-NEXT:    .cfi_def_cfa_offset 16
 ; O3-NEXT:    .cfi_offset %rbx, -16
 ; O3-NEXT:    movq foo(%rip), %rax
-; O3-NEXT:  .Lpcsection631:
+; O3-NEXT:  .Lpcsection676:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection632:
+; O3-NEXT:  .Lpcsection677:
 ; O3-NEXT:    movl $1, %ebx
-; O3-NEXT:  .Lpcsection633:
+; O3-NEXT:  .Lpcsection678:
 ; O3-NEXT:    xorl %edx, %edx
-; O3-NEXT:  .Lpcsection634:
+; O3-NEXT:  .Lpcsection679:
 ; O3-NEXT:    xorl %ecx, %ecx
-; O3-NEXT:  .Lpcsection635:
+; O3-NEXT:  .Lpcsection680:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection636:
+; O3-NEXT:  .Lpcsection681:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection637:
+; O3-NEXT:  .Lpcsection682:
 ; O3-NEXT:    xorl %edx, %edx
-; O3-NEXT:  .Lpcsection638:
+; O3-NEXT:  .Lpcsection683:
 ; O3-NEXT:    xorl %ecx, %ecx
-; O3-NEXT:  .Lpcsection639:
+; O3-NEXT:  .Lpcsection684:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
-; O3-NEXT:  .Lpcsection640:
+; O3-NEXT:  .Lpcsection685:
 ; O3-NEXT:    movl $42, %eax
-; O3-NEXT:  .Lpcsection641:
+; O3-NEXT:  .Lpcsection686:
 ; O3-NEXT:    xorl %edx, %edx
-; O3-NEXT:  .Lpcsection642:
+; O3-NEXT:  .Lpcsection687:
 ; O3-NEXT:    xorl %ecx, %ecx
-; O3-NEXT:  .Lpcsection643:
+; O3-NEXT:  .Lpcsection688:
 ; O3-NEXT:    lock cmpxchg16b (%rdi)
 ; O3-NEXT:    movq $3, foo(%rip)
 ; O3-NEXT:    popq %rbx
diff --git a/llvm/test/CodeGen/X86/peephole-copy.mir b/llvm/test/CodeGen/X86/peephole-copy.mir
new file mode 100644
index 0000000000..e875465dfc
--- /dev/null
+++ b/llvm/test/CodeGen/X86/peephole-copy.mir
@@ -0,0 +1,34 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 3
+# RUN: llc -mtriple=i686-- -run-pass=peephole-opt %s -o - | FileCheck %s
+--- |
+  define void @c() {
+  entry:
+    tail call void asm sideeffect "", "q,~{dirflag},~{fpsr},~{flags}"(i32 512)
+    tail call void asm sideeffect "", "q,~{dirflag},~{fpsr},~{flags}"(i32 512)
+    ret void
+  }
+...
+---
+# In peephole optimization the modified COPY instruction should not cause
+# compiler failure.
+name: c
+registers:
+  - { id: 0, class: gr32_abcd }
+  - { id: 1, class: gr32_abcd }
+  - { id: 2, class: gr32 }
+
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: c
+    ; CHECK: [[MOV32ri:%[0-9]+]]:gr32_abcd = MOV32ri 512
+    ; CHECK-NEXT: INLINEASM &"", 1 /* sideeffect attdialect */, 2359305 /* reguse:GR32 */, [[MOV32ri]], 1 /* reguse */, implicit-def early-clobber $df
+    ; CHECK-NEXT: [[MOV32ri1:%[0-9]+]]:gr32_abcd = MOV32ri 512
+    ; CHECK-NEXT: INLINEASM &"", 1 /* sideeffect attdialect */, 2359305 /* reguse:GR32 */, [[MOV32ri1]], 1 /* reguse */, implicit-def early-clobber $df
+    ; CHECK-NEXT: RET 0
+    %2 = MOV32ri 512
+    %0 = COPY %2
+    INLINEASM &"", 1 /* sideeffect attdialect */, 2359305 /* reguse:GR32_ABCD */, %0:gr32_abcd, 1 /* clobber */, implicit-def early-clobber $df
+    %1 = COPY %2
+    INLINEASM &"", 1 /* sideeffect attdialect */, 2359305 /* reguse:GR32_ABCD */, %1:gr32_abcd, 1 /* clobber */, implicit-def early-clobber $df
+    RET 0
+...
diff --git a/llvm/test/CodeGen/X86/physreg-pairs.ll b/llvm/test/CodeGen/X86/physreg-pairs.ll
index 5e1d430311..07ee803709 100644
--- a/llvm/test/CodeGen/X86/physreg-pairs.ll
+++ b/llvm/test/CodeGen/X86/physreg-pairs.ll
@@ -145,8 +145,8 @@ define dso_local i64 @test_ebp(i64 %in) local_unnamed_addr nounwind {
 ; CHECK-LABEL: test_ebp:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pushl %ebp
-; CHECK-NEXT:    movl $19088743, %esp # imm = 0x1234567
 ; CHECK-NEXT:    movl $-1985229329, %ebp # imm = 0x89ABCDEF
+; CHECK-NEXT:    movl $19088743, %esp # imm = 0x1234567
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    movl %ebp, %eax
 ; CHECK-NEXT:    #NO_APP
diff --git a/llvm/test/CodeGen/X86/popcnt.ll b/llvm/test/CodeGen/X86/popcnt.ll
index 5ed14ab6e0..a0879ad930 100644
--- a/llvm/test/CodeGen/X86/popcnt.ll
+++ b/llvm/test/CodeGen/X86/popcnt.ll
@@ -1044,12 +1044,11 @@ define i32 @cnt32_pgso(i32 %x) nounwind readnone !prof !14 {
 ; X86-NEXT:    shrl %ecx
 ; X86-NEXT:    andl $1431655765, %ecx # imm = 0x55555555
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    movl $858993459, %ecx # imm = 0x33333333
-; X86-NEXT:    movl %eax, %edx
-; X86-NEXT:    andl %ecx, %edx
+; X86-NEXT:    movl %eax, %ecx
+; X86-NEXT:    andl $858993459, %ecx # imm = 0x33333333
 ; X86-NEXT:    shrl $2, %eax
-; X86-NEXT:    andl %ecx, %eax
-; X86-NEXT:    addl %edx, %eax
+; X86-NEXT:    andl $858993459, %eax # imm = 0x33333333
+; X86-NEXT:    addl %ecx, %eax
 ; X86-NEXT:    movl %eax, %ecx
 ; X86-NEXT:    shrl $4, %ecx
 ; X86-NEXT:    addl %eax, %ecx
@@ -1064,12 +1063,11 @@ define i32 @cnt32_pgso(i32 %x) nounwind readnone !prof !14 {
 ; X64-NEXT:    shrl %eax
 ; X64-NEXT:    andl $1431655765, %eax # imm = 0x55555555
 ; X64-NEXT:    subl %eax, %edi
-; X64-NEXT:    movl $858993459, %eax # imm = 0x33333333
-; X64-NEXT:    movl %edi, %ecx
-; X64-NEXT:    andl %eax, %ecx
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    andl $858993459, %eax # imm = 0x33333333
 ; X64-NEXT:    shrl $2, %edi
-; X64-NEXT:    andl %eax, %edi
-; X64-NEXT:    addl %ecx, %edi
+; X64-NEXT:    andl $858993459, %edi # imm = 0x33333333
+; X64-NEXT:    addl %eax, %edi
 ; X64-NEXT:    movl %edi, %eax
 ; X64-NEXT:    shrl $4, %eax
 ; X64-NEXT:    addl %edi, %eax
@@ -1094,49 +1092,40 @@ define i32 @cnt32_pgso(i32 %x) nounwind readnone !prof !14 {
 define i64 @cnt64_pgso(i64 %x) nounwind readnone !prof !14 {
 ; X86-NOSSE-LABEL: cnt64_pgso:
 ; X86-NOSSE:       # %bb.0:
-; X86-NOSSE-NEXT:    pushl %ebx
-; X86-NOSSE-NEXT:    pushl %edi
-; X86-NOSSE-NEXT:    pushl %esi
 ; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NOSSE-NEXT:    movl %esi, %ecx
-; X86-NOSSE-NEXT:    shrl %ecx
-; X86-NOSSE-NEXT:    movl $1431655765, %edx # imm = 0x55555555
-; X86-NOSSE-NEXT:    andl %edx, %ecx
-; X86-NOSSE-NEXT:    subl %ecx, %esi
-; X86-NOSSE-NEXT:    movl $858993459, %ecx # imm = 0x33333333
-; X86-NOSSE-NEXT:    movl %esi, %edi
-; X86-NOSSE-NEXT:    andl %ecx, %edi
-; X86-NOSSE-NEXT:    shrl $2, %esi
-; X86-NOSSE-NEXT:    andl %ecx, %esi
-; X86-NOSSE-NEXT:    addl %edi, %esi
-; X86-NOSSE-NEXT:    movl %esi, %ebx
-; X86-NOSSE-NEXT:    shrl $4, %ebx
-; X86-NOSSE-NEXT:    addl %esi, %ebx
-; X86-NOSSE-NEXT:    movl $252645135, %edi # imm = 0xF0F0F0F
-; X86-NOSSE-NEXT:    andl %edi, %ebx
-; X86-NOSSE-NEXT:    imull $16843009, %ebx, %esi # imm = 0x1010101
-; X86-NOSSE-NEXT:    shrl $24, %esi
-; X86-NOSSE-NEXT:    movl %eax, %ebx
-; X86-NOSSE-NEXT:    shrl %ebx
-; X86-NOSSE-NEXT:    andl %edx, %ebx
-; X86-NOSSE-NEXT:    subl %ebx, %eax
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NOSSE-NEXT:    movl %ecx, %edx
+; X86-NOSSE-NEXT:    shrl %edx
+; X86-NOSSE-NEXT:    andl $1431655765, %edx # imm = 0x55555555
+; X86-NOSSE-NEXT:    subl %edx, %ecx
+; X86-NOSSE-NEXT:    movl %ecx, %edx
+; X86-NOSSE-NEXT:    andl $858993459, %edx # imm = 0x33333333
+; X86-NOSSE-NEXT:    shrl $2, %ecx
+; X86-NOSSE-NEXT:    andl $858993459, %ecx # imm = 0x33333333
+; X86-NOSSE-NEXT:    addl %edx, %ecx
+; X86-NOSSE-NEXT:    movl %ecx, %edx
+; X86-NOSSE-NEXT:    shrl $4, %edx
+; X86-NOSSE-NEXT:    addl %ecx, %edx
+; X86-NOSSE-NEXT:    andl $252645135, %edx # imm = 0xF0F0F0F
+; X86-NOSSE-NEXT:    imull $16843009, %edx, %ecx # imm = 0x1010101
+; X86-NOSSE-NEXT:    shrl $24, %ecx
 ; X86-NOSSE-NEXT:    movl %eax, %edx
-; X86-NOSSE-NEXT:    andl %ecx, %edx
+; X86-NOSSE-NEXT:    shrl %edx
+; X86-NOSSE-NEXT:    andl $1431655765, %edx # imm = 0x55555555
+; X86-NOSSE-NEXT:    subl %edx, %eax
+; X86-NOSSE-NEXT:    movl %eax, %edx
+; X86-NOSSE-NEXT:    andl $858993459, %edx # imm = 0x33333333
 ; X86-NOSSE-NEXT:    shrl $2, %eax
-; X86-NOSSE-NEXT:    andl %ecx, %eax
+; X86-NOSSE-NEXT:    andl $858993459, %eax # imm = 0x33333333
 ; X86-NOSSE-NEXT:    addl %edx, %eax
-; X86-NOSSE-NEXT:    movl %eax, %ecx
-; X86-NOSSE-NEXT:    shrl $4, %ecx
-; X86-NOSSE-NEXT:    addl %eax, %ecx
-; X86-NOSSE-NEXT:    andl %edi, %ecx
-; X86-NOSSE-NEXT:    imull $16843009, %ecx, %eax # imm = 0x1010101
+; X86-NOSSE-NEXT:    movl %eax, %edx
+; X86-NOSSE-NEXT:    shrl $4, %edx
+; X86-NOSSE-NEXT:    addl %eax, %edx
+; X86-NOSSE-NEXT:    andl $252645135, %edx # imm = 0xF0F0F0F
+; X86-NOSSE-NEXT:    imull $16843009, %edx, %eax # imm = 0x1010101
 ; X86-NOSSE-NEXT:    shrl $24, %eax
-; X86-NOSSE-NEXT:    addl %esi, %eax
+; X86-NOSSE-NEXT:    addl %ecx, %eax
 ; X86-NOSSE-NEXT:    xorl %edx, %edx
-; X86-NOSSE-NEXT:    popl %esi
-; X86-NOSSE-NEXT:    popl %edi
-; X86-NOSSE-NEXT:    popl %ebx
 ; X86-NOSSE-NEXT:    retl
 ;
 ; X64-LABEL: cnt64_pgso:
@@ -1223,92 +1212,85 @@ define i64 @cnt64_pgso(i64 %x) nounwind readnone !prof !14 {
 define i128 @cnt128_pgso(i128 %x) nounwind readnone !prof !14 {
 ; X86-NOSSE-LABEL: cnt128_pgso:
 ; X86-NOSSE:       # %bb.0:
-; X86-NOSSE-NEXT:    pushl %ebp
 ; X86-NOSSE-NEXT:    pushl %ebx
 ; X86-NOSSE-NEXT:    pushl %edi
 ; X86-NOSSE-NEXT:    pushl %esi
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %ebx
-; X86-NOSSE-NEXT:    movl %ebx, %ecx
-; X86-NOSSE-NEXT:    shrl %ecx
-; X86-NOSSE-NEXT:    movl $1431655765, %edi # imm = 0x55555555
-; X86-NOSSE-NEXT:    andl %edi, %ecx
-; X86-NOSSE-NEXT:    subl %ecx, %ebx
-; X86-NOSSE-NEXT:    movl $858993459, %ecx # imm = 0x33333333
-; X86-NOSSE-NEXT:    movl %ebx, %ebp
-; X86-NOSSE-NEXT:    andl %ecx, %ebp
-; X86-NOSSE-NEXT:    shrl $2, %ebx
-; X86-NOSSE-NEXT:    andl %ecx, %ebx
-; X86-NOSSE-NEXT:    addl %ebp, %ebx
-; X86-NOSSE-NEXT:    movl %ebx, %ebp
-; X86-NOSSE-NEXT:    shrl $4, %ebp
-; X86-NOSSE-NEXT:    addl %ebx, %ebp
-; X86-NOSSE-NEXT:    movl %eax, %ebx
+; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NOSSE-NEXT:    movl %edi, %ebx
 ; X86-NOSSE-NEXT:    shrl %ebx
-; X86-NOSSE-NEXT:    andl %edi, %ebx
-; X86-NOSSE-NEXT:    subl %ebx, %eax
-; X86-NOSSE-NEXT:    movl %eax, %ebx
-; X86-NOSSE-NEXT:    andl %ecx, %ebx
-; X86-NOSSE-NEXT:    shrl $2, %eax
-; X86-NOSSE-NEXT:    andl %ecx, %eax
-; X86-NOSSE-NEXT:    addl %ebx, %eax
-; X86-NOSSE-NEXT:    movl %eax, %edi
-; X86-NOSSE-NEXT:    shrl $4, %edi
-; X86-NOSSE-NEXT:    addl %eax, %edi
-; X86-NOSSE-NEXT:    movl $252645135, %ebx # imm = 0xF0F0F0F
-; X86-NOSSE-NEXT:    andl %ebx, %ebp
-; X86-NOSSE-NEXT:    imull $16843009, %ebp, %eax # imm = 0x1010101
-; X86-NOSSE-NEXT:    shrl $24, %eax
-; X86-NOSSE-NEXT:    andl %ebx, %edi
-; X86-NOSSE-NEXT:    imull $16843009, %edi, %edi # imm = 0x1010101
+; X86-NOSSE-NEXT:    andl $1431655765, %ebx # imm = 0x55555555
+; X86-NOSSE-NEXT:    subl %ebx, %edi
+; X86-NOSSE-NEXT:    movl %edi, %ebx
+; X86-NOSSE-NEXT:    andl $858993459, %ebx # imm = 0x33333333
+; X86-NOSSE-NEXT:    shrl $2, %edi
+; X86-NOSSE-NEXT:    andl $858993459, %edi # imm = 0x33333333
+; X86-NOSSE-NEXT:    addl %ebx, %edi
+; X86-NOSSE-NEXT:    movl %edi, %ebx
+; X86-NOSSE-NEXT:    shrl $4, %ebx
+; X86-NOSSE-NEXT:    addl %edi, %ebx
+; X86-NOSSE-NEXT:    andl $252645135, %ebx # imm = 0xF0F0F0F
+; X86-NOSSE-NEXT:    imull $16843009, %ebx, %edi # imm = 0x1010101
 ; X86-NOSSE-NEXT:    shrl $24, %edi
-; X86-NOSSE-NEXT:    addl %eax, %edi
-; X86-NOSSE-NEXT:    movl %esi, %eax
-; X86-NOSSE-NEXT:    shrl %eax
-; X86-NOSSE-NEXT:    movl $1431655765, %ebp # imm = 0x55555555
-; X86-NOSSE-NEXT:    andl %ebp, %eax
-; X86-NOSSE-NEXT:    subl %eax, %esi
-; X86-NOSSE-NEXT:    movl %esi, %eax
-; X86-NOSSE-NEXT:    andl %ecx, %eax
+; X86-NOSSE-NEXT:    movl %esi, %ebx
+; X86-NOSSE-NEXT:    shrl %ebx
+; X86-NOSSE-NEXT:    andl $1431655765, %ebx # imm = 0x55555555
+; X86-NOSSE-NEXT:    subl %ebx, %esi
+; X86-NOSSE-NEXT:    movl %esi, %ebx
+; X86-NOSSE-NEXT:    andl $858993459, %ebx # imm = 0x33333333
 ; X86-NOSSE-NEXT:    shrl $2, %esi
-; X86-NOSSE-NEXT:    andl %ecx, %esi
-; X86-NOSSE-NEXT:    addl %eax, %esi
-; X86-NOSSE-NEXT:    movl %esi, %ebp
-; X86-NOSSE-NEXT:    shrl $4, %ebp
-; X86-NOSSE-NEXT:    addl %esi, %ebp
-; X86-NOSSE-NEXT:    movl %edx, %eax
-; X86-NOSSE-NEXT:    shrl %eax
-; X86-NOSSE-NEXT:    movl $1431655765, %esi # imm = 0x55555555
-; X86-NOSSE-NEXT:    andl %esi, %eax
-; X86-NOSSE-NEXT:    subl %eax, %edx
-; X86-NOSSE-NEXT:    movl %edx, %eax
-; X86-NOSSE-NEXT:    andl %ecx, %eax
+; X86-NOSSE-NEXT:    andl $858993459, %esi # imm = 0x33333333
+; X86-NOSSE-NEXT:    addl %ebx, %esi
+; X86-NOSSE-NEXT:    movl %esi, %ebx
+; X86-NOSSE-NEXT:    shrl $4, %ebx
+; X86-NOSSE-NEXT:    addl %esi, %ebx
+; X86-NOSSE-NEXT:    andl $252645135, %ebx # imm = 0xF0F0F0F
+; X86-NOSSE-NEXT:    imull $16843009, %ebx, %esi # imm = 0x1010101
+; X86-NOSSE-NEXT:    shrl $24, %esi
+; X86-NOSSE-NEXT:    addl %edi, %esi
+; X86-NOSSE-NEXT:    movl %edx, %edi
+; X86-NOSSE-NEXT:    shrl %edi
+; X86-NOSSE-NEXT:    andl $1431655765, %edi # imm = 0x55555555
+; X86-NOSSE-NEXT:    subl %edi, %edx
+; X86-NOSSE-NEXT:    movl %edx, %edi
+; X86-NOSSE-NEXT:    andl $858993459, %edi # imm = 0x33333333
 ; X86-NOSSE-NEXT:    shrl $2, %edx
-; X86-NOSSE-NEXT:    andl %ecx, %edx
-; X86-NOSSE-NEXT:    addl %eax, %edx
-; X86-NOSSE-NEXT:    movl %edx, %eax
-; X86-NOSSE-NEXT:    shrl $4, %eax
-; X86-NOSSE-NEXT:    addl %edx, %eax
-; X86-NOSSE-NEXT:    andl %ebx, %ebp
-; X86-NOSSE-NEXT:    andl %ebx, %eax
-; X86-NOSSE-NEXT:    imull $16843009, %ebp, %ecx # imm = 0x1010101
-; X86-NOSSE-NEXT:    shrl $24, %ecx
-; X86-NOSSE-NEXT:    imull $16843009, %eax, %edx # imm = 0x1010101
-; X86-NOSSE-NEXT:    shrl $24, %edx
-; X86-NOSSE-NEXT:    addl %ecx, %edx
-; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    andl $858993459, %edx # imm = 0x33333333
 ; X86-NOSSE-NEXT:    addl %edi, %edx
-; X86-NOSSE-NEXT:    xorl %ecx, %ecx
-; X86-NOSSE-NEXT:    movl %ecx, 12(%eax)
-; X86-NOSSE-NEXT:    movl %ecx, 8(%eax)
-; X86-NOSSE-NEXT:    movl %ecx, 4(%eax)
-; X86-NOSSE-NEXT:    movl %edx, (%eax)
+; X86-NOSSE-NEXT:    movl %edx, %edi
+; X86-NOSSE-NEXT:    shrl $4, %edi
+; X86-NOSSE-NEXT:    addl %edx, %edi
+; X86-NOSSE-NEXT:    andl $252645135, %edi # imm = 0xF0F0F0F
+; X86-NOSSE-NEXT:    imull $16843009, %edi, %edx # imm = 0x1010101
+; X86-NOSSE-NEXT:    shrl $24, %edx
+; X86-NOSSE-NEXT:    movl %ecx, %edi
+; X86-NOSSE-NEXT:    shrl %edi
+; X86-NOSSE-NEXT:    andl $1431655765, %edi # imm = 0x55555555
+; X86-NOSSE-NEXT:    subl %edi, %ecx
+; X86-NOSSE-NEXT:    movl %ecx, %edi
+; X86-NOSSE-NEXT:    andl $858993459, %edi # imm = 0x33333333
+; X86-NOSSE-NEXT:    shrl $2, %ecx
+; X86-NOSSE-NEXT:    andl $858993459, %ecx # imm = 0x33333333
+; X86-NOSSE-NEXT:    addl %edi, %ecx
+; X86-NOSSE-NEXT:    movl %ecx, %edi
+; X86-NOSSE-NEXT:    shrl $4, %edi
+; X86-NOSSE-NEXT:    addl %ecx, %edi
+; X86-NOSSE-NEXT:    andl $252645135, %edi # imm = 0xF0F0F0F
+; X86-NOSSE-NEXT:    imull $16843009, %edi, %ecx # imm = 0x1010101
+; X86-NOSSE-NEXT:    shrl $24, %ecx
+; X86-NOSSE-NEXT:    addl %edx, %ecx
+; X86-NOSSE-NEXT:    addl %esi, %ecx
+; X86-NOSSE-NEXT:    xorl %edx, %edx
+; X86-NOSSE-NEXT:    movl %edx, 12(%eax)
+; X86-NOSSE-NEXT:    movl %edx, 8(%eax)
+; X86-NOSSE-NEXT:    movl %edx, 4(%eax)
+; X86-NOSSE-NEXT:    movl %ecx, (%eax)
 ; X86-NOSSE-NEXT:    popl %esi
 ; X86-NOSSE-NEXT:    popl %edi
 ; X86-NOSSE-NEXT:    popl %ebx
-; X86-NOSSE-NEXT:    popl %ebp
 ; X86-NOSSE-NEXT:    retl $4
 ;
 ; X64-LABEL: cnt128_pgso:
diff --git a/llvm/test/CodeGen/X86/pr69080.ll b/llvm/test/CodeGen/X86/pr69080.ll
new file mode 100644
index 0000000000..1b27adcb1a
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr69080.ll
@@ -0,0 +1,38 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s --check-prefixes=SSE
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512f | FileCheck %s --check-prefixes=AVX
+
+define { <4 x i1>, <4 x i1> } @uaddo(<4 x i1> %a) {
+; SSE-LABEL: uaddo:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps %xmm0, %xmm1
+; SSE-NEXT:    xorps %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: uaddo:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovaps %xmm0, %xmm1
+; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %f = call { <4 x i1>, <4 x i1> } @llvm.uadd.with.overflow.v4i1(<4 x i1> %a, <4 x i1> %a)
+  ret { <4 x i1>, <4 x i1> } %f
+}
+declare { <4 x i1>, <4 x i1> } @llvm.uadd.with.overflow.v4i1(<4 x i1>, <4 x i1>)
+
+define { <4 x i1>, <4 x i1> } @saddo(<4 x i1> %a) {
+; SSE-LABEL: saddo:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps %xmm0, %xmm1
+; SSE-NEXT:    xorps %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: saddo:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovaps %xmm0, %xmm1
+; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %f = call { <4 x i1>, <4 x i1> } @llvm.sadd.with.overflow.v4i1(<4 x i1> %a, <4 x i1> %a)
+  ret { <4 x i1>, <4 x i1> } %f
+}
+declare { <4 x i1>, <4 x i1> } @llvm.sadd.with.overflow.v4i1(<4 x i1>, <4 x i1>)
diff --git a/llvm/test/CodeGen/X86/prologue-epilogue-remarks.mir b/llvm/test/CodeGen/X86/prologue-epilogue-remarks.mir
index f0c6121bd9..39d06c2909 100644
--- a/llvm/test/CodeGen/X86/prologue-epilogue-remarks.mir
+++ b/llvm/test/CodeGen/X86/prologue-epilogue-remarks.mir
@@ -17,7 +17,9 @@ stack:
 # CHECK-NEXT: Function:        fun0
 # CHECK-NEXT: Args:
 # CHECK-NEXT:   - NumStackBytes:   '8'
-# CHECK-NEXT:   - String:          ' stack bytes in function'
+# CHECK-NEXT:   - String:          ' stack bytes in function '''
+# CHECK-NEXT:   - Function: fun0
+# CHECK-NEXT:   - String: ''''
 # CHECK-NEXT: ...
 
 constants:
@@ -36,7 +38,9 @@ stack:
 # CHECK-NEXT: Function:        fun1
 # CHECK-NEXT: Args:
 # CHECK-NEXT:   - NumStackBytes:   '20'
-# CHECK-NEXT:   - String:          ' stack bytes in function'
+# CHECK-NEXT:   - String:          ' stack bytes in function '''
+# CHECK-NEXT:   - Function: fun1
+# CHECK-NEXT:   - String: ''''
 # CHECK-NEXT: ...
 constants:
 body:             |
@@ -54,7 +58,9 @@ stack:
 # CHECK-NEXT: Function:        fun2
 # CHECK-NEXT: Args:
 # CHECK-NEXT:   - NumStackBytes:   '1024'
-# CHECK-NEXT:   - String:          ' stack bytes in function'
+# CHECK-NEXT:   - String:          ' stack bytes in function '''
+# CHECK-NEXT:   - Function: fun2
+# CHECK-NEXT:   - String: ''''
 # CHECK-NEXT: ...
 constants:
 body:             |
diff --git a/llvm/test/CodeGen/X86/ragreedy-hoist-spill.ll b/llvm/test/CodeGen/X86/ragreedy-hoist-spill.ll
index e9448a800f..c0bfb71e18 100644
--- a/llvm/test/CodeGen/X86/ragreedy-hoist-spill.ll
+++ b/llvm/test/CodeGen/X86/ragreedy-hoist-spill.ll
@@ -81,8 +81,8 @@ define ptr @SyFgets(ptr %line, i64 %length, i64 %fid) {
 ; CHECK-NEXT:    imulq $1040, %rdx, %rax ## imm = 0x410
 ; CHECK-NEXT:    movq _syBuf@GOTPCREL(%rip), %rcx
 ; CHECK-NEXT:    leaq 8(%rcx,%rax), %rdx
-; CHECK-NEXT:    movl $1, %r13d
 ; CHECK-NEXT:    movq _syCTRO@GOTPCREL(%rip), %rax
+; CHECK-NEXT:    movl $1, %r13d
 ; CHECK-NEXT:    movb $1, %cl
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  LBB0_9: ## %do.body
diff --git a/llvm/test/CodeGen/X86/remat-phys-dead.ll b/llvm/test/CodeGen/X86/remat-phys-dead.ll
index 09f2e4320b..9a0a219869 100644
--- a/llvm/test/CodeGen/X86/remat-phys-dead.ll
+++ b/llvm/test/CodeGen/X86/remat-phys-dead.ll
@@ -18,6 +18,6 @@ define i8 @test_remat() {
 define i32 @test_remat32() {
   ret i32 0
 ; CHECK: REGISTER COALESCER
-; CHECK: Remat: $eax = MOV32r0 implicit-def dead $eflags
+; CHECK: $eax = MOV32r0 implicit-def dead $eflags
 }
 
diff --git a/llvm/test/CodeGen/X86/scheduler-backtracking.ll b/llvm/test/CodeGen/X86/scheduler-backtracking.ll
index 0b3ef70d2b..a9f3e8b22f 100644
--- a/llvm/test/CodeGen/X86/scheduler-backtracking.ll
+++ b/llvm/test/CodeGen/X86/scheduler-backtracking.ll
@@ -218,6 +218,8 @@ define i256 @test2(i256 %a) nounwind {
 ; ILP-LABEL: test2:
 ; ILP:       # %bb.0:
 ; ILP-NEXT:    movq %rdi, %rax
+; ILP-NEXT:    xorps %xmm0, %xmm0
+; ILP-NEXT:    movaps %xmm0, 16(%rdi)
 ; ILP-NEXT:    xorl %edi, %edi
 ; ILP-NEXT:    movq %rsi, %r11
 ; ILP-NEXT:    negq %r11
@@ -250,14 +252,14 @@ define i256 @test2(i256 %a) nounwind {
 ; ILP-NEXT:    orq %rdi, %r9
 ; ILP-NEXT:    cmovneq %rcx, %r8
 ; ILP-NEXT:    movq %r8, (%rax)
-; ILP-NEXT:    movq $0, 24(%rax)
-; ILP-NEXT:    movq $0, 16(%rax)
 ; ILP-NEXT:    movq $0, 8(%rax)
 ; ILP-NEXT:    retq
 ;
 ; HYBRID-LABEL: test2:
 ; HYBRID:       # %bb.0:
 ; HYBRID-NEXT:    movq %rdi, %rax
+; HYBRID-NEXT:    xorps %xmm0, %xmm0
+; HYBRID-NEXT:    movaps %xmm0, 16(%rdi)
 ; HYBRID-NEXT:    xorl %edi, %edi
 ; HYBRID-NEXT:    movq %rsi, %r11
 ; HYBRID-NEXT:    negq %r11
@@ -290,14 +292,14 @@ define i256 @test2(i256 %a) nounwind {
 ; HYBRID-NEXT:    orq %rdi, %r9
 ; HYBRID-NEXT:    cmovneq %rcx, %r8
 ; HYBRID-NEXT:    movq %r8, (%rax)
-; HYBRID-NEXT:    movq $0, 24(%rax)
-; HYBRID-NEXT:    movq $0, 16(%rax)
 ; HYBRID-NEXT:    movq $0, 8(%rax)
 ; HYBRID-NEXT:    retq
 ;
 ; BURR-LABEL: test2:
 ; BURR:       # %bb.0:
 ; BURR-NEXT:    movq %rdi, %rax
+; BURR-NEXT:    xorps %xmm0, %xmm0
+; BURR-NEXT:    movaps %xmm0, 16(%rdi)
 ; BURR-NEXT:    xorl %edi, %edi
 ; BURR-NEXT:    movq %rsi, %r11
 ; BURR-NEXT:    negq %r11
@@ -330,8 +332,6 @@ define i256 @test2(i256 %a) nounwind {
 ; BURR-NEXT:    orq %rdi, %r9
 ; BURR-NEXT:    cmovneq %rcx, %r8
 ; BURR-NEXT:    movq %r8, (%rax)
-; BURR-NEXT:    movq $0, 24(%rax)
-; BURR-NEXT:    movq $0, 16(%rax)
 ; BURR-NEXT:    movq $0, 8(%rax)
 ; BURR-NEXT:    retq
 ;
@@ -369,15 +369,17 @@ define i256 @test2(i256 %a) nounwind {
 ; SRC-NEXT:    subq $-128, %r8
 ; SRC-NEXT:    orq %r9, %rdi
 ; SRC-NEXT:    cmovneq %rdx, %r8
+; SRC-NEXT:    xorps %xmm0, %xmm0
+; SRC-NEXT:    movaps %xmm0, 16(%rax)
 ; SRC-NEXT:    movq %r8, (%rax)
-; SRC-NEXT:    movq $0, 24(%rax)
-; SRC-NEXT:    movq $0, 16(%rax)
 ; SRC-NEXT:    movq $0, 8(%rax)
 ; SRC-NEXT:    retq
 ;
 ; LIN-LABEL: test2:
 ; LIN:       # %bb.0:
 ; LIN-NEXT:    movq %rdi, %rax
+; LIN-NEXT:    xorps %xmm0, %xmm0
+; LIN-NEXT:    movaps %xmm0, 16(%rdi)
 ; LIN-NEXT:    movq %rsi, %rdi
 ; LIN-NEXT:    negq %rdi
 ; LIN-NEXT:    andq %rsi, %rdi
@@ -411,8 +413,6 @@ define i256 @test2(i256 %a) nounwind {
 ; LIN-NEXT:    cmoveq %rdx, %r8
 ; LIN-NEXT:    movq %r8, (%rax)
 ; LIN-NEXT:    movq $0, 8(%rax)
-; LIN-NEXT:    movq $0, 16(%rax)
-; LIN-NEXT:    movq $0, 24(%rax)
 ; LIN-NEXT:    retq
   %b = sub i256 0, %a
   %c = and i256 %b, %a
@@ -425,6 +425,8 @@ define i256 @test3(i256 %n) nounwind {
 ; ILP:       # %bb.0:
 ; ILP-NEXT:    pushq %rbx
 ; ILP-NEXT:    movq %rdi, %rax
+; ILP-NEXT:    xorps %xmm0, %xmm0
+; ILP-NEXT:    movaps %xmm0, 16(%rdi)
 ; ILP-NEXT:    xorl %r9d, %r9d
 ; ILP-NEXT:    movq %rsi, %rdi
 ; ILP-NEXT:    negq %rdi
@@ -439,13 +441,13 @@ define i256 @test3(i256 %n) nounwind {
 ; ILP-NEXT:    notq %rdx
 ; ILP-NEXT:    andq %r10, %rdx
 ; ILP-NEXT:    bsrq %rdx, %r9
-; ILP-NEXT:    notq %rsi
 ; ILP-NEXT:    xorq $63, %rbx
 ; ILP-NEXT:    notq %rcx
 ; ILP-NEXT:    andq %r11, %rcx
 ; ILP-NEXT:    bsrq %rcx, %r10
 ; ILP-NEXT:    xorq $63, %r10
 ; ILP-NEXT:    addq $64, %r10
+; ILP-NEXT:    notq %rsi
 ; ILP-NEXT:    testq %r8, %r8
 ; ILP-NEXT:    cmovneq %rbx, %r10
 ; ILP-NEXT:    xorq $63, %r9
@@ -461,8 +463,6 @@ define i256 @test3(i256 %n) nounwind {
 ; ILP-NEXT:    orq %r8, %rcx
 ; ILP-NEXT:    cmovneq %r10, %rsi
 ; ILP-NEXT:    movq %rsi, (%rax)
-; ILP-NEXT:    movq $0, 24(%rax)
-; ILP-NEXT:    movq $0, 16(%rax)
 ; ILP-NEXT:    movq $0, 8(%rax)
 ; ILP-NEXT:    popq %rbx
 ; ILP-NEXT:    retq
@@ -471,6 +471,8 @@ define i256 @test3(i256 %n) nounwind {
 ; HYBRID:       # %bb.0:
 ; HYBRID-NEXT:    pushq %rbx
 ; HYBRID-NEXT:    movq %rdi, %rax
+; HYBRID-NEXT:    xorps %xmm0, %xmm0
+; HYBRID-NEXT:    movaps %xmm0, 16(%rdi)
 ; HYBRID-NEXT:    xorl %r9d, %r9d
 ; HYBRID-NEXT:    movq %rsi, %rdi
 ; HYBRID-NEXT:    negq %rdi
@@ -507,8 +509,6 @@ define i256 @test3(i256 %n) nounwind {
 ; HYBRID-NEXT:    orq %r8, %rcx
 ; HYBRID-NEXT:    cmovneq %r9, %rsi
 ; HYBRID-NEXT:    movq %rsi, (%rax)
-; HYBRID-NEXT:    movq $0, 24(%rax)
-; HYBRID-NEXT:    movq $0, 16(%rax)
 ; HYBRID-NEXT:    movq $0, 8(%rax)
 ; HYBRID-NEXT:    popq %rbx
 ; HYBRID-NEXT:    retq
@@ -517,6 +517,8 @@ define i256 @test3(i256 %n) nounwind {
 ; BURR:       # %bb.0:
 ; BURR-NEXT:    pushq %rbx
 ; BURR-NEXT:    movq %rdi, %rax
+; BURR-NEXT:    xorps %xmm0, %xmm0
+; BURR-NEXT:    movaps %xmm0, 16(%rdi)
 ; BURR-NEXT:    xorl %r9d, %r9d
 ; BURR-NEXT:    movq %rsi, %rdi
 ; BURR-NEXT:    negq %rdi
@@ -553,8 +555,6 @@ define i256 @test3(i256 %n) nounwind {
 ; BURR-NEXT:    orq %r8, %rcx
 ; BURR-NEXT:    cmovneq %r9, %rsi
 ; BURR-NEXT:    movq %rsi, (%rax)
-; BURR-NEXT:    movq $0, 24(%rax)
-; BURR-NEXT:    movq $0, 16(%rax)
 ; BURR-NEXT:    movq $0, 8(%rax)
 ; BURR-NEXT:    popq %rbx
 ; BURR-NEXT:    retq
@@ -597,15 +597,17 @@ define i256 @test3(i256 %n) nounwind {
 ; SRC-NEXT:    subq $-128, %r10
 ; SRC-NEXT:    orq %rcx, %r8
 ; SRC-NEXT:    cmovneq %r9, %r10
+; SRC-NEXT:    xorps %xmm0, %xmm0
+; SRC-NEXT:    movaps %xmm0, 16(%rax)
 ; SRC-NEXT:    movq %r10, (%rax)
-; SRC-NEXT:    movq $0, 24(%rax)
-; SRC-NEXT:    movq $0, 16(%rax)
 ; SRC-NEXT:    movq $0, 8(%rax)
 ; SRC-NEXT:    retq
 ;
 ; LIN-LABEL: test3:
 ; LIN:       # %bb.0:
 ; LIN-NEXT:    movq %rdi, %rax
+; LIN-NEXT:    xorps %xmm0, %xmm0
+; LIN-NEXT:    movaps %xmm0, 16(%rdi)
 ; LIN-NEXT:    movq %rsi, %rdi
 ; LIN-NEXT:    negq %rdi
 ; LIN-NEXT:    notq %rsi
@@ -643,8 +645,6 @@ define i256 @test3(i256 %n) nounwind {
 ; LIN-NEXT:    cmoveq %rsi, %rdi
 ; LIN-NEXT:    movq %rdi, (%rax)
 ; LIN-NEXT:    movq $0, 8(%rax)
-; LIN-NEXT:    movq $0, 16(%rax)
-; LIN-NEXT:    movq $0, 24(%rax)
 ; LIN-NEXT:    retq
   %m = sub i256 -1, %n
   %x = sub i256 0, %n
diff --git a/llvm/test/CodeGen/X86/sdiv_fix.ll b/llvm/test/CodeGen/X86/sdiv_fix.ll
index 336aa216d1..ce0b212aa4 100644
--- a/llvm/test/CodeGen/X86/sdiv_fix.ll
+++ b/llvm/test/CodeGen/X86/sdiv_fix.ll
@@ -306,8 +306,8 @@ define i64 @func5(i64 %x, i64 %y) nounwind {
 ; X86-NEXT:    pushl %ebx
 ; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    andl $-8, %esp
-; X86-NEXT:    subl $72, %esp
+; X86-NEXT:    andl $-16, %esp
+; X86-NEXT:    subl $80, %esp
 ; X86-NEXT:    movl 8(%ebp), %ecx
 ; X86-NEXT:    movl 12(%ebp), %eax
 ; X86-NEXT:    movl 20(%ebp), %edx
diff --git a/llvm/test/CodeGen/X86/sdiv_fix_sat.ll b/llvm/test/CodeGen/X86/sdiv_fix_sat.ll
index 371484e015..31297a06f8 100644
--- a/llvm/test/CodeGen/X86/sdiv_fix_sat.ll
+++ b/llvm/test/CodeGen/X86/sdiv_fix_sat.ll
@@ -369,8 +369,8 @@ define i64 @func5(i64 %x, i64 %y) nounwind {
 ; X86-NEXT:    pushl %ebx
 ; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    andl $-8, %esp
-; X86-NEXT:    subl $88, %esp
+; X86-NEXT:    andl $-16, %esp
+; X86-NEXT:    subl $96, %esp
 ; X86-NEXT:    movl 8(%ebp), %ecx
 ; X86-NEXT:    movl 12(%ebp), %eax
 ; X86-NEXT:    movl 20(%ebp), %esi
@@ -805,8 +805,8 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X86-NEXT:    pushl %ebx
 ; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    andl $-8, %esp
-; X86-NEXT:    subl $192, %esp
+; X86-NEXT:    andl $-16, %esp
+; X86-NEXT:    subl $208, %esp
 ; X86-NEXT:    movl 36(%ebp), %esi
 ; X86-NEXT:    movl 16(%ebp), %ebx
 ; X86-NEXT:    movl 32(%ebp), %eax
diff --git a/llvm/test/CodeGen/X86/select_const_i128.ll b/llvm/test/CodeGen/X86/select_const_i128.ll
index af38bd6ce9..503cf13ec4 100644
--- a/llvm/test/CodeGen/X86/select_const_i128.ll
+++ b/llvm/test/CodeGen/X86/select_const_i128.ll
@@ -9,10 +9,9 @@ define i128 @select_eq_i128(ptr %a) {
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    ptest %xmm0, %xmm0
 ; CHECK-NEXT:    setne %al
-; CHECK-NEXT:    xorl %ecx, %ecx
 ; CHECK-NEXT:    addq $-1, %rax
 ; CHECK-NEXT:    movabsq $9223372036854775807, %rdx # imm = 0x7FFFFFFFFFFFFFFF
-; CHECK-NEXT:    adcq %rcx, %rdx
+; CHECK-NEXT:    adcq $0, %rdx
 ; CHECK-NEXT:    retq
   %1 = load i128, ptr %a, align 16
   %cmp = icmp eq i128 %1, 1
diff --git a/llvm/test/CodeGen/X86/setcc-wide-types.ll b/llvm/test/CodeGen/X86/setcc-wide-types.ll
index 61254d5e5c..5aa266db65 100644
--- a/llvm/test/CodeGen/X86/setcc-wide-types.ll
+++ b/llvm/test/CodeGen/X86/setcc-wide-types.ll
@@ -774,13 +774,11 @@ define i1 @ne_v4i256(<4 x i256> %a0) {
 define i32 @ne_i128_pair(ptr %a, ptr %b) {
 ; SSE2-LABEL: ne_i128_pair:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqu (%rdi), %xmm0
-; SSE2-NEXT:    movdqu 16(%rdi), %xmm1
-; SSE2-NEXT:    movdqu (%rsi), %xmm2
-; SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
-; SSE2-NEXT:    movdqu 16(%rsi), %xmm0
-; SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
-; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    movdqa (%rdi), %xmm0
+; SSE2-NEXT:    movdqa 16(%rdi), %xmm1
+; SSE2-NEXT:    pcmpeqb 16(%rsi), %xmm1
+; SSE2-NEXT:    pcmpeqb (%rsi), %xmm0
+; SSE2-NEXT:    pand %xmm1, %xmm0
 ; SSE2-NEXT:    pmovmskb %xmm0, %ecx
 ; SSE2-NEXT:    xorl %eax, %eax
 ; SSE2-NEXT:    cmpl $65535, %ecx # imm = 0xFFFF
@@ -789,13 +787,11 @@ define i32 @ne_i128_pair(ptr %a, ptr %b) {
 ;
 ; SSE41-LABEL: ne_i128_pair:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqu (%rdi), %xmm0
-; SSE41-NEXT:    movdqu 16(%rdi), %xmm1
-; SSE41-NEXT:    movdqu (%rsi), %xmm2
-; SSE41-NEXT:    pxor %xmm0, %xmm2
-; SSE41-NEXT:    movdqu 16(%rsi), %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    movdqa (%rdi), %xmm0
+; SSE41-NEXT:    movdqa 16(%rdi), %xmm1
+; SSE41-NEXT:    pxor 16(%rsi), %xmm1
+; SSE41-NEXT:    pxor (%rsi), %xmm0
+; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    xorl %eax, %eax
 ; SSE41-NEXT:    ptest %xmm0, %xmm0
 ; SSE41-NEXT:    setne %al
@@ -803,8 +799,8 @@ define i32 @ne_i128_pair(ptr %a, ptr %b) {
 ;
 ; AVXANY-LABEL: ne_i128_pair:
 ; AVXANY:       # %bb.0:
-; AVXANY-NEXT:    vmovdqu (%rdi), %xmm0
-; AVXANY-NEXT:    vmovdqu 16(%rdi), %xmm1
+; AVXANY-NEXT:    vmovdqa (%rdi), %xmm0
+; AVXANY-NEXT:    vmovdqa 16(%rdi), %xmm1
 ; AVXANY-NEXT:    vpxor 16(%rsi), %xmm1, %xmm1
 ; AVXANY-NEXT:    vpxor (%rsi), %xmm0, %xmm0
 ; AVXANY-NEXT:    vpor %xmm1, %xmm0, %xmm0
@@ -832,13 +828,11 @@ define i32 @ne_i128_pair(ptr %a, ptr %b) {
 define i32 @eq_i128_pair(ptr %a, ptr %b) {
 ; SSE2-LABEL: eq_i128_pair:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqu (%rdi), %xmm0
-; SSE2-NEXT:    movdqu 16(%rdi), %xmm1
-; SSE2-NEXT:    movdqu (%rsi), %xmm2
-; SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
-; SSE2-NEXT:    movdqu 16(%rsi), %xmm0
-; SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
-; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    movdqa (%rdi), %xmm0
+; SSE2-NEXT:    movdqa 16(%rdi), %xmm1
+; SSE2-NEXT:    pcmpeqb 16(%rsi), %xmm1
+; SSE2-NEXT:    pcmpeqb (%rsi), %xmm0
+; SSE2-NEXT:    pand %xmm1, %xmm0
 ; SSE2-NEXT:    pmovmskb %xmm0, %ecx
 ; SSE2-NEXT:    xorl %eax, %eax
 ; SSE2-NEXT:    cmpl $65535, %ecx # imm = 0xFFFF
@@ -847,13 +841,11 @@ define i32 @eq_i128_pair(ptr %a, ptr %b) {
 ;
 ; SSE41-LABEL: eq_i128_pair:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqu (%rdi), %xmm0
-; SSE41-NEXT:    movdqu 16(%rdi), %xmm1
-; SSE41-NEXT:    movdqu (%rsi), %xmm2
-; SSE41-NEXT:    pxor %xmm0, %xmm2
-; SSE41-NEXT:    movdqu 16(%rsi), %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    movdqa (%rdi), %xmm0
+; SSE41-NEXT:    movdqa 16(%rdi), %xmm1
+; SSE41-NEXT:    pxor 16(%rsi), %xmm1
+; SSE41-NEXT:    pxor (%rsi), %xmm0
+; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    xorl %eax, %eax
 ; SSE41-NEXT:    ptest %xmm0, %xmm0
 ; SSE41-NEXT:    sete %al
@@ -861,8 +853,8 @@ define i32 @eq_i128_pair(ptr %a, ptr %b) {
 ;
 ; AVXANY-LABEL: eq_i128_pair:
 ; AVXANY:       # %bb.0:
-; AVXANY-NEXT:    vmovdqu (%rdi), %xmm0
-; AVXANY-NEXT:    vmovdqu 16(%rdi), %xmm1
+; AVXANY-NEXT:    vmovdqa (%rdi), %xmm0
+; AVXANY-NEXT:    vmovdqa 16(%rdi), %xmm1
 ; AVXANY-NEXT:    vpxor 16(%rsi), %xmm1, %xmm1
 ; AVXANY-NEXT:    vpxor (%rsi), %xmm0, %xmm0
 ; AVXANY-NEXT:    vpor %xmm1, %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/shrink_vmul.ll b/llvm/test/CodeGen/X86/shrink_vmul.ll
index 524ecf2aec..3d80c1554b 100644
--- a/llvm/test/CodeGen/X86/shrink_vmul.ll
+++ b/llvm/test/CodeGen/X86/shrink_vmul.ll
@@ -1986,29 +1986,29 @@ define void @PR34947(ptr %p0, ptr %p1) nounwind {
 ; X86-SSE-NEXT:    movdqa (%eax), %xmm3
 ; X86-SSE-NEXT:    movdqa (%ecx), %xmm0
 ; X86-SSE-NEXT:    movdqa 16(%ecx), %xmm1
-; X86-SSE-NEXT:    pxor %xmm4, %xmm4
+; X86-SSE-NEXT:    pxor %xmm5, %xmm5
 ; X86-SSE-NEXT:    movdqa %xmm3, %xmm2
 ; X86-SSE-NEXT:    pextrw $7, %xmm3, %eax
 ; X86-SSE-NEXT:    pextrw $4, %xmm3, %edi
 ; X86-SSE-NEXT:    pextrw $0, %xmm3, %ebp
 ; X86-SSE-NEXT:    pextrw $1, %xmm3, %esi
 ; X86-SSE-NEXT:    pextrw $3, %xmm3, %ebx
-; X86-SSE-NEXT:    movdqa %xmm3, %xmm5
-; X86-SSE-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
-; X86-SSE-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
+; X86-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X86-SSE-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
+; X86-SSE-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm5[4],xmm2[5],xmm5[5],xmm2[6],xmm5[6],xmm2[7],xmm5[7]
 ; X86-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[3,3,3,3]
 ; X86-SSE-NEXT:    movd %xmm3, %ecx
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl %ecx
 ; X86-SSE-NEXT:    movd %edx, %xmm3
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
-; X86-SSE-NEXT:    movd %xmm4, %eax
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
-; X86-SSE-NEXT:    movd %xmm4, %ecx
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[2,3,2,3]
+; X86-SSE-NEXT:    movd %xmm5, %eax
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
+; X86-SSE-NEXT:    movd %xmm5, %ecx
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl %ecx
-; X86-SSE-NEXT:    movd %edx, %xmm4
-; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; X86-SSE-NEXT:    movd %edx, %xmm5
+; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
 ; X86-SSE-NEXT:    movl %edi, %eax
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %edi
@@ -2022,7 +2022,7 @@ define void @PR34947(ptr %p0, ptr %p1) nounwind {
 ; X86-SSE-NEXT:    divl %ecx
 ; X86-SSE-NEXT:    movd %edx, %xmm1
 ; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
-; X86-SSE-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm4[0]
+; X86-SSE-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
 ; X86-SSE-NEXT:    movl %ebp, %eax
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl (%edi)
@@ -2040,7 +2040,7 @@ define void @PR34947(ptr %p0, ptr %p1) nounwind {
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl %ecx
 ; X86-SSE-NEXT:    movd %edx, %xmm2
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[2,3,2,3]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]
 ; X86-SSE-NEXT:    movd %xmm4, %eax
 ; X86-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; X86-SSE-NEXT:    movd %xmm0, %ecx
@@ -2207,29 +2207,29 @@ define void @PR34947(ptr %p0, ptr %p1) nounwind {
 ; X64-SSE-NEXT:    movdqa (%rdi), %xmm3
 ; X64-SSE-NEXT:    movdqa (%rsi), %xmm0
 ; X64-SSE-NEXT:    movdqa 16(%rsi), %xmm1
-; X64-SSE-NEXT:    pxor %xmm4, %xmm4
+; X64-SSE-NEXT:    pxor %xmm5, %xmm5
 ; X64-SSE-NEXT:    movdqa %xmm3, %xmm2
 ; X64-SSE-NEXT:    pextrw $7, %xmm3, %eax
 ; X64-SSE-NEXT:    pextrw $4, %xmm3, %r8d
 ; X64-SSE-NEXT:    pextrw $0, %xmm3, %r10d
 ; X64-SSE-NEXT:    pextrw $1, %xmm3, %edi
 ; X64-SSE-NEXT:    pextrw $3, %xmm3, %r9d
-; X64-SSE-NEXT:    movdqa %xmm3, %xmm5
-; X64-SSE-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
-; X64-SSE-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
+; X64-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X64-SSE-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
+; X64-SSE-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm5[4],xmm2[5],xmm5[5],xmm2[6],xmm5[6],xmm2[7],xmm5[7]
 ; X64-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[3,3,3,3]
 ; X64-SSE-NEXT:    movd %xmm3, %r11d
 ; X64-SSE-NEXT:    xorl %edx, %edx
 ; X64-SSE-NEXT:    divl %r11d
 ; X64-SSE-NEXT:    movd %edx, %xmm3
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
-; X64-SSE-NEXT:    movd %xmm4, %eax
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
-; X64-SSE-NEXT:    movd %xmm4, %r11d
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[2,3,2,3]
+; X64-SSE-NEXT:    movd %xmm5, %eax
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
+; X64-SSE-NEXT:    movd %xmm5, %r11d
 ; X64-SSE-NEXT:    xorl %edx, %edx
 ; X64-SSE-NEXT:    divl %r11d
-; X64-SSE-NEXT:    movd %edx, %xmm4
-; X64-SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; X64-SSE-NEXT:    movd %edx, %xmm5
+; X64-SSE-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
 ; X64-SSE-NEXT:    movl %r8d, %eax
 ; X64-SSE-NEXT:    xorl %edx, %edx
 ; X64-SSE-NEXT:    divl 16(%rsi)
@@ -2242,7 +2242,7 @@ define void @PR34947(ptr %p0, ptr %p1) nounwind {
 ; X64-SSE-NEXT:    divl %r8d
 ; X64-SSE-NEXT:    movd %edx, %xmm1
 ; X64-SSE-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
-; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm4[0]
+; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
 ; X64-SSE-NEXT:    movl %r10d, %eax
 ; X64-SSE-NEXT:    xorl %edx, %edx
 ; X64-SSE-NEXT:    divl (%rsi)
@@ -2260,7 +2260,7 @@ define void @PR34947(ptr %p0, ptr %p1) nounwind {
 ; X64-SSE-NEXT:    xorl %edx, %edx
 ; X64-SSE-NEXT:    divl %edi
 ; X64-SSE-NEXT:    movd %edx, %xmm2
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[2,3,2,3]
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]
 ; X64-SSE-NEXT:    movd %xmm4, %eax
 ; X64-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; X64-SSE-NEXT:    movd %xmm0, %edi
diff --git a/llvm/test/CodeGen/X86/smul-with-overflow.ll b/llvm/test/CodeGen/X86/smul-with-overflow.ll
index 0e17af441d..da0e3fdc1a 100644
--- a/llvm/test/CodeGen/X86/smul-with-overflow.ll
+++ b/llvm/test/CodeGen/X86/smul-with-overflow.ll
@@ -804,7 +804,7 @@ define { i129, i1 } @smul_ovf(i129 %x, i129 %y) nounwind {
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
 ; X86-NEXT:    movl %edx, 12(%eax)
 ; X86-NEXT:    movb %cl, 16(%eax)
-; X86-NEXT:    setne 20(%eax)
+; X86-NEXT:    setne 32(%eax)
 ; X86-NEXT:    addl $188, %esp
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    popl %edi
@@ -990,7 +990,7 @@ define { i129, i1 } @smul_ovf(i129 %x, i129 %y) nounwind {
 ; X64-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
 ; X64-NEXT:    movq %rcx, (%rax)
 ; X64-NEXT:    movb %sil, 16(%rax)
-; X64-NEXT:    setne 24(%rax)
+; X64-NEXT:    setne 32(%rax)
 ; X64-NEXT:    popq %rbx
 ; X64-NEXT:    popq %r12
 ; X64-NEXT:    popq %r13
diff --git a/llvm/test/CodeGen/X86/speculative-load-hardening-call-and-ret.ll b/llvm/test/CodeGen/X86/speculative-load-hardening-call-and-ret.ll
index 3f2226aec2..064812323d 100644
--- a/llvm/test/CodeGen/X86/speculative-load-hardening-call-and-ret.ll
+++ b/llvm/test/CodeGen/X86/speculative-load-hardening-call-and-ret.ll
@@ -283,15 +283,14 @@ define i32 @test_call_setjmp(ptr%ptr) nounwind {
 ; X64-NOPIC-NEXT:    pushq %rbp
 ; X64-NOPIC-NEXT:    pushq %r15
 ; X64-NOPIC-NEXT:    pushq %r14
-; X64-NOPIC-NEXT:    pushq %r13
 ; X64-NOPIC-NEXT:    pushq %r12
 ; X64-NOPIC-NEXT:    pushq %rbx
-; X64-NOPIC-NEXT:    subq $24, %rsp
+; X64-NOPIC-NEXT:    subq $16, %rsp
 ; X64-NOPIC-NEXT:    movq %rsp, %rax
 ; X64-NOPIC-NEXT:    movq %rdi, %rbx
 ; X64-NOPIC-NEXT:    movq $-1, %r15
 ; X64-NOPIC-NEXT:    sarq $63, %rax
-; X64-NOPIC-NEXT:    leaq {{[0-9]+}}(%rsp), %r14
+; X64-NOPIC-NEXT:    movq %rsp, %r14
 ; X64-NOPIC-NEXT:    shlq $47, %rax
 ; X64-NOPIC-NEXT:    movq %r14, %rdi
 ; X64-NOPIC-NEXT:    orq %rax, %rsp
@@ -302,24 +301,23 @@ define i32 @test_call_setjmp(ptr%ptr) nounwind {
 ; X64-NOPIC-NEXT:    sarq $63, %rax
 ; X64-NOPIC-NEXT:    cmpq $.Lslh_ret_addr4, %r12
 ; X64-NOPIC-NEXT:    cmovneq %r15, %rax
-; X64-NOPIC-NEXT:    movl (%rbx), %r12d
-; X64-NOPIC-NEXT:    movl $42, %ebp
+; X64-NOPIC-NEXT:    movl (%rbx), %ebp
 ; X64-NOPIC-NEXT:    shlq $47, %rax
 ; X64-NOPIC-NEXT:    movq %r14, %rdi
-; X64-NOPIC-NEXT:    movl %ebp, %esi
+; X64-NOPIC-NEXT:    movl $42, %esi
 ; X64-NOPIC-NEXT:    orq %rax, %rsp
-; X64-NOPIC-NEXT:    movq $.Lslh_ret_addr5, %r13
+; X64-NOPIC-NEXT:    movq $.Lslh_ret_addr5, %r12
 ; X64-NOPIC-NEXT:    callq sigsetjmp@PLT
 ; X64-NOPIC-NEXT:  .Lslh_ret_addr5:
 ; X64-NOPIC-NEXT:    movq %rsp, %rax
 ; X64-NOPIC-NEXT:    sarq $63, %rax
-; X64-NOPIC-NEXT:    cmpq $.Lslh_ret_addr5, %r13
+; X64-NOPIC-NEXT:    cmpq $.Lslh_ret_addr5, %r12
 ; X64-NOPIC-NEXT:    cmovneq %r15, %rax
-; X64-NOPIC-NEXT:    addl (%rbx), %r12d
+; X64-NOPIC-NEXT:    addl (%rbx), %ebp
 ; X64-NOPIC-NEXT:    shlq $47, %rax
 ; X64-NOPIC-NEXT:    movq %r14, %rdi
 ; X64-NOPIC-NEXT:    movq %r14, %rsi
-; X64-NOPIC-NEXT:    movl %ebp, %edx
+; X64-NOPIC-NEXT:    movl $42, %edx
 ; X64-NOPIC-NEXT:    orq %rax, %rsp
 ; X64-NOPIC-NEXT:    movq $.Lslh_ret_addr6, %r14
 ; X64-NOPIC-NEXT:    callq __sigsetjmp@PLT
@@ -329,15 +327,14 @@ define i32 @test_call_setjmp(ptr%ptr) nounwind {
 ; X64-NOPIC-NEXT:    cmpq $.Lslh_ret_addr6, %r14
 ; X64-NOPIC-NEXT:    movq %rax, %rcx
 ; X64-NOPIC-NEXT:    cmovneq %r15, %rcx
-; X64-NOPIC-NEXT:    addl (%rbx), %r12d
-; X64-NOPIC-NEXT:    movl %r12d, %eax
+; X64-NOPIC-NEXT:    addl (%rbx), %ebp
+; X64-NOPIC-NEXT:    movl %ebp, %eax
 ; X64-NOPIC-NEXT:    orl %ecx, %eax
 ; X64-NOPIC-NEXT:    shlq $47, %rcx
 ; X64-NOPIC-NEXT:    orq %rcx, %rsp
-; X64-NOPIC-NEXT:    addq $24, %rsp
+; X64-NOPIC-NEXT:    addq $16, %rsp
 ; X64-NOPIC-NEXT:    popq %rbx
 ; X64-NOPIC-NEXT:    popq %r12
-; X64-NOPIC-NEXT:    popq %r13
 ; X64-NOPIC-NEXT:    popq %r14
 ; X64-NOPIC-NEXT:    popq %r15
 ; X64-NOPIC-NEXT:    popq %rbp
@@ -348,15 +345,14 @@ define i32 @test_call_setjmp(ptr%ptr) nounwind {
 ; X64-NOPIC-MCM-NEXT:    pushq %rbp
 ; X64-NOPIC-MCM-NEXT:    pushq %r15
 ; X64-NOPIC-MCM-NEXT:    pushq %r14
-; X64-NOPIC-MCM-NEXT:    pushq %r13
 ; X64-NOPIC-MCM-NEXT:    pushq %r12
 ; X64-NOPIC-MCM-NEXT:    pushq %rbx
-; X64-NOPIC-MCM-NEXT:    subq $24, %rsp
+; X64-NOPIC-MCM-NEXT:    subq $16, %rsp
 ; X64-NOPIC-MCM-NEXT:    movq %rsp, %rax
 ; X64-NOPIC-MCM-NEXT:    movq %rdi, %rbx
 ; X64-NOPIC-MCM-NEXT:    movq $-1, %r15
 ; X64-NOPIC-MCM-NEXT:    sarq $63, %rax
-; X64-NOPIC-MCM-NEXT:    leaq {{[0-9]+}}(%rsp), %r14
+; X64-NOPIC-MCM-NEXT:    movq %rsp, %r14
 ; X64-NOPIC-MCM-NEXT:    shlq $47, %rax
 ; X64-NOPIC-MCM-NEXT:    movq %r14, %rdi
 ; X64-NOPIC-MCM-NEXT:    orq %rax, %rsp
@@ -368,25 +364,24 @@ define i32 @test_call_setjmp(ptr%ptr) nounwind {
 ; X64-NOPIC-MCM-NEXT:    leaq .Lslh_ret_addr4(%rip), %rcx
 ; X64-NOPIC-MCM-NEXT:    cmpq %rcx, %r12
 ; X64-NOPIC-MCM-NEXT:    cmovneq %r15, %rax
-; X64-NOPIC-MCM-NEXT:    movl (%rbx), %r12d
-; X64-NOPIC-MCM-NEXT:    movl $42, %ebp
+; X64-NOPIC-MCM-NEXT:    movl (%rbx), %ebp
 ; X64-NOPIC-MCM-NEXT:    shlq $47, %rax
 ; X64-NOPIC-MCM-NEXT:    movq %r14, %rdi
-; X64-NOPIC-MCM-NEXT:    movl %ebp, %esi
+; X64-NOPIC-MCM-NEXT:    movl $42, %esi
 ; X64-NOPIC-MCM-NEXT:    orq %rax, %rsp
-; X64-NOPIC-MCM-NEXT:    leaq .Lslh_ret_addr5(%rip), %r13
+; X64-NOPIC-MCM-NEXT:    leaq .Lslh_ret_addr5(%rip), %r12
 ; X64-NOPIC-MCM-NEXT:    callq sigsetjmp@PLT
 ; X64-NOPIC-MCM-NEXT:  .Lslh_ret_addr5:
 ; X64-NOPIC-MCM-NEXT:    movq %rsp, %rax
 ; X64-NOPIC-MCM-NEXT:    sarq $63, %rax
 ; X64-NOPIC-MCM-NEXT:    leaq .Lslh_ret_addr5(%rip), %rcx
-; X64-NOPIC-MCM-NEXT:    cmpq %rcx, %r13
+; X64-NOPIC-MCM-NEXT:    cmpq %rcx, %r12
 ; X64-NOPIC-MCM-NEXT:    cmovneq %r15, %rax
-; X64-NOPIC-MCM-NEXT:    addl (%rbx), %r12d
+; X64-NOPIC-MCM-NEXT:    addl (%rbx), %ebp
 ; X64-NOPIC-MCM-NEXT:    shlq $47, %rax
 ; X64-NOPIC-MCM-NEXT:    movq %r14, %rdi
 ; X64-NOPIC-MCM-NEXT:    movq %r14, %rsi
-; X64-NOPIC-MCM-NEXT:    movl %ebp, %edx
+; X64-NOPIC-MCM-NEXT:    movl $42, %edx
 ; X64-NOPIC-MCM-NEXT:    orq %rax, %rsp
 ; X64-NOPIC-MCM-NEXT:    leaq .Lslh_ret_addr6(%rip), %r14
 ; X64-NOPIC-MCM-NEXT:    callq __sigsetjmp@PLT
@@ -397,15 +392,14 @@ define i32 @test_call_setjmp(ptr%ptr) nounwind {
 ; X64-NOPIC-MCM-NEXT:    cmpq %rcx, %r14
 ; X64-NOPIC-MCM-NEXT:    movq %rax, %rcx
 ; X64-NOPIC-MCM-NEXT:    cmovneq %r15, %rcx
-; X64-NOPIC-MCM-NEXT:    addl (%rbx), %r12d
-; X64-NOPIC-MCM-NEXT:    movl %r12d, %eax
+; X64-NOPIC-MCM-NEXT:    addl (%rbx), %ebp
+; X64-NOPIC-MCM-NEXT:    movl %ebp, %eax
 ; X64-NOPIC-MCM-NEXT:    orl %ecx, %eax
 ; X64-NOPIC-MCM-NEXT:    shlq $47, %rcx
 ; X64-NOPIC-MCM-NEXT:    orq %rcx, %rsp
-; X64-NOPIC-MCM-NEXT:    addq $24, %rsp
+; X64-NOPIC-MCM-NEXT:    addq $16, %rsp
 ; X64-NOPIC-MCM-NEXT:    popq %rbx
 ; X64-NOPIC-MCM-NEXT:    popq %r12
-; X64-NOPIC-MCM-NEXT:    popq %r13
 ; X64-NOPIC-MCM-NEXT:    popq %r14
 ; X64-NOPIC-MCM-NEXT:    popq %r15
 ; X64-NOPIC-MCM-NEXT:    popq %rbp
@@ -416,15 +410,14 @@ define i32 @test_call_setjmp(ptr%ptr) nounwind {
 ; X64-PIC-NEXT:    pushq %rbp
 ; X64-PIC-NEXT:    pushq %r15
 ; X64-PIC-NEXT:    pushq %r14
-; X64-PIC-NEXT:    pushq %r13
 ; X64-PIC-NEXT:    pushq %r12
 ; X64-PIC-NEXT:    pushq %rbx
-; X64-PIC-NEXT:    subq $24, %rsp
+; X64-PIC-NEXT:    subq $16, %rsp
 ; X64-PIC-NEXT:    movq %rsp, %rax
 ; X64-PIC-NEXT:    movq %rdi, %rbx
 ; X64-PIC-NEXT:    movq $-1, %r15
 ; X64-PIC-NEXT:    sarq $63, %rax
-; X64-PIC-NEXT:    leaq {{[0-9]+}}(%rsp), %r14
+; X64-PIC-NEXT:    movq %rsp, %r14
 ; X64-PIC-NEXT:    shlq $47, %rax
 ; X64-PIC-NEXT:    movq %r14, %rdi
 ; X64-PIC-NEXT:    orq %rax, %rsp
@@ -436,25 +429,24 @@ define i32 @test_call_setjmp(ptr%ptr) nounwind {
 ; X64-PIC-NEXT:    leaq .Lslh_ret_addr4(%rip), %rcx
 ; X64-PIC-NEXT:    cmpq %rcx, %r12
 ; X64-PIC-NEXT:    cmovneq %r15, %rax
-; X64-PIC-NEXT:    movl (%rbx), %r12d
-; X64-PIC-NEXT:    movl $42, %ebp
+; X64-PIC-NEXT:    movl (%rbx), %ebp
 ; X64-PIC-NEXT:    shlq $47, %rax
 ; X64-PIC-NEXT:    movq %r14, %rdi
-; X64-PIC-NEXT:    movl %ebp, %esi
+; X64-PIC-NEXT:    movl $42, %esi
 ; X64-PIC-NEXT:    orq %rax, %rsp
-; X64-PIC-NEXT:    leaq .Lslh_ret_addr5(%rip), %r13
+; X64-PIC-NEXT:    leaq .Lslh_ret_addr5(%rip), %r12
 ; X64-PIC-NEXT:    callq sigsetjmp@PLT
 ; X64-PIC-NEXT:  .Lslh_ret_addr5:
 ; X64-PIC-NEXT:    movq %rsp, %rax
 ; X64-PIC-NEXT:    sarq $63, %rax
 ; X64-PIC-NEXT:    leaq .Lslh_ret_addr5(%rip), %rcx
-; X64-PIC-NEXT:    cmpq %rcx, %r13
+; X64-PIC-NEXT:    cmpq %rcx, %r12
 ; X64-PIC-NEXT:    cmovneq %r15, %rax
-; X64-PIC-NEXT:    addl (%rbx), %r12d
+; X64-PIC-NEXT:    addl (%rbx), %ebp
 ; X64-PIC-NEXT:    shlq $47, %rax
 ; X64-PIC-NEXT:    movq %r14, %rdi
 ; X64-PIC-NEXT:    movq %r14, %rsi
-; X64-PIC-NEXT:    movl %ebp, %edx
+; X64-PIC-NEXT:    movl $42, %edx
 ; X64-PIC-NEXT:    orq %rax, %rsp
 ; X64-PIC-NEXT:    leaq .Lslh_ret_addr6(%rip), %r14
 ; X64-PIC-NEXT:    callq __sigsetjmp@PLT
@@ -465,15 +457,14 @@ define i32 @test_call_setjmp(ptr%ptr) nounwind {
 ; X64-PIC-NEXT:    cmpq %rcx, %r14
 ; X64-PIC-NEXT:    movq %rax, %rcx
 ; X64-PIC-NEXT:    cmovneq %r15, %rcx
-; X64-PIC-NEXT:    addl (%rbx), %r12d
-; X64-PIC-NEXT:    movl %r12d, %eax
+; X64-PIC-NEXT:    addl (%rbx), %ebp
+; X64-PIC-NEXT:    movl %ebp, %eax
 ; X64-PIC-NEXT:    orl %ecx, %eax
 ; X64-PIC-NEXT:    shlq $47, %rcx
 ; X64-PIC-NEXT:    orq %rcx, %rsp
-; X64-PIC-NEXT:    addq $24, %rsp
+; X64-PIC-NEXT:    addq $16, %rsp
 ; X64-PIC-NEXT:    popq %rbx
 ; X64-PIC-NEXT:    popq %r12
-; X64-PIC-NEXT:    popq %r13
 ; X64-PIC-NEXT:    popq %r14
 ; X64-PIC-NEXT:    popq %r15
 ; X64-PIC-NEXT:    popq %rbp
diff --git a/llvm/test/CodeGen/X86/sret-implicit.ll b/llvm/test/CodeGen/X86/sret-implicit.ll
index 8cd2e63263..6cd64812f2 100644
--- a/llvm/test/CodeGen/X86/sret-implicit.ll
+++ b/llvm/test/CodeGen/X86/sret-implicit.ll
@@ -25,7 +25,8 @@ define i256 @sret_demoted() {
 
 ; X64-LABEL: sret_demoted
 ; X64-DAG: movq %rdi, %rax
-; X64-DAG: movq $0, (%rdi)
+; X64-DAG: xorps %xmm0, %xmm0
+; X64-DAG: movaps %xmm0, (%rdi)
 ; X64: retq
 
 ; X86-LABEL: sret_demoted
diff --git a/llvm/test/CodeGen/X86/stack-frame-layout-remarks-64.ll b/llvm/test/CodeGen/X86/stack-frame-layout-remarks-64.ll
deleted file mode 100644
index f4b00e3132..0000000000
--- a/llvm/test/CodeGen/X86/stack-frame-layout-remarks-64.ll
+++ /dev/null
@@ -1,76 +0,0 @@
-; Test remark output for stack-frame-layout
-
-; ensure basic output works
-; RUN: llc -mcpu=corei7 -O1 -pass-remarks-analysis=stack-frame-layout < %s 2>&1 >/dev/null | FileCheck %s
-
-; check additional slots are displayed when stack is not optimized
-; RUN: llc -mcpu=corei7 -O0 -pass-remarks-analysis=stack-frame-layout < %s 2>&1 >/dev/null | FileCheck %s --check-prefix=NO_COLORING
-
-target triple = "x86_64-unknown-linux-gnu"
-
-@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
-declare i32 @printf(ptr, ...)
-
-; CHECK: Function: stackSizeWarning
-; CHECK: Offset: [SP-88], Type: Variable, Align: 16, Size: 80
-; CHECK:    buffer @ frame-diags.c:30
-; NO_COLORING: Offset: [SP-168], Type: Variable, Align: 16, Size: 80
-; CHECK:    buffer2 @ frame-diags.c:33
-define void @stackSizeWarning() {
-entry:
-  %buffer = alloca [80 x i8], align 16
-  %buffer2 = alloca [80 x i8], align 16
-  call void @llvm.dbg.declare(metadata ptr %buffer, metadata !25, metadata !DIExpression()), !dbg !39
-  call void @llvm.dbg.declare(metadata ptr %buffer2, metadata !31, metadata !DIExpression()), !dbg !40
-  ret void
-}
-
-; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
-declare void @llvm.dbg.declare(metadata, metadata, metadata) #0
-
-attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
-
-!llvm.dbg.cu = !{!0, !2}
-!llvm.module.flags = !{!18, !19, !20, !21, !22, !23, !24}
-
-!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
-!1 = !DIFile(filename: "frame-diags.c", directory: "")
-!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, retainedTypes: !4, splitDebugInlining: false, nameTableKind: None)
-!3 = !DIFile(filename: "dot.c", directory: "")
-!4 = !{!5, !6, !10, !13}
-!5 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: null, size: 64)
-!6 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !7, size: 64)
-!7 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Array", file: !3, line: 3, size: 128, elements: !8)
-!8 = !{!9, !12}
-!9 = !DIDerivedType(tag: DW_TAG_member, name: "data", scope: !7, file: !3, line: 4, baseType: !10, size: 64)
-!10 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !11, size: 64)
-!11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
-!12 = !DIDerivedType(tag: DW_TAG_member, name: "size", scope: !7, file: !3, line: 5, baseType: !11, size: 32, offset: 64)
-!13 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !14, size: 64)
-!14 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Result", file: !3, line: 8, size: 128, elements: !15)
-!15 = !{!16, !17}
-!16 = !DIDerivedType(tag: DW_TAG_member, name: "data", scope: !14, file: !3, line: 9, baseType: !6, size: 64)
-!17 = !DIDerivedType(tag: DW_TAG_member, name: "sum", scope: !14, file: !3, line: 10, baseType: !11, size: 32, offset: 64)
-!18 = !{i32 7, !"Dwarf Version", i32 5}
-!19 = !{i32 2, !"Debug Info Version", i32 3}
-!20 = !{i32 1, !"wchar_size", i32 4}
-!21 = !{i32 8, !"PIC Level", i32 2}
-!22 = !{i32 7, !"PIE Level", i32 2}
-!23 = !{i32 7, !"uwtable", i32 2}
-!24 = !{i32 7, !"frame-pointer", i32 2}
-!25 = !DILocalVariable(name: "buffer", scope: !26, file: !1, line: 30, type: !32)
-!26 = distinct !DILexicalBlock(scope: !27, file: !1, line: 29, column: 3)
-!27 = distinct !DISubprogram(name: "stackSizeWarning", scope: !1, file: !1, line: 28, type: !28, scopeLine: 28, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !30)
-!28 = !DISubroutineType(types: !29)
-!29 = !{null}
-!30 = !{!25, !31, !36, !37}
-!31 = !DILocalVariable(name: "buffer2", scope: !27, file: !1, line: 33, type: !32)
-!32 = !DICompositeType(tag: DW_TAG_array_type, baseType: !33, size: 640, elements: !34)
-!33 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
-!34 = !{!35}
-!35 = !DISubrange(count: 80)
-!36 = !DILocalVariable(name: "a", scope: !27, file: !1, line: 34, type: !11)
-!37 = !DILocalVariable(name: "b", scope: !27, file: !1, line: 35, type: !38)
-!38 = !DIBasicType(name: "long", size: 64, encoding: DW_ATE_signed)
-!39 = !DILocation(line: 30, column: 10, scope: !26)
-!40 = !DILocation(line: 33, column: 8, scope: !27)
diff --git a/llvm/test/CodeGen/X86/stack-frame-layout-remarks-32.ll b/llvm/test/CodeGen/X86/stack-frame-layout-remarks.ll
similarity index 91%
rename from llvm/test/CodeGen/X86/stack-frame-layout-remarks-32.ll
rename to llvm/test/CodeGen/X86/stack-frame-layout-remarks.ll
index dda5459149..d32a37efcb 100644
--- a/llvm/test/CodeGen/X86/stack-frame-layout-remarks-32.ll
+++ b/llvm/test/CodeGen/X86/stack-frame-layout-remarks.ll
@@ -1,16 +1,36 @@
 ; Test remark output for stack-frame-layout
 
+; ensure basic output works
+; RUN: llc -mcpu=corei7 -O1 -pass-remarks-analysis=stack-frame-layout < %s 2>&1 >/dev/null | FileCheck %s
+
+; check additional slots are displayed when stack is not optimized
+; RUN: llc -mcpu=corei7 -O0 -pass-remarks-analysis=stack-frame-layout < %s 2>&1 >/dev/null | FileCheck %s --check-prefix=NO_COLORING
+
 ; check more complex cases
 ; RUN: llc %s -pass-remarks-analysis=stack-frame-layout -o /dev/null --march=x86 -mcpu=i386 2>&1 | FileCheck %s --check-prefix=BOTH --check-prefix=DEBUG
 
 ; check output without debug info
 ; RUN: opt %s -passes=strip -S | llc  -pass-remarks-analysis=stack-frame-layout -o /dev/null --march=x86 -mcpu=i386 2>&1 | FileCheck %s --check-prefix=BOTH --check-prefix=STRIPPED
 
-target triple = "i386-unknown-linux-gnu"
+target triple = "x86_64-unknown-linux-gnu"
 
 @.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
 declare i32 @printf(ptr, ...)
 
+; CHECK: Function: stackSizeWarning
+; CHECK: Offset: [SP-88], Type: Variable, Align: 16, Size: 80
+; CHECK:    buffer @ frame-diags.c:30
+; NO_COLORING: Offset: [SP-168], Type: Variable, Align: 16, Size: 80
+; CHECK:    buffer2 @ frame-diags.c:33
+define void @stackSizeWarning() {
+entry:
+  %buffer = alloca [80 x i8], align 16
+  %buffer2 = alloca [80 x i8], align 16
+  call void @llvm.dbg.declare(metadata ptr %buffer, metadata !25, metadata !DIExpression()), !dbg !39
+  call void @llvm.dbg.declare(metadata ptr %buffer2, metadata !31, metadata !DIExpression()), !dbg !40
+  ret void
+}
+
 ; Function Attrs: nocallback nofree nosync nounwind readnone speculatable willreturn
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #0
 
@@ -188,7 +208,7 @@ entry:
 }
 
 ; uselistorder directives
-uselistorder ptr @llvm.dbg.declare, { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17, 16 }
+uselistorder ptr @llvm.dbg.declare, { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 19, 18 }
 
 attributes #0 = { nocallback nofree nosync nounwind readnone speculatable willreturn }
 attributes #1 = { "frame-pointer"="all" }
@@ -202,19 +222,19 @@ attributes #2 = { ssp "stack-protector-buffer-size"="5" "frame-pointer"="all" }
 !2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, retainedTypes: !4, splitDebugInlining: false, nameTableKind: None)
 !3 = !DIFile(filename: "dot.c", directory: "")
 !4 = !{!5, !6, !10, !13}
-!5 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: null, size: 32)
-!6 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !7, size: 32)
-!7 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Array", file: !3, line: 3, size: 64, elements: !8)
+!5 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: null, size: 64)
+!6 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !7, size: 64)
+!7 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Array", file: !3, line: 3, size: 128, elements: !8)
 !8 = !{!9, !12}
-!9 = !DIDerivedType(tag: DW_TAG_member, name: "data", scope: !7, file: !3, line: 4, baseType: !10, size: 32)
-!10 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !11, size: 32)
+!9 = !DIDerivedType(tag: DW_TAG_member, name: "data", scope: !7, file: !3, line: 4, baseType: !10, size: 64)
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !11, size: 64)
 !11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
-!12 = !DIDerivedType(tag: DW_TAG_member, name: "size", scope: !7, file: !3, line: 5, baseType: !11, size: 32, offset: 32)
-!13 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !14, size: 32)
-!14 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Result", file: !3, line: 8, size: 64, elements: !15)
+!12 = !DIDerivedType(tag: DW_TAG_member, name: "size", scope: !7, file: !3, line: 5, baseType: !11, size: 32, offset: 64)
+!13 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !14, size: 64)
+!14 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Result", file: !3, line: 8, size: 128, elements: !15)
 !15 = !{!16, !17}
-!16 = !DIDerivedType(tag: DW_TAG_member, name: "data", scope: !14, file: !3, line: 9, baseType: !6, size: 32)
-!17 = !DIDerivedType(tag: DW_TAG_member, name: "sum", scope: !14, file: !3, line: 10, baseType: !11, size: 32, offset: 32)
+!16 = !DIDerivedType(tag: DW_TAG_member, name: "data", scope: !14, file: !3, line: 9, baseType: !6, size: 64)
+!17 = !DIDerivedType(tag: DW_TAG_member, name: "sum", scope: !14, file: !3, line: 10, baseType: !11, size: 32, offset: 64)
 !18 = !{i32 7, !"Dwarf Version", i32 5}
 !19 = !{i32 2, !"Debug Info Version", i32 3}
 !20 = !{i32 1, !"wchar_size", i32 4}
diff --git a/llvm/test/CodeGen/X86/statepoint-call-lowering.ll b/llvm/test/CodeGen/X86/statepoint-call-lowering.ll
index 66088046ef..758cb8b7b6 100644
--- a/llvm/test/CodeGen/X86/statepoint-call-lowering.ll
+++ b/llvm/test/CodeGen/X86/statepoint-call-lowering.ll
@@ -247,8 +247,6 @@ define i8 @test_signext_return(ptr) gc "statepoint-example" {
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    callq signext_return_i1@PLT
 ; CHECK-NEXT:  .Ltmp10:
-; CHECK-NEXT:    andb $1, %al
-; CHECK-NEXT:    negb %al
 ; CHECK-NEXT:    popq %rcx
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
@@ -266,7 +264,6 @@ define i8 @test_zeroext_return() gc "statepoint-example" {
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    callq return_i1@PLT
 ; CHECK-NEXT:  .Ltmp11:
-; CHECK-NEXT:    andb $1, %al
 ; CHECK-NEXT:    popq %rcx
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
@@ -277,6 +274,24 @@ entry:
   ret i8 %ext
 }
 
+define signext i1 @test_noext_signext_return() gc "statepoint-example" {
+; CHECK-LABEL: test_noext_signext_return:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    callq return_i1@PLT
+; CHECK-NEXT:  .Ltmp12:
+; CHECK-NEXT:    andb $1, %al
+; CHECK-NEXT:    negb %al
+; CHECK-NEXT:    popq %rcx
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+entry:
+  %safepoint_token = tail call token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0(i64 0, i32 0, ptr elementtype(i1 ()) @return_i1, i32 0, i32 0, i32 0, i32 0)
+  %call1 = call i1 @llvm.experimental.gc.result.i1(token %safepoint_token)
+  ret i1 %call1
+}
+
 declare token @llvm.experimental.gc.statepoint.p0(i64, i32, ptr, i32, i32, ...)
 declare i1 @llvm.experimental.gc.result.i1(token)
 
diff --git a/llvm/test/CodeGen/X86/statepoint-deopt-lowering.ll b/llvm/test/CodeGen/X86/statepoint-deopt-lowering.ll
index cd882450c1..72f4fa37dc 100644
--- a/llvm/test/CodeGen/X86/statepoint-deopt-lowering.ll
+++ b/llvm/test/CodeGen/X86/statepoint-deopt-lowering.ll
@@ -43,35 +43,25 @@ define void @test_illegal_constants() gc "statepoint-example" {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    subq $248, %rsp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 256
-; CHECK-NEXT:    movq $0, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq $0, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq $0, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq $144, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    movaps %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movaps %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movaps %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movaps %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movaps %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movaps %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movaps %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movaps %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movaps %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movaps %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movaps %xmm0, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movq $0, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movq $144, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movq $0, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq $0, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq $0, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq $0, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq $0, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq $0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movq $144, (%rsp)
 ; CHECK-NEXT:    movq $0, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movq $144, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movq $0, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq $0, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq $0, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq $0, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq $0, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq $0, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq $0, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq $0, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq $0, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq $0, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq $0, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq $0, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq $0, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq $0, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq $0, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movq $144, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    callq foo@PLT
 ; CHECK-NEXT:  .Ltmp2:
@@ -89,60 +79,36 @@ define void @test_illegal_values(i128 %v1, i256 %v2, i512 %v3, i1024 %v4) gc "st
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    subq $248, %rsp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 256
+; CHECK-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm0
+; CHECK-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm1
+; CHECK-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm2
+; CHECK-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm3
+; CHECK-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm4
+; CHECK-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm5
+; CHECK-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm6
+; CHECK-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm7
+; CHECK-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm8
+; CHECK-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm9
+; CHECK-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm10
+; CHECK-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm11
 ; CHECK-NEXT:    movq %r9, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movq %r8, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movq %rcx, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movq %rdx, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movq %rsi, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq %rdi, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; CHECK-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; CHECK-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; CHECK-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; CHECK-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; CHECK-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; CHECK-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; CHECK-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; CHECK-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; CHECK-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; CHECK-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; CHECK-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; CHECK-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; CHECK-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; CHECK-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; CHECK-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; CHECK-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; CHECK-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; CHECK-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; CHECK-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; CHECK-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; CHECK-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; CHECK-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; CHECK-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; CHECK-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movq %rdi, (%rsp)
+; CHECK-NEXT:    movaps %xmm11, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movaps %xmm10, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movaps %xmm9, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movaps %xmm8, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movaps %xmm7, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movaps %xmm6, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movaps %xmm5, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movaps %xmm4, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movaps %xmm3, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movaps %xmm2, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movaps %xmm1, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movaps %xmm0, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    callq foo@PLT
 ; CHECK-NEXT:  .Ltmp3:
 ; CHECK-NEXT:    addq $248, %rsp
diff --git a/llvm/test/CodeGen/X86/statepoint-vector.ll b/llvm/test/CodeGen/X86/statepoint-vector.ll
index c8ddb89ee5..f53ac2f8f6 100644
--- a/llvm/test/CodeGen/X86/statepoint-vector.ll
+++ b/llvm/test/CodeGen/X86/statepoint-vector.ll
@@ -122,9 +122,9 @@ define void @test5() gc "statepoint-example" {
 ; CHECK-NEXT:    subq $40, %rsp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 48
 ; CHECK-NEXT:    xorps %xmm0, %xmm0
-; CHECK-NEXT:    movups %xmm0, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    movq $-1, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movaps %xmm0, {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    movq $-1, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movq $-1, (%rsp)
 ; CHECK-NEXT:    callq do_safepoint@PLT
 ; CHECK-NEXT:  .Ltmp4:
 ; CHECK-NEXT:    addq $40, %rsp
diff --git a/llvm/test/CodeGen/X86/swifterror.ll b/llvm/test/CodeGen/X86/swifterror.ll
index 6d77e04504..5814146a54 100644
--- a/llvm/test/CodeGen/X86/swifterror.ll
+++ b/llvm/test/CodeGen/X86/swifterror.ll
@@ -1566,11 +1566,11 @@ define swiftcc { i64, i64, i64, i64} @params_and_return_in_reg(i64, i64, i64, i6
 ; CHECK-APPLE-NEXT:    .cfi_offset %r14, -32
 ; CHECK-APPLE-NEXT:    .cfi_offset %r15, -24
 ; CHECK-APPLE-NEXT:    .cfi_offset %rbp, -16
-; CHECK-APPLE-NEXT:    movq %r12, %rbx
-; CHECK-APPLE-NEXT:    movq %r13, (%rsp) ## 8-byte Spill
+; CHECK-APPLE-NEXT:    movq %r12, (%rsp) ## 8-byte Spill
+; CHECK-APPLE-NEXT:    movq %r13, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
 ; CHECK-APPLE-NEXT:    movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
 ; CHECK-APPLE-NEXT:    movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
-; CHECK-APPLE-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-APPLE-NEXT:    movq %rcx, %rbx
 ; CHECK-APPLE-NEXT:    movq %rdx, %r14
 ; CHECK-APPLE-NEXT:    movq %rsi, %r15
 ; CHECK-APPLE-NEXT:    movq %rdi, %rbp
@@ -1587,16 +1587,16 @@ define swiftcc { i64, i64, i64, i64} @params_and_return_in_reg(i64, i64, i64, i6
 ; CHECK-APPLE-NEXT:    movq %rbp, %rdi
 ; CHECK-APPLE-NEXT:    movq %r15, %rsi
 ; CHECK-APPLE-NEXT:    movq %r14, %rdx
-; CHECK-APPLE-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx ## 8-byte Reload
+; CHECK-APPLE-NEXT:    movq %rbx, %rcx
 ; CHECK-APPLE-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 ## 8-byte Reload
 ; CHECK-APPLE-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r9 ## 8-byte Reload
-; CHECK-APPLE-NEXT:    movq (%rsp), %r13 ## 8-byte Reload
-; CHECK-APPLE-NEXT:    movq %rbx, %r12
+; CHECK-APPLE-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 ## 8-byte Reload
+; CHECK-APPLE-NEXT:    movq (%rsp), %r12 ## 8-byte Reload
 ; CHECK-APPLE-NEXT:    callq _params_and_return_in_reg2
-; CHECK-APPLE-NEXT:    movq %rax, %r14
-; CHECK-APPLE-NEXT:    movq %rdx, %r15
-; CHECK-APPLE-NEXT:    movq %rcx, %rbp
-; CHECK-APPLE-NEXT:    movq %r8, %rbx
+; CHECK-APPLE-NEXT:    movq %rax, %rbx
+; CHECK-APPLE-NEXT:    movq %rdx, %r14
+; CHECK-APPLE-NEXT:    movq %rcx, %r15
+; CHECK-APPLE-NEXT:    movq %r8, %rbp
 ; CHECK-APPLE-NEXT:    movq %r12, (%rsp) ## 8-byte Spill
 ; CHECK-APPLE-NEXT:    movl $1, %edi
 ; CHECK-APPLE-NEXT:    movl $2, %esi
@@ -1607,10 +1607,10 @@ define swiftcc { i64, i64, i64, i64} @params_and_return_in_reg(i64, i64, i64, i6
 ; CHECK-APPLE-NEXT:    xorl %r13d, %r13d
 ; CHECK-APPLE-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %r12 ## 8-byte Reload
 ; CHECK-APPLE-NEXT:    callq _params_in_reg2
-; CHECK-APPLE-NEXT:    movq %r14, %rax
-; CHECK-APPLE-NEXT:    movq %r15, %rdx
-; CHECK-APPLE-NEXT:    movq %rbp, %rcx
-; CHECK-APPLE-NEXT:    movq %rbx, %r8
+; CHECK-APPLE-NEXT:    movq %rbx, %rax
+; CHECK-APPLE-NEXT:    movq %r14, %rdx
+; CHECK-APPLE-NEXT:    movq %r15, %rcx
+; CHECK-APPLE-NEXT:    movq %rbp, %r8
 ; CHECK-APPLE-NEXT:    movq (%rsp), %r12 ## 8-byte Reload
 ; CHECK-APPLE-NEXT:    addq $48, %rsp
 ; CHECK-APPLE-NEXT:    popq %rbx
diff --git a/llvm/test/CodeGen/X86/ternlog.ll b/llvm/test/CodeGen/X86/ternlog.ll
index da3b7d5baf..cef044acbc 100644
--- a/llvm/test/CodeGen/X86/ternlog.ll
+++ b/llvm/test/CodeGen/X86/ternlog.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: opt < %s -passes=instcombine -mtriple=x86_64-unknown-unknown -S | llc -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s
+; RUN: opt < %s -passes=instcombine -mtriple=x86_64-unknown-unknown -S | llc -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl -early-live-intervals | FileCheck %s
 
 ;; This is just a simple test to make sure there are no regressions
 ;; cause by splitting/recombining ternlog intrinsics.
diff --git a/llvm/test/CodeGen/X86/udiv_fix.ll b/llvm/test/CodeGen/X86/udiv_fix.ll
index 8d3319eb59..5b1e054550 100644
--- a/llvm/test/CodeGen/X86/udiv_fix.ll
+++ b/llvm/test/CodeGen/X86/udiv_fix.ll
@@ -152,8 +152,8 @@ define i64 @func5(i64 %x, i64 %y) nounwind {
 ; X86-NEXT:    pushl %ebp
 ; X86-NEXT:    movl %esp, %ebp
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    andl $-8, %esp
-; X86-NEXT:    subl $24, %esp
+; X86-NEXT:    andl $-16, %esp
+; X86-NEXT:    subl $32, %esp
 ; X86-NEXT:    movl 8(%ebp), %eax
 ; X86-NEXT:    movl 12(%ebp), %ecx
 ; X86-NEXT:    movl %ecx, %edx
diff --git a/llvm/test/CodeGen/X86/udiv_fix_sat.ll b/llvm/test/CodeGen/X86/udiv_fix_sat.ll
index 67d3e5b16e..30a7f80b23 100644
--- a/llvm/test/CodeGen/X86/udiv_fix_sat.ll
+++ b/llvm/test/CodeGen/X86/udiv_fix_sat.ll
@@ -193,8 +193,8 @@ define i64 @func5(i64 %x, i64 %y) nounwind {
 ; X86-NEXT:    pushl %ebp
 ; X86-NEXT:    movl %esp, %ebp
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    andl $-8, %esp
-; X86-NEXT:    subl $24, %esp
+; X86-NEXT:    andl $-16, %esp
+; X86-NEXT:    subl $32, %esp
 ; X86-NEXT:    movl 8(%ebp), %eax
 ; X86-NEXT:    movl 12(%ebp), %ecx
 ; X86-NEXT:    movl %ecx, %edx
diff --git a/llvm/test/CodeGen/X86/usermsr-intrinsics.ll b/llvm/test/CodeGen/X86/usermsr-intrinsics.ll
new file mode 100644
index 0000000000..29801a494f
--- /dev/null
+++ b/llvm/test/CodeGen/X86/usermsr-intrinsics.ll
@@ -0,0 +1,64 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc < %s -verify-machineinstrs -mtriple=x86_64-unknown-unknown --show-mc-encoding -mattr=+usermsr | FileCheck %s --check-prefixes=X64
+
+define i64 @test_int_x86_urdmsr(i64 %A) nounwind {
+; X64-LABEL: test_int_x86_urdmsr:
+; X64:       # %bb.0:
+; X64-NEXT:    urdmsr %rdi, %rax # encoding: [0xf2,0x0f,0x38,0xf8,0xc7]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %ret = call i64 @llvm.x86.urdmsr(i64 %A)
+  ret i64 %ret
+}
+
+define i64 @test_int_x86_urdmsr_const() nounwind {
+; X64-LABEL: test_int_x86_urdmsr_const:
+; X64:       # %bb.0:
+; X64-NEXT:    urdmsr $123, %rax # encoding: [0xc4,0xe7,0x7b,0xf8,0xc0,0x7b,0x00,0x00,0x00]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %ret = call i64 @llvm.x86.urdmsr(i64 123)
+  ret i64 %ret
+}
+
+define i64 @test_int_x86_urdmsr_const_i64() nounwind {
+; X64-LABEL: test_int_x86_urdmsr_const_i64:
+; X64:       # %bb.0:
+; X64-NEXT:    movabsq $8589934591, %rax # encoding: [0x48,0xb8,0xff,0xff,0xff,0xff,0x01,0x00,0x00,0x00]
+; X64-NEXT:    # imm = 0x1FFFFFFFF
+; X64-NEXT:    urdmsr %rax, %rax # encoding: [0xf2,0x0f,0x38,0xf8,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %ret = call i64 @llvm.x86.urdmsr(i64 8589934591)
+  ret i64 %ret
+}
+
+declare i64 @llvm.x86.urdmsr(i64 %A)
+
+define void @test_int_x86_uwrmsr(i64 %A, i64 %B) nounwind {
+; X64-LABEL: test_int_x86_uwrmsr:
+; X64:       # %bb.0:
+; X64-NEXT:    uwrmsr %rdi, %rsi # encoding: [0xf3,0x0f,0x38,0xf8,0xfe]
+; X64-NEXT:    retq # encoding: [0xc3]
+  call void @llvm.x86.uwrmsr(i64 %A, i64 %B)
+  ret void
+}
+
+define void @test_int_x86_uwrmsr_const(i64 %A) nounwind {
+; X64-LABEL: test_int_x86_uwrmsr_const:
+; X64:       # %bb.0:
+; X64-NEXT:    uwrmsr %rdi, $123 # encoding: [0xc4,0xe7,0x7a,0xf8,0xc7,0x7b,0x00,0x00,0x00]
+; X64-NEXT:    retq # encoding: [0xc3]
+  call void @llvm.x86.uwrmsr(i64 %A, i64 123)
+  ret void
+}
+
+define void @test_int_x86_uwrmsr_const_i64(i64 %A) nounwind {
+; X64-LABEL: test_int_x86_uwrmsr_const_i64:
+; X64:       # %bb.0:
+; X64-NEXT:    movabsq $8589934591, %rax # encoding: [0x48,0xb8,0xff,0xff,0xff,0xff,0x01,0x00,0x00,0x00]
+; X64-NEXT:    # imm = 0x1FFFFFFFF
+; X64-NEXT:    uwrmsr %rdi, %rax # encoding: [0xf3,0x0f,0x38,0xf8,0xf8]
+; X64-NEXT:    retq # encoding: [0xc3]
+  call void @llvm.x86.uwrmsr(i64 %A, i64 8589934591)
+  ret void
+}
+
+declare void @llvm.x86.uwrmsr(i64 %A, i64 %B)
diff --git a/llvm/test/CodeGen/X86/vec_saddo.ll b/llvm/test/CodeGen/X86/vec_saddo.ll
index 7631367ba5..eae9b96921 100644
--- a/llvm/test/CodeGen/X86/vec_saddo.ll
+++ b/llvm/test/CodeGen/X86/vec_saddo.ll
@@ -976,34 +976,24 @@ define <4 x i32> @saddo_v4i24(<4 x i24> %a0, <4 x i24> %a1, ptr %p2) nounwind {
 define <4 x i32> @saddo_v4i1(<4 x i1> %a0, <4 x i1> %a1, ptr %p2) nounwind {
 ; SSE-LABEL: saddo_v4i1:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pslld $31, %xmm1
-; SSE-NEXT:    psrad $31, %xmm1
+; SSE-NEXT:    movdqa %xmm0, %xmm2
+; SSE-NEXT:    pxor %xmm1, %xmm2
+; SSE-NEXT:    pslld $31, %xmm2
+; SSE-NEXT:    movmskps %xmm2, %eax
+; SSE-NEXT:    pand %xmm1, %xmm0
 ; SSE-NEXT:    pslld $31, %xmm0
 ; SSE-NEXT:    psrad $31, %xmm0
-; SSE-NEXT:    paddd %xmm1, %xmm0
-; SSE-NEXT:    movdqa %xmm0, %xmm1
-; SSE-NEXT:    pslld $31, %xmm1
-; SSE-NEXT:    movmskps %xmm1, %eax
-; SSE-NEXT:    psrad $31, %xmm1
-; SSE-NEXT:    pcmpeqd %xmm1, %xmm0
-; SSE-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE-NEXT:    pxor %xmm1, %xmm0
 ; SSE-NEXT:    movb %al, (%rdi)
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: saddo_v4i1:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpslld $31, %xmm1, %xmm1
-; AVX-NEXT:    vpsrad $31, %xmm1, %xmm1
+; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm2
+; AVX-NEXT:    vpslld $31, %xmm2, %xmm2
+; AVX-NEXT:    vmovmskps %xmm2, %eax
+; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpslld $31, %xmm0, %xmm0
 ; AVX-NEXT:    vpsrad $31, %xmm0, %xmm0
-; AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpslld $31, %xmm0, %xmm1
-; AVX-NEXT:    vpsrad $31, %xmm1, %xmm2
-; AVX-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
-; AVX-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpxor %xmm2, %xmm0, %xmm0
-; AVX-NEXT:    vmovmskps %xmm1, %eax
 ; AVX-NEXT:    movb %al, (%rdi)
 ; AVX-NEXT:    retq
 ;
@@ -1011,11 +1001,10 @@ define <4 x i32> @saddo_v4i1(<4 x i1> %a0, <4 x i1> %a1, ptr %p2) nounwind {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpslld $31, %xmm0, %xmm0
 ; AVX512-NEXT:    vptestmd %xmm0, %xmm0, %k0
-; AVX512-NEXT:    vpslld $31, %xmm1, %xmm1
-; AVX512-NEXT:    vptestmd %xmm1, %xmm1, %k1
+; AVX512-NEXT:    vpslld $31, %xmm1, %xmm0
+; AVX512-NEXT:    vptestmd %xmm0, %xmm0, %k1
 ; AVX512-NEXT:    kxorw %k1, %k0, %k2
-; AVX512-NEXT:    vptestnmd %xmm0, %xmm0, %k0 {%k2}
-; AVX512-NEXT:    kxorw %k0, %k1, %k1
+; AVX512-NEXT:    kandw %k1, %k0, %k1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; AVX512-NEXT:    kshiftlw $12, %k2, %k0
diff --git a/llvm/test/CodeGen/X86/vec_ssubo.ll b/llvm/test/CodeGen/X86/vec_ssubo.ll
index d634457069..f8cf543cb9 100644
--- a/llvm/test/CodeGen/X86/vec_ssubo.ll
+++ b/llvm/test/CodeGen/X86/vec_ssubo.ll
@@ -985,34 +985,24 @@ define <4 x i32> @ssubo_v4i24(<4 x i24> %a0, <4 x i24> %a1, ptr %p2) nounwind {
 define <4 x i32> @ssubo_v4i1(<4 x i1> %a0, <4 x i1> %a1, ptr %p2) nounwind {
 ; SSE-LABEL: ssubo_v4i1:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pslld $31, %xmm1
-; SSE-NEXT:    psrad $31, %xmm1
+; SSE-NEXT:    movdqa %xmm0, %xmm2
+; SSE-NEXT:    pxor %xmm1, %xmm2
+; SSE-NEXT:    pslld $31, %xmm2
+; SSE-NEXT:    movmskps %xmm2, %eax
+; SSE-NEXT:    pandn %xmm1, %xmm0
 ; SSE-NEXT:    pslld $31, %xmm0
 ; SSE-NEXT:    psrad $31, %xmm0
-; SSE-NEXT:    psubd %xmm1, %xmm0
-; SSE-NEXT:    movdqa %xmm0, %xmm1
-; SSE-NEXT:    pslld $31, %xmm1
-; SSE-NEXT:    movmskps %xmm1, %eax
-; SSE-NEXT:    psrad $31, %xmm1
-; SSE-NEXT:    pcmpeqd %xmm1, %xmm0
-; SSE-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE-NEXT:    pxor %xmm1, %xmm0
 ; SSE-NEXT:    movb %al, (%rdi)
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: ssubo_v4i1:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpslld $31, %xmm1, %xmm1
-; AVX-NEXT:    vpsrad $31, %xmm1, %xmm1
+; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm2
+; AVX-NEXT:    vpslld $31, %xmm2, %xmm2
+; AVX-NEXT:    vmovmskps %xmm2, %eax
+; AVX-NEXT:    vpandn %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpslld $31, %xmm0, %xmm0
 ; AVX-NEXT:    vpsrad $31, %xmm0, %xmm0
-; AVX-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpslld $31, %xmm0, %xmm1
-; AVX-NEXT:    vpsrad $31, %xmm1, %xmm2
-; AVX-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
-; AVX-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpxor %xmm2, %xmm0, %xmm0
-; AVX-NEXT:    vmovmskps %xmm1, %eax
 ; AVX-NEXT:    movb %al, (%rdi)
 ; AVX-NEXT:    retq
 ;
@@ -1022,11 +1012,11 @@ define <4 x i32> @ssubo_v4i1(<4 x i1> %a0, <4 x i1> %a1, ptr %p2) nounwind {
 ; AVX512-NEXT:    vptestmd %xmm0, %xmm0, %k0
 ; AVX512-NEXT:    vpslld $31, %xmm1, %xmm1
 ; AVX512-NEXT:    vptestmd %xmm1, %xmm1, %k1
-; AVX512-NEXT:    kxorw %k1, %k0, %k1
-; AVX512-NEXT:    vptestnmd %xmm0, %xmm0, %k2 {%k1}
+; AVX512-NEXT:    kxorw %k1, %k0, %k0
+; AVX512-NEXT:    vptestnmd %xmm0, %xmm0, %k1 {%k1}
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k2} {z}
-; AVX512-NEXT:    kshiftlw $12, %k1, %k0
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    kshiftlw $12, %k0, %k0
 ; AVX512-NEXT:    kshiftrw $12, %k0, %k0
 ; AVX512-NEXT:    kmovd %k0, %eax
 ; AVX512-NEXT:    movb %al, (%rdi)
diff --git a/llvm/test/CodeGen/X86/vec_uaddo.ll b/llvm/test/CodeGen/X86/vec_uaddo.ll
index 653c3a9969..950e943bd9 100644
--- a/llvm/test/CodeGen/X86/vec_uaddo.ll
+++ b/llvm/test/CodeGen/X86/vec_uaddo.ll
@@ -1075,49 +1075,26 @@ define <4 x i32> @uaddo_v4i24(<4 x i24> %a0, <4 x i24> %a1, ptr %p2) nounwind {
 define <4 x i32> @uaddo_v4i1(<4 x i1> %a0, <4 x i1> %a1, ptr %p2) nounwind {
 ; SSE-LABEL: uaddo_v4i1:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [1,1,1,1]
-; SSE-NEXT:    pand %xmm2, %xmm1
-; SSE-NEXT:    pand %xmm2, %xmm0
-; SSE-NEXT:    paddd %xmm1, %xmm0
-; SSE-NEXT:    pand %xmm0, %xmm2
-; SSE-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE-NEXT:    pxor %xmm2, %xmm1
+; SSE-NEXT:    movdqa %xmm0, %xmm2
+; SSE-NEXT:    pxor %xmm1, %xmm2
+; SSE-NEXT:    pslld $31, %xmm2
+; SSE-NEXT:    movmskps %xmm2, %eax
+; SSE-NEXT:    pand %xmm1, %xmm0
 ; SSE-NEXT:    pslld $31, %xmm0
-; SSE-NEXT:    movmskps %xmm0, %eax
+; SSE-NEXT:    psrad $31, %xmm0
 ; SSE-NEXT:    movb %al, (%rdi)
-; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: uaddo_v4i1:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1,1,1,1]
-; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpslld $31, %xmm1, %xmm1
-; AVX1-NEXT:    vmovmskps %xmm1, %eax
-; AVX1-NEXT:    movb %al, (%rdi)
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: uaddo_v4i1:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
-; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm0
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpslld $31, %xmm1, %xmm1
-; AVX2-NEXT:    vmovmskps %xmm1, %eax
-; AVX2-NEXT:    movb %al, (%rdi)
-; AVX2-NEXT:    retq
+; AVX-LABEL: uaddo_v4i1:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm2
+; AVX-NEXT:    vpslld $31, %xmm2, %xmm2
+; AVX-NEXT:    vmovmskps %xmm2, %eax
+; AVX-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX-NEXT:    vpsrad $31, %xmm0, %xmm0
+; AVX-NEXT:    movb %al, (%rdi)
+; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: uaddo_v4i1:
 ; AVX512:       # %bb.0:
@@ -1125,11 +1102,11 @@ define <4 x i32> @uaddo_v4i1(<4 x i1> %a0, <4 x i1> %a1, ptr %p2) nounwind {
 ; AVX512-NEXT:    vptestmd %xmm0, %xmm0, %k0
 ; AVX512-NEXT:    vpslld $31, %xmm1, %xmm0
 ; AVX512-NEXT:    vptestmd %xmm0, %xmm0, %k1
-; AVX512-NEXT:    kxorw %k1, %k0, %k1
-; AVX512-NEXT:    kandnw %k0, %k1, %k2
+; AVX512-NEXT:    kxorw %k1, %k0, %k2
+; AVX512-NEXT:    kandw %k1, %k0, %k1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k2} {z}
-; AVX512-NEXT:    kshiftlw $12, %k1, %k0
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    kshiftlw $12, %k2, %k0
 ; AVX512-NEXT:    kshiftrw $12, %k0, %k0
 ; AVX512-NEXT:    kmovd %k0, %eax
 ; AVX512-NEXT:    movb %al, (%rdi)
diff --git a/llvm/test/CodeGen/X86/vec_usubo.ll b/llvm/test/CodeGen/X86/vec_usubo.ll
index a58c3dd0d5..7de972770d 100644
--- a/llvm/test/CodeGen/X86/vec_usubo.ll
+++ b/llvm/test/CodeGen/X86/vec_usubo.ll
@@ -1122,49 +1122,26 @@ define <4 x i32> @usubo_v4i24(<4 x i24> %a0, <4 x i24> %a1, ptr %p2) nounwind {
 define <4 x i32> @usubo_v4i1(<4 x i1> %a0, <4 x i1> %a1, ptr %p2) nounwind {
 ; SSE-LABEL: usubo_v4i1:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [1,1,1,1]
-; SSE-NEXT:    pand %xmm2, %xmm1
-; SSE-NEXT:    pand %xmm2, %xmm0
-; SSE-NEXT:    psubd %xmm1, %xmm0
-; SSE-NEXT:    pand %xmm0, %xmm2
-; SSE-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE-NEXT:    pxor %xmm2, %xmm1
+; SSE-NEXT:    movdqa %xmm0, %xmm2
+; SSE-NEXT:    pxor %xmm1, %xmm2
+; SSE-NEXT:    pslld $31, %xmm2
+; SSE-NEXT:    movmskps %xmm2, %eax
+; SSE-NEXT:    pandn %xmm1, %xmm0
 ; SSE-NEXT:    pslld $31, %xmm0
-; SSE-NEXT:    movmskps %xmm0, %eax
+; SSE-NEXT:    psrad $31, %xmm0
 ; SSE-NEXT:    movb %al, (%rdi)
-; SSE-NEXT:    movdqa %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: usubo_v4i1:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1,1,1,1]
-; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpslld $31, %xmm1, %xmm1
-; AVX1-NEXT:    vmovmskps %xmm1, %eax
-; AVX1-NEXT:    movb %al, (%rdi)
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: usubo_v4i1:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
-; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm0
-; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpslld $31, %xmm1, %xmm1
-; AVX2-NEXT:    vmovmskps %xmm1, %eax
-; AVX2-NEXT:    movb %al, (%rdi)
-; AVX2-NEXT:    retq
+; AVX-LABEL: usubo_v4i1:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm2
+; AVX-NEXT:    vpslld $31, %xmm2, %xmm2
+; AVX-NEXT:    vmovmskps %xmm2, %eax
+; AVX-NEXT:    vpandn %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX-NEXT:    vpsrad $31, %xmm0, %xmm0
+; AVX-NEXT:    movb %al, (%rdi)
+; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: usubo_v4i1:
 ; AVX512:       # %bb.0:
@@ -1172,11 +1149,11 @@ define <4 x i32> @usubo_v4i1(<4 x i1> %a0, <4 x i1> %a1, ptr %p2) nounwind {
 ; AVX512-NEXT:    vptestmd %xmm0, %xmm0, %k0
 ; AVX512-NEXT:    vpslld $31, %xmm1, %xmm1
 ; AVX512-NEXT:    vptestmd %xmm1, %xmm1, %k1
-; AVX512-NEXT:    kxorw %k1, %k0, %k1
-; AVX512-NEXT:    vptestnmd %xmm0, %xmm0, %k2 {%k1}
+; AVX512-NEXT:    kxorw %k1, %k0, %k0
+; AVX512-NEXT:    vptestnmd %xmm0, %xmm0, %k1 {%k1}
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k2} {z}
-; AVX512-NEXT:    kshiftlw $12, %k1, %k0
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    kshiftlw $12, %k0, %k0
 ; AVX512-NEXT:    kshiftrw $12, %k0, %k0
 ; AVX512-NEXT:    kmovd %k0, %eax
 ; AVX512-NEXT:    movb %al, (%rdi)
diff --git a/llvm/test/CodeGen/X86/vector-llrint.ll b/llvm/test/CodeGen/X86/vector-llrint.ll
new file mode 100644
index 0000000000..46904f82fd
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vector-llrint.ll
@@ -0,0 +1,290 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown | FileCheck %s --check-prefix=X64-SSE
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=avx | FileCheck %s --check-prefix=X64-AVX
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=avx512f | FileCheck %s --check-prefix=X64-AVX
+
+define <1 x i64> @llrint_v1i64_v1f32(<1 x float> %x) {
+; X64-SSE-LABEL: llrint_v1i64_v1f32:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    cvtss2si %xmm0, %rax
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: llrint_v1i64_v1f32:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vcvtss2si %xmm0, %rax
+; X64-AVX-NEXT:    retq
+  %a = call <1 x i64> @llvm.llrint.v1i64.v1f32(<1 x float> %x)
+  ret <1 x i64> %a
+}
+declare <1 x i64> @llvm.llrint.v1i64.v1f32(<1 x float>)
+
+define <2 x i64> @llrint_v2i64_v2f32(<2 x float> %x) {
+; X64-SSE-LABEL: llrint_v2i64_v2f32:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    cvtss2si %xmm0, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm1
+; X64-SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; X64-SSE-NEXT:    cvtss2si %xmm0, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm0
+; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; X64-SSE-NEXT:    movdqa %xmm1, %xmm0
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: llrint_v2i64_v2f32:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vcvtss2si %xmm0, %rax
+; X64-AVX-NEXT:    vmovq %rax, %xmm1
+; X64-AVX-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; X64-AVX-NEXT:    vcvtss2si %xmm0, %rax
+; X64-AVX-NEXT:    vmovq %rax, %xmm0
+; X64-AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; X64-AVX-NEXT:    retq
+  %a = call <2 x i64> @llvm.llrint.v2i64.v2f32(<2 x float> %x)
+  ret <2 x i64> %a
+}
+declare <2 x i64> @llvm.llrint.v2i64.v2f32(<2 x float>)
+
+define <4 x i64> @llrint_v4i64_v4f32(<4 x float> %x) {
+; X64-SSE-LABEL: llrint_v4i64_v4f32:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    cvtss2si %xmm0, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm2
+; X64-SSE-NEXT:    movaps %xmm0, %xmm1
+; X64-SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
+; X64-SSE-NEXT:    cvtss2si %xmm1, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm1
+; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
+; X64-SSE-NEXT:    movaps %xmm0, %xmm1
+; X64-SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3],xmm0[3,3]
+; X64-SSE-NEXT:    cvtss2si %xmm1, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm3
+; X64-SSE-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; X64-SSE-NEXT:    cvtss2si %xmm0, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm1
+; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; X64-SSE-NEXT:    movdqa %xmm2, %xmm0
+; X64-SSE-NEXT:    retq
+  %a = call <4 x i64> @llvm.llrint.v4i64.v4f32(<4 x float> %x)
+  ret <4 x i64> %a
+}
+declare <4 x i64> @llvm.llrint.v4i64.v4f32(<4 x float>)
+
+define <8 x i64> @llrint_v8i64_v8f32(<8 x float> %x) {
+; X64-SSE-LABEL: llrint_v8i64_v8f32:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    movaps %xmm0, %xmm2
+; X64-SSE-NEXT:    cvtss2si %xmm0, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm0
+; X64-SSE-NEXT:    movaps %xmm2, %xmm3
+; X64-SSE-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm2[1,1]
+; X64-SSE-NEXT:    cvtss2si %xmm3, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm3
+; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; X64-SSE-NEXT:    movaps %xmm2, %xmm3
+; X64-SSE-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,3],xmm2[3,3]
+; X64-SSE-NEXT:    cvtss2si %xmm3, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm3
+; X64-SSE-NEXT:    movhlps {{.*#+}} xmm2 = xmm2[1,1]
+; X64-SSE-NEXT:    cvtss2si %xmm2, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm4
+; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm3[0]
+; X64-SSE-NEXT:    cvtss2si %xmm1, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm2
+; X64-SSE-NEXT:    movaps %xmm1, %xmm3
+; X64-SSE-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm1[1,1]
+; X64-SSE-NEXT:    cvtss2si %xmm3, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm3
+; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; X64-SSE-NEXT:    movaps %xmm1, %xmm3
+; X64-SSE-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,3],xmm1[3,3]
+; X64-SSE-NEXT:    cvtss2si %xmm3, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm5
+; X64-SSE-NEXT:    movhlps {{.*#+}} xmm1 = xmm1[1,1]
+; X64-SSE-NEXT:    cvtss2si %xmm1, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm3
+; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; X64-SSE-NEXT:    movdqa %xmm4, %xmm1
+; X64-SSE-NEXT:    retq
+  %a = call <8 x i64> @llvm.llrint.v8i64.v8f32(<8 x float> %x)
+  ret <8 x i64> %a
+}
+declare <8 x i64> @llvm.llrint.v8i64.v8f32(<8 x float>)
+
+define <16 x i64> @llrint_v16i64_v16f32(<16 x float> %x) {
+; X64-SSE-LABEL: llrint_v16i64_v16f32:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    movq %rdi, %rax
+; X64-SSE-NEXT:    cvtss2si %xmm0, %rcx
+; X64-SSE-NEXT:    movq %rcx, %xmm4
+; X64-SSE-NEXT:    movaps %xmm0, %xmm5
+; X64-SSE-NEXT:    shufps {{.*#+}} xmm5 = xmm5[1,1],xmm0[1,1]
+; X64-SSE-NEXT:    cvtss2si %xmm5, %rcx
+; X64-SSE-NEXT:    movq %rcx, %xmm5
+; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm5[0]
+; X64-SSE-NEXT:    movaps %xmm0, %xmm5
+; X64-SSE-NEXT:    shufps {{.*#+}} xmm5 = xmm5[3,3],xmm0[3,3]
+; X64-SSE-NEXT:    cvtss2si %xmm5, %rcx
+; X64-SSE-NEXT:    movq %rcx, %xmm5
+; X64-SSE-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; X64-SSE-NEXT:    cvtss2si %xmm0, %rcx
+; X64-SSE-NEXT:    movq %rcx, %xmm0
+; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm5[0]
+; X64-SSE-NEXT:    cvtss2si %xmm1, %rcx
+; X64-SSE-NEXT:    movq %rcx, %xmm5
+; X64-SSE-NEXT:    movaps %xmm1, %xmm6
+; X64-SSE-NEXT:    shufps {{.*#+}} xmm6 = xmm6[1,1],xmm1[1,1]
+; X64-SSE-NEXT:    cvtss2si %xmm6, %rcx
+; X64-SSE-NEXT:    movq %rcx, %xmm6
+; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm6[0]
+; X64-SSE-NEXT:    movaps %xmm1, %xmm6
+; X64-SSE-NEXT:    shufps {{.*#+}} xmm6 = xmm6[3,3],xmm1[3,3]
+; X64-SSE-NEXT:    cvtss2si %xmm6, %rcx
+; X64-SSE-NEXT:    movq %rcx, %xmm6
+; X64-SSE-NEXT:    movhlps {{.*#+}} xmm1 = xmm1[1,1]
+; X64-SSE-NEXT:    cvtss2si %xmm1, %rcx
+; X64-SSE-NEXT:    movq %rcx, %xmm1
+; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm6[0]
+; X64-SSE-NEXT:    cvtss2si %xmm2, %rcx
+; X64-SSE-NEXT:    movq %rcx, %xmm6
+; X64-SSE-NEXT:    movaps %xmm2, %xmm7
+; X64-SSE-NEXT:    shufps {{.*#+}} xmm7 = xmm7[1,1],xmm2[1,1]
+; X64-SSE-NEXT:    cvtss2si %xmm7, %rcx
+; X64-SSE-NEXT:    movq %rcx, %xmm7
+; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm7[0]
+; X64-SSE-NEXT:    movaps %xmm2, %xmm7
+; X64-SSE-NEXT:    shufps {{.*#+}} xmm7 = xmm7[3,3],xmm2[3,3]
+; X64-SSE-NEXT:    cvtss2si %xmm7, %rcx
+; X64-SSE-NEXT:    movq %rcx, %xmm7
+; X64-SSE-NEXT:    movhlps {{.*#+}} xmm2 = xmm2[1,1]
+; X64-SSE-NEXT:    cvtss2si %xmm2, %rcx
+; X64-SSE-NEXT:    movq %rcx, %xmm2
+; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm7[0]
+; X64-SSE-NEXT:    cvtss2si %xmm3, %rcx
+; X64-SSE-NEXT:    movq %rcx, %xmm7
+; X64-SSE-NEXT:    movaps %xmm3, %xmm8
+; X64-SSE-NEXT:    shufps {{.*#+}} xmm8 = xmm8[1,1],xmm3[1,1]
+; X64-SSE-NEXT:    cvtss2si %xmm8, %rcx
+; X64-SSE-NEXT:    movq %rcx, %xmm8
+; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm7 = xmm7[0],xmm8[0]
+; X64-SSE-NEXT:    movaps %xmm3, %xmm8
+; X64-SSE-NEXT:    shufps {{.*#+}} xmm8 = xmm8[3,3],xmm3[3,3]
+; X64-SSE-NEXT:    cvtss2si %xmm8, %rcx
+; X64-SSE-NEXT:    movq %rcx, %xmm8
+; X64-SSE-NEXT:    movhlps {{.*#+}} xmm3 = xmm3[1,1]
+; X64-SSE-NEXT:    cvtss2si %xmm3, %rcx
+; X64-SSE-NEXT:    movq %rcx, %xmm3
+; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm8[0]
+; X64-SSE-NEXT:    movdqa %xmm3, 112(%rdi)
+; X64-SSE-NEXT:    movdqa %xmm7, 96(%rdi)
+; X64-SSE-NEXT:    movdqa %xmm2, 80(%rdi)
+; X64-SSE-NEXT:    movdqa %xmm6, 64(%rdi)
+; X64-SSE-NEXT:    movdqa %xmm1, 48(%rdi)
+; X64-SSE-NEXT:    movdqa %xmm5, 32(%rdi)
+; X64-SSE-NEXT:    movdqa %xmm0, 16(%rdi)
+; X64-SSE-NEXT:    movdqa %xmm4, (%rdi)
+; X64-SSE-NEXT:    retq
+  %a = call <16 x i64> @llvm.llrint.v16i64.v16f32(<16 x float> %x)
+  ret <16 x i64> %a
+}
+declare <16 x i64> @llvm.llrint.v16i64.v16f32(<16 x float>)
+
+define <1 x i64> @llrint_v1i64_v1f64(<1 x double> %x) {
+; X64-SSE-LABEL: llrint_v1i64_v1f64:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    cvtsd2si %xmm0, %rax
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: llrint_v1i64_v1f64:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vcvtsd2si %xmm0, %rax
+; X64-AVX-NEXT:    retq
+  %a = call <1 x i64> @llvm.llrint.v1i64.v1f64(<1 x double> %x)
+  ret <1 x i64> %a
+}
+declare <1 x i64> @llvm.llrint.v1i64.v1f64(<1 x double>)
+
+define <2 x i64> @llrint_v2i64_v2f64(<2 x double> %x) {
+; X64-SSE-LABEL: llrint_v2i64_v2f64:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    cvtsd2si %xmm0, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm1
+; X64-SSE-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1,1]
+; X64-SSE-NEXT:    cvtsd2si %xmm0, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm0
+; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; X64-SSE-NEXT:    movdqa %xmm1, %xmm0
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: llrint_v2i64_v2f64:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vcvtsd2si %xmm0, %rax
+; X64-AVX-NEXT:    vmovq %rax, %xmm1
+; X64-AVX-NEXT:    vshufpd {{.*#+}} xmm0 = xmm0[1,0]
+; X64-AVX-NEXT:    vcvtsd2si %xmm0, %rax
+; X64-AVX-NEXT:    vmovq %rax, %xmm0
+; X64-AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; X64-AVX-NEXT:    retq
+  %a = call <2 x i64> @llvm.llrint.v2i64.v2f64(<2 x double> %x)
+  ret <2 x i64> %a
+}
+declare <2 x i64> @llvm.llrint.v2i64.v2f64(<2 x double>)
+
+define <4 x i64> @llrint_v4i64_v4f64(<4 x double> %x) {
+; X64-SSE-LABEL: llrint_v4i64_v4f64:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    cvtsd2si %xmm0, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm2
+; X64-SSE-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1,1]
+; X64-SSE-NEXT:    cvtsd2si %xmm0, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm0
+; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0]
+; X64-SSE-NEXT:    cvtsd2si %xmm1, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm3
+; X64-SSE-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1,1]
+; X64-SSE-NEXT:    cvtsd2si %xmm1, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm0
+; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm0[0]
+; X64-SSE-NEXT:    movdqa %xmm2, %xmm0
+; X64-SSE-NEXT:    movdqa %xmm3, %xmm1
+; X64-SSE-NEXT:    retq
+  %a = call <4 x i64> @llvm.llrint.v4i64.v4f64(<4 x double> %x)
+  ret <4 x i64> %a
+}
+declare <4 x i64> @llvm.llrint.v4i64.v4f64(<4 x double>)
+
+define <8 x i64> @llrint_v8i64_v8f64(<8 x double> %x) {
+; X64-SSE-LABEL: llrint_v8i64_v8f64:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    cvtsd2si %xmm0, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm4
+; X64-SSE-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1,1]
+; X64-SSE-NEXT:    cvtsd2si %xmm0, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm0
+; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm0[0]
+; X64-SSE-NEXT:    cvtsd2si %xmm1, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm5
+; X64-SSE-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1,1]
+; X64-SSE-NEXT:    cvtsd2si %xmm1, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm0
+; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm0[0]
+; X64-SSE-NEXT:    cvtsd2si %xmm2, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm6
+; X64-SSE-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1,1]
+; X64-SSE-NEXT:    cvtsd2si %xmm2, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm0
+; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm0[0]
+; X64-SSE-NEXT:    cvtsd2si %xmm3, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm7
+; X64-SSE-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1,1]
+; X64-SSE-NEXT:    cvtsd2si %xmm3, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm0
+; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm7 = xmm7[0],xmm0[0]
+; X64-SSE-NEXT:    movdqa %xmm4, %xmm0
+; X64-SSE-NEXT:    movdqa %xmm5, %xmm1
+; X64-SSE-NEXT:    movdqa %xmm6, %xmm2
+; X64-SSE-NEXT:    movdqa %xmm7, %xmm3
+; X64-SSE-NEXT:    retq
+  %a = call <8 x i64> @llvm.llrint.v8i64.v8f64(<8 x double> %x)
+  ret <8 x i64> %a
+}
+declare <8 x i64> @llvm.llrint.v8i64.v8f64(<8 x double>)
diff --git a/llvm/test/CodeGen/X86/vector-lrint.ll b/llvm/test/CodeGen/X86/vector-lrint.ll
new file mode 100644
index 0000000000..7373cd32df
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vector-lrint.ll
@@ -0,0 +1,429 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=i686-unknown -mattr=sse2 | FileCheck %s --check-prefix=X86-SSE2
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=i686-unknown -mattr=avx | FileCheck %s --check-prefix=X86-AVX
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=i686-unknown -mattr=avx512f | FileCheck %s --check-prefix=X86-AVX
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=x86_64-unknown | FileCheck %s --check-prefix=X64-SSE
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=x86_64-unknown -mattr=avx | FileCheck %s --check-prefix=X64-AVX
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=x86_64-unknown -mattr=avx512f | FileCheck %s --check-prefix=X64-AVX
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=x86_64-unknown | FileCheck %s --check-prefix=X64-SSE
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=x86_64-unknown -mattr=avx | FileCheck %s --check-prefix=X64-AVX
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=x86_64-unknown -mattr=avx512f | FileCheck %s --check-prefix=X64-AVX
+
+define <1 x iXLen> @lrint_v1f32(<1 x float> %x) {
+; X86-SSE2-LABEL: lrint_v1f32:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    cvtss2si {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX-LABEL: lrint_v1f32:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    vcvtss2si {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    retl
+  %a = call <1 x iXLen> @llvm.lrint.v1iXLen.v1f32(<1 x float> %x)
+  ret <1 x iXLen> %a
+}
+declare <1 x iXLen> @llvm.lrint.v1iXLen.v1f32(<1 x float>)
+
+define <2 x iXLen> @lrint_v2f32(<2 x float> %x) {
+; X86-SSE2-LABEL: lrint_v2f32:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movaps %xmm0, %xmm1
+; X86-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3],xmm0[3,3]
+; X86-SSE2-NEXT:    cvtss2si %xmm1, %eax
+; X86-SSE2-NEXT:    movd %eax, %xmm1
+; X86-SSE2-NEXT:    movaps %xmm0, %xmm2
+; X86-SSE2-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm0[1]
+; X86-SSE2-NEXT:    cvtss2si %xmm2, %eax
+; X86-SSE2-NEXT:    movd %eax, %xmm2
+; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; X86-SSE2-NEXT:    cvtss2si %xmm0, %eax
+; X86-SSE2-NEXT:    movd %eax, %xmm1
+; X86-SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; X86-SSE2-NEXT:    cvtss2si %xmm0, %eax
+; X86-SSE2-NEXT:    movd %eax, %xmm0
+; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; X86-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; X86-SSE2-NEXT:    movdqa %xmm1, %xmm0
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX-LABEL: lrint_v2f32:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; X86-AVX-NEXT:    vcvtss2si %xmm1, %eax
+; X86-AVX-NEXT:    vcvtss2si %xmm0, %ecx
+; X86-AVX-NEXT:    vmovd %ecx, %xmm1
+; X86-AVX-NEXT:    vpinsrd $1, %eax, %xmm1, %xmm1
+; X86-AVX-NEXT:    vshufpd {{.*#+}} xmm2 = xmm0[1,0]
+; X86-AVX-NEXT:    vcvtss2si %xmm2, %eax
+; X86-AVX-NEXT:    vpinsrd $2, %eax, %xmm1, %xmm1
+; X86-AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; X86-AVX-NEXT:    vcvtss2si %xmm0, %eax
+; X86-AVX-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm0
+; X86-AVX-NEXT:    retl
+  %a = call <2 x iXLen> @llvm.lrint.v2iXLen.v2f32(<2 x float> %x)
+  ret <2 x iXLen> %a
+}
+declare <2 x iXLen> @llvm.lrint.v2iXLen.v2f32(<2 x float>)
+
+define <4 x iXLen> @lrint_v4f32(<4 x float> %x) {
+; X86-SSE2-LABEL: lrint_v4f32:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movaps %xmm0, %xmm1
+; X86-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3],xmm0[3,3]
+; X86-SSE2-NEXT:    cvtss2si %xmm1, %eax
+; X86-SSE2-NEXT:    movd %eax, %xmm1
+; X86-SSE2-NEXT:    movaps %xmm0, %xmm2
+; X86-SSE2-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm0[1]
+; X86-SSE2-NEXT:    cvtss2si %xmm2, %eax
+; X86-SSE2-NEXT:    movd %eax, %xmm2
+; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; X86-SSE2-NEXT:    cvtss2si %xmm0, %eax
+; X86-SSE2-NEXT:    movd %eax, %xmm1
+; X86-SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; X86-SSE2-NEXT:    cvtss2si %xmm0, %eax
+; X86-SSE2-NEXT:    movd %eax, %xmm0
+; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; X86-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; X86-SSE2-NEXT:    movdqa %xmm1, %xmm0
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX-LABEL: lrint_v4f32:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; X86-AVX-NEXT:    vcvtss2si %xmm1, %eax
+; X86-AVX-NEXT:    vcvtss2si %xmm0, %ecx
+; X86-AVX-NEXT:    vmovd %ecx, %xmm1
+; X86-AVX-NEXT:    vpinsrd $1, %eax, %xmm1, %xmm1
+; X86-AVX-NEXT:    vshufpd {{.*#+}} xmm2 = xmm0[1,0]
+; X86-AVX-NEXT:    vcvtss2si %xmm2, %eax
+; X86-AVX-NEXT:    vpinsrd $2, %eax, %xmm1, %xmm1
+; X86-AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; X86-AVX-NEXT:    vcvtss2si %xmm0, %eax
+; X86-AVX-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm0
+; X86-AVX-NEXT:    retl
+  %a = call <4 x iXLen> @llvm.lrint.v4iXLen.v4f32(<4 x float> %x)
+  ret <4 x iXLen> %a
+}
+declare <4 x iXLen> @llvm.lrint.v4iXLen.v4f32(<4 x float>)
+
+define <8 x iXLen> @lrint_v8f32(<8 x float> %x) {
+; X86-SSE2-LABEL: lrint_v8f32:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movaps %xmm0, %xmm2
+; X86-SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; X86-SSE2-NEXT:    cvtss2si %xmm0, %eax
+; X86-SSE2-NEXT:    movd %eax, %xmm0
+; X86-SSE2-NEXT:    movaps %xmm2, %xmm3
+; X86-SSE2-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm2[1]
+; X86-SSE2-NEXT:    cvtss2si %xmm3, %eax
+; X86-SSE2-NEXT:    movd %eax, %xmm3
+; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
+; X86-SSE2-NEXT:    cvtss2si %xmm2, %eax
+; X86-SSE2-NEXT:    movd %eax, %xmm0
+; X86-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; X86-SSE2-NEXT:    cvtss2si %xmm2, %eax
+; X86-SSE2-NEXT:    movd %eax, %xmm2
+; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; X86-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; X86-SSE2-NEXT:    movaps %xmm1, %xmm2
+; X86-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,3],xmm1[3,3]
+; X86-SSE2-NEXT:    cvtss2si %xmm2, %eax
+; X86-SSE2-NEXT:    movd %eax, %xmm2
+; X86-SSE2-NEXT:    movaps %xmm1, %xmm3
+; X86-SSE2-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm1[1]
+; X86-SSE2-NEXT:    cvtss2si %xmm3, %eax
+; X86-SSE2-NEXT:    movd %eax, %xmm3
+; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; X86-SSE2-NEXT:    cvtss2si %xmm1, %eax
+; X86-SSE2-NEXT:    movd %eax, %xmm2
+; X86-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; X86-SSE2-NEXT:    cvtss2si %xmm1, %eax
+; X86-SSE2-NEXT:    movd %eax, %xmm1
+; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; X86-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; X86-SSE2-NEXT:    movdqa %xmm2, %xmm1
+; X86-SSE2-NEXT:    retl
+  %a = call <8 x iXLen> @llvm.lrint.v8iXLen.v8f32(<8 x float> %x)
+  ret <8 x iXLen> %a
+}
+declare <8 x iXLen> @llvm.lrint.v8iXLen.v8f32(<8 x float>)
+
+define <16 x iXLen> @lrint_v16iXLen_v16f32(<16 x float> %x) {
+  %a = call <16 x iXLen> @llvm.lrint.v16iXLen.v16f32(<16 x float> %x)
+  ret <16 x iXLen> %a
+}
+declare <16 x iXLen> @llvm.lrint.v16iXLen.v16f32(<16 x float>)
+
+define <1 x i64> @lrint_v1f64(<1 x double> %x) {
+; X86-SSE2-LABEL: lrint_v1f64:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE2-NEXT:    .cfi_offset %ebp, -8
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    .cfi_def_cfa_register %ebp
+; X86-SSE2-NEXT:    andl $-8, %esp
+; X86-SSE2-NEXT:    subl $8, %esp
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    movsd %xmm0, (%esp)
+; X86-SSE2-NEXT:    fldl (%esp)
+; X86-SSE2-NEXT:    fistpll (%esp)
+; X86-SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    movd %xmm0, %eax
+; X86-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; X86-SSE2-NEXT:    movd %xmm0, %edx
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    .cfi_def_cfa %esp, 4
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX-LABEL: lrint_v1f64:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    pushl %ebp
+; X86-AVX-NEXT:    .cfi_def_cfa_offset 8
+; X86-AVX-NEXT:    .cfi_offset %ebp, -8
+; X86-AVX-NEXT:    movl %esp, %ebp
+; X86-AVX-NEXT:    .cfi_def_cfa_register %ebp
+; X86-AVX-NEXT:    andl $-8, %esp
+; X86-AVX-NEXT:    subl $8, %esp
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vmovsd %xmm0, (%esp)
+; X86-AVX-NEXT:    fldl (%esp)
+; X86-AVX-NEXT:    fistpll (%esp)
+; X86-AVX-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vmovd %xmm0, %eax
+; X86-AVX-NEXT:    vpextrd $1, %xmm0, %edx
+; X86-AVX-NEXT:    movl %ebp, %esp
+; X86-AVX-NEXT:    popl %ebp
+; X86-AVX-NEXT:    .cfi_def_cfa %esp, 4
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE-LABEL: lrint_v1f64:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    cvtsd2si %xmm0, %rax
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: lrint_v1f64:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vcvtsd2si %xmm0, %rax
+; X64-AVX-NEXT:    retq
+  %a = call <1 x i64> @llvm.lrint.v1i64.v1f64(<1 x double> %x)
+  ret <1 x i64> %a
+}
+declare <1 x i64> @llvm.lrint.v1i64.v1f64(<1 x double>)
+
+define <2 x i64> @lrint_v2f64(<2 x double> %x) {
+; X86-SSE2-LABEL: lrint_v2f64:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE2-NEXT:    .cfi_offset %ebp, -8
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    .cfi_def_cfa_register %ebp
+; X86-SSE2-NEXT:    andl $-8, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    movhps %xmm0, (%esp)
+; X86-SSE2-NEXT:    movlps %xmm0, {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    fldl (%esp)
+; X86-SSE2-NEXT:    fistpll (%esp)
+; X86-SSE2-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    .cfi_def_cfa %esp, 4
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX-LABEL: lrint_v2f64:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    pushl %ebp
+; X86-AVX-NEXT:    .cfi_def_cfa_offset 8
+; X86-AVX-NEXT:    .cfi_offset %ebp, -8
+; X86-AVX-NEXT:    movl %esp, %ebp
+; X86-AVX-NEXT:    .cfi_def_cfa_register %ebp
+; X86-AVX-NEXT:    andl $-8, %esp
+; X86-AVX-NEXT:    subl $16, %esp
+; X86-AVX-NEXT:    vmovlps %xmm0, {{[0-9]+}}(%esp)
+; X86-AVX-NEXT:    vmovhps %xmm0, (%esp)
+; X86-AVX-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-AVX-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-AVX-NEXT:    fldl (%esp)
+; X86-AVX-NEXT:    fistpll (%esp)
+; X86-AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-AVX-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; X86-AVX-NEXT:    vpinsrd $2, (%esp), %xmm0, %xmm0
+; X86-AVX-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; X86-AVX-NEXT:    movl %ebp, %esp
+; X86-AVX-NEXT:    popl %ebp
+; X86-AVX-NEXT:    .cfi_def_cfa %esp, 4
+; X86-AVX-NEXT:    retl
+;
+; X64-SSE-LABEL: lrint_v2f64:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    cvtsd2si %xmm0, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm1
+; X64-SSE-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1,1]
+; X64-SSE-NEXT:    cvtsd2si %xmm0, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm0
+; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; X64-SSE-NEXT:    movdqa %xmm1, %xmm0
+; X64-SSE-NEXT:    retq
+;
+; X64-AVX-LABEL: lrint_v2f64:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vcvtsd2si %xmm0, %rax
+; X64-AVX-NEXT:    vmovq %rax, %xmm1
+; X64-AVX-NEXT:    vshufpd {{.*#+}} xmm0 = xmm0[1,0]
+; X64-AVX-NEXT:    vcvtsd2si %xmm0, %rax
+; X64-AVX-NEXT:    vmovq %rax, %xmm0
+; X64-AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; X64-AVX-NEXT:    retq
+  %a = call <2 x i64> @llvm.lrint.v2i64.v2f64(<2 x double> %x)
+  ret <2 x i64> %a
+}
+declare <2 x i64> @llvm.lrint.v2i64.v2f64(<2 x double>)
+
+define <4 x i64> @lrint_v4f64(<4 x double> %x) {
+; X86-SSE2-LABEL: lrint_v4f64:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE2-NEXT:    .cfi_offset %ebp, -8
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    .cfi_def_cfa_register %ebp
+; X86-SSE2-NEXT:    andl $-8, %esp
+; X86-SSE2-NEXT:    subl $32, %esp
+; X86-SSE2-NEXT:    movhps %xmm0, {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    movlps %xmm0, {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    movhps %xmm1, (%esp)
+; X86-SSE2-NEXT:    movlps %xmm1, {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    fldl (%esp)
+; X86-SSE2-NEXT:    fistpll (%esp)
+; X86-SSE2-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; X86-SSE2-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    .cfi_def_cfa %esp, 4
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE-LABEL: lrint_v4f64:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    cvtsd2si %xmm0, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm2
+; X64-SSE-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1,1]
+; X64-SSE-NEXT:    cvtsd2si %xmm0, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm0
+; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0]
+; X64-SSE-NEXT:    cvtsd2si %xmm1, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm3
+; X64-SSE-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1,1]
+; X64-SSE-NEXT:    cvtsd2si %xmm1, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm0
+; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm0[0]
+; X64-SSE-NEXT:    movdqa %xmm2, %xmm0
+; X64-SSE-NEXT:    movdqa %xmm3, %xmm1
+; X64-SSE-NEXT:    retq
+  %a = call <4 x i64> @llvm.lrint.v4i64.v4f64(<4 x double> %x)
+  ret <4 x i64> %a
+}
+declare <4 x i64> @llvm.lrint.v4i64.v4f64(<4 x double>)
+
+define <8 x i64> @lrint_v8f64(<8 x double> %x) {
+; X86-SSE2-LABEL: lrint_v8f64:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE2-NEXT:    .cfi_offset %ebp, -8
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    .cfi_def_cfa_register %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $80, %esp
+; X86-SSE2-NEXT:    movaps 8(%ebp), %xmm3
+; X86-SSE2-NEXT:    movhps %xmm0, {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    movlps %xmm0, {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    movhps %xmm1, {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    movlps %xmm1, {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    movhps %xmm2, {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    movlps %xmm2, {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    movhps %xmm3, {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    movlps %xmm3, {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE2-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; X86-SSE2-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm2 = mem[0],zero
+; X86-SSE2-NEXT:    movlhps {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm4 = mem[0],zero
+; X86-SSE2-NEXT:    movsd {{.*#+}} xmm3 = mem[0],zero
+; X86-SSE2-NEXT:    movlhps {{.*#+}} xmm3 = xmm3[0],xmm4[0]
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    .cfi_def_cfa %esp, 4
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE-LABEL: lrint_v8f64:
+; X64-SSE:       # %bb.0:
+; X64-SSE-NEXT:    cvtsd2si %xmm0, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm4
+; X64-SSE-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1,1]
+; X64-SSE-NEXT:    cvtsd2si %xmm0, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm0
+; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm0[0]
+; X64-SSE-NEXT:    cvtsd2si %xmm1, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm5
+; X64-SSE-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1,1]
+; X64-SSE-NEXT:    cvtsd2si %xmm1, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm0
+; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm0[0]
+; X64-SSE-NEXT:    cvtsd2si %xmm2, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm6
+; X64-SSE-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1,1]
+; X64-SSE-NEXT:    cvtsd2si %xmm2, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm0
+; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm0[0]
+; X64-SSE-NEXT:    cvtsd2si %xmm3, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm7
+; X64-SSE-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1,1]
+; X64-SSE-NEXT:    cvtsd2si %xmm3, %rax
+; X64-SSE-NEXT:    movq %rax, %xmm0
+; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm7 = xmm7[0],xmm0[0]
+; X64-SSE-NEXT:    movdqa %xmm4, %xmm0
+; X64-SSE-NEXT:    movdqa %xmm5, %xmm1
+; X64-SSE-NEXT:    movdqa %xmm6, %xmm2
+; X64-SSE-NEXT:    movdqa %xmm7, %xmm3
+; X64-SSE-NEXT:    retq
+  %a = call <8 x i64> @llvm.lrint.v8i64.v8f64(<8 x double> %x)
+  ret <8 x i64> %a
+}
+declare <8 x i64> @llvm.lrint.v8i64.v8f64(<8 x double>)
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512bwvl.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512bwvl.ll
index 8f46209689..23c37af1db 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512bwvl.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512bwvl.ll
@@ -173,13 +173,14 @@ define <8 x i32> @PR46393(<8 x i16> %a0, i8 %a1) {
 define i64 @PR55050() {
 ; X86-LABEL: PR55050:
 ; X86:       # %bb.0: # %entry
+; X86-NEXT:    xorl %edx, %edx
 ; X86-NEXT:    xorl %eax, %eax
-; X86-NEXT:    testb %al, %al
+; X86-NEXT:    testb %dl, %dl
 ; X86-NEXT:    jne .LBB10_2
 ; X86-NEXT:  # %bb.1: # %if
 ; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    xorl %edx, %edx
 ; X86-NEXT:  .LBB10_2: # %exit
-; X86-NEXT:    movl %eax, %edx
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: PR55050:
diff --git a/llvm/test/CodeGen/X86/vselect.ll b/llvm/test/CodeGen/X86/vselect.ll
index 0c57f497aa..784d32bde1 100644
--- a/llvm/test/CodeGen/X86/vselect.ll
+++ b/llvm/test/CodeGen/X86/vselect.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,SSE41
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=AVX
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2
 
 ; Verify that we don't emit packed vector shifts instructions if the
 ; condition used by the vector select is a vector of constants.
@@ -425,6 +425,79 @@ define <2 x i64> @test25(<2 x i64> %a, <2 x i64> %b) {
   ret <2 x i64> %1
 }
 
+define <16 x i8> @test26(<16 x i8> %a, <16 x i8> %b) {
+; SSE2-LABEL: test26:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movaps {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; SSE2-NEXT:    andps %xmm2, %xmm1
+; SSE2-NEXT:    andnps %xmm0, %xmm2
+; SSE2-NEXT:    orps %xmm1, %xmm2
+; SSE2-NEXT:    movaps %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: test26:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    movaps {{.*#+}} xmm0 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; SSE41-NEXT:    pblendvb %xmm0, %xmm1, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: test26:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: test26:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastw {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX2-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+  %1 = select <16 x i1> <i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true>, <16 x i8> %a, <16 x i8> %b
+  ret <16 x i8> %1
+}
+
+define <32 x i8> @test27(<32 x i8> %a, <32 x i8> %b) {
+; SSE2-LABEL: test27:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movaps {{.*#+}} xmm4 = [255,0,0,255,255,0,0,255,255,0,0,255,255,0,0,255]
+; SSE2-NEXT:    movaps %xmm4, %xmm5
+; SSE2-NEXT:    andnps %xmm2, %xmm5
+; SSE2-NEXT:    andps %xmm4, %xmm0
+; SSE2-NEXT:    orps %xmm5, %xmm0
+; SSE2-NEXT:    andps %xmm4, %xmm1
+; SSE2-NEXT:    andnps %xmm3, %xmm4
+; SSE2-NEXT:    orps %xmm4, %xmm1
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: test27:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    movaps {{.*#+}} xmm0 = [255,0,0,255,255,0,0,255,255,0,0,255,255,0,0,255]
+; SSE41-NEXT:    pblendvb %xmm0, %xmm4, %xmm2
+; SSE41-NEXT:    pblendvb %xmm0, %xmm1, %xmm3
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm3, %xmm1
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: test27:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vbroadcastss {{.*#+}} ymm2 = [255,0,0,255,255,0,0,255,255,0,0,255,255,0,0,255,255,0,0,255,255,0,0,255,255,0,0,255,255,0,0,255]
+; AVX1-NEXT:    vandnps %ymm1, %ymm2, %ymm1
+; AVX1-NEXT:    vandps %ymm2, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: test27:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [255,0,0,255,255,0,0,255,255,0,0,255,255,0,0,255,255,0,0,255,255,0,0,255,255,0,0,255,255,0,0,255]
+; AVX2-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    retq
+  %1 = select <32 x i1> <i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true>, <32 x i8> %a, <32 x i8> %b
+  ret <32 x i8> %1
+}
+
 define <4 x float> @select_of_shuffles_0(<2 x float> %a0, <2 x float> %b0, <2 x float> %a1, <2 x float> %b1) {
 ; SSE-LABEL: select_of_shuffles_0:
 ; SSE:       # %bb.0:
diff --git a/llvm/test/DebugInfo/ARM/sdag-split-arg.ll b/llvm/test/DebugInfo/ARM/sdag-split-arg.ll
index de1d822a8c..9699c102c0 100644
--- a/llvm/test/DebugInfo/ARM/sdag-split-arg.ll
+++ b/llvm/test/DebugInfo/ARM/sdag-split-arg.ll
@@ -19,8 +19,8 @@ target triple = "thumbv7k-apple-watchos2.0.0"
 ; Function Attrs: optsize ssp
 define i64 @_Z3foox(i64 returned) local_unnamed_addr #0 !dbg !13 {
   tail call void @llvm.dbg.value(metadata i64 %0, metadata !17, metadata !DIExpression()), !dbg !18
-  ; CHECK: @DEBUG_VALUE: foo:offset <- [DW_OP_LLVM_fragment 0 32] $r0
-  ; CHECK: @DEBUG_VALUE: foo:offset <- [DW_OP_LLVM_fragment 32 32] $r1
+  ; CHECK: @DEBUG_VALUE: foo:offset <- [DW_OP_LLVM_fragment 0 32] $r5
+  ; CHECK: @DEBUG_VALUE: foo:offset <- [DW_OP_LLVM_fragment 32 32] $r4
 
   %2 = load i64, ptr @g, align 8, !dbg !19, !tbaa !21
   %3 = icmp eq i64 %2, %0, !dbg !19
diff --git a/llvm/test/DebugInfo/X86/PR20038.ll b/llvm/test/DebugInfo/Generic/PR20038.ll
similarity index 96%
rename from llvm/test/DebugInfo/X86/PR20038.ll
rename to llvm/test/DebugInfo/Generic/PR20038.ll
index 0879cd1680..024a6abf05 100644
--- a/llvm/test/DebugInfo/X86/PR20038.ll
+++ b/llvm/test/DebugInfo/Generic/PR20038.ll
@@ -1,4 +1,7 @@
-; RUN: %llc_dwarf -mtriple=x86_64-unknown-linux-gnu -O0 -filetype=obj -dwarf-linkage-names=All < %s | llvm-dwarfdump -debug-info - | FileCheck %s --implicit-check-not=DW_TAG
+; For some reason, the output when targetting sparc is not quite as expected.
+; XFAIL: target=sparc{{.*}}
+
+; RUN: %llc_dwarf -O0 -filetype=obj -dwarf-linkage-names=All < %s | llvm-dwarfdump -debug-info - | FileCheck %s --implicit-check-not=DW_TAG
 
 ; IR generated from clang -O0 with:
 ; struct C {
diff --git a/llvm/test/DebugInfo/Generic/discriminated-union.ll b/llvm/test/DebugInfo/Generic/discriminated-union.ll
index 6a42fea338..0acf478f65 100644
--- a/llvm/test/DebugInfo/Generic/discriminated-union.ll
+++ b/llvm/test/DebugInfo/Generic/discriminated-union.ll
@@ -25,14 +25,14 @@
 ;         CHECK: DW_AT_alignment
 ;         CHECK: DW_AT_data_member_location [DW_FORM_data1]	(0x00)
 
-%F = type { [0 x i8], i64, [8 x i8] }
+%F = type { [0 x i8], ptr, [8 x i8] }
 %"F::Nope" = type {}
 
 define internal void @_ZN2e34main17h934ff72f9a38d4bbE() unnamed_addr #0 !dbg !5 {
 start:
   %qq = alloca %F, align 8
   call void @llvm.dbg.declare(metadata ptr %qq, metadata !10, metadata !28), !dbg !29
-  store i64 0, ptr %qq, !dbg !29
+  store ptr null, ptr %qq, !dbg !29
   ret void, !dbg !30
 }
 
diff --git a/llvm/test/DebugInfo/Generic/dwarf-public-names.ll b/llvm/test/DebugInfo/Generic/dwarf-public-names.ll
new file mode 100644
index 0000000000..bcb16172b7
--- /dev/null
+++ b/llvm/test/DebugInfo/Generic/dwarf-public-names.ll
@@ -0,0 +1,137 @@
+; RUN: %llc_dwarf -debugger-tune=gdb -filetype=obj -o %t.o < %s
+; RUN: llvm-dwarfdump -debug-pubnames %t.o | FileCheck %s
+; ModuleID = 'dwarf-public-names.cpp'
+;
+; Generated from:
+;
+; struct C {
+;   void member_function();
+;   static int static_member_function();
+;   static int static_member_variable;
+; };
+;
+; int C::static_member_variable = 0;
+;
+; void C::member_function() {
+;   static_member_variable = 0;
+; }
+;
+; int C::static_member_function() {
+;   return static_member_variable;
+; }
+;
+; C global_variable;
+;
+; int global_function() {
+;   return -1;
+; }
+;
+; namespace ns {
+;   void global_namespace_function() {
+;     global_variable.member_function();
+;   }
+;   int global_namespace_variable = 1;
+; }
+
+; Skip the output to the header of the pubnames section.
+; CHECK: debug_pubnames
+; CHECK: version = 0x0002
+
+; Check for each name in the output.
+; CHECK-DAG: "ns"
+; CHECK-DAG: "C::static_member_function"
+; CHECK-DAG: "global_variable"
+; CHECK-DAG: "ns::global_namespace_variable"
+; CHECK-DAG: "ns::global_namespace_function"
+; CHECK-DAG: "global_function"
+; CHECK-DAG: "C::static_member_variable"
+; CHECK-DAG: "C::member_function"
+
+source_filename = "test/DebugInfo/Generic/dwarf-public-names.ll"
+
+%struct.C = type { i8 }
+
+@_ZN1C22static_member_variableE = global i32 0, align 4, !dbg !0
+@global_variable = global %struct.C zeroinitializer, align 1, !dbg !15
+@_ZN2ns25global_namespace_variableE = global i32 1, align 4, !dbg !17
+
+; Function Attrs: nounwind uwtable
+define void @_ZN1C15member_functionEv(ptr %this) #0 align 2 !dbg !23 {
+entry:
+  %this.addr = alloca ptr, align 8
+  store ptr %this, ptr %this.addr, align 8
+  call void @llvm.dbg.declare(metadata ptr %this.addr, metadata !24, metadata !26), !dbg !27
+  %this1 = load ptr, ptr %this.addr
+  store i32 0, ptr @_ZN1C22static_member_variableE, align 4, !dbg !28
+  ret void, !dbg !29
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+; Function Attrs: nounwind uwtable
+define i32 @_ZN1C22static_member_functionEv() #0 align 2 !dbg !30 {
+entry:
+  %0 = load i32, ptr @_ZN1C22static_member_variableE, align 4, !dbg !31
+  ret i32 %0, !dbg !31
+}
+
+; Function Attrs: nounwind uwtable
+define i32 @_Z15global_functionv() #0 !dbg !32 {
+entry:
+  ret i32 -1, !dbg !33
+}
+
+; Function Attrs: nounwind uwtable
+define void @_ZN2ns25global_namespace_functionEv() #0 !dbg !34 {
+entry:
+  call void @_ZN1C15member_functionEv(ptr @global_variable), !dbg !37
+  ret void, !dbg !38
+}
+
+attributes #0 = { nounwind uwtable }
+attributes #1 = { nounwind readnone }
+
+!llvm.dbg.cu = !{!20}
+!llvm.module.flags = !{!22}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = !DIGlobalVariable(name: "static_member_variable", linkageName: "_ZN1C22static_member_variableE", scope: !2, file: !3, line: 7, type: !6, isLocal: false, isDefinition: true, declaration: !5)
+!2 = !DICompositeType(tag: DW_TAG_structure_type, name: "C", file: !3, line: 1, size: 8, align: 8, elements: !4)
+!3 = !DIFile(filename: "dwarf-public-names.cpp", directory: "/usr2/kparzysz/s.hex/t")
+!4 = !{!5, !7, !12}
+!5 = !DIDerivedType(tag: DW_TAG_member, name: "static_member_variable", scope: !2, file: !3, line: 4, baseType: !6, flags: DIFlagStaticMember)
+!6 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!7 = !DISubprogram(name: "member_function", linkageName: "_ZN1C15member_functionEv", scope: !2, file: !3, line: 2, type: !8, isLocal: false, isDefinition: false, scopeLine: 2, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, retainedNodes: !11)
+!8 = !DISubroutineType(types: !9)
+!9 = !{null, !10}
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !2, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer)
+!11 = !{}
+!12 = !DISubprogram(name: "static_member_function", linkageName: "_ZN1C22static_member_functionEv", scope: !2, file: !3, line: 3, type: !13, isLocal: false, isDefinition: false, scopeLine: 3, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, retainedNodes: !11)
+!13 = !DISubroutineType(types: !14)
+!14 = !{!6}
+!15 = !DIGlobalVariableExpression(var: !16, expr: !DIExpression())
+!16 = !DIGlobalVariable(name: "global_variable", scope: null, file: !3, line: 17, type: !2, isLocal: false, isDefinition: true) ; previously: invalid DW_TAG_base_type
+!17 = !DIGlobalVariableExpression(var: !18, expr: !DIExpression())
+!18 = !DIGlobalVariable(name: "global_namespace_variable", linkageName: "_ZN2ns25global_namespace_variableE", scope: !19, file: !3, line: 27, type: !6, isLocal: false, isDefinition: true)
+!19 = !DINamespace(name: "ns", scope: null)
+!20 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !3, producer: "clang version 3.3", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !11, retainedTypes: !11, globals: !21, imports: !11) ; previously: invalid DW_TAG_base_type
+!21 = !{!0, !15, !17}
+!22 = !{i32 1, !"Debug Info Version", i32 3}
+!23 = distinct !DISubprogram(name: "member_function", linkageName: "_ZN1C15member_functionEv", scope: null, file: !3, line: 9, type: !8, isLocal: false, isDefinition: true, scopeLine: 9, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !20, declaration: !7, retainedNodes: !11)
+!24 = !DILocalVariable(name: "this", arg: 1, scope: !23, file: !3, line: 9, type: !25, flags: DIFlagArtificial | DIFlagObjectPointer)
+!25 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !2, size: 64, align: 64)
+!26 = !DIExpression()
+!27 = !DILocation(line: 9, scope: !23)
+!28 = !DILocation(line: 10, scope: !23)
+!29 = !DILocation(line: 11, scope: !23)
+!30 = distinct !DISubprogram(name: "static_member_function", linkageName: "_ZN1C22static_member_functionEv", scope: null, file: !3, line: 13, type: !13, isLocal: false, isDefinition: true, scopeLine: 13, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !20, declaration: !12, retainedNodes: !11)
+!31 = !DILocation(line: 14, scope: !30)
+!32 = distinct !DISubprogram(name: "global_function", linkageName: "_Z15global_functionv", scope: !3, file: !3, line: 19, type: !13, isLocal: false, isDefinition: true, scopeLine: 19, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !20, retainedNodes: !11)
+!33 = !DILocation(line: 20, scope: !32)
+!34 = distinct !DISubprogram(name: "global_namespace_function", linkageName: "_ZN2ns25global_namespace_functionEv", scope: !19, file: !3, line: 24, type: !35, isLocal: false, isDefinition: true, scopeLine: 24, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !20, retainedNodes: !11)
+!35 = !DISubroutineType(types: !36)
+!36 = !{null}
+!37 = !DILocation(line: 25, scope: !34)
+!38 = !DILocation(line: 26, scope: !34)
+
diff --git a/llvm/test/DebugInfo/X86/member-order.ll b/llvm/test/DebugInfo/Generic/member-order.ll
similarity index 96%
rename from llvm/test/DebugInfo/X86/member-order.ll
rename to llvm/test/DebugInfo/Generic/member-order.ll
index 6b39d79ec9..a2965cc0dd 100644
--- a/llvm/test/DebugInfo/X86/member-order.ll
+++ b/llvm/test/DebugInfo/Generic/member-order.ll
@@ -1,4 +1,4 @@
-; RUN: %llc_dwarf -mtriple=x86_64-unknown-linux -filetype=obj -O0 < %s | llvm-dwarfdump -debug-info - | FileCheck %s
+; RUN: %llc_dwarf -filetype=obj -O0 < %s | llvm-dwarfdump -debug-info - | FileCheck %s
 
 ; generated by clang from:
 ; struct foo {
diff --git a/llvm/test/DebugInfo/X86/tu-composite.ll b/llvm/test/DebugInfo/Generic/tu-composite.ll
similarity index 99%
rename from llvm/test/DebugInfo/X86/tu-composite.ll
rename to llvm/test/DebugInfo/Generic/tu-composite.ll
index 9cb4a7ff75..bcfe049e73 100644
--- a/llvm/test/DebugInfo/X86/tu-composite.ll
+++ b/llvm/test/DebugInfo/Generic/tu-composite.ll
@@ -1,4 +1,4 @@
-; RUN: %llc_dwarf -mtriple=x86_64-unknown-linux-gnu -filetype=obj -O0 < %s > %t
+; RUN: %llc_dwarf -filetype=obj -O0 < %s > %t
 ; RUN: llvm-dwarfdump -v -debug-info %t | FileCheck %s
 ; CHECK: [[TYPE:.*]]: DW_TAG_structure_type
 ; Make sure we correctly handle containing type of a struct being a type identifier.
diff --git a/llvm/test/DebugInfo/Generic/univariant-discriminated-union.ll b/llvm/test/DebugInfo/Generic/univariant-discriminated-union.ll
index a1849c715f..628c2f8b1d 100644
--- a/llvm/test/DebugInfo/Generic/univariant-discriminated-union.ll
+++ b/llvm/test/DebugInfo/Generic/univariant-discriminated-union.ll
@@ -12,14 +12,14 @@
 ;       CHECK: DW_AT_alignment
 ;       CHECK: DW_AT_data_member_location [DW_FORM_data1]	(0x00)
 
-%F = type { [0 x i8], i64, [8 x i8] }
+%F = type { [0 x i8], ptr, [8 x i8] }
 %"F::Nope" = type {}
 
 define internal void @_ZN2e34main17h934ff72f9a38d4bbE() unnamed_addr #0 !dbg !5 {
 start:
   %qq = alloca %F, align 8
   call void @llvm.dbg.declare(metadata ptr %qq, metadata !10, metadata !28), !dbg !29
-  store i64 0, ptr %qq, !dbg !29
+  store ptr null, ptr %qq, !dbg !29
   ret void, !dbg !30
 }
 
diff --git a/llvm/test/DebugInfo/M68k/calling-convention.ll b/llvm/test/DebugInfo/M68k/calling-convention.ll
new file mode 100644
index 0000000000..c6e8049771
--- /dev/null
+++ b/llvm/test/DebugInfo/M68k/calling-convention.ll
@@ -0,0 +1,21 @@
+; RUN: llc --mtriple=m68k -filetype=obj %s -o %t
+; RUN: llvm-dwarfdump -v %t | FileCheck %s
+
+; CHECK-LABEL: DW_TAG_subprogram
+; CHECK: DW_AT_calling_convention [DW_FORM_data1]        (DW_CC_LLVM_M68kRTD)
+define m68k_rtdcc void @foo() !dbg !3 {
+entry:
+  ret void
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "clang version 17.0.0 (https://github.com/llvm/llvm-project.git)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "foo.c", directory: "/path/to/file")
+!2 = !{i32 2, !"Debug Info Version", i32 3}
+!3 = distinct !DISubprogram(name: "foo", scope: !4, file: !4, line: 4, type: !5, scopeLine: 4, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !7)
+!4 = !DIFile(filename: "./foo.c", directory: "/path/to/file")
+!5 = !DISubroutineType(cc: DW_CC_LLVM_M68kRTD, types: !6)
+!6 = !{null}
+!7 = !{}
diff --git a/llvm/test/DebugInfo/M68k/lit.local.cfg b/llvm/test/DebugInfo/M68k/lit.local.cfg
new file mode 100644
index 0000000000..dd33fe312c
--- /dev/null
+++ b/llvm/test/DebugInfo/M68k/lit.local.cfg
@@ -0,0 +1,2 @@
+if not "M68k" in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/DebugInfo/Symbolize/XCOFF/xcoff-symbolize-data.ll b/llvm/test/DebugInfo/Symbolize/XCOFF/xcoff-symbolize-data.ll
index 968c175ffd..5432b59d58 100644
--- a/llvm/test/DebugInfo/Symbolize/XCOFF/xcoff-symbolize-data.ll
+++ b/llvm/test/DebugInfo/Symbolize/XCOFF/xcoff-symbolize-data.ll
@@ -35,23 +35,20 @@
 ; CHECK-EMPTY:
 
 ;; Test a function scope static variable.
-;; FIXME: fix the wrong size 152
 ; CHECK: f()::function_global
-; CHECK-NEXT: 144 152
+; CHECK-NEXT: 144 4
 ; CHECK-NEXT: /t.cpp:8
 ; CHECK-EMPTY:
 
 ;; Test a global scope static variable that is used in current compilation unit.
-;; FIXME: fix the wrong size 152
 ; CHECK: beta
-; CHECK-NEXT: 148 152
+; CHECK-NEXT: 148 4
 ; CHECK-NEXT: /t.cpp:13
 ; CHECK-EMPTY:
 
 ;; Test another global scope static variable that is used in current compilation unit.
-;; FIXME: fix the wrong size 152
 ; CHECK: alpha
-; CHECK-NEXT: 152 152
+; CHECK-NEXT: 152 4
 ; CHECK-NEXT: /t.cpp:12
 ; CHECK-EMPTY:
 
diff --git a/llvm/test/DebugInfo/X86/fi-piece.ll b/llvm/test/DebugInfo/X86/fi-piece.ll
index b763b4f126..30144d7875 100644
--- a/llvm/test/DebugInfo/X86/fi-piece.ll
+++ b/llvm/test/DebugInfo/X86/fi-piece.ll
@@ -5,7 +5,7 @@
 ; CHECK: DW_TAG_subprogram
 ; CHECK:   DW_AT_abstract_origin
 ; CHECK: DW_TAG_variable
-; CHECK-NEXT:   DW_AT_location [DW_FORM_exprloc]	(DW_OP_fbreg -8, DW_OP_piece 0x2, DW_OP_fbreg -12, DW_OP_piece 0x2)
+; CHECK-NEXT:   DW_AT_location [DW_FORM_exprloc]	(DW_OP_fbreg -4, DW_OP_piece 0x2, DW_OP_fbreg -8, DW_OP_piece 0x2)
 ; CHECK-NEXT:   DW_AT_abstract_origin {{.*}}"a"
 ; Inlined variable, not to be merged.
 ; CHECK-NOT: DW_TAG
@@ -25,11 +25,10 @@ define void @f() #0 !dbg !8 {
 entry:
   %a = alloca i16, align 4
   %b = alloca i16, align 4
-  %c = alloca { i16, i16 }, align 4
   call void @llvm.dbg.declare(metadata ptr %a, metadata !11, metadata !DIExpression(DW_OP_LLVM_fragment, 0, 16)), !dbg !14
   store i16 1, ptr %a, align 4, !dbg !14
   call void @llvm.dbg.declare(metadata ptr %b, metadata !11, metadata !DIExpression(DW_OP_LLVM_fragment, 16, 16)), !dbg !16
-  call void @llvm.dbg.declare(metadata ptr %c, metadata !11, metadata !13), !dbg !17
+  call void @llvm.dbg.declare(metadata ptr %a, metadata !11, metadata !13), !dbg !17
   store i16 2, ptr %b, align 4, !dbg !17
   ret void
 }
diff --git a/llvm/test/DebugInfo/invalid-sizes.ll b/llvm/test/DebugInfo/invalid-sizes.ll
deleted file mode 100644
index 6562f708d0..0000000000
--- a/llvm/test/DebugInfo/invalid-sizes.ll
+++ /dev/null
@@ -1,80 +0,0 @@
-; RUN: llvm-as -disable-output < %s 2>&1 | FileCheck %s
-
-; CHECK: llvm.dbg.declare has larger fragment size than alloca size
-; CHECK-NEXT: call void @llvm.dbg.declare(metadata ptr %slice.dbg.spill, metadata !23, metadata !DIExpression())
-; CHECK: llvm.dbg.declare has larger fragment size than alloca size
-; CHECK-NEXT: call void @llvm.dbg.declare(metadata ptr %slice.dbg.spill1, metadata !23, metadata !DIExpression())
-
-%"EndianSlice<'_>" = type { { ptr, i64 }, i32, [1 x i32] }
-
-; example::test
-; Function Attrs: nonlazybind uwtable
-define void @_ZN7example4test17h64a501af0fe536ddE(ptr align 1 %s.0, i64 %s.1) unnamed_addr #0 !dbg !7 {
-start:
-  %slice.dbg.spill1 = alloca i32, align 4
-  %slice.dbg.spill = alloca { ptr, i64 }, align 8
-  %s.dbg.spill = alloca { ptr, i64 }, align 8
-  %_2 = alloca %"EndianSlice<'_>", align 8
-  %0 = getelementptr inbounds { ptr, i64 }, ptr %s.dbg.spill, i32 0, i32 0
-  store ptr %s.0, ptr %0, align 8
-  %1 = getelementptr inbounds { ptr, i64 }, ptr %s.dbg.spill, i32 0, i32 1
-  store i64 %s.1, ptr %1, align 8
-  call void @llvm.dbg.declare(metadata ptr %s.dbg.spill, metadata !22, metadata !DIExpression()), !dbg !33
-  %2 = getelementptr inbounds { ptr, i64 }, ptr %slice.dbg.spill, i32 0, i32 0, !dbg !34
-  store ptr %s.0, ptr %2, align 8, !dbg !34
-  %3 = getelementptr inbounds { ptr, i64 }, ptr %slice.dbg.spill, i32 0, i32 1, !dbg !34
-  store i64 %s.1, ptr %3, align 8, !dbg !34
-  call void @llvm.dbg.declare(metadata ptr %slice.dbg.spill, metadata !23, metadata !DIExpression()), !dbg !35
-  store i32 1, ptr %slice.dbg.spill1, align 4, !dbg !34
-  call void @llvm.dbg.declare(metadata ptr %slice.dbg.spill1, metadata !23, metadata !DIExpression()), !dbg !35
-  ret void, !dbg !36
-}
-
-; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
-declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
-
-attributes #0 = { nonlazybind uwtable "probe-stack"="inline-asm" "target-cpu"="x86-64" }
-attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
-
-!llvm.module.flags = !{!0, !1, !2, !3}
-!llvm.ident = !{!4}
-!llvm.dbg.cu = !{!5}
-
-!0 = !{i32 8, !"PIC Level", i32 2}
-!1 = !{i32 2, !"RtLibUseGOT", i32 1}
-!2 = !{i32 2, !"Dwarf Version", i32 4}
-!3 = !{i32 2, !"Debug Info Version", i32 3}
-!4 = !{!"rustc version 1.74.0-nightly (5c6a7e71c 2023-08-20)"}
-!5 = distinct !DICompileUnit(language: DW_LANG_Rust, file: !6, producer: "clang LLVM (rustc version 1.74.0-nightly (5c6a7e71c 2023-08-20))", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false)
-!6 = !DIFile(filename: "/app/example.rs/@/example.a6c375ed18e8f6d3-cgu.0", directory: "/app")
-!7 = distinct !DISubprogram(name: "test", linkageName: "_ZN7example4test17h64a501af0fe536ddE", scope: !9, file: !8, line: 9, type: !10, scopeLine: 9, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !5, templateParams: !20, retainedNodes: !21)
-!8 = !DIFile(filename: "example.rs", directory: "/app", checksumkind: CSK_MD5, checksum: "bd53c9e80c244adbeae5aa0d57de599d")
-!9 = !DINamespace(name: "example", scope: null)
-!10 = !DISubroutineType(types: !11)
-!11 = !{null, !12}
-!12 = !DICompositeType(tag: DW_TAG_structure_type, name: "&[u8]", file: !13, size: 128, align: 64, elements: !14, templateParams: !20, identifier: "4f7d759e2003ffb713a77bd933fd0146")
-!13 = !DIFile(filename: "<unknown>", directory: "")
-!14 = !{!15, !18}
-!15 = !DIDerivedType(tag: DW_TAG_member, name: "data_ptr", scope: !12, file: !13, baseType: !16, size: 64, align: 64)
-!16 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !17, size: 64, align: 64, dwarfAddressSpace: 0)
-!17 = !DIBasicType(name: "u8", size: 8, encoding: DW_ATE_unsigned)
-!18 = !DIDerivedType(tag: DW_TAG_member, name: "length", scope: !12, file: !13, baseType: !19, size: 64, align: 64, offset: 64)
-!19 = !DIBasicType(name: "usize", size: 64, encoding: DW_ATE_unsigned)
-!20 = !{}
-!21 = !{!22, !23}
-!22 = !DILocalVariable(name: "s", arg: 1, scope: !7, file: !8, line: 9, type: !12)
-!23 = !DILocalVariable(name: "slice", scope: !24, file: !8, line: 10, type: !25, align: 8)
-!24 = distinct !DILexicalBlock(scope: !7, file: !8, line: 10, column: 5)
-!25 = !DICompositeType(tag: DW_TAG_structure_type, name: "EndianSlice", scope: !9, file: !13, size: 192, align: 64, elements: !26, templateParams: !20, identifier: "f1b6e593370159e9df4228aa26ace4b5")
-!26 = !{!27, !28}
-!27 = !DIDerivedType(tag: DW_TAG_member, name: "slice", scope: !25, file: !13, baseType: !12, size: 128, align: 64)
-!28 = !DIDerivedType(tag: DW_TAG_member, name: "endian", scope: !25, file: !13, baseType: !29, size: 32, align: 32, offset: 128)
-!29 = !DICompositeType(tag: DW_TAG_structure_type, name: "Endian", scope: !9, file: !13, size: 32, align: 32, elements: !30, templateParams: !20, identifier: "a76092aada82685a5b963f3da7ae1bd9")
-!30 = !{!31}
-!31 = !DIDerivedType(tag: DW_TAG_member, name: "__0", scope: !29, file: !13, baseType: !32, size: 32, align: 32)
-!32 = !DIBasicType(name: "i32", size: 32, encoding: DW_ATE_signed)
-!33 = !DILocation(line: 9, column: 13, scope: !7)
-!34 = !DILocation(line: 10, column: 17, scope: !7)
-!35 = !DILocation(line: 10, column: 9, scope: !24)
-!36 = !DILocation(line: 11, column: 2, scope: !37)
-!37 = !DILexicalBlockFile(scope: !7, file: !8, discriminator: 0)
diff --git a/llvm/test/ExecutionEngine/2010-01-15-UndefValue.ll b/llvm/test/ExecutionEngine/2010-01-15-UndefValue.ll
index c208490f1b..5376fa2329 100644
--- a/llvm/test/ExecutionEngine/2010-01-15-UndefValue.ll
+++ b/llvm/test/ExecutionEngine/2010-01-15-UndefValue.ll
@@ -4,5 +4,6 @@ define i32 @main() {
        %a = add i32 0, undef
        %b = fadd float 0.0, undef
        %c = fadd double 0.0, undef
+       %d = insertvalue {i32, [4 x i32]} undef, i32 1, 1, 2
        ret i32 0
 }
diff --git a/llvm/test/ExecutionEngine/JITLink/ppc64/ELF_ppc64_relocations.s b/llvm/test/ExecutionEngine/JITLink/ppc64/ELF_ppc64_relocations.s
index 7e39a20ef6..8f28a8662c 100644
--- a/llvm/test/ExecutionEngine/JITLink/ppc64/ELF_ppc64_relocations.s
+++ b/llvm/test/ExecutionEngine/JITLink/ppc64/ELF_ppc64_relocations.s
@@ -1,3 +1,4 @@
+# REQUIRES: system-linux
 # RUN: rm -rf %t && mkdir -p %t
 # RUN: llvm-mc --triple=powerpc64le-unknown-linux-gnu --filetype=obj -o \
 # RUN:   %t/elf_reloc.o --defsym LE=1 %s
@@ -8,6 +9,8 @@
 # RUN:              --abs external_addr14_func=0x0880 \
 # RUN:              --abs external_addr16_data=0x6000 \
 # RUN:              --abs external_addr32_data=0x36668840 \
+# RUN:              --abs pcrel_external_var=0x36668860 \
+# RUN:              --abs pcrel_external_tls=0x36668880 \
 # RUN:              --check %s %t/elf_reloc.o
 # RUN: llvm-mc --triple=powerpc64-unknown-linux-gnu --filetype=obj -o \
 # RUN:   %t/elf_reloc.o %s
@@ -18,6 +21,8 @@
 # RUN:              --abs external_addr14_func=0x0880 \
 # RUN:              --abs external_addr16_data=0x6000 \
 # RUN:              --abs external_addr32_data=0x36668840 \
+# RUN:              --abs pcrel_external_var=0x36668860 \
+# RUN:              --abs pcrel_external_tls=0x36668880 \
 # RUN:              --check %s %t/elf_reloc.o
 
 # jitlink-check: section_addr(elf_reloc.o, $__GOT) + 0x8000 = __TOC__
@@ -240,6 +245,35 @@ reloc_rel16:
   blr
   .size reloc_rel16, .-reloc_rel16
 
+# Check R_PPC64_GOT_PCREL34
+# jitlink-check: (got_addr(elf_reloc.o, pcrel_external_var) - reloc_got_pcrel34)[33:0] = \
+# jitlink-check:   ((((*{4}(reloc_got_pcrel34)) & 0x3ffff) << 16) | ((*{4}(reloc_got_pcrel34 + 4)) & 0xffff))[33:0]
+  .global reloc_got_pcrel34
+  .p2align 4
+  .type reloc_got_pcrel34,@function
+reloc_got_pcrel34:
+  pld 3,pcrel_external_var@got@pcrel(0),1
+.Lpcrel0:
+  .reloc .Lpcrel0-8,R_PPC64_PCREL_OPT,.-(.Lpcrel0-8)
+  blr
+  .size reloc_got_pcrel34,.-reloc_got_pcrel34
+
+  .global reloc_tlsgd_pcrel34
+  .p2align 4
+  .type reloc_tlsgd_pcrel34,@function
+reloc_tlsgd_pcrel34:
+  mflr 0
+  std 0, 16(1)
+  stdu 1, -32(1)
+  paddi 3, 0, pcrel_external_tls@got@tlsgd@pcrel, 1
+  bl __tls_get_addr@notoc(a@tlsgd)
+  lwa 3, 0(3)
+  addi 1, 1, 32
+  ld 0, 16(1)
+  mtlr 0
+  blr
+  .size reloc_tlsgd_pcrel34,.-reloc_tlsgd_pcrel34
+
   .type	.L.str,@object
 	.section	.rodata.str1.1,"aMS",@progbits,1
 .L.str:
diff --git a/llvm/test/ExecutionEngine/OrcLazy/minimal-throw-catch.ll b/llvm/test/ExecutionEngine/OrcLazy/minimal-throw-catch.ll
index b1e0aea05a..7bbaa0575a 100644
--- a/llvm/test/ExecutionEngine/OrcLazy/minimal-throw-catch.ll
+++ b/llvm/test/ExecutionEngine/OrcLazy/minimal-throw-catch.ll
@@ -4,7 +4,7 @@
 ; Basic correctness testing for eh-frame processing and registration.
 
 source_filename = "minimal-throw-catch.cpp"
-target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.14.0"
 
 @_ZTIi = external constant ptr
diff --git a/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_constant_address_space.ll b/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_constant_address_space.ll
index bf2038994a..911e8021a7 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_constant_address_space.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_constant_address_space.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
 ; RUN: opt < %s -passes=asan -S | FileCheck %s
 target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8"
 target triple = "amdgcn-amd-amdhsa"
@@ -5,30 +6,31 @@ target triple = "amdgcn-amd-amdhsa"
 @x = addrspace(4) global [2 x i32] zeroinitializer, align 4
 
 define protected amdgpu_kernel void @constant_load(i64 %i) sanitize_address {
-entry:
-; CHECK-LABEL: @constant_load
-; CHECK-NOT: load
-;
-; CHECK:   %[[LOAD_ADDR:[^ ]*]] = ptrtoint ptr addrspace(4) %a to i64
-; CHECK:   lshr i64 %[[LOAD_ADDR]], 3
-; CHECK:   add i64 %{{.*}}, 2147450880
-; CHECK:   %[[LOAD_SHADOW_PTR:[^ ]*]] = inttoptr
-; CHECK:   %[[LOAD_SHADOW:[^ ]*]] = load i8, ptr %[[LOAD_SHADOW_PTR]]
-; CHECK:   icmp ne i8
-; CHECK:   br i1 %{{.*}}, label %{{.*}}, label %{{.*}}
-;
-; CHECK:   add i64 %{{.*}}, 3
-; CHECK:   trunc i64 %{{.*}} to i8
-; CHECK:   icmp sge i8 %{{.*}}, %[[LOAD_SHADOW]]
-; CHECK:   br i1 %{{.*}}, label %{{.*}}, label %{{.*}}
+; CHECK-LABEL: define protected amdgpu_kernel void @constant_load(
+; CHECK-SAME: i64 [[I:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(4) @x, i64 0, i64 [[I]]
+; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoint ptr addrspace(4) [[A]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], 2147450880
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr [[TMP3]], align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[TMP6:%.*]], label [[TMP12:%.*]], !prof [[PROF1:![0-9]+]]
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = and i64 [[TMP0]], 7
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[TMP7]], 3
+; CHECK-NEXT:    [[TMP9:%.*]] = trunc i64 [[TMP8]] to i8
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp sge i8 [[TMP9]], [[TMP4]]
+; CHECK-NEXT:    br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP12]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__asan_report_load4(i64 [[TMP0]]) #[[ATTR3:[0-9]+]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    [[Q:%.*]] = load i32, ptr addrspace(4) [[A]], align 4
+; CHECK-NEXT:    ret void
 ;
-; The crash block reports the error.
-; CHECK:   call void @__asan_report_load4(i64 %[[LOAD_ADDR]])
-; CHECK:   unreachable
-;
-; The actual load.
-; CHECK:   load i32, ptr addrspace(4) %a
-; CHECK:   ret void
+entry:
 
   %a = getelementptr inbounds [2 x i32], ptr  addrspace(4) @x, i64 0, i64 %i
   %q = load i32, ptr addrspace(4) %a, align 4
diff --git a/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_generic_address_space.ll b/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_generic_address_space.ll
index 465d7c9c4e..34b7f04592 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_generic_address_space.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_generic_address_space.ll
@@ -1,38 +1,42 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
 ; RUN: opt < %s -passes=asan -S | FileCheck %s
 target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8"
 target triple = "amdgcn-amd-amdhsa"
 
 define protected amdgpu_kernel void @generic_store(ptr addrspace(1) %p, i32 %i) sanitize_address {
-entry:
-; CHECK-LABEL: @generic_store
-; CHECK-NOT: store
-; CHECK:   %[[GENERIC_ADDR:[^ ]*]] = addrspacecast ptr addrspace(1) %p to ptr
-; CHECK:   call i1 @llvm.amdgcn.is.shared(ptr %[[GENERIC_ADDR]])
-; CHECK:   call i1 @llvm.amdgcn.is.private(ptr %[[GENERIC_ADDR]])
-; CHECK:   or
-; CHECK:   xor i1 %{{.*}}, true
-; CHECK:   br i1 %{{.*}}, label %{{.*}}, label %{{.*}}
-;
-; CHECK:   %[[STORE_ADDR:[^ ]*]] = ptrtoint ptr %q to i64
-; CHECK:   lshr i64 %[[STORE_ADDR]], 3
-; CHECK:   add i64 %{{.*}}, 2147450880
-; CHECK:   %[[STORE_SHADOW_PTR:[^ ]*]] = inttoptr
-; CHECK:   %[[STORE_SHADOW:[^ ]*]] = load i8, ptr %[[STORE_SHADOW_PTR]]
-; CHECK:   icmp ne i8
-; CHECK:   br i1 %{{.*}}, label %{{.*}}, label %{{.*}}
-;
-; CHECK:   add i64 %{{.*}}, 3
-; CHECK:   trunc i64 %{{.*}} to i8
-; CHECK:   icmp sge i8 %{{.*}}, %[[STORE_SHADOW]]
-; CHECK:   br i1 %{{.*}}, label %{{.*}}, label %{{.*}}
+; CHECK-LABEL: define protected amdgpu_kernel void @generic_store(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i32 [[I:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[Q:%.*]] = addrspacecast ptr addrspace(1) [[P]] to ptr
+; CHECK-NEXT:    [[TMP0:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr [[Q]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[Q]])
+; CHECK-NEXT:    [[TMP2:%.*]] = or i1 [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i1 [[TMP2]], true
+; CHECK-NEXT:    br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP18:%.*]]
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[Q]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = lshr i64 [[TMP5]], 3
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[TMP6]], 2147450880
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[TMP9:%.*]] = load i8, ptr [[TMP8]], align 1
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne i8 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP17:%.*]], !prof [[PROF0:![0-9]+]]
+; CHECK:       11:
+; CHECK-NEXT:    [[TMP12:%.*]] = and i64 [[TMP5]], 7
+; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[TMP12]], 3
+; CHECK-NEXT:    [[TMP14:%.*]] = trunc i64 [[TMP13]] to i8
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp sge i8 [[TMP14]], [[TMP9]]
+; CHECK-NEXT:    br i1 [[TMP15]], label [[TMP16:%.*]], label [[TMP17]]
+; CHECK:       16:
+; CHECK-NEXT:    call void @__asan_report_store4(i64 [[TMP5]]) #[[ATTR3:[0-9]+]]
+; CHECK-NEXT:    unreachable
+; CHECK:       17:
+; CHECK-NEXT:    br label [[TMP18]]
+; CHECK:       18:
+; CHECK-NEXT:    store i32 0, ptr [[Q]], align 4
+; CHECK-NEXT:    ret void
 ;
-; The crash block reports the error.
-; CHECK:   call void @__asan_report_store4(i64 %[[STORE_ADDR]])
-; CHECK:   unreachable
-;
-; The actual store.
-; CHECK:   store i32 0, ptr %q
-; CHECK:   ret void
+entry:
 
   %q = addrspacecast ptr addrspace(1) %p to ptr
   store i32 0, ptr %q, align 4
@@ -40,35 +44,39 @@ entry:
 }
 
 define protected amdgpu_kernel void @generic_load(ptr addrspace(1) %p, i32 %i) sanitize_address {
-entry:
-; CHECK-LABEL: @generic_load
-; CHECK-NOT: load
-; CHECK:   call i1 @llvm.amdgcn.is.shared(ptr %[[GENERIC_ADDR]])
-; CHECK:   call i1 @llvm.amdgcn.is.private(ptr %[[GENERIC_ADDR]])
-; CHECK:   or
-; CHECK:   xor i1 %{{.*}}, true
-; CHECK:   br i1 %{{.*}}, label %{{.*}}, label %{{.*}}
-;
-; CHECK:   %[[STORE_ADDR:[^ ]*]] = ptrtoint ptr %q to i64
-; CHECK:   lshr i64 %[[STORE_ADDR]], 3
-; CHECK:   add i64 %{{.*}}, 2147450880
-; CHECK:   %[[STORE_SHADOW_PTR:[^ ]*]] = inttoptr
-; CHECK:   %[[STORE_SHADOW:[^ ]*]] = load i8, ptr %[[STORE_SHADOW_PTR]]
-; CHECK:   icmp ne i8
-; CHECK:   br i1 %{{.*}}, label %{{.*}}, label %{{.*}}
-;
-; CHECK:   add i64 %{{.*}}, 3
-; CHECK:   trunc i64 %{{.*}} to i8
-; CHECK:   icmp sge i8 %{{.*}}, %[[STORE_SHADOW]]
-; CHECK:   br i1 %{{.*}}, label %{{.*}}, label %{{.*}}
+; CHECK-LABEL: define protected amdgpu_kernel void @generic_load(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i32 [[I:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[Q:%.*]] = addrspacecast ptr addrspace(1) [[P]] to ptr
+; CHECK-NEXT:    [[TMP0:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr [[Q]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[Q]])
+; CHECK-NEXT:    [[TMP2:%.*]] = or i1 [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i1 [[TMP2]], true
+; CHECK-NEXT:    br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP18:%.*]]
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[Q]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = lshr i64 [[TMP5]], 3
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[TMP6]], 2147450880
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[TMP9:%.*]] = load i8, ptr [[TMP8]], align 1
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne i8 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP17:%.*]], !prof [[PROF0]]
+; CHECK:       11:
+; CHECK-NEXT:    [[TMP12:%.*]] = and i64 [[TMP5]], 7
+; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[TMP12]], 3
+; CHECK-NEXT:    [[TMP14:%.*]] = trunc i64 [[TMP13]] to i8
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp sge i8 [[TMP14]], [[TMP9]]
+; CHECK-NEXT:    br i1 [[TMP15]], label [[TMP16:%.*]], label [[TMP17]]
+; CHECK:       16:
+; CHECK-NEXT:    call void @__asan_report_load4(i64 [[TMP5]]) #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       17:
+; CHECK-NEXT:    br label [[TMP18]]
+; CHECK:       18:
+; CHECK-NEXT:    [[R:%.*]] = load i32, ptr [[Q]], align 4
+; CHECK-NEXT:    ret void
 ;
-; The crash block reports the error.
-; CHECK:   call void @__asan_report_load4(i64 %[[STORE_ADDR]])
-; CHECK:   unreachable
-;
-; The actual store.
-; CHECK:   load i32, ptr %q
-; CHECK:   ret void
+entry:
 
   %q = addrspacecast ptr addrspace(1) %p to ptr
   %r = load i32, ptr %q, align 4
diff --git a/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_global_address_space.ll b/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_global_address_space.ll
index a051b41970..d8708e7448 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_global_address_space.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_global_address_space.ll
@@ -1,62 +1,63 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
 ; RUN: opt < %s -passes=asan -S | FileCheck %s
 target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8"
 target triple = "amdgcn-amd-amdhsa"
 
 define protected amdgpu_kernel void @global_store(ptr addrspace(1) %p, i32 %i) sanitize_address {
-entry:
-; CHECK-LABEL: @global_store
-; CHECK-NOT: store
-;
-; CHECK:   %[[STORE_ADDR:[^ ]*]] = ptrtoint ptr addrspace(1) %p to i64
-; CHECK:   lshr i64 %[[STORE_ADDR]], 3
-; CHECK:   add i64 %{{.*}}, 2147450880
-; CHECK:   %[[STORE_SHADOW_PTR:[^ ]*]] = inttoptr
-; CHECK:   %[[STORE_SHADOW:[^ ]*]] = load i8, ptr %[[STORE_SHADOW_PTR]]
-; CHECK:   icmp ne i8
-; CHECK:   br i1 %{{.*}}, label %{{.*}}, label %{{.*}}
-;
-; CHECK:   add i64 %{{.*}}, 3
-; CHECK:   trunc i64 %{{.*}} to i8
-; CHECK:   icmp sge i8 %{{.*}}, %[[STORE_SHADOW]]
-; CHECK:   br i1 %{{.*}}, label %{{.*}}, label %{{.*}}
+; CHECK-LABEL: define protected amdgpu_kernel void @global_store(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i32 [[I:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoint ptr addrspace(1) [[P]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], 2147450880
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr [[TMP3]], align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[TMP6:%.*]], label [[TMP12:%.*]], !prof [[PROF0:![0-9]+]]
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = and i64 [[TMP0]], 7
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[TMP7]], 3
+; CHECK-NEXT:    [[TMP9:%.*]] = trunc i64 [[TMP8]] to i8
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp sge i8 [[TMP9]], [[TMP4]]
+; CHECK-NEXT:    br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP12]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__asan_report_store4(i64 [[TMP0]]) #[[ATTR3:[0-9]+]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    store i32 0, ptr addrspace(1) [[P]], align 4
+; CHECK-NEXT:    ret void
 ;
-; The crash block reports the error.
-; CHECK:   call void @__asan_report_store4(i64 %[[STORE_ADDR]])
-; CHECK:   unreachable
-;
-; The actual store.
-; CHECK:   store i32 0, ptr addrspace(1) %p
-; CHECK:   ret void
+entry:
 
   store i32 0, ptr addrspace(1) %p, align 4
   ret void
 }
 
 define protected amdgpu_kernel void @global_load(ptr addrspace(1) %p, i32 %i) sanitize_address {
-entry:
-; CHECK-LABEL: @global_load
-; CHECK-NOT: load
-;
-; CHECK:   %[[LOAD_ADDR:[^ ]*]] = ptrtoint ptr addrspace(1) %p to i64
-; CHECK:   lshr i64 %[[LOAD_ADDR]], 3
-; CHECK:   add i64 %{{.*}}, 2147450880
-; CHECK:   %[[LOAD_SHADOW_PTR:[^ ]*]] = inttoptr
-; CHECK:   %[[LOAD_SHADOW:[^ ]*]] = load i8, ptr %[[LOAD_SHADOW_PTR]]
-; CHECK:   icmp ne i8
-; CHECK:   br i1 %{{.*}}, label %{{.*}}, label %{{.*}}
-;
-; CHECK:   add i64 %{{.*}}, 3
-; CHECK:   trunc i64 %{{.*}} to i8
-; CHECK:   icmp sge i8 %{{.*}}, %[[LOAD_SHADOW]]
-; CHECK:   br i1 %{{.*}}, label %{{.*}}, label %{{.*}}
+; CHECK-LABEL: define protected amdgpu_kernel void @global_load(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i32 [[I:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoint ptr addrspace(1) [[P]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], 2147450880
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr [[TMP3]], align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[TMP6:%.*]], label [[TMP12:%.*]], !prof [[PROF0]]
+; CHECK:       6:
+; CHECK-NEXT:    [[TMP7:%.*]] = and i64 [[TMP0]], 7
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[TMP7]], 3
+; CHECK-NEXT:    [[TMP9:%.*]] = trunc i64 [[TMP8]] to i8
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp sge i8 [[TMP9]], [[TMP4]]
+; CHECK-NEXT:    br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP12]]
+; CHECK:       11:
+; CHECK-NEXT:    call void @__asan_report_load4(i64 [[TMP0]]) #[[ATTR3]]
+; CHECK-NEXT:    unreachable
+; CHECK:       12:
+; CHECK-NEXT:    [[Q:%.*]] = load i32, ptr addrspace(1) [[P]], align 4
+; CHECK-NEXT:    ret void
 ;
-; The crash block reports the error.
-; CHECK:   call void @__asan_report_load4(i64 %[[LOAD_ADDR]])
-; CHECK:   unreachable
-;
-; The actual load.
-; CHECK:   load i32, ptr addrspace(1) %p
-; CHECK:   ret void
+entry:
 
   %q = load i32, ptr addrspace(1) %p, align 4
   ret void
diff --git a/llvm/test/Instrumentation/AddressSanitizer/basic.ll b/llvm/test/Instrumentation/AddressSanitizer/basic.ll
index 068d6d18cd..2064db5a09 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/basic.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/basic.ll
@@ -210,8 +210,10 @@ define void @test_swifterror_3() sanitize_address {
 
 ;; ctor/dtor have the nounwind attribute. See uwtable.ll, they additionally have
 ;; the uwtable attribute with the module flag "uwtable".
-; CHECK: define internal void @asan.module_ctor() #[[#ATTR:]] {{(comdat )?}}{
+; CHECK: define internal void @asan.module_ctor() #[[#ATTR:]] comdat {
 ; CHECK: call void @__asan_init()
+;; __asan_register_elf_globals is called even if this module does not contain instrumented global variables.
+; CHECK: call void @__asan_register_elf_globals(i64 ptrtoint (ptr @___asan_globals_registered to i64), i64 ptrtoint (ptr @__start_asan_globals to i64), i64 ptrtoint (ptr @__stop_asan_globals to i64))
 
 ; CHECK: attributes #[[#ATTR]] = { nounwind }
 
diff --git a/llvm/test/Instrumentation/AddressSanitizer/global_metadata_array.ll b/llvm/test/Instrumentation/AddressSanitizer/global_metadata_array.ll
index 0e4d1f168d..1617bf9b67 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/global_metadata_array.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/global_metadata_array.ll
@@ -1,8 +1,12 @@
-; RUN: opt < %s -passes=asan -asan-globals-live-support=0 -mtriple=x86_64-unknown-linux-gnu -S | FileCheck --check-prefix=CHECK %s
-; RUN: opt < %s -passes=asan -asan-globals-live-support=0 -mtriple=x86_64-apple-macosx10.11.0 -S | FileCheck --check-prefix=CHECK %s
-; RUN: opt < %s -passes=asan -asan-globals-live-support=0 -mtriple=x86_64-pc-windows-msvc19.0.24215 -S | FileCheck --check-prefix=CHECK %s
-; RUN: opt < %s -passes=asan -asan-globals-live-support=0 -asan-mapping-scale=5 -mtriple=x86_64-unknown-linux-gnu -S | FileCheck --check-prefixes=CHECK,CHECK-S5 %s
+; RUN: rm -rf %t && split-file %s %t && cd %t
+; RUN: opt < a.ll -passes=asan -asan-globals-live-support=0 -mtriple=x86_64-unknown-linux-gnu -S | FileCheck --check-prefix=CHECK %s
+; RUN: opt < a.ll -passes=asan -asan-globals-live-support=0 -mtriple=x86_64-apple-macosx10.11.0 -S | FileCheck --check-prefix=CHECK %s
+; RUN: opt < a.ll -passes=asan -asan-globals-live-support=0 -mtriple=x86_64-pc-windows-msvc19.0.24215 -S | FileCheck --check-prefix=CHECK %s
+; RUN: opt < a.ll -passes=asan -asan-globals-live-support=0 -asan-mapping-scale=5 -mtriple=x86_64-unknown-linux-gnu -S | FileCheck --check-prefixes=CHECK,CHECK-S5 %s
 
+; RUN: opt < empty.ll -passes=asan -asan-globals-live-support=0 -mtriple=x86_64-unknown-linux-gnu -S | FileCheck --check-prefix=ELF-NOGC %s
+
+;--- a.ll
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 ; Globals:
@@ -59,3 +63,10 @@ attributes #1 = { nounwind sanitize_address "less-precise-fpmad"="false" "frame-
 !7 = !{!"/tmp/asan-globals.cpp", i32 7, i32 5}
 !8 = !{!"/tmp/asan-globals.cpp", i32 12, i32 14}
 !9 = !{!"/tmp/asan-globals.cpp", i32 14, i32 25}
+
+;; In the presence of instrumented global variables, asan.module_ctor do not use comdat.
+; CHECK: define internal void @asan.module_ctor() #[[#ATTR:]] {
+
+; ELF-NOGC: define internal void @asan.module_ctor() #[[#ATTR:]] comdat {
+
+;--- empty.ll
diff --git a/llvm/test/Instrumentation/AddressSanitizer/global_with_comdat.ll b/llvm/test/Instrumentation/AddressSanitizer/global_with_comdat.ll
index 47bb1f102e..699b8287d3 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/global_with_comdat.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/global_with_comdat.ll
@@ -4,10 +4,14 @@
 ; We keep using comdats for garbage collection if odr indicators are
 ; enabled as indicator symbols will cause link time odr violations.
 ; This is to fix PR 47925.
-;
-; RUN: opt < %s -passes=asan -asan-globals-live-support=1 -asan-use-odr-indicator=0 -S | FileCheck %s --check-prefixes=CHECK,NOCOMDAT
+; RUN: rm -rf %t && split-file %s %t && cd %t
+; RUN: opt < a.ll -passes=asan -asan-globals-live-support=1 -asan-use-odr-indicator=0 -S | FileCheck %s --check-prefixes=CHECK,NOCOMDAT
 ; Check that enabling odr indicators enables comdat for globals.
-; RUN: opt < %s -passes=asan -asan-globals-live-support=1 -S | FileCheck %s --check-prefixes=CHECK,COMDAT
+; RUN: opt < a.ll -passes=asan -asan-globals-live-support=1 -S | FileCheck %s --check-prefixes=CHECK,COMDAT
+
+; RUN: opt < no_module_id.ll -passes=asan -S | FileCheck %s --check-prefix=NOMODULEID
+
+;--- a.ll
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
@@ -87,3 +91,43 @@ attributes #1 = { nounwind sanitize_address "less-precise-fpmad"="false" "frame-
 !7 = !{!"/tmp/asan-globals.cpp", i32 7, i32 5}
 !8 = !{!"/tmp/asan-globals.cpp", i32 12, i32 14}
 !9 = !{!"/tmp/asan-globals.cpp", i32 14, i32 25}
+
+;--- no_module_id.ll
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; NOMODULEID:      $asan.module_ctor = comdat any
+; NOMODULEID:      $asan.module_dtor = comdat any
+
+;; Don't place the instrumented globals in a comdat when the unique module ID is empty.
+; NOMODULEID:      @.str = internal constant { [4 x i8], [28 x i8] } { [4 x i8] c"str\00", [28 x i8] zeroinitializer }, align 32
+; NOMODULEID:      @_ZL3buf = internal global { [4 x i8], [28 x i8] } zeroinitializer, align 32
+; NOMODULEID:      @__asan_global_.str = private global {{.*}}, section "asan_globals", !associated !0
+; NOMODULEID:      @__asan_global__ZL3buf = private global {{.*}}, section "asan_globals", !associated !1
+; NOMODULEID:      @llvm.compiler.used = appending global [4 x ptr] [ptr @.str, ptr @_ZL3buf, ptr @__asan_global_.str, ptr @__asan_global__ZL3buf]
+
+; NOMODULEID:      define internal void @asan.module_ctor() #[[#]] comdat {
+; NOMODULEID-NEXT:   call void @__asan_init()
+; NOMODULEID-NEXT:   call void @__asan_version_mismatch_check_v8()
+; NOMODULEID-NEXT:   call void @__asan_register_elf_globals(i64 ptrtoint (ptr @___asan_globals_registered to i64), i64 ptrtoint (ptr @__start_asan_globals to i64), i64 ptrtoint (ptr @__stop_asan_globals to i64))
+; NOMODULEID-NEXT:   ret void
+; NOMODULEID-NEXT: }
+
+; NOMODULEID:      !0 = !{ptr @.str}
+; NOMODULEID:      !1 = !{ptr @_ZL3buf}
+
+@.str = private unnamed_addr constant [4 x i8] c"str\00", align 1
+@_ZL3buf = internal unnamed_addr global [4 x i8] zeroinitializer, align 1
+@llvm.global_ctors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 65535, ptr @_GLOBAL__sub_I_a.cc, ptr null }]
+
+declare void @ext(ptr noundef)
+
+; Function Attrs: uwtable
+define internal void @_GLOBAL__sub_I_a.cc() #2 section ".text.startup" {
+entry:
+  %0 = load i8, ptr @_ZL3buf, align 1
+  %inc = add i8 %0, 1
+  store i8 %inc, ptr @_ZL3buf, align 1
+  tail call void @ext(ptr noundef nonnull @.str)
+  ret void
+}
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/str-nobuiltin.ll b/llvm/test/Instrumentation/HWAddressSanitizer/str-nobuiltin.ll
new file mode 100644
index 0000000000..8faa906027
--- /dev/null
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/str-nobuiltin.ll
@@ -0,0 +1,33 @@
+; Test marking string functions as nobuiltin in address sanitizer.
+;
+; RUN: opt < %s -passes=hwasan -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare ptr @memchr(ptr %a, i32 %b, i64 %c)
+declare i32 @memcmp(ptr %a, ptr %b, i64 %c)
+declare i32 @strcmp(ptr %a, ptr %b)
+declare ptr @strcpy(ptr %a, ptr %b)
+declare ptr @stpcpy(ptr %a, ptr %b)
+declare i64 @strlen(ptr %a)
+declare i64 @strnlen(ptr %a, i64 %b)
+
+; CHECK: call{{.*}}@memchr{{.*}} #[[ATTR:[0-9]+]]
+; CHECK: call{{.*}}@memcmp{{.*}} #[[ATTR]]
+; CHECK: call{{.*}}@strcmp{{.*}} #[[ATTR]]
+; CHECK: call{{.*}}@strcpy{{.*}} #[[ATTR]]
+; CHECK: call{{.*}}@stpcpy{{.*}} #[[ATTR]]
+; CHECK: call{{.*}}@strlen{{.*}} #[[ATTR]]
+; CHECK: call{{.*}}@strnlen{{.*}} #[[ATTR]]
+; attributes #[[ATTR]] = { nobuiltin }
+
+define void @f1(ptr %a, ptr %b) nounwind uwtable sanitize_hwaddress {
+  tail call ptr @memchr(ptr %a, i32 1, i64 12)
+  tail call i32 @memcmp(ptr %a, ptr %b, i64 12)
+  tail call i32 @strcmp(ptr %a, ptr %b)
+  tail call ptr @strcpy(ptr %a, ptr %b)
+  tail call ptr @stpcpy(ptr %a, ptr %b)
+  tail call i64 @strlen(ptr %a)
+  tail call i64 @strnlen(ptr %a, i64 12)
+  ret void
+}
diff --git a/llvm/test/Linker/type-unique-odr-a.ll b/llvm/test/Linker/type-unique-odr-a.ll
index 831d42b35c..9a911c7f2d 100644
--- a/llvm/test/Linker/type-unique-odr-a.ll
+++ b/llvm/test/Linker/type-unique-odr-a.ll
@@ -1,4 +1,4 @@
-; REQUIRES: x86-registered-target
+; REQUIRES: object-emission
 ;
 ; RUN: llvm-link %s %p/type-unique-odr-b.ll -S -o - \
 ; RUN:   | %llc_dwarf -dwarf-linkage-names=All -filetype=obj -O0 \
@@ -56,8 +56,6 @@
 
 ; ModuleID = 'type-unique-odr-a.cpp'
 
-target triple = "x86_64-unknown-linux-gnu"
-
 %class.A = type { i32 }
 
 ; Function Attrs: nounwind
diff --git a/llvm/test/Linker/type-unique-odr-b.ll b/llvm/test/Linker/type-unique-odr-b.ll
index 92b489a10a..0526b2a6db 100644
--- a/llvm/test/Linker/type-unique-odr-b.ll
+++ b/llvm/test/Linker/type-unique-odr-b.ll
@@ -19,8 +19,6 @@
 
 ; ModuleID = 'type-unique-odr-b.cpp'
 
-target triple = "x86_64-unknown-linux-gnu"
-
 %class.A = type { i32 }
 
 ; Function Attrs: nounwind
diff --git a/llvm/test/Linker/type-unique-simple2-a.ll b/llvm/test/Linker/type-unique-simple2-a.ll
index 28dddc298d..1032a43fd6 100644
--- a/llvm/test/Linker/type-unique-simple2-a.ll
+++ b/llvm/test/Linker/type-unique-simple2-a.ll
@@ -1,4 +1,4 @@
-; REQUIRES: x86-registered-target
+; REQUIRES: object-emission
 ;
 ; RUN: llvm-link %s %p/type-unique-simple2-b.ll -S -o - | %llc_dwarf -filetype=obj -O0 | llvm-dwarfdump -v -debug-info - | FileCheck %s
 ;
@@ -26,8 +26,6 @@
 
 ; ModuleID = 'a.cpp'
 
-target triple = "x86_64-unknown-linux-gnu"
-
 %class.A = type { ptr }
 
 @_ZTV1A = external unnamed_addr constant [4 x ptr]
diff --git a/llvm/test/Linker/type-unique-simple2-b.ll b/llvm/test/Linker/type-unique-simple2-b.ll
index 0b61b7fb72..38263314de 100644
--- a/llvm/test/Linker/type-unique-simple2-b.ll
+++ b/llvm/test/Linker/type-unique-simple2-b.ll
@@ -10,8 +10,6 @@
 ; target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 ; target triple = "x86_64-apple-macosx10.9.0"
 
-target triple = "x86_64-unknown-linux-gnu"
-
 %class.A = type { ptr }
 
 @_ZTV1A = unnamed_addr constant [4 x ptr] [ptr null, ptr @_ZTI1A, ptr @_ZN1A6setFooEv, ptr @_ZN1A6getFooEv]
diff --git a/llvm/test/MC/AArch64/FP8/system-regs.s b/llvm/test/MC/AArch64/FP8/system-regs.s
new file mode 100644
index 0000000000..4a396d4dff
--- /dev/null
+++ b/llvm/test/MC/AArch64/FP8/system-regs.s
@@ -0,0 +1,32 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+fpmr < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+fpmr < %s \
+// RUN:        | llvm-objdump -d --mattr=+fpmr - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+fpmr < %s \
+// RUN:        | llvm-objdump --mattr=-fpmr -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+// --------------------------------------------------------------------------//
+// read
+
+mrs x3, FPMR
+// CHECK-INST: mrs x3, FPMR
+// CHECK-ENCODING: [0x43,0x44,0x3b,0xd5]
+// CHECK-ERROR: expected readable system register
+// CHECK-UNKNOWN: d53b4443   mrs   x3, S3_3_C4_C4_2
+
+mrs x3, ID_AA64FPFR0_EL1
+// CHECK-INST: mrs x3, ID_AA64FPFR0_EL1
+// CHECK-ENCODING: [0xe3,0x04,0x38,0xd5]
+// CHECK-ERROR: expected readable system register
+// CHECK-UNKNOWN: d53804e3   mrs   x3, S3_0_C0_C4_7
+
+// --------------------------------------------------------------------------//
+// write
+
+msr FPMR, x3
+// CHECK-INST: msr FPMR, x3
+// CHECK-ENCODING: [0x43,0x44,0x1b,0xd5]
+// CHECK-ERROR: expected writable system register or pstate
+// CHECK-UNKNOWN: d51b4443   msr   S3_3_C4_C4_2, x3
diff --git a/llvm/test/MC/AArch64/arm64-system-encoding.s b/llvm/test/MC/AArch64/arm64-system-encoding.s
index b30a0c4b0a..313ec91177 100644
--- a/llvm/test/MC/AArch64/arm64-system-encoding.s
+++ b/llvm/test/MC/AArch64/arm64-system-encoding.s
@@ -336,6 +336,7 @@ foo:
   mrs x3, ID_AA64ISAR0_EL1
   mrs x3, ID_AA64ISAR1_EL1
   mrs x3, ID_AA64ISAR2_EL1
+  mrs x3, ID_AA64ISAR3_EL1
   mrs x3, ID_AA64MMFR0_EL1
   mrs x3, ID_AA64MMFR1_EL1
   mrs x3, ID_AA64MMFR2_EL1
@@ -556,6 +557,7 @@ foo:
 ; CHECK: mrs x3, ID_AA64ISAR0_EL1       ; encoding: [0x03,0x06,0x38,0xd5]
 ; CHECK: mrs x3, ID_AA64ISAR1_EL1       ; encoding: [0x23,0x06,0x38,0xd5]
 ; CHECK: mrs x3, ID_AA64ISAR2_EL1       ; encoding: [0x43,0x06,0x38,0xd5]
+; CHECK: mrs x3, ID_AA64ISAR3_EL1       ; encoding: [0x63,0x06,0x38,0xd5]
 ; CHECK: mrs x3, ID_AA64MMFR0_EL1       ; encoding: [0x03,0x07,0x38,0xd5]
 ; CHECK: mrs x3, ID_AA64MMFR1_EL1       ; encoding: [0x23,0x07,0x38,0xd5]
 ; CHECK: mrs x3, ID_AA64MMFR2_EL1       ; encoding: [0x43,0x07,0x38,0xd5]
diff --git a/llvm/test/MC/AMDGPU/gfx1030_err.s b/llvm/test/MC/AMDGPU/gfx1030_err.s
index 10ffa9b73a..ba8784a39c 100644
--- a/llvm/test/MC/AMDGPU/gfx1030_err.s
+++ b/llvm/test/MC/AMDGPU/gfx1030_err.s
@@ -141,12 +141,6 @@ ds_write_src2_b32 v1 offset:65535
 ds_write_src2_b64 v1 offset:65535
 // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
 
-buffer_atomic_csub v5, off, s[8:11], s3 offset:4095
-// GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: instruction must use glc
-
-global_atomic_csub v2, v[0:1], v2, off offset:100 slc
-// GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: instruction must use glc
-
 image_msaa_load v[1:4], v5, s[8:15] dmask:0xf dim:SQ_RSRC_IMG_1D
 // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: invalid dim; must be MSAA type
 
diff --git a/llvm/test/MC/AMDGPU/gfx1030_new.s b/llvm/test/MC/AMDGPU/gfx1030_new.s
index e03bc9b0df..eb7424caf6 100644
--- a/llvm/test/MC/AMDGPU/gfx1030_new.s
+++ b/llvm/test/MC/AMDGPU/gfx1030_new.s
@@ -30,18 +30,30 @@ global_store_dword v254, v1, s[2:3] offset:16
 global_atomic_csub v2, v[0:1], v2, off offset:100 glc slc
 // GFX10: encoding: [0x64,0x80,0xd3,0xdc,0x00,0x02,0x7d,0x02]
 
+global_atomic_csub v[0:1], v2, off offset:100 slc
+// GFX10: encoding: [0x64,0x80,0xd2,0xdc,0x00,0x02,0x7d,0x00]
+
 global_atomic_csub v2, v[0:1], v2, off glc
 // GFX10: encoding: [0x00,0x80,0xd1,0xdc,0x00,0x02,0x7d,0x02]
 
+global_atomic_csub v[0:1], v2, off
+// GFX10: encoding: [0x00,0x80,0xd0,0xdc,0x00,0x02,0x7d,0x00]
+
 global_atomic_csub v2, v0, v2, s[2:3] glc
 // GFX10: encoding: [0x00,0x80,0xd1,0xdc,0x00,0x02,0x02,0x02]
 
+global_atomic_csub v0, v2, s[2:3]
+// GFX10: encoding: [0x00,0x80,0xd0,0xdc,0x00,0x02,0x02,0x00]
+
 global_atomic_csub v2, v0, v2, s[2:3] offset:100 glc slc
 // GFX10: encoding: [0x64,0x80,0xd3,0xdc,0x00,0x02,0x02,0x02]
 
 buffer_atomic_csub v5, off, s[8:11], s3 glc
 // GFX10: encoding: [0x00,0x40,0xd0,0xe0,0x00,0x05,0x02,0x03]
 
+buffer_atomic_csub v5, off, s[8:11], s3
+// GFX10: encoding: [0x00,0x00,0xd0,0xe0,0x00,0x05,0x02,0x03]
+
 buffer_atomic_csub v5, off, s[8:11], s3 offset:4095 glc
 // GFX10: encoding: [0xff,0x4f,0xd0,0xe0,0x00,0x05,0x02,0x03]
 
@@ -51,12 +63,21 @@ buffer_atomic_csub v5, off, s[8:11], -1 offset:4095 glc
 buffer_atomic_csub v5, v0, s[8:11], s3 offen offset:4095 glc
 // GFX10: encoding: [0xff,0x5f,0xd0,0xe0,0x00,0x05,0x02,0x03]
 
+buffer_atomic_csub v5, v0, s[8:11], s3 offen offset:4095
+// GFX10: encoding: [0xff,0x1f,0xd0,0xe0,0x00,0x05,0x02,0x03]
+
 buffer_atomic_csub v5, v0, s[8:11], s3 idxen offset:4095 glc
 // GFX10: encoding: [0xff,0x6f,0xd0,0xe0,0x00,0x05,0x02,0x03]
 
+buffer_atomic_csub v5, v0, s[8:11], s3 idxen offset:4095
+// GFX10: encoding: [0xff,0x2f,0xd0,0xe0,0x00,0x05,0x02,0x03]
+
 buffer_atomic_csub v5, off, s[8:11], s3 glc slc
 // GFX10: encoding: [0x00,0x40,0xd0,0xe0,0x00,0x05,0x42,0x03]
 
+buffer_atomic_csub v5, off, s[8:11], s3 slc
+// GFX10: encoding: [0x00,0x00,0xd0,0xe0,0x00,0x05,0x42,0x03]
+
 s_getreg_b32 s2, hwreg(HW_REG_SHADER_CYCLES)
 // GFX10: encoding: [0x1d,0xf8,0x02,0xb9]
 
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_flat.s b/llvm/test/MC/AMDGPU/gfx11_asm_flat.s
index 300988646d..17deec42bc 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_flat.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_flat.s
@@ -1536,6 +1536,9 @@ global_atomic_cmpswap_x2 v[254:255], v255, v[252:255], ttmp[14:15] offset:-4096
 global_atomic_csub v5, v[1:2], v2, off glc
 // GFX11: [0x00,0x40,0xde,0xdc,0x01,0x02,0x7c,0x05]
 
+global_atomic_csub v[1:2], v2, off
+// GFX11: [0x00,0x00,0xde,0xdc,0x01,0x02,0x7c,0x00]
+
 global_atomic_csub v5, v[254:255], v2, off glc
 // GFX11: [0x00,0x40,0xde,0xdc,0xfe,0x02,0x7c,0x05]
 
@@ -1563,9 +1566,15 @@ global_atomic_csub v5, v255, v2, vcc offset:4095 glc
 global_atomic_csub v255, v255, v255, ttmp[14:15] offset:-4096 glc slc dlc
 // GFX11: [0x00,0xf0,0xde,0xdc,0xff,0xff,0x7a,0xff]
 
+global_atomic_csub v255, v255, ttmp[14:15] offset:-4096 slc dlc
+// GFX11: [0x00,0xb0,0xde,0xdc,0xff,0xff,0x7a,0x00]
+
 global_atomic_csub_u32 v5, v[1:2], v2, off glc
 // GFX11: [0x00,0x40,0xde,0xdc,0x01,0x02,0x7c,0x05]
 
+global_atomic_csub_u32 v[1:2], v2, off
+// GFX11: [0x00,0x00,0xde,0xdc,0x01,0x02,0x7c,0x00]
+
 global_atomic_csub_u32 v5, v[254:255], v2, off glc
 // GFX11: [0x00,0x40,0xde,0xdc,0xfe,0x02,0x7c,0x05]
 
@@ -1593,6 +1602,9 @@ global_atomic_csub_u32 v5, v255, v2, vcc offset:4095 glc
 global_atomic_csub_u32 v255, v255, v255, ttmp[14:15] offset:-4096 glc slc dlc
 // GFX11: [0x00,0xf0,0xde,0xdc,0xff,0xff,0x7a,0xff]
 
+global_atomic_csub_u32 v255, v255, ttmp[14:15] offset:-4096 slc dlc
+// GFX11: [0x00,0xb0,0xde,0xdc,0xff,0xff,0x7a,0x00]
+
 global_atomic_dec v[1:2], v2, off
 // GFX11: [0x00,0x00,0x02,0xdd,0x01,0x02,0x7c,0x00]
 
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_mubuf.s b/llvm/test/MC/AMDGPU/gfx11_asm_mubuf.s
index a5efde4219..6dfb237713 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_mubuf.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_mubuf.s
@@ -3091,12 +3091,12 @@ buffer_atomic_cmpswap_f32 v[5:6], off, s[8:11], s3 offset:4095 dlc
 buffer_atomic_cmpswap_f32 v[5:6], off, s[8:11], s3 offset:4095 glc slc dlc
 // GFX11: encoding: [0xff,0x7f,0x40,0xe1,0x00,0x05,0x02,0x03]
 
-buffer_atomic_csub_u32 v5, off, s[8:11], s3 offset:4095
-// GFX11-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction must use glc
-
 buffer_atomic_csub_u32 v255, off, s[8:11], s3 offset:4095 glc
 // GFX11: encoding: [0xff,0x4f,0xdc,0xe0,0x00,0xff,0x02,0x03]
 
+buffer_atomic_csub_u32 v255, off, s[8:11], s3 offset:4095
+// GFX11: encoding: [0xff,0x0f,0xdc,0xe0,0x00,0xff,0x02,0x03]
+
 buffer_atomic_csub_u32 v5, off, s[12:15], s3 offset:4095 glc
 // GFX11: encoding: [0xff,0x4f,0xdc,0xe0,0x00,0x05,0x03,0x03]
 
@@ -3142,12 +3142,21 @@ buffer_atomic_csub_u32 v5, off, s[8:11], s3 offset:4095 glc
 buffer_atomic_csub_u32 v5, off, s[8:11], s3 offset:4095 glc slc
 // GFX11: encoding: [0xff,0x5f,0xdc,0xe0,0x00,0x05,0x02,0x03]
 
+buffer_atomic_csub_u32 v5, off, s[8:11], s3 offset:4095 slc
+// GFX11: encoding: [0xff,0x1f,0xdc,0xe0,0x00,0x05,0x02,0x03]
+
 buffer_atomic_csub_u32 v5, off, s[8:11], s3 offset:4095 glc dlc
 // GFX11: encoding: [0xff,0x6f,0xdc,0xe0,0x00,0x05,0x02,0x03]
 
+buffer_atomic_csub_u32 v5, off, s[8:11], s3 offset:4095 dlc
+// GFX11: encoding: [0xff,0x2f,0xdc,0xe0,0x00,0x05,0x02,0x03]
+
 buffer_atomic_csub_u32 v5, off, s[8:11], s3 offset:4095 glc slc dlc
 // GFX11: encoding: [0xff,0x7f,0xdc,0xe0,0x00,0x05,0x02,0x03]
 
+buffer_atomic_csub_u32 v5, off, s[8:11], s3 offset:4095 slc dlc
+// GFX11: encoding: [0xff,0x3f,0xdc,0xe0,0x00,0x05,0x02,0x03]
+
 buffer_atomic_dec_u32 v5, off, s[8:11], s3 offset:4095
 // GFX11: encoding: [0xff,0x0f,0x00,0xe1,0x00,0x05,0x02,0x03]
 
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_mubuf_alias.s b/llvm/test/MC/AMDGPU/gfx11_asm_mubuf_alias.s
index 5456d68f4c..9cf7503e5f 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_mubuf_alias.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_mubuf_alias.s
@@ -2210,12 +2210,12 @@ buffer_atomic_fcmpswap v[5:6], off, s[8:11], s3 offset:4095 slc
 buffer_atomic_fcmpswap v[5:6], off, s[8:11], s3 offset:4095 glc slc
 // GFX11: encoding: [0xff,0x5f,0x40,0xe1,0x00,0x05,0x02,0x03]
 
-buffer_atomic_csub v5, off, s[8:11], s3 offset:4095
-// GFX11-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction must use glc
-
 buffer_atomic_csub v255, off, s[8:11], s3 offset:4095 glc
 // GFX11: encoding: [0xff,0x4f,0xdc,0xe0,0x00,0xff,0x02,0x03]
 
+buffer_atomic_csub v255, off, s[8:11], s3 offset:4095
+// GFX11: encoding: [0xff,0x0f,0xdc,0xe0,0x00,0xff,0x02,0x03]
+
 buffer_atomic_csub v5, off, s[12:15], s3 offset:4095 glc
 // GFX11: encoding: [0xff,0x4f,0xdc,0xe0,0x00,0x05,0x03,0x03]
 
@@ -2261,6 +2261,9 @@ buffer_atomic_csub v5, off, s[8:11], s3 offset:4095 glc
 buffer_atomic_csub v5, off, s[8:11], s3 offset:4095 glc slc
 // GFX11: encoding: [0xff,0x5f,0xdc,0xe0,0x00,0x05,0x02,0x03]
 
+buffer_atomic_csub v5, off, s[8:11], s3 offset:4095 slc
+// GFX11: encoding: [0xff,0x1f,0xdc,0xe0,0x00,0x05,0x02,0x03]
+
 buffer_atomic_dec v5, off, s[8:11], s3 offset:4095
 // GFX11: encoding: [0xff,0x0f,0x00,0xe1,0x00,0x05,0x02,0x03]
 
diff --git a/llvm/test/MC/AMDGPU/literals.s b/llvm/test/MC/AMDGPU/literals.s
index 8e5e8fd188..00575619c4 100644
--- a/llvm/test/MC/AMDGPU/literals.s
+++ b/llvm/test/MC/AMDGPU/literals.s
@@ -889,3 +889,49 @@ v_pk_add_f16 v255, private_base, private_limit
 // NOSICIVI: :[[@LINE+2]]:{{[0-9]+}}: error: instruction not supported on this GPU
 // NOGFX9: :[[@LINE+1]]:{{[0-9]+}}: error: invalid operand (violates constant bus restrictions)
 v_pk_add_f16 v255, vccz, execz
+
+//---------------------------------------------------------------------------//
+// check dummy lit() syntax for sp3 compatibility.
+//---------------------------------------------------------------------------//
+
+// SICI: v_sqrt_f32_e32 v2, 0x7b                 ; encoding: [0xff,0x66,0x04,0x7e,0x7b,0x00,0x00,0x00]
+// GFX89: v_sqrt_f32_e32 v2, 0x7b                 ; encoding: [0xff,0x4e,0x04,0x7e,0x7b,0x00,0x00,0x00]
+v_sqrt_f32 v2, lit(123)
+
+// SICI: v_sqrt_f32_e32 v2, 0x7b                 ; encoding: [0xff,0x66,0x04,0x7e,0x7b,0x00,0x00,0x00]
+// GFX89: v_sqrt_f32_e32 v2, 0x7b                 ; encoding: [0xff,0x4e,0x04,0x7e,0x7b,0x00,0x00,0x00]
+v_sqrt_f32 v2, abs(lit(123))
+
+// SICI: v_sqrt_f32_e32 v2, 0x42f60000           ; encoding: [0xff,0x66,0x04,0x7e,0x00,0x00,0xf6,0x42
+// GFX89: v_sqrt_f32_e32 v2, 0x42f60000           ; encoding: [0xff,0x4e,0x04,0x7e,0x00,0x00,0xf6,0x42]
+v_sqrt_f32 v2, lit(123.0)
+
+// SICI: v_sqrt_f64_e32 v[2:3], 0x405ec000       ; encoding: [0xff,0x68,0x04,0x7e,0x00,0xc0,0x5e,0x40]
+// GFX89: v_sqrt_f64_e32 v[2:3], 0x405ec000       ; encoding: [0xff,0x50,0x04,0x7e,0x00,0xc0,0x5e,0x40]
+v_sqrt_f64 v[2:3], lit(123.0)
+
+// SICI: v_sqrt_f64_e32 v[2:3], 0x7b             ; encoding: [0xff,0x68,0x04,0x7e,0x7b,0x00,0x00,0x00]
+// GFX89: v_sqrt_f64_e32 v[2:3], 0x7b             ; encoding: [0xff,0x50,0x04,0x7e,0x7b,0x00,0x00,0x00]
+v_sqrt_f64 v[2:3], lit(123)
+
+// NOSICI: :[[@LINE+2]]:{{[0-9]+}}: error: expected left paren after lit
+// NOGFX89: :[[@LINE+1]]:{{[0-9]+}}: error: expected left paren after lit
+v_sqrt_f32 v2, lit 123.0
+
+// NOSICI: :[[@LINE+2]]:{{[0-9]+}}: error: expected closing parentheses
+// NOGFX89: :[[@LINE+1]]:{{[0-9]+}}: error: expected closing parentheses
+v_sqrt_f32 v2, lit(123.0
+
+// NOSICI: :[[@LINE+2]]:{{[0-9]+}}: error: expected immediate with lit modifier
+// NOGFX89: :[[@LINE+1]]:{{[0-9]+}}: error: expected immediate with lit modifier
+v_sqrt_f32 v2, lit(v1)
+
+// Make sure lit() is accepted on operands without modifiers.
+
+// SICI: v_madak_f32 v4, 0x7e8, v8, 0x7e8        ; encoding: [0xff,0x10,0x08,0x42,0xe8,0x07,0x00,0x00]
+// GFX89: v_madak_f32 v4, 0x7e8, v8, 0x7e8        ; encoding: [0xff,0x10,0x08,0x30,0xe8,0x07,0x00,0x00]
+v_madak_f32 v4, lit(0x7e8), v8, lit(0x7e8)
+
+// NOSICI: :[[@LINE+2]]:{{[0-9]+}}: error: not a valid operand.
+// NOGFX89: :[[@LINE+1]]:{{[0-9]+}}: error: not a valid operand.
+v_madak_f32 v4, lit(lit(0x7e8)), v8, lit(0x7e8)
diff --git a/llvm/test/MC/ARM/inst-directive-it-vpt.s b/llvm/test/MC/ARM/inst-directive-it-vpt.s
new file mode 100644
index 0000000000..8550d720ed
--- /dev/null
+++ b/llvm/test/MC/ARM/inst-directive-it-vpt.s
@@ -0,0 +1,26 @@
+// RUN: llvm-mc %s -triple armv8m.main -mattr=+mve -filetype asm -o - | FileCheck %s
+
+  .thumb
+
+// CHECK: it      eq
+// CHECK: .inst.n 0x3001
+// CHECK: add.w r0, r0, #1
+  it eq
+  .inst.n 0x3001 // addeq r0, #1
+  add r0, #1
+
+// CHECK: vpst
+// CHECK: .inst.w 0xef220844
+// CHECK: vadd.i32 q0, q1, q2
+  vpst
+  .inst.w 0xef220844 // vaddt.i32 q0, q1, q2
+  vadd.i32 q0, q1, q2
+
+// CHECK: ite eq
+// CHECK: .inst.n 0x3001
+// CHECK: addne r0, #1
+// CHECK: add.w r0, r0, #1
+  ite eq
+  .inst.n 0x3001 // addeq r0, #1
+  addne r0, #1
+  add r0, #1
diff --git a/llvm/test/MC/Disassembler/AMDGPU/decode-err.txt b/llvm/test/MC/Disassembler/AMDGPU/decode-err.txt
index bf2c55ef2e..22fec90318 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/decode-err.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/decode-err.txt
@@ -9,10 +9,6 @@
 # GFX11: warning: invalid instruction encoding
 0x34,0x12,0x93,0xbf
 
-# this is buffer_atomic_csub_u32 v5, off, s[8:11], s3 offset:4095. Invalid without glc
-# GFX11: warning: invalid instruction encoding
-0xff,0x0f,0xdc,0xe0,0x00,0x05,0x02,0x03
-
 # W32: v_dual_add_f32 v5, 0xaf123456, v2 :: v_dual_fmaak_f32 v6, v3, v1, 0xaf123456 ; encoding: [0xff,0x04,0x02,0xc9,0x03,0x03,0x06,0x05,0x56,0x34,0x12,0xaf]
 # W64: warning: invalid instruction encoding
 0xff,0x04,0x02,0xc9,0x03,0x03,0x06,0x05,0x56,0x34,0x12,0xaf
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1030_new.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1030_new.txt
index 129f70d291..fb02c8be1d 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1030_new.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1030_new.txt
@@ -30,18 +30,27 @@
 # GFX10: global_atomic_csub v2, v[0:1], v2, off offset:100 glc slc
 0x64,0x80,0xd3,0xdc,0x00,0x02,0x7d,0x02
 
+# GFX10: global_atomic_csub v[0:1], v2, off offset:100 slc
+0x64,0x80,0xd2,0xdc,0x00,0x02,0x7d,0x00
+
 # GFX10: global_atomic_csub v2, v[0:1], v2, off glc
 0x00,0x80,0xd1,0xdc,0x00,0x02,0x7d,0x02
 
+# GFX10: global_atomic_csub v[0:1], v2, off
+0x00,0x80,0xd0,0xdc,0x00,0x02,0x7d,0x00
+
 # GFX10: global_atomic_csub v2, v0, v2, s[2:3] glc
 0x00,0x80,0xd1,0xdc,0x00,0x02,0x02,0x02
 
 # GFX10: global_atomic_csub v2, v0, v2, s[2:3] offset:100 glc slc
 0x64,0x80,0xd3,0xdc,0x00,0x02,0x02,0x02
 
-# GFX10: buffer_atomic_csub v5, off, s[8:11], s3
+# GFX10: buffer_atomic_csub v5, off, s[8:11], s3 glc
 0x00,0x40,0xd0,0xe0,0x00,0x05,0x02,0x03
 
+# GFX10: buffer_atomic_csub v5, off, s[8:11], s3
+0x00,0x00,0xd0,0xe0,0x00,0x05,0x02,0x03
+
 # GFX10: buffer_atomic_csub v5, off, s[8:11], s3 offset:4095 glc
 0xff,0x4f,0xd0,0xe0,0x00,0x05,0x02,0x03
 
@@ -57,6 +66,9 @@
 # GFX10: buffer_atomic_csub v5, off, s[8:11], s3 glc slc
 0x00,0x40,0xd0,0xe0,0x00,0x05,0x42,0x03
 
+# GFX10: buffer_atomic_csub v5, off, s[8:11], s3 slc
+0x00,0x00,0xd0,0xe0,0x00,0x05,0x42,0x03
+
 # GFX10: v_fma_legacy_f32 v0, v1, v2, v3
 0x00,0x00,0x40,0xd5,0x01,0x05,0x0e,0x04
 
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_flat.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_flat.txt
index e83fbe3180..30bfaa2cba 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_flat.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_flat.txt
@@ -719,6 +719,9 @@
 # GFX11: global_atomic_csub_u32 v5, v[1:2], v2, off glc ; encoding: [0x00,0x40,0xde,0xdc,0x01,0x02,0x7c,0x05]
 0x00,0x40,0xde,0xdc,0x01,0x02,0x7c,0x05
 
+# GFX11: global_atomic_csub_u32 v[1:2], v2, off ; encoding: [0x00,0x00,0xde,0xdc,0x01,0x02,0x7c,0x00]
+0x00,0x00,0xde,0xdc,0x01,0x02,0x7c,0x00
+
 # GFX11: global_atomic_csub_u32 v5, v[254:255], v2, off glc ; encoding: [0x00,0x40,0xde,0xdc,0xfe,0x02,0x7c,0x05]
 0x00,0x40,0xde,0xdc,0xfe,0x02,0x7c,0x05
 
@@ -740,12 +743,18 @@
 # GFX11: global_atomic_csub_u32 v5, v255, v2, s[104:105] offset:-1 glc ; encoding: [0xff,0x5f,0xde,0xdc,0xff,0x02,0x68,0x05]
 0xff,0x5f,0xde,0xdc,0xff,0x02,0x68,0x05
 
+# GFX11: global_atomic_csub_u32 v255, v2, s[104:105] offset:-1 ; encoding: [0xff,0x1f,0xde,0xdc,0xff,0x02,0x68,0x00]
+0xff,0x1f,0xde,0xdc,0xff,0x02,0x68,0x00
+
 # GFX11: global_atomic_csub_u32 v5, v255, v2, vcc offset:4095 glc ; encoding: [0xff,0x4f,0xde,0xdc,0xff,0x02,0x6a,0x05]
 0xff,0x4f,0xde,0xdc,0xff,0x02,0x6a,0x05
 
 # GFX11: global_atomic_csub_u32 v255, v255, v255, ttmp[14:15] offset:-4096 glc slc dlc ; encoding: [0x00,0xf0,0xde,0xdc,0xff,0xff,0x7a,0xff]
 0x00,0xf0,0xde,0xdc,0xff,0xff,0x7a,0xff
 
+# GFX11: global_atomic_csub_u32 v255, v255, ttmp[14:15] offset:-4096 slc dlc ; encoding: [0x00,0xb0,0xde,0xdc,0xff,0xff,0x7a,0x00]
+0x00,0xb0,0xde,0xdc,0xff,0xff,0x7a,0x00
+
 # GFX11: global_atomic_dec_u32 v[1:2], v2, off ; encoding: [0x00,0x00,0x02,0xdd,0x01,0x02,0x7c,0x00]
 0x00,0x00,0x02,0xdd,0x01,0x02,0x7c,0x00
 
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_mubuf.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_mubuf.txt
index f231b1bb07..5f4d40a9f8 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_mubuf.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_mubuf.txt
@@ -3084,6 +3084,9 @@
 # GFX11: buffer_atomic_csub_u32 v5, off, s[8:11], s3 offset:4095 glc                    ; encoding: [0xff,0x4f,0xdc,0xe0,0x00,0x05,0x02,0x03]
 0xff,0x4f,0xdc,0xe0,0x00,0x05,0x02,0x03
 
+# GFX11: buffer_atomic_csub_u32 v5, off, s[8:11], s3 offset:4095                        ; encoding: [0xff,0x0f,0xdc,0xe0,0x00,0x05,0x02,0x03]
+0xff,0x0f,0xdc,0xe0,0x00,0x05,0x02,0x03
+
 # GFX11: buffer_atomic_csub_u32 v255, off, s[8:11], s3 offset:4095 glc                  ; encoding: [0xff,0x4f,0xdc,0xe0,0x00,0xff,0x02,0x03]
 0xff,0x4f,0xdc,0xe0,0x00,0xff,0x02,0x03
 
@@ -3132,12 +3135,21 @@
 # GFX11: buffer_atomic_csub_u32 v5, off, s[8:11], s3 offset:4095 glc slc                ; encoding: [0xff,0x5f,0xdc,0xe0,0x00,0x05,0x02,0x03]
 0xff,0x5f,0xdc,0xe0,0x00,0x05,0x02,0x03
 
+# GFX11: buffer_atomic_csub_u32 v5, off, s[8:11], s3 offset:4095 slc                    ; encoding: [0xff,0x1f,0xdc,0xe0,0x00,0x05,0x02,0x03]
+0xff,0x1f,0xdc,0xe0,0x00,0x05,0x02,0x03
+
 # GFX11: buffer_atomic_csub_u32 v5, off, s[8:11], s3 offset:4095 glc dlc                ; encoding: [0xff,0x6f,0xdc,0xe0,0x00,0x05,0x02,0x03]
 0xff,0x6f,0xdc,0xe0,0x00,0x05,0x02,0x03
 
+# GFX11: buffer_atomic_csub_u32 v5, off, s[8:11], s3 offset:4095 dlc                    ; encoding: [0xff,0x2f,0xdc,0xe0,0x00,0x05,0x02,0x03]
+0xff,0x2f,0xdc,0xe0,0x00,0x05,0x02,0x03
+
 # GFX11: buffer_atomic_csub_u32 v5, off, s[8:11], s3 offset:4095 glc slc dlc            ; encoding: [0xff,0x7f,0xdc,0xe0,0x00,0x05,0x02,0x03]
 0xff,0x7f,0xdc,0xe0,0x00,0x05,0x02,0x03
 
+# GFX11: buffer_atomic_csub_u32 v5, off, s[8:11], s3 offset:4095 slc dlc                ; encoding: [0xff,0x3f,0xdc,0xe0,0x00,0x05,0x02,0x03]
+0xff,0x3f,0xdc,0xe0,0x00,0x05,0x02,0x03
+
 # GFX11: buffer_atomic_dec_u32 v5, off, s[8:11], s3 offset:4095                         ; encoding: [0xff,0x0f,0x00,0xe1,0x00,0x05,0x02,0x03]
 0xff,0x0f,0x00,0xe1,0x00,0x05,0x02,0x03
 
diff --git a/llvm/test/MC/Disassembler/X86/usermsr-64.txt b/llvm/test/MC/Disassembler/X86/usermsr-64.txt
new file mode 100644
index 0000000000..592a1a204f
--- /dev/null
+++ b/llvm/test/MC/Disassembler/X86/usermsr-64.txt
@@ -0,0 +1,28 @@
+# RUN: llvm-mc --disassemble %s -triple=x86_64 | FileCheck %s --check-prefixes=ATT
+# RUN: llvm-mc --disassemble %s -triple=x86_64 --output-asm-variant=1 | FileCheck %s --check-prefixes=INTEL
+
+# ATT:   urdmsr $123, %r9
+# INTEL: urdmsr r9, 123
+0xc4,0xc7,0x7b,0xf8,0xc1,0x7b,0x00,0x00,0x00
+
+# ATT:   urdmsr %r9, %r9
+# INTEL: urdmsr r9, r9
+0xf2,0x45,0x0f,0x38,0xf8,0xc9
+
+# Test if WIG is supported for this instruction/form.
+# ATT:   urdmsr %r9, %r9
+# INTEL: urdmsr r9, r9
+0xf2,0x4d,0x0f,0x38,0xf8,0xc9
+
+# ATT:   uwrmsr %r9, $123
+# INTEL: uwrmsr 123, r9
+0xc4,0xc7,0x7a,0xf8,0xc1,0x7b,0x00,0x00,0x00
+
+# ATT:   uwrmsr %r9, %r9
+# INTEL: uwrmsr r9, r9
+0xf3,0x45,0x0f,0x38,0xf8,0xc9
+
+# Test if WIG is supported for this instruction/form.
+# ATT:   uwrmsr %r9, %r9
+# INTEL: uwrmsr r9, r9
+0xf3,0x4d,0x0f,0x38,0xf8,0xc9
diff --git a/llvm/test/MC/RISCV/compress-rv32i.s b/llvm/test/MC/RISCV/compress-rv32i.s
index 7869481bb7..165defc343 100644
--- a/llvm/test/MC/RISCV/compress-rv32i.s
+++ b/llvm/test/MC/RISCV/compress-rv32i.s
@@ -154,16 +154,36 @@ jal zero, -2048
 # CHECK: # encoding: [0x01,0xd0]
 beq s0, zero, -256
 
+# CHECK-BYTES: 01 d0
+# CHECK-ALIASASM: beqz s0, -256
+# CHECK-ALIASOBJ32: beqz s0, 0xffffff2a
+# CHECK-ALIASOBJ64: beqz s0, 0xffffffffffffff2a
+# CHECK-INSTASM: c.beqz s0, -256
+# CHECK-INSTOBJ32: c.beqz s0, 0xffffff2a
+# CHECK-INSTOBJ64: c.beqz s0, 0xffffffffffffff2a
+# CHECK: # encoding: [0x01,0xd0]
+beq zero, s0, -256
+
 # CHECK-BYTES: 7d ec
 # CHECK-ALIASASM: bnez s0, 254
-# CHECK-ALIASOBJ32: bnez s0, 0x128
-# CHECK-ALIASOBJ64: bnez s0, 0x128
+# CHECK-ALIASOBJ32: bnez s0, 0x12a
+# CHECK-ALIASOBJ64: bnez s0, 0x12a
 # CHECK-INSTASM: c.bnez s0, 254
-# CHECK-INSTOBJ32: c.bnez s0, 0x128
-# CHECK-INSTOBJ64: c.bnez s0, 0x128
+# CHECK-INSTOBJ32: c.bnez s0, 0x12a
+# CHECK-INSTOBJ64: c.bnez s0, 0x12a
 # CHECK: # encoding: [0x7d,0xec]
 bne s0, zero, 254
 
+# CHECK-BYTES: 7d ec
+# CHECK-ALIASASM: bnez s0, 254
+# CHECK-ALIASOBJ32: bnez s0, 0x12c
+# CHECK-ALIASOBJ64: bnez s0, 0x12c
+# CHECK-INSTASM: c.bnez s0, 254
+# CHECK-INSTOBJ32: c.bnez s0, 0x12c
+# CHECK-INSTOBJ64: c.bnez s0, 0x12c
+# CHECK: # encoding: [0x7d,0xec]
+bne zero, s0, 254
+
 # CHECK-BYTES: 7e 04
 # CHECK-ALIAS: slli s0, s0, 31
 # CHECK-INST: c.slli s0, 31
diff --git a/llvm/test/MC/RISCV/rvv/xsfvqmacc.s b/llvm/test/MC/RISCV/rvv/xsfvqmacc.s
new file mode 100644
index 0000000000..ba19f21844
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/xsfvqmacc.s
@@ -0,0 +1,57 @@
+# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+v,+xsfvqmaccqoq,+xsfvqmaccdod %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv64 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+v,+xsfvqmaccqoq,+xsfvqmaccdod %s \
+# RUN:        | llvm-objdump -d --mattr=+v,+xsfvqmaccqoq,+xsfvqmaccdod - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+v,+xsfvqmaccqoq,+xsfvqmaccdod %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+sf.vqmaccu.2x8x2 v8, v4, v20
+# CHECK-INST: sf.vqmaccu.2x8x2 v8, v4, v20
+# CHECK-ENCODING: [0x5b,0x24,0x42,0xb3]
+# CHECK-ERROR: instruction requires the following: 'XSfvqmaccdod' (SiFive Int8 Matrix Multiplication Instructions (2-by-8 and 8-by-2))
+# CHECK-UNKNOWN: 5b 24 42 b3 <unknown>
+
+sf.vqmacc.2x8x2 v8, v4, v20
+# CHECK-INST: sf.vqmacc.2x8x2 v8, v4, v20
+# CHECK-ENCODING: [0x5b,0x24,0x42,0xb7]
+# CHECK-ERROR: instruction requires the following: 'XSfvqmaccdod' (SiFive Int8 Matrix Multiplication Instructions (2-by-8 and 8-by-2))
+# CHECK-UNKNOWN: 5b 24 42 b7 <unknown>
+
+sf.vqmaccus.2x8x2 v8, v4, v20
+# CHECK-INST: sf.vqmaccus.2x8x2 v8, v4, v20
+# CHECK-ENCODING: [0x5b,0x24,0x42,0xbb]
+# CHECK-ERROR: instruction requires the following: 'XSfvqmaccdod' (SiFive Int8 Matrix Multiplication Instructions (2-by-8 and 8-by-2))
+# CHECK-UNKNOWN: 5b 24 42 bb <unknown>
+
+sf.vqmaccsu.2x8x2 v8, v4, v20
+# CHECK-INST: sf.vqmaccsu.2x8x2 v8, v4, v20
+# CHECK-ENCODING: [0x5b,0x24,0x42,0xbf]
+# CHECK-ERROR: instruction requires the following: 'XSfvqmaccdod' (SiFive Int8 Matrix Multiplication Instructions (2-by-8 and 8-by-2))
+# CHECK-UNKNOWN: 5b 24 42 bf <unknown>
+
+sf.vqmaccu.4x8x4 v8, v4, v20
+# CHECK-INST: sf.vqmaccu.4x8x4 v8, v4, v20
+# CHECK-ENCODING: [0x5b,0x24,0x42,0xf3]
+# CHECK-ERROR: instruction requires the following: 'XSfvqmaccqoq' (SiFive Int8 Matrix Multiplication Instructions (4-by-8 and 8-by-4))
+# CHECK-UNKNOWN: 5b 24 42 f3 <unknown>
+
+sf.vqmacc.4x8x4 v8, v4, v20
+# CHECK-INST: sf.vqmacc.4x8x4 v8, v4, v20
+# CHECK-ENCODING: [0x5b,0x24,0x42,0xf7]
+# CHECK-ERROR: instruction requires the following: 'XSfvqmaccqoq' (SiFive Int8 Matrix Multiplication Instructions (4-by-8 and 8-by-4))
+# CHECK-UNKNOWN: 5b 24 42 f7 <unknown>
+
+sf.vqmaccus.4x8x4 v8, v4, v20
+# CHECK-INST: sf.vqmaccus.4x8x4 v8, v4, v20
+# CHECK-ENCODING: [0x5b,0x24,0x42,0xfb]
+# CHECK-ERROR: instruction requires the following: 'XSfvqmaccqoq' (SiFive Int8 Matrix Multiplication Instructions (4-by-8 and 8-by-4))
+# CHECK-UNKNOWN: 5b 24 42 fb <unknown>
+
+sf.vqmaccsu.4x8x4 v8, v4, v20
+# CHECK-INST: sf.vqmaccsu.4x8x4 v8, v4, v20
+# CHECK-ENCODING: [0x5b,0x24,0x42,0xff]
+# CHECK-ERROR: instruction requires the following: 'XSfvqmaccqoq' (SiFive Int8 Matrix Multiplication Instructions (4-by-8 and 8-by-4))
+# CHECK-UNKNOWN: 5b 24 42 ff <unknown>
diff --git a/llvm/test/MC/X86/usermsr-64-att.s b/llvm/test/MC/X86/usermsr-64-att.s
new file mode 100644
index 0000000000..e89d0a800a
--- /dev/null
+++ b/llvm/test/MC/X86/usermsr-64-att.s
@@ -0,0 +1,18 @@
+// RUN: llvm-mc -triple x86_64 --show-encoding %s | FileCheck %s
+
+// CHECK: urdmsr $123, %r9
+// CHECK: encoding: [0xc4,0xc7,0x7b,0xf8,0xc1,0x7b,0x00,0x00,0x00]
+          urdmsr $123, %r9
+
+// CHECK: urdmsr %r9, %r9
+// CHECK: encoding: [0xf2,0x45,0x0f,0x38,0xf8,0xc9]
+          urdmsr %r9, %r9
+
+// CHECK: uwrmsr %r9, $123
+// CHECK: encoding: [0xc4,0xc7,0x7a,0xf8,0xc1,0x7b,0x00,0x00,0x00]
+          uwrmsr %r9, $123
+
+// CHECK: uwrmsr %r9, %r9
+// CHECK: encoding: [0xf3,0x45,0x0f,0x38,0xf8,0xc9]
+          uwrmsr %r9, %r9
+
diff --git a/llvm/test/MC/X86/usermsr-64-intel.s b/llvm/test/MC/X86/usermsr-64-intel.s
new file mode 100644
index 0000000000..13d9161080
--- /dev/null
+++ b/llvm/test/MC/X86/usermsr-64-intel.s
@@ -0,0 +1,18 @@
+// RUN: llvm-mc -triple x86_64 -x86-asm-syntax=intel -output-asm-variant=1 --show-encoding %s | FileCheck %s
+
+// CHECK: urdmsr r9, 123
+// CHECK: encoding: [0xc4,0xc7,0x7b,0xf8,0xc1,0x7b,0x00,0x00,0x00]
+          urdmsr r9, 123
+
+// CHECK: urdmsr r9, r9
+// CHECK: encoding: [0xf2,0x45,0x0f,0x38,0xf8,0xc9]
+          urdmsr r9, r9
+
+// CHECK: uwrmsr 123, r9
+// CHECK: encoding: [0xc4,0xc7,0x7a,0xf8,0xc1,0x7b,0x00,0x00,0x00]
+          uwrmsr 123, r9
+
+// CHECK: uwrmsr r9, r9
+// CHECK: encoding: [0xf3,0x45,0x0f,0x38,0xf8,0xc9]
+          uwrmsr r9, r9
+
diff --git a/llvm/test/Other/print-at-pass-number.ll b/llvm/test/Other/print-at-pass-number.ll
index 3f7cb7c8e5..8b2d3144e0 100644
--- a/llvm/test/Other/print-at-pass-number.ll
+++ b/llvm/test/Other/print-at-pass-number.ll
@@ -24,8 +24,8 @@ bb4:                                              ; preds = %bb1
   ret i32 %add3
 }
 
-; NUMBER: Running pass 1 LoopSimplifyPass
-; NUMBER-NEXT: Running pass 2 LCSSAPass
-; NUMBER-NEXT: Running pass 3 IndVarSimplifyPass
-; NUMBER-NEXT: Running pass 4 LoopDeletionPass
+; NUMBER:  Running pass 1 LoopSimplifyPass on bar
+; NUMBER-NEXT: Running pass 2 LCSSAPass on bar
+; NUMBER-NEXT: Running pass 3 IndVarSimplifyPass on bb1
+; NUMBER-NEXT: Running pass 4 LoopDeletionPass on bb1
 ; NUMBER-NOT: Running pass
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-imms.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-imms.td
index efe1b4b50d..0495a66a7c 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-imms.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-imms.td
@@ -34,7 +34,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 
 // CHECK:      const int64_t *GenMyCombiner::getMatchTable() const {
 // CHECK-NEXT:   constexpr static int64_t MatchTable0[] = {
-// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/19, 126, /*)*//*default:*//*Label 3*/ 202,
+// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/19, 126, /*)*//*default:*//*Label 3*/ 194,
 // CHECK-NEXT:     /*TargetOpcode::COPY*//*Label 0*/ 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 // CHECK-NEXT:     /*TargetOpcode::G_CONSTANT*//*Label 1*/ 138, 0, 0, 0, 0, 0,
 // CHECK-NEXT:     /*TargetOpcode::G_ZEXT*//*Label 2*/ 165,
@@ -69,25 +69,23 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:     // Label 5: @164
 // CHECK-NEXT:     GIM_Reject,
 // CHECK-NEXT:     // Label 2: @165
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 6*/ 201, // Rule ID 1 //
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 6*/ 193, // Rule ID 1 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule1Enabled,
 // CHECK-NEXT:       // MIs[0] a
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // MIs[0] Operand 1
 // CHECK-NEXT:       GIM_CheckConstantInt, /*MI*/0, /*Op*/1, 0,
 // CHECK-NEXT:       GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/1, /*Opcode*/TargetOpcode::G_CONSTANT,
-// CHECK-NEXT:       GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/0,
-// CHECK-NEXT:       GIR_AddCImm, /*InsnID*/1, /*Type*/GILLT_s32, /*Imm*/0,
+// CHECK-NEXT:       GIR_BuildConstant, /*TempRegID*/0, /*Val*/0,
 // CHECK-NEXT:       // Combiner Rule #1: InstTest1
 // CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::COPY,
 // CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // a
 // CHECK-NEXT:       GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0,
 // CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 6: @201
+// CHECK-NEXT:     // Label 6: @193
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 3: @202
+// CHECK-NEXT:     // Label 3: @194
 // CHECK-NEXT:     GIM_Reject,
 // CHECK-NEXT:     };
 // CHECK-NEXT:   return MatchTable0;
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-patfrag-root.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-patfrag-root.td
index b6296cf902..5cb9206ca5 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-patfrag-root.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-patfrag-root.td
@@ -28,31 +28,29 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 
 // CHECK:      const int64_t *GenMyCombiner::getMatchTable() const {
 // CHECK-NEXT:   constexpr static int64_t MatchTable0[] = {
-// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/118, 181, /*)*//*default:*//*Label 3*/ 176,
+// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/118, 181, /*)*//*default:*//*Label 3*/ 152,
 // CHECK-NEXT:     /*TargetOpcode::G_TRUNC*//*Label 0*/ 68, 0, 0, 0, 0, 0, 0,
-// CHECK-NEXT:     /*TargetOpcode::G_ZEXT*//*Label 1*/ 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-// CHECK-NEXT:     /*TargetOpcode::G_FPEXT*//*Label 2*/ 143,
+// CHECK-NEXT:     /*TargetOpcode::G_ZEXT*//*Label 1*/ 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+// CHECK-NEXT:     /*TargetOpcode::G_FPEXT*//*Label 2*/ 127,
 // CHECK-NEXT:     // Label 0: @68
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 4*/ 100, // Rule ID 1 //
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 4*/ 92, // Rule ID 1 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled,
 // CHECK-NEXT:       // MIs[0] root
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // MIs[0] __Test0_match_0.z
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/1, /*Opcode*/TargetOpcode::G_CONSTANT,
-// CHECK-NEXT:       GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/0,
-// CHECK-NEXT:       GIR_AddCImm, /*InsnID*/1, /*Type*/GILLT_s32, /*Imm*/0,
+// CHECK-NEXT:       GIR_BuildConstant, /*TempRegID*/0, /*Val*/0,
 // CHECK-NEXT:       // Combiner Rule #0: Test0 @ [__Test0_match_0[1]]
 // CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::COPY,
 // CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // root
 // CHECK-NEXT:       GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0,
 // CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 4: @100
+// CHECK-NEXT:     // Label 4: @92
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 1: @101
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 5*/ 142, // Rule ID 0 //
+// CHECK-NEXT:     // Label 1: @93
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 5*/ 126, // Rule ID 0 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled,
 // CHECK-NEXT:       // MIs[0] root
 // CHECK-NEXT:       // No operand predicates
@@ -63,37 +61,33 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       GIM_CheckIsSafeToFold, /*InsnID*/1,
 // CHECK-NEXT:       GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/1, /*Opcode*/TargetOpcode::G_CONSTANT,
-// CHECK-NEXT:       GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/0,
-// CHECK-NEXT:       GIR_AddCImm, /*InsnID*/1, /*Type*/GILLT_s32, /*Imm*/0,
+// CHECK-NEXT:       GIR_BuildConstant, /*TempRegID*/0, /*Val*/0,
 // CHECK-NEXT:       // Combiner Rule #0: Test0 @ [__Test0_match_0[0]]
 // CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::COPY,
 // CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // root
 // CHECK-NEXT:       GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0,
 // CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 5: @142
+// CHECK-NEXT:     // Label 5: @126
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 2: @143
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 6*/ 175, // Rule ID 2 //
+// CHECK-NEXT:     // Label 2: @127
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 6*/ 151, // Rule ID 2 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled,
 // CHECK-NEXT:       // MIs[0] root
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // MIs[0] __Test0_match_0.z
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/1, /*Opcode*/TargetOpcode::G_CONSTANT,
-// CHECK-NEXT:       GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/0,
-// CHECK-NEXT:       GIR_AddCImm, /*InsnID*/1, /*Type*/GILLT_s32, /*Imm*/0,
+// CHECK-NEXT:       GIR_BuildConstant, /*TempRegID*/0, /*Val*/0,
 // CHECK-NEXT:       // Combiner Rule #0: Test0 @ [__Test0_match_0[2]]
 // CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::COPY,
 // CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // root
 // CHECK-NEXT:       GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0,
 // CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 6: @175
+// CHECK-NEXT:     // Label 6: @151
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 3: @176
+// CHECK-NEXT:     // Label 3: @152
 // CHECK-NEXT:     GIM_Reject,
 // CHECK-NEXT:     };
 // CHECK-NEXT:   return MatchTable0;
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-permutations.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-permutations.td
index b0651c971c..22cd2be819 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-permutations.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-permutations.td
@@ -159,9 +159,9 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 
 // CHECK:      const int64_t *GenMyCombiner::getMatchTable() const {
 // CHECK-NEXT:   constexpr static int64_t MatchTable0[] = {
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 0*/ 746,
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 0*/ 682,
 // CHECK-NEXT:       GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_AND,
-// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 1*/ 84, // Rule ID 7 //
+// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 1*/ 76, // Rule ID 7 //
 // CHECK-NEXT:         GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled,
 // CHECK-NEXT:         // MIs[0] dst
 // CHECK-NEXT:         // No operand predicates
@@ -191,9 +191,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/3,
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/4,
 // CHECK-NEXT:         GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT:         GIR_BuildMI, /*InsnID*/1, /*Opcode*/TargetOpcode::G_CONSTANT,
-// CHECK-NEXT:         GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/0,
-// CHECK-NEXT:         GIR_AddCImm, /*InsnID*/1, /*Type*/GILLT_s32, /*Imm*/0,
+// CHECK-NEXT:         GIR_BuildConstant, /*TempRegID*/0, /*Val*/0,
 // CHECK-NEXT:         // Combiner Rule #0: Test0 @ [a[1], b[1], c[1]]
 // CHECK-NEXT:         GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::COPY,
 // CHECK-NEXT:         GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst
@@ -201,8 +199,8 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:         GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT:         GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner0,
 // CHECK-NEXT:         GIR_Done,
-// CHECK-NEXT:       // Label 1: @84
-// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 2*/ 172, // Rule ID 6 //
+// CHECK-NEXT:       // Label 1: @76
+// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 2*/ 156, // Rule ID 6 //
 // CHECK-NEXT:         GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled,
 // CHECK-NEXT:         // MIs[0] dst
 // CHECK-NEXT:         // No operand predicates
@@ -236,9 +234,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/4,
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/5,
 // CHECK-NEXT:         GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT:         GIR_BuildMI, /*InsnID*/1, /*Opcode*/TargetOpcode::G_CONSTANT,
-// CHECK-NEXT:         GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/0,
-// CHECK-NEXT:         GIR_AddCImm, /*InsnID*/1, /*Type*/GILLT_s32, /*Imm*/0,
+// CHECK-NEXT:         GIR_BuildConstant, /*TempRegID*/0, /*Val*/0,
 // CHECK-NEXT:         // Combiner Rule #0: Test0 @ [a[1], b[1], c[0]]
 // CHECK-NEXT:         GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::COPY,
 // CHECK-NEXT:         GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst
@@ -246,8 +242,8 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:         GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT:         GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner0,
 // CHECK-NEXT:         GIR_Done,
-// CHECK-NEXT:       // Label 2: @172
-// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 3*/ 260, // Rule ID 5 //
+// CHECK-NEXT:       // Label 2: @156
+// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 3*/ 236, // Rule ID 5 //
 // CHECK-NEXT:         GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled,
 // CHECK-NEXT:         // MIs[0] dst
 // CHECK-NEXT:         // No operand predicates
@@ -281,9 +277,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/4,
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/5,
 // CHECK-NEXT:         GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT:         GIR_BuildMI, /*InsnID*/1, /*Opcode*/TargetOpcode::G_CONSTANT,
-// CHECK-NEXT:         GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/0,
-// CHECK-NEXT:         GIR_AddCImm, /*InsnID*/1, /*Type*/GILLT_s32, /*Imm*/0,
+// CHECK-NEXT:         GIR_BuildConstant, /*TempRegID*/0, /*Val*/0,
 // CHECK-NEXT:         // Combiner Rule #0: Test0 @ [a[1], b[0], c[1]]
 // CHECK-NEXT:         GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::COPY,
 // CHECK-NEXT:         GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst
@@ -291,8 +285,8 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:         GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT:         GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner0,
 // CHECK-NEXT:         GIR_Done,
-// CHECK-NEXT:       // Label 3: @260
-// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 4*/ 357, // Rule ID 4 //
+// CHECK-NEXT:       // Label 3: @236
+// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 4*/ 325, // Rule ID 4 //
 // CHECK-NEXT:         GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled,
 // CHECK-NEXT:         // MIs[0] dst
 // CHECK-NEXT:         // No operand predicates
@@ -330,9 +324,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/5,
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/6,
 // CHECK-NEXT:         GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT:         GIR_BuildMI, /*InsnID*/1, /*Opcode*/TargetOpcode::G_CONSTANT,
-// CHECK-NEXT:         GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/0,
-// CHECK-NEXT:         GIR_AddCImm, /*InsnID*/1, /*Type*/GILLT_s32, /*Imm*/0,
+// CHECK-NEXT:         GIR_BuildConstant, /*TempRegID*/0, /*Val*/0,
 // CHECK-NEXT:         // Combiner Rule #0: Test0 @ [a[1], b[0], c[0]]
 // CHECK-NEXT:         GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::COPY,
 // CHECK-NEXT:         GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst
@@ -340,8 +332,8 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:         GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT:         GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner0,
 // CHECK-NEXT:         GIR_Done,
-// CHECK-NEXT:       // Label 4: @357
-// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 5*/ 445, // Rule ID 3 //
+// CHECK-NEXT:       // Label 4: @325
+// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 5*/ 405, // Rule ID 3 //
 // CHECK-NEXT:         GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled,
 // CHECK-NEXT:         // MIs[0] dst
 // CHECK-NEXT:         // No operand predicates
@@ -375,9 +367,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/4,
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/5,
 // CHECK-NEXT:         GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT:         GIR_BuildMI, /*InsnID*/1, /*Opcode*/TargetOpcode::G_CONSTANT,
-// CHECK-NEXT:         GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/0,
-// CHECK-NEXT:         GIR_AddCImm, /*InsnID*/1, /*Type*/GILLT_s32, /*Imm*/0,
+// CHECK-NEXT:         GIR_BuildConstant, /*TempRegID*/0, /*Val*/0,
 // CHECK-NEXT:         // Combiner Rule #0: Test0 @ [a[0], b[1], c[1]]
 // CHECK-NEXT:         GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::COPY,
 // CHECK-NEXT:         GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst
@@ -385,8 +375,8 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:         GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT:         GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner0,
 // CHECK-NEXT:         GIR_Done,
-// CHECK-NEXT:       // Label 5: @445
-// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 6*/ 542, // Rule ID 2 //
+// CHECK-NEXT:       // Label 5: @405
+// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 6*/ 494, // Rule ID 2 //
 // CHECK-NEXT:         GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled,
 // CHECK-NEXT:         // MIs[0] dst
 // CHECK-NEXT:         // No operand predicates
@@ -424,9 +414,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/5,
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/6,
 // CHECK-NEXT:         GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT:         GIR_BuildMI, /*InsnID*/1, /*Opcode*/TargetOpcode::G_CONSTANT,
-// CHECK-NEXT:         GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/0,
-// CHECK-NEXT:         GIR_AddCImm, /*InsnID*/1, /*Type*/GILLT_s32, /*Imm*/0,
+// CHECK-NEXT:         GIR_BuildConstant, /*TempRegID*/0, /*Val*/0,
 // CHECK-NEXT:         // Combiner Rule #0: Test0 @ [a[0], b[1], c[0]]
 // CHECK-NEXT:         GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::COPY,
 // CHECK-NEXT:         GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst
@@ -434,8 +422,8 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:         GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT:         GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner0,
 // CHECK-NEXT:         GIR_Done,
-// CHECK-NEXT:       // Label 6: @542
-// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 7*/ 639, // Rule ID 1 //
+// CHECK-NEXT:       // Label 6: @494
+// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 7*/ 583, // Rule ID 1 //
 // CHECK-NEXT:         GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled,
 // CHECK-NEXT:         // MIs[0] dst
 // CHECK-NEXT:         // No operand predicates
@@ -473,9 +461,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/5,
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/6,
 // CHECK-NEXT:         GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT:         GIR_BuildMI, /*InsnID*/1, /*Opcode*/TargetOpcode::G_CONSTANT,
-// CHECK-NEXT:         GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/0,
-// CHECK-NEXT:         GIR_AddCImm, /*InsnID*/1, /*Type*/GILLT_s32, /*Imm*/0,
+// CHECK-NEXT:         GIR_BuildConstant, /*TempRegID*/0, /*Val*/0,
 // CHECK-NEXT:         // Combiner Rule #0: Test0 @ [a[0], b[0], c[1]]
 // CHECK-NEXT:         GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::COPY,
 // CHECK-NEXT:         GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst
@@ -483,8 +469,8 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:         GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT:         GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner0,
 // CHECK-NEXT:         GIR_Done,
-// CHECK-NEXT:       // Label 7: @639
-// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 8*/ 745, // Rule ID 0 //
+// CHECK-NEXT:       // Label 7: @583
+// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 8*/ 681, // Rule ID 0 //
 // CHECK-NEXT:         GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled,
 // CHECK-NEXT:         // MIs[0] dst
 // CHECK-NEXT:         // No operand predicates
@@ -526,9 +512,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/6,
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/7,
 // CHECK-NEXT:         GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT:         GIR_BuildMI, /*InsnID*/1, /*Opcode*/TargetOpcode::G_CONSTANT,
-// CHECK-NEXT:         GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/0,
-// CHECK-NEXT:         GIR_AddCImm, /*InsnID*/1, /*Type*/GILLT_s32, /*Imm*/0,
+// CHECK-NEXT:         GIR_BuildConstant, /*TempRegID*/0, /*Val*/0,
 // CHECK-NEXT:         // Combiner Rule #0: Test0 @ [a[0], b[0], c[0]]
 // CHECK-NEXT:         GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::COPY,
 // CHECK-NEXT:         GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst
@@ -536,9 +520,9 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:         GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT:         GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner0,
 // CHECK-NEXT:         GIR_Done,
-// CHECK-NEXT:       // Label 8: @745
+// CHECK-NEXT:       // Label 8: @681
 // CHECK-NEXT:       GIM_Reject,
-// CHECK-NEXT:     // Label 0: @746
+// CHECK-NEXT:     // Label 0: @682
 // CHECK-NEXT:     GIM_Reject,
 // CHECK-NEXT:     };
 // CHECK-NEXT:   return MatchTable0;
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td
index b810c519d2..a74f7fbbe1 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td
@@ -93,12 +93,12 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK:      bool GenMyCombiner::tryCombineAll(MachineInstr &I) const {
 // CHECK-NEXT:   const TargetSubtargetInfo &ST = MF.getSubtarget();
 // CHECK-NEXT:   const PredicateBitset AvailableFeatures = getAvailableFeatures();
-// CHECK-NEXT:   NewMIVector OutMIs;
+// CHECK-NEXT:   B.setInstrAndDebugLoc(I);
 // CHECK-NEXT:   State.MIs.clear();
 // CHECK-NEXT:   State.MIs.push_back(&I);
 // CHECK-NEXT:   MatchInfos = MatchInfosTy();
 // CHECK-EMPTY:
-// CHECK-NEXT:   if (executeMatchTable(*this, OutMIs, State, ExecInfo, getMatchTable(), *ST.getInstrInfo(), MRI, *MRI.getTargetRegisterInfo(), *ST.getRegBankInfo(), AvailableFeatures, /*CoverageInfo*/ nullptr, &Observer))
+// CHECK-NEXT:   if (executeMatchTable(*this, State, ExecInfo, B, getMatchTable(), *ST.getInstrInfo(), MRI, *MRI.getTargetRegisterInfo(), *ST.getRegBankInfo(), AvailableFeatures, /*CoverageInfo*/ nullptr))
 // CHECK-NEXT:     return true;
 // CHECK-NEXT:   }
 // CHECK-EMPTY:
@@ -132,7 +132,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // Verify match table.
 // CHECK:      const int64_t *GenMyCombiner::getMatchTable() const {
 // CHECK-NEXT:   constexpr static int64_t MatchTable0[] = {
-// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/19, 126, /*)*//*default:*//*Label 6*/ 275,
+// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/19, 126, /*)*//*default:*//*Label 6*/ 267,
 // CHECK-NEXT:     /*TargetOpcode::COPY*//*Label 0*/ 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 // CHECK-NEXT:     /*TargetOpcode::G_AND*//*Label 1*/ 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 // CHECK-NEXT:     /*TargetOpcode::G_STORE*//*Label 2*/ 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -232,7 +232,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:     // Label 13: @238
 // CHECK-NEXT:     GIM_Reject,
 // CHECK-NEXT:     // Label 5: @239
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 14*/ 274, // Rule ID 7 //
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 14*/ 266, // Rule ID 7 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule6Enabled,
 // CHECK-NEXT:       // MIs[0] dst
 // CHECK-NEXT:       // No operand predicates
@@ -240,18 +240,16 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_GICombiner1,
 // CHECK-NEXT:       GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/1, /*Opcode*/TargetOpcode::G_CONSTANT,
-// CHECK-NEXT:       GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/0,
-// CHECK-NEXT:       GIR_AddCImm, /*InsnID*/1, /*Type*/GILLT_s32, /*Imm*/0,
+// CHECK-NEXT:       GIR_BuildConstant, /*TempRegID*/0, /*Val*/0,
 // CHECK-NEXT:       // Combiner Rule #6: PatFragTest0 @ [__PatFragTest0_match_1[0]]
 // CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::COPY,
 // CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst
 // CHECK-NEXT:       GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0,
 // CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 14: @274
+// CHECK-NEXT:     // Label 14: @266
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 6: @275
+// CHECK-NEXT:     // Label 6: @267
 // CHECK-NEXT:     GIM_Reject,
 // CHECK-NEXT:     };
 // CHECK-NEXT:   return MatchTable0;
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/misc/redundant-combine-in-list.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/misc/redundant-combine-in-list.td
new file mode 100644
index 0000000000..da38a228f6
--- /dev/null
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/misc/redundant-combine-in-list.td
@@ -0,0 +1,30 @@
+// RUN: llvm-tblgen -I %p/../../../../include -gen-global-isel-combiner \
+// RUN:     -combiners=Combiner %s 2>&1 | FileCheck %s
+
+include "llvm/Target/Target.td"
+include "llvm/Target/GlobalISel/Combine.td"
+
+// Check we don't crash if a combine is present twice in the list.
+
+def MyTargetISA : InstrInfo;
+def MyTarget : Target { let InstructionSet = MyTargetISA; }
+
+def dummy;
+
+// CHECK: :[[@LINE+1]]:{{[0-9]+}}: warning: skipping rule 'Foo' because it has already been processed
+def Foo : GICombineRule<
+  (defs root:$root),
+  (match (G_ZEXT $root, $x)),
+  (apply (G_TRUNC $root, $x))>;
+
+def Bar : GICombineRule<
+  (defs root:$root),
+  (match (G_TRUNC $root, $x)),
+  (apply (G_ZEXT $root, $x))>;
+
+def FooBar : GICombineGroup<[ Foo, Bar ]>;
+
+def Combiner: GICombiner<"GenMyCombiner", [
+  FooBar,
+  Foo
+]>;
diff --git a/llvm/test/TableGen/GlobalISelEmitter.td b/llvm/test/TableGen/GlobalISelEmitter.td
index 7cca2d52e4..b7a81894f6 100644
--- a/llvm/test/TableGen/GlobalISelEmitter.td
+++ b/llvm/test/TableGen/GlobalISelEmitter.td
@@ -216,11 +216,11 @@ def HasC : Predicate<"Subtarget->hasC()"> { let RecomputePerFunction = 1; }
 
 // CHECK: bool MyTargetInstructionSelector::selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const {
 // CHECK-NEXT: const PredicateBitset AvailableFeatures = getAvailableFeatures();
-// CHECK-NEXT: NewMIVector OutMIs;
+// CHECK-NEXT: MachineIRBuilder B(I);
 // CHECK-NEXT: State.MIs.clear();
 // CHECK-NEXT: State.MIs.push_back(&I);
 
-// CHECK:      if (executeMatchTable(*this, OutMIs, State, ExecInfo, getMatchTable(), TII, MF->getRegInfo(), TRI, RBI, AvailableFeatures, &CoverageInfo)) {
+// CHECK:      if (executeMatchTable(*this, State, ExecInfo, B, getMatchTable(), TII, MF->getRegInfo(), TRI, RBI, AvailableFeatures, &CoverageInfo)) {
 // CHECK-NEXT:   return true;
 // CHECK-NEXT: }
 
diff --git a/llvm/test/TableGen/GlobalISelEmitterHwModes.td b/llvm/test/TableGen/GlobalISelEmitterHwModes.td
index 678acb4cd0..b185feaf00 100644
--- a/llvm/test/TableGen/GlobalISelEmitterHwModes.td
+++ b/llvm/test/TableGen/GlobalISelEmitterHwModes.td
@@ -113,11 +113,11 @@ class I<dag OOps, dag IOps, list<dag> Pat>
 
 // CHECK: bool MyTargetInstructionSelector::selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const {
 // CHECK-NEXT: const PredicateBitset AvailableFeatures = getAvailableFeatures();
-// CHECK-NEXT: NewMIVector OutMIs;
+// CHECK-NEXT: MachineIRBuilder B(I);
 // CHECK-NEXT: State.MIs.clear();
 // CHECK-NEXT: State.MIs.push_back(&I);
 
-// CHECK:      if (executeMatchTable(*this, OutMIs, State, ExecInfo, getMatchTable(), TII, MF->getRegInfo(), TRI, RBI, AvailableFeatures, &CoverageInfo)) {
+// CHECK:      if (executeMatchTable(*this, State, ExecInfo, B, getMatchTable(), TII, MF->getRegInfo(), TRI, RBI, AvailableFeatures, &CoverageInfo)) {
 // CHECK-NEXT:   return true;
 // CHECK-NEXT: }
 
diff --git a/llvm/test/TableGen/VarLenEncoderHwModes.td b/llvm/test/TableGen/VarLenEncoderHwModes.td
new file mode 100644
index 0000000000..e0da0c9b93
--- /dev/null
+++ b/llvm/test/TableGen/VarLenEncoderHwModes.td
@@ -0,0 +1,110 @@
+// RUN: llvm-tblgen -gen-emitter -I %p/../../include %s | FileCheck %s
+
+// Verify VarLenCodeEmitterGen using EncodingInfos with different HwModes.
+
+include "llvm/Target/Target.td"
+
+def ArchInstrInfo : InstrInfo { }
+
+def Arch : Target {
+  let InstructionSet = ArchInstrInfo;
+}
+
+def Reg : Register<"reg">;
+
+def RegClass : RegisterClass<"foo", [i64], 0, (add Reg)>;
+
+def GR64 : RegisterOperand<RegClass>;
+
+def HasA : Predicate<"Subtarget->hasA()">;
+def HasB : Predicate<"Subtarget->hasB()">;
+
+def ModeA : HwMode<"+a", [HasA]>;
+def ModeB : HwMode<"+b", [HasB]>;
+
+def fooTypeEncA : InstructionEncoding {
+  dag Inst = (descend
+    (operand "$src", 4),
+    (operand "$dst", 4),
+    0b00000001
+  );
+}
+
+def fooTypeEncB : InstructionEncoding {
+  dag Inst = (descend
+    (operand "$dst", 4),
+    (operand "$src", 4),
+    0b00000010
+  );
+}
+
+def fooTypeEncC : InstructionEncoding {
+  dag Inst = (descend
+    (operand "$dst", 4),
+    (operand "$src", 4),
+    0b00000100
+  );
+}
+
+class VarLenInst : Instruction {
+  let AsmString = "foo $src, $dst";
+  let OutOperandList = (outs GR64:$dst);
+  let InOperandList  = (ins GR64:$src);
+}
+
+// Defined in both HwModes
+def foo : VarLenInst {
+  let EncodingInfos = EncodingByHwMode<
+    [ModeA, ModeB],
+    [fooTypeEncA, fooTypeEncB]
+  >;
+}
+
+// Same encoding in any HwMode
+def bar : VarLenInst {
+  dag Inst = (descend
+    (operand "$dst", 4),
+    (operand "$src", 4),
+    0b00000011
+  );
+}
+
+// Only defined in HwMode B.
+def baz : VarLenInst {
+  let EncodingInfos = EncodingByHwMode<
+    [ModeB],
+    [fooTypeEncC]
+  >;
+}
+
+// CHECK:     static const uint64_t InstBits_ModeA[] = {
+// CHECK:       UINT64_C(3),        // bar
+// CHECK:       UINT64_C(1),        // foo
+
+// CHECK:     static const uint64_t InstBits_ModeB[] = {
+// CHECK:       UINT64_C(3),        // bar
+// CHECK:       UINT64_C(4),        // baz
+// CHECK:       UINT64_C(2),        // foo
+
+// CHECK:     auto getInstBits_ModeA =
+// CHECK:       Idx = Index_ModeA
+
+// CHECK:     auto getInstBits_ModeB =
+// CHECK:       Idx = Index_ModeB
+
+// CHECK:     case ::bar: {
+// CHECK-NOT:   switch (Mode) {
+// CHECK:       Inst = getInstBits_ModeA
+
+// CHECK:     case ::foo: {
+// CHECK:       switch (Mode) {
+// CHECK:       case 1: {
+// CHECK:       Inst = getInstBits_ModeA
+// CHECK:       case 2: {
+// CHECK:       Inst = getInstBits_ModeB
+
+// CHECK:     case ::baz: {
+// CHECK:       case 1: {
+// CHECK:       llvm_unreachable("Undefined encoding in this mode");
+// CHECK:       case 2: {
+// CHECK:       Inst = getInstBits_ModeB
diff --git a/llvm/test/TableGen/dump.td b/llvm/test/TableGen/dump.td
new file mode 100644
index 0000000000..633e73802f
--- /dev/null
+++ b/llvm/test/TableGen/dump.td
@@ -0,0 +1,103 @@
+// RUN: llvm-tblgen %s -o -  2>&1 >/dev/null | FileCheck %s -DFILE=%s
+
+// CHECK: [[FILE]]:[[@LINE+1]]:1: note: Debug message
+dump "Debug message";
+
+def op;
+class A {
+  string A = "some text";
+  dag X =(op op);
+}
+def a : A;
+// CHECK: [[FILE]]:[[@LINE+5]]:1: note: The Value of A is:
+// CHECK-NEXT: a {     // A
+// CHECK-NEXT: string A = "some text";
+// CHECK-NEXT: dag X = (op op);
+// CHECK-NEXT: }
+dump "The Value of A is: \n" # !repr(a);
+
+def b : A;
+// CHECK: [[FILE]]:[[@LINE+4]]:1: note: b {     // A
+// CHECK-NEXT: string A = "some text";
+// CHECK-NEXT: dag X = (op op);
+// CHECK-NEXT: }
+dump b;
+
+defvar value_A = "some other text";
+// CHECK: [[FILE]]:[[@LINE+1]]:1: note: some other text
+dump value_A;
+
+defvar value_B = 12;
+def X;
+// CHECK: [[FILE]]:[[@LINE+3]]:1: note: got a pair of values ["some other text" : 12], and an empty record:
+// CHECK-NEXT: X {
+// CHECK-NEXT: }
+dump "got a pair of values [" # !repr(value_A) # " : " # !repr(value_B) # "], " # "and an empty record:\n" # !repr(X);
+
+multiclass MC<dag s> {
+// CHECK: [[FILE]]:[[@LINE+1]]:3: note: s = (op a)
+  dump "s = " # !repr(s);
+// CHECK: [[FILE]]:[[@LINE+4]]:3: note: args[0] = a {        // A
+// CHECK-NEXT:   string A = "some text";
+// CHECK-NEXT: dag X = (op op);
+// CHECK-NEXT: }
+  dump "args[0] = " # !repr(!getdagarg<A>(s,0));
+  def A;
+}
+defm X : MC<(op a)>;
+
+multiclass MMC<dag s> {
+// CHECK: [[FILE]]:[[@LINE+1]]:3: note: the operand of s is op
+  dump "the operand of s is " # !getdagop(s);
+// CHECK: [[FILE]]:[[@LINE-13]]:3: note: s = (op a, a)
+// CHECK: [[FILE]]:[[@LINE-9]]:3: note: args[0] = a {        // A
+// CHECK-NEXT:   string A = "some text";
+// CHECK-NEXT: dag X = (op op);
+// CHECK-NEXT: }
+  defm : MC<s>;
+}
+
+defm XX : MMC<(op a, a)>;
+
+
+foreach i = [-1, 2] in {
+// CHECK: [[FILE]]:[[@LINE+4]]:3: note: i = -1 (negative)
+// CHECK: [[FILE]]:[[@LINE+8]]:5: note: i + 1  <= 0
+// CHECK: [[FILE]]:[[@LINE+2]]:3: note: i = 2 (positive)
+// CHECK: [[FILE]]:[[@LINE+4]]:5: note: i + 1  > 0 (i + 1 = 3)
+  dump "i = " # !repr(i) # !if(!ge(i,0), " (positive)", " (negative)");
+  defvar ip1 = !add(i, 1);
+  if !gt(ip1,0) then {
+    dump "i + 1 > 0 (i + 1 = " # !repr(ip1) # ")";
+  } else {
+    dump "i + 1 <= 0" ;
+  }
+}
+
+class Code<code val> {
+  dump "val = " # !repr(val);
+  code Val = val;
+  int number = 0;
+}
+// CHECK: [[FILE]]:[[@LINE-4]]:3: note: val = [{a = a +1;}]
+def IncrementA : Code<[{a = a +1;}]>;
+class InheritFromCode : Code<[{f(x);}]>{
+  let number = 33;
+  dump "number = " # !repr(number);
+}
+// CHECK: [[FILE]]:[[@LINE-10]]:3: note: val = [{f(x);}]
+// CHECK: [[FILE]]:[[@LINE-3]]:3: note: number = 33
+def ModeCode : InheritFromCode;
+
+
+class BaseClassForSet;
+multiclass DefineSubSet {
+  def _One : BaseClassForSet;
+  def _Two : BaseClassForSet;
+}
+defset list<BaseClassForSet> TheSet = {
+defm Subset: DefineSubSet;
+def Three : BaseClassForSet;
+}
+// CHECK: [[FILE]]:[[@LINE+1]]:1: note: TheSet = [Subset_One, Subset_Two, Three]
+dump "TheSet = " # !repr(TheSet);
diff --git a/llvm/test/TableGen/repr.td b/llvm/test/TableGen/repr.td
new file mode 100644
index 0000000000..20635f3840
--- /dev/null
+++ b/llvm/test/TableGen/repr.td
@@ -0,0 +1,51 @@
+// RUN: llvm-tblgen %s | FileCheck %s
+
+class PrintString<string s> {
+ string S = s;
+}
+
+def op;
+def case_01 : PrintString<!repr((op op, op))>;
+// CHECK-LABEL: def case_01 { // PrintString
+// CHECK-NEXT:    string S = "(op op, op)";
+// CHECK-NEXT:  }
+
+def case_02 : PrintString<!repr(32)>;
+// CHECK-LABEL: def case_02 { // PrintString
+// CHECK-NEXT:    string S = "32";
+// CHECK-NEXT:  }
+
+multiclass Multi<int i, bits<3> b> {
+  defvar IncI = !repr(!add(i,1));
+  def _IncI : PrintString<IncI>;
+  def _b : PrintString<!repr(b)>;
+}
+
+defm case_03 : Multi<2, 0b111>;
+// CHECK-LABEL: def case_03_IncI {      // PrintString
+// CHECK-NEXT:    string S = "3";
+// CHECK-NEXT:  }
+// CHECK-LABEL: def case_03_b { // PrintString
+// CHECK-NEXT:    string S = "{ 1, 1, 1 }";
+// CHECK-NEXT:  }
+
+def case_04 : PrintString<!repr(!foreach(i, [1,2,3], !mul(i,2)))>;
+// CHECK-LABEL: def case_04 { // PrintString
+// CHECK-NEXT:    string S = "[2, 4, 6]";
+// CHECK-NEXT:  }
+
+// Show the dumping of full records...
+def case_05 : PrintString<!repr(case_04)>;
+// CHECK-LABEL: def case_05 { // PrintString
+// CHECK-NEXT:    string S = "case_04 { // PrintString
+// CHECK-NEXT:    string S = "[2, 4, 6]";
+// CHECK-NEXT:  }
+// CHECK-NEXT:  ";
+// CHECK-NEXT:  }
+
+// ... and how !repr differs compared to !cast<string>.
+def case_06 : PrintString<!cast<string>(case_04)>;
+// CHECK-LABEL: def case_06 { // PrintString
+// CHECK-NEXT:    string S = "case_04";
+// CHECK-NEXT:  }
+
diff --git a/llvm/test/Transforms/Attributor/lvi-after-jumpthreading.ll b/llvm/test/Transforms/Attributor/lvi-after-jumpthreading.ll
index 562b1a4a71..2aa95216a6 100644
--- a/llvm/test/Transforms/Attributor/lvi-after-jumpthreading.ll
+++ b/llvm/test/Transforms/Attributor/lvi-after-jumpthreading.ll
@@ -185,7 +185,6 @@ declare void @llvm.experimental.guard(i1, ...)
 ; CHECK: attributes #[[ATTR0]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) }
 ; CHECK: attributes #[[ATTR1:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write) }
 ; CHECK: attributes #[[ATTR2]] = { nounwind }
-; CHECK: attributes #[[ATTR3:[0-9]+]] = { nocallback nofree nosync willreturn }
 ;.
 ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
 ; CGSCC: {{.*}}
diff --git a/llvm/test/Transforms/Attributor/value-simplify-gpu.ll b/llvm/test/Transforms/Attributor/value-simplify-gpu.ll
index e78d937ec0..a7e0a92912 100644
--- a/llvm/test/Transforms/Attributor/value-simplify-gpu.ll
+++ b/llvm/test/Transforms/Attributor/value-simplify-gpu.ll
@@ -8,11 +8,13 @@ target triple = "amdgcn-amd-amdhsa"
 @ReachableKernel = internal addrspace(3) global i32 3, align 4
 @UnreachableKernel = internal addrspace(3) global i32 42, align 4
 @ReachableKernelAS0 = internal global i32 7, align 4
+@AS3OneKernelAtATime = internal addrspace(3) global i32 42, align 4
 
 ;.
 ; CHECK: @[[REACHABLEKERNEL:[a-zA-Z0-9_$"\\.-]+]] = internal addrspace(3) global i32 3, align 4
 ; CHECK: @[[UNREACHABLEKERNEL:[a-zA-Z0-9_$"\\.-]+]] = internal addrspace(3) global i32 42, align 4
 ; CHECK: @[[REACHABLEKERNELAS0:[a-zA-Z0-9_$"\\.-]+]] = internal global i32 7, align 4
+; CHECK: @[[AS3ONEKERNELATATIME:[a-zA-Z0-9_$"\\.-]+]] = internal addrspace(3) global i32 42, align 4
 ; CHECK: @[[REACHABLENONKERNEL:[a-zA-Z0-9_$"\\.-]+]] = internal addrspace(3) global i32 0, align 4
 ; CHECK: @[[UNREACHABLENONKERNEL:[a-zA-Z0-9_$"\\.-]+]] = internal addrspace(3) global i32 0, align 4
 ;.
@@ -354,6 +356,77 @@ entry:
   ret void
 }
 
+define dso_local void @kernel2(i32 %C) norecurse "kernel" {
+; TUNIT: Function Attrs: norecurse nosync nounwind
+; TUNIT-LABEL: define {{[^@]+}}@kernel2
+; TUNIT-SAME: (i32 [[C:%.*]]) #[[ATTR0]] {
+; TUNIT-NEXT:    [[I:%.*]] = icmp eq i32 [[C]], 42
+; TUNIT-NEXT:    br i1 [[I]], label [[T:%.*]], label [[F:%.*]]
+; TUNIT:       t:
+; TUNIT-NEXT:    store i32 333, ptr addrspace(3) @AS3OneKernelAtATime, align 4
+; TUNIT-NEXT:    br label [[F]]
+; TUNIT:       f:
+; TUNIT-NEXT:    [[L:%.*]] = load i32, ptr addrspace(3) @AS3OneKernelAtATime, align 4
+; TUNIT-NEXT:    call void @use(i32 noundef [[L]], i32 noundef [[L]], i32 noundef [[L]]) #[[ATTR7]]
+; TUNIT-NEXT:    ret void
+;
+; CGSCC: Function Attrs: norecurse nosync nounwind
+; CGSCC-LABEL: define {{[^@]+}}@kernel2
+; CGSCC-SAME: (i32 [[C:%.*]]) #[[ATTR0]] {
+; CGSCC-NEXT:    [[I:%.*]] = icmp eq i32 [[C]], 42
+; CGSCC-NEXT:    br i1 [[I]], label [[T:%.*]], label [[F:%.*]]
+; CGSCC:       t:
+; CGSCC-NEXT:    store i32 333, ptr addrspace(3) @AS3OneKernelAtATime, align 4
+; CGSCC-NEXT:    br label [[F]]
+; CGSCC:       f:
+; CGSCC-NEXT:    [[L:%.*]] = load i32, ptr addrspace(3) @AS3OneKernelAtATime, align 4
+; CGSCC-NEXT:    call void @use(i32 noundef [[L]], i32 noundef [[L]], i32 noundef [[L]]) #[[ATTR4]]
+; CGSCC-NEXT:    ret void
+;
+  %i = icmp eq i32 %C, 42
+  br i1 %i, label %t, label %f
+t:
+  store i32 333, ptr addrspace(3) @AS3OneKernelAtATime
+  br label %f
+f:
+  %l = load i32, ptr addrspace(3) @AS3OneKernelAtATime
+  call void @use(i32 %l,i32 %l, i32 %l)
+  ret void
+}
+
+define dso_local void @kernel3(i32 %C) norecurse "kernel" {
+; TUNIT: Function Attrs: norecurse nosync nounwind
+; TUNIT-LABEL: define {{[^@]+}}@kernel3
+; TUNIT-SAME: (i32 [[C:%.*]]) #[[ATTR0]] {
+; TUNIT-NEXT:    [[I:%.*]] = icmp eq i32 [[C]], 42
+; TUNIT-NEXT:    br i1 [[I]], label [[T:%.*]], label [[F:%.*]]
+; TUNIT:       t:
+; TUNIT-NEXT:    call void @use(i32 noundef 42, i32 noundef 42, i32 noundef 42) #[[ATTR7]]
+; TUNIT-NEXT:    ret void
+; TUNIT:       f:
+; TUNIT-NEXT:    ret void
+;
+; CGSCC: Function Attrs: norecurse nosync nounwind
+; CGSCC-LABEL: define {{[^@]+}}@kernel3
+; CGSCC-SAME: (i32 [[C:%.*]]) #[[ATTR0]] {
+; CGSCC-NEXT:    [[I:%.*]] = icmp eq i32 [[C]], 42
+; CGSCC-NEXT:    br i1 [[I]], label [[T:%.*]], label [[F:%.*]]
+; CGSCC:       t:
+; CGSCC-NEXT:    call void @use(i32 noundef 42, i32 noundef 42, i32 noundef 42) #[[ATTR4]]
+; CGSCC-NEXT:    ret void
+; CGSCC:       f:
+; CGSCC-NEXT:    ret void
+;
+  %i = icmp eq i32 %C, 42
+  br i1 %i, label %t, label %f
+t:
+  %l = load i32, ptr addrspace(3) @AS3OneKernelAtATime
+  call void @use(i32 %l,i32 %l, i32 %l)
+  ret void
+f:
+  ret void
+}
+
 declare dso_local void @use(i32, i32, i32) nosync norecurse nounwind
 
 ;.
diff --git a/llvm/test/Transforms/Attributor/value-simplify.ll b/llvm/test/Transforms/Attributor/value-simplify.ll
index cbdc190c6e..0ee06afe92 100644
--- a/llvm/test/Transforms/Attributor/value-simplify.ll
+++ b/llvm/test/Transforms/Attributor/value-simplify.ll
@@ -9,10 +9,16 @@ declare ptr @llvm.call.preallocated.arg(token, i32)
 
 @str = private unnamed_addr addrspace(4) constant [1 x i8] c"\00", align 1
 @ConstAS3Ptr = addrspace(3) global i32 0, align 4
+@ConstPtr = constant i32 0, align 4
+@ConstWeakPtr = weak constant i32 0, align 4
+@ConstWeakODRPtr = weak_odr constant i32 0, align 4
 
 ;.
 ; CHECK: @[[STR:[a-zA-Z0-9_$"\\.-]+]] = private unnamed_addr addrspace(4) constant [1 x i8] zeroinitializer, align 1
 ; CHECK: @[[CONSTAS3PTR:[a-zA-Z0-9_$"\\.-]+]] = addrspace(3) global i32 0, align 4
+; CHECK: @[[CONSTPTR:[a-zA-Z0-9_$"\\.-]+]] = constant i32 0, align 4
+; CHECK: @[[CONSTWEAKPTR:[a-zA-Z0-9_$"\\.-]+]] = weak constant i32 0, align 4
+; CHECK: @[[CONSTWEAKODRPTR:[a-zA-Z0-9_$"\\.-]+]] = weak_odr constant i32 0, align 4
 ; CHECK: @[[S:[a-zA-Z0-9_$"\\.-]+]] = external global [[STRUCT_X:%.*]]
 ; CHECK: @[[G:[a-zA-Z0-9_$"\\.-]+]] = internal constant { [2 x ptr] } { [2 x ptr] [ptr @f1, ptr @f2] }
 ; CHECK: @[[X:[a-zA-Z0-9_$"\\.-]+]] = external global i32
@@ -1632,6 +1638,53 @@ dead5:
   ret i8 5
 }
 
+define i32 @readConst() {
+; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
+; TUNIT-LABEL: define {{[^@]+}}@readConst
+; TUNIT-SAME: () #[[ATTR2]] {
+; TUNIT-NEXT:    ret i32 0
+;
+; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
+; CGSCC-LABEL: define {{[^@]+}}@readConst
+; CGSCC-SAME: () #[[ATTR1]] {
+; CGSCC-NEXT:    ret i32 0
+;
+  %l = load i32, ptr @ConstPtr
+  ret i32 %l
+}
+
+define i32 @readWeakConst() {
+; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
+; TUNIT-LABEL: define {{[^@]+}}@readWeakConst
+; TUNIT-SAME: () #[[ATTR2]] {
+; TUNIT-NEXT:    [[L:%.*]] = load i32, ptr @ConstWeakPtr, align 4
+; TUNIT-NEXT:    ret i32 [[L]]
+;
+; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
+; CGSCC-LABEL: define {{[^@]+}}@readWeakConst
+; CGSCC-SAME: () #[[ATTR1]] {
+; CGSCC-NEXT:    [[L:%.*]] = load i32, ptr @ConstWeakPtr, align 4
+; CGSCC-NEXT:    ret i32 [[L]]
+;
+  %l = load i32, ptr @ConstWeakPtr
+  ret i32 %l
+}
+
+define i32 @readWeakOdrConst() {
+; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
+; TUNIT-LABEL: define {{[^@]+}}@readWeakOdrConst
+; TUNIT-SAME: () #[[ATTR2]] {
+; TUNIT-NEXT:    ret i32 0
+;
+; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
+; CGSCC-LABEL: define {{[^@]+}}@readWeakOdrConst
+; CGSCC-SAME: () #[[ATTR1]] {
+; CGSCC-NEXT:    ret i32 0
+;
+  %l = load i32, ptr @ConstWeakODRPtr
+  ret i32 %l
+}
+
 ;.
 ; TUNIT: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind willreturn }
 ; TUNIT: attributes #[[ATTR1]] = { memory(readwrite, argmem: none) }
diff --git a/llvm/test/Transforms/CodeGenPrepare/AArch64/sink-gather-scatter-addressing.ll b/llvm/test/Transforms/CodeGenPrepare/AArch64/sink-gather-scatter-addressing.ll
new file mode 100644
index 0000000000..73322836d1
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/AArch64/sink-gather-scatter-addressing.ll
@@ -0,0 +1,231 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
+; RUN: opt -S --codegenprepare < %s | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+; Sink the GEP to make use of scalar+vector addressing modes.
+define <vscale x 4 x float> @gather_offsets_sink_gep(ptr %base, <vscale x 4 x i32> %indices, <vscale x 4 x i1> %mask, i1 %cond) {
+; CHECK-LABEL: define <vscale x 4 x float> @gather_offsets_sink_gep(
+; CHECK-SAME: ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND]], label [[COND_BLOCK:%.*]], label [[EXIT:%.*]]
+; CHECK:       cond.block:
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr float, ptr [[BASE]], <vscale x 4 x i32> [[INDICES]]
+; CHECK-NEXT:    [[LOAD:%.*]] = tail call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0(<vscale x 4 x ptr> [[TMP0]], i32 4, <vscale x 4 x i1> [[MASK]], <vscale x 4 x float> poison)
+; CHECK-NEXT:    ret <vscale x 4 x float> [[LOAD]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret <vscale x 4 x float> zeroinitializer
+;
+entry:
+  %ptrs = getelementptr float, ptr %base, <vscale x 4 x i32> %indices
+  br i1 %cond, label %cond.block, label %exit
+
+cond.block:
+  %load = tail call <vscale x 4 x float> @llvm.masked.gather.nxv4f32(<vscale x 4 x ptr> %ptrs, i32 4, <vscale x 4 x i1> %mask, <vscale x 4 x float> poison)
+  br label %exit
+
+exit:
+  %ret = phi <vscale x 4 x float> [ zeroinitializer, %entry ], [ %load, %cond.block ]
+  ret <vscale x 4 x float> %ret
+}
+
+; Sink sext to make use of scalar+sxtw(vector) addressing modes.
+define <vscale x 4 x float> @gather_offsets_sink_sext(ptr %base, <vscale x 4 x i32> %indices, <vscale x 4 x i1> %mask, i1 %cond) {
+; CHECK-LABEL: define <vscale x 4 x float> @gather_offsets_sink_sext(
+; CHECK-SAME: ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND]], label [[COND_BLOCK:%.*]], label [[EXIT:%.*]]
+; CHECK:       cond.block:
+; CHECK-NEXT:    [[TMP0:%.*]] = sext <vscale x 4 x i32> [[INDICES]] to <vscale x 4 x i64>
+; CHECK-NEXT:    [[PTRS:%.*]] = getelementptr float, ptr [[BASE]], <vscale x 4 x i64> [[TMP0]]
+; CHECK-NEXT:    [[LOAD:%.*]] = tail call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0(<vscale x 4 x ptr> [[PTRS]], i32 4, <vscale x 4 x i1> [[MASK]], <vscale x 4 x float> poison)
+; CHECK-NEXT:    ret <vscale x 4 x float> [[LOAD]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret <vscale x 4 x float> zeroinitializer
+;
+entry:
+  %indices.sext = sext <vscale x 4 x i32> %indices to <vscale x 4 x i64>
+  br i1 %cond, label %cond.block, label %exit
+
+cond.block:
+  %ptrs = getelementptr float, ptr %base, <vscale x 4 x i64> %indices.sext
+  %load = tail call <vscale x 4 x float> @llvm.masked.gather.nxv4f32(<vscale x 4 x ptr> %ptrs, i32 4, <vscale x 4 x i1> %mask, <vscale x 4 x float> poison)
+  br label %exit
+
+exit:
+  %ret = phi <vscale x 4 x float> [ zeroinitializer, %entry ], [ %load, %cond.block ]
+  ret <vscale x 4 x float> %ret
+}
+
+; As above but ensure both the GEP and sext is sunk.
+define <vscale x 4 x float> @gather_offsets_sink_sext_get(ptr %base, <vscale x 4 x i32> %indices, <vscale x 4 x i1> %mask, i1 %cond) {
+; CHECK-LABEL: define <vscale x 4 x float> @gather_offsets_sink_sext_get(
+; CHECK-SAME: ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND]], label [[COND_BLOCK:%.*]], label [[EXIT:%.*]]
+; CHECK:       cond.block:
+; CHECK-NEXT:    [[TMP0:%.*]] = sext <vscale x 4 x i32> [[INDICES]] to <vscale x 4 x i64>
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr float, ptr [[BASE]], <vscale x 4 x i64> [[TMP0]]
+; CHECK-NEXT:    [[LOAD:%.*]] = tail call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0(<vscale x 4 x ptr> [[TMP1]], i32 4, <vscale x 4 x i1> [[MASK]], <vscale x 4 x float> poison)
+; CHECK-NEXT:    ret <vscale x 4 x float> [[LOAD]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret <vscale x 4 x float> zeroinitializer
+;
+entry:
+  %indices.sext = sext <vscale x 4 x i32> %indices to <vscale x 4 x i64>
+  %ptrs = getelementptr float, ptr %base, <vscale x 4 x i64> %indices.sext
+  br i1 %cond, label %cond.block, label %exit
+
+cond.block:
+  %load = tail call <vscale x 4 x float> @llvm.masked.gather.nxv4f32(<vscale x 4 x ptr> %ptrs, i32 4, <vscale x 4 x i1> %mask, <vscale x 4 x float> poison)
+  br label %exit
+
+exit:
+  %ret = phi <vscale x 4 x float> [ zeroinitializer, %entry ], [ %load, %cond.block ]
+  ret <vscale x 4 x float> %ret
+}
+
+; Don't sink GEPs that cannot benefit from SVE's scalar+vector addressing modes.
+define <vscale x 4 x float> @gather_no_scalar_base(<vscale x 4 x ptr> %bases, <vscale x 4 x i32> %indices, <vscale x 4 x i1> %mask, i1 %cond) {
+; CHECK-LABEL: define <vscale x 4 x float> @gather_no_scalar_base(
+; CHECK-SAME: <vscale x 4 x ptr> [[BASES:%.*]], <vscale x 4 x i32> [[INDICES:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PTRS:%.*]] = getelementptr float, <vscale x 4 x ptr> [[BASES]], <vscale x 4 x i32> [[INDICES]]
+; CHECK-NEXT:    br i1 [[COND]], label [[COND_BLOCK:%.*]], label [[EXIT:%.*]]
+; CHECK:       cond.block:
+; CHECK-NEXT:    [[LOAD:%.*]] = tail call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0(<vscale x 4 x ptr> [[PTRS]], i32 4, <vscale x 4 x i1> [[MASK]], <vscale x 4 x float> poison)
+; CHECK-NEXT:    ret <vscale x 4 x float> [[LOAD]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret <vscale x 4 x float> zeroinitializer
+;
+entry:
+  %ptrs = getelementptr float, <vscale x 4 x ptr> %bases, <vscale x 4 x i32> %indices
+  br i1 %cond, label %cond.block, label %exit
+
+cond.block:
+  %load = tail call <vscale x 4 x float> @llvm.masked.gather.nxv4f32(<vscale x 4 x ptr> %ptrs, i32 4, <vscale x 4 x i1> %mask, <vscale x 4 x float> poison)
+  br label %exit
+
+exit:
+  %ret = phi <vscale x 4 x float> [ zeroinitializer, %entry ], [ %load, %cond.block ]
+  ret <vscale x 4 x float> %ret
+}
+
+; Don't sink extends whose result type is already favourable for SVE's sxtw/uxtw addressing modes.
+; NOTE: We still want to sink the GEP.
+define <vscale x 4 x float> @gather_offset_type_too_small(ptr %base, <vscale x 4 x i8> %indices, <vscale x 4 x i1> %mask, i1 %cond) {
+; CHECK-LABEL: define <vscale x 4 x float> @gather_offset_type_too_small(
+; CHECK-SAME: ptr [[BASE:%.*]], <vscale x 4 x i8> [[INDICES:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INDICES_SEXT:%.*]] = sext <vscale x 4 x i8> [[INDICES]] to <vscale x 4 x i32>
+; CHECK-NEXT:    br i1 [[COND]], label [[COND_BLOCK:%.*]], label [[EXIT:%.*]]
+; CHECK:       cond.block:
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr float, ptr [[BASE]], <vscale x 4 x i32> [[INDICES_SEXT]]
+; CHECK-NEXT:    [[LOAD:%.*]] = tail call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0(<vscale x 4 x ptr> [[TMP0]], i32 4, <vscale x 4 x i1> [[MASK]], <vscale x 4 x float> poison)
+; CHECK-NEXT:    ret <vscale x 4 x float> [[LOAD]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret <vscale x 4 x float> zeroinitializer
+;
+entry:
+  %indices.sext = sext <vscale x 4 x i8> %indices to <vscale x 4 x i32>
+  %ptrs = getelementptr float, ptr %base, <vscale x 4 x i32> %indices.sext
+  br i1 %cond, label %cond.block, label %exit
+
+cond.block:
+  %load = tail call <vscale x 4 x float> @llvm.masked.gather.nxv4f32(<vscale x 4 x ptr> %ptrs, i32 4, <vscale x 4 x i1> %mask, <vscale x 4 x float> poison)
+  br label %exit
+
+exit:
+  %ret = phi <vscale x 4 x float> [ zeroinitializer, %entry ], [ %load, %cond.block ]
+  ret <vscale x 4 x float> %ret
+}
+
+; Don't sink extends that cannot benefit from SVE's sxtw/uxtw addressing modes.
+; NOTE: We still want to sink the GEP.
+define <vscale x 4 x float> @gather_offset_type_too_big(ptr %base, <vscale x 4 x i48> %indices, <vscale x 4 x i1> %mask, i1 %cond) {
+; CHECK-LABEL: define <vscale x 4 x float> @gather_offset_type_too_big(
+; CHECK-SAME: ptr [[BASE:%.*]], <vscale x 4 x i48> [[INDICES:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INDICES_SEXT:%.*]] = sext <vscale x 4 x i48> [[INDICES]] to <vscale x 4 x i64>
+; CHECK-NEXT:    br i1 [[COND]], label [[COND_BLOCK:%.*]], label [[EXIT:%.*]]
+; CHECK:       cond.block:
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr float, ptr [[BASE]], <vscale x 4 x i64> [[INDICES_SEXT]]
+; CHECK-NEXT:    [[LOAD:%.*]] = tail call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0(<vscale x 4 x ptr> [[TMP0]], i32 4, <vscale x 4 x i1> [[MASK]], <vscale x 4 x float> poison)
+; CHECK-NEXT:    ret <vscale x 4 x float> [[LOAD]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret <vscale x 4 x float> zeroinitializer
+;
+entry:
+  %indices.sext = sext <vscale x 4 x i48> %indices to <vscale x 4 x i64>
+  %ptrs = getelementptr float, ptr %base, <vscale x 4 x i64> %indices.sext
+  br i1 %cond, label %cond.block, label %exit
+
+cond.block:
+  %load = tail call <vscale x 4 x float> @llvm.masked.gather.nxv4f32(<vscale x 4 x ptr> %ptrs, i32 4, <vscale x 4 x i1> %mask, <vscale x 4 x float> poison)
+  br label %exit
+
+exit:
+  %ret = phi <vscale x 4 x float> [ zeroinitializer, %entry ], [ %load, %cond.block ]
+  ret <vscale x 4 x float> %ret
+}
+
+; Sink zext to make use of scalar+uxtw(vector) addressing modes.
+; TODO: There's an argument here to split the extend into i8->i32 and i32->i64,
+; which would be especially useful if the i8s are the result of a load because
+; it would maintain the use of sign-extending loads.
+define <vscale x 4 x float> @gather_offset_sink_zext(ptr %base, <vscale x 4 x i8> %indices, <vscale x 4 x i1> %mask, i1 %cond) {
+; CHECK-LABEL: define <vscale x 4 x float> @gather_offset_sink_zext(
+; CHECK-SAME: ptr [[BASE:%.*]], <vscale x 4 x i8> [[INDICES:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND]], label [[COND_BLOCK:%.*]], label [[EXIT:%.*]]
+; CHECK:       cond.block:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext <vscale x 4 x i8> [[INDICES]] to <vscale x 4 x i64>
+; CHECK-NEXT:    [[PTRS:%.*]] = getelementptr float, ptr [[BASE]], <vscale x 4 x i64> [[TMP0]]
+; CHECK-NEXT:    [[LOAD:%.*]] = tail call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0(<vscale x 4 x ptr> [[PTRS]], i32 4, <vscale x 4 x i1> [[MASK]], <vscale x 4 x float> poison)
+; CHECK-NEXT:    ret <vscale x 4 x float> [[LOAD]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret <vscale x 4 x float> zeroinitializer
+;
+entry:
+  %indices.zext = zext <vscale x 4 x i8> %indices to <vscale x 4 x i64>
+  br i1 %cond, label %cond.block, label %exit
+
+cond.block:
+  %ptrs = getelementptr float, ptr %base, <vscale x 4 x i64> %indices.zext
+  %load = tail call <vscale x 4 x float> @llvm.masked.gather.nxv4f32(<vscale x 4 x ptr> %ptrs, i32 4, <vscale x 4 x i1> %mask, <vscale x 4 x float> poison)
+  br label %exit
+
+exit:
+  %ret = phi <vscale x 4 x float> [ zeroinitializer, %entry ], [ %load, %cond.block ]
+  ret <vscale x 4 x float> %ret
+}
+
+; Ensure we support scatters as well as gathers.
+define void @scatter_offsets_sink_sext_get(<vscale x 4 x float> %data, ptr %base, <vscale x 4 x i32> %indices, <vscale x 4 x i1> %mask, i1 %cond) {
+; CHECK-LABEL: define void @scatter_offsets_sink_sext_get(
+; CHECK-SAME: <vscale x 4 x float> [[DATA:%.*]], ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDICES:%.*]], <vscale x 4 x i1> [[MASK:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND]], label [[COND_BLOCK:%.*]], label [[EXIT:%.*]]
+; CHECK:       cond.block:
+; CHECK-NEXT:    [[TMP0:%.*]] = sext <vscale x 4 x i32> [[INDICES]] to <vscale x 4 x i64>
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr float, ptr [[BASE]], <vscale x 4 x i64> [[TMP0]]
+; CHECK-NEXT:    tail call void @llvm.masked.scatter.nxv4f32.nxv4p0(<vscale x 4 x float> [[DATA]], <vscale x 4 x ptr> [[TMP1]], i32 4, <vscale x 4 x i1> [[MASK]])
+; CHECK-NEXT:    ret void
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %indices.sext = sext <vscale x 4 x i32> %indices to <vscale x 4 x i64>
+  %ptrs = getelementptr float, ptr %base, <vscale x 4 x i64> %indices.sext
+  br i1 %cond, label %cond.block, label %exit
+
+cond.block:
+  tail call void @llvm.masked.scatter.nxv4f32(<vscale x 4 x float> %data, <vscale x 4 x ptr> %ptrs, i32 4, <vscale x 4 x i1> %mask)
+  br label %exit
+
+exit:
+  ret void
+}
+
+declare <vscale x 4 x float> @llvm.masked.gather.nxv4f32(<vscale x 4 x ptr>, i32, <vscale x 4 x i1>, <vscale x 4 x float>)
+declare void @llvm.masked.scatter.nxv4f32(<vscale x 4 x float>, <vscale x 4 x ptr>, i32, <vscale x 4 x i1>)
diff --git a/llvm/test/Transforms/ConstraintElimination/large-constant-ints.ll b/llvm/test/Transforms/ConstraintElimination/large-constant-ints.ll
index 8e6fa126a5..9568b155af 100644
--- a/llvm/test/Transforms/ConstraintElimination/large-constant-ints.ll
+++ b/llvm/test/Transforms/ConstraintElimination/large-constant-ints.ll
@@ -96,6 +96,7 @@ else:
   ret i1 false
 }
 
+; TODO: This could be folded.
 define i1 @sub_decomp_i80(i80 %a) {
 ; CHECK-LABEL: @sub_decomp_i80(
 ; CHECK-NEXT:  entry:
@@ -104,7 +105,8 @@ define i1 @sub_decomp_i80(i80 %a) {
 ; CHECK-NEXT:    br i1 [[C]], label [[THEN:%.*]], label [[ELSE:%.*]]
 ; CHECK:       then:
 ; CHECK-NEXT:    [[SUB_1:%.*]] = sub nuw i80 [[A]], 1973801615886922022913
-; CHECK-NEXT:    ret i1 true
+; CHECK-NEXT:    [[C_1:%.*]] = icmp ult i80 [[SUB_1]], 1346612317380797267967
+; CHECK-NEXT:    ret i1 [[C_1]]
 ; CHECK:       else:
 ; CHECK-NEXT:    ret i1 false
 ;
@@ -418,4 +420,17 @@ entry:
   ret i1 %res
 }
 
+define i1 @pr68751(i128 %arg) {
+; CHECK-LABEL: @pr68751(
+; CHECK-NEXT:    [[SHL1:%.*]] = shl nuw nsw i128 [[ARG:%.*]], 32
+; CHECK-NEXT:    [[SHL2:%.*]] = shl nuw nsw i128 [[SHL1]], 32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i128 [[SHL2]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl1 = shl nuw nsw i128 %arg, 32
+  %shl2 = shl nuw nsw i128 %shl1, 32
+  %cmp = icmp eq i128 %shl2, 0
+  ret i1 %cmp
+}
+
 declare void @llvm.assume(i1)
diff --git a/llvm/test/Transforms/ConstraintElimination/shl.ll b/llvm/test/Transforms/ConstraintElimination/shl.ll
index 982e0e7458..9f98a9d3a5 100644
--- a/llvm/test/Transforms/ConstraintElimination/shl.ll
+++ b/llvm/test/Transforms/ConstraintElimination/shl.ll
@@ -1277,7 +1277,8 @@ define i1 @shl_55() {
 ; CHECK-LABEL: @shl_55(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SHL_UB:%.*]] = shl nuw nsw i256 1, 55
-; CHECK-NEXT:    ret i1 true
+; CHECK-NEXT:    [[SHL_CMP:%.*]] = icmp uge i256 [[SHL_UB]], 1
+; CHECK-NEXT:    ret i1 [[SHL_CMP]]
 ;
 entry:
   %shl.ub = shl nuw nsw i256 1, 55
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/abs.ll b/llvm/test/Transforms/CorrelatedValuePropagation/abs.ll
index 6231b05a85..7f10ce63e2 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/abs.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/abs.ll
@@ -2,6 +2,7 @@
 ; RUN: opt < %s -passes=correlated-propagation -S | FileCheck %s
 
 declare void @llvm.assume(i1)
+declare i32 @llvm.abs.i32(i32, i1)
 declare i8 @llvm.abs.i8(i8, i1)
 declare i1 @llvm.abs.i1(i1, i1)
 
@@ -379,11 +380,139 @@ define i8 @test27(i8 %x) {
 
 define i1 @pr59887(i1 %x, i1 %c) {
 ; CHECK-LABEL: @pr59887(
-; CHECK-NEXT:    [[ABS:%.*]] = call i1 @llvm.abs.i1(i1 [[X:%.*]], i1 false)
-; CHECK-NEXT:    [[RES:%.*]] = select i1 [[C:%.*]], i1 [[ABS]], i1 false
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[C:%.*]], i1 [[X:%.*]], i1 false
 ; CHECK-NEXT:    ret i1 [[RES]]
 ;
   %abs = call i1 @llvm.abs.i1(i1 %x, i1 false)
   %res = select i1 %c, i1 %abs, i1 false
   ret i1 %res
 }
+
+; Because of `undef`, We can't delete `abs`.
+; We can't replace the `abs` argument with true either.
+define i32 @pr68381_undef_abs_false(i1 %c0, i1 %c1, i8 %v1) {
+; CHECK-LABEL: @pr68381_undef_abs_false(
+; CHECK-NEXT:  start:
+; CHECK-NEXT:    br i1 [[C0:%.*]], label [[BB0:%.*]], label [[BB1:%.*]]
+; CHECK:       bb0:
+; CHECK-NEXT:    [[V1_I32:%.*]] = zext i8 [[V1:%.*]] to i32
+; CHECK-NEXT:    br label [[BB1]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[X:%.*]] = phi i32 [ [[V1_I32]], [[BB0]] ], [ undef, [[START:%.*]] ]
+; CHECK-NEXT:    br i1 [[C1:%.*]], label [[BB0]], label [[BB2:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[Z:%.*]] = call i32 @llvm.abs.i32(i32 [[X]], i1 false)
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+start:
+  br i1 %c0, label %bb0, label %bb1
+
+bb0:
+  %v1_i32 = zext i8 %v1 to i32
+  br label %bb1
+
+bb1:
+  %x = phi i32 [ %v1_i32, %bb0 ], [ undef, %start ]
+  br i1 %c1, label %bb0, label %bb2
+
+bb2:
+  %z = call i32 @llvm.abs.i32(i32 %x, i1 false)
+  ret i32 %z
+}
+
+; Because of `and`, we can delete `abs`.
+define i32 @pr68381_undef_abs_false_and(i1 %c0, i1 %c1, i8 %v1) {
+; CHECK-LABEL: @pr68381_undef_abs_false_and(
+; CHECK-NEXT:  start:
+; CHECK-NEXT:    br i1 [[C0:%.*]], label [[BB0:%.*]], label [[BB1:%.*]]
+; CHECK:       bb0:
+; CHECK-NEXT:    [[V1_I32:%.*]] = zext i8 [[V1:%.*]] to i32
+; CHECK-NEXT:    br label [[BB1]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[X:%.*]] = phi i32 [ [[V1_I32]], [[BB0]] ], [ undef, [[START:%.*]] ]
+; CHECK-NEXT:    br i1 [[C1:%.*]], label [[BB0]], label [[BB2:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[Y:%.*]] = and i32 [[X]], 255
+; CHECK-NEXT:    ret i32 [[Y]]
+;
+start:
+  br i1 %c0, label %bb0, label %bb1
+
+bb0:
+  %v1_i32 = zext i8 %v1 to i32
+  br label %bb1
+
+bb1:
+  %x = phi i32 [ %v1_i32, %bb0 ], [ undef, %start ]
+  br i1 %c1, label %bb0, label %bb2
+
+bb2:
+  %y = and i32 %x, 255
+  %z = call i32 @llvm.abs.i32(i32 %y, i1 false)
+  ret i32 %z
+}
+
+; Because of `undef`, we can't replace `abs` with `sub`.
+define i32 @pr68381_undef_abs_false_sub(i1 %c0, i1 %c1, i32 %v1, i32 %v2) {
+; CHECK-LABEL: @pr68381_undef_abs_false_sub(
+; CHECK-NEXT:  start:
+; CHECK-NEXT:    br i1 [[C0:%.*]], label [[BB0:%.*]], label [[BB1:%.*]]
+; CHECK:       bb0:
+; CHECK-NEXT:    [[V3:%.*]] = add i32 [[V1:%.*]], [[V2:%.*]]
+; CHECK-NEXT:    [[LIM:%.*]] = icmp sle i32 [[V3]], -1
+; CHECK-NEXT:    call void @llvm.assume(i1 [[LIM]])
+; CHECK-NEXT:    br label [[BB1]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[X:%.*]] = phi i32 [ [[V3]], [[BB0]] ], [ undef, [[START:%.*]] ]
+; CHECK-NEXT:    br i1 [[C1:%.*]], label [[BB0]], label [[BB2:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[Z:%.*]] = call i32 @llvm.abs.i32(i32 [[X]], i1 false)
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+start:
+  br i1 %c0, label %bb0, label %bb1
+
+bb0:
+  %v3 = add i32 %v1, %v2
+  %lim = icmp sle i32 %v3, -1
+  call void @llvm.assume(i1 %lim)
+  br label %bb1
+
+bb1:
+  %x = phi i32 [ %v3, %bb0 ], [ undef, %start ]
+  br i1 %c1, label %bb0, label %bb2
+
+bb2:
+  %z = call i32 @llvm.abs.i32(i32 %x, i1 false)
+  ret i32 %z
+}
+
+; We can delete `abs`.
+define i32 @pr68381_undef_abs_true(i1 %c0, i1 %c1, i8 %v1) {
+; CHECK-LABEL: @pr68381_undef_abs_true(
+; CHECK-NEXT:  start:
+; CHECK-NEXT:    br i1 [[C0:%.*]], label [[BB0:%.*]], label [[BB1:%.*]]
+; CHECK:       bb0:
+; CHECK-NEXT:    [[V1_I32:%.*]] = zext i8 [[V1:%.*]] to i32
+; CHECK-NEXT:    br label [[BB1]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[X:%.*]] = phi i32 [ [[V1_I32]], [[BB0]] ], [ undef, [[START:%.*]] ]
+; CHECK-NEXT:    br i1 [[C1:%.*]], label [[BB0]], label [[BB2:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    ret i32 [[X]]
+;
+start:
+  br i1 %c0, label %bb0, label %bb1
+
+bb0:
+  %v1_i32 = zext i8 %v1 to i32
+  br label %bb1
+
+bb1:
+  %x = phi i32 [ %v1_i32, %bb0 ], [ undef, %start ]
+  br i1 %c1, label %bb0, label %bb2
+
+bb2:
+  %z = call i32 @llvm.abs.i32(i32 %x, i1 true)
+  ret i32 %z
+}
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/icmp.ll b/llvm/test/Transforms/CorrelatedValuePropagation/icmp.ll
index c4f0ade399..b1f4b5246b 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/icmp.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/icmp.ll
@@ -1240,6 +1240,110 @@ define <2 x i1> @non_const_range_minmax_vec(<2 x i8> %a, <2 x i8> %b) {
   ret <2 x i1> %cmp1
 }
 
+define void @ashr_sgt(i8 %x) {
+; CHECK-LABEL: @ashr_sgt(
+; CHECK-NEXT:    [[S:%.*]] = ashr i8 [[X:%.*]], 2
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i8 [[S]], 1
+; CHECK-NEXT:    br i1 [[C]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    call void @check1(i1 true)
+; CHECK-NEXT:    [[C3:%.*]] = icmp ugt i8 [[X]], 8
+; CHECK-NEXT:    call void @check1(i1 [[C3]])
+; CHECK-NEXT:    ret void
+; CHECK:       else:
+; CHECK-NEXT:    ret void
+;
+  %s = ashr i8 %x, 2
+  %c = icmp sgt i8 %s, 1
+  br i1 %c, label %if, label %else
+if:
+  %c2 = icmp sgt i8 %x, 7
+  call void @check1(i1 %c2)
+  %c3 = icmp sgt i8 %x, 8
+  call void @check1(i1 %c3)
+  ret void
+else:
+  ret void
+}
+
+define void @ashr_sge(i8 %x) {
+; CHECK-LABEL: @ashr_sge(
+; CHECK-NEXT:    [[S:%.*]] = ashr i8 [[X:%.*]], 2
+; CHECK-NEXT:    [[C:%.*]] = icmp sge i8 [[S]], 1
+; CHECK-NEXT:    br i1 [[C]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    call void @check1(i1 true)
+; CHECK-NEXT:    [[C3:%.*]] = icmp uge i8 [[X]], 5
+; CHECK-NEXT:    call void @check1(i1 [[C3]])
+; CHECK-NEXT:    ret void
+; CHECK:       else:
+; CHECK-NEXT:    ret void
+;
+  %s = ashr i8 %x, 2
+  %c = icmp sge i8 %s, 1
+  br i1 %c, label %if, label %else
+if:
+  %c2 = icmp sge i8 %x, 4
+  call void @check1(i1 %c2)
+  %c3 = icmp sge i8 %x, 5
+  call void @check1(i1 %c3)
+  ret void
+else:
+  ret void
+}
+
+define void @ashr_slt(i8 %x) {
+; CHECK-LABEL: @ashr_slt(
+; CHECK-NEXT:    [[S:%.*]] = ashr i8 [[X:%.*]], 2
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i8 [[S]], 1
+; CHECK-NEXT:    br i1 [[C]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    call void @check1(i1 true)
+; CHECK-NEXT:    [[C3:%.*]] = icmp slt i8 [[X]], 3
+; CHECK-NEXT:    call void @check1(i1 [[C3]])
+; CHECK-NEXT:    ret void
+; CHECK:       else:
+; CHECK-NEXT:    ret void
+;
+  %s = ashr i8 %x, 2
+  %c = icmp slt i8 %s, 1
+  br i1 %c, label %if, label %else
+if:
+  %c2 = icmp slt i8 %x, 4
+  call void @check1(i1 %c2)
+  %c3 = icmp slt i8 %x, 3
+  call void @check1(i1 %c3)
+  ret void
+else:
+  ret void
+}
+
+define void @ashr_sle(i8 %x) {
+; CHECK-LABEL: @ashr_sle(
+; CHECK-NEXT:    [[S:%.*]] = ashr i8 [[X:%.*]], 2
+; CHECK-NEXT:    [[C:%.*]] = icmp sle i8 [[S]], 1
+; CHECK-NEXT:    br i1 [[C]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    call void @check1(i1 true)
+; CHECK-NEXT:    [[C3:%.*]] = icmp sle i8 [[X]], 6
+; CHECK-NEXT:    call void @check1(i1 [[C3]])
+; CHECK-NEXT:    ret void
+; CHECK:       else:
+; CHECK-NEXT:    ret void
+;
+  %s = ashr i8 %x, 2
+  %c = icmp sle i8 %s, 1
+  br i1 %c, label %if, label %else
+if:
+  %c2 = icmp sle i8 %x, 7
+  call void @check1(i1 %c2)
+  %c3 = icmp sle i8 %x, 6
+  call void @check1(i1 %c3)
+  ret void
+else:
+  ret void
+}
+
 declare i8 @llvm.umin.i8(i8, i8)
 declare i8 @llvm.umax.i8(i8, i8)
 declare <2 x i8> @llvm.umin.v2i8(<2 x i8>, <2 x i8>)
diff --git a/llvm/test/Transforms/FunctionAttrs/nonnull.ll b/llvm/test/Transforms/FunctionAttrs/nonnull.ll
index 33e9b377d7..7ca07e5493 100644
--- a/llvm/test/Transforms/FunctionAttrs/nonnull.ll
+++ b/llvm/test/Transforms/FunctionAttrs/nonnull.ll
@@ -18,12 +18,12 @@ define ptr @test1() {
 
 ; Return a pointer trivially nonnull (argument attribute)
 define ptr @test2(ptr nonnull %p) {
-; FNATTRS-LABEL: define nonnull ptr @test2
-; FNATTRS-SAME: (ptr nonnull readnone returned [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; FNATTRS-LABEL: define nonnull ptr @test2(
+; FNATTRS-SAME: ptr nonnull readnone returned [[P:%.*]]) #[[ATTR0:[0-9]+]] {
 ; FNATTRS-NEXT:    ret ptr [[P]]
 ;
-; ATTRIBUTOR-LABEL: define nonnull ptr @test2
-; ATTRIBUTOR-SAME: (ptr nofree nonnull readnone [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; ATTRIBUTOR-LABEL: define nonnull ptr @test2(
+; ATTRIBUTOR-SAME: ptr nofree nonnull readnone [[P:%.*]]) #[[ATTR0:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:    ret ptr [[P]]
 ;
   ret ptr %p
@@ -32,8 +32,8 @@ define ptr @test2(ptr nonnull %p) {
 ; Given an SCC where one of the functions can not be marked nonnull,
 ; can we still mark the other one which is trivially nonnull
 define ptr @scc_binder(i1 %c) {
-; COMMON-LABEL: define ptr @scc_binder
-; COMMON-SAME: (i1 [[C:%.*]]) {
+; COMMON-LABEL: define ptr @scc_binder(
+; COMMON-SAME: i1 [[C:%.*]]) {
 ; COMMON-NEXT:    br i1 [[C]], label [[REC:%.*]], label [[END:%.*]]
 ; COMMON:       rec:
 ; COMMON-NEXT:    [[TMP1:%.*]] = call ptr @test3(i1 [[C]])
@@ -50,8 +50,8 @@ end:
 }
 
 define ptr @test3(i1 %c) {
-; COMMON-LABEL: define nonnull ptr @test3
-; COMMON-SAME: (i1 [[C:%.*]]) {
+; COMMON-LABEL: define nonnull ptr @test3(
+; COMMON-SAME: i1 [[C:%.*]]) {
 ; COMMON-NEXT:    [[TMP1:%.*]] = call ptr @scc_binder(i1 [[C]])
 ; COMMON-NEXT:    [[RET:%.*]] = call ptr @ret_nonnull()
 ; COMMON-NEXT:    ret ptr [[RET]]
@@ -65,13 +65,13 @@ define ptr @test3(i1 %c) {
 ; nonnull if neither can ever return null.  (In this case, they
 ; just never return period.)
 define ptr @test4_helper() {
-; FNATTRS-LABEL: define noalias nonnull ptr @test4_helper
-; FNATTRS-SAME: () #[[ATTR1:[0-9]+]] {
+; FNATTRS-LABEL: define noalias nonnull ptr @test4_helper(
+; FNATTRS-SAME: ) #[[ATTR1:[0-9]+]] {
 ; FNATTRS-NEXT:    [[RET:%.*]] = call ptr @test4()
 ; FNATTRS-NEXT:    ret ptr [[RET]]
 ;
-; ATTRIBUTOR-LABEL: define ptr @test4_helper
-; ATTRIBUTOR-SAME: () #[[ATTR1:[0-9]+]] {
+; ATTRIBUTOR-LABEL: define ptr @test4_helper(
+; ATTRIBUTOR-SAME: ) #[[ATTR1:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:    [[RET:%.*]] = call ptr @test4() #[[ATTR1]]
 ; ATTRIBUTOR-NEXT:    ret ptr [[RET]]
 ;
@@ -80,13 +80,13 @@ define ptr @test4_helper() {
 }
 
 define ptr @test4() {
-; FNATTRS-LABEL: define noalias nonnull ptr @test4
-; FNATTRS-SAME: () #[[ATTR1]] {
+; FNATTRS-LABEL: define noalias nonnull ptr @test4(
+; FNATTRS-SAME: ) #[[ATTR1]] {
 ; FNATTRS-NEXT:    [[RET:%.*]] = call ptr @test4_helper()
 ; FNATTRS-NEXT:    ret ptr [[RET]]
 ;
-; ATTRIBUTOR-LABEL: define ptr @test4
-; ATTRIBUTOR-SAME: () #[[ATTR1]] {
+; ATTRIBUTOR-LABEL: define ptr @test4(
+; ATTRIBUTOR-SAME: ) #[[ATTR1]] {
 ; ATTRIBUTOR-NEXT:    [[RET:%.*]] = call ptr @test4_helper() #[[ATTR1]]
 ; ATTRIBUTOR-NEXT:    ret ptr [[RET]]
 ;
@@ -97,8 +97,8 @@ define ptr @test4() {
 ; Given a mutual recursive set of functions which *can* return null
 ; make sure we haven't marked them as nonnull.
 define ptr @test5_helper(i1 %c) {
-; FNATTRS-LABEL: define noalias ptr @test5_helper
-; FNATTRS-SAME: (i1 [[C:%.*]]) #[[ATTR1]] {
+; FNATTRS-LABEL: define noalias ptr @test5_helper(
+; FNATTRS-SAME: i1 [[C:%.*]]) #[[ATTR1]] {
 ; FNATTRS-NEXT:    br i1 [[C]], label [[REC:%.*]], label [[END:%.*]]
 ; FNATTRS:       rec:
 ; FNATTRS-NEXT:    [[RET:%.*]] = call ptr @test5(i1 [[C]])
@@ -106,8 +106,8 @@ define ptr @test5_helper(i1 %c) {
 ; FNATTRS:       end:
 ; FNATTRS-NEXT:    ret ptr null
 ;
-; ATTRIBUTOR-LABEL: define ptr @test5_helper
-; ATTRIBUTOR-SAME: (i1 [[C:%.*]]) #[[ATTR1]] {
+; ATTRIBUTOR-LABEL: define ptr @test5_helper(
+; ATTRIBUTOR-SAME: i1 [[C:%.*]]) #[[ATTR1]] {
 ; ATTRIBUTOR-NEXT:    br i1 [[C]], label [[REC:%.*]], label [[END:%.*]]
 ; ATTRIBUTOR:       rec:
 ; ATTRIBUTOR-NEXT:    [[RET:%.*]] = call ptr @test5(i1 [[C]]) #[[ATTR1]]
@@ -124,13 +124,13 @@ end:
 }
 
 define ptr @test5(i1 %c) {
-; FNATTRS-LABEL: define noalias ptr @test5
-; FNATTRS-SAME: (i1 [[C:%.*]]) #[[ATTR1]] {
+; FNATTRS-LABEL: define noalias ptr @test5(
+; FNATTRS-SAME: i1 [[C:%.*]]) #[[ATTR1]] {
 ; FNATTRS-NEXT:    [[RET:%.*]] = call ptr @test5_helper(i1 [[C]])
 ; FNATTRS-NEXT:    ret ptr [[RET]]
 ;
-; ATTRIBUTOR-LABEL: define ptr @test5
-; ATTRIBUTOR-SAME: (i1 [[C:%.*]]) #[[ATTR1]] {
+; ATTRIBUTOR-LABEL: define ptr @test5(
+; ATTRIBUTOR-SAME: i1 [[C:%.*]]) #[[ATTR1]] {
 ; ATTRIBUTOR-NEXT:    [[RET:%.*]] = call ptr @test5_helper(i1 [[C]]) #[[ATTR1]]
 ; ATTRIBUTOR-NEXT:    ret ptr [[RET]]
 ;
@@ -161,8 +161,8 @@ exit:
 }
 
 define ptr @test6b(i1 %c) {
-; COMMON-LABEL: define nonnull ptr @test6b
-; COMMON-SAME: (i1 [[C:%.*]]) {
+; COMMON-LABEL: define nonnull ptr @test6b(
+; COMMON-SAME: i1 [[C:%.*]]) {
 ; COMMON-NEXT:  entry:
 ; COMMON-NEXT:    [[RET:%.*]] = call ptr @ret_nonnull()
 ; COMMON-NEXT:    br label [[LOOP:%.*]]
@@ -183,25 +183,25 @@ exit:
 }
 
 define ptr @test7(ptr %a) {
-; FNATTRS-LABEL: define ptr @test7
-; FNATTRS-SAME: (ptr readnone returned [[A:%.*]]) #[[ATTR0]] {
+; FNATTRS-LABEL: define ptr @test7(
+; FNATTRS-SAME: ptr readnone returned [[A:%.*]]) #[[ATTR0]] {
 ; FNATTRS-NEXT:    ret ptr [[A]]
 ;
-; ATTRIBUTOR-LABEL: define ptr @test7
-; ATTRIBUTOR-SAME: (ptr nofree readnone [[A:%.*]]) #[[ATTR0]] {
+; ATTRIBUTOR-LABEL: define ptr @test7(
+; ATTRIBUTOR-SAME: ptr nofree readnone [[A:%.*]]) #[[ATTR0]] {
 ; ATTRIBUTOR-NEXT:    ret ptr [[A]]
 ;
   ret ptr %a
 }
 
 define ptr @test8(ptr %a) {
-; FNATTRS-LABEL: define nonnull ptr @test8
-; FNATTRS-SAME: (ptr readnone [[A:%.*]]) #[[ATTR0]] {
+; FNATTRS-LABEL: define nonnull ptr @test8(
+; FNATTRS-SAME: ptr readnone [[A:%.*]]) #[[ATTR0]] {
 ; FNATTRS-NEXT:    [[B:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 1
 ; FNATTRS-NEXT:    ret ptr [[B]]
 ;
-; ATTRIBUTOR-LABEL: define nonnull ptr @test8
-; ATTRIBUTOR-SAME: (ptr nofree readnone [[A:%.*]]) #[[ATTR0]] {
+; ATTRIBUTOR-LABEL: define nonnull ptr @test8(
+; ATTRIBUTOR-SAME: ptr nofree readnone [[A:%.*]]) #[[ATTR0]] {
 ; ATTRIBUTOR-NEXT:    [[B:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 1
 ; ATTRIBUTOR-NEXT:    ret ptr [[B]]
 ;
@@ -210,13 +210,13 @@ define ptr @test8(ptr %a) {
 }
 
 define ptr @test9(ptr %a, i64 %n) {
-; FNATTRS-LABEL: define ptr @test9
-; FNATTRS-SAME: (ptr readnone [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; FNATTRS-LABEL: define ptr @test9(
+; FNATTRS-SAME: ptr readnone [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; FNATTRS-NEXT:    [[B:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[N]]
 ; FNATTRS-NEXT:    ret ptr [[B]]
 ;
-; ATTRIBUTOR-LABEL: define ptr @test9
-; ATTRIBUTOR-SAME: (ptr nofree readnone [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; ATTRIBUTOR-LABEL: define ptr @test9(
+; ATTRIBUTOR-SAME: ptr nofree readnone [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; ATTRIBUTOR-NEXT:    [[B:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[N]]
 ; ATTRIBUTOR-NEXT:    ret ptr [[B]]
 ;
@@ -227,15 +227,15 @@ define ptr @test9(ptr %a, i64 %n) {
 declare void @llvm.assume(i1)
 ; FIXME: missing nonnull
 define ptr @test10(ptr %a, i64 %n) {
-; FNATTRS-LABEL: define ptr @test10
-; FNATTRS-SAME: (ptr readnone [[A:%.*]], i64 [[N:%.*]]) #[[ATTR3:[0-9]+]] {
+; FNATTRS-LABEL: define ptr @test10(
+; FNATTRS-SAME: ptr readnone [[A:%.*]], i64 [[N:%.*]]) #[[ATTR3:[0-9]+]] {
 ; FNATTRS-NEXT:    [[CMP:%.*]] = icmp ne i64 [[N]], 0
 ; FNATTRS-NEXT:    call void @llvm.assume(i1 [[CMP]])
 ; FNATTRS-NEXT:    [[B:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[N]]
 ; FNATTRS-NEXT:    ret ptr [[B]]
 ;
-; ATTRIBUTOR-LABEL: define ptr @test10
-; ATTRIBUTOR-SAME: (ptr nofree readnone [[A:%.*]], i64 [[N:%.*]]) #[[ATTR3:[0-9]+]] {
+; ATTRIBUTOR-LABEL: define ptr @test10(
+; ATTRIBUTOR-SAME: ptr nofree readnone [[A:%.*]], i64 [[N:%.*]]) #[[ATTR3:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:    [[CMP:%.*]] = icmp ne i64 [[N]], 0
 ; ATTRIBUTOR-NEXT:    call void @llvm.assume(i1 [[CMP]]) #[[ATTR14:[0-9]+]]
 ; ATTRIBUTOR-NEXT:    [[B:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[N]]
@@ -252,8 +252,8 @@ define ptr @test10(ptr %a, i64 %n) {
 ;   return p? p: nonnull();
 ; }
 define ptr @test11(ptr) local_unnamed_addr {
-; FNATTRS-LABEL: define nonnull ptr @test11
-; FNATTRS-SAME: (ptr readnone [[TMP0:%.*]]) local_unnamed_addr {
+; FNATTRS-LABEL: define nonnull ptr @test11(
+; FNATTRS-SAME: ptr readnone [[TMP0:%.*]]) local_unnamed_addr {
 ; FNATTRS-NEXT:    [[TMP2:%.*]] = icmp eq ptr [[TMP0]], null
 ; FNATTRS-NEXT:    br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP5:%.*]]
 ; FNATTRS:       3:
@@ -263,8 +263,8 @@ define ptr @test11(ptr) local_unnamed_addr {
 ; FNATTRS-NEXT:    [[TMP6:%.*]] = phi ptr [ [[TMP4]], [[TMP3]] ], [ [[TMP0]], [[TMP1:%.*]] ]
 ; FNATTRS-NEXT:    ret ptr [[TMP6]]
 ;
-; ATTRIBUTOR-LABEL: define nonnull ptr @test11
-; ATTRIBUTOR-SAME: (ptr [[TMP0:%.*]]) local_unnamed_addr {
+; ATTRIBUTOR-LABEL: define nonnull ptr @test11(
+; ATTRIBUTOR-SAME: ptr [[TMP0:%.*]]) local_unnamed_addr {
 ; ATTRIBUTOR-NEXT:    [[TMP2:%.*]] = icmp eq ptr [[TMP0]], null
 ; ATTRIBUTOR-NEXT:    br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP5:%.*]]
 ; ATTRIBUTOR:       3:
@@ -290,8 +290,8 @@ define ptr @test11(ptr) local_unnamed_addr {
 ; Simple CallSite Test
 declare void @test12_helper(ptr)
 define void @test12(ptr nonnull %a) {
-; COMMON-LABEL: define void @test12
-; COMMON-SAME: (ptr nonnull [[A:%.*]]) {
+; COMMON-LABEL: define void @test12(
+; COMMON-SAME: ptr nonnull [[A:%.*]]) {
 ; COMMON-NEXT:    tail call void @test12_helper(ptr [[A]])
 ; COMMON-NEXT:    ret void
 ;
@@ -324,12 +324,12 @@ define void @test13_helper() {
   ret void
 }
 define internal void @test13(ptr %a, ptr %b, ptr %c) {
-; FNATTRS-LABEL: define internal void @test13
-; FNATTRS-SAME: (ptr nocapture readnone [[A:%.*]], ptr nocapture readnone [[B:%.*]], ptr nocapture readnone [[C:%.*]]) #[[ATTR0]] {
+; FNATTRS-LABEL: define internal void @test13(
+; FNATTRS-SAME: ptr nocapture readnone [[A:%.*]], ptr nocapture readnone [[B:%.*]], ptr nocapture readnone [[C:%.*]]) #[[ATTR0]] {
 ; FNATTRS-NEXT:    ret void
 ;
-; ATTRIBUTOR-LABEL: define internal void @test13
-; ATTRIBUTOR-SAME: (ptr nocapture nofree readnone [[A:%.*]], ptr nocapture nofree readnone [[B:%.*]], ptr nocapture nofree readnone [[C:%.*]]) #[[ATTR4:[0-9]+]] {
+; ATTRIBUTOR-LABEL: define internal void @test13(
+; ATTRIBUTOR-SAME: ptr nocapture nofree readnone [[A:%.*]], ptr nocapture nofree readnone [[B:%.*]], ptr nocapture nofree readnone [[C:%.*]]) #[[ATTR4:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:    ret void
 ;
   ret void
@@ -351,8 +351,8 @@ declare nonnull ptr @nonnull()
 
 define internal ptr @f1(ptr %arg) {
 ; FIXME: missing nonnull It should be nonnull @f1(ptr nonnull readonly %arg)
-; FNATTRS-LABEL: define internal nonnull ptr @f1
-; FNATTRS-SAME: (ptr readonly [[ARG:%.*]]) #[[ATTR4:[0-9]+]] {
+; FNATTRS-LABEL: define internal nonnull ptr @f1(
+; FNATTRS-SAME: ptr readonly [[ARG:%.*]]) #[[ATTR4:[0-9]+]] {
 ; FNATTRS-NEXT:  bb:
 ; FNATTRS-NEXT:    [[TMP:%.*]] = icmp eq ptr [[ARG]], null
 ; FNATTRS-NEXT:    br i1 [[TMP]], label [[BB9:%.*]], label [[BB1:%.*]]
@@ -372,8 +372,8 @@ define internal ptr @f1(ptr %arg) {
 ; FNATTRS-NEXT:    [[TMP10:%.*]] = phi ptr [ [[TMP5C]], [[BB4]] ], [ inttoptr (i64 4 to ptr), [[BB:%.*]] ]
 ; FNATTRS-NEXT:    ret ptr [[TMP10]]
 ;
-; ATTRIBUTOR-LABEL: define internal ptr @f1
-; ATTRIBUTOR-SAME: (ptr nofree readonly [[ARG:%.*]]) #[[ATTR5:[0-9]+]] {
+; ATTRIBUTOR-LABEL: define internal ptr @f1(
+; ATTRIBUTOR-SAME: ptr nofree readonly [[ARG:%.*]]) #[[ATTR5:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:  bb:
 ; ATTRIBUTOR-NEXT:    [[TMP:%.*]] = icmp eq ptr [[ARG]], null
 ; ATTRIBUTOR-NEXT:    br i1 [[TMP]], label [[BB9:%.*]], label [[BB1:%.*]]
@@ -420,14 +420,14 @@ bb9:                                              ; preds = %bb4, %bb
 
 define internal ptr @f2(ptr %arg) {
 ; FIXME: missing nonnull. It should be nonnull @f2(ptr nonnull %arg)
-; FNATTRS-LABEL: define internal nonnull ptr @f2
-; FNATTRS-SAME: (ptr [[ARG:%.*]]) #[[ATTR4]] {
+; FNATTRS-LABEL: define internal nonnull ptr @f2(
+; FNATTRS-SAME: ptr [[ARG:%.*]]) #[[ATTR4]] {
 ; FNATTRS-NEXT:  bb:
 ; FNATTRS-NEXT:    [[TMP:%.*]] = tail call ptr @f1(ptr [[ARG]])
 ; FNATTRS-NEXT:    ret ptr [[TMP]]
 ;
-; ATTRIBUTOR-LABEL: define internal ptr @f2
-; ATTRIBUTOR-SAME: (ptr readonly [[ARG:%.*]]) #[[ATTR5]] {
+; ATTRIBUTOR-LABEL: define internal ptr @f2(
+; ATTRIBUTOR-SAME: ptr readonly [[ARG:%.*]]) #[[ATTR5]] {
 ; ATTRIBUTOR-NEXT:  bb:
 ; ATTRIBUTOR-NEXT:    [[TMP:%.*]] = tail call ptr @f1(ptr readonly [[ARG]]) #[[ATTR15]]
 ; ATTRIBUTOR-NEXT:    ret ptr [[TMP]]
@@ -441,14 +441,14 @@ bb:
 
 define dso_local noalias ptr @f3(ptr %arg) {
 ; FIXME: missing nonnull. It should be nonnull @f3(ptr nonnull readonly %arg)
-; FNATTRS-LABEL: define dso_local noalias nonnull ptr @f3
-; FNATTRS-SAME: (ptr [[ARG:%.*]]) #[[ATTR4]] {
+; FNATTRS-LABEL: define dso_local noalias nonnull ptr @f3(
+; FNATTRS-SAME: ptr [[ARG:%.*]]) #[[ATTR4]] {
 ; FNATTRS-NEXT:  bb:
 ; FNATTRS-NEXT:    [[TMP:%.*]] = call ptr @f1(ptr [[ARG]])
 ; FNATTRS-NEXT:    ret ptr [[TMP]]
 ;
-; ATTRIBUTOR-LABEL: define dso_local noalias ptr @f3
-; ATTRIBUTOR-SAME: (ptr nofree readonly [[ARG:%.*]]) #[[ATTR5]] {
+; ATTRIBUTOR-LABEL: define dso_local noalias ptr @f3(
+; ATTRIBUTOR-SAME: ptr nofree readonly [[ARG:%.*]]) #[[ATTR5]] {
 ; ATTRIBUTOR-NEXT:  bb:
 ; ATTRIBUTOR-NEXT:    [[TMP:%.*]] = call ptr @f1(ptr nofree readonly [[ARG]]) #[[ATTR15]]
 ; ATTRIBUTOR-NEXT:    ret ptr [[TMP]]
@@ -461,13 +461,13 @@ bb:
 
 ; TEST 15
 define void @f15(ptr %arg) {
-; FNATTRS-LABEL: define void @f15
-; FNATTRS-SAME: (ptr [[ARG:%.*]]) {
+; FNATTRS-LABEL: define void @f15(
+; FNATTRS-SAME: ptr [[ARG:%.*]]) {
 ; FNATTRS-NEXT:    tail call void @use1(ptr dereferenceable(4) [[ARG]])
 ; FNATTRS-NEXT:    ret void
 ;
-; ATTRIBUTOR-LABEL: define void @f15
-; ATTRIBUTOR-SAME: (ptr nonnull [[ARG:%.*]]) {
+; ATTRIBUTOR-LABEL: define void @f15(
+; ATTRIBUTOR-SAME: ptr nonnull [[ARG:%.*]]) {
 ; ATTRIBUTOR-NEXT:    tail call void @use1(ptr nonnull dereferenceable(4) [[ARG]])
 ; ATTRIBUTOR-NEXT:    ret void
 ;
@@ -487,8 +487,8 @@ declare void @fun3(ptr, ptr, ptr) #1
 ; We can say that %a is nonnull but %b is not.
 define void @f16(ptr %a, ptr %b, i8 %c) {
 ; FIXME: missing nonnull on %a
-; FNATTRS-LABEL: define void @f16
-; FNATTRS-SAME: (ptr [[A:%.*]], ptr [[B:%.*]], i8 [[C:%.*]]) #[[ATTR6:[0-9]+]] {
+; FNATTRS-LABEL: define void @f16(
+; FNATTRS-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i8 [[C:%.*]]) #[[ATTR6:[0-9]+]] {
 ; FNATTRS-NEXT:    [[CMP:%.*]] = icmp eq i8 [[C]], 0
 ; FNATTRS-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
 ; FNATTRS:       if.then:
@@ -498,8 +498,8 @@ define void @f16(ptr %a, ptr %b, i8 %c) {
 ; FNATTRS-NEXT:    tail call void @fun2(ptr nonnull [[A]], ptr [[B]])
 ; FNATTRS-NEXT:    ret void
 ;
-; ATTRIBUTOR-LABEL: define void @f16
-; ATTRIBUTOR-SAME: (ptr nonnull [[A:%.*]], ptr [[B:%.*]], i8 [[C:%.*]]) #[[ATTR7:[0-9]+]] {
+; ATTRIBUTOR-LABEL: define void @f16(
+; ATTRIBUTOR-SAME: ptr nonnull [[A:%.*]], ptr [[B:%.*]], i8 [[C:%.*]]) #[[ATTR7:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:    [[CMP:%.*]] = icmp eq i8 [[C]], 0
 ; ATTRIBUTOR-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
 ; ATTRIBUTOR:       if.then:
@@ -526,8 +526,8 @@ if.else:
 ; fun1(nonnull %a)
 ; We can say that %a is nonnull
 define void @f17(ptr %a, i8 %c) {
-; FNATTRS-LABEL: define void @f17
-; FNATTRS-SAME: (ptr [[A:%.*]], i8 [[C:%.*]]) #[[ATTR6]] {
+; FNATTRS-LABEL: define void @f17(
+; FNATTRS-SAME: ptr [[A:%.*]], i8 [[C:%.*]]) #[[ATTR6]] {
 ; FNATTRS-NEXT:    [[CMP:%.*]] = icmp eq i8 [[C]], 0
 ; FNATTRS-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
 ; FNATTRS:       if.then:
@@ -540,8 +540,8 @@ define void @f17(ptr %a, i8 %c) {
 ; FNATTRS-NEXT:    tail call void @fun1(ptr nonnull [[A]])
 ; FNATTRS-NEXT:    ret void
 ;
-; ATTRIBUTOR-LABEL: define void @f17
-; ATTRIBUTOR-SAME: (ptr nonnull [[A:%.*]], i8 [[C:%.*]]) #[[ATTR7]] {
+; ATTRIBUTOR-LABEL: define void @f17(
+; ATTRIBUTOR-SAME: ptr nonnull [[A:%.*]], i8 [[C:%.*]]) #[[ATTR7]] {
 ; ATTRIBUTOR-NEXT:    [[CMP:%.*]] = icmp eq i8 [[C]], 0
 ; ATTRIBUTOR-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
 ; ATTRIBUTOR:       if.then:
@@ -578,8 +578,8 @@ cont:
 ; fun1(nonnull %a)
 
 define void @f18(ptr %a, ptr %b, i8 %c) {
-; FNATTRS-LABEL: define void @f18
-; FNATTRS-SAME: (ptr [[A:%.*]], ptr [[B:%.*]], i8 [[C:%.*]]) #[[ATTR6]] {
+; FNATTRS-LABEL: define void @f18(
+; FNATTRS-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i8 [[C:%.*]]) #[[ATTR6]] {
 ; FNATTRS-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[C]], 0
 ; FNATTRS-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
 ; FNATTRS:       if.then:
@@ -601,8 +601,8 @@ define void @f18(ptr %a, ptr %b, i8 %c) {
 ; FNATTRS-NEXT:    tail call void @fun1(ptr nonnull [[A]])
 ; FNATTRS-NEXT:    ret void
 ;
-; ATTRIBUTOR-LABEL: define void @f18
-; ATTRIBUTOR-SAME: (ptr nonnull [[A:%.*]], ptr [[B:%.*]], i8 [[C:%.*]]) #[[ATTR7]] {
+; ATTRIBUTOR-LABEL: define void @f18(
+; ATTRIBUTOR-SAME: ptr nonnull [[A:%.*]], ptr [[B:%.*]], i8 [[C:%.*]]) #[[ATTR7]] {
 ; ATTRIBUTOR-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[C]], 0
 ; ATTRIBUTOR-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
 ; ATTRIBUTOR:       if.then:
@@ -650,8 +650,8 @@ cont2:
 
 define void @f19(ptr %a, ptr %b, i8 %c) {
 ; FIXME: missing nonnull on %b
-; FNATTRS-LABEL: define void @f19
-; FNATTRS-SAME: (ptr [[A:%.*]], ptr [[B:%.*]], i8 [[C:%.*]]) #[[ATTR7:[0-9]+]] {
+; FNATTRS-LABEL: define void @f19(
+; FNATTRS-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i8 [[C:%.*]]) #[[ATTR7:[0-9]+]] {
 ; FNATTRS-NEXT:    br label [[LOOP_HEADER:%.*]]
 ; FNATTRS:       loop.header:
 ; FNATTRS-NEXT:    [[CMP2:%.*]] = icmp eq i8 [[C]], 0
@@ -664,8 +664,8 @@ define void @f19(ptr %a, ptr %b, i8 %c) {
 ; FNATTRS-NEXT:    tail call void @fun1(ptr nonnull [[B]])
 ; FNATTRS-NEXT:    ret void
 ;
-; ATTRIBUTOR-LABEL: define void @f19
-; ATTRIBUTOR-SAME: (ptr [[A:%.*]], ptr nonnull [[B:%.*]], i8 [[C:%.*]]) #[[ATTR8:[0-9]+]] {
+; ATTRIBUTOR-LABEL: define void @f19(
+; ATTRIBUTOR-SAME: ptr [[A:%.*]], ptr nonnull [[B:%.*]], i8 [[C:%.*]]) #[[ATTR8:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:    br label [[LOOP_HEADER:%.*]]
 ; ATTRIBUTOR:       loop.header:
 ; ATTRIBUTOR-NEXT:    [[CMP2:%.*]] = icmp eq i8 [[C]], 0
@@ -708,13 +708,13 @@ declare i8 @use1safecall(ptr %x) nounwind willreturn ; nounwind+willreturn guara
 
 define void @parent_poison(ptr %a) {
 ; FNATTR-LABEL: @parent_poison(ptr %a)
-; FNATTRS-LABEL: define void @parent_poison
-; FNATTRS-SAME: (ptr [[A:%.*]]) {
+; FNATTRS-LABEL: define void @parent_poison(
+; FNATTRS-SAME: ptr [[A:%.*]]) {
 ; FNATTRS-NEXT:    call void @use1nonnull_without_noundef(ptr [[A]])
 ; FNATTRS-NEXT:    ret void
 ;
-; ATTRIBUTOR-LABEL: define void @parent_poison
-; ATTRIBUTOR-SAME: (ptr nonnull [[A:%.*]]) {
+; ATTRIBUTOR-LABEL: define void @parent_poison(
+; ATTRIBUTOR-SAME: ptr nonnull [[A:%.*]]) {
 ; ATTRIBUTOR-NEXT:    call void @use1nonnull_without_noundef(ptr nonnull [[A]])
 ; ATTRIBUTOR-NEXT:    ret void
 ;
@@ -725,8 +725,8 @@ define void @parent_poison(ptr %a) {
 ; Can't extend non-null to parent for any argument because the 2nd call is not guaranteed to execute.
 
 define void @parent1(ptr %a, ptr %b, ptr %c) {
-; COMMON-LABEL: define void @parent1
-; COMMON-SAME: (ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) {
+; COMMON-LABEL: define void @parent1(
+; COMMON-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) {
 ; COMMON-NEXT:    call void @use3(ptr [[C]], ptr [[A]], ptr [[B]])
 ; COMMON-NEXT:    call void @use3nonnull(ptr [[B]], ptr [[C]], ptr [[A]])
 ; COMMON-NEXT:    ret void
@@ -739,14 +739,14 @@ define void @parent1(ptr %a, ptr %b, ptr %c) {
 ; Extend non-null to parent for all arguments.
 
 define void @parent2(ptr %a, ptr %b, ptr %c) {
-; FNATTRS-LABEL: define void @parent2
-; FNATTRS-SAME: (ptr nonnull [[A:%.*]], ptr nonnull [[B:%.*]], ptr nonnull [[C:%.*]]) {
+; FNATTRS-LABEL: define void @parent2(
+; FNATTRS-SAME: ptr nonnull [[A:%.*]], ptr nonnull [[B:%.*]], ptr nonnull [[C:%.*]]) {
 ; FNATTRS-NEXT:    call void @use3nonnull(ptr [[B]], ptr [[C]], ptr [[A]])
 ; FNATTRS-NEXT:    call void @use3(ptr [[C]], ptr [[A]], ptr [[B]])
 ; FNATTRS-NEXT:    ret void
 ;
-; ATTRIBUTOR-LABEL: define void @parent2
-; ATTRIBUTOR-SAME: (ptr nonnull [[A:%.*]], ptr nonnull [[B:%.*]], ptr nonnull [[C:%.*]]) {
+; ATTRIBUTOR-LABEL: define void @parent2(
+; ATTRIBUTOR-SAME: ptr nonnull [[A:%.*]], ptr nonnull [[B:%.*]], ptr nonnull [[C:%.*]]) {
 ; ATTRIBUTOR-NEXT:    call void @use3nonnull(ptr nonnull [[B]], ptr nonnull [[C]], ptr nonnull [[A]])
 ; ATTRIBUTOR-NEXT:    call void @use3(ptr [[C]], ptr [[A]], ptr [[B]])
 ; ATTRIBUTOR-NEXT:    ret void
@@ -761,14 +761,14 @@ define void @parent2(ptr %a, ptr %b, ptr %c) {
 ; Extend non-null to parent for 1st argument.
 
 define void @parent3(ptr %a, ptr %b, ptr %c) {
-; FNATTRS-LABEL: define void @parent3
-; FNATTRS-SAME: (ptr nonnull [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) {
+; FNATTRS-LABEL: define void @parent3(
+; FNATTRS-SAME: ptr nonnull [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) {
 ; FNATTRS-NEXT:    call void @use1nonnull(ptr [[A]])
 ; FNATTRS-NEXT:    call void @use3(ptr [[C]], ptr [[B]], ptr [[A]])
 ; FNATTRS-NEXT:    ret void
 ;
-; ATTRIBUTOR-LABEL: define void @parent3
-; ATTRIBUTOR-SAME: (ptr nonnull [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) {
+; ATTRIBUTOR-LABEL: define void @parent3(
+; ATTRIBUTOR-SAME: ptr nonnull [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) {
 ; ATTRIBUTOR-NEXT:    call void @use1nonnull(ptr nonnull [[A]])
 ; ATTRIBUTOR-NEXT:    call void @use3(ptr [[C]], ptr [[B]], ptr [[A]])
 ; ATTRIBUTOR-NEXT:    ret void
@@ -788,15 +788,15 @@ define void @parent4(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    call void @use2nonnull(ptr %c, ptr %b)
 ; CHECK-NEXT:    call void @use2(ptr %a, ptr %c)
 ; CHECK-NEXT:    call void @use1(ptr %b)
-; FNATTRS-LABEL: define void @parent4
-; FNATTRS-SAME: (ptr [[A:%.*]], ptr nonnull [[B:%.*]], ptr nonnull [[C:%.*]]) {
+; FNATTRS-LABEL: define void @parent4(
+; FNATTRS-SAME: ptr [[A:%.*]], ptr nonnull [[B:%.*]], ptr nonnull [[C:%.*]]) {
 ; FNATTRS-NEXT:    call void @use2nonnull(ptr [[C]], ptr [[B]])
 ; FNATTRS-NEXT:    call void @use2(ptr [[A]], ptr [[C]])
 ; FNATTRS-NEXT:    call void @use1(ptr [[B]])
 ; FNATTRS-NEXT:    ret void
 ;
-; ATTRIBUTOR-LABEL: define void @parent4
-; ATTRIBUTOR-SAME: (ptr [[A:%.*]], ptr nonnull [[B:%.*]], ptr nonnull [[C:%.*]]) {
+; ATTRIBUTOR-LABEL: define void @parent4(
+; ATTRIBUTOR-SAME: ptr [[A:%.*]], ptr nonnull [[B:%.*]], ptr nonnull [[C:%.*]]) {
 ; ATTRIBUTOR-NEXT:    call void @use2nonnull(ptr nonnull [[C]], ptr nonnull [[B]])
 ; ATTRIBUTOR-NEXT:    call void @use2(ptr [[A]], ptr [[C]])
 ; ATTRIBUTOR-NEXT:    call void @use1(ptr [[B]])
@@ -814,8 +814,8 @@ define void @parent4(ptr %a, ptr %b, ptr %c) {
 ; because it would incorrectly propagate the wrong information to its callers.
 
 define void @parent5(ptr %a, i1 %a_is_notnull) {
-; FNATTRS-LABEL: define void @parent5
-; FNATTRS-SAME: (ptr [[A:%.*]], i1 [[A_IS_NOTNULL:%.*]]) {
+; FNATTRS-LABEL: define void @parent5(
+; FNATTRS-SAME: ptr [[A:%.*]], i1 [[A_IS_NOTNULL:%.*]]) {
 ; FNATTRS-NEXT:    br i1 [[A_IS_NOTNULL]], label [[T:%.*]], label [[F:%.*]]
 ; FNATTRS:       t:
 ; FNATTRS-NEXT:    call void @use1nonnull(ptr [[A]])
@@ -823,8 +823,8 @@ define void @parent5(ptr %a, i1 %a_is_notnull) {
 ; FNATTRS:       f:
 ; FNATTRS-NEXT:    ret void
 ;
-; ATTRIBUTOR-LABEL: define void @parent5
-; ATTRIBUTOR-SAME: (ptr [[A:%.*]], i1 [[A_IS_NOTNULL:%.*]]) {
+; ATTRIBUTOR-LABEL: define void @parent5(
+; ATTRIBUTOR-SAME: ptr [[A:%.*]], i1 [[A_IS_NOTNULL:%.*]]) {
 ; ATTRIBUTOR-NEXT:    br i1 [[A_IS_NOTNULL]], label [[T:%.*]], label [[F:%.*]]
 ; ATTRIBUTOR:       t:
 ; ATTRIBUTOR-NEXT:    call void @use1nonnull(ptr nonnull [[A]])
@@ -845,14 +845,14 @@ f:
 ; The volatile load can't trap, so we can guarantee that we'll get to the call.
 
 define i8 @parent6(ptr %a, ptr %b) {
-; FNATTRS-LABEL: define i8 @parent6
-; FNATTRS-SAME: (ptr nonnull [[A:%.*]], ptr [[B:%.*]]) {
+; FNATTRS-LABEL: define i8 @parent6(
+; FNATTRS-SAME: ptr nonnull [[A:%.*]], ptr [[B:%.*]]) {
 ; FNATTRS-NEXT:    [[C:%.*]] = load volatile i8, ptr [[B]], align 1
 ; FNATTRS-NEXT:    call void @use1nonnull(ptr [[A]])
 ; FNATTRS-NEXT:    ret i8 [[C]]
 ;
-; ATTRIBUTOR-LABEL: define i8 @parent6
-; ATTRIBUTOR-SAME: (ptr nonnull [[A:%.*]], ptr nofree [[B:%.*]]) {
+; ATTRIBUTOR-LABEL: define i8 @parent6(
+; ATTRIBUTOR-SAME: ptr nonnull [[A:%.*]], ptr nofree [[B:%.*]]) {
 ; ATTRIBUTOR-NEXT:    [[C:%.*]] = load volatile i8, ptr [[B]], align 1
 ; ATTRIBUTOR-NEXT:    call void @use1nonnull(ptr nonnull [[A]])
 ; ATTRIBUTOR-NEXT:    ret i8 [[C]]
@@ -866,14 +866,14 @@ define i8 @parent6(ptr %a, ptr %b) {
 ; The nonnull callsite is guaranteed to execute, so the argument must be nonnull throughout the parent.
 
 define i8 @parent7(ptr %a) {
-; FNATTRS-LABEL: define i8 @parent7
-; FNATTRS-SAME: (ptr nonnull [[A:%.*]]) {
+; FNATTRS-LABEL: define i8 @parent7(
+; FNATTRS-SAME: ptr nonnull [[A:%.*]]) {
 ; FNATTRS-NEXT:    [[RET:%.*]] = call i8 @use1safecall(ptr [[A]])
 ; FNATTRS-NEXT:    call void @use1nonnull(ptr [[A]])
 ; FNATTRS-NEXT:    ret i8 [[RET]]
 ;
-; ATTRIBUTOR-LABEL: define i8 @parent7
-; ATTRIBUTOR-SAME: (ptr nonnull [[A:%.*]]) {
+; ATTRIBUTOR-LABEL: define i8 @parent7(
+; ATTRIBUTOR-SAME: ptr nonnull [[A:%.*]]) {
 ; ATTRIBUTOR-NEXT:    [[RET:%.*]] = call i8 @use1safecall(ptr nonnull [[A]]) #[[ATTR16]]
 ; ATTRIBUTOR-NEXT:    call void @use1nonnull(ptr nonnull [[A]])
 ; ATTRIBUTOR-NEXT:    ret i8 [[RET]]
@@ -892,8 +892,8 @@ define i8 @parent7(ptr %a) {
 declare i32 @esfp(...)
 
 define i1 @parent8(ptr %a, ptr %bogus1, ptr %b) personality ptr @esfp{
-; FNATTRS-LABEL: define i1 @parent8
-; FNATTRS-SAME: (ptr nonnull [[A:%.*]], ptr nocapture readnone [[BOGUS1:%.*]], ptr nonnull [[B:%.*]]) #[[ATTR7]] personality ptr @esfp {
+; FNATTRS-LABEL: define i1 @parent8(
+; FNATTRS-SAME: ptr nonnull [[A:%.*]], ptr nocapture readnone [[BOGUS1:%.*]], ptr nonnull [[B:%.*]]) #[[ATTR7]] personality ptr @esfp {
 ; FNATTRS-NEXT:  entry:
 ; FNATTRS-NEXT:    invoke void @use2nonnull(ptr [[A]], ptr [[B]])
 ; FNATTRS-NEXT:    to label [[CONT:%.*]] unwind label [[EXC:%.*]]
@@ -905,8 +905,8 @@ define i1 @parent8(ptr %a, ptr %bogus1, ptr %b) personality ptr @esfp{
 ; FNATTRS-NEXT:    filter [0 x ptr] zeroinitializer
 ; FNATTRS-NEXT:    unreachable
 ;
-; ATTRIBUTOR-LABEL: define i1 @parent8
-; ATTRIBUTOR-SAME: (ptr nonnull [[A:%.*]], ptr nocapture nofree readnone [[BOGUS1:%.*]], ptr nonnull [[B:%.*]]) #[[ATTR8]] personality ptr @esfp {
+; ATTRIBUTOR-LABEL: define i1 @parent8(
+; ATTRIBUTOR-SAME: ptr nonnull [[A:%.*]], ptr nocapture nofree readnone [[BOGUS1:%.*]], ptr nonnull [[B:%.*]]) #[[ATTR8]] personality ptr @esfp {
 ; ATTRIBUTOR-NEXT:  entry:
 ; ATTRIBUTOR-NEXT:    invoke void @use2nonnull(ptr nonnull [[A]], ptr nonnull [[B]])
 ; ATTRIBUTOR-NEXT:    to label [[CONT:%.*]] unwind label [[EXC:%.*]]
@@ -934,13 +934,13 @@ exc:
 }
 
 define ptr @gep1(ptr %p) {
-; FNATTRS-LABEL: define nonnull ptr @gep1
-; FNATTRS-SAME: (ptr readnone [[P:%.*]]) #[[ATTR0]] {
+; FNATTRS-LABEL: define nonnull ptr @gep1(
+; FNATTRS-SAME: ptr readnone [[P:%.*]]) #[[ATTR0]] {
 ; FNATTRS-NEXT:    [[Q:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 1
 ; FNATTRS-NEXT:    ret ptr [[Q]]
 ;
-; ATTRIBUTOR-LABEL: define nonnull ptr @gep1
-; ATTRIBUTOR-SAME: (ptr nofree readnone [[P:%.*]]) #[[ATTR0]] {
+; ATTRIBUTOR-LABEL: define nonnull ptr @gep1(
+; ATTRIBUTOR-SAME: ptr nofree readnone [[P:%.*]]) #[[ATTR0]] {
 ; ATTRIBUTOR-NEXT:    [[Q:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 1
 ; ATTRIBUTOR-NEXT:    ret ptr [[Q]]
 ;
@@ -950,13 +950,13 @@ define ptr @gep1(ptr %p) {
 
 define ptr @gep1_no_null_opt(ptr %p) #0 {
 ; Should't be able to derive nonnull based on gep.
-; FNATTRS-LABEL: define ptr @gep1_no_null_opt
-; FNATTRS-SAME: (ptr readnone [[P:%.*]]) #[[ATTR8:[0-9]+]] {
+; FNATTRS-LABEL: define ptr @gep1_no_null_opt(
+; FNATTRS-SAME: ptr readnone [[P:%.*]]) #[[ATTR8:[0-9]+]] {
 ; FNATTRS-NEXT:    [[Q:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 1
 ; FNATTRS-NEXT:    ret ptr [[Q]]
 ;
-; ATTRIBUTOR-LABEL: define ptr @gep1_no_null_opt
-; ATTRIBUTOR-SAME: (ptr nofree readnone [[P:%.*]]) #[[ATTR9:[0-9]+]] {
+; ATTRIBUTOR-LABEL: define ptr @gep1_no_null_opt(
+; ATTRIBUTOR-SAME: ptr nofree readnone [[P:%.*]]) #[[ATTR9:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:    [[Q:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 1
 ; ATTRIBUTOR-NEXT:    ret ptr [[Q]]
 ;
@@ -965,13 +965,13 @@ define ptr @gep1_no_null_opt(ptr %p) #0 {
 }
 
 define ptr addrspace(3) @gep2(ptr addrspace(3) %p) {
-; FNATTRS-LABEL: define ptr addrspace(3) @gep2
-; FNATTRS-SAME: (ptr addrspace(3) readnone [[P:%.*]]) #[[ATTR0]] {
+; FNATTRS-LABEL: define ptr addrspace(3) @gep2(
+; FNATTRS-SAME: ptr addrspace(3) readnone [[P:%.*]]) #[[ATTR0]] {
 ; FNATTRS-NEXT:    [[Q:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[P]], i32 1
 ; FNATTRS-NEXT:    ret ptr addrspace(3) [[Q]]
 ;
-; ATTRIBUTOR-LABEL: define ptr addrspace(3) @gep2
-; ATTRIBUTOR-SAME: (ptr addrspace(3) nofree readnone [[P:%.*]]) #[[ATTR0]] {
+; ATTRIBUTOR-LABEL: define ptr addrspace(3) @gep2(
+; ATTRIBUTOR-SAME: ptr addrspace(3) nofree readnone [[P:%.*]]) #[[ATTR0]] {
 ; ATTRIBUTOR-NEXT:    [[Q:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[P]], i32 1
 ; ATTRIBUTOR-NEXT:    ret ptr addrspace(3) [[Q]]
 ;
@@ -981,37 +981,37 @@ define ptr addrspace(3) @gep2(ptr addrspace(3) %p) {
 
 ; FIXME: We should propagate dereferenceable here but *not* nonnull
 define ptr addrspace(3) @as(ptr addrspace(3) dereferenceable(4) %p) {
-; FNATTRS-LABEL: define ptr addrspace(3) @as
-; FNATTRS-SAME: (ptr addrspace(3) readnone returned dereferenceable(4) [[P:%.*]]) #[[ATTR0]] {
+; FNATTRS-LABEL: define ptr addrspace(3) @as(
+; FNATTRS-SAME: ptr addrspace(3) readnone returned dereferenceable(4) [[P:%.*]]) #[[ATTR0]] {
 ; FNATTRS-NEXT:    ret ptr addrspace(3) [[P]]
 ;
-; ATTRIBUTOR-LABEL: define ptr addrspace(3) @as
-; ATTRIBUTOR-SAME: (ptr addrspace(3) nofree readnone dereferenceable(4) [[P:%.*]]) #[[ATTR0]] {
+; ATTRIBUTOR-LABEL: define ptr addrspace(3) @as(
+; ATTRIBUTOR-SAME: ptr addrspace(3) nofree readnone dereferenceable(4) [[P:%.*]]) #[[ATTR0]] {
 ; ATTRIBUTOR-NEXT:    ret ptr addrspace(3) [[P]]
 ;
   ret ptr addrspace(3) %p
 }
 
 define internal ptr @g2() {
-; FNATTRS-LABEL: define internal nonnull ptr @g2
-; FNATTRS-SAME: () #[[ATTR0]] {
+; FNATTRS-LABEL: define internal nonnull ptr @g2(
+; FNATTRS-SAME: ) #[[ATTR0]] {
 ; FNATTRS-NEXT:    ret ptr inttoptr (i64 4 to ptr)
 ;
-; ATTRIBUTOR-LABEL: define internal ptr @g2
-; ATTRIBUTOR-SAME: () #[[ATTR10:[0-9]+]] {
+; ATTRIBUTOR-LABEL: define internal ptr @g2(
+; ATTRIBUTOR-SAME: ) #[[ATTR10:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:    ret ptr inttoptr (i64 4 to ptr)
 ;
   ret ptr inttoptr (i64 4 to ptr)
 }
 
 define  ptr @g1() {
-; FNATTRS-LABEL: define nonnull ptr @g1
-; FNATTRS-SAME: () #[[ATTR0]] {
+; FNATTRS-LABEL: define nonnull ptr @g1(
+; FNATTRS-SAME: ) #[[ATTR0]] {
 ; FNATTRS-NEXT:    [[C:%.*]] = call ptr @g2()
 ; FNATTRS-NEXT:    ret ptr [[C]]
 ;
-; ATTRIBUTOR-LABEL: define ptr @g1
-; ATTRIBUTOR-SAME: () #[[ATTR0]] {
+; ATTRIBUTOR-LABEL: define ptr @g1(
+; ATTRIBUTOR-SAME: ) #[[ATTR0]] {
 ; ATTRIBUTOR-NEXT:    [[C:%.*]] = call ptr @g2() #[[ATTR10]]
 ; ATTRIBUTOR-NEXT:    ret ptr [[C]]
 ;
@@ -1021,13 +1021,13 @@ define  ptr @g1() {
 
 declare void @use_i32_ptr(ptr) readnone nounwind
 define internal void @called_by_weak(ptr %a) {
-; FNATTRS-LABEL: define internal void @called_by_weak
-; FNATTRS-SAME: (ptr nocapture readnone [[A:%.*]]) #[[ATTR1]] {
+; FNATTRS-LABEL: define internal void @called_by_weak(
+; FNATTRS-SAME: ptr nocapture readnone [[A:%.*]]) #[[ATTR1]] {
 ; FNATTRS-NEXT:    call void @use_i32_ptr(ptr [[A]])
 ; FNATTRS-NEXT:    ret void
 ;
-; ATTRIBUTOR-LABEL: define internal void @called_by_weak
-; ATTRIBUTOR-SAME: (ptr nocapture readnone [[A:%.*]]) #[[ATTR11:[0-9]+]] {
+; ATTRIBUTOR-LABEL: define internal void @called_by_weak(
+; ATTRIBUTOR-SAME: ptr nocapture readnone [[A:%.*]]) #[[ATTR11:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:    call void @use_i32_ptr(ptr [[A]])
 ; ATTRIBUTOR-NEXT:    ret void
 ;
@@ -1037,13 +1037,13 @@ define internal void @called_by_weak(ptr %a) {
 
 ; Check we do not annotate the function interface of this weak function.
 define weak_odr void @weak_caller(ptr nonnull %a) {
-; FNATTRS-LABEL: define weak_odr void @weak_caller
-; FNATTRS-SAME: (ptr nonnull [[A:%.*]]) {
+; FNATTRS-LABEL: define weak_odr void @weak_caller(
+; FNATTRS-SAME: ptr nonnull [[A:%.*]]) {
 ; FNATTRS-NEXT:    call void @called_by_weak(ptr [[A]])
 ; FNATTRS-NEXT:    ret void
 ;
-; ATTRIBUTOR-LABEL: define weak_odr void @weak_caller
-; ATTRIBUTOR-SAME: (ptr nonnull [[A:%.*]]) {
+; ATTRIBUTOR-LABEL: define weak_odr void @weak_caller(
+; ATTRIBUTOR-SAME: ptr nonnull [[A:%.*]]) {
 ; ATTRIBUTOR-NEXT:    call void @called_by_weak(ptr nocapture nonnull readnone [[A]])
 ; ATTRIBUTOR-NEXT:    ret void
 ;
@@ -1053,13 +1053,13 @@ define weak_odr void @weak_caller(ptr nonnull %a) {
 
 ; Expect nonnull
 define internal void @control(ptr dereferenceable(4) %a) {
-; FNATTRS-LABEL: define internal void @control
-; FNATTRS-SAME: (ptr nocapture readnone dereferenceable(4) [[A:%.*]]) #[[ATTR1]] {
+; FNATTRS-LABEL: define internal void @control(
+; FNATTRS-SAME: ptr nocapture readnone dereferenceable(4) [[A:%.*]]) #[[ATTR1]] {
 ; FNATTRS-NEXT:    call void @use_i32_ptr(ptr [[A]])
 ; FNATTRS-NEXT:    ret void
 ;
-; ATTRIBUTOR-LABEL: define internal void @control
-; ATTRIBUTOR-SAME: (ptr nocapture readnone dereferenceable(4) [[A:%.*]]) #[[ATTR11]] {
+; ATTRIBUTOR-LABEL: define internal void @control(
+; ATTRIBUTOR-SAME: ptr nocapture readnone dereferenceable(4) [[A:%.*]]) #[[ATTR11]] {
 ; ATTRIBUTOR-NEXT:    call void @use_i32_ptr(ptr [[A]])
 ; ATTRIBUTOR-NEXT:    ret void
 ;
@@ -1068,13 +1068,13 @@ define internal void @control(ptr dereferenceable(4) %a) {
 }
 ; Avoid nonnull as we do not touch naked functions
 define internal void @naked(ptr dereferenceable(4) %a) naked {
-; FNATTRS-LABEL: define internal void @naked
-; FNATTRS-SAME: (ptr dereferenceable(4) [[A:%.*]]) #[[ATTR10:[0-9]+]] {
+; FNATTRS-LABEL: define internal void @naked(
+; FNATTRS-SAME: ptr dereferenceable(4) [[A:%.*]]) #[[ATTR10:[0-9]+]] {
 ; FNATTRS-NEXT:    call void @use_i32_ptr(ptr [[A]])
 ; FNATTRS-NEXT:    ret void
 ;
-; ATTRIBUTOR-LABEL: define internal void @naked
-; ATTRIBUTOR-SAME: (ptr dereferenceable(4) [[A:%.*]]) #[[ATTR12:[0-9]+]] {
+; ATTRIBUTOR-LABEL: define internal void @naked(
+; ATTRIBUTOR-SAME: ptr dereferenceable(4) [[A:%.*]]) #[[ATTR12:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:    call void @use_i32_ptr(ptr [[A]])
 ; ATTRIBUTOR-NEXT:    ret void
 ;
@@ -1083,13 +1083,13 @@ define internal void @naked(ptr dereferenceable(4) %a) naked {
 }
 ; Avoid nonnull as we do not touch optnone
 define internal void @optnone(ptr dereferenceable(4) %a) optnone noinline {
-; FNATTRS-LABEL: define internal void @optnone
-; FNATTRS-SAME: (ptr dereferenceable(4) [[A:%.*]]) #[[ATTR11:[0-9]+]] {
+; FNATTRS-LABEL: define internal void @optnone(
+; FNATTRS-SAME: ptr dereferenceable(4) [[A:%.*]]) #[[ATTR11:[0-9]+]] {
 ; FNATTRS-NEXT:    call void @use_i32_ptr(ptr [[A]])
 ; FNATTRS-NEXT:    ret void
 ;
-; ATTRIBUTOR-LABEL: define internal void @optnone
-; ATTRIBUTOR-SAME: (ptr dereferenceable(4) [[A:%.*]]) #[[ATTR13:[0-9]+]] {
+; ATTRIBUTOR-LABEL: define internal void @optnone(
+; ATTRIBUTOR-SAME: ptr dereferenceable(4) [[A:%.*]]) #[[ATTR13:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:    call void @use_i32_ptr(ptr [[A]])
 ; ATTRIBUTOR-NEXT:    ret void
 ;
@@ -1097,15 +1097,15 @@ define internal void @optnone(ptr dereferenceable(4) %a) optnone noinline {
   ret void
 }
 define void @make_live(ptr nonnull dereferenceable(8) %a) {
-; FNATTRS-LABEL: define void @make_live
-; FNATTRS-SAME: (ptr nonnull dereferenceable(8) [[A:%.*]]) {
+; FNATTRS-LABEL: define void @make_live(
+; FNATTRS-SAME: ptr nonnull dereferenceable(8) [[A:%.*]]) {
 ; FNATTRS-NEXT:    call void @naked(ptr nonnull align 16 dereferenceable(8) [[A]])
 ; FNATTRS-NEXT:    call void @control(ptr nonnull align 16 dereferenceable(8) [[A]])
 ; FNATTRS-NEXT:    call void @optnone(ptr nonnull align 16 dereferenceable(8) [[A]])
 ; FNATTRS-NEXT:    ret void
 ;
-; ATTRIBUTOR-LABEL: define void @make_live
-; ATTRIBUTOR-SAME: (ptr nonnull dereferenceable(8) [[A:%.*]]) {
+; ATTRIBUTOR-LABEL: define void @make_live(
+; ATTRIBUTOR-SAME: ptr nonnull dereferenceable(8) [[A:%.*]]) {
 ; ATTRIBUTOR-NEXT:    call void @naked(ptr nonnull align 16 dereferenceable(8) [[A]])
 ; ATTRIBUTOR-NEXT:    call void @control(ptr nocapture nonnull readnone align 16 dereferenceable(8) [[A]])
 ; ATTRIBUTOR-NEXT:    call void @optnone(ptr nonnull align 16 dereferenceable(8) [[A]])
@@ -1126,8 +1126,8 @@ define void @make_live(ptr nonnull dereferenceable(8) %a) {
 declare void @h(ptr) willreturn nounwind
 declare i32 @g(ptr) willreturn nounwind
 define i32 @nonnull_exec_ctx_1(ptr %a, i32 %b) {
-; FNATTRS-LABEL: define i32 @nonnull_exec_ctx_1
-; FNATTRS-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR7]] {
+; FNATTRS-LABEL: define i32 @nonnull_exec_ctx_1(
+; FNATTRS-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR7]] {
 ; FNATTRS-NEXT:  en:
 ; FNATTRS-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[B]], 0
 ; FNATTRS-NEXT:    br i1 [[TMP3]], label [[EX:%.*]], label [[HD:%.*]]
@@ -1141,8 +1141,8 @@ define i32 @nonnull_exec_ctx_1(ptr %a, i32 %b) {
 ; FNATTRS-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP8]], [[B]]
 ; FNATTRS-NEXT:    br i1 [[TMP9]], label [[EX]], label [[HD]]
 ;
-; ATTRIBUTOR-LABEL: define i32 @nonnull_exec_ctx_1
-; ATTRIBUTOR-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR8]] {
+; ATTRIBUTOR-LABEL: define i32 @nonnull_exec_ctx_1(
+; ATTRIBUTOR-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR8]] {
 ; ATTRIBUTOR-NEXT:  en:
 ; ATTRIBUTOR-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[B]], 0
 ; ATTRIBUTOR-NEXT:    br i1 [[TMP3]], label [[EX:%.*]], label [[HD:%.*]]
@@ -1173,8 +1173,8 @@ hd:
 }
 
 define i32 @nonnull_exec_ctx_1b(ptr %a, i32 %b) {
-; FNATTRS-LABEL: define i32 @nonnull_exec_ctx_1b
-; FNATTRS-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR7]] {
+; FNATTRS-LABEL: define i32 @nonnull_exec_ctx_1b(
+; FNATTRS-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR7]] {
 ; FNATTRS-NEXT:  en:
 ; FNATTRS-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[B]], 0
 ; FNATTRS-NEXT:    br i1 [[TMP3]], label [[EX:%.*]], label [[HD:%.*]]
@@ -1190,8 +1190,8 @@ define i32 @nonnull_exec_ctx_1b(ptr %a, i32 %b) {
 ; FNATTRS-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP8]], [[B]]
 ; FNATTRS-NEXT:    br i1 [[TMP9]], label [[EX]], label [[HD]]
 ;
-; ATTRIBUTOR-LABEL: define i32 @nonnull_exec_ctx_1b
-; ATTRIBUTOR-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR8]] {
+; ATTRIBUTOR-LABEL: define i32 @nonnull_exec_ctx_1b(
+; ATTRIBUTOR-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR8]] {
 ; ATTRIBUTOR-NEXT:  en:
 ; ATTRIBUTOR-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[B]], 0
 ; ATTRIBUTOR-NEXT:    br i1 [[TMP3]], label [[EX:%.*]], label [[HD:%.*]]
@@ -1227,8 +1227,8 @@ hd2:
 }
 
 define i32 @nonnull_exec_ctx_2(ptr %a, i32 %b) willreturn nounwind {
-; FNATTRS-LABEL: define i32 @nonnull_exec_ctx_2
-; FNATTRS-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR6]] {
+; FNATTRS-LABEL: define i32 @nonnull_exec_ctx_2(
+; FNATTRS-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR6]] {
 ; FNATTRS-NEXT:  en:
 ; FNATTRS-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[B]], 0
 ; FNATTRS-NEXT:    br i1 [[TMP3]], label [[EX:%.*]], label [[HD:%.*]]
@@ -1242,8 +1242,8 @@ define i32 @nonnull_exec_ctx_2(ptr %a, i32 %b) willreturn nounwind {
 ; FNATTRS-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP8]], [[B]]
 ; FNATTRS-NEXT:    br i1 [[TMP9]], label [[EX]], label [[HD]]
 ;
-; ATTRIBUTOR-LABEL: define i32 @nonnull_exec_ctx_2
-; ATTRIBUTOR-SAME: (ptr nonnull [[A:%.*]], i32 [[B:%.*]]) #[[ATTR7]] {
+; ATTRIBUTOR-LABEL: define i32 @nonnull_exec_ctx_2(
+; ATTRIBUTOR-SAME: ptr nonnull [[A:%.*]], i32 [[B:%.*]]) #[[ATTR7]] {
 ; ATTRIBUTOR-NEXT:  en:
 ; ATTRIBUTOR-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[B]], 0
 ; ATTRIBUTOR-NEXT:    br i1 [[TMP3]], label [[EX:%.*]], label [[HD:%.*]]
@@ -1274,8 +1274,8 @@ hd:
 }
 
 define i32 @nonnull_exec_ctx_2b(ptr %a, i32 %b) willreturn nounwind {
-; FNATTRS-LABEL: define i32 @nonnull_exec_ctx_2b
-; FNATTRS-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR6]] {
+; FNATTRS-LABEL: define i32 @nonnull_exec_ctx_2b(
+; FNATTRS-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR6]] {
 ; FNATTRS-NEXT:  en:
 ; FNATTRS-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[B]], 0
 ; FNATTRS-NEXT:    br i1 [[TMP3]], label [[EX:%.*]], label [[HD:%.*]]
@@ -1291,8 +1291,8 @@ define i32 @nonnull_exec_ctx_2b(ptr %a, i32 %b) willreturn nounwind {
 ; FNATTRS-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP8]], [[B]]
 ; FNATTRS-NEXT:    br i1 [[TMP9]], label [[EX]], label [[HD]]
 ;
-; ATTRIBUTOR-LABEL: define i32 @nonnull_exec_ctx_2b
-; ATTRIBUTOR-SAME: (ptr nonnull [[A:%.*]], i32 [[B:%.*]]) #[[ATTR7]] {
+; ATTRIBUTOR-LABEL: define i32 @nonnull_exec_ctx_2b(
+; ATTRIBUTOR-SAME: ptr nonnull [[A:%.*]], i32 [[B:%.*]]) #[[ATTR7]] {
 ; ATTRIBUTOR-NEXT:  en:
 ; ATTRIBUTOR-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[B]], 0
 ; ATTRIBUTOR-NEXT:    br i1 [[TMP3]], label [[EX:%.*]], label [[HD:%.*]]
@@ -1332,8 +1332,8 @@ declare void @sink(ptr)
 
 ; FIXME: the sink argument should be marked nonnull as in @PR43833_simple.
 define void @PR43833(ptr %0, i32 %1) {
-; COMMON-LABEL: define void @PR43833
-; COMMON-SAME: (ptr [[TMP0:%.*]], i32 [[TMP1:%.*]]) {
+; COMMON-LABEL: define void @PR43833(
+; COMMON-SAME: ptr [[TMP0:%.*]], i32 [[TMP1:%.*]]) {
 ; COMMON-NEXT:    [[TMP3:%.*]] = icmp sgt i32 [[TMP1]], 1
 ; COMMON-NEXT:    br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP7:%.*]]
 ; COMMON:       4:
@@ -1370,8 +1370,8 @@ define void @PR43833(ptr %0, i32 %1) {
 
 ; Adjusted from PR43833
 define void @PR43833_simple(ptr %0, i32 %1) {
-; COMMON-LABEL: define void @PR43833_simple
-; COMMON-SAME: (ptr [[TMP0:%.*]], i32 [[TMP1:%.*]]) {
+; COMMON-LABEL: define void @PR43833_simple(
+; COMMON-SAME: ptr [[TMP0:%.*]], i32 [[TMP1:%.*]]) {
 ; COMMON-NEXT:    [[TMP3:%.*]] = icmp ne i32 [[TMP1]], 0
 ; COMMON-NEXT:    br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP7:%.*]]
 ; COMMON:       4:
diff --git a/llvm/test/Transforms/FunctionAttrs/readnone.ll b/llvm/test/Transforms/FunctionAttrs/readnone.ll
index 40146633cc..7a6d47cbab 100644
--- a/llvm/test/Transforms/FunctionAttrs/readnone.ll
+++ b/llvm/test/Transforms/FunctionAttrs/readnone.ll
@@ -3,13 +3,13 @@
 ; RUN: opt -passes=attributor-light -S < %s | FileCheck --check-prefixes=COMMON,ATTRIBUTOR %s
 
 define void @bar(ptr readonly %0) {
-; FNATTRS-LABEL: define void @bar
-; FNATTRS-SAME: (ptr nocapture readnone [[TMP0:%.*]]) #[[ATTR0:[0-9]+]] {
+; FNATTRS-LABEL: define void @bar(
+; FNATTRS-SAME: ptr nocapture readnone [[TMP0:%.*]]) #[[ATTR0:[0-9]+]] {
 ; FNATTRS-NEXT:    call void @foo(ptr [[TMP0]])
 ; FNATTRS-NEXT:    ret void
 ;
-; ATTRIBUTOR-LABEL: define void @bar
-; ATTRIBUTOR-SAME: (ptr nocapture nofree readnone [[TMP0:%.*]]) #[[ATTR0:[0-9]+]] {
+; ATTRIBUTOR-LABEL: define void @bar(
+; ATTRIBUTOR-SAME: ptr nocapture nofree readnone [[TMP0:%.*]]) #[[ATTR0:[0-9]+]] {
 ; ATTRIBUTOR-NEXT:    call void @foo(ptr nocapture nofree readnone [[TMP0]]) #[[ATTR0]]
 ; ATTRIBUTOR-NEXT:    ret void
 ;
@@ -18,13 +18,13 @@ define void @bar(ptr readonly %0) {
 }
 
 define void @foo(ptr readonly %0) {
-; FNATTRS-LABEL: define void @foo
-; FNATTRS-SAME: (ptr nocapture readnone [[TMP0:%.*]]) #[[ATTR0]] {
+; FNATTRS-LABEL: define void @foo(
+; FNATTRS-SAME: ptr nocapture readnone [[TMP0:%.*]]) #[[ATTR0]] {
 ; FNATTRS-NEXT:    call void @bar(ptr [[TMP0]])
 ; FNATTRS-NEXT:    ret void
 ;
-; ATTRIBUTOR-LABEL: define void @foo
-; ATTRIBUTOR-SAME: (ptr nocapture nofree readnone [[TMP0:%.*]]) #[[ATTR0]] {
+; ATTRIBUTOR-LABEL: define void @foo(
+; ATTRIBUTOR-SAME: ptr nocapture nofree readnone [[TMP0:%.*]]) #[[ATTR0]] {
 ; ATTRIBUTOR-NEXT:    call void @bar(ptr nocapture nofree readnone [[TMP0]]) #[[ATTR0]]
 ; ATTRIBUTOR-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/FunctionAttrs/writeonly.ll b/llvm/test/Transforms/FunctionAttrs/writeonly.ll
index 633068129e..8676d52846 100644
--- a/llvm/test/Transforms/FunctionAttrs/writeonly.ll
+++ b/llvm/test/Transforms/FunctionAttrs/writeonly.ll
@@ -215,7 +215,7 @@ define void @direct2(ptr %p) {
 define void @direct2b(ptr %p) {
 ; FNATTRS: Function Attrs: memory(write)
 ; FNATTRS-LABEL: define {{[^@]+}}@direct2b
-; FNATTRS-SAME: (ptr nocapture [[P:%.*]]) #[[ATTR8]] {
+; FNATTRS-SAME: (ptr nocapture writeonly [[P:%.*]]) #[[ATTR8]] {
 ; FNATTRS-NEXT:    call void @direct2_callee(ptr nocapture [[P]])
 ; FNATTRS-NEXT:    ret void
 ;
@@ -304,7 +304,7 @@ define void @fptr_test2(ptr %p, ptr %f) {
 define void @fptr_test3(ptr %p, ptr %f) {
 ; FNATTRS: Function Attrs: memory(write)
 ; FNATTRS-LABEL: define {{[^@]+}}@fptr_test3
-; FNATTRS-SAME: (ptr nocapture [[P:%.*]], ptr nocapture readonly [[F:%.*]]) #[[ATTR8]] {
+; FNATTRS-SAME: (ptr nocapture writeonly [[P:%.*]], ptr nocapture readonly [[F:%.*]]) #[[ATTR8]] {
 ; FNATTRS-NEXT:    call void [[F]](ptr nocapture [[P]]) #[[ATTR8]]
 ; FNATTRS-NEXT:    ret void
 ;
@@ -317,3 +317,48 @@ define void @fptr_test3(ptr %p, ptr %f) {
   call void %f(ptr nocapture %p) writeonly
   ret void
 }
+
+define void @test_argmem_none_callee(ptr %p) {
+; FNATTRS-LABEL: define {{[^@]+}}@test_argmem_none_callee
+; FNATTRS-SAME: (ptr nocapture readnone [[P:%.*]]) {
+; FNATTRS-NEXT:    call void @direct1_callee(ptr nocapture [[P]]) #[[ATTR9:[0-9]+]]
+; FNATTRS-NEXT:    ret void
+;
+; ATTRIBUTOR-LABEL: define {{[^@]+}}@test_argmem_none_callee
+; ATTRIBUTOR-SAME: (ptr nocapture [[P:%.*]]) {
+; ATTRIBUTOR-NEXT:    call void @direct1_callee(ptr nocapture [[P]]) #[[ATTR8:[0-9]+]]
+; ATTRIBUTOR-NEXT:    ret void
+;
+  call void @direct1_callee(ptr nocapture %p) memory(readwrite, argmem: none)
+  ret void
+}
+
+define void @test_argmem_read_callee(ptr %p) {
+; FNATTRS-LABEL: define {{[^@]+}}@test_argmem_read_callee
+; FNATTRS-SAME: (ptr nocapture readonly [[P:%.*]]) {
+; FNATTRS-NEXT:    call void @direct1_callee(ptr nocapture [[P]]) #[[ATTR10:[0-9]+]]
+; FNATTRS-NEXT:    ret void
+;
+; ATTRIBUTOR-LABEL: define {{[^@]+}}@test_argmem_read_callee
+; ATTRIBUTOR-SAME: (ptr nocapture [[P:%.*]]) {
+; ATTRIBUTOR-NEXT:    call void @direct1_callee(ptr nocapture [[P]]) #[[ATTR9:[0-9]+]]
+; ATTRIBUTOR-NEXT:    ret void
+;
+  call void @direct1_callee(ptr nocapture %p) memory(readwrite, argmem: read)
+  ret void
+}
+
+define void @test_argmem_write_callee(ptr %p) {
+; FNATTRS-LABEL: define {{[^@]+}}@test_argmem_write_callee
+; FNATTRS-SAME: (ptr nocapture writeonly [[P:%.*]]) {
+; FNATTRS-NEXT:    call void @direct1_callee(ptr nocapture [[P]]) #[[ATTR11:[0-9]+]]
+; FNATTRS-NEXT:    ret void
+;
+; ATTRIBUTOR-LABEL: define {{[^@]+}}@test_argmem_write_callee
+; ATTRIBUTOR-SAME: (ptr nocapture [[P:%.*]]) {
+; ATTRIBUTOR-NEXT:    call void @direct1_callee(ptr nocapture [[P]]) #[[ATTR10:[0-9]+]]
+; ATTRIBUTOR-NEXT:    ret void
+;
+  call void @direct1_callee(ptr nocapture %p) memory(readwrite, argmem: write)
+  ret void
+}
diff --git a/llvm/test/Transforms/GVN/captured-before.ll b/llvm/test/Transforms/GVN/captured-before.ll
new file mode 100644
index 0000000000..4e025b3f77
--- /dev/null
+++ b/llvm/test/Transforms/GVN/captured-before.ll
@@ -0,0 +1,80 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
+; RUN: opt -S -passes=gvn < %s | FileCheck %s
+
+declare void @capture(ptr)
+declare void @some_call()
+
+define i32 @test_call_before_capture(ptr %p) {
+; CHECK-LABEL: define i32 @test_call_before_capture(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    store i32 123, ptr [[A]], align 4
+; CHECK-NEXT:    call void @some_call()
+; CHECK-NEXT:    call void @capture(ptr [[A]])
+; CHECK-NEXT:    ret i32 123
+;
+  %a = alloca i32
+  store i32 123, ptr %a
+  call void @some_call()
+  %v = load i32, ptr %a
+  call void @capture(ptr %a)
+  ret i32 %v
+}
+
+define i32 @test_call_after_capture(ptr %p) {
+; CHECK-LABEL: define i32 @test_call_after_capture(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    store i32 123, ptr [[A]], align 4
+; CHECK-NEXT:    call void @capture(ptr [[A]])
+; CHECK-NEXT:    call void @some_call()
+; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  %a = alloca i32
+  store i32 123, ptr %a
+  call void @capture(ptr %a)
+  call void @some_call()
+  %v = load i32, ptr %a
+  ret i32 %v
+}
+
+define i32 @test_store_before_capture(ptr %p) {
+; CHECK-LABEL: define i32 @test_store_before_capture(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    store i32 123, ptr [[A]], align 4
+; CHECK-NEXT:    [[P2:%.*]] = load ptr, ptr [[P]], align 8
+; CHECK-NEXT:    store i32 42, ptr [[P2]], align 4
+; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    call void @capture(ptr [[A]])
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  %a = alloca i32
+  store i32 123, ptr %a
+  %p2 = load ptr, ptr %p
+  store i32 42, ptr %p2
+  %v = load i32, ptr %a
+  call void @capture(ptr %a)
+  ret i32 %v
+}
+
+define i32 @test_store_after_capture(ptr %p) {
+; CHECK-LABEL: define i32 @test_store_after_capture(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    store i32 123, ptr [[A]], align 4
+; CHECK-NEXT:    call void @capture(ptr [[A]])
+; CHECK-NEXT:    [[P2:%.*]] = load ptr, ptr [[P]], align 8
+; CHECK-NEXT:    store i32 42, ptr [[P2]], align 4
+; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  %a = alloca i32
+  store i32 123, ptr %a
+  call void @capture(ptr %a)
+  %p2 = load ptr, ptr %p
+  store i32 42, ptr %p2
+  %v = load i32, ptr %a
+  ret i32 %v
+}
diff --git a/llvm/test/Transforms/GVN/pr69301.ll b/llvm/test/Transforms/GVN/pr69301.ll
new file mode 100644
index 0000000000..dc54ef559d
--- /dev/null
+++ b/llvm/test/Transforms/GVN/pr69301.ll
@@ -0,0 +1,59 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
+; RUN: opt -S -passes=gvn < %s | FileCheck %s
+
+; Make sure we don't have use-after-free due to dangling values in
+; select available value.
+
+define i64 @test(i1 %c, ptr %p) {
+; CHECK-LABEL: define i64 @test(
+; CHECK-SAME: i1 [[C:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[ADD:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[P]], [[ENTRY]] ], [ [[SELECT:%.*]], [[LOOP_LATCH]] ]
+; CHECK-NEXT:    [[ICMP:%.*]] = icmp eq i64 [[IV]], 0
+; CHECK-NEXT:    br i1 [[ICMP]], label [[LOOP_EXIT_CRIT_EDGE:%.*]], label [[LOOP_CONT:%.*]]
+; CHECK:       loop.exit_crit_edge:
+; CHECK-NEXT:    [[RES_PRE:%.*]] = load i64, ptr [[PTR_IV]], align 8
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       loop.cont:
+; CHECK-NEXT:    [[ADD]] = add i64 [[IV]], -1
+; CHECK-NEXT:    [[RES_PRE1:%.*]] = load i64, ptr [[PTR_IV]], align 8
+; CHECK-NEXT:    br i1 [[C]], label [[EXITSPLIT:%.*]], label [[LOOP_LATCH]]
+; CHECK:       loop.latch:
+; CHECK-NEXT:    [[LOAD6:%.*]] = load i64, ptr [[P]], align 8
+; CHECK-NEXT:    [[ICMP7:%.*]] = icmp ugt i64 [[RES_PRE1]], [[LOAD6]]
+; CHECK-NEXT:    [[TMP0:%.*]] = select i1 [[ICMP7]], i64 [[RES_PRE1]], i64 [[LOAD6]]
+; CHECK-NEXT:    [[SELECT]] = select i1 [[ICMP7]], ptr [[PTR_IV]], ptr [[P]]
+; CHECK-NEXT:    br label [[LOOP]]
+; CHECK:       exitsplit:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[RES_PRE1]], [[EXITSPLIT]] ], [ [[RES_PRE]], [[LOOP_EXIT_CRIT_EDGE]] ]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %add, %loop.latch ]
+  %ptr.iv = phi ptr [ %p, %entry ], [ %select, %loop.latch ]
+  %icmp = icmp eq i64 %iv, 0
+  br i1 %icmp, label %exit, label %loop.cont
+
+loop.cont:
+  %add = add i64 %iv, -1
+  br i1 %c, label %exit, label %loop.latch
+
+loop.latch:
+  %load = load i64, ptr %ptr.iv, align 8
+  %load6 = load i64, ptr %p, align 8
+  %icmp7 = icmp ugt i64 %load, %load6
+  %select = select i1 %icmp7, ptr %ptr.iv, ptr %p
+  br label %loop
+
+exit:
+  %res = load i64, ptr %ptr.iv, align 8
+  ret i64 %res
+}
diff --git a/llvm/test/Transforms/HipStdPar/accelerator-code-selection.ll b/llvm/test/Transforms/HipStdPar/accelerator-code-selection.ll
new file mode 100644
index 0000000000..2d41bd9b34
--- /dev/null
+++ b/llvm/test/Transforms/HipStdPar/accelerator-code-selection.ll
@@ -0,0 +1,116 @@
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=hipstdpar-select-accelerator-code \
+; RUN: %s | FileCheck %s
+
+$_ZNK8CallableclEPi = comdat any
+$_ZNK8CallableclEPf = comdat any
+$_ZNK8Callable6mem_fnEPKi = comdat any
+$_ZN8Callable13static_mem_fnEPKi = comdat any
+; CHECK-NOT: $_ZNK8Callable37another_mem_fn_which_will_get_removedEPKf
+$_ZNK8Callable37another_mem_fn_which_will_get_removedEPKf = comdat any
+; CHECK-NOT: $_ZN8Callable44another_static_mem_fn_which_will_get_removedEPKf
+$_ZN8Callable44another_static_mem_fn_which_will_get_removedEPKf = comdat any
+
+%struct.Callable = type { [64 x i8] }
+
+; CHECK-NOT: @should_be_removed
+@llvm.compiler.used = appending addrspace(1) global [1 x ptr] [ptr @should_be_removed], section "llvm.metadata"
+
+define void @should_be_removed(ptr %p) {
+  ret void
+}
+
+declare void @llvm.trap()
+
+; CHECK: define {{.*}} @called_via_chain
+define void @called_via_chain(ptr %p) {
+  entry:
+    %tobool.not = icmp eq ptr %p, null
+    br i1 %tobool.not, label %if.then, label %if.end
+
+  if.then:
+    tail call void @llvm.trap()
+    unreachable
+
+  if.end:
+    ret void
+}
+
+; CHECK: define {{.*}} @directly_called
+define void @directly_called(ptr %p) {
+  tail call void @called_via_chain(ptr %p)
+  ret void
+}
+
+; CHECK: define {{.*}} amdgpu_kernel {{.*}} @accelerator_execution_root
+define hidden amdgpu_kernel void @accelerator_execution_root(ptr %p) {
+  tail call void @directly_called(ptr %p)
+  ret void
+}
+
+; CHECK-NOT: @defined_elsewhere_should_be_removed
+declare void @defined_elsewhere_should_be_removed(ptr)
+
+; CHECK: declare {{.*}} @defined_elsewhere_directly_called
+declare void @defined_elsewhere_directly_called(ptr)
+
+; CHECK: define {{.*}} amdgpu_kernel {{.*}} @another_accelerator_execution_root
+define hidden amdgpu_kernel void @another_accelerator_execution_root(ptr %p) {
+  tail call void @defined_elsewhere_directly_called(ptr %p)
+  ret void
+}
+
+; Also test passing a callable object (functor / lambda) to a kernel, which is
+; the common pattern for customising algorithms.
+
+; CHECK: define {{.*}} amdgpu_kernel {{.*}} @_Z22accelerator_execution_root_taking_callablePi8Callable
+define hidden amdgpu_kernel void @_Z22accelerator_execution_root_taking_callablePi8Callable(ptr noundef %p, ptr addrspace(4) nocapture readonly byref(%struct.Callable) align 8 %callable) {
+  %callable_in_generic = addrspacecast ptr addrspace(4) %callable to ptr
+  call void @_ZNK8CallableclEPi(ptr noundef nonnull align 1 dereferenceable(64) %callable_in_generic, ptr noundef %p)
+
+  ret void
+}
+
+; CHECK: define {{.*}} @_ZNK8CallableclEPi
+define linkonce_odr dso_local void @_ZNK8CallableclEPi(ptr noundef nonnull align 1 dereferenceable(64) %this, ptr noundef %p) {
+  call void @_ZNK8Callable6mem_fnEPKi(ptr noundef nonnull align 1 dereferenceable(1) %this, ptr noundef %p)
+
+  ret void
+}
+
+; CHECK: define {{.*}} @_ZNK8Callable6mem_fnEPKi
+define linkonce_odr dso_local void @_ZNK8Callable6mem_fnEPKi(ptr noundef nonnull align 1 dereferenceable(1) %this, ptr noundef %p) {
+  call void @_ZN8Callable13static_mem_fnEPKi(ptr noundef %p)
+
+  ret void
+}
+
+; CHECK: define {{.*}} @_ZN8Callable13static_mem_fnEPKi
+define linkonce_odr dso_local void @_ZN8Callable13static_mem_fnEPKi(ptr noundef %p) {
+  ret void
+}
+
+; CHECK-NOT: define {{.*}} @_Z26non_kernel_taking_callablePf8Callable
+define dso_local void @_Z26non_kernel_taking_callablePf8Callable(ptr noundef %p, ptr noundef byval(%struct.Callable) align 8 %callable) {
+  call void @_ZNK8CallableclEPf(ptr noundef nonnull align 1 dereferenceable(64) %callable, ptr noundef %p)
+
+  ret void
+}
+
+; CHECK-NOT: define {{.*}} @_ZNK8CallableclEPf
+define linkonce_odr dso_local void @_ZNK8CallableclEPf(ptr noundef nonnull align 1 dereferenceable(64) %this, ptr noundef %p) {
+  call void @_ZNK8Callable37another_mem_fn_which_will_get_removedEPKf(ptr noundef nonnull align 1 dereferenceable(64) %this, ptr noundef %p)
+
+  ret void
+}
+
+; CHECK-NOT: @_ZNK8Callable37another_mem_fn_which_will_get_removedEPKf
+define linkonce_odr dso_local void @_ZNK8Callable37another_mem_fn_which_will_get_removedEPKf(ptr noundef nonnull align 1 dereferenceable(64) %this, ptr noundef %p) {
+  call void @_ZN8Callable44another_static_mem_fn_which_will_get_removedEPKf(ptr noundef %p)
+
+  ret void
+}
+
+; CHECK-NOT: @_ZN8Callable44another_static_mem_fn_which_will_get_removedEPKf
+define linkonce_odr dso_local void @_ZN8Callable44another_static_mem_fn_which_will_get_removedEPKf(ptr noundef %p) {
+  ret void
+}
\ No newline at end of file
diff --git a/llvm/test/Transforms/HipStdPar/allocation-interposition.ll b/llvm/test/Transforms/HipStdPar/allocation-interposition.ll
new file mode 100644
index 0000000000..291b06ed0c
--- /dev/null
+++ b/llvm/test/Transforms/HipStdPar/allocation-interposition.ll
@@ -0,0 +1,221 @@
+; RUN: opt -S -passes=hipstdpar-interpose-alloc %s | FileCheck %s
+
+%"struct.std::nothrow_t" = type { i8 }
+
+@_ZSt7nothrow = external global %"struct.std::nothrow_t", align 1
+
+declare ptr @__hipstdpar_aligned_alloc(i64, i64)
+
+declare ptr @__hipstdpar_malloc(i64)
+
+declare ptr @__hipstdpar_calloc(i64, i64)
+
+declare i32 @__hipstdpar_posix_aligned_alloc(ptr, i64, i64)
+
+declare void @__hipstdpar_hidden_free(ptr)
+
+declare ptr @__hipstdpar_realloc(ptr, i64)
+
+declare ptr @__hipstdpar_realloc_array(ptr, i64, i64)
+
+declare void @__hipstdpar_free(ptr)
+
+declare ptr @__hipstdpar_operator_new_aligned(i64, i64)
+
+declare ptr @__hipstdpar_operator_new(i64)
+
+declare ptr @__hipstdpar_operator_new_nothrow(i64, %"struct.std::nothrow_t")
+
+declare ptr @__hipstdpar_operator_new_aligned_nothrow(i64, i64, %"struct.std::nothrow_t")
+
+declare void @__hipstdpar_operator_delete_aligned_sized(ptr, i64, i64)
+
+declare void @__hipstdpar_operator_delete(ptr)
+
+declare void @__hipstdpar_operator_delete_aligned(ptr, i64)
+
+declare void @__hipstdpar_operator_delete_sized(ptr, i64)
+
+define dso_local noundef i32 @allocs() {
+  ; CHECK: %1 = call noalias align 8 ptr @__hipstdpar_aligned_alloc(i64 noundef 8, i64 noundef 42)
+  %1 = call noalias align 8 ptr @aligned_alloc(i64 noundef 8, i64 noundef 42)
+  ; CHECK: call void @__hipstdpar_free(ptr noundef %1)
+  call void @free(ptr noundef %1)
+
+  ; CHECK: %2 = call noalias ptr @__hipstdpar_calloc(i64 noundef 1, i64 noundef 42)
+  %2 = call noalias ptr @calloc(i64 noundef 1, i64 noundef 42)
+  ; CHECK: call void @__hipstdpar_free(ptr noundef %2)
+  call void @free(ptr noundef %2)
+
+  ; CHECK: %3 = call noalias ptr @__hipstdpar_malloc(i64 noundef 42)
+  %3 = call noalias ptr @malloc(i64 noundef 42)
+  ; CHECK: call void @__hipstdpar_free(ptr noundef %3)
+  call void @free(ptr noundef %3)
+
+  ; CHECK: %4 = call noalias align 8 ptr @__hipstdpar_aligned_alloc(i64 noundef 8, i64 noundef 42)
+  %4 = call noalias align 8 ptr @memalign(i64 noundef 8, i64 noundef 42)
+  ; CHECK: call void @__hipstdpar_free(ptr noundef %4)
+  call void @free(ptr noundef %4)
+
+  %tmp = alloca ptr, align 8
+  ; CHECK: %5 = call i32 @__hipstdpar_posix_aligned_alloc(ptr noundef %tmp, i64 noundef 8, i64 noundef 42)
+  %5 = call i32 @posix_memalign(ptr noundef %tmp, i64 noundef 8, i64 noundef 42)
+  ; CHECK: call void @__hipstdpar_free(ptr noundef %tmp)
+  call void @free(ptr noundef %tmp)
+
+  ; CHECK: %6 = call noalias ptr @__hipstdpar_malloc(i64 noundef 42)
+  %6 = call noalias ptr @malloc(i64 noundef 42)
+  ; CHECK: %7 = call ptr @__hipstdpar_realloc(ptr noundef %6, i64 noundef 42)
+  %7 = call ptr @realloc(ptr noundef %6, i64 noundef 42)
+  ; CHECK: call void @__hipstdpar_free(ptr noundef %7)
+  call void @free(ptr noundef %7)
+
+  ; CHECK: %8 = call noalias ptr @__hipstdpar_calloc(i64 noundef 1, i64 noundef 42)
+  %8 = call noalias ptr @calloc(i64 noundef 1, i64 noundef 42)
+  ; CHECK: %9 = call ptr @__hipstdpar_realloc_array(ptr noundef %8, i64 noundef 1, i64 noundef 42)
+  %9 = call ptr @reallocarray(ptr noundef %8, i64 noundef 1, i64 noundef 42)
+  ; CHECK: call void @__hipstdpar_free(ptr noundef %9)
+  call void @free(ptr noundef %9)
+
+  ; CHECK: %10 = call noalias noundef nonnull ptr @__hipstdpar_operator_new(i64 noundef 1)
+  %10 = call noalias noundef nonnull ptr @_Znwm(i64 noundef 1)
+  ; CHECK: call void @__hipstdpar_operator_delete(ptr noundef %10)
+  call void @_ZdlPv(ptr noundef %10)
+
+  ; CHECK: %11 = call noalias noundef nonnull align 8 ptr @__hipstdpar_operator_new_aligned(i64 noundef 1, i64 noundef 8)
+  %11 = call noalias noundef nonnull align 8 ptr @_ZnwmSt11align_val_t(i64 noundef 1, i64 noundef 8)
+  ; CHECK: call void @__hipstdpar_operator_delete_aligned(ptr noundef %11, i64 noundef 8)
+  call void @_ZdlPvSt11align_val_t(ptr noundef %11, i64 noundef 8)
+
+  ; CHECK: %12 = call noalias noundef ptr @__hipstdpar_operator_new_nothrow(i64 noundef 1, ptr noundef nonnull align 1 dereferenceable(1) @_ZSt7nothrow)
+  %12 = call noalias noundef ptr @_ZnwmRKSt9nothrow_t(i64 noundef 1, ptr noundef nonnull align 1 dereferenceable(1) @_ZSt7nothrow)
+  ; CHECK: call void @__hipstdpar_operator_delete(ptr noundef %12)
+  call void @_ZdlPv(ptr noundef %12)
+
+  ; CHECK: %13 = call noalias noundef align 8 ptr @__hipstdpar_operator_new_aligned_nothrow(i64 noundef 1, i64 noundef 8, ptr noundef nonnull align 1 dereferenceable(1) @_ZSt7nothrow)
+  %13 = call noalias noundef align 8 ptr @_ZnwmSt11align_val_tRKSt9nothrow_t(i64 noundef 1, i64 noundef 8, ptr noundef nonnull align 1 dereferenceable(1) @_ZSt7nothrow)
+  ; CHECK: call void @__hipstdpar_operator_delete_aligned(ptr noundef %13, i64 noundef 8)
+  call void @_ZdlPvSt11align_val_t(ptr noundef %13, i64 noundef 8)
+
+  ; CHECK: %14 = call noalias noundef nonnull ptr @__hipstdpar_operator_new(i64 noundef 42)
+  %14 = call noalias noundef nonnull ptr @_Znam(i64 noundef 42)
+  ; CHECK: call void @__hipstdpar_operator_delete(ptr noundef %14)
+  call void @_ZdaPv(ptr noundef %14)
+
+  ; CHECK: %15 = call noalias noundef nonnull align 8 ptr @__hipstdpar_operator_new_aligned(i64 noundef 42, i64 noundef 8)
+  %15 = call noalias noundef nonnull align 8 ptr @_ZnamSt11align_val_t(i64 noundef 42, i64 noundef 8)
+  ; CHECK: call void @__hipstdpar_operator_delete_aligned(ptr noundef %15, i64 noundef 8)
+  call void @_ZdaPvSt11align_val_t(ptr noundef %15, i64 noundef 8)
+
+  ; CHECK:  %16 = call noalias noundef ptr @__hipstdpar_operator_new_nothrow(i64 noundef 42, ptr noundef nonnull align 1 dereferenceable(1) @_ZSt7nothrow)
+  %16 = call noalias noundef ptr @_ZnamRKSt9nothrow_t(i64 noundef 42, ptr noundef nonnull align 1 dereferenceable(1) @_ZSt7nothrow)
+  ; CHECK: call void @__hipstdpar_operator_delete(ptr noundef %16)
+  call void @_ZdaPv(ptr noundef %16)
+
+  ; CHECK: %17 = call noalias noundef align 8 ptr @__hipstdpar_operator_new_aligned_nothrow(i64 noundef 42, i64 noundef 8, ptr noundef nonnull align 1 dereferenceable(1) @_ZSt7nothrow)
+  %17 = call noalias noundef align 8 ptr @_ZnamSt11align_val_tRKSt9nothrow_t(i64 noundef 42, i64 noundef 8, ptr noundef nonnull align 1 dereferenceable(1) @_ZSt7nothrow)
+  ; CHECK: call void @__hipstdpar_operator_delete_aligned(ptr noundef %17, i64 noundef 8)
+  call void @_ZdaPvSt11align_val_t(ptr noundef %17, i64 noundef 8)
+
+  ; CHECK:  %18 = call ptr @__hipstdpar_calloc(i64 noundef 1, i64 noundef 42)
+  %18 = call ptr @calloc(i64 noundef 1, i64 noundef 42)
+  ; CHECK: call void @__hipstdpar_free(ptr noundef %18)
+  call void @free(ptr noundef %18)
+
+  ; CHECK: %19 = call ptr @__hipstdpar_malloc(i64 noundef 42)
+  %19 = call ptr @malloc(i64 noundef 42)
+  ; CHECK: call void @__hipstdpar_free(ptr noundef %19)
+  call void @free(ptr noundef %19)
+
+  ; CHECK: %20 = call noalias noundef nonnull ptr @__hipstdpar_operator_new(i64 noundef 42)
+  %20 = call noalias noundef nonnull ptr @_Znwm(i64 noundef 42)
+  ; CHECK: call void @__hipstdpar_operator_delete(ptr noundef %20)
+  call void @_ZdlPv(ptr noundef %20)
+
+  ; CHECK:  %21 = call noalias noundef nonnull align 8 ptr @__hipstdpar_operator_new_aligned(i64 noundef 42, i64 noundef 8)
+  %21 = call noalias noundef nonnull align 8 ptr @_ZnwmSt11align_val_t(i64 noundef 42, i64 noundef 8)
+  ; CHECK: call void @__hipstdpar_operator_delete_aligned(ptr noundef %21, i64 noundef 8)
+  call void @_ZdlPvSt11align_val_t(ptr noundef %21, i64 noundef 8)
+
+  ; CHECK: %22 = call noalias noundef ptr @__hipstdpar_operator_new_nothrow(i64 noundef 42, ptr noundef nonnull align 1 dereferenceable(1) @_ZSt7nothrow)
+  %22 = call noalias noundef ptr @_ZnwmRKSt9nothrow_t(i64 noundef 42, ptr noundef nonnull align 1 dereferenceable(1) @_ZSt7nothrow)
+  ; CHECK: call void @__hipstdpar_operator_delete(ptr noundef %22)
+  call void @_ZdlPv(ptr noundef %22)
+
+  ; CHECK:  %23 = call noalias noundef align 8 ptr @__hipstdpar_operator_new_aligned_nothrow(i64 noundef 42, i64 noundef 8, ptr noundef nonnull align 1 dereferenceable(1) @_ZSt7nothrow)
+  %23 = call noalias noundef align 8 ptr @_ZnwmSt11align_val_tRKSt9nothrow_t(i64 noundef 42, i64 noundef 8, ptr noundef nonnull align 1 dereferenceable(1) @_ZSt7nothrow)
+  ; CHECK: call void @__hipstdpar_operator_delete_aligned(ptr noundef %23, i64 noundef 8)
+  call void @_ZdlPvSt11align_val_t(ptr noundef %23, i64 noundef 8)
+
+  ; CHECK: %24 = call ptr @__hipstdpar_malloc(i64 noundef 42)
+  %24 = call ptr @malloc(i64 noundef 42)
+  ; CHECK: %25 = call ptr @__hipstdpar_realloc(ptr noundef %24, i64 noundef 41)
+  %25 = call ptr @realloc(ptr noundef %24, i64 noundef 41)
+  ; CHECK: call void @__hipstdpar_free(ptr noundef %25)
+  call void @free(ptr noundef %25)
+
+  ; CHECK: %26 = call ptr @__hipstdpar_calloc(i64 noundef 1, i64 noundef 42)
+  %26 = call ptr @__libc_calloc(i64 noundef 1, i64 noundef 42)
+  ; CHECK: call void @__hipstdpar_free(ptr noundef %26)
+  call void @__libc_free(ptr noundef %26)
+
+  ; CHECK: %27 = call ptr @__hipstdpar_malloc(i64 noundef 42)
+  %27 = call ptr @__libc_malloc(i64 noundef 42)
+  ; CHECK: call void @__hipstdpar_free(ptr noundef %27)
+  call void @__libc_free(ptr noundef %27)
+
+  ; CHECK: %28 = call ptr @__hipstdpar_aligned_alloc(i64 noundef 8, i64 noundef 42)
+  %28 = call ptr @__libc_memalign(i64 noundef 8, i64 noundef 42)
+  ; CHECK: call void @__hipstdpar_free(ptr noundef %28)
+  call void @__libc_free(ptr noundef %28)
+
+  ret i32 0
+}
+
+declare noalias ptr @aligned_alloc(i64 noundef, i64 noundef)
+
+declare void @free(ptr noundef)
+
+declare noalias ptr @calloc(i64 noundef, i64 noundef)
+
+declare noalias ptr @malloc(i64 noundef)
+
+declare noalias ptr @memalign(i64 noundef, i64 noundef)
+
+declare i32 @posix_memalign(ptr noundef, i64 noundef, i64 noundef)
+
+declare ptr @realloc(ptr noundef, i64 noundef)
+
+declare ptr @reallocarray(ptr noundef, i64 noundef, i64 noundef)
+
+declare noundef nonnull ptr @_Znwm(i64 noundef)
+
+declare void @_ZdlPv(ptr noundef)
+
+declare noalias noundef nonnull ptr @_ZnwmSt11align_val_t(i64 noundef, i64 noundef)
+
+declare void @_ZdlPvSt11align_val_t(ptr noundef, i64 noundef)
+
+declare noalias noundef ptr @_ZnwmRKSt9nothrow_t(i64 noundef, ptr noundef nonnull align 1 dereferenceable(1))
+
+declare noalias noundef ptr @_ZnwmSt11align_val_tRKSt9nothrow_t(i64 noundef, i64 noundef, ptr noundef nonnull align 1 dereferenceable(1))
+
+declare noundef nonnull ptr @_Znam(i64 noundef)
+
+declare void @_ZdaPv(ptr noundef)
+
+declare noalias noundef nonnull ptr @_ZnamSt11align_val_t(i64 noundef, i64 noundef)
+
+declare void @_ZdaPvSt11align_val_t(ptr noundef, i64 noundef)
+
+declare noalias noundef ptr @_ZnamRKSt9nothrow_t(i64 noundef, ptr noundef nonnull align 1 dereferenceable(1))
+
+declare noalias noundef ptr @_ZnamSt11align_val_tRKSt9nothrow_t(i64 noundef, i64 noundef, ptr noundef nonnull align 1 dereferenceable(1))
+
+declare ptr @__libc_calloc(i64 noundef, i64 noundef)
+
+declare void @__libc_free(ptr noundef)
+
+declare ptr @__libc_malloc(i64 noundef)
+
+declare ptr @__libc_memalign(i64 noundef, i64 noundef)
\ No newline at end of file
diff --git a/llvm/test/Transforms/HipStdPar/allocation-no-interposition.ll b/llvm/test/Transforms/HipStdPar/allocation-no-interposition.ll
new file mode 100644
index 0000000000..15640c6ae9
--- /dev/null
+++ b/llvm/test/Transforms/HipStdPar/allocation-no-interposition.ll
@@ -0,0 +1,161 @@
+; RUN: opt < %s -passes=hipstdpar-interpose-alloc -S 2>&1 | FileCheck %s
+
+; CHECK: warning: {{.*}} aligned_alloc {{.*}} cannot be interposed, missing: __hipstdpar_aligned_alloc. Tried to run the allocation interposition pass without the replacement functions available.
+; CHECK: warning: {{.*}} free {{.*}} cannot be interposed, missing: __hipstdpar_free. Tried to run the allocation interposition pass without the replacement functions available.
+; CHECK: warning: {{.*}} calloc {{.*}} cannot be interposed, missing: __hipstdpar_calloc. Tried to run the allocation interposition pass without the replacement functions available.
+; CHECK: warning: {{.*}} malloc {{.*}} cannot be interposed, missing: __hipstdpar_malloc. Tried to run the allocation interposition pass without the replacement functions available.
+; CHECK: warning: {{.*}} memalign {{.*}} cannot be interposed, missing: __hipstdpar_aligned_alloc. Tried to run the allocation interposition pass without the replacement functions available.
+; CHECK: warning: {{.*}} posix_memalign {{.*}} cannot be interposed, missing: __hipstdpar_posix_aligned_alloc. Tried to run the allocation interposition pass without the replacement functions available.
+; CHECK: warning: {{.*}} realloc {{.*}} cannot be interposed, missing: __hipstdpar_realloc. Tried to run the allocation interposition pass without the replacement functions available.
+; CHECK: warning: {{.*}} reallocarray {{.*}} cannot be interposed, missing: __hipstdpar_realloc_array. Tried to run the allocation interposition pass without the replacement functions available.
+; CHECK: warning: {{.*}} _Znwm {{.*}} cannot be interposed, missing: __hipstdpar_operator_new. Tried to run the allocation interposition pass without the replacement functions available.
+; CHECK: warning: {{.*}} _ZdlPv {{.*}} cannot be interposed, missing: __hipstdpar_operator_delete. Tried to run the allocation interposition pass without the replacement functions available.
+; CHECK: warning: {{.*}} _ZnwmSt11align_val_t {{.*}} cannot be interposed, missing: __hipstdpar_operator_new_aligned. Tried to run the allocation interposition pass without the replacement functions available.
+; CHECK: warning: {{.*}} _ZdlPvSt11align_val_t {{.*}} cannot be interposed, missing: __hipstdpar_operator_delete_aligned. Tried to run the allocation interposition pass without the replacement functions available.
+; CHECK: warning: {{.*}} _ZnwmRKSt9nothrow_t {{.*}} cannot be interposed, missing: __hipstdpar_operator_new_nothrow. Tried to run the allocation interposition pass without the replacement functions available.
+; CHECK: warning: {{.*}} _ZnwmSt11align_val_tRKSt9nothrow_t {{.*}} cannot be interposed, missing: __hipstdpar_operator_new_aligned_nothrow. Tried to run the allocation interposition pass without the replacement functions available.
+; CHECK: warning: {{.*}} _Znam {{.*}} cannot be interposed, missing: __hipstdpar_operator_new. Tried to run the allocation interposition pass without the replacement functions available.
+; CHECK: warning: {{.*}} _ZdaPv {{.*}} cannot be interposed, missing: __hipstdpar_operator_delete. Tried to run the allocation interposition pass without the replacement functions available.
+; CHECK: warning: {{.*}} _ZnamSt11align_val_t {{.*}} cannot be interposed, missing: __hipstdpar_operator_new_aligned. Tried to run the allocation interposition pass without the replacement functions available.
+; CHECK: warning: {{.*}} _ZdaPvSt11align_val_t {{.*}} cannot be interposed, missing: __hipstdpar_operator_delete_aligned. Tried to run the allocation interposition pass without the replacement functions available.
+; CHECK: warning: {{.*}} _ZnamRKSt9nothrow_t {{.*}} cannot be interposed, missing: __hipstdpar_operator_new_nothrow. Tried to run the allocation interposition pass without the replacement functions available.
+; CHECK: warning: {{.*}} _ZnamSt11align_val_tRKSt9nothrow_t {{.*}} cannot be interposed, missing: __hipstdpar_operator_new_aligned_nothrow. Tried to run the allocation interposition pass without the replacement functions available.
+; CHECK: warning: {{.*}} __libc_calloc {{.*}} cannot be interposed, missing: __hipstdpar_calloc. Tried to run the allocation interposition pass without the replacement functions available.
+; CHECK: warning: {{.*}} __libc_free {{.*}} cannot be interposed, missing: __hipstdpar_free. Tried to run the allocation interposition pass without the replacement functions available.
+; CHECK: warning: {{.*}} __libc_malloc {{.*}} cannot be interposed, missing: __hipstdpar_malloc. Tried to run the allocation interposition pass without the replacement functions available.
+; CHECK: warning: {{.*}} __libc_memalign {{.*}} cannot be interposed, missing: __hipstdpar_aligned_alloc. Tried to run the allocation interposition pass without the replacement functions available.
+
+%"struct.std::nothrow_t" = type { i8 }
+
+@_ZSt7nothrow = external global %"struct.std::nothrow_t", align 1
+
+define dso_local noundef i32 @allocs() {
+  %1 = call noalias align 8 ptr @aligned_alloc(i64 noundef 8, i64 noundef 42)
+  call void @free(ptr noundef %1)
+
+  %2 = call noalias ptr @calloc(i64 noundef 1, i64 noundef 42)
+  call void @free(ptr noundef %2)
+
+  %3 = call noalias ptr @malloc(i64 noundef 42)
+  call void @free(ptr noundef %3)
+
+  %4 = call noalias align 8 ptr @memalign(i64 noundef 8, i64 noundef 42)
+  call void @free(ptr noundef %4)
+
+  %tmp = alloca ptr, align 8
+  %5 = call i32 @posix_memalign(ptr noundef %tmp, i64 noundef 8, i64 noundef 42)
+  call void @free(ptr noundef %tmp)
+
+  %6 = call noalias ptr @malloc(i64 noundef 42)
+  %7 = call ptr @realloc(ptr noundef %6, i64 noundef 42)
+  call void @free(ptr noundef %7)
+
+  %8 = call noalias ptr @calloc(i64 noundef 1, i64 noundef 42)
+  %9 = call ptr @reallocarray(ptr noundef %8, i64 noundef 1, i64 noundef 42)
+  call void @free(ptr noundef %9)
+
+  %10 = call noalias noundef nonnull ptr @_Znwm(i64 noundef 1)
+  call void @_ZdlPv(ptr noundef %10)
+
+  %11 = call noalias noundef nonnull align 8 ptr @_ZnwmSt11align_val_t(i64 noundef 1, i64 noundef 8)
+  call void @_ZdlPvSt11align_val_t(ptr noundef %11, i64 noundef 8)
+
+  %12 = call noalias noundef ptr @_ZnwmRKSt9nothrow_t(i64 noundef 1, ptr noundef nonnull align 1 dereferenceable(1) @_ZSt7nothrow)
+  call void @_ZdlPv(ptr noundef %12)
+
+  %13 = call noalias noundef align 8 ptr @_ZnwmSt11align_val_tRKSt9nothrow_t(i64 noundef 1, i64 noundef 8, ptr noundef nonnull align 1 dereferenceable(1) @_ZSt7nothrow)
+  call void @_ZdlPvSt11align_val_t(ptr noundef %13, i64 noundef 8)
+
+  %14 = call noalias noundef nonnull ptr @_Znam(i64 noundef 42)
+  call void @_ZdaPv(ptr noundef %14)
+
+  %15 = call noalias noundef nonnull align 8 ptr @_ZnamSt11align_val_t(i64 noundef 42, i64 noundef 8)
+  call void @_ZdaPvSt11align_val_t(ptr noundef %15, i64 noundef 8)
+
+  %16 = call noalias noundef ptr @_ZnamRKSt9nothrow_t(i64 noundef 42, ptr noundef nonnull align 1 dereferenceable(1) @_ZSt7nothrow)
+  call void @_ZdaPv(ptr noundef %16)
+
+  %17 = call noalias noundef align 8 ptr @_ZnamSt11align_val_tRKSt9nothrow_t(i64 noundef 42, i64 noundef 8, ptr noundef nonnull align 1 dereferenceable(1) @_ZSt7nothrow)
+  call void @_ZdaPvSt11align_val_t(ptr noundef %17, i64 noundef 8)
+
+  %18 = call ptr @calloc(i64 noundef 1, i64 noundef 42)
+  call void @free(ptr noundef %18)
+
+  %19 = call ptr @malloc(i64 noundef 42)
+  call void @free(ptr noundef %19)
+
+  %20 = call noalias noundef nonnull ptr @_Znwm(i64 noundef 42)
+  call void @_ZdlPv(ptr noundef %20)
+
+  %21 = call noalias noundef nonnull align 8 ptr @_ZnwmSt11align_val_t(i64 noundef 42, i64 noundef 8)
+  call void @_ZdlPvSt11align_val_t(ptr noundef %21, i64 noundef 8)
+
+  %22 = call noalias noundef ptr @_ZnwmRKSt9nothrow_t(i64 noundef 42, ptr noundef nonnull align 1 dereferenceable(1) @_ZSt7nothrow)
+  call void @_ZdlPv(ptr noundef %22)
+
+  %23 = call noalias noundef align 8 ptr @_ZnwmSt11align_val_tRKSt9nothrow_t(i64 noundef 42, i64 noundef 8, ptr noundef nonnull align 1 dereferenceable(1) @_ZSt7nothrow)
+  call void @_ZdlPvSt11align_val_t(ptr noundef %23, i64 noundef 8)
+
+  %24 = call ptr @malloc(i64 noundef 42)
+  %25 = call ptr @realloc(ptr noundef %24, i64 noundef 41)
+  call void @free(ptr noundef %25)
+
+  %26 = call ptr @__libc_calloc(i64 noundef 1, i64 noundef 42)
+  call void @__libc_free(ptr noundef %26)
+
+  %27 = call ptr @__libc_malloc(i64 noundef 42)
+  call void @__libc_free(ptr noundef %27)
+
+  %28 = call ptr @__libc_memalign(i64 noundef 8, i64 noundef 42)
+  call void @__libc_free(ptr noundef %28)
+
+  ret i32 0
+}
+
+declare noalias ptr @aligned_alloc(i64 noundef, i64 noundef)
+
+declare void @free(ptr noundef)
+
+declare noalias ptr @calloc(i64 noundef, i64 noundef)
+
+declare noalias ptr @malloc(i64 noundef)
+
+declare noalias ptr @memalign(i64 noundef, i64 noundef)
+
+declare i32 @posix_memalign(ptr noundef, i64 noundef, i64 noundef)
+
+declare ptr @realloc(ptr noundef, i64 noundef)
+
+declare ptr @reallocarray(ptr noundef, i64 noundef, i64 noundef)
+
+declare noundef nonnull ptr @_Znwm(i64 noundef)
+
+declare void @_ZdlPv(ptr noundef)
+
+declare noalias noundef nonnull ptr @_ZnwmSt11align_val_t(i64 noundef, i64 noundef)
+
+declare void @_ZdlPvSt11align_val_t(ptr noundef, i64 noundef)
+
+declare noalias noundef ptr @_ZnwmRKSt9nothrow_t(i64 noundef, ptr noundef nonnull align 1 dereferenceable(1))
+
+declare noalias noundef ptr @_ZnwmSt11align_val_tRKSt9nothrow_t(i64 noundef, i64 noundef, ptr noundef nonnull align 1 dereferenceable(1))
+
+declare noundef nonnull ptr @_Znam(i64 noundef)
+
+declare void @_ZdaPv(ptr noundef)
+
+declare noalias noundef nonnull ptr @_ZnamSt11align_val_t(i64 noundef, i64 noundef)
+
+declare void @_ZdaPvSt11align_val_t(ptr noundef, i64 noundef)
+
+declare noalias noundef ptr @_ZnamRKSt9nothrow_t(i64 noundef, ptr noundef nonnull align 1 dereferenceable(1))
+
+declare noalias noundef ptr @_ZnamSt11align_val_tRKSt9nothrow_t(i64 noundef, i64 noundef, ptr noundef nonnull align 1 dereferenceable(1))
+
+declare ptr @__libc_calloc(i64 noundef, i64 noundef)
+
+declare void @__libc_free(ptr noundef)
+
+declare ptr @__libc_malloc(i64 noundef)
+
+declare ptr @__libc_memalign(i64 noundef, i64 noundef)
\ No newline at end of file
diff --git a/llvm/test/Transforms/HipStdPar/unsupported-asm.ll b/llvm/test/Transforms/HipStdPar/unsupported-asm.ll
new file mode 100644
index 0000000000..19e6b28cc9
--- /dev/null
+++ b/llvm/test/Transforms/HipStdPar/unsupported-asm.ll
@@ -0,0 +1,12 @@
+; RUN: not opt -S -mtriple=amdgcn-amd-amdhsa -passes=hipstdpar-select-accelerator-code \
+; RUN: %s 2>&1 | FileCheck %s
+
+; CHECK: error: {{.*}} in function foo void (): Accelerator does not support the ASM block:
+; CHECK-NEXT: {{.*}}Invalid ASM block{{.*}}
+define amdgpu_kernel void @foo() {
+entry:
+  call void @__ASM__hipstdpar_unsupported([18 x i8] c"Invalid ASM block\00")
+  ret void
+}
+
+declare void @__ASM__hipstdpar_unsupported([18 x i8])
\ No newline at end of file
diff --git a/llvm/test/Transforms/HipStdPar/unsupported-builtins.ll b/llvm/test/Transforms/HipStdPar/unsupported-builtins.ll
new file mode 100644
index 0000000000..19490dad80
--- /dev/null
+++ b/llvm/test/Transforms/HipStdPar/unsupported-builtins.ll
@@ -0,0 +1,11 @@
+; RUN: not opt -S -mtriple=amdgcn-amd-amdhsa -passes=hipstdpar-select-accelerator-code \
+; RUN: %s 2>&1 | FileCheck %s
+
+; CHECK: error: {{.*}} in function foo void (): Accelerator does not support the __builtin_ia32_pause function
+define amdgpu_kernel void @foo() {
+entry:
+  call void @__builtin_ia32_pause__hipstdpar_unsupported()
+  ret void
+}
+
+declare void @__builtin_ia32_pause__hipstdpar_unsupported()
\ No newline at end of file
diff --git a/llvm/test/Transforms/HipStdPar/unsupported-thread-local-direct-use.ll b/llvm/test/Transforms/HipStdPar/unsupported-thread-local-direct-use.ll
new file mode 100644
index 0000000000..fa17bb114e
--- /dev/null
+++ b/llvm/test/Transforms/HipStdPar/unsupported-thread-local-direct-use.ll
@@ -0,0 +1,14 @@
+; RUN: not opt -S -mtriple=amdgcn-amd-amdhsa -passes=hipstdpar-select-accelerator-code \
+; RUN:   %s 2>&1 | FileCheck %s
+
+@tls = hidden thread_local addrspace(1) global i32 0, align 4
+
+; CHECK: error: {{.*}} in function direct_use void (): Accelerator does not support the thread_local variable tls
+define amdgpu_kernel void @direct_use() {
+entry:
+  %0 = call align 4 ptr addrspace(1) @llvm.threadlocal.address.p1(ptr addrspace(1) @tls)
+  %1 = load i32, ptr addrspace(1) %0, align 4
+  ret void
+}
+
+declare nonnull ptr addrspace(1) @llvm.threadlocal.address.p1(ptr addrspace(1) nonnull)
diff --git a/llvm/test/Transforms/HipStdPar/unsupported-thread-local-indirect-use.ll b/llvm/test/Transforms/HipStdPar/unsupported-thread-local-indirect-use.ll
new file mode 100644
index 0000000000..40014853d8
--- /dev/null
+++ b/llvm/test/Transforms/HipStdPar/unsupported-thread-local-indirect-use.ll
@@ -0,0 +1,14 @@
+; RUN: not opt -S -mtriple=amdgcn-amd-amdhsa -passes=hipstdpar-select-accelerator-code \
+; RUN:   %s 2>&1 | FileCheck %s
+
+@tls = hidden thread_local addrspace(1) global i32 0, align 4
+
+; CHECK: error: {{.*}} in function indirect_use void (): Accelerator does not support the thread_local variable tls
+define amdgpu_kernel void @indirect_use() {
+entry:
+  %0 = call align 4 ptr @llvm.threadlocal.address.p0(ptr addrspacecast (ptr addrspace(1) @tls to ptr))
+  %1 = load i32, ptr %0, align 4
+  ret void
+}
+
+declare nonnull ptr @llvm.threadlocal.address.p0(ptr nonnull)
diff --git a/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-sel.ll b/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-sel.ll
index b0f059c9de..c6f08ce828 100644
--- a/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-sel.ll
+++ b/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-sel.ll
@@ -12,6 +12,16 @@ define <vscale x 4 x i32> @replace_sel_intrinsic(<vscale x 4 x i1> %p, <vscale x
   ret <vscale x 4 x i32> %1
 }
 
+define <vscale x 4 x i32> @sel_ptrue(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: @sel_ptrue(
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[A:%.*]]
+;
+  %pred = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
+  %res = call <vscale x 4 x i32> @llvm.aarch64.sve.sel.nxv4i32(<vscale x 4 x i1> %pred, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %res
+}
+
+declare <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32)
 declare <vscale x 4 x i32> @llvm.aarch64.sve.sel.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
 
 attributes #0 = { "target-features"="+sve" }
diff --git a/llvm/test/Transforms/InstCombine/and-add-shl.ll b/llvm/test/Transforms/InstCombine/and-add-shl.ll
index 28778f3413..92b3a8144d 100644
--- a/llvm/test/Transforms/InstCombine/and-add-shl.ll
+++ b/llvm/test/Transforms/InstCombine/and-add-shl.ll
@@ -29,7 +29,7 @@ define i8 @and_not_shl(i8 %x) {
 ; CHECK-SAME: (i8 [[X:%.*]]) {
 ; CHECK-NEXT:    [[OP1_P2:%.*]] = icmp ult i8 [[X]], 6
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[OP1_P2]])
-; CHECK-NEXT:    [[SHIFT:%.*]] = shl i8 -1, [[X]]
+; CHECK-NEXT:    [[SHIFT:%.*]] = shl nsw i8 -1, [[X]]
 ; CHECK-NEXT:    [[NOT:%.*]] = and i8 [[SHIFT]], 32
 ; CHECK-NEXT:    [[R:%.*]] = xor i8 [[NOT]], 32
 ; CHECK-NEXT:    ret i8 [[R]]
diff --git a/llvm/test/Transforms/InstCombine/and.ll b/llvm/test/Transforms/InstCombine/and.ll
index 48e21a581b..95b1b0e73e 100644
--- a/llvm/test/Transforms/InstCombine/and.ll
+++ b/llvm/test/Transforms/InstCombine/and.ll
@@ -1595,8 +1595,8 @@ define <2 x i8> @flip_masked_bit_uniform(<2 x i8> %A) {
 
 define <2 x i8> @flip_masked_bit_undef(<2 x i8> %A) {
 ; CHECK-LABEL: @flip_masked_bit_undef(
-; CHECK-NEXT:    [[B:%.*]] = add <2 x i8> [[A:%.*]], <i8 16, i8 undef>
-; CHECK-NEXT:    [[C:%.*]] = and <2 x i8> [[B]], <i8 16, i8 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i8> [[A:%.*]], <i8 -1, i8 -1>
+; CHECK-NEXT:    [[C:%.*]] = and <2 x i8> [[TMP1]], <i8 16, i8 undef>
 ; CHECK-NEXT:    ret <2 x i8> [[C]]
 ;
   %B = add <2 x i8> %A, <i8 16, i8 undef>
@@ -1606,8 +1606,8 @@ define <2 x i8> @flip_masked_bit_undef(<2 x i8> %A) {
 
 define <2 x i8> @flip_masked_bit_nonuniform(<2 x i8> %A) {
 ; CHECK-LABEL: @flip_masked_bit_nonuniform(
-; CHECK-NEXT:    [[B:%.*]] = add <2 x i8> [[A:%.*]], <i8 16, i8 4>
-; CHECK-NEXT:    [[C:%.*]] = and <2 x i8> [[B]], <i8 16, i8 4>
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i8> [[A:%.*]], <i8 -1, i8 -1>
+; CHECK-NEXT:    [[C:%.*]] = and <2 x i8> [[TMP1]], <i8 16, i8 4>
 ; CHECK-NEXT:    ret <2 x i8> [[C]]
 ;
   %B = add <2 x i8> %A, <i8 16, i8 4>
@@ -2553,8 +2553,8 @@ define i32 @canonicalize_and_add_power2_or_zero(i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[P2:%.*]] = and i32 [[NY]], [[Y]]
 ; CHECK-NEXT:    call void @use32(i32 [[P2]])
 ; CHECK-NEXT:    [[X2:%.*]] = mul i32 [[X:%.*]], [[X]]
-; CHECK-NEXT:    [[VAL:%.*]] = add i32 [[X2]], [[P2]]
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[VAL]], [[P2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[X2]], -1
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[P2]], [[TMP1]]
 ; CHECK-NEXT:    ret i32 [[AND]]
 ;
   %ny = sub i32 0, %y
@@ -2572,8 +2572,8 @@ define i32 @canonicalize_and_sub_power2_or_zero(i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[NY:%.*]] = sub i32 0, [[Y:%.*]]
 ; CHECK-NEXT:    [[P2:%.*]] = and i32 [[NY]], [[Y]]
 ; CHECK-NEXT:    call void @use32(i32 [[P2]])
-; CHECK-NEXT:    [[VAL:%.*]] = sub i32 [[X:%.*]], [[P2]]
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[VAL]], [[P2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[P2]], [[TMP1]]
 ; CHECK-NEXT:    ret i32 [[AND]]
 ;
   %ny = sub i32 0, %y
@@ -2590,8 +2590,8 @@ define i32 @canonicalize_and_add_power2_or_zero_commuted1(i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[NY:%.*]] = sub i32 0, [[Y:%.*]]
 ; CHECK-NEXT:    [[P2:%.*]] = and i32 [[NY]], [[Y]]
 ; CHECK-NEXT:    call void @use32(i32 [[P2]])
-; CHECK-NEXT:    [[VAL:%.*]] = add i32 [[P2]], [[X:%.*]]
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[VAL]], [[P2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[P2]], [[TMP1]]
 ; CHECK-NEXT:    ret i32 [[AND]]
 ;
   %ny = sub i32 0, %y
@@ -2609,8 +2609,8 @@ define i32 @canonicalize_and_add_power2_or_zero_commuted2(i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[P2:%.*]] = and i32 [[NY]], [[Y]]
 ; CHECK-NEXT:    call void @use32(i32 [[P2]])
 ; CHECK-NEXT:    [[X2:%.*]] = mul i32 [[X:%.*]], [[X]]
-; CHECK-NEXT:    [[VAL:%.*]] = add i32 [[X2]], [[P2]]
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[P2]], [[VAL]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[X2]], -1
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[P2]], [[TMP1]]
 ; CHECK-NEXT:    ret i32 [[AND]]
 ;
   %ny = sub i32 0, %y
@@ -2628,8 +2628,8 @@ define i32 @canonicalize_and_add_power2_or_zero_commuted3(i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[NY:%.*]] = sub i32 0, [[Y:%.*]]
 ; CHECK-NEXT:    [[P2:%.*]] = and i32 [[NY]], [[Y]]
 ; CHECK-NEXT:    call void @use32(i32 [[P2]])
-; CHECK-NEXT:    [[VAL:%.*]] = add i32 [[P2]], [[X:%.*]]
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[P2]], [[VAL]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[P2]], [[TMP1]]
 ; CHECK-NEXT:    ret i32 [[AND]]
 ;
   %ny = sub i32 0, %y
diff --git a/llvm/test/Transforms/InstCombine/dbg-scalable-store-fixed-frag.ll b/llvm/test/Transforms/InstCombine/dbg-scalable-store-fixed-frag.ll
index 8743ac6f14..a8a7ee4608 100644
--- a/llvm/test/Transforms/InstCombine/dbg-scalable-store-fixed-frag.ll
+++ b/llvm/test/Transforms/InstCombine/dbg-scalable-store-fixed-frag.ll
@@ -4,14 +4,14 @@
 define i32 @foo(<vscale x 2 x i32> %x) {
 ; CHECK-LABEL: @foo(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[ARR:%.*]] = alloca [4 x i32], align 4
-; CHECK-NEXT:    call void @llvm.dbg.declare(metadata ptr [[ARR]], metadata [[META8:![0-9]+]], metadata !DIExpression()), !dbg [[DBG14:![0-9]+]]
+; CHECK-NEXT:    [[ARR:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata <vscale x 2 x i32> undef, metadata [[META8:![0-9]+]], metadata !DIExpression()), !dbg [[DBG14:![0-9]+]]
 ; CHECK-NEXT:    store <vscale x 2 x i32> [[X:%.*]], ptr [[ARR]], align 4
 ; CHECK-NEXT:    [[RES:%.*]] = load i32, ptr [[ARR]], align 4
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
 entry:
-  %arr = alloca [4 x i32], align 4
+  %arr = alloca i32, align 4
   call void @llvm.dbg.declare(metadata ptr %arr, metadata !8, metadata !DIExpression()), !dbg !14
   store <vscale x 2 x i32> %x, ptr %arr, align 4
   %res = load i32, ptr %arr
diff --git a/llvm/test/Transforms/InstCombine/dbg-simplify-alloca-size.ll b/llvm/test/Transforms/InstCombine/dbg-simplify-alloca-size.ll
index 6a807ba378..028b19fadf 100644
--- a/llvm/test/Transforms/InstCombine/dbg-simplify-alloca-size.ll
+++ b/llvm/test/Transforms/InstCombine/dbg-simplify-alloca-size.ll
@@ -7,13 +7,13 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata)
 
 ; CHECK-LABEL: @toplevel(
 ; CHECK:  entry:
-; CHECK-NEXT:    %pixels1 = alloca [4 x i8], align 1
+; CHECK-NEXT:    %pixels1 = alloca [3 x i8], align 1
 ; CHECK-NEXT:    call void @llvm.dbg.declare(metadata ptr %pixels1, metadata ![[MD:[0-9]+]], metadata !DIExpression()), !dbg ![[DBG:[0-9]+]]
 ; CHECK-NEXT:    call void @foo(ptr nonnull %pixels1)
 ; CHECK-NEXT:    ret void
 define dso_local void @toplevel() {
 entry:
-  %pixels = alloca i8, i32 4
+  %pixels = alloca i8, i32 3
   call void @llvm.dbg.declare(metadata ptr %pixels, metadata !11, metadata !DIExpression()), !dbg !12
   call void @foo(ptr %pixels)
   ret void
diff --git a/llvm/test/Transforms/InstCombine/div-shift.ll b/llvm/test/Transforms/InstCombine/div-shift.ll
index 76c5328dc8..d208837f04 100644
--- a/llvm/test/Transforms/InstCombine/div-shift.ll
+++ b/llvm/test/Transforms/InstCombine/div-shift.ll
@@ -2,6 +2,7 @@
 ; RUN: opt < %s -passes=instcombine -S | FileCheck %s
 
 declare void @use(i8)
+declare void @use32(i32)
 
 declare i8 @llvm.umin.i8(i8, i8)
 declare i8 @llvm.umax.i8(i8, i8)
@@ -1025,3 +1026,271 @@ define i8 @udiv_shl_no_overflow(i8 %x, i8 %y) {
   %mul = udiv i8 %x, %min
   ret i8 %mul
 }
+
+; (X<<Y) / (X<<Z) -> 1 << Y >> Z
+
+define i32 @sdiv_shl_pair_const(i32 %a) {
+; CHECK-LABEL: @sdiv_shl_pair_const(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i32 2
+;
+entry:
+  %lhs = shl nsw i32 %a, 2
+  %rhs = shl nsw i32 %a, 1
+  %div = sdiv i32 %lhs, %rhs
+  ret i32 %div
+}
+
+define i32 @udiv_shl_pair_const(i32 %a) {
+; CHECK-LABEL: @udiv_shl_pair_const(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i32 2
+;
+entry:
+  %lhs = shl nuw i32 %a, 2
+  %rhs = shl nuw i32 %a, 1
+  %div = udiv i32 %lhs, %rhs
+  ret i32 %div
+}
+
+define i32 @sdiv_shl_pair1(i32 %a, i32 %x, i32 %y) {
+; CHECK-LABEL: @sdiv_shl_pair1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SHL_DIVIDEND:%.*]] = shl nuw nsw i32 1, [[X:%.*]]
+; CHECK-NEXT:    [[DIV:%.*]] = lshr i32 [[SHL_DIVIDEND]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+entry:
+  %lhs = shl nsw i32 %a, %x
+  %rhs = shl nuw nsw i32 %a, %y
+  %div = sdiv i32 %lhs, %rhs
+  ret i32 %div
+}
+
+define i32 @sdiv_shl_pair2(i32 %a, i32 %x, i32 %y) {
+; CHECK-LABEL: @sdiv_shl_pair2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SHL_DIVIDEND:%.*]] = shl nuw nsw i32 1, [[X:%.*]]
+; CHECK-NEXT:    [[DIV:%.*]] = lshr i32 [[SHL_DIVIDEND]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+entry:
+  %lhs = shl nuw nsw i32 %a, %x
+  %rhs = shl nsw i32 %a, %y
+  %div = sdiv i32 %lhs, %rhs
+  ret i32 %div
+}
+
+define i32 @sdiv_shl_pair3(i32 %a, i32 %x, i32 %y) {
+; CHECK-LABEL: @sdiv_shl_pair3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SHL_DIVIDEND:%.*]] = shl nuw i32 1, [[X:%.*]]
+; CHECK-NEXT:    [[DIV:%.*]] = lshr i32 [[SHL_DIVIDEND]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+entry:
+  %lhs = shl nsw i32 %a, %x
+  %rhs = shl nsw i32 %a, %y
+  %div = sdiv i32 %lhs, %rhs
+  ret i32 %div
+}
+
+define i32 @sdiv_shl_no_pair_fail(i32 %a, i32 %b, i32 %x, i32 %y) {
+; CHECK-LABEL: @sdiv_shl_no_pair_fail(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LHS:%.*]] = shl nuw nsw i32 [[A:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[RHS:%.*]] = shl nuw i32 [[B:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 [[LHS]], [[RHS]]
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+entry:
+  %lhs = shl nuw nsw i32 %a, %x
+  %rhs = shl nuw i32 %b, %y
+  %div = sdiv i32 %lhs, %rhs
+  ret i32 %div
+}
+
+define i32 @udiv_shl_pair1(i32 %a, i32 %x, i32 %y) {
+; CHECK-LABEL: @udiv_shl_pair1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SHL_DIVIDEND:%.*]] = shl nuw i32 1, [[X:%.*]]
+; CHECK-NEXT:    [[DIV:%.*]] = lshr i32 [[SHL_DIVIDEND]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+entry:
+  %lhs = shl nuw i32 %a, %x
+  %rhs = shl nuw i32 %a, %y
+  %div = udiv i32 %lhs, %rhs
+  ret i32 %div
+}
+
+define i32 @udiv_shl_pair2(i32 %a, i32 %x, i32 %y) {
+; CHECK-LABEL: @udiv_shl_pair2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SHL_DIVIDEND:%.*]] = shl nuw nsw i32 1, [[X:%.*]]
+; CHECK-NEXT:    [[DIV:%.*]] = lshr i32 [[SHL_DIVIDEND]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+entry:
+  %lhs = shl nuw nsw i32 %a, %x
+  %rhs = shl nuw i32 %a, %y
+  %div = udiv i32 %lhs, %rhs
+  ret i32 %div
+}
+
+define i32 @udiv_shl_pair3(i32 %a, i32 %x, i32 %y) {
+; CHECK-LABEL: @udiv_shl_pair3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SHL_DIVIDEND:%.*]] = shl nuw i32 1, [[X:%.*]]
+; CHECK-NEXT:    [[DIV:%.*]] = lshr i32 [[SHL_DIVIDEND]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+entry:
+  %lhs = shl nuw i32 %a, %x
+  %rhs = shl nuw nsw i32 %a, %y
+  %div = udiv i32 %lhs, %rhs
+  ret i32 %div
+}
+
+define i32 @sdiv_shl_pair_overflow_fail1(i32 %a, i32 %x, i32 %y) {
+; CHECK-LABEL: @sdiv_shl_pair_overflow_fail1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LHS:%.*]] = shl i32 [[A:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[RHS:%.*]] = shl nsw i32 [[A]], [[Y:%.*]]
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 [[LHS]], [[RHS]]
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+entry:
+  %lhs = shl i32 %a, %x
+  %rhs = shl nsw i32 %a, %y
+  %div = sdiv i32 %lhs, %rhs
+  ret i32 %div
+}
+
+define i32 @sdiv_shl_pair_overflow_fail2(i32 %a, i32 %x, i32 %y) {
+; CHECK-LABEL: @sdiv_shl_pair_overflow_fail2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LHS:%.*]] = shl nsw i32 [[A:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[RHS:%.*]] = shl nuw i32 [[A]], [[Y:%.*]]
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 [[LHS]], [[RHS]]
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+entry:
+  %lhs = shl nsw i32 %a, %x
+  %rhs = shl nuw i32 %a, %y
+  %div = sdiv i32 %lhs, %rhs
+  ret i32 %div
+}
+
+define i32 @udiv_shl_pair_overflow_fail1(i32 %a, i32 %x, i32 %y) {
+; CHECK-LABEL: @udiv_shl_pair_overflow_fail1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LHS:%.*]] = shl nsw i32 [[A:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[RHS:%.*]] = shl nuw i32 [[A]], [[Y:%.*]]
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i32 [[LHS]], [[RHS]]
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+entry:
+  %lhs = shl nsw i32 %a, %x
+  %rhs = shl nuw i32 %a, %y
+  %div = udiv i32 %lhs, %rhs
+  ret i32 %div
+}
+
+define i32 @udiv_shl_pair_overflow_fail2(i32 %a, i32 %x, i32 %y) {
+; CHECK-LABEL: @udiv_shl_pair_overflow_fail2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LHS:%.*]] = shl nsw i32 [[A:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[RHS:%.*]] = shl i32 [[A]], [[Y:%.*]]
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i32 [[LHS]], [[RHS]]
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+entry:
+  %lhs = shl nsw i32 %a, %x
+  %rhs = shl i32 %a, %y
+  %div = udiv i32 %lhs, %rhs
+  ret i32 %div
+}
+
+define i32 @udiv_shl_pair_overflow_fail3(i32 %a, i32 %x, i32 %y) {
+; CHECK-LABEL: @udiv_shl_pair_overflow_fail3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LHS:%.*]] = shl nuw nsw i32 [[A:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[RHS:%.*]] = shl i32 [[A]], [[Y:%.*]]
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i32 [[LHS]], [[RHS]]
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+entry:
+  %lhs = shl nuw nsw i32 %a, %x
+  %rhs = shl i32 %a, %y
+  %div = udiv i32 %lhs, %rhs
+  ret i32 %div
+}
+
+define i32 @sdiv_shl_pair_multiuse1(i32 %a, i32 %x, i32 %y) {
+; CHECK-LABEL: @sdiv_shl_pair_multiuse1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LHS:%.*]] = shl nuw nsw i32 [[A:%.*]], [[X:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[LHS]])
+; CHECK-NEXT:    [[SHL_DIVIDEND:%.*]] = shl nuw nsw i32 1, [[X]]
+; CHECK-NEXT:    [[DIV:%.*]] = lshr i32 [[SHL_DIVIDEND]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+entry:
+  %lhs = shl nuw nsw i32 %a, %x
+  call void @use32(i32 %lhs)
+  %rhs = shl nsw i32 %a, %y
+  %div = sdiv i32 %lhs, %rhs
+  ret i32 %div
+}
+
+define i32 @sdiv_shl_pair_multiuse2(i32 %a, i32 %x, i32 %y) {
+; CHECK-LABEL: @sdiv_shl_pair_multiuse2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[RHS:%.*]] = shl nsw i32 [[A:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[RHS]])
+; CHECK-NEXT:    [[SHL_DIVIDEND:%.*]] = shl nuw nsw i32 1, [[X:%.*]]
+; CHECK-NEXT:    [[DIV:%.*]] = lshr i32 [[SHL_DIVIDEND]], [[Y]]
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+entry:
+  %lhs = shl nuw nsw i32 %a, %x
+  %rhs = shl nsw i32 %a, %y
+  call void @use32(i32 %rhs)
+  %div = sdiv i32 %lhs, %rhs
+  ret i32 %div
+}
+
+define i32 @sdiv_shl_pair_multiuse3(i32 %a, i32 %x, i32 %y) {
+; CHECK-LABEL: @sdiv_shl_pair_multiuse3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LHS:%.*]] = shl nuw nsw i32 [[A:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[RHS:%.*]] = shl nsw i32 [[A]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[LHS]])
+; CHECK-NEXT:    call void @use32(i32 [[RHS]])
+; CHECK-NEXT:    [[SHL_DIVIDEND:%.*]] = shl nuw nsw i32 1, [[X]]
+; CHECK-NEXT:    [[DIV:%.*]] = lshr i32 [[SHL_DIVIDEND]], [[Y]]
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+entry:
+  %lhs = shl nuw nsw i32 %a, %x
+  %rhs = shl nsw i32 %a, %y
+  call void @use32(i32 %lhs)
+  call void @use32(i32 %rhs)
+  %div = sdiv i32 %lhs, %rhs
+  ret i32 %div
+}
+
+@a = external global i32
+define i32 @pr69291() {
+; CHECK-LABEL: @pr69291(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i32 1
+;
+entry:
+  %conv = load i32, ptr @a, align 1
+  %add = shl nuw nsw i32 %conv, 1
+  %add2 = shl nuw nsw i32 %conv, 1
+  %div = sdiv i32 %add, %add2
+  ret i32 %div
+}
diff --git a/llvm/test/Transforms/InstCombine/icmp-add.ll b/llvm/test/Transforms/InstCombine/icmp-add.ll
index 832db40fda..b99ed20d7d 100644
--- a/llvm/test/Transforms/InstCombine/icmp-add.ll
+++ b/llvm/test/Transforms/InstCombine/icmp-add.ll
@@ -515,6 +515,19 @@ bb:
   ret i1 %t
 }
 
+define <2 x i1> @cvt_icmp_2_sext_plus_zext_ne_vec(<2 x i1> %arg, <2 x i1> %arg1) {
+; CHECK-LABEL: @cvt_icmp_2_sext_plus_zext_ne_vec(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+bb:
+  %i = sext <2 x i1> %arg to <2 x i32>
+  %i2 = zext <2 x i1> %arg1 to <2 x i32>
+  %i3 = add nsw <2 x i32> %i, %i2
+  %i4 = icmp ne <2 x i32> %i3, <i32 2, i32 2>
+  ret <2 x i1> %i4
+}
+
 ; test if zext i1 X + sext i1 Y converted to sext i1 X + zext i1 Y
 ; and then processed
 
@@ -531,6 +544,19 @@ bb:
   ret i1 %t
 }
 
+define <2 x i1> @cvt_icmp_neg_2_zext_plus_sext_eq_vec(<2 x i1> %arg, <2 x i1> %arg1) {
+; CHECK-LABEL: @cvt_icmp_neg_2_zext_plus_sext_eq_vec(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+bb:
+  %i = zext <2 x i1> %arg to <2 x i32>
+  %i2 = sext <2 x i1> %arg1 to <2 x i32>
+  %i3 = add nsw <2 x i32> %i2, %i
+  %i4 = icmp eq <2 x i32> %i3, <i32 2, i32 2>
+  ret <2 x i1> %i4
+}
+
 define i1 @cvt_icmp_neg_1_zext_plus_sext_eq(i1 %arg, i1 %arg1) {
 ; CHECK-LABEL: @cvt_icmp_neg_1_zext_plus_sext_eq(
 ; CHECK-NEXT:  bb:
diff --git a/llvm/test/Transforms/InstCombine/icmp-shr-lt-gt.ll b/llvm/test/Transforms/InstCombine/icmp-shr-lt-gt.ll
index 33c9b59551..08a763a50b 100644
--- a/llvm/test/Transforms/InstCombine/icmp-shr-lt-gt.ll
+++ b/llvm/test/Transforms/InstCombine/icmp-shr-lt-gt.ll
@@ -897,6 +897,19 @@ define i1 @ashrsgt_01_00(i4 %x) {
   ret i1 %c
 }
 
+define i1 @ashrsgt_01_00_multiuse(i4 %x, ptr %p) {
+; CHECK-LABEL: @ashrsgt_01_00_multiuse(
+; CHECK-NEXT:    [[S:%.*]] = ashr i4 [[X:%.*]], 1
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 [[X]], 1
+; CHECK-NEXT:    store i4 [[S]], ptr [[P:%.*]], align 1
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr i4 %x, 1
+  %c = icmp sgt i4 %s, 0
+  store i4 %s, ptr %p
+  ret i1 %c
+}
+
 define i1 @ashrsgt_01_01(i4 %x) {
 ; CHECK-LABEL: @ashrsgt_01_01(
 ; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 [[X:%.*]], 3
diff --git a/llvm/test/Transforms/InstCombine/icmp-shr.ll b/llvm/test/Transforms/InstCombine/icmp-shr.ll
index f4dfa2edfa..253b38c8bf 100644
--- a/llvm/test/Transforms/InstCombine/icmp-shr.ll
+++ b/llvm/test/Transforms/InstCombine/icmp-shr.ll
@@ -576,6 +576,19 @@ define i1 @ashr_ugt_0(i4 %x) {
   ret i1 %r
 }
 
+define i1 @ashr_ugt_0_multiuse(i4 %x, ptr %p) {
+; CHECK-LABEL: @ashr_ugt_0_multiuse(
+; CHECK-NEXT:    [[S:%.*]] = ashr i4 [[X:%.*]], 1
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i4 [[X]], 1
+; CHECK-NEXT:    store i4 [[S]], ptr [[P:%.*]], align 1
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %s = ashr i4 %x, 1
+  %r = icmp ugt i4 %s, 0 ; 0b0000
+  store i4 %s, ptr %p
+  ret i1 %r
+}
+
 define i1 @ashr_ugt_1(i4 %x) {
 ; CHECK-LABEL: @ashr_ugt_1(
 ; CHECK-NEXT:    [[R:%.*]] = icmp ugt i4 [[X:%.*]], 3
@@ -764,6 +777,19 @@ define i1 @ashr_ult_2(i4 %x) {
   ret i1 %r
 }
 
+define i1 @ashr_ult_2_multiuse(i4 %x, ptr %p) {
+; CHECK-LABEL: @ashr_ult_2_multiuse(
+; CHECK-NEXT:    [[S:%.*]] = ashr i4 [[X:%.*]], 1
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i4 [[X]], 4
+; CHECK-NEXT:    store i4 [[S]], ptr [[P:%.*]], align 1
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %s = ashr i4 %x, 1
+  %r = icmp ult i4 %s, 2 ; 0b0010
+  store i4 %s, ptr %p
+  ret i1 %r
+}
+
 ; negative test
 
 define i1 @ashr_ult_3(i4 %x) {
@@ -1302,9 +1328,9 @@ define i1 @lshr_neg_sgt_zero(i8 %x) {
 
 define i1 @exactly_one_set_signbit(i8 %x, i8 %y) {
 ; CHECK-LABEL: @exactly_one_set_signbit(
-; CHECK-NEXT:    [[XOR_SIGNBITS:%.*]] = xor i8 [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = icmp slt i8 [[XOR_SIGNBITS]], 0
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i8 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
   %xsign = lshr i8 %x, 7
   %ypos = icmp sgt i8 %y, -1
@@ -1317,9 +1343,9 @@ define i1 @exactly_one_set_signbit_use1(i8 %x, i8 %y) {
 ; CHECK-LABEL: @exactly_one_set_signbit_use1(
 ; CHECK-NEXT:    [[XSIGN:%.*]] = lshr i8 [[X:%.*]], 7
 ; CHECK-NEXT:    call void @use(i8 [[XSIGN]])
-; CHECK-NEXT:    [[XOR_SIGNBITS:%.*]] = xor i8 [[X]], [[Y:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = icmp slt i8 [[XOR_SIGNBITS]], 0
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i8 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
   %xsign = lshr i8 %x, 7
   call void @use(i8 %xsign)
@@ -1331,9 +1357,9 @@ define i1 @exactly_one_set_signbit_use1(i8 %x, i8 %y) {
 
 define <2 x i1> @same_signbit(<2 x i8> %x, <2 x i8> %y) {
 ; CHECK-LABEL: @same_signbit(
-; CHECK-NEXT:    [[XOR_SIGNBITS:%.*]] = xor <2 x i8> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = icmp sgt <2 x i8> [[XOR_SIGNBITS]], <i8 -1, i8 -1>
-; CHECK-NEXT:    ret <2 x i1> [[R]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i8> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R1:%.*]] = icmp sgt <2 x i8> [[TMP1]], <i8 -1, i8 -1>
+; CHECK-NEXT:    ret <2 x i1> [[R1]]
 ;
   %xsign = lshr <2 x i8> %x, <i8 7, i8 7>
   %ypos = icmp sgt <2 x i8> %y, <i8 -1, i8 -1>
@@ -1347,9 +1373,9 @@ define i1 @same_signbit_use2(i8 %x, i8 %y) {
 ; CHECK-NEXT:    [[YPOS:%.*]] = icmp sgt i8 [[Y:%.*]], -1
 ; CHECK-NEXT:    [[YPOSZ:%.*]] = zext i1 [[YPOS]] to i8
 ; CHECK-NEXT:    call void @use(i8 [[YPOSZ]])
-; CHECK-NEXT:    [[XOR_SIGNBITS:%.*]] = xor i8 [[X:%.*]], [[Y]]
-; CHECK-NEXT:    [[R:%.*]] = icmp sgt i8 [[XOR_SIGNBITS]], -1
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[X:%.*]], [[Y]]
+; CHECK-NEXT:    [[R1:%.*]] = icmp sgt i8 [[TMP1]], -1
+; CHECK-NEXT:    ret i1 [[R1]]
 ;
   %xsign = lshr i8 %x, 7
   %ypos = icmp sgt i8 %y, -1
@@ -1382,9 +1408,10 @@ define i1 @same_signbit_use3(i8 %x, i8 %y) {
 
 define <2 x i1> @same_signbit_poison_elts(<2 x i8> %x, <2 x i8> %y) {
 ; CHECK-LABEL: @same_signbit_poison_elts(
-; CHECK-NEXT:    [[XOR_SIGNBITS:%.*]] = xor <2 x i8> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = icmp sgt <2 x i8> [[XOR_SIGNBITS]], <i8 -1, i8 -1>
-; CHECK-NEXT:    ret <2 x i1> [[R]]
+; CHECK-NEXT:    [[YPOS:%.*]] = icmp sgt <2 x i8> [[Y:%.*]], <i8 -1, i8 poison>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <2 x i8> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    [[R1:%.*]] = xor <2 x i1> [[TMP1]], [[YPOS]]
+; CHECK-NEXT:    ret <2 x i1> [[R1]]
 ;
   %xsign = lshr <2 x i8> %x, <i8 7, i8 poison>
   %ypos = icmp sgt <2 x i8> %y, <i8 -1, i8 poison>
@@ -1397,11 +1424,10 @@ define <2 x i1> @same_signbit_poison_elts(<2 x i8> %x, <2 x i8> %y) {
 
 define i1 @same_signbit_wrong_type(i8 %x, i32 %y) {
 ; CHECK-LABEL: @same_signbit_wrong_type(
-; CHECK-NEXT:    [[XSIGN:%.*]] = lshr i8 [[X:%.*]], 7
 ; CHECK-NEXT:    [[YPOS:%.*]] = icmp sgt i32 [[Y:%.*]], -1
-; CHECK-NEXT:    [[YPOSZ:%.*]] = zext i1 [[YPOS]] to i8
-; CHECK-NEXT:    [[R:%.*]] = icmp ne i8 [[XSIGN]], [[YPOSZ]]
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i8 [[X:%.*]], 0
+; CHECK-NEXT:    [[R1:%.*]] = xor i1 [[TMP1]], [[YPOS]]
+; CHECK-NEXT:    ret i1 [[R1]]
 ;
   %xsign = lshr i8 %x, 7
   %ypos = icmp sgt i32 %y, -1
@@ -1450,11 +1476,9 @@ define i1 @exactly_one_set_signbit_wrong_shr(i8 %x, i8 %y) {
 
 define i1 @exactly_one_set_signbit_wrong_pred(i8 %x, i8 %y) {
 ; CHECK-LABEL: @exactly_one_set_signbit_wrong_pred(
-; CHECK-NEXT:    [[XSIGN:%.*]] = lshr i8 [[X:%.*]], 7
-; CHECK-NEXT:    [[YPOS:%.*]] = icmp sgt i8 [[Y:%.*]], -1
-; CHECK-NEXT:    [[YPOSZ:%.*]] = zext i1 [[YPOS]] to i8
-; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[XSIGN]], [[YPOSZ]]
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[R1:%.*]] = icmp slt i8 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[R1]]
 ;
   %xsign = lshr i8 %x, 7
   %ypos = icmp sgt i8 %y, -1
@@ -1465,9 +1489,9 @@ define i1 @exactly_one_set_signbit_wrong_pred(i8 %x, i8 %y) {
 
 define i1 @exactly_one_set_signbit_signed(i8 %x, i8 %y) {
 ; CHECK-LABEL: @exactly_one_set_signbit_signed(
-; CHECK-NEXT:    [[XOR_SIGNBITS:%.*]] = xor i8 [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = icmp slt i8 [[XOR_SIGNBITS]], 0
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i8 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
   %xsign = ashr i8 %x, 7
   %ypos = icmp sgt i8 %y, -1
@@ -1480,9 +1504,9 @@ define i1 @exactly_one_set_signbit_use1_signed(i8 %x, i8 %y) {
 ; CHECK-LABEL: @exactly_one_set_signbit_use1_signed(
 ; CHECK-NEXT:    [[XSIGN:%.*]] = ashr i8 [[X:%.*]], 7
 ; CHECK-NEXT:    call void @use(i8 [[XSIGN]])
-; CHECK-NEXT:    [[XOR_SIGNBITS:%.*]] = xor i8 [[X]], [[Y:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = icmp slt i8 [[XOR_SIGNBITS]], 0
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i8 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
   %xsign = ashr i8 %x, 7
   call void @use(i8 %xsign)
@@ -1494,9 +1518,9 @@ define i1 @exactly_one_set_signbit_use1_signed(i8 %x, i8 %y) {
 
 define <2 x i1> @same_signbit_signed(<2 x i8> %x, <2 x i8> %y) {
 ; CHECK-LABEL: @same_signbit_signed(
-; CHECK-NEXT:    [[XOR_SIGNBITS:%.*]] = xor <2 x i8> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = icmp sgt <2 x i8> [[XOR_SIGNBITS]], <i8 -1, i8 -1>
-; CHECK-NEXT:    ret <2 x i1> [[R]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i8> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R1:%.*]] = icmp sgt <2 x i8> [[TMP1]], <i8 -1, i8 -1>
+; CHECK-NEXT:    ret <2 x i1> [[R1]]
 ;
   %xsign = ashr <2 x i8> %x, <i8 7, i8 7>
   %ypos = icmp sgt <2 x i8> %y, <i8 -1, i8 -1>
@@ -1510,9 +1534,9 @@ define i1 @same_signbit_use2_signed(i8 %x, i8 %y) {
 ; CHECK-NEXT:    [[YPOS:%.*]] = icmp sgt i8 [[Y:%.*]], -1
 ; CHECK-NEXT:    [[YPOSZ:%.*]] = sext i1 [[YPOS]] to i8
 ; CHECK-NEXT:    call void @use(i8 [[YPOSZ]])
-; CHECK-NEXT:    [[XOR_SIGNBITS:%.*]] = xor i8 [[X:%.*]], [[Y]]
-; CHECK-NEXT:    [[R:%.*]] = icmp sgt i8 [[XOR_SIGNBITS]], -1
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[X:%.*]], [[Y]]
+; CHECK-NEXT:    [[R1:%.*]] = icmp sgt i8 [[TMP1]], -1
+; CHECK-NEXT:    ret i1 [[R1]]
 ;
   %xsign = ashr i8 %x, 7
   %ypos = icmp sgt i8 %y, -1
@@ -1545,9 +1569,10 @@ define i1 @same_signbit_use3_signed(i8 %x, i8 %y) {
 
 define <2 x i1> @same_signbit_poison_elts_signed(<2 x i8> %x, <2 x i8> %y) {
 ; CHECK-LABEL: @same_signbit_poison_elts_signed(
-; CHECK-NEXT:    [[XOR_SIGNBITS:%.*]] = xor <2 x i8> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = icmp sgt <2 x i8> [[XOR_SIGNBITS]], <i8 -1, i8 -1>
-; CHECK-NEXT:    ret <2 x i1> [[R]]
+; CHECK-NEXT:    [[YPOS:%.*]] = icmp sgt <2 x i8> [[Y:%.*]], <i8 -1, i8 poison>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <2 x i8> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    [[R1:%.*]] = xor <2 x i1> [[TMP1]], [[YPOS]]
+; CHECK-NEXT:    ret <2 x i1> [[R1]]
 ;
   %xsign = ashr <2 x i8> %x, <i8 7, i8 poison>
   %ypos = icmp sgt <2 x i8> %y, <i8 -1, i8 poison>
@@ -1560,11 +1585,10 @@ define <2 x i1> @same_signbit_poison_elts_signed(<2 x i8> %x, <2 x i8> %y) {
 
 define i1 @same_signbit_wrong_type_signed(i8 %x, i32 %y) {
 ; CHECK-LABEL: @same_signbit_wrong_type_signed(
-; CHECK-NEXT:    [[XSIGN:%.*]] = ashr i8 [[X:%.*]], 7
 ; CHECK-NEXT:    [[YPOS:%.*]] = icmp sgt i32 [[Y:%.*]], -1
-; CHECK-NEXT:    [[YPOSZ:%.*]] = sext i1 [[YPOS]] to i8
-; CHECK-NEXT:    [[R:%.*]] = icmp ne i8 [[XSIGN]], [[YPOSZ]]
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i8 [[X:%.*]], 0
+; CHECK-NEXT:    [[R1:%.*]] = xor i1 [[TMP1]], [[YPOS]]
+; CHECK-NEXT:    ret i1 [[R1]]
 ;
   %xsign = ashr i8 %x, 7
   %ypos = icmp sgt i32 %y, -1
@@ -1589,3 +1613,80 @@ define i1 @exactly_one_set_signbit_wrong_shamt_signed(i8 %x, i8 %y) {
   %r = icmp eq i8 %xsign, %yposz
   ret i1 %r
 }
+
+define i1 @slt_zero_ult_i1(i32 %a, i1 %b) {
+; CHECK-LABEL: @slt_zero_ult_i1(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i1 [[B:%.*]], true
+; CHECK-NEXT:    [[CMP21:%.*]] = and i1 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i1 [[CMP21]]
+;
+  %conv = zext i1 %b to i32
+  %cmp1 = lshr i32 %a, 31
+  %cmp2 = icmp ult i32 %conv, %cmp1
+  ret i1 %cmp2
+}
+
+define i1 @slt_zero_ult_i1_fail1(i32 %a, i1 %b) {
+; CHECK-LABEL: @slt_zero_ult_i1_fail1(
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[B:%.*]] to i32
+; CHECK-NEXT:    [[CMP1:%.*]] = lshr i32 [[A:%.*]], 30
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i32 [[CMP1]], [[CONV]]
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %conv = zext i1 %b to i32
+  %cmp1 = lshr i32 %a, 30
+  %cmp2 = icmp ult i32 %conv, %cmp1
+  ret i1 %cmp2
+}
+
+define i1 @slt_zero_ult_i1_fail2(i32 %a, i1 %b) {
+; CHECK-LABEL: @slt_zero_ult_i1_fail2(
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[B:%.*]] to i32
+; CHECK-NEXT:    [[CMP1:%.*]] = ashr i32 [[A:%.*]], 31
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i32 [[CMP1]], [[CONV]]
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %conv = zext i1 %b to i32
+  %cmp1 = ashr i32 %a, 31
+  %cmp2 = icmp ult i32 %conv, %cmp1
+  ret i1 %cmp2
+}
+
+define i1 @slt_zero_slt_i1_fail(i32 %a, i1 %b) {
+; CHECK-LABEL: @slt_zero_slt_i1_fail(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i1 [[B:%.*]], true
+; CHECK-NEXT:    [[CMP21:%.*]] = and i1 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i1 [[CMP21]]
+;
+  %conv = zext i1 %b to i32
+  %cmp1 = lshr i32 %a, 31
+  %cmp2 = icmp slt i32 %conv, %cmp1
+  ret i1 %cmp2
+}
+
+define i1 @slt_zero_eq_i1_signed(i32 %a, i1 %b) {
+; CHECK-LABEL: @slt_zero_eq_i1_signed(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[A:%.*]], -1
+; CHECK-NEXT:    [[CMP21:%.*]] = xor i1 [[TMP1]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP21]]
+;
+  %conv = sext i1 %b to i32
+  %cmp1 = ashr i32 %a, 31
+  %cmp2 = icmp eq i32 %conv, %cmp1
+  ret i1 %cmp2
+}
+
+define i1 @slt_zero_eq_i1_fail_signed(i32 %a, i1 %b) {
+; CHECK-LABEL: @slt_zero_eq_i1_fail_signed(
+; CHECK-NEXT:    [[CONV:%.*]] = sext i1 [[B:%.*]] to i32
+; CHECK-NEXT:    [[CMP1:%.*]] = lshr i32 [[A:%.*]], 31
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32 [[CMP1]], [[CONV]]
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %conv = sext i1 %b to i32
+  %cmp1 = lshr i32 %a, 31
+  %cmp2 = icmp eq i32 %conv, %cmp1
+  ret i1 %cmp2
+}
diff --git a/llvm/test/Transforms/InstCombine/icmp-xor-signbit.ll b/llvm/test/Transforms/InstCombine/icmp-xor-signbit.ll
index 29a18ebbdd..d08dca2253 100644
--- a/llvm/test/Transforms/InstCombine/icmp-xor-signbit.ll
+++ b/llvm/test/Transforms/InstCombine/icmp-xor-signbit.ll
@@ -217,3 +217,107 @@ define <2 x i1> @negative_simplify_splat(<4 x i8> %x) {
   ret <2 x i1> %c
 }
 
+define i1 @slt_zero_eq_i1(i32 %a, i1 %b) {
+; CHECK-LABEL: @slt_zero_eq_i1(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[A:%.*]], -1
+; CHECK-NEXT:    [[CMP21:%.*]] = xor i1 [[TMP1]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP21]]
+;
+  %conv = zext i1 %b to i32
+  %cmp1 = lshr i32 %a, 31
+  %cmp2 = icmp eq i32 %conv, %cmp1
+  ret i1 %cmp2
+}
+
+define i1 @slt_zero_eq_i1_fail(i32 %a, i1 %b) {
+; CHECK-LABEL: @slt_zero_eq_i1_fail(
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[B:%.*]] to i32
+; CHECK-NEXT:    [[CMP1:%.*]] = ashr i32 [[A:%.*]], 31
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32 [[CMP1]], [[CONV]]
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %conv = zext i1 %b to i32
+  %cmp1 = ashr i32 %a, 31
+  %cmp2 = icmp eq i32 %conv, %cmp1
+  ret i1 %cmp2
+}
+
+define i1 @slt_zero_eq_ne_0(i32 %a) {
+; CHECK-LABEL: @slt_zero_eq_ne_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[A:%.*]], 1
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = icmp ne i32 %a, 0
+  %conv = zext i1 %cmp to i32
+  %cmp1 = lshr i32 %a, 31
+  %cmp2 = icmp eq i32 %conv, %cmp1
+  ret i1 %cmp2
+}
+
+define i1 @slt_zero_ne_ne_0(i32 %a) {
+; CHECK-LABEL: @slt_zero_ne_ne_0(
+; CHECK-NEXT:    [[CMP21:%.*]] = icmp sgt i32 [[A:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP21]]
+;
+  %cmp = icmp ne i32 %a, 0
+  %conv = zext i1 %cmp to i32
+  %cmp1 = lshr i32 %a, 31
+  %cmp2 = icmp ne i32 %conv, %cmp1
+  ret i1 %cmp2
+}
+
+define <4 x i1> @slt_zero_eq_ne_0_vec(<4 x i32> %a) {
+; CHECK-LABEL: @slt_zero_eq_ne_0_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <4 x i32> [[A:%.*]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    ret <4 x i1> [[TMP1]]
+;
+  %cmp = icmp ne <4 x i32> %a, zeroinitializer
+  %conv = zext <4 x i1> %cmp to <4 x i32>
+  %cmp1 = lshr <4 x i32> %a, <i32 31, i32 31, i32 31, i32 31>
+  %cmp2 = icmp eq <4 x i32> %conv, %cmp1
+  ret <4 x i1> %cmp2
+}
+
+define i1 @slt_zero_ne_ne_b(i32 %a, i32 %b) {
+; CHECK-LABEL: @slt_zero_ne_ne_b(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[A]], 0
+; CHECK-NEXT:    [[CMP21:%.*]] = xor i1 [[TMP1]], [[CMP]]
+; CHECK-NEXT:    ret i1 [[CMP21]]
+;
+  %cmp = icmp ne i32 %a, %b
+  %conv = zext i1 %cmp to i32
+  %cmp1 = lshr i32 %a, 31
+  %cmp2 = icmp ne i32 %conv, %cmp1
+  ret i1 %cmp2
+}
+
+define i1 @slt_zero_eq_ne_0_fail1(i32 %a) {
+; CHECK-LABEL: @slt_zero_eq_ne_0_fail1(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    [[CMP1:%.*]] = ashr i32 [[A]], 31
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32 [[CMP1]], [[CONV]]
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp = icmp ne i32 %a, 0
+  %conv = zext i1 %cmp to i32
+  %cmp1 = ashr i32 %a, 31
+  %cmp2 = icmp eq i32 %conv, %cmp1
+  ret i1 %cmp2
+}
+
+define i1 @slt_zero_eq_ne_0_fail2(i32 %a) {
+; CHECK-LABEL: @slt_zero_eq_ne_0_fail2(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    [[CMP1:%.*]] = lshr i32 [[A]], 30
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32 [[CMP1]], [[CONV]]
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp = icmp ne i32 %a, 0
+  %conv = zext i1 %cmp to i32
+  %cmp1 = lshr i32 %a, 30
+  %cmp2 = icmp eq i32 %conv, %cmp1
+  ret i1 %cmp2
+}
diff --git a/llvm/test/Transforms/InstCombine/malloc-free.ll b/llvm/test/Transforms/InstCombine/malloc-free.ll
index dc918a7fc8..10725950a1 100644
--- a/llvm/test/Transforms/InstCombine/malloc-free.ll
+++ b/llvm/test/Transforms/InstCombine/malloc-free.ll
@@ -26,9 +26,64 @@ define i32 @dead_aligned_alloc(i32 %size, i32 %alignment, i8 %value) {
   ret i32 0
 }
 
+define i1 @aligned_alloc_only_pointe(i32 %size, i32 %alignment, i8 %value) {
+; CHECK-LABEL: @aligned_alloc_only_pointe(
+; CHECK-NEXT:    [[ALIGNED_ALLOCATION:%.*]] = tail call ptr @aligned_alloc(i32 [[ALIGNMENT:%.*]], i32 [[SIZE:%.*]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne ptr [[ALIGNED_ALLOCATION]], null
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %aligned_allocation = tail call ptr @aligned_alloc(i32 %alignment, i32 %size)
+  %cmp = icmp ne ptr %aligned_allocation, null
+  ret i1 %cmp
+}
+
+define i1 @aligned_alloc_pointer_only_used_by_cmp_alignment_and_value_known_ok(i32 %size, i32 %alignment, i8 %value) {
+; CHECK-LABEL: @aligned_alloc_pointer_only_used_by_cmp_alignment_and_value_known_ok(
+; CHECK-NEXT:    ret i1 true
+;
+  %aligned_allocation = tail call ptr @aligned_alloc(i32 8, i32 32)
+  %cmp = icmp ne ptr %aligned_allocation, null
+  ret i1 %cmp
+}
+
+define i1 @aligned_alloc_pointer_only_used_by_cmp_alignment_no_power_of_2(i32 %size, i32 %alignment, i8 %value) {
+; CHECK-LABEL: @aligned_alloc_pointer_only_used_by_cmp_alignment_no_power_of_2(
+; CHECK-NEXT:    [[ALIGNED_ALLOCATION:%.*]] = tail call dereferenceable_or_null(32) ptr @aligned_alloc(i32 3, i32 32)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne ptr [[ALIGNED_ALLOCATION]], null
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %aligned_allocation = tail call ptr @aligned_alloc(i32 3, i32 32)
+  %cmp = icmp ne ptr %aligned_allocation, null
+  ret i1 %cmp
+}
+
+define i1 @aligned_alloc_pointer_only_used_by_cmp_size_not_multiple_of_alignment(i32 %size, i32 %alignment, i8 %value) {
+; CHECK-LABEL: @aligned_alloc_pointer_only_used_by_cmp_size_not_multiple_of_alignment(
+; CHECK-NEXT:    [[ALIGNED_ALLOCATION:%.*]] = tail call dereferenceable_or_null(31) ptr @aligned_alloc(i32 8, i32 31)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne ptr [[ALIGNED_ALLOCATION]], null
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %aligned_allocation = tail call ptr @aligned_alloc(i32 8, i32 31)
+  %cmp = icmp ne ptr %aligned_allocation, null
+  ret i1 %cmp
+}
+
+; This test uses a aligned allocation function different to @aligned_alloc,
+; and should be treated as having @aligned_alloc's constraints on alignment
+; and size operands.
+define i1 @other_aligned_allocation_function(i32 %size, i32 %alignment, i8 %value) {
+; CHECK-LABEL: @other_aligned_allocation_function(
+; CHECK-NEXT:    ret i1 true
+;
+  %aligned_allocation = tail call ptr @other_aligned_alloc(i32 %alignment, i32 %size)
+  %cmp = icmp ne ptr %aligned_allocation, null
+  ret i1 %cmp
+}
+
 declare noalias ptr @calloc(i32, i32) nounwind allockind("alloc,zeroed") allocsize(0,1) "alloc-family"="malloc"
 declare noalias ptr @malloc(i32) allockind("alloc,uninitialized") allocsize(0) "alloc-family"="malloc"
 declare noalias ptr @aligned_alloc(i32, i32) allockind("alloc,uninitialized,aligned") allocsize(1) "alloc-family"="malloc"
+declare noalias ptr @other_aligned_alloc(i32, i32) allockind("alloc,uninitialized,aligned") allocsize(1) "alloc-family"="malloc"
 declare void @free(ptr) allockind("free") "alloc-family"="malloc"
 
 define i1 @foo() {
diff --git a/llvm/test/Transforms/InstCombine/powi.ll b/llvm/test/Transforms/InstCombine/powi.ll
index 20fe25c50a..89efbb6f45 100644
--- a/llvm/test/Transforms/InstCombine/powi.ll
+++ b/llvm/test/Transforms/InstCombine/powi.ll
@@ -2,6 +2,7 @@
 ; RUN: opt -passes=instcombine -S < %s | FileCheck %s
 
 declare double @llvm.powi.f64.i32(double, i32)
+declare float @llvm.powi.f32.i32(float, i32)
 declare double @llvm.powi.f64.i64(double, i64)
 declare double @llvm.fabs.f64(double)
 declare double @llvm.copysign.f64(double, double)
@@ -258,3 +259,85 @@ define double @different_types_powi(double %x, i32 %y, i64 %z) {
   %mul = fmul reassoc double %p2, %p1
   ret double %mul
 }
+
+define double @fdiv_pow_powi(double %x) {
+; CHECK-LABEL: @fdiv_pow_powi(
+; CHECK-NEXT:    [[DIV:%.*]] = fmul reassoc nnan double [[X:%.*]], [[X]]
+; CHECK-NEXT:    ret double [[DIV]]
+;
+  %p1 = call double @llvm.powi.f64.i32(double %x, i32 3)
+  %div = fdiv reassoc nnan double %p1, %x
+  ret double %div
+}
+
+define float @fdiv_powf_powi(float %x) {
+; CHECK-LABEL: @fdiv_powf_powi(
+; CHECK-NEXT:    [[DIV:%.*]] = call reassoc nnan float @llvm.powi.f32.i32(float [[X:%.*]], i32 99)
+; CHECK-NEXT:    ret float [[DIV]]
+;
+  %p1 = call float @llvm.powi.f32.i32(float %x, i32 100)
+  %div = fdiv reassoc nnan float %p1, %x
+  ret float %div
+}
+
+; TODO: Multi-use may be also better off creating Powi(x,y-1) then creating
+; (mul, Powi(x,y-1),x) to replace the Powi(x,y).
+define double @fdiv_pow_powi_multi_use(double %x) {
+; CHECK-LABEL: @fdiv_pow_powi_multi_use(
+; CHECK-NEXT:    [[P1:%.*]] = call double @llvm.powi.f64.i32(double [[X:%.*]], i32 3)
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv reassoc nnan double [[P1]], [[X]]
+; CHECK-NEXT:    tail call void @use(double [[P1]])
+; CHECK-NEXT:    ret double [[DIV]]
+;
+  %p1 = call double @llvm.powi.f64.i32(double %x, i32 3)
+  %div = fdiv reassoc nnan double %p1, %x
+  tail call void @use(double %p1)
+  ret double %div
+}
+
+; Negative test: Miss part of the fmf flag for the fdiv instruction
+define float @fdiv_powf_powi_missing_reassoc(float %x) {
+; CHECK-LABEL: @fdiv_powf_powi_missing_reassoc(
+; CHECK-NEXT:    [[P1:%.*]] = call float @llvm.powi.f32.i32(float [[X:%.*]], i32 100)
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv nnan float [[P1]], [[X]]
+; CHECK-NEXT:    ret float [[DIV]]
+;
+  %p1 = call float @llvm.powi.f32.i32(float %x, i32 100)
+  %div = fdiv nnan float %p1, %x
+  ret float %div
+}
+
+define float @fdiv_powf_powi_missing_nnan(float %x) {
+; CHECK-LABEL: @fdiv_powf_powi_missing_nnan(
+; CHECK-NEXT:    [[P1:%.*]] = call float @llvm.powi.f32.i32(float [[X:%.*]], i32 100)
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv reassoc float [[P1]], [[X]]
+; CHECK-NEXT:    ret float [[DIV]]
+;
+  %p1 = call float @llvm.powi.f32.i32(float %x, i32 100)
+  %div = fdiv reassoc float %p1, %x
+  ret float %div
+}
+
+; Negative test: Illegal because (Y - 1) wraparound
+define double @fdiv_pow_powi_negative(double %x) {
+; CHECK-LABEL: @fdiv_pow_powi_negative(
+; CHECK-NEXT:    [[P1:%.*]] = call double @llvm.powi.f64.i32(double [[X:%.*]], i32 -2147483648)
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv reassoc nnan double [[P1]], [[X]]
+; CHECK-NEXT:    ret double [[DIV]]
+;
+  %p1 = call double @llvm.powi.f64.i32(double %x, i32 -2147483648) ; INT_MIN
+  %div = fdiv reassoc nnan double %p1, %x
+  ret double %div
+}
+
+; Negative test: The 2nd powi argument is a variable
+define double @fdiv_pow_powi_negative_variable(double %x, i32 %y) {
+; CHECK-LABEL: @fdiv_pow_powi_negative_variable(
+; CHECK-NEXT:    [[P1:%.*]] = call double @llvm.powi.f64.i32(double [[X:%.*]], i32 [[Y:%.*]])
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv reassoc nnan double [[P1]], [[X]]
+; CHECK-NEXT:    ret double [[DIV]]
+;
+  %p1 = call double @llvm.powi.f64.i32(double %x, i32 %y)
+  %div = fdiv reassoc nnan double %p1, %x
+  ret double %div
+}
diff --git a/llvm/test/Transforms/InstCombine/pr69059.ll b/llvm/test/Transforms/InstCombine/pr69059.ll
new file mode 100644
index 0000000000..75690b8396
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr69059.ll
@@ -0,0 +1,16 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
+
+define i64 @pr69059() {
+; CHECK-LABEL: define i64 @pr69059() {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL:%.*]] = call i64 @labs(i32 0)
+; CHECK-NEXT:    ret i64 [[CALL]]
+;
+entry:
+  %call = call i64 @labs(i32 0)
+  ret i64 %call
+}
+
+; negative test: not a valid libfunc proto
+declare i64 @labs(i32)
diff --git a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-pr49778.ll b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-pr49778.ll
index 96dc6c68f4..b06a90e2cd 100644
--- a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-pr49778.ll
+++ b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-pr49778.ll
@@ -5,10 +5,10 @@
 define i32 @src(i1 %x2) {
 ; CHECK-LABEL: @src(
 ; CHECK-NEXT:    [[X13:%.*]] = zext i1 [[X2:%.*]] to i32
-; CHECK-NEXT:    [[_7:%.*]] = shl i32 -1, [[X13]]
+; CHECK-NEXT:    [[_7:%.*]] = shl nsw i32 -1, [[X13]]
 ; CHECK-NEXT:    [[MASK:%.*]] = xor i32 [[_7]], -1
 ; CHECK-NEXT:    [[_8:%.*]] = and i32 [[MASK]], [[X13]]
-; CHECK-NEXT:    [[_9:%.*]] = shl i32 [[_8]], [[X13]]
+; CHECK-NEXT:    [[_9:%.*]] = shl nuw nsw i32 [[_8]], [[X13]]
 ; CHECK-NEXT:    ret i32 [[_9]]
 ;
   %x13 = zext i1 %x2 to i32
diff --git a/llvm/test/Transforms/InstCombine/rotate.ll b/llvm/test/Transforms/InstCombine/rotate.ll
index fece475348..ed5145255b 100644
--- a/llvm/test/Transforms/InstCombine/rotate.ll
+++ b/llvm/test/Transforms/InstCombine/rotate.ll
@@ -705,7 +705,7 @@ define i9 @rotateleft_9_neg_mask_wide_amount_commute(i9 %v, i33 %shamt) {
 ; CHECK-NEXT:    [[LSHAMT:%.*]] = and i33 [[SHAMT]], 8
 ; CHECK-NEXT:    [[RSHAMT:%.*]] = and i33 [[NEG]], 8
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i9 [[V:%.*]] to i33
-; CHECK-NEXT:    [[SHL:%.*]] = shl i33 [[CONV]], [[LSHAMT]]
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw nsw i33 [[CONV]], [[LSHAMT]]
 ; CHECK-NEXT:    [[SHR:%.*]] = lshr i33 [[CONV]], [[RSHAMT]]
 ; CHECK-NEXT:    [[OR:%.*]] = or i33 [[SHL]], [[SHR]]
 ; CHECK-NEXT:    [[RET:%.*]] = trunc i33 [[OR]] to i9
diff --git a/llvm/test/Transforms/InstCombine/sdiv-guard.ll b/llvm/test/Transforms/InstCombine/sdiv-guard.ll
index ba96709241..cff2f6aefd 100644
--- a/llvm/test/Transforms/InstCombine/sdiv-guard.ll
+++ b/llvm/test/Transforms/InstCombine/sdiv-guard.ll
@@ -6,8 +6,9 @@ declare void @llvm.experimental.guard(i1, ...)
 ; Regression test. If %flag is false then %s == 0 and guard should be triggered.
 define i32 @a(i1 %flag, i32 %X) nounwind readnone {
 ; CHECK-LABEL: @a(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[X:%.*]], 0
-; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[CMP]]) #[[ATTR2:[0-9]+]] [ "deopt"() ]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[CMP:%.*]] = select i1 [[FLAG:%.*]], i1 [[CMP1]], i1 false
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[CMP]]) #[[ATTR1:[0-9]+]] [ "deopt"() ]
 ; CHECK-NEXT:    [[R:%.*]] = sdiv i32 100, [[X]]
 ; CHECK-NEXT:    ret i32 [[R]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/shift-flags.ll b/llvm/test/Transforms/InstCombine/shift-flags.ll
new file mode 100644
index 0000000000..08cf4821d8
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/shift-flags.ll
@@ -0,0 +1,119 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
+
+define i8 @shl_add_nuw(i8 %amt_in, i8 %cnt_in) {
+; CHECK-LABEL: @shl_add_nuw(
+; CHECK-NEXT:    [[AMT:%.*]] = and i8 [[AMT_IN:%.*]], 63
+; CHECK-NEXT:    [[CNT:%.*]] = and i8 [[CNT_IN:%.*]], 2
+; CHECK-NEXT:    [[R:%.*]] = shl nuw i8 [[AMT]], [[CNT]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %amt = and i8 %amt_in, 63
+  %cnt = and i8 %cnt_in, 2
+  %r = shl i8 %amt, %cnt
+  ret i8 %r
+}
+
+define i8 @shl_add_nuw_fail(i8 %amt_in, i8 %cnt_in) {
+; CHECK-LABEL: @shl_add_nuw_fail(
+; CHECK-NEXT:    [[AMT:%.*]] = and i8 [[AMT_IN:%.*]], 63
+; CHECK-NEXT:    [[CNT:%.*]] = and i8 [[CNT_IN:%.*]], 3
+; CHECK-NEXT:    [[R:%.*]] = shl i8 [[AMT]], [[CNT]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %amt = and i8 %amt_in, 63
+  %cnt = and i8 %cnt_in, 3
+  %r = shl i8 %amt, %cnt
+  ret i8 %r
+}
+
+define i8 @shl_add_nuw_and_nsw(i8 %amt_in, i8 %cnt_in) {
+; CHECK-LABEL: @shl_add_nuw_and_nsw(
+; CHECK-NEXT:    [[AMT:%.*]] = and i8 [[AMT_IN:%.*]], 31
+; CHECK-NEXT:    [[CNT:%.*]] = and i8 [[CNT_IN:%.*]], 2
+; CHECK-NEXT:    [[R:%.*]] = shl nuw nsw i8 [[AMT]], [[CNT]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %amt = and i8 %amt_in, 31
+  %cnt = and i8 %cnt_in, 2
+  %r = shl i8 %amt, %cnt
+  ret i8 %r
+}
+
+define i8 @shl_add_nsw(i8 %amt_in, i8 %cnt_in) {
+; CHECK-LABEL: @shl_add_nsw(
+; CHECK-NEXT:    [[AMT:%.*]] = or i8 [[AMT_IN:%.*]], -32
+; CHECK-NEXT:    [[CNT:%.*]] = and i8 [[CNT_IN:%.*]], 2
+; CHECK-NEXT:    [[R:%.*]] = shl nsw i8 [[AMT]], [[CNT]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %amt = or i8 %amt_in, 224
+  %cnt = and i8 %cnt_in, 2
+  %r = shl i8 %amt, %cnt
+  ret i8 %r
+}
+
+define i8 @shl_add_nsw_fail(i8 %amt_in, i8 %cnt_in) {
+; CHECK-LABEL: @shl_add_nsw_fail(
+; CHECK-NEXT:    [[AMT:%.*]] = or i8 [[AMT_IN:%.*]], -64
+; CHECK-NEXT:    [[CNT:%.*]] = and i8 [[CNT_IN:%.*]], 2
+; CHECK-NEXT:    [[R:%.*]] = shl i8 [[AMT]], [[CNT]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %amt = or i8 %amt_in, 192
+  %cnt = and i8 %cnt_in, 2
+  %r = shl i8 %amt, %cnt
+  ret i8 %r
+}
+
+define i8 @lshr_add_exact(i8 %amt_in, i8 %cnt_in) {
+; CHECK-LABEL: @lshr_add_exact(
+; CHECK-NEXT:    [[AMT:%.*]] = and i8 [[AMT_IN:%.*]], -4
+; CHECK-NEXT:    [[CNT:%.*]] = and i8 [[CNT_IN:%.*]], 2
+; CHECK-NEXT:    [[R:%.*]] = lshr exact i8 [[AMT]], [[CNT]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %amt = and i8 %amt_in, -4
+  %cnt = and i8 %cnt_in, 2
+  %r = lshr i8 %amt, %cnt
+  ret i8 %r
+}
+
+define i8 @lshr_add_exact_fail(i8 %amt_in, i8 %cnt_in) {
+; CHECK-LABEL: @lshr_add_exact_fail(
+; CHECK-NEXT:    [[AMT:%.*]] = and i8 [[AMT_IN:%.*]], -7
+; CHECK-NEXT:    [[CNT:%.*]] = and i8 [[CNT_IN:%.*]], 2
+; CHECK-NEXT:    [[R:%.*]] = lshr i8 [[AMT]], [[CNT]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %amt = and i8 %amt_in, -7
+  %cnt = and i8 %cnt_in, 2
+  %r = lshr i8 %amt, %cnt
+  ret i8 %r
+}
+
+define i8 @ashr_add_exact(i8 %amt_in, i8 %cnt_in) {
+; CHECK-LABEL: @ashr_add_exact(
+; CHECK-NEXT:    [[AMT:%.*]] = and i8 [[AMT_IN:%.*]], -14
+; CHECK-NEXT:    [[CNT:%.*]] = and i8 [[CNT_IN:%.*]], 1
+; CHECK-NEXT:    [[R:%.*]] = ashr exact i8 [[AMT]], [[CNT]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %amt = and i8 %amt_in, -14
+  %cnt = and i8 %cnt_in, 1
+  %r = ashr i8 %amt, %cnt
+  ret i8 %r
+}
+
+define i8 @ashr_add_exact_fail(i8 %amt_in, i8 %cnt_in) {
+; CHECK-LABEL: @ashr_add_exact_fail(
+; CHECK-NEXT:    [[AMT:%.*]] = and i8 [[AMT_IN:%.*]], -14
+; CHECK-NEXT:    [[CNT:%.*]] = and i8 [[CNT_IN:%.*]], 2
+; CHECK-NEXT:    [[R:%.*]] = ashr i8 [[AMT]], [[CNT]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %amt = and i8 %amt_in, -14
+  %cnt = and i8 %cnt_in, 2
+  %r = ashr i8 %amt, %cnt
+  ret i8 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/simplify-demanded-fpclass.ll b/llvm/test/Transforms/InstCombine/simplify-demanded-fpclass.ll
index 4f9396add2..9817b6e13c 100644
--- a/llvm/test/Transforms/InstCombine/simplify-demanded-fpclass.ll
+++ b/llvm/test/Transforms/InstCombine/simplify-demanded-fpclass.ll
@@ -42,7 +42,7 @@ define nofpclass(inf) float @ret_nofpclass_inf_undef() {
 define nofpclass(all) float @ret_nofpclass_all_var(float %arg) {
 ; CHECK-LABEL: define nofpclass(all) float @ret_nofpclass_all_var
 ; CHECK-SAME: (float [[ARG:%.*]]) {
-; CHECK-NEXT:    ret float poison
+; CHECK-NEXT:    ret float [[ARG]]
 ;
   ret float %arg
 }
@@ -51,7 +51,7 @@ define nofpclass(all) float @ret_nofpclass_all_var(float %arg) {
 define nofpclass(all) <2 x float> @ret_nofpclass_all_var_vector(<2 x float> %arg) {
 ; CHECK-LABEL: define nofpclass(all) <2 x float> @ret_nofpclass_all_var_vector
 ; CHECK-SAME: (<2 x float> [[ARG:%.*]]) {
-; CHECK-NEXT:    ret <2 x float> poison
+; CHECK-NEXT:    ret <2 x float> [[ARG]]
 ;
   ret <2 x float> %arg
 }
@@ -65,14 +65,14 @@ define nofpclass(inf) float @ret_nofpclass_inf__0() {
 
 define nofpclass(inf) float @ret_nofpclass_inf__pinf() {
 ; CHECK-LABEL: define nofpclass(inf) float @ret_nofpclass_inf__pinf() {
-; CHECK-NEXT:    ret float poison
+; CHECK-NEXT:    ret float 0x7FF0000000000000
 ;
   ret float 0x7FF0000000000000
 }
 
 define nofpclass(pinf) float @ret_nofpclass_pinf__pinf() {
 ; CHECK-LABEL: define nofpclass(pinf) float @ret_nofpclass_pinf__pinf() {
-; CHECK-NEXT:    ret float poison
+; CHECK-NEXT:    ret float 0x7FF0000000000000
 ;
   ret float 0x7FF0000000000000
 }
@@ -86,7 +86,7 @@ define nofpclass(pinf) float @ret_nofpclass_pinf__ninf() {
 
 define nofpclass(inf) float @ret_nofpclass_inf__ninf() {
 ; CHECK-LABEL: define nofpclass(inf) float @ret_nofpclass_inf__ninf() {
-; CHECK-NEXT:    ret float poison
+; CHECK-NEXT:    ret float 0xFFF0000000000000
 ;
   ret float 0xFFF0000000000000
 }
@@ -106,7 +106,8 @@ define nofpclass(inf) float @ret_nofpclass_inf__select_nofpclass_inf_lhs(i1 %con
 define nofpclass(inf) float @ret_nofpclass_inf__select_nofpclass_arg_only_inf_lhs(i1 %cond, float nofpclass(nan norm zero sub) %x, float %y) {
 ; CHECK-LABEL: define nofpclass(inf) float @ret_nofpclass_inf__select_nofpclass_arg_only_inf_lhs
 ; CHECK-SAME: (i1 [[COND:%.*]], float nofpclass(nan zero sub norm) [[X:%.*]], float [[Y:%.*]]) {
-; CHECK-NEXT:    ret float [[Y]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND]], float [[X]], float [[Y]]
+; CHECK-NEXT:    ret float [[SELECT]]
 ;
   %select = select i1 %cond, float %x, float %y
   ret float %select
@@ -116,7 +117,8 @@ define nofpclass(inf) float @ret_nofpclass_inf__select_nofpclass_arg_only_inf_lh
 define nofpclass(inf) float @ret_nofpclass_inf__select_nofpclass_arg_only_inf_rhs(i1 %cond, float %x, float nofpclass(nan norm zero sub) %y) {
 ; CHECK-LABEL: define nofpclass(inf) float @ret_nofpclass_inf__select_nofpclass_arg_only_inf_rhs
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]], float nofpclass(nan zero sub norm) [[Y:%.*]]) {
-; CHECK-NEXT:    ret float [[X]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND]], float [[X]], float [[Y]]
+; CHECK-NEXT:    ret float [[SELECT]]
 ;
   %select = select i1 %cond, float %x, float %y
   ret float %select
@@ -126,7 +128,8 @@ define nofpclass(inf) float @ret_nofpclass_inf__select_nofpclass_arg_only_inf_rh
 define nofpclass(inf) [3 x [2 x float]] @ret_float_array(i1 %cond, [3 x [2 x float]] nofpclass(nan norm zero sub) %x, [3 x [2 x float]] %y) {
 ; CHECK-LABEL: define nofpclass(inf) [3 x [2 x float]] @ret_float_array
 ; CHECK-SAME: (i1 [[COND:%.*]], [3 x [2 x float]] nofpclass(nan zero sub norm) [[X:%.*]], [3 x [2 x float]] [[Y:%.*]]) {
-; CHECK-NEXT:    ret [3 x [2 x float]] [[Y]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND]], [3 x [2 x float]] [[X]], [3 x [2 x float]] [[Y]]
+; CHECK-NEXT:    ret [3 x [2 x float]] [[SELECT]]
 ;
   %select = select i1 %cond, [3 x [2 x float]] %x, [3 x [2 x float]] %y
   ret [3 x [2 x float ]] %select
@@ -136,7 +139,8 @@ define nofpclass(inf) [3 x [2 x float]] @ret_float_array(i1 %cond, [3 x [2 x flo
 define nofpclass(inf) float @ret_nofpclass_inf__select_pinf_lhs(i1 %cond, float %x) {
 ; CHECK-LABEL: define nofpclass(inf) float @ret_nofpclass_inf__select_pinf_lhs
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]]) {
-; CHECK-NEXT:    ret float [[X]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND]], float 0x7FF0000000000000, float [[X]]
+; CHECK-NEXT:    ret float [[SELECT]]
 ;
   %select = select i1 %cond, float 0x7FF0000000000000, float %x
   ret float %select
@@ -146,7 +150,8 @@ define nofpclass(inf) float @ret_nofpclass_inf__select_pinf_lhs(i1 %cond, float
 define nofpclass(inf) float @ret_nofpclass_inf__select_pinf_rhs(i1 %cond, float %x) {
 ; CHECK-LABEL: define nofpclass(inf) float @ret_nofpclass_inf__select_pinf_rhs
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]]) {
-; CHECK-NEXT:    ret float [[X]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND]], float [[X]], float 0x7FF0000000000000
+; CHECK-NEXT:    ret float [[SELECT]]
 ;
   %select = select i1 %cond, float %x, float 0x7FF0000000000000
   ret float %select
@@ -156,7 +161,8 @@ define nofpclass(inf) float @ret_nofpclass_inf__select_pinf_rhs(i1 %cond, float
 define nofpclass(inf) float @ret_nofpclass_inf__select_pinf_or_ninf(i1 %cond, float %x) {
 ; CHECK-LABEL: define nofpclass(inf) float @ret_nofpclass_inf__select_pinf_or_ninf
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]]) {
-; CHECK-NEXT:    ret float poison
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND]], float 0x7FF0000000000000, float 0xFFF0000000000000
+; CHECK-NEXT:    ret float [[SELECT]]
 ;
   %select = select i1 %cond, float 0x7FF0000000000000, float 0xFFF0000000000000
   ret float %select
@@ -166,7 +172,8 @@ define nofpclass(inf) float @ret_nofpclass_inf__select_pinf_or_ninf(i1 %cond, fl
 define nofpclass(inf) float @ret_nofpclass_inf__select_ninf_or_pinf(i1 %cond, float %x) {
 ; CHECK-LABEL: define nofpclass(inf) float @ret_nofpclass_inf__select_ninf_or_pinf
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]]) {
-; CHECK-NEXT:    ret float poison
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND]], float 0xFFF0000000000000, float 0x7FF0000000000000
+; CHECK-NEXT:    ret float [[SELECT]]
 ;
   %select = select i1 %cond, float 0xFFF0000000000000, float 0x7FF0000000000000
   ret float %select
@@ -176,7 +183,8 @@ define nofpclass(inf) float @ret_nofpclass_inf__select_ninf_or_pinf(i1 %cond, fl
 define nofpclass(ninf) float @ret_nofpclass_ninf__select_ninf_or_pinf(i1 %cond, float %x) {
 ; CHECK-LABEL: define nofpclass(ninf) float @ret_nofpclass_ninf__select_ninf_or_pinf
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]]) {
-; CHECK-NEXT:    ret float 0x7FF0000000000000
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND]], float 0xFFF0000000000000, float 0x7FF0000000000000
+; CHECK-NEXT:    ret float [[SELECT]]
 ;
   %select = select i1 %cond, float 0xFFF0000000000000, float 0x7FF0000000000000
   ret float %select
@@ -186,7 +194,8 @@ define nofpclass(ninf) float @ret_nofpclass_ninf__select_ninf_or_pinf(i1 %cond,
 define nofpclass(pinf) float @ret_nofpclass_pinf__select_ninf_or_pinf(i1 %cond, float %x) {
 ; CHECK-LABEL: define nofpclass(pinf) float @ret_nofpclass_pinf__select_ninf_or_pinf
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]]) {
-; CHECK-NEXT:    ret float 0xFFF0000000000000
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND]], float 0xFFF0000000000000, float 0x7FF0000000000000
+; CHECK-NEXT:    ret float [[SELECT]]
 ;
   %select = select i1 %cond, float 0xFFF0000000000000, float 0x7FF0000000000000
   ret float %select
@@ -196,7 +205,8 @@ define nofpclass(pinf) float @ret_nofpclass_pinf__select_ninf_or_pinf(i1 %cond,
 define nofpclass(zero) float @ret_nofpclass_zero__select_pzero_or_nzero(i1 %cond, float %x) {
 ; CHECK-LABEL: define nofpclass(zero) float @ret_nofpclass_zero__select_pzero_or_nzero
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]]) {
-; CHECK-NEXT:    ret float poison
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND]], float 0.000000e+00, float -0.000000e+00
+; CHECK-NEXT:    ret float [[SELECT]]
 ;
   %select = select i1 %cond, float 0.0, float -0.0
   ret float %select
@@ -206,7 +216,8 @@ define nofpclass(zero) float @ret_nofpclass_zero__select_pzero_or_nzero(i1 %cond
 define nofpclass(nzero) float @ret_nofpclass_nzero__select_pzero_or_nzero(i1 %cond, float %x) {
 ; CHECK-LABEL: define nofpclass(nzero) float @ret_nofpclass_nzero__select_pzero_or_nzero
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]]) {
-; CHECK-NEXT:    ret float 0.000000e+00
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND]], float 0.000000e+00, float -0.000000e+00
+; CHECK-NEXT:    ret float [[SELECT]]
 ;
   %select = select i1 %cond, float 0.0, float -0.0
   ret float %select
@@ -216,7 +227,8 @@ define nofpclass(nzero) float @ret_nofpclass_nzero__select_pzero_or_nzero(i1 %co
 define nofpclass(pzero) float @ret_nofpclass_pzero__select_pzero_or_nzero(i1 %cond, float %x) {
 ; CHECK-LABEL: define nofpclass(pzero) float @ret_nofpclass_pzero__select_pzero_or_nzero
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]]) {
-; CHECK-NEXT:    ret float -0.000000e+00
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND]], float 0.000000e+00, float -0.000000e+00
+; CHECK-NEXT:    ret float [[SELECT]]
 ;
   %select = select i1 %cond, float 0.0, float -0.0
   ret float %select
@@ -226,7 +238,8 @@ define nofpclass(pzero) float @ret_nofpclass_pzero__select_pzero_or_nzero(i1 %co
 define nofpclass(inf) <2 x float> @ret_nofpclass_inf__select_pinf_lhs_vector(<2 x i1> %cond, <2 x float> %x) {
 ; CHECK-LABEL: define nofpclass(inf) <2 x float> @ret_nofpclass_inf__select_pinf_lhs_vector
 ; CHECK-SAME: (<2 x i1> [[COND:%.*]], <2 x float> [[X:%.*]]) {
-; CHECK-NEXT:    ret <2 x float> [[X]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select <2 x i1> [[COND]], <2 x float> <float 0x7FF0000000000000, float 0x7FF0000000000000>, <2 x float> [[X]]
+; CHECK-NEXT:    ret <2 x float> [[SELECT]]
 ;
   %select = select <2 x i1> %cond, <2 x float> <float 0x7FF0000000000000, float 0x7FF0000000000000>, <2 x float> %x
   ret <2 x float> %select
@@ -236,7 +249,8 @@ define nofpclass(inf) <2 x float> @ret_nofpclass_inf__select_pinf_lhs_vector(<2
 define nofpclass(inf) <2 x float> @ret_nofpclass_inf__select_pinf_lhs_vector_undef(<2 x i1> %cond, <2 x float> %x) {
 ; CHECK-LABEL: define nofpclass(inf) <2 x float> @ret_nofpclass_inf__select_pinf_lhs_vector_undef
 ; CHECK-SAME: (<2 x i1> [[COND:%.*]], <2 x float> [[X:%.*]]) {
-; CHECK-NEXT:    ret <2 x float> [[X]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select <2 x i1> [[COND]], <2 x float> <float 0x7FF0000000000000, float poison>, <2 x float> [[X]]
+; CHECK-NEXT:    ret <2 x float> [[SELECT]]
 ;
   %select = select <2 x i1> %cond, <2 x float> <float 0x7FF0000000000000, float poison>, <2 x float> %x
   ret <2 x float> %select
@@ -246,7 +260,8 @@ define nofpclass(inf) <2 x float> @ret_nofpclass_inf__select_pinf_lhs_vector_und
 define nofpclass(inf) <2 x float> @ret_nofpclass_inf__select_mixed_inf_lhs_vector(<2 x i1> %cond, <2 x float> %x) {
 ; CHECK-LABEL: define nofpclass(inf) <2 x float> @ret_nofpclass_inf__select_mixed_inf_lhs_vector
 ; CHECK-SAME: (<2 x i1> [[COND:%.*]], <2 x float> [[X:%.*]]) {
-; CHECK-NEXT:    ret <2 x float> [[X]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select <2 x i1> [[COND]], <2 x float> <float 0x7FF0000000000000, float 0xFFF0000000000000>, <2 x float> [[X]]
+; CHECK-NEXT:    ret <2 x float> [[SELECT]]
 ;
   %select = select <2 x i1> %cond, <2 x float> <float 0x7FF0000000000000, float 0xFFF0000000000000>, <2 x float> %x
   ret <2 x float> %select
@@ -312,7 +327,8 @@ define nofpclass(nan) float @ret_nofpclass_nan__select_pinf_rhs(i1 %cond, float
 define nofpclass(inf nan) float @ret_nofpclass_inf_nan__select_chain_inf_nan_0(i1 %cond, float %x) {
 ; CHECK-LABEL: define nofpclass(nan inf) float @ret_nofpclass_inf_nan__select_chain_inf_nan_0
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]]) {
-; CHECK-NEXT:    ret float [[X]]
+; CHECK-NEXT:    [[SELECT1:%.*]] = select i1 [[COND]], float 0x7FF0000000000000, float [[X]]
+; CHECK-NEXT:    ret float [[SELECT1]]
 ;
   %select0 = select i1 %cond, float 0x7FF8000000000000, float %x
   %select1 = select i1 %cond, float 0x7FF0000000000000, float %select0
@@ -322,7 +338,8 @@ define nofpclass(inf nan) float @ret_nofpclass_inf_nan__select_chain_inf_nan_0(i
 define nofpclass(inf nan) float @ret_nofpclass_inf_nan__select_chain_inf_nan_1(i1 %cond, float %x) {
 ; CHECK-LABEL: define nofpclass(nan inf) float @ret_nofpclass_inf_nan__select_chain_inf_nan_1
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]]) {
-; CHECK-NEXT:    ret float poison
+; CHECK-NEXT:    [[SELECT1:%.*]] = select i1 [[COND]], float 0x7FF0000000000000, float 0x7FF8000000000000
+; CHECK-NEXT:    ret float [[SELECT1]]
 ;
   %select0 = select i1 %cond, float %x, float 0x7FF8000000000000
   %select1 = select i1 %cond, float 0x7FF0000000000000, float %select0
@@ -343,7 +360,8 @@ define nofpclass(nan) float @ret_nofpclass_nan__select_chain_inf_nan(i1 %cond, f
 define nofpclass(inf) float @ret_nofpclass_inf__select_chain_inf_nan_0(i1 %cond, float %x) {
 ; CHECK-LABEL: define nofpclass(inf) float @ret_nofpclass_inf__select_chain_inf_nan_0
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]]) {
-; CHECK-NEXT:    ret float [[X]]
+; CHECK-NEXT:    [[SELECT1:%.*]] = select i1 [[COND]], float 0x7FF0000000000000, float [[X]]
+; CHECK-NEXT:    ret float [[SELECT1]]
 ;
   %select0 = select i1 %cond, float 0x7FF8000000000000, float %x
   %select1 = select i1 %cond, float 0x7FF0000000000000, float %select0
@@ -353,7 +371,8 @@ define nofpclass(inf) float @ret_nofpclass_inf__select_chain_inf_nan_0(i1 %cond,
 define nofpclass(inf) float @ret_nofpclass_inf__select_chain_inf_nan_1(i1 %cond, float %x) {
 ; CHECK-LABEL: define nofpclass(inf) float @ret_nofpclass_inf__select_chain_inf_nan_1
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]]) {
-; CHECK-NEXT:    ret float 0x7FF8000000000000
+; CHECK-NEXT:    [[SELECT1:%.*]] = select i1 [[COND]], float 0x7FF8000000000000, float 0x7FF0000000000000
+; CHECK-NEXT:    ret float [[SELECT1]]
 ;
   %select0 = select i1 %cond, float 0x7FF8000000000000, float %x
   %select1 = select i1 %cond, float %select0, float 0x7FF0000000000000
@@ -364,7 +383,8 @@ define nofpclass(inf) float @ret_nofpclass_inf__select_chain_inf_nan_1(i1 %cond,
 define nofpclass(inf) float @ret_nofpclass_inf__fabs_select_ninf_rhs(i1 %cond, float %x) {
 ; CHECK-LABEL: define nofpclass(inf) float @ret_nofpclass_inf__fabs_select_ninf_rhs
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]]) {
-; CHECK-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND]], float [[X]], float 0xFFF0000000000000
+; CHECK-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float [[SELECT]])
 ; CHECK-NEXT:    ret float [[FABS]]
 ;
   %select = select i1 %cond, float %x, float 0xFFF0000000000000
@@ -376,7 +396,8 @@ define nofpclass(inf) float @ret_nofpclass_inf__fabs_select_ninf_rhs(i1 %cond, f
 define nofpclass(inf) float @ret_nofpclass_inf__fabs_select_pinf_rhs(i1 %cond, float %x) {
 ; CHECK-LABEL: define nofpclass(inf) float @ret_nofpclass_inf__fabs_select_pinf_rhs
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]]) {
-; CHECK-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND]], float [[X]], float 0x7FF0000000000000
+; CHECK-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float [[SELECT]])
 ; CHECK-NEXT:    ret float [[FABS]]
 ;
   %select = select i1 %cond, float %x, float 0x7FF0000000000000
@@ -400,7 +421,8 @@ define nofpclass(ninf nnorm nsub nzero) float @ret_nofpclass_no_negatives__fabs_
 define nofpclass(pinf pnorm psub pzero) float @ret_nofpclass_no_positives__fabs_select_pinf_rhs(i1 %cond, float %x) {
 ; CHECK-LABEL: define nofpclass(pinf pzero psub pnorm) float @ret_nofpclass_no_positives__fabs_select_pinf_rhs
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]]) {
-; CHECK-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND]], float [[X]], float 0x7FF0000000000000
+; CHECK-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float [[SELECT]])
 ; CHECK-NEXT:    ret float [[FABS]]
 ;
   %select = select i1 %cond, float %x, float 0x7FF0000000000000
@@ -424,7 +446,9 @@ define nofpclass(nan ninf nnorm nsub nzero) float @ret_nofpclass_no_negatives_na
 define nofpclass(nan pinf pnorm psub pzero) float @ret_nofpclass_no_positives_nan__fabs_select_pinf_rhs(i1 %cond, float %x) {
 ; CHECK-LABEL: define nofpclass(nan pinf pzero psub pnorm) float @ret_nofpclass_no_positives_nan__fabs_select_pinf_rhs
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]]) {
-; CHECK-NEXT:    ret float poison
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND]], float [[X]], float 0x7FF0000000000000
+; CHECK-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float [[SELECT]])
+; CHECK-NEXT:    ret float [[FABS]]
 ;
   %select = select i1 %cond, float %x, float 0x7FF0000000000000
   %fabs = call float @llvm.fabs.f32(float %select)
@@ -435,7 +459,8 @@ define nofpclass(nan pinf pnorm psub pzero) float @ret_nofpclass_no_positives_na
 define nofpclass(inf) float @ret_nofpclass_inf__fneg_select_ninf_rhs(i1 %cond, float %x) {
 ; CHECK-LABEL: define nofpclass(inf) float @ret_nofpclass_inf__fneg_select_ninf_rhs
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]]) {
-; CHECK-NEXT:    [[FNEG:%.*]] = fneg float [[X]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND]], float [[X]], float 0xFFF0000000000000
+; CHECK-NEXT:    [[FNEG:%.*]] = fneg float [[SELECT]]
 ; CHECK-NEXT:    ret float [[FNEG]]
 ;
   %select = select i1 %cond, float %x, float 0xFFF0000000000000
@@ -447,7 +472,8 @@ define nofpclass(inf) float @ret_nofpclass_inf__fneg_select_ninf_rhs(i1 %cond, f
 define nofpclass(inf nnorm nsub nzero) float @ret_nofpclass_nonegatives_noinf___fneg_select_pinf_rhs(i1 %cond, float %x) {
 ; CHECK-LABEL: define nofpclass(inf nzero nsub nnorm) float @ret_nofpclass_nonegatives_noinf___fneg_select_pinf_rhs
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]]) {
-; CHECK-NEXT:    [[FNEG:%.*]] = fneg float [[X]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND]], float [[X]], float 0x7FF0000000000000
+; CHECK-NEXT:    [[FNEG:%.*]] = fneg float [[SELECT]]
 ; CHECK-NEXT:    ret float [[FNEG]]
 ;
   %select = select i1 %cond, float %x, float 0x7FF0000000000000
@@ -459,7 +485,8 @@ define nofpclass(inf nnorm nsub nzero) float @ret_nofpclass_nonegatives_noinf___
 define nofpclass(inf nnorm nsub nzero) float @ret_nofpclass_nonegatives_noinf___fneg_select_ninf_lhs(i1 %cond, float %x) {
 ; CHECK-LABEL: define nofpclass(inf nzero nsub nnorm) float @ret_nofpclass_nonegatives_noinf___fneg_select_ninf_lhs
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]]) {
-; CHECK-NEXT:    [[FNEG:%.*]] = fneg float [[X]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND]], float 0xFFF0000000000000, float [[X]]
+; CHECK-NEXT:    [[FNEG:%.*]] = fneg float [[SELECT]]
 ; CHECK-NEXT:    ret float [[FNEG]]
 ;
   %select = select i1 %cond, float 0xFFF0000000000000, float %x
@@ -483,7 +510,8 @@ define nofpclass(pzero psub pnorm pinf) float @ret_nofpclass_nopositives___fneg_
 define nofpclass(inf) float @ret_nofpclass_inf__fneg_fabs_select_pinf_rhs(i1 %cond, float %x) {
 ; CHECK-LABEL: define nofpclass(inf) float @ret_nofpclass_inf__fneg_fabs_select_pinf_rhs
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]]) {
-; CHECK-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND]], float [[X]], float 0x7FF0000000000000
+; CHECK-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float [[SELECT]])
 ; CHECK-NEXT:    [[FNEG:%.*]] = fneg float [[FABS]]
 ; CHECK-NEXT:    ret float [[FNEG]]
 ;
@@ -497,7 +525,8 @@ define nofpclass(inf) float @ret_nofpclass_inf__fneg_fabs_select_pinf_rhs(i1 %co
 define nofpclass(ninf nnorm nsub nzero) float @ret_nofpclass_nonegatives__fneg_fabs_select_pinf_rhs(i1 %cond, float %x) {
 ; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) float @ret_nofpclass_nonegatives__fneg_fabs_select_pinf_rhs
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]]) {
-; CHECK-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND]], float [[X]], float 0x7FF0000000000000
+; CHECK-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float [[SELECT]])
 ; CHECK-NEXT:    [[FNEG:%.*]] = fneg float [[FABS]]
 ; CHECK-NEXT:    ret float [[FNEG]]
 ;
@@ -512,7 +541,10 @@ define nofpclass(ninf nnorm nsub nzero) float @ret_nofpclass_nonegatives__fneg_f
 define nofpclass(nan ninf nnorm nsub nzero) float @ret_nofpclass_nonegatives_nonan__fneg_fabs_select_pinf_rhs(i1 %cond, float %x) {
 ; CHECK-LABEL: define nofpclass(nan ninf nzero nsub nnorm) float @ret_nofpclass_nonegatives_nonan__fneg_fabs_select_pinf_rhs
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]]) {
-; CHECK-NEXT:    ret float poison
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND]], float [[X]], float 0x7FF0000000000000
+; CHECK-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float [[SELECT]])
+; CHECK-NEXT:    [[FNEG:%.*]] = fneg float [[FABS]]
+; CHECK-NEXT:    ret float [[FNEG]]
 ;
   %select = select i1 %cond, float %x, float 0x7FF0000000000000
   %fabs = call float @llvm.fabs.f32(float %select)
@@ -524,7 +556,8 @@ define nofpclass(nan ninf nnorm nsub nzero) float @ret_nofpclass_nonegatives_non
 define nofpclass(inf) float @ret_nofpclass_inf__copysign_unknown_select_pinf_rhs(i1 %cond, float %x, float %unknown.sign) {
 ; CHECK-LABEL: define nofpclass(inf) float @ret_nofpclass_inf__copysign_unknown_select_pinf_rhs
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]], float [[UNKNOWN_SIGN:%.*]]) {
-; CHECK-NEXT:    [[COPYSIGN:%.*]] = call float @llvm.copysign.f32(float [[X]], float [[UNKNOWN_SIGN]])
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND]], float [[X]], float 0x7FF0000000000000
+; CHECK-NEXT:    [[COPYSIGN:%.*]] = call float @llvm.copysign.f32(float [[SELECT]], float [[UNKNOWN_SIGN]])
 ; CHECK-NEXT:    ret float [[COPYSIGN]]
 ;
   %select = select i1 %cond, float %x, float 0x7FF0000000000000
@@ -535,7 +568,8 @@ define nofpclass(inf) float @ret_nofpclass_inf__copysign_unknown_select_pinf_rhs
 define nofpclass(inf) float @ret_nofpclass_inf__copysign_positive_select_pinf_rhs(i1 %cond, float %x) {
 ; CHECK-LABEL: define nofpclass(inf) float @ret_nofpclass_inf__copysign_positive_select_pinf_rhs
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]]) {
-; CHECK-NEXT:    [[COPYSIGN:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND]], float [[X]], float 0x7FF0000000000000
+; CHECK-NEXT:    [[COPYSIGN:%.*]] = call float @llvm.fabs.f32(float [[SELECT]])
 ; CHECK-NEXT:    ret float [[COPYSIGN]]
 ;
   %select = select i1 %cond, float %x, float 0x7FF0000000000000
@@ -546,7 +580,8 @@ define nofpclass(inf) float @ret_nofpclass_inf__copysign_positive_select_pinf_rh
 define nofpclass(inf) float @ret_nofpclass_inf__copysign_negative_select_pinf_rhs(i1 %cond, float %x) {
 ; CHECK-LABEL: define nofpclass(inf) float @ret_nofpclass_inf__copysign_negative_select_pinf_rhs
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND]], float [[X]], float 0x7FF0000000000000
+; CHECK-NEXT:    [[TMP1:%.*]] = call float @llvm.fabs.f32(float [[SELECT]])
 ; CHECK-NEXT:    [[COPYSIGN:%.*]] = fneg float [[TMP1]]
 ; CHECK-NEXT:    ret float [[COPYSIGN]]
 ;
@@ -559,8 +594,7 @@ define nofpclass(inf) float @ret_nofpclass_inf__copysign_negative_select_pinf_rh
 define nofpclass(pinf pnorm psub pzero) float @ret_nofpclass_nopositives_copysign(float %x, float %unknown.sign) {
 ; CHECK-LABEL: define nofpclass(pinf pzero psub pnorm) float @ret_nofpclass_nopositives_copysign
 ; CHECK-SAME: (float [[X:%.*]], float [[UNKNOWN_SIGN:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = call float @llvm.fabs.f32(float [[X]])
-; CHECK-NEXT:    [[COPYSIGN:%.*]] = fneg float [[TMP1]]
+; CHECK-NEXT:    [[COPYSIGN:%.*]] = call float @llvm.copysign.f32(float [[X]], float [[UNKNOWN_SIGN]])
 ; CHECK-NEXT:    ret float [[COPYSIGN]]
 ;
   %copysign = call float @llvm.copysign.f32(float %x, float %unknown.sign)
@@ -571,8 +605,7 @@ define nofpclass(pinf pnorm psub pzero) float @ret_nofpclass_nopositives_copysig
 define nofpclass(pinf pnorm psub pzero) float @ret_nofpclass_nopositives_copysign_nnan_flag(float %x, float %unknown.sign) {
 ; CHECK-LABEL: define nofpclass(pinf pzero psub pnorm) float @ret_nofpclass_nopositives_copysign_nnan_flag
 ; CHECK-SAME: (float [[X:%.*]], float [[UNKNOWN_SIGN:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = call nnan float @llvm.fabs.f32(float [[X]])
-; CHECK-NEXT:    [[COPYSIGN:%.*]] = fneg nnan float [[TMP1]]
+; CHECK-NEXT:    [[COPYSIGN:%.*]] = call nnan float @llvm.copysign.f32(float [[X]], float [[UNKNOWN_SIGN]])
 ; CHECK-NEXT:    ret float [[COPYSIGN]]
 ;
   %copysign = call nnan float @llvm.copysign.f32(float %x, float %unknown.sign)
@@ -583,8 +616,7 @@ define nofpclass(pinf pnorm psub pzero) float @ret_nofpclass_nopositives_copysig
 define nofpclass(nan pinf pnorm psub pzero) float @ret_nofpclass_nopositives_nonan_copysign(float %x, float %unknown.sign) {
 ; CHECK-LABEL: define nofpclass(nan pinf pzero psub pnorm) float @ret_nofpclass_nopositives_nonan_copysign
 ; CHECK-SAME: (float [[X:%.*]], float [[UNKNOWN_SIGN:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = call float @llvm.fabs.f32(float [[X]])
-; CHECK-NEXT:    [[COPYSIGN:%.*]] = fneg float [[TMP1]]
+; CHECK-NEXT:    [[COPYSIGN:%.*]] = call float @llvm.copysign.f32(float [[X]], float [[UNKNOWN_SIGN]])
 ; CHECK-NEXT:    ret float [[COPYSIGN]]
 ;
   %copysign = call float @llvm.copysign.f32(float %x, float %unknown.sign)
@@ -595,7 +627,7 @@ define nofpclass(nan pinf pnorm psub pzero) float @ret_nofpclass_nopositives_non
 define nofpclass(ninf nnorm nsub nzero) float @ret_nofpclass_nonegatives_copysign(float %x, float %unknown.sign) {
 ; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) float @ret_nofpclass_nonegatives_copysign
 ; CHECK-SAME: (float [[X:%.*]], float [[UNKNOWN_SIGN:%.*]]) {
-; CHECK-NEXT:    [[COPYSIGN:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT:    [[COPYSIGN:%.*]] = call float @llvm.copysign.f32(float [[X]], float [[UNKNOWN_SIGN]])
 ; CHECK-NEXT:    ret float [[COPYSIGN]]
 ;
   %copysign = call float @llvm.copysign.f32(float %x, float %unknown.sign)
@@ -606,7 +638,7 @@ define nofpclass(ninf nnorm nsub nzero) float @ret_nofpclass_nonegatives_copysig
 define nofpclass(ninf nnorm nsub nzero) float @ret_nofpclass_nonegatives_copysign_nnan_flag(float %x, float %unknown.sign) {
 ; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) float @ret_nofpclass_nonegatives_copysign_nnan_flag
 ; CHECK-SAME: (float [[X:%.*]], float [[UNKNOWN_SIGN:%.*]]) {
-; CHECK-NEXT:    [[COPYSIGN:%.*]] = call nnan float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT:    [[COPYSIGN:%.*]] = call nnan float @llvm.copysign.f32(float [[X]], float [[UNKNOWN_SIGN]])
 ; CHECK-NEXT:    ret float [[COPYSIGN]]
 ;
   %copysign = call nnan float @llvm.copysign.f32(float %x, float %unknown.sign)
@@ -617,7 +649,7 @@ define nofpclass(ninf nnorm nsub nzero) float @ret_nofpclass_nonegatives_copysig
 define nofpclass(nan ninf nnorm nsub nzero) float @ret_nofpclass_nonegatives_nonan_copysign(float %x, float %unknown.sign) {
 ; CHECK-LABEL: define nofpclass(nan ninf nzero nsub nnorm) float @ret_nofpclass_nonegatives_nonan_copysign
 ; CHECK-SAME: (float [[X:%.*]], float [[UNKNOWN_SIGN:%.*]]) {
-; CHECK-NEXT:    [[COPYSIGN:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT:    [[COPYSIGN:%.*]] = call float @llvm.copysign.f32(float [[X]], float [[UNKNOWN_SIGN]])
 ; CHECK-NEXT:    ret float [[COPYSIGN]]
 ;
   %copysign = call float @llvm.copysign.f32(float %x, float %unknown.sign)
@@ -627,7 +659,8 @@ define nofpclass(nan ninf nnorm nsub nzero) float @ret_nofpclass_nonegatives_non
 define nofpclass(pinf pnorm psub pzero) float @ret_nofpclass_nopositives__copysign_fabs_select_pinf_rhs(i1 %cond, float %x, float %sign) {
 ; CHECK-LABEL: define nofpclass(pinf pzero psub pnorm) float @ret_nofpclass_nopositives__copysign_fabs_select_pinf_rhs
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]], float [[SIGN:%.*]]) {
-; CHECK-NEXT:    [[COPYSIGN:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND]], float [[X]], float 0x7FF0000000000000
+; CHECK-NEXT:    [[COPYSIGN:%.*]] = call float @llvm.fabs.f32(float [[SELECT]])
 ; CHECK-NEXT:    ret float [[COPYSIGN]]
 ;
   %select = select i1 %cond, float %x, float 0x7FF0000000000000
@@ -640,7 +673,8 @@ define nofpclass(pinf pnorm psub pzero) float @ret_nofpclass_nopositives__copysi
 define nofpclass(inf nnorm nsub nzero) float @ret_nofpclass_no_negatives_noinf__copysign_unknown_select_pinf_rhs(i1 %cond, float %x, float %unknown.sign) {
 ; CHECK-LABEL: define nofpclass(inf nzero nsub nnorm) float @ret_nofpclass_no_negatives_noinf__copysign_unknown_select_pinf_rhs
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]], float [[UNKNOWN_SIGN:%.*]]) {
-; CHECK-NEXT:    [[COPYSIGN:%.*]] = call float @llvm.fabs.f32(float [[X]])
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND]], float [[X]], float 0x7FF0000000000000
+; CHECK-NEXT:    [[COPYSIGN:%.*]] = call float @llvm.copysign.f32(float [[SELECT]], float [[UNKNOWN_SIGN]])
 ; CHECK-NEXT:    ret float [[COPYSIGN]]
 ;
   %select = select i1 %cond, float %x, float 0x7FF0000000000000
@@ -652,8 +686,8 @@ define nofpclass(inf nnorm nsub nzero) float @ret_nofpclass_no_negatives_noinf__
 define nofpclass(inf pnorm psub pzero) float @ret_nofpclass_no_positives_noinf__copysign_unknown_select_pinf_rhs(i1 %cond, float %x, float %unknown.sign) {
 ; CHECK-LABEL: define nofpclass(inf pzero psub pnorm) float @ret_nofpclass_no_positives_noinf__copysign_unknown_select_pinf_rhs
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]], float [[UNKNOWN_SIGN:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = call float @llvm.fabs.f32(float [[X]])
-; CHECK-NEXT:    [[COPYSIGN:%.*]] = fneg float [[TMP1]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND]], float [[X]], float 0x7FF0000000000000
+; CHECK-NEXT:    [[COPYSIGN:%.*]] = call float @llvm.copysign.f32(float [[SELECT]], float [[UNKNOWN_SIGN]])
 ; CHECK-NEXT:    ret float [[COPYSIGN]]
 ;
   %select = select i1 %cond, float %x, float 0x7FF0000000000000
@@ -666,7 +700,7 @@ define nofpclass(ninf nnorm nsub nzero) float @ret_nofpclass_no_negatives__copys
 ; CHECK-LABEL: define nofpclass(ninf nzero nsub nnorm) float @ret_nofpclass_no_negatives__copysign_unknown_select_pinf_rhs
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]], float [[UNKNOWN_SIGN:%.*]]) {
 ; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND]], float [[X]], float 0x7FF0000000000000
-; CHECK-NEXT:    [[COPYSIGN:%.*]] = call float @llvm.fabs.f32(float [[SELECT]])
+; CHECK-NEXT:    [[COPYSIGN:%.*]] = call float @llvm.copysign.f32(float [[SELECT]], float [[UNKNOWN_SIGN]])
 ; CHECK-NEXT:    ret float [[COPYSIGN]]
 ;
   %select = select i1 %cond, float %x, float 0x7FF0000000000000
@@ -679,8 +713,7 @@ define nofpclass(pinf pnorm psub pzero) float @ret_nofpclass_no_positives__copys
 ; CHECK-LABEL: define nofpclass(pinf pzero psub pnorm) float @ret_nofpclass_no_positives__copysign_unknown_select_pinf_rhs
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]], float [[UNKNOWN_SIGN:%.*]]) {
 ; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND]], float [[X]], float 0x7FF0000000000000
-; CHECK-NEXT:    [[TMP1:%.*]] = call float @llvm.fabs.f32(float [[SELECT]])
-; CHECK-NEXT:    [[COPYSIGN:%.*]] = fneg float [[TMP1]]
+; CHECK-NEXT:    [[COPYSIGN:%.*]] = call float @llvm.copysign.f32(float [[SELECT]], float [[UNKNOWN_SIGN]])
 ; CHECK-NEXT:    ret float [[COPYSIGN]]
 ;
   %select = select i1 %cond, float %x, float 0x7FF0000000000000
@@ -693,7 +726,7 @@ define nofpclass(nan ninf nnorm nsub nzero) float @ret_nofpclass_no_negatives_no
 ; CHECK-LABEL: define nofpclass(nan ninf nzero nsub nnorm) float @ret_nofpclass_no_negatives_nonan__copysign_unknown_select_pinf_rhs
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]], float [[UNKNOWN_SIGN:%.*]]) {
 ; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND]], float [[X]], float 0x7FF0000000000000
-; CHECK-NEXT:    [[COPYSIGN:%.*]] = call float @llvm.fabs.f32(float [[SELECT]])
+; CHECK-NEXT:    [[COPYSIGN:%.*]] = call float @llvm.copysign.f32(float [[SELECT]], float [[UNKNOWN_SIGN]])
 ; CHECK-NEXT:    ret float [[COPYSIGN]]
 ;
   %select = select i1 %cond, float %x, float 0x7FF0000000000000
@@ -706,8 +739,7 @@ define nofpclass(nan pinf pnorm psub pzero) float @ret_nofpclass_no_positives_no
 ; CHECK-LABEL: define nofpclass(nan pinf pzero psub pnorm) float @ret_nofpclass_no_positives_nonan__copysign_unknown_select_pinf_rhs
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]], float [[UNKNOWN_SIGN:%.*]]) {
 ; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND]], float [[X]], float 0x7FF0000000000000
-; CHECK-NEXT:    [[TMP1:%.*]] = call float @llvm.fabs.f32(float [[SELECT]])
-; CHECK-NEXT:    [[COPYSIGN:%.*]] = fneg float [[TMP1]]
+; CHECK-NEXT:    [[COPYSIGN:%.*]] = call float @llvm.copysign.f32(float [[SELECT]], float [[UNKNOWN_SIGN]])
 ; CHECK-NEXT:    ret float [[COPYSIGN]]
 ;
   %select = select i1 %cond, float %x, float 0x7FF0000000000000
@@ -758,7 +790,9 @@ define nofpclass(nan ninf nnorm nsub nzero) float @ret_nofpclass_nan_negatives__
 define nofpclass(nan ninf nnorm nsub zero) float @ret_nofpclass_nan_negatives_zero__select_clamp_pos_to_zero(float %x) {
 ; CHECK-LABEL: define nofpclass(nan ninf zero nsub nnorm) float @ret_nofpclass_nan_negatives_zero__select_clamp_pos_to_zero
 ; CHECK-SAME: (float [[X:%.*]]) {
-; CHECK-NEXT:    ret float [[X]]
+; CHECK-NEXT:    [[IS_GT_ZERO:%.*]] = fcmp ogt float [[X]], 0.000000e+00
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[IS_GT_ZERO]], float 0.000000e+00, float [[X]]
+; CHECK-NEXT:    ret float [[SELECT]]
 ;
   %is.gt.zero = fcmp ogt float %x, 0.0
   %select = select i1 %is.gt.zero, float 0.0, float %x
@@ -769,7 +803,9 @@ define nofpclass(nan ninf nnorm nsub zero) float @ret_nofpclass_nan_negatives_ze
 define nofpclass(ninf nnorm nsub zero) float @ret_nofpclass_negatives_zero__select_clamp_pos_to_zero(float %x) {
 ; CHECK-LABEL: define nofpclass(ninf zero nsub nnorm) float @ret_nofpclass_negatives_zero__select_clamp_pos_to_zero
 ; CHECK-SAME: (float [[X:%.*]]) {
-; CHECK-NEXT:    ret float [[X]]
+; CHECK-NEXT:    [[IS_GT_ZERO:%.*]] = fcmp ogt float [[X]], 0.000000e+00
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[IS_GT_ZERO]], float 0.000000e+00, float [[X]]
+; CHECK-NEXT:    ret float [[SELECT]]
 ;
   %is.gt.zero = fcmp ogt float %x, 0.0
   %select = select i1 %is.gt.zero, float 0.0, float %x
@@ -783,7 +819,8 @@ define nofpclass(inf) float @ret_nofpclass_noinfs__assumed_isinf__select_pinf_lh
 ; CHECK-NEXT:    [[FABS_X:%.*]] = call float @llvm.fabs.f32(float [[X]])
 ; CHECK-NEXT:    [[X_IS_INF:%.*]] = fcmp oeq float [[FABS_X]], 0x7FF0000000000000
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[X_IS_INF]])
-; CHECK-NEXT:    ret float [[Y]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND]], float [[X]], float [[Y]]
+; CHECK-NEXT:    ret float [[SELECT]]
 ;
   %fabs.x = call float @llvm.fabs.f32(float %x)
   %x.is.inf = fcmp oeq float %fabs.x, 0x7FF0000000000000
@@ -801,13 +838,18 @@ define nofpclass(nan inf nzero nsub nnorm) float @powr_issue64870(float nofpclas
 ; CHECK-NEXT:    [[I1:%.*]] = tail call float @llvm.log2.f32(float [[I]])
 ; CHECK-NEXT:    [[I2:%.*]] = fmul float [[I1]], [[Y]]
 ; CHECK-NEXT:    [[I3:%.*]] = tail call nofpclass(ninf nzero nsub nnorm) float @llvm.exp2.f32(float [[I2]])
+; CHECK-NEXT:    [[I4:%.*]] = fcmp olt float [[Y]], 0.000000e+00
+; CHECK-NEXT:    [[I5:%.*]] = select i1 [[I4]], float 0x7FF0000000000000, float 0.000000e+00
 ; CHECK-NEXT:    [[I6:%.*]] = fcmp oeq float [[X]], 0.000000e+00
-; CHECK-NEXT:    [[I7:%.*]] = select i1 [[I6]], float 0.000000e+00, float [[I3]]
+; CHECK-NEXT:    [[I7:%.*]] = select i1 [[I6]], float [[I5]], float [[I3]]
 ; CHECK-NEXT:    [[I8:%.*]] = fcmp oeq float [[Y]], 0.000000e+00
+; CHECK-NEXT:    [[I9:%.*]] = select i1 [[I6]], float 0x7FF8000000000000, float 1.000000e+00
+; CHECK-NEXT:    [[I10:%.*]] = select i1 [[I8]], float [[I9]], float [[I7]]
 ; CHECK-NEXT:    [[I11:%.*]] = fcmp oeq float [[X]], 1.000000e+00
-; CHECK-NEXT:    [[TMP0:%.*]] = select i1 [[I11]], i1 true, i1 [[I8]]
-; CHECK-NEXT:    [[I12:%.*]] = select i1 [[TMP0]], float 1.000000e+00, float [[I7]]
-; CHECK-NEXT:    ret float [[I12]]
+; CHECK-NEXT:    [[I12:%.*]] = select i1 [[I11]], float 1.000000e+00, float [[I10]]
+; CHECK-NEXT:    [[I13:%.*]] = fcmp olt float [[X]], 0.000000e+00
+; CHECK-NEXT:    [[I14:%.*]] = select i1 [[I13]], float 0x7FF8000000000000, float [[I12]]
+; CHECK-NEXT:    ret float [[I14]]
 ;
 entry:
   %i = tail call float @llvm.fabs.f32(float %x)
@@ -839,8 +881,12 @@ define nofpclass(nan inf nzero nsub nnorm) float @test_powr_issue64870_2(float n
 ; CHECK-NEXT:    [[I4:%.*]] = select i1 [[I]], float 0x7FF8000000000000, float [[ARG1]]
 ; CHECK-NEXT:    [[I5:%.*]] = fmul float [[I4]], [[I3]]
 ; CHECK-NEXT:    [[I6:%.*]] = tail call noundef nofpclass(ninf nzero nsub nnorm) float @llvm.exp2.f32(float noundef [[I5]])
+; CHECK-NEXT:    [[I7:%.*]] = fcmp olt float [[I4]], 0.000000e+00
+; CHECK-NEXT:    [[I8:%.*]] = select i1 [[I7]], float 0x7FF0000000000000, float 0.000000e+00
+; CHECK-NEXT:    [[I9:%.*]] = fcmp ueq float [[I4]], 0.000000e+00
 ; CHECK-NEXT:    [[I10:%.*]] = fcmp oeq float [[I2]], 0.000000e+00
-; CHECK-NEXT:    [[I12:%.*]] = select i1 [[I10]], float 0.000000e+00, float [[I6]]
+; CHECK-NEXT:    [[I11:%.*]] = select i1 [[I9]], float 0x7FF8000000000000, float [[I8]]
+; CHECK-NEXT:    [[I12:%.*]] = select i1 [[I10]], float [[I11]], float [[I6]]
 ; CHECK-NEXT:    ret float [[I12]]
 ;
 bb:
@@ -877,10 +923,16 @@ define nofpclass(nan inf) float @pow_f32(float nofpclass(nan inf) %arg, float no
 ; CHECK-NEXT:    [[I11:%.*]] = and i1 [[I7]], [[I10]]
 ; CHECK-NEXT:    [[I12:%.*]] = select i1 [[I11]], float [[ARG]], float 1.000000e+00
 ; CHECK-NEXT:    [[I13:%.*]] = tail call noundef float @llvm.copysign.f32(float noundef [[I4]], float noundef [[I12]])
+; CHECK-NEXT:    [[I14:%.*]] = fcmp olt float [[ARG]], 0.000000e+00
+; CHECK-NEXT:    [[I15:%.*]] = select i1 [[I7]], float [[I13]], float 0x7FF8000000000000
+; CHECK-NEXT:    [[I16:%.*]] = select i1 [[I14]], float [[I15]], float [[I13]]
 ; CHECK-NEXT:    [[I17:%.*]] = fcmp oeq float [[ARG]], 0.000000e+00
+; CHECK-NEXT:    [[I18:%.*]] = fcmp olt float [[ARG1]], 0.000000e+00
+; CHECK-NEXT:    [[I19:%.*]] = xor i1 [[I17]], [[I18]]
+; CHECK-NEXT:    [[I20:%.*]] = select i1 [[I19]], float 0.000000e+00, float 0x7FF0000000000000
 ; CHECK-NEXT:    [[I21:%.*]] = select i1 [[I11]], float [[ARG]], float 0.000000e+00
-; CHECK-NEXT:    [[I22:%.*]] = tail call noundef nofpclass(nan sub norm) float @llvm.copysign.f32(float noundef 0.000000e+00, float noundef [[I21]])
-; CHECK-NEXT:    [[I23:%.*]] = select i1 [[I17]], float [[I22]], float [[I13]]
+; CHECK-NEXT:    [[I22:%.*]] = tail call noundef nofpclass(nan sub norm) float @llvm.copysign.f32(float noundef [[I20]], float noundef [[I21]])
+; CHECK-NEXT:    [[I23:%.*]] = select i1 [[I17]], float [[I22]], float [[I16]]
 ; CHECK-NEXT:    [[I24:%.*]] = fcmp oeq float [[ARG]], 1.000000e+00
 ; CHECK-NEXT:    [[I25:%.*]] = fcmp oeq float [[ARG1]], 0.000000e+00
 ; CHECK-NEXT:    [[I26:%.*]] = or i1 [[I24]], [[I25]]
@@ -925,7 +977,8 @@ define nofpclass(inf) float @ret_nofpclass_inf__select_nofpclass_call_only_inf(i
 ; CHECK-LABEL: define nofpclass(inf) float @ret_nofpclass_inf__select_nofpclass_call_only_inf
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[Y:%.*]]) {
 ; CHECK-NEXT:    [[MUST_BE_INF:%.*]] = call nofpclass(nan zero sub norm) float @extern()
-; CHECK-NEXT:    ret float [[Y]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND]], float [[MUST_BE_INF]], float [[Y]]
+; CHECK-NEXT:    ret float [[SELECT]]
 ;
   %must.be.inf = call nofpclass(nan norm zero sub) float @extern()
   %select = select i1 %cond, float %must.be.inf, float %y
@@ -936,7 +989,7 @@ define nofpclass(pinf) float @ret_nofpclass_pinf__nofpclass_call_only_inf(i1 %co
 ; CHECK-LABEL: define nofpclass(pinf) float @ret_nofpclass_pinf__nofpclass_call_only_inf
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[Y:%.*]]) {
 ; CHECK-NEXT:    [[MUST_BE_INF:%.*]] = call nofpclass(nan zero sub norm) float @extern()
-; CHECK-NEXT:    ret float 0xFFF0000000000000
+; CHECK-NEXT:    ret float [[MUST_BE_INF]]
 ;
   %must.be.inf = call nofpclass(nan norm zero sub) float @extern()
   ret float %must.be.inf
@@ -946,7 +999,7 @@ define nofpclass(ninf) float @ret_nofpclass_ninf__nofpclass_call_only_inf(i1 %co
 ; CHECK-LABEL: define nofpclass(ninf) float @ret_nofpclass_ninf__nofpclass_call_only_inf
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[Y:%.*]]) {
 ; CHECK-NEXT:    [[MUST_BE_INF:%.*]] = call nofpclass(nan zero sub norm) float @extern()
-; CHECK-NEXT:    ret float 0x7FF0000000000000
+; CHECK-NEXT:    ret float [[MUST_BE_INF]]
 ;
   %must.be.inf = call nofpclass(nan norm zero sub) float @extern()
   ret float %must.be.inf
@@ -956,7 +1009,7 @@ define nofpclass(nzero) float @ret_nofpclass_nzero__nofpclass_call_only_zero(i1
 ; CHECK-LABEL: define nofpclass(nzero) float @ret_nofpclass_nzero__nofpclass_call_only_zero
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[Y:%.*]]) {
 ; CHECK-NEXT:    [[MUST_BE_ZERO:%.*]] = call nofpclass(nan inf sub norm) float @extern()
-; CHECK-NEXT:    ret float 0.000000e+00
+; CHECK-NEXT:    ret float [[MUST_BE_ZERO]]
 ;
   %must.be.zero = call nofpclass(nan sub norm inf) float @extern()
   ret float %must.be.zero
@@ -966,7 +1019,7 @@ define nofpclass(pzero) float @ret_nofpclass_pzero__nofpclass_call_only_zero(i1
 ; CHECK-LABEL: define nofpclass(pzero) float @ret_nofpclass_pzero__nofpclass_call_only_zero
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[Y:%.*]]) {
 ; CHECK-NEXT:    [[MUST_BE_ZERO:%.*]] = call nofpclass(nan inf sub norm) float @extern()
-; CHECK-NEXT:    ret float -0.000000e+00
+; CHECK-NEXT:    ret float [[MUST_BE_ZERO]]
 ;
   %must.be.zero = call nofpclass(nan sub norm inf) float @extern()
   ret float %must.be.zero
@@ -1080,7 +1133,8 @@ define nofpclass(inf) float @ret_nofpclass_inf__recursive_phi_0(i1 %cond0, float
 ; CHECK-NEXT:    [[LOOP_COND:%.*]] = call i1 @loop.cond()
 ; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[RET]], label [[LOOP]]
 ; CHECK:       ret:
-; CHECK-NEXT:    ret float 0.000000e+00
+; CHECK-NEXT:    [[PHI_RET:%.*]] = phi float [ 0.000000e+00, [[ENTRY:%.*]] ], [ 0x7FF0000000000000, [[LOOP]] ]
+; CHECK-NEXT:    ret float [[PHI_RET]]
 ;
 entry:
   br i1 %cond0, label %loop, label %ret
@@ -1105,7 +1159,7 @@ define nofpclass(inf) float @ret_nofpclass_inf__recursive_phi_1(i1 %cond0, float
 ; CHECK-NEXT:    [[LOOP_COND:%.*]] = call i1 @loop.cond()
 ; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[RET]], label [[LOOP]]
 ; CHECK:       ret:
-; CHECK-NEXT:    ret float poison
+; CHECK-NEXT:    ret float 0x7FF0000000000000
 ;
 entry:
   br i1 %cond0, label %loop, label %ret
@@ -1157,7 +1211,8 @@ ret:
 define nofpclass(inf) float @ret_nofpclass_inf__arithmetic_fence_select_pinf_rhs(i1 %cond, float %x) {
 ; CHECK-LABEL: define nofpclass(inf) float @ret_nofpclass_inf__arithmetic_fence_select_pinf_rhs
 ; CHECK-SAME: (i1 [[COND:%.*]], float [[X:%.*]]) {
-; CHECK-NEXT:    [[FENCE:%.*]] = call float @llvm.arithmetic.fence.f32(float [[X]])
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[COND]], float [[X]], float 0x7FF0000000000000
+; CHECK-NEXT:    [[FENCE:%.*]] = call float @llvm.arithmetic.fence.f32(float [[SELECT]])
 ; CHECK-NEXT:    ret float [[FENCE]]
 ;
   %select = select i1 %cond, float %x, float 0x7FF0000000000000
diff --git a/llvm/test/Transforms/InstCombine/trunc-inseltpoison.ll b/llvm/test/Transforms/InstCombine/trunc-inseltpoison.ll
index ac0115a0f5..b5dcb9b67d 100644
--- a/llvm/test/Transforms/InstCombine/trunc-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/trunc-inseltpoison.ll
@@ -345,7 +345,7 @@ define i64 @test11(i32 %A, i32 %B) {
 ; CHECK-NEXT:    [[C:%.*]] = zext i32 [[A:%.*]] to i64
 ; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[B:%.*]], 31
 ; CHECK-NEXT:    [[E:%.*]] = zext i32 [[TMP1]] to i64
-; CHECK-NEXT:    [[F:%.*]] = shl i64 [[C]], [[E]]
+; CHECK-NEXT:    [[F:%.*]] = shl nuw nsw i64 [[C]], [[E]]
 ; CHECK-NEXT:    ret i64 [[F]]
 ;
   %C = zext i32 %A to i128
@@ -361,7 +361,7 @@ define <2 x i64> @test11_vec(<2 x i32> %A, <2 x i32> %B) {
 ; CHECK-NEXT:    [[C:%.*]] = zext <2 x i32> [[A:%.*]] to <2 x i64>
 ; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[B:%.*]], <i32 31, i32 31>
 ; CHECK-NEXT:    [[E:%.*]] = zext <2 x i32> [[TMP1]] to <2 x i64>
-; CHECK-NEXT:    [[F:%.*]] = shl <2 x i64> [[C]], [[E]]
+; CHECK-NEXT:    [[F:%.*]] = shl nuw nsw <2 x i64> [[C]], [[E]]
 ; CHECK-NEXT:    ret <2 x i64> [[F]]
 ;
   %C = zext <2 x i32> %A to <2 x i128>
@@ -377,7 +377,7 @@ define <2 x i64> @test11_vec_nonuniform(<2 x i32> %A, <2 x i32> %B) {
 ; CHECK-NEXT:    [[C:%.*]] = zext <2 x i32> [[A:%.*]] to <2 x i64>
 ; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[B:%.*]], <i32 31, i32 15>
 ; CHECK-NEXT:    [[E:%.*]] = zext <2 x i32> [[TMP1]] to <2 x i64>
-; CHECK-NEXT:    [[F:%.*]] = shl <2 x i64> [[C]], [[E]]
+; CHECK-NEXT:    [[F:%.*]] = shl nuw nsw <2 x i64> [[C]], [[E]]
 ; CHECK-NEXT:    ret <2 x i64> [[F]]
 ;
   %C = zext <2 x i32> %A to <2 x i128>
diff --git a/llvm/test/Transforms/InstCombine/trunc.ll b/llvm/test/Transforms/InstCombine/trunc.ll
index e04bcaf073..33baee8584 100644
--- a/llvm/test/Transforms/InstCombine/trunc.ll
+++ b/llvm/test/Transforms/InstCombine/trunc.ll
@@ -345,7 +345,7 @@ define i64 @test11(i32 %A, i32 %B) {
 ; CHECK-NEXT:    [[C:%.*]] = zext i32 [[A:%.*]] to i64
 ; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[B:%.*]], 31
 ; CHECK-NEXT:    [[E:%.*]] = zext i32 [[TMP1]] to i64
-; CHECK-NEXT:    [[F:%.*]] = shl i64 [[C]], [[E]]
+; CHECK-NEXT:    [[F:%.*]] = shl nuw nsw i64 [[C]], [[E]]
 ; CHECK-NEXT:    ret i64 [[F]]
 ;
   %C = zext i32 %A to i128
@@ -361,7 +361,7 @@ define <2 x i64> @test11_vec(<2 x i32> %A, <2 x i32> %B) {
 ; CHECK-NEXT:    [[C:%.*]] = zext <2 x i32> [[A:%.*]] to <2 x i64>
 ; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[B:%.*]], <i32 31, i32 31>
 ; CHECK-NEXT:    [[E:%.*]] = zext <2 x i32> [[TMP1]] to <2 x i64>
-; CHECK-NEXT:    [[F:%.*]] = shl <2 x i64> [[C]], [[E]]
+; CHECK-NEXT:    [[F:%.*]] = shl nuw nsw <2 x i64> [[C]], [[E]]
 ; CHECK-NEXT:    ret <2 x i64> [[F]]
 ;
   %C = zext <2 x i32> %A to <2 x i128>
@@ -377,7 +377,7 @@ define <2 x i64> @test11_vec_nonuniform(<2 x i32> %A, <2 x i32> %B) {
 ; CHECK-NEXT:    [[C:%.*]] = zext <2 x i32> [[A:%.*]] to <2 x i64>
 ; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[B:%.*]], <i32 31, i32 15>
 ; CHECK-NEXT:    [[E:%.*]] = zext <2 x i32> [[TMP1]] to <2 x i64>
-; CHECK-NEXT:    [[F:%.*]] = shl <2 x i64> [[C]], [[E]]
+; CHECK-NEXT:    [[F:%.*]] = shl nuw nsw <2 x i64> [[C]], [[E]]
 ; CHECK-NEXT:    ret <2 x i64> [[F]]
 ;
   %C = zext <2 x i32> %A to <2 x i128>
diff --git a/llvm/test/Transforms/InstSimplify/ctpop-pow2.ll b/llvm/test/Transforms/InstSimplify/ctpop-pow2.ll
index eae368f03c..48cc8895ae 100644
--- a/llvm/test/Transforms/InstSimplify/ctpop-pow2.ll
+++ b/llvm/test/Transforms/InstSimplify/ctpop-pow2.ll
@@ -41,10 +41,7 @@ define i16 @ctpop_x_and_negx(i16 %x) {
 
 define i8 @ctpop_x_nz_and_negx(i8 %x) {
 ; CHECK-LABEL: @ctpop_x_nz_and_negx(
-; CHECK-NEXT:    [[X1:%.*]] = or i8 [[X:%.*]], 1
-; CHECK-NEXT:    [[V0:%.*]] = sub i8 0, [[X1]]
-; CHECK-NEXT:    [[V1:%.*]] = and i8 [[X1]], [[V0]]
-; CHECK-NEXT:    ret i8 [[V1]]
+; CHECK-NEXT:    ret i8 1
 ;
   %x1 = or i8 %x, 1
   %v0 = sub i8 0, %x1
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
index 904dcd4fed..a78878c55a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -passes=loop-vectorize,instsimplify -force-vector-interleave=1 -S | FileCheck %s --check-prefixes=TFNONE
-; RUN: opt < %s -passes=loop-vectorize,instsimplify -force-vector-interleave=1 -prefer-predicate-over-epilogue=predicate-dont-vectorize -S | FileCheck %s --check-prefixes=TFALWAYS
-; RUN: opt < %s -passes=loop-vectorize,instsimplify -force-vector-interleave=1 -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -S | FileCheck %s --check-prefixes=TFFALLBACK
+; RUN: opt < %s -passes=loop-vectorize,instsimplify,simplifycfg -force-vector-interleave=1 -prefer-predicate-over-epilogue=predicate-dont-vectorize -S | FileCheck %s --check-prefixes=TFCOMMON,TFALWAYS
+; RUN: opt < %s -passes=loop-vectorize,instsimplify,simplifycfg -force-vector-interleave=1 -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -S | FileCheck %s --check-prefixes=TFCOMMON,TFFALLBACK
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -41,7 +41,7 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #4 {
 ; TFNONE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; TFNONE-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDVARS_IV]]
 ; TFNONE-NEXT:    [[LOAD:%.*]] = load i64, ptr [[GEP]], align 4
-; TFNONE-NEXT:    [[CALL:%.*]] = call i64 @foo(i64 [[LOAD]]) #[[ATTR2:[0-9]+]]
+; TFNONE-NEXT:    [[CALL:%.*]] = call i64 @foo(i64 [[LOAD]]) #[[ATTR3:[0-9]+]]
 ; TFNONE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDVARS_IV]]
 ; TFNONE-NEXT:    store i64 [[CALL]], ptr [[ARRAYIDX]], align 4
 ; TFNONE-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
@@ -50,99 +50,35 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #4 {
 ; TFNONE:       for.cond.cleanup:
 ; TFNONE-NEXT:    ret void
 ;
-; TFALWAYS-LABEL: @test_widen(
-; TFALWAYS-NEXT:  entry:
-; TFALWAYS-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; TFALWAYS:       vector.ph:
-; TFALWAYS-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; TFALWAYS-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
-; TFALWAYS-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; TFALWAYS-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
-; TFALWAYS-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP3]], 1
-; TFALWAYS-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]]
-; TFALWAYS-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; TFALWAYS-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; TFALWAYS-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
-; TFALWAYS-NEXT:    br label [[VECTOR_BODY:%.*]]
-; TFALWAYS:       vector.body:
-; TFALWAYS-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; TFALWAYS-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
-; TFALWAYS-NEXT:    [[TMP5:%.*]] = getelementptr i64, ptr [[B:%.*]], i64 [[INDEX]]
-; TFALWAYS-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[TMP5]], i32 4, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
-; TFALWAYS-NEXT:    [[TMP6:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; TFALWAYS-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
-; TFALWAYS-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP6]], ptr [[TMP7]], i32 4, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; TFALWAYS-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; TFALWAYS-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 2
-; TFALWAYS-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]]
-; TFALWAYS-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; TFALWAYS-NEXT:    [[TMP10:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)
-; TFALWAYS-NEXT:    [[TMP11:%.*]] = extractelement <vscale x 2 x i1> [[TMP10]], i32 0
-; TFALWAYS-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; TFALWAYS:       middle.block:
-; TFALWAYS-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
-; TFALWAYS:       scalar.ph:
-; TFALWAYS-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; TFALWAYS-NEXT:    br label [[FOR_BODY:%.*]]
-; TFALWAYS:       for.body:
-; TFALWAYS-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; TFALWAYS-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDVARS_IV]]
-; TFALWAYS-NEXT:    [[LOAD:%.*]] = load i64, ptr [[GEP]], align 4
-; TFALWAYS-NEXT:    [[CALL:%.*]] = call i64 @foo(i64 [[LOAD]]) #[[ATTR4:[0-9]+]]
-; TFALWAYS-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDVARS_IV]]
-; TFALWAYS-NEXT:    store i64 [[CALL]], ptr [[ARRAYIDX]], align 4
-; TFALWAYS-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; TFALWAYS-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1025
-; TFALWAYS-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; TFALWAYS:       for.cond.cleanup:
-; TFALWAYS-NEXT:    ret void
-;
-; TFFALLBACK-LABEL: @test_widen(
-; TFFALLBACK-NEXT:  entry:
-; TFFALLBACK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; TFFALLBACK:       vector.ph:
-; TFFALLBACK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; TFFALLBACK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
-; TFFALLBACK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; TFFALLBACK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
-; TFFALLBACK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP3]], 1
-; TFFALLBACK-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]]
-; TFFALLBACK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; TFFALLBACK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; TFFALLBACK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
-; TFFALLBACK-NEXT:    br label [[VECTOR_BODY:%.*]]
-; TFFALLBACK:       vector.body:
-; TFFALLBACK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; TFFALLBACK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
-; TFFALLBACK-NEXT:    [[TMP5:%.*]] = getelementptr i64, ptr [[B:%.*]], i64 [[INDEX]]
-; TFFALLBACK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[TMP5]], i32 4, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
-; TFFALLBACK-NEXT:    [[TMP6:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; TFFALLBACK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
-; TFFALLBACK-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP6]], ptr [[TMP7]], i32 4, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; TFFALLBACK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; TFFALLBACK-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 2
-; TFFALLBACK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]]
-; TFFALLBACK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; TFFALLBACK-NEXT:    [[TMP10:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)
-; TFFALLBACK-NEXT:    [[TMP11:%.*]] = extractelement <vscale x 2 x i1> [[TMP10]], i32 0
-; TFFALLBACK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; TFFALLBACK:       middle.block:
-; TFFALLBACK-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
-; TFFALLBACK:       scalar.ph:
-; TFFALLBACK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; TFFALLBACK-NEXT:    br label [[FOR_BODY:%.*]]
-; TFFALLBACK:       for.body:
-; TFFALLBACK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; TFFALLBACK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDVARS_IV]]
-; TFFALLBACK-NEXT:    [[LOAD:%.*]] = load i64, ptr [[GEP]], align 4
-; TFFALLBACK-NEXT:    [[CALL:%.*]] = call i64 @foo(i64 [[LOAD]]) #[[ATTR4:[0-9]+]]
-; TFFALLBACK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDVARS_IV]]
-; TFFALLBACK-NEXT:    store i64 [[CALL]], ptr [[ARRAYIDX]], align 4
-; TFFALLBACK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; TFFALLBACK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1025
-; TFFALLBACK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; TFFALLBACK:       for.cond.cleanup:
-; TFFALLBACK-NEXT:    ret void
+; TFCOMMON-LABEL: @test_widen(
+; TFCOMMON-NEXT:  entry:
+; TFCOMMON-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; TFCOMMON-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; TFCOMMON-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; TFCOMMON-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; TFCOMMON-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP3]], 1
+; TFCOMMON-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]]
+; TFCOMMON-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
+; TFCOMMON-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; TFCOMMON-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
+; TFCOMMON-NEXT:    br label [[VECTOR_BODY:%.*]]
+; TFCOMMON:       vector.body:
+; TFCOMMON-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; TFCOMMON-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; TFCOMMON-NEXT:    [[TMP5:%.*]] = getelementptr i64, ptr [[B:%.*]], i64 [[INDEX]]
+; TFCOMMON-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[TMP5]], i32 4, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
+; TFCOMMON-NEXT:    [[TMP6:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; TFCOMMON-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
+; TFCOMMON-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP6]], ptr [[TMP7]], i32 4, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; TFCOMMON-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
+; TFCOMMON-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 2
+; TFCOMMON-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]]
+; TFCOMMON-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
+; TFCOMMON-NEXT:    [[TMP10:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)
+; TFCOMMON-NEXT:    [[TMP11:%.*]] = extractelement <vscale x 2 x i1> [[TMP10]], i32 0
+; TFCOMMON-NEXT:    br i1 [[TMP11]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; TFCOMMON:       for.cond.cleanup:
+; TFCOMMON-NEXT:    ret void
 ;
 entry:
   br label %for.body
@@ -203,7 +139,7 @@ define void @test_if_then(ptr noalias %a, ptr readnone %b) #4 {
 ; TFNONE-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[TMP12]], 50
 ; TFNONE-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END]]
 ; TFNONE:       if.then:
-; TFNONE-NEXT:    [[TMP13:%.*]] = call i64 @foo(i64 [[TMP12]]) #[[ATTR2]]
+; TFNONE-NEXT:    [[TMP13:%.*]] = call i64 @foo(i64 [[TMP12]]) #[[ATTR3]]
 ; TFNONE-NEXT:    br label [[IF_END]]
 ; TFNONE:       if.end:
 ; TFNONE-NEXT:    [[TMP14:%.*]] = phi i64 [ [[TMP13]], [[IF_THEN]] ], [ 0, [[FOR_BODY]] ]
@@ -215,123 +151,41 @@ define void @test_if_then(ptr noalias %a, ptr readnone %b) #4 {
 ; TFNONE:       for.cond.cleanup:
 ; TFNONE-NEXT:    ret void
 ;
-; TFALWAYS-LABEL: @test_if_then(
-; TFALWAYS-NEXT:  entry:
-; TFALWAYS-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; TFALWAYS:       vector.ph:
-; TFALWAYS-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; TFALWAYS-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
-; TFALWAYS-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; TFALWAYS-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
-; TFALWAYS-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP3]], 1
-; TFALWAYS-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]]
-; TFALWAYS-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; TFALWAYS-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; TFALWAYS-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
-; TFALWAYS-NEXT:    br label [[VECTOR_BODY:%.*]]
-; TFALWAYS:       vector.body:
-; TFALWAYS-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; TFALWAYS-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
-; TFALWAYS-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
-; TFALWAYS-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[TMP5]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
-; TFALWAYS-NEXT:    [[TMP6:%.*]] = icmp ugt <vscale x 2 x i64> [[WIDE_MASKED_LOAD]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 50, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; TFALWAYS-NEXT:    [[TMP7:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i1> [[TMP6]], <vscale x 2 x i1> zeroinitializer
-; TFALWAYS-NEXT:    [[TMP8:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[TMP7]])
-; TFALWAYS-NEXT:    [[TMP9:%.*]] = xor <vscale x 2 x i1> [[TMP6]], shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)
-; TFALWAYS-NEXT:    [[TMP10:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i1> [[TMP9]], <vscale x 2 x i1> zeroinitializer
-; TFALWAYS-NEXT:    [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP10]], <vscale x 2 x i64> zeroinitializer, <vscale x 2 x i64> [[TMP8]]
-; TFALWAYS-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[INDEX]]
-; TFALWAYS-NEXT:    [[TMP12:%.*]] = or <vscale x 2 x i1> [[TMP7]], [[TMP10]]
-; TFALWAYS-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI]], ptr [[TMP11]], i32 8, <vscale x 2 x i1> [[TMP12]])
-; TFALWAYS-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; TFALWAYS-NEXT:    [[TMP14:%.*]] = mul i64 [[TMP13]], 2
-; TFALWAYS-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP14]]
-; TFALWAYS-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; TFALWAYS-NEXT:    [[TMP15:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)
-; TFALWAYS-NEXT:    [[TMP16:%.*]] = extractelement <vscale x 2 x i1> [[TMP15]], i32 0
-; TFALWAYS-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; TFALWAYS:       middle.block:
-; TFALWAYS-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
-; TFALWAYS:       scalar.ph:
-; TFALWAYS-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; TFALWAYS-NEXT:    br label [[FOR_BODY:%.*]]
-; TFALWAYS:       for.body:
-; TFALWAYS-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[IF_END:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; TFALWAYS-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDVARS_IV]]
-; TFALWAYS-NEXT:    [[TMP17:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
-; TFALWAYS-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[TMP17]], 50
-; TFALWAYS-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END]]
-; TFALWAYS:       if.then:
-; TFALWAYS-NEXT:    [[TMP18:%.*]] = call i64 @foo(i64 [[TMP17]]) #[[ATTR4]]
-; TFALWAYS-NEXT:    br label [[IF_END]]
-; TFALWAYS:       if.end:
-; TFALWAYS-NEXT:    [[TMP19:%.*]] = phi i64 [ [[TMP18]], [[IF_THEN]] ], [ 0, [[FOR_BODY]] ]
-; TFALWAYS-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDVARS_IV]]
-; TFALWAYS-NEXT:    store i64 [[TMP19]], ptr [[ARRAYIDX1]], align 8
-; TFALWAYS-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; TFALWAYS-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1025
-; TFALWAYS-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
-; TFALWAYS:       for.cond.cleanup:
-; TFALWAYS-NEXT:    ret void
-;
-; TFFALLBACK-LABEL: @test_if_then(
-; TFFALLBACK-NEXT:  entry:
-; TFFALLBACK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; TFFALLBACK:       vector.ph:
-; TFFALLBACK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; TFFALLBACK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
-; TFFALLBACK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; TFFALLBACK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
-; TFFALLBACK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP3]], 1
-; TFFALLBACK-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]]
-; TFFALLBACK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; TFFALLBACK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; TFFALLBACK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
-; TFFALLBACK-NEXT:    br label [[VECTOR_BODY:%.*]]
-; TFFALLBACK:       vector.body:
-; TFFALLBACK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; TFFALLBACK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
-; TFFALLBACK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
-; TFFALLBACK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[TMP5]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
-; TFFALLBACK-NEXT:    [[TMP6:%.*]] = icmp ugt <vscale x 2 x i64> [[WIDE_MASKED_LOAD]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 50, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; TFFALLBACK-NEXT:    [[TMP7:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i1> [[TMP6]], <vscale x 2 x i1> zeroinitializer
-; TFFALLBACK-NEXT:    [[TMP8:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[TMP7]])
-; TFFALLBACK-NEXT:    [[TMP9:%.*]] = xor <vscale x 2 x i1> [[TMP6]], shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)
-; TFFALLBACK-NEXT:    [[TMP10:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i1> [[TMP9]], <vscale x 2 x i1> zeroinitializer
-; TFFALLBACK-NEXT:    [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP10]], <vscale x 2 x i64> zeroinitializer, <vscale x 2 x i64> [[TMP8]]
-; TFFALLBACK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[INDEX]]
-; TFFALLBACK-NEXT:    [[TMP12:%.*]] = or <vscale x 2 x i1> [[TMP7]], [[TMP10]]
-; TFFALLBACK-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI]], ptr [[TMP11]], i32 8, <vscale x 2 x i1> [[TMP12]])
-; TFFALLBACK-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; TFFALLBACK-NEXT:    [[TMP14:%.*]] = mul i64 [[TMP13]], 2
-; TFFALLBACK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP14]]
-; TFFALLBACK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; TFFALLBACK-NEXT:    [[TMP15:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)
-; TFFALLBACK-NEXT:    [[TMP16:%.*]] = extractelement <vscale x 2 x i1> [[TMP15]], i32 0
-; TFFALLBACK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; TFFALLBACK:       middle.block:
-; TFFALLBACK-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
-; TFFALLBACK:       scalar.ph:
-; TFFALLBACK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; TFFALLBACK-NEXT:    br label [[FOR_BODY:%.*]]
-; TFFALLBACK:       for.body:
-; TFFALLBACK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[IF_END:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; TFFALLBACK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDVARS_IV]]
-; TFFALLBACK-NEXT:    [[TMP17:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
-; TFFALLBACK-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[TMP17]], 50
-; TFFALLBACK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END]]
-; TFFALLBACK:       if.then:
-; TFFALLBACK-NEXT:    [[TMP18:%.*]] = call i64 @foo(i64 [[TMP17]]) #[[ATTR4]]
-; TFFALLBACK-NEXT:    br label [[IF_END]]
-; TFFALLBACK:       if.end:
-; TFFALLBACK-NEXT:    [[TMP19:%.*]] = phi i64 [ [[TMP18]], [[IF_THEN]] ], [ 0, [[FOR_BODY]] ]
-; TFFALLBACK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDVARS_IV]]
-; TFFALLBACK-NEXT:    store i64 [[TMP19]], ptr [[ARRAYIDX1]], align 8
-; TFFALLBACK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; TFFALLBACK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1025
-; TFFALLBACK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
-; TFFALLBACK:       for.cond.cleanup:
-; TFFALLBACK-NEXT:    ret void
+; TFCOMMON-LABEL: @test_if_then(
+; TFCOMMON-NEXT:  entry:
+; TFCOMMON-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; TFCOMMON-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; TFCOMMON-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; TFCOMMON-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; TFCOMMON-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP3]], 1
+; TFCOMMON-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]]
+; TFCOMMON-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
+; TFCOMMON-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; TFCOMMON-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
+; TFCOMMON-NEXT:    br label [[VECTOR_BODY:%.*]]
+; TFCOMMON:       vector.body:
+; TFCOMMON-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; TFCOMMON-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; TFCOMMON-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
+; TFCOMMON-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[TMP5]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
+; TFCOMMON-NEXT:    [[TMP6:%.*]] = icmp ugt <vscale x 2 x i64> [[WIDE_MASKED_LOAD]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 50, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
+; TFCOMMON-NEXT:    [[TMP7:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i1> [[TMP6]], <vscale x 2 x i1> zeroinitializer
+; TFCOMMON-NEXT:    [[TMP8:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[TMP7]])
+; TFCOMMON-NEXT:    [[TMP9:%.*]] = xor <vscale x 2 x i1> [[TMP6]], shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)
+; TFCOMMON-NEXT:    [[TMP10:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i1> [[TMP9]], <vscale x 2 x i1> zeroinitializer
+; TFCOMMON-NEXT:    [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP10]], <vscale x 2 x i64> zeroinitializer, <vscale x 2 x i64> [[TMP8]]
+; TFCOMMON-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[INDEX]]
+; TFCOMMON-NEXT:    [[TMP12:%.*]] = or <vscale x 2 x i1> [[TMP7]], [[TMP10]]
+; TFCOMMON-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI]], ptr [[TMP11]], i32 8, <vscale x 2 x i1> [[TMP12]])
+; TFCOMMON-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
+; TFCOMMON-NEXT:    [[TMP14:%.*]] = mul i64 [[TMP13]], 2
+; TFCOMMON-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP14]]
+; TFCOMMON-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
+; TFCOMMON-NEXT:    [[TMP15:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)
+; TFCOMMON-NEXT:    [[TMP16:%.*]] = extractelement <vscale x 2 x i1> [[TMP15]], i32 0
+; TFCOMMON-NEXT:    br i1 [[TMP16]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; TFCOMMON:       for.cond.cleanup:
+; TFCOMMON-NEXT:    ret void
 ;
 entry:
   br label %for.body
@@ -404,10 +258,10 @@ define void @test_widen_if_then_else(ptr noalias %a, ptr readnone %b) #4 {
 ; TFNONE-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[TMP13]], 50
 ; TFNONE-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
 ; TFNONE:       if.then:
-; TFNONE-NEXT:    [[TMP14:%.*]] = call i64 @foo(i64 [[TMP13]]) #[[ATTR3:[0-9]+]]
+; TFNONE-NEXT:    [[TMP14:%.*]] = call i64 @foo(i64 [[TMP13]]) #[[ATTR4:[0-9]+]]
 ; TFNONE-NEXT:    br label [[IF_END]]
 ; TFNONE:       if.else:
-; TFNONE-NEXT:    [[TMP15:%.*]] = call i64 @foo(i64 0) #[[ATTR3]]
+; TFNONE-NEXT:    [[TMP15:%.*]] = call i64 @foo(i64 0) #[[ATTR4]]
 ; TFNONE-NEXT:    br label [[IF_END]]
 ; TFNONE:       if.end:
 ; TFNONE-NEXT:    [[TMP16:%.*]] = phi i64 [ [[TMP14]], [[IF_THEN]] ], [ [[TMP15]], [[IF_ELSE]] ]
@@ -419,131 +273,42 @@ define void @test_widen_if_then_else(ptr noalias %a, ptr readnone %b) #4 {
 ; TFNONE:       for.cond.cleanup:
 ; TFNONE-NEXT:    ret void
 ;
-; TFALWAYS-LABEL: @test_widen_if_then_else(
-; TFALWAYS-NEXT:  entry:
-; TFALWAYS-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; TFALWAYS:       vector.ph:
-; TFALWAYS-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; TFALWAYS-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
-; TFALWAYS-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; TFALWAYS-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
-; TFALWAYS-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP3]], 1
-; TFALWAYS-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]]
-; TFALWAYS-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; TFALWAYS-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; TFALWAYS-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
-; TFALWAYS-NEXT:    br label [[VECTOR_BODY:%.*]]
-; TFALWAYS:       vector.body:
-; TFALWAYS-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; TFALWAYS-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
-; TFALWAYS-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
-; TFALWAYS-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[TMP5]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
-; TFALWAYS-NEXT:    [[TMP6:%.*]] = icmp ugt <vscale x 2 x i64> [[WIDE_MASKED_LOAD]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 50, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; TFALWAYS-NEXT:    [[TMP7:%.*]] = xor <vscale x 2 x i1> [[TMP6]], shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)
-; TFALWAYS-NEXT:    [[TMP8:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i1> [[TMP7]], <vscale x 2 x i1> zeroinitializer
-; TFALWAYS-NEXT:    [[TMP9:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> [[TMP8]])
-; TFALWAYS-NEXT:    [[TMP10:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i1> [[TMP6]], <vscale x 2 x i1> zeroinitializer
-; TFALWAYS-NEXT:    [[TMP11:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[TMP10]])
-; TFALWAYS-NEXT:    [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP8]], <vscale x 2 x i64> [[TMP9]], <vscale x 2 x i64> [[TMP11]]
-; TFALWAYS-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[INDEX]]
-; TFALWAYS-NEXT:    [[TMP13:%.*]] = or <vscale x 2 x i1> [[TMP8]], [[TMP10]]
-; TFALWAYS-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI]], ptr [[TMP12]], i32 8, <vscale x 2 x i1> [[TMP13]])
-; TFALWAYS-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; TFALWAYS-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 2
-; TFALWAYS-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP15]]
-; TFALWAYS-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; TFALWAYS-NEXT:    [[TMP16:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)
-; TFALWAYS-NEXT:    [[TMP17:%.*]] = extractelement <vscale x 2 x i1> [[TMP16]], i32 0
-; TFALWAYS-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; TFALWAYS:       middle.block:
-; TFALWAYS-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
-; TFALWAYS:       scalar.ph:
-; TFALWAYS-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; TFALWAYS-NEXT:    br label [[FOR_BODY:%.*]]
-; TFALWAYS:       for.body:
-; TFALWAYS-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[IF_END:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; TFALWAYS-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDVARS_IV]]
-; TFALWAYS-NEXT:    [[TMP18:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
-; TFALWAYS-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[TMP18]], 50
-; TFALWAYS-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
-; TFALWAYS:       if.then:
-; TFALWAYS-NEXT:    [[TMP19:%.*]] = call i64 @foo(i64 [[TMP18]]) #[[ATTR5:[0-9]+]]
-; TFALWAYS-NEXT:    br label [[IF_END]]
-; TFALWAYS:       if.else:
-; TFALWAYS-NEXT:    [[TMP20:%.*]] = call i64 @foo(i64 0) #[[ATTR5]]
-; TFALWAYS-NEXT:    br label [[IF_END]]
-; TFALWAYS:       if.end:
-; TFALWAYS-NEXT:    [[TMP21:%.*]] = phi i64 [ [[TMP19]], [[IF_THEN]] ], [ [[TMP20]], [[IF_ELSE]] ]
-; TFALWAYS-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDVARS_IV]]
-; TFALWAYS-NEXT:    store i64 [[TMP21]], ptr [[ARRAYIDX1]], align 8
-; TFALWAYS-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; TFALWAYS-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1025
-; TFALWAYS-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
-; TFALWAYS:       for.cond.cleanup:
-; TFALWAYS-NEXT:    ret void
-;
-; TFFALLBACK-LABEL: @test_widen_if_then_else(
-; TFFALLBACK-NEXT:  entry:
-; TFFALLBACK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; TFFALLBACK:       vector.ph:
-; TFFALLBACK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; TFFALLBACK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
-; TFFALLBACK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; TFFALLBACK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
-; TFFALLBACK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP3]], 1
-; TFFALLBACK-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]]
-; TFFALLBACK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; TFFALLBACK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; TFFALLBACK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
-; TFFALLBACK-NEXT:    br label [[VECTOR_BODY:%.*]]
-; TFFALLBACK:       vector.body:
-; TFFALLBACK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; TFFALLBACK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
-; TFFALLBACK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
-; TFFALLBACK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[TMP5]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
-; TFFALLBACK-NEXT:    [[TMP6:%.*]] = icmp ugt <vscale x 2 x i64> [[WIDE_MASKED_LOAD]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 50, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
-; TFFALLBACK-NEXT:    [[TMP7:%.*]] = xor <vscale x 2 x i1> [[TMP6]], shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)
-; TFFALLBACK-NEXT:    [[TMP8:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i1> [[TMP7]], <vscale x 2 x i1> zeroinitializer
-; TFFALLBACK-NEXT:    [[TMP9:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> [[TMP8]])
-; TFFALLBACK-NEXT:    [[TMP10:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i1> [[TMP6]], <vscale x 2 x i1> zeroinitializer
-; TFFALLBACK-NEXT:    [[TMP11:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[TMP10]])
-; TFFALLBACK-NEXT:    [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP8]], <vscale x 2 x i64> [[TMP9]], <vscale x 2 x i64> [[TMP11]]
-; TFFALLBACK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[INDEX]]
-; TFFALLBACK-NEXT:    [[TMP13:%.*]] = or <vscale x 2 x i1> [[TMP8]], [[TMP10]]
-; TFFALLBACK-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI]], ptr [[TMP12]], i32 8, <vscale x 2 x i1> [[TMP13]])
-; TFFALLBACK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; TFFALLBACK-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 2
-; TFFALLBACK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP15]]
-; TFFALLBACK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; TFFALLBACK-NEXT:    [[TMP16:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)
-; TFFALLBACK-NEXT:    [[TMP17:%.*]] = extractelement <vscale x 2 x i1> [[TMP16]], i32 0
-; TFFALLBACK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; TFFALLBACK:       middle.block:
-; TFFALLBACK-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
-; TFFALLBACK:       scalar.ph:
-; TFFALLBACK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; TFFALLBACK-NEXT:    br label [[FOR_BODY:%.*]]
-; TFFALLBACK:       for.body:
-; TFFALLBACK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[IF_END:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; TFFALLBACK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDVARS_IV]]
-; TFFALLBACK-NEXT:    [[TMP18:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
-; TFFALLBACK-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[TMP18]], 50
-; TFFALLBACK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
-; TFFALLBACK:       if.then:
-; TFFALLBACK-NEXT:    [[TMP19:%.*]] = call i64 @foo(i64 [[TMP18]]) #[[ATTR5:[0-9]+]]
-; TFFALLBACK-NEXT:    br label [[IF_END]]
-; TFFALLBACK:       if.else:
-; TFFALLBACK-NEXT:    [[TMP20:%.*]] = call i64 @foo(i64 0) #[[ATTR5]]
-; TFFALLBACK-NEXT:    br label [[IF_END]]
-; TFFALLBACK:       if.end:
-; TFFALLBACK-NEXT:    [[TMP21:%.*]] = phi i64 [ [[TMP19]], [[IF_THEN]] ], [ [[TMP20]], [[IF_ELSE]] ]
-; TFFALLBACK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDVARS_IV]]
-; TFFALLBACK-NEXT:    store i64 [[TMP21]], ptr [[ARRAYIDX1]], align 8
-; TFFALLBACK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; TFFALLBACK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1025
-; TFFALLBACK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
-; TFFALLBACK:       for.cond.cleanup:
-; TFFALLBACK-NEXT:    ret void
+; TFCOMMON-LABEL: @test_widen_if_then_else(
+; TFCOMMON-NEXT:  entry:
+; TFCOMMON-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; TFCOMMON-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; TFCOMMON-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; TFCOMMON-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; TFCOMMON-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP3]], 1
+; TFCOMMON-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]]
+; TFCOMMON-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
+; TFCOMMON-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; TFCOMMON-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
+; TFCOMMON-NEXT:    br label [[VECTOR_BODY:%.*]]
+; TFCOMMON:       vector.body:
+; TFCOMMON-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; TFCOMMON-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; TFCOMMON-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
+; TFCOMMON-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[TMP5]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
+; TFCOMMON-NEXT:    [[TMP6:%.*]] = icmp ugt <vscale x 2 x i64> [[WIDE_MASKED_LOAD]], shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> poison, i64 50, i64 0), <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer)
+; TFCOMMON-NEXT:    [[TMP7:%.*]] = xor <vscale x 2 x i1> [[TMP6]], shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)
+; TFCOMMON-NEXT:    [[TMP8:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i1> [[TMP7]], <vscale x 2 x i1> zeroinitializer
+; TFCOMMON-NEXT:    [[TMP9:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> [[TMP8]])
+; TFCOMMON-NEXT:    [[TMP10:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i1> [[TMP6]], <vscale x 2 x i1> zeroinitializer
+; TFCOMMON-NEXT:    [[TMP11:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[TMP10]])
+; TFCOMMON-NEXT:    [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP8]], <vscale x 2 x i64> [[TMP9]], <vscale x 2 x i64> [[TMP11]]
+; TFCOMMON-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[INDEX]]
+; TFCOMMON-NEXT:    [[TMP13:%.*]] = or <vscale x 2 x i1> [[TMP8]], [[TMP10]]
+; TFCOMMON-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI]], ptr [[TMP12]], i32 8, <vscale x 2 x i1> [[TMP13]])
+; TFCOMMON-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
+; TFCOMMON-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 2
+; TFCOMMON-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP15]]
+; TFCOMMON-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
+; TFCOMMON-NEXT:    [[TMP16:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)
+; TFCOMMON-NEXT:    [[TMP17:%.*]] = extractelement <vscale x 2 x i1> [[TMP16]], i32 0
+; TFCOMMON-NEXT:    br i1 [[TMP17]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; TFCOMMON:       for.cond.cleanup:
+; TFCOMMON-NEXT:    ret void
 ;
 entry:
   br label %for.body
@@ -612,7 +377,7 @@ define void @test_widen_nomask(ptr noalias %a, ptr readnone %b) #4 {
 ; TFNONE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; TFNONE-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDVARS_IV]]
 ; TFNONE-NEXT:    [[LOAD:%.*]] = load i64, ptr [[GEP]], align 4
-; TFNONE-NEXT:    [[CALL:%.*]] = call i64 @foo(i64 [[LOAD]]) #[[ATTR4:[0-9]+]]
+; TFNONE-NEXT:    [[CALL:%.*]] = call i64 @foo(i64 [[LOAD]]) #[[ATTR5:[0-9]+]]
 ; TFNONE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDVARS_IV]]
 ; TFNONE-NEXT:    store i64 [[CALL]], ptr [[ARRAYIDX]], align 4
 ; TFNONE-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
@@ -628,7 +393,7 @@ define void @test_widen_nomask(ptr noalias %a, ptr readnone %b) #4 {
 ; TFALWAYS-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; TFALWAYS-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[B:%.*]], i64 [[INDVARS_IV]]
 ; TFALWAYS-NEXT:    [[LOAD:%.*]] = load i64, ptr [[GEP]], align 4
-; TFALWAYS-NEXT:    [[CALL:%.*]] = call i64 @foo(i64 [[LOAD]]) #[[ATTR6:[0-9]+]]
+; TFALWAYS-NEXT:    [[CALL:%.*]] = call i64 @foo(i64 [[LOAD]]) #[[ATTR5:[0-9]+]]
 ; TFALWAYS-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDVARS_IV]]
 ; TFALWAYS-NEXT:    store i64 [[CALL]], ptr [[ARRAYIDX]], align 4
 ; TFALWAYS-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
@@ -660,22 +425,20 @@ define void @test_widen_nomask(ptr noalias %a, ptr readnone %b) #4 {
 ; TFFALLBACK-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 2
 ; TFFALLBACK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
 ; TFFALLBACK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; TFFALLBACK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; TFFALLBACK:       middle.block:
-; TFFALLBACK-NEXT:    br i1 false, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; TFFALLBACK-NEXT:    br i1 [[TMP9]], label [[SCALAR_PH]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; TFFALLBACK:       scalar.ph:
-; TFFALLBACK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; TFFALLBACK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[N_VEC]], [[VECTOR_BODY]] ]
 ; TFFALLBACK-NEXT:    br label [[FOR_BODY:%.*]]
 ; TFFALLBACK:       for.body:
 ; TFFALLBACK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; TFFALLBACK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDVARS_IV]]
 ; TFFALLBACK-NEXT:    [[LOAD:%.*]] = load i64, ptr [[GEP]], align 4
-; TFFALLBACK-NEXT:    [[CALL:%.*]] = call i64 @foo(i64 [[LOAD]]) #[[ATTR6:[0-9]+]]
+; TFFALLBACK-NEXT:    [[CALL:%.*]] = call i64 @foo(i64 [[LOAD]]) #[[ATTR5:[0-9]+]]
 ; TFFALLBACK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDVARS_IV]]
 ; TFFALLBACK-NEXT:    store i64 [[CALL]], ptr [[ARRAYIDX]], align 4
 ; TFFALLBACK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; TFFALLBACK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1025
-; TFFALLBACK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; TFFALLBACK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; TFFALLBACK:       for.cond.cleanup:
 ; TFFALLBACK-NEXT:    ret void
 ;
@@ -734,7 +497,7 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 {
 ; TFNONE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; TFNONE-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDVARS_IV]]
 ; TFNONE-NEXT:    [[LOAD:%.*]] = load i64, ptr [[GEP]], align 4
-; TFNONE-NEXT:    [[CALL:%.*]] = call i64 @foo(i64 [[LOAD]]) #[[ATTR5:[0-9]+]]
+; TFNONE-NEXT:    [[CALL:%.*]] = call i64 @foo(i64 [[LOAD]]) #[[ATTR6:[0-9]+]]
 ; TFNONE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDVARS_IV]]
 ; TFNONE-NEXT:    store i64 [[CALL]], ptr [[ARRAYIDX]], align 4
 ; TFNONE-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
@@ -745,8 +508,6 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 {
 ;
 ; TFALWAYS-LABEL: @test_widen_optmask(
 ; TFALWAYS-NEXT:  entry:
-; TFALWAYS-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; TFALWAYS:       vector.ph:
 ; TFALWAYS-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; TFALWAYS-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
 ; TFALWAYS-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
@@ -758,8 +519,8 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 {
 ; TFALWAYS-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
 ; TFALWAYS-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; TFALWAYS:       vector.body:
-; TFALWAYS-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; TFALWAYS-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; TFALWAYS-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; TFALWAYS-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; TFALWAYS-NEXT:    [[TMP5:%.*]] = getelementptr i64, ptr [[B:%.*]], i64 [[INDEX]]
 ; TFALWAYS-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[TMP5]], i32 4, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
 ; TFALWAYS-NEXT:    [[TMP6:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
@@ -771,29 +532,12 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 {
 ; TFALWAYS-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
 ; TFALWAYS-NEXT:    [[TMP10:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)
 ; TFALWAYS-NEXT:    [[TMP11:%.*]] = extractelement <vscale x 2 x i1> [[TMP10]], i32 0
-; TFALWAYS-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; TFALWAYS:       middle.block:
-; TFALWAYS-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
-; TFALWAYS:       scalar.ph:
-; TFALWAYS-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; TFALWAYS-NEXT:    br label [[FOR_BODY:%.*]]
-; TFALWAYS:       for.body:
-; TFALWAYS-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; TFALWAYS-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDVARS_IV]]
-; TFALWAYS-NEXT:    [[LOAD:%.*]] = load i64, ptr [[GEP]], align 4
-; TFALWAYS-NEXT:    [[CALL:%.*]] = call i64 @foo(i64 [[LOAD]]) #[[ATTR7:[0-9]+]]
-; TFALWAYS-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDVARS_IV]]
-; TFALWAYS-NEXT:    store i64 [[CALL]], ptr [[ARRAYIDX]], align 4
-; TFALWAYS-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; TFALWAYS-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1025
-; TFALWAYS-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; TFALWAYS-NEXT:    br i1 [[TMP11]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; TFALWAYS:       for.cond.cleanup:
 ; TFALWAYS-NEXT:    ret void
 ;
 ; TFFALLBACK-LABEL: @test_widen_optmask(
 ; TFFALLBACK-NEXT:  entry:
-; TFFALLBACK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; TFFALLBACK:       vector.ph:
 ; TFFALLBACK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; TFFALLBACK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
 ; TFFALLBACK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
@@ -805,8 +549,8 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 {
 ; TFFALLBACK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
 ; TFFALLBACK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; TFFALLBACK:       vector.body:
-; TFFALLBACK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; TFFALLBACK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; TFFALLBACK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; TFFALLBACK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; TFFALLBACK-NEXT:    [[TMP5:%.*]] = getelementptr i64, ptr [[B:%.*]], i64 [[INDEX]]
 ; TFFALLBACK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[TMP5]], i32 4, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
 ; TFFALLBACK-NEXT:    [[TMP6:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
@@ -818,22 +562,7 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 {
 ; TFFALLBACK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
 ; TFFALLBACK-NEXT:    [[TMP10:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)
 ; TFFALLBACK-NEXT:    [[TMP11:%.*]] = extractelement <vscale x 2 x i1> [[TMP10]], i32 0
-; TFFALLBACK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
-; TFFALLBACK:       middle.block:
-; TFFALLBACK-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
-; TFFALLBACK:       scalar.ph:
-; TFFALLBACK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; TFFALLBACK-NEXT:    br label [[FOR_BODY:%.*]]
-; TFFALLBACK:       for.body:
-; TFFALLBACK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; TFFALLBACK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDVARS_IV]]
-; TFFALLBACK-NEXT:    [[LOAD:%.*]] = load i64, ptr [[GEP]], align 4
-; TFFALLBACK-NEXT:    [[CALL:%.*]] = call i64 @foo(i64 [[LOAD]]) #[[ATTR7:[0-9]+]]
-; TFFALLBACK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDVARS_IV]]
-; TFFALLBACK-NEXT:    store i64 [[CALL]], ptr [[ARRAYIDX]], align 4
-; TFFALLBACK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; TFFALLBACK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1025
-; TFFALLBACK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; TFFALLBACK-NEXT:    br i1 [[TMP11]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; TFFALLBACK:       for.cond.cleanup:
 ; TFFALLBACK-NEXT:    ret void
 ;
@@ -855,7 +584,160 @@ for.cond.cleanup:
   ret void
 }
 
+
+; An fmuladd intrinsic followed by a call; we want to make sure we correctly
+; pick up the second call and assign a vector variant to it.
+define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, double %m) #4 {
+; TFNONE-LABEL: @test_widen_fmuladd_and_call(
+; TFNONE-NEXT:  entry:
+; TFNONE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; TFNONE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; TFNONE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1025, [[TMP1]]
+; TFNONE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; TFNONE:       vector.ph:
+; TFNONE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; TFNONE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; TFNONE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
+; TFNONE-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
+; TFNONE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x double> poison, double [[M:%.*]], i64 0
+; TFNONE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x double> [[BROADCAST_SPLATINSERT]], <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
+; TFNONE-NEXT:    br label [[VECTOR_BODY:%.*]]
+; TFNONE:       vector.body:
+; TFNONE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; TFNONE-NEXT:    [[VEC_PHI:%.*]] = phi double [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ]
+; TFNONE-NEXT:    [[TMP4:%.*]] = getelementptr double, ptr [[B:%.*]], i64 [[INDEX]]
+; TFNONE-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x double>, ptr [[TMP4]], align 8
+; TFNONE-NEXT:    [[TMP5:%.*]] = fmul <vscale x 2 x double> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; TFNONE-NEXT:    [[TMP6:%.*]] = fptoui <vscale x 2 x double> [[WIDE_LOAD]] to <vscale x 2 x i64>
+; TFNONE-NEXT:    [[TMP7:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[TMP6]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; TFNONE-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
+; TFNONE-NEXT:    store <vscale x 2 x i64> [[TMP7]], ptr [[TMP8]], align 4
+; TFNONE-NEXT:    [[TMP9]] = call double @llvm.vector.reduce.fadd.nxv2f64(double [[VEC_PHI]], <vscale x 2 x double> [[TMP5]])
+; TFNONE-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
+; TFNONE-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 2
+; TFNONE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]]
+; TFNONE-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; TFNONE-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; TFNONE:       middle.block:
+; TFNONE-NEXT:    br i1 false, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; TFNONE:       scalar.ph:
+; TFNONE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; TFNONE-NEXT:    [[BC_MERGE_RDX:%.*]] = phi double [ 0.000000e+00, [[ENTRY]] ], [ [[TMP9]], [[MIDDLE_BLOCK]] ]
+; TFNONE-NEXT:    br label [[FOR_BODY:%.*]]
+; TFNONE:       for.body:
+; TFNONE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; TFNONE-NEXT:    [[FMA_SUM:%.*]] = phi double [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MULADD:%.*]], [[FOR_BODY]] ]
+; TFNONE-NEXT:    [[GEP:%.*]] = getelementptr double, ptr [[B]], i64 [[INDVARS_IV]]
+; TFNONE-NEXT:    [[LOAD:%.*]] = load double, ptr [[GEP]], align 8
+; TFNONE-NEXT:    [[MULADD]] = tail call double @llvm.fmuladd.f64(double [[LOAD]], double [[M]], double [[FMA_SUM]])
+; TFNONE-NEXT:    [[TOINT:%.*]] = fptoui double [[LOAD]] to i64
+; TFNONE-NEXT:    [[CALL:%.*]] = call i64 @foo(i64 [[TOINT]]) #[[ATTR3]]
+; TFNONE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDVARS_IV]]
+; TFNONE-NEXT:    store i64 [[CALL]], ptr [[ARRAYIDX]], align 4
+; TFNONE-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; TFNONE-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1025
+; TFNONE-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; TFNONE:       for.cond.cleanup:
+; TFNONE-NEXT:    [[MULADD_LCSSA:%.*]] = phi double [ [[MULADD]], [[FOR_BODY]] ], [ [[TMP9]], [[MIDDLE_BLOCK]] ]
+; TFNONE-NEXT:    ret double [[MULADD_LCSSA]]
+;
+; TFALWAYS-LABEL: @test_widen_fmuladd_and_call(
+; TFALWAYS-NEXT:  entry:
+; TFALWAYS-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; TFALWAYS-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; TFALWAYS-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; TFALWAYS-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; TFALWAYS-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP3]], 1
+; TFALWAYS-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]]
+; TFALWAYS-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
+; TFALWAYS-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; TFALWAYS-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
+; TFALWAYS-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x double> poison, double [[M:%.*]], i64 0
+; TFALWAYS-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x double> [[BROADCAST_SPLATINSERT]], <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
+; TFALWAYS-NEXT:    br label [[VECTOR_BODY:%.*]]
+; TFALWAYS:       vector.body:
+; TFALWAYS-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; TFALWAYS-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; TFALWAYS-NEXT:    [[VEC_PHI:%.*]] = phi double [ 0.000000e+00, [[ENTRY]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ]
+; TFALWAYS-NEXT:    [[TMP5:%.*]] = getelementptr double, ptr [[B:%.*]], i64 [[INDEX]]
+; TFALWAYS-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr [[TMP5]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
+; TFALWAYS-NEXT:    [[TMP6:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT]]
+; TFALWAYS-NEXT:    [[TMP7:%.*]] = fptoui <vscale x 2 x double> [[WIDE_MASKED_LOAD]] to <vscale x 2 x i64>
+; TFALWAYS-NEXT:    [[TMP8:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[TMP7]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; TFALWAYS-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
+; TFALWAYS-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP8]], ptr [[TMP9]], i32 4, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; TFALWAYS-NEXT:    [[TMP10:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> [[TMP6]], <vscale x 2 x double> shufflevector (<vscale x 2 x double> insertelement (<vscale x 2 x double> poison, double -0.000000e+00, i64 0), <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer)
+; TFALWAYS-NEXT:    [[TMP11]] = call double @llvm.vector.reduce.fadd.nxv2f64(double [[VEC_PHI]], <vscale x 2 x double> [[TMP10]])
+; TFALWAYS-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
+; TFALWAYS-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 2
+; TFALWAYS-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP13]]
+; TFALWAYS-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
+; TFALWAYS-NEXT:    [[TMP14:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)
+; TFALWAYS-NEXT:    [[TMP15:%.*]] = extractelement <vscale x 2 x i1> [[TMP14]], i32 0
+; TFALWAYS-NEXT:    br i1 [[TMP15]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; TFALWAYS:       for.cond.cleanup:
+; TFALWAYS-NEXT:    ret double [[TMP11]]
+;
+; TFFALLBACK-LABEL: @test_widen_fmuladd_and_call(
+; TFFALLBACK-NEXT:  entry:
+; TFFALLBACK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; TFFALLBACK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; TFFALLBACK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; TFFALLBACK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; TFFALLBACK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP3]], 1
+; TFFALLBACK-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]]
+; TFFALLBACK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
+; TFFALLBACK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; TFFALLBACK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
+; TFFALLBACK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x double> poison, double [[M:%.*]], i64 0
+; TFFALLBACK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x double> [[BROADCAST_SPLATINSERT]], <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
+; TFFALLBACK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; TFFALLBACK:       vector.body:
+; TFFALLBACK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; TFFALLBACK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; TFFALLBACK-NEXT:    [[VEC_PHI:%.*]] = phi double [ 0.000000e+00, [[ENTRY]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ]
+; TFFALLBACK-NEXT:    [[TMP5:%.*]] = getelementptr double, ptr [[B:%.*]], i64 [[INDEX]]
+; TFFALLBACK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr [[TMP5]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
+; TFFALLBACK-NEXT:    [[TMP6:%.*]] = fmul <vscale x 2 x double> [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT]]
+; TFFALLBACK-NEXT:    [[TMP7:%.*]] = fptoui <vscale x 2 x double> [[WIDE_MASKED_LOAD]] to <vscale x 2 x i64>
+; TFFALLBACK-NEXT:    [[TMP8:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[TMP7]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; TFFALLBACK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
+; TFFALLBACK-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP8]], ptr [[TMP9]], i32 4, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; TFFALLBACK-NEXT:    [[TMP10:%.*]] = select <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> [[TMP6]], <vscale x 2 x double> shufflevector (<vscale x 2 x double> insertelement (<vscale x 2 x double> poison, double -0.000000e+00, i64 0), <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer)
+; TFFALLBACK-NEXT:    [[TMP11]] = call double @llvm.vector.reduce.fadd.nxv2f64(double [[VEC_PHI]], <vscale x 2 x double> [[TMP10]])
+; TFFALLBACK-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
+; TFFALLBACK-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 2
+; TFFALLBACK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP13]]
+; TFFALLBACK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
+; TFFALLBACK-NEXT:    [[TMP14:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)
+; TFFALLBACK-NEXT:    [[TMP15:%.*]] = extractelement <vscale x 2 x i1> [[TMP14]], i32 0
+; TFFALLBACK-NEXT:    br i1 [[TMP15]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; TFFALLBACK:       for.cond.cleanup:
+; TFFALLBACK-NEXT:    ret double [[TMP11]]
+;
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %fma_sum = phi double [ 0.000000e+00, %entry ], [ %muladd, %for.body ]
+  %gep = getelementptr double, ptr %b, i64 %indvars.iv
+  %load = load double, ptr %gep
+  %muladd = tail call double @llvm.fmuladd.f64(double %load, double %m, double %fma_sum)
+  %toint = fptoui double %load to i64
+  %call = call i64 @foo(i64 %toint) #1
+  %arrayidx = getelementptr inbounds i64, ptr %a, i64 %indvars.iv
+  store i64 %call, ptr %arrayidx
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1025
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret double %muladd
+}
+
 declare i64 @foo(i64)
+declare double @llvm.fmuladd.f64(double, double, double)
 
 ;; scalable vector variants of foo
 declare <vscale x 2 x i64> @foo_uniform(i64, <vscale x 2 x i1>)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll b/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll
new file mode 100644
index 0000000000..ca3c935669
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll
@@ -0,0 +1,117 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt -S -mtriple aarch64 -mattr=+sve -passes=loop-vectorize -enable-vplan-native-path < %s | FileCheck %s
+
+@A = external local_unnamed_addr global [1024 x float], align 4
+@B = external local_unnamed_addr global [512 x float], align 4
+
+; Test if the vplan-native-path successfully vectorizes a loop using scalable vectors if the target preferes scalable vectors.
+define void @foo() {
+; CHECK-LABEL: define void @foo
+; CHECK-SAME: () #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
+; CHECK-NEXT:    [[TMP5:%.*]] = add <vscale x 4 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = mul <vscale x 4 x i64> [[TMP5]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP6]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 1, [[TMP8]]
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP9]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[OUTER_LOOP_LATCH4:%.*]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[OUTER_LOOP_LATCH4]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [1024 x float], ptr @A, i64 0, <vscale x 4 x i64> [[VEC_IND]]
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0(<vscale x 4 x ptr> [[TMP10]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), <vscale x 4 x float> poison)
+; CHECK-NEXT:    br label [[INNER_LOOP1:%.*]]
+; CHECK:       inner_loop1:
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i64> [ zeroinitializer, [[VECTOR_BODY]] ], [ [[TMP13:%.*]], [[INNER_LOOP1]] ]
+; CHECK-NEXT:    [[VEC_PHI2:%.*]] = phi <vscale x 4 x float> [ [[WIDE_MASKED_GATHER]], [[VECTOR_BODY]] ], [ [[TMP12:%.*]], [[INNER_LOOP1]] ]
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [512 x float], ptr @B, i64 0, <vscale x 4 x i64> [[VEC_PHI]]
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0(<vscale x 4 x ptr> [[TMP11]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), <vscale x 4 x float> poison)
+; CHECK-NEXT:    [[TMP12]] = fmul <vscale x 4 x float> [[VEC_PHI2]], [[WIDE_MASKED_GATHER3]]
+; CHECK-NEXT:    [[TMP13]] = add nuw nsw <vscale x 4 x i64> [[VEC_PHI]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq <vscale x 4 x i64> [[TMP13]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 512, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <vscale x 4 x i1> [[TMP14]], i32 0
+; CHECK-NEXT:    br i1 [[TMP15]], label [[OUTER_LOOP_LATCH4]], label [[INNER_LOOP1]]
+; CHECK:       outer_loop_latch4:
+; CHECK-NEXT:    [[VEC_PHI5:%.*]] = phi <vscale x 4 x float> [ [[TMP12]], [[INNER_LOOP1]] ]
+; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4f32.nxv4p0(<vscale x 4 x float> [[VEC_PHI5]], <vscale x 4 x ptr> [[TMP10]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT:    [[TMP16:%.*]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq <vscale x 4 x i64> [[TMP16]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1024, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP19:%.*]] = mul i64 [[TMP18]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]]
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
+; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[OUTER_LOOP:%.*]]
+; CHECK:       outer_loop:
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[OUTER_LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds [1024 x float], ptr @A, i64 0, i64 [[I]]
+; CHECK-NEXT:    [[X_START:%.*]] = load float, ptr [[ARRAYIDX1]], align 4
+; CHECK-NEXT:    br label [[INNER_LOOP:%.*]]
+; CHECK:       inner_loop:
+; CHECK-NEXT:    [[J:%.*]] = phi i64 [ 0, [[OUTER_LOOP]] ], [ [[J_NEXT:%.*]], [[INNER_LOOP]] ]
+; CHECK-NEXT:    [[X:%.*]] = phi float [ [[X_START]], [[OUTER_LOOP]] ], [ [[X_NEXT:%.*]], [[INNER_LOOP]] ]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds [512 x float], ptr @B, i64 0, i64 [[J]]
+; CHECK-NEXT:    [[B:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[X_NEXT]] = fmul float [[X]], [[B]]
+; CHECK-NEXT:    [[J_NEXT]] = add nuw nsw i64 [[J]], 1
+; CHECK-NEXT:    [[INNER_EXITCOND:%.*]] = icmp eq i64 [[J_NEXT]], 512
+; CHECK-NEXT:    br i1 [[INNER_EXITCOND]], label [[OUTER_LOOP_LATCH]], label [[INNER_LOOP]]
+; CHECK:       outer_loop_latch:
+; CHECK-NEXT:    [[X_NEXT_LCSSA:%.*]] = phi float [ [[X_NEXT]], [[INNER_LOOP]] ]
+; CHECK-NEXT:    store float [[X_NEXT_LCSSA]], ptr [[ARRAYIDX1]], align 4
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
+; CHECK-NEXT:    [[OUTER_EXITCOND:%.*]] = icmp eq i64 [[I_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[OUTER_EXITCOND]], label [[EXIT]], label [[OUTER_LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %outer_loop
+
+outer_loop:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %outer_loop_latch ]
+  %arrayidx1 = getelementptr inbounds [1024 x float], ptr @A, i64 0, i64 %i
+  %x.start = load float, ptr %arrayidx1, align 4
+  br label %inner_loop
+
+inner_loop:
+  %j = phi i64 [ 0, %outer_loop ], [ %j.next, %inner_loop ]
+  %x = phi float [ %x.start, %outer_loop ], [ %x.next, %inner_loop ]
+  %arrayidx2 = getelementptr inbounds [512 x float], ptr @B, i64 0, i64 %j
+  %b = load float, ptr %arrayidx2, align 4
+  %x.next = fmul float %x, %b
+  %j.next = add nuw nsw i64 %j, 1
+  %inner_exitcond = icmp eq i64 %j.next, 512
+  br i1 %inner_exitcond, label %outer_loop_latch, label %inner_loop
+
+outer_loop_latch:
+  store float %x.next, ptr %arrayidx1, align 4
+  %i.next = add nuw nsw i64 %i, 1
+  %outer_exitcond = icmp eq i64 %i.next, 1024
+  br i1 %outer_exitcond, label %exit, label %outer_loop, !llvm.loop !1
+
+exit:
+  ret void
+}
+
+!1 = distinct !{!1, !2}
+!2 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sleef-calls-aarch64.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sleef-calls-aarch64.ll
index d25e24efd5..d7dc122eda 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sleef-calls-aarch64.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sleef-calls-aarch64.ll
@@ -1,21 +1,26 @@
-; Do NOT use -O3. It will lower exp2 to ldexp, and the test will fail.
-; RUN: opt -vector-library=sleefgnuabi -replace-with-veclib < %s | opt -vector-library=sleefgnuabi -passes=inject-tli-mappings,loop-unroll,loop-vectorize -S | FileCheck %s --check-prefixes=CHECK,NEON
-; RUN: opt -mattr=+sve -vector-library=sleefgnuabi -replace-with-veclib < %s | opt -vector-library=sleefgnuabi -passes=inject-tli-mappings,loop-unroll,loop-vectorize -S | FileCheck %s --check-prefixes=CHECK,SVE
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter "(_)|(cos|exp[^e]|fmod|gamma|log|pow|sin|sqrt|tan)|(ret)" --version 2
+; RUN: opt -vector-library=sleefgnuabi -passes=inject-tli-mappings,loop-vectorize -force-vector-interleave=1 -S < %s | FileCheck %s --check-prefix=NEON
+; RUN: opt -mattr=+sve -vector-library=sleefgnuabi -passes=inject-tli-mappings,loop-vectorize -force-vector-interleave=1 -S < %s | FileCheck %s --check-prefix=SVE
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64-unknown-linux-gnu"
 
-declare double @acos(double) #0
-declare float @acosf(float) #0
-declare double @llvm.acos.f64(double) #0
-declare float @llvm.acos.f32(float) #0
+declare double @acos(double)
+declare float @acosf(float)
 
 define void @acos_f64(double* nocapture %varray) {
-  ; CHECK-LABEL: @acos_f64(
-  ; NEON:     [[TMP5:%.*]] = call <2 x double> @_ZGVnN2v_acos(<2 x double> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_acos(<vscale x 2 x double> [[TMP4:%.*]], <vscale x 2 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @acos_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_acos(<2 x double> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call double @acos(double [[CONV:%.*]]) #[[ATTR0:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @acos_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0:[0-9]+]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_acos(<vscale x 2 x double> [[TMP11:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call double @acos(double [[CONV:%.*]]) #[[ATTR2:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -35,11 +40,18 @@ define void @acos_f64(double* nocapture %varray) {
 }
 
 define void @acos_f32(float* nocapture %varray) {
-  ; CHECK-LABEL: @acos_f32(
-  ; NEON:    [[TMP5:%.*]] = call <4 x float> @_ZGVnN4v_acosf(<4 x float> [[TMP4:%.*]])
-  ; SVE:     [[TMP5:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_acosf(<vscale x 4 x float> [[TMP4:%.*]], <vscale x 4 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @acos_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_acosf(<4 x float> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call float @acosf(float [[CONV:%.*]]) #[[ATTR1:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @acos_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_acosf(<vscale x 4 x float> [[TMP11:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call float @acosf(float [[CONV:%.*]]) #[[ATTR3:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -58,17 +70,22 @@ define void @acos_f32(float* nocapture %varray) {
   ret void
 }
 
-declare double @asin(double) #0
-declare float @asinf(float) #0
-declare double @llvm.asin.f64(double) #0
-declare float @llvm.asin.f32(float) #0
+declare double @asin(double)
+declare float @asinf(float)
 
 define void @asin_f64(double* nocapture %varray) {
-  ; CHECK-LABEL: @asin_f64(
-  ; NEON:     [[TMP5:%.*]] = call <2 x double> @_ZGVnN2v_asin(<2 x double> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_asin(<vscale x 2 x double> [[TMP4:%.*]], <vscale x 2 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @asin_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_asin(<2 x double> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call double @asin(double [[CONV:%.*]]) #[[ATTR2:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @asin_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_asin(<vscale x 2 x double> [[TMP11:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call double @asin(double [[CONV:%.*]]) #[[ATTR4:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -88,11 +105,18 @@ define void @asin_f64(double* nocapture %varray) {
 }
 
 define void @asin_f32(float* nocapture %varray) {
-  ; CHECK-LABEL: @asin_f32(
-  ; NEON:     [[TMP5:%.*]] = call <4 x float> @_ZGVnN4v_asinf(<4 x float> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_asinf(<vscale x 4 x float> [[TMP4:%.*]], <vscale x 4 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @asin_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_asinf(<4 x float> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call float @asinf(float [[CONV:%.*]]) #[[ATTR3:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @asin_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_asinf(<vscale x 4 x float> [[TMP11:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call float @asinf(float [[CONV:%.*]]) #[[ATTR5:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -111,17 +135,22 @@ define void @asin_f32(float* nocapture %varray) {
   ret void
 }
 
-declare double @atan(double) #0
-declare float @atanf(float) #0
-declare double @llvm.atan.f64(double) #0
-declare float @llvm.atan.f32(float) #0
+declare double @atan(double)
+declare float @atanf(float)
 
 define void @atan_f64(double* nocapture %varray) {
-  ; CHECK-LABEL: @atan_f64(
-  ; NEON:     [[TMP5:%.*]] = call <2 x double> @_ZGVnN2v_atan(<2 x double> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_atan(<vscale x 2 x double> [[TMP4:%.*]], <vscale x 2 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @atan_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_atan(<2 x double> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call double @atan(double [[CONV:%.*]]) #[[ATTR4:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @atan_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_atan(<vscale x 2 x double> [[TMP11:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call double @atan(double [[CONV:%.*]]) #[[ATTR6:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -141,11 +170,18 @@ define void @atan_f64(double* nocapture %varray) {
 }
 
 define void @atan_f32(float* nocapture %varray) {
-  ; CHECK-LABEL: @atan_f32(
-  ; NEON:     [[TMP5:%.*]] = call <4 x float> @_ZGVnN4v_atanf(<4 x float> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_atanf(<vscale x 4 x float> [[TMP4:%.*]], <vscale x 4 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @atan_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_atanf(<4 x float> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call float @atanf(float [[CONV:%.*]]) #[[ATTR5:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @atan_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_atanf(<vscale x 4 x float> [[TMP11:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call float @atanf(float [[CONV:%.*]]) #[[ATTR7:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -164,17 +200,22 @@ define void @atan_f32(float* nocapture %varray) {
   ret void
 }
 
-declare double @atan2(double, double) #0
-declare float @atan2f(float, float) #0
-declare double @llvm.atan2.f64(double, double) #0
-declare float @llvm.atan2.f32(float, float) #0
+declare double @atan2(double, double)
+declare float @atan2f(float, float)
 
 define void @atan2_f64(double* nocapture %varray) {
-  ; CHECK-LABEL: @atan2_f64(
-  ; NEON:     [[TMP5:%.*]] = call <2 x double> @_ZGVnN2vv_atan2(<2 x double> [[TMP4:%.*]], <2 x double> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_atan2(<vscale x 2 x double> [[TMP4:%.*]], <vscale x 2 x double> [[TMP4:%.*]], <vscale x 2 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @atan2_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2vv_atan2(<2 x double> [[TMP1:%.*]], <2 x double> [[TMP1]])
+; NEON:    [[CALL:%.*]] = tail call double @atan2(double [[CONV:%.*]], double [[CONV]]) #[[ATTR6:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @atan2_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_atan2(<vscale x 2 x double> [[TMP11:%.*]], <vscale x 2 x double> [[TMP11]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call double @atan2(double [[CONV:%.*]], double [[CONV]]) #[[ATTR8:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -194,11 +235,18 @@ define void @atan2_f64(double* nocapture %varray) {
 }
 
 define void @atan2_f32(float* nocapture %varray) {
-  ; CHECK-LABEL: @atan2_f32(
-  ; NEON:     [[TMP5:%.*]] = call <4 x float> @_ZGVnN4vv_atan2f(<4 x float> [[TMP4:%.*]], <4 x float> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_atan2f(<vscale x 4 x float> [[TMP4:%.*]], <vscale x 4 x float> [[TMP4:%.*]], <vscale x 4 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @atan2_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4vv_atan2f(<4 x float> [[TMP1:%.*]], <4 x float> [[TMP1]])
+; NEON:    [[CALL:%.*]] = tail call float @atan2f(float [[CONV:%.*]], float [[CONV]]) #[[ATTR7:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @atan2_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_atan2f(<vscale x 4 x float> [[TMP11:%.*]], <vscale x 4 x float> [[TMP11]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call float @atan2f(float [[CONV:%.*]], float [[CONV]]) #[[ATTR9:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -217,17 +265,22 @@ define void @atan2_f32(float* nocapture %varray) {
   ret void
 }
 
-declare double @atanh(double) #0
-declare float @atanhf(float) #0
-declare double @llvm.atanh.f64(double) #0
-declare float @llvm.atanh.f32(float) #0
+declare double @atanh(double)
+declare float @atanhf(float)
 
 define void @atanh_f64(double* nocapture %varray) {
-  ; CHECK-LABEL: @atanh_f64(
-  ; NEON:     [[TMP5:%.*]] = call <2 x double> @_ZGVnN2v_atanh(<2 x double> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_atanh(<vscale x 2 x double> [[TMP4:%.*]], <vscale x 2 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @atanh_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_atanh(<2 x double> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call double @atanh(double [[CONV:%.*]]) #[[ATTR8:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @atanh_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_atanh(<vscale x 2 x double> [[TMP11:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call double @atanh(double [[CONV:%.*]]) #[[ATTR10:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -247,11 +300,18 @@ define void @atanh_f64(double* nocapture %varray) {
 }
 
 define void @atanh_f32(float* nocapture %varray) {
-  ; CHECK-LABEL: @atanh_f32(
-  ; NEON:     [[TMP5:%.*]] = call <4 x float> @_ZGVnN4v_atanhf(<4 x float> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_atanhf(<vscale x 4 x float> [[TMP4:%.*]], <vscale x 4 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @atanh_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_atanhf(<4 x float> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call float @atanhf(float [[CONV:%.*]]) #[[ATTR9:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @atanh_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_atanhf(<vscale x 4 x float> [[TMP11:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call float @atanhf(float [[CONV:%.*]]) #[[ATTR11:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -270,17 +330,22 @@ define void @atanh_f32(float* nocapture %varray) {
   ret void
 }
 
-declare double @cos(double) #0
-declare float @cosf(float) #0
-declare double @llvm.cos.f64(double) #0
-declare float @llvm.cos.f32(float) #0
+declare double @cos(double)
+declare float @cosf(float)
 
 define void @cos_f64(double* nocapture %varray) {
-  ; CHECK-LABEL: @cos_f64(
-  ; NEON:     [[TMP5:%.*]] = call <2 x double> @_ZGVnN2v_cos(<2 x double> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cos(<vscale x 2 x double> [[TMP4:%.*]], <vscale x 2 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @cos_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_cos(<2 x double> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call double @cos(double [[CONV:%.*]]) #[[ATTR10:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @cos_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cos(<vscale x 2 x double> [[TMP11:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call double @cos(double [[CONV:%.*]]) #[[ATTR12:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -300,11 +365,18 @@ define void @cos_f64(double* nocapture %varray) {
 }
 
 define void @cos_f32(float* nocapture %varray) {
-  ; CHECK-LABEL: @cos_f32(
-  ; NEON:     [[TMP5:%.*]] = call <4 x float> @_ZGVnN4v_cosf(<4 x float> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_cosf(<vscale x 4 x float> [[TMP4:%.*]], <vscale x 4 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @cos_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_cosf(<4 x float> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call float @cosf(float [[CONV:%.*]]) #[[ATTR11:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @cos_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_cosf(<vscale x 4 x float> [[TMP11:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call float @cosf(float [[CONV:%.*]]) #[[ATTR13:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -323,17 +395,22 @@ define void @cos_f32(float* nocapture %varray) {
   ret void
 }
 
-declare double @cosh(double) #0
-declare float @coshf(float) #0
-declare double @llvm.cosh.f64(double) #0
-declare float @llvm.cosh.f32(float) #0
+declare double @cosh(double)
+declare float @coshf(float)
 
 define void @cosh_f64(double* nocapture %varray) {
-  ; CHECK-LABEL: @cosh_f64(
-  ; NEON:     [[TMP5:%.*]] = call <2 x double> @_ZGVnN2v_cosh(<2 x double> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cosh(<vscale x 2 x double> [[TMP4:%.*]], <vscale x 2 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @cosh_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_cosh(<2 x double> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call double @cosh(double [[CONV:%.*]]) #[[ATTR12:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @cosh_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cosh(<vscale x 2 x double> [[TMP11:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call double @cosh(double [[CONV:%.*]]) #[[ATTR14:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -353,11 +430,18 @@ define void @cosh_f64(double* nocapture %varray) {
 }
 
 define void @cosh_f32(float* nocapture %varray) {
-  ; CHECK-LABEL: @cosh_f32(
-  ; NEON:     [[TMP5:%.*]] = call <4 x float> @_ZGVnN4v_coshf(<4 x float> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_coshf(<vscale x 4 x float> [[TMP4:%.*]], <vscale x 4 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @cosh_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_coshf(<4 x float> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call float @coshf(float [[CONV:%.*]]) #[[ATTR13:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @cosh_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_coshf(<vscale x 4 x float> [[TMP11:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call float @coshf(float [[CONV:%.*]]) #[[ATTR15:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -376,17 +460,22 @@ define void @cosh_f32(float* nocapture %varray) {
   ret void
 }
 
-declare double @exp(double) #0
-declare float @expf(float) #0
-declare double @llvm.exp.f64(double) #0
-declare float @llvm.exp.f32(float) #0
+declare double @exp(double)
+declare float @expf(float)
 
 define void @exp_f64(double* nocapture %varray) {
-  ; CHECK-LABEL: @exp_f64(
-  ; NEON:     [[TMP5:%.*]] = call <2 x double> @_ZGVnN2v_exp(<2 x double> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp(<vscale x 2 x double> [[TMP4:%.*]], <vscale x 2 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @exp_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_exp(<2 x double> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call double @exp(double [[CONV:%.*]]) #[[ATTR14:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @exp_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp(<vscale x 2 x double> [[TMP11:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call double @exp(double [[CONV:%.*]]) #[[ATTR16:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -406,11 +495,18 @@ define void @exp_f64(double* nocapture %varray) {
 }
 
 define void @exp_f32(float* nocapture %varray) {
-  ; CHECK-LABEL: @exp_f32(
-  ; NEON:     [[TMP5:%.*]] = call <4 x float> @_ZGVnN4v_expf(<4 x float> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_expf(<vscale x 4 x float> [[TMP4:%.*]], <vscale x 4 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @exp_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_expf(<4 x float> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call float @expf(float [[CONV:%.*]]) #[[ATTR15:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @exp_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_expf(<vscale x 4 x float> [[TMP11:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call float @expf(float [[CONV:%.*]]) #[[ATTR17:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -429,17 +525,22 @@ define void @exp_f32(float* nocapture %varray) {
   ret void
 }
 
-declare double @exp2(double) #0
-declare float @exp2f(float) #0
-declare double @llvm.exp2.f64(double) #0
-declare float @llvm.exp2.f32(float) #0
+declare double @exp2(double)
+declare float @exp2f(float)
 
 define void @exp2_f64(double* nocapture %varray) {
-  ; CHECK-LABEL: @exp2_f64(
-  ; NEON:     [[TMP5:%.*]] = call <2 x double> @_ZGVnN2v_exp2(<2 x double> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp2(<vscale x 2 x double> [[TMP4:%.*]], <vscale x 2 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @exp2_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_exp2(<2 x double> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call double @exp2(double [[CONV:%.*]]) #[[ATTR16:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @exp2_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp2(<vscale x 2 x double> [[TMP11:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call double @exp2(double [[CONV:%.*]]) #[[ATTR18:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -459,11 +560,18 @@ define void @exp2_f64(double* nocapture %varray) {
 }
 
 define void @exp2_f32(float* nocapture %varray) {
-  ; CHECK-LABEL: @exp2_f32(
-  ; NEON:     [[TMP5:%.*]] = call <4 x float> @_ZGVnN4v_exp2f(<4 x float> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_exp2f(<vscale x 4 x float> [[TMP4:%.*]], <vscale x 4 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @exp2_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_exp2f(<4 x float> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call float @exp2f(float [[CONV:%.*]]) #[[ATTR17:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @exp2_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_exp2f(<vscale x 4 x float> [[TMP11:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call float @exp2f(float [[CONV:%.*]]) #[[ATTR19:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -482,17 +590,22 @@ define void @exp2_f32(float* nocapture %varray) {
   ret void
 }
 
-declare double @exp10(double) #0
-declare float @exp10f(float) #0
-declare double @llvm.exp10.f64(double) #0
-declare float @llvm.exp10.f32(float) #0
+declare double @exp10(double)
+declare float @exp10f(float)
 
 define void @exp10_f64(double* nocapture %varray) {
-  ; CHECK-LABEL: @exp10_f64(
-  ; NEON:     [[TMP5:%.*]] = call <2 x double> @_ZGVnN2v_exp10(<2 x double> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp10(<vscale x 2 x double> [[TMP4:%.*]], <vscale x 2 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @exp10_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_exp10(<2 x double> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call double @exp10(double [[CONV:%.*]]) #[[ATTR18:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @exp10_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp10(<vscale x 2 x double> [[TMP11:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call double @exp10(double [[CONV:%.*]]) #[[ATTR20:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -512,11 +625,18 @@ define void @exp10_f64(double* nocapture %varray) {
 }
 
 define void @exp10_f32(float* nocapture %varray) {
-  ; CHECK-LABEL: @exp10_f32(
-  ; NEON:     [[TMP5:%.*]] = call <4 x float> @_ZGVnN4v_exp10f(<4 x float> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_exp10f(<vscale x 4 x float> [[TMP4:%.*]], <vscale x 4 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @exp10_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_exp10f(<4 x float> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call float @exp10f(float [[CONV:%.*]]) #[[ATTR19:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @exp10_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_exp10f(<vscale x 4 x float> [[TMP11:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call float @exp10f(float [[CONV:%.*]]) #[[ATTR21:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -535,14 +655,25 @@ define void @exp10_f32(float* nocapture %varray) {
   ret void
 }
 
-declare double @fmod(double, double) #0
-declare float @fmodf(float, float) #0
+; There are no TLI mappings to fixed vector functions for fmod and fmodf.
+
+declare double @fmod(double, double)
+declare float @fmodf(float, float)
 
 define void @fmod_f64(double* nocapture %varray) {
-  ; CHECK-LABEL: @fmod_f64(
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_fmod(<vscale x 2 x double> [[TMP4:%.*]], <vscale x 2 x double> [[TMP4:%.*]], <vscale x 2 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @fmod_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP3:%.*]] = tail call double @fmod(double [[TMP2:%.*]], double [[TMP2]]) #[[ATTR20:[0-9]+]]
+; NEON:    [[TMP5:%.*]] = tail call double @fmod(double [[TMP4:%.*]], double [[TMP4]]) #[[ATTR20]]
+; NEON:    [[CALL:%.*]] = tail call double @fmod(double [[CONV:%.*]], double [[CONV]]) #[[ATTR20]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @fmod_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_fmod(<vscale x 2 x double> [[TMP11:%.*]], <vscale x 2 x double> [[TMP11]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call double @fmod(double [[CONV:%.*]], double [[CONV]]) #[[ATTR22:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -562,10 +693,19 @@ define void @fmod_f64(double* nocapture %varray) {
 }
 
 define void @fmod_f32(float* nocapture %varray) {
-  ; CHECK-LABEL: @fmod_f32(
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_fmodf(<vscale x 4 x float> [[TMP4:%.*]], <vscale x 4 x float> [[TMP4:%.*]], <vscale x 4 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @fmod_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP3:%.*]] = tail call float @fmodf(float [[TMP2:%.*]], float [[TMP2]]) #[[ATTR21:[0-9]+]]
+; NEON:    [[TMP5:%.*]] = tail call float @fmodf(float [[TMP4:%.*]], float [[TMP4]]) #[[ATTR21]]
+; NEON:    [[CALL:%.*]] = tail call float @fmodf(float [[CONV:%.*]], float [[CONV]]) #[[ATTR21]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @fmod_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_fmodf(<vscale x 4 x float> [[TMP11:%.*]], <vscale x 4 x float> [[TMP11]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call float @fmodf(float [[CONV:%.*]], float [[CONV]]) #[[ATTR23:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -584,17 +724,22 @@ define void @fmod_f32(float* nocapture %varray) {
   ret void
 }
 
-declare double @lgamma(double) #0
-declare float @lgammaf(float) #0
-declare double @llvm.lgamma.f64(double) #0
-declare float @llvm.lgamma.f32(float) #0
+declare double @lgamma(double)
+declare float @lgammaf(float)
 
 define void @lgamma_f64(double* nocapture %varray) {
-  ; CHECK-LABEL: @lgamma_f64(
-  ; NEON:     [[TMP5:%.*]] = call <2 x double> @_ZGVnN2v_lgamma(<2 x double> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_lgamma(<vscale x 2 x double> [[TMP4:%.*]], <vscale x 2 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @lgamma_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_lgamma(<2 x double> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call double @lgamma(double [[CONV:%.*]]) #[[ATTR22:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @lgamma_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_lgamma(<vscale x 2 x double> [[TMP11:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call double @lgamma(double [[CONV:%.*]]) #[[ATTR24:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -614,11 +759,18 @@ define void @lgamma_f64(double* nocapture %varray) {
 }
 
 define void @lgamma_f32(float* nocapture %varray) {
-  ; CHECK-LABEL: @lgamma_f32(
-  ; NEON:     [[TMP5:%.*]] = call <4 x float> @_ZGVnN4v_lgammaf(<4 x float> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_lgammaf(<vscale x 4 x float> [[TMP4:%.*]], <vscale x 4 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @lgamma_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_lgammaf(<4 x float> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call float @lgammaf(float [[CONV:%.*]]) #[[ATTR23:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @lgamma_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_lgammaf(<vscale x 4 x float> [[TMP11:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call float @lgammaf(float [[CONV:%.*]]) #[[ATTR25:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -637,17 +789,22 @@ define void @lgamma_f32(float* nocapture %varray) {
   ret void
 }
 
-declare double @log10(double) #0
-declare float @log10f(float) #0
-declare double @llvm.log10.f64(double) #0
-declare float @llvm.log10.f32(float) #0
+declare double @log10(double)
+declare float @log10f(float)
 
 define void @log10_f64(double* nocapture %varray) {
-  ; CHECK-LABEL: @log10_f64(
-  ; NEON:     [[TMP5:%.*]] = call <2 x double> @_ZGVnN2v_log10(<2 x double> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log10(<vscale x 2 x double> [[TMP4:%.*]], <vscale x 2 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @log10_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_log10(<2 x double> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call double @log10(double [[CONV:%.*]]) #[[ATTR24:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @log10_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log10(<vscale x 2 x double> [[TMP11:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call double @log10(double [[CONV:%.*]]) #[[ATTR26:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -667,11 +824,18 @@ define void @log10_f64(double* nocapture %varray) {
 }
 
 define void @log10_f32(float* nocapture %varray) {
-  ; CHECK-LABEL: @log10_f32(
-  ; NEON:     [[TMP5:%.*]] = call <4 x float> @_ZGVnN4v_log10f(<4 x float> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_log10f(<vscale x 4 x float> [[TMP4:%.*]], <vscale x 4 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @log10_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_log10f(<4 x float> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call float @log10f(float [[CONV:%.*]]) #[[ATTR25:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @log10_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_log10f(<vscale x 4 x float> [[TMP11:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call float @log10f(float [[CONV:%.*]]) #[[ATTR27:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -690,17 +854,22 @@ define void @log10_f32(float* nocapture %varray) {
   ret void
 }
 
-declare double @log2(double) #0
-declare float @log2f(float) #0
-declare double @llvm.log2.f64(double) #0
-declare float @llvm.log2.f32(float) #0
+declare double @log2(double)
+declare float @log2f(float)
 
 define void @log2_f64(double* nocapture %varray) {
-  ; CHECK-LABEL: @log2_f64(
-  ; NEON:    [[TMP5:%.*]] = call <2 x double> @_ZGVnN2v_log2(<2 x double> [[TMP4:%.*]])
-  ; SVE:     [[TMP5:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log2(<vscale x 2 x double> [[TMP4:%.*]], <vscale x 2 x i1> {{.*}})
-  ; CHECK:   ret void
-  ;
+; NEON-LABEL: define void @log2_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_log2(<2 x double> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call double @log2(double [[CONV:%.*]]) #[[ATTR26:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @log2_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log2(<vscale x 2 x double> [[TMP11:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call double @log2(double [[CONV:%.*]]) #[[ATTR28:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -720,11 +889,18 @@ define void @log2_f64(double* nocapture %varray) {
 }
 
 define void @log2_f32(float* nocapture %varray) {
-  ; CHECK-LABEL: @log2_f32(
-  ; NEON:    [[TMP5:%.*]] = call <4 x float> @_ZGVnN4v_log2f(<4 x float> [[TMP4:%.*]])
-  ; SVE:     [[TMP5:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_log2f(<vscale x 4 x float> [[TMP4:%.*]], <vscale x 4 x i1> {{.*}})
-  ; CHECK:   ret void
-  ;
+; NEON-LABEL: define void @log2_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_log2f(<4 x float> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call float @log2f(float [[CONV:%.*]]) #[[ATTR27:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @log2_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_log2f(<vscale x 4 x float> [[TMP11:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call float @log2f(float [[CONV:%.*]]) #[[ATTR29:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -743,17 +919,22 @@ define void @log2_f32(float* nocapture %varray) {
   ret void
 }
 
-declare double @log(double) #0
-declare float @logf(float) #0
-declare double @llvm.log.f64(double) #0
-declare float @llvm.log.f32(float) #0
+declare double @log(double)
+declare float @logf(float)
 
 define void @log_f64(double* nocapture %varray) {
-  ; CHECK-LABEL: @log_f64(
-  ; NEON:     [[TMP5:%.*]] = call <2 x double> @_ZGVnN2v_log(<2 x double> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log(<vscale x 2 x double> [[TMP4:%.*]], <vscale x 2 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @log_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_log(<2 x double> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call double @log(double [[CONV:%.*]]) #[[ATTR28:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @log_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log(<vscale x 2 x double> [[TMP11:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call double @log(double [[CONV:%.*]]) #[[ATTR30:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -773,11 +954,18 @@ define void @log_f64(double* nocapture %varray) {
 }
 
 define void @log_f32(float* nocapture %varray) {
-  ; CHECK-LABEL: @log_f32(
-  ; NEON:     [[TMP5:%.*]] = call <4 x float> @_ZGVnN4v_logf(<4 x float> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_logf(<vscale x 4 x float> [[TMP4:%.*]], <vscale x 4 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @log_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_logf(<4 x float> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call float @logf(float [[CONV:%.*]]) #[[ATTR29:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @log_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_logf(<vscale x 4 x float> [[TMP11:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call float @logf(float [[CONV:%.*]]) #[[ATTR31:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -796,17 +984,22 @@ define void @log_f32(float* nocapture %varray) {
   ret void
 }
 
-declare double @pow(double, double) #0
-declare float @powf(float, float) #0
-declare double @llvm.pow.f64(double, double) #0
-declare float @llvm.pow.f32(float, float) #0
+declare double @pow(double, double)
+declare float @powf(float, float)
 
 define void @pow_f64(double* nocapture %varray) {
-  ; CHECK-LABEL: @pow_f64(
-  ; NEON:     [[TMP5:%.*]] = call <2 x double> @_ZGVnN2vv_pow(<2 x double> [[TMP4:%.*]], <2 x double> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_pow(<vscale x 2 x double> [[TMP4:%.*]], <vscale x 2 x double> [[TMP4:%.*]], <vscale x 2 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @pow_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2vv_pow(<2 x double> [[TMP1:%.*]], <2 x double> [[TMP1]])
+; NEON:    [[CALL:%.*]] = tail call double @pow(double [[CONV:%.*]], double [[CONV]]) #[[ATTR30:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @pow_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_pow(<vscale x 2 x double> [[TMP11:%.*]], <vscale x 2 x double> [[TMP11]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call double @pow(double [[CONV:%.*]], double [[CONV]]) #[[ATTR32:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -826,11 +1019,18 @@ define void @pow_f64(double* nocapture %varray) {
 }
 
 define void @pow_f32(float* nocapture %varray) {
-  ; CHECK-LABEL: @pow_f32(
-  ; NEON:     [[TMP5:%.*]] = call <4 x float> @_ZGVnN4vv_powf(<4 x float> [[TMP4:%.*]], <4 x float> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_powf(<vscale x 4 x float> [[TMP4:%.*]], <vscale x 4 x float> [[TMP4:%.*]], <vscale x 4 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @pow_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4vv_powf(<4 x float> [[TMP1:%.*]], <4 x float> [[TMP1]])
+; NEON:    [[CALL:%.*]] = tail call float @powf(float [[CONV:%.*]], float [[CONV]]) #[[ATTR31:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @pow_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_powf(<vscale x 4 x float> [[TMP11:%.*]], <vscale x 4 x float> [[TMP11]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call float @powf(float [[CONV:%.*]], float [[CONV]]) #[[ATTR33:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -849,17 +1049,22 @@ define void @pow_f32(float* nocapture %varray) {
   ret void
 }
 
-declare double @sin(double) #0
-declare float @sinf(float) #0
-declare double @llvm.sin.f64(double) #0
-declare float @llvm.sin.f32(float) #0
+declare double @sin(double)
+declare float @sinf(float)
 
 define void @sin_f64(double* nocapture %varray) {
-  ; CHECK-LABEL: @sin_f64(
-  ; NEON:     [[TMP5:%.*]] = call <2 x double> @_ZGVnN2v_sin(<2 x double> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sin(<vscale x 2 x double> [[TMP4:%.*]], <vscale x 2 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @sin_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_sin(<2 x double> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call double @sin(double [[CONV:%.*]]) #[[ATTR32:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @sin_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sin(<vscale x 2 x double> [[TMP11:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call double @sin(double [[CONV:%.*]]) #[[ATTR34:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -879,11 +1084,18 @@ define void @sin_f64(double* nocapture %varray) {
 }
 
 define void @sin_f32(float* nocapture %varray) {
-  ; CHECK-LABEL: @sin_f32(
-  ; NEON:     [[TMP5:%.*]] = call <4 x float> @_ZGVnN4v_sinf(<4 x float> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sinf(<vscale x 4 x float> [[TMP4:%.*]], <vscale x 4 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @sin_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_sinf(<4 x float> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call float @sinf(float [[CONV:%.*]]) #[[ATTR33:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @sin_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sinf(<vscale x 4 x float> [[TMP11:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call float @sinf(float [[CONV:%.*]]) #[[ATTR35:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -902,17 +1114,22 @@ define void @sin_f32(float* nocapture %varray) {
   ret void
 }
 
-declare double @sinh(double) #0
-declare float @sinhf(float) #0
-declare double @llvm.sinh.f64(double) #0
-declare float @llvm.sinh.f32(float) #0
+declare double @sinh(double)
+declare float @sinhf(float)
 
 define void @sinh_f64(double* nocapture %varray) {
-  ; CHECK-LABEL: @sinh_f64(
-  ; NEON:     [[TMP5:%.*]] = call <2 x double> @_ZGVnN2v_sinh(<2 x double> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sinh(<vscale x 2 x double> [[TMP4:%.*]], <vscale x 2 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @sinh_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_sinh(<2 x double> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call double @sinh(double [[CONV:%.*]]) #[[ATTR34:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @sinh_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sinh(<vscale x 2 x double> [[TMP11:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call double @sinh(double [[CONV:%.*]]) #[[ATTR36:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -932,11 +1149,18 @@ define void @sinh_f64(double* nocapture %varray) {
 }
 
 define void @sinh_f32(float* nocapture %varray) {
-  ; CHECK-LABEL: @sinh_f32(
-  ; NEON:     [[TMP5:%.*]] = call <4 x float> @_ZGVnN4v_sinhf(<4 x float> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sinhf(<vscale x 4 x float> [[TMP4:%.*]], <vscale x 4 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @sinh_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_sinhf(<4 x float> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call float @sinhf(float [[CONV:%.*]]) #[[ATTR35:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @sinh_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sinhf(<vscale x 4 x float> [[TMP11:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call float @sinhf(float [[CONV:%.*]]) #[[ATTR37:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -955,17 +1179,22 @@ define void @sinh_f32(float* nocapture %varray) {
   ret void
 }
 
-declare double @sqrt(double) #0
-declare float @sqrtf(float) #0
-declare double @llvm.sqrt.f64(double) #0
-declare float @llvm.sqrt.f32(float) #0
+declare double @sqrt(double)
+declare float @sqrtf(float)
 
 define void @sqrt_f64(double* nocapture %varray) {
-  ; CHECK-LABEL: @sqrt_f64(
-  ; NEON:     [[TMP5:%.*]] = call <2 x double> @_ZGVnN2v_sqrt(<2 x double> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sqrt(<vscale x 2 x double> [[TMP4:%.*]], <vscale x 2 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @sqrt_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_sqrt(<2 x double> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call double @sqrt(double [[CONV:%.*]]) #[[ATTR36:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @sqrt_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sqrt(<vscale x 2 x double> [[TMP11:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call double @sqrt(double [[CONV:%.*]]) #[[ATTR38:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -985,11 +1214,18 @@ define void @sqrt_f64(double* nocapture %varray) {
 }
 
 define void @sqrt_f32(float* nocapture %varray) {
-  ; CHECK-LABEL: @sqrt_f32(
-  ; NEON:     [[TMP5:%.*]] = call <4 x float> @_ZGVnN4v_sqrtf(<4 x float> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sqrtf(<vscale x 4 x float> [[TMP4:%.*]], <vscale x 4 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @sqrt_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_sqrtf(<4 x float> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call float @sqrtf(float [[CONV:%.*]]) #[[ATTR37:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @sqrt_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sqrtf(<vscale x 4 x float> [[TMP11:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call float @sqrtf(float [[CONV:%.*]]) #[[ATTR39:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -1008,65 +1244,22 @@ define void @sqrt_f32(float* nocapture %varray) {
   ret void
 }
 
-define void @llvm_sqrt_f64(double* nocapture %varray) {
-  ; CHECK-LABEL: @llvm_sqrt_f64(
-  ; NEON:     [[TMP5:%.*]] = call fast <2 x double> @llvm.sqrt.v2f64(<2 x double> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call fast <vscale x 2 x double> @llvm.sqrt.nxv2f64(<vscale x 2 x double> [[TMP4:%.*]])
-  ; CHECK:    ret void
-  ;
-  entry:
-  br label %for.body
-
-  for.body:
-  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
-  %tmp = trunc i64 %iv to i32
-  %conv = sitofp i32 %tmp to double
-  %call = tail call fast double @llvm.sqrt.f64(double %conv)
-  %arrayidx = getelementptr inbounds double, double* %varray, i64 %iv
-  store double %call, double* %arrayidx, align 8
-  %iv.next = add nuw nsw i64 %iv, 1
-  %exitcond = icmp eq i64 %iv.next, 1000
-  br i1 %exitcond, label %for.end, label %for.body
-
-  for.end:
-  ret void
-}
-
-define void @llvm_sqrt_f32(float* nocapture %varray) {
-  ; CHECK-LABEL: @llvm_sqrt_f32(
-  ; NEON:     [[TMP5:%.*]] = call fast <4 x float> @llvm.sqrt.v4f32(<4 x float> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call fast <vscale x 4 x float> @llvm.sqrt.nxv4f32(<vscale x 4 x float> [[TMP4:%.*]])
-  ; CHECK:    ret void
-  ;
-  entry:
-  br label %for.body
-
-  for.body:
-  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
-  %tmp = trunc i64 %iv to i32
-  %conv = sitofp i32 %tmp to float
-  %call = tail call fast float @llvm.sqrt.f32(float %conv)
-  %arrayidx = getelementptr inbounds float, float* %varray, i64 %iv
-  store float %call, float* %arrayidx, align 4
-  %iv.next = add nuw nsw i64 %iv, 1
-  %exitcond = icmp eq i64 %iv.next, 1000
-  br i1 %exitcond, label %for.end, label %for.body
-
-  for.end:
-  ret void
-}
-
-declare double @tan(double) #0
-declare float @tanf(float) #0
-declare double @llvm.tan.f64(double) #0
-declare float @llvm.tan.f32(float) #0
+declare double @tan(double)
+declare float @tanf(float)
 
 define void @tan_f64(double* nocapture %varray) {
-  ; CHECK-LABEL: @tan_f64(
-  ; NEON:     [[TMP5:%.*]] = call <2 x double> @_ZGVnN2v_tan(<2 x double> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_tan(<vscale x 2 x double> [[TMP4:%.*]], <vscale x 2 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @tan_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_tan(<2 x double> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call double @tan(double [[CONV:%.*]]) #[[ATTR38:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @tan_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_tan(<vscale x 2 x double> [[TMP11:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call double @tan(double [[CONV:%.*]]) #[[ATTR40:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -1086,11 +1279,18 @@ define void @tan_f64(double* nocapture %varray) {
 }
 
 define void @tan_f32(float* nocapture %varray) {
-  ; CHECK-LABEL: @tan_f32(
-  ; NEON:     [[TMP5:%.*]] = call <4 x float> @_ZGVnN4v_tanf(<4 x float> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_tanf(<vscale x 4 x float> [[TMP4:%.*]], <vscale x 4 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @tan_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_tanf(<4 x float> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call float @tanf(float [[CONV:%.*]]) #[[ATTR39:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @tan_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_tanf(<vscale x 4 x float> [[TMP11:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call float @tanf(float [[CONV:%.*]]) #[[ATTR41:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -1109,17 +1309,22 @@ define void @tan_f32(float* nocapture %varray) {
   ret void
 }
 
-declare double @tanh(double) #0
-declare float @tanhf(float) #0
-declare double @llvm.tanh.f64(double) #0
-declare float @llvm.tanh.f32(float) #0
+declare double @tanh(double)
+declare float @tanhf(float)
 
 define void @tanh_f64(double* nocapture %varray) {
-  ; CHECK-LABEL: @tanh_f64(
-  ; NEON:     [[TMP5:%.*]] = call <2 x double> @_ZGVnN2v_tanh(<2 x double> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_tanh(<vscale x 2 x double> [[TMP4:%.*]], <vscale x 2 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @tanh_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_tanh(<2 x double> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call double @tanh(double [[CONV:%.*]]) #[[ATTR40:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @tanh_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_tanh(<vscale x 2 x double> [[TMP11:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call double @tanh(double [[CONV:%.*]]) #[[ATTR42:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -1139,11 +1344,18 @@ define void @tanh_f64(double* nocapture %varray) {
 }
 
 define void @tanh_f32(float* nocapture %varray) {
-  ; CHECK-LABEL: @tanh_f32(
-  ; NEON:     [[TMP5:%.*]] = call <4 x float> @_ZGVnN4v_tanhf(<4 x float> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_tanhf(<vscale x 4 x float> [[TMP4:%.*]], <vscale x 4 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @tanh_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_tanhf(<4 x float> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call float @tanhf(float [[CONV:%.*]]) #[[ATTR41:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @tanh_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_tanhf(<vscale x 4 x float> [[TMP11:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call float @tanhf(float [[CONV:%.*]]) #[[ATTR43:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -1162,17 +1374,22 @@ define void @tanh_f32(float* nocapture %varray) {
   ret void
 }
 
-declare double @tgamma(double) #0
-declare float @tgammaf(float) #0
-declare double @llvm.tgamma.f64(double) #0
-declare float @llvm.tgamma.f32(float) #0
+declare double @tgamma(double)
+declare float @tgammaf(float)
 
 define void @tgamma_f64(double* nocapture %varray) {
-  ; CHECK-LABEL: @tgamma_f64(
-  ; NEON:     [[TMP5:%.*]] = call <2 x double> @_ZGVnN2v_tgamma(<2 x double> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_tgamma(<vscale x 2 x double> [[TMP4:%.*]], <vscale x 2 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @tgamma_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_tgamma(<2 x double> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call double @tgamma(double [[CONV:%.*]]) #[[ATTR42:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @tgamma_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_tgamma(<vscale x 2 x double> [[TMP11:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call double @tgamma(double [[CONV:%.*]]) #[[ATTR44:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
@@ -1192,11 +1409,18 @@ define void @tgamma_f64(double* nocapture %varray) {
 }
 
 define void @tgamma_f32(float* nocapture %varray) {
-  ; CHECK-LABEL: @tgamma_f32(
-  ; NEON:     [[TMP5:%.*]] = call <4 x float> @_ZGVnN4v_tgammaf(<4 x float> [[TMP4:%.*]])
-  ; SVE:      [[TMP5:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_tgammaf(<vscale x 4 x float> [[TMP4:%.*]], <vscale x 4 x i1> {{.*}})
-  ; CHECK:    ret void
-  ;
+; NEON-LABEL: define void @tgamma_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_tgammaf(<4 x float> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call float @tgammaf(float [[CONV:%.*]]) #[[ATTR43:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @tgamma_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR0]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_tgammaf(<vscale x 4 x float> [[TMP11:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call float @tgammaf(float [[CONV:%.*]]) #[[ATTR45:[0-9]+]]
+; SVE:    ret void
+;
   entry:
   br label %for.body
 
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sleef-intrinsic-calls-aarch64.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sleef-intrinsic-calls-aarch64.ll
new file mode 100644
index 0000000000..715c2c352b
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sleef-intrinsic-calls-aarch64.ll
@@ -0,0 +1,1311 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter "(\.|_)(ceil|copysign|cos|exp[^e]|exp2|fabs|floor|fma|log|m..num|pow|.*int|round|sin|sqrt|trunc)|(ret)" --version 2
+; RUN: opt -vector-library=sleefgnuabi -passes=inject-tli-mappings,loop-vectorize -force-vector-interleave=1 -S < %s | FileCheck %s --check-prefix=NEON
+; RUN: opt -mattr=+sve -vector-library=sleefgnuabi -passes=inject-tli-mappings,loop-vectorize -force-vector-interleave=1 -S < %s | FileCheck %s --check-prefix=SVE
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-unknown-linux-gnu"
+
+
+; Tests are checking if LV can vectorize loops with llvm math intrinsics using mappings
+; from TLI (if such mappings exist) for scalable and fixed width vectors.
+
+declare double @llvm.ceil.f64(double)
+declare float @llvm.ceil.f32(float)
+
+define void @llvm_ceil_f64(double* nocapture %varray) {
+; NEON-LABEL: define void @llvm_ceil_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @llvm.ceil.v2f64(<2 x double> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call double @llvm.ceil.f64(double [[CONV:%.*]])
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_ceil_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1:[0-9]+]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @llvm.ceil.nxv2f64(<vscale x 2 x double> [[TMP11:%.*]])
+; SVE:    [[CALL:%.*]] = tail call double @llvm.ceil.f64(double [[CONV:%.*]])
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @llvm.ceil.f64(double %conv)
+  %arrayidx = getelementptr inbounds double, double* %varray, i64 %iv
+  store double %call, double* %arrayidx, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+define void @llvm_ceil_f32(float* nocapture %varray) {
+; NEON-LABEL: define void @llvm_ceil_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @llvm.ceil.v4f32(<4 x float> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call float @llvm.ceil.f32(float [[CONV:%.*]])
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_ceil_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.ceil.nxv4f32(<vscale x 4 x float> [[TMP11:%.*]])
+; SVE:    [[CALL:%.*]] = tail call float @llvm.ceil.f32(float [[CONV:%.*]])
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @llvm.ceil.f32(float %conv)
+  %arrayidx = getelementptr inbounds float, float* %varray, i64 %iv
+  store float %call, float* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+declare double @llvm.copysign.f64(double, double)
+declare float @llvm.copysign.f32(float, float)
+
+define void @llvm_copysign_f64(double* nocapture %varray) {
+; NEON-LABEL: define void @llvm_copysign_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @llvm.copysign.v2f64(<2 x double> [[TMP1:%.*]], <2 x double> [[TMP1]])
+; NEON:    [[CALL:%.*]] = tail call double @llvm.copysign.f64(double [[CONV:%.*]], double [[CONV]])
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_copysign_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @llvm.copysign.nxv2f64(<vscale x 2 x double> [[TMP11:%.*]], <vscale x 2 x double> [[TMP11]])
+; SVE:    [[CALL:%.*]] = tail call double @llvm.copysign.f64(double [[CONV:%.*]], double [[CONV]])
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @llvm.copysign.f64(double %conv, double %conv)
+  %arrayidx = getelementptr inbounds double, double* %varray, i64 %iv
+  store double %call, double* %arrayidx, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+define void @llvm_copysign_f32(float* nocapture %varray) {
+; NEON-LABEL: define void @llvm_copysign_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @llvm.copysign.v4f32(<4 x float> [[TMP1:%.*]], <4 x float> [[TMP1]])
+; NEON:    [[CALL:%.*]] = tail call float @llvm.copysign.f32(float [[CONV:%.*]], float [[CONV]])
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_copysign_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.copysign.nxv4f32(<vscale x 4 x float> [[TMP11:%.*]], <vscale x 4 x float> [[TMP11]])
+; SVE:    [[CALL:%.*]] = tail call float @llvm.copysign.f32(float [[CONV:%.*]], float [[CONV]])
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @llvm.copysign.f32(float %conv, float %conv)
+  %arrayidx = getelementptr inbounds float, float* %varray, i64 %iv
+  store float %call, float* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+declare double @llvm.cos.f64(double)
+declare float @llvm.cos.f32(float)
+
+define void @llvm_cos_f64(double* nocapture %varray) {
+; NEON-LABEL: define void @llvm_cos_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_cos(<2 x double> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call double @llvm.cos.f64(double [[CONV:%.*]]) #[[ATTR1:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_cos_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_cos(<vscale x 2 x double> [[TMP11:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call double @llvm.cos.f64(double [[CONV:%.*]]) #[[ATTR4:[0-9]+]]
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @llvm.cos.f64(double %conv)
+  %arrayidx = getelementptr inbounds double, double* %varray, i64 %iv
+  store double %call, double* %arrayidx, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+define void @llvm_cos_f32(float* nocapture %varray) {
+; NEON-LABEL: define void @llvm_cos_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_cosf(<4 x float> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call float @llvm.cos.f32(float [[CONV:%.*]]) #[[ATTR2:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_cos_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_cosf(<vscale x 4 x float> [[TMP11:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call float @llvm.cos.f32(float [[CONV:%.*]]) #[[ATTR5:[0-9]+]]
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @llvm.cos.f32(float %conv)
+  %arrayidx = getelementptr inbounds float, float* %varray, i64 %iv
+  store float %call, float* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+declare double @llvm.exp.f64(double)
+declare float @llvm.exp.f32(float)
+
+define void @llvm_exp_f64(double* nocapture %varray) {
+; NEON-LABEL: define void @llvm_exp_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_exp(<2 x double> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call double @llvm.exp.f64(double [[CONV:%.*]]) #[[ATTR3:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_exp_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp(<vscale x 2 x double> [[TMP11:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call double @llvm.exp.f64(double [[CONV:%.*]]) #[[ATTR6:[0-9]+]]
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @llvm.exp.f64(double %conv)
+  %arrayidx = getelementptr inbounds double, double* %varray, i64 %iv
+  store double %call, double* %arrayidx, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+define void @llvm_exp_f32(float* nocapture %varray) {
+; NEON-LABEL: define void @llvm_exp_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_expf(<4 x float> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call float @llvm.exp.f32(float [[CONV:%.*]]) #[[ATTR4:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_exp_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_expf(<vscale x 4 x float> [[TMP11:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call float @llvm.exp.f32(float [[CONV:%.*]]) #[[ATTR7:[0-9]+]]
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @llvm.exp.f32(float %conv)
+  %arrayidx = getelementptr inbounds float, float* %varray, i64 %iv
+  store float %call, float* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+declare double @llvm.exp2.f64(double)
+declare float @llvm.exp2.f32(float)
+
+define void @llvm_exp2_f64(double* nocapture %varray) {
+; NEON-LABEL: define void @llvm_exp2_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_exp2(<2 x double> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call double @llvm.exp2.f64(double [[CONV:%.*]]) #[[ATTR5:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_exp2_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_exp2(<vscale x 2 x double> [[TMP11:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call double @llvm.exp2.f64(double [[CONV:%.*]]) #[[ATTR8:[0-9]+]]
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @llvm.exp2.f64(double %conv)
+  %arrayidx = getelementptr inbounds double, double* %varray, i64 %iv
+  store double %call, double* %arrayidx, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+define void @llvm_exp2_f32(float* nocapture %varray) {
+; NEON-LABEL: define void @llvm_exp2_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_exp2f(<4 x float> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call float @llvm.exp2.f32(float [[CONV:%.*]]) #[[ATTR6:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_exp2_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_exp2f(<vscale x 4 x float> [[TMP11:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call float @llvm.exp2.f32(float [[CONV:%.*]]) #[[ATTR9:[0-9]+]]
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @llvm.exp2.f32(float %conv)
+  %arrayidx = getelementptr inbounds float, float* %varray, i64 %iv
+  store float %call, float* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+declare double @llvm.fabs.f64(double)
+declare float @llvm.fabs.f32(float)
+
+define void @llvm_fabs_f64(double* nocapture %varray) {
+; NEON-LABEL: define void @llvm_fabs_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @llvm.fabs.v2f64(<2 x double> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call double @llvm.fabs.f64(double [[CONV:%.*]])
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_fabs_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @llvm.fabs.nxv2f64(<vscale x 2 x double> [[TMP11:%.*]])
+; SVE:    [[CALL:%.*]] = tail call double @llvm.fabs.f64(double [[CONV:%.*]])
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @llvm.fabs.f64(double %conv)
+  %arrayidx = getelementptr inbounds double, double* %varray, i64 %iv
+  store double %call, double* %arrayidx, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+
+define void @llvm_fabs_f32(float* nocapture %varray) {
+; NEON-LABEL: define void @llvm_fabs_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call float @llvm.fabs.f32(float [[CONV:%.*]])
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_fabs_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.fabs.nxv4f32(<vscale x 4 x float> [[TMP11:%.*]])
+; SVE:    [[CALL:%.*]] = tail call float @llvm.fabs.f32(float [[CONV:%.*]])
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @llvm.fabs.f32(float %conv)
+  %arrayidx = getelementptr inbounds float, float* %varray, i64 %iv
+  store float %call, float* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+declare double @llvm.floor.f64(double)
+declare float @llvm.floor.f32(float)
+
+define void @llvm_floor_f64(double* nocapture %varray) {
+; NEON-LABEL: define void @llvm_floor_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @llvm.floor.v2f64(<2 x double> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call double @llvm.floor.f64(double [[CONV:%.*]])
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_floor_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @llvm.floor.nxv2f64(<vscale x 2 x double> [[TMP11:%.*]])
+; SVE:    [[CALL:%.*]] = tail call double @llvm.floor.f64(double [[CONV:%.*]])
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @llvm.floor.f64(double %conv)
+  %arrayidx = getelementptr inbounds double, double* %varray, i64 %iv
+  store double %call, double* %arrayidx, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+define void @llvm_floor_f32(float* nocapture %varray) {
+; NEON-LABEL: define void @llvm_floor_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @llvm.floor.v4f32(<4 x float> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call float @llvm.floor.f32(float [[CONV:%.*]])
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_floor_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.floor.nxv4f32(<vscale x 4 x float> [[TMP11:%.*]])
+; SVE:    [[CALL:%.*]] = tail call float @llvm.floor.f32(float [[CONV:%.*]])
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @llvm.floor.f32(float %conv)
+  %arrayidx = getelementptr inbounds float, float* %varray, i64 %iv
+  store float %call, float* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+declare double @llvm.fma.f64(double, double, double)
+declare float @llvm.fma.f32(float, float, float)
+
+define void @llvm_fma_f64(double* nocapture %varray) {
+; NEON-LABEL: define void @llvm_fma_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @llvm.fma.v2f64(<2 x double> [[TMP1:%.*]], <2 x double> [[TMP1]], <2 x double> [[TMP1]])
+; NEON:    [[CALL:%.*]] = tail call double @llvm.fma.f64(double [[CONV:%.*]], double [[CONV]], double [[CONV]])
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_fma_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @llvm.fma.nxv2f64(<vscale x 2 x double> [[TMP11:%.*]], <vscale x 2 x double> [[TMP11]], <vscale x 2 x double> [[TMP11]])
+; SVE:    [[CALL:%.*]] = tail call double @llvm.fma.f64(double [[CONV:%.*]], double [[CONV]], double [[CONV]])
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @llvm.fma.f64(double %conv, double %conv, double %conv)
+  %arrayidx = getelementptr inbounds double, double* %varray, i64 %iv
+  store double %call, double* %arrayidx, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+define void @llvm_fma_f32(float* nocapture %varray) {
+; NEON-LABEL: define void @llvm_fma_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[TMP1:%.*]], <4 x float> [[TMP1]], <4 x float> [[TMP1]])
+; NEON:    [[CALL:%.*]] = tail call float @llvm.fma.f32(float [[CONV:%.*]], float [[CONV]], float [[CONV]])
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_fma_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.fma.nxv4f32(<vscale x 4 x float> [[TMP11:%.*]], <vscale x 4 x float> [[TMP11]], <vscale x 4 x float> [[TMP11]])
+; SVE:    [[CALL:%.*]] = tail call float @llvm.fma.f32(float [[CONV:%.*]], float [[CONV]], float [[CONV]])
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @llvm.fma.f32(float %conv, float %conv, float %conv)
+  %arrayidx = getelementptr inbounds float, float* %varray, i64 %iv
+  store float %call, float* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+declare double @llvm.log.f64(double)
+declare float @llvm.log.f32(float)
+
+define void @llvm_log_f64(double* nocapture %varray) {
+; NEON-LABEL: define void @llvm_log_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_log(<2 x double> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call double @llvm.log.f64(double [[CONV:%.*]]) #[[ATTR7:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_log_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log(<vscale x 2 x double> [[TMP11:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call double @llvm.log.f64(double [[CONV:%.*]]) #[[ATTR10:[0-9]+]]
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @llvm.log.f64(double %conv)
+  %arrayidx = getelementptr inbounds double, double* %varray, i64 %iv
+  store double %call, double* %arrayidx, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+define void @llvm_log_f32(float* nocapture %varray) {
+; NEON-LABEL: define void @llvm_log_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_logf(<4 x float> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call float @llvm.log.f32(float [[CONV:%.*]]) #[[ATTR8:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_log_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_logf(<vscale x 4 x float> [[TMP11:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call float @llvm.log.f32(float [[CONV:%.*]]) #[[ATTR11:[0-9]+]]
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @llvm.log.f32(float %conv)
+  %arrayidx = getelementptr inbounds float, float* %varray, i64 %iv
+  store float %call, float* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+declare double @llvm.log10.f64(double)
+declare float @llvm.log10.f32(float)
+
+define void @llvm_log10_f64(double* nocapture %varray) {
+; NEON-LABEL: define void @llvm_log10_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_log10(<2 x double> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call double @llvm.log10.f64(double [[CONV:%.*]]) #[[ATTR9:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_log10_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log10(<vscale x 2 x double> [[TMP11:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call double @llvm.log10.f64(double [[CONV:%.*]]) #[[ATTR12:[0-9]+]]
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @llvm.log10.f64(double %conv)
+  %arrayidx = getelementptr inbounds double, double* %varray, i64 %iv
+  store double %call, double* %arrayidx, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+define void @llvm_log10_f32(float* nocapture %varray) {
+; NEON-LABEL: define void @llvm_log10_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_log10f(<4 x float> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call float @llvm.log10.f32(float [[CONV:%.*]]) #[[ATTR10:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_log10_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_log10f(<vscale x 4 x float> [[TMP11:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call float @llvm.log10.f32(float [[CONV:%.*]]) #[[ATTR13:[0-9]+]]
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @llvm.log10.f32(float %conv)
+  %arrayidx = getelementptr inbounds float, float* %varray, i64 %iv
+  store float %call, float* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+declare double @llvm.log2.f64(double)
+declare float @llvm.log2.f32(float)
+
+define void @llvm_log2_f64(double* nocapture %varray) {
+; NEON-LABEL: define void @llvm_log2_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_log2(<2 x double> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call double @llvm.log2.f64(double [[CONV:%.*]]) #[[ATTR11:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_log2_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_log2(<vscale x 2 x double> [[TMP11:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call double @llvm.log2.f64(double [[CONV:%.*]]) #[[ATTR14:[0-9]+]]
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @llvm.log2.f64(double %conv)
+  %arrayidx = getelementptr inbounds double, double* %varray, i64 %iv
+  store double %call, double* %arrayidx, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+define void @llvm_log2_f32(float* nocapture %varray) {
+; NEON-LABEL: define void @llvm_log2_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_log2f(<4 x float> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call float @llvm.log2.f32(float [[CONV:%.*]]) #[[ATTR12:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_log2_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_log2f(<vscale x 4 x float> [[TMP11:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call float @llvm.log2.f32(float [[CONV:%.*]]) #[[ATTR15:[0-9]+]]
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @llvm.log2.f32(float %conv)
+  %arrayidx = getelementptr inbounds float, float* %varray, i64 %iv
+  store float %call, float* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+declare double @llvm.maxnum.f64(double, double)
+declare float @llvm.maxnum.f32(float, float)
+
+define void @llvm_maxnum_f64(double* nocapture %varray) {
+; NEON-LABEL: define void @llvm_maxnum_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @llvm.maxnum.v2f64(<2 x double> [[TMP1:%.*]], <2 x double> [[TMP1]])
+; NEON:    [[CALL:%.*]] = tail call double @llvm.maxnum.f64(double [[CONV:%.*]], double [[CONV]])
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_maxnum_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @llvm.maxnum.nxv2f64(<vscale x 2 x double> [[TMP11:%.*]], <vscale x 2 x double> [[TMP11]])
+; SVE:    [[CALL:%.*]] = tail call double @llvm.maxnum.f64(double [[CONV:%.*]], double [[CONV]])
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @llvm.maxnum.f64(double %conv, double %conv)
+  %arrayidx = getelementptr inbounds double, double* %varray, i64 %iv
+  store double %call, double* %arrayidx, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+define void @llvm_maxnum_f32(float* nocapture %varray) {
+; NEON-LABEL: define void @llvm_maxnum_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP1:%.*]], <4 x float> [[TMP1]])
+; NEON:    [[CALL:%.*]] = tail call float @llvm.maxnum.f32(float [[CONV:%.*]], float [[CONV]])
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_maxnum_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.maxnum.nxv4f32(<vscale x 4 x float> [[TMP11:%.*]], <vscale x 4 x float> [[TMP11]])
+; SVE:    [[CALL:%.*]] = tail call float @llvm.maxnum.f32(float [[CONV:%.*]], float [[CONV]])
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @llvm.maxnum.f32(float %conv, float %conv)
+  %arrayidx = getelementptr inbounds float, float* %varray, i64 %iv
+  store float %call, float* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+declare double @llvm.minnum.f64(double, double)
+declare float @llvm.minnum.f32(float, float)
+
+define void @llvm_minnum_f64(double* nocapture %varray) {
+; NEON-LABEL: define void @llvm_minnum_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @llvm.minnum.v2f64(<2 x double> [[TMP1:%.*]], <2 x double> [[TMP1]])
+; NEON:    [[CALL:%.*]] = tail call double @llvm.minnum.f64(double [[CONV:%.*]], double [[CONV]])
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_minnum_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @llvm.minnum.nxv2f64(<vscale x 2 x double> [[TMP11:%.*]], <vscale x 2 x double> [[TMP11]])
+; SVE:    [[CALL:%.*]] = tail call double @llvm.minnum.f64(double [[CONV:%.*]], double [[CONV]])
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @llvm.minnum.f64(double %conv, double %conv)
+  %arrayidx = getelementptr inbounds double, double* %varray, i64 %iv
+  store double %call, double* %arrayidx, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+define void @llvm_minnum_f32(float* nocapture %varray) {
+; NEON-LABEL: define void @llvm_minnum_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[TMP1:%.*]], <4 x float> [[TMP1]])
+; NEON:    [[CALL:%.*]] = tail call float @llvm.minnum.f32(float [[CONV:%.*]], float [[CONV]])
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_minnum_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.minnum.nxv4f32(<vscale x 4 x float> [[TMP11:%.*]], <vscale x 4 x float> [[TMP11]])
+; SVE:    [[CALL:%.*]] = tail call float @llvm.minnum.f32(float [[CONV:%.*]], float [[CONV]])
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @llvm.minnum.f32(float %conv, float %conv)
+  %arrayidx = getelementptr inbounds float, float* %varray, i64 %iv
+  store float %call, float* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+declare double @llvm.nearbyint.f64(double)
+declare float @llvm.nearbyint.f32(float)
+
+define void @llvm_nearbyint_f64(double* nocapture %varray) {
+; NEON-LABEL: define void @llvm_nearbyint_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @llvm.nearbyint.v2f64(<2 x double> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call double @llvm.nearbyint.f64(double [[CONV:%.*]])
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_nearbyint_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @llvm.nearbyint.nxv2f64(<vscale x 2 x double> [[TMP11:%.*]])
+; SVE:    [[CALL:%.*]] = tail call double @llvm.nearbyint.f64(double [[CONV:%.*]])
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @llvm.nearbyint.f64(double %conv)
+  %arrayidx = getelementptr inbounds double, double* %varray, i64 %iv
+  store double %call, double* %arrayidx, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+define void @llvm_nearbyint_f32(float* nocapture %varray) {
+; NEON-LABEL: define void @llvm_nearbyint_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @llvm.nearbyint.v4f32(<4 x float> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call float @llvm.nearbyint.f32(float [[CONV:%.*]])
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_nearbyint_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.nearbyint.nxv4f32(<vscale x 4 x float> [[TMP11:%.*]])
+; SVE:    [[CALL:%.*]] = tail call float @llvm.nearbyint.f32(float [[CONV:%.*]])
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @llvm.nearbyint.f32(float %conv)
+  %arrayidx = getelementptr inbounds float, float* %varray, i64 %iv
+  store float %call, float* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+declare double @llvm.pow.f64(double, double)
+declare float @llvm.pow.f32(float, float)
+
+define void @llvm_pow_f64(double* nocapture %varray) {
+; NEON-LABEL: define void @llvm_pow_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2vv_pow(<2 x double> [[TMP1:%.*]], <2 x double> [[TMP1]])
+; NEON:    [[CALL:%.*]] = tail call double @llvm.pow.f64(double [[CONV:%.*]], double [[CONV]]) #[[ATTR13:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_pow_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxvv_pow(<vscale x 2 x double> [[TMP11:%.*]], <vscale x 2 x double> [[TMP11]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call double @llvm.pow.f64(double [[CONV:%.*]], double [[CONV]]) #[[ATTR16:[0-9]+]]
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @llvm.pow.f64(double %conv, double %conv)
+  %arrayidx = getelementptr inbounds double, double* %varray, i64 %iv
+  store double %call, double* %arrayidx, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+define void @llvm_pow_f32(float* nocapture %varray) {
+; NEON-LABEL: define void @llvm_pow_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4vv_powf(<4 x float> [[TMP1:%.*]], <4 x float> [[TMP1]])
+; NEON:    [[CALL:%.*]] = tail call float @llvm.pow.f32(float [[CONV:%.*]], float [[CONV]]) #[[ATTR14:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_pow_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxvv_powf(<vscale x 4 x float> [[TMP11:%.*]], <vscale x 4 x float> [[TMP11]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call float @llvm.pow.f32(float [[CONV:%.*]], float [[CONV]]) #[[ATTR17:[0-9]+]]
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @llvm.pow.f32(float %conv, float %conv)
+  %arrayidx = getelementptr inbounds float, float* %varray, i64 %iv
+  store float %call, float* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+declare double @llvm.rint.f64(double)
+declare float @llvm.rint.f32(float)
+
+define void @llvm_rint_f64(double* nocapture %varray) {
+; NEON-LABEL: define void @llvm_rint_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @llvm.rint.v2f64(<2 x double> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call double @llvm.rint.f64(double [[CONV:%.*]])
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_rint_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @llvm.rint.nxv2f64(<vscale x 2 x double> [[TMP11:%.*]])
+; SVE:    [[CALL:%.*]] = tail call double @llvm.rint.f64(double [[CONV:%.*]])
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @llvm.rint.f64(double %conv)
+  %arrayidx = getelementptr inbounds double, double* %varray, i64 %iv
+  store double %call, double* %arrayidx, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+define void @llvm_rint_f32(float* nocapture %varray) {
+; NEON-LABEL: define void @llvm_rint_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @llvm.rint.v4f32(<4 x float> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call float @llvm.rint.f32(float [[CONV:%.*]])
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_rint_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.rint.nxv4f32(<vscale x 4 x float> [[TMP11:%.*]])
+; SVE:    [[CALL:%.*]] = tail call float @llvm.rint.f32(float [[CONV:%.*]])
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @llvm.rint.f32(float %conv)
+  %arrayidx = getelementptr inbounds float, float* %varray, i64 %iv
+  store float %call, float* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+declare double @llvm.round.f64(double)
+declare float @llvm.round.f32(float)
+
+define void @llvm_round_f64(double* nocapture %varray) {
+; NEON-LABEL: define void @llvm_round_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @llvm.round.v2f64(<2 x double> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call double @llvm.round.f64(double [[CONV:%.*]])
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_round_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @llvm.round.nxv2f64(<vscale x 2 x double> [[TMP11:%.*]])
+; SVE:    [[CALL:%.*]] = tail call double @llvm.round.f64(double [[CONV:%.*]])
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @llvm.round.f64(double %conv)
+  %arrayidx = getelementptr inbounds double, double* %varray, i64 %iv
+  store double %call, double* %arrayidx, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+define void @llvm_round_f32(float* nocapture %varray) {
+; NEON-LABEL: define void @llvm_round_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @llvm.round.v4f32(<4 x float> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call float @llvm.round.f32(float [[CONV:%.*]])
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_round_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.round.nxv4f32(<vscale x 4 x float> [[TMP11:%.*]])
+; SVE:    [[CALL:%.*]] = tail call float @llvm.round.f32(float [[CONV:%.*]])
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @llvm.round.f32(float %conv)
+  %arrayidx = getelementptr inbounds float, float* %varray, i64 %iv
+  store float %call, float* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+declare double @llvm.sin.f64(double)
+declare float @llvm.sin.f32(float)
+
+define void @llvm_sin_f64(double* nocapture %varray) {
+; NEON-LABEL: define void @llvm_sin_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @_ZGVnN2v_sin(<2 x double> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call double @llvm.sin.f64(double [[CONV:%.*]]) #[[ATTR15:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_sin_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @_ZGVsMxv_sin(<vscale x 2 x double> [[TMP11:%.*]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call double @llvm.sin.f64(double [[CONV:%.*]]) #[[ATTR18:[0-9]+]]
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @llvm.sin.f64(double %conv)
+  %arrayidx = getelementptr inbounds double, double* %varray, i64 %iv
+  store double %call, double* %arrayidx, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+define void @llvm_sin_f32(float* nocapture %varray) {
+; NEON-LABEL: define void @llvm_sin_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @_ZGVnN4v_sinf(<4 x float> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call float @llvm.sin.f32(float [[CONV:%.*]]) #[[ATTR16:[0-9]+]]
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_sin_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @_ZGVsMxv_sinf(<vscale x 4 x float> [[TMP11:%.*]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; SVE:    [[CALL:%.*]] = tail call float @llvm.sin.f32(float [[CONV:%.*]]) #[[ATTR19:[0-9]+]]
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @llvm.sin.f32(float %conv)
+  %arrayidx = getelementptr inbounds float, float* %varray, i64 %iv
+  store float %call, float* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+declare double @llvm.sqrt.f64(double)
+declare float @llvm.sqrt.f32(float)
+
+define void @llvm_sqrt_f64(double* nocapture %varray) {
+; NEON-LABEL: define void @llvm_sqrt_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @llvm.sqrt.v2f64(<2 x double> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call double @llvm.sqrt.f64(double [[CONV:%.*]])
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_sqrt_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @llvm.sqrt.nxv2f64(<vscale x 2 x double> [[TMP11:%.*]])
+; SVE:    [[CALL:%.*]] = tail call double @llvm.sqrt.f64(double [[CONV:%.*]])
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @llvm.sqrt.f64(double %conv)
+  %arrayidx = getelementptr inbounds double, double* %varray, i64 %iv
+  store double %call, double* %arrayidx, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+define void @llvm_sqrt_f32(float* nocapture %varray) {
+; NEON-LABEL: define void @llvm_sqrt_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @llvm.sqrt.v4f32(<4 x float> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call float @llvm.sqrt.f32(float [[CONV:%.*]])
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_sqrt_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.sqrt.nxv4f32(<vscale x 4 x float> [[TMP11:%.*]])
+; SVE:    [[CALL:%.*]] = tail call float @llvm.sqrt.f32(float [[CONV:%.*]])
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @llvm.sqrt.f32(float %conv)
+  %arrayidx = getelementptr inbounds float, float* %varray, i64 %iv
+  store float %call, float* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+declare double @llvm.trunc.f64(double)
+declare float @llvm.trunc.f32(float)
+
+define void @llvm_trunc_f64(double* nocapture %varray) {
+; NEON-LABEL: define void @llvm_trunc_f64
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <2 x double> @llvm.trunc.v2f64(<2 x double> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call double @llvm.trunc.f64(double [[CONV:%.*]])
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_trunc_f64
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 2 x double> @llvm.trunc.nxv2f64(<vscale x 2 x double> [[TMP11:%.*]])
+; SVE:    [[CALL:%.*]] = tail call double @llvm.trunc.f64(double [[CONV:%.*]])
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @llvm.trunc.f64(double %conv)
+  %arrayidx = getelementptr inbounds double, double* %varray, i64 %iv
+  store double %call, double* %arrayidx, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
+
+define void @llvm_trunc_f32(float* nocapture %varray) {
+; NEON-LABEL: define void @llvm_trunc_f32
+; NEON-SAME: (ptr nocapture [[VARRAY:%.*]]) {
+; NEON:    [[TMP2:%.*]] = call <4 x float> @llvm.trunc.v4f32(<4 x float> [[TMP1:%.*]])
+; NEON:    [[CALL:%.*]] = tail call float @llvm.trunc.f32(float [[CONV:%.*]])
+; NEON:    ret void
+;
+; SVE-LABEL: define void @llvm_trunc_f32
+; SVE-SAME: (ptr nocapture [[VARRAY:%.*]]) #[[ATTR1]] {
+; SVE:    [[TMP12:%.*]] = call <vscale x 4 x float> @llvm.trunc.nxv4f32(<vscale x 4 x float> [[TMP11:%.*]])
+; SVE:    [[CALL:%.*]] = tail call float @llvm.trunc.f32(float [[CONV:%.*]])
+; SVE:    ret void
+;
+  entry:
+  br label %for.body
+
+  for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @llvm.trunc.f32(float %conv)
+  %arrayidx = getelementptr inbounds float, float* %varray, i64 %iv
+  store float %call, float* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+  for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-known-trip-count.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-known-trip-count.ll
index fcfa79e7ce..c3e30f1f81 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-known-trip-count.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-known-trip-count.ll
@@ -387,10 +387,10 @@ for.body:                                         ; preds = %entry, %for.body
 }
 
 ; Larger example with predication that should also not be vectorized
-; CHECK-LABEL: predicated
+; CHECK-LABEL: predicated_test
 ; CHECK: LV: Selecting VF: 1
 ; CHECK: LV: Selecting VF: 1
-define dso_local i32 @predicated(i32 noundef %0, ptr %glob) #0 {
+define dso_local i32 @predicated_test(i32 noundef %0, ptr %glob) #0 {
   %2 = alloca [101 x i32], align 4
   %3 = alloca [21 x i32], align 4
   call void @llvm.lifetime.start.p0(i64 404, ptr nonnull %2)
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
index d2534d7d18..cad64f5c7e 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
@@ -51,7 +51,7 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:  LV: Scalarizing: %cmp = icmp ugt i64 %indvars.iv, 1
 ; CHECK-NEXT:  LV: Scalarizing: %indvars.iv.next = add nsw i64 %indvars.iv, -1
 ; CHECK-NEXT:  VPlan 'Initial VPlan for VF={vscale x 4},UF>=1' {
-; CHECK-NEXT:  Live-in vp<%0> = vector-trip-count
+; CHECK-NEXT:  Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
 ; CHECK-NEXT:  vp<%1> = original trip-count
 ; CHECK:       ph:
 ; CHECK-NEXT:    EMIT vp<%1> = EXPAND SCEV (zext i32 %n to i64)
@@ -60,18 +60,18 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:  Successor(s): vector loop
 ; CHECK:       <x1> vector loop: {
 ; CHECK-NEXT:    vector.body:
-; CHECK-NEXT:    EMIT vp<%2> = CANONICAL-INDUCTION
-; CHECK-NEXT:    vp<%3> = DERIVED-IV ir<%n> + vp<%2> * ir<-1>
-; CHECK-NEXT:    vp<%4> = SCALAR-STEPS vp<%3>, ir<-1>
-; CHECK-NEXT:    CLONE ir<%i.0> = add nsw vp<%4>, ir<-1>
+; CHECK-NEXT:    EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
+; CHECK-NEXT:    vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1>
+; CHECK-NEXT:    vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<-1>
+; CHECK-NEXT:    CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1>
 ; CHECK-NEXT:    CLONE ir<%idxprom> = zext ir<%i.0>
 ; CHECK-NEXT:    CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom>
 ; CHECK-NEXT:    WIDEN ir<%1> = load ir<%arrayidx>
 ; CHECK-NEXT:    WIDEN ir<%add9> = add ir<%1>, ir<1>
 ; CHECK-NEXT:    CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom>
 ; CHECK-NEXT:    WIDEN store ir<%arrayidx3>, ir<%add9>
-; CHECK-NEXT:    EMIT vp<%11> = VF * UF + nuw vp<%2>
-; CHECK-NEXT:    EMIT branch-on-count vp<%11>, vp<%0>
+; CHECK-NEXT:    EMIT vp<[[IV_INC:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]>
+; CHECK-NEXT:    EMIT branch-on-count vp<[[IV_INC]]>, vp<[[VEC_TC]]>
 ; CHECK-NEXT:    No successors
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  Successor(s): middle.block
@@ -188,7 +188,7 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:  LV: Scalarizing: %cmp = icmp ugt i64 %indvars.iv, 1
 ; CHECK-NEXT:  LV: Scalarizing: %indvars.iv.next = add nsw i64 %indvars.iv, -1
 ; CHECK-NEXT:  VPlan 'Initial VPlan for VF={vscale x 4},UF>=1' {
-; CHECK-NEXT:  Live-in vp<%0> = vector-trip-count
+; CHECK-NEXT:  Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
 ; CHECK-NEXT:  vp<%1> = original trip-count
 ; CHECK:       ph:
 ; CHECK-NEXT:    EMIT vp<%1> = EXPAND SCEV (zext i32 %n to i64)
@@ -197,18 +197,18 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:  Successor(s): vector loop
 ; CHECK:       <x1> vector loop: {
 ; CHECK-NEXT:    vector.body:
-; CHECK-NEXT:    EMIT vp<%2> = CANONICAL-INDUCTION
-; CHECK-NEXT:    vp<%3> = DERIVED-IV ir<%n> + vp<%2> * ir<-1>
-; CHECK-NEXT:    vp<%4> = SCALAR-STEPS vp<%3>, ir<-1>
-; CHECK-NEXT:    CLONE ir<%i.0> = add nsw vp<%4>, ir<-1>
+; CHECK-NEXT:    EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
+; CHECK-NEXT:    vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1>
+; CHECK-NEXT:    vp<[[STEPS]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<-1>
+; CHECK-NEXT:    CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1>
 ; CHECK-NEXT:    CLONE ir<%idxprom> = zext ir<%i.0>
 ; CHECK-NEXT:    CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom>
 ; CHECK-NEXT:    WIDEN ir<%1> = load ir<%arrayidx>
 ; CHECK-NEXT:    WIDEN ir<%conv1> = fadd ir<%1>, ir<1.000000e+00>
 ; CHECK-NEXT:    CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom>
 ; CHECK-NEXT:    WIDEN store ir<%arrayidx3>, ir<%conv1>
-; CHECK-NEXT:    EMIT vp<%11> = VF * UF + nuw vp<%2>
-; CHECK-NEXT:    EMIT branch-on-count vp<%11>, vp<%0>
+; CHECK-NEXT:    EMIT vp<[[IV_INC:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]>
+; CHECK-NEXT:    EMIT branch-on-count vp<[[IV_INC]]>, vp<[[VEC_TC]]>
 ; CHECK-NEXT:    No successors
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  Successor(s): middle.block
diff --git a/llvm/test/Transforms/LoopVectorize/X86/interleaving.ll b/llvm/test/Transforms/LoopVectorize/X86/interleaving.ll
index e8b9386005..24d9e824f6 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/interleaving.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/interleaving.ll
@@ -24,15 +24,15 @@ define void @foo(ptr noalias nocapture %a, ptr noalias nocapture readonly %b) {
 ; SSE-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <8 x i32> [[WIDE_VEC1]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
 ; SSE-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
 ; SSE-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <8 x i32> [[WIDE_VEC1]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; SSE-NEXT:    [[TMP7:%.*]] = add nsw <4 x i32> [[STRIDED_VEC3]], [[STRIDED_VEC]]
-; SSE-NEXT:    [[TMP8:%.*]] = add nsw <4 x i32> [[STRIDED_VEC4]], [[STRIDED_VEC2]]
-; SSE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
-; SSE-NEXT:    store <4 x i32> [[TMP7]], ptr [[TMP9]], align 4
-; SSE-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP9]], i64 4
-; SSE-NEXT:    store <4 x i32> [[TMP8]], ptr [[TMP11]], align 4
+; SSE-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[STRIDED_VEC3]], [[STRIDED_VEC]]
+; SSE-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> [[STRIDED_VEC4]], [[STRIDED_VEC2]]
+; SSE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
+; SSE-NEXT:    store <4 x i32> [[TMP5]], ptr [[TMP7]], align 4
+; SSE-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i64 4
+; SSE-NEXT:    store <4 x i32> [[TMP6]], ptr [[TMP8]], align 4
 ; SSE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; SSE-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; SSE-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; SSE-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; SSE-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; SSE:       middle.block:
 ; SSE-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
 ; SSE:       scalar.ph:
@@ -40,7 +40,7 @@ define void @foo(ptr noalias nocapture %a, ptr noalias nocapture readonly %b) {
 ; SSE:       for.cond.cleanup:
 ; SSE-NEXT:    ret void
 ; SSE:       for.body:
-; SSE-NEXT:    br i1 poison, label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]]
+; SSE-NEXT:    br i1 poison, label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ;
 ; AVX1-LABEL: @foo(
 ; AVX1-NEXT:  entry:
@@ -72,21 +72,21 @@ define void @foo(ptr noalias nocapture %a, ptr noalias nocapture readonly %b) {
 ; AVX1-NEXT:    [[STRIDED_VEC8:%.*]] = shufflevector <8 x i32> [[WIDE_VEC1]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
 ; AVX1-NEXT:    [[STRIDED_VEC9:%.*]] = shufflevector <8 x i32> [[WIDE_VEC2]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
 ; AVX1-NEXT:    [[STRIDED_VEC10:%.*]] = shufflevector <8 x i32> [[WIDE_VEC3]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; AVX1-NEXT:    [[TMP15:%.*]] = add nsw <4 x i32> [[STRIDED_VEC7]], [[STRIDED_VEC]]
-; AVX1-NEXT:    [[TMP16:%.*]] = add nsw <4 x i32> [[STRIDED_VEC8]], [[STRIDED_VEC4]]
-; AVX1-NEXT:    [[TMP17:%.*]] = add nsw <4 x i32> [[STRIDED_VEC9]], [[STRIDED_VEC5]]
-; AVX1-NEXT:    [[TMP18:%.*]] = add nsw <4 x i32> [[STRIDED_VEC10]], [[STRIDED_VEC6]]
-; AVX1-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
-; AVX1-NEXT:    store <4 x i32> [[TMP15]], ptr [[TMP19]], align 4
-; AVX1-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[TMP19]], i64 4
-; AVX1-NEXT:    store <4 x i32> [[TMP16]], ptr [[TMP21]], align 4
-; AVX1-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i32, ptr [[TMP19]], i64 8
-; AVX1-NEXT:    store <4 x i32> [[TMP17]], ptr [[TMP23]], align 4
-; AVX1-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[TMP19]], i64 12
-; AVX1-NEXT:    store <4 x i32> [[TMP18]], ptr [[TMP25]], align 4
+; AVX1-NEXT:    [[TMP11:%.*]] = add nsw <4 x i32> [[STRIDED_VEC7]], [[STRIDED_VEC]]
+; AVX1-NEXT:    [[TMP12:%.*]] = add nsw <4 x i32> [[STRIDED_VEC8]], [[STRIDED_VEC4]]
+; AVX1-NEXT:    [[TMP13:%.*]] = add nsw <4 x i32> [[STRIDED_VEC9]], [[STRIDED_VEC5]]
+; AVX1-NEXT:    [[TMP14:%.*]] = add nsw <4 x i32> [[STRIDED_VEC10]], [[STRIDED_VEC6]]
+; AVX1-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
+; AVX1-NEXT:    store <4 x i32> [[TMP11]], ptr [[TMP15]], align 4
+; AVX1-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[TMP15]], i64 4
+; AVX1-NEXT:    store <4 x i32> [[TMP12]], ptr [[TMP16]], align 4
+; AVX1-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP15]], i64 8
+; AVX1-NEXT:    store <4 x i32> [[TMP13]], ptr [[TMP17]], align 4
+; AVX1-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[TMP15]], i64 12
+; AVX1-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP18]], align 4
 ; AVX1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; AVX1-NEXT:    [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; AVX1-NEXT:    br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; AVX1-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; AVX1-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; AVX1:       middle.block:
 ; AVX1-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
 ; AVX1:       scalar.ph:
@@ -94,7 +94,7 @@ define void @foo(ptr noalias nocapture %a, ptr noalias nocapture readonly %b) {
 ; AVX1:       for.cond.cleanup:
 ; AVX1-NEXT:    ret void
 ; AVX1:       for.body:
-; AVX1-NEXT:    br i1 poison, label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]]
+; AVX1-NEXT:    br i1 poison, label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ;
 ; AVX2-LABEL: @foo(
 ; AVX2-NEXT:  entry:
@@ -126,21 +126,21 @@ define void @foo(ptr noalias nocapture %a, ptr noalias nocapture readonly %b) {
 ; AVX2-NEXT:    [[STRIDED_VEC8:%.*]] = shufflevector <16 x i32> [[WIDE_VEC1]], <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
 ; AVX2-NEXT:    [[STRIDED_VEC9:%.*]] = shufflevector <16 x i32> [[WIDE_VEC2]], <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
 ; AVX2-NEXT:    [[STRIDED_VEC10:%.*]] = shufflevector <16 x i32> [[WIDE_VEC3]], <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; AVX2-NEXT:    [[TMP15:%.*]] = add nsw <8 x i32> [[STRIDED_VEC7]], [[STRIDED_VEC]]
-; AVX2-NEXT:    [[TMP16:%.*]] = add nsw <8 x i32> [[STRIDED_VEC8]], [[STRIDED_VEC4]]
-; AVX2-NEXT:    [[TMP17:%.*]] = add nsw <8 x i32> [[STRIDED_VEC9]], [[STRIDED_VEC5]]
-; AVX2-NEXT:    [[TMP18:%.*]] = add nsw <8 x i32> [[STRIDED_VEC10]], [[STRIDED_VEC6]]
-; AVX2-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
-; AVX2-NEXT:    store <8 x i32> [[TMP15]], ptr [[TMP19]], align 4
-; AVX2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[TMP19]], i64 8
-; AVX2-NEXT:    store <8 x i32> [[TMP16]], ptr [[TMP21]], align 4
-; AVX2-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i32, ptr [[TMP19]], i64 16
-; AVX2-NEXT:    store <8 x i32> [[TMP17]], ptr [[TMP23]], align 4
-; AVX2-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[TMP19]], i64 24
-; AVX2-NEXT:    store <8 x i32> [[TMP18]], ptr [[TMP25]], align 4
+; AVX2-NEXT:    [[TMP11:%.*]] = add nsw <8 x i32> [[STRIDED_VEC7]], [[STRIDED_VEC]]
+; AVX2-NEXT:    [[TMP12:%.*]] = add nsw <8 x i32> [[STRIDED_VEC8]], [[STRIDED_VEC4]]
+; AVX2-NEXT:    [[TMP13:%.*]] = add nsw <8 x i32> [[STRIDED_VEC9]], [[STRIDED_VEC5]]
+; AVX2-NEXT:    [[TMP14:%.*]] = add nsw <8 x i32> [[STRIDED_VEC10]], [[STRIDED_VEC6]]
+; AVX2-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
+; AVX2-NEXT:    store <8 x i32> [[TMP11]], ptr [[TMP15]], align 4
+; AVX2-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[TMP15]], i64 8
+; AVX2-NEXT:    store <8 x i32> [[TMP12]], ptr [[TMP16]], align 4
+; AVX2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP15]], i64 16
+; AVX2-NEXT:    store <8 x i32> [[TMP13]], ptr [[TMP17]], align 4
+; AVX2-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[TMP15]], i64 24
+; AVX2-NEXT:    store <8 x i32> [[TMP14]], ptr [[TMP18]], align 4
 ; AVX2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
-; AVX2-NEXT:    [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; AVX2-NEXT:    br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; AVX2-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; AVX2-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; AVX2:       middle.block:
 ; AVX2-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
 ; AVX2:       scalar.ph:
@@ -148,7 +148,7 @@ define void @foo(ptr noalias nocapture %a, ptr noalias nocapture readonly %b) {
 ; AVX2:       for.cond.cleanup:
 ; AVX2-NEXT:    ret void
 ; AVX2:       for.body:
-; AVX2-NEXT:    br i1 poison, label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]]
+; AVX2-NEXT:    br i1 poison, label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ;
 ; ATOM-LABEL: @foo(
 ; ATOM-NEXT:  entry:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/invariant-store-vectorization.ll b/llvm/test/Transforms/LoopVectorize/X86/invariant-store-vectorization.ll
index ccee24887f..995973a645 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/invariant-store-vectorization.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/invariant-store-vectorization.ll
@@ -15,12 +15,12 @@ define i32 @inv_val_store_to_inv_address_with_reduction(ptr %a, i64 %n, ptr %b)
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX2]], 8
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; CHECK:       vector.memcheck:
-; CHECK-NEXT:    [[UGLYGEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 4
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 4
 ; CHECK-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
 ; CHECK-NEXT:    [[TMP0:%.*]] = shl i64 [[SMAX]], 2
-; CHECK-NEXT:    [[UGLYGEP1:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP0]]
-; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ugt ptr [[UGLYGEP1]], [[A]]
-; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ugt ptr [[UGLYGEP]], [[B]]
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ugt ptr [[SCEVGEP1]], [[A]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ugt ptr [[SCEVGEP]], [[B]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
 ; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label [[VEC_EPILOG_SCALAR_PH]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
 ; CHECK:       vector.main.loop.iter.check:
@@ -132,10 +132,10 @@ define void @inv_val_store_to_inv_address_conditional(ptr %a, i64 %n, ptr %b, i3
 ; CHECK:       vector.memcheck:
 ; CHECK-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
 ; CHECK-NEXT:    [[TMP0:%.*]] = shl i64 [[SMAX]], 2
-; CHECK-NEXT:    [[UGLYGEP:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP0]]
-; CHECK-NEXT:    [[UGLYGEP1:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 4
-; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ugt ptr [[UGLYGEP1]], [[B]]
-; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ugt ptr [[UGLYGEP]], [[A]]
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 4
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ugt ptr [[SCEVGEP1]], [[B]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ugt ptr [[SCEVGEP]], [[A]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
 ; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label [[VEC_EPILOG_SCALAR_PH]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
 ; CHECK:       vector.main.loop.iter.check:
@@ -245,18 +245,18 @@ define void @variant_val_store_to_inv_address_conditional(ptr %a, i64 %n, ptr %b
 ; CHECK:       vector.memcheck:
 ; CHECK-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
 ; CHECK-NEXT:    [[TMP0:%.*]] = shl i64 [[SMAX]], 2
-; CHECK-NEXT:    [[UGLYGEP:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP0]]
-; CHECK-NEXT:    [[UGLYGEP1:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 4
-; CHECK-NEXT:    [[UGLYGEP2:%.*]] = getelementptr i8, ptr [[C:%.*]], i64 [[TMP0]]
-; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ugt ptr [[UGLYGEP1]], [[B]]
-; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ugt ptr [[UGLYGEP]], [[A]]
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 4
+; CHECK-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[C:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ugt ptr [[SCEVGEP1]], [[B]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ugt ptr [[SCEVGEP]], [[A]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; CHECK-NEXT:    [[BOUND03:%.*]] = icmp ugt ptr [[UGLYGEP2]], [[B]]
-; CHECK-NEXT:    [[BOUND14:%.*]] = icmp ugt ptr [[UGLYGEP]], [[C]]
+; CHECK-NEXT:    [[BOUND03:%.*]] = icmp ugt ptr [[SCEVGEP2]], [[B]]
+; CHECK-NEXT:    [[BOUND14:%.*]] = icmp ugt ptr [[SCEVGEP]], [[C]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT5:%.*]] = and i1 [[BOUND03]], [[BOUND14]]
 ; CHECK-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT5]]
-; CHECK-NEXT:    [[BOUND06:%.*]] = icmp ugt ptr [[UGLYGEP2]], [[A]]
-; CHECK-NEXT:    [[BOUND17:%.*]] = icmp ugt ptr [[UGLYGEP1]], [[C]]
+; CHECK-NEXT:    [[BOUND06:%.*]] = icmp ugt ptr [[SCEVGEP2]], [[A]]
+; CHECK-NEXT:    [[BOUND17:%.*]] = icmp ugt ptr [[SCEVGEP1]], [[C]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT8:%.*]] = and i1 [[BOUND06]], [[BOUND17]]
 ; CHECK-NEXT:    [[CONFLICT_RDX9:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT8]]
 ; CHECK-NEXT:    br i1 [[CONFLICT_RDX9]], label [[VEC_EPILOG_SCALAR_PH]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-store-accesses-with-gaps.ll b/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-store-accesses-with-gaps.ll
index d28f1f6767..0ccdc22eea 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-store-accesses-with-gaps.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-store-accesses-with-gaps.ll
@@ -27,67 +27,67 @@ define dso_local void @test1(ptr noalias nocapture %points, ptr noalias nocaptur
 ; DISABLED_MASKED_STRIDED-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i64 [[INDEX]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = shl nuw nsw <4 x i64> [[VEC_IND]], <i64 2, i64 2, i64 2, i64 2>
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = extractelement <4 x i64> [[TMP2]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i16, ptr [[POINTS:%.*]], i64 [[TMP3]]
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP2]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP5]]
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP2]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP7]]
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP9:%.*]] = extractelement <4 x i64> [[TMP2]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP9]]
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP11:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP11]], ptr [[TMP4]], align 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP12:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP12]], ptr [[TMP6]], align 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP13:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP13]], ptr [[TMP8]], align 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP14:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP14]], ptr [[TMP10]], align 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i16, ptr [[Y:%.*]], i64 [[INDEX]]
-; DISABLED_MASKED_STRIDED-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i16>, ptr [[TMP15]], align 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP17:%.*]] = or <4 x i64> [[TMP2]], <i64 1, i64 1, i64 1, i64 1>
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP18:%.*]] = extractelement <4 x i64> [[TMP17]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP1:%.*]] = shl nuw nsw <4 x i64> [[VEC_IND]], <i64 2, i64 2, i64 2, i64 2>
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i16, ptr [[POINTS:%.*]], i64 [[TMP2]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP4]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP6]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP8:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP8]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP10:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP10]], ptr [[TMP3]], align 2
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP11:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 1
+; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP11]], ptr [[TMP5]], align 2
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP12:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP12]], ptr [[TMP7]], align 2
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP13:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 3
+; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP13]], ptr [[TMP9]], align 2
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i16, ptr [[Y:%.*]], i64 [[INDEX]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i16>, ptr [[TMP14]], align 2
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP15:%.*]] = or <4 x i64> [[TMP1]], <i64 1, i64 1, i64 1, i64 1>
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP16:%.*]] = extractelement <4 x i64> [[TMP15]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP16]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP18:%.*]] = extractelement <4 x i64> [[TMP15]], i64 1
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP18]]
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP20:%.*]] = extractelement <4 x i64> [[TMP17]], i64 1
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP20:%.*]] = extractelement <4 x i64> [[TMP15]], i64 2
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP20]]
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP22:%.*]] = extractelement <4 x i64> [[TMP17]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP22:%.*]] = extractelement <4 x i64> [[TMP15]], i64 3
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP22]]
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP24:%.*]] = extractelement <4 x i64> [[TMP17]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP24]]
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP26:%.*]] = extractelement <4 x i16> [[WIDE_LOAD1]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP26]], ptr [[TMP19]], align 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP27:%.*]] = extractelement <4 x i16> [[WIDE_LOAD1]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP27]], ptr [[TMP21]], align 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP28:%.*]] = extractelement <4 x i16> [[WIDE_LOAD1]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP28]], ptr [[TMP23]], align 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP29:%.*]] = extractelement <4 x i16> [[WIDE_LOAD1]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP29]], ptr [[TMP25]], align 2
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP24:%.*]] = extractelement <4 x i16> [[WIDE_LOAD1]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP24]], ptr [[TMP17]], align 2
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP25:%.*]] = extractelement <4 x i16> [[WIDE_LOAD1]], i64 1
+; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP25]], ptr [[TMP19]], align 2
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP26:%.*]] = extractelement <4 x i16> [[WIDE_LOAD1]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP26]], ptr [[TMP21]], align 2
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP27:%.*]] = extractelement <4 x i16> [[WIDE_LOAD1]], i64 3
+; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP27]], ptr [[TMP23]], align 2
 ; DISABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; DISABLED_MASKED_STRIDED-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], <i64 4, i64 4, i64 4, i64 4>
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP30]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP28]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; DISABLED_MASKED_STRIDED:       for.end:
 ; DISABLED_MASKED_STRIDED-NEXT:    ret void
 ;
 ; ENABLED_MASKED_STRIDED-LABEL: @test1(
 ; ENABLED_MASKED_STRIDED-NEXT:  entry:
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = getelementptr i16, ptr [[POINTS:%.*]], i64 -1
+; ENABLED_MASKED_STRIDED-NEXT:    [[INVARIANT_GEP:%.*]] = getelementptr i16, ptr [[POINTS:%.*]], i64 -1
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; ENABLED_MASKED_STRIDED:       vector.body:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i64 [[INDEX]]
-; ENABLED_MASKED_STRIDED-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP1]], align 2
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = shl nuw nsw i64 [[INDEX]], 2
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i16, ptr [[Y:%.*]], i64 [[INDEX]]
-; ENABLED_MASKED_STRIDED-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i16>, ptr [[TMP4]], align 2
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = or i64 [[TMP3]], 1
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP7:%.*]] = getelementptr i16, ptr [[TMP0]], i64 [[TMP6]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i64 [[INDEX]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[INDEX]], 2
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[Y:%.*]], i64 [[INDEX]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i16>, ptr [[TMP2]], align 2
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = or i64 [[TMP1]], 1
+; ENABLED_MASKED_STRIDED-NEXT:    [[GEP:%.*]] = getelementptr i16, ptr [[INVARIANT_GEP]], i64 [[TMP3]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i16> [[WIDE_LOAD]], <4 x i16> [[WIDE_LOAD1]], <16 x i32> <i32 0, i32 4, i32 poison, i32 poison, i32 1, i32 5, i32 poison, i32 poison, i32 2, i32 6, i32 poison, i32 poison, i32 3, i32 7, i32 poison, i32 poison>
-; ENABLED_MASKED_STRIDED-NEXT:    call void @llvm.masked.store.v16i16.p0(<16 x i16> [[INTERLEAVED_VEC]], ptr [[TMP7]], i32 2, <16 x i1> <i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false>)
+; ENABLED_MASKED_STRIDED-NEXT:    call void @llvm.masked.store.v16i16.p0(<16 x i16> [[INTERLEAVED_VEC]], ptr [[GEP]], i32 2, <16 x i1> <i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false>)
 ; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP9]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP4]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; ENABLED_MASKED_STRIDED:       for.end:
 ; ENABLED_MASKED_STRIDED-NEXT:    ret void
 ;
@@ -143,86 +143,86 @@ define dso_local void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i64 [[INDEX]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr [[TMP1]], i32 2, <4 x i1> [[TMP0]], <4 x i16> poison)
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = shl nsw <4 x i64> [[VEC_IND]], <i64 2, i64 2, i64 2, i64 2>
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = extractelement <4 x i1> [[TMP0]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP4]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = shl nsw <4 x i64> [[VEC_IND]], <i64 2, i64 2, i64 2, i64 2>
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = extractelement <4 x i1> [[TMP0]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP3]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if:
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i16, ptr [[POINTS:%.*]], i64 [[TMP5]]
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP7:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP7]], ptr [[TMP6]], align 2
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP2]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[POINTS:%.*]], i64 [[TMP4]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP6]], ptr [[TMP5]], align 2
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; DISABLED_MASKED_STRIDED:       pred.store.continue:
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP8:%.*]] = extractelement <4 x i1> [[TMP0]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP8]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2:%.*]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP7:%.*]] = extractelement <4 x i1> [[TMP0]], i64 1
+; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP7]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if1:
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP9:%.*]] = extractelement <4 x i64> [[TMP3]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP9]]
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP11:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP11]], ptr [[TMP10]], align 2
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP8:%.*]] = extractelement <4 x i64> [[TMP2]], i64 1
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP8]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP10:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD]], i64 1
+; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP10]], ptr [[TMP9]], align 2
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE2]]
 ; DISABLED_MASKED_STRIDED:       pred.store.continue2:
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP12:%.*]] = extractelement <4 x i1> [[TMP0]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP12]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4:%.*]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP11:%.*]] = extractelement <4 x i1> [[TMP0]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP11]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if3:
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP13:%.*]] = extractelement <4 x i64> [[TMP3]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP13]]
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP15:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP15]], ptr [[TMP14]], align 2
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP12:%.*]] = extractelement <4 x i64> [[TMP2]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP12]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP14:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP14]], ptr [[TMP13]], align 2
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE4]]
 ; DISABLED_MASKED_STRIDED:       pred.store.continue4:
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP16:%.*]] = extractelement <4 x i1> [[TMP0]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP16]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6:%.*]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP15:%.*]] = extractelement <4 x i1> [[TMP0]], i64 3
+; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP15]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if5:
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP17:%.*]] = extractelement <4 x i64> [[TMP3]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP17]]
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP19:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP19]], ptr [[TMP18]], align 2
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP16:%.*]] = extractelement <4 x i64> [[TMP2]], i64 3
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP16]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP18:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD]], i64 3
+; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP18]], ptr [[TMP17]], align 2
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE6]]
 ; DISABLED_MASKED_STRIDED:       pred.store.continue6:
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i16, ptr [[Y:%.*]], i64 [[INDEX]]
-; DISABLED_MASKED_STRIDED-NEXT:    [[WIDE_MASKED_LOAD7:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr [[TMP20]], i32 2, <4 x i1> [[TMP0]], <4 x i16> poison)
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP22:%.*]] = or <4 x i64> [[TMP3]], <i64 1, i64 1, i64 1, i64 1>
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP23:%.*]] = extractelement <4 x i1> [[TMP0]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP23]], label [[PRED_STORE_IF8:%.*]], label [[PRED_STORE_CONTINUE9:%.*]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i16, ptr [[Y:%.*]], i64 [[INDEX]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[WIDE_MASKED_LOAD7:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr [[TMP19]], i32 2, <4 x i1> [[TMP0]], <4 x i16> poison)
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP20:%.*]] = or <4 x i64> [[TMP2]], <i64 1, i64 1, i64 1, i64 1>
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP21:%.*]] = extractelement <4 x i1> [[TMP0]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP21]], label [[PRED_STORE_IF8:%.*]], label [[PRED_STORE_CONTINUE9:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if8:
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP24:%.*]] = extractelement <4 x i64> [[TMP22]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP24]]
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP26:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD7]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP26]], ptr [[TMP25]], align 2
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP22:%.*]] = extractelement <4 x i64> [[TMP20]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP22]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP24:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD7]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP24]], ptr [[TMP23]], align 2
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE9]]
 ; DISABLED_MASKED_STRIDED:       pred.store.continue9:
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP27:%.*]] = extractelement <4 x i1> [[TMP0]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP27]], label [[PRED_STORE_IF10:%.*]], label [[PRED_STORE_CONTINUE11:%.*]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP25:%.*]] = extractelement <4 x i1> [[TMP0]], i64 1
+; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP25]], label [[PRED_STORE_IF10:%.*]], label [[PRED_STORE_CONTINUE11:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if10:
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP28:%.*]] = extractelement <4 x i64> [[TMP22]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP28]]
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP30:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD7]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP30]], ptr [[TMP29]], align 2
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP26:%.*]] = extractelement <4 x i64> [[TMP20]], i64 1
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP26]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP28:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD7]], i64 1
+; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP28]], ptr [[TMP27]], align 2
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE11]]
 ; DISABLED_MASKED_STRIDED:       pred.store.continue11:
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP31:%.*]] = extractelement <4 x i1> [[TMP0]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP31]], label [[PRED_STORE_IF12:%.*]], label [[PRED_STORE_CONTINUE13:%.*]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP29:%.*]] = extractelement <4 x i1> [[TMP0]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP29]], label [[PRED_STORE_IF12:%.*]], label [[PRED_STORE_CONTINUE13:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if12:
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP32:%.*]] = extractelement <4 x i64> [[TMP22]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP32]]
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP34:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD7]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP34]], ptr [[TMP33]], align 2
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP30:%.*]] = extractelement <4 x i64> [[TMP20]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP30]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP32:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD7]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP32]], ptr [[TMP31]], align 2
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE13]]
 ; DISABLED_MASKED_STRIDED:       pred.store.continue13:
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP35:%.*]] = extractelement <4 x i1> [[TMP0]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP35]], label [[PRED_STORE_IF14:%.*]], label [[PRED_STORE_CONTINUE15]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP33:%.*]] = extractelement <4 x i1> [[TMP0]], i64 3
+; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP33]], label [[PRED_STORE_IF14:%.*]], label [[PRED_STORE_CONTINUE15]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if14:
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP36:%.*]] = extractelement <4 x i64> [[TMP22]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP37:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP36]]
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP38:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD7]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP38]], ptr [[TMP37]], align 2
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP34:%.*]] = extractelement <4 x i64> [[TMP20]], i64 3
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP35:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP34]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP36:%.*]] = extractelement <4 x i16> [[WIDE_MASKED_LOAD7]], i64 3
+; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP36]], ptr [[TMP35]], align 2
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE15]]
 ; DISABLED_MASKED_STRIDED:       pred.store.continue15:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
 ; DISABLED_MASKED_STRIDED-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], <i64 4, i64 4, i64 4, i64 4>
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP39:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP39]], label [[FOR_END_LOOPEXIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP37:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP37]], label [[FOR_END_LOOPEXIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; DISABLED_MASKED_STRIDED:       for.end.loopexit:
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[FOR_END]]
 ; DISABLED_MASKED_STRIDED:       for.end:
@@ -239,28 +239,28 @@ define dso_local void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = add nsw i64 [[WIDE_TRIP_COUNT]], -1
 ; ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
 ; ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = getelementptr i16, ptr [[POINTS:%.*]], i64 -1
+; ENABLED_MASKED_STRIDED-NEXT:    [[INVARIANT_GEP:%.*]] = getelementptr i16, ptr [[POINTS:%.*]], i64 -1
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; ENABLED_MASKED_STRIDED:       vector.body:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
 ; ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
 ; ENABLED_MASKED_STRIDED-NEXT:    [[VEC_IV:%.*]] = or <4 x i64> [[BROADCAST_SPLAT2]], <i64 0, i64 1, i64 2, i64 3>
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i64 [[INDEX]]
-; ENABLED_MASKED_STRIDED-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr [[TMP2]], i32 2, <4 x i1> [[TMP1]], <4 x i16> poison)
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = shl nsw i64 [[INDEX]], 2
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[Y:%.*]], i64 [[INDEX]]
-; ENABLED_MASKED_STRIDED-NEXT:    [[WIDE_MASKED_LOAD3:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr [[TMP5]], i32 2, <4 x i1> [[TMP1]], <4 x i16> poison)
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP7:%.*]] = or i64 [[TMP4]], 1
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP8:%.*]] = getelementptr i16, ptr [[TMP0]], i64 [[TMP7]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i64 [[INDEX]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr [[TMP1]], i32 2, <4 x i1> [[TMP0]], <4 x i16> poison)
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = shl nsw i64 [[INDEX]], 2
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i16, ptr [[Y:%.*]], i64 [[INDEX]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[WIDE_MASKED_LOAD3:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr [[TMP3]], i32 2, <4 x i1> [[TMP0]], <4 x i16> poison)
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = or i64 [[TMP2]], 1
+; ENABLED_MASKED_STRIDED-NEXT:    [[GEP:%.*]] = getelementptr i16, ptr [[INVARIANT_GEP]], i64 [[TMP4]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i16> [[WIDE_MASKED_LOAD]], <4 x i16> [[WIDE_MASKED_LOAD3]], <16 x i32> <i32 0, i32 4, i32 poison, i32 poison, i32 1, i32 5, i32 poison, i32 poison, i32 2, i32 6, i32 poison, i32 poison, i32 3, i32 7, i32 poison, i32 poison>
-; ENABLED_MASKED_STRIDED-NEXT:    [[INTERLEAVED_MASK:%.*]] = shufflevector <4 x i1> [[TMP1]], <4 x i1> poison, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3, i32 3>
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP10:%.*]] = and <16 x i1> [[INTERLEAVED_MASK]], <i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false>
-; ENABLED_MASKED_STRIDED-NEXT:    call void @llvm.masked.store.v16i16.p0(<16 x i16> [[INTERLEAVED_VEC]], ptr [[TMP8]], i32 2, <16 x i1> [[TMP10]])
+; ENABLED_MASKED_STRIDED-NEXT:    [[INTERLEAVED_MASK:%.*]] = shufflevector <4 x i1> [[TMP0]], <4 x i1> poison, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3, i32 3>
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = and <16 x i1> [[INTERLEAVED_MASK]], <i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false>
+; ENABLED_MASKED_STRIDED-NEXT:    call void @llvm.masked.store.v16i16.p0(<16 x i16> [[INTERLEAVED_VEC]], ptr [[GEP]], i32 2, <16 x i1> [[TMP5]])
 ; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP11]], label [[FOR_END_LOOPEXIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP6]], label [[FOR_END_LOOPEXIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; ENABLED_MASKED_STRIDED:       for.end.loopexit:
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[FOR_END]]
 ; ENABLED_MASKED_STRIDED:       for.end:
@@ -317,48 +317,48 @@ define dso_local void @test(ptr noalias nocapture %points, ptr noalias nocapture
 ; DISABLED_MASKED_STRIDED-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE6]] ]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i64 [[INDEX]]
 ; DISABLED_MASKED_STRIDED-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], zeroinitializer
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = mul nuw nsw <4 x i64> [[VEC_IND]], <i64 3, i64 3, i64 3, i64 3>
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP4]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], zeroinitializer
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = mul nuw nsw <4 x i64> [[VEC_IND]], <i64 3, i64 3, i64 3, i64 3>
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP3]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if:
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i16, ptr [[POINTS:%.*]], i64 [[TMP5]]
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP7:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 0
-; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP7]], ptr [[TMP6]], align 2
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP2]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[POINTS:%.*]], i64 [[TMP4]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 0
+; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP6]], ptr [[TMP5]], align 2
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; DISABLED_MASKED_STRIDED:       pred.store.continue:
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP8:%.*]] = extractelement <4 x i1> [[TMP2]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP8]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2:%.*]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP7:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1
+; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP7]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if1:
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP9:%.*]] = extractelement <4 x i64> [[TMP3]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP9]]
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP11:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 1
-; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP11]], ptr [[TMP10]], align 2
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP8:%.*]] = extractelement <4 x i64> [[TMP2]], i64 1
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP8]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP10:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 1
+; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP10]], ptr [[TMP9]], align 2
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE2]]
 ; DISABLED_MASKED_STRIDED:       pred.store.continue2:
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP12:%.*]] = extractelement <4 x i1> [[TMP2]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP12]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4:%.*]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP11:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP11]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4:%.*]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if3:
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP13:%.*]] = extractelement <4 x i64> [[TMP3]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP13]]
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP15:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 2
-; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP15]], ptr [[TMP14]], align 2
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP12:%.*]] = extractelement <4 x i64> [[TMP2]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP12]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP14:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 2
+; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP14]], ptr [[TMP13]], align 2
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE4]]
 ; DISABLED_MASKED_STRIDED:       pred.store.continue4:
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP16:%.*]] = extractelement <4 x i1> [[TMP2]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP16]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP15:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3
+; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP15]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6]]
 ; DISABLED_MASKED_STRIDED:       pred.store.if5:
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP17:%.*]] = extractelement <4 x i64> [[TMP3]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP17]]
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP19:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 3
-; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP19]], ptr [[TMP18]], align 2
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP16:%.*]] = extractelement <4 x i64> [[TMP2]], i64 3
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP16]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP18:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 3
+; DISABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP18]], ptr [[TMP17]], align 2
 ; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE6]]
 ; DISABLED_MASKED_STRIDED:       pred.store.continue6:
 ; DISABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; DISABLED_MASKED_STRIDED-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], <i64 4, i64 4, i64 4, i64 4>
-; DISABLED_MASKED_STRIDED-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP20]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP19]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; DISABLED_MASKED_STRIDED:       for.end:
 ; DISABLED_MASKED_STRIDED-NEXT:    ret void
 ;
@@ -370,48 +370,48 @@ define dso_local void @test(ptr noalias nocapture %points, ptr noalias nocapture
 ; ENABLED_MASKED_STRIDED-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE6]] ]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i64 [[INDEX]]
 ; ENABLED_MASKED_STRIDED-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP0]], align 2
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], zeroinitializer
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = mul nuw nsw <4 x i64> [[VEC_IND]], <i64 3, i64 3, i64 3, i64 3>
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
-; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP4]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], zeroinitializer
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = mul nuw nsw <4 x i64> [[VEC_IND]], <i64 3, i64 3, i64 3, i64 3>
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0
+; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP3]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; ENABLED_MASKED_STRIDED:       pred.store.if:
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i64 0
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i16, ptr [[POINTS:%.*]], i64 [[TMP5]]
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP7:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 0
-; ENABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP7]], ptr [[TMP6]], align 2
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP2]], i64 0
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[POINTS:%.*]], i64 [[TMP4]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 0
+; ENABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP6]], ptr [[TMP5]], align 2
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; ENABLED_MASKED_STRIDED:       pred.store.continue:
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP8:%.*]] = extractelement <4 x i1> [[TMP2]], i64 1
-; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP8]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2:%.*]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP7:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1
+; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP7]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2:%.*]]
 ; ENABLED_MASKED_STRIDED:       pred.store.if1:
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP9:%.*]] = extractelement <4 x i64> [[TMP3]], i64 1
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP9]]
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP11:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 1
-; ENABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP11]], ptr [[TMP10]], align 2
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP8:%.*]] = extractelement <4 x i64> [[TMP2]], i64 1
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP8]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP10:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 1
+; ENABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP10]], ptr [[TMP9]], align 2
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE2]]
 ; ENABLED_MASKED_STRIDED:       pred.store.continue2:
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP12:%.*]] = extractelement <4 x i1> [[TMP2]], i64 2
-; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP12]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4:%.*]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP11:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2
+; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP11]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4:%.*]]
 ; ENABLED_MASKED_STRIDED:       pred.store.if3:
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP13:%.*]] = extractelement <4 x i64> [[TMP3]], i64 2
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP13]]
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP15:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 2
-; ENABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP15]], ptr [[TMP14]], align 2
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP12:%.*]] = extractelement <4 x i64> [[TMP2]], i64 2
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP12]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP14:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 2
+; ENABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP14]], ptr [[TMP13]], align 2
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE4]]
 ; ENABLED_MASKED_STRIDED:       pred.store.continue4:
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP16:%.*]] = extractelement <4 x i1> [[TMP2]], i64 3
-; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP16]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP15:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3
+; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP15]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6]]
 ; ENABLED_MASKED_STRIDED:       pred.store.if5:
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP17:%.*]] = extractelement <4 x i64> [[TMP3]], i64 3
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP17]]
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP19:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 3
-; ENABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP19]], ptr [[TMP18]], align 2
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP16:%.*]] = extractelement <4 x i64> [[TMP2]], i64 3
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i16, ptr [[POINTS]], i64 [[TMP16]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP18:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i64 3
+; ENABLED_MASKED_STRIDED-NEXT:    store i16 [[TMP18]], ptr [[TMP17]], align 2
 ; ENABLED_MASKED_STRIDED-NEXT:    br label [[PRED_STORE_CONTINUE6]]
 ; ENABLED_MASKED_STRIDED:       pred.store.continue6:
 ; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; ENABLED_MASKED_STRIDED-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], <i64 4, i64 4, i64 4, i64 4>
-; ENABLED_MASKED_STRIDED-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP20]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP19]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; ENABLED_MASKED_STRIDED:       for.end:
 ; ENABLED_MASKED_STRIDED-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll
index 7a3c7d6fbf..03903d80cf 100644
--- a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll
@@ -207,10 +207,10 @@ define i16 @reduction_or_trunc(ptr noalias nocapture %ptr) {
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP3]], align 2
 ; CHECK-NEXT:    [[TMP4:%.*]] = zext <4 x i16> [[WIDE_LOAD]] to <4 x i32>
 ; CHECK-NEXT:    [[TMP5:%.*]] = or <4 x i32> [[TMP1]], [[TMP4]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
 ; CHECK-NEXT:    [[TMP7:%.*]] = trunc <4 x i32> [[TMP5]] to <4 x i16>
 ; CHECK-NEXT:    [[TMP8]] = zext <4 x i16> [[TMP7]] to <4 x i32>
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
 ; CHECK-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[TMP9:%.*]] = trunc <4 x i32> [[TMP8]] to <4 x i16>
@@ -234,10 +234,10 @@ define i16 @reduction_or_trunc(ptr noalias nocapture %ptr) {
 ; CHECK-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x i16>, ptr [[TMP16]], align 2
 ; CHECK-NEXT:    [[TMP17:%.*]] = zext <4 x i16> [[WIDE_LOAD4]] to <4 x i32>
 ; CHECK-NEXT:    [[TMP18:%.*]] = or <4 x i32> [[TMP14]], [[TMP17]]
-; CHECK-NEXT:    [[INDEX_NEXT5]] = add nuw i32 [[INDEX2]], 4
-; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i32 [[INDEX_NEXT5]], 256
 ; CHECK-NEXT:    [[TMP20:%.*]] = trunc <4 x i32> [[TMP18]] to <4 x i16>
 ; CHECK-NEXT:    [[TMP21]] = zext <4 x i16> [[TMP20]] to <4 x i32>
+; CHECK-NEXT:    [[INDEX_NEXT5]] = add nuw i32 [[INDEX2]], 4
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i32 [[INDEX_NEXT5]], 256
 ; CHECK-NEXT:    br i1 [[TMP19]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; CHECK:       vec.epilog.middle.block:
 ; CHECK-NEXT:    [[TMP22:%.*]] = trunc <4 x i32> [[TMP21]] to <4 x i16>
diff --git a/llvm/test/Transforms/LoopVectorize/extract-last-veclane.ll b/llvm/test/Transforms/LoopVectorize/extract-last-veclane.ll
index 90ce053183..6de93c381d 100644
--- a/llvm/test/Transforms/LoopVectorize/extract-last-veclane.ll
+++ b/llvm/test/Transforms/LoopVectorize/extract-last-veclane.ll
@@ -13,14 +13,14 @@ define void @inv_store_last_lane(ptr noalias nocapture %a, ptr noalias nocapture
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = shl nsw <4 x i32> [[WIDE_LOAD]], <i32 1, i32 1, i32 1, i32 1>
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nsw <4 x i32> [[WIDE_LOAD]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-NEXT:    store <4 x i32> [[TMP1]], ptr [[TMP2]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP2]], i64 3
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP1]], i64 3
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
@@ -29,15 +29,15 @@ define void @inv_store_last_lane(ptr noalias nocapture %a, ptr noalias nocapture
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV]]
-; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; CHECK-NEXT:    [[MUL:%.*]] = shl nsw i32 [[TMP7]], 1
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = shl nsw i32 [[TMP5]], 1
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    store i32 [[MUL]], ptr [[ARRAYIDX2]], align 4
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       exit:
-; CHECK-NEXT:    [[MUL_LCSSA:%.*]] = phi i32 [ [[MUL]], [[FOR_BODY]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[MUL_LCSSA:%.*]] = phi i32 [ [[MUL]], [[FOR_BODY]] ], [ [[TMP4]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, ptr [[INV:%.*]], i64 42
 ; CHECK-NEXT:    store i32 [[MUL_LCSSA]], ptr [[ARRAYIDX5]], align 4
 ; CHECK-NEXT:    ret void
@@ -74,14 +74,14 @@ define float @ret_last_lane(ptr noalias nocapture %a, ptr noalias nocapture read
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = fmul <4 x float> [[WIDE_LOAD]], <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    store <4 x float> [[TMP2]], ptr [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <4 x float> [[WIDE_LOAD]], <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-NEXT:    store <4 x float> [[TMP1]], ptr [[TMP2]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[TMP2]], i64 3
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i64 3
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
@@ -90,15 +90,15 @@ define float @ret_last_lane(ptr noalias nocapture %a, ptr noalias nocapture read
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDVARS_IV]]
-; CHECK-NEXT:    [[TMP7:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP7]], 2.000000e+00
+; CHECK-NEXT:    [[TMP5:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP5]], 2.000000e+00
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    store float [[MUL]], ptr [[ARRAYIDX2]], align 4
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       exit:
-; CHECK-NEXT:    [[MUL_LCSSA:%.*]] = phi float [ [[MUL]], [[FOR_BODY]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[MUL_LCSSA:%.*]] = phi float [ [[MUL]], [[FOR_BODY]] ], [ [[TMP4]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret float [[MUL_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll b/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
index 926168c622..bb0e2b19cf 100644
--- a/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
@@ -432,17 +432,13 @@ define void @minimal_bit_widths(i1 %c) {
 ; UNROLL-NEXT:    [[TMP5:%.*]] = load i8, ptr [[TMP3]], align 1
 ; UNROLL-NEXT:    br i1 [[C:%.*]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE3]]
 ; UNROLL:       pred.store.if:
-; UNROLL-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP4]] to i32
-; UNROLL-NEXT:    [[TMP7:%.*]] = trunc i32 [[TMP6]] to i8
-; UNROLL-NEXT:    store i8 [[TMP7]], ptr [[TMP2]], align 1
-; UNROLL-NEXT:    [[TMP8:%.*]] = zext i8 [[TMP5]] to i32
-; UNROLL-NEXT:    [[TMP9:%.*]] = trunc i32 [[TMP8]] to i8
-; UNROLL-NEXT:    store i8 [[TMP9]], ptr [[TMP3]], align 1
+; UNROLL-NEXT:    store i8 [[TMP4]], ptr [[TMP2]], align 1
+; UNROLL-NEXT:    store i8 [[TMP5]], ptr [[TMP3]], align 1
 ; UNROLL-NEXT:    br label [[PRED_STORE_CONTINUE3]]
 ; UNROLL:       pred.store.continue3:
 ; UNROLL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; UNROLL-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; UNROLL-NEXT:    br i1 [[TMP10]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; UNROLL-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; UNROLL-NEXT:    br i1 [[TMP6]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; UNROLL:       for.end:
 ; UNROLL-NEXT:    ret void
 ;
@@ -461,21 +457,17 @@ define void @minimal_bit_widths(i1 %c) {
 ; UNROLL-NOSIMPLIFY-NEXT:    [[TMP5:%.*]] = load i8, ptr [[TMP3]], align 1
 ; UNROLL-NOSIMPLIFY-NEXT:    br i1 [[C:%.*]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; UNROLL-NOSIMPLIFY:       pred.store.if:
-; UNROLL-NOSIMPLIFY-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP4]] to i32
-; UNROLL-NOSIMPLIFY-NEXT:    [[TMP7:%.*]] = trunc i32 [[TMP6]] to i8
-; UNROLL-NOSIMPLIFY-NEXT:    store i8 [[TMP7]], ptr [[TMP2]], align 1
+; UNROLL-NOSIMPLIFY-NEXT:    store i8 [[TMP4]], ptr [[TMP2]], align 1
 ; UNROLL-NOSIMPLIFY-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; UNROLL-NOSIMPLIFY:       pred.store.continue:
 ; UNROLL-NOSIMPLIFY-NEXT:    br i1 [[C]], label [[PRED_STORE_IF2:%.*]], label [[PRED_STORE_CONTINUE3]]
 ; UNROLL-NOSIMPLIFY:       pred.store.if2:
-; UNROLL-NOSIMPLIFY-NEXT:    [[TMP8:%.*]] = zext i8 [[TMP5]] to i32
-; UNROLL-NOSIMPLIFY-NEXT:    [[TMP9:%.*]] = trunc i32 [[TMP8]] to i8
-; UNROLL-NOSIMPLIFY-NEXT:    store i8 [[TMP9]], ptr [[TMP3]], align 1
+; UNROLL-NOSIMPLIFY-NEXT:    store i8 [[TMP5]], ptr [[TMP3]], align 1
 ; UNROLL-NOSIMPLIFY-NEXT:    br label [[PRED_STORE_CONTINUE3]]
 ; UNROLL-NOSIMPLIFY:       pred.store.continue3:
 ; UNROLL-NOSIMPLIFY-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; UNROLL-NOSIMPLIFY-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; UNROLL-NOSIMPLIFY-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; UNROLL-NOSIMPLIFY-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; UNROLL-NOSIMPLIFY-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; UNROLL-NOSIMPLIFY:       middle.block:
 ; UNROLL-NOSIMPLIFY-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; UNROLL-NOSIMPLIFY:       scalar.ph:
@@ -515,27 +507,23 @@ define void @minimal_bit_widths(i1 %c) {
 ; VEC-NEXT:    [[TMP3:%.*]] = extractelement <2 x i1> [[BROADCAST_SPLAT]], i32 0
 ; VEC-NEXT:    br i1 [[TMP3]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; VEC:       pred.store.if:
-; VEC-NEXT:    [[TMP4:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i32 0
-; VEC-NEXT:    [[TMP5:%.*]] = zext i8 [[TMP4]] to i32
-; VEC-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr undef, i64 [[TMP0]]
-; VEC-NEXT:    [[TMP7:%.*]] = trunc i32 [[TMP5]] to i8
-; VEC-NEXT:    store i8 [[TMP7]], ptr [[TMP6]], align 1
+; VEC-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr undef, i64 [[TMP0]]
+; VEC-NEXT:    [[TMP5:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i32 0
+; VEC-NEXT:    store i8 [[TMP5]], ptr [[TMP4]], align 1
 ; VEC-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; VEC:       pred.store.continue:
-; VEC-NEXT:    [[TMP8:%.*]] = extractelement <2 x i1> [[BROADCAST_SPLAT]], i32 1
-; VEC-NEXT:    br i1 [[TMP8]], label [[PRED_STORE_IF2:%.*]], label [[PRED_STORE_CONTINUE3]]
+; VEC-NEXT:    [[TMP6:%.*]] = extractelement <2 x i1> [[BROADCAST_SPLAT]], i32 1
+; VEC-NEXT:    br i1 [[TMP6]], label [[PRED_STORE_IF2:%.*]], label [[PRED_STORE_CONTINUE3]]
 ; VEC:       pred.store.if2:
-; VEC-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 1
-; VEC-NEXT:    [[TMP10:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i32 1
-; VEC-NEXT:    [[TMP11:%.*]] = zext i8 [[TMP10]] to i32
-; VEC-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr undef, i64 [[TMP9]]
-; VEC-NEXT:    [[TMP13:%.*]] = trunc i32 [[TMP11]] to i8
-; VEC-NEXT:    store i8 [[TMP13]], ptr [[TMP12]], align 1
+; VEC-NEXT:    [[TMP7:%.*]] = add i64 [[INDEX]], 1
+; VEC-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr undef, i64 [[TMP7]]
+; VEC-NEXT:    [[TMP9:%.*]] = extractelement <2 x i8> [[WIDE_LOAD]], i32 1
+; VEC-NEXT:    store i8 [[TMP9]], ptr [[TMP8]], align 1
 ; VEC-NEXT:    br label [[PRED_STORE_CONTINUE3]]
 ; VEC:       pred.store.continue3:
 ; VEC-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; VEC-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; VEC-NEXT:    br i1 [[TMP14]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; VEC-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; VEC-NEXT:    br i1 [[TMP10]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; VEC:       for.end:
 ; VEC-NEXT:    ret void
 ;
@@ -606,21 +594,17 @@ define void @minimal_bit_widths_with_aliasing_store(i1 %c, ptr %ptr) {
 ; UNROLL-NOSIMPLIFY-NEXT:    store i8 0, ptr [[TMP3]], align 1
 ; UNROLL-NOSIMPLIFY-NEXT:    br i1 [[C:%.*]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; UNROLL-NOSIMPLIFY:       pred.store.if:
-; UNROLL-NOSIMPLIFY-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP4]] to i32
-; UNROLL-NOSIMPLIFY-NEXT:    [[TMP7:%.*]] = trunc i32 [[TMP6]] to i8
-; UNROLL-NOSIMPLIFY-NEXT:    store i8 [[TMP7]], ptr [[TMP2]], align 1
+; UNROLL-NOSIMPLIFY-NEXT:    store i8 [[TMP4]], ptr [[TMP2]], align 1
 ; UNROLL-NOSIMPLIFY-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; UNROLL-NOSIMPLIFY:       pred.store.continue:
 ; UNROLL-NOSIMPLIFY-NEXT:    br i1 [[C]], label [[PRED_STORE_IF2:%.*]], label [[PRED_STORE_CONTINUE3]]
 ; UNROLL-NOSIMPLIFY:       pred.store.if2:
-; UNROLL-NOSIMPLIFY-NEXT:    [[TMP8:%.*]] = zext i8 [[TMP5]] to i32
-; UNROLL-NOSIMPLIFY-NEXT:    [[TMP9:%.*]] = trunc i32 [[TMP8]] to i8
-; UNROLL-NOSIMPLIFY-NEXT:    store i8 [[TMP9]], ptr [[TMP3]], align 1
+; UNROLL-NOSIMPLIFY-NEXT:    store i8 [[TMP5]], ptr [[TMP3]], align 1
 ; UNROLL-NOSIMPLIFY-NEXT:    br label [[PRED_STORE_CONTINUE3]]
 ; UNROLL-NOSIMPLIFY:       pred.store.continue3:
 ; UNROLL-NOSIMPLIFY-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; UNROLL-NOSIMPLIFY-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 0
-; UNROLL-NOSIMPLIFY-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; UNROLL-NOSIMPLIFY-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 0
+; UNROLL-NOSIMPLIFY-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; UNROLL-NOSIMPLIFY:       middle.block:
 ; UNROLL-NOSIMPLIFY-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; UNROLL-NOSIMPLIFY:       scalar.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll
index b74829c698..541dc9a70c 100644
--- a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll
@@ -232,3 +232,38 @@ loop:
 exit:
   ret void
 }
+
+define i16 @reduction_with_casts() {
+; CHECK-LABEL: define i16 @reduction_with_casts() {
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH:%.+]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY:%.+]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP2:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = and i32 [[VEC_PHI]], 65535
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[VEC_PHI1]], 65535
+; CHECK-NEXT:    [[TMP2]] = add i32 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP3]] = add i32 [[TMP1]], 1
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 9998
+; CHECK-NEXT:    br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add i32 [[TMP3]], [[TMP2]]
+; CHECK-NEXT:    br i1 false, label [[EXIT:%.*]], label %scalar.ph
+;
+entry:
+  br label %loop
+
+loop:
+  %count.0.in1 = phi i32 [ 0, %entry ], [ %add, %loop ]
+  %iv = phi i16 [ 1, %entry ], [ %iv.next, %loop ]
+  %conv1 = and i32 %count.0.in1, 65535
+  %add = add nuw nsw i32 %conv1, 1
+  %iv.next = add i16 %iv, 1
+  %cmp = icmp eq i16 %iv.next, 10000
+  br i1 %cmp, label %exit, label %loop
+
+exit:
+  %add.lcssa = phi i32 [ %add, %loop ]
+  %count.0 = trunc i32 %add.lcssa to i16
+  ret i16 %count.0
+}
diff --git a/llvm/test/Transforms/LoopVectorize/outer_loop_scalable.ll b/llvm/test/Transforms/LoopVectorize/outer_loop_scalable.ll
new file mode 100644
index 0000000000..c7d7ac3cb9
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/outer_loop_scalable.ll
@@ -0,0 +1,150 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt -S -pass-remarks=loop-vectorize -passes=loop-vectorize -enable-vplan-native-path -force-target-supports-scalable-vectors < %s 2>&1 | FileCheck %s
+; RUN: opt -S -pass-remarks=loop-vectorize -passes=loop-vectorize -enable-vplan-native-path < %s 2>&1 | FileCheck %s -check-prefix=NO_SCALABLE_VECS
+
+; Test if the vplan-native-path successfully vectorizes a loop using scalable
+; vectors if the target supports scalable vectors and rejects vectorization
+; if a scalable VF is requested but not supported by the target.
+
+; CHECK: remark: <unknown>:0:0: vectorized outer loop (vectorization width: vscale x 4, interleaved count: 1)
+; NO_SCALABLE_VECS: remark: <unknown>:0:0: loop not vectorized: the scalable user-specified vectorization width for outer-loop vectorization cannot be used because the target does not support scalable vectors.
+
+@A = external local_unnamed_addr global [1024 x float], align 4
+@B = external local_unnamed_addr global [512 x float], align 4
+
+define void @foo() {
+; CHECK-LABEL: define void @foo() {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
+; CHECK-NEXT:    [[TMP5:%.*]] = add <vscale x 4 x i64> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = mul <vscale x 4 x i64> [[TMP5]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP6]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 1, [[TMP8]]
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP9]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[OUTER_LOOP_LATCH4:%.*]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[OUTER_LOOP_LATCH4]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [1024 x float], ptr @A, i64 0, <vscale x 4 x i64> [[VEC_IND]]
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0(<vscale x 4 x ptr> [[TMP10]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), <vscale x 4 x float> poison)
+; CHECK-NEXT:    br label [[INNER_LOOP1:%.*]]
+; CHECK:       inner_loop1:
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i64> [ zeroinitializer, [[VECTOR_BODY]] ], [ [[TMP13:%.*]], [[INNER_LOOP1]] ]
+; CHECK-NEXT:    [[VEC_PHI2:%.*]] = phi <vscale x 4 x float> [ [[WIDE_MASKED_GATHER]], [[VECTOR_BODY]] ], [ [[TMP12:%.*]], [[INNER_LOOP1]] ]
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [512 x float], ptr @B, i64 0, <vscale x 4 x i64> [[VEC_PHI]]
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0(<vscale x 4 x ptr> [[TMP11]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer), <vscale x 4 x float> poison)
+; CHECK-NEXT:    [[TMP12]] = fmul <vscale x 4 x float> [[VEC_PHI2]], [[WIDE_MASKED_GATHER3]]
+; CHECK-NEXT:    [[TMP13]] = add nuw nsw <vscale x 4 x i64> [[VEC_PHI]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq <vscale x 4 x i64> [[TMP13]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 512, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <vscale x 4 x i1> [[TMP14]], i32 0
+; CHECK-NEXT:    br i1 [[TMP15]], label [[OUTER_LOOP_LATCH4]], label [[INNER_LOOP1]]
+; CHECK:       outer_loop_latch4:
+; CHECK-NEXT:    [[VEC_PHI5:%.*]] = phi <vscale x 4 x float> [ [[TMP12]], [[INNER_LOOP1]] ]
+; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4f32.nxv4p0(<vscale x 4 x float> [[VEC_PHI5]], <vscale x 4 x ptr> [[TMP10]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT:    [[TMP16:%.*]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq <vscale x 4 x i64> [[TMP16]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1024, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP19:%.*]] = mul i64 [[TMP18]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]]
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
+; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[OUTER_LOOP:%.*]]
+; CHECK:       outer_loop:
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[OUTER_LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds [1024 x float], ptr @A, i64 0, i64 [[I]]
+; CHECK-NEXT:    [[X_START:%.*]] = load float, ptr [[ARRAYIDX1]], align 4
+; CHECK-NEXT:    br label [[INNER_LOOP:%.*]]
+; CHECK:       inner_loop:
+; CHECK-NEXT:    [[J:%.*]] = phi i64 [ 0, [[OUTER_LOOP]] ], [ [[J_NEXT:%.*]], [[INNER_LOOP]] ]
+; CHECK-NEXT:    [[X:%.*]] = phi float [ [[X_START]], [[OUTER_LOOP]] ], [ [[X_NEXT:%.*]], [[INNER_LOOP]] ]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds [512 x float], ptr @B, i64 0, i64 [[J]]
+; CHECK-NEXT:    [[B:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[X_NEXT]] = fmul float [[X]], [[B]]
+; CHECK-NEXT:    [[J_NEXT]] = add nuw nsw i64 [[J]], 1
+; CHECK-NEXT:    [[INNER_EXITCOND:%.*]] = icmp eq i64 [[J_NEXT]], 512
+; CHECK-NEXT:    br i1 [[INNER_EXITCOND]], label [[OUTER_LOOP_LATCH]], label [[INNER_LOOP]]
+; CHECK:       outer_loop_latch:
+; CHECK-NEXT:    [[X_NEXT_LCSSA:%.*]] = phi float [ [[X_NEXT]], [[INNER_LOOP]] ]
+; CHECK-NEXT:    store float [[X_NEXT_LCSSA]], ptr [[ARRAYIDX1]], align 4
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
+; CHECK-NEXT:    [[OUTER_EXITCOND:%.*]] = icmp eq i64 [[I_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[OUTER_EXITCOND]], label [[EXIT]], label [[OUTER_LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+; NO_SCALABLE_VECS-LABEL: define void @foo() {
+; NO_SCALABLE_VECS-NEXT:  entry:
+; NO_SCALABLE_VECS-NEXT:    br label [[OUTER_LOOP:%.*]]
+; NO_SCALABLE_VECS:       outer_loop:
+; NO_SCALABLE_VECS-NEXT:    [[I:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[I_NEXT:%.*]], [[OUTER_LOOP_LATCH:%.*]] ]
+; NO_SCALABLE_VECS-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds [1024 x float], ptr @A, i64 0, i64 [[I]]
+; NO_SCALABLE_VECS-NEXT:    [[X_START:%.*]] = load float, ptr [[ARRAYIDX1]], align 4
+; NO_SCALABLE_VECS-NEXT:    br label [[INNER_LOOP:%.*]]
+; NO_SCALABLE_VECS:       inner_loop:
+; NO_SCALABLE_VECS-NEXT:    [[J:%.*]] = phi i64 [ 0, [[OUTER_LOOP]] ], [ [[J_NEXT:%.*]], [[INNER_LOOP]] ]
+; NO_SCALABLE_VECS-NEXT:    [[X:%.*]] = phi float [ [[X_START]], [[OUTER_LOOP]] ], [ [[X_NEXT:%.*]], [[INNER_LOOP]] ]
+; NO_SCALABLE_VECS-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds [512 x float], ptr @B, i64 0, i64 [[J]]
+; NO_SCALABLE_VECS-NEXT:    [[B:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; NO_SCALABLE_VECS-NEXT:    [[X_NEXT]] = fmul float [[X]], [[B]]
+; NO_SCALABLE_VECS-NEXT:    [[J_NEXT]] = add nuw nsw i64 [[J]], 1
+; NO_SCALABLE_VECS-NEXT:    [[INNER_EXITCOND:%.*]] = icmp eq i64 [[J_NEXT]], 512
+; NO_SCALABLE_VECS-NEXT:    br i1 [[INNER_EXITCOND]], label [[OUTER_LOOP_LATCH]], label [[INNER_LOOP]]
+; NO_SCALABLE_VECS:       outer_loop_latch:
+; NO_SCALABLE_VECS-NEXT:    [[X_NEXT_LCSSA:%.*]] = phi float [ [[X_NEXT]], [[INNER_LOOP]] ]
+; NO_SCALABLE_VECS-NEXT:    store float [[X_NEXT_LCSSA]], ptr [[ARRAYIDX1]], align 4
+; NO_SCALABLE_VECS-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
+; NO_SCALABLE_VECS-NEXT:    [[OUTER_EXITCOND:%.*]] = icmp eq i64 [[I_NEXT]], 1024
+; NO_SCALABLE_VECS-NEXT:    br i1 [[OUTER_EXITCOND]], label [[EXIT:%.*]], label [[OUTER_LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
+; NO_SCALABLE_VECS:       exit:
+; NO_SCALABLE_VECS-NEXT:    ret void
+entry:
+  br label %outer_loop
+
+outer_loop:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %outer_loop_latch ]
+  %arrayidx1 = getelementptr inbounds [1024 x float], ptr @A, i64 0, i64 %i
+  %x.start = load float, ptr %arrayidx1, align 4
+  br label %inner_loop
+
+inner_loop:
+  %j = phi i64 [ 0, %outer_loop ], [ %j.next, %inner_loop ]
+  %x = phi float [ %x.start, %outer_loop ], [ %x.next, %inner_loop ]
+  %arrayidx2 = getelementptr inbounds [512 x float], ptr @B, i64 0, i64 %j
+  %b = load float, ptr %arrayidx2, align 4
+  %x.next = fmul float %x, %b
+  %j.next = add nuw nsw i64 %j, 1
+  %inner_exitcond = icmp eq i64 %j.next, 512
+  br i1 %inner_exitcond, label %outer_loop_latch, label %inner_loop
+
+outer_loop_latch:
+  store float %x.next, ptr %arrayidx1, align 4
+  %i.next = add nuw nsw i64 %i, 1
+  %outer_exitcond = icmp eq i64 %i.next, 1024
+  br i1 %outer_exitcond, label %exit, label %outer_loop, !llvm.loop !1
+
+exit:
+  ret void
+}
+
+!1 = distinct !{!1, !2, !3, !4}
+!2 = !{!"llvm.loop.vectorize.enable", i1 true}
+!3 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}
+!4 = !{!"llvm.loop.vectorize.width", i32 4}
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-inloop-uf4.ll b/llvm/test/Transforms/LoopVectorize/reduction-inloop-uf4.ll
index 2cfe93fb6c..44e609439c 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction-inloop-uf4.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction-inloop-uf4.ll
@@ -11,38 +11,38 @@ define i32 @reduction_sum_single(ptr noalias nocapture %A) {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI2:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI3:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI2:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI3:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 4
-; CHECK-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 8
-; CHECK-NEXT:    [[WIDE_LOAD5:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 12
-; CHECK-NEXT:    [[WIDE_LOAD6:%.*]] = load <4 x i32>, ptr [[TMP6]], align 4
-; CHECK-NEXT:    [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD]])
-; CHECK-NEXT:    [[TMP9]] = add i32 [[TMP8]], [[VEC_PHI]]
-; CHECK-NEXT:    [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD4]])
-; CHECK-NEXT:    [[TMP11]] = add i32 [[TMP10]], [[VEC_PHI1]]
-; CHECK-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD5]])
-; CHECK-NEXT:    [[TMP13]] = add i32 [[TMP12]], [[VEC_PHI2]]
-; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD6]])
-; CHECK-NEXT:    [[TMP15]] = add i32 [[TMP14]], [[VEC_PHI3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 4
+; CHECK-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 8
+; CHECK-NEXT:    [[WIDE_LOAD5:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 12
+; CHECK-NEXT:    [[WIDE_LOAD6:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[TMP5]] = add i32 [[TMP4]], [[VEC_PHI]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD4]])
+; CHECK-NEXT:    [[TMP7]] = add i32 [[TMP6]], [[VEC_PHI1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD5]])
+; CHECK-NEXT:    [[TMP9]] = add i32 [[TMP8]], [[VEC_PHI2]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD6]])
+; CHECK-NEXT:    [[TMP11]] = add i32 [[TMP10]], [[VEC_PHI3]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
-; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
+; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[BIN_RDX:%.*]] = add i32 [[TMP11]], [[TMP9]]
-; CHECK-NEXT:    [[BIN_RDX7:%.*]] = add i32 [[TMP13]], [[BIN_RDX]]
-; CHECK-NEXT:    [[BIN_RDX8:%.*]] = add i32 [[TMP15]], [[BIN_RDX7]]
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add i32 [[TMP7]], [[TMP5]]
+; CHECK-NEXT:    [[BIN_RDX7:%.*]] = add i32 [[TMP9]], [[BIN_RDX]]
+; CHECK-NEXT:    [[BIN_RDX8:%.*]] = add i32 [[TMP11]], [[BIN_RDX7]]
 ; CHECK-NEXT:    br i1 true, label [[DOT_CRIT_EDGE:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
 ; CHECK-NEXT:    br label [[DOTLR_PH:%.*]]
 ; CHECK:       .lr.ph:
-; CHECK-NEXT:    br i1 poison, label [[DOT_CRIT_EDGE]], label [[DOTLR_PH]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK-NEXT:    br i1 poison, label [[DOT_CRIT_EDGE]], label [[DOTLR_PH]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       ._crit_edge:
 ; CHECK-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ poison, [[DOTLR_PH]] ], [ [[BIN_RDX8]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[SUM_0_LCSSA]]
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll b/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll
index 837d663f4a..a4a075463b 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll
@@ -22,10 +22,10 @@ define i8 @PR34687(i1 %c, i32 %x, i32 %n) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i32> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
 ; CHECK-NEXT:    [[TMP1:%.*]] = and <4 x i32> [[VEC_PHI]], <i32 255, i32 255, i32 255, i32 255>
 ; CHECK-NEXT:    [[TMP2:%.*]] = add <4 x i32> [[TMP1]], [[BROADCAST_SPLAT2]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = trunc <4 x i32> [[TMP2]] to <4 x i8>
 ; CHECK-NEXT:    [[TMP5]] = zext <4 x i8> [[TMP4]] to <4 x i32>
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[TMP6:%.*]] = trunc <4 x i32> [[TMP5]] to <4 x i8>
@@ -99,10 +99,10 @@ define i32 @PR35734(i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ [[TMP2]], [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP3:%.*]] = and <4 x i32> [[VEC_PHI]], <i32 1, i32 1, i32 1, i32 1>
 ; CHECK-NEXT:    [[TMP4:%.*]] = add <4 x i32> [[TMP3]], <i32 -1, i32 -1, i32 -1, i32 -1>
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = trunc <4 x i32> [[TMP4]] to <4 x i1>
 ; CHECK-NEXT:    [[TMP7]] = sext <4 x i1> [[TMP6]] to <4 x i32>
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[TMP8:%.*]] = trunc <4 x i32> [[TMP7]] to <4 x i1>
diff --git a/llvm/test/Transforms/LoopVectorize/reduction.ll b/llvm/test/Transforms/LoopVectorize/reduction.ll
index 278917b006..aba2532706 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction.ll
@@ -1,14 +1,63 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
 ; RUN: opt < %s -passes=loop-vectorize,dce,instcombine -force-vector-interleave=1 -force-vector-width=4 -S | FileCheck %s
 
-target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
-
-;CHECK-LABEL: @reduction_sum(
-;CHECK: phi <4 x i32>
-;CHECK: load <4 x i32>
-;CHECK: add <4 x i32>
-;CHECK: call i32 @llvm.vector.reduce.add.v4i32(<4 x i32>
-;CHECK: ret i32
-define i32 @reduction_sum(i32 %n, ptr noalias nocapture %A, ptr noalias nocapture %B) nounwind uwtable readonly noinline ssp {
+define i32 @reduction_sum(i32 %n, ptr %A, ptr %B) {
+; CHECK-LABEL: define i32 @reduction_sum(
+; CHECK-SAME: i32 [[N:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[TMP1]], label [[DOTLR_PH_PREHEADER:%.*]], label [[DOT_CRIT_EDGE:%.*]]
+; CHECK:       .lr.ph.preheader:
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[N]] to i64
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[TMP2]], 4294967292
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i32> [[VEC_PHI]], [[VEC_IND]]
+; CHECK-NEXT:    [[TMP6:%.*]] = add <4 x i32> [[TMP5]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP7]] = add <4 x i32> [[TMP6]], [[WIDE_LOAD1]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP7]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP2]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[DOT_CRIT_EDGE_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[DOTLR_PH_PREHEADER]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], [[MIDDLE_BLOCK]] ], [ 0, [[DOTLR_PH_PREHEADER]] ]
+; CHECK-NEXT:    br label [[DOTLR_PH:%.*]]
+; CHECK:       .lr.ph:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[DOTLR_PH]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[SUM_02:%.*]] = phi i32 [ [[TMP17:%.*]], [[DOTLR_PH]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4
+; CHECK-NEXT:    [[TMP14:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[TMP15:%.*]] = add i32 [[SUM_02]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = add i32 [[TMP15]], [[TMP11]]
+; CHECK-NEXT:    [[TMP17]] = add i32 [[TMP16]], [[TMP13]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[DOT_CRIT_EDGE_LOOPEXIT]], label [[DOTLR_PH]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       ._crit_edge.loopexit:
+; CHECK-NEXT:    [[DOTLCSSA:%.*]] = phi i32 [ [[TMP17]], [[DOTLR_PH]] ], [ [[TMP9]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    br label [[DOT_CRIT_EDGE]]
+; CHECK:       ._crit_edge:
+; CHECK-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[DOTLCSSA]], [[DOT_CRIT_EDGE_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[SUM_0_LCSSA]]
+;
   %1 = icmp sgt i32 %n, 0
   br i1 %1, label %.lr.ph, label %._crit_edge
 
@@ -33,13 +82,63 @@ define i32 @reduction_sum(i32 %n, ptr noalias nocapture %A, ptr noalias nocaptur
   ret i32 %sum.0.lcssa
 }
 
-;CHECK-LABEL: @reduction_prod(
-;CHECK: phi <4 x i32>
-;CHECK: load <4 x i32>
-;CHECK: mul <4 x i32>
-;CHECK: call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32>
-;CHECK: ret i32
-define i32 @reduction_prod(i32 %n, ptr noalias nocapture %A, ptr noalias nocapture %B) nounwind uwtable readonly noinline ssp {
+define i32 @reduction_prod(i32 %n, ptr %A, ptr %B) {
+; CHECK-LABEL: define i32 @reduction_prod(
+; CHECK-SAME: i32 [[N:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[TMP1]], label [[DOTLR_PH_PREHEADER:%.*]], label [[DOT_CRIT_EDGE:%.*]]
+; CHECK:       .lr.ph.preheader:
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[N]] to i64
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[TMP2]], 4294967292
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ <i32 1, i32 1, i32 1, i32 1>, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = mul <4 x i32> [[VEC_PHI]], [[VEC_IND]]
+; CHECK-NEXT:    [[TMP6:%.*]] = mul <4 x i32> [[TMP5]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP7]] = mul <4 x i32> [[TMP6]], [[WIDE_LOAD1]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[TMP7]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP2]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[DOT_CRIT_EDGE_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[DOTLR_PH_PREHEADER]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], [[MIDDLE_BLOCK]] ], [ 1, [[DOTLR_PH_PREHEADER]] ]
+; CHECK-NEXT:    br label [[DOTLR_PH:%.*]]
+; CHECK:       .lr.ph:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[DOTLR_PH]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[PROD_02:%.*]] = phi i32 [ [[TMP17:%.*]], [[DOTLR_PH]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4
+; CHECK-NEXT:    [[TMP14:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[TMP15:%.*]] = mul i32 [[PROD_02]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = mul i32 [[TMP15]], [[TMP11]]
+; CHECK-NEXT:    [[TMP17]] = mul i32 [[TMP16]], [[TMP13]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[DOT_CRIT_EDGE_LOOPEXIT]], label [[DOTLR_PH]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       ._crit_edge.loopexit:
+; CHECK-NEXT:    [[DOTLCSSA:%.*]] = phi i32 [ [[TMP17]], [[DOTLR_PH]] ], [ [[TMP9]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    br label [[DOT_CRIT_EDGE]]
+; CHECK:       ._crit_edge:
+; CHECK-NEXT:    [[PROD_0_LCSSA:%.*]] = phi i32 [ 1, [[TMP0:%.*]] ], [ [[DOTLCSSA]], [[DOT_CRIT_EDGE_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[PROD_0_LCSSA]]
+;
   %1 = icmp sgt i32 %n, 0
   br i1 %1, label %.lr.ph, label %._crit_edge
 
@@ -64,13 +163,63 @@ define i32 @reduction_prod(i32 %n, ptr noalias nocapture %A, ptr noalias nocaptu
   ret i32 %prod.0.lcssa
 }
 
-;CHECK-LABEL: @reduction_mix(
-;CHECK: phi <4 x i32>
-;CHECK: load <4 x i32>
-;CHECK: mul nsw <4 x i32>
-;CHECK: call i32 @llvm.vector.reduce.add.v4i32(<4 x i32>
-;CHECK: ret i32
-define i32 @reduction_mix(i32 %n, ptr noalias nocapture %A, ptr noalias nocapture %B) nounwind uwtable readonly noinline ssp {
+define i32 @reduction_mix(i32 %n, ptr %A, ptr %B) {
+; CHECK-LABEL: define i32 @reduction_mix(
+; CHECK-SAME: i32 [[N:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[TMP1]], label [[DOTLR_PH_PREHEADER:%.*]], label [[DOT_CRIT_EDGE:%.*]]
+; CHECK:       .lr.ph.preheader:
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[N]] to i64
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[TMP2]], 4294967292
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nsw <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP6:%.*]] = add <4 x i32> [[VEC_PHI]], [[VEC_IND]]
+; CHECK-NEXT:    [[TMP7]] = add <4 x i32> [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP7]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP2]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[DOT_CRIT_EDGE_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[DOTLR_PH_PREHEADER]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], [[MIDDLE_BLOCK]] ], [ 0, [[DOTLR_PH_PREHEADER]] ]
+; CHECK-NEXT:    br label [[DOTLR_PH:%.*]]
+; CHECK:       .lr.ph:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[DOTLR_PH]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[SUM_02:%.*]] = phi i32 [ [[TMP17:%.*]], [[DOTLR_PH]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4
+; CHECK-NEXT:    [[TMP14:%.*]] = mul nsw i32 [[TMP13]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[TMP16:%.*]] = add i32 [[SUM_02]], [[TMP15]]
+; CHECK-NEXT:    [[TMP17]] = add i32 [[TMP16]], [[TMP14]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[DOT_CRIT_EDGE_LOOPEXIT]], label [[DOTLR_PH]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK:       ._crit_edge.loopexit:
+; CHECK-NEXT:    [[DOTLCSSA:%.*]] = phi i32 [ [[TMP17]], [[DOTLR_PH]] ], [ [[TMP9]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    br label [[DOT_CRIT_EDGE]]
+; CHECK:       ._crit_edge:
+; CHECK-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[DOTLCSSA]], [[DOT_CRIT_EDGE_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[SUM_0_LCSSA]]
+;
   %1 = icmp sgt i32 %n, 0
   br i1 %1, label %.lr.ph, label %._crit_edge
 
@@ -95,11 +244,63 @@ define i32 @reduction_mix(i32 %n, ptr noalias nocapture %A, ptr noalias nocaptur
   ret i32 %sum.0.lcssa
 }
 
-;CHECK-LABEL: @reduction_mul(
-;CHECK: mul <4 x i32>
-;CHECK: call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32>
-;CHECK: ret i32
-define i32 @reduction_mul(i32 %n, ptr noalias nocapture %A, ptr noalias nocapture %B) nounwind uwtable readonly noinline ssp {
+define i32 @reduction_mul(i32 %n, ptr %A, ptr %B) {
+; CHECK-LABEL: define i32 @reduction_mul(
+; CHECK-SAME: i32 [[N:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[TMP1]], label [[DOTLR_PH_PREHEADER:%.*]], label [[DOT_CRIT_EDGE:%.*]]
+; CHECK:       .lr.ph.preheader:
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[N]] to i64
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[TMP2]], 4294967292
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ <i32 19, i32 1, i32 1, i32 1>, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[VEC_IND]]
+; CHECK-NEXT:    [[TMP6:%.*]] = add <4 x i32> [[TMP5]], [[WIDE_LOAD1]]
+; CHECK-NEXT:    [[TMP7]] = mul <4 x i32> [[TMP6]], [[VEC_PHI]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[TMP7]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP2]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[DOT_CRIT_EDGE_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[DOTLR_PH_PREHEADER]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], [[MIDDLE_BLOCK]] ], [ 19, [[DOTLR_PH_PREHEADER]] ]
+; CHECK-NEXT:    br label [[DOTLR_PH:%.*]]
+; CHECK:       .lr.ph:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[DOTLR_PH]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[SUM_02:%.*]] = phi i32 [ [[TMP17:%.*]], [[DOTLR_PH]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4
+; CHECK-NEXT:    [[TMP14:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[TMP15:%.*]] = add i32 [[TMP11]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = add i32 [[TMP15]], [[TMP13]]
+; CHECK-NEXT:    [[TMP17]] = mul i32 [[TMP16]], [[SUM_02]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[DOT_CRIT_EDGE_LOOPEXIT]], label [[DOTLR_PH]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK:       ._crit_edge.loopexit:
+; CHECK-NEXT:    [[DOTLCSSA:%.*]] = phi i32 [ [[TMP17]], [[DOTLR_PH]] ], [ [[TMP9]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    br label [[DOT_CRIT_EDGE]]
+; CHECK:       ._crit_edge:
+; CHECK-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[DOTLCSSA]], [[DOT_CRIT_EDGE_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[SUM_0_LCSSA]]
+;
   %1 = icmp sgt i32 %n, 0
   br i1 %1, label %.lr.ph, label %._crit_edge
 
@@ -124,12 +325,59 @@ define i32 @reduction_mul(i32 %n, ptr noalias nocapture %A, ptr noalias nocaptur
   ret i32 %sum.0.lcssa
 }
 
-;CHECK-LABEL: @start_at_non_zero(
-;CHECK: phi <4 x i32>
-;CHECK: <i32 120, i32 0, i32 0, i32 0>
-;CHECK: call i32 @llvm.vector.reduce.add.v4i32(<4 x i32>
-;CHECK: ret i32
-define i32 @start_at_non_zero(ptr nocapture %in, ptr nocapture %coeff, ptr nocapture %out, i32 %n) nounwind uwtable readonly ssp {
+define i32 @start_at_non_zero(ptr %in, ptr %coeff, ptr %out, i32 %n) {
+; CHECK-LABEL: define i32 @start_at_non_zero(
+; CHECK-SAME: ptr [[IN:%.*]], ptr [[COEFF:%.*]], ptr [[OUT:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP7:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[CMP7]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[TMP0]], 4294967292
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ <i32 120, i32 0, i32 0, i32 0>, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[IN]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[COEFF]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nsw <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP4]] = add <4 x i32> [[TMP3]], [[VEC_PHI]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP4]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP0]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP6]], [[MIDDLE_BLOCK]] ], [ 120, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[SUM_09:%.*]] = phi i32 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[IN]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[COEFF]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[ADD]] = add nsw i32 [[MUL]], [[SUM_09]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ 120, [[ENTRY:%.*]] ], [ [[ADD_LCSSA]], [[FOR_END_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[SUM_0_LCSSA]]
+;
 entry:
   %cmp7 = icmp sgt i32 %n, 0
   br i1 %cmp7, label %for.body, label %for.end
@@ -153,12 +401,59 @@ for.end:                                          ; preds = %for.body, %entry
   ret i32 %sum.0.lcssa
 }
 
-;CHECK-LABEL: @reduction_and(
-;CHECK: <i32 -1, i32 -1, i32 -1, i32 -1>
-;CHECK: and <4 x i32>
-;CHECK: call i32 @llvm.vector.reduce.and.v4i32(<4 x i32>
-;CHECK: ret i32
-define i32 @reduction_and(i32 %n, ptr nocapture %A, ptr nocapture %B) nounwind uwtable readonly {
+define i32 @reduction_and(i32 %n, ptr %A, ptr %B) {
+; CHECK-LABEL: define i32 @reduction_and(
+; CHECK-SAME: i32 [[N:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP7:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[CMP7]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[TMP0]], 4294967292
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ <i32 -1, i32 -1, i32 -1, i32 -1>, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP4]] = and <4 x i32> [[TMP3]], [[VEC_PHI]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[TMP4]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP0]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP6]], [[MIDDLE_BLOCK]] ], [ -1, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[RESULT_08:%.*]] = phi i32 [ [[AND:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[AND]] = and i32 [[ADD]], [[RESULT_08]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    [[AND_LCSSA:%.*]] = phi i32 [ [[AND]], [[FOR_BODY]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[RESULT_0_LCSSA:%.*]] = phi i32 [ -1, [[ENTRY:%.*]] ], [ [[AND_LCSSA]], [[FOR_END_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT_0_LCSSA]]
+;
 entry:
   %cmp7 = icmp sgt i32 %n, 0
   br i1 %cmp7, label %for.body, label %for.end
@@ -182,11 +477,59 @@ for.end:                                          ; preds = %for.body, %entry
   ret i32 %result.0.lcssa
 }
 
-;CHECK-LABEL: @reduction_or(
-;CHECK: or <4 x i32>
-;CHECK: call i32 @llvm.vector.reduce.or.v4i32(<4 x i32>
-;CHECK: ret i32
-define i32 @reduction_or(i32 %n, ptr nocapture %A, ptr nocapture %B) nounwind uwtable readonly {
+define i32 @reduction_or(i32 %n, ptr %A, ptr %B) {
+; CHECK-LABEL: define i32 @reduction_or(
+; CHECK-SAME: i32 [[N:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP7:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[CMP7]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[TMP0]], 4294967292
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP4]] = or <4 x i32> [[TMP3]], [[VEC_PHI]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP4]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP0]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP6]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[RESULT_08:%.*]] = phi i32 [ [[OR:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[OR]] = or i32 [[ADD]], [[RESULT_08]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    [[OR_LCSSA:%.*]] = phi i32 [ [[OR]], [[FOR_BODY]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[RESULT_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[OR_LCSSA]], [[FOR_END_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT_0_LCSSA]]
+;
 entry:
   %cmp7 = icmp sgt i32 %n, 0
   br i1 %cmp7, label %for.body, label %for.end
@@ -210,11 +553,59 @@ for.end:                                          ; preds = %for.body, %entry
   ret i32 %result.0.lcssa
 }
 
-;CHECK-LABEL: @reduction_xor(
-;CHECK: xor <4 x i32>
-;CHECK: call i32 @llvm.vector.reduce.xor.v4i32(<4 x i32>
-;CHECK: ret i32
-define i32 @reduction_xor(i32 %n, ptr nocapture %A, ptr nocapture %B) nounwind uwtable readonly {
+define i32 @reduction_xor(i32 %n, ptr %A, ptr %B) {
+; CHECK-LABEL: define i32 @reduction_xor(
+; CHECK-SAME: i32 [[N:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP7:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[CMP7]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[TMP0]], 4294967292
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP4]] = xor <4 x i32> [[TMP3]], [[VEC_PHI]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.xor.v4i32(<4 x i32> [[TMP4]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP0]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP6]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[RESULT_08:%.*]] = phi i32 [ [[XOR:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[XOR]] = xor i32 [[ADD]], [[RESULT_08]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    [[XOR_LCSSA:%.*]] = phi i32 [ [[XOR]], [[FOR_BODY]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[RESULT_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[XOR_LCSSA]], [[FOR_END_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT_0_LCSSA]]
+;
 entry:
   %cmp7 = icmp sgt i32 %n, 0
   br i1 %cmp7, label %for.body, label %for.end
@@ -239,11 +630,30 @@ for.end:                                          ; preds = %for.body, %entry
 }
 
 ; In this code the subtracted variable is on the RHS and this is not an induction variable.
-;CHECK-LABEL: @reduction_sub_rhs(
-;CHECK-NOT: phi <4 x i32>
-;CHECK-NOT: sub nsw <4 x i32>
-;CHECK: ret i32
-define i32 @reduction_sub_rhs(i32 %n, ptr noalias nocapture %A) nounwind uwtable readonly {
+define i32 @reduction_sub_rhs(i32 %n, ptr %A) {
+; CHECK-LABEL: define i32 @reduction_sub_rhs(
+; CHECK-SAME: i32 [[N:%.*]], ptr [[A:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP4:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[CMP4]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[X_05:%.*]] = phi i32 [ [[SUB:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[SUB]] = sub nsw i32 [[TMP0]], [[X_05]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[X_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[SUB]], [[FOR_END_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[X_0_LCSSA]]
+;
 entry:
   %cmp4 = icmp sgt i32 %n, 0
   br i1 %cmp4, label %for.body, label %for.end
@@ -266,11 +676,53 @@ for.end:                                          ; preds = %for.body, %entry
 
 
 ; In this test the reduction variable is on the LHS and we can vectorize it.
-;CHECK-LABEL: @reduction_sub_lhs(
-;CHECK: phi <4 x i32>
-;CHECK: sub <4 x i32>
-;CHECK: ret i32
-define i32 @reduction_sub_lhs(i32 %n, ptr noalias nocapture %A) nounwind uwtable readonly {
+define i32 @reduction_sub_lhs(i32 %n, ptr %A) {
+; CHECK-LABEL: define i32 @reduction_sub_lhs(
+; CHECK-SAME: i32 [[N:%.*]], ptr [[A:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP4:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[CMP4]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[TMP0]], 4294967292
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP2:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP2]] = sub <4 x i32> [[VEC_PHI]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP2]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP0]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP4]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[X_05:%.*]] = phi i32 [ [[SUB:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[SUB]] = sub nsw i32 [[X_05]], [[TMP5]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    [[SUB_LCSSA:%.*]] = phi i32 [ [[SUB]], [[FOR_BODY]] ], [ [[TMP4]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[X_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[SUB_LCSSA]], [[FOR_END_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[X_0_LCSSA]]
+;
 entry:
   %cmp4 = icmp sgt i32 %n, 0
   br i1 %cmp4, label %for.body, label %for.end
@@ -292,10 +744,57 @@ for.end:                                          ; preds = %for.body, %entry
 }
 
 ; We can vectorize conditional reductions with multi-input phis.
-; CHECK: reduction_conditional
-; CHECK: fadd fast <4 x float>
-
 define float @reduction_conditional(ptr %A, ptr %B, ptr %C, float %S) {
+; CHECK-LABEL: define float @reduction_conditional(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], float [[S:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x float> <float poison, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00>, float [[S]], i64 0
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ [[TMP0]], [[VECTOR_PH]] ], [ [[PREDPHI3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = fcmp ogt <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fcmp ule <4 x float> [[WIDE_LOAD1]], <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+; CHECK-NEXT:    [[TMP5:%.*]] = fcmp ogt <4 x float> [[WIDE_LOAD]], <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+; CHECK-NEXT:    [[TMP6:%.*]] = and <4 x i1> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = and <4 x i1> [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <4 x i1> [[TMP5]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP9:%.*]] = and <4 x i1> [[TMP6]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = xor <4 x i1> [[TMP3]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[PREDPHI_V:%.*]] = select <4 x i1> [[TMP7]], <4 x float> [[WIDE_LOAD1]], <4 x float> [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[TMP10]], <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i1> [[TMP9]]
+; CHECK-NEXT:    [[PREDPHI2:%.*]] = select <4 x i1> [[TMP11]], <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, <4 x float> [[PREDPHI_V]]
+; CHECK-NEXT:    [[PREDPHI3]] = fadd fast <4 x float> [[VEC_PHI]], [[PREDPHI2]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[TMP13:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[PREDPHI3]])
+; CHECK-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    br i1 poison, label [[IF_THEN:%.*]], label [[FOR_INC:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    br i1 poison, label [[IF_THEN8:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then8:
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       if.else:
+; CHECK-NEXT:    br i1 poison, label [[IF_THEN16:%.*]], label [[FOR_INC]]
+; CHECK:       if.then16:
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    br i1 poison, label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[SUM_1_LCSSA:%.*]] = phi float [ poison, [[FOR_INC]] ], [ [[TMP13]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret float [[SUM_1_LCSSA]]
+;
 entry:
   br label %for.body
 
@@ -338,9 +837,41 @@ for.end:
 }
 
 ; We can't vectorize reductions with phi inputs from outside the reduction.
-; CHECK: noreduction_phi
-; CHECK-NOT: fadd <4 x float>
 define float @noreduction_phi(ptr %A, ptr %B, ptr %C, float %S) {
+; CHECK-LABEL: define float @noreduction_phi(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], float [[S:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-NEXT:    [[SUM_033:%.*]] = phi float [ [[S]], [[ENTRY]] ], [ [[SUM_1:%.*]], [[FOR_INC]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[CMP3:%.*]] = fcmp ogt float [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[CMP3]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[CMP6:%.*]] = fcmp ogt float [[TMP1]], 1.000000e+00
+; CHECK-NEXT:    br i1 [[CMP6]], label [[IF_THEN8:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then8:
+; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float [[SUM_033]], [[TMP0]]
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       if.else:
+; CHECK-NEXT:    [[CMP14:%.*]] = fcmp ogt float [[TMP0]], 2.000000e+00
+; CHECK-NEXT:    br i1 [[CMP14]], label [[IF_THEN16:%.*]], label [[FOR_INC]]
+; CHECK:       if.then16:
+; CHECK-NEXT:    [[ADD19:%.*]] = fadd fast float [[SUM_033]], [[TMP1]]
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[SUM_1]] = phi float [ [[ADD]], [[IF_THEN8]] ], [ [[ADD19]], [[IF_THEN16]] ], [ 0.000000e+00, [[IF_ELSE]] ], [ [[SUM_033]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], 128
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret float [[SUM_1]]
+;
 entry:
   br label %for.body
 
@@ -383,10 +914,27 @@ for.end:
 }
 
 ; We can't vectorize reductions that feed another header PHI.
-; CHECK: noredux_header_phi
-; CHECK-NOT: fadd <4 x float>
-
 define float @noredux_header_phi(ptr %A, ptr %B, ptr %C, float %S)  {
+; CHECK-LABEL: define float @noredux_header_phi(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], float [[S:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUM2_09:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ], [ [[ADD1:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUM_08:%.*]] = phi float [ [[S]], [[ENTRY]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[ADD]] = fadd fast float [[SUM_08]], [[TMP0]]
+; CHECK-NEXT:    [[ADD1]] = fadd fast float [[SUM2_09]], [[ADD]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], 128
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[ADD2:%.*]] = fadd fast float [[ADD]], [[ADD1]]
+; CHECK-NEXT:    ret float [[ADD2]]
+;
 entry:
   br label %for.body
 
@@ -415,11 +963,21 @@ for.end:
 ; loop special care must be taken. Otherwise, the reduced value feeding into the
 ; outside user misses a few iterations (VF-1) of the loop.
 ; PR16522
-
-; CHECK-LABEL: @phivalueredux(
-; CHECK-NOT: x i32>
-
 define i32 @phivalueredux(i32 %p) {
+; CHECK-LABEL: define i32 @phivalueredux(
+; CHECK-SAME: i32 [[P:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[T_03:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[P_ADDR_02:%.*]] = phi i32 [ [[P]], [[ENTRY]] ], [ [[XOR:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[XOR]] = xor i32 [[P_ADDR_02]], -1
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[T_03]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 16
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret i32 [[P_ADDR_02]]
+;
 entry:
   br label %for.body
 
@@ -438,10 +996,25 @@ for.end:
 ; Don't vectorize a reduction value that is not the last in a reduction cyle. We
 ; would loose iterations (VF-1) on the operations after that use.
 ; PR17498
-
-; CHECK-LABEL: not_last_operation
-; CHECK-NOT: x i32>
 define i32 @not_last_operation(i32 %p, i32 %val) {
+; CHECK-LABEL: define i32 @not_last_operation(
+; CHECK-SAME: i32 [[P:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[P]], 0
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INC613_1:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC6_1:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[INC511_1:%.*]] = phi i32 [ [[VAL]], [[ENTRY]] ], [ [[INC5_1:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[INC4_1:%.*]] = zext i1 [[TOBOOL]] to i32
+; CHECK-NEXT:    [[INC511_1_INC4_1:%.*]] = add nsw i32 [[INC511_1]], [[INC4_1]]
+; CHECK-NEXT:    [[INC5_1]] = add nsw i32 [[INC511_1_INC4_1]], 1
+; CHECK-NEXT:    [[INC6_1]] = add nuw nsw i32 [[INC613_1]], 1
+; CHECK-NEXT:    [[EXITCOND_1:%.*]] = icmp eq i32 [[INC6_1]], 22
+; CHECK-NEXT:    br i1 [[EXITCOND_1]], label [[EXIT:%.*]], label [[FOR_BODY]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[INC_2:%.*]] = add nsw i32 [[INC511_1_INC4_1]], 2
+; CHECK-NEXT:    ret i32 [[INC_2]]
+;
 entry:
   %tobool = icmp eq i32 %p, 0
   br label %for.body
@@ -462,14 +1035,64 @@ exit:
   ret i32 %inc.2
 }
 
-;CHECK-LABEL: @reduction_sum_multiuse(
-;CHECK: phi <4 x i32>
-;CHECK: load <4 x i32>
-;CHECK: add <4 x i32>
-;CHECK: call i32 @llvm.vector.reduce.add.v4i32(<4 x i32>
-;CHECK: %sum.copy = phi i32 [ %[[SCALAR:.*]], %.lr.ph ], [ %[[VECTOR:.*]], %middle.block ]
-;CHECK: ret i32
-define i32 @reduction_sum_multiuse(i32 %n, ptr noalias nocapture %A, ptr noalias nocapture %B) {
+define i32 @reduction_sum_multiuse(i32 %n, ptr %A, ptr %B) {
+; CHECK-LABEL: define i32 @reduction_sum_multiuse(
+; CHECK-SAME: i32 [[N:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[TMP1]], label [[DOTLR_PH_PREHEADER:%.*]], label [[END:%.*]]
+; CHECK:       .lr.ph.preheader:
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[N]] to i64
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[TMP2]], 4294967292
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i32> [[VEC_PHI]], [[VEC_IND]]
+; CHECK-NEXT:    [[TMP6:%.*]] = add <4 x i32> [[TMP5]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP7]] = add <4 x i32> [[TMP6]], [[WIDE_LOAD1]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP7]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP2]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[DOT_CRIT_EDGE:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[DOTLR_PH_PREHEADER]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], [[MIDDLE_BLOCK]] ], [ 0, [[DOTLR_PH_PREHEADER]] ]
+; CHECK-NEXT:    br label [[DOTLR_PH:%.*]]
+; CHECK:       .lr.ph:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[DOTLR_PH]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[SUM_02:%.*]] = phi i32 [ [[TMP17:%.*]], [[DOTLR_PH]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4
+; CHECK-NEXT:    [[TMP14:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[TMP15:%.*]] = add i32 [[SUM_02]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = add i32 [[TMP15]], [[TMP11]]
+; CHECK-NEXT:    [[TMP17]] = add i32 [[TMP16]], [[TMP13]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[DOT_CRIT_EDGE]], label [[DOTLR_PH]], !llvm.loop [[LOOP23:![0-9]+]]
+; CHECK:       ._crit_edge:
+; CHECK-NEXT:    [[SUM_COPY:%.*]] = phi i32 [ [[TMP17]], [[DOTLR_PH]] ], [ [[TMP9]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[TMP18:%.*]] = shl i32 [[SUM_COPY]], 1
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[F2:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP18]], [[DOT_CRIT_EDGE]] ]
+; CHECK-NEXT:    ret i32 [[F2]]
+;
   %1 = icmp sgt i32 %n, 0
   br i1 %1, label %.lr.ph.preheader, label %end
 .lr.ph.preheader:                                 ; preds = %0
@@ -505,9 +1128,37 @@ end:
 
 ; This looks like a predicated reduction, but it is a reset of the reduction
 ; variable. We cannot vectorize this.
-; CHECK-LABEL: reduction_reset(
-; CHECK-NOT: <4 x i32>
-define void @reduction_reset(i32 %N, ptr nocapture readonly %arrayA, ptr nocapture %arrayB) {
+define void @reduction_reset(i32 %N, ptr %arrayA, ptr %arrayB) {
+; CHECK-LABEL: define void @reduction_reset(
+; CHECK-SAME: i32 [[N:%.*]], ptr [[ARRAYA:%.*]], ptr [[ARRAYB:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[C4:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[C4]], label [[DOTLR_PH_PREHEADER:%.*]], label [[DOT_CRIT_EDGE:%.*]]
+; CHECK:       .lr.ph.preheader:
+; CHECK-NEXT:    [[C5:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64
+; CHECK-NEXT:    br label [[DOTLR_PH:%.*]]
+; CHECK:       .lr.ph:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[DOTLR_PH_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[DOTLR_PH]] ]
+; CHECK-NEXT:    [[DOT017:%.*]] = phi i32 [ 100, [[DOTLR_PH_PREHEADER]] ], [ [[CSEL:%.*]], [[DOTLR_PH]] ]
+; CHECK-NEXT:    [[C6:%.*]] = getelementptr inbounds i32, ptr [[ARRAYA]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[C7:%.*]] = load i32, ptr [[C6]], align 4
+; CHECK-NEXT:    [[C8:%.*]] = icmp sgt i32 [[C7]], 0
+; CHECK-NEXT:    [[C9:%.*]] = add nsw i32 [[C7]], [[DOT017]]
+; CHECK-NEXT:    [[CSEL]] = select i1 [[C8]], i32 [[C9]], i32 0
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[DOT_CRIT_EDGE_LOOPEXIT:%.*]], label [[DOTLR_PH]]
+; CHECK:       ._crit_edge.loopexit:
+; CHECK-NEXT:    [[PHITMP19:%.*]] = sext i32 [[C5]] to i64
+; CHECK-NEXT:    br label [[DOT_CRIT_EDGE]]
+; CHECK:       ._crit_edge:
+; CHECK-NEXT:    [[DOT015_LCSSA:%.*]] = phi i64 [ -1, [[ENTRY:%.*]] ], [ [[PHITMP19]], [[DOT_CRIT_EDGE_LOOPEXIT]] ]
+; CHECK-NEXT:    [[DOT0_LCSSA:%.*]] = phi i32 [ 100, [[ENTRY]] ], [ [[CSEL]], [[DOT_CRIT_EDGE_LOOPEXIT]] ]
+; CHECK-NEXT:    [[C10:%.*]] = getelementptr inbounds i32, ptr [[ARRAYB]], i64 [[DOT015_LCSSA]]
+; CHECK-NEXT:    store i32 [[DOT0_LCSSA]], ptr [[C10]], align 4
+; CHECK-NEXT:    ret void
+;
 entry:
   %c4 = icmp sgt i32 %N, 0
   br i1 %c4, label %.lr.ph.preheader, label %._crit_edge
@@ -544,8 +1195,56 @@ entry:
 
 ; Can vectorize reduction with redundant single-operand phi input.
 define i64 @reduction_with_phi_with_one_incoming_on_backedge(i16 %n, ptr %A) {
-; CHECK-LABEL: @reduction_with_phi_with_one_incoming_on_backedge
-; CHECK: add <4 x i64>
+; CHECK-LABEL: define i64 @reduction_with_phi_with_one_incoming_on_backedge(
+; CHECK-SAME: i16 [[N:%.*]], ptr [[A:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SMAX:%.*]] = call i16 @llvm.smax.i16(i16 [[N]], i16 2)
+; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i16 [[SMAX]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[TMP0]] to i32
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i16 [[SMAX]], 5
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i32 [[TMP1]], 65532
+; CHECK-NEXT:    [[DOTCAST:%.*]] = trunc i32 [[N_VEC]] to i16
+; CHECK-NEXT:    [[IND_END:%.*]] = or i16 [[DOTCAST]], 1
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[DOTCAST1:%.*]] = trunc i32 [[INDEX]] to i16
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = or i16 [[DOTCAST1]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i16 [[OFFSET_IDX]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i64, ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP4]] = add <4 x i64> [[VEC_PHI]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP4]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N_VEC]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP6]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
+; CHECK:       loop.header:
+; CHECK-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[SUM:%.*]] = phi i64 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[SUM_NEXT:%.*]], [[LOOP_LATCH]] ]
+; CHECK-NEXT:    [[TMP7:%.*]] = sext i16 [[IV]] to i64
+; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr i64, ptr [[A]], i64 [[TMP7]]
+; CHECK-NEXT:    [[LV_A:%.*]] = load i64, ptr [[GEP_A]], align 4
+; CHECK-NEXT:    [[SUM_NEXT]] = add nsw i64 [[SUM]], [[LV_A]]
+; CHECK-NEXT:    br label [[LOOP_BB:%.*]]
+; CHECK:       loop.bb:
+; CHECK-NEXT:    br label [[LOOP_LATCH]]
+; CHECK:       loop.latch:
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i16 [[IV]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp slt i16 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[COND]], label [[LOOP_HEADER]], label [[EXIT]], !llvm.loop [[LOOP25:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[LCSSA_EXIT:%.*]] = phi i64 [ [[SUM_NEXT]], [[LOOP_LATCH]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i64 [[LCSSA_EXIT]]
 ;
 entry:
   br label %loop.header
@@ -574,8 +1273,57 @@ exit:
 
 ; Can vectorize reduction with redundant two-operand phi input.
 define i64 @reduction_with_phi_with_two_incoming_on_backedge(i16 %n, ptr %A) {
-; CHECK-LABEL: @reduction_with_phi_with_two_incoming_on_backedge
-; CHECK: add <4 x i64>
+; CHECK-LABEL: define i64 @reduction_with_phi_with_two_incoming_on_backedge(
+; CHECK-SAME: i16 [[N:%.*]], ptr [[A:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SMAX:%.*]] = call i16 @llvm.smax.i16(i16 [[N]], i16 2)
+; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i16 [[SMAX]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[TMP0]] to i32
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i16 [[SMAX]], 5
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i32 [[TMP1]], 65532
+; CHECK-NEXT:    [[DOTCAST:%.*]] = trunc i32 [[N_VEC]] to i16
+; CHECK-NEXT:    [[IND_END:%.*]] = or i16 [[DOTCAST]], 1
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[DOTCAST1:%.*]] = trunc i32 [[INDEX]] to i16
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = or i16 [[DOTCAST1]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i16 [[OFFSET_IDX]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i64, ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP4]] = add <4 x i64> [[VEC_PHI]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP4]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N_VEC]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP6]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
+; CHECK:       loop.header:
+; CHECK-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[SUM:%.*]] = phi i64 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[SUM_NEXT:%.*]], [[LOOP_LATCH]] ]
+; CHECK-NEXT:    [[TMP7:%.*]] = sext i16 [[IV]] to i64
+; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr i64, ptr [[A]], i64 [[TMP7]]
+; CHECK-NEXT:    [[LV_A:%.*]] = load i64, ptr [[GEP_A]], align 4
+; CHECK-NEXT:    [[SUM_NEXT]] = add nsw i64 [[SUM]], [[LV_A]]
+; CHECK-NEXT:    [[CMP_0:%.*]] = icmp eq i64 [[LV_A]], 29
+; CHECK-NEXT:    br i1 [[CMP_0]], label [[LOOP_BB:%.*]], label [[LOOP_LATCH]]
+; CHECK:       loop.bb:
+; CHECK-NEXT:    br label [[LOOP_LATCH]]
+; CHECK:       loop.latch:
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i16 [[IV]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp slt i16 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[COND]], label [[LOOP_HEADER]], label [[EXIT]], !llvm.loop [[LOOP27:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[LCSSA_EXIT:%.*]] = phi i64 [ [[SUM_NEXT]], [[LOOP_LATCH]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i64 [[LCSSA_EXIT]]
 ;
 entry:
   br label %loop.header
@@ -602,6 +1350,3 @@ exit:
   %lcssa.exit = phi i64 [ %phi.sum.next, %loop.latch ]
   ret i64 %lcssa.exit
 }
-
-; Make sure any check-not directives are not triggered by function declarations.
-; CHECK: declare
diff --git a/llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll b/llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll
index 3cc6e5fa7b..afe16c71f7 100644
--- a/llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll
@@ -17,14 +17,14 @@ define i8 @reduction_add_trunc(ptr noalias nocapture %A) {
 ; CHECK-NEXT:    [[TMP27:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD2]] to <vscale x 8 x i32>
 ; CHECK-NEXT:    [[TMP28:%.*]] = add <vscale x 8 x i32> [[TMP14]], [[TMP26]]
 ; CHECK-NEXT:    [[TMP29:%.*]] = add <vscale x 8 x i32> [[TMP15]], [[TMP27]]
+; CHECK-NEXT:    [[TMP33:%.*]] = trunc <vscale x 8 x i32> [[TMP28]] to <vscale x 8 x i8>
+; CHECK-NEXT:    [[TMP35:%.*]] = trunc <vscale x 8 x i32> [[TMP29]] to <vscale x 8 x i8>
+; CHECK-NEXT:    [[TMP34]] = zext <vscale x 8 x i8> [[TMP33]] to <vscale x 8 x i32>
+; CHECK-NEXT:    [[TMP36]] = zext <vscale x 8 x i8> [[TMP35]] to <vscale x 8 x i32>
 ; CHECK-NEXT:    [[TMP30:%.*]] = call i32 @llvm.vscale.i32()
 ; CHECK-NEXT:    [[TMP31:%.*]] = mul i32 [[TMP30]], 16
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP31]]
 ; CHECK-NEXT:    [[TMP32:%.*]] = icmp eq i32 [[INDEX_NEXT]], {{%.*}}
-; CHECK-NEXT:    [[TMP33:%.*]] = trunc <vscale x 8 x i32> [[TMP28]] to <vscale x 8 x i8>
-; CHECK-NEXT:    [[TMP34]] = zext <vscale x 8 x i8> [[TMP33]] to <vscale x 8 x i32>
-; CHECK-NEXT:    [[TMP35:%.*]] = trunc <vscale x 8 x i32> [[TMP29]] to <vscale x 8 x i8>
-; CHECK-NEXT:    [[TMP36]] = zext <vscale x 8 x i8> [[TMP35]] to <vscale x 8 x i32>
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[TMP37:%.*]] = trunc <vscale x 8 x i32> [[TMP34]] to <vscale x 8 x i8>
 ; CHECK-NEXT:    [[TMP38:%.*]] = trunc <vscale x 8 x i32> [[TMP36]] to <vscale x 8 x i8>
diff --git a/llvm/test/Transforms/LoopVectorize/scev-invalidation.ll b/llvm/test/Transforms/LoopVectorize/scev-invalidation.ll
new file mode 100644
index 0000000000..08163293c1
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/scev-invalidation.ll
@@ -0,0 +1,22 @@
+; RUN: opt < %s -passes="require<scalar-evolution>,print<scalar-evolution>,loop-vectorize" --verify-scev -force-vector-interleave=2 -force-vector-width=8 -S | FileCheck %s
+
+; CHECK-LABEL: @main(
+; CHECK: vector.body
+define i32 @main(i32 %.pre) {
+entry:
+  br label %for.body
+
+for.body:
+  %g.019 = phi i16 [ 0, %entry ], [ %dec7, %for.body ]
+  %and = and i32 %.pre, 40
+  %0 = sub i32 0, %and
+  %dec7 = add i16 %g.019, 1
+  %cmp.not = icmp eq i16 %dec7, 0
+  br i1 %cmp.not, label %for.inc16, label %for.body
+
+for.inc16:
+  %1 = phi i32 [ %inc, %for.inc16 ], [ 0, %for.body ]
+  %inc = add i32 %1, 1
+  %add12 = add i32 %0, %1
+  br label %for.inc16
+}
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll
index db2ca36352..6f54d37383 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll
@@ -26,7 +26,7 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounw
 ; CHECK-NEXT:    label="\<x1\> vector loop"
 ; CHECK-NEXT:    N2 [label =
 ; CHECK-NEXT:    "vector.body:\l" +
-; CHECK-NEXT:    "  EMIT vp\<[[CAN_IV:%.+]]\> = CANONICAL-INDUCTION\l" +
+; CHECK-NEXT:    "  EMIT vp\<[[CAN_IV:%.+]]\> = CANONICAL-INDUCTION ir\<0\>, vp\<%7\>\l" +
 ; CHECK-NEXT:    "  vp\<[[STEPS:%.+]]\> = SCALAR-STEPS vp\<[[CAN_IV]]\>, ir\<1\>\l" +
 ; CHECK-NEXT:    "  CLONE ir\<%arrayidx\> = getelementptr inbounds ir\<%y\>, vp\<[[STEPS]]\>\l" +
 ; CHECK-NEXT:    "  WIDEN ir\<%lv\> = load ir\<%arrayidx\>\l" +
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
index bbede3f79d..3626b8c9dc 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
@@ -17,14 +17,14 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounw
 ; CHECK-EMPTY:
 ; CHECK-NEXT: <x1> vector loop: {
 ; CHECK-NEXT: vector.body:
-; CHECK-NEXT:   EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
+; CHECK-NEXT:   EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
 ; CHECK-NEXT:   vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
 ; CHECK-NEXT:   CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[STEPS]]>
 ; CHECK-NEXT:   WIDEN ir<%lv> = load ir<%arrayidx>
 ; CHECK-NEXT:   WIDEN-CALL ir<%call> = call @llvm.sqrt.f32(ir<%lv>)
 ; CHECK-NEXT:   CLONE ir<%arrayidx2> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
 ; CHECK-NEXT:   WIDEN store ir<%arrayidx2>, ir<%call>
-; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]>
+; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]>
 ; CHECK-NEXT:   EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
 ; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
@@ -64,7 +64,7 @@ define void @print_widen_gep_and_select(i64 %n, ptr noalias %y, ptr noalias %x,
 ; CHECK-EMPTY:
 ; CHECK-NEXT: <x1> vector loop: {
 ; CHECK-NEXT: vector.body:
-; CHECK-NEXT:   EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
+; CHECK-NEXT:   EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
 ; CHECK-NEXT:   WIDEN-INDUCTION %iv = phi %iv.next, 0, ir<1>
 ; CHECK-NEXT:   vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
 ; CHECK-NEXT:   WIDEN-GEP Inv[Var] ir<%arrayidx> = getelementptr inbounds ir<%y>, ir<%iv>
@@ -74,7 +74,7 @@ define void @print_widen_gep_and_select(i64 %n, ptr noalias %y, ptr noalias %x,
 ; CHECK-NEXT:   WIDEN ir<%add> = fadd ir<%lv>, ir<%sel>
 ; CHECK-NEXT:   CLONE ir<%arrayidx2> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
 ; CHECK-NEXT:   WIDEN store ir<%arrayidx2>, ir<%add>
-; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]>
+; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]>
 ; CHECK-NEXT:   EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
 ; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
@@ -116,13 +116,13 @@ define float @print_reduction(i64 %n, ptr noalias %y) {
 ; CHECK-EMPTY:
 ; CHECK-NEXT: <x1> vector loop: {
 ; CHECK-NEXT: vector.body:
-; CHECK-NEXT:   EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
+; CHECK-NEXT:   EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
 ; CHECK-NEXT:   WIDEN-REDUCTION-PHI ir<%red> = phi ir<0.000000e+00>, ir<%red.next>
 ; CHECK-NEXT:   vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
 ; CHECK-NEXT:   CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[STEPS]]>
 ; CHECK-NEXT:   WIDEN ir<%lv> = load ir<%arrayidx>
 ; CHECK-NEXT:   REDUCE ir<%red.next> = ir<%red> + fast reduce.fadd (ir<%lv>)
-; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]>
+; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]>
 ; CHECK-NEXT:   EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
 ; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
@@ -162,13 +162,13 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no
 ; CHECK-EMPTY:
 ; CHECK-NEXT: <x1> vector loop: {
 ; CHECK-NEXT: vector.body:
-; CHECK-NEXT:   EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
+; CHECK-NEXT:   EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
 ; CHECK-NEXT:   WIDEN-REDUCTION-PHI ir<%red> = phi ir<0.000000e+00>, ir<%red.next>
 ; CHECK-NEXT:   vp<[[IV:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
 ; CHECK-NEXT:   CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[IV]]>
 ; CHECK-NEXT:   WIDEN ir<%lv> = load ir<%arrayidx>
 ; CHECK-NEXT:   REDUCE ir<%red.next> = ir<%red> + fast reduce.fadd (ir<%lv>) (with final reduction value stored in invariant address sank outside of loop)
-; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]>
+; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]>
 ; CHECK-NEXT:   EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
 ; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
@@ -211,7 +211,7 @@ define void @print_replicate_predicated_phi(i64 %n, ptr %x) {
 ; CHECK-EMPTY:
 ; CHECK-NEXT: <x1> vector loop: {
 ; CHECK-NEXT: vector.body:
-; CHECK-NEXT:   EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
+; CHECK-NEXT:   EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
 ; CHECK-NEXT:   WIDEN-INDUCTION %i = phi 0, %i.next, ir<1>
 ; CHECK-NEXT:   vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
 ; CHECK-NEXT:   WIDEN ir<%cmp> = icmp ult ir<%i>, ir<5>
@@ -237,7 +237,7 @@ define void @print_replicate_predicated_phi(i64 %n, ptr %x) {
 ; CHECK-NEXT:   BLEND ir<%d> = ir<0>/vp<[[NOT]]> vp<[[PRED]]>/ir<%cmp>
 ; CHECK-NEXT:   CLONE ir<%idx> = getelementptr ir<%x>, vp<[[STEPS]]>
 ; CHECK-NEXT:   WIDEN store ir<%idx>, ir<%d>
-; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]>
+; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]>
 ; CHECK-NEXT:   EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
 ; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
@@ -285,7 +285,7 @@ define void @print_interleave_groups(i32 %C, i32 %D) {
 ; CHECK-EMPTY:
 ; CHECK-NEXT: <x1> vector loop: {
 ; CHECK-NEXT:  vector.body:
-; CHECK-NEXT:   EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
+; CHECK-NEXT:   EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
 ; CHECK-NEXT:   vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<0> + vp<[[CAN_IV]]> * ir<4>
 ; CHECK-NEXT:   vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<4>
 ; CHECK-NEXT:   CLONE ir<%gep.AB.0> = getelementptr inbounds ir<@AB>, ir<0>, vp<[[STEPS]]>
@@ -301,7 +301,7 @@ define void @print_interleave_groups(i32 %C, i32 %D) {
 ; CHECK-NEXT:     store ir<1> to index 1
 ; CHECK-NEXT:     store ir<2> to index 2
 ; CHECK-NEXT:     store ir<%AB.3> to index 3
-; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]>
+; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]>
 ; CHECK-NEXT:   EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
 ; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
@@ -353,7 +353,7 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) {
 ; CHECK-EMPTY:
 ; CHECK-NEXT: <x1> vector loop: {
 ; CHECK-NEXT: vector.body:
-; CHECK-NEXT:   EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
+; CHECK-NEXT:   EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
 ; CHECK-NEXT:   WIDEN-REDUCTION-PHI ir<%sum.07> = phi ir<0.000000e+00>, ir<%muladd>
 ; CHECK-NEXT:   vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
 ; CHECK-NEXT:   CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
@@ -362,7 +362,7 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) {
 ; CHECK-NEXT:   WIDEN ir<%l.b> = load ir<%arrayidx2>
 ; CHECK-NEXT:   EMIT vp<[[FMUL:%.+]]> = fmul nnan ninf nsz ir<%l.a>, ir<%l.b>
 ; CHECK-NEXT:   REDUCE ir<[[MULADD:%.+]]> = ir<%sum.07> + nnan ninf nsz reduce.fadd (vp<[[FMUL]]>)
-; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]>
+; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]>
 ; CHECK-NEXT:   EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
 ; CHECK-NEXT:   No successors
 ; CHECK-NEXT: }
@@ -404,7 +404,7 @@ define void @debug_loc_vpinstruction(ptr nocapture %asd, ptr nocapture %bsd) !db
 ; CHECK-EMPTY:
 ; CHECK-NEXT: <x1> vector loop: {
 ; CHECK-NEXT:  vector.body:
-; CHECK-NEXT:    EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
+; CHECK-NEXT:    EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
 ; CHECK-NEXT:    vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
 ; CHECK-NEXT:    CLONE ir<%isd> = getelementptr inbounds ir<%asd>, vp<[[STEPS]]>
 ; CHECK-NEXT:    WIDEN ir<%lsd> = load ir<%isd>
@@ -436,7 +436,7 @@ define void @debug_loc_vpinstruction(ptr nocapture %asd, ptr nocapture %bsd) !db
 ; CHECK-NEXT:    EMIT vp<[[SEL2:%.+]]> = select vp<[[NOT1]]>, vp<[[NOT2]]>, ir<false>
 ; CHECK-NEXT:    BLEND ir<%ysd.0> = vp<[[PHI]]>/vp<[[OR1]]> ir<%psd>/vp<[[SEL2]]>
 ; CHECK-NEXT:    WIDEN store ir<%isd>, ir<%ysd.0>
-; CHECK-NEXT:    EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]>
+; CHECK-NEXT:    EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]>
 ; CHECK-NEXT:    EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
 ; CHECK-NEXT:  No successors
 ; CHECK-NEXT:}
@@ -495,7 +495,7 @@ define void @print_expand_scev(i64 %y, ptr %ptr) {
 ; CHECK-EMPTY:
 ; CHECK-NEXT: <x1> vector loop: {
 ; CHECK-NEXT:   vector.body:
-; CHECK-NEXT:     EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
+; CHECK-NEXT:    EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
 ; CHECK-NEXT:     WIDEN-INDUCTION\l" +
 ; CHECK-NEXT:     "  %iv = phi %iv.next, 0\l" +
 ; CHECK-NEXT:     "  ir<%v2>, vp<[[EXP_SCEV]]>
@@ -504,8 +504,8 @@ define void @print_expand_scev(i64 %y, ptr %ptr) {
 ; CHECK-NEXT:     WIDEN ir<%v3> = add nuw ir<%v2>, ir<1>
 ; CHECK-NEXT:     REPLICATE ir<%gep> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]>
 ; CHECK-NEXT:     REPLICATE store ir<%v3>, ir<%gep>
-; CHECK-NEXT:     EMIT vp<[[CAN_INC:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]>
-; CHECK-NEXT:     EMIT branch-on-count  vp<[[CAN_INC]]>, vp<[[VTC]]>
+; CHECK-NEXT:     EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]>
+; CHECK-NEXT:     EMIT branch-on-count  vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
 ; CHECK-NEXT:   No successors
 ; CHECK-NEXT: }
 ; CHECK-NEXT: Successor(s): middle.block
@@ -545,13 +545,13 @@ define i32 @print_exit_value(ptr %ptr, i32 %off) {
 ; CHECK-EMPTY:
 ; CHECK-NEXT: <x1> vector loop: {
 ; CHECK-NEXT:   vector.body:
-; CHECK-NEXT:     EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
+; CHECK-NEXT:    EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
 ; CHECK-NEXT:     WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1>
 ; CHECK-NEXT:     vp<[[STEPS:%.+]]>    = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
 ; CHECK-NEXT:     CLONE ir<%gep> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]>
 ; CHECK-NEXT:     WIDEN ir<%add> = add ir<%iv>, ir<%off>
 ; CHECK-NEXT:     WIDEN store ir<%gep>, ir<0>
-; CHECK-NEXT:     EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw  vp<[[CAN_IV]]>
+; CHECK-NEXT:     EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw  vp<[[CAN_IV]]>
 ; CHECK-NEXT:     EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
 ; CHECK-NEXT:   No successors
 ; CHECK-NEXT: }
@@ -591,7 +591,7 @@ define void @print_fast_math_flags(i64 %n, ptr noalias %y, ptr noalias %x, ptr %
 ; CHECK-EMPTY:
 ; CHECK-NEXT: <x1> vector loop: {
 ; CHECK-NEXT: vector.body:
-; CHECK-NEXT:   EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
+; CHECK-NEXT:   EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
 ; CHECK-NEXT:   vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
 ; CHECK-NEXT:   CLONE ir<%gep.y> = getelementptr inbounds ir<%y>, vp<[[STEPS]]>
 ; CHECK-NEXT:   WIDEN ir<%lv> = load ir<%gep.y>
@@ -600,7 +600,7 @@ define void @print_fast_math_flags(i64 %n, ptr noalias %y, ptr noalias %x, ptr %
 ; CHECK-NEXT:   WIDEN ir<%div> = fdiv reassoc nsz contract ir<%mul>, ir<2.000000e+00>
 ; CHECK-NEXT:   CLONE ir<%gep.x> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
 ; CHECK-NEXT:   WIDEN store ir<%gep.x>, ir<%div>
-; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]>
+; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]>
 ; CHECK-NEXT:   EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
 ; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
@@ -641,7 +641,7 @@ define void @print_exact_flags(i64 %n, ptr noalias %x) {
 ; CHECK-EMPTY:
 ; CHECK-NEXT: <x1> vector loop: {
 ; CHECK-NEXT: vector.body:
-; CHECK-NEXT:   EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
+; CHECK-NEXT:   EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
 ; CHECK-NEXT:   vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
 ; CHECK-NEXT:   CLONE ir<%gep.x> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
 ; CHECK-NEXT:   WIDEN ir<%lv> = load ir<%gep.x>
@@ -649,7 +649,7 @@ define void @print_exact_flags(i64 %n, ptr noalias %x) {
 ; CHECK-NEXT:   WIDEN ir<%div.2> = udiv ir<%lv>, ir<60>
 ; CHECK-NEXT:   WIDEN ir<%add> = add nuw nsw ir<%div.1>, ir<%div.2>
 ; CHECK-NEXT:   WIDEN store ir<%gep.x>, ir<%add>
-; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]>
+; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]>
 ; CHECK-NEXT:   EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
 ; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
@@ -689,9 +689,9 @@ define void @print_call_flags(ptr readonly %src, ptr noalias %dest, i64 %n) {
 ; CHECK-EMPTY:
 ; CHECK-NEXT: <x1> vector loop: {
 ; CHECK-NEXT: vector.body:
-; CHECK-NEXT:   EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
+; CHECK-NEXT:   EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
 ; CHECK-NEXT:   vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
-; CHECK-NEXT:   CLONE ir<%ld.addr> = getelementptr inbounds ir<%src>, vp<%2>
+; CHECK-NEXT:   CLONE ir<%ld.addr> = getelementptr inbounds ir<%src>, vp<[[STEPS]]>
 ; CHECK-NEXT:   WIDEN ir<%ld.value> = load ir<%ld.addr>
 ; CHECK-NEXT:   WIDEN ir<%ifcond> = fcmp oeq ir<%ld.value>, ir<5.000000e+00>
 ; CHECK-NEXT:  Successor(s): pred.call
@@ -707,19 +707,19 @@ define void @print_call_flags(ptr readonly %src, ptr noalias %dest, i64 %n) {
 ; CHECK-NEXT:    Successor(s): pred.call.continue
 ; CHECK-EMPTY:
 ; CHECK-NEXT:    pred.call.continue:
-; CHECK-NEXT:      PHI-PREDICATED-INSTRUCTION vp<%8> = ir<%foo.ret.1>
-; CHECK-NEXT:      PHI-PREDICATED-INSTRUCTION vp<%9> = ir<%foo.ret.2>
+; CHECK-NEXT:      PHI-PREDICATED-INSTRUCTION vp<[[PHI1:%.+]]> = ir<%foo.ret.1>
+; CHECK-NEXT:      PHI-PREDICATED-INSTRUCTION vp<[[PHI2:%.+]]> = ir<%foo.ret.2>
 ; CHECK-NEXT:    No successors
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  Successor(s): if.then.1
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  if.then.1:
-; CHECK-NEXT:    WIDEN ir<%fadd> = fadd vp<%8>, vp<%9>
-; CHECK-NEXT:    EMIT vp<%11> = not ir<%ifcond>
-; CHECK-NEXT:    BLEND ir<%st.value> = ir<%ld.value>/vp<%11> ir<%fadd>/ir<%ifcond>
-; CHECK-NEXT:    CLONE ir<%st.addr> = getelementptr inbounds ir<%dest>, vp<%2>
+; CHECK-NEXT:    WIDEN ir<%fadd> = fadd vp<[[PHI1]]>, vp<[[PHI2]]>
+; CHECK-NEXT:    EMIT vp<[[NOT_COND:%.+]]> = not ir<%ifcond>
+; CHECK-NEXT:    BLEND ir<%st.value> = ir<%ld.value>/vp<[[NOT_COND]]> ir<%fadd>/ir<%ifcond>
+; CHECK-NEXT:    CLONE ir<%st.addr> = getelementptr inbounds ir<%dest>, vp<[[STEPS]]>
 ; CHECK-NEXT:    WIDEN store ir<%st.addr>, ir<%st.value>
-; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]>
+; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]>
 ; CHECK-NEXT:   EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]>
 ; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll
index 444f36ad3a..0dac63d545 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll
@@ -29,9 +29,7 @@ define void @sink_with_sideeffects(i1 %c, ptr %ptr) {
 ; CHECK-NEXT:   Successor(s): pred.store.if, pred.store.continue
 
 ; CHECK:      pred.store.if:
-; CHECK-NEXT:   CLONE ir<%tmp4> = zext ir<%tmp3>
-; CHECK-NEXT:   CLONE ir<%tmp5> = trunc ir<%tmp4>
-; CHECK-NEXT:   CLONE store ir<%tmp5>, ir<%tmp2>
+; CHECK-NEXT:   CLONE store ir<%tmp3>, ir<%tmp2>
 ; CHECK-NEXT:   Successor(s): pred.store.continue
 
 ; CHECK:      pred.store.continue:
diff --git a/llvm/test/Transforms/LowerTypeTests/function-weak.ll b/llvm/test/Transforms/LowerTypeTests/function-weak.ll
index c765937f19..5f9041cd21 100644
--- a/llvm/test/Transforms/LowerTypeTests/function-weak.ll
+++ b/llvm/test/Transforms/LowerTypeTests/function-weak.ll
@@ -48,6 +48,19 @@ entry:
   ret void
 }
 
+define void @struct() {
+; CHECK-LABEL: define void @struct() {
+; CHECK: %0 = select i1 icmp ne (ptr @f, ptr null), ptr @.cfi.jumptable, ptr null
+; CHECK-NEXT: %1 = icmp ne ptr %0, null
+; CHECK-NEXT: %2 = insertvalue { i1, i8 } poison, i1 %1, 0
+; CHECK-NEXT: %3 = insertvalue { i1, i8 } %2, i8 0, 1
+; CHECK-NEXT: %x = extractvalue { i1, i8 } %3, 0
+
+entry:
+  %x = extractvalue { i1, i8 } { i1 icmp ne (ptr @f, ptr null), i8 0 }, 0
+  ret void
+}
+
 define void @phi(i1 %c) {
 ; CHECK-LABEL: define void @phi(i1 %c) {
 ; CHECK: entry:
diff --git a/llvm/test/Transforms/MergeFunc/cfi-thunk-merging.ll b/llvm/test/Transforms/MergeFunc/cfi-thunk-merging.ll
new file mode 100644
index 0000000000..d35d777282
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/cfi-thunk-merging.ll
@@ -0,0 +1,210 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --include-generated-funcs --version 2
+;; Check the cases involving internal CFI instrumented functions where we do not expect functions to be merged.
+; RUN: opt -S -passes=mergefunc < %s | FileCheck %s
+; RUN: opt -S -passes=mergefunc,lowertypetests < %s | FileCheck --check-prefix=LOWERTYPETESTS %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux"
+
+@0 = private unnamed_addr constant { i16, i16, [12 x i8] } { i16 -1, i16 0, [12 x i8] c"'int (int)'\00" }
+
+; Function Attrs: noinline nounwind optnone
+define dso_local i32 @f(i32 noundef %arg) #0 !type !3 !type !4 {
+entry:
+  %arg.addr = alloca i32, align 4
+  %a = alloca i32, align 4
+  %b = alloca i32, align 4
+  store i32 %arg, ptr %arg.addr, align 4
+  store i32 0, ptr %b, align 4
+  %0 = load i32, ptr %arg.addr, align 4
+  %cmp = icmp sgt i32 %0, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  store i32 1, ptr %a, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  %1 = load i32, ptr %a, align 4
+  %2 = load i32, ptr %b, align 4
+  %add = add nsw i32 %1, %2
+  ret i32 %add
+}
+
+; Function Attrs: noinline nounwind optnone
+define dso_local i32 @f_thunk(i32 noundef %arg) #0 !type !3 !type !4 {
+entry:
+  %arg.addr = alloca i32, align 4
+  %a = alloca i32, align 4
+  %b = alloca i32, align 4
+  store i32 %arg, ptr %arg.addr, align 4
+  store i32 0, ptr %b, align 4
+  %0 = load i32, ptr %arg.addr, align 4
+  %cmp = icmp sgt i32 %0, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  store i32 1, ptr %a, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  %1 = load i32, ptr %a, align 4
+  %2 = load i32, ptr %b, align 4
+  %add = add nsw i32 %1, %2
+  ret i32 %add
+}
+
+; Function Attrs: noinline nounwind optnone
+define dso_local i32 @g(i32 noundef %b) #0 !type !3 !type !4 {
+entry:
+  %b.addr = alloca i32, align 4
+  %fp = alloca ptr, align 8
+  store i32 %b, ptr %b.addr, align 4
+  %0 = load i32, ptr %b.addr, align 4
+  %tobool = icmp ne i32 %0, 0
+  %1 = zext i1 %tobool to i64
+  %cond = select i1 %tobool, ptr @f, ptr @f_thunk
+  store ptr %cond, ptr %fp, align 8
+  %2 = load ptr, ptr %fp, align 8
+  %3 = call i1 @llvm.type.test(ptr %2, metadata !"_ZTSFiiE"), !nosanitize !5
+  br i1 %3, label %cont, label %trap, !nosanitize !5
+
+trap:                                             ; preds = %entry
+  call void @llvm.ubsantrap(i8 2) #3, !nosanitize !5
+  unreachable, !nosanitize !5
+
+cont:                                             ; preds = %entry
+  %4 = load i32, ptr %b.addr, align 4
+  %call = call i32 %2(i32 noundef %4)
+  ret i32 %call
+}
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare i1 @llvm.type.test(ptr, metadata) #1
+
+; Function Attrs: cold noreturn nounwind
+declare void @llvm.ubsantrap(i8 immarg) #2
+
+attributes #0 = { noinline nounwind optnone "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+cx8,+mmx,+sse,+sse2,+x87" }
+attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+attributes #2 = { cold noreturn nounwind }
+attributes #3 = { noreturn nounwind }
+
+!llvm.module.flags = !{!0, !1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 4, !"CFI Canonical Jump Tables", i32 0}
+!3 = !{i64 0, !"_ZTSFiiE"}
+!4 = !{i64 0, !"_ZTSFiiE.generalized"}
+!5 = !{}
+; CHECK-LABEL: define dso_local i32 @f
+; CHECK-SAME: (i32 noundef [[ARG:%.*]]) #[[ATTR0:[0-9]+]] !type !2 !type !3 {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARG_ADDR:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[B:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    store i32 [[ARG]], ptr [[ARG_ADDR]], align 4
+; CHECK-NEXT:    store i32 0, ptr [[B]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARG_ADDR]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP0]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    store i32 1, ptr [[A]], align 4
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[B]], align 4
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+;
+; CHECK-LABEL: define dso_local i32 @g
+; CHECK-SAME: (i32 noundef [[B:%.*]]) #[[ATTR0]] !type !2 !type !3 {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[FP:%.*]] = alloca ptr, align 8
+; CHECK-NEXT:    store i32 [[B]], ptr [[B_ADDR]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[B_ADDR]], align 4
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i1 [[TOBOOL]] to i64
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], ptr @f, ptr @f_thunk
+; CHECK-NEXT:    store ptr [[COND]], ptr [[FP]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[FP]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = call i1 @llvm.type.test(ptr [[TMP2]], metadata !"_ZTSFiiE"), !nosanitize !4
+; CHECK-NEXT:    br i1 [[TMP3]], label [[CONT:%.*]], label [[TRAP:%.*]], !nosanitize !4
+; CHECK:       trap:
+; CHECK-NEXT:    call void @llvm.ubsantrap(i8 2) #[[ATTR3:[0-9]+]], !nosanitize !4
+; CHECK-NEXT:    unreachable, !nosanitize !4
+; CHECK:       cont:
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[B_ADDR]], align 4
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 [[TMP2]](i32 noundef [[TMP4]])
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+;
+; CHECK-LABEL: define dso_local i32 @f_thunk
+; CHECK-SAME: (i32 noundef [[TMP0:%.*]]) #[[ATTR0]] !type !2 {
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call i32 @f(i32 noundef [[TMP0]]) #[[ATTR0]]
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+;
+; LOWERTYPETESTS-LABEL: define dso_local i32 @f
+; LOWERTYPETESTS-SAME: (i32 noundef [[ARG:%.*]]) #[[ATTR0:[0-9]+]] !type !2 !type !3 {
+; LOWERTYPETESTS-NEXT:  entry:
+; LOWERTYPETESTS-NEXT:    [[ARG_ADDR:%.*]] = alloca i32, align 4
+; LOWERTYPETESTS-NEXT:    [[A:%.*]] = alloca i32, align 4
+; LOWERTYPETESTS-NEXT:    [[B:%.*]] = alloca i32, align 4
+; LOWERTYPETESTS-NEXT:    store i32 [[ARG]], ptr [[ARG_ADDR]], align 4
+; LOWERTYPETESTS-NEXT:    store i32 0, ptr [[B]], align 4
+; LOWERTYPETESTS-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARG_ADDR]], align 4
+; LOWERTYPETESTS-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP0]], 0
+; LOWERTYPETESTS-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; LOWERTYPETESTS:       if.then:
+; LOWERTYPETESTS-NEXT:    store i32 1, ptr [[A]], align 4
+; LOWERTYPETESTS-NEXT:    br label [[IF_END]]
+; LOWERTYPETESTS:       if.end:
+; LOWERTYPETESTS-NEXT:    [[TMP1:%.*]] = load i32, ptr [[A]], align 4
+; LOWERTYPETESTS-NEXT:    [[TMP2:%.*]] = load i32, ptr [[B]], align 4
+; LOWERTYPETESTS-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP1]], [[TMP2]]
+; LOWERTYPETESTS-NEXT:    ret i32 [[ADD]]
+;
+;
+; LOWERTYPETESTS-LABEL: define dso_local i32 @g
+; LOWERTYPETESTS-SAME: (i32 noundef [[B:%.*]]) #[[ATTR0]] !type !2 !type !3 {
+; LOWERTYPETESTS-NEXT:  entry:
+; LOWERTYPETESTS-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
+; LOWERTYPETESTS-NEXT:    [[FP:%.*]] = alloca ptr, align 8
+; LOWERTYPETESTS-NEXT:    store i32 [[B]], ptr [[B_ADDR]], align 4
+; LOWERTYPETESTS-NEXT:    [[TMP0:%.*]] = load i32, ptr [[B_ADDR]], align 4
+; LOWERTYPETESTS-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[TMP0]], 0
+; LOWERTYPETESTS-NEXT:    [[TMP1:%.*]] = zext i1 [[TOBOOL]] to i64
+; LOWERTYPETESTS-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], ptr @.cfi.jumptable, ptr getelementptr inbounds ([2 x [8 x i8]], ptr @.cfi.jumptable, i64 0, i64 1)
+; LOWERTYPETESTS-NEXT:    store ptr [[COND]], ptr [[FP]], align 8
+; LOWERTYPETESTS-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[FP]], align 8
+; LOWERTYPETESTS-NEXT:    [[TMP3:%.*]] = ptrtoint ptr [[TMP2]] to i64
+; LOWERTYPETESTS-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP3]], ptrtoint (ptr @.cfi.jumptable to i64)
+; LOWERTYPETESTS-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP4]], 3
+; LOWERTYPETESTS-NEXT:    [[TMP6:%.*]] = shl i64 [[TMP4]], 61
+; LOWERTYPETESTS-NEXT:    [[TMP7:%.*]] = or i64 [[TMP5]], [[TMP6]]
+; LOWERTYPETESTS-NEXT:    [[TMP8:%.*]] = icmp ule i64 [[TMP7]], 1
+; LOWERTYPETESTS-NEXT:    br i1 [[TMP8]], label [[CONT:%.*]], label [[TRAP:%.*]], !nosanitize !4
+; LOWERTYPETESTS:       trap:
+; LOWERTYPETESTS-NEXT:    call void @llvm.ubsantrap(i8 2) #[[ATTR4:[0-9]+]], !nosanitize !4
+; LOWERTYPETESTS-NEXT:    unreachable, !nosanitize !4
+; LOWERTYPETESTS:       cont:
+; LOWERTYPETESTS-NEXT:    [[TMP9:%.*]] = load i32, ptr [[B_ADDR]], align 4
+; LOWERTYPETESTS-NEXT:    [[CALL:%.*]] = call i32 [[TMP2]](i32 noundef [[TMP9]])
+; LOWERTYPETESTS-NEXT:    ret i32 [[CALL]]
+;
+;
+; LOWERTYPETESTS-LABEL: define dso_local i32 @f_thunk
+; LOWERTYPETESTS-SAME: (i32 noundef [[TMP0:%.*]]) #[[ATTR0]] !type !2 {
+; LOWERTYPETESTS-NEXT:    [[TMP2:%.*]] = tail call i32 @f(i32 noundef [[TMP0]]) #[[ATTR0]]
+; LOWERTYPETESTS-NEXT:    ret i32 [[TMP2]]
+;
+;
+; LOWERTYPETESTS-LABEL: define private void @.cfi.jumptable
+; LOWERTYPETESTS-SAME: () #[[ATTR3:[0-9]+]] align 8 {
+; LOWERTYPETESTS-NEXT:  entry:
+; LOWERTYPETESTS-NEXT:    call void asm sideeffect "jmp ${0:c}@plt\0Aint3\0Aint3\0Aint3\0Ajmp ${1:c}@plt\0Aint3\0Aint3\0Aint3\0A", "s,s"(ptr @f, ptr @f_thunk)
+; LOWERTYPETESTS-NEXT:    unreachable
+;
diff --git a/llvm/test/Transforms/MergeFunc/mergefunc-preserve-debug-info.ll b/llvm/test/Transforms/MergeFunc/mergefunc-preserve-debug-info.ll
index 13d94f8cf6..72aa647ae0 100644
--- a/llvm/test/Transforms/MergeFunc/mergefunc-preserve-debug-info.ll
+++ b/llvm/test/Transforms/MergeFunc/mergefunc-preserve-debug-info.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; RUN: opt -passes='default<O0>,mergefunc' -S -mergefunc-preserve-debug-info < %s | FileCheck %s --check-prefix=OPTIMIZATION_LEVEL_0
 ; RUN: opt -passes='default<O2>,mergefunc' -S -mergefunc-preserve-debug-info < %s | FileCheck %s --check-prefix=OPTIMIZATION_LEVEL_2
 
@@ -43,6 +44,48 @@
 
 ; Function Attrs: nounwind uwtable
 define i32 @maxA(i32 %x, i32 %y) !dbg !6 {
+; OPTIMIZATION_LEVEL_0-LABEL: define i32 @maxA
+; OPTIMIZATION_LEVEL_0-SAME: (i32 [[X:%.*]], i32 [[Y:%.*]]) !dbg [[DBG6:![0-9]+]] {
+; OPTIMIZATION_LEVEL_0-NEXT:  entry:
+; OPTIMIZATION_LEVEL_0-NEXT:    [[X_ADDR:%.*]] = alloca i32, align 4
+; OPTIMIZATION_LEVEL_0-NEXT:    [[Y_ADDR:%.*]] = alloca i32, align 4
+; OPTIMIZATION_LEVEL_0-NEXT:    [[I:%.*]] = alloca i32, align 4
+; OPTIMIZATION_LEVEL_0-NEXT:    [[M:%.*]] = alloca i32, align 4
+; OPTIMIZATION_LEVEL_0-NEXT:    [[J:%.*]] = alloca i32, align 4
+; OPTIMIZATION_LEVEL_0-NEXT:    store i32 [[X]], ptr [[X_ADDR]], align 4
+; OPTIMIZATION_LEVEL_0-NEXT:    call void @llvm.dbg.declare(metadata ptr [[X_ADDR]], metadata [[META11:![0-9]+]], metadata !DIExpression()), !dbg [[DBG12:![0-9]+]]
+; OPTIMIZATION_LEVEL_0-NEXT:    store i32 [[Y]], ptr [[Y_ADDR]], align 4
+; OPTIMIZATION_LEVEL_0-NEXT:    call void @llvm.dbg.declare(metadata ptr [[Y_ADDR]], metadata [[META13:![0-9]+]], metadata !DIExpression()), !dbg [[DBG14:![0-9]+]]
+; OPTIMIZATION_LEVEL_0-NEXT:    call void @llvm.dbg.declare(metadata ptr [[I]], metadata [[META15:![0-9]+]], metadata !DIExpression()), !dbg [[DBG16:![0-9]+]]
+; OPTIMIZATION_LEVEL_0-NEXT:    call void @llvm.dbg.declare(metadata ptr [[M]], metadata [[META17:![0-9]+]], metadata !DIExpression()), !dbg [[DBG18:![0-9]+]]
+; OPTIMIZATION_LEVEL_0-NEXT:    call void @llvm.dbg.declare(metadata ptr [[J]], metadata [[META19:![0-9]+]], metadata !DIExpression()), !dbg [[DBG20:![0-9]+]]
+; OPTIMIZATION_LEVEL_0-NEXT:    [[TMP0:%.*]] = load i32, ptr [[X_ADDR]], align 4, !dbg [[DBG21:![0-9]+]]
+; OPTIMIZATION_LEVEL_0-NEXT:    [[TMP1:%.*]] = load i32, ptr [[Y_ADDR]], align 4, !dbg [[DBG23:![0-9]+]]
+; OPTIMIZATION_LEVEL_0-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP0]], [[TMP1]], !dbg [[DBG24:![0-9]+]]
+; OPTIMIZATION_LEVEL_0-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]], !dbg [[DBG25:![0-9]+]]
+; OPTIMIZATION_LEVEL_0:       if.then:
+; OPTIMIZATION_LEVEL_0-NEXT:    [[TMP2:%.*]] = load i32, ptr [[X_ADDR]], align 4, !dbg [[DBG26:![0-9]+]]
+; OPTIMIZATION_LEVEL_0-NEXT:    store i32 [[TMP2]], ptr [[M]], align 4, !dbg [[DBG27:![0-9]+]]
+; OPTIMIZATION_LEVEL_0-NEXT:    br label [[IF_END:%.*]], !dbg [[DBG28:![0-9]+]]
+; OPTIMIZATION_LEVEL_0:       if.else:
+; OPTIMIZATION_LEVEL_0-NEXT:    [[TMP3:%.*]] = load i32, ptr [[Y_ADDR]], align 4, !dbg [[DBG29:![0-9]+]]
+; OPTIMIZATION_LEVEL_0-NEXT:    store i32 [[TMP3]], ptr [[M]], align 4, !dbg [[DBG30:![0-9]+]]
+; OPTIMIZATION_LEVEL_0-NEXT:    br label [[IF_END]]
+; OPTIMIZATION_LEVEL_0:       if.end:
+; OPTIMIZATION_LEVEL_0-NEXT:    [[TMP4:%.*]] = load i32, ptr [[M]], align 4, !dbg [[DBG31:![0-9]+]]
+; OPTIMIZATION_LEVEL_0-NEXT:    ret i32 [[TMP4]], !dbg [[DBG32:![0-9]+]]
+;
+; OPTIMIZATION_LEVEL_2-LABEL: define i32 @maxA
+; OPTIMIZATION_LEVEL_2-SAME: (i32 [[X:%.*]], i32 [[Y:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] !dbg [[DBG6:![0-9]+]] {
+; OPTIMIZATION_LEVEL_2-NEXT:  entry:
+; OPTIMIZATION_LEVEL_2-NEXT:    call void @llvm.dbg.value(metadata i32 [[X]], metadata [[META11:![0-9]+]], metadata !DIExpression()), !dbg [[DBG12:![0-9]+]]
+; OPTIMIZATION_LEVEL_2-NEXT:    call void @llvm.dbg.value(metadata i32 [[Y]], metadata [[META13:![0-9]+]], metadata !DIExpression()), !dbg [[DBG12]]
+; OPTIMIZATION_LEVEL_2-NEXT:    call void @llvm.dbg.declare(metadata ptr undef, metadata [[META14:![0-9]+]], metadata !DIExpression()), !dbg [[DBG15:![0-9]+]]
+; OPTIMIZATION_LEVEL_2-NEXT:    call void @llvm.dbg.declare(metadata ptr undef, metadata [[META16:![0-9]+]], metadata !DIExpression()), !dbg [[DBG17:![0-9]+]]
+; OPTIMIZATION_LEVEL_2-NEXT:    [[X_Y:%.*]] = tail call i32 @llvm.smax.i32(i32 [[X]], i32 [[Y]])
+; OPTIMIZATION_LEVEL_2-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_Y]], metadata [[META18:![0-9]+]], metadata !DIExpression()), !dbg [[DBG12]]
+; OPTIMIZATION_LEVEL_2-NEXT:    ret i32 [[X_Y]], !dbg [[DBG19:![0-9]+]]
+;
 entry:
   %x.addr = alloca i32, align 4
   %y.addr = alloca i32, align 4
@@ -81,26 +124,27 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata)
 
 ; Function Attrs: nounwind uwtable
 define i32 @maxB(i32 %x, i32 %y) !dbg !34 {
+; OPTIMIZATION_LEVEL_0-LABEL: define i32 @maxB
+; OPTIMIZATION_LEVEL_0-SAME: (i32 [[X:%.*]], i32 [[Y:%.*]]) !dbg [[DBG33:![0-9]+]] {
+; OPTIMIZATION_LEVEL_0-NEXT:  entry:
+; OPTIMIZATION_LEVEL_0-NEXT:    [[X_ADDR:%.*]] = alloca i32, align 4
+; OPTIMIZATION_LEVEL_0-NEXT:    [[Y_ADDR:%.*]] = alloca i32, align 4
+; OPTIMIZATION_LEVEL_0-NEXT:    store i32 [[X]], ptr [[X_ADDR]], align 4
+; OPTIMIZATION_LEVEL_0-NEXT:    call void @llvm.dbg.declare(metadata ptr [[X_ADDR]], metadata [[META34:![0-9]+]], metadata !DIExpression()), !dbg [[DBG35:![0-9]+]]
+; OPTIMIZATION_LEVEL_0-NEXT:    store i32 [[Y]], ptr [[Y_ADDR]], align 4
+; OPTIMIZATION_LEVEL_0-NEXT:    call void @llvm.dbg.declare(metadata ptr [[Y_ADDR]], metadata [[META36:![0-9]+]], metadata !DIExpression()), !dbg [[DBG37:![0-9]+]]
+; OPTIMIZATION_LEVEL_0-NEXT:    [[TMP0:%.*]] = tail call i32 @maxA(i32 [[X]], i32 [[Y]]), !dbg [[DBG38:![0-9]+]]
+; OPTIMIZATION_LEVEL_0-NEXT:    ret i32 [[TMP0]], !dbg [[DBG38]]
+;
+; OPTIMIZATION_LEVEL_2-LABEL: define i32 @maxB
+; OPTIMIZATION_LEVEL_2-SAME: (i32 [[X:%.*]], i32 [[Y:%.*]]) local_unnamed_addr #[[ATTR0]] !dbg [[DBG20:![0-9]+]] {
+; OPTIMIZATION_LEVEL_2-NEXT:  entry:
+; OPTIMIZATION_LEVEL_2-NEXT:    call void @llvm.dbg.value(metadata i32 [[X]], metadata [[META21:![0-9]+]], metadata !DIExpression()), !dbg [[DBG22:![0-9]+]]
+; OPTIMIZATION_LEVEL_2-NEXT:    call void @llvm.dbg.value(metadata i32 [[Y]], metadata [[META23:![0-9]+]], metadata !DIExpression()), !dbg [[DBG22]]
+; OPTIMIZATION_LEVEL_2-NEXT:    [[TMP0:%.*]] = tail call i32 @maxA(i32 [[X]], i32 [[Y]]) #[[ATTR0]], !dbg [[DBG24:![0-9]+]]
+; OPTIMIZATION_LEVEL_2-NEXT:    ret i32 [[TMP0]], !dbg [[DBG24]]
+;
 
-; OPTIMIZATION_LEVEL_0: define i32 @maxB(i32 %x, i32 %y)
-; OPTIMIZATION_LEVEL_0-NEXT: entry:
-; OPTIMIZATION_LEVEL_0-NEXT: %x.addr = alloca i32, align 4
-; OPTIMIZATION_LEVEL_0-NEXT: %y.addr = alloca i32, align 4
-; OPTIMIZATION_LEVEL_0-NEXT: store i32 %x, ptr %x.addr, align 4
-; OPTIMIZATION_LEVEL_0-NEXT: call void @llvm.dbg.declare(metadata ptr %x.addr, metadata !{{[0-9]+}}, metadata !DIExpression()), !dbg !{{[0-9]+}}
-; OPTIMIZATION_LEVEL_0-NEXT: store i32 %y, ptr %y.addr, align 4
-; OPTIMIZATION_LEVEL_0-NEXT: call void @llvm.dbg.declare(metadata ptr %y.addr, metadata !{{[0-9]+}}, metadata !DIExpression()), !dbg !{{[0-9]+}}
-; OPTIMIZATION_LEVEL_0-NEXT: %0 = tail call i32 @maxA(i32 %x, i32 %y), !dbg !{{[0-9]+}}
-; OPTIMIZATION_LEVEL_0-NEXT: ret i32 %0, !dbg !{{[0-9]+}}
-; OPTIMIZATION_LEVEL_0-NEXT: }
-
-; OPTIMIZATION_LEVEL_2: define i32 @maxB(i32 %x, i32 %y)
-; OPTIMIZATION_LEVEL_2-NEXT: entry:
-; OPTIMIZATION_LEVEL_2-NEXT: call void @llvm.dbg.value(metadata i32 %x, metadata !{{[0-9]+}}, metadata !DIExpression()), !dbg !{{[0-9]+}}
-; OPTIMIZATION_LEVEL_2-NEXT: call void @llvm.dbg.value(metadata i32 %y, metadata !{{[0-9]+}}, metadata !DIExpression()), !dbg !{{[0-9]+}}
-; OPTIMIZATION_LEVEL_2-NEXT: %0 = tail call i32 @maxA(i32 %x, i32 %y) #{{[0-9]+}}, !dbg !{{[0-9]+}}
-; OPTIMIZATION_LEVEL_2-NEXT: ret i32 %0, !dbg !{{[0-9]+}}
-; OPTIMIZATION_LEVEL_2-NEXT: }
 
 entry:
   %x.addr = alloca i32, align 4
@@ -137,18 +181,19 @@ if.end:                                           ; preds = %if.else, %if.then
 
 ; Function Attrs: nounwind uwtable
 define void @f() !dbg !57 {
+; OPTIMIZATION_LEVEL_0-LABEL: define void @f
+; OPTIMIZATION_LEVEL_0-SAME: () !dbg [[DBG39:![0-9]+]] {
+; OPTIMIZATION_LEVEL_0-NEXT:  entry:
+; OPTIMIZATION_LEVEL_0-NEXT:    [[CALL:%.*]] = call i32 @maxA(i32 3, i32 4), !dbg [[DBG42:![0-9]+]]
+; OPTIMIZATION_LEVEL_0-NEXT:    [[CALL1:%.*]] = call i32 @maxB(i32 1, i32 9), !dbg [[DBG43:![0-9]+]]
+; OPTIMIZATION_LEVEL_0-NEXT:    ret void, !dbg [[DBG44:![0-9]+]]
+;
+; OPTIMIZATION_LEVEL_2-LABEL: define void @f
+; OPTIMIZATION_LEVEL_2-SAME: () local_unnamed_addr #[[ATTR2:[0-9]+]] !dbg [[DBG25:![0-9]+]] {
+; OPTIMIZATION_LEVEL_2-NEXT:  entry:
+; OPTIMIZATION_LEVEL_2-NEXT:    ret void, !dbg [[DBG28:![0-9]+]]
+;
 entry:
-
-; OPTIMIZATION_LEVEL_0: define void @f()
-; OPTIMIZATION_LEVEL_0-NEXT: entry:
-; OPTIMIZATION_LEVEL_0-NEXT: %call = call i32 @maxA(i32 3, i32 4), !dbg !{{[0-9]+}}
-; OPTIMIZATION_LEVEL_0-NEXT: %call1 = call i32 @maxB(i32 1, i32 9), !dbg !{{[0-9]+}}
-; OPTIMIZATION_LEVEL_0-NEXT: ret void, !dbg !{{[0-9]+}}
-
-; OPTIMIZATION_LEVEL_2: define void @f()
-; OPTIMIZATION_LEVEL_2-NEXT: entry:
-; OPTIMIZATION_LEVEL_2-NEXT: ret void, !dbg !{{[0-9]+}}
-
   %call = call i32 @maxA(i32 3, i32 4), !dbg !60
   %call1 = call i32 @maxB(i32 1, i32 9), !dbg !61
   ret void, !dbg !62
diff --git a/llvm/test/Transforms/MergeFunc/mergefunc-preserve-vfe-intrinsics.ll b/llvm/test/Transforms/MergeFunc/mergefunc-preserve-vfe-intrinsics.ll
new file mode 100644
index 0000000000..1c29aec8d8
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/mergefunc-preserve-vfe-intrinsics.ll
@@ -0,0 +1,44 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
+; RUN: opt -passes=mergefunc -S %s | FileCheck %s
+
+; This test contains three identical functions, aside from the metadata
+; they pass to a function call. This test verifies that the function merger
+; pass is able to merge the two functions that are truly identical,
+; but the third that passes different metadata is preserved
+
+declare { ptr, i1 } @llvm.type.checked.load(ptr, i32, metadata)
+
+define i1 @merge_candidate_a(ptr %ptr, i32 %offset) {
+; CHECK-LABEL: define i1 @merge_candidate_a(
+; CHECK-SAME: ptr [[PTR:%.*]], i32 [[OFFSET:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call { ptr, i1 } @llvm.type.checked.load(ptr [[PTR]], i32 [[OFFSET]], metadata !"common_metadata")
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { ptr, i1 } [[TMP1]], 1
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %1 = call { ptr, i1 } @llvm.type.checked.load(ptr %ptr, i32 %offset, metadata !"common_metadata")
+  %2 = extractvalue { ptr, i1 } %1, 1
+  ret i1 %2
+}
+
+define i1 @merge_candidate_c(ptr %ptr, i32 %offset) {
+; CHECK-LABEL: define i1 @merge_candidate_c(
+; CHECK-SAME: ptr [[PTR:%.*]], i32 [[OFFSET:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call { ptr, i1 } @llvm.type.checked.load(ptr [[PTR]], i32 [[OFFSET]], metadata !"different_metadata")
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { ptr, i1 } [[TMP1]], 1
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %1 = call { ptr, i1 } @llvm.type.checked.load(ptr %ptr, i32 %offset, metadata !"different_metadata")
+  %2 = extractvalue { ptr, i1 } %1, 1
+  ret i1 %2
+}
+
+define i1 @merge_candidate_b(ptr %ptr, i32 %offset) {
+; CHECK-LABEL: define i1 @merge_candidate_b(
+; CHECK-SAME: ptr [[TMP0:%.*]], i32 [[TMP1:%.*]]) {
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call i1 @merge_candidate_a(ptr [[TMP0]], i32 [[TMP1]])
+; CHECK-NEXT:    ret i1 [[TMP3]]
+;
+  %1 = call { ptr, i1 } @llvm.type.checked.load(ptr %ptr, i32 %offset, metadata !"common_metadata")
+  %2 = extractvalue { ptr, i1 } %1, 1
+  ret i1 %2
+}
diff --git a/llvm/test/Transforms/MergeFunc/no-merge-debug-thunks.ll b/llvm/test/Transforms/MergeFunc/no-merge-debug-thunks.ll
new file mode 100644
index 0000000000..80467c7aee
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/no-merge-debug-thunks.ll
@@ -0,0 +1,61 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+;; Make sure debug instructions are not counted when deciding to merge functions
+; RUN: opt -S -passes=mergefunc < %s | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+; Function Attrs: nounwind readnone
+define hidden i32 @f(i32 %t) {
+; CHECK-LABEL: define hidden i32 @f
+; CHECK-SAME: (i32 [[T:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i32 [[T]], metadata [[META6:![0-9]+]], metadata !DIExpression()), !dbg [[DBG12:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i32 [[T]], metadata [[META6]], metadata !DIExpression()), !dbg [[DBG12]]
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  call void @llvm.dbg.value(metadata i32 %t, metadata !12, metadata !DIExpression()), !dbg !13
+  call void @llvm.dbg.value(metadata i32 %t, metadata !12, metadata !DIExpression()), !dbg !13
+  ret i32 0
+}
+
+; Function Attrs: nounwind readnone
+define hidden i32 @f_thunk(i32 %t) {
+; CHECK-LABEL: define hidden i32 @f_thunk
+; CHECK-SAME: (i32 [[T:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i32 [[T]], metadata [[META6]], metadata !DIExpression()), !dbg [[DBG12]]
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i32 [[T]], metadata [[META6]], metadata !DIExpression()), !dbg [[DBG12]]
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  call void @llvm.dbg.value(metadata i32 %t, metadata !12, metadata !DIExpression()), !dbg !13
+  call void @llvm.dbg.value(metadata i32 %t, metadata !12, metadata !DIExpression()), !dbg !13
+  ret i32 0
+}
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "no-merge-debug-thunks.c", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!7 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 3, type: !8, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !11)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10, !10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !{!12}
+!12 = !DILocalVariable(name: "t", arg: 1, scope: !7, file: !1, line: 3, type: !10)
+!13 = !DILocation(line: 3, column: 14, scope: !7)
+!14 = !DILocation(line: 4, column: 12, scope: !7)
+!16 = distinct !DISubprogram(name: "_start", scope: !1, file: !1, line: 7, type: !17, isLocal: false, isDefinition: true, scopeLine: 7, isOptimized: true, unit: !0, retainedNodes: !2)
+!17 = !DISubroutineType(types: !18)
+!18 = !{!10}
+!19 = !DILocation(line: 8, column: 3, scope: !16)
+!20 = !DILocation(line: 9, column: 3, scope: !16)
diff --git a/llvm/test/Transforms/OpenMP/parallel_level_fold.ll b/llvm/test/Transforms/OpenMP/parallel_level_fold.ll
index ac8a49ddfb..eb75d030a0 100644
--- a/llvm/test/Transforms/OpenMP/parallel_level_fold.ll
+++ b/llvm/test/Transforms/OpenMP/parallel_level_fold.ll
@@ -14,7 +14,7 @@ target triple = "nvptx64"
 ; CHECK: @[[G:[a-zA-Z0-9_$"\\.-]+]] = external global i16
 ; CHECK: @[[NONE_SPMD_KERNEL_ENVIRONMENT:[a-zA-Z0-9_$"\\.-]+]] = local_unnamed_addr constant [[STRUCT_KERNELENVIRONMENTTY:%.*]] { [[STRUCT_CONFIGURATIONENVIRONMENTTY:%.*]] { i8 0, i8 0, i8 1 }, ptr null, ptr null }
 ; CHECK: @[[SPMD_KERNEL_ENVIRONMENT:[a-zA-Z0-9_$"\\.-]+]] = local_unnamed_addr constant [[STRUCT_KERNELENVIRONMENTTY:%.*]] { [[STRUCT_CONFIGURATIONENVIRONMENTTY:%.*]] { i8 0, i8 0, i8 2 }, ptr null, ptr null }
-; CHECK: @[[PARALLEL_KERNEL_ENVIRONMENT:[a-zA-Z0-9_$"\\.-]+]] = local_unnamed_addr constant [[STRUCT_KERNELENVIRONMENTTY:%.*]] { [[STRUCT_CONFIGURATIONENVIRONMENTTY:%.*]] { i8 0, i8 0, i8 2 }, ptr null, ptr null }
+; CHECK: @[[PARALLEL_KERNEL_ENVIRONMENT:[a-zA-Z0-9_$"\\.-]+]] = local_unnamed_addr constant [[STRUCT_KERNELENVIRONMENTTY:%.*]] { [[STRUCT_CONFIGURATIONENVIRONMENTTY:%.*]] { i8 0, i8 1, i8 2 }, ptr null, ptr null }
 ;.
 define weak void @none_spmd() "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@none_spmd
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/pixel-splat.ll b/llvm/test/Transforms/PhaseOrdering/X86/pixel-splat.ll
index f484b04a90..e169f2570c 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/pixel-splat.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/pixel-splat.ll
@@ -34,21 +34,21 @@ define void @loop_or(ptr noalias %pIn, ptr noalias %pOut, i32 %s) {
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[PIN:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 4
-; CHECK-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x i8>, ptr [[TMP2]], align 1
-; CHECK-NEXT:    [[TMP4:%.*]] = zext <4 x i8> [[WIDE_LOAD]] to <4 x i32>
-; CHECK-NEXT:    [[TMP5:%.*]] = zext <4 x i8> [[WIDE_LOAD4]] to <4 x i32>
-; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw nsw <4 x i32> [[TMP4]], <i32 65793, i32 65793, i32 65793, i32 65793>
-; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw nsw <4 x i32> [[TMP5]], <i32 65793, i32 65793, i32 65793, i32 65793>
-; CHECK-NEXT:    [[TMP8:%.*]] = or <4 x i32> [[TMP6]], <i32 -16777216, i32 -16777216, i32 -16777216, i32 -16777216>
-; CHECK-NEXT:    [[TMP9:%.*]] = or <4 x i32> [[TMP7]], <i32 -16777216, i32 -16777216, i32 -16777216, i32 -16777216>
-; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[POUT:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    store <4 x i32> [[TMP8]], ptr [[TMP10]], align 4
-; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP10]], i64 4
-; CHECK-NEXT:    store <4 x i32> [[TMP9]], ptr [[TMP12]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 4
+; CHECK-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = zext <4 x i8> [[WIDE_LOAD]] to <4 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = zext <4 x i8> [[WIDE_LOAD4]] to <4 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw nsw <4 x i32> [[TMP2]], <i32 65793, i32 65793, i32 65793, i32 65793>
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw nsw <4 x i32> [[TMP3]], <i32 65793, i32 65793, i32 65793, i32 65793>
+; CHECK-NEXT:    [[TMP6:%.*]] = or <4 x i32> [[TMP4]], <i32 -16777216, i32 -16777216, i32 -16777216, i32 -16777216>
+; CHECK-NEXT:    [[TMP7:%.*]] = or <4 x i32> [[TMP5]], <i32 -16777216, i32 -16777216, i32 -16777216, i32 -16777216>
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[POUT:%.*]], i64 [[INDEX]]
+; CHECK-NEXT:    store <4 x i32> [[TMP6]], ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP8]], i64 4
+; CHECK-NEXT:    store <4 x i32> [[TMP7]], ptr [[TMP9]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[WIDE_TRIP_COUNT]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END]], label [[FOR_BODY_PREHEADER5]]
@@ -58,15 +58,15 @@ define void @loop_or(ptr noalias %pIn, ptr noalias %pOut, i32 %s) {
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[INDVARS_IV_PH]], [[FOR_BODY_PREHEADER5]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[PIN]], i64 [[INDVARS_IV]]
-; CHECK-NEXT:    [[TMP15:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
-; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[TMP15]] to i32
+; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[TMP11]] to i32
 ; CHECK-NEXT:    [[OR2:%.*]] = mul nuw nsw i32 [[CONV]], 65793
 ; CHECK-NEXT:    [[OR3:%.*]] = or i32 [[OR2]], -16777216
 ; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, ptr [[POUT]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    store i32 [[OR3]], ptr [[ARRAYIDX5]], align 4
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/pr50555.ll b/llvm/test/Transforms/PhaseOrdering/X86/pr50555.ll
index 7b7658279a..18d1c64268 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/pr50555.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/pr50555.ll
@@ -4,29 +4,29 @@
 
 define void @trunc_through_one_add(ptr noalias %0, ptr noalias readonly %1) {
 ; SSE-LABEL: @trunc_through_one_add(
-; SSE-NEXT:    [[TMP4:%.*]] = load <8 x i8>, ptr [[TMP1:%.*]], align 1
-; SSE-NEXT:    [[TMP5:%.*]] = zext <8 x i8> [[TMP4]] to <8 x i16>
-; SSE-NEXT:    [[TMP6:%.*]] = lshr <8 x i16> [[TMP5]], <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
-; SSE-NEXT:    [[TMP7:%.*]] = add nuw nsw <8 x i16> [[TMP6]], [[TMP5]]
-; SSE-NEXT:    [[TMP8:%.*]] = lshr <8 x i16> [[TMP7]], <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
-; SSE-NEXT:    store <8 x i16> [[TMP8]], ptr [[TMP0:%.*]], align 2
-; SSE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 8
-; SSE-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i16, ptr [[TMP0]], i64 8
-; SSE-NEXT:    [[TMP13:%.*]] = load <8 x i8>, ptr [[TMP10]], align 1
-; SSE-NEXT:    [[TMP14:%.*]] = zext <8 x i8> [[TMP13]] to <8 x i16>
-; SSE-NEXT:    [[TMP15:%.*]] = lshr <8 x i16> [[TMP14]], <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
-; SSE-NEXT:    [[TMP16:%.*]] = add nuw nsw <8 x i16> [[TMP15]], [[TMP14]]
-; SSE-NEXT:    [[TMP17:%.*]] = lshr <8 x i16> [[TMP16]], <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
-; SSE-NEXT:    store <8 x i16> [[TMP17]], ptr [[TMP11]], align 2
+; SSE-NEXT:    [[TMP3:%.*]] = load <8 x i8>, ptr [[TMP1:%.*]], align 1
+; SSE-NEXT:    [[TMP4:%.*]] = zext <8 x i8> [[TMP3]] to <8 x i16>
+; SSE-NEXT:    [[TMP5:%.*]] = lshr <8 x i16> [[TMP4]], <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+; SSE-NEXT:    [[TMP6:%.*]] = add nuw nsw <8 x i16> [[TMP5]], [[TMP4]]
+; SSE-NEXT:    [[TMP7:%.*]] = lshr <8 x i16> [[TMP6]], <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+; SSE-NEXT:    store <8 x i16> [[TMP7]], ptr [[TMP0:%.*]], align 2
+; SSE-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 8
+; SSE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i16, ptr [[TMP0]], i64 8
+; SSE-NEXT:    [[TMP10:%.*]] = load <8 x i8>, ptr [[TMP8]], align 1
+; SSE-NEXT:    [[TMP11:%.*]] = zext <8 x i8> [[TMP10]] to <8 x i16>
+; SSE-NEXT:    [[TMP12:%.*]] = lshr <8 x i16> [[TMP11]], <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+; SSE-NEXT:    [[TMP13:%.*]] = add nuw nsw <8 x i16> [[TMP12]], [[TMP11]]
+; SSE-NEXT:    [[TMP14:%.*]] = lshr <8 x i16> [[TMP13]], <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+; SSE-NEXT:    store <8 x i16> [[TMP14]], ptr [[TMP9]], align 2
 ; SSE-NEXT:    ret void
 ;
 ; AVX-LABEL: @trunc_through_one_add(
-; AVX-NEXT:    [[TMP4:%.*]] = load <16 x i8>, ptr [[TMP1:%.*]], align 1
-; AVX-NEXT:    [[TMP5:%.*]] = zext <16 x i8> [[TMP4]] to <16 x i16>
-; AVX-NEXT:    [[TMP6:%.*]] = lshr <16 x i16> [[TMP5]], <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
-; AVX-NEXT:    [[TMP7:%.*]] = add nuw nsw <16 x i16> [[TMP6]], [[TMP5]]
-; AVX-NEXT:    [[TMP8:%.*]] = lshr <16 x i16> [[TMP7]], <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
-; AVX-NEXT:    store <16 x i16> [[TMP8]], ptr [[TMP0:%.*]], align 2
+; AVX-NEXT:    [[TMP3:%.*]] = load <16 x i8>, ptr [[TMP1:%.*]], align 1
+; AVX-NEXT:    [[TMP4:%.*]] = zext <16 x i8> [[TMP3]] to <16 x i16>
+; AVX-NEXT:    [[TMP5:%.*]] = lshr <16 x i16> [[TMP4]], <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+; AVX-NEXT:    [[TMP6:%.*]] = add nuw nsw <16 x i16> [[TMP5]], [[TMP4]]
+; AVX-NEXT:    [[TMP7:%.*]] = lshr <16 x i16> [[TMP6]], <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+; AVX-NEXT:    store <16 x i16> [[TMP7]], ptr [[TMP0:%.*]], align 2
 ; AVX-NEXT:    ret void
 ;
   %3 = load i8, ptr %1, align 1
@@ -176,39 +176,39 @@ define void @trunc_through_one_add(ptr noalias %0, ptr noalias readonly %1) {
 
 define void @trunc_through_two_adds(ptr noalias %0, ptr noalias readonly %1, ptr noalias readonly %2) {
 ; SSE-LABEL: @trunc_through_two_adds(
-; SSE-NEXT:    [[TMP5:%.*]] = load <8 x i8>, ptr [[TMP1:%.*]], align 1
-; SSE-NEXT:    [[TMP6:%.*]] = zext <8 x i8> [[TMP5]] to <8 x i16>
-; SSE-NEXT:    [[TMP8:%.*]] = load <8 x i8>, ptr [[TMP2:%.*]], align 1
-; SSE-NEXT:    [[TMP9:%.*]] = zext <8 x i8> [[TMP8]] to <8 x i16>
-; SSE-NEXT:    [[TMP10:%.*]] = add nuw nsw <8 x i16> [[TMP9]], [[TMP6]]
-; SSE-NEXT:    [[TMP11:%.*]] = lshr <8 x i16> [[TMP10]], <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
-; SSE-NEXT:    [[TMP12:%.*]] = add nuw nsw <8 x i16> [[TMP11]], [[TMP10]]
-; SSE-NEXT:    [[TMP13:%.*]] = lshr <8 x i16> [[TMP12]], <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
-; SSE-NEXT:    store <8 x i16> [[TMP13]], ptr [[TMP0:%.*]], align 2
-; SSE-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 8
-; SSE-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 8
-; SSE-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i16, ptr [[TMP0]], i64 8
-; SSE-NEXT:    [[TMP19:%.*]] = load <8 x i8>, ptr [[TMP15]], align 1
-; SSE-NEXT:    [[TMP20:%.*]] = zext <8 x i8> [[TMP19]] to <8 x i16>
-; SSE-NEXT:    [[TMP22:%.*]] = load <8 x i8>, ptr [[TMP16]], align 1
-; SSE-NEXT:    [[TMP23:%.*]] = zext <8 x i8> [[TMP22]] to <8 x i16>
-; SSE-NEXT:    [[TMP24:%.*]] = add nuw nsw <8 x i16> [[TMP23]], [[TMP20]]
-; SSE-NEXT:    [[TMP25:%.*]] = lshr <8 x i16> [[TMP24]], <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
-; SSE-NEXT:    [[TMP26:%.*]] = add nuw nsw <8 x i16> [[TMP25]], [[TMP24]]
-; SSE-NEXT:    [[TMP27:%.*]] = lshr <8 x i16> [[TMP26]], <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
-; SSE-NEXT:    store <8 x i16> [[TMP27]], ptr [[TMP17]], align 2
+; SSE-NEXT:    [[TMP4:%.*]] = load <8 x i8>, ptr [[TMP1:%.*]], align 1
+; SSE-NEXT:    [[TMP5:%.*]] = zext <8 x i8> [[TMP4]] to <8 x i16>
+; SSE-NEXT:    [[TMP6:%.*]] = load <8 x i8>, ptr [[TMP2:%.*]], align 1
+; SSE-NEXT:    [[TMP7:%.*]] = zext <8 x i8> [[TMP6]] to <8 x i16>
+; SSE-NEXT:    [[TMP8:%.*]] = add nuw nsw <8 x i16> [[TMP7]], [[TMP5]]
+; SSE-NEXT:    [[TMP9:%.*]] = lshr <8 x i16> [[TMP8]], <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+; SSE-NEXT:    [[TMP10:%.*]] = add nuw nsw <8 x i16> [[TMP9]], [[TMP8]]
+; SSE-NEXT:    [[TMP11:%.*]] = lshr <8 x i16> [[TMP10]], <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+; SSE-NEXT:    store <8 x i16> [[TMP11]], ptr [[TMP0:%.*]], align 2
+; SSE-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 8
+; SSE-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 8
+; SSE-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i16, ptr [[TMP0]], i64 8
+; SSE-NEXT:    [[TMP15:%.*]] = load <8 x i8>, ptr [[TMP12]], align 1
+; SSE-NEXT:    [[TMP16:%.*]] = zext <8 x i8> [[TMP15]] to <8 x i16>
+; SSE-NEXT:    [[TMP17:%.*]] = load <8 x i8>, ptr [[TMP13]], align 1
+; SSE-NEXT:    [[TMP18:%.*]] = zext <8 x i8> [[TMP17]] to <8 x i16>
+; SSE-NEXT:    [[TMP19:%.*]] = add nuw nsw <8 x i16> [[TMP18]], [[TMP16]]
+; SSE-NEXT:    [[TMP20:%.*]] = lshr <8 x i16> [[TMP19]], <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+; SSE-NEXT:    [[TMP21:%.*]] = add nuw nsw <8 x i16> [[TMP20]], [[TMP19]]
+; SSE-NEXT:    [[TMP22:%.*]] = lshr <8 x i16> [[TMP21]], <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+; SSE-NEXT:    store <8 x i16> [[TMP22]], ptr [[TMP14]], align 2
 ; SSE-NEXT:    ret void
 ;
 ; AVX-LABEL: @trunc_through_two_adds(
-; AVX-NEXT:    [[TMP5:%.*]] = load <16 x i8>, ptr [[TMP1:%.*]], align 1
-; AVX-NEXT:    [[TMP6:%.*]] = zext <16 x i8> [[TMP5]] to <16 x i16>
-; AVX-NEXT:    [[TMP8:%.*]] = load <16 x i8>, ptr [[TMP2:%.*]], align 1
-; AVX-NEXT:    [[TMP9:%.*]] = zext <16 x i8> [[TMP8]] to <16 x i16>
-; AVX-NEXT:    [[TMP10:%.*]] = add nuw nsw <16 x i16> [[TMP9]], [[TMP6]]
-; AVX-NEXT:    [[TMP11:%.*]] = lshr <16 x i16> [[TMP10]], <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
-; AVX-NEXT:    [[TMP12:%.*]] = add nuw nsw <16 x i16> [[TMP11]], [[TMP10]]
-; AVX-NEXT:    [[TMP13:%.*]] = lshr <16 x i16> [[TMP12]], <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
-; AVX-NEXT:    store <16 x i16> [[TMP13]], ptr [[TMP0:%.*]], align 2
+; AVX-NEXT:    [[TMP4:%.*]] = load <16 x i8>, ptr [[TMP1:%.*]], align 1
+; AVX-NEXT:    [[TMP5:%.*]] = zext <16 x i8> [[TMP4]] to <16 x i16>
+; AVX-NEXT:    [[TMP6:%.*]] = load <16 x i8>, ptr [[TMP2:%.*]], align 1
+; AVX-NEXT:    [[TMP7:%.*]] = zext <16 x i8> [[TMP6]] to <16 x i16>
+; AVX-NEXT:    [[TMP8:%.*]] = add nuw nsw <16 x i16> [[TMP7]], [[TMP5]]
+; AVX-NEXT:    [[TMP9:%.*]] = lshr <16 x i16> [[TMP8]], <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+; AVX-NEXT:    [[TMP10:%.*]] = add nuw nsw <16 x i16> [[TMP9]], [[TMP8]]
+; AVX-NEXT:    [[TMP11:%.*]] = lshr <16 x i16> [[TMP10]], <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
+; AVX-NEXT:    store <16 x i16> [[TMP11]], ptr [[TMP0:%.*]], align 2
 ; AVX-NEXT:    ret void
 ;
   %4 = load i8, ptr %1, align 1
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/speculation-vs-tbaa.ll b/llvm/test/Transforms/PhaseOrdering/X86/speculation-vs-tbaa.ll
index f691f8dfae..f1ddee00ca 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/speculation-vs-tbaa.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/speculation-vs-tbaa.ll
@@ -41,11 +41,11 @@ define void @licm(ptr align 8 dereferenceable(8) %_M_start.i, i64 %numElem) {
 ; O23-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; O23-NEXT:    [[TMP1:%.*]] = getelementptr inbounds double, ptr [[TMP0]], i64 [[INDEX]]
 ; O23-NEXT:    store <2 x double> <double 2.000000e+00, double 2.000000e+00>, ptr [[TMP1]], align 8, !tbaa [[TBAA8:![0-9]+]]
-; O23-NEXT:    [[TMP3:%.*]] = getelementptr inbounds double, ptr [[TMP1]], i64 2
-; O23-NEXT:    store <2 x double> <double 2.000000e+00, double 2.000000e+00>, ptr [[TMP3]], align 8, !tbaa [[TBAA8]]
+; O23-NEXT:    [[TMP2:%.*]] = getelementptr inbounds double, ptr [[TMP1]], i64 2
+; O23-NEXT:    store <2 x double> <double 2.000000e+00, double 2.000000e+00>, ptr [[TMP2]], align 8, !tbaa [[TBAA8]]
 ; O23-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; O23-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; O23-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; O23-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; O23-NEXT:    br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; O23:       middle.block:
 ; O23-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[NUMELEM]]
 ; O23-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY_PREHEADER]]
@@ -58,7 +58,7 @@ define void @licm(ptr align 8 dereferenceable(8) %_M_start.i, i64 %numElem) {
 ; O23-NEXT:    store double 2.000000e+00, ptr [[ADD_PTR_I]], align 8, !tbaa [[TBAA8]]
 ; O23-NEXT:    [[INC]] = add nuw i64 [[K_02]], 1
 ; O23-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[NUMELEM]]
-; O23-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; O23-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
 ; O23:       for.cond.cleanup:
 ; O23-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/PhaseOrdering/icmp-ashr-breaking-select-idiom.ll b/llvm/test/Transforms/PhaseOrdering/icmp-ashr-breaking-select-idiom.ll
new file mode 100644
index 0000000000..8559f973f2
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/icmp-ashr-breaking-select-idiom.ll
@@ -0,0 +1,38 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
+; RUN: opt -O1 -S < %s  | FileCheck %s
+
+define i32 @testa(i32 %mul) {
+; CHECK-LABEL: define i32 @testa(
+; CHECK-SAME: i32 [[MUL:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 [[MUL]], 15
+; CHECK-NEXT:    [[CMP4_I:%.*]] = icmp slt i32 [[MUL]], 1073741824
+; CHECK-NEXT:    [[SPEC_SELECT_I:%.*]] = select i1 [[CMP4_I]], i32 [[SHR]], i32 32767
+; CHECK-NEXT:    ret i32 [[SPEC_SELECT_I]]
+;
+  %shr = ashr i32 %mul, 15
+  %cmp4.i = icmp sgt i32 %shr, 32767
+  %switch.i = icmp ult i1 %cmp4.i, true
+  %spec.select.i = select i1 %switch.i, i32 %shr, i32 32767
+  ret i32 %spec.select.i
+}
+
+define i32 @testb(i32 %mul) {
+; CHECK-LABEL: define i32 @testb(
+; CHECK-SAME: i32 [[MUL:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; CHECK-NEXT:    [[SHR102:%.*]] = ashr i32 [[MUL]], 7
+; CHECK-NEXT:    [[CMP4_I:%.*]] = icmp sgt i32 [[MUL]], 16383
+; CHECK-NEXT:    [[RETVAL_0_I:%.*]] = select i1 [[CMP4_I]], i32 127, i32 -128
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[MUL]], 16384
+; CHECK-NEXT:    [[CLEANUP_DEST_SLOT_0_I:%.*]] = icmp ult i32 [[TMP1]], 32768
+; CHECK-NEXT:    [[SPEC_SELECT_I:%.*]] = select i1 [[CLEANUP_DEST_SLOT_0_I]], i32 [[SHR102]], i32 [[RETVAL_0_I]]
+; CHECK-NEXT:    ret i32 [[SPEC_SELECT_I]]
+;
+  %shr102 = ashr i32 %mul, 7
+  %cmp4.i = icmp sgt i32 %shr102, 127
+  %cmp6.i = icmp slt i32 %shr102, -128
+  %retval.0.i = select i1 %cmp4.i, i32 127, i32 -128
+  %cleanup.dest.slot.0.i = select i1 %cmp4.i, i1 true, i1 %cmp6.i
+  %switch.i = icmp ult i1 %cleanup.dest.slot.0.i, true
+  %spec.select.i = select i1 %switch.i, i32 %shr102, i32 %retval.0.i
+  ret i32 %spec.select.i
+}
diff --git a/llvm/test/Transforms/PhaseOrdering/pr39282.ll b/llvm/test/Transforms/PhaseOrdering/pr39282.ll
index 62d96ecdd6..776c5c482b 100644
--- a/llvm/test/Transforms/PhaseOrdering/pr39282.ll
+++ b/llvm/test/Transforms/PhaseOrdering/pr39282.ll
@@ -17,22 +17,22 @@ define void @copy(ptr noalias %to, ptr noalias %from) {
 define void @pr39282(ptr %addr1, ptr %addr2) {
 ; CHECK-LABEL: @pr39282(
 ; CHECK-NEXT:  start:
-; CHECK-NEXT:    call void @llvm.experimental.noalias.scope.decl
-; CHECK-NEXT:    call void @llvm.experimental.noalias.scope.decl
+; CHECK-NEXT:    tail call void @llvm.experimental.noalias.scope.decl(metadata [[META0:![0-9]+]])
+; CHECK-NEXT:    tail call void @llvm.experimental.noalias.scope.decl(metadata [[META3:![0-9]+]])
 ; CHECK-NEXT:    [[X_I:%.*]] = load i32, ptr [[ADDR1:%.*]], align 4, !alias.scope !3, !noalias !0
 ; CHECK-NEXT:    store i32 [[X_I]], ptr [[ADDR2:%.*]], align 4, !alias.scope !0, !noalias !3
 ; CHECK-NEXT:    [[ADDR1I_1:%.*]] = getelementptr inbounds i32, ptr [[ADDR1]], i64 1
 ; CHECK-NEXT:    [[ADDR2I_1:%.*]] = getelementptr inbounds i32, ptr [[ADDR2]], i64 1
-; CHECK-NEXT:    call void @llvm.experimental.noalias.scope.decl
-; CHECK-NEXT:    call void @llvm.experimental.noalias.scope.decl
+; CHECK-NEXT:    tail call void @llvm.experimental.noalias.scope.decl(metadata [[META5:![0-9]+]])
+; CHECK-NEXT:    tail call void @llvm.experimental.noalias.scope.decl(metadata [[META7:![0-9]+]])
 ; CHECK-NEXT:    [[X_I_1:%.*]] = load i32, ptr [[ADDR1I_1]], align 4, !alias.scope !7, !noalias !5
 ; CHECK-NEXT:    store i32 [[X_I_1]], ptr [[ADDR2I_1]], align 4, !alias.scope !5, !noalias !7
-; CHECK-NEXT:    call void @llvm.experimental.noalias.scope.decl
-; CHECK-NEXT:    call void @llvm.experimental.noalias.scope.decl
+; CHECK-NEXT:    tail call void @llvm.experimental.noalias.scope.decl(metadata [[META9:![0-9]+]])
+; CHECK-NEXT:    tail call void @llvm.experimental.noalias.scope.decl(metadata [[META11:![0-9]+]])
 ; CHECK-NEXT:    [[X_I_2:%.*]] = load i32, ptr [[ADDR1]], align 4, !alias.scope !11, !noalias !9
 ; CHECK-NEXT:    store i32 [[X_I_2]], ptr [[ADDR2]], align 4, !alias.scope !9, !noalias !11
-; CHECK-NEXT:    call void @llvm.experimental.noalias.scope.decl
-; CHECK-NEXT:    call void @llvm.experimental.noalias.scope.decl
+; CHECK-NEXT:    tail call void @llvm.experimental.noalias.scope.decl(metadata [[META13:![0-9]+]])
+; CHECK-NEXT:    tail call void @llvm.experimental.noalias.scope.decl(metadata [[META15:![0-9]+]])
 ; CHECK-NEXT:    [[X_I_3:%.*]] = load i32, ptr [[ADDR1I_1]], align 4, !alias.scope !15, !noalias !13
 ; CHECK-NEXT:    store i32 [[X_I_3]], ptr [[ADDR2I_1]], align 4, !alias.scope !13, !noalias !15
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/PhaseOrdering/switch_different_gep_types.ll b/llvm/test/Transforms/PhaseOrdering/switch_different_gep_types.ll
new file mode 100644
index 0000000000..29a3fc03f4
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/switch_different_gep_types.ll
@@ -0,0 +1,75 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
+; RUN: opt -S -passes='default<O1>' < %s | FileCheck %s
+; RUN: opt -S -passes='default<O2>' < %s | FileCheck %s
+; RUN: opt -S -passes='default<O3>' < %s | FileCheck %s
+
+%"OpKind::Zero" = type { [1 x i32], i32 }
+%"OpKind::One" = type { [1 x i32], i32, i16, [1 x i16] }
+%"OpKind::Two" = type { [1 x i32], i32, i16, i16 }
+%"OpKind::Three" = type { [1 x i32], i32, i16, i16, i16, [1 x i16] }
+
+; FIXME: The switch should be optimized away.
+define i32 @test(ptr %ptr) {
+; CHECK-LABEL: define i32 @test(
+; CHECK-SAME: ptr nocapture readonly [[PTR:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  start:
+; CHECK-NEXT:    [[T:%.*]] = load i32, ptr [[PTR]], align 4
+; CHECK-NEXT:    switch i32 [[T]], label [[DEFAULT:%.*]] [
+; CHECK-NEXT:    i32 0, label [[BB4:%.*]]
+; CHECK-NEXT:    i32 1, label [[BB5:%.*]]
+; CHECK-NEXT:    i32 2, label [[BB6:%.*]]
+; CHECK-NEXT:    i32 3, label [[BB7:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       default:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb4:
+; CHECK-NEXT:    [[GEP0:%.*]] = getelementptr inbounds %"OpKind::Zero", ptr [[PTR]], i64 0, i32 1
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       bb5:
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds %"OpKind::One", ptr [[PTR]], i64 0, i32 1
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       bb6:
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds %"OpKind::Two", ptr [[PTR]], i64 0, i32 1
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       bb7:
+; CHECK-NEXT:    [[GEP3:%.*]] = getelementptr inbounds %"OpKind::Three", ptr [[PTR]], i64 0, i32 1
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[PHI:%.*]] = phi ptr [ [[GEP3]], [[BB7]] ], [ [[GEP2]], [[BB6]] ], [ [[GEP1]], [[BB5]] ], [ [[GEP0]], [[BB4]] ]
+; CHECK-NEXT:    [[RET:%.*]] = load i32, ptr [[PHI]], align 4
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+start:
+  %t = load i32, ptr %ptr, align 4
+  switch i32 %t, label %default [
+  i32 0, label %bb4
+  i32 1, label %bb5
+  i32 2, label %bb6
+  i32 3, label %bb7
+  ]
+
+default:
+  unreachable
+
+bb4:
+  %gep0 = getelementptr inbounds %"OpKind::Zero", ptr %ptr, i64 0, i32 1
+  br label %exit
+
+bb5:
+  %gep1 = getelementptr inbounds %"OpKind::One", ptr %ptr, i64 0, i32 1
+  br label %exit
+
+bb6:
+  %gep2 = getelementptr inbounds %"OpKind::Two", ptr %ptr, i64 0, i32 1
+  br label %exit
+
+bb7:
+  %gep3 = getelementptr inbounds %"OpKind::Three", ptr %ptr, i64 0, i32 1
+  br label %exit
+
+exit:
+  %phi = phi ptr [ %gep3, %bb7 ], [ %gep2, %bb6 ], [ %gep1, %bb5 ], [ %gep0, %bb4 ]
+  %ret = load i32, ptr %phi, align 4
+  ret i32 %ret
+}
+
diff --git a/llvm/test/Transforms/PreISelIntrinsicLowering/X86/objc-arc.ll b/llvm/test/Transforms/PreISelIntrinsicLowering/X86/objc-arc.ll
index 861c8586be..b36737f12e 100644
--- a/llvm/test/Transforms/PreISelIntrinsicLowering/X86/objc-arc.ll
+++ b/llvm/test/Transforms/PreISelIntrinsicLowering/X86/objc-arc.ll
@@ -9,7 +9,7 @@ declare i32 @__gxx_personality_v0(...)
 define ptr @test_objc_autorelease(ptr %arg0) {
 ; CHECK-LABEL: test_objc_autorelease
 ; CHECK-NEXT: entry
-; CHECK-NEXT: %0 = notail call ptr @objc_autorelease(ptr returned %arg0)
+; CHECK-NEXT: %0 = notail call ptr @objc_autorelease(ptr %arg0)
 ; CHECK-NEXT: ret ptr %0
 entry:
   %0 = call ptr @llvm.objc.autorelease(ptr %arg0)
@@ -39,7 +39,7 @@ entry:
 define ptr @test_objc_autoreleaseReturnValue(ptr %arg0) {
 ; CHECK-LABEL: test_objc_autoreleaseReturnValue
 ; CHECK-NEXT: entry
-; CHECK-NEXT: %0 = tail call ptr @objc_autoreleaseReturnValue(ptr returned %arg0)
+; CHECK-NEXT: %0 = tail call ptr @objc_autoreleaseReturnValue(ptr %arg0)
 ; CHECK-NEXT: ret ptr %0
 entry:
   %0 = call ptr @llvm.objc.autoreleaseReturnValue(ptr %arg0)
@@ -119,7 +119,7 @@ entry:
 define ptr @test_objc_retain(ptr %arg0) {
 ; CHECK-LABEL: test_objc_retain
 ; CHECK-NEXT: entry
-; CHECK-NEXT: %0 = tail call ptr @objc_retain(ptr returned %arg0)
+; CHECK-NEXT: %0 = tail call ptr @objc_retain(ptr %arg0)
 ; CHECK-NEXT: ret ptr %0
 entry:
   %0 = call ptr @llvm.objc.retain(ptr %arg0)
@@ -129,7 +129,7 @@ entry:
 define ptr @test_objc_retainAutorelease(ptr %arg0) {
 ; CHECK-LABEL: test_objc_retainAutorelease
 ; CHECK-NEXT: entry
-; CHECK-NEXT: %0 = call ptr @objc_retainAutorelease(ptr returned %arg0)
+; CHECK-NEXT: %0 = call ptr @objc_retainAutorelease(ptr %arg0)
 ; CHECK-NEXT: ret ptr %0
 entry:
   %0 = call ptr @llvm.objc.retainAutorelease(ptr %arg0)
@@ -139,7 +139,7 @@ entry:
 define ptr @test_objc_retainAutoreleaseReturnValue(ptr %arg0) {
 ; CHECK-LABEL: test_objc_retainAutoreleaseReturnValue
 ; CHECK-NEXT: entry
-; CHECK-NEXT: %0 = tail call ptr @objc_retainAutoreleaseReturnValue(ptr returned %arg0)
+; CHECK-NEXT: %0 = tail call ptr @objc_retainAutoreleaseReturnValue(ptr %arg0)
 ; CHECK-NEXT: ret ptr %0
 entry:
   %0 = tail call ptr @llvm.objc.retainAutoreleaseReturnValue(ptr %arg0)
@@ -149,7 +149,7 @@ entry:
 define ptr @test_objc_retainAutoreleasedReturnValue(ptr %arg0) {
 ; CHECK-LABEL: test_objc_retainAutoreleasedReturnValue
 ; CHECK-NEXT: entry
-; CHECK-NEXT: %0 = tail call ptr @objc_retainAutoreleasedReturnValue(ptr returned %arg0)
+; CHECK-NEXT: %0 = tail call ptr @objc_retainAutoreleasedReturnValue(ptr %arg0)
 ; CHECK-NEXT: ret ptr %0
 entry:
   %0 = call ptr @llvm.objc.retainAutoreleasedReturnValue(ptr %arg0)
@@ -213,7 +213,7 @@ entry:
 define ptr @test_objc_unsafeClaimAutoreleasedReturnValue(ptr %arg0) {
 ; CHECK-LABEL: test_objc_unsafeClaimAutoreleasedReturnValue
 ; CHECK-NEXT: entry
-; CHECK-NEXT: %0 = tail call ptr @objc_unsafeClaimAutoreleasedReturnValue(ptr returned %arg0)
+; CHECK-NEXT: %0 = tail call ptr @objc_unsafeClaimAutoreleasedReturnValue(ptr %arg0)
 ; CHECK-NEXT: ret ptr %0
 entry:
   %0 = call ptr @llvm.objc.unsafeClaimAutoreleasedReturnValue(ptr %arg0)
@@ -260,7 +260,7 @@ entry:
 define ptr @test_objc_retain_autorelease(ptr %arg0) {
 ; CHECK-LABEL: test_objc_retain_autorelease
 ; CHECK-NEXT: entry
-; CHECK-NEXT: %0 = call ptr @objc_retain_autorelease(ptr returned %arg0)
+; CHECK-NEXT: %0 = call ptr @objc_retain_autorelease(ptr %arg0)
 ; CHECK-NEXT: ret ptr %0
 entry:
   %0 = call ptr @llvm.objc.retain.autorelease(ptr %arg0)
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/call-argument-attributes.ll b/llvm/test/Transforms/RewriteStatepointsForGC/call-argument-attributes.ll
new file mode 100644
index 0000000000..4a7088f953
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/call-argument-attributes.ll
@@ -0,0 +1,42 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
+; RUN: opt < %s -passes=rewrite-statepoints-for-gc -S | FileCheck %s
+
+declare i8 @callee(ptr, i8, float, ptr)
+
+define i8 @test(ptr %arg) gc "statepoint-example" {
+; CHECK-LABEL: define i8 @test(
+; CHECK-SAME: ptr [[ARG:%.*]]) gc "statepoint-example" {
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0(i64 2882400000, i32 0, ptr elementtype(i8 (ptr, i8, float, ptr)) @callee, i32 4, i32 0, ptr nocapture sret({ i64, i64 }) align 8 null, i8 signext 8, float inreg 1.000000e+00, ptr [[ARG]], i32 0, i32 0)
+; CHECK-NEXT:    [[R1:%.*]] = call zeroext i8 @llvm.experimental.gc.result.i8(token [[STATEPOINT_TOKEN]])
+; CHECK-NEXT:    ret i8 [[R1]]
+;
+  %r = call zeroext i8 @callee(ptr sret({i64, i64}) noalias align 8 nocapture null, i8 signext 8, float inreg 1.0, ptr writeonly %arg)
+  ret i8 %r
+}
+
+declare i32 @personality_function()
+
+define i8 @test_invoke(ptr %arg) gc "statepoint-example" personality ptr @personality_function {
+; CHECK-LABEL: define i8 @test_invoke(
+; CHECK-SAME: ptr [[ARG:%.*]]) gc "statepoint-example" personality ptr @personality_function {
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = invoke token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0(i64 2882400000, i32 0, ptr elementtype(i8 (ptr, i8, float, ptr)) @callee, i32 4, i32 0, ptr nocapture sret({ i64, i64 }) align 8 null, i8 signext 8, float inreg 1.000000e+00, ptr [[ARG]], i32 0, i32 0)
+; CHECK-NEXT:    to label [[NORMAL_RETURN:%.*]] unwind label [[EXCEPTIONAL_RETURN:%.*]]
+; CHECK:       normal_return:
+; CHECK-NEXT:    [[R1:%.*]] = call zeroext i8 @llvm.experimental.gc.result.i8(token [[STATEPOINT_TOKEN]])
+; CHECK-NEXT:    ret i8 [[R1]]
+; CHECK:       exceptional_return:
+; CHECK-NEXT:    [[LANDING_PAD4:%.*]] = landingpad token
+; CHECK-NEXT:    cleanup
+; CHECK-NEXT:    ret i8 0
+;
+  %r = invoke zeroext i8 @callee(ptr sret({i64, i64}) noalias align 8 nocapture null, i8 signext 8, float inreg 1.0, ptr writeonly %arg)
+  to label %normal_return unwind label %exceptional_return
+
+normal_return:
+  ret i8 %r
+
+exceptional_return:
+  %landing_pad4 = landingpad token
+  cleanup
+  ret i8 0
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/matching-gather-nodes-phi-users.ll b/llvm/test/Transforms/SLPVectorizer/X86/matching-gather-nodes-phi-users.ll
new file mode 100644
index 0000000000..28e0b06f69
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/matching-gather-nodes-phi-users.ll
@@ -0,0 +1,124 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
+; RUN: opt -passes=slp-vectorizer -mtriple=x86_64 -S -mcpu=core-avx2 -pass-remarks-output=%t < %s | FileCheck %s
+; RUN: FileCheck --input-file=%t --check-prefix=YAML %s
+
+; YAML: --- !Passed
+; YAML: Pass:            slp-vectorizer
+; YAML: Name:            StoresVectorized
+; YAML: Function:        test
+; YAML: Args:
+; YAML:   - String:          'Stores SLP vectorized with cost '
+; YAML:   - Cost:            '-3'
+; YAML:   - String:          ' and with tree size '
+; YAML:   - TreeSize:        '14'
+; YAML: ...
+
+; Test that SLP cost modeling is able to match gathering tree
+; entries: two instances of { call0 .. call3 }. Both used by
+; vectorized PHI entries: { phi0 .. phi3 } and { phi4 .. phi7 }.
+; Check that cost is not added twice for gathered calls.
+
+define void @test(ptr %dst, float %a, float %b, float %c, float %d) {
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: ptr [[DST:%.*]], float [[A:%.*]], float [[B:%.*]], float [[C:%.*]], float [[D:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  Entry:
+; CHECK-NEXT:    br i1 poison, label [[LOOP0:%.*]], label [[EXIT:%.*]]
+; CHECK:       loop0:
+; CHECK-NEXT:    [[TMP0:%.*]] = phi <4 x float> [ <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>, [[ENTRY:%.*]] ], [ [[TMP7:%.*]], [[USERBLOCK1:%.*]] ]
+; CHECK-NEXT:    br i1 poison, label [[USERBLOCK0:%.*]], label [[BLKX:%.*]]
+; CHECK:       UserBlock0:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <4 x float> [ zeroinitializer, [[LOOP0]] ], [ [[TMP5:%.*]], [[BLKX]] ]
+; CHECK-NEXT:    br label [[LOOP_INNER:%.*]]
+; CHECK:       blkx:
+; CHECK-NEXT:    [[CALL0:%.*]] = call fast float @foo(float [[A]])
+; CHECK-NEXT:    [[CALL1:%.*]] = call fast float @foo(float [[B]])
+; CHECK-NEXT:    [[CALL2:%.*]] = call fast float @foo(float [[C]])
+; CHECK-NEXT:    [[CALL3:%.*]] = call fast float @foo(float [[D]])
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> poison, float [[CALL0]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x float> [[TMP2]], float [[CALL1]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x float> [[TMP3]], float [[CALL2]], i32 2
+; CHECK-NEXT:    [[TMP5]] = insertelement <4 x float> [[TMP4]], float [[CALL3]], i32 3
+; CHECK-NEXT:    br i1 poison, label [[USERBLOCK0]], label [[USERBLOCK1]]
+; CHECK:       UserBlock1:
+; CHECK-NEXT:    [[TMP6:%.*]] = phi <4 x float> [ [[TMP5]], [[BLKX]] ], [ [[TMP9:%.*]], [[LOOP_INNER]] ]
+; CHECK-NEXT:    [[TMP7]] = fadd fast <4 x float> [[TMP0]], [[TMP6]]
+; CHECK-NEXT:    br i1 poison, label [[EXIT]], label [[LOOP0]]
+; CHECK:       loop.inner:
+; CHECK-NEXT:    [[TMP8:%.*]] = phi <4 x float> [ [[TMP1]], [[USERBLOCK0]] ], [ [[TMP9]], [[LOOP_INNER]] ]
+; CHECK-NEXT:    [[TMP9]] = fadd fast <4 x float> [[TMP8]], poison
+; CHECK-NEXT:    br i1 poison, label [[USERBLOCK1]], label [[LOOP_INNER]]
+; CHECK:       Exit:
+; CHECK-NEXT:    [[TMP10:%.*]] = phi <4 x float> [ <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>, [[ENTRY]] ], [ [[TMP7]], [[USERBLOCK1]] ]
+; CHECK-NEXT:    [[IDX0:%.*]] = add i64 0, poison
+; CHECK-NEXT:    [[GEP0:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[IDX0]]
+; CHECK-NEXT:    store <4 x float> [[TMP10]], ptr [[GEP0]], align 4
+; CHECK-NEXT:    ret void
+;
+Entry:
+  br i1 poison, label %loop0, label %Exit
+
+loop0:
+  %i0 = phi float [ 5.000000e-01, %Entry ], [ %add3, %UserBlock1 ]
+  %i1 = phi float [ 5.000000e-01, %Entry ], [ %add2, %UserBlock1 ]
+  %i2 = phi float [ 5.000000e-01, %Entry ], [ %add1, %UserBlock1 ]
+  %i3 = phi float [ 5.000000e-01, %Entry ], [ %add0, %UserBlock1 ]
+  br i1 poison, label %UserBlock0, label %blkx
+
+UserBlock0:
+  %phi0 = phi float [ 0.000000e+00, %loop0 ], [ %call3, %blkx ]
+  %phi1 = phi float [ 0.000000e+00, %loop0 ], [ %call2, %blkx ]
+  %phi2 = phi float [ 0.000000e+00, %loop0 ], [ %call1, %blkx ]
+  %phi3 = phi float [ 0.000000e+00, %loop0 ], [ %call0, %blkx ]
+  br label %loop.inner
+
+blkx:
+  %call0 = call fast float @foo(float %a)
+  %call1 = call fast float @foo(float %b)
+  %call2 = call fast float @foo(float %c)
+  %call3 = call fast float @foo(float %d)
+  br i1 poison, label %UserBlock0, label %UserBlock1
+
+UserBlock1:
+  %phi4 = phi float [ %call0, %blkx ], [ %add4, %loop.inner ]
+  %phi5 = phi float [ %call1, %blkx ], [ %add5, %loop.inner ]
+  %phi6 = phi float [ %call2, %blkx ], [ %add6, %loop.inner ]
+  %phi7 = phi float [ %call3, %blkx ], [ %add7, %loop.inner ]
+  %add0 = fadd fast float %i3, %phi4
+  %add1 = fadd fast float %i2, %phi5
+  %add2 = fadd fast float %i1, %phi6
+  %add3 = fadd fast float %i0, %phi7
+  br i1 poison, label %Exit, label %loop0
+
+loop.inner:
+  %t20 = phi float [ %phi3, %UserBlock0 ], [ %add4, %loop.inner ]
+  %t19 = phi float [ %phi2, %UserBlock0 ], [ %add5, %loop.inner ]
+  %t18 = phi float [ %phi1, %UserBlock0 ], [ %add6, %loop.inner ]
+  %t17 = phi float [ %phi0, %UserBlock0 ], [ %add7, %loop.inner ]
+  %add4 = fadd fast float %t20, poison
+  %add5 = fadd fast float %t19, poison
+  %add6 = fadd fast float %t18, poison
+  %add7 = fadd fast float %t17, poison
+
+  br i1 poison, label %UserBlock1, label %loop.inner
+
+Exit:
+  %t48 = phi float [ 5.000000e-01, %Entry ], [ %add3, %UserBlock1 ]
+  %t47 = phi float [ 5.000000e-01, %Entry ], [ %add2, %UserBlock1 ]
+  %t46 = phi float [ 5.000000e-01, %Entry ], [ %add1, %UserBlock1 ]
+  %t45 = phi float [ 5.000000e-01, %Entry ], [ %add0, %UserBlock1 ]
+  %idx0 = add i64 0, poison
+  %gep0 = getelementptr inbounds float, ptr %dst, i64 %idx0
+  store float %t45, ptr %gep0, align 4
+  %idx1 = add i64 1, poison
+  %gep1 = getelementptr inbounds float, ptr %dst, i64 %idx1
+  store float %t46, ptr %gep1, align 4
+  %idx2 = add i64 2, poison
+  %gep2 = getelementptr inbounds float, ptr %dst, i64 %idx2
+  store float %t47, ptr %gep2, align 4
+  %idx3 = add i64 3, poison
+  %gep3 = getelementptr inbounds float, ptr %dst, i64 %idx3
+  store float %t48, ptr %gep3, align 4
+  ret void
+}
+
+declare float @foo(float)
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/multi-nodes-to-shuffle.ll b/llvm/test/Transforms/SLPVectorizer/X86/multi-nodes-to-shuffle.ll
index cf3d40df15..21aac98aa3 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/multi-nodes-to-shuffle.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/multi-nodes-to-shuffle.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -passes=slp-vectorizer -S < %s -mtriple=x86_64-unknown-linux -slp-threshold=-107 | FileCheck %s
+; RUN: opt -passes=slp-vectorizer -S < %s -mtriple=x86_64-unknown-linux -slp-threshold=-107 -mattr=+avx2 | FileCheck %s
 
 define void @test(i64 %p0, i64 %p1, i64 %p2, i64 %p3) {
 ; CHECK-LABEL: @test(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/non-scheduled-inst-reused-as-last-inst.ll b/llvm/test/Transforms/SLPVectorizer/X86/non-scheduled-inst-reused-as-last-inst.ll
new file mode 100644
index 0000000000..3a9eca2bf2
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/non-scheduled-inst-reused-as-last-inst.ll
@@ -0,0 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
+; RUN: opt -S -passes=slp-vectorizer -slp-threshold=-9999 -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define void @foo() {
+; CHECK-LABEL: define void @foo() {
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> <i32 poison, i32 0>, i32 0, i32 0
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <2 x i32> [ zeroinitializer, [[BB:%.*]] ], [ [[TMP6:%.*]], [[BB4:%.*]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = shl <2 x i32> [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i32> [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> [[TMP1]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP6]] = or <2 x i32> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i32> [[TMP6]], i32 0
+; CHECK-NEXT:    [[CALL:%.*]] = call i64 null(i32 [[TMP7]])
+; CHECK-NEXT:    br label [[BB4]]
+; CHECK:       bb4:
+; CHECK-NEXT:    br i1 false, label [[BB5:%.*]], label [[BB1]]
+; CHECK:       bb5:
+; CHECK-NEXT:    [[TMP8:%.*]] = phi <2 x i32> [ [[TMP4]], [[BB4]] ]
+; CHECK-NEXT:    ret void
+;
+bb:
+  br label %bb1
+
+bb1:
+  %phi = phi i32 [ 0, %bb ], [ %or, %bb4 ]
+  %phi2 = phi i32 [ 0, %bb ], [ %or3, %bb4 ]
+  %and = and i32 0, 0
+  %shl = shl i32 %phi, %and
+  %or = or i32 %shl, 0
+  %call = call i64 null(i32 %or)
+  %or3 = or i32 %phi2, 0
+  br label %bb4
+
+bb4:
+  br i1 false, label %bb5, label %bb1
+
+bb5:
+  %phi6 = phi i32 [ %shl, %bb4 ]
+  %phi7 = phi i32 [ %or3, %bb4 ]
+  ret void
+}
diff --git a/llvm/test/Transforms/Util/dbg-user-of-aext.ll b/llvm/test/Transforms/Util/dbg-user-of-aext.ll
index 7c9188f551..c91b68a68e 100644
--- a/llvm/test/Transforms/Util/dbg-user-of-aext.ll
+++ b/llvm/test/Transforms/Util/dbg-user-of-aext.ll
@@ -3,8 +3,6 @@
 ; opposed to the operand of a [s|z]ext).
 ; RUN: opt -S -passes='sroa' %s | FileCheck %s
 
-target datalayout = "i64:64"
-
 ; Built from:
 ; struct foo { bool b; long i; };
 ; void f(bool b, bool expr, foo g) {
@@ -14,15 +12,15 @@ target datalayout = "i64:64"
 
 ; Expect two fragments:
 ; * first starting at bit 0, 8 bits (for the bool)
-; * second starting at bit 64, 64 bits (for the long)
-; (this happens to create/demonstrate a gap from bits [7, 64))
+; * second starting at bit 32, 32 bits (for the long)
+; (this happens to create/demonstrate a gap from bits [7, 32))
 
 ; But also check that a complex expression is not used for a lone bool
 ; parameter. It can reference the register it's in directly without masking off
 ; high bits or anything
 
 ; CHECK: call void @llvm.dbg.value(metadata i8 %g.coerce0, metadata ![[VAR_STRUCT:[0-9]+]], metadata !DIExpression(DW_OP_LLVM_fragment, 0, 8))
-; CHECK: call void @llvm.dbg.value(metadata i64 %g.coerce1, metadata ![[VAR_STRUCT]], metadata !DIExpression(DW_OP_LLVM_fragment, 64, 64))
+; CHECK: call void @llvm.dbg.value(metadata i64 %g.coerce1, metadata ![[VAR_STRUCT]], metadata !DIExpression(DW_OP_LLVM_fragment, 32, 64))
 ; CHECK: call void @llvm.dbg.value(metadata i8 %frombool, metadata ![[VAR_BOOL:[0-9]+]], metadata !DIExpression())
 ; CHECK: call void @llvm.dbg.value(metadata i8 %frombool1, metadata ![[VAR_FRAG:[0-9]+]], metadata !DIExpression(DW_OP_LLVM_fragment, 0, 4))
 
diff --git a/llvm/test/Transforms/VectorCombine/RISCV/vpintrin-scalarization.ll b/llvm/test/Transforms/VectorCombine/RISCV/vpintrin-scalarization.ll
index ea1831210b..e95aea4eb4 100644
--- a/llvm/test/Transforms/VectorCombine/RISCV/vpintrin-scalarization.ll
+++ b/llvm/test/Transforms/VectorCombine/RISCV/vpintrin-scalarization.ll
@@ -166,14 +166,23 @@ define <vscale x 1 x i64> @mul_nxv1i64_anymask(<vscale x 1 x i64> %x, i64 %y, <v
 }
 
 define <vscale x 1 x i64> @sdiv_nxv1i64_allonesmask(<vscale x 1 x i64> %x, i64 %y, i32 zeroext %evl) {
-; ALL-LABEL: @sdiv_nxv1i64_allonesmask(
-; ALL-NEXT:    [[SPLAT:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
-; ALL-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[SPLAT]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
-; ALL-NEXT:    [[TMP1:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[Y:%.*]], i64 0
-; ALL-NEXT:    [[TMP2:%.*]] = shufflevector <vscale x 1 x i64> [[TMP1]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-; ALL-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x i64> @llvm.vp.sdiv.nxv1i64(<vscale x 1 x i64> [[TMP2]], <vscale x 1 x i64> shufflevector (<vscale x 1 x i64> insertelement (<vscale x 1 x i64> poison, i64 42, i32 0), <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> [[MASK]], i32 [[EVL:%.*]])
-; ALL-NEXT:    [[TMP4:%.*]] = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> [[X:%.*]], <vscale x 1 x i64> [[TMP3]], <vscale x 1 x i1> [[MASK]], i32 [[EVL]])
-; ALL-NEXT:    ret <vscale x 1 x i64> [[TMP4]]
+; VEC-COMBINE-LABEL: @sdiv_nxv1i64_allonesmask(
+; VEC-COMBINE-NEXT:    [[SPLAT:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
+; VEC-COMBINE-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[SPLAT]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
+; VEC-COMBINE-NEXT:    [[TMP1:%.*]] = sdiv i64 [[Y:%.*]], 42
+; VEC-COMBINE-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[TMP1]], i64 0
+; VEC-COMBINE-NEXT:    [[TMP2:%.*]] = shufflevector <vscale x 1 x i64> [[DOTSPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; VEC-COMBINE-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> [[X:%.*]], <vscale x 1 x i64> [[TMP2]], <vscale x 1 x i1> [[MASK]], i32 [[EVL:%.*]])
+; VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> [[TMP3]]
+;
+; NO-VEC-COMBINE-LABEL: @sdiv_nxv1i64_allonesmask(
+; NO-VEC-COMBINE-NEXT:    [[SPLAT:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
+; NO-VEC-COMBINE-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[SPLAT]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
+; NO-VEC-COMBINE-NEXT:    [[TMP1:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[Y:%.*]], i64 0
+; NO-VEC-COMBINE-NEXT:    [[TMP2:%.*]] = shufflevector <vscale x 1 x i64> [[TMP1]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; NO-VEC-COMBINE-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x i64> @llvm.vp.sdiv.nxv1i64(<vscale x 1 x i64> [[TMP2]], <vscale x 1 x i64> shufflevector (<vscale x 1 x i64> insertelement (<vscale x 1 x i64> poison, i64 42, i32 0), <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> [[MASK]], i32 [[EVL:%.*]])
+; NO-VEC-COMBINE-NEXT:    [[TMP4:%.*]] = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> [[X:%.*]], <vscale x 1 x i64> [[TMP3]], <vscale x 1 x i1> [[MASK]], i32 [[EVL]])
+; NO-VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> [[TMP4]]
 ;
   %splat = insertelement <vscale x 1 x i1> poison, i1 -1, i32 0
   %mask = shufflevector <vscale x 1 x i1> %splat, <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
@@ -199,15 +208,45 @@ define <vscale x 1 x i64> @sdiv_nxv1i64_anymask(<vscale x 1 x i64> %x, i64 %y, <
   ret <vscale x 1 x i64> %4
 }
 
+define <vscale x 1 x i64> @sdiv_nxv1i64_unspeculatable(i64 %x, i64 %y, i32 zeroext %evl) {
+; ALL-LABEL: @sdiv_nxv1i64_unspeculatable(
+; ALL-NEXT:    [[MASK_HEAD:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
+; ALL-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[MASK_HEAD]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
+; ALL-NEXT:    [[X_HEAD:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[X:%.*]], i64 0
+; ALL-NEXT:    [[X_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[X_HEAD]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; ALL-NEXT:    [[Y_HEAD:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[X]], i64 0
+; ALL-NEXT:    [[Y_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[Y_HEAD]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; ALL-NEXT:    [[RES:%.*]] = call <vscale x 1 x i64> @llvm.vp.sdiv.nxv1i64(<vscale x 1 x i64> [[X_SPLAT]], <vscale x 1 x i64> [[Y_SPLAT]], <vscale x 1 x i1> [[MASK]], i32 [[EVL:%.*]])
+; ALL-NEXT:    ret <vscale x 1 x i64> [[RES]]
+;
+  %mask.head = insertelement <vscale x 1 x i1> poison, i1 -1, i32 0
+  %mask = shufflevector <vscale x 1 x i1> %mask.head, <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
+  %x.head = insertelement <vscale x 1 x i64> poison, i64 %x, i64 0
+  %x.splat = shufflevector <vscale x 1 x i64> %x.head, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+  %y.head = insertelement <vscale x 1 x i64> poison, i64 %x, i64 0
+  %y.splat = shufflevector <vscale x 1 x i64> %y.head, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+  %res = call <vscale x 1 x i64> @llvm.vp.sdiv.nxv1i64(<vscale x 1 x i64> %x.splat, <vscale x 1 x i64> %y.splat, <vscale x 1 x i1> %mask, i32 %evl)
+  ret <vscale x 1 x i64> %res
+}
+
 define <vscale x 1 x i64> @udiv_nxv1i64_allonesmask(<vscale x 1 x i64> %x, i64 %y, i32 zeroext %evl) {
-; ALL-LABEL: @udiv_nxv1i64_allonesmask(
-; ALL-NEXT:    [[SPLAT:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
-; ALL-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[SPLAT]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
-; ALL-NEXT:    [[TMP1:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[Y:%.*]], i64 0
-; ALL-NEXT:    [[TMP2:%.*]] = shufflevector <vscale x 1 x i64> [[TMP1]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-; ALL-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x i64> @llvm.vp.udiv.nxv1i64(<vscale x 1 x i64> [[TMP2]], <vscale x 1 x i64> shufflevector (<vscale x 1 x i64> insertelement (<vscale x 1 x i64> poison, i64 42, i32 0), <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> [[MASK]], i32 [[EVL:%.*]])
-; ALL-NEXT:    [[TMP4:%.*]] = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> [[X:%.*]], <vscale x 1 x i64> [[TMP3]], <vscale x 1 x i1> [[MASK]], i32 [[EVL]])
-; ALL-NEXT:    ret <vscale x 1 x i64> [[TMP4]]
+; VEC-COMBINE-LABEL: @udiv_nxv1i64_allonesmask(
+; VEC-COMBINE-NEXT:    [[SPLAT:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
+; VEC-COMBINE-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[SPLAT]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
+; VEC-COMBINE-NEXT:    [[TMP1:%.*]] = udiv i64 [[Y:%.*]], 42
+; VEC-COMBINE-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[TMP1]], i64 0
+; VEC-COMBINE-NEXT:    [[TMP2:%.*]] = shufflevector <vscale x 1 x i64> [[DOTSPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; VEC-COMBINE-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> [[X:%.*]], <vscale x 1 x i64> [[TMP2]], <vscale x 1 x i1> [[MASK]], i32 [[EVL:%.*]])
+; VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> [[TMP3]]
+;
+; NO-VEC-COMBINE-LABEL: @udiv_nxv1i64_allonesmask(
+; NO-VEC-COMBINE-NEXT:    [[SPLAT:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
+; NO-VEC-COMBINE-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[SPLAT]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
+; NO-VEC-COMBINE-NEXT:    [[TMP1:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[Y:%.*]], i64 0
+; NO-VEC-COMBINE-NEXT:    [[TMP2:%.*]] = shufflevector <vscale x 1 x i64> [[TMP1]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; NO-VEC-COMBINE-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x i64> @llvm.vp.udiv.nxv1i64(<vscale x 1 x i64> [[TMP2]], <vscale x 1 x i64> shufflevector (<vscale x 1 x i64> insertelement (<vscale x 1 x i64> poison, i64 42, i32 0), <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> [[MASK]], i32 [[EVL:%.*]])
+; NO-VEC-COMBINE-NEXT:    [[TMP4:%.*]] = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> [[X:%.*]], <vscale x 1 x i64> [[TMP3]], <vscale x 1 x i1> [[MASK]], i32 [[EVL]])
+; NO-VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> [[TMP4]]
 ;
   %splat = insertelement <vscale x 1 x i1> poison, i1 -1, i32 0
   %mask = shufflevector <vscale x 1 x i1> %splat, <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
@@ -233,15 +272,45 @@ define <vscale x 1 x i64> @udiv_nxv1i64_anymask(<vscale x 1 x i64> %x, i64 %y, <
   ret <vscale x 1 x i64> %4
 }
 
+define <vscale x 1 x i64> @udiv_nxv1i64_unspeculatable(i64 %x, i64 %y, i32 zeroext %evl) {
+; ALL-LABEL: @udiv_nxv1i64_unspeculatable(
+; ALL-NEXT:    [[MASK_HEAD:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
+; ALL-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[MASK_HEAD]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
+; ALL-NEXT:    [[X_HEAD:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[X:%.*]], i64 0
+; ALL-NEXT:    [[X_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[X_HEAD]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; ALL-NEXT:    [[Y_HEAD:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[X]], i64 0
+; ALL-NEXT:    [[Y_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[Y_HEAD]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; ALL-NEXT:    [[RES:%.*]] = call <vscale x 1 x i64> @llvm.vp.udiv.nxv1i64(<vscale x 1 x i64> [[X_SPLAT]], <vscale x 1 x i64> [[Y_SPLAT]], <vscale x 1 x i1> [[MASK]], i32 [[EVL:%.*]])
+; ALL-NEXT:    ret <vscale x 1 x i64> [[RES]]
+;
+  %mask.head = insertelement <vscale x 1 x i1> poison, i1 -1, i32 0
+  %mask = shufflevector <vscale x 1 x i1> %mask.head, <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
+  %x.head = insertelement <vscale x 1 x i64> poison, i64 %x, i64 0
+  %x.splat = shufflevector <vscale x 1 x i64> %x.head, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+  %y.head = insertelement <vscale x 1 x i64> poison, i64 %x, i64 0
+  %y.splat = shufflevector <vscale x 1 x i64> %y.head, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+  %res = call <vscale x 1 x i64> @llvm.vp.udiv.nxv1i64(<vscale x 1 x i64> %x.splat, <vscale x 1 x i64> %y.splat, <vscale x 1 x i1> %mask, i32 %evl)
+  ret <vscale x 1 x i64> %res
+}
+
 define <vscale x 1 x i64> @srem_nxv1i64_allonesmask(<vscale x 1 x i64> %x, i64 %y, i32 zeroext %evl) {
-; ALL-LABEL: @srem_nxv1i64_allonesmask(
-; ALL-NEXT:    [[SPLAT:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
-; ALL-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[SPLAT]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
-; ALL-NEXT:    [[TMP1:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[Y:%.*]], i64 0
-; ALL-NEXT:    [[TMP2:%.*]] = shufflevector <vscale x 1 x i64> [[TMP1]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-; ALL-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x i64> @llvm.vp.srem.nxv1i64(<vscale x 1 x i64> [[TMP2]], <vscale x 1 x i64> shufflevector (<vscale x 1 x i64> insertelement (<vscale x 1 x i64> poison, i64 42, i32 0), <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> [[MASK]], i32 [[EVL:%.*]])
-; ALL-NEXT:    [[TMP4:%.*]] = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> [[X:%.*]], <vscale x 1 x i64> [[TMP3]], <vscale x 1 x i1> [[MASK]], i32 [[EVL]])
-; ALL-NEXT:    ret <vscale x 1 x i64> [[TMP4]]
+; VEC-COMBINE-LABEL: @srem_nxv1i64_allonesmask(
+; VEC-COMBINE-NEXT:    [[SPLAT:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
+; VEC-COMBINE-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[SPLAT]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
+; VEC-COMBINE-NEXT:    [[TMP1:%.*]] = srem i64 [[Y:%.*]], 42
+; VEC-COMBINE-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[TMP1]], i64 0
+; VEC-COMBINE-NEXT:    [[TMP2:%.*]] = shufflevector <vscale x 1 x i64> [[DOTSPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; VEC-COMBINE-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> [[X:%.*]], <vscale x 1 x i64> [[TMP2]], <vscale x 1 x i1> [[MASK]], i32 [[EVL:%.*]])
+; VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> [[TMP3]]
+;
+; NO-VEC-COMBINE-LABEL: @srem_nxv1i64_allonesmask(
+; NO-VEC-COMBINE-NEXT:    [[SPLAT:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
+; NO-VEC-COMBINE-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[SPLAT]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
+; NO-VEC-COMBINE-NEXT:    [[TMP1:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[Y:%.*]], i64 0
+; NO-VEC-COMBINE-NEXT:    [[TMP2:%.*]] = shufflevector <vscale x 1 x i64> [[TMP1]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; NO-VEC-COMBINE-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x i64> @llvm.vp.srem.nxv1i64(<vscale x 1 x i64> [[TMP2]], <vscale x 1 x i64> shufflevector (<vscale x 1 x i64> insertelement (<vscale x 1 x i64> poison, i64 42, i32 0), <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> [[MASK]], i32 [[EVL:%.*]])
+; NO-VEC-COMBINE-NEXT:    [[TMP4:%.*]] = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> [[X:%.*]], <vscale x 1 x i64> [[TMP3]], <vscale x 1 x i1> [[MASK]], i32 [[EVL]])
+; NO-VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> [[TMP4]]
 ;
   %splat = insertelement <vscale x 1 x i1> poison, i1 -1, i32 0
   %mask = shufflevector <vscale x 1 x i1> %splat, <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
@@ -267,210 +336,265 @@ define <vscale x 1 x i64> @srem_nxv1i64_anymask(<vscale x 1 x i64> %x, i64 %y, <
   ret <vscale x 1 x i64> %4
 }
 
-define <vscale x 1 x i64> @urem_nxv1i64_allonesmask(<vscale x 1 x i64> %x, i64 %y, i32 zeroext %evl) {
-; ALL-LABEL: @urem_nxv1i64_allonesmask(
-; ALL-NEXT:    [[SPLAT:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
-; ALL-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[SPLAT]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
-; ALL-NEXT:    [[TMP1:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[Y:%.*]], i64 0
-; ALL-NEXT:    [[TMP2:%.*]] = shufflevector <vscale x 1 x i64> [[TMP1]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-; ALL-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x i64> @llvm.vp.urem.nxv1i64(<vscale x 1 x i64> [[TMP2]], <vscale x 1 x i64> shufflevector (<vscale x 1 x i64> insertelement (<vscale x 1 x i64> poison, i64 42, i32 0), <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> [[MASK]], i32 [[EVL:%.*]])
-; ALL-NEXT:    [[TMP4:%.*]] = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> [[X:%.*]], <vscale x 1 x i64> [[TMP3]], <vscale x 1 x i1> [[MASK]], i32 [[EVL]])
-; ALL-NEXT:    ret <vscale x 1 x i64> [[TMP4]]
+define <vscale x 1 x i64> @srem_nxv1i64_unspeculatable(i64 %x, i64 %y, i32 zeroext %evl) {
+; ALL-LABEL: @srem_nxv1i64_unspeculatable(
+; ALL-NEXT:    [[MASK_HEAD:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
+; ALL-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[MASK_HEAD]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
+; ALL-NEXT:    [[X_HEAD:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[X:%.*]], i64 0
+; ALL-NEXT:    [[X_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[X_HEAD]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; ALL-NEXT:    [[Y_HEAD:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[X]], i64 0
+; ALL-NEXT:    [[Y_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[Y_HEAD]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; ALL-NEXT:    [[RES:%.*]] = call <vscale x 1 x i64> @llvm.vp.srem.nxv1i64(<vscale x 1 x i64> [[X_SPLAT]], <vscale x 1 x i64> [[Y_SPLAT]], <vscale x 1 x i1> [[MASK]], i32 [[EVL:%.*]])
+; ALL-NEXT:    ret <vscale x 1 x i64> [[RES]]
 ;
-  %splat = insertelement <vscale x 1 x i1> poison, i1 -1, i32 0
-  %mask = shufflevector <vscale x 1 x i1> %splat, <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
-  %1 = insertelement <vscale x 1 x i64> poison, i64 %y, i64 0
-  %2 = shufflevector <vscale x 1 x i64> %1, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-  %3 = call <vscale x 1 x i64> @llvm.vp.urem.nxv1i64(<vscale x 1 x i64> %2, <vscale x 1 x i64> shufflevector(<vscale x 1 x i64> insertelement(<vscale x 1 x i64> poison, i64 42, i32 0), <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> %mask, i32 %evl)
-  %4 = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> %x, <vscale x 1 x i64> %3, <vscale x 1 x i1> %mask, i32 %evl)
-  ret <vscale x 1 x i64> %4
+  %mask.head = insertelement <vscale x 1 x i1> poison, i1 -1, i32 0
+  %mask = shufflevector <vscale x 1 x i1> %mask.head, <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
+  %x.head = insertelement <vscale x 1 x i64> poison, i64 %x, i64 0
+  %x.splat = shufflevector <vscale x 1 x i64> %x.head, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+  %y.head = insertelement <vscale x 1 x i64> poison, i64 %x, i64 0
+  %y.splat = shufflevector <vscale x 1 x i64> %y.head, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+  %res = call <vscale x 1 x i64> @llvm.vp.srem.nxv1i64(<vscale x 1 x i64> %x.splat, <vscale x 1 x i64> %y.splat, <vscale x 1 x i1> %mask, i32 %evl)
+  ret <vscale x 1 x i64> %res
 }
 
-define <vscale x 1 x i64> @urem_nxv1i64_anymask(<vscale x 1 x i64> %x, i64 %y, <vscale x 1 x i1> %mask, i32 zeroext %evl) {
-; ALL-LABEL: @urem_nxv1i64_anymask(
-; ALL-NEXT:    [[TMP1:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[Y:%.*]], i64 0
-; ALL-NEXT:    [[TMP2:%.*]] = shufflevector <vscale x 1 x i64> [[TMP1]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-; ALL-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x i64> @llvm.vp.urem.nxv1i64(<vscale x 1 x i64> [[TMP2]], <vscale x 1 x i64> shufflevector (<vscale x 1 x i64> insertelement (<vscale x 1 x i64> poison, i64 42, i32 0), <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> [[MASK:%.*]], i32 [[EVL:%.*]])
-; ALL-NEXT:    [[TMP4:%.*]] = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> [[X:%.*]], <vscale x 1 x i64> [[TMP3]], <vscale x 1 x i1> [[MASK]], i32 [[EVL]])
-; ALL-NEXT:    ret <vscale x 1 x i64> [[TMP4]]
-;
-  %1 = insertelement <vscale x 1 x i64> poison, i64 %y, i64 0
-  %2 = shufflevector <vscale x 1 x i64> %1, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-  %3 = call <vscale x 1 x i64> @llvm.vp.urem.nxv1i64(<vscale x 1 x i64> %2, <vscale x 1 x i64> shufflevector(<vscale x 1 x i64> insertelement(<vscale x 1 x i64> poison, i64 42, i32 0), <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> %mask, i32 %evl)
-  %4 = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> %x, <vscale x 1 x i64> %3, <vscale x 1 x i1> %mask, i32 %evl)
-  ret <vscale x 1 x i64> %4
-}
-
-define <vscale x 1 x i64> @sdiv_nxv1i64_allonesmask_knownvl(<vscale x 1 x i64> %x, i64 %y) {
-; VEC-COMBINE-LABEL: @sdiv_nxv1i64_allonesmask_knownvl(
+define <vscale x 1 x i64> @urem_nxv1i64_allonesmask(<vscale x 1 x i64> %x, i64 %y, i32 zeroext %evl) {
+; VEC-COMBINE-LABEL: @urem_nxv1i64_allonesmask(
 ; VEC-COMBINE-NEXT:    [[SPLAT:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
 ; VEC-COMBINE-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[SPLAT]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
-; VEC-COMBINE-NEXT:    [[TMP1:%.*]] = sdiv i64 [[Y:%.*]], 42
+; VEC-COMBINE-NEXT:    [[TMP1:%.*]] = urem i64 [[Y:%.*]], 42
 ; VEC-COMBINE-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[TMP1]], i64 0
 ; VEC-COMBINE-NEXT:    [[TMP2:%.*]] = shufflevector <vscale x 1 x i64> [[DOTSPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-; VEC-COMBINE-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> [[X:%.*]], <vscale x 1 x i64> [[TMP2]], <vscale x 1 x i1> [[MASK]], i32 4)
+; VEC-COMBINE-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> [[X:%.*]], <vscale x 1 x i64> [[TMP2]], <vscale x 1 x i1> [[MASK]], i32 [[EVL:%.*]])
 ; VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> [[TMP3]]
 ;
-; NO-VEC-COMBINE-LABEL: @sdiv_nxv1i64_allonesmask_knownvl(
+; NO-VEC-COMBINE-LABEL: @urem_nxv1i64_allonesmask(
 ; NO-VEC-COMBINE-NEXT:    [[SPLAT:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
 ; NO-VEC-COMBINE-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[SPLAT]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
 ; NO-VEC-COMBINE-NEXT:    [[TMP1:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[Y:%.*]], i64 0
 ; NO-VEC-COMBINE-NEXT:    [[TMP2:%.*]] = shufflevector <vscale x 1 x i64> [[TMP1]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-; NO-VEC-COMBINE-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x i64> @llvm.vp.sdiv.nxv1i64(<vscale x 1 x i64> [[TMP2]], <vscale x 1 x i64> shufflevector (<vscale x 1 x i64> insertelement (<vscale x 1 x i64> poison, i64 42, i32 0), <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> [[MASK]], i32 4)
-; NO-VEC-COMBINE-NEXT:    [[TMP4:%.*]] = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> [[X:%.*]], <vscale x 1 x i64> [[TMP3]], <vscale x 1 x i1> [[MASK]], i32 4)
+; NO-VEC-COMBINE-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x i64> @llvm.vp.urem.nxv1i64(<vscale x 1 x i64> [[TMP2]], <vscale x 1 x i64> shufflevector (<vscale x 1 x i64> insertelement (<vscale x 1 x i64> poison, i64 42, i32 0), <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> [[MASK]], i32 [[EVL:%.*]])
+; NO-VEC-COMBINE-NEXT:    [[TMP4:%.*]] = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> [[X:%.*]], <vscale x 1 x i64> [[TMP3]], <vscale x 1 x i1> [[MASK]], i32 [[EVL]])
 ; NO-VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> [[TMP4]]
 ;
   %splat = insertelement <vscale x 1 x i1> poison, i1 -1, i32 0
   %mask = shufflevector <vscale x 1 x i1> %splat, <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
   %1 = insertelement <vscale x 1 x i64> poison, i64 %y, i64 0
   %2 = shufflevector <vscale x 1 x i64> %1, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-  %3 = call <vscale x 1 x i64> @llvm.vp.sdiv.nxv1i64(<vscale x 1 x i64> %2, <vscale x 1 x i64> shufflevector(<vscale x 1 x i64> insertelement(<vscale x 1 x i64> poison, i64 42, i32 0), <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> %mask, i32 4)
-  %4 = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> %x, <vscale x 1 x i64> %3, <vscale x 1 x i1> %mask, i32 4)
+  %3 = call <vscale x 1 x i64> @llvm.vp.urem.nxv1i64(<vscale x 1 x i64> %2, <vscale x 1 x i64> shufflevector(<vscale x 1 x i64> insertelement(<vscale x 1 x i64> poison, i64 42, i32 0), <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> %mask, i32 %evl)
+  %4 = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> %x, <vscale x 1 x i64> %3, <vscale x 1 x i1> %mask, i32 %evl)
   ret <vscale x 1 x i64> %4
 }
 
-define <vscale x 1 x i64> @sdiv_nxv1i64_anymask_knownvl(<vscale x 1 x i64> %x, i64 %y, <vscale x 1 x i1> %mask) {
-; ALL-LABEL: @sdiv_nxv1i64_anymask_knownvl(
+define <vscale x 1 x i64> @urem_nxv1i64_anymask(<vscale x 1 x i64> %x, i64 %y, <vscale x 1 x i1> %mask, i32 zeroext %evl) {
+; ALL-LABEL: @urem_nxv1i64_anymask(
 ; ALL-NEXT:    [[TMP1:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[Y:%.*]], i64 0
 ; ALL-NEXT:    [[TMP2:%.*]] = shufflevector <vscale x 1 x i64> [[TMP1]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-; ALL-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x i64> @llvm.vp.sdiv.nxv1i64(<vscale x 1 x i64> [[TMP2]], <vscale x 1 x i64> shufflevector (<vscale x 1 x i64> insertelement (<vscale x 1 x i64> poison, i64 42, i32 0), <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> [[MASK:%.*]], i32 4)
-; ALL-NEXT:    [[TMP4:%.*]] = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> [[X:%.*]], <vscale x 1 x i64> [[TMP3]], <vscale x 1 x i1> [[MASK]], i32 4)
+; ALL-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x i64> @llvm.vp.urem.nxv1i64(<vscale x 1 x i64> [[TMP2]], <vscale x 1 x i64> shufflevector (<vscale x 1 x i64> insertelement (<vscale x 1 x i64> poison, i64 42, i32 0), <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> [[MASK:%.*]], i32 [[EVL:%.*]])
+; ALL-NEXT:    [[TMP4:%.*]] = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> [[X:%.*]], <vscale x 1 x i64> [[TMP3]], <vscale x 1 x i1> [[MASK]], i32 [[EVL]])
 ; ALL-NEXT:    ret <vscale x 1 x i64> [[TMP4]]
 ;
   %1 = insertelement <vscale x 1 x i64> poison, i64 %y, i64 0
   %2 = shufflevector <vscale x 1 x i64> %1, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-  %3 = call <vscale x 1 x i64> @llvm.vp.sdiv.nxv1i64(<vscale x 1 x i64> %2, <vscale x 1 x i64> shufflevector(<vscale x 1 x i64> insertelement(<vscale x 1 x i64> poison, i64 42, i32 0), <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> %mask, i32 4)
-  %4 = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> %x, <vscale x 1 x i64> %3, <vscale x 1 x i1> %mask, i32 4)
+  %3 = call <vscale x 1 x i64> @llvm.vp.urem.nxv1i64(<vscale x 1 x i64> %2, <vscale x 1 x i64> shufflevector(<vscale x 1 x i64> insertelement(<vscale x 1 x i64> poison, i64 42, i32 0), <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> %mask, i32 %evl)
+  %4 = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> %x, <vscale x 1 x i64> %3, <vscale x 1 x i1> %mask, i32 %evl)
   ret <vscale x 1 x i64> %4
 }
 
-define <vscale x 1 x i64> @udiv_nxv1i64_allonesmask_knownvl(<vscale x 1 x i64> %x, i64 %y) {
+define <vscale x 1 x i64> @urem_nxv1i64_unspeculatable(i64 %x, i64 %y, i32 zeroext %evl) {
+; ALL-LABEL: @urem_nxv1i64_unspeculatable(
+; ALL-NEXT:    [[MASK_HEAD:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
+; ALL-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[MASK_HEAD]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
+; ALL-NEXT:    [[X_HEAD:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[X:%.*]], i64 0
+; ALL-NEXT:    [[X_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[X_HEAD]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; ALL-NEXT:    [[Y_HEAD:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[X]], i64 0
+; ALL-NEXT:    [[Y_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[Y_HEAD]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; ALL-NEXT:    [[RES:%.*]] = call <vscale x 1 x i64> @llvm.vp.urem.nxv1i64(<vscale x 1 x i64> [[X_SPLAT]], <vscale x 1 x i64> [[Y_SPLAT]], <vscale x 1 x i1> [[MASK]], i32 [[EVL:%.*]])
+; ALL-NEXT:    ret <vscale x 1 x i64> [[RES]]
+;
+  %mask.head = insertelement <vscale x 1 x i1> poison, i1 -1, i32 0
+  %mask = shufflevector <vscale x 1 x i1> %mask.head, <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
+  %x.head = insertelement <vscale x 1 x i64> poison, i64 %x, i64 0
+  %x.splat = shufflevector <vscale x 1 x i64> %x.head, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+  %y.head = insertelement <vscale x 1 x i64> poison, i64 %x, i64 0
+  %y.splat = shufflevector <vscale x 1 x i64> %y.head, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+  %res = call <vscale x 1 x i64> @llvm.vp.urem.nxv1i64(<vscale x 1 x i64> %x.splat, <vscale x 1 x i64> %y.splat, <vscale x 1 x i1> %mask, i32 %evl)
+  ret <vscale x 1 x i64> %res
+}
+
+define <vscale x 1 x i64> @sdiv_nxv1i64_allonesmask_knownvl(i64 %x, i64 %y) {
+; VEC-COMBINE-LABEL: @sdiv_nxv1i64_allonesmask_knownvl(
+; VEC-COMBINE-NEXT:    [[TMP1:%.*]] = sdiv i64 [[X:%.*]], [[X]]
+; VEC-COMBINE-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[TMP1]], i64 0
+; VEC-COMBINE-NEXT:    [[RES:%.*]] = shufflevector <vscale x 1 x i64> [[DOTSPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> [[RES]]
+;
+; NO-VEC-COMBINE-LABEL: @sdiv_nxv1i64_allonesmask_knownvl(
+; NO-VEC-COMBINE-NEXT:    [[MASK_HEAD:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
+; NO-VEC-COMBINE-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[MASK_HEAD]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
+; NO-VEC-COMBINE-NEXT:    [[X_HEAD:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[X:%.*]], i64 0
+; NO-VEC-COMBINE-NEXT:    [[X_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[X_HEAD]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; NO-VEC-COMBINE-NEXT:    [[Y_HEAD:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[X]], i64 0
+; NO-VEC-COMBINE-NEXT:    [[Y_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[Y_HEAD]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; NO-VEC-COMBINE-NEXT:    [[RES:%.*]] = call <vscale x 1 x i64> @llvm.vp.sdiv.nxv1i64(<vscale x 1 x i64> [[X_SPLAT]], <vscale x 1 x i64> [[Y_SPLAT]], <vscale x 1 x i1> [[MASK]], i32 4)
+; NO-VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> [[RES]]
+;
+  %mask.head = insertelement <vscale x 1 x i1> poison, i1 -1, i32 0
+  %mask = shufflevector <vscale x 1 x i1> %mask.head, <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
+  %x.head = insertelement <vscale x 1 x i64> poison, i64 %x, i64 0
+  %x.splat = shufflevector <vscale x 1 x i64> %x.head, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+  %y.head = insertelement <vscale x 1 x i64> poison, i64 %x, i64 0
+  %y.splat = shufflevector <vscale x 1 x i64> %y.head, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+  %res = call <vscale x 1 x i64> @llvm.vp.sdiv.nxv1i64(<vscale x 1 x i64> %x.splat, <vscale x 1 x i64> %y.splat, <vscale x 1 x i1> %mask, i32 4)
+  ret <vscale x 1 x i64> %res
+}
+
+define <vscale x 1 x i64> @sdiv_nxv1i64_anymask_knownvl(i64 %x, i64 %y, <vscale x 1 x i1> %mask) {
+; ALL-LABEL: @sdiv_nxv1i64_anymask_knownvl(
+; ALL-NEXT:    [[X_HEAD:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[X:%.*]], i64 0
+; ALL-NEXT:    [[X_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[X_HEAD]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; ALL-NEXT:    [[Y_HEAD:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[X]], i64 0
+; ALL-NEXT:    [[Y_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[Y_HEAD]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; ALL-NEXT:    [[RES:%.*]] = call <vscale x 1 x i64> @llvm.vp.sdiv.nxv1i64(<vscale x 1 x i64> [[X_SPLAT]], <vscale x 1 x i64> [[Y_SPLAT]], <vscale x 1 x i1> [[MASK:%.*]], i32 4)
+; ALL-NEXT:    ret <vscale x 1 x i64> [[RES]]
+;
+  %x.head = insertelement <vscale x 1 x i64> poison, i64 %x, i64 0
+  %x.splat = shufflevector <vscale x 1 x i64> %x.head, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+  %y.head = insertelement <vscale x 1 x i64> poison, i64 %x, i64 0
+  %y.splat = shufflevector <vscale x 1 x i64> %y.head, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+  %res = call <vscale x 1 x i64> @llvm.vp.sdiv.nxv1i64(<vscale x 1 x i64> %x.splat, <vscale x 1 x i64> %y.splat, <vscale x 1 x i1> %mask, i32 4)
+  ret <vscale x 1 x i64> %res
+}
+
+define <vscale x 1 x i64> @udiv_nxv1i64_allonesmask_knownvl(i64 %x, i64 %y) {
 ; VEC-COMBINE-LABEL: @udiv_nxv1i64_allonesmask_knownvl(
-; VEC-COMBINE-NEXT:    [[SPLAT:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
-; VEC-COMBINE-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[SPLAT]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
-; VEC-COMBINE-NEXT:    [[TMP1:%.*]] = udiv i64 [[Y:%.*]], 42
+; VEC-COMBINE-NEXT:    [[TMP1:%.*]] = udiv i64 [[X:%.*]], [[X]]
 ; VEC-COMBINE-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[TMP1]], i64 0
-; VEC-COMBINE-NEXT:    [[TMP2:%.*]] = shufflevector <vscale x 1 x i64> [[DOTSPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-; VEC-COMBINE-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> [[X:%.*]], <vscale x 1 x i64> [[TMP2]], <vscale x 1 x i1> [[MASK]], i32 4)
-; VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> [[TMP3]]
+; VEC-COMBINE-NEXT:    [[RES:%.*]] = shufflevector <vscale x 1 x i64> [[DOTSPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> [[RES]]
 ;
 ; NO-VEC-COMBINE-LABEL: @udiv_nxv1i64_allonesmask_knownvl(
-; NO-VEC-COMBINE-NEXT:    [[SPLAT:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
-; NO-VEC-COMBINE-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[SPLAT]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
-; NO-VEC-COMBINE-NEXT:    [[TMP1:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[Y:%.*]], i64 0
-; NO-VEC-COMBINE-NEXT:    [[TMP2:%.*]] = shufflevector <vscale x 1 x i64> [[TMP1]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-; NO-VEC-COMBINE-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x i64> @llvm.vp.udiv.nxv1i64(<vscale x 1 x i64> [[TMP2]], <vscale x 1 x i64> shufflevector (<vscale x 1 x i64> insertelement (<vscale x 1 x i64> poison, i64 42, i32 0), <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> [[MASK]], i32 4)
-; NO-VEC-COMBINE-NEXT:    [[TMP4:%.*]] = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> [[X:%.*]], <vscale x 1 x i64> [[TMP3]], <vscale x 1 x i1> [[MASK]], i32 4)
-; NO-VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> [[TMP4]]
-;
-  %splat = insertelement <vscale x 1 x i1> poison, i1 -1, i32 0
-  %mask = shufflevector <vscale x 1 x i1> %splat, <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
-  %1 = insertelement <vscale x 1 x i64> poison, i64 %y, i64 0
-  %2 = shufflevector <vscale x 1 x i64> %1, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-  %3 = call <vscale x 1 x i64> @llvm.vp.udiv.nxv1i64(<vscale x 1 x i64> %2, <vscale x 1 x i64> shufflevector(<vscale x 1 x i64> insertelement(<vscale x 1 x i64> poison, i64 42, i32 0), <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> %mask, i32 4)
-  %4 = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> %x, <vscale x 1 x i64> %3, <vscale x 1 x i1> %mask, i32 4)
-  ret <vscale x 1 x i64> %4
-}
-
-define <vscale x 1 x i64> @udiv_nxv1i64_anymask_knownvl(<vscale x 1 x i64> %x, i64 %y, <vscale x 1 x i1> %mask) {
+; NO-VEC-COMBINE-NEXT:    [[MASK_HEAD:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
+; NO-VEC-COMBINE-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[MASK_HEAD]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
+; NO-VEC-COMBINE-NEXT:    [[X_HEAD:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[X:%.*]], i64 0
+; NO-VEC-COMBINE-NEXT:    [[X_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[X_HEAD]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; NO-VEC-COMBINE-NEXT:    [[Y_HEAD:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[X]], i64 0
+; NO-VEC-COMBINE-NEXT:    [[Y_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[Y_HEAD]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; NO-VEC-COMBINE-NEXT:    [[RES:%.*]] = call <vscale x 1 x i64> @llvm.vp.udiv.nxv1i64(<vscale x 1 x i64> [[X_SPLAT]], <vscale x 1 x i64> [[Y_SPLAT]], <vscale x 1 x i1> [[MASK]], i32 4)
+; NO-VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> [[RES]]
+;
+  %mask.head = insertelement <vscale x 1 x i1> poison, i1 -1, i32 0
+  %mask = shufflevector <vscale x 1 x i1> %mask.head, <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
+  %x.head = insertelement <vscale x 1 x i64> poison, i64 %x, i64 0
+  %x.splat = shufflevector <vscale x 1 x i64> %x.head, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+  %y.head = insertelement <vscale x 1 x i64> poison, i64 %x, i64 0
+  %y.splat = shufflevector <vscale x 1 x i64> %y.head, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+  %res = call <vscale x 1 x i64> @llvm.vp.udiv.nxv1i64(<vscale x 1 x i64> %x.splat, <vscale x 1 x i64> %y.splat, <vscale x 1 x i1> %mask, i32 4)
+  ret <vscale x 1 x i64> %res
+}
+
+define <vscale x 1 x i64> @udiv_nxv1i64_anymask_knownvl(i64 %x, i64 %y, <vscale x 1 x i1> %mask) {
 ; ALL-LABEL: @udiv_nxv1i64_anymask_knownvl(
-; ALL-NEXT:    [[TMP1:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[Y:%.*]], i64 0
-; ALL-NEXT:    [[TMP2:%.*]] = shufflevector <vscale x 1 x i64> [[TMP1]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-; ALL-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x i64> @llvm.vp.udiv.nxv1i64(<vscale x 1 x i64> [[TMP2]], <vscale x 1 x i64> shufflevector (<vscale x 1 x i64> insertelement (<vscale x 1 x i64> poison, i64 42, i32 0), <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> [[MASK:%.*]], i32 4)
-; ALL-NEXT:    [[TMP4:%.*]] = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> [[X:%.*]], <vscale x 1 x i64> [[TMP3]], <vscale x 1 x i1> [[MASK]], i32 4)
-; ALL-NEXT:    ret <vscale x 1 x i64> [[TMP4]]
-;
-  %1 = insertelement <vscale x 1 x i64> poison, i64 %y, i64 0
-  %2 = shufflevector <vscale x 1 x i64> %1, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-  %3 = call <vscale x 1 x i64> @llvm.vp.udiv.nxv1i64(<vscale x 1 x i64> %2, <vscale x 1 x i64> shufflevector(<vscale x 1 x i64> insertelement(<vscale x 1 x i64> poison, i64 42, i32 0), <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> %mask, i32 4)
-  %4 = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> %x, <vscale x 1 x i64> %3, <vscale x 1 x i1> %mask, i32 4)
-  ret <vscale x 1 x i64> %4
-}
-
-define <vscale x 1 x i64> @srem_nxv1i64_allonesmask_knownvl(<vscale x 1 x i64> %x, i64 %y) {
+; ALL-NEXT:    [[X_HEAD:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[X:%.*]], i64 0
+; ALL-NEXT:    [[X_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[X_HEAD]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; ALL-NEXT:    [[Y_HEAD:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[X]], i64 0
+; ALL-NEXT:    [[Y_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[Y_HEAD]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; ALL-NEXT:    [[RES:%.*]] = call <vscale x 1 x i64> @llvm.vp.udiv.nxv1i64(<vscale x 1 x i64> [[X_SPLAT]], <vscale x 1 x i64> [[Y_SPLAT]], <vscale x 1 x i1> [[MASK:%.*]], i32 4)
+; ALL-NEXT:    ret <vscale x 1 x i64> [[RES]]
+;
+  %x.head = insertelement <vscale x 1 x i64> poison, i64 %x, i64 0
+  %x.splat = shufflevector <vscale x 1 x i64> %x.head, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+  %y.head = insertelement <vscale x 1 x i64> poison, i64 %x, i64 0
+  %y.splat = shufflevector <vscale x 1 x i64> %y.head, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+  %res = call <vscale x 1 x i64> @llvm.vp.udiv.nxv1i64(<vscale x 1 x i64> %x.splat, <vscale x 1 x i64> %y.splat, <vscale x 1 x i1> %mask, i32 4)
+  ret <vscale x 1 x i64> %res
+}
+
+define <vscale x 1 x i64> @srem_nxv1i64_allonesmask_knownvl(i64 %x, i64 %y) {
 ; VEC-COMBINE-LABEL: @srem_nxv1i64_allonesmask_knownvl(
-; VEC-COMBINE-NEXT:    [[SPLAT:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
-; VEC-COMBINE-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[SPLAT]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
-; VEC-COMBINE-NEXT:    [[TMP1:%.*]] = srem i64 [[Y:%.*]], 42
+; VEC-COMBINE-NEXT:    [[TMP1:%.*]] = srem i64 [[X:%.*]], [[X]]
 ; VEC-COMBINE-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[TMP1]], i64 0
-; VEC-COMBINE-NEXT:    [[TMP2:%.*]] = shufflevector <vscale x 1 x i64> [[DOTSPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-; VEC-COMBINE-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> [[X:%.*]], <vscale x 1 x i64> [[TMP2]], <vscale x 1 x i1> [[MASK]], i32 4)
-; VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> [[TMP3]]
+; VEC-COMBINE-NEXT:    [[RES:%.*]] = shufflevector <vscale x 1 x i64> [[DOTSPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> [[RES]]
 ;
 ; NO-VEC-COMBINE-LABEL: @srem_nxv1i64_allonesmask_knownvl(
-; NO-VEC-COMBINE-NEXT:    [[SPLAT:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
-; NO-VEC-COMBINE-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[SPLAT]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
-; NO-VEC-COMBINE-NEXT:    [[TMP1:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[Y:%.*]], i64 0
-; NO-VEC-COMBINE-NEXT:    [[TMP2:%.*]] = shufflevector <vscale x 1 x i64> [[TMP1]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-; NO-VEC-COMBINE-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x i64> @llvm.vp.srem.nxv1i64(<vscale x 1 x i64> [[TMP2]], <vscale x 1 x i64> shufflevector (<vscale x 1 x i64> insertelement (<vscale x 1 x i64> poison, i64 42, i32 0), <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> [[MASK]], i32 4)
-; NO-VEC-COMBINE-NEXT:    [[TMP4:%.*]] = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> [[X:%.*]], <vscale x 1 x i64> [[TMP3]], <vscale x 1 x i1> [[MASK]], i32 4)
-; NO-VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> [[TMP4]]
-;
-  %splat = insertelement <vscale x 1 x i1> poison, i1 -1, i32 0
-  %mask = shufflevector <vscale x 1 x i1> %splat, <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
-  %1 = insertelement <vscale x 1 x i64> poison, i64 %y, i64 0
-  %2 = shufflevector <vscale x 1 x i64> %1, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-  %3 = call <vscale x 1 x i64> @llvm.vp.srem.nxv1i64(<vscale x 1 x i64> %2, <vscale x 1 x i64> shufflevector(<vscale x 1 x i64> insertelement(<vscale x 1 x i64> poison, i64 42, i32 0), <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> %mask, i32 4)
-  %4 = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> %x, <vscale x 1 x i64> %3, <vscale x 1 x i1> %mask, i32 4)
-  ret <vscale x 1 x i64> %4
-}
-
-define <vscale x 1 x i64> @srem_nxv1i64_anymask_knownvl(<vscale x 1 x i64> %x, i64 %y, <vscale x 1 x i1> %mask) {
+; NO-VEC-COMBINE-NEXT:    [[MASK_HEAD:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
+; NO-VEC-COMBINE-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[MASK_HEAD]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
+; NO-VEC-COMBINE-NEXT:    [[X_HEAD:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[X:%.*]], i64 0
+; NO-VEC-COMBINE-NEXT:    [[X_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[X_HEAD]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; NO-VEC-COMBINE-NEXT:    [[Y_HEAD:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[X]], i64 0
+; NO-VEC-COMBINE-NEXT:    [[Y_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[Y_HEAD]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; NO-VEC-COMBINE-NEXT:    [[RES:%.*]] = call <vscale x 1 x i64> @llvm.vp.srem.nxv1i64(<vscale x 1 x i64> [[X_SPLAT]], <vscale x 1 x i64> [[Y_SPLAT]], <vscale x 1 x i1> [[MASK]], i32 4)
+; NO-VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> [[RES]]
+;
+  %mask.head = insertelement <vscale x 1 x i1> poison, i1 -1, i32 0
+  %mask = shufflevector <vscale x 1 x i1> %mask.head, <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
+  %x.head = insertelement <vscale x 1 x i64> poison, i64 %x, i64 0
+  %x.splat = shufflevector <vscale x 1 x i64> %x.head, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+  %y.head = insertelement <vscale x 1 x i64> poison, i64 %x, i64 0
+  %y.splat = shufflevector <vscale x 1 x i64> %y.head, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+  %res = call <vscale x 1 x i64> @llvm.vp.srem.nxv1i64(<vscale x 1 x i64> %x.splat, <vscale x 1 x i64> %y.splat, <vscale x 1 x i1> %mask, i32 4)
+  ret <vscale x 1 x i64> %res
+}
+
+define <vscale x 1 x i64> @srem_nxv1i64_anymask_knownvl(i64 %x, i64 %y, <vscale x 1 x i1> %mask) {
 ; ALL-LABEL: @srem_nxv1i64_anymask_knownvl(
-; ALL-NEXT:    [[TMP1:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[Y:%.*]], i64 0
-; ALL-NEXT:    [[TMP2:%.*]] = shufflevector <vscale x 1 x i64> [[TMP1]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-; ALL-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x i64> @llvm.vp.srem.nxv1i64(<vscale x 1 x i64> [[TMP2]], <vscale x 1 x i64> shufflevector (<vscale x 1 x i64> insertelement (<vscale x 1 x i64> poison, i64 42, i32 0), <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> [[MASK:%.*]], i32 4)
-; ALL-NEXT:    [[TMP4:%.*]] = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> [[X:%.*]], <vscale x 1 x i64> [[TMP3]], <vscale x 1 x i1> [[MASK]], i32 4)
-; ALL-NEXT:    ret <vscale x 1 x i64> [[TMP4]]
-;
-  %1 = insertelement <vscale x 1 x i64> poison, i64 %y, i64 0
-  %2 = shufflevector <vscale x 1 x i64> %1, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-  %3 = call <vscale x 1 x i64> @llvm.vp.srem.nxv1i64(<vscale x 1 x i64> %2, <vscale x 1 x i64> shufflevector(<vscale x 1 x i64> insertelement(<vscale x 1 x i64> poison, i64 42, i32 0), <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> %mask, i32 4)
-  %4 = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> %x, <vscale x 1 x i64> %3, <vscale x 1 x i1> %mask, i32 4)
-  ret <vscale x 1 x i64> %4
-}
-
-define <vscale x 1 x i64> @urem_nxv1i64_allonesmask_knownvl(<vscale x 1 x i64> %x, i64 %y) {
+; ALL-NEXT:    [[X_HEAD:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[X:%.*]], i64 0
+; ALL-NEXT:    [[X_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[X_HEAD]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; ALL-NEXT:    [[Y_HEAD:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[X]], i64 0
+; ALL-NEXT:    [[Y_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[Y_HEAD]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; ALL-NEXT:    [[RES:%.*]] = call <vscale x 1 x i64> @llvm.vp.srem.nxv1i64(<vscale x 1 x i64> [[X_SPLAT]], <vscale x 1 x i64> [[Y_SPLAT]], <vscale x 1 x i1> [[MASK:%.*]], i32 4)
+; ALL-NEXT:    ret <vscale x 1 x i64> [[RES]]
+;
+  %x.head = insertelement <vscale x 1 x i64> poison, i64 %x, i64 0
+  %x.splat = shufflevector <vscale x 1 x i64> %x.head, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+  %y.head = insertelement <vscale x 1 x i64> poison, i64 %x, i64 0
+  %y.splat = shufflevector <vscale x 1 x i64> %y.head, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+  %res = call <vscale x 1 x i64> @llvm.vp.srem.nxv1i64(<vscale x 1 x i64> %x.splat, <vscale x 1 x i64> %y.splat, <vscale x 1 x i1> %mask, i32 4)
+  ret <vscale x 1 x i64> %res
+}
+
+define <vscale x 1 x i64> @urem_nxv1i64_allonesmask_knownvl(i64 %x, i64 %y) {
 ; VEC-COMBINE-LABEL: @urem_nxv1i64_allonesmask_knownvl(
-; VEC-COMBINE-NEXT:    [[SPLAT:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
-; VEC-COMBINE-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[SPLAT]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
-; VEC-COMBINE-NEXT:    [[TMP1:%.*]] = urem i64 [[Y:%.*]], 42
+; VEC-COMBINE-NEXT:    [[TMP1:%.*]] = urem i64 [[X:%.*]], [[X]]
 ; VEC-COMBINE-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[TMP1]], i64 0
-; VEC-COMBINE-NEXT:    [[TMP2:%.*]] = shufflevector <vscale x 1 x i64> [[DOTSPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-; VEC-COMBINE-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> [[X:%.*]], <vscale x 1 x i64> [[TMP2]], <vscale x 1 x i1> [[MASK]], i32 4)
-; VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> [[TMP3]]
+; VEC-COMBINE-NEXT:    [[RES:%.*]] = shufflevector <vscale x 1 x i64> [[DOTSPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> [[RES]]
 ;
 ; NO-VEC-COMBINE-LABEL: @urem_nxv1i64_allonesmask_knownvl(
-; NO-VEC-COMBINE-NEXT:    [[SPLAT:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
-; NO-VEC-COMBINE-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[SPLAT]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
-; NO-VEC-COMBINE-NEXT:    [[TMP1:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[Y:%.*]], i64 0
-; NO-VEC-COMBINE-NEXT:    [[TMP2:%.*]] = shufflevector <vscale x 1 x i64> [[TMP1]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-; NO-VEC-COMBINE-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x i64> @llvm.vp.urem.nxv1i64(<vscale x 1 x i64> [[TMP2]], <vscale x 1 x i64> shufflevector (<vscale x 1 x i64> insertelement (<vscale x 1 x i64> poison, i64 42, i32 0), <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> [[MASK]], i32 4)
-; NO-VEC-COMBINE-NEXT:    [[TMP4:%.*]] = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> [[X:%.*]], <vscale x 1 x i64> [[TMP3]], <vscale x 1 x i1> [[MASK]], i32 4)
-; NO-VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> [[TMP4]]
-;
-  %splat = insertelement <vscale x 1 x i1> poison, i1 -1, i32 0
-  %mask = shufflevector <vscale x 1 x i1> %splat, <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
-  %1 = insertelement <vscale x 1 x i64> poison, i64 %y, i64 0
-  %2 = shufflevector <vscale x 1 x i64> %1, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-  %3 = call <vscale x 1 x i64> @llvm.vp.urem.nxv1i64(<vscale x 1 x i64> %2, <vscale x 1 x i64> shufflevector(<vscale x 1 x i64> insertelement(<vscale x 1 x i64> poison, i64 42, i32 0), <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> %mask, i32 4)
-  %4 = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> %x, <vscale x 1 x i64> %3, <vscale x 1 x i1> %mask, i32 4)
-  ret <vscale x 1 x i64> %4
-}
-
-define <vscale x 1 x i64> @urem_nxv1i64_anymask_knownvl(<vscale x 1 x i64> %x, i64 %y, <vscale x 1 x i1> %mask) {
+; NO-VEC-COMBINE-NEXT:    [[MASK_HEAD:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
+; NO-VEC-COMBINE-NEXT:    [[MASK:%.*]] = shufflevector <vscale x 1 x i1> [[MASK_HEAD]], <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
+; NO-VEC-COMBINE-NEXT:    [[X_HEAD:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[X:%.*]], i64 0
+; NO-VEC-COMBINE-NEXT:    [[X_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[X_HEAD]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; NO-VEC-COMBINE-NEXT:    [[Y_HEAD:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[X]], i64 0
+; NO-VEC-COMBINE-NEXT:    [[Y_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[Y_HEAD]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; NO-VEC-COMBINE-NEXT:    [[RES:%.*]] = call <vscale x 1 x i64> @llvm.vp.urem.nxv1i64(<vscale x 1 x i64> [[X_SPLAT]], <vscale x 1 x i64> [[Y_SPLAT]], <vscale x 1 x i1> [[MASK]], i32 4)
+; NO-VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> [[RES]]
+;
+  %mask.head = insertelement <vscale x 1 x i1> poison, i1 -1, i32 0
+  %mask = shufflevector <vscale x 1 x i1> %mask.head, <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer
+  %x.head = insertelement <vscale x 1 x i64> poison, i64 %x, i64 0
+  %x.splat = shufflevector <vscale x 1 x i64> %x.head, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+  %y.head = insertelement <vscale x 1 x i64> poison, i64 %x, i64 0
+  %y.splat = shufflevector <vscale x 1 x i64> %y.head, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+  %res = call <vscale x 1 x i64> @llvm.vp.urem.nxv1i64(<vscale x 1 x i64> %x.splat, <vscale x 1 x i64> %y.splat, <vscale x 1 x i1> %mask, i32 4)
+  ret <vscale x 1 x i64> %res
+}
+
+define <vscale x 1 x i64> @urem_nxv1i64_anymask_knownvl(i64 %x, i64 %y, <vscale x 1 x i1> %mask) {
 ; ALL-LABEL: @urem_nxv1i64_anymask_knownvl(
-; ALL-NEXT:    [[TMP1:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[Y:%.*]], i64 0
-; ALL-NEXT:    [[TMP2:%.*]] = shufflevector <vscale x 1 x i64> [[TMP1]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-; ALL-NEXT:    [[TMP3:%.*]] = call <vscale x 1 x i64> @llvm.vp.urem.nxv1i64(<vscale x 1 x i64> [[TMP2]], <vscale x 1 x i64> shufflevector (<vscale x 1 x i64> insertelement (<vscale x 1 x i64> poison, i64 42, i32 0), <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> [[MASK:%.*]], i32 4)
-; ALL-NEXT:    [[TMP4:%.*]] = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> [[X:%.*]], <vscale x 1 x i64> [[TMP3]], <vscale x 1 x i1> [[MASK]], i32 4)
-; ALL-NEXT:    ret <vscale x 1 x i64> [[TMP4]]
-;
-  %1 = insertelement <vscale x 1 x i64> poison, i64 %y, i64 0
-  %2 = shufflevector <vscale x 1 x i64> %1, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-  %3 = call <vscale x 1 x i64> @llvm.vp.urem.nxv1i64(<vscale x 1 x i64> %2, <vscale x 1 x i64> shufflevector(<vscale x 1 x i64> insertelement(<vscale x 1 x i64> poison, i64 42, i32 0), <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i1> %mask, i32 4)
-  %4 = call <vscale x 1 x i64> @llvm.vp.mul.nxv1i64(<vscale x 1 x i64> %x, <vscale x 1 x i64> %3, <vscale x 1 x i1> %mask, i32 4)
-  ret <vscale x 1 x i64> %4
+; ALL-NEXT:    [[X_HEAD:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[X:%.*]], i64 0
+; ALL-NEXT:    [[X_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[X_HEAD]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; ALL-NEXT:    [[Y_HEAD:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[X]], i64 0
+; ALL-NEXT:    [[Y_SPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[Y_HEAD]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; ALL-NEXT:    [[RES:%.*]] = call <vscale x 1 x i64> @llvm.vp.urem.nxv1i64(<vscale x 1 x i64> [[X_SPLAT]], <vscale x 1 x i64> [[Y_SPLAT]], <vscale x 1 x i1> [[MASK:%.*]], i32 4)
+; ALL-NEXT:    ret <vscale x 1 x i64> [[RES]]
+;
+  %x.head = insertelement <vscale x 1 x i64> poison, i64 %x, i64 0
+  %x.splat = shufflevector <vscale x 1 x i64> %x.head, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+  %y.head = insertelement <vscale x 1 x i64> poison, i64 %x, i64 0
+  %y.splat = shufflevector <vscale x 1 x i64> %y.head, <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+  %res = call <vscale x 1 x i64> @llvm.vp.urem.nxv1i64(<vscale x 1 x i64> %x.splat, <vscale x 1 x i64> %y.splat, <vscale x 1 x i1> %mask, i32 4)
+  ret <vscale x 1 x i64> %res
 }
 
 define <vscale x 1 x i64> @ashr_nxv1i64_allonesmask(<vscale x 1 x i64> %x, i64 %y, i32 zeroext %evl) {
diff --git a/llvm/test/Unit/lit.cfg.py b/llvm/test/Unit/lit.cfg.py
index f15c30dbcd..61296d7ea0 100644
--- a/llvm/test/Unit/lit.cfg.py
+++ b/llvm/test/Unit/lit.cfg.py
@@ -19,7 +19,11 @@
 config.test_source_root = config.test_exec_root
 
 # testFormat: The test format to use to interpret tests.
-config.test_format = lit.formats.GoogleTest(config.llvm_build_mode, "Tests")
+config.test_format = lit.formats.GoogleTest(
+    config.llvm_build_mode,
+    "Tests",
+    run_under=config.gtest_run_under,
+)
 
 # Propagate the temp directory. Windows requires this because it uses \Windows\
 # if none of these are present.
diff --git a/llvm/test/Unit/lit.site.cfg.py.in b/llvm/test/Unit/lit.site.cfg.py.in
index 1d7d765801..3536a34f79 100644
--- a/llvm/test/Unit/lit.site.cfg.py.in
+++ b/llvm/test/Unit/lit.site.cfg.py.in
@@ -7,6 +7,7 @@ config.llvm_obj_root = path(r"@LLVM_BINARY_DIR@")
 config.llvm_tools_dir = lit_config.substitute(path(r"@LLVM_TOOLS_DIR@"))
 config.llvm_build_mode = lit_config.substitute("@LLVM_BUILD_MODE@")
 config.shlibdir = lit_config.substitute(path(r"@SHLIBDIR@"))
+config.gtest_run_under = lit_config.substitute(r"@LLVM_GTEST_RUN_UNDER@")
 
 # Let the main config do the real work.
 lit_config.load_config(
diff --git a/llvm/test/YAMLParser/plain-characters.test b/llvm/test/YAMLParser/plain-characters.test
new file mode 100644
index 0000000000..f22016bcb9
--- /dev/null
+++ b/llvm/test/YAMLParser/plain-characters.test
@@ -0,0 +1,30 @@
+# RUN: yaml-bench -canonical %s | FileCheck %s
+# Example from https://yaml.org/spec/1.2.2/#example-plain-characters
+
+# Outside flow collection:
+- ::vector
+- ": - ()"
+- Up, up, and away!
+- -123
+- https://example.com/foo#bar
+# Inside flow collection:
+- [ ::vector,
+  ": - ()",
+  "Up, up and away!",
+  -123,
+  https://example.com/foo#bar ]
+
+# CHECK: !!seq [
+# CHECK-NEXT:   !!str "::vector",
+# CHECK-NEXT:   !!str ": - ()",
+# CHECK-NEXT:   !!str "Up, up, and away!",
+# CHECK-NEXT:   !!str "-123",
+# CHECK-NEXT:   !!str "https://example.com/foo#bar",
+# CHECK-NEXT:   !!seq [
+# CHECK-NEXT:     !!str "::vector",
+# CHECK-NEXT:     !!str ": - ()",
+# CHECK-NEXT:     !!str "Up, up and away!",
+# CHECK-NEXT:     !!str "-123",
+# CHECK-NEXT:     !!str "https://example.com/foo#bar",
+# CHECK-NEXT:   ],
+# CHECK-NEXT: ]
diff --git a/llvm/test/tools/llc/aix-pic-setting.ll b/llvm/test/tools/llc/aix-pic-setting.ll
index 70e08e2513..3537baf1cd 100644
--- a/llvm/test/tools/llc/aix-pic-setting.ll
+++ b/llvm/test/tools/llc/aix-pic-setting.ll
@@ -6,4 +6,4 @@
 ; RUN: not llc -mtriple=powerpc64-ibm-aix --relocation-model=ropi-rwpi < %s 2>&1 | FileCheck --check-prefix=CHECK-NON-PIC %s
 
 ; CHECK-NOT: {{.}}
-; CHECK-NON-PIC: error: '<stdin>': invalid relocation model, AIX only supports PIC
+; CHECK-NON-PIC: invalid relocation model, AIX only supports PIC
diff --git a/llvm/test/tools/llvm-dwp/X86/cu_tu_units_manual_v5_invalid.s b/llvm/test/tools/llvm-dwp/X86/cu_tu_units_manual_v5_invalid.s
new file mode 100644
index 0000000000..1f63b21217
--- /dev/null
+++ b/llvm/test/tools/llvm-dwp/X86/cu_tu_units_manual_v5_invalid.s
@@ -0,0 +1,73 @@
+# This test checks that llvm-dwarfdump correctly reports errors when parsing
+# DWARF Unit Headers in DWP files
+
+# RUN: llvm-mc -triple x86_64-unknown-linux %s -filetype=obj -o %t.o \
+# RUN:         -split-dwarf-file=%t.dwo -dwarf-version=5
+# RUN: llvm-dwp %t.dwo -o %t.dwp
+# RUN: llvm-dwarfdump -debug-info -debug-cu-index -debug-tu-index \
+# RUN:                -manaully-generate-unit-index %t.dwp 2>&1 | FileCheck %s
+
+## Note: In order to check whether the type unit index is generated
+## there is no need to add the missing DIEs for the structure type of the type unit.
+
+# CHECK-NOT: .debug_info.dwo contents:
+
+# CHECK-DAG: .debug_cu_index contents:
+# CHECK: warning: Failed to parse CU header in DWP file: DWARF unit at offset 0x00000000 has unsupported version 6, supported are 2-5
+
+# CHECK-DAG: .debug_tu_index contents:
+# CHECK: warning: Failed to parse CU header in DWP file: DWARF unit at offset 0x00000000 has unsupported version 6, supported are 2-5
+
+    .section	.debug_info.dwo,"e",@progbits
+    .long	.Ldebug_info_dwo_end0-.Ldebug_info_dwo_start0 # Length of Unit
+.Ldebug_info_dwo_start0:
+    .short	6                               # DWARF version number
+    .byte	6                               # DWARF Unit Type (DW_UT_split_type)
+    .byte	8                               # Address Size (in bytes)
+    .long	0                               # Offset Into Abbrev. Section
+    .quad	5657452045627120676             # Type Signature
+    .long	25                              # Type DIE Offset
+    .byte	2                               # Abbrev [2] DW_TAG_type_unit
+    .byte	3                               # Abbrev [3] DW_TAG_structure_type
+    .byte	0                               # End Of Children Mark
+.Ldebug_info_dwo_end0:
+    .section	.debug_info.dwo,"e",@progbits
+    .long	.Ldebug_info_dwo_end1-.Ldebug_info_dwo_start1 # Length of Unit
+.Ldebug_info_dwo_start1:
+    .short	6                               # DWARF version number
+    .byte	6                               # DWARF Unit Type (DW_UT_split_type)
+    .byte	8                               # Address Size (in bytes)
+    .long	0                               # Offset Into Abbrev. Section
+    .quad	-8528522068957683993            # Type Signature
+    .long	25                              # Type DIE Offset
+    .byte	4                               # Abbrev [4] DW_TAG_type_unit
+    .byte	5                               # Abbrev [5] DW_TAG_structure_type
+    .byte	0                               # End Of Children Mark
+.Ldebug_info_dwo_end1:
+    .section	.debug_info.dwo,"e",@progbits
+    .long	.Ldebug_info_dwo_end2-.Ldebug_info_dwo_start2 # Length of Unit
+.Ldebug_info_dwo_start2:
+    .short	6                               # DWARF version number
+    .byte	5                               # DWARF Unit Type (DW_UT_split_compile)
+    .byte	8                               # Address Size (in bytes)
+    .long	0                               # Offset Into Abbrev. Section
+    .quad	1152943841751211454
+    .byte	1                               # Abbrev [1] DW_TAG_compile_unit
+.Ldebug_info_dwo_end2:
+    .section	.debug_abbrev.dwo,"e",@progbits
+    .byte	1                               # Abbreviation Code
+    .byte	17                              # DW_TAG_compile_unit
+    .byte	0                               # DW_CHILDREN_no
+    .byte	0                               # EOM(1)
+    .byte	0                               # EOM(2)
+    .byte	2                               # Abbreviation Code
+    .byte	65                              # DW_TAG_type_unit
+    .byte	1                               # DW_CHILDREN_yes
+    .byte	0                               # EOM
+    .byte	0                               # EOM
+    .byte	4                               # Abbreviation Code
+    .byte	65                              # DW_TAG_type_unit
+    .byte	1                               # DW_CHILDREN_yes
+    .byte	0                               # EOM
+    .byte	0                               # EOM
+    .byte	0                               # EOM
diff --git a/llvm/test/tools/llvm-lib/arm64ec-implib.test b/llvm/test/tools/llvm-lib/arm64ec-implib.test
new file mode 100644
index 0000000000..ee8b134d06
--- /dev/null
+++ b/llvm/test/tools/llvm-lib/arm64ec-implib.test
@@ -0,0 +1,54 @@
+Test creating ARM64EC importlib.
+
+RUN: split-file %s %t.dir && cd %t.dir
+RUN: llvm-lib -machine:arm64ec -def:test.def -out:test.lib
+
+RUN: llvm-nm --print-armap test.lib | FileCheck -check-prefix=ARMAP %s
+
+ARMAP:      Archive EC map
+ARMAP-NEXT: __IMPORT_DESCRIPTOR_test in test.dll
+ARMAP-NEXT: __NULL_IMPORT_DESCRIPTOR in test.dll
+ARMAP-NEXT: __imp_dataexp in test.dll
+ARMAP-NEXT: __imp_funcexp in test.dll
+ARMAP-NEXT: funcexp in test.dll
+ARMAP-NEXT: test_NULL_THUNK_DATA in test.dll
+
+RUN: llvm-readobj test.lib | FileCheck -check-prefix=READOBJ %s
+
+READOBJ:      File: test.lib(test.dll)
+READOBJ-NEXT: Format: COFF-ARM64EC
+READOBJ-NEXT: Arch: aarch64
+READOBJ-NEXT: AddressSize: 64bit
+READOBJ-EMPTY:
+READOBJ-NEXT: File: test.lib(test.dll)
+READOBJ-NEXT: Format: COFF-ARM64EC
+READOBJ-NEXT: Arch: aarch64
+READOBJ-NEXT: AddressSize: 64bit
+READOBJ-EMPTY:
+READOBJ-NEXT: File: test.lib(test.dll)
+READOBJ-NEXT: Format: COFF-ARM64
+READOBJ-NEXT: Arch: aarch64
+READOBJ-NEXT: AddressSize: 64bit
+READOBJ-EMPTY:
+READOBJ-NEXT: File: test.dll
+READOBJ-NEXT: Format: COFF-import-file
+READOBJ-NEXT: Type: code
+READOBJ-NEXT: Name type: name
+READOBJ-NEXT: Symbol: __imp_funcexp
+READOBJ-NEXT: Symbol: funcexp
+READOBJ-EMPTY:
+READOBJ-NEXT: File: test.dll
+READOBJ-NEXT: Format: COFF-import-file
+READOBJ-NEXT: Type: data
+READOBJ-NEXT: Name type: name
+READOBJ-NEXT: Symbol: __imp_dataexp
+
+Creating a new lib containing the existing lib:
+RUN: llvm-lib -machine:arm64ec test.lib -out:test2.lib
+RUN: llvm-nm --print-armap test2.lib | FileCheck -check-prefix=ARMAP %s
+
+#--- test.def
+LIBRARY test.dll
+EXPORTS
+    funcexp
+    dataexp DATA
diff --git a/llvm/test/tools/llvm-lto2/X86/pipeline.ll b/llvm/test/tools/llvm-lto2/X86/pipeline.ll
index 37568dcd0b..9896cf4da2 100644
--- a/llvm/test/tools/llvm-lto2/X86/pipeline.ll
+++ b/llvm/test/tools/llvm-lto2/X86/pipeline.ll
@@ -15,7 +15,7 @@
 ; is accepted).
 ; RUN: llvm-lto2 run %t1.bc -o %t.o -r %t1.bc,patatino,px
 
-target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @patatino() {
diff --git a/llvm/test/tools/llvm-lto2/X86/slp-vectorize-pm.ll b/llvm/test/tools/llvm-lto2/X86/slp-vectorize-pm.ll
index f5cadc127e..6b54981c80 100644
--- a/llvm/test/tools/llvm-lto2/X86/slp-vectorize-pm.ll
+++ b/llvm/test/tools/llvm-lto2/X86/slp-vectorize-pm.ll
@@ -26,7 +26,7 @@
 ; CHECK-O2-LPV: = !{!"llvm.loop.isvectorized", i32 1}
 ; CHECK-O3-LPV: = !{!"llvm.loop.isvectorized", i32 1}
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-i64:64-i128:128-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define i32 @foo(ptr %a) {
diff --git a/llvm/test/tools/llvm-lto2/X86/stats-file-option.ll b/llvm/test/tools/llvm-lto2/X86/stats-file-option.ll
index 7daedd7017..99c2486827 100644
--- a/llvm/test/tools/llvm-lto2/X86/stats-file-option.ll
+++ b/llvm/test/tools/llvm-lto2/X86/stats-file-option.ll
@@ -6,7 +6,7 @@
 ; RUN: llvm-lto2 run %t1.bc -o %t.o -r %t1.bc,patatino,px -stats-file=%t2.stats
 ; RUN: FileCheck --input-file=%t2.stats %s
 
-target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @patatino() {
diff --git a/llvm/test/tools/llvm-mca/AArch64/Cortex/A57-writeback.s b/llvm/test/tools/llvm-mca/AArch64/Cortex/A57-writeback.s
index 6993401ffa..5248392188 100644
--- a/llvm/test/tools/llvm-mca/AArch64/Cortex/A57-writeback.s
+++ b/llvm/test/tools/llvm-mca/AArch64/Cortex/A57-writeback.s
@@ -1162,28 +1162,28 @@ add x0, x27, 1
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2504
+# CHECK-NEXT: Total Cycles:      507
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.60
-# CHECK-NEXT: IPC:               0.40
+# CHECK-NEXT: uOps Per Cycle:    2.96
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          012345678
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeER  .    .    .    .  .   ld1	{ v1.1d }, [x27], #8
-# CHECK-NEXT: [0,1]     D=====eER .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D====eeeeeER  .    .    .  .   ld1	{ v1.2d }, [x27], #16
-# CHECK-NEXT: [0,3]     .D=========eER .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D========eeeeeER  .    .  .   ld1	{ v1.2s }, [x27], #8
-# CHECK-NEXT: [0,5]     . D=============eER .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D============eeeeeER  .  .   ld1	{ v1.4h }, [x27], #8
-# CHECK-NEXT: [0,7]     .  D=================eER .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D================eeeeeER.   ld1	{ v1.4s }, [x27], #16
-# CHECK-NEXT: [0,9]     .   D=====================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeER  ..   ld1	{ v1.1d }, [x27], #8
+# CHECK-NEXT: [0,1]     D=eE---R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeER ..   ld1	{ v1.2d }, [x27], #16
+# CHECK-NEXT: [0,3]     .D=eE---R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeER..   ld1	{ v1.2s }, [x27], #8
+# CHECK-NEXT: [0,5]     . D=eE---R..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeER.   ld1	{ v1.4h }, [x27], #8
+# CHECK-NEXT: [0,7]     .  D=eE---R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeER   ld1	{ v1.4s }, [x27], #16
+# CHECK-NEXT: [0,9]     .   D=eE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1193,43 +1193,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.1d }, [x27], #8
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       ld1	{ v1.2d }, [x27], #16
-# CHECK-NEXT: 3.     1     10.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     9.0    0.0    0.0       ld1	{ v1.2s }, [x27], #8
-# CHECK-NEXT: 5.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     13.0   0.0    0.0       ld1	{ v1.4h }, [x27], #8
-# CHECK-NEXT: 7.     1     18.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     17.0   0.0    0.0       ld1	{ v1.4s }, [x27], #16
-# CHECK-NEXT: 9.     1     22.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     11.5   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.2d }, [x27], #16
+# CHECK-NEXT: 3.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1	{ v1.2s }, [x27], #8
+# CHECK-NEXT: 5.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld1	{ v1.4h }, [x27], #8
+# CHECK-NEXT: 7.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ld1	{ v1.4s }, [x27], #16
+# CHECK-NEXT: 9.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    1.5       <total>
 
 # CHECK:      [1] Code Region - G02
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2504
+# CHECK-NEXT: Total Cycles:      507
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.60
-# CHECK-NEXT: IPC:               0.40
+# CHECK-NEXT: uOps Per Cycle:    2.96
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          012345678
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeER  .    .    .    .  .   ld1	{ v1.8b }, [x27], #8
-# CHECK-NEXT: [0,1]     D=====eER .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D====eeeeeER  .    .    .  .   ld1	{ v1.8h }, [x27], #16
-# CHECK-NEXT: [0,3]     .D=========eER .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D========eeeeeER  .    .  .   ld1	{ v1.16b }, [x27], #16
-# CHECK-NEXT: [0,5]     . D=============eER .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D============eeeeeER  .  .   ld1	{ v1.1d }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D=================eER .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D================eeeeeER.   ld1	{ v1.2d }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D=====================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeER  ..   ld1	{ v1.8b }, [x27], #8
+# CHECK-NEXT: [0,1]     D=eE---R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeER ..   ld1	{ v1.8h }, [x27], #16
+# CHECK-NEXT: [0,3]     .D=eE---R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeER..   ld1	{ v1.16b }, [x27], #16
+# CHECK-NEXT: [0,5]     . D=eE---R..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeER.   ld1	{ v1.1d }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D=eE---R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeER   ld1	{ v1.2d }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D=eE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1239,43 +1239,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.8b }, [x27], #8
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       ld1	{ v1.8h }, [x27], #16
-# CHECK-NEXT: 3.     1     10.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     9.0    0.0    0.0       ld1	{ v1.16b }, [x27], #16
-# CHECK-NEXT: 5.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     13.0   0.0    0.0       ld1	{ v1.1d }, [x27], x28
-# CHECK-NEXT: 7.     1     18.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     17.0   0.0    0.0       ld1	{ v1.2d }, [x27], x28
-# CHECK-NEXT: 9.     1     22.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     11.5   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.8h }, [x27], #16
+# CHECK-NEXT: 3.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1	{ v1.16b }, [x27], #16
+# CHECK-NEXT: 5.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld1	{ v1.1d }, [x27], x28
+# CHECK-NEXT: 7.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ld1	{ v1.2d }, [x27], x28
+# CHECK-NEXT: 9.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    1.5       <total>
 
 # CHECK:      [2] Code Region - G03
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2504
+# CHECK-NEXT: Total Cycles:      507
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.60
-# CHECK-NEXT: IPC:               0.40
+# CHECK-NEXT: uOps Per Cycle:    2.96
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          012345678
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeER  .    .    .    .  .   ld1	{ v1.2s }, [x27], x28
-# CHECK-NEXT: [0,1]     D=====eER .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D====eeeeeER  .    .    .  .   ld1	{ v1.4h }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=========eER .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D========eeeeeER  .    .  .   ld1	{ v1.4s }, [x27], x28
-# CHECK-NEXT: [0,5]     . D=============eER .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D============eeeeeER  .  .   ld1	{ v1.8b }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D=================eER .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D================eeeeeER.   ld1	{ v1.8h }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D=====================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeER  ..   ld1	{ v1.2s }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE---R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeER ..   ld1	{ v1.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE---R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeER..   ld1	{ v1.4s }, [x27], x28
+# CHECK-NEXT: [0,5]     . D=eE---R..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeER.   ld1	{ v1.8b }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D=eE---R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeER   ld1	{ v1.8h }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D=eE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1285,43 +1285,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.2s }, [x27], x28
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       ld1	{ v1.4h }, [x27], x28
-# CHECK-NEXT: 3.     1     10.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     9.0    0.0    0.0       ld1	{ v1.4s }, [x27], x28
-# CHECK-NEXT: 5.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     13.0   0.0    0.0       ld1	{ v1.8b }, [x27], x28
-# CHECK-NEXT: 7.     1     18.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     17.0   0.0    0.0       ld1	{ v1.8h }, [x27], x28
-# CHECK-NEXT: 9.     1     22.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     11.5   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.4h }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1	{ v1.4s }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld1	{ v1.8b }, [x27], x28
+# CHECK-NEXT: 7.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ld1	{ v1.8h }, [x27], x28
+# CHECK-NEXT: 9.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    1.5       <total>
 
 # CHECK:      [3] Code Region - G04
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2604
+# CHECK-NEXT: Total Cycles:      607
 # CHECK-NEXT: Total uOps:        1600
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.61
-# CHECK-NEXT: IPC:               0.38
+# CHECK-NEXT: uOps Per Cycle:    2.64
+# CHECK-NEXT: IPC:               1.65
 # CHECK-NEXT: Block RThroughput: 6.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeER  .    .    .    .   .   ld1	{ v1.16b }, [x27], x28
-# CHECK-NEXT: [0,1]     D=====eER .    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D====eeeeeER  .    .    .   .   ld1	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: [0,3]     .D=========eER .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D========eeeeeeER .    .   .   ld1	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: [0,5]     .  D=============eER.    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=============eeeeeER .   .   ld1	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: [0,7]     .   D=================eER.   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D=================eeeeeER.   ld1	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: [0,9]     .    D=====================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeER  . .   ld1	{ v1.16b }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE---R  . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeER . .   ld1	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: [0,3]     .D=eE---R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeER .   ld1	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: [0,5]     .  DeE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeeeeER.   ld1	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: [0,7]     .   D=eE---R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D=eeeeeER   ld1	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: [0,9]     .    D=eE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1331,43 +1331,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.16b }, [x27], x28
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       ld1	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: 3.     1     10.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     9.0    0.0    0.0       ld1	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: 5.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     14.0   0.0    0.0       ld1	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: 7.     1     18.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     18.0   0.0    0.0       ld1	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: 9.     1     22.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     11.7   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: 3.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: 5.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    1.0    0.0       ld1	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: 7.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     2.0    0.0    0.0       ld1	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: 9.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.6    0.2    1.6       <total>
 
 # CHECK:      [4] Code Region - G05
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2804
+# CHECK-NEXT: Total Cycles:      807
 # CHECK-NEXT: Total uOps:        1800
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.64
-# CHECK-NEXT: IPC:               0.36
+# CHECK-NEXT: uOps Per Cycle:    2.23
+# CHECK-NEXT: IPC:               1.24
 # CHECK-NEXT: Block RThroughput: 8.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          01
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    ..   ld1	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: [0,1]     .D=====eER.    .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=====eeeeeER .    .    .    ..   ld1	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: [0,3]     . D=========eER.    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .  D========eeeeeeER.    .    ..   ld1	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: [0,5]     .   D=============eER    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    D============eeeeeeER    ..   ld1	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: [0,7]     .    .D=================eER   ..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .D=================eeeeeER.   ld1	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: [0,9]     .    . D=====================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER .   .   ld1	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: [0,1]     .DeE----R .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .D=eeeeeER.   .   ld1	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: [0,3]     . D=eE---R.   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .  DeeeeeeER  .   ld1	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: [0,5]     .   DeE----R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    DeeeeeeER.   ld1	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: [0,7]     .    .DeE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .D=eeeeeER   ld1	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: [0,9]     .    . D=eE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1377,43 +1377,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     6.0    0.0    0.0       ld1	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: 3.     1     10.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     9.0    0.0    0.0       ld1	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: 5.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     13.0   0.0    0.0       ld1	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: 7.     1     18.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     18.0   0.0    0.0       ld1	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: 9.     1     22.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     11.7   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    1.0    0.0       ld1	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: 3.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: 5.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld1	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: 7.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     2.0    1.0    0.0       ld1	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: 9.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.4    0.4    1.8       <total>
 
 # CHECK:      [5] Code Region - G06
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2704
+# CHECK-NEXT: Total Cycles:      707
 # CHECK-NEXT: Total uOps:        1700
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.63
-# CHECK-NEXT: IPC:               0.37
+# CHECK-NEXT: uOps Per Cycle:    2.40
+# CHECK-NEXT: IPC:               1.41
 # CHECK-NEXT: Block RThroughput: 7.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     0123
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .   ld1	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: [0,1]     .D=====eER.    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=====eeeeeER .    .    .    .   ld1	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: [0,3]     . D=========eER.    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=========eeeeeER .    .    .   ld1	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: [0,5]     .  D=============eER.    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .   D============eeeeeeER.    .   ld1	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: [0,7]     .    D=================eER    .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    D=================eeeeeER.   ld1	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: [0,9]     .    .D=====================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER .  .   ld1	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE----R .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .D=eeeeeER.  .   ld1	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,3]     . D=eE---R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeeeeER  .   ld1	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,5]     .  D=eE---R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .   DeeeeeeER.   ld1	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,7]     .    DeE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    D=eeeeeER   ld1	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .D=eE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1423,43 +1423,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     6.0    0.0    0.0       ld1	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: 3.     1     10.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     10.0   0.0    0.0       ld1	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: 5.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     13.0   0.0    0.0       ld1	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: 7.     1     18.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     18.0   0.0    0.0       ld1	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: 9.     1     22.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     11.8   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    1.0    0.0       ld1	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld1	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     2.0    1.0    0.0       ld1	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: 9.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.6    0.3    1.7       <total>
 
 # CHECK:      [6] Code Region - G07
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3104
+# CHECK-NEXT: Total Cycles:      1107
 # CHECK-NEXT: Total uOps:        2100
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.68
-# CHECK-NEXT: IPC:               0.32
+# CHECK-NEXT: uOps Per Cycle:    1.90
+# CHECK-NEXT: IPC:               0.90
 # CHECK-NEXT: Block RThroughput: 11.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          01234
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     01234567
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .   .   ld1	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: [0,1]     .D=====eER.    .    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D====eeeeeeER.    .    .    .   .   ld1	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: [0,3]     .  D=========eER    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D========eeeeeeER    .    .   .   ld1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: [0,5]     .    D=============eER   .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D============eeeeeeeER  .   .   ld1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: [0,7]     .    . D==================eER .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .  D=================eeeeeeER.   ld1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: [0,9]     .    .   D======================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER .    . .   ld1	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE----R .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeER    . .   ld1	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE----R    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeER  . .   ld1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
+# CHECK-NEXT: [0,5]     .    DeE----R  . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeER .   ld1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: [0,7]     .    . DeE-----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  D=eeeeeeER   ld1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: [0,9]     .    .   D=eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1469,43 +1469,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       ld1	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: 3.     1     10.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     9.0    0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: 5.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     13.0   0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: 7.     1     19.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     18.0   0.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: 9.     1     23.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     11.8   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld1	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
+# CHECK-NEXT: 5.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: 7.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     2.0    2.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: 9.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.2    0.6    2.1       <total>
 
 # CHECK:      [7] Code Region - G08
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3304
+# CHECK-NEXT: Total Cycles:      1307
 # CHECK-NEXT: Total uOps:        2300
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.70
-# CHECK-NEXT: IPC:               0.30
+# CHECK-NEXT: uOps Per Cycle:    1.76
+# CHECK-NEXT: IPC:               0.77
 # CHECK-NEXT: Block RThroughput: 13.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .    ..   ld1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: [0,1]     .D=====eER.    .    .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D====eeeeeeeER    .    .    .    ..   ld1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: [0,3]     .  D==========eER   .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D=========eeeeeeER   .    .    ..   ld1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: [0,5]     .    D==============eER  .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D=============eeeeeeeER .    ..   ld1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: [0,7]     .    . D===================eER.    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .  D==================eeeeeeeER.   ld1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: [0,9]     .    .   D========================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER .    .   .   ld1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: [0,1]     .DeE----R .    .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeER   .   .   ld1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: [0,3]     .  DeE-----R   .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   D=eeeeeeER .   .   ld1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: [0,5]     .    D=eE----R .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .D=eeeeeeeER  .   ld1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: [0,7]     .    . D=eE-----R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  D==eeeeeeeER   ld1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: [0,9]     .    .   D==eE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1515,43 +1515,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: 3.     1     11.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     10.0   0.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: 5.     1     15.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     14.0   0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: 7.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     19.0   0.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: 9.     1     25.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     12.6   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: 3.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    2.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: 5.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    1.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: 7.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    2.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: 9.     1     3.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.8    0.7    2.3       <total>
 
 # CHECK:      [8] Code Region - G09
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3204
+# CHECK-NEXT: Total Cycles:      1207
 # CHECK-NEXT: Total uOps:        2200
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.69
-# CHECK-NEXT: IPC:               0.31
+# CHECK-NEXT: uOps Per Cycle:    1.82
+# CHECK-NEXT: IPC:               0.83
 # CHECK-NEXT: Block RThroughput: 12.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          012345
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     012345678
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .    .   ld1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: [0,1]     .D=====eER.    .    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D====eeeeeeeER    .    .    .    .   ld1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: [0,3]     .  D==========eER   .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D=========eeeeeeER   .    .    .   ld1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: [0,5]     .    D==============eER  .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D=============eeeeeeER  .    .   ld1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: [0,7]     .    . D==================eER .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .  D=================eeeeeeeER.   ld1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: [0,9]     .    .   D=======================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER .    .  .   ld1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE----R .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeER   .  .   ld1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE-----R   .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   D=eeeeeeER .  .   ld1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,5]     .    D=eE----R .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .D=eeeeeeER  .   ld1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,7]     .    . D=eE----R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  D=eeeeeeeER   ld1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .   D=eE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1561,43 +1561,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: 3.     1     11.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     10.0   0.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: 5.     1     15.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     14.0   0.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: 7.     1     19.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     18.0   0.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: 9.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     12.3   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    2.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    1.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 7.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     2.0    1.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: 9.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.6    0.6    2.2       <total>
 
 # CHECK:      [9] Code Region - G10
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3404
+# CHECK-NEXT: Total Cycles:      1407
 # CHECK-NEXT: Total uOps:        2400
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.71
-# CHECK-NEXT: IPC:               0.29
+# CHECK-NEXT: uOps Per Cycle:    1.71
+# CHECK-NEXT: IPC:               0.71
 # CHECK-NEXT: Block RThroughput: 14.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          01234567
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          0
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .    . .   ld1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: [0,1]     .D=====eER.    .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D====eeeeeeeER    .    .    .    . .   ld1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: [0,3]     .  D==========eER   .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D=========eeeeeeeER  .    .    . .   ld1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: [0,5]     .    D===============eER .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D==============eeeeeeER .    . .   ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: [0,7]     .    . D===================eER.    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .  D==================eeeeeeeeER.   ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: [0,9]     .    .   D=========================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER .    .    .   ld1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE----R .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeER   .    .   ld1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE-----R   .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   D=eeeeeeeER.    .   ld1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,5]     .    D=eE-----R.    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .D==eeeeeeER   .   ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: [0,7]     .    . D==eE----R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  D==eeeeeeeeER   ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: [0,9]     .    .   D==eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1607,43 +1607,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: 3.     1     11.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     10.0   0.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: 5.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     15.0   0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: 7.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     19.0   0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: 9.     1     26.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     12.9   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    2.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    2.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: 7.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    1.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: 9.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.0    0.7    2.4       <total>
 
 # CHECK:      [10] Code Region - G11
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3404
+# CHECK-NEXT: Total Cycles:      1407
 # CHECK-NEXT: Total uOps:        2400
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.71
-# CHECK-NEXT: IPC:               0.29
+# CHECK-NEXT: uOps Per Cycle:    1.71
+# CHECK-NEXT: IPC:               0.71
 # CHECK-NEXT: Block RThroughput: 14.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          01234567
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          0
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .    . .   ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: [0,1]     .D=====eER.    .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D====eeeeeeER.    .    .    .    . .   ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: [0,3]     .  D=========eER    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D========eeeeeeeeER  .    .    . .   ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: [0,5]     .    D===============eER .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D==============eeeeeeER .    . .   ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: [0,7]     .    . D===================eER.    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .  D==================eeeeeeeeER.   ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: [0,9]     .    .   D=========================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER .    .    .   ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: [0,1]     .DeE----R .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeER    .    .   ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: [0,3]     .  DeE----R    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeER.    .   ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: [0,5]     .    DeE------R.    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .D==eeeeeeER   .   ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: [0,7]     .    . D==eE----R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  D==eeeeeeeeER   ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: [0,9]     .    .   D==eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1653,43 +1653,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: 3.     1     10.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     9.0    0.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: 5.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     15.0   0.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: 7.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     19.0   0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: 9.     1     26.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     12.7   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: 3.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    3.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: 7.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    1.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: 9.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.8    0.7    2.4       <total>
 
 # CHECK:      [11] Code Region - G12
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3404
+# CHECK-NEXT: Total Cycles:      1407
 # CHECK-NEXT: Total uOps:        2400
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.71
-# CHECK-NEXT: IPC:               0.29
+# CHECK-NEXT: uOps Per Cycle:    1.71
+# CHECK-NEXT: IPC:               0.71
 # CHECK-NEXT: Block RThroughput: 14.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          01234567
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          0
 
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    . .   ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: [0,1]     .D=======eER   .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D======eeeeeeER   .    .    .    . .   ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: [0,3]     .  D===========eER  .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D==========eeeeeeeeER.    .    . .   ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: [0,5]     .    D=================eER    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D================eeeeeeER    . .   ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: [0,7]     .    . D=====================eER   . .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .  D====================eeeeeeER.   ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: [0,9]     .    .   D=========================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeeER    .    .   ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: [0,1]     .DeE------R    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . D==eeeeeeER  .    .   ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK-NEXT: [0,3]     .  D==eE----R  .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   D==eeeeeeeeER   .   ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,5]     .    D==eE------R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .D====eeeeeeER .   ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,7]     .    . D====eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  D====eeeeeeER   ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .   D====eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1699,43 +1699,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: 3.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     11.0   0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: 5.     1     18.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     17.0   0.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: 7.     1     22.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     21.0   0.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: 9.     1     26.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     14.3   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     3.0    3.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    1.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     5.0    3.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 7.     1     5.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     5.0    1.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: 9.     1     5.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.4    0.9    2.4       <total>
 
 # CHECK:      [12] Code Region - G13
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3804
+# CHECK-NEXT: Total Cycles:      1910
 # CHECK-NEXT: Total uOps:        2600
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.68
-# CHECK-NEXT: IPC:               0.26
+# CHECK-NEXT: uOps Per Cycle:    1.36
+# CHECK-NEXT: IPC:               0.52
 # CHECK-NEXT: Block RThroughput: 15.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          01
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    ..   ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: [0,1]     .D=======eER   .    .    .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D======eeeeeeER   .    .    .    .    ..   ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: [0,3]     .  D===========eER  .    .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D==========eeeeeeeeER.    .    .    ..   ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: [0,5]     .    D=================eER    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D================eeeeeeeeER  .    ..   ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: [0,7]     .    . D=======================eER .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .  D======================eeeeeeeeER.   ld1	{ v1.b }[0], [x27], #1
-# CHECK-NEXT: [0,9]     .    .   D=============================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          012345678
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    .  .   ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE------R    .    .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . D==eeeeeeER  .    .    .  .   ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: [0,3]     .  D==eE----R  .    .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   D==eeeeeeeeER   .    .  .   ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,5]     .    D==eE------R   .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .D====eeeeeeeeER    .  .   ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,7]     .    . D====eE------R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  D==========eeeeeeeeER   ld1	{ v1.b }[0], [x27], #1
+# CHECK-NEXT: [0,9]     .    .   D==========eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1745,22 +1745,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: 3.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     11.0   0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: 5.     1     18.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     17.0   0.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: 7.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     23.0   0.0    0.0       ld1	{ v1.b }[0], [x27], #1
-# CHECK-NEXT: 9.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     15.1   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     3.0    3.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    1.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     5.0    3.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: 7.     1     5.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     11.0   0.0    0.0       ld1	{ v1.b }[0], [x27], #1
+# CHECK-NEXT: 9.     1     11.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     4.6    0.8    2.8       <total>
 
 # CHECK:      [13] Code Region - G14
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    3
@@ -1770,18 +1770,18 @@ add x0, x27, 1
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld1	{ v1.b }[8], [x27], #1
-# CHECK-NEXT: [0,1]     .D=======eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D======eeeeeeeeER .    .    .    .    .  .   ld1	{ v1.b }[0], [x27], x28
-# CHECK-NEXT: [0,3]     .  D=============eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D============eeeeeeeeER   .    .    .  .   ld1	{ v1.b }[8], [x27], x28
-# CHECK-NEXT: [0,5]     .    D===================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D==================eeeeeeeeER.    .  .   ld1	{ v1.h }[0], [x27], #2
-# CHECK-NEXT: [0,7]     .    . D=========================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .  D========================eeeeeeeeER.   ld1	{ v1.h }[4], [x27], #2
-# CHECK-NEXT: [0,9]     .    .   D===============================eER   add	x0, x27, #1
+# CHECK-NEXT: Index     0123456789          0123456789          012
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld1	{ v1.b }[8], [x27], #1
+# CHECK-NEXT: [0,1]     .DeE------R    .    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . D======eeeeeeeeER .    .    .    .    . .   ld1	{ v1.b }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .  D======eE------R .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   D============eeeeeeeeER   .    .    . .   ld1	{ v1.b }[8], [x27], x28
+# CHECK-NEXT: [0,5]     .    D============eE------R   .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .D==================eeeeeeeeER.    . .   ld1	{ v1.h }[0], [x27], #2
+# CHECK-NEXT: [0,7]     .    . D==================eE------R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  D========================eeeeeeeeER   ld1	{ v1.h }[4], [x27], #2
+# CHECK-NEXT: [0,9]     .    .   D========================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1791,22 +1791,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.b }[8], [x27], #1
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld1	{ v1.b }[0], [x27], x28
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 3.     1     7.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld1	{ v1.b }[8], [x27], x28
-# CHECK-NEXT: 5.     1     20.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 5.     1     13.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld1	{ v1.h }[0], [x27], #2
-# CHECK-NEXT: 7.     1     26.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 7.     1     19.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld1	{ v1.h }[4], [x27], #2
-# CHECK-NEXT: 9.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.5   0.1    0.0       <total>
+# CHECK-NEXT: 9.     1     25.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     13.0   0.1    3.0       <total>
 
 # CHECK:      [14] Code Region - G15
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3704
+# CHECK-NEXT: Total Cycles:      3703
 # CHECK-NEXT: Total uOps:        1900
 
 # CHECK:      Dispatch Width:    3
@@ -1816,18 +1816,18 @@ add x0, x27, 1
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0
+# CHECK-NEXT: Index     0123456789          0123456789
 
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .   ld1	{ v1.h }[0], [x27], x28
-# CHECK-NEXT: [0,1]     .D=======eER   .    .    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D======eeeeeeeeER .    .    .    .    .   ld1	{ v1.h }[4], [x27], x28
-# CHECK-NEXT: [0,3]     .  D=============eER.    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D============eeeeeeeeER   .    .    .   ld1	{ v1.s }[0], [x27], #4
-# CHECK-NEXT: [0,5]     .    D===================eER  .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D==================eeeeeeeeER.    .   ld1	{ v1.s }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .    . D=========================eER    .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    . D=========================eeeeeER.   ld1	{ v1.d }[0], [x27], #8
-# CHECK-NEXT: [0,9]     .    .  D=============================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .   .   ld1	{ v1.h }[0], [x27], x28
+# CHECK-NEXT: [0,1]     .DeE------R    .    .    .    .    .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . D======eeeeeeeeER .    .    .    .   .   ld1	{ v1.h }[4], [x27], x28
+# CHECK-NEXT: [0,3]     .  D======eE------R .    .    .    .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   D============eeeeeeeeER   .    .   .   ld1	{ v1.s }[0], [x27], #4
+# CHECK-NEXT: [0,5]     .    D============eE------R   .    .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .D==================eeeeeeeeER.   .   ld1	{ v1.s }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .    . D==================eE------R.   .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    . D=========================eeeeeER   ld1	{ v1.d }[0], [x27], #8
+# CHECK-NEXT: [0,9]     .    .  D=========================eE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1837,43 +1837,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.h }[0], [x27], x28
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld1	{ v1.h }[4], [x27], x28
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 3.     1     7.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld1	{ v1.s }[0], [x27], #4
-# CHECK-NEXT: 5.     1     20.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 5.     1     13.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld1	{ v1.s }[0], [x27], x28
-# CHECK-NEXT: 7.     1     26.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 7.     1     19.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 8.     1     26.0   0.0    0.0       ld1	{ v1.d }[0], [x27], #8
-# CHECK-NEXT: 9.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.4   0.1    0.0       <total>
+# CHECK-NEXT: 9.     1     26.0   0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     13.2   0.1    2.7       <total>
 
 # CHECK:      [15] Code Region - G16
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3404
+# CHECK-NEXT: Total Cycles:      1205
 # CHECK-NEXT: Total uOps:        1800
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.53
-# CHECK-NEXT: IPC:               0.29
+# CHECK-NEXT: uOps Per Cycle:    1.49
+# CHECK-NEXT: IPC:               0.83
 # CHECK-NEXT: Block RThroughput: 6.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          01234567
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     0123456
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeER  .    .    .    .    .    . .   ld1	{ v1.d }[0], [x27], x28
-# CHECK-NEXT: [0,1]     D=====eER .    .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D====eeeeeER  .    .    .    .    . .   ld1r	{ v1.1d }, [x27], #8
-# CHECK-NEXT: [0,3]     .D=========eER .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D========eeeeeeeeER    .    .    . .   ld1r	{ v1.2d }, [x27], #8
-# CHECK-NEXT: [0,5]     .  D===============eER   .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .   D==============eeeeeeeeER .    . .   ld1r	{ v1.2s }, [x27], #4
-# CHECK-NEXT: [0,7]     .    D=====================eER.    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .D====================eeeeeeeeER.   ld1r	{ v1.4h }, [x27], #2
-# CHECK-NEXT: [0,9]     .    . D===========================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeER  .    ..   ld1	{ v1.d }[0], [x27], x28
+# CHECK-NEXT: [0,1]     D=eE---R  .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeER .    ..   ld1r	{ v1.1d }, [x27], #8
+# CHECK-NEXT: [0,3]     .D=eE---R .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeeeER  ..   ld1r	{ v1.2d }, [x27], #8
+# CHECK-NEXT: [0,5]     .  DeE------R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .   DeeeeeeeeER..   ld1r	{ v1.2s }, [x27], #4
+# CHECK-NEXT: [0,7]     .    DeE------R..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .DeeeeeeeeER   ld1r	{ v1.4h }, [x27], #2
+# CHECK-NEXT: [0,9]     .    . DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1883,43 +1883,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.d }[0], [x27], x28
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       ld1r	{ v1.1d }, [x27], #8
-# CHECK-NEXT: 3.     1     10.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     9.0    0.0    0.0       ld1r	{ v1.2d }, [x27], #8
-# CHECK-NEXT: 5.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     15.0   0.0    0.0       ld1r	{ v1.2s }, [x27], #4
-# CHECK-NEXT: 7.     1     22.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     21.0   0.0    0.0       ld1r	{ v1.4h }, [x27], #2
-# CHECK-NEXT: 9.     1     28.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     13.3   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1r	{ v1.1d }, [x27], #8
+# CHECK-NEXT: 3.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1r	{ v1.2d }, [x27], #8
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld1r	{ v1.2s }, [x27], #4
+# CHECK-NEXT: 7.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld1r	{ v1.4h }, [x27], #2
+# CHECK-NEXT: 9.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.2    0.3    2.4       <total>
 
 # CHECK:      [16] Code Region - G17
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3704
+# CHECK-NEXT: Total Cycles:      908
 # CHECK-NEXT: Total uOps:        1900
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.51
-# CHECK-NEXT: IPC:               0.27
+# CHECK-NEXT: uOps Per Cycle:    2.09
+# CHECK-NEXT: IPC:               1.10
 # CHECK-NEXT: Block RThroughput: 6.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0
+# CHECK-NEXT:                     0123456
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .   ld1r	{ v1.4s }, [x27], #4
-# CHECK-NEXT: [0,1]     .D=======eER   .    .    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D======eeeeeeeeER .    .    .    .    .   ld1r	{ v1.8b }, [x27], #1
-# CHECK-NEXT: [0,3]     .  D=============eER.    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D============eeeeeeeeER   .    .    .   ld1r	{ v1.8h }, [x27], #2
-# CHECK-NEXT: [0,5]     .    D===================eER  .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D==================eeeeeeeeER.    .   ld1r	{ v1.16b }, [x27], #1
-# CHECK-NEXT: [0,7]     .    . D=========================eER    .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    . D=========================eeeeeER.   ld1r	{ v1.1d }, [x27], x28
-# CHECK-NEXT: [0,9]     .    .  D=============================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeeER    ..   ld1r	{ v1.4s }, [x27], #4
+# CHECK-NEXT: [0,1]     .DeE------R    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeER  ..   ld1r	{ v1.8b }, [x27], #1
+# CHECK-NEXT: [0,3]     .  DeE------R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeER..   ld1r	{ v1.8h }, [x27], #2
+# CHECK-NEXT: [0,5]     .    DeE------R..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeeER   ld1r	{ v1.16b }, [x27], #1
+# CHECK-NEXT: [0,7]     .    . DeE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    . DeeeeeE--R   ld1r	{ v1.1d }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .  DeE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1929,43 +1929,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1r	{ v1.4s }, [x27], #4
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld1r	{ v1.8b }, [x27], #1
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld1r	{ v1.8h }, [x27], #2
-# CHECK-NEXT: 5.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld1r	{ v1.16b }, [x27], #1
-# CHECK-NEXT: 7.     1     26.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     26.0   0.0    0.0       ld1r	{ v1.1d }, [x27], x28
-# CHECK-NEXT: 9.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.4   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld1r	{ v1.8b }, [x27], #1
+# CHECK-NEXT: 3.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld1r	{ v1.8h }, [x27], #2
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld1r	{ v1.16b }, [x27], #1
+# CHECK-NEXT: 7.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    2.0       ld1r	{ v1.1d }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.4    3.1       <total>
 
 # CHECK:      [17] Code Region - G18
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      1009
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.50
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    1.98
+# CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 6.7
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
+# CHECK-NEXT:                     012345678
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld1r	{ v1.2d }, [x27], x28
-# CHECK-NEXT: [0,1]     .D=======eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D======eeeeeeeeER .    .    .    .    .  .   ld1r	{ v1.2s }, [x27], x28
-# CHECK-NEXT: [0,3]     .  D=============eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D============eeeeeeeeER   .    .    .  .   ld1r	{ v1.4h }, [x27], x28
-# CHECK-NEXT: [0,5]     .    D===================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D==================eeeeeeeeER.    .  .   ld1r	{ v1.4s }, [x27], x28
-# CHECK-NEXT: [0,7]     .    . D=========================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .  D========================eeeeeeeeER.   ld1r	{ v1.8b }, [x27], x28
-# CHECK-NEXT: [0,9]     .    .   D===============================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeeER    .  .   ld1r	{ v1.2d }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE------R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeER  .  .   ld1r	{ v1.2s }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE------R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeER.  .   ld1r	{ v1.4h }, [x27], x28
+# CHECK-NEXT: [0,5]     .    DeE------R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeeER .   ld1r	{ v1.4s }, [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeeeER   ld1r	{ v1.8b }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1975,43 +1975,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1r	{ v1.2d }, [x27], x28
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld1r	{ v1.2s }, [x27], x28
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld1r	{ v1.4h }, [x27], x28
-# CHECK-NEXT: 5.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld1r	{ v1.4s }, [x27], x28
-# CHECK-NEXT: 7.     1     26.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld1r	{ v1.8b }, [x27], x28
-# CHECK-NEXT: 9.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.5   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld1r	{ v1.2s }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld1r	{ v1.4h }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld1r	{ v1.4s }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld1r	{ v1.8b }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    3.0       <total>
 
 # CHECK:      [18] Code Region - G19
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3804
+# CHECK-NEXT: Total Cycles:      1009
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.53
-# CHECK-NEXT: IPC:               0.26
+# CHECK-NEXT: uOps Per Cycle:    1.98
+# CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 6.7
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          01
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    ..   ld1r	{ v1.8h }, [x27], x28
-# CHECK-NEXT: [0,1]     .D=======eER   .    .    .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D======eeeeeeeeER .    .    .    .    ..   ld1r	{ v1.16b }, [x27], x28
-# CHECK-NEXT: [0,3]     .  D=============eER.    .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D============eeeeeeER.    .    .    ..   ld2	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: [0,5]     .    D=================eER    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D================eeeeeeeeER  .    ..   ld2	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: [0,7]     .    . D=======================eER .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .  D======================eeeeeeeeER.   ld2	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: [0,9]     .    .   D=============================eER   add	x0, x27, #1
+# CHECK-NEXT:                     012345678
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER    .  .   ld1r	{ v1.8h }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE------R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeER  .  .   ld1r	{ v1.16b }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE------R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeER  .  .   ld2	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: [0,5]     .    DeE----R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeeER .   ld2	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: [0,7]     .    . DeE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeeeER   ld2	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: [0,9]     .    .   DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2021,43 +2021,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1r	{ v1.8h }, [x27], x28
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld1r	{ v1.16b }, [x27], x28
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld2	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: 5.     1     18.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     17.0   0.0    0.0       ld2	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: 7.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     23.0   0.0    0.0       ld2	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: 9.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     15.5   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld1r	{ v1.16b }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld2	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: 5.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld2	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: 7.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld2	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: 9.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    2.8       <total>
 
 # CHECK:      [19] Code Region - G20
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4104
+# CHECK-NEXT: Total Cycles:      1008
 # CHECK-NEXT: Total uOps:        2600
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.63
-# CHECK-NEXT: IPC:               0.24
+# CHECK-NEXT: uOps Per Cycle:    2.58
+# CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 9.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          01234
-
-# CHECK:      [0,0]     DeeeeeeeeeER   .    .    .    .    .    .   .   ld2	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: [0,1]     .D========eER  .    .    .    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D=======eeeeeeeeER.    .    .    .    .   .   ld2	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: [0,3]     .  D==============eER    .    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D=============eeeeeeeeeER .    .    .   .   ld2	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: [0,5]     .    D=====================eER.    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D====================eeeeeeeeeER  .   .   ld2	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: [0,7]     .    . D============================eER .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .  D===========================eeeeeeER.   ld2	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: [0,9]     .    .   D================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01234567
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeeER   . .   ld2	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: [0,1]     .DeE-------R   . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeER  . .   ld2	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: [0,3]     .  DeE------R  . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeeER .   ld2	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: [0,5]     .    DeE-------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeeeER   ld2	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: [0,7]     .    . DeE-------R   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeE-R   ld2	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2067,43 +2067,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld2	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: 3.     1     15.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     14.0   0.0    0.0       ld2	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: 5.     1     22.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     21.0   0.0    0.0       ld2	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: 7.     1     29.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     28.0   0.0    0.0       ld2	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: 9.     1     33.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     18.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld2	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: 3.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld2	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: 5.     1     1.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld2	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: 7.     1     1.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    1.0       ld2	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    3.3       <total>
 
 # CHECK:      [20] Code Region - G21
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4204
+# CHECK-NEXT: Total Cycles:      1010
 # CHECK-NEXT: Total uOps:        2400
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.57
-# CHECK-NEXT: IPC:               0.24
+# CHECK-NEXT: uOps Per Cycle:    2.38
+# CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 8.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          012345
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .    .   ld2	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: [0,1]     .D=======eER   .    .    .    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D======eeeeeeeeER .    .    .    .    .    .   ld2	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: [0,3]     .  D=============eER.    .    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D============eeeeeeeeeER  .    .    .    .   ld2	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: [0,5]     .    D====================eER .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D===================eeeeeeeeER    .    .   ld2	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: [0,7]     .    . D==========================eER   .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .  D=========================eeeeeeeeeER.   ld2	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: [0,9]     .    .   D=================================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeeER    .   .   ld2	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE------R    .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeER  .   .   ld2	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE------R  .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeeER   .   ld2	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,5]     .    DeE-------R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeeER  .   ld2	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeeeeER   ld2	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeE-------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2113,43 +2113,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld2	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld2	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: 5.     1     21.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     20.0   0.0    0.0       ld2	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: 7.     1     27.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     26.0   0.0    0.0       ld2	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: 9.     1     34.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     17.1   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld2	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld2	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld2	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld2	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    3.2       <total>
 
 # CHECK:      [21] Code Region - G22
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4104
+# CHECK-NEXT: Total Cycles:      3410
 # CHECK-NEXT: Total uOps:        2600
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.63
-# CHECK-NEXT: IPC:               0.24
+# CHECK-NEXT: uOps Per Cycle:    0.76
+# CHECK-NEXT: IPC:               0.29
 # CHECK-NEXT: Block RThroughput: 8.7
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          01234
-
-# CHECK:      [0,0]     DeeeeeeeeeER   .    .    .    .    .    .   .   ld2	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: [0,1]     .D========eER  .    .    .    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D=======eeeeeeeeER.    .    .    .    .   .   ld2	{ v1.b, v2.b }[0], [x27], #2
-# CHECK-NEXT: [0,3]     .  D==============eER    .    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D=============eeeeeeeeER  .    .    .   .   ld2	{ v1.b, v2.b }[8], [x27], #2
-# CHECK-NEXT: [0,5]     .    D====================eER .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D===================eeeeeeeeER    .   .   ld2	{ v1.b, v2.b }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .    . D==========================eER   .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .  D=========================eeeeeeeeER.   ld2	{ v1.b, v2.b }[8], [x27], x28
-# CHECK-NEXT: [0,9]     .    .   D================================eER   add	x0, x27, #1
+# CHECK-NEXT: Index     0123456789          0123456789          0123
+
+# CHECK:      [0,0]     DeeeeeeeeeER   .    .    .    .    .    .  .   ld2	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE-------R   .    .    .    .    .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . D=======eeeeeeeeER.    .    .    .    .  .   ld2	{ v1.b, v2.b }[0], [x27], #2
+# CHECK-NEXT: [0,3]     .  D=======eE------R.    .    .    .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   D=============eeeeeeeeER  .    .    .  .   ld2	{ v1.b, v2.b }[8], [x27], #2
+# CHECK-NEXT: [0,5]     .    D=============eE------R  .    .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .D===================eeeeeeeeER    .  .   ld2	{ v1.b, v2.b }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .    . D===================eE------R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  D=========================eeeeeeeeER   ld2	{ v1.b, v2.b }[8], [x27], x28
+# CHECK-NEXT: [0,9]     .    .   D=========================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2159,22 +2159,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     1.0    0.0    7.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld2	{ v1.b, v2.b }[0], [x27], #2
-# CHECK-NEXT: 3.     1     15.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 3.     1     8.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     14.0   0.0    0.0       ld2	{ v1.b, v2.b }[8], [x27], #2
-# CHECK-NEXT: 5.     1     21.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 5.     1     14.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     20.0   0.0    0.0       ld2	{ v1.b, v2.b }[0], [x27], x28
-# CHECK-NEXT: 7.     1     27.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 7.     1     20.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 8.     1     26.0   0.0    0.0       ld2	{ v1.b, v2.b }[8], [x27], x28
-# CHECK-NEXT: 9.     1     33.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     17.4   0.1    0.0       <total>
+# CHECK-NEXT: 9.     1     26.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     13.8   0.1    3.1       <total>
 
 # CHECK:      [22] Code Region - G23
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3804
+# CHECK-NEXT: Total Cycles:      3803
 # CHECK-NEXT: Total uOps:        2400
 
 # CHECK:      Dispatch Width:    3
@@ -2184,18 +2184,18 @@ add x0, x27, 1
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          01
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    ..   ld2	{ v1.h, v2.h }[0], [x27], #4
-# CHECK-NEXT: [0,1]     .D=======eER   .    .    .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D======eeeeeeeeER .    .    .    .    ..   ld2	{ v1.h, v2.h }[4], [x27], #4
-# CHECK-NEXT: [0,3]     .  D=============eER.    .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D============eeeeeeeeER   .    .    ..   ld2	{ v1.h, v2.h }[0], [x27], x28
-# CHECK-NEXT: [0,5]     .    D===================eER  .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D==================eeeeeeeeER.    ..   ld2	{ v1.h, v2.h }[4], [x27], x28
-# CHECK-NEXT: [0,7]     .    . D=========================eER    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .  D========================eeeeeeER.   ld2	{ v1.s, v2.s }[0], [x27], #8
-# CHECK-NEXT: [0,9]     .    .   D=============================eER   add	x0, x27, #1
+# CHECK-NEXT: Index     0123456789          0123456789          0
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .   ld2	{ v1.h, v2.h }[0], [x27], #4
+# CHECK-NEXT: [0,1]     .DeE------R    .    .    .    .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . D======eeeeeeeeER .    .    .    .    .   ld2	{ v1.h, v2.h }[4], [x27], #4
+# CHECK-NEXT: [0,3]     .  D======eE------R .    .    .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   D============eeeeeeeeER   .    .    .   ld2	{ v1.h, v2.h }[0], [x27], x28
+# CHECK-NEXT: [0,5]     .    D============eE------R   .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .D==================eeeeeeeeER.    .   ld2	{ v1.h, v2.h }[4], [x27], x28
+# CHECK-NEXT: [0,7]     .    . D==================eE------R.    .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  D========================eeeeeeER   ld2	{ v1.s, v2.s }[0], [x27], #8
+# CHECK-NEXT: [0,9]     .    .   D========================eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2205,43 +2205,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.h, v2.h }[0], [x27], #4
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld2	{ v1.h, v2.h }[4], [x27], #4
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 3.     1     7.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld2	{ v1.h, v2.h }[0], [x27], x28
-# CHECK-NEXT: 5.     1     20.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 5.     1     13.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld2	{ v1.h, v2.h }[4], [x27], x28
-# CHECK-NEXT: 7.     1     26.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 7.     1     19.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld2	{ v1.s, v2.s }[0], [x27], #8
-# CHECK-NEXT: 9.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.3   0.1    0.0       <total>
+# CHECK-NEXT: 9.     1     25.0   0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     13.0   0.1    2.8       <total>
 
 # CHECK:      [23] Code Region - G24
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3504
+# CHECK-NEXT: Total Cycles:      2403
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.57
-# CHECK-NEXT: IPC:               0.29
+# CHECK-NEXT: uOps Per Cycle:    0.83
+# CHECK-NEXT: IPC:               0.42
 # CHECK-NEXT: Block RThroughput: 6.7
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          012345678
-# CHECK-NEXT: Index     0123456789          0123456789
-
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .    .  .   ld2	{ v1.s, v2.s }[0], [x27], x28
-# CHECK-NEXT: [0,1]     .D=====eER.    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D====eeeeeeeeER   .    .    .    .  .   ld2	{ v1.d, v2.d }[0], [x27], #16
-# CHECK-NEXT: [0,3]     .  D===========eER  .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D==========eeeeeeeeER.    .    .  .   ld2	{ v1.d, v2.d }[0], [x27], x28
-# CHECK-NEXT: [0,5]     .    D=================eER    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    D=================eeeeeER.    .  .   ld2r	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: [0,7]     .    .D=====================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    . D====================eeeeeeeeER.   ld2r	{ v1.2d, v2.2d }, [x27], #16
-# CHECK-NEXT: [0,9]     .    .  D===========================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          0123456
+
+# CHECK:      [0,0]     DeeeeeeER .    .    .    ..   ld2	{ v1.s, v2.s }[0], [x27], x28
+# CHECK-NEXT: [0,1]     .DeE----R .    .    .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . D====eeeeeeeeER   .    ..   ld2	{ v1.d, v2.d }[0], [x27], #16
+# CHECK-NEXT: [0,3]     .  D====eE------R   .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   D==========eeeeeeeeER..   ld2	{ v1.d, v2.d }[0], [x27], x28
+# CHECK-NEXT: [0,5]     .    D==========eE------R..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    D==========eeeeeE--R..   ld2r	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: [0,7]     .    .D==========eE-----R..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    . D=========eeeeeeeeER   ld2r	{ v1.2d, v2.2d }, [x27], #16
+# CHECK-NEXT: [0,9]     .    .  D=========eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2251,43 +2251,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.s, v2.s }[0], [x27], x28
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     1.0    0.0    4.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     5.0    0.0    0.0       ld2	{ v1.d, v2.d }[0], [x27], #16
-# CHECK-NEXT: 3.     1     12.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 3.     1     5.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     11.0   0.0    0.0       ld2	{ v1.d, v2.d }[0], [x27], x28
-# CHECK-NEXT: 5.     1     18.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     18.0   0.0    0.0       ld2r	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: 7.     1     22.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     21.0   0.0    0.0       ld2r	{ v1.2d, v2.2d }, [x27], #16
-# CHECK-NEXT: 9.     1     28.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     14.2   0.1    0.0       <total>
+# CHECK-NEXT: 5.     1     11.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     11.0   0.0    2.0       ld2r	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: 7.     1     11.0   0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     10.0   0.0    0.0       ld2r	{ v1.2d, v2.2d }, [x27], #16
+# CHECK-NEXT: 9.     1     10.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     7.6    0.1    2.9       <total>
 
 # CHECK:      [24] Code Region - G25
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      1009
 # CHECK-NEXT: Total uOps:        2200
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.55
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    2.18
+# CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 7.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
+# CHECK-NEXT:                     012345678
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld2r	{ v1.2s, v2.2s }, [x27], #8
-# CHECK-NEXT: [0,1]     .D=======eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D======eeeeeeeeER .    .    .    .    .  .   ld2r	{ v1.4h, v2.4h }, [x27], #4
-# CHECK-NEXT: [0,3]     .  D=============eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D============eeeeeeeeER   .    .    .  .   ld2r	{ v1.4s, v2.4s }, [x27], #8
-# CHECK-NEXT: [0,5]     .    D===================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D==================eeeeeeeeER.    .  .   ld2r	{ v1.8b, v2.8b }, [x27], #2
-# CHECK-NEXT: [0,7]     .    . D=========================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .  D========================eeeeeeeeER.   ld2r	{ v1.8h, v2.8h }, [x27], #4
-# CHECK-NEXT: [0,9]     .    .   D===============================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeeER    .  .   ld2r	{ v1.2s, v2.2s }, [x27], #8
+# CHECK-NEXT: [0,1]     .DeE------R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeER  .  .   ld2r	{ v1.4h, v2.4h }, [x27], #4
+# CHECK-NEXT: [0,3]     .  DeE------R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeER.  .   ld2r	{ v1.4s, v2.4s }, [x27], #8
+# CHECK-NEXT: [0,5]     .    DeE------R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeeER .   ld2r	{ v1.8b, v2.8b }, [x27], #2
+# CHECK-NEXT: [0,7]     .    . DeE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeeeER   ld2r	{ v1.8h, v2.8h }, [x27], #4
+# CHECK-NEXT: [0,9]     .    .   DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2297,43 +2297,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2r	{ v1.2s, v2.2s }, [x27], #8
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld2r	{ v1.4h, v2.4h }, [x27], #4
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld2r	{ v1.4s, v2.4s }, [x27], #8
-# CHECK-NEXT: 5.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld2r	{ v1.8b, v2.8b }, [x27], #2
-# CHECK-NEXT: 7.     1     26.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld2r	{ v1.8h, v2.8h }, [x27], #4
-# CHECK-NEXT: 9.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.5   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld2r	{ v1.4h, v2.4h }, [x27], #4
+# CHECK-NEXT: 3.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld2r	{ v1.4s, v2.4s }, [x27], #8
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld2r	{ v1.8b, v2.8b }, [x27], #2
+# CHECK-NEXT: 7.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld2r	{ v1.8h, v2.8h }, [x27], #4
+# CHECK-NEXT: 9.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    3.0       <total>
 
 # CHECK:      [25] Code Region - G26
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3704
+# CHECK-NEXT: Total Cycles:      909
 # CHECK-NEXT: Total uOps:        2100
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.57
-# CHECK-NEXT: IPC:               0.27
+# CHECK-NEXT: uOps Per Cycle:    2.31
+# CHECK-NEXT: IPC:               1.10
 # CHECK-NEXT: Block RThroughput: 7.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0
+# CHECK-NEXT:                     01234567
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .   ld2r	{ v1.16b, v2.16b }, [x27], #2
-# CHECK-NEXT: [0,1]     .D=======eER   .    .    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D======eeeeeER    .    .    .    .    .   ld2r	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: [0,3]     . D===========eER   .    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .  D==========eeeeeeeeER .    .    .    .   ld2r	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: [0,5]     .   D=================eER.    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    D================eeeeeeeeER   .    .   ld2r	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: [0,7]     .    .D=======================eER  .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    . D======================eeeeeeeeER.   ld2r	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: [0,9]     .    .  D=============================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeeER    . .   ld2r	{ v1.16b, v2.16b }, [x27], #2
+# CHECK-NEXT: [0,1]     .DeE------R    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeE-R    . .   ld2r	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: [0,3]     . D=eE----R    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .  DeeeeeeeeER . .   ld2r	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,5]     .   DeE------R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    DeeeeeeeeER .   ld2r	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,7]     .    .DeE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    . DeeeeeeeeER   ld2r	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .  DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2343,43 +2343,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2r	{ v1.16b, v2.16b }, [x27], #2
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld2r	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: 3.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     11.0   0.0    0.0       ld2r	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: 5.     1     18.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     17.0   0.0    0.0       ld2r	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: 7.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     23.0   0.0    0.0       ld2r	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: 9.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     15.1   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    1.0       ld2r	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld2r	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld2r	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld2r	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.1    0.4    2.9       <total>
 
 # CHECK:      [26] Code Region - G27
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      1009
 # CHECK-NEXT: Total uOps:        2500
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.62
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    2.48
+# CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 8.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
+# CHECK-NEXT:                     012345678
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld2r	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: [0,1]     .D=======eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D======eeeeeeeeER .    .    .    .    .  .   ld2r	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: [0,3]     .  D=============eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D============eeeeeeeeER   .    .    .  .   ld2r	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: [0,5]     .    D===================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D==================eeeeeeeeER.    .  .   ld2r	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: [0,7]     .    . D=========================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .  D========================eeeeeeeeER.   ld3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: [0,9]     .    .   D===============================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeeER    .  .   ld2r	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE------R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeER  .  .   ld2r	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE------R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeER.  .   ld2r	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,5]     .    DeE------R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeeER .   ld2r	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeeeER   ld3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: [0,9]     .    .   DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2389,43 +2389,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2r	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld2r	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld2r	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: 5.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld2r	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: 7.     1     26.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: 9.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.5   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld2r	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld2r	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld2r	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: 9.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    3.0       <total>
 
 # CHECK:      [27] Code Region - G28
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4704
+# CHECK-NEXT: Total Cycles:      1210
 # CHECK-NEXT: Total uOps:        3600
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.77
-# CHECK-NEXT: IPC:               0.21
+# CHECK-NEXT: uOps Per Cycle:    2.98
+# CHECK-NEXT: IPC:               0.83
 # CHECK-NEXT: Block RThroughput: 12.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789          0
-# CHECK-NEXT: Index     0123456789          0123456789          0123456789
-
-# CHECK:      [0,0]     DeeeeeeeeeER   .    .    .    .    .    .    .    .   ld3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: [0,1]     .D========eER  .    .    .    .    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D=======eeeeeeeeeER    .    .    .    .    .    .   ld3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: [0,3]     .  D===============eER   .    .    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D==============eeeeeeeeeeER    .    .    .    .   ld3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: [0,5]     .    .D======================eER   .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    . D=====================eeeeeeeeeER.    .    .   ld3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: [0,7]     .    .  D=============================eER    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .   D============================eeeeeeeeeeER.   ld3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: [0,9]     .    .    .D====================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          01
+
+# CHECK:      [0,0]     DeeeeeeeeeER   .    ..   ld3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: [0,1]     .DeE-------R   .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeeER .    ..   ld3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: [0,3]     .  DeE-------R .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeeeER   ..   ld3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: [0,5]     .    .DeE-------R   ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    . DeeeeeeeeeER ..   ld3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: [0,7]     .    .  DeE-------R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .   DeeeeeeeeeeER   ld3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: [0,9]     .    .    .DeE-------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2435,43 +2435,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: 3.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: 5.     1     23.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: 7.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: 9.     1     37.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     19.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: 3.     1     1.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: 5.     1     1.0    1.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: 7.     1     1.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: 9.     1     1.0    1.0    7.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.7    3.5       <total>
 
 # CHECK:      [28] Code Region - G29
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4604
+# CHECK-NEXT: Total Cycles:      1410
 # CHECK-NEXT: Total uOps:        3600
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.78
-# CHECK-NEXT: IPC:               0.22
+# CHECK-NEXT: uOps Per Cycle:    2.55
+# CHECK-NEXT: IPC:               0.71
 # CHECK-NEXT: Block RThroughput: 14.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123456789
-
-# CHECK:      [0,0]     DeeeeeeeeeeER  .    .    .    .    .    .    .   .   ld3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: [0,1]     . D========eER .    .    .    .    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .  D=======eeeeeeeeER    .    .    .    .    .   .   ld3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: [0,3]     .   D==============eER   .    .    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    D=============eeeeeeeeeER.    .    .    .   .   ld3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: [0,5]     .    .D=====================eER    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    . D====================eeeeeeeeeER .    .   .   ld3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: [0,7]     .    .  D============================eER.    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .   D===========================eeeeeeeeeeER.   ld3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: [0,9]     .    .    .D===================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          0123
+
+# CHECK:      [0,0]     DeeeeeeeeeeER  .    .  .   ld3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: [0,1]     . DeE-------R  .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .  DeeeeeeeeER .    .  .   ld3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: [0,3]     .   DeE------R .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    D==eeeeeeeeeER .  .   ld3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,5]     .    .D==eE-------R .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    . D==eeeeeeeeeER  .   ld3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,7]     .    .  D==eE-------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .   D==eeeeeeeeeeER   ld3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .    .D=eE--------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2481,43 +2481,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: 3.     1     15.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     14.0   0.0    0.0       ld3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: 5.     1     22.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     21.0   0.0    0.0       ld3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: 7.     1     29.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     28.0   0.0    0.0       ld3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: 9.     1     36.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     18.3   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    1.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    3.0    0.0       ld3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    1.0    0.0       ld3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 7.     1     3.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    1.0    0.0       ld3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: 9.     1     2.0    0.0    8.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.1    0.8    3.5       <total>
 
 # CHECK:      [29] Code Region - G30
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4704
+# CHECK-NEXT: Total Cycles:      2511
 # CHECK-NEXT: Total uOps:        3600
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.77
-# CHECK-NEXT: IPC:               0.21
+# CHECK-NEXT: uOps Per Cycle:    1.43
+# CHECK-NEXT: IPC:               0.40
 # CHECK-NEXT: Block RThroughput: 12.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789          0
-# CHECK-NEXT: Index     0123456789          0123456789          0123456789
-
-# CHECK:      [0,0]     DeeeeeeeeeER   .    .    .    .    .    .    .    .   ld3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: [0,1]     .D========eER  .    .    .    .    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D=======eeeeeeeeeeER   .    .    .    .    .    .   ld3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: [0,3]     .   D===============eER  .    .    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    D==============eeeeeeeeeeER   .    .    .    .   ld3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: [0,5]     .    . D======================eER  .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .  D=====================eeeeeeeeeER    .    .   ld3	{ v1.b, v2.b, v3.b }[0], [x27], #3
-# CHECK-NEXT: [0,7]     .    .   D=============================eER   .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    D============================eeeeeeeeeER.   ld3	{ v1.b, v2.b, v3.b }[8], [x27], #3
-# CHECK-NEXT: [0,9]     .    .    .D====================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0123456789          012345
+# CHECK-NEXT: Index     0123456789          0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeeER   .    .    .    .    .   ld3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE-------R   .    .    .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeeeER.    .    .    .    .   ld3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: [0,3]     .   DeE-------R.    .    .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    DeeeeeeeeeeER  .    .    .    .   ld3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,5]     .    . DeE-------R  .    .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .  D=======eeeeeeeeeER   .    .   ld3	{ v1.b, v2.b, v3.b }[0], [x27], #3
+# CHECK-NEXT: [0,7]     .    .   D=======eE-------R   .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    D==============eeeeeeeeeER   ld3	{ v1.b, v2.b, v3.b }[8], [x27], #3
+# CHECK-NEXT: [0,9]     .    .    .D==============eE-------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2527,22 +2527,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: 3.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: 5.     1     23.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[0], [x27], #3
-# CHECK-NEXT: 7.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[8], [x27], #3
-# CHECK-NEXT: 9.     1     37.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     19.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    1.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    1.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     8.0    0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[0], [x27], #3
+# CHECK-NEXT: 7.     1     8.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     15.0   0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[8], [x27], #3
+# CHECK-NEXT: 9.     1     15.0   0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT:        1     5.2    0.5    3.5       <total>
 
 # CHECK:      [30] Code Region - G31
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4504
+# CHECK-NEXT: Total Cycles:      4503
 # CHECK-NEXT: Total uOps:        3000
 
 # CHECK:      Dispatch Width:    3
@@ -2552,18 +2552,18 @@ add x0, x27, 1
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          012345678
-
-# CHECK:      [0,0]     DeeeeeeeeeER   .    .    .    .    .    .    .  .   ld3	{ v1.b, v2.b, v3.b }[0], [x27], x28
-# CHECK-NEXT: [0,1]     .D========eER  .    .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D=======eeeeeeeeeER    .    .    .    .    .  .   ld3	{ v1.b, v2.b, v3.b }[8], [x27], x28
-# CHECK-NEXT: [0,3]     .  D===============eER   .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D==============eeeeeeeeeER.    .    .    .  .   ld3	{ v1.h, v2.h, v3.h }[0], [x27], #6
-# CHECK-NEXT: [0,5]     .    D======================eER    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D=====================eeeeeeeeeER .    .  .   ld3	{ v1.h, v2.h, v3.h }[4], [x27], #6
-# CHECK-NEXT: [0,7]     .    . D=============================eER.    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .  D============================eeeeeeeeeER.   ld3	{ v1.h, v2.h, v3.h }[0], [x27], x28
-# CHECK-NEXT: [0,9]     .    .   D====================================eER   add	x0, x27, #1
+# CHECK-NEXT: Index     0123456789          0123456789          01234567
+
+# CHECK:      [0,0]     DeeeeeeeeeER   .    .    .    .    .    .    . .   ld3	{ v1.b, v2.b, v3.b }[0], [x27], x28
+# CHECK-NEXT: [0,1]     .DeE-------R   .    .    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . D=======eeeeeeeeeER    .    .    .    .    . .   ld3	{ v1.b, v2.b, v3.b }[8], [x27], x28
+# CHECK-NEXT: [0,3]     .  D=======eE-------R    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   D==============eeeeeeeeeER.    .    .    . .   ld3	{ v1.h, v2.h, v3.h }[0], [x27], #6
+# CHECK-NEXT: [0,5]     .    D==============eE-------R.    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .D=====================eeeeeeeeeER .    . .   ld3	{ v1.h, v2.h, v3.h }[4], [x27], #6
+# CHECK-NEXT: [0,7]     .    . D=====================eE-------R .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  D============================eeeeeeeeeER   ld3	{ v1.h, v2.h, v3.h }[0], [x27], x28
+# CHECK-NEXT: [0,9]     .    .   D============================eE-------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2573,22 +2573,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.b, v2.b, v3.b }[0], [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     1.0    0.0    7.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[8], [x27], x28
-# CHECK-NEXT: 3.     1     16.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 3.     1     8.0    0.0    7.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld3	{ v1.h, v2.h, v3.h }[0], [x27], #6
-# CHECK-NEXT: 5.     1     23.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 5.     1     15.0   0.0    7.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld3	{ v1.h, v2.h, v3.h }[4], [x27], #6
-# CHECK-NEXT: 7.     1     30.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 7.     1     22.0   0.0    7.0       add	x0, x27, #1
 # CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld3	{ v1.h, v2.h, v3.h }[0], [x27], x28
-# CHECK-NEXT: 9.     1     37.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     19.0   0.1    0.0       <total>
+# CHECK-NEXT: 9.     1     29.0   0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT:        1     15.0   0.1    3.5       <total>
 
 # CHECK:      [31] Code Region - G32
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3704
+# CHECK-NEXT: Total Cycles:      3703
 # CHECK-NEXT: Total uOps:        2400
 
 # CHECK:      Dispatch Width:    3
@@ -2598,18 +2598,18 @@ add x0, x27, 1
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0
+# CHECK-NEXT: Index     0123456789          0123456789
 
-# CHECK:      [0,0]     DeeeeeeeeeER   .    .    .    .    .    .   ld3	{ v1.h, v2.h, v3.h }[4], [x27], x28
-# CHECK-NEXT: [0,1]     .D========eER  .    .    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D=======eeeeeeeeER.    .    .    .    .   ld3	{ v1.s, v2.s, v3.s }[0], [x27], #12
-# CHECK-NEXT: [0,3]     .  D==============eER    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D=============eeeeeeeeER  .    .    .   ld3	{ v1.s, v2.s, v3.s }[0], [x27], x28
-# CHECK-NEXT: [0,5]     .    D====================eER .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D===================eeeeeeER .    .   ld3	{ v1.d, v2.d, v3.d }[0], [x27], #24
-# CHECK-NEXT: [0,7]     .    . D========================eER.    .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .  D=======================eeeeeeER.   ld3	{ v1.d, v2.d, v3.d }[0], [x27], x28
-# CHECK-NEXT: [0,9]     .    .   D============================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeeeER   .    .    .    .    .   .   ld3	{ v1.h, v2.h, v3.h }[4], [x27], x28
+# CHECK-NEXT: [0,1]     .DeE-------R   .    .    .    .    .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . D=======eeeeeeeeER.    .    .    .   .   ld3	{ v1.s, v2.s, v3.s }[0], [x27], #12
+# CHECK-NEXT: [0,3]     .  D=======eE------R.    .    .    .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   D=============eeeeeeeeER  .    .   .   ld3	{ v1.s, v2.s, v3.s }[0], [x27], x28
+# CHECK-NEXT: [0,5]     .    D=============eE------R  .    .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .D===================eeeeeeER .   .   ld3	{ v1.d, v2.d, v3.d }[0], [x27], #24
+# CHECK-NEXT: [0,7]     .    . D===================eE----R .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  D=======================eeeeeeER   ld3	{ v1.d, v2.d, v3.d }[0], [x27], x28
+# CHECK-NEXT: [0,9]     .    .   D=======================eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2619,43 +2619,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.h, v2.h, v3.h }[4], [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     1.0    0.0    7.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld3	{ v1.s, v2.s, v3.s }[0], [x27], #12
-# CHECK-NEXT: 3.     1     15.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 3.     1     8.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     14.0   0.0    0.0       ld3	{ v1.s, v2.s, v3.s }[0], [x27], x28
-# CHECK-NEXT: 5.     1     21.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 5.     1     14.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     20.0   0.0    0.0       ld3	{ v1.d, v2.d, v3.d }[0], [x27], #24
-# CHECK-NEXT: 7.     1     25.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 7.     1     20.0   0.0    4.0       add	x0, x27, #1
 # CHECK-NEXT: 8.     1     24.0   0.0    0.0       ld3	{ v1.d, v2.d, v3.d }[0], [x27], x28
-# CHECK-NEXT: 9.     1     29.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.6   0.1    0.0       <total>
+# CHECK-NEXT: 9.     1     24.0   0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     13.4   0.1    2.7       <total>
 
 # CHECK:      [32] Code Region - G33
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      1110
 # CHECK-NEXT: Total uOps:        2700
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.67
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    2.43
+# CHECK-NEXT: IPC:               0.90
 # CHECK-NEXT: Block RThroughput: 9.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          0
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .    .    .  .   ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: [0,1]     .D=====eER.    .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D====eeeeeeeeeER  .    .    .    .    .  .   ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], #24
-# CHECK-NEXT: [0,3]     .   D===========eER .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    D==========eeeeeeeeER    .    .    .  .   ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], #12
-# CHECK-NEXT: [0,5]     .    .D=================eER   .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    . D================eeeeeeeeER .    .  .   ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], #6
-# CHECK-NEXT: [0,7]     .    .  D=======================eER.    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .   D======================eeeeeeeeeER.   ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], #12
-# CHECK-NEXT: [0,9]     .    .    D==============================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER .    .    .   ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], #24
+# CHECK-NEXT: [0,1]     .DeE----R .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeeER .    .   ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], #24
+# CHECK-NEXT: [0,3]     .   DeE------R .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    DeeeeeeeeER    .   ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], #12
+# CHECK-NEXT: [0,5]     .    .DeE------R    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    . DeeeeeeeeER  .   ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], #6
+# CHECK-NEXT: [0,7]     .    .  DeE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .   DeeeeeeeeeER   ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], #12
+# CHECK-NEXT: [0,9]     .    .    DeE-------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2665,43 +2665,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], #24
-# CHECK-NEXT: 3.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     11.0   0.0    0.0       ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], #12
-# CHECK-NEXT: 5.     1     18.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     17.0   0.0    0.0       ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], #6
-# CHECK-NEXT: 7.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     23.0   0.0    0.0       ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], #12
-# CHECK-NEXT: 9.     1     31.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     14.8   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], #24
+# CHECK-NEXT: 3.     1     1.0    1.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], #12
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], #6
+# CHECK-NEXT: 7.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], #12
+# CHECK-NEXT: 9.     1     1.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.6    2.9       <total>
 
 # CHECK:      [33] Code Region - G34
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4104
+# CHECK-NEXT: Total Cycles:      1109
 # CHECK-NEXT: Total uOps:        2800
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.68
-# CHECK-NEXT: IPC:               0.24
+# CHECK-NEXT: uOps Per Cycle:    2.52
+# CHECK-NEXT: IPC:               0.90
 # CHECK-NEXT: Block RThroughput: 9.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          01234
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .   .   ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], #3
-# CHECK-NEXT: [0,1]     .D=======eER   .    .    .    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D======eeeeeeeeeER.    .    .    .    .   .   ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], #6
-# CHECK-NEXT: [0,3]     .  D==============eER    .    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D=============eeeeeeeeeER .    .    .   .   ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], #3
-# CHECK-NEXT: [0,5]     .    D=====================eER.    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D====================eeeeeeER.    .   .   ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: [0,7]     .    . D=========================eER    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .  D========================eeeeeeeeeER.   ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: [0,9]     .    .    D===============================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER    .   .   ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], #3
+# CHECK-NEXT: [0,1]     .DeE------R    .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeeER .   .   ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], #6
+# CHECK-NEXT: [0,3]     .  DeE-------R .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeeER   .   ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], #3
+# CHECK-NEXT: [0,5]     .    DeE-------R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeE-R   .   ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeE-----R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeeeeER   ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .    DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2711,43 +2711,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], #3
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], #6
-# CHECK-NEXT: 3.     1     15.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     14.0   0.0    0.0       ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], #3
-# CHECK-NEXT: 5.     1     22.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     21.0   0.0    0.0       ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: 7.     1     26.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: 9.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     17.1   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], #6
+# CHECK-NEXT: 3.     1     1.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], #3
+# CHECK-NEXT: 5.     1     1.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    1.0       ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    1.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.6    3.2       <total>
 
 # CHECK:      [34] Code Region - G35
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4204
+# CHECK-NEXT: Total Cycles:      1010
 # CHECK-NEXT: Total uOps:        2700
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.64
-# CHECK-NEXT: IPC:               0.24
+# CHECK-NEXT: uOps Per Cycle:    2.67
+# CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 9.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          012345
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .    .   ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: [0,1]     .D=======eER   .    .    .    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D======eeeeeeeeER .    .    .    .    .    .   ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: [0,3]     .  D=============eER.    .    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D============eeeeeeeeeER  .    .    .    .   ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: [0,5]     .    D====================eER .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D===================eeeeeeeeER    .    .   ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: [0,7]     .    . D==========================eER   .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .  D=========================eeeeeeeeeER.   ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: [0,9]     .    .   D=================================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeeER    .   .   ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE------R    .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeER  .   .   ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE------R  .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeeER   .   ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: [0,5]     .    DeE-------R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeeER  .   ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeeeeER   ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeE-------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2757,43 +2757,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: 5.     1     21.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     20.0   0.0    0.0       ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: 7.     1     27.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     26.0   0.0    0.0       ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: 9.     1     34.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     17.1   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    3.2       <total>
 
 # CHECK:      [35] Code Region - G36
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4604
+# CHECK-NEXT: Total Cycles:      1311
 # CHECK-NEXT: Total uOps:        3400
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.74
-# CHECK-NEXT: IPC:               0.22
+# CHECK-NEXT: uOps Per Cycle:    2.59
+# CHECK-NEXT: IPC:               0.76
 # CHECK-NEXT: Block RThroughput: 13.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123456789
-
-# CHECK:      [0,0]     DeeeeeeeeeER   .    .    .    .    .    .    .   .   ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: [0,1]     .D========eER  .    .    .    .    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D=======eeeeeeeeER.    .    .    .    .    .   .   ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: [0,3]     .  D==============eER    .    .    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D=============eeeeeeeeeER .    .    .    .   .   ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: [0,5]     .    D=====================eER.    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D====================eeeeeeeeeER  .    .   .   ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: [0,7]     .    . D============================eER .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .  D===========================eeeeeeeeeeeER.   ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: [0,9]     .    .    .D===================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          0123
+
+# CHECK:      [0,0]     DeeeeeeeeeER   .    .  .   ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE-------R   .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeER  .    .  .   ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: [0,3]     .  DeE------R  .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   D==eeeeeeeeeER  .  .   ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: [0,5]     .    D==eE-------R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .D==eeeeeeeeeER.  .   ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: [0,7]     .    . D==eE-------R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  D==eeeeeeeeeeeER   ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: [0,9]     .    .    .DeE---------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2803,43 +2803,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: 3.     1     15.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     14.0   0.0    0.0       ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: 5.     1     22.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     21.0   0.0    0.0       ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: 7.     1     29.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     28.0   0.0    0.0       ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: 9.     1     36.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     18.3   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: 3.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    3.0    0.0       ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: 5.     1     3.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    1.0    0.0       ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: 7.     1     3.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    1.0    0.0       ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: 9.     1     1.0    0.0    9.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.0    0.7    3.6       <total>
 
 # CHECK:      [36] Code Region - G37
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4804
+# CHECK-NEXT: Total Cycles:      1610
 # CHECK-NEXT: Total uOps:        3800
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.79
-# CHECK-NEXT: IPC:               0.21
+# CHECK-NEXT: uOps Per Cycle:    2.36
+# CHECK-NEXT: IPC:               0.62
 # CHECK-NEXT: Block RThroughput: 16.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789          01
-# CHECK-NEXT: Index     0123456789          0123456789          0123456789
-
-# CHECK:      [0,0]     DeeeeeeeeeER   .    .    .    .    .    .    .    ..   ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: [0,1]     .D========eER  .    .    .    .    .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D=======eeeeeeeeeeeER  .    .    .    .    .    ..   ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: [0,3]     .    D===============eER .    .    .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    .D==============eeeeeeeeeeeER .    .    .    ..   ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: [0,5]     .    .   D======================eER.    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .    D=====================eeeeeeeeER   .    ..   ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: [0,7]     .    .    .D============================eER  .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    . D===========================eeeeeeeeeER.   ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: [0,9]     .    .    .  D===================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          012345
+
+# CHECK:      [0,0]     DeeeeeeeeeER   .    .    .   ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: [0,1]     .DeE-------R   .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeeeeER    .    .   ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: [0,3]     .    DeE-------R    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .DeeeeeeeeeeeER.    .   ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: [0,5]     .    .   DeE-------R.    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .    DeeeeeeeeER    .   ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,7]     .    .    .DeE------R    .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    . D==eeeeeeeeeER   ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .    .  D==eE-------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2849,43 +2849,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: 3.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: 5.     1     23.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: 7.     1     29.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     28.0   0.0    0.0       ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: 9.     1     36.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     18.7   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: 3.     1     1.0    1.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: 5.     1     1.0    1.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    3.0    0.0       ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 9.     1     3.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.4    0.9    3.4       <total>
 
 # CHECK:      [37] Code Region - G38
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      5104
+# CHECK-NEXT: Total Cycles:      1610
 # CHECK-NEXT: Total uOps:        4200
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.82
-# CHECK-NEXT: IPC:               0.20
+# CHECK-NEXT: uOps Per Cycle:    2.61
+# CHECK-NEXT: IPC:               0.62
 # CHECK-NEXT: Block RThroughput: 16.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789          01234
-# CHECK-NEXT: Index     0123456789          0123456789          0123456789
-
-# CHECK:      [0,0]     DeeeeeeeeeER   .    .    .    .    .    .    .    .   .   ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: [0,1]     .D========eER  .    .    .    .    .    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D=======eeeeeeeeeeeER  .    .    .    .    .    .   .   ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: [0,3]     .    D===============eER .    .    .    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    .D==============eeeeeeeeeER   .    .    .    .   .   ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: [0,5]     .    . D======================eER  .    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .  D=====================eeeeeeeeeeeER  .    .   .   ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: [0,7]     .    .    .D=============================eER .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    . D============================eeeeeeeeeeeER.   ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: [0,9]     .    .    .    D====================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          012345
+
+# CHECK:      [0,0]     DeeeeeeeeeER   .    .    .   ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE-------R   .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeeeeER    .    .   ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,3]     .    DeE-------R    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .DeeeeeeeeeER  .    .   ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: [0,5]     .    . DeE-------R  .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .  DeeeeeeeeeeeER   .   ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,7]     .    .    .DeE-------R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    . DeeeeeeeeeeeER   ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .    .    DeE-------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2895,22 +2895,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: 3.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: 5.     1     23.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: 7.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: 9.     1     37.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     19.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    1.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    1.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    1.0    7.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.8    3.5       <total>
 
 # CHECK:      [38] Code Region - G39
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4504
+# CHECK-NEXT: Total Cycles:      4503
 # CHECK-NEXT: Total uOps:        3500
 
 # CHECK:      Dispatch Width:    3
@@ -2920,18 +2920,18 @@ add x0, x27, 1
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          012345678
-
-# CHECK:      [0,0]     DeeeeeeeeeER   .    .    .    .    .    .    .  .   ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
-# CHECK-NEXT: [0,1]     . D=======eER  .    .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .  D======eeeeeeeeeER    .    .    .    .    .  .   ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
-# CHECK-NEXT: [0,3]     .    D=============eER   .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    .D============eeeeeeeeeER.    .    .    .  .   ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
-# CHECK-NEXT: [0,5]     .    .  D===================eER    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .   D==================eeeeeeeeeER .    .  .   ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
-# CHECK-NEXT: [0,7]     .    .    .D=========================eER.    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    . D========================eeeeeeeeeER.   ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
-# CHECK-NEXT: [0,9]     .    .    .   D===============================eER   add	x0, x27, #1
+# CHECK-NEXT: Index     0123456789          0123456789          01234567
+
+# CHECK:      [0,0]     DeeeeeeeeeER   .    .    .    .    .    .    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
+# CHECK-NEXT: [0,1]     . DeE------R   .    .    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .  D======eeeeeeeeeER    .    .    .    .    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
+# CHECK-NEXT: [0,3]     .    D=====eE-------R    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .D============eeeeeeeeeER.    .    .    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
+# CHECK-NEXT: [0,5]     .    .  D===========eE-------R.    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .   D==================eeeeeeeeeER .    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
+# CHECK-NEXT: [0,7]     .    .    .D=================eE-------R .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    . D========================eeeeeeeeeER   ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
+# CHECK-NEXT: [0,9]     .    .    .   D=======================eE-------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2941,22 +2941,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     1.0    1.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 3.     1     6.0    0.0    7.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
-# CHECK-NEXT: 5.     1     20.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 5.     1     12.0   0.0    7.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
-# CHECK-NEXT: 7.     1     26.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 7.     1     18.0   0.0    7.0       add	x0, x27, #1
 # CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
-# CHECK-NEXT: 9.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.5   0.1    0.0       <total>
+# CHECK-NEXT: 9.     1     24.0   0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT:        1     12.6   0.2    3.4       <total>
 
 # CHECK:      [39] Code Region - G40
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4304
+# CHECK-NEXT: Total Cycles:      4303
 # CHECK-NEXT: Total uOps:        3100
 
 # CHECK:      Dispatch Width:    3
@@ -2966,18 +2966,18 @@ add x0, x27, 1
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123456
-
-# CHECK:      [0,0]     DeeeeeeeeeER   .    .    .    .    .    .    ..   ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
-# CHECK-NEXT: [0,1]     . D=======eER  .    .    .    .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .  D======eeeeeeeeeER    .    .    .    .    ..   ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
-# CHECK-NEXT: [0,3]     .    D=============eER   .    .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    .D============eeeeeeeeeER.    .    .    ..   ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
-# CHECK-NEXT: [0,5]     .    .  D===================eER    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .   D==================eeeeeeeeER  .    ..   ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
-# CHECK-NEXT: [0,7]     .    .    D=========================eER .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    .D========================eeeeeeeeER.   ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
-# CHECK-NEXT: [0,9]     .    .    . D===============================eER   add	x0, x27, #1
+# CHECK-NEXT: Index     0123456789          0123456789          012345
+
+# CHECK:      [0,0]     DeeeeeeeeeER   .    .    .    .    .    .    .   ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
+# CHECK-NEXT: [0,1]     . DeE------R   .    .    .    .    .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .  D======eeeeeeeeeER    .    .    .    .    .   ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .    D=====eE-------R    .    .    .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .D============eeeeeeeeeER.    .    .    .   ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
+# CHECK-NEXT: [0,5]     .    .  D===========eE-------R.    .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .   D==================eeeeeeeeER  .    .   ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
+# CHECK-NEXT: [0,7]     .    .    D==================eE------R  .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    .D========================eeeeeeeeER   ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
+# CHECK-NEXT: [0,9]     .    .    . D========================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2987,43 +2987,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     1.0    1.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 3.     1     6.0    0.0    7.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
-# CHECK-NEXT: 5.     1     20.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 5.     1     12.0   0.0    7.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
-# CHECK-NEXT: 7.     1     26.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 7.     1     19.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
-# CHECK-NEXT: 9.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.5   0.1    0.0       <total>
+# CHECK-NEXT: 9.     1     25.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     12.8   0.2    3.2       <total>
 
 # CHECK:      [40] Code Region - G41
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4104
+# CHECK-NEXT: Total Cycles:      2303
 # CHECK-NEXT: Total uOps:        3100
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.76
-# CHECK-NEXT: IPC:               0.24
+# CHECK-NEXT: uOps Per Cycle:    1.35
+# CHECK-NEXT: IPC:               0.43
 # CHECK-NEXT: Block RThroughput: 10.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          01234
-
-# CHECK:      [0,0]     DeeeeeeeeeER   .    .    .    .    .    .   .   ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
-# CHECK-NEXT: [0,1]     . D=======eER  .    .    .    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .  D======eeeeeeeeeER    .    .    .    .   .   ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
-# CHECK-NEXT: [0,3]     .    D=============eER   .    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    .D============eeeeeeER   .    .    .   .   ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: [0,5]     .    . D=================eER  .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .  D================eeeeeeeeeER    .   .   ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #32
-# CHECK-NEXT: [0,7]     .    .    D=======================eER   .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    .D======================eeeeeeeeER.   ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #16
-# CHECK-NEXT: [0,9]     .    .    . D=============================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          012345
+
+# CHECK:      [0,0]     DeeeeeeeeeER   .    .    .   ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
+# CHECK-NEXT: [0,1]     . DeE------R   .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .  D======eeeeeeeeeER    .   ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .    D=====eE-------R    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .D=====eeeeeeE-R    .   ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: [0,5]     .    . D=====eE-----R    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .  D=====eeeeeeeeeER.   ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #32
+# CHECK-NEXT: [0,7]     .    .    D====eE-------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    .D====eeeeeeeeER   ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #16
+# CHECK-NEXT: [0,9]     .    .    . D====eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3033,43 +3033,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     1.0    1.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: 5.     1     18.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     17.0   0.0    0.0       ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #32
-# CHECK-NEXT: 7.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     23.0   0.0    0.0       ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #16
-# CHECK-NEXT: 9.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     15.5   0.1    0.0       <total>
+# CHECK-NEXT: 3.     1     6.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     6.0    1.0    1.0       ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: 5.     1     6.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     6.0    1.0    0.0       ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #32
+# CHECK-NEXT: 7.     1     5.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     5.0    1.0    0.0       ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #16
+# CHECK-NEXT: 9.     1     5.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     4.8    0.5    3.2       <total>
 
 # CHECK:      [41] Code Region - G42
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4304
+# CHECK-NEXT: Total Cycles:      1309
 # CHECK-NEXT: Total uOps:        3100
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.72
-# CHECK-NEXT: IPC:               0.23
+# CHECK-NEXT: uOps Per Cycle:    2.37
+# CHECK-NEXT: IPC:               0.76
 # CHECK-NEXT: Block RThroughput: 10.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123456
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .    ..   ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #8
-# CHECK-NEXT: [0,1]     .D=======eER   .    .    .    .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D======eeeeeeeeeER.    .    .    .    .    ..   ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #16
-# CHECK-NEXT: [0,3]     .   D=============eER    .    .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    D============eeeeeeeeER  .    .    .    ..   ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #4
-# CHECK-NEXT: [0,5]     .    .D===================eER .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    . D==================eeeeeeeeeER   .    ..   ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #8
-# CHECK-NEXT: [0,7]     .    .   D=========================eER  .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    D========================eeeeeeeeeER.   ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #4
-# CHECK-NEXT: [0,9]     .    .    . D===============================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          01
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    ..   ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #8
+# CHECK-NEXT: [0,1]     .DeE------R    .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeeER .    ..   ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #16
+# CHECK-NEXT: [0,3]     .   DeE------R .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    DeeeeeeeeER    ..   ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #4
+# CHECK-NEXT: [0,5]     .    .DeE------R    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    . DeeeeeeeeeER ..   ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #8
+# CHECK-NEXT: [0,7]     .    .   DeE------R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    DeeeeeeeeeER   ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #4
+# CHECK-NEXT: [0,9]     .    .    . DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3079,43 +3079,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #8
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #16
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #4
-# CHECK-NEXT: 5.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #8
-# CHECK-NEXT: 7.     1     26.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #4
-# CHECK-NEXT: 9.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.5   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #16
+# CHECK-NEXT: 3.     1     1.0    1.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #4
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #8
+# CHECK-NEXT: 7.     1     1.0    1.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #4
+# CHECK-NEXT: 9.     1     1.0    1.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.8    3.0       <total>
 
 # CHECK:      [42] Code Region - G43
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      1209
 # CHECK-NEXT: Total uOps:        2900
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.72
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    2.40
+# CHECK-NEXT: IPC:               0.83
 # CHECK-NEXT: Block RThroughput: 9.7
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          0
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .    .    .  .   ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: [0,1]     .D=====eER.    .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D====eeeeeeeeeER  .    .    .    .    .  .   ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: [0,3]     .   D===========eER .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    D==========eeeeeeeeER    .    .    .  .   ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: [0,5]     .    .D=================eER   .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    . D================eeeeeeeeER .    .  .   ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: [0,7]     .    .  D=======================eER.    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .   D======================eeeeeeeeeER.   ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: [0,9]     .    .    .D=============================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER .    .    .   ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE----R .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeeER .    .   ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,3]     .   DeE------R .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    DeeeeeeeeER    .   ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,5]     .    .DeE------R    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    . DeeeeeeeeER  .   ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,7]     .    .  DeE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .   DeeeeeeeeeER   ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .    .DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3125,43 +3125,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: 3.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     11.0   0.0    0.0       ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: 5.     1     18.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     17.0   0.0    0.0       ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: 7.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     23.0   0.0    0.0       ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: 9.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     14.7   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    1.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    1.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.7    2.8       <total>
 
 # CHECK:      [43] Code Region - G44
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3604
+# CHECK-NEXT: Total Cycles:      1206
 # CHECK-NEXT: Total uOps:        2700
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.75
-# CHECK-NEXT: IPC:               0.28
+# CHECK-NEXT: uOps Per Cycle:    2.24
+# CHECK-NEXT: IPC:               0.83
 # CHECK-NEXT: Block RThroughput: 9.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     01234567
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .   .   ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: [0,1]     .D=======eER   .    .    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D======eeeeeeeeeER.    .    .    .   .   ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: [0,3]     .   D=============eER    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    D============eeeeeeeeeER .    .   .   ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: [0,5]     .    . D===================eER.    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .  D==================eeeeeER .   .   ldp	s1, s2, [x27], #248
-# CHECK-NEXT: [0,7]     .    .   D======================eER.   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    D=====================eeeeeER.   ldp	d1, d2, [x27], #496
-# CHECK-NEXT: [0,9]     .    .    .D=========================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeeER    . .   ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE------R    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeeER . .   ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,3]     .   DeE------R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    DeeeeeeeeeER.   ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,5]     .    . DeE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .  DeeeeeE-R.   ldp	s1, s2, [x27], #248
+# CHECK-NEXT: [0,7]     .    .   DeE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    DeeeeeER   ldp	d1, d2, [x27], #496
+# CHECK-NEXT: [0,9]     .    .    .DeE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3171,43 +3171,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: 5.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ldp	s1, s2, [x27], #248
-# CHECK-NEXT: 7.     1     23.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     22.0   0.0    0.0       ldp	d1, d2, [x27], #496
-# CHECK-NEXT: 9.     1     26.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     15.3   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    1.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    1.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    1.0       ldp	s1, s2, [x27], #248
+# CHECK-NEXT: 7.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ldp	d1, d2, [x27], #496
+# CHECK-NEXT: 9.     1     1.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.7    2.6       <total>
 
 # CHECK:      [44] Code Region - G45
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2306
+# CHECK-NEXT: Total Cycles:      1005
 # CHECK-NEXT: Total uOps:        2200
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.95
-# CHECK-NEXT: IPC:               0.43
+# CHECK-NEXT: uOps Per Cycle:    2.19
+# CHECK-NEXT: IPC:               1.00
 # CHECK-NEXT: Block RThroughput: 7.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          012345678
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .  .   ldp	q1, q2, [x27], #992
-# CHECK-NEXT: [0,1]     .D=====eER.    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D====eeeeeER .    .    .  .   ldp	s1, s2, [x27, #248]!
-# CHECK-NEXT: [0,3]     .  D========eER.    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D=======eeeeeER .    .  .   ldp	d1, d2, [x27, #496]!
-# CHECK-NEXT: [0,5]     .    D===========eER.    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D==========eeeeeeER.  .   ldp	q1, q2, [x27, #992]!
-# CHECK-NEXT: [0,7]     .    . D===============eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .  D==============eeeeER   ldp	w1, w2, [x27], #248
-# CHECK-NEXT: [0,9]     .    .   D==============eE--R   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER .   .   ldp	q1, q2, [x27], #992
+# CHECK-NEXT: [0,1]     .DeE----R .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeER.   .   ldp	s1, s2, [x27, #248]!
+# CHECK-NEXT: [0,3]     .  DeE---R.   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeER  .   ldp	d1, d2, [x27, #496]!
+# CHECK-NEXT: [0,5]     .    DeE---R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeER   ldp	q1, q2, [x27, #992]!
+# CHECK-NEXT: [0,7]     .    . DeE----R   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeER   ldp	w1, w2, [x27], #248
+# CHECK-NEXT: [0,9]     .    .   DeE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3217,43 +3217,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldp	q1, q2, [x27], #992
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       ldp	s1, s2, [x27, #248]!
-# CHECK-NEXT: 3.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     8.0    0.0    0.0       ldp	d1, d2, [x27, #496]!
-# CHECK-NEXT: 5.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     11.0   0.0    0.0       ldp	q1, q2, [x27, #992]!
-# CHECK-NEXT: 7.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     15.0   0.0    0.0       ldp	w1, w2, [x27], #248
-# CHECK-NEXT: 9.     1     15.0   0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT:        1     9.8    0.1    0.2       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ldp	s1, s2, [x27, #248]!
+# CHECK-NEXT: 3.     1     1.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ldp	d1, d2, [x27, #496]!
+# CHECK-NEXT: 5.     1     1.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ldp	q1, q2, [x27, #992]!
+# CHECK-NEXT: 7.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ldp	w1, w2, [x27], #248
+# CHECK-NEXT: 9.     1     1.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    1.6       <total>
 
 # CHECK:      [45] Code Region - G46
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1307
+# CHECK-NEXT: Total Cycles:      1006
 # CHECK-NEXT: Total uOps:        2400
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    1.84
-# CHECK-NEXT: IPC:               0.77
+# CHECK-NEXT: uOps Per Cycle:    2.39
+# CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 8.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
+# CHECK-NEXT:                     012345
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeER   .    .   .   ldp	x1, x2, [x27], #496
-# CHECK-NEXT: [0,1]     .DeE--R   .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . DeeeeER .    .   .   ldp	w1, w2, [x27, #248]!
-# CHECK-NEXT: [0,3]     .  DeE--R .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   DeeeeER    .   .   ldp	x1, x2, [x27, #496]!
-# CHECK-NEXT: [0,5]     .    DeE--R    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .DeeeeeER .   .   ldpsw	x1, x2, [x27], #248
-# CHECK-NEXT: [0,7]     .    . D====eER.   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .  D===eeeeeER.   ldpsw	x1, x2, [x27, #248]!
-# CHECK-NEXT: [0,9]     .    .   D=======eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeER   .    .   ldp	x1, x2, [x27], #496
+# CHECK-NEXT: [0,1]     .DeE--R   .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeER .    .   ldp	w1, w2, [x27, #248]!
+# CHECK-NEXT: [0,3]     .  DeE--R .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeER    .   ldp	x1, x2, [x27, #496]!
+# CHECK-NEXT: [0,5]     .    DeE--R    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeER .   ldpsw	x1, x2, [x27], #248
+# CHECK-NEXT: [0,7]     .    . DeE---R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeER   ldpsw	x1, x2, [x27, #248]!
+# CHECK-NEXT: [0,9]     .    .   DeE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3269,10 +3269,10 @@ add x0, x27, 1
 # CHECK-NEXT: 4.     1     1.0    1.0    0.0       ldp	x1, x2, [x27, #496]!
 # CHECK-NEXT: 5.     1     1.0    0.0    2.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     1.0    1.0    0.0       ldpsw	x1, x2, [x27], #248
-# CHECK-NEXT: 7.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ldpsw	x1, x2, [x27, #248]!
-# CHECK-NEXT: 9.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.4    0.4    0.6       <total>
+# CHECK-NEXT: 7.     1     1.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ldpsw	x1, x2, [x27, #248]!
+# CHECK-NEXT: 9.     1     1.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    1.2       <total>
 
 # CHECK:      [46] Code Region - G47
 
@@ -3324,28 +3324,28 @@ add x0, x27, 1
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2504
+# CHECK-NEXT: Total Cycles:      507
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    0.60
-# CHECK-NEXT: IPC:               0.40
+# CHECK-NEXT: uOps Per Cycle:    2.96
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          012345678
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeER  .    .    .    .  .   ldr	b1, [x27, #254]!
-# CHECK-NEXT: [0,1]     D=====eER .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D====eeeeeER  .    .    .  .   ldr	h1, [x27, #254]!
-# CHECK-NEXT: [0,3]     .D=========eER .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D========eeeeeER  .    .  .   ldr	s1, [x27, #254]!
-# CHECK-NEXT: [0,5]     . D=============eER .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D============eeeeeER  .  .   ldr	d1, [x27, #254]!
-# CHECK-NEXT: [0,7]     .  D=================eER .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D================eeeeeER.   ldr	q1, [x27, #254]!
-# CHECK-NEXT: [0,9]     .   D=====================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeER  ..   ldr	b1, [x27, #254]!
+# CHECK-NEXT: [0,1]     D=eE---R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeER ..   ldr	h1, [x27, #254]!
+# CHECK-NEXT: [0,3]     .D=eE---R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeER..   ldr	s1, [x27, #254]!
+# CHECK-NEXT: [0,5]     . D=eE---R..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeER.   ldr	d1, [x27, #254]!
+# CHECK-NEXT: [0,7]     .  D=eE---R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeER   ldr	q1, [x27, #254]!
+# CHECK-NEXT: [0,9]     .   D=eE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3355,16 +3355,16 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldr	b1, [x27, #254]!
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       ldr	h1, [x27, #254]!
-# CHECK-NEXT: 3.     1     10.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     9.0    0.0    0.0       ldr	s1, [x27, #254]!
-# CHECK-NEXT: 5.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     13.0   0.0    0.0       ldr	d1, [x27, #254]!
-# CHECK-NEXT: 7.     1     18.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     17.0   0.0    0.0       ldr	q1, [x27, #254]!
-# CHECK-NEXT: 9.     1     22.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     11.5   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ldr	h1, [x27, #254]!
+# CHECK-NEXT: 3.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ldr	s1, [x27, #254]!
+# CHECK-NEXT: 5.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ldr	d1, [x27, #254]!
+# CHECK-NEXT: 7.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ldr	q1, [x27, #254]!
+# CHECK-NEXT: 9.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    1.5       <total>
 
 # CHECK:      [48] Code Region - G49
 
@@ -3508,7 +3508,7 @@ add x0, x27, 1
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      604
+# CHECK-NEXT: Total Cycles:      603
 # CHECK-NEXT: Total uOps:        1600
 
 # CHECK:      Dispatch Width:    3
@@ -3517,18 +3517,18 @@ add x0, x27, 1
 # CHECK-NEXT: Block RThroughput: 5.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     012345678
 
-# CHECK:      [0,0]     DeeeeER  .   ldrsh	x1, [x27, #254]!
-# CHECK-NEXT: [0,1]     D=eE--R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeER .   ldrsw	x1, [x27], #254
-# CHECK-NEXT: [0,3]     .D=eE--R .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . DeeeeER.   ldrsw	x1, [x27, #254]!
-# CHECK-NEXT: [0,5]     . D=eE--R.   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  DeE--R.   st1	{ v1.1d }, [x27], #8
-# CHECK-NEXT: [0,7]     .  D=eE-R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   DeeER.   st1	{ v1.2d }, [x27], #16
-# CHECK-NEXT: [0,9]     .    D=eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeER .   ldrsh	x1, [x27, #254]!
+# CHECK-NEXT: [0,1]     D=eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeER.   ldrsw	x1, [x27], #254
+# CHECK-NEXT: [0,3]     .D=eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeER   ldrsw	x1, [x27, #254]!
+# CHECK-NEXT: [0,5]     . D=eE--R   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeE--R   st1	{ v1.1d }, [x27], #8
+# CHECK-NEXT: [0,7]     .  D=eE-R   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeER   st1	{ v1.2d }, [x27], #16
+# CHECK-NEXT: [0,9]     .    DeER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3546,35 +3546,34 @@ add x0, x27, 1
 # CHECK-NEXT: 6.     1     1.0    0.0    2.0       st1	{ v1.1d }, [x27], #8
 # CHECK-NEXT: 7.     1     2.0    0.0    1.0       add	x0, x27, #1
 # CHECK-NEXT: 8.     1     1.0    0.0    0.0       st1	{ v1.2d }, [x27], #16
-# CHECK-NEXT: 9.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     1.5    0.1    0.9       <total>
+# CHECK-NEXT: 9.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.4    0.1    0.9       <total>
 
 # CHECK:      [52] Code Region - G53
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      704
+# CHECK-NEXT: Total Cycles:      703
 # CHECK-NEXT: Total uOps:        1700
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    2.41
+# CHECK-NEXT: uOps Per Cycle:    2.42
 # CHECK-NEXT: IPC:               1.42
 # CHECK-NEXT: Block RThroughput: 7.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeER .    .   st1	{ v1.2s }, [x27], #8
-# CHECK-NEXT: [0,1]     D=eER.    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeER.    .   st1	{ v1.4h }, [x27], #8
-# CHECK-NEXT: [0,3]     .D=eER    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . DeeER   .   st1	{ v1.4s }, [x27], #16
-# CHECK-NEXT: [0,5]     .  D=eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eER  .   st1	{ v1.8b }, [x27], #8
-# CHECK-NEXT: [0,7]     .   D=eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    DeeER.   st1	{ v1.8h }, [x27], #16
-# CHECK-NEXT: [0,9]     .    .D=eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeER .   .   st1	{ v1.2s }, [x27], #8
+# CHECK-NEXT: [0,1]     D=eER.   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeER.   .   st1	{ v1.4h }, [x27], #8
+# CHECK-NEXT: [0,3]     .D=eER   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeER  .   st1	{ v1.4s }, [x27], #16
+# CHECK-NEXT: [0,5]     .  DeER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eER .   st1	{ v1.8b }, [x27], #8
+# CHECK-NEXT: [0,7]     .   D=eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    DeeER   st1	{ v1.8h }, [x27], #16
+# CHECK-NEXT: [0,9]     .    .DeER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3588,12 +3587,12 @@ add x0, x27, 1
 # CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.4h }, [x27], #8
 # CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     1.0    0.0    0.0       st1	{ v1.4s }, [x27], #16
-# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st1	{ v1.8b }, [x27], #8
+# CHECK-NEXT: 5.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    1.0    0.0       st1	{ v1.8b }, [x27], #8
 # CHECK-NEXT: 7.     1     2.0    0.0    0.0       add	x0, x27, #1
 # CHECK-NEXT: 8.     1     1.0    0.0    0.0       st1	{ v1.8h }, [x27], #16
-# CHECK-NEXT: 9.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     1.6    0.1    0.0       <total>
+# CHECK-NEXT: 9.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.4    0.2    0.0       <total>
 
 # CHECK:      [53] Code Region - G54
 
@@ -3612,11 +3611,11 @@ add x0, x27, 1
 # CHECK-NEXT: Index     0123456789
 
 # CHECK:      [0,0]     DeeER.    .   st1	{ v1.16b }, [x27], #16
-# CHECK-NEXT: [0,1]     .D=eER    .   add	x0, x27, #1
+# CHECK-NEXT: [0,1]     .DeER.    .   add	x0, x27, #1
 # CHECK-NEXT: [0,2]     .D=eER    .   st1	{ v1.1d }, [x27], x28
 # CHECK-NEXT: [0,3]     . D=eER   .   add	x0, x27, #1
 # CHECK-NEXT: [0,4]     .  DeeER  .   st1	{ v1.2d }, [x27], x28
-# CHECK-NEXT: [0,5]     .   D=eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,5]     .   DeER  .   add	x0, x27, #1
 # CHECK-NEXT: [0,6]     .   D=eER .   st1	{ v1.2s }, [x27], x28
 # CHECK-NEXT: [0,7]     .    D=eER.   add	x0, x27, #1
 # CHECK-NEXT: [0,8]     .    D=eER.   st1	{ v1.4h }, [x27], x28
@@ -3630,22 +3629,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.16b }, [x27], #16
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.1d }, [x27], x28
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    1.0    0.0       st1	{ v1.1d }, [x27], x28
 # CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     1.0    0.0    0.0       st1	{ v1.2d }, [x27], x28
-# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st1	{ v1.2s }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    1.0    0.0       st1	{ v1.2s }, [x27], x28
 # CHECK-NEXT: 7.     1     2.0    0.0    0.0       add	x0, x27, #1
 # CHECK-NEXT: 8.     1     2.0    0.0    0.0       st1	{ v1.4h }, [x27], x28
 # CHECK-NEXT: 9.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     1.8    0.1    0.0       <total>
+# CHECK-NEXT:        1     1.6    0.3    0.0       <total>
 
 # CHECK:      [54] Code Region - G55
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      904
+# CHECK-NEXT: Total Cycles:      903
 # CHECK-NEXT: Total uOps:        1900
 
 # CHECK:      Dispatch Width:    3
@@ -3654,19 +3653,19 @@ add x0, x27, 1
 # CHECK-NEXT: Block RThroughput: 9.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     012
+# CHECK-NEXT:                     01
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeER.    . .   st1	{ v1.4s }, [x27], x28
-# CHECK-NEXT: [0,1]     .D=eER    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=eER    . .   st1	{ v1.8b }, [x27], x28
-# CHECK-NEXT: [0,3]     . D=eER   . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .  DeeER  . .   st1	{ v1.8h }, [x27], x28
-# CHECK-NEXT: [0,5]     .   D=eER . .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    DeeER. .   st1	{ v1.16b }, [x27], x28
-# CHECK-NEXT: [0,7]     .    .D=eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    . DeeER.   st1	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: [0,9]     .    .  D=eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.    ..   st1	{ v1.4s }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeER.    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .D=eER    ..   st1	{ v1.8b }, [x27], x28
+# CHECK-NEXT: [0,3]     . D=eER   ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .  DeeER  ..   st1	{ v1.8h }, [x27], x28
+# CHECK-NEXT: [0,5]     .   DeER  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    DeeER..   st1	{ v1.16b }, [x27], x28
+# CHECK-NEXT: [0,7]     .    .DeER..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    . DeeER   st1	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: [0,9]     .    .  DeER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3676,22 +3675,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.4s }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.8b }, [x27], x28
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    1.0    0.0       st1	{ v1.8b }, [x27], x28
 # CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     1.0    0.0    0.0       st1	{ v1.8h }, [x27], x28
-# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st1	{ v1.16b }, [x27], x28
-# CHECK-NEXT: 7.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     1.0    0.0    0.0       st1	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: 9.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     1.6    0.1    0.0       <total>
+# CHECK-NEXT: 5.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       st1	{ v1.16b }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st1	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: 9.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.2    0.4    0.0       <total>
 
 # CHECK:      [55] Code Region - G56
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1404
+# CHECK-NEXT: Total Cycles:      1403
 # CHECK-NEXT: Total uOps:        2400
 
 # CHECK:      Dispatch Width:    3
@@ -3700,19 +3699,19 @@ add x0, x27, 1
 # CHECK-NEXT: Block RThroughput: 14.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01234567
+# CHECK-NEXT:                     0123456
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeER   .    . .   st1	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: [0,1]     .D===eER  .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D==eeER .    . .   st1	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: [0,3]     .  D===eER.    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D==eeER    . .   st1	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: [0,5]     .    D===eER   . .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D==eeeeER. .   st1	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: [0,7]     .    . D=====eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .  D====eeER.   st1	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: [0,9]     .    .   D=====eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeER   .    ..   st1	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: [0,1]     .DeE--R   .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . D==eeER .    ..   st1	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: [0,3]     .  D==eER .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   D==eeER    ..   st1	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: [0,5]     .    D==eER    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .D==eeeeER..   st1	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: [0,7]     .    . D==eE--R..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  D====eeER   st1	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: [0,9]     .    .   D====eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3722,22 +3721,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: 1.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st1	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: 5.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st1	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: 7.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     5.0    0.0    0.0       st1	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: 9.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.9    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     3.0    3.0    0.0       st1	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    1.0    0.0       st1	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    1.0    0.0       st1	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: 7.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     5.0    3.0    0.0       st1	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: 9.     1     5.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.0    0.9    0.4       <total>
 
 # CHECK:      [56] Code Region - G57
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1604
+# CHECK-NEXT: Total Cycles:      1603
 # CHECK-NEXT: Total uOps:        2600
 
 # CHECK:      Dispatch Width:    3
@@ -3746,19 +3745,19 @@ add x0, x27, 1
 # CHECK-NEXT: Block RThroughput: 16.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
+# CHECK-NEXT:                     012345678
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeER   .    .   .   st1	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: [0,1]     .D===eER  .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D==eeeeER    .   .   st1	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: [0,3]     .  D=====eER   .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D====eeER  .   .   st1	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: [0,5]     .    D=====eER .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D====eeeeER  .   st1	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: [0,7]     .    . D=======eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .  D======eeER.   st1	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: [0,9]     .    .   D=======eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeER   .    .  .   st1	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: [0,1]     .DeE--R   .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . D==eeeeER    .  .   st1	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: [0,3]     .  D==eE--R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   D====eeER  .  .   st1	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: [0,5]     .    D====eER  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .D====eeeeER .   st1	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,7]     .    . D====eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  D======eeER   st1	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .   D======eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3768,22 +3767,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: 1.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st1	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: 3.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       st1	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: 5.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     5.0    0.0    0.0       st1	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: 7.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     7.0    0.0    0.0       st1	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: 9.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     5.3    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     3.0    3.0    0.0       st1	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     5.0    3.0    0.0       st1	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: 5.     1     5.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     5.0    1.0    0.0       st1	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: 7.     1     5.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     7.0    3.0    0.0       st1	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: 9.     1     7.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     4.2    1.1    0.6       <total>
 
 # CHECK:      [57] Code Region - G58
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1604
+# CHECK-NEXT: Total Cycles:      1603
 # CHECK-NEXT: Total uOps:        2600
 
 # CHECK:      Dispatch Width:    3
@@ -3792,19 +3791,19 @@ add x0, x27, 1
 # CHECK-NEXT: Block RThroughput: 16.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
+# CHECK-NEXT:                     012345678
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeER.    .    .   .   st1	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: [0,1]     .D=eER    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . DeeeeER .    .   .   st1	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: [0,3]     .  D===eER.    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D==eeER    .   .   st1	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: [0,5]     .    D===eER   .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D==eeeeER.   .   st1	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: [0,7]     .    . D=====eER   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .  D====eeeeER.   st1	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: [0,9]     .    .   D=======eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.    .    .  .   st1	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeER.    .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeER .    .  .   st1	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE--R .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   D==eeER    .  .   st1	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,5]     .    D==eER    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .D==eeeeER.  .   st1	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,7]     .    . D==eE--R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  D====eeeeER   st1	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .   D====eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3814,22 +3813,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: 5.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st1	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: 7.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     5.0    0.0    0.0       st1	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: 9.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.7    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st1	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    3.0    0.0       st1	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    1.0    0.0       st1	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: 7.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     5.0    3.0    0.0       st1	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: 9.     1     5.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.6    0.9    0.6       <total>
 
 # CHECK:      [58] Code Region - G59
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2104
+# CHECK-NEXT: Total Cycles:      2103
 # CHECK-NEXT: Total uOps:        3100
 
 # CHECK:      Dispatch Width:    3
@@ -3839,18 +3838,18 @@ add x0, x27, 1
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          01234
-
-# CHECK:      [0,0]     DeeeER    .    .    .   .   st1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: [0,1]     .D==eER   .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D=eeeeeeER   .    .   .   st1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: [0,3]     .   D=====eER  .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    D====eeeER.    .   .   st1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: [0,5]     .    .D======eER    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    . D=====eeeER  .   .   st1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: [0,7]     .    .  D=======eER .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .   D======eeeeeeER.   st1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: [0,9]     .    .    .D==========eER   add	x0, x27, #1
+# CHECK-NEXT: Index     0123456789          0123
+
+# CHECK:      [0,0]     DeeeER    .    .    .  .   st1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
+# CHECK-NEXT: [0,1]     .DeE-R    .    .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . D=eeeeeeER   .    .  .   st1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: [0,3]     .   DeE----R   .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    D====eeeER.    .  .   st1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: [0,5]     .    .D====eE-R.    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    . D=====eeeER  .  .   st1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: [0,7]     .    .  D=====eE-R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .   D======eeeeeeER   st1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: [0,9]     .    .    .D=====eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3860,22 +3859,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: 3.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       st1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: 5.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     6.0    0.0    0.0       st1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: 7.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     7.0    0.0    0.0       st1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: 9.     1     11.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     5.6    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    2.0    0.0       st1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: 3.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     5.0    5.0    0.0       st1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: 5.     1     5.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     6.0    2.0    0.0       st1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: 7.     1     6.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     7.0    2.0    0.0       st1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: 9.     1     6.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     4.0    1.2    1.1       <total>
 
 # CHECK:      [59] Code Region - G60
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2404
+# CHECK-NEXT: Total Cycles:      2403
 # CHECK-NEXT: Total uOps:        3400
 
 # CHECK:      Dispatch Width:    3
@@ -3885,18 +3884,18 @@ add x0, x27, 1
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          01234567
-
-# CHECK:      [0,0]     DeeeER    .    .    .    . .   st1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: [0,1]     .D==eER   .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D=eeeeeeER   .    .    . .   st1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: [0,3]     .   D=====eER  .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    D====eeeeeeER  .    . .   st1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: [0,5]     .    . D========eER .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .  D=======eeeER    . .   st1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: [0,7]     .    .   D=========eER   . .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    D========eeeeeeER.   st1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: [0,9]     .    .    . D============eER   add	x0, x27, #1
+# CHECK-NEXT: Index     0123456789          0123456
+
+# CHECK:      [0,0]     DeeeER    .    .    .    ..   st1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: [0,1]     .DeE-R    .    .    .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . D=eeeeeeER   .    .    ..   st1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: [0,3]     .   DeE----R   .    .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    D====eeeeeeER  .    ..   st1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: [0,5]     .    . D===eE----R  .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .  D=======eeeER    ..   st1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: [0,7]     .    .   D=======eE-R    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    D========eeeeeeER   st1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .    . D=======eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3906,22 +3905,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: 3.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       st1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: 5.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     8.0    0.0    0.0       st1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: 7.     1     10.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     9.0    0.0    0.0       st1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: 9.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     6.6    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    2.0    0.0       st1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: 3.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     5.0    5.0    0.0       st1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: 5.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     8.0    5.0    0.0       st1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: 7.     1     8.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     9.0    2.0    0.0       st1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: 9.     1     8.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     4.7    1.5    1.4       <total>
 
 # CHECK:      [60] Code Region - G61
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2104
+# CHECK-NEXT: Total Cycles:      2103
 # CHECK-NEXT: Total uOps:        3100
 
 # CHECK:      Dispatch Width:    3
@@ -3931,18 +3930,18 @@ add x0, x27, 1
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          01234
-
-# CHECK:      [0,0]     DeeeER    .    .    .   .   st1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: [0,1]     .D==eER   .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D=eeeER .    .    .   .   st1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: [0,3]     .  D===eER.    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D==eeeeeeER.    .   .   st1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: [0,5]     .    .D======eER    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    . D=====eeeER  .   .   st1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: [0,7]     .    .  D=======eER .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .   D======eeeeeeER.   st1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: [0,9]     .    .    .D==========eER   add	x0, x27, #1
+# CHECK-NEXT: Index     0123456789          0123
+
+# CHECK:      [0,0]     DeeeER    .    .    .  .   st1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE-R    .    .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . D=eeeER .    .    .  .   st1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     .  D=eE-R .    .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   D==eeeeeeER.    .  .   st1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: [0,5]     .    .D=eE----R.    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    . D=====eeeER  .  .   st1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,7]     .    .  D=====eE-R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .   D======eeeeeeER   st1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .    .D=====eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3952,22 +3951,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: 5.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     6.0    0.0    0.0       st1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: 7.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     7.0    0.0    0.0       st1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: 9.     1     11.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     5.2    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    2.0    0.0       st1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    2.0    0.0       st1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     6.0    5.0    0.0       st1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: 7.     1     6.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     7.0    2.0    0.0       st1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: 9.     1     6.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.6    1.2    1.1       <total>
 
 # CHECK:      [61] Code Region - G62
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2604
+# CHECK-NEXT: Total Cycles:      2603
 # CHECK-NEXT: Total uOps:        3600
 
 # CHECK:      Dispatch Width:    3
@@ -3977,18 +3976,18 @@ add x0, x27, 1
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT: Index     0123456789          012345678
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .   .   st1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: [0,1]     . D====eER.    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .  D===eeeeER  .    .    .   .   st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: [0,3]     .   D======eER .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    D=====eeeeeeeeER    .   .   st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: [0,5]     .    .  D==========eER   .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .   D=========eeeeER.   .   st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: [0,7]     .    .    D============eER   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    .D===========eeeeER.   st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: [0,9]     .    .    . D==============eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER .    .    .    .  .   st1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,1]     . DeE---R .    .    .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .  D===eeeeER  .    .    .  .   st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: [0,3]     .   D===eE--R  .    .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    D=====eeeeeeeeER    .  .   st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: [0,5]     .    .  D===eE------R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .   D=========eeeeER.  .   st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: [0,7]     .    .    D=========eE--R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    .D===========eeeeER   st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: [0,9]     .    .    . D===========eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3998,22 +3997,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: 1.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     4.0    0.0    0.0       st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: 3.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     6.0    0.0    0.0       st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: 5.     1     11.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     10.0   0.0    0.0       st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: 7.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     12.0   0.0    0.0       st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: 9.     1     15.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     8.4    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    1.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     4.0    4.0    0.0       st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: 3.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     6.0    3.0    0.0       st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: 5.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     10.0   7.0    0.0       st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: 7.     1     10.0   0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     12.0   3.0    0.0       st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: 9.     1     12.0   0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     6.4    1.9    1.5       <total>
 
 # CHECK:      [62] Code Region - G63
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3204
+# CHECK-NEXT: Total Cycles:      3203
 # CHECK-NEXT: Total uOps:        4200
 
 # CHECK:      Dispatch Width:    3
@@ -4022,19 +4021,19 @@ add x0, x27, 1
 # CHECK-NEXT: Block RThroughput: 32.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          012345
+# CHECK-NEXT:                     0123456789          01234
 # CHECK-NEXT: Index     0123456789          0123456789
 
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .   st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: [0,1]     .  D=====eER   .    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .   D====eeeeER.    .    .    .    .   st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: [0,3]     .    D=======eER    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    .D======eeeeeeeeER  .    .    .   st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: [0,5]     .    .   D===========eER .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .    D==========eeeeeeeeER    .   st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: [0,7]     .    .    .  D===============eER   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    .   D==============eeeeER.   st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: [0,9]     .    .    .    D=================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .   .   st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: [0,1]     .  DeE----R    .    .    .    .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .   D====eeeeER.    .    .    .   .   st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: [0,3]     .    D====eE--R.    .    .    .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .D======eeeeeeeeER  .    .   .   st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: [0,5]     .    .   D====eE------R  .    .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .    D==========eeeeeeeeER   .   st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: [0,7]     .    .    .  D========eE------R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    .   D==============eeeeER   st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .    .    D==============eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4044,22 +4043,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: 3.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     7.0    0.0    0.0       st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: 5.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     11.0   0.0    0.0       st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: 7.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     15.0   0.0    0.0       st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: 9.     1     18.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     9.9    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    1.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     5.0    5.0    0.0       st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: 3.     1     5.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     7.0    3.0    0.0       st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: 5.     1     5.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     11.0   7.0    0.0       st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: 7.     1     9.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     15.0   7.0    0.0       st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK-NEXT: 9.     1     15.0   0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     7.4    2.4    2.0       <total>
 
 # CHECK:      [63] Code Region - G64
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2804
+# CHECK-NEXT: Total Cycles:      2803
 # CHECK-NEXT: Total uOps:        3800
 
 # CHECK:      Dispatch Width:    3
@@ -4068,19 +4067,19 @@ add x0, x27, 1
 # CHECK-NEXT: Block RThroughput: 28.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          01
+# CHECK-NEXT:                     0123456789          0
 # CHECK-NEXT: Index     0123456789          0123456789
 
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    ..   st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: [0,1]     .  D=====eER   .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .   D====eeeeER.    .    .    ..   st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: [0,3]     .    D=======eER    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    .D======eeeeER .    .    ..   st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: [0,5]     .    . D=========eER.    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .  D========eeeeeeeeER   ..   st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: [0,7]     .    .    .D=============eER  ..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    . D============eeeeER.   st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: [0,9]     .    .    .  D===============eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .   st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,1]     .  DeE----R    .    .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .   D====eeeeER.    .    .    .   st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,3]     .    D====eE--R.    .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .D======eeeeER .    .    .   st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,5]     .    . D======eE--R .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .  D========eeeeeeeeER   .   st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,7]     .    .    .D======eE------R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    . D============eeeeER   st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .    .  D============eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4090,16 +4089,16 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: 3.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     7.0    0.0    0.0       st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: 5.     1     10.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     9.0    0.0    0.0       st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: 7.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     13.0   0.0    0.0       st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: 9.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     8.9    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    1.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     5.0    5.0    0.0       st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 3.     1     5.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     7.0    3.0    0.0       st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: 5.     1     7.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     9.0    3.0    0.0       st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: 7.     1     7.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     13.0   7.0    0.0       st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: 9.     1     13.0   0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     6.8    2.0    1.6       <total>
 
 # CHECK:      [64] Code Region - G65
 
@@ -4118,9 +4117,9 @@ add x0, x27, 1
 # CHECK-NEXT: Index     0123456789          012
 
 # CHECK:      [0,0]     DeeeeeeeeER    .    . .   st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: [0,1]     .  D=====eER   .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,1]     .  DeE----R    .    . .   add	x0, x27, #1
 # CHECK-NEXT: [0,2]     .   D====eeeeeeeeER . .   st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: [0,3]     .    . D=========eER. .   add	x0, x27, #1
+# CHECK-NEXT: [0,3]     .    . D==eE------R . .   add	x0, x27, #1
 # CHECK-NEXT: [0,4]     .    . D=========eER. .   st1	{ v1.b }[0], [x27], #1
 # CHECK-NEXT: [0,5]     .    .  D=========eER .   add	x0, x27, #1
 # CHECK-NEXT: [0,6]     .    .  D=========eER .   st1	{ v1.b }[8], [x27], #1
@@ -4136,16 +4135,16 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: 3.     1     10.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     10.0   0.0    0.0       st1	{ v1.b }[0], [x27], #1
+# CHECK-NEXT: 1.     1     1.0    1.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     5.0    5.0    0.0       st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     10.0   7.0    0.0       st1	{ v1.b }[0], [x27], #1
 # CHECK-NEXT: 5.     1     10.0   0.0    0.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     10.0   0.0    0.0       st1	{ v1.b }[8], [x27], #1
 # CHECK-NEXT: 7.     1     10.0   0.0    0.0       add	x0, x27, #1
 # CHECK-NEXT: 8.     1     10.0   0.0    0.0       st1	{ v1.b }[0], [x27], x28
 # CHECK-NEXT: 9.     1     10.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     8.2    0.1    0.0       <total>
+# CHECK-NEXT:        1     7.0    1.4    1.0       <total>
 
 # CHECK:      [65] Code Region - G66
 
@@ -4196,28 +4195,28 @@ add x0, x27, 1
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1204
+# CHECK-NEXT: Total Cycles:      805
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    1.66
-# CHECK-NEXT: IPC:               0.83
+# CHECK-NEXT: uOps Per Cycle:    2.48
+# CHECK-NEXT: IPC:               1.24
 # CHECK-NEXT: Block RThroughput: 8.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     012345
+# CHECK-NEXT:                     012
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeER .    .    .   st1	{ v1.s }[0], [x27], #4
-# CHECK-NEXT: [0,1]     D=eER.    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeER.    .    .   st1	{ v1.s }[0], [x27], x28
-# CHECK-NEXT: [0,3]     .D=eER    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . DeeeER  .    .   st1	{ v1.d }[0], [x27], #8
-# CHECK-NEXT: [0,5]     .  D==eER .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .   D=eeeER    .   st1	{ v1.d }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .    D===eER   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .D==eeeeER.   st2	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: [0,9]     .    . D=====eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeER .    . .   st1	{ v1.s }[0], [x27], #4
+# CHECK-NEXT: [0,1]     D=eER.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeER.    . .   st1	{ v1.s }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .D=eER    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeER  . .   st1	{ v1.d }[0], [x27], #8
+# CHECK-NEXT: [0,5]     .  DeE-R  . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .   DeeeER. .   st1	{ v1.d }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .    DeE-R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .DeeeeER   st2	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: [0,9]     .    . DeE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4231,39 +4230,39 @@ add x0, x27, 1
 # CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.s }[0], [x27], x28
 # CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     1.0    0.0    0.0       st1	{ v1.d }[0], [x27], #8
-# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st1	{ v1.d }[0], [x27], x28
-# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st2	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: 9.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.5    0.1    0.0       <total>
+# CHECK-NEXT: 5.     1     1.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       st1	{ v1.d }[0], [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st2	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: 9.     1     1.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.2    0.3    0.4       <total>
 
 # CHECK:      [67] Code Region - G68
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1704
+# CHECK-NEXT: Total Cycles:      1403
 # CHECK-NEXT: Total uOps:        3100
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    1.82
-# CHECK-NEXT: IPC:               0.59
+# CHECK-NEXT: uOps Per Cycle:    2.21
+# CHECK-NEXT: IPC:               0.71
 # CHECK-NEXT: Block RThroughput: 14.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0
+# CHECK-NEXT:                     0123456
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeER    .    .    .   st2	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: [0,1]     .D==eER   .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D=eeeER .    .    .   st2	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: [0,3]     .  D===eER.    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D==eeeeER  .    .   st2	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: [0,5]     .    .D====eER .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    . D===eeeER    .   st2	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: [0,7]     .    .  D=====eER   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .   D====eeeeER.   st2	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: [0,9]     .    .    .D======eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeER    .    ..   st2	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: [0,1]     .DeE-R    .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeER  .    ..   st2	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: [0,3]     .  DeE-R  .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeER    ..   st2	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: [0,5]     .    .DeE-R    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    . D=eeeER ..   st2	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: [0,7]     .    .  D=eE-R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .   D=eeeeER   st2	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: [0,9]     .    .    .DeE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4273,43 +4272,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st2	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st2	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: 5.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     4.0    0.0    0.0       st2	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: 7.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     5.0    0.0    0.0       st2	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: 9.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     4.0    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st2	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: 3.     1     1.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st2	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: 5.     1     1.0    1.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    2.0    0.0       st2	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: 7.     1     2.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     2.0    1.0    0.0       st2	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: 9.     1     1.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.3    0.7    0.6       <total>
 
 # CHECK:      [68] Code Region - G69
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1804
+# CHECK-NEXT: Total Cycles:      1603
 # CHECK-NEXT: Total uOps:        3200
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    1.77
-# CHECK-NEXT: IPC:               0.55
+# CHECK-NEXT: uOps Per Cycle:    2.00
+# CHECK-NEXT: IPC:               0.62
 # CHECK-NEXT: Block RThroughput: 16.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          01
+# CHECK-NEXT:                     012345678
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeER   .    .    ..   st2	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: [0,1]     . D==eER  .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .  D=eeeeER    .    ..   st2	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: [0,3]     .   D====eER   .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    D===eeeER .    ..   st2	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: [0,5]     .    .D=====eER.    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    . D====eeeER   ..   st2	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: [0,7]     .    .  D======eER  ..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .   D=====eeeeER.   st2	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: [0,9]     .    .    .D=======eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeER   .    .  .   st2	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: [0,1]     . DeE-R   .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .  D=eeeeER    .  .   st2	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,3]     .   D=eE--R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    D===eeeER .  .   st2	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,5]     .    .D===eE-R .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    . D===eeeER  .   st2	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,7]     .    .  D===eE-R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .   D===eeeeER   st2	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .    .D==eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4319,43 +4318,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st2	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: 3.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       st2	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: 5.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     5.0    0.0    0.0       st2	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: 7.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     6.0    0.0    0.0       st2	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: 9.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     4.7    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    1.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    2.0    0.0       st2	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     4.0    3.0    0.0       st2	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: 5.     1     4.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     4.0    1.0    0.0       st2	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    1.0    0.0       st2	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: 9.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.9    0.9    0.7       <total>
 
 # CHECK:      [69] Code Region - G70
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1704
+# CHECK-NEXT: Total Cycles:      1205
 # CHECK-NEXT: Total uOps:        2900
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    1.70
-# CHECK-NEXT: IPC:               0.59
+# CHECK-NEXT: uOps Per Cycle:    2.41
+# CHECK-NEXT: IPC:               0.83
 # CHECK-NEXT: Block RThroughput: 12.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0
+# CHECK-NEXT:                     0123456
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeER    .    .    .   st2	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: [0,1]     .D==eER   .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D=eeeeER.    .    .   st2	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: [0,3]     .   D===eER    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    D==eeeeER .    .   st2	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: [0,5]     .    . D====eER.    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .  D===eeeER   .   st2	{ v1.b, v2.b }[0], [x27], #2
-# CHECK-NEXT: [0,7]     .    .   D=====eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    D====eeeER.   st2	{ v1.b, v2.b }[8], [x27], #2
-# CHECK-NEXT: [0,9]     .    .    .D======eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeER    .    ..   st2	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE-R    .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeER .    ..   st2	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,3]     .   DeE-R .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    D=eeeeER  ..   st2	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,5]     .    . DeE--R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .  D==eeeER.   st2	{ v1.b, v2.b }[0], [x27], #2
+# CHECK-NEXT: [0,7]     .    .   D==eE-R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    D=eeeER   st2	{ v1.b, v2.b }[8], [x27], #2
+# CHECK-NEXT: [0,9]     .    .    .D=eE-R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4365,43 +4364,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st2	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st2	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: 5.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     4.0    0.0    0.0       st2	{ v1.b, v2.b }[0], [x27], #2
-# CHECK-NEXT: 7.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     5.0    0.0    0.0       st2	{ v1.b, v2.b }[8], [x27], #2
-# CHECK-NEXT: 9.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     4.0    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st2	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    1.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    2.0    0.0       st2	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    3.0    0.0       st2	{ v1.b, v2.b }[0], [x27], #2
+# CHECK-NEXT: 7.     1     3.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     2.0    0.0    0.0       st2	{ v1.b, v2.b }[8], [x27], #2
+# CHECK-NEXT: 9.     1     2.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.7    0.8    0.6       <total>
 
 # CHECK:      [70] Code Region - G71
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1504
+# CHECK-NEXT: Total Cycles:      1004
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    1.33
-# CHECK-NEXT: IPC:               0.66
+# CHECK-NEXT: uOps Per Cycle:    1.99
+# CHECK-NEXT: IPC:               1.00
 # CHECK-NEXT: Block RThroughput: 6.7
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     012345678
+# CHECK-NEXT:                     0123
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeER    .    .  .   st2	{ v1.b, v2.b }[0], [x27], x28
-# CHECK-NEXT: [0,1]     .D==eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D=eeeER .    .  .   st2	{ v1.b, v2.b }[8], [x27], x28
-# CHECK-NEXT: [0,3]     .  D===eER.    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D==eeeER   .  .   st2	{ v1.h, v2.h }[0], [x27], #4
-# CHECK-NEXT: [0,5]     .    D====eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D===eeeER.  .   st2	{ v1.h, v2.h }[4], [x27], #4
-# CHECK-NEXT: [0,7]     .    . D=====eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .  D====eeeER.   st2	{ v1.h, v2.h }[0], [x27], x28
-# CHECK-NEXT: [0,9]     .    .   D======eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeER    .  .   st2	{ v1.b, v2.b }[0], [x27], x28
+# CHECK-NEXT: [0,1]     .DeE-R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeER  .  .   st2	{ v1.b, v2.b }[8], [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE-R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeER.  .   st2	{ v1.h, v2.h }[0], [x27], #4
+# CHECK-NEXT: [0,5]     .    DeE-R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeER .   st2	{ v1.h, v2.h }[4], [x27], #4
+# CHECK-NEXT: [0,7]     .    . DeE-R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeER   st2	{ v1.h, v2.h }[0], [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeE-R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4411,43 +4410,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.b, v2.b }[0], [x27], x28
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st2	{ v1.b, v2.b }[8], [x27], x28
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st2	{ v1.h, v2.h }[0], [x27], #4
-# CHECK-NEXT: 5.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     4.0    0.0    0.0       st2	{ v1.h, v2.h }[4], [x27], #4
-# CHECK-NEXT: 7.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     5.0    0.0    0.0       st2	{ v1.h, v2.h }[0], [x27], x28
-# CHECK-NEXT: 9.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     4.0    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st2	{ v1.b, v2.b }[8], [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st2	{ v1.h, v2.h }[0], [x27], #4
+# CHECK-NEXT: 5.     1     1.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       st2	{ v1.h, v2.h }[4], [x27], #4
+# CHECK-NEXT: 7.     1     1.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st2	{ v1.h, v2.h }[0], [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    0.5       <total>
 
 # CHECK:      [71] Code Region - G72
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1304
+# CHECK-NEXT: Total Cycles:      1003
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    1.53
-# CHECK-NEXT: IPC:               0.77
+# CHECK-NEXT: uOps Per Cycle:    1.99
+# CHECK-NEXT: IPC:               1.00
 # CHECK-NEXT: Block RThroughput: 7.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456
+# CHECK-NEXT:                     012
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeER    .    ..   st2	{ v1.h, v2.h }[4], [x27], x28
-# CHECK-NEXT: [0,1]     .D==eER   .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D=eeeER .    ..   st2	{ v1.s, v2.s }[0], [x27], #8
-# CHECK-NEXT: [0,3]     .  D===eER.    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D==eeeER   ..   st2	{ v1.s, v2.s }[0], [x27], x28
-# CHECK-NEXT: [0,5]     .    D====eER  ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D===eeER ..   st2	{ v1.d, v2.d }[0], [x27], #16
-# CHECK-NEXT: [0,7]     .    . D====eER..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .  D===eeER.   st2	{ v1.d, v2.d }[0], [x27], x28
-# CHECK-NEXT: [0,9]     .    .   D====eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeER    . .   st2	{ v1.h, v2.h }[4], [x27], x28
+# CHECK-NEXT: [0,1]     .DeE-R    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeER  . .   st2	{ v1.s, v2.s }[0], [x27], #8
+# CHECK-NEXT: [0,3]     .  DeE-R  . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeER. .   st2	{ v1.s, v2.s }[0], [x27], x28
+# CHECK-NEXT: [0,5]     .    DeE-R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeER .   st2	{ v1.d, v2.d }[0], [x27], #16
+# CHECK-NEXT: [0,7]     .    . DeER .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeER   st2	{ v1.d, v2.d }[0], [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4457,22 +4456,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.h, v2.h }[4], [x27], x28
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st2	{ v1.s, v2.s }[0], [x27], #8
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st2	{ v1.s, v2.s }[0], [x27], x28
-# CHECK-NEXT: 5.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     4.0    0.0    0.0       st2	{ v1.d, v2.d }[0], [x27], #16
-# CHECK-NEXT: 7.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     4.0    0.0    0.0       st2	{ v1.d, v2.d }[0], [x27], x28
-# CHECK-NEXT: 9.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.6    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st2	{ v1.s, v2.s }[0], [x27], #8
+# CHECK-NEXT: 3.     1     1.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st2	{ v1.s, v2.s }[0], [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       st2	{ v1.d, v2.d }[0], [x27], #16
+# CHECK-NEXT: 7.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st2	{ v1.d, v2.d }[0], [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    0.3       <total>
 
 # CHECK:      [72] Code Region - G73
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      600
-# CHECK-NEXT: Total Cycles:      1204
+# CHECK-NEXT: Total Cycles:      1203
 # CHECK-NEXT: Total uOps:        2200
 
 # CHECK:      Dispatch Width:    3
@@ -4481,15 +4480,15 @@ add x0, x27, 1
 # CHECK-NEXT: Block RThroughput: 12.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     012345
+# CHECK-NEXT:                     01234
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    .   st3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: [0,1]     . D====eER.    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .  D===eeeER   .   st3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: [0,3]     .    D====eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    .D===eeeER.   st3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: [0,5]     .    .  D====eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER .   .   st3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: [0,1]     . DeE---R .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .  D===eeeER  .   st3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: [0,3]     .    D==eE-R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .D===eeeER   st3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: [0,5]     .    .  D==eE-R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4499,18 +4498,18 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: 1.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     4.0    0.0    0.0       st3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: 3.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       st3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: 5.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     4.0    0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    1.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     4.0    4.0    0.0       st3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: 3.     1     3.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     4.0    2.0    0.0       st3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: 5.     1     3.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    1.3    0.8       <total>
 
 # CHECK:      [73] Code Region - G74
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2704
+# CHECK-NEXT: Total Cycles:      2703
 # CHECK-NEXT: Total uOps:        5100
 
 # CHECK:      Dispatch Width:    3
@@ -4519,19 +4518,19 @@ add x0, x27, 1
 # CHECK-NEXT: Block RThroughput: 27.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0
+# CHECK-NEXT:                     0123456789
 # CHECK-NEXT: Index     0123456789          0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .   st3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: [0,1]     .  D===eER.    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .   D==eeeER   .    .    .    .   st3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: [0,3]     .    .D===eER  .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    . D==eeeeeeER  .    .    .   st3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: [0,5]     .    .    D=====eER .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .    .D====eeeeeeER .    .   st3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: [0,7]     .    .    .   D=======eER.    .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    .    D======eeeeeeER.   st3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: [0,9]     .    .    .    . D==========eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER .    .    .    .   .   st3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: [0,1]     .  DeE--R .    .    .    .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .   D==eeeER   .    .    .   .   st3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: [0,3]     .    .D=eE-R   .    .    .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    . D==eeeeeeER  .    .   .   st3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: [0,5]     .    .    DeE----R  .    .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .    .D====eeeeeeER .   .   st3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: [0,7]     .    .    .   D==eE----R .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    .    D======eeeeeeER   st3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .    .    . D=====eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4541,22 +4540,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: 1.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: 5.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     5.0    0.0    0.0       st3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: 7.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     7.0    0.0    0.0       st3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: 9.     1     11.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     5.2    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    1.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     3.0    3.0    0.0       st3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: 3.     1     2.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    2.0    0.0       st3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: 5.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     5.0    5.0    0.0       st3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: 7.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     7.0    5.0    0.0       st3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: 9.     1     6.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.2    1.7    1.5       <total>
 
 # CHECK:      [74] Code Region - G75
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2104
+# CHECK-NEXT: Total Cycles:      2103
 # CHECK-NEXT: Total uOps:        4500
 
 # CHECK:      Dispatch Width:    3
@@ -4566,18 +4565,18 @@ add x0, x27, 1
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          01234
-
-# CHECK:      [0,0]     DeeeER    .    .    .   .   st3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: [0,1]     . D=eER   .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .  DeeeER .    .    .   .   st3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: [0,3]     .    D=eER.    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    .DeeeeeeER.    .   .   st3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: [0,5]     .    .   D===eER    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .    D==eeeER  .   .   st3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: [0,7]     .    .    . D===eER .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    .  D==eeeeeeER.   st3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: [0,9]     .    .    .    .D=====eER   add	x0, x27, #1
+# CHECK-NEXT: Index     0123456789          0123
+
+# CHECK:      [0,0]     DeeeER    .    .    .  .   st3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,1]     . DeER    .    .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .  DeeeER .    .    .  .   st3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     .    DeER .    .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .DeeeeeeER.    .  .   st3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: [0,5]     .    .   DeE--R.    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .    D==eeeER  .  .   st3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,7]     .    .    . D=eE-R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    .  D==eeeeeeER   st3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .    .    .DeE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4587,43 +4586,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: 5.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: 9.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    1.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    1.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    1.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    3.0    0.0       st3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: 7.     1     2.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    2.0    0.0       st3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    1.1    0.7       <total>
 
 # CHECK:      [75] Code Region - G76
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1804
+# CHECK-NEXT: Total Cycles:      1204
 # CHECK-NEXT: Total uOps:        2800
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    1.55
-# CHECK-NEXT: IPC:               0.55
+# CHECK-NEXT: uOps Per Cycle:    2.33
+# CHECK-NEXT: IPC:               0.83
 # CHECK-NEXT: Block RThroughput: 10.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          01
+# CHECK-NEXT:                     012345
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    .    ..   st3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: [0,1]     .  D===eER.    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .   D==eeeER   .    ..   st3	{ v1.b, v2.b, v3.b }[0], [x27], #3
-# CHECK-NEXT: [0,3]     .    D====eER  .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    .D===eeeER.    ..   st3	{ v1.b, v2.b, v3.b }[8], [x27], #3
-# CHECK-NEXT: [0,5]     .    . D=====eER    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .  D====eeeER  ..   st3	{ v1.b, v2.b, v3.b }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .    .   D======eER ..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    D=====eeeER.   st3	{ v1.b, v2.b, v3.b }[8], [x27], x28
-# CHECK-NEXT: [0,9]     .    .    .D=======eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER .    .   st3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,1]     .  DeE--R .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .   D==eeeER   .   st3	{ v1.b, v2.b, v3.b }[0], [x27], #3
+# CHECK-NEXT: [0,3]     .    D==eE-R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .D=eeeER  .   st3	{ v1.b, v2.b, v3.b }[8], [x27], #3
+# CHECK-NEXT: [0,5]     .    . D=eE-R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .  DeeeER .   st3	{ v1.b, v2.b, v3.b }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .    .   DeE-R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    DeeeER   st3	{ v1.b, v2.b, v3.b }[8], [x27], x28
+# CHECK-NEXT: [0,9]     .    .    .DeE-R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4633,43 +4632,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: 1.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st3	{ v1.b, v2.b, v3.b }[0], [x27], #3
-# CHECK-NEXT: 3.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       st3	{ v1.b, v2.b, v3.b }[8], [x27], #3
-# CHECK-NEXT: 5.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     5.0    0.0    0.0       st3	{ v1.b, v2.b, v3.b }[0], [x27], x28
-# CHECK-NEXT: 7.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     6.0    0.0    0.0       st3	{ v1.b, v2.b, v3.b }[8], [x27], x28
-# CHECK-NEXT: 9.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     4.9    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    1.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     3.0    3.0    0.0       st3	{ v1.b, v2.b, v3.b }[0], [x27], #3
+# CHECK-NEXT: 3.     1     3.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st3	{ v1.b, v2.b, v3.b }[8], [x27], #3
+# CHECK-NEXT: 5.     1     2.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st3	{ v1.b, v2.b, v3.b }[0], [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st3	{ v1.b, v2.b, v3.b }[8], [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.6    0.6    0.6       <total>
 
 # CHECK:      [76] Code Region - G77
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1504
+# CHECK-NEXT: Total Cycles:      1004
 # CHECK-NEXT: Total uOps:        2100
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    1.40
-# CHECK-NEXT: IPC:               0.66
+# CHECK-NEXT: uOps Per Cycle:    2.09
+# CHECK-NEXT: IPC:               1.00
 # CHECK-NEXT: Block RThroughput: 7.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     012345678
+# CHECK-NEXT:                     0123
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeER    .    .  .   st3	{ v1.h, v2.h, v3.h }[0], [x27], #6
-# CHECK-NEXT: [0,1]     .D==eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D=eeeER .    .  .   st3	{ v1.h, v2.h, v3.h }[4], [x27], #6
-# CHECK-NEXT: [0,3]     .  D===eER.    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D==eeeER   .  .   st3	{ v1.h, v2.h, v3.h }[0], [x27], x28
-# CHECK-NEXT: [0,5]     .    D====eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D===eeeER.  .   st3	{ v1.h, v2.h, v3.h }[4], [x27], x28
-# CHECK-NEXT: [0,7]     .    . D=====eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .  D====eeeER.   st3	{ v1.s, v2.s, v3.s }[0], [x27], #12
-# CHECK-NEXT: [0,9]     .    .   D======eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeER    .  .   st3	{ v1.h, v2.h, v3.h }[0], [x27], #6
+# CHECK-NEXT: [0,1]     .DeE-R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeER  .  .   st3	{ v1.h, v2.h, v3.h }[4], [x27], #6
+# CHECK-NEXT: [0,3]     .  DeE-R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeER.  .   st3	{ v1.h, v2.h, v3.h }[0], [x27], x28
+# CHECK-NEXT: [0,5]     .    DeE-R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeER .   st3	{ v1.h, v2.h, v3.h }[4], [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeE-R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeER   st3	{ v1.s, v2.s, v3.s }[0], [x27], #12
+# CHECK-NEXT: [0,9]     .    .   DeE-R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4679,43 +4678,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.h, v2.h, v3.h }[0], [x27], #6
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st3	{ v1.h, v2.h, v3.h }[4], [x27], #6
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st3	{ v1.h, v2.h, v3.h }[0], [x27], x28
-# CHECK-NEXT: 5.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     4.0    0.0    0.0       st3	{ v1.h, v2.h, v3.h }[4], [x27], x28
-# CHECK-NEXT: 7.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     5.0    0.0    0.0       st3	{ v1.s, v2.s, v3.s }[0], [x27], #12
-# CHECK-NEXT: 9.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     4.0    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st3	{ v1.h, v2.h, v3.h }[4], [x27], #6
+# CHECK-NEXT: 3.     1     1.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st3	{ v1.h, v2.h, v3.h }[0], [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       st3	{ v1.h, v2.h, v3.h }[4], [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st3	{ v1.s, v2.s, v3.s }[0], [x27], #12
+# CHECK-NEXT: 9.     1     1.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    0.5       <total>
 
 # CHECK:      [77] Code Region - G78
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2104
+# CHECK-NEXT: Total Cycles:      1903
 # CHECK-NEXT: Total uOps:        3300
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    1.57
-# CHECK-NEXT: IPC:               0.48
+# CHECK-NEXT: uOps Per Cycle:    1.73
+# CHECK-NEXT: IPC:               0.53
 # CHECK-NEXT: Block RThroughput: 19.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          01234
-
-# CHECK:      [0,0]     DeeeER    .    .    .   .   st3	{ v1.s, v2.s, v3.s }[0], [x27], x28
-# CHECK-NEXT: [0,1]     .D==eER   .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D=eeeER .    .    .   .   st3	{ v1.d, v2.d, v3.d }[0], [x27], #24
-# CHECK-NEXT: [0,3]     .  D===eER.    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D==eeeER   .    .   .   st3	{ v1.d, v2.d, v3.d }[0], [x27], x28
-# CHECK-NEXT: [0,5]     .    D====eER  .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D===eeeeeeeeER.   .   st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: [0,7]     .    .   D========eER   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    D=======eeeeER.   st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: [0,9]     .    .    . D=========eER   add	x0, x27, #1
+# CHECK-NEXT: Index     0123456789          01
+
+# CHECK:      [0,0]     DeeeER    .    .    ..   st3	{ v1.s, v2.s, v3.s }[0], [x27], x28
+# CHECK-NEXT: [0,1]     .DeE-R    .    .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . D=eeeER .    .    ..   st3	{ v1.d, v2.d, v3.d }[0], [x27], #24
+# CHECK-NEXT: [0,3]     .  D=eE-R .    .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   D=eeeER    .    ..   st3	{ v1.d, v2.d, v3.d }[0], [x27], x28
+# CHECK-NEXT: [0,5]     .    D=eE-R    .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .D=eeeeeeeeER  ..   st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: [0,7]     .    .   DeE-----R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    D=====eeeeER   st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: [0,9]     .    .    . D====eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4725,22 +4724,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.s, v2.s, v3.s }[0], [x27], x28
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st3	{ v1.d, v2.d, v3.d }[0], [x27], #24
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st3	{ v1.d, v2.d, v3.d }[0], [x27], x28
-# CHECK-NEXT: 5.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     4.0    0.0    0.0       st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: 7.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     8.0    0.0    0.0       st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: 9.     1     10.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     4.9    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    2.0    0.0       st3	{ v1.d, v2.d, v3.d }[0], [x27], #24
+# CHECK-NEXT: 3.     1     2.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st3	{ v1.d, v2.d, v3.d }[0], [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    1.0    0.0       st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: 7.     1     1.0    1.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     6.0    6.0    0.0       st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: 9.     1     5.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.4    1.2    1.0       <total>
 
 # CHECK:      [78] Code Region - G79
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3204
+# CHECK-NEXT: Total Cycles:      3203
 # CHECK-NEXT: Total uOps:        5800
 
 # CHECK:      Dispatch Width:    3
@@ -4749,19 +4748,19 @@ add x0, x27, 1
 # CHECK-NEXT: Block RThroughput: 32.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          012345
+# CHECK-NEXT:                     0123456789          01234
 # CHECK-NEXT: Index     0123456789          0123456789
 
-# CHECK:      [0,0]     DeeeeER   .    .    .    .    .    .   st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: [0,1]     . D==eER  .    .    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .  D=eeeeeeeeER.    .    .    .    .   st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: [0,3]     .    . D=====eER    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    .  D====eeeeER .    .    .    .   st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: [0,5]     .    .    D======eER.    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .    .D=====eeeeeeeeER   .    .   st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: [0,7]     .    .    .    D=========eER  .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    .    .D========eeeeeeeeER.   st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: [0,9]     .    .    .    .    D============eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeER   .    .    .    .    .   .   st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: [0,1]     . DeE-R   .    .    .    .    .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .  D=eeeeeeeeER.    .    .    .   .   st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: [0,3]     .    . DeE----R.    .    .    .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .  D====eeeeER .    .    .   .   st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: [0,5]     .    .    D===eE--R .    .    .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .    .D=====eeeeeeeeER   .   .   st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: [0,7]     .    .    .    D==eE------R   .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    .    .D========eeeeeeeeER   st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: [0,9]     .    .    .    .    D=====eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4771,22 +4770,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: 3.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: 5.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     6.0    0.0    0.0       st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: 7.     1     10.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     9.0    0.0    0.0       st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: 9.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     6.2    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    1.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    2.0    0.0       st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: 3.     1     1.0    1.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     5.0    5.0    0.0       st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: 5.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     6.0    3.0    0.0       st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: 7.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     9.0    7.0    0.0       st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: 9.     1     6.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.8    2.0    1.9       <total>
 
 # CHECK:      [79] Code Region - G80
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2804
+# CHECK-NEXT: Total Cycles:      2803
 # CHECK-NEXT: Total uOps:        4800
 
 # CHECK:      Dispatch Width:    3
@@ -4795,19 +4794,19 @@ add x0, x27, 1
 # CHECK-NEXT: Block RThroughput: 28.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          01
+# CHECK-NEXT:                     0123456789          0
 # CHECK-NEXT: Index     0123456789          0123456789
 
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    ..   st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: [0,1]     .  D=====eER   .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .   D====eeeeER.    .    .    ..   st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: [0,3]     .    .D======eER    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    . D=====eeeeER .    .    ..   st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: [0,5]     .    .   D=======eER.    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .    D======eeeeeeeeER   ..   st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: [0,7]     .    .    .   D==========eER  ..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    .    D=========eeeeER.   st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: [0,9]     .    .    .    . D===========eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .   st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,1]     .  DeE----R    .    .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .   D====eeeeER.    .    .    .   st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,3]     .    .D===eE--R.    .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    . D=====eeeeER .    .    .   st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,5]     .    .   D====eE--R .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .    D======eeeeeeeeER   .   st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,7]     .    .    .   D===eE------R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    .    D=========eeeeER   st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .    .    . D========eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4817,43 +4816,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: 3.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     6.0    0.0    0.0       st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: 5.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     7.0    0.0    0.0       st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: 7.     1     11.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     10.0   0.0    0.0       st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: 9.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     7.3    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    1.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     5.0    5.0    0.0       st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 3.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     6.0    3.0    0.0       st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: 5.     1     5.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     7.0    3.0    0.0       st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     10.0   7.0    0.0       st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: 9.     1     9.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     5.2    2.0    1.6       <total>
 
 # CHECK:      [80] Code Region - G81
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2504
+# CHECK-NEXT: Total Cycles:      1905
 # CHECK-NEXT: Total uOps:        4000
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    1.60
-# CHECK-NEXT: IPC:               0.40
+# CHECK-NEXT: uOps Per Cycle:    2.10
+# CHECK-NEXT: IPC:               0.52
 # CHECK-NEXT: Block RThroughput: 19.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          012345678
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .  .   st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: [0,1]     .   D====eER   .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .    D===eeeeeeeeER .    .  .   st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: [0,3]     .    .   D=======eER.    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .    .    D======eeeER   .  .   st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
-# CHECK-NEXT: [0,5]     .    .    .D========eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .    . D=======eeeER.  .   st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
-# CHECK-NEXT: [0,7]     .    .    .  D=========eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .    .   D========eeeER.   st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
-# CHECK-NEXT: [0,9]     .    .    .    D==========eER   add	x0, x27, #1
+# CHECK-NEXT: Index     0123456789          0123
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .  .   st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,1]     .   DeE---R    .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .    D===eeeeeeeeER .  .   st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,3]     .    .   DeE------R .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .    D======eeeER .   st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
+# CHECK-NEXT: [0,5]     .    .    .D======eE-R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .    . D=====eeeER.   st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
+# CHECK-NEXT: [0,7]     .    .    .  D=====eE-R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    .   D====eeeER   st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
+# CHECK-NEXT: [0,9]     .    .    .    D====eE-R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4863,43 +4862,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: 1.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     4.0    0.0    0.0       st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: 3.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     7.0    0.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
-# CHECK-NEXT: 5.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     8.0    0.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
-# CHECK-NEXT: 7.     1     10.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     9.0    0.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
-# CHECK-NEXT: 9.     1     11.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     7.2    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    1.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     4.0    4.0    0.0       st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     7.0    7.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
+# CHECK-NEXT: 5.     1     7.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     6.0    0.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
+# CHECK-NEXT: 7.     1     6.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     5.0    0.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
+# CHECK-NEXT: 9.     1     5.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT:        1     4.3    1.3    1.2       <total>
 
 # CHECK:      [81] Code Region - G82
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1504
+# CHECK-NEXT: Total Cycles:      1004
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    1.33
-# CHECK-NEXT: IPC:               0.66
+# CHECK-NEXT: uOps Per Cycle:    1.99
+# CHECK-NEXT: IPC:               1.00
 # CHECK-NEXT: Block RThroughput: 6.7
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     012345678
+# CHECK-NEXT:                     0123
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeER    .    .  .   st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
-# CHECK-NEXT: [0,1]     .D==eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D=eeeER .    .  .   st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
-# CHECK-NEXT: [0,3]     .  D===eER.    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D==eeeER   .  .   st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
-# CHECK-NEXT: [0,5]     .    D====eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D===eeeER.  .   st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .    . D=====eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .  D====eeeER.   st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
-# CHECK-NEXT: [0,9]     .    .   D======eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeER    .  .   st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
+# CHECK-NEXT: [0,1]     .DeE-R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeER  .  .   st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
+# CHECK-NEXT: [0,3]     .  DeE-R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeER.  .   st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
+# CHECK-NEXT: [0,5]     .    DeE-R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeER .   st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeE-R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeER   st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeE-R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4909,41 +4908,41 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
-# CHECK-NEXT: 5.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     4.0    0.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
-# CHECK-NEXT: 7.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     5.0    0.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
-# CHECK-NEXT: 9.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     4.0    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
+# CHECK-NEXT: 3.     1     1.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
+# CHECK-NEXT: 5.     1     1.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    0.5       <total>
 
 # CHECK:      [82] Code Region - G83
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      800
-# CHECK-NEXT: Total Cycles:      1404
+# CHECK-NEXT: Total Cycles:      1204
 # CHECK-NEXT: Total uOps:        2200
 
 # CHECK:      Dispatch Width:    3
-# CHECK-NEXT: uOps Per Cycle:    1.57
-# CHECK-NEXT: IPC:               0.57
+# CHECK-NEXT: uOps Per Cycle:    1.83
+# CHECK-NEXT: IPC:               0.66
 # CHECK-NEXT: Block RThroughput: 12.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01234567
+# CHECK-NEXT:                     012345
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeER   .    . .   st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
-# CHECK-NEXT: [0,1]     .D===eER  .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D==eeeeER    . .   st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
-# CHECK-NEXT: [0,3]     .  D=====eER   . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D====eeeER . .   st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
-# CHECK-NEXT: [0,5]     .    D======eER. .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D=====eeeER.   st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .    . D=======eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeER   .    .   st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
+# CHECK-NEXT: [0,1]     .DeE--R   .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . D==eeeeER    .   st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .  D==eE--R    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   D====eeeER .   st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
+# CHECK-NEXT: [0,5]     .    D====eE-R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .D====eeeER   st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .    . D====eE-R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4953,14 +4952,14 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
-# CHECK-NEXT: 1.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
-# CHECK-NEXT: 3.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
-# CHECK-NEXT: 5.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     6.0    0.0    0.0       st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
-# CHECK-NEXT: 7.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     5.0    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     3.0    3.0    0.0       st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     5.0    3.0    0.0       st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
+# CHECK-NEXT: 5.     1     5.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     5.0    1.0    0.0       st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
+# CHECK-NEXT: 7.     1     5.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.5    1.0    0.8       <total>
 
 # CHECK:      [83] Code Region - G84
 
diff --git a/llvm/test/tools/llvm-mca/AArch64/HiSilicon/tsv110-writeback.s b/llvm/test/tools/llvm-mca/AArch64/HiSilicon/tsv110-writeback.s
new file mode 100644
index 0000000000..2738f0b54c
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/AArch64/HiSilicon/tsv110-writeback.s
@@ -0,0 +1,3954 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=aarch64 -mcpu=tsv110 --instruction-info=0 --resource-pressure=0 --timeline --timeline-max-iterations=1 < %s | FileCheck %s
+
+# LLVM-MCA-BEGIN G01
+ld1  { v1.1d }, [x27], #8
+ld1  { v1.2d }, [x27], #16
+ld1  { v1.2s }, [x27], #8
+ld1  { v1.4h }, [x27], #8
+ld1  { v1.4s }, [x27], #16
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G02
+ld1  { v1.8b }, [x27], #8
+ld1  { v1.8h }, [x27], #16
+ld1  { v1.16b }, [x27], #16
+ld1  { v1.1d }, [x27], x28
+ld1  { v1.2d }, [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G03
+ld1  { v1.2s }, [x27], x28
+ld1  { v1.4h }, [x27], x28
+ld1  { v1.4s }, [x27], x28
+ld1  { v1.8b }, [x27], x28
+ld1  { v1.8h }, [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G04
+ld1  { v1.16b }, [x27], x28
+ld1  { v1.1d, v2.1d }, [x27], #16
+ld1  { v1.2d, v2.2d }, [x27], #32
+ld1  { v1.2s, v2.2s }, [x27], #16
+ld1  { v1.4h, v2.4h }, [x27], #16
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G05
+ld1  { v1.4s, v2.4s }, [x27], #32
+ld1  { v1.8b, v2.8b }, [x27], #16
+ld1  { v1.8h, v2.8h }, [x27], #32
+ld1  { v1.16b, v2.16b }, [x27], #32
+ld1  { v1.1d, v2.1d }, [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G06
+ld1  { v1.2d, v2.2d }, [x27], x28
+ld1  { v1.2s, v2.2s }, [x27], x28
+ld1  { v1.4h, v2.4h }, [x27], x28
+ld1  { v1.4s, v2.4s }, [x27], x28
+ld1  { v1.8b, v2.8b }, [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G07
+ld1  { v1.8h, v2.8h }, [x27], x28
+ld1  { v1.16b, v2.16b }, [x27], x28
+ld1  { v1.1d, v2.1d, v3.1d }, [x27], #24
+ld1  { v1.2d, v2.2d, v3.2d }, [x27], #48
+ld1  { v1.2s, v2.2s, v3.2s }, [x27], #24
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G08
+ld1  { v1.4h, v2.4h, v3.4h }, [x27], #24
+ld1  { v1.4s, v2.4s, v3.4s }, [x27], #48
+ld1  { v1.8b, v2.8b, v3.8b }, [x27], #24
+ld1  { v1.8h, v2.8h, v3.8h }, [x27], #48
+ld1  { v1.16b, v2.16b, v3.16b }, [x27], #48
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G09
+ld1  { v1.1d, v2.1d, v3.1d }, [x27], x28
+ld1  { v1.2d, v2.2d, v3.2d }, [x27], x28
+ld1  { v1.2s, v2.2s, v3.2s }, [x27], x28
+ld1  { v1.4h, v2.4h, v3.4h }, [x27], x28
+ld1  { v1.4s, v2.4s, v3.4s }, [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G10
+ld1  { v1.8b, v2.8b, v3.8b }, [x27], x28
+ld1  { v1.8h, v2.8h, v3.8h }, [x27], x28
+ld1  { v1.16b, v2.16b, v3.16b }, [x27], x28
+ld1  { v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+ld1  { v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G11
+ld1  { v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+ld1  { v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+ld1  { v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+ld1  { v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+ld1  { v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G12
+ld1  { v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+ld1  { v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+ld1  { v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+ld1  { v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+ld1  { v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G13
+ld1  { v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+ld1  { v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+ld1  { v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+ld1  { v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+ld1  { v1.b }[0], [x27], #1
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G14
+ld1  { v1.b }[8], [x27], #1
+ld1  { v1.b }[0], [x27], x28
+ld1  { v1.b }[8], [x27], x28
+ld1  { v1.h }[0], [x27], #2
+ld1  { v1.h }[4], [x27], #2
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G15
+ld1  { v1.h }[0], [x27], x28
+ld1  { v1.h }[4], [x27], x28
+ld1  { v1.s }[0], [x27], #4
+ld1  { v1.s }[0], [x27], x28
+ld1  { v1.d }[0], [x27], #8
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G16
+ld1  { v1.d }[0], [x27], x28
+ld1r  { v1.1d }, [x27], #8
+ld1r  { v1.2d }, [x27], #8
+ld1r  { v1.2s }, [x27], #4
+ld1r  { v1.4h }, [x27], #2
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G17
+ld1r  { v1.4s }, [x27], #4
+ld1r  { v1.8b }, [x27], #1
+ld1r  { v1.8h }, [x27], #2
+ld1r  { v1.16b }, [x27], #1
+ld1r  { v1.1d }, [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G18
+ld1r  { v1.2d }, [x27], x28
+ld1r  { v1.2s }, [x27], x28
+ld1r  { v1.4h }, [x27], x28
+ld1r  { v1.4s }, [x27], x28
+ld1r  { v1.8b }, [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G19
+ld1r  { v1.8h }, [x27], x28
+ld1r  { v1.16b }, [x27], x28
+ld2  { v1.2d, v2.2d }, [x27], #32
+ld2  { v1.2s, v2.2s }, [x27], #16
+ld2  { v1.4h, v2.4h }, [x27], #16
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G20
+ld2  { v1.4s, v2.4s }, [x27], #32
+ld2  { v1.8b, v2.8b }, [x27], #16
+ld2  { v1.8h, v2.8h }, [x27], #32
+ld2  { v1.16b, v2.16b }, [x27], #32
+ld2  { v1.2d, v2.2d }, [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G21
+ld2  { v1.2s, v2.2s }, [x27], x28
+ld2  { v1.4h, v2.4h }, [x27], x28
+ld2  { v1.4s, v2.4s }, [x27], x28
+ld2  { v1.8b, v2.8b }, [x27], x28
+ld2  { v1.8h, v2.8h }, [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G22
+ld2  { v1.16b, v2.16b }, [x27], x28
+ld2  { v1.b, v2.b }[0], [x27], #2
+ld2  { v1.b, v2.b }[8], [x27], #2
+ld2  { v1.b, v2.b }[0], [x27], x28
+ld2  { v1.b, v2.b }[8], [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G23
+ld2  { v1.h, v2.h }[0], [x27], #4
+ld2  { v1.h, v2.h }[4], [x27], #4
+ld2  { v1.h, v2.h }[0], [x27], x28
+ld2  { v1.h, v2.h }[4], [x27], x28
+ld2  { v1.s, v2.s }[0], [x27], #8
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G24
+ld2  { v1.s, v2.s }[0], [x27], x28
+ld2  { v1.d, v2.d }[0], [x27], #16
+ld2  { v1.d, v2.d }[0], [x27], x28
+ld2r  { v1.1d, v2.1d }, [x27], #16
+ld2r  { v1.2d, v2.2d }, [x27], #16
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G25
+ld2r  { v1.2s, v2.2s }, [x27], #8
+ld2r  { v1.4h, v2.4h }, [x27], #4
+ld2r  { v1.4s, v2.4s }, [x27], #8
+ld2r  { v1.8b, v2.8b }, [x27], #2
+ld2r  { v1.8h, v2.8h }, [x27], #4
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G26
+ld2r  { v1.16b, v2.16b }, [x27], #2
+ld2r  { v1.1d, v2.1d }, [x27], x28
+ld2r  { v1.2d, v2.2d }, [x27], x28
+ld2r  { v1.2s, v2.2s }, [x27], x28
+ld2r  { v1.4h, v2.4h }, [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G27
+ld2r  { v1.4s, v2.4s }, [x27], x28
+ld2r  { v1.8b, v2.8b }, [x27], x28
+ld2r  { v1.8h, v2.8h }, [x27], x28
+ld2r  { v1.16b, v2.16b }, [x27], x28
+ld3  { v1.2d, v2.2d, v3.2d }, [x27], #48
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G28
+ld3  { v1.2s, v2.2s, v3.2s }, [x27], #24
+ld3  { v1.4h, v2.4h, v3.4h }, [x27], #24
+ld3  { v1.4s, v2.4s, v3.4s }, [x27], #48
+ld3  { v1.8b, v2.8b, v3.8b }, [x27], #24
+ld3  { v1.8h, v2.8h, v3.8h }, [x27], #48
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G29
+ld3  { v1.16b, v2.16b, v3.16b }, [x27], #48
+ld3  { v1.2d, v2.2d, v3.2d }, [x27], x28
+ld3  { v1.2s, v2.2s, v3.2s }, [x27], x28
+ld3  { v1.4h, v2.4h, v3.4h }, [x27], x28
+ld3  { v1.4s, v2.4s, v3.4s }, [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G30
+ld3  { v1.8b, v2.8b, v3.8b }, [x27], x28
+ld3  { v1.8h, v2.8h, v3.8h }, [x27], x28
+ld3  { v1.16b, v2.16b, v3.16b }, [x27], x28
+ld3  { v1.b, v2.b, v3.b }[0], [x27], #3
+ld3  { v1.b, v2.b, v3.b }[8], [x27], #3
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G31
+ld3  { v1.b, v2.b, v3.b }[0], [x27], x28
+ld3  { v1.b, v2.b, v3.b }[8], [x27], x28
+ld3  { v1.h, v2.h, v3.h }[0], [x27], #6
+ld3  { v1.h, v2.h, v3.h }[4], [x27], #6
+ld3  { v1.h, v2.h, v3.h }[0], [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G32
+ld3  { v1.h, v2.h, v3.h }[4], [x27], x28
+ld3  { v1.s, v2.s, v3.s }[0], [x27], #12
+ld3  { v1.s, v2.s, v3.s }[0], [x27], x28
+ld3  { v1.d, v2.d, v3.d }[0], [x27], #24
+ld3  { v1.d, v2.d, v3.d }[0], [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G33
+ld3r  { v1.1d, v2.1d, v3.1d }, [x27], #24
+ld3r  { v1.2d, v2.2d, v3.2d }, [x27], #24
+ld3r  { v1.2s, v2.2s, v3.2s }, [x27], #12
+ld3r  { v1.4h, v2.4h, v3.4h }, [x27], #6
+ld3r  { v1.4s, v2.4s, v3.4s }, [x27], #12
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G34
+ld3r  { v1.8b, v2.8b, v3.8b }, [x27], #3
+ld3r  { v1.8h, v2.8h, v3.8h }, [x27], #6
+ld3r  { v1.16b, v2.16b, v3.16b }, [x27], #3
+ld3r  { v1.1d, v2.1d, v3.1d }, [x27], x28
+ld3r  { v1.2d, v2.2d, v3.2d }, [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G35
+ld3r  { v1.2s, v2.2s, v3.2s }, [x27], x28
+ld3r  { v1.4h, v2.4h, v3.4h }, [x27], x28
+ld3r  { v1.4s, v2.4s, v3.4s }, [x27], x28
+ld3r  { v1.8b, v2.8b, v3.8b }, [x27], x28
+ld3r  { v1.8h, v2.8h, v3.8h }, [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G36
+ld3r  { v1.16b, v2.16b, v3.16b }, [x27], x28
+ld4  { v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+ld4  { v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+ld4  { v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+ld4  { v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G37
+ld4  { v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+ld4  { v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+ld4  { v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+ld4  { v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+ld4  { v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G38
+ld4  { v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+ld4  { v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+ld4  { v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+ld4  { v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+ld4  { v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G39
+ld4  { v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
+ld4  { v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
+ld4  { v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
+ld4  { v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
+ld4  { v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G40
+ld4  { v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
+ld4  { v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
+ld4  { v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
+ld4  { v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
+ld4  { v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G41
+ld4  { v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
+ld4  { v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
+ld4r  { v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+ld4r  { v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #32
+ld4r  { v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #16
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G42
+ld4r  { v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #8
+ld4r  { v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #16
+ld4r  { v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #4
+ld4r  { v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #8
+ld4r  { v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #4
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G43
+ld4r  { v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+ld4r  { v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+ld4r  { v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+ld4r  { v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+ld4r  { v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G44
+ld4r  { v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+ld4r  { v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+ld4r  { v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+ldp  s1, s2, [x27], #248
+ldp  d1, d2, [x27], #496
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G45
+ldp  q1, q2, [x27], #992
+ldp  s1, s2, [x27, #248]!
+ldp  d1, d2, [x27, #496]!
+ldp  q1, q2, [x27, #992]!
+ldp  w1, w2, [x27], #248
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G46
+ldp  x1, x2, [x27], #496
+ldp  w1, w2, [x27, #248]!
+ldp  x1, x2, [x27, #496]!
+ldpsw  x1, x2, [x27], #248
+ldpsw  x1, x2, [x27, #248]!
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G47
+ldr  b1, [x27], #254
+ldr  h1, [x27], #254
+ldr  s1, [x27], #254
+ldr  d1, [x27], #254
+ldr  q1, [x27], #254
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G48
+ldr  b1, [x27, #254]!
+ldr  h1, [x27, #254]!
+ldr  s1, [x27, #254]!
+ldr  d1, [x27, #254]!
+ldr  q1, [x27, #254]!
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G49
+ldr  w1, [x27], #254
+ldr  x1, [x27], #254
+ldr  w1, [x27, #254]!
+ldr  x1, [x27, #254]!
+ldrb  w1, [x27], #254
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G50
+ldrb  w1, [x27, #254]!
+ldrh  w1, [x27], #254
+ldrh  w1, [x27, #254]!
+ldrsb  w1, [x27], #254
+ldrsb  x1, [x27], #254
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G51
+ldrsb  w1, [x27, #254]!
+ldrsb  x1, [x27, #254]!
+ldrsh  w1, [x27], #254
+ldrsh  x1, [x27], #254
+ldrsh  w1, [x27, #254]!
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G52
+ldrsh  x1, [x27, #254]!
+ldrsw  x1, [x27], #254
+ldrsw  x1, [x27, #254]!
+st1  { v1.1d }, [x27], #8
+st1  { v1.2d }, [x27], #16
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G53
+st1  { v1.2s }, [x27], #8
+st1  { v1.4h }, [x27], #8
+st1  { v1.4s }, [x27], #16
+st1  { v1.8b }, [x27], #8
+st1  { v1.8h }, [x27], #16
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G54
+st1  { v1.16b }, [x27], #16
+st1  { v1.1d }, [x27], x28
+st1  { v1.2d }, [x27], x28
+st1  { v1.2s }, [x27], x28
+st1  { v1.4h }, [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G55
+st1  { v1.4s }, [x27], x28
+st1  { v1.8b }, [x27], x28
+st1  { v1.8h }, [x27], x28
+st1  { v1.16b }, [x27], x28
+st1  { v1.1d, v2.1d }, [x27], #16
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G56
+st1  { v1.2d, v2.2d }, [x27], #32
+st1  { v1.2s, v2.2s }, [x27], #16
+st1  { v1.4h, v2.4h }, [x27], #16
+st1  { v1.4s, v2.4s }, [x27], #32
+st1  { v1.8b, v2.8b }, [x27], #16
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G57
+st1  { v1.8h, v2.8h }, [x27], #32
+st1  { v1.16b, v2.16b }, [x27], #32
+st1  { v1.1d, v2.1d }, [x27], x28
+st1  { v1.2d, v2.2d }, [x27], x28
+st1  { v1.2s, v2.2s }, [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G58
+st1  { v1.4h, v2.4h }, [x27], x28
+st1  { v1.4s, v2.4s }, [x27], x28
+st1  { v1.8b, v2.8b }, [x27], x28
+st1  { v1.8h, v2.8h }, [x27], x28
+st1  { v1.16b, v2.16b }, [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G59
+st1  { v1.1d, v2.1d, v3.1d }, [x27], #24
+st1  { v1.2d, v2.2d, v3.2d }, [x27], #48
+st1  { v1.2s, v2.2s, v3.2s }, [x27], #24
+st1  { v1.4h, v2.4h, v3.4h }, [x27], #24
+st1  { v1.4s, v2.4s, v3.4s }, [x27], #48
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G60
+st1  { v1.8b, v2.8b, v3.8b }, [x27], #24
+st1  { v1.8h, v2.8h, v3.8h }, [x27], #48
+st1  { v1.16b, v2.16b, v3.16b }, [x27], #48
+st1  { v1.1d, v2.1d, v3.1d }, [x27], x28
+st1  { v1.2d, v2.2d, v3.2d }, [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G61
+st1  { v1.2s, v2.2s, v3.2s }, [x27], x28
+st1  { v1.4h, v2.4h, v3.4h }, [x27], x28
+st1  { v1.4s, v2.4s, v3.4s }, [x27], x28
+st1  { v1.8b, v2.8b, v3.8b }, [x27], x28
+st1  { v1.8h, v2.8h, v3.8h }, [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G62
+st1  { v1.16b, v2.16b, v3.16b }, [x27], x28
+st1  { v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+st1  { v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+st1  { v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+st1  { v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G63
+st1  { v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+st1  { v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+st1  { v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+st1  { v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+st1  { v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G64
+st1  { v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+st1  { v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+st1  { v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+st1  { v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+st1  { v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G65
+st1  { v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+st1  { v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+st1  { v1.b }[0], [x27], #1
+st1  { v1.b }[8], [x27], #1
+st1  { v1.b }[0], [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G66
+st1  { v1.b }[8], [x27], x28
+st1  { v1.h }[0], [x27], #2
+st1  { v1.h }[4], [x27], #2
+st1  { v1.h }[0], [x27], x28
+st1  { v1.h }[4], [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G67
+st1  { v1.s }[0], [x27], #4
+st1  { v1.s }[0], [x27], x28
+st1  { v1.d }[0], [x27], #8
+st1  { v1.d }[0], [x27], x28
+st2  { v1.2d, v2.2d }, [x27], #32
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G68
+st2  { v1.2s, v2.2s }, [x27], #16
+st2  { v1.4h, v2.4h }, [x27], #16
+st2  { v1.4s, v2.4s }, [x27], #32
+st2  { v1.8b, v2.8b }, [x27], #16
+st2  { v1.8h, v2.8h }, [x27], #32
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G69
+st2  { v1.16b, v2.16b }, [x27], #32
+st2  { v1.2d, v2.2d }, [x27], x28
+st2  { v1.2s, v2.2s }, [x27], x28
+st2  { v1.4h, v2.4h }, [x27], x28
+st2  { v1.4s, v2.4s }, [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G70
+st2  { v1.8b, v2.8b }, [x27], x28
+st2  { v1.8h, v2.8h }, [x27], x28
+st2  { v1.16b, v2.16b }, [x27], x28
+st2  { v1.b, v2.b }[0], [x27], #2
+st2  { v1.b, v2.b }[8], [x27], #2
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G71
+st2  { v1.b, v2.b }[0], [x27], x28
+st2  { v1.b, v2.b }[8], [x27], x28
+st2  { v1.h, v2.h }[0], [x27], #4
+st2  { v1.h, v2.h }[4], [x27], #4
+st2  { v1.h, v2.h }[0], [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G72
+st2  { v1.h, v2.h }[4], [x27], x28
+st2  { v1.s, v2.s }[0], [x27], #8
+st2  { v1.s, v2.s }[0], [x27], x28
+st2  { v1.d, v2.d }[0], [x27], #16
+st2  { v1.d, v2.d }[0], [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G73
+st3  { v1.2d, v2.2d, v3.2d }, [x27], #48
+st3  { v1.2s, v2.2s, v3.2s }, [x27], #24
+st3  { v1.4h, v2.4h, v3.4h }, [x27], #24
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G74
+st3  { v1.4s, v2.4s, v3.4s }, [x27], #48
+st3  { v1.8b, v2.8b, v3.8b }, [x27], #24
+st3  { v1.8h, v2.8h, v3.8h }, [x27], #48
+st3  { v1.16b, v2.16b, v3.16b }, [x27], #48
+st3  { v1.2d, v2.2d, v3.2d }, [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G75
+st3  { v1.2s, v2.2s, v3.2s }, [x27], x28
+st3  { v1.4h, v2.4h, v3.4h }, [x27], x28
+st3  { v1.4s, v2.4s, v3.4s }, [x27], x28
+st3  { v1.8b, v2.8b, v3.8b }, [x27], x28
+st3  { v1.8h, v2.8h, v3.8h }, [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G76
+st3  { v1.16b, v2.16b, v3.16b }, [x27], x28
+st3  { v1.b, v2.b, v3.b }[0], [x27], #3
+st3  { v1.b, v2.b, v3.b }[8], [x27], #3
+st3  { v1.b, v2.b, v3.b }[0], [x27], x28
+st3  { v1.b, v2.b, v3.b }[8], [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G77
+st3  { v1.h, v2.h, v3.h }[0], [x27], #6
+st3  { v1.h, v2.h, v3.h }[4], [x27], #6
+st3  { v1.h, v2.h, v3.h }[0], [x27], x28
+st3  { v1.h, v2.h, v3.h }[4], [x27], x28
+st3  { v1.s, v2.s, v3.s }[0], [x27], #12
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G78
+st3  { v1.s, v2.s, v3.s }[0], [x27], x28
+st3  { v1.d, v2.d, v3.d }[0], [x27], #24
+st3  { v1.d, v2.d, v3.d }[0], [x27], x28
+st4  { v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+st4  { v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G79
+st4  { v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+st4  { v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+st4  { v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+st4  { v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+st4  { v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G80
+st4  { v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+st4  { v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+st4  { v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+st4  { v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+st4  { v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G81
+st4  { v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+st4  { v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+st4  { v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
+st4  { v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
+st4  { v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G82
+st4  { v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
+st4  { v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
+st4  { v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
+st4  { v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
+st4  { v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G83
+st4  { v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
+st4  { v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
+st4  { v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
+st4  { v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G84
+stp  s1, s2, [x27], #248
+stp  d1, d2, [x27], #496
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G85
+stp  q1, q2, [x27], #992
+stp  s1, s2, [x27, #248]!
+stp  d1, d2, [x27, #496]!
+stp  q1, q2, [x27, #992]!
+stp  w1, w2, [x27], #248
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G86
+stp  x1, x2, [x27], #496
+stp  w1, w2, [x27, #248]!
+stp  x1, x2, [x27, #496]!
+str  b1, [x27], #254
+str  h1, [x27], #254
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G87
+str  s1, [x27], #254
+str  d1, [x27], #254
+str  q1, [x27], #254
+str  b1, [x27, #254]!
+str  h1, [x27, #254]!
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G88
+str  s1, [x27, #254]!
+str  d1, [x27, #254]!
+str  q1, [x27, #254]!
+str  w1, [x27], #254
+str  x1, [x27], #254
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G89
+str  w1, [x27, #254]!
+str  x1, [x27, #254]!
+strb  w1, [x27], #254
+strb  w1, [x27, #254]!
+strh  w1, [x27], #254
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G90
+strh  w1, [x27, #254]!
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN G91
+ldr  x1, [x27], #254
+ldr  x2, [x1], #254
+# LLVM-MCA-END
+
+# CHECK:      [0] Code Region - G01
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      507
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.97
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeER  ..   ld1	{ v1.1d }, [x27], #8
+# CHECK-NEXT: [0,1]     D=eeeeeER ..   ld1	{ v1.2d }, [x27], #16
+# CHECK-NEXT: [0,2]     .D=eeeeeER..   ld1	{ v1.2s }, [x27], #8
+# CHECK-NEXT: [0,3]     .D==eeeeeER.   ld1	{ v1.4h }, [x27], #8
+# CHECK-NEXT: [0,4]     . D==eeeeeER   ld1	{ v1.4s }, [x27], #16
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.1d }, [x27], #8
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       ld1	{ v1.2d }, [x27], #16
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.2s }, [x27], #8
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       ld1	{ v1.4h }, [x27], #8
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1	{ v1.4s }, [x27], #16
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [1] Code Region - G02
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      507
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.97
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeER  ..   ld1	{ v1.8b }, [x27], #8
+# CHECK-NEXT: [0,1]     D=eeeeeER ..   ld1	{ v1.8h }, [x27], #16
+# CHECK-NEXT: [0,2]     .D=eeeeeER..   ld1	{ v1.16b }, [x27], #16
+# CHECK-NEXT: [0,3]     .D==eeeeeER.   ld1	{ v1.1d }, [x27], x28
+# CHECK-NEXT: [0,4]     . D==eeeeeER   ld1	{ v1.2d }, [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.8b }, [x27], #8
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       ld1	{ v1.8h }, [x27], #16
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.16b }, [x27], #16
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       ld1	{ v1.1d }, [x27], x28
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1	{ v1.2d }, [x27], x28
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [2] Code Region - G03
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      507
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.97
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeER  ..   ld1	{ v1.2s }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eeeeeER ..   ld1	{ v1.4h }, [x27], x28
+# CHECK-NEXT: [0,2]     .D=eeeeeER..   ld1	{ v1.4s }, [x27], x28
+# CHECK-NEXT: [0,3]     .D==eeeeeER.   ld1	{ v1.8b }, [x27], x28
+# CHECK-NEXT: [0,4]     . D==eeeeeER   ld1	{ v1.8h }, [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.2s }, [x27], x28
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       ld1	{ v1.4h }, [x27], x28
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.4s }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       ld1	{ v1.8b }, [x27], x28
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1	{ v1.8h }, [x27], x28
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [3] Code Region - G04
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      507
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.97
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeER  ..   ld1	{ v1.16b }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eeeeeER ..   ld1	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: [0,2]     .D=eeeeeER..   ld1	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: [0,3]     .D==eeeeeER.   ld1	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: [0,4]     . D==eeeeeER   ld1	{ v1.4h, v2.4h }, [x27], #16
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.16b }, [x27], x28
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       ld1	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       ld1	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [4] Code Region - G05
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      507
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.97
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeER  ..   ld1	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: [0,1]     D=eeeeeER ..   ld1	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: [0,2]     .D=eeeeeER..   ld1	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: [0,3]     .D==eeeeeER.   ld1	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: [0,4]     . D==eeeeeER   ld1	{ v1.1d, v2.1d }, [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       ld1	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       ld1	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [5] Code Region - G06
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      507
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.97
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeER  ..   ld1	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eeeeeER ..   ld1	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,2]     .D=eeeeeER..   ld1	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     .D==eeeeeER.   ld1	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,4]     . D==eeeeeER   ld1	{ v1.8b, v2.8b }, [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       ld1	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       ld1	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [6] Code Region - G07
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      608
+# CHECK-NEXT: Total uOps:        1600
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    2.63
+# CHECK-NEXT: IPC:               0.82
+# CHECK-NEXT: Block RThroughput: 5.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeER  .  .   ld1	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eeeeeER .  .   ld1	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,2]     .D=eeeeeeER  .   ld1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
+# CHECK-NEXT: [0,3]     . D=eeeeeeER .   ld1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: [0,4]     .  D==eeeeeeER   ld1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       ld1	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: 4.     1     3.0    1.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT:        1     2.0    0.4    0.0       <total>
+
+# CHECK:      [7] Code Region - G08
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      757
+# CHECK-NEXT: Total uOps:        2000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    2.64
+# CHECK-NEXT: IPC:               0.66
+# CHECK-NEXT: Block RThroughput: 7.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeER .   .   ld1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: [0,1]     .DeeeeeeER.   .   ld1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: [0,2]     . D=eeeeeeER  .   ld1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: [0,3]     .  D=eeeeeeER .   ld1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: [0,4]     .   D==eeeeeeER   ld1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: 2.     1     2.0    1.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: 4.     1     3.0    1.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT:        1     1.8    0.6    0.0       <total>
+
+# CHECK:      [8] Code Region - G09
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      757
+# CHECK-NEXT: Total uOps:        2000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    2.64
+# CHECK-NEXT: IPC:               0.66
+# CHECK-NEXT: Block RThroughput: 7.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeER .   .   ld1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeeeeeeER.   .   ld1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: [0,2]     . D=eeeeeeER  .   ld1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,3]     .  D=eeeeeeER .   ld1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,4]     .   D==eeeeeeER   ld1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: 2.     1     2.0    1.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 4.     1     3.0    1.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT:        1     1.8    0.6    0.0       <total>
+
+# CHECK:      [9] Code Region - G10
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      708
+# CHECK-NEXT: Total uOps:        1800
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    2.54
+# CHECK-NEXT: IPC:               0.71
+# CHECK-NEXT: Block RThroughput: 6.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeER .   .   ld1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeeeeeeER.   .   ld1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: [0,2]     . D=eeeeeeER  .   ld1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,3]     .  D=eeeeeeER .   ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: [0,4]     .   D==eeeeeeER   ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: 2.     1     2.0    1.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: 4.     1     3.0    1.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT:        1     1.8    0.6    0.0       <total>
+
+# CHECK:      [10] Code Region - G11
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      508
+# CHECK-NEXT: Total uOps:        1500
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    2.95
+# CHECK-NEXT: IPC:               0.98
+# CHECK-NEXT: Block RThroughput: 5.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: [0,1]     .DeeeeeeER. .   ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: [0,2]     . DeeeeeeER .   ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: [0,3]     .  DeeeeeeER.   ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: [0,4]     .   DeeeeeeER   ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT:        1     1.0    0.2    0.0       <total>
+
+# CHECK:      [11] Code Region - G12
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      508
+# CHECK-NEXT: Total uOps:        1500
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    2.95
+# CHECK-NEXT: IPC:               0.98
+# CHECK-NEXT: Block RThroughput: 5.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: [0,1]     .DeeeeeeER. .   ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK-NEXT: [0,2]     . DeeeeeeER .   ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeeeeeeER.   ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,4]     .   DeeeeeeER   ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT:        1     1.0    0.2    0.0       <total>
+
+# CHECK:      [12] Code Region - G13
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      1009
+# CHECK-NEXT: Total uOps:        1500
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.49
+# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: Block RThroughput: 4.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     012345678
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeER .    .  .   ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeeeeeeER.    .  .   ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: [0,2]     . DeeeeeeER    .  .   ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeeeeeeER   .  .   ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,4]     .   D=====eeeeeeeER   ld1	{ v1.b }[0], [x27], #1
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: 4.     1     6.0    0.0    0.0       ld1	{ v1.b }[0], [x27], #1
+# CHECK-NEXT:        1     2.0    0.2    0.0       <total>
+
+# CHECK:      [13] Code Region - G14
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      3503
+# CHECK-NEXT: Total uOps:        1500
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    0.43
+# CHECK-NEXT: IPC:               0.14
+# CHECK-NEXT: Block RThroughput: 3.8
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789          01234567
+# CHECK-NEXT: Index     0123456789          0123456789
+
+# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    . .   ld1	{ v1.b }[8], [x27], #1
+# CHECK-NEXT: [0,1]     .D======eeeeeeeER   .    .    .    . .   ld1	{ v1.b }[0], [x27], x28
+# CHECK-NEXT: [0,2]     . D============eeeeeeeER .    .    . .   ld1	{ v1.b }[8], [x27], x28
+# CHECK-NEXT: [0,3]     .  D==================eeeeeeeER    . .   ld1	{ v1.h }[0], [x27], #2
+# CHECK-NEXT: [0,4]     .   D========================eeeeeeeER   ld1	{ v1.h }[4], [x27], #2
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.b }[8], [x27], #1
+# CHECK-NEXT: 1.     1     7.0    0.0    0.0       ld1	{ v1.b }[0], [x27], x28
+# CHECK-NEXT: 2.     1     13.0   0.0    0.0       ld1	{ v1.b }[8], [x27], x28
+# CHECK-NEXT: 3.     1     19.0   0.0    0.0       ld1	{ v1.h }[0], [x27], #2
+# CHECK-NEXT: 4.     1     25.0   0.0    0.0       ld1	{ v1.h }[4], [x27], #2
+# CHECK-NEXT:        1     13.0   0.2    0.0       <total>
+
+# CHECK:      [14] Code Region - G15
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      3503
+# CHECK-NEXT: Total uOps:        1500
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    0.43
+# CHECK-NEXT: IPC:               0.14
+# CHECK-NEXT: Block RThroughput: 3.8
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789          01234567
+# CHECK-NEXT: Index     0123456789          0123456789
+
+# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    . .   ld1	{ v1.h }[0], [x27], x28
+# CHECK-NEXT: [0,1]     .D======eeeeeeeER   .    .    .    . .   ld1	{ v1.h }[4], [x27], x28
+# CHECK-NEXT: [0,2]     . D============eeeeeeeER .    .    . .   ld1	{ v1.s }[0], [x27], #4
+# CHECK-NEXT: [0,3]     .  D==================eeeeeeeER    . .   ld1	{ v1.s }[0], [x27], x28
+# CHECK-NEXT: [0,4]     .   D========================eeeeeeeER   ld1	{ v1.d }[0], [x27], #8
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.h }[0], [x27], x28
+# CHECK-NEXT: 1.     1     7.0    0.0    0.0       ld1	{ v1.h }[4], [x27], x28
+# CHECK-NEXT: 2.     1     13.0   0.0    0.0       ld1	{ v1.s }[0], [x27], #4
+# CHECK-NEXT: 3.     1     19.0   0.0    0.0       ld1	{ v1.s }[0], [x27], x28
+# CHECK-NEXT: 4.     1     25.0   0.0    0.0       ld1	{ v1.d }[0], [x27], #8
+# CHECK-NEXT:        1     13.0   0.2    0.0       <total>
+
+# CHECK:      [15] Code Region - G16
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      1103
+# CHECK-NEXT: Total uOps:        1500
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.36
+# CHECK-NEXT: IPC:               0.45
+# CHECK-NEXT: Block RThroughput: 3.8
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeER.  .   ld1	{ v1.d }[0], [x27], x28
+# CHECK-NEXT: [0,1]     .DeeeeeeeER  .   ld1r	{ v1.1d }, [x27], #8
+# CHECK-NEXT: [0,2]     . DeeeeeeeER .   ld1r	{ v1.2d }, [x27], #8
+# CHECK-NEXT: [0,3]     .  DeeeeeeeER.   ld1r	{ v1.2s }, [x27], #4
+# CHECK-NEXT: [0,4]     .   DeeeeeeeER   ld1r	{ v1.4h }, [x27], #2
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.d }[0], [x27], x28
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ld1r	{ v1.1d }, [x27], #8
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1r	{ v1.2d }, [x27], #8
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       ld1r	{ v1.2s }, [x27], #4
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1r	{ v1.4h }, [x27], #2
+# CHECK-NEXT:        1     1.0    0.2    0.0       <total>
+
+# CHECK:      [16] Code Region - G17
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      509
+# CHECK-NEXT: Total uOps:        1500
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    2.95
+# CHECK-NEXT: IPC:               0.98
+# CHECK-NEXT: Block RThroughput: 3.8
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeER.  .   ld1r	{ v1.4s }, [x27], #4
+# CHECK-NEXT: [0,1]     .DeeeeeeeER  .   ld1r	{ v1.8b }, [x27], #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeER .   ld1r	{ v1.8h }, [x27], #2
+# CHECK-NEXT: [0,3]     .  DeeeeeeeER.   ld1r	{ v1.16b }, [x27], #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeER   ld1r	{ v1.1d }, [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1r	{ v1.4s }, [x27], #4
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ld1r	{ v1.8b }, [x27], #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1r	{ v1.8h }, [x27], #2
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       ld1r	{ v1.16b }, [x27], #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1r	{ v1.1d }, [x27], x28
+# CHECK-NEXT:        1     1.0    0.2    0.0       <total>
+
+# CHECK:      [17] Code Region - G18
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      509
+# CHECK-NEXT: Total uOps:        1500
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    2.95
+# CHECK-NEXT: IPC:               0.98
+# CHECK-NEXT: Block RThroughput: 3.8
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeER.  .   ld1r	{ v1.2d }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeeeeeeeER  .   ld1r	{ v1.2s }, [x27], x28
+# CHECK-NEXT: [0,2]     . DeeeeeeeER .   ld1r	{ v1.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeeeeeeeER.   ld1r	{ v1.4s }, [x27], x28
+# CHECK-NEXT: [0,4]     .   DeeeeeeeER   ld1r	{ v1.8b }, [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1r	{ v1.2d }, [x27], x28
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ld1r	{ v1.2s }, [x27], x28
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1r	{ v1.4h }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       ld1r	{ v1.4s }, [x27], x28
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1r	{ v1.8b }, [x27], x28
+# CHECK-NEXT:        1     1.0    0.2    0.0       <total>
+
+# CHECK:      [18] Code Region - G19
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      509
+# CHECK-NEXT: Total uOps:        1800
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    3.54
+# CHECK-NEXT: IPC:               0.98
+# CHECK-NEXT: Block RThroughput: 4.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeER.  .   ld1r	{ v1.8h }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeeeeeeeER  .   ld1r	{ v1.16b }, [x27], x28
+# CHECK-NEXT: [0,2]     . DeeeeeeeER .   ld2	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: [0,3]     .  DeeeeeeeER.   ld2	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: [0,4]     .   DeeeeeeeER   ld2	{ v1.4h, v2.4h }, [x27], #16
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1r	{ v1.8h }, [x27], x28
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ld1r	{ v1.16b }, [x27], x28
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld2	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       ld2	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld2	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT:        1     1.0    0.2    0.0       <total>
+
+# CHECK:      [19] Code Region - G20
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      509
+# CHECK-NEXT: Total uOps:        2000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    3.93
+# CHECK-NEXT: IPC:               0.98
+# CHECK-NEXT: Block RThroughput: 5.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeER.  .   ld2	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: [0,1]     .DeeeeeeeER  .   ld2	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: [0,2]     . DeeeeeeeER .   ld2	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: [0,3]     .  DeeeeeeeER.   ld2	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: [0,4]     .   DeeeeeeeER   ld2	{ v1.2d, v2.2d }, [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ld2	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld2	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       ld2	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld2	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT:        1     1.0    0.2    0.0       <total>
+
+# CHECK:      [20] Code Region - G21
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      509
+# CHECK-NEXT: Total uOps:        2000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    3.93
+# CHECK-NEXT: IPC:               0.98
+# CHECK-NEXT: Block RThroughput: 5.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeER.  .   ld2	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeeeeeeeER  .   ld2	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,2]     . DeeeeeeeER .   ld2	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeeeeeeeER.   ld2	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,4]     .   DeeeeeeeER   ld2	{ v1.8h, v2.8h }, [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ld2	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld2	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       ld2	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld2	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT:        1     1.0    0.2    0.0       <total>
+
+# CHECK:      [21] Code Region - G22
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      2909
+# CHECK-NEXT: Total uOps:        2000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    0.69
+# CHECK-NEXT: IPC:               0.17
+# CHECK-NEXT: Block RThroughput: 5.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789          01234567
+# CHECK-NEXT: Index     0123456789          0123456789
+
+# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    . .   ld2	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,1]     .D======eeeeeeeER   .    .    .    . .   ld2	{ v1.b, v2.b }[0], [x27], #2
+# CHECK-NEXT: [0,2]     . D============eeeeeeeER .    .    . .   ld2	{ v1.b, v2.b }[8], [x27], #2
+# CHECK-NEXT: [0,3]     .  D==================eeeeeeeER    . .   ld2	{ v1.b, v2.b }[0], [x27], x28
+# CHECK-NEXT: [0,4]     .   D========================eeeeeeeER   ld2	{ v1.b, v2.b }[8], [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: 1.     1     7.0    0.0    0.0       ld2	{ v1.b, v2.b }[0], [x27], #2
+# CHECK-NEXT: 2.     1     13.0   0.0    0.0       ld2	{ v1.b, v2.b }[8], [x27], #2
+# CHECK-NEXT: 3.     1     19.0   0.0    0.0       ld2	{ v1.b, v2.b }[0], [x27], x28
+# CHECK-NEXT: 4.     1     25.0   0.0    0.0       ld2	{ v1.b, v2.b }[8], [x27], x28
+# CHECK-NEXT:        1     13.0   0.2    0.0       <total>
+
+# CHECK:      [22] Code Region - G23
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      3503
+# CHECK-NEXT: Total uOps:        2000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    0.57
+# CHECK-NEXT: IPC:               0.14
+# CHECK-NEXT: Block RThroughput: 5.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789          01234567
+# CHECK-NEXT: Index     0123456789          0123456789
+
+# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    . .   ld2	{ v1.h, v2.h }[0], [x27], #4
+# CHECK-NEXT: [0,1]     .D======eeeeeeeER   .    .    .    . .   ld2	{ v1.h, v2.h }[4], [x27], #4
+# CHECK-NEXT: [0,2]     . D============eeeeeeeER .    .    . .   ld2	{ v1.h, v2.h }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .  D==================eeeeeeeER    . .   ld2	{ v1.h, v2.h }[4], [x27], x28
+# CHECK-NEXT: [0,4]     .   D========================eeeeeeeER   ld2	{ v1.s, v2.s }[0], [x27], #8
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.h, v2.h }[0], [x27], #4
+# CHECK-NEXT: 1.     1     7.0    0.0    0.0       ld2	{ v1.h, v2.h }[4], [x27], #4
+# CHECK-NEXT: 2.     1     13.0   0.0    0.0       ld2	{ v1.h, v2.h }[0], [x27], x28
+# CHECK-NEXT: 3.     1     19.0   0.0    0.0       ld2	{ v1.h, v2.h }[4], [x27], x28
+# CHECK-NEXT: 4.     1     25.0   0.0    0.0       ld2	{ v1.s, v2.s }[0], [x27], #8
+# CHECK-NEXT:        1     13.0   0.2    0.0       <total>
+
+# CHECK:      [23] Code Region - G24
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      2303
+# CHECK-NEXT: Total uOps:        2000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    0.87
+# CHECK-NEXT: IPC:               0.22
+# CHECK-NEXT: Block RThroughput: 5.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          012345
+
+# CHECK:      [0,0]     DeeeeeeeER.    .    .    .   ld2	{ v1.s, v2.s }[0], [x27], x28
+# CHECK-NEXT: [0,1]     .D======eeeeeeeER   .    .   ld2	{ v1.d, v2.d }[0], [x27], #16
+# CHECK-NEXT: [0,2]     . D============eeeeeeeER .   ld2	{ v1.d, v2.d }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .  D============eeeeeeeER.   ld2r	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: [0,4]     .   D============eeeeeeeER   ld2r	{ v1.2d, v2.2d }, [x27], #16
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.s, v2.s }[0], [x27], x28
+# CHECK-NEXT: 1.     1     7.0    0.0    0.0       ld2	{ v1.d, v2.d }[0], [x27], #16
+# CHECK-NEXT: 2.     1     13.0   0.0    0.0       ld2	{ v1.d, v2.d }[0], [x27], x28
+# CHECK-NEXT: 3.     1     13.0   0.0    0.0       ld2r	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld2r	{ v1.2d, v2.2d }, [x27], #16
+# CHECK-NEXT:        1     9.4    0.2    0.0       <total>
+
+# CHECK:      [24] Code Region - G25
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      509
+# CHECK-NEXT: Total uOps:        2000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    3.93
+# CHECK-NEXT: IPC:               0.98
+# CHECK-NEXT: Block RThroughput: 5.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeER.  .   ld2r	{ v1.2s, v2.2s }, [x27], #8
+# CHECK-NEXT: [0,1]     .DeeeeeeeER  .   ld2r	{ v1.4h, v2.4h }, [x27], #4
+# CHECK-NEXT: [0,2]     . DeeeeeeeER .   ld2r	{ v1.4s, v2.4s }, [x27], #8
+# CHECK-NEXT: [0,3]     .  DeeeeeeeER.   ld2r	{ v1.8b, v2.8b }, [x27], #2
+# CHECK-NEXT: [0,4]     .   DeeeeeeeER   ld2r	{ v1.8h, v2.8h }, [x27], #4
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2r	{ v1.2s, v2.2s }, [x27], #8
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ld2r	{ v1.4h, v2.4h }, [x27], #4
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld2r	{ v1.4s, v2.4s }, [x27], #8
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       ld2r	{ v1.8b, v2.8b }, [x27], #2
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld2r	{ v1.8h, v2.8h }, [x27], #4
+# CHECK-NEXT:        1     1.0    0.2    0.0       <total>
+
+# CHECK:      [25] Code Region - G26
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      509
+# CHECK-NEXT: Total uOps:        2000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    3.93
+# CHECK-NEXT: IPC:               0.98
+# CHECK-NEXT: Block RThroughput: 5.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeER.  .   ld2r	{ v1.16b, v2.16b }, [x27], #2
+# CHECK-NEXT: [0,1]     .DeeeeeeeER  .   ld2r	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: [0,2]     . DeeeeeeeER .   ld2r	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeeeeeeeER.   ld2r	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,4]     .   DeeeeeeeER   ld2r	{ v1.4h, v2.4h }, [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2r	{ v1.16b, v2.16b }, [x27], #2
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ld2r	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld2r	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       ld2r	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld2r	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT:        1     1.0    0.2    0.0       <total>
+
+# CHECK:      [26] Code Region - G27
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      609
+# CHECK-NEXT: Total uOps:        2100
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    3.45
+# CHECK-NEXT: IPC:               0.82
+# CHECK-NEXT: Block RThroughput: 5.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeER.   .   ld2r	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeeeeeeeER   .   ld2r	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,2]     . DeeeeeeeER  .   ld2r	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeeeeeeeER .   ld2r	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeER   ld3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2r	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ld2r	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld2r	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       ld2r	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT:        1     1.0    0.2    0.0       <total>
+
+# CHECK:      [27] Code Region - G28
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      1009
+# CHECK-NEXT: Total uOps:        2500
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    2.48
+# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: Block RThroughput: 7.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     012345678
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER    .  .   ld3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: [0,1]     . DeeeeeeeeER  .  .   ld3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: [0,2]     .   DeeeeeeeeER.  .   ld3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: [0,3]     .    .DeeeeeeeeER .   ld3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: [0,4]     .    .  DeeeeeeeeER   ld3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: 1.     1     1.0    1.0    0.0       ld3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: 3.     1     1.0    1.0    0.0       ld3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT:        1     1.0    1.0    0.0       <total>
+
+# CHECK:      [28] Code Region - G29
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      1009
+# CHECK-NEXT: Total uOps:        2500
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    2.48
+# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: Block RThroughput: 7.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     012345678
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER    .  .   ld3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: [0,1]     . DeeeeeeeeER  .  .   ld3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: [0,2]     .   DeeeeeeeeER.  .   ld3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,3]     .    .DeeeeeeeeER .   ld3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,4]     .    .  DeeeeeeeeER   ld3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: 1.     1     1.0    1.0    0.0       ld3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    1.0    0.0       ld3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT:        1     1.0    1.0    0.0       <total>
+
+# CHECK:      [29] Code Region - G30
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      2011
+# CHECK-NEXT: Total uOps:        2500
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.24
+# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: Block RThroughput: 7.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789          0
+# CHECK-NEXT: Index     0123456789          0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .   ld3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,1]     . DeeeeeeeeER  .    .    .    .   ld3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: [0,2]     .   DeeeeeeeeER.    .    .    .   ld3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,3]     .    .D======eeeeeeeeER  .    .   ld3	{ v1.b, v2.b, v3.b }[0], [x27], #3
+# CHECK-NEXT: [0,4]     .    .  D============eeeeeeeeER   ld3	{ v1.b, v2.b, v3.b }[8], [x27], #3
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: 1.     1     1.0    1.0    0.0       ld3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: 3.     1     7.0    0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[0], [x27], #3
+# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[8], [x27], #3
+# CHECK-NEXT:        1     4.6    0.6    0.0       <total>
+
+# CHECK:      [30] Code Region - G31
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      4003
+# CHECK-NEXT: Total uOps:        2500
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    0.62
+# CHECK-NEXT: IPC:               0.12
+# CHECK-NEXT: Block RThroughput: 7.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789          0123456789
+# CHECK-NEXT: Index     0123456789          0123456789          012
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld3	{ v1.b, v2.b, v3.b }[0], [x27], x28
+# CHECK-NEXT: [0,1]     . D======eeeeeeeeER .    .    .    .    . .   ld3	{ v1.b, v2.b, v3.b }[8], [x27], x28
+# CHECK-NEXT: [0,2]     .   D============eeeeeeeeER   .    .    . .   ld3	{ v1.h, v2.h, v3.h }[0], [x27], #6
+# CHECK-NEXT: [0,3]     .    .D==================eeeeeeeeER.    . .   ld3	{ v1.h, v2.h, v3.h }[4], [x27], #6
+# CHECK-NEXT: [0,4]     .    .  D========================eeeeeeeeER   ld3	{ v1.h, v2.h, v3.h }[0], [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.b, v2.b, v3.b }[0], [x27], x28
+# CHECK-NEXT: 1.     1     7.0    0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[8], [x27], x28
+# CHECK-NEXT: 2.     1     13.0   0.0    0.0       ld3	{ v1.h, v2.h, v3.h }[0], [x27], #6
+# CHECK-NEXT: 3.     1     19.0   0.0    0.0       ld3	{ v1.h, v2.h, v3.h }[4], [x27], #6
+# CHECK-NEXT: 4.     1     25.0   0.0    0.0       ld3	{ v1.h, v2.h, v3.h }[0], [x27], x28
+# CHECK-NEXT:        1     13.0   0.2    0.0       <total>
+
+# CHECK:      [31] Code Region - G32
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      4003
+# CHECK-NEXT: Total uOps:        2500
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    0.62
+# CHECK-NEXT: IPC:               0.12
+# CHECK-NEXT: Block RThroughput: 7.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789          0123456789
+# CHECK-NEXT: Index     0123456789          0123456789          012
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld3	{ v1.h, v2.h, v3.h }[4], [x27], x28
+# CHECK-NEXT: [0,1]     . D======eeeeeeeeER .    .    .    .    . .   ld3	{ v1.s, v2.s, v3.s }[0], [x27], #12
+# CHECK-NEXT: [0,2]     .   D============eeeeeeeeER   .    .    . .   ld3	{ v1.s, v2.s, v3.s }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .    .D==================eeeeeeeeER.    . .   ld3	{ v1.d, v2.d, v3.d }[0], [x27], #24
+# CHECK-NEXT: [0,4]     .    .  D========================eeeeeeeeER   ld3	{ v1.d, v2.d, v3.d }[0], [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.h, v2.h, v3.h }[4], [x27], x28
+# CHECK-NEXT: 1.     1     7.0    0.0    0.0       ld3	{ v1.s, v2.s, v3.s }[0], [x27], #12
+# CHECK-NEXT: 2.     1     13.0   0.0    0.0       ld3	{ v1.s, v2.s, v3.s }[0], [x27], x28
+# CHECK-NEXT: 3.     1     19.0   0.0    0.0       ld3	{ v1.d, v2.d, v3.d }[0], [x27], #24
+# CHECK-NEXT: 4.     1     25.0   0.0    0.0       ld3	{ v1.d, v2.d, v3.d }[0], [x27], x28
+# CHECK-NEXT:        1     13.0   0.2    0.0       <total>
+
+# CHECK:      [32] Code Region - G33
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      1009
+# CHECK-NEXT: Total uOps:        2500
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    2.48
+# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: Block RThroughput: 7.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     012345678
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER    .  .   ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], #24
+# CHECK-NEXT: [0,1]     . DeeeeeeeeER  .  .   ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], #24
+# CHECK-NEXT: [0,2]     .   DeeeeeeeeER.  .   ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], #12
+# CHECK-NEXT: [0,3]     .    .DeeeeeeeeER .   ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], #6
+# CHECK-NEXT: [0,4]     .    .  DeeeeeeeeER   ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], #12
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], #24
+# CHECK-NEXT: 1.     1     1.0    1.0    0.0       ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], #24
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], #12
+# CHECK-NEXT: 3.     1     1.0    1.0    0.0       ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], #6
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], #12
+# CHECK-NEXT:        1     1.0    1.0    0.0       <total>
+
+# CHECK:      [33] Code Region - G34
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      1009
+# CHECK-NEXT: Total uOps:        2500
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    2.48
+# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: Block RThroughput: 7.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     012345678
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER    .  .   ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], #3
+# CHECK-NEXT: [0,1]     . DeeeeeeeeER  .  .   ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], #6
+# CHECK-NEXT: [0,2]     .   DeeeeeeeeER.  .   ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], #3
+# CHECK-NEXT: [0,3]     .    .DeeeeeeeeER .   ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: [0,4]     .    .  DeeeeeeeeER   ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], #3
+# CHECK-NEXT: 1.     1     1.0    1.0    0.0       ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], #6
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], #3
+# CHECK-NEXT: 3.     1     1.0    1.0    0.0       ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT:        1     1.0    1.0    0.0       <total>
+
+# CHECK:      [34] Code Region - G35
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      1009
+# CHECK-NEXT: Total uOps:        2500
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    2.48
+# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: Block RThroughput: 7.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     012345678
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER    .  .   ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,1]     . DeeeeeeeeER  .  .   ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,2]     .   DeeeeeeeeER.  .   ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: [0,3]     .    .DeeeeeeeeER .   ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,4]     .    .  DeeeeeeeeER   ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: 1.     1     1.0    1.0    0.0       ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    1.0    0.0       ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT:        1     1.0    1.0    0.0       <total>
+
+# CHECK:      [35] Code Region - G36
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      1410
+# CHECK-NEXT: Total uOps:        4100
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    2.91
+# CHECK-NEXT: IPC:               0.35
+# CHECK-NEXT: Block RThroughput: 10.3
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          0123
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .  .   ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,1]     . DeeeeeeeeeeER.    .  .   ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: [0,2]     .    DeeeeeeeeeeER  .  .   ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: [0,3]     .    .  DeeeeeeeeeeER  .   ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: [0,4]     .    .    .DeeeeeeeeeeER   ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: 1.     1     1.0    1.0    0.0       ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: 3.     1     1.0    1.0    0.0       ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT:        1     1.0    1.0    0.0       <total>
+
+# CHECK:      [36] Code Region - G37
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      1510
+# CHECK-NEXT: Total uOps:        4500
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    2.98
+# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: Block RThroughput: 11.3
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          01234
+
+# CHECK:      [0,0]     DeeeeeeeeeeER  .    .   .   ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: [0,1]     .  DeeeeeeeeeeER    .   .   ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: [0,2]     .    .DeeeeeeeeeeER .   .   ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: [0,3]     .    .   DeeeeeeeeeeER  .   ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,4]     .    .    . DeeeeeeeeeeER   ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: 1.     1     1.0    1.0    0.0       ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: 3.     1     1.0    1.0    0.0       ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT:        1     1.0    1.0    0.0       <total>
+
+# CHECK:      [37] Code Region - G38
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      1510
+# CHECK-NEXT: Total uOps:        4500
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    2.98
+# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: Block RThroughput: 11.3
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          01234
+
+# CHECK:      [0,0]     DeeeeeeeeeeER  .    .   .   ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,1]     .  DeeeeeeeeeeER    .   .   ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,2]     .    .DeeeeeeeeeeER .   .   ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: [0,3]     .    .   DeeeeeeeeeeER  .   ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,4]     .    .    . DeeeeeeeeeeER   ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: 1.     1     1.0    1.0    0.0       ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    1.0    0.0       ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT:        1     1.0    1.0    0.0       <total>
+
+# CHECK:      [38] Code Region - G39
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      4003
+# CHECK-NEXT: Total uOps:        3000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    0.75
+# CHECK-NEXT: IPC:               0.12
+# CHECK-NEXT: Block RThroughput: 7.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789          0123456789
+# CHECK-NEXT: Index     0123456789          0123456789          012
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
+# CHECK-NEXT: [0,1]     . D======eeeeeeeeER .    .    .    .    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
+# CHECK-NEXT: [0,2]     .   D============eeeeeeeeER   .    .    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .    .D==================eeeeeeeeER.    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
+# CHECK-NEXT: [0,4]     .    .  D========================eeeeeeeeER   ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
+# CHECK-NEXT: 1.     1     7.0    0.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
+# CHECK-NEXT: 2.     1     13.0   0.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
+# CHECK-NEXT: 3.     1     19.0   0.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
+# CHECK-NEXT: 4.     1     25.0   0.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
+# CHECK-NEXT:        1     13.0   0.2    0.0       <total>
+
+# CHECK:      [39] Code Region - G40
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      4003
+# CHECK-NEXT: Total uOps:        3000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    0.75
+# CHECK-NEXT: IPC:               0.12
+# CHECK-NEXT: Block RThroughput: 7.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789          0123456789
+# CHECK-NEXT: Index     0123456789          0123456789          012
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
+# CHECK-NEXT: [0,1]     . D======eeeeeeeeER .    .    .    .    . .   ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
+# CHECK-NEXT: [0,2]     .   D============eeeeeeeeER   .    .    . .   ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
+# CHECK-NEXT: [0,3]     .    .D==================eeeeeeeeER.    . .   ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
+# CHECK-NEXT: [0,4]     .    .  D========================eeeeeeeeER   ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
+# CHECK-NEXT: 1.     1     7.0    0.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
+# CHECK-NEXT: 2.     1     13.0   0.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
+# CHECK-NEXT: 3.     1     19.0   0.0    0.0       ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
+# CHECK-NEXT: 4.     1     25.0   0.0    0.0       ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
+# CHECK-NEXT:        1     13.0   0.2    0.0       <total>
+
+# CHECK:      [40] Code Region - G41
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      2003
+# CHECK-NEXT: Total uOps:        3000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.50
+# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: Block RThroughput: 7.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          012
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    . .   ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
+# CHECK-NEXT: [0,1]     . D======eeeeeeeeER . .   ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
+# CHECK-NEXT: [0,2]     .   D=====eeeeeeeeER. .   ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: [0,3]     .    .D=====eeeeeeeeER.   ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #32
+# CHECK-NEXT: [0,4]     .    .  D====eeeeeeeeER   ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #16
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
+# CHECK-NEXT: 1.     1     7.0    0.0    0.0       ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
+# CHECK-NEXT: 2.     1     6.0    0.0    0.0       ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: 3.     1     6.0    1.0    0.0       ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #32
+# CHECK-NEXT: 4.     1     5.0    0.0    0.0       ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #16
+# CHECK-NEXT:        1     5.0    0.4    0.0       <total>
+
+# CHECK:      [41] Code Region - G42
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      1009
+# CHECK-NEXT: Total uOps:        3000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    2.97
+# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: Block RThroughput: 7.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     012345678
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER    .  .   ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #8
+# CHECK-NEXT: [0,1]     . DeeeeeeeeER  .  .   ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #16
+# CHECK-NEXT: [0,2]     .   DeeeeeeeeER.  .   ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #4
+# CHECK-NEXT: [0,3]     .    .DeeeeeeeeER .   ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #8
+# CHECK-NEXT: [0,4]     .    .  DeeeeeeeeER   ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #4
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #8
+# CHECK-NEXT: 1.     1     1.0    1.0    0.0       ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #16
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #4
+# CHECK-NEXT: 3.     1     1.0    1.0    0.0       ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #8
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #4
+# CHECK-NEXT:        1     1.0    1.0    0.0       <total>
+
+# CHECK:      [42] Code Region - G43
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      1009
+# CHECK-NEXT: Total uOps:        3000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    2.97
+# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: Block RThroughput: 7.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     012345678
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER    .  .   ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK-NEXT: [0,1]     . DeeeeeeeeER  .  .   ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,2]     .   DeeeeeeeeER.  .   ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,3]     .    .DeeeeeeeeER .   ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,4]     .    .  DeeeeeeeeER   ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK-NEXT: 1.     1     1.0    1.0    0.0       ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    1.0    0.0       ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT:        1     1.0    1.0    0.0       <total>
+
+# CHECK:      [43] Code Region - G44
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      807
+# CHECK-NEXT: Total uOps:        2400
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    2.97
+# CHECK-NEXT: IPC:               0.62
+# CHECK-NEXT: Block RThroughput: 6.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER   .   ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: [0,1]     . DeeeeeeeeER .   ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,2]     .   DeeeeeeeeER   ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,3]     .    .DeeeeeE-R   ldp	s1, s2, [x27], #248
+# CHECK-NEXT: [0,4]     .    . DeeeeeER   ldp	d1, d2, [x27], #496
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: 1.     1     1.0    1.0    0.0       ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    1.0    1.0       ldp	s1, s2, [x27], #248
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ldp	d1, d2, [x27], #496
+# CHECK-NEXT:        1     1.0    0.8    0.2       <total>
+
+# CHECK:      [44] Code Region - G45
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      506
+# CHECK-NEXT: Total uOps:        1600
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    3.16
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 4.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeER  .   ldp	q1, q2, [x27], #992
+# CHECK-NEXT: [0,1]     .DeeeeeER .   ldp	s1, s2, [x27, #248]!
+# CHECK-NEXT: [0,2]     . DeeeeeER.   ldp	d1, d2, [x27, #496]!
+# CHECK-NEXT: [0,3]     .  DeeeeeER   ldp	q1, q2, [x27, #992]!
+# CHECK-NEXT: [0,4]     .   DeeeeER   ldp	w1, w2, [x27], #248
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldp	q1, q2, [x27], #992
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ldp	s1, s2, [x27, #248]!
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ldp	d1, d2, [x27, #496]!
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       ldp	q1, q2, [x27, #992]!
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ldp	w1, w2, [x27], #248
+# CHECK-NEXT:        1     1.0    0.2    0.0       <total>
+
+# CHECK:      [45] Code Region - G46
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      506
+# CHECK-NEXT: Total uOps:        1800
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    3.56
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 4.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeER   .   ldp	x1, x2, [x27], #496
+# CHECK-NEXT: [0,1]     .DeeeeER  .   ldp	w1, w2, [x27, #248]!
+# CHECK-NEXT: [0,2]     . DeeeeER .   ldp	x1, x2, [x27, #496]!
+# CHECK-NEXT: [0,3]     .  DeeeeER.   ldpsw	x1, x2, [x27], #248
+# CHECK-NEXT: [0,4]     .   DeeeeER   ldpsw	x1, x2, [x27, #248]!
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldp	x1, x2, [x27], #496
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       ldp	w1, w2, [x27, #248]!
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ldp	x1, x2, [x27, #496]!
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       ldpsw	x1, x2, [x27], #248
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ldpsw	x1, x2, [x27, #248]!
+# CHECK-NEXT:        1     1.0    0.2    0.0       <total>
+
+# CHECK:      [46] Code Region - G47
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      507
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.97
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeER  ..   ldr	b1, [x27], #254
+# CHECK-NEXT: [0,1]     D=eeeeeER ..   ldr	h1, [x27], #254
+# CHECK-NEXT: [0,2]     .D=eeeeeER..   ldr	s1, [x27], #254
+# CHECK-NEXT: [0,3]     .D==eeeeeER.   ldr	d1, [x27], #254
+# CHECK-NEXT: [0,4]     . D==eeeeeER   ldr	q1, [x27], #254
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldr	b1, [x27], #254
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       ldr	h1, [x27], #254
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ldr	s1, [x27], #254
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       ldr	d1, [x27], #254
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ldr	q1, [x27], #254
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [47] Code Region - G48
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      507
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.97
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeER  ..   ldr	b1, [x27, #254]!
+# CHECK-NEXT: [0,1]     D=eeeeeER ..   ldr	h1, [x27, #254]!
+# CHECK-NEXT: [0,2]     .D=eeeeeER..   ldr	s1, [x27, #254]!
+# CHECK-NEXT: [0,3]     .D==eeeeeER.   ldr	d1, [x27, #254]!
+# CHECK-NEXT: [0,4]     . D==eeeeeER   ldr	q1, [x27, #254]!
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldr	b1, [x27, #254]!
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       ldr	h1, [x27, #254]!
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ldr	s1, [x27, #254]!
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       ldr	d1, [x27, #254]!
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ldr	q1, [x27, #254]!
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [48] Code Region - G49
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      506
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.98
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeER   .   ldr	w1, [x27], #254
+# CHECK-NEXT: [0,1]     D=eeeeER  .   ldr	x1, [x27], #254
+# CHECK-NEXT: [0,2]     .D=eeeeER .   ldr	w1, [x27, #254]!
+# CHECK-NEXT: [0,3]     .D==eeeeER.   ldr	x1, [x27, #254]!
+# CHECK-NEXT: [0,4]     . D==eeeeER   ldrb	w1, [x27], #254
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldr	w1, [x27], #254
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       ldr	x1, [x27], #254
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ldr	w1, [x27, #254]!
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       ldr	x1, [x27, #254]!
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ldrb	w1, [x27], #254
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [49] Code Region - G50
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      506
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.98
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeER   .   ldrb	w1, [x27, #254]!
+# CHECK-NEXT: [0,1]     D=eeeeER  .   ldrh	w1, [x27], #254
+# CHECK-NEXT: [0,2]     .D=eeeeER .   ldrh	w1, [x27, #254]!
+# CHECK-NEXT: [0,3]     .D==eeeeER.   ldrsb	w1, [x27], #254
+# CHECK-NEXT: [0,4]     . D==eeeeER   ldrsb	x1, [x27], #254
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldrb	w1, [x27, #254]!
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       ldrh	w1, [x27], #254
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ldrh	w1, [x27, #254]!
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       ldrsb	w1, [x27], #254
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ldrsb	x1, [x27], #254
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [50] Code Region - G51
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      506
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.98
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeER   .   ldrsb	w1, [x27, #254]!
+# CHECK-NEXT: [0,1]     D=eeeeER  .   ldrsb	x1, [x27, #254]!
+# CHECK-NEXT: [0,2]     .D=eeeeER .   ldrsh	w1, [x27], #254
+# CHECK-NEXT: [0,3]     .D==eeeeER.   ldrsh	x1, [x27], #254
+# CHECK-NEXT: [0,4]     . D==eeeeER   ldrsh	w1, [x27, #254]!
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldrsb	w1, [x27, #254]!
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       ldrsb	x1, [x27, #254]!
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ldrsh	w1, [x27], #254
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       ldrsh	x1, [x27], #254
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ldrsh	w1, [x27, #254]!
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [51] Code Region - G52
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      505
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.98
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeER  .   ldrsh	x1, [x27, #254]!
+# CHECK-NEXT: [0,1]     D=eeeeER .   ldrsw	x1, [x27], #254
+# CHECK-NEXT: [0,2]     .D=eeeeER.   ldrsw	x1, [x27, #254]!
+# CHECK-NEXT: [0,3]     .D==eeeER.   st1	{ v1.1d }, [x27], #8
+# CHECK-NEXT: [0,4]     . D==eeeER   st1	{ v1.2d }, [x27], #16
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldrsh	x1, [x27, #254]!
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       ldrsw	x1, [x27], #254
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ldrsw	x1, [x27, #254]!
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st1	{ v1.1d }, [x27], #8
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.2d }, [x27], #16
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [52] Code Region - G53
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      505
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.98
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeER   .   st1	{ v1.2s }, [x27], #8
+# CHECK-NEXT: [0,1]     D=eeeER  .   st1	{ v1.4h }, [x27], #8
+# CHECK-NEXT: [0,2]     .D=eeeER .   st1	{ v1.4s }, [x27], #16
+# CHECK-NEXT: [0,3]     .D==eeeER.   st1	{ v1.8b }, [x27], #8
+# CHECK-NEXT: [0,4]     . D==eeeER   st1	{ v1.8h }, [x27], #16
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.2s }, [x27], #8
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st1	{ v1.4h }, [x27], #8
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.4s }, [x27], #16
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st1	{ v1.8b }, [x27], #8
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.8h }, [x27], #16
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [53] Code Region - G54
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      505
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.98
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeER   .   st1	{ v1.16b }, [x27], #16
+# CHECK-NEXT: [0,1]     D=eeeER  .   st1	{ v1.1d }, [x27], x28
+# CHECK-NEXT: [0,2]     .D=eeeER .   st1	{ v1.2d }, [x27], x28
+# CHECK-NEXT: [0,3]     .D==eeeER.   st1	{ v1.2s }, [x27], x28
+# CHECK-NEXT: [0,4]     . D==eeeER   st1	{ v1.4h }, [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.16b }, [x27], #16
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st1	{ v1.1d }, [x27], x28
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.2d }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st1	{ v1.2s }, [x27], x28
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.4h }, [x27], x28
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [54] Code Region - G55
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      506
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.98
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeER    .   st1	{ v1.4s }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eeeER   .   st1	{ v1.8b }, [x27], x28
+# CHECK-NEXT: [0,2]     .D=eeeER  .   st1	{ v1.8h }, [x27], x28
+# CHECK-NEXT: [0,3]     .D==eeeER .   st1	{ v1.16b }, [x27], x28
+# CHECK-NEXT: [0,4]     . D==eeeeER   st1	{ v1.1d, v2.1d }, [x27], #16
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.4s }, [x27], x28
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st1	{ v1.8b }, [x27], x28
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.8h }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st1	{ v1.16b }, [x27], x28
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [55] Code Region - G56
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      506
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.98
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeER   .   st1	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: [0,1]     D=eeeeER  .   st1	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: [0,2]     .D=eeeeER .   st1	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: [0,3]     .D==eeeeER.   st1	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: [0,4]     . D==eeeeER   st1	{ v1.8b, v2.8b }, [x27], #16
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st1	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st1	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [56] Code Region - G57
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      506
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.98
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeER   .   st1	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: [0,1]     D=eeeeER  .   st1	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: [0,2]     .D=eeeeER .   st1	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: [0,3]     .D==eeeeER.   st1	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,4]     . D==eeeeER   st1	{ v1.2s, v2.2s }, [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st1	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st1	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [57] Code Region - G58
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      506
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.98
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeER   .   st1	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eeeeER  .   st1	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,2]     .D=eeeeER .   st1	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,3]     .D==eeeeER.   st1	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,4]     . D==eeeeER   st1	{ v1.16b, v2.16b }, [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st1	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st1	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [58] Code Region - G59
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      507
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.97
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeER  ..   st1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
+# CHECK-NEXT: [0,1]     D=eeeeeER ..   st1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: [0,2]     .D=eeeeeER..   st1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: [0,3]     .D==eeeeeER.   st1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: [0,4]     . D==eeeeeER   st1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [59] Code Region - G60
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      507
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.97
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeER  ..   st1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: [0,1]     D=eeeeeER ..   st1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: [0,2]     .D=eeeeeER..   st1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: [0,3]     .D==eeeeeER.   st1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: [0,4]     . D==eeeeeER   st1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [60] Code Region - G61
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      507
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.97
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeER  ..   st1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eeeeeER ..   st1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,2]     .D=eeeeeER..   st1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: [0,3]     .D==eeeeeER.   st1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,4]     . D==eeeeeER   st1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [61] Code Region - G62
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      508
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.97
+# CHECK-NEXT: IPC:               0.98
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeER  . .   st1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eeeeeeER. .   st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: [0,2]     .D=eeeeeeER .   st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: [0,3]     .D==eeeeeeER.   st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: [0,4]     . D==eeeeeeER   st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [62] Code Region - G63
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      508
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.97
+# CHECK-NEXT: IPC:               0.98
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeER . .   st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: [0,1]     D=eeeeeeER. .   st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: [0,2]     .D=eeeeeeER .   st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: [0,3]     .D==eeeeeeER.   st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: [0,4]     . D==eeeeeeER   st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [63] Code Region - G64
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      508
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.97
+# CHECK-NEXT: IPC:               0.98
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeER . .   st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eeeeeeER. .   st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,2]     .D=eeeeeeER .   st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     .D==eeeeeeER.   st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,4]     . D==eeeeeeER   st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [64] Code Region - G65
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      505
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.98
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeER.   st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eeeeeeER   st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,2]     .D=eeeE--R   st1	{ v1.b }[0], [x27], #1
+# CHECK-NEXT: [0,3]     .D==eeeE-R   st1	{ v1.b }[8], [x27], #1
+# CHECK-NEXT: [0,4]     . D==eeeER   st1	{ v1.b }[0], [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: 2.     1     2.0    0.0    2.0       st1	{ v1.b }[0], [x27], #1
+# CHECK-NEXT: 3.     1     3.0    0.0    1.0       st1	{ v1.b }[8], [x27], #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.b }[0], [x27], x28
+# CHECK-NEXT:        1     2.2    0.2    0.6       <total>
+
+# CHECK:      [65] Code Region - G66
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      505
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.98
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeER   .   st1	{ v1.b }[8], [x27], x28
+# CHECK-NEXT: [0,1]     D=eeeER  .   st1	{ v1.h }[0], [x27], #2
+# CHECK-NEXT: [0,2]     .D=eeeER .   st1	{ v1.h }[4], [x27], #2
+# CHECK-NEXT: [0,3]     .D==eeeER.   st1	{ v1.h }[0], [x27], x28
+# CHECK-NEXT: [0,4]     . D==eeeER   st1	{ v1.h }[4], [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.b }[8], [x27], x28
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st1	{ v1.h }[0], [x27], #2
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.h }[4], [x27], #2
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st1	{ v1.h }[0], [x27], x28
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.h }[4], [x27], x28
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [66] Code Region - G67
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      506
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.98
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeER    .   st1	{ v1.s }[0], [x27], #4
+# CHECK-NEXT: [0,1]     D=eeeER   .   st1	{ v1.s }[0], [x27], x28
+# CHECK-NEXT: [0,2]     .D=eeeER  .   st1	{ v1.d }[0], [x27], #8
+# CHECK-NEXT: [0,3]     .D==eeeER .   st1	{ v1.d }[0], [x27], x28
+# CHECK-NEXT: [0,4]     . D==eeeeER   st2	{ v1.2d, v2.2d }, [x27], #32
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.s }[0], [x27], #4
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st1	{ v1.s }[0], [x27], x28
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.d }[0], [x27], #8
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st1	{ v1.d }[0], [x27], x28
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st2	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [67] Code Region - G68
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      506
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.98
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeER   .   st2	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: [0,1]     D=eeeeER  .   st2	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: [0,2]     .D=eeeeER .   st2	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: [0,3]     .D==eeeeER.   st2	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: [0,4]     . D==eeeeER   st2	{ v1.8h, v2.8h }, [x27], #32
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st2	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st2	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st2	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st2	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [68] Code Region - G69
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      506
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.98
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeER   .   st2	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: [0,1]     D=eeeeER  .   st2	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,2]     .D=eeeeER .   st2	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,3]     .D==eeeeER.   st2	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,4]     . D==eeeeER   st2	{ v1.4s, v2.4s }, [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st2	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st2	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st2	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st2	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [69] Code Region - G70
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      506
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.98
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeER   .   st2	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eeeeER  .   st2	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,2]     .D=eeeeER .   st2	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,3]     .D==eeeeER.   st2	{ v1.b, v2.b }[0], [x27], #2
+# CHECK-NEXT: [0,4]     . D==eeeeER   st2	{ v1.b, v2.b }[8], [x27], #2
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st2	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st2	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st2	{ v1.b, v2.b }[0], [x27], #2
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st2	{ v1.b, v2.b }[8], [x27], #2
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [70] Code Region - G71
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      506
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.98
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeER   .   st2	{ v1.b, v2.b }[0], [x27], x28
+# CHECK-NEXT: [0,1]     D=eeeeER  .   st2	{ v1.b, v2.b }[8], [x27], x28
+# CHECK-NEXT: [0,2]     .D=eeeeER .   st2	{ v1.h, v2.h }[0], [x27], #4
+# CHECK-NEXT: [0,3]     .D==eeeeER.   st2	{ v1.h, v2.h }[4], [x27], #4
+# CHECK-NEXT: [0,4]     . D==eeeeER   st2	{ v1.h, v2.h }[0], [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.b, v2.b }[0], [x27], x28
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st2	{ v1.b, v2.b }[8], [x27], x28
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st2	{ v1.h, v2.h }[0], [x27], #4
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st2	{ v1.h, v2.h }[4], [x27], #4
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st2	{ v1.h, v2.h }[0], [x27], x28
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [71] Code Region - G72
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      506
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.98
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeER   .   st2	{ v1.h, v2.h }[4], [x27], x28
+# CHECK-NEXT: [0,1]     D=eeeeER  .   st2	{ v1.s, v2.s }[0], [x27], #8
+# CHECK-NEXT: [0,2]     .D=eeeeER .   st2	{ v1.s, v2.s }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .D==eeeeER.   st2	{ v1.d, v2.d }[0], [x27], #16
+# CHECK-NEXT: [0,4]     . D==eeeeER   st2	{ v1.d, v2.d }[0], [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.h, v2.h }[4], [x27], x28
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st2	{ v1.s, v2.s }[0], [x27], #8
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st2	{ v1.s, v2.s }[0], [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st2	{ v1.d, v2.d }[0], [x27], #16
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st2	{ v1.d, v2.d }[0], [x27], x28
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [72] Code Region - G73
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      300
+# CHECK-NEXT: Total Cycles:      307
+# CHECK-NEXT: Total uOps:        600
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.95
+# CHECK-NEXT: IPC:               0.98
+# CHECK-NEXT: Block RThroughput: 1.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeER .   st3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: [0,1]     D=eeeeeER.   st3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: [0,2]     .D=eeeeeER   st3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT:        1     1.7    0.3    0.0       <total>
+
+# CHECK:      [73] Code Region - G74
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      507
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.97
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeER  ..   st3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: [0,1]     D=eeeeeER ..   st3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: [0,2]     .D=eeeeeER..   st3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: [0,3]     .D==eeeeeER.   st3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: [0,4]     . D==eeeeeER   st3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [74] Code Region - G75
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      507
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.97
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeER  ..   st3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eeeeeER ..   st3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,2]     .D=eeeeeER..   st3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: [0,3]     .D==eeeeeER.   st3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,4]     . D==eeeeeER   st3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [75] Code Region - G76
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      507
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.97
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeER  ..   st3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eeeeeER ..   st3	{ v1.b, v2.b, v3.b }[0], [x27], #3
+# CHECK-NEXT: [0,2]     .D=eeeeeER..   st3	{ v1.b, v2.b, v3.b }[8], [x27], #3
+# CHECK-NEXT: [0,3]     .D==eeeeeER.   st3	{ v1.b, v2.b, v3.b }[0], [x27], x28
+# CHECK-NEXT: [0,4]     . D==eeeeeER   st3	{ v1.b, v2.b, v3.b }[8], [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st3	{ v1.b, v2.b, v3.b }[0], [x27], #3
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st3	{ v1.b, v2.b, v3.b }[8], [x27], #3
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st3	{ v1.b, v2.b, v3.b }[0], [x27], x28
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st3	{ v1.b, v2.b, v3.b }[8], [x27], x28
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [76] Code Region - G77
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      507
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.97
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeER  ..   st3	{ v1.h, v2.h, v3.h }[0], [x27], #6
+# CHECK-NEXT: [0,1]     D=eeeeeER ..   st3	{ v1.h, v2.h, v3.h }[4], [x27], #6
+# CHECK-NEXT: [0,2]     .D=eeeeeER..   st3	{ v1.h, v2.h, v3.h }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .D==eeeeeER.   st3	{ v1.h, v2.h, v3.h }[4], [x27], x28
+# CHECK-NEXT: [0,4]     . D==eeeeeER   st3	{ v1.s, v2.s, v3.s }[0], [x27], #12
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.h, v2.h, v3.h }[0], [x27], #6
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st3	{ v1.h, v2.h, v3.h }[4], [x27], #6
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st3	{ v1.h, v2.h, v3.h }[0], [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st3	{ v1.h, v2.h, v3.h }[4], [x27], x28
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st3	{ v1.s, v2.s, v3.s }[0], [x27], #12
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [77] Code Region - G78
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      510
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.96
+# CHECK-NEXT: IPC:               0.98
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeER  .   .   st3	{ v1.s, v2.s, v3.s }[0], [x27], x28
+# CHECK-NEXT: [0,1]     D=eeeeeER .   .   st3	{ v1.d, v2.d, v3.d }[0], [x27], #24
+# CHECK-NEXT: [0,2]     .D=eeeeeER.   .   st3	{ v1.d, v2.d, v3.d }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .D==eeeeeeeeER.   st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: [0,4]     . D==eeeeeeeeER   st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.s, v2.s, v3.s }[0], [x27], x28
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st3	{ v1.d, v2.d, v3.d }[0], [x27], #24
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st3	{ v1.d, v2.d, v3.d }[0], [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [78] Code Region - G79
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      510
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.96
+# CHECK-NEXT: IPC:               0.98
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER   .   st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: [0,1]     D=eeeeeeeeER  .   st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: [0,2]     .D=eeeeeeeeER .   st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: [0,3]     .D==eeeeeeeeER.   st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: [0,4]     . D==eeeeeeeeER   st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [79] Code Region - G80
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      510
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.96
+# CHECK-NEXT: IPC:               0.98
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER   .   st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eeeeeeeeER  .   st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,2]     .D=eeeeeeeeER .   st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     .D==eeeeeeeeER.   st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,4]     . D==eeeeeeeeER   st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [80] Code Region - G81
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      508
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.97
+# CHECK-NEXT: IPC:               0.98
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER .   st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eeeeeeeeER.   st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,2]     .D=eeeeeeE-R.   st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
+# CHECK-NEXT: [0,3]     .D==eeeeeeER.   st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
+# CHECK-NEXT: [0,4]     . D==eeeeeeER   st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: 2.     1     2.0    0.0    1.0       st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
+# CHECK-NEXT:        1     2.2    0.2    0.2       <total>
+
+# CHECK:      [81] Code Region - G82
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      508
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.97
+# CHECK-NEXT: IPC:               0.98
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeER . .   st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
+# CHECK-NEXT: [0,1]     D=eeeeeeER. .   st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
+# CHECK-NEXT: [0,2]     .D=eeeeeeER .   st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
+# CHECK-NEXT: [0,3]     .D==eeeeeeER.   st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
+# CHECK-NEXT: [0,4]     . D==eeeeeeER   st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [82] Code Region - G83
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      400
+# CHECK-NEXT: Total Cycles:      408
+# CHECK-NEXT: Total uOps:        800
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.96
+# CHECK-NEXT: IPC:               0.98
+# CHECK-NEXT: Block RThroughput: 2.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeER ..   st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
+# CHECK-NEXT: [0,1]     D=eeeeeeER..   st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
+# CHECK-NEXT: [0,2]     .D=eeeeeeER.   st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
+# CHECK-NEXT: [0,3]     .D==eeeeeeER   st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
+# CHECK-NEXT:        1     2.0    0.3    0.0       <total>
+
+# CHECK:      [83] Code Region - G84
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      200
+# CHECK-NEXT: Total Cycles:      403
+# CHECK-NEXT: Total uOps:        600
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.49
+# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: Block RThroughput: 4.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT: Index     0123456
+
+# CHECK:      [0,0]     DeeER..   stp	s1, s2, [x27], #248
+# CHECK-NEXT: [0,1]     .D=eeER   stp	d1, d2, [x27], #496
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       stp	s1, s2, [x27], #248
+# CHECK-NEXT: 1.     1     2.0    1.0    0.0       stp	d1, d2, [x27], #496
+# CHECK-NEXT:        1     1.5    1.0    0.0       <total>
+
+# CHECK:      [84] Code Region - G85
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      903
+# CHECK-NEXT: Total uOps:        1400
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.55
+# CHECK-NEXT: IPC:               0.55
+# CHECK-NEXT: Block RThroughput: 9.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeER.    ..   stp	q1, q2, [x27], #992
+# CHECK-NEXT: [0,1]     .D=eeER   ..   stp	s1, s2, [x27, #248]!
+# CHECK-NEXT: [0,2]     . D==eeER ..   stp	d1, d2, [x27, #496]!
+# CHECK-NEXT: [0,3]     .  D===eeER.   stp	q1, q2, [x27, #992]!
+# CHECK-NEXT: [0,4]     .   D====eER   stp	w1, w2, [x27], #248
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       stp	q1, q2, [x27], #992
+# CHECK-NEXT: 1.     1     2.0    1.0    0.0       stp	s1, s2, [x27, #248]!
+# CHECK-NEXT: 2.     1     3.0    1.0    0.0       stp	d1, d2, [x27, #496]!
+# CHECK-NEXT: 3.     1     4.0    1.0    0.0       stp	q1, q2, [x27, #992]!
+# CHECK-NEXT: 4.     1     5.0    1.0    0.0       stp	w1, w2, [x27], #248
+# CHECK-NEXT:        1     3.0    1.0    0.0       <total>
+
+# CHECK:      [85] Code Region - G86
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      503
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.99
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 5.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT: Index     01234567
+
+# CHECK:      [0,0]     DeER . .   stp	x1, x2, [x27], #496
+# CHECK-NEXT: [0,1]     D=eER. .   stp	w1, w2, [x27, #248]!
+# CHECK-NEXT: [0,2]     .D=eER .   stp	x1, x2, [x27, #496]!
+# CHECK-NEXT: [0,3]     .D==eER.   str	b1, [x27], #254
+# CHECK-NEXT: [0,4]     . D==eER   str	h1, [x27], #254
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       stp	x1, x2, [x27], #496
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       stp	w1, w2, [x27, #248]!
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       stp	x1, x2, [x27, #496]!
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       str	b1, [x27], #254
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       str	h1, [x27], #254
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [86] Code Region - G87
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      503
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.99
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 5.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT: Index     01234567
+
+# CHECK:      [0,0]     DeER . .   str	s1, [x27], #254
+# CHECK-NEXT: [0,1]     D=eER. .   str	d1, [x27], #254
+# CHECK-NEXT: [0,2]     .D=eER .   str	q1, [x27], #254
+# CHECK-NEXT: [0,3]     .D==eER.   str	b1, [x27, #254]!
+# CHECK-NEXT: [0,4]     . D==eER   str	h1, [x27, #254]!
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       str	s1, [x27], #254
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       str	d1, [x27], #254
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       str	q1, [x27], #254
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       str	b1, [x27, #254]!
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       str	h1, [x27, #254]!
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [87] Code Region - G88
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      503
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.99
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 5.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT: Index     01234567
+
+# CHECK:      [0,0]     DeER . .   str	s1, [x27, #254]!
+# CHECK-NEXT: [0,1]     D=eER. .   str	d1, [x27, #254]!
+# CHECK-NEXT: [0,2]     .D=eER .   str	q1, [x27, #254]!
+# CHECK-NEXT: [0,3]     .D==eER.   str	w1, [x27], #254
+# CHECK-NEXT: [0,4]     . D==eER   str	x1, [x27], #254
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       str	s1, [x27, #254]!
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       str	d1, [x27, #254]!
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       str	q1, [x27, #254]!
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       str	w1, [x27], #254
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       str	x1, [x27], #254
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [88] Code Region - G89
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      503
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.99
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 5.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT: Index     01234567
+
+# CHECK:      [0,0]     DeER . .   str	w1, [x27, #254]!
+# CHECK-NEXT: [0,1]     D=eER. .   str	x1, [x27, #254]!
+# CHECK-NEXT: [0,2]     .D=eER .   strb	w1, [x27], #254
+# CHECK-NEXT: [0,3]     .D==eER.   strb	w1, [x27, #254]!
+# CHECK-NEXT: [0,4]     . D==eER   strh	w1, [x27], #254
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       str	w1, [x27, #254]!
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       str	x1, [x27, #254]!
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       strb	w1, [x27], #254
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       strb	w1, [x27, #254]!
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       strh	w1, [x27], #254
+# CHECK-NEXT:        1     2.2    0.2    0.0       <total>
+
+# CHECK:      [89] Code Region - G90
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      100
+# CHECK-NEXT: Total Cycles:      103
+# CHECK-NEXT: Total uOps:        200
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.94
+# CHECK-NEXT: IPC:               0.97
+# CHECK-NEXT: Block RThroughput: 1.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT: Index     0123
+
+# CHECK:      [0,0]     DeER   strh	w1, [x27, #254]!
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       strh	w1, [x27, #254]!
+
+# CHECK:      [90] Code Region - G91
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      200
+# CHECK-NEXT: Total Cycles:      110
+# CHECK-NEXT: Total uOps:        400
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    3.64
+# CHECK-NEXT: IPC:               1.82
+# CHECK-NEXT: Block RThroughput: 1.0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeER   .   ldr	x1, [x27], #254
+# CHECK-NEXT: [0,1]     D====eeeeER   ldr	x2, [x1], #254
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldr	x1, [x27], #254
+# CHECK-NEXT: 1.     1     5.0    0.0    0.0       ldr	x2, [x1], #254
+# CHECK-NEXT:        1     3.0    0.5    0.0       <total>
diff --git a/llvm/test/tools/llvm-mca/AArch64/Neoverse/N1-writeback.s b/llvm/test/tools/llvm-mca/AArch64/Neoverse/N1-writeback.s
index b4e7a4e40b..8fe21167a5 100644
--- a/llvm/test/tools/llvm-mca/AArch64/Neoverse/N1-writeback.s
+++ b/llvm/test/tools/llvm-mca/AArch64/Neoverse/N1-writeback.s
@@ -1162,28 +1162,28 @@ add x0, x27, 1
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2504
+# CHECK-NEXT: Total Cycles:      507
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    0.60
-# CHECK-NEXT: IPC:               0.40
+# CHECK-NEXT: uOps Per Cycle:    2.96
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 3.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          012345678
-
-# CHECK:      [0,0]     DeeeeeER  .    .    .    .  .   ld1	{ v1.1d }, [x27], #8
-# CHECK-NEXT: [0,1]     D=====eER .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=====eeeeeER  .    .    .  .   ld1	{ v1.2d }, [x27], #16
-# CHECK-NEXT: [0,3]     D==========eER .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D==========eeeeeER  .    .  .   ld1	{ v1.2s }, [x27], #8
-# CHECK-NEXT: [0,5]     .D==============eER .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==============eeeeeER  .  .   ld1	{ v1.4h }, [x27], #8
-# CHECK-NEXT: [0,7]     .D===================eER .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===================eeeeeER.   ld1	{ v1.4s }, [x27], #16
-# CHECK-NEXT: [0,9]     .D========================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeER  ..   ld1	{ v1.1d }, [x27], #8
+# CHECK-NEXT: [0,1]     D=eE---R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeER ..   ld1	{ v1.2d }, [x27], #16
+# CHECK-NEXT: [0,3]     D==eE---R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeeER..   ld1	{ v1.2s }, [x27], #8
+# CHECK-NEXT: [0,5]     .D==eE---R..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeER.   ld1	{ v1.4h }, [x27], #8
+# CHECK-NEXT: [0,7]     .D===eE---R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeeeeER   ld1	{ v1.4s }, [x27], #16
+# CHECK-NEXT: [0,9]     .D====eE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1193,43 +1193,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.1d }, [x27], #8
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     6.0    0.0    0.0       ld1	{ v1.2d }, [x27], #16
-# CHECK-NEXT: 3.     1     11.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     11.0   0.0    0.0       ld1	{ v1.2s }, [x27], #8
-# CHECK-NEXT: 5.     1     15.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     15.0   0.0    0.0       ld1	{ v1.4h }, [x27], #8
-# CHECK-NEXT: 7.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     20.0   0.0    0.0       ld1	{ v1.4s }, [x27], #16
-# CHECK-NEXT: 9.     1     25.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     13.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.2d }, [x27], #16
+# CHECK-NEXT: 3.     1     3.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1	{ v1.2s }, [x27], #8
+# CHECK-NEXT: 5.     1     3.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.4h }, [x27], #8
+# CHECK-NEXT: 7.     1     4.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ld1	{ v1.4s }, [x27], #16
+# CHECK-NEXT: 9.     1     5.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.0    0.1    1.5       <total>
 
 # CHECK:      [1] Code Region - G02
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2504
+# CHECK-NEXT: Total Cycles:      507
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    0.60
-# CHECK-NEXT: IPC:               0.40
+# CHECK-NEXT: uOps Per Cycle:    2.96
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 3.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          012345678
-
-# CHECK:      [0,0]     DeeeeeER  .    .    .    .  .   ld1	{ v1.8b }, [x27], #8
-# CHECK-NEXT: [0,1]     D=====eER .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=====eeeeeER  .    .    .  .   ld1	{ v1.8h }, [x27], #16
-# CHECK-NEXT: [0,3]     D==========eER .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D==========eeeeeER  .    .  .   ld1	{ v1.16b }, [x27], #16
-# CHECK-NEXT: [0,5]     .D==============eER .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==============eeeeeER  .  .   ld1	{ v1.1d }, [x27], x28
-# CHECK-NEXT: [0,7]     .D===================eER .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===================eeeeeER.   ld1	{ v1.2d }, [x27], x28
-# CHECK-NEXT: [0,9]     .D========================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeER  ..   ld1	{ v1.8b }, [x27], #8
+# CHECK-NEXT: [0,1]     D=eE---R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeER ..   ld1	{ v1.8h }, [x27], #16
+# CHECK-NEXT: [0,3]     D==eE---R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeeER..   ld1	{ v1.16b }, [x27], #16
+# CHECK-NEXT: [0,5]     .D==eE---R..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeER.   ld1	{ v1.1d }, [x27], x28
+# CHECK-NEXT: [0,7]     .D===eE---R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeeeeER   ld1	{ v1.2d }, [x27], x28
+# CHECK-NEXT: [0,9]     .D====eE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1239,43 +1239,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.8b }, [x27], #8
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     6.0    0.0    0.0       ld1	{ v1.8h }, [x27], #16
-# CHECK-NEXT: 3.     1     11.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     11.0   0.0    0.0       ld1	{ v1.16b }, [x27], #16
-# CHECK-NEXT: 5.     1     15.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     15.0   0.0    0.0       ld1	{ v1.1d }, [x27], x28
-# CHECK-NEXT: 7.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     20.0   0.0    0.0       ld1	{ v1.2d }, [x27], x28
-# CHECK-NEXT: 9.     1     25.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     13.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.8h }, [x27], #16
+# CHECK-NEXT: 3.     1     3.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1	{ v1.16b }, [x27], #16
+# CHECK-NEXT: 5.     1     3.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.1d }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ld1	{ v1.2d }, [x27], x28
+# CHECK-NEXT: 9.     1     5.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.0    0.1    1.5       <total>
 
 # CHECK:      [2] Code Region - G03
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2504
+# CHECK-NEXT: Total Cycles:      507
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    0.60
-# CHECK-NEXT: IPC:               0.40
+# CHECK-NEXT: uOps Per Cycle:    2.96
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 3.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          012345678
-
-# CHECK:      [0,0]     DeeeeeER  .    .    .    .  .   ld1	{ v1.2s }, [x27], x28
-# CHECK-NEXT: [0,1]     D=====eER .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=====eeeeeER  .    .    .  .   ld1	{ v1.4h }, [x27], x28
-# CHECK-NEXT: [0,3]     D==========eER .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D==========eeeeeER  .    .  .   ld1	{ v1.4s }, [x27], x28
-# CHECK-NEXT: [0,5]     .D==============eER .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==============eeeeeER  .  .   ld1	{ v1.8b }, [x27], x28
-# CHECK-NEXT: [0,7]     .D===================eER .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===================eeeeeER.   ld1	{ v1.8h }, [x27], x28
-# CHECK-NEXT: [0,9]     .D========================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeER  ..   ld1	{ v1.2s }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE---R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeER ..   ld1	{ v1.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eE---R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeeER..   ld1	{ v1.4s }, [x27], x28
+# CHECK-NEXT: [0,5]     .D==eE---R..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeER.   ld1	{ v1.8b }, [x27], x28
+# CHECK-NEXT: [0,7]     .D===eE---R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeeeeER   ld1	{ v1.8h }, [x27], x28
+# CHECK-NEXT: [0,9]     .D====eE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1285,43 +1285,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.2s }, [x27], x28
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     6.0    0.0    0.0       ld1	{ v1.4h }, [x27], x28
-# CHECK-NEXT: 3.     1     11.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     11.0   0.0    0.0       ld1	{ v1.4s }, [x27], x28
-# CHECK-NEXT: 5.     1     15.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     15.0   0.0    0.0       ld1	{ v1.8b }, [x27], x28
-# CHECK-NEXT: 7.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     20.0   0.0    0.0       ld1	{ v1.8h }, [x27], x28
-# CHECK-NEXT: 9.     1     25.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     13.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.4h }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1	{ v1.4s }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.8b }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ld1	{ v1.8h }, [x27], x28
+# CHECK-NEXT: 9.     1     5.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.0    0.1    1.5       <total>
 
 # CHECK:      [3] Code Region - G04
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2504
+# CHECK-NEXT: Total Cycles:      507
 # CHECK-NEXT: Total uOps:        1900
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    0.76
-# CHECK-NEXT: IPC:               0.40
+# CHECK-NEXT: uOps Per Cycle:    3.75
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 4.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          012345678
-
-# CHECK:      [0,0]     DeeeeeER  .    .    .    .  .   ld1	{ v1.16b }, [x27], x28
-# CHECK-NEXT: [0,1]     D=====eER .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=====eeeeeER  .    .    .  .   ld1	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: [0,3]     D==========eER .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=========eeeeeER  .    .  .   ld1	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: [0,5]     .D==============eER .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==============eeeeeER  .  .   ld1	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: [0,7]     .D===================eER .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==================eeeeeER.   ld1	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: [0,9]     . D=======================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeER  ..   ld1	{ v1.16b }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE---R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeER ..   ld1	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: [0,3]     D==eE---R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeER..   ld1	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: [0,5]     .D==eE---R..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeER.   ld1	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: [0,7]     .D===eE---R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeER   ld1	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: [0,9]     . D===eE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1331,43 +1331,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.16b }, [x27], x28
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     6.0    0.0    0.0       ld1	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: 3.     1     11.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     10.0   0.0    0.0       ld1	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: 5.     1     15.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     15.0   0.0    0.0       ld1	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: 7.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     19.0   0.0    0.0       ld1	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: 9.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     12.7   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: 3.     1     3.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: 5.     1     3.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: 7.     1     4.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: 9.     1     4.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    1.5       <total>
 
 # CHECK:      [4] Code Region - G05
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2504
+# CHECK-NEXT: Total Cycles:      507
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    0.80
-# CHECK-NEXT: IPC:               0.40
+# CHECK-NEXT: uOps Per Cycle:    3.94
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          012345678
-
-# CHECK:      [0,0]     DeeeeeER  .    .    .    .  .   ld1	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: [0,1]     D=====eER .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=====eeeeeER  .    .    .  .   ld1	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: [0,3]     D==========eER .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=========eeeeeER  .    .  .   ld1	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: [0,5]     .D==============eER .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==============eeeeeER  .  .   ld1	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: [0,7]     .D===================eER .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==================eeeeeER.   ld1	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: [0,9]     . D=======================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeER  ..   ld1	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: [0,1]     D=eE---R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeER ..   ld1	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: [0,3]     D==eE---R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeER..   ld1	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: [0,5]     .D==eE---R..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeER.   ld1	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: [0,7]     .D===eE---R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeER   ld1	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: [0,9]     . D===eE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1377,43 +1377,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     6.0    0.0    0.0       ld1	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: 3.     1     11.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     10.0   0.0    0.0       ld1	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: 5.     1     15.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     15.0   0.0    0.0       ld1	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: 7.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     19.0   0.0    0.0       ld1	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: 9.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     12.7   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: 3.     1     3.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: 5.     1     3.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: 7.     1     4.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    1.5       <total>
 
 # CHECK:      [5] Code Region - G06
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2504
+# CHECK-NEXT: Total Cycles:      507
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    0.80
-# CHECK-NEXT: IPC:               0.40
+# CHECK-NEXT: uOps Per Cycle:    3.94
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          012345678
-
-# CHECK:      [0,0]     DeeeeeER  .    .    .    .  .   ld1	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: [0,1]     D=====eER .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=====eeeeeER  .    .    .  .   ld1	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: [0,3]     D==========eER .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=========eeeeeER  .    .  .   ld1	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: [0,5]     .D==============eER .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==============eeeeeER  .  .   ld1	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: [0,7]     .D===================eER .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==================eeeeeER.   ld1	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: [0,9]     . D=======================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeER  ..   ld1	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE---R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeER ..   ld1	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eE---R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeER..   ld1	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,5]     .D==eE---R..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeER.   ld1	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,7]     .D===eE---R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeER   ld1	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,9]     . D===eE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1423,43 +1423,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     6.0    0.0    0.0       ld1	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: 3.     1     11.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     10.0   0.0    0.0       ld1	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: 5.     1     15.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     15.0   0.0    0.0       ld1	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: 7.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     19.0   0.0    0.0       ld1	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: 9.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     12.7   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    1.5       <total>
 
 # CHECK:      [6] Code Region - G07
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2804
+# CHECK-NEXT: Total Cycles:      707
 # CHECK-NEXT: Total uOps:        2300
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    0.82
-# CHECK-NEXT: IPC:               0.36
+# CHECK-NEXT: uOps Per Cycle:    3.25
+# CHECK-NEXT: IPC:               1.41
 # CHECK-NEXT: Block RThroughput: 6.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          01
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     0123
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeER  .    .    .    .    ..   ld1	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: [0,1]     D=====eER .    .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=====eeeeeER  .    .    .    ..   ld1	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: [0,3]     D==========eER .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=========eeeeeeER .    .    ..   ld1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: [0,5]     .D===============eER.    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     . D==============eeeeeeER.    ..   ld1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: [0,7]     . D====================eER    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .  D===================eeeeeeER.   ld1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: [0,9]     .  D=========================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeER  .  .   ld1	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE---R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeER .  .   ld1	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eE---R .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeeER  .   ld1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
+# CHECK-NEXT: [0,5]     .D==eE----R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     . D=eeeeeeER .   ld1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: [0,7]     . D==eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .  D==eeeeeeER   ld1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: [0,9]     .  D===eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1469,43 +1469,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     6.0    0.0    0.0       ld1	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: 3.     1     11.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     10.0   0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: 5.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     15.0   0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: 7.     1     21.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     20.0   0.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: 9.     1     26.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     13.2   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
+# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: 7.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    1.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.5    0.2    1.8       <total>
 
 # CHECK:      [7] Code Region - G08
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3004
+# CHECK-NEXT: Total Cycles:      757
 # CHECK-NEXT: Total uOps:        2500
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    0.83
-# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: uOps Per Cycle:    3.30
+# CHECK-NEXT: IPC:               1.32
 # CHECK-NEXT: Block RThroughput: 7.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .  .   ld1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=====eeeeeeER.    .    .    .  .   ld1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: [0,3]     .D===========eER    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==========eeeeeeER    .    .  .   ld1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: [0,5]     . D================eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D===============eeeeeeER   .  .   ld1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: [0,7]     .  D=====================eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D====================eeeeeeER.   ld1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: [0,9]     .   D==========================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER .   .   ld1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: [0,1]     D=eE----R .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeER.   .   ld1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: [0,3]     .D=eE----R.   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeeeeeER  .   ld1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: [0,5]     . D==eE----R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeeeeeER .   ld1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: [0,7]     .  D==eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D==eeeeeeER   ld1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: [0,9]     .   D===eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1515,43 +1515,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     6.0    0.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: 3.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     11.0   0.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: 5.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     16.0   0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: 7.     1     22.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     21.0   0.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: 9.     1     27.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     14.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: 7.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    1.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.3    2.0       <total>
 
 # CHECK:      [8] Code Region - G09
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3004
+# CHECK-NEXT: Total Cycles:      757
 # CHECK-NEXT: Total uOps:        2500
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    0.83
-# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: uOps Per Cycle:    3.30
+# CHECK-NEXT: IPC:               1.32
 # CHECK-NEXT: Block RThroughput: 7.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .  .   ld1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=====eeeeeeER.    .    .    .  .   ld1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: [0,3]     .D===========eER    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==========eeeeeeER    .    .  .   ld1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: [0,5]     . D================eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D===============eeeeeeER   .  .   ld1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D=====================eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D====================eeeeeeER.   ld1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D==========================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER .   .   ld1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE----R .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeER.   .   ld1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE----R.   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeeeeeER  .   ld1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,5]     . D==eE----R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeeeeeER .   ld1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D==eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D==eeeeeeER   ld1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D===eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1561,43 +1561,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     6.0    0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: 3.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     11.0   0.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: 5.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     16.0   0.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: 7.     1     22.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     21.0   0.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: 9.     1     27.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     14.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 7.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    1.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.3    2.0       <total>
 
 # CHECK:      [9] Code Region - G10
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3004
+# CHECK-NEXT: Total Cycles:      856
 # CHECK-NEXT: Total uOps:        2700
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    0.90
-# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: uOps Per Cycle:    3.15
+# CHECK-NEXT: IPC:               1.17
 # CHECK-NEXT: Block RThroughput: 8.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .  .   ld1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=====eeeeeeER.    .    .    .  .   ld1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: [0,3]     .D===========eER    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==========eeeeeeER    .    .  .   ld1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: [0,5]     . D================eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D===============eeeeeeER   .  .   ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: [0,7]     .  D=====================eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D====================eeeeeeER.   ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: [0,9]     .   D==========================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER .   .   ld1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE----R .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeER.   .   ld1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE----R.   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeeeeeER  .   ld1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,5]     . D==eE----R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeeeeeER .   ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: [0,7]     .  D==eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D==eeeeeeER   ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: [0,9]     .   D===eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1607,43 +1607,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     6.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: 3.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     11.0   0.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: 5.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     16.0   0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: 7.     1     22.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     21.0   0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: 9.     1     27.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     14.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: 7.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    1.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.3    2.0       <total>
 
 # CHECK:      [10] Code Region - G11
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3004
+# CHECK-NEXT: Total Cycles:      1006
 # CHECK-NEXT: Total uOps:        3000
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    1.00
-# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: uOps Per Cycle:    2.98
+# CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 10.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     0123456
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .  .   ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=====eeeeeeER.    .    .    .  .   ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: [0,3]     .D===========eER    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==========eeeeeeER    .    .  .   ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: [0,5]     . D================eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D===============eeeeeeER   .  .   ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: [0,7]     .  D=====================eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D====================eeeeeeER.   ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: [0,9]     .   D==========================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER .    ..   ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: [0,1]     D=eE----R .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeER.    ..   ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: [0,3]     .D=eE----R.    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D==eeeeeeER  ..   ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: [0,5]     . D===eE----R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D==eeeeeeER ..   ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: [0,7]     .  D===eE----R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D====eeeeeeER   ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: [0,9]     .   D=====eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1653,43 +1653,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     6.0    0.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: 3.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     11.0   0.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: 5.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     16.0   0.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: 7.     1     22.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     21.0   0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: 9.     1     27.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     14.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    2.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: 5.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     5.0    2.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: 9.     1     6.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.1    0.5    2.0       <total>
 
 # CHECK:      [11] Code Region - G12
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3004
+# CHECK-NEXT: Total Cycles:      1006
 # CHECK-NEXT: Total uOps:        3000
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    1.00
-# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: uOps Per Cycle:    2.98
+# CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 10.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     0123456
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .  .   ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=====eeeeeeER.    .    .    .  .   ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: [0,3]     .D===========eER    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==========eeeeeeER    .    .  .   ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: [0,5]     . D================eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D===============eeeeeeER   .  .   ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D=====================eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D====================eeeeeeER.   ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D==========================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER .    ..   ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: [0,1]     D=eE----R .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeER.    ..   ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE----R.    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D==eeeeeeER  ..   ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,5]     . D===eE----R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D==eeeeeeER ..   ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D===eE----R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D====eeeeeeER   ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D=====eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1699,43 +1699,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     6.0    0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: 3.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     11.0   0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: 5.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     16.0   0.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: 7.     1     22.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     21.0   0.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: 9.     1     27.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     14.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    2.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: 5.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     5.0    2.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: 9.     1     6.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.1    0.5    2.0       <total>
 
 # CHECK:      [12] Code Region - G13
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3104
+# CHECK-NEXT: Total Cycles:      1209
 # CHECK-NEXT: Total uOps:        2800
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    0.90
-# CHECK-NEXT: IPC:               0.32
+# CHECK-NEXT: uOps Per Cycle:    2.32
+# CHECK-NEXT: IPC:               0.83
 # CHECK-NEXT: Block RThroughput: 8.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          01234
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          0
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .   .   ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=====eeeeeeER.    .    .    .   .   ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: [0,3]     .D===========eER    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==========eeeeeeER    .    .   .   ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: [0,5]     . D================eER   .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D===============eeeeeeER   .   .   ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D=====================eER  .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D====================eeeeeeeER.   ld1	{ v1.b }[0], [x27], #1
-# CHECK-NEXT: [0,9]     .   D===========================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER .    .    .   ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE----R .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeER.    .    .   ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE----R.    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D==eeeeeeER  .    .   ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,5]     . D===eE----R  .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D==eeeeeeER .    .   ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D===eE----R .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D=======eeeeeeeER   ld1	{ v1.b }[0], [x27], #1
+# CHECK-NEXT: [0,9]     .   D========eE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1745,22 +1745,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     6.0    0.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: 3.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     11.0   0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: 5.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     16.0   0.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: 7.     1     22.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     21.0   0.0    0.0       ld1	{ v1.b }[0], [x27], #1
-# CHECK-NEXT: 9.     1     28.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     14.1   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    2.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: 5.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     8.0    0.0    0.0       ld1	{ v1.b }[0], [x27], #1
+# CHECK-NEXT: 9.     1     9.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.7    0.3    2.1       <total>
 
 # CHECK:      [13] Code Region - G14
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3504
+# CHECK-NEXT: Total Cycles:      3503
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    8
@@ -1769,19 +1769,19 @@ add x0, x27, 1
 # CHECK-NEXT: Block RThroughput: 3.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          012345678
+# CHECK-NEXT:                     0123456789          01234567
 # CHECK-NEXT: Index     0123456789          0123456789
 
-# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    .  .   ld1	{ v1.b }[8], [x27], #1
-# CHECK-NEXT: [0,1]     D=======eER    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=======eeeeeeeER   .    .    .    .  .   ld1	{ v1.b }[0], [x27], x28
-# CHECK-NEXT: [0,3]     D==============eER  .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=============eeeeeeeER .    .    .  .   ld1	{ v1.b }[8], [x27], x28
-# CHECK-NEXT: [0,5]     .D====================eER.    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D====================eeeeeeeER    .  .   ld1	{ v1.h }[0], [x27], #2
-# CHECK-NEXT: [0,7]     .D===========================eER   .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==========================eeeeeeeER.   ld1	{ v1.h }[4], [x27], #2
-# CHECK-NEXT: [0,9]     . D=================================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    . .   ld1	{ v1.b }[8], [x27], #1
+# CHECK-NEXT: [0,1]     D=eE-----R.    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=======eeeeeeeER   .    .    .    . .   ld1	{ v1.b }[0], [x27], x28
+# CHECK-NEXT: [0,3]     D========eE-----R   .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=============eeeeeeeER .    .    . .   ld1	{ v1.b }[8], [x27], x28
+# CHECK-NEXT: [0,5]     .D==============eE-----R .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D====================eeeeeeeER    . .   ld1	{ v1.h }[0], [x27], #2
+# CHECK-NEXT: [0,7]     .D=====================eE-----R    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==========================eeeeeeeER   ld1	{ v1.h }[4], [x27], #2
+# CHECK-NEXT: [0,9]     . D===========================eE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1791,22 +1791,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.b }[8], [x27], #1
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     2.0    0.0    5.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld1	{ v1.b }[0], [x27], x28
-# CHECK-NEXT: 3.     1     15.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 3.     1     9.0    0.0    5.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     14.0   0.0    0.0       ld1	{ v1.b }[8], [x27], x28
-# CHECK-NEXT: 5.     1     21.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 5.     1     15.0   0.0    5.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     21.0   0.0    0.0       ld1	{ v1.h }[0], [x27], #2
-# CHECK-NEXT: 7.     1     28.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 7.     1     22.0   0.0    5.0       add	x0, x27, #1
 # CHECK-NEXT: 8.     1     27.0   0.0    0.0       ld1	{ v1.h }[4], [x27], #2
-# CHECK-NEXT: 9.     1     34.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     17.7   0.1    0.0       <total>
+# CHECK-NEXT: 9.     1     28.0   0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     14.7   0.1    2.5       <total>
 
 # CHECK:      [14] Code Region - G15
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3504
+# CHECK-NEXT: Total Cycles:      3503
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    8
@@ -1815,19 +1815,19 @@ add x0, x27, 1
 # CHECK-NEXT: Block RThroughput: 3.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          012345678
+# CHECK-NEXT:                     0123456789          01234567
 # CHECK-NEXT: Index     0123456789          0123456789
 
-# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    .  .   ld1	{ v1.h }[0], [x27], x28
-# CHECK-NEXT: [0,1]     D=======eER    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=======eeeeeeeER   .    .    .    .  .   ld1	{ v1.h }[4], [x27], x28
-# CHECK-NEXT: [0,3]     D==============eER  .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=============eeeeeeeER .    .    .  .   ld1	{ v1.s }[0], [x27], #4
-# CHECK-NEXT: [0,5]     .D====================eER.    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D====================eeeeeeeER    .  .   ld1	{ v1.s }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .D===========================eER   .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==========================eeeeeeeER.   ld1	{ v1.d }[0], [x27], #8
-# CHECK-NEXT: [0,9]     . D=================================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    . .   ld1	{ v1.h }[0], [x27], x28
+# CHECK-NEXT: [0,1]     D=eE-----R.    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=======eeeeeeeER   .    .    .    . .   ld1	{ v1.h }[4], [x27], x28
+# CHECK-NEXT: [0,3]     D========eE-----R   .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=============eeeeeeeER .    .    . .   ld1	{ v1.s }[0], [x27], #4
+# CHECK-NEXT: [0,5]     .D==============eE-----R .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D====================eeeeeeeER    . .   ld1	{ v1.s }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .D=====================eE-----R    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==========================eeeeeeeER   ld1	{ v1.d }[0], [x27], #8
+# CHECK-NEXT: [0,9]     . D===========================eE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1837,43 +1837,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.h }[0], [x27], x28
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     2.0    0.0    5.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld1	{ v1.h }[4], [x27], x28
-# CHECK-NEXT: 3.     1     15.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 3.     1     9.0    0.0    5.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     14.0   0.0    0.0       ld1	{ v1.s }[0], [x27], #4
-# CHECK-NEXT: 5.     1     21.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 5.     1     15.0   0.0    5.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     21.0   0.0    0.0       ld1	{ v1.s }[0], [x27], x28
-# CHECK-NEXT: 7.     1     28.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 7.     1     22.0   0.0    5.0       add	x0, x27, #1
 # CHECK-NEXT: 8.     1     27.0   0.0    0.0       ld1	{ v1.d }[0], [x27], #8
-# CHECK-NEXT: 9.     1     34.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     17.7   0.1    0.0       <total>
+# CHECK-NEXT: 9.     1     28.0   0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     14.7   0.1    2.5       <total>
 
 # CHECK:      [15] Code Region - G16
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3504
+# CHECK-NEXT: Total Cycles:      1103
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    0.57
-# CHECK-NEXT: IPC:               0.29
+# CHECK-NEXT: uOps Per Cycle:    1.81
+# CHECK-NEXT: IPC:               0.91
 # CHECK-NEXT: Block RThroughput: 3.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          012345678
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     0123
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    .  .   ld1	{ v1.d }[0], [x27], x28
-# CHECK-NEXT: [0,1]     D=======eER    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=======eeeeeeeER   .    .    .    .  .   ld1r	{ v1.1d }, [x27], #8
-# CHECK-NEXT: [0,3]     D==============eER  .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=============eeeeeeeER .    .    .  .   ld1r	{ v1.2d }, [x27], #8
-# CHECK-NEXT: [0,5]     .D====================eER.    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D====================eeeeeeeER    .  .   ld1r	{ v1.2s }, [x27], #4
-# CHECK-NEXT: [0,7]     .D===========================eER   .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==========================eeeeeeeER.   ld1r	{ v1.4h }, [x27], #2
-# CHECK-NEXT: [0,9]     . D=================================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeER.  .   ld1	{ v1.d }[0], [x27], x28
+# CHECK-NEXT: [0,1]     D=eE-----R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeeER  .   ld1r	{ v1.1d }, [x27], #8
+# CHECK-NEXT: [0,3]     D==eE-----R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeeeER .   ld1r	{ v1.2d }, [x27], #8
+# CHECK-NEXT: [0,5]     .D==eE-----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeeER.   ld1r	{ v1.2s }, [x27], #4
+# CHECK-NEXT: [0,7]     .D===eE-----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeeeER   ld1r	{ v1.4h }, [x27], #2
+# CHECK-NEXT: [0,9]     . D===eE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1883,43 +1883,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.d }[0], [x27], x28
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld1r	{ v1.1d }, [x27], #8
-# CHECK-NEXT: 3.     1     15.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     14.0   0.0    0.0       ld1r	{ v1.2d }, [x27], #8
-# CHECK-NEXT: 5.     1     21.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     21.0   0.0    0.0       ld1r	{ v1.2s }, [x27], #4
-# CHECK-NEXT: 7.     1     28.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     27.0   0.0    0.0       ld1r	{ v1.4h }, [x27], #2
-# CHECK-NEXT: 9.     1     34.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     17.7   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1r	{ v1.1d }, [x27], #8
+# CHECK-NEXT: 3.     1     3.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1r	{ v1.2d }, [x27], #8
+# CHECK-NEXT: 5.     1     3.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1r	{ v1.2s }, [x27], #4
+# CHECK-NEXT: 7.     1     4.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1r	{ v1.4h }, [x27], #2
+# CHECK-NEXT: 9.     1     4.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    2.5       <total>
 
 # CHECK:      [16] Code Region - G17
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3504
+# CHECK-NEXT: Total Cycles:      509
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    0.57
-# CHECK-NEXT: IPC:               0.29
+# CHECK-NEXT: uOps Per Cycle:    3.93
+# CHECK-NEXT: IPC:               1.96
 # CHECK-NEXT: Block RThroughput: 3.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          012345678
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     0123
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    .  .   ld1r	{ v1.4s }, [x27], #4
-# CHECK-NEXT: [0,1]     D=======eER    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=======eeeeeeeER   .    .    .    .  .   ld1r	{ v1.8b }, [x27], #1
-# CHECK-NEXT: [0,3]     D==============eER  .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=============eeeeeeeER .    .    .  .   ld1r	{ v1.8h }, [x27], #2
-# CHECK-NEXT: [0,5]     .D====================eER.    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D====================eeeeeeeER    .  .   ld1r	{ v1.16b }, [x27], #1
-# CHECK-NEXT: [0,7]     .D===========================eER   .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==========================eeeeeeeER.   ld1r	{ v1.1d }, [x27], x28
-# CHECK-NEXT: [0,9]     . D=================================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeER.  .   ld1r	{ v1.4s }, [x27], #4
+# CHECK-NEXT: [0,1]     D=eE-----R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeeER  .   ld1r	{ v1.8b }, [x27], #1
+# CHECK-NEXT: [0,3]     D==eE-----R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeeeER .   ld1r	{ v1.8h }, [x27], #2
+# CHECK-NEXT: [0,5]     .D==eE-----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeeER.   ld1r	{ v1.16b }, [x27], #1
+# CHECK-NEXT: [0,7]     .D===eE-----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeeeER   ld1r	{ v1.1d }, [x27], x28
+# CHECK-NEXT: [0,9]     . D===eE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1929,43 +1929,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1r	{ v1.4s }, [x27], #4
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld1r	{ v1.8b }, [x27], #1
-# CHECK-NEXT: 3.     1     15.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     14.0   0.0    0.0       ld1r	{ v1.8h }, [x27], #2
-# CHECK-NEXT: 5.     1     21.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     21.0   0.0    0.0       ld1r	{ v1.16b }, [x27], #1
-# CHECK-NEXT: 7.     1     28.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     27.0   0.0    0.0       ld1r	{ v1.1d }, [x27], x28
-# CHECK-NEXT: 9.     1     34.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     17.7   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1r	{ v1.8b }, [x27], #1
+# CHECK-NEXT: 3.     1     3.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1r	{ v1.8h }, [x27], #2
+# CHECK-NEXT: 5.     1     3.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1r	{ v1.16b }, [x27], #1
+# CHECK-NEXT: 7.     1     4.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1r	{ v1.1d }, [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    2.5       <total>
 
 # CHECK:      [17] Code Region - G18
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3504
+# CHECK-NEXT: Total Cycles:      509
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    0.57
-# CHECK-NEXT: IPC:               0.29
+# CHECK-NEXT: uOps Per Cycle:    3.93
+# CHECK-NEXT: IPC:               1.96
 # CHECK-NEXT: Block RThroughput: 3.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          012345678
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     0123
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    .  .   ld1r	{ v1.2d }, [x27], x28
-# CHECK-NEXT: [0,1]     D=======eER    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=======eeeeeeeER   .    .    .    .  .   ld1r	{ v1.2s }, [x27], x28
-# CHECK-NEXT: [0,3]     D==============eER  .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=============eeeeeeeER .    .    .  .   ld1r	{ v1.4h }, [x27], x28
-# CHECK-NEXT: [0,5]     .D====================eER.    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D====================eeeeeeeER    .  .   ld1r	{ v1.4s }, [x27], x28
-# CHECK-NEXT: [0,7]     .D===========================eER   .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==========================eeeeeeeER.   ld1r	{ v1.8b }, [x27], x28
-# CHECK-NEXT: [0,9]     . D=================================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeER.  .   ld1r	{ v1.2d }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE-----R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeeER  .   ld1r	{ v1.2s }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eE-----R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeeeER .   ld1r	{ v1.4h }, [x27], x28
+# CHECK-NEXT: [0,5]     .D==eE-----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeeER.   ld1r	{ v1.4s }, [x27], x28
+# CHECK-NEXT: [0,7]     .D===eE-----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeeeER   ld1r	{ v1.8b }, [x27], x28
+# CHECK-NEXT: [0,9]     . D===eE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1975,43 +1975,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1r	{ v1.2d }, [x27], x28
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld1r	{ v1.2s }, [x27], x28
-# CHECK-NEXT: 3.     1     15.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     14.0   0.0    0.0       ld1r	{ v1.4h }, [x27], x28
-# CHECK-NEXT: 5.     1     21.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     21.0   0.0    0.0       ld1r	{ v1.4s }, [x27], x28
-# CHECK-NEXT: 7.     1     28.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     27.0   0.0    0.0       ld1r	{ v1.8b }, [x27], x28
-# CHECK-NEXT: 9.     1     34.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     17.7   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1r	{ v1.2s }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1r	{ v1.4h }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1r	{ v1.4s }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1r	{ v1.8b }, [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    2.5       <total>
 
 # CHECK:      [18] Code Region - G19
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3504
+# CHECK-NEXT: Total Cycles:      509
 # CHECK-NEXT: Total uOps:        2600
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    0.74
-# CHECK-NEXT: IPC:               0.29
+# CHECK-NEXT: uOps Per Cycle:    5.11
+# CHECK-NEXT: IPC:               1.96
 # CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          012345678
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     0123
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    .  .   ld1r	{ v1.8h }, [x27], x28
-# CHECK-NEXT: [0,1]     D=======eER    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=======eeeeeeeER   .    .    .    .  .   ld1r	{ v1.16b }, [x27], x28
-# CHECK-NEXT: [0,3]     D==============eER  .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=============eeeeeeeER .    .    .  .   ld2	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: [0,5]     .D====================eER.    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     . D===================eeeeeeeER    .  .   ld2	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: [0,7]     . D==========================eER   .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .  D=========================eeeeeeeER.   ld2	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: [0,9]     .  D================================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeER.  .   ld1r	{ v1.8h }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE-----R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeeER  .   ld1r	{ v1.16b }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eE-----R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeeeER .   ld2	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: [0,5]     .D==eE-----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     . D=eeeeeeeER.   ld2	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: [0,7]     . D==eE-----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .  D=eeeeeeeER   ld2	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: [0,9]     .  D==eE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2021,43 +2021,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1r	{ v1.8h }, [x27], x28
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld1r	{ v1.16b }, [x27], x28
-# CHECK-NEXT: 3.     1     15.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     14.0   0.0    0.0       ld2	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: 5.     1     21.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     20.0   0.0    0.0       ld2	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: 7.     1     27.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     26.0   0.0    0.0       ld2	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: 9.     1     33.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     17.3   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1r	{ v1.16b }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld2	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: 5.     1     3.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ld2	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: 7.     1     3.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     2.0    0.0    0.0       ld2	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: 9.     1     3.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.1    2.5       <total>
 
 # CHECK:      [19] Code Region - G20
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3504
+# CHECK-NEXT: Total Cycles:      509
 # CHECK-NEXT: Total uOps:        3000
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    0.86
-# CHECK-NEXT: IPC:               0.29
+# CHECK-NEXT: uOps Per Cycle:    5.89
+# CHECK-NEXT: IPC:               1.96
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          012345678
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     0123
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    .  .   ld2	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: [0,1]     D=======eER    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D======eeeeeeeER   .    .    .    .  .   ld2	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: [0,3]     .D=============eER  .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D============eeeeeeeER .    .    .  .   ld2	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: [0,5]     . D===================eER.    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D==================eeeeeeeER    .  .   ld2	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: [0,7]     .  D=========================eER   .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D========================eeeeeeeER.   ld2	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D===============================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeER.  .   ld2	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: [0,1]     D=eE-----R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeER  .   ld2	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: [0,3]     .D=eE-----R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeeER .   ld2	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: [0,5]     . D=eE-----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeeER.   ld2	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: [0,7]     .  D=eE-----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeeeER   ld2	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D=eE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2067,43 +2067,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld2	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld2	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: 5.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld2	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: 7.     1     26.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld2	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: 9.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.5   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld2	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: 3.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld2	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: 5.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld2	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: 7.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ld2	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: 9.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    2.5       <total>
 
 # CHECK:      [20] Code Region - G21
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3504
+# CHECK-NEXT: Total Cycles:      509
 # CHECK-NEXT: Total uOps:        3000
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    0.86
-# CHECK-NEXT: IPC:               0.29
+# CHECK-NEXT: uOps Per Cycle:    5.89
+# CHECK-NEXT: IPC:               1.96
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          012345678
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     0123
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    .  .   ld2	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: [0,1]     D=======eER    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D======eeeeeeeER   .    .    .    .  .   ld2	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=============eER  .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D============eeeeeeeER .    .    .  .   ld2	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: [0,5]     . D===================eER.    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D==================eeeeeeeER    .  .   ld2	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D=========================eER   .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D========================eeeeeeeER.   ld2	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D===============================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeER.  .   ld2	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE-----R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeER  .   ld2	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE-----R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeeER .   ld2	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,5]     . D=eE-----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeeER.   ld2	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D=eE-----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeeeER   ld2	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D=eE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2113,43 +2113,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld2	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld2	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: 5.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld2	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: 7.     1     26.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld2	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: 9.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.5   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld2	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld2	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld2	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: 7.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ld2	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: 9.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    2.5       <total>
 
 # CHECK:      [21] Code Region - G22
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3504
+# CHECK-NEXT: Total Cycles:      2909
 # CHECK-NEXT: Total uOps:        3000
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    0.86
-# CHECK-NEXT: IPC:               0.29
+# CHECK-NEXT: uOps Per Cycle:    1.03
+# CHECK-NEXT: IPC:               0.34
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          012345678
+# CHECK-NEXT:                     0123456789          01234567
 # CHECK-NEXT: Index     0123456789          0123456789
 
-# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    .  .   ld2	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: [0,1]     D=======eER    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D======eeeeeeeER   .    .    .    .  .   ld2	{ v1.b, v2.b }[0], [x27], #2
-# CHECK-NEXT: [0,3]     .D=============eER  .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D============eeeeeeeER .    .    .  .   ld2	{ v1.b, v2.b }[8], [x27], #2
-# CHECK-NEXT: [0,5]     . D===================eER.    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D==================eeeeeeeER    .  .   ld2	{ v1.b, v2.b }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .  D=========================eER   .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D========================eeeeeeeER.   ld2	{ v1.b, v2.b }[8], [x27], x28
-# CHECK-NEXT: [0,9]     .   D===============================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    . .   ld2	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE-----R.    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .D======eeeeeeeER   .    .    .    . .   ld2	{ v1.b, v2.b }[0], [x27], #2
+# CHECK-NEXT: [0,3]     .D=======eE-----R   .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D============eeeeeeeER .    .    . .   ld2	{ v1.b, v2.b }[8], [x27], #2
+# CHECK-NEXT: [0,5]     . D=============eE-----R .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D==================eeeeeeeER    . .   ld2	{ v1.b, v2.b }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .  D===================eE-----R    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D========================eeeeeeeER   ld2	{ v1.b, v2.b }[8], [x27], x28
+# CHECK-NEXT: [0,9]     .   D=========================eE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2159,22 +2159,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     2.0    0.0    5.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld2	{ v1.b, v2.b }[0], [x27], #2
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 3.     1     8.0    0.0    5.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld2	{ v1.b, v2.b }[8], [x27], #2
-# CHECK-NEXT: 5.     1     20.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 5.     1     14.0   0.0    5.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld2	{ v1.b, v2.b }[0], [x27], x28
-# CHECK-NEXT: 7.     1     26.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 7.     1     20.0   0.0    5.0       add	x0, x27, #1
 # CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld2	{ v1.b, v2.b }[8], [x27], x28
-# CHECK-NEXT: 9.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.5   0.1    0.0       <total>
+# CHECK-NEXT: 9.     1     26.0   0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     13.5   0.1    2.5       <total>
 
 # CHECK:      [22] Code Region - G23
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3504
+# CHECK-NEXT: Total Cycles:      3503
 # CHECK-NEXT: Total uOps:        3000
 
 # CHECK:      Dispatch Width:    8
@@ -2183,19 +2183,19 @@ add x0, x27, 1
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          012345678
+# CHECK-NEXT:                     0123456789          01234567
 # CHECK-NEXT: Index     0123456789          0123456789
 
-# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    .  .   ld2	{ v1.h, v2.h }[0], [x27], #4
-# CHECK-NEXT: [0,1]     D=======eER    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D======eeeeeeeER   .    .    .    .  .   ld2	{ v1.h, v2.h }[4], [x27], #4
-# CHECK-NEXT: [0,3]     .D=============eER  .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D============eeeeeeeER .    .    .  .   ld2	{ v1.h, v2.h }[0], [x27], x28
-# CHECK-NEXT: [0,5]     . D===================eER.    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D==================eeeeeeeER    .  .   ld2	{ v1.h, v2.h }[4], [x27], x28
-# CHECK-NEXT: [0,7]     .  D=========================eER   .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D========================eeeeeeeER.   ld2	{ v1.s, v2.s }[0], [x27], #8
-# CHECK-NEXT: [0,9]     .   D===============================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    . .   ld2	{ v1.h, v2.h }[0], [x27], #4
+# CHECK-NEXT: [0,1]     D=eE-----R.    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .D======eeeeeeeER   .    .    .    . .   ld2	{ v1.h, v2.h }[4], [x27], #4
+# CHECK-NEXT: [0,3]     .D=======eE-----R   .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D============eeeeeeeER .    .    . .   ld2	{ v1.h, v2.h }[0], [x27], x28
+# CHECK-NEXT: [0,5]     . D=============eE-----R .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D==================eeeeeeeER    . .   ld2	{ v1.h, v2.h }[4], [x27], x28
+# CHECK-NEXT: [0,7]     .  D===================eE-----R    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D========================eeeeeeeER   ld2	{ v1.s, v2.s }[0], [x27], #8
+# CHECK-NEXT: [0,9]     .   D=========================eE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2205,43 +2205,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.h, v2.h }[0], [x27], #4
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     2.0    0.0    5.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld2	{ v1.h, v2.h }[4], [x27], #4
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 3.     1     8.0    0.0    5.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld2	{ v1.h, v2.h }[0], [x27], x28
-# CHECK-NEXT: 5.     1     20.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 5.     1     14.0   0.0    5.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld2	{ v1.h, v2.h }[4], [x27], x28
-# CHECK-NEXT: 7.     1     26.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 7.     1     20.0   0.0    5.0       add	x0, x27, #1
 # CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld2	{ v1.s, v2.s }[0], [x27], #8
-# CHECK-NEXT: 9.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.5   0.1    0.0       <total>
+# CHECK-NEXT: 9.     1     26.0   0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     13.5   0.1    2.5       <total>
 
 # CHECK:      [23] Code Region - G24
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3504
+# CHECK-NEXT: Total Cycles:      2303
 # CHECK-NEXT: Total uOps:        3000
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    0.86
-# CHECK-NEXT: IPC:               0.29
+# CHECK-NEXT: uOps Per Cycle:    1.30
+# CHECK-NEXT: IPC:               0.43
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          012345678
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          012345
 
-# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    .  .   ld2	{ v1.s, v2.s }[0], [x27], x28
-# CHECK-NEXT: [0,1]     D=======eER    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D======eeeeeeeER   .    .    .    .  .   ld2	{ v1.d, v2.d }[0], [x27], #16
-# CHECK-NEXT: [0,3]     .D=============eER  .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D============eeeeeeeER .    .    .  .   ld2	{ v1.d, v2.d }[0], [x27], x28
-# CHECK-NEXT: [0,5]     . D===================eER.    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D==================eeeeeeeER    .  .   ld2r	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: [0,7]     .  D=========================eER   .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D========================eeeeeeeER.   ld2r	{ v1.2d, v2.2d }, [x27], #16
-# CHECK-NEXT: [0,9]     .   D===============================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeER.    .    .    .   ld2	{ v1.s, v2.s }[0], [x27], x28
+# CHECK-NEXT: [0,1]     D=eE-----R.    .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .D======eeeeeeeER   .    .   ld2	{ v1.d, v2.d }[0], [x27], #16
+# CHECK-NEXT: [0,3]     .D=======eE-----R   .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D============eeeeeeeER .   ld2	{ v1.d, v2.d }[0], [x27], x28
+# CHECK-NEXT: [0,5]     . D=============eE-----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D============eeeeeeeER.   ld2r	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: [0,7]     .  D=============eE-----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D============eeeeeeeER   ld2r	{ v1.2d, v2.2d }, [x27], #16
+# CHECK-NEXT: [0,9]     .   D=============eE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2251,43 +2251,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.s, v2.s }[0], [x27], x28
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     2.0    0.0    5.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld2	{ v1.d, v2.d }[0], [x27], #16
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 3.     1     8.0    0.0    5.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld2	{ v1.d, v2.d }[0], [x27], x28
-# CHECK-NEXT: 5.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld2r	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: 7.     1     26.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld2r	{ v1.2d, v2.2d }, [x27], #16
-# CHECK-NEXT: 9.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.5   0.1    0.0       <total>
+# CHECK-NEXT: 5.     1     14.0   0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     13.0   0.0    0.0       ld2r	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: 7.     1     14.0   0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     13.0   0.0    0.0       ld2r	{ v1.2d, v2.2d }, [x27], #16
+# CHECK-NEXT: 9.     1     14.0   0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     9.9    0.1    2.5       <total>
 
 # CHECK:      [24] Code Region - G25
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3504
+# CHECK-NEXT: Total Cycles:      509
 # CHECK-NEXT: Total uOps:        3000
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    0.86
-# CHECK-NEXT: IPC:               0.29
+# CHECK-NEXT: uOps Per Cycle:    5.89
+# CHECK-NEXT: IPC:               1.96
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          012345678
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     0123
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    .  .   ld2r	{ v1.2s, v2.2s }, [x27], #8
-# CHECK-NEXT: [0,1]     D=======eER    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D======eeeeeeeER   .    .    .    .  .   ld2r	{ v1.4h, v2.4h }, [x27], #4
-# CHECK-NEXT: [0,3]     .D=============eER  .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D============eeeeeeeER .    .    .  .   ld2r	{ v1.4s, v2.4s }, [x27], #8
-# CHECK-NEXT: [0,5]     . D===================eER.    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D==================eeeeeeeER    .  .   ld2r	{ v1.8b, v2.8b }, [x27], #2
-# CHECK-NEXT: [0,7]     .  D=========================eER   .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D========================eeeeeeeER.   ld2r	{ v1.8h, v2.8h }, [x27], #4
-# CHECK-NEXT: [0,9]     .   D===============================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeER.  .   ld2r	{ v1.2s, v2.2s }, [x27], #8
+# CHECK-NEXT: [0,1]     D=eE-----R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeER  .   ld2r	{ v1.4h, v2.4h }, [x27], #4
+# CHECK-NEXT: [0,3]     .D=eE-----R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeeER .   ld2r	{ v1.4s, v2.4s }, [x27], #8
+# CHECK-NEXT: [0,5]     . D=eE-----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeeER.   ld2r	{ v1.8b, v2.8b }, [x27], #2
+# CHECK-NEXT: [0,7]     .  D=eE-----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeeeER   ld2r	{ v1.8h, v2.8h }, [x27], #4
+# CHECK-NEXT: [0,9]     .   D=eE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2297,43 +2297,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2r	{ v1.2s, v2.2s }, [x27], #8
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld2r	{ v1.4h, v2.4h }, [x27], #4
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld2r	{ v1.4s, v2.4s }, [x27], #8
-# CHECK-NEXT: 5.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld2r	{ v1.8b, v2.8b }, [x27], #2
-# CHECK-NEXT: 7.     1     26.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld2r	{ v1.8h, v2.8h }, [x27], #4
-# CHECK-NEXT: 9.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.5   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld2r	{ v1.4h, v2.4h }, [x27], #4
+# CHECK-NEXT: 3.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld2r	{ v1.4s, v2.4s }, [x27], #8
+# CHECK-NEXT: 5.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld2r	{ v1.8b, v2.8b }, [x27], #2
+# CHECK-NEXT: 7.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ld2r	{ v1.8h, v2.8h }, [x27], #4
+# CHECK-NEXT: 9.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    2.5       <total>
 
 # CHECK:      [25] Code Region - G26
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3504
+# CHECK-NEXT: Total Cycles:      509
 # CHECK-NEXT: Total uOps:        3000
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    0.86
-# CHECK-NEXT: IPC:               0.29
+# CHECK-NEXT: uOps Per Cycle:    5.89
+# CHECK-NEXT: IPC:               1.96
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          012345678
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     0123
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    .  .   ld2r	{ v1.16b, v2.16b }, [x27], #2
-# CHECK-NEXT: [0,1]     D=======eER    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D======eeeeeeeER   .    .    .    .  .   ld2r	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=============eER  .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D============eeeeeeeER .    .    .  .   ld2r	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: [0,5]     . D===================eER.    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D==================eeeeeeeER    .  .   ld2r	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D=========================eER   .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D========================eeeeeeeER.   ld2r	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D===============================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeER.  .   ld2r	{ v1.16b, v2.16b }, [x27], #2
+# CHECK-NEXT: [0,1]     D=eE-----R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeER  .   ld2r	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE-----R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeeER .   ld2r	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,5]     . D=eE-----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeeER.   ld2r	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D=eE-----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeeeER   ld2r	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D=eE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2343,43 +2343,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2r	{ v1.16b, v2.16b }, [x27], #2
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld2r	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld2r	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: 5.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld2r	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: 7.     1     26.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld2r	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: 9.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.5   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld2r	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld2r	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld2r	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: 7.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ld2r	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: 9.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    2.5       <total>
 
 # CHECK:      [26] Code Region - G27
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3604
+# CHECK-NEXT: Total Cycles:      609
 # CHECK-NEXT: Total uOps:        3200
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    0.89
-# CHECK-NEXT: IPC:               0.28
+# CHECK-NEXT: uOps Per Cycle:    5.25
+# CHECK-NEXT: IPC:               1.64
 # CHECK-NEXT: Block RThroughput: 5.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    .   .   ld2r	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: [0,1]     D=======eER    .    .    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D======eeeeeeeER   .    .    .    .   .   ld2r	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=============eER  .    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D============eeeeeeeER .    .    .   .   ld2r	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: [0,5]     . D===================eER.    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D==================eeeeeeeER    .   .   ld2r	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D=========================eER   .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D========================eeeeeeeeER.   ld3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: [0,9]     .   D================================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeER.   .   ld2r	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE-----R.   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeER   .   ld2r	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE-----R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeeER  .   ld2r	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,5]     . D=eE-----R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeeER .   ld2r	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D=eE-----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeeeeER   ld3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: [0,9]     .   D=eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2389,43 +2389,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2r	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld2r	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld2r	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: 5.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld2r	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: 7.     1     26.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: 9.     1     33.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.6   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld2r	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld2r	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld2r	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: 7.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ld3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: 9.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    2.6       <total>
 
 # CHECK:      [27] Code Region - G28
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      759
 # CHECK-NEXT: Total uOps:        4000
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    1.00
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    5.27
+# CHECK-NEXT: IPC:               1.32
 # CHECK-NEXT: Block RThroughput: 7.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    .  .   ld3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: [0,3]     .D===============eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    .  .   ld3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: [0,5]     . D======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    .  .   ld3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: [0,7]     .  D=============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER.   ld3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: [0,9]     .   D====================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0123456
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER    ..   ld3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: [0,1]     D=eE------R    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeER   ..   ld3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: [0,3]     .D=eE------R   ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeeeeeeeER ..   ld3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: [0,5]     . D==eE------R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeeeeeeeER..   ld3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: [0,7]     .  D==eE------R..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D==eeeeeeeeER   ld3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: [0,9]     .   D===eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2435,43 +2435,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: 3.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: 5.     1     23.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: 7.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: 9.     1     37.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     19.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       ld3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ld3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: 7.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    1.0    0.0       ld3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.3    3.0       <total>
 
 # CHECK:      [28] Code Region - G29
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      759
 # CHECK-NEXT: Total uOps:        4000
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    1.00
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    5.27
+# CHECK-NEXT: IPC:               1.32
 # CHECK-NEXT: Block RThroughput: 7.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    .  .   ld3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: [0,3]     .D===============eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    .  .   ld3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: [0,5]     . D======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    .  .   ld3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D=============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER.   ld3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D====================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0123456
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER    ..   ld3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: [0,1]     D=eE------R    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeER   ..   ld3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE------R   ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeeeeeeeER ..   ld3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,5]     . D==eE------R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeeeeeeeER..   ld3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D==eE------R..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D==eeeeeeeeER   ld3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D===eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2481,43 +2481,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: 3.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: 5.     1     23.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: 7.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: 9.     1     37.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     19.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       ld3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ld3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 7.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    1.0    0.0       ld3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.3    3.0       <total>
 
 # CHECK:      [29] Code Region - G30
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3804
+# CHECK-NEXT: Total Cycles:      1909
 # CHECK-NEXT: Total uOps:        3800
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    1.00
-# CHECK-NEXT: IPC:               0.26
+# CHECK-NEXT: uOps Per Cycle:    1.99
+# CHECK-NEXT: IPC:               0.52
 # CHECK-NEXT: Block RThroughput: 7.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          01
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    ..   ld3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    ..   ld3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: [0,3]     .D===============eER.    .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    ..   ld3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: [0,5]     . D======================eER  .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeER .    ..   ld3	{ v1.b, v2.b, v3.b }[0], [x27], #3
-# CHECK-NEXT: [0,7]     .  D============================eER.    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D===========================eeeeeeeER.   ld3	{ v1.b, v2.b, v3.b }[8], [x27], #3
-# CHECK-NEXT: [0,9]     .   D==================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          01234567
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    . .   ld3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE------R    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeER   .    .    . .   ld3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE------R   .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeeeeeeeER .    .    . .   ld3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,5]     . D==eE------R .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D========eeeeeeeER    . .   ld3	{ v1.b, v2.b, v3.b }[0], [x27], #3
+# CHECK-NEXT: [0,7]     .  D=========eE-----R    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D==============eeeeeeeER   ld3	{ v1.b, v2.b, v3.b }[8], [x27], #3
+# CHECK-NEXT: [0,9]     .   D===============eE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2527,22 +2527,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: 3.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: 5.     1     23.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[0], [x27], #3
-# CHECK-NEXT: 7.     1     29.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     28.0   0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[8], [x27], #3
-# CHECK-NEXT: 9.     1     35.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     18.6   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       ld3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     9.0    0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[0], [x27], #3
+# CHECK-NEXT: 7.     1     10.0   0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     15.0   0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[8], [x27], #3
+# CHECK-NEXT: 9.     1     16.0   0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     6.1    0.2    2.8       <total>
 
 # CHECK:      [30] Code Region - G31
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3504
+# CHECK-NEXT: Total Cycles:      3503
 # CHECK-NEXT: Total uOps:        3500
 
 # CHECK:      Dispatch Width:    8
@@ -2551,19 +2551,19 @@ add x0, x27, 1
 # CHECK-NEXT: Block RThroughput: 7.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          012345678
+# CHECK-NEXT:                     0123456789          01234567
 # CHECK-NEXT: Index     0123456789          0123456789
 
-# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    .  .   ld3	{ v1.b, v2.b, v3.b }[0], [x27], x28
-# CHECK-NEXT: [0,1]     D=======eER    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D======eeeeeeeER   .    .    .    .  .   ld3	{ v1.b, v2.b, v3.b }[8], [x27], x28
-# CHECK-NEXT: [0,3]     .D=============eER  .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D============eeeeeeeER .    .    .  .   ld3	{ v1.h, v2.h, v3.h }[0], [x27], #6
-# CHECK-NEXT: [0,5]     . D===================eER.    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D==================eeeeeeeER    .  .   ld3	{ v1.h, v2.h, v3.h }[4], [x27], #6
-# CHECK-NEXT: [0,7]     .  D=========================eER   .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D========================eeeeeeeER.   ld3	{ v1.h, v2.h, v3.h }[0], [x27], x28
-# CHECK-NEXT: [0,9]     .   D===============================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    . .   ld3	{ v1.b, v2.b, v3.b }[0], [x27], x28
+# CHECK-NEXT: [0,1]     D=eE-----R.    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .D======eeeeeeeER   .    .    .    . .   ld3	{ v1.b, v2.b, v3.b }[8], [x27], x28
+# CHECK-NEXT: [0,3]     .D=======eE-----R   .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D============eeeeeeeER .    .    . .   ld3	{ v1.h, v2.h, v3.h }[0], [x27], #6
+# CHECK-NEXT: [0,5]     . D=============eE-----R .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D==================eeeeeeeER    . .   ld3	{ v1.h, v2.h, v3.h }[4], [x27], #6
+# CHECK-NEXT: [0,7]     .  D===================eE-----R    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D========================eeeeeeeER   ld3	{ v1.h, v2.h, v3.h }[0], [x27], x28
+# CHECK-NEXT: [0,9]     .   D=========================eE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2573,22 +2573,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.b, v2.b, v3.b }[0], [x27], x28
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     2.0    0.0    5.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[8], [x27], x28
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 3.     1     8.0    0.0    5.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld3	{ v1.h, v2.h, v3.h }[0], [x27], #6
-# CHECK-NEXT: 5.     1     20.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 5.     1     14.0   0.0    5.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld3	{ v1.h, v2.h, v3.h }[4], [x27], #6
-# CHECK-NEXT: 7.     1     26.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 7.     1     20.0   0.0    5.0       add	x0, x27, #1
 # CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld3	{ v1.h, v2.h, v3.h }[0], [x27], x28
-# CHECK-NEXT: 9.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.5   0.1    0.0       <total>
+# CHECK-NEXT: 9.     1     26.0   0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     13.5   0.1    2.5       <total>
 
 # CHECK:      [31] Code Region - G32
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3504
+# CHECK-NEXT: Total Cycles:      3503
 # CHECK-NEXT: Total uOps:        3500
 
 # CHECK:      Dispatch Width:    8
@@ -2597,19 +2597,19 @@ add x0, x27, 1
 # CHECK-NEXT: Block RThroughput: 7.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          012345678
+# CHECK-NEXT:                     0123456789          01234567
 # CHECK-NEXT: Index     0123456789          0123456789
 
-# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    .  .   ld3	{ v1.h, v2.h, v3.h }[4], [x27], x28
-# CHECK-NEXT: [0,1]     D=======eER    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D======eeeeeeeER   .    .    .    .  .   ld3	{ v1.s, v2.s, v3.s }[0], [x27], #12
-# CHECK-NEXT: [0,3]     .D=============eER  .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D============eeeeeeeER .    .    .  .   ld3	{ v1.s, v2.s, v3.s }[0], [x27], x28
-# CHECK-NEXT: [0,5]     . D===================eER.    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D==================eeeeeeeER    .  .   ld3	{ v1.d, v2.d, v3.d }[0], [x27], #24
-# CHECK-NEXT: [0,7]     .  D=========================eER   .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D========================eeeeeeeER.   ld3	{ v1.d, v2.d, v3.d }[0], [x27], x28
-# CHECK-NEXT: [0,9]     .   D===============================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    . .   ld3	{ v1.h, v2.h, v3.h }[4], [x27], x28
+# CHECK-NEXT: [0,1]     D=eE-----R.    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .D======eeeeeeeER   .    .    .    . .   ld3	{ v1.s, v2.s, v3.s }[0], [x27], #12
+# CHECK-NEXT: [0,3]     .D=======eE-----R   .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D============eeeeeeeER .    .    . .   ld3	{ v1.s, v2.s, v3.s }[0], [x27], x28
+# CHECK-NEXT: [0,5]     . D=============eE-----R .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D==================eeeeeeeER    . .   ld3	{ v1.d, v2.d, v3.d }[0], [x27], #24
+# CHECK-NEXT: [0,7]     .  D===================eE-----R    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D========================eeeeeeeER   ld3	{ v1.d, v2.d, v3.d }[0], [x27], x28
+# CHECK-NEXT: [0,9]     .   D=========================eE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2619,43 +2619,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.h, v2.h, v3.h }[4], [x27], x28
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     2.0    0.0    5.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld3	{ v1.s, v2.s, v3.s }[0], [x27], #12
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 3.     1     8.0    0.0    5.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld3	{ v1.s, v2.s, v3.s }[0], [x27], x28
-# CHECK-NEXT: 5.     1     20.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 5.     1     14.0   0.0    5.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld3	{ v1.d, v2.d, v3.d }[0], [x27], #24
-# CHECK-NEXT: 7.     1     26.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 7.     1     20.0   0.0    5.0       add	x0, x27, #1
 # CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld3	{ v1.d, v2.d, v3.d }[0], [x27], x28
-# CHECK-NEXT: 9.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.5   0.1    0.0       <total>
+# CHECK-NEXT: 9.     1     26.0   0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     13.5   0.1    2.5       <total>
 
 # CHECK:      [32] Code Region - G33
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3504
+# CHECK-NEXT: Total Cycles:      758
 # CHECK-NEXT: Total uOps:        3500
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    1.00
-# CHECK-NEXT: IPC:               0.29
+# CHECK-NEXT: uOps Per Cycle:    4.62
+# CHECK-NEXT: IPC:               1.32
 # CHECK-NEXT: Block RThroughput: 7.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          012345678
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     012345
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    .  .   ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: [0,1]     D=======eER    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D======eeeeeeeER   .    .    .    .  .   ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], #24
-# CHECK-NEXT: [0,3]     .D=============eER  .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D============eeeeeeeER .    .    .  .   ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], #12
-# CHECK-NEXT: [0,5]     . D===================eER.    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D==================eeeeeeeER    .  .   ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], #6
-# CHECK-NEXT: [0,7]     .  D=========================eER   .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D========================eeeeeeeER.   ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], #12
-# CHECK-NEXT: [0,9]     .   D===============================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeER.    .   ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], #24
+# CHECK-NEXT: [0,1]     D=eE-----R.    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeER    .   ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], #24
+# CHECK-NEXT: [0,3]     .D=eE-----R    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeeeeeeER  .   ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], #12
+# CHECK-NEXT: [0,5]     . D==eE-----R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeeeeeeER .   ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], #6
+# CHECK-NEXT: [0,7]     .  D==eE-----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D==eeeeeeeER   ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], #12
+# CHECK-NEXT: [0,9]     .   D===eE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2665,43 +2665,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], #24
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], #12
-# CHECK-NEXT: 5.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], #6
-# CHECK-NEXT: 7.     1     26.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], #12
-# CHECK-NEXT: 9.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.5   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], #24
+# CHECK-NEXT: 3.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], #12
+# CHECK-NEXT: 5.     1     3.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], #6
+# CHECK-NEXT: 7.     1     3.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    1.0    0.0       ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], #12
+# CHECK-NEXT: 9.     1     4.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.3    2.5       <total>
 
 # CHECK:      [33] Code Region - G34
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3504
+# CHECK-NEXT: Total Cycles:      758
 # CHECK-NEXT: Total uOps:        3500
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    1.00
-# CHECK-NEXT: IPC:               0.29
+# CHECK-NEXT: uOps Per Cycle:    4.62
+# CHECK-NEXT: IPC:               1.32
 # CHECK-NEXT: Block RThroughput: 7.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          012345678
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     012345
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    .  .   ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], #3
-# CHECK-NEXT: [0,1]     D=======eER    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D======eeeeeeeER   .    .    .    .  .   ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], #6
-# CHECK-NEXT: [0,3]     .D=============eER  .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D============eeeeeeeER .    .    .  .   ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], #3
-# CHECK-NEXT: [0,5]     . D===================eER.    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D==================eeeeeeeER    .  .   ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D=========================eER   .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D========================eeeeeeeER.   ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D===============================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeER.    .   ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], #3
+# CHECK-NEXT: [0,1]     D=eE-----R.    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeER    .   ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], #6
+# CHECK-NEXT: [0,3]     .D=eE-----R    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeeeeeeER  .   ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], #3
+# CHECK-NEXT: [0,5]     . D==eE-----R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeeeeeeER .   ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D==eE-----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D==eeeeeeeER   ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D===eE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2711,43 +2711,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], #3
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], #6
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], #3
-# CHECK-NEXT: 5.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: 7.     1     26.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: 9.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.5   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], #6
+# CHECK-NEXT: 3.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], #3
+# CHECK-NEXT: 5.     1     3.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: 7.     1     3.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    1.0    0.0       ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.3    2.5       <total>
 
 # CHECK:      [34] Code Region - G35
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3504
+# CHECK-NEXT: Total Cycles:      758
 # CHECK-NEXT: Total uOps:        3500
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    1.00
-# CHECK-NEXT: IPC:               0.29
+# CHECK-NEXT: uOps Per Cycle:    4.62
+# CHECK-NEXT: IPC:               1.32
 # CHECK-NEXT: Block RThroughput: 7.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          012345678
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     012345
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    .  .   ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: [0,1]     D=======eER    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D======eeeeeeeER   .    .    .    .  .   ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=============eER  .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D============eeeeeeeER .    .    .  .   ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: [0,5]     . D===================eER.    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D==================eeeeeeeER    .  .   ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D=========================eER   .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D========================eeeeeeeER.   ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D===============================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeER.    .   ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE-----R.    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeER    .   ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE-----R    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeeeeeeER  .   ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: [0,5]     . D==eE-----R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeeeeeeER .   ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D==eE-----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D==eeeeeeeER   ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D===eE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2757,43 +2757,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: 5.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: 7.     1     26.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: 9.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.5   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: 7.     1     3.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    1.0    0.0       ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.3    2.5       <total>
 
 # CHECK:      [35] Code Region - G36
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4304
+# CHECK-NEXT: Total Cycles:      960
 # CHECK-NEXT: Total uOps:        4500
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    1.05
-# CHECK-NEXT: IPC:               0.23
+# CHECK-NEXT: uOps Per Cycle:    4.69
+# CHECK-NEXT: IPC:               1.04
 # CHECK-NEXT: Block RThroughput: 9.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123456
-
-# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    .    .    ..   ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: [0,1]     D=======eER    .    .    .    .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D======eeeeeeeeeeER.    .    .    .    .    ..   ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: [0,3]     . D===============eER    .    .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .  D==============eeeeeeeeER  .    .    .    ..   ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: [0,5]     .   D=====================eER .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    D====================eeeeeeeeER    .    ..   ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: [0,7]     .    .D===========================eER   .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    . D==========================eeeeeeeeeeER.   ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: [0,9]     .    .  D===================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeER.    .   .   ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE-----R.    .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeeeER .   .   ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: [0,3]     . DeE--------R .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .  DeeeeeeeeER .   .   ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: [0,5]     .   DeE------R .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    DeeeeeeeeER   .   ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: [0,7]     .    .DeE------R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    . DeeeeeeeeeeER   ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: [0,9]     .    .  DeE--------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2803,43 +2803,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: 3.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: 5.     1     22.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     21.0   0.0    0.0       ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: 7.     1     28.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     27.0   0.0    0.0       ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: 9.     1     36.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     18.1   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: 3.     1     1.0    0.0    8.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: 7.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: 9.     1     1.0    0.0    8.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.1    0.4    3.3       <total>
 
 # CHECK:      [36] Code Region - G37
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4604
+# CHECK-NEXT: Total Cycles:      1009
 # CHECK-NEXT: Total uOps:        4800
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    1.04
-# CHECK-NEXT: IPC:               0.22
+# CHECK-NEXT: uOps Per Cycle:    4.76
+# CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 10.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123456789
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .    .   .   ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: [0,1]     .D=======eER   .    .    .    .    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D======eeeeeeeeeeER    .    .    .    .    .   .   ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: [0,3]     .  D===============eER   .    .    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D==============eeeeeeeeeeER    .    .    .   .   ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: [0,5]     .    D=======================eER   .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D======================eeeeeeeeeeER    .   .   ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: [0,7]     .    . D===============================eER   .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .  D==============================eeeeeeeeER.   ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: [0,9]     .    .   D=====================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     012345678
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER    .  .   ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: [0,1]     .DeE------R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeeeER.  .   ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: [0,3]     .  DeE--------R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeeeER .   ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: [0,5]     .    DeE--------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeeeeER   ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeE--------R   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeeeER   ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2849,43 +2849,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: 3.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: 5.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     23.0   0.0    0.0       ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: 7.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     31.0   0.0    0.0       ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: 9.     1     38.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     19.5   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: 3.     1     1.0    0.0    8.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: 5.     1     1.0    0.0    8.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    8.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    3.6       <total>
 
 # CHECK:      [37] Code Region - G38
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4604
+# CHECK-NEXT: Total Cycles:      1011
 # CHECK-NEXT: Total uOps:        4800
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    1.04
-# CHECK-NEXT: IPC:               0.22
+# CHECK-NEXT: uOps Per Cycle:    4.75
+# CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 10.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123456789
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .    .   .   ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: [0,1]     .D=======eER   .    .    .    .    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D======eeeeeeeeeeER    .    .    .    .    .   .   ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: [0,3]     .  D===============eER   .    .    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D==============eeeeeeeeER .    .    .    .   .   ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: [0,5]     .    D=====================eER.    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D====================eeeeeeeeeeER .    .   .   ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: [0,7]     .    . D=============================eER.    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .  D============================eeeeeeeeeeER.   ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: [0,9]     .    .   D=====================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          0
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .   ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE------R    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeeeER.    .   ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE--------R.    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeER.    .   ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: [0,5]     .    DeE------R.    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeeeeER .   ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeE--------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeeeeeER   ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeE--------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2895,22 +2895,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: 3.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: 5.     1     22.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     21.0   0.0    0.0       ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: 7.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: 9.     1     38.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     18.7   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    8.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    8.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    8.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    3.6       <total>
 
 # CHECK:      [38] Code Region - G39
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        5000
 
 # CHECK:      Dispatch Width:    8
@@ -2920,18 +2920,18 @@ add x0, x27, 1
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
-# CHECK-NEXT: [0,1]     .D=======eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D======eeeeeeeeER .    .    .    .    .  .   ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
-# CHECK-NEXT: [0,3]     .  D=============eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D============eeeeeeeeER   .    .    .  .   ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
-# CHECK-NEXT: [0,5]     .    D===================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D==================eeeeeeeeER.    .  .   ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
-# CHECK-NEXT: [0,7]     .    . D=========================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .  D========================eeeeeeeeER.   ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
-# CHECK-NEXT: [0,9]     .    .   D===============================eER   add	x0, x27, #1
+# CHECK-NEXT: Index     0123456789          0123456789          012
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
+# CHECK-NEXT: [0,1]     .DeE------R    .    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . D======eeeeeeeeER .    .    .    .    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
+# CHECK-NEXT: [0,3]     .  D======eE------R .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   D============eeeeeeeeER   .    .    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
+# CHECK-NEXT: [0,5]     .    D============eE------R   .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .D==================eeeeeeeeER.    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
+# CHECK-NEXT: [0,7]     .    . D==================eE------R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  D========================eeeeeeeeER   ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
+# CHECK-NEXT: [0,9]     .    .   D========================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2941,22 +2941,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 3.     1     7.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
-# CHECK-NEXT: 5.     1     20.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 5.     1     13.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
-# CHECK-NEXT: 7.     1     26.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 7.     1     19.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
-# CHECK-NEXT: 9.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.5   0.1    0.0       <total>
+# CHECK-NEXT: 9.     1     25.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     13.0   0.1    3.0       <total>
 
 # CHECK:      [39] Code Region - G40
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        5000
 
 # CHECK:      Dispatch Width:    8
@@ -2966,18 +2966,18 @@ add x0, x27, 1
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
-# CHECK-NEXT: [0,1]     .D=======eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D======eeeeeeeeER .    .    .    .    .  .   ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
-# CHECK-NEXT: [0,3]     .  D=============eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D============eeeeeeeeER   .    .    .  .   ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
-# CHECK-NEXT: [0,5]     .    D===================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D==================eeeeeeeeER.    .  .   ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
-# CHECK-NEXT: [0,7]     .    . D=========================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .  D========================eeeeeeeeER.   ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
-# CHECK-NEXT: [0,9]     .    .   D===============================eER   add	x0, x27, #1
+# CHECK-NEXT: Index     0123456789          0123456789          012
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
+# CHECK-NEXT: [0,1]     .DeE------R    .    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . D======eeeeeeeeER .    .    .    .    . .   ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .  D======eE------R .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   D============eeeeeeeeER   .    .    . .   ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
+# CHECK-NEXT: [0,5]     .    D============eE------R   .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .D==================eeeeeeeeER.    . .   ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
+# CHECK-NEXT: [0,7]     .    . D==================eE------R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  D========================eeeeeeeeER   ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
+# CHECK-NEXT: [0,9]     .    .   D========================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2987,43 +2987,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 3.     1     7.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
-# CHECK-NEXT: 5.     1     20.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 5.     1     13.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
-# CHECK-NEXT: 7.     1     26.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 7.     1     19.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
-# CHECK-NEXT: 9.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.5   0.1    0.0       <total>
+# CHECK-NEXT: 9.     1     25.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     13.0   0.1    3.0       <total>
 
 # CHECK:      [40] Code Region - G41
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      2103
 # CHECK-NEXT: Total uOps:        5000
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    1.25
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    2.38
+# CHECK-NEXT: IPC:               0.48
 # CHECK-NEXT: Block RThroughput: 10.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
-# CHECK-NEXT: [0,1]     .D=======eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D======eeeeeeeeER .    .    .    .    .  .   ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
-# CHECK-NEXT: [0,3]     .  D=============eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D============eeeeeeeeER   .    .    .  .   ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: [0,5]     .    D===================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D==================eeeeeeeeER.    .  .   ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #32
-# CHECK-NEXT: [0,7]     .    . D=========================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .  D========================eeeeeeeeER.   ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #16
-# CHECK-NEXT: [0,9]     .    .   D===============================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          0123
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .  .   ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
+# CHECK-NEXT: [0,1]     .DeE------R    .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . D======eeeeeeeeER .  .   ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .  D======eE------R .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   D=====eeeeeeeeER.  .   ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: [0,5]     .    D=====eE------R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .D======eeeeeeeeER.   ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #32
+# CHECK-NEXT: [0,7]     .    . D======eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  D=====eeeeeeeeER   ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #16
+# CHECK-NEXT: [0,9]     .    .   D=====eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3033,43 +3033,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: 5.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #32
-# CHECK-NEXT: 7.     1     26.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #16
-# CHECK-NEXT: 9.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.5   0.1    0.0       <total>
+# CHECK-NEXT: 3.     1     7.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     6.0    0.0    0.0       ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: 5.     1     6.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     7.0    2.0    0.0       ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #32
+# CHECK-NEXT: 7.     1     7.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     6.0    0.0    0.0       ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #16
+# CHECK-NEXT: 9.     1     6.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     5.4    0.3    3.0       <total>
 
 # CHECK:      [41] Code Region - G42
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      1009
 # CHECK-NEXT: Total uOps:        5000
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    1.25
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    4.96
+# CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 10.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #8
-# CHECK-NEXT: [0,1]     .D=======eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D======eeeeeeeeER .    .    .    .    .  .   ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #16
-# CHECK-NEXT: [0,3]     .  D=============eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D============eeeeeeeeER   .    .    .  .   ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #4
-# CHECK-NEXT: [0,5]     .    D===================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D==================eeeeeeeeER.    .  .   ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #8
-# CHECK-NEXT: [0,7]     .    . D=========================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .  D========================eeeeeeeeER.   ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #4
-# CHECK-NEXT: [0,9]     .    .   D===============================eER   add	x0, x27, #1
+# CHECK-NEXT:                     012345678
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER    .  .   ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #8
+# CHECK-NEXT: [0,1]     .DeE------R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeER  .  .   ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #16
+# CHECK-NEXT: [0,3]     .  DeE------R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeER.  .   ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #4
+# CHECK-NEXT: [0,5]     .    DeE------R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeeER .   ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #8
+# CHECK-NEXT: [0,7]     .    . DeE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeeeER   ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #4
+# CHECK-NEXT: [0,9]     .    .   DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3079,43 +3079,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #8
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #16
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #4
-# CHECK-NEXT: 5.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #8
-# CHECK-NEXT: 7.     1     26.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #4
-# CHECK-NEXT: 9.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.5   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #16
+# CHECK-NEXT: 3.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #4
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #8
+# CHECK-NEXT: 7.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #4
+# CHECK-NEXT: 9.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    3.0       <total>
 
 # CHECK:      [42] Code Region - G43
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      1009
 # CHECK-NEXT: Total uOps:        5000
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    1.25
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    4.96
+# CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 10.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: [0,1]     .D=======eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D======eeeeeeeeER .    .    .    .    .  .   ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: [0,3]     .  D=============eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D============eeeeeeeeER   .    .    .  .   ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: [0,5]     .    D===================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    .D==================eeeeeeeeER.    .  .   ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: [0,7]     .    . D=========================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .  D========================eeeeeeeeER.   ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: [0,9]     .    .   D===============================eER   add	x0, x27, #1
+# CHECK-NEXT:                     012345678
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER    .  .   ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE------R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeER  .  .   ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE------R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeER.  .   ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,5]     .    DeE------R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeeER .   ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeeeER   ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3125,43 +3125,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: 5.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: 7.     1     26.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: 9.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.5   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    3.0       <total>
 
 # CHECK:      [43] Code Region - G44
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3404
+# CHECK-NEXT: Total Cycles:      808
 # CHECK-NEXT: Total uOps:        3800
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    1.12
-# CHECK-NEXT: IPC:               0.29
+# CHECK-NEXT: uOps Per Cycle:    4.70
+# CHECK-NEXT: IPC:               1.24
 # CHECK-NEXT: Block RThroughput: 8.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          01234567
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     012345
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    . .   ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: [0,1]     .D=======eER   .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D======eeeeeeeeER .    .    .    . .   ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: [0,3]     .  D=============eER.    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D============eeeeeeeeER   .    . .   ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: [0,5]     .    D===================eER  .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    D===================eeeeeER   . .   ldp	s1, s2, [x27], #248
-# CHECK-NEXT: [0,7]     .    D========================eER  . .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .D=======================eeeeeER.   ldp	d1, d2, [x27], #496
-# CHECK-NEXT: [0,9]     .    .D============================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeeER    .   ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE------R    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeER  .   ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeER.   ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,5]     .    DeE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    D=eeeeeE-R.   ldp	s1, s2, [x27], #248
+# CHECK-NEXT: [0,7]     .    D==eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .D==eeeeeER   ldp	d1, d2, [x27], #496
+# CHECK-NEXT: [0,9]     .    .D===eE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3171,43 +3171,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: 5.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     20.0   0.0    0.0       ldp	s1, s2, [x27], #248
-# CHECK-NEXT: 7.     1     25.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     24.0   0.0    0.0       ldp	d1, d2, [x27], #496
-# CHECK-NEXT: 9.     1     29.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.1   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    1.0    1.0       ldp	s1, s2, [x27], #248
+# CHECK-NEXT: 7.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    1.0    0.0       ldp	d1, d2, [x27], #496
+# CHECK-NEXT: 9.     1     4.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.8    0.5    2.6       <total>
 
 # CHECK:      [44] Code Region - G45
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2506
+# CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    0.80
-# CHECK-NEXT: IPC:               0.40
+# CHECK-NEXT: uOps Per Cycle:    3.94
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .   ldp	q1, q2, [x27], #992
-# CHECK-NEXT: [0,1]     D=======eER    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=======eeeeeER.    .    .    .   ldp	s1, s2, [x27, #248]!
-# CHECK-NEXT: [0,3]     D============eER    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===========eeeeeER.    .    .   ldp	d1, d2, [x27, #496]!
-# CHECK-NEXT: [0,5]     .D================eER    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D================eeeeeeeER   .   ldp	q1, q2, [x27, #992]!
-# CHECK-NEXT: [0,7]     .D=======================eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D======================eeeeER   ldp	w1, w2, [x27], #248
-# CHECK-NEXT: [0,9]     . D=======================eE--R   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeER. .   ldp	q1, q2, [x27], #992
+# CHECK-NEXT: [0,1]     D=eE-----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeE-R. .   ldp	s1, s2, [x27, #248]!
+# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeER. .   ldp	d1, d2, [x27, #496]!
+# CHECK-NEXT: [0,5]     .D==eE---R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeeER   ldp	q1, q2, [x27, #992]!
+# CHECK-NEXT: [0,7]     .D===eE-----R   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeE--R   ldp	w1, w2, [x27], #248
+# CHECK-NEXT: [0,9]     . D===eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3217,43 +3217,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldp	q1, q2, [x27], #992
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ldp	s1, s2, [x27, #248]!
-# CHECK-NEXT: 3.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     12.0   0.0    0.0       ldp	d1, d2, [x27, #496]!
-# CHECK-NEXT: 5.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     17.0   0.0    0.0       ldp	q1, q2, [x27, #992]!
-# CHECK-NEXT: 7.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     23.0   0.0    0.0       ldp	w1, w2, [x27], #248
-# CHECK-NEXT: 9.     1     24.0   0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT:        1     14.7   0.1    0.2       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    1.0       ldp	s1, s2, [x27, #248]!
+# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ldp	d1, d2, [x27, #496]!
+# CHECK-NEXT: 5.     1     3.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ldp	q1, q2, [x27, #992]!
+# CHECK-NEXT: 7.     1     4.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    2.0       ldp	w1, w2, [x27], #248
+# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    2.4       <total>
 
 # CHECK:      [45] Code Region - G46
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1304
+# CHECK-NEXT: Total Cycles:      507
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    1.53
-# CHECK-NEXT: IPC:               0.77
+# CHECK-NEXT: uOps Per Cycle:    3.94
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456
+# CHECK-NEXT:                     01
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeER   .    ..   ldp	x1, x2, [x27], #496
-# CHECK-NEXT: [0,1]     D=eE--R   .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeER  .    ..   ldp	w1, w2, [x27, #248]!
-# CHECK-NEXT: [0,3]     D==eE--R  .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeER .    ..   ldp	x1, x2, [x27, #496]!
-# CHECK-NEXT: [0,5]     .D==eE--R .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeER    ..   ldpsw	x1, x2, [x27], #248
-# CHECK-NEXT: [0,7]     .D=======eER   ..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D======eeeeeER.   ldpsw	x1, x2, [x27, #248]!
-# CHECK-NEXT: [0,9]     . D===========eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeER   ..   ldp	x1, x2, [x27], #496
+# CHECK-NEXT: [0,1]     D=eE--R   ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeER  ..   ldp	w1, w2, [x27, #248]!
+# CHECK-NEXT: [0,3]     D==eE--R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeER ..   ldp	x1, x2, [x27, #496]!
+# CHECK-NEXT: [0,5]     .D==eE--R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeER.   ldpsw	x1, x2, [x27], #248
+# CHECK-NEXT: [0,7]     .D===eE---R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeER   ldpsw	x1, x2, [x27, #248]!
+# CHECK-NEXT: [0,9]     . D===eE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3269,37 +3269,37 @@ add x0, x27, 1
 # CHECK-NEXT: 4.     1     2.0    0.0    0.0       ldp	x1, x2, [x27, #496]!
 # CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     3.0    0.0    0.0       ldpsw	x1, x2, [x27], #248
-# CHECK-NEXT: 7.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     7.0    0.0    0.0       ldpsw	x1, x2, [x27, #248]!
-# CHECK-NEXT: 9.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     4.3    0.1    0.6       <total>
+# CHECK-NEXT: 7.     1     4.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ldpsw	x1, x2, [x27, #248]!
+# CHECK-NEXT: 9.     1     4.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    1.2       <total>
 
 # CHECK:      [46] Code Region - G47
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2504
+# CHECK-NEXT: Total Cycles:      507
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    0.60
-# CHECK-NEXT: IPC:               0.40
+# CHECK-NEXT: uOps Per Cycle:    2.96
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 3.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          012345678
-
-# CHECK:      [0,0]     DeeeeeER  .    .    .    .  .   ldr	b1, [x27], #254
-# CHECK-NEXT: [0,1]     D=====eER .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=====eeeeeER  .    .    .  .   ldr	h1, [x27], #254
-# CHECK-NEXT: [0,3]     D==========eER .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D==========eeeeeER  .    .  .   ldr	s1, [x27], #254
-# CHECK-NEXT: [0,5]     .D==============eER .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==============eeeeeER  .  .   ldr	d1, [x27], #254
-# CHECK-NEXT: [0,7]     .D===================eER .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===================eeeeeER.   ldr	q1, [x27], #254
-# CHECK-NEXT: [0,9]     .D========================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeER  ..   ldr	b1, [x27], #254
+# CHECK-NEXT: [0,1]     D=eE---R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeER ..   ldr	h1, [x27], #254
+# CHECK-NEXT: [0,3]     D==eE---R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeeER..   ldr	s1, [x27], #254
+# CHECK-NEXT: [0,5]     .D==eE---R..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeER.   ldr	d1, [x27], #254
+# CHECK-NEXT: [0,7]     .D===eE---R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeeeeER   ldr	q1, [x27], #254
+# CHECK-NEXT: [0,9]     .D====eE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3309,43 +3309,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldr	b1, [x27], #254
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     6.0    0.0    0.0       ldr	h1, [x27], #254
-# CHECK-NEXT: 3.     1     11.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     11.0   0.0    0.0       ldr	s1, [x27], #254
-# CHECK-NEXT: 5.     1     15.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     15.0   0.0    0.0       ldr	d1, [x27], #254
-# CHECK-NEXT: 7.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     20.0   0.0    0.0       ldr	q1, [x27], #254
-# CHECK-NEXT: 9.     1     25.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     13.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ldr	h1, [x27], #254
+# CHECK-NEXT: 3.     1     3.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ldr	s1, [x27], #254
+# CHECK-NEXT: 5.     1     3.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ldr	d1, [x27], #254
+# CHECK-NEXT: 7.     1     4.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ldr	q1, [x27], #254
+# CHECK-NEXT: 9.     1     5.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.0    0.1    1.5       <total>
 
 # CHECK:      [47] Code Region - G48
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2504
+# CHECK-NEXT: Total Cycles:      507
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    0.60
-# CHECK-NEXT: IPC:               0.40
+# CHECK-NEXT: uOps Per Cycle:    2.96
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 3.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          012345678
-
-# CHECK:      [0,0]     DeeeeeER  .    .    .    .  .   ldr	b1, [x27, #254]!
-# CHECK-NEXT: [0,1]     D=====eER .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=====eeeeeER  .    .    .  .   ldr	h1, [x27, #254]!
-# CHECK-NEXT: [0,3]     D==========eER .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D==========eeeeeER  .    .  .   ldr	s1, [x27, #254]!
-# CHECK-NEXT: [0,5]     .D==============eER .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==============eeeeeER  .  .   ldr	d1, [x27, #254]!
-# CHECK-NEXT: [0,7]     .D===================eER .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===================eeeeeER.   ldr	q1, [x27, #254]!
-# CHECK-NEXT: [0,9]     .D========================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeER  ..   ldr	b1, [x27, #254]!
+# CHECK-NEXT: [0,1]     D=eE---R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeER ..   ldr	h1, [x27, #254]!
+# CHECK-NEXT: [0,3]     D==eE---R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeeER..   ldr	s1, [x27, #254]!
+# CHECK-NEXT: [0,5]     .D==eE---R..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeER.   ldr	d1, [x27, #254]!
+# CHECK-NEXT: [0,7]     .D===eE---R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeeeeER   ldr	q1, [x27, #254]!
+# CHECK-NEXT: [0,9]     .D====eE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3355,16 +3355,16 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldr	b1, [x27, #254]!
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     6.0    0.0    0.0       ldr	h1, [x27, #254]!
-# CHECK-NEXT: 3.     1     11.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     11.0   0.0    0.0       ldr	s1, [x27, #254]!
-# CHECK-NEXT: 5.     1     15.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     15.0   0.0    0.0       ldr	d1, [x27, #254]!
-# CHECK-NEXT: 7.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     20.0   0.0    0.0       ldr	q1, [x27, #254]!
-# CHECK-NEXT: 9.     1     25.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     13.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ldr	h1, [x27, #254]!
+# CHECK-NEXT: 3.     1     3.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ldr	s1, [x27, #254]!
+# CHECK-NEXT: 5.     1     3.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ldr	d1, [x27, #254]!
+# CHECK-NEXT: 7.     1     4.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ldr	q1, [x27, #254]!
+# CHECK-NEXT: 9.     1     5.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.0    0.1    1.5       <total>
 
 # CHECK:      [48] Code Region - G49
 
@@ -3508,28 +3508,27 @@ add x0, x27, 1
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      704
+# CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        1700
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    2.41
-# CHECK-NEXT: IPC:               1.42
+# CHECK-NEXT: uOps Per Cycle:    3.37
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 3.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     012345678
 
-# CHECK:      [0,0]     DeeeeER   .   ldrsh	x1, [x27, #254]!
-# CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeER  .   ldrsw	x1, [x27], #254
-# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D==eeeeER .   ldrsw	x1, [x27, #254]!
-# CHECK-NEXT: [0,5]     .D==eE--R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeE-R .   st1	{ v1.1d }, [x27], #8
-# CHECK-NEXT: [0,7]     .D====eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D====eeER.   st1	{ v1.2d }, [x27], #16
-# CHECK-NEXT: [0,9]     . D=====eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeER .   ldrsh	x1, [x27, #254]!
+# CHECK-NEXT: [0,1]     D=eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeER.   ldrsw	x1, [x27], #254
+# CHECK-NEXT: [0,3]     D==eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeER   ldrsw	x1, [x27, #254]!
+# CHECK-NEXT: [0,5]     .D==eE--R   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeE-R   st1	{ v1.1d }, [x27], #8
+# CHECK-NEXT: [0,7]     .D===eE-R   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeER   st1	{ v1.2d }, [x27], #16
+# CHECK-NEXT: [0,9]     . D===eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3545,37 +3544,36 @@ add x0, x27, 1
 # CHECK-NEXT: 4.     1     3.0    0.0    0.0       ldrsw	x1, [x27, #254]!
 # CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     3.0    0.0    1.0       st1	{ v1.1d }, [x27], #8
-# CHECK-NEXT: 7.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     5.0    0.0    0.0       st1	{ v1.2d }, [x27], #16
-# CHECK-NEXT: 9.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.3    0.1    0.7       <total>
+# CHECK-NEXT: 7.     1     4.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       st1	{ v1.2d }, [x27], #16
+# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.9    0.1    0.8       <total>
 
 # CHECK:      [52] Code Region - G53
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1004
+# CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    1.99
-# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: uOps Per Cycle:    3.97
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 3.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     012345678
 
-# CHECK:      [0,0]     DeeER.    .  .   st1	{ v1.2s }, [x27], #8
-# CHECK-NEXT: [0,1]     D==eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D==eeER   .  .   st1	{ v1.4h }, [x27], #8
-# CHECK-NEXT: [0,3]     D====eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===eeER .  .   st1	{ v1.4s }, [x27], #16
-# CHECK-NEXT: [0,5]     .D=====eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=====eeER  .   st1	{ v1.8b }, [x27], #8
-# CHECK-NEXT: [0,7]     .D=======eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D======eeER.   st1	{ v1.8h }, [x27], #16
-# CHECK-NEXT: [0,9]     . D========eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.  .   st1	{ v1.2s }, [x27], #8
+# CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeER  .   st1	{ v1.4h }, [x27], #8
+# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeER .   st1	{ v1.4s }, [x27], #16
+# CHECK-NEXT: [0,5]     .D==eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeER.   st1	{ v1.8b }, [x27], #8
+# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeER   st1	{ v1.8h }, [x27], #16
+# CHECK-NEXT: [0,9]     . D===eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3585,43 +3583,42 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.2s }, [x27], #8
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st1	{ v1.4h }, [x27], #8
-# CHECK-NEXT: 3.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       st1	{ v1.4s }, [x27], #16
-# CHECK-NEXT: 5.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     6.0    0.0    0.0       st1	{ v1.8b }, [x27], #8
-# CHECK-NEXT: 7.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     7.0    0.0    0.0       st1	{ v1.8h }, [x27], #16
-# CHECK-NEXT: 9.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     5.2    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.4h }, [x27], #8
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st1	{ v1.4s }, [x27], #16
+# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st1	{ v1.8b }, [x27], #8
+# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st1	{ v1.8h }, [x27], #16
+# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    0.0       <total>
 
 # CHECK:      [53] Code Region - G54
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1004
+# CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    1.99
-# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: uOps Per Cycle:    3.97
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 3.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     012345678
 
-# CHECK:      [0,0]     DeeER.    .  .   st1	{ v1.16b }, [x27], #16
-# CHECK-NEXT: [0,1]     D==eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D==eeER   .  .   st1	{ v1.1d }, [x27], x28
-# CHECK-NEXT: [0,3]     D====eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===eeER .  .   st1	{ v1.2d }, [x27], x28
-# CHECK-NEXT: [0,5]     .D=====eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=====eeER  .   st1	{ v1.2s }, [x27], x28
-# CHECK-NEXT: [0,7]     .D=======eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D======eeER.   st1	{ v1.4h }, [x27], x28
-# CHECK-NEXT: [0,9]     . D========eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.  .   st1	{ v1.16b }, [x27], #16
+# CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeER  .   st1	{ v1.1d }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeER .   st1	{ v1.2d }, [x27], x28
+# CHECK-NEXT: [0,5]     .D==eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeER.   st1	{ v1.2s }, [x27], x28
+# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeER   st1	{ v1.4h }, [x27], x28
+# CHECK-NEXT: [0,9]     . D===eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3631,43 +3628,42 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.16b }, [x27], #16
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st1	{ v1.1d }, [x27], x28
-# CHECK-NEXT: 3.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       st1	{ v1.2d }, [x27], x28
-# CHECK-NEXT: 5.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     6.0    0.0    0.0       st1	{ v1.2s }, [x27], x28
-# CHECK-NEXT: 7.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     7.0    0.0    0.0       st1	{ v1.4h }, [x27], x28
-# CHECK-NEXT: 9.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     5.2    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.1d }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st1	{ v1.2d }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st1	{ v1.2s }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st1	{ v1.4h }, [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    0.0       <total>
 
 # CHECK:      [54] Code Region - G55
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1004
+# CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2100
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    2.09
-# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: uOps Per Cycle:    4.17
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 3.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     012345678
 
-# CHECK:      [0,0]     DeeER.    .  .   st1	{ v1.4s }, [x27], x28
-# CHECK-NEXT: [0,1]     D==eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D==eeER   .  .   st1	{ v1.8b }, [x27], x28
-# CHECK-NEXT: [0,3]     D====eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===eeER .  .   st1	{ v1.8h }, [x27], x28
-# CHECK-NEXT: [0,5]     .D=====eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=====eeER  .   st1	{ v1.16b }, [x27], x28
-# CHECK-NEXT: [0,7]     .D=======eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D======eeER.   st1	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: [0,9]     . D========eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.  .   st1	{ v1.4s }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeER  .   st1	{ v1.8b }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeER .   st1	{ v1.8h }, [x27], x28
+# CHECK-NEXT: [0,5]     .D==eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeER.   st1	{ v1.16b }, [x27], x28
+# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeER   st1	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: [0,9]     . D===eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3677,43 +3673,42 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.4s }, [x27], x28
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st1	{ v1.8b }, [x27], x28
-# CHECK-NEXT: 3.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       st1	{ v1.8h }, [x27], x28
-# CHECK-NEXT: 5.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     6.0    0.0    0.0       st1	{ v1.16b }, [x27], x28
-# CHECK-NEXT: 7.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     7.0    0.0    0.0       st1	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: 9.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     5.2    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.8b }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st1	{ v1.8h }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st1	{ v1.16b }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st1	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    0.0       <total>
 
 # CHECK:      [55] Code Region - G56
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1004
+# CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2700
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    2.69
-# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: uOps Per Cycle:    5.36
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     012345678
 
-# CHECK:      [0,0]     DeeER.    .  .   st1	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: [0,1]     D==eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=eeER   .  .   st1	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: [0,3]     .D===eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==eeER .  .   st1	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: [0,5]     . D====eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D===eeER  .   st1	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: [0,7]     .  D=====eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D====eeER.   st1	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: [0,9]     .   D======eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.  .   st1	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeER  .   st1	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: [0,3]     .D=eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeER .   st1	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: [0,5]     . D=eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeER.   st1	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: [0,7]     .  D=eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeER   st1	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: [0,9]     .   D=eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3723,43 +3718,42 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: 5.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     4.0    0.0    0.0       st1	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: 7.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     5.0    0.0    0.0       st1	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: 9.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     4.0    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st1	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st1	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: 7.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       st1	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: 9.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    0.0       <total>
 
 # CHECK:      [56] Code Region - G57
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1004
+# CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2800
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    2.79
-# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: uOps Per Cycle:    5.56
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     012345678
 
-# CHECK:      [0,0]     DeeER.    .  .   st1	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: [0,1]     D==eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=eeER   .  .   st1	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: [0,3]     .D===eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==eeER .  .   st1	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: [0,5]     . D====eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D===eeER  .   st1	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D=====eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D====eeER.   st1	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D======eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.  .   st1	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeER  .   st1	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: [0,3]     .D=eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeER .   st1	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: [0,5]     . D=eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeER.   st1	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D=eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeER   st1	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D=eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3769,43 +3763,42 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: 5.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     4.0    0.0    0.0       st1	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: 7.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     5.0    0.0    0.0       st1	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: 9.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     4.0    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st1	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st1	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: 7.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       st1	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: 9.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    0.0       <total>
 
 # CHECK:      [57] Code Region - G58
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1004
+# CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2800
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    2.79
-# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: uOps Per Cycle:    5.56
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     012345678
 
-# CHECK:      [0,0]     DeeER.    .  .   st1	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: [0,1]     D==eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=eeER   .  .   st1	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: [0,3]     .D===eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==eeER .  .   st1	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: [0,5]     . D====eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D===eeER  .   st1	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D=====eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D====eeER.   st1	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D======eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.  .   st1	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeER  .   st1	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeER .   st1	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,5]     . D=eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeER.   st1	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D=eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeER   st1	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D=eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3815,43 +3808,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: 5.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     4.0    0.0    0.0       st1	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: 7.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     5.0    0.0    0.0       st1	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: 9.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     4.0    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st1	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st1	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: 7.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       st1	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: 9.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    0.0       <total>
 
 # CHECK:      [58] Code Region - G59
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1704
+# CHECK-NEXT: Total Cycles:      755
 # CHECK-NEXT: Total uOps:        3700
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    2.17
-# CHECK-NEXT: IPC:               0.59
+# CHECK-NEXT: uOps Per Cycle:    4.90
+# CHECK-NEXT: IPC:               1.32
 # CHECK-NEXT: Block RThroughput: 7.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeER    .    .    .   st1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: [0,1]     D===eER   .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D==eeeeER.    .    .   st1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: [0,3]     .D======eER    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=====eeeER  .    .   st1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: [0,5]     . D========eER .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=======eeeER    .   st1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: [0,7]     .  D==========eER   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D=========eeeeER.   st1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: [0,9]     .   D=============eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeER    . .   st1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
+# CHECK-NEXT: [0,1]     D=eE-R    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeER  . .   st1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: [0,3]     .D=eE--R  . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeeER . .   st1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: [0,5]     . D==eE-R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeeER. .   st1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: [0,7]     .  D==eE-R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D==eeeeER   st1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: [0,9]     .   D===eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3861,43 +3854,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: 1.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: 3.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     6.0    0.0    0.0       st1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: 5.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     8.0    0.0    0.0       st1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: 7.     1     11.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     10.0   0.0    0.0       st1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: 9.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     7.3    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: 3.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: 5.     1     3.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: 7.     1     3.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    1.0    0.0       st1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: 9.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.3    0.7       <total>
 
 # CHECK:      [59] Code Region - G60
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1804
+# CHECK-NEXT: Total Cycles:      755
 # CHECK-NEXT: Total uOps:        3800
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    2.11
-# CHECK-NEXT: IPC:               0.55
+# CHECK-NEXT: uOps Per Cycle:    5.03
+# CHECK-NEXT: IPC:               1.32
 # CHECK-NEXT: Block RThroughput: 7.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          01
-
-# CHECK:      [0,0]     DeeeER    .    .    ..   st1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: [0,1]     D===eER   .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D==eeeeER.    .    ..   st1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: [0,3]     .D======eER    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=====eeeeER .    ..   st1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: [0,5]     . D=========eER.    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D========eeeER   ..   st1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D===========eER  ..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D==========eeeeER.   st1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D==============eER   add	x0, x27, #1
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeER    . .   st1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: [0,1]     D=eE-R    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeER  . .   st1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: [0,3]     .D=eE--R  . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeeeER. .   st1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: [0,5]     . D==eE--R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeeER. .   st1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D==eE-R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D==eeeeER   st1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D===eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3907,43 +3900,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: 1.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: 3.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     6.0    0.0    0.0       st1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: 5.     1     10.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     9.0    0.0    0.0       st1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: 7.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     11.0   0.0    0.0       st1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: 9.     1     15.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     7.8    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: 3.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: 7.     1     3.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    1.0    0.0       st1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.3    0.8       <total>
 
 # CHECK:      [60] Code Region - G61
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1704
+# CHECK-NEXT: Total Cycles:      755
 # CHECK-NEXT: Total uOps:        3700
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    2.17
-# CHECK-NEXT: IPC:               0.59
+# CHECK-NEXT: uOps Per Cycle:    4.90
+# CHECK-NEXT: IPC:               1.32
 # CHECK-NEXT: Block RThroughput: 7.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeER    .    .    .   st1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: [0,1]     D===eER   .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D==eeeER .    .    .   st1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=====eER.    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D====eeeeER  .    .   st1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: [0,5]     . D========eER .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=======eeeER    .   st1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D==========eER   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D=========eeeeER.   st1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D=============eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeER    . .   st1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE-R    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeER   . .   st1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE-R   . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeeeER. .   st1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: [0,5]     . D==eE--R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeeER. .   st1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D==eE-R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D==eeeeER   st1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D===eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3953,43 +3946,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: 1.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: 3.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       st1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: 5.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     8.0    0.0    0.0       st1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: 7.     1     11.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     10.0   0.0    0.0       st1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: 9.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     7.1    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: 7.     1     3.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    1.0    0.0       st1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.3    0.7       <total>
 
 # CHECK:      [61] Code Region - G62
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1504
+# CHECK-NEXT: Total Cycles:      704
 # CHECK-NEXT: Total uOps:        3600
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    2.39
-# CHECK-NEXT: IPC:               0.66
+# CHECK-NEXT: uOps Per Cycle:    5.11
+# CHECK-NEXT: IPC:               1.42
 # CHECK-NEXT: Block RThroughput: 6.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     012345678
+# CHECK-NEXT:                     0
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeER   .    .  .   st1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: [0,1]     D====eER  .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D===eeER .    .  .   st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: [0,3]     .D=====eER.    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D====eeeeeER .  .   st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: [0,5]     .  D========eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D========eeER  .   st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: [0,7]     .  D==========eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D=========eeER.   st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: [0,9]     .   D===========eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeER   .   st1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeE-R   .   st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: [0,3]     .D=eE-R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeeeeER   st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: [0,5]     .  D=eE---R   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeE--R   st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: [0,7]     .  D==eE--R   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D==eeER   st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: [0,9]     .   D===eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3999,43 +3992,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: 1.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     4.0    0.0    0.0       st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: 3.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: 5.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     9.0    0.0    0.0       st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: 7.     1     11.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     10.0   0.0    0.0       st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: 9.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     7.2    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    1.0       st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: 3.     1     2.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: 5.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    2.0       st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: 7.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    1.0    0.0       st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.2    0.3    1.1       <total>
 
 # CHECK:      [62] Code Region - G63
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1904
+# CHECK-NEXT: Total Cycles:      804
 # CHECK-NEXT: Total uOps:        4200
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    2.21
-# CHECK-NEXT: IPC:               0.53
+# CHECK-NEXT: uOps Per Cycle:    5.22
+# CHECK-NEXT: IPC:               1.24
 # CHECK-NEXT: Block RThroughput: 8.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          012
-
-# CHECK:      [0,0]     DeeeeeER  .    .    . .   st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: [0,1]     .D====eER .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D====eeER.    .    . .   st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: [0,3]     .D======eER    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=====eeeeeER.    . .   st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: [0,5]     .  D=========eER    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .   D========eeeeeER. .   st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: [0,7]     .    D============eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    D============eeER.   st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: [0,9]     .    D==============eER   add	x0, x27, #1
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeER  ..   st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: [0,1]     .DeE---R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeE--R  ..   st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: [0,3]     .D=eE--R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeeeeER.   st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: [0,5]     .  D=eE---R.   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .   DeeeeeER   st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: [0,7]     .    DeE---R   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    D==eeER   st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK-NEXT: [0,9]     .    D===eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4045,43 +4038,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: 1.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: 3.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     6.0    0.0    0.0       st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: 5.     1     10.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     9.0    0.0    0.0       st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: 7.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     13.0   0.0    0.0       st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: 9.     1     15.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     8.4    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    2.0       st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: 3.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: 5.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: 7.     1     1.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    2.0    0.0       st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.8    0.4    1.3       <total>
 
 # CHECK:      [63] Code Region - G64
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1604
+# CHECK-NEXT: Total Cycles:      705
 # CHECK-NEXT: Total uOps:        3800
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    2.37
-# CHECK-NEXT: IPC:               0.62
+# CHECK-NEXT: uOps Per Cycle:    5.39
+# CHECK-NEXT: IPC:               1.42
 # CHECK-NEXT: Block RThroughput: 7.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
+# CHECK-NEXT:                     01
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeER  .    .   .   st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: [0,1]     .D====eER .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D====eeER.    .   .   st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: [0,3]     .D======eER    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=====eeER   .   .   st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: [0,5]     . D=======eER  .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D======eeeeeER  .   st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: [0,7]     .   D==========eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D==========eeER.   st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D============eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeER  ..   st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE---R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeE--R  ..   st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE--R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeER  ..   st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,5]     . D==eER  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeeeeER   st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,7]     .   D=eE---R   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D=eeE--R   st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D==eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4091,43 +4084,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: 1.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: 3.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     6.0    0.0    0.0       st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: 5.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     7.0    0.0    0.0       st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: 7.     1     11.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     11.0   0.0    0.0       st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: 9.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     7.4    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    2.0       st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: 7.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     2.0    0.0    2.0       st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: 9.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.9    0.2    1.4       <total>
 
 # CHECK:      [64] Code Region - G65
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2204
+# CHECK-NEXT: Total Cycles:      706
 # CHECK-NEXT: Total uOps:        3200
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    1.45
-# CHECK-NEXT: IPC:               0.45
+# CHECK-NEXT: uOps Per Cycle:    4.53
+# CHECK-NEXT: IPC:               1.42
 # CHECK-NEXT: Block RThroughput: 5.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          012345
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeER  .    .    .    .   st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: [0,1]     .D====eER .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D===eeeeeER  .    .    .   st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: [0,3]     .  D=======eER .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .  D=======eeeeER   .    .   st1	{ v1.b }[0], [x27], #1
-# CHECK-NEXT: [0,5]     .  D===========eER  .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .   D==========eeeeER    .   st1	{ v1.b }[8], [x27], #1
-# CHECK-NEXT: [0,7]     .   D==============eER   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D==============eeeeER.   st1	{ v1.b }[0], [x27], x28
-# CHECK-NEXT: [0,9]     .   D==================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeER  . .   st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE---R  . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeER. .   st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE---R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .  D=eeeeER .   st1	{ v1.b }[0], [x27], #1
+# CHECK-NEXT: [0,5]     .  D==eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .   D=eeeeER.   st1	{ v1.b }[8], [x27], #1
+# CHECK-NEXT: [0,7]     .   D==eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D==eeeeER   st1	{ v1.b }[0], [x27], x28
+# CHECK-NEXT: [0,9]     .   D===eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4137,43 +4130,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: 1.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     4.0    0.0    0.0       st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: 3.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     8.0    0.0    0.0       st1	{ v1.b }[0], [x27], #1
-# CHECK-NEXT: 5.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     11.0   0.0    0.0       st1	{ v1.b }[8], [x27], #1
-# CHECK-NEXT: 7.     1     15.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     15.0   0.0    0.0       st1	{ v1.b }[0], [x27], x28
-# CHECK-NEXT: 9.     1     19.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     9.8    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st1	{ v1.b }[0], [x27], #1
+# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st1	{ v1.b }[8], [x27], #1
+# CHECK-NEXT: 7.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st1	{ v1.b }[0], [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.1    0.3    1.2       <total>
 
 # CHECK:      [65] Code Region - G66
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2004
+# CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    1.00
-# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: uOps Per Cycle:    3.95
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 3.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0123
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeER   .    .    .  .   st1	{ v1.b }[8], [x27], x28
-# CHECK-NEXT: [0,1]     D====eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D====eeeeER    .    .  .   st1	{ v1.h }[0], [x27], #2
-# CHECK-NEXT: [0,3]     D========eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=======eeeeER.    .  .   st1	{ v1.h }[4], [x27], #2
-# CHECK-NEXT: [0,5]     .D===========eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D===========eeeeER .  .   st1	{ v1.h }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .D===============eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============eeeeER.   st1	{ v1.h }[4], [x27], x28
-# CHECK-NEXT: [0,9]     . D==================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeER   .   st1	{ v1.b }[8], [x27], x28
+# CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeER  .   st1	{ v1.h }[0], [x27], #2
+# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeER .   st1	{ v1.h }[4], [x27], #2
+# CHECK-NEXT: [0,5]     .D==eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeER.   st1	{ v1.h }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .D===eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeER   st1	{ v1.h }[4], [x27], x28
+# CHECK-NEXT: [0,9]     . D===eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4183,43 +4176,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.b }[8], [x27], x28
-# CHECK-NEXT: 1.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       st1	{ v1.h }[0], [x27], #2
-# CHECK-NEXT: 3.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     8.0    0.0    0.0       st1	{ v1.h }[4], [x27], #2
-# CHECK-NEXT: 5.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     12.0   0.0    0.0       st1	{ v1.h }[0], [x27], x28
-# CHECK-NEXT: 7.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     15.0   0.0    0.0       st1	{ v1.h }[4], [x27], x28
-# CHECK-NEXT: 9.     1     19.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     10.2   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.h }[0], [x27], #2
+# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st1	{ v1.h }[4], [x27], #2
+# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st1	{ v1.h }[0], [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st1	{ v1.h }[4], [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    1.0       <total>
 
 # CHECK:      [66] Code Region - G67
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2104
+# CHECK-NEXT: Total Cycles:      507
 # CHECK-NEXT: Total uOps:        2200
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    1.05
-# CHECK-NEXT: IPC:               0.48
+# CHECK-NEXT: uOps Per Cycle:    4.34
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 3.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          01234
-
-# CHECK:      [0,0]     DeeeeER   .    .    .   .   st1	{ v1.s }[0], [x27], #4
-# CHECK-NEXT: [0,1]     D====eER  .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D====eeeeER    .    .   .   st1	{ v1.s }[0], [x27], x28
-# CHECK-NEXT: [0,3]     D========eER   .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=======eeeeER.    .   .   st1	{ v1.d }[0], [x27], #8
-# CHECK-NEXT: [0,5]     .D===========eER    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D===========eeeeER .   .   st1	{ v1.d }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .D===============eER.   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============eeeeeER.   st2	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: [0,9]     . D===================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeER   ..   st1	{ v1.s }[0], [x27], #4
+# CHECK-NEXT: [0,1]     D=eE--R   ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeER  ..   st1	{ v1.s }[0], [x27], x28
+# CHECK-NEXT: [0,3]     D==eE--R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeER ..   st1	{ v1.d }[0], [x27], #8
+# CHECK-NEXT: [0,5]     .D==eE--R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeER..   st1	{ v1.d }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .D===eE--R..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeER   st2	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: [0,9]     . D===eE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4229,43 +4222,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.s }[0], [x27], #4
-# CHECK-NEXT: 1.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       st1	{ v1.s }[0], [x27], x28
-# CHECK-NEXT: 3.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     8.0    0.0    0.0       st1	{ v1.d }[0], [x27], #8
-# CHECK-NEXT: 5.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     12.0   0.0    0.0       st1	{ v1.d }[0], [x27], x28
-# CHECK-NEXT: 7.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     15.0   0.0    0.0       st2	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: 9.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     10.3   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.s }[0], [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st1	{ v1.d }[0], [x27], #8
+# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st1	{ v1.d }[0], [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st2	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: 9.     1     4.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    1.1       <total>
 
 # CHECK:      [67] Code Region - G68
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2204
+# CHECK-NEXT: Total Cycles:      507
 # CHECK-NEXT: Total uOps:        2400
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    1.09
-# CHECK-NEXT: IPC:               0.45
+# CHECK-NEXT: uOps Per Cycle:    4.73
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 3.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          012345
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeER   .    .    .    .   st2	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: [0,1]     D====eER  .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D====eeeeER    .    .    .   st2	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: [0,3]     D========eER   .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=======eeeeeER    .    .   st2	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: [0,5]     .D============eER   .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     . D===========eeeeER.    .   st2	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: [0,7]     . D===============eER    .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .  D==============eeeeeER.   st2	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: [0,9]     .  D===================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeER   ..   st2	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: [0,1]     D=eE--R   ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeER  ..   st2	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: [0,3]     D==eE--R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeER..   st2	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: [0,5]     .D==eE---R..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     . D=eeeeER..   st2	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: [0,7]     . D==eE--R..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .  D=eeeeeER   st2	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: [0,9]     .  D==eE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4275,43 +4268,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: 1.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       st2	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: 3.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     8.0    0.0    0.0       st2	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: 5.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     12.0   0.0    0.0       st2	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: 7.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     15.0   0.0    0.0       st2	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: 9.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     10.4   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st2	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st2	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: 5.     1     3.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st2	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: 7.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     2.0    0.0    0.0       st2	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: 9.     1     3.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.1    1.2       <total>
 
 # CHECK:      [68] Code Region - G69
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2304
+# CHECK-NEXT: Total Cycles:      507
 # CHECK-NEXT: Total uOps:        2600
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    1.13
-# CHECK-NEXT: IPC:               0.43
+# CHECK-NEXT: uOps Per Cycle:    5.13
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0123456
-
-# CHECK:      [0,0]     DeeeeeER  .    .    .    ..   st2	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: [0,1]     D=====eER .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D====eeeeeER  .    .    ..   st2	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=========eER .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D========eeeeER   .    ..   st2	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: [0,5]     . D============eER  .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     . D============eeeeER    ..   st2	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: [0,7]     . D================eER   ..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .  D===============eeeeeER.   st2	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: [0,9]     .  D====================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeER  ..   st2	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: [0,1]     D=eE---R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeER ..   st2	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE---R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeER ..   st2	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,5]     . D=eE--R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     . D=eeeeER..   st2	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,7]     . D==eE--R..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .  D=eeeeeER   st2	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,9]     .  D==eE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4321,43 +4314,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       st2	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: 3.     1     10.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     9.0    0.0    0.0       st2	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: 5.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     13.0   0.0    0.0       st2	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: 7.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     16.0   0.0    0.0       st2	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: 9.     1     21.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     11.1   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st2	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st2	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st2	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: 7.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     2.0    0.0    0.0       st2	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: 9.     1     3.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.9    0.1    1.3       <total>
 
 # CHECK:      [69] Code Region - G70
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2204
+# CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        2400
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    1.09
-# CHECK-NEXT: IPC:               0.45
+# CHECK-NEXT: uOps Per Cycle:    4.74
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 3.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          012345
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeER   .    .    .    .   st2	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: [0,1]     D====eER  .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D===eeeeeER   .    .    .   st2	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: [0,3]     .D========eER  .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=======eeeeeER   .    .   st2	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: [0,5]     . D============eER  .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D===========eeeeER    .   st2	{ v1.b, v2.b }[0], [x27], #2
-# CHECK-NEXT: [0,7]     .  D===============eER   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .  D===============eeeeER.   st2	{ v1.b, v2.b }[8], [x27], #2
-# CHECK-NEXT: [0,9]     .  D===================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeER   .   st2	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeER .   st2	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE---R .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeER.   st2	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,5]     . D=eE---R.   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeER.   st2	{ v1.b, v2.b }[0], [x27], #2
+# CHECK-NEXT: [0,7]     .  D=eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .  D=eeeeER   st2	{ v1.b, v2.b }[8], [x27], #2
+# CHECK-NEXT: [0,9]     .  D==eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4367,43 +4360,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: 1.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     4.0    0.0    0.0       st2	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: 3.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     8.0    0.0    0.0       st2	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: 5.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     12.0   0.0    0.0       st2	{ v1.b, v2.b }[0], [x27], #2
-# CHECK-NEXT: 7.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     16.0   0.0    0.0       st2	{ v1.b, v2.b }[8], [x27], #2
-# CHECK-NEXT: 9.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     10.4   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st2	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st2	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st2	{ v1.b, v2.b }[0], [x27], #2
+# CHECK-NEXT: 7.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     2.0    0.0    0.0       st2	{ v1.b, v2.b }[8], [x27], #2
+# CHECK-NEXT: 9.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.7    0.1    1.2       <total>
 
 # CHECK:      [70] Code Region - G71
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2004
+# CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    1.00
-# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: uOps Per Cycle:    3.95
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 3.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0123
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeER   .    .    .  .   st2	{ v1.b, v2.b }[0], [x27], x28
-# CHECK-NEXT: [0,1]     D====eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D====eeeeER    .    .  .   st2	{ v1.b, v2.b }[8], [x27], x28
-# CHECK-NEXT: [0,3]     D========eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=======eeeeER.    .  .   st2	{ v1.h, v2.h }[0], [x27], #4
-# CHECK-NEXT: [0,5]     .D===========eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D===========eeeeER .  .   st2	{ v1.h, v2.h }[4], [x27], #4
-# CHECK-NEXT: [0,7]     .D===============eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============eeeeER.   st2	{ v1.h, v2.h }[0], [x27], x28
-# CHECK-NEXT: [0,9]     . D==================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeER   .   st2	{ v1.b, v2.b }[0], [x27], x28
+# CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeER  .   st2	{ v1.b, v2.b }[8], [x27], x28
+# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeER .   st2	{ v1.h, v2.h }[0], [x27], #4
+# CHECK-NEXT: [0,5]     .D==eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeER.   st2	{ v1.h, v2.h }[4], [x27], #4
+# CHECK-NEXT: [0,7]     .D===eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeER   st2	{ v1.h, v2.h }[0], [x27], x28
+# CHECK-NEXT: [0,9]     . D===eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4413,43 +4406,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.b, v2.b }[0], [x27], x28
-# CHECK-NEXT: 1.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       st2	{ v1.b, v2.b }[8], [x27], x28
-# CHECK-NEXT: 3.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     8.0    0.0    0.0       st2	{ v1.h, v2.h }[0], [x27], #4
-# CHECK-NEXT: 5.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     12.0   0.0    0.0       st2	{ v1.h, v2.h }[4], [x27], #4
-# CHECK-NEXT: 7.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     15.0   0.0    0.0       st2	{ v1.h, v2.h }[0], [x27], x28
-# CHECK-NEXT: 9.     1     19.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     10.2   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st2	{ v1.b, v2.b }[8], [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st2	{ v1.h, v2.h }[0], [x27], #4
+# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st2	{ v1.h, v2.h }[4], [x27], #4
+# CHECK-NEXT: 7.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st2	{ v1.h, v2.h }[0], [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    1.0       <total>
 
 # CHECK:      [71] Code Region - G72
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2004
+# CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    1.00
-# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: uOps Per Cycle:    3.95
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 3.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0123
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeER   .    .    .  .   st2	{ v1.h, v2.h }[4], [x27], x28
-# CHECK-NEXT: [0,1]     D====eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D====eeeeER    .    .  .   st2	{ v1.s, v2.s }[0], [x27], #8
-# CHECK-NEXT: [0,3]     D========eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=======eeeeER.    .  .   st2	{ v1.s, v2.s }[0], [x27], x28
-# CHECK-NEXT: [0,5]     .D===========eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D===========eeeeER .  .   st2	{ v1.d, v2.d }[0], [x27], #16
-# CHECK-NEXT: [0,7]     .D===============eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============eeeeER.   st2	{ v1.d, v2.d }[0], [x27], x28
-# CHECK-NEXT: [0,9]     . D==================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeER   .   st2	{ v1.h, v2.h }[4], [x27], x28
+# CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeER  .   st2	{ v1.s, v2.s }[0], [x27], #8
+# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeER .   st2	{ v1.s, v2.s }[0], [x27], x28
+# CHECK-NEXT: [0,5]     .D==eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeER.   st2	{ v1.d, v2.d }[0], [x27], #16
+# CHECK-NEXT: [0,7]     .D===eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeER   st2	{ v1.d, v2.d }[0], [x27], x28
+# CHECK-NEXT: [0,9]     . D===eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4459,39 +4452,39 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.h, v2.h }[4], [x27], x28
-# CHECK-NEXT: 1.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       st2	{ v1.s, v2.s }[0], [x27], #8
-# CHECK-NEXT: 3.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     8.0    0.0    0.0       st2	{ v1.s, v2.s }[0], [x27], x28
-# CHECK-NEXT: 5.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     12.0   0.0    0.0       st2	{ v1.d, v2.d }[0], [x27], #16
-# CHECK-NEXT: 7.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     15.0   0.0    0.0       st2	{ v1.d, v2.d }[0], [x27], x28
-# CHECK-NEXT: 9.     1     19.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     10.2   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st2	{ v1.s, v2.s }[0], [x27], #8
+# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st2	{ v1.s, v2.s }[0], [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st2	{ v1.d, v2.d }[0], [x27], #16
+# CHECK-NEXT: 7.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st2	{ v1.d, v2.d }[0], [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    1.0       <total>
 
 # CHECK:      [72] Code Region - G73
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      600
-# CHECK-NEXT: Total Cycles:      1604
+# CHECK-NEXT: Total Cycles:      407
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    1.25
-# CHECK-NEXT: IPC:               0.37
+# CHECK-NEXT: uOps Per Cycle:    4.91
+# CHECK-NEXT: IPC:               1.47
 # CHECK-NEXT: Block RThroughput: 3.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
+# CHECK-NEXT:                     0
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    .   .   st3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: [0,1]     D======eER.    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=====eeeeeER .   .   st3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: [0,3]     .D==========eER.   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=========eeeeeER.   st3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: [0,5]     . D==============eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER .   st3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: [0,1]     D=eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeER .   st3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: [0,3]     .D=eE---R .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeeeeER   st3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: [0,5]     . D==eE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4501,39 +4494,39 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     6.0    0.0    0.0       st3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: 3.     1     11.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     10.0   0.0    0.0       st3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: 5.     1     15.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     8.3    0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: 3.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: 5.     1     3.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.8    0.3    1.7       <total>
 
 # CHECK:      [73] Code Region - G74
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2904
+# CHECK-NEXT: Total Cycles:      708
 # CHECK-NEXT: Total uOps:        3800
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    1.31
-# CHECK-NEXT: IPC:               0.34
+# CHECK-NEXT: uOps Per Cycle:    5.37
+# CHECK-NEXT: IPC:               1.41
 # CHECK-NEXT: Block RThroughput: 7.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          012
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    . .   st3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=====eeeeeER .    .    .    . .   st3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: [0,3]     .D==========eER.    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=========eeeeeeER.    .    . .   st3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: [0,5]     . D===============eER    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D==============eeeeeeER    . .   st3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: [0,7]     .  D====================eER   . .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D===================eeeeeeER.   st3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D=========================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER .   .   st3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: [0,1]     D=eE----R .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeER .   .   st3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: [0,3]     .D=eE---R .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeeeeeER  .   st3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: [0,5]     . D==eE----R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeeeeeER .   st3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: [0,7]     .  D==eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D==eeeeeeER   st3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D===eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4543,43 +4536,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     6.0    0.0    0.0       st3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: 3.     1     11.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     10.0   0.0    0.0       st3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: 5.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     15.0   0.0    0.0       st3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: 7.     1     21.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     20.0   0.0    0.0       st3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: 9.     1     26.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     13.3   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: 3.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: 7.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    1.0    0.0       st3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.3    1.9       <total>
 
 # CHECK:      [74] Code Region - G75
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2704
+# CHECK-NEXT: Total Cycles:      707
 # CHECK-NEXT: Total uOps:        3400
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    1.26
-# CHECK-NEXT: IPC:               0.37
+# CHECK-NEXT: uOps Per Cycle:    4.81
+# CHECK-NEXT: IPC:               1.41
 # CHECK-NEXT: Block RThroughput: 6.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     0123
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeER  .    .    .    .    .   st3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: [0,1]     D=====eER .    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D====eeeeeER  .    .    .    .   st3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=========eER .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D========eeeeeeER .    .    .   st3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: [0,5]     . D==============eER.    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=============eeeeeER .    .   st3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D==================eER.    .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D=================eeeeeeER.   st3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D=======================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeER  .  .   st3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE---R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeER .  .   st3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE---R .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeER  .   st3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: [0,5]     . D=eE----R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeER  .   st3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D=eE---R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D=eeeeeeER   st3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D==eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4589,43 +4582,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       st3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: 3.     1     10.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     9.0    0.0    0.0       st3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: 5.     1     15.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     14.0   0.0    0.0       st3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: 7.     1     19.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     18.0   0.0    0.0       st3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: 9.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     12.1   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: 7.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     2.0    1.0    0.0       st3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: 9.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.7    0.2    1.7       <total>
 
 # CHECK:      [75] Code Region - G76
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2204
+# CHECK-NEXT: Total Cycles:      755
 # CHECK-NEXT: Total uOps:        4000
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    1.81
-# CHECK-NEXT: IPC:               0.45
+# CHECK-NEXT: uOps Per Cycle:    5.30
+# CHECK-NEXT: IPC:               1.32
 # CHECK-NEXT: Block RThroughput: 7.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          012345
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .   st3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=====eeeeER  .    .    .   st3	{ v1.b, v2.b, v3.b }[0], [x27], #3
-# CHECK-NEXT: [0,3]     .D=========eER .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D========eeeeER   .    .   st3	{ v1.b, v2.b, v3.b }[8], [x27], #3
-# CHECK-NEXT: [0,5]     . D============eER  .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D===========eeeeER    .   st3	{ v1.b, v2.b, v3.b }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .  D===============eER   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D==============eeeeER.   st3	{ v1.b, v2.b, v3.b }[8], [x27], x28
-# CHECK-NEXT: [0,9]     .   D==================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER . .   st3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeE-R . .   st3	{ v1.b, v2.b, v3.b }[0], [x27], #3
+# CHECK-NEXT: [0,3]     .D=eE---R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeeeER. .   st3	{ v1.b, v2.b, v3.b }[8], [x27], #3
+# CHECK-NEXT: [0,5]     . D==eE--R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeeeER .   st3	{ v1.b, v2.b, v3.b }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .  D==eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D==eeeeER   st3	{ v1.b, v2.b, v3.b }[8], [x27], x28
+# CHECK-NEXT: [0,9]     .   D===eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4635,43 +4628,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     6.0    0.0    0.0       st3	{ v1.b, v2.b, v3.b }[0], [x27], #3
-# CHECK-NEXT: 3.     1     10.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     9.0    0.0    0.0       st3	{ v1.b, v2.b, v3.b }[8], [x27], #3
-# CHECK-NEXT: 5.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     12.0   0.0    0.0       st3	{ v1.b, v2.b, v3.b }[0], [x27], x28
-# CHECK-NEXT: 7.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     15.0   0.0    0.0       st3	{ v1.b, v2.b, v3.b }[8], [x27], x28
-# CHECK-NEXT: 9.     1     19.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     10.8   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    1.0       st3	{ v1.b, v2.b, v3.b }[0], [x27], #3
+# CHECK-NEXT: 3.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st3	{ v1.b, v2.b, v3.b }[8], [x27], #3
+# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st3	{ v1.b, v2.b, v3.b }[0], [x27], x28
+# CHECK-NEXT: 7.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    1.0    0.0       st3	{ v1.b, v2.b, v3.b }[8], [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.3    1.4       <total>
 
 # CHECK:      [76] Code Region - G77
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2004
+# CHECK-NEXT: Total Cycles:      755
 # CHECK-NEXT: Total uOps:        4000
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    2.00
-# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: uOps Per Cycle:    5.30
+# CHECK-NEXT: IPC:               1.32
 # CHECK-NEXT: Block RThroughput: 7.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0123
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeER   .    .    .  .   st3	{ v1.h, v2.h, v3.h }[0], [x27], #6
-# CHECK-NEXT: [0,1]     D====eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D===eeeeER    .    .  .   st3	{ v1.h, v2.h, v3.h }[4], [x27], #6
-# CHECK-NEXT: [0,3]     .D=======eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D======eeeeER.    .  .   st3	{ v1.h, v2.h, v3.h }[0], [x27], x28
-# CHECK-NEXT: [0,5]     . D==========eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=========eeeeER .  .   st3	{ v1.h, v2.h, v3.h }[4], [x27], x28
-# CHECK-NEXT: [0,7]     .  D=============eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D============eeeeER.   st3	{ v1.s, v2.s, v3.s }[0], [x27], #12
-# CHECK-NEXT: [0,9]     .   D================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeER   . .   st3	{ v1.h, v2.h, v3.h }[0], [x27], #6
+# CHECK-NEXT: [0,1]     D=eE--R   . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeER  . .   st3	{ v1.h, v2.h, v3.h }[4], [x27], #6
+# CHECK-NEXT: [0,3]     .D=eE--R  . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeeeER. .   st3	{ v1.h, v2.h, v3.h }[0], [x27], x28
+# CHECK-NEXT: [0,5]     . D==eE--R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeeeER .   st3	{ v1.h, v2.h, v3.h }[4], [x27], x28
+# CHECK-NEXT: [0,7]     .  D==eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D==eeeeER   st3	{ v1.s, v2.s, v3.s }[0], [x27], #12
+# CHECK-NEXT: [0,9]     .   D===eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4681,43 +4674,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.h, v2.h, v3.h }[0], [x27], #6
-# CHECK-NEXT: 1.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     4.0    0.0    0.0       st3	{ v1.h, v2.h, v3.h }[4], [x27], #6
-# CHECK-NEXT: 3.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     7.0    0.0    0.0       st3	{ v1.h, v2.h, v3.h }[0], [x27], x28
-# CHECK-NEXT: 5.     1     11.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     10.0   0.0    0.0       st3	{ v1.h, v2.h, v3.h }[4], [x27], x28
-# CHECK-NEXT: 7.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     13.0   0.0    0.0       st3	{ v1.s, v2.s, v3.s }[0], [x27], #12
-# CHECK-NEXT: 9.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     9.0    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st3	{ v1.h, v2.h, v3.h }[4], [x27], #6
+# CHECK-NEXT: 3.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st3	{ v1.h, v2.h, v3.h }[0], [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st3	{ v1.h, v2.h, v3.h }[4], [x27], x28
+# CHECK-NEXT: 7.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    1.0    0.0       st3	{ v1.s, v2.s, v3.s }[0], [x27], #12
+# CHECK-NEXT: 9.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.3    1.0       <total>
 
 # CHECK:      [77] Code Region - G78
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2704
+# CHECK-NEXT: Total Cycles:      808
 # CHECK-NEXT: Total uOps:        4200
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    1.55
-# CHECK-NEXT: IPC:               0.37
+# CHECK-NEXT: uOps Per Cycle:    5.20
+# CHECK-NEXT: IPC:               1.24
 # CHECK-NEXT: Block RThroughput: 8.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     012345
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeER   .    .    .    .    .   st3	{ v1.s, v2.s, v3.s }[0], [x27], x28
-# CHECK-NEXT: [0,1]     D====eER  .    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D===eeeeeER   .    .    .    .   st3	{ v1.d, v2.d, v3.d }[0], [x27], #24
-# CHECK-NEXT: [0,3]     .D========eER  .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=======eeeeeER   .    .    .   st3	{ v1.d, v2.d, v3.d }[0], [x27], x28
-# CHECK-NEXT: [0,5]     . D============eER  .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D===========eeeeeeER  .    .   st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: [0,7]     .   D================eER .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    D===============eeeeeeeER.   st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: [0,9]     .    D======================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeER   .    .   st3	{ v1.s, v2.s, v3.s }[0], [x27], x28
+# CHECK-NEXT: [0,1]     D=eE--R   .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeER .    .   st3	{ v1.d, v2.d, v3.d }[0], [x27], #24
+# CHECK-NEXT: [0,3]     .D=eE---R .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeeeeER    .   st3	{ v1.d, v2.d, v3.d }[0], [x27], x28
+# CHECK-NEXT: [0,5]     . D==eE---R    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeeeeeER  .   st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: [0,7]     .   D=eE----R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    D=eeeeeeeER   st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: [0,9]     .    D==eE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4727,43 +4720,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.s, v2.s, v3.s }[0], [x27], x28
-# CHECK-NEXT: 1.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     4.0    0.0    0.0       st3	{ v1.d, v2.d, v3.d }[0], [x27], #24
-# CHECK-NEXT: 3.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     8.0    0.0    0.0       st3	{ v1.d, v2.d, v3.d }[0], [x27], x28
-# CHECK-NEXT: 5.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     12.0   0.0    0.0       st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: 7.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     16.0   0.0    0.0       st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: 9.     1     23.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     10.8   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st3	{ v1.d, v2.d, v3.d }[0], [x27], #24
+# CHECK-NEXT: 3.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st3	{ v1.d, v2.d, v3.d }[0], [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: 7.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     2.0    1.0    0.0       st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: 9.     1     3.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.0    0.3    1.7       <total>
 
 # CHECK:      [78] Code Region - G79
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4104
+# CHECK-NEXT: Total Cycles:      1207
 # CHECK-NEXT: Total uOps:        5800
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    1.41
-# CHECK-NEXT: IPC:               0.24
+# CHECK-NEXT: uOps Per Cycle:    4.81
+# CHECK-NEXT: IPC:               0.83
 # CHECK-NEXT: Block RThroughput: 12.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          01234
-
-# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    .    .   .   st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: [0,1]     D=======eER    .    .    .    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D======eeeeeeeeeER .    .    .    .    .   .   st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: [0,3]     . D==============eER.    .    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .  D=============eeeeeeeER    .    .    .   .   st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: [0,5]     .  D====================eER   .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .   D===================eeeeeeeeeER.    .   .   st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: [0,7]     .    D===========================eER    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .D==========================eeeeeeeeeER.   st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: [0,9]     .    . D==================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     012345678
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeER.    .  .   st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: [0,1]     D=eE-----R.    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeeER  .  .   st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: [0,3]     . DeE-------R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .  DeeeeeeeER  .  .   st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: [0,5]     .  D=eE-----R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .   D==eeeeeeeeeER.   st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: [0,7]     .    D==eE-------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .D=eeeeeeeeeER   st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: [0,9]     .    . D=eE-------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4773,43 +4766,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: 3.     1     15.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     14.0   0.0    0.0       st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: 5.     1     21.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     20.0   0.0    0.0       st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: 7.     1     28.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     27.0   0.0    0.0       st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: 9.     1     35.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     17.6   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: 3.     1     1.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: 5.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    2.0    0.0       st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: 7.     1     3.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     2.0    0.0    0.0       st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: 9.     1     2.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.8    0.4    3.1       <total>
 
 # CHECK:      [79] Code Region - G80
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3604
+# CHECK-NEXT: Total Cycles:      1007
 # CHECK-NEXT: Total uOps:        4800
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    1.33
-# CHECK-NEXT: IPC:               0.28
+# CHECK-NEXT: uOps Per Cycle:    4.77
+# CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 9.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     0123456
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .    .   .   st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: [0,1]     .D=====eER.    .    .    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D====eeeeeeeER    .    .    .    .   .   st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: [0,3]     . D===========eER   .    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .  D==========eeeeeeeER  .    .    .   .   st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: [0,5]     .  D=================eER .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .   D================eeeeeeeeeER   .   .   st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: [0,7]     .    D========================eER  .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .D=======================eeeeeeeER.   st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: [0,9]     .    .D==============================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER .    ..   st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE----R .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeER   ..   st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,3]     . D=eE-----R   ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .  D=eeeeeeeER ..   st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,5]     .  D==eE-----R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .   D=eeeeeeeeeER   st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,7]     .    D=eE-------R   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .D=eeeeeeeER   st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .D==eE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4819,43 +4812,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: 3.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     11.0   0.0    0.0       st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: 5.     1     18.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     17.0   0.0    0.0       st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: 7.     1     25.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     24.0   0.0    0.0       st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: 9.     1     31.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     15.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: 7.     1     2.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     2.0    1.0    0.0       st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: 9.     1     3.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.9    0.4    2.6       <total>
 
 # CHECK:      [80] Code Region - G81
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3304
+# CHECK-NEXT: Total Cycles:      1057
 # CHECK-NEXT: Total uOps:        5200
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    1.57
-# CHECK-NEXT: IPC:               0.30
+# CHECK-NEXT: uOps Per Cycle:    4.92
+# CHECK-NEXT: IPC:               0.95
 # CHECK-NEXT: Block RThroughput: 10.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     0123456
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeeeER   .    .    .    .    ..   st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: [0,1]     .D========eER  .    .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D=======eeeeeeeeeER    .    .    ..   st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: [0,3]     .  D===============eER   .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D==============eeeeeER    .    ..   st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
-# CHECK-NEXT: [0,5]     .   D===================eER   .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    D==================eeeeeER    ..   st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
-# CHECK-NEXT: [0,7]     .    D=======================eER   ..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .D======================eeeeeER.   st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
-# CHECK-NEXT: [0,9]     .    .D===========================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeeeER   ..   st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE-------R   ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeeER ..   st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE-------R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   D==eeeeeER ..   st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
+# CHECK-NEXT: [0,5]     .   D===eE---R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    D===eeeeeER.   st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
+# CHECK-NEXT: [0,7]     .    D====eE---R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .D===eeeeeER   st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
+# CHECK-NEXT: [0,9]     .    .D====eE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4865,43 +4858,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: 3.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     15.0   0.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
-# CHECK-NEXT: 5.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     19.0   0.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
-# CHECK-NEXT: 7.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     23.0   0.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
-# CHECK-NEXT: 9.     1     28.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.3   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    3.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
+# CHECK-NEXT: 5.     1     4.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     4.0    1.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
+# CHECK-NEXT: 7.     1     5.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
+# CHECK-NEXT: 9.     1     5.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.9    0.6    2.3       <total>
 
 # CHECK:      [81] Code Region - G82
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2504
+# CHECK-NEXT: Total Cycles:      756
 # CHECK-NEXT: Total uOps:        4000
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    1.60
-# CHECK-NEXT: IPC:               0.40
+# CHECK-NEXT: uOps Per Cycle:    5.29
+# CHECK-NEXT: IPC:               1.32
 # CHECK-NEXT: Block RThroughput: 7.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          012345678
-
-# CHECK:      [0,0]     DeeeeeER  .    .    .    .  .   st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
-# CHECK-NEXT: [0,1]     D=====eER .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D====eeeeeER  .    .    .  .   st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
-# CHECK-NEXT: [0,3]     .D=========eER .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D========eeeeeER  .    .  .   st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
-# CHECK-NEXT: [0,5]     . D=============eER .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D============eeeeeER  .  .   st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .  D=================eER .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D================eeeeeER.   st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
-# CHECK-NEXT: [0,9]     .   D=====================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0123
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeER  .  .   st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
+# CHECK-NEXT: [0,1]     D=eE---R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeER .  .   st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
+# CHECK-NEXT: [0,3]     .D=eE---R .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeeeeER  .   st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
+# CHECK-NEXT: [0,5]     . D==eE---R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeeeeER .   st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .  D==eE---R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D==eeeeeER   st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
+# CHECK-NEXT: [0,9]     .   D===eE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4911,41 +4904,41 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
-# CHECK-NEXT: 3.     1     10.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     9.0    0.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
-# CHECK-NEXT: 5.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     13.0   0.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
-# CHECK-NEXT: 7.     1     18.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     17.0   0.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
-# CHECK-NEXT: 9.     1     22.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     11.5   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
+# CHECK-NEXT: 3.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
+# CHECK-NEXT: 5.     1     3.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
+# CHECK-NEXT: 7.     1     3.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    1.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.3    1.5       <total>
 
 # CHECK:      [82] Code Region - G83
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      800
-# CHECK-NEXT: Total Cycles:      1804
+# CHECK-NEXT: Total Cycles:      605
 # CHECK-NEXT: Total uOps:        3200
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    1.77
-# CHECK-NEXT: IPC:               0.44
+# CHECK-NEXT: uOps Per Cycle:    5.29
+# CHECK-NEXT: IPC:               1.32
 # CHECK-NEXT: Block RThroughput: 6.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          01
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeER  .    .    ..   st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
-# CHECK-NEXT: [0,1]     D=====eER .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D====eeeeeER  .    ..   st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
-# CHECK-NEXT: [0,3]     .D=========eER .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D========eeeeER   ..   st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
-# CHECK-NEXT: [0,5]     . D============eER  ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D===========eeeeER.   st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .  D===============eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeER  .   st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
+# CHECK-NEXT: [0,1]     D=eE---R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeER .   st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE---R .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeeeER.   st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
+# CHECK-NEXT: [0,5]     . D==eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeeeER   st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .  D==eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4955,34 +4948,34 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
-# CHECK-NEXT: 3.     1     10.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     9.0    0.0    0.0       st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
-# CHECK-NEXT: 5.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     12.0   0.0    0.0       st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
-# CHECK-NEXT: 7.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     9.0    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
+# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
+# CHECK-NEXT: 7.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.0    0.3    1.3       <total>
 
 # CHECK:      [83] Code Region - G84
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      400
-# CHECK-NEXT: Total Cycles:      404
+# CHECK-NEXT: Total Cycles:      204
 # CHECK-NEXT: Total uOps:        1000
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    2.48
-# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: uOps Per Cycle:    4.90
+# CHECK-NEXT: IPC:               1.96
 # CHECK-NEXT: Block RThroughput: 1.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT: Index     01234567
+# CHECK-NEXT: Index     012345
 
-# CHECK:      [0,0]     DeeER. .   stp	s1, s2, [x27], #248
-# CHECK-NEXT: [0,1]     D==eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=eeER.   stp	d1, d2, [x27], #496
-# CHECK-NEXT: [0,3]     .D===eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.   stp	s1, s2, [x27], #248
+# CHECK-NEXT: [0,1]     D=eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeER   stp	d1, d2, [x27], #496
+# CHECK-NEXT: [0,3]     .D=eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4992,37 +4985,36 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       stp	s1, s2, [x27], #248
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       stp	d1, d2, [x27], #496
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.5    0.3    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       stp	d1, d2, [x27], #496
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.3    0.0       <total>
 
 # CHECK:      [84] Code Region - G85
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1104
+# CHECK-NEXT: Total Cycles:      703
 # CHECK-NEXT: Total uOps:        3100
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    2.81
-# CHECK-NEXT: IPC:               0.91
+# CHECK-NEXT: uOps Per Cycle:    4.41
+# CHECK-NEXT: IPC:               1.42
 # CHECK-NEXT: Block RThroughput: 6.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01234
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeER    .   .   stp	q1, q2, [x27], #992
-# CHECK-NEXT: [0,1]     D===eER   .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D==eeER  .   .   stp	s1, s2, [x27, #248]!
-# CHECK-NEXT: [0,3]     .D====eER .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D===eeER.   .   stp	d1, d2, [x27, #496]!
-# CHECK-NEXT: [0,5]     . D=====eER   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D====eeeER .   stp	q1, q2, [x27, #992]!
-# CHECK-NEXT: [0,7]     .  D=======eER.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D======eER.   stp	w1, w2, [x27], #248
-# CHECK-NEXT: [0,9]     .   D=======eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeER   .   stp	q1, q2, [x27], #992
+# CHECK-NEXT: [0,1]     D=eE-R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeER   .   stp	s1, s2, [x27, #248]!
+# CHECK-NEXT: [0,3]     .D=eER   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeER  .   stp	d1, d2, [x27, #496]!
+# CHECK-NEXT: [0,5]     . D=eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeeER   stp	q1, q2, [x27, #992]!
+# CHECK-NEXT: [0,7]     .  D==eE-R   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D=eE-R   stp	w1, w2, [x27], #248
+# CHECK-NEXT: [0,9]     .   D==eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -5032,43 +5024,42 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       stp	q1, q2, [x27], #992
-# CHECK-NEXT: 1.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       stp	s1, s2, [x27, #248]!
-# CHECK-NEXT: 3.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       stp	d1, d2, [x27, #496]!
-# CHECK-NEXT: 5.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     5.0    0.0    0.0       stp	q1, q2, [x27, #992]!
-# CHECK-NEXT: 7.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     7.0    0.0    0.0       stp	w1, w2, [x27], #248
-# CHECK-NEXT: 9.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     5.1    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       stp	s1, s2, [x27, #248]!
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       stp	d1, d2, [x27, #496]!
+# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    1.0    0.0       stp	q1, q2, [x27, #992]!
+# CHECK-NEXT: 7.     1     3.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     2.0    0.0    1.0       stp	w1, w2, [x27], #248
+# CHECK-NEXT: 9.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.9    0.2    0.3       <total>
 
 # CHECK:      [85] Code Region - G86
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      704
+# CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2300
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    3.27
-# CHECK-NEXT: IPC:               1.42
+# CHECK-NEXT: uOps Per Cycle:    4.56
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     012345678
 
-# CHECK:      [0,0]     DeER .    .   stp	x1, x2, [x27], #496
-# CHECK-NEXT: [0,1]     D=eER.    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeER.    .   stp	w1, w2, [x27, #248]!
-# CHECK-NEXT: [0,3]     .D=eER    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . DeER    .   stp	x1, x2, [x27, #496]!
-# CHECK-NEXT: [0,5]     . D=eER   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     . D=eeER  .   str	b1, [x27], #254
-# CHECK-NEXT: [0,7]     .  D==eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .  D==eeER.   str	h1, [x27], #254
-# CHECK-NEXT: [0,9]     .  D====eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeER .  .   stp	x1, x2, [x27], #496
+# CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeER.  .   stp	w1, w2, [x27, #248]!
+# CHECK-NEXT: [0,3]     .D=eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeER  .   stp	x1, x2, [x27, #496]!
+# CHECK-NEXT: [0,5]     . D=eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     . D=eeER.   str	b1, [x27], #254
+# CHECK-NEXT: [0,7]     .  D=eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .  D=eeER   str	h1, [x27], #254
+# CHECK-NEXT: [0,9]     .  D==eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -5084,37 +5075,36 @@ add x0, x27, 1
 # CHECK-NEXT: 4.     1     1.0    0.0    0.0       stp	x1, x2, [x27, #496]!
 # CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     2.0    0.0    0.0       str	b1, [x27], #254
-# CHECK-NEXT: 7.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       str	h1, [x27], #254
-# CHECK-NEXT: 9.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.2    0.1    0.0       <total>
+# CHECK-NEXT: 7.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     2.0    0.0    0.0       str	h1, [x27], #254
+# CHECK-NEXT: 9.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.8    0.1    0.0       <total>
 
 # CHECK:      [86] Code Region - G87
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1004
+# CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2200
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    2.19
-# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: uOps Per Cycle:    4.37
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 3.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     012345678
 
-# CHECK:      [0,0]     DeeER.    .  .   str	s1, [x27], #254
-# CHECK-NEXT: [0,1]     D==eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D==eeER   .  .   str	d1, [x27], #254
-# CHECK-NEXT: [0,3]     D====eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===eeER .  .   str	q1, [x27], #254
-# CHECK-NEXT: [0,5]     .D=====eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     . D====eeER  .   str	b1, [x27, #254]!
-# CHECK-NEXT: [0,7]     . D======eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D======eeER.   str	h1, [x27, #254]!
-# CHECK-NEXT: [0,9]     . D========eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.  .   str	s1, [x27], #254
+# CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeER  .   str	d1, [x27], #254
+# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeER .   str	q1, [x27], #254
+# CHECK-NEXT: [0,5]     .D==eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     . D=eeER.   str	b1, [x27, #254]!
+# CHECK-NEXT: [0,7]     . D==eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeER   str	h1, [x27, #254]!
+# CHECK-NEXT: [0,9]     . D===eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -5124,43 +5114,42 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       str	s1, [x27], #254
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       str	d1, [x27], #254
-# CHECK-NEXT: 3.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       str	q1, [x27], #254
-# CHECK-NEXT: 5.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     5.0    0.0    0.0       str	b1, [x27, #254]!
-# CHECK-NEXT: 7.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     7.0    0.0    0.0       str	h1, [x27, #254]!
-# CHECK-NEXT: 9.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     5.0    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       str	d1, [x27], #254
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       str	q1, [x27], #254
+# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       str	b1, [x27, #254]!
+# CHECK-NEXT: 7.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       str	h1, [x27, #254]!
+# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.5    0.1    0.0       <total>
 
 # CHECK:      [87] Code Region - G88
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      804
+# CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2200
 
 # CHECK:      Dispatch Width:    8
-# CHECK-NEXT: uOps Per Cycle:    2.74
-# CHECK-NEXT: IPC:               1.24
+# CHECK-NEXT: uOps Per Cycle:    4.37
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 3.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     012345678
 
-# CHECK:      [0,0]     DeeER.    ..   str	s1, [x27, #254]!
-# CHECK-NEXT: [0,1]     D==eER    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D==eeER   ..   str	d1, [x27, #254]!
-# CHECK-NEXT: [0,3]     D====eER  ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===eeER ..   str	q1, [x27, #254]!
-# CHECK-NEXT: [0,5]     .D=====eER..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     . D====eER..   str	w1, [x27], #254
-# CHECK-NEXT: [0,7]     . D=====eER.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D=====eER.   str	x1, [x27], #254
-# CHECK-NEXT: [0,9]     . D======eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.  .   str	s1, [x27, #254]!
+# CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeER  .   str	d1, [x27, #254]!
+# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeER .   str	q1, [x27, #254]!
+# CHECK-NEXT: [0,5]     .D==eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     . D=eER .   str	w1, [x27], #254
+# CHECK-NEXT: [0,7]     . D==eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eER.   str	x1, [x27], #254
+# CHECK-NEXT: [0,9]     . D===eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -5170,16 +5159,16 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       str	s1, [x27, #254]!
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       str	d1, [x27, #254]!
-# CHECK-NEXT: 3.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       str	q1, [x27, #254]!
-# CHECK-NEXT: 5.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     5.0    0.0    0.0       str	w1, [x27], #254
-# CHECK-NEXT: 7.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     6.0    0.0    0.0       str	x1, [x27], #254
-# CHECK-NEXT: 9.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     4.6    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       str	d1, [x27, #254]!
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       str	q1, [x27, #254]!
+# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       str	w1, [x27], #254
+# CHECK-NEXT: 7.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       str	x1, [x27], #254
+# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.5    0.1    0.0       <total>
 
 # CHECK:      [88] Code Region - G89
 
diff --git a/llvm/test/tools/llvm-mca/AArch64/Neoverse/N2-writeback.s b/llvm/test/tools/llvm-mca/AArch64/Neoverse/N2-writeback.s
index 7ee97a088e..0c6ccc1fac 100644
--- a/llvm/test/tools/llvm-mca/AArch64/Neoverse/N2-writeback.s
+++ b/llvm/test/tools/llvm-mca/AArch64/Neoverse/N2-writeback.s
@@ -1182,28 +1182,28 @@ add x0, x27, 1
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3004
+# CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    0.50
-# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: uOps Per Cycle:    2.95
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .  .   ld1	{ v1.1d }, [x27], #8
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D======eeeeeeER.    .    .    .  .   ld1	{ v1.2d }, [x27], #16
-# CHECK-NEXT: [0,3]     D============eER    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D============eeeeeeER    .    .  .   ld1	{ v1.2s }, [x27], #8
-# CHECK-NEXT: [0,5]     D==================eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=================eeeeeeER   .  .   ld1	{ v1.4h }, [x27], #8
-# CHECK-NEXT: [0,7]     .D=======================eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D=======================eeeeeeER.   ld1	{ v1.4s }, [x27], #16
-# CHECK-NEXT: [0,9]     .D=============================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.1d }, [x27], #8
+# CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ld1	{ v1.2d }, [x27], #16
+# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeeeER .   ld1	{ v1.2s }, [x27], #8
+# CHECK-NEXT: [0,5]     D===eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeER.   ld1	{ v1.4h }, [x27], #8
+# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeeeeeER   ld1	{ v1.4s }, [x27], #16
+# CHECK-NEXT: [0,9]     .D====eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1213,43 +1213,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.1d }, [x27], #8
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld1	{ v1.2d }, [x27], #16
-# CHECK-NEXT: 3.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld1	{ v1.2s }, [x27], #8
-# CHECK-NEXT: 5.     1     19.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     18.0   0.0    0.0       ld1	{ v1.4h }, [x27], #8
-# CHECK-NEXT: 7.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     24.0   0.0    0.0       ld1	{ v1.4s }, [x27], #16
-# CHECK-NEXT: 9.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     15.6   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.2d }, [x27], #16
+# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1	{ v1.2s }, [x27], #8
+# CHECK-NEXT: 5.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.4h }, [x27], #8
+# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ld1	{ v1.4s }, [x27], #16
+# CHECK-NEXT: 9.     1     5.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.1    0.1    2.0       <total>
 
 # CHECK:      [1] Code Region - G02
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3004
+# CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    0.50
-# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: uOps Per Cycle:    2.95
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .  .   ld1	{ v1.8b }, [x27], #8
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D======eeeeeeER.    .    .    .  .   ld1	{ v1.8h }, [x27], #16
-# CHECK-NEXT: [0,3]     D============eER    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D============eeeeeeER    .    .  .   ld1	{ v1.16b }, [x27], #16
-# CHECK-NEXT: [0,5]     D==================eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=================eeeeeeER   .  .   ld1	{ v1.1d }, [x27], x28
-# CHECK-NEXT: [0,7]     .D=======================eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D=======================eeeeeeER.   ld1	{ v1.2d }, [x27], x28
-# CHECK-NEXT: [0,9]     .D=============================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.8b }, [x27], #8
+# CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ld1	{ v1.8h }, [x27], #16
+# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeeeER .   ld1	{ v1.16b }, [x27], #16
+# CHECK-NEXT: [0,5]     D===eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeER.   ld1	{ v1.1d }, [x27], x28
+# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeeeeeER   ld1	{ v1.2d }, [x27], x28
+# CHECK-NEXT: [0,9]     .D====eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1259,43 +1259,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.8b }, [x27], #8
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld1	{ v1.8h }, [x27], #16
-# CHECK-NEXT: 3.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld1	{ v1.16b }, [x27], #16
-# CHECK-NEXT: 5.     1     19.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     18.0   0.0    0.0       ld1	{ v1.1d }, [x27], x28
-# CHECK-NEXT: 7.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     24.0   0.0    0.0       ld1	{ v1.2d }, [x27], x28
-# CHECK-NEXT: 9.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     15.6   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.8h }, [x27], #16
+# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1	{ v1.16b }, [x27], #16
+# CHECK-NEXT: 5.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.1d }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ld1	{ v1.2d }, [x27], x28
+# CHECK-NEXT: 9.     1     5.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.1    0.1    2.0       <total>
 
 # CHECK:      [2] Code Region - G03
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3004
+# CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    0.50
-# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: uOps Per Cycle:    2.95
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .  .   ld1	{ v1.2s }, [x27], x28
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D======eeeeeeER.    .    .    .  .   ld1	{ v1.4h }, [x27], x28
-# CHECK-NEXT: [0,3]     D============eER    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D============eeeeeeER    .    .  .   ld1	{ v1.4s }, [x27], x28
-# CHECK-NEXT: [0,5]     D==================eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=================eeeeeeER   .  .   ld1	{ v1.8b }, [x27], x28
-# CHECK-NEXT: [0,7]     .D=======================eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D=======================eeeeeeER.   ld1	{ v1.8h }, [x27], x28
-# CHECK-NEXT: [0,9]     .D=============================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.2s }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ld1	{ v1.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeeeER .   ld1	{ v1.4s }, [x27], x28
+# CHECK-NEXT: [0,5]     D===eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeER.   ld1	{ v1.8b }, [x27], x28
+# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeeeeeER   ld1	{ v1.8h }, [x27], x28
+# CHECK-NEXT: [0,9]     .D====eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1305,43 +1305,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.2s }, [x27], x28
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld1	{ v1.4h }, [x27], x28
-# CHECK-NEXT: 3.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld1	{ v1.4s }, [x27], x28
-# CHECK-NEXT: 5.     1     19.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     18.0   0.0    0.0       ld1	{ v1.8b }, [x27], x28
-# CHECK-NEXT: 7.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     24.0   0.0    0.0       ld1	{ v1.8h }, [x27], x28
-# CHECK-NEXT: 9.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     15.6   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.4h }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1	{ v1.4s }, [x27], x28
+# CHECK-NEXT: 5.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.8b }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ld1	{ v1.8h }, [x27], x28
+# CHECK-NEXT: 9.     1     5.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.1    0.1    2.0       <total>
 
 # CHECK:      [3] Code Region - G04
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3004
+# CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        1900
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    0.63
-# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: uOps Per Cycle:    3.74
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 3.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .  .   ld1	{ v1.16b }, [x27], x28
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D======eeeeeeER.    .    .    .  .   ld1	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: [0,3]     D============eER    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D============eeeeeeER    .    .  .   ld1	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: [0,5]     .D=================eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=================eeeeeeER   .  .   ld1	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: [0,7]     .D=======================eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D=======================eeeeeeER.   ld1	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: [0,9]     .D=============================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.16b }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ld1	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeeeER .   ld1	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: [0,5]     .D==eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeER.   ld1	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeeeeeER   ld1	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: [0,9]     .D====eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1351,43 +1351,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.16b }, [x27], x28
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld1	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: 3.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld1	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: 5.     1     18.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     18.0   0.0    0.0       ld1	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: 7.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     24.0   0.0    0.0       ld1	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: 9.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     15.5   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ld1	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: 9.     1     5.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.0    0.1    2.0       <total>
 
 # CHECK:      [4] Code Region - G05
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3004
+# CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    0.67
-# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: uOps Per Cycle:    3.94
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 3.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .  .   ld1	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D======eeeeeeER.    .    .    .  .   ld1	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: [0,3]     D============eER    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===========eeeeeeER    .    .  .   ld1	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: [0,5]     .D=================eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=================eeeeeeER   .  .   ld1	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: [0,7]     .D=======================eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D======================eeeeeeER.   ld1	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: [0,9]     . D============================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ld1	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeeER .   ld1	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: [0,5]     .D==eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeER.   ld1	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeeER   ld1	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: [0,9]     . D===eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1397,43 +1397,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld1	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: 3.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     12.0   0.0    0.0       ld1	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: 5.     1     18.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     18.0   0.0    0.0       ld1	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: 7.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     23.0   0.0    0.0       ld1	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: 9.     1     29.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     15.2   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    2.0       <total>
 
 # CHECK:      [5] Code Region - G06
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3004
+# CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    0.67
-# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: uOps Per Cycle:    3.94
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 3.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .  .   ld1	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D======eeeeeeER.    .    .    .  .   ld1	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: [0,3]     D============eER    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===========eeeeeeER    .    .  .   ld1	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: [0,5]     .D=================eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=================eeeeeeER   .  .   ld1	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: [0,7]     .D=======================eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D======================eeeeeeER.   ld1	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: [0,9]     . D============================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ld1	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeeER .   ld1	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,5]     .D==eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeER.   ld1	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeeER   ld1	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,9]     . D===eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1443,43 +1443,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld1	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: 3.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     12.0   0.0    0.0       ld1	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: 5.     1     18.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     18.0   0.0    0.0       ld1	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: 7.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     23.0   0.0    0.0       ld1	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: 9.     1     29.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     15.2   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    2.0       <total>
 
 # CHECK:      [6] Code Region - G07
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3004
+# CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        2300
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    0.77
-# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: uOps Per Cycle:    4.53
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 4.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .  .   ld1	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D======eeeeeeER.    .    .    .  .   ld1	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: [0,3]     D============eER    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===========eeeeeeER    .    .  .   ld1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: [0,5]     .D=================eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=================eeeeeeER   .  .   ld1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: [0,7]     .D=======================eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D======================eeeeeeER.   ld1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: [0,9]     . D============================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ld1	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeeER .   ld1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
+# CHECK-NEXT: [0,5]     .D==eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeER.   ld1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeeER   ld1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: [0,9]     . D===eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1489,43 +1489,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld1	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: 3.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     12.0   0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: 5.     1     18.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     18.0   0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: 7.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     23.0   0.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: 9.     1     29.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     15.2   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
+# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    2.0       <total>
 
 # CHECK:      [7] Code Region - G08
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3004
+# CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        2500
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    0.83
-# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: uOps Per Cycle:    4.92
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .  .   ld1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D======eeeeeeER.    .    .    .  .   ld1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: [0,3]     D============eER    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===========eeeeeeER    .    .  .   ld1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: [0,5]     .D=================eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=================eeeeeeER   .  .   ld1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: [0,7]     .D=======================eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D======================eeeeeeER.   ld1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: [0,9]     . D============================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ld1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeeER .   ld1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: [0,5]     .D==eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeER.   ld1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeeER   ld1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: [0,9]     . D===eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1535,43 +1535,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: 3.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     12.0   0.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: 5.     1     18.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     18.0   0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: 7.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     23.0   0.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: 9.     1     29.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     15.2   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    2.0       <total>
 
 # CHECK:      [8] Code Region - G09
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3004
+# CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        2500
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    0.83
-# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: uOps Per Cycle:    4.92
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .  .   ld1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D======eeeeeeER.    .    .    .  .   ld1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: [0,3]     D============eER    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===========eeeeeeER    .    .  .   ld1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: [0,5]     .D=================eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=================eeeeeeER   .  .   ld1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: [0,7]     .D=======================eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D======================eeeeeeER.   ld1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: [0,9]     . D============================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ld1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeeER .   ld1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,5]     .D==eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeER.   ld1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeeER   ld1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: [0,9]     . D===eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1581,43 +1581,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: 3.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     12.0   0.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: 5.     1     18.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     18.0   0.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: 7.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     23.0   0.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: 9.     1     29.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     15.2   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    2.0       <total>
 
 # CHECK:      [9] Code Region - G10
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3204
+# CHECK-NEXT: Total Cycles:      608
 # CHECK-NEXT: Total uOps:        2700
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    0.84
-# CHECK-NEXT: IPC:               0.31
+# CHECK-NEXT: uOps Per Cycle:    4.44
+# CHECK-NEXT: IPC:               1.64
 # CHECK-NEXT: Block RThroughput: 5.7
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          012345
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     0123
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .    .   ld1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D======eeeeeeER.    .    .    .    .   ld1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: [0,3]     D============eER    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===========eeeeeeER    .    .    .   ld1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: [0,5]     .D=================eER   .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=================eeeeeeeER  .    .   ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: [0,7]     . D=======================eER .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D=======================eeeeeeeER.   ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: [0,9]     . D==============================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER .  .   ld1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE----R .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeER.  .   ld1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eE----R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeeER  .   ld1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,5]     .D==eE----R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeeER.   ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: [0,7]     . D==eE-----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeeeER   ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: [0,9]     . D===eE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1627,43 +1627,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: 3.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     12.0   0.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: 5.     1     18.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     18.0   0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: 7.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     24.0   0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: 9.     1     31.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     15.5   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: 7.     1     3.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: 9.     1     4.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.6    0.1    2.2       <total>
 
 # CHECK:      [10] Code Region - G11
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3504
+# CHECK-NEXT: Total Cycles:      675
 # CHECK-NEXT: Total uOps:        3000
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    0.86
-# CHECK-NEXT: IPC:               0.29
+# CHECK-NEXT: uOps Per Cycle:    4.44
+# CHECK-NEXT: IPC:               1.48
 # CHECK-NEXT: Block RThroughput: 6.7
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          012345678
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    .  .   ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: [0,1]     D=======eER    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D======eeeeeeeER   .    .    .    .  .   ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: [0,3]     .D=============eER  .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D============eeeeeeeER .    .    .  .   ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: [0,5]     . D===================eER.    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D==================eeeeeeeER    .  .   ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: [0,7]     .  D=========================eER   .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D========================eeeeeeeER.   ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: [0,9]     .   D===============================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeER.   .   ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: [0,1]     D=eE-----R.   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeER   .   ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: [0,3]     .D=eE-----R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeeER  .   ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: [0,5]     . D=eE-----R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeeeeeeER.   ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: [0,7]     .  D==eE-----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D=eeeeeeeER   ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: [0,9]     .   D==eE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1673,43 +1673,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: 5.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: 7.     1     26.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: 9.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.5   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: 3.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: 5.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    1.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: 7.     1     3.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     2.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: 9.     1     3.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.9    0.2    2.5       <total>
 
 # CHECK:      [11] Code Region - G12
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3504
+# CHECK-NEXT: Total Cycles:      675
 # CHECK-NEXT: Total uOps:        3000
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    0.86
-# CHECK-NEXT: IPC:               0.29
+# CHECK-NEXT: uOps Per Cycle:    4.44
+# CHECK-NEXT: IPC:               1.48
 # CHECK-NEXT: Block RThroughput: 6.7
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          012345678
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    .  .   ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: [0,1]     D=======eER    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D======eeeeeeeER   .    .    .    .  .   ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=============eER  .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D============eeeeeeeER .    .    .  .   ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: [0,5]     . D===================eER.    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D==================eeeeeeeER    .  .   ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D=========================eER   .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D========================eeeeeeeER.   ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D===============================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeER.   .   ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: [0,1]     D=eE-----R.   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeER   .   ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE-----R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeeER  .   ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,5]     . D=eE-----R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeeeeeeER.   ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D==eE-----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D=eeeeeeeER   ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D==eE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1719,43 +1719,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: 5.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: 7.     1     26.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: 9.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.5   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    1.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 7.     1     3.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     2.0    0.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: 9.     1     3.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.9    0.2    2.5       <total>
 
 # CHECK:      [12] Code Region - G13
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3604
+# CHECK-NEXT: Total Cycles:      1210
 # CHECK-NEXT: Total uOps:        2800
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    0.78
-# CHECK-NEXT: IPC:               0.28
+# CHECK-NEXT: uOps Per Cycle:    2.31
+# CHECK-NEXT: IPC:               0.83
 # CHECK-NEXT: Block RThroughput: 5.7
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          01
 
-# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    .   .   ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: [0,1]     D=======eER    .    .    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D======eeeeeeeER   .    .    .    .   .   ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=============eER  .    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D============eeeeeeeER .    .    .   .   ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: [0,5]     . D===================eER.    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D==================eeeeeeeER    .   .   ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D=========================eER   .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .  D=========================eeeeeeeeER.   ld1	{ v1.b }[0], [x27], #1
-# CHECK-NEXT: [0,9]     .  D=================================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeER.    .    ..   ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE-----R.    .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeER    .    ..   ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE-----R    .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeeER   .    ..   ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,5]     . D=eE-----R   .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeeeeeeER .    ..   ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D==eE-----R .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .  D========eeeeeeeeER   ld1	{ v1.b }[0], [x27], #1
+# CHECK-NEXT: [0,9]     .  D=========eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1765,22 +1765,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: 5.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: 7.     1     26.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     26.0   0.0    0.0       ld1	{ v1.b }[0], [x27], #1
-# CHECK-NEXT: 9.     1     34.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.8   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    1.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: 7.     1     3.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     9.0    0.0    0.0       ld1	{ v1.b }[0], [x27], #1
+# CHECK-NEXT: 9.     1     10.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.3    0.2    2.6       <total>
 
 # CHECK:      [13] Code Region - G14
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    10
@@ -1790,18 +1790,18 @@ add x0, x27, 1
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld1	{ v1.b }[8], [x27], #1
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    .  .   ld1	{ v1.b }[0], [x27], x28
-# CHECK-NEXT: [0,3]     D================eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER   .    .    .  .   ld1	{ v1.b }[8], [x27], x28
-# CHECK-NEXT: [0,5]     .D=======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    .  .   ld1	{ v1.h }[0], [x27], #2
-# CHECK-NEXT: [0,7]     .D===============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============================eeeeeeeeER.   ld1	{ v1.h }[4], [x27], #2
-# CHECK-NEXT: [0,9]     . D======================================eER   add	x0, x27, #1
+# CHECK-NEXT: Index     0123456789          0123456789          012
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld1	{ v1.b }[8], [x27], #1
+# CHECK-NEXT: [0,1]     D=eE------R    .    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    . .   ld1	{ v1.b }[0], [x27], x28
+# CHECK-NEXT: [0,3]     D=========eE------R .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER   .    .    . .   ld1	{ v1.b }[8], [x27], x28
+# CHECK-NEXT: [0,5]     .D================eE------R   .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    . .   ld1	{ v1.h }[0], [x27], #2
+# CHECK-NEXT: [0,7]     .D========================eE------R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==============================eeeeeeeeER   ld1	{ v1.h }[4], [x27], #2
+# CHECK-NEXT: [0,9]     . D===============================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1811,22 +1811,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.b }[8], [x27], #1
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld1	{ v1.b }[0], [x27], x28
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 3.     1     10.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld1	{ v1.b }[8], [x27], x28
-# CHECK-NEXT: 5.     1     24.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 5.     1     17.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld1	{ v1.h }[0], [x27], #2
-# CHECK-NEXT: 7.     1     32.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 7.     1     25.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 8.     1     31.0   0.0    0.0       ld1	{ v1.h }[4], [x27], #2
-# CHECK-NEXT: 9.     1     39.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.2   0.1    0.0       <total>
+# CHECK-NEXT: 9.     1     32.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     16.7   0.1    3.0       <total>
 
 # CHECK:      [14] Code Region - G15
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    10
@@ -1836,18 +1836,18 @@ add x0, x27, 1
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld1	{ v1.h }[0], [x27], x28
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    .  .   ld1	{ v1.h }[4], [x27], x28
-# CHECK-NEXT: [0,3]     D================eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER   .    .    .  .   ld1	{ v1.s }[0], [x27], #4
-# CHECK-NEXT: [0,5]     .D=======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    .  .   ld1	{ v1.s }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .D===============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============================eeeeeeeeER.   ld1	{ v1.d }[0], [x27], #8
-# CHECK-NEXT: [0,9]     . D======================================eER   add	x0, x27, #1
+# CHECK-NEXT: Index     0123456789          0123456789          012
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld1	{ v1.h }[0], [x27], x28
+# CHECK-NEXT: [0,1]     D=eE------R    .    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    . .   ld1	{ v1.h }[4], [x27], x28
+# CHECK-NEXT: [0,3]     D=========eE------R .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER   .    .    . .   ld1	{ v1.s }[0], [x27], #4
+# CHECK-NEXT: [0,5]     .D================eE------R   .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    . .   ld1	{ v1.s }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .D========================eE------R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==============================eeeeeeeeER   ld1	{ v1.d }[0], [x27], #8
+# CHECK-NEXT: [0,9]     . D===============================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1857,43 +1857,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.h }[0], [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld1	{ v1.h }[4], [x27], x28
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 3.     1     10.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld1	{ v1.s }[0], [x27], #4
-# CHECK-NEXT: 5.     1     24.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 5.     1     17.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld1	{ v1.s }[0], [x27], x28
-# CHECK-NEXT: 7.     1     32.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 7.     1     25.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 8.     1     31.0   0.0    0.0       ld1	{ v1.d }[0], [x27], #8
-# CHECK-NEXT: 9.     1     39.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.2   0.1    0.0       <total>
+# CHECK-NEXT: 9.     1     32.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     16.7   0.1    3.0       <total>
 
 # CHECK:      [15] Code Region - G16
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      1203
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    0.50
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    1.66
+# CHECK-NEXT: IPC:               0.83
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld1	{ v1.d }[0], [x27], x28
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    .  .   ld1r	{ v1.1d }, [x27], #8
-# CHECK-NEXT: [0,3]     D================eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER   .    .    .  .   ld1r	{ v1.2d }, [x27], #8
-# CHECK-NEXT: [0,5]     .D=======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    .  .   ld1r	{ v1.2s }, [x27], #4
-# CHECK-NEXT: [0,7]     .D===============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============================eeeeeeeeER.   ld1r	{ v1.4h }, [x27], #2
-# CHECK-NEXT: [0,9]     . D======================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER   .   ld1	{ v1.d }[0], [x27], x28
+# CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld1r	{ v1.1d }, [x27], #8
+# CHECK-NEXT: [0,3]     D==eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeeeeER .   ld1r	{ v1.2d }, [x27], #8
+# CHECK-NEXT: [0,5]     .D==eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeeeER.   ld1r	{ v1.2s }, [x27], #4
+# CHECK-NEXT: [0,7]     .D===eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeeeeER   ld1r	{ v1.4h }, [x27], #2
+# CHECK-NEXT: [0,9]     . D===eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1903,43 +1903,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.d }[0], [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld1r	{ v1.1d }, [x27], #8
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld1r	{ v1.2d }, [x27], #8
-# CHECK-NEXT: 5.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld1r	{ v1.2s }, [x27], #4
-# CHECK-NEXT: 7.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     31.0   0.0    0.0       ld1r	{ v1.4h }, [x27], #2
-# CHECK-NEXT: 9.     1     39.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.2   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1r	{ v1.1d }, [x27], #8
+# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1r	{ v1.2d }, [x27], #8
+# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1r	{ v1.2s }, [x27], #4
+# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1r	{ v1.4h }, [x27], #2
+# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    3.0       <total>
 
 # CHECK:      [16] Code Region - G17
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    0.50
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    3.92
+# CHECK-NEXT: IPC:               1.96
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld1r	{ v1.4s }, [x27], #4
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    .  .   ld1r	{ v1.8b }, [x27], #1
-# CHECK-NEXT: [0,3]     D================eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER   .    .    .  .   ld1r	{ v1.8h }, [x27], #2
-# CHECK-NEXT: [0,5]     .D=======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    .  .   ld1r	{ v1.16b }, [x27], #1
-# CHECK-NEXT: [0,7]     .D===============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============================eeeeeeeeER.   ld1r	{ v1.1d }, [x27], x28
-# CHECK-NEXT: [0,9]     . D======================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER   .   ld1r	{ v1.4s }, [x27], #4
+# CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld1r	{ v1.8b }, [x27], #1
+# CHECK-NEXT: [0,3]     D==eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeeeeER .   ld1r	{ v1.8h }, [x27], #2
+# CHECK-NEXT: [0,5]     .D==eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeeeER.   ld1r	{ v1.16b }, [x27], #1
+# CHECK-NEXT: [0,7]     .D===eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeeeeER   ld1r	{ v1.1d }, [x27], x28
+# CHECK-NEXT: [0,9]     . D===eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1949,43 +1949,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1r	{ v1.4s }, [x27], #4
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld1r	{ v1.8b }, [x27], #1
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld1r	{ v1.8h }, [x27], #2
-# CHECK-NEXT: 5.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld1r	{ v1.16b }, [x27], #1
-# CHECK-NEXT: 7.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     31.0   0.0    0.0       ld1r	{ v1.1d }, [x27], x28
-# CHECK-NEXT: 9.     1     39.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.2   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1r	{ v1.8b }, [x27], #1
+# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1r	{ v1.8h }, [x27], #2
+# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1r	{ v1.16b }, [x27], #1
+# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1r	{ v1.1d }, [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    3.0       <total>
 
 # CHECK:      [17] Code Region - G18
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    0.50
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    3.92
+# CHECK-NEXT: IPC:               1.96
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld1r	{ v1.2d }, [x27], x28
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    .  .   ld1r	{ v1.2s }, [x27], x28
-# CHECK-NEXT: [0,3]     D================eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER   .    .    .  .   ld1r	{ v1.4h }, [x27], x28
-# CHECK-NEXT: [0,5]     .D=======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    .  .   ld1r	{ v1.4s }, [x27], x28
-# CHECK-NEXT: [0,7]     .D===============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============================eeeeeeeeER.   ld1r	{ v1.8b }, [x27], x28
-# CHECK-NEXT: [0,9]     . D======================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER   .   ld1r	{ v1.2d }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld1r	{ v1.2s }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeeeeER .   ld1r	{ v1.4h }, [x27], x28
+# CHECK-NEXT: [0,5]     .D==eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeeeER.   ld1r	{ v1.4s }, [x27], x28
+# CHECK-NEXT: [0,7]     .D===eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeeeeER   ld1r	{ v1.8b }, [x27], x28
+# CHECK-NEXT: [0,9]     . D===eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1995,43 +1995,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1r	{ v1.2d }, [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld1r	{ v1.2s }, [x27], x28
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld1r	{ v1.4h }, [x27], x28
-# CHECK-NEXT: 5.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld1r	{ v1.4s }, [x27], x28
-# CHECK-NEXT: 7.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     31.0   0.0    0.0       ld1r	{ v1.8b }, [x27], x28
-# CHECK-NEXT: 9.     1     39.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.2   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1r	{ v1.2s }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1r	{ v1.4h }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1r	{ v1.4s }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1r	{ v1.8b }, [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    3.0       <total>
 
 # CHECK:      [18] Code Region - G19
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        2400
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    0.60
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    4.71
+# CHECK-NEXT: IPC:               1.96
 # CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld1r	{ v1.8h }, [x27], x28
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    .  .   ld1r	{ v1.16b }, [x27], x28
-# CHECK-NEXT: [0,3]     D================eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER   .    .    .  .   ld2	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: [0,5]     .D=======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    .  .   ld2	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: [0,7]     . D==============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============================eeeeeeeeER.   ld2	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: [0,9]     . D======================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER   .   ld1r	{ v1.8h }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld1r	{ v1.16b }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeeeeER .   ld2	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: [0,5]     .D==eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeeeER.   ld2	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: [0,7]     . D==eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeeeeER   ld2	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: [0,9]     . D===eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2041,43 +2041,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1r	{ v1.8h }, [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld1r	{ v1.16b }, [x27], x28
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld2	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: 5.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld2	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: 7.     1     31.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     31.0   0.0    0.0       ld2	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: 9.     1     39.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.1   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1r	{ v1.16b }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld2	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld2	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: 7.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld2	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.6    0.1    3.0       <total>
 
 # CHECK:      [19] Code Region - G20
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        2900
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    0.72
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    5.69
+# CHECK-NEXT: IPC:               1.96
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld2	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    .  .   ld2	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: [0,3]     .D===============eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER   .    .    .  .   ld2	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: [0,5]     .D=======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     . D======================eeeeeeeeER.    .  .   ld2	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: [0,7]     . D==============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .  D=============================eeeeeeeeER.   ld2	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: [0,9]     .  D=====================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER   .   ld2	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld2	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: [0,3]     .D=eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeeeeER .   ld2	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: [0,5]     .D==eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     . D=eeeeeeeeER.   ld2	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: [0,7]     . D==eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .  D=eeeeeeeeER   ld2	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,9]     .  D==eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2087,43 +2087,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld2	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: 3.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld2	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: 5.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     23.0   0.0    0.0       ld2	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: 7.     1     31.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     30.0   0.0    0.0       ld2	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: 9.     1     38.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     19.7   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld2	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld2	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ld2	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: 7.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     2.0    0.0    0.0       ld2	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: 9.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.2    0.1    3.0       <total>
 
 # CHECK:      [20] Code Region - G21
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        2700
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    0.67
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    5.29
+# CHECK-NEXT: IPC:               1.96
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld2	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    .  .   ld2	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: [0,3]     D================eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER   .    .    .  .   ld2	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: [0,5]     .D=======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    .  .   ld2	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: [0,7]     . D==============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============================eeeeeeeeER.   ld2	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: [0,9]     . D======================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER   .   ld2	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld2	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeeeeER .   ld2	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,5]     .D==eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeeeER.   ld2	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,7]     . D==eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeeeeER   ld2	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,9]     . D===eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2133,43 +2133,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld2	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld2	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: 5.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld2	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: 7.     1     31.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     31.0   0.0    0.0       ld2	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: 9.     1     39.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.1   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld2	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld2	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld2	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: 7.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld2	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.6    0.1    3.0       <total>
 
 # CHECK:      [21] Code Region - G22
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      3310
 # CHECK-NEXT: Total uOps:        2600
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    0.65
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    0.79
+# CHECK-NEXT: IPC:               0.30
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld2	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    .  .   ld2	{ v1.b, v2.b }[0], [x27], #2
-# CHECK-NEXT: [0,3]     .D===============eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER   .    .    .  .   ld2	{ v1.b, v2.b }[8], [x27], #2
-# CHECK-NEXT: [0,5]     .D=======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    .  .   ld2	{ v1.b, v2.b }[0], [x27], x28
-# CHECK-NEXT: [0,7]     . D==============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============================eeeeeeeeER.   ld2	{ v1.b, v2.b }[8], [x27], x28
-# CHECK-NEXT: [0,9]     . D======================================eER   add	x0, x27, #1
+# CHECK-NEXT: Index     0123456789          0123456789          012
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld2	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE------R    .    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    . .   ld2	{ v1.b, v2.b }[0], [x27], #2
+# CHECK-NEXT: [0,3]     .D========eE------R .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER   .    .    . .   ld2	{ v1.b, v2.b }[8], [x27], #2
+# CHECK-NEXT: [0,5]     .D================eE------R   .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    . .   ld2	{ v1.b, v2.b }[0], [x27], x28
+# CHECK-NEXT: [0,7]     . D=======================eE------R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==============================eeeeeeeeER   ld2	{ v1.b, v2.b }[8], [x27], x28
+# CHECK-NEXT: [0,9]     . D===============================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2179,22 +2179,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld2	{ v1.b, v2.b }[0], [x27], #2
-# CHECK-NEXT: 3.     1     16.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 3.     1     9.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld2	{ v1.b, v2.b }[8], [x27], #2
-# CHECK-NEXT: 5.     1     24.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 5.     1     17.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld2	{ v1.b, v2.b }[0], [x27], x28
-# CHECK-NEXT: 7.     1     31.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 7.     1     24.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 8.     1     31.0   0.0    0.0       ld2	{ v1.b, v2.b }[8], [x27], x28
-# CHECK-NEXT: 9.     1     39.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.0   0.1    0.0       <total>
+# CHECK-NEXT: 9.     1     32.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     16.5   0.1    3.0       <total>
 
 # CHECK:      [22] Code Region - G23
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        2500
 
 # CHECK:      Dispatch Width:    10
@@ -2204,18 +2204,18 @@ add x0, x27, 1
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld2	{ v1.h, v2.h }[0], [x27], #4
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    .  .   ld2	{ v1.h, v2.h }[4], [x27], #4
-# CHECK-NEXT: [0,3]     D================eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER   .    .    .  .   ld2	{ v1.h, v2.h }[0], [x27], x28
-# CHECK-NEXT: [0,5]     .D=======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    .  .   ld2	{ v1.h, v2.h }[4], [x27], x28
-# CHECK-NEXT: [0,7]     .D===============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============================eeeeeeeeER.   ld2	{ v1.s, v2.s }[0], [x27], #8
-# CHECK-NEXT: [0,9]     . D======================================eER   add	x0, x27, #1
+# CHECK-NEXT: Index     0123456789          0123456789          012
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld2	{ v1.h, v2.h }[0], [x27], #4
+# CHECK-NEXT: [0,1]     D=eE------R    .    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    . .   ld2	{ v1.h, v2.h }[4], [x27], #4
+# CHECK-NEXT: [0,3]     D=========eE------R .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER   .    .    . .   ld2	{ v1.h, v2.h }[0], [x27], x28
+# CHECK-NEXT: [0,5]     .D================eE------R   .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    . .   ld2	{ v1.h, v2.h }[4], [x27], x28
+# CHECK-NEXT: [0,7]     .D========================eE------R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==============================eeeeeeeeER   ld2	{ v1.s, v2.s }[0], [x27], #8
+# CHECK-NEXT: [0,9]     . D===============================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2225,43 +2225,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.h, v2.h }[0], [x27], #4
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld2	{ v1.h, v2.h }[4], [x27], #4
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 3.     1     10.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld2	{ v1.h, v2.h }[0], [x27], x28
-# CHECK-NEXT: 5.     1     24.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 5.     1     17.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld2	{ v1.h, v2.h }[4], [x27], x28
-# CHECK-NEXT: 7.     1     32.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 7.     1     25.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 8.     1     31.0   0.0    0.0       ld2	{ v1.s, v2.s }[0], [x27], #8
-# CHECK-NEXT: 9.     1     39.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.2   0.1    0.0       <total>
+# CHECK-NEXT: 9.     1     32.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     16.7   0.1    3.0       <total>
 
 # CHECK:      [23] Code Region - G24
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      2603
 # CHECK-NEXT: Total uOps:        2500
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    0.62
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    0.96
+# CHECK-NEXT: IPC:               0.38
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld2	{ v1.s, v2.s }[0], [x27], x28
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    .  .   ld2	{ v1.d, v2.d }[0], [x27], #16
-# CHECK-NEXT: [0,3]     D================eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER   .    .    .  .   ld2	{ v1.d, v2.d }[0], [x27], x28
-# CHECK-NEXT: [0,5]     .D=======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    .  .   ld2r	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: [0,7]     .D===============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============================eeeeeeeeER.   ld2r	{ v1.2d, v2.2d }, [x27], #16
-# CHECK-NEXT: [0,9]     . D======================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          012345678
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    .  .   ld2	{ v1.s, v2.s }[0], [x27], x28
+# CHECK-NEXT: [0,1]     D=eE------R    .    .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .  .   ld2	{ v1.d, v2.d }[0], [x27], #16
+# CHECK-NEXT: [0,3]     D=========eE------R .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER .   ld2	{ v1.d, v2.d }[0], [x27], x28
+# CHECK-NEXT: [0,5]     .D================eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D================eeeeeeeeER.   ld2r	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: [0,7]     .D=================eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D================eeeeeeeeER   ld2r	{ v1.2d, v2.2d }, [x27], #16
+# CHECK-NEXT: [0,9]     . D=================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2271,43 +2271,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.s, v2.s }[0], [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld2	{ v1.d, v2.d }[0], [x27], #16
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 3.     1     10.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld2	{ v1.d, v2.d }[0], [x27], x28
-# CHECK-NEXT: 5.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld2r	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: 7.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     31.0   0.0    0.0       ld2r	{ v1.2d, v2.2d }, [x27], #16
-# CHECK-NEXT: 9.     1     39.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.2   0.1    0.0       <total>
+# CHECK-NEXT: 5.     1     17.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     17.0   0.0    0.0       ld2r	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: 7.     1     18.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     17.0   0.0    0.0       ld2r	{ v1.2d, v2.2d }, [x27], #16
+# CHECK-NEXT: 9.     1     18.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     12.5   0.1    3.0       <total>
 
 # CHECK:      [24] Code Region - G25
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        2500
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    0.62
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    4.90
+# CHECK-NEXT: IPC:               1.96
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld2r	{ v1.2s, v2.2s }, [x27], #8
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    .  .   ld2r	{ v1.4h, v2.4h }, [x27], #4
-# CHECK-NEXT: [0,3]     D================eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER   .    .    .  .   ld2r	{ v1.4s, v2.4s }, [x27], #8
-# CHECK-NEXT: [0,5]     .D=======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    .  .   ld2r	{ v1.8b, v2.8b }, [x27], #2
-# CHECK-NEXT: [0,7]     .D===============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============================eeeeeeeeER.   ld2r	{ v1.8h, v2.8h }, [x27], #4
-# CHECK-NEXT: [0,9]     . D======================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER   .   ld2r	{ v1.2s, v2.2s }, [x27], #8
+# CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld2r	{ v1.4h, v2.4h }, [x27], #4
+# CHECK-NEXT: [0,3]     D==eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeeeeER .   ld2r	{ v1.4s, v2.4s }, [x27], #8
+# CHECK-NEXT: [0,5]     .D==eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeeeER.   ld2r	{ v1.8b, v2.8b }, [x27], #2
+# CHECK-NEXT: [0,7]     .D===eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeeeeER   ld2r	{ v1.8h, v2.8h }, [x27], #4
+# CHECK-NEXT: [0,9]     . D===eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2317,43 +2317,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2r	{ v1.2s, v2.2s }, [x27], #8
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld2r	{ v1.4h, v2.4h }, [x27], #4
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld2r	{ v1.4s, v2.4s }, [x27], #8
-# CHECK-NEXT: 5.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld2r	{ v1.8b, v2.8b }, [x27], #2
-# CHECK-NEXT: 7.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     31.0   0.0    0.0       ld2r	{ v1.8h, v2.8h }, [x27], #4
-# CHECK-NEXT: 9.     1     39.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.2   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld2r	{ v1.4h, v2.4h }, [x27], #4
+# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld2r	{ v1.4s, v2.4s }, [x27], #8
+# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld2r	{ v1.8b, v2.8b }, [x27], #2
+# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld2r	{ v1.8h, v2.8h }, [x27], #4
+# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    3.0       <total>
 
 # CHECK:      [25] Code Region - G26
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        2500
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    0.62
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    4.90
+# CHECK-NEXT: IPC:               1.96
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld2r	{ v1.16b, v2.16b }, [x27], #2
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    .  .   ld2r	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: [0,3]     D================eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER   .    .    .  .   ld2r	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: [0,5]     .D=======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    .  .   ld2r	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: [0,7]     .D===============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============================eeeeeeeeER.   ld2r	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: [0,9]     . D======================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER   .   ld2r	{ v1.16b, v2.16b }, [x27], #2
+# CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld2r	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeeeeER .   ld2r	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,5]     .D==eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeeeER.   ld2r	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,7]     .D===eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeeeeER   ld2r	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,9]     . D===eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2363,43 +2363,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2r	{ v1.16b, v2.16b }, [x27], #2
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld2r	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld2r	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: 5.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld2r	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: 7.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     31.0   0.0    0.0       ld2r	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: 9.     1     39.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.2   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld2r	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld2r	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld2r	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld2r	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    3.0       <total>
 
 # CHECK:      [26] Code Region - G27
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      609
 # CHECK-NEXT: Total uOps:        2800
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    0.70
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    4.60
+# CHECK-NEXT: IPC:               1.64
 # CHECK-NEXT: Block RThroughput: 5.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld2r	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    .  .   ld2r	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: [0,3]     D================eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER   .    .    .  .   ld2r	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: [0,5]     .D=======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    .  .   ld2r	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: [0,7]     .D===============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============================eeeeeeeeER.   ld3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: [0,9]     . D======================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER   .   ld2r	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld2r	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeeeeER .   ld2r	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,5]     .D==eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeeeER.   ld2r	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,7]     .D===eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeeeeER   ld3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: [0,9]     . D===eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2409,43 +2409,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2r	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld2r	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld2r	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: 5.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld2r	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: 7.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     31.0   0.0    0.0       ld3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: 9.     1     39.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.2   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld2r	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld2r	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld2r	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    3.0       <total>
 
 # CHECK:      [27] Code Region - G28
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      759
 # CHECK-NEXT: Total uOps:        3700
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    0.92
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    4.87
+# CHECK-NEXT: IPC:               1.32
 # CHECK-NEXT: Block RThroughput: 7.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    .  .   ld3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: [0,3]     .D===============eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    .  .   ld3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: [0,5]     . D======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    .  .   ld3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: [0,7]     .  D=============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER.   ld3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: [0,9]     .   D====================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0123456
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER    ..   ld3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: [0,1]     D=eE------R    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeER   ..   ld3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: [0,3]     .D=eE------R   ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeeeeeeeER ..   ld3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: [0,5]     . D==eE------R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeeeeeeeER..   ld3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: [0,7]     .  D==eE------R..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D==eeeeeeeeER   ld3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: [0,9]     .   D===eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2455,43 +2455,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: 3.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: 5.     1     23.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: 7.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: 9.     1     37.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     19.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       ld3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ld3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: 7.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    1.0    0.0       ld3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.3    3.0       <total>
 
 # CHECK:      [28] Code Region - G29
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      759
 # CHECK-NEXT: Total uOps:        3800
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    0.95
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    5.01
+# CHECK-NEXT: IPC:               1.32
 # CHECK-NEXT: Block RThroughput: 7.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    .  .   ld3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: [0,3]     .D===============eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    .  .   ld3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: [0,5]     . D======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    .  .   ld3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D=============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER.   ld3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D====================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0123456
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER    ..   ld3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: [0,1]     D=eE------R    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeER   ..   ld3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE------R   ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeeeeeeeER ..   ld3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,5]     . D==eE------R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeeeeeeeER..   ld3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D==eE------R..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D==eeeeeeeeER   ld3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D===eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2501,43 +2501,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: 3.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: 5.     1     23.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: 7.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: 9.     1     37.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     19.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       ld3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ld3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 7.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    1.0    0.0       ld3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.3    3.0       <total>
 
 # CHECK:      [29] Code Region - G30
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      2010
 # CHECK-NEXT: Total uOps:        3700
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    0.92
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    1.84
+# CHECK-NEXT: IPC:               0.50
 # CHECK-NEXT: Block RThroughput: 7.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    .  .   ld3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: [0,3]     .D===============eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    .  .   ld3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: [0,5]     . D======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    .  .   ld3	{ v1.b, v2.b, v3.b }[0], [x27], #3
-# CHECK-NEXT: [0,7]     .  D=============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER.   ld3	{ v1.b, v2.b, v3.b }[8], [x27], #3
-# CHECK-NEXT: [0,9]     .   D====================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    .   .   ld3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE------R    .    .    .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeER   .    .    .   .   ld3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE------R   .    .    .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeeeeeeeER .    .    .   .   ld3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,5]     . D==eE------R .    .    .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D========eeeeeeeeER   .   .   ld3	{ v1.b, v2.b, v3.b }[0], [x27], #3
+# CHECK-NEXT: [0,7]     .  D=========eE------R   .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D===============eeeeeeeeER   ld3	{ v1.b, v2.b, v3.b }[8], [x27], #3
+# CHECK-NEXT: [0,9]     .   D================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2547,22 +2547,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: 3.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: 5.     1     23.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[0], [x27], #3
-# CHECK-NEXT: 7.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[8], [x27], #3
-# CHECK-NEXT: 9.     1     37.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     19.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       ld3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     9.0    0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[0], [x27], #3
+# CHECK-NEXT: 7.     1     10.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     16.0   0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[8], [x27], #3
+# CHECK-NEXT: 9.     1     17.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     6.3    0.2    3.0       <total>
 
 # CHECK:      [30] Code Region - G31
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        3500
 
 # CHECK:      Dispatch Width:    10
@@ -2572,18 +2572,18 @@ add x0, x27, 1
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld3	{ v1.b, v2.b, v3.b }[0], [x27], x28
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    .  .   ld3	{ v1.b, v2.b, v3.b }[8], [x27], x28
-# CHECK-NEXT: [0,3]     .D===============eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    .  .   ld3	{ v1.h, v2.h, v3.h }[0], [x27], #6
-# CHECK-NEXT: [0,5]     . D======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    .  .   ld3	{ v1.h, v2.h, v3.h }[4], [x27], #6
-# CHECK-NEXT: [0,7]     .  D=============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER.   ld3	{ v1.h, v2.h, v3.h }[0], [x27], x28
-# CHECK-NEXT: [0,9]     .   D====================================eER   add	x0, x27, #1
+# CHECK-NEXT: Index     0123456789          0123456789          012
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld3	{ v1.b, v2.b, v3.b }[0], [x27], x28
+# CHECK-NEXT: [0,1]     D=eE------R    .    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    . .   ld3	{ v1.b, v2.b, v3.b }[8], [x27], x28
+# CHECK-NEXT: [0,3]     .D========eE------R .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    . .   ld3	{ v1.h, v2.h, v3.h }[0], [x27], #6
+# CHECK-NEXT: [0,5]     . D===============eE------R   .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    . .   ld3	{ v1.h, v2.h, v3.h }[4], [x27], #6
+# CHECK-NEXT: [0,7]     .  D======================eE------R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER   ld3	{ v1.h, v2.h, v3.h }[0], [x27], x28
+# CHECK-NEXT: [0,9]     .   D=============================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2593,22 +2593,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.b, v2.b, v3.b }[0], [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[8], [x27], x28
-# CHECK-NEXT: 3.     1     16.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 3.     1     9.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld3	{ v1.h, v2.h, v3.h }[0], [x27], #6
-# CHECK-NEXT: 5.     1     23.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 5.     1     16.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld3	{ v1.h, v2.h, v3.h }[4], [x27], #6
-# CHECK-NEXT: 7.     1     30.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 7.     1     23.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld3	{ v1.h, v2.h, v3.h }[0], [x27], x28
-# CHECK-NEXT: 9.     1     37.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     19.0   0.1    0.0       <total>
+# CHECK-NEXT: 9.     1     30.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     15.5   0.1    3.0       <total>
 
 # CHECK:      [31] Code Region - G32
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        3500
 
 # CHECK:      Dispatch Width:    10
@@ -2618,18 +2618,18 @@ add x0, x27, 1
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld3	{ v1.h, v2.h, v3.h }[4], [x27], x28
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    .  .   ld3	{ v1.s, v2.s, v3.s }[0], [x27], #12
-# CHECK-NEXT: [0,3]     .D===============eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    .  .   ld3	{ v1.s, v2.s, v3.s }[0], [x27], x28
-# CHECK-NEXT: [0,5]     . D======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    .  .   ld3	{ v1.d, v2.d, v3.d }[0], [x27], #24
-# CHECK-NEXT: [0,7]     .  D=============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER.   ld3	{ v1.d, v2.d, v3.d }[0], [x27], x28
-# CHECK-NEXT: [0,9]     .   D====================================eER   add	x0, x27, #1
+# CHECK-NEXT: Index     0123456789          0123456789          012
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld3	{ v1.h, v2.h, v3.h }[4], [x27], x28
+# CHECK-NEXT: [0,1]     D=eE------R    .    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    . .   ld3	{ v1.s, v2.s, v3.s }[0], [x27], #12
+# CHECK-NEXT: [0,3]     .D========eE------R .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    . .   ld3	{ v1.s, v2.s, v3.s }[0], [x27], x28
+# CHECK-NEXT: [0,5]     . D===============eE------R   .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    . .   ld3	{ v1.d, v2.d, v3.d }[0], [x27], #24
+# CHECK-NEXT: [0,7]     .  D======================eE------R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER   ld3	{ v1.d, v2.d, v3.d }[0], [x27], x28
+# CHECK-NEXT: [0,9]     .   D=============================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2639,43 +2639,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.h, v2.h, v3.h }[4], [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld3	{ v1.s, v2.s, v3.s }[0], [x27], #12
-# CHECK-NEXT: 3.     1     16.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 3.     1     9.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld3	{ v1.s, v2.s, v3.s }[0], [x27], x28
-# CHECK-NEXT: 5.     1     23.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 5.     1     16.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld3	{ v1.d, v2.d, v3.d }[0], [x27], #24
-# CHECK-NEXT: 7.     1     30.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 7.     1     23.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld3	{ v1.d, v2.d, v3.d }[0], [x27], x28
-# CHECK-NEXT: 9.     1     37.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     19.0   0.1    0.0       <total>
+# CHECK-NEXT: 9.     1     30.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     15.5   0.1    3.0       <total>
 
 # CHECK:      [32] Code Region - G33
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      759
 # CHECK-NEXT: Total uOps:        3700
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    0.92
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    4.87
+# CHECK-NEXT: IPC:               1.32
 # CHECK-NEXT: Block RThroughput: 7.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    .  .   ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], #24
-# CHECK-NEXT: [0,3]     .D===============eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    .  .   ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], #12
-# CHECK-NEXT: [0,5]     . D======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    .  .   ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], #6
-# CHECK-NEXT: [0,7]     .  D=============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER.   ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], #12
-# CHECK-NEXT: [0,9]     .   D====================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0123456
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER    ..   ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], #24
+# CHECK-NEXT: [0,1]     D=eE------R    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeER   ..   ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], #24
+# CHECK-NEXT: [0,3]     .D=eE------R   ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeeeeeeeER ..   ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], #12
+# CHECK-NEXT: [0,5]     . D==eE------R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeeeeeeeER..   ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], #6
+# CHECK-NEXT: [0,7]     .  D==eE------R..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D==eeeeeeeeER   ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], #12
+# CHECK-NEXT: [0,9]     .   D===eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2685,43 +2685,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], #24
-# CHECK-NEXT: 3.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], #12
-# CHECK-NEXT: 5.     1     23.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], #6
-# CHECK-NEXT: 7.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], #12
-# CHECK-NEXT: 9.     1     37.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     19.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], #24
+# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], #12
+# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], #6
+# CHECK-NEXT: 7.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    1.0    0.0       ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], #12
+# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.3    3.0       <total>
 
 # CHECK:      [33] Code Region - G34
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      759
 # CHECK-NEXT: Total uOps:        3800
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    0.95
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    5.01
+# CHECK-NEXT: IPC:               1.32
 # CHECK-NEXT: Block RThroughput: 7.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], #3
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    .  .   ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], #6
-# CHECK-NEXT: [0,3]     .D===============eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    .  .   ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], #3
-# CHECK-NEXT: [0,5]     . D======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    .  .   ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D=============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER.   ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D====================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0123456
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER    ..   ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], #3
+# CHECK-NEXT: [0,1]     D=eE------R    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeER   ..   ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], #6
+# CHECK-NEXT: [0,3]     .D=eE------R   ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeeeeeeeER ..   ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], #3
+# CHECK-NEXT: [0,5]     . D==eE------R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeeeeeeeER..   ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D==eE------R..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D==eeeeeeeeER   ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D===eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2731,43 +2731,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], #3
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], #6
-# CHECK-NEXT: 3.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], #3
-# CHECK-NEXT: 5.     1     23.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: 7.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: 9.     1     37.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     19.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], #6
+# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], #3
+# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: 7.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    1.0    0.0       ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.3    3.0       <total>
 
 # CHECK:      [34] Code Region - G35
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      759
 # CHECK-NEXT: Total uOps:        3700
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    0.92
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    4.87
+# CHECK-NEXT: IPC:               1.32
 # CHECK-NEXT: Block RThroughput: 7.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    .  .   ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: [0,3]     .D===============eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    .  .   ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: [0,5]     . D======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    .  .   ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D=============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER.   ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D====================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0123456
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER    ..   ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE------R    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeER   ..   ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE------R   ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeeeeeeeER ..   ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: [0,5]     . D==eE------R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeeeeeeeER..   ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D==eE------R..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D==eeeeeeeeER   ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D===eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2777,43 +2777,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: 3.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: 5.     1     23.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: 7.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: 9.     1     37.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     19.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: 7.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    1.0    0.0       ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.3    3.0       <total>
 
 # CHECK:      [35] Code Region - G36
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4204
+# CHECK-NEXT: Total Cycles:      959
 # CHECK-NEXT: Total uOps:        4600
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    1.09
-# CHECK-NEXT: IPC:               0.24
+# CHECK-NEXT: uOps Per Cycle:    4.80
+# CHECK-NEXT: IPC:               1.04
 # CHECK-NEXT: Block RThroughput: 9.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          012345
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .    .   ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeeER.    .    .    .    .    .   ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: [0,3]     .D================eER    .    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D===============eeeeeeeeER  .    .    .    .   ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: [0,5]     . D=======================eER .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D======================eeeeeeeeER    .    .   ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: [0,7]     .  D==============================eER   .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D=============================eeeeeeeeeER.   ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: [0,9]     .   D======================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     012345678
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER    .  .   ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE------R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeeER  .  .   ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: [0,3]     .D=eE-------R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeeeeeeeER .  .   ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: [0,5]     . D==eE------R .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D==eeeeeeeeER  .   ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: [0,7]     .  D===eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D===eeeeeeeeeER   ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: [0,9]     .   D====eE-------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2823,43 +2823,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: 5.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     23.0   0.0    0.0       ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: 7.     1     31.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     30.0   0.0    0.0       ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: 9.     1     39.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     19.8   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: 3.     1     2.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    1.0    0.0       ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    1.0    0.0       ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: 9.     1     5.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.4    3.2       <total>
 
 # CHECK:      [36] Code Region - G37
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4304
+# CHECK-NEXT: Total Cycles:      1008
 # CHECK-NEXT: Total uOps:        4800
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    1.12
-# CHECK-NEXT: IPC:               0.23
+# CHECK-NEXT: uOps Per Cycle:    4.76
+# CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 10.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123456
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .    ..   ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeeER.    .    .    .    .    ..   ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: [0,3]     .D================eER    .    .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D===============eeeeeeeeeER .    .    .    ..   ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: [0,5]     . D========================eER.    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=======================eeeeeeeeeER  .    ..   ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D================================eER .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D===============================eeeeeeeeER.   ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D=======================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     012345678
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER    .  .   ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: [0,1]     D=eE------R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeeER  .  .   ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: [0,3]     .D=eE-------R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D==eeeeeeeeeER  .   ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: [0,5]     . D===eE-------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D==eeeeeeeeeER .   ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D===eE-------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D====eeeeeeeeER   ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D=====eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2869,43 +2869,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: 5.     1     25.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: 7.     1     33.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     32.0   0.0    0.0       ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: 9.     1     40.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.5   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: 3.     1     2.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    2.0    0.0       ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: 5.     1     4.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     5.0    2.0    0.0       ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 9.     1     6.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.1    0.5    3.3       <total>
 
 # CHECK:      [37] Code Region - G38
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4304
+# CHECK-NEXT: Total Cycles:      1009
 # CHECK-NEXT: Total uOps:        4800
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    1.12
-# CHECK-NEXT: IPC:               0.23
+# CHECK-NEXT: uOps Per Cycle:    4.76
+# CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 10.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123456
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .    ..   ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeeER.    .    .    .    .    ..   ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: [0,3]     .D================eER    .    .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D===============eeeeeeeeER  .    .    .    ..   ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: [0,5]     . D=======================eER .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D======================eeeeeeeeeER   .    ..   ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D===============================eER  .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D==============================eeeeeeeeeER.   ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D=======================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER    .   .   ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE------R    .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeeER  .   .   ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE-------R  .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D==eeeeeeeeER.   .   ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: [0,5]     . D===eE------R.   .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D==eeeeeeeeeER  .   ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D===eE-------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D====eeeeeeeeeER   ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D=====eE-------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2915,22 +2915,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: 5.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     23.0   0.0    0.0       ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: 7.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     31.0   0.0    0.0       ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: 9.     1     40.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.1   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    2.0    0.0       ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: 5.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     5.0    2.0    0.0       ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: 9.     1     6.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.1    0.5    3.3       <total>
 
 # CHECK:      [38] Code Region - G39
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        4500
 
 # CHECK:      Dispatch Width:    10
@@ -2940,18 +2940,18 @@ add x0, x27, 1
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    .  .   ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
-# CHECK-NEXT: [0,3]     .D===============eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    .  .   ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
-# CHECK-NEXT: [0,5]     . D======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    .  .   ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
-# CHECK-NEXT: [0,7]     .  D=============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER.   ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
-# CHECK-NEXT: [0,9]     .   D====================================eER   add	x0, x27, #1
+# CHECK-NEXT: Index     0123456789          0123456789          012
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
+# CHECK-NEXT: [0,1]     D=eE------R    .    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
+# CHECK-NEXT: [0,3]     .D========eE------R .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
+# CHECK-NEXT: [0,5]     . D===============eE------R   .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
+# CHECK-NEXT: [0,7]     .  D======================eE------R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER   ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
+# CHECK-NEXT: [0,9]     .   D=============================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2961,22 +2961,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
-# CHECK-NEXT: 3.     1     16.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 3.     1     9.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
-# CHECK-NEXT: 5.     1     23.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 5.     1     16.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
-# CHECK-NEXT: 7.     1     30.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 7.     1     23.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
-# CHECK-NEXT: 9.     1     37.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     19.0   0.1    0.0       <total>
+# CHECK-NEXT: 9.     1     30.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     15.5   0.1    3.0       <total>
 
 # CHECK:      [39] Code Region - G40
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        4500
 
 # CHECK:      Dispatch Width:    10
@@ -2986,18 +2986,18 @@ add x0, x27, 1
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    .  .   ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
-# CHECK-NEXT: [0,3]     .D===============eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    .  .   ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
-# CHECK-NEXT: [0,5]     . D======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    .  .   ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
-# CHECK-NEXT: [0,7]     .  D=============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER.   ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
-# CHECK-NEXT: [0,9]     .   D====================================eER   add	x0, x27, #1
+# CHECK-NEXT: Index     0123456789          0123456789          012
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
+# CHECK-NEXT: [0,1]     D=eE------R    .    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    . .   ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .D========eE------R .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    . .   ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
+# CHECK-NEXT: [0,5]     . D===============eE------R   .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    . .   ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
+# CHECK-NEXT: [0,7]     .  D======================eE------R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER   ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
+# CHECK-NEXT: [0,9]     .   D=============================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3007,43 +3007,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
-# CHECK-NEXT: 3.     1     16.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 3.     1     9.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
-# CHECK-NEXT: 5.     1     23.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 5.     1     16.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
-# CHECK-NEXT: 7.     1     30.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 7.     1     23.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
-# CHECK-NEXT: 9.     1     37.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     19.0   0.1    0.0       <total>
+# CHECK-NEXT: 9.     1     30.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     15.5   0.1    3.0       <total>
 
 # CHECK:      [40] Code Region - G41
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      2103
 # CHECK-NEXT: Total uOps:        4600
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    1.15
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    2.19
+# CHECK-NEXT: IPC:               0.48
 # CHECK-NEXT: Block RThroughput: 10.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    .  .   ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
-# CHECK-NEXT: [0,3]     .D===============eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    .  .   ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: [0,5]     . D======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    .  .   ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #32
-# CHECK-NEXT: [0,7]     .  D=============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER.   ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #16
-# CHECK-NEXT: [0,9]     .   D====================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          0123
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .  .   ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
+# CHECK-NEXT: [0,1]     D=eE------R    .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .  .   ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .D========eE------R .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=======eeeeeeeeER.  .   ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: [0,5]     . D========eE------R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=========eeeeeeeeER.   ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #32
+# CHECK-NEXT: [0,7]     .  D==========eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D=========eeeeeeeeER   ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #16
+# CHECK-NEXT: [0,9]     .   D==========eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3053,43 +3053,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
-# CHECK-NEXT: 3.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: 5.     1     23.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #32
-# CHECK-NEXT: 7.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #16
-# CHECK-NEXT: 9.     1     37.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     19.0   0.1    0.0       <total>
+# CHECK-NEXT: 3.     1     9.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     8.0    0.0    0.0       ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: 5.     1     9.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     10.0   2.0    0.0       ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #32
+# CHECK-NEXT: 7.     1     11.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     10.0   0.0    0.0       ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #16
+# CHECK-NEXT: 9.     1     11.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     7.9    0.3    3.0       <total>
 
 # CHECK:      [41] Code Region - G42
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      1008
 # CHECK-NEXT: Total uOps:        4800
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    1.20
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    4.76
+# CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 10.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #8
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    .  .   ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #16
-# CHECK-NEXT: [0,3]     .D===============eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    .  .   ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #4
-# CHECK-NEXT: [0,5]     . D======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    .  .   ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #8
-# CHECK-NEXT: [0,7]     .  D=============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER.   ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #4
-# CHECK-NEXT: [0,9]     .   D====================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     012345678
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER    .  .   ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #8
+# CHECK-NEXT: [0,1]     D=eE------R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeER   .  .   ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #16
+# CHECK-NEXT: [0,3]     .D=eE------R   .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D==eeeeeeeeER.  .   ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #4
+# CHECK-NEXT: [0,5]     . D===eE------R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D==eeeeeeeeER  .   ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #8
+# CHECK-NEXT: [0,7]     .  D===eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D====eeeeeeeeER   ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #4
+# CHECK-NEXT: [0,9]     .   D=====eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3099,43 +3099,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #8
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #16
-# CHECK-NEXT: 3.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #4
-# CHECK-NEXT: 5.     1     23.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #8
-# CHECK-NEXT: 7.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #4
-# CHECK-NEXT: 9.     1     37.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     19.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #16
+# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    2.0    0.0       ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #4
+# CHECK-NEXT: 5.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #8
+# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     5.0    2.0    0.0       ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #4
+# CHECK-NEXT: 9.     1     6.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.1    0.5    3.0       <total>
 
 # CHECK:      [42] Code Region - G43
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      1008
 # CHECK-NEXT: Total uOps:        4700
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    1.17
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    4.66
+# CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 10.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    .  .   ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: [0,3]     .D===============eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    .  .   ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: [0,5]     . D======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    .  .   ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D=============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER.   ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D====================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     012345678
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER    .  .   ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE------R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeER   .  .   ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE------R   .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D==eeeeeeeeER.  .   ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,5]     . D===eE------R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D==eeeeeeeeER  .   ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D===eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D====eeeeeeeeER   ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D=====eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3145,43 +3145,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: 3.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: 5.     1     23.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: 7.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: 9.     1     37.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     19.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    2.0    0.0       ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 5.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     5.0    2.0    0.0       ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: 9.     1     6.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.1    0.5    3.0       <total>
 
 # CHECK:      [43] Code Region - G44
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3604
+# CHECK-NEXT: Total Cycles:      708
 # CHECK-NEXT: Total uOps:        3900
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    1.08
-# CHECK-NEXT: IPC:               0.28
+# CHECK-NEXT: uOps Per Cycle:    5.51
+# CHECK-NEXT: IPC:               1.41
 # CHECK-NEXT: Block RThroughput: 6.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .   .   ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .   .   ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: [0,3]     .D===============eER.    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .   .   ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: [0,5]     . D======================eER  .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=====================eeeeeeER  .   .   ldp	s1, s2, [x27], #248
-# CHECK-NEXT: [0,7]     .  D===========================eER .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .  D===========================eeeeeeER.   ldp	d1, d2, [x27], #496
-# CHECK-NEXT: [0,9]     .  D=================================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeeER   .   ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeER  .   ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D==eeeeeeeeER   ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,5]     . D===eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D==eeeeeeE-R   ldp	s1, s2, [x27], #248
+# CHECK-NEXT: [0,7]     .  D===eE-----R   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .  D===eeeeeeER   ldp	d1, d2, [x27], #496
+# CHECK-NEXT: [0,9]     .  D====eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3191,43 +3191,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: 3.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: 5.     1     23.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     22.0   0.0    0.0       ldp	s1, s2, [x27], #248
-# CHECK-NEXT: 7.     1     28.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     28.0   0.0    0.0       ldp	d1, d2, [x27], #496
-# CHECK-NEXT: 9.     1     34.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     18.4   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    2.0    0.0       ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: 5.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    1.0       ldp	s1, s2, [x27], #248
+# CHECK-NEXT: 7.     1     4.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ldp	d1, d2, [x27], #496
+# CHECK-NEXT: 9.     1     5.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.9    0.3    2.8       <total>
 
 # CHECK:      [44] Code Region - G45
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2506
+# CHECK-NEXT: Total Cycles:      507
 # CHECK-NEXT: Total uOps:        2800
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    1.12
-# CHECK-NEXT: IPC:               0.40
+# CHECK-NEXT: uOps Per Cycle:    5.52
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .   ldp	q1, q2, [x27], #992
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=====eeeeeeER.    .    .    .   ldp	s1, s2, [x27, #248]!
-# CHECK-NEXT: [0,3]     .D===========eER    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===========eeeeeeER    .    .   ldp	d1, d2, [x27, #496]!
-# CHECK-NEXT: [0,5]     .D=================eER   .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     . D================eeeeeeER   .   ldp	q1, q2, [x27, #992]!
-# CHECK-NEXT: [0,7]     . D======================eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D======================eeeeER   ldp	w1, w2, [x27], #248
-# CHECK-NEXT: [0,9]     .  D======================eE--R   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER ..   ldp	q1, q2, [x27], #992
+# CHECK-NEXT: [0,1]     D=eE----R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeER..   ldp	s1, s2, [x27, #248]!
+# CHECK-NEXT: [0,3]     .D=eE----R..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeeER.   ldp	d1, d2, [x27, #496]!
+# CHECK-NEXT: [0,5]     .D==eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     . D=eeeeeeER   ldp	q1, q2, [x27, #992]!
+# CHECK-NEXT: [0,7]     . D==eE----R   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeE-R   ldp	w1, w2, [x27], #248
+# CHECK-NEXT: [0,9]     .  D==eE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3237,43 +3237,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldp	q1, q2, [x27], #992
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     6.0    0.0    0.0       ldp	s1, s2, [x27, #248]!
-# CHECK-NEXT: 3.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     12.0   0.0    0.0       ldp	d1, d2, [x27, #496]!
-# CHECK-NEXT: 5.     1     18.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     17.0   0.0    0.0       ldp	q1, q2, [x27, #992]!
-# CHECK-NEXT: 7.     1     23.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     23.0   0.0    0.0       ldp	w1, w2, [x27], #248
-# CHECK-NEXT: 9.     1     23.0   0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT:        1     14.2   0.1    0.2       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ldp	s1, s2, [x27, #248]!
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ldp	d1, d2, [x27, #496]!
+# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ldp	q1, q2, [x27, #992]!
+# CHECK-NEXT: 7.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    1.0       ldp	w1, w2, [x27], #248
+# CHECK-NEXT: 9.     1     3.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.2    0.1    2.0       <total>
 
 # CHECK:      [45] Code Region - G46
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1304
+# CHECK-NEXT: Total Cycles:      1006
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    1.53
-# CHECK-NEXT: IPC:               0.77
+# CHECK-NEXT: uOps Per Cycle:    1.99
+# CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 10.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456
+# CHECK-NEXT:                     012345
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeER   .    ..   ldp	x1, x2, [x27], #496
-# CHECK-NEXT: [0,1]     D=eE--R   .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeER  .    ..   ldp	w1, w2, [x27, #248]!
-# CHECK-NEXT: [0,3]     D==eE--R  .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeER .    ..   ldp	x1, x2, [x27, #496]!
-# CHECK-NEXT: [0,5]     .D==eE--R .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeER    ..   ldpsw	x1, x2, [x27], #248
-# CHECK-NEXT: [0,7]     .D=======eER   ..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D======eeeeeER.   ldpsw	x1, x2, [x27, #248]!
-# CHECK-NEXT: [0,9]     . D===========eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeER   .    .   ldp	x1, x2, [x27], #496
+# CHECK-NEXT: [0,1]     D=eE--R   .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeER  .    .   ldp	w1, w2, [x27, #248]!
+# CHECK-NEXT: [0,3]     D==eE--R  .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeER .    .   ldp	x1, x2, [x27, #496]!
+# CHECK-NEXT: [0,5]     .D==eE--R .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeER    .   ldpsw	x1, x2, [x27], #248
+# CHECK-NEXT: [0,7]     .D===eE---R    .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D======eeeeeER   ldpsw	x1, x2, [x27, #248]!
+# CHECK-NEXT: [0,9]     . D=======eE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3289,10 +3289,10 @@ add x0, x27, 1
 # CHECK-NEXT: 4.     1     2.0    0.0    0.0       ldp	x1, x2, [x27, #496]!
 # CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     3.0    0.0    0.0       ldpsw	x1, x2, [x27], #248
-# CHECK-NEXT: 7.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     7.0    0.0    0.0       ldpsw	x1, x2, [x27, #248]!
-# CHECK-NEXT: 9.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     4.3    0.1    0.6       <total>
+# CHECK-NEXT: 7.     1     4.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     7.0    4.0    0.0       ldpsw	x1, x2, [x27, #248]!
+# CHECK-NEXT: 9.     1     8.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.5    0.5    1.2       <total>
 
 # CHECK:      [46] Code Region - G47
 
@@ -3344,28 +3344,28 @@ add x0, x27, 1
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3004
+# CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    0.67
-# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: uOps Per Cycle:    3.94
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 3.8
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .  .   ldr	b1, [x27, #254]!
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D======eeeeeeER.    .    .    .  .   ldr	h1, [x27, #254]!
-# CHECK-NEXT: [0,3]     D============eER    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===========eeeeeeER    .    .  .   ldr	s1, [x27, #254]!
-# CHECK-NEXT: [0,5]     .D=================eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=================eeeeeeER   .  .   ldr	d1, [x27, #254]!
-# CHECK-NEXT: [0,7]     .D=======================eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D======================eeeeeeER.   ldr	q1, [x27, #254]!
-# CHECK-NEXT: [0,9]     . D============================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER . .   ldr	b1, [x27, #254]!
+# CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ldr	h1, [x27, #254]!
+# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeeER .   ldr	s1, [x27, #254]!
+# CHECK-NEXT: [0,5]     .D==eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeER.   ldr	d1, [x27, #254]!
+# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeeER   ldr	q1, [x27, #254]!
+# CHECK-NEXT: [0,9]     . D===eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3375,16 +3375,16 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldr	b1, [x27, #254]!
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ldr	h1, [x27, #254]!
-# CHECK-NEXT: 3.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     12.0   0.0    0.0       ldr	s1, [x27, #254]!
-# CHECK-NEXT: 5.     1     18.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     18.0   0.0    0.0       ldr	d1, [x27, #254]!
-# CHECK-NEXT: 7.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     23.0   0.0    0.0       ldr	q1, [x27, #254]!
-# CHECK-NEXT: 9.     1     29.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     15.2   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ldr	h1, [x27, #254]!
+# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ldr	s1, [x27, #254]!
+# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ldr	d1, [x27, #254]!
+# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ldr	q1, [x27, #254]!
+# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    2.0       <total>
 
 # CHECK:      [48] Code Region - G49
 
@@ -3528,28 +3528,27 @@ add x0, x27, 1
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      704
+# CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        1700
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    2.41
-# CHECK-NEXT: IPC:               1.42
+# CHECK-NEXT: uOps Per Cycle:    3.37
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     012345678
 
-# CHECK:      [0,0]     DeeeeER   .   ldrsh	x1, [x27, #254]!
-# CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeER  .   ldrsw	x1, [x27], #254
-# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D==eeeeER .   ldrsw	x1, [x27, #254]!
-# CHECK-NEXT: [0,5]     D===eE--R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeE-R .   st1	{ v1.1d }, [x27], #8
-# CHECK-NEXT: [0,7]     .D====eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D====eeER.   st1	{ v1.2d }, [x27], #16
-# CHECK-NEXT: [0,9]     .D======eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeER .   ldrsh	x1, [x27, #254]!
+# CHECK-NEXT: [0,1]     D=eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeER.   ldrsw	x1, [x27], #254
+# CHECK-NEXT: [0,3]     D==eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeER   ldrsw	x1, [x27, #254]!
+# CHECK-NEXT: [0,5]     D===eE--R   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeE-R   st1	{ v1.1d }, [x27], #8
+# CHECK-NEXT: [0,7]     .D===eE-R   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeER   st1	{ v1.2d }, [x27], #16
+# CHECK-NEXT: [0,9]     .D====eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3565,37 +3564,36 @@ add x0, x27, 1
 # CHECK-NEXT: 4.     1     3.0    0.0    0.0       ldrsw	x1, [x27, #254]!
 # CHECK-NEXT: 5.     1     4.0    0.0    2.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     3.0    0.0    1.0       st1	{ v1.1d }, [x27], #8
-# CHECK-NEXT: 7.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     5.0    0.0    0.0       st1	{ v1.2d }, [x27], #16
-# CHECK-NEXT: 9.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.5    0.1    0.7       <total>
+# CHECK-NEXT: 7.     1     4.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       st1	{ v1.2d }, [x27], #16
+# CHECK-NEXT: 9.     1     5.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.1    0.1    0.8       <total>
 
 # CHECK:      [52] Code Region - G53
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1004
+# CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    1.99
-# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: uOps Per Cycle:    3.97
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     012345678
 
-# CHECK:      [0,0]     DeeER.    .  .   st1	{ v1.2s }, [x27], #8
-# CHECK-NEXT: [0,1]     D==eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D==eeER   .  .   st1	{ v1.4h }, [x27], #8
-# CHECK-NEXT: [0,3]     D====eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===eeER .  .   st1	{ v1.4s }, [x27], #16
-# CHECK-NEXT: [0,5]     .D=====eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=====eeER  .   st1	{ v1.8b }, [x27], #8
-# CHECK-NEXT: [0,7]     .D=======eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D======eeER.   st1	{ v1.8h }, [x27], #16
-# CHECK-NEXT: [0,9]     . D========eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.  .   st1	{ v1.2s }, [x27], #8
+# CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeER  .   st1	{ v1.4h }, [x27], #8
+# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeER .   st1	{ v1.4s }, [x27], #16
+# CHECK-NEXT: [0,5]     .D==eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeER.   st1	{ v1.8b }, [x27], #8
+# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeER   st1	{ v1.8h }, [x27], #16
+# CHECK-NEXT: [0,9]     . D===eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3605,43 +3603,42 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.2s }, [x27], #8
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st1	{ v1.4h }, [x27], #8
-# CHECK-NEXT: 3.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       st1	{ v1.4s }, [x27], #16
-# CHECK-NEXT: 5.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     6.0    0.0    0.0       st1	{ v1.8b }, [x27], #8
-# CHECK-NEXT: 7.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     7.0    0.0    0.0       st1	{ v1.8h }, [x27], #16
-# CHECK-NEXT: 9.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     5.2    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.4h }, [x27], #8
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st1	{ v1.4s }, [x27], #16
+# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st1	{ v1.8b }, [x27], #8
+# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st1	{ v1.8h }, [x27], #16
+# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    0.0       <total>
 
 # CHECK:      [53] Code Region - G54
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1004
+# CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    1.99
-# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: uOps Per Cycle:    3.97
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     012345678
 
-# CHECK:      [0,0]     DeeER.    .  .   st1	{ v1.16b }, [x27], #16
-# CHECK-NEXT: [0,1]     D==eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D==eeER   .  .   st1	{ v1.1d }, [x27], x28
-# CHECK-NEXT: [0,3]     D====eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===eeER .  .   st1	{ v1.2d }, [x27], x28
-# CHECK-NEXT: [0,5]     .D=====eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=====eeER  .   st1	{ v1.2s }, [x27], x28
-# CHECK-NEXT: [0,7]     .D=======eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D======eeER.   st1	{ v1.4h }, [x27], x28
-# CHECK-NEXT: [0,9]     . D========eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.  .   st1	{ v1.16b }, [x27], #16
+# CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeER  .   st1	{ v1.1d }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeER .   st1	{ v1.2d }, [x27], x28
+# CHECK-NEXT: [0,5]     .D==eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeER.   st1	{ v1.2s }, [x27], x28
+# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeER   st1	{ v1.4h }, [x27], x28
+# CHECK-NEXT: [0,9]     . D===eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3651,43 +3648,42 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.16b }, [x27], #16
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st1	{ v1.1d }, [x27], x28
-# CHECK-NEXT: 3.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       st1	{ v1.2d }, [x27], x28
-# CHECK-NEXT: 5.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     6.0    0.0    0.0       st1	{ v1.2s }, [x27], x28
-# CHECK-NEXT: 7.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     7.0    0.0    0.0       st1	{ v1.4h }, [x27], x28
-# CHECK-NEXT: 9.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     5.2    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.1d }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st1	{ v1.2d }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st1	{ v1.2s }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st1	{ v1.4h }, [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    0.0       <total>
 
 # CHECK:      [54] Code Region - G55
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1004
+# CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    1.99
-# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: uOps Per Cycle:    3.97
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     012345678
 
-# CHECK:      [0,0]     DeeER.    .  .   st1	{ v1.4s }, [x27], x28
-# CHECK-NEXT: [0,1]     D==eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D==eeER   .  .   st1	{ v1.8b }, [x27], x28
-# CHECK-NEXT: [0,3]     D====eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===eeER .  .   st1	{ v1.8h }, [x27], x28
-# CHECK-NEXT: [0,5]     .D=====eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=====eeER  .   st1	{ v1.16b }, [x27], x28
-# CHECK-NEXT: [0,7]     .D=======eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D======eeER.   st1	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: [0,9]     . D========eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.  .   st1	{ v1.4s }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeER  .   st1	{ v1.8b }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeER .   st1	{ v1.8h }, [x27], x28
+# CHECK-NEXT: [0,5]     .D==eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeER.   st1	{ v1.16b }, [x27], x28
+# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeER   st1	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: [0,9]     . D===eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3697,43 +3693,42 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.4s }, [x27], x28
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st1	{ v1.8b }, [x27], x28
-# CHECK-NEXT: 3.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       st1	{ v1.8h }, [x27], x28
-# CHECK-NEXT: 5.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     6.0    0.0    0.0       st1	{ v1.16b }, [x27], x28
-# CHECK-NEXT: 7.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     7.0    0.0    0.0       st1	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: 9.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     5.2    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.8b }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st1	{ v1.8h }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st1	{ v1.16b }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st1	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    0.0       <total>
 
 # CHECK:      [55] Code Region - G56
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1004
+# CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2400
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    2.39
-# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: uOps Per Cycle:    4.76
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 3.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     012345678
 
-# CHECK:      [0,0]     DeeER.    .  .   st1	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: [0,1]     D==eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D==eeER   .  .   st1	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: [0,3]     D====eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===eeER .  .   st1	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: [0,5]     .D=====eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=====eeER  .   st1	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: [0,7]     .D=======eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D======eeER.   st1	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: [0,9]     . D========eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.  .   st1	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeER  .   st1	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeER .   st1	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: [0,5]     .D==eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeER.   st1	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeER   st1	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: [0,9]     . D===eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3743,43 +3738,42 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st1	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: 3.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       st1	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: 5.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     6.0    0.0    0.0       st1	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: 7.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     7.0    0.0    0.0       st1	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: 9.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     5.2    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st1	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st1	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st1	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    0.0       <total>
 
 # CHECK:      [56] Code Region - G57
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1004
+# CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2600
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    2.59
-# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: uOps Per Cycle:    5.16
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     012345678
 
-# CHECK:      [0,0]     DeeER.    .  .   st1	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: [0,1]     D==eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=eeER   .  .   st1	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: [0,3]     .D===eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===eeER .  .   st1	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: [0,5]     .D=====eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     . D====eeER  .   st1	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: [0,7]     . D======eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D======eeER.   st1	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: [0,9]     . D========eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.  .   st1	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeER  .   st1	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: [0,3]     .D=eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeER .   st1	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: [0,5]     .D==eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     . D=eeER.   st1	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,7]     . D==eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeER   st1	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,9]     . D===eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3789,43 +3783,42 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       st1	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: 5.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     5.0    0.0    0.0       st1	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: 7.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     7.0    0.0    0.0       st1	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: 9.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     4.8    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st1	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st1	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: 7.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st1	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.1    0.0       <total>
 
 # CHECK:      [57] Code Region - G58
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1004
+# CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2600
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    2.59
-# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: uOps Per Cycle:    5.16
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     012345678
 
-# CHECK:      [0,0]     DeeER.    .  .   st1	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: [0,1]     D==eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D==eeER   .  .   st1	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: [0,3]     D====eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===eeER .  .   st1	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: [0,5]     .D=====eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=====eeER  .   st1	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: [0,7]     .D=======eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D======eeER.   st1	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: [0,9]     . D========eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.  .   st1	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeER  .   st1	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeER .   st1	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,5]     .D==eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeER.   st1	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeER   st1	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,9]     . D===eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3835,43 +3828,42 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st1	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: 3.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       st1	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: 5.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     6.0    0.0    0.0       st1	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: 7.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     7.0    0.0    0.0       st1	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: 9.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     5.2    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st1	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st1	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st1	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    0.0       <total>
 
 # CHECK:      [58] Code Region - G59
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1004
+# CHECK-NEXT: Total Cycles:      703
 # CHECK-NEXT: Total uOps:        3400
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    3.39
-# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: uOps Per Cycle:    4.84
+# CHECK-NEXT: IPC:               1.42
 # CHECK-NEXT: Block RThroughput: 6.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeER.    .  .   st1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: [0,1]     D==eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=eeER   .  .   st1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: [0,3]     .D===eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==eeER .  .   st1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: [0,5]     . D====eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D===eeER  .   st1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: [0,7]     .  D=====eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D====eeER.   st1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: [0,9]     .   D======eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.   .   st1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
+# CHECK-NEXT: [0,1]     D=eER.   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeER   .   st1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: [0,3]     .D=eER   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeER  .   st1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: [0,5]     . D=eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeER.   st1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: [0,7]     .  D==eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D=eeER   st1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: [0,9]     .   D==eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3881,43 +3873,42 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: 5.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     4.0    0.0    0.0       st1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: 7.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     5.0    0.0    0.0       st1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: 9.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     4.0    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    1.0    0.0       st1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: 7.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     2.0    0.0    0.0       st1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: 9.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.9    0.2    0.0       <total>
 
 # CHECK:      [59] Code Region - G60
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1004
+# CHECK-NEXT: Total Cycles:      703
 # CHECK-NEXT: Total uOps:        3600
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    3.59
-# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: uOps Per Cycle:    5.12
+# CHECK-NEXT: IPC:               1.42
 # CHECK-NEXT: Block RThroughput: 6.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeER.    .  .   st1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: [0,1]     D==eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=eeER   .  .   st1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: [0,3]     .D===eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==eeER .  .   st1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: [0,5]     . D====eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D===eeER  .   st1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D=====eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D====eeER.   st1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D======eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.   .   st1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: [0,1]     D=eER.   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeER   .   st1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: [0,3]     .D=eER   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeER  .   st1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: [0,5]     . D=eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeER.   st1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D==eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D=eeER   st1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D==eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3927,43 +3918,42 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: 5.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     4.0    0.0    0.0       st1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: 7.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     5.0    0.0    0.0       st1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: 9.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     4.0    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    1.0    0.0       st1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: 7.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     2.0    0.0    0.0       st1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: 9.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.9    0.2    0.0       <total>
 
 # CHECK:      [60] Code Region - G61
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1004
+# CHECK-NEXT: Total Cycles:      703
 # CHECK-NEXT: Total uOps:        3400
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    3.39
-# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: uOps Per Cycle:    4.84
+# CHECK-NEXT: IPC:               1.42
 # CHECK-NEXT: Block RThroughput: 6.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeER.    .  .   st1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: [0,1]     D==eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=eeER   .  .   st1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: [0,3]     .D===eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==eeER .  .   st1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: [0,5]     . D====eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D===eeER  .   st1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D=====eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D====eeER.   st1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D======eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.   .   st1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eER.   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeER   .   st1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eER   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeER  .   st1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: [0,5]     . D=eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeER .   st1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D=eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D=eeER   st1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D==eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3973,43 +3963,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: 5.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     4.0    0.0    0.0       st1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: 7.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     5.0    0.0    0.0       st1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: 9.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     4.0    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: 7.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     2.0    1.0    0.0       st1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: 9.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.7    0.2    0.0       <total>
 
 # CHECK:      [61] Code Region - G62
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1004
+# CHECK-NEXT: Total Cycles:      704
 # CHECK-NEXT: Total uOps:        3600
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    3.59
-# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: uOps Per Cycle:    5.11
+# CHECK-NEXT: IPC:               1.42
 # CHECK-NEXT: Block RThroughput: 6.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123
+# CHECK-NEXT:                     0
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeER.    .  .   st1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: [0,1]     D==eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=eeER   .  .   st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: [0,3]     .D===eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==eeER .  .   st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: [0,5]     . D====eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D===eeER  .   st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: [0,7]     .  D=====eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D====eeER.   st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: [0,9]     .   D======eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.    .   st1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eER.    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeER    .   st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: [0,3]     .D=eER    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeER  .   st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: [0,5]     . D==eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeER .   st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: [0,7]     .  D==eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D==eeER   st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: [0,9]     .   D===eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4019,43 +4009,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: 5.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     4.0    0.0    0.0       st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: 7.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     5.0    0.0    0.0       st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: 9.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     4.0    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: 7.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    1.0    0.0       st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.3    0.0       <total>
 
 # CHECK:      [62] Code Region - G63
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1004
+# CHECK-NEXT: Total Cycles:      804
 # CHECK-NEXT: Total uOps:        4200
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    4.18
-# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: uOps Per Cycle:    5.22
+# CHECK-NEXT: IPC:               1.24
 # CHECK-NEXT: Block RThroughput: 8.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123
+# CHECK-NEXT:                     01
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeER.    .  .   st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: [0,1]     D==eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=eeER   .  .   st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: [0,3]     .D===eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==eeER .  .   st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: [0,5]     . D====eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D===eeER  .   st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: [0,7]     .  D=====eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D====eeER.   st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D======eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.    ..   st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: [0,1]     D=eER.    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeER    ..   st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: [0,3]     .D=eER    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeER  ..   st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: [0,5]     . D==eER  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeER ..   st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: [0,7]     .  D==eER ..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D===eeER   st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D====eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4065,43 +4055,42 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: 5.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     4.0    0.0    0.0       st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: 7.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     5.0    0.0    0.0       st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: 9.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     4.0    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: 7.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    2.0    0.0       st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK-NEXT: 9.     1     5.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.5    0.4    0.0       <total>
 
 # CHECK:      [63] Code Region - G64
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1004
+# CHECK-NEXT: Total Cycles:      703
 # CHECK-NEXT: Total uOps:        3800
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    3.78
-# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: uOps Per Cycle:    5.41
+# CHECK-NEXT: IPC:               1.42
 # CHECK-NEXT: Block RThroughput: 7.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeER.    .  .   st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: [0,1]     D==eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=eeER   .  .   st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: [0,3]     .D===eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==eeER .  .   st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: [0,5]     . D====eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D===eeER  .   st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D=====eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D====eeER.   st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D======eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.   .   st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eER.   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeER   .   st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eER   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeER .   st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,5]     . D==eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeER.   st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D==eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D=eeER   st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D==eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4111,43 +4100,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: 5.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     4.0    0.0    0.0       st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: 7.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     5.0    0.0    0.0       st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: 9.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     4.0    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: 7.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     2.0    0.0    0.0       st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: 9.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.1    0.2    0.0       <total>
 
 # CHECK:      [64] Code Region - G65
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1604
+# CHECK-NEXT: Total Cycles:      706
 # CHECK-NEXT: Total uOps:        3200
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    2.00
-# CHECK-NEXT: IPC:               0.62
+# CHECK-NEXT: uOps Per Cycle:    4.53
+# CHECK-NEXT: IPC:               1.42
 # CHECK-NEXT: Block RThroughput: 5.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
+# CHECK-NEXT:                     012
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeER.    .    .   .   st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: [0,1]     D==eER    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=eeER   .    .   .   st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: [0,3]     .D===eER  .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==eeeeER    .   .   st1	{ v1.b }[0], [x27], #1
-# CHECK-NEXT: [0,5]     . D======eER   .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     . D======eeeeER.   .   st1	{ v1.b }[8], [x27], #1
-# CHECK-NEXT: [0,7]     . D==========eER   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .  D=========eeeeER.   st1	{ v1.b }[0], [x27], x28
-# CHECK-NEXT: [0,9]     .  D=============eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.    . .   st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eER.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeER    . .   st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eER    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D==eeeeER .   st1	{ v1.b }[0], [x27], #1
+# CHECK-NEXT: [0,5]     . D===eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     . D===eeeeER.   st1	{ v1.b }[8], [x27], #1
+# CHECK-NEXT: [0,7]     . D====eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .  D===eeeeER   st1	{ v1.b }[0], [x27], x28
+# CHECK-NEXT: [0,9]     .  D====eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4157,43 +4146,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.b }[0], [x27], #1
-# CHECK-NEXT: 5.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     7.0    0.0    0.0       st1	{ v1.b }[8], [x27], #1
-# CHECK-NEXT: 7.     1     11.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     10.0   0.0    0.0       st1	{ v1.b }[0], [x27], x28
-# CHECK-NEXT: 9.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     6.2    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    2.0    0.0       st1	{ v1.b }[0], [x27], #1
+# CHECK-NEXT: 5.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     4.0    0.0    0.0       st1	{ v1.b }[8], [x27], #1
+# CHECK-NEXT: 7.     1     5.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       st1	{ v1.b }[0], [x27], x28
+# CHECK-NEXT: 9.     1     5.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.1    0.3    0.6       <total>
 
 # CHECK:      [65] Code Region - G66
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2004
+# CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    1.00
-# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: uOps Per Cycle:    3.95
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0123
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeER   .    .    .  .   st1	{ v1.b }[8], [x27], x28
-# CHECK-NEXT: [0,1]     D====eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D====eeeeER    .    .  .   st1	{ v1.h }[0], [x27], #2
-# CHECK-NEXT: [0,3]     D========eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=======eeeeER.    .  .   st1	{ v1.h }[4], [x27], #2
-# CHECK-NEXT: [0,5]     .D===========eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D===========eeeeER .  .   st1	{ v1.h }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .D===============eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============eeeeER.   st1	{ v1.h }[4], [x27], x28
-# CHECK-NEXT: [0,9]     . D==================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeER   .   st1	{ v1.b }[8], [x27], x28
+# CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeER  .   st1	{ v1.h }[0], [x27], #2
+# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeER .   st1	{ v1.h }[4], [x27], #2
+# CHECK-NEXT: [0,5]     .D==eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeER.   st1	{ v1.h }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .D===eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeER   st1	{ v1.h }[4], [x27], x28
+# CHECK-NEXT: [0,9]     . D===eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4203,43 +4192,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.b }[8], [x27], x28
-# CHECK-NEXT: 1.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       st1	{ v1.h }[0], [x27], #2
-# CHECK-NEXT: 3.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     8.0    0.0    0.0       st1	{ v1.h }[4], [x27], #2
-# CHECK-NEXT: 5.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     12.0   0.0    0.0       st1	{ v1.h }[0], [x27], x28
-# CHECK-NEXT: 7.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     15.0   0.0    0.0       st1	{ v1.h }[4], [x27], x28
-# CHECK-NEXT: 9.     1     19.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     10.2   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.h }[0], [x27], #2
+# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st1	{ v1.h }[4], [x27], #2
+# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st1	{ v1.h }[0], [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st1	{ v1.h }[4], [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    1.0       <total>
 
 # CHECK:      [66] Code Region - G67
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2004
+# CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        2200
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    1.10
-# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: uOps Per Cycle:    4.35
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 3.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0123
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeER   .    .    .  .   st1	{ v1.s }[0], [x27], #4
-# CHECK-NEXT: [0,1]     D====eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D====eeeeER    .    .  .   st1	{ v1.s }[0], [x27], x28
-# CHECK-NEXT: [0,3]     D========eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=======eeeeER.    .  .   st1	{ v1.d }[0], [x27], #8
-# CHECK-NEXT: [0,5]     .D===========eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D===========eeeeER .  .   st1	{ v1.d }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .D===============eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============eeeeER.   st2	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: [0,9]     . D==================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeER   .   st1	{ v1.s }[0], [x27], #4
+# CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeER  .   st1	{ v1.s }[0], [x27], x28
+# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeER .   st1	{ v1.d }[0], [x27], #8
+# CHECK-NEXT: [0,5]     .D==eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeER.   st1	{ v1.d }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .D===eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeER   st2	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: [0,9]     . D===eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4249,43 +4238,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.s }[0], [x27], #4
-# CHECK-NEXT: 1.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       st1	{ v1.s }[0], [x27], x28
-# CHECK-NEXT: 3.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     8.0    0.0    0.0       st1	{ v1.d }[0], [x27], #8
-# CHECK-NEXT: 5.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     12.0   0.0    0.0       st1	{ v1.d }[0], [x27], x28
-# CHECK-NEXT: 7.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     15.0   0.0    0.0       st2	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: 9.     1     19.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     10.2   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.s }[0], [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st1	{ v1.d }[0], [x27], #8
+# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st1	{ v1.d }[0], [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st2	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: 9.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    1.0       <total>
 
 # CHECK:      [67] Code Region - G68
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2004
+# CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        2400
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    1.20
-# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: uOps Per Cycle:    4.74
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 3.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0123
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeER   .    .    .  .   st2	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: [0,1]     D====eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D====eeeeER    .    .  .   st2	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: [0,3]     D========eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=======eeeeER.    .  .   st2	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: [0,5]     .D===========eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D===========eeeeER .  .   st2	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: [0,7]     .D===============eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============eeeeER.   st2	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: [0,9]     . D==================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeER   .   st2	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeER  .   st2	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeER .   st2	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: [0,5]     .D==eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeER.   st2	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: [0,7]     .D===eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeER   st2	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: [0,9]     . D===eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4295,43 +4284,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: 1.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       st2	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: 3.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     8.0    0.0    0.0       st2	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: 5.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     12.0   0.0    0.0       st2	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: 7.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     15.0   0.0    0.0       st2	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: 9.     1     19.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     10.2   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st2	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st2	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st2	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: 7.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st2	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: 9.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    1.0       <total>
 
 # CHECK:      [68] Code Region - G69
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2004
+# CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        2600
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    1.30
-# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: uOps Per Cycle:    5.14
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0123
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeER   .    .    .  .   st2	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: [0,1]     D====eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D===eeeeER    .    .  .   st2	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=======eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=======eeeeER.    .  .   st2	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: [0,5]     .D===========eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     . D==========eeeeER .  .   st2	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: [0,7]     . D==============eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============eeeeER.   st2	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: [0,9]     . D==================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeER   .   st2	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeER  .   st2	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE--R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeER .   st2	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,5]     .D==eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     . D=eeeeER.   st2	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,7]     . D==eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeER   st2	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,9]     . D===eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4341,43 +4330,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: 1.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     4.0    0.0    0.0       st2	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: 3.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     8.0    0.0    0.0       st2	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: 5.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     11.0   0.0    0.0       st2	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: 7.     1     15.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     15.0   0.0    0.0       st2	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: 9.     1     19.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     9.8    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st2	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st2	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st2	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: 7.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st2	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.1    1.0       <total>
 
 # CHECK:      [69] Code Region - G70
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2004
+# CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        2400
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    1.20
-# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: uOps Per Cycle:    4.74
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 3.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0123
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeER   .    .    .  .   st2	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: [0,1]     D====eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D====eeeeER    .    .  .   st2	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: [0,3]     D========eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=======eeeeER.    .  .   st2	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: [0,5]     .D===========eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D===========eeeeER .  .   st2	{ v1.b, v2.b }[0], [x27], #2
-# CHECK-NEXT: [0,7]     .D===============eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============eeeeER.   st2	{ v1.b, v2.b }[8], [x27], #2
-# CHECK-NEXT: [0,9]     . D==================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeER   .   st2	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeER  .   st2	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeER .   st2	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,5]     .D==eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeER.   st2	{ v1.b, v2.b }[0], [x27], #2
+# CHECK-NEXT: [0,7]     .D===eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeER   st2	{ v1.b, v2.b }[8], [x27], #2
+# CHECK-NEXT: [0,9]     . D===eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4387,43 +4376,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: 1.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       st2	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: 3.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     8.0    0.0    0.0       st2	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: 5.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     12.0   0.0    0.0       st2	{ v1.b, v2.b }[0], [x27], #2
-# CHECK-NEXT: 7.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     15.0   0.0    0.0       st2	{ v1.b, v2.b }[8], [x27], #2
-# CHECK-NEXT: 9.     1     19.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     10.2   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st2	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st2	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st2	{ v1.b, v2.b }[0], [x27], #2
+# CHECK-NEXT: 7.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st2	{ v1.b, v2.b }[8], [x27], #2
+# CHECK-NEXT: 9.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    1.0       <total>
 
 # CHECK:      [70] Code Region - G71
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2004
+# CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    1.00
-# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: uOps Per Cycle:    3.95
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0123
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeER   .    .    .  .   st2	{ v1.b, v2.b }[0], [x27], x28
-# CHECK-NEXT: [0,1]     D====eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D====eeeeER    .    .  .   st2	{ v1.b, v2.b }[8], [x27], x28
-# CHECK-NEXT: [0,3]     D========eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=======eeeeER.    .  .   st2	{ v1.h, v2.h }[0], [x27], #4
-# CHECK-NEXT: [0,5]     .D===========eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D===========eeeeER .  .   st2	{ v1.h, v2.h }[4], [x27], #4
-# CHECK-NEXT: [0,7]     .D===============eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============eeeeER.   st2	{ v1.h, v2.h }[0], [x27], x28
-# CHECK-NEXT: [0,9]     . D==================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeER   .   st2	{ v1.b, v2.b }[0], [x27], x28
+# CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeER  .   st2	{ v1.b, v2.b }[8], [x27], x28
+# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeER .   st2	{ v1.h, v2.h }[0], [x27], #4
+# CHECK-NEXT: [0,5]     .D==eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeER.   st2	{ v1.h, v2.h }[4], [x27], #4
+# CHECK-NEXT: [0,7]     .D===eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeER   st2	{ v1.h, v2.h }[0], [x27], x28
+# CHECK-NEXT: [0,9]     . D===eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4433,43 +4422,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.b, v2.b }[0], [x27], x28
-# CHECK-NEXT: 1.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       st2	{ v1.b, v2.b }[8], [x27], x28
-# CHECK-NEXT: 3.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     8.0    0.0    0.0       st2	{ v1.h, v2.h }[0], [x27], #4
-# CHECK-NEXT: 5.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     12.0   0.0    0.0       st2	{ v1.h, v2.h }[4], [x27], #4
-# CHECK-NEXT: 7.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     15.0   0.0    0.0       st2	{ v1.h, v2.h }[0], [x27], x28
-# CHECK-NEXT: 9.     1     19.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     10.2   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st2	{ v1.b, v2.b }[8], [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st2	{ v1.h, v2.h }[0], [x27], #4
+# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st2	{ v1.h, v2.h }[4], [x27], #4
+# CHECK-NEXT: 7.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st2	{ v1.h, v2.h }[0], [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    1.0       <total>
 
 # CHECK:      [71] Code Region - G72
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2004
+# CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    1.00
-# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: uOps Per Cycle:    3.95
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0123
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeER   .    .    .  .   st2	{ v1.h, v2.h }[4], [x27], x28
-# CHECK-NEXT: [0,1]     D====eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D====eeeeER    .    .  .   st2	{ v1.s, v2.s }[0], [x27], #8
-# CHECK-NEXT: [0,3]     D========eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=======eeeeER.    .  .   st2	{ v1.s, v2.s }[0], [x27], x28
-# CHECK-NEXT: [0,5]     .D===========eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D===========eeeeER .  .   st2	{ v1.d, v2.d }[0], [x27], #16
-# CHECK-NEXT: [0,7]     .D===============eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============eeeeER.   st2	{ v1.d, v2.d }[0], [x27], x28
-# CHECK-NEXT: [0,9]     . D==================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeER   .   st2	{ v1.h, v2.h }[4], [x27], x28
+# CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeER  .   st2	{ v1.s, v2.s }[0], [x27], #8
+# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeER .   st2	{ v1.s, v2.s }[0], [x27], x28
+# CHECK-NEXT: [0,5]     .D==eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeER.   st2	{ v1.d, v2.d }[0], [x27], #16
+# CHECK-NEXT: [0,7]     .D===eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeER   st2	{ v1.d, v2.d }[0], [x27], x28
+# CHECK-NEXT: [0,9]     . D===eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4479,43 +4468,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.h, v2.h }[4], [x27], x28
-# CHECK-NEXT: 1.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       st2	{ v1.s, v2.s }[0], [x27], #8
-# CHECK-NEXT: 3.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     8.0    0.0    0.0       st2	{ v1.s, v2.s }[0], [x27], x28
-# CHECK-NEXT: 5.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     12.0   0.0    0.0       st2	{ v1.d, v2.d }[0], [x27], #16
-# CHECK-NEXT: 7.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     15.0   0.0    0.0       st2	{ v1.d, v2.d }[0], [x27], x28
-# CHECK-NEXT: 9.     1     19.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     10.2   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st2	{ v1.s, v2.s }[0], [x27], #8
+# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st2	{ v1.s, v2.s }[0], [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st2	{ v1.d, v2.d }[0], [x27], #16
+# CHECK-NEXT: 7.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st2	{ v1.d, v2.d }[0], [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    1.0       <total>
 
 # CHECK:      [72] Code Region - G73
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1804
+# CHECK-NEXT: Total Cycles:      607
 # CHECK-NEXT: Total uOps:        2800
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    1.55
-# CHECK-NEXT: IPC:               0.55
+# CHECK-NEXT: uOps Per Cycle:    4.61
+# CHECK-NEXT: IPC:               1.65
 # CHECK-NEXT: Block RThroughput: 4.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          01
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeER .    .    .    ..   st2g	x26, [x27], #4064
-# CHECK-NEXT: [0,1]     D=eER.    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eER.    .    .    ..   st2g	x26, [x27, #4064]!
-# CHECK-NEXT: [0,3]     D==eER    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeER    .    ..   st3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: [0,5]     .D=======eER   .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     . D======eeeeeER    ..   st3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: [0,7]     . D===========eER   ..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .  D==========eeeeeER.   st3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: [0,9]     .  D===============eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeER .    . .   st2g	x26, [x27], #4064
+# CHECK-NEXT: [0,1]     D=eER.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eER.    . .   st2g	x26, [x27, #4064]!
+# CHECK-NEXT: [0,3]     D==eER    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeeER .   st3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: [0,5]     .D==eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     . D=eeeeeER .   st3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: [0,7]     . D==eE---R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .  D==eeeeeER   st3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: [0,9]     .  D===eE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4529,39 +4518,39 @@ add x0, x27, 1
 # CHECK-NEXT: 2.     1     2.0    0.0    0.0       st2g	x26, [x27, #4064]!
 # CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     2.0    0.0    0.0       st3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: 5.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     7.0    0.0    0.0       st3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: 7.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     11.0   0.0    0.0       st3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: 9.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     6.4    0.1    0.0       <total>
+# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: 7.     1     3.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    1.0    0.0       st3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: 9.     1     4.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.5    0.2    1.0       <total>
 
 # CHECK:      [73] Code Region - G74
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2904
+# CHECK-NEXT: Total Cycles:      708
 # CHECK-NEXT: Total uOps:        3800
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    1.31
-# CHECK-NEXT: IPC:               0.34
+# CHECK-NEXT: uOps Per Cycle:    5.37
+# CHECK-NEXT: IPC:               1.41
 # CHECK-NEXT: Block RThroughput: 7.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          012
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    . .   st3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=====eeeeeER .    .    .    . .   st3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: [0,3]     .D==========eER.    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=========eeeeeeER.    .    . .   st3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: [0,5]     . D===============eER    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D==============eeeeeeER    . .   st3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: [0,7]     .  D====================eER   . .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D===================eeeeeeER.   st3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D=========================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER .   .   st3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: [0,1]     D=eE----R .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeER .   .   st3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: [0,3]     .D=eE---R .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeeeeeER  .   st3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: [0,5]     . D==eE----R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeeeeeER .   st3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: [0,7]     .  D==eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D==eeeeeeER   st3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D===eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4571,43 +4560,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     6.0    0.0    0.0       st3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: 3.     1     11.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     10.0   0.0    0.0       st3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: 5.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     15.0   0.0    0.0       st3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: 7.     1     21.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     20.0   0.0    0.0       st3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: 9.     1     26.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     13.3   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: 3.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: 7.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    1.0    0.0       st3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.3    1.9       <total>
 
 # CHECK:      [74] Code Region - G75
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2704
+# CHECK-NEXT: Total Cycles:      707
 # CHECK-NEXT: Total uOps:        3400
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    1.26
-# CHECK-NEXT: IPC:               0.37
+# CHECK-NEXT: uOps Per Cycle:    4.81
+# CHECK-NEXT: IPC:               1.41
 # CHECK-NEXT: Block RThroughput: 6.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     0123
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeER  .    .    .    .    .   st3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: [0,1]     D=====eER .    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D====eeeeeER  .    .    .    .   st3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=========eER .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D========eeeeeeER .    .    .   st3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: [0,5]     . D==============eER.    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=============eeeeeER .    .   st3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D==================eER.    .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D=================eeeeeeER.   st3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D=======================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeER  .  .   st3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE---R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeER .  .   st3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE---R .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeER  .   st3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: [0,5]     . D=eE----R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeER  .   st3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D=eE---R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D=eeeeeeER   st3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D==eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4617,43 +4606,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       st3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: 3.     1     10.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     9.0    0.0    0.0       st3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: 5.     1     15.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     14.0   0.0    0.0       st3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: 7.     1     19.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     18.0   0.0    0.0       st3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: 9.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     12.1   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: 7.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     2.0    1.0    0.0       st3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: 9.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.7    0.2    1.7       <total>
 
 # CHECK:      [75] Code Region - G76
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3004
+# CHECK-NEXT: Total Cycles:      757
 # CHECK-NEXT: Total uOps:        4000
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    1.33
-# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: uOps Per Cycle:    5.28
+# CHECK-NEXT: IPC:               1.32
 # CHECK-NEXT: Block RThroughput: 7.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .  .   st3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=====eeeeeeER.    .    .    .  .   st3	{ v1.b, v2.b, v3.b }[0], [x27], #3
-# CHECK-NEXT: [0,3]     .D===========eER    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==========eeeeeeER    .    .  .   st3	{ v1.b, v2.b, v3.b }[8], [x27], #3
-# CHECK-NEXT: [0,5]     . D================eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D===============eeeeeeER   .  .   st3	{ v1.b, v2.b, v3.b }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .  D=====================eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D====================eeeeeeER.   st3	{ v1.b, v2.b, v3.b }[8], [x27], x28
-# CHECK-NEXT: [0,9]     .   D==========================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER .   .   st3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE----R .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeER.   .   st3	{ v1.b, v2.b, v3.b }[0], [x27], #3
+# CHECK-NEXT: [0,3]     .D=eE----R.   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeeeeeER  .   st3	{ v1.b, v2.b, v3.b }[8], [x27], #3
+# CHECK-NEXT: [0,5]     . D==eE----R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeeeeeER .   st3	{ v1.b, v2.b, v3.b }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .  D==eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D==eeeeeeER   st3	{ v1.b, v2.b, v3.b }[8], [x27], x28
+# CHECK-NEXT: [0,9]     .   D===eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4663,43 +4652,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     6.0    0.0    0.0       st3	{ v1.b, v2.b, v3.b }[0], [x27], #3
-# CHECK-NEXT: 3.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     11.0   0.0    0.0       st3	{ v1.b, v2.b, v3.b }[8], [x27], #3
-# CHECK-NEXT: 5.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     16.0   0.0    0.0       st3	{ v1.b, v2.b, v3.b }[0], [x27], x28
-# CHECK-NEXT: 7.     1     22.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     21.0   0.0    0.0       st3	{ v1.b, v2.b, v3.b }[8], [x27], x28
-# CHECK-NEXT: 9.     1     27.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     14.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st3	{ v1.b, v2.b, v3.b }[0], [x27], #3
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st3	{ v1.b, v2.b, v3.b }[8], [x27], #3
+# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st3	{ v1.b, v2.b, v3.b }[0], [x27], x28
+# CHECK-NEXT: 7.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    1.0    0.0       st3	{ v1.b, v2.b, v3.b }[8], [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.3    2.0       <total>
 
 # CHECK:      [76] Code Region - G77
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3004
+# CHECK-NEXT: Total Cycles:      757
 # CHECK-NEXT: Total uOps:        4000
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    1.33
-# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: uOps Per Cycle:    5.28
+# CHECK-NEXT: IPC:               1.32
 # CHECK-NEXT: Block RThroughput: 7.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .  .   st3	{ v1.h, v2.h, v3.h }[0], [x27], #6
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=====eeeeeeER.    .    .    .  .   st3	{ v1.h, v2.h, v3.h }[4], [x27], #6
-# CHECK-NEXT: [0,3]     .D===========eER    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==========eeeeeeER    .    .  .   st3	{ v1.h, v2.h, v3.h }[0], [x27], x28
-# CHECK-NEXT: [0,5]     . D================eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D===============eeeeeeER   .  .   st3	{ v1.h, v2.h, v3.h }[4], [x27], x28
-# CHECK-NEXT: [0,7]     .  D=====================eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D====================eeeeeeER.   st3	{ v1.s, v2.s, v3.s }[0], [x27], #12
-# CHECK-NEXT: [0,9]     .   D==========================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER .   .   st3	{ v1.h, v2.h, v3.h }[0], [x27], #6
+# CHECK-NEXT: [0,1]     D=eE----R .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeER.   .   st3	{ v1.h, v2.h, v3.h }[4], [x27], #6
+# CHECK-NEXT: [0,3]     .D=eE----R.   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeeeeeER  .   st3	{ v1.h, v2.h, v3.h }[0], [x27], x28
+# CHECK-NEXT: [0,5]     . D==eE----R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeeeeeER .   st3	{ v1.h, v2.h, v3.h }[4], [x27], x28
+# CHECK-NEXT: [0,7]     .  D==eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D==eeeeeeER   st3	{ v1.s, v2.s, v3.s }[0], [x27], #12
+# CHECK-NEXT: [0,9]     .   D===eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4709,43 +4698,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.h, v2.h, v3.h }[0], [x27], #6
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     6.0    0.0    0.0       st3	{ v1.h, v2.h, v3.h }[4], [x27], #6
-# CHECK-NEXT: 3.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     11.0   0.0    0.0       st3	{ v1.h, v2.h, v3.h }[0], [x27], x28
-# CHECK-NEXT: 5.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     16.0   0.0    0.0       st3	{ v1.h, v2.h, v3.h }[4], [x27], x28
-# CHECK-NEXT: 7.     1     22.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     21.0   0.0    0.0       st3	{ v1.s, v2.s, v3.s }[0], [x27], #12
-# CHECK-NEXT: 9.     1     27.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     14.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st3	{ v1.h, v2.h, v3.h }[4], [x27], #6
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st3	{ v1.h, v2.h, v3.h }[0], [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st3	{ v1.h, v2.h, v3.h }[4], [x27], x28
+# CHECK-NEXT: 7.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    1.0    0.0       st3	{ v1.s, v2.s, v3.s }[0], [x27], #12
+# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.3    2.0       <total>
 
 # CHECK:      [77] Code Region - G78
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2904
+# CHECK-NEXT: Total Cycles:      807
 # CHECK-NEXT: Total uOps:        4200
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    1.45
-# CHECK-NEXT: IPC:               0.34
+# CHECK-NEXT: uOps Per Cycle:    5.20
+# CHECK-NEXT: IPC:               1.24
 # CHECK-NEXT: Block RThroughput: 8.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          012
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    . .   st3	{ v1.s, v2.s, v3.s }[0], [x27], x28
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=====eeeeeeER.    .    .    . .   st3	{ v1.d, v2.d, v3.d }[0], [x27], #24
-# CHECK-NEXT: [0,3]     .D===========eER    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==========eeeeeeER    .    . .   st3	{ v1.d, v2.d, v3.d }[0], [x27], x28
-# CHECK-NEXT: [0,5]     . D================eER   .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D===============eeeeeER    . .   st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: [0,7]     .  D====================eER   . .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D===================eeeeeeER.   st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: [0,9]     .   D=========================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER .   .   st3	{ v1.s, v2.s, v3.s }[0], [x27], x28
+# CHECK-NEXT: [0,1]     D=eE----R .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeER.   .   st3	{ v1.d, v2.d, v3.d }[0], [x27], #24
+# CHECK-NEXT: [0,3]     .D=eE----R.   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeeeeeER  .   st3	{ v1.d, v2.d, v3.d }[0], [x27], x28
+# CHECK-NEXT: [0,5]     . D==eE----R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeeeeER  .   st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: [0,7]     .  D==eE---R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D==eeeeeeER   st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: [0,9]     .   D===eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4755,43 +4744,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.s, v2.s, v3.s }[0], [x27], x28
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     6.0    0.0    0.0       st3	{ v1.d, v2.d, v3.d }[0], [x27], #24
-# CHECK-NEXT: 3.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     11.0   0.0    0.0       st3	{ v1.d, v2.d, v3.d }[0], [x27], x28
-# CHECK-NEXT: 5.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     16.0   0.0    0.0       st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: 7.     1     21.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     20.0   0.0    0.0       st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: 9.     1     26.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     13.7   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st3	{ v1.d, v2.d, v3.d }[0], [x27], #24
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st3	{ v1.d, v2.d, v3.d }[0], [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: 7.     1     3.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    1.0    0.0       st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.3    1.9       <total>
 
 # CHECK:      [78] Code Region - G79
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3304
+# CHECK-NEXT: Total Cycles:      1205
 # CHECK-NEXT: Total uOps:        5800
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    1.76
-# CHECK-NEXT: IPC:               0.30
+# CHECK-NEXT: uOps Per Cycle:    4.81
+# CHECK-NEXT: IPC:               0.83
 # CHECK-NEXT: Block RThroughput: 12.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     0123456
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .    ..   st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=====eeeeeeeER    .    .    .    ..   st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: [0,3]     . D===========eER   .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .  D==========eeeeeeER   .    .    ..   st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: [0,5]     .  D================eER  .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .   D===============eeeeeeeER .    ..   st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: [0,7]     .    D=====================eER.    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .D====================eeeeeeeER.   st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: [0,9]     .    . D==========================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER .    ..   st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: [0,1]     D=eE----R .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeER    ..   st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: [0,3]     . DeE-----R    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .  DeeeeeeER   ..   st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: [0,5]     .  D=eE----R   ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .   D==eeeeeeeER.   st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: [0,7]     .    D==eE-----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .D=eeeeeeeER   st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: [0,9]     .    . D=eE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4801,43 +4790,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     6.0    0.0    0.0       st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: 3.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     11.0   0.0    0.0       st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: 5.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     16.0   0.0    0.0       st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: 7.     1     22.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     21.0   0.0    0.0       st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: 9.     1     27.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     14.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: 3.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: 5.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    2.0    0.0       st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: 7.     1     3.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     2.0    0.0    0.0       st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: 9.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.8    0.4    2.3       <total>
 
 # CHECK:      [79] Code Region - G80
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3004
+# CHECK-NEXT: Total Cycles:      1006
 # CHECK-NEXT: Total uOps:        4800
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    1.60
-# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: uOps Per Cycle:    4.77
+# CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 9.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     012345
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeER  .    .    .    .    .  .   st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: [0,1]     D=====eER .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D====eeeeeeER .    .    .    .  .   st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: [0,3]     .D==========eER.    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=========eeeeeeER.    .    .  .   st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: [0,5]     . D===============eER    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D==============eeeeeeeER   .  .   st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: [0,7]     .   D====================eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    D===================eeeeeeER.   st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: [0,9]     .    D=========================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeER  .    .   st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE---R  .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeER.    .   st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE----R.    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D==eeeeeeER  .   st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,5]     . D===eE----R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D==eeeeeeeER.   st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,7]     .   D==eE-----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    D==eeeeeeER   st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: [0,9]     .    D===eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4847,43 +4836,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: 3.     1     11.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     10.0   0.0    0.0       st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: 5.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     15.0   0.0    0.0       st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: 7.     1     21.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     20.0   0.0    0.0       st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: 9.     1     26.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     13.1   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    2.0    0.0       st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: 5.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: 7.     1     3.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    1.0    0.0       st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.6    0.4    2.0       <total>
 
 # CHECK:      [80] Code Region - G81
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3204
+# CHECK-NEXT: Total Cycles:      1058
 # CHECK-NEXT: Total uOps:        5200
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    1.62
-# CHECK-NEXT: IPC:               0.31
+# CHECK-NEXT: uOps Per Cycle:    4.91
+# CHECK-NEXT: IPC:               0.95
 # CHECK-NEXT: Block RThroughput: 10.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          012345
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     01234567
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    .   st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: [0,1]     .D======eER    .    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . D=====eeeeeeeER   .    .    .    .   st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: [0,3]     .  D===========eER  .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D==========eeeeeeER  .    .    .   st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
-# CHECK-NEXT: [0,5]     .   D================eER .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    D===============eeeeeeER .    .   st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
-# CHECK-NEXT: [0,7]     .    D=====================eER.    .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .D====================eeeeeeER.   st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
-# CHECK-NEXT: [0,9]     .    .D==========================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeER.    . .   st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE-----R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeER   . .   st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE-----R   . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   D==eeeeeeER. .   st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
+# CHECK-NEXT: [0,5]     .   D===eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    D===eeeeeeER.   st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
+# CHECK-NEXT: [0,7]     .    D====eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .D===eeeeeeER   st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
+# CHECK-NEXT: [0,9]     .    .D====eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4893,43 +4882,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     6.0    0.0    0.0       st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: 3.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     11.0   0.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
-# CHECK-NEXT: 5.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     16.0   0.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
-# CHECK-NEXT: 7.     1     22.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     21.0   0.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
-# CHECK-NEXT: 9.     1     27.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     14.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    3.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
+# CHECK-NEXT: 5.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     4.0    1.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
+# CHECK-NEXT: 7.     1     5.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
+# CHECK-NEXT: 9.     1     5.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.9    0.6    2.2       <total>
 
 # CHECK:      [81] Code Region - G82
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3004
+# CHECK-NEXT: Total Cycles:      757
 # CHECK-NEXT: Total uOps:        4000
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    1.33
-# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: uOps Per Cycle:    5.28
+# CHECK-NEXT: IPC:               1.32
 # CHECK-NEXT: Block RThroughput: 7.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123
-# CHECK-NEXT: Index     0123456789          0123456789
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .  .   st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=====eeeeeeER.    .    .    .  .   st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
-# CHECK-NEXT: [0,3]     .D===========eER    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==========eeeeeeER    .    .  .   st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
-# CHECK-NEXT: [0,5]     . D================eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D===============eeeeeeER   .  .   st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .  D=====================eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D====================eeeeeeER.   st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
-# CHECK-NEXT: [0,9]     .   D==========================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER .   .   st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
+# CHECK-NEXT: [0,1]     D=eE----R .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeER.   .   st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
+# CHECK-NEXT: [0,3]     .D=eE----R.   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeeeeeER  .   st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
+# CHECK-NEXT: [0,5]     . D==eE----R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeeeeeER .   st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .  D==eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D==eeeeeeER   st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
+# CHECK-NEXT: [0,9]     .   D===eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4939,43 +4928,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     6.0    0.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
-# CHECK-NEXT: 3.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     11.0   0.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
-# CHECK-NEXT: 5.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     16.0   0.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
-# CHECK-NEXT: 7.     1     22.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     21.0   0.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
-# CHECK-NEXT: 9.     1     27.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     14.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
+# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
+# CHECK-NEXT: 7.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    1.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.3    2.0       <total>
 
 # CHECK:      [82] Code Region - G83
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2104
+# CHECK-NEXT: Total Cycles:      704
 # CHECK-NEXT: Total uOps:        3600
 
 # CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    1.71
-# CHECK-NEXT: IPC:               0.48
+# CHECK-NEXT: uOps Per Cycle:    5.11
+# CHECK-NEXT: IPC:               1.42
 # CHECK-NEXT: Block RThroughput: 6.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          01234
-
-# CHECK:      [0,0]     DeeeeeeER .    .    .   .   st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
-# CHECK-NEXT: [0,1]     D======eER.    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=====eeeeeeER.    .   .   st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
-# CHECK-NEXT: [0,3]     .D===========eER    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==========eeeeER .   .   st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
-# CHECK-NEXT: [0,5]     . D==============eER.   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=============eeeeER .   st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .  D=================eER.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D================eER.   stg	x26, [x27], #4064
-# CHECK-NEXT: [0,9]     .   D=================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeER .   st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
+# CHECK-NEXT: [0,1]     D=eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeER.   st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeeeER.   st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
+# CHECK-NEXT: [0,5]     . D==eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeeeER   st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .  D==eE--R   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D==eE-R   stg	x26, [x27], #4064
+# CHECK-NEXT: [0,9]     .   D===eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4985,16 +4974,16 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     6.0    0.0    0.0       st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
-# CHECK-NEXT: 3.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     11.0   0.0    0.0       st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
-# CHECK-NEXT: 5.     1     15.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     14.0   0.0    0.0       st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
-# CHECK-NEXT: 7.     1     18.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     17.0   0.0    0.0       stg	x26, [x27], #4064
-# CHECK-NEXT: 9.     1     18.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     11.9   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
+# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
+# CHECK-NEXT: 7.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    1.0    1.0       stg	x26, [x27], #4064
+# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.3    1.3       <total>
 
 # CHECK:      [83] Code Region - G84
 
diff --git a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-writeback.s b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-writeback.s
index 91adecac8d..264ad8bccc 100644
--- a/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-writeback.s
+++ b/llvm/test/tools/llvm-mca/AArch64/Neoverse/V1-writeback.s
@@ -1162,28 +1162,28 @@ add x0, x27, 1
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3004
+# CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    0.50
-# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: uOps Per Cycle:    2.95
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123
-# CHECK-NEXT: Index     0123456789          0123456789
-
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .  .   ld1	{ v1.1d }, [x27], #8
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D======eeeeeeER.    .    .    .  .   ld1	{ v1.2d }, [x27], #16
-# CHECK-NEXT: [0,3]     D============eER    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D============eeeeeeER    .    .  .   ld1	{ v1.2s }, [x27], #8
-# CHECK-NEXT: [0,5]     D==================eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     D==================eeeeeeER   .  .   ld1	{ v1.4h }, [x27], #8
-# CHECK-NEXT: [0,7]     D========================eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     D========================eeeeeeER.   ld1	{ v1.4s }, [x27], #16
-# CHECK-NEXT: [0,9]     D==============================eER   add	x0, x27, #1
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.1d }, [x27], #8
+# CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ld1	{ v1.2d }, [x27], #16
+# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeeeER .   ld1	{ v1.2s }, [x27], #8
+# CHECK-NEXT: [0,5]     D===eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     D===eeeeeeER.   ld1	{ v1.4h }, [x27], #8
+# CHECK-NEXT: [0,7]     D====eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     D====eeeeeeER   ld1	{ v1.4s }, [x27], #16
+# CHECK-NEXT: [0,9]     D=====eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1193,43 +1193,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.1d }, [x27], #8
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld1	{ v1.2d }, [x27], #16
-# CHECK-NEXT: 3.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld1	{ v1.2s }, [x27], #8
-# CHECK-NEXT: 5.     1     19.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld1	{ v1.4h }, [x27], #8
-# CHECK-NEXT: 7.     1     25.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld1	{ v1.4s }, [x27], #16
-# CHECK-NEXT: 9.     1     31.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.2d }, [x27], #16
+# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1	{ v1.2s }, [x27], #8
+# CHECK-NEXT: 5.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     4.0    0.0    0.0       ld1	{ v1.4h }, [x27], #8
+# CHECK-NEXT: 7.     1     5.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     5.0    0.0    0.0       ld1	{ v1.4s }, [x27], #16
+# CHECK-NEXT: 9.     1     6.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.5    0.1    2.0       <total>
 
 # CHECK:      [1] Code Region - G02
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3004
+# CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    0.50
-# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: uOps Per Cycle:    2.95
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123
-# CHECK-NEXT: Index     0123456789          0123456789
-
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .  .   ld1	{ v1.8b }, [x27], #8
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D======eeeeeeER.    .    .    .  .   ld1	{ v1.8h }, [x27], #16
-# CHECK-NEXT: [0,3]     D============eER    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D============eeeeeeER    .    .  .   ld1	{ v1.16b }, [x27], #16
-# CHECK-NEXT: [0,5]     D==================eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     D==================eeeeeeER   .  .   ld1	{ v1.1d }, [x27], x28
-# CHECK-NEXT: [0,7]     D========================eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     D========================eeeeeeER.   ld1	{ v1.2d }, [x27], x28
-# CHECK-NEXT: [0,9]     D==============================eER   add	x0, x27, #1
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.8b }, [x27], #8
+# CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ld1	{ v1.8h }, [x27], #16
+# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeeeER .   ld1	{ v1.16b }, [x27], #16
+# CHECK-NEXT: [0,5]     D===eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     D===eeeeeeER.   ld1	{ v1.1d }, [x27], x28
+# CHECK-NEXT: [0,7]     D====eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     D====eeeeeeER   ld1	{ v1.2d }, [x27], x28
+# CHECK-NEXT: [0,9]     D=====eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1239,43 +1239,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.8b }, [x27], #8
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld1	{ v1.8h }, [x27], #16
-# CHECK-NEXT: 3.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld1	{ v1.16b }, [x27], #16
-# CHECK-NEXT: 5.     1     19.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld1	{ v1.1d }, [x27], x28
-# CHECK-NEXT: 7.     1     25.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld1	{ v1.2d }, [x27], x28
-# CHECK-NEXT: 9.     1     31.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.8h }, [x27], #16
+# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1	{ v1.16b }, [x27], #16
+# CHECK-NEXT: 5.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     4.0    0.0    0.0       ld1	{ v1.1d }, [x27], x28
+# CHECK-NEXT: 7.     1     5.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     5.0    0.0    0.0       ld1	{ v1.2d }, [x27], x28
+# CHECK-NEXT: 9.     1     6.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.5    0.1    2.0       <total>
 
 # CHECK:      [2] Code Region - G03
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3004
+# CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    0.50
-# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: uOps Per Cycle:    2.95
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123
-# CHECK-NEXT: Index     0123456789          0123456789
-
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .  .   ld1	{ v1.2s }, [x27], x28
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D======eeeeeeER.    .    .    .  .   ld1	{ v1.4h }, [x27], x28
-# CHECK-NEXT: [0,3]     D============eER    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D============eeeeeeER    .    .  .   ld1	{ v1.4s }, [x27], x28
-# CHECK-NEXT: [0,5]     D==================eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     D==================eeeeeeER   .  .   ld1	{ v1.8b }, [x27], x28
-# CHECK-NEXT: [0,7]     D========================eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     D========================eeeeeeER.   ld1	{ v1.8h }, [x27], x28
-# CHECK-NEXT: [0,9]     D==============================eER   add	x0, x27, #1
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.2s }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ld1	{ v1.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeeeER .   ld1	{ v1.4s }, [x27], x28
+# CHECK-NEXT: [0,5]     D===eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     D===eeeeeeER.   ld1	{ v1.8b }, [x27], x28
+# CHECK-NEXT: [0,7]     D====eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     D====eeeeeeER   ld1	{ v1.8h }, [x27], x28
+# CHECK-NEXT: [0,9]     D=====eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1285,43 +1285,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.2s }, [x27], x28
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld1	{ v1.4h }, [x27], x28
-# CHECK-NEXT: 3.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld1	{ v1.4s }, [x27], x28
-# CHECK-NEXT: 5.     1     19.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld1	{ v1.8b }, [x27], x28
-# CHECK-NEXT: 7.     1     25.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld1	{ v1.8h }, [x27], x28
-# CHECK-NEXT: 9.     1     31.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.4h }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1	{ v1.4s }, [x27], x28
+# CHECK-NEXT: 5.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     4.0    0.0    0.0       ld1	{ v1.8b }, [x27], x28
+# CHECK-NEXT: 7.     1     5.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     5.0    0.0    0.0       ld1	{ v1.8h }, [x27], x28
+# CHECK-NEXT: 9.     1     6.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.5    0.1    2.0       <total>
 
 # CHECK:      [3] Code Region - G04
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3004
+# CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        1900
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    0.63
-# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: uOps Per Cycle:    3.74
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 3.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123
-# CHECK-NEXT: Index     0123456789          0123456789
-
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .  .   ld1	{ v1.16b }, [x27], x28
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D======eeeeeeER.    .    .    .  .   ld1	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: [0,3]     D============eER    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D============eeeeeeER    .    .  .   ld1	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: [0,5]     D==================eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     D==================eeeeeeER   .  .   ld1	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: [0,7]     D========================eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D=======================eeeeeeER.   ld1	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: [0,9]     .D=============================eER   add	x0, x27, #1
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.16b }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ld1	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeeeER .   ld1	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: [0,5]     D===eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     D===eeeeeeER.   ld1	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: [0,7]     D====eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeeeeeER   ld1	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: [0,9]     .D====eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1331,43 +1331,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.16b }, [x27], x28
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld1	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: 3.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld1	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: 5.     1     19.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld1	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: 7.     1     25.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     24.0   0.0    0.0       ld1	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: 9.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     15.8   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: 5.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     4.0    0.0    0.0       ld1	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: 7.     1     5.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ld1	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: 9.     1     5.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.3    0.1    2.0       <total>
 
 # CHECK:      [4] Code Region - G05
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3004
+# CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    0.67
-# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: uOps Per Cycle:    3.94
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 3.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123
-# CHECK-NEXT: Index     0123456789          0123456789
-
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .  .   ld1	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D======eeeeeeER.    .    .    .  .   ld1	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: [0,3]     D============eER    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D============eeeeeeER    .    .  .   ld1	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: [0,5]     D==================eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     D==================eeeeeeER   .  .   ld1	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: [0,7]     .D=======================eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D=======================eeeeeeER.   ld1	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: [0,9]     .D=============================eER   add	x0, x27, #1
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ld1	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeeeER .   ld1	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: [0,5]     D===eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     D===eeeeeeER.   ld1	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeeeeeER   ld1	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: [0,9]     .D====eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1377,43 +1377,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld1	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: 3.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld1	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: 5.     1     19.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld1	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: 7.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     24.0   0.0    0.0       ld1	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: 9.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     15.7   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: 5.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     4.0    0.0    0.0       ld1	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ld1	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: 9.     1     5.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.2    0.1    2.0       <total>
 
 # CHECK:      [5] Code Region - G06
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3004
+# CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    0.67
-# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: uOps Per Cycle:    3.94
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 3.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123
-# CHECK-NEXT: Index     0123456789          0123456789
-
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .  .   ld1	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D======eeeeeeER.    .    .    .  .   ld1	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: [0,3]     D============eER    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D============eeeeeeER    .    .  .   ld1	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: [0,5]     D==================eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     D==================eeeeeeER   .  .   ld1	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: [0,7]     .D=======================eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D=======================eeeeeeER.   ld1	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: [0,9]     .D=============================eER   add	x0, x27, #1
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ld1	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeeeER .   ld1	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,5]     D===eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     D===eeeeeeER.   ld1	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeeeeeER   ld1	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,9]     .D====eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1423,43 +1423,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld1	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: 3.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld1	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: 5.     1     19.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld1	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: 7.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     24.0   0.0    0.0       ld1	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: 9.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     15.7   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: 5.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     4.0    0.0    0.0       ld1	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ld1	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: 9.     1     5.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.2    0.1    2.0       <total>
 
 # CHECK:      [6] Code Region - G07
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3004
+# CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        2300
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    0.77
-# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: uOps Per Cycle:    4.53
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 4.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123
-# CHECK-NEXT: Index     0123456789          0123456789
-
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .  .   ld1	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D======eeeeeeER.    .    .    .  .   ld1	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: [0,3]     D============eER    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D============eeeeeeER    .    .  .   ld1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: [0,5]     D==================eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=================eeeeeeER   .  .   ld1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: [0,7]     .D=======================eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D=======================eeeeeeER.   ld1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: [0,9]     .D=============================eER   add	x0, x27, #1
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ld1	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeeeER .   ld1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
+# CHECK-NEXT: [0,5]     D===eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeER.   ld1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeeeeeER   ld1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: [0,9]     .D====eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1469,43 +1469,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld1	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: 3.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: 5.     1     19.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     18.0   0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: 7.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     24.0   0.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: 9.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     15.6   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
+# CHECK-NEXT: 5.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: 9.     1     5.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.1    0.1    2.0       <total>
 
 # CHECK:      [7] Code Region - G08
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3004
+# CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        2500
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    0.83
-# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: uOps Per Cycle:    4.92
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123
-# CHECK-NEXT: Index     0123456789          0123456789
-
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .  .   ld1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D======eeeeeeER.    .    .    .  .   ld1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: [0,3]     D============eER    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D============eeeeeeER    .    .  .   ld1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: [0,5]     D==================eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=================eeeeeeER   .  .   ld1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: [0,7]     .D=======================eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D=======================eeeeeeER.   ld1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: [0,9]     .D=============================eER   add	x0, x27, #1
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ld1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeeeER .   ld1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: [0,5]     D===eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeER.   ld1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeeeeeER   ld1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: [0,9]     .D====eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1515,43 +1515,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: 3.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: 5.     1     19.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     18.0   0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: 7.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     24.0   0.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: 9.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     15.6   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: 5.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: 9.     1     5.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.1    0.1    2.0       <total>
 
 # CHECK:      [8] Code Region - G09
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3004
+# CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        2500
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    0.83
-# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: uOps Per Cycle:    4.92
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123
-# CHECK-NEXT: Index     0123456789          0123456789
-
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .  .   ld1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D======eeeeeeER.    .    .    .  .   ld1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: [0,3]     D============eER    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D============eeeeeeER    .    .  .   ld1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: [0,5]     D==================eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=================eeeeeeER   .  .   ld1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: [0,7]     .D=======================eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D=======================eeeeeeER.   ld1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: [0,9]     .D=============================eER   add	x0, x27, #1
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ld1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeeeER .   ld1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,5]     D===eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeER.   ld1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeeeeeER   ld1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: [0,9]     .D====eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1561,43 +1561,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: 3.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: 5.     1     19.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     18.0   0.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: 7.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     24.0   0.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: 9.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     15.6   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: 5.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: 9.     1     5.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.1    0.1    2.0       <total>
 
 # CHECK:      [9] Code Region - G10
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3104
+# CHECK-NEXT: Total Cycles:      608
 # CHECK-NEXT: Total uOps:        2500
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    0.81
-# CHECK-NEXT: IPC:               0.32
+# CHECK-NEXT: uOps Per Cycle:    4.11
+# CHECK-NEXT: IPC:               1.64
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          01234
-# CHECK-NEXT: Index     0123456789          0123456789
-
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .   .   ld1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D======eeeeeeER.    .    .    .   .   ld1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: [0,3]     D============eER    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D============eeeeeeER    .    .   .   ld1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: [0,5]     D==================eER   .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=================eeeeeeER   .   .   ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: [0,7]     .D=======================eER  .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D=======================eeeeeeeER.   ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: [0,9]     .D==============================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0123
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeER .  .   ld1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE----R .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeER.  .   ld1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eE----R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeeeER  .   ld1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,5]     D===eE----R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeER .   ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: [0,7]     .D===eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeeeeeeER   ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: [0,9]     .D====eE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1607,43 +1607,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: 3.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: 5.     1     19.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     18.0   0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: 7.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     24.0   0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: 9.     1     31.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     15.7   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: 5.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: 9.     1     5.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.1    0.1    2.1       <total>
 
 # CHECK:      [10] Code Region - G11
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3204
+# CHECK-NEXT: Total Cycles:      509
 # CHECK-NEXT: Total uOps:        2400
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    0.75
-# CHECK-NEXT: IPC:               0.31
+# CHECK-NEXT: uOps Per Cycle:    4.72
+# CHECK-NEXT: IPC:               1.96
 # CHECK-NEXT: Block RThroughput: 4.7
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          012345
-# CHECK-NEXT: Index     0123456789          0123456789
-
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .    .   ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D======eeeeeeER.    .    .    .    .   ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: [0,3]     D============eER    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D============eeeeeeeER   .    .    .   ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: [0,5]     D===================eER  .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==================eeeeeeER  .    .   ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: [0,7]     .D========================eER .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D========================eeeeeeeER.   ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: [0,9]     .D===============================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0123
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeER .  .   ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: [0,1]     D=eE----R .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeER.  .   ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: [0,3]     D==eE----R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeeeeER .   ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: [0,5]     D===eE-----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeER .   ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: [0,7]     .D===eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeeeeeeER   ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: [0,9]     .D====eE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1653,43 +1653,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: 3.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: 5.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: 7.     1     25.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: 9.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.2   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: 5.     1     4.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: 9.     1     5.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.1    0.1    2.2       <total>
 
 # CHECK:      [11] Code Region - G12
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3204
+# CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        2400
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    0.75
-# CHECK-NEXT: IPC:               0.31
+# CHECK-NEXT: uOps Per Cycle:    4.72
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 4.7
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          012345
-# CHECK-NEXT: Index     0123456789          0123456789
-
-# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    .   ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: [0,1]     D=======eER    .    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=======eeeeeeER    .    .    .    .   ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: [0,3]     D=============eER   .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D=============eeeeeeeER  .    .    .   ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: [0,5]     .D===================eER .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D===================eeeeeeER .    .   ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: [0,7]     .D=========================eER.    .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D=========================eeeeeeER.   ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: [0,9]     .D===============================eER   add	x0, x27, #1
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeER. .   ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: [0,1]     D=eE-----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeeeeER.   ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,5]     .D==eE-----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeER.   ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeeeeeER   ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,9]     .D====eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1699,43 +1699,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     14.0   0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: 5.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     20.0   0.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: 7.     1     26.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     26.0   0.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: 9.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.9   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: 9.     1     5.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.0    0.1    2.2       <total>
 
 # CHECK:      [12] Code Region - G13
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3504
+# CHECK-NEXT: Total Cycles:      1110
 # CHECK-NEXT: Total uOps:        2600
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    0.74
-# CHECK-NEXT: IPC:               0.29
+# CHECK-NEXT: uOps Per Cycle:    2.34
+# CHECK-NEXT: IPC:               0.90
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          012345678
-# CHECK-NEXT: Index     0123456789          0123456789
-
-# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    .  .   ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: [0,1]     D=======eER    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=======eeeeeeER    .    .    .    .  .   ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: [0,3]     D=============eER   .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D=============eeeeeeeER  .    .    .  .   ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: [0,5]     .D===================eER .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D===================eeeeeeeER.    .  .   ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: [0,7]     .D==========================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D==========================eeeeeeeeER.   ld1	{ v1.b }[0], [x27], #1
-# CHECK-NEXT: [0,9]     .D==================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          0
+
+# CHECK:      [0,0]     DeeeeeeeER.    .    .   ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE-----R.    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeER.    .    .   ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eE----R.    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeeeeER   .    .   ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,5]     .D==eE-----R   .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeeER  .    .   ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,7]     .D===eE-----R  .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D=========eeeeeeeeER   ld1	{ v1.b }[0], [x27], #1
+# CHECK-NEXT: [0,9]     .D==========eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1745,22 +1745,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     14.0   0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: 5.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     20.0   0.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: 7.     1     27.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     27.0   0.0    0.0       ld1	{ v1.b }[0], [x27], #1
-# CHECK-NEXT: 9.     1     35.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     17.4   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     10.0   0.0    0.0       ld1	{ v1.b }[0], [x27], #1
+# CHECK-NEXT: 9.     1     11.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     4.2    0.1    2.5       <total>
 
 # CHECK:      [13] Code Region - G14
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    15
@@ -1770,18 +1770,18 @@ add x0, x27, 1
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld1	{ v1.b }[8], [x27], #1
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    .  .   ld1	{ v1.b }[0], [x27], x28
-# CHECK-NEXT: [0,3]     D================eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D================eeeeeeeeER   .    .    .  .   ld1	{ v1.b }[8], [x27], x28
-# CHECK-NEXT: [0,5]     D========================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     D========================eeeeeeeeER.    .  .   ld1	{ v1.h }[0], [x27], #2
-# CHECK-NEXT: [0,7]     .D===============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===============================eeeeeeeeER.   ld1	{ v1.h }[4], [x27], #2
-# CHECK-NEXT: [0,9]     .D=======================================eER   add	x0, x27, #1
+# CHECK-NEXT: Index     0123456789          0123456789          012
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld1	{ v1.b }[8], [x27], #1
+# CHECK-NEXT: [0,1]     D=eE------R    .    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    . .   ld1	{ v1.b }[0], [x27], x28
+# CHECK-NEXT: [0,3]     D=========eE------R .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D================eeeeeeeeER   .    .    . .   ld1	{ v1.b }[8], [x27], x28
+# CHECK-NEXT: [0,5]     D=================eE------R   .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     D========================eeeeeeeeER.    . .   ld1	{ v1.h }[0], [x27], #2
+# CHECK-NEXT: [0,7]     .D========================eE------R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===============================eeeeeeeeER   ld1	{ v1.h }[4], [x27], #2
+# CHECK-NEXT: [0,9]     .D================================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1791,22 +1791,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.b }[8], [x27], #1
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld1	{ v1.b }[0], [x27], x28
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 3.     1     10.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     17.0   0.0    0.0       ld1	{ v1.b }[8], [x27], x28
-# CHECK-NEXT: 5.     1     25.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 5.     1     18.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     25.0   0.0    0.0       ld1	{ v1.h }[0], [x27], #2
-# CHECK-NEXT: 7.     1     32.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 7.     1     25.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 8.     1     32.0   0.0    0.0       ld1	{ v1.h }[4], [x27], #2
-# CHECK-NEXT: 9.     1     40.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.7   0.1    0.0       <total>
+# CHECK-NEXT: 9.     1     33.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     17.2   0.1    3.0       <total>
 
 # CHECK:      [14] Code Region - G15
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    15
@@ -1816,18 +1816,18 @@ add x0, x27, 1
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld1	{ v1.h }[0], [x27], x28
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    .  .   ld1	{ v1.h }[4], [x27], x28
-# CHECK-NEXT: [0,3]     D================eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D================eeeeeeeeER   .    .    .  .   ld1	{ v1.s }[0], [x27], #4
-# CHECK-NEXT: [0,5]     D========================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     D========================eeeeeeeeER.    .  .   ld1	{ v1.s }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .D===============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===============================eeeeeeeeER.   ld1	{ v1.d }[0], [x27], #8
-# CHECK-NEXT: [0,9]     .D=======================================eER   add	x0, x27, #1
+# CHECK-NEXT: Index     0123456789          0123456789          012
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld1	{ v1.h }[0], [x27], x28
+# CHECK-NEXT: [0,1]     D=eE------R    .    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    . .   ld1	{ v1.h }[4], [x27], x28
+# CHECK-NEXT: [0,3]     D=========eE------R .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D================eeeeeeeeER   .    .    . .   ld1	{ v1.s }[0], [x27], #4
+# CHECK-NEXT: [0,5]     D=================eE------R   .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     D========================eeeeeeeeER.    . .   ld1	{ v1.s }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .D========================eE------R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===============================eeeeeeeeER   ld1	{ v1.d }[0], [x27], #8
+# CHECK-NEXT: [0,9]     .D================================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1837,43 +1837,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.h }[0], [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld1	{ v1.h }[4], [x27], x28
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 3.     1     10.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     17.0   0.0    0.0       ld1	{ v1.s }[0], [x27], #4
-# CHECK-NEXT: 5.     1     25.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 5.     1     18.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     25.0   0.0    0.0       ld1	{ v1.s }[0], [x27], x28
-# CHECK-NEXT: 7.     1     32.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 7.     1     25.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 8.     1     32.0   0.0    0.0       ld1	{ v1.d }[0], [x27], #8
-# CHECK-NEXT: 9.     1     40.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.7   0.1    0.0       <total>
+# CHECK-NEXT: 9.     1     33.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     17.2   0.1    3.0       <total>
 
 # CHECK:      [15] Code Region - G16
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      1203
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    0.50
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    1.66
+# CHECK-NEXT: IPC:               0.83
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld1	{ v1.d }[0], [x27], x28
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    .  .   ld1r	{ v1.1d }, [x27], #8
-# CHECK-NEXT: [0,3]     D================eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D================eeeeeeeeER   .    .    .  .   ld1r	{ v1.2d }, [x27], #8
-# CHECK-NEXT: [0,5]     D========================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     D========================eeeeeeeeER.    .  .   ld1r	{ v1.2s }, [x27], #4
-# CHECK-NEXT: [0,7]     .D===============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===============================eeeeeeeeER.   ld1r	{ v1.4h }, [x27], #2
-# CHECK-NEXT: [0,9]     .D=======================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER   .   ld1	{ v1.d }[0], [x27], x28
+# CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld1r	{ v1.1d }, [x27], #8
+# CHECK-NEXT: [0,3]     D==eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeeeeeER .   ld1r	{ v1.2d }, [x27], #8
+# CHECK-NEXT: [0,5]     D===eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     D===eeeeeeeeER.   ld1r	{ v1.2s }, [x27], #4
+# CHECK-NEXT: [0,7]     .D===eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeeeeeeeER   ld1r	{ v1.4h }, [x27], #2
+# CHECK-NEXT: [0,9]     .D====eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1883,43 +1883,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.d }[0], [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld1r	{ v1.1d }, [x27], #8
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     17.0   0.0    0.0       ld1r	{ v1.2d }, [x27], #8
-# CHECK-NEXT: 5.     1     25.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     25.0   0.0    0.0       ld1r	{ v1.2s }, [x27], #4
-# CHECK-NEXT: 7.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     32.0   0.0    0.0       ld1r	{ v1.4h }, [x27], #2
-# CHECK-NEXT: 9.     1     40.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.7   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1r	{ v1.1d }, [x27], #8
+# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1r	{ v1.2d }, [x27], #8
+# CHECK-NEXT: 5.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     4.0    0.0    0.0       ld1r	{ v1.2s }, [x27], #4
+# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ld1r	{ v1.4h }, [x27], #2
+# CHECK-NEXT: 9.     1     5.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.2    0.1    3.0       <total>
 
 # CHECK:      [16] Code Region - G17
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    0.50
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    3.92
+# CHECK-NEXT: IPC:               1.96
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld1r	{ v1.4s }, [x27], #4
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    .  .   ld1r	{ v1.8b }, [x27], #1
-# CHECK-NEXT: [0,3]     D================eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D================eeeeeeeeER   .    .    .  .   ld1r	{ v1.8h }, [x27], #2
-# CHECK-NEXT: [0,5]     D========================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     D========================eeeeeeeeER.    .  .   ld1r	{ v1.16b }, [x27], #1
-# CHECK-NEXT: [0,7]     .D===============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===============================eeeeeeeeER.   ld1r	{ v1.1d }, [x27], x28
-# CHECK-NEXT: [0,9]     .D=======================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER   .   ld1r	{ v1.4s }, [x27], #4
+# CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld1r	{ v1.8b }, [x27], #1
+# CHECK-NEXT: [0,3]     D==eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeeeeeER .   ld1r	{ v1.8h }, [x27], #2
+# CHECK-NEXT: [0,5]     D===eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     D===eeeeeeeeER.   ld1r	{ v1.16b }, [x27], #1
+# CHECK-NEXT: [0,7]     .D===eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeeeeeeeER   ld1r	{ v1.1d }, [x27], x28
+# CHECK-NEXT: [0,9]     .D====eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1929,43 +1929,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1r	{ v1.4s }, [x27], #4
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld1r	{ v1.8b }, [x27], #1
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     17.0   0.0    0.0       ld1r	{ v1.8h }, [x27], #2
-# CHECK-NEXT: 5.     1     25.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     25.0   0.0    0.0       ld1r	{ v1.16b }, [x27], #1
-# CHECK-NEXT: 7.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     32.0   0.0    0.0       ld1r	{ v1.1d }, [x27], x28
-# CHECK-NEXT: 9.     1     40.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.7   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1r	{ v1.8b }, [x27], #1
+# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1r	{ v1.8h }, [x27], #2
+# CHECK-NEXT: 5.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     4.0    0.0    0.0       ld1r	{ v1.16b }, [x27], #1
+# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ld1r	{ v1.1d }, [x27], x28
+# CHECK-NEXT: 9.     1     5.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.2    0.1    3.0       <total>
 
 # CHECK:      [17] Code Region - G18
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    0.50
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    3.92
+# CHECK-NEXT: IPC:               1.96
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld1r	{ v1.2d }, [x27], x28
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    .  .   ld1r	{ v1.2s }, [x27], x28
-# CHECK-NEXT: [0,3]     D================eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D================eeeeeeeeER   .    .    .  .   ld1r	{ v1.4h }, [x27], x28
-# CHECK-NEXT: [0,5]     D========================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     D========================eeeeeeeeER.    .  .   ld1r	{ v1.4s }, [x27], x28
-# CHECK-NEXT: [0,7]     .D===============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===============================eeeeeeeeER.   ld1r	{ v1.8b }, [x27], x28
-# CHECK-NEXT: [0,9]     .D=======================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER   .   ld1r	{ v1.2d }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld1r	{ v1.2s }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeeeeeER .   ld1r	{ v1.4h }, [x27], x28
+# CHECK-NEXT: [0,5]     D===eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     D===eeeeeeeeER.   ld1r	{ v1.4s }, [x27], x28
+# CHECK-NEXT: [0,7]     .D===eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeeeeeeeER   ld1r	{ v1.8b }, [x27], x28
+# CHECK-NEXT: [0,9]     .D====eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1975,43 +1975,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1r	{ v1.2d }, [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld1r	{ v1.2s }, [x27], x28
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     17.0   0.0    0.0       ld1r	{ v1.4h }, [x27], x28
-# CHECK-NEXT: 5.     1     25.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     25.0   0.0    0.0       ld1r	{ v1.4s }, [x27], x28
-# CHECK-NEXT: 7.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     32.0   0.0    0.0       ld1r	{ v1.8b }, [x27], x28
-# CHECK-NEXT: 9.     1     40.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.7   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1r	{ v1.2s }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1r	{ v1.4h }, [x27], x28
+# CHECK-NEXT: 5.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     4.0    0.0    0.0       ld1r	{ v1.4s }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ld1r	{ v1.8b }, [x27], x28
+# CHECK-NEXT: 9.     1     5.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.2    0.1    3.0       <total>
 
 # CHECK:      [18] Code Region - G19
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        2400
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    0.60
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    4.71
+# CHECK-NEXT: IPC:               1.96
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld1r	{ v1.8h }, [x27], x28
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    .  .   ld1r	{ v1.16b }, [x27], x28
-# CHECK-NEXT: [0,3]     D================eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D================eeeeeeeeER   .    .    .  .   ld2	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: [0,5]     D========================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    .  .   ld2	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: [0,7]     .D===============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===============================eeeeeeeeER.   ld2	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: [0,9]     .D=======================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER   .   ld1r	{ v1.8h }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld1r	{ v1.16b }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeeeeeER .   ld2	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: [0,5]     D===eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeeeER.   ld2	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: [0,7]     .D===eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeeeeeeeER   ld2	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: [0,9]     .D====eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2021,43 +2021,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1r	{ v1.8h }, [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld1r	{ v1.16b }, [x27], x28
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     17.0   0.0    0.0       ld2	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: 5.     1     25.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld2	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: 7.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     32.0   0.0    0.0       ld2	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: 9.     1     40.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.6   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1r	{ v1.16b }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld2	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: 5.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld2	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ld2	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: 9.     1     5.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.1    0.1    3.0       <total>
 
 # CHECK:      [19] Code Region - G20
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        2900
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    0.72
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    5.69
+# CHECK-NEXT: IPC:               1.96
 # CHECK-NEXT: Block RThroughput: 3.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld2	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    .  .   ld2	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: [0,3]     D================eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER   .    .    .  .   ld2	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: [0,5]     .D=======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    .  .   ld2	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: [0,7]     .D===============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============================eeeeeeeeER.   ld2	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: [0,9]     . D======================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER   .   ld2	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld2	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: [0,3]     D==eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeeeeER .   ld2	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: [0,5]     .D==eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeeeER.   ld2	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: [0,7]     .D===eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeeeeER   ld2	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,9]     . D===eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2067,43 +2067,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld2	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld2	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: 5.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld2	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: 7.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     31.0   0.0    0.0       ld2	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: 9.     1     39.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.2   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld2	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld2	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld2	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld2	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    3.0       <total>
 
 # CHECK:      [20] Code Region - G21
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        2700
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    0.67
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    5.29
+# CHECK-NEXT: IPC:               1.96
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld2	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    .  .   ld2	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: [0,3]     D================eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D================eeeeeeeeER   .    .    .  .   ld2	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: [0,5]     .D=======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    .  .   ld2	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: [0,7]     .D===============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===============================eeeeeeeeER.   ld2	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: [0,9]     .D=======================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER   .   ld2	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld2	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeeeeeER .   ld2	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,5]     .D==eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeeeER.   ld2	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,7]     .D===eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeeeeeeeER   ld2	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,9]     .D====eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2113,43 +2113,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld2	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     17.0   0.0    0.0       ld2	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: 5.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld2	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: 7.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     32.0   0.0    0.0       ld2	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: 9.     1     40.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.5   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld2	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld2	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld2	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ld2	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: 9.     1     5.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.0    0.1    3.0       <total>
 
 # CHECK:      [21] Code Region - G22
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      3310
 # CHECK-NEXT: Total uOps:        2600
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    0.65
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    0.79
+# CHECK-NEXT: IPC:               0.30
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld2	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    .  .   ld2	{ v1.b, v2.b }[0], [x27], #2
-# CHECK-NEXT: [0,3]     D================eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D================eeeeeeeeER   .    .    .  .   ld2	{ v1.b, v2.b }[8], [x27], #2
-# CHECK-NEXT: [0,5]     .D=======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    .  .   ld2	{ v1.b, v2.b }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .D===============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===============================eeeeeeeeER.   ld2	{ v1.b, v2.b }[8], [x27], x28
-# CHECK-NEXT: [0,9]     .D=======================================eER   add	x0, x27, #1
+# CHECK-NEXT: Index     0123456789          0123456789          012
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld2	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE------R    .    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    . .   ld2	{ v1.b, v2.b }[0], [x27], #2
+# CHECK-NEXT: [0,3]     D=========eE------R .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D================eeeeeeeeER   .    .    . .   ld2	{ v1.b, v2.b }[8], [x27], #2
+# CHECK-NEXT: [0,5]     .D================eE------R   .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    . .   ld2	{ v1.b, v2.b }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .D========================eE------R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===============================eeeeeeeeER   ld2	{ v1.b, v2.b }[8], [x27], x28
+# CHECK-NEXT: [0,9]     .D================================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2159,22 +2159,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld2	{ v1.b, v2.b }[0], [x27], #2
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 3.     1     10.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     17.0   0.0    0.0       ld2	{ v1.b, v2.b }[8], [x27], #2
-# CHECK-NEXT: 5.     1     24.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 5.     1     17.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld2	{ v1.b, v2.b }[0], [x27], x28
-# CHECK-NEXT: 7.     1     32.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 7.     1     25.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 8.     1     32.0   0.0    0.0       ld2	{ v1.b, v2.b }[8], [x27], x28
-# CHECK-NEXT: 9.     1     40.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.5   0.1    0.0       <total>
+# CHECK-NEXT: 9.     1     33.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     17.0   0.1    3.0       <total>
 
 # CHECK:      [22] Code Region - G23
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        2500
 
 # CHECK:      Dispatch Width:    15
@@ -2184,18 +2184,18 @@ add x0, x27, 1
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld2	{ v1.h, v2.h }[0], [x27], #4
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    .  .   ld2	{ v1.h, v2.h }[4], [x27], #4
-# CHECK-NEXT: [0,3]     D================eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D================eeeeeeeeER   .    .    .  .   ld2	{ v1.h, v2.h }[0], [x27], x28
-# CHECK-NEXT: [0,5]     D========================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    .  .   ld2	{ v1.h, v2.h }[4], [x27], x28
-# CHECK-NEXT: [0,7]     .D===============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===============================eeeeeeeeER.   ld2	{ v1.s, v2.s }[0], [x27], #8
-# CHECK-NEXT: [0,9]     .D=======================================eER   add	x0, x27, #1
+# CHECK-NEXT: Index     0123456789          0123456789          012
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld2	{ v1.h, v2.h }[0], [x27], #4
+# CHECK-NEXT: [0,1]     D=eE------R    .    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    . .   ld2	{ v1.h, v2.h }[4], [x27], #4
+# CHECK-NEXT: [0,3]     D=========eE------R .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D================eeeeeeeeER   .    .    . .   ld2	{ v1.h, v2.h }[0], [x27], x28
+# CHECK-NEXT: [0,5]     D=================eE------R   .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    . .   ld2	{ v1.h, v2.h }[4], [x27], x28
+# CHECK-NEXT: [0,7]     .D========================eE------R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===============================eeeeeeeeER   ld2	{ v1.s, v2.s }[0], [x27], #8
+# CHECK-NEXT: [0,9]     .D================================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2205,43 +2205,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.h, v2.h }[0], [x27], #4
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld2	{ v1.h, v2.h }[4], [x27], #4
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 3.     1     10.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     17.0   0.0    0.0       ld2	{ v1.h, v2.h }[0], [x27], x28
-# CHECK-NEXT: 5.     1     25.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 5.     1     18.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld2	{ v1.h, v2.h }[4], [x27], x28
-# CHECK-NEXT: 7.     1     32.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 7.     1     25.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 8.     1     32.0   0.0    0.0       ld2	{ v1.s, v2.s }[0], [x27], #8
-# CHECK-NEXT: 9.     1     40.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.6   0.1    0.0       <total>
+# CHECK-NEXT: 9.     1     33.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     17.1   0.1    3.0       <total>
 
 # CHECK:      [23] Code Region - G24
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      2603
 # CHECK-NEXT: Total uOps:        2500
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    0.62
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    0.96
+# CHECK-NEXT: IPC:               0.38
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld2	{ v1.s, v2.s }[0], [x27], x28
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    .  .   ld2	{ v1.d, v2.d }[0], [x27], #16
-# CHECK-NEXT: [0,3]     D================eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D================eeeeeeeeER   .    .    .  .   ld2	{ v1.d, v2.d }[0], [x27], x28
-# CHECK-NEXT: [0,5]     D========================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    .  .   ld2r	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: [0,7]     .D===============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===============================eeeeeeeeER.   ld2r	{ v1.2d, v2.2d }, [x27], #16
-# CHECK-NEXT: [0,9]     .D=======================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          012345678
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    .  .   ld2	{ v1.s, v2.s }[0], [x27], x28
+# CHECK-NEXT: [0,1]     D=eE------R    .    .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .  .   ld2	{ v1.d, v2.d }[0], [x27], #16
+# CHECK-NEXT: [0,3]     D=========eE------R .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D================eeeeeeeeER .   ld2	{ v1.d, v2.d }[0], [x27], x28
+# CHECK-NEXT: [0,5]     D=================eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D================eeeeeeeeER.   ld2r	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: [0,7]     .D=================eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D=================eeeeeeeeER   ld2r	{ v1.2d, v2.2d }, [x27], #16
+# CHECK-NEXT: [0,9]     .D==================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2251,43 +2251,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.s, v2.s }[0], [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld2	{ v1.d, v2.d }[0], [x27], #16
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 3.     1     10.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     17.0   0.0    0.0       ld2	{ v1.d, v2.d }[0], [x27], x28
-# CHECK-NEXT: 5.     1     25.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld2r	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: 7.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     32.0   0.0    0.0       ld2r	{ v1.2d, v2.2d }, [x27], #16
-# CHECK-NEXT: 9.     1     40.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.6   0.1    0.0       <total>
+# CHECK-NEXT: 5.     1     18.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     17.0   0.0    0.0       ld2r	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: 7.     1     18.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     18.0   0.0    0.0       ld2r	{ v1.2d, v2.2d }, [x27], #16
+# CHECK-NEXT: 9.     1     19.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     12.9   0.1    3.0       <total>
 
 # CHECK:      [24] Code Region - G25
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        2500
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    0.62
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    4.90
+# CHECK-NEXT: IPC:               1.96
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld2r	{ v1.2s, v2.2s }, [x27], #8
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    .  .   ld2r	{ v1.4h, v2.4h }, [x27], #4
-# CHECK-NEXT: [0,3]     D================eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D================eeeeeeeeER   .    .    .  .   ld2r	{ v1.4s, v2.4s }, [x27], #8
-# CHECK-NEXT: [0,5]     D========================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    .  .   ld2r	{ v1.8b, v2.8b }, [x27], #2
-# CHECK-NEXT: [0,7]     .D===============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===============================eeeeeeeeER.   ld2r	{ v1.8h, v2.8h }, [x27], #4
-# CHECK-NEXT: [0,9]     .D=======================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER   .   ld2r	{ v1.2s, v2.2s }, [x27], #8
+# CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld2r	{ v1.4h, v2.4h }, [x27], #4
+# CHECK-NEXT: [0,3]     D==eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeeeeeER .   ld2r	{ v1.4s, v2.4s }, [x27], #8
+# CHECK-NEXT: [0,5]     D===eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeeeER.   ld2r	{ v1.8b, v2.8b }, [x27], #2
+# CHECK-NEXT: [0,7]     .D===eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeeeeeeeER   ld2r	{ v1.8h, v2.8h }, [x27], #4
+# CHECK-NEXT: [0,9]     .D====eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2297,43 +2297,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2r	{ v1.2s, v2.2s }, [x27], #8
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld2r	{ v1.4h, v2.4h }, [x27], #4
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     17.0   0.0    0.0       ld2r	{ v1.4s, v2.4s }, [x27], #8
-# CHECK-NEXT: 5.     1     25.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld2r	{ v1.8b, v2.8b }, [x27], #2
-# CHECK-NEXT: 7.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     32.0   0.0    0.0       ld2r	{ v1.8h, v2.8h }, [x27], #4
-# CHECK-NEXT: 9.     1     40.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.6   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld2r	{ v1.4h, v2.4h }, [x27], #4
+# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld2r	{ v1.4s, v2.4s }, [x27], #8
+# CHECK-NEXT: 5.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld2r	{ v1.8b, v2.8b }, [x27], #2
+# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ld2r	{ v1.8h, v2.8h }, [x27], #4
+# CHECK-NEXT: 9.     1     5.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.1    0.1    3.0       <total>
 
 # CHECK:      [25] Code Region - G26
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        2500
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    0.62
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    4.90
+# CHECK-NEXT: IPC:               1.96
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld2r	{ v1.16b, v2.16b }, [x27], #2
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    .  .   ld2r	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: [0,3]     D================eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D================eeeeeeeeER   .    .    .  .   ld2r	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: [0,5]     D========================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    .  .   ld2r	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: [0,7]     .D===============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===============================eeeeeeeeER.   ld2r	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: [0,9]     .D=======================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER   .   ld2r	{ v1.16b, v2.16b }, [x27], #2
+# CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld2r	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeeeeeER .   ld2r	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,5]     D===eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeeeER.   ld2r	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,7]     .D===eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeeeeeeeER   ld2r	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,9]     .D====eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2343,43 +2343,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2r	{ v1.16b, v2.16b }, [x27], #2
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld2r	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     17.0   0.0    0.0       ld2r	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: 5.     1     25.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld2r	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: 7.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     32.0   0.0    0.0       ld2r	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: 9.     1     40.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.6   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld2r	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld2r	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: 5.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld2r	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ld2r	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: 9.     1     5.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.1    0.1    3.0       <total>
 
 # CHECK:      [26] Code Region - G27
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        2800
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    0.70
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    5.49
+# CHECK-NEXT: IPC:               1.96
 # CHECK-NEXT: Block RThroughput: 2.8
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld2r	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    .  .   ld2r	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: [0,3]     D================eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D================eeeeeeeeER   .    .    .  .   ld2r	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: [0,5]     D========================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    .  .   ld2r	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: [0,7]     .D===============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===============================eeeeeeeeER.   ld3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: [0,9]     .D=======================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER   .   ld2r	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld2r	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeeeeeER .   ld2r	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,5]     D===eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeeeER.   ld2r	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,7]     .D===eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeeeeeeeER   ld3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: [0,9]     .D====eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2389,43 +2389,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2r	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld2r	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     17.0   0.0    0.0       ld2r	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: 5.     1     25.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld2r	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: 7.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     32.0   0.0    0.0       ld3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: 9.     1     40.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.6   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld2r	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld2r	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: 5.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld2r	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ld3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: 9.     1     5.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.1    0.1    3.0       <total>
 
 # CHECK:      [27] Code Region - G28
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        3700
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    0.92
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    7.25
+# CHECK-NEXT: IPC:               1.96
 # CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    .  .   ld3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: [0,3]     D================eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER   .    .    .  .   ld3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: [0,5]     .D=======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    .  .   ld3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: [0,7]     .D===============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============================eeeeeeeeER.   ld3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: [0,9]     . D======================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER   .   ld3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: [0,3]     D==eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeeeeER .   ld3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: [0,5]     .D==eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeeeER.   ld3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: [0,7]     .D===eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeeeeER   ld3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: [0,9]     . D===eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2435,43 +2435,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: 5.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: 7.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     31.0   0.0    0.0       ld3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: 9.     1     39.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.2   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    3.0       <total>
 
 # CHECK:      [28] Code Region - G29
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        3800
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    0.95
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    7.45
+# CHECK-NEXT: IPC:               1.96
 # CHECK-NEXT: Block RThroughput: 4.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    .  .   ld3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: [0,3]     .D===============eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER   .    .    .  .   ld3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: [0,5]     .D=======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    .  .   ld3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: [0,7]     .D===============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============================eeeeeeeeER.   ld3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: [0,9]     . D======================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER   .   ld3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeeeeER .   ld3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,5]     .D==eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeeeER.   ld3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,7]     .D===eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeeeeER   ld3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: [0,9]     . D===eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2481,43 +2481,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: 3.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: 5.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: 7.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     31.0   0.0    0.0       ld3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: 9.     1     39.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.1   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.6    0.1    3.0       <total>
 
 # CHECK:      [29] Code Region - G30
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      1910
 # CHECK-NEXT: Total uOps:        3700
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    0.92
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    1.94
+# CHECK-NEXT: IPC:               0.52
 # CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    .  .   ld3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: [0,3]     D================eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER   .    .    .  .   ld3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: [0,5]     .D=======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    .  .   ld3	{ v1.b, v2.b, v3.b }[0], [x27], #3
-# CHECK-NEXT: [0,7]     .D===============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============================eeeeeeeeER.   ld3	{ v1.b, v2.b, v3.b }[8], [x27], #3
-# CHECK-NEXT: [0,9]     . D======================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          012345678
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    .  .   ld3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE------R    .    .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeeeER   .    .    .  .   ld3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eE------R   .    .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeeeeER  .    .    .  .   ld3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,5]     .D==eE------R  .    .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D=========eeeeeeeeER    .  .   ld3	{ v1.b, v2.b, v3.b }[0], [x27], #3
+# CHECK-NEXT: [0,7]     .D==========eE------R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D================eeeeeeeeER   ld3	{ v1.b, v2.b, v3.b }[8], [x27], #3
+# CHECK-NEXT: [0,9]     . D=================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2527,22 +2527,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: 5.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[0], [x27], #3
-# CHECK-NEXT: 7.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     31.0   0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[8], [x27], #3
-# CHECK-NEXT: 9.     1     39.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.2   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     10.0   0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[0], [x27], #3
+# CHECK-NEXT: 7.     1     11.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     17.0   0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[8], [x27], #3
+# CHECK-NEXT: 9.     1     18.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     6.9    0.1    3.0       <total>
 
 # CHECK:      [30] Code Region - G31
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        3500
 
 # CHECK:      Dispatch Width:    15
@@ -2552,18 +2552,18 @@ add x0, x27, 1
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld3	{ v1.b, v2.b, v3.b }[0], [x27], x28
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    .  .   ld3	{ v1.b, v2.b, v3.b }[8], [x27], x28
-# CHECK-NEXT: [0,3]     D================eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER   .    .    .  .   ld3	{ v1.h, v2.h, v3.h }[0], [x27], #6
-# CHECK-NEXT: [0,5]     .D=======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    .  .   ld3	{ v1.h, v2.h, v3.h }[4], [x27], #6
-# CHECK-NEXT: [0,7]     .D===============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============================eeeeeeeeER.   ld3	{ v1.h, v2.h, v3.h }[0], [x27], x28
-# CHECK-NEXT: [0,9]     . D======================================eER   add	x0, x27, #1
+# CHECK-NEXT: Index     0123456789          0123456789          012
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld3	{ v1.b, v2.b, v3.b }[0], [x27], x28
+# CHECK-NEXT: [0,1]     D=eE------R    .    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    . .   ld3	{ v1.b, v2.b, v3.b }[8], [x27], x28
+# CHECK-NEXT: [0,3]     D=========eE------R .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER   .    .    . .   ld3	{ v1.h, v2.h, v3.h }[0], [x27], #6
+# CHECK-NEXT: [0,5]     .D================eE------R   .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    . .   ld3	{ v1.h, v2.h, v3.h }[4], [x27], #6
+# CHECK-NEXT: [0,7]     .D========================eE------R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==============================eeeeeeeeER   ld3	{ v1.h, v2.h, v3.h }[0], [x27], x28
+# CHECK-NEXT: [0,9]     . D===============================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2573,22 +2573,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.b, v2.b, v3.b }[0], [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[8], [x27], x28
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 3.     1     10.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld3	{ v1.h, v2.h, v3.h }[0], [x27], #6
-# CHECK-NEXT: 5.     1     24.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 5.     1     17.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld3	{ v1.h, v2.h, v3.h }[4], [x27], #6
-# CHECK-NEXT: 7.     1     32.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 7.     1     25.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 8.     1     31.0   0.0    0.0       ld3	{ v1.h, v2.h, v3.h }[0], [x27], x28
-# CHECK-NEXT: 9.     1     39.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.2   0.1    0.0       <total>
+# CHECK-NEXT: 9.     1     32.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     16.7   0.1    3.0       <total>
 
 # CHECK:      [31] Code Region - G32
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        3500
 
 # CHECK:      Dispatch Width:    15
@@ -2598,18 +2598,18 @@ add x0, x27, 1
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld3	{ v1.h, v2.h, v3.h }[4], [x27], x28
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    .  .   ld3	{ v1.s, v2.s, v3.s }[0], [x27], #12
-# CHECK-NEXT: [0,3]     D================eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER   .    .    .  .   ld3	{ v1.s, v2.s, v3.s }[0], [x27], x28
-# CHECK-NEXT: [0,5]     .D=======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    .  .   ld3	{ v1.d, v2.d, v3.d }[0], [x27], #24
-# CHECK-NEXT: [0,7]     .D===============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============================eeeeeeeeER.   ld3	{ v1.d, v2.d, v3.d }[0], [x27], x28
-# CHECK-NEXT: [0,9]     . D======================================eER   add	x0, x27, #1
+# CHECK-NEXT: Index     0123456789          0123456789          012
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld3	{ v1.h, v2.h, v3.h }[4], [x27], x28
+# CHECK-NEXT: [0,1]     D=eE------R    .    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    . .   ld3	{ v1.s, v2.s, v3.s }[0], [x27], #12
+# CHECK-NEXT: [0,3]     D=========eE------R .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER   .    .    . .   ld3	{ v1.s, v2.s, v3.s }[0], [x27], x28
+# CHECK-NEXT: [0,5]     .D================eE------R   .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    . .   ld3	{ v1.d, v2.d, v3.d }[0], [x27], #24
+# CHECK-NEXT: [0,7]     .D========================eE------R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==============================eeeeeeeeER   ld3	{ v1.d, v2.d, v3.d }[0], [x27], x28
+# CHECK-NEXT: [0,9]     . D===============================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2619,43 +2619,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.h, v2.h, v3.h }[4], [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld3	{ v1.s, v2.s, v3.s }[0], [x27], #12
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 3.     1     10.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld3	{ v1.s, v2.s, v3.s }[0], [x27], x28
-# CHECK-NEXT: 5.     1     24.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 5.     1     17.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld3	{ v1.d, v2.d, v3.d }[0], [x27], #24
-# CHECK-NEXT: 7.     1     32.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 7.     1     25.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 8.     1     31.0   0.0    0.0       ld3	{ v1.d, v2.d, v3.d }[0], [x27], x28
-# CHECK-NEXT: 9.     1     39.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.2   0.1    0.0       <total>
+# CHECK-NEXT: 9.     1     32.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     16.7   0.1    3.0       <total>
 
 # CHECK:      [32] Code Region - G33
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        3500
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    0.87
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    6.86
+# CHECK-NEXT: IPC:               1.96
 # CHECK-NEXT: Block RThroughput: 3.8
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    .  .   ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], #24
-# CHECK-NEXT: [0,3]     D================eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER   .    .    .  .   ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], #12
-# CHECK-NEXT: [0,5]     .D=======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    .  .   ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], #6
-# CHECK-NEXT: [0,7]     .D===============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============================eeeeeeeeER.   ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], #12
-# CHECK-NEXT: [0,9]     . D======================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER   .   ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], #24
+# CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], #24
+# CHECK-NEXT: [0,3]     D==eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeeeeER .   ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], #12
+# CHECK-NEXT: [0,5]     .D==eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeeeER.   ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], #6
+# CHECK-NEXT: [0,7]     .D===eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeeeeER   ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], #12
+# CHECK-NEXT: [0,9]     . D===eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2665,43 +2665,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], #24
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], #12
-# CHECK-NEXT: 5.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], #6
-# CHECK-NEXT: 7.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     31.0   0.0    0.0       ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], #12
-# CHECK-NEXT: 9.     1     39.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.2   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], #24
+# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], #12
+# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], #6
+# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], #12
+# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    3.0       <total>
 
 # CHECK:      [33] Code Region - G34
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        3500
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    0.87
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    6.86
+# CHECK-NEXT: IPC:               1.96
 # CHECK-NEXT: Block RThroughput: 3.8
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], #3
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    .  .   ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], #6
-# CHECK-NEXT: [0,3]     D================eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER   .    .    .  .   ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], #3
-# CHECK-NEXT: [0,5]     .D=======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    .  .   ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: [0,7]     .D===============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============================eeeeeeeeER.   ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: [0,9]     . D======================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER   .   ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], #3
+# CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], #6
+# CHECK-NEXT: [0,3]     D==eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeeeeER .   ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], #3
+# CHECK-NEXT: [0,5]     .D==eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeeeER.   ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: [0,7]     .D===eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeeeeER   ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: [0,9]     . D===eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2711,43 +2711,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], #3
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], #6
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], #3
-# CHECK-NEXT: 5.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: 7.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     31.0   0.0    0.0       ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: 9.     1     39.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.2   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], #6
+# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], #3
+# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    3.0       <total>
 
 # CHECK:      [34] Code Region - G35
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        3500
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    0.87
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    6.86
+# CHECK-NEXT: IPC:               1.96
 # CHECK-NEXT: Block RThroughput: 3.8
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    .  .   ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: [0,3]     D================eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER   .    .    .  .   ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: [0,5]     .D=======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    .  .   ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: [0,7]     .D===============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============================eeeeeeeeER.   ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: [0,9]     . D======================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER   .   ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeeeeER .   ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: [0,5]     .D==eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeeeeeER.   ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,7]     .D===eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeeeeER   ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: [0,9]     . D===eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2757,43 +2757,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: 5.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: 7.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     31.0   0.0    0.0       ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: 9.     1     39.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.2   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    3.0       <total>
 
 # CHECK:      [35] Code Region - G36
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4204
+# CHECK-NEXT: Total Cycles:      611
 # CHECK-NEXT: Total uOps:        4500
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    1.07
-# CHECK-NEXT: IPC:               0.24
+# CHECK-NEXT: uOps Per Cycle:    7.36
+# CHECK-NEXT: IPC:               1.64
 # CHECK-NEXT: Block RThroughput: 5.3
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          012345
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .    .   ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeeER.    .    .    .    .    .   ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: [0,3]     .D================eER    .    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D===============eeeeeeeeER  .    .    .    .   ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: [0,5]     . D=======================eER .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D======================eeeeeeeeER    .    .   ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: [0,7]     .  D==============================eER   .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D=============================eeeeeeeeeER.   ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: [0,9]     .   D======================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0123456
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER    ..   ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE------R    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeeER  ..   ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: [0,3]     .D=eE-------R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeeeER  ..   ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: [0,5]     . D=eE------R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeeeER ..   ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: [0,7]     .  D=eE------R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D=eeeeeeeeeER   ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: [0,9]     .   D==eE-------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2803,43 +2803,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: 5.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     23.0   0.0    0.0       ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: 7.     1     31.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     30.0   0.0    0.0       ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: 9.     1     39.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     19.8   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: 3.     1     2.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: 5.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: 7.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     2.0    1.0    0.0       ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: 9.     1     3.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.7    0.2    3.2       <total>
 
 # CHECK:      [36] Code Region - G37
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4304
+# CHECK-NEXT: Total Cycles:      610
 # CHECK-NEXT: Total uOps:        4800
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    1.12
-# CHECK-NEXT: IPC:               0.23
+# CHECK-NEXT: uOps Per Cycle:    7.87
+# CHECK-NEXT: IPC:               1.64
 # CHECK-NEXT: Block RThroughput: 6.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123456
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .    ..   ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeeER.    .    .    .    .    ..   ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: [0,3]     .D================eER    .    .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D===============eeeeeeeeeER .    .    .    ..   ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: [0,5]     . D========================eER.    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=======================eeeeeeeeeER  .    ..   ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D================================eER .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D===============================eeeeeeeeER.   ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D=======================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     012345
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER    .   ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: [0,1]     D=eE------R    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeeER  .   ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: [0,3]     .D=eE-------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeeeeER .   ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: [0,5]     . D=eE-------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeeeeER.   ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D=eE-------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D=eeeeeeeeER   ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D==eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2849,43 +2849,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: 5.     1     25.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: 7.     1     33.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     32.0   0.0    0.0       ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: 9.     1     40.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.5   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: 3.     1     2.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: 5.     1     2.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: 7.     1     2.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     2.0    1.0    0.0       ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 9.     1     3.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.7    0.2    3.3       <total>
 
 # CHECK:      [37] Code Region - G38
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4304
+# CHECK-NEXT: Total Cycles:      660
 # CHECK-NEXT: Total uOps:        4800
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    1.12
-# CHECK-NEXT: IPC:               0.23
+# CHECK-NEXT: uOps Per Cycle:    7.27
+# CHECK-NEXT: IPC:               1.52
 # CHECK-NEXT: Block RThroughput: 6.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123456
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .    ..   ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeeER.    .    .    .    .    ..   ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: [0,3]     .D================eER    .    .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D===============eeeeeeeeER  .    .    .    ..   ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: [0,5]     . D=======================eER .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D======================eeeeeeeeeER   .    ..   ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D===============================eER  .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D==============================eeeeeeeeeER.   ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D=======================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0123456
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER    ..   ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE------R    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeeER  ..   ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE-------R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeeeER  ..   ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: [0,5]     . D=eE------R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeeeeER..   ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D=eE-------R..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D=eeeeeeeeeER   ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D==eE-------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2895,22 +2895,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: 3.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: 5.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     23.0   0.0    0.0       ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: 7.     1     32.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     31.0   0.0    0.0       ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: 9.     1     40.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     20.1   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: 7.     1     2.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     2.0    1.0    0.0       ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: 9.     1     3.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.7    0.2    3.3       <total>
 
 # CHECK:      [38] Code Region - G39
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        4500
 
 # CHECK:      Dispatch Width:    15
@@ -2920,18 +2920,18 @@ add x0, x27, 1
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    .  .   ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
-# CHECK-NEXT: [0,3]     .D===============eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    .  .   ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
-# CHECK-NEXT: [0,5]     . D======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    .  .   ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
-# CHECK-NEXT: [0,7]     .  D=============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER.   ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
-# CHECK-NEXT: [0,9]     .   D====================================eER   add	x0, x27, #1
+# CHECK-NEXT: Index     0123456789          0123456789          012
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
+# CHECK-NEXT: [0,1]     D=eE------R    .    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
+# CHECK-NEXT: [0,3]     .D========eE------R .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
+# CHECK-NEXT: [0,5]     . D===============eE------R   .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
+# CHECK-NEXT: [0,7]     .  D======================eE------R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER   ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
+# CHECK-NEXT: [0,9]     .   D=============================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2941,22 +2941,22 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
-# CHECK-NEXT: 3.     1     16.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 3.     1     9.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
-# CHECK-NEXT: 5.     1     23.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 5.     1     16.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
-# CHECK-NEXT: 7.     1     30.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 7.     1     23.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
-# CHECK-NEXT: 9.     1     37.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     19.0   0.1    0.0       <total>
+# CHECK-NEXT: 9.     1     30.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     15.5   0.1    3.0       <total>
 
 # CHECK:      [39] Code Region - G40
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        4500
 
 # CHECK:      Dispatch Width:    15
@@ -2966,18 +2966,18 @@ add x0, x27, 1
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    .  .   ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
-# CHECK-NEXT: [0,3]     .D===============eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    .  .   ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
-# CHECK-NEXT: [0,5]     . D======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    .  .   ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
-# CHECK-NEXT: [0,7]     .  D=============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER.   ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
-# CHECK-NEXT: [0,9]     .   D====================================eER   add	x0, x27, #1
+# CHECK-NEXT: Index     0123456789          0123456789          012
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
+# CHECK-NEXT: [0,1]     D=eE------R    .    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    . .   ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .D========eE------R .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    . .   ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
+# CHECK-NEXT: [0,5]     . D===============eE------R   .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    . .   ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
+# CHECK-NEXT: [0,7]     .  D======================eE------R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER   ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
+# CHECK-NEXT: [0,9]     .   D=============================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2987,43 +2987,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
-# CHECK-NEXT: 3.     1     16.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 3.     1     9.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
-# CHECK-NEXT: 5.     1     23.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 5.     1     16.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
-# CHECK-NEXT: 7.     1     30.0   0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 7.     1     23.0   0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
-# CHECK-NEXT: 9.     1     37.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     19.0   0.1    0.0       <total>
+# CHECK-NEXT: 9.     1     30.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     15.5   0.1    3.0       <total>
 
 # CHECK:      [40] Code Region - G41
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      1903
 # CHECK-NEXT: Total uOps:        4500
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    1.12
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    2.36
+# CHECK-NEXT: IPC:               0.53
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    .  .   ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
-# CHECK-NEXT: [0,3]     .D===============eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    .  .   ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: [0,5]     . D======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    .  .   ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #32
-# CHECK-NEXT: [0,7]     .  D=============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER.   ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #16
-# CHECK-NEXT: [0,9]     .   D====================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          01
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    ..   ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
+# CHECK-NEXT: [0,1]     D=eE------R    .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER ..   ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .D========eE------R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=======eeeeeeeeER..   ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: [0,5]     . D========eE------R..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=======eeeeeeeeER.   ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #32
+# CHECK-NEXT: [0,7]     .  D========eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D=======eeeeeeeeER   ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #16
+# CHECK-NEXT: [0,9]     .   D========eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3033,43 +3033,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
-# CHECK-NEXT: 3.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: 5.     1     23.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #32
-# CHECK-NEXT: 7.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #16
-# CHECK-NEXT: 9.     1     37.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     19.0   0.1    0.0       <total>
+# CHECK-NEXT: 3.     1     9.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     8.0    0.0    0.0       ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: 5.     1     9.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     8.0    0.0    0.0       ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #32
+# CHECK-NEXT: 7.     1     9.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     8.0    0.0    0.0       ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #16
+# CHECK-NEXT: 9.     1     9.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     7.1    0.1    3.0       <total>
 
 # CHECK:      [41] Code Region - G42
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        4500
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    1.12
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    8.82
+# CHECK-NEXT: IPC:               1.96
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #8
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    .  .   ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #16
-# CHECK-NEXT: [0,3]     .D===============eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    .  .   ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #4
-# CHECK-NEXT: [0,5]     . D======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    .  .   ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #8
-# CHECK-NEXT: [0,7]     .  D=============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER.   ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #4
-# CHECK-NEXT: [0,9]     .   D====================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER   .   ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #8
+# CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeER  .   ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #16
+# CHECK-NEXT: [0,3]     .D=eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeeeER .   ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #4
+# CHECK-NEXT: [0,5]     . D=eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeeeER.   ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #8
+# CHECK-NEXT: [0,7]     .  D=eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeeeeER   ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #4
+# CHECK-NEXT: [0,9]     .   D=eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3079,43 +3079,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #8
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #16
-# CHECK-NEXT: 3.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #4
-# CHECK-NEXT: 5.     1     23.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #8
-# CHECK-NEXT: 7.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #4
-# CHECK-NEXT: 9.     1     37.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     19.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #16
+# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #4
+# CHECK-NEXT: 5.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #8
+# CHECK-NEXT: 7.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #4
+# CHECK-NEXT: 9.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    3.0       <total>
 
 # CHECK:      [42] Code Region - G43
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      4004
+# CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        4500
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    1.12
-# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: uOps Per Cycle:    8.82
+# CHECK-NEXT: IPC:               1.96
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789          0123
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    .  .   ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    .  .   ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: [0,3]     .D===============eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    .  .   ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: [0,5]     . D======================eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    .  .   ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D=============================eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER.   ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D====================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER   .   ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeER  .   ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeeeER .   ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,5]     . D=eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeeeER.   ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D=eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeeeeER   ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D=eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3125,43 +3125,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: 3.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: 5.     1     23.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: 7.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: 9.     1     37.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     19.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: 7.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: 9.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    3.0       <total>
 
 # CHECK:      [43] Code Region - G44
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3604
+# CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        3300
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    0.92
-# CHECK-NEXT: IPC:               0.28
+# CHECK-NEXT: uOps Per Cycle:    6.50
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 3.7
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456789
-# CHECK-NEXT: Index     0123456789          0123456789
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .   .   ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: [0,1]     D========eER   .    .    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .   .   ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: [0,3]     .D===============eER.    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .   .   ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: [0,5]     . D======================eER  .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     . D======================eeeeeeER  .   .   ldp	s1, s2, [x27], #248
-# CHECK-NEXT: [0,7]     . D============================eER .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D============================eeeeeeER.   ldp	d1, d2, [x27], #496
-# CHECK-NEXT: [0,9]     . D==================================eER   add	x0, x27, #1
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeeER .   ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeER.   ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeeeER   ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,5]     . D=eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     . D=eeeeeeE-R   ldp	s1, s2, [x27], #248
+# CHECK-NEXT: [0,7]     . D==eE-----R   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeeER   ldp	d1, d2, [x27], #496
+# CHECK-NEXT: [0,9]     . D===eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3171,43 +3171,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: 1.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: 3.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: 5.     1     23.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     23.0   0.0    0.0       ldp	s1, s2, [x27], #248
-# CHECK-NEXT: 7.     1     29.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     29.0   0.0    0.0       ldp	d1, d2, [x27], #496
-# CHECK-NEXT: 9.     1     35.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     18.8   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    1.0       ldp	s1, s2, [x27], #248
+# CHECK-NEXT: 7.     1     3.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ldp	d1, d2, [x27], #496
+# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.1    0.1    2.8       <total>
 
 # CHECK:      [44] Code Region - G45
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2804
+# CHECK-NEXT: Total Cycles:      507
 # CHECK-NEXT: Total uOps:        1700
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    0.61
-# CHECK-NEXT: IPC:               0.36
+# CHECK-NEXT: uOps Per Cycle:    3.35
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          01
-# CHECK-NEXT: Index     0123456789          0123456789
-
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    ..   ldp	q1, q2, [x27], #992
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D======eeeeeeER.    .    .    ..   ldp	s1, s2, [x27, #248]!
-# CHECK-NEXT: [0,3]     D============eER    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D============eeeeeeER    .    ..   ldp	d1, d2, [x27, #496]!
-# CHECK-NEXT: [0,5]     D==================eER   .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     D==================eeeeeeER   ..   ldp	q1, q2, [x27, #992]!
-# CHECK-NEXT: [0,7]     D========================eER  ..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D=======================eeeeER.   ldp	w1, w2, [x27], #248
-# CHECK-NEXT: [0,9]     .D===========================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeER ..   ldp	q1, q2, [x27], #992
+# CHECK-NEXT: [0,1]     D=eE----R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeER..   ldp	s1, s2, [x27, #248]!
+# CHECK-NEXT: [0,3]     D==eE----R..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeeeER.   ldp	d1, d2, [x27, #496]!
+# CHECK-NEXT: [0,5]     D===eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     D===eeeeeeER   ldp	q1, q2, [x27, #992]!
+# CHECK-NEXT: [0,7]     D====eE----R   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeeeE-R   ldp	w1, w2, [x27], #248
+# CHECK-NEXT: [0,9]     .D====eE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3217,43 +3217,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldp	q1, q2, [x27], #992
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ldp	s1, s2, [x27, #248]!
-# CHECK-NEXT: 3.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ldp	d1, d2, [x27, #496]!
-# CHECK-NEXT: 5.     1     19.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ldp	q1, q2, [x27, #992]!
-# CHECK-NEXT: 7.     1     25.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     24.0   0.0    0.0       ldp	w1, w2, [x27], #248
-# CHECK-NEXT: 9.     1     28.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     15.6   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ldp	s1, s2, [x27, #248]!
+# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ldp	d1, d2, [x27, #496]!
+# CHECK-NEXT: 5.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     4.0    0.0    0.0       ldp	q1, q2, [x27, #992]!
+# CHECK-NEXT: 7.     1     5.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    1.0       ldp	w1, w2, [x27], #248
+# CHECK-NEXT: 9.     1     5.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.3    0.1    2.0       <total>
 
 # CHECK:      [45] Code Region - G46
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1604
+# CHECK-NEXT: Total Cycles:      507
 # CHECK-NEXT: Total uOps:        1900
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    1.18
-# CHECK-NEXT: IPC:               0.62
+# CHECK-NEXT: uOps Per Cycle:    3.75
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 3.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
+# CHECK-NEXT:                     01
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeER   .    .   .   ldp	x1, x2, [x27], #496
-# CHECK-NEXT: [0,1]     D=eE--R   .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeER  .    .   .   ldp	w1, w2, [x27, #248]!
-# CHECK-NEXT: [0,3]     D=====eER .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D=====eeeeER   .   .   ldp	x1, x2, [x27, #496]!
-# CHECK-NEXT: [0,5]     D======eE--R   .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     D======eeeeeER .   .   ldpsw	x1, x2, [x27], #248
-# CHECK-NEXT: [0,7]     D===========eER.   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D==========eeeeeER.   ldpsw	x1, x2, [x27, #248]!
-# CHECK-NEXT: [0,9]     .D===============eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeER   ..   ldp	x1, x2, [x27], #496
+# CHECK-NEXT: [0,1]     D=eE--R   ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeER  ..   ldp	w1, w2, [x27, #248]!
+# CHECK-NEXT: [0,3]     D==eE--R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeER ..   ldp	x1, x2, [x27, #496]!
+# CHECK-NEXT: [0,5]     D===eE--R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     D===eeeeeER.   ldpsw	x1, x2, [x27], #248
+# CHECK-NEXT: [0,7]     D====eE---R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeeeeER   ldpsw	x1, x2, [x27, #248]!
+# CHECK-NEXT: [0,9]     .D====eE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3265,41 +3265,41 @@ add x0, x27, 1
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldp	x1, x2, [x27], #496
 # CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     2.0    0.0    0.0       ldp	w1, w2, [x27, #248]!
-# CHECK-NEXT: 3.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     6.0    0.0    0.0       ldp	x1, x2, [x27, #496]!
-# CHECK-NEXT: 5.     1     7.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     7.0    0.0    0.0       ldpsw	x1, x2, [x27], #248
-# CHECK-NEXT: 7.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     11.0   0.0    0.0       ldpsw	x1, x2, [x27, #248]!
-# CHECK-NEXT: 9.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     7.0    0.1    0.4       <total>
+# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ldp	x1, x2, [x27, #496]!
+# CHECK-NEXT: 5.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     4.0    0.0    0.0       ldpsw	x1, x2, [x27], #248
+# CHECK-NEXT: 7.     1     5.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ldpsw	x1, x2, [x27, #248]!
+# CHECK-NEXT: 9.     1     5.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.3    0.1    1.2       <total>
 
 # CHECK:      [46] Code Region - G47
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3004
+# CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    0.50
-# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: uOps Per Cycle:    2.95
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123
-# CHECK-NEXT: Index     0123456789          0123456789
-
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .  .   ldr	b1, [x27], #254
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D======eeeeeeER.    .    .    .  .   ldr	h1, [x27], #254
-# CHECK-NEXT: [0,3]     D============eER    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D============eeeeeeER    .    .  .   ldr	s1, [x27], #254
-# CHECK-NEXT: [0,5]     D==================eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     D==================eeeeeeER   .  .   ldr	d1, [x27], #254
-# CHECK-NEXT: [0,7]     D========================eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     D========================eeeeeeER.   ldr	q1, [x27], #254
-# CHECK-NEXT: [0,9]     D==============================eER   add	x0, x27, #1
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeER . .   ldr	b1, [x27], #254
+# CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ldr	h1, [x27], #254
+# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeeeER .   ldr	s1, [x27], #254
+# CHECK-NEXT: [0,5]     D===eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     D===eeeeeeER.   ldr	d1, [x27], #254
+# CHECK-NEXT: [0,7]     D====eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     D====eeeeeeER   ldr	q1, [x27], #254
+# CHECK-NEXT: [0,9]     D=====eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3309,43 +3309,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldr	b1, [x27], #254
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ldr	h1, [x27], #254
-# CHECK-NEXT: 3.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ldr	s1, [x27], #254
-# CHECK-NEXT: 5.     1     19.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ldr	d1, [x27], #254
-# CHECK-NEXT: 7.     1     25.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     25.0   0.0    0.0       ldr	q1, [x27], #254
-# CHECK-NEXT: 9.     1     31.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ldr	h1, [x27], #254
+# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ldr	s1, [x27], #254
+# CHECK-NEXT: 5.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     4.0    0.0    0.0       ldr	d1, [x27], #254
+# CHECK-NEXT: 7.     1     5.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     5.0    0.0    0.0       ldr	q1, [x27], #254
+# CHECK-NEXT: 9.     1     6.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.5    0.1    2.0       <total>
 
 # CHECK:      [47] Code Region - G48
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3004
+# CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        1500
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    0.50
-# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: uOps Per Cycle:    2.95
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123
-# CHECK-NEXT: Index     0123456789          0123456789
-
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .  .   ldr	b1, [x27, #254]!
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D======eeeeeeER.    .    .    .  .   ldr	h1, [x27, #254]!
-# CHECK-NEXT: [0,3]     D============eER    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D============eeeeeeER    .    .  .   ldr	s1, [x27, #254]!
-# CHECK-NEXT: [0,5]     D==================eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     D==================eeeeeeER   .  .   ldr	d1, [x27, #254]!
-# CHECK-NEXT: [0,7]     D========================eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     D========================eeeeeeER.   ldr	q1, [x27, #254]!
-# CHECK-NEXT: [0,9]     D==============================eER   add	x0, x27, #1
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeER . .   ldr	b1, [x27, #254]!
+# CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ldr	h1, [x27, #254]!
+# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeeeER .   ldr	s1, [x27, #254]!
+# CHECK-NEXT: [0,5]     D===eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     D===eeeeeeER.   ldr	d1, [x27, #254]!
+# CHECK-NEXT: [0,7]     D====eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     D====eeeeeeER   ldr	q1, [x27, #254]!
+# CHECK-NEXT: [0,9]     D=====eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3355,16 +3355,16 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldr	b1, [x27, #254]!
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ldr	h1, [x27, #254]!
-# CHECK-NEXT: 3.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ldr	s1, [x27, #254]!
-# CHECK-NEXT: 5.     1     19.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ldr	d1, [x27, #254]!
-# CHECK-NEXT: 7.     1     25.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     25.0   0.0    0.0       ldr	q1, [x27, #254]!
-# CHECK-NEXT: 9.     1     31.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ldr	h1, [x27, #254]!
+# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ldr	s1, [x27, #254]!
+# CHECK-NEXT: 5.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     4.0    0.0    0.0       ldr	d1, [x27, #254]!
+# CHECK-NEXT: 7.     1     5.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     5.0    0.0    0.0       ldr	q1, [x27, #254]!
+# CHECK-NEXT: 9.     1     6.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.5    0.1    2.0       <total>
 
 # CHECK:      [48] Code Region - G49
 
@@ -3508,28 +3508,27 @@ add x0, x27, 1
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      704
+# CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        1700
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    2.41
-# CHECK-NEXT: IPC:               1.42
+# CHECK-NEXT: uOps Per Cycle:    3.37
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     012345678
 
-# CHECK:      [0,0]     DeeeeER   .   ldrsh	x1, [x27, #254]!
-# CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeER  .   ldrsw	x1, [x27], #254
-# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D==eeeeER .   ldrsw	x1, [x27, #254]!
-# CHECK-NEXT: [0,5]     D===eE--R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     D===eeE-R .   st1	{ v1.1d }, [x27], #8
-# CHECK-NEXT: [0,7]     D=====eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D====eeER.   st1	{ v1.2d }, [x27], #16
-# CHECK-NEXT: [0,9]     .D======eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeER .   ldrsh	x1, [x27, #254]!
+# CHECK-NEXT: [0,1]     D=eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeER.   ldrsw	x1, [x27], #254
+# CHECK-NEXT: [0,3]     D==eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeER   ldrsw	x1, [x27, #254]!
+# CHECK-NEXT: [0,5]     D===eE--R   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     D===eeE-R   st1	{ v1.1d }, [x27], #8
+# CHECK-NEXT: [0,7]     D====eE-R   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeER   st1	{ v1.2d }, [x27], #16
+# CHECK-NEXT: [0,9]     .D====eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3545,37 +3544,36 @@ add x0, x27, 1
 # CHECK-NEXT: 4.     1     3.0    0.0    0.0       ldrsw	x1, [x27, #254]!
 # CHECK-NEXT: 5.     1     4.0    0.0    2.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     4.0    0.0    1.0       st1	{ v1.1d }, [x27], #8
-# CHECK-NEXT: 7.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     5.0    0.0    0.0       st1	{ v1.2d }, [x27], #16
-# CHECK-NEXT: 9.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.7    0.1    0.7       <total>
+# CHECK-NEXT: 7.     1     5.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       st1	{ v1.2d }, [x27], #16
+# CHECK-NEXT: 9.     1     5.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.3    0.1    0.8       <total>
 
 # CHECK:      [52] Code Region - G53
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1004
+# CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    1.99
-# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: uOps Per Cycle:    3.97
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     012345678
 
-# CHECK:      [0,0]     DeeER.    .  .   st1	{ v1.2s }, [x27], #8
-# CHECK-NEXT: [0,1]     D==eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D==eeER   .  .   st1	{ v1.4h }, [x27], #8
-# CHECK-NEXT: [0,3]     D====eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D====eeER .  .   st1	{ v1.4s }, [x27], #16
-# CHECK-NEXT: [0,5]     D======eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     D======eeER  .   st1	{ v1.8b }, [x27], #8
-# CHECK-NEXT: [0,7]     .D=======eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D=======eeER.   st1	{ v1.8h }, [x27], #16
-# CHECK-NEXT: [0,9]     .D=========eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.  .   st1	{ v1.2s }, [x27], #8
+# CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeER  .   st1	{ v1.4h }, [x27], #8
+# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeER .   st1	{ v1.4s }, [x27], #16
+# CHECK-NEXT: [0,5]     D===eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     D===eeER.   st1	{ v1.8b }, [x27], #8
+# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeER   st1	{ v1.8h }, [x27], #16
+# CHECK-NEXT: [0,9]     .D====eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3585,43 +3583,42 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.2s }, [x27], #8
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st1	{ v1.4h }, [x27], #8
-# CHECK-NEXT: 3.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       st1	{ v1.4s }, [x27], #16
-# CHECK-NEXT: 5.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     7.0    0.0    0.0       st1	{ v1.8b }, [x27], #8
-# CHECK-NEXT: 7.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     8.0    0.0    0.0       st1	{ v1.8h }, [x27], #16
-# CHECK-NEXT: 9.     1     10.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     5.7    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.4h }, [x27], #8
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.4s }, [x27], #16
+# CHECK-NEXT: 5.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     4.0    0.0    0.0       st1	{ v1.8b }, [x27], #8
+# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       st1	{ v1.8h }, [x27], #16
+# CHECK-NEXT: 9.     1     5.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.2    0.1    0.0       <total>
 
 # CHECK:      [53] Code Region - G54
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1004
+# CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    1.99
-# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: uOps Per Cycle:    3.97
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     012345678
 
-# CHECK:      [0,0]     DeeER.    .  .   st1	{ v1.16b }, [x27], #16
-# CHECK-NEXT: [0,1]     D==eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D==eeER   .  .   st1	{ v1.1d }, [x27], x28
-# CHECK-NEXT: [0,3]     D====eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D====eeER .  .   st1	{ v1.2d }, [x27], x28
-# CHECK-NEXT: [0,5]     D======eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     D======eeER  .   st1	{ v1.2s }, [x27], x28
-# CHECK-NEXT: [0,7]     .D=======eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D=======eeER.   st1	{ v1.4h }, [x27], x28
-# CHECK-NEXT: [0,9]     .D=========eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.  .   st1	{ v1.16b }, [x27], #16
+# CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeER  .   st1	{ v1.1d }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeER .   st1	{ v1.2d }, [x27], x28
+# CHECK-NEXT: [0,5]     D===eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     D===eeER.   st1	{ v1.2s }, [x27], x28
+# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeER   st1	{ v1.4h }, [x27], x28
+# CHECK-NEXT: [0,9]     .D====eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3631,43 +3628,42 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.16b }, [x27], #16
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st1	{ v1.1d }, [x27], x28
-# CHECK-NEXT: 3.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       st1	{ v1.2d }, [x27], x28
-# CHECK-NEXT: 5.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     7.0    0.0    0.0       st1	{ v1.2s }, [x27], x28
-# CHECK-NEXT: 7.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     8.0    0.0    0.0       st1	{ v1.4h }, [x27], x28
-# CHECK-NEXT: 9.     1     10.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     5.7    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.1d }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.2d }, [x27], x28
+# CHECK-NEXT: 5.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     4.0    0.0    0.0       st1	{ v1.2s }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       st1	{ v1.4h }, [x27], x28
+# CHECK-NEXT: 9.     1     5.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.2    0.1    0.0       <total>
 
 # CHECK:      [54] Code Region - G55
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1004
+# CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    1.99
-# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: uOps Per Cycle:    3.97
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     012345678
 
-# CHECK:      [0,0]     DeeER.    .  .   st1	{ v1.4s }, [x27], x28
-# CHECK-NEXT: [0,1]     D==eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D==eeER   .  .   st1	{ v1.8b }, [x27], x28
-# CHECK-NEXT: [0,3]     D====eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D====eeER .  .   st1	{ v1.8h }, [x27], x28
-# CHECK-NEXT: [0,5]     D======eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     D======eeER  .   st1	{ v1.16b }, [x27], x28
-# CHECK-NEXT: [0,7]     .D=======eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D=======eeER.   st1	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: [0,9]     .D=========eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.  .   st1	{ v1.4s }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeER  .   st1	{ v1.8b }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeER .   st1	{ v1.8h }, [x27], x28
+# CHECK-NEXT: [0,5]     D===eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     D===eeER.   st1	{ v1.16b }, [x27], x28
+# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeER   st1	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: [0,9]     .D====eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3677,43 +3673,42 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.4s }, [x27], x28
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st1	{ v1.8b }, [x27], x28
-# CHECK-NEXT: 3.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       st1	{ v1.8h }, [x27], x28
-# CHECK-NEXT: 5.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     7.0    0.0    0.0       st1	{ v1.16b }, [x27], x28
-# CHECK-NEXT: 7.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     8.0    0.0    0.0       st1	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: 9.     1     10.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     5.7    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.8b }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.8h }, [x27], x28
+# CHECK-NEXT: 5.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     4.0    0.0    0.0       st1	{ v1.16b }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       st1	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: 9.     1     5.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.2    0.1    0.0       <total>
 
 # CHECK:      [55] Code Region - G56
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1004
+# CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2400
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    2.39
-# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: uOps Per Cycle:    4.76
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 3.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     012345678
 
-# CHECK:      [0,0]     DeeER.    .  .   st1	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: [0,1]     D==eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D==eeER   .  .   st1	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: [0,3]     D====eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D====eeER .  .   st1	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: [0,5]     D======eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=====eeER  .   st1	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: [0,7]     .D=======eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D=======eeER.   st1	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: [0,9]     .D=========eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.  .   st1	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeER  .   st1	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeER .   st1	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: [0,5]     D===eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeER.   st1	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeER   st1	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: [0,9]     .D====eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3723,43 +3718,42 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st1	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: 3.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       st1	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: 5.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     6.0    0.0    0.0       st1	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: 7.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     8.0    0.0    0.0       st1	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: 9.     1     10.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     5.6    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: 5.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st1	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       st1	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: 9.     1     5.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.1    0.1    0.0       <total>
 
 # CHECK:      [56] Code Region - G57
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1004
+# CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2600
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    2.59
-# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: uOps Per Cycle:    5.16
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     012345678
 
-# CHECK:      [0,0]     DeeER.    .  .   st1	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: [0,1]     D==eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D==eeER   .  .   st1	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: [0,3]     D====eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D====eeER .  .   st1	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: [0,5]     .D=====eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=====eeER  .   st1	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: [0,7]     .D=======eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D=======eeER.   st1	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: [0,9]     .D=========eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.  .   st1	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeER  .   st1	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeER .   st1	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: [0,5]     .D==eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeER.   st1	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeER   st1	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,9]     .D====eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3769,43 +3763,42 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st1	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: 3.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       st1	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: 5.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     6.0    0.0    0.0       st1	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: 7.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     8.0    0.0    0.0       st1	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: 9.     1     10.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     5.5    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st1	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       st1	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: 9.     1     5.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.0    0.1    0.0       <total>
 
 # CHECK:      [57] Code Region - G58
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1004
+# CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2600
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    2.59
-# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: uOps Per Cycle:    5.16
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     012345678
 
-# CHECK:      [0,0]     DeeER.    .  .   st1	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: [0,1]     D==eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D==eeER   .  .   st1	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: [0,3]     D====eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D====eeER .  .   st1	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: [0,5]     D======eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=====eeER  .   st1	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: [0,7]     .D=======eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D=======eeER.   st1	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: [0,9]     .D=========eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.  .   st1	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeER  .   st1	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeER .   st1	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,5]     D===eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeER.   st1	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeER   st1	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,9]     .D====eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3815,43 +3808,42 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st1	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: 3.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       st1	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: 5.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     6.0    0.0    0.0       st1	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: 7.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     8.0    0.0    0.0       st1	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: 9.     1     10.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     5.6    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: 5.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st1	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       st1	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: 9.     1     5.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.1    0.1    0.0       <total>
 
 # CHECK:      [58] Code Region - G59
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1004
+# CHECK-NEXT: Total Cycles:      703
 # CHECK-NEXT: Total uOps:        3400
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    3.39
-# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: uOps Per Cycle:    4.84
+# CHECK-NEXT: IPC:               1.42
 # CHECK-NEXT: Block RThroughput: 6.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeER.    .  .   st1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: [0,1]     D==eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D==eeER   .  .   st1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: [0,3]     D====eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===eeER .  .   st1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: [0,5]     .D=====eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=====eeER  .   st1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: [0,7]     .D=======eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D======eeER.   st1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: [0,9]     . D========eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.   .   st1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
+# CHECK-NEXT: [0,1]     D=eER.   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeER   .   st1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: [0,3]     D==eER   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeER  .   st1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: [0,5]     .D==eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D===eeER.   st1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: [0,7]     .D====eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D===eeER   st1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: [0,9]     . D====eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3861,43 +3853,42 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: 3.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       st1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: 5.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     6.0    0.0    0.0       st1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: 7.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     7.0    0.0    0.0       st1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: 9.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     5.2    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     4.0    1.0    0.0       st1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: 7.     1     5.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       st1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: 9.     1     5.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.1    0.2    0.0       <total>
 
 # CHECK:      [59] Code Region - G60
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1004
+# CHECK-NEXT: Total Cycles:      703
 # CHECK-NEXT: Total uOps:        3600
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    3.59
-# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: uOps Per Cycle:    5.12
+# CHECK-NEXT: IPC:               1.42
 # CHECK-NEXT: Block RThroughput: 6.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeER.    .  .   st1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: [0,1]     D==eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D==eeER   .  .   st1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: [0,3]     D====eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===eeER .  .   st1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: [0,5]     .D=====eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=====eeER  .   st1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: [0,7]     .D=======eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D======eeER.   st1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: [0,9]     . D========eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.   .   st1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: [0,1]     D=eER.   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeER   .   st1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: [0,3]     D==eER   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeER  .   st1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: [0,5]     .D==eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D===eeER.   st1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: [0,7]     .D====eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D===eeER   st1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: [0,9]     . D====eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3907,43 +3898,42 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: 3.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       st1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: 5.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     6.0    0.0    0.0       st1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: 7.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     7.0    0.0    0.0       st1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: 9.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     5.2    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     4.0    1.0    0.0       st1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: 7.     1     5.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       st1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: 9.     1     5.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.1    0.2    0.0       <total>
 
 # CHECK:      [60] Code Region - G61
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1004
+# CHECK-NEXT: Total Cycles:      703
 # CHECK-NEXT: Total uOps:        3400
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    3.39
-# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: uOps Per Cycle:    4.84
+# CHECK-NEXT: IPC:               1.42
 # CHECK-NEXT: Block RThroughput: 6.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeER.    .  .   st1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: [0,1]     D==eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D==eeER   .  .   st1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: [0,3]     D====eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===eeER .  .   st1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: [0,5]     .D=====eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=====eeER  .   st1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: [0,7]     .D=======eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D======eeER.   st1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: [0,9]     . D========eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.   .   st1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eER.   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeER   .   st1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eER   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeER  .   st1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: [0,5]     .D==eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeER .   st1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,7]     .D===eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D===eeER   st1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: [0,9]     . D====eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3953,43 +3943,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: 3.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       st1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: 5.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     6.0    0.0    0.0       st1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: 7.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     7.0    0.0    0.0       st1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: 9.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     5.2    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    1.0    0.0       st1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: 9.     1     5.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.9    0.2    0.0       <total>
 
 # CHECK:      [61] Code Region - G62
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1004
+# CHECK-NEXT: Total Cycles:      704
 # CHECK-NEXT: Total uOps:        3600
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    3.59
-# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: uOps Per Cycle:    5.11
+# CHECK-NEXT: IPC:               1.42
 # CHECK-NEXT: Block RThroughput: 6.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123
+# CHECK-NEXT:                     0
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeER.    .  .   st1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: [0,1]     D==eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D==eeER   .  .   st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: [0,3]     D====eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===eeER .  .   st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: [0,5]     .D=====eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=====eeER  .   st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: [0,7]     . D======eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D======eeER.   st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: [0,9]     . D========eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.    .   st1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eER.    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeER    .   st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: [0,3]     D==eER    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D==eeER  .   st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: [0,5]     .D===eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D===eeER .   st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: [0,7]     . D===eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D====eeER   st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: [0,9]     . D=====eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3999,43 +3989,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: 3.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: 5.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     6.0    0.0    0.0       st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: 7.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     7.0    0.0    0.0       st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: 9.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     5.1    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    1.0    0.0       st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: 5.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     4.0    0.0    0.0       st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     5.0    1.0    0.0       st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: 9.     1     6.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.4    0.3    0.0       <total>
 
 # CHECK:      [62] Code Region - G63
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1004
+# CHECK-NEXT: Total Cycles:      804
 # CHECK-NEXT: Total uOps:        4200
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    4.18
-# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: uOps Per Cycle:    5.22
+# CHECK-NEXT: IPC:               1.24
 # CHECK-NEXT: Block RThroughput: 8.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123
+# CHECK-NEXT:                     01
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeER.    .  .   st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: [0,1]     D==eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D==eeER   .  .   st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: [0,3]     .D===eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===eeER .  .   st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: [0,5]     .D=====eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     . D====eeER  .   st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: [0,7]     . D======eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D======eeER.   st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: [0,9]     .  D=======eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.    ..   st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: [0,1]     D=eER.    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeER    ..   st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: [0,3]     .D=eER    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D==eeER  ..   st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: [0,5]     .D===eER  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     . D==eeER ..   st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: [0,7]     . D===eER ..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D=====eeER   st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK-NEXT: [0,9]     .  D=====eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4045,43 +4035,42 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: 5.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     5.0    0.0    0.0       st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: 7.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     7.0    0.0    0.0       st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: 9.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     4.8    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    1.0    0.0       st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: 5.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     6.0    2.0    0.0       st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK-NEXT: 9.     1     6.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.3    0.4    0.0       <total>
 
 # CHECK:      [63] Code Region - G64
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1004
+# CHECK-NEXT: Total Cycles:      703
 # CHECK-NEXT: Total uOps:        3800
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    3.78
-# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: uOps Per Cycle:    5.41
+# CHECK-NEXT: IPC:               1.42
 # CHECK-NEXT: Block RThroughput: 7.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeER.    .  .   st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: [0,1]     D==eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D==eeER   .  .   st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: [0,3]     .D===eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===eeER .  .   st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: [0,5]     .D=====eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     . D====eeER  .   st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: [0,7]     . D======eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D======eeER.   st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: [0,9]     .  D=======eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.   .   st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eER.   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeER   .   st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eER   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D==eeER .   st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,5]     .D===eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     . D==eeER.   st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,7]     . D===eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D===eeER   st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: [0,9]     .  D===eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4091,43 +4080,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: 5.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     5.0    0.0    0.0       st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: 7.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     7.0    0.0    0.0       st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: 9.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     4.8    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    1.0    0.0       st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: 5.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.9    0.2    0.0       <total>
 
 # CHECK:      [64] Code Region - G65
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1604
+# CHECK-NEXT: Total Cycles:      706
 # CHECK-NEXT: Total uOps:        3200
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    2.00
-# CHECK-NEXT: IPC:               0.62
+# CHECK-NEXT: uOps Per Cycle:    4.53
+# CHECK-NEXT: IPC:               1.42
 # CHECK-NEXT: Block RThroughput: 5.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
+# CHECK-NEXT:                     012
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeER.    .    .   .   st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: [0,1]     D==eER    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=eeER   .    .   .   st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: [0,3]     .D===eER  .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===eeeeER    .   .   st1	{ v1.b }[0], [x27], #1
-# CHECK-NEXT: [0,5]     .D=======eER   .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     . D======eeeeER.   .   st1	{ v1.b }[8], [x27], #1
-# CHECK-NEXT: [0,7]     . D==========eER   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==========eeeeER.   st1	{ v1.b }[0], [x27], x28
-# CHECK-NEXT: [0,9]     . D==============eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.    . .   st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eER.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeER    . .   st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eER    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D===eeeeER .   st1	{ v1.b }[0], [x27], #1
+# CHECK-NEXT: [0,5]     .D====eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     . D===eeeeER.   st1	{ v1.b }[8], [x27], #1
+# CHECK-NEXT: [0,7]     . D====eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D====eeeeER   st1	{ v1.b }[0], [x27], x28
+# CHECK-NEXT: [0,9]     . D=====eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4137,43 +4126,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     4.0    0.0    0.0       st1	{ v1.b }[0], [x27], #1
-# CHECK-NEXT: 5.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     7.0    0.0    0.0       st1	{ v1.b }[8], [x27], #1
-# CHECK-NEXT: 7.     1     11.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     11.0   0.0    0.0       st1	{ v1.b }[0], [x27], x28
-# CHECK-NEXT: 9.     1     15.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     6.6    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     4.0    2.0    0.0       st1	{ v1.b }[0], [x27], #1
+# CHECK-NEXT: 5.     1     5.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     4.0    0.0    0.0       st1	{ v1.b }[8], [x27], #1
+# CHECK-NEXT: 7.     1     5.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     5.0    0.0    0.0       st1	{ v1.b }[0], [x27], x28
+# CHECK-NEXT: 9.     1     6.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.5    0.3    0.6       <total>
 
 # CHECK:      [65] Code Region - G66
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2004
+# CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    1.00
-# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: uOps Per Cycle:    3.95
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0123
-
-# CHECK:      [0,0]     DeeeeER   .    .    .  .   st1	{ v1.b }[8], [x27], x28
-# CHECK-NEXT: [0,1]     D====eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D====eeeeER    .    .  .   st1	{ v1.h }[0], [x27], #2
-# CHECK-NEXT: [0,3]     D========eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D========eeeeER.    .  .   st1	{ v1.h }[4], [x27], #2
-# CHECK-NEXT: [0,5]     D============eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     D============eeeeER .  .   st1	{ v1.h }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .D===============eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===============eeeeER.   st1	{ v1.h }[4], [x27], x28
-# CHECK-NEXT: [0,9]     .D===================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeER   .   st1	{ v1.b }[8], [x27], x28
+# CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeER  .   st1	{ v1.h }[0], [x27], #2
+# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeER .   st1	{ v1.h }[4], [x27], #2
+# CHECK-NEXT: [0,5]     D===eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     D===eeeeER.   st1	{ v1.h }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .D===eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeeeER   st1	{ v1.h }[4], [x27], x28
+# CHECK-NEXT: [0,9]     .D====eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4183,43 +4172,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.b }[8], [x27], x28
-# CHECK-NEXT: 1.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       st1	{ v1.h }[0], [x27], #2
-# CHECK-NEXT: 3.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     9.0    0.0    0.0       st1	{ v1.h }[4], [x27], #2
-# CHECK-NEXT: 5.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     13.0   0.0    0.0       st1	{ v1.h }[0], [x27], x28
-# CHECK-NEXT: 7.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     16.0   0.0    0.0       st1	{ v1.h }[4], [x27], x28
-# CHECK-NEXT: 9.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     10.7   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.h }[0], [x27], #2
+# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.h }[4], [x27], #2
+# CHECK-NEXT: 5.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     4.0    0.0    0.0       st1	{ v1.h }[0], [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       st1	{ v1.h }[4], [x27], x28
+# CHECK-NEXT: 9.     1     5.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.2    0.1    1.0       <total>
 
 # CHECK:      [66] Code Region - G67
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2004
+# CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        2200
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    1.10
-# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: uOps Per Cycle:    4.35
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 3.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0123
-
-# CHECK:      [0,0]     DeeeeER   .    .    .  .   st1	{ v1.s }[0], [x27], #4
-# CHECK-NEXT: [0,1]     D====eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D====eeeeER    .    .  .   st1	{ v1.s }[0], [x27], x28
-# CHECK-NEXT: [0,3]     D========eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D========eeeeER.    .  .   st1	{ v1.d }[0], [x27], #8
-# CHECK-NEXT: [0,5]     D============eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     D============eeeeER .  .   st1	{ v1.d }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .D===============eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===============eeeeER.   st2	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: [0,9]     .D===================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeER   .   st1	{ v1.s }[0], [x27], #4
+# CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeER  .   st1	{ v1.s }[0], [x27], x28
+# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeER .   st1	{ v1.d }[0], [x27], #8
+# CHECK-NEXT: [0,5]     D===eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     D===eeeeER.   st1	{ v1.d }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .D===eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeeeER   st2	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: [0,9]     .D====eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4229,43 +4218,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.s }[0], [x27], #4
-# CHECK-NEXT: 1.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       st1	{ v1.s }[0], [x27], x28
-# CHECK-NEXT: 3.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     9.0    0.0    0.0       st1	{ v1.d }[0], [x27], #8
-# CHECK-NEXT: 5.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     13.0   0.0    0.0       st1	{ v1.d }[0], [x27], x28
-# CHECK-NEXT: 7.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     16.0   0.0    0.0       st2	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: 9.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     10.7   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.s }[0], [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st1	{ v1.d }[0], [x27], #8
+# CHECK-NEXT: 5.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     4.0    0.0    0.0       st1	{ v1.d }[0], [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       st2	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: 9.     1     5.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.2    0.1    1.0       <total>
 
 # CHECK:      [67] Code Region - G68
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2004
+# CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        2400
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    1.20
-# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: uOps Per Cycle:    4.74
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 3.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0123
-
-# CHECK:      [0,0]     DeeeeER   .    .    .  .   st2	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: [0,1]     D====eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D====eeeeER    .    .  .   st2	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: [0,3]     D========eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D========eeeeER.    .  .   st2	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: [0,5]     D============eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D===========eeeeER .  .   st2	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: [0,7]     .D===============eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===============eeeeER.   st2	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: [0,9]     .D===================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeER   .   st2	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeER  .   st2	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeER .   st2	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: [0,5]     D===eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeER.   st2	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: [0,7]     .D===eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeeeER   st2	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: [0,9]     .D====eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4275,43 +4264,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: 1.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       st2	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: 3.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     9.0    0.0    0.0       st2	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: 5.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     12.0   0.0    0.0       st2	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: 7.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     16.0   0.0    0.0       st2	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: 9.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     10.6   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st2	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st2	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: 5.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st2	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: 7.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       st2	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: 9.     1     5.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.1    0.1    1.0       <total>
 
 # CHECK:      [68] Code Region - G69
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2004
+# CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        2600
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    1.30
-# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: uOps Per Cycle:    5.14
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0123
-
-# CHECK:      [0,0]     DeeeeER   .    .    .  .   st2	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: [0,1]     D====eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D====eeeeER    .    .  .   st2	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: [0,3]     D========eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D========eeeeER.    .  .   st2	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: [0,5]     .D===========eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D===========eeeeER .  .   st2	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: [0,7]     .D===============eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===============eeeeER.   st2	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: [0,9]     .D===================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeER   .   st2	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeER  .   st2	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeER .   st2	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,5]     .D==eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeER.   st2	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,7]     .D===eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeeeER   st2	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,9]     .D====eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4321,43 +4310,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: 1.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       st2	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: 3.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     9.0    0.0    0.0       st2	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: 5.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     12.0   0.0    0.0       st2	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: 7.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     16.0   0.0    0.0       st2	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: 9.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     10.5   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st2	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st2	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st2	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       st2	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: 9.     1     5.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.0    0.1    1.0       <total>
 
 # CHECK:      [69] Code Region - G70
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2004
+# CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        2400
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    1.20
-# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: uOps Per Cycle:    4.74
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 3.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0123
-
-# CHECK:      [0,0]     DeeeeER   .    .    .  .   st2	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: [0,1]     D====eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D====eeeeER    .    .  .   st2	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: [0,3]     D========eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D========eeeeER.    .  .   st2	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: [0,5]     .D===========eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D===========eeeeER .  .   st2	{ v1.b, v2.b }[0], [x27], #2
-# CHECK-NEXT: [0,7]     .D===============eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===============eeeeER.   st2	{ v1.b, v2.b }[8], [x27], #2
-# CHECK-NEXT: [0,9]     .D===================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeER   .   st2	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeER  .   st2	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeER .   st2	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,5]     .D==eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeER.   st2	{ v1.b, v2.b }[0], [x27], #2
+# CHECK-NEXT: [0,7]     .D===eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeeeER   st2	{ v1.b, v2.b }[8], [x27], #2
+# CHECK-NEXT: [0,9]     .D====eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4367,43 +4356,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: 1.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       st2	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: 3.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     9.0    0.0    0.0       st2	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: 5.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     12.0   0.0    0.0       st2	{ v1.b, v2.b }[0], [x27], #2
-# CHECK-NEXT: 7.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     16.0   0.0    0.0       st2	{ v1.b, v2.b }[8], [x27], #2
-# CHECK-NEXT: 9.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     10.5   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st2	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st2	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st2	{ v1.b, v2.b }[0], [x27], #2
+# CHECK-NEXT: 7.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       st2	{ v1.b, v2.b }[8], [x27], #2
+# CHECK-NEXT: 9.     1     5.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.0    0.1    1.0       <total>
 
 # CHECK:      [70] Code Region - G71
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2004
+# CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    1.00
-# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: uOps Per Cycle:    3.95
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0123
-
-# CHECK:      [0,0]     DeeeeER   .    .    .  .   st2	{ v1.b, v2.b }[0], [x27], x28
-# CHECK-NEXT: [0,1]     D====eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D====eeeeER    .    .  .   st2	{ v1.b, v2.b }[8], [x27], x28
-# CHECK-NEXT: [0,3]     D========eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D========eeeeER.    .  .   st2	{ v1.h, v2.h }[0], [x27], #4
-# CHECK-NEXT: [0,5]     D============eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     D============eeeeER .  .   st2	{ v1.h, v2.h }[4], [x27], #4
-# CHECK-NEXT: [0,7]     .D===============eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===============eeeeER.   st2	{ v1.h, v2.h }[0], [x27], x28
-# CHECK-NEXT: [0,9]     .D===================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeER   .   st2	{ v1.b, v2.b }[0], [x27], x28
+# CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeER  .   st2	{ v1.b, v2.b }[8], [x27], x28
+# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeER .   st2	{ v1.h, v2.h }[0], [x27], #4
+# CHECK-NEXT: [0,5]     D===eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     D===eeeeER.   st2	{ v1.h, v2.h }[4], [x27], #4
+# CHECK-NEXT: [0,7]     .D===eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeeeER   st2	{ v1.h, v2.h }[0], [x27], x28
+# CHECK-NEXT: [0,9]     .D====eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4413,43 +4402,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.b, v2.b }[0], [x27], x28
-# CHECK-NEXT: 1.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       st2	{ v1.b, v2.b }[8], [x27], x28
-# CHECK-NEXT: 3.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     9.0    0.0    0.0       st2	{ v1.h, v2.h }[0], [x27], #4
-# CHECK-NEXT: 5.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     13.0   0.0    0.0       st2	{ v1.h, v2.h }[4], [x27], #4
-# CHECK-NEXT: 7.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     16.0   0.0    0.0       st2	{ v1.h, v2.h }[0], [x27], x28
-# CHECK-NEXT: 9.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     10.7   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st2	{ v1.b, v2.b }[8], [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st2	{ v1.h, v2.h }[0], [x27], #4
+# CHECK-NEXT: 5.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     4.0    0.0    0.0       st2	{ v1.h, v2.h }[4], [x27], #4
+# CHECK-NEXT: 7.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       st2	{ v1.h, v2.h }[0], [x27], x28
+# CHECK-NEXT: 9.     1     5.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.2    0.1    1.0       <total>
 
 # CHECK:      [71] Code Region - G72
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2004
+# CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    1.00
-# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: uOps Per Cycle:    3.95
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0123
-
-# CHECK:      [0,0]     DeeeeER   .    .    .  .   st2	{ v1.h, v2.h }[4], [x27], x28
-# CHECK-NEXT: [0,1]     D====eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D====eeeeER    .    .  .   st2	{ v1.s, v2.s }[0], [x27], #8
-# CHECK-NEXT: [0,3]     D========eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D========eeeeER.    .  .   st2	{ v1.s, v2.s }[0], [x27], x28
-# CHECK-NEXT: [0,5]     D============eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     D============eeeeER .  .   st2	{ v1.d, v2.d }[0], [x27], #16
-# CHECK-NEXT: [0,7]     .D===============eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===============eeeeER.   st2	{ v1.d, v2.d }[0], [x27], x28
-# CHECK-NEXT: [0,9]     .D===================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeER   .   st2	{ v1.h, v2.h }[4], [x27], x28
+# CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeER  .   st2	{ v1.s, v2.s }[0], [x27], #8
+# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeeeER .   st2	{ v1.s, v2.s }[0], [x27], x28
+# CHECK-NEXT: [0,5]     D===eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     D===eeeeER.   st2	{ v1.d, v2.d }[0], [x27], #16
+# CHECK-NEXT: [0,7]     .D===eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeeeER   st2	{ v1.d, v2.d }[0], [x27], x28
+# CHECK-NEXT: [0,9]     .D====eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4459,39 +4448,38 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.h, v2.h }[4], [x27], x28
-# CHECK-NEXT: 1.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       st2	{ v1.s, v2.s }[0], [x27], #8
-# CHECK-NEXT: 3.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     9.0    0.0    0.0       st2	{ v1.s, v2.s }[0], [x27], x28
-# CHECK-NEXT: 5.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     13.0   0.0    0.0       st2	{ v1.d, v2.d }[0], [x27], #16
-# CHECK-NEXT: 7.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     16.0   0.0    0.0       st2	{ v1.d, v2.d }[0], [x27], x28
-# CHECK-NEXT: 9.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     10.7   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st2	{ v1.s, v2.s }[0], [x27], #8
+# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       st2	{ v1.s, v2.s }[0], [x27], x28
+# CHECK-NEXT: 5.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     4.0    0.0    0.0       st2	{ v1.d, v2.d }[0], [x27], #16
+# CHECK-NEXT: 7.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       st2	{ v1.d, v2.d }[0], [x27], x28
+# CHECK-NEXT: 9.     1     5.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.2    0.1    1.0       <total>
 
 # CHECK:      [72] Code Region - G73
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      600
-# CHECK-NEXT: Total Cycles:      1304
+# CHECK-NEXT: Total Cycles:      406
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    1.53
-# CHECK-NEXT: IPC:               0.46
+# CHECK-NEXT: uOps Per Cycle:    4.93
+# CHECK-NEXT: IPC:               1.48
 # CHECK-NEXT: Block RThroughput: 3.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeER  .    ..   st3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: [0,1]     D=====eER .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=====eeeeER   ..   st3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: [0,3]     D=========eER  ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D========eeeeER.   st3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: [0,5]     .D============eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeER .   st3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: [0,1]     D=eE---R .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeER .   st3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: [0,3]     D==eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D==eeeeER   st3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: [0,5]     .D===eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4501,39 +4489,39 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     6.0    0.0    0.0       st3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: 3.     1     10.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     9.0    0.0    0.0       st3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: 5.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     7.5    0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    1.0    0.0       st3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: 5.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.5    0.3    1.2       <total>
 
 # CHECK:      [73] Code Region - G74
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2404
+# CHECK-NEXT: Total Cycles:      707
 # CHECK-NEXT: Total uOps:        3800
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    1.58
-# CHECK-NEXT: IPC:               0.42
+# CHECK-NEXT: uOps Per Cycle:    5.37
+# CHECK-NEXT: IPC:               1.41
 # CHECK-NEXT: Block RThroughput: 7.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          01234567
-
-# CHECK:      [0,0]     DeeeeeER  .    .    .    . .   st3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: [0,1]     D=====eER .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=====eeeeER   .    .    . .   st3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: [0,3]     D=========eER  .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D========eeeeeER   .    . .   st3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: [0,5]     .D=============eER  .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=============eeeeeER   . .   st3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: [0,7]     . D=================eER  . .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D=================eeeeeER.   st3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: [0,9]     . D======================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0123
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeER  .  .   st3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: [0,1]     D=eE---R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeER  .  .   st3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: [0,3]     D==eE--R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D==eeeeeER  .   st3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: [0,5]     .D===eE---R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D===eeeeeER .   st3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: [0,7]     . D===eE---R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D====eeeeeER   st3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: [0,9]     . D=====eE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4543,43 +4531,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     6.0    0.0    0.0       st3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: 3.     1     10.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     9.0    0.0    0.0       st3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: 5.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     14.0   0.0    0.0       st3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: 7.     1     18.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     18.0   0.0    0.0       st3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: 9.     1     23.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     11.9   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    1.0    0.0       st3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: 5.     1     4.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     4.0    0.0    0.0       st3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: 7.     1     4.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     5.0    1.0    0.0       st3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: 9.     1     6.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.4    0.3    1.4       <total>
 
 # CHECK:      [74] Code Region - G75
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2204
+# CHECK-NEXT: Total Cycles:      706
 # CHECK-NEXT: Total uOps:        3400
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    1.54
-# CHECK-NEXT: IPC:               0.45
+# CHECK-NEXT: uOps Per Cycle:    4.82
+# CHECK-NEXT: IPC:               1.42
 # CHECK-NEXT: Block RThroughput: 6.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          012345
-
-# CHECK:      [0,0]     DeeeeER   .    .    .    .   st3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: [0,1]     D====eER  .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D====eeeeER    .    .    .   st3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: [0,3]     D========eER   .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=======eeeeeER    .    .   st3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: [0,5]     .D============eER   .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D============eeeeER.    .   st3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: [0,7]     .D================eER    .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D===============eeeeeER.   st3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: [0,9]     . D====================eER   add	x0, x27, #1
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeER   . .   st3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE--R   . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeER  . .   st3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     D==eE--R  . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeER. .   st3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: [0,5]     .D==eE---R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeER. .   st3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,7]     .D===eE--R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D===eeeeeER   st3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: [0,9]     . D====eE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4589,43 +4577,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: 1.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       st3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: 3.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     8.0    0.0    0.0       st3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: 5.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     13.0   0.0    0.0       st3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: 7.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     16.0   0.0    0.0       st3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: 9.     1     21.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     10.8   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    1.0    0.0       st3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: 9.     1     5.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.9    0.2    1.2       <total>
 
 # CHECK:      [75] Code Region - G76
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2104
+# CHECK-NEXT: Total Cycles:      606
 # CHECK-NEXT: Total uOps:        3200
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    1.52
-# CHECK-NEXT: IPC:               0.48
+# CHECK-NEXT: uOps Per Cycle:    5.28
+# CHECK-NEXT: IPC:               1.65
 # CHECK-NEXT: Block RThroughput: 5.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          01234
-
-# CHECK:      [0,0]     DeeeeeER  .    .    .   .   st3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: [0,1]     D=====eER .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=====eeeeER   .    .   .   st3	{ v1.b, v2.b, v3.b }[0], [x27], #3
-# CHECK-NEXT: [0,3]     D=========eER  .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D========eeeeER    .   .   st3	{ v1.b, v2.b, v3.b }[8], [x27], #3
-# CHECK-NEXT: [0,5]     .D============eER   .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D============eeeeER.   .   st3	{ v1.b, v2.b, v3.b }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .D================eER   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D===============eeeeER.   st3	{ v1.b, v2.b, v3.b }[8], [x27], x28
-# CHECK-NEXT: [0,9]     . D===================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeER  ..   st3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE---R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeER  ..   st3	{ v1.b, v2.b, v3.b }[0], [x27], #3
+# CHECK-NEXT: [0,3]     D==eE--R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D==eeeeER..   st3	{ v1.b, v2.b, v3.b }[8], [x27], #3
+# CHECK-NEXT: [0,5]     .D===eE--R..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D===eeeeER.   st3	{ v1.b, v2.b, v3.b }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .D====eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D===eeeeER   st3	{ v1.b, v2.b, v3.b }[8], [x27], x28
+# CHECK-NEXT: [0,9]     . D====eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4635,43 +4623,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: 1.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     6.0    0.0    0.0       st3	{ v1.b, v2.b, v3.b }[0], [x27], #3
-# CHECK-NEXT: 3.     1     10.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     9.0    0.0    0.0       st3	{ v1.b, v2.b, v3.b }[8], [x27], #3
-# CHECK-NEXT: 5.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     13.0   0.0    0.0       st3	{ v1.b, v2.b, v3.b }[0], [x27], x28
-# CHECK-NEXT: 7.     1     17.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     16.0   0.0    0.0       st3	{ v1.b, v2.b, v3.b }[8], [x27], x28
-# CHECK-NEXT: 9.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     11.1   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st3	{ v1.b, v2.b, v3.b }[0], [x27], #3
+# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    1.0    0.0       st3	{ v1.b, v2.b, v3.b }[8], [x27], #3
+# CHECK-NEXT: 5.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     4.0    0.0    0.0       st3	{ v1.b, v2.b, v3.b }[0], [x27], x28
+# CHECK-NEXT: 7.     1     5.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       st3	{ v1.b, v2.b, v3.b }[8], [x27], x28
+# CHECK-NEXT: 9.     1     5.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.3    0.2    1.1       <total>
 
 # CHECK:      [76] Code Region - G77
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2004
+# CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        3000
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    1.50
-# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: uOps Per Cycle:    5.93
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0123
-
-# CHECK:      [0,0]     DeeeeER   .    .    .  .   st3	{ v1.h, v2.h, v3.h }[0], [x27], #6
-# CHECK-NEXT: [0,1]     D====eER  .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D====eeeeER    .    .  .   st3	{ v1.h, v2.h, v3.h }[4], [x27], #6
-# CHECK-NEXT: [0,3]     D========eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=======eeeeER.    .  .   st3	{ v1.h, v2.h, v3.h }[0], [x27], x28
-# CHECK-NEXT: [0,5]     .D===========eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D===========eeeeER .  .   st3	{ v1.h, v2.h, v3.h }[4], [x27], x28
-# CHECK-NEXT: [0,7]     .D===============eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============eeeeER.   st3	{ v1.s, v2.s, v3.s }[0], [x27], #12
-# CHECK-NEXT: [0,9]     . D==================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeER   .   st3	{ v1.h, v2.h, v3.h }[0], [x27], #6
+# CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeER  .   st3	{ v1.h, v2.h, v3.h }[4], [x27], #6
+# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeER .   st3	{ v1.h, v2.h, v3.h }[0], [x27], x28
+# CHECK-NEXT: [0,5]     .D==eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeER.   st3	{ v1.h, v2.h, v3.h }[4], [x27], x28
+# CHECK-NEXT: [0,7]     .D===eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeER   st3	{ v1.s, v2.s, v3.s }[0], [x27], #12
+# CHECK-NEXT: [0,9]     . D===eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4681,43 +4669,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.h, v2.h, v3.h }[0], [x27], #6
-# CHECK-NEXT: 1.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       st3	{ v1.h, v2.h, v3.h }[4], [x27], #6
-# CHECK-NEXT: 3.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     8.0    0.0    0.0       st3	{ v1.h, v2.h, v3.h }[0], [x27], x28
-# CHECK-NEXT: 5.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     12.0   0.0    0.0       st3	{ v1.h, v2.h, v3.h }[4], [x27], x28
-# CHECK-NEXT: 7.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     15.0   0.0    0.0       st3	{ v1.s, v2.s, v3.s }[0], [x27], #12
-# CHECK-NEXT: 9.     1     19.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     10.2   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st3	{ v1.h, v2.h, v3.h }[4], [x27], #6
+# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st3	{ v1.h, v2.h, v3.h }[0], [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st3	{ v1.h, v2.h, v3.h }[4], [x27], x28
+# CHECK-NEXT: 7.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st3	{ v1.s, v2.s, v3.s }[0], [x27], #12
+# CHECK-NEXT: 9.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.7    0.1    1.0       <total>
 
 # CHECK:      [77] Code Region - G78
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2204
+# CHECK-NEXT: Total Cycles:      706
 # CHECK-NEXT: Total uOps:        3600
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    1.63
-# CHECK-NEXT: IPC:               0.45
+# CHECK-NEXT: uOps Per Cycle:    5.10
+# CHECK-NEXT: IPC:               1.42
 # CHECK-NEXT: Block RThroughput: 6.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          012345
-
-# CHECK:      [0,0]     DeeeeER   .    .    .    .   st3	{ v1.s, v2.s, v3.s }[0], [x27], x28
-# CHECK-NEXT: [0,1]     D====eER  .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D====eeeeER    .    .    .   st3	{ v1.d, v2.d, v3.d }[0], [x27], #24
-# CHECK-NEXT: [0,3]     D========eER   .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=======eeeeER.    .    .   st3	{ v1.d, v2.d, v3.d }[0], [x27], x28
-# CHECK-NEXT: [0,5]     .D===========eER    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D===========eeeeER .    .   st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: [0,7]     . D==============eER.    .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============eeeeeeER.   st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: [0,9]     . D====================eER   add	x0, x27, #1
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeER   . .   st3	{ v1.s, v2.s, v3.s }[0], [x27], x28
+# CHECK-NEXT: [0,1]     D=eE--R   . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeER  . .   st3	{ v1.d, v2.d, v3.d }[0], [x27], #24
+# CHECK-NEXT: [0,3]     D==eE--R  . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeER . .   st3	{ v1.d, v2.d, v3.d }[0], [x27], x28
+# CHECK-NEXT: [0,5]     .D==eE--R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeeeER. .   st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: [0,7]     . D==eE--R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeeER   st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: [0,9]     . D===eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4727,43 +4715,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.s, v2.s, v3.s }[0], [x27], x28
-# CHECK-NEXT: 1.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     5.0    0.0    0.0       st3	{ v1.d, v2.d, v3.d }[0], [x27], #24
-# CHECK-NEXT: 3.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     8.0    0.0    0.0       st3	{ v1.d, v2.d, v3.d }[0], [x27], x28
-# CHECK-NEXT: 5.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     12.0   0.0    0.0       st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: 7.     1     15.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     15.0   0.0    0.0       st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: 9.     1     21.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     10.3   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st3	{ v1.d, v2.d, v3.d }[0], [x27], #24
+# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st3	{ v1.d, v2.d, v3.d }[0], [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: 7.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.6    0.1    1.2       <total>
 
 # CHECK:      [78] Code Region - G79
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3304
+# CHECK-NEXT: Total Cycles:      1205
 # CHECK-NEXT: Total uOps:        5800
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    1.76
-# CHECK-NEXT: IPC:               0.30
+# CHECK-NEXT: uOps Per Cycle:    4.81
+# CHECK-NEXT: IPC:               0.83
 # CHECK-NEXT: Block RThroughput: 12.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123456
-# CHECK-NEXT: Index     0123456789          0123456789
-
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .    ..   st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=====eeeeeeeER    .    .    .    ..   st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: [0,3]     .D============eER   .    .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D===========eeeeeeER   .    .    ..   st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: [0,5]     . D=================eER  .    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D================eeeeeeeER .    ..   st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: [0,7]     .  D=======================eER.    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D======================eeeeeeeER.   st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: [0,9]     .   D=============================eER   add	x0, x27, #1
+# CHECK-NEXT:                     0123456
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeER .    ..   st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: [0,1]     D=eE----R .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeER    ..   st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: [0,3]     .D=eE-----R    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeeeeeER   ..   st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: [0,5]     . D==eE----R   ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D===eeeeeeeER.   st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: [0,7]     .  D====eE-----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D===eeeeeeeER   st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: [0,9]     .   D====eE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4773,43 +4761,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     6.0    0.0    0.0       st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: 3.     1     13.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     12.0   0.0    0.0       st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: 5.     1     18.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     17.0   0.0    0.0       st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: 7.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     23.0   0.0    0.0       st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: 9.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     15.1   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: 3.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     4.0    2.0    0.0       st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: 7.     1     5.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: 9.     1     5.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.9    0.4    2.3       <total>
 
 # CHECK:      [79] Code Region - G80
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2904
+# CHECK-NEXT: Total Cycles:      1006
 # CHECK-NEXT: Total uOps:        4800
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    1.65
-# CHECK-NEXT: IPC:               0.34
+# CHECK-NEXT: uOps Per Cycle:    4.77
+# CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 9.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          012
-# CHECK-NEXT: Index     0123456789          0123456789
-
-# CHECK:      [0,0]     DeeeeER   .    .    .    .    . .   st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: [0,1]     D====eER  .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D===eeeeeeER  .    .    .    . .   st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=========eER .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=========eeeeeeER .    .    . .   st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: [0,5]     . D==============eER.    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     . D==============eeeeeeeER    . .   st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: [0,7]     . D=====================eER   . .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .  D====================eeeeeeER.   st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: [0,9]     .  D==========================eER   add	x0, x27, #1
+# CHECK-NEXT:                     012345
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeER   .    .   st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE--R   .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeER.    .   st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE----R.    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D===eeeeeeER  .   st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,5]     . D===eE----R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     . D===eeeeeeeER.   st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,7]     . D====eE-----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .  D====eeeeeeER   st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: [0,9]     .  D=====eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4819,43 +4807,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: 1.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     4.0    0.0    0.0       st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: 3.     1     10.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     10.0   0.0    0.0       st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: 5.     1     15.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     15.0   0.0    0.0       st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: 7.     1     22.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     21.0   0.0    0.0       st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: 9.     1     27.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     13.0   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     4.0    2.0    0.0       st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: 5.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     4.0    0.0    0.0       st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: 7.     1     5.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     5.0    1.0    0.0       st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: 9.     1     6.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.4    0.4    1.9       <total>
 
 # CHECK:      [80] Code Region - G81
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3204
+# CHECK-NEXT: Total Cycles:      807
 # CHECK-NEXT: Total uOps:        5200
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    1.62
-# CHECK-NEXT: IPC:               0.31
+# CHECK-NEXT: uOps Per Cycle:    6.44
+# CHECK-NEXT: IPC:               1.24
 # CHECK-NEXT: Block RThroughput: 6.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          012345
-# CHECK-NEXT: Index     0123456789          0123456789
-
-# CHECK:      [0,0]     DeeeeeeeER.    .    .    .    .    .   st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: [0,1]     D=======eER    .    .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D======eeeeeeeER   .    .    .    .   st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=============eER  .    .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D============eeeeeeER  .    .    .   st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
-# CHECK-NEXT: [0,5]     . D==================eER .    .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     . D==================eeeeeeER .    .   st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
-# CHECK-NEXT: [0,7]     .  D=======================eER.    .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .  D=======================eeeeeeER.   st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
-# CHECK-NEXT: [0,9]     .  D=============================eER   add	x0, x27, #1
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeeER.   .   st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE-----R.   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeER   .   st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE-----R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeER   .   st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
+# CHECK-NEXT: [0,5]     . D=eE----R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     . D===eeeeeeER.   st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
+# CHECK-NEXT: [0,7]     .  D===eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .  D===eeeeeeER   st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
+# CHECK-NEXT: [0,9]     .  D====eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4865,43 +4853,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: 1.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: 3.     1     14.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     13.0   0.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
-# CHECK-NEXT: 5.     1     19.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     19.0   0.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
-# CHECK-NEXT: 7.     1     24.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     24.0   0.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
-# CHECK-NEXT: 9.     1     30.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     15.9   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
+# CHECK-NEXT: 5.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     4.0    2.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
+# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
+# CHECK-NEXT: 9.     1     5.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.6    0.3    2.2       <total>
 
 # CHECK:      [81] Code Region - G82
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      3004
+# CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        4000
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    1.33
-# CHECK-NEXT: IPC:               0.33
+# CHECK-NEXT: uOps Per Cycle:    7.87
+# CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789          0123
-# CHECK-NEXT: Index     0123456789          0123456789
-
-# CHECK:      [0,0]     DeeeeeeER .    .    .    .    .  .   st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
-# CHECK-NEXT: [0,1]     D======eER.    .    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D======eeeeeeER.    .    .    .  .   st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
-# CHECK-NEXT: [0,3]     .D===========eER    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===========eeeeeeER    .    .  .   st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
-# CHECK-NEXT: [0,5]     .D=================eER   .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     . D================eeeeeeER   .  .   st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
-# CHECK-NEXT: [0,7]     . D======================eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D======================eeeeeeER.   st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
-# CHECK-NEXT: [0,9]     .  D===========================eER   add	x0, x27, #1
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeeeeeeER . .   st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
+# CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeER. .   st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
+# CHECK-NEXT: [0,3]     .D=eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeeER .   st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
+# CHECK-NEXT: [0,5]     .D==eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     . D=eeeeeeER.   st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
+# CHECK-NEXT: [0,7]     . D==eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeeER   st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
+# CHECK-NEXT: [0,9]     .  D==eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4911,41 +4899,41 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
-# CHECK-NEXT: 3.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     12.0   0.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
-# CHECK-NEXT: 5.     1     18.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     17.0   0.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
-# CHECK-NEXT: 7.     1     23.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     23.0   0.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
-# CHECK-NEXT: 9.     1     28.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     14.8   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
+# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
+# CHECK-NEXT: 7.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
+# CHECK-NEXT: 9.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.1    2.0       <total>
 
 # CHECK:      [82] Code Region - G83
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      800
-# CHECK-NEXT: Total Cycles:      2004
+# CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        2800
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    1.40
-# CHECK-NEXT: IPC:               0.40
+# CHECK-NEXT: uOps Per Cycle:    5.53
+# CHECK-NEXT: IPC:               1.58
 # CHECK-NEXT: Block RThroughput: 2.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0123
+# CHECK-NEXT:                     0
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeER .    .    .  .   st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
-# CHECK-NEXT: [0,1]     D======eER.    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D======eeeeeeER.    .  .   st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
-# CHECK-NEXT: [0,3]     .D===========eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===========eeeeER .  .   st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
-# CHECK-NEXT: [0,5]     .D===============eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D===============eeeeER.   st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .D===================eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeER .   st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
+# CHECK-NEXT: [0,1]     D=eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeeeeeER.   st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D==eeeeER.   st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
+# CHECK-NEXT: [0,5]     .D===eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D===eeeeER   st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .D====eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4955,34 +4943,34 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
-# CHECK-NEXT: 1.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     7.0    0.0    0.0       st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
-# CHECK-NEXT: 3.     1     12.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     12.0   0.0    0.0       st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
-# CHECK-NEXT: 5.     1     16.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     16.0   0.0    0.0       st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
-# CHECK-NEXT: 7.     1     20.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     11.4   0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    1.0    0.0       st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
+# CHECK-NEXT: 5.     1     4.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     4.0    0.0    0.0       st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
+# CHECK-NEXT: 7.     1     5.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.9    0.3    1.5       <total>
 
 # CHECK:      [83] Code Region - G84
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      400
-# CHECK-NEXT: Total Cycles:      404
+# CHECK-NEXT: Total Cycles:      204
 # CHECK-NEXT: Total uOps:        800
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    1.98
-# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: uOps Per Cycle:    3.92
+# CHECK-NEXT: IPC:               1.96
 # CHECK-NEXT: Block RThroughput: 1.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT: Index     01234567
+# CHECK-NEXT: Index     012345
 
-# CHECK:      [0,0]     DeeER. .   stp	s1, s2, [x27], #248
-# CHECK-NEXT: [0,1]     D==eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D==eeER.   stp	d1, d2, [x27], #496
-# CHECK-NEXT: [0,3]     D====eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.   stp	s1, s2, [x27], #248
+# CHECK-NEXT: [0,1]     D=eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeER   stp	d1, d2, [x27], #496
+# CHECK-NEXT: [0,3]     D==eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4992,37 +4980,36 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       stp	s1, s2, [x27], #248
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       stp	d1, d2, [x27], #496
-# CHECK-NEXT: 3.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.0    0.3    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       stp	d1, d2, [x27], #496
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.0    0.3    0.0       <total>
 
 # CHECK:      [84] Code Region - G85
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      904
+# CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2200
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    2.43
-# CHECK-NEXT: IPC:               1.11
+# CHECK-NEXT: uOps Per Cycle:    4.37
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 3.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     012
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     012345678
 
-# CHECK:      [0,0]     DeeER.    . .   stp	q1, q2, [x27], #992
-# CHECK-NEXT: [0,1]     D==eER    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D==eeER   . .   stp	s1, s2, [x27, #248]!
-# CHECK-NEXT: [0,3]     D====eER  . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D====eeER . .   stp	d1, d2, [x27, #496]!
-# CHECK-NEXT: [0,5]     D======eER. .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=====eeER .   stp	q1, q2, [x27, #992]!
-# CHECK-NEXT: [0,7]     .D=======eER.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D=======eER.   stp	w1, w2, [x27], #248
-# CHECK-NEXT: [0,9]     .D========eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.  .   stp	q1, q2, [x27], #992
+# CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeER  .   stp	s1, s2, [x27, #248]!
+# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeER .   stp	d1, d2, [x27, #496]!
+# CHECK-NEXT: [0,5]     D===eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .D==eeER.   stp	q1, q2, [x27, #992]!
+# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eER.   stp	w1, w2, [x27], #248
+# CHECK-NEXT: [0,9]     .D====eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -5032,43 +5019,42 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       stp	q1, q2, [x27], #992
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       stp	s1, s2, [x27, #248]!
-# CHECK-NEXT: 3.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       stp	d1, d2, [x27, #496]!
-# CHECK-NEXT: 5.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     6.0    0.0    0.0       stp	q1, q2, [x27, #992]!
-# CHECK-NEXT: 7.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     8.0    0.0    0.0       stp	w1, w2, [x27], #248
-# CHECK-NEXT: 9.     1     9.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     5.5    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       stp	s1, s2, [x27, #248]!
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       stp	d1, d2, [x27, #496]!
+# CHECK-NEXT: 5.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     3.0    0.0    0.0       stp	q1, q2, [x27, #992]!
+# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       stp	w1, w2, [x27], #248
+# CHECK-NEXT: 9.     1     5.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.1    0.1    0.0       <total>
 
 # CHECK:      [85] Code Region - G86
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      704
+# CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    2.84
-# CHECK-NEXT: IPC:               1.42
+# CHECK-NEXT: uOps Per Cycle:    3.97
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     012345678
 
-# CHECK:      [0,0]     DeER .    .   stp	x1, x2, [x27], #496
-# CHECK-NEXT: [0,1]     D=eER.    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eER.    .   stp	w1, w2, [x27, #248]!
-# CHECK-NEXT: [0,3]     D==eER    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D==eER    .   stp	x1, x2, [x27, #496]!
-# CHECK-NEXT: [0,5]     D===eER   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     D===eeER  .   str	b1, [x27], #254
-# CHECK-NEXT: [0,7]     .D====eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D====eeER.   str	h1, [x27], #254
-# CHECK-NEXT: [0,9]     .D======eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeER .  .   stp	x1, x2, [x27], #496
+# CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eER.  .   stp	w1, w2, [x27, #248]!
+# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eER  .   stp	x1, x2, [x27, #496]!
+# CHECK-NEXT: [0,5]     D===eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     D===eeER.   str	b1, [x27], #254
+# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeER   str	h1, [x27], #254
+# CHECK-NEXT: [0,9]     .D====eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -5084,37 +5070,36 @@ add x0, x27, 1
 # CHECK-NEXT: 4.     1     3.0    0.0    0.0       stp	x1, x2, [x27, #496]!
 # CHECK-NEXT: 5.     1     4.0    0.0    0.0       add	x0, x27, #1
 # CHECK-NEXT: 6.     1     4.0    0.0    0.0       str	b1, [x27], #254
-# CHECK-NEXT: 7.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     5.0    0.0    0.0       str	h1, [x27], #254
-# CHECK-NEXT: 9.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.6    0.1    0.0       <total>
+# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       str	h1, [x27], #254
+# CHECK-NEXT: 9.     1     5.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.2    0.1    0.0       <total>
 
 # CHECK:      [86] Code Region - G87
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1004
+# CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    1.99
-# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: uOps Per Cycle:    3.97
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     012345678
 
-# CHECK:      [0,0]     DeeER.    .  .   str	s1, [x27], #254
-# CHECK-NEXT: [0,1]     D==eER    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D==eeER   .  .   str	d1, [x27], #254
-# CHECK-NEXT: [0,3]     D====eER  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D====eeER .  .   str	q1, [x27], #254
-# CHECK-NEXT: [0,5]     D======eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     D======eeER  .   str	b1, [x27, #254]!
-# CHECK-NEXT: [0,7]     .D=======eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D=======eeER.   str	h1, [x27, #254]!
-# CHECK-NEXT: [0,9]     .D=========eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.  .   str	s1, [x27], #254
+# CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeER  .   str	d1, [x27], #254
+# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeER .   str	q1, [x27], #254
+# CHECK-NEXT: [0,5]     D===eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     D===eeER.   str	b1, [x27, #254]!
+# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eeER   str	h1, [x27, #254]!
+# CHECK-NEXT: [0,9]     .D====eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -5124,43 +5109,42 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       str	s1, [x27], #254
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       str	d1, [x27], #254
-# CHECK-NEXT: 3.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       str	q1, [x27], #254
-# CHECK-NEXT: 5.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     7.0    0.0    0.0       str	b1, [x27, #254]!
-# CHECK-NEXT: 7.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     8.0    0.0    0.0       str	h1, [x27, #254]!
-# CHECK-NEXT: 9.     1     10.0   0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     5.7    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       str	d1, [x27], #254
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       str	q1, [x27], #254
+# CHECK-NEXT: 5.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     4.0    0.0    0.0       str	b1, [x27, #254]!
+# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       str	h1, [x27, #254]!
+# CHECK-NEXT: 9.     1     5.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.2    0.1    0.0       <total>
 
 # CHECK:      [87] Code Region - G88
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      804
+# CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2000
 
 # CHECK:      Dispatch Width:    15
-# CHECK-NEXT: uOps Per Cycle:    2.49
-# CHECK-NEXT: IPC:               1.24
+# CHECK-NEXT: uOps Per Cycle:    3.97
+# CHECK-NEXT: IPC:               1.98
 # CHECK-NEXT: Block RThroughput: 2.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     012345678
 
-# CHECK:      [0,0]     DeeER.    ..   str	s1, [x27, #254]!
-# CHECK-NEXT: [0,1]     D==eER    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D==eeER   ..   str	d1, [x27, #254]!
-# CHECK-NEXT: [0,3]     D====eER  ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D====eeER ..   str	q1, [x27, #254]!
-# CHECK-NEXT: [0,5]     D======eER..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     D======eER..   str	w1, [x27], #254
-# CHECK-NEXT: [0,7]     .D======eER.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D======eER.   str	x1, [x27], #254
-# CHECK-NEXT: [0,9]     .D=======eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.  .   str	s1, [x27, #254]!
+# CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     D=eeER  .   str	d1, [x27, #254]!
+# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     D==eeER .   str	q1, [x27, #254]!
+# CHECK-NEXT: [0,5]     D===eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     D===eER .   str	w1, [x27], #254
+# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .D===eER.   str	x1, [x27], #254
+# CHECK-NEXT: [0,9]     .D====eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -5170,16 +5154,16 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       str	s1, [x27, #254]!
-# CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       str	d1, [x27, #254]!
-# CHECK-NEXT: 3.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     5.0    0.0    0.0       str	q1, [x27, #254]!
-# CHECK-NEXT: 5.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     7.0    0.0    0.0       str	w1, [x27], #254
-# CHECK-NEXT: 7.     1     7.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     7.0    0.0    0.0       str	x1, [x27], #254
-# CHECK-NEXT: 9.     1     8.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     5.3    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       str	d1, [x27, #254]!
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     3.0    0.0    0.0       str	q1, [x27, #254]!
+# CHECK-NEXT: 5.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     4.0    0.0    0.0       str	w1, [x27], #254
+# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     4.0    0.0    0.0       str	x1, [x27], #254
+# CHECK-NEXT: 9.     1     5.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     3.2    0.1    0.0       <total>
 
 # CHECK:      [88] Code Region - G89
 
diff --git a/llvm/test/tools/llvm-mca/RISCV/vle-vse.s b/llvm/test/tools/llvm-mca/RISCV/vle-vse.s
new file mode 100644
index 0000000000..15b8f854c5
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/RISCV/vle-vse.s
@@ -0,0 +1,1249 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=riscv64 -mcpu=sifive-x280 -iterations=1 < %s | FileCheck %s
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vle8.v    v1, (a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vle8.v    v1, (a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vle8.v    v1, (a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vle8.v    v1, (a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vle8.v    v1, (a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vle8.v    v1, (a0)
+vsetvli zero, zero, e8, m8, tu, mu
+vle8.v    v1, (a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vle8.v    v1, (a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vle8.v    v1, (a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vle8.v    v1, (a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vle8.v    v1, (a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vle8.v    v1, (a0)
+vsetvli zero, zero, e16, m8, tu, mu
+vle8.v    v1, (a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vle8.v    v1, (a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vle8.v    v1, (a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vle8.v    v1, (a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vle8.v    v1, (a0)
+vsetvli zero, zero, e32, m8, tu, mu
+vle8.v    v1, (a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vle8.v    v1, (a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vle8.v    v1, (a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vle8.v    v1, (a0)
+vsetvli zero, zero, e64, m8, tu, mu
+vle8.v    v1, (a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vle16.v    v1, (a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vle16.v    v1, (a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vle16.v    v1, (a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vle16.v    v1, (a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vle16.v    v1, (a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vle16.v    v1, (a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vle16.v    v1, (a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vle16.v    v1, (a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vle16.v    v1, (a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vle16.v    v1, (a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vle16.v    v1, (a0)
+vsetvli zero, zero, e16, m8, tu, mu
+vle16.v    v1, (a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vle16.v    v1, (a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vle16.v    v1, (a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vle16.v    v1, (a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vle16.v    v1, (a0)
+vsetvli zero, zero, e32, m8, tu, mu
+vle16.v    v1, (a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vle16.v    v1, (a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vle16.v    v1, (a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vle16.v    v1, (a0)
+vsetvli zero, zero, e64, m8, tu, mu
+vle16.v    v1, (a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vle32.v    v1, (a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vle32.v    v1, (a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vle32.v    v1, (a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vle32.v    v1, (a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vle32.v    v1, (a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vle32.v    v1, (a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vle32.v    v1, (a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vle32.v    v1, (a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vle32.v    v1, (a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vle32.v    v1, (a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vle32.v    v1, (a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vle32.v    v1, (a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vle32.v    v1, (a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vle32.v    v1, (a0)
+vsetvli zero, zero, e32, m8, tu, mu
+vle32.v    v1, (a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vle32.v    v1, (a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vle32.v    v1, (a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vle32.v    v1, (a0)
+vsetvli zero, zero, e64, m8, tu, mu
+vle32.v    v1, (a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vle64.v    v1, (a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vle64.v    v1, (a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vle64.v    v1, (a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vle64.v    v1, (a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vle64.v    v1, (a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vle64.v    v1, (a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vle64.v    v1, (a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vle64.v    v1, (a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vle64.v    v1, (a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vle64.v    v1, (a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vle64.v    v1, (a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vle64.v    v1, (a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vle64.v    v1, (a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vle64.v    v1, (a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vle64.v    v1, (a0)
+vsetvli zero, zero, e64, m8, tu, mu
+vle64.v    v1, (a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vse8.v    v1, (a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vse8.v    v1, (a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vse8.v    v1, (a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vse8.v    v1, (a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vse8.v    v1, (a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vse8.v    v1, (a0)
+vsetvli zero, zero, e8, m8, tu, mu
+vse8.v    v1, (a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vse8.v    v1, (a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vse8.v    v1, (a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vse8.v    v1, (a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vse8.v    v1, (a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vse8.v    v1, (a0)
+vsetvli zero, zero, e16, m8, tu, mu
+vse8.v    v1, (a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vse8.v    v1, (a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vse8.v    v1, (a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vse8.v    v1, (a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vse8.v    v1, (a0)
+vsetvli zero, zero, e32, m8, tu, mu
+vse8.v    v1, (a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vse8.v    v1, (a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vse8.v    v1, (a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vse8.v    v1, (a0)
+vsetvli zero, zero, e64, m8, tu, mu
+vse8.v    v1, (a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vse16.v    v1, (a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vse16.v    v1, (a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vse16.v    v1, (a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vse16.v    v1, (a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vse16.v    v1, (a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vse16.v    v1, (a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vse16.v    v1, (a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vse16.v    v1, (a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vse16.v    v1, (a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vse16.v    v1, (a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vse16.v    v1, (a0)
+vsetvli zero, zero, e16, m8, tu, mu
+vse16.v    v1, (a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vse16.v    v1, (a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vse16.v    v1, (a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vse16.v    v1, (a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vse16.v    v1, (a0)
+vsetvli zero, zero, e32, m8, tu, mu
+vse16.v    v1, (a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vse16.v    v1, (a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vse16.v    v1, (a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vse16.v    v1, (a0)
+vsetvli zero, zero, e64, m8, tu, mu
+vse16.v    v1, (a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vse32.v    v1, (a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vse32.v    v1, (a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vse32.v    v1, (a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vse32.v    v1, (a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vse32.v    v1, (a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vse32.v    v1, (a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vse32.v    v1, (a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vse32.v    v1, (a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vse32.v    v1, (a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vse32.v    v1, (a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vse32.v    v1, (a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vse32.v    v1, (a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vse32.v    v1, (a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vse32.v    v1, (a0)
+vsetvli zero, zero, e32, m8, tu, mu
+vse32.v    v1, (a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vse32.v    v1, (a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vse32.v    v1, (a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vse32.v    v1, (a0)
+vsetvli zero, zero, e64, m8, tu, mu
+vse32.v    v1, (a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vse64.v    v1, (a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vse64.v    v1, (a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vse64.v    v1, (a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vse64.v    v1, (a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vse64.v    v1, (a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vse64.v    v1, (a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vse64.v    v1, (a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vse64.v    v1, (a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vse64.v    v1, (a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vse64.v    v1, (a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vse64.v    v1, (a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vse64.v    v1, (a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vse64.v    v1, (a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vse64.v    v1, (a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vse64.v    v1, (a0)
+vsetvli zero, zero, e64, m8, tu, mu
+vse64.v    v1, (a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlm.v    v1, (a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vlm.v    v1, (a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vlm.v    v1, (a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vlm.v    v1, (a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vlm.v    v1, (a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vlm.v    v1, (a0)
+vsetvli zero, zero, e8, m8, tu, mu
+vlm.v    v1, (a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vlm.v    v1, (a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vlm.v    v1, (a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vlm.v    v1, (a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vlm.v    v1, (a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vlm.v    v1, (a0)
+vsetvli zero, zero, e16, m8, tu, mu
+vlm.v    v1, (a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vlm.v    v1, (a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vlm.v    v1, (a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vlm.v    v1, (a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vlm.v    v1, (a0)
+vsetvli zero, zero, e32, m8, tu, mu
+vlm.v    v1, (a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vlm.v    v1, (a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vlm.v    v1, (a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vlm.v    v1, (a0)
+vsetvli zero, zero, e64, m8, tu, mu
+vlm.v    v1, (a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsm.v    v1, (a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vsm.v    v1, (a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vsm.v    v1, (a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vsm.v    v1, (a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vsm.v    v1, (a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vsm.v    v1, (a0)
+vsetvli zero, zero, e8, m8, tu, mu
+vsm.v    v1, (a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vsm.v    v1, (a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vsm.v    v1, (a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vsm.v    v1, (a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vsm.v    v1, (a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vsm.v    v1, (a0)
+vsetvli zero, zero, e16, m8, tu, mu
+vsm.v    v1, (a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vsm.v    v1, (a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vsm.v    v1, (a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vsm.v    v1, (a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vsm.v    v1, (a0)
+vsetvli zero, zero, e32, m8, tu, mu
+vsm.v    v1, (a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vsm.v    v1, (a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vsm.v    v1, (a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vsm.v    v1, (a0)
+vsetvli zero, zero, e64, m8, tu, mu
+vsm.v    v1, (a0)
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      400
+# CHECK-NEXT: Total Cycles:      1084
+# CHECK-NEXT: Total uOps:        400
+
+# CHECK:      Dispatch Width:    2
+# CHECK-NEXT: uOps Per Cycle:    0.37
+# CHECK-NEXT: IPC:               0.37
+# CHECK-NEXT: Block RThroughput: 848.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      4     1.00    *                   vle8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      4     1.00    *                   vle8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      4     1.00    *                   vle8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vle8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      4     4.00    *                   vle8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      4     8.00    *                   vle8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      4     16.00   *                   vle8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      4     1.00    *                   vle8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      4     1.00    *                   vle8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      4     1.00    *                   vle8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vle8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      4     4.00    *                   vle8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      4     8.00    *                   vle8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      4     1.00    *                   vle8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      4     1.00    *                   vle8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      4     1.00    *                   vle8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vle8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      4     4.00    *                   vle8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      4     1.00    *                   vle8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      4     1.00    *                   vle8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      4     1.00    *                   vle8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vle8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      4     1.00    *                   vle16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      4     1.00    *                   vle16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vle16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      4     4.00    *                   vle16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      4     8.00    *                   vle16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      4     16.00   *                   vle16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      4     1.00    *                   vle16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      4     1.00    *                   vle16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vle16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      4     4.00    *                   vle16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      4     8.00    *                   vle16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      4     16.00   *                   vle16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      4     1.00    *                   vle16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      4     1.00    *                   vle16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vle16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      4     4.00    *                   vle16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      4     8.00    *                   vle16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      4     1.00    *                   vle16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      4     1.00    *                   vle16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vle16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      4     4.00    *                   vle16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      4     1.00    *                   vle32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vle32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      4     4.00    *                   vle32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      4     8.00    *                   vle32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      4     16.00   *                   vle32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      4     1.00    *                   vle32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vle32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      4     4.00    *                   vle32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      4     8.00    *                   vle32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      4     16.00   *                   vle32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      4     1.00    *                   vle32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vle32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      4     4.00    *                   vle32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      4     8.00    *                   vle32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      4     16.00   *                   vle32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      4     1.00    *                   vle32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vle32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      4     4.00    *                   vle32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      4     8.00    *                   vle32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vle64.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      4     4.00    *                   vle64.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      4     8.00    *                   vle64.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      4     16.00   *                   vle64.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vle64.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      4     4.00    *                   vle64.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      4     8.00    *                   vle64.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      4     16.00   *                   vle64.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vle64.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      4     4.00    *                   vle64.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      4     8.00    *                   vle64.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      4     16.00   *                   vle64.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vle64.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      4     4.00    *                   vle64.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      4     8.00    *                   vle64.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      4     16.00   *                   vle64.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      1     1.00           *            vse8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      1     1.00           *            vse8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      1     1.00           *            vse8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vse8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      1     4.00           *            vse8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      1     8.00           *            vse8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      1     16.00          *            vse8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      1     1.00           *            vse8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      1     1.00           *            vse8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      1     1.00           *            vse8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vse8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      1     4.00           *            vse8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      1     8.00           *            vse8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      1     1.00           *            vse8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      1     1.00           *            vse8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      1     1.00           *            vse8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vse8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      1     4.00           *            vse8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      1     1.00           *            vse8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      1     1.00           *            vse8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      1     1.00           *            vse8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vse8.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      1     1.00           *            vse16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      1     1.00           *            vse16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vse16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      1     4.00           *            vse16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      1     8.00           *            vse16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      1     16.00          *            vse16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      1     1.00           *            vse16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      1     1.00           *            vse16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vse16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      1     4.00           *            vse16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      1     8.00           *            vse16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      1     16.00          *            vse16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      1     1.00           *            vse16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      1     1.00           *            vse16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vse16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      1     4.00           *            vse16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      1     8.00           *            vse16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      1     1.00           *            vse16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      1     1.00           *            vse16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vse16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      1     4.00           *            vse16.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      1     1.00           *            vse32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vse32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      1     4.00           *            vse32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      1     8.00           *            vse32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      1     16.00          *            vse32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      1     1.00           *            vse32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vse32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      1     4.00           *            vse32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      1     8.00           *            vse32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      1     16.00          *            vse32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      1     1.00           *            vse32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vse32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      1     4.00           *            vse32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      1     8.00           *            vse32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      1     16.00          *            vse32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      1     1.00           *            vse32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vse32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      1     4.00           *            vse32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      1     8.00           *            vse32.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vse64.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      1     4.00           *            vse64.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      1     8.00           *            vse64.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      1     16.00          *            vse64.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vse64.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      1     4.00           *            vse64.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      1     8.00           *            vse64.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      1     16.00          *            vse64.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vse64.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      1     4.00           *            vse64.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      1     8.00           *            vse64.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      1     16.00          *            vse64.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vse64.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      1     4.00           *            vse64.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      1     8.00           *            vse64.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      1     16.00          *            vse64.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vlm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vlm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vlm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vlm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vlm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vlm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vlm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vlm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vlm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vlm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vlm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vlm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vlm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vlm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vlm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vlm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vlm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vlm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vlm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vlm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vlm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      4     2.00    *                   vlm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vsm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vsm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vsm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vsm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vsm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vsm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vsm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vsm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vsm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vsm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vsm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vsm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vsm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vsm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vsm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vsm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vsm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vsm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vsm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vsm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vsm.v	v1, (a0)
+# CHECK-NEXT:  1      3     1.00                  U     vsetvli	zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      1     2.00           *            vsm.v	v1, (a0)
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SiFive7FDiv
+# CHECK-NEXT: [1]   - SiFive7IDiv
+# CHECK-NEXT: [2]   - SiFive7PipeA
+# CHECK-NEXT: [3]   - SiFive7PipeB
+# CHECK-NEXT: [4]   - SiFive7PipeV
+# CHECK-NEXT: [5]   - SiFive7VA
+# CHECK-NEXT: [6]   - SiFive7VL
+# CHECK-NEXT: [7]   - SiFive7VS
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]
+# CHECK-NEXT:  -      -     200.00  -     848.00  -     424.00 424.00
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    Instructions:
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -     1.00    -     vle8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -     1.00    -     vle8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -     1.00    -     vle8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vle8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     4.00    -     4.00    -     vle8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -     8.00    -     8.00    -     vle8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -     16.00   -     16.00   -     vle8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -     1.00    -     vle8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -     1.00    -     vle8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -     1.00    -     vle8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vle8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -     4.00    -     4.00    -     vle8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -     8.00    -     8.00    -     vle8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -     1.00    -     vle8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -     1.00    -     vle8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -     1.00    -     vle8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vle8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -     4.00    -     4.00    -     vle8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -     1.00    -     vle8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -     1.00    -     vle8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -     1.00    -     vle8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vle8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -     1.00    -     vle16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -     1.00    -     vle16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vle16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     4.00    -     4.00    -     vle16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     8.00    -     8.00    -     vle16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -     16.00   -     16.00   -     vle16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -     1.00    -     vle16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -     1.00    -     vle16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vle16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     4.00    -     4.00    -     vle16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -     8.00    -     8.00    -     vle16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -     16.00   -     16.00   -     vle16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -     1.00    -     vle16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -     1.00    -     vle16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vle16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -     4.00    -     4.00    -     vle16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -     8.00    -     8.00    -     vle16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -     1.00    -     vle16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -     1.00    -     vle16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vle16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -     4.00    -     4.00    -     vle16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -     1.00    -     vle32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vle32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -     4.00    -     4.00    -     vle32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     8.00    -     8.00    -     vle32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     16.00   -     16.00   -     vle32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -     1.00    -     vle32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vle32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     4.00    -     4.00    -     vle32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     8.00    -     8.00    -     vle32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -     16.00   -     16.00   -     vle32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -     1.00    -     vle32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vle32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     4.00    -     4.00    -     vle32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -     8.00    -     8.00    -     vle32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -     16.00   -     16.00   -     vle32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -     1.00    -     vle32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vle32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -     4.00    -     4.00    -     vle32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -     8.00    -     8.00    -     vle32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vle64.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -     4.00    -     4.00    -     vle64.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -     8.00    -     8.00    -     vle64.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     16.00   -     16.00   -     vle64.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vle64.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -     4.00    -     4.00    -     vle64.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     8.00    -     8.00    -     vle64.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     16.00   -     16.00   -     vle64.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vle64.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     4.00    -     4.00    -     vle64.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     8.00    -     8.00    -     vle64.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -     16.00   -     16.00   -     vle64.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vle64.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     4.00    -     4.00    -     vle64.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -     8.00    -     8.00    -     vle64.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -     16.00   -     16.00   -     vle64.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   vse8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   vse8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   vse8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vse8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     4.00    -      -     4.00   vse8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -     8.00    -      -     8.00   vse8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -     16.00   -      -     16.00  vse8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   vse8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   vse8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   vse8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vse8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -     4.00    -      -     4.00   vse8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -     8.00    -      -     8.00   vse8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   vse8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   vse8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   vse8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vse8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -     4.00    -      -     4.00   vse8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   vse8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   vse8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   vse8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vse8.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   vse16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   vse16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vse16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     4.00    -      -     4.00   vse16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     8.00    -      -     8.00   vse16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -     16.00   -      -     16.00  vse16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   vse16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   vse16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vse16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     4.00    -      -     4.00   vse16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -     8.00    -      -     8.00   vse16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -     16.00   -      -     16.00  vse16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   vse16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   vse16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vse16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -     4.00    -      -     4.00   vse16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -     8.00    -      -     8.00   vse16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   vse16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   vse16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vse16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -     4.00    -      -     4.00   vse16.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   vse32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vse32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -     4.00    -      -     4.00   vse32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     8.00    -      -     8.00   vse32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     16.00   -      -     16.00  vse32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   vse32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vse32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     4.00    -      -     4.00   vse32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     8.00    -      -     8.00   vse32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -     16.00   -      -     16.00  vse32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   vse32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vse32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     4.00    -      -     4.00   vse32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -     8.00    -      -     8.00   vse32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -     16.00   -      -     16.00  vse32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   vse32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vse32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -     4.00    -      -     4.00   vse32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -     8.00    -      -     8.00   vse32.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vse64.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -     4.00    -      -     4.00   vse64.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -     8.00    -      -     8.00   vse64.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     16.00   -      -     16.00  vse64.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vse64.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -     4.00    -      -     4.00   vse64.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     8.00    -      -     8.00   vse64.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     16.00   -      -     16.00  vse64.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vse64.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     4.00    -      -     4.00   vse64.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     8.00    -      -     8.00   vse64.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -     16.00   -      -     16.00  vse64.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vse64.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     4.00    -      -     4.00   vse64.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -     8.00    -      -     8.00   vse64.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -     16.00   -      -     16.00  vse64.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vlm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vlm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vlm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vlm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vlm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vlm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vlm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vlm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vlm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vlm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vlm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vlm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vlm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vlm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vlm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vlm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vlm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vlm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vlm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vlm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vlm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -     2.00    -     vlm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vsm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vsm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vsm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vsm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vsm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vsm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vsm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vsm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vsm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vsm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vsm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vsm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vsm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vsm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vsm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vsm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vsm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vsm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vsm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vsm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vsm.v	v1, (a0)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vsetvli	zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -     2.00    -      -     2.00   vsm.v	v1, (a0)
diff --git a/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-gfx10.s b/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-gfx10.s
index 52b399e4f0..58b01031af 100644
--- a/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-gfx10.s
+++ b/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-gfx10.s
@@ -12,10 +12,6 @@
 ; CHECK-NEXT: .amdhsa_private_segment_fixed_size 0
 ; CHECK-NEXT: .amdhsa_kernarg_size 0
 ; CHECK-NEXT: ; SHARED_VGPR_COUNT 0
-; CHECK-NEXT: ; INST_PREF_SIZE 0
-; CHECK-NEXT: ; TRAP_ON_START 0
-; CHECK-NEXT: ; TRAP_ON_END 0
-; CHECK-NEXT: ; IMAGE_OP 0
 ; CHECK-NEXT: .amdhsa_next_free_vgpr 32
 ; CHECK-NEXT: .amdhsa_reserve_vcc 0
 ; CHECK-NEXT: .amdhsa_reserve_flat_scratch 0
@@ -69,10 +65,6 @@
 ; CHECK-NEXT: .amdhsa_private_segment_fixed_size 0
 ; CHECK-NEXT: .amdhsa_kernarg_size 0
 ; CHECK-NEXT: .amdhsa_shared_vgpr_count 0
-; CHECK-NEXT: ; INST_PREF_SIZE 0
-; CHECK-NEXT: ; TRAP_ON_START 0
-; CHECK-NEXT: ; TRAP_ON_END 0
-; CHECK-NEXT: ; IMAGE_OP 0
 ; CHECK-NEXT: .amdhsa_next_free_vgpr 32
 ; CHECK-NEXT: .amdhsa_reserve_vcc 0
 ; CHECK-NEXT: .amdhsa_reserve_flat_scratch 0
@@ -126,10 +118,6 @@
 ; CHECK-NEXT: .amdhsa_private_segment_fixed_size 0
 ; CHECK-NEXT: .amdhsa_kernarg_size 0
 ; CHECK-NEXT: .amdhsa_shared_vgpr_count 1
-; CHECK-NEXT: ; INST_PREF_SIZE 0
-; CHECK-NEXT: ; TRAP_ON_START 0
-; CHECK-NEXT: ; TRAP_ON_END 0
-; CHECK-NEXT: ; IMAGE_OP 0
 ; CHECK-NEXT: .amdhsa_next_free_vgpr 32
 ; CHECK-NEXT: .amdhsa_reserve_vcc 0
 ; CHECK-NEXT: .amdhsa_reserve_flat_scratch 0
@@ -183,10 +171,6 @@
 ; CHECK-NEXT: .amdhsa_private_segment_fixed_size 0
 ; CHECK-NEXT: .amdhsa_kernarg_size 0
 ; CHECK-NEXT: .amdhsa_shared_vgpr_count 1
-; CHECK-NEXT: ; INST_PREF_SIZE 0
-; CHECK-NEXT: ; TRAP_ON_START 0
-; CHECK-NEXT: ; TRAP_ON_END 0
-; CHECK-NEXT: ; IMAGE_OP 0
 ; CHECK-NEXT: .amdhsa_next_free_vgpr 32
 ; CHECK-NEXT: .amdhsa_reserve_vcc 0
 ; CHECK-NEXT: .amdhsa_reserve_flat_scratch 0
diff --git a/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-gfx11.s b/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-gfx11.s
new file mode 100644
index 0000000000..2133002908
--- /dev/null
+++ b/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-gfx11.s
@@ -0,0 +1,220 @@
+;; Test disassembly for gfx11 kernel descriptor.
+
+; RUN: rm -rf %t && split-file %s %t && cd %t
+
+;--- 1.s
+; RUN: llvm-mc --triple=amdgcn-amd-amdhsa -mattr=+wavefrontsize32,-wavefrontsize64 -filetype=obj -mcpu=gfx1100 < 1.s > 1.o
+; RUN: llvm-objdump --disassemble-symbols=kernel.kd 1.o | tail -n +7 | tee 1-disasm.s | FileCheck 1.s
+; RUN: llvm-mc --triple=amdgcn-amd-amdhsa -mattr=+wavefrontsize32,-wavefrontsize64 -filetype=obj -mcpu=gfx1100 < 1-disasm.s > 1-disasm.o
+; RUN: cmp 1.o 1-disasm.o
+; CHECK: .amdhsa_kernel kernel
+; CHECK-NEXT: .amdhsa_group_segment_fixed_size 0
+; CHECK-NEXT: .amdhsa_private_segment_fixed_size 0
+; CHECK-NEXT: .amdhsa_kernarg_size 0
+; CHECK-NEXT: ; SHARED_VGPR_COUNT 0
+; CHECK-NEXT: ; INST_PREF_SIZE 0
+; CHECK-NEXT: ; TRAP_ON_START 0
+; CHECK-NEXT: ; TRAP_ON_END 0
+; CHECK-NEXT: ; IMAGE_OP 0
+; CHECK-NEXT: .amdhsa_next_free_vgpr 32
+; CHECK-NEXT: .amdhsa_reserve_vcc 0
+; CHECK-NEXT: .amdhsa_reserve_xnack_mask 0
+; CHECK-NEXT: .amdhsa_next_free_sgpr 8
+; CHECK-NEXT: .amdhsa_float_round_mode_32 0
+; CHECK-NEXT: .amdhsa_float_round_mode_16_64 0
+; CHECK-NEXT: .amdhsa_float_denorm_mode_32 0
+; CHECK-NEXT: .amdhsa_float_denorm_mode_16_64 3
+; CHECK-NEXT: .amdhsa_dx10_clamp 1
+; CHECK-NEXT: .amdhsa_ieee_mode 1
+; CHECK-NEXT: .amdhsa_fp16_overflow 0
+; CHECK-NEXT: .amdhsa_workgroup_processor_mode 1
+; CHECK-NEXT: .amdhsa_memory_ordered 1
+; CHECK-NEXT: .amdhsa_forward_progress 0
+; CHECK-NEXT: .amdhsa_enable_private_segment 0
+; CHECK-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1
+; CHECK-NEXT: .amdhsa_system_sgpr_workgroup_id_y 0
+; CHECK-NEXT: .amdhsa_system_sgpr_workgroup_id_z 0
+; CHECK-NEXT: .amdhsa_system_sgpr_workgroup_info 0
+; CHECK-NEXT: .amdhsa_system_vgpr_workitem_id 0
+; CHECK-NEXT: .amdhsa_exception_fp_ieee_invalid_op 0
+; CHECK-NEXT: .amdhsa_exception_fp_denorm_src 0
+; CHECK-NEXT: .amdhsa_exception_fp_ieee_div_zero 0
+; CHECK-NEXT: .amdhsa_exception_fp_ieee_overflow 0
+; CHECK-NEXT: .amdhsa_exception_fp_ieee_underflow 0
+; CHECK-NEXT: .amdhsa_exception_fp_ieee_inexact 0
+; CHECK-NEXT: .amdhsa_exception_int_div_zero 0
+; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
+; CHECK-NEXT: .amdhsa_user_sgpr_queue_ptr 0
+; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
+; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_id 0
+; CHECK-NEXT: .amdhsa_user_sgpr_private_segment_size 0
+; CHECK-NEXT: .amdhsa_wavefront_size32 1
+; CHECK-NEXT: .end_amdhsa_kernel
+.amdhsa_kernel kernel
+  .amdhsa_next_free_vgpr 32
+  .amdhsa_next_free_sgpr 32
+  .amdhsa_wavefront_size32 1
+.end_amdhsa_kernel
+
+;--- 2.s
+; RUN: llvm-mc --triple=amdgcn-amd-amdhsa -mattr=+wavefrontsize64,-wavefrontsize32 -filetype=obj -mcpu=gfx1100 < 2.s > 2.o
+; RUN: llvm-objdump --disassemble-symbols=kernel.kd 2.o | tail -n +7 | tee 2-disasm.s | FileCheck 2.s
+; RUN: llvm-mc --triple=amdgcn-amd-amdhsa -mattr=+wavefrontsize64,-wavefrontsize32 -filetype=obj -mcpu=gfx1100 < 2-disasm.s > 2-disasm.o
+; RUN: cmp 2.o 2-disasm.o
+; CHECK: .amdhsa_kernel kernel
+; CHECK-NEXT: .amdhsa_group_segment_fixed_size 0
+; CHECK-NEXT: .amdhsa_private_segment_fixed_size 0
+; CHECK-NEXT: .amdhsa_kernarg_size 0
+; CHECK-NEXT: .amdhsa_shared_vgpr_count 0
+; CHECK-NEXT: ; INST_PREF_SIZE 0
+; CHECK-NEXT: ; TRAP_ON_START 0
+; CHECK-NEXT: ; TRAP_ON_END 0
+; CHECK-NEXT: ; IMAGE_OP 0
+; CHECK-NEXT: .amdhsa_next_free_vgpr 32
+; CHECK-NEXT: .amdhsa_reserve_vcc 0
+; CHECK-NEXT: .amdhsa_reserve_xnack_mask 0
+; CHECK-NEXT: .amdhsa_next_free_sgpr 8
+; CHECK-NEXT: .amdhsa_float_round_mode_32 0
+; CHECK-NEXT: .amdhsa_float_round_mode_16_64 0
+; CHECK-NEXT: .amdhsa_float_denorm_mode_32 0
+; CHECK-NEXT: .amdhsa_float_denorm_mode_16_64 3
+; CHECK-NEXT: .amdhsa_dx10_clamp 1
+; CHECK-NEXT: .amdhsa_ieee_mode 1
+; CHECK-NEXT: .amdhsa_fp16_overflow 0
+; CHECK-NEXT: .amdhsa_workgroup_processor_mode 1
+; CHECK-NEXT: .amdhsa_memory_ordered 1
+; CHECK-NEXT: .amdhsa_forward_progress 0
+; CHECK-NEXT: .amdhsa_enable_private_segment 0
+; CHECK-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1
+; CHECK-NEXT: .amdhsa_system_sgpr_workgroup_id_y 0
+; CHECK-NEXT: .amdhsa_system_sgpr_workgroup_id_z 0
+; CHECK-NEXT: .amdhsa_system_sgpr_workgroup_info 0
+; CHECK-NEXT: .amdhsa_system_vgpr_workitem_id 0
+; CHECK-NEXT: .amdhsa_exception_fp_ieee_invalid_op 0
+; CHECK-NEXT: .amdhsa_exception_fp_denorm_src 0
+; CHECK-NEXT: .amdhsa_exception_fp_ieee_div_zero 0
+; CHECK-NEXT: .amdhsa_exception_fp_ieee_overflow 0
+; CHECK-NEXT: .amdhsa_exception_fp_ieee_underflow 0
+; CHECK-NEXT: .amdhsa_exception_fp_ieee_inexact 0
+; CHECK-NEXT: .amdhsa_exception_int_div_zero 0
+; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
+; CHECK-NEXT: .amdhsa_user_sgpr_queue_ptr 0
+; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
+; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_id 0
+; CHECK-NEXT: .amdhsa_user_sgpr_private_segment_size 0
+; CHECK-NEXT: .amdhsa_wavefront_size32 0
+; CHECK-NEXT: .end_amdhsa_kernel
+.amdhsa_kernel kernel
+  .amdhsa_next_free_vgpr 32
+  .amdhsa_next_free_sgpr 32
+  .amdhsa_shared_vgpr_count 0
+.end_amdhsa_kernel
+
+;--- 3.s
+; RUN: llvm-mc --triple=amdgcn-amd-amdhsa -mattr=+wavefrontsize64,-wavefrontsize32 -filetype=obj -mcpu=gfx1100 < 3.s > 3.o
+; RUN: llvm-objdump --disassemble-symbols=kernel.kd 3.o | tail -n +7 | tee 3-disasm.s | FileCheck 3.s
+; RUN: llvm-mc --triple=amdgcn-amd-amdhsa -mattr=+wavefrontsize64,-wavefrontsize32 -filetype=obj -mcpu=gfx1100 < 3-disasm.s > 3-disasm.o
+; RUN: cmp 3.o 3-disasm.o
+; CHECK: .amdhsa_kernel kernel
+; CHECK-NEXT: .amdhsa_group_segment_fixed_size 0
+; CHECK-NEXT: .amdhsa_private_segment_fixed_size 0
+; CHECK-NEXT: .amdhsa_kernarg_size 0
+; CHECK-NEXT: .amdhsa_shared_vgpr_count 1
+; CHECK-NEXT: ; INST_PREF_SIZE 0
+; CHECK-NEXT: ; TRAP_ON_START 0
+; CHECK-NEXT: ; TRAP_ON_END 0
+; CHECK-NEXT: ; IMAGE_OP 0
+; CHECK-NEXT: .amdhsa_next_free_vgpr 32
+; CHECK-NEXT: .amdhsa_reserve_vcc 0
+; CHECK-NEXT: .amdhsa_reserve_xnack_mask 0
+; CHECK-NEXT: .amdhsa_next_free_sgpr 8
+; CHECK-NEXT: .amdhsa_float_round_mode_32 0
+; CHECK-NEXT: .amdhsa_float_round_mode_16_64 0
+; CHECK-NEXT: .amdhsa_float_denorm_mode_32 0
+; CHECK-NEXT: .amdhsa_float_denorm_mode_16_64 3
+; CHECK-NEXT: .amdhsa_dx10_clamp 1
+; CHECK-NEXT: .amdhsa_ieee_mode 1
+; CHECK-NEXT: .amdhsa_fp16_overflow 0
+; CHECK-NEXT: .amdhsa_workgroup_processor_mode 1
+; CHECK-NEXT: .amdhsa_memory_ordered 1
+; CHECK-NEXT: .amdhsa_forward_progress 0
+; CHECK-NEXT: .amdhsa_enable_private_segment 0
+; CHECK-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1
+; CHECK-NEXT: .amdhsa_system_sgpr_workgroup_id_y 0
+; CHECK-NEXT: .amdhsa_system_sgpr_workgroup_id_z 0
+; CHECK-NEXT: .amdhsa_system_sgpr_workgroup_info 0
+; CHECK-NEXT: .amdhsa_system_vgpr_workitem_id 0
+; CHECK-NEXT: .amdhsa_exception_fp_ieee_invalid_op 0
+; CHECK-NEXT: .amdhsa_exception_fp_denorm_src 0
+; CHECK-NEXT: .amdhsa_exception_fp_ieee_div_zero 0
+; CHECK-NEXT: .amdhsa_exception_fp_ieee_overflow 0
+; CHECK-NEXT: .amdhsa_exception_fp_ieee_underflow 0
+; CHECK-NEXT: .amdhsa_exception_fp_ieee_inexact 0
+; CHECK-NEXT: .amdhsa_exception_int_div_zero 0
+; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
+; CHECK-NEXT: .amdhsa_user_sgpr_queue_ptr 0
+; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
+; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_id 0
+; CHECK-NEXT: .amdhsa_user_sgpr_private_segment_size 0
+; CHECK-NEXT: .amdhsa_wavefront_size32 0
+; CHECK-NEXT: .end_amdhsa_kernel
+.amdhsa_kernel kernel
+  .amdhsa_next_free_vgpr 32
+  .amdhsa_next_free_sgpr 32
+  .amdhsa_shared_vgpr_count 1
+.end_amdhsa_kernel
+
+;--- 4.s
+; RUN: llvm-mc --triple=amdgcn-amd-amdhsa -mattr=+wavefrontsize64,-wavefrontsize32 -filetype=obj -mcpu=gfx1100 < 4.s > 4.o
+; RUN: llvm-objdump --disassemble-symbols=kernel.kd 4.o | tail -n +7 | tee 4-disasm.s | FileCheck 4.s
+; RUN: llvm-mc --triple=amdgcn-amd-amdhsa -mattr=+wavefrontsize64,-wavefrontsize32 -filetype=obj -mcpu=gfx1100 < 4-disasm.s > 4-disasm.o
+; RUN: cmp 4.o 4-disasm.o
+; CHECK: .amdhsa_kernel kernel
+; CHECK-NEXT: .amdhsa_group_segment_fixed_size 0
+; CHECK-NEXT: .amdhsa_private_segment_fixed_size 0
+; CHECK-NEXT: .amdhsa_kernarg_size 0
+; CHECK-NEXT: .amdhsa_shared_vgpr_count 1
+; CHECK-NEXT: ; INST_PREF_SIZE 0
+; CHECK-NEXT: ; TRAP_ON_START 0
+; CHECK-NEXT: ; TRAP_ON_END 0
+; CHECK-NEXT: ; IMAGE_OP 0
+; CHECK-NEXT: .amdhsa_next_free_vgpr 32
+; CHECK-NEXT: .amdhsa_reserve_vcc 0
+; CHECK-NEXT: .amdhsa_reserve_xnack_mask 0
+; CHECK-NEXT: .amdhsa_next_free_sgpr 8
+; CHECK-NEXT: .amdhsa_float_round_mode_32 0
+; CHECK-NEXT: .amdhsa_float_round_mode_16_64 0
+; CHECK-NEXT: .amdhsa_float_denorm_mode_32 0
+; CHECK-NEXT: .amdhsa_float_denorm_mode_16_64 3
+; CHECK-NEXT: .amdhsa_dx10_clamp 1
+; CHECK-NEXT: .amdhsa_ieee_mode 1
+; CHECK-NEXT: .amdhsa_fp16_overflow 0
+; CHECK-NEXT: .amdhsa_workgroup_processor_mode 1
+; CHECK-NEXT: .amdhsa_memory_ordered 1
+; CHECK-NEXT: .amdhsa_forward_progress 0
+; CHECK-NEXT: .amdhsa_enable_private_segment 0
+; CHECK-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1
+; CHECK-NEXT: .amdhsa_system_sgpr_workgroup_id_y 0
+; CHECK-NEXT: .amdhsa_system_sgpr_workgroup_id_z 0
+; CHECK-NEXT: .amdhsa_system_sgpr_workgroup_info 0
+; CHECK-NEXT: .amdhsa_system_vgpr_workitem_id 0
+; CHECK-NEXT: .amdhsa_exception_fp_ieee_invalid_op 0
+; CHECK-NEXT: .amdhsa_exception_fp_denorm_src 0
+; CHECK-NEXT: .amdhsa_exception_fp_ieee_div_zero 0
+; CHECK-NEXT: .amdhsa_exception_fp_ieee_overflow 0
+; CHECK-NEXT: .amdhsa_exception_fp_ieee_underflow 0
+; CHECK-NEXT: .amdhsa_exception_fp_ieee_inexact 0
+; CHECK-NEXT: .amdhsa_exception_int_div_zero 0
+; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
+; CHECK-NEXT: .amdhsa_user_sgpr_queue_ptr 0
+; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
+; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_id 0
+; CHECK-NEXT: .amdhsa_user_sgpr_private_segment_size 0
+; CHECK-NEXT: .amdhsa_wavefront_size32 0
+; CHECK-NEXT: .end_amdhsa_kernel
+.amdhsa_kernel kernel
+  .amdhsa_next_free_vgpr 32
+  .amdhsa_next_free_sgpr 32
+  .amdhsa_shared_vgpr_count 1
+  .amdhsa_wavefront_size32 0
+.end_amdhsa_kernel
diff --git a/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-zeroed-gfx10.s b/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-zeroed-gfx10.s
index 04cf28f89e..39cef4da42 100644
--- a/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-zeroed-gfx10.s
+++ b/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-zeroed-gfx10.s
@@ -26,10 +26,6 @@
 ; OBJDUMP-NEXT:         .amdhsa_private_segment_fixed_size 0
 ; OBJDUMP-NEXT:         .amdhsa_kernarg_size 0
 ; OBJDUMP-NEXT:         .amdhsa_shared_vgpr_count 0
-; OBJDUMP-NEXT:         ; INST_PREF_SIZE 0
-; OBJDUMP-NEXT:         ; TRAP_ON_START 0
-; OBJDUMP-NEXT:         ; TRAP_ON_END 0
-; OBJDUMP-NEXT:         ; IMAGE_OP 0
 ; OBJDUMP-NEXT:         .amdhsa_next_free_vgpr 8
 ; OBJDUMP-NEXT:         .amdhsa_reserve_vcc 0
 ; OBJDUMP-NEXT:         .amdhsa_reserve_flat_scratch 0
diff --git a/llvm/test/tools/llvm-objdump/ELF/RISCV/branches.s b/llvm/test/tools/llvm-objdump/ELF/RISCV/branches.s
index 5fec4e6e25..ebd86a702b 100644
--- a/llvm/test/tools/llvm-objdump/ELF/RISCV/branches.s
+++ b/llvm/test/tools/llvm-objdump/ELF/RISCV/branches.s
@@ -57,11 +57,11 @@ c.jal bar
 c.j bar
 
 # CHECK: auipc ra, 0
-# CHECK: jalr	ra, 16(ra){{$}}
+# CHECK: jalr	ra, 16(ra) <foo+0x58>
 call .Llocal
 
 # CHECK: auipc ra, 0
-# CHECK: jalr	ra, 16(ra){{$}}
+# CHECK: jalr	ra, 16(ra) <bar>
 call bar
 
 .Llocal:
diff --git a/llvm/test/tools/llvm-objdump/ELF/RISCV/multi-instr-target.s b/llvm/test/tools/llvm-objdump/ELF/RISCV/multi-instr-target.s
new file mode 100644
index 0000000000..91b643e961
--- /dev/null
+++ b/llvm/test/tools/llvm-objdump/ELF/RISCV/multi-instr-target.s
@@ -0,0 +1,45 @@
+# RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+c < %s | \
+# RUN:     llvm-objdump -d -M no-aliases --no-show-raw-insn - | \
+# RUN:     FileCheck %s
+
+## Test multiple interleaved auipc/jalr pairs
+# CHECK: auipc t0, 0
+1: auipc t0, %pcrel_hi(bar)
+# CHECK: auipc t1, 0
+2: auipc t1, %pcrel_hi(bar)
+# CHECK: jalr ra, {{[0-9]+}}(t0) <bar>
+jalr %pcrel_lo(1b)(t0)
+## Target should not be printed because the call above clobbers register state
+# CHECK: jalr ra, {{[0-9]+}}(t1){{$}}
+jalr %pcrel_lo(2b)(t1)
+
+## Test that auipc+jalr with a write to the target register in between does not
+## print the target
+# CHECK: auipc t0, 0
+1: auipc t0, %pcrel_hi(bar)
+# CHECK: c.li t0, 0
+li t0, 0
+# CHECK: jalr ra, {{[0-9]+}}(t0){{$}}
+jalr %pcrel_lo(1b)(t0)
+
+## Test that auipc+jalr with a write to an unrelated register in between does
+## print the target
+# CHECK: auipc t0, 0
+1: auipc t0, %pcrel_hi(bar)
+# CHECK: c.li t1, 0
+li t1, 0
+# CHECK: jalr ra, {{[0-9]+}}(t0) <bar>
+jalr %pcrel_lo(1b)(t0)
+
+## Test that auipc+jalr with a terminator in between does not print the target
+# CHECK: auipc t0, 0
+1: auipc t0, %pcrel_hi(bar)
+# CHECK: c.j {{.*}} <bar>
+j bar
+# CHECK: jalr ra, {{[0-9]+}}(t0){{$}}
+jalr %pcrel_lo(1b)(t0)
+
+# CHECK-LABEL: <bar>:
+bar:
+# CHECK: c.nop
+nop
diff --git a/llvm/test/tools/llvm-rc/Inputs/split-path.rc b/llvm/test/tools/llvm-rc/Inputs/split-path.rc
new file mode 100644
index 0000000000..fb510e8969
--- /dev/null
+++ b/llvm/test/tools/llvm-rc/Inputs/split-path.rc
@@ -0,0 +1,2 @@
+100 ICON "subdir" "/icon-new.ico"
+101 24 "subdir" "/empty.manifest"
diff --git a/llvm/test/tools/llvm-rc/split-path.test b/llvm/test/tools/llvm-rc/split-path.test
new file mode 100644
index 0000000000..a12fd2bc32
--- /dev/null
+++ b/llvm/test/tools/llvm-rc/split-path.test
@@ -0,0 +1,7 @@
+; RUN: rm -rf %t
+; RUN: mkdir %t
+; RUN: cd %t
+; RUN: mkdir subdir
+; RUN: cp %p/Inputs/icon-new.ico subdir
+; RUN: touch subdir/empty.manifest
+; RUN: llvm-windres --no-preprocess %p/Inputs/split-path.rc %t/split-path.res
diff --git a/llvm/test/tools/llvm-readobj/ELF/note-core.test b/llvm/test/tools/llvm-readobj/ELF/note-core.test
index 4c79460f94..84ec96b170 100644
--- a/llvm/test/tools/llvm-readobj/ELF/note-core.test
+++ b/llvm/test/tools/llvm-readobj/ELF/note-core.test
@@ -240,40 +240,45 @@
 # RUN: llvm-readelf --notes %t48.o | FileCheck %s --check-prefix=CHECK-GNU  -DDESC="NT_ARM_PAC_MASK (AArch64 Pointer Authentication code masks)"
 # RUN: llvm-readobj --notes %t48.o | FileCheck %s --check-prefix=CHECK-LLVM -DDESC="NT_ARM_PAC_MASK (AArch64 Pointer Authentication code masks)"
 
+## Check ELF::NT_ARM_TAGGED_ADDR_CTRL
+# RUN: yaml2obj %s -DTYPE=0x409 -o %t49.o
+# RUN: llvm-readelf --notes %t49.o | FileCheck %s --check-prefix=CHECK-GNU  -DDESC="NT_ARM_TAGGED_ADDR_CTRL (AArch64 Tagged Address Control)"
+# RUN: llvm-readobj --notes %t49.o | FileCheck %s --check-prefix=CHECK-LLVM -DDESC="NT_ARM_TAGGED_ADDR_CTRL (AArch64 Tagged Address Control)"
+
 ## Check ELF::NT_ARM_SSVE
-# RUN: yaml2obj %s -DTYPE=0x40b -o %t49.o
-# RUN: llvm-readelf --notes %t49.o | FileCheck %s --check-prefix=CHECK-GNU  -DDESC="NT_ARM_SSVE (AArch64 Streaming SVE registers)"
-# RUN: llvm-readobj --notes %t49.o | FileCheck %s --check-prefix=CHECK-LLVM -DDESC="NT_ARM_SSVE (AArch64 Streaming SVE registers)"
+# RUN: yaml2obj %s -DTYPE=0x40b -o %t50.o
+# RUN: llvm-readelf --notes %t50.o | FileCheck %s --check-prefix=CHECK-GNU  -DDESC="NT_ARM_SSVE (AArch64 Streaming SVE registers)"
+# RUN: llvm-readobj --notes %t50.o | FileCheck %s --check-prefix=CHECK-LLVM -DDESC="NT_ARM_SSVE (AArch64 Streaming SVE registers)"
 
 ## Check ELF::NT_ARM_ZA
-# RUN: yaml2obj %s -DTYPE=0x40c -o %t50.o
-# RUN: llvm-readelf --notes %t50.o | FileCheck %s --check-prefix=CHECK-GNU  -DDESC="NT_ARM_ZA (AArch64 SME ZA registers)"
-# RUN: llvm-readobj --notes %t50.o | FileCheck %s --check-prefix=CHECK-LLVM -DDESC="NT_ARM_ZA (AArch64 SME ZA registers)"
+# RUN: yaml2obj %s -DTYPE=0x40c -o %t51.o
+# RUN: llvm-readelf --notes %t51.o | FileCheck %s --check-prefix=CHECK-GNU  -DDESC="NT_ARM_ZA (AArch64 SME ZA registers)"
+# RUN: llvm-readobj --notes %t51.o | FileCheck %s --check-prefix=CHECK-LLVM -DDESC="NT_ARM_ZA (AArch64 SME ZA registers)"
 
 ## Check ELF::NT_ARM_ZT
-# RUN: yaml2obj %s -DTYPE=0x40d -o %t51.o
-# RUN: llvm-readelf --notes %t51.o | FileCheck %s --check-prefix=CHECK-GNU  -DDESC="NT_ARM_ZT (AArch64 SME ZT registers)"
-# RUN: llvm-readobj --notes %t51.o | FileCheck %s --check-prefix=CHECK-LLVM -DDESC="NT_ARM_ZT (AArch64 SME ZT registers)"
+# RUN: yaml2obj %s -DTYPE=0x40d -o %t52.o
+# RUN: llvm-readelf --notes %t52.o | FileCheck %s --check-prefix=CHECK-GNU  -DDESC="NT_ARM_ZT (AArch64 SME ZT registers)"
+# RUN: llvm-readobj --notes %t52.o | FileCheck %s --check-prefix=CHECK-LLVM -DDESC="NT_ARM_ZT (AArch64 SME ZT registers)"
 
 ## Check ELF::NT_FILE.
-# RUN: yaml2obj %s -DTYPE=0x46494c45 -o %t52.o
-# RUN: llvm-readelf --notes %t52.o | FileCheck %s --check-prefix=CHECK-GNU  -DDESC="NT_FILE (mapped files)"
-# RUN: llvm-readobj --notes %t52.o | FileCheck %s --check-prefix=CHECK-LLVM -DDESC="NT_FILE (mapped files)"
+# RUN: yaml2obj %s -DTYPE=0x46494c45 -o %t53.o
+# RUN: llvm-readelf --notes %t53.o | FileCheck %s --check-prefix=CHECK-GNU  -DDESC="NT_FILE (mapped files)"
+# RUN: llvm-readobj --notes %t53.o | FileCheck %s --check-prefix=CHECK-LLVM -DDESC="NT_FILE (mapped files)"
 
 ## Check ELF::NT_PRXFPREG.
-# RUN: yaml2obj %s -DTYPE=0x46e62b7f -o %t53.o
-# RUN: llvm-readelf --notes %t53.o | FileCheck %s --check-prefix=CHECK-GNU  -DDESC="NT_PRXFPREG (user_xfpregs structure)"
-# RUN: llvm-readobj --notes %t53.o | FileCheck %s --check-prefix=CHECK-LLVM -DDESC="NT_PRXFPREG (user_xfpregs structure)"
+# RUN: yaml2obj %s -DTYPE=0x46e62b7f -o %t54.o
+# RUN: llvm-readelf --notes %t54.o | FileCheck %s --check-prefix=CHECK-GNU  -DDESC="NT_PRXFPREG (user_xfpregs structure)"
+# RUN: llvm-readobj --notes %t54.o | FileCheck %s --check-prefix=CHECK-LLVM -DDESC="NT_PRXFPREG (user_xfpregs structure)"
 
 ## Check ELF::NT_SIGINFO.
-# RUN: yaml2obj %s -DTYPE=0x53494749 -o %t54.o
-# RUN: llvm-readelf --notes %t54.o | FileCheck %s --check-prefix=CHECK-GNU  -DDESC="NT_SIGINFO (siginfo_t data)"
-# RUN: llvm-readobj --notes %t54.o | FileCheck %s --check-prefix=CHECK-LLVM -DDESC="NT_SIGINFO (siginfo_t data)"
+# RUN: yaml2obj %s -DTYPE=0x53494749 -o %t55.o
+# RUN: llvm-readelf --notes %t55.o | FileCheck %s --check-prefix=CHECK-GNU  -DDESC="NT_SIGINFO (siginfo_t data)"
+# RUN: llvm-readobj --notes %t55.o | FileCheck %s --check-prefix=CHECK-LLVM -DDESC="NT_SIGINFO (siginfo_t data)"
 
 ## Check an arbitrary unknown type.
-# RUN: yaml2obj %s -DTYPE=0x12345678 -o %t55.o
-# RUN: llvm-readelf --notes %t55.o | FileCheck %s --check-prefix=CHECK-GNU  -DDESC="Unknown note type: (0x12345678)"
-# RUN: llvm-readobj --notes %t55.o | FileCheck %s --check-prefix=CHECK-LLVM -DDESC="Unknown (0x12345678)"
+# RUN: yaml2obj %s -DTYPE=0x12345678 -o %t56.o
+# RUN: llvm-readelf --notes %t56.o | FileCheck %s --check-prefix=CHECK-GNU  -DDESC="Unknown note type: (0x12345678)"
+# RUN: llvm-readobj --notes %t56.o | FileCheck %s --check-prefix=CHECK-LLVM -DDESC="Unknown (0x12345678)"
 
 # CHECK-GNU:      Owner Data size  Description
 # CHECK-GNU-NEXT: CORE  0x00000000 [[DESC]]
diff --git a/llvm/test/tools/llvm-remarkutil/Inputs/annotation-count-with-dbg-loc.yaml b/llvm/test/tools/llvm-remarkutil/Inputs/annotation-count-with-dbg-loc.yaml
index 6262ee262f..ebdd722774 100644
--- a/llvm/test/tools/llvm-remarkutil/Inputs/annotation-count-with-dbg-loc.yaml
+++ b/llvm/test/tools/llvm-remarkutil/Inputs/annotation-count-with-dbg-loc.yaml
@@ -12,7 +12,7 @@ Args:
 Pass:            annotation-remarks
 Name:            AnnotationSummary
 DebugLoc:        { File: path/to/anno2.c, Line: 1, Column: 2 }
-Function:        func2 
+Function:        func2
 Args:
   - String:          'Annotated '
   - count:           '2'
diff --git a/llvm/test/tools/llvm-remarkutil/Inputs/annotation-count.yaml b/llvm/test/tools/llvm-remarkutil/Inputs/annotation-count.yaml
index f29808a777..9b428f013c 100644
--- a/llvm/test/tools/llvm-remarkutil/Inputs/annotation-count.yaml
+++ b/llvm/test/tools/llvm-remarkutil/Inputs/annotation-count.yaml
@@ -1,7 +1,7 @@
 --- !Analysis
 Pass:            annotation-remarks
 Name:            AnnotationSummary
-Function:        func1 
+Function:        func1
 Args:
   - String:          'Annotated '
   - count:           '1'
@@ -10,7 +10,7 @@ Args:
 --- !Analysis
 Pass:            annotation-remarks
 Name:            AnnotationSummary
-Function:        func2 
+Function:        func2
 Args:
   - String:          'Annotated '
   - count:           '2'
diff --git a/llvm/test/tools/llvm-remarkutil/annotation-count-with-dbg-loc.test b/llvm/test/tools/llvm-remarkutil/annotation-count-with-dbg-loc.test
index cdc731c003..673a38001f 100644
--- a/llvm/test/tools/llvm-remarkutil/annotation-count-with-dbg-loc.test
+++ b/llvm/test/tools/llvm-remarkutil/annotation-count-with-dbg-loc.test
@@ -1,7 +1,14 @@
 RUN: llvm-remarkutil annotation-count --use-debug-loc --parser=yaml --annotation-type=remark %p/Inputs/annotation-count-with-dbg-loc.yaml | FileCheck %s
 RUN: llvm-remarkutil yaml2bitstream %p/Inputs/annotation-count-with-dbg-loc.yaml | llvm-remarkutil annotation-count --use-debug-loc --parser=bitstream --annotation-type=remark | FileCheck %s
+RUN: llvm-remarkutil count --parser=yaml --count-by=arg --group-by=function-with-loc --remark-name="AnnotationSummary" %p/Inputs/annotation-count-with-dbg-loc.yaml | FileCheck %s --check-prefix=COUNT-CHECK
+RUN: llvm-remarkutil yaml2bitstream %p/Inputs/annotation-count-with-dbg-loc.yaml | llvm-remarkutil count --parser=bitstream --count-by=arg --group-by=function-with-loc --remark-name="AnnotationSummary" | FileCheck %s --check-prefix=COUNT-CHECK
 
 ; CHECK-LABEL: Source,Function,Count
 ; CHECK: path/to/anno.c:1:2,func1,1
 ; CHECK: path/to/anno2.c:1:2,func2,2
 ; CHECK: path/to/anno3.c:1:2,func3,3
+
+; COUNT-CHECK-LABEL: FuctionWithDebugLoc,count
+; COUNT-CHECK: path/to/anno.c:func1,1
+; COUNT-CHECK: path/to/anno2.c:func2,2
+; COUNT-CHECK: path/to/anno3.c:func3,3
diff --git a/llvm/test/tools/llvm-remarkutil/annotation-count.test b/llvm/test/tools/llvm-remarkutil/annotation-count.test
index 7358240220..e006220c64 100644
--- a/llvm/test/tools/llvm-remarkutil/annotation-count.test
+++ b/llvm/test/tools/llvm-remarkutil/annotation-count.test
@@ -1,7 +1,14 @@
 RUN: llvm-remarkutil annotation-count --parser=yaml --annotation-type=remark %p/Inputs/annotation-count.yaml | FileCheck %s
 RUN: llvm-remarkutil yaml2bitstream %p/Inputs/annotation-count.yaml | llvm-remarkutil annotation-count --parser=bitstream --annotation-type=remark | FileCheck %s
+RUN: llvm-remarkutil count --parser=yaml --count-by=arg --group-by=function --remark-name="AnnotationSummary" %p/Inputs/annotation-count.yaml | FileCheck %s --check-prefix=COUNT-CHECK
+RUN: llvm-remarkutil yaml2bitstream %p/Inputs/annotation-count.yaml | llvm-remarkutil count --parser=bitstream --count-by=arg --group-by=function --remark-name="AnnotationSummary" | FileCheck %s --check-prefix=COUNT-CHECK
 
 ; CHECK-LABEL: Function,Count
 ; CHECK: func1,1
 ; CHECK: func2,2
 ; CHECK: func3,3
+
+; COUNT-CHECK-LABEL: Function,count
+; COUNT-CHECK: func1,1
+; COUNT-CHECK: func2,2
+; COUNT-CHECK: func3,3
diff --git a/llvm/test/tools/llvm-remarkutil/broken-bitstream-remark.test b/llvm/test/tools/llvm-remarkutil/broken-bitstream-remark.test
index 515c096f54..78011aece0 100644
--- a/llvm/test/tools/llvm-remarkutil/broken-bitstream-remark.test
+++ b/llvm/test/tools/llvm-remarkutil/broken-bitstream-remark.test
@@ -1,4 +1,6 @@
 RUN: not llvm-remarkutil bitstream2yaml %p/Inputs/broken-remark -o - 2>&1 | FileCheck %s
 RUN: not llvm-remarkutil instruction-count --parser=bitstream %p/Inputs/broken-remark -o - 2>&1 | FileCheck %s
 RUN: not llvm-remarkutil annotation-count --parser=bitstream --annotation-type=remark %p/Inputs/broken-remark -o - 2>&1 | FileCheck %s
+RUN: not llvm-remarkutil count --parser=bitstream %p/Inputs/broken-remark -o - 2>&1 | FileCheck %s
+
 CHECK: error: Unknown magic number: expecting RMRK, got --- .
diff --git a/llvm/test/tools/llvm-remarkutil/broken-yaml-remark.test b/llvm/test/tools/llvm-remarkutil/broken-yaml-remark.test
index df87e3db80..0f06506603 100644
--- a/llvm/test/tools/llvm-remarkutil/broken-yaml-remark.test
+++ b/llvm/test/tools/llvm-remarkutil/broken-yaml-remark.test
@@ -1,4 +1,6 @@
 RUN: not llvm-remarkutil yaml2bitstream %p/Inputs/broken-remark -o - 2>&1 | FileCheck %s
 RUN: not llvm-remarkutil instruction-count --parser=yaml %p/Inputs/broken-remark -o - 2>&1 | FileCheck %s
 RUN: not llvm-remarkutil annotation-count --parser=yaml --annotation-type=remark %p/Inputs/broken-remark -o - 2>&1 | FileCheck %s
+RUN: not llvm-remarkutil count --parser=yaml %p/Inputs/broken-remark -o - 2>&1 | FileCheck %s
+
 CHECK: error: Type, Pass, Name or Function missing
diff --git a/llvm/test/tools/llvm-remarkutil/count/Inputs/remark-count-by.yaml b/llvm/test/tools/llvm-remarkutil/count/Inputs/remark-count-by.yaml
new file mode 100644
index 0000000000..7d1b76eaf0
--- /dev/null
+++ b/llvm/test/tools/llvm-remarkutil/count/Inputs/remark-count-by.yaml
@@ -0,0 +1,43 @@
+--- !Analysis
+Pass:            generic-remarks-pass
+Name:            Remark
+DebugLoc:        { File: path/to/anno.c, Line: 1, Column: 2 }
+Function:        func1
+Args:
+  - count1:           '1'
+  - count2:           '2'
+  - count3:           '3'
+  - count4:           '4'
+  - String:          ' instructions with '
+  - type:            remark
+--- !Analysis
+Pass:            generic-remarks-pass
+Name:            Remark2
+DebugLoc:        { File: path/to/anno2.c, Line: 1, Column: 2 }
+Function:        func1
+Args:
+  - count1:           '1'
+  - count2:           '2'
+  - count3:           '3'
+  - String:          ' instructions with '
+  - type:            remark
+--- !Analysis
+Pass:            generic-remarks-pass
+Name:            Remark3
+DebugLoc:        { File: path/to/anno.c, Line: 1, Column: 2 }
+Function:        func1
+Args:
+  - count1:           '1'
+  - String:          ' instructions with '
+  - type:            remark
+--- !Analysis
+Pass:            generic-remarks-pass
+Name:            Remark
+DebugLoc:        { File: path/to/anno.c, Line: 1, Column: 2 }
+Function:        func2
+Args:
+  - count1:           '1'
+  - count2:           '2'
+  - count3:           '3'
+  - String:          ' instructions with '
+  - type:            remark
diff --git a/llvm/test/tools/llvm-remarkutil/count/Inputs/remark-filter-by.yaml b/llvm/test/tools/llvm-remarkutil/count/Inputs/remark-filter-by.yaml
new file mode 100644
index 0000000000..b4318370b2
--- /dev/null
+++ b/llvm/test/tools/llvm-remarkutil/count/Inputs/remark-filter-by.yaml
@@ -0,0 +1,40 @@
+--- !Analysis
+Pass:            generic-remarks-pass
+Name:            Remark
+DebugLoc:        { File: path/to/anno2.c, Line: 1, Column: 2 }
+Function:        func1
+Args:
+  - count1:           '1'
+  - String:          ' instructions with '
+  - type:            remark
+--- !Analysis
+Pass:            generic-remarks-pass2
+Name:            Remark2
+DebugLoc:        { File: path/to/anno2.c, Line: 1, Column: 2 }
+Function:        func1
+Args:
+  - count1:           '1'
+  - count2:           '2'
+  - count3:           '3'
+  - String:          ' instructions with '
+  - type:            remark
+--- !Missed
+Pass:            generic-remarks-pass
+Name:            Remark3
+DebugLoc:        { File: path/to/anno.c, Line: 1, Column: 2 }
+Function:        func1
+Args:
+  - count1:           '1'
+  - String:          ' instructions with '
+  - type:            remark
+--- !Passed
+Pass:            generic-remarks-pass
+Name:            Remark
+DebugLoc:        { File: path/to/anno.c, Line: 1, Column: 2 }
+Function:        func1
+Args:
+  - count1:           '1'
+  - count2:           '2'
+  - count3:           '3'
+  - String:          ' instructions with '
+  - type:            remark
diff --git a/llvm/test/tools/llvm-remarkutil/count/Inputs/remark-group-by.yaml b/llvm/test/tools/llvm-remarkutil/count/Inputs/remark-group-by.yaml
new file mode 100644
index 0000000000..a61870a87e
--- /dev/null
+++ b/llvm/test/tools/llvm-remarkutil/count/Inputs/remark-group-by.yaml
@@ -0,0 +1,54 @@
+--- !Analysis
+Pass:            generic-remarks-pass
+Name:            Remark
+DebugLoc:        { File: path/to/anno.c, Line: 1, Column: 2 }
+Function:        func1
+Args:
+  - count:           '1'
+  - String:          ' instructions with '
+  - type:            remark
+--- !Missed
+Pass:            generic-remarks-pass
+Name:            Remark
+DebugLoc:        { File: path/to/anno.c, Line: 1, Column: 2 }
+Function:        func1
+Args:
+  - count:           '3'
+  - String:          ' instructions with '
+  - type:            remark
+--- !Passed
+Pass:            generic-remarks-pass
+Name:            Remark
+DebugLoc:        { File: path/to/anno.c, Line: 1, Column: 2 }
+Function:        func2
+Args:
+  - count:           '3'
+  - String:          ' instructions with '
+  - type:            remark
+--- !Analysis
+Pass:            generic-remarks-pass2
+Name:            Remark
+DebugLoc:        { File: path/to/anno3.c, Line: 1, Column: 2 }
+Function:        func1
+Args:
+  - count:           '3'
+  - String:          ' instructions with '
+  - type:            remark
+--- !Analysis
+Pass:            generic-remarks-pass3
+Name:            Remark
+DebugLoc:        { File: path/to/anno.c, Line: 1, Column: 2 }
+Function:        func2
+Args:
+  - count:           '2'
+  - String:          ' instructions with '
+  - type:            remark
+--- !Analysis
+Pass:            generic-remarks-pass4
+Name:            Remark
+DebugLoc:        { File: path/to/anno2.c, Line: 1, Column: 2 }
+Function:        func3
+Args:
+  - count:           '2'
+  - String:          ' instructions with '
+  - type:            remark
diff --git a/llvm/test/tools/llvm-remarkutil/count/count-by-keys.test b/llvm/test/tools/llvm-remarkutil/count/count-by-keys.test
new file mode 100644
index 0000000000..c0dfbec501
--- /dev/null
+++ b/llvm/test/tools/llvm-remarkutil/count/count-by-keys.test
@@ -0,0 +1,20 @@
+RUN: llvm-remarkutil count --parser=yaml --count-by=arg --group-by=source %p/Inputs/remark-count-by.yaml | FileCheck %s
+RUN: llvm-remarkutil count --parser=yaml --count-by=arg --group-by=function %p/Inputs/remark-count-by.yaml | FileCheck %s --check-prefix=CHECKFUNC
+RUN: llvm-remarkutil count --parser=yaml --count-by=arg --group-by=function-with-loc %p/Inputs/remark-count-by.yaml | FileCheck %s --check-prefix=CHECKFUNCLOC
+RUN: llvm-remarkutil count --parser=yaml --count-by=arg --group-by=total %p/Inputs/remark-count-by.yaml | FileCheck %s --check-prefix=CHECKTOTAL
+
+; CHECK-LABEL: Source,count1,count2,count3,count4
+; CHECK: path/to/anno.c,3,4,6,4
+; CHECK: path/to/anno2.c,1,2,3,0
+
+; CHECKFUNC-LABEL: Function,count1,count2,count3,count4
+; CHECKFUNC: func1,3,4,6,4
+; CHECKFUNC: func2,1,2,3,0
+
+; CHECKFUNCLOC-LABEL: FuctionWithDebugLoc,count1,count2,count3,count4
+; CHECKFUNCLOC: path/to/anno.c:func1,2,2,3,4
+; CHECKFUNCLOC: path/to/anno.c:func2,1,2,3,0
+; CHECKFUNCLOC: path/to/anno2.c:func1,1,2,3,0
+
+; CHECKTOTAL-LABEL: Total,count1,count2,count3,count4
+; CHECKTOTAL: Total,4,6,9,4
diff --git a/llvm/test/tools/llvm-remarkutil/count/count-by-remark.test b/llvm/test/tools/llvm-remarkutil/count/count-by-remark.test
new file mode 100644
index 0000000000..607ef78669
--- /dev/null
+++ b/llvm/test/tools/llvm-remarkutil/count/count-by-remark.test
@@ -0,0 +1,20 @@
+RUN: llvm-remarkutil count --parser=yaml --count-by=remark-name --group-by=source %p/Inputs/remark-count-by.yaml | FileCheck %s
+RUN: llvm-remarkutil count --parser=yaml --count-by=remark-name --group-by=function %p/Inputs/remark-count-by.yaml | FileCheck %s --check-prefix=CHECKFUNC
+RUN: llvm-remarkutil count --parser=yaml --count-by=remark-name --group-by=function-with-loc %p/Inputs/remark-count-by.yaml | FileCheck %s --check-prefix=CHECKFUNCLOC
+RUN: llvm-remarkutil count --parser=yaml --count-by=remark-name --group-by=total %p/Inputs/remark-count-by.yaml | FileCheck %s --check-prefix=CHECKTOTAL
+
+; CHECK-LABEL: Source,Count
+; CHECK: path/to/anno.c,3
+; CHECK: path/to/anno2.c,1
+
+; CHECKFUNC-LABEL: Function,Count
+; CHECKFUNC: func1,3
+; CHECKFUNC: func2,1
+
+; CHECKFUNCLOC-LABEL: FuctionWithDebugLoc,Count
+; CHECKFUNCLOC: path/to/anno.c:func1,2
+; CHECKFUNCLOC: path/to/anno.c:func2,1
+; CHECKFUNCLOC: path/to/anno2.c:func1,1
+
+; CHECKTOTAL-LABEL: Total,Count
+; CHECKTOTAL: Total,4
diff --git a/llvm/test/tools/llvm-remarkutil/count/filter-by-pass-name.test b/llvm/test/tools/llvm-remarkutil/count/filter-by-pass-name.test
new file mode 100644
index 0000000000..caf630484c
--- /dev/null
+++ b/llvm/test/tools/llvm-remarkutil/count/filter-by-pass-name.test
@@ -0,0 +1,10 @@
+RUN: llvm-remarkutil count --parser=yaml --pass-name=generic-remarks-pass %p/Inputs/remark-filter-by.yaml | FileCheck %s
+RUN: llvm-remarkutil count --parser=yaml --rpass-name=.* %p/Inputs/remark-filter-by.yaml | FileCheck %s --check-prefix=CHECKALL
+
+; CHECK-LABEL: Source,Count
+; CHECK: path/to/anno.c,2
+; CHECK: path/to/anno2.c,1
+
+; CHECKALL-LABEL: Source,Count
+; CHECKALL: path/to/anno.c,2
+; CHECKALL: path/to/anno2.c,2
diff --git a/llvm/test/tools/llvm-remarkutil/count/filter-by-remark-name.test b/llvm/test/tools/llvm-remarkutil/count/filter-by-remark-name.test
new file mode 100644
index 0000000000..24d0399e36
--- /dev/null
+++ b/llvm/test/tools/llvm-remarkutil/count/filter-by-remark-name.test
@@ -0,0 +1,10 @@
+RUN: llvm-remarkutil count --parser=yaml --remark-name=Remark %p/Inputs/remark-filter-by.yaml | FileCheck %s
+RUN: llvm-remarkutil count --parser=yaml --rremark-name=R.* %p/Inputs/remark-filter-by.yaml | FileCheck %s --check-prefix=CHECKALL
+
+; CHECK-LABEL: Source,Count
+; CHECK: path/to/anno.c,1
+; CHECK: path/to/anno2.c,1
+
+; CHECKALL-LABEL: Source,Count
+; CHECKALL: path/to/anno.c,2
+; CHECKALL: path/to/anno2.c,2
diff --git a/llvm/test/tools/llvm-remarkutil/count/filter-by-type.test b/llvm/test/tools/llvm-remarkutil/count/filter-by-type.test
new file mode 100644
index 0000000000..db4f4610ae
--- /dev/null
+++ b/llvm/test/tools/llvm-remarkutil/count/filter-by-type.test
@@ -0,0 +1,16 @@
+RUN: llvm-remarkutil count --parser=yaml --remark-type=missed %p/Inputs/remark-filter-by.yaml | FileCheck %s --check-prefix=MISSED
+RUN: llvm-remarkutil count --parser=yaml --remark-type=passed %p/Inputs/remark-filter-by.yaml | FileCheck %s --check-prefix=PASSED
+RUN: llvm-remarkutil count --parser=yaml --remark-type=analysis %p/Inputs/remark-filter-by.yaml | FileCheck %s --check-prefix=ANALYSIS
+RUN: llvm-remarkutil count --parser=yaml --remark-type=unknown %p/Inputs/remark-filter-by.yaml | FileCheck %s --check-prefix=UNKNOWN
+
+; MISSED-LABEL: Source,Count
+; MISSED: path/to/anno.c,1
+
+; PASSED-LABEL: Source,Count
+; PASSED: path/to/anno.c,1
+
+; ANALYSIS-LABEL: Source,Count
+; ANALYSIS: path/to/anno2.c,2
+
+; UNKNOWN: Source,Count
+; UNKNOWN-EMPTY:
diff --git a/llvm/test/tools/llvm-remarkutil/count/group-by-function-with-loc.test b/llvm/test/tools/llvm-remarkutil/count/group-by-function-with-loc.test
new file mode 100644
index 0000000000..2ceb45b38f
--- /dev/null
+++ b/llvm/test/tools/llvm-remarkutil/count/group-by-function-with-loc.test
@@ -0,0 +1,7 @@
+RUN: llvm-remarkutil count --parser=yaml --group-by=function-with-loc %p/Inputs/remark-group-by.yaml | FileCheck %s
+
+; CHECK-LABEL: FuctionWithDebugLoc,Count
+; CHECK: path/to/anno.c:func1,2
+; CHECK: path/to/anno.c:func2,2
+; CHECK: path/to/anno2.c:func3,1
+; CHECK: path/to/anno3.c:func1,1
diff --git a/llvm/test/tools/llvm-remarkutil/count/group-by-function.test b/llvm/test/tools/llvm-remarkutil/count/group-by-function.test
new file mode 100644
index 0000000000..67643089d4
--- /dev/null
+++ b/llvm/test/tools/llvm-remarkutil/count/group-by-function.test
@@ -0,0 +1,7 @@
+
+RUN: llvm-remarkutil count --parser=yaml --group-by=function %p/Inputs/remark-group-by.yaml | FileCheck %s
+
+; CHECK-LABEL: Function,Count
+; CHECK: func1,3
+; CHECK: func2,2
+; CHECK: func3,1
diff --git a/llvm/test/tools/llvm-remarkutil/count/group-by-source.test b/llvm/test/tools/llvm-remarkutil/count/group-by-source.test
new file mode 100644
index 0000000000..0a11ac6392
--- /dev/null
+++ b/llvm/test/tools/llvm-remarkutil/count/group-by-source.test
@@ -0,0 +1,6 @@
+RUN: llvm-remarkutil count --parser=yaml --group-by=source %p/Inputs/remark-group-by.yaml | FileCheck %s
+
+; CHECK-LABEL: Source,Count
+; CHECK: path/to/anno.c,4
+; CHECK: path/to/anno2.c,1
+; CHECK: path/to/anno3.c,1
diff --git a/llvm/test/tools/llvm-remarkutil/empty-file.test b/llvm/test/tools/llvm-remarkutil/empty-file.test
index aa2ee41dd7..abbf8e02cf 100644
--- a/llvm/test/tools/llvm-remarkutil/empty-file.test
+++ b/llvm/test/tools/llvm-remarkutil/empty-file.test
@@ -1,9 +1,11 @@
 RUN: not llvm-remarkutil yaml2bitstream %p/Inputs/empty-file -o - 2>&1 | FileCheck %s --check-prefix=YAMLPARSER
 RUN: not llvm-remarkutil instruction-count --parser=yaml %p/Inputs/empty-file -o - 2>&1 | FileCheck %s --check-prefix=YAMLPARSER
 RUN: not llvm-remarkutil annotation-count --parser=yaml --annotation-type=remark %p/Inputs/empty-file -o - 2>&1 | FileCheck %s --check-prefix=YAMLPARSER
+RUN: not llvm-remarkutil count --parser=yaml %p/Inputs/empty-file -o - 2>&1 | FileCheck %s --check-prefix=YAMLPARSER
 RUN: llvm-remarkutil bitstream2yaml %p/Inputs/empty-file -o - 2>&1 | FileCheck %s --allow-empty --check-prefix=BITSTREAM2YAML
 RUN: llvm-remarkutil instruction-count --parser=bitstream %p/Inputs/empty-file -o - 2>&1 | FileCheck %s --allow-empty --check-prefix=SIZEBITSTREAM
 RUN: llvm-remarkutil annotation-count --parser=bitstream --annotation-type=remark %p/Inputs/empty-file -o - 2>&1 | FileCheck %s --allow-empty --check-prefix=ANNOTATIONBITSTREAM
+RUN: llvm-remarkutil count --parser=bitstream %p/Inputs/empty-file -o - 2>&1 | FileCheck %s --allow-empty --check-prefix=COUNTBITSTREAM
 
 ; YAMLPARSER: error: document root is not of mapping type.
 
@@ -15,3 +17,6 @@ RUN: llvm-remarkutil annotation-count --parser=bitstream --annotation-type=remar
 
 ; ANNOTATIONBITSTREAM-LABEL: Function,Count
 ; ANNOTATIONBITSTREAM-EMPTY: 
+
+; COUNTBITSTREAM-LABEL: Source,Count
+; COUNTBITSTREAM-EMPTY:
diff --git a/llvm/test/tools/llvm-remarkutil/instruction-count-with-dbg-loc.test b/llvm/test/tools/llvm-remarkutil/instruction-count-with-dbg-loc.test
index ac6c8256fe..f1efd44d68 100644
--- a/llvm/test/tools/llvm-remarkutil/instruction-count-with-dbg-loc.test
+++ b/llvm/test/tools/llvm-remarkutil/instruction-count-with-dbg-loc.test
@@ -1,7 +1,14 @@
 RUN: llvm-remarkutil instruction-count --use-debug-loc --parser=yaml %p/Inputs/instruction-count-with-dbg-loc.yaml | FileCheck %s
 RUN: llvm-remarkutil yaml2bitstream %p/Inputs/instruction-count-with-dbg-loc.yaml | llvm-remarkutil instruction-count --use-debug-loc --parser=bitstream | FileCheck %s
+RUN: llvm-remarkutil count --parser=yaml --count-by=arg --group-by=function-with-loc --remark-name="InstructionCount" %p/Inputs/instruction-count-with-dbg-loc.yaml | FileCheck %s --check-prefix=COUNT-CHECK
+RUN: llvm-remarkutil yaml2bitstream %p/Inputs/instruction-count-with-dbg-loc.yaml  | llvm-remarkutil count --parser=bitstream --count-by=arg --group-by=function-with-loc --remark-name="InstructionCount" | FileCheck %s --check-prefix=COUNT-CHECK
 
 ; CHECK-LABEL: Source,Function,InstructionCount
 ; CHECK: path/to/inst.c:1:2,func1,1
 ; CHECK: path/to/inst2.c:1:2,func2,2
 ; CHECK: path/to/inst3.c:1:2,func3,3
+
+; COUNT-CHECK-LABEL: FuctionWithDebugLoc,NumInstructions
+; COUNT-CHECK: path/to/inst.c:func1,1
+; COUNT-CHECK: path/to/inst2.c:func2,2
+; COUNT-CHECK: path/to/inst3.c:func3,3
diff --git a/llvm/test/tools/llvm-remarkutil/instruction-count.test b/llvm/test/tools/llvm-remarkutil/instruction-count.test
index d5f62e36c1..c4bc3e5780 100644
--- a/llvm/test/tools/llvm-remarkutil/instruction-count.test
+++ b/llvm/test/tools/llvm-remarkutil/instruction-count.test
@@ -1,7 +1,14 @@
 RUN: llvm-remarkutil instruction-count --parser=yaml %p/Inputs/instruction-count.yaml | FileCheck %s
 RUN: llvm-remarkutil yaml2bitstream %p/Inputs/instruction-count.yaml | llvm-remarkutil instruction-count --parser=bitstream | FileCheck %s
+RUN: llvm-remarkutil count --parser=yaml --count-by=arg --group-by=function --remark-name="InstructionCount" %p/Inputs/instruction-count.yaml | FileCheck %s --check-prefix=COUNT-CHECK
+RUN: llvm-remarkutil yaml2bitstream %p/Inputs/instruction-count.yaml  | llvm-remarkutil count --parser=bitstream --count-by=arg --group-by=function --remark-name="InstructionCount" | FileCheck %s --check-prefix=COUNT-CHECK
 
 ; CHECK-LABEL: Function,InstructionCount
 ; CHECK: func1,1
 ; CHECK: func2,2
 ; CHECK: func3,3
+
+; COUNT-CHECK-LABEL: Function,NumInstructions
+; COUNT-CHECK: func1,1
+; COUNT-CHECK: func2,2
+; COUNT-CHECK: func3,3
diff --git a/llvm/test/tools/llvm-symbolizer/xcoff-sd-symbol.ll b/llvm/test/tools/llvm-symbolizer/xcoff-sd-symbol.ll
new file mode 100644
index 0000000000..781ac72933
--- /dev/null
+++ b/llvm/test/tools/llvm-symbolizer/xcoff-sd-symbol.ll
@@ -0,0 +1,25 @@
+;; FIXME: use assembly rather than LLVM IR once integrated assembler supports
+;; AIX assembly syntax.
+
+; REQUIRES: powerpc-registered-target
+; RUN: llc -filetype=obj -o %t -mtriple=powerpc-aix-ibm-xcoff -function-sections < %s
+; RUN: llvm-symbolizer --obj=%t 'CODE 0x0' 'CODE 0x20' | \
+; RUN:   FileCheck %s
+
+define void @foo() {
+entry:
+  ret void
+}
+
+define void @foo1() {
+entry:
+  ret void
+}
+
+; CHECK: ??
+; CHECK: ??:0:0
+; CHECK-EMPTY:
+
+; CHECK: ??
+; CHECK: ??:0:0
+; CHECK-EMPTY:
diff --git a/llvm/tools/dsymutil/DebugMap.cpp b/llvm/tools/dsymutil/DebugMap.cpp
index 5ae7217711..d4e2c2b2cf 100644
--- a/llvm/tools/dsymutil/DebugMap.cpp
+++ b/llvm/tools/dsymutil/DebugMap.cpp
@@ -11,7 +11,6 @@
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
-#include "llvm/ADT/iterator_range.h"
 #include "llvm/BinaryFormat/MachO.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/Chrono.h"
diff --git a/llvm/tools/dsymutil/DwarfLinkerForBinary.cpp b/llvm/tools/dsymutil/DwarfLinkerForBinary.cpp
index 9057a2d640..39776ae5a9 100644
--- a/llvm/tools/dsymutil/DwarfLinkerForBinary.cpp
+++ b/llvm/tools/dsymutil/DwarfLinkerForBinary.cpp
@@ -21,7 +21,6 @@
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallString.h"
-#include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/BinaryFormat/Dwarf.h"
diff --git a/llvm/tools/llc/llc.cpp b/llvm/tools/llc/llc.cpp
index 0ca06cda20..0b174afc22 100644
--- a/llvm/tools/llc/llc.cpp
+++ b/llvm/tools/llc/llc.cpp
@@ -558,12 +558,6 @@ static int compileModule(char **argv, LLVMContext &Context) {
         exit(1);
       }
 
-      // On AIX, setting the relocation model to anything other than PIC is
-      // considered a user error.
-      if (TheTriple.isOSAIX() && RM && *RM != Reloc::PIC_)
-        reportError("invalid relocation model, AIX only supports PIC",
-                    InputFilename);
-
       InitializeOptions(TheTriple);
       Target = std::unique_ptr<TargetMachine>(TheTarget->createTargetMachine(
           TheTriple.getTriple(), CPUStr, FeaturesStr, Options, RM, CM, OLvl));
@@ -607,14 +601,6 @@ static int compileModule(char **argv, LLVMContext &Context) {
       return 1;
     }
 
-    // On AIX, setting the relocation model to anything other than PIC is
-    // considered a user error.
-    if (TheTriple.isOSAIX() && RM && *RM != Reloc::PIC_) {
-      WithColor::error(errs(), argv[0])
-          << "invalid relocation model, AIX only supports PIC.\n";
-      return 1;
-    }
-
     InitializeOptions(TheTriple);
     Target = std::unique_ptr<TargetMachine>(TheTarget->createTargetMachine(
         TheTriple.getTriple(), CPUStr, FeaturesStr, Options, RM, CM, OLvl));
diff --git a/llvm/tools/llvm-dwarfutil/DebugInfoLinker.cpp b/llvm/tools/llvm-dwarfutil/DebugInfoLinker.cpp
index f52e9092f5..d97dd7392b 100644
--- a/llvm/tools/llvm-dwarfutil/DebugInfoLinker.cpp
+++ b/llvm/tools/llvm-dwarfutil/DebugInfoLinker.cpp
@@ -15,7 +15,6 @@
 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
 #include "llvm/DebugInfo/DWARF/DWARFExpression.h"
 #include "llvm/Object/ObjectFile.h"
-#include "llvm/Support/Endian.h"
 #include <memory>
 #include <vector>
 
diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkResult.cpp b/llvm/tools/llvm-exegesis/lib/BenchmarkResult.cpp
index b8e53de57b..02c4da11e0 100644
--- a/llvm/tools/llvm-exegesis/lib/BenchmarkResult.cpp
+++ b/llvm/tools/llvm-exegesis/lib/BenchmarkResult.cpp
@@ -11,7 +11,6 @@
 #include "Error.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/ScopeExit.h"
-#include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/bit.h"
 #include "llvm/ObjectYAML/YAML.h"
diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkResult.h b/llvm/tools/llvm-exegesis/lib/BenchmarkResult.h
index 77e0994fe0..8a7faa0176 100644
--- a/llvm/tools/llvm-exegesis/lib/BenchmarkResult.h
+++ b/llvm/tools/llvm-exegesis/lib/BenchmarkResult.h
@@ -17,7 +17,6 @@
 
 #include "LlvmState.h"
 #include "RegisterValue.h"
-#include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/MC/MCInst.h"
diff --git a/llvm/tools/llvm-exegesis/lib/LlvmState.h b/llvm/tools/llvm-exegesis/lib/LlvmState.h
index 137ba1b5a5..16f0def518 100644
--- a/llvm/tools/llvm-exegesis/lib/LlvmState.h
+++ b/llvm/tools/llvm-exegesis/lib/LlvmState.h
@@ -16,7 +16,6 @@
 
 #include "MCInstrDescView.h"
 #include "RegisterAliasing.h"
-#include "llvm/ADT/StringMap.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstrInfo.h"
diff --git a/llvm/tools/llvm-exegesis/lib/X86/X86Counter.cpp b/llvm/tools/llvm-exegesis/lib/X86/X86Counter.cpp
index 3fee0e5d52..423c45e22b 100644
--- a/llvm/tools/llvm-exegesis/lib/X86/X86Counter.cpp
+++ b/llvm/tools/llvm-exegesis/lib/X86/X86Counter.cpp
@@ -87,7 +87,8 @@ static llvm::Error parseDataBuffer(const char *DataBuf, size_t DataSize,
       continue;
     }
     DataPtr += sizeof(Header);
-    uint64_t Count = llvm::support::endian::read64(DataPtr, support::native);
+    uint64_t Count =
+        llvm::support::endian::read64(DataPtr, llvm::endianness::native);
     DataPtr += sizeof(Count);
 
     struct perf_branch_entry Entry;
diff --git a/llvm/tools/llvm-gsymutil/llvm-gsymutil.cpp b/llvm/tools/llvm-gsymutil/llvm-gsymutil.cpp
index dc351c60e8..46ec4bdc28 100644
--- a/llvm/tools/llvm-gsymutil/llvm-gsymutil.cpp
+++ b/llvm/tools/llvm-gsymutil/llvm-gsymutil.cpp
@@ -370,8 +370,9 @@ static llvm::Error handleObjectFile(ObjectFile &Obj,
     return Err;
 
   // Save the GSYM file to disk.
-  support::endianness Endian =
-      Obj.makeTriple().isLittleEndian() ? support::little : support::big;
+  llvm::endianness Endian = Obj.makeTriple().isLittleEndian()
+                                ? llvm::endianness::little
+                                : llvm::endianness::big;
 
   std::optional<uint64_t> OptSegmentSize;
   if (SegmentSize > 0)
diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
index 0709f292a4..d73d247599 100644
--- a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
+++ b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
@@ -1903,7 +1903,8 @@ static Error runChecks(Session &S, Triple TT, SubtargetFeatures Features) {
 
   RuntimeDyldChecker Checker(
       IsSymbolValid, GetSymbolInfo, GetSectionInfo, GetStubInfo, GetGOTInfo,
-      S.ES.getTargetTriple().isLittleEndian() ? support::little : support::big,
+      S.ES.getTargetTriple().isLittleEndian() ? llvm::endianness::little
+                                              : llvm::endianness::big,
       TT, StringRef(), Features, dbgs());
 
   std::string CheckLineStart = "# " + CheckName + ":";
diff --git a/llvm/tools/llvm-objdump/MachODump.cpp b/llvm/tools/llvm-objdump/MachODump.cpp
index 0e4aa25005..f80a12d2f8 100644
--- a/llvm/tools/llvm-objdump/MachODump.cpp
+++ b/llvm/tools/llvm-objdump/MachODump.cpp
@@ -7629,16 +7629,13 @@ namespace {
 
 template <typename T>
 static uint64_t read(StringRef Contents, ptrdiff_t Offset) {
-  using llvm::support::little;
-  using llvm::support::unaligned;
-
   if (Offset + sizeof(T) > Contents.size()) {
     outs() << "warning: attempt to read past end of buffer\n";
     return T();
   }
 
-  uint64_t Val =
-      support::endian::read<T, little, unaligned>(Contents.data() + Offset);
+  uint64_t Val = support::endian::read<T, llvm::endianness::little>(
+      Contents.data() + Offset);
   return Val;
 }
 
diff --git a/llvm/tools/llvm-objdump/llvm-objdump.cpp b/llvm/tools/llvm-objdump/llvm-objdump.cpp
index d3edc85e46..a112c50bf7 100644
--- a/llvm/tools/llvm-objdump/llvm-objdump.cpp
+++ b/llvm/tools/llvm-objdump/llvm-objdump.cpp
@@ -685,8 +685,9 @@ class AMDGCNPrettyPrinter : public PrettyPrinter {
       // using the .long directive, or .byte directive if fewer than 4 bytes
       // remaining
       if (Bytes.size() >= 4) {
-        OS << format("\t.long 0x%08" PRIx32 " ",
-                     support::endian::read32<support::little>(Bytes.data()));
+        OS << format(
+            "\t.long 0x%08" PRIx32 " ",
+            support::endian::read32<llvm::endianness::little>(Bytes.data()));
         OS.indent(42);
       } else {
           OS << format("\t.byte 0x%02" PRIx8, Bytes[0]);
@@ -788,7 +789,7 @@ class ARMPrettyPrinter : public PrettyPrinter {
   }
 
 private:
-  llvm::endianness InstructionEndianness = llvm::support::little;
+  llvm::endianness InstructionEndianness = llvm::endianness::little;
 };
 ARMPrettyPrinter ARMPrettyPrinterInst;
 
@@ -811,8 +812,8 @@ class AArch64PrettyPrinter : public PrettyPrinter {
       for (; Pos + 4 <= End; Pos += 4)
         OS << ' '
            << format_hex_no_prefix(
-                  llvm::support::endian::read<uint32_t>(Bytes.data() + Pos,
-                                                        llvm::support::little),
+                  llvm::support::endian::read<uint32_t>(
+                      Bytes.data() + Pos, llvm::endianness::little),
                   8);
       if (Pos < End) {
         OS << ' ';
@@ -859,7 +860,7 @@ class DisassemblerTarget {
   std::unique_ptr<const MCSubtargetInfo> SubtargetInfo;
   std::shared_ptr<MCContext> Context;
   std::unique_ptr<MCDisassembler> DisAsm;
-  std::shared_ptr<const MCInstrAnalysis> InstrAnalysis;
+  std::shared_ptr<MCInstrAnalysis> InstrAnalysis;
   std::shared_ptr<MCInstPrinter> InstPrinter;
   PrettyPrinter *Printer;
 
@@ -1167,8 +1168,8 @@ static uint64_t dumpARMELFData(uint64_t SectionAddr, uint64_t Index,
                                ArrayRef<uint8_t> Bytes,
                                ArrayRef<MappingSymbolPair> MappingSymbols,
                                const MCSubtargetInfo &STI, raw_ostream &OS) {
-  support::endianness Endian =
-      Obj.isLittleEndian() ? support::little : support::big;
+  llvm::endianness Endian =
+      Obj.isLittleEndian() ? llvm::endianness::little : llvm::endianness::big;
   size_t Start = OS.tell();
   OS << format("%8" PRIx64 ": ", SectionAddr + Index);
   if (Index + 4 <= End) {
@@ -1282,14 +1283,19 @@ collectBBAddrMapLabels(const std::unordered_map<uint64_t, BBAddrMap> &AddrToBBAd
   }
 }
 
-static void collectLocalBranchTargets(
-    ArrayRef<uint8_t> Bytes, const MCInstrAnalysis *MIA, MCDisassembler *DisAsm,
-    MCInstPrinter *IP, const MCSubtargetInfo *STI, uint64_t SectionAddr,
-    uint64_t Start, uint64_t End, std::unordered_map<uint64_t, std::string> &Labels) {
+static void
+collectLocalBranchTargets(ArrayRef<uint8_t> Bytes, MCInstrAnalysis *MIA,
+                          MCDisassembler *DisAsm, MCInstPrinter *IP,
+                          const MCSubtargetInfo *STI, uint64_t SectionAddr,
+                          uint64_t Start, uint64_t End,
+                          std::unordered_map<uint64_t, std::string> &Labels) {
   // So far only supports PowerPC and X86.
   if (!STI->getTargetTriple().isPPC() && !STI->getTargetTriple().isX86())
     return;
 
+  if (MIA)
+    MIA->resetState();
+
   Labels.clear();
   unsigned LabelCount = 0;
   Start += SectionAddr;
@@ -1315,6 +1321,9 @@ static void collectLocalBranchTargets(
           !Labels.count(Target) &&
           !(STI->getTargetTriple().isPPC() && Target == Index))
         Labels[Target] = ("L" + Twine(LabelCount++)).str();
+      MIA->updateState(Inst, Index);
+    } else if (!Disassembled && MIA) {
+      MIA->resetState();
     }
     Index += Size;
   }
@@ -1966,6 +1975,9 @@ disassembleObject(ObjectFile &Obj, const ObjectFile &DbgObj,
                                BBAddrMapLabels);
       }
 
+      if (DT->InstrAnalysis)
+        DT->InstrAnalysis->resetState();
+
       while (Index < End) {
         // ARM and AArch64 ELF binaries can interleave data and text in the
         // same section. We rely on the markers introduced to understand what
@@ -2182,6 +2194,10 @@ disassembleObject(ObjectFile &Obj, const ObjectFile &DbgObj,
               if (TargetOS == &CommentStream)
                 *TargetOS << "\n";
             }
+
+            DT->InstrAnalysis->updateState(Inst, SectionAddr + Index);
+          } else if (!Disassembled && DT->InstrAnalysis) {
+            DT->InstrAnalysis->resetState();
           }
         }
 
@@ -2289,9 +2305,9 @@ static void disassembleObject(ObjectFile *Obj, bool InlineRelocs) {
     if (Elf32BE && (Elf32BE->isRelocatableObject() ||
                     !(Elf32BE->getPlatformFlags() & ELF::EF_ARM_BE8))) {
       Features.AddFeature("+big-endian-instructions");
-      ARMPrettyPrinterInst.setInstructionEndianness(llvm::support::big);
+      ARMPrettyPrinterInst.setInstructionEndianness(llvm::endianness::big);
     } else {
-      ARMPrettyPrinterInst.setInstructionEndianness(llvm::support::little);
+      ARMPrettyPrinterInst.setInstructionEndianness(llvm::endianness::little);
     }
   }
 
diff --git a/llvm/tools/llvm-pdbutil/DumpOutputStyle.cpp b/llvm/tools/llvm-pdbutil/DumpOutputStyle.cpp
index 447a9cb6b3..5ce663e2ef 100644
--- a/llvm/tools/llvm-pdbutil/DumpOutputStyle.cpp
+++ b/llvm/tools/llvm-pdbutil/DumpOutputStyle.cpp
@@ -1286,7 +1286,7 @@ Error DumpOutputStyle::dumpTypesFromObjectFile() {
       return ContentsOrErr.takeError();
 
     uint32_t Magic;
-    BinaryStreamReader Reader(*ContentsOrErr, llvm::support::little);
+    BinaryStreamReader Reader(*ContentsOrErr, llvm::endianness::little);
     if (auto EC = Reader.readInteger(Magic))
       return EC;
     if (Magic != COFF::DEBUG_SECTION_MAGIC)
diff --git a/llvm/tools/llvm-pdbutil/ExplainOutputStyle.cpp b/llvm/tools/llvm-pdbutil/ExplainOutputStyle.cpp
index 3e57e318d7..ce9d659335 100644
--- a/llvm/tools/llvm-pdbutil/ExplainOutputStyle.cpp
+++ b/llvm/tools/llvm-pdbutil/ExplainOutputStyle.cpp
@@ -66,9 +66,8 @@ Error ExplainOutputStyle::explainPdbFile() {
 }
 
 Error ExplainOutputStyle::explainBinaryFile() {
-  std::unique_ptr<BinaryByteStream> Stream =
-      std::make_unique<BinaryByteStream>(File.unknown().getBuffer(),
-                                          llvm::support::little);
+  std::unique_ptr<BinaryByteStream> Stream = std::make_unique<BinaryByteStream>(
+      File.unknown().getBuffer(), llvm::endianness::little);
   switch (opts::explain::InputType) {
   case opts::explain::InputFileType::DBIStream: {
     DbiStream Dbi(std::move(Stream));
diff --git a/llvm/tools/llvm-pdbutil/llvm-pdbutil.cpp b/llvm/tools/llvm-pdbutil/llvm-pdbutil.cpp
index 10839383cb..fa870666fe 100644
--- a/llvm/tools/llvm-pdbutil/llvm-pdbutil.cpp
+++ b/llvm/tools/llvm-pdbutil/llvm-pdbutil.cpp
@@ -1415,7 +1415,7 @@ static void exportStream() {
   SourceStream = File.createIndexedStream(Index);
   auto OutFile = ExitOnErr(
       FileOutputBuffer::create(OutFileName, SourceStream->getLength()));
-  FileBufferByteStream DestStream(std::move(OutFile), llvm::support::little);
+  FileBufferByteStream DestStream(std::move(OutFile), llvm::endianness::little);
   BinaryStreamWriter Writer(DestStream);
   ExitOnErr(Writer.writeStreamRef(*SourceStream));
   ExitOnErr(DestStream.commit());
diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp
index 18307935c7..7d665a8005 100644
--- a/llvm/tools/llvm-profdata/llvm-profdata.cpp
+++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp
@@ -199,6 +199,14 @@ class SymbolRemapper {
     StringRef New = RemappingTable.lookup(Name);
     return New.empty() ? Name : New;
   }
+
+  FunctionId operator()(FunctionId Name) {
+    // MD5 name cannot be remapped.
+    if (!Name.isStringRef())
+      return Name;
+    StringRef New = RemappingTable.lookup(Name.stringRef());
+    return New.empty() ? Name : FunctionId(New);
+  }
 };
 }
 
@@ -709,14 +717,15 @@ adjustInstrProfile(std::unique_ptr<WriterContext> &WC,
   //
   // Note that goo's count will remain in bar.cc:bar() as it does not have an
   // entry in InstrProfile.
-  DenseMap<StringRef, std::pair<uint64_t, uint64_t>> FlattenSampleMap;
+  llvm::StringMap<std::pair<uint64_t, uint64_t>> FlattenSampleMap;
   auto BuildMaxSampleMap = [&FlattenSampleMap, &StaticFuncMap,
                             &InstrProfileMap](const FunctionSamples &FS,
                                               const StringRef &RootName) {
     auto BuildMaxSampleMapImpl = [&](const FunctionSamples &FS,
                                      const StringRef &RootName,
                                      auto &BuildImpl) -> void {
-      const StringRef &Name = FS.getName();
+      std::string NameStr = FS.getFunction().str();
+      const StringRef Name = NameStr;
       const StringRef *NewRootName = &RootName;
       uint64_t EntrySample = FS.getHeadSamplesEstimate();
       uint64_t MaxBodySample = FS.getMaxCountInside(/* SkipCallSite*/ true);
@@ -770,7 +779,8 @@ adjustInstrProfile(std::unique_ptr<WriterContext> &WC,
 
   for (auto &PD : Reader->getProfiles()) {
     sampleprof::FunctionSamples &FS = PD.second;
-    BuildMaxSampleMap(FS, FS.getName());
+    std::string Name = FS.getFunction().str();
+    BuildMaxSampleMap(FS, Name);
   }
 
   ProfileSummary InstrPS = *IPBuilder.getSummary();
@@ -806,7 +816,7 @@ adjustInstrProfile(std::unique_ptr<WriterContext> &WC,
     uint64_t SampleMaxCount = std::max(E.second.first, E.second.second);
     if (SampleMaxCount < ColdSampleThreshold)
       continue;
-    const StringRef &Name = E.first;
+    StringRef Name = E.first();
     auto It = InstrProfileMap.find(Name);
     if (It == InstrProfileMap.end()) {
       auto NewName = StaticFuncMap.find(Name);
@@ -885,7 +895,7 @@ static sampleprof::FunctionSamples
 remapSamples(const sampleprof::FunctionSamples &Samples,
              SymbolRemapper &Remapper, sampleprof_error &Error) {
   sampleprof::FunctionSamples Result;
-  Result.setName(Remapper(Samples.getName()));
+  Result.setFunction(Remapper(Samples.getFunction()));
   Result.addTotalSamples(Samples.getTotalSamples());
   Result.addHeadSamples(Samples.getHeadSamples());
   for (const auto &BodySample : Samples.getBodySamples()) {
@@ -896,7 +906,7 @@ remapSamples(const sampleprof::FunctionSamples &Samples,
     for (const auto &Target : BodySample.second.getCallTargets()) {
       Result.addCalledTargetSamples(BodySample.first.LineOffset,
                                     MaskedDiscriminator,
-                                    Remapper(Target.first()), Target.second);
+                                    Remapper(Target.first), Target.second);
     }
   }
   for (const auto &CallsiteSamples : Samples.getCallsiteSamples()) {
@@ -905,8 +915,7 @@ remapSamples(const sampleprof::FunctionSamples &Samples,
     for (const auto &Callsite : CallsiteSamples.second) {
       sampleprof::FunctionSamples Remapped =
           remapSamples(Callsite.second, Remapper, Error);
-      MergeResult(Error,
-                  Target[std::string(Remapped.getName())].merge(Remapped));
+      MergeResult(Error, Target[Remapped.getFunction()].merge(Remapped));
     }
   }
   return Result;
diff --git a/llvm/tools/llvm-profgen/CSPreInliner.cpp b/llvm/tools/llvm-profgen/CSPreInliner.cpp
index e891ea8df4..025d3ca5a6 100644
--- a/llvm/tools/llvm-profgen/CSPreInliner.cpp
+++ b/llvm/tools/llvm-profgen/CSPreInliner.cpp
@@ -74,8 +74,8 @@ CSPreInliner::CSPreInliner(SampleContextTracker &Tracker,
     ProfileInlineLimitMax = 50000;
 }
 
-std::vector<StringRef> CSPreInliner::buildTopDownOrder() {
-  std::vector<StringRef> Order;
+std::vector<FunctionId> CSPreInliner::buildTopDownOrder() {
+  std::vector<FunctionId> Order;
   // Trim cold edges to get a more stable call graph. This allows for a more
   // stable top-down order which in turns helps the stablity of the generated
   // profile from run to run.
@@ -129,7 +129,7 @@ bool CSPreInliner::getInlineCandidates(ProfiledCandidateQueue &CQueue,
     LineLocation Callsite = CalleeNode->getCallSiteLoc();
     if (auto CallTargets = CallerSamples->findCallTargetMapAt(Callsite)) {
       SampleRecord::CallTargetMap &TargetCounts = CallTargets.get();
-      auto It = TargetCounts.find(CalleeSamples->getName());
+      auto It = TargetCounts.find(CalleeSamples->getFunction());
       if (It != TargetCounts.end())
         CallsiteCount = It->second;
     }
@@ -196,7 +196,7 @@ bool CSPreInliner::shouldInline(ProfiledInlineCandidate &Candidate) {
   return (Candidate.SizeCost < SampleThreshold);
 }
 
-void CSPreInliner::processFunction(const StringRef Name) {
+void CSPreInliner::processFunction(const FunctionId Name) {
   FunctionSamples *FSamples = ContextTracker.getBaseSamplesFor(Name);
   if (!FSamples)
     return;
@@ -297,7 +297,7 @@ void CSPreInliner::run() {
   // It also helps better compress context profile to control profile
   // size, as we now only need context profile for functions going to
   // be inlined.
-  for (StringRef FuncName : buildTopDownOrder()) {
+  for (FunctionId FuncName : buildTopDownOrder()) {
     processFunction(FuncName);
   }
 
diff --git a/llvm/tools/llvm-profgen/CSPreInliner.h b/llvm/tools/llvm-profgen/CSPreInliner.h
index 4d848aafda..8a3f16a4f1 100644
--- a/llvm/tools/llvm-profgen/CSPreInliner.h
+++ b/llvm/tools/llvm-profgen/CSPreInliner.h
@@ -57,8 +57,7 @@ struct ProfiledCandidateComparer {
     // Tie breaker using GUID so we have stable/deterministic inlining order
     assert(LHS.CalleeSamples && RHS.CalleeSamples &&
            "Expect non-null FunctionSamples");
-    return LHS.CalleeSamples->getGUID(LHS.CalleeSamples->getName()) <
-           RHS.CalleeSamples->getGUID(RHS.CalleeSamples->getName());
+    return LHS.CalleeSamples->getGUID() < RHS.CalleeSamples->getGUID();
   }
 };
 
@@ -81,8 +80,8 @@ class CSPreInliner {
 private:
   bool getInlineCandidates(ProfiledCandidateQueue &CQueue,
                            const FunctionSamples *FCallerContextSamples);
-  std::vector<StringRef> buildTopDownOrder();
-  void processFunction(StringRef Name);
+  std::vector<FunctionId> buildTopDownOrder();
+  void processFunction(FunctionId Name);
   bool shouldInline(ProfiledInlineCandidate &Candidate);
   uint32_t getFuncSize(const ContextTrieNode *ContextNode);
   bool UseContextCost;
diff --git a/llvm/tools/llvm-profgen/CallContext.h b/llvm/tools/llvm-profgen/CallContext.h
index 5e552130d0..ce9423f118 100644
--- a/llvm/tools/llvm-profgen/CallContext.h
+++ b/llvm/tools/llvm-profgen/CallContext.h
@@ -18,7 +18,7 @@ namespace llvm {
 namespace sampleprof {
 
 inline std::string getCallSite(const SampleContextFrame &Callsite) {
-  std::string CallsiteStr = Callsite.FuncName.str();
+  std::string CallsiteStr = Callsite.Func.str();
   CallsiteStr += ":";
   CallsiteStr += Twine(Callsite.Location.LineOffset).str();
   if (Callsite.Location.Discriminator > 0) {
diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.cpp b/llvm/tools/llvm-profgen/ProfileGenerator.cpp
index d307ab4656..c4028e6b13 100644
--- a/llvm/tools/llvm-profgen/ProfileGenerator.cpp
+++ b/llvm/tools/llvm-profgen/ProfileGenerator.cpp
@@ -208,7 +208,7 @@ double ProfileGeneratorBase::calculateDensity(const SampleProfileMap &Profiles,
   }
 
   for (auto *FuncSamples : HotFuncs) {
-    auto *Func = Binary->getBinaryFunction(FuncSamples->getName());
+    auto *Func = Binary->getBinaryFunction(FuncSamples->getFunction());
     if (!Func)
       continue;
     uint64_t FuncSize = Func->getFuncSize();
@@ -449,7 +449,7 @@ bool ProfileGeneratorBase::collectFunctionsFromRawProfile(
 bool ProfileGenerator::collectFunctionsFromLLVMProfile(
     std::unordered_set<const BinaryFunction *> &ProfiledFunctions) {
   for (const auto &FS : ProfileMap) {
-    if (auto *Func = Binary->getBinaryFunction(FS.second.getName()))
+    if (auto *Func = Binary->getBinaryFunction(FS.second.getFunction()))
       ProfiledFunctions.insert(Func);
   }
   return true;
@@ -466,7 +466,7 @@ bool CSProfileGenerator::collectFunctionsFromLLVMProfile(
 }
 
 FunctionSamples &
-ProfileGenerator::getTopLevelFunctionProfile(StringRef FuncName) {
+ProfileGenerator::getTopLevelFunctionProfile(FunctionId FuncName) {
   SampleContext Context(FuncName);
   return ProfileMap.Create(Context);
 }
@@ -586,7 +586,7 @@ void ProfileGenerator::populateBoundarySamplesWithProbesForAllFunctions(
       FunctionProfile.addCalledTargetSamples(
           FrameVec.back().Location.LineOffset,
           FrameVec.back().Location.Discriminator,
-          CalleeName, Count);
+          FunctionId(CalleeName), Count);
     }
   }
 }
@@ -595,11 +595,11 @@ FunctionSamples &ProfileGenerator::getLeafProfileAndAddTotalSamples(
     const SampleContextFrameVector &FrameVec, uint64_t Count) {
   // Get top level profile
   FunctionSamples *FunctionProfile =
-      &getTopLevelFunctionProfile(FrameVec[0].FuncName);
+      &getTopLevelFunctionProfile(FrameVec[0].Func);
   FunctionProfile->addTotalSamples(Count);
   if (Binary->usePseudoProbes()) {
     const auto *FuncDesc = Binary->getFuncDescForGUID(
-        Function::getGUID(FunctionProfile->getName()));
+        FunctionProfile->getFunction().getHashCode());
     FunctionProfile->setFunctionHash(FuncDesc->FuncHash);
   }
 
@@ -610,16 +610,16 @@ FunctionSamples &ProfileGenerator::getLeafProfileAndAddTotalSamples(
     FunctionSamplesMap &SamplesMap =
         FunctionProfile->functionSamplesAt(Callsite);
     auto Ret =
-        SamplesMap.emplace(FrameVec[I].FuncName.str(), FunctionSamples());
+        SamplesMap.emplace(FrameVec[I].Func, FunctionSamples());
     if (Ret.second) {
-      SampleContext Context(FrameVec[I].FuncName);
+      SampleContext Context(FrameVec[I].Func);
       Ret.first->second.setContext(Context);
     }
     FunctionProfile = &Ret.first->second;
     FunctionProfile->addTotalSamples(Count);
     if (Binary->usePseudoProbes()) {
       const auto *FuncDesc = Binary->getFuncDescForGUID(
-          Function::getGUID(FunctionProfile->getName()));
+          FunctionProfile->getFunction().getHashCode());
       FunctionProfile->setFunctionHash(FuncDesc->FuncHash);
     }
   }
@@ -716,10 +716,11 @@ void ProfileGenerator::populateBoundarySamplesForAllFunctions(
       FunctionProfile.addCalledTargetSamples(
           FrameVec.back().Location.LineOffset,
           getBaseDiscriminator(FrameVec.back().Location.Discriminator),
-          CalleeName, Count);
+          FunctionId(CalleeName), Count);
     }
     // Add head samples for callee.
-    FunctionSamples &CalleeProfile = getTopLevelFunctionProfile(CalleeName);
+    FunctionSamples &CalleeProfile =
+        getTopLevelFunctionProfile(FunctionId(CalleeName));
     CalleeProfile.addHeadSamples(Count);
   }
 }
@@ -737,7 +738,7 @@ CSProfileGenerator::getOrCreateFunctionSamples(ContextTrieNode *ContextNode,
   if (!FProfile) {
     FSamplesList.emplace_back();
     FProfile = &FSamplesList.back();
-    FProfile->setName(ContextNode->getFuncName());
+    FProfile->setFunction(ContextNode->getFuncName());
     ContextNode->setFunctionSamples(FProfile);
   }
   // Update ContextWasInlined attribute for existing contexts.
@@ -899,7 +900,8 @@ void CSProfileGenerator::populateBoundarySamplesForFunction(
       if (LeafLoc) {
         CallerNode->getFunctionSamples()->addCalledTargetSamples(
             LeafLoc->Location.LineOffset,
-            getBaseDiscriminator(LeafLoc->Location.Discriminator), CalleeName,
+            getBaseDiscriminator(LeafLoc->Location.Discriminator),
+            FunctionId(CalleeName),
             Count);
         // Record head sample for called target(callee)
         CalleeCallSite = LeafLoc->Location;
@@ -907,7 +909,8 @@ void CSProfileGenerator::populateBoundarySamplesForFunction(
     }
 
     ContextTrieNode *CalleeNode =
-        CallerNode->getOrCreateChildContext(CalleeCallSite, CalleeName);
+        CallerNode->getOrCreateChildContext(CalleeCallSite,
+                                            FunctionId(CalleeName));
     FunctionSamples *CalleeProfile = getOrCreateFunctionSamples(CalleeNode);
     CalleeProfile->addHeadSamples(Count);
   }
@@ -1212,7 +1215,7 @@ void CSProfileGenerator::populateBoundarySamplesWithProbes(
       continue;
     FunctionProfile.addCalledTargetSamples(CallProbe->getIndex(),
                                            CallProbe->getDiscriminator(),
-                                           CalleeName, Count);
+                                           FunctionId(CalleeName), Count);
   }
 }
 
diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.h b/llvm/tools/llvm-profgen/ProfileGenerator.h
index 471792ec71..88cf2dc0d4 100644
--- a/llvm/tools/llvm-profgen/ProfileGenerator.h
+++ b/llvm/tools/llvm-profgen/ProfileGenerator.h
@@ -157,7 +157,7 @@ class ProfileGenerator : public ProfileGeneratorBase {
   void generateLineNumBasedProfile();
   void generateProbeBasedProfile();
   RangeSample preprocessRangeCounter(const RangeSample &RangeCounter);
-  FunctionSamples &getTopLevelFunctionProfile(StringRef FuncName);
+  FunctionSamples &getTopLevelFunctionProfile(FunctionId FuncName);
   // Helper function to get the leaf frame's FunctionProfile by traversing the
   // inline stack and meanwhile it adds the total samples for each frame's
   // function profile.
diff --git a/llvm/tools/llvm-profgen/ProfiledBinary.cpp b/llvm/tools/llvm-profgen/ProfiledBinary.cpp
index 85d7c1123f..b40c5f8cbb 100644
--- a/llvm/tools/llvm-profgen/ProfiledBinary.cpp
+++ b/llvm/tools/llvm-profgen/ProfiledBinary.cpp
@@ -77,7 +77,7 @@ void BinarySizeContextTracker::addInstructionForContext(
   ContextTrieNode *CurNode = &RootContext;
   bool IsLeaf = true;
   for (const auto &Callsite : reverse(Context)) {
-    StringRef CallerName = Callsite.FuncName;
+    FunctionId CallerName = Callsite.Func;
     LineLocation CallsiteLoc = IsLeaf ? LineLocation(0, 0) : Callsite.Location;
     CurNode = CurNode->getOrCreateChildContext(CallsiteLoc, CallerName);
     IsLeaf = false;
@@ -145,7 +145,8 @@ void BinarySizeContextTracker::trackInlineesOptimizedAway(
       StringRef CallerName = ProbeFrame.first;
       LineLocation CallsiteLoc(ProbeFrame.second, 0);
       SizeContext =
-          SizeContext->getOrCreateChildContext(CallsiteLoc, CallerName);
+          SizeContext->getOrCreateChildContext(CallsiteLoc,
+                                               FunctionId(CallerName));
     }
     // Add 0 size to make known.
     SizeContext->addFunctionSize(0);
@@ -479,12 +480,6 @@ bool ProfiledBinary::dissassembleSymbol(std::size_t SI, ArrayRef<uint8_t> Bytes,
   if (ShowDisassembly)
     outs() << '<' << SymbolName << ">:\n";
 
-  auto WarnInvalidInsts = [](uint64_t Start, uint64_t End) {
-    WithColor::warning() << "Invalid instructions at "
-                         << format("%8" PRIx64, Start) << " - "
-                         << format("%8" PRIx64, End) << "\n";
-  };
-
   uint64_t Address = StartAddress;
   // Size of a consecutive invalid instruction range starting from Address -1
   // backwards.
@@ -577,7 +572,8 @@ bool ProfiledBinary::dissassembleSymbol(std::size_t SI, ArrayRef<uint8_t> Bytes,
       }
 
       if (InvalidInstLength) {
-        WarnInvalidInsts(Address - InvalidInstLength, Address - 1);
+        AddrsWithInvalidInstruction.insert(
+            {Address - InvalidInstLength, Address - 1});
         InvalidInstLength = 0;
       }
     } else {
@@ -588,7 +584,8 @@ bool ProfiledBinary::dissassembleSymbol(std::size_t SI, ArrayRef<uint8_t> Bytes,
   }
 
   if (InvalidInstLength)
-    WarnInvalidInsts(Address - InvalidInstLength, Address - 1);
+    AddrsWithInvalidInstruction.insert(
+        {Address - InvalidInstLength, Address - 1});
 
   if (ShowDisassembly)
     outs() << "\n";
@@ -707,6 +704,19 @@ void ProfiledBinary::disassemble(const ELFObjectFileBase *Obj) {
     }
   }
 
+  if (!AddrsWithInvalidInstruction.empty()) {
+    if (ShowDetailedWarning) {
+      for (auto &Addr : AddrsWithInvalidInstruction) {
+        WithColor::warning()
+            << "Invalid instructions at " << format("%8" PRIx64, Addr.first)
+            << " - " << format("%8" PRIx64, Addr.second) << "\n";
+      }
+    }
+    WithColor::warning() << "Found " << AddrsWithInvalidInstruction.size()
+                         << " invalid instructions\n";
+    AddrsWithInvalidInstruction.clear();
+  }
+
   // Dissassemble rodata section to check if FS discriminator symbol exists.
   checkUseFSDiscriminator(Obj, AllSymbols);
 }
@@ -791,10 +801,12 @@ void ProfiledBinary::loadSymbolsFromDWARFUnit(DWARFUnit &CompilationUnit) {
         FRange.StartAddress = StartAddress;
         FRange.EndAddress = EndAddress;
       } else {
-        WithColor::warning()
-            << "Duplicated symbol start address at "
-            << format("%8" PRIx64, StartAddress) << " "
-            << R.first->second.getFuncName() << " and " << Name << "\n";
+        AddrsWithMultipleSymbols.insert(StartAddress);
+        if (ShowDetailedWarning)
+          WithColor::warning()
+              << "Duplicated symbol start address at "
+              << format("%8" PRIx64, StartAddress) << " "
+              << R.first->second.getFuncName() << " and " << Name << "\n";
       }
     }
   }
@@ -838,6 +850,18 @@ void ProfiledBinary::loadSymbolsFromDWARF(ObjectFile &Obj) {
   if (BinaryFunctions.empty())
     WithColor::warning() << "Loading of DWARF info completed, but no binary "
                             "functions have been retrieved.\n";
+  // Populate the hash binary function map for MD5 function name lookup. This
+  // is done after BinaryFunctions are finalized.
+  for (auto &BinaryFunction : BinaryFunctions) {
+    HashBinaryFunctions[MD5Hash(StringRef(BinaryFunction.first))] =
+        &BinaryFunction.second;
+  }
+
+  if (!AddrsWithMultipleSymbols.empty()) {
+    WithColor::warning() << "Found " << AddrsWithMultipleSymbols.size()
+                         << " start addresses with multiple symbols\n";
+    AddrsWithMultipleSymbols.clear();
+  }
 }
 
 void ProfiledBinary::populateSymbolListFromDWARF(
@@ -872,7 +896,8 @@ SampleContextFrameVector ProfiledBinary::symbolize(const InstructionPointer &IP,
   SampleContextFrameVector CallStack;
   for (int32_t I = InlineStack.getNumberOfFrames() - 1; I >= 0; I--) {
     const auto &CallerFrame = InlineStack.getFrame(I);
-    if (CallerFrame.FunctionName.empty() || (CallerFrame.FunctionName == "<invalid>"))
+    if (CallerFrame.FunctionName.empty() ||
+        (CallerFrame.FunctionName == "<invalid>"))
       break;
 
     StringRef FunctionName(CallerFrame.FunctionName);
@@ -889,7 +914,7 @@ SampleContextFrameVector ProfiledBinary::symbolize(const InstructionPointer &IP,
 
     LineLocation Line(LineOffset, Discriminator);
     auto It = NameStrings.insert(FunctionName.str());
-    CallStack.emplace_back(*It.first, Line);
+    CallStack.emplace_back(FunctionId(StringRef(*It.first)), Line);
   }
 
   return CallStack;
diff --git a/llvm/tools/llvm-profgen/ProfiledBinary.h b/llvm/tools/llvm-profgen/ProfiledBinary.h
index a6d78c661c..8f42ff11a3 100644
--- a/llvm/tools/llvm-profgen/ProfiledBinary.h
+++ b/llvm/tools/llvm-profgen/ProfiledBinary.h
@@ -220,12 +220,20 @@ class ProfiledBinary {
   // A map of mapping function name to BinaryFunction info.
   std::unordered_map<std::string, BinaryFunction> BinaryFunctions;
 
+  // Lookup BinaryFunctions using the function name's MD5 hash. Needed if the
+  // profile is using MD5.
+  std::unordered_map<uint64_t, BinaryFunction *> HashBinaryFunctions;
+
   // A list of binary functions that have samples.
   std::unordered_set<const BinaryFunction *> ProfiledFunctions;
 
   // GUID to Elf symbol start address map
   DenseMap<uint64_t, uint64_t> SymbolStartAddrs;
 
+  // These maps are for temporary use of warning diagnosis.
+  DenseSet<int64_t> AddrsWithMultipleSymbols;
+  DenseSet<std::pair<uint64_t, uint64_t>> AddrsWithInvalidInstruction;
+
   // Start address to Elf symbol GUID map
   std::unordered_multimap<uint64_t, uint64_t> StartAddrToSymMap;
 
@@ -476,12 +484,18 @@ class ProfiledBinary {
   void setProfiledFunctions(std::unordered_set<const BinaryFunction *> &Funcs) {
     ProfiledFunctions = Funcs;
   }
-
-  BinaryFunction *getBinaryFunction(StringRef FName) {
-    auto I = BinaryFunctions.find(FName.str());
-    if (I == BinaryFunctions.end())
+  
+  BinaryFunction *getBinaryFunction(FunctionId FName) {
+    if (FName.isStringRef()) {
+      auto I = BinaryFunctions.find(FName.str());
+      if (I == BinaryFunctions.end())
+        return nullptr;
+      return &I->second;
+    }
+    auto I = HashBinaryFunctions.find(FName.getHashCode());
+    if (I == HashBinaryFunctions.end())
       return nullptr;
-    return &I->second;
+    return I->second;
   }
 
   uint32_t getFuncSizeForContext(const ContextTrieNode *ContextNode) {
@@ -519,7 +533,7 @@ class ProfiledBinary {
 
   void flushSymbolizer() { Symbolizer.reset(); }
 
-  MissingFrameInferrer* getMissingContextInferrer() {
+  MissingFrameInferrer *getMissingContextInferrer() {
     return MissingContextInferrer.get();
   }
 
@@ -556,7 +570,7 @@ class ProfiledBinary {
         InlineContextStack.clear();
         continue;
       }
-      InlineContextStack.emplace_back(Callsite.first,
+      InlineContextStack.emplace_back(FunctionId(Callsite.first),
                                       LineLocation(Callsite.second, 0));
     }
   }
diff --git a/llvm/tools/llvm-rc/ResourceFileWriter.cpp b/llvm/tools/llvm-rc/ResourceFileWriter.cpp
index dd9db338ec..9738fd4934 100644
--- a/llvm/tools/llvm-rc/ResourceFileWriter.cpp
+++ b/llvm/tools/llvm-rc/ResourceFileWriter.cpp
@@ -891,7 +891,7 @@ Error ResourceFileWriter::visitIconOrCursorResource(const RCResource *Base) {
   if (!File)
     return File.takeError();
 
-  BinaryStreamReader Reader((*File)->getBuffer(), support::little);
+  BinaryStreamReader Reader((*File)->getBuffer(), llvm::endianness::little);
 
   // Read the file headers.
   //   - At the beginning, ICONDIR/NEWHEADER header.
diff --git a/llvm/tools/llvm-rc/ResourceFileWriter.h b/llvm/tools/llvm-rc/ResourceFileWriter.h
index d809890ee8..9413a0eecd 100644
--- a/llvm/tools/llvm-rc/ResourceFileWriter.h
+++ b/llvm/tools/llvm-rc/ResourceFileWriter.h
@@ -183,8 +183,8 @@ class ResourceFileWriter : public Visitor {
   uint64_t writeObject(const ArrayRef<uint8_t> Data);
 
   template <typename T> uint64_t writeInt(const T &Value) {
-    support::detail::packed_endian_specific_integral<T, support::little,
-                                                     support::unaligned>
+    support::detail::packed_endian_specific_integral<
+        T, llvm::endianness::little, support::unaligned>
         Object(Value);
     return writeObject(Object);
   }
diff --git a/llvm/tools/llvm-rc/ResourceScriptParser.cpp b/llvm/tools/llvm-rc/ResourceScriptParser.cpp
index 9e10474488..4f02fa502d 100644
--- a/llvm/tools/llvm-rc/ResourceScriptParser.cpp
+++ b/llvm/tools/llvm-rc/ResourceScriptParser.cpp
@@ -238,7 +238,24 @@ Expected<StringRef> RCParser::readString() {
 Expected<StringRef> RCParser::readFilename() {
   if (!isNextTokenKind(Kind::String) && !isNextTokenKind(Kind::Identifier))
     return getExpectedError("string");
-  return read().value();
+  const RCToken &Token = read();
+  StringRef Str = Token.value();
+  if (Token.kind() != Kind::String)
+    return Str;
+  while (isNextTokenKind(Kind::String)) {
+    const RCToken &NextToken = read();
+    StringRef Next = NextToken.value();
+    bool IsWide = Str.consume_front_insensitive("L");
+    Next.consume_front_insensitive("L");
+    bool StrUnquoted = Str.consume_front("\"") && Str.consume_back("\"");
+    bool NextUnquoted = Next.consume_front("\"") && Next.consume_back("\"");
+    assert(StrUnquoted && NextUnquoted);
+    (void)StrUnquoted;
+    (void)NextUnquoted;
+
+    Str = Saver.save(Twine(IsWide ? "L" : "") + "\"" + Str + Next + "\"");
+  }
+  return Str;
 }
 
 Expected<StringRef> RCParser::readIdentifier() {
@@ -499,9 +516,10 @@ RCParser::ParseType RCParser::parseUserDefinedResource(IntOrString Type) {
   // Check if this is a file resource.
   switch (look().kind()) {
   case Kind::String:
-  case Kind::Identifier:
-    return std::make_unique<UserDefinedResource>(Type, read().value(),
-                                                 MemoryFlags);
+  case Kind::Identifier: {
+    ASSIGN_OR_RETURN(Filename, readFilename());
+    return std::make_unique<UserDefinedResource>(Type, *Filename, MemoryFlags);
+  }
   default:
     break;
   }
diff --git a/llvm/tools/llvm-rc/ResourceScriptParser.h b/llvm/tools/llvm-rc/ResourceScriptParser.h
index 5c01cec0f1..603afd8d73 100644
--- a/llvm/tools/llvm-rc/ResourceScriptParser.h
+++ b/llvm/tools/llvm-rc/ResourceScriptParser.h
@@ -18,6 +18,7 @@
 #include "ResourceScriptToken.h"
 
 #include "llvm/Support/Compiler.h"
+#include "llvm/Support/StringSaver.h"
 #include "llvm/Support/raw_ostream.h"
 
 #include <system_error>
@@ -185,6 +186,9 @@ class RCParser {
   std::vector<RCToken> Tokens;
   LocIter CurLoc;
   const LocIter End;
+
+  BumpPtrAllocator Alloc;
+  StringSaver Saver{Alloc};
 };
 
 } // namespace rc
diff --git a/llvm/tools/llvm-readobj/COFFDumper.cpp b/llvm/tools/llvm-readobj/COFFDumper.cpp
index 6e32a7eeb9..9c24b0b8db 100644
--- a/llvm/tools/llvm-readobj/COFFDumper.cpp
+++ b/llvm/tools/llvm-readobj/COFFDumper.cpp
@@ -1106,7 +1106,7 @@ void COFFDumper::initializeFileAndStringTables(BinaryStreamReader &Reader) {
     if (Error E = Reader.readFixedString(Contents, SubSectionSize))
       reportError(std::move(E), Obj->getFileName());
 
-    BinaryStreamRef ST(Contents, support::little);
+    BinaryStreamRef ST(Contents, llvm::endianness::little);
     switch (DebugSubsectionKind(SubType)) {
     case DebugSubsectionKind::FileChecksums:
       if (Error E = CVFileChecksumTable.initialize(ST))
@@ -1148,7 +1148,7 @@ void COFFDumper::printCodeViewSymbolSection(StringRef SectionName,
     reportError(errorCodeToError(object_error::parse_failed),
                 Obj->getFileName());
 
-  BinaryStreamReader FSReader(Data, support::little);
+  BinaryStreamReader FSReader(Data, llvm::endianness::little);
   initializeFileAndStringTables(FSReader);
 
   // TODO: Convert this over to using ModuleSubstreamVisitor.
@@ -1237,7 +1237,7 @@ void COFFDumper::printCodeViewSymbolSection(StringRef SectionName,
     }
     case DebugSubsectionKind::FrameData: {
       // First four bytes is a relocation against the function.
-      BinaryStreamReader SR(Contents, llvm::support::little);
+      BinaryStreamReader SR(Contents, llvm::endianness::little);
 
       DebugFrameDataSubsectionRef FrameData;
       if (Error E = FrameData.initialize(SR))
@@ -1302,7 +1302,8 @@ void COFFDumper::printCodeViewSymbolSection(StringRef SectionName,
     ListScope S(W, "FunctionLineTable");
     W.printString("LinkageName", Name);
 
-    BinaryStreamReader Reader(FunctionLineTables[Name], support::little);
+    BinaryStreamReader Reader(FunctionLineTables[Name],
+                              llvm::endianness::little);
 
     DebugLinesSubsectionRef LineInfo;
     if (Error E = LineInfo.initialize(Reader))
@@ -1354,7 +1355,7 @@ void COFFDumper::printCodeViewSymbolsSubsection(StringRef Subsection,
   CVSymbolDumper CVSD(W, Types, CodeViewContainer::ObjectFile, std::move(CODD),
                       CompilationCPUType, opts::CodeViewSubsectionBytes);
   CVSymbolArray Symbols;
-  BinaryStreamReader Reader(BinaryData, llvm::support::little);
+  BinaryStreamReader Reader(BinaryData, llvm::endianness::little);
   if (Error E = Reader.readArray(Symbols, Reader.getLength())) {
     W.flush();
     reportError(std::move(E), Obj->getFileName());
@@ -1369,7 +1370,7 @@ void COFFDumper::printCodeViewSymbolsSubsection(StringRef Subsection,
 }
 
 void COFFDumper::printCodeViewFileChecksums(StringRef Subsection) {
-  BinaryStreamRef Stream(Subsection, llvm::support::little);
+  BinaryStreamRef Stream(Subsection, llvm::endianness::little);
   DebugChecksumsSubsectionRef Checksums;
   if (Error E = Checksums.initialize(Stream))
     reportError(std::move(E), Obj->getFileName());
@@ -1389,7 +1390,7 @@ void COFFDumper::printCodeViewFileChecksums(StringRef Subsection) {
 }
 
 void COFFDumper::printCodeViewInlineeLines(StringRef Subsection) {
-  BinaryStreamReader SR(Subsection, llvm::support::little);
+  BinaryStreamReader SR(Subsection, llvm::endianness::little);
   DebugInlineeLinesSubsectionRef Lines;
   if (Error E = Lines.initialize(SR))
     reportError(std::move(E), Obj->getFileName());
@@ -1449,7 +1450,7 @@ void COFFDumper::mergeCodeViewTypes(MergingTypeTableBuilder &CVIDs,
                     Obj->getFileName());
 
       CVTypeArray Types;
-      BinaryStreamReader Reader(Data, llvm::support::little);
+      BinaryStreamReader Reader(Data, llvm::endianness::little);
       if (auto EC = Reader.readArray(Types, Reader.getLength())) {
         consumeError(std::move(EC));
         W.flush();
@@ -2091,10 +2092,10 @@ void COFFDumper::printStackMap() const {
 
   if (Obj->isLittleEndian())
     prettyPrintStackMap(
-        W, StackMapParser<support::little>(StackMapContentsArray));
+        W, StackMapParser<llvm::endianness::little>(StackMapContentsArray));
   else
     prettyPrintStackMap(
-        W, StackMapParser<support::big>(StackMapContentsArray));
+        W, StackMapParser<llvm::endianness::big>(StackMapContentsArray));
 }
 
 void COFFDumper::printAddrsig() {
@@ -2150,7 +2151,7 @@ void COFFDumper::printCGProfile() {
 
   StringRef CGProfileContents =
       unwrapOrError(Obj->getFileName(), CGProfileSection.getContents());
-  BinaryStreamReader Reader(CGProfileContents, llvm::support::little);
+  BinaryStreamReader Reader(CGProfileContents, llvm::endianness::little);
 
   ListScope L(W, "CGProfile");
   while (!Reader.empty()) {
diff --git a/llvm/tools/llvm-readobj/DwarfCFIEHPrinter.h b/llvm/tools/llvm-readobj/DwarfCFIEHPrinter.h
index 365a5985e4..687d97abd0 100644
--- a/llvm/tools/llvm-readobj/DwarfCFIEHPrinter.h
+++ b/llvm/tools/llvm-readobj/DwarfCFIEHPrinter.h
@@ -113,8 +113,7 @@ void PrinterContext<ELFT>::printEHFrameHdr(const Elf_Phdr *EHFramePHdr) const {
   if (!Content)
     reportError(Content.takeError(), ObjF.getFileName());
 
-  DataExtractor DE(*Content,
-                   ELFT::TargetEndianness == support::endianness::little,
+  DataExtractor DE(*Content, ELFT::TargetEndianness == llvm::endianness::little,
                    ELFT::Is64Bits ? 8 : 4);
 
   DictScope D(W, "Header");
@@ -189,7 +188,7 @@ void PrinterContext<ELFT>::printEHFrame(const Elf_Shdr *EHFrameShdr) const {
       ObjF, DWARFContext::ProcessDebugRelocations::Process, nullptr);
   DWARFDataExtractor DE(DICtx->getDWARFObj(),
                         DICtx->getDWARFObj().getEHFrameSection(),
-                        ELFT::TargetEndianness == support::endianness::little,
+                        ELFT::TargetEndianness == llvm::endianness::little,
                         ELFT::Is64Bits ? 8 : 4);
   DWARFDebugFrame EHFrame(Triple::ArchType(ObjF.getArch()), /*IsEH=*/true,
                           /*EHFrameAddress=*/Address);
diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp
index eda3982113..658c651add 100644
--- a/llvm/tools/llvm-readobj/ELFDumper.cpp
+++ b/llvm/tools/llvm-readobj/ELFDumper.cpp
@@ -363,7 +363,7 @@ template <typename ELFT> class ELFDumper : public ObjDumper {
   }
 
   void printAttributes(unsigned, std::unique_ptr<ELFAttributeParser>,
-                       support::endianness);
+                       llvm::endianness);
   void printMipsReginfo();
   void printMipsOptions();
 
@@ -2842,7 +2842,7 @@ template <class ELFT> void ELFDumper<ELFT>::printArchSpecificInfo() {
     if (Obj.isLE())
       printAttributes(ELF::SHT_ARM_ATTRIBUTES,
                       std::make_unique<ARMAttributeParser>(&W),
-                      support::little);
+                      llvm::endianness::little);
     else
       reportUniqueWarning("attribute printing not implemented for big-endian "
                           "ARM objects");
@@ -2851,7 +2851,7 @@ template <class ELFT> void ELFDumper<ELFT>::printArchSpecificInfo() {
     if (Obj.isLE())
       printAttributes(ELF::SHT_RISCV_ATTRIBUTES,
                       std::make_unique<RISCVAttributeParser>(&W),
-                      support::little);
+                      llvm::endianness::little);
     else
       reportUniqueWarning("attribute printing not implemented for big-endian "
                           "RISC-V objects");
@@ -2859,7 +2859,7 @@ template <class ELFT> void ELFDumper<ELFT>::printArchSpecificInfo() {
   case EM_MSP430:
     printAttributes(ELF::SHT_MSP430_ATTRIBUTES,
                     std::make_unique<MSP430AttributeParser>(&W),
-                    support::little);
+                    llvm::endianness::little);
     break;
   case EM_MIPS: {
     printMipsABIFlags();
@@ -2885,7 +2885,7 @@ template <class ELFT> void ELFDumper<ELFT>::printArchSpecificInfo() {
 template <class ELFT>
 void ELFDumper<ELFT>::printAttributes(
     unsigned AttrShType, std::unique_ptr<ELFAttributeParser> AttrParser,
-    support::endianness Endianness) {
+    llvm::endianness Endianness) {
   assert((AttrShType != ELF::SHT_NULL) && AttrParser &&
          "Incomplete ELF attribute implementation");
   DictScope BA(W, "BuildAttributes");
@@ -5565,7 +5565,7 @@ static AMDGPUNote getAMDGPUNote(uint32_t NoteType, ArrayRef<uint8_t> Desc) {
     // FIXME: Metadata Verifier only works with AMDHSA.
     //  This is an ugly workaround to avoid the verifier for other MD
     //  formats (e.g. amdpal)
-    if (MsgPackString.find("amdhsa.") != StringRef::npos) {
+    if (MsgPackString.contains("amdhsa.")) {
       AMDGPU::HSAMD::V3::MetadataVerifier Verifier(true);
       if (!Verifier.verify(MsgPackDoc.getRoot()))
         MetadataString = "Invalid AMDGPU Metadata\n";
@@ -5806,6 +5806,8 @@ const NoteType CoreNoteTypes[] = {
     {ELF::NT_ARM_SVE, "NT_ARM_SVE (AArch64 SVE registers)"},
     {ELF::NT_ARM_PAC_MASK,
      "NT_ARM_PAC_MASK (AArch64 Pointer Authentication code masks)"},
+    {ELF::NT_ARM_TAGGED_ADDR_CTRL,
+     "NT_ARM_TAGGED_ADDR_CTRL (AArch64 Tagged Address Control)"},
     {ELF::NT_ARM_SSVE, "NT_ARM_SSVE (AArch64 Streaming SVE registers)"},
     {ELF::NT_ARM_ZA, "NT_ARM_ZA (AArch64 SME ZA registers)"},
     {ELF::NT_ARM_ZT, "NT_ARM_ZT (AArch64 SME ZT registers)"},
@@ -6002,9 +6004,9 @@ template <class ELFT> void GNUELFDumper<ELFT>::printNotes() {
         return Error::success();
     } else if (Name == "CORE") {
       if (Type == ELF::NT_FILE) {
-        DataExtractor DescExtractor(Descriptor,
-                                    ELFT::TargetEndianness == support::little,
-                                    sizeof(Elf_Addr));
+        DataExtractor DescExtractor(
+            Descriptor, ELFT::TargetEndianness == llvm::endianness::little,
+            sizeof(Elf_Addr));
         if (Expected<CoreNote> NoteOrErr = readCoreNote(DescExtractor)) {
           printCoreNote<ELFT>(OS, *NoteOrErr);
           return Error::success();
@@ -7699,9 +7701,9 @@ template <class ELFT> void LLVMELFDumper<ELFT>::printNotes() {
         return Error::success();
     } else if (Name == "CORE") {
       if (Type == ELF::NT_FILE) {
-        DataExtractor DescExtractor(Descriptor,
-                                    ELFT::TargetEndianness == support::little,
-                                    sizeof(Elf_Addr));
+        DataExtractor DescExtractor(
+            Descriptor, ELFT::TargetEndianness == llvm::endianness::little,
+            sizeof(Elf_Addr));
         if (Expected<CoreNote> N = readCoreNote(DescExtractor)) {
           printCoreNoteLLVMStyle(*N, W);
           return Error::success();
diff --git a/llvm/tools/llvm-readobj/MachODumper.cpp b/llvm/tools/llvm-readobj/MachODumper.cpp
index ce5ba46695..0a23ad772e 100644
--- a/llvm/tools/llvm-readobj/MachODumper.cpp
+++ b/llvm/tools/llvm-readobj/MachODumper.cpp
@@ -734,10 +734,10 @@ void MachODumper::printStackMap() const {
 
   if (Obj->isLittleEndian())
     prettyPrintStackMap(
-        W, StackMapParser<support::little>(StackMapContentsArray));
+        W, StackMapParser<llvm::endianness::little>(StackMapContentsArray));
   else
     prettyPrintStackMap(
-        W, StackMapParser<support::big>(StackMapContentsArray));
+        W, StackMapParser<llvm::endianness::big>(StackMapContentsArray));
 }
 
 void MachODumper::printCGProfile() {
@@ -760,8 +760,8 @@ void MachODumper::printCGProfile() {
   StringRef CGProfileContents =
       unwrapOrError(Obj->getFileName(), CGProfileSection.getContents());
   BinaryStreamReader Reader(CGProfileContents, Obj->isLittleEndian()
-                                                   ? llvm::support::little
-                                                   : llvm::support::big);
+                                                   ? llvm::endianness::little
+                                                   : llvm::endianness::big);
 
   ListScope L(W, "CGProfile");
   while (!Reader.empty()) {
diff --git a/llvm/tools/llvm-readobj/ObjDumper.h b/llvm/tools/llvm-readobj/ObjDumper.h
index a44fa42b85..1d67945358 100644
--- a/llvm/tools/llvm-readobj/ObjDumper.h
+++ b/llvm/tools/llvm-readobj/ObjDumper.h
@@ -13,9 +13,7 @@
 #include <memory>
 #include <system_error>
 
-#include "llvm/ADT/STLFunctionalExtras.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/CommandLine.h"
diff --git a/llvm/tools/llvm-readobj/WindowsResourceDumper.cpp b/llvm/tools/llvm-readobj/WindowsResourceDumper.cpp
index fb085ecaa7..53370054f5 100644
--- a/llvm/tools/llvm-readobj/WindowsResourceDumper.cpp
+++ b/llvm/tools/llvm-readobj/WindowsResourceDumper.cpp
@@ -26,7 +26,7 @@ std::string stripUTF16(const ArrayRef<UTF16> &UTF16Str) {
   for (UTF16 Ch : UTF16Str) {
     // UTF16Str will have swapped byte order in case of big-endian machines.
     // Swap it back in such a case.
-    uint16_t ChValue = support::endian::byte_swap(Ch, support::little);
+    uint16_t ChValue = support::endian::byte_swap(Ch, llvm::endianness::little);
     if (ChValue <= 0xFF)
       Result += ChValue;
     else
diff --git a/llvm/tools/llvm-readtapi/CMakeLists.txt b/llvm/tools/llvm-readtapi/CMakeLists.txt
index bc859ee57c..277011cf13 100644
--- a/llvm/tools/llvm-readtapi/CMakeLists.txt
+++ b/llvm/tools/llvm-readtapi/CMakeLists.txt
@@ -9,4 +9,6 @@ add_llvm_tool(llvm-readtapi
   DiffEngine.cpp
   )
 
-add_llvm_tool_symlink(readtapi llvm-readtapi)
+if(LLVM_INSTALL_BINUTILS_SYMLINKS)
+  add_llvm_tool_symlink(readtapi llvm-readtapi)
+endif()
diff --git a/llvm/tools/llvm-reduce/deltas/ReduceAttributes.cpp b/llvm/tools/llvm-reduce/deltas/ReduceAttributes.cpp
index df87ce7cfa..538fd6a845 100644
--- a/llvm/tools/llvm-reduce/deltas/ReduceAttributes.cpp
+++ b/llvm/tools/llvm-reduce/deltas/ReduceAttributes.cpp
@@ -19,7 +19,6 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/Sequence.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/iterator_range.h"
 #include "llvm/IR/Attributes.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/GlobalVariable.h"
diff --git a/llvm/tools/llvm-reduce/deltas/ReduceOperandBundles.cpp b/llvm/tools/llvm-reduce/deltas/ReduceOperandBundles.cpp
index b048042e6e..a211b6ac8d 100644
--- a/llvm/tools/llvm-reduce/deltas/ReduceOperandBundles.cpp
+++ b/llvm/tools/llvm-reduce/deltas/ReduceOperandBundles.cpp
@@ -18,7 +18,6 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/Sequence.h"
-#include "llvm/ADT/iterator_range.h"
 #include "llvm/IR/InstVisitor.h"
 #include "llvm/IR/InstrTypes.h"
 #include "llvm/Support/raw_ostream.h"
diff --git a/llvm/tools/llvm-remarkutil/CMakeLists.txt b/llvm/tools/llvm-remarkutil/CMakeLists.txt
index c1dd311e97..48aeb9397c 100644
--- a/llvm/tools/llvm-remarkutil/CMakeLists.txt
+++ b/llvm/tools/llvm-remarkutil/CMakeLists.txt
@@ -7,6 +7,7 @@ set(LLVM_LINK_COMPONENTS
 add_llvm_tool(llvm-remarkutil
   RemarkConvert.cpp
   RemarkCount.cpp
+  RemarkCounter.cpp
   RemarkSizeDiff.cpp
   RemarkUtil.cpp
   RemarkUtilHelpers.cpp
diff --git a/llvm/tools/llvm-remarkutil/RemarkCounter.cpp b/llvm/tools/llvm-remarkutil/RemarkCounter.cpp
new file mode 100644
index 0000000000..dc0685f342
--- /dev/null
+++ b/llvm/tools/llvm-remarkutil/RemarkCounter.cpp
@@ -0,0 +1,337 @@
+//===- RemarkCounter.cpp --------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Generic tool to count remarks based on properties
+//
+//===----------------------------------------------------------------------===//
+
+#include "RemarkCounter.h"
+#include "RemarkUtilRegistry.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Regex.h"
+
+using namespace llvm;
+using namespace remarks;
+using namespace llvm::remarkutil;
+
+static cl::SubCommand CountSub("count",
+                               "Collect remarks based on specified criteria.");
+
+INPUT_FORMAT_COMMAND_LINE_OPTIONS(CountSub)
+INPUT_OUTPUT_COMMAND_LINE_OPTIONS(CountSub)
+
+static cl::list<std::string>
+    Keys("args", cl::desc("Specify remark argument/s to count by."),
+         cl::value_desc("arguments"), cl::sub(CountSub), cl::ValueOptional);
+static cl::list<std::string> RKeys(
+    "rargs",
+    cl::desc(
+        "Specify remark argument/s to count (accepts regular expressions)."),
+    cl::value_desc("arguments"), cl::sub(CountSub), cl::ValueOptional);
+static cl::opt<std::string>
+    RemarkNameOpt("remark-name",
+                  cl::desc("Optional remark name to filter collection by."),
+                  cl::ValueOptional, cl::sub(CountSub));
+static cl::opt<std::string>
+    PassNameOpt("pass-name", cl::ValueOptional,
+                cl::desc("Optional remark pass name to filter collection by."),
+                cl::sub(CountSub));
+
+static cl::opt<std::string> RemarkFilterArgByOpt(
+    "filter-arg-by", cl::desc("Optional remark arg to filter collection by."),
+    cl::ValueOptional, cl::sub(CountSub));
+static cl::opt<std::string>
+    RemarkNameOptRE("rremark-name",
+                    cl::desc("Optional remark name to filter collection by "
+                             "(accepts regular expressions)."),
+                    cl::ValueOptional, cl::sub(CountSub));
+static cl::opt<std::string>
+    RemarkArgFilterOptRE("rfilter-arg-by",
+                         cl::desc("Optional remark arg to filter collection by "
+                                  "(accepts regular expressions)."),
+                         cl::sub(CountSub), cl::ValueOptional);
+static cl::opt<std::string>
+    PassNameOptRE("rpass-name", cl::ValueOptional,
+                  cl::desc("Optional remark pass name to filter collection "
+                           "by (accepts regular expressions)."),
+                  cl::sub(CountSub));
+static cl::opt<Type> RemarkTypeOpt(
+    "remark-type", cl::desc("Optional remark type to filter collection by."),
+    cl::values(clEnumValN(Type::Unknown, "unknown", "UNKOWN"),
+               clEnumValN(Type::Passed, "passed", "PASSED"),
+               clEnumValN(Type::Missed, "missed", "MISSED"),
+               clEnumValN(Type::Analysis, "analysis", "ANALYSIS"),
+               clEnumValN(Type::AnalysisFPCommute, "analysis-fp-commute",
+                          "ANALYSIS_FP_COMMUTE"),
+               clEnumValN(Type::AnalysisAliasing, "analysis-aliasing",
+                          "ANALYSIS_ALIASING"),
+               clEnumValN(Type::Failure, "failure", "FAILURE")),
+    cl::init(Type::Failure), cl::sub(CountSub));
+static cl::opt<CountBy> CountByOpt(
+    "count-by", cl::desc("Specify the property to collect remarks by."),
+    cl::values(
+        clEnumValN(CountBy::REMARK, "remark-name",
+                   "Counts individual remarks based on how many of the remark "
+                   "exists."),
+        clEnumValN(CountBy::ARGUMENT, "arg",
+                   "Counts based on the value each specified argument has. The "
+                   "argument has to have a number value to be considered.")),
+    cl::init(CountBy::REMARK), cl::sub(CountSub));
+static cl::opt<GroupBy> GroupByOpt(
+    "group-by", cl::desc("Specify the property to group remarks by."),
+    cl::values(
+        clEnumValN(
+            GroupBy::PER_SOURCE, "source",
+            "Display the count broken down by the filepath of each remark "
+            "emitted. Requires remarks to have DebugLoc information."),
+        clEnumValN(GroupBy::PER_FUNCTION, "function",
+                   "Breakdown the count by function name."),
+        clEnumValN(
+            GroupBy::PER_FUNCTION_WITH_DEBUG_LOC, "function-with-loc",
+            "Breakdown the count by function name taking into consideration "
+            "the filepath info from the DebugLoc of the remark."),
+        clEnumValN(GroupBy::TOTAL, "total",
+                   "Output the total number corresponding to the count for the "
+                   "provided input file.")),
+    cl::init(GroupBy::PER_SOURCE), cl::sub(CountSub));
+
+/// Look for matching argument with \p Key in \p Remark and return the parsed
+/// integer value or 0 if it is has no integer value.
+static unsigned getValForKey(StringRef Key, const Remark &Remark) {
+  auto *RemarkArg = find_if(Remark.Args, [&Key](const Argument &Arg) {
+    return Arg.Key == Key && Arg.isValInt();
+  });
+  if (RemarkArg == Remark.Args.end())
+    return 0;
+  return *RemarkArg->getValAsInt();
+}
+
+Error Filters::regexArgumentsValid() {
+  if (RemarkNameFilter && RemarkNameFilter->IsRegex)
+    if (auto E = checkRegex(RemarkNameFilter->FilterRE))
+      return E;
+  if (PassNameFilter && PassNameFilter->IsRegex)
+    if (auto E = checkRegex(PassNameFilter->FilterRE))
+      return E;
+  if (ArgFilter && ArgFilter->IsRegex)
+    if (auto E = checkRegex(ArgFilter->FilterRE))
+      return E;
+  return Error::success();
+}
+
+bool Filters::filterRemark(const Remark &Remark) {
+  if (RemarkNameFilter && !RemarkNameFilter->match(Remark.RemarkName))
+    return false;
+  if (PassNameFilter && !PassNameFilter->match(Remark.PassName))
+    return false;
+  if (RemarkTypeFilter)
+    return *RemarkTypeFilter == Remark.RemarkType;
+  if (ArgFilter) {
+    if (!any_of(Remark.Args,
+                [this](Argument Arg) { return ArgFilter->match(Arg.Val); }))
+      return false;
+  }
+  return true;
+}
+
+Error ArgumentCounter::getAllMatchingArgumentsInRemark(
+    StringRef Buffer, ArrayRef<FilterMatcher> Arguments, Filters &Filter) {
+  auto MaybeParser = createRemarkParser(InputFormat, Buffer);
+  if (!MaybeParser)
+    return MaybeParser.takeError();
+  auto &Parser = **MaybeParser;
+  auto MaybeRemark = Parser.next();
+  for (; MaybeRemark; MaybeRemark = Parser.next()) {
+    auto &Remark = **MaybeRemark;
+    // Only collect keys from remarks included in the filter.
+    if (!Filter.filterRemark(Remark))
+      continue;
+    for (auto &Key : Arguments) {
+      for (Argument Arg : Remark.Args)
+        if (Key.match(Arg.Key) && Arg.isValInt())
+          ArgumentSetIdxMap.insert({Arg.Key, ArgumentSetIdxMap.size()});
+    }
+  }
+
+  auto E = MaybeRemark.takeError();
+  if (!E.isA<EndOfFileError>())
+    return E;
+  consumeError(std::move(E));
+  return Error::success();
+}
+
+std::optional<std::string> Counter::getGroupByKey(const Remark &Remark) {
+  switch (Group) {
+  case GroupBy::PER_FUNCTION:
+    return Remark.FunctionName.str();
+  case GroupBy::TOTAL:
+    return "Total";
+  case GroupBy::PER_SOURCE:
+  case GroupBy::PER_FUNCTION_WITH_DEBUG_LOC:
+    if (!Remark.Loc.has_value())
+      return std::nullopt;
+
+    if (Group == GroupBy::PER_FUNCTION_WITH_DEBUG_LOC)
+      return Remark.Loc->SourceFilePath.str() + ":" + Remark.FunctionName.str();
+    return Remark.Loc->SourceFilePath.str();
+  }
+  llvm_unreachable("Fully covered switch above!");
+}
+
+void ArgumentCounter::collect(const Remark &Remark) {
+  SmallVector<unsigned, 4> Row(ArgumentSetIdxMap.size());
+  std::optional<std::string> GroupByKey = getGroupByKey(Remark);
+  // Early return if we don't have a value
+  if (!GroupByKey)
+    return;
+  auto GroupVal = *GroupByKey;
+  CountByKeysMap.insert({GroupVal, Row});
+  for (auto [Key, Idx] : ArgumentSetIdxMap) {
+    auto Count = getValForKey(Key, Remark);
+    CountByKeysMap[GroupVal][Idx] += Count;
+  }
+}
+
+void RemarkCounter::collect(const Remark &Remark) {
+  std::optional<std::string> Key = getGroupByKey(Remark);
+  if (!Key.has_value())
+    return;
+  auto Iter = CountedByRemarksMap.insert({*Key, 1});
+  if (!Iter.second)
+    Iter.first->second += 1;
+}
+
+Error ArgumentCounter::print(StringRef OutputFileName) {
+  auto MaybeOF =
+      getOutputFileWithFlags(OutputFileName, sys::fs::OF_TextWithCRLF);
+  if (!MaybeOF)
+    return MaybeOF.takeError();
+
+  auto OF = std::move(*MaybeOF);
+  OF->os() << groupByToStr(Group) << ",";
+  unsigned Idx = 0;
+  for (auto [Key, _] : ArgumentSetIdxMap) {
+    OF->os() << Key;
+    if (Idx != ArgumentSetIdxMap.size() - 1)
+      OF->os() << ",";
+    Idx++;
+  }
+  OF->os() << "\n";
+  for (auto [Header, CountVector] : CountByKeysMap) {
+    OF->os() << Header << ",";
+    unsigned Idx = 0;
+    for (auto Count : CountVector) {
+      OF->os() << Count;
+      if (Idx != ArgumentSetIdxMap.size() - 1)
+        OF->os() << ",";
+      Idx++;
+    }
+    OF->os() << "\n";
+  }
+  return Error::success();
+}
+
+Error RemarkCounter::print(StringRef OutputFileName) {
+  auto MaybeOF =
+      getOutputFileWithFlags(OutputFileName, sys::fs::OF_TextWithCRLF);
+  if (!MaybeOF)
+    return MaybeOF.takeError();
+
+  auto OF = std::move(*MaybeOF);
+  OF->os() << groupByToStr(Group) << ","
+           << "Count\n";
+  for (auto [Key, Count] : CountedByRemarksMap)
+    OF->os() << Key << "," << Count << "\n";
+  OF->keep();
+  return Error::success();
+}
+
+Expected<Filters> getRemarkFilter() {
+  // Create Filter properties.
+  std::optional<FilterMatcher> RemarkNameFilter;
+  std::optional<FilterMatcher> PassNameFilter;
+  std::optional<FilterMatcher> RemarkArgFilter;
+  std::optional<Type> RemarkType;
+  if (!RemarkNameOpt.empty())
+    RemarkNameFilter = {RemarkNameOpt, false};
+  else if (!RemarkNameOptRE.empty())
+    RemarkNameFilter = {RemarkNameOptRE, true};
+  if (!PassNameOpt.empty())
+    PassNameFilter = {PassNameOpt, false};
+  else if (!PassNameOptRE.empty())
+    PassNameFilter = {PassNameOptRE, true};
+  if (RemarkTypeOpt != Type::Failure)
+    RemarkType = RemarkTypeOpt;
+  if (!RemarkFilterArgByOpt.empty())
+    RemarkArgFilter = {RemarkFilterArgByOpt, false};
+  else if (!RemarkArgFilterOptRE.empty())
+    RemarkArgFilter = {RemarkArgFilterOptRE, true};
+  // Create RemarkFilter.
+  return Filters::createRemarkFilter(std::move(RemarkNameFilter),
+                                     std::move(PassNameFilter),
+                                     std::move(RemarkArgFilter), RemarkType);
+}
+
+Error useCollectRemark(StringRef Buffer, Counter &Counter, Filters &Filter) {
+  // Create Parser.
+  auto MaybeParser = createRemarkParser(InputFormat, Buffer);
+  if (!MaybeParser)
+    return MaybeParser.takeError();
+  auto &Parser = **MaybeParser;
+  auto MaybeRemark = Parser.next();
+  for (; MaybeRemark; MaybeRemark = Parser.next()) {
+    const Remark &Remark = **MaybeRemark;
+    if (Filter.filterRemark(Remark))
+      Counter.collect(Remark);
+  }
+
+  if (auto E = Counter.print(OutputFileName))
+    return E;
+  auto E = MaybeRemark.takeError();
+  if (!E.isA<EndOfFileError>())
+    return E;
+  consumeError(std::move(E));
+  return Error::success();
+}
+
+static Error collectRemarks() {
+  // Create a parser for the user-specified input format.
+  auto MaybeBuf = getInputMemoryBuffer(InputFileName);
+  if (!MaybeBuf)
+    return MaybeBuf.takeError();
+  StringRef Buffer = (*MaybeBuf)->getBuffer();
+  auto MaybeFilter = getRemarkFilter();
+  if (!MaybeFilter)
+    return MaybeFilter.takeError();
+  auto &Filter = *MaybeFilter;
+  if (CountByOpt == CountBy::REMARK) {
+    RemarkCounter RC(GroupByOpt);
+    if (auto E = useCollectRemark(Buffer, RC, Filter))
+      return E;
+  } else if (CountByOpt == CountBy::ARGUMENT) {
+    SmallVector<FilterMatcher, 4> ArgumentsVector;
+    if (!Keys.empty()) {
+      for (auto &Key : Keys)
+        ArgumentsVector.push_back({Key, false});
+    } else if (!RKeys.empty())
+      for (auto Key : RKeys)
+        ArgumentsVector.push_back({Key, true});
+    else
+      ArgumentsVector.push_back({".*", true});
+
+    Expected<ArgumentCounter> AC = ArgumentCounter::createArgumentCounter(
+        GroupByOpt, ArgumentsVector, Buffer, Filter);
+    if (!AC)
+      return AC.takeError();
+    if (auto E = useCollectRemark(Buffer, *AC, Filter))
+      return E;
+  }
+  return Error::success();
+}
+
+static CommandRegistration CountReg(&CountSub, collectRemarks);
diff --git a/llvm/tools/llvm-remarkutil/RemarkCounter.h b/llvm/tools/llvm-remarkutil/RemarkCounter.h
new file mode 100644
index 0000000000..3dd06622bc
--- /dev/null
+++ b/llvm/tools/llvm-remarkutil/RemarkCounter.h
@@ -0,0 +1,217 @@
+//===- RemarkCounter.h ----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Generic tool to count remarks based on properties
+//
+//===----------------------------------------------------------------------===//
+#ifndef TOOLS_LLVM_REMARKCOUNTER_H
+#define TOOLS_LLVM_REMARKCOUNTER_H
+#include "RemarkUtilHelpers.h"
+#include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/SmallSet.h"
+#include "llvm/Support/Regex.h"
+#include <list>
+
+namespace llvm {
+namespace remarks {
+
+/// Collect remarks by counting the existance of a remark or by looking through
+/// the keys and summing through the total count.
+enum class CountBy { REMARK, ARGUMENT };
+
+/// Summarize the count by either emitting one count for the remark file, or
+/// grouping the count by source file or by function name.
+enum class GroupBy {
+  TOTAL,
+  PER_SOURCE,
+  PER_FUNCTION,
+  PER_FUNCTION_WITH_DEBUG_LOC
+};
+
+/// Convert \p GroupBy to a std::string.
+inline std::string groupByToStr(GroupBy GroupBy) {
+  switch (GroupBy) {
+  default:
+    return "Total";
+  case GroupBy::PER_FUNCTION:
+    return "Function";
+  case GroupBy::PER_FUNCTION_WITH_DEBUG_LOC:
+    return "FuctionWithDebugLoc";
+  case GroupBy::PER_SOURCE:
+    return "Source";
+  }
+}
+
+/// Filter object which can be either a string or a regex to match with the
+/// remark properties.
+struct FilterMatcher {
+  Regex FilterRE;
+  std::string FilterStr;
+  bool IsRegex;
+  FilterMatcher(std::string Filter, bool IsRegex) : IsRegex(IsRegex) {
+    if (IsRegex)
+      FilterRE = Regex(Filter);
+    else
+      FilterStr = Filter;
+  }
+
+  bool match(StringRef StringToMatch) const {
+    if (IsRegex)
+      return FilterRE.match(StringToMatch);
+    return FilterStr == StringToMatch.trim().str();
+  }
+};
+
+/// Filter out remarks based on remark properties based on name, pass name,
+/// argument and type.
+struct Filters {
+  std::optional<FilterMatcher> RemarkNameFilter;
+  std::optional<FilterMatcher> PassNameFilter;
+  std::optional<FilterMatcher> ArgFilter;
+  std::optional<Type> RemarkTypeFilter;
+  /// Returns a filter object if all the arguments provided are valid regex
+  /// types otherwise return an error.
+  static Expected<Filters>
+  createRemarkFilter(std::optional<FilterMatcher> RemarkNameFilter,
+                     std::optional<FilterMatcher> PassNameFilter,
+                     std::optional<FilterMatcher> ArgFilter,
+                     std::optional<Type> RemarkTypeFilter) {
+    Filters Filter;
+    Filter.RemarkNameFilter = std::move(RemarkNameFilter);
+    Filter.PassNameFilter = std::move(PassNameFilter);
+    Filter.ArgFilter = std::move(ArgFilter);
+    Filter.RemarkTypeFilter = std::move(RemarkTypeFilter);
+    if (auto E = Filter.regexArgumentsValid())
+      return std::move(E);
+    return std::move(Filter);
+  }
+  /// Returns true if \p Remark satisfies all the provided filters.
+  bool filterRemark(const Remark &Remark);
+
+private:
+  /// Check if arguments can be parsed as valid regex types.
+  Error regexArgumentsValid();
+};
+
+/// Convert Regex string error to an error object.
+inline Error checkRegex(const Regex &Regex) {
+  std::string Error;
+  if (!Regex.isValid(Error))
+    return createStringError(make_error_code(std::errc::invalid_argument),
+                             Twine("Regex: ", Error));
+  return Error::success();
+}
+
+/// Abstract counter class used to define the general required methods for
+/// counting a remark.
+struct Counter {
+  GroupBy Group = GroupBy::TOTAL;
+  Counter() = default;
+  Counter(enum GroupBy GroupBy) : Group(GroupBy) {}
+  /// Obtain the field for collecting remark info based on how we are
+  /// collecting. Remarks are grouped by FunctionName, Source, Source and
+  /// Function or collect by file.
+  std::optional<std::string> getGroupByKey(const Remark &Remark);
+
+  /// Collect count information from \p Remark organized based on \p Group
+  /// property.
+  virtual void collect(const Remark &) = 0;
+  /// Output the final count to the file \p OutputFileName
+  virtual Error print(StringRef OutputFileName) = 0;
+  virtual ~Counter() = default;
+};
+
+/// Count remarks based on the provided \p Keys argument and summing up the
+/// value for each matching key organized by source, function or reporting a
+/// total for the specified remark file.
+/// Reporting count grouped by source:
+///
+///  | source        | key1 | key2 | key3 |
+///  |---------------|------|------|------|
+///  | path/to/file1 | 0    | 1    | 3    |
+///  | path/to/file2 | 1    | 0    | 2    |
+///  | path/to/file3 | 2    | 3    | 1    |
+///
+/// Reporting count grouped by function:
+///
+///  | Function      | key1 | key2 | key3 |
+///  |---------------|------|------|------|
+///  | function1     | 0    | 1    | 3    |
+///  | function2     | 1    | 0    | 2    |
+///  | function3     | 2    | 3    | 1    |
+struct ArgumentCounter : Counter {
+  /// The internal object to keep the count for the remarks. The first argument
+  /// corresponds to the property we are collecting for this can be either a
+  /// source or function. The second argument is a row of integers where each
+  /// item in the row is the count for a specified key.
+  std::map<std::string, SmallVector<unsigned, 4>> CountByKeysMap;
+  /// A set of all the remark argument found in the remark file. The second
+  /// argument is the index of each of those arguments which can be used in
+  /// `CountByKeysMap` to fill count information for that argument.
+  MapVector<StringRef, unsigned> ArgumentSetIdxMap;
+  /// Create an argument counter. If the provided \p Arguments represent a regex
+  /// vector then we need to check that the provided regular expressions are
+  /// valid if not we return an Error.
+  static Expected<ArgumentCounter>
+  createArgumentCounter(GroupBy Group, ArrayRef<FilterMatcher> Arguments,
+                        StringRef Buffer, Filters &Filter) {
+    ArgumentCounter AC;
+    AC.Group = Group;
+    for (auto &Arg : Arguments) {
+      if (Arg.IsRegex) {
+        if (auto E = checkRegex(Arg.FilterRE))
+          return std::move(E);
+      }
+    }
+    if (auto E = AC.getAllMatchingArgumentsInRemark(Buffer, Arguments, Filter))
+      return std::move(E);
+    return AC;
+  }
+
+  /// Update the internal count map based on the remark integer arguments that
+  /// correspond the the user specified argument keys to collect for.
+  void collect(const Remark &) override;
+
+  /// Print a CSV table consisting of an index which is specified by \p
+  /// `Group` and can be a function name, source file name or function name
+  /// with the full source path and columns of user specified remark arguments
+  /// to collect the count for.
+  Error print(StringRef OutputFileName) override;
+
+private:
+  /// collect all the arguments that match the list of \p Arguments provided by
+  /// parsing through \p Buffer of remarks and filling \p ArgumentSetIdxMap
+  /// acting as a row for for all the keys that we are interested in collecting
+  /// information for.
+  Error getAllMatchingArgumentsInRemark(StringRef Buffer,
+                                        ArrayRef<FilterMatcher> Arguments,
+                                        Filters &Filter);
+};
+
+/// Collect remarks based by counting the existance of individual remarks. The
+/// reported table will be structured based on the provided \p Group argument
+/// by reporting count for functions, source or total count for the provided
+/// remark file.
+struct RemarkCounter : Counter {
+  std::map<std::string, unsigned> CountedByRemarksMap;
+  RemarkCounter(GroupBy Group) : Counter(Group) {}
+
+  /// Advance the internal map count broken by \p Group when
+  /// seeing \p Remark.
+  void collect(const Remark &) override;
+
+  /// Print a CSV table consisting of an index which is specified by \p
+  /// `Group` and can be a function name, source file name or function name
+  /// with the full source path and a counts column corresponding to the count
+  /// of each individual remark at th index.
+  Error print(StringRef OutputFileName) override;
+};
+} // namespace remarks
+
+} // namespace llvm
+#endif // TOOLS_LLVM_REMARKCOUNTER_H
diff --git a/llvm/tools/llvm-rtdyld/llvm-rtdyld.cpp b/llvm/tools/llvm-rtdyld/llvm-rtdyld.cpp
index d947be7f1f..107b555a99 100644
--- a/llvm/tools/llvm-rtdyld/llvm-rtdyld.cpp
+++ b/llvm/tools/llvm-rtdyld/llvm-rtdyld.cpp
@@ -978,7 +978,9 @@ static int linkAndVerify() {
     if (!Checker)
       Checker = std::make_unique<RuntimeDyldChecker>(
           IsSymbolValid, GetSymbolInfo, GetSectionInfo, GetStubInfo,
-          GetStubInfo, Obj.isLittleEndian() ? support::little : support::big,
+          GetStubInfo,
+          Obj.isLittleEndian() ? llvm::endianness::little
+                               : llvm::endianness::big,
           TheTriple, MCPU, SubtargetFeatures(), dbgs());
 
     auto FileName = sys::path::filename(InputFile);
diff --git a/llvm/tools/llvm-xray/xray-converter.cpp b/llvm/tools/llvm-xray/xray-converter.cpp
index 0a3271a156..34832ebda0 100644
--- a/llvm/tools/llvm-xray/xray-converter.cpp
+++ b/llvm/tools/llvm-xray/xray-converter.cpp
@@ -104,7 +104,7 @@ void TraceConverter::exportAsYAML(const Trace &Records, raw_ostream &OS) {
 void TraceConverter::exportAsRAWv1(const Trace &Records, raw_ostream &OS) {
   // First write out the file header, in the correct endian-appropriate format
   // (XRay assumes currently little endian).
-  support::endian::Writer Writer(OS, support::endianness::little);
+  support::endian::Writer Writer(OS, llvm::endianness::little);
   const auto &FH = Records.getFileHeader();
   Writer.write(FH.Version);
   Writer.write(FH.Type);
diff --git a/llvm/tools/llvm-xray/xray-graph-diff.cpp b/llvm/tools/llvm-xray/xray-graph-diff.cpp
index 8ccfd304ff..899a6725a5 100644
--- a/llvm/tools/llvm-xray/xray-graph-diff.cpp
+++ b/llvm/tools/llvm-xray/xray-graph-diff.cpp
@@ -20,7 +20,6 @@
 #include "xray-registry.h"
 
 #include "xray-color-helper.h"
-#include "llvm/ADT/iterator_range.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/XRay/Trace.h"
diff --git a/llvm/tools/llvm-xray/xray-graph-diff.h b/llvm/tools/llvm-xray/xray-graph-diff.h
index 5d12c563f4..c2b2a938bf 100644
--- a/llvm/tools/llvm-xray/xray-graph-diff.h
+++ b/llvm/tools/llvm-xray/xray-graph-diff.h
@@ -15,7 +15,6 @@
 #define XRAY_GRAPH_DIFF_H
 
 #include "xray-graph.h"
-#include "llvm/ADT/StringMap.h"
 #include "llvm/XRay/Graph.h"
 
 namespace llvm {
diff --git a/llvm/tools/obj2yaml/coff2yaml.cpp b/llvm/tools/obj2yaml/coff2yaml.cpp
index 604799fb27..2f80e62f87 100644
--- a/llvm/tools/obj2yaml/coff2yaml.cpp
+++ b/llvm/tools/obj2yaml/coff2yaml.cpp
@@ -123,7 +123,7 @@ initializeFileAndStringTable(const llvm::object::COFFObjectFile &Obj,
 
     cantFail(Obj.getSectionContents(COFFSection, sectionData));
 
-    BinaryStreamReader Reader(sectionData, support::little);
+    BinaryStreamReader Reader(sectionData, llvm::endianness::little);
     uint32_t Magic;
 
     Err(Reader.readInteger(Magic));
diff --git a/llvm/tools/opt/opt.cpp b/llvm/tools/opt/opt.cpp
index ed9c10d971..52c8c17ea4 100644
--- a/llvm/tools/opt/opt.cpp
+++ b/llvm/tools/opt/opt.cpp
@@ -351,7 +351,6 @@ static bool shouldPinPassToLegacyPM(StringRef Pass) {
       "expandmemcmp",
       "loop-reduce",
       "lower-amx-type",
-      "pre-amx-config",
       "lower-amx-intrinsics",
       "polyhedral-info",
       "print-polyhedral-info",
diff --git a/llvm/unittests/ADT/CMakeLists.txt b/llvm/unittests/ADT/CMakeLists.txt
index 42634cef6d..12d7325036 100644
--- a/llvm/unittests/ADT/CMakeLists.txt
+++ b/llvm/unittests/ADT/CMakeLists.txt
@@ -35,6 +35,7 @@ add_llvm_unittest(ADTTests
   HashingTest.cpp
   IListBaseTest.cpp
   IListIteratorTest.cpp
+  IListIteratorBitsTest.cpp
   IListNodeBaseTest.cpp
   IListNodeTest.cpp
   IListSentinelTest.cpp
diff --git a/llvm/unittests/ADT/CombinationGeneratorTest.cpp b/llvm/unittests/ADT/CombinationGeneratorTest.cpp
index 79c6b19afc..a3e36fb322 100644
--- a/llvm/unittests/ADT/CombinationGeneratorTest.cpp
+++ b/llvm/unittests/ADT/CombinationGeneratorTest.cpp
@@ -9,7 +9,6 @@
 #include "llvm/ADT/CombinationGenerator.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/iterator_range.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
diff --git a/llvm/unittests/ADT/IListIteratorBitsTest.cpp b/llvm/unittests/ADT/IListIteratorBitsTest.cpp
new file mode 100644
index 0000000000..167b30a5e3
--- /dev/null
+++ b/llvm/unittests/ADT/IListIteratorBitsTest.cpp
@@ -0,0 +1,138 @@
+//==- unittests/ADT/IListIteratorBitsTest.cpp - ilist_iterator_w_bits tests -=//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/simple_ilist.h"
+#include "gtest/gtest.h"
+
+// Test that ilist_iterator_w_bits can be used to store extra information about
+// what we're iterating over, that it's only enabled when given the relevant
+// option, and it can be fed into various iteration utilities.
+
+using namespace llvm;
+
+namespace {
+
+class dummy;
+
+struct Node : ilist_node<Node, ilist_iterator_bits<true>> {
+  friend class dummy;
+};
+
+struct PlainNode : ilist_node<PlainNode> {
+  friend class dummy;
+};
+
+TEST(IListIteratorBitsTest, DefaultConstructor) {
+  simple_ilist<Node, ilist_iterator_bits<true>>::iterator I;
+  simple_ilist<Node, ilist_iterator_bits<true>>::reverse_iterator RI;
+  simple_ilist<Node, ilist_iterator_bits<true>>::const_iterator CI;
+  simple_ilist<Node, ilist_iterator_bits<true>>::const_reverse_iterator CRI;
+  EXPECT_EQ(nullptr, I.getNodePtr());
+  EXPECT_EQ(nullptr, CI.getNodePtr());
+  EXPECT_EQ(nullptr, RI.getNodePtr());
+  EXPECT_EQ(nullptr, CRI.getNodePtr());
+  EXPECT_EQ(I, I);
+  EXPECT_EQ(I, CI);
+  EXPECT_EQ(CI, I);
+  EXPECT_EQ(CI, CI);
+  EXPECT_EQ(RI, RI);
+  EXPECT_EQ(RI, CRI);
+  EXPECT_EQ(CRI, RI);
+  EXPECT_EQ(CRI, CRI);
+  EXPECT_EQ(I, RI.getReverse());
+  EXPECT_EQ(RI, I.getReverse());
+}
+
+TEST(IListIteratorBitsTest, ConsAndAssignment) {
+  simple_ilist<Node, ilist_iterator_bits<true>> L;
+  Node A;
+  L.insert(L.end(), A);
+
+  simple_ilist<Node, ilist_iterator_bits<true>>::iterator I, I2;
+
+// Two sets of tests: if we've compiled in the iterator bits, then check that
+// HeadInclusiveBit and TailInclusiveBit are preserved on assignment and copy
+// construction, but not on other operations.
+#ifdef EXPERIMENTAL_DEBUGINFO_ITERATORS
+  I = L.begin();
+  EXPECT_FALSE(I.getHeadBit());
+  EXPECT_FALSE(I.getTailBit());
+  I.setHeadBit(true);
+  I.setTailBit(true);
+  EXPECT_TRUE(I.getHeadBit());
+  EXPECT_TRUE(I.getTailBit());
+
+  ++I;
+
+  EXPECT_FALSE(I.getHeadBit());
+  EXPECT_FALSE(I.getTailBit());
+
+  I = L.begin();
+  I.setHeadBit(true);
+  I.setTailBit(true);
+  I2 = I;
+  EXPECT_TRUE(I2.getHeadBit());
+  EXPECT_TRUE(I2.getTailBit());
+
+  I = L.begin();
+  I.setHeadBit(true);
+  I.setTailBit(true);
+  simple_ilist<Node, ilist_iterator_bits<true>>::iterator I3(I);
+  EXPECT_TRUE(I3.getHeadBit());
+  EXPECT_TRUE(I3.getTailBit());
+#else
+  // The calls should be available, but shouldn't actually store information.
+  I = L.begin();
+  EXPECT_FALSE(I.getHeadBit());
+  EXPECT_FALSE(I.getTailBit());
+  I.setHeadBit(true);
+  I.setTailBit(true);
+  EXPECT_FALSE(I.getHeadBit());
+  EXPECT_FALSE(I.getTailBit());
+  // Suppress warnings as we don't test with this variable.
+  (void)I2;
+#endif
+}
+
+class dummy {
+  // Test that we get an ilist_iterator_w_bits out of the node given that the
+  // options are enabled.
+  using node_options = typename ilist_detail::compute_node_options<
+      Node, ilist_iterator_bits<true>>::type;
+  static_assert(std::is_same<Node::self_iterator,
+                             llvm::ilist_iterator_w_bits<node_options, false,
+                                                         false>>::value);
+
+  // Now test that a plain node, without the option, gets a plain
+  // ilist_iterator.
+  using plain_node_options =
+      typename ilist_detail::compute_node_options<PlainNode>::type;
+  static_assert(std::is_same<
+                PlainNode::self_iterator,
+                llvm::ilist_iterator<plain_node_options, false, false>>::value);
+};
+
+TEST(IListIteratorBitsTest, RangeIteration) {
+  // Check that we can feed ilist_iterator_w_bits into make_range and similar.
+  // Plus, we should be able to convert it to a reverse iterator and use that.
+  simple_ilist<Node, ilist_iterator_bits<true>> L;
+  Node A;
+  L.insert(L.end(), A);
+
+  for (Node &N : make_range(L.begin(), L.end()))
+    (void)N;
+
+  simple_ilist<Node, ilist_iterator_bits<true>>::iterator It =
+      L.begin()->getIterator();
+  auto RevIt = It.getReverse();
+
+  for (Node &N : make_range(RevIt, L.rend()))
+    (void)N;
+}
+
+} // end namespace
diff --git a/llvm/unittests/ADT/RangeAdapterTest.cpp b/llvm/unittests/ADT/RangeAdapterTest.cpp
index 5c27225360..eb75ac3018 100644
--- a/llvm/unittests/ADT/RangeAdapterTest.cpp
+++ b/llvm/unittests/ADT/RangeAdapterTest.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/iterator_range.h"
 #include "gtest/gtest.h"
 
 #include <iterator>
diff --git a/llvm/unittests/Analysis/FunctionPropertiesAnalysisTest.cpp b/llvm/unittests/Analysis/FunctionPropertiesAnalysisTest.cpp
index f5f4a54ae0..3b38235894 100644
--- a/llvm/unittests/Analysis/FunctionPropertiesAnalysisTest.cpp
+++ b/llvm/unittests/Analysis/FunctionPropertiesAnalysisTest.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Analysis/FunctionPropertiesAnalysis.h"
-#include "llvm/ADT/iterator_range.h"
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/AsmParser/Parser.h"
@@ -25,9 +24,11 @@
 
 using namespace llvm;
 
+namespace llvm {
 extern cl::opt<bool> EnableDetailedFunctionProperties;
 extern cl::opt<bool> BigBasicBlockInstructionThreshold;
 extern cl::opt<bool> MediumBasicBlockInstrutionThreshold;
+} // namespace llvm
 
 namespace {
 
diff --git a/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp b/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp
index 8c2328ee1c..292b5cade9 100644
--- a/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp
+++ b/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp
@@ -69,6 +69,16 @@ TEST_F(TargetLibraryInfoTest, InvalidProto) {
         M->getOrInsertFunction(TLI.getName(LF), InvalidFTy).getCallee());
     EXPECT_FALSE(isLibFunc(F, LF));
   }
+
+  // i64 @labs(i32)
+  {
+    auto *InvalidLabsFTy = FunctionType::get(Type::getInt64Ty(Context),
+                                             {Type::getInt32Ty(Context)},
+                                             /*isVarArg=*/false);
+    auto *F = cast<Function>(
+        M->getOrInsertFunction("labs", InvalidLabsFTy).getCallee());
+    EXPECT_FALSE(isLibFunc(F, LibFunc_labs));
+  }
 }
 
 // Check that we do accept know-correct prototypes.
diff --git a/llvm/unittests/Bitcode/DataLayoutUpgradeTest.cpp b/llvm/unittests/Bitcode/DataLayoutUpgradeTest.cpp
index 63878995ef..5d798daf66 100644
--- a/llvm/unittests/Bitcode/DataLayoutUpgradeTest.cpp
+++ b/llvm/unittests/Bitcode/DataLayoutUpgradeTest.cpp
@@ -19,14 +19,16 @@ TEST(DataLayoutUpgradeTest, ValidDataLayoutUpgrade) {
                               "x86_64-unknown-linux-gnu");
   std::string DL2 = UpgradeDataLayoutString(
       "e-m:w-p:32:32-i64:64-f80:32-n8:16:32-S32", "i686-pc-windows-msvc");
-  std::string DL3 = UpgradeDataLayoutString("e-m:o-i64:64-i128:128-n32:64-S128",
-                                            "x86_64-apple-macosx");
-  EXPECT_EQ(DL1, "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64"
-                 "-f80:128-n8:16:32:64-S128");
-  EXPECT_EQ(DL2, "e-m:w-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64"
-                 "-f80:128-n8:16:32-S32");
-  EXPECT_EQ(DL3, "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128"
-                 "-n32:64-S128");
+  std::string DL3 = UpgradeDataLayoutString(
+      "e-m:o-i64:64-f80:128-n8:16:32:64-S128", "x86_64-apple-macosx");
+  EXPECT_EQ(DL1,
+            "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128"
+            "-f80:128-n8:16:32:64-S128");
+  EXPECT_EQ(DL2,
+            "e-m:w-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128"
+            "-f80:128-n8:16:32-S32");
+  EXPECT_EQ(DL3, "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:"
+                 "128-n8:16:32:64-S128");
 
   // Check that AMDGPU targets add -G1 if it's not present.
   EXPECT_EQ(UpgradeDataLayoutString("e-p:32:32", "r600"), "e-p:32:32-G1");
@@ -58,21 +60,22 @@ TEST(DataLayoutUpgradeTest, ValidDataLayoutUpgrade) {
 
 TEST(DataLayoutUpgradeTest, NoDataLayoutUpgrade) {
   std::string DL1 = UpgradeDataLayoutString(
-      "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32"
+      "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-i128:128:128-"
+      "f32:32:32"
       "-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
       "-n8:16:32:64-S128",
       "x86_64-unknown-linux-gnu");
-  std::string DL2 = UpgradeDataLayoutString("e-p:32:32", "i686-apple-darwin9");
-  std::string DL3 = UpgradeDataLayoutString("e-m:e-i64:64-n32:64",
+  std::string DL2 = UpgradeDataLayoutString("e-m:e-i64:64-n32:64",
                                             "powerpc64le-unknown-linux-gnu");
-  std::string DL4 =
+  std::string DL3 =
       UpgradeDataLayoutString("e-m:o-i64:64-i128:128-n32:64-S128", "aarch64--");
-  EXPECT_EQ(DL1, "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64"
-                 "-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64"
-                 "-f80:128:128-n8:16:32:64-S128");
-  EXPECT_EQ(DL2, "e-p:32:32");
-  EXPECT_EQ(DL3, "e-m:e-i64:64-n32:64");
-  EXPECT_EQ(DL4, "e-m:o-i64:64-i128:128-n32:64-S128");
+  EXPECT_EQ(
+      DL1,
+      "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-i128:128:128"
+      "-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64"
+      "-f80:128:128-n8:16:32:64-S128");
+  EXPECT_EQ(DL2, "e-m:e-i64:64-n32:64");
+  EXPECT_EQ(DL3, "e-m:o-i64:64-i128:128-n32:64-S128");
 
   // Check that AMDGPU targets don't add -G1 if there is already a -G flag.
   EXPECT_EQ(UpgradeDataLayoutString("e-p:32:32-G2", "r600"), "e-p:32:32-G2");
diff --git a/llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp b/llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp
index 0e1f273690..bb8e76a2ee 100644
--- a/llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp
+++ b/llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp
@@ -254,6 +254,59 @@ TEST_F(AArch64SelectionDAGTest, ComputeKnownBits_ADD) {
   EXPECT_EQ(Known.One, APInt(8, 0x55));
 }
 
+// Piggy-backing on the AArch64 tests to verify SelectionDAG::computeKnownBits.
+TEST_F(AArch64SelectionDAGTest, ComputeKnownBits_UADDO_CARRY) {
+  SDLoc Loc;
+  auto IntVT = EVT::getIntegerVT(Context, 8);
+  auto UnknownOp = DAG->getRegister(0, IntVT);
+  auto Mask_Zero = DAG->getConstant(0x28, Loc, IntVT);
+  auto Mask_One = DAG->getConstant(0x20, Loc, IntVT);
+  auto N0 = DAG->getNode(ISD::AND, Loc, IntVT, Mask_Zero, UnknownOp);
+  N0 = DAG->getNode(ISD::OR, Loc, IntVT, Mask_One, N0);
+  auto N1 = DAG->getConstant(0x65, Loc, IntVT);
+
+  KnownBits Known;
+
+  auto UnknownBorrow = DAG->getRegister(1, IntVT);
+  auto OpUnknownBorrow =
+      DAG->getNode(ISD::UADDO_CARRY, Loc, IntVT, N0, N1, UnknownBorrow);
+  // N0 = 0010?000
+  // N1 = 01100101
+  // B  =        ?
+  //  =>
+  // Known.Zero = 01110000 (0x70)
+  // Known.One  = 10000100 (0x84)
+  Known = DAG->computeKnownBits(OpUnknownBorrow);
+  EXPECT_EQ(Known.Zero, APInt(8, 0x70));
+  EXPECT_EQ(Known.One, APInt(8, 0x84));
+
+  auto ZeroBorrow = DAG->getConstant(0x0, Loc, IntVT);
+  auto OpZeroBorrow =
+      DAG->getNode(ISD::UADDO_CARRY, Loc, IntVT, N0, N1, ZeroBorrow);
+  // N0 = 0010?000
+  // N1 = 01100101
+  // B  =        0
+  //  =>
+  // Known.Zero = 01110010 (0x72)
+  // Known.One  = 10000101 (0x85)
+  Known = DAG->computeKnownBits(OpZeroBorrow);
+  EXPECT_EQ(Known.Zero, APInt(8, 0x72));
+  EXPECT_EQ(Known.One, APInt(8, 0x85));
+
+  auto OneBorrow = DAG->getConstant(0x1, Loc, IntVT);
+  auto OpOneBorrow =
+      DAG->getNode(ISD::UADDO_CARRY, Loc, IntVT, N0, N1, OneBorrow);
+  // N0 = 0010?000
+  // N1 = 01100101
+  // B  =        1
+  //  =>
+  // Known.Zero = 01110001 (0x71)
+  // Known.One  = 10000110 (0x86)
+  Known = DAG->computeKnownBits(OpOneBorrow);
+  EXPECT_EQ(Known.Zero, APInt(8, 0x71));
+  EXPECT_EQ(Known.One, APInt(8, 0x86));
+}
+
 // Piggy-backing on the AArch64 tests to verify SelectionDAG::computeKnownBits.
 TEST_F(AArch64SelectionDAGTest, ComputeKnownBits_SUB) {
   SDLoc Loc;
@@ -273,6 +326,60 @@ TEST_F(AArch64SelectionDAGTest, ComputeKnownBits_SUB) {
   EXPECT_EQ(Known.One, APInt(8, 0x1));
 }
 
+// Piggy-backing on the AArch64 tests to verify SelectionDAG::computeKnownBits.
+TEST_F(AArch64SelectionDAGTest, ComputeKnownBits_USUBO_CARRY) {
+  SDLoc Loc;
+  auto IntVT = EVT::getIntegerVT(Context, 8);
+  auto N0 = DAG->getConstant(0x5a, Loc, IntVT);
+  auto UnknownOp = DAG->getRegister(0, IntVT);         // ????????
+  auto Mask1_Zero = DAG->getConstant(0x8, Loc, IntVT); // 00001000
+  auto Mask1_One = DAG->getConstant(0x20, Loc, IntVT); // 00100000
+  // N1 = (???????? & 00001000) | 00100000 = 0010?000
+  auto N1 = DAG->getNode(ISD::AND, Loc, IntVT, Mask1_Zero, UnknownOp);
+  N1 = DAG->getNode(ISD::OR, Loc, IntVT, Mask1_One, N1);
+
+  KnownBits Known;
+
+  auto UnknownBorrow = DAG->getRegister(1, IntVT);
+  auto OpUnknownBorrow =
+      DAG->getNode(ISD::USUBO_CARRY, Loc, IntVT, N0, N1, UnknownBorrow);
+  // N0 = 01011010
+  // N1 = 0010?000
+  // B  =        ?
+  //  =>
+  // Known.Zero = 11000100 (0xc4)
+  // Known.One  = 00110000 (0x30)
+  Known = DAG->computeKnownBits(OpUnknownBorrow);
+  EXPECT_EQ(Known.Zero, APInt(8, 0xc4));
+  EXPECT_EQ(Known.One, APInt(8, 0x30));
+
+  auto ZeroBorrow = DAG->getConstant(0x0, Loc, IntVT);
+  auto OpZeroBorrow =
+      DAG->getNode(ISD::USUBO_CARRY, Loc, IntVT, N0, N1, ZeroBorrow);
+  // N0 = 01011010
+  // N1 = 0010?000
+  // B  =        0
+  //  =>
+  // Known.Zero = 11000101 (0xc5)
+  // Known.One  = 00110010 (0x32)
+  Known = DAG->computeKnownBits(OpZeroBorrow);
+  EXPECT_EQ(Known.Zero, APInt(8, 0xc5));
+  EXPECT_EQ(Known.One, APInt(8, 0x32));
+
+  auto OneBorrow = DAG->getConstant(0x1, Loc, IntVT);
+  auto OpOneBorrow =
+      DAG->getNode(ISD::USUBO_CARRY, Loc, IntVT, N0, N1, OneBorrow);
+  // N0 = 01011010
+  // N1 = 0010?000
+  // B  =        1
+  //  =>
+  // Known.Zero = 11000110 (0xc6)
+  // Known.One  = 00110001 (0x31)
+  Known = DAG->computeKnownBits(OpOneBorrow);
+  EXPECT_EQ(Known.Zero, APInt(8, 0xc6));
+  EXPECT_EQ(Known.One, APInt(8, 0x31));
+}
+
 TEST_F(AArch64SelectionDAGTest, isSplatValue_Fixed_BUILD_VECTOR) {
   TargetLowering TL(*TM);
 
diff --git a/llvm/unittests/CodeGen/InstrRefLDVTest.cpp b/llvm/unittests/CodeGen/InstrRefLDVTest.cpp
index c0272c337d..5a050bc520 100644
--- a/llvm/unittests/CodeGen/InstrRefLDVTest.cpp
+++ b/llvm/unittests/CodeGen/InstrRefLDVTest.cpp
@@ -70,8 +70,8 @@ class InstrRefLDVTest : public testing::Test {
   void SetUp() {
     // Boilerplate that creates a MachineFunction and associated blocks.
 
-    Mod->setDataLayout("e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-"
-                       "n8:16:32:64-S128");
+    Mod->setDataLayout("e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-"
+                       "f80:128-n8:16:32:64-S128");
     Triple TargetTriple("x86_64--");
     std::string Error;
     const Target *T = TargetRegistry::lookupTarget("", TargetTriple, Error);
@@ -475,8 +475,8 @@ body:  |
     auto MIRParse = createMIRParser(std::move(MemBuf), Ctx);
     Mod = MIRParse->parseIRModule();
     assert(Mod);
-    Mod->setDataLayout("e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-"
-                       "n8:16:32:64-S128");
+    Mod->setDataLayout("e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-"
+                       "f80:128-n8:16:32:64-S128");
 
     bool Result = MIRParse->parseMachineFunctions(*Mod, *MMI);
     assert(!Result && "Failed to parse unit test machine function?");
diff --git a/llvm/unittests/DebugInfo/CodeView/RandomAccessVisitorTest.cpp b/llvm/unittests/DebugInfo/CodeView/RandomAccessVisitorTest.cpp
index 0686b47c69..5c961998a4 100644
--- a/llvm/unittests/DebugInfo/CodeView/RandomAccessVisitorTest.cpp
+++ b/llvm/unittests/DebugInfo/CodeView/RandomAccessVisitorTest.cpp
@@ -152,7 +152,8 @@ class RandomAccessVisitorTest : public testing::Test {
   }
 
   struct GlobalTestState {
-    GlobalTestState() : Strings(Allocator), ItemStream(llvm::support::little) {}
+    GlobalTestState()
+        : Strings(Allocator), ItemStream(llvm::endianness::little) {}
 
     BumpPtrAllocator Allocator;
     StringSaver Strings;
@@ -181,7 +182,7 @@ class RandomAccessVisitorTest : public testing::Test {
     uint32_t Size = Count * sizeof(TypeIndexOffset);
     uint8_t *Buffer = GlobalState->Allocator.Allocate<uint8_t>(Size);
     MutableArrayRef<uint8_t> Bytes(Buffer, Size);
-    Storage = MutableBinaryByteStream(Bytes, support::little);
+    Storage = MutableBinaryByteStream(Bytes, llvm::endianness::little);
     BinaryStreamWriter Writer(Storage);
     for (const auto I : Indices)
       consumeError(Writer.writeObject(GlobalState->AllOffsets[I]));
@@ -370,7 +371,7 @@ TEST_F(RandomAccessVisitorTest, CrossChunkName) {
       {Builder.records()[0]},
       {Builder.records()[1]},
   };
-  BinaryItemStream<CVType> ItemStream(llvm::support::little);
+  BinaryItemStream<CVType> ItemStream(llvm::endianness::little);
   ItemStream.setItems(TypeArray);
   VarStreamArray<CVType> TypeStream(ItemStream);
 
@@ -386,7 +387,7 @@ TEST_F(RandomAccessVisitorTest, CrossChunkName) {
   ArrayRef<uint8_t> Buffer(reinterpret_cast<const uint8_t *>(TIO.data()),
                            TIO.size() * sizeof(TypeIndexOffset));
 
-  BinaryStreamReader Reader(Buffer, llvm::support::little);
+  BinaryStreamReader Reader(Buffer, llvm::endianness::little);
   FixedStreamArray<TypeIndexOffset> PartialOffsets;
   ASSERT_THAT_ERROR(Reader.readArray(PartialOffsets, 2), Succeeded());
 
diff --git a/llvm/unittests/DebugInfo/DWARF/DWARFDebugInfoTest.cpp b/llvm/unittests/DebugInfo/DWARF/DWARFDebugInfoTest.cpp
index 2adc2403ea..d81557d756 100644
--- a/llvm/unittests/DebugInfo/DWARF/DWARFDebugInfoTest.cpp
+++ b/llvm/unittests/DebugInfo/DWARF/DWARFDebugInfoTest.cpp
@@ -9,7 +9,6 @@
 #include "DwarfGenerator.h"
 #include "DwarfUtils.h"
 #include "llvm/ADT/ArrayRef.h"
-#include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/BinaryFormat/Dwarf.h"
@@ -2170,7 +2169,11 @@ TEST(DWARFDebugInfo, TestDWARF64UnitLength) {
     DWARFDataExtractor Data(Obj, Sec, /* IsLittleEndian = */ true,
                             /* AddressSize = */ 4);
     uint64_t Offset = 0;
-    EXPECT_FALSE(Header.extract(*Context, Data, &Offset, DW_SECT_INFO));
+    ASSERT_THAT_ERROR(
+        Header.extract(*Context, Data, &Offset, DW_SECT_INFO),
+        FailedWithMessage(
+            "DWARF unit from offset 0x00000000 incl. to offset  "
+            "0x1122334455667794 excl. extends past section size 0x00000018"));
     // Header.extract() returns false because there is not enough space
     // in the section for the declared length. Anyway, we can check that
     // the properties are read correctly.
diff --git a/llvm/unittests/DebugInfo/GSYM/GSYMTest.cpp b/llvm/unittests/DebugInfo/GSYM/GSYMTest.cpp
index 3f40ff56d7..ad81a2fcd1 100644
--- a/llvm/unittests/DebugInfo/GSYM/GSYMTest.cpp
+++ b/llvm/unittests/DebugInfo/GSYM/GSYMTest.cpp
@@ -181,7 +181,7 @@ static void TestFunctionInfoDecodeError(llvm::endianness ByteOrder,
                                         const uint64_t BaseAddr,
                                         std::string ExpectedErrorMsg) {
   uint8_t AddressSize = 4;
-  DataExtractor Data(Bytes, ByteOrder == llvm::support::little, AddressSize);
+  DataExtractor Data(Bytes, ByteOrder == llvm::endianness::little, AddressSize);
   llvm::Expected<FunctionInfo> Decoded = FunctionInfo::decode(Data, BaseAddr);
   // Make sure decoding fails.
   ASSERT_FALSE((bool)Decoded);
@@ -192,7 +192,7 @@ static void TestFunctionInfoDecodeError(llvm::endianness ByteOrder,
 TEST(GSYMTest, TestFunctionInfoDecodeErrors) {
   // Test decoding FunctionInfo objects that ensure we report an appropriate
   // error message.
-  const llvm::endianness ByteOrder = llvm::support::little;
+  const llvm::endianness ByteOrder = llvm::endianness::little;
   SmallString<512> Str;
   raw_svector_ostream OutStrm(Str);
   FileWriter FW(OutStrm, ByteOrder);
@@ -237,22 +237,23 @@ TEST(GSYMTest, TestFunctionInfoEncodeErrors) {
   const uint32_t InvalidName = 0;
   const uint32_t ValidName = 1;
   FunctionInfo InvalidNameFI(FuncAddr, FuncSize, InvalidName);
-  TestFunctionInfoEncodeError(llvm::support::little, InvalidNameFI,
+  TestFunctionInfoEncodeError(
+      llvm::endianness::little, InvalidNameFI,
       "attempted to encode invalid FunctionInfo object");
 
   FunctionInfo InvalidLineTableFI(FuncAddr, FuncSize, ValidName);
   // Empty line tables are not valid. Verify if the encoding of anything
   // in our line table fails, that we see get the error propagated.
   InvalidLineTableFI.OptLineTable = LineTable();
-  TestFunctionInfoEncodeError(llvm::support::little, InvalidLineTableFI,
-      "attempted to encode invalid LineTable object");
+  TestFunctionInfoEncodeError(llvm::endianness::little, InvalidLineTableFI,
+                              "attempted to encode invalid LineTable object");
 
   FunctionInfo InvalidInlineInfoFI(FuncAddr, FuncSize, ValidName);
   // Empty line tables are not valid. Verify if the encoding of anything
   // in our line table fails, that we see get the error propagated.
   InvalidInlineInfoFI.Inline = InlineInfo();
-  TestFunctionInfoEncodeError(llvm::support::little, InvalidInlineInfoFI,
-      "attempted to encode invalid InlineInfo object");
+  TestFunctionInfoEncodeError(llvm::endianness::little, InvalidInlineInfoFI,
+                              "attempted to encode invalid InlineInfo object");
 }
 
 static void TestFunctionInfoEncodeDecode(llvm::endianness ByteOrder,
@@ -267,7 +268,7 @@ static void TestFunctionInfoEncodeDecode(llvm::endianness ByteOrder,
   ASSERT_EQ(ExpectedOffset.get(), 0ULL);
   std::string Bytes(OutStrm.str());
   uint8_t AddressSize = 4;
-  DataExtractor Data(Bytes, ByteOrder == llvm::support::little, AddressSize);
+  DataExtractor Data(Bytes, ByteOrder == llvm::endianness::little, AddressSize);
   llvm::Expected<FunctionInfo> Decoded =
       FunctionInfo::decode(Data, FI.Range.start());
   // Make sure decoding succeeded.
@@ -306,30 +307,30 @@ TEST(GSYMTest, TestFunctionInfoEncoding) {
   // Make sure that we can encode and decode a FunctionInfo with no line table
   // or inline info.
   FunctionInfo FI(FuncAddr, FuncSize, FuncName);
-  TestFunctionInfoEncodeDecode(llvm::support::little, FI);
-  TestFunctionInfoEncodeDecode(llvm::support::big, FI);
+  TestFunctionInfoEncodeDecode(llvm::endianness::little, FI);
+  TestFunctionInfoEncodeDecode(llvm::endianness::big, FI);
 
   // Make sure that we can encode and decode a FunctionInfo with a line table
   // and no inline info.
   FunctionInfo FILines(FuncAddr, FuncSize, FuncName);
   AddLines(FuncAddr, FileIdx, FILines);
-  TestFunctionInfoEncodeDecode(llvm::support::little, FILines);
-  TestFunctionInfoEncodeDecode(llvm::support::big, FILines);
+  TestFunctionInfoEncodeDecode(llvm::endianness::little, FILines);
+  TestFunctionInfoEncodeDecode(llvm::endianness::big, FILines);
 
   // Make sure that we can encode and decode a FunctionInfo with no line table
   // and with inline info.
   FunctionInfo FIInline(FuncAddr, FuncSize, FuncName);
   AddInline(FuncAddr, FuncSize, FIInline);
-  TestFunctionInfoEncodeDecode(llvm::support::little, FIInline);
-  TestFunctionInfoEncodeDecode(llvm::support::big, FIInline);
+  TestFunctionInfoEncodeDecode(llvm::endianness::little, FIInline);
+  TestFunctionInfoEncodeDecode(llvm::endianness::big, FIInline);
 
   // Make sure that we can encode and decode a FunctionInfo with no line table
   // and with inline info.
   FunctionInfo FIBoth(FuncAddr, FuncSize, FuncName);
   AddLines(FuncAddr, FileIdx, FIBoth);
   AddInline(FuncAddr, FuncSize, FIBoth);
-  TestFunctionInfoEncodeDecode(llvm::support::little, FIBoth);
-  TestFunctionInfoEncodeDecode(llvm::support::big, FIBoth);
+  TestFunctionInfoEncodeDecode(llvm::endianness::little, FIBoth);
+  TestFunctionInfoEncodeDecode(llvm::endianness::big, FIBoth);
 }
 
 static void TestInlineInfoEncodeDecode(llvm::endianness ByteOrder,
@@ -343,7 +344,7 @@ static void TestInlineInfoEncodeDecode(llvm::endianness ByteOrder,
   ASSERT_FALSE(Err);
   std::string Bytes(OutStrm.str());
   uint8_t AddressSize = 4;
-  DataExtractor Data(Bytes, ByteOrder == llvm::support::little, AddressSize);
+  DataExtractor Data(Bytes, ByteOrder == llvm::endianness::little, AddressSize);
   llvm::Expected<InlineInfo> Decoded = InlineInfo::decode(Data, BaseAddr);
   // Make sure decoding succeeded.
   ASSERT_TRUE((bool)Decoded);
@@ -355,7 +356,7 @@ static void TestInlineInfoDecodeError(llvm::endianness ByteOrder,
                                       StringRef Bytes, const uint64_t BaseAddr,
                                       std::string ExpectedErrorMsg) {
   uint8_t AddressSize = 4;
-  DataExtractor Data(Bytes, ByteOrder == llvm::support::little, AddressSize);
+  DataExtractor Data(Bytes, ByteOrder == llvm::endianness::little, AddressSize);
   llvm::Expected<InlineInfo> Decoded = InlineInfo::decode(Data, BaseAddr);
   // Make sure decoding fails.
   ASSERT_FALSE((bool)Decoded);
@@ -469,8 +470,8 @@ TEST(GSYMTest, TestInlineInfo) {
   ASSERT_EQ(*InlineInfos->at(1), Inline1);
 
   // Test encoding and decoding InlineInfo objects
-  TestInlineInfoEncodeDecode(llvm::support::little, Root);
-  TestInlineInfoEncodeDecode(llvm::support::big, Root);
+  TestInlineInfoEncodeDecode(llvm::endianness::little, Root);
+  TestInlineInfoEncodeDecode(llvm::endianness::big, Root);
 }
 
 TEST(GSYMTest, TestInlineInfoEncodeErrors) {
@@ -480,16 +481,16 @@ TEST(GSYMTest, TestInlineInfoEncodeErrors) {
   // that has no ranges.
   InlineInfo Empty;
   std::string EmptyErr("attempted to encode invalid InlineInfo object");
-  TestInlineInfoEncodeError(llvm::support::little, Empty, EmptyErr);
-  TestInlineInfoEncodeError(llvm::support::big, Empty, EmptyErr);
+  TestInlineInfoEncodeError(llvm::endianness::little, Empty, EmptyErr);
+  TestInlineInfoEncodeError(llvm::endianness::big, Empty, EmptyErr);
 
   // Verify that we get an error trying to encode an InlineInfo object that has
   // a child InlineInfo that has no ranges.
   InlineInfo ContainsEmpty;
   ContainsEmpty.Ranges.insert({0x100, 0x200});
   ContainsEmpty.Children.push_back(Empty);
-  TestInlineInfoEncodeError(llvm::support::little, ContainsEmpty, EmptyErr);
-  TestInlineInfoEncodeError(llvm::support::big, ContainsEmpty, EmptyErr);
+  TestInlineInfoEncodeError(llvm::endianness::little, ContainsEmpty, EmptyErr);
+  TestInlineInfoEncodeError(llvm::endianness::big, ContainsEmpty, EmptyErr);
 
   // Verify that we get an error trying to encode an InlineInfo object that has
   // a child whose address range is not contained in the parent address range.
@@ -499,17 +500,16 @@ TEST(GSYMTest, TestInlineInfoEncodeErrors) {
   InlineInfo ChildNotContainedChild;
   ChildNotContainedChild.Ranges.insert({0x200, 0x300});
   ChildNotContained.Children.push_back(ChildNotContainedChild);
-  TestInlineInfoEncodeError(llvm::support::little, ChildNotContained,
+  TestInlineInfoEncodeError(llvm::endianness::little, ChildNotContained,
                             ChildNotContainedErr);
-  TestInlineInfoEncodeError(llvm::support::big, ChildNotContained,
+  TestInlineInfoEncodeError(llvm::endianness::big, ChildNotContained,
                             ChildNotContainedErr);
-
 }
 
 TEST(GSYMTest, TestInlineInfoDecodeErrors) {
   // Test decoding InlineInfo objects that ensure we report an appropriate
   // error message.
-  const llvm::endianness ByteOrder = llvm::support::little;
+  const llvm::endianness ByteOrder = llvm::endianness::little;
   SmallString<512> Str;
   raw_svector_ostream OutStrm(Str);
   FileWriter FW(OutStrm, ByteOrder);
@@ -601,7 +601,7 @@ static void TestFileWriterHelper(llvm::endianness ByteOrder) {
 
   std::string Bytes(OutStrm.str());
   uint8_t AddressSize = 4;
-  DataExtractor Data(Bytes, ByteOrder == llvm::support::little, AddressSize);
+  DataExtractor Data(Bytes, ByteOrder == llvm::endianness::little, AddressSize);
   uint64_t Offset = 0;
   EXPECT_EQ(Data.getU8(&Offset), U8);
   EXPECT_EQ(Data.getU16(&Offset), U16);
@@ -617,8 +617,8 @@ static void TestFileWriterHelper(llvm::endianness ByteOrder) {
 }
 
 TEST(GSYMTest, TestFileWriter) {
-  TestFileWriterHelper(llvm::support::little);
-  TestFileWriterHelper(llvm::support::big);
+  TestFileWriterHelper(llvm::endianness::little);
+  TestFileWriterHelper(llvm::endianness::big);
 }
 
 TEST(GSYMTest, TestAddressRangeEncodeDecode) {
@@ -637,7 +637,7 @@ TEST(GSYMTest, TestAddressRangeEncodeDecode) {
   encodeRange(Range2, FW, BaseAddr);
   std::string Bytes(OutStrm.str());
   uint8_t AddressSize = 4;
-  DataExtractor Data(Bytes, ByteOrder == llvm::support::little, AddressSize);
+  DataExtractor Data(Bytes, ByteOrder == llvm::endianness::little, AddressSize);
 
   AddressRange DecodedRange1, DecodedRange2;
   uint64_t Offset = 0;
@@ -657,7 +657,7 @@ static void TestAddressRangeEncodeDecodeHelper(const AddressRanges &Ranges,
 
   std::string Bytes(OutStrm.str());
   uint8_t AddressSize = 4;
-  DataExtractor Data(Bytes, ByteOrder == llvm::support::little, AddressSize);
+  DataExtractor Data(Bytes, ByteOrder == llvm::endianness::little, AddressSize);
 
   AddressRanges DecodedRanges;
   uint64_t Offset = 0;
@@ -696,7 +696,7 @@ static void TestLineTableHelper(llvm::endianness ByteOrder,
   ASSERT_FALSE(Err);
   std::string Bytes(OutStrm.str());
   uint8_t AddressSize = 4;
-  DataExtractor Data(Bytes, ByteOrder == llvm::support::little, AddressSize);
+  DataExtractor Data(Bytes, ByteOrder == llvm::endianness::little, AddressSize);
   llvm::Expected<LineTable> Decoded = LineTable::decode(Data, BaseAddr);
   // Make sure decoding succeeded.
   ASSERT_TRUE((bool)Decoded);
@@ -730,8 +730,8 @@ TEST(GSYMTest, TestLineTable) {
   ASSERT_EQ(LT[2], Line2);
 
   // Test encoding and decoding line tables.
-  TestLineTableHelper(llvm::support::little, LT);
-  TestLineTableHelper(llvm::support::big, LT);
+  TestLineTableHelper(llvm::endianness::little, LT);
+  TestLineTableHelper(llvm::endianness::big, LT);
 
   // Verify the clear method works as expected.
   LT.clear();
@@ -766,7 +766,7 @@ static void TestLineTableDecodeError(llvm::endianness ByteOrder,
                                      StringRef Bytes, const uint64_t BaseAddr,
                                      std::string ExpectedErrorMsg) {
   uint8_t AddressSize = 4;
-  DataExtractor Data(Bytes, ByteOrder == llvm::support::little, AddressSize);
+  DataExtractor Data(Bytes, ByteOrder == llvm::endianness::little, AddressSize);
   llvm::Expected<LineTable> Decoded = LineTable::decode(Data, BaseAddr);
   // Make sure decoding fails.
   ASSERT_FALSE((bool)Decoded);
@@ -777,7 +777,7 @@ static void TestLineTableDecodeError(llvm::endianness ByteOrder,
 TEST(GSYMTest, TestLineTableDecodeErrors) {
   // Test decoding InlineInfo objects that ensure we report an appropriate
   // error message.
-  const llvm::endianness ByteOrder = llvm::support::little;
+  const llvm::endianness ByteOrder = llvm::endianness::little;
   SmallString<512> Str;
   raw_svector_ostream OutStrm(Str);
   FileWriter FW(OutStrm, ByteOrder);
@@ -813,7 +813,7 @@ TEST(GSYMTest, TestLineTableDecodeErrors) {
 TEST(GSYMTest, TestLineTableEncodeErrors) {
   const uint64_t BaseAddr = 0x1000;
   const uint32_t FileIdx = 1;
-  const llvm::endianness ByteOrder = llvm::support::little;
+  const llvm::endianness ByteOrder = llvm::endianness::little;
   SmallString<512> Str;
   raw_svector_ostream OutStrm(Str);
   FileWriter FW(OutStrm, ByteOrder);
@@ -842,7 +842,7 @@ TEST(GSYMTest, TestLineTableEncodeErrors) {
 
 static void TestHeaderEncodeError(const Header &H,
                                   std::string ExpectedErrorMsg) {
-  const support::endianness ByteOrder = llvm::support::little;
+  const llvm::endianness ByteOrder = llvm::endianness::little;
   SmallString<512> Str;
   raw_svector_ostream OutStrm(Str);
   FileWriter FW(OutStrm, ByteOrder);
@@ -852,9 +852,9 @@ static void TestHeaderEncodeError(const Header &H,
 
 static void TestHeaderDecodeError(StringRef Bytes,
                                   std::string ExpectedErrorMsg) {
-  const support::endianness ByteOrder = llvm::support::little;
+  const llvm::endianness ByteOrder = llvm::endianness::little;
   uint8_t AddressSize = 4;
-  DataExtractor Data(Bytes, ByteOrder == llvm::support::little, AddressSize);
+  DataExtractor Data(Bytes, ByteOrder == llvm::endianness::little, AddressSize);
   llvm::Expected<Header> Decoded = Header::decode(Data);
   // Make sure decoding fails.
   ASSERT_FALSE((bool)Decoded);
@@ -897,7 +897,7 @@ TEST(GSYMTest, TestHeaderEncodeErrors) {
 }
 
 TEST(GSYMTest, TestHeaderDecodeErrors) {
-  const llvm::endianness ByteOrder = llvm::support::little;
+  const llvm::endianness ByteOrder = llvm::endianness::little;
   SmallString<512> Str;
   raw_svector_ostream OutStrm(Str);
   FileWriter FW(OutStrm, ByteOrder);
@@ -919,7 +919,7 @@ TEST(GSYMTest, TestHeaderDecodeErrors) {
 }
 
 static void TestHeaderEncodeDecode(const Header &H,
-                                   support::endianness ByteOrder) {
+                                   llvm::endianness ByteOrder) {
   uint8_t AddressSize = 4;
   SmallString<512> Str;
   raw_svector_ostream OutStrm(Str);
@@ -927,18 +927,17 @@ static void TestHeaderEncodeDecode(const Header &H,
   llvm::Error Err = H.encode(FW);
   ASSERT_FALSE(Err);
   std::string Bytes(OutStrm.str());
-  DataExtractor Data(Bytes, ByteOrder == llvm::support::little, AddressSize);
+  DataExtractor Data(Bytes, ByteOrder == llvm::endianness::little, AddressSize);
   llvm::Expected<Header> Decoded = Header::decode(Data);
   // Make sure decoding succeeded.
   ASSERT_TRUE((bool)Decoded);
   EXPECT_EQ(H, Decoded.get());
-
 }
 TEST(GSYMTest, TestHeaderEncodeDecode) {
   Header H;
   InitHeader(H);
-  TestHeaderEncodeDecode(H, llvm::support::little);
-  TestHeaderEncodeDecode(H, llvm::support::big);
+  TestHeaderEncodeDecode(H, llvm::endianness::little);
+  TestHeaderEncodeDecode(H, llvm::endianness::big);
 }
 
 static void TestGsymCreatorEncodeError(llvm::endianness ByteOrder,
@@ -961,7 +960,7 @@ TEST(GSYMTest, TestGsymCreatorEncodeErrors) {
   // function infos. We shouldn't be saving a GSYM file in this case since
   // there is nothing inside of it.
   GsymCreator GC;
-  TestGsymCreatorEncodeError(llvm::support::little, GC,
+  TestGsymCreatorEncodeError(llvm::endianness::little, GC,
                              "no functions to encode");
   const uint64_t FuncAddr = 0x1000;
   const uint64_t FuncSize = 0x100;
@@ -969,7 +968,7 @@ TEST(GSYMTest, TestGsymCreatorEncodeErrors) {
   // Verify we get an error trying to encode a GsymCreator that isn't
   // finalized.
   GC.addFunctionInfo(FunctionInfo(FuncAddr, FuncSize, FuncName));
-  TestGsymCreatorEncodeError(llvm::support::little, GC,
+  TestGsymCreatorEncodeError(llvm::endianness::little, GC,
                              "GsymCreator wasn't finalized prior to encoding");
   std::string finalizeIssues;
   raw_string_ostream OS(finalizeIssues);
@@ -981,7 +980,7 @@ TEST(GSYMTest, TestGsymCreatorEncodeErrors) {
   // Verify we get an error trying to encode a GsymCreator with a UUID that is
   // too long.
   GC.setUUID(InvalidUUID);
-  TestGsymCreatorEncodeError(llvm::support::little, GC,
+  TestGsymCreatorEncodeError(llvm::endianness::little, GC,
                              "invalid UUID size 21");
   GC.setUUID(ValidUUID);
   // Verify errors are propagated when we try to encoding an invalid line
@@ -990,7 +989,7 @@ TEST(GSYMTest, TestGsymCreatorEncodeErrors) {
     FI.OptLineTable = LineTable(); // Invalid line table.
     return false; // Stop iterating
   });
-  TestGsymCreatorEncodeError(llvm::support::little, GC,
+  TestGsymCreatorEncodeError(llvm::endianness::little, GC,
                              "attempted to encode invalid LineTable object");
   // Verify errors are propagated when we try to encoding an invalid inline
   // info.
@@ -999,7 +998,7 @@ TEST(GSYMTest, TestGsymCreatorEncodeErrors) {
     FI.Inline = InlineInfo(); // Invalid InlineInfo.
     return false; // Stop iterating
   });
-  TestGsymCreatorEncodeError(llvm::support::little, GC,
+  TestGsymCreatorEncodeError(llvm::endianness::little, GC,
                              "attempted to encode invalid InlineInfo object");
 }
 
@@ -1014,10 +1013,10 @@ static void Compare(const GsymCreator &GC, const GsymReader &GR) {
   });
 }
 
-static void TestEncodeDecode(const GsymCreator &GC,
-                             support::endianness ByteOrder, uint16_t Version,
-                             uint8_t AddrOffSize, uint64_t BaseAddress,
-                             uint32_t NumAddresses, ArrayRef<uint8_t> UUID) {
+static void TestEncodeDecode(const GsymCreator &GC, llvm::endianness ByteOrder,
+                             uint16_t Version, uint8_t AddrOffSize,
+                             uint64_t BaseAddress, uint32_t NumAddresses,
+                             ArrayRef<uint8_t> UUID) {
   SmallString<512> Str;
   raw_svector_ostream OutStrm(Str);
   FileWriter FW(OutStrm, ByteOrder);
@@ -1047,15 +1046,11 @@ TEST(GSYMTest, TestGsymCreator1ByteAddrOffsets) {
   GC.addFunctionInfo(FunctionInfo(BaseAddr+0x20, 0x10, Func2Name));
   Error Err = GC.finalize(llvm::nulls());
   ASSERT_FALSE(Err);
-  TestEncodeDecode(GC, llvm::support::little,
-                   GSYM_VERSION,
-                   AddrOffSize,
+  TestEncodeDecode(GC, llvm::endianness::little, GSYM_VERSION, AddrOffSize,
                    BaseAddr,
                    2, // NumAddresses
                    ArrayRef<uint8_t>(UUID));
-  TestEncodeDecode(GC, llvm::support::big,
-                   GSYM_VERSION,
-                   AddrOffSize,
+  TestEncodeDecode(GC, llvm::endianness::big, GSYM_VERSION, AddrOffSize,
                    BaseAddr,
                    2, // NumAddresses
                    ArrayRef<uint8_t>(UUID));
@@ -1073,15 +1068,11 @@ TEST(GSYMTest, TestGsymCreator2ByteAddrOffsets) {
   GC.addFunctionInfo(FunctionInfo(BaseAddr+0x200, 0x100, Func2Name));
   Error Err = GC.finalize(llvm::nulls());
   ASSERT_FALSE(Err);
-  TestEncodeDecode(GC, llvm::support::little,
-                   GSYM_VERSION,
-                   AddrOffSize,
+  TestEncodeDecode(GC, llvm::endianness::little, GSYM_VERSION, AddrOffSize,
                    BaseAddr,
                    2, // NumAddresses
                    ArrayRef<uint8_t>(UUID));
-  TestEncodeDecode(GC, llvm::support::big,
-                   GSYM_VERSION,
-                   AddrOffSize,
+  TestEncodeDecode(GC, llvm::endianness::big, GSYM_VERSION, AddrOffSize,
                    BaseAddr,
                    2, // NumAddresses
                    ArrayRef<uint8_t>(UUID));
@@ -1099,15 +1090,11 @@ TEST(GSYMTest, TestGsymCreator4ByteAddrOffsets) {
   GC.addFunctionInfo(FunctionInfo(BaseAddr+0x20000, 0x100, Func2Name));
   Error Err = GC.finalize(llvm::nulls());
   ASSERT_FALSE(Err);
-  TestEncodeDecode(GC, llvm::support::little,
-                   GSYM_VERSION,
-                   AddrOffSize,
+  TestEncodeDecode(GC, llvm::endianness::little, GSYM_VERSION, AddrOffSize,
                    BaseAddr,
                    2, // NumAddresses
                    ArrayRef<uint8_t>(UUID));
-  TestEncodeDecode(GC, llvm::support::big,
-                   GSYM_VERSION,
-                   AddrOffSize,
+  TestEncodeDecode(GC, llvm::endianness::big, GSYM_VERSION, AddrOffSize,
                    BaseAddr,
                    2, // NumAddresses
                    ArrayRef<uint8_t>(UUID));
@@ -1125,15 +1112,11 @@ TEST(GSYMTest, TestGsymCreator8ByteAddrOffsets) {
   GC.addFunctionInfo(FunctionInfo(BaseAddr+0x100000000, 0x100, Func2Name));
   Error Err = GC.finalize(llvm::nulls());
   ASSERT_FALSE(Err);
-  TestEncodeDecode(GC, llvm::support::little,
-                   GSYM_VERSION,
-                   AddrOffSize,
+  TestEncodeDecode(GC, llvm::endianness::little, GSYM_VERSION, AddrOffSize,
                    BaseAddr,
                    2, // NumAddresses
                    ArrayRef<uint8_t>(UUID));
-  TestEncodeDecode(GC, llvm::support::big,
-                   GSYM_VERSION,
-                   AddrOffSize,
+  TestEncodeDecode(GC, llvm::endianness::big, GSYM_VERSION, AddrOffSize,
                    BaseAddr,
                    2, // NumAddresses
                    ArrayRef<uint8_t>(UUID));
@@ -2450,11 +2433,12 @@ TEST(GSYMTest, TestGsymCreatorMultipleSymbolsWithNoSize) {
   GC.addFunctionInfo(FunctionInfo(BaseAddr, 0, Func2Name));
   Error Err = GC.finalize(llvm::nulls());
   ASSERT_FALSE(Err);
-  TestEncodeDecode(GC, llvm::support::little, GSYM_VERSION, AddrOffSize,
+  TestEncodeDecode(GC, llvm::endianness::little, GSYM_VERSION, AddrOffSize,
                    BaseAddr,
                    1, // NumAddresses
                    ArrayRef<uint8_t>(UUID));
-  TestEncodeDecode(GC, llvm::support::big, GSYM_VERSION, AddrOffSize, BaseAddr,
+  TestEncodeDecode(GC, llvm::endianness::big, GSYM_VERSION, AddrOffSize,
+                   BaseAddr,
                    1, // NumAddresses
                    ArrayRef<uint8_t>(UUID));
 }
@@ -4021,3 +4005,155 @@ TEST(GSYMTest, TestEmptyRangeWarnings) {
   // Make sure we don't see spurious errors in the output:
   EXPECT_TRUE(errors.find("error:") == std::string::npos);
 }
+
+
+TEST(GSYMTest, TestEmptyLinkageName) {
+  // This example has a single compile unit that has a DW_TAG_subprogram that
+  // has a function that has an empty linkage name and a valid normal name.
+  // Previously this would cause an encoding error:
+  //
+  // DWARF conversion failed: attempted to encode invalid FunctionInfo object
+  //
+  // This was because we would get a valid but empty linkage name and we would
+  // try to use this in the GSYM FunctionInfo and that would cause the error
+  // as the name was empty.
+  //
+  // 0x0000000b: DW_TAG_compile_unit
+  //               DW_AT_name        ("/tmp/main.cpp")
+  //               DW_AT_language    (DW_LANG_C)
+  //               DW_AT_stmt_list   (0x00000000)
+  //
+  // 0x00000015:   DW_TAG_subprogram
+  //                 DW_AT_name      ("foo")
+  //                 DW_AT_linkage_name      ("")
+  //                 DW_AT_low_pc    (0x0000000000001000)
+  //                 DW_AT_high_pc   (0x0000000000001050)
+  //
+  // 0x0000002e:   NULL
+
+
+  StringRef yamldata = R"(
+  debug_str:
+    - ''
+    - '/tmp/main.cpp'
+    - foo
+    - ''
+  debug_abbrev:
+    - ID:              0
+      Table:
+        - Code:            0x1
+          Tag:             DW_TAG_compile_unit
+          Children:        DW_CHILDREN_yes
+          Attributes:
+            - Attribute:       DW_AT_name
+              Form:            DW_FORM_strp
+            - Attribute:       DW_AT_language
+              Form:            DW_FORM_udata
+            - Attribute:       DW_AT_stmt_list
+              Form:            DW_FORM_sec_offset
+        - Code:            0x2
+          Tag:             DW_TAG_subprogram
+          Children:        DW_CHILDREN_no
+          Attributes:
+            - Attribute:       DW_AT_name
+              Form:            DW_FORM_strp
+            - Attribute:       DW_AT_linkage_name
+              Form:            DW_FORM_strp
+            - Attribute:       DW_AT_low_pc
+              Form:            DW_FORM_addr
+            - Attribute:       DW_AT_high_pc
+              Form:            DW_FORM_addr
+  debug_info:
+    - Length:          0x2B
+      Version:         4
+      AbbrevTableID:   0
+      AbbrOffset:      0x0
+      AddrSize:        8
+      Entries:
+        - AbbrCode:        0x1
+          Values:
+            - Value:           0x1
+            - Value:           0x2
+            - Value:           0x0
+        - AbbrCode:        0x2
+          Values:
+            - Value:           0xF
+            - Value:           0x13
+            - Value:           0x1000
+            - Value:           0x1050
+        - AbbrCode:        0x0
+  debug_line:
+    - Length:          68
+      Version:         2
+      PrologueLength:  36
+      MinInstLength:   1
+      DefaultIsStmt:   1
+      LineBase:        251
+      LineRange:       14
+      OpcodeBase:      13
+      StandardOpcodeLengths: [ 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1 ]
+      IncludeDirs:
+        - '/tmp'
+      Files:
+        - Name:            main.cpp
+          DirIdx:          1
+          ModTime:         0
+          Length:          0
+      Opcodes:
+        - Opcode:          DW_LNS_extended_op
+          ExtLen:          9
+          SubOpcode:       DW_LNE_set_address
+          Data:            4096
+        - Opcode:          DW_LNS_advance_line
+          SData:           9
+          Data:            0
+        - Opcode:          DW_LNS_copy
+          Data:            0
+        - Opcode:          DW_LNS_advance_pc
+          Data:            256
+        - Opcode:          DW_LNS_advance_line
+          SData:           1
+          Data:            0
+        - Opcode:          DW_LNS_copy
+          Data:            0
+        - Opcode:          DW_LNS_advance_pc
+          Data:            256
+        - Opcode:          DW_LNS_extended_op
+          ExtLen:          1
+          SubOpcode:       DW_LNE_end_sequence
+          Data:            0
+  )";
+  auto ErrOrSections = DWARFYAML::emitDebugSections(yamldata);
+  ASSERT_THAT_EXPECTED(ErrOrSections, Succeeded());
+  std::unique_ptr<DWARFContext> DwarfContext =
+      DWARFContext::create(*ErrOrSections, 8);
+  ASSERT_TRUE(DwarfContext.get() != nullptr);
+  std::string errors;
+  raw_string_ostream OS(errors);
+  GsymCreator GC;
+  DwarfTransformer DT(*DwarfContext, GC);
+  const uint32_t ThreadCount = 1;
+  ASSERT_THAT_ERROR(DT.convert(ThreadCount, &OS), Succeeded());
+  ASSERT_THAT_ERROR(GC.finalize(OS), Succeeded());
+  OS.flush();
+  SmallString<512> Str;
+  raw_svector_ostream OutStrm(Str);
+  const auto ByteOrder = llvm::endianness::native;
+  FileWriter FW(OutStrm, ByteOrder);
+  ASSERT_THAT_ERROR(GC.encode(FW), Succeeded());
+  Expected<GsymReader> GR = GsymReader::copyBuffer(OutStrm.str());
+  ASSERT_THAT_EXPECTED(GR, Succeeded());
+  // There should be one function in our GSYM.
+  EXPECT_EQ(GR->getNumAddresses(), 1u);
+  // Verify "foo" is present and has a line table and no inline info.
+  auto ExpFI = GR->getFunctionInfo(0x1000);
+  ASSERT_THAT_EXPECTED(ExpFI, Succeeded());
+  ASSERT_EQ(ExpFI->Range, AddressRange(0x1000, 0x1050));
+  EXPECT_TRUE(ExpFI->OptLineTable.has_value());
+  EXPECT_FALSE(ExpFI->Inline.has_value());
+  StringRef FuncName = GR->getString(ExpFI->Name);
+  EXPECT_EQ(FuncName, "foo");
+
+  // Make sure we don't see spurious errors in the output:
+  EXPECT_TRUE(errors.find("error:") == std::string::npos);
+}
diff --git a/llvm/unittests/DebugInfo/MSF/MappedBlockStreamTest.cpp b/llvm/unittests/DebugInfo/MSF/MappedBlockStreamTest.cpp
index 60fb14dd91..d1f04e9b28 100644
--- a/llvm/unittests/DebugInfo/MSF/MappedBlockStreamTest.cpp
+++ b/llvm/unittests/DebugInfo/MSF/MappedBlockStreamTest.cpp
@@ -34,7 +34,9 @@ class DiscontiguousStream : public WritableBinaryStream {
   uint32_t block_size() const { return 1; }
   uint32_t block_count() const { return Blocks.size(); }
 
-  endianness getEndian() const override { return little; }
+  llvm::endianness getEndian() const override {
+    return llvm::endianness::little;
+  }
 
   Error readBytes(uint64_t Offset, uint64_t Size,
                   ArrayRef<uint8_t> &Buffer) override {
@@ -412,7 +414,7 @@ TEST(MappedBlockStreamTest, TestWriteContiguousStreamRef) {
       F.block_size(), F.layout(), F, F.Allocator);
 
   // First write "Test Str" into the source stream.
-  MutableBinaryByteStream SourceStream(SrcData, little);
+  MutableBinaryByteStream SourceStream(SrcData, llvm::endianness::little);
   BinaryStreamWriter SourceWriter(SourceStream);
   EXPECT_THAT_ERROR(SourceWriter.writeCString("Test Str"), Succeeded());
   EXPECT_EQ(SrcDataBytes, std::vector<uint8_t>(
@@ -511,7 +513,7 @@ TEST(MappedBlockStreamTest, CreateFpmStream) {
   constexpr uint32_t NumFileBlocks = 4096 * 4;
 
   std::vector<uint8_t> MsfBuffer(NumFileBlocks * SB.BlockSize);
-  MutableBinaryByteStream MsfStream(MsfBuffer, llvm::support::little);
+  MutableBinaryByteStream MsfStream(MsfBuffer, llvm::endianness::little);
 
   SB.NumBlocks = NumFileBlocks;
   auto FpmStream =
diff --git a/llvm/unittests/DebugInfo/PDB/HashTableTest.cpp b/llvm/unittests/DebugInfo/PDB/HashTableTest.cpp
index 57f0dcf23d..6d17332f49 100644
--- a/llvm/unittests/DebugInfo/PDB/HashTableTest.cpp
+++ b/llvm/unittests/DebugInfo/PDB/HashTableTest.cpp
@@ -147,7 +147,7 @@ TEST(HashTableTest, Serialization) {
   }
 
   std::vector<uint8_t> Buffer(Table.calculateSerializedLength());
-  MutableBinaryByteStream Stream(Buffer, little);
+  MutableBinaryByteStream Stream(Buffer, llvm::endianness::little);
   BinaryStreamWriter Writer(Stream);
   EXPECT_THAT_ERROR(Table.commit(Writer), Succeeded());
   // We should have written precisely the number of bytes we calculated earlier.
@@ -251,7 +251,7 @@ TEST(HashTableTest, NonTrivialValueType) {
   }
 
   std::vector<uint8_t> Buffer(Table.calculateSerializedLength());
-  MutableBinaryByteStream Stream(Buffer, little);
+  MutableBinaryByteStream Stream(Buffer, llvm::endianness::little);
   BinaryStreamWriter Writer(Stream);
   EXPECT_THAT_ERROR(Table.commit(Writer), Succeeded());
   // We should have written precisely the number of bytes we calculated earlier.
diff --git a/llvm/unittests/DebugInfo/PDB/StringTableBuilderTest.cpp b/llvm/unittests/DebugInfo/PDB/StringTableBuilderTest.cpp
index 2f7c061944..1253f7c7ea 100644
--- a/llvm/unittests/DebugInfo/PDB/StringTableBuilderTest.cpp
+++ b/llvm/unittests/DebugInfo/PDB/StringTableBuilderTest.cpp
@@ -47,12 +47,12 @@ TEST(StringTableBuilderTest, Simple) {
   EXPECT_EQ(6U, Distinct.size());
 
   std::vector<uint8_t> Buffer(Builder.calculateSerializedSize());
-  MutableBinaryByteStream OutStream(Buffer, little);
+  MutableBinaryByteStream OutStream(Buffer, llvm::endianness::little);
   BinaryStreamWriter Writer(OutStream);
   EXPECT_THAT_ERROR(Builder.commit(Writer), Succeeded());
 
   // Reads the contents back.
-  BinaryByteStream InStream(Buffer, little);
+  BinaryByteStream InStream(Buffer, llvm::endianness::little);
   BinaryStreamReader Reader(InStream);
   PDBStringTable Table;
   EXPECT_THAT_ERROR(Table.reload(Reader), Succeeded());
diff --git a/llvm/unittests/DebugInfo/Symbolizer/MarkupTest.cpp b/llvm/unittests/DebugInfo/Symbolizer/MarkupTest.cpp
index c91d32b9a3..a93d9deb2f 100644
--- a/llvm/unittests/DebugInfo/Symbolizer/MarkupTest.cpp
+++ b/llvm/unittests/DebugInfo/Symbolizer/MarkupTest.cpp
@@ -9,7 +9,6 @@
 
 #include "llvm/DebugInfo/Symbolize/Markup.h"
 
-#include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/FormatVariadic.h"
 
diff --git a/llvm/unittests/ExecutionEngine/JITLink/AArch32Tests.cpp b/llvm/unittests/ExecutionEngine/JITLink/AArch32Tests.cpp
index 3f581445a2..dcc8d3b237 100644
--- a/llvm/unittests/ExecutionEngine/JITLink/AArch32Tests.cpp
+++ b/llvm/unittests/ExecutionEngine/JITLink/AArch32Tests.cpp
@@ -110,9 +110,11 @@ TEST(AArch32_Relocations, Thumb_Call_J1J2) {
   constexpr HalfWords ImmMask = FixupInfo<Thumb_Call>::ImmMask;
 
   static std::array<HalfWords, 3> MemPresets{
-      makeHalfWords<little>({0xff, 0xf7, 0xfe, 0xef}), // common
-      makeHalfWords<little>({0x00, 0x00, 0x00, 0x00}), // zeros
-      makeHalfWords<little>({0xff, 0xff, 0xff, 0xff}), // ones
+      makeHalfWords<llvm::endianness::little>(
+          {0xff, 0xf7, 0xfe, 0xef}), // common
+      makeHalfWords<llvm::endianness::little>(
+          {0x00, 0x00, 0x00, 0x00}), // zeros
+      makeHalfWords<llvm::endianness::little>({0xff, 0xff, 0xff, 0xff}), // ones
   };
 
   auto EncodeDecode = [ImmMask](int64_t In, MutableHalfWords &Mem) {
@@ -146,9 +148,11 @@ TEST(AArch32_Relocations, Thumb_Call_Bare) {
   constexpr HalfWords ImmMask = FixupInfo<Thumb_Call>::ImmMask;
 
   static std::array<HalfWords, 3> MemPresets{
-      makeHalfWords<little>({0xff, 0xf7, 0xfe, 0xef}), // common
-      makeHalfWords<little>({0x00, 0x00, 0x00, 0x00}), // zeros
-      makeHalfWords<little>({0xff, 0xff, 0xff, 0xff}), // ones
+      makeHalfWords<llvm::endianness::little>(
+          {0xff, 0xf7, 0xfe, 0xef}), // common
+      makeHalfWords<llvm::endianness::little>(
+          {0x00, 0x00, 0x00, 0x00}), // zeros
+      makeHalfWords<llvm::endianness::little>({0xff, 0xff, 0xff, 0xff}), // ones
   };
 
   auto EncodeDecode = [ImmMask](int64_t In, MutableHalfWords &Mem) {
@@ -217,9 +221,11 @@ TEST(AArch32_Relocations, Thumb_MovtAbs) {
 
   static std::array<uint8_t, 3> Registers{0, 5, 12};
   static std::array<HalfWords, 3> MemPresets{
-      makeHalfWords<little>({0xff, 0xf7, 0xfe, 0xef}), // common
-      makeHalfWords<little>({0x00, 0x00, 0x00, 0x00}), // zeros
-      makeHalfWords<little>({0xff, 0xff, 0xff, 0xff}), // ones
+      makeHalfWords<llvm::endianness::little>(
+          {0xff, 0xf7, 0xfe, 0xef}), // common
+      makeHalfWords<llvm::endianness::little>(
+          {0x00, 0x00, 0x00, 0x00}), // zeros
+      makeHalfWords<llvm::endianness::little>({0xff, 0xff, 0xff, 0xff}), // ones
   };
 
   auto EncodeDecode = [ImmMask](uint32_t In, MutableHalfWords &Mem) {
diff --git a/llvm/unittests/ExecutionEngine/JITLink/JITLinkMocks.cpp b/llvm/unittests/ExecutionEngine/JITLink/JITLinkMocks.cpp
index 158333b106..c40ce7adb0 100644
--- a/llvm/unittests/ExecutionEngine/JITLink/JITLinkMocks.cpp
+++ b/llvm/unittests/ExecutionEngine/JITLink/JITLinkMocks.cpp
@@ -52,7 +52,8 @@ TEST(JITLinkMocks, SmokeTest) {
   // Check that the testing infrastructure defaults can "link" a graph
   // successfully.
   auto G = std::make_unique<LinkGraph>("foo", Triple("x86_64-apple-darwin"), 8,
-                                       support::little, getGenericEdgeKindName);
+                                       llvm::endianness::little,
+                                       getGenericEdgeKindName);
 
   ArrayRef<char> Content = "hello, world!";
   auto &Sec =
diff --git a/llvm/unittests/ExecutionEngine/JITLink/LinkGraphTests.cpp b/llvm/unittests/ExecutionEngine/JITLink/LinkGraphTests.cpp
index 711f35fc76..a94ad0859e 100644
--- a/llvm/unittests/ExecutionEngine/JITLink/LinkGraphTests.cpp
+++ b/llvm/unittests/ExecutionEngine/JITLink/LinkGraphTests.cpp
@@ -61,12 +61,12 @@ static ArrayRef<char> BlockContent(BlockContentBytes);
 
 TEST(LinkGraphTest, Construction) {
   // Check that LinkGraph construction works as expected.
-  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, support::little,
+  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
   EXPECT_EQ(G.getName(), "foo");
   EXPECT_EQ(G.getTargetTriple().str(), "x86_64-apple-darwin");
   EXPECT_EQ(G.getPointerSize(), 8U);
-  EXPECT_EQ(G.getEndianness(), support::little);
+  EXPECT_EQ(G.getEndianness(), llvm::endianness::little);
   EXPECT_TRUE(G.external_symbols().empty());
   EXPECT_TRUE(G.absolute_symbols().empty());
   EXPECT_TRUE(G.defined_symbols().empty());
@@ -75,7 +75,7 @@ TEST(LinkGraphTest, Construction) {
 
 TEST(LinkGraphTest, AddressAccess) {
   // Check that we can get addresses for blocks, symbols, and edges.
-  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, support::little,
+  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
 
   auto &Sec1 =
@@ -94,7 +94,7 @@ TEST(LinkGraphTest, AddressAccess) {
 
 TEST(LinkGraphTest, SectionEmpty) {
   // Check that Section::empty behaves as expected.
-  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, support::little,
+  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
   auto &Sec1 =
       G.createSection("__data.1", orc::MemProt::Read | orc::MemProt::Write);
@@ -112,7 +112,7 @@ TEST(LinkGraphTest, SectionEmpty) {
 
 TEST(LinkGraphTest, BlockAndSymbolIteration) {
   // Check that we can iterate over blocks within Sections and across sections.
-  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, support::little,
+  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
   auto &Sec1 =
       G.createSection("__data.1", orc::MemProt::Read | orc::MemProt::Write);
@@ -165,7 +165,7 @@ TEST(LinkGraphTest, BlockAndSymbolIteration) {
 
 TEST(LinkGraphTest, ContentAccessAndUpdate) {
   // Check that we can make a defined symbol external.
-  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, support::little,
+  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
   auto &Sec =
       G.createSection("__data", orc::MemProt::Read | orc::MemProt::Write);
@@ -254,7 +254,7 @@ TEST(LinkGraphTest, ContentAccessAndUpdate) {
 
 TEST(LinkGraphTest, MakeExternal) {
   // Check that we can make defined and absolute symbols external.
-  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, support::little,
+  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
   auto &Sec =
       G.createSection("__data", orc::MemProt::Read | orc::MemProt::Write);
@@ -324,7 +324,7 @@ TEST(LinkGraphTest, MakeExternal) {
 
 TEST(LinkGraphTest, MakeAbsolute) {
   // Check that we can make defined and external symbols absolute.
-  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, support::little,
+  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
   auto &Sec =
       G.createSection("__data", orc::MemProt::Read | orc::MemProt::Write);
@@ -393,7 +393,7 @@ TEST(LinkGraphTest, MakeAbsolute) {
 
 TEST(LinkGraphTest, MakeDefined) {
   // Check that we can make an external symbol defined.
-  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, support::little,
+  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
   auto &Sec =
       G.createSection("__data", orc::MemProt::Read | orc::MemProt::Write);
@@ -441,7 +441,7 @@ TEST(LinkGraphTest, MakeDefined) {
 
 TEST(LinkGraphTest, TransferDefinedSymbol) {
   // Check that we can transfer a defined symbol from one block to another.
-  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, support::little,
+  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
   auto &Sec =
       G.createSection("__data", orc::MemProt::Read | orc::MemProt::Write);
@@ -476,7 +476,7 @@ TEST(LinkGraphTest, TransferDefinedSymbol) {
 TEST(LinkGraphTest, TransferDefinedSymbolAcrossSections) {
   // Check that we can transfer a defined symbol from an existing block in one
   // section to another.
-  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, support::little,
+  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
   auto &Sec1 =
       G.createSection("__data.1", orc::MemProt::Read | orc::MemProt::Write);
@@ -510,7 +510,7 @@ TEST(LinkGraphTest, TransferDefinedSymbolAcrossSections) {
 TEST(LinkGraphTest, TransferBlock) {
   // Check that we can transfer a block (and all associated symbols) from one
   // section to another.
-  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, support::little,
+  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
   auto &Sec1 =
       G.createSection("__data.1", orc::MemProt::Read | orc::MemProt::Write);
@@ -558,7 +558,7 @@ TEST(LinkGraphTest, TransferBlock) {
 TEST(LinkGraphTest, MergeSections) {
   // Check that we can transfer a block (and all associated symbols) from one
   // section to another.
-  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, support::little,
+  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
   auto &Sec1 =
       G.createSection("__data.1", orc::MemProt::Read | orc::MemProt::Write);
@@ -644,7 +644,7 @@ TEST(LinkGraphTest, MergeSections) {
 
 TEST(LinkGraphTest, SplitBlock) {
   // Check that the LinkGraph::splitBlock test works as expected.
-  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, support::little,
+  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
   auto &Sec =
       G.createSection("__data", orc::MemProt::Read | orc::MemProt::Write);
@@ -740,7 +740,7 @@ TEST(LinkGraphTest, SplitBlock) {
 }
 
 TEST(LinkGraphTest, GraphAllocationMethods) {
-  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, support::little,
+  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
 
   // Test allocation of sized, uninitialized buffer.
@@ -761,7 +761,7 @@ TEST(LinkGraphTest, GraphAllocationMethods) {
 
 TEST(LinkGraphTest, IsCStringBlockTest) {
   // Check that the LinkGraph::splitBlock test works as expected.
-  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, support::little,
+  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
   auto &Sec =
       G.createSection("__data", orc::MemProt::Read | orc::MemProt::Write);
@@ -786,7 +786,7 @@ TEST(LinkGraphTest, IsCStringBlockTest) {
 
 TEST(LinkGraphTest, BasicLayoutHonorsNoAlloc) {
   // Check that BasicLayout honors NoAlloc.
-  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, support::little,
+  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
 
   // Create a regular section and block.
diff --git a/llvm/unittests/ExecutionEngine/JITLink/MemoryManagerErrorTests.cpp b/llvm/unittests/ExecutionEngine/JITLink/MemoryManagerErrorTests.cpp
index c6f4b962a0..f0f3dd117c 100644
--- a/llvm/unittests/ExecutionEngine/JITLink/MemoryManagerErrorTests.cpp
+++ b/llvm/unittests/ExecutionEngine/JITLink/MemoryManagerErrorTests.cpp
@@ -19,7 +19,8 @@ using namespace llvm::jitlink;
 TEST(MemoryManagerErrorTest, ErrorOnFirstAllocate) {
   // Check that we can get addresses for blocks, symbols, and edges.
   auto G = std::make_unique<LinkGraph>("foo", Triple("x86_64-apple-darwin"), 8,
-                                       support::little, getGenericEdgeKindName);
+                                       llvm::endianness::little,
+                                       getGenericEdgeKindName);
 
   ArrayRef<char> Content = "hello, world!";
   auto &Sec =
diff --git a/llvm/unittests/ExecutionEngine/JITLink/StubsTests.cpp b/llvm/unittests/ExecutionEngine/JITLink/StubsTests.cpp
index fb932e756c..e33aa63b5e 100644
--- a/llvm/unittests/ExecutionEngine/JITLink/StubsTests.cpp
+++ b/llvm/unittests/ExecutionEngine/JITLink/StubsTests.cpp
@@ -60,7 +60,7 @@ GenerateStub(LinkGraph &G, size_t PointerSize, Edge::Kind PointerEdgeKind) {
 TEST(StubsTest, StubsGeneration_x86_64) {
   const char PointerJumpStubContent[6] = {
       static_cast<char>(0xFFu), 0x25, 0x00, 0x00, 0x00, 0x00};
-  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, support::little,
+  LinkGraph G("foo", Triple("x86_64-apple-darwin"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
   auto [PointerSym, StubSym] = GenerateStub(G, 8U, x86_64::Pointer64);
 
@@ -80,7 +80,7 @@ TEST(StubsTest, StubsGeneration_aarch64) {
       0x10, 0x02, 0x40, (char)0xf9u, // LDR x16, [x16, <imm>@pageoff12]
       0x00, 0x02, 0x1f, (char)0xd6u  // BR  x16
   };
-  LinkGraph G("foo", Triple("aarch64-linux-gnu"), 8, support::little,
+  LinkGraph G("foo", Triple("aarch64-linux-gnu"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
   auto [PointerSym, StubSym] = GenerateStub(G, 8U, aarch64::Pointer64);
 
@@ -100,8 +100,8 @@ TEST(StubsTest, StubsGeneration_aarch64) {
 TEST(StubsTest, StubsGeneration_i386) {
   const char PointerJumpStubContent[6] = {
       static_cast<char>(0xFFu), 0x25, 0x00, 0x00, 0x00, 0x00};
-  LinkGraph G("foo", Triple("i386-unknown-linux-gnu"), 4, support::little,
-              getGenericEdgeKindName);
+  LinkGraph G("foo", Triple("i386-unknown-linux-gnu"), 4,
+              llvm::endianness::little, getGenericEdgeKindName);
   auto [PointerSym, StubSym] = GenerateStub(G, 4U, i386::Pointer32);
 
   EXPECT_EQ(std::distance(StubSym.getBlock().edges().begin(),
@@ -129,7 +129,7 @@ TEST(StubsTest, StubsGeneration_loongarch32) {
       0x00,
       0x4c // jr $t8
   };
-  LinkGraph G("foo", Triple("loongarch32"), 4, support::little,
+  LinkGraph G("foo", Triple("loongarch32"), 4, llvm::endianness::little,
               getGenericEdgeKindName);
   auto [PointerSym, StubSym] = GenerateStub(G, 4U, loongarch::Pointer32);
 
@@ -161,7 +161,7 @@ TEST(StubsTest, StubsGeneration_loongarch64) {
       0x00,
       0x4c // jr $t8
   };
-  LinkGraph G("foo", Triple("loongarch64"), 8, support::little,
+  LinkGraph G("foo", Triple("loongarch64"), 8, llvm::endianness::little,
               getGenericEdgeKindName);
   auto [PointerSym, StubSym] = GenerateStub(G, 8U, loongarch::Pointer64);
 
diff --git a/llvm/unittests/ExecutionEngine/Orc/ObjectLinkingLayerTest.cpp b/llvm/unittests/ExecutionEngine/Orc/ObjectLinkingLayerTest.cpp
index c943eaf3bd..91659240c9 100644
--- a/llvm/unittests/ExecutionEngine/Orc/ObjectLinkingLayerTest.cpp
+++ b/llvm/unittests/ExecutionEngine/Orc/ObjectLinkingLayerTest.cpp
@@ -39,9 +39,9 @@ class ObjectLinkingLayerTest : public testing::Test {
 };
 
 TEST_F(ObjectLinkingLayerTest, AddLinkGraph) {
-  auto G =
-      std::make_unique<LinkGraph>("foo", Triple("x86_64-apple-darwin"), 8,
-                                  support::little, x86_64::getEdgeKindName);
+  auto G = std::make_unique<LinkGraph>("foo", Triple("x86_64-apple-darwin"), 8,
+                                       llvm::endianness::little,
+                                       x86_64::getEdgeKindName);
 
   auto &Sec1 = G->createSection("__data", MemProt::Read | MemProt::Write);
   auto &B1 = G->createContentBlock(Sec1, BlockContent,
@@ -104,9 +104,9 @@ TEST_F(ObjectLinkingLayerTest, ClaimLateDefinedWeakSymbols) {
 
   ObjLinkingLayer.addPlugin(std::make_unique<TestPlugin>());
 
-  auto G =
-      std::make_unique<LinkGraph>("foo", Triple("x86_64-apple-darwin"), 8,
-                                  support::little, x86_64::getEdgeKindName);
+  auto G = std::make_unique<LinkGraph>("foo", Triple("x86_64-apple-darwin"), 8,
+                                       llvm::endianness::little,
+                                       x86_64::getEdgeKindName);
 
   auto &DataSec = G->createSection("__data", MemProt::Read | MemProt::Write);
   auto &DataBlock = G->createContentBlock(DataSec, BlockContent,
@@ -158,9 +158,9 @@ TEST_F(ObjectLinkingLayerTest, HandleErrorDuringPostAllocationPass) {
 
   ObjLinkingLayer.addPlugin(std::make_unique<TestPlugin>());
 
-  auto G =
-      std::make_unique<LinkGraph>("foo", Triple("x86_64-apple-darwin"), 8,
-                                  support::little, x86_64::getEdgeKindName);
+  auto G = std::make_unique<LinkGraph>("foo", Triple("x86_64-apple-darwin"), 8,
+                                       llvm::endianness::little,
+                                       x86_64::getEdgeKindName);
 
   auto &DataSec = G->createSection("__data", MemProt::Read | MemProt::Write);
   auto &DataBlock = G->createContentBlock(DataSec, BlockContent,
diff --git a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
index c60aca259d..97cfc33967 100644
--- a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
+++ b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
@@ -4033,7 +4033,9 @@ TEST_F(OpenMPIRBuilderTest, CreateTeams) {
   };
 
   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
-  Builder.restoreIP(OMPBuilder.createTeams(Builder, BodyGenCB));
+  Builder.restoreIP(OMPBuilder.createTeams(
+      Builder, BodyGenCB, /*NumTeamsLower=*/nullptr, /*NumTeamsUpper=*/nullptr,
+      /*ThreadLimit=*/nullptr, /*IfExpr=*/nullptr));
 
   OMPBuilder.finalize();
   Builder.CreateRetVoid();
@@ -4074,6 +4076,348 @@ TEST_F(OpenMPIRBuilderTest, CreateTeams) {
                      [](Instruction &inst) { return isa<ICmpInst>(&inst); }));
 }
 
+TEST_F(OpenMPIRBuilderTest, CreateTeamsWithThreadLimit) {
+  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
+  OpenMPIRBuilder OMPBuilder(*M);
+  OMPBuilder.initialize();
+  F->setName("func");
+  IRBuilder<> &Builder = OMPBuilder.Builder;
+  Builder.SetInsertPoint(BB);
+
+  Function *FakeFunction =
+      Function::Create(FunctionType::get(Builder.getVoidTy(), false),
+                       GlobalValue::ExternalLinkage, "fakeFunction", M.get());
+
+  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+    Builder.restoreIP(CodeGenIP);
+    Builder.CreateCall(FakeFunction, {});
+  };
+
+  // `F` has an argument - an integer, so we use that as the thread limit.
+  Builder.restoreIP(OMPBuilder.createTeams(/*=*/Builder, BodyGenCB,
+                                           /*NumTeamsLower=*/nullptr,
+                                           /*NumTeamsUpper=*/nullptr,
+                                           /*ThreadLimit=*/F->arg_begin(),
+                                           /*IfExpr=*/nullptr));
+
+  Builder.CreateRetVoid();
+  OMPBuilder.finalize();
+
+  ASSERT_FALSE(verifyModule(*M));
+
+  CallInst *PushNumTeamsCallInst =
+      findSingleCall(F, OMPRTL___kmpc_push_num_teams_51, OMPBuilder);
+  ASSERT_NE(PushNumTeamsCallInst, nullptr);
+
+  EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(2), Builder.getInt32(0));
+  EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(3), Builder.getInt32(0));
+  EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(4), &*F->arg_begin());
+
+  // Verifying that the next instruction to execute is kmpc_fork_teams
+  BranchInst *BrInst =
+      dyn_cast<BranchInst>(PushNumTeamsCallInst->getNextNonDebugInstruction());
+  ASSERT_NE(BrInst, nullptr);
+  ASSERT_EQ(BrInst->getNumSuccessors(), 1U);
+  Instruction *NextInstruction =
+      BrInst->getSuccessor(0)->getFirstNonPHIOrDbgOrLifetime();
+  CallInst *ForkTeamsCI = dyn_cast_if_present<CallInst>(NextInstruction);
+  ASSERT_NE(ForkTeamsCI, nullptr);
+  EXPECT_EQ(ForkTeamsCI->getCalledFunction(),
+            OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_teams));
+}
+
+TEST_F(OpenMPIRBuilderTest, CreateTeamsWithNumTeamsUpper) {
+  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
+  OpenMPIRBuilder OMPBuilder(*M);
+  OMPBuilder.initialize();
+  F->setName("func");
+  IRBuilder<> &Builder = OMPBuilder.Builder;
+  Builder.SetInsertPoint(BB);
+
+  Function *FakeFunction =
+      Function::Create(FunctionType::get(Builder.getVoidTy(), false),
+                       GlobalValue::ExternalLinkage, "fakeFunction", M.get());
+
+  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+    Builder.restoreIP(CodeGenIP);
+    Builder.CreateCall(FakeFunction, {});
+  };
+
+  // `F` already has an integer argument, so we use that as upper bound to
+  // `num_teams`
+  Builder.restoreIP(OMPBuilder.createTeams(Builder, BodyGenCB,
+                                           /*NumTeamsLower=*/nullptr,
+                                           /*NumTeamsUpper=*/F->arg_begin(),
+                                           /*ThreadLimit=*/nullptr,
+                                           /*IfExpr=*/nullptr));
+
+  Builder.CreateRetVoid();
+  OMPBuilder.finalize();
+
+  ASSERT_FALSE(verifyModule(*M));
+
+  CallInst *PushNumTeamsCallInst =
+      findSingleCall(F, OMPRTL___kmpc_push_num_teams_51, OMPBuilder);
+  ASSERT_NE(PushNumTeamsCallInst, nullptr);
+
+  EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(2), &*F->arg_begin());
+  EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(3), &*F->arg_begin());
+  EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(4), Builder.getInt32(0));
+
+  // Verifying that the next instruction to execute is kmpc_fork_teams
+  BranchInst *BrInst =
+      dyn_cast<BranchInst>(PushNumTeamsCallInst->getNextNonDebugInstruction());
+  ASSERT_NE(BrInst, nullptr);
+  ASSERT_EQ(BrInst->getNumSuccessors(), 1U);
+  Instruction *NextInstruction =
+      BrInst->getSuccessor(0)->getFirstNonPHIOrDbgOrLifetime();
+  CallInst *ForkTeamsCI = dyn_cast_if_present<CallInst>(NextInstruction);
+  ASSERT_NE(ForkTeamsCI, nullptr);
+  EXPECT_EQ(ForkTeamsCI->getCalledFunction(),
+            OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_teams));
+}
+
+TEST_F(OpenMPIRBuilderTest, CreateTeamsWithNumTeamsBoth) {
+  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
+  OpenMPIRBuilder OMPBuilder(*M);
+  OMPBuilder.initialize();
+  F->setName("func");
+  IRBuilder<> &Builder = OMPBuilder.Builder;
+  Builder.SetInsertPoint(BB);
+
+  Function *FakeFunction =
+      Function::Create(FunctionType::get(Builder.getVoidTy(), false),
+                       GlobalValue::ExternalLinkage, "fakeFunction", M.get());
+
+  Value *NumTeamsLower =
+      Builder.CreateAdd(F->arg_begin(), Builder.getInt32(5), "numTeamsLower");
+  Value *NumTeamsUpper =
+      Builder.CreateAdd(F->arg_begin(), Builder.getInt32(10), "numTeamsUpper");
+
+  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+    Builder.restoreIP(CodeGenIP);
+    Builder.CreateCall(FakeFunction, {});
+  };
+
+  // `F` already has an integer argument, so we use that as upper bound to
+  // `num_teams`
+  Builder.restoreIP(
+      OMPBuilder.createTeams(Builder, BodyGenCB, NumTeamsLower, NumTeamsUpper,
+                             /*ThreadLimit=*/nullptr, /*IfExpr=*/nullptr));
+
+  Builder.CreateRetVoid();
+  OMPBuilder.finalize();
+
+  ASSERT_FALSE(verifyModule(*M));
+
+  CallInst *PushNumTeamsCallInst =
+      findSingleCall(F, OMPRTL___kmpc_push_num_teams_51, OMPBuilder);
+  ASSERT_NE(PushNumTeamsCallInst, nullptr);
+
+  EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(2), NumTeamsLower);
+  EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(3), NumTeamsUpper);
+  EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(4), Builder.getInt32(0));
+
+  // Verifying that the next instruction to execute is kmpc_fork_teams
+  BranchInst *BrInst =
+      dyn_cast<BranchInst>(PushNumTeamsCallInst->getNextNonDebugInstruction());
+  ASSERT_NE(BrInst, nullptr);
+  ASSERT_EQ(BrInst->getNumSuccessors(), 1U);
+  Instruction *NextInstruction =
+      BrInst->getSuccessor(0)->getFirstNonPHIOrDbgOrLifetime();
+  CallInst *ForkTeamsCI = dyn_cast_if_present<CallInst>(NextInstruction);
+  ASSERT_NE(ForkTeamsCI, nullptr);
+  EXPECT_EQ(ForkTeamsCI->getCalledFunction(),
+            OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_teams));
+}
+
+TEST_F(OpenMPIRBuilderTest, CreateTeamsWithNumTeamsAndThreadLimit) {
+  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
+  OpenMPIRBuilder OMPBuilder(*M);
+  OMPBuilder.initialize();
+  F->setName("func");
+  IRBuilder<> &Builder = OMPBuilder.Builder;
+  Builder.SetInsertPoint(BB);
+
+  BasicBlock *CodegenBB = splitBB(Builder, true);
+  Builder.SetInsertPoint(CodegenBB);
+
+  // Generate values for `num_teams` and `thread_limit` using the first argument
+  // of the testing function.
+  Value *NumTeamsLower =
+      Builder.CreateAdd(F->arg_begin(), Builder.getInt32(5), "numTeamsLower");
+  Value *NumTeamsUpper =
+      Builder.CreateAdd(F->arg_begin(), Builder.getInt32(10), "numTeamsUpper");
+  Value *ThreadLimit =
+      Builder.CreateAdd(F->arg_begin(), Builder.getInt32(20), "threadLimit");
+
+  Function *FakeFunction =
+      Function::Create(FunctionType::get(Builder.getVoidTy(), false),
+                       GlobalValue::ExternalLinkage, "fakeFunction", M.get());
+
+  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+    Builder.restoreIP(CodeGenIP);
+    Builder.CreateCall(FakeFunction, {});
+  };
+
+  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
+  Builder.restoreIP(OMPBuilder.createTeams(
+      Builder, BodyGenCB, NumTeamsLower, NumTeamsUpper, ThreadLimit, nullptr));
+
+  Builder.CreateRetVoid();
+  OMPBuilder.finalize();
+
+  ASSERT_FALSE(verifyModule(*M));
+
+  CallInst *PushNumTeamsCallInst =
+      findSingleCall(F, OMPRTL___kmpc_push_num_teams_51, OMPBuilder);
+  ASSERT_NE(PushNumTeamsCallInst, nullptr);
+
+  EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(2), NumTeamsLower);
+  EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(3), NumTeamsUpper);
+  EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(4), ThreadLimit);
+
+  // Verifying that the next instruction to execute is kmpc_fork_teams
+  BranchInst *BrInst =
+      dyn_cast<BranchInst>(PushNumTeamsCallInst->getNextNonDebugInstruction());
+  ASSERT_NE(BrInst, nullptr);
+  ASSERT_EQ(BrInst->getNumSuccessors(), 1U);
+  Instruction *NextInstruction =
+      BrInst->getSuccessor(0)->getFirstNonPHIOrDbgOrLifetime();
+  CallInst *ForkTeamsCI = dyn_cast_if_present<CallInst>(NextInstruction);
+  ASSERT_NE(ForkTeamsCI, nullptr);
+  EXPECT_EQ(ForkTeamsCI->getCalledFunction(),
+            OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_teams));
+}
+
+TEST_F(OpenMPIRBuilderTest, CreateTeamsWithIfCondition) {
+  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
+  OpenMPIRBuilder OMPBuilder(*M);
+  OMPBuilder.initialize();
+  F->setName("func");
+  IRBuilder<> &Builder = OMPBuilder.Builder;
+  Builder.SetInsertPoint(BB);
+
+  Value *IfExpr = Builder.CreateLoad(Builder.getInt1Ty(),
+                                     Builder.CreateAlloca(Builder.getInt1Ty()));
+
+  Function *FakeFunction =
+      Function::Create(FunctionType::get(Builder.getVoidTy(), false),
+                       GlobalValue::ExternalLinkage, "fakeFunction", M.get());
+
+  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+    Builder.restoreIP(CodeGenIP);
+    Builder.CreateCall(FakeFunction, {});
+  };
+
+  // `F` already has an integer argument, so we use that as upper bound to
+  // `num_teams`
+  Builder.restoreIP(OMPBuilder.createTeams(
+      Builder, BodyGenCB, /*NumTeamsLower=*/nullptr, /*NumTeamsUpper=*/nullptr,
+      /*ThreadLimit=*/nullptr, IfExpr));
+
+  Builder.CreateRetVoid();
+  OMPBuilder.finalize();
+
+  ASSERT_FALSE(verifyModule(*M));
+
+  CallInst *PushNumTeamsCallInst =
+      findSingleCall(F, OMPRTL___kmpc_push_num_teams_51, OMPBuilder);
+  ASSERT_NE(PushNumTeamsCallInst, nullptr);
+  Value *NumTeamsLower = PushNumTeamsCallInst->getArgOperand(2);
+  Value *NumTeamsUpper = PushNumTeamsCallInst->getArgOperand(3);
+  Value *ThreadLimit = PushNumTeamsCallInst->getArgOperand(4);
+
+  // Check the lower_bound
+  ASSERT_NE(NumTeamsLower, nullptr);
+  SelectInst *NumTeamsLowerSelectInst = dyn_cast<SelectInst>(NumTeamsLower);
+  ASSERT_NE(NumTeamsLowerSelectInst, nullptr);
+  EXPECT_EQ(NumTeamsLowerSelectInst->getCondition(), IfExpr);
+  EXPECT_EQ(NumTeamsLowerSelectInst->getTrueValue(), Builder.getInt32(0));
+  EXPECT_EQ(NumTeamsLowerSelectInst->getFalseValue(), Builder.getInt32(1));
+
+  // Check the upper_bound
+  ASSERT_NE(NumTeamsUpper, nullptr);
+  SelectInst *NumTeamsUpperSelectInst = dyn_cast<SelectInst>(NumTeamsUpper);
+  ASSERT_NE(NumTeamsUpperSelectInst, nullptr);
+  EXPECT_EQ(NumTeamsUpperSelectInst->getCondition(), IfExpr);
+  EXPECT_EQ(NumTeamsUpperSelectInst->getTrueValue(), Builder.getInt32(0));
+  EXPECT_EQ(NumTeamsUpperSelectInst->getFalseValue(), Builder.getInt32(1));
+
+  // Check thread_limit
+  EXPECT_EQ(ThreadLimit, Builder.getInt32(0));
+}
+
+TEST_F(OpenMPIRBuilderTest, CreateTeamsWithIfConditionAndNumTeams) {
+  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
+  OpenMPIRBuilder OMPBuilder(*M);
+  OMPBuilder.initialize();
+  F->setName("func");
+  IRBuilder<> &Builder = OMPBuilder.Builder;
+  Builder.SetInsertPoint(BB);
+
+  Value *IfExpr = Builder.CreateLoad(
+      Builder.getInt32Ty(), Builder.CreateAlloca(Builder.getInt32Ty()));
+  Value *NumTeamsLower = Builder.CreateAdd(F->arg_begin(), Builder.getInt32(5));
+  Value *NumTeamsUpper =
+      Builder.CreateAdd(F->arg_begin(), Builder.getInt32(10));
+  Value *ThreadLimit = Builder.CreateAdd(F->arg_begin(), Builder.getInt32(20));
+
+  Function *FakeFunction =
+      Function::Create(FunctionType::get(Builder.getVoidTy(), false),
+                       GlobalValue::ExternalLinkage, "fakeFunction", M.get());
+
+  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+    Builder.restoreIP(CodeGenIP);
+    Builder.CreateCall(FakeFunction, {});
+  };
+
+  // `F` already has an integer argument, so we use that as upper bound to
+  // `num_teams`
+  Builder.restoreIP(OMPBuilder.createTeams(Builder, BodyGenCB, NumTeamsLower,
+                                           NumTeamsUpper, ThreadLimit, IfExpr));
+
+  Builder.CreateRetVoid();
+  OMPBuilder.finalize();
+
+  ASSERT_FALSE(verifyModule(*M));
+
+  CallInst *PushNumTeamsCallInst =
+      findSingleCall(F, OMPRTL___kmpc_push_num_teams_51, OMPBuilder);
+  ASSERT_NE(PushNumTeamsCallInst, nullptr);
+  Value *NumTeamsLowerArg = PushNumTeamsCallInst->getArgOperand(2);
+  Value *NumTeamsUpperArg = PushNumTeamsCallInst->getArgOperand(3);
+  Value *ThreadLimitArg = PushNumTeamsCallInst->getArgOperand(4);
+
+  // Get the boolean conversion of if expression
+  ASSERT_EQ(IfExpr->getNumUses(), 1U);
+  User *IfExprInst = IfExpr->user_back();
+  ICmpInst *IfExprCmpInst = dyn_cast<ICmpInst>(IfExprInst);
+  ASSERT_NE(IfExprCmpInst, nullptr);
+  EXPECT_EQ(IfExprCmpInst->getPredicate(), ICmpInst::Predicate::ICMP_NE);
+  EXPECT_EQ(IfExprCmpInst->getOperand(0), IfExpr);
+  EXPECT_EQ(IfExprCmpInst->getOperand(1), Builder.getInt32(0));
+
+  // Check the lower_bound
+  ASSERT_NE(NumTeamsLowerArg, nullptr);
+  SelectInst *NumTeamsLowerSelectInst = dyn_cast<SelectInst>(NumTeamsLowerArg);
+  ASSERT_NE(NumTeamsLowerSelectInst, nullptr);
+  EXPECT_EQ(NumTeamsLowerSelectInst->getCondition(), IfExprCmpInst);
+  EXPECT_EQ(NumTeamsLowerSelectInst->getTrueValue(), NumTeamsLower);
+  EXPECT_EQ(NumTeamsLowerSelectInst->getFalseValue(), Builder.getInt32(1));
+
+  // Check the upper_bound
+  ASSERT_NE(NumTeamsUpperArg, nullptr);
+  SelectInst *NumTeamsUpperSelectInst = dyn_cast<SelectInst>(NumTeamsUpperArg);
+  ASSERT_NE(NumTeamsUpperSelectInst, nullptr);
+  EXPECT_EQ(NumTeamsUpperSelectInst->getCondition(), IfExprCmpInst);
+  EXPECT_EQ(NumTeamsUpperSelectInst->getTrueValue(), NumTeamsUpper);
+  EXPECT_EQ(NumTeamsUpperSelectInst->getFalseValue(), Builder.getInt32(1));
+
+  // Check thread_limit
+  EXPECT_EQ(ThreadLimitArg, ThreadLimit);
+}
+
 /// Returns the single instruction of InstTy type in BB that uses the value V.
 /// If there is more than one such instruction, returns null.
 template <typename InstTy>
diff --git a/llvm/unittests/MC/DwarfLineTableHeaders.cpp b/llvm/unittests/MC/DwarfLineTableHeaders.cpp
index ed3671aa6c..691d319f54 100644
--- a/llvm/unittests/MC/DwarfLineTableHeaders.cpp
+++ b/llvm/unittests/MC/DwarfLineTableHeaders.cpp
@@ -172,8 +172,8 @@ class DwarfLineTableHeaders : public ::testing::Test {
       Expected<StringRef> ContentsOrErr = Section.getContents();
       ASSERT_TRUE(static_cast<bool>(ContentsOrErr));
       StringRef Contents = *ContentsOrErr;
-      ASSERT_TRUE(Contents.find("dir") != StringRef::npos);
-      ASSERT_TRUE(Contents.find("file") != StringRef::npos);
+      ASSERT_TRUE(Contents.contains("dir"));
+      ASSERT_TRUE(Contents.contains("file"));
       ASSERT_TRUE(Contents.size() == 9);
       return;
     }
diff --git a/llvm/unittests/MC/StringTableBuilderTest.cpp b/llvm/unittests/MC/StringTableBuilderTest.cpp
index 91f2b3b97e..05f469a229 100644
--- a/llvm/unittests/MC/StringTableBuilderTest.cpp
+++ b/llvm/unittests/MC/StringTableBuilderTest.cpp
@@ -58,8 +58,8 @@ TEST(StringTableBuilderTest, BasicWinCOFF) {
 
   std::string Expected;
 
-  ExpectedSize =
-      support::endian::byte_swap<uint32_t, support::little>(ExpectedSize);
+  ExpectedSize = support::endian::byte_swap<uint32_t, llvm::endianness::little>(
+      ExpectedSize);
   Expected.append((const char*)&ExpectedSize, 4);
   Expected += "pygmy hippopotamus";
   Expected += '\x00';
diff --git a/llvm/unittests/ObjCopy/ObjCopyTest.cpp b/llvm/unittests/ObjCopy/ObjCopyTest.cpp
index db74dc3b7d..4382c73e88 100644
--- a/llvm/unittests/ObjCopy/ObjCopyTest.cpp
+++ b/llvm/unittests/ObjCopy/ObjCopyTest.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ObjCopy/ObjCopy.h"
-#include "llvm/ADT/SmallString.h"
 #include "llvm/ObjCopy/ConfigManager.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/ObjectYAML/yaml2obj.h"
diff --git a/llvm/unittests/ProfileData/InstrProfTest.cpp b/llvm/unittests/ProfileData/InstrProfTest.cpp
index 9c6d155521..494e3c18c8 100644
--- a/llvm/unittests/ProfileData/InstrProfTest.cpp
+++ b/llvm/unittests/ProfileData/InstrProfTest.cpp
@@ -844,13 +844,13 @@ TEST_P(MaybeSparseInstrProfTest, get_icall_data_read_write_big_endian) {
   Writer.addRecord({"callee3", 0x1235, {3, 4}}, Err);
 
   // Set big endian output.
-  Writer.setValueProfDataEndianness(support::big);
+  Writer.setValueProfDataEndianness(llvm::endianness::big);
 
   auto Profile = Writer.writeBuffer();
   readProfile(std::move(Profile));
 
   // Set big endian input.
-  Reader->setValueProfDataEndianness(support::big);
+  Reader->setValueProfDataEndianness(llvm::endianness::big);
 
   Expected<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
   EXPECT_THAT_ERROR(R.takeError(), Succeeded());
@@ -867,7 +867,7 @@ TEST_P(MaybeSparseInstrProfTest, get_icall_data_read_write_big_endian) {
   ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee1"));
 
   // Restore little endian default:
-  Writer.setValueProfDataEndianness(support::little);
+  Writer.setValueProfDataEndianness(llvm::endianness::little);
 }
 
 TEST_P(MaybeSparseInstrProfTest, get_icall_data_merge1) {
diff --git a/llvm/unittests/ProfileData/SampleProfTest.cpp b/llvm/unittests/ProfileData/SampleProfTest.cpp
index e01025440e..3abba47430 100644
--- a/llvm/unittests/ProfileData/SampleProfTest.cpp
+++ b/llvm/unittests/ProfileData/SampleProfTest.cpp
@@ -145,7 +145,7 @@ struct SampleProfTest : ::testing::Test {
 
     StringRef FooName("_Z3fooi");
     FunctionSamples FooSamples;
-    FooSamples.setName(FooName);
+    FooSamples.setFunction(FunctionId(FooName));
     FooSamples.addTotalSamples(7711);
     FooSamples.addHeadSamples(610);
     FooSamples.addBodySamples(1, 0, 610);
@@ -155,43 +155,43 @@ struct SampleProfTest : ::testing::Test {
     FooSamples.addBodySamples(10, 0, 605);
 
     // Add inline instance with name "_Z3gooi".
-    StringRef GooName("_Z3gooi");
+    FunctionId GooName(StringRef("_Z3gooi"));
     auto &GooSamples =
-        FooSamples.functionSamplesAt(LineLocation(7, 0))[GooName.str()];
-    GooSamples.setName(GooName);
+        FooSamples.functionSamplesAt(LineLocation(7, 0))[GooName];
+    GooSamples.setFunction(GooName);
     GooSamples.addTotalSamples(502);
     GooSamples.addBodySamples(3, 0, 502);
 
     // Add inline instance with name "_Z3hooi".
-    StringRef HooName("_Z3hooi");
+    FunctionId HooName(StringRef("_Z3hooi"));
     auto &HooSamples =
-        GooSamples.functionSamplesAt(LineLocation(9, 0))[HooName.str()];
-    HooSamples.setName(HooName);
+        GooSamples.functionSamplesAt(LineLocation(9, 0))[HooName];
+    HooSamples.setFunction(HooName);
     HooSamples.addTotalSamples(317);
     HooSamples.addBodySamples(4, 0, 317);
 
     StringRef BarName("_Z3bari");
     FunctionSamples BarSamples;
-    BarSamples.setName(BarName);
+    BarSamples.setFunction(FunctionId(BarName));
     BarSamples.addTotalSamples(20301);
     BarSamples.addHeadSamples(1437);
     BarSamples.addBodySamples(1, 0, 1437);
     // Test how reader/writer handles unmangled names.
     StringRef MconstructName("_M_construct<char *>");
     StringRef StringviewName("string_view<std::allocator<char> >");
-    BarSamples.addCalledTargetSamples(1, 0, MconstructName, 1000);
-    BarSamples.addCalledTargetSamples(1, 0, StringviewName, 437);
+    BarSamples.addCalledTargetSamples(1, 0, FunctionId(MconstructName), 1000);
+    BarSamples.addCalledTargetSamples(1, 0, FunctionId(StringviewName), 437);
 
     StringRef BazName("_Z3bazi");
     FunctionSamples BazSamples;
-    BazSamples.setName(BazName);
+    BazSamples.setFunction(FunctionId(BazName));
     BazSamples.addTotalSamples(12557);
     BazSamples.addHeadSamples(1257);
     BazSamples.addBodySamples(1, 0, 12557);
 
     StringRef BooName("_Z3booi");
     FunctionSamples BooSamples;
-    BooSamples.setName(BooName);
+    BooSamples.setFunction(FunctionId(BooName));
     BooSamples.addTotalSamples(1232);
     BooSamples.addHeadSamples(1);
     BooSamples.addBodySamples(1, 0, 1232);
@@ -210,8 +210,8 @@ struct SampleProfTest : ::testing::Test {
     if (Remap) {
       FooName = "_Z4fauxi";
       BarName = "_Z3barl";
-      GooName = "_Z3gool";
-      HooName = "_Z3hool";
+      GooName = FunctionId(StringRef("_Z3gool"));
+      HooName = FunctionId(StringRef("_Z3hool"));
     }
 
     M.getOrInsertFunction(FooName, fn_type);
@@ -245,7 +245,7 @@ struct SampleProfTest : ::testing::Test {
     FunctionSamples *ReadFooSamples = Reader->getSamplesFor(FooName);
     ASSERT_TRUE(ReadFooSamples != nullptr);
     if (!UseMD5) {
-      ASSERT_EQ("_Z3fooi", ReadFooSamples->getName());
+      ASSERT_EQ("_Z3fooi", ReadFooSamples->getFunction().str());
     }
     ASSERT_EQ(7711u, ReadFooSamples->getTotalSamples());
     ASSERT_EQ(610u, ReadFooSamples->getHeadSamples());
@@ -254,7 +254,8 @@ struct SampleProfTest : ::testing::Test {
     // inline instance for GooName. Test the correct FunctionSamples can be
     // found with Remapper support.
     const FunctionSamples *ReadGooSamples =
-        ReadFooSamples->findFunctionSamplesAt(LineLocation(7, 0), GooName,
+        ReadFooSamples->findFunctionSamplesAt(LineLocation(7, 0),
+                                              GooName.stringRef(),
                                               Reader->getRemapper());
     ASSERT_TRUE(ReadGooSamples != nullptr);
     ASSERT_EQ(502u, ReadGooSamples->getTotalSamples());
@@ -263,7 +264,8 @@ struct SampleProfTest : ::testing::Test {
     // no inline instance for GooName. Test no FunctionSamples will be
     // found with Remapper support.
     const FunctionSamples *ReadGooSamplesAgain =
-        ReadFooSamples->findFunctionSamplesAt(LineLocation(9, 0), GooName,
+        ReadFooSamples->findFunctionSamplesAt(LineLocation(9, 0),
+                                              GooName.stringRef(),
                                               Reader->getRemapper());
     ASSERT_TRUE(ReadGooSamplesAgain == nullptr);
 
@@ -272,7 +274,8 @@ struct SampleProfTest : ::testing::Test {
     // inline instance for HooName. Test the correct FunctionSamples can be
     // found with Remapper support.
     const FunctionSamples *ReadHooSamples =
-        ReadGooSamples->findFunctionSamplesAt(LineLocation(9, 0), HooName,
+        ReadGooSamples->findFunctionSamplesAt(LineLocation(9, 0),
+                                              HooName.stringRef(),
                                               Reader->getRemapper());
     ASSERT_TRUE(ReadHooSamples != nullptr);
     ASSERT_EQ(317u, ReadHooSamples->getTotalSamples());
@@ -280,7 +283,7 @@ struct SampleProfTest : ::testing::Test {
     FunctionSamples *ReadBarSamples = Reader->getSamplesFor(BarName);
     ASSERT_TRUE(ReadBarSamples != nullptr);
     if (!UseMD5) {
-      ASSERT_EQ("_Z3bari", ReadBarSamples->getName());
+      ASSERT_EQ("_Z3bari", ReadBarSamples->getFunction().str());
     }
     ASSERT_EQ(20301u, ReadBarSamples->getTotalSamples());
     ASSERT_EQ(1437u, ReadBarSamples->getHeadSamples());
@@ -304,13 +307,9 @@ struct SampleProfTest : ::testing::Test {
     FunctionSamples *ReadBooSamples = Reader->getSamplesFor(BooName);
     ASSERT_TRUE(ReadBooSamples != nullptr);
     ASSERT_EQ(1232u, ReadBooSamples->getTotalSamples());
-
-    std::string MconstructGUID;
-    StringRef MconstructRep =
-        getRepInFormat(MconstructName, UseMD5, MconstructGUID);
-    std::string StringviewGUID;
-    StringRef StringviewRep =
-        getRepInFormat(StringviewName, UseMD5, StringviewGUID);
+    
+    FunctionId MconstructRep = getRepInFormat(MconstructName);
+    FunctionId StringviewRep = getRepInFormat(StringviewName);
     ASSERT_EQ(1000u, CTMap.get()[MconstructRep]);
     ASSERT_EQ(437u, CTMap.get()[StringviewRep]);
 
@@ -333,7 +332,7 @@ struct SampleProfTest : ::testing::Test {
                           uint64_t TotalSamples, uint64_t HeadSamples) {
     StringRef Name(Fname);
     FunctionSamples FcnSamples;
-    FcnSamples.setName(Name);
+    FcnSamples.setFunction(FunctionId(Name));
     FcnSamples.addTotalSamples(TotalSamples);
     FcnSamples.addHeadSamples(HeadSamples);
     FcnSamples.addBodySamples(1, 0, HeadSamples);
diff --git a/llvm/unittests/Support/ARMAttributeParser.cpp b/llvm/unittests/Support/ARMAttributeParser.cpp
index ef3da3fc75..4bde0eeb95 100644
--- a/llvm/unittests/Support/ARMAttributeParser.cpp
+++ b/llvm/unittests/Support/ARMAttributeParser.cpp
@@ -35,7 +35,7 @@ bool testBuildAttr(unsigned Tag, unsigned Value,
     reinterpret_cast<const uint8_t*>(OS.str().c_str()), OS.str().size());
 
   ARMAttributeParser Parser;
-  cantFail(Parser.parse(Bytes, support::little));
+  cantFail(Parser.parse(Bytes, llvm::endianness::little));
 
   std::optional<unsigned> Attr = Parser.getAttributeValue(ExpectedTag);
   return Attr && *Attr == ExpectedValue;
@@ -43,7 +43,7 @@ bool testBuildAttr(unsigned Tag, unsigned Value,
 
 void testParseError(ArrayRef<uint8_t> bytes, const char *msg) {
   ARMAttributeParser parser;
-  Error e = parser.parse(bytes, support::little);
+  Error e = parser.parse(bytes, llvm::endianness::little);
   EXPECT_STREQ(toString(std::move(e)).c_str(), msg);
 }
 
diff --git a/llvm/unittests/Support/BLAKE3Test.cpp b/llvm/unittests/Support/BLAKE3Test.cpp
index a729524112..dc6bfb46c1 100644
--- a/llvm/unittests/Support/BLAKE3Test.cpp
+++ b/llvm/unittests/Support/BLAKE3Test.cpp
@@ -62,7 +62,7 @@ TEST(BLAKE3Test, BLAKE3) {
             "616F575A1B58D4C9797D4217B9730AE5E6EB319D76EDEF6549B46F4EFE31FF8B");
 
   // Using generic HashBuilder.
-  HashBuilder<BLAKE3, support::endianness::native> HashBuilder;
+  HashBuilder<BLAKE3, llvm::endianness::native> HashBuilder;
   HashBuilder.update(std::get<0>(testvectors[2]));
   BLAKE3Result<> HBHash1 = HashBuilder.final();
   BLAKE3Result<> HBHash2 = HashBuilder.result();
@@ -84,7 +84,7 @@ TEST(BLAKE3Test, SmallerHashSize) {
   EXPECT_EQ(hashStr1, "6437B3AC38465133FFB63B75273A8DB5");
 
   // Using generic HashBuilder.
-  HashBuilder<TruncatedBLAKE3<16>, support::endianness::native> HashBuilder;
+  HashBuilder<TruncatedBLAKE3<16>, llvm::endianness::native> HashBuilder;
   HashBuilder.update(Input);
   BLAKE3Result<16> hash3 = HashBuilder.final();
   BLAKE3Result<16> hash4 = HashBuilder.result();
diff --git a/llvm/unittests/Support/BinaryStreamTest.cpp b/llvm/unittests/Support/BinaryStreamTest.cpp
index 01b49ca65e..70cd4036fb 100644
--- a/llvm/unittests/Support/BinaryStreamTest.cpp
+++ b/llvm/unittests/Support/BinaryStreamTest.cpp
@@ -102,7 +102,8 @@ class BrokenStream : public WritableBinaryStream {
   BumpPtrAllocator Allocator;
 };
 
-constexpr endianness Endians[] = {big, little, native};
+constexpr llvm::endianness Endians[] = {
+    llvm::endianness::big, llvm::endianness::little, llvm::endianness::native};
 constexpr uint32_t NumEndians = std::size(Endians);
 constexpr uint32_t NumStreams = 2 * NumEndians;
 
@@ -266,7 +267,7 @@ TEST_F(BinaryStreamTest, StreamRefBounds) {
 
 TEST_F(BinaryStreamTest, StreamRefDynamicSize) {
   StringRef Strings[] = {"1", "2", "3", "4"};
-  AppendingBinaryByteStream Stream(support::little);
+  AppendingBinaryByteStream Stream(llvm::endianness::little);
 
   BinaryStreamWriter Writer(Stream);
   BinaryStreamReader Reader(Stream);
@@ -320,7 +321,7 @@ TEST_F(BinaryStreamTest, DropOperations) {
   initializeInput(InputData, 1);
 
   ArrayRef<uint8_t> Result;
-  BinaryStreamRef Original(InputData, support::little);
+  BinaryStreamRef Original(InputData, llvm::endianness::little);
   ASSERT_EQ(InputData.size(), Original.getLength());
 
   EXPECT_THAT_ERROR(Original.readBytes(0, InputData.size(), Result),
@@ -393,7 +394,7 @@ TEST_F(BinaryStreamTest, MutableBinaryByteStreamBounds) {
 }
 
 TEST_F(BinaryStreamTest, AppendingStream) {
-  AppendingBinaryByteStream Stream(llvm::support::little);
+  AppendingBinaryByteStream Stream(llvm::endianness::little);
   EXPECT_EQ(0U, Stream.getLength());
 
   std::vector<uint8_t> InputData = {'T', 'e', 's', 't', 'T', 'e', 's', 't'};
@@ -835,7 +836,7 @@ TEST_F(BinaryStreamTest, StreamWriterPadToAlignment) {
   // This test may seem excessive but it is checking for past bugs and corner
   // cases by making sure that the stream is allowed to grow and that
   // both multiple pad chunks and single chunk extensions work.
-  AppendingBinaryByteStream Stream(support::little);
+  AppendingBinaryByteStream Stream(llvm::endianness::little);
   BinaryStreamWriter Writer(Stream);
 
   // Offset 0: '0'
@@ -874,7 +875,7 @@ TEST_F(BinaryStreamTest, StreamWriterPadToAlignment) {
 
 TEST_F(BinaryStreamTest, StreamWriterAppend) {
   StringRef Strings[] = {"First", "Second", "Third", "Fourth"};
-  AppendingBinaryByteStream Stream(support::little);
+  AppendingBinaryByteStream Stream(llvm::endianness::little);
   BinaryStreamWriter Writer(Stream);
 
   for (auto &Str : Strings) {
@@ -925,13 +926,13 @@ TEST_F(BinaryStreamTest, BinaryItemStream) {
     uint8_t *Ptr = static_cast<uint8_t *>(Allocator.Allocate(sizeof(Foo),
                                                              alignof(Foo)));
     MutableArrayRef<uint8_t> Buffer(Ptr, sizeof(Foo));
-    MutableBinaryByteStream Stream(Buffer, llvm::support::big);
+    MutableBinaryByteStream Stream(Buffer, llvm::endianness::big);
     BinaryStreamWriter Writer(Stream);
     ASSERT_THAT_ERROR(Writer.writeObject(F), Succeeded());
     Objects.push_back(BinaryItemStreamObject(Buffer));
   }
 
-  BinaryItemStream<BinaryItemStreamObject> ItemStream(big);
+  BinaryItemStream<BinaryItemStreamObject> ItemStream(llvm::endianness::big);
   ItemStream.setItems(Objects);
   BinaryStreamReader Reader(ItemStream);
 
diff --git a/llvm/unittests/Support/CMakeLists.txt b/llvm/unittests/Support/CMakeLists.txt
index dfd55b2289..e1bf793536 100644
--- a/llvm/unittests/Support/CMakeLists.txt
+++ b/llvm/unittests/Support/CMakeLists.txt
@@ -91,6 +91,7 @@ add_llvm_unittest(SupportTests
   TypeTraitsTest.cpp
   TrailingObjectsTest.cpp
   UnicodeTest.cpp
+  UTCTimeTest.cpp
   VersionTupleTest.cpp
   VirtualFileSystemTest.cpp
   WithColorTest.cpp
diff --git a/llvm/unittests/Support/CSKYAttributeParserTest.cpp b/llvm/unittests/Support/CSKYAttributeParserTest.cpp
index d3967fb0ea..1d39d14899 100644
--- a/llvm/unittests/Support/CSKYAttributeParserTest.cpp
+++ b/llvm/unittests/Support/CSKYAttributeParserTest.cpp
@@ -81,7 +81,7 @@ static bool testAttributeInt(unsigned Tag, unsigned Value, unsigned ExpectedTag,
                           OS.str().size());
 
   CSKYAttributeParser Parser;
-  cantFail(Parser.parse(Bytes, support::little));
+  cantFail(Parser.parse(Bytes, llvm::endianness::little));
 
   std::optional<unsigned> Attr = Parser.getAttributeValue(ExpectedTag);
   return Attr && *Attr == ExpectedValue;
@@ -98,7 +98,7 @@ static bool testAttributeString(unsigned Tag, const char *Value,
                           OS.str().size());
 
   CSKYAttributeParser Parser;
-  cantFail(Parser.parse(Bytes, support::little));
+  cantFail(Parser.parse(Bytes, llvm::endianness::little));
 
   std::optional<StringRef> Attr = Parser.getAttributeString(ExpectedTag);
   return Attr && *Attr == ExpectedValue;
@@ -113,7 +113,7 @@ static void testParseError(unsigned Tag, unsigned Value, const char *msg) {
                           OS.str().size());
 
   CSKYAttributeParser Parser;
-  Error e = Parser.parse(Bytes, support::little);
+  Error e = Parser.parse(Bytes, llvm::endianness::little);
   EXPECT_STREQ(toString(std::move(e)).c_str(), msg);
 }
 
diff --git a/llvm/unittests/Support/CompressionTest.cpp b/llvm/unittests/Support/CompressionTest.cpp
index f17bfa136c..5d326cafbe 100644
--- a/llvm/unittests/Support/CompressionTest.cpp
+++ b/llvm/unittests/Support/CompressionTest.cpp
@@ -11,7 +11,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Support/Compression.h"
-#include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Config/config.h"
diff --git a/llvm/unittests/Support/ELFAttributeParserTest.cpp b/llvm/unittests/Support/ELFAttributeParserTest.cpp
index 74d031e58a..38e7b09cc3 100644
--- a/llvm/unittests/Support/ELFAttributeParserTest.cpp
+++ b/llvm/unittests/Support/ELFAttributeParserTest.cpp
@@ -31,7 +31,7 @@ class AttributeHeaderParser : public ELFAttributeParser {
 
 static void testParseError(ArrayRef<uint8_t> bytes, const char *msg) {
   AttributeHeaderParser parser;
-  Error e = parser.parse(bytes, support::little);
+  Error e = parser.parse(bytes, llvm::endianness::little);
   EXPECT_STREQ(toString(std::move(e)).c_str(), msg);
 }
 
diff --git a/llvm/unittests/Support/EndianStreamTest.cpp b/llvm/unittests/Support/EndianStreamTest.cpp
index 1e800ff557..2bab71c547 100644
--- a/llvm/unittests/Support/EndianStreamTest.cpp
+++ b/llvm/unittests/Support/EndianStreamTest.cpp
@@ -20,7 +20,7 @@ TEST(EndianStream, WriteInt32LE) {
 
   {
     raw_svector_ostream OS(data);
-    endian::Writer LE(OS, little);
+    endian::Writer LE(OS, llvm::endianness::little);
     LE.write(static_cast<int32_t>(-1362446643));
   }
 
@@ -35,7 +35,7 @@ TEST(EndianStream, WriteInt32BE) {
 
   {
     raw_svector_ostream OS(data);
-    endian::Writer BE(OS, big);
+    endian::Writer BE(OS, llvm::endianness::big);
     BE.write(static_cast<int32_t>(-1362446643));
   }
 
@@ -51,7 +51,7 @@ TEST(EndianStream, WriteFloatLE) {
 
   {
     raw_svector_ostream OS(data);
-    endian::Writer LE(OS, little);
+    endian::Writer LE(OS, llvm::endianness::little);
     LE.write(12345.0f);
   }
 
@@ -66,7 +66,7 @@ TEST(EndianStream, WriteFloatBE) {
 
   {
     raw_svector_ostream OS(data);
-    endian::Writer BE(OS, big);
+    endian::Writer BE(OS, llvm::endianness::big);
     BE.write(12345.0f);
   }
 
@@ -81,7 +81,7 @@ TEST(EndianStream, WriteInt64LE) {
 
   {
     raw_svector_ostream OS(data);
-    endian::Writer LE(OS, little);
+    endian::Writer LE(OS, llvm::endianness::little);
     LE.write(static_cast<int64_t>(-136244664332342323));
   }
 
@@ -100,7 +100,7 @@ TEST(EndianStream, WriteInt64BE) {
 
   {
     raw_svector_ostream OS(data);
-    endian::Writer BE(OS, big);
+    endian::Writer BE(OS, llvm::endianness::big);
     BE.write(static_cast<int64_t>(-136244664332342323));
   }
 
@@ -119,7 +119,7 @@ TEST(EndianStream, WriteDoubleLE) {
 
   {
     raw_svector_ostream OS(data);
-    endian::Writer LE(OS, little);
+    endian::Writer LE(OS, llvm::endianness::little);
     LE.write(-2349214918.58107);
   }
 
@@ -138,7 +138,7 @@ TEST(EndianStream, WriteDoubleBE) {
 
   {
     raw_svector_ostream OS(data);
-    endian::Writer BE(OS, big);
+    endian::Writer BE(OS, llvm::endianness::big);
     BE.write(-2349214918.58107);
   }
 
@@ -157,7 +157,7 @@ TEST(EndianStream, WriteArrayLE) {
 
   {
     raw_svector_ostream OS(Data);
-    endian::Writer LE(OS, little);
+    endian::Writer LE(OS, llvm::endianness::little);
     LE.write<uint16_t>({0x1234, 0x5678});
   }
 
@@ -172,7 +172,7 @@ TEST(EndianStream, WriteVectorLE) {
 
   {
     raw_svector_ostream OS(Data);
-    endian::Writer LE(OS, little);
+    endian::Writer LE(OS, llvm::endianness::little);
     std::vector<uint16_t> Vec{0x1234, 0x5678};
     LE.write<uint16_t>(Vec);
   }
@@ -188,7 +188,7 @@ TEST(EndianStream, WriteFloatArrayLE) {
 
   {
     raw_svector_ostream OS(Data);
-    endian::Writer LE(OS, little);
+    endian::Writer LE(OS, llvm::endianness::little);
     LE.write<float>({12345.0f, 12346.0f});
   }
 
diff --git a/llvm/unittests/Support/EndianTest.cpp b/llvm/unittests/Support/EndianTest.cpp
index b5e4a9c8d1..ab7dfc3800 100644
--- a/llvm/unittests/Support/EndianTest.cpp
+++ b/llvm/unittests/Support/EndianTest.cpp
@@ -23,12 +23,17 @@ TEST(Endian, Read) {
   unsigned char bigval[] = {0x00, 0x01, 0x02, 0x03, 0x04};
   unsigned char littleval[] = {0x00, 0x04, 0x03, 0x02, 0x01};
   int32_t BigAsHost = 0x00010203;
-  EXPECT_EQ(BigAsHost, (endian::read<int32_t, big, unaligned>(bigval)));
+  EXPECT_EQ(BigAsHost,
+            (endian::read<int32_t, llvm::endianness::big, unaligned>(bigval)));
   int32_t LittleAsHost = 0x02030400;
-  EXPECT_EQ(LittleAsHost,(endian::read<int32_t, little, unaligned>(littleval)));
+  EXPECT_EQ(
+      LittleAsHost,
+      (endian::read<int32_t, llvm::endianness::little, unaligned>(littleval)));
 
-  EXPECT_EQ((endian::read<int32_t, big, unaligned>(bigval + 1)),
-            (endian::read<int32_t, little, unaligned>(littleval + 1)));
+  EXPECT_EQ(
+      (endian::read<int32_t, llvm::endianness::big, unaligned>(bigval + 1)),
+      (endian::read<int32_t, llvm::endianness::little, unaligned>(littleval +
+                                                                  1)));
 }
 
 TEST(Endian, ReadBitAligned) {
@@ -36,35 +41,43 @@ TEST(Endian, ReadBitAligned) {
   unsigned char littleval[] = {0x3f, 0x00, 0x00, 0x00, 0xc0, 0xff, 0xff, 0xff};
   unsigned char bigval[] = {0x00, 0x00, 0x00, 0x3f, 0xff, 0xff, 0xff, 0xc0};
   EXPECT_EQ(
-      (endian::readAtBitAlignment<int, little, unaligned>(&littleval[0], 6)),
+      (endian::readAtBitAlignment<int, llvm::endianness::little, unaligned>(
+          &littleval[0], 6)),
       0x0);
-  EXPECT_EQ((endian::readAtBitAlignment<int, big, unaligned>(&bigval[0], 6)),
+  EXPECT_EQ((endian::readAtBitAlignment<int, llvm::endianness::big, unaligned>(
+                &bigval[0], 6)),
             0x0);
   // Test to make sure that signed right shift of 0xf0000000 is masked
   // properly.
   unsigned char littleval2[] = {0x00, 0x00, 0x00, 0xf0, 0x00, 0x00, 0x00, 0x00};
   unsigned char bigval2[] = {0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
   EXPECT_EQ(
-      (endian::readAtBitAlignment<int, little, unaligned>(&littleval2[0], 4)),
+      (endian::readAtBitAlignment<int, llvm::endianness::little, unaligned>(
+          &littleval2[0], 4)),
       0x0f000000);
-  EXPECT_EQ((endian::readAtBitAlignment<int, big, unaligned>(&bigval2[0], 4)),
+  EXPECT_EQ((endian::readAtBitAlignment<int, llvm::endianness::big, unaligned>(
+                &bigval2[0], 4)),
             0x0f000000);
   // Test to make sure left shift of start bit doesn't overflow.
   EXPECT_EQ(
-      (endian::readAtBitAlignment<int, little, unaligned>(&littleval2[0], 1)),
+      (endian::readAtBitAlignment<int, llvm::endianness::little, unaligned>(
+          &littleval2[0], 1)),
       0x78000000);
-  EXPECT_EQ((endian::readAtBitAlignment<int, big, unaligned>(&bigval2[0], 1)),
+  EXPECT_EQ((endian::readAtBitAlignment<int, llvm::endianness::big, unaligned>(
+                &bigval2[0], 1)),
             0x78000000);
   // Test to make sure 64-bit int doesn't overflow.
   unsigned char littleval3[] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0,
                                 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
   unsigned char bigval3[] = {0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
                              0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
-  EXPECT_EQ((endian::readAtBitAlignment<int64_t, little, unaligned>(
-                &littleval3[0], 4)),
-            0x0f00000000000000);
   EXPECT_EQ(
-      (endian::readAtBitAlignment<int64_t, big, unaligned>(&bigval3[0], 4)),
+      (endian::readAtBitAlignment<int64_t, llvm::endianness::little, unaligned>(
+          &littleval3[0], 4)),
+      0x0f00000000000000);
+  EXPECT_EQ(
+      (endian::readAtBitAlignment<int64_t, llvm::endianness::big, unaligned>(
+          &bigval3[0], 4)),
       0x0f00000000000000);
 }
 
@@ -72,8 +85,8 @@ TEST(Endian, WriteBitAligned) {
   // This test ensures that signed right shift of 0xffffaa is masked
   // properly.
   unsigned char bigval[8] = {0x00};
-  endian::writeAtBitAlignment<int32_t, big, unaligned>(bigval, (int)0xffffaaaa,
-                                                       4);
+  endian::writeAtBitAlignment<int32_t, llvm::endianness::big, unaligned>(
+      bigval, (int)0xffffaaaa, 4);
   EXPECT_EQ(bigval[0], 0xff);
   EXPECT_EQ(bigval[1], 0xfa);
   EXPECT_EQ(bigval[2], 0xaa);
@@ -84,8 +97,8 @@ TEST(Endian, WriteBitAligned) {
   EXPECT_EQ(bigval[7], 0x0f);
 
   unsigned char littleval[8] = {0x00};
-  endian::writeAtBitAlignment<int32_t, little, unaligned>(littleval,
-                                                          (int)0xffffaaaa, 4);
+  endian::writeAtBitAlignment<int32_t, llvm::endianness::little, unaligned>(
+      littleval, (int)0xffffaaaa, 4);
   EXPECT_EQ(littleval[0], 0xa0);
   EXPECT_EQ(littleval[1], 0xaa);
   EXPECT_EQ(littleval[2], 0xfa);
@@ -98,8 +111,8 @@ TEST(Endian, WriteBitAligned) {
   // This test makes sure 1<<31 doesn't overflow.
   // Test to make sure left shift of start bit doesn't overflow.
   unsigned char bigval2[8] = {0x00};
-  endian::writeAtBitAlignment<int32_t, big, unaligned>(bigval2, (int)0xffffffff,
-                                                       1);
+  endian::writeAtBitAlignment<int32_t, llvm::endianness::big, unaligned>(
+      bigval2, (int)0xffffffff, 1);
   EXPECT_EQ(bigval2[0], 0xff);
   EXPECT_EQ(bigval2[1], 0xff);
   EXPECT_EQ(bigval2[2], 0xff);
@@ -110,8 +123,8 @@ TEST(Endian, WriteBitAligned) {
   EXPECT_EQ(bigval2[7], 0x01);
 
   unsigned char littleval2[8] = {0x00};
-  endian::writeAtBitAlignment<int32_t, little, unaligned>(littleval2,
-                                                          (int)0xffffffff, 1);
+  endian::writeAtBitAlignment<int32_t, llvm::endianness::little, unaligned>(
+      littleval2, (int)0xffffffff, 1);
   EXPECT_EQ(littleval2[0], 0xfe);
   EXPECT_EQ(littleval2[1], 0xff);
   EXPECT_EQ(littleval2[2], 0xff);
@@ -123,7 +136,7 @@ TEST(Endian, WriteBitAligned) {
 
   // Test to make sure 64-bit int doesn't overflow.
   unsigned char bigval64[16] = {0x00};
-  endian::writeAtBitAlignment<int64_t, big, unaligned>(
+  endian::writeAtBitAlignment<int64_t, llvm::endianness::big, unaligned>(
       bigval64, (int64_t)0xffffffffffffffff, 1);
   EXPECT_EQ(bigval64[0], 0xff);
   EXPECT_EQ(bigval64[1], 0xff);
@@ -143,7 +156,7 @@ TEST(Endian, WriteBitAligned) {
   EXPECT_EQ(bigval64[15], 0x01);
 
   unsigned char littleval64[16] = {0x00};
-  endian::writeAtBitAlignment<int64_t, little, unaligned>(
+  endian::writeAtBitAlignment<int64_t, llvm::endianness::little, unaligned>(
       littleval64, (int64_t)0xffffffffffffffff, 1);
   EXPECT_EQ(littleval64[0], 0xfe);
   EXPECT_EQ(littleval64[1], 0xff);
@@ -165,23 +178,26 @@ TEST(Endian, WriteBitAligned) {
 
 TEST(Endian, Write) {
   unsigned char data[5];
-  endian::write<int32_t, big, unaligned>(data, -1362446643);
+  endian::write<int32_t, llvm::endianness::big, unaligned>(data, -1362446643);
   EXPECT_EQ(data[0], 0xAE);
   EXPECT_EQ(data[1], 0xCA);
   EXPECT_EQ(data[2], 0xB6);
   EXPECT_EQ(data[3], 0xCD);
-  endian::write<int32_t, big, unaligned>(data + 1, -1362446643);
+  endian::write<int32_t, llvm::endianness::big, unaligned>(data + 1,
+                                                           -1362446643);
   EXPECT_EQ(data[1], 0xAE);
   EXPECT_EQ(data[2], 0xCA);
   EXPECT_EQ(data[3], 0xB6);
   EXPECT_EQ(data[4], 0xCD);
 
-  endian::write<int32_t, little, unaligned>(data, -1362446643);
+  endian::write<int32_t, llvm::endianness::little, unaligned>(data,
+                                                              -1362446643);
   EXPECT_EQ(data[0], 0xCD);
   EXPECT_EQ(data[1], 0xB6);
   EXPECT_EQ(data[2], 0xCA);
   EXPECT_EQ(data[3], 0xAE);
-  endian::write<int32_t, little, unaligned>(data + 1, -1362446643);
+  endian::write<int32_t, llvm::endianness::little, unaligned>(data + 1,
+                                                              -1362446643);
   EXPECT_EQ(data[1], 0xCD);
   EXPECT_EQ(data[2], 0xB6);
   EXPECT_EQ(data[3], 0xCA);
diff --git a/llvm/unittests/Support/HashBuilderTest.cpp b/llvm/unittests/Support/HashBuilderTest.cpp
index 461b7c3b25..0aacfcd185 100644
--- a/llvm/unittests/Support/HashBuilderTest.cpp
+++ b/llvm/unittests/Support/HashBuilderTest.cpp
@@ -26,16 +26,16 @@ struct HasherTAndEndianness {
   using HasherT = _HasherT;
   static constexpr llvm::endianness Endianness = _Endianness;
 };
-using HasherTAndEndiannessToTest =
-    ::testing::Types<HasherTAndEndianness<llvm::MD5, llvm::support::big>,
-                     HasherTAndEndianness<llvm::MD5, llvm::support::little>,
-                     HasherTAndEndianness<llvm::MD5, llvm::support::native>,
-                     HasherTAndEndianness<llvm::SHA1, llvm::support::big>,
-                     HasherTAndEndianness<llvm::SHA1, llvm::support::little>,
-                     HasherTAndEndianness<llvm::SHA1, llvm::support::native>,
-                     HasherTAndEndianness<llvm::SHA256, llvm::support::big>,
-                     HasherTAndEndianness<llvm::SHA256, llvm::support::little>,
-                     HasherTAndEndianness<llvm::SHA256, llvm::support::native>>;
+using HasherTAndEndiannessToTest = ::testing::Types<
+    HasherTAndEndianness<llvm::MD5, llvm::endianness::big>,
+    HasherTAndEndianness<llvm::MD5, llvm::endianness::little>,
+    HasherTAndEndianness<llvm::MD5, llvm::endianness::native>,
+    HasherTAndEndianness<llvm::SHA1, llvm::endianness::big>,
+    HasherTAndEndianness<llvm::SHA1, llvm::endianness::little>,
+    HasherTAndEndianness<llvm::SHA1, llvm::endianness::native>,
+    HasherTAndEndianness<llvm::SHA256, llvm::endianness::big>,
+    HasherTAndEndianness<llvm::SHA256, llvm::endianness::little>,
+    HasherTAndEndianness<llvm::SHA256, llvm::endianness::native>>;
 template <typename HasherT> class HashBuilderTest : public testing::Test {};
 TYPED_TEST_SUITE(HashBuilderTest, HasherTAndEndiannessToTest, );
 
diff --git a/llvm/unittests/Support/KnownBitsTest.cpp b/llvm/unittests/Support/KnownBitsTest.cpp
index 9d184beea3..c0377d45c3 100644
--- a/llvm/unittests/Support/KnownBitsTest.cpp
+++ b/llvm/unittests/Support/KnownBitsTest.cpp
@@ -213,6 +213,37 @@ TEST(KnownBitsTest, AddSubExhaustive) {
   TestAddSubExhaustive(false);
 }
 
+TEST(KnownBitsTest, SubBorrowExhaustive) {
+  unsigned Bits = 4;
+  ForeachKnownBits(Bits, [&](const KnownBits &Known1) {
+    ForeachKnownBits(Bits, [&](const KnownBits &Known2) {
+      ForeachKnownBits(1, [&](const KnownBits &KnownBorrow) {
+        // Explicitly compute known bits of the subtraction by trying all
+        // possibilities.
+        KnownBits Known(Bits);
+        Known.Zero.setAllBits();
+        Known.One.setAllBits();
+        ForeachNumInKnownBits(Known1, [&](const APInt &N1) {
+          ForeachNumInKnownBits(Known2, [&](const APInt &N2) {
+            ForeachNumInKnownBits(KnownBorrow, [&](const APInt &Borrow) {
+              APInt Sub = N1 - N2;
+              if (Borrow.getBoolValue())
+                --Sub;
+
+              Known.One &= Sub;
+              Known.Zero &= ~Sub;
+            });
+          });
+        });
+
+        KnownBits KnownComputed =
+            KnownBits::computeForSubBorrow(Known1, Known2, KnownBorrow);
+        EXPECT_EQ(Known, KnownComputed);
+      });
+    });
+  });
+}
+
 TEST(KnownBitsTest, BinaryExhaustive) {
   testBinaryOpExhaustive(
       [](const KnownBits &Known1, const KnownBits &Known2) {
diff --git a/llvm/unittests/Support/NativeFormatTests.cpp b/llvm/unittests/Support/NativeFormatTests.cpp
index 5568727bcc..11753068b0 100644
--- a/llvm/unittests/Support/NativeFormatTests.cpp
+++ b/llvm/unittests/Support/NativeFormatTests.cpp
@@ -6,7 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/ADT/SmallString.h"
 #include "llvm/Support/NativeFormatting.h"
 #include "llvm/Support/raw_ostream.h"
 #include "gtest/gtest.h"
diff --git a/llvm/unittests/Support/RISCVAttributeParserTest.cpp b/llvm/unittests/Support/RISCVAttributeParserTest.cpp
index cdbec0cf2d..a9ede29c65 100644
--- a/llvm/unittests/Support/RISCVAttributeParserTest.cpp
+++ b/llvm/unittests/Support/RISCVAttributeParserTest.cpp
@@ -42,7 +42,7 @@ static bool testAttribute(unsigned Tag, unsigned Value, unsigned ExpectedTag,
                           OS.str().size());
 
   RISCVAttributeParser Parser;
-  cantFail(Parser.parse(Bytes, support::little));
+  cantFail(Parser.parse(Bytes, llvm::endianness::little));
 
   std::optional<unsigned> Attr = Parser.getAttributeValue(ExpectedTag);
   return Attr && *Attr == ExpectedValue;
diff --git a/llvm/unittests/Support/RISCVISAInfoTest.cpp b/llvm/unittests/Support/RISCVISAInfoTest.cpp
index 90e26a23e8..885ec88f8f 100644
--- a/llvm/unittests/Support/RISCVISAInfoTest.cpp
+++ b/llvm/unittests/Support/RISCVISAInfoTest.cpp
@@ -629,6 +629,7 @@ TEST(getTargetFeatureForExtension, RetrieveTargetFeatureFromOneExt) {
 }
 
 TEST(RiscvExtensionsHelp, CheckExtensions) {
+  // clang-format off
   std::string ExpectedOutput =
 R"(All available -march extensions for RISC-V
 
@@ -713,6 +714,8 @@ R"(All available -march extensions for RISC-V
     xcvsimd             1.0
     xsfcie              1.0
     xsfvcp              1.0
+    xsfvqmaccdod        1.0
+    xsfvqmaccqoq        1.0
     xtheadba            1.0
     xtheadbb            1.0
     xtheadbs            1.0
@@ -755,6 +758,7 @@ Experimental extensions
 
 Use -march to specify the target's extension.
 For example, clang -march=rv32i_v1p0)";
+  // clang-format on
 
   StringMap<StringRef> DummyMap;
   DummyMap["i"] = "This is a long dummy description";
diff --git a/llvm/unittests/Support/RegexTest.cpp b/llvm/unittests/Support/RegexTest.cpp
index 78f37cdbd1..e3c721b466 100644
--- a/llvm/unittests/Support/RegexTest.cpp
+++ b/llvm/unittests/Support/RegexTest.cpp
@@ -127,6 +127,34 @@ TEST_F(RegexTest, Substitution) {
 
   EXPECT_EQ("aber", Regex("a[0-9]+b").sub("a\\100b", "a1234ber", &Error));
   EXPECT_EQ(Error, "invalid backreference string '100'");
+
+  EXPECT_EQ("012345", Regex("a([0-9]+).*").sub("0\\g<1>5", "a1234ber", &Error));
+  EXPECT_EQ("", Error);
+
+  EXPECT_EQ("0a1234ber5",
+            Regex("a([0-9]+).*").sub("0\\g<0>5", "a1234ber", &Error));
+  EXPECT_EQ("", Error);
+
+  EXPECT_EQ("0A5", Regex("a(.)(.)(.)(.)(.)(.)(.)(.)(.)(.).*")
+                       .sub("0\\g<10>5", "a123456789Aber", &Error));
+  EXPECT_EQ("", Error);
+
+  EXPECT_EQ("0g<-1>5",
+            Regex("a([0-9]+).*").sub("0\\g<-1>5", "a1234ber", &Error));
+  EXPECT_EQ("", Error);
+
+  EXPECT_EQ("0g<15", Regex("a([0-9]+).*").sub("0\\g<15", "a1234ber", &Error));
+  EXPECT_EQ("", Error);
+
+  EXPECT_EQ("0g<>15", Regex("a([0-9]+).*").sub("0\\g<>15", "a1234ber", &Error));
+  EXPECT_EQ("", Error);
+
+  EXPECT_EQ("0g<3e>1",
+            Regex("a([0-9]+).*").sub("0\\g<3e>1", "a1234ber", &Error));
+  EXPECT_EQ("", Error);
+
+  EXPECT_EQ("aber", Regex("a([0-9]+)b").sub("a\\g<100>b", "a1234ber", &Error));
+  EXPECT_EQ(Error, "invalid backreference string 'g<100>'");
 }
 
 TEST_F(RegexTest, IsLiteralERE) {
diff --git a/llvm/unittests/Support/SHA256.cpp b/llvm/unittests/Support/SHA256.cpp
index a50ce7ba64..f33d2d5fcf 100644
--- a/llvm/unittests/Support/SHA256.cpp
+++ b/llvm/unittests/Support/SHA256.cpp
@@ -13,7 +13,6 @@
 
 #include "llvm/Support/SHA256.h"
 #include "llvm/ADT/ArrayRef.h"
-#include "llvm/ADT/SmallString.h"
 #include "gtest/gtest.h"
 
 using namespace llvm;
diff --git a/llvm/unittests/Support/UTCTimeTest.cpp b/llvm/unittests/Support/UTCTimeTest.cpp
new file mode 100644
index 0000000000..64e04d2937
--- /dev/null
+++ b/llvm/unittests/Support/UTCTimeTest.cpp
@@ -0,0 +1,41 @@
+//===- unittests/Support/UTCTimeTest.cpp ----------------- ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/Chrono.h"
+#include "gtest/gtest.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/FormatProviders.h"
+#include "llvm/Support/FormatVariadic.h"
+
+namespace llvm {
+namespace sys {
+namespace {
+
+TEST(UTCTime, convertutc) {
+  // Get the current time.
+  time_t currentTime;
+  time(&currentTime);
+
+  // Convert with toUtcTime.
+  SmallString<15> customResultString;
+  raw_svector_ostream T(customResultString);
+  T << formatv("{0:%Y-%m-%d %H:%M:%S}", llvm::sys::toUtcTime(currentTime));
+
+  // Convert with gmtime.
+  char gmtimeResultString[20];
+  std::tm *gmtimeResult = std::gmtime(&currentTime);
+  assert(gmtimeResult != NULL);
+  std::strftime(gmtimeResultString, 20, "%Y-%m-%d %H:%M:%S", gmtimeResult);
+
+  // Compare the formatted strings.
+  EXPECT_EQ(customResultString, StringRef(gmtimeResultString, 19));
+
+}
+} // namespace
+} // namespace sys
+} // namespace llvm
diff --git a/llvm/unittests/Support/YAMLIOTest.cpp b/llvm/unittests/Support/YAMLIOTest.cpp
index dc68f945e0..488746764a 100644
--- a/llvm/unittests/Support/YAMLIOTest.cpp
+++ b/llvm/unittests/Support/YAMLIOTest.cpp
@@ -550,10 +550,10 @@ TEST(YAMLIO, TestReadWriteBuiltInTypes) {
 
 struct EndianTypes {
   typedef llvm::support::detail::packed_endian_specific_integral<
-      float, llvm::support::little, llvm::support::unaligned>
+      float, llvm::endianness::little, llvm::support::unaligned>
       ulittle_float;
   typedef llvm::support::detail::packed_endian_specific_integral<
-      double, llvm::support::little, llvm::support::unaligned>
+      double, llvm::endianness::little, llvm::support::unaligned>
       ulittle_double;
 
   llvm::support::ulittle64_t u64;
@@ -2392,6 +2392,7 @@ TEST(YAMLIO, TestMalformedMapFailsGracefully) {
 
 struct OptionalTest {
   std::vector<int> Numbers;
+  std::optional<int> MaybeNumber;
 };
 
 struct OptionalTestSeq {
@@ -2405,6 +2406,7 @@ namespace yaml {
   struct MappingTraits<OptionalTest> {
     static void mapping(IO& IO, OptionalTest &OT) {
       IO.mapOptional("Numbers", OT.Numbers);
+      IO.mapOptional("MaybeNumber", OT.MaybeNumber);
     }
   };
 
@@ -2466,6 +2468,7 @@ TEST(YAMLIO, TestEmptyStringSucceedsForMapWithOptionalFields) {
   Input yin("");
   yin >> doc;
   EXPECT_FALSE(yin.error());
+  EXPECT_FALSE(doc.MaybeNumber.has_value());
 }
 
 TEST(YAMLIO, TestEmptyStringSucceedsForSequence) {
@@ -3153,7 +3156,7 @@ TEST(YAMLIO, TestFlowSequenceTokenErrors) {
 
 TEST(YAMLIO, TestDirectiveMappingNoValue) {
   Input yin("%YAML\n{5:");
-  EXPECT_FALSE(yin.setCurrentDocument());
+  yin.setCurrentDocument();
   EXPECT_TRUE(yin.error());
 
   Input yin2("%TAG\n'\x98!< :\n");
diff --git a/llvm/unittests/Support/YAMLParserTest.cpp b/llvm/unittests/Support/YAMLParserTest.cpp
index b52a3850c0..247e707568 100644
--- a/llvm/unittests/Support/YAMLParserTest.cpp
+++ b/llvm/unittests/Support/YAMLParserTest.cpp
@@ -47,6 +47,10 @@ TEST(YAMLParser, ParsesEmptyArray) {
   ExpectParseSuccess("Empty array", "[]");
 }
 
+TEST(YAMLParser, ParsesComplexMap) {
+  ExpectParseSuccess("Complex block map", "? a\n: b");
+}
+
 TEST(YAMLParser, FailsIfNotClosingArray) {
   ExpectParseError("Not closing array", "[");
   ExpectParseError("Not closing array", "  [  ");
@@ -82,7 +86,10 @@ TEST(YAMLParser, FailsIfMissingColon) {
 }
 
 TEST(YAMLParser, FailsOnMissingQuote) {
-  ExpectParseError("Missing open quote", "[{a\":\"b\"}]");
+  // Missing open quote counts as a plain scalar per YAML spec
+  // (Following is equivalent to JSON [{"a\":\"b\"": null}])
+  ExpectParseSuccess("Missing open quote", "[{a\":\"b\"}]");
+  // Closing quote is more strict -- plain scalars cannot start with a quote
   ExpectParseError("Missing closing quote", "[{\"a\":\"b}]");
 }
 
@@ -128,6 +135,48 @@ TEST(YAMLParser, ParsesArrayOfArrays) {
   ExpectParseSuccess("Array of arrays", "[[]]");
 }
 
+TEST(YAMLParser, ParsesPlainScalars) {
+  ExpectParseSuccess("Plain scalar", "hello");
+  ExpectParseSuccess("Plain scalar beginning with a question mark", "?hello");
+  ExpectParseSuccess("Plain scalar beginning with a colon", ":hello");
+  ExpectParseSuccess("Plain scalar beginning with two colons", "::hello");
+  ExpectParseSuccess("Plain scalar beginning with a hyphen", "-hello");
+  ExpectParseSuccess("Multi-line plain scalar", "Hello\nworld");
+  ExpectParseSuccess("Plain scalar with indicator characters",
+                     "He-!l*lo, []world{}");
+  ExpectParseSuccess("Plain scalar with indicator characters used as block key",
+                     "He-!l*lo, []world{}: value");
+  ExpectParseSuccess("Plain scalar in flow sequence", "hello");
+  ExpectParseSuccess(
+      "Plain scalar beginning with a question mark in flow sequence",
+      "[ ?hello ]");
+  ExpectParseSuccess("Plain scalar beginning with a colon in flow sequence",
+                     "[ :hello ]");
+  ExpectParseSuccess("Plain scalar beginning with two colons in flow sequence",
+                     "[ ::hello ]");
+  ExpectParseSuccess("Plain scalar beginning with a hyphen in flow sequence",
+                     "[ -hello ]");
+  ExpectParseSuccess("Multi-line plain scalar in flow sequence",
+                     "[ Hello\nworld ]");
+  ExpectParseSuccess(
+      "Plain scalar with non-flow indicator characters in flow sequence",
+      "[ He-!l*lo, world ]");
+  ExpectParseSuccess(
+      "Plain scalar with non-flow indicator characters used as flow key",
+      "{ He-!l*lo, world: value } ");
+  ExpectParseError(
+      "Plain scalar with flow indicator characters inside flow sequence",
+      "[ Hello[world ]");
+  ExpectParseError(
+      "Plain scalar with flow indicator characters inside flow key",
+      "{ Hello[world: value }");
+  // Multi-line plain scalar in keys is strictly invalid per the spec, but many
+  // implementations accept it in flow keys nonetheless.  Block keys are not
+  // accepted by any other implementation I can find.
+  ExpectParseSuccess("Multi-line plain scalar in block key", "a\nb: c");
+  ExpectParseSuccess("Multi-line plain scalar in flow key", "{\na\nb: c\n}");
+}
+
 TEST(YAMLParser, ParsesBlockLiteralScalars) {
   ExpectParseSuccess("Block literal scalar", "test: |\n  Hello\n  World\n");
   ExpectParseSuccess("Block literal scalar EOF", "test: |\n  Hello\n  World");
@@ -176,6 +225,10 @@ TEST(YAMLParser, HandlesEndOfFileGracefully) {
   ExpectParseError("In array hitting EOF", "[[] ");
   ExpectParseError("In array hitting EOF", "[[]");
   ExpectParseError("In object hitting EOF", "{\"\"");
+  // This one is valid, equivalent to the JSON {"": null}
+  ExpectParseSuccess("In complex block map hitting EOF", "?");
+  // Equivalent to JSON [null]
+  ExpectParseSuccess("In block sequence hitting EOF", "-");
 }
 
 TEST(YAMLParser, HandlesNullValuesInKeyValueNodesGracefully) {
@@ -183,6 +236,12 @@ TEST(YAMLParser, HandlesNullValuesInKeyValueNodesGracefully) {
   ExpectParseError("KeyValueNode with null value", "test: '");
 }
 
+TEST(YAMLParser, BlockSequenceEOF) {
+  SourceMgr SM;
+  yaml::Stream Stream("-", SM);
+  EXPECT_TRUE(isa_and_present<yaml::SequenceNode>(Stream.begin()->getRoot()));
+}
+
 // Checks that the given string can be parsed into an identical string inside
 // of an array.
 static void ExpectCanParseString(StringRef String) {
diff --git a/llvm/unittests/Target/LoongArch/InstSizes.cpp b/llvm/unittests/Target/LoongArch/InstSizes.cpp
index e027f25682..f38af4a8d9 100644
--- a/llvm/unittests/Target/LoongArch/InstSizes.cpp
+++ b/llvm/unittests/Target/LoongArch/InstSizes.cpp
@@ -121,7 +121,7 @@ TEST(InstSizes, AtomicPseudo) {
       "    dead early-clobber renamable $r10, dead early-clobber renamable $r11 = PseudoAtomicLoadAdd32 renamable $r7, renamable $r6, renamable $r8\n"
       "    dead early-clobber renamable $r5, dead early-clobber renamable $r9, dead early-clobber renamable $r10 = PseudoMaskedAtomicLoadUMax32 renamable $r7, renamable $r6, renamable $r8, 4\n"
       "    early-clobber renamable $r9, dead early-clobber renamable $r10, dead early-clobber renamable $r11 = PseudoMaskedAtomicLoadMax32 killed renamable $r6, killed renamable $r5, killed renamable $r7, killed renamable $r8, 4\n"
-      "    dead early-clobber renamable $r5, dead early-clobber renamable $r9 = PseudoCmpXchg32 renamable $r7, renamable $r4, renamable $r6\n"
+      "    dead early-clobber renamable $r5, dead early-clobber renamable $r9 = PseudoCmpXchg32 renamable $r7, renamable $r4, renamable $r6, 4\n"
       "    dead early-clobber renamable $r5, dead early-clobber renamable $r9 = PseudoMaskedCmpXchg32 killed renamable $r7, killed renamable $r4, killed renamable $r6, killed renamable $r8, 4\n",
       // clang-format on
       [](LoongArchInstrInfo &II, MachineFunction &MF) {
diff --git a/llvm/unittests/Target/RISCV/CMakeLists.txt b/llvm/unittests/Target/RISCV/CMakeLists.txt
index 2c757b82e5..9d0bf7244c 100644
--- a/llvm/unittests/Target/RISCV/CMakeLists.txt
+++ b/llvm/unittests/Target/RISCV/CMakeLists.txt
@@ -13,6 +13,7 @@ set(LLVM_LINK_COMPONENTS
 
 add_llvm_target_unittest(RISCVTests
   MCInstrAnalysisTest.cpp
+  RISCVBaseInfoTest.cpp
   )
 
 set_property(TARGET RISCVTests PROPERTY FOLDER "Tests/UnitTests/TargetTests")
diff --git a/llvm/unittests/Target/RISCV/RISCVBaseInfoTest.cpp b/llvm/unittests/Target/RISCV/RISCVBaseInfoTest.cpp
new file mode 100644
index 0000000000..0e4c90caaa
--- /dev/null
+++ b/llvm/unittests/Target/RISCV/RISCVBaseInfoTest.cpp
@@ -0,0 +1,34 @@
+//===- RISCVBaseInfoTest.cpp - RISCVBaseInfo unit tests ----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MCTargetDesc/RISCVBaseInfo.h"
+
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+namespace {
+TEST(RISCVBaseInfo, CheckSameRatioLMUL) {
+  // Smaller LMUL.
+  EXPECT_EQ(RISCVII::LMUL_1,
+            RISCVVType::getSameRatioLMUL(16, RISCVII::LMUL_2, 8));
+  EXPECT_EQ(RISCVII::LMUL_F2,
+            RISCVVType::getSameRatioLMUL(16, RISCVII::LMUL_1, 8));
+  // Smaller fractional LMUL.
+  EXPECT_EQ(RISCVII::LMUL_F8,
+            RISCVVType::getSameRatioLMUL(16, RISCVII::LMUL_F4, 8));
+  // Bigger LMUL.
+  EXPECT_EQ(RISCVII::LMUL_2,
+            RISCVVType::getSameRatioLMUL(8, RISCVII::LMUL_1, 16));
+  EXPECT_EQ(RISCVII::LMUL_1,
+            RISCVVType::getSameRatioLMUL(8, RISCVII::LMUL_F2, 16));
+  // Bigger fractional LMUL.
+  EXPECT_EQ(RISCVII::LMUL_F2,
+            RISCVVType::getSameRatioLMUL(8, RISCVII::LMUL_F4, 16));
+}
+} // namespace
diff --git a/llvm/unittests/TargetParser/TargetParserTest.cpp b/llvm/unittests/TargetParser/TargetParserTest.cpp
index f8ef18efe5..b432e7ac2d 100644
--- a/llvm/unittests/TargetParser/TargetParserTest.cpp
+++ b/llvm/unittests/TargetParser/TargetParserTest.cpp
@@ -1731,7 +1731,7 @@ TEST(TargetParserTest, AArch64ExtensionFeatures) {
       AArch64::AEK_B16B16,  AArch64::AEK_SMEF16F16, AArch64::AEK_CSSC,
       AArch64::AEK_RCPC3,   AArch64::AEK_THE,       AArch64::AEK_D128,
       AArch64::AEK_LSE128,  AArch64::AEK_SPECRES2,  AArch64::AEK_RASv2,
-      AArch64::AEK_ITE,     AArch64::AEK_GCS,
+      AArch64::AEK_ITE,     AArch64::AEK_GCS,       AArch64::AEK_FPMR,
   };
 
   std::vector<StringRef> Features;
@@ -1804,6 +1804,7 @@ TEST(TargetParserTest, AArch64ExtensionFeatures) {
   EXPECT_TRUE(llvm::is_contained(Features, "+specres2"));
   EXPECT_TRUE(llvm::is_contained(Features, "+ite"));
   EXPECT_TRUE(llvm::is_contained(Features, "+gcs"));
+  EXPECT_TRUE(llvm::is_contained(Features, "+fpmr"));
 
   // Assuming we listed every extension above, this should produce the same
   // result. (note that AEK_NONE doesn't have a name so it won't be in the
@@ -1927,6 +1928,7 @@ TEST(TargetParserTest, AArch64ArchExtFeature) {
       {"predres2", "nopredres2", "+specres2", "-specres2"},
       {"rasv2", "norasv2", "+rasv2", "-rasv2"},
       {"gcs", "nogcs", "+gcs", "-gcs"},
+      {"fpmr", "nofpmr", "+fpmr", "-fpmr"},
   };
 
   for (unsigned i = 0; i < std::size(ArchExt); i++) {
diff --git a/llvm/unittests/Transforms/Utils/CodeExtractorTest.cpp b/llvm/unittests/Transforms/Utils/CodeExtractorTest.cpp
index c142729e2c..528d332393 100644
--- a/llvm/unittests/Transforms/Utils/CodeExtractorTest.cpp
+++ b/llvm/unittests/Transforms/Utils/CodeExtractorTest.cpp
@@ -555,4 +555,64 @@ TEST(CodeExtractor, PartialAggregateArgs) {
   EXPECT_FALSE(verifyFunction(*Outlined));
   EXPECT_FALSE(verifyFunction(*Func));
 }
+
+TEST(CodeExtractor, OpenMPAggregateArgs) {
+  LLVMContext Ctx;
+  SMDiagnostic Err;
+  std::unique_ptr<Module> M(parseAssemblyString(R"ir(
+    target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8"
+    target triple = "amdgcn-amd-amdhsa"
+
+    define void @foo(ptr %0) {
+      %2= alloca ptr, align 8, addrspace(5)
+      %3 = addrspacecast ptr addrspace(5) %2 to ptr
+      store ptr %0, ptr %3, align 8
+      %4 = load ptr, ptr %3, align 8
+      br label %entry
+
+   entry:
+      br label %extract
+
+    extract:
+      store i64 10, ptr %4, align 4
+      br label %exit
+
+    exit:
+      ret void
+    }
+  )ir",
+                                                Err, Ctx));
+  Function *Func = M->getFunction("foo");
+  SmallVector<BasicBlock *, 1> Blocks{getBlockByName(Func, "extract")};
+
+  // Create the CodeExtractor with arguments aggregation enabled.
+  // Outlined function argument should be declared in 0 address space
+  // even if the default alloca address space is 5.
+  CodeExtractor CE(Blocks, /* DominatorTree */ nullptr,
+                   /* AggregateArgs */ true, /* BlockFrequencyInfo */ nullptr,
+                   /* BranchProbabilityInfo */ nullptr,
+                   /* AssumptionCache */ nullptr,
+                   /* AllowVarArgs */ true,
+                   /* AllowAlloca */ true,
+                   /* AllocaBlock*/ &Func->getEntryBlock(),
+                   /* Suffix */ ".outlined",
+                   /* ArgsInZeroAddressSpace */ true);
+
+  EXPECT_TRUE(CE.isEligible());
+
+  CodeExtractorAnalysisCache CEAC(*Func);
+  SetVector<Value *> Inputs, Outputs, SinkingCands, HoistingCands;
+  BasicBlock *CommonExit = nullptr;
+  CE.findAllocas(CEAC, SinkingCands, HoistingCands, CommonExit);
+  CE.findInputsOutputs(Inputs, Outputs, SinkingCands);
+
+  Function *Outlined = CE.extractCodeRegion(CEAC, Inputs, Outputs);
+  EXPECT_TRUE(Outlined);
+  EXPECT_EQ(Outlined->arg_size(), 1U);
+  // Check address space of outlined argument is ptr in address space 0
+  EXPECT_EQ(Outlined->getArg(0)->getType(),
+            PointerType::get(M->getContext(), 0));
+  EXPECT_FALSE(verifyFunction(*Outlined));
+  EXPECT_FALSE(verifyFunction(*Func));
+}
 } // end anonymous namespace
diff --git a/llvm/unittests/Transforms/Utils/CodeLayoutTest.cpp b/llvm/unittests/Transforms/Utils/CodeLayoutTest.cpp
index ce42f70322..ef9aa9a763 100644
--- a/llvm/unittests/Transforms/Utils/CodeLayoutTest.cpp
+++ b/llvm/unittests/Transforms/Utils/CodeLayoutTest.cpp
@@ -40,6 +40,14 @@ TEST(CodeLayout, HotChain) {
     const std::vector<uint64_t> CallOffsets(std::size(Edges), 5);
     auto Order = computeCacheDirectedLayout(Sizes, Counts, Edges, CallOffsets);
     EXPECT_THAT(Order, ElementsAreArray({0, 3, 4, 2, 1}));
+
+    // -cdsort-max-chain-size disables forming a larger chain and therefore may
+    // change the result.
+    CDSortConfig Config;
+    Config.MaxChainSize = 3;
+    Order =
+        computeCacheDirectedLayout(Config, Sizes, Counts, Edges, CallOffsets);
+    EXPECT_THAT(Order, ElementsAreArray({0, 3, 4, 1, 2}));
   }
 }
 
diff --git a/llvm/utils/TableGen/DirectiveEmitter.cpp b/llvm/utils/TableGen/DirectiveEmitter.cpp
index 67033c6290..b6aee665f8 100644
--- a/llvm/utils/TableGen/DirectiveEmitter.cpp
+++ b/llvm/utils/TableGen/DirectiveEmitter.cpp
@@ -736,6 +736,7 @@ static void GenerateFlangClausesParser(const DirectiveLanguage &DirLang,
             .Case("Name", "name")
             .Case("ScalarIntConstantExpr", "scalarIntConstantExpr")
             .Case("ScalarIntExpr", "scalarIntExpr")
+            .Case("ScalarExpr", "scalarExpr")
             .Case("ScalarLogicalExpr", "scalarLogicalExpr")
             .Default(("Parser<" + Clause.getFlangClass() + ">{}")
                          .toStringRef(Scratch));
diff --git a/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp b/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp
index b28915148e..7992cb4362 100644
--- a/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp
+++ b/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp
@@ -3108,13 +3108,10 @@ bool CombineRuleBuilder::emitCodeGenInstructionApplyImmOperand(
   }
 
   unsigned TempRegID = M.allocateTempRegID();
-  auto ActIt = M.insertAction<BuildMIAction>(
-      M.actions_begin(), M.allocateOutputInsnID(), &getGConstant());
-  // Ensure MakeTempReg occurs before the BuildMI of th G_CONSTANT.
-  M.insertAction<MakeTempRegisterAction>(ActIt, LLT, TempRegID);
-  auto &ConstantMI = *static_cast<BuildMIAction *>(ActIt->get());
-  ConstantMI.addRenderer<TempRegRenderer>(TempRegID);
-  ConstantMI.addRenderer<ImmRenderer>(O.getImmValue(), LLT);
+  // Ensure MakeTempReg & the BuildConstantAction occur at the beginning.
+  auto InsertIt =
+      M.insertAction<MakeTempRegisterAction>(M.actions_begin(), LLT, TempRegID);
+  M.insertAction<BuildConstantAction>(++InsertIt, TempRegID, O.getImmValue());
   DstMI.addRenderer<TempRegRenderer>(TempRegID);
   return true;
 }
@@ -3310,6 +3307,10 @@ class GICombinerEmitter final : public GlobalISelMatchTableExecutorEmitter {
   // combine rule used to disable/enable it.
   std::vector<std::pair<unsigned, std::string>> AllCombineRules;
 
+  // Keep track of all rules we've seen so far to ensure we don't process
+  // the same rule twice.
+  StringSet<> RulesSeen;
+
   MatchTable buildMatchTable(MutableArrayRef<RuleMatcher> Rules);
 
   void emitRuleConfigImpl(raw_ostream &OS);
@@ -3465,15 +3466,15 @@ void GICombinerEmitter::emitAdditionalImpl(raw_ostream &OS) {
      << "  const TargetSubtargetInfo &ST = MF.getSubtarget();\n"
      << "  const PredicateBitset AvailableFeatures = "
         "getAvailableFeatures();\n"
-     << "  NewMIVector OutMIs;\n"
+     << "  B.setInstrAndDebugLoc(I);\n"
      << "  State.MIs.clear();\n"
      << "  State.MIs.push_back(&I);\n"
      << "  " << MatchDataInfo::StructName << " = "
      << MatchDataInfo::StructTypeName << "();\n\n"
-     << "  if (executeMatchTable(*this, OutMIs, State, ExecInfo"
+     << "  if (executeMatchTable(*this, State, ExecInfo, B"
      << ", getMatchTable(), *ST.getInstrInfo(), MRI, "
         "*MRI.getTargetRegisterInfo(), *ST.getRegBankInfo(), AvailableFeatures"
-     << ", /*CoverageInfo*/ nullptr, &Observer)) {\n"
+     << ", /*CoverageInfo*/ nullptr)) {\n"
      << "    return true;\n"
      << "  }\n\n"
      << "  return false;\n"
@@ -3627,27 +3628,37 @@ void GICombinerEmitter::gatherRules(
     std::vector<RuleMatcher> &ActiveRules,
     const std::vector<Record *> &&RulesAndGroups) {
   for (Record *Rec : RulesAndGroups) {
-    if (Rec->isValueUnset("Rules")) {
-      AllCombineRules.emplace_back(NextRuleID, Rec->getName().str());
-      CombineRuleBuilder CRB(Target, SubtargetFeatures, *Rec, NextRuleID++,
-                             ActiveRules);
+    if (!Rec->isValueUnset("Rules")) {
+      gatherRules(ActiveRules, Rec->getValueAsListOfDefs("Rules"));
+      continue;
+    }
 
-      if (!CRB.parseAll()) {
-        assert(ErrorsPrinted && "Parsing failed without errors!");
-        continue;
-      }
+    StringRef RuleName = Rec->getName();
+    if (!RulesSeen.insert(RuleName).second) {
+      PrintWarning(Rec->getLoc(),
+                   "skipping rule '" + Rec->getName() +
+                       "' because it has already been processed");
+      continue;
+    }
 
-      if (StopAfterParse) {
-        CRB.print(outs());
-        continue;
-      }
+    AllCombineRules.emplace_back(NextRuleID, Rec->getName().str());
+    CombineRuleBuilder CRB(Target, SubtargetFeatures, *Rec, NextRuleID++,
+                           ActiveRules);
 
-      if (!CRB.emitRuleMatchers()) {
-        assert(ErrorsPrinted && "Emission failed without errors!");
-        continue;
-      }
-    } else
-      gatherRules(ActiveRules, Rec->getValueAsListOfDefs("Rules"));
+    if (!CRB.parseAll()) {
+      assert(ErrorsPrinted && "Parsing failed without errors!");
+      continue;
+    }
+
+    if (StopAfterParse) {
+      CRB.print(outs());
+      continue;
+    }
+
+    if (!CRB.emitRuleMatchers()) {
+      assert(ErrorsPrinted && "Emission failed without errors!");
+      continue;
+    }
   }
 }
 
diff --git a/llvm/utils/TableGen/GlobalISelEmitter.cpp b/llvm/utils/TableGen/GlobalISelEmitter.cpp
index 2ea4890446..8d9ded1b2a 100644
--- a/llvm/utils/TableGen/GlobalISelEmitter.cpp
+++ b/llvm/utils/TableGen/GlobalISelEmitter.cpp
@@ -2267,10 +2267,10 @@ void GlobalISelEmitter::emitAdditionalImpl(raw_ostream &OS) {
         "&CoverageInfo) const {\n"
      << "  const PredicateBitset AvailableFeatures = "
         "getAvailableFeatures();\n"
-     << "  NewMIVector OutMIs;\n"
+     << "  MachineIRBuilder B(I);\n"
      << "  State.MIs.clear();\n"
      << "  State.MIs.push_back(&I);\n\n"
-     << "  if (executeMatchTable(*this, OutMIs, State, ExecInfo"
+     << "  if (executeMatchTable(*this, State, ExecInfo, B"
      << ", getMatchTable(), TII, MF->getRegInfo(), TRI, RBI, AvailableFeatures"
      << ", &CoverageInfo)) {\n"
      << "    return true;\n"
diff --git a/llvm/utils/TableGen/GlobalISelMatchTable.cpp b/llvm/utils/TableGen/GlobalISelMatchTable.cpp
index dcfd0a34be..9a4a375f34 100644
--- a/llvm/utils/TableGen/GlobalISelMatchTable.cpp
+++ b/llvm/utils/TableGen/GlobalISelMatchTable.cpp
@@ -2014,6 +2014,16 @@ void BuildMIAction::emitActionOpcodes(MatchTable &Table,
     EraseInstAction::emitActionOpcodes(Table, Rule, /*InsnID*/ 0);
 }
 
+//===- BuildConstantAction ------------------------------------------------===//
+
+void BuildConstantAction::emitActionOpcodes(MatchTable &Table,
+                                            RuleMatcher &Rule) const {
+  Table << MatchTable::Opcode("GIR_BuildConstant")
+        << MatchTable::Comment("TempRegID") << MatchTable::IntValue(TempRegID)
+        << MatchTable::Comment("Val") << MatchTable::IntValue(Val)
+        << MatchTable::LineBreak;
+}
+
 //===- EraseInstAction ----------------------------------------------------===//
 
 void EraseInstAction::emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule,
diff --git a/llvm/utils/TableGen/GlobalISelMatchTable.h b/llvm/utils/TableGen/GlobalISelMatchTable.h
index 549d7ccde1..5608bab482 100644
--- a/llvm/utils/TableGen/GlobalISelMatchTable.h
+++ b/llvm/utils/TableGen/GlobalISelMatchTable.h
@@ -2093,6 +2093,7 @@ class MatchAction {
     AK_DebugComment,
     AK_CustomCXX,
     AK_BuildMI,
+    AK_BuildConstantMI,
     AK_EraseInst,
     AK_ReplaceReg,
     AK_ConstraintOpsToDef,
@@ -2187,6 +2188,24 @@ class BuildMIAction : public MatchAction {
   void emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const override;
 };
 
+/// Generates code to create a constant that defines a TempReg.
+/// The instruction created is usually a G_CONSTANT but it could also be a
+/// G_BUILD_VECTOR for vector types.
+class BuildConstantAction : public MatchAction {
+  unsigned TempRegID;
+  int64_t Val;
+
+public:
+  BuildConstantAction(unsigned TempRegID, int64_t Val)
+      : MatchAction(AK_BuildConstantMI), TempRegID(TempRegID), Val(Val) {}
+
+  static bool classof(const MatchAction *A) {
+    return A->getKind() == AK_BuildConstantMI;
+  }
+
+  void emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const override;
+};
+
 class EraseInstAction : public MatchAction {
   unsigned InsnID;
 
diff --git a/llvm/utils/TableGen/SubtargetEmitter.cpp b/llvm/utils/TableGen/SubtargetEmitter.cpp
index 5e822078b9..f7a7172d61 100644
--- a/llvm/utils/TableGen/SubtargetEmitter.cpp
+++ b/llvm/utils/TableGen/SubtargetEmitter.cpp
@@ -1935,7 +1935,7 @@ void SubtargetEmitter::run(raw_ostream &OS) {
   if (NumProcs)
     OS << Target << "SubTypeKV, ";
   else
-    OS << "None, ";
+    OS << "std::nullopt, ";
   OS << '\n'; OS.indent(22);
   OS << Target << "WriteProcResTable, "
      << Target << "WriteLatencyTable, "
@@ -2028,7 +2028,7 @@ void SubtargetEmitter::run(raw_ostream &OS) {
   if (NumProcs)
     OS << "ArrayRef(" << Target << "SubTypeKV, " << NumProcs << "), ";
   else
-    OS << "None, ";
+    OS << "std::nullopt, ";
   OS << '\n'; OS.indent(24);
   OS << Target << "WriteProcResTable, "
      << Target << "WriteLatencyTable, "
diff --git a/llvm/utils/TableGen/VarLenCodeEmitterGen.cpp b/llvm/utils/TableGen/VarLenCodeEmitterGen.cpp
index 7a24030e17..24f116bbea 100644
--- a/llvm/utils/TableGen/VarLenCodeEmitterGen.cpp
+++ b/llvm/utils/TableGen/VarLenCodeEmitterGen.cpp
@@ -67,17 +67,26 @@ namespace {
 class VarLenCodeEmitterGen {
   RecordKeeper &Records;
 
-  DenseMap<Record *, VarLenInst> VarLenInsts;
+  // Representaton of alternative encodings used for HwModes.
+  using AltEncodingTy = int;
+  // Mode identifier when only one encoding is defined.
+  const AltEncodingTy Universal = -1;
+  // The set of alternative instruction encodings with a descriptive
+  // name suffix to improve readability of the generated code.
+  std::map<AltEncodingTy, std::string> Modes;
+
+  DenseMap<Record *, DenseMap<AltEncodingTy, VarLenInst>> VarLenInsts;
 
   // Emit based values (i.e. fixed bits in the encoded instructions)
   void emitInstructionBaseValues(
       raw_ostream &OS,
       ArrayRef<const CodeGenInstruction *> NumberedInstructions,
-      CodeGenTarget &Target, int HwMode = -1);
+      CodeGenTarget &Target, AltEncodingTy Mode);
 
-  std::string getInstructionCase(Record *R, CodeGenTarget &Target);
-  std::string getInstructionCaseForEncoding(Record *R, Record *EncodingDef,
-                                            CodeGenTarget &Target);
+  std::string getInstructionCases(Record *R, CodeGenTarget &Target);
+  std::string getInstructionCaseForEncoding(Record *R, AltEncodingTy Mode,
+                                            const VarLenInst &VLI,
+                                            CodeGenTarget &Target, int I);
 
 public:
   explicit VarLenCodeEmitterGen(RecordKeeper &R) : Records(R) {}
@@ -214,36 +223,38 @@ void VarLenCodeEmitterGen::run(raw_ostream &OS) {
   auto Insts = Records.getAllDerivedDefinitions("Instruction");
 
   auto NumberedInstructions = Target.getInstructionsByEnumValue();
-  const CodeGenHwModes &HWM = Target.getHwModes();
 
-  // The set of HwModes used by instruction encodings.
-  std::set<unsigned> HwModes;
   for (const CodeGenInstruction *CGI : NumberedInstructions) {
     Record *R = CGI->TheDef;
-
     // Create the corresponding VarLenInst instance.
     if (R->getValueAsString("Namespace") == "TargetOpcode" ||
         R->getValueAsBit("isPseudo"))
       continue;
 
+    // Setup alternative encodings according to HwModes
     if (const RecordVal *RV = R->getValue("EncodingInfos")) {
       if (auto *DI = dyn_cast_or_null<DefInit>(RV->getValue())) {
+        const CodeGenHwModes &HWM = Target.getHwModes();
         EncodingInfoByHwMode EBM(DI->getDef(), HWM);
         for (auto &KV : EBM) {
-          HwModes.insert(KV.first);
+          AltEncodingTy Mode = KV.first;
+          Modes.insert({Mode, "_" + HWM.getMode(Mode).Name.str()});
           Record *EncodingDef = KV.second;
           RecordVal *RV = EncodingDef->getValue("Inst");
           DagInit *DI = cast<DagInit>(RV->getValue());
-          VarLenInsts.insert({EncodingDef, VarLenInst(DI, RV)});
+          VarLenInsts[R].insert({Mode, VarLenInst(DI, RV)});
         }
         continue;
       }
     }
     RecordVal *RV = R->getValue("Inst");
     DagInit *DI = cast<DagInit>(RV->getValue());
-    VarLenInsts.insert({R, VarLenInst(DI, RV)});
+    VarLenInsts[R].insert({Universal, VarLenInst(DI, RV)});
   }
 
+  if (Modes.empty())
+    Modes.insert({Universal, ""}); // Base case, skip suffix.
+
   // Emit function declaration
   OS << "void " << Target.getName()
      << "MCCodeEmitter::getBinaryCodeForInstr(const MCInst &MI,\n"
@@ -253,36 +264,26 @@ void VarLenCodeEmitterGen::run(raw_ostream &OS) {
      << "    const MCSubtargetInfo &STI) const {\n";
 
   // Emit instruction base values
-  if (HwModes.empty()) {
-    emitInstructionBaseValues(OS, NumberedInstructions, Target);
-  } else {
-    for (unsigned HwMode : HwModes)
-      emitInstructionBaseValues(OS, NumberedInstructions, Target, (int)HwMode);
-  }
+  for (const auto &Mode : Modes)
+    emitInstructionBaseValues(OS, NumberedInstructions, Target, Mode.first);
 
-  if (!HwModes.empty()) {
-    OS << "  const unsigned **Index;\n";
-    OS << "  const uint64_t *InstBits;\n";
-    OS << "  unsigned HwMode = STI.getHwMode();\n";
-    OS << "  switch (HwMode) {\n";
-    OS << "  default: llvm_unreachable(\"Unknown hardware mode!\"); break;\n";
-    for (unsigned I : HwModes) {
-      OS << "  case " << I << ": InstBits = InstBits_" << HWM.getMode(I).Name
-         << "; Index = Index_" << HWM.getMode(I).Name << "; break;\n";
-    }
-    OS << "  };\n";
+  if (Modes.size() > 1) {
+    OS << "  unsigned Mode = STI.getHwMode();\n";
   }
 
-  // Emit helper function to retrieve base values.
-  OS << "  auto getInstBits = [&](unsigned Opcode) -> APInt {\n"
-     << "    unsigned NumBits = Index[Opcode][0];\n"
-     << "    if (!NumBits)\n"
-     << "      return APInt::getZeroWidth();\n"
-     << "    unsigned Idx = Index[Opcode][1];\n"
-     << "    ArrayRef<uint64_t> Data(&InstBits[Idx], "
-     << "APInt::getNumWords(NumBits));\n"
-     << "    return APInt(NumBits, Data);\n"
-     << "  };\n";
+  for (const auto &Mode : Modes) {
+    // Emit helper function to retrieve base values.
+    OS << "  auto getInstBits" << Mode.second
+       << " = [&](unsigned Opcode) -> APInt {\n"
+       << "    unsigned NumBits = Index" << Mode.second << "[Opcode][0];\n"
+       << "    if (!NumBits)\n"
+       << "      return APInt::getZeroWidth();\n"
+       << "    unsigned Idx = Index" << Mode.second << "[Opcode][1];\n"
+       << "    ArrayRef<uint64_t> Data(&InstBits" << Mode.second << "[Idx], "
+       << "APInt::getNumWords(NumBits));\n"
+       << "    return APInt(NumBits, Data);\n"
+       << "  };\n";
+  }
 
   // Map to accumulate all the cases.
   std::map<std::string, std::vector<std::string>> CaseMap;
@@ -294,7 +295,7 @@ void VarLenCodeEmitterGen::run(raw_ostream &OS) {
       continue;
     std::string InstName =
         (R->getValueAsString("Namespace") + "::" + R->getName()).str();
-    std::string Case = getInstructionCase(R, Target);
+    std::string Case = getInstructionCases(R, Target);
 
     CaseMap[Case].push_back(std::move(InstName));
   }
@@ -344,19 +345,12 @@ static void emitInstBits(raw_ostream &IS, raw_ostream &SS, const APInt &Bits,
 
 void VarLenCodeEmitterGen::emitInstructionBaseValues(
     raw_ostream &OS, ArrayRef<const CodeGenInstruction *> NumberedInstructions,
-    CodeGenTarget &Target, int HwMode) {
+    CodeGenTarget &Target, AltEncodingTy Mode) {
   std::string IndexArray, StorageArray;
   raw_string_ostream IS(IndexArray), SS(StorageArray);
 
-  const CodeGenHwModes &HWM = Target.getHwModes();
-  if (HwMode == -1) {
-    IS << "  static const unsigned Index[][2] = {\n";
-    SS << "  static const uint64_t InstBits[] = {\n";
-  } else {
-    StringRef Name = HWM.getMode(HwMode).Name;
-    IS << "  static const unsigned Index_" << Name << "[][2] = {\n";
-    SS << "  static const uint64_t InstBits_" << Name << "[] = {\n";
-  }
+  IS << "  static const unsigned Index" << Modes[Mode] << "[][2] = {\n";
+  SS << "  static const uint64_t InstBits" << Modes[Mode] << "[] = {\n";
 
   unsigned NumFixedValueWords = 0U;
   for (const CodeGenInstruction *CGI : NumberedInstructions) {
@@ -368,20 +362,18 @@ void VarLenCodeEmitterGen::emitInstructionBaseValues(
       continue;
     }
 
-    Record *EncodingDef = R;
-    if (const RecordVal *RV = R->getValue("EncodingInfos")) {
-      if (auto *DI = dyn_cast_or_null<DefInit>(RV->getValue())) {
-        EncodingInfoByHwMode EBM(DI->getDef(), HWM);
-        if (EBM.hasMode(HwMode))
-          EncodingDef = EBM.get(HwMode);
-      }
+    const auto InstIt = VarLenInsts.find(R);
+    if (InstIt == VarLenInsts.end())
+      PrintFatalError(R, "VarLenInst not found for this record");
+    auto ModeIt = InstIt->second.find(Mode);
+    if (ModeIt == InstIt->second.end())
+      ModeIt = InstIt->second.find(Universal);
+    if (ModeIt == InstIt->second.end()) {
+      IS.indent(4) << "{/*NumBits*/0, /*Index*/0},\t"
+                   << "// " << R->getName() << " no encoding\n";
+      continue;
     }
-
-    auto It = VarLenInsts.find(EncodingDef);
-    if (It == VarLenInsts.end())
-      PrintFatalError(EncodingDef, "VarLenInst not found for this record");
-    const VarLenInst &VLI = It->second;
-
+    const VarLenInst &VLI = ModeIt->second;
     unsigned i = 0U, BitWidth = VLI.size();
 
     // Start by filling in fixed values.
@@ -414,34 +406,45 @@ void VarLenCodeEmitterGen::emitInstructionBaseValues(
   OS << IS.str() << SS.str();
 }
 
-std::string VarLenCodeEmitterGen::getInstructionCase(Record *R,
-                                                     CodeGenTarget &Target) {
+std::string VarLenCodeEmitterGen::getInstructionCases(Record *R,
+                                                      CodeGenTarget &Target) {
+  auto It = VarLenInsts.find(R);
+  if (It == VarLenInsts.end())
+    PrintFatalError(R, "Parsed encoding record not found");
+  const auto &Map = It->second;
+
+  // Is this instructions encoding universal (same for all modes)?
+  // Allways true if there is only one mode.
+  if (Map.size() == 1 && Map.begin()->first == Universal) {
+    // Universal, just pick the first mode.
+    AltEncodingTy Mode = Modes.begin()->first;
+    const auto &Encoding = Map.begin()->second;
+    return getInstructionCaseForEncoding(R, Mode, Encoding, Target, 6);
+  }
+
   std::string Case;
-  if (const RecordVal *RV = R->getValue("EncodingInfos")) {
-    if (auto *DI = dyn_cast_or_null<DefInit>(RV->getValue())) {
-      const CodeGenHwModes &HWM = Target.getHwModes();
-      EncodingInfoByHwMode EBM(DI->getDef(), HWM);
-      Case += "      switch (HwMode) {\n";
-      Case += "      default: llvm_unreachable(\"Unhandled HwMode\");\n";
-      for (auto &KV : EBM) {
-        Case += "      case " + itostr(KV.first) + ": {\n";
-        Case += getInstructionCaseForEncoding(R, KV.second, Target);
-        Case += "      break;\n";
-        Case += "      }\n";
-      }
-      Case += "      }\n";
-      return Case;
+  Case += "      switch (Mode) {\n";
+  Case += "      default: llvm_unreachable(\"Unhandled Mode\");\n";
+  for (const auto &Mode : Modes) {
+    Case += "      case " + itostr(Mode.first) + ": {\n";
+    const auto &It = Map.find(Mode.first);
+    if (It == Map.end()) {
+      Case +=
+          "        llvm_unreachable(\"Undefined encoding in this mode\");\n";
+    } else {
+      Case +=
+          getInstructionCaseForEncoding(R, It->first, It->second, Target, 8);
     }
+    Case += "        break;\n";
+    Case += "      }\n";
   }
-  return getInstructionCaseForEncoding(R, R, Target);
+  Case += "      }\n";
+  return Case;
 }
 
 std::string VarLenCodeEmitterGen::getInstructionCaseForEncoding(
-    Record *R, Record *EncodingDef, CodeGenTarget &Target) {
-  auto It = VarLenInsts.find(EncodingDef);
-  if (It == VarLenInsts.end())
-    PrintFatalError(EncodingDef, "Parsed encoding record not found");
-  const VarLenInst &VLI = It->second;
+    Record *R, AltEncodingTy Mode, const VarLenInst &VLI, CodeGenTarget &Target,
+    int I) {
   size_t BitWidth = VLI.size();
 
   CodeGenInstruction &CGI = Target.getInstruction(R);
@@ -450,9 +453,9 @@ std::string VarLenCodeEmitterGen::getInstructionCaseForEncoding(
   raw_string_ostream SS(Case);
   // Resize the scratch buffer.
   if (BitWidth && !VLI.isFixedValueOnly())
-    SS.indent(6) << "Scratch = Scratch.zext(" << BitWidth << ");\n";
+    SS.indent(I) << "Scratch = Scratch.zext(" << BitWidth << ");\n";
   // Populate based value.
-  SS.indent(6) << "Inst = getInstBits(opcode);\n";
+  SS.indent(I) << "Inst = getInstBits" << Modes[Mode] << "(opcode);\n";
 
   // Process each segment in VLI.
   size_t Offset = 0U;
@@ -480,17 +483,17 @@ std::string VarLenCodeEmitterGen::getInstructionCaseForEncoding(
       if (ES.CustomEncoder.size())
         CustomEncoder = ES.CustomEncoder;
 
-      SS.indent(6) << "Scratch.clearAllBits();\n";
-      SS.indent(6) << "// op: " << OperandName.drop_front(1) << "\n";
+      SS.indent(I) << "Scratch.clearAllBits();\n";
+      SS.indent(I) << "// op: " << OperandName.drop_front(1) << "\n";
       if (CustomEncoder.empty())
-        SS.indent(6) << "getMachineOpValue(MI, MI.getOperand("
+        SS.indent(I) << "getMachineOpValue(MI, MI.getOperand("
                      << utostr(FlatOpIdx) << ")";
       else
-        SS.indent(6) << CustomEncoder << "(MI, /*OpIdx=*/" << utostr(FlatOpIdx);
+        SS.indent(I) << CustomEncoder << "(MI, /*OpIdx=*/" << utostr(FlatOpIdx);
 
       SS << ", /*Pos=*/" << utostr(Offset) << ", Scratch, Fixups, STI);\n";
 
-      SS.indent(6) << "Inst.insertBits("
+      SS.indent(I) << "Inst.insertBits("
                    << "Scratch.extractBits(" << utostr(NumBits) << ", "
                    << utostr(LoBit) << ")"
                    << ", " << Offset << ");\n";
@@ -500,7 +503,7 @@ std::string VarLenCodeEmitterGen::getInstructionCaseForEncoding(
 
   StringRef PostEmitter = R->getValueAsString("PostEncoderMethod");
   if (!PostEmitter.empty())
-    SS.indent(6) << "Inst = " << PostEmitter << "(MI, Inst, STI);\n";
+    SS.indent(I) << "Inst = " << PostEmitter << "(MI, Inst, STI);\n";
 
   return Case;
 }
diff --git a/llvm/utils/TableGen/X86DisassemblerTables.cpp b/llvm/utils/TableGen/X86DisassemblerTables.cpp
index 708c92aecf..ba51bf4858 100644
--- a/llvm/utils/TableGen/X86DisassemblerTables.cpp
+++ b/llvm/utils/TableGen/X86DisassemblerTables.cpp
@@ -982,6 +982,7 @@ void DisassemblerTables::emitContextDecisions(raw_ostream &o1, raw_ostream &o2,
   emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[7], THREEDNOW_MAP_STR);
   emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[8], MAP5_STR);
   emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[9], MAP6_STR);
+  emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[10], MAP7_STR);
 }
 
 void DisassemblerTables::emit(raw_ostream &o) const {
diff --git a/llvm/utils/TableGen/X86DisassemblerTables.h b/llvm/utils/TableGen/X86DisassemblerTables.h
index 966f7406ef..4b6f6543ac 100644
--- a/llvm/utils/TableGen/X86DisassemblerTables.h
+++ b/llvm/utils/TableGen/X86DisassemblerTables.h
@@ -46,7 +46,8 @@ class DisassemblerTables {
   /// [7] 3dnow map opcode
   /// [8] fixed length MAP5 opcode
   /// [9] fixed length MAP6 opcode
-  std::unique_ptr<ContextDecision> Tables[10];
+  /// [10] fixed length MAP7 opcode
+  std::unique_ptr<ContextDecision> Tables[11];
 
   // Table of ModRM encodings.
   typedef std::map<std::vector<unsigned>, unsigned> ModRMMapTy;
diff --git a/llvm/utils/TableGen/X86RecognizableInstr.cpp b/llvm/utils/TableGen/X86RecognizableInstr.cpp
index b2f51ba016..962da623b1 100644
--- a/llvm/utils/TableGen/X86RecognizableInstr.cpp
+++ b/llvm/utils/TableGen/X86RecognizableInstr.cpp
@@ -791,6 +791,7 @@ void RecognizableInstr::emitDecodePath(DisassemblerTables &tables) const {
   case X86Local::ThreeDNow: opcodeType = THREEDNOW_MAP; break;
   case X86Local::T_MAP5:    opcodeType = MAP5;          break;
   case X86Local::T_MAP6:    opcodeType = MAP6;          break;
+  case X86Local::T_MAP7:    opcodeType = MAP7;          break;
   }
 
   std::unique_ptr<ModRMFilter> filter;
diff --git a/llvm/utils/TableGen/X86RecognizableInstr.h b/llvm/utils/TableGen/X86RecognizableInstr.h
index 5efacdb274..38bca87bfe 100644
--- a/llvm/utils/TableGen/X86RecognizableInstr.h
+++ b/llvm/utils/TableGen/X86RecognizableInstr.h
@@ -137,7 +137,7 @@ namespace X86Local {
 
   enum {
     OB = 0, TB = 1, T8 = 2, TA = 3, XOP8 = 4, XOP9 = 5, XOPA = 6, ThreeDNow = 7,
-    T_MAP5 = 8, T_MAP6 = 9
+    T_MAP5 = 8, T_MAP6 = 9, T_MAP7 = 10
   };
 
   enum {
diff --git a/llvm/utils/TableGen/jupyter/README.md b/llvm/utils/TableGen/jupyter/README.md
index 4356a90787..79c986a3fc 100644
--- a/llvm/utils/TableGen/jupyter/README.md
+++ b/llvm/utils/TableGen/jupyter/README.md
@@ -7,6 +7,8 @@ TableGen.
 
 [LLVM_TableGen.ipynb](LLVM_TableGen.ipynb) - A demo of the kernel's capabilities.
 
+[tablegen_tutorial_part_1.ipynb](tablegen_tutorial_part_1.ipynb) - A tutorial on the TableGen language.
+
 [sql_query_backend.ipynb](sql_query_backend.ipynb) - How to write a backend using
 JSON output and Python.
 
diff --git a/llvm/utils/emacs/llvm-mode.el b/llvm/utils/emacs/llvm-mode.el
index e37cc693a1..53381cf91b 100644
--- a/llvm/utils/emacs/llvm-mode.el
+++ b/llvm/utils/emacs/llvm-mode.el
@@ -25,7 +25,7 @@
        '("alwaysinline" "argmemonly" "allocsize" "builtin" "cold" "convergent" "dereferenceable" "dereferenceable_or_null" "hot" "immarg" "inaccessiblememonly"
          "inaccessiblemem_or_argmemonly" "inalloca" "inlinehint" "jumptable" "minsize" "mustprogress" "naked" "nobuiltin" "nonnull" "nocapture"
          "nocallback" "nocf_check" "noduplicate" "nofree" "noimplicitfloat" "noinline" "nomerge" "nonlazybind" "noprofile" "noredzone" "noreturn"
-         "norecurse" "nosync" "noundef" "nounwind" "nosanitize_bounds" "nosanitize_coverage" "null_pointer_is_valid" "optforfuzzing" "optnone" "optsize" "preallocated" "readnone" "readonly" "returned" "returns_twice"
+         "norecurse" "nosync" "noundef" "nounwind" "nosanitize_bounds" "nosanitize_coverage" "null_pointer_is_valid" "optdebug" "optforfuzzing" "optnone" "optsize" "preallocated" "readnone" "readonly" "returned" "returns_twice"
          "shadowcallstack" "signext" "speculatable" "speculative_load_hardening" "ssp" "sspreq" "sspstrong" "safestack" "sanitize_address" "sanitize_hwaddress" "sanitize_memtag"
          "sanitize_thread" "sanitize_memory" "strictfp" "swifterror" "uwtable" "vscale_range" "willreturn" "writeonly" "zeroext") 'symbols) . font-lock-constant-face)
    ;; Variables
diff --git a/llvm/utils/emacs/tablegen-mode.el b/llvm/utils/emacs/tablegen-mode.el
index 572823fa56..330da46588 100644
--- a/llvm/utils/emacs/tablegen-mode.el
+++ b/llvm/utils/emacs/tablegen-mode.el
@@ -19,7 +19,7 @@
 (defvar tablegen-font-lock-keywords
   (let ((kw (regexp-opt '("class" "defm" "def" "field" "include" "in"
                          "let" "multiclass" "foreach" "if" "then" "else"
-                         "defvar" "defset")
+                         "defvar" "defset" "dump" "assert")
                         'words))
         (type-kw (regexp-opt '("bit" "bits" "code" "dag" "int" "list" "string")
                              'words))
diff --git a/llvm/utils/git/code-format-helper.py b/llvm/utils/git/code-format-helper.py
index 8d3c30b309..742fa5a2ee 100644
--- a/llvm/utils/git/code-format-helper.py
+++ b/llvm/utils/git/code-format-helper.py
@@ -20,14 +20,21 @@
 
 class FormatHelper:
     COMMENT_TAG = "<!--LLVM CODE FORMAT COMMENT: {fmt}-->"
-    name = "unknown"
+    name: str
+    friendly_name: str
 
     @property
     def comment_tag(self) -> str:
         return self.COMMENT_TAG.replace("fmt", self.name)
 
-    def format_run(self, changed_files: [str], args: argparse.Namespace) -> str | None:
-        pass
+    @property
+    def instructions(self) -> str:
+        raise NotImplementedError()
+
+    def format_run(
+        self, changed_files: list[str], args: argparse.Namespace
+    ) -> str | None:
+        raise NotImplementedError()
 
     def pr_comment_text(self, diff: str) -> str:
         return f"""
@@ -66,7 +73,7 @@ def find_comment(
                 return comment
         return None
 
-    def update_pr(self, diff: str, args: argparse.Namespace):
+    def update_pr(self, diff: str, args: argparse.Namespace) -> None:
         repo = github.Github(args.token).get_repo(args.repo)
         pr = repo.get_issue(args.issue_number).as_pull_request()
 
@@ -78,7 +85,7 @@ def update_pr(self, diff: str, args: argparse.Namespace):
         else:
             pr.as_issue().create_comment(pr_text)
 
-    def update_pr_success(self, args: argparse.Namespace):
+    def update_pr_success(self, args: argparse.Namespace) -> None:
         repo = github.Github(args.token).get_repo(args.repo)
         pr = repo.get_issue(args.issue_number).as_pull_request()
 
@@ -91,7 +98,7 @@ def update_pr_success(self, args: argparse.Namespace):
 """
             )
 
-    def run(self, changed_files: [str], args: argparse.Namespace):
+    def run(self, changed_files: list[str], args: argparse.Namespace) -> bool:
         diff = self.format_run(changed_files, args)
         if diff:
             self.update_pr(diff, args)
@@ -106,11 +113,11 @@ class ClangFormatHelper(FormatHelper):
     friendly_name = "C/C++ code formatter"
 
     @property
-    def instructions(self):
+    def instructions(self) -> str:
         return " ".join(self.cf_cmd)
 
     @cached_property
-    def libcxx_excluded_files(self):
+    def libcxx_excluded_files(self) -> list[str]:
         with open("libcxx/utils/data/ignore_format.txt", "r") as ifd:
             return [excl.strip() for excl in ifd.readlines()]
 
@@ -120,7 +127,7 @@ def should_be_excluded(self, path: str) -> bool:
             return True
         return False
 
-    def filter_changed_files(self, changed_files: [str]) -> [str]:
+    def filter_changed_files(self, changed_files: list[str]) -> list[str]:
         filtered_files = []
         for path in changed_files:
             _, ext = os.path.splitext(path)
@@ -129,10 +136,12 @@ def filter_changed_files(self, changed_files: [str]) -> [str]:
                     filtered_files.append(path)
         return filtered_files
 
-    def format_run(self, changed_files: [str], args: argparse.Namespace) -> str | None:
+    def format_run(
+        self, changed_files: list[str], args: argparse.Namespace
+    ) -> str | None:
         cpp_files = self.filter_changed_files(changed_files)
         if not cpp_files:
-            return
+            return None
         cf_cmd = [
             "git-clang-format",
             "--diff",
@@ -156,10 +165,10 @@ class DarkerFormatHelper(FormatHelper):
     friendly_name = "Python code formatter"
 
     @property
-    def instructions(self):
+    def instructions(self) -> str:
         return " ".join(self.darker_cmd)
 
-    def filter_changed_files(self, changed_files: [str]) -> [str]:
+    def filter_changed_files(self, changed_files: list[str]) -> list[str]:
         filtered_files = []
         for path in changed_files:
             name, ext = os.path.splitext(path)
@@ -168,10 +177,12 @@ def filter_changed_files(self, changed_files: [str]) -> [str]:
 
         return filtered_files
 
-    def format_run(self, changed_files: [str], args: argparse.Namespace) -> str | None:
+    def format_run(
+        self, changed_files: list[str], args: argparse.Namespace
+    ) -> str | None:
         py_files = self.filter_changed_files(changed_files)
         if not py_files:
-            return
+            return None
         darker_cmd = [
             "darker",
             "--check",
diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/bugprone/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/bugprone/BUILD.gn
index b9b5714b92..dcc2520006 100644
--- a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/bugprone/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/bugprone/BUILD.gn
@@ -23,6 +23,7 @@ static_library("bugprone") {
     "BoolPointerImplicitConversionCheck.cpp",
     "BranchCloneCheck.cpp",
     "BugproneTidyModule.cpp",
+    "CastingThroughVoidCheck.cpp",
     "ComparePointerToMemberVirtualFunctionCheck.cpp",
     "CopyConstructorInitCheck.cpp",
     "DanglingHandleCheck.cpp",
diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/misc/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/misc/BUILD.gn
index 8a811bc990..36957f502c 100644
--- a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/misc/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/misc/BUILD.gn
@@ -34,6 +34,7 @@ static_library("misc") {
   sources = [
     "ConfusableIdentifierCheck.cpp",
     "ConstCorrectnessCheck.cpp",
+    "CoroutineHostileRAIICheck.cpp",
     "DefinitionsInHeadersCheck.cpp",
     "HeaderIncludeCycleCheck.cpp",
     "IncludeCleanerCheck.cpp",
diff --git a/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn
index 9fe2dda6f2..a70ff97299 100644
--- a/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn
@@ -264,6 +264,7 @@ copy("Headers") {
     "stdarg.h",
     "stdatomic.h",
     "stdbool.h",
+    "stdckdint.h",
     "stddef.h",
     "stdint.h",
     "stdnoreturn.h",
@@ -273,6 +274,7 @@ copy("Headers") {
     "tsxldtrkintrin.h",
     "uintrintrin.h",
     "unwind.h",
+    "usermsrintrin.h",
     "vadefs.h",
     "vaesintrin.h",
     "varargs.h",
diff --git a/llvm/utils/gn/secondary/clang/unittests/AST/Interp/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/AST/Interp/BUILD.gn
new file mode 100644
index 0000000000..dd47a3d2d3
--- /dev/null
+++ b/llvm/utils/gn/secondary/clang/unittests/AST/Interp/BUILD.gn
@@ -0,0 +1,13 @@
+import("//third-party/unittest/unittest.gni")
+
+unittest("InterpTests") {
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/AST",
+    "//clang/lib/Basic",
+    "//clang/lib/Frontend",
+    "//clang/lib/Testing",
+    "//clang/lib/Tooling",
+  ]
+  sources = [ "Descriptor.cpp" ]
+}
diff --git a/llvm/utils/gn/secondary/clang/unittests/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/BUILD.gn
index f438103902..b60c5264d6 100644
--- a/llvm/utils/gn/secondary/clang/unittests/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/unittests/BUILD.gn
@@ -3,6 +3,7 @@ import("//clang/lib/StaticAnalyzer/Frontend/enable.gni")
 group("unittests") {
   deps = [
     "AST:ASTTests",
+    "AST/Interp:InterpTests",
     "ASTMatchers:ASTMatchersTests",
     "ASTMatchers/Dynamic:DynamicASTMatchersTests",
     "Basic:BasicTests",
diff --git a/llvm/utils/gn/secondary/clang/unittests/Driver/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/Driver/BUILD.gn
index 14900e2254..4c7e3c110b 100644
--- a/llvm/utils/gn/secondary/clang/unittests/Driver/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/unittests/Driver/BUILD.gn
@@ -14,6 +14,7 @@ unittest("ClangDriverTests") {
   sources = [
     "DXCModeTest.cpp",
     "DistroTest.cpp",
+    "GCCVersionTest.cpp",
     "ModuleCacheTest.cpp",
     "MultilibBuilderTest.cpp",
     "MultilibTest.cpp",
diff --git a/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn
index 303a6c29d7..3a19729bb8 100644
--- a/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn
+++ b/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn
@@ -271,6 +271,7 @@ static_library("builtins") {
     sources += [
       "cpu_model.c",
       "divxc3.c",
+      "extendxftf2.c",
       "fixunsxfdi.c",
       "fixunsxfsi.c",
       "fixunsxfti.c",
@@ -285,6 +286,7 @@ static_library("builtins") {
       "powixf2.c",
       "truncdfbf2.c",
       "truncsfbf2.c",
+      "trunctfxf2.c",
     ]
   }
   if (current_cpu == "x86") {
diff --git a/llvm/utils/gn/secondary/compiler-rt/lib/scudo/standalone/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/lib/scudo/standalone/BUILD.gn
index ed581d8f75..02a9595e26 100644
--- a/llvm/utils/gn/secondary/compiler-rt/lib/scudo/standalone/BUILD.gn
+++ b/llvm/utils/gn/secondary/compiler-rt/lib/scudo/standalone/BUILD.gn
@@ -20,6 +20,10 @@ source_set("sources") {
     "combined.h",
     "common.cpp",
     "common.h",
+    "condition_variable.h",
+    "condition_variable_base.h",
+    "condition_variable_linux.cpp",
+    "condition_variable_linux.h",
     "crc32_hw.cpp",
     "flags.cpp",
     "flags.h",
@@ -51,6 +55,8 @@ source_set("sources") {
     "release.h",
     "report.cpp",
     "report.h",
+    "report_linux.cpp",
+    "report_linux.h",
     "secondary.h",
     "size_class_map.h",
     "stack_depot.h",
diff --git a/llvm/utils/gn/secondary/compiler-rt/lib/scudo/standalone/tests/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/lib/scudo/standalone/tests/BUILD.gn
index b8fb774729..f69f1c413b 100644
--- a/llvm/utils/gn/secondary/compiler-rt/lib/scudo/standalone/tests/BUILD.gn
+++ b/llvm/utils/gn/secondary/compiler-rt/lib/scudo/standalone/tests/BUILD.gn
@@ -14,6 +14,7 @@ unittest("ScudoUnitTest") {
     "chunk_test.cpp",
     "combined_test.cpp",
     "common_test.cpp",
+    "condition_variable_test.cpp",
     "flags_test.cpp",
     "list_test.cpp",
     "map_test.cpp",
diff --git a/llvm/utils/gn/secondary/compiler-rt/lib/tsan/rtl/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/lib/tsan/rtl/BUILD.gn
index 7abf53f94d..ce5f863158 100644
--- a/llvm/utils/gn/secondary/compiler-rt/lib/tsan/rtl/BUILD.gn
+++ b/llvm/utils/gn/secondary/compiler-rt/lib/tsan/rtl/BUILD.gn
@@ -127,6 +127,8 @@ target(tsan_target_type, "rtl") {
     sources += [ "tsan_rtl_mips64.S" ]
   } else if (target_cpu == "powerpc64") {
     sources += [ "tsan_rtl_ppc64.S" ]
+  } else if (target_cpu == "riscv64") {
+    sources += [ "tsan_rtl_riscv64.S" ]
   } else if (target_cpu == "s390x") {
     sources += [ "tsan_rtl_s390x.S" ]
   }
diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index b57a353962..f350303068 100644
--- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -168,6 +168,7 @@ if (current_toolchain == default_toolchain) {
       "__algorithm/pstl_generate.h",
       "__algorithm/pstl_is_partitioned.h",
       "__algorithm/pstl_merge.h",
+      "__algorithm/pstl_move.h",
       "__algorithm/pstl_replace.h",
       "__algorithm/pstl_sort.h",
       "__algorithm/pstl_stable_sort.h",
@@ -582,6 +583,7 @@ if (current_toolchain == default_toolchain) {
       "__mdspan/extents.h",
       "__mdspan/layout_left.h",
       "__mdspan/layout_right.h",
+      "__mdspan/layout_stride.h",
       "__mdspan/mdspan.h",
       "__memory/addressof.h",
       "__memory/align.h",
@@ -1046,7 +1048,6 @@ if (current_toolchain == default_toolchain) {
       "scoped_allocator",
       "semaphore",
       "set",
-      "setjmp.h",
       "shared_mutex",
       "span",
       "sstream",
diff --git a/llvm/utils/gn/secondary/lldb/test/BUILD.gn b/llvm/utils/gn/secondary/lldb/test/BUILD.gn
index 2a5100d755..06ef7383ad 100644
--- a/llvm/utils/gn/secondary/lldb/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/test/BUILD.gn
@@ -163,12 +163,12 @@ group("test") {
     ":lit_unit_site_cfg",
     "//clang/tools/driver:symlinks",
     "//lld/tools/lld:symlinks",
+    "//lldb/tools/lldb-dap",
     "//lldb/tools/driver:lldb",
 
     # XXX lldb-instr, darwin-debug, etc
     "//lldb/tools/lldb-server",
     "//lldb/tools/lldb-test",
-    "//lldb/tools/lldb-vscode",
     "//lldb/utils/lit-cpuid",
 
     #"//lldb/unittests",
diff --git a/llvm/utils/gn/secondary/lldb/tools/lldb-vscode/BUILD.gn b/llvm/utils/gn/secondary/lldb/tools/lldb-dap/BUILD.gn
similarity index 86%
rename from llvm/utils/gn/secondary/lldb/tools/lldb-vscode/BUILD.gn
rename to llvm/utils/gn/secondary/lldb/tools/lldb-dap/BUILD.gn
index f6a22670fb..d8292df8c0 100644
--- a/llvm/utils/gn/secondary/lldb/tools/lldb-vscode/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/tools/lldb-dap/BUILD.gn
@@ -3,19 +3,19 @@ import("//llvm/utils/gn/build/write_cmake_config.gni")
 import("//llvm/version.gni")
 
 tablegen("Options") {
-  visibility = [ ":lldb-vscode" ]
+  visibility = [ ":lldb-dap" ]
   args = [ "-gen-opt-parser-defs" ]
 }
 
 if (host_os == "mac") {
   write_cmake_config("write_info_plist") {
-    input = "lldb-vscode-Info.plist.in"
-    output = "$target_gen_dir/lldb-vscode-Info.plist"
+    input = "lldb-dap-Info.plist.in"
+    output = "$target_gen_dir/lldb-dap-Info.plist"
     values = [ "LLDB_VERSION=$llvm_version" ]
   }
 }
 
-executable("lldb-vscode") {
+executable("lldb-dap") {
   configs += [ "//llvm/utils/gn/build:lldb_code" ]
   deps = [
     ":Options",
@@ -39,6 +39,7 @@ executable("lldb-vscode") {
 
   sources = [
     "BreakpointBase.cpp",
+    "DAP.cpp",
     "ExceptionBreakpoint.cpp",
     "FifoFiles.cpp",
     "FunctionBreakpoint.cpp",
@@ -49,7 +50,6 @@ executable("lldb-vscode") {
     "ProgressEvent.cpp",
     "RunInTerminal.cpp",
     "SourceBreakpoint.cpp",
-    "VSCode.cpp",
-    "lldb-vscode.cpp",
+    "lldb-dap.cpp",
   ]
 }
diff --git a/llvm/utils/gn/secondary/llvm/lib/Passes/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Passes/BUILD.gn
index cba3bf6cd3..d98420100d 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Passes/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Passes/BUILD.gn
@@ -9,6 +9,7 @@ static_library("Passes") {
     "//llvm/lib/Target",
     "//llvm/lib/Transforms/AggressiveInstCombine",
     "//llvm/lib/Transforms/Coroutines",
+    "//llvm/lib/Transforms/HipStdPar",
     "//llvm/lib/Transforms/IPO",
     "//llvm/lib/Transforms/InstCombine",
     "//llvm/lib/Transforms/Instrumentation",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
index eb9df8ac23..1afff26bca 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
@@ -120,6 +120,7 @@ static_library("LLVMAMDGPUCodeGen") {
     "//llvm/lib/Support",
     "//llvm/lib/Target",
     "//llvm/lib/TargetParser",
+    "//llvm/lib/Transforms/HipStdPar",
     "//llvm/lib/Transforms/IPO",
     "//llvm/lib/Transforms/Scalar",
     "//llvm/lib/Transforms/Utils",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/X86/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/X86/BUILD.gn
index 921cac1a6f..e8aa57fc9b 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/X86/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/X86/BUILD.gn
@@ -122,7 +122,6 @@ static_library("LLVMX86CodeGen") {
     "X86OptimizeLEAs.cpp",
     "X86PadShortFunction.cpp",
     "X86PartialReduction.cpp",
-    "X86PreAMXConfig.cpp",
     "X86PreTileConfig.cpp",
     "X86RegisterInfo.cpp",
     "X86ReturnThunks.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Transforms/HipStdPar/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Transforms/HipStdPar/BUILD.gn
new file mode 100644
index 0000000000..f564817957
--- /dev/null
+++ b/llvm/utils/gn/secondary/llvm/lib/Transforms/HipStdPar/BUILD.gn
@@ -0,0 +1,10 @@
+static_library("HipStdPar") {
+  output_name = "LLVMHipStdPar"
+  deps = [
+    "//llvm/lib/Analysis",
+    "//llvm/lib/IR",
+    "//llvm/lib/Support",
+    "//llvm/lib/Transforms/Utils",
+  ]
+  sources = [ "HipStdPar.cpp" ]
+}
diff --git a/llvm/utils/gn/secondary/llvm/test/BUILD.gn b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
index f859af249f..dd9fd0c10d 100644
--- a/llvm/utils/gn/secondary/llvm/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
@@ -222,7 +222,10 @@ write_lit_config("lit_site_cfg") {
 write_lit_config("lit_unit_site_cfg") {
   input = "//llvm/test/Unit/lit.site.cfg.py.in"
   output = llvm_lit_unit_site_cfg_file
-  extra_values = [ "LLVM_BUILD_MODE=." ]
+  extra_values = [
+    "LLVM_BUILD_MODE=.",
+    "LLVM_GTEST_RUN_UNDER=",
+  ]
 }
 
 # This target should contain all dependencies of check-llvm.
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-readtapi/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-readtapi/BUILD.gn
index c9d8afb2f1..bcd883b0d9 100644
--- a/llvm/utils/gn/secondary/llvm/tools/llvm-readtapi/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-readtapi/BUILD.gn
@@ -1,3 +1,22 @@
+import("//llvm/tools/binutils_symlinks.gni")
+import("//llvm/utils/gn/build/symlink_or_copy.gni")
+
+if (llvm_install_binutils_symlinks) {
+  symlink_or_copy("readtapi") {
+    deps = [ ":llvm-readtapi" ]
+    source = "llvm-readtapi"
+    output = "$root_out_dir/bin/readtapi"
+  }
+}
+
+# //:llvm-readtapi depends on this symlink target, see comment in //BUILD.gn.
+group("symlinks") {
+  deps = [ ":llvm-readtapi" ]
+  if (llvm_install_binutils_symlinks) {
+    deps += [ ":readtapi" ]
+  }
+}
+
 executable("llvm-readtapi") {
   deps = [
     "//llvm/lib/Object",
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-remarkutil/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-remarkutil/BUILD.gn
index e55e82ff6f..920abdc65f 100644
--- a/llvm/utils/gn/secondary/llvm/tools/llvm-remarkutil/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-remarkutil/BUILD.gn
@@ -9,6 +9,7 @@ executable("llvm-remarkutil") {
   sources = [
     "RemarkConvert.cpp",
     "RemarkCount.cpp",
+    "RemarkCounter.cpp",
     "RemarkSizeDiff.cpp",
     "RemarkUtil.cpp",
     "RemarkUtilHelpers.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/unittests/ADT/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/ADT/BUILD.gn
index 00a3b599c8..15c198c73f 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/ADT/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/ADT/BUILD.gn
@@ -42,6 +42,7 @@ unittest("ADTTests") {
     "FunctionRefTest.cpp",
     "HashingTest.cpp",
     "IListBaseTest.cpp",
+    "IListIteratorBitsTest.cpp",
     "IListIteratorTest.cpp",
     "IListNodeBaseTest.cpp",
     "IListNodeTest.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn
index c7e2b49f74..fddee57954 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn
@@ -93,6 +93,7 @@ unittest("SupportTests") {
     "TypeNameTest.cpp",
     "TypeSizeTest.cpp",
     "TypeTraitsTest.cpp",
+    "UTCTimeTest.cpp",
     "UnicodeTest.cpp",
     "VersionTupleTest.cpp",
     "VirtualFileSystemTest.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/unittests/Target/RISCV/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/Target/RISCV/BUILD.gn
index 707704fb9d..6d19d22a1a 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/Target/RISCV/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/Target/RISCV/BUILD.gn
@@ -9,5 +9,8 @@ unittest("RISCVTests") {
     "//llvm/lib/TargetParser",
   ]
   include_dirs = [ "//llvm/lib/Target/RISCV" ]
-  sources = [ "MCInstrAnalysisTest.cpp" ]
+  sources = [
+    "MCInstrAnalysisTest.cpp",
+    "RISCVBaseInfoTest.cpp",
+  ]
 }
diff --git a/llvm/utils/kate/llvm-tablegen.xml b/llvm/utils/kate/llvm-tablegen.xml
index 486d373a11..a460710495 100644
--- a/llvm/utils/kate/llvm-tablegen.xml
+++ b/llvm/utils/kate/llvm-tablegen.xml
@@ -44,6 +44,7 @@
       <item> !ne </item>
       <item> !tolower </item>
       <item> !toupper </item>
+      <item> !repr </item>
     </list>
     <list name="objects">
       <item> class </item>
@@ -53,6 +54,8 @@
       <item> let </item>
       <item> defvar </item>
       <item> multiclass </item>
+      <item> assert </item>
+      <item> dump </item>
     </list>
     <list name="class-like">
       <item> class </item>
diff --git a/llvm/utils/kate/llvm.xml b/llvm/utils/kate/llvm.xml
index 9f7ec77bf3..0e7aec3880 100644
--- a/llvm/utils/kate/llvm.xml
+++ b/llvm/utils/kate/llvm.xml
@@ -111,6 +111,7 @@
       <item> nosync </item>
       <item> nounwind </item>
       <item> null_pointer_is_valid </item>
+      <item> optdebug </item>
       <item> optforfuzzing </item>
       <item> optnone </item>
       <item> optsize </item>
diff --git a/llvm/utils/lit/lit/LitConfig.py b/llvm/utils/lit/lit/LitConfig.py
index d7e79b60f3..5dc712ae28 100644
--- a/llvm/utils/lit/lit/LitConfig.py
+++ b/llvm/utils/lit/lit/LitConfig.py
@@ -37,6 +37,7 @@ def __init__(
         maxIndividualTestTime=0,
         parallelism_groups={},
         per_test_coverage=False,
+        gtest_sharding=True,
     ):
         # The name of the test runner.
         self.progname = progname
@@ -87,6 +88,7 @@ def __init__(
         self.maxIndividualTestTime = maxIndividualTestTime
         self.parallelism_groups = parallelism_groups
         self.per_test_coverage = per_test_coverage
+        self.gtest_sharding = bool(gtest_sharding)
 
     @property
     def maxIndividualTestTime(self):
diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py
index a6314e35c1..788152846e 100644
--- a/llvm/utils/lit/lit/TestRunner.py
+++ b/llvm/utils/lit/lit/TestRunner.py
@@ -12,6 +12,8 @@
 import shutil
 import tempfile
 import threading
+import typing
+from typing import Optional, Tuple
 
 import io
 
@@ -34,6 +36,17 @@ def __init__(self, command, message):
         self.message = message
 
 
+class ScriptFatal(Exception):
+    """
+    A script had a fatal error such that there's no point in retrying.  The
+    message has not been emitted on stdout or stderr but is instead included in
+    this exception.
+    """
+
+    def __init__(self, message):
+        super().__init__(message)
+
+
 kIsWindows = platform.system() == "Windows"
 
 # Don't use close_fds on Windows.
@@ -1009,7 +1022,8 @@ def formatOutput(title, data, limit=None):
     return out
 
 
-# Normally returns out, err, exitCode, timeoutInfo.
+# Always either returns the tuple (out, err, exitCode, timeoutInfo) or raises a
+# ScriptFatal exception.
 #
 # If debug is True (the normal lit behavior), err is empty, and out contains an
 # execution trace, including stdout and stderr shown per command executed.
@@ -1017,7 +1031,9 @@ def formatOutput(title, data, limit=None):
 # If debug is False (set by some custom lit test formats that call this
 # function), out contains only stdout from the script, err contains only stderr
 # from the script, and there is no execution trace.
-def executeScriptInternal(test, litConfig, tmpBase, commands, cwd, debug=True):
+def executeScriptInternal(
+    test, litConfig, tmpBase, commands, cwd, debug=True
+) -> Tuple[str, str, int, Optional[str]]:
     cmds = []
     for i, ln in enumerate(commands):
         # Within lit, we try to always add '%dbg(...)' to command lines in order
@@ -1043,9 +1059,9 @@ def executeScriptInternal(test, litConfig, tmpBase, commands, cwd, debug=True):
                 ShUtil.ShParser(ln, litConfig.isWindows, test.config.pipefail).parse()
             )
         except:
-            return lit.Test.Result(
-                Test.FAIL, f"shell parser error on {dbg}: {command.lstrip()}\n"
-            )
+            raise ScriptFatal(
+                f"shell parser error on {dbg}: {command.lstrip()}\n"
+            ) from None
 
     cmd = cmds[0]
     for c in cmds[1:]:
@@ -2130,8 +2146,11 @@ def parseIntegratedTestScript(test, additional_parsers=[], require_script=True):
     return script
 
 
-def _runShTest(test, litConfig, useExternalSh, script, tmpBase):
-    def runOnce(execdir):
+def _runShTest(test, litConfig, useExternalSh, script, tmpBase) -> lit.Test.Result:
+    # Always returns the tuple (out, err, exitCode, timeoutInfo, status).
+    def runOnce(
+        execdir,
+    ) -> Tuple[str, str, int, Optional[str], Test.ResultCode]:
         # script is modified below (for litConfig.per_test_coverage, and for
         # %dbg expansions).  runOnce can be called multiple times, but applying
         # the modifications multiple times can corrupt script, so always modify
@@ -2158,12 +2177,16 @@ def runOnce(execdir):
                     command = buildPdbgCommand(dbg, command)
                 scriptCopy[i] = command
 
-        if useExternalSh:
-            res = executeScript(test, litConfig, tmpBase, scriptCopy, execdir)
-        else:
-            res = executeScriptInternal(test, litConfig, tmpBase, scriptCopy, execdir)
-        if isinstance(res, lit.Test.Result):
-            return res
+        try:
+            if useExternalSh:
+                res = executeScript(test, litConfig, tmpBase, scriptCopy, execdir)
+            else:
+                res = executeScriptInternal(
+                    test, litConfig, tmpBase, scriptCopy, execdir
+                )
+        except ScriptFatal as e:
+            out = f"# " + "\n# ".join(str(e).splitlines()) + "\n"
+            return out, "", 1, None, Test.UNRESOLVED
 
         out, err, exitCode, timeoutInfo = res
         if exitCode == 0:
@@ -2183,9 +2206,6 @@ def runOnce(execdir):
     attempts = test.allowed_retries + 1
     for i in range(attempts):
         res = runOnce(execdir)
-        if isinstance(res, lit.Test.Result):
-            return res
-
         out, err, exitCode, timeoutInfo, status = res
         if status != Test.FAIL:
             break
diff --git a/llvm/utils/lit/lit/TestingConfig.py b/llvm/utils/lit/lit/TestingConfig.py
index 541bd01ef8..eb9f8de2a7 100644
--- a/llvm/utils/lit/lit/TestingConfig.py
+++ b/llvm/utils/lit/lit/TestingConfig.py
@@ -43,6 +43,7 @@ def fromdefaults(litConfig):
             "TSAN_OPTIONS",
             "UBSAN_OPTIONS",
             "ADB",
+            "ADB_SERVER_SOCKET",
             "ANDROID_SERIAL",
             "SSH_AUTH_SOCK",
             "SANITIZER_IGNORE_CVE_2016_2143",
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index 132476fb2a..ba37066595 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -118,6 +118,18 @@ def parse_args():
         )
 
     execution_group = parser.add_argument_group("Test Execution")
+    execution_group.add_argument(
+        "--gtest-sharding",
+        help="Enable sharding for GoogleTest format",
+        action="store_true",
+        default=True,
+    )
+    execution_group.add_argument(
+        "--no-gtest-sharding",
+        dest="gtest_sharding",
+        help="Disable sharding for GoogleTest format",
+        action="store_false",
+    )
     execution_group.add_argument(
         "--path",
         help="Additional paths to add to testing environment",
diff --git a/llvm/utils/lit/lit/formats/googletest.py b/llvm/utils/lit/lit/formats/googletest.py
index f8304cbd05..8037094a91 100644
--- a/llvm/utils/lit/lit/formats/googletest.py
+++ b/llvm/utils/lit/lit/formats/googletest.py
@@ -68,24 +68,49 @@ def getTestsInDirectory(self, testSuite, path_in_suite, litConfig, localConfig):
                     self.seen_executables.add(execpath)
                 num_tests = self.get_num_tests(execpath, litConfig, localConfig)
                 if num_tests is not None:
-                    # Compute the number of shards.
-                    shard_size = init_shard_size
-                    nshard = int(math.ceil(num_tests / shard_size))
-                    while nshard < core_count and shard_size > 1:
-                        shard_size = shard_size // 2
+                    if litConfig.gtest_sharding:
+                        # Compute the number of shards.
+                        shard_size = init_shard_size
                         nshard = int(math.ceil(num_tests / shard_size))
-
-                    # Create one lit test for each shard.
-                    for idx in range(nshard):
-                        testPath = path_in_suite + (subdir, fn, str(idx), str(nshard))
+                        while nshard < core_count and shard_size > 1:
+                            shard_size = shard_size // 2
+                            nshard = int(math.ceil(num_tests / shard_size))
+
+                        # Create one lit test for each shard.
+                        for idx in range(nshard):
+                            testPath = path_in_suite + (
+                                subdir,
+                                fn,
+                                str(idx),
+                                str(nshard),
+                            )
+                            json_file = (
+                                "-".join(
+                                    [
+                                        execpath,
+                                        testSuite.config.name,
+                                        str(os.getpid()),
+                                        str(idx),
+                                        str(nshard),
+                                    ]
+                                )
+                                + ".json"
+                            )
+                            yield lit.Test.Test(
+                                testSuite,
+                                testPath,
+                                localConfig,
+                                file_path=execpath,
+                                gtest_json_file=json_file,
+                            )
+                    else:
+                        testPath = path_in_suite + (subdir, fn)
                         json_file = (
                             "-".join(
                                 [
                                     execpath,
                                     testSuite.config.name,
                                     str(os.getpid()),
-                                    str(idx),
-                                    str(nshard),
                                 ]
                             )
                             + ".json"
@@ -118,24 +143,32 @@ def execute(self, test, litConfig):
         if test.gtest_json_file is None:
             return lit.Test.FAIL, ""
 
-        testPath, testName = os.path.split(test.getSourcePath())
-        while not os.path.exists(testPath):
-            # Handle GTest parametrized and typed tests, whose name includes
-            # some '/'s.
-            testPath, namePrefix = os.path.split(testPath)
-            testName = namePrefix + "/" + testName
-
-        testName, total_shards = os.path.split(testName)
-        testName, shard_idx = os.path.split(testName)
+        testPath = test.getSourcePath()
         from lit.cl_arguments import TestOrder
 
         use_shuffle = TestOrder(litConfig.order) == TestOrder.RANDOM
         shard_env = {
             "GTEST_OUTPUT": "json:" + test.gtest_json_file,
             "GTEST_SHUFFLE": "1" if use_shuffle else "0",
-            "GTEST_TOTAL_SHARDS": os.environ.get("GTEST_TOTAL_SHARDS", total_shards),
-            "GTEST_SHARD_INDEX": os.environ.get("GTEST_SHARD_INDEX", shard_idx),
         }
+        if litConfig.gtest_sharding:
+            testPath, testName = os.path.split(test.getSourcePath())
+            while not os.path.exists(testPath):
+                # Handle GTest parameterized and typed tests, whose name includes
+                # some '/'s.
+                testPath, namePrefix = os.path.split(testPath)
+                testName = namePrefix + "/" + testName
+
+            testName, total_shards = os.path.split(testName)
+            testName, shard_idx = os.path.split(testName)
+            shard_env.update(
+                {
+                    "GTEST_TOTAL_SHARDS": os.environ.get(
+                        "GTEST_TOTAL_SHARDS", total_shards
+                    ),
+                    "GTEST_SHARD_INDEX": os.environ.get("GTEST_SHARD_INDEX", shard_idx),
+                }
+            )
         test.config.environment.update(shard_env)
 
         cmd = [testPath]
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index b2a578e0ae..db9f24f748 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -41,6 +41,7 @@ def main(builtin_params={}):
         params=params,
         config_prefix=opts.configPrefix,
         per_test_coverage=opts.per_test_coverage,
+        gtest_sharding=opts.gtest_sharding,
     )
 
     discovered_tests = lit.discovery.find_tests_for_inputs(
@@ -310,6 +311,7 @@ def print_histogram(tests):
 
 def print_results(tests, elapsed, opts):
     tests_by_code = {code: [] for code in lit.Test.ResultCode.all_codes()}
+    total_tests = len(tests)
     for test in tests:
         tests_by_code[test.result.code].append(test)
 
@@ -320,7 +322,7 @@ def print_results(tests, elapsed, opts):
             opts.shown_codes,
         )
 
-    print_summary(tests_by_code, opts.quiet, elapsed)
+    print_summary(total_tests, tests_by_code, opts.quiet, elapsed)
 
 
 def print_group(tests, code, shown_codes):
@@ -335,10 +337,11 @@ def print_group(tests, code, shown_codes):
     sys.stdout.write("\n")
 
 
-def print_summary(tests_by_code, quiet, elapsed):
+def print_summary(total_tests, tests_by_code, quiet, elapsed):
     if not quiet:
         print("\nTesting Time: %.2fs" % elapsed)
 
+    print("\nTotal Discovered Tests: %s" % (total_tests))
     codes = [c for c in lit.Test.ResultCode.all_codes() if not quiet or c.isFailure]
     groups = [(c.label, len(tests_by_code[c])) for c in codes]
     groups = [(label, count) for label, count in groups if count]
@@ -351,4 +354,4 @@ def print_summary(tests_by_code, quiet, elapsed):
     for (label, count) in groups:
         label = label.ljust(max_label_len)
         count = str(count).rjust(max_count_len)
-        print("  %s: %s" % (label, count))
+        print("  %s: %s (%.2f%%)" % (label, count, float(count) / total_tests * 100))
diff --git a/llvm/utils/lit/tests/Inputs/googletest-no-sharding/DummySubDir/OneTest.py b/llvm/utils/lit/tests/Inputs/googletest-no-sharding/DummySubDir/OneTest.py
new file mode 100644
index 0000000000..38d76cd637
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/googletest-no-sharding/DummySubDir/OneTest.py
@@ -0,0 +1,105 @@
+#!/usr/bin/env python
+
+import os
+import sys
+
+if len(sys.argv) == 3 and sys.argv[1] == "--gtest_list_tests":
+    if sys.argv[2] != "--gtest_filter=-*DISABLED_*":
+        raise ValueError(f"unexpected argument: {sys.argv[2]}")
+    print(
+        """\
+FirstTest.
+  subTestA
+  subTestB
+  subTestC
+  subTestD
+ParameterizedTest/0.
+  subTest
+ParameterizedTest/1.
+  subTest"""
+    )
+    sys.exit(0)
+elif len(sys.argv) != 1:
+    # sharding and json output are specified using environment variables
+    raise ValueError(f"unexpected argument: {' '.join(sys.argv[1:])!r}")
+
+for e in ["GTEST_OUTPUT"]:
+    if e not in os.environ:
+        raise ValueError(f"missing environment variables: {e}")
+
+if not os.environ["GTEST_OUTPUT"].startswith("json:"):
+    raise ValueError(f"must emit json output: {os.environ['GTEST_OUTPUT']}")
+
+output = """\
+{
+"random_seed": 123,
+"testsuites": [
+    {
+        "name": "FirstTest",
+        "testsuite": [
+            {
+                "name": "subTestA",
+                "result": "COMPLETED",
+                "time": "0.001s"
+            },
+            {
+                "name": "subTestB",
+                "result": "COMPLETED",
+                "time": "0.001s",
+                "failures": [
+                    {
+                        "failure": "I am subTest B, I FAIL\\nAnd I have two lines of output",
+                        "type": ""
+                    }
+                ]
+            },
+            {
+                "name": "subTestC",
+                "result": "SKIPPED",
+                "time": "0.001s"
+            },
+            {
+                "name": "subTestD",
+                "result": "UNRESOLVED",
+                "time": "0.001s"
+            }
+        ]
+    },
+    {
+        "name": "ParameterizedTest/0",
+        "testsuite": [
+            {
+                "name": "subTest",
+                "result": "COMPLETED",
+                "time": "0.001s"
+            }
+        ]
+    },
+    {
+        "name": "ParameterizedTest/1",
+        "testsuite": [
+            {
+                "name": "subTest",
+                "result": "COMPLETED",
+                "time": "0.001s"
+            }
+        ]
+    }
+]
+}"""
+
+dummy_output = """\
+{
+"testsuites": [
+]
+}"""
+
+json_filename = os.environ["GTEST_OUTPUT"].split(":", 1)[1]
+with open(json_filename, "w", encoding="utf-8") as f:
+    print("[ RUN      ] FirstTest.subTestB", flush=True)
+    print("I am subTest B output", file=sys.stderr, flush=True)
+    print("[  FAILED  ] FirstTest.subTestB (8 ms)", flush=True)
+    f.write(output)
+    exit_code = 1
+
+sys.exit(exit_code)
diff --git a/llvm/utils/lit/tests/Inputs/googletest-no-sharding/lit.cfg b/llvm/utils/lit/tests/Inputs/googletest-no-sharding/lit.cfg
new file mode 100644
index 0000000000..fd75709efa
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/googletest-no-sharding/lit.cfg
@@ -0,0 +1,4 @@
+import lit.formats
+
+config.name = "googletest-no-sharding"
+config.test_format = lit.formats.GoogleTest("DummySubDir", "Test")
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/before-name.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/before-name.txt
index 2ef4704790..12a7652be4 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/before-name.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/before-name.txt
@@ -4,4 +4,4 @@
 #      CHECK: Substitution name 'foo %{name}' is malformed {{.*}}
 # CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/between-name-equals.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/between-name-equals.txt
index dae61298be..ab38ffb511 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/between-name-equals.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/between-name-equals.txt
@@ -6,4 +6,4 @@
 #      CHECK: Substitution name '%{name}:' is malformed {{.*}}
 # CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/braces-empty.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/braces-empty.txt
index c6c7ececa8..07ea6563e8 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/braces-empty.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/braces-empty.txt
@@ -4,4 +4,4 @@
 #      CHECK: Substitution name '%{}' is malformed {{.*}}
 # CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/braces-with-dot.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/braces-with-dot.txt
index c0528c695d..ba89991625 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/braces-with-dot.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/braces-with-dot.txt
@@ -9,4 +9,4 @@
 #      CHECK: Substitution name '%{foo.bar}' is malformed {{.*}}
 # CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/braces-with-equals.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/braces-with-equals.txt
index 1029665015..52743b76fa 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/braces-with-equals.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/braces-with-equals.txt
@@ -6,4 +6,4 @@
 #      CHECK: Substitution name '%{foo' is malformed {{.*}}
 # CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/braces-with-newline.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/braces-with-newline.txt
index 9138beb9ef..71d6e6b216 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/braces-with-newline.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/braces-with-newline.txt
@@ -5,4 +5,4 @@
 #      CHECK: Substitution name '%{foo bar}' is malformed {{.*}}
 # CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/braces-with-number.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/braces-with-number.txt
index 72f96f3ec6..371cb244a5 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/braces-with-number.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/braces-with-number.txt
@@ -9,4 +9,4 @@
 #      CHECK: Substitution name '%{3}' is malformed {{.*}}
 # CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/braces-with-ws.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/braces-with-ws.txt
index a64531b7b5..30b58b3e74 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/braces-with-ws.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/braces-with-ws.txt
@@ -4,4 +4,4 @@
 #      CHECK: Substitution name '%{foo bar}' is malformed {{.*}}
 # CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/empty.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/empty.txt
index 7fec3c49c1..aa9e802f2d 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/empty.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/empty.txt
@@ -4,4 +4,4 @@
 #      CHECK: Substitution's definition does not contain '='
 # CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/no-equals.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/no-equals.txt
index c57283bcec..81c362cbc3 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/no-equals.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/no-equals.txt
@@ -4,4 +4,4 @@
 #      CHECK: Substitution's definition does not contain '='
 # CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/no-name.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/no-name.txt
index a39681f1f5..d11da872f4 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/no-name.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/no-name.txt
@@ -4,4 +4,4 @@
 #      CHECK: Substitution name '' is malformed {{.*}}
 # CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/ws-only.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/ws-only.txt
index eee0c77946..198a79fc27 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/ws-only.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/assignment/ws-only.txt
@@ -5,4 +5,4 @@
 #      CHECK: Substitution's definition does not contain '='
 # CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/empty.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/empty.txt
index 6a85adc04f..79326bfc3f 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/empty.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/empty.txt
@@ -7,4 +7,4 @@
 #      CHECK: Substitution's continuation is empty
 # CHECK-NEXT: in DEFINE: directive on test line [[#@LINE-4]]
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/end-in-double-backslash.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/end-in-double-backslash.txt
index acb51cfd1f..1125638011 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/end-in-double-backslash.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/end-in-double-backslash.txt
@@ -8,4 +8,4 @@
 #      CHECK: Substitution's continuation is empty
 # CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-4]]
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-define-bad-redefine.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-define-bad-redefine.txt
index 0083ccbf2e..65d3d37fd2 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-define-bad-redefine.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-define-bad-redefine.txt
@@ -10,4 +10,4 @@
 #      CHECK: Substitution's definition does not contain '='
 # CHECK-NEXT: in {{REDEFINE}}: directive on test line [[#@LINE-5]]
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-define-continuation.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-define-continuation.txt
index 40bab49f38..951266628c 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-define-continuation.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-define-continuation.txt
@@ -6,4 +6,4 @@
 
 # CHECK: Test has unterminated '{{DEFINE}}:' directive (with '\') from line [[#@LINE-3]] to [[#@LINE-2]]
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-define-redefine.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-define-redefine.txt
index 327b72df5b..ea14ba903b 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-define-redefine.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-define-redefine.txt
@@ -10,4 +10,4 @@
 
 # CHECK: Test has unterminated '{{DEFINE}}:' directive (with '\') at line [[#@LINE-5]]
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-define-run.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-define-run.txt
index 65fe09c846..a31c9a13d2 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-define-run.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-define-run.txt
@@ -6,4 +6,4 @@
 
 # CHECK: Test has unterminated '{{DEFINE}}:' directive (with '\') at line [[#@LINE-4]]
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-define.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-define.txt
index 58a978a386..7de93e8a30 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-define.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-define.txt
@@ -5,4 +5,4 @@
 
 # CHECK: Test has unterminated '{{DEFINE}}:' directive (with '\') at line [[#@LINE-2]]
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-redefine-bad-define.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-redefine-bad-define.txt
index 5d91231d84..bcbe8a0152 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-redefine-bad-define.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-redefine-bad-define.txt
@@ -10,4 +10,4 @@
 #      CHECK: Substitution's definition does not contain '='
 # CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-5]]
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-redefine-continuation.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-redefine-continuation.txt
index d697d1f560..29b55a1f5e 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-redefine-continuation.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-redefine-continuation.txt
@@ -6,4 +6,4 @@
 
 # CHECK: Test has unterminated '{{REDEFINE}}:' directive (with '\') from line [[#@LINE-3]] to [[#@LINE-2]]
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-redefine-define.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-redefine-define.txt
index a259e210b0..b5ea18c52a 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-redefine-define.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-redefine-define.txt
@@ -10,4 +10,4 @@
 
 # CHECK: Test has unterminated '{{REDEFINE}}:' directive (with '\') at line [[#@LINE-5]]
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-redefine-run.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-redefine-run.txt
index 6385180719..828aa9593e 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-redefine-run.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-redefine-run.txt
@@ -6,4 +6,4 @@
 
 # CHECK: Test has unterminated '{{REDEFINE}}:' directive (with '\') at line [[#@LINE-4]]
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-redefine.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-redefine.txt
index e6f9143a98..5cf7731b00 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-redefine.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-redefine.txt
@@ -5,4 +5,4 @@
 
 # CHECK: Test has unterminated '{{REDEFINE}}:' directive (with '\') at line [[#@LINE-2]]
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-run-define.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-run-define.txt
index 870a707ad2..a5e95c8162 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-run-define.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-run-define.txt
@@ -6,4 +6,4 @@
 
 # CHECK: Test has unterminated '{{RUN}}:' directive (with '\') at line [[#@LINE-4]]
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-run-redefine.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-run-redefine.txt
index 986e1d53b4..fe79fa1e39 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-run-redefine.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/unterminated-run-redefine.txt
@@ -6,4 +6,4 @@
 
 # CHECK: Test has unterminated '{{RUN}}:' directive (with '\') at line [[#@LINE-4]]
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/ws-only.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/ws-only.txt
index c4b063055d..d6e534d06b 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/ws-only.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/continuation/ws-only.txt
@@ -8,4 +8,4 @@
 #      CHECK: Substitution's continuation is empty
 # CHECK-NEXT: in {{DEFINE}}: directive on test line [[#@LINE-5]]
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/define-already-by-config.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/define-already-by-config.txt
index 56c9f6ac85..35fdf4f0e0 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/define-already-by-config.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/define-already-by-config.txt
@@ -5,4 +5,4 @@
 # CHECK-NEXT: Existing pattern: %{global:greeting}
 #  CHECK-NOT: Existing pattern:
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/define-already-by-test.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/define-already-by-test.txt
index 922201ec45..7ece43de90 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/define-already-by-test.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/define-already-by-test.txt
@@ -7,4 +7,4 @@
 # CHECK-NEXT: Existing pattern: %{foo}
 #  CHECK-NOT: Existing pattern:
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/define-inside-pattern.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/define-inside-pattern.txt
index 0ed22d5029..17673fdf80 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/define-inside-pattern.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/define-inside-pattern.txt
@@ -5,4 +5,4 @@
 # CHECK-NEXT: Existing pattern: <%{global:inside}>
 #  CHECK-NOT: Existing pattern:
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/define-multiple-exact.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/define-multiple-exact.txt
index b222bc2f62..5b01204837 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/define-multiple-exact.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/define-multiple-exact.txt
@@ -6,4 +6,4 @@
 # CHECK-NEXT: Existing pattern: %{global:multiple-exact}
 #  CHECK-NOT: Existing pattern:
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/define-multiple-once-exact.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/define-multiple-once-exact.txt
index 2da630e027..05a53beb89 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/define-multiple-once-exact.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/define-multiple-once-exact.txt
@@ -6,4 +6,4 @@
 # CHECK-NEXT: Existing pattern: %{global:multiple-once-exact}
 #  CHECK-NOT: Existing pattern:
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/define-prefixes-pattern.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/define-prefixes-pattern.txt
index a0c7402a39..3a936c9133 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/define-prefixes-pattern.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/define-prefixes-pattern.txt
@@ -5,4 +5,4 @@
 # CHECK-NEXT: Existing pattern: %{global:prefix}\((.*)\)
 #  CHECK-NOT: Existing pattern:
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/define-suffixes-pattern.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/define-suffixes-pattern.txt
index 3189271d3e..8cfb8b3d41 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/define-suffixes-pattern.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/define-suffixes-pattern.txt
@@ -5,4 +5,4 @@
 # CHECK-NEXT: Existing pattern: @%{global:suffix}
 #  CHECK-NOT: Existing pattern:
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/redefine-inside-pattern.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/redefine-inside-pattern.txt
index 7d308a3d52..d354d07499 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/redefine-inside-pattern.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/redefine-inside-pattern.txt
@@ -6,4 +6,4 @@
 # CHECK-NEXT: Existing pattern: <%{global:inside}>
 #  CHECK-NOT: Existing pattern:
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/redefine-multiple-exact.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/redefine-multiple-exact.txt
index ee61c8aea2..e0c87e162b 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/redefine-multiple-exact.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/redefine-multiple-exact.txt
@@ -10,4 +10,4 @@
 # CHECK-NEXT: Existing pattern: %{global:multiple-exact}
 #  CHECK-NOT: Existing pattern:
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/redefine-multiple-once-exact.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/redefine-multiple-once-exact.txt
index 3fa86a78e8..7c0052475b 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/redefine-multiple-once-exact.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/redefine-multiple-once-exact.txt
@@ -9,4 +9,4 @@
 # CHECK-NEXT: Existing pattern: %{global:multiple-once-exact}
 #  CHECK-NOT: Existing pattern:
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/redefine-none.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/redefine-none.txt
index 28ad04c71e..20987eb7f9 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/redefine-none.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/redefine-none.txt
@@ -4,4 +4,4 @@
 #     CHECK: ValueError: No substitution for '%{local}' is defined before '{{REDEFINE}}:' directive at line [[#@LINE-3]]
 # CHECK-NOT: Existing pattern:
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/redefine-prefixes-pattern.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/redefine-prefixes-pattern.txt
index cc82a0e094..a34cae4a3b 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/redefine-prefixes-pattern.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/redefine-prefixes-pattern.txt
@@ -6,4 +6,4 @@
 # CHECK-NEXT: Existing pattern: %{global:prefix}\((.*)\)
 #  CHECK-NOT: Existing pattern:
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/redefine-suffixes-pattern.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/redefine-suffixes-pattern.txt
index edbc124169..293e8ab28a 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/redefine-suffixes-pattern.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/defined-check/redefine-suffixes-pattern.txt
@@ -6,4 +6,4 @@
 # CHECK-NEXT: Existing pattern: @%{global:suffix}
 #  CHECK-NOT: Existing pattern:
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/location-range.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/location-range.txt
index a230298ffd..d8186055cb 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/location-range.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/location-range.txt
@@ -5,4 +5,4 @@
 
 # CHECK: ValueError: Substitution whose pattern contains '%{foo}' is already defined before '{{DEFINE}}:' directive from line [[#@LINE-4]] to [[#@LINE-3]]
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/errors/no-run.txt b/llvm/utils/lit/tests/Inputs/shtest-define/errors/no-run.txt
index 842b9c4755..ff2d10981d 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/errors/no-run.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/errors/no-run.txt
@@ -5,4 +5,4 @@
 
 # CHECK: Test has no '{{RUN}}:' line
 
-# CHECK: Unresolved: 1
+# CHECK: Unresolved: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/examples/param-subst.txt b/llvm/utils/lit/tests/Inputs/shtest-define/examples/param-subst.txt
index 1d94c12137..91418cd5e8 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/examples/param-subst.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/examples/param-subst.txt
@@ -31,4 +31,4 @@
 ; RUN: %{check}
 ; CHECK:# | %clang_cc1 -verify -fopenmp -fopenmp-version=51 -triple x86_64-unknown-linux-gnu -emit-llvm -o - {{.*}} | FileCheck -check-prefix=NO-SIMD {{.*}}
 
-; CHECK: Passed: 1
+; CHECK: Passed: 1 {{\([0-9]*\.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/expansion-order.txt b/llvm/utils/lit/tests/Inputs/shtest-define/expansion-order.txt
index 23c0a6d507..8d3e961e77 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/expansion-order.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/expansion-order.txt
@@ -38,4 +38,4 @@
 # RUN: %{local:echo}
 # CHECK:# | LOCAL: So Long World World
 
-# CHECK: Passed: 1
+# CHECK: Passed: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/line-number-substitutions.txt b/llvm/utils/lit/tests/Inputs/shtest-define/line-number-substitutions.txt
index 5a1d7f2e19..100bbb6c16 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/line-number-substitutions.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/line-number-substitutions.txt
@@ -29,4 +29,4 @@
 # RUN: echo '%{lines-rel}'
 # CHECK:# | [[#@LINE+1]], [[#@LINE-3]], [[#@LINE-12]]
 
-# CHECK: Passed: 1
+# CHECK: Passed: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/name-chars.txt b/llvm/utils/lit/tests/Inputs/shtest-define/name-chars.txt
index d27fda2e2f..cce90e8947 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/name-chars.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/name-chars.txt
@@ -22,4 +22,4 @@
 # RUN: echo '%{_foo_bar_}'
 # CHECK:# | ok at [[#@LINE - 2]]
 
-# CHECK: Passed: 1
+# CHECK: Passed: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/recursiveExpansionLimit.txt b/llvm/utils/lit/tests/Inputs/shtest-define/recursiveExpansionLimit.txt
index e504b822ac..982f7fd43a 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/recursiveExpansionLimit.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/recursiveExpansionLimit.txt
@@ -9,4 +9,4 @@
 # CHECK-NON-RECUR:# | %{inner}
 # CHECK-RECUR:# | expanded
 
-# CHECK: Passed: 1
+# CHECK: Passed: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/shared-substs-0.txt b/llvm/utils/lit/tests/Inputs/shtest-define/shared-substs-0.txt
index dfff9ddd43..034bdd7469 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/shared-substs-0.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/shared-substs-0.txt
@@ -19,4 +19,4 @@
 # DEFINE: %{local:echo}=echo LOCAL0: subst
 # RUN: %{local:echo}
 
-# CHECK: Passed: 1
+# CHECK: Passed: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/shared-substs-1.txt b/llvm/utils/lit/tests/Inputs/shtest-define/shared-substs-1.txt
index 723165a671..66d1a80330 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/shared-substs-1.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/shared-substs-1.txt
@@ -19,4 +19,4 @@
 # DEFINE: %{local:echo}=echo LOCAL1: subst
 # RUN: %{local:echo}
 
-# CHECK: Passed: 1
+# CHECK: Passed: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/value-equals.txt b/llvm/utils/lit/tests/Inputs/shtest-define/value-equals.txt
index c1bdfef30b..88e10d4f64 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/value-equals.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/value-equals.txt
@@ -19,4 +19,4 @@
 # RUN: echo '%{continue-equals}'
 # CHECK:# | FileCheck -input-file=test.txt -implicit-check-not=foobar -check-prefixes=FOO,BAR
 
-# CHECK: Passed: 1
+# CHECK: Passed: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/value-escaped.txt b/llvm/utils/lit/tests/Inputs/shtest-define/value-escaped.txt
index 9143796edd..68cf35825e 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/value-escaped.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/value-escaped.txt
@@ -13,4 +13,4 @@
 # RUN: echo '%{escape}'
 # CHECK:# | {{\\?}}\n {{\\?}}\g<param>
 
-# CHECK: Passed: 1
+# CHECK: Passed: 1 {{\([0-9]*.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/Inputs/shtest-define/ws-and-continuations.txt b/llvm/utils/lit/tests/Inputs/shtest-define/ws-and-continuations.txt
index 3e4db1b15d..32075797c4 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-define/ws-and-continuations.txt
+++ b/llvm/utils/lit/tests/Inputs/shtest-define/ws-and-continuations.txt
@@ -132,4 +132,4 @@
 # RUN: %{deceptive-continue}
 # CHECK:# | x=3
 
-# CHECK:{{ *}}Passed: 1
+# CHECK:{{ *}}Passed: 1 {{\([0-9]*\.[0-9]*%\)}}
diff --git a/llvm/utils/lit/tests/discovery.py b/llvm/utils/lit/tests/discovery.py
old mode 100644
new mode 100755
index 67f940f925..60e7c8d6d6
--- a/llvm/utils/lit/tests/discovery.py
+++ b/llvm/utils/lit/tests/discovery.py
@@ -28,6 +28,13 @@
 # CHECK-BASIC-OUT: top-level-suite :: test-one
 # CHECK-BASIC-OUT: top-level-suite :: test-two
 
+# RUN: %{lit} %{inputs}/discovery \
+# RUN:   -v > %t.out 2> %t.err
+# RUN: FileCheck --check-prefix=CHECK-PERCENTAGES-OUT < %/t.out %s
+#
+# CHECK-PERCENTAGES-OUT:  Total Discovered Tests: {{[0-9]*}}
+# CHECK-PERCENTAGES-OUT:  Passed: {{[0-9]*}} {{\([0-9]*\.[0-9]*%\)}}
+
 # Check discovery when providing the special builtin 'config_map'
 # RUN: %{python} %{inputs}/config-map-discovery/driver.py \
 # RUN:           %{inputs}/config-map-discovery/main-config/lit.cfg \
diff --git a/llvm/utils/lit/tests/googletest-no-sharding.py b/llvm/utils/lit/tests/googletest-no-sharding.py
new file mode 100644
index 0000000000..bb008effb8
--- /dev/null
+++ b/llvm/utils/lit/tests/googletest-no-sharding.py
@@ -0,0 +1,43 @@
+# Check the various features of the GoogleTest format.
+
+# RUN: not %{lit} -v --no-gtest-sharding --order=random %{inputs}/googletest-no-sharding > %t.out
+# FIXME: Temporarily dump test output so we can debug failing tests on
+# buildbots.
+# RUN: cat %t.out
+# RUN: FileCheck < %t.out %s
+#
+# END.
+
+# CHECK: -- Testing:
+# CHECK: FAIL: googletest-no-sharding :: [[PATH:[Dd]ummy[Ss]ub[Dd]ir/]][[FILE:OneTest\.py]]
+# CHECK: *** TEST 'googletest-no-sharding :: [[PATH]][[FILE]]' FAILED ***
+# CHECK-NEXT: Script(shard):
+# CHECK-NEXT: --
+# CHECK-NEXT: GTEST_OUTPUT=json:{{[^[:space:]]*}} GTEST_SHUFFLE=1 GTEST_RANDOM_SEED=123 {{.*}}[[FILE]]
+# CHECK-NEXT: --
+# CHECK-EMPTY:
+# CHECK-NEXT: Script:
+# CHECK-NEXT: --
+# CHECK-NEXT: [[FILE]] --gtest_filter=FirstTest.subTestB
+# CHECK-NEXT: --
+# CHECK-NEXT: I am subTest B output
+# CHECK-EMPTY:
+# CHECK-NEXT: I am subTest B, I FAIL
+# CHECK-NEXT: And I have two lines of output
+# CHECK-EMPTY:
+# CHECK: Script:
+# CHECK-NEXT: --
+# CHECK-NEXT: [[FILE]] --gtest_filter=FirstTest.subTestD
+# CHECK-NEXT: --
+# CHECK-NEXT: unresolved test result
+# CHECK: ***
+# CHECK: ***
+# CHECK: Unresolved Tests (1):
+# CHECK-NEXT:   googletest-no-sharding :: FirstTest/subTestD
+# CHECK: ***
+# CHECK-NEXT: Failed Tests (1):
+# CHECK-NEXT:   googletest-no-sharding :: FirstTest/subTestB
+# CHECK: Skipped{{ *}}: 1
+# CHECK: Passed{{ *}}: 3
+# CHECK: Unresolved{{ *}}: 1
+# CHECK: Failed{{ *}}: 1
diff --git a/llvm/utils/lit/tests/ignore-fail.py b/llvm/utils/lit/tests/ignore-fail.py
index 0644959828..494c6e092c 100644
--- a/llvm/utils/lit/tests/ignore-fail.py
+++ b/llvm/utils/lit/tests/ignore-fail.py
@@ -12,8 +12,9 @@
 # CHECK-DAG: XPASS: ignore-fail :: xpass.txt
 
 #      CHECK: Testing Time:
-# CHECK-NEXT:   Expectedly Failed : 1
-# CHECK-NEXT:   Unresolved : 1
-# CHECK-NEXT:   Failed : 1
-# CHECK-NEXT:   Unexpectedly Passed: 1
+# CHECK: Total Discovered Tests:
+# CHECK-NEXT:   Expectedly Failed : 1 {{\([0-9]*\.[0-9]*%\)}}
+# CHECK-NEXT:   Unresolved : 1 {{\([0-9]*\.[0-9]*%\)}}
+# CHECK-NEXT:   Failed : 1 {{\([0-9]*\.[0-9]*%\)}}
+# CHECK-NEXT:   Unexpectedly Passed: 1 {{\([0-9]*\.[0-9]*%\)}}
 #  CHECK-NOT: {{.}}
diff --git a/llvm/utils/lit/tests/shtest-env.py b/llvm/utils/lit/tests/shtest-env.py
index c093b62e54..03bb4a3cae 100644
--- a/llvm/utils/lit/tests/shtest-env.py
+++ b/llvm/utils/lit/tests/shtest-env.py
@@ -120,6 +120,7 @@
 # CHECK-NOT: {{^[^#]}}
 # CHECK: --
 
-# CHECK: Passed:  4
-# CHECK: Failed: 12
+# CHECK: Total Discovered Tests: 16
+# CHECK: Passed:  4 {{\([0-9]*\.[0-9]*%\)}}
+# CHECK: Failed: 12 {{\([0-9]*\.[0-9]*%\)}}
 # CHECK-NOT: {{.}}
diff --git a/llvm/utils/lit/tests/shtest-not.py b/llvm/utils/lit/tests/shtest-not.py
index f514cddf1d..b42769ffd9 100644
--- a/llvm/utils/lit/tests/shtest-not.py
+++ b/llvm/utils/lit/tests/shtest-not.py
@@ -184,6 +184,7 @@
 # CHECK: # | Error: 'not --crash' cannot call 'rm'
 # CHECK: # error: command failed with exit status: {{.*}}
 
-# CHECK: Passed:  1
-# CHECK: Failed: 16
+# CHECK: Total Discovered Tests: 17
+# CHECK: Passed:  1 {{\([0-9]*\.[0-9]*%\)}}
+# CHECK: Failed: 16 {{\([0-9]*\.[0-9]*%\)}}
 # CHECK-NOT: {{.}}
diff --git a/llvm/utils/lit/tests/shtest-pushd-popd.py b/llvm/utils/lit/tests/shtest-pushd-popd.py
index 6d7e93c74a..f917c1a4a4 100644
--- a/llvm/utils/lit/tests/shtest-pushd-popd.py
+++ b/llvm/utils/lit/tests/shtest-pushd-popd.py
@@ -19,6 +19,7 @@
 # CHECK: pushd a b
 # CHECK: # | 'pushd' supports only one argument
 
-# CHECK: Passed:  1
-# CHECK: Failed:  3
+# CHECK: Total Discovered Tests: 4
+# CHECK: Passed:  1 {{\([0-9]*\.[0-9]*%\)}}
+# CHECK: Failed:  3 {{\([0-9]*\.[0-9]*%\)}}
 # CHECK-NOT: {{.}}
diff --git a/llvm/utils/lit/tests/shtest-shell.py b/llvm/utils/lit/tests/shtest-shell.py
index a043582d6a..8685119488 100644
--- a/llvm/utils/lit/tests/shtest-shell.py
+++ b/llvm/utils/lit/tests/shtest-shell.py
@@ -561,10 +561,17 @@
 
 # FIXME: The output here sucks.
 #
-# CHECK: FAIL: shtest-shell :: error-1.txt
-# CHECK: *** TEST 'shtest-shell :: error-1.txt' FAILED ***
-# CHECK: shell parser error on RUN: at line 3: echo "missing quote
-# CHECK: ***
+#       CHECK: UNRESOLVED: shtest-shell :: error-1.txt
+#  CHECK-NEXT: *** TEST 'shtest-shell :: error-1.txt' FAILED ***
+#  CHECK-NEXT: Exit Code: 1
+# CHECK-EMPTY:
+#  CHECK-NEXT: Command Output (stdout):
+#  CHECK-NEXT: --
+#  CHECK-NEXT: # shell parser error on RUN: at line 3: echo "missing quote
+# CHECK-EMPTY:
+#  CHECK-NEXT: --
+# CHECK-EMPTY:
+#  CHECK-NEXT: ***
 
 # CHECK: FAIL: shtest-shell :: error-2.txt
 # CHECK: *** TEST 'shtest-shell :: error-2.txt' FAILED ***
@@ -643,4 +650,5 @@
 #      CHECK: ***
 
 # CHECK: PASS: shtest-shell :: valid-shell.txt
-# CHECK: Failed Tests (39)
+# CHECK: Unresolved Tests (1)
+# CHECK: Failed Tests (38)
diff --git a/llvm/utils/vim/syntax/llvm.vim b/llvm/utils/vim/syntax/llvm.vim
index 9185a029a2..d86e3d1ddb 100644
--- a/llvm/utils/vim/syntax/llvm.vim
+++ b/llvm/utils/vim/syntax/llvm.vim
@@ -142,6 +142,7 @@ syn keyword llvmKeyword
       \ nosanitize_bounds
       \ nosanitize_coverage
       \ null_pointer_is_valid
+      \ optdebug
       \ optforfuzzing
       \ optnone
       \ optsize
diff --git a/llvm/utils/vim/syntax/tablegen.vim b/llvm/utils/vim/syntax/tablegen.vim
index 341c8bef28..c2a2230663 100644
--- a/llvm/utils/vim/syntax/tablegen.vim
+++ b/llvm/utils/vim/syntax/tablegen.vim
@@ -14,7 +14,7 @@ syntax sync minlines=100
 
 syn case match
 
-syn keyword tgKeyword   def let in code dag field include defm foreach defset defvar if then else
+syn keyword tgKeyword   def let in code dag field include defm foreach defset defvar if then else assert dump
 syn keyword tgType      class int string list bit bits multiclass
 
 syn match   tgNumber    /\<\d\+\>/
diff --git a/llvm/utils/vscode/llvm/syntaxes/TableGen.tmLanguage b/llvm/utils/vscode/llvm/syntaxes/TableGen.tmLanguage
index 0b08ec262a..15f266ca6e 100644
--- a/llvm/utils/vscode/llvm/syntaxes/TableGen.tmLanguage
+++ b/llvm/utils/vscode/llvm/syntaxes/TableGen.tmLanguage
@@ -18,7 +18,7 @@
 		</dict>
 		<dict>
 			<key>match</key>
-			<string>\b(def|let|in|code|dag|string|list|bits|bit|field|include|defm|foreach|class|multiclass|int|defvar|defset|if|then|else)\b</string>
+			<string>\b(def|let|in|code|dag|string|list|bits|bit|field|include|defm|foreach|class|multiclass|int|defvar|defset|if|then|else|assert|dump)\b</string>
 			<key>name</key>
 			<string>keyword.control.tablegen</string>
 		</dict>
diff --git a/mlir/cmake/modules/AddMLIR.cmake b/mlir/cmake/modules/AddMLIR.cmake
index 4622d4d05f..544abe4368 100644
--- a/mlir/cmake/modules/AddMLIR.cmake
+++ b/mlir/cmake/modules/AddMLIR.cmake
@@ -188,7 +188,8 @@ endfunction()
 # Generate Documentation
 function(add_mlir_doc doc_filename output_file output_directory command)
   set(LLVM_TARGET_DEFINITIONS ${doc_filename}.td)
-  tablegen(MLIR ${output_file}.md ${command} ${ARGN})
+  # The MLIR docs use Hugo, so we allow Hugo specific features here.
+  tablegen(MLIR ${output_file}.md ${command} -allow-hugo-specific-features ${ARGN})
   set(GEN_DOC_FILE ${MLIR_BINARY_DIR}/docs/${output_directory}${output_file}.md)
   add_custom_command(
           OUTPUT ${GEN_DOC_FILE}
diff --git a/mlir/docs/Bindings/Python.md b/mlir/docs/Bindings/Python.md
index bf54efee1f..bc2e676a87 100644
--- a/mlir/docs/Bindings/Python.md
+++ b/mlir/docs/Bindings/Python.md
@@ -1017,90 +1017,79 @@ very generic signature.
 
 #### Extending Generated Op Classes
 
-Note that this is a rather complex mechanism and this section errs on the side
-of explicitness. Users are encouraged to find an example and duplicate it if
-they don't feel the need to understand the subtlety. The `builtin` dialect
-provides some relatively simple examples.
-
 As mentioned above, the build system generates Python sources like
 `_{DIALECT_NAMESPACE}_ops_gen.py` for each dialect with Python bindings. It is
-often desirable to to use these generated classes as a starting point for
-further customization, so an extension mechanism is provided to make this easy
-(you are always free to do ad-hoc patching in your `{DIALECT_NAMESPACE}.py` file
-but we prefer a more standard mechanism that is applied uniformly).
+often desirable to use these generated classes as a starting point for
+further customization, so an extension mechanism is provided to make this easy.
+This mechanism uses conventional inheritance combined with `OpView` registration.
+For example, the default builder for `arith.constant`
+
+```python
+class ConstantOp(_ods_ir.OpView):
+  OPERATION_NAME = "arith.constant"
+
+  _ODS_REGIONS = (0, True)
+
+  def __init__(self, value, *, loc=None, ip=None):
+    ...
+```
 
-To provide extensions, add a `_{DIALECT_NAMESPACE}_ops_ext.py` file to the
-`dialects` module (i.e. adjacent to your `{DIALECT_NAMESPACE}.py` top-level and
-the `*_ops_gen.py` file). Using the `builtin` dialect and `FuncOp` as an
-example, the generated code will include an import like this:
+expects `value` to be a `TypedAttr` (e.g., `IntegerAttr` or `FloatAttr`). 
+Thus, a natural extension is a builder that accepts a MLIR type and a Python value and instantiates the appropriate `TypedAttr`:
 
 ```python
-try:
-  from . import _builtin_ops_ext as _ods_ext_module
-except ImportError:
-  _ods_ext_module = None
+from typing import Union
+
+from mlir.ir import Type, IntegerAttr, FloatAttr
+from mlir.dialects._arith_ops_gen import _Dialect, ConstantOp
+from mlir.dialects._ods_common import _cext
+
+@_cext.register_operation(_Dialect, replace=True)
+class ConstantOpExt(ConstantOp):
+    def __init__(
+        self, result: Type, value: Union[int, float], *, loc=None, ip=None
+    ):
+        if isinstance(value, int):
+            super().__init__(IntegerAttr.get(result, value), loc=loc, ip=ip)
+        elif isinstance(value, float):
+            super().__init__(FloatAttr.get(result, value), loc=loc, ip=ip)
+        else:
+            raise NotImplementedError(f"Building `arith.constant` not supported for {result=} {value=}")
 ```
 
-Then for each generated concrete `OpView` subclass, it will apply a decorator
-like:
+which enables building an instance of `arith.constant` like so:
 
 ```python
-@_ods_cext.register_operation(_Dialect)
-@_ods_extend_opview_class(_ods_ext_module)
-class FuncOp(_ods_ir.OpView):
+from mlir.ir import F32Type
+
+a = ConstantOpExt(F32Type.get(), 42.42)
+b = ConstantOpExt(IntegerType.get_signless(32), 42)
 ```
 
-See the `_ods_common.py` `extend_opview_class` function for details of the
-mechanism. At a high level:
-
-*   If the extension module exists, locate an extension class for the op (in
-    this example, `FuncOp`):
-    *   First by looking for an attribute with the exact name in the extension
-        module.
-    *   Falling back to calling a `select_opview_mixin(parent_opview_cls)`
-        function defined in the extension module.
-*   If a mixin class is found, a new subclass is dynamically created that
-    multiply inherits from `({_builtin_ops_ext.FuncOp},
-    _builtin_ops_gen.FuncOp)`.
-
-The mixin class should not inherit from anything (i.e. directly extends `object`
-only). The facility is typically used to define custom `__init__` methods,
-properties, instance methods and static methods. Due to the inheritance
-ordering, the mixin class can act as though it extends the generated `OpView`
-subclass in most contexts (i.e. `issubclass(_builtin_ops_ext.FuncOp, OpView)`
-will return `False` but usage generally allows you treat it as duck typed as an
-`OpView`).
-
-There are a couple of recommendations, given how the class hierarchy is defined:
-
-*   For static methods that need to instantiate the actual "leaf" op (which is
-    dynamically generated and would result in circular dependencies to try to
-    reference by name), prefer to use `@classmethod` and the concrete subclass
-    will be provided as your first `cls` argument. See
-    `_builtin_ops_ext.FuncOp.from_py_func` as an example.
-*   If seeking to replace the generated `__init__` method entirely, you may
-    actually want to invoke the super-super-class `mlir.ir.OpView` constructor
-    directly, as it takes an `mlir.ir.Operation`, which is likely what you are
-    constructing (i.e. the generated `__init__` method likely adds more API
-    constraints than you want to expose in a custom builder).
-
-A pattern that comes up frequently is wanting to provide a sugared `__init__`
-method which has optional or type-polymorphism/implicit conversions but to
-otherwise want to invoke the default op building logic. For such cases, it is
-recommended to use an idiom such as:
+Note, three key aspects of the extension mechanism in this example:
+
+1. `ConstantOpExt` directly inherits from the generated `ConstantOp`;
+2. in this, simplest, case all that's required is a call to the super class' initializer, i.e., `super().__init__(...)`;
+3. in order to register `ConstantOpExt` as the preferred `OpView` that is returned by `mlir.ir.Operation.opview` (see [Operations, Regions and Blocks](#operations-regions-and-blocks))
+   we decorate the class with `@_cext.register_operation(_Dialect, replace=True)`, **where the `replace=True` must be used**.
+
+In some more complex cases it might be necessary to explicitly build the `OpView` through `OpView.build_generic` (see [Default Builder](#default-builder)), just as is performed by the generated builders.
+I.e., we must call `OpView.build_generic` **and pass the result to `OpView.__init__`**, where the small issue becomes that the latter is already overridden by the generated builder.
+Thus, we must call a method of a super class' super class (the "grandparent"); for example:
 
 ```python
-  def __init__(self, sugar, spice, *, loc=None, ip=None):
-    ... massage into result_type, operands, attributes ...
-    OpView.__init__(self, self.build_generic(
-        results=[result_type],
-        operands=operands,
-        attributes=attributes,
-        loc=loc,
-        ip=ip))
+from mlir.dialects._scf_ops_gen import _Dialect, ForOp
+from mlir.dialects._ods_common import _cext
+
+@_cext.register_operation(_Dialect, replace=True)
+class ForOpExt(ForOp):
+    def __init__(self, lower_bound, upper_bound, step, iter_args, *, loc=None, ip=None):
+        ...
+        super(ForOp, self).__init__(self.build_generic(...))
 ```
 
-Refer to the documentation for `build_generic` for more information.
+where `OpView.__init__` is called via `super(ForOp, self).__init__`.
+Note, there are alternatives ways to implement this (e.g., explicitly writing `OpView.__init__`); see any discussion on Python inheritance.
 
 ## Providing Python bindings for a dialect
 
diff --git a/mlir/docs/Dialects/Linalg/OpDSL.md b/mlir/docs/Dialects/Linalg/OpDSL.md
index b2868efa09..5c4c8b4e18 100644
--- a/mlir/docs/Dialects/Linalg/OpDSL.md
+++ b/mlir/docs/Dialects/Linalg/OpDSL.md
@@ -12,7 +12,7 @@ corresponding `linalg.generic` IR for the composition.
 
 The tool is bundled with the MLIR Python bindings. To use from the CMake build
 tree, MLIR must be build with Python bindings enabled
-(`-DMLIR_ENALBE_BINDINGS_PYTHON=ON`). Then add the `python` directory in the
+(`-DMLIR_ENABLE_BINDINGS_PYTHON=ON`). Then add the `python` directory in the
 build tree to your `PYTHONPATH` environment variable (i.e. `export
 PYTHONPATH=$PWD/build/tools/mlir/python_packages/mlir_core`). Optionally, use an
 installed MLIR package, if available, to avoid building.
diff --git a/mlir/docs/Interfaces.md b/mlir/docs/Interfaces.md
index e9236c9dfe..343522c5a3 100644
--- a/mlir/docs/Interfaces.md
+++ b/mlir/docs/Interfaces.md
@@ -409,6 +409,10 @@ comprised of the following components:
         the instance is the interface class. In the trait declaration, the
         type of the instance is the concrete entity class
         (e.g. `IntegerAttr`, `FuncOp`, etc.).
+*   Extra Trait Class Declarations (Optional: `extraTraitClassDeclaration`)
+    -   Additional C++ code that is injected into the interface trait
+        declaration.
+    -   Allows the same replacements as extra shared class declarations.
 
 `OpInterface` classes may additionally contain the following:
 
diff --git a/mlir/docs/Passes.md b/mlir/docs/Passes.md
index 1e2a2b1d20..add48876e8 100644
--- a/mlir/docs/Passes.md
+++ b/mlir/docs/Passes.md
@@ -60,6 +60,10 @@ This document describes the available MLIR passes and their contracts.
 
 [include "MemRefPasses.md"]
 
+## 'ml\_program' Dialect Passes
+
+[include "MLProgramPasses.md"]
+
 ## 'nvgpu' Dialect Passes
 
 [include "NVGPUPasses.md"]
diff --git a/mlir/examples/minimal-opt/README.md b/mlir/examples/minimal-opt/README.md
index 99179c4c11..b8a455f7a7 100644
--- a/mlir/examples/minimal-opt/README.md
+++ b/mlir/examples/minimal-opt/README.md
@@ -14,10 +14,10 @@ Below are some example measurements taken at the time of the LLVM 17 release,
 using clang-14 on a X86 Ubuntu and [bloaty](https://github.com/google/bloaty).
 
 |                                  | Base   | Os     | Oz     | Os LTO | Oz LTO |
-| :------------------------------: | ------ | ------ | ------ | ------ | ------ |
-| `mlir-cat`                       | 1018kB | 836KB  | 879KB  | 697KB  | 649KB  |
-| `mlir-minimal-opt`               | 1.54MB | 1.25MB | 1.29MB | 1.10MB | 1.00MB |
-| `mlir-minimal-opt-canonicalizer` | 2.24MB | 1.81MB | 1.86MB | 1.62MB | 1.48MB |
+| :-----------------------------: | ------ | ------ | ------ | ------ | ------ |
+| `mlir-cat`                      | 1018kB | 836KB  | 879KB  | 697KB  | 649KB  |
+| `mlir-minimal-opt`              | 1.54MB | 1.25MB | 1.29MB | 1.10MB | 1.00MB |
+| `mlir-minimal-opt-canonicalize` | 2.24MB | 1.81MB | 1.86MB | 1.62MB | 1.48MB |
 
 Base configuration:
 
diff --git a/mlir/examples/minimal-opt/mlir-cat.cpp b/mlir/examples/minimal-opt/mlir-cat.cpp
index 518654c0ab..2bc89b807a 100644
--- a/mlir/examples/minimal-opt/mlir-cat.cpp
+++ b/mlir/examples/minimal-opt/mlir-cat.cpp
@@ -7,14 +7,17 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/IR/MLIRContext.h"
+#include "mlir/IR/Operation.h"
+#include "mlir/IR/OwningOpRef.h"
 #include "mlir/Parser/Parser.h"
 #include "mlir/Support/FileUtilities.h"
 #include "mlir/Support/LLVM.h"
-#include "mlir/Support/LogicalResult.h"
-#include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/SourceMgr.h"
 #include "llvm/Support/ToolOutputFile.h"
 #include "llvm/Support/raw_ostream.h"
+#include <cstdlib>
+#include <string>
+#include <utility>
 
 using namespace mlir;
 
diff --git a/mlir/examples/standalone/lib/Standalone/StandaloneOps.cpp b/mlir/examples/standalone/lib/Standalone/StandaloneOps.cpp
index 497eb98705..55b66b5123 100644
--- a/mlir/examples/standalone/lib/Standalone/StandaloneOps.cpp
+++ b/mlir/examples/standalone/lib/Standalone/StandaloneOps.cpp
@@ -8,7 +8,6 @@
 
 #include "Standalone/StandaloneOps.h"
 #include "Standalone/StandaloneDialect.h"
-#include "mlir/IR/OpImplementation.h"
 
 #define GET_OP_CLASSES
 #include "Standalone/StandaloneOps.cpp.inc"
diff --git a/mlir/examples/standalone/lib/Standalone/StandalonePasses.cpp b/mlir/examples/standalone/lib/Standalone/StandalonePasses.cpp
index 6af45c9769..d438cb46ec 100644
--- a/mlir/examples/standalone/lib/Standalone/StandalonePasses.cpp
+++ b/mlir/examples/standalone/lib/Standalone/StandalonePasses.cpp
@@ -6,9 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 #include "mlir/Dialect/Func/IR/FuncOps.h"
-#include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/Pass/Pass.h"
+#include "mlir/Rewrite/FrozenRewritePatternSet.h"
+#include "mlir/Support/LogicalResult.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 #include "Standalone/StandalonePasses.h"
diff --git a/mlir/examples/standalone/python/StandaloneExtension.cpp b/mlir/examples/standalone/python/StandaloneExtension.cpp
index e8f574f2c6..5e83060cd4 100644
--- a/mlir/examples/standalone/python/StandaloneExtension.cpp
+++ b/mlir/examples/standalone/python/StandaloneExtension.cpp
@@ -9,7 +9,6 @@
 #include "Standalone-c/Dialects.h"
 #include "mlir/Bindings/Python/PybindAdaptors.h"
 
-namespace py = pybind11;
 using namespace mlir::python::adaptors;
 
 PYBIND11_MODULE(_standaloneDialects, m) {
diff --git a/mlir/examples/standalone/standalone-opt/standalone-opt.cpp b/mlir/examples/standalone/standalone-opt/standalone-opt.cpp
index e75db35e52..e39fa96701 100644
--- a/mlir/examples/standalone/standalone-opt/standalone-opt.cpp
+++ b/mlir/examples/standalone/standalone-opt/standalone-opt.cpp
@@ -6,19 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Dialect/Arith/IR/Arith.h"
-#include "mlir/IR/Dialect.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/InitAllDialects.h"
 #include "mlir/InitAllPasses.h"
-#include "mlir/Pass/Pass.h"
-#include "mlir/Pass/PassManager.h"
 #include "mlir/Support/FileUtilities.h"
 #include "mlir/Tools/mlir-opt/MlirOptMain.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/InitLLVM.h"
-#include "llvm/Support/SourceMgr.h"
-#include "llvm/Support/ToolOutputFile.h"
 
 #include "Standalone/StandaloneDialect.h"
 #include "Standalone/StandalonePasses.h"
diff --git a/mlir/examples/standalone/standalone-plugin/standalone-plugin.cpp b/mlir/examples/standalone/standalone-plugin/standalone-plugin.cpp
index 129b86b9db..d2dcdc96b3 100644
--- a/mlir/examples/standalone/standalone-plugin/standalone-plugin.cpp
+++ b/mlir/examples/standalone/standalone-plugin/standalone-plugin.cpp
@@ -6,16 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Dialect/Arith/IR/Arith.h"
-#include "mlir/IR/Dialect.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/InitAllDialects.h"
-#include "mlir/InitAllPasses.h"
-#include "mlir/Pass/Pass.h"
 #include "mlir/Tools/Plugins/DialectPlugin.h"
 
 #include "Standalone/StandaloneDialect.h"
 #include "Standalone/StandalonePasses.h"
+#include "mlir/Tools/Plugins/PassPlugin.h"
+#include "llvm/Config/llvm-config.h"
+#include "llvm/Support/Compiler.h"
 
 using namespace mlir;
 
diff --git a/mlir/examples/standalone/standalone-translate/standalone-translate.cpp b/mlir/examples/standalone/standalone-translate/standalone-translate.cpp
index 94277c1786..a4da328fd7 100644
--- a/mlir/examples/standalone/standalone-translate/standalone-translate.cpp
+++ b/mlir/examples/standalone/standalone-translate/standalone-translate.cpp
@@ -12,10 +12,13 @@
 //===----------------------------------------------------------------------===//
 
 #include "Standalone/StandaloneDialect.h"
+#include "mlir/IR/DialectRegistry.h"
+#include "mlir/IR/Operation.h"
 #include "mlir/InitAllTranslations.h"
 #include "mlir/Support/LogicalResult.h"
 #include "mlir/Tools/mlir-translate/MlirTranslateMain.h"
 #include "mlir/Tools/mlir-translate/Translation.h"
+#include "llvm/Support/raw_ostream.h"
 
 int main(int argc, char **argv) {
   mlir::registerAllTranslations();
diff --git a/mlir/examples/toy/Ch1/parser/AST.cpp b/mlir/examples/toy/Ch1/parser/AST.cpp
index 2eaabb1b52..2546f2a972 100644
--- a/mlir/examples/toy/Ch1/parser/AST.cpp
+++ b/mlir/examples/toy/Ch1/parser/AST.cpp
@@ -12,9 +12,12 @@
 
 #include "toy/AST.h"
 
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/ADT/TypeSwitch.h"
+#include "llvm/Support/Casting.h"
 #include "llvm/Support/raw_ostream.h"
+#include <string>
 
 using namespace toy;
 
diff --git a/mlir/examples/toy/Ch1/toyc.cpp b/mlir/examples/toy/Ch1/toyc.cpp
index ca2a1a1fa8..fb7b484a92 100644
--- a/mlir/examples/toy/Ch1/toyc.cpp
+++ b/mlir/examples/toy/Ch1/toyc.cpp
@@ -10,6 +10,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "toy/AST.h"
+#include "toy/Lexer.h"
 #include "toy/Parser.h"
 
 #include "llvm/ADT/StringRef.h"
@@ -17,6 +19,9 @@
 #include "llvm/Support/ErrorOr.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/raw_ostream.h"
+#include <memory>
+#include <string>
+#include <system_error>
 
 using namespace toy;
 namespace cl = llvm::cl;
diff --git a/mlir/examples/toy/Ch2/mlir/Dialect.cpp b/mlir/examples/toy/Ch2/mlir/Dialect.cpp
index 042d1de2d2..d35bd9fc5f 100644
--- a/mlir/examples/toy/Ch2/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch2/mlir/Dialect.cpp
@@ -13,10 +13,22 @@
 
 #include "toy/Dialect.h"
 
+#include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/OpImplementation.h"
+#include "mlir/IR/Operation.h"
+#include "mlir/IR/OperationSupport.h"
+#include "mlir/IR/Value.h"
 #include "mlir/Interfaces/FunctionImplementation.h"
+#include "mlir/Support/LLVM.h"
+#include "mlir/Support/LogicalResult.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Casting.h"
+#include <algorithm>
+#include <string>
 
 using namespace mlir;
 using namespace mlir::toy;
diff --git a/mlir/examples/toy/Ch2/mlir/MLIRGen.cpp b/mlir/examples/toy/Ch2/mlir/MLIRGen.cpp
index b1abd37c57..2f0a88f709 100644
--- a/mlir/examples/toy/Ch2/mlir/MLIRGen.cpp
+++ b/mlir/examples/toy/Ch2/mlir/MLIRGen.cpp
@@ -12,20 +12,31 @@
 //===----------------------------------------------------------------------===//
 
 #include "toy/MLIRGen.h"
+#include "mlir/IR/Block.h"
+#include "mlir/IR/Diagnostics.h"
+#include "mlir/IR/Value.h"
+#include "mlir/Support/LogicalResult.h"
 #include "toy/AST.h"
 #include "toy/Dialect.h"
 
-#include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/Verifier.h"
+#include "toy/Lexer.h"
 
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/ScopedHashTable.h"
-#include "llvm/Support/raw_ostream.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/Twine.h"
+#include <cassert>
+#include <cstdint>
+#include <functional>
 #include <numeric>
+#include <optional>
+#include <vector>
 
 using namespace mlir::toy;
 using namespace toy;
diff --git a/mlir/examples/toy/Ch2/parser/AST.cpp b/mlir/examples/toy/Ch2/parser/AST.cpp
index 2eaabb1b52..2546f2a972 100644
--- a/mlir/examples/toy/Ch2/parser/AST.cpp
+++ b/mlir/examples/toy/Ch2/parser/AST.cpp
@@ -12,9 +12,12 @@
 
 #include "toy/AST.h"
 
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/ADT/TypeSwitch.h"
+#include "llvm/Support/Casting.h"
 #include "llvm/Support/raw_ostream.h"
+#include <string>
 
 using namespace toy;
 
diff --git a/mlir/examples/toy/Ch6/mlir/LowerToLLVM.cpp b/mlir/examples/toy/Ch6/mlir/LowerToLLVM.cpp
index 684ce37b23..67b82f91b1 100644
--- a/mlir/examples/toy/Ch6/mlir/LowerToLLVM.cpp
+++ b/mlir/examples/toy/Ch6/mlir/LowerToLLVM.cpp
@@ -61,7 +61,7 @@ class PrintOpLowering : public ConversionPattern {
   LogicalResult
   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
-    auto context = rewriter.getContext();
+    auto *context = rewriter.getContext();
     auto memRefType = llvm::cast<MemRefType>((*op->operand_type_begin()));
     auto memRefShape = memRefType.getShape();
     auto loc = op->getLoc();
@@ -117,8 +117,8 @@ class PrintOpLowering : public ConversionPattern {
   ///   * `i32 (i8*, ...)`
   static LLVM::LLVMFunctionType getPrintfType(MLIRContext *context) {
     auto llvmI32Ty = IntegerType::get(context, 32);
-    auto llvmI8PtrTy = LLVM::LLVMPointerType::get(IntegerType::get(context, 8));
-    auto llvmFnType = LLVM::LLVMFunctionType::get(llvmI32Ty, llvmI8PtrTy,
+    auto llvmPtrTy = LLVM::LLVMPointerType::get(context);
+    auto llvmFnType = LLVM::LLVMFunctionType::get(llvmI32Ty, llvmPtrTy,
                                                   /*isVarArg=*/true);
     return llvmFnType;
   }
@@ -162,8 +162,7 @@ class PrintOpLowering : public ConversionPattern {
     Value cst0 = builder.create<LLVM::ConstantOp>(loc, builder.getI64Type(),
                                                   builder.getIndexAttr(0));
     return builder.create<LLVM::GEPOp>(
-        loc,
-        LLVM::LLVMPointerType::get(IntegerType::get(builder.getContext(), 8)),
+        loc, LLVM::LLVMPointerType::get(builder.getContext()), global.getType(),
         globalPtr, ArrayRef<Value>({cst0, cst0}));
   }
 };
diff --git a/mlir/examples/toy/Ch7/mlir/LowerToLLVM.cpp b/mlir/examples/toy/Ch7/mlir/LowerToLLVM.cpp
index 684ce37b23..67b82f91b1 100644
--- a/mlir/examples/toy/Ch7/mlir/LowerToLLVM.cpp
+++ b/mlir/examples/toy/Ch7/mlir/LowerToLLVM.cpp
@@ -61,7 +61,7 @@ class PrintOpLowering : public ConversionPattern {
   LogicalResult
   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
-    auto context = rewriter.getContext();
+    auto *context = rewriter.getContext();
     auto memRefType = llvm::cast<MemRefType>((*op->operand_type_begin()));
     auto memRefShape = memRefType.getShape();
     auto loc = op->getLoc();
@@ -117,8 +117,8 @@ class PrintOpLowering : public ConversionPattern {
   ///   * `i32 (i8*, ...)`
   static LLVM::LLVMFunctionType getPrintfType(MLIRContext *context) {
     auto llvmI32Ty = IntegerType::get(context, 32);
-    auto llvmI8PtrTy = LLVM::LLVMPointerType::get(IntegerType::get(context, 8));
-    auto llvmFnType = LLVM::LLVMFunctionType::get(llvmI32Ty, llvmI8PtrTy,
+    auto llvmPtrTy = LLVM::LLVMPointerType::get(context);
+    auto llvmFnType = LLVM::LLVMFunctionType::get(llvmI32Ty, llvmPtrTy,
                                                   /*isVarArg=*/true);
     return llvmFnType;
   }
@@ -162,8 +162,7 @@ class PrintOpLowering : public ConversionPattern {
     Value cst0 = builder.create<LLVM::ConstantOp>(loc, builder.getI64Type(),
                                                   builder.getIndexAttr(0));
     return builder.create<LLVM::GEPOp>(
-        loc,
-        LLVM::LLVMPointerType::get(IntegerType::get(builder.getContext(), 8)),
+        loc, LLVM::LLVMPointerType::get(builder.getContext()), global.getType(),
         globalPtr, ArrayRef<Value>({cst0, cst0}));
   }
 };
diff --git a/mlir/include/mlir-c/Dialect/SparseTensor.h b/mlir/include/mlir-c/Dialect/SparseTensor.h
index 7e47e54e73..859a4f0dd9 100644
--- a/mlir/include/mlir-c/Dialect/SparseTensor.h
+++ b/mlir/include/mlir-c/Dialect/SparseTensor.h
@@ -51,11 +51,10 @@ MLIR_CAPI_EXPORTED bool
 mlirAttributeIsASparseTensorEncodingAttr(MlirAttribute attr);
 
 /// Creates a `sparse_tensor.encoding` attribute with the given parameters.
-/// TODO: add a version that supplied lvlToDim when it cannot be inferred
 MLIR_CAPI_EXPORTED MlirAttribute mlirSparseTensorEncodingAttrGet(
     MlirContext ctx, intptr_t lvlRank,
     enum MlirSparseTensorDimLevelType const *lvlTypes, MlirAffineMap dimToLvl,
-    int posWidth, int crdWidth);
+    MlirAffineMap lvlTodim, int posWidth, int crdWidth);
 
 /// Returns the level-rank of the `sparse_tensor.encoding` attribute.
 MLIR_CAPI_EXPORTED intptr_t
diff --git a/mlir/include/mlir-c/IR.h b/mlir/include/mlir-c/IR.h
index e361f33a0d..7b121d4df3 100644
--- a/mlir/include/mlir-c/IR.h
+++ b/mlir/include/mlir-c/IR.h
@@ -73,7 +73,6 @@ DEFINE_C_API_STRUCT(MlirValue, const void);
 ///
 /// A named attribute is essentially a (name, attribute) pair where the name is
 /// a string.
-
 struct MlirNamedAttribute {
   MlirIdentifier name;
   MlirAttribute attribute;
@@ -698,6 +697,24 @@ MLIR_CAPI_EXPORTED void mlirOperationMoveAfter(MlirOperation op,
 /// ownership is transferred to the block of the other operation.
 MLIR_CAPI_EXPORTED void mlirOperationMoveBefore(MlirOperation op,
                                                 MlirOperation other);
+
+/// Traversal order for operation walk.
+typedef enum MlirWalkOrder {
+  MlirWalkPreOrder,
+  MlirWalkPostOrder
+} MlirWalkOrder;
+
+/// Operation walker type. The handler is passed an (opaque) reference to an
+/// operation a pointer to a `userData`.
+typedef void (*MlirOperationWalkCallback)(MlirOperation, void *userData);
+
+/// Walks operation `op` in `walkOrder` and calls `callback` on that operation.
+/// `*userData` is passed to the callback as well and can be used to tunnel some
+/// some context or other data into the callback.
+MLIR_CAPI_EXPORTED
+void mlirOperationWalk(MlirOperation op, MlirOperationWalkCallback callback,
+                       void *userData, MlirWalkOrder walkOrder);
+
 //===----------------------------------------------------------------------===//
 // Region API.
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Analysis/DataFlow/DeadCodeAnalysis.h b/mlir/include/mlir/Analysis/DataFlow/DeadCodeAnalysis.h
index 7a6fea8326..10ef8b6ba5 100644
--- a/mlir/include/mlir/Analysis/DataFlow/DeadCodeAnalysis.h
+++ b/mlir/include/mlir/Analysis/DataFlow/DeadCodeAnalysis.h
@@ -208,8 +208,7 @@ class DeadCodeAnalysis : public DataFlowAnalysis {
   /// Visit the given terminator operation that exits a region under an
   /// operation with control-flow semantics. These are terminators with no CFG
   /// successors.
-  void visitRegionTerminator(RegionBranchTerminatorOpInterface op,
-                             RegionBranchOpInterface branch);
+  void visitRegionTerminator(Operation *op, RegionBranchOpInterface branch);
 
   /// Visit the given terminator operation that exits a callable region. These
   /// are terminators with no CFG successors.
diff --git a/mlir/include/mlir/Analysis/Presburger/Fraction.h b/mlir/include/mlir/Analysis/Presburger/Fraction.h
index a410f528e1..afcbed84c6 100644
--- a/mlir/include/mlir/Analysis/Presburger/Fraction.h
+++ b/mlir/include/mlir/Analysis/Presburger/Fraction.h
@@ -38,7 +38,7 @@ struct Fraction {
     }
   }
   /// Overloads for passing literals.
-  Fraction(const MPInt &num, int64_t den = 1) : Fraction(num, MPInt(den)) {}
+  Fraction(const MPInt &num, int64_t den) : Fraction(num, MPInt(den)) {}
   Fraction(int64_t num, const MPInt &den = MPInt(1))
       : Fraction(MPInt(num), den) {}
   Fraction(int64_t num, int64_t den) : Fraction(MPInt(num), MPInt(den)) {}
diff --git a/mlir/include/mlir/Analysis/Presburger/LinearTransform.h b/mlir/include/mlir/Analysis/Presburger/LinearTransform.h
index 67dc9e87fa..b5c761439f 100644
--- a/mlir/include/mlir/Analysis/Presburger/LinearTransform.h
+++ b/mlir/include/mlir/Analysis/Presburger/LinearTransform.h
@@ -50,10 +50,6 @@ class LinearTransform {
     return matrix.postMultiplyWithColumn(colVec);
   }
 
-  // Compute the determinant of the transform by converting it to row echelon
-  // form and then taking the product of the diagonal.
-  MPInt determinant();
-
 private:
   IntMatrix matrix;
 };
diff --git a/mlir/include/mlir/Analysis/Presburger/Matrix.h b/mlir/include/mlir/Analysis/Presburger/Matrix.h
index bed3a5f75e..4d9f13832e 100644
--- a/mlir/include/mlir/Analysis/Presburger/Matrix.h
+++ b/mlir/include/mlir/Analysis/Presburger/Matrix.h
@@ -15,8 +15,8 @@
 #ifndef MLIR_ANALYSIS_PRESBURGER_MATRIX_H
 #define MLIR_ANALYSIS_PRESBURGER_MATRIX_H
 
-#include "mlir/Support/LLVM.h"
 #include "mlir/Analysis/Presburger/Fraction.h"
+#include "mlir/Support/LLVM.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -36,9 +36,11 @@ namespace presburger {
 /// This class only works for the types MPInt and Fraction, since the method
 /// implementations are in the Matrix.cpp file. Only these two types have
 /// been explicitly instantiated there.
-template<typename T>
+template <typename T>
 class Matrix {
-static_assert(std::is_same_v<T,MPInt> || std::is_same_v<T,Fraction>, "T must be MPInt or Fraction.");
+  static_assert(std::is_same_v<T, MPInt> || std::is_same_v<T, Fraction>,
+                "T must be MPInt or Fraction.");
+
 public:
   Matrix() = delete;
 
@@ -69,9 +71,7 @@ static_assert(std::is_same_v<T,MPInt> || std::is_same_v<T,Fraction>, "T must be
 
   T &operator()(unsigned row, unsigned column) { return at(row, column); }
 
-  T operator()(unsigned row, unsigned column) const {
-    return at(row, column);
-  }
+  T operator()(unsigned row, unsigned column) const { return at(row, column); }
 
   /// Swap the given columns.
   void swapColumns(unsigned column, unsigned otherColumn);
@@ -189,7 +189,7 @@ static_assert(std::is_same_v<T,MPInt> || std::is_same_v<T,Fraction>, "T must be
   /// invariants satisfied.
   bool hasConsistentState() const;
 
-private:
+protected:
   /// The current number of rows, columns, and reserved columns. The underlying
   /// data vector is viewed as an nRows x nReservedColumns matrix, of which the
   /// first nColumns columns are currently in use, and the remaining are
@@ -204,21 +204,14 @@ static_assert(std::is_same_v<T,MPInt> || std::is_same_v<T,Fraction>, "T must be
 // An inherited class for integer matrices, with no new data attributes.
 // This is only used for the matrix-related methods which apply only
 // to integers (hermite normal form computation and row normalisation).
-class IntMatrix : public Matrix<MPInt>
-{
+class IntMatrix : public Matrix<MPInt> {
 public:
   IntMatrix(unsigned rows, unsigned columns, unsigned reservedRows = 0,
-            unsigned reservedColumns = 0) :
-    Matrix<MPInt>(rows, columns, reservedRows, reservedColumns) {};
-
-  IntMatrix(Matrix<MPInt> m) :
-    Matrix<MPInt>(m.getNumRows(), m.getNumColumns(), m.getNumReservedRows(), m.getNumReservedColumns())
-  {
-    for (unsigned i = 0; i < m.getNumRows(); i++)
-      for (unsigned j = 0; j < m.getNumColumns(); j++)
-        at(i, j) = m(i, j);
-  };
-  
+            unsigned reservedColumns = 0)
+      : Matrix<MPInt>(rows, columns, reservedRows, reservedColumns){};
+
+  IntMatrix(Matrix<MPInt> m) : Matrix<MPInt>(std::move(m)){};
+
   /// Return the identity matrix of the specified dimension.
   static IntMatrix identity(unsigned dimension);
 
@@ -241,6 +234,37 @@ class IntMatrix : public Matrix<MPInt>
   /// Returns the GCD of the columns of the specified row.
   MPInt normalizeRow(unsigned row);
 
+  // Compute the determinant of the matrix (cubic time).
+  // Stores the integer inverse of the matrix in the pointer
+  // passed (if any). The pointer is unchanged if the inverse
+  // does not exist, which happens iff det = 0.
+  // For a matrix M, the integer inverse is the matrix M' such that
+  // M x M' = M'  M = det(M) x I.
+  // Assert-fails if the matrix is not square.
+  MPInt determinant(IntMatrix *inverse = nullptr) const;
+};
+
+// An inherited class for rational matrices, with no new data attributes.
+// This class is for functionality that only applies to matrices of fractions.
+class FracMatrix : public Matrix<Fraction> {
+public:
+  FracMatrix(unsigned rows, unsigned columns, unsigned reservedRows = 0,
+             unsigned reservedColumns = 0)
+      : Matrix<Fraction>(rows, columns, reservedRows, reservedColumns){};
+
+  FracMatrix(Matrix<Fraction> m) : Matrix<Fraction>(std::move(m)){};
+
+  explicit FracMatrix(IntMatrix m);
+
+  /// Return the identity matrix of the specified dimension.
+  static FracMatrix identity(unsigned dimension);
+
+  // Compute the determinant of the matrix (cubic time).
+  // Stores the inverse of the matrix in the pointer
+  // passed (if any). The pointer is unchanged if the inverse
+  // does not exist, which happens iff det = 0.
+  // Assert-fails if the matrix is not square.
+  Fraction determinant(FracMatrix *inverse = nullptr) const;
 };
 
 } // namespace presburger
diff --git a/mlir/include/mlir/Conversion/FuncToLLVM/ConvertFuncToLLVM.h b/mlir/include/mlir/Conversion/FuncToLLVM/ConvertFuncToLLVM.h
index 21bd191aa9..107718eeba 100644
--- a/mlir/include/mlir/Conversion/FuncToLLVM/ConvertFuncToLLVM.h
+++ b/mlir/include/mlir/Conversion/FuncToLLVM/ConvertFuncToLLVM.h
@@ -14,13 +14,28 @@
 #ifndef MLIR_CONVERSION_FUNCTOLLVM_CONVERTFUNCTOLLVM_H
 #define MLIR_CONVERSION_FUNCTOLLVM_CONVERTFUNCTOLLVM_H
 
+#include "mlir/Interfaces/FunctionInterfaces.h"
+#include "mlir/Support/LogicalResult.h"
+
 namespace mlir {
 
+namespace LLVM {
+class LLVMFuncOp;
+} // namespace LLVM
+
+class ConversionPatternRewriter;
 class DialectRegistry;
 class LLVMTypeConverter;
 class RewritePatternSet;
 class SymbolTable;
 
+/// Convert input FunctionOpInterface operation to LLVMFuncOp by using the
+/// provided LLVMTypeConverter. Return failure if failed to so.
+FailureOr<LLVM::LLVMFuncOp>
+convertFuncOpToLLVMFuncOp(FunctionOpInterface funcOp,
+                          ConversionPatternRewriter &rewriter,
+                          const LLVMTypeConverter &converter);
+
 /// Collect the default pattern to convert a FuncOp to the LLVM dialect. If
 /// `emitCWrappers` is set, the pattern will also produce functions
 /// that pass memref descriptors by pointer-to-structure in addition to the
diff --git a/mlir/include/mlir/Conversion/IndexToSPIRV/IndexToSPIRV.h b/mlir/include/mlir/Conversion/IndexToSPIRV/IndexToSPIRV.h
new file mode 100644
index 0000000000..58a1c5246e
--- /dev/null
+++ b/mlir/include/mlir/Conversion/IndexToSPIRV/IndexToSPIRV.h
@@ -0,0 +1,30 @@
+//===- IndexToSPIRV.h - Index to SPIRV dialect conversion -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_CONVERSION_INDEXTOSPIRV_INDEXTOSPIRV_H
+#define MLIR_CONVERSION_INDEXTOSPIRV_INDEXTOSPIRV_H
+
+#include "mlir/Pass/Pass.h"
+#include <memory>
+
+namespace mlir {
+class RewritePatternSet;
+class SPIRVTypeConverter;
+class Pass;
+
+#define GEN_PASS_DECL_CONVERTINDEXTOSPIRVPASS
+#include "mlir/Conversion/Passes.h.inc"
+
+namespace index {
+void populateIndexToSPIRVPatterns(SPIRVTypeConverter &converter,
+                                  RewritePatternSet &patterns);
+std::unique_ptr<OperationPass<>> createConvertIndexToSPIRVPass();
+} // namespace index
+} // namespace mlir
+
+#endif // MLIR_CONVERSION_INDEXTOSPIRV_INDEXTOSPIRV_H
diff --git a/mlir/include/mlir/Conversion/Passes.h b/mlir/include/mlir/Conversion/Passes.h
index e714f5070f..c13c457fd9 100644
--- a/mlir/include/mlir/Conversion/Passes.h
+++ b/mlir/include/mlir/Conversion/Passes.h
@@ -35,6 +35,7 @@
 #include "mlir/Conversion/GPUToSPIRV/GPUToSPIRVPass.h"
 #include "mlir/Conversion/GPUToVulkan/ConvertGPUToVulkanPass.h"
 #include "mlir/Conversion/IndexToLLVM/IndexToLLVM.h"
+#include "mlir/Conversion/IndexToSPIRV/IndexToSPIRV.h"
 #include "mlir/Conversion/LinalgToStandard/LinalgToStandard.h"
 #include "mlir/Conversion/MathToFuncs/MathToFuncs.h"
 #include "mlir/Conversion/MathToLLVM/MathToLLVM.h"
diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td
index 38b05c792d..274784fe4a 100644
--- a/mlir/include/mlir/Conversion/Passes.td
+++ b/mlir/include/mlir/Conversion/Passes.td
@@ -644,6 +644,28 @@ def ConvertIndexToLLVMPass : Pass<"convert-index-to-llvm"> {
   ];
 }
 
+//===----------------------------------------------------------------------===//
+// ConvertIndexToSPIRVPass
+//===----------------------------------------------------------------------===//
+
+def ConvertIndexToSPIRVPass : Pass<"convert-index-to-spirv"> {
+  let summary = "Lower the `index` dialect to the `spirv` dialect.";
+  let description = [{
+    This pass lowers Index dialect operations to SPIR-V dialect operations.
+    Operation conversions are 1-to-1 except for the exotic divides: `ceildivs`,
+    `ceildivu`, and `floordivs`. The index bitwidth will be 32 or 64 as
+    specified by use-64bit-index.
+  }];
+
+  let dependentDialects = ["::mlir::spirv::SPIRVDialect"];
+
+  let options = [
+    Option<"use64bitIndex", "use-64bit-index",
+           "bool", /*default=*/"false",
+           "Use 64-bit integers to convert index types">
+  ];
+}
+
 //===----------------------------------------------------------------------===//
 // LinalgToStandard
 //===----------------------------------------------------------------------===//
@@ -792,10 +814,10 @@ def ConvertMemRefToSPIRV : Pass<"convert-memref-to-spirv"> {
 //===----------------------------------------------------------------------===//
 
 def ConvertNVVMToLLVMPass : Pass<"convert-nvvm-to-llvm"> {
-  let summary = "Convert NVVM dialect to LLVM dialect";
+  let summary = "Convert NVVM to PTX with Inline Assembly in LLVM dialect";
   let description = [{
-    This pass generates inline assembly for the NVVM ops which is not 
-    implemented in LLVM core.
+    This pass generates PTX instructions using inline assembly for NVVM 
+    operations implements `BasicPtxBuilderInterface`.
   }];
   let dependentDialects = [
     "NVVM::NVVMDialect",
@@ -1082,6 +1104,14 @@ def TosaToLinalg
   }];
 
   let constructor = "tosa::createTosaToLinalg()";
+  let options = [
+    Option<"disableTosaDecompositions", "disable-tosa-decompositions",
+           "bool", /*default=*/"false",
+           "Disable tosa decompositions pass">,
+    Option<"aggressiveReduceConstant", "aggressive-reduce-constant",
+           "bool", /*default=*/"false",
+           "Always perform the reduce constant optimization">
+  ];
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Conversion/TosaToLinalg/TosaToLinalg.h b/mlir/include/mlir/Conversion/TosaToLinalg/TosaToLinalg.h
index 818d43ffe4..c411010603 100644
--- a/mlir/include/mlir/Conversion/TosaToLinalg/TosaToLinalg.h
+++ b/mlir/include/mlir/Conversion/TosaToLinalg/TosaToLinalg.h
@@ -33,10 +33,10 @@ std::unique_ptr<Pass> createTosaToLinalgNamed();
 /// pipeline succeeds.  The option to disable decompositions is available for
 /// benchmarking performance improvements from the canonicalizations.
 void addTosaToLinalgPasses(
-    OpPassManager &pm, bool disableTosaDecompositions = false,
+    OpPassManager &pm, const TosaToLinalgOptions &options,
     // Note: Default to 'none' level unless otherwise specified.
-    tosa::ValidationOptions const &validationOptions =
-        tosa::ValidationOptions().setLevel(tosa::TosaLevelEnum::None));
+    tosa::TosaValidationOptions const &validationOptions = {
+        tosa::TosaProfileEnum::Undefined, false, tosa::TosaLevelEnum::None});
 
 /// Populates conversion passes from TOSA dialect to Linalg dialect.
 void populateTosaToLinalgConversionPatterns(RewritePatternSet *patterns);
diff --git a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td
index f90a7b23ec..36fdf390a7 100644
--- a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td
+++ b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td
@@ -121,7 +121,8 @@ def AffineForOp : Affine_Op<"for",
      ImplicitAffineTerminator, ConditionallySpeculatable,
      RecursiveMemoryEffects, DeclareOpInterfaceMethods<LoopLikeOpInterface,
      ["getSingleInductionVar", "getSingleLowerBound", "getSingleStep",
-      "getSingleUpperBound", "replaceWithAdditionalYields"]>,
+      "getSingleUpperBound", "getYieldedValues",
+      "replaceWithAdditionalYields"]>,
      DeclareOpInterfaceMethods<RegionBranchOpInterface,
      ["getEntrySuccessorOperands"]>]> {
   let summary = "for operation";
diff --git a/mlir/include/mlir/Dialect/Affine/IR/ValueBoundsOpInterfaceImpl.h b/mlir/include/mlir/Dialect/Affine/IR/ValueBoundsOpInterfaceImpl.h
index 2abbabc5bb..6e617ef40a 100644
--- a/mlir/include/mlir/Dialect/Affine/IR/ValueBoundsOpInterfaceImpl.h
+++ b/mlir/include/mlir/Dialect/Affine/IR/ValueBoundsOpInterfaceImpl.h
@@ -9,11 +9,27 @@
 #ifndef MLIR_DIALECT_AFFINE_IR_VALUEBOUNDSOPINTERFACEIMPL_H
 #define MLIR_DIALECT_AFFINE_IR_VALUEBOUNDSOPINTERFACEIMPL_H
 
+#include "mlir/Support/LogicalResult.h"
+
 namespace mlir {
 class DialectRegistry;
+class Value;
 
 namespace affine {
 void registerValueBoundsOpInterfaceExternalModels(DialectRegistry &registry);
+
+/// Compute a constant delta of the given two values. Return "failure" if we
+/// cannot determine a constant delta. `value1`/`value2` must be index-typed.
+///
+/// This function is similar to
+/// `ValueBoundsConstraintSet::computeConstantDistance`. To work around
+/// limitations in `FlatLinearConstraints`, this function fully composes
+/// `value1` and `value2` (if they are the result of affine.apply ops) before
+/// populating the constraint set. The folding/composing logic can see
+/// opportunities for simplifications that the constraint set implementation
+/// cannot see.
+FailureOr<int64_t> fullyComposeAndComputeConstantDelta(Value value1,
+                                                       Value value2);
 } // namespace affine
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Dialect/ArmSME/IR/ArmSME.h b/mlir/include/mlir/Dialect/ArmSME/IR/ArmSME.h
index b27ceca215..fe1f9062a3 100644
--- a/mlir/include/mlir/Dialect/ArmSME/IR/ArmSME.h
+++ b/mlir/include/mlir/Dialect/ArmSME/IR/ArmSME.h
@@ -14,6 +14,7 @@
 #define MLIR_DIALECT_ARMSME_IR_ARMSME_H
 
 #include "mlir/Bytecode/BytecodeOpInterface.h"
+#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/IR/BuiltinTypes.h"
diff --git a/mlir/include/mlir/Dialect/ArmSME/IR/ArmSMEIntrinsicOps.td b/mlir/include/mlir/Dialect/ArmSME/IR/ArmSMEIntrinsicOps.td
index feeac3b8a0..df837ebcf2 100644
--- a/mlir/include/mlir/Dialect/ArmSME/IR/ArmSMEIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/ArmSME/IR/ArmSMEIntrinsicOps.td
@@ -38,16 +38,16 @@ class ArmSME_IntrOp<string mnemonic, list<int> overloadedOperands = [],
 
 // Zero
 def LLVM_aarch64_sme_zero : ArmSME_IntrOp<"zero">,
-                            Arguments<(ins Arg<I32, "Tile mask">)>;
+                            Arguments<(ins Arg<I32, "Tile mask">:$tile_mask)>;
 
 // MOP's
 class ArmSME_IntrMopOverloadedOp<string mnemonic>
     : ArmSME_IntrOp<mnemonic, [4]>,
-      Arguments<(ins Arg<I32, "Virtual tile ID">,
-                 Arg<MOPPredicate, "LHS predicate">,
-                 Arg<MOPPredicate, "RHS predicate">,
-                 Arg<MOPVector, "LHS vector operand">,
-                 Arg<MOPVector, "RHS vector operand">)>;
+      Arguments<(ins Arg<I32, "Virtual tile ID">:$tile_id,
+                 Arg<MOPPredicate, "LHS predicate">:$lhs_predicate,
+                 Arg<MOPPredicate, "RHS predicate">:$rhs_predicate,
+                 Arg<MOPVector, "LHS vector operand">:$lhs_vector,
+                 Arg<MOPVector, "RHS vector operand">:$rhs_vector)>;
 
 def LLVM_aarch64_sme_mopa : ArmSME_IntrMopOverloadedOp<"mopa">;
 def LLVM_aarch64_sme_mops : ArmSME_IntrMopOverloadedOp<"mops">;
@@ -65,10 +65,10 @@ def LLVM_aarch64_sme_usmops_wide : ArmSME_IntrMopOverloadedOp<"usmops.wide">;
 // Loads
 class ArmSME_IntrLoadOp<string mnemonic>
     : ArmSME_IntrOp<mnemonic>,
-      Arguments<(ins Arg<LDSTPredicate, "Vector predicate">,
-                 Arg<LLVM_AnyPointer, "Load address">,
-                 Arg<I32, "Virtual tile ID">,
-                 Arg<I32, "Tile slice">)>;
+      Arguments<(ins Arg<LDSTPredicate, "Vector predicate">:$predicate,
+                 Arg<LLVM_AnyPointer, "Load address">:$load_address,
+                 Arg<I32, "Virtual tile ID">:$tile_id,
+                 Arg<I32, "Tile slice">:$tile_slice_index)>;
 
 def LLVM_aarch64_sme_ld1b_horiz : ArmSME_IntrLoadOp<"ld1b.horiz">;
 def LLVM_aarch64_sme_ld1h_horiz : ArmSME_IntrLoadOp<"ld1h.horiz">;
@@ -84,10 +84,10 @@ def LLVM_aarch64_sme_ld1q_vert : ArmSME_IntrLoadOp<"ld1q.vert">;
 // Stores
 class ArmSME_IntrStoreOp<string mnemonic>
     : ArmSME_IntrOp<mnemonic>,
-      Arguments<(ins Arg<LDSTPredicate, "Vector predicate">,
-                 Arg<LLVM_AnyPointer, "Store address", [MemWrite]>,
-                 Arg<I32, "Virtual tile ID">,
-                 Arg<I32, "Tile slice">)>;
+      Arguments<(ins Arg<LDSTPredicate, "Vector predicate">:$predicate,
+                 Arg<LLVM_AnyPointer, "Store address", [MemWrite]>:$store_address,
+                 Arg<I32, "Virtual tile ID">:$tild_id,
+                 Arg<I32, "Tile slice">:$tile_slice_index)>;
 
 def LLVM_aarch64_sme_st1b_horiz : ArmSME_IntrStoreOp<"st1b.horiz">;
 def LLVM_aarch64_sme_st1h_horiz : ArmSME_IntrStoreOp<"st1h.horiz">;
@@ -102,28 +102,28 @@ def LLVM_aarch64_sme_st1q_vert : ArmSME_IntrStoreOp<"st1q.vert">;
 
 def LLVM_aarch64_sme_str
     : ArmSME_IntrOp<"str">,
-      Arguments<(ins Arg<I32, "Index">,
-                 Arg<LLVM_AnyPointer, "Store address", [MemWrite]>)>;
+      Arguments<(ins Arg<I32, "Index">:$index,
+                 Arg<LLVM_AnyPointer, "Store address", [MemWrite]>:$store_address)>;
 
 // Vector to tile slice
 class LLVM_aarch64_sme_write<string direction>
     : ArmSME_IntrOp<"write." # direction, /*overloadedOperands=*/[3],
-                    [AllShapesMatch<["pg", "vector"]>]>,
-      Arguments<(ins Arg<I32, "Virtual tile ID">,
-                     Arg<I32, "Tile slice">,
-                     Arg<SVEPredicate, "Vector predicate">:$pg,
+                    [AllShapesMatch<["predicate", "vector"]>]>,
+      Arguments<(ins Arg<I32, "Virtual tile ID">:$tile_id,
+                     Arg<I32, "Tile slice">:$tile_slice_index,
+                     Arg<SVEPredicate, "Vector predicate">:$predicate,
                      Arg<SVEVector, "Vector operand">:$vector)>;
 
 // Tile slice to vector
 class LLVM_aarch64_sme_read<string direction>
     : ArmSME_IntrOp<"read." # direction, /*overloadedOperands=*/[],
-                    [AllShapesMatch<["vector", "pg", "res"]>,
+                    [AllShapesMatch<["vector", "predicate", "res"]>,
                      AllElementTypesMatch<["vector", "res"]>],
                     /*numResults=*/1, /*overloadedResults=*/[0]>,
       Arguments<(ins Arg<SVEVector, "Vector operand">:$vector,
-                     Arg<SVEPredicate, "Vector predicate">:$pg,
-                     Arg<I32, "Virtual tile ID">,
-                     Arg<I32, "Tile slice">)>;
+                     Arg<SVEPredicate, "Vector predicate">:$predicate,
+                     Arg<I32, "Virtual tile ID">:$tile_id,
+                     Arg<I32, "Tile slice">:$tile_slice_index)>;
 
 def LLVM_aarch64_sme_write_horiz : LLVM_aarch64_sme_write<"horiz">;
 def LLVM_aarch64_sme_write_vert : LLVM_aarch64_sme_write<"vert">;
diff --git a/mlir/include/mlir/Dialect/ArmSME/IR/ArmSMEOps.td b/mlir/include/mlir/Dialect/ArmSME/IR/ArmSMEOps.td
index e090922680..dab54b63d8 100644
--- a/mlir/include/mlir/Dialect/ArmSME/IR/ArmSMEOps.td
+++ b/mlir/include/mlir/Dialect/ArmSME/IR/ArmSMEOps.td
@@ -27,7 +27,7 @@ include "mlir/Interfaces/InferTypeOpInterface.td"
 
 class SMETileType<Type datatype, list<int> dims, string description>
   : ShapedContainerType<[datatype],
-      And<[IsVectorOfRankPred<[2]>, allDimsScalableVectorTypePred,
+      And<[IsVectorOfRankPred<[2]>, IsVectorTypeWithAllDimsScalablePred,
            IsVectorOfShape<dims>]>,
   description>;
 
@@ -76,6 +76,7 @@ def TileSliceLayout : I32EnumAttr<"TileSliceLayout", "Layout of a tile slice", [
 def ArmSME_TileSliceLayoutAttr : EnumAttr<ArmSME_Dialect, TileSliceLayout,
                                           "layout"> {
   let assemblyFormat = "`<` $value `>`";
+  let defaultValue = "TileSliceLayout::Horizontal";
 }
 
 //===----------------------------------------------------------------------===//
@@ -248,19 +249,18 @@ def TileLoadOp : ArmSME_Op<"tile_load"> {
 
     Example 2: Load a FP 32-bit element ZA tile with vertical layout from memory.
     ```mlir
-    %tile = arm_sme.tile_load %base[%c0, %c0], <vertical> : memref<?x?xf32>, vector<[4]x[4]xf32>
+    %tile = arm_sme.tile_load %base[%c0, %c0] layout<vertical> : memref<?x?xf32>, vector<[4]x[4]xf32>
     ```
 
     Example 3: Load a 128-bit element ZA tile with horizontal layout (default) from memory.
     ```mlir
-    %tile = arm_sme.tile_load %base[%c0, %c0], <horizontal> : memref<?x?xi128>, vector<[1]x[1]xi128>
+    %tile = arm_sme.tile_load %base[%c0, %c0] layout<horizontal> : memref<?x?xi128>, vector<[1]x[1]xi128>
     ```
   }];
   let arguments = (ins
     Arg<AnyMemRef, "the reference to load from", [MemRead]>:$base,
     Variadic<Index>:$indices,
-    DefaultValuedAttr<ArmSME_TileSliceLayoutAttr,
-                      "::mlir::arm_sme::TileSliceLayout::Horizontal">:$layout
+    ArmSME_TileSliceLayoutAttr:$layout
   );
   let results = (outs SMETile:$result);
 
@@ -274,7 +274,7 @@ def TileLoadOp : ArmSME_Op<"tile_load"> {
   }];
 
   let assemblyFormat =
-    "$base `[` $indices `]` (`,` $layout^)? attr-dict "
+    "$base `[` $indices `]` (`layout` `` $layout^)? attr-dict "
       "`:` type($base) `,` type($result)";
 }
 
@@ -296,19 +296,17 @@ def TileStoreOp : ArmSME_Op<"tile_store"> {
 
     Example 2: Store a FP 32-bit element ZA tile with vertical layout to memory.
     ```mlir
-    arm_sme.tile_store %tile, %base[%c0, %c0], <vertical> : vector<[4]x[4]xf32>, memref<?x?xf32>
+    arm_sme.tile_store %tile, %base[%c0, %c0] layout<vertical> : vector<[4]x[4]xf32>, memref<?x?xf32>
     ```
 
     Example 3: Store a 128-bit element ZA tile with horizontal (default) layout to memory.
     ```mlir
-    arm_sme.tile_store %tile, %base[%c0, %c0], <horizontal> : vector<[1]x[1]xi128>, memref<?x?xi128>
+    arm_sme.tile_store %tile, %base[%c0, %c0] layout<horizontal> : vector<[1]x[1]xi128>, memref<?x?xi128>
     ```
   }];
   let arguments = (ins SMETile:$valueToStore,
     Arg<AnyMemRef, "the reference to store to", [MemWrite]>:$base,
-    Variadic<Index>:$indices,
-    DefaultValuedAttr<ArmSME_TileSliceLayoutAttr,
-                      "::mlir::arm_sme::TileSliceLayout::Horizontal">:$layout
+    Variadic<Index>:$indices, ArmSME_TileSliceLayoutAttr:$layout
   );
   let extraClassDeclaration = [{
     MemRefType getMemRefType() {
@@ -320,7 +318,7 @@ def TileStoreOp : ArmSME_Op<"tile_store"> {
   }];
 
   let assemblyFormat =
-    "$valueToStore `,` $base `[` $indices `]` (`,` $layout^)? attr-dict "
+    "$valueToStore `,` $base `[` $indices `]` (`layout` `` $layout^)? attr-dict "
       "`:` type($base) `,` type($valueToStore)";
 }
 
@@ -348,19 +346,18 @@ def LoadTileSliceOp : ArmSME_Op<"load_tile_slice", [
 
     Example 2: Load a vector<[4]xf32> tile slice from memory into tile vertically at given index.
     ```mlir
-    %tile_update = arm_sme.load_tile_slice %base[%c0], %tile, %tile_slice_index, <vertical> : memref<?x?xf32>, vector<[4]x[4]xf32>
+    %tile_update = arm_sme.load_tile_slice %base[%c0], %tile, %tile_slice_index layout<vertical> : memref<?x?xf32>, vector<[4]x[4]xf32>
     ```
 
     Example 3: Load a vector<[1]xi128> tile slice from memory into tile vertically at given index.
     ```mlir
-    %tile_update = arm_sme.load_tile_slice %base[%c0], %tile, %tile_slice_index, <vertical> : memref<?x?xi128>, vector<[1]x[1]xi128>
+    %tile_update = arm_sme.load_tile_slice %base[%c0], %tile, %tile_slice_index layout<vertical> : memref<?x?xi128>, vector<[1]x[1]xi128>
     ```
   }];
   let arguments = (ins
     Arg<AnyMemRef, "the reference to load from">:$base,
     SMETile:$tile, Variadic<Index>:$indices, Index:$tile_slice_index,
-    DefaultValuedAttr<ArmSME_TileSliceLayoutAttr,
-                      "::mlir::arm_sme::TileSliceLayout::Horizontal">:$layout
+    ArmSME_TileSliceLayoutAttr:$layout
   );
   let results = (outs SMETile:$result);
 
@@ -374,7 +371,7 @@ def LoadTileSliceOp : ArmSME_Op<"load_tile_slice", [
   }];
 
   let assemblyFormat = [{
-    $base `[` $indices `]` `,` $tile `,` $tile_slice_index (`,` $layout^)?
+    $base `[` $indices `]` `,` $tile `,` $tile_slice_index (`layout` `` $layout^)?
       attr-dict `:` type($base) `,` type($result)
   }];
 }
@@ -401,19 +398,17 @@ def StoreTileSliceOp : ArmSME_Op<"store_tile_slice"> {
 
     Example 2: Store vector<[4]xf32> vertical tile slice from tile at given index to memory.
     ```mlir
-    arm_sme.store_tile_slice %tile, %tile_slice_index, %base[%c0], <vertical> : vector<[4]x[4]xf32>, memref<?x?xf32>
+    arm_sme.store_tile_slice %tile, %tile_slice_index, %base[%c0] layout<vertical> : vector<[4]x[4]xf32>, memref<?x?xf32>
     ```
 
     Example 3: Store a vector<[1]xi128> vertical tile slice from tile at given index to memory.
     ```mlir
-    arm_sme.store_tile_slice %tile, %tile_slice_index, %base[%c0], <vertical> : vector<[1]x[1]xi128>, memref<?x?xi128>
+    arm_sme.store_tile_slice %tile, %tile_slice_index, %base[%c0] layout<vertical> : vector<[1]x[1]xi128>, memref<?x?xi128>
     ```
   }];
   let arguments = (ins SMETile:$tile, Index:$tile_slice_index,
     Arg<AnyMemRef, "the reference to store to", [MemWrite]>:$base,
-    Variadic<Index>:$indices,
-    DefaultValuedAttr<ArmSME_TileSliceLayoutAttr,
-                      "::mlir::arm_sme::TileSliceLayout::Horizontal">:$layout
+    Variadic<Index>:$indices, ArmSME_TileSliceLayoutAttr:$layout
   );
   let extraClassDeclaration = [{
     MemRefType getMemRefType() {
@@ -425,7 +420,7 @@ def StoreTileSliceOp : ArmSME_Op<"store_tile_slice"> {
   }];
 
   let assemblyFormat = [{
-    $tile `,` $tile_slice_index `,` $base `[` $indices `]` (`,` $layout^)?
+    $tile `,` $tile_slice_index `,` $base `[` $indices `]` (`layout` `` $layout^)?
       attr-dict `:` type($base) `,` type($tile)
   }];
 }
diff --git a/mlir/include/mlir/Dialect/ArmSVE/IR/ArmSVE.td b/mlir/include/mlir/Dialect/ArmSVE/IR/ArmSVE.td
index d4294b4dd9..e3f3d9e62e 100644
--- a/mlir/include/mlir/Dialect/ArmSVE/IR/ArmSVE.td
+++ b/mlir/include/mlir/Dialect/ArmSVE/IR/ArmSVE.td
@@ -28,6 +28,8 @@ def ArmSVE_Dialect : Dialect {
     This dialect contains the definitions necessary to target specific Arm SVE
     scalable vector operations.
   }];
+
+  let dependentDialects = ["vector::VectorDialect"];
 }
 
 //===----------------------------------------------------------------------===//
@@ -40,6 +42,13 @@ def SVBool : ScalableVectorOfRankAndLengthAndType<
 def SVEPredicate : ScalableVectorOfRankAndLengthAndType<
   [1], [16, 8, 4, 2, 1], [I1]>;
 
+// Generalizations of SVBool and SVEPredicate to ranks >= 1.
+// These are masks with a single trailing scalable dimension.
+def SVBoolMask : VectorWithTrailingDimScalableOfSizeAndType<
+  [16], [I1]>;
+def SVEPredicateMask : VectorWithTrailingDimScalableOfSizeAndType<
+  [16, 8, 4, 2, 1], [I1]>;
+
 //===----------------------------------------------------------------------===//
 // ArmSVE op definitions
 //===----------------------------------------------------------------------===//
@@ -236,6 +245,81 @@ def UmmlaOp : ArmSVE_Op<"ummla",
     "$acc `,` $src1 `,` $src2 attr-dict `:` type($src1) `to` type($dst)";
 }
 
+class SvboolTypeConstraint<string lhsArg, string rhsArg> : TypesMatchWith<
+      "expected corresponding svbool type widened to [16]xi1",
+      lhsArg, rhsArg,
+      "VectorType(VectorType::Builder(::llvm::cast<VectorType>($_self)).setDim(::llvm::cast<VectorType>($_self).getRank() - 1, 16))">;
+
+def ConvertFromSvboolOp : ArmSVE_Op<"convert_from_svbool",
+                            [Pure, SvboolTypeConstraint<"result", "source">]>
+{
+  let summary = "Convert a svbool type to a SVE predicate type";
+  let description = [{
+    Converts svbool types (`vector<[16]xi1>` or vectors of that type, e.g.
+    `vector<2x3x[16]xi1>`) to SVE predicate types. Note: Only the trailing
+    dimension can be scalable.
+
+    Example 1: Convert a 1-D svbool mask to a SVE predicate.
+    ```mlir
+    %source = vector.load %memref[%c0] : memref<?xi1>, vector<[16]xi1>
+    %result = arm_sve.convert_from_svbool %source : vector<[4]xi1>
+    ```
+
+    Example 2: Convert a 2-D svbool mask to a mask of SVE predicates.
+    ```mlir
+    %source = vector.load %memref[%c0, %c0] : memref<2x?xi1>, vector<2x[16]xi1>
+    %result = arm_sve.convert_from_svbool %source : vector<2x[8]xi1>
+    ```
+
+    ---
+
+    A `svbool` is the smallest SVE predicate type that has a in-memory
+    representation (and maps to a full predicate register). In MLIR `svbool` is
+    represented as `vector<[16]xi1>`. Smaller SVE predicate types
+    (`vector<[1|2|4|8]xi1>`) must be stored as a `svbool` then converted back to
+    the original predicate type after loading.
+  }];
+  let arguments = (ins SVBoolMask:$source);
+  let results = (outs SVEPredicateMask:$result);
+  let assemblyFormat = "$source attr-dict `:` type($result)";
+}
+
+def ConvertToSvboolOp : ArmSVE_Op<"convert_to_svbool",
+                            [Pure, SvboolTypeConstraint<"source", "result">]>
+{
+  let summary = "Convert a SVE predicate type to a svbool type";
+  let description = [{
+    Converts SVE predicate types (or vectors of predicate types, e.g.
+    `vector<4x[4]xi1>`) to svbool types. Note: Only the trailing dimension can
+    be scalable.
+
+    Example 1: Convert a 1-D SVE predicate to a svbool mask.
+    ```mlir
+    %source = vector.create_mask %dim_size : vector<[4]xi1>
+    %result = arm_sve.convert_to_svbool %source : vector<[4]xi1>
+    // => Results in vector<[16]xi1>
+    ```
+
+    Example 2: Convert a 2-D mask of SVE predicates to a svbool mask.
+    ```mlir
+    %source = vector.create_mask %c2, %dim_size : vector<2x[2]xi1>
+    %result = arm_sve.convert_to_svbool %source : vector<2x[2]xi1>
+    // => Results in vector<2x[16]xi1>
+    ```
+
+    ---
+
+    A `svbool` is the smallest SVE predicate type that has a in-memory
+    representation (and maps to a full predicate register). In MLIR `svbool` is
+    represented as `vector<[16]xi1>`. Smaller SVE predicate types
+    (`vector<[1|2|4|8]xi1>`) must be converted to a `svbool` before they can be
+    stored.
+  }];
+  let arguments = (ins SVEPredicateMask:$source);
+  let results = (outs SVBoolMask:$result);
+  let assemblyFormat = "$source attr-dict `:` type($source)";
+}
+
 def ScalableMaskedAddIOp : ScalableMaskedIOp<"masked.addi", "addition",
                                              [Commutative]>;
 
diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationBase.td b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationBase.td
index 0d509e6934..818e931aeb 100644
--- a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationBase.td
+++ b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationBase.td
@@ -60,6 +60,17 @@ def Bufferization_Dialect : Dialect {
     /// arguments during One-Shot Module Bufferize.
     constexpr const static ::llvm::StringLiteral
         kBufferLayoutAttrName = "bufferization.buffer_layout";
+
+    /// An attribute that can be attached to ops with an allocation and/or
+    /// deallocation side effect. It indicates that the op is under a "manual
+    /// deallocation" scheme. In the case of an allocation op, the returned
+    /// value is *not* an automatically managed allocation and assigned an
+    /// ownership of "false". Furthermore, only deallocation ops that are
+    /// guaranteed to deallocate a buffer under "manual deallocation" are
+    /// allowed to have this attribute. (Deallocation ops without this
+    /// attribute are rejected by the ownership-based buffer deallocation pass.)
+    constexpr const static ::llvm::StringLiteral
+        kManualDeallocation = "bufferization.manual_deallocation";
   }];
   let hasOperationAttrVerify = 1;
 }
diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td
index 34a6f5d74b..c779d1f843 100644
--- a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td
+++ b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td
@@ -246,28 +246,32 @@ def Bufferization_MaterializeInDestinationOp
     rewrite IR such that a computation is performed directly in `dest` and no
     memcpy is needed.
 
-    If `dest` is a buffer, the `restrict` and `writable` attributes must be
-    specified. These attributes have the same meaning as the respective
-    attributes of `bufferization.to_tensor`. `writable` indicates that the
-    `dest` buffer is considered writable. It does not make sense to materialize
-    a computation in a read-only buffer, so `writable` is required. `restrict`
-    indicates that this op is the only way for the tensor IR to access `dest`
-    (or an alias thereof). E.g., there must be no other `to_tensor` ops with
-    `dest` or with an alias of `dest`. Such IR is not supported by
-    One-Shot Bufferize. Ops that have incorrect usage of `restrict` may
-    bufferize incorrectly.
-
-    Note: `restrict` and `writable` could be removed from this op because they
-    must always be set for memref destinations. This op has these attributes to
-    make clear the requirements on the `dest` operand in the op assembly format.
-    Moreover, these requirements may be relaxed at some point in the future.
+    If `dest` is a buffer, the `writable` attribute must be specified and the
+    `restrict` keyword can be specified. These attributes have the same meaning
+    as the respective attributes of `bufferization.to_tensor`.
+
+    `writable` indicates that the `dest` buffer is considered writable. It does
+    not make sense to materialize a computation in a read-only buffer, so
+    `writable` is required.
+
+    `restrict` indicates that there is no `bufferization.to_tensor` op and no
+    other `bufferization.materialize_in_destination` op with `dest` (or an alias
+    thereof) and "restrict". Only ops with this attribute are considered for
+    "empty tensor elimination". As part of empty tensor elimination, a new
+    `to_tensor` op with `dest` may be inserted and the `restrict` attribute is
+    transferred from this op to the new `to_tensor` op. Having "restrict" on
+    this op guarantees that performing empty tensor elimination would not create
+    invalid IR (i.e., having multiple `to_tensor restrict` with aliasing
+    buffers).
+
+    Note: `writable` could be removed from this op because it must always be set
+    for memref destinations. This op has that attribute to make clear the
+    requirements on the `dest` operand in the op assembly format.
 
     Note: If `dest` is a tensor, `tensor.insert_slice` could be used for the
     same purpose, but since tensor dialect ops only indicate *what* should be
     computed but not *where*, it could fold away, causing the computation to
-    materialize in a different buffer. It is also possible that the
-    `tensor.insert_slice` destination bufferizes out-of-place, which would also
-    cause the computation to materialize in a buffer different buffer.
+    materialize in a different buffer.
   }];
 
   let arguments = (ins AnyTensor:$source, AnyShaped:$dest,
diff --git a/mlir/include/mlir/Dialect/ControlFlow/IR/ControlFlowOps.td b/mlir/include/mlir/Dialect/ControlFlow/IR/ControlFlowOps.td
index b396b2c2c1..f77b8cbbbc 100644
--- a/mlir/include/mlir/Dialect/ControlFlow/IR/ControlFlowOps.td
+++ b/mlir/include/mlir/Dialect/ControlFlow/IR/ControlFlowOps.td
@@ -40,7 +40,8 @@ class CF_Op<string mnemonic, list<Trait> traits = []> :
 def AssertOp : CF_Op<"assert"> {
   let summary = "Assert operation with message attribute";
   let description = [{
-    Assert operation with single boolean operand and an error message attribute.
+    Assert operation at runtime with single boolean operand and an error
+    message attribute.
     If the argument is `true` this operation has no effect. Otherwise, the
     program execution will abort. The provided error message may be used by a
     runtime to propagate the error to the user.
diff --git a/mlir/include/mlir/Dialect/LLVMIR/BasicPtxBuilderInterface.h b/mlir/include/mlir/Dialect/LLVMIR/BasicPtxBuilderInterface.h
new file mode 100644
index 0000000000..5b7df69b62
--- /dev/null
+++ b/mlir/include/mlir/Dialect/LLVMIR/BasicPtxBuilderInterface.h
@@ -0,0 +1,80 @@
+//===- BasicPtxBuilderInterface.td - PTX builder interface -*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines the interface to build PTX (Parallel Thread Execution) from NVVM Ops
+// automatically. It is used by NVVM to LLVM pass.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef NVVM_DIALECT_NVVM_IR_BASICPTXBUILDERINTERFACE_H_
+#define NVVM_DIALECT_NVVM_IR_BASICPTXBUILDERINTERFACE_H_
+
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/IR/PatternMatch.h"
+#include "mlir/IR/Value.h"
+
+namespace mlir {
+namespace NVVM {
+/// Register read/write modifier to build constraint string for PTX inline
+/// https://docs.nvidia.com/cuda/inline-ptx-assembly/index.html#parameters
+enum class PTXRegisterMod {
+  /// Read register with no modifier
+  Read = 0,
+  /// Read register with '+' modifier
+  Write = 2,
+  /// Read register with '=' modifier.
+  /// Note that, this is not natively supported by LLVM, but it is possible to
+  /// set read and write for the same operand.
+  ReadWrite = 1,
+};
+} // namespace NVVM
+} // namespace mlir
+
+/// Include the generated interface declarations.
+#include "mlir/Dialect/LLVMIR/BasicPtxBuilderInterface.h.inc"
+
+namespace mlir {
+
+namespace NVVM {
+
+/// A class to build PTX assembly automatically. It is used by
+/// BasicPtxBuilderInterface.
+class PtxBuilder {
+  // The interface op that is used to build the PTX.
+  BasicPtxBuilderInterface interfaceOp;
+  // Rewriter to create new operations.
+  PatternRewriter &rewriter;
+  // The operands for the PTX instruction
+  SmallVector<Value> ptxOperands;
+  // Register constraints (read, write, readwrite) and register data types
+  std::string registerConstraints;
+
+  bool hasResult = false;
+
+public:
+  /// Single constructor that only initializes members.
+  PtxBuilder(Operation *op, PatternRewriter &rewriter)
+      : interfaceOp(op), rewriter(rewriter) {}
+
+  /// Add an operand with the read/write input type.
+  void insertValue(Value v, PTXRegisterMod itype = PTXRegisterMod::Read);
+
+  /// Builds the inline assembly Op and returns it. The `insertValue` needs to
+  /// be called to pass operands before building the PTX.
+  LLVM::InlineAsmOp build();
+
+  /// Shortcut to build the inline assembly Op and replace or erase the original
+  /// op with
+  void buildAndReplaceOp();
+};
+
+} // namespace NVVM
+} // namespace mlir
+
+#endif // NVVM_DIALECT_NVVM_IR_BASICPTXBUILDERINTERFACE_H_
diff --git a/mlir/include/mlir/Dialect/LLVMIR/BasicPtxBuilderInterface.td b/mlir/include/mlir/Dialect/LLVMIR/BasicPtxBuilderInterface.td
new file mode 100644
index 0000000000..df5a2448bd
--- /dev/null
+++ b/mlir/include/mlir/Dialect/LLVMIR/BasicPtxBuilderInterface.td
@@ -0,0 +1,157 @@
+//===- BasicPtxBuilderInterface.td - PTX builder interface -*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines the interface to build PTX (Parallel Thread Execution) from NVVM Ops 
+// automatically. It is used by NVVM to LLVM pass. 
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef BASICPTXBUILDER_OP_INTERFACE
+#define BASICPTXBUILDER_OP_INTERFACE
+
+include "mlir/IR/EnumAttr.td"
+include "mlir/Dialect/GPU/IR/CompilationAttrInterfaces.td"
+include "mlir/Dialect/LLVMIR/LLVMOpBase.td"
+
+//===----------------------------------------------------------------------===//
+// Basic PTX Builder Interface
+//===----------------------------------------------------------------------===//
+
+def PtxPredicate : Optional<I1>;
+
+def BasicPtxBuilderOpInterface : OpInterface<"BasicPtxBuilderInterface"> {
+  let description = [{
+    This interface is used to generate inline assembly with PTX for basic 
+    operations. It's utilized in the `convert-nvvm-to-llvm pass` to lower 
+    NVVM Ops that implement this interface to PTX (parallel thread execution) 
+    using inline assembly Ops. Interface methods play a crucial role in this 
+    lowering process.
+
+    Here's an example of an Op with the `BasicPtxBuilderOpInterface`:    
+    ```tablegen
+      def NVVM_SpecialOp : NVVM_Op<"special.op",
+          [DeclareOpInterfaceMethods<BasicPtxBuilderOpInterface>]>,  
+        Results<(outs LLVM_Type:$res)>,
+        Arguments<(ins LLVM_i64ptr_any:$op1, I32:$op2)> {
+        ...
+        let extraClassDefinition = [{
+          std::string $cppClass::getPtx() { 
+            return std::string("special.op %0, %1, %2;"); 
+          }
+     } ];
+    ```
+
+    In the above NVVM Op example:
+    ```mlir
+      %0 = nvvm.special.op %1, %2 : !llvm.ptr, i32 -> i32
+    ```
+
+    The `convert-nvvm-to-llvm` pass generates the inline assembly like below. 
+    The order of arguments is retained, and the read and write modifiers are 
+    set based on the input and result types:
+    ```mlir
+      %0 = llvm.inline_asm 
+                has_side_effects 
+                asm_dialect = 
+                att "special.op %0, %1, %2;", "=r,l,r" %arg0, %arg1 
+                : (!llvm.ptr, i32) -> i32
+    ```
+  }];
+  let cppNamespace = "::mlir::NVVM";
+  let methods = [
+    InterfaceMethod<
+        /*desc=*/[{
+          Optional function for setting a predicate, which 
+          always returns a `PtxPredicate` value of type i1. If no predicate is 
+          provided, the instruction is unguarded; otherwise, it's guarded by the 
+          predicate value. The `PtxPredicate` value must always be the last argument. 
+          The provided PTX code by `getPtx` should not include the predicate usage.
+          The interface automatically handles predicate usage in the generated
+          PTX code when necessary.
+        }],
+        /*retType=*/"std::optional<::mlir::Value>",
+        /*methodName=*/"getPredicate",
+        /*args=*/(ins),
+        /*methodBody=*/"",
+        /*defaultImplementation=*/"return {};"
+      >,
+    InterfaceMethod<
+        /*desc=*/[{ Returns PTX assembly with operand number. }],
+        /*retType=*/"std::string",
+        /*methodName=*/"getPtx"
+      >,
+    InterfaceMethod<
+        /*desc=*/[{
+          This function indicates whether the operation is supported by LLVM 
+          intrinsics. It's particularly useful for operations that have 
+          specific cases with LLVM intrinsic support.
+        }],
+        /*retType=*/"bool",
+        /*methodName=*/"hasIntrinsic",
+        /*args=*/(ins),
+        /*methodBody=*/"",
+        /*defaultImplementation=*/"return false;"
+      >,
+    InterfaceMethod<
+        /*desc=*/[{Return whether the operation has memory side effects.}],
+        /*retType=*/"bool",
+        /*methodName=*/"hasSideEffect",
+        /*args=*/(ins),
+        /*methodBody=*/"",
+        /*defaultImplementation=*/"return true;"
+      >,
+    
+    InterfaceMethod<
+        /*desc=*/[{Helper function to generate i32 constant value.}],
+        /*retType=*/"::mlir::Value",
+        /*methodName=*/"makeConstantI32",
+        /*args=*/(ins "::mlir::RewriterBase &":$rewriter, "int" : $val),
+        /*methodBody=*/"",
+        /*defaultImpl=*/ [{
+            mlir::Operation* op = $_op;
+            return rewriter.create<LLVM::ConstantOp>(
+              op->getLoc(), rewriter.getIntegerType(32), val);
+        }]
+     >,
+     InterfaceMethod<
+         /*desc=*/[{ 
+            This function supplies the necessary arguments for passing PTX code,
+            following this order:
+             1) Adds results 
+             2) Adds operands 
+             3) Adds attributes             
+          }],
+         /*retType=*/"void",
+         /*methodName=*/"getAsmValues",
+         /*args=*/(ins "::mlir::RewriterBase &":$rewriter, 
+         "llvm::SmallVectorImpl<std::pair<mlir::Value, mlir::NVVM::PTXRegisterMod>>&" : $asmValues),
+         /*methodBody=*/"",
+         /*defaultImpl=*/ [{         
+           mlir::Operation* op = $_op;
+           
+           // Step 1. Add results
+           for (auto val : op->getResults()) 
+            asmValues.push_back({val, mlir::NVVM::PTXRegisterMod::Write});
+
+           // Step 2. Add operands
+           for (auto val : op->getOperands()) 
+            asmValues.push_back({val, mlir::NVVM::PTXRegisterMod::Read});
+           
+           // Step 3. Add attributes
+           for (auto attr : op->getAttrs()) {
+            if (auto intAttr = dyn_cast<mlir::IntegerAttr>(attr.getValue())) {
+             ::mlir::Value val = makeConstantI32(rewriter, intAttr.getInt());
+             asmValues.push_back({val, mlir::NVVM::PTXRegisterMod::Read});
+             }
+           }
+         }]
+       >
+  ];
+}
+
+#endif // BASICPTXBUILDER_OP_INTERFACE
\ No newline at end of file
diff --git a/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt b/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt
index aaf64e6332..64de028c7f 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt
@@ -46,14 +46,18 @@ mlir_tablegen(LLVMIntrinsicFromLLVMIRConversions.inc -gen-intr-from-llvmir-conve
 mlir_tablegen(LLVMConvertibleLLVMIRIntrinsics.inc -gen-convertible-llvmir-intrinsics)
 add_public_tablegen_target(MLIRLLVMIntrinsicConversionsIncGen)
 
+set(LLVM_TARGET_DEFINITIONS BasicPtxBuilderInterface.td)
+mlir_tablegen(BasicPtxBuilderInterface.h.inc -gen-op-interface-decls)
+mlir_tablegen(BasicPtxBuilderInterface.cpp.inc -gen-op-interface-defs)
+add_public_tablegen_target(MLIRBasicPtxBuilderInterfaceIncGen)
+add_dependencies(mlir-headers MLIRBasicPtxBuilderInterfaceIncGen)
+
 add_mlir_dialect(NVVMOps nvvm)
 add_mlir_doc(NVVMOps NVVMDialect Dialects/ -gen-dialect-doc -dialect=nvvm)
 set(LLVM_TARGET_DEFINITIONS NVVMOps.td)
 mlir_tablegen(NVVMConversions.inc -gen-llvmir-conversions)
 mlir_tablegen(NVVMOpsEnums.h.inc -gen-enum-decls)
 mlir_tablegen(NVVMOpsEnums.cpp.inc -gen-enum-defs)
-mlir_tablegen(NVVMOpsInterface.h.inc -gen-op-interface-decls)
-mlir_tablegen(NVVMOpsInterface.cpp.inc -gen-op-interface-defs)
 mlir_tablegen(NVVMOpsAttributes.h.inc -gen-attrdef-decls -attrdefs-dialect=nvvm)
 mlir_tablegen(NVVMOpsAttributes.cpp.inc -gen-attrdef-defs -attrdefs-dialect=nvvm)
 add_public_tablegen_target(MLIRNVVMConversionsIncGen)
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
index 8745d14c8d..2a572ab4de 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
@@ -1071,7 +1071,7 @@ def LLVM_AddressOfOp : LLVM_Op<"mlir.addressof",
       CArg<"ArrayRef<NamedAttribute>", "{}">:$attrs),
     [{
       build($_builder, $_state,
-            LLVM::LLVMPointerType::get(global.getType(), global.getAddrSpace()),
+            LLVM::LLVMPointerType::get($_builder.getContext(), global.getAddrSpace()),
             global.getSymName());
       $_state.addAttributes(attrs);
     }]>,
@@ -1079,7 +1079,7 @@ def LLVM_AddressOfOp : LLVM_Op<"mlir.addressof",
       CArg<"ArrayRef<NamedAttribute>", "{}">:$attrs),
     [{
       build($_builder, $_state,
-            LLVM::LLVMPointerType::get(func.getFunctionType()), func.getName());
+            LLVM::LLVMPointerType::get($_builder.getContext()), func.getName());
       $_state.addAttributes(attrs);
     }]>
   ];
diff --git a/mlir/include/mlir/Dialect/LLVMIR/NVVMDialect.h b/mlir/include/mlir/Dialect/LLVMIR/NVVMDialect.h
index 1644d00293..8ff8f850a9 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/NVVMDialect.h
+++ b/mlir/include/mlir/Dialect/LLVMIR/NVVMDialect.h
@@ -15,6 +15,7 @@
 #define MLIR_DIALECT_LLVMIR_NVVMDIALECT_H_
 
 #include "mlir/Bytecode/BytecodeOpInterface.h"
+#include "mlir/Dialect/LLVMIR/BasicPtxBuilderInterface.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/OpDefinition.h"
@@ -26,8 +27,6 @@
 namespace mlir {
 namespace NVVM {
 
-#include "mlir/Dialect/LLVMIR/NVVMOpsInterface.h.inc"
-
 /// NVVM memory space identifiers.
 enum NVVMMemorySpace {
   /// Global memory space identifier.
diff --git a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
index ecfc5dd2e8..9cda7862cc 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
@@ -17,6 +17,7 @@ include "mlir/IR/EnumAttr.td"
 include "mlir/Dialect/GPU/IR/CompilationAttrInterfaces.td"
 include "mlir/Dialect/LLVMIR/LLVMOpBase.td"
 include "mlir/Interfaces/SideEffectInterfaces.td"
+include "mlir/Dialect/LLVMIR/BasicPtxBuilderInterface.td"
 
 def LLVM_i8Ptr_global : LLVM_IntPtrBase<8, 1>;
 def LLVM_i8Ptr_shared : LLVM_IntPtrBase<8, 3>;
@@ -73,6 +74,12 @@ class NVVM_Op<string mnemonic, list<Trait> traits = []> :
   LLVM_OpBase<NVVM_Dialect, mnemonic, traits> {
 }
 
+/// Base class that defines BasicPtxBuilderOpInterface. 
+class NVVM_PTXBuilder_Op<string mnemonic, 
+  list<Trait> traits = [DeclareOpInterfaceMethods<BasicPtxBuilderOpInterface>]> :
+  LLVM_OpBase<NVVM_Dialect, mnemonic, traits> {
+}
+
 //===----------------------------------------------------------------------===//
 // NVVM attribute definitions
 //===----------------------------------------------------------------------===//
@@ -82,135 +89,6 @@ class NVVM_Attr<string attrName, string attrMnemonic, list<Trait> traits = []>
   let mnemonic = attrMnemonic;
 }
 
-//===----------------------------------------------------------------------===//
-// Basic PTX Builder Interface
-//===----------------------------------------------------------------------===//
-
-// https://docs.nvidia.com/cuda/inline-ptx-assembly/index.html#parameters
-def Read : I32EnumAttrCase<"Read", 0, "read">;
-def Write : I32EnumAttrCase<"Write", 2, "write">;
-def ReadWrite : I32EnumAttrCase<"ReadWrite", 1, "readwrite">;
-
-def PTXRegisterMod : I32EnumAttr<"PTXRegisterMod", 
-  "Register read/write modifier to build cosntraint string for PTX inline",
-  [Read, Write, ReadWrite]> {
-  let cppNamespace = "::mlir::NVVM";
-}
-
-
-def BasicPtxBuilderOpInterface : OpInterface<"BasicPtxBuilderInterface"> {
-  let description = [{
-    Interface to generate inline assembly with PTX for basic operations. 
-
-    Interface is used in `convert-nvvm-to-llvm` pass that lowers Ops supports 
-    this interface to inline assembly Op. Interface has several methods and 
-    they are used for this lowering. 
-
-    `getPtx` method returns PTX code. 
-
-    `hasSideEffect` is used to set whether the op has any side effect on the 
-    memory.
-
-    `hasIntrinsic` returns whether the operation has intrinsic support in LLVM. 
-    This is useful for the Ops that don't have intrinsic support for each case.
-
-    `getAsmValues` returns arguments to pass PTX code. The order of arguments 
-    is started from the results and they are used as write, followed by the 
-    operands and attributes.
-
-    Example:
-    If we have following Op definition that returns PTX code by `getPtx`. 
-    
-    ```tablegen
-      def NVVM_MyOp : NVVM_Op<"myop",
-          [DeclareOpInterfaceMethods<BasicPtxBuilderOpInterface>]>,  
-        Results<(outs LLVM_Type:$res)>,
-        Arguments<(ins LLVM_i64ptr_any:$op1, I32:$op2)> {
-        ...
-        let extraClassDefinition = [{
-          std::string $cppClass::getPtx() { 
-            return std::string("my.ptx.code %0, %1, %2;"); 
-          }
-      } ];
-    ```
-
-    The NVVM Op will look like below:
-    ```mlir
-      %0 = my.ptx.code %1, %2 : !llvm.ptr, i32 -> i32
-    ```
-
-    The `convert-nvvm-to-llvm` Pass generates the PTX code below. The order of 
-    arguments are kept the same. The read and write modifiers are set based on
-    the input and result types.
-    ```mlir
-      %0 = llvm.inline_asm has_side_effects asm_dialect = att "my.ptx.code %0, %1, %2;", "=r,l,r" %arg0, %arg1 : (!llvm.ptr, i32) -> i32
-    ```
-
-  }];
-  let methods = [
-    InterfaceMethod<
-        /*desc=*/[{
-          Returns whether the operation has intrinsic support in LLVM.
-        }],
-        /*retType=*/"bool",
-        /*methodName=*/"hasIntrinsic",
-        /*args=*/(ins),
-        /*methodBody=*/"",
-        /*defaultImplementation=*/"return false;"
-      >,
-    InterfaceMethod<
-        /*desc=*/[{ Return whether the operation has memory side effects. }],
-        /*retType=*/"bool",
-        /*methodName=*/"hasSideEffect",
-        /*args=*/(ins),
-        /*methodBody=*/"",
-        /*defaultImplementation=*/"return true;"
-      >,
-    InterfaceMethod<
-        /*desc=*/[{ Returns PTX code. }],
-        /*retType=*/"std::string",
-        /*methodName=*/"getPtx"
-      >,
-    InterfaceMethod<
-        /*desc=*/[{Generate constant value.}],
-        /*retType=*/"::mlir::Value",
-        /*methodName=*/"makeConstantI32",
-        /*args=*/(ins "::mlir::RewriterBase &":$rewriter, "int" : $val),
-        /*methodBody=*/"",
-        /*defaultImpl=*/ [{
-            mlir::Operation* op = $_op;
-            return rewriter.create<LLVM::ConstantOp>(
-              op->getLoc(), rewriter.getIntegerType(32), val);
-        }]
-     >,
-     InterfaceMethod<
-         /*desc=*/[{ 
-            Returns arguments to pass PTX code.
-            The order of arguments is started from the results and they are 
-            used as write, followed by the operands and attributes.
-          }],
-         /*retType=*/"void",
-         /*methodName=*/"getAsmValues",
-         /*args=*/(ins "::mlir::RewriterBase &":$rewriter, 
-         "llvm::SmallVectorImpl<std::pair<mlir::Value, mlir::NVVM::PTXRegisterMod>>&" : $asmValues),
-         /*methodBody=*/"",
-         /*defaultImpl=*/ [{         
-           mlir::Operation* op = $_op;
-           for (auto val : op->getResults()) 
-            asmValues.push_back({val, mlir::NVVM::PTXRegisterMod::Write});
-           for (auto val : op->getOperands()) 
-            asmValues.push_back({val, mlir::NVVM::PTXRegisterMod::Read});
-           for (auto attr : op->getAttrs()) {
-            if (auto intAttr = dyn_cast<mlir::IntegerAttr>(attr.getValue())) {
-             Value val = makeConstantI32(rewriter, intAttr.getInt());
-             asmValues.push_back({val ,mlir::NVVM::PTXRegisterMod::Read});
-             }
-           }
-         }]
-       >
-  ];
-}
-
 //===----------------------------------------------------------------------===//
 // NVVM intrinsic operations
 //===----------------------------------------------------------------------===//
@@ -334,21 +212,31 @@ def NVVM_ReduxOp :
 //===----------------------------------------------------------------------===//
 
 /// mbarrier.init instruction with generic pointer type
-def NVVM_MBarrierInitOp : NVVM_Op<"mbarrier.init">,
-  Arguments<(ins LLVM_i64ptr_any:$addr, I32:$count)> {
+def NVVM_MBarrierInitOp : NVVM_PTXBuilder_Op<"mbarrier.init">,
+  Arguments<(ins LLVM_i64ptr_any:$addr, I32:$count, PtxPredicate:$predicate)> {
   string llvmBuilder = [{
       createIntrinsicCall(builder, llvm::Intrinsic::nvvm_mbarrier_init, {$addr, $count});
   }];
-  let assemblyFormat = "$addr `,` $count attr-dict `:` type(operands)";
+  let assemblyFormat = "$addr `,` $count (`,` `predicate` `=` $predicate^)? attr-dict `:` type(operands)";
+  let extraClassDeclaration = [{
+    bool hasIntrinsic() { if(getPredicate()) return false; return true; }
+  }];
+  let extraClassDefinition = [{
+    std::string $cppClass::getPtx() { return std::string("mbarrier.init.b64 [%0], %1;"); }
+  }];
 }
 
 /// mbarrier.init instruction with shared pointer type
-def NVVM_MBarrierInitSharedOp : NVVM_Op<"mbarrier.init.shared">,
-  Arguments<(ins LLVM_i64ptr_shared:$addr, I32:$count)> {
+def NVVM_MBarrierInitSharedOp : NVVM_PTXBuilder_Op<"mbarrier.init.shared">,
+  Arguments<(ins LLVM_i64ptr_shared:$addr, I32:$count, PtxPredicate:$predicate)> {
   string llvmBuilder = [{
       createIntrinsicCall(builder, llvm::Intrinsic::nvvm_mbarrier_init_shared, {$addr, $count});
   }];
-  let assemblyFormat = "$addr `,` $count attr-dict `:` type(operands)";
+  let assemblyFormat = "$addr `,` $count (`,` `predicate` `=` $predicate^)? attr-dict `:` type(operands)";
+  let extraClassDeclaration = "bool hasIntrinsic() { return !getPredicate(); }";
+  let extraClassDefinition = [{
+    std::string $cppClass::getPtx() { return std::string("mbarrier.init.shared.b64 [%0], %1;"); }
+  }];
 }
 
 def NVVM_MBarrierInvalOp : NVVM_Op<"mbarrier.inval">,
@@ -403,26 +291,23 @@ def NVVM_MBarrierArriveNocompleteSharedOp : NVVM_Op<"mbarrier.arrive.nocomplete.
   let assemblyFormat = "$addr `,` $count attr-dict `:` type(operands) `->` type($res)";
 }
 
-def NVVM_MBarrierArriveExpectTxOp : NVVM_Op<"mbarrier.arrive.expect_tx",
-                    [DeclareOpInterfaceMethods<BasicPtxBuilderOpInterface>]>,  
-  Arguments<(ins LLVM_i64ptr_any:$addr, I32:$txcount)> {
-  let assemblyFormat = "$addr `,` $txcount attr-dict `:` type(operands)";
+def NVVM_MBarrierArriveExpectTxOp : NVVM_PTXBuilder_Op<"mbarrier.arrive.expect_tx">,  
+  Arguments<(ins LLVM_i64ptr_any:$addr, I32:$txcount, PtxPredicate:$predicate)> {
+  let assemblyFormat = "$addr `,` $txcount (`,` `predicate` `=` $predicate^)? attr-dict `:` type(operands)";
   let extraClassDefinition = [{
     std::string $cppClass::getPtx() { return std::string("mbarrier.arrive.expect_tx.b64 _, [%0], %1;"); }
   }];
 }
 
-def NVVM_MBarrierArriveExpectTxSharedOp : NVVM_Op<"mbarrier.arrive.expect_tx.shared", 
-                    [DeclareOpInterfaceMethods<BasicPtxBuilderOpInterface>]>,  
-  Arguments<(ins LLVM_i64ptr_shared:$addr, I32:$txcount)> {    
-  let assemblyFormat = "$addr `,` $txcount attr-dict `:` type(operands)";
+def NVVM_MBarrierArriveExpectTxSharedOp : NVVM_PTXBuilder_Op<"mbarrier.arrive.expect_tx.shared">,  
+  Arguments<(ins LLVM_i64ptr_shared:$addr, I32:$txcount, PtxPredicate:$predicate)> {    
+  let assemblyFormat = "$addr `,` $txcount (`,` `predicate` `=` $predicate^)? attr-dict `:` type(operands)";
   let extraClassDefinition = [{
     std::string $cppClass::getPtx() { return std::string("mbarrier.arrive.expect_tx.shared.b64 _, [%0], %1;"); }
   }];
 }
 
-def NVVM_MBarrierTryWaitParityOp : NVVM_Op<"mbarrier.try_wait.parity", 
-                    [DeclareOpInterfaceMethods<BasicPtxBuilderOpInterface>]>,  
+def NVVM_MBarrierTryWaitParityOp : NVVM_PTXBuilder_Op<"mbarrier.try_wait.parity">,  
   Arguments<(ins LLVM_i64ptr_any:$addr, I32:$phase, I32:$ticks)> {  
   let assemblyFormat = "$addr `,` $phase `,` $ticks attr-dict `:` type(operands)";
   let extraClassDefinition = [{
@@ -441,8 +326,7 @@ def NVVM_MBarrierTryWaitParityOp : NVVM_Op<"mbarrier.try_wait.parity",
   }];
 }
 
-def NVVM_MBarrierTryWaitParitySharedOp : NVVM_Op<"mbarrier.try_wait.parity.shared", 
-                    [DeclareOpInterfaceMethods<BasicPtxBuilderOpInterface>]>,  
+def NVVM_MBarrierTryWaitParitySharedOp : NVVM_PTXBuilder_Op<"mbarrier.try_wait.parity.shared">,  
   Arguments<(ins LLVM_i64ptr_shared:$addr, I32:$phase, I32:$ticks)> {  
   let assemblyFormat = "$addr `,` $phase `,` $ticks attr-dict `:` type(operands)";
   let extraClassDefinition = [{
@@ -616,7 +500,7 @@ def LoadCacheModifierKind : I32EnumAttr<"LoadCacheModifierKind",
 
 def LoadCacheModifierAttr : EnumAttr<NVVM_Dialect, LoadCacheModifierKind, "load_cache_modifier">;
 
-def NVVM_CpAsyncOp : NVVM_Op<"cp.async.shared.global", [DeclareOpInterfaceMethods<BasicPtxBuilderOpInterface>]>,
+def NVVM_CpAsyncOp : NVVM_PTXBuilder_Op<"cp.async.shared.global">,
   Arguments<(ins LLVM_i8Ptr_shared:$dst,
                  LLVM_i8Ptr_global:$src,
                  I32Attr:$size,
@@ -1302,6 +1186,35 @@ def NVVM_WMMAMmaOp : NVVM_Op<"wmma.mma">,
   let hasVerifier = 1;
 }
 
+def NVVM_StMatrixOp: NVVM_PTXBuilder_Op<"stmatrix">, 
+  Arguments<(ins LLVM_i8Ptr_shared:$ptr, 
+                 Variadic<I32>:$sources, 
+                 MMALayoutAttr:$layout)> {
+  let summary = "cooperative matrix store";
+  let description = [{
+    Collectively store one or more matrices across all threads in a warp to the
+    location indicated by the address operand $ptr in shared memory.
+    [For more information, see PTX ISA]
+    (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#warp-level-matrix-store-instruction-stmatrix)
+  }];
+  
+  let assemblyFormat = "$ptr `,` $sources attr-dict `:` type(operands)";
+  let extraClassDefinition = [{
+    std::string $cppClass::getPtx() {
+      int d = getSources().size();
+      std::string ptx = "stmatrix.sync.aligned";
+      ptx += ".x" + std::to_string(d);
+      if (getLayout() == NVVM::MMALayout::col)
+        ptx += ".trans";
+      if(d == 1) ptx += ".m8n8.shared.b16 [%0], {%1}";
+      if(d == 2) ptx += ".m8n8.shared.b16 [%0], {%1, %2}";
+      if(d == 4) ptx += ".m8n8.shared.b16 [%0], {%1, %2, %3, %4};";
+      return ptx;
+    }
+  }];
+  let hasVerifier = 1;
+}
+
 def NVVM_LdMatrixOp: NVVM_Op<"ldmatrix">,
   Results<(outs AnyType:$res)>,
   Arguments<(ins LLVM_AnyPointer: $ptr, I32Attr:$num, MMALayoutAttr:$layout)> {
@@ -1487,12 +1400,24 @@ def NVVM_MmaOp : NVVM_Op<"mma.sync", [AttrSizedOperandSegments]> {
 // NVVM TMA Ops
 //===----------------------------------------------------------------------===//
 
-def NVVM_CpAsyncBulkTensorGlobalToSharedClusterOp : NVVM_Op<"cp.async.bulk.tensor.shared.cluster.global", [DeclareOpInterfaceMethods<BasicPtxBuilderOpInterface>]>,
+def NVVM_CpAsyncBulkTensorGlobalToSharedClusterOp : 
+  NVVM_Op<"cp.async.bulk.tensor.shared.cluster.global", 
+  [DeclareOpInterfaceMethods<BasicPtxBuilderOpInterface>, 
+  AttrSizedOperandSegments]>,
   Arguments<(ins  LLVM_i64ptr_shared:$dstMem,
                   LLVM_i64ptr_any:$tmaDescriptor,
                   LLVM_i64ptr_shared:$mbar,
-                  Variadic<I32>:$coordinates)> {
-  let assemblyFormat = "$dstMem `,` $tmaDescriptor `,` $mbar `,` `box` `[`$coordinates `]` attr-dict  `:` type(operands)";
+                  Variadic<I32>:$coordinates,
+                  PtxPredicate:$predicate)> {
+  let assemblyFormat = [{ 
+    $dstMem `,` 
+    $tmaDescriptor `,` 
+    $mbar `,` 
+    `box` `[`$coordinates `]` 
+    (`,` `predicate` `=` $predicate^)? 
+    attr-dict  `:` type(operands)
+  }];
+
   let extraClassDefinition = [{
     std::string $cppClass::getPtx() {
       int dim = getCoordinates().size();
@@ -1510,11 +1435,21 @@ def NVVM_CpAsyncBulkTensorGlobalToSharedClusterOp : NVVM_Op<"cp.async.bulk.tenso
   let hasVerifier = 1;
 }
 
-def NVVM_CpAsyncBulkTensorSharedCTAToGlobalOp : NVVM_Op<"cp.async.bulk.tensor.global.shared.cta", [DeclareOpInterfaceMethods<BasicPtxBuilderOpInterface>]>,
+def NVVM_CpAsyncBulkTensorSharedCTAToGlobalOp : 
+  NVVM_Op<"cp.async.bulk.tensor.global.shared.cta", 
+  [DeclareOpInterfaceMethods<BasicPtxBuilderOpInterface>, 
+  AttrSizedOperandSegments]>,
   Arguments<(ins  LLVM_i64ptr_any:$tmaDescriptor,
                   LLVM_i64ptr_shared:$srcMem,
-                  Variadic<I32>:$coordinates)> {
-  let assemblyFormat = "$tmaDescriptor `,` $srcMem `,` `box` `[`$coordinates `]` attr-dict  `:` type(operands)";
+                  Variadic<I32>:$coordinates,
+                  PtxPredicate:$predicate)> {
+  let assemblyFormat = [{ 
+    $tmaDescriptor `,` 
+    $srcMem `,` 
+    `box` `[`$coordinates `]` 
+    (`,` `predicate` `=` $predicate^)?  
+    attr-dict  `:` type(operands)
+  }];
   let extraClassDefinition = [{
     std::string $cppClass::getPtx() {
       int dim = getCoordinates().size();
@@ -1532,12 +1467,22 @@ def NVVM_CpAsyncBulkTensorSharedCTAToGlobalOp : NVVM_Op<"cp.async.bulk.tensor.gl
   let hasVerifier = 1;
 }
 
+def NVVM_PrefetchTensorMapOp : NVVM_Op<"prefetch.tensormap",
+                    [DeclareOpInterfaceMethods<BasicPtxBuilderOpInterface>]>,
+  Arguments<(ins LLVM_i64ptr_any:$tmaDescriptor, PtxPredicate:$predicate)> {
+  let assemblyFormat = "$tmaDescriptor (`,` `predicate` `=` $predicate^)? attr-dict `:` type(operands)";
+  let extraClassDefinition = [{
+    std::string $cppClass::getPtx() { 
+      return std::string("prefetch.tensormap [%0];");
+    }
+  }];
+}
+
 //===----------------------------------------------------------------------===//
 // NVVM Wgmma Ops
 //===----------------------------------------------------------------------===//
 
-def NVVM_WgmmaFenceAlignedOp : NVVM_Op<"wgmma.fence.aligned", 
-                    [DeclareOpInterfaceMethods<BasicPtxBuilderOpInterface>]> {
+def NVVM_WgmmaFenceAlignedOp : NVVM_PTXBuilder_Op<"wgmma.fence.aligned"> {
   let arguments = (ins);
   let description = [{
     Enforce an ordering of register accesses between warpgroup level matrix 
@@ -1551,8 +1496,7 @@ def NVVM_WgmmaFenceAlignedOp : NVVM_Op<"wgmma.fence.aligned",
   }];
 }
 
-def NVVM_WgmmaGroupSyncAlignedOp : NVVM_Op<"wgmma.commit.group.sync.aligned", 
-                    [DeclareOpInterfaceMethods<BasicPtxBuilderOpInterface>]>,
+def NVVM_WgmmaGroupSyncAlignedOp : NVVM_PTXBuilder_Op<"wgmma.commit.group.sync.aligned">,
   Arguments<(ins )> {
   let assemblyFormat = "attr-dict";
   let description = [{
@@ -1565,8 +1509,7 @@ def NVVM_WgmmaGroupSyncAlignedOp : NVVM_Op<"wgmma.commit.group.sync.aligned",
   }];
 }
 
-def NVVM_WgmmaWaitGroupSyncOp : NVVM_Op<"wgmma.wait.group.sync.aligned", 
-                    [DeclareOpInterfaceMethods<BasicPtxBuilderOpInterface>]>{
+def NVVM_WgmmaWaitGroupSyncOp : NVVM_PTXBuilder_Op<"wgmma.wait.group.sync.aligned">{
   let arguments = (ins I32Attr:$group);
   let assemblyFormat = "attr-dict $group";
   let description = [{
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgInterfaces.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgInterfaces.td
index 9ca029b489..44e82f452b 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgInterfaces.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgInterfaces.td
@@ -86,6 +86,39 @@ def LinalgContractionOpInterface : OpInterface<"ContractionOpInterface"> {
     /*methodBody=*/[{
         return mlir::isRowMajorBatchMatmul($_op.getIndexingMaps());
     }]>,
+    InterfaceMethod<
+    /*desc=*/[{
+      Returns whether the given op has indexing maps that correspond to a
+      vector-matrix multiplication.
+    }],
+    /*retTy=*/"bool",
+    /*methodName=*/"isVecmat",
+    /*args=*/(ins),
+    /*methodBody=*/[{
+        return mlir::isVecmat($_op.getIndexingMaps());
+    }]>,
+    InterfaceMethod<
+    /*desc=*/[{
+      Returns whether the given op has indexing maps that correspond to a
+      matrix-vector multiplication.
+    }],
+    /*retTy=*/"bool",
+    /*methodName=*/"isMatvec",
+    /*args=*/(ins),
+    /*methodBody=*/[{
+        return mlir::isMatvec($_op.getIndexingMaps());
+    }]>,
+    InterfaceMethod<
+    /*desc=*/[{
+      Returns whether the given op has indexing maps that correspond to a
+      batched matrix-vector multiplication.
+    }],
+    /*retTy=*/"bool",
+    /*methodName=*/"isBatchMatvec",
+    /*args=*/(ins),
+    /*methodBody=*/[{
+        return mlir::isBatchMatvec($_op.getIndexingMaps());
+    }]>,
   ];
 }
 
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
index 21a5e5cc47..751edd0228 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
@@ -442,10 +442,16 @@ def TransposeOp : LinalgStructuredBase_Op<"transpose", [
     // Implement functions necessary for DestinationStyleOpInterface.
     MutableOperandRange getDpsInitsMutable() { return getInitMutable(); }
 
+    static void regionBuilder(mlir::ImplicitLocOpBuilder &b, mlir::Block &block,
+        mlir::ArrayRef<mlir::NamedAttribute>) {
+      OpBuilder::InsertionGuard guard(b);
+      b.create<linalg::YieldOp>(b.getLoc(), block.getArgument(0));
+    }
+
     static std::function<void(mlir::ImplicitLocOpBuilder &, mlir::Block &,
         mlir::ArrayRef<mlir::NamedAttribute>)>
       getRegionBuilder() {
-      return nullptr;
+      return regionBuilder;
     }
 
     static void createRegion(::mlir::OpBuilder &opBuilder,
@@ -510,10 +516,16 @@ def BroadcastOp : LinalgStructuredBase_Op<"broadcast", [
     // Implement functions necessary for DestinationStyleOpInterface.
     MutableOperandRange getDpsInitsMutable() { return getInitMutable(); }
 
+    static void regionBuilder(mlir::ImplicitLocOpBuilder &b, mlir::Block &block,
+        mlir::ArrayRef<mlir::NamedAttribute>) {
+      OpBuilder::InsertionGuard guard(b);
+      b.create<linalg::YieldOp>(b.getLoc(), block.getArgument(0));
+    }
+
     static std::function<void(mlir::ImplicitLocOpBuilder &, mlir::Block &,
         mlir::ArrayRef<mlir::NamedAttribute>)>
       getRegionBuilder() {
-      return nullptr;
+      return regionBuilder;
     }
   }];
 
diff --git a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
index 07a192f7b8..fbe2923c71 100644
--- a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
+++ b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
@@ -1047,16 +1047,18 @@ bool isDimSequencePreserved(AffineMap map, ReassociationIndicesRef dimSequence);
 bool areDimSequencesPreserved(ArrayRef<AffineMap> maps,
                               ArrayRef<ReassociationIndices> dimSequences);
 
-/// Collapses dimensions of linalg.generic operation. A precondition to
-/// calling this method is that for each list in `foldedIterationDim`, the
+/// Collapses dimensions of linalg.generic/linalg.copy operation. A precondition
+/// to calling this method is that for each list in `foldedIterationDim`, the
 /// sequence of dimensions is contiguous in domains of all `indexing_maps` of
-/// the `genericOp`. This can be checked using `areDimSequencePreserved` method.
+/// the `linalgOp`. This can be checked using `areDimSequencePreserved` method.
 /// When valid, the method also collapses the operands of the op. Returns
-/// replacement values of the results of the original `genericOp` by inserting
+/// replacement values of the results of the original `linalgOp` by inserting
 /// reshapes to get back values of compatible types.
-FailureOr<SmallVector<Value>> collapseGenericOpIterationDims(
-    GenericOp genericOp, ArrayRef<ReassociationIndices> foldedIterationDims,
-    RewriterBase &rewriter);
+template <typename LinalgType>
+FailureOr<SmallVector<Value>>
+collapseOpIterationDims(LinalgType op,
+                        ArrayRef<ReassociationIndices> foldedIterationDims,
+                        RewriterBase &rewriter);
 
 struct LowerPackResult {
   tensor::PadOp padOp;
@@ -1175,6 +1177,14 @@ FailureOr<Operation *> rewriteInDestinationPassingStyle(RewriterBase &rewriter,
 FailureOr<std::pair<Operation *, Operation *>>
 rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcHwcfOp convOp);
 
+/// Same as the above but for Fhwc channel orderings in the filter. In this case
+/// the matrix multiplication is actually a row-wise dot-product rather than a
+/// row-column dot-product. This is to avoid transposing the filter matrix which
+/// would be required for a regular matrix multiplication to produce the correct
+/// output dimensions.
+FailureOr<std::pair<Operation *, Operation *>>
+rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp);
+
 /// Similar to rewriteInIm2Col with linalg::Conv2DNhwcHwcfOp except there is no
 /// reduction among the input channels so each convolution can be a
 /// matrix-vector product and by transposing both input filter so channels are
@@ -1507,7 +1517,7 @@ void populateEraseUnnecessaryInputsPatterns(RewritePatternSet &patterns);
 /// to return an array of `ReassociationIndices` representing dimensions that
 /// should be merged.
 using GetCollapsableDimensionsFn =
-    std::function<SmallVector<ReassociationIndices>(linalg::GenericOp)>;
+    std::function<SmallVector<ReassociationIndices>(linalg::LinalgOp)>;
 
 /// Pattern to collapse dimensions in a linalg.generic op. This will collapse
 /// tensor operands when needed and expand back the result tensors.
diff --git a/mlir/include/mlir/Dialect/Mesh/IR/MeshBase.td b/mlir/include/mlir/Dialect/Mesh/IR/MeshBase.td
index d761743a82..39d24595ec 100644
--- a/mlir/include/mlir/Dialect/Mesh/IR/MeshBase.td
+++ b/mlir/include/mlir/Dialect/Mesh/IR/MeshBase.td
@@ -58,8 +58,8 @@ def MeshSharding : AttrDef<Mesh_Dialect, "MeshSharding"> {
 
   let parameters = (ins
     AttrParameter<"::mlir::SymbolRefAttr", "cluster placed">:$cluster,
-    ArrayRefParameter<"::mlir::DenseI8ArrayAttr">:$split_axes,
-    OptionalArrayRefParameter<"int8_t">:$partial_axes,
+    ArrayRefParameter<"::mlir::DenseI32ArrayAttr">:$split_axes,
+    OptionalArrayRefParameter<"int32_t">:$partial_axes,
     OptionalParameter<"::mlir::mesh::Partial">:$partial_type
   );
 
diff --git a/mlir/include/mlir/Dialect/Mesh/IR/MeshOps.td b/mlir/include/mlir/Dialect/Mesh/IR/MeshOps.td
index 8ca4b66531..a8aa0a694b 100644
--- a/mlir/include/mlir/Dialect/Mesh/IR/MeshOps.td
+++ b/mlir/include/mlir/Dialect/Mesh/IR/MeshOps.td
@@ -70,7 +70,7 @@ def Mesh_ClusterOp : Mesh_Op<"cluster", [Symbol]> {
   }];
   let arguments = (ins
     SymbolNameAttr:$sym_name,
-    I8Attr:$rank,
+    I64Attr:$rank,
     DefaultValuedAttr<DenseI64ArrayAttr, "{}">:$dim_sizes
   );
   let assemblyFormat = [{
diff --git a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td
index 57cd1a3806..440f7d0380 100644
--- a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td
+++ b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td
@@ -522,8 +522,8 @@ def NVGPU_MBarrierInitOp : NVGPU_Op<"mbarrier.init", []> {
       nvgpu.mbarrier.init %barrier, %num_threads : !nvgpu.mbarrier.barrier<memorySpace = #gpu.address_space<workgroup>>
     ```
   }];
-  let arguments = (ins NVGPU_MBarrierGroup:$barriers, Index:$count, Index:$mbarId);
-  let assemblyFormat = "$barriers `[` $mbarId `]` `,` $count attr-dict `:` type($barriers)";
+  let arguments = (ins NVGPU_MBarrierGroup:$barriers, Index:$count, Index:$mbarId, Optional<I1>:$predicate);
+  let assemblyFormat = "$barriers `[` $mbarId `]` `,` $count (`,` `predicate` `=` $predicate^)? attr-dict `:` type($barriers)";
 }
 
 def NVGPU_MBarrierTestWaitOp : NVGPU_Op<"mbarrier.test.wait", []> {
@@ -597,8 +597,8 @@ def NVGPU_MBarrierArriveExpectTxOp : NVGPU_Op<"mbarrier.arrive.expect_tx", []> {
       nvgpu.mbarrier.arrive.expect_tx %barrier, %ic0 : !nvgpu.mbarrier.barrier<memorySpace = #gpu.address_space<workgroup>>
     ```
   }];
-  let arguments = (ins NVGPU_MBarrierGroup:$barriers, Index:$txcount, Index:$mbarId);
-  let assemblyFormat = "$barriers `[` $mbarId `]` `,` $txcount  attr-dict `:` type($barriers)";
+  let arguments = (ins NVGPU_MBarrierGroup:$barriers, Index:$txcount, Index:$mbarId, Optional<I1>:$predicate);
+  let assemblyFormat = "$barriers `[` $mbarId `]` `,` $txcount  (`,` `predicate` `=` $predicate^)? attr-dict `:` type($barriers)";
 }
 
 def NVGPU_MBarrierTryWaitParityOp : NVGPU_Op<"mbarrier.try_wait.parity", []> {
@@ -619,7 +619,19 @@ def NVGPU_MBarrierTryWaitParityOp : NVGPU_Op<"mbarrier.try_wait.parity", []> {
   let assemblyFormat = "$barriers `[` $mbarId `]` `,` $phase `,` $ticks attr-dict `:` type($barriers)";  
 }
 
-def NVGPU_TmaAsyncLoadOp : NVGPU_Op<"tma.async.load", []> {
+def NVGPU_TmaPrefetchOp : NVGPU_Op<"tma.prefetch.descriptor", []> {
+  let summary = "Prefetch given `nvgpu.tensormap.descriptor` ";
+  let description = [{
+    The Op brings the cache line containing the given `$tmaDescriptor` for 
+    subsequent use by the `tma.async.load` instruction.
+  }];
+  let arguments = (ins NVGPU_TensorMapDescriptor:$tensorMapDescriptor, Optional<I1>:$predicate);
+  let assemblyFormat = [{
+    $tensorMapDescriptor (`,` `predicate` `=` $predicate^)? attr-dict `:` type($tensorMapDescriptor)
+  }];
+}
+
+def NVGPU_TmaAsyncLoadOp : NVGPU_Op<"tma.async.load", [AttrSizedOperandSegments]> {
   let summary = "TMA asynchronous load";
   let description = [{
     The Op loads a tile memory region from global memory to shared memory by 
@@ -634,10 +646,14 @@ def NVGPU_TmaAsyncLoadOp : NVGPU_Op<"tma.async.load", []> {
                        NVGPU_MBarrierGroup:$barriers,
                        NVGPU_TensorMapDescriptor:$tensorMapDescriptor,
                        Variadic<Index>:$coordinates, 
-                       Index:$mbarId);
+                       Index:$mbarId,
+                       Optional<I1>:$predicate);
   let assemblyFormat = [{
-    $tensorMapDescriptor `[` $coordinates `]` `,` $barriers `[` $mbarId `]` `to` $dst
-      attr-dict `:` type($tensorMapDescriptor) `,` type($barriers) `->` type($dst)
+    $tensorMapDescriptor `[` $coordinates `]` `,` $barriers `[` $mbarId `]` 
+      `to` $dst
+      (`,` `predicate` `=` $predicate^)? 
+      attr-dict `:` type($tensorMapDescriptor) `,` type($barriers) 
+      `->` type($dst)
   }];
   let hasVerifier = 1;
 
@@ -719,8 +735,8 @@ def NVGPU_WarpgroupMmaOp : NVGPU_Op<"warpgroup.mma"> {
                        DefaultValuedOptionalAttr<I32Attr, "1">:$waitGroup,
                        OptionalAttr<UnitAttr>:$transposeA,
                        OptionalAttr<UnitAttr>:$transposeB,
-                       Variadic<NVGPU_WarpgroupAccumulator>:$matrixC);
-  let results = (outs Variadic<NVGPU_WarpgroupAccumulator>:$matrixD);
+                       NVGPU_WarpgroupAccumulator:$matrixC);
+  let results = (outs NVGPU_WarpgroupAccumulator:$matrixD);
   let assemblyFormat = [{    
     $descriptorA`,` $descriptorB`,` $matrixC attr-dict
     `:` type($descriptorA) `,` type($descriptorB) `,` type($matrixC) `->` type($matrixD)
@@ -739,12 +755,24 @@ def NVGPU_WarpgroupMmaStoreOp : NVGPU_Op<"warpgroup.mma.store"> {
     Note that, the op must be run with warp group.
   }];
 
-  let arguments = (ins Variadic<NVGPU_WarpgroupAccumulator>:$matrixD,
+  let arguments = (ins NVGPU_WarpgroupAccumulator:$matrixD,
                        Arg<AnyMemRef, "", [MemWrite]>:$dstMemref);
   
   let assemblyFormat = [{
-    `[` $matrixD `]` `,` $dstMemref attr-dict `:` type($matrixD) `to` type($dstMemref)
+    $matrixD `,` $dstMemref attr-dict `:` type($matrixD) `to` type($dstMemref)
+  }];
+  let hasVerifier = 1;
+}
+
+def NVGPU_WarpgroupMmaInitAccumulatorOp : NVGPU_Op<"warpgroup.mma.init.accumulator"> {  
+  let summary = "Initializes the accumulator matrix";
+
+  let description = [{
+    This Op generates and initializes the accumulator matrix for 
+    `nvgpu.warpgroup.mma` op to perform matrix-multiply-and-accumulate.
   }];
+  let results = (outs NVGPU_WarpgroupAccumulator:$matrixC);
+  let assemblyFormat = "attr-dict `->` type($matrixC)";
   let hasVerifier = 1;
 }
 
diff --git a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUDialect.h b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUDialect.h
index 96af26842d..e6bba7e608 100644
--- a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUDialect.h
+++ b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPUDialect.h
@@ -23,6 +23,9 @@
 
 constexpr int kWarpSize = 32;
 
+/// M size of wgmma.mma_async instruction
+constexpr int kWgmmaSizeM = 64;
+
 #define GET_ATTRDEF_CLASSES
 #include "mlir/Dialect/NVGPU/IR/NVGPUAttrDefs.h.inc"
 
diff --git a/mlir/include/mlir/Dialect/SCF/IR/SCFOps.td b/mlir/include/mlir/Dialect/SCF/IR/SCFOps.td
index e1a604a887..6ac0912f6f 100644
--- a/mlir/include/mlir/Dialect/SCF/IR/SCFOps.td
+++ b/mlir/include/mlir/Dialect/SCF/IR/SCFOps.td
@@ -122,8 +122,8 @@ def ExecuteRegionOp : SCF_Op<"execute_region", [
 def ForOp : SCF_Op<"for",
       [AutomaticAllocationScope, DeclareOpInterfaceMethods<LoopLikeOpInterface,
        ["getInits", "getSingleInductionVar", "getSingleLowerBound",
-        "getSingleStep", "getSingleUpperBound", "promoteIfSingleIteration",
-        "replaceWithAdditionalYields"]>,
+        "getSingleStep", "getSingleUpperBound", "getYieldedValues",
+        "promoteIfSingleIteration", "replaceWithAdditionalYields"]>,
        AllTypesMatch<["lowerBound", "upperBound", "step"]>,
        ConditionallySpeculatable,
        DeclareOpInterfaceMethods<RegionBranchOpInterface,
@@ -243,9 +243,11 @@ def ForOp : SCF_Op<"for",
         function_ref<void(OpBuilder &, Location, Value, ValueRange)>;
 
     Value getInductionVar() { return getBody()->getArgument(0); }
+
     Block::BlockArgListType getRegionIterArgs() {
       return getBody()->getArguments().drop_front(getNumInductionVars());
     }
+
     /// Return the `index`-th region iteration argument.
     BlockArgument getRegionIterArg(unsigned index) {
       assert(index < getNumRegionIterArgs() &&
@@ -331,7 +333,8 @@ def ForallOp : SCF_Op<"forall", [
        AttrSizedOperandSegments,
        AutomaticAllocationScope,
        DeclareOpInterfaceMethods<LoopLikeOpInterface,
-          ["promoteIfSingleIteration"]>,
+          ["promoteIfSingleIteration", "getSingleInductionVar",
+          "getSingleLowerBound", "getSingleUpperBound", "getSingleStep"]>,
        RecursiveMemoryEffects,
        SingleBlockImplicitTerminator<"scf::InParallelOp">,
        DeclareOpInterfaceMethods<RegionBranchOpInterface>,
@@ -788,7 +791,8 @@ def IfOp : SCF_Op<"if", [DeclareOpInterfaceMethods<RegionBranchOpInterface, [
 def ParallelOp : SCF_Op<"parallel",
     [AutomaticAllocationScope,
      AttrSizedOperandSegments,
-     DeclareOpInterfaceMethods<LoopLikeOpInterface>,
+     DeclareOpInterfaceMethods<LoopLikeOpInterface, ["getSingleInductionVar",
+          "getSingleLowerBound", "getSingleUpperBound", "getSingleStep"]>,
      RecursiveMemoryEffects,
      DeclareOpInterfaceMethods<RegionBranchOpInterface>,
      SingleBlockImplicitTerminator<"scf::YieldOp">]> {
@@ -1086,6 +1090,11 @@ def WhileOp : SCF_Op<"while",
 
     ConditionOp getConditionOp();
     YieldOp getYieldOp();
+
+    /// Return the values that are yielded from the "after" region (by the
+    /// scf.yield op).
+    ValueRange getYieldedValues();
+
     Block::BlockArgListType getBeforeArguments();
     Block::BlockArgListType getAfterArguments();
     Block *getBeforeBody() { return &getBefore().front(); }
diff --git a/mlir/include/mlir/Dialect/SCF/Transforms/TileUsingInterface.h b/mlir/include/mlir/Dialect/SCF/Transforms/TileUsingInterface.h
index 9f49d97e14..81325b6279 100644
--- a/mlir/include/mlir/Dialect/SCF/Transforms/TileUsingInterface.h
+++ b/mlir/include/mlir/Dialect/SCF/Transforms/TileUsingInterface.h
@@ -51,6 +51,17 @@ struct SCFTilingOptions {
     interchangeVector = llvm::to_vector(interchange);
     return *this;
   }
+
+  /// Specify mapping of loops to devices. This is only respected when the loop
+  /// constructs support such a mapping (like `scf.forall`). Will be ignored
+  /// when using loop constructs that dont support such a mapping (like
+  /// `scf.for`)
+  SmallVector<Attribute> mappingVector = {};
+  SCFTilingOptions &setMapping(ArrayRef<DeviceMappingAttrInterface> mapping) {
+    mappingVector = llvm::map_to_vector(
+        mapping, [](auto attr) -> Attribute { return attr; });
+    return *this;
+  }
 };
 
 /// Transformation information returned after tiling.
@@ -82,6 +93,12 @@ struct SCFTileAndFuseOptions {
   }
 };
 
+/// Method to tile an op that implements the `TilingInterface` using
+/// `scf.forall`.
+FailureOr<SCFTilingResult>
+tileUsingSCFForallOp(RewriterBase &rewriter, TilingInterface op,
+                     const SCFTilingOptions &options);
+
 /// Fuse the producer of the source of `candidateSliceOp` by computing the
 /// required slice of the producer in-place.  Note that the method
 /// replaces the uses of `candidateSliceOp` with the tiled and fused producer
diff --git a/mlir/include/mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h b/mlir/include/mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h
index 89ded981d3..933d62e35f 100644
--- a/mlir/include/mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h
+++ b/mlir/include/mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h
@@ -55,13 +55,13 @@ struct SPIRVConversionOptions {
   /// values will be packed into one 32-bit value to be memory efficient.
   bool emulateLT32BitScalarTypes{true};
 
-  /// Use 64-bit integers to convert index types.
-  bool use64bitIndex{false};
-
   /// Whether to enable fast math mode during conversion. If true, various
   /// patterns would assume no NaN/infinity numbers as inputs, and thus there
   /// will be no special guards emitted to check and handle such cases.
   bool enableFastMathMode{false};
+
+  /// Use 64-bit integers when converting index types.
+  bool use64bitIndex{false};
 };
 
 /// Type conversion from builtin types to SPIR-V types for shader interface.
@@ -77,6 +77,11 @@ class SPIRVTypeConverter : public TypeConverter {
   /// Gets the SPIR-V correspondence for the standard index type.
   Type getIndexType() const;
 
+  /// Gets the bitwidth of the index type when converted to SPIR-V.
+  unsigned getIndexTypeBitwidth() const {
+    return options.use64bitIndex ? 64 : 32;
+  }
+
   const spirv::TargetEnv &getTargetEnv() const { return targetEnv; }
 
   /// Returns the options controlling the SPIR-V type converter.
diff --git a/mlir/include/mlir/Dialect/SparseTensor/IR/CMakeLists.txt b/mlir/include/mlir/Dialect/SparseTensor/IR/CMakeLists.txt
index 25a2e4869c..54ad9491cc 100644
--- a/mlir/include/mlir/Dialect/SparseTensor/IR/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/SparseTensor/IR/CMakeLists.txt
@@ -12,3 +12,9 @@ set(LLVM_TARGET_DEFINITIONS SparseTensorTypes.td)
 mlir_tablegen(SparseTensorTypes.h.inc -gen-typedef-decls)
 mlir_tablegen(SparseTensorTypes.cpp.inc -gen-typedef-defs)
 add_public_tablegen_target(MLIRSparseTensorTypesIncGen)
+
+set(LLVM_TARGET_DEFINITIONS SparseTensorInterfaces.td)
+mlir_tablegen(SparseTensorInterfaces.h.inc -gen-op-interface-decls)
+mlir_tablegen(SparseTensorInterfaces.cpp.inc -gen-op-interface-defs)
+add_public_tablegen_target(MLIRSparseTensorInterfacesIncGen)
+add_dependencies(mlir-headers MLIRSparseTensorInterfacesIncGen)
diff --git a/mlir/include/mlir/Dialect/SparseTensor/IR/Enums.h b/mlir/include/mlir/Dialect/SparseTensor/IR/Enums.h
index 2920ef79f4..c65a27567d 100644
--- a/mlir/include/mlir/Dialect/SparseTensor/IR/Enums.h
+++ b/mlir/include/mlir/Dialect/SparseTensor/IR/Enums.h
@@ -31,6 +31,7 @@
 // NOTE: Client code will need to include "mlir/ExecutionEngine/Float16bits.h"
 // if they want to use the `MLIR_SPARSETENSOR_FOREVERY_V` macro.
 
+#include <cassert>
 #include <cinttypes>
 #include <complex>
 #include <optional>
@@ -143,14 +144,12 @@ constexpr bool isComplexPrimaryType(PrimaryType valTy) {
 /// The actions performed by @newSparseTensor.
 enum class Action : uint32_t {
   kEmpty = 0,
-  // newSparseTensor no longer handles `kFromFile=1`, so we leave this
-  // number reserved to help catch any code that still needs updating.
+  kEmptyForward = 1,
   kFromCOO = 2,
-  kSparseToSparse = 3,
-  kEmptyCOO = 4,
+  kFromReader = 4,
   kToCOO = 5,
-  kToIterator = 6,
   kPack = 7,
+  kSortCOOInPlace = 8,
 };
 
 /// This enum defines all the sparse representations supportable by
@@ -445,6 +444,56 @@ static_assert((isUniqueDLT(DimLevelType::Dense) &&
                !isUniqueDLT(DimLevelType::LooseCompressedNuNo)),
               "isUniqueDLT definition is broken");
 
+/// Bit manipulations for affine encoding.
+///
+/// Note that because the indices in the mappings refer to dimensions
+/// and levels (and *not* the sizes of these dimensions and levels), the
+/// 64-bit encoding gives ample room for a compact encoding of affine
+/// operations in the higher bits. Pure permutations still allow for
+/// 60-bit indices. But non-permutations reserve 20-bits for the
+/// potential three components (index i, constant, index ii).
+///
+/// The compact encoding is as follows:
+///
+///  0xffffffffffffffff
+/// |0000      |                        60-bit idx| e.g. i
+/// |0001 floor|           20-bit const|20-bit idx| e.g. i floor c
+/// |0010 mod  |           20-bit const|20-bit idx| e.g. i mod c
+/// |0011 mul  |20-bit idx|20-bit const|20-bit idx| e.g. i + c * ii
+///
+/// This encoding provides sufficient generality for currently supported
+/// sparse tensor types. To generalize this more, we will need to provide
+/// a broader encoding scheme for affine functions. Also, the library
+/// encoding may be replaced with pure "direct-IR" code in the future.
+///
+constexpr uint64_t encodeDim(uint64_t i, uint64_t cf, uint64_t cm) {
+  if (cf != 0) {
+    assert(cf <= 0xfffff && cm == 0 && i <= 0xfffff);
+    return (0x01L << 60) | (cf << 20) | i;
+  }
+  if (cm != 0) {
+    assert(cm <= 0xfffff && i <= 0xfffff);
+    return (0x02L << 60) | (cm << 20) | i;
+  }
+  assert(i <= 0x0fffffffffffffffu);
+  return i;
+}
+constexpr uint64_t encodeLvl(uint64_t i, uint64_t c, uint64_t ii) {
+  if (c != 0) {
+    assert(c <= 0xfffff && ii <= 0xfffff && i <= 0xfffff);
+    return (0x03L << 60) | (c << 20) | (ii << 40) | i;
+  }
+  assert(i <= 0x0fffffffffffffffu);
+  return i;
+}
+constexpr bool isEncodedFloor(uint64_t v) { return (v >> 60) == 0x01; }
+constexpr bool isEncodedMod(uint64_t v) { return (v >> 60) == 0x02; }
+constexpr bool isEncodedMul(uint64_t v) { return (v >> 60) == 0x03; }
+constexpr uint64_t decodeIndex(uint64_t v) { return v & 0xfffffu; }
+constexpr uint64_t decodeConst(uint64_t v) { return (v >> 20) & 0xfffffu; }
+constexpr uint64_t decodeMulc(uint64_t v) { return (v >> 20) & 0xfffffu; }
+constexpr uint64_t decodeMuli(uint64_t v) { return (v >> 40) & 0xfffffu; }
+
 } // namespace sparse_tensor
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensor.h b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensor.h
index 3eb9ce010c..51aacd8cc2 100644
--- a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensor.h
+++ b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensor.h
@@ -11,6 +11,7 @@
 
 #include "mlir/Bytecode/BytecodeOpInterface.h"
 #include "mlir/Dialect/SparseTensor/IR/Enums.h"
+#include "mlir/Dialect/SparseTensor/IR/SparseTensorInterfaces.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/OpDefinition.h"
@@ -159,6 +160,34 @@ inline bool hasAnySparseOperandOrResult(Operation *op) {
   return hasAnySparseOperand(op) || hasAnySparseResult(op);
 }
 
+//
+// Inference.
+//
+
+/// Given the dimToLvl map, infers the lvlToDim map, or returns
+/// empty Affine map when inference fails.
+AffineMap inferLvlToDim(AffineMap dimToLvl, MLIRContext *context);
+
+/// Returns the lvlToDim map for the given dimToLvl map specific
+/// to the block sparse cases.
+/// Asserts on failure (so only use when known to succeed).
+AffineMap inverseBlockSparsity(AffineMap dimToLvl, MLIRContext *context);
+
+/// Given the dimToLvl map, returns the block sizes in a vector.
+/// For instance, a 2x3 block will return [2, 3]. Unblocked dimension i
+/// will return 0, and i floordiv 1, i mod 1 will return 1. Therefore,
+/// the example below will return [0, 1].
+/// map = ( i, j ) ->
+///       ( i : dense,
+///         j floordiv 1 : compressed,
+///         j mod 1      : dense
+///       )
+/// Only valid block sparsity will be accepted.
+SmallVector<unsigned> getBlockSize(AffineMap dimToLvl);
+
+/// Given the dimToLvl map, returns if it's block sparsity.
+bool isBlockSparsity(AffineMap dimToLvl);
+
 //
 // Reordering.
 //
diff --git a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.td b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.td
index afd978c1c5..2dd7f8e961 100644
--- a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.td
+++ b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.td
@@ -134,7 +134,7 @@ def SparseTensorEncodingAttr : SparseTensor_Attr<"SparseTensorEncoding",
     level-coordinates. The dimension-expressions collectively define the inverse map,
     which only needs to be provided for elaborate cases where it cannot be inferred
     automatically.
-    
+
     Each dimension could also have an optional `SparseTensorDimSliceAttr`.
     Within the sparse storage format, we refer to indices that are stored explicitly
     as **coordinates** and offsets into the storage format as **positions**.
@@ -237,10 +237,10 @@ def SparseTensorEncodingAttr : SparseTensor_Attr<"SparseTensorEncoding",
     }>
     ... tensor<20x30xf32, #BSR_explicit> ...
 
-    // ELL format. 
+    // ELL format.
     // In the simple format for matrix, one array stores values and another
     // array stores column indices. The arrays have the same number of rows
-    // as the original matrix, but only have as many columns as 
+    // as the original matrix, but only have as many columns as
     // the maximum number of nonzeros on a row of the original matrix.
     // There are many variants for ELL such as jagged diagonal scheme.
     // To implement ELL, map provides a notion of "counting a
@@ -307,6 +307,9 @@ def SparseTensorEncodingAttr : SparseTensor_Attr<"SparseTensorEncoding",
                      "AffineMap":$lvlToDim,
                      "unsigned":$posWidth,
                      "unsigned":$crdWidth), [{
+      if (!lvlToDim) {
+        lvlToDim = ::mlir::sparse_tensor::inferLvlToDim(dimToLvl, $_ctxt);
+      }
       return $_get($_ctxt, lvlTypes, dimToLvl, lvlToDim, posWidth, crdWidth,
         ArrayRef<::mlir::sparse_tensor::SparseTensorDimSliceAttr>{});
     }]>
@@ -376,6 +379,9 @@ def SparseTensorEncodingAttr : SparseTensor_Attr<"SparseTensorEncoding",
     /// the null encoding (since dense-tensors are always all-dense).
     bool isAllDense() const;
 
+    /// Returns true if it is a sparse tensor encoding in COO format.
+    bool isCOO() const;
+
     /// Returns true if every level is ordered.  Also returns true for
     /// the null encoding (since dense-tensors are always all-ordered).
     bool isAllOrdered() const;
@@ -423,6 +429,11 @@ def SparseTensorEncodingAttr : SparseTensor_Attr<"SparseTensorEncoding",
     std::optional<uint64_t> getStaticLvlSliceSize(::mlir::sparse_tensor::Level lvl) const;
     std::optional<uint64_t> getStaticLvlSliceStride(::mlir::sparse_tensor::Level lvl) const;
 
+    //
+    // Helper function to build IR related to the encoding.
+    //
+    ValueRange translateCrds(::mlir::OpBuilder &builder, ::mlir::Location loc, ::mlir::ValueRange crds, ::mlir::sparse_tensor::CrdTransDirectionKind) const;
+
     //
     // Printing methods.
     //
@@ -468,6 +479,10 @@ def SparseTensorStorageSpecifierKindAttr
 def IsSparseTensorPred
   : CPred<"!!::mlir::sparse_tensor::getSparseTensorEncoding($_self)">;
 
+def IsCOOPred
+  : CPred<"!!::mlir::sparse_tensor::getSparseTensorEncoding($_self) && "
+          "  ::mlir::sparse_tensor::getSparseTensorEncoding($_self).isCOO()">;
+
 def IsSparseTensorSlicePred
   : CPred<"!!::mlir::sparse_tensor::getSparseTensorEncoding($_self) && "
           "  ::mlir::sparse_tensor::getSparseTensorEncoding($_self).isSlice()">;
@@ -478,10 +493,14 @@ def IsSparseTensorSlicePred
 class SparseTensorOf<list<Type> allowedTypes>
   : TensorOf<allowedTypes, [IsSparseTensorPred], "sparse tensor">;
 
+class COOSparseTensorOf<list<Type> allowedTypes>
+  : TensorOf<allowedTypes, [IsCOOPred], "COO sparse tensor">;
+
 class SparseTensorSliceOf<list<Type> allowedTypes>
   : TensorOf<allowedTypes, [IsSparseTensorSlicePred], "sparse tensor slice">;
 
 def AnySparseTensor : SparseTensorOf<[AnyType]>;
+def AnyCOOSparseTensor : COOSparseTensorOf<[AnyType]>;
 def AnySparseTensorSlice : SparseTensorSliceOf<[AnyType]>;
 
 class RankedSparseTensorOf<list<Type> allowedTypes>
@@ -527,4 +546,26 @@ def SparseTensorSortKindAttr
                "SparseTensorSortAlgorithm"> {
 }
 
+
+//===----------------------------------------------------------------------===//
+// Sparse Tensor Coordinate Translation Direction Attribute.
+//===----------------------------------------------------------------------===//
+
+// The C++ enum for sparse tensor coordinate translation direction enum.
+def SparseTensorCrdTransDirectionEnum
+    : I32EnumAttr<"CrdTransDirectionKind", "sparse tensor coordinate translation direction", [
+        I32EnumAttrCase<"dim2lvl", 0, "dim_to_lvl">,
+        I32EnumAttrCase<"lvl2dim", 1, "lvl_to_dim">,
+      ]> {
+  let genSpecializedAttr = 0;
+  let cppNamespace = SparseTensor_Dialect.cppNamespace;
+}
+
+// The C++ enum for sparse tensor coordinate translation direction attribute.
+def SparseTensorCrdTransDirectionAttr
+    : EnumAttr<SparseTensor_Dialect, SparseTensorCrdTransDirectionEnum,
+               "CrdTransDirection"> {
+}
+
+
 #endif // SPARSETENSOR_ATTRDEFS
diff --git a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorInterfaces.h b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorInterfaces.h
new file mode 100644
index 0000000000..ebbc522123
--- /dev/null
+++ b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorInterfaces.h
@@ -0,0 +1,31 @@
+//===- SparseTensorInterfaces.h - sparse tensor operations
+//interfaces-------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_SPARSETENSOR_IR_SPARSETENSORINTERFACES_H_
+#define MLIR_DIALECT_SPARSETENSOR_IR_SPARSETENSORINTERFACES_H_
+
+#include "mlir/IR/OpDefinition.h"
+
+namespace mlir {
+class PatternRewriter;
+
+namespace sparse_tensor {
+class StageWithSortSparseOp;
+
+namespace detail {
+LogicalResult stageWithSortImpl(sparse_tensor::StageWithSortSparseOp op,
+                                PatternRewriter &rewriter);
+} // namespace detail
+} // namespace sparse_tensor
+} // namespace mlir
+
+/// Include the generated interface declarations.
+#include "mlir/Dialect/SparseTensor/IR/SparseTensorInterfaces.h.inc"
+
+#endif // MLIR_DIALECT_SPARSETENSOR_IR_SPARSETENSORINTERFACES_H_
diff --git a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorInterfaces.td b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorInterfaces.td
new file mode 100644
index 0000000000..1379363ff7
--- /dev/null
+++ b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorInterfaces.td
@@ -0,0 +1,45 @@
+//===- SparseTensorInterfaces.td --------------------------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SPARSETENSOR_IR_SPARSETENSORINTERFACES
+#define SPARSETENSOR_IR_SPARSETENSORINTERFACES
+
+include "mlir/IR/OpBase.td"
+
+def StageWithSortSparseOpInterface : OpInterface<"StageWithSortSparseOp"> {
+  let description = [{
+    A stage-with-sort sparse tensor operation is an operation that produces
+    unordered intermediate output. An extra sort is required to obtain the final
+    ordered result.
+
+    E.g., convert csr -> csc need to be implemented as
+          convert csr -> unordered coo -> sort by column -> csc; and
+          concatenate csr, csc -> csr can be staged into
+          concatenate csr, csr -> unordered coo -> sort by row -> csr.
+  }];
+  let cppNamespace = "::mlir::sparse_tensor";
+  let methods = [
+    InterfaceMethod<
+    /*desc=*/"Return true if the operation needs an extra sort to produce the final result.",
+    /*retTy=*/"bool",
+    /*methodName=*/"needsExtraSort",
+    /*args=*/(ins),
+    /*methodBody=*/"">,
+    InterfaceMethod<
+    /*desc=*/"Stage the operation, return the final result value after staging.",
+    /*retTy=*/"::mlir::LogicalResult",
+    /*methodName=*/"stageWithSort",
+    /*args=*/(ins "::mlir::PatternRewriter &":$rewriter),
+    /*methodBody=*/[{
+        return detail::stageWithSortImpl($_op, rewriter);
+    }]>,
+  ];
+}
+
+
+#endif // SPARSETENSOR_IR_SPARSETENSORINTERFACES
diff --git a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td
index 7ea5ca23f1..7209f2ef84 100644
--- a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td
+++ b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td
@@ -12,6 +12,7 @@
 include "mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.td"
 include "mlir/Dialect/SparseTensor/IR/SparseTensorBase.td"
 include "mlir/Dialect/SparseTensor/IR/SparseTensorTypes.td"
+include "mlir/Dialect/SparseTensor/IR/SparseTensorInterfaces.td"
 include "mlir/Interfaces/InferTypeOpInterface.td"
 include "mlir/Interfaces/SideEffectInterfaces.td"
 
@@ -153,7 +154,7 @@ def SparseTensor_DisassembleOp : SparseTensor_Op<"disassemble", [Pure, SameVaria
 }
 
 def SparseTensor_ConvertOp : SparseTensor_Op<"convert",
-  [Pure]>,
+  [Pure, StageWithSortSparseOpInterface]>,
     Arguments<(ins AnyTensor:$source)>,
     Results<(outs AnyTensor:$dest)> {
   string summary = "Converts between different tensor types";
@@ -195,6 +196,13 @@ def SparseTensor_ConvertOp : SparseTensor_Op<"convert",
     ```
 
   }];
+
+  let extraClassDeclaration = [{
+     // Whether the convert can be done by a single step or it would require
+     // an extra sort. Inherited from StageWithSortSparseOpInterface.
+     bool needsExtraSort();
+  }];
+
   let assemblyFormat = "$source attr-dict `:` type($source) `to` type($dest)";
   let hasFolder = 1;
   let hasVerifier = 1;
@@ -327,7 +335,8 @@ def SparseTensor_NumberOfEntriesOp : SparseTensor_Op<"number_of_entries", [Pure]
   let assemblyFormat = "$tensor attr-dict `:` type($tensor)";
 }
 
-def SparseTensor_ConcatenateOp : SparseTensor_Op<"concatenate", [Pure]>,
+def SparseTensor_ConcatenateOp : SparseTensor_Op<"concatenate",
+                                 [Pure, StageWithSortSparseOpInterface]>,
     Arguments<(ins Variadic<AnyRankedTensor>:$inputs, DimensionAttr:$dimension)>,
     Results<(outs AnyRankedTensor:$result)> {
 
@@ -350,6 +359,12 @@ def SparseTensor_ConcatenateOp : SparseTensor_Op<"concatenate", [Pure]>,
      ```
    }];
 
+  let extraClassDeclaration = [{
+     // Whether the concatenate can be done by a single step or it would require
+     // an extra sort. Inherited from StageWithSortSparseOpInterface.
+     bool needsExtraSort();
+  }];
+
   let assemblyFormat = "$inputs attr-dict `:` type($inputs) `to` type($result)";
   let hasVerifier = 1;
 }
@@ -431,10 +446,13 @@ def SparseTensor_StorageSpecifierInitOp : SparseTensor_Op<"storage_specifier.ini
     Example:
 
     ```mlir
-    #CSR = #sparse_tensor.encoding<{ dimLevelType = [ "dense", "compressed" ]}>
+    #CSR = #sparse_tensor.encoding<{
+      map = (i, j) -> (i : dense, j : compressed)
+    }>
     #CSR_SLICE = #sparse_tensor.encoding<{
-      dimLevelType = [ "dense", "compressed" ],
-      slice = [ (1, 4, 1), (1, 4, 2) ]
+      map = (d0 : #sparse_tensor<slice(1, 4, 1)>,
+             d1 : #sparse_tensor<slice(1, 4, 2)>) ->
+            (d0 : dense, d1 : compressed)
     }>
 
     %0 = sparse_tensor.storage_specifier.init :  !sparse_tensor.storage_specifier<#CSR>
@@ -502,6 +520,32 @@ def SparseTensor_SetStorageSpecifierOp : SparseTensor_Op<"storage_specifier.set"
   let hasVerifier = 1;
 }
 
+//===----------------------------------------------------------------------===//
+// Sparse Tensor Coordinate Translation Operation.
+//===----------------------------------------------------------------------===//
+
+def SparseTensor_CrdTranslateOp : SparseTensor_Op<"crd_translate", [Pure]>,
+    Arguments<(ins Variadic<Index>:$in_crds,
+                   SparseTensorCrdTransDirectionAttr:$direction,
+                   SparseTensorEncodingAttr:$encoder)>,
+    Results<(outs Variadic<Index>:$out_crds)> {
+  string summary = "Performs coordinate translation between level and dimension coordinate space.";
+  string description = [{
+    Performs coordinate translation between level and dimension coordinate space according
+    to the affine maps defined by $encoder.
+
+    Example:
+
+    ```mlir
+    %l0, %l1, %l2, %l3 = sparse_tensor.crd_translate dim_to_lvl [%d0, %d1] as #BSR
+                       : index, index, index, index
+    ```
+  }];
+  let assemblyFormat = "$direction `[` $in_crds `]` `as` $encoder attr-dict `:` type($out_crds)";
+  let hasVerifier = 1;
+  let hasFolder = 1;
+}
+
 //===----------------------------------------------------------------------===//
 // Sparse Tensor Management Operations. These operations are "impure" in the
 // sense that some behavior is defined by side-effects. These operations provide
@@ -759,7 +803,7 @@ def SparseTensor_OutOp : SparseTensor_Op<"out", []>,
 }
 
 //===----------------------------------------------------------------------===//
-// Sparse Tensor Sorting Operations.
+// Sparse Tensor Sorting/Ordering Operations.
 //===----------------------------------------------------------------------===//
 
 def SparseTensor_SortOp : SparseTensor_Op<"sort">,
@@ -798,6 +842,36 @@ def SparseTensor_SortOp : SparseTensor_Op<"sort">,
   let hasVerifier = 1;
 }
 
+def SparseTensor_ReorderCOOOp : SparseTensor_Op<"reorder_coo", [Pure]>,
+    Arguments<(ins AnyCOOSparseTensor: $input_coo,
+                   SparseTensorSortKindAttr:$algorithm)>,
+    Results<(outs AnyCOOSparseTensor: $result_coo)> {
+  let summary = "Reorder the input COO such that it has the the same order as "
+                "the output COO";
+  let description = [{
+    sparse_tensor.reorder_coo reorder input COO to the same order as specified by
+    the output format. E.g., reorder an unordered COO into an ordered one.
+
+    The input and result COO tensor must have the same element type, position type and
+    coordinate type. At the moment, the operation also only supports ordering
+    input and result COO with the same dim2lvl map.
+
+    Example:
+
+    ```mlir
+    %res = sparse_tensor.reorder_coo quick_sort %coo : tensor<?x?xf64 : #Unordered_COO> to
+                                                       tensor<?x?xf64 : #Ordered_COO>
+
+    ```
+  }];
+
+  let assemblyFormat = "$algorithm $input_coo attr-dict"
+                       "`:` type($input_coo) `to` type($result_coo)";
+
+  let hasFolder = 1;
+  let hasVerifier = 1;
+}
+
 //===----------------------------------------------------------------------===//
 // Sparse Tensor Syntax Operations.
 //===----------------------------------------------------------------------===//
@@ -1215,8 +1289,7 @@ def SparseTensor_ForeachOp : SparseTensor_Op<"foreach",
      For example:
      ```mlir
      #COL_MAJOR = #sparse_tensor.encoding<{
-       dimLevelType = [ "compressed", "compressed" ],
-       dimOrdering = affine_map<(i,j) -> (j,i)>
+       map = (d0, d1) -> (d1 : compressed, d0 : compressed)
      }>
 
      // foreach on a column-major sparse tensor
@@ -1226,7 +1299,7 @@ def SparseTensor_ForeachOp : SparseTensor_Op<"foreach",
      }
 
      #ROW_MAJOR = #sparse_tensor.encoding<{
-       dimLevelType = [ "compressed", "compressed" ],
+       map = (d0, d1) -> (d0 : compressed, d1 : compressed)
      }>
 
      // foreach on a row-major sparse tensor
diff --git a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorType.h b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorType.h
index b5dbc67781..c3e967fdcd 100644
--- a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorType.h
+++ b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorType.h
@@ -176,6 +176,10 @@ class SparseTensorType {
   /// Returns the encoding (or the null-attribute for dense-tensors).
   SparseTensorEncodingAttr getEncoding() const { return enc; }
 
+  //
+  // SparseTensorEncodingAttr delegators
+  //
+
   /// Returns true for tensors which have an encoding, and false for
   /// those which do not.  Therefore tensors with an all-dense encoding
   /// return true.
@@ -189,14 +193,20 @@ class SparseTensorType {
   /// (This is always true for dense-tensors.)
   bool isAllOrdered() const { return enc.isAllOrdered(); }
 
-  /// Returns true if the dimToLvl mapping is the identity.
-  /// (This is always true for dense-tensors.)
-  bool isIdentity() const { return !dimToLvl; }
+  /// Translates between level / dimension coordinate space.
+  ValueRange translateCrds(OpBuilder &builder, Location loc, ValueRange crds,
+                           CrdTransDirectionKind dir) const {
+    return enc.translateCrds(builder, loc, crds, dir);
+  }
 
   /// Returns true if the dimToLvl mapping is a permutation.
   /// (This is always true for dense-tensors.)
   bool isPermutation() const { return enc.isPermutation(); }
 
+  /// Returns true if the dimToLvl mapping is the identity.
+  /// (This is always true for dense-tensors.)
+  bool isIdentity() const { return enc.isIdentity(); }
+
   /// Returns the dimToLvl mapping (or the null-map for the identity).
   /// If you intend to compare the results of this method for equality,
   /// see `hasSameDimToLvl` instead.
diff --git a/mlir/include/mlir/Dialect/SparseTensor/Pipelines/Passes.h b/mlir/include/mlir/Dialect/SparseTensor/Pipelines/Passes.h
index c88963d399..57d8ffb356 100644
--- a/mlir/include/mlir/Dialect/SparseTensor/Pipelines/Passes.h
+++ b/mlir/include/mlir/Dialect/SparseTensor/Pipelines/Passes.h
@@ -88,6 +88,8 @@ struct SparseCompilerOptions
       *this, "enable-buffer-initialization",
       desc("Enable zero-initialization of memory buffers"), init(false)};
 
+  // TODO: Delete the option, it should also be false after switching to
+  // buffer-deallocation-pass
   PassOptions::Option<bool> createSparseDeallocs{
       *this, "create-sparse-deallocs",
       desc("Specify if the temporary buffers created by the sparse "
@@ -100,11 +102,6 @@ struct SparseCompilerOptions
       *this, "vl", desc("Set the vector length (0 disables vectorization)"),
       init(0)};
 
-  // These options must be kept in sync with `SparseTensorConversionBase`.
-  PassOptions::Option<int32_t> sparseToSparse{
-      *this, "s2s-strategy",
-      desc("Set the strategy for sparse-to-sparse conversion"), init(0)};
-
   // These options must be kept in sync with the `ConvertVectorToLLVM`
   // (defined in include/mlir/Dialect/SparseTensor/Pipelines/Passes.h).
   PassOptions::Option<bool> reassociateFPReductions{
@@ -174,12 +171,6 @@ struct SparseCompilerOptions
                                  enableRuntimeLibrary);
   }
 
-  /// Projects out the options for `createSparseTensorConversionPass`.
-  SparseTensorConversionOptions sparseTensorConversionOptions() const {
-    return SparseTensorConversionOptions(
-        sparseToSparseConversionStrategy(sparseToSparse));
-  }
-
   /// Projects out the options for `createConvertVectorToLLVMPass`.
   ConvertVectorToLLVMPassOptions lowerVectorToLLVMOptions() const {
     ConvertVectorToLLVMPassOptions opts{};
diff --git a/mlir/include/mlir/Dialect/SparseTensor/Transforms/Passes.h b/mlir/include/mlir/Dialect/SparseTensor/Transforms/Passes.h
index c537e92a51..204bc1ec2d 100644
--- a/mlir/include/mlir/Dialect/SparseTensor/Transforms/Passes.h
+++ b/mlir/include/mlir/Dialect/SparseTensor/Transforms/Passes.h
@@ -119,37 +119,11 @@ class SparseTensorTypeToPtrConverter : public TypeConverter {
   SparseTensorTypeToPtrConverter();
 };
 
-/// Defines a strategy for implementing sparse-to-sparse conversion.
-/// `kAuto` leaves it up to the compiler to automatically determine
-/// the method used.  `kViaCOO` converts the source tensor to COO and
-/// then converts the COO to the target format.  `kDirect` converts
-/// directly via the algorithm in <https://arxiv.org/abs/2001.02609>;
-/// however, beware that there are many formats not supported by this
-/// conversion method.
-enum class SparseToSparseConversionStrategy { kAuto, kViaCOO, kDirect };
-
-/// Converts command-line sparse2sparse flag to the strategy enum.
-SparseToSparseConversionStrategy sparseToSparseConversionStrategy(int32_t flag);
-
-/// SparseTensorConversion options.
-struct SparseTensorConversionOptions {
-  SparseTensorConversionOptions(SparseToSparseConversionStrategy s2s)
-      : sparseToSparseStrategy(s2s) {}
-  SparseTensorConversionOptions()
-      : SparseTensorConversionOptions(SparseToSparseConversionStrategy::kAuto) {
-  }
-  SparseToSparseConversionStrategy sparseToSparseStrategy;
-};
-
 /// Sets up sparse tensor conversion rules.
-void populateSparseTensorConversionPatterns(
-    TypeConverter &typeConverter, RewritePatternSet &patterns,
-    const SparseTensorConversionOptions &options =
-        SparseTensorConversionOptions());
+void populateSparseTensorConversionPatterns(TypeConverter &typeConverter,
+                                            RewritePatternSet &patterns);
 
 std::unique_ptr<Pass> createSparseTensorConversionPass();
-std::unique_ptr<Pass>
-createSparseTensorConversionPass(const SparseTensorConversionOptions &options);
 
 //===----------------------------------------------------------------------===//
 // The SparseTensorCodegen pass.
@@ -235,7 +209,6 @@ std::unique_ptr<Pass> createSparsificationAndBufferizationPass();
 std::unique_ptr<Pass> createSparsificationAndBufferizationPass(
     const bufferization::OneShotBufferizationOptions &bufferizationOptions,
     const SparsificationOptions &sparsificationOptions,
-    const SparseTensorConversionOptions &sparseTensorConversionOptions,
     bool createSparseDeallocs, bool enableRuntimeLibrary,
     bool enableBufferInitialization, unsigned vectorLength,
     bool enableVLAVectorization, bool enableSIMDIndex32);
diff --git a/mlir/include/mlir/Dialect/SparseTensor/Transforms/Passes.td b/mlir/include/mlir/Dialect/SparseTensor/Transforms/Passes.td
index 8f116bff9b..73ecf5061f 100644
--- a/mlir/include/mlir/Dialect/SparseTensor/Transforms/Passes.td
+++ b/mlir/include/mlir/Dialect/SparseTensor/Transforms/Passes.td
@@ -143,6 +143,7 @@ def PostSparsificationRewrite : Pass<"post-sparsification-rewrite", "ModuleOp">
   }];
   let constructor = "mlir::createPostSparsificationRewritePass()";
   let dependentDialects = [
+    "affine::AffineDialect",
     "arith::ArithDialect",
     "bufferization::BufferizationDialect",
     "linalg::LinalgDialect",
@@ -201,10 +202,6 @@ def SparseTensorConversionPass : Pass<"sparse-tensor-conversion", "ModuleOp"> {
     "scf::SCFDialect",
     "sparse_tensor::SparseTensorDialect",
   ];
-  let options = [
-    Option<"sparseToSparse", "s2s-strategy", "int32_t", "0",
-           "Set the strategy for sparse-to-sparse conversion">,
-  ];
 }
 
 def SparseTensorCodegen : Pass<"sparse-tensor-codegen", "ModuleOp"> {
diff --git a/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td b/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td
index 86a250b77d..21e1f87bfa 100644
--- a/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td
+++ b/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td
@@ -1168,16 +1168,17 @@ def Tensor_PadOp : Tensor_Op<"pad", [
 
     * source: the "base" tensor on which to pad.
     * low: A list contains the padding along the start of each
-           dimension, i.e `low`.
+           dimension, i.e., how many padded values are prepended
+           to the beginning of the tensor in each dimension.
     * high: A list contains the padding along the end of each
-            dimension, i.e. `high`.
+            dimension, i.e., how many padded values are appended
+            to the end of the tensor in each dimension.
     * nofold: indicates that the operation should not be folded when source and
               result types are equal.
 
-    The result tensor dimensions are `low` + `dim` + `high` along that
-    dimension. The number of elements of `low` and `high` must match
-    the rank of the input tensor. They can be either a constant or a
-    dynamic value.
+    The result tensor dimensions are `low[i]` + `dim[i]` + `high[i]` for each
+    dimension `i`. The number of elements of `low` and `high` must match the
+    rank of the input tensor. They can be either a constant or a dynamic value.
 
     The region of the `tensor.pad` operation returns the value to use
     for the padding. The arguments of the region represent the index
@@ -1189,7 +1190,21 @@ def Tensor_PadOp : Tensor_Op<"pad", [
     if the source type and the padded type have the same static shape. This can
     be used, e.g., for packing or promotion to faster memory.
 
-    Example 1:
+    Example 1: add 3 zeros to the beginning and 5 zeros to the end of a 1D
+    tensor.
+
+    ```mlir
+      %arg0 = ... : tensor<10xi32>
+      %c0_i32 = arith.constant 0 : i32
+      %padded = tensor.pad %arg0 low[3] high[5] {
+      ^bb0(%arg1: index):
+        tensor.yield %c0_i32 : i32
+      } : tensor<10xi32> to tensor<18xi32>
+    ```
+
+    Example 2: add 1 value to the beginning of dimension 0, 2 values to the end
+    of dimension 0, 2 values to the start of dimension 1, and 3 values to the
+    end of dimension 1.
 
     ```mlir
       %pad_value = ... : f32
@@ -1199,7 +1214,7 @@ def Tensor_PadOp : Tensor_Op<"pad", [
       } : tensor<?x?xf32> to tensor<?x?xf32>
     ```
 
-    Example 2:
+    Example 3:
 
     ```mlir
       %pad_value = ... : f32
@@ -1209,7 +1224,7 @@ def Tensor_PadOp : Tensor_Op<"pad", [
       } : tensor<1x2x2x?xf32> to tensor<6x?x?x?xf32>
     ```
 
-    Example 3:
+    Example 4:
 
     ```mlir
       %pad_value = ... : f32
@@ -1219,10 +1234,11 @@ def Tensor_PadOp : Tensor_Op<"pad", [
       } : tensor<2x3xf32> to tensor<?x?xf32>
     ```
 
-    Example 4:
+    Example 5: Force a padded value to be always exist with `nofold`, even
+    though the padding config specifies that no new elements will be added to
+    the tensor.
 
     ```mlir
-      // Force a padded value to be always exist with `nofold`.
       %pad_value = ... : f32
       %0 = tensor.pad %arg0 nofold low[0, 0] high[0, 0] {
       ^bb0(%arg1: index, %arg2: index):
diff --git a/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.h b/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.h
index 555d9bea18..a9bc3351f4 100644
--- a/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.h
+++ b/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.h
@@ -34,6 +34,11 @@ class PatternRewriter;
 
 namespace tosa {
 
+ParseResult parseTypeOrAttr(OpAsmParser &parser, TypeAttr &typeAttr,
+                            Attribute &attr);
+void printTypeOrAttr(OpAsmPrinter &p, Operation *op, TypeAttr type,
+                     Attribute attr);
+
 #include "mlir/Dialect/Tosa/IR/TosaInterfaces.h.inc"
 
 } // namespace tosa
diff --git a/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td b/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td
index a80111aedf..5cc97469d1 100644
--- a/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td
+++ b/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td
@@ -380,8 +380,8 @@ def Tosa_ClampOp : Tosa_ElementwiseOp<"clamp"> {
     Tosa_Tensor:$input,
     I64Attr:$min_int,
     I64Attr:$max_int,
-    F32Attr:$min_fp,
-    F32Attr:$max_fp
+    Tosa_FloatAttr:$min_fp,
+    Tosa_FloatAttr:$max_fp
   );
 
   let results = (outs
@@ -389,6 +389,7 @@ def Tosa_ClampOp : Tosa_ElementwiseOp<"clamp"> {
   );
 
   let hasCanonicalizer = 1;
+  let hasVerifier = 1;
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/Tosa/IR/TosaTypesBase.td b/mlir/include/mlir/Dialect/Tosa/IR/TosaTypesBase.td
index e39f4662e7..c55ddaafdd 100644
--- a/mlir/include/mlir/Dialect/Tosa/IR/TosaTypesBase.td
+++ b/mlir/include/mlir/Dialect/Tosa/IR/TosaTypesBase.td
@@ -197,6 +197,12 @@ def Tosa_IntArrayAttrUpto2 : ConfinedAttr<DenseI64ArrayAttr, [DenseArrayMaxCt<2>
 def Tosa_IntArrayAttrUpto4 : ConfinedAttr<DenseI64ArrayAttr, [DenseArrayMaxCt<4>]>;
 def Tosa_IntArrayAttrUpto5 : ConfinedAttr<DenseI64ArrayAttr, [DenseArrayMaxCt<5>]>;
 
+def Tosa_FloatAttr : Attr<CPred<"::llvm::isa<::mlir::FloatAttr>($_self)">,
+                          "arbitrary float attribute"> {
+  let storageType = [{ ::mlir::FloatAttr }];
+  let returnType = [{ ::mlir::APFloat }];
+}
+
 //===----------------------------------------------------------------------===//
 // Iterable attributes.
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/Tosa/IR/TosaUtilOps.td b/mlir/include/mlir/Dialect/Tosa/IR/TosaUtilOps.td
index d75f5dffa8..f9f25da1b6 100644
--- a/mlir/include/mlir/Dialect/Tosa/IR/TosaUtilOps.td
+++ b/mlir/include/mlir/Dialect/Tosa/IR/TosaUtilOps.td
@@ -79,4 +79,71 @@ def Tosa_YieldOp : Tosa_Op<"yield", [
   let assemblyFormat = "$inputs attr-dict `:` type($inputs)";
 }
 
+//===----------------------------------------------------------------------===//
+// Operator: variable
+//===----------------------------------------------------------------------===//
+def Tosa_VariableOp : Tosa_Op<"variable", []> {
+  let summary = "Defines a variable";
+
+  let description = [{
+    Defines a new TOSA variable. This is a mutable value.
+    Modifications are expressed using read/write semantics.
+  }];
+
+  let arguments = (ins
+    SymbolNameAttr:$name,
+    TypeAttr:$type,
+    OptionalAttr<AnyAttr>:$initial_value
+  );
+
+  let assemblyFormat = [{
+    $name
+    attr-dict
+    custom<TypeOrAttr>($type, $initial_value)
+  }];
+}
+
+//===----------------------------------------------------------------------===//
+// Operator: variable.write
+//===----------------------------------------------------------------------===//
+def Tosa_VariableWriteOp : Tosa_Op<"variable.write", []> {
+  let summary = "write_buffer operator";
+
+  let description = [{
+    Assigns a value to pseudo-buffer resource holding a mutable tensor.
+  }];
+
+  let arguments = (ins
+    SymbolNameAttr:$name,
+    AnyType:$value
+  );
+
+  let assemblyFormat = [{
+    $name attr-dict `,` $value `:` type($value)
+  }];
+}
+
+//===----------------------------------------------------------------------===//
+// Operator: variable.read
+//===----------------------------------------------------------------------===//
+def Tosa_VariableReadOp : Tosa_Op<"variable.read", []> {
+  let summary = "read_buffer operator";
+
+  let description = [{
+    Reads the value from a pseudo-buffer resource holding a mutable tensor.
+  }];
+
+  let arguments = (ins
+    SymbolNameAttr:$name
+  );
+
+  let results = (outs
+    AnyType:$value
+  );
+
+  let assemblyFormat = [{
+    $name attr-dict `:` type($value)
+  }];
+}
+
 #endif // TOSA_UTIL_OPS
diff --git a/mlir/include/mlir/Dialect/Tosa/Transforms/Passes.h b/mlir/include/mlir/Dialect/Tosa/Transforms/Passes.h
index 6b5dd9c970..fbfc56dfe2 100644
--- a/mlir/include/mlir/Dialect/Tosa/Transforms/Passes.h
+++ b/mlir/include/mlir/Dialect/Tosa/Transforms/Passes.h
@@ -35,9 +35,12 @@ void populateTosaFoldConstantReciprocalPatterns(MLIRContext *ctx,
 void populateTosaFoldConstantTransposePatterns(MLIRContext *ctx,
                                                RewritePatternSet &patterns);
 void populateTosaConstantReduction(MLIRContext *ctx,
-                                   RewritePatternSet &patterns);
+                                   RewritePatternSet &patterns,
+                                   bool aggressiveReduceConstant);
 
 std::unique_ptr<Pass> createTosaLayerwiseConstantFoldPass();
+std::unique_ptr<Pass> createTosaLayerwiseConstantFoldPass(
+    const TosaLayerwiseConstantFoldPassOptions &options);
 std::unique_ptr<Pass> createTosaInferShapesPass();
 std::unique_ptr<Pass> createTosaMakeBroadcastablePass();
 std::unique_ptr<Pass> createTosaTestQuantUtilAPIPass();
@@ -65,9 +68,6 @@ struct ValidationOptions {
   }
 };
 
-std::unique_ptr<Pass> createTosaValidationPass(
-    ValidationOptions const &options = ValidationOptions());
-
 #define GEN_PASS_REGISTRATION
 #include "mlir/Dialect/Tosa/Transforms/Passes.h.inc"
 
diff --git a/mlir/include/mlir/Dialect/Tosa/Transforms/Passes.td b/mlir/include/mlir/Dialect/Tosa/Transforms/Passes.td
index 18402b3e70..a0f670de20 100644
--- a/mlir/include/mlir/Dialect/Tosa/Transforms/Passes.td
+++ b/mlir/include/mlir/Dialect/Tosa/Transforms/Passes.td
@@ -23,6 +23,13 @@ def TosaLayerwiseConstantFoldPass : Pass<"tosa-layerwise-constant-fold", "func::
   }];
 
   let constructor = "createTosaLayerwiseConstantFoldPass()";
+
+  let options = [
+      Option<"aggressiveReduceConstant", "aggressive-reduce-constant", "bool",
+             /*default=*/"false",
+             "Always perform the reduce constant optimization"
+             "May add more tosa.const but would reduce runtime calculations">,
+   ];
 }
 
 def TosaInferShapes : Pass<"tosa-infer-shapes", "func::FuncOp"> {
@@ -82,13 +89,12 @@ def TosaLevelType : I32EnumAttr<"TosaLevelEnum", "Tosa level",
   let cppNamespace = "mlir::tosa";
 }
 
-def TosaValidation : Pass<"tosa-validate", "func::FuncOp"> {
+def TosaValidation : Pass<"tosa-validate", "mlir::ModuleOp"> {
   let summary = "Validates TOSA dialect";
   let description = [{
     This pass validates if input TOSA operations match the specification for given
     criteria, e.g. TOSA profile.
   }];
-  let constructor = "createTosaValidationPass()";
 
   let options = [
       Option<"profile", "profile", "mlir::tosa::TosaProfileEnum",
diff --git a/mlir/include/mlir/Dialect/Transform/IR/TransformInterfaces.h b/mlir/include/mlir/Dialect/Transform/IR/TransformInterfaces.h
index 7b37245fc3..5fcde11d52 100644
--- a/mlir/include/mlir/Dialect/Transform/IR/TransformInterfaces.h
+++ b/mlir/include/mlir/Dialect/Transform/IR/TransformInterfaces.h
@@ -95,11 +95,23 @@ class TransformOptions {
     return *this;
   }
 
+  // Ensures that only a single top-level transform op is present in the IR.
+  TransformOptions &enableEnforceSingleToplevelTransformOp(bool enable = true) {
+    enforceSingleToplevelTransformOp = enable;
+    return *this;
+  }
+
   /// Returns true if the expensive checks are requested.
   bool getExpensiveChecksEnabled() const { return expensiveChecksEnabled; }
 
+  // Returns true if enforcing a single top-level transform op is requested.
+  bool getEnforceSingleToplevelTransformOp() const {
+    return enforceSingleToplevelTransformOp;
+  }
+
 private:
   bool expensiveChecksEnabled = true;
+  bool enforceSingleToplevelTransformOp = true;
 };
 
 /// Entry point to the Transform dialect infrastructure. Applies the
diff --git a/mlir/include/mlir/Dialect/Transform/Transforms/Passes.td b/mlir/include/mlir/Dialect/Transform/Transforms/Passes.td
index 2400066c8a..286f69bc52 100644
--- a/mlir/include/mlir/Dialect/Transform/Transforms/Passes.td
+++ b/mlir/include/mlir/Dialect/Transform/Transforms/Passes.td
@@ -42,4 +42,37 @@ def InferEffectsPass : Pass<"transform-infer-effects"> {
   }];
 }
 
+def PreloadLibraryPass : Pass<"transform-preload-library"> {
+  let summary = "preload transform dialect library";
+  let description = [{
+    This pass preloads a transform library and makes it available to subsequent
+    transform interpreter passes. The preloading occurs into the Transform
+    dialect and thus provides very limited functionality that does not scale.
+
+    Warning: Only a single such pass should exist for a given MLIR context.
+    This is a temporary solution until a resource-based solution is available.
+  }];
+  // TODO: investigate using a resource blob if some ownership mode allows it.
+  let dependentDialects = ["::mlir::transform::TransformDialect"];
+  let options = [
+    ListOption<"transformLibraryPaths", "transform-library-paths", "std::string",
+    "Optional paths to files with modules that should be merged into the "
+    "transform module to provide the definitions of external named sequences.">
+  ];
+}
+
+def InterpreterPass : Pass<"transform-interpreter"> {
+  let summary = "transform dialect interpreter";
+  let description = [{
+    This pass runs the transform dialect interpreter and applies the named
+    sequence transformation specified by the provided name (defaults to
+    `__transform_main`).
+  }];
+  let dependentDialects = ["::mlir::transform::TransformDialect"];
+  let options = [
+    Option<"entryPoint", "entry-point", "std::string",
+           /*default=*/[{"__transform_main"}],
+           "Entry point of the pass pipeline.">,
+  ];
+}
 #endif // MLIR_DIALECT_TRANSFORM_TRANSFORMS_PASSES
diff --git a/mlir/include/mlir/Dialect/Transform/Transforms/TransformInterpreterUtils.h b/mlir/include/mlir/Dialect/Transform/Transforms/TransformInterpreterUtils.h
index 36c80e6fd6..3fc02267f2 100644
--- a/mlir/include/mlir/Dialect/Transform/Transforms/TransformInterpreterUtils.h
+++ b/mlir/include/mlir/Dialect/Transform/Transforms/TransformInterpreterUtils.h
@@ -26,6 +26,16 @@ class Region;
 
 namespace transform {
 namespace detail {
+
+/// Expands the given list of `paths` to a list of `.mlir` files.
+///
+/// Each entry in `paths` may either be a regular file, in which case it ends up
+/// in the result list, or a directory, in which case all (regular) `.mlir`
+/// files in that directory are added. Any other file types lead to a failure.
+LogicalResult expandPathsToMLIRFiles(ArrayRef<std::string> &paths,
+                                     MLIRContext *context,
+                                     SmallVectorImpl<std::string> &fileNames);
+
 /// Utility to parse and verify the content of a `transformFileName` MLIR file
 /// containing a transform dialect specification.
 LogicalResult
@@ -33,6 +43,14 @@ parseTransformModuleFromFile(MLIRContext *context,
                              llvm::StringRef transformFileName,
                              OwningOpRef<ModuleOp> &transformModule);
 
+/// Utility to parse, verify, aggregate and link the content of all mlir files
+/// nested under `transformLibraryPaths` and containing transform dialect
+/// specifications.
+LogicalResult
+assembleTransformLibraryFromPaths(MLIRContext *context,
+                                  ArrayRef<std::string> transformLibraryPaths,
+                                  OwningOpRef<ModuleOp> &transformModule);
+
 /// Utility to load a transform interpreter `module` from a module that has
 /// already been preloaded in the context.
 /// This mode is useful in cases where explicit parsing of a transform library
diff --git a/mlir/include/mlir/Dialect/Utils/StructuredOpsUtils.h b/mlir/include/mlir/Dialect/Utils/StructuredOpsUtils.h
index dab24bd930..225b9f287d 100644
--- a/mlir/include/mlir/Dialect/Utils/StructuredOpsUtils.h
+++ b/mlir/include/mlir/Dialect/Utils/StructuredOpsUtils.h
@@ -49,6 +49,24 @@ bool isColumnMajorMatmul(ArrayAttr indexingMaps);
 /// the reduction.
 bool isRowMajorBatchMatmul(ArrayAttr indexingMaps);
 
+/// Tests whether the given maps describe a vector matrix multiplication. The
+/// test is permutation-invariant. Note that this only checks the affine maps
+/// from an operation, so does not perform any checks on the math being
+/// performed within the reduction.
+bool isVecmat(ArrayAttr indexingMaps);
+
+/// Tests whether the given maps describe a matrix vector multiplication. The
+/// test is permutation-invariant. Note that this only checks the affine maps
+/// from an operation, so does not perform any checks on the math being
+/// performed within the reduction.
+bool isMatvec(ArrayAttr indexingMaps);
+
+/// Tests whether the given maps describe a batch matrix vector multiplication.
+/// The test is permutation-invariant. Note that this only checks the affine
+/// maps from an operation, so does not perform any checks on the math being
+/// performed within the reduction.
+bool isBatchMatvec(ArrayAttr indexingMaps);
+
 /// Return positions in `iteratorTypes` that match `iteratorTypeName`.
 inline void findPositionsOfType(ArrayRef<utils::IteratorType> iteratorTypes,
                                 utils::IteratorType iteratorTypeName,
diff --git a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.h b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.h
index fc0c80036f..9ab20e20d9 100644
--- a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.h
+++ b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.h
@@ -105,16 +105,23 @@ bool checkSameValueRAW(TransferWriteOp defWrite, TransferReadOp read);
 /// op.
 bool checkSameValueWAW(TransferWriteOp write, TransferWriteOp priorWrite);
 
-/// Same behavior as `isDisjointTransferSet` but doesn't require the operations
-/// to have the same tensor/memref. This allows comparing operations accessing
-/// different tensors.
+/// Return true if we can prove that the transfer operations access disjoint
+/// memory, without requring the accessed tensor/memref to be the same.
+///
+/// If `testDynamicValueUsingBounds` is true, tries to test dynamic values
+/// via ValueBoundsOpInterface.
 bool isDisjointTransferIndices(VectorTransferOpInterface transferA,
-                               VectorTransferOpInterface transferB);
+                               VectorTransferOpInterface transferB,
+                               bool testDynamicValueUsingBounds = false);
 
 /// Return true if we can prove that the transfer operations access disjoint
-/// memory.
+/// memory, requiring the operations to access the same tensor/memref.
+///
+/// If `testDynamicValueUsingBounds` is true, tries to test dynamic values
+/// via ValueBoundsOpInterface.
 bool isDisjointTransferSet(VectorTransferOpInterface transferA,
-                           VectorTransferOpInterface transferB);
+                           VectorTransferOpInterface transferB,
+                           bool testDynamicValueUsingBounds = false);
 
 /// Return the result value of reducing two scalar/vector values with the
 /// corresponding arith operation.
diff --git a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
index 2df2fe4c5c..917b27a40f 100644
--- a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
+++ b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
@@ -215,6 +215,8 @@ def Vector_ReductionOp :
   Vector_Op<"reduction", [Pure,
      PredOpTrait<"source operand and result have same element type",
                  TCresVTEtIsSameAsOpBase<0, 0>>,
+     OptionalTypesMatchWith<"dest and acc have the same type",
+                            "dest", "acc", "::llvm::cast<Type>($_self)">,
      DeclareOpInterfaceMethods<ArithFastMathInterface>,
      DeclareOpInterfaceMethods<MaskableOpInterface>,
      DeclareOpInterfaceMethods<VectorUnrollOpInterface, ["getShapeForUnroll"]>
@@ -263,9 +265,8 @@ def Vector_ReductionOp :
                          "::mlir::arith::FastMathFlags::none">:$fastMathFlags)>
   ];
 
-  // TODO: Migrate to assemblyFormat once `AllTypesMatch` supports optional
-  // operands.
-  let hasCustomAssemblyFormat = 1;
+  let assemblyFormat = "$kind `,` $vector (`,` $acc^)? (`fastmath` `` $fastmath^)?"
+                       " attr-dict `:` type($vector) `into` type($dest)";
   let hasCanonicalizer = 1;
   let hasVerifier = 1;
 }
diff --git a/mlir/include/mlir/ExecutionEngine/SparseTensor/COO.h b/mlir/include/mlir/ExecutionEngine/SparseTensor/COO.h
index f6eb45defc..721e9bc69a 100644
--- a/mlir/include/mlir/ExecutionEngine/SparseTensor/COO.h
+++ b/mlir/include/mlir/ExecutionEngine/SparseTensor/COO.h
@@ -58,21 +58,13 @@ struct ElementLT final {
   const uint64_t rank;
 };
 
-/// The type of callback functions which receive an element.
-template <typename V>
-using ElementConsumer =
-    const std::function<void(const std::vector<uint64_t> &, V)> &;
-
 /// A memory-resident sparse tensor in coordinate-scheme representation
-/// (a collection of `Element`s).  This data structure is used as
-/// an intermediate representation; e.g., for reading sparse tensors
-/// from external formats into memory, or for certain conversions between
-/// different `SparseTensorStorage` formats.
+/// (a collection of `Element`s). This data structure is used as an
+/// intermediate representation, e.g., for reading sparse tensors from
+/// external formats into memory.
 template <typename V>
 class SparseTensorCOO final {
 public:
-  using const_iterator = typename std::vector<Element<V>>::const_iterator;
-
   /// Constructs a new coordinate-scheme sparse tensor with the given
   /// sizes and an optional initial storage capacity.
   explicit SparseTensorCOO(const std::vector<uint64_t> &dimSizes,
@@ -106,7 +98,7 @@ class SparseTensorCOO final {
   /// Returns the `operator<` closure object for the COO's element type.
   ElementLT<V> getElementLT() const { return ElementLT<V>(getRank()); }
 
-  /// Adds an element to the tensor. This method invalidates all iterators.
+  /// Adds an element to the tensor.
   void add(const std::vector<uint64_t> &dimCoords, V val) {
     const uint64_t *base = coordinates.data();
     const uint64_t size = coordinates.size();
@@ -135,12 +127,9 @@ class SparseTensorCOO final {
     elements.push_back(addedElem);
   }
 
-  const_iterator begin() const { return elements.cbegin(); }
-  const_iterator end() const { return elements.cend(); }
-
   /// Sorts elements lexicographically by coordinates. If a coordinate
   /// is mapped to multiple values, then the relative order of those
-  /// values is unspecified. This method invalidates all iterators.
+  /// values is unspecified.
   void sort() {
     if (isSorted)
       return;
diff --git a/mlir/include/mlir/ExecutionEngine/SparseTensor/File.h b/mlir/include/mlir/ExecutionEngine/SparseTensor/File.h
index 9157bfa7e7..1b5f0553a3 100644
--- a/mlir/include/mlir/ExecutionEngine/SparseTensor/File.h
+++ b/mlir/include/mlir/ExecutionEngine/SparseTensor/File.h
@@ -201,10 +201,11 @@ class SparseTensorReader final {
                    const uint64_t *lvl2dim) {
     const uint64_t dimRank = getRank();
     MapRef map(dimRank, lvlRank, dim2lvl, lvl2dim);
-    auto *coo = readCOO<V>(map, lvlSizes);
+    auto *lvlCOO = readCOO<V>(map, lvlSizes);
     auto *tensor = SparseTensorStorage<P, I, V>::newFromCOO(
-        dimRank, getDimSizes(), lvlRank, lvlTypes, lvl2dim, *coo);
-    delete coo;
+        dimRank, getDimSizes(), lvlRank, lvlSizes, lvlTypes, dim2lvl, lvl2dim,
+        *lvlCOO);
+    delete lvlCOO;
     return tensor;
   }
 
diff --git a/mlir/include/mlir/ExecutionEngine/SparseTensor/MapRef.h b/mlir/include/mlir/ExecutionEngine/SparseTensor/MapRef.h
index 22ae70a61d..5c09aa4e4b 100644
--- a/mlir/include/mlir/ExecutionEngine/SparseTensor/MapRef.h
+++ b/mlir/include/mlir/ExecutionEngine/SparseTensor/MapRef.h
@@ -38,13 +38,23 @@ class MapRef final {
   // Push forward maps from dimensions to levels.
   //
 
+  // Map from dimRank in to lvlRank out.
   template <typename T>
   inline void pushforward(const T *in, T *out) const {
     if (isPermutation) {
-      for (uint64_t i = 0; i < lvlRank; ++i)
-        out[i] = in[lvl2dim[i]];
+      for (uint64_t l = 0; l < lvlRank; l++) {
+        out[l] = in[dim2lvl[l]];
+      }
     } else {
-      assert(0 && "coming soon");
+      uint64_t i, c;
+      for (uint64_t l = 0; l < lvlRank; l++)
+        if (isFloor(l, i, c)) {
+          out[l] = in[i] / c;
+        } else if (isMod(l, i, c)) {
+          out[l] = in[i] % c;
+        } else {
+          out[l] = in[dim2lvl[l]];
+        }
     }
   }
 
@@ -52,13 +62,20 @@ class MapRef final {
   // Push backward maps from levels to dimensions.
   //
 
+  // Map from lvlRank in to dimRank out.
   template <typename T>
   inline void pushbackward(const T *in, T *out) const {
     if (isPermutation) {
-      for (uint64_t i = 0; i < dimRank; ++i)
-        out[i] = in[dim2lvl[i]];
+      for (uint64_t d = 0; d < dimRank; d++)
+        out[d] = in[lvl2dim[d]];
     } else {
-      assert(0 && "coming soon");
+      uint64_t i, c, ii;
+      for (uint64_t d = 0; d < dimRank; d++)
+        if (isMul(d, i, c, ii)) {
+          out[d] = in[i] + c * in[ii];
+        } else {
+          out[d] = in[lvl2dim[d]];
+        }
     }
   }
 
@@ -68,6 +85,10 @@ class MapRef final {
 private:
   bool isPermutationMap() const;
 
+  bool isFloor(uint64_t l, uint64_t &i, uint64_t &c) const;
+  bool isMod(uint64_t l, uint64_t &i, uint64_t &c) const;
+  bool isMul(uint64_t d, uint64_t &i, uint64_t &c, uint64_t &ii) const;
+
   const uint64_t dimRank;
   const uint64_t lvlRank;
   const uint64_t *const dim2lvl; // non-owning pointer
diff --git a/mlir/include/mlir/ExecutionEngine/SparseTensor/Storage.h b/mlir/include/mlir/ExecutionEngine/SparseTensor/Storage.h
index 0407bccaae..ad92ee1f89 100644
--- a/mlir/include/mlir/ExecutionEngine/SparseTensor/Storage.h
+++ b/mlir/include/mlir/ExecutionEngine/SparseTensor/Storage.h
@@ -10,9 +10,6 @@
 //
 // * `SparseTensorStorageBase`
 // * `SparseTensorStorage<P, C, V>`
-// * `SparseTensorEnumeratorBase<V>`
-// * `SparseTensorEnumerator<P, C, V>`
-// * `SparseTensorNNZ`
 //
 //===----------------------------------------------------------------------===//
 
@@ -24,34 +21,20 @@
 #include "mlir/ExecutionEngine/SparseTensor/ArithmeticUtils.h"
 #include "mlir/ExecutionEngine/SparseTensor/COO.h"
 #include "mlir/ExecutionEngine/SparseTensor/ErrorHandling.h"
-
-#define ASSERT_VALID_DIM(d)                                                    \
-  assert(d < getDimRank() && "Dimension is out of bounds");
-#define ASSERT_VALID_LVL(l)                                                    \
-  assert(l < getLvlRank() && "Level is out of bounds");
-#define ASSERT_COMPRESSED_LVL(l)                                               \
-  assert(isCompressedLvl(l) && "Level is not compressed");
-#define ASSERT_COMPRESSED_OR_SINGLETON_LVL(l)                                  \
-  do {                                                                         \
-    const DimLevelType dlt = getLvlType(l);                                    \
-    (void)dlt;                                                                 \
-    assert((isCompressedDLT(dlt) || isSingletonDLT(dlt)) &&                    \
-           "Level is neither compressed nor singleton");                       \
-  } while (false)
-#define ASSERT_DENSE_DLT(dlt) assert(isDenseDLT(dlt) && "Level is not dense");
+#include "mlir/ExecutionEngine/SparseTensor/MapRef.h"
 
 namespace mlir {
 namespace sparse_tensor {
 
-// Forward references.
-template <typename V>
-class SparseTensorEnumeratorBase;
-template <typename P, typename C, typename V>
-class SparseTensorEnumerator;
+//===----------------------------------------------------------------------===//
+//
+//  SparseTensorStorage Classes
+//
+//===----------------------------------------------------------------------===//
 
-/// Abstract base class for `SparseTensorStorage<P,C,V>`.  This class
+/// Abstract base class for `SparseTensorStorage<P,C,V>`. This class
 /// takes responsibility for all the `<P,C,V>`-independent aspects
-/// of the tensor (e.g., shape, sparsity, permutation).  In addition,
+/// of the tensor (e.g., sizes, sparsity, mapping). In addition,
 /// we use function overloading to implement "partial" method
 /// specialization, which the C-API relies on to catch type errors
 /// arising from our use of opaque pointers.
@@ -59,27 +42,23 @@ class SparseTensorEnumerator;
 /// Because this class forms a bridge between the denotational semantics
 /// of "tensors" and the operational semantics of how we store and
 /// compute with them, it also distinguishes between two different
-/// coordinate spaces (and their associated rank, shape, sizes, etc).
+/// coordinate spaces (and their associated rank, sizes, etc).
 /// Denotationally, we have the *dimensions* of the tensor represented
 /// by this object.  Operationally, we have the *levels* of the storage
-/// representation itself.  We use this "dimension" vs "level" terminology
-/// throughout, since alternative terminology like "tensor-dimension",
-/// "original-dimension", "storage-dimension", etc, is both more verbose
-/// and prone to introduce confusion whenever the qualifiers are dropped.
-/// Where necessary, we use "axis" as the generic term.
+/// representation itself.
 ///
 /// The *size* of an axis is the cardinality of possible coordinate
 /// values along that axis (regardless of which coordinates have stored
-/// element values).  As such, each size must be non-zero since if any
+/// element values). As such, each size must be non-zero since if any
 /// axis has size-zero then the whole tensor would have trivial storage
-/// (since there are no possible coordinates).  Thus we use the plural
+/// (since there are no possible coordinates). Thus we use the plural
 /// term *sizes* for a collection of non-zero cardinalities, and use
-/// this term whenever referring to run-time cardinalities.  Whereas we
+/// this term whenever referring to run-time cardinalities. Whereas we
 /// use the term *shape* for a collection of compile-time cardinalities,
 /// where zero is used to indicate cardinalities which are dynamic (i.e.,
-/// unknown/unspecified at compile-time).  At run-time, these dynamic
+/// unknown/unspecified at compile-time). At run-time, these dynamic
 /// cardinalities will be inferred from or checked against sizes otherwise
-/// specified.  Thus, dynamic cardinalities always have an "immutable but
+/// specified. Thus, dynamic cardinalities always have an "immutable but
 /// unknown" value; so the term "dynamic" should not be taken to indicate
 /// run-time mutability.
 class SparseTensorStorageBase {
@@ -89,25 +68,10 @@ class SparseTensorStorageBase {
 
 public:
   /// Constructs a new sparse-tensor storage object with the given encoding.
-  ///
-  /// Preconditions:
-  /// * `dimSizes`, `lvlSizes`, `lvlTypes`, and `lvl2dim` must be nonnull.
-  /// * `dimSizes` must be valid for `dimRank`.
-  /// * `lvlSizes`, `lvlTypes`, and `lvl2dim` must be valid for `lvlRank`.
-  /// * `lvl2dim` must map `lvlSizes`-coordinates to `dimSizes`-coordinates.
-  ///
-  /// Asserts:
-  /// * `dimRank` and `lvlRank` are nonzero.
-  /// * `dimSizes` and `lvlSizes` contain only nonzero sizes.
   SparseTensorStorageBase(uint64_t dimRank, const uint64_t *dimSizes,
                           uint64_t lvlRank, const uint64_t *lvlSizes,
-                          const DimLevelType *lvlTypes,
+                          const DimLevelType *lvlTypes, const uint64_t *dim2lvl,
                           const uint64_t *lvl2dim);
-  // NOTE: For the most part we only need the `dimRank`.  But we need
-  // `dimSizes` for `toCOO` to support the identity permutation nicely
-  // (i.e., without the caller needing to already know the tensor's
-  // dimension-sizes; e.g., as in `fromMLIRSparseTensor`).
-
   virtual ~SparseTensorStorageBase() = default;
 
   /// Gets the number of tensor-dimensions.
@@ -121,7 +85,7 @@ class SparseTensorStorageBase {
 
   /// Safely looks up the size of the given tensor-dimension.
   uint64_t getDimSize(uint64_t d) const {
-    ASSERT_VALID_DIM(d);
+    assert(d < getDimRank());
     return dimSizes[d];
   }
 
@@ -130,19 +94,16 @@ class SparseTensorStorageBase {
 
   /// Safely looks up the size of the given storage-level.
   uint64_t getLvlSize(uint64_t l) const {
-    ASSERT_VALID_LVL(l);
+    assert(l < getLvlRank());
     return lvlSizes[l];
   }
 
-  /// Gets the level-to-dimension mapping.
-  const std::vector<uint64_t> &getLvl2Dim() const { return lvl2dim; }
-
   /// Gets the level-types array.
   const std::vector<DimLevelType> &getLvlTypes() const { return lvlTypes; }
 
   /// Safely looks up the type of the given level.
   DimLevelType getLvlType(uint64_t l) const {
-    ASSERT_VALID_LVL(l);
+    assert(l < getLvlRank());
     return lvlTypes[l];
   }
 
@@ -165,18 +126,6 @@ class SparseTensorStorageBase {
   /// Safely checks if the level is unique.
   bool isUniqueLvl(uint64_t l) const { return isUniqueDLT(getLvlType(l)); }
 
-  /// Allocates a new enumerator.  Callers must make sure to delete
-  /// the enumerator when they're done with it. The first argument
-  /// is the out-parameter for storing the newly allocated enumerator;
-  /// all other arguments are passed along to the `SparseTensorEnumerator`
-  /// ctor and must satisfy the preconditions/assertions thereof.
-#define DECL_NEWENUMERATOR(VNAME, V)                                           \
-  virtual void newEnumerator(SparseTensorEnumeratorBase<V> **, uint64_t,       \
-                             const uint64_t *, uint64_t, const uint64_t *)     \
-      const;
-  MLIR_SPARSETENSOR_FOREVERY_V(DECL_NEWENUMERATOR)
-#undef DECL_NEWENUMERATOR
-
   /// Gets positions-overhead storage for the given level.
 #define DECL_GETPOSITIONS(PNAME, P)                                            \
   virtual void getPositions(std::vector<P> **, uint64_t);
@@ -188,6 +137,7 @@ class SparseTensorStorageBase {
   virtual void getCoordinates(std::vector<C> **, uint64_t);
   MLIR_SPARSETENSOR_FOREVERY_FIXED_O(DECL_GETCOORDINATES)
 #undef DECL_GETCOORDINATES
+
   /// Gets the coordinate-value stored at the given level and position.
   virtual uint64_t getCrd(uint64_t lvl, uint64_t pos) const = 0;
 
@@ -196,6 +146,13 @@ class SparseTensorStorageBase {
   MLIR_SPARSETENSOR_FOREVERY_V(DECL_GETVALUES)
 #undef DECL_GETVALUES
 
+  /// Element-wise forwarding insertions. The first argument is the
+  /// dimension-coordinates for the value being inserted.
+#define DECL_FORWARDINGINSERT(VNAME, V)                                        \
+  virtual void forwardingInsert(const uint64_t *, V);
+  MLIR_SPARSETENSOR_FOREVERY_V(DECL_FORWARDINGINSERT)
+#undef DECL_FORWARDINGINSERT
+
   /// Element-wise insertion in lexicographic coordinate order. The first
   /// argument is the level-coordinates for the value being inserted.
 #define DECL_LEXINSERT(VNAME, V) virtual void lexInsert(const uint64_t *, V);
@@ -205,35 +162,32 @@ class SparseTensorStorageBase {
   /// Expanded insertion.  Note that this method resets the
   /// values/filled-switch array back to all-zero/false while only
   /// iterating over the nonzero elements.
-  ///
-  /// Arguments:
-  /// * `lvlCoords` the level-coordinates shared by the values being inserted.
-  /// * `values` a map from last-level coordinates to their associated value.
-  /// * `filled` a map from last-level coordinates to bool, indicating
-  ///   whether `values` contains a valid value to be inserted.
-  /// * `added` a map from `[0..count)` to last-level coordinates for
-  ///   which `filled` is true and `values` contains the assotiated value.
-  /// * `count` the size of `added`.
-  /// * `expsz` the size of the expanded vector (verification only).
 #define DECL_EXPINSERT(VNAME, V)                                               \
   virtual void expInsert(uint64_t *, V *, bool *, uint64_t *, uint64_t,        \
                          uint64_t);
   MLIR_SPARSETENSOR_FOREVERY_V(DECL_EXPINSERT)
 #undef DECL_EXPINSERT
 
-  /// Finishes insertion.
-  virtual void endInsert() = 0;
+  /// Finalizes forwarding insertions.
+  virtual void endForwardingInsert() = 0;
+
+  /// Finalizes lexicographic insertions.
+  virtual void endLexInsert() = 0;
 
 private:
   const std::vector<uint64_t> dimSizes;
   const std::vector<uint64_t> lvlSizes;
   const std::vector<DimLevelType> lvlTypes;
-  const std::vector<uint64_t> lvl2dim;
+  const std::vector<uint64_t> dim2lvlVec;
+  const std::vector<uint64_t> lvl2dimVec;
+
+protected:
+  const MapRef map; // non-owning pointers into dim2lvl/lvl2dim vectors
 };
 
 /// A memory-resident sparse tensor using a storage scheme based on
-/// per-level sparse/dense annotations.  This data structure provides
-/// a bufferized form of a sparse tensor type.  In contrast to generating
+/// per-level sparse/dense annotations. This data structure provides
+/// a bufferized form of a sparse tensor type. In contrast to generating
 /// setup methods for each differently annotated sparse tensor, this
 /// method provides a convenient "one-size-fits-all" solution that simply
 /// takes an input tensor and annotations to implement all required setup
@@ -244,58 +198,38 @@ class SparseTensorStorage final : public SparseTensorStorageBase {
   /// Beware that the object is not necessarily guaranteed to be in a
   /// valid state after this constructor alone; e.g., `isCompressedLvl(l)`
   /// doesn't entail `!(positions[l].empty())`.
-  ///
-  /// Preconditions/assertions are as per the `SparseTensorStorageBase` ctor.
   SparseTensorStorage(uint64_t dimRank, const uint64_t *dimSizes,
                       uint64_t lvlRank, const uint64_t *lvlSizes,
-                      const DimLevelType *lvlTypes, const uint64_t *lvl2dim)
+                      const DimLevelType *lvlTypes, const uint64_t *dim2lvl,
+                      const uint64_t *lvl2dim)
       : SparseTensorStorageBase(dimRank, dimSizes, lvlRank, lvlSizes, lvlTypes,
-                                lvl2dim),
-        positions(lvlRank), coordinates(lvlRank), lvlCursor(lvlRank) {}
+                                dim2lvl, lvl2dim),
+        positions(lvlRank), coordinates(lvlRank), lvlCursor(lvlRank), coo() {}
 
 public:
   /// Constructs a sparse tensor with the given encoding, and allocates
-  /// overhead storage according to some simple heuristics.  When the
+  /// overhead storage according to some simple heuristics. When the
   /// `bool` argument is true and `lvlTypes` are all dense, then this
-  /// ctor will also initialize the values array with zeros.  That
+  /// ctor will also initialize the values array with zeros. That
   /// argument should be true when an empty tensor is intended; whereas
   /// it should usually be false when the ctor will be followed up by
   /// some other form of initialization.
-  ///
-  /// Preconditions/assertions are as per the `SparseTensorStorageBase` ctor.
   SparseTensorStorage(uint64_t dimRank, const uint64_t *dimSizes,
                       uint64_t lvlRank, const uint64_t *lvlSizes,
-                      const DimLevelType *lvlTypes, const uint64_t *lvl2dim,
+                      const DimLevelType *lvlTypes, const uint64_t *dim2lvl,
+                      const uint64_t *lvl2dim, SparseTensorCOO<V> *lvlCOO,
                       bool initializeValuesIfAllDense);
 
   /// Constructs a sparse tensor with the given encoding, and initializes
-  /// the contents from the COO.  This ctor performs the same heuristic
-  /// overhead-storage allocation as the ctor taking a `bool`, and
-  /// has the same preconditions/assertions (where we define `lvlSizes =
-  /// lvlCOO.getDimSizes().data()`), with the following addition:
-  ///
-  /// Asserts:
-  /// * `lvlRank == lvlCOO.getRank()`.
+  /// the contents from the COO. This ctor performs the same heuristic
+  /// overhead-storage allocation as the ctor above.
   SparseTensorStorage(uint64_t dimRank, const uint64_t *dimSizes,
-                      uint64_t lvlRank, const DimLevelType *lvlTypes,
+                      uint64_t lvlRank, const uint64_t *lvlSizes,
+                      const DimLevelType *lvlTypes, const uint64_t *dim2lvl,
                       const uint64_t *lvl2dim, SparseTensorCOO<V> &lvlCOO);
 
   /// Constructs a sparse tensor with the given encoding, and initializes
-  /// the contents from the enumerator.  This ctor allocates exactly
-  /// the required amount of overhead storage, not using any heuristics.
-  /// Preconditions/assertions are as per the `SparseTensorStorageBase`
-  /// ctor (where we define `lvlSizes = lvlEnumerator.getTrgSizes().data()`),
-  /// with the following addition:
-  ///
-  /// Asserts:
-  /// * `lvlRank == lvlEnumerator.getTrgRank()`.
-  SparseTensorStorage(uint64_t dimRank, const uint64_t *dimSizes,
-                      uint64_t lvlRank, const DimLevelType *lvlTypes,
-                      const uint64_t *lvl2dim,
-                      SparseTensorEnumeratorBase<V> &lvlEnumerator);
-
-  /// Constructs a sparse tensor with the given encoding, and initializes
-  /// the contents from the level buffers.  This ctor allocates exactly
+  /// the contents from the level buffers. This ctor allocates exactly
   /// the required amount of overhead storage, not using any heuristics.
   /// It assumes that the data provided by `lvlBufs` can be directly used to
   /// interpret the result sparse tensor and performs *NO* integrity test on the
@@ -303,88 +237,29 @@ class SparseTensorStorage final : public SparseTensorStorageBase {
   /// passed in as a single AoS memory.
   SparseTensorStorage(uint64_t dimRank, const uint64_t *dimSizes,
                       uint64_t lvlRank, const uint64_t *lvlSizes,
-                      const DimLevelType *lvlTypes, const uint64_t *lvl2dim,
-                      const intptr_t *lvlBufs);
+                      const DimLevelType *lvlTypes, const uint64_t *dim2lvl,
+                      const uint64_t *lvl2dim, const intptr_t *lvlBufs);
 
-  /// Allocates a new empty sparse tensor. The preconditions/assertions
-  /// are as per the `SparseTensorStorageBase` ctor; which is to say,
-  /// the `dimSizes` and `lvlSizes` must both be "sizes" not "shapes",
-  /// since there's nowhere to reconstruct dynamic sizes from.
+  /// Allocates a new empty sparse tensor.
   static SparseTensorStorage<P, C, V> *
   newEmpty(uint64_t dimRank, const uint64_t *dimSizes, uint64_t lvlRank,
            const uint64_t *lvlSizes, const DimLevelType *lvlTypes,
-           const uint64_t *lvl2dim) {
-    return new SparseTensorStorage<P, C, V>(dimRank, dimSizes, lvlRank,
-                                            lvlSizes, lvlTypes, lvl2dim, true);
-  }
+           const uint64_t *dim2lvl, const uint64_t *lvl2dim, bool forwarding);
 
   /// Allocates a new sparse tensor and initializes it from the given COO.
-  /// The preconditions are as per the `SparseTensorStorageBase` ctor
-  /// (where we define `lvlSizes = lvlCOO.getDimSizes().data()`), but
-  /// using the following assertions in lieu of the base ctor's assertions:
-  ///
-  /// Asserts:
-  /// * `dimRank` and `lvlRank` are nonzero.
-  /// * `lvlRank == lvlCOO.getRank()`.
-  /// * `lvlCOO.getDimSizes()` under the `lvl2dim` mapping is a refinement
-  ///   of `dimShape`.
-  //
-  // TODO: The ability to reconstruct dynamic dimensions-sizes does not
-  // easily generalize to arbitrary `lvl2dim` mappings.  When compiling
-  // MLIR programs to use this library, we should be able to generate
-  // code for effectively computing the reconstruction, but it's not clear
-  // that there's a feasible way to do so from within the library itself.
-  // Therefore, when we functionalize the `lvl2dim` mapping we'll have
-  // to update the type/preconditions of this factory too.
   static SparseTensorStorage<P, C, V> *
-  newFromCOO(uint64_t dimRank, const uint64_t *dimShape, uint64_t lvlRank,
-             const DimLevelType *lvlTypes, const uint64_t *lvl2dim,
+  newFromCOO(uint64_t dimRank, const uint64_t *dimSizes, uint64_t lvlRank,
+             const uint64_t *lvlSizes, const DimLevelType *lvlTypes,
+             const uint64_t *dim2lvl, const uint64_t *lvl2dim,
              SparseTensorCOO<V> &lvlCOO);
 
-  /// Allocates a new sparse tensor and initializes it with the contents
-  /// of another sparse tensor.
-  ///
-  /// Preconditions:
-  /// * as per the `SparseTensorStorageBase` ctor.
-  /// * `src2lvl` must be valid for `srcRank`, must map coordinates valid
-  ///    for `source.getDimSizes()` to coordinates valid for `lvlSizes`,
-  ///    and therefore must be the inverse of `lvl2dim`.
-  /// * `source` must have the same value type `V`.
-  ///
-  /// Asserts:
-  /// * `dimRank` and `lvlRank` are nonzero.
-  /// * `srcRank == source.getDimRank()`.
-  /// * `lvlSizes` contains only nonzero sizes.
-  /// * `source.getDimSizes()` is a refinement of `dimShape`.
-  //
-  // TODO: The `dimRank` and `dimShape` arguments are only used for
-  // verifying that the source tensor has the expected shape.  So if we
-  // wanted to skip that verification, then we could remove those arguments.
-  // Alternatively, if we required the `dimShape` to be "sizes" instead,
-  // then that would remove any constraints on `source.getDimSizes()`
-  // (other than compatibility with `src2lvl`) as well as removing the
-  // requirement that `src2lvl` be the inverse of `lvl2dim`.  Which would
-  // enable this factory to be used for performing a much larger class of
-  // transformations (which can already be handled by the `SparseTensorNNZ`
-  // implementation).
-  static SparseTensorStorage<P, C, V> *
-  newFromSparseTensor(uint64_t dimRank, const uint64_t *dimShape,
-                      uint64_t lvlRank, const uint64_t *lvlSizes,
-                      const DimLevelType *lvlTypes, const uint64_t *lvl2dim,
-                      uint64_t srcRank, const uint64_t *src2lvl,
-                      const SparseTensorStorageBase &source);
-
   /// Allocates a new sparse tensor and initialize it with the data stored level
   /// buffers directly.
-  ///
-  /// Precondition:
-  /// * as per the `SparseTensorStorageBase` ctor.
-  /// * the data integrity stored in `buffers` is guaranteed by users already.
   static SparseTensorStorage<P, C, V> *
-  packFromLvlBuffers(uint64_t dimRank, const uint64_t *dimShape,
+  packFromLvlBuffers(uint64_t dimRank, const uint64_t *dimSizes,
                      uint64_t lvlRank, const uint64_t *lvlSizes,
-                     const DimLevelType *lvlTypes, const uint64_t *lvl2dim,
-                     uint64_t srcRank, const uint64_t *src2lvl,
+                     const DimLevelType *lvlTypes, const uint64_t *dim2lvl,
+                     const uint64_t *lvl2dim, uint64_t srcRank,
                      const intptr_t *buffers);
 
   ~SparseTensorStorage() final = default;
@@ -392,12 +267,12 @@ class SparseTensorStorage final : public SparseTensorStorageBase {
   /// Partially specialize these getter methods based on template types.
   void getPositions(std::vector<P> **out, uint64_t lvl) final {
     assert(out && "Received nullptr for out parameter");
-    ASSERT_VALID_LVL(lvl);
+    assert(lvl < getLvlRank());
     *out = &positions[lvl];
   }
   void getCoordinates(std::vector<C> **out, uint64_t lvl) final {
     assert(out && "Received nullptr for out parameter");
-    ASSERT_VALID_LVL(lvl);
+    assert(lvl < getLvlRank());
     *out = &coordinates[lvl];
   }
   void getValues(std::vector<V> **out) final {
@@ -405,15 +280,34 @@ class SparseTensorStorage final : public SparseTensorStorageBase {
     *out = &values;
   }
 
+  /// Returns coordinate at given position.
   uint64_t getCrd(uint64_t lvl, uint64_t pos) const final {
-    ASSERT_COMPRESSED_OR_SINGLETON_LVL(lvl);
-    assert(pos < coordinates[lvl].size() && "Position is out of bounds");
+    assert(isCompressedDLT(getLvlType(lvl)) || isSingletonDLT(getLvlType(lvl)));
+    assert(pos < coordinates[lvl].size());
     return coordinates[lvl][pos]; // Converts the stored `C` into `uint64_t`.
   }
 
+  /// Partially specialize forwarding insertions based on template types.
+  void forwardingInsert(const uint64_t *dimCoords, V val) final {
+    assert(dimCoords && coo);
+    map.pushforward(dimCoords, lvlCursor.data());
+    coo->add(lvlCursor, val);
+  }
+
   /// Partially specialize lexicographical insertions based on template types.
   void lexInsert(const uint64_t *lvlCoords, V val) final {
-    assert(lvlCoords && "Received nullptr for level-coordinates");
+    assert(lvlCoords);
+    bool allDense = std::all_of(getLvlTypes().begin(), getLvlTypes().end(),
+                                [](DimLevelType lt) { return isDenseDLT(lt); });
+    if (allDense) {
+      uint64_t lvlRank = getLvlRank();
+      uint64_t valIdx = 0;
+      // Linearize the address
+      for (uint64_t lvl = 0; lvl < lvlRank; lvl++)
+        valIdx = valIdx * getLvlSize(lvl) + lvlCoords[lvl];
+      values[valIdx] = val;
+      return;
+    }
     // First, wrap up pending insertion path.
     uint64_t diffLvl = 0;
     uint64_t full = 0;
@@ -444,7 +338,7 @@ class SparseTensorStorage final : public SparseTensorStorageBase {
     values[c] = 0;
     filled[c] = false;
     // Subsequent insertions are quick.
-    for (uint64_t i = 1; i < count; ++i) {
+    for (uint64_t i = 1; i < count; i++) {
       assert(c < added[i] && "non-lexicographic insertion");
       c = added[i];
       assert(c <= expsz);
@@ -456,54 +350,103 @@ class SparseTensorStorage final : public SparseTensorStorageBase {
     }
   }
 
+  /// Finalizes forwarding insertions.
+  void endForwardingInsert() final {
+    // Ensure COO is sorted.
+    assert(coo);
+    coo->sort();
+    // Now actually insert the `elements`.
+    const auto &elements = coo->getElements();
+    const uint64_t nse = elements.size();
+    assert(values.size() == 0);
+    values.reserve(nse);
+    fromCOO(elements, 0, nse, 0);
+    delete coo;
+    coo = nullptr;
+  }
+
   /// Finalizes lexicographic insertions.
-  void endInsert() final {
+  void endLexInsert() final {
     if (values.empty())
       finalizeSegment(0);
     else
       endPath(0);
   }
 
-  /// Allocates a new enumerator for this class's `<P,C,V>` types and
-  /// erase the `<P,C>` parts from the type.  Callers must make sure to
-  /// delete the enumerator when they're done with it.
-  void newEnumerator(SparseTensorEnumeratorBase<V> **out, uint64_t trgRank,
-                     const uint64_t *trgSizes, uint64_t srcRank,
-                     const uint64_t *src2trg) const final {
-    assert(out && "Received nullptr for out parameter");
-    *out = new SparseTensorEnumerator<P, C, V>(*this, trgRank, trgSizes,
-                                               srcRank, src2trg);
-  }
-
-  /// Allocates a new COO object and initializes it with the contents
-  /// of this tensor under the given mapping from the `getDimSizes()`
-  /// coordinate-space to the `trgSizes` coordinate-space.  Callers must
-  /// make sure to delete the COO when they're done with it.
-  ///
-  /// Preconditions/assertions are as per the `SparseTensorEnumerator` ctor.
-  SparseTensorCOO<V> *toCOO(uint64_t trgRank, const uint64_t *trgSizes,
-                            uint64_t srcRank, const uint64_t *src2trg) const {
-    // We inline `newEnumerator` to avoid virtual dispatch and allocation.
-    // TODO: use MapRef here too for the translation
-    SparseTensorEnumerator<P, C, V> enumerator(*this, trgRank, trgSizes,
-                                               srcRank, src2trg);
-    auto *coo = new SparseTensorCOO<V>(trgRank, trgSizes, values.size());
-    enumerator.forallElements(
-        [&coo](const auto &trgCoords, V val) { coo->add(trgCoords, val); });
-    // TODO: This assertion assumes there are no stored zeros,
-    // or if there are then that we don't filter them out.
-    // <https://github.com/llvm/llvm-project/issues/54179>
+  /// Allocates a new COO object and initializes it with the contents.
+  /// Callers must make sure to delete the COO when they're done with it.
+  SparseTensorCOO<V> *toCOO() {
+    std::vector<uint64_t> dimCoords(getDimRank());
+    coo = new SparseTensorCOO<V>(getDimSizes(), values.size());
+    toCOO(0, 0, dimCoords);
     assert(coo->getElements().size() == values.size());
     return coo;
   }
 
+  /// Sort the unordered tensor in place, the method assumes that it is
+  /// an unordered COO tensor.
+  void sortInPlace() {
+    uint64_t nnz = values.size();
+#ifndef NDEBUG
+    for (uint64_t l = 0; l < getLvlRank(); l++)
+      assert(nnz == coordinates[l].size());
+#endif
+
+    // In-place permutation.
+    auto applyPerm = [this](std::vector<uint64_t> &perm) {
+      uint64_t length = perm.size();
+      uint64_t lvlRank = getLvlRank();
+      // Cache for the current level coordinates.
+      std::vector<P> lvlCrds(lvlRank);
+      for (uint64_t i = 0; i < length; i++) {
+        uint64_t current = i;
+        if (i != perm[current]) {
+          for (uint64_t l = 0; l < lvlRank; l++)
+            lvlCrds[l] = coordinates[l][i];
+          V val = values[i];
+          // Deals with a permutation cycle.
+          while (i != perm[current]) {
+            uint64_t next = perm[current];
+            // Swaps the level coordinates and value.
+            for (uint64_t l = 0; l < lvlRank; l++)
+              coordinates[l][current] = coordinates[l][next];
+            values[current] = values[next];
+            perm[current] = current;
+            current = next;
+          }
+          for (uint64_t l = 0; l < lvlRank; l++)
+            coordinates[l][current] = lvlCrds[l];
+          values[current] = val;
+          perm[current] = current;
+        }
+      }
+    };
+
+    std::vector<uint64_t> sortedIdx(nnz, 0);
+    for (uint64_t i = 0; i < nnz; i++)
+      sortedIdx[i] = i;
+
+    std::sort(sortedIdx.begin(), sortedIdx.end(),
+              [this](uint64_t lhs, uint64_t rhs) {
+                for (uint64_t l = 0; l < getLvlRank(); l++) {
+                  if (coordinates[l][lhs] == coordinates[l][rhs])
+                    continue;
+                  return coordinates[l][lhs] < coordinates[l][rhs];
+                }
+                assert(lhs == rhs && "duplicate coordinates");
+                return false;
+              });
+
+    applyPerm(sortedIdx);
+  }
+
 private:
   /// Appends an arbitrary new position to `positions[lvl]`.  This method
   /// checks that `pos` is representable in the `P` type; however, it
   /// does not check that `pos` is semantically valid (i.e., larger than
   /// the previous position and smaller than `coordinates[lvl].capacity()`).
   void appendPos(uint64_t lvl, uint64_t pos, uint64_t count = 1) {
-    ASSERT_COMPRESSED_LVL(lvl);
+    assert(isCompressedLvl(lvl));
     positions[lvl].insert(positions[lvl].end(), count,
                           detail::checkOverflowCast<P>(pos));
   }
@@ -522,7 +465,7 @@ class SparseTensorStorage final : public SparseTensorStorageBase {
     if (isCompressedDLT(dlt) || isSingletonDLT(dlt)) {
       coordinates[lvl].push_back(detail::checkOverflowCast<C>(crd));
     } else { // Dense level.
-      ASSERT_DENSE_DLT(dlt);
+      assert(isDenseDLT(dlt));
       assert(crd >= full && "Coordinate was already filled");
       if (crd == full)
         return; // Short-circuit, since it'll be a nop.
@@ -533,27 +476,11 @@ class SparseTensorStorage final : public SparseTensorStorageBase {
     }
   }
 
-  /// Writes the given coordinate to `coordinates[lvl][pos]`.  This method
-  /// checks that `crd` is representable in the `C` type; however, it
-  /// does not check that `crd` is semantically valid (i.e., in bounds
-  /// for `dimSizes[lvl]` and not elsewhere occurring in the same segment).
-  void writeCrd(uint64_t lvl, uint64_t pos, uint64_t crd) {
-    ASSERT_COMPRESSED_OR_SINGLETON_LVL(lvl);
-    // Subscript assignment to `std::vector` requires that the `pos`-th
-    // entry has been initialized; thus we must be sure to check `size()`
-    // here, instead of `capacity()` as would be ideal.
-    assert(pos < coordinates[lvl].size() && "Position is out of bounds");
-    coordinates[lvl][pos] = detail::checkOverflowCast<C>(crd);
-  }
-
   /// Computes the assembled-size associated with the `l`-th level,
   /// given the assembled-size associated with the `(l-1)`-th level.
   /// "Assembled-sizes" correspond to the (nominal) sizes of overhead
   /// storage, as opposed to "level-sizes" which are the cardinality
   /// of possible coordinates for that level.
-  ///
-  /// Precondition: the `positions[l]` array must be fully initialized
-  /// before calling this method.
   uint64_t assembledSize(uint64_t parentSz, uint64_t l) const {
     const auto dlt = getLvlType(l); // Avoid redundant bounds checking.
     if (isCompressedDLT(dlt))
@@ -561,7 +488,7 @@ class SparseTensorStorage final : public SparseTensorStorageBase {
     if (isSingletonDLT(dlt))
       return parentSz; // New size is same as the parent.
     if (isDenseDLT(dlt))
-      return parentSz * getLvlSizes()[l];
+      return parentSz * getLvlSize(l);
     MLIR_SPARSETENSOR_FATAL("unsupported level type: %d\n",
                             static_cast<uint8_t>(dlt));
   }
@@ -569,11 +496,6 @@ class SparseTensorStorage final : public SparseTensorStorageBase {
   /// Initializes sparse tensor storage scheme from a memory-resident sparse
   /// tensor in coordinate scheme. This method prepares the positions and
   /// coordinates arrays under the given per-level dense/sparse annotations.
-  ///
-  /// Preconditions:
-  /// * the `lvlElements` must be lexicographically sorted.
-  /// * the coordinates of every element are valid for `getLvlSizes()`
-  ///   (i.e., equal rank and pointwise less-than).
   void fromCOO(const std::vector<Element<V>> &lvlElements, uint64_t lo,
                uint64_t hi, uint64_t l) {
     const uint64_t lvlRank = getLvlRank();
@@ -614,7 +536,7 @@ class SparseTensorStorage final : public SparseTensorStorageBase {
     } else if (isSingletonDLT(dlt)) {
       return; // Nothing to finalize.
     } else {  // Dense dimension.
-      ASSERT_DENSE_DLT(dlt);
+      assert(isDenseDLT(dlt));
       const uint64_t sz = getLvlSizes()[l];
       assert(sz >= full && "Segment is overfull");
       count = detail::checkedMul(count, sz - full);
@@ -633,9 +555,9 @@ class SparseTensorStorage final : public SparseTensorStorageBase {
   void endPath(uint64_t diffLvl) {
     const uint64_t lvlRank = getLvlRank();
     const uint64_t lastLvl = lvlRank - 1;
-    assert(diffLvl <= lvlRank && "Level-diff is out of bounds");
+    assert(diffLvl <= lvlRank);
     const uint64_t stop = lvlRank - diffLvl;
-    for (uint64_t i = 0; i < stop; ++i) {
+    for (uint64_t i = 0; i < stop; i++) {
       const uint64_t l = lastLvl - i;
       finalizeSegment(l, lvlCursor[l] + 1);
     }
@@ -646,8 +568,8 @@ class SparseTensorStorage final : public SparseTensorStorageBase {
   void insPath(const uint64_t *lvlCoords, uint64_t diffLvl, uint64_t full,
                V val) {
     const uint64_t lvlRank = getLvlRank();
-    assert(diffLvl <= lvlRank && "Level-diff is out of bounds");
-    for (uint64_t l = diffLvl; l < lvlRank; ++l) {
+    assert(diffLvl <= lvlRank);
+    for (uint64_t l = diffLvl; l < lvlRank; l++) {
       const uint64_t c = lvlCoords[l];
       appendCrd(l, full, c);
       full = 0;
@@ -660,7 +582,7 @@ class SparseTensorStorage final : public SparseTensorStorageBase {
   /// in the argument differ from those in the current cursor.
   uint64_t lexDiff(const uint64_t *lvlCoords) const {
     const uint64_t lvlRank = getLvlRank();
-    for (uint64_t l = 0; l < lvlRank; ++l) {
+    for (uint64_t l = 0; l < lvlRank; l++) {
       const auto crd = lvlCoords[l];
       const auto cur = lvlCursor[l];
       if (crd > cur || (crd == cur && !isUniqueLvl(l)) ||
@@ -677,331 +599,105 @@ class SparseTensorStorage final : public SparseTensorStorageBase {
     return -1u;
   }
 
-  // Allow `SparseTensorEnumerator` to access the data-members (to avoid
-  // the cost of virtual-function dispatch in inner loops), without
-  // making them public to other client code.
-  friend class SparseTensorEnumerator<P, C, V>;
-
-  std::vector<std::vector<P>> positions;
-  std::vector<std::vector<C>> coordinates;
-  std::vector<V> values;
-  std::vector<uint64_t> lvlCursor; // cursor for lexicographic insertion.
-};
-
-#undef ASSERT_COMPRESSED_OR_SINGLETON_LVL
-#undef ASSERT_COMPRESSED_LVL
-#undef ASSERT_VALID_LVL
-#undef ASSERT_VALID_DIM
-
-//===----------------------------------------------------------------------===//
-/// A (higher-order) function object for enumerating the elements of some
-/// `SparseTensorStorage` under a permutation.  That is, the `forallElements`
-/// method encapsulates the loop-nest for enumerating the elements of
-/// the source tensor (in whatever order is best for the source tensor),
-/// and applies a permutation to the coordinates before handing
-/// each element to the callback.  A single enumerator object can be
-/// freely reused for several calls to `forallElements`, just so long
-/// as each call is sequential with respect to one another.
-///
-/// N.B., this class stores a reference to the `SparseTensorStorageBase`
-/// passed to the constructor; thus, objects of this class must not
-/// outlive the sparse tensor they depend on.
-///
-/// Design Note: The reason we define this class instead of simply using
-/// `SparseTensorEnumerator<P,C,V>` is because we need to hide/generalize
-/// the `<P,C>` template parameters from MLIR client code (to simplify the
-/// type parameters used for direct sparse-to-sparse conversion).  And the
-/// reason we define the `SparseTensorEnumerator<P,C,V>` subclasses rather
-/// than simply using this class, is to avoid the cost of virtual-method
-/// dispatch within the loop-nest.
-template <typename V>
-class SparseTensorEnumeratorBase {
-public:
-  /// Constructs an enumerator which automatically applies the given
-  /// mapping from the source tensor's dimensions to the desired
-  /// target tensor dimensions.
-  ///
-  /// Preconditions:
-  /// * the `src` must have the same `V` value type.
-  /// * `trgSizes` must be valid for `trgRank`.
-  /// * `src2trg` must be valid for `srcRank`, and must map coordinates
-  ///   valid for `src.getDimSizes()` to coordinates valid for `trgSizes`.
-  ///
-  /// Asserts:
-  /// * `trgSizes` must be nonnull and must contain only nonzero sizes.
-  /// * `srcRank == src.getDimRank()`.
-  /// * `src2trg` must be nonnull.
-  SparseTensorEnumeratorBase(const SparseTensorStorageBase &src,
-                             uint64_t trgRank, const uint64_t *trgSizes,
-                             uint64_t srcRank, const uint64_t *src2trg)
-      : src(src), trgSizes(trgSizes, trgSizes + trgRank),
-        lvl2trg(src.getLvlRank()), trgCursor(trgRank) {
-    assert(trgSizes && "Received nullptr for target-sizes");
-    assert(src2trg && "Received nullptr for source-to-target mapping");
-    assert(srcRank == src.getDimRank() && "Source-rank mismatch");
-    for (uint64_t t = 0; t < trgRank; ++t)
-      assert(trgSizes[t] > 0 && "Target-size zero has trivial storage");
-    const auto &lvl2src = src.getLvl2Dim();
-    for (uint64_t lvlRank = src.getLvlRank(), l = 0; l < lvlRank; ++l)
-      lvl2trg[l] = src2trg[lvl2src[l]];
-  }
-
-  virtual ~SparseTensorEnumeratorBase() = default;
-
-  // We disallow copying to help avoid leaking the `src` reference.
-  // (In addition to avoiding the problem of slicing.)
-  SparseTensorEnumeratorBase(const SparseTensorEnumeratorBase &) = delete;
-  SparseTensorEnumeratorBase &
-  operator=(const SparseTensorEnumeratorBase &) = delete;
-
-  /// Gets the source's dimension-rank.
-  uint64_t getSrcDimRank() const { return src.getDimRank(); }
-
-  /// Gets the target's dimension-/level-rank.  (This is usually
-  /// "dimension-rank", though that may coincide with "level-rank"
-  /// depending on usage.)
-  uint64_t getTrgRank() const { return trgSizes.size(); }
-
-  /// Gets the target's dimension-/level-sizes.  (These are usually
-  /// "dimensions", though that may coincide with "level-rank" depending
-  /// on usage.)
-  const std::vector<uint64_t> &getTrgSizes() const { return trgSizes; }
-
-  /// Enumerates all elements of the source tensor, permutes their
-  /// coordinates, and passes the permuted element to the callback.
-  /// The callback must not store the cursor reference directly,
-  /// since this function reuses the storage.  Instead, the callback
-  /// must copy it if they want to keep it.
-  virtual void forallElements(ElementConsumer<V> yield) = 0;
-
-protected:
-  const SparseTensorStorageBase &src;
-  std::vector<uint64_t> trgSizes;  // in target order.
-  std::vector<uint64_t> lvl2trg;   // source-levels -> target-dims/lvls.
-  std::vector<uint64_t> trgCursor; // in target order.
-};
-
-//===----------------------------------------------------------------------===//
-template <typename P, typename C, typename V>
-class SparseTensorEnumerator final : public SparseTensorEnumeratorBase<V> {
-  using Base = SparseTensorEnumeratorBase<V>;
-  using StorageImpl = SparseTensorStorage<P, C, V>;
-
-public:
-  /// Constructs an enumerator which automatically applies the given
-  /// mapping from the source tensor's dimensions to the desired
-  /// target tensor dimensions.
-  ///
-  /// Preconditions/assertions are as per the `SparseTensorEnumeratorBase` ctor.
-  SparseTensorEnumerator(const StorageImpl &src, uint64_t trgRank,
-                         const uint64_t *trgSizes, uint64_t srcRank,
-                         const uint64_t *src2trg)
-      : Base(src, trgRank, trgSizes, srcRank, src2trg) {}
-
-  ~SparseTensorEnumerator() final = default;
-
-  void forallElements(ElementConsumer<V> yield) final {
-    forallElements(yield, 0, 0);
-  }
-
-private:
-  // TODO: Once we functionalize the mappings, then we'll no longer
-  // be able to use the current approach of constructing `lvl2trg` in the
-  // ctor and using it to incrementally fill the `trgCursor` cursor as we
-  // recurse through `forallElements`.  Instead we'll want to incrementally
-  // fill a `lvlCursor` as we recurse, and then use `src.getLvl2Dim()`
-  // and `src2trg` to convert it just before yielding to the callback.
-  // It's probably most efficient to just store the `srcCursor` and
-  // `trgCursor` buffers in this object, but we may want to benchmark
-  // that against using `std::calloc` to stack-allocate them instead.
-  //
-  /// The recursive component of the public `forallElements`.
-  void forallElements(ElementConsumer<V> yield, uint64_t parentPos,
-                      uint64_t l) {
-    // Recover the `<P,C,V>` type parameters of `src`.
-    const auto &src = static_cast<const StorageImpl &>(this->src);
-    if (l == src.getLvlRank()) {
-      assert(parentPos < src.values.size() &&
-             "Value position is out of bounds");
-      // TODO: <https://github.com/llvm/llvm-project/issues/54179>
-      yield(this->trgCursor, src.values[parentPos]);
+  // Performs forall on level entries and inserts into dim COO.
+  void toCOO(uint64_t parentPos, uint64_t l, std::vector<uint64_t> &dimCoords) {
+    if (l == getLvlRank()) {
+      map.pushbackward(lvlCursor.data(), dimCoords.data());
+      assert(coo);
+      assert(parentPos < values.size());
+      coo->add(dimCoords, values[parentPos]);
       return;
     }
-    uint64_t &cursorL = this->trgCursor[this->lvl2trg[l]];
-    const auto dlt = src.getLvlType(l); // Avoid redundant bounds checking.
-    if (isCompressedDLT(dlt)) {
+    if (isCompressedLvl(l)) {
       // Look up the bounds of the `l`-level segment determined by the
       // `(l - 1)`-level position `parentPos`.
-      const std::vector<P> &positionsL = src.positions[l];
-      assert(parentPos + 1 < positionsL.size() &&
-             "Parent position is out of bounds");
+      const std::vector<P> &positionsL = positions[l];
+      assert(parentPos + 1 < positionsL.size());
       const uint64_t pstart = static_cast<uint64_t>(positionsL[parentPos]);
       const uint64_t pstop = static_cast<uint64_t>(positionsL[parentPos + 1]);
       // Loop-invariant code for looking up the `l`-level coordinates.
-      const std::vector<C> &coordinatesL = src.coordinates[l];
-      assert(pstop <= coordinatesL.size() && "Stop position is out of bounds");
+      const std::vector<C> &coordinatesL = coordinates[l];
+      assert(pstop <= coordinatesL.size());
       for (uint64_t pos = pstart; pos < pstop; ++pos) {
-        cursorL = static_cast<uint64_t>(coordinatesL[pos]);
-        forallElements(yield, pos, l + 1);
+        lvlCursor[l] = static_cast<uint64_t>(coordinatesL[pos]);
+        toCOO(pos, l + 1, dimCoords);
       }
-    } else if (isSingletonDLT(dlt)) {
-      cursorL = src.getCrd(l, parentPos);
-      forallElements(yield, parentPos, l + 1);
+    } else if (isSingletonLvl(l)) {
+      lvlCursor[l] = getCrd(l, parentPos);
+      toCOO(parentPos, l + 1, dimCoords);
     } else { // Dense level.
-      ASSERT_DENSE_DLT(dlt);
-      const uint64_t sz = src.getLvlSizes()[l];
+      assert(isDenseLvl(l));
+      const uint64_t sz = getLvlSizes()[l];
       const uint64_t pstart = parentPos * sz;
       for (uint64_t c = 0; c < sz; ++c) {
-        cursorL = c;
-        forallElements(yield, pstart + c, l + 1);
+        lvlCursor[l] = c;
+        toCOO(pstart + c, l + 1, dimCoords);
       }
     }
   }
-};
-
-//===----------------------------------------------------------------------===//
-/// Statistics regarding the number of nonzero subtensors in
-/// a source tensor, for direct sparse=>sparse conversion a la
-/// <https://arxiv.org/abs/2001.02609>.
-///
-/// N.B., this class stores references to the parameters passed to
-/// the constructor; thus, objects of this class must not outlive
-/// those parameters.
-///
-/// This class does not have the "dimension" vs "level" distinction, but
-/// since it is used for initializing the levels of a `SparseTensorStorage`
-/// object, we use the "level" name throughout for the sake of consistency.
-class SparseTensorNNZ final {
-public:
-  /// Allocates the statistics structure for the desired target-tensor
-  /// level structure (i.e., sizes and types).  This constructor does not
-  /// actually populate the statistics, however; for that see `initialize`.
-  ///
-  /// Precondition: `lvlSizes` must not contain zeros.
-  /// Asserts: `lvlSizes.size() == lvlTypes.size()`.
-  SparseTensorNNZ(const std::vector<uint64_t> &lvlSizes,
-                  const std::vector<DimLevelType> &lvlTypes);
-
-  // We disallow copying to help avoid leaking the stored references.
-  SparseTensorNNZ(const SparseTensorNNZ &) = delete;
-  SparseTensorNNZ &operator=(const SparseTensorNNZ &) = delete;
-
-  /// Gets the target-tensor's level-rank.
-  uint64_t getLvlRank() const { return lvlSizes.size(); }
 
-  /// Enumerates the source tensor to fill in the statistics.
-  /// The enumerator should already incorporate the mapping from
-  /// the source tensor-dimensions to the target storage-levels.
-  ///
-  /// Asserts:
-  /// * `enumerator.getTrgRank() == getLvlRank()`.
-  /// * `enumerator.getTrgSizes() == lvlSizes`.
-  template <typename V>
-  void initialize(SparseTensorEnumeratorBase<V> &enumerator) {
-    assert(enumerator.getTrgRank() == getLvlRank() && "Tensor rank mismatch");
-    assert(enumerator.getTrgSizes() == lvlSizes && "Tensor size mismatch");
-    enumerator.forallElements(
-        [this](const std::vector<uint64_t> &lvlCoords, V) { add(lvlCoords); });
-  }
-
-  /// The type of callback functions which receive an nnz-statistic.
-  using NNZConsumer = const std::function<void(uint64_t)> &;
-
-  /// Lexicographically enumerates all coordinates for levels strictly
-  /// less than `stopLvl`, and passes their nnz statistic to the callback.
-  /// Since our use-case only requires the statistic not the coordinates
-  /// themselves, we do not bother to construct those coordinates.
-  void forallCoords(uint64_t stopLvl, NNZConsumer yield) const;
-
-private:
-  /// Adds a new element (i.e., increment its statistics).  We use
-  /// a method rather than inlining into the lambda in `initialize`,
-  /// to avoid spurious templating over `V`.  And this method is private
-  /// to avoid needing to re-assert validity of `lvlCoords` (which is
-  /// guaranteed by `forallElements`).
-  void add(const std::vector<uint64_t> &lvlCoords);
-
-  /// Recursive component of the public `forallCoords`.
-  void forallCoords(NNZConsumer yield, uint64_t stopLvl, uint64_t parentPos,
-                    uint64_t l) const;
-
-  // All of these are in the target storage-order.
-  const std::vector<uint64_t> &lvlSizes;
-  const std::vector<DimLevelType> &lvlTypes;
-  std::vector<std::vector<uint64_t>> nnz;
+  std::vector<std::vector<P>> positions;
+  std::vector<std::vector<C>> coordinates;
+  std::vector<V> values;
+  std::vector<uint64_t> lvlCursor;
+  SparseTensorCOO<V> *coo;
 };
 
 //===----------------------------------------------------------------------===//
-// Definitions of the ctors and factories of `SparseTensorStorage<P,C,V>`.
+//
+//  SparseTensorStorage Factories
+//
+//===----------------------------------------------------------------------===//
 
 template <typename P, typename C, typename V>
-SparseTensorStorage<P, C, V> *SparseTensorStorage<P, C, V>::newFromCOO(
-    uint64_t dimRank, const uint64_t *dimShape, uint64_t lvlRank,
-    const DimLevelType *lvlTypes, const uint64_t *lvl2dim,
-    SparseTensorCOO<V> &lvlCOO) {
-  assert(dimShape && "Got nullptr for dimension shape");
-  assert(lvl2dim && "Got nullptr for level-to-dimension mapping");
-  const auto &lvlSizes = lvlCOO.getDimSizes();
-  assert(lvlRank == lvlSizes.size() && "Level-rank mismatch");
-  // Must reconstruct `dimSizes` from `lvlSizes`.  While this is easy
-  // enough to do when `lvl2dim` is a permutation, this approach will
-  // not work for more general mappings; so we will need to move this
-  // computation off to codegen.
-  std::vector<uint64_t> dimSizes(dimRank);
-  for (uint64_t l = 0; l < lvlRank; ++l) {
-    const uint64_t d = lvl2dim[l];
-    assert((dimShape[d] == 0 || dimShape[d] == lvlSizes[l]) &&
-           "Dimension sizes do not match expected shape");
-    dimSizes[d] = lvlSizes[l];
-  }
-  return new SparseTensorStorage<P, C, V>(dimRank, dimSizes.data(), lvlRank,
-                                          lvlTypes, lvl2dim, lvlCOO);
+SparseTensorStorage<P, C, V> *SparseTensorStorage<P, C, V>::newEmpty(
+    uint64_t dimRank, const uint64_t *dimSizes, uint64_t lvlRank,
+    const uint64_t *lvlSizes, const DimLevelType *lvlTypes,
+    const uint64_t *dim2lvl, const uint64_t *lvl2dim, bool forwarding) {
+  SparseTensorCOO<V> *lvlCOO = nullptr;
+  if (forwarding)
+    lvlCOO = new SparseTensorCOO<V>(lvlRank, lvlSizes);
+  return new SparseTensorStorage<P, C, V>(dimRank, dimSizes, lvlRank, lvlSizes,
+                                          lvlTypes, dim2lvl, lvl2dim, lvlCOO,
+                                          !forwarding);
 }
 
 template <typename P, typename C, typename V>
-SparseTensorStorage<P, C, V> *SparseTensorStorage<P, C, V>::newFromSparseTensor(
-    uint64_t dimRank, const uint64_t *dimShape, uint64_t lvlRank,
+SparseTensorStorage<P, C, V> *SparseTensorStorage<P, C, V>::newFromCOO(
+    uint64_t dimRank, const uint64_t *dimSizes, uint64_t lvlRank,
     const uint64_t *lvlSizes, const DimLevelType *lvlTypes,
-    const uint64_t *lvl2dim, uint64_t srcRank, const uint64_t *src2lvl,
-    const SparseTensorStorageBase &source) {
-  // Verify that the `source` dimensions match the expected `dimShape`.
-  assert(dimShape && "Got nullptr for dimension shape");
-  assert(dimRank == source.getDimRank() && "Dimension-rank mismatch");
-  const auto &dimSizes = source.getDimSizes();
-#ifndef NDEBUG
-  for (uint64_t d = 0; d < dimRank; ++d) {
-    const uint64_t sz = dimShape[d];
-    assert((sz == 0 || sz == dimSizes[d]) &&
-           "Dimension-sizes do not match expected shape");
-  }
-#endif
-  SparseTensorEnumeratorBase<V> *lvlEnumerator;
-  source.newEnumerator(&lvlEnumerator, lvlRank, lvlSizes, srcRank, src2lvl);
-  auto *tensor = new SparseTensorStorage<P, C, V>(
-      dimRank, dimSizes.data(), lvlRank, lvlTypes, lvl2dim, *lvlEnumerator);
-  delete lvlEnumerator;
-  return tensor;
+    const uint64_t *dim2lvl, const uint64_t *lvl2dim,
+    SparseTensorCOO<V> &lvlCOO) {
+  return new SparseTensorStorage<P, C, V>(dimRank, dimSizes, lvlRank, lvlSizes,
+                                          lvlTypes, dim2lvl, lvl2dim, lvlCOO);
 }
 
 template <typename P, typename C, typename V>
 SparseTensorStorage<P, C, V> *SparseTensorStorage<P, C, V>::packFromLvlBuffers(
-    uint64_t dimRank, const uint64_t *dimShape, uint64_t lvlRank,
+    uint64_t dimRank, const uint64_t *dimSizes, uint64_t lvlRank,
     const uint64_t *lvlSizes, const DimLevelType *lvlTypes,
-    const uint64_t *lvl2dim, uint64_t srcRank, const uint64_t *src2lvl,
+    const uint64_t *dim2lvl, const uint64_t *lvl2dim, uint64_t srcRank,
     const intptr_t *buffers) {
-  assert(dimShape && "Got nullptr for dimension shape");
-  auto *tensor = new SparseTensorStorage<P, C, V>(
-      dimRank, dimShape, lvlRank, lvlSizes, lvlTypes, lvl2dim, buffers);
-  return tensor;
+  return new SparseTensorStorage<P, C, V>(dimRank, dimSizes, lvlRank, lvlSizes,
+                                          lvlTypes, dim2lvl, lvl2dim, buffers);
 }
 
+//===----------------------------------------------------------------------===//
+//
+//  SparseTensorStorage Constructors
+//
+//===----------------------------------------------------------------------===//
+
 template <typename P, typename C, typename V>
 SparseTensorStorage<P, C, V>::SparseTensorStorage(
     uint64_t dimRank, const uint64_t *dimSizes, uint64_t lvlRank,
     const uint64_t *lvlSizes, const DimLevelType *lvlTypes,
-    const uint64_t *lvl2dim, bool initializeValuesIfAllDense)
+    const uint64_t *dim2lvl, const uint64_t *lvl2dim,
+    SparseTensorCOO<V> *lvlCOO, bool initializeValuesIfAllDense)
     : SparseTensorStorage(dimRank, dimSizes, lvlRank, lvlSizes, lvlTypes,
-                          lvl2dim) {
+                          dim2lvl, lvl2dim) {
+  assert(!lvlCOO || lvlRank == lvlCOO->getRank());
+  coo = lvlCOO;
   // Provide hints on capacity of positions and coordinates.
   // TODO: needs much fine-tuning based on actual sparsity; currently
   // we reserve position/coordinate space based on all previous dense
@@ -1009,11 +705,9 @@ SparseTensorStorage<P, C, V>::SparseTensorStorage(
   // really use nnz and dense/sparse distribution.
   bool allDense = true;
   uint64_t sz = 1;
-  for (uint64_t l = 0; l < lvlRank; ++l) {
+  for (uint64_t l = 0; l < lvlRank; l++) {
     const DimLevelType dlt = lvlTypes[l]; // Avoid redundant bounds checking.
     if (isCompressedDLT(dlt)) {
-      // TODO: Take a parameter between 1 and `lvlSizes[l]`, and multiply
-      // `sz` by that before reserving. (For now we just use 1.)
       positions[l].reserve(sz + 1);
       positions[l].push_back(0);
       coordinates[l].reserve(sz);
@@ -1024,7 +718,7 @@ SparseTensorStorage<P, C, V>::SparseTensorStorage(
       sz = 1;
       allDense = false;
     } else { // Dense level.
-      ASSERT_DENSE_DLT(dlt);
+      assert(isDenseDLT(dlt));
       sz = detail::checkedMul(sz, lvlSizes[l]);
     }
   }
@@ -1035,131 +729,29 @@ SparseTensorStorage<P, C, V>::SparseTensorStorage(
 template <typename P, typename C, typename V>
 SparseTensorStorage<P, C, V>::SparseTensorStorage( // NOLINT
     uint64_t dimRank, const uint64_t *dimSizes, uint64_t lvlRank,
-    const DimLevelType *lvlTypes, const uint64_t *lvl2dim,
+    const uint64_t *lvlSizes, const DimLevelType *lvlTypes,
+    const uint64_t *dim2lvl, const uint64_t *lvl2dim,
     SparseTensorCOO<V> &lvlCOO)
-    : SparseTensorStorage(dimRank, dimSizes, lvlRank,
-                          lvlCOO.getDimSizes().data(), lvlTypes, lvl2dim,
-                          false) {
-  assert(lvlRank == lvlCOO.getDimSizes().size() && "Level-rank mismatch");
-  // Ensure the preconditions of `fromCOO`.  (One is already ensured by
-  // using `lvlSizes = lvlCOO.getDimSizes()` in the ctor above.)
+    : SparseTensorStorage(dimRank, dimSizes, lvlRank, lvlSizes, lvlTypes,
+                          dim2lvl, lvl2dim, nullptr, false) {
+  // Ensure lvlCOO is sorted.
+  assert(lvlRank == lvlCOO.getRank());
   lvlCOO.sort();
   // Now actually insert the `elements`.
   const auto &elements = lvlCOO.getElements();
   const uint64_t nse = elements.size();
+  assert(values.size() == 0);
   values.reserve(nse);
   fromCOO(elements, 0, nse, 0);
 }
 
-template <typename P, typename C, typename V>
-SparseTensorStorage<P, C, V>::SparseTensorStorage(
-    uint64_t dimRank, const uint64_t *dimSizes, uint64_t lvlRank,
-    const DimLevelType *lvlTypes, const uint64_t *lvl2dim,
-    SparseTensorEnumeratorBase<V> &lvlEnumerator)
-    : SparseTensorStorage(dimRank, dimSizes, lvlRank,
-                          lvlEnumerator.getTrgSizes().data(), lvlTypes,
-                          lvl2dim) {
-  assert(lvlRank == lvlEnumerator.getTrgRank() && "Level-rank mismatch");
-  {
-    // Initialize the statistics structure.
-    SparseTensorNNZ nnz(getLvlSizes(), getLvlTypes());
-    nnz.initialize(lvlEnumerator);
-    // Initialize "positions" overhead (and allocate "coordinates", "values").
-    uint64_t parentSz = 1; // assembled-size of the `(l - 1)`-level.
-    for (uint64_t l = 0; l < lvlRank; ++l) {
-      const auto dlt = lvlTypes[l]; // Avoid redundant bounds checking.
-      if (isCompressedDLT(dlt)) {
-        positions[l].reserve(parentSz + 1);
-        positions[l].push_back(0);
-        uint64_t currentPos = 0;
-        nnz.forallCoords(l, [this, &currentPos, l](uint64_t n) {
-          currentPos += n;
-          appendPos(l, currentPos);
-        });
-        assert(positions[l].size() == parentSz + 1 &&
-               "Final positions size doesn't match allocated size");
-        // That assertion entails `assembledSize(parentSz, l)`
-        // is now in a valid state.  That is, `positions[l][parentSz]`
-        // equals the present value of `currentPos`, which is the
-        // correct assembled-size for `coordinates[l]`.
-      }
-      // Update assembled-size for the next iteration.
-      parentSz = assembledSize(parentSz, l);
-      // Ideally we need only `coordinates[l].reserve(parentSz)`, however
-      // the `std::vector` implementation forces us to initialize it too.
-      // That is, in the yieldPos loop we need random-access assignment
-      // to `coordinates[l]`; however, `std::vector`'s subscript-assignment
-      // only allows assigning to already-initialized positions.
-      if (isCompressedDLT(dlt) || isSingletonDLT(dlt))
-        coordinates[l].resize(parentSz, 0);
-      else
-        ASSERT_DENSE_DLT(dlt); // Future-proofing.
-    }
-    values.resize(parentSz, 0); // Both allocate and zero-initialize.
-  }
-  // The yieldPos loop
-  lvlEnumerator.forallElements([this](const auto &lvlCoords, V val) {
-    uint64_t parentSz = 1, parentPos = 0;
-    for (uint64_t lvlRank = getLvlRank(), l = 0; l < lvlRank; ++l) {
-      const auto dlt = getLvlTypes()[l]; // Avoid redundant bounds checking.
-      if (isCompressedDLT(dlt)) {
-        // If `parentPos == parentSz` then it's valid as an array-lookup;
-        // however, it's semantically invalid here since that entry
-        // does not represent a segment of `coordinates[l]`.  Moreover, that
-        // entry must be immutable for `assembledSize` to remain valid.
-        assert(parentPos < parentSz && "Parent position is out of bounds");
-        const uint64_t currentPos = positions[l][parentPos];
-        // This increment won't overflow the `P` type, since it can't
-        // exceed the original value of `positions[l][parentPos+1]`
-        // which was already verified to be within bounds for `P`
-        // when it was written to the array.
-        positions[l][parentPos]++;
-        writeCrd(l, currentPos, lvlCoords[l]);
-        parentPos = currentPos;
-      } else if (isSingletonDLT(dlt)) {
-        writeCrd(l, parentPos, lvlCoords[l]);
-        // the new parentPos equals the old parentPos.
-      } else { // Dense level.
-        ASSERT_DENSE_DLT(dlt);
-        parentPos = parentPos * getLvlSizes()[l] + lvlCoords[l];
-      }
-      parentSz = assembledSize(parentSz, l);
-    }
-    assert(parentPos < values.size() && "Value position is out of bounds");
-    values[parentPos] = val;
-  });
-  // The finalizeYieldPos loop
-  for (uint64_t parentSz = 1, l = 0; l < lvlRank; ++l) {
-    const auto dlt = lvlTypes[l]; // Avoid redundant bounds checking.
-    if (isCompressedDLT(dlt)) {
-      assert(parentSz == positions[l].size() - 1 &&
-             "Actual positions size doesn't match the expected size");
-      // Can't check all of them, but at least we can check the last one.
-      assert(positions[l][parentSz - 1] == positions[l][parentSz] &&
-             "Positions got corrupted");
-      // TODO: optimize this by using `memmove` or similar.
-      for (uint64_t n = 0; n < parentSz; ++n) {
-        const uint64_t parentPos = parentSz - n;
-        positions[l][parentPos] = positions[l][parentPos - 1];
-      }
-      positions[l][0] = 0;
-    } else {
-      // Both dense and singleton are no-ops for the finalizeYieldPos loop.
-      // This assertion is for future-proofing.
-      assert((isDenseDLT(dlt) || isSingletonDLT(dlt)) &&
-             "Level is neither dense nor singleton");
-    }
-    parentSz = assembledSize(parentSz, l);
-  }
-}
-
 template <typename P, typename C, typename V>
 SparseTensorStorage<P, C, V>::SparseTensorStorage(
     uint64_t dimRank, const uint64_t *dimSizes, uint64_t lvlRank,
     const uint64_t *lvlSizes, const DimLevelType *lvlTypes,
-    const uint64_t *lvl2dim, const intptr_t *lvlBufs)
+    const uint64_t *dim2lvl, const uint64_t *lvl2dim, const intptr_t *lvlBufs)
     : SparseTensorStorage(dimRank, dimSizes, lvlRank, lvlSizes, lvlTypes,
-                          lvl2dim) {
+                          dim2lvl, lvl2dim) {
   uint64_t trailCOOLen = 0, parentSz = 1, bufIdx = 0;
   for (uint64_t l = 0; l < lvlRank; l++) {
     if (!isUniqueLvl(l) && isCompressedLvl(l)) {
@@ -1181,7 +773,7 @@ SparseTensorStorage<P, C, V>::SparseTensorStorage(
       positions[l].assign(posPtr, posPtr + parentSz + 1);
       coordinates[l].assign(crdPtr, crdPtr + positions[l][parentSz]);
     } else {
-      assert(isDenseLvl(l) && "Level is not dense");
+      assert(isDenseLvl(l));
     }
     parentSz = assembledSize(parentSz, l);
   }
@@ -1206,8 +798,6 @@ SparseTensorStorage<P, C, V>::SparseTensorStorage(
   values.assign(valPtr, valPtr + parentSz);
 }
 
-#undef ASSERT_DENSE_DLT
-
 } // namespace sparse_tensor
 } // namespace mlir
 
diff --git a/mlir/include/mlir/ExecutionEngine/SparseTensorRuntime.h b/mlir/include/mlir/ExecutionEngine/SparseTensorRuntime.h
index e723a35434..bbfe6ed19e 100644
--- a/mlir/include/mlir/ExecutionEngine/SparseTensorRuntime.h
+++ b/mlir/include/mlir/ExecutionEngine/SparseTensorRuntime.h
@@ -24,8 +24,6 @@
 
 using namespace mlir::sparse_tensor;
 
-using SparseTensorWriter = std::ostream;
-
 extern "C" {
 
 //===----------------------------------------------------------------------===//
@@ -37,22 +35,20 @@ extern "C" {
 //
 //===----------------------------------------------------------------------===//
 
-/// The @newSparseTensor function for constructing a new sparse tensor.
 /// This is the "swiss army knife" method for materializing sparse
 /// tensors into the computation.  The types of the `ptr` argument and
-/// the result depend on the action, as explained in the following table
-/// (where "STS" means a sparse-tensor-storage object, "COO" means
-/// a coordinate-scheme object, and "Iterator" means an iterator object).
+/// the result depend on the action, as explained in the following table,
+/// where "STS" means a sparse-tensor-storage object and "COO" means
+/// a coordinate-scheme object.
 ///
 /// Action:         `ptr`:          Returns:
-/// kEmpty          unused          STS, empty
-/// kEmptyCOO       unused          COO, empty
-/// kFromFile       char* filename  STS, read from the file
+/// kEmpty          -               STS, empty
+/// kEmptyForward   -               STS, empty, with forwarding COO
 /// kFromCOO        COO             STS, copied from the COO source
+/// kFromReader     reader          STS, input from reader
 /// kToCOO          STS             COO, copied from the STS source
-/// kSparseToSparse STS             STS, copied from the STS source
-/// kToIterator     STS             Iterator, call @getNext to use and
-///                                 @delSparseTensorIterator to free.
+/// kPack           buffers         STS, from level buffers
+/// kSortCOOInPlace STS             STS, sorted in place
 MLIR_CRUNNERUTILS_EXPORT void *_mlir_ciface_newSparseTensor( // NOLINT
     StridedMemRefType<index_type, 1> *dimSizesRef,
     StridedMemRefType<index_type, 1> *lvlSizesRef,
@@ -84,25 +80,13 @@ MLIR_SPARSETENSOR_FOREVERY_O(DECL_SPARSEPOSITIONS)
 MLIR_SPARSETENSOR_FOREVERY_O(DECL_SPARSECOORDINATES)
 #undef DECL_SPARSECOORDINATES
 
-/// Coordinate-scheme method for adding a new element.
-/// TODO: remove dim2lvl
-#define DECL_ADDELT(VNAME, V)                                                  \
-  MLIR_CRUNNERUTILS_EXPORT void *_mlir_ciface_addElt##VNAME(                   \
-      void *lvlCOO, StridedMemRefType<V, 0> *vref,                             \
-      StridedMemRefType<index_type, 1> *dimCoordsRef,                          \
-      StridedMemRefType<index_type, 1> *dim2lvlRef);
-MLIR_SPARSETENSOR_FOREVERY_V(DECL_ADDELT)
-#undef DECL_ADDELT
-
-/// Coordinate-scheme method for getting the next element while iterating.
-/// The `cref` argument uses the same coordinate-space as the `iter` (which
-/// can be either dim- or lvl-coords, depending on context).
-#define DECL_GETNEXT(VNAME, V)                                                 \
-  MLIR_CRUNNERUTILS_EXPORT bool _mlir_ciface_getNext##VNAME(                   \
-      void *iter, StridedMemRefType<index_type, 1> *cref,                      \
-      StridedMemRefType<V, 0> *vref);
-MLIR_SPARSETENSOR_FOREVERY_V(DECL_GETNEXT)
-#undef DECL_GETNEXT
+/// Tensor-storage method for a dim to lvl forwarding insertion.
+#define DECL_FORWARDINGINSERT(VNAME, V)                                        \
+  MLIR_CRUNNERUTILS_EXPORT void _mlir_ciface_forwardingInsert##VNAME(          \
+      void *tensor, StridedMemRefType<V, 0> *vref,                             \
+      StridedMemRefType<index_type, 1> *dimCoordsRef);                         \
+  MLIR_SPARSETENSOR_FOREVERY_V(DECL_FORWARDINGINSERT)
+#undef DECL_FORWARDINGINSERT
 
 /// Tensor-storage method to insert elements in lexicographical
 /// level-coordinate order.
@@ -129,31 +113,22 @@ MLIR_CRUNNERUTILS_EXPORT void *_mlir_ciface_createCheckedSparseTensorReader(
     char *filename, StridedMemRefType<index_type, 1> *dimShapeRef,
     PrimaryType valTp);
 
-/// Constructs a new sparse-tensor storage object with the given encoding,
-/// initializes it by reading all the elements from the file, and then
-/// closes the file.
-MLIR_CRUNNERUTILS_EXPORT void *_mlir_ciface_newSparseTensorFromReader(
-    void *p, StridedMemRefType<index_type, 1> *lvlSizesRef,
-    StridedMemRefType<DimLevelType, 1> *lvlTypesRef,
-    StridedMemRefType<index_type, 1> *dim2lvlRef,
-    StridedMemRefType<index_type, 1> *lvl2dimRef, OverheadType posTp,
-    OverheadType crdTp, PrimaryType valTp);
-
 /// SparseTensorReader method to obtain direct access to the
 /// dimension-sizes array.
 MLIR_CRUNNERUTILS_EXPORT void _mlir_ciface_getSparseTensorReaderDimSizes(
     StridedMemRefType<index_type, 1> *out, void *p);
 
 /// Reads the sparse tensor, stores the coordinates and values to the given
-/// memrefs. Returns a boolean to indicate whether the COO elements are sorted.
-#define DECL_GETNEXT(VNAME, V, CNAME, C)                                       \
+/// memrefs of a COO in AoS format. Returns a boolean to indicate whether
+/// the COO elements are sorted.
+#define DECL_READTOBUFFERS(VNAME, V, CNAME, C)                                 \
   MLIR_CRUNNERUTILS_EXPORT bool                                                \
       _mlir_ciface_getSparseTensorReaderReadToBuffers##CNAME##VNAME(           \
           void *p, StridedMemRefType<index_type, 1> *dim2lvlRef,               \
           StridedMemRefType<index_type, 1> *lvl2dimRef,                        \
           StridedMemRefType<C, 1> *cref, StridedMemRefType<V, 1> *vref)        \
-          MLIR_SPARSETENSOR_FOREVERY_V_O(DECL_GETNEXT)
-#undef DECL_GETNEXT
+          MLIR_SPARSETENSOR_FOREVERY_V_O(DECL_READTOBUFFERS)
+#undef DECL_READTOBUFFERS
 
 /// Outputs the sparse tensor dim-rank, nse, and dim-shape.
 MLIR_CRUNNERUTILS_EXPORT void _mlir_ciface_outSparseTensorWriterMetaData(
@@ -185,8 +160,11 @@ MLIR_CRUNNERUTILS_EXPORT index_type sparseLvlSize(void *tensor, index_type l);
 /// Tensor-storage method to get the size of the given dimension.
 MLIR_CRUNNERUTILS_EXPORT index_type sparseDimSize(void *tensor, index_type d);
 
+/// Tensor-storage method to finalize forwarding insertions.
+MLIR_CRUNNERUTILS_EXPORT void endForwardingInsert(void *tensor);
+
 /// Tensor-storage method to finalize lexicographic insertions.
-MLIR_CRUNNERUTILS_EXPORT void endInsert(void *tensor);
+MLIR_CRUNNERUTILS_EXPORT void endLexInsert(void *tensor);
 
 /// Coordinate-scheme method to write to file in extended FROSTT format.
 #define DECL_OUTSPARSETENSOR(VNAME, V)                                         \
@@ -204,34 +182,13 @@ MLIR_CRUNNERUTILS_EXPORT void delSparseTensor(void *tensor);
 MLIR_SPARSETENSOR_FOREVERY_V(DECL_DELCOO)
 #undef DECL_DELCOO
 
-/// Releases the memory for an iterator object.
-#define DECL_DELITER(VNAME, V)                                                 \
-  MLIR_CRUNNERUTILS_EXPORT void delSparseTensorIterator##VNAME(void *iter);
-MLIR_SPARSETENSOR_FOREVERY_V(DECL_DELITER)
-#undef DECL_DELITER
-
 /// Helper function to read a sparse tensor filename from the environment,
 /// defined with the naming convention ${TENSOR0}, ${TENSOR1}, etc.
 MLIR_CRUNNERUTILS_EXPORT char *getTensorFilename(index_type id);
 
-/// Helper function to read the header of a file and return the
-/// shape/sizes, without parsing the elements of the file.
-MLIR_CRUNNERUTILS_EXPORT void readSparseTensorShape(char *filename,
-                                                    std::vector<uint64_t> *out);
-
-/// Returns the rank of the sparse tensor being read.
-MLIR_CRUNNERUTILS_EXPORT index_type getSparseTensorReaderRank(void *p);
-
-/// Returns the is_symmetric bit for the sparse tensor being read.
-MLIR_CRUNNERUTILS_EXPORT bool getSparseTensorReaderIsSymmetric(void *p);
-
 /// Returns the number of stored elements for the sparse tensor being read.
 MLIR_CRUNNERUTILS_EXPORT index_type getSparseTensorReaderNSE(void *p);
 
-/// Returns the size of a dimension for the sparse tensor being read.
-MLIR_CRUNNERUTILS_EXPORT index_type getSparseTensorReaderDimSize(void *p,
-                                                                 index_type d);
-
 /// Releases the SparseTensorReader and closes the associated file.
 MLIR_CRUNNERUTILS_EXPORT void delSparseTensorReader(void *p);
 
diff --git a/mlir/include/mlir/IR/Attributes.h b/mlir/include/mlir/IR/Attributes.h
index ac5dd49f0b..f433363e5d 100644
--- a/mlir/include/mlir/IR/Attributes.h
+++ b/mlir/include/mlir/IR/Attributes.h
@@ -409,7 +409,6 @@ struct CastInfo<To, From,
     /// Return a constant true instead of a dynamic true when casting to self or
     /// up the hierarchy.
     if constexpr (std::is_base_of_v<To, From>) {
-      (void)ty;
       return true;
     } else {
       return To::classof(ty);
diff --git a/mlir/include/mlir/IR/BuiltinTypes.h b/mlir/include/mlir/IR/BuiltinTypes.h
index 9df5548cd5..92ce053ad5 100644
--- a/mlir/include/mlir/IR/BuiltinTypes.h
+++ b/mlir/include/mlir/IR/BuiltinTypes.h
@@ -11,6 +11,7 @@
 
 #include "mlir/IR/BuiltinAttributeInterfaces.h"
 #include "mlir/IR/BuiltinTypeInterfaces.h"
+#include "mlir/Support/ADTExtras.h"
 
 namespace llvm {
 class BitVector;
@@ -274,20 +275,14 @@ class RankedTensorType::Builder {
   /// Erase a dim from shape @pos.
   Builder &dropDim(unsigned pos) {
     assert(pos < shape.size() && "overflow");
-    if (storage.empty())
-      storage.append(shape.begin(), shape.end());
-    storage.erase(storage.begin() + pos);
-    shape = {storage.data(), storage.size()};
+    shape.erase(pos);
     return *this;
   }
 
   /// Insert a val into shape @pos.
   Builder &insertDim(int64_t val, unsigned pos) {
     assert(pos <= shape.size() && "overflow");
-    if (storage.empty())
-      storage.append(shape.begin(), shape.end());
-    storage.insert(storage.begin() + pos, val);
-    shape = {storage.data(), storage.size()};
+    shape.insert(pos, val);
     return *this;
   }
 
@@ -296,9 +291,7 @@ class RankedTensorType::Builder {
   }
 
 private:
-  ArrayRef<int64_t> shape;
-  // Owning shape data for copy-on-write operations.
-  SmallVector<int64_t> storage;
+  CopyOnWriteArrayRef<int64_t> shape;
   Type elementType;
   Attribute encoding;
 };
@@ -313,27 +306,18 @@ class VectorType::Builder {
 public:
   /// Build from another VectorType.
   explicit Builder(VectorType other)
-      : shape(other.getShape()), elementType(other.getElementType()),
+      : elementType(other.getElementType()), shape(other.getShape()),
         scalableDims(other.getScalableDims()) {}
 
   /// Build from scratch.
   Builder(ArrayRef<int64_t> shape, Type elementType,
-          unsigned numScalableDims = 0, ArrayRef<bool> scalableDims = {})
-      : shape(shape), elementType(elementType) {
-    if (scalableDims.empty())
-      scalableDims = SmallVector<bool>(shape.size(), false);
-    else
-      this->scalableDims = scalableDims;
-  }
+          ArrayRef<bool> scalableDims = {})
+      : elementType(elementType), shape(shape), scalableDims(scalableDims) {}
 
   Builder &setShape(ArrayRef<int64_t> newShape,
                     ArrayRef<bool> newIsScalableDim = {}) {
-    if (newIsScalableDim.empty())
-      scalableDims = SmallVector<bool>(shape.size(), false);
-    else
-      scalableDims = newIsScalableDim;
-
     shape = newShape;
+    scalableDims = newIsScalableDim;
     return *this;
   }
 
@@ -345,25 +329,16 @@ class VectorType::Builder {
   /// Erase a dim from shape @pos.
   Builder &dropDim(unsigned pos) {
     assert(pos < shape.size() && "overflow");
-    if (storage.empty())
-      storage.append(shape.begin(), shape.end());
-    if (storageScalableDims.empty())
-      storageScalableDims.append(scalableDims.begin(), scalableDims.end());
-    storage.erase(storage.begin() + pos);
-    storageScalableDims.erase(storageScalableDims.begin() + pos);
-    shape = {storage.data(), storage.size()};
-    scalableDims =
-        ArrayRef<bool>(storageScalableDims.data(), storageScalableDims.size());
+    shape.erase(pos);
+    if (!scalableDims.empty())
+      scalableDims.erase(pos);
     return *this;
   }
 
   /// Set a dim in shape @pos to val.
   Builder &setDim(unsigned pos, int64_t val) {
-    if (storage.empty())
-      storage.append(shape.begin(), shape.end());
-    assert(pos < storage.size() && "overflow");
-    storage[pos] = val;
-    shape = {storage.data(), storage.size()};
+    assert(pos < shape.size() && "overflow");
+    shape.set(pos, val);
     return *this;
   }
 
@@ -372,13 +347,9 @@ class VectorType::Builder {
   }
 
 private:
-  ArrayRef<int64_t> shape;
-  // Owning shape data for copy-on-write operations.
-  SmallVector<int64_t> storage;
   Type elementType;
-  ArrayRef<bool> scalableDims;
-  // Owning scalableDims data for copy-on-write operations.
-  SmallVector<bool> storageScalableDims;
+  CopyOnWriteArrayRef<int64_t> shape;
+  CopyOnWriteArrayRef<bool> scalableDims;
 };
 
 /// Given an `originalShape` and a `reducedShape` assumed to be a subset of
diff --git a/mlir/include/mlir/IR/CommonTypeConstraints.td b/mlir/include/mlir/IR/CommonTypeConstraints.td
index 4fc14e30b8..b0b5348baa 100644
--- a/mlir/include/mlir/IR/CommonTypeConstraints.td
+++ b/mlir/include/mlir/IR/CommonTypeConstraints.td
@@ -34,11 +34,25 @@ def IsFixedVectorTypePred : CPred<[{::llvm::isa<::mlir::VectorType>($_self) &&
                                   !::llvm::cast<VectorType>($_self).isScalable()}]>;
 
 // Whether a type is a scalable VectorType.
-def IsScalableVectorTypePred : CPred<[{::llvm::isa<::mlir::VectorType>($_self) &&
-                                   ::llvm::cast<VectorType>($_self).isScalable()}]>;
+def IsVectorTypeWithAnyDimScalablePred 
+        : CPred<[{::llvm::isa<::mlir::VectorType>($_self) &&
+                  ::llvm::cast<VectorType>($_self).isScalable()}]>;
+
+// Whether a type is a scalable VectorType, with a single trailing scalable dimension.
+// Examples:
+// Valid:
+//   - vector<[4]xf32>, vector<2x3x[2]xi64>, vector<32x[8]xi32>
+// Invalid
+//   - vector<[4]x8xi32>, vector<[2]x[2]xf64>, vector<2x[8]x4xi32>
+def IsVectorTypeWithOnlyTrailingDimScalablePred : And<[
+  CPred<"::llvm::isa<::mlir::VectorType>($_self)">,
+  CPred<"::llvm::cast<::mlir::VectorType>($_self).getRank() > 0">,
+  CPred<"::llvm::cast<::mlir::VectorType>($_self).getScalableDims().back()">,
+  CPred<"!llvm::is_contained(::llvm::cast<::mlir::VectorType>($_self).getScalableDims().drop_back(), true)">
+]>;
 
 // Whether a type is a VectorType and all dimensions are scalable.
-def allDimsScalableVectorTypePred : And<[
+def IsVectorTypeWithAllDimsScalablePred : And<[
   IsVectorTypePred,
   CPred<[{::llvm::cast<::mlir::VectorType>($_self).allDimsScalable()}]>
 ]>;
@@ -103,7 +117,8 @@ class DialectType<Dialect d, Pred condition, string descr = "",
 
 // A variadic type constraint. It expands to zero or more of the base type. This
 // class is used for supporting variadic operands/results.
-class Variadic<Type type> : TypeConstraint<type.predicate, type.summary,
+class Variadic<Type type> : TypeConstraint<type.predicate,
+                                           "variadic of " # type.summary,
                                            type.cppClassName> {
   Type baseType = type;
   int minSize = 0;
@@ -401,9 +416,18 @@ class FixedVectorOf<list<Type> allowedTypes> :
           "fixed-length vector", "::mlir::VectorType">;
 
 class ScalableVectorOf<list<Type> allowedTypes> :
-  ShapedContainerType<allowedTypes, IsScalableVectorTypePred,
+  ShapedContainerType<allowedTypes, IsVectorTypeWithAnyDimScalablePred,
           "scalable vector", "::mlir::VectorType">;
 
+// Any vector with a single trailing scalable dimension, with an element type in
+// the `allowedTypes` list.
+//
+// Note: This Similar to ScalableVectorOf, with the extra requirement that only
+// the trailing dim is scalable.
+class VectorWithTrailingDimScalableOf<list<Type> allowedTypes> :
+  ShapedContainerType<allowedTypes, IsVectorTypeWithOnlyTrailingDimScalablePred,
+          "trailing scalable vector", "::mlir::VectorType">;
+
 // Whether the number of elements of a vector is from the given
 // `allowedRanks` list
 class IsVectorOfRankPred<list<int> allowedRanks> :
@@ -425,7 +449,7 @@ class IsFixedVectorOfRankPred<list<int> allowedRanks> :
 // Whether the number of elements of a scalable vector is from the given
 // `allowedRanks` list
 class IsScalableVectorOfRankPred<list<int> allowedRanks> :
-  And<[IsScalableVectorTypePred,
+  And<[IsVectorTypeWithAnyDimScalablePred,
        Or<!foreach(allowedlength, allowedRanks,
                    CPred<[{::llvm::cast<::mlir::VectorType>($_self).getRank()
                            == }]
@@ -475,12 +499,46 @@ class IsFixedVectorOfLengthPred<list<int> allowedLengths> :
 // Whether the number of elements of a scalable vector is from the given
 // `allowedLengths` list
 class IsScalableVectorOfLengthPred<list<int> allowedLengths> :
-  And<[IsScalableVectorTypePred,
+  And<[IsVectorTypeWithAnyDimScalablePred,
        Or<!foreach(allowedlength, allowedLengths,
                    CPred<[{::llvm::cast<::mlir::VectorType>($_self).getNumElements()
                            == }]
                          # allowedlength>)>]>;
 
+// Normalizes an index so the indices in both directions have the same value.
+// For example, when indexing forwards index 2 is the third element. When
+// indexing in reverse the third element is -3. This helper would map both of
+// these to the "normalized" index of 3. This makes the bounds checking in
+// IsNthDimSizeIsOneOfPred simpler (see first CPred).
+class NormalizeIndex<int value> {
+  int ret = !if(!lt(value, 0),
+    !sub(0, value)  /* -value if negative */,
+    !add(value, 1)  /* value + 1 if positive*/);
+}
+
+// Whether the n-th dim of the shape is contained within `allowedSizes`.
+// Negative values for `n` index in reverse.
+//
+// Examples:
+// IsNthDimSizeIsOneOfPred<0, {2, 3, 4}>
+//  - Accepts any shape where the first dim is 2, 3, or 4.
+//    * This means shapes like: 2x8x9x5, 4, 3x1, 4x?, etc
+// IsNthDimSizeIsOneOfPred<-1, {16}>
+//  - Accepts any shape where the last dim is 16.
+//    * This means shapes like 2x16, 16, 1x2x3x4x16, etc
+// IsNthDimSizeIsOneOfPred<-2, {10, 5}>
+//  - Accepts any shape where the second to last dim is 10 or 5.
+//    * This means shapes like: 1x10x2, 2x1x4x5x6, 8x10x?, etc
+class IsNthDimSizeIsOneOfPred<int n, list<int> allowedSizes>
+  : And<[
+      CPred<"::llvm::cast<::mlir::ShapedType>($_self).getRank() >= " # NormalizeIndex<n>.ret>,
+      CPred<"::llvm::is_contained(ArrayRef<int64_t>({" # !interleave(allowedSizes, ", ") # "}), "
+        # "::llvm::cast<::mlir::ShapedType>($_self).getDimSize("
+        #   !if(!lt(n, 0),
+              "::llvm::cast<::mlir::ShapedType>($_self).getRank() + " # n,
+              "" # n)
+        # "))">]>;
+
 // Whether the shape of a vector matches the given `shape` list.
 class IsVectorOfShape<list<int> shape>
   : CPred<"::llvm::cast<::mlir::VectorType>($_self).getShape() == ArrayRef<int64_t>({" # !interleave(shape, ", ") # "})">;
@@ -546,6 +604,24 @@ class ScalableVectorOfRankAndLengthAndType<list<int> allowedRanks,
   ScalableVectorOfLength<allowedLengths>.summary,
   "::mlir::VectorType">;
 
+// Any ShapedType where the size of the n-th dim is contained in `allowedSizes`.
+// Negative values for `n` index in reverse.
+class ShapedTypeWithNthDimOfSize<int n, list<int> allowedSizes> : Type<
+  IsNthDimSizeIsOneOfPred<n, allowedSizes>,
+  " with dim " # n # " having a size of {" # !interleave(allowedSizes, ", ") # "}",
+  "::mlir::ShapedType">;
+
+// Any scalable vector with a single trailing scalable dimensions, where the
+// size of the trailing dimension is in `allowedTrailingSizes` list, and the
+// type is in the `allowedTypes` list.
+class VectorWithTrailingDimScalableOfSizeAndType<list<int> allowedTrailingSizes,
+                                           list<Type> allowedTypes> : AllOfType<
+  [VectorWithTrailingDimScalableOf<allowedTypes>,
+   ShapedTypeWithNthDimOfSize<-1, allowedTrailingSizes>],
+   VectorWithTrailingDimScalableOf<allowedTypes>.summary #
+   ShapedTypeWithNthDimOfSize<-1, allowedTrailingSizes>.summary,
+  "::mlir::VectorType">;
+
 def AnyVector : VectorOf<[AnyType]>;
 // Temporary vector type clone that allows gradual transition to 0-D vectors.
 def AnyVectorOfAnyRank : VectorOfAnyRankOf<[AnyType]>;
diff --git a/mlir/include/mlir/IR/OpBase.td b/mlir/include/mlir/IR/OpBase.td
index 236dd74839..7866ac24c1 100644
--- a/mlir/include/mlir/IR/OpBase.td
+++ b/mlir/include/mlir/IR/OpBase.td
@@ -568,6 +568,14 @@ class TypesMatchWith<string summary, string lhsArg, string rhsArg,
   string transformer = transform;
 }
 
+// The same as TypesMatchWith but if either `lhsArg` or `rhsArg` are optional
+// and not present returns success.
+class OptionalTypesMatchWith<string summary, string lhsArg, string rhsArg,
+                     string transform, string comparator = "std::equal_to<>()">
+  : TypesMatchWith<summary, lhsArg, rhsArg, transform,
+     "!get" # snakeCaseToCamelCase<lhsArg>.ret # "()"
+     # " || !get" # snakeCaseToCamelCase<rhsArg>.ret # "() || " # comparator>;
+
 // Special variant of `TypesMatchWith` that provides a comparator suitable for
 // ranged arguments.
 class RangedTypesMatchWith<string summary, string lhsArg, string rhsArg,
diff --git a/mlir/include/mlir/IR/OpImplementation.h b/mlir/include/mlir/IR/OpImplementation.h
index 379392ace4..f1fabf95a6 100644
--- a/mlir/include/mlir/IR/OpImplementation.h
+++ b/mlir/include/mlir/IR/OpImplementation.h
@@ -350,8 +350,7 @@ template <typename AsmPrinterT, typename T,
                                !std::is_convertible<T &, Attribute &>::value &&
                                !std::is_convertible<T &, ValueRange>::value &&
                                !std::is_convertible<T &, APFloat &>::value &&
-                               !llvm::is_one_of<T, bool, int8_t, uint8_t, float,
-                                                double>::value,
+                               !llvm::is_one_of<T, bool, float, double>::value,
                            T> * = nullptr>
 inline std::enable_if_t<std::is_base_of<AsmPrinter, AsmPrinterT>::value,
                         AsmPrinterT &>
@@ -367,17 +366,6 @@ operator<<(AsmPrinterT &p, bool value) {
   return p << (value ? StringRef("true") : "false");
 }
 
-/// Specialization for 8-bit integers to ensure values are printed as integers
-// and not characters.
-template <
-    typename AsmPrinterT, typename T,
-    std::enable_if_t<llvm::is_one_of<T, int8_t, uint8_t>::value, T> * = nullptr>
-inline std::enable_if_t<std::is_base_of<AsmPrinter, AsmPrinterT>::value,
-                        AsmPrinterT &>
-operator<<(AsmPrinterT &p, T value) {
-  return p << static_cast<int16_t>(value);
-}
-
 template <typename AsmPrinterT, typename ValueRangeT>
 inline std::enable_if_t<std::is_base_of<AsmPrinter, AsmPrinterT>::value,
                         AsmPrinterT &>
diff --git a/mlir/include/mlir/IR/Types.h b/mlir/include/mlir/IR/Types.h
index 832794928a..46bb733101 100644
--- a/mlir/include/mlir/IR/Types.h
+++ b/mlir/include/mlir/IR/Types.h
@@ -406,7 +406,6 @@ struct CastInfo<
     /// Return a constant true instead of a dynamic true when casting to self or
     /// up the hierarchy.
     if constexpr (std::is_base_of_v<To, From>) {
-      (void)ty;
       return true;
     } else {
       return To::classof(ty);
diff --git a/mlir/include/mlir/IR/Utils.td b/mlir/include/mlir/IR/Utils.td
index 651706099d..e75b3ae7a5 100644
--- a/mlir/include/mlir/IR/Utils.td
+++ b/mlir/include/mlir/IR/Utils.td
@@ -66,4 +66,28 @@ class CArg<string ty, string value = ""> {
   string defaultValue = value;
 }
 
+// Helper which makes the first letter of a string uppercase.
+// e.g. cat -> Cat
+class firstCharToUpper<string str>
+{
+  string ret = !if(!gt(!size(str), 0),
+    !toupper(!substr(str, 0, 1)) # !substr(str, 1),
+    "");
+}
+
+class _snakeCaseHelper<string str> {
+  int idx = !find(str, "_");
+  string ret = !if(!ge(idx, 0),
+    !substr(str, 0, idx) # firstCharToUpper<!substr(str, !add(idx, 1))>.ret,
+    str);
+}
+
+// Converts a snake_case string to CamelCase.
+// TODO: Replace with a !tocamelcase bang operator.
+class snakeCaseToCamelCase<string str>
+{
+  string ret = !foldl(firstCharToUpper<str>.ret,
+    !range(0, !size(str)), acc, idx, _snakeCaseHelper<acc>.ret);
+}
+
 #endif // UTILS_TD
diff --git a/mlir/include/mlir/IR/Value.h b/mlir/include/mlir/IR/Value.h
index 4e550fe3e3..dbcc10d4f4 100644
--- a/mlir/include/mlir/IR/Value.h
+++ b/mlir/include/mlir/IR/Value.h
@@ -605,7 +605,6 @@ struct CastInfo<
     /// Return a constant true instead of a dynamic true when casting to self or
     /// up the hierarchy.
     if constexpr (std::is_base_of_v<To, From>) {
-      (void)ty;
       return true;
     } else {
       return To::classof(ty);
diff --git a/mlir/include/mlir/Interfaces/LoopLikeInterface.h b/mlir/include/mlir/Interfaces/LoopLikeInterface.h
index 0eebb984e5..7c7d378d05 100644
--- a/mlir/include/mlir/Interfaces/LoopLikeInterface.h
+++ b/mlir/include/mlir/Interfaces/LoopLikeInterface.h
@@ -24,6 +24,11 @@ class RewriterBase;
 /// arguments in `newBbArgs`.
 using NewYieldValuesFn = std::function<SmallVector<Value>(
     OpBuilder &b, Location loc, ArrayRef<BlockArgument> newBbArgs)>;
+
+namespace detail {
+/// Verify invariants of the LoopLikeOpInterface.
+LogicalResult verifyLoopLikeOpInterface(Operation *op);
+} // namespace detail
 } // namespace mlir
 
 /// Include the generated interface declarations.
diff --git a/mlir/include/mlir/Interfaces/LoopLikeInterface.td b/mlir/include/mlir/Interfaces/LoopLikeInterface.td
index ded0a29292..4d2a66dd31 100644
--- a/mlir/include/mlir/Interfaces/LoopLikeInterface.td
+++ b/mlir/include/mlir/Interfaces/LoopLikeInterface.td
@@ -20,6 +20,19 @@ def LoopLikeOpInterface : OpInterface<"LoopLikeOpInterface"> {
     Contains helper functions to query properties and perform transformations
     of a loop. Operations that implement this interface will be considered by
     loop-invariant code motion.
+
+    Loop-carried variables can be exposed through this interface. There are
+    3 components to a loop-carried variable.
+    - The "region iter_arg" is the block argument of the entry block that
+      represents the loop-carried variable in each iteration.
+    - The "init value" is an operand of the loop op that serves as the initial
+      region iter_arg value for the first iteration (if any).
+    - The "yielded" value is the value that is forwarded from one iteration to
+      serve as the region iter_arg of the next iteration.
+
+    If one of the respective interface methods is implemented, so must the other
+    two. The interface verifier ensures that the number of types of the region
+    iter_args, init values and yielded values match.
   }];
   let cppNamespace = "::mlir";
 
@@ -141,6 +154,17 @@ def LoopLikeOpInterface : OpInterface<"LoopLikeOpInterface"> {
         return ::mlir::Block::BlockArgListType();
       }]
     >,
+    InterfaceMethod<[{
+        Return the values that are yielded to the next iteration.
+      }],
+      /*retTy=*/"::mlir::ValueRange",
+      /*methodName=*/"getYieldedValues",
+      /*args=*/(ins),
+      /*methodBody=*/"",
+      /*defaultImplementation=*/[{
+        return ::mlir::ValueRange();
+      }]
+    >,
     InterfaceMethod<[{
         Append the specified additional "init" operands: replace this loop with
         a new loop that has the additional init operands. The loop body of
@@ -192,6 +216,12 @@ def LoopLikeOpInterface : OpInterface<"LoopLikeOpInterface"> {
           });
     }
   }];
+
+  let verifyWithRegions = 1;
+
+  let verify = [{
+    return detail::verifyLoopLikeOpInterface($_op);
+  }];
 }
 
 #endif // MLIR_INTERFACES_LOOPLIKEINTERFACE
diff --git a/mlir/include/mlir/Interfaces/ValueBoundsOpInterface.h b/mlir/include/mlir/Interfaces/ValueBoundsOpInterface.h
index 2687d79aec..8f11c563e0 100644
--- a/mlir/include/mlir/Interfaces/ValueBoundsOpInterface.h
+++ b/mlir/include/mlir/Interfaces/ValueBoundsOpInterface.h
@@ -176,6 +176,16 @@ class ValueBoundsConstraintSet {
       presburger::BoundType type, AffineMap map, ValueDimList mapOperands,
       StopConditionFn stopCondition = nullptr, bool closedUB = false);
 
+  /// Compute a constant delta between the given two values. Return "failure"
+  /// if a constant delta could not be determined.
+  ///
+  /// `dim1`/`dim2` must be `nullopt` if and only if `value1`/`value2` are
+  /// index-typed.
+  static FailureOr<int64_t>
+  computeConstantDelta(Value value1, Value value2,
+                       std::optional<int64_t> dim1 = std::nullopt,
+                       std::optional<int64_t> dim2 = std::nullopt);
+
   /// Compute whether the given values/dimensions are equal. Return "failure" if
   /// equality could not be determined.
   ///
diff --git a/mlir/include/mlir/Query/Matcher/ErrorBuilder.h b/mlir/include/mlir/Query/Matcher/ErrorBuilder.h
new file mode 100644
index 0000000000..1073daed87
--- /dev/null
+++ b/mlir/include/mlir/Query/Matcher/ErrorBuilder.h
@@ -0,0 +1,63 @@
+//===--- ErrorBuilder.h - Helper for building error messages ----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// ErrorBuilder to manage error messages.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_TOOLS_MLIRQUERY_MATCHER_ERRORBUILDER_H
+#define MLIR_TOOLS_MLIRQUERY_MATCHER_ERRORBUILDER_H
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/Twine.h"
+#include <initializer_list>
+
+namespace mlir::query::matcher::internal {
+class Diagnostics;
+
+// Represents the line and column numbers in a source query.
+struct SourceLocation {
+  unsigned line{};
+  unsigned column{};
+};
+
+// Represents a range in a source query, defined by its start and end locations.
+struct SourceRange {
+  SourceLocation start{};
+  SourceLocation end{};
+};
+
+// All errors from the system.
+enum class ErrorType {
+  None,
+
+  // Parser Errors
+  ParserFailedToBuildMatcher,
+  ParserInvalidToken,
+  ParserNoCloseParen,
+  ParserNoCode,
+  ParserNoComma,
+  ParserNoOpenParen,
+  ParserNotAMatcher,
+  ParserOverloadedType,
+  ParserStringError,
+  ParserTrailingCode,
+
+  // Registry Errors
+  RegistryMatcherNotFound,
+  RegistryValueNotFound,
+  RegistryWrongArgCount,
+  RegistryWrongArgType
+};
+
+void addError(Diagnostics *error, SourceRange range, ErrorType errorType,
+              std::initializer_list<llvm::Twine> errorTexts);
+
+} // namespace mlir::query::matcher::internal
+
+#endif // MLIR_TOOLS_MLIRQUERY_MATCHER_ERRORBUILDER_H
diff --git a/mlir/include/mlir/Query/Matcher/Marshallers.h b/mlir/include/mlir/Query/Matcher/Marshallers.h
new file mode 100644
index 0000000000..6ed35ac0dd
--- /dev/null
+++ b/mlir/include/mlir/Query/Matcher/Marshallers.h
@@ -0,0 +1,199 @@
+//===--- Marshallers.h - Generic matcher function marshallers ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains function templates and classes to wrap matcher construct
+// functions. It provides a collection of template function and classes that
+// present a generic marshalling layer on top of matcher construct functions.
+// The registry uses these to export all marshaller constructors with a uniform
+// interface. This mechanism takes inspiration from clang-query.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_TOOLS_MLIRQUERY_MATCHER_MARSHALLERS_H
+#define MLIR_TOOLS_MLIRQUERY_MATCHER_MARSHALLERS_H
+
+#include "ErrorBuilder.h"
+#include "VariantValue.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/StringRef.h"
+
+namespace mlir::query::matcher::internal {
+
+// Helper template class for jumping from argument type to the correct is/get
+// functions in VariantValue. This is used for verifying and extracting the
+// matcher arguments.
+template <class T>
+struct ArgTypeTraits;
+template <class T>
+struct ArgTypeTraits<const T &> : public ArgTypeTraits<T> {};
+
+template <>
+struct ArgTypeTraits<llvm::StringRef> {
+
+  static bool hasCorrectType(const VariantValue &value) {
+    return value.isString();
+  }
+
+  static const llvm::StringRef &get(const VariantValue &value) {
+    return value.getString();
+  }
+
+  static ArgKind getKind() { return ArgKind::String; }
+
+  static std::optional<std::string> getBestGuess(const VariantValue &) {
+    return std::nullopt;
+  }
+};
+
+template <>
+struct ArgTypeTraits<DynMatcher> {
+
+  static bool hasCorrectType(const VariantValue &value) {
+    return value.isMatcher();
+  }
+
+  static DynMatcher get(const VariantValue &value) {
+    return *value.getMatcher().getDynMatcher();
+  }
+
+  static ArgKind getKind() { return ArgKind::Matcher; }
+
+  static std::optional<std::string> getBestGuess(const VariantValue &) {
+    return std::nullopt;
+  }
+};
+
+// Interface for generic matcher descriptor.
+// Offers a create() method that constructs the matcher from the provided
+// arguments.
+class MatcherDescriptor {
+public:
+  virtual ~MatcherDescriptor() = default;
+  virtual VariantMatcher create(SourceRange nameRange,
+                                const llvm::ArrayRef<ParserValue> args,
+                                Diagnostics *error) const = 0;
+
+  // Returns the number of arguments accepted by the matcher.
+  virtual unsigned getNumArgs() const = 0;
+
+  // Append the set of argument types accepted for argument 'argNo' to
+  // 'argKinds'.
+  virtual void getArgKinds(unsigned argNo,
+                           std::vector<ArgKind> &argKinds) const = 0;
+};
+
+class FixedArgCountMatcherDescriptor : public MatcherDescriptor {
+public:
+  using MarshallerType = VariantMatcher (*)(void (*matcherFunc)(),
+                                            llvm::StringRef matcherName,
+                                            SourceRange nameRange,
+                                            llvm::ArrayRef<ParserValue> args,
+                                            Diagnostics *error);
+
+  // Marshaller Function to unpack the arguments and call Func. Func is the
+  // Matcher construct function. This is the function that the matcher
+  // expressions would use to create the matcher.
+  FixedArgCountMatcherDescriptor(MarshallerType marshaller,
+                                 void (*matcherFunc)(),
+                                 llvm::StringRef matcherName,
+                                 llvm::ArrayRef<ArgKind> argKinds)
+      : marshaller(marshaller), matcherFunc(matcherFunc),
+        matcherName(matcherName), argKinds(argKinds.begin(), argKinds.end()) {}
+
+  VariantMatcher create(SourceRange nameRange, llvm::ArrayRef<ParserValue> args,
+                        Diagnostics *error) const override {
+    return marshaller(matcherFunc, matcherName, nameRange, args, error);
+  }
+
+  unsigned getNumArgs() const override { return argKinds.size(); }
+
+  void getArgKinds(unsigned argNo, std::vector<ArgKind> &kinds) const override {
+    kinds.push_back(argKinds[argNo]);
+  }
+
+private:
+  const MarshallerType marshaller;
+  void (*const matcherFunc)();
+  const llvm::StringRef matcherName;
+  const std::vector<ArgKind> argKinds;
+};
+
+// Helper function to check if argument count matches expected count
+inline bool checkArgCount(SourceRange nameRange, size_t expectedArgCount,
+                          llvm::ArrayRef<ParserValue> args,
+                          Diagnostics *error) {
+  if (args.size() != expectedArgCount) {
+    addError(error, nameRange, ErrorType::RegistryWrongArgCount,
+             {llvm::Twine(expectedArgCount), llvm::Twine(args.size())});
+    return false;
+  }
+  return true;
+}
+
+// Helper function for checking argument type
+template <typename ArgType, size_t Index>
+inline bool checkArgTypeAtIndex(llvm::StringRef matcherName,
+                                llvm::ArrayRef<ParserValue> args,
+                                Diagnostics *error) {
+  if (!ArgTypeTraits<ArgType>::hasCorrectType(args[Index].value)) {
+    addError(error, args[Index].range, ErrorType::RegistryWrongArgType,
+             {llvm::Twine(matcherName), llvm::Twine(Index + 1)});
+    return false;
+  }
+  return true;
+}
+
+// Marshaller function for fixed number of arguments
+template <typename ReturnType, typename... ArgTypes, size_t... Is>
+static VariantMatcher
+matcherMarshallFixedImpl(void (*matcherFunc)(), llvm::StringRef matcherName,
+                         SourceRange nameRange,
+                         llvm::ArrayRef<ParserValue> args, Diagnostics *error,
+                         std::index_sequence<Is...>) {
+  using FuncType = ReturnType (*)(ArgTypes...);
+
+  // Check if the argument count matches the expected count
+  if (!checkArgCount(nameRange, sizeof...(ArgTypes), args, error))
+    return VariantMatcher();
+
+  // Check if each argument at the corresponding index has the correct type
+  if ((... && checkArgTypeAtIndex<ArgTypes, Is>(matcherName, args, error))) {
+    ReturnType fnPointer = reinterpret_cast<FuncType>(matcherFunc)(
+        ArgTypeTraits<ArgTypes>::get(args[Is].value)...);
+    return VariantMatcher::SingleMatcher(
+        *DynMatcher::constructDynMatcherFromMatcherFn(fnPointer));
+  }
+
+  return VariantMatcher();
+}
+
+template <typename ReturnType, typename... ArgTypes>
+static VariantMatcher
+matcherMarshallFixed(void (*matcherFunc)(), llvm::StringRef matcherName,
+                     SourceRange nameRange, llvm::ArrayRef<ParserValue> args,
+                     Diagnostics *error) {
+  return matcherMarshallFixedImpl<ReturnType, ArgTypes...>(
+      matcherFunc, matcherName, nameRange, args, error,
+      std::index_sequence_for<ArgTypes...>{});
+}
+
+// Fixed number of arguments overload
+template <typename ReturnType, typename... ArgTypes>
+std::unique_ptr<MatcherDescriptor>
+makeMatcherAutoMarshall(ReturnType (*matcherFunc)(ArgTypes...),
+                        llvm::StringRef matcherName) {
+  // Create a vector of argument kinds
+  std::vector<ArgKind> argKinds = {ArgTypeTraits<ArgTypes>::getKind()...};
+  return std::make_unique<FixedArgCountMatcherDescriptor>(
+      matcherMarshallFixed<ReturnType, ArgTypes...>,
+      reinterpret_cast<void (*)()>(matcherFunc), matcherName, argKinds);
+}
+
+} // namespace mlir::query::matcher::internal
+
+#endif // MLIR_TOOLS_MLIRQUERY_MATCHER_MARSHALLERS_H
diff --git a/mlir/include/mlir/Query/Matcher/MatchFinder.h b/mlir/include/mlir/Query/Matcher/MatchFinder.h
new file mode 100644
index 0000000000..b008a21f53
--- /dev/null
+++ b/mlir/include/mlir/Query/Matcher/MatchFinder.h
@@ -0,0 +1,41 @@
+//===- MatchFinder.h - ------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the MatchFinder class, which is used to find operations
+// that match a given matcher.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_TOOLS_MLIRQUERY_MATCHER_MATCHERFINDER_H
+#define MLIR_TOOLS_MLIRQUERY_MATCHER_MATCHERFINDER_H
+
+#include "MatchersInternal.h"
+
+namespace mlir::query::matcher {
+
+// MatchFinder is used to find all operations that match a given matcher.
+class MatchFinder {
+public:
+  // Returns all operations that match the given matcher.
+  static std::vector<Operation *> getMatches(Operation *root,
+                                             DynMatcher matcher) {
+    std::vector<Operation *> matches;
+
+    // Simple match finding with walk.
+    root->walk([&](Operation *subOp) {
+      if (matcher.match(subOp))
+        matches.push_back(subOp);
+    });
+
+    return matches;
+  }
+};
+
+} // namespace mlir::query::matcher
+
+#endif // MLIR_TOOLS_MLIRQUERY_MATCHER_MATCHERFINDER_H
diff --git a/mlir/include/mlir/Query/Matcher/MatchersInternal.h b/mlir/include/mlir/Query/Matcher/MatchersInternal.h
new file mode 100644
index 0000000000..67455be592
--- /dev/null
+++ b/mlir/include/mlir/Query/Matcher/MatchersInternal.h
@@ -0,0 +1,72 @@
+//===- MatchersInternal.h - Structural query framework ----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implements the base layer of the matcher framework.
+//
+// Matchers are methods that return a Matcher which provides a method
+// match(Operation *op)
+//
+// The matcher functions are defined in include/mlir/IR/Matchers.h.
+// This file contains the wrapper classes needed to construct matchers for
+// mlir-query.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_TOOLS_MLIRQUERY_MATCHER_MATCHERSINTERNAL_H
+#define MLIR_TOOLS_MLIRQUERY_MATCHER_MATCHERSINTERNAL_H
+
+#include "mlir/IR/Matchers.h"
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
+
+namespace mlir::query::matcher {
+
+// Generic interface for matchers on an MLIR operation.
+class MatcherInterface
+    : public llvm::ThreadSafeRefCountedBase<MatcherInterface> {
+public:
+  virtual ~MatcherInterface() = default;
+
+  virtual bool match(Operation *op) = 0;
+};
+
+// MatcherFnImpl takes a matcher function object and implements
+// MatcherInterface.
+template <typename MatcherFn>
+class MatcherFnImpl : public MatcherInterface {
+public:
+  MatcherFnImpl(MatcherFn &matcherFn) : matcherFn(matcherFn) {}
+  bool match(Operation *op) override { return matcherFn.match(op); }
+
+private:
+  MatcherFn matcherFn;
+};
+
+// Matcher wraps a MatcherInterface implementation and provides a match()
+// method that redirects calls to the underlying implementation.
+class DynMatcher {
+public:
+  // Takes ownership of the provided implementation pointer.
+  DynMatcher(MatcherInterface *implementation)
+      : implementation(implementation) {}
+
+  template <typename MatcherFn>
+  static std::unique_ptr<DynMatcher>
+  constructDynMatcherFromMatcherFn(MatcherFn &matcherFn) {
+    auto impl = std::make_unique<MatcherFnImpl<MatcherFn>>(matcherFn);
+    return std::make_unique<DynMatcher>(impl.release());
+  }
+
+  bool match(Operation *op) const { return implementation->match(op); }
+
+private:
+  llvm::IntrusiveRefCntPtr<MatcherInterface> implementation;
+};
+
+} // namespace mlir::query::matcher
+
+#endif // MLIR_TOOLS_MLIRQUERY_MATCHER_MATCHERSINTERNAL_H
diff --git a/mlir/include/mlir/Query/Matcher/Registry.h b/mlir/include/mlir/Query/Matcher/Registry.h
new file mode 100644
index 0000000000..e929b4a04d
--- /dev/null
+++ b/mlir/include/mlir/Query/Matcher/Registry.h
@@ -0,0 +1,51 @@
+//===--- Registry.h - Matcher Registry --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Registry class to manage the registry of matchers using a map.
+//
+// This class provides a convenient interface for registering and accessing
+// matcher constructors using a string-based map.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_TOOLS_MLIRQUERY_MATCHER_REGISTRY_H
+#define MLIR_TOOLS_MLIRQUERY_MATCHER_REGISTRY_H
+
+#include "Marshallers.h"
+#include "llvm/ADT/StringMap.h"
+#include <string>
+
+namespace mlir::query::matcher {
+
+using ConstructorMap =
+    llvm::StringMap<std::unique_ptr<const internal::MatcherDescriptor>>;
+
+class Registry {
+public:
+  Registry() = default;
+  ~Registry() = default;
+
+  const ConstructorMap &constructors() const { return constructorMap; }
+
+  template <typename MatcherType>
+  void registerMatcher(const std::string &name, MatcherType matcher) {
+    registerMatcherDescriptor(name,
+                              internal::makeMatcherAutoMarshall(matcher, name));
+  }
+
+private:
+  void registerMatcherDescriptor(
+      llvm::StringRef matcherName,
+      std::unique_ptr<internal::MatcherDescriptor> callback);
+
+  ConstructorMap constructorMap;
+};
+
+} // namespace mlir::query::matcher
+
+#endif // MLIR_TOOLS_MLIRQUERY_MATCHER_REGISTRY_H
diff --git a/mlir/include/mlir/Query/Matcher/VariantValue.h b/mlir/include/mlir/Query/Matcher/VariantValue.h
new file mode 100644
index 0000000000..449f8b3a01
--- /dev/null
+++ b/mlir/include/mlir/Query/Matcher/VariantValue.h
@@ -0,0 +1,128 @@
+//===--- VariantValue.h -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Supports all the types required for dynamic Matcher construction.
+// Used by the registry to construct matchers in a generic way.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_TOOLS_MLIRQUERY_MATCHER_VARIANTVALUE_H
+#define MLIR_TOOLS_MLIRQUERY_MATCHER_VARIANTVALUE_H
+
+#include "ErrorBuilder.h"
+#include "MatchersInternal.h"
+#include "llvm/ADT/StringRef.h"
+
+namespace mlir::query::matcher {
+
+// All types that VariantValue can contain.
+enum class ArgKind { Matcher, String };
+
+// A variant matcher object to abstract simple and complex matchers into a
+// single object type.
+class VariantMatcher {
+  class MatcherOps;
+
+  // Payload interface to be specialized by each matcher type. It follows a
+  // similar interface as VariantMatcher itself.
+  class Payload {
+  public:
+    virtual ~Payload();
+    virtual std::optional<DynMatcher> getDynMatcher() const = 0;
+    virtual std::string getTypeAsString() const = 0;
+  };
+
+public:
+  // A null matcher.
+  VariantMatcher();
+
+  // Clones the provided matcher.
+  static VariantMatcher SingleMatcher(DynMatcher matcher);
+
+  // Makes the matcher the "null" matcher.
+  void reset();
+
+  // Checks if the matcher is null.
+  bool isNull() const { return !value; }
+
+  // Returns the matcher
+  std::optional<DynMatcher> getDynMatcher() const;
+
+  // String representation of the type of the value.
+  std::string getTypeAsString() const;
+
+private:
+  explicit VariantMatcher(std::shared_ptr<Payload> value)
+      : value(std::move(value)) {}
+
+  class SinglePayload;
+
+  std::shared_ptr<const Payload> value;
+};
+
+// Variant value class with a tagged union with value type semantics. It is used
+// by the registry as the return value and argument type for the matcher factory
+// methods. It can be constructed from any of the supported types:
+//  - StringRef
+//  - VariantMatcher
+class VariantValue {
+public:
+  VariantValue() : type(ValueType::Nothing) {}
+
+  VariantValue(const VariantValue &other);
+  ~VariantValue();
+  VariantValue &operator=(const VariantValue &other);
+
+  // Specific constructors for each supported type.
+  VariantValue(const llvm::StringRef string);
+  VariantValue(const VariantMatcher &matcher);
+
+  // String value functions.
+  bool isString() const;
+  const llvm::StringRef &getString() const;
+  void setString(const llvm::StringRef &string);
+
+  // Matcher value functions.
+  bool isMatcher() const;
+  const VariantMatcher &getMatcher() const;
+  void setMatcher(const VariantMatcher &matcher);
+
+  // String representation of the type of the value.
+  std::string getTypeAsString() const;
+
+private:
+  void reset();
+
+  // All supported value types.
+  enum class ValueType {
+    Nothing,
+    String,
+    Matcher,
+  };
+
+  // All supported value types.
+  union AllValues {
+    llvm::StringRef *String;
+    VariantMatcher *Matcher;
+  };
+
+  ValueType type;
+  AllValues value;
+};
+
+// A VariantValue instance annotated with its parser context.
+struct ParserValue {
+  ParserValue() {}
+  llvm::StringRef text;
+  internal::SourceRange range;
+  VariantValue value;
+};
+
+} // namespace mlir::query::matcher
+
+#endif // MLIR_TOOLS_MLIRQUERY_MATCHER_VARIANTVALUE_H
diff --git a/mlir/include/mlir/Query/Query.h b/mlir/include/mlir/Query/Query.h
new file mode 100644
index 0000000000..447fc7ca21
--- /dev/null
+++ b/mlir/include/mlir/Query/Query.h
@@ -0,0 +1,109 @@
+//===--- Query.h ------------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_TOOLS_MLIRQUERY_QUERY_H
+#define MLIR_TOOLS_MLIRQUERY_QUERY_H
+
+#include "Matcher/VariantValue.h"
+#include "mlir/Support/LogicalResult.h"
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/LineEditor/LineEditor.h"
+#include <string>
+
+namespace mlir::query {
+
+enum class QueryKind { Invalid, NoOp, Help, Match, Quit };
+
+class QuerySession;
+
+struct Query : llvm::RefCountedBase<Query> {
+  Query(QueryKind kind) : kind(kind) {}
+  virtual ~Query();
+
+  // Perform the query on qs and print output to os.
+  virtual mlir::LogicalResult run(llvm::raw_ostream &os,
+                                  QuerySession &qs) const = 0;
+
+  llvm::StringRef remainingContent;
+  const QueryKind kind;
+};
+
+typedef llvm::IntrusiveRefCntPtr<Query> QueryRef;
+
+QueryRef parse(llvm::StringRef line, const QuerySession &qs);
+
+std::vector<llvm::LineEditor::Completion>
+complete(llvm::StringRef line, size_t pos, const QuerySession &qs);
+
+// Any query which resulted in a parse error. The error message is in ErrStr.
+struct InvalidQuery : Query {
+  InvalidQuery(const llvm::Twine &errStr)
+      : Query(QueryKind::Invalid), errStr(errStr.str()) {}
+  mlir::LogicalResult run(llvm::raw_ostream &os,
+                          QuerySession &qs) const override;
+
+  std::string errStr;
+
+  static bool classof(const Query *query) {
+    return query->kind == QueryKind::Invalid;
+  }
+};
+
+// No-op query (i.e. a blank line).
+struct NoOpQuery : Query {
+  NoOpQuery() : Query(QueryKind::NoOp) {}
+  mlir::LogicalResult run(llvm::raw_ostream &os,
+                          QuerySession &qs) const override;
+
+  static bool classof(const Query *query) {
+    return query->kind == QueryKind::NoOp;
+  }
+};
+
+// Query for "help".
+struct HelpQuery : Query {
+  HelpQuery() : Query(QueryKind::Help) {}
+  mlir::LogicalResult run(llvm::raw_ostream &os,
+                          QuerySession &qs) const override;
+
+  static bool classof(const Query *query) {
+    return query->kind == QueryKind::Help;
+  }
+};
+
+// Query for "quit".
+struct QuitQuery : Query {
+  QuitQuery() : Query(QueryKind::Quit) {}
+  mlir::LogicalResult run(llvm::raw_ostream &os,
+                          QuerySession &qs) const override;
+
+  static bool classof(const Query *query) {
+    return query->kind == QueryKind::Quit;
+  }
+};
+
+// Query for "match MATCHER".
+struct MatchQuery : Query {
+  MatchQuery(llvm::StringRef source, const matcher::DynMatcher &matcher)
+      : Query(QueryKind::Match), matcher(matcher), source(source) {}
+  mlir::LogicalResult run(llvm::raw_ostream &os,
+                          QuerySession &qs) const override;
+
+  const matcher::DynMatcher matcher;
+
+  llvm::StringRef source;
+
+  static bool classof(const Query *query) {
+    return query->kind == QueryKind::Match;
+  }
+};
+
+} // namespace mlir::query
+
+#endif
diff --git a/mlir/include/mlir/Query/QuerySession.h b/mlir/include/mlir/Query/QuerySession.h
new file mode 100644
index 0000000000..b03a8cae8f
--- /dev/null
+++ b/mlir/include/mlir/Query/QuerySession.h
@@ -0,0 +1,42 @@
+//===--- QuerySession.h -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_TOOLS_MLIRQUERY_QUERYSESSION_H
+#define MLIR_TOOLS_MLIRQUERY_QUERYSESSION_H
+
+#include "llvm/ADT/StringMap.h"
+
+namespace mlir::query {
+
+class Registry;
+// Represents the state for a particular mlir-query session.
+class QuerySession {
+public:
+  QuerySession(Operation *rootOp, llvm::SourceMgr &sourceMgr, unsigned bufferId,
+               const matcher::Registry &matcherRegistry)
+      : rootOp(rootOp), sourceMgr(sourceMgr), bufferId(bufferId),
+        matcherRegistry(matcherRegistry) {}
+
+  Operation *getRootOp() { return rootOp; }
+  llvm::SourceMgr &getSourceManager() const { return sourceMgr; }
+  unsigned getBufferId() { return bufferId; }
+  const matcher::Registry &getRegistryData() const { return matcherRegistry; }
+
+  llvm::StringMap<matcher::VariantValue> namedValues;
+  bool terminate = false;
+
+private:
+  Operation *rootOp;
+  llvm::SourceMgr &sourceMgr;
+  unsigned bufferId;
+  const matcher::Registry &matcherRegistry;
+};
+
+} // namespace mlir::query
+
+#endif // MLIR_TOOLS_MLIRQUERY_QUERYSESSION_H
diff --git a/mlir/include/mlir/Support/ADTExtras.h b/mlir/include/mlir/Support/ADTExtras.h
new file mode 100644
index 0000000000..51ec7ac25d
--- /dev/null
+++ b/mlir/include/mlir/Support/ADTExtras.h
@@ -0,0 +1,83 @@
+//===- ADTExtras.h - Extra ADTs for use in MLIR -----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_SUPPORT_ADTEXTRAS_H
+#define MLIR_SUPPORT_ADTEXTRAS_H
+
+#include "mlir/Support/LLVM.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallVector.h"
+
+namespace mlir {
+
+//===----------------------------------------------------------------------===//
+// CopyOnWriteArrayRef<T>
+//===----------------------------------------------------------------------===//
+
+// A wrapper around an ArrayRef<T> that copies to a SmallVector<T> on
+// modification. This is for use in the mlir::<Type>::Builders.
+template <typename T>
+class CopyOnWriteArrayRef {
+public:
+  CopyOnWriteArrayRef(ArrayRef<T> array) : nonOwning(array){};
+
+  CopyOnWriteArrayRef &operator=(ArrayRef<T> array) {
+    nonOwning = array;
+    owningStorage = {};
+    return *this;
+  }
+
+  void insert(size_t index, T value) {
+    SmallVector<T> &vector = ensureCopy();
+    vector.insert(vector.begin() + index, value);
+  }
+
+  void erase(size_t index) {
+    // Note: A copy can be avoided when just dropping the front/back dims.
+    if (isNonOwning() && index == 0) {
+      nonOwning = nonOwning.drop_front();
+    } else if (isNonOwning() && index == size() - 1) {
+      nonOwning = nonOwning.drop_back();
+    } else {
+      SmallVector<T> &vector = ensureCopy();
+      vector.erase(vector.begin() + index);
+    }
+  }
+
+  void set(size_t index, T value) { ensureCopy()[index] = value; }
+
+  size_t size() const { return ArrayRef<T>(*this).size(); }
+
+  bool empty() const { return ArrayRef<T>(*this).empty(); }
+
+  operator ArrayRef<T>() const {
+    return nonOwning.empty() ? ArrayRef<T>(owningStorage) : nonOwning;
+  }
+
+private:
+  bool isNonOwning() const { return !nonOwning.empty(); }
+
+  SmallVector<T> &ensureCopy() {
+    // Empty non-owning storage signals the array has been copied to the owning
+    // storage (or both are empty). Note: `nonOwning` should never reference
+    // `owningStorage`. This can lead to dangling references if the
+    // CopyOnWriteArrayRef<T> is copied.
+    if (isNonOwning()) {
+      owningStorage = SmallVector<T>(nonOwning);
+      nonOwning = {};
+    }
+    return owningStorage;
+  }
+
+  ArrayRef<T> nonOwning;
+  SmallVector<T> owningStorage;
+};
+
+} // namespace mlir
+
+#endif
diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
index 68b5224055..f9026f8493 100644
--- a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
+++ b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
@@ -380,9 +380,6 @@ namespace detail {
 /// to the results of preceding blocks.
 void connectPHINodes(Region &region, const ModuleTranslation &state);
 
-/// Get a topologically sorted list of blocks of the given region.
-SetVector<Block *> getTopologicallySortedBlocks(Region &region);
-
 /// Create an LLVM IR constant of `llvmType` from the MLIR attribute `attr`.
 /// This currently supports integer, floating point, splat and dense element
 /// attributes and combinations thereof. Also, an array attribute with two
diff --git a/mlir/include/mlir/Tools/mlir-query/MlirQueryMain.h b/mlir/include/mlir/Tools/mlir-query/MlirQueryMain.h
new file mode 100644
index 0000000000..fa1cd5d817
--- /dev/null
+++ b/mlir/include/mlir/Tools/mlir-query/MlirQueryMain.h
@@ -0,0 +1,30 @@
+//===- MlirQueryMain.h - MLIR Query main ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Main entry function for mlir-query for when built as standalone
+// binary.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_TOOLS_MLIRQUERY_MLIRQUERYMAIN_H
+#define MLIR_TOOLS_MLIRQUERY_MLIRQUERYMAIN_H
+
+#include "mlir/Query/Matcher/Registry.h"
+#include "mlir/Support/LogicalResult.h"
+
+namespace mlir {
+
+class MLIRContext;
+
+LogicalResult
+mlirQueryMain(int argc, char **argv, MLIRContext &context,
+              const mlir::query::matcher::Registry &matcherRegistry);
+
+} // namespace mlir
+
+#endif // MLIR_TOOLS_MLIRQUERY_MLIRQUERYMAIN_H
diff --git a/mlir/include/mlir/Transforms/RegionUtils.h b/mlir/include/mlir/Transforms/RegionUtils.h
index 06eebff201..192ff71384 100644
--- a/mlir/include/mlir/Transforms/RegionUtils.h
+++ b/mlir/include/mlir/Transforms/RegionUtils.h
@@ -87,6 +87,9 @@ LogicalResult eraseUnreachableBlocks(RewriterBase &rewriter,
 LogicalResult runRegionDCE(RewriterBase &rewriter,
                            MutableArrayRef<Region> regions);
 
+/// Get a topologically sorted list of blocks of the given region.
+SetVector<Block *> getTopologicallySortedBlocks(Region &region);
+
 } // namespace mlir
 
 #endif // MLIR_TRANSFORMS_REGIONUTILS_H_
diff --git a/mlir/lib/Analysis/CallGraph.cpp b/mlir/lib/Analysis/CallGraph.cpp
index 614e8e24ff..1129226471 100644
--- a/mlir/lib/Analysis/CallGraph.cpp
+++ b/mlir/lib/Analysis/CallGraph.cpp
@@ -133,7 +133,7 @@ CallGraphNode *CallGraph::getOrAddNode(Region *region,
 /// Lookup a call graph node for the given region, or nullptr if none is
 /// registered.
 CallGraphNode *CallGraph::lookupNode(Region *region) const {
-  auto it = nodes.find(region);
+  const auto *it = nodes.find(region);
   return it == nodes.end() ? nullptr : it->second.get();
 }
 
diff --git a/mlir/lib/Analysis/DataFlow/DeadCodeAnalysis.cpp b/mlir/lib/Analysis/DataFlow/DeadCodeAnalysis.cpp
index bbe139fd4f..7a39e3ae34 100644
--- a/mlir/lib/Analysis/DataFlow/DeadCodeAnalysis.cpp
+++ b/mlir/lib/Analysis/DataFlow/DeadCodeAnalysis.cpp
@@ -259,8 +259,7 @@ LogicalResult DeadCodeAnalysis::visit(ProgramPoint point) {
   if (isRegionOrCallableReturn(op)) {
     if (auto branch = dyn_cast<RegionBranchOpInterface>(op->getParentOp())) {
       // Visit the exiting terminator of a region.
-      visitRegionTerminator(cast<RegionBranchTerminatorOpInterface>(op),
-                            branch);
+      visitRegionTerminator(op, branch);
     } else if (auto callable =
                    dyn_cast<CallableOpInterface>(op->getParentOp())) {
       // Visit the exiting terminator of a callable.
@@ -379,14 +378,17 @@ void DeadCodeAnalysis::visitRegionBranchOperation(
   }
 }
 
-void DeadCodeAnalysis::visitRegionTerminator(
-    RegionBranchTerminatorOpInterface op, RegionBranchOpInterface branch) {
+void DeadCodeAnalysis::visitRegionTerminator(Operation *op,
+                                             RegionBranchOpInterface branch) {
   std::optional<SmallVector<Attribute>> operands = getOperandValues(op);
   if (!operands)
     return;
 
   SmallVector<RegionSuccessor> successors;
-  op.getSuccessorRegions(*operands, successors);
+  if (auto terminator = dyn_cast<RegionBranchTerminatorOpInterface>(op))
+    terminator.getSuccessorRegions(*operands, successors);
+  else
+    branch.getSuccessorRegions(op->getParentRegion(), successors);
 
   // Mark successor region entry blocks as executable and add this op to the
   // list of predecessors.
diff --git a/mlir/lib/Analysis/Presburger/Matrix.cpp b/mlir/lib/Analysis/Presburger/Matrix.cpp
index f0bcb09fb2..ae97e456d9 100644
--- a/mlir/lib/Analysis/Presburger/Matrix.cpp
+++ b/mlir/lib/Analysis/Presburger/Matrix.cpp
@@ -14,35 +14,41 @@
 using namespace mlir;
 using namespace presburger;
 
-template <typename T> Matrix<T>::Matrix(unsigned rows, unsigned columns, unsigned reservedRows,
-               unsigned reservedColumns)
+template <typename T>
+Matrix<T>::Matrix(unsigned rows, unsigned columns, unsigned reservedRows,
+                  unsigned reservedColumns)
     : nRows(rows), nColumns(columns),
       nReservedColumns(std::max(nColumns, reservedColumns)),
       data(nRows * nReservedColumns) {
   data.reserve(std::max(nRows, reservedRows) * nReservedColumns);
 }
 
-template <typename T> Matrix<T> Matrix<T>::identity(unsigned dimension) {
+template <typename T>
+Matrix<T> Matrix<T>::identity(unsigned dimension) {
   Matrix matrix(dimension, dimension);
   for (unsigned i = 0; i < dimension; ++i)
     matrix(i, i) = 1;
   return matrix;
 }
 
-template <typename T> unsigned Matrix<T>::getNumReservedRows() const {
+template <typename T>
+unsigned Matrix<T>::getNumReservedRows() const {
   return data.capacity() / nReservedColumns;
 }
 
-template <typename T> void Matrix<T>::reserveRows(unsigned rows) {
+template <typename T>
+void Matrix<T>::reserveRows(unsigned rows) {
   data.reserve(rows * nReservedColumns);
 }
 
-template <typename T> unsigned Matrix<T>::appendExtraRow() {
+template <typename T>
+unsigned Matrix<T>::appendExtraRow() {
   resizeVertically(nRows + 1);
   return nRows - 1;
 }
 
-template <typename T> unsigned Matrix<T>::appendExtraRow(ArrayRef<T> elems) {
+template <typename T>
+unsigned Matrix<T>::appendExtraRow(ArrayRef<T> elems) {
   assert(elems.size() == nColumns && "elems must match row length!");
   unsigned row = appendExtraRow();
   for (unsigned col = 0; col < nColumns; ++col)
@@ -50,24 +56,28 @@ template <typename T> unsigned Matrix<T>::appendExtraRow(ArrayRef<T> elems) {
   return row;
 }
 
-template <typename T> void Matrix<T>::resizeHorizontally(unsigned newNColumns) {
+template <typename T>
+void Matrix<T>::resizeHorizontally(unsigned newNColumns) {
   if (newNColumns < nColumns)
     removeColumns(newNColumns, nColumns - newNColumns);
   if (newNColumns > nColumns)
     insertColumns(nColumns, newNColumns - nColumns);
 }
 
-template <typename T> void Matrix<T>::resize(unsigned newNRows, unsigned newNColumns) {
+template <typename T>
+void Matrix<T>::resize(unsigned newNRows, unsigned newNColumns) {
   resizeHorizontally(newNColumns);
   resizeVertically(newNRows);
 }
 
-template <typename T> void Matrix<T>::resizeVertically(unsigned newNRows) {
+template <typename T>
+void Matrix<T>::resizeVertically(unsigned newNRows) {
   nRows = newNRows;
   data.resize(nRows * nReservedColumns);
 }
 
-template <typename T> void Matrix<T>::swapRows(unsigned row, unsigned otherRow) {
+template <typename T>
+void Matrix<T>::swapRows(unsigned row, unsigned otherRow) {
   assert((row < getNumRows() && otherRow < getNumRows()) &&
          "Given row out of bounds");
   if (row == otherRow)
@@ -76,7 +86,8 @@ template <typename T> void Matrix<T>::swapRows(unsigned row, unsigned otherRow)
     std::swap(at(row, col), at(otherRow, col));
 }
 
-template <typename T> void Matrix<T>::swapColumns(unsigned column, unsigned otherColumn) {
+template <typename T>
+void Matrix<T>::swapColumns(unsigned column, unsigned otherColumn) {
   assert((column < getNumColumns() && otherColumn < getNumColumns()) &&
          "Given column out of bounds");
   if (column == otherColumn)
@@ -85,23 +96,30 @@ template <typename T> void Matrix<T>::swapColumns(unsigned column, unsigned othe
     std::swap(at(row, column), at(row, otherColumn));
 }
 
-template <typename T> MutableArrayRef<T> Matrix<T>::getRow(unsigned row) {
+template <typename T>
+MutableArrayRef<T> Matrix<T>::getRow(unsigned row) {
   return {&data[row * nReservedColumns], nColumns};
 }
 
-template <typename T> ArrayRef<T> Matrix<T>::getRow(unsigned row) const {
+template <typename T>
+ArrayRef<T> Matrix<T>::getRow(unsigned row) const {
   return {&data[row * nReservedColumns], nColumns};
 }
 
-template <typename T> void Matrix<T>::setRow(unsigned row, ArrayRef<T> elems) {
+template <typename T>
+void Matrix<T>::setRow(unsigned row, ArrayRef<T> elems) {
   assert(elems.size() == getNumColumns() &&
          "elems size must match row length!");
   for (unsigned i = 0, e = getNumColumns(); i < e; ++i)
     at(row, i) = elems[i];
 }
 
-template <typename T> void Matrix<T>::insertColumn(unsigned pos) { insertColumns(pos, 1); }
-template <typename T> void Matrix<T>::insertColumns(unsigned pos, unsigned count) {
+template <typename T>
+void Matrix<T>::insertColumn(unsigned pos) {
+  insertColumns(pos, 1);
+}
+template <typename T>
+void Matrix<T>::insertColumns(unsigned pos, unsigned count) {
   if (count == 0)
     return;
   assert(pos <= nColumns);
@@ -142,8 +160,12 @@ template <typename T> void Matrix<T>::insertColumns(unsigned pos, unsigned count
   }
 }
 
-template <typename T> void Matrix<T>::removeColumn(unsigned pos) { removeColumns(pos, 1); }
-template <typename T> void Matrix<T>::removeColumns(unsigned pos, unsigned count) {
+template <typename T>
+void Matrix<T>::removeColumn(unsigned pos) {
+  removeColumns(pos, 1);
+}
+template <typename T>
+void Matrix<T>::removeColumns(unsigned pos, unsigned count) {
   if (count == 0)
     return;
   assert(pos + count - 1 < nColumns);
@@ -156,8 +178,12 @@ template <typename T> void Matrix<T>::removeColumns(unsigned pos, unsigned count
   nColumns -= count;
 }
 
-template <typename T> void Matrix<T>::insertRow(unsigned pos) { insertRows(pos, 1); }
-template <typename T> void Matrix<T>::insertRows(unsigned pos, unsigned count) {
+template <typename T>
+void Matrix<T>::insertRow(unsigned pos) {
+  insertRows(pos, 1);
+}
+template <typename T>
+void Matrix<T>::insertRows(unsigned pos, unsigned count) {
   if (count == 0)
     return;
 
@@ -170,8 +196,12 @@ template <typename T> void Matrix<T>::insertRows(unsigned pos, unsigned count) {
       at(r, c) = 0;
 }
 
-template <typename T> void Matrix<T>::removeRow(unsigned pos) { removeRows(pos, 1); }
-template <typename T> void Matrix<T>::removeRows(unsigned pos, unsigned count) {
+template <typename T>
+void Matrix<T>::removeRow(unsigned pos) {
+  removeRows(pos, 1);
+}
+template <typename T>
+void Matrix<T>::removeRows(unsigned pos, unsigned count) {
   if (count == 0)
     return;
   assert(pos + count - 1 <= nRows);
@@ -180,50 +210,57 @@ template <typename T> void Matrix<T>::removeRows(unsigned pos, unsigned count) {
   resizeVertically(nRows - count);
 }
 
-template <typename T> void Matrix<T>::copyRow(unsigned sourceRow, unsigned targetRow) {
+template <typename T>
+void Matrix<T>::copyRow(unsigned sourceRow, unsigned targetRow) {
   if (sourceRow == targetRow)
     return;
   for (unsigned c = 0; c < nColumns; ++c)
     at(targetRow, c) = at(sourceRow, c);
 }
 
-template <typename T> void Matrix<T>::fillRow(unsigned row, const T &value) {
+template <typename T>
+void Matrix<T>::fillRow(unsigned row, const T &value) {
   for (unsigned col = 0; col < nColumns; ++col)
     at(row, col) = value;
 }
 
-template <typename T> void Matrix<T>::addToRow(unsigned sourceRow, unsigned targetRow,
-                      const T &scale) {
+template <typename T>
+void Matrix<T>::addToRow(unsigned sourceRow, unsigned targetRow,
+                         const T &scale) {
   addToRow(targetRow, getRow(sourceRow), scale);
 }
 
-template <typename T> void Matrix<T>::addToRow(unsigned row, ArrayRef<T> rowVec,
-                      const T &scale) {
+template <typename T>
+void Matrix<T>::addToRow(unsigned row, ArrayRef<T> rowVec, const T &scale) {
   if (scale == 0)
     return;
   for (unsigned col = 0; col < nColumns; ++col)
     at(row, col) += scale * rowVec[col];
 }
 
-template <typename T> void Matrix<T>::addToColumn(unsigned sourceColumn, unsigned targetColumn,
-                         const T &scale) {
+template <typename T>
+void Matrix<T>::addToColumn(unsigned sourceColumn, unsigned targetColumn,
+                            const T &scale) {
   if (scale == 0)
     return;
   for (unsigned row = 0, e = getNumRows(); row < e; ++row)
     at(row, targetColumn) += scale * at(row, sourceColumn);
 }
 
-template <typename T> void Matrix<T>::negateColumn(unsigned column) {
+template <typename T>
+void Matrix<T>::negateColumn(unsigned column) {
   for (unsigned row = 0, e = getNumRows(); row < e; ++row)
     at(row, column) = -at(row, column);
 }
 
-template <typename T> void Matrix<T>::negateRow(unsigned row) {
+template <typename T>
+void Matrix<T>::negateRow(unsigned row) {
   for (unsigned column = 0, e = getNumColumns(); column < e; ++column)
     at(row, column) = -at(row, column);
 }
 
-template <typename T> SmallVector<T, 8> Matrix<T>::preMultiplyWithRow(ArrayRef<T> rowVec) const {
+template <typename T>
+SmallVector<T, 8> Matrix<T>::preMultiplyWithRow(ArrayRef<T> rowVec) const {
   assert(rowVec.size() == getNumRows() && "Invalid row vector dimension!");
 
   SmallVector<T, 8> result(getNumColumns(), T(0));
@@ -233,8 +270,8 @@ template <typename T> SmallVector<T, 8> Matrix<T>::preMultiplyWithRow(ArrayRef<T
   return result;
 }
 
-template <typename T> SmallVector<T, 8>
-Matrix<T>::postMultiplyWithColumn(ArrayRef<T> colVec) const {
+template <typename T>
+SmallVector<T, 8> Matrix<T>::postMultiplyWithColumn(ArrayRef<T> colVec) const {
   assert(getNumColumns() == colVec.size() &&
          "Invalid column vector dimension!");
 
@@ -250,8 +287,9 @@ Matrix<T>::postMultiplyWithColumn(ArrayRef<T> colVec) const {
 /// sourceCol. This brings M(row, targetCol) to the range [0, M(row,
 /// sourceCol)). Apply the same column operation to otherMatrix, with the same
 /// integer multiple.
-static void modEntryColumnOperation(Matrix<MPInt> &m, unsigned row, unsigned sourceCol,
-                                    unsigned targetCol, Matrix<MPInt> &otherMatrix) {
+static void modEntryColumnOperation(Matrix<MPInt> &m, unsigned row,
+                                    unsigned sourceCol, unsigned targetCol,
+                                    Matrix<MPInt> &otherMatrix) {
   assert(m(row, sourceCol) != 0 && "Cannot divide by zero!");
   assert(m(row, sourceCol) > 0 && "Source must be positive!");
   MPInt ratio = -floorDiv(m(row, targetCol), m(row, sourceCol));
@@ -259,7 +297,8 @@ static void modEntryColumnOperation(Matrix<MPInt> &m, unsigned row, unsigned sou
   otherMatrix.addToColumn(sourceCol, targetCol, ratio);
 }
 
-template <typename T> void Matrix<T>::print(raw_ostream &os) const {
+template <typename T>
+void Matrix<T>::print(raw_ostream &os) const {
   for (unsigned row = 0; row < nRows; ++row) {
     for (unsigned column = 0; column < nColumns; ++column)
       os << at(row, column) << ' ';
@@ -267,9 +306,13 @@ template <typename T> void Matrix<T>::print(raw_ostream &os) const {
   }
 }
 
-template <typename T> void Matrix<T>::dump() const { print(llvm::errs()); }
+template <typename T>
+void Matrix<T>::dump() const {
+  print(llvm::errs());
+}
 
-template <typename T> bool Matrix<T>::hasConsistentState() const {
+template <typename T>
+bool Matrix<T>::hasConsistentState() const {
   if (data.size() != nRows * nReservedColumns)
     return false;
   if (nColumns > nReservedColumns)
@@ -287,8 +330,8 @@ namespace mlir {
 namespace presburger {
 template class Matrix<MPInt>;
 template class Matrix<Fraction>;
-}
-}
+} // namespace presburger
+} // namespace mlir
 
 IntMatrix IntMatrix::identity(unsigned dimension) {
   IntMatrix matrix(dimension, dimension);
@@ -297,7 +340,6 @@ IntMatrix IntMatrix::identity(unsigned dimension) {
   return matrix;
 }
 
-
 std::pair<IntMatrix, IntMatrix> IntMatrix::computeHermiteNormalForm() const {
   // We start with u as an identity matrix and perform operations on h until h
   // is in hermite normal form. We apply the same sequence of operations on u to
@@ -390,4 +432,120 @@ MPInt IntMatrix::normalizeRow(unsigned row, unsigned cols) {
 
 MPInt IntMatrix::normalizeRow(unsigned row) {
   return normalizeRow(row, getNumColumns());
+}
+
+MPInt IntMatrix::determinant(IntMatrix *inverse) const {
+  assert(nRows == nColumns &&
+         "determinant can only be calculated for square matrices!");
+
+  FracMatrix m(*this);
+
+  FracMatrix fracInverse(nRows, nColumns);
+  MPInt detM = m.determinant(&fracInverse).getAsInteger();
+
+  if (detM == 0)
+    return MPInt(0);
+
+  *inverse = IntMatrix(nRows, nColumns);
+  for (unsigned i = 0; i < nRows; i++)
+    for (unsigned j = 0; j < nColumns; j++)
+      inverse->at(i, j) = (fracInverse.at(i, j) * detM).getAsInteger();
+
+  return detM;
+}
+
+FracMatrix FracMatrix::identity(unsigned dimension) {
+  return Matrix::identity(dimension);
+}
+
+FracMatrix::FracMatrix(IntMatrix m)
+    : FracMatrix(m.getNumRows(), m.getNumColumns()) {
+  for (unsigned i = 0; i < m.getNumRows(); i++)
+    for (unsigned j = 0; j < m.getNumColumns(); j++)
+      this->at(i, j) = m.at(i, j);
+}
+
+Fraction FracMatrix::determinant(FracMatrix *inverse) const {
+  assert(nRows == nColumns &&
+         "determinant can only be calculated for square matrices!");
+
+  FracMatrix m(*this);
+  FracMatrix tempInv(nRows, nColumns);
+  if (inverse)
+    tempInv = FracMatrix::identity(nRows);
+
+  Fraction a, b;
+  // Make the matrix into upper triangular form using
+  // gaussian elimination with row operations.
+  // If inverse is required, we apply more operations
+  // to turn the matrix into diagonal form. We apply
+  // the same operations to the inverse matrix,
+  // which is initially identity.
+  // Either way, the product of the diagonal elements
+  // is then the determinant.
+  for (unsigned i = 0; i < nRows; i++) {
+    if (m(i, i) == 0)
+      // First ensure that the diagonal
+      // element is nonzero, by swapping
+      // it with a nonzero row.
+      for (unsigned j = i + 1; j < nRows; j++) {
+        if (m(j, i) != 0) {
+          m.swapRows(j, i);
+          if (inverse)
+            tempInv.swapRows(j, i);
+          break;
+        }
+      }
+
+    b = m.at(i, i);
+    if (b == 0)
+      return 0;
+
+    // Set all elements above the
+    // diagonal to zero.
+    if (inverse) {
+      for (unsigned j = 0; j < i; j++) {
+        if (m.at(j, i) == 0)
+          continue;
+        a = m.at(j, i);
+        // Set element (j, i) to zero
+        // by subtracting the ith row,
+        // appropriately scaled.
+        m.addToRow(i, j, -a / b);
+        tempInv.addToRow(i, j, -a / b);
+      }
+    }
+
+    // Set all elements below the
+    // diagonal to zero.
+    for (unsigned j = i + 1; j < nRows; j++) {
+      if (m.at(j, i) == 0)
+        continue;
+      a = m.at(j, i);
+      // Set element (j, i) to zero
+      // by subtracting the ith row,
+      // appropriately scaled.
+      m.addToRow(i, j, -a / b);
+      if (inverse)
+        tempInv.addToRow(i, j, -a / b);
+    }
+  }
+
+  // Now only diagonal elements of m are nonzero, but they are
+  // not necessarily 1. To get the true inverse, we should
+  // normalize them and apply the same scale to the inverse matrix.
+  // For efficiency we skip scaling m and just scale tempInv appropriately.
+  if (inverse) {
+    for (unsigned i = 0; i < nRows; i++)
+      for (unsigned j = 0; j < nRows; j++)
+        tempInv.at(i, j) = tempInv.at(i, j) / m(i, i);
+
+    *inverse = std::move(tempInv);
+  }
+
+  Fraction determinant = 1;
+  for (unsigned i = 0; i < nRows; i++)
+    determinant *= m.at(i, i);
+
+  return determinant;
 }
\ No newline at end of file
diff --git a/mlir/lib/Analysis/Presburger/PresburgerRelation.cpp b/mlir/lib/Analysis/Presburger/PresburgerRelation.cpp
index 0b3f6a3912..5a9cf71fc8 100644
--- a/mlir/lib/Analysis/Presburger/PresburgerRelation.cpp
+++ b/mlir/lib/Analysis/Presburger/PresburgerRelation.cpp
@@ -43,7 +43,7 @@ void PresburgerRelation::convertVarKind(VarKind srcKind, unsigned srcPos,
                                         unsigned num, VarKind dstKind,
                                         unsigned dstPos) {
   assert(srcKind != VarKind::Local && dstKind != VarKind::Local &&
-      "srcKind/dstKind cannot be local");
+         "srcKind/dstKind cannot be local");
   assert(srcKind != dstKind && "cannot convert variables to the same kind");
   assert(srcPos + num <= space.getNumVarKind(srcKind) &&
          "invalid range for source variables");
@@ -636,17 +636,13 @@ bool PresburgerRelation::isPlainEqual(const PresburgerRelation &set) const {
 /// one unconstrained disjunct, indicating the absence of constraints or
 /// conditions.
 bool PresburgerRelation::isPlainUniverse() const {
-  for (auto &disjunct : getAllDisjuncts()) {
-    if (disjunct.getNumConstraints() == 0)
-      return true;
-  }
-  return false;
+  return llvm::any_of(getAllDisjuncts(), [](const IntegerRelation &disjunct) {
+    return disjunct.getNumConstraints() == 0;
+  });
 }
 
 bool PresburgerRelation::isConvexNoLocals() const {
-  if (getNumDisjuncts() == 1 && getSpace().getNumLocalVars() == 0)
-    return true;
-  return false;
+  return getNumDisjuncts() == 1 && getSpace().getNumLocalVars() == 0;
 }
 
 /// Return true if there is no disjunct, false otherwise.
@@ -823,8 +819,8 @@ PresburgerRelation SetCoalescer::coalesce() {
   }
 
   PresburgerRelation newSet = PresburgerRelation::getEmpty(space);
-  for (unsigned i = 0, e = disjuncts.size(); i < e; ++i)
-    newSet.unionInPlace(disjuncts[i]);
+  for (const IntegerRelation &disjunct : disjuncts)
+    newSet.unionInPlace(disjunct);
 
   return newSet;
 }
diff --git a/mlir/lib/Analysis/Presburger/Utils.cpp b/mlir/lib/Analysis/Presburger/Utils.cpp
index e7fd2843b9..9aef2f5de1 100644
--- a/mlir/lib/Analysis/Presburger/Utils.cpp
+++ b/mlir/lib/Analysis/Presburger/Utils.cpp
@@ -502,8 +502,8 @@ void DivisionRepr::print(raw_ostream &os) const {
   os << "Dividends:\n";
   dividends.print(os);
   os << "Denominators\n";
-  for (unsigned i = 0, e = denoms.size(); i < e; ++i)
-    os << denoms[i] << " ";
+  for (const MPInt &denom : denoms)
+    os << denom << " ";
   os << "\n";
 }
 
diff --git a/mlir/lib/AsmParser/AttributeParser.cpp b/mlir/lib/AsmParser/AttributeParser.cpp
index b1991ce06f..d085fb6af6 100644
--- a/mlir/lib/AsmParser/AttributeParser.cpp
+++ b/mlir/lib/AsmParser/AttributeParser.cpp
@@ -309,7 +309,7 @@ ParseResult Parser::parseAttributeDict(NamedAttrList &attributes) {
     else
       return emitWrongTokenError("expected attribute name");
 
-    if (nameId->size() == 0)
+    if (nameId->empty())
       return emitError("expected valid attribute name");
 
     if (!seenKeys.insert(*nameId).second)
@@ -1225,6 +1225,7 @@ Attribute Parser::parseStridedLayoutAttr() {
 ///                         `[` integer-literal `]<` attribute-value `>`
 ///
 Attribute Parser::parseDistinctAttr(Type type) {
+  SMLoc loc = getToken().getLoc();
   consumeToken(Token::kw_distinct);
   if (parseToken(Token::l_square, "expected '[' after 'distinct'"))
     return {};
@@ -1269,7 +1270,7 @@ Attribute Parser::parseDistinctAttr(Type type) {
     DistinctAttr distinctAttr = DistinctAttr::create(referencedAttr);
     it = distinctAttrs.try_emplace(*value, distinctAttr).first;
   } else if (it->getSecond().getReferencedAttr() != referencedAttr) {
-    emitError("referenced attribute does not match previous definition: ")
+    emitError(loc, "referenced attribute does not match previous definition: ")
         << it->getSecond().getReferencedAttr();
     return {};
   }
diff --git a/mlir/lib/Bindings/Python/DialectSparseTensor.cpp b/mlir/lib/Bindings/Python/DialectSparseTensor.cpp
index 8e9e0b6baf..9bde3a443e 100644
--- a/mlir/lib/Bindings/Python/DialectSparseTensor.cpp
+++ b/mlir/lib/Bindings/Python/DialectSparseTensor.cpp
@@ -41,16 +41,17 @@ static void populateDialectSparseTensorSubmodule(const py::module &m) {
       .def_classmethod(
           "get",
           [](py::object cls, std::vector<MlirSparseTensorDimLevelType> lvlTypes,
-             std::optional<MlirAffineMap> dimToLvl, int posWidth, int crdWidth,
+             std::optional<MlirAffineMap> dimToLvl,
+             std::optional<MlirAffineMap> lvlToDim, int posWidth, int crdWidth,
              MlirContext context) {
-            // TODO: provide dimToLvl
             return cls(mlirSparseTensorEncodingAttrGet(
                 context, lvlTypes.size(), lvlTypes.data(),
-                dimToLvl ? *dimToLvl : MlirAffineMap{nullptr}, posWidth,
+                dimToLvl ? *dimToLvl : MlirAffineMap{nullptr},
+                lvlToDim ? *lvlToDim : MlirAffineMap{nullptr}, posWidth,
                 crdWidth));
           },
           py::arg("cls"), py::arg("lvl_types"), py::arg("dim_to_lvl"),
-          py::arg("pos_width"), py::arg("crd_width"),
+          py::arg("lvl_to_dim"), py::arg("pos_width"), py::arg("crd_width"),
           py::arg("context") = py::none(),
           "Gets a sparse_tensor.encoding from parameters.")
       .def_property_readonly(
@@ -71,6 +72,14 @@ static void populateDialectSparseTensorSubmodule(const py::module &m) {
               return {};
             return ret;
           })
+      .def_property_readonly(
+          "lvl_to_dim",
+          [](MlirAttribute self) -> std::optional<MlirAffineMap> {
+            MlirAffineMap ret = mlirSparseTensorEncodingAttrGetLvlToDim(self);
+            if (mlirAffineMapIsNull(ret))
+              return {};
+            return ret;
+          })
       .def_property_readonly("pos_width",
                              mlirSparseTensorEncodingAttrGetPosWidth)
       .def_property_readonly("crd_width",
diff --git a/mlir/lib/Bindings/Python/Globals.h b/mlir/lib/Bindings/Python/Globals.h
index 97cd70089a..21899bdce2 100644
--- a/mlir/lib/Bindings/Python/Globals.h
+++ b/mlir/lib/Bindings/Python/Globals.h
@@ -77,10 +77,10 @@ class PyGlobals {
                            pybind11::object pyClass);
 
   /// Adds a concrete implementation operation class.
-  /// Raises an exception if the mapping already exists.
+  /// Raises an exception if the mapping already exists and replace == false.
   /// This is intended to be called by implementation code.
   void registerOperationImpl(const std::string &operationName,
-                             pybind11::object pyClass);
+                             pybind11::object pyClass, bool replace = false);
 
   /// Returns the custom Attribute builder for Attribute kind.
   std::optional<pybind11::function>
diff --git a/mlir/lib/Bindings/Python/IRCore.cpp b/mlir/lib/Bindings/Python/IRCore.cpp
index c8373e06f0..a8ea1a381e 100644
--- a/mlir/lib/Bindings/Python/IRCore.cpp
+++ b/mlir/lib/Bindings/Python/IRCore.cpp
@@ -635,6 +635,11 @@ size_t PyMlirContext::clearLiveOperations() {
   return numInvalidated;
 }
 
+void PyMlirContext::setOperationInvalid(MlirOperation op) {
+  if (liveOperations.contains(op.ptr))
+    liveOperations[op.ptr].second->setInvalid();
+}
+
 size_t PyMlirContext::getLiveModuleCount() { return liveModules.size(); }
 
 pybind11::object PyMlirContext::contextEnter() {
@@ -3207,7 +3212,18 @@ void mlir::python::populateIRCore(py::module &m) {
            "Inserts an operation.")
       .def_property_readonly(
           "block", [](PyInsertionPoint &self) { return self.getBlock(); },
-          "Returns the block that this InsertionPoint points to.");
+          "Returns the block that this InsertionPoint points to.")
+      .def_property_readonly(
+          "ref_operation",
+          [](PyInsertionPoint &self) -> py::object {
+            auto ref_operation = self.getRefOperation();
+            if (ref_operation)
+              return ref_operation->getObject();
+            return py::none();
+          },
+          "The reference operation before which new operations are "
+          "inserted, or None if the insertion point is at the end of "
+          "the block");
 
   //----------------------------------------------------------------------------
   // Mapping of PyAttribute.
diff --git a/mlir/lib/Bindings/Python/IRModule.cpp b/mlir/lib/Bindings/Python/IRModule.cpp
index 2cc66277ab..a1c8ab7a09 100644
--- a/mlir/lib/Bindings/Python/IRModule.cpp
+++ b/mlir/lib/Bindings/Python/IRModule.cpp
@@ -96,9 +96,9 @@ void PyGlobals::registerDialectImpl(const std::string &dialectNamespace,
 }
 
 void PyGlobals::registerOperationImpl(const std::string &operationName,
-                                      py::object pyClass) {
+                                      py::object pyClass, bool replace) {
   py::object &found = operationClassMap[operationName];
-  if (found) {
+  if (found && !replace) {
     throw std::runtime_error((llvm::Twine("Operation '") + operationName +
                               "' is already registered.")
                                  .str());
diff --git a/mlir/lib/Bindings/Python/IRModule.h b/mlir/lib/Bindings/Python/IRModule.h
index 3ca7dd8519..2629288571 100644
--- a/mlir/lib/Bindings/Python/IRModule.h
+++ b/mlir/lib/Bindings/Python/IRModule.h
@@ -209,6 +209,11 @@ class PyMlirContext {
   /// place.
   size_t clearLiveOperations();
 
+  /// Sets an operation invalid. This is useful for when some non-bindings
+  /// code destroys the operation and the bindings need to made aware. For
+  /// example, in the case when pass manager is run.
+  void setOperationInvalid(MlirOperation op);
+
   /// Gets the count of live modules associated with this context.
   /// Used for testing.
   size_t getLiveModuleCount();
@@ -833,6 +838,7 @@ class PyInsertionPoint {
                    const pybind11::object &excTb);
 
   PyBlock &getBlock() { return block; }
+  std::optional<PyOperationRef> &getRefOperation() { return refOperation; }
 
 private:
   // Trampoline constructor that avoids null initializing members while
diff --git a/mlir/lib/Bindings/Python/MainModule.cpp b/mlir/lib/Bindings/Python/MainModule.cpp
index cdddfbe506..a936becf67 100644
--- a/mlir/lib/Bindings/Python/MainModule.cpp
+++ b/mlir/lib/Bindings/Python/MainModule.cpp
@@ -41,7 +41,7 @@ PYBIND11_MODULE(_mlir, m) {
            "dialect_namespace"_a, "dialect_class"_a,
            "Testing hook for directly registering a dialect")
       .def("_register_operation_impl", &PyGlobals::registerOperationImpl,
-           "operation_name"_a, "operation_class"_a,
+           "operation_name"_a, "operation_class"_a, "replace"_a = false,
            "Testing hook for directly registering an operation");
 
   // Aside from making the globals accessible to python, having python manage
@@ -63,12 +63,13 @@ PYBIND11_MODULE(_mlir, m) {
       "Class decorator for registering a custom Dialect wrapper");
   m.def(
       "register_operation",
-      [](const py::object &dialectClass) -> py::cpp_function {
+      [](const py::object &dialectClass, bool replace) -> py::cpp_function {
         return py::cpp_function(
-            [dialectClass](py::object opClass) -> py::object {
+            [dialectClass, replace](py::object opClass) -> py::object {
               std::string operationName =
                   opClass.attr("OPERATION_NAME").cast<std::string>();
-              PyGlobals::get().registerOperationImpl(operationName, opClass);
+              PyGlobals::get().registerOperationImpl(operationName, opClass,
+                                                     replace);
 
               // Dict-stuff the new opClass by name onto the dialect class.
               py::object opClassName = opClass.attr("__name__");
@@ -76,7 +77,7 @@ PYBIND11_MODULE(_mlir, m) {
               return opClass;
             });
       },
-      "dialect_class"_a,
+      "dialect_class"_a, "replace"_a = false,
       "Produce a class decorator for registering an Operation class as part of "
       "a dialect");
   m.def(
diff --git a/mlir/lib/Bindings/Python/Pass.cpp b/mlir/lib/Bindings/Python/Pass.cpp
index cdbfcfbc22..2175cea799 100644
--- a/mlir/lib/Bindings/Python/Pass.cpp
+++ b/mlir/lib/Bindings/Python/Pass.cpp
@@ -13,6 +13,7 @@
 #include "mlir-c/Pass.h"
 
 namespace py = pybind11;
+using namespace py::literals;
 using namespace mlir;
 using namespace mlir::python;
 
@@ -63,8 +64,7 @@ void mlir::python::populatePassManagerSubmodule(py::module &m) {
                  mlirStringRefCreate(anchorOp.data(), anchorOp.size()));
              return new PyPassManager(passManager);
            }),
-           py::arg("anchor_op") = py::str("any"),
-           py::arg("context") = py::none(),
+           "anchor_op"_a = py::str("any"), "context"_a = py::none(),
            "Create a new PassManager for the current (or provided) Context.")
       .def_property_readonly(MLIR_PYTHON_CAPI_PTR_ATTR,
                              &PyPassManager::getCapsule)
@@ -82,7 +82,7 @@ void mlir::python::populatePassManagerSubmodule(py::module &m) {
           [](PyPassManager &passManager, bool enable) {
             mlirPassManagerEnableVerifier(passManager.get(), enable);
           },
-          py::arg("enable"), "Enable / disable verify-each.")
+          "enable"_a, "Enable / disable verify-each.")
       .def_static(
           "parse",
           [](const std::string &pipeline, DefaultingPyMlirContext context) {
@@ -96,7 +96,7 @@ void mlir::python::populatePassManagerSubmodule(py::module &m) {
               throw py::value_error(std::string(errorMsg.join()));
             return new PyPassManager(passManager);
           },
-          py::arg("pipeline"), py::arg("context") = py::none(),
+          "pipeline"_a, "context"_a = py::none(),
           "Parse a textual pass-pipeline and return a top-level PassManager "
           "that can be applied on a Module. Throw a ValueError if the pipeline "
           "can't be parsed")
@@ -111,12 +111,35 @@ void mlir::python::populatePassManagerSubmodule(py::module &m) {
             if (mlirLogicalResultIsFailure(status))
               throw py::value_error(std::string(errorMsg.join()));
           },
-          py::arg("pipeline"),
+          "pipeline"_a,
           "Add textual pipeline elements to the pass manager. Throws a "
           "ValueError if the pipeline can't be parsed.")
       .def(
           "run",
-          [](PyPassManager &passManager, PyOperationBase &op) {
+          [](PyPassManager &passManager, PyOperationBase &op,
+             bool invalidateOps) {
+            if (invalidateOps) {
+              typedef struct {
+                PyOperation &rootOp;
+                bool rootSeen;
+              } callBackData;
+              callBackData data{op.getOperation(), false};
+              // Mark all ops below the op that the passmanager will be rooted
+              // at (but not op itself - note the preorder) as invalid.
+              MlirOperationWalkCallback invalidatingCallback =
+                  [](MlirOperation op, void *userData) {
+                    callBackData *data = static_cast<callBackData *>(userData);
+                    if (LLVM_LIKELY(data->rootSeen))
+                      data->rootOp.getOperation()
+                          .getContext()
+                          ->setOperationInvalid(op);
+                    else
+                      data->rootSeen = true;
+                  };
+              mlirOperationWalk(op.getOperation(), invalidatingCallback,
+                                static_cast<void *>(&data), MlirWalkPreOrder);
+            }
+            // Actually run the pass manager.
             PyMlirContext::ErrorCapture errors(op.getOperation().getContext());
             MlirLogicalResult status = mlirPassManagerRunOnOp(
                 passManager.get(), op.getOperation().get());
@@ -124,7 +147,7 @@ void mlir::python::populatePassManagerSubmodule(py::module &m) {
               throw MLIRError("Failure while executing pass pipeline",
                               errors.take());
           },
-          py::arg("operation"),
+          "operation"_a, "invalidate_ops"_a = true,
           "Run the pass manager on the provided operation, raising an "
           "MLIRError on failure.")
       .def(
diff --git a/mlir/lib/Bytecode/Writer/IRNumbering.cpp b/mlir/lib/Bytecode/Writer/IRNumbering.cpp
index 67f929059e..74c45723c2 100644
--- a/mlir/lib/Bytecode/Writer/IRNumbering.cpp
+++ b/mlir/lib/Bytecode/Writer/IRNumbering.cpp
@@ -557,7 +557,7 @@ struct NumberingResourceBuilder : public AsmResourceBuilder {
   void numberEntry(StringRef key) {
     // TODO: We could pre-number resource key strings here as well.
 
-    auto it = dialect->resourceMap.find(key);
+    auto *it = dialect->resourceMap.find(key);
     if (it != dialect->resourceMap.end()) {
       it->second->number = nextResourceID++;
       it->second->isDeclaration = false;
diff --git a/mlir/lib/CAPI/Dialect/SparseTensor.cpp b/mlir/lib/CAPI/Dialect/SparseTensor.cpp
index bf3a4ad5e7..c3ad95527d 100644
--- a/mlir/lib/CAPI/Dialect/SparseTensor.cpp
+++ b/mlir/lib/CAPI/Dialect/SparseTensor.cpp
@@ -48,15 +48,14 @@ bool mlirAttributeIsASparseTensorEncodingAttr(MlirAttribute attr) {
 MlirAttribute
 mlirSparseTensorEncodingAttrGet(MlirContext ctx, intptr_t lvlRank,
                                 MlirSparseTensorDimLevelType const *lvlTypes,
-                                MlirAffineMap dimToLvl, int posWidth,
-                                int crdWidth) {
+                                MlirAffineMap dimToLvl, MlirAffineMap lvlToDim,
+                                int posWidth, int crdWidth) {
   SmallVector<DimLevelType> cppLvlTypes;
   cppLvlTypes.reserve(lvlRank);
   for (intptr_t l = 0; l < lvlRank; ++l)
     cppLvlTypes.push_back(static_cast<DimLevelType>(lvlTypes[l]));
-  mlir::AffineMap lvlToDim; // TODO: provide in API
   return wrap(SparseTensorEncodingAttr::get(unwrap(ctx), cppLvlTypes,
-                                            unwrap(dimToLvl), lvlToDim,
+                                            unwrap(dimToLvl), unwrap(lvlToDim),
                                             posWidth, crdWidth));
 }
 
diff --git a/mlir/lib/CAPI/IR/IR.cpp b/mlir/lib/CAPI/IR/IR.cpp
index c1abbbe364..0a51517518 100644
--- a/mlir/lib/CAPI/IR/IR.cpp
+++ b/mlir/lib/CAPI/IR/IR.cpp
@@ -25,6 +25,7 @@
 #include "mlir/IR/Types.h"
 #include "mlir/IR/Value.h"
 #include "mlir/IR/Verifier.h"
+#include "mlir/IR/Visitors.h"
 #include "mlir/Interfaces/InferTypeOpInterface.h"
 #include "mlir/Parser/Parser.h"
 
@@ -705,6 +706,20 @@ void mlirOperationMoveBefore(MlirOperation op, MlirOperation other) {
   return unwrap(op)->moveBefore(unwrap(other));
 }
 
+void mlirOperationWalk(MlirOperation op, MlirOperationWalkCallback callback,
+                       void *userData, MlirWalkOrder walkOrder) {
+  switch (walkOrder) {
+
+  case MlirWalkPreOrder:
+    unwrap(op)->walk<mlir::WalkOrder::PreOrder>(
+        [callback, userData](Operation *op) { callback(wrap(op), userData); });
+    break;
+  case MlirWalkPostOrder:
+    unwrap(op)->walk<mlir::WalkOrder::PostOrder>(
+        [callback, userData](Operation *op) { callback(wrap(op), userData); });
+  }
+}
+
 //===----------------------------------------------------------------------===//
 // Region API.
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/CMakeLists.txt b/mlir/lib/CMakeLists.txt
index c71664a3f0..d25c84a397 100644
--- a/mlir/lib/CMakeLists.txt
+++ b/mlir/lib/CMakeLists.txt
@@ -11,6 +11,7 @@ add_subdirectory(IR)
 add_subdirectory(Interfaces)
 add_subdirectory(Parser)
 add_subdirectory(Pass)
+add_subdirectory(Query)
 add_subdirectory(Reducer)
 add_subdirectory(Rewrite)
 add_subdirectory(Support)
diff --git a/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp b/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp
index 881cc8575f..0ec51b7430 100644
--- a/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp
+++ b/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp
@@ -134,7 +134,7 @@ struct TileLoadOpConversion : public OpRewritePattern<arm_sme::TileLoadOp> {
 ///
 ///  BEFORE:
 ///  ```mlir
-///  arm_sme.tile_store %tile, %dest[%c0, %c0], <vertical>
+///  arm_sme.tile_store %tile, %dest[%c0, %c0] layout<vertical>
 ///    : memref<?x?xi32>, vector<[4]x[4]xi32
 ///  ```
 ///
@@ -147,7 +147,7 @@ struct TileLoadOpConversion : public OpRewritePattern<arm_sme::TileLoadOp> {
 ///  %svl_s = arith.muli %min_svl_s, %vscale : index
 ///  scf.for %tile_slice_idx = %c0 to %svl_s step %c1 {
 ///    arm_sme.store_tile_slice %tile, %tile_slice_idx, %dest[%tile_slice_idx],
-///      <vertical> : memref<?x?xi32>, vector<[4]x[4]xi32>
+///      layout<vertical> : memref<?x?xi32>, vector<[4]x[4]xi32>
 ///  }
 ///  ```
 struct TileStoreOpConversion : public OpRewritePattern<arm_sme::TileStoreOp> {
diff --git a/mlir/lib/Conversion/CMakeLists.txt b/mlir/lib/Conversion/CMakeLists.txt
index 35790254be..7e1c7bcf9a 100644
--- a/mlir/lib/Conversion/CMakeLists.txt
+++ b/mlir/lib/Conversion/CMakeLists.txt
@@ -24,6 +24,7 @@ add_subdirectory(GPUToROCDL)
 add_subdirectory(GPUToSPIRV)
 add_subdirectory(GPUToVulkan)
 add_subdirectory(IndexToLLVM)
+add_subdirectory(IndexToSPIRV)
 add_subdirectory(LinalgToStandard)
 add_subdirectory(LLVMCommon)
 add_subdirectory(MathToFuncs)
diff --git a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
index 4aacb47a7f..3506f50916 100644
--- a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
+++ b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
@@ -74,7 +74,7 @@ static bool shouldUseBarePtrCallConv(Operation *op,
 
 /// Only retain those attributes that are not constructed by
 /// `LLVMFuncOp::build`.
-static void filterFuncAttributes(func::FuncOp func,
+static void filterFuncAttributes(FunctionOpInterface func,
                                  SmallVectorImpl<NamedAttribute> &result) {
   for (const NamedAttribute &attr : func->getDiscardableAttrs()) {
     if (attr.getName() == linkageAttrName ||
@@ -87,26 +87,26 @@ static void filterFuncAttributes(func::FuncOp func,
 
 /// Propagate argument/results attributes.
 static void propagateArgResAttrs(OpBuilder &builder, bool resultStructType,
-                                 func::FuncOp funcOp,
+                                 FunctionOpInterface funcOp,
                                  LLVM::LLVMFuncOp wrapperFuncOp) {
-  auto argAttrs = funcOp.getArgAttrs();
+  auto argAttrs = funcOp.getAllArgAttrs();
   if (!resultStructType) {
     if (auto resAttrs = funcOp.getAllResultAttrs())
       wrapperFuncOp.setAllResultAttrs(resAttrs);
     if (argAttrs)
-      wrapperFuncOp.setAllArgAttrs(*argAttrs);
+      wrapperFuncOp.setAllArgAttrs(argAttrs);
   } else {
     SmallVector<Attribute> argAttributes;
     // Only modify the argument and result attributes when the result is now
     // an argument.
     if (argAttrs) {
       argAttributes.push_back(builder.getDictionaryAttr({}));
-      argAttributes.append(argAttrs->begin(), argAttrs->end());
+      argAttributes.append(argAttrs.begin(), argAttrs.end());
       wrapperFuncOp.setAllArgAttrs(argAttributes);
     }
   }
-  if (funcOp.getSymVisibilityAttr())
-    wrapperFuncOp.setSymVisibility(funcOp.getSymVisibilityAttr());
+  cast<FunctionOpInterface>(wrapperFuncOp.getOperation())
+      .setVisibility(funcOp.getVisibility());
 }
 
 /// Creates an auxiliary function with pointer-to-memref-descriptor-struct
@@ -119,9 +119,9 @@ static void propagateArgResAttrs(OpBuilder &builder, bool resultStructType,
 /// the extra arguments.
 static void wrapForExternalCallers(OpBuilder &rewriter, Location loc,
                                    const LLVMTypeConverter &typeConverter,
-                                   func::FuncOp funcOp,
+                                   FunctionOpInterface funcOp,
                                    LLVM::LLVMFuncOp newFuncOp) {
-  auto type = funcOp.getFunctionType();
+  auto type = cast<FunctionType>(funcOp.getFunctionType());
   auto [wrapperFuncType, resultStructType] =
       typeConverter.convertFunctionTypeCWrapper(type);
 
@@ -179,12 +179,13 @@ static void wrapForExternalCallers(OpBuilder &rewriter, Location loc,
 /// corresponding to a memref descriptor.
 static void wrapExternalFunction(OpBuilder &builder, Location loc,
                                  const LLVMTypeConverter &typeConverter,
-                                 func::FuncOp funcOp,
+                                 FunctionOpInterface funcOp,
                                  LLVM::LLVMFuncOp newFuncOp) {
   OpBuilder::InsertionGuard guard(builder);
 
   auto [wrapperType, resultStructType] =
-      typeConverter.convertFunctionTypeCWrapper(funcOp.getFunctionType());
+      typeConverter.convertFunctionTypeCWrapper(
+          cast<FunctionType>(funcOp.getFunctionType()));
   // This conversion can only fail if it could not convert one of the argument
   // types. But since it has been applied to a non-wrapper function before, it
   // should have failed earlier and not reach this point at all.
@@ -205,7 +206,7 @@ static void wrapExternalFunction(OpBuilder &builder, Location loc,
   builder.setInsertionPointToStart(newFuncOp.addEntryBlock());
 
   // Get a ValueRange containing arguments.
-  FunctionType type = funcOp.getFunctionType();
+  FunctionType type = cast<FunctionType>(funcOp.getFunctionType());
   SmallVector<Value, 8> args;
   args.reserve(type.getNumInputs());
   ValueRange wrapperArgsRange(newFuncOp.getArguments());
@@ -317,6 +318,140 @@ static void modifyFuncOpToUseBarePtrCallingConv(
   }
 }
 
+FailureOr<LLVM::LLVMFuncOp>
+mlir::convertFuncOpToLLVMFuncOp(FunctionOpInterface funcOp,
+                                ConversionPatternRewriter &rewriter,
+                                const LLVMTypeConverter &converter) {
+  // Check the funcOp has `FunctionType`.
+  auto funcTy = dyn_cast<FunctionType>(funcOp.getFunctionType());
+  if (!funcTy)
+    return rewriter.notifyMatchFailure(
+        funcOp, "Only support FunctionOpInterface with FunctionType");
+
+  // Convert the original function arguments. They are converted using the
+  // LLVMTypeConverter provided to this legalization pattern.
+  auto varargsAttr = funcOp->getAttrOfType<BoolAttr>(varargsAttrName);
+  TypeConverter::SignatureConversion result(funcOp.getNumArguments());
+  auto llvmType = converter.convertFunctionSignature(
+      funcTy, varargsAttr && varargsAttr.getValue(),
+      shouldUseBarePtrCallConv(funcOp, &converter), result);
+  if (!llvmType)
+    return rewriter.notifyMatchFailure(funcOp, "signature conversion failed");
+
+  // Create an LLVM function, use external linkage by default until MLIR
+  // functions have linkage.
+  LLVM::Linkage linkage = LLVM::Linkage::External;
+  if (funcOp->hasAttr(linkageAttrName)) {
+    auto attr =
+        dyn_cast<mlir::LLVM::LinkageAttr>(funcOp->getAttr(linkageAttrName));
+    if (!attr) {
+      funcOp->emitError() << "Contains " << linkageAttrName
+                          << " attribute not of type LLVM::LinkageAttr";
+      return rewriter.notifyMatchFailure(
+          funcOp, "Contains linkage attribute not of type LLVM::LinkageAttr");
+    }
+    linkage = attr.getLinkage();
+  }
+
+  SmallVector<NamedAttribute, 4> attributes;
+  filterFuncAttributes(funcOp, attributes);
+  auto newFuncOp = rewriter.create<LLVM::LLVMFuncOp>(
+      funcOp.getLoc(), funcOp.getName(), llvmType, linkage,
+      /*dsoLocal=*/false, /*cconv=*/LLVM::CConv::C, /*comdat=*/nullptr,
+      attributes);
+  cast<FunctionOpInterface>(newFuncOp.getOperation())
+      .setVisibility(funcOp.getVisibility());
+
+  // Create a memory effect attribute corresponding to readnone.
+  StringRef readnoneAttrName = LLVM::LLVMDialect::getReadnoneAttrName();
+  if (funcOp->hasAttr(readnoneAttrName)) {
+    auto attr = funcOp->getAttrOfType<UnitAttr>(readnoneAttrName);
+    if (!attr) {
+      funcOp->emitError() << "Contains " << readnoneAttrName
+                          << " attribute not of type UnitAttr";
+      return rewriter.notifyMatchFailure(
+          funcOp, "Contains readnone attribute not of type UnitAttr");
+    }
+    auto memoryAttr = LLVM::MemoryEffectsAttr::get(
+        rewriter.getContext(),
+        {LLVM::ModRefInfo::NoModRef, LLVM::ModRefInfo::NoModRef,
+         LLVM::ModRefInfo::NoModRef});
+    newFuncOp.setMemoryAttr(memoryAttr);
+  }
+
+  // Propagate argument/result attributes to all converted arguments/result
+  // obtained after converting a given original argument/result.
+  if (ArrayAttr resAttrDicts = funcOp.getAllResultAttrs()) {
+    assert(!resAttrDicts.empty() && "expected array to be non-empty");
+    if (funcOp.getNumResults() == 1)
+      newFuncOp.setAllResultAttrs(resAttrDicts);
+  }
+  if (ArrayAttr argAttrDicts = funcOp.getAllArgAttrs()) {
+    SmallVector<Attribute> newArgAttrs(
+        cast<LLVM::LLVMFunctionType>(llvmType).getNumParams());
+    for (unsigned i = 0, e = funcOp.getNumArguments(); i < e; ++i) {
+      // Some LLVM IR attribute have a type attached to them. During FuncOp ->
+      // LLVMFuncOp conversion these types may have changed. Account for that
+      // change by converting attributes' types as well.
+      SmallVector<NamedAttribute, 4> convertedAttrs;
+      auto attrsDict = cast<DictionaryAttr>(argAttrDicts[i]);
+      convertedAttrs.reserve(attrsDict.size());
+      for (const NamedAttribute &attr : attrsDict) {
+        const auto convert = [&](const NamedAttribute &attr) {
+          return TypeAttr::get(converter.convertType(
+              cast<TypeAttr>(attr.getValue()).getValue()));
+        };
+        if (attr.getName().getValue() ==
+            LLVM::LLVMDialect::getByValAttrName()) {
+          convertedAttrs.push_back(rewriter.getNamedAttr(
+              LLVM::LLVMDialect::getByValAttrName(), convert(attr)));
+        } else if (attr.getName().getValue() ==
+                   LLVM::LLVMDialect::getByRefAttrName()) {
+          convertedAttrs.push_back(rewriter.getNamedAttr(
+              LLVM::LLVMDialect::getByRefAttrName(), convert(attr)));
+        } else if (attr.getName().getValue() ==
+                   LLVM::LLVMDialect::getStructRetAttrName()) {
+          convertedAttrs.push_back(rewriter.getNamedAttr(
+              LLVM::LLVMDialect::getStructRetAttrName(), convert(attr)));
+        } else if (attr.getName().getValue() ==
+                   LLVM::LLVMDialect::getInAllocaAttrName()) {
+          convertedAttrs.push_back(rewriter.getNamedAttr(
+              LLVM::LLVMDialect::getInAllocaAttrName(), convert(attr)));
+        } else {
+          convertedAttrs.push_back(attr);
+        }
+      }
+      auto mapping = result.getInputMapping(i);
+      assert(mapping && "unexpected deletion of function argument");
+      // Only attach the new argument attributes if there is a one-to-one
+      // mapping from old to new types. Otherwise, attributes might be
+      // attached to types that they do not support.
+      if (mapping->size == 1) {
+        newArgAttrs[mapping->inputNo] =
+            DictionaryAttr::get(rewriter.getContext(), convertedAttrs);
+        continue;
+      }
+      // TODO: Implement custom handling for types that expand to multiple
+      // function arguments.
+      for (size_t j = 0; j < mapping->size; ++j)
+        newArgAttrs[mapping->inputNo + j] =
+            DictionaryAttr::get(rewriter.getContext(), {});
+    }
+    if (!newArgAttrs.empty())
+      newFuncOp.setAllArgAttrs(rewriter.getArrayAttr(newArgAttrs));
+  }
+
+  rewriter.inlineRegionBefore(funcOp.getFunctionBody(), newFuncOp.getBody(),
+                              newFuncOp.end());
+  if (failed(rewriter.convertRegionTypes(&newFuncOp.getBody(), converter,
+                                         &result))) {
+    return rewriter.notifyMatchFailure(funcOp,
+                                       "region types conversion failed");
+  }
+
+  return newFuncOp;
+}
+
 namespace {
 
 struct FuncOpConversionBase : public ConvertOpToLLVMPattern<func::FuncOp> {
@@ -328,128 +463,9 @@ struct FuncOpConversionBase : public ConvertOpToLLVMPattern<func::FuncOp> {
   FailureOr<LLVM::LLVMFuncOp>
   convertFuncOpToLLVMFuncOp(func::FuncOp funcOp,
                             ConversionPatternRewriter &rewriter) const {
-    // Convert the original function arguments. They are converted using the
-    // LLVMTypeConverter provided to this legalization pattern.
-    auto varargsAttr = funcOp->getAttrOfType<BoolAttr>(varargsAttrName);
-    TypeConverter::SignatureConversion result(funcOp.getNumArguments());
-    auto llvmType = getTypeConverter()->convertFunctionSignature(
-        funcOp.getFunctionType(), varargsAttr && varargsAttr.getValue(),
-        shouldUseBarePtrCallConv(funcOp, getTypeConverter()), result);
-    if (!llvmType)
-      return rewriter.notifyMatchFailure(funcOp, "signature conversion failed");
-
-    // Create an LLVM function, use external linkage by default until MLIR
-    // functions have linkage.
-    LLVM::Linkage linkage = LLVM::Linkage::External;
-    if (funcOp->hasAttr(linkageAttrName)) {
-      auto attr =
-          dyn_cast<mlir::LLVM::LinkageAttr>(funcOp->getAttr(linkageAttrName));
-      if (!attr) {
-        funcOp->emitError() << "Contains " << linkageAttrName
-                            << " attribute not of type LLVM::LinkageAttr";
-        return rewriter.notifyMatchFailure(
-            funcOp, "Contains linkage attribute not of type LLVM::LinkageAttr");
-      }
-      linkage = attr.getLinkage();
-    }
-
-    SmallVector<NamedAttribute, 4> attributes;
-    filterFuncAttributes(funcOp, attributes);
-    auto newFuncOp = rewriter.create<LLVM::LLVMFuncOp>(
-        funcOp.getLoc(), funcOp.getName(), llvmType, linkage,
-        /*dsoLocal=*/false, /*cconv=*/LLVM::CConv::C, /*comdat=*/nullptr,
-        attributes);
-    if (funcOp.getSymVisibilityAttr())
-      newFuncOp.setSymVisibility(funcOp.getSymVisibilityAttr());
-
-    // Create a memory effect attribute corresponding to readnone.
-    StringRef readnoneAttrName = LLVM::LLVMDialect::getReadnoneAttrName();
-    if (funcOp->hasAttr(readnoneAttrName)) {
-      auto attr = funcOp->getAttrOfType<UnitAttr>(readnoneAttrName);
-      if (!attr) {
-        funcOp->emitError() << "Contains " << readnoneAttrName
-                            << " attribute not of type UnitAttr";
-        return rewriter.notifyMatchFailure(
-            funcOp, "Contains readnone attribute not of type UnitAttr");
-      }
-      auto memoryAttr = LLVM::MemoryEffectsAttr::get(
-          rewriter.getContext(),
-          {LLVM::ModRefInfo::NoModRef, LLVM::ModRefInfo::NoModRef,
-           LLVM::ModRefInfo::NoModRef});
-      newFuncOp.setMemoryAttr(memoryAttr);
-    }
-
-    // Propagate argument/result attributes to all converted arguments/result
-    // obtained after converting a given original argument/result.
-    if (ArrayAttr resAttrDicts = funcOp.getAllResultAttrs()) {
-      assert(!resAttrDicts.empty() && "expected array to be non-empty");
-      if (funcOp.getNumResults() == 1)
-        newFuncOp.setAllResultAttrs(resAttrDicts);
-    }
-    if (ArrayAttr argAttrDicts = funcOp.getAllArgAttrs()) {
-      SmallVector<Attribute> newArgAttrs(
-          cast<LLVM::LLVMFunctionType>(llvmType).getNumParams());
-      for (unsigned i = 0, e = funcOp.getNumArguments(); i < e; ++i) {
-        // Some LLVM IR attribute have a type attached to them. During FuncOp ->
-        // LLVMFuncOp conversion these types may have changed. Account for that
-        // change by converting attributes' types as well.
-        SmallVector<NamedAttribute, 4> convertedAttrs;
-        auto attrsDict = cast<DictionaryAttr>(argAttrDicts[i]);
-        convertedAttrs.reserve(attrsDict.size());
-        for (const NamedAttribute &attr : attrsDict) {
-          const auto convert = [&](const NamedAttribute &attr) {
-            return TypeAttr::get(getTypeConverter()->convertType(
-                cast<TypeAttr>(attr.getValue()).getValue()));
-          };
-          if (attr.getName().getValue() ==
-              LLVM::LLVMDialect::getByValAttrName()) {
-            convertedAttrs.push_back(rewriter.getNamedAttr(
-                LLVM::LLVMDialect::getByValAttrName(), convert(attr)));
-          } else if (attr.getName().getValue() ==
-                     LLVM::LLVMDialect::getByRefAttrName()) {
-            convertedAttrs.push_back(rewriter.getNamedAttr(
-                LLVM::LLVMDialect::getByRefAttrName(), convert(attr)));
-          } else if (attr.getName().getValue() ==
-                     LLVM::LLVMDialect::getStructRetAttrName()) {
-            convertedAttrs.push_back(rewriter.getNamedAttr(
-                LLVM::LLVMDialect::getStructRetAttrName(), convert(attr)));
-          } else if (attr.getName().getValue() ==
-                     LLVM::LLVMDialect::getInAllocaAttrName()) {
-            convertedAttrs.push_back(rewriter.getNamedAttr(
-                LLVM::LLVMDialect::getInAllocaAttrName(), convert(attr)));
-          } else {
-            convertedAttrs.push_back(attr);
-          }
-        }
-        auto mapping = result.getInputMapping(i);
-        assert(mapping && "unexpected deletion of function argument");
-        // Only attach the new argument attributes if there is a one-to-one
-        // mapping from old to new types. Otherwise, attributes might be
-        // attached to types that they do not support.
-        if (mapping->size == 1) {
-          newArgAttrs[mapping->inputNo] =
-              DictionaryAttr::get(rewriter.getContext(), convertedAttrs);
-          continue;
-        }
-        // TODO: Implement custom handling for types that expand to multiple
-        // function arguments.
-        for (size_t j = 0; j < mapping->size; ++j)
-          newArgAttrs[mapping->inputNo + j] =
-              DictionaryAttr::get(rewriter.getContext(), {});
-      }
-      if (!newArgAttrs.empty())
-        newFuncOp.setAllArgAttrs(rewriter.getArrayAttr(newArgAttrs));
-    }
-
-    rewriter.inlineRegionBefore(funcOp.getBody(), newFuncOp.getBody(),
-                                newFuncOp.end());
-    if (failed(rewriter.convertRegionTypes(&newFuncOp.getBody(), *typeConverter,
-                                           &result))) {
-      return rewriter.notifyMatchFailure(funcOp,
-                                         "region types conversion failed");
-    }
-
-    return newFuncOp;
+    return mlir::convertFuncOpToLLVMFuncOp(
+        cast<FunctionOpInterface>(funcOp.getOperation()), rewriter,
+        *getTypeConverter());
   }
 };
 
diff --git a/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp b/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp
index 96d8fceba7..6d2585aa30 100644
--- a/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp
+++ b/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.cpp
@@ -441,7 +441,7 @@ LogicalResult GPUPrintfOpToVPrintfLowering::matchAndRewrite(
   Location loc = gpuPrintfOp->getLoc();
 
   mlir::Type llvmI8 = typeConverter->convertType(rewriter.getIntegerType(8));
-  mlir::Type i8Ptr = LLVM::LLVMPointerType::get(llvmI8);
+  mlir::Type ptrType = LLVM::LLVMPointerType::get(rewriter.getContext());
 
   // Note: this is the GPUModule op, not the ModuleOp that surrounds it
   // This ensures that global constants and declarations are placed within
@@ -449,7 +449,7 @@ LogicalResult GPUPrintfOpToVPrintfLowering::matchAndRewrite(
   auto moduleOp = gpuPrintfOp->getParentOfType<gpu::GPUModuleOp>();
 
   auto vprintfType =
-      LLVM::LLVMFunctionType::get(rewriter.getI32Type(), {i8Ptr, i8Ptr});
+      LLVM::LLVMFunctionType::get(rewriter.getI32Type(), {ptrType, ptrType});
   LLVM::LLVMFuncOp vprintfDecl =
       getOrDefineFunction(moduleOp, loc, rewriter, "vprintf", vprintfType);
 
@@ -473,7 +473,8 @@ LogicalResult GPUPrintfOpToVPrintfLowering::matchAndRewrite(
   // Get a pointer to the format string's first element
   Value globalPtr = rewriter.create<LLVM::AddressOfOp>(loc, global);
   Value stringStart = rewriter.create<LLVM::GEPOp>(
-      loc, i8Ptr, globalPtr, ArrayRef<LLVM::GEPArg>{0, 0});
+      loc, getTypeConverter()->getPointerType(globalType), globalType,
+      globalPtr, ArrayRef<LLVM::GEPArg>{0, 0});
   SmallVector<Type> types;
   SmallVector<Value> args;
   // Promote and pack the arguments into a stack allocation.
@@ -490,18 +491,17 @@ LogicalResult GPUPrintfOpToVPrintfLowering::matchAndRewrite(
   }
   Type structType =
       LLVM::LLVMStructType::getLiteral(gpuPrintfOp.getContext(), types);
-  Type structPtrType = LLVM::LLVMPointerType::get(structType);
   Value one = rewriter.create<LLVM::ConstantOp>(loc, rewriter.getI64Type(),
                                                 rewriter.getIndexAttr(1));
-  Value tempAlloc = rewriter.create<LLVM::AllocaOp>(loc, structPtrType, one,
-                                                    /*alignment=*/0);
+  Value tempAlloc =
+      rewriter.create<LLVM::AllocaOp>(loc, ptrType, structType, one,
+                                      /*alignment=*/0);
   for (auto [index, arg] : llvm::enumerate(args)) {
     Value ptr = rewriter.create<LLVM::GEPOp>(
-        loc, LLVM::LLVMPointerType::get(arg.getType()), tempAlloc,
-        ArrayRef<LLVM::GEPArg>{0, index});
+        loc, getTypeConverter()->getPointerType(structType), structType,
+        tempAlloc, ArrayRef<LLVM::GEPArg>{0, index});
     rewriter.create<LLVM::StoreOp>(loc, arg, ptr);
   }
-  tempAlloc = rewriter.create<LLVM::BitcastOp>(loc, i8Ptr, tempAlloc);
   std::array<Value, 2> printfArgs = {stringStart, tempAlloc};
 
   rewriter.create<LLVM::CallOp>(loc, vprintfDecl, printfArgs);
diff --git a/mlir/lib/Conversion/IndexToSPIRV/CMakeLists.txt b/mlir/lib/Conversion/IndexToSPIRV/CMakeLists.txt
new file mode 100644
index 0000000000..a02d2d40a9
--- /dev/null
+++ b/mlir/lib/Conversion/IndexToSPIRV/CMakeLists.txt
@@ -0,0 +1,17 @@
+add_mlir_conversion_library(MLIRIndexToSPIRV
+  IndexToSPIRV.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/IndexToSPIRV
+
+  DEPENDS
+  MLIRConversionPassIncGen
+
+  LINK_COMPONENTS
+  Core
+
+  LINK_LIBS PUBLIC
+  MLIRIndexDialect
+  MLIRSPIRVConversion
+  MLIRSPIRVDialect
+  )
diff --git a/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp b/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp
new file mode 100644
index 0000000000..b58efc096e
--- /dev/null
+++ b/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp
@@ -0,0 +1,418 @@
+//===- IndexToSPIRV.cpp - Index to SPIRV dialect conversion -----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Conversion/IndexToSPIRV/IndexToSPIRV.h"
+#include "../SPIRVCommon/Pattern.h"
+#include "mlir/Dialect/Index/IR/IndexDialect.h"
+#include "mlir/Dialect/Index/IR/IndexOps.h"
+#include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
+#include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
+#include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
+#include "mlir/Pass/Pass.h"
+
+using namespace mlir;
+using namespace index;
+
+namespace {
+
+//===----------------------------------------------------------------------===//
+// Trivial Conversions
+//===----------------------------------------------------------------------===//
+
+using ConvertIndexAdd = spirv::ElementwiseOpPattern<AddOp, spirv::IAddOp>;
+using ConvertIndexSub = spirv::ElementwiseOpPattern<SubOp, spirv::ISubOp>;
+using ConvertIndexMul = spirv::ElementwiseOpPattern<MulOp, spirv::IMulOp>;
+using ConvertIndexDivS = spirv::ElementwiseOpPattern<DivSOp, spirv::SDivOp>;
+using ConvertIndexDivU = spirv::ElementwiseOpPattern<DivUOp, spirv::UDivOp>;
+using ConvertIndexRemS = spirv::ElementwiseOpPattern<RemSOp, spirv::SRemOp>;
+using ConvertIndexRemU = spirv::ElementwiseOpPattern<RemUOp, spirv::UModOp>;
+using ConvertIndexMaxS = spirv::ElementwiseOpPattern<MaxSOp, spirv::GLSMaxOp>;
+using ConvertIndexMaxU = spirv::ElementwiseOpPattern<MaxUOp, spirv::GLUMaxOp>;
+using ConvertIndexMinS = spirv::ElementwiseOpPattern<MinSOp, spirv::GLSMinOp>;
+using ConvertIndexMinU = spirv::ElementwiseOpPattern<MinUOp, spirv::GLUMinOp>;
+
+using ConvertIndexShl =
+    spirv::ElementwiseOpPattern<ShlOp, spirv::ShiftLeftLogicalOp>;
+using ConvertIndexShrS =
+    spirv::ElementwiseOpPattern<ShrSOp, spirv::ShiftRightArithmeticOp>;
+using ConvertIndexShrU =
+    spirv::ElementwiseOpPattern<ShrUOp, spirv::ShiftRightLogicalOp>;
+
+/// It is the case that when we convert bitwise operations to SPIR-V operations
+/// we must take into account the special pattern in SPIR-V that if the
+/// operands are boolean values, then SPIR-V uses `SPIRVLogicalOp`. Otherwise,
+/// for non-boolean operands, SPIR-V should use `SPIRVBitwiseOp`. However,
+/// index.add is never a boolean operation so we can directly convert it to the
+/// Bitwise[And|Or]Op.
+using ConvertIndexAnd = spirv::ElementwiseOpPattern<AndOp, spirv::BitwiseAndOp>;
+using ConvertIndexOr = spirv::ElementwiseOpPattern<OrOp, spirv::BitwiseOrOp>;
+using ConvertIndexXor = spirv::ElementwiseOpPattern<XOrOp, spirv::BitwiseXorOp>;
+
+//===----------------------------------------------------------------------===//
+// ConvertConstantBool
+//===----------------------------------------------------------------------===//
+
+// Converts index.bool.constant operation to spirv.Constant.
+struct ConvertIndexConstantBoolOpPattern final
+    : OpConversionPattern<BoolConstantOp> {
+  using OpConversionPattern::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(BoolConstantOp op, BoolConstantOpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    rewriter.replaceOpWithNewOp<spirv::ConstantOp>(op, op.getType(),
+                                                   op.getValueAttr());
+    return success();
+  }
+};
+
+//===----------------------------------------------------------------------===//
+// ConvertConstant
+//===----------------------------------------------------------------------===//
+
+// Converts index.constant op to spirv.Constant. Will truncate from i64 to i32
+// when required.
+struct ConvertIndexConstantOpPattern final : OpConversionPattern<ConstantOp> {
+  using OpConversionPattern::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(ConstantOp op, ConstantOpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    auto *typeConverter = this->template getTypeConverter<SPIRVTypeConverter>();
+    Type indexType = typeConverter->getIndexType();
+
+    APInt value = op.getValue().trunc(typeConverter->getIndexTypeBitwidth());
+    rewriter.replaceOpWithNewOp<spirv::ConstantOp>(
+        op, indexType, IntegerAttr::get(indexType, value));
+    return success();
+  }
+};
+
+//===----------------------------------------------------------------------===//
+// ConvertIndexCeilDivS
+//===----------------------------------------------------------------------===//
+
+/// Convert `ceildivs(n, m)` into `x = m > 0 ? -1 : 1` and then
+/// `n*m > 0 ? (n+x)/m + 1 : -(-n/m)`. Formula taken from the equivalent
+/// conversion in IndexToLLVM.
+struct ConvertIndexCeilDivSPattern final : OpConversionPattern<CeilDivSOp> {
+  using OpConversionPattern::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(CeilDivSOp op, CeilDivSOpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    Location loc = op.getLoc();
+    Value n = adaptor.getLhs();
+    Type n_type = n.getType();
+    Value m = adaptor.getRhs();
+
+    // Define the constants
+    Value zero = rewriter.create<spirv::ConstantOp>(
+        loc, n_type, IntegerAttr::get(n_type, 0));
+    Value posOne = rewriter.create<spirv::ConstantOp>(
+        loc, n_type, IntegerAttr::get(n_type, 1));
+    Value negOne = rewriter.create<spirv::ConstantOp>(
+        loc, n_type, IntegerAttr::get(n_type, -1));
+
+    // Compute `x`.
+    Value mPos = rewriter.create<spirv::SGreaterThanOp>(loc, m, zero);
+    Value x = rewriter.create<spirv::SelectOp>(loc, mPos, negOne, posOne);
+
+    // Compute the positive result.
+    Value nPlusX = rewriter.create<spirv::IAddOp>(loc, n, x);
+    Value nPlusXDivM = rewriter.create<spirv::SDivOp>(loc, nPlusX, m);
+    Value posRes = rewriter.create<spirv::IAddOp>(loc, nPlusXDivM, posOne);
+
+    // Compute the negative result.
+    Value negN = rewriter.create<spirv::ISubOp>(loc, zero, n);
+    Value negNDivM = rewriter.create<spirv::SDivOp>(loc, negN, m);
+    Value negRes = rewriter.create<spirv::ISubOp>(loc, zero, negNDivM);
+
+    // Pick the positive result if `n` and `m` have the same sign and `n` is
+    // non-zero, i.e. `(n > 0) == (m > 0) && n != 0`.
+    Value nPos = rewriter.create<spirv::SGreaterThanOp>(loc, n, zero);
+    Value sameSign = rewriter.create<spirv::LogicalEqualOp>(loc, nPos, mPos);
+    Value nNonZero = rewriter.create<spirv::INotEqualOp>(loc, n, zero);
+    Value cmp = rewriter.create<spirv::LogicalAndOp>(loc, sameSign, nNonZero);
+    rewriter.replaceOpWithNewOp<spirv::SelectOp>(op, cmp, posRes, negRes);
+    return success();
+  }
+};
+
+//===----------------------------------------------------------------------===//
+// ConvertIndexCeilDivU
+//===----------------------------------------------------------------------===//
+
+/// Convert `ceildivu(n, m)` into `n == 0 ? 0 : (n-1)/m + 1`. Formula taken
+/// from the equivalent conversion in IndexToLLVM.
+struct ConvertIndexCeilDivUPattern final : OpConversionPattern<CeilDivUOp> {
+  using OpConversionPattern::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(CeilDivUOp op, CeilDivUOpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    Location loc = op.getLoc();
+    Value n = adaptor.getLhs();
+    Type n_type = n.getType();
+    Value m = adaptor.getRhs();
+
+    // Define the constants
+    Value zero = rewriter.create<spirv::ConstantOp>(
+        loc, n_type, IntegerAttr::get(n_type, 0));
+    Value one = rewriter.create<spirv::ConstantOp>(loc, n_type,
+                                                   IntegerAttr::get(n_type, 1));
+
+    // Compute the non-zero result.
+    Value minusOne = rewriter.create<spirv::ISubOp>(loc, n, one);
+    Value quotient = rewriter.create<spirv::UDivOp>(loc, minusOne, m);
+    Value plusOne = rewriter.create<spirv::IAddOp>(loc, quotient, one);
+
+    // Pick the result
+    Value cmp = rewriter.create<spirv::IEqualOp>(loc, n, zero);
+    rewriter.replaceOpWithNewOp<spirv::SelectOp>(op, cmp, zero, plusOne);
+    return success();
+  }
+};
+
+//===----------------------------------------------------------------------===//
+// ConvertIndexFloorDivS
+//===----------------------------------------------------------------------===//
+
+/// Convert `floordivs(n, m)` into `x = m < 0 ? 1 : -1` and then
+/// `n*m < 0 ? -1 - (x-n)/m : n/m`. Formula taken from the equivalent conversion
+/// in IndexToLLVM.
+struct ConvertIndexFloorDivSPattern final : OpConversionPattern<FloorDivSOp> {
+  using OpConversionPattern::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(FloorDivSOp op, FloorDivSOpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    Location loc = op.getLoc();
+    Value n = adaptor.getLhs();
+    Type n_type = n.getType();
+    Value m = adaptor.getRhs();
+
+    // Define the constants
+    Value zero = rewriter.create<spirv::ConstantOp>(
+        loc, n_type, IntegerAttr::get(n_type, 0));
+    Value posOne = rewriter.create<spirv::ConstantOp>(
+        loc, n_type, IntegerAttr::get(n_type, 1));
+    Value negOne = rewriter.create<spirv::ConstantOp>(
+        loc, n_type, IntegerAttr::get(n_type, -1));
+
+    // Compute `x`.
+    Value mNeg = rewriter.create<spirv::SLessThanOp>(loc, m, zero);
+    Value x = rewriter.create<spirv::SelectOp>(loc, mNeg, posOne, negOne);
+
+    // Compute the negative result
+    Value xMinusN = rewriter.create<spirv::ISubOp>(loc, x, n);
+    Value xMinusNDivM = rewriter.create<spirv::SDivOp>(loc, xMinusN, m);
+    Value negRes = rewriter.create<spirv::ISubOp>(loc, negOne, xMinusNDivM);
+
+    // Compute the positive result.
+    Value posRes = rewriter.create<spirv::SDivOp>(loc, n, m);
+
+    // Pick the negative result if `n` and `m` have different signs and `n` is
+    // non-zero, i.e. `(n < 0) != (m < 0) && n != 0`.
+    Value nNeg = rewriter.create<spirv::SLessThanOp>(loc, n, zero);
+    Value diffSign = rewriter.create<spirv::LogicalNotEqualOp>(loc, nNeg, mNeg);
+    Value nNonZero = rewriter.create<spirv::INotEqualOp>(loc, n, zero);
+
+    Value cmp = rewriter.create<spirv::LogicalAndOp>(loc, diffSign, nNonZero);
+    rewriter.replaceOpWithNewOp<spirv::SelectOp>(op, cmp, posRes, negRes);
+    return success();
+  }
+};
+
+//===----------------------------------------------------------------------===//
+// ConvertIndexCast
+//===----------------------------------------------------------------------===//
+
+/// Convert a cast op. If the materialized index type is the same as the other
+/// type, fold away the op. Otherwise, use the Convert SPIR-V operation.
+/// Signed casts sign extend when the result bitwidth is larger. Unsigned casts
+/// zero extend when the result bitwidth is larger.
+template <typename CastOp, typename ConvertOp>
+struct ConvertIndexCast final : OpConversionPattern<CastOp> {
+  using OpConversionPattern<CastOp>::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(CastOp op, typename CastOp::Adaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    auto *typeConverter = this->template getTypeConverter<SPIRVTypeConverter>();
+    Type indexType = typeConverter->getIndexType();
+
+    Type srcType = adaptor.getInput().getType();
+    Type dstType = op.getType();
+    if (isa<IndexType>(srcType)) {
+      srcType = indexType;
+    }
+    if (isa<IndexType>(dstType)) {
+      dstType = indexType;
+    }
+
+    if (srcType == dstType) {
+      rewriter.replaceOp(op, adaptor.getInput());
+    } else {
+      rewriter.template replaceOpWithNewOp<ConvertOp>(op, dstType,
+                                                      adaptor.getOperands());
+    }
+    return success();
+  }
+};
+
+using ConvertIndexCastS = ConvertIndexCast<CastSOp, spirv::SConvertOp>;
+using ConvertIndexCastU = ConvertIndexCast<CastUOp, spirv::UConvertOp>;
+
+//===----------------------------------------------------------------------===//
+// ConvertIndexCmp
+//===----------------------------------------------------------------------===//
+
+// Helper template to replace the operation
+template <typename ICmpOp>
+static LogicalResult rewriteCmpOp(CmpOp op, CmpOpAdaptor adaptor,
+                                  ConversionPatternRewriter &rewriter) {
+  rewriter.replaceOpWithNewOp<ICmpOp>(op, adaptor.getLhs(), adaptor.getRhs());
+  return success();
+}
+
+struct ConvertIndexCmpPattern final : OpConversionPattern<CmpOp> {
+  using OpConversionPattern::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(CmpOp op, CmpOpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    // We must convert the predicates to the corresponding int comparions.
+    switch (op.getPred()) {
+    case IndexCmpPredicate::EQ:
+      return rewriteCmpOp<spirv::IEqualOp>(op, adaptor, rewriter);
+    case IndexCmpPredicate::NE:
+      return rewriteCmpOp<spirv::INotEqualOp>(op, adaptor, rewriter);
+    case IndexCmpPredicate::SGE:
+      return rewriteCmpOp<spirv::SGreaterThanEqualOp>(op, adaptor, rewriter);
+    case IndexCmpPredicate::SGT:
+      return rewriteCmpOp<spirv::SGreaterThanOp>(op, adaptor, rewriter);
+    case IndexCmpPredicate::SLE:
+      return rewriteCmpOp<spirv::SLessThanEqualOp>(op, adaptor, rewriter);
+    case IndexCmpPredicate::SLT:
+      return rewriteCmpOp<spirv::SLessThanOp>(op, adaptor, rewriter);
+    case IndexCmpPredicate::UGE:
+      return rewriteCmpOp<spirv::UGreaterThanEqualOp>(op, adaptor, rewriter);
+    case IndexCmpPredicate::UGT:
+      return rewriteCmpOp<spirv::UGreaterThanOp>(op, adaptor, rewriter);
+    case IndexCmpPredicate::ULE:
+      return rewriteCmpOp<spirv::ULessThanEqualOp>(op, adaptor, rewriter);
+    case IndexCmpPredicate::ULT:
+      return rewriteCmpOp<spirv::ULessThanOp>(op, adaptor, rewriter);
+    }
+  }
+};
+
+//===----------------------------------------------------------------------===//
+// ConvertIndexSizeOf
+//===----------------------------------------------------------------------===//
+
+/// Lower `index.sizeof` to a constant with the value of the index bitwidth.
+struct ConvertIndexSizeOf final : OpConversionPattern<SizeOfOp> {
+  using OpConversionPattern::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(SizeOfOp op, SizeOfOpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    auto *typeConverter = this->template getTypeConverter<SPIRVTypeConverter>();
+    Type indexType = typeConverter->getIndexType();
+    unsigned bitwidth = typeConverter->getIndexTypeBitwidth();
+    rewriter.replaceOpWithNewOp<spirv::ConstantOp>(
+        op, indexType, IntegerAttr::get(indexType, bitwidth));
+    return success();
+  }
+};
+} // namespace
+
+//===----------------------------------------------------------------------===//
+// Pattern Population
+//===----------------------------------------------------------------------===//
+
+void index::populateIndexToSPIRVPatterns(SPIRVTypeConverter &typeConverter,
+                                         RewritePatternSet &patterns) {
+  patterns.add<
+      // clang-format off
+    ConvertIndexAdd,
+    ConvertIndexSub,
+    ConvertIndexMul,
+    ConvertIndexDivS,
+    ConvertIndexDivU,
+    ConvertIndexRemS,
+    ConvertIndexRemU,
+    ConvertIndexMaxS,
+    ConvertIndexMaxU,
+    ConvertIndexMinS,
+    ConvertIndexMinU,
+    ConvertIndexShl,
+    ConvertIndexShrS,
+    ConvertIndexShrU,
+    ConvertIndexAnd,
+    ConvertIndexOr,
+    ConvertIndexXor,
+    ConvertIndexConstantBoolOpPattern,
+    ConvertIndexConstantOpPattern,
+    ConvertIndexCeilDivSPattern,
+    ConvertIndexCeilDivUPattern,
+    ConvertIndexFloorDivSPattern,
+    ConvertIndexCastS,
+    ConvertIndexCastU,
+    ConvertIndexCmpPattern,
+    ConvertIndexSizeOf
+  >(typeConverter, patterns.getContext());
+}
+
+//===----------------------------------------------------------------------===//
+// ODS-Generated Definitions
+//===----------------------------------------------------------------------===//
+
+namespace mlir {
+#define GEN_PASS_DEF_CONVERTINDEXTOSPIRVPASS
+#include "mlir/Conversion/Passes.h.inc"
+} // namespace mlir
+
+//===----------------------------------------------------------------------===//
+// Pass Definition
+//===----------------------------------------------------------------------===//
+
+namespace {
+struct ConvertIndexToSPIRVPass
+    : public impl::ConvertIndexToSPIRVPassBase<ConvertIndexToSPIRVPass> {
+  using Base::Base;
+
+  void runOnOperation() override {
+    Operation *op = getOperation();
+    spirv::TargetEnvAttr targetAttr = spirv::lookupTargetEnvOrDefault(op);
+    std::unique_ptr<SPIRVConversionTarget> target =
+      SPIRVConversionTarget::get(targetAttr);
+
+    SPIRVConversionOptions options;
+    options.use64bitIndex = this->use64bitIndex;
+    SPIRVTypeConverter typeConverter(targetAttr, options);
+
+    // Use UnrealizedConversionCast as the bridge so that we don't need to pull
+    // in patterns for other dialects.
+    target->addLegalOp<UnrealizedConversionCastOp>();
+
+    // Allow the spirv operations we are converting to
+    target->addLegalDialect<spirv::SPIRVDialect>();
+    // Fail hard when there are any remaining 'index' ops.
+    target->addIllegalDialect<index::IndexDialect>();
+
+    RewritePatternSet patterns(&getContext());
+    index::populateIndexToSPIRVPatterns(typeConverter, patterns);
+
+    if (failed(applyPartialConversion(op, *target, std::move(patterns))))
+      signalPassFailure();
+  }
+};
+} // namespace
diff --git a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
index 99c4d42233..efcde2ba58 100644
--- a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
+++ b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
@@ -28,6 +28,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
+#include <optional>
 
 #define DEBUG_TYPE "nvgpu-to-nvvm"
 #define DBGS() (llvm::dbgs() << '[' << DEBUG_TYPE << "] ")
@@ -412,10 +413,28 @@ struct ConvertNVGPUToNVVMPass
       return converter.convertType(IntegerType::get(type.getContext(), 32));
     });
     converter.addConversion([&](nvgpu::WarpgroupAccumulatorType type) -> Type {
-      VectorType vtype = type.getFragmented();
+      Type elemType = type.getFragmented().getElementType();
+      int64_t sizeM = type.getFragmented().getDimSize(0);
+      int64_t sizeN = type.getFragmented().getDimSize(1);
+
+      unsigned numMembers;
+      if (elemType.isF32() || elemType.isInteger(32))
+        numMembers = sizeN / 2;
+      else if (elemType.isF16())
+        numMembers = sizeN / 4;
+      else
+        llvm_unreachable("unsupported type for warpgroup accumulator");
+
+      SmallVector<Type> innerStructBody;
+      for (unsigned i = 0; i < numMembers; i++)
+        innerStructBody.push_back(elemType);
+      auto innerStructType =
+          LLVM::LLVMStructType::getLiteral(type.getContext(), innerStructBody);
+
       SmallVector<Type> structBody;
-      for (unsigned i = 0; i < vtype.getDimSize(0); i++)
-        structBody.push_back(vtype.getElementType());
+      for (int i = 0; i < sizeM; i += kWgmmaSizeM)
+        structBody.push_back(innerStructType);
+
       auto convertedType =
           LLVM::LLVMStructType::getLiteral(type.getContext(), structBody);
       return converter.convertType(convertedType);
@@ -811,10 +830,11 @@ struct NVGPUMBarrierInitLowering
                                    adaptor.getMbarId(), rewriter);
     Value count = truncToI32(b, adaptor.getCount());
     if (isMbarrierShared(mbarrierType)) {
-      rewriter.replaceOpWithNewOp<NVVM::MBarrierInitSharedOp>(op, barrier,
-                                                              count);
+      rewriter.replaceOpWithNewOp<NVVM::MBarrierInitSharedOp>(
+          op, barrier, count, adaptor.getPredicate());
     } else {
-      rewriter.replaceOpWithNewOp<NVVM::MBarrierInitOp>(op, barrier, count);
+      rewriter.replaceOpWithNewOp<NVVM::MBarrierInitOp>(op, barrier, count,
+                                                        adaptor.getPredicate());
     }
     return success();
   }
@@ -909,12 +929,12 @@ struct NVGPUMBarrierArriveExpectTxLowering
 
     if (isMbarrierShared(op.getBarriers().getType())) {
       rewriter.replaceOpWithNewOp<NVVM::MBarrierArriveExpectTxSharedOp>(
-          op, barrier, txcount);
+          op, barrier, txcount, adaptor.getPredicate());
       return success();
     }
 
-    rewriter.replaceOpWithNewOp<NVVM::MBarrierArriveExpectTxOp>(op, barrier,
-                                                                txcount);
+    rewriter.replaceOpWithNewOp<NVVM::MBarrierArriveExpectTxOp>(
+        op, barrier, txcount, adaptor.getPredicate());
     return success();
   }
 };
@@ -965,7 +985,8 @@ struct NVGPUTmaAsyncLoadOpLowering
     }
 
     rewriter.replaceOpWithNewOp<NVVM::CpAsyncBulkTensorGlobalToSharedClusterOp>(
-        op, dest, adaptor.getTensorMapDescriptor(), barrier, coords);
+        op, dest, adaptor.getTensorMapDescriptor(), barrier, coords,
+        adaptor.getPredicate());
     return success();
   }
 };
@@ -1186,7 +1207,6 @@ struct NVGPUWarpgroupMmaOpLowering
     nvgpu::WarpgroupMmaOp op;
     ImplicitLocOpBuilder b;
     OpAdaptor adaptor;
-    const LLVMTypeConverter &typeConverter;
 
     // Entire shape of the given Op
     int64_t totalM, totalN, totalK;
@@ -1330,7 +1350,7 @@ struct NVGPUWarpgroupMmaOpLowering
 
     /// This function generates a WgmmaMmaAsyncOp using provided GMMA matrix
     /// descriptors and arranges them based on induction variables: i, j, and k.
-    Value generateWgmma(int i, int j, int k, Value matrixC, Value matrixD) {
+    Value generateWgmma(int i, int j, int k, Value matrixC) {
       LLVM_DEBUG(DBGS() << "\t wgmma."
                         << "m" << wgmmaM << "n" << wgmmaN << "k" << wgmmaK
                         << "(A[" << (iterationM * wgmmaM) << ":"
@@ -1359,34 +1379,36 @@ struct NVGPUWarpgroupMmaOpLowering
       auto overflow = NVVM::MMAIntOverflowAttr::get(
           op->getContext(), NVVM::MMAIntOverflow::wrapped);
 
-      Type resultStructType = typeConverter.convertType(matrixD.getType());
-
       return b.create<NVVM::WgmmaMmaAsyncOp>(
-          resultStructType, matrixC, descriptorA, descriptorB, shape, itypeA,
+          matrixC.getType(), matrixC, descriptorA, descriptorB, shape, itypeA,
           itypeB, scaleOut, scaleIn, scaleIn, layoutA, layoutB, overflow);
     }
 
     /// Generates multiple wgmma instructions to complete the given GEMM shape
-    SmallVector<Value> generateWgmmaGroup() {
-      SmallVector<Value> wgmmaResults;
+    Value generateWgmmaGroup() {
+      Value wgmmaResult =
+          b.create<LLVM::UndefOp>(adaptor.getMatrixC().getType());
 
       // Perform GEMM
+      SmallVector<Value> wgmmaResults;
       for (int i = 0; i < iterationM; ++i) {
-        Value matrixC = adaptor.getMatrixC()[i];
-        Value matrixD = op.getMatrixD()[i];
+        Value matrixC = b.create<LLVM::ExtractValueOp>(adaptor.getMatrixC(), i);
         for (int j = 0; j < iterationN; ++j)
           for (int k = 0; k < iterationK; ++k)
-            matrixC = generateWgmma(i, j, k, matrixC, matrixD);
+            matrixC = generateWgmma(i, j, k, matrixC);
         wgmmaResults.push_back(matrixC);
       }
-
-      return wgmmaResults;
+      for (auto [idx, matrix] : llvm::enumerate(wgmmaResults)) {
+        wgmmaResult = b.create<LLVM::InsertValueOp>(wgmmaResult.getType(),
+                                                    wgmmaResult, matrix, idx);
+      }
+      return wgmmaResult;
     }
 
   public:
     WarpgroupGemm(nvgpu::WarpgroupMmaOp op, ImplicitLocOpBuilder &b,
-                  OpAdaptor adaptor, const LLVMTypeConverter &typeConverter)
-        : op(op), b(b), adaptor(adaptor), typeConverter(typeConverter) {
+                  OpAdaptor adaptor)
+        : op(op), b(b), adaptor(adaptor) {
       // Find the entire GEMM Shape
       totalM = op.getDescriptorA().getType().getTensor().getDimSize(0);
       totalN = op.getDescriptorB().getType().getTensor().getDimSize(1);
@@ -1411,27 +1433,27 @@ struct NVGPUWarpgroupMmaOpLowering
     /// instructions and group synchronization, as well as waiting
     /// (WgmmaGroupSyncAlignedOp) for group synchronization
     /// (WgmmaWaitGroupSyncOp) after the instructions.
-    SmallVector<Value> generateWarpgroupMma() {
+    Value generateWarpgroupMma() {
       b.create<NVVM::WgmmaFenceAlignedOp>();
-      SmallVector<Value> wgmmaResults = generateWgmmaGroup();
+      Value wgmmaResult = generateWgmmaGroup();
       b.create<NVVM::WgmmaGroupSyncAlignedOp>();
       b.create<NVVM::WgmmaWaitGroupSyncOp>(op.getWaitGroup());
-      return wgmmaResults;
+      return wgmmaResult;
     }
   };
-
   LogicalResult
   matchAndRewrite(nvgpu::WarpgroupMmaOp op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     ImplicitLocOpBuilder b(op->getLoc(), rewriter);
+
     // Step 1. Build a helper class
-    WarpgroupGemm warpgroupGemm(op, b, adaptor, *this->getTypeConverter());
+    WarpgroupGemm warpgroupGemm(op, b, adaptor);
 
     // Step 2. Get the entire GEMM Shape
-    SmallVector<Value> wgmmaResults = warpgroupGemm.generateWarpgroupMma();
+    Value wgmmaResult = warpgroupGemm.generateWarpgroupMma();
 
     // Step 3. Replace fragmented result struct with the op results
-    rewriter.replaceOp(op, wgmmaResults);
+    rewriter.replaceOp(op, wgmmaResult);
     return success();
   }
 };
@@ -1535,10 +1557,13 @@ struct NVGPUWarpgroupMmaStoreOpLowering
   matchAndRewrite(nvgpu::WarpgroupMmaStoreOp op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     int offset = 0;
-    ImplicitLocOpBuilder lb(op->getLoc(), rewriter);
-    for (Value matrixD : adaptor.getMatrixD()) {
-      auto structType = matrixD.getType().cast<LLVM::LLVMStructType>();
-      storeFragmentedMatrix(lb, matrixD, op.getDstMemref(), offset);
+    ImplicitLocOpBuilder b(op->getLoc(), rewriter);
+    Value matriDValue = adaptor.getMatrixD();
+    auto stype = matriDValue.getType().cast<LLVM::LLVMStructType>();
+    for (auto [idx, matrixD] : llvm::enumerate(stype.getBody())) {
+      auto structType = matrixD.cast<LLVM::LLVMStructType>();
+      Value innerStructValue = b.create<LLVM::ExtractValueOp>(matriDValue, idx);
+      storeFragmentedMatrix(b, innerStructValue, op.getDstMemref(), offset);
       offset += structType.getBody().size();
     }
     rewriter.eraseOp(op);
@@ -1546,6 +1571,58 @@ struct NVGPUWarpgroupMmaStoreOpLowering
   }
 };
 
+struct NVGPUWarpgroupMmaInitAccumulatorOpLowering
+    : public ConvertOpToLLVMPattern<nvgpu::WarpgroupMmaInitAccumulatorOp> {
+  using ConvertOpToLLVMPattern<
+      nvgpu::WarpgroupMmaInitAccumulatorOp>::ConvertOpToLLVMPattern;
+  LogicalResult
+  matchAndRewrite(nvgpu::WarpgroupMmaInitAccumulatorOp op, OpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    ImplicitLocOpBuilder b(op->getLoc(), rewriter);
+    LLVM::LLVMStructType packStructType =
+        getTypeConverter()
+            ->convertType(op.getMatrixC().getType())
+            .cast<LLVM::LLVMStructType>();
+    Type elemType = packStructType.getBody()
+                        .front()
+                        .cast<LLVM::LLVMStructType>()
+                        .getBody()
+                        .front();
+    Value zero = b.create<LLVM::ConstantOp>(elemType, b.getZeroAttr(elemType));
+    Value packStruct = b.create<LLVM::UndefOp>(packStructType);
+    SmallVector<Value> innerStructs;
+    // Unpack the structs and set all values to zero
+    for (auto [idx, s] : llvm::enumerate(packStructType.getBody())) {
+      auto structType = s.cast<LLVM::LLVMStructType>();
+      Value structValue = b.create<LLVM::ExtractValueOp>(packStruct, idx);
+      for (unsigned i = 0; i < structType.getBody().size(); ++i) {
+        structValue = b.create<LLVM::InsertValueOp>(
+            structType, structValue, zero, ArrayRef<int64_t>({i}));
+      }
+      innerStructs.push_back(structValue);
+    }
+    // Pack the inner structs into a single struct
+    for (auto [idx, matrix] : llvm::enumerate(innerStructs)) {
+      packStruct = b.create<LLVM::InsertValueOp>(packStruct.getType(),
+                                                 packStruct, matrix, idx);
+    }
+    rewriter.replaceOp(op, packStruct);
+    return success();
+  }
+};
+
+struct NVGPUTmaPrefetchOpLowering
+    : public ConvertOpToLLVMPattern<nvgpu::TmaPrefetchOp> {
+  using ConvertOpToLLVMPattern<nvgpu::TmaPrefetchOp>::ConvertOpToLLVMPattern;
+  LogicalResult
+  matchAndRewrite(nvgpu::TmaPrefetchOp op, OpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    rewriter.replaceOpWithNewOp<NVVM::PrefetchTensorMapOp>(
+        op, adaptor.getTensorMapDescriptor(), adaptor.getPredicate());
+    return success();
+  }
+};
+
 } // namespace
 
 void mlir::populateNVGPUToNVVMConversionPatterns(LLVMTypeConverter &converter,
@@ -1559,10 +1636,12 @@ void mlir::populateNVGPUToNVVMConversionPatterns(LLVMTypeConverter &converter,
       NVGPUMBarrierTryWaitParityLowering,    // nvgpu.mbarrier.try_wait_parity
       NVGPUTmaAsyncLoadOpLowering,           // nvgpu.tma.async.load
       NVGPUTmaCreateDescriptorOpLowering,    // nvgpu.tma.create.descriptor
+      NVGPUTmaPrefetchOpLowering,            // nvgpu.tma.prefetch.descriptor
       NVGPUMBarrierArriveExpectTxLowering,   // nvgpu.mbarrier.arrive.expect_tx
       NVGPUGenerateWarpgroupDescriptorLowering, // nvgpu.warpgroup.generate.descriptor
       NVGPUWarpgroupMmaOpLowering,              // nvgpu.warpgroup.mma
       NVGPUWarpgroupMmaStoreOpLowering,         // nvgpu.warpgroup.mma.store
+      NVGPUWarpgroupMmaInitAccumulatorOpLowering, // nvgpu.warpgroup.mma.init.accumulator
       MmaSyncOptoNVVM, MmaLdMatrixOpToNVVM, NVGPUAsyncCopyLowering,
       NVGPUAsyncCreateGroupLowering, NVGPUAsyncWaitLowering,
       NVGPUMmaSparseSyncLowering>(converter);
diff --git a/mlir/lib/Conversion/NVVMToLLVM/NVVMToLLVM.cpp b/mlir/lib/Conversion/NVVMToLLVM/NVVMToLLVM.cpp
index 15703fb993..d1d68e3c9c 100644
--- a/mlir/lib/Conversion/NVVMToLLVM/NVVMToLLVM.cpp
+++ b/mlir/lib/Conversion/NVVMToLLVM/NVVMToLLVM.cpp
@@ -14,16 +14,11 @@
 #include "mlir/Conversion/NVVMToLLVM/NVVMToLLVM.h"
 
 #include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h"
-#include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
 #include "mlir/Conversion/LLVMCommon/Pattern.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
-#include "mlir/Dialect/GPU/IR/GPUDialect.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/Dialect/LLVMIR/NVVMDialect.h"
-#include "mlir/Dialect/MemRef/IR/MemRef.h"
-#include "mlir/IR/Attributes.h"
-#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
@@ -31,11 +26,7 @@
 #include "mlir/Pass/Pass.h"
 #include "mlir/Support/LLVM.h"
 #include "mlir/Support/LogicalResult.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/StringRef.h"
 #include "llvm/Support/raw_ostream.h"
-#include <regex>
-#include <string>
 
 #define DEBUG_TYPE "nvvm-to-llvm"
 #define DBGS() (llvm::dbgs() << "[" DEBUG_TYPE "]: ")
@@ -49,151 +40,17 @@ namespace mlir {
 using namespace mlir;
 using namespace NVVM;
 
-#include "mlir/Dialect/LLVMIR/NVVMOpsInterface.cpp.inc"
 namespace {
 
-class PtxBuilder {
-  NVVM::BasicPtxBuilderInterface op;
-  PatternRewriter &rewriter;
-  std::string asmStr;
-  SmallVector<Value> asmVals;
-  std::string asmConstraints;
-  bool sideEffects;
-  bool hasResult = false;
-
-  // https://docs.nvidia.com/cuda/inline-ptx-assembly/index.html#constraints
-  char getRegisterType(Type type) {
-    if (type.isInteger(1))
-      return 'b';
-    if (type.isInteger(16))
-      return 'h';
-    if (type.isInteger(32))
-      return 'r';
-    if (type.isInteger(64))
-      return 'l';
-    if (type.isF32())
-      return 'f';
-    if (type.isF64())
-      return 'd';
-    if (auto ptr = type.dyn_cast<LLVM::LLVMPointerType>()) {
-      // Shared address spaces is addressed with 32-bit pointers.
-      if (ptr.getAddressSpace() == NVVM::kSharedMemorySpace) {
-        return 'r';
-      }
-      return 'l';
-    }
-    op->emitError() << "Register type could not deduced from MLIR type: "
-                    << type;
-    return ' ';
-  }
-
-  char getRegisterType(Value v) {
-    if (v.getDefiningOp<LLVM::ConstantOp>())
-      return 'n';
-    return getRegisterType(v.getType());
-  }
-
-public:
-  PtxBuilder(Operation *op, PatternRewriter &rewriter, std::string ptxAsm,
-             bool sideEffects = false)
-      : op(op), rewriter(rewriter), asmStr(std::move(ptxAsm)),
-        sideEffects(sideEffects) {}
-
-  void insertValue(Value v, PTXRegisterMod itype = PTXRegisterMod::Read) {
-    LLVM_DEBUG(DBGS() << v << "\t Modifier : " << itype << "\n");
-    auto getModifier = [&]() -> const char * {
-      if (itype == PTXRegisterMod::ReadWrite) {
-        assert(false && "Read-Write modifier is not supported. Try setting the "
-                        "same value as Write and Read seperately.");
-        return "+";
-      }
-      if (itype == PTXRegisterMod::Write) {
-        return "=";
-      }
-      return "";
-    };
-    auto addValue = [&](Value v) {
-      if (itype == PTXRegisterMod::Read) {
-        asmVals.push_back(v);
-        return;
-      }
-      if (itype == PTXRegisterMod::ReadWrite)
-        asmVals.push_back(v);
-      hasResult = true;
-    };
-
-    llvm::raw_string_ostream ss(asmConstraints);
-    // Handle Structs
-    if (auto stype = dyn_cast<LLVM::LLVMStructType>(v.getType())) {
-      if (itype == PTXRegisterMod::Write) {
-        addValue(v);
-      }
-      for (auto [idx, t] : llvm::enumerate(stype.getBody())) {
-        if (itype != PTXRegisterMod::Write) {
-          Value extractValue =
-              rewriter.create<LLVM::ExtractValueOp>(op->getLoc(), v, idx);
-          addValue(extractValue);
-        }
-        if (itype == PTXRegisterMod::ReadWrite) {
-          ss << idx << ",";
-        } else {
-          ss << getModifier() << getRegisterType(t) << ",";
-        }
-        ss.flush();
-      }
-      return;
-    }
-    // Handle Scalars
-    addValue(v);
-    ss << getModifier() << getRegisterType(v) << ",";
-    ss.flush();
-  }
-
-  LLVM::InlineAsmOp build() {
-    auto asmDialectAttr =
-        LLVM::AsmDialectAttr::get(op->getContext(), LLVM::AsmDialect::AD_ATT);
-
-    auto resultTypes = op->getResultTypes();
-
-    // Remove the last comma from the constraints string.
-    if (!asmConstraints.empty() &&
-        asmConstraints[asmConstraints.size() - 1] == ',')
-      asmConstraints.pop_back();
-
-    // asm keywords expects %, but inline assembly uses $. Replace all % with $
-    std::replace(asmStr.begin(), asmStr.end(), '%', '$');
-
-    return rewriter.create<LLVM::InlineAsmOp>(
-        op->getLoc(),
-        /*result types=*/resultTypes,
-        /*operands=*/asmVals,
-        /*asm_string=*/llvm::StringRef(asmStr),
-        /*constraints=*/asmConstraints.data(),
-        /*has_side_effects=*/sideEffects,
-        /*is_align_stack=*/false,
-        /*asm_dialect=*/asmDialectAttr,
-        /*operand_attrs=*/ArrayAttr());
-  }
-
-  void buildAndReplaceOp() {
-    LLVM::InlineAsmOp inlineAsmOp = build();
-    LLVM_DEBUG(DBGS() << "\n Generated PTX \n\t" << inlineAsmOp << "\n");
-    if (inlineAsmOp->getNumResults() == op->getNumResults())
-      rewriter.replaceOp(op, inlineAsmOp);
-    else
-      rewriter.eraseOp(op);
-  }
-};
-
 struct PtxLowering
-    : public OpInterfaceRewritePattern<NVVM::BasicPtxBuilderInterface> {
+    : public OpInterfaceRewritePattern<BasicPtxBuilderInterface> {
   using OpInterfaceRewritePattern<
-      NVVM::BasicPtxBuilderInterface>::OpInterfaceRewritePattern;
+      BasicPtxBuilderInterface>::OpInterfaceRewritePattern;
 
   PtxLowering(MLIRContext *context, PatternBenefit benefit = 2)
       : OpInterfaceRewritePattern(context, benefit) {}
 
-  LogicalResult matchAndRewrite(NVVM::BasicPtxBuilderInterface op,
+  LogicalResult matchAndRewrite(BasicPtxBuilderInterface op,
                                 PatternRewriter &rewriter) const override {
     if (op.hasIntrinsic()) {
       LLVM_DEBUG(DBGS() << "Ptx Builder does not lower \n\t" << op << "\n");
@@ -202,11 +59,11 @@ struct PtxLowering
 
     SmallVector<std::pair<Value, PTXRegisterMod>> asmValues;
     LLVM_DEBUG(DBGS() << op.getPtx() << "\n");
-    PtxBuilder generator(op, rewriter, op.getPtx(), op.hasSideEffect());
+    PtxBuilder generator(op, rewriter);
 
     op.getAsmValues(rewriter, asmValues);
     for (auto &[asmValue, modifier] : asmValues) {
-      LLVM_DEBUG(DBGSNL() << asmValue << "\t Modifier : " << modifier);
+      LLVM_DEBUG(DBGSNL() << asmValue << "\t Modifier : " << &modifier);
       generator.insertValue(asmValue, modifier);
     }
 
@@ -234,7 +91,7 @@ struct ConvertNVVMToLLVMPass
   }
 };
 
-/// Implement the interface to convert NNVM to LLVM.
+/// Implement the interface to convert NVVM to LLVM.
 struct NVVMToLLVMDialectInterface : public ConvertToLLVMPatternInterface {
   using ConvertToLLVMPatternInterface::ConvertToLLVMPatternInterface;
   void loadDependentDialects(MLIRContext *context) const final {
diff --git a/mlir/lib/Conversion/SCFToSPIRV/CMakeLists.txt b/mlir/lib/Conversion/SCFToSPIRV/CMakeLists.txt
index 987a1e84b7..e509fb025e 100644
--- a/mlir/lib/Conversion/SCFToSPIRV/CMakeLists.txt
+++ b/mlir/lib/Conversion/SCFToSPIRV/CMakeLists.txt
@@ -11,6 +11,7 @@ add_mlir_conversion_library(MLIRSCFToSPIRV
   LINK_LIBS PUBLIC
   MLIRArithToSPIRV
   MLIRFuncToSPIRV
+  MLIRIndexToSPIRV
   MLIRMemRefToSPIRV
   MLIRSPIRVDialect
   MLIRSPIRVConversion
diff --git a/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRVPass.cpp b/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRVPass.cpp
index 1e8fe4423a..3ef1d84ee2 100644
--- a/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRVPass.cpp
+++ b/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRVPass.cpp
@@ -14,6 +14,7 @@
 
 #include "mlir/Conversion/ArithToSPIRV/ArithToSPIRV.h"
 #include "mlir/Conversion/FuncToSPIRV/FuncToSPIRV.h"
+#include "mlir/Conversion/IndexToSPIRV/IndexToSPIRV.h"
 #include "mlir/Conversion/MemRefToSPIRV/MemRefToSPIRV.h"
 #include "mlir/Conversion/SCFToSPIRV/SCFToSPIRV.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
@@ -52,6 +53,7 @@ void SCFToSPIRVPass::runOnOperation() {
   populateFuncToSPIRVPatterns(typeConverter, patterns);
   populateMemRefToSPIRVPatterns(typeConverter, patterns);
   populateBuiltinFuncToSPIRVPatterns(typeConverter, patterns);
+  index::populateIndexToSPIRVPatterns(typeConverter, patterns);
 
   if (failed(applyPartialConversion(op, *target, std::move(patterns))))
     return signalPassFailure();
diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
index 4214bb5756..ee8f52dead 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
@@ -691,7 +691,7 @@ class MaxPool2dConverter : public OpRewritePattern<tosa::MaxPool2dOp> {
 
     // Determine what the initial value needs to be for the max pool op.
     TypedAttr initialAttr;
-    if (resultETy.isF32())
+    if (resultETy.isF32() || resultETy.isBF16() || resultETy.isF16())
       initialAttr = rewriter.getFloatAttr(
           resultETy, APFloat::getLargest(
                          cast<FloatType>(resultETy).getFloatSemantics(), true));
diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgPass.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgPass.cpp
index d7e867d922..2072fabc29 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgPass.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgPass.cpp
@@ -75,10 +75,10 @@ std::unique_ptr<Pass> mlir::tosa::createTosaToLinalg() {
 }
 
 void mlir::tosa::addTosaToLinalgPasses(
-    OpPassManager &pm, bool disableTosaDecompositions,
-    tosa::ValidationOptions const &validationOptions) {
+    OpPassManager &pm, const TosaToLinalgOptions &options,
+    tosa::TosaValidationOptions const &validationOptions) {
   // Optional decompositions are designed to benefit linalg.
-  if (!disableTosaDecompositions)
+  if (!options.disableTosaDecompositions)
     pm.addNestedPass<func::FuncOp>(tosa::createTosaOptionalDecompositions());
   pm.addNestedPass<func::FuncOp>(createCanonicalizerPass());
 
@@ -87,9 +87,10 @@ void mlir::tosa::addTosaToLinalgPasses(
   pm.addNestedPass<func::FuncOp>(tosa::createTosaToLinalgNamed());
   pm.addNestedPass<func::FuncOp>(createCanonicalizerPass());
   // TODO: Remove pass that operates on const tensor and enable optionality
-  pm.addNestedPass<func::FuncOp>(tosa::createTosaLayerwiseConstantFoldPass());
+  pm.addNestedPass<func::FuncOp>(tosa::createTosaLayerwiseConstantFoldPass(
+      {options.aggressiveReduceConstant}));
   pm.addNestedPass<func::FuncOp>(tosa::createTosaMakeBroadcastablePass());
-  pm.addNestedPass<func::FuncOp>(
-      tosa::createTosaValidationPass(validationOptions));
+  pm.addNestedPass<mlir::ModuleOp>(
+      tosa::createTosaValidation(validationOptions));
   pm.addNestedPass<func::FuncOp>(tosa::createTosaToLinalg());
 }
diff --git a/mlir/lib/Conversion/VectorToArmSME/VectorToArmSME.cpp b/mlir/lib/Conversion/VectorToArmSME/VectorToArmSME.cpp
index cbc5e468c7..d06eb4f5b0 100644
--- a/mlir/lib/Conversion/VectorToArmSME/VectorToArmSME.cpp
+++ b/mlir/lib/Conversion/VectorToArmSME/VectorToArmSME.cpp
@@ -67,7 +67,7 @@ namespace {
 ///
 /// is converted to:
 ///
-///   arm_sme.tile_load ... <vertical>
+///   arm_sme.tile_load ... layout<vertical>
 struct TransferReadPermutationToArmSMELowering
     : public OpRewritePattern<vector::TransferReadOp> {
   using OpRewritePattern<vector::TransferReadOp>::OpRewritePattern;
@@ -368,8 +368,8 @@ struct SplatOpToArmSMELowering : public OpRewritePattern<vector::SplatOp> {
 ///   %alloca = memref.alloca(%svl_s, %svl_s) : memref<?x?xi32>
 ///   %arm_sme.tile_store %src, <hor>, %alloca[%c0, %c0]
 ///     : memref<?x?xi32>, vector<[4]x[4]xi32>
-///   %transposed_src = arm_sme.tile_load %alloca[%c0, %c0], <vertical>
-///     : memref<?x?xi32>, vector<[4]x[4]xi32>
+///   %transposed_src = arm_sme.tile_load %alloca[%c0, %c0]
+///     layout<vertical> : memref<?x?xi32>, vector<[4]x[4]xi32>
 ///
 /// NOTE: Tranposing via memory is obviously expensive, the current intention
 /// is to avoid the transpose if possible, this is therefore intended as a
diff --git a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
index 113f4cfc31..f2b3171c1a 100644
--- a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
+++ b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
@@ -2215,6 +2215,10 @@ unsigned AffineForOp::getNumIterOperands() {
   return getNumOperands() - lbMap.getNumInputs() - ubMap.getNumInputs();
 }
 
+ValueRange AffineForOp::getYieldedValues() {
+  return cast<AffineYieldOp>(getBody()->getTerminator()).getOperands();
+}
+
 void AffineForOp::print(OpAsmPrinter &p) {
   p << ' ';
   p.printRegionArgument(getBody()->getArgument(0), /*argAttrs=*/{},
diff --git a/mlir/lib/Dialect/Affine/IR/ValueBoundsOpInterfaceImpl.cpp b/mlir/lib/Dialect/Affine/IR/ValueBoundsOpInterfaceImpl.cpp
index 97dd70e4f1..e0c3abe7a0 100644
--- a/mlir/lib/Dialect/Affine/IR/ValueBoundsOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Affine/IR/ValueBoundsOpInterfaceImpl.cpp
@@ -27,12 +27,22 @@ struct AffineApplyOpInterface
     assert(applyOp.getAffineMap().getNumResults() == 1 &&
            "expected single result");
 
+    // Fully compose this affine.apply with other ops because the folding logic
+    // can see opportunities for simplifying the affine map that
+    // `FlatLinearConstraints` can currently not see.
+    AffineMap map = applyOp.getAffineMap();
+    SmallVector<Value> operands = llvm::to_vector(applyOp.getOperands());
+    fullyComposeAffineMapAndOperands(&map, &operands);
+
     // Align affine map result with dims/symbols in the constraint set.
-    AffineExpr expr = applyOp.getAffineMap().getResult(0);
-    SmallVector<AffineExpr> dimReplacements = llvm::to_vector(llvm::map_range(
-        applyOp.getDimOperands(), [&](Value v) { return cstr.getExpr(v); }));
-    SmallVector<AffineExpr> symReplacements = llvm::to_vector(llvm::map_range(
-        applyOp.getSymbolOperands(), [&](Value v) { return cstr.getExpr(v); }));
+    AffineExpr expr = map.getResult(0);
+    SmallVector<AffineExpr> dimReplacements, symReplacements;
+    for (int64_t i = 0, e = map.getNumDims(); i < e; ++i)
+      dimReplacements.push_back(cstr.getExpr(operands[i]));
+    for (int64_t i = map.getNumDims(),
+                 e = map.getNumDims() + map.getNumSymbols();
+         i < e; ++i)
+      symReplacements.push_back(cstr.getExpr(operands[i]));
     AffineExpr bound =
         expr.replaceDimsAndSymbols(dimReplacements, symReplacements);
     cstr.bound(value) == bound;
@@ -92,3 +102,27 @@ void mlir::affine::registerValueBoundsOpInterfaceExternalModels(
     AffineMinOp::attachInterface<AffineMinOpInterface>(*ctx);
   });
 }
+
+FailureOr<int64_t>
+mlir::affine::fullyComposeAndComputeConstantDelta(Value value1, Value value2) {
+  assert(value1.getType().isIndex() && "expected index type");
+  assert(value2.getType().isIndex() && "expected index type");
+
+  // Subtract the two values/dimensions from each other. If the result is 0,
+  // both are equal.
+  Builder b(value1.getContext());
+  AffineMap map = AffineMap::get(/*dimCount=*/2, /*symbolCount=*/0,
+                                 b.getAffineDimExpr(0) - b.getAffineDimExpr(1));
+  // Fully compose the affine map with other ops because the folding logic
+  // can see opportunities for simplifying the affine map that
+  // `FlatLinearConstraints` can currently not see.
+  SmallVector<Value> mapOperands;
+  mapOperands.push_back(value1);
+  mapOperands.push_back(value2);
+  affine::fullyComposeAffineMapAndOperands(&map, &mapOperands);
+  ValueDimList valueDims;
+  for (Value v : mapOperands)
+    valueDims.push_back({v, std::nullopt});
+  return ValueBoundsConstraintSet::computeConstantBound(
+      presburger::BoundType::EQ, map, valueDims);
+}
diff --git a/mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td b/mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td
index 9d38513215..ef951647cc 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td
+++ b/mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td
@@ -237,6 +237,9 @@ def CmpIExtUI :
 // SelectOp
 //===----------------------------------------------------------------------===//
 
+def GetScalarOrVectorTrueAttribute :
+  NativeCodeCall<"cast<TypedAttr>(getBoolAttribute($0.getType(), true))">;
+
 // select(not(pred), a, b) => select(pred, b, a)
 def SelectNotCond :
     Pat<(SelectOp (Arith_XOrIOp $pred, (ConstantLikeMatcher APIntAttr:$ones)), $a, $b),
@@ -262,7 +265,8 @@ def SelectAndCond :
 def SelectAndNotCond :
     Pat<(SelectOp $predA, (SelectOp $predB, $y, $x), $y),
         (SelectOp (Arith_AndIOp $predA,
-                                (Arith_XOrIOp $predB, (Arith_ConstantOp ConstantAttr<I1Attr, "1">))),
+                                (Arith_XOrIOp $predB,
+                                (Arith_ConstantOp (GetScalarOrVectorTrueAttribute $predB)))),
                   $x, $y)>;
 
 // select(predA, x, select(predB, x, y)) => select(or(predA, predB), x, y)
@@ -274,7 +278,8 @@ def SelectOrCond :
 def SelectOrNotCond :
     Pat<(SelectOp $predA, $x, (SelectOp $predB, $y, $x)),
         (SelectOp (Arith_OrIOp $predA,
-                               (Arith_XOrIOp $predB, (Arith_ConstantOp ConstantAttr<I1Attr, "1">))),
+                               (Arith_XOrIOp $predB,
+                               (Arith_ConstantOp (GetScalarOrVectorTrueAttribute $predB)))),
                   $x, $y)>;
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
index 0ecc288f3b..1002719f0b 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
+++ b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
@@ -39,26 +39,26 @@ using namespace mlir::arith;
 static IntegerAttr
 applyToIntegerAttrs(PatternRewriter &builder, Value res, Attribute lhs,
                     Attribute rhs,
-                    function_ref<int64_t(int64_t, int64_t)> binFn) {
-  return builder.getIntegerAttr(res.getType(),
-                                binFn(llvm::cast<IntegerAttr>(lhs).getInt(),
-                                      llvm::cast<IntegerAttr>(rhs).getInt()));
+                    function_ref<APInt(const APInt &, const APInt &)> binFn) {
+  APInt lhsVal = llvm::cast<IntegerAttr>(lhs).getValue();
+  APInt rhsVal = llvm::cast<IntegerAttr>(rhs).getValue();
+  APInt value = binFn(lhsVal, rhsVal);
+  return IntegerAttr::get(res.getType(), value);
 }
 
 static IntegerAttr addIntegerAttrs(PatternRewriter &builder, Value res,
                                    Attribute lhs, Attribute rhs) {
-  return applyToIntegerAttrs(builder, res, lhs, rhs, std::plus<int64_t>());
+  return applyToIntegerAttrs(builder, res, lhs, rhs, std::plus<APInt>());
 }
 
 static IntegerAttr subIntegerAttrs(PatternRewriter &builder, Value res,
                                    Attribute lhs, Attribute rhs) {
-  return applyToIntegerAttrs(builder, res, lhs, rhs, std::minus<int64_t>());
+  return applyToIntegerAttrs(builder, res, lhs, rhs, std::minus<APInt>());
 }
 
 static IntegerAttr mulIntegerAttrs(PatternRewriter &builder, Value res,
                                    Attribute lhs, Attribute rhs) {
-  return applyToIntegerAttrs(builder, res, lhs, rhs,
-                             std::multiplies<int64_t>());
+  return applyToIntegerAttrs(builder, res, lhs, rhs, std::multiplies<APInt>());
 }
 
 /// Invert an integer comparison predicate.
@@ -113,6 +113,14 @@ static FailureOr<APInt> getIntOrSplatIntValue(Attribute attr) {
   return failure();
 }
 
+static Attribute getBoolAttribute(Type type, bool value) {
+  auto boolAttr = BoolAttr::get(type.getContext(), value);
+  ShapedType shapedType = llvm::dyn_cast_or_null<ShapedType>(type);
+  if (!shapedType)
+    return boolAttr;
+  return DenseElementsAttr::get(shapedType, boolAttr);
+}
+
 //===----------------------------------------------------------------------===//
 // TableGen'd canonicalization patterns
 //===----------------------------------------------------------------------===//
@@ -1696,14 +1704,6 @@ static bool applyCmpPredicateToEqualOperands(arith::CmpIPredicate predicate) {
   llvm_unreachable("unknown cmpi predicate kind");
 }
 
-static Attribute getBoolAttribute(Type type, MLIRContext *ctx, bool value) {
-  auto boolAttr = BoolAttr::get(ctx, value);
-  ShapedType shapedType = llvm::dyn_cast_or_null<ShapedType>(type);
-  if (!shapedType)
-    return boolAttr;
-  return DenseElementsAttr::get(shapedType, boolAttr);
-}
-
 static std::optional<int64_t> getIntegerWidth(Type t) {
   if (auto intType = llvm::dyn_cast<IntegerType>(t)) {
     return intType.getWidth();
@@ -1718,7 +1718,7 @@ OpFoldResult arith::CmpIOp::fold(FoldAdaptor adaptor) {
   // cmpi(pred, x, x)
   if (getLhs() == getRhs()) {
     auto val = applyCmpPredicateToEqualOperands(getPredicate());
-    return getBoolAttribute(getType(), getContext(), val);
+    return getBoolAttribute(getType(), val);
   }
 
   if (matchPattern(adaptor.getRhs(), m_Zero())) {
diff --git a/mlir/lib/Dialect/ArmSVE/IR/ArmSVEDialect.cpp b/mlir/lib/Dialect/ArmSVE/IR/ArmSVEDialect.cpp
index b7f1020deb..594c9b4c27 100644
--- a/mlir/lib/Dialect/ArmSVE/IR/ArmSVEDialect.cpp
+++ b/mlir/lib/Dialect/ArmSVE/IR/ArmSVEDialect.cpp
@@ -12,6 +12,7 @@
 
 #include "mlir/Dialect/ArmSVE/IR/ArmSVEDialect.h"
 #include "mlir/Dialect/LLVMIR/LLVMTypes.h"
+#include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/OpImplementation.h"
diff --git a/mlir/lib/Dialect/ArmSVE/IR/CMakeLists.txt b/mlir/lib/Dialect/ArmSVE/IR/CMakeLists.txt
index fffc77245d..9ef7384fc5 100644
--- a/mlir/lib/Dialect/ArmSVE/IR/CMakeLists.txt
+++ b/mlir/lib/Dialect/ArmSVE/IR/CMakeLists.txt
@@ -10,5 +10,6 @@ add_mlir_dialect_library(MLIRArmSVEDialect
   LINK_LIBS PUBLIC
   MLIRIR
   MLIRLLVMDialect
+  MLIRVectorDialect
   MLIRSideEffectInterfaces
   )
diff --git a/mlir/lib/Dialect/ArmSVE/Transforms/CMakeLists.txt b/mlir/lib/Dialect/ArmSVE/Transforms/CMakeLists.txt
index 7031ab4f79..2f1c43fae2 100644
--- a/mlir/lib/Dialect/ArmSVE/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/ArmSVE/Transforms/CMakeLists.txt
@@ -7,6 +7,7 @@ add_mlir_dialect_library(MLIRArmSVETransforms
   LINK_LIBS PUBLIC
   MLIRArmSVEDialect
   MLIRFuncDialect
+  MLIRVectorDialect
   MLIRIR
   MLIRLLVMCommonConversion
   MLIRLLVMDialect
diff --git a/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp b/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp
index abbb978304..92278c0d74 100644
--- a/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp
+++ b/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp
@@ -12,6 +12,8 @@
 #include "mlir/Dialect/ArmSVE/Transforms/Transforms.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/Dialect/Utils/IndexingUtils.h"
+#include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/PatternMatch.h"
 
@@ -66,6 +68,77 @@ using ScalableMaskedDivFOpLowering =
     OneToOneConvertToLLVMPattern<ScalableMaskedDivFOp,
                                  ScalableMaskedDivFIntrOp>;
 
+namespace {
+
+/// Unrolls a conversion to/from equivalent vector types, to allow using a
+/// conversion intrinsic that only supports 1-D vector types.
+///
+/// Example:
+/// ```
+/// %result = arm_sve.convert_to_svbool %source : vector<2x[4]xi1>
+/// ```
+/// is rewritten into:
+/// ```
+/// %cst = arith.constant dense<false> : vector<2x[16]xi1>
+/// %1 = vector.extract %source[0] : vector<[4]xi1> from vector<2x[4]xi1>
+/// %2 = "arm_sve.intr.convert.to.svbool"(%1)
+///                : (vector<[4]xi1>) -> vector<[16]xi1>
+/// %3 = vector.insert %2, %cst[0] : vector<[16]xi1> into vector<2x[16]xi1>
+/// %4 = vector.extract %source[1] : vector<[4]xi1> from vector<2x[4]xi1>
+/// %5 = "arm_sve.intr.convert.to.svbool"(%4)
+///                : (vector<[4]xi1>) -> vector<[16]xi1>
+/// %result = vector.insert %5, %3[1] : vector<[16]xi1> into vector<2x[16]xi1>
+/// ```
+template <typename Op, typename IntrOp>
+struct SvboolConversionOpLowering : public ConvertOpToLLVMPattern<Op> {
+  using ConvertOpToLLVMPattern<Op>::ConvertOpToLLVMPattern;
+
+  LogicalResult
+  matchAndRewrite(Op convertOp, typename Op::Adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    auto loc = convertOp.getLoc();
+
+    auto source = convertOp.getSource();
+    VectorType sourceType = source.getType();
+    VectorType resultType = convertOp.getResult().getType();
+
+    Value result = rewriter.create<arith::ConstantOp>(
+        loc, resultType, rewriter.getZeroAttr(resultType));
+
+    // We want to iterate over the input vector in steps of the trailing
+    // dimension. So this creates tile shape where all leading dimensions are 1,
+    // and the trailing dimension step is the size of the dimension.
+    SmallVector<int64_t> tileShape(sourceType.getRank(), 1);
+    tileShape.back() = sourceType.getShape().back();
+
+    // Iterate over all scalable mask/predicate slices of the source vector.
+    for (SmallVector<int64_t> index :
+         StaticTileOffsetRange(sourceType.getShape(), tileShape)) {
+      auto extractOrInsertPosition = ArrayRef(index).drop_back();
+      auto sourceVector = rewriter.create<vector::ExtractOp>(
+          loc, source, extractOrInsertPosition);
+      VectorType convertedType =
+          VectorType::Builder(llvm::cast<VectorType>(sourceVector.getType()))
+              .setDim(0, resultType.getShape().back());
+      auto convertedVector =
+          rewriter.create<IntrOp>(loc, TypeRange{convertedType}, sourceVector);
+      result = rewriter.create<vector::InsertOp>(loc, convertedVector, result,
+                                                 extractOrInsertPosition);
+    }
+
+    rewriter.replaceOp(convertOp, result);
+    return success();
+  }
+};
+
+using ConvertToSvboolOpLowering =
+    SvboolConversionOpLowering<ConvertToSvboolOp, ConvertToSvboolIntrOp>;
+
+using ConvertFromSvboolOpLowering =
+    SvboolConversionOpLowering<ConvertFromSvboolOp, ConvertFromSvboolIntrOp>;
+
+} // namespace
+
 /// Populate the given list with patterns that convert from ArmSVE to LLVM.
 void mlir::populateArmSVELegalizeForLLVMExportPatterns(
     LLVMTypeConverter &converter, RewritePatternSet &patterns) {
@@ -88,7 +161,9 @@ void mlir::populateArmSVELegalizeForLLVMExportPatterns(
                ScalableMaskedMulFOpLowering,
                ScalableMaskedSDivIOpLowering,
                ScalableMaskedUDivIOpLowering,
-               ScalableMaskedDivFOpLowering>(converter);
+               ScalableMaskedDivFOpLowering,
+               ConvertToSvboolOpLowering,
+               ConvertFromSvboolOpLowering>(converter);
   // clang-format on
 }
 
@@ -107,7 +182,9 @@ void mlir::configureArmSVELegalizeForExportTarget(
                     ScalableMaskedMulFIntrOp,
                     ScalableMaskedSDivIIntrOp,
                     ScalableMaskedUDivIIntrOp,
-                    ScalableMaskedDivFIntrOp>();
+                    ScalableMaskedDivFIntrOp,
+                    ConvertToSvboolIntrOp,
+                    ConvertFromSvboolIntrOp>();
   target.addIllegalOp<SdotOp,
                       SmmlaOp,
                       UdotOp,
@@ -120,6 +197,8 @@ void mlir::configureArmSVELegalizeForExportTarget(
                       ScalableMaskedMulFOp,
                       ScalableMaskedSDivIOp,
                       ScalableMaskedUDivIOp,
-                      ScalableMaskedDivFOp>();
+                      ScalableMaskedDivFOp,
+                      ConvertToSvboolOp,
+                      ConvertFromSvboolOp>();
   // clang-format on
 }
diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp
index 8d21446f1e..36b3c12502 100644
--- a/mlir/lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferDeallocationOpInterface.cpp
@@ -120,8 +120,7 @@ void DeallocationState::addMemrefToDeallocate(Value memref, Block *block) {
 }
 
 void DeallocationState::dropMemrefToDeallocate(Value memref, Block *block) {
-  llvm::erase_if(memrefsToDeallocatePerBlock[block],
-                 [&](const auto &mr) { return mr == memref; });
+  llvm::erase_value(memrefsToDeallocatePerBlock[block], memref);
 }
 
 void DeallocationState::getLiveMemrefsIn(Block *block,
diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferizationDialect.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferizationDialect.cpp
index 802bd52269..e5a0c3c45b 100644
--- a/mlir/lib/Dialect/Bufferization/IR/BufferizationDialect.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferizationDialect.cpp
@@ -28,6 +28,16 @@ constexpr const ::llvm::StringLiteral BufferizationDialect::kWritableAttrName;
 constexpr const ::llvm::StringLiteral
     BufferizationDialect::kBufferLayoutAttrName;
 
+/// An attribute that can be attached to ops with an allocation and/or
+/// deallocation side effect. It indicates that the op is under a "manual
+/// deallocation" scheme. In the case of an allocation op, the returned
+/// value is *not* an automatically managed allocation and assigned an
+/// ownership of "false". Furthermore, only deallocation ops that are
+/// guaranteed to deallocate a buffer under "manual deallocation" are
+/// allowed to have this attribute. (Deallocation ops without this
+/// attribute are rejected by the ownership-based buffer deallocation pass.)
+constexpr const ::llvm::StringLiteral BufferizationDialect::kManualDeallocation;
+
 //===----------------------------------------------------------------------===//
 // Bufferization Dialect Interfaces
 //===----------------------------------------------------------------------===//
@@ -105,6 +115,16 @@ BufferizationDialect::verifyOperationAttribute(Operation *op,
                                                NamedAttribute attr) {
   using bufferization::BufferizableOpInterface;
 
+  if (attr.getName() == kManualDeallocation) {
+    if (!mlir::hasEffect<MemoryEffects::Allocate>(op) &&
+        !mlir::hasEffect<MemoryEffects::Free>(op))
+      return op->emitOpError("attribute '")
+             << kManualDeallocation
+             << "' can be used only on ops that have an allocation and/or free "
+                "side effect";
+    return success();
+  }
+
   return op->emitError()
          << "attribute '" << attr.getName()
          << "' not supported as an op attribute by the bufferization dialect";
diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
index 738c8374d7..5716dcc9d9 100644
--- a/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
@@ -613,11 +613,19 @@ Value MaterializeInDestinationOp::buildSubsetExtraction(OpBuilder &builder,
     return getDest();
   }
 
+  // The "restrict" attribute is transferred from this op to the newly created
+  // to_tensor op. If this op does not the "restrict" attribute, the subset
+  // extraction cannot be built because there is no guarantee that there is no
+  // pre-existing "restrict" to_tensor op with the same/an aliasing destination.
+  if (!getRestrict())
+    return {};
+
   // Build a bufferization.to_tensor op.
   assert(isa<BaseMemRefType>(getDest().getType()) && "expected memref type");
   assert(getRestrict() &&
          "expected that ops with memrefs dest have 'restrict'");
-  return builder.create<ToTensorOp>(loc, getDest(), getRestrict(),
+  setRestrict(false);
+  return builder.create<ToTensorOp>(loc, getDest(), /*restrict=*/true,
                                     getWritable());
 }
 
@@ -647,9 +655,8 @@ LogicalResult MaterializeInDestinationOp::verify() {
   if (isa<BaseMemRefType>(getDest().getType()) &&
       getOperation()->getNumResults() != 0)
     return emitOpError("memref 'dest' implies zero results");
-  if (getRestrict() != isa<BaseMemRefType>(getDest().getType()))
-    return emitOpError("'restrict' must be specified if and only if the "
-                       "destination is of memref type");
+  if (getRestrict() && !isa<BaseMemRefType>(getDest().getType()))
+    return emitOpError("'restrict' is valid only for memref destinations");
   if (getWritable() != isa<BaseMemRefType>(getDest().getType()))
     return emitOpError("'writable' must be specified if and only if the "
                        "destination is of memref type");
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp b/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp
index fd36716163..b2b77eda92 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp
@@ -21,6 +21,7 @@
 #include "mlir/Dialect/Bufferization/IR/BufferDeallocationOpInterface.h"
 #include "mlir/Dialect/Bufferization/IR/Bufferization.h"
 #include "mlir/Dialect/Bufferization/Transforms/Passes.h"
+#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
@@ -856,13 +857,32 @@ FailureOr<Operation *> BufferDeallocation::handleInterface(CallOpInterface op) {
 FailureOr<Operation *>
 BufferDeallocation::handleInterface(MemoryEffectOpInterface op) {
   auto *block = op->getBlock();
+  OpBuilder builder = OpBuilder::atBlockBegin(block);
 
-  for (auto operand : llvm::make_filter_range(op->getOperands(), isMemref))
-    if (op.getEffectOnValue<MemoryEffects::Free>(operand).has_value())
-      return op->emitError(
-          "memory free side-effect on MemRef value not supported!");
+  for (auto operand : llvm::make_filter_range(op->getOperands(), isMemref)) {
+    if (op.getEffectOnValue<MemoryEffects::Free>(operand).has_value()) {
+      if (!op->hasAttr(BufferizationDialect::kManualDeallocation))
+        return op->emitError(
+            "memory free side-effect on MemRef value not supported!");
+
+      // Buffers that were allocated under "manual deallocation" may be
+      // manually deallocated. We insert a runtime assertion to cover certain
+      // cases of invalid IR where an automatically managed buffer allocation
+      // is manually deallocated. This is not a bulletproof check!
+      OpBuilder::InsertionGuard g(builder);
+      builder.setInsertionPoint(op);
+      Ownership ownership = state.getOwnership(operand, block);
+      if (ownership.isUnique()) {
+        Value ownershipInverted = builder.create<arith::XOrIOp>(
+            op.getLoc(), ownership.getIndicator(),
+            buildBoolValue(builder, op.getLoc(), true));
+        builder.create<cf::AssertOp>(
+            op.getLoc(), ownershipInverted,
+            "expected that the block does not have ownership");
+      }
+    }
+  }
 
-  OpBuilder builder = OpBuilder::atBlockBegin(block);
   for (auto res : llvm::make_filter_range(op->getResults(), isMemref)) {
     auto allocEffect = op.getEffectOnValue<MemoryEffects::Allocate>(res);
     if (allocEffect.has_value()) {
@@ -880,6 +900,15 @@ BufferDeallocation::handleInterface(MemoryEffectOpInterface op) {
         continue;
       }
 
+      if (op->hasAttr(BufferizationDialect::kManualDeallocation)) {
+        // This allocation will be deallocated manually. Assign an ownership of
+        // "false", so that it will never be deallocated by the buffer
+        // deallocation pass.
+        state.resetOwnerships(res, block);
+        state.updateOwnership(res, buildBoolValue(builder, op.getLoc(), false));
+        continue;
+      }
+
       state.updateOwnership(res, buildBoolValue(builder, op.getLoc(), true));
       state.addMemrefToDeallocate(res, block);
     }
diff --git a/mlir/lib/Dialect/GPU/Transforms/SerializeToCubin.cpp b/mlir/lib/Dialect/GPU/Transforms/SerializeToCubin.cpp
index 6ea7f57ad3..fb9f14ddbc 100644
--- a/mlir/lib/Dialect/GPU/Transforms/SerializeToCubin.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/SerializeToCubin.cpp
@@ -110,7 +110,11 @@ SerializeToCubinPass::serializeISA(const std::string &isa) {
   CUdevice device;
   RETURN_ON_CUDA_ERROR(cuDeviceGet(&device, 0));
   CUcontext context;
-  RETURN_ON_CUDA_ERROR(cuCtxCreate(&context, 0, device));
+  // Use the primary context.
+  RETURN_ON_CUDA_ERROR(cuDevicePrimaryCtxRetain(&context, device));
+  // Push the primary context so that the next CUDA operations
+  // actually use it.
+  RETURN_ON_CUDA_ERROR(cuCtxPushCurrent(context));
   CUlinkState linkState;
 
   CUjit_option jitOptions[] = {CU_JIT_ERROR_LOG_BUFFER,
@@ -146,7 +150,10 @@ SerializeToCubinPass::serializeISA(const std::string &isa) {
 
   // This will also destroy the cubin data.
   RETURN_ON_CUDA_ERROR(cuLinkDestroy(linkState));
-  RETURN_ON_CUDA_ERROR(cuCtxDestroy(context));
+  // Pop and release the primary context.
+  CUcontext poppedContext;
+  RETURN_ON_CUDA_ERROR(cuCtxPopCurrent(&poppedContext));
+  RETURN_ON_CUDA_ERROR(cuDevicePrimaryCtxRelease(device));
 
   return result;
 }
diff --git a/mlir/lib/Dialect/LLVMIR/CMakeLists.txt b/mlir/lib/Dialect/LLVMIR/CMakeLists.txt
index 230ffec900..b002596776 100644
--- a/mlir/lib/Dialect/LLVMIR/CMakeLists.txt
+++ b/mlir/lib/Dialect/LLVMIR/CMakeLists.txt
@@ -42,6 +42,7 @@ add_mlir_dialect_library(MLIRLLVMDialect
 
 add_mlir_dialect_library(MLIRNVVMDialect
   IR/NVVMDialect.cpp
+  IR/BasicPtxBuilderInterface.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/LLVMIR
@@ -50,6 +51,7 @@ add_mlir_dialect_library(MLIRNVVMDialect
   MLIRGPUCompilationAttrInterfacesIncGen
   MLIRNVVMOpsIncGen
   MLIRNVVMConversionsIncGen
+  MLIRBasicPtxBuilderInterfaceIncGen
   intrinsics_gen
 
   LINK_COMPONENTS
diff --git a/mlir/lib/Dialect/LLVMIR/IR/BasicPtxBuilderInterface.cpp b/mlir/lib/Dialect/LLVMIR/IR/BasicPtxBuilderInterface.cpp
new file mode 100644
index 0000000000..f3b674fdb5
--- /dev/null
+++ b/mlir/lib/Dialect/LLVMIR/IR/BasicPtxBuilderInterface.cpp
@@ -0,0 +1,158 @@
+//===- BasicPtxBuilderInterface.td - PTX builder interface -*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines the interface to build PTX (Parallel Thread Execution) from NVVM Ops
+// automatically. It is used by NVVM to LLVM pass.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/LLVMIR/BasicPtxBuilderInterface.h"
+#include "mlir/Support/LogicalResult.h"
+
+#define DEBUG_TYPE "ptx-builder"
+#define DBGS() (llvm::dbgs() << "[" DEBUG_TYPE "]: ")
+#define DBGSNL() (llvm::dbgs() << "\n")
+
+//===----------------------------------------------------------------------===//
+// BasicPtxBuilderInterface
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/LLVMIR/BasicPtxBuilderInterface.cpp.inc"
+
+using namespace mlir;
+using namespace NVVM;
+
+static constexpr int64_t kSharedMemorySpace = 3;
+
+static char getRegisterType(Type type) {
+  if (type.isInteger(1))
+    return 'b';
+  if (type.isInteger(16))
+    return 'h';
+  if (type.isInteger(32))
+    return 'r';
+  if (type.isInteger(64))
+    return 'l';
+  if (type.isF32())
+    return 'f';
+  if (type.isF64())
+    return 'd';
+  if (auto ptr = type.dyn_cast<LLVM::LLVMPointerType>()) {
+    // Shared address spaces is addressed with 32-bit pointers.
+    if (ptr.getAddressSpace() == kSharedMemorySpace) {
+      return 'r';
+    }
+    return 'l';
+  }
+  // register type for struct is not supported.
+  llvm_unreachable("The register type could not deduced from MLIR type");
+  return '?';
+}
+
+static char getRegisterType(Value v) {
+  if (v.getDefiningOp<LLVM::ConstantOp>())
+    return 'n';
+  return getRegisterType(v.getType());
+}
+
+void PtxBuilder::insertValue(Value v, PTXRegisterMod itype) {
+  LLVM_DEBUG(DBGS() << v << "\t Modifier : " << &itype << "\n");
+  auto getModifier = [&]() -> const char * {
+    if (itype == PTXRegisterMod::ReadWrite) {
+      assert(false && "Read-Write modifier is not supported. Try setting the "
+                      "same value as Write and Read seperately.");
+      return "+";
+    }
+    if (itype == PTXRegisterMod::Write) {
+      return "=";
+    }
+    return "";
+  };
+  auto addValue = [&](Value v) {
+    if (itype == PTXRegisterMod::Read) {
+      ptxOperands.push_back(v);
+      return;
+    }
+    if (itype == PTXRegisterMod::ReadWrite)
+      ptxOperands.push_back(v);
+    hasResult = true;
+  };
+
+  llvm::raw_string_ostream ss(registerConstraints);
+  // Handle Structs
+  if (auto stype = dyn_cast<LLVM::LLVMStructType>(v.getType())) {
+    if (itype == PTXRegisterMod::Write) {
+      addValue(v);
+    }
+    for (auto [idx, t] : llvm::enumerate(stype.getBody())) {
+      if (itype != PTXRegisterMod::Write) {
+        Value extractValue = rewriter.create<LLVM::ExtractValueOp>(
+            interfaceOp->getLoc(), v, idx);
+        addValue(extractValue);
+      }
+      if (itype == PTXRegisterMod::ReadWrite) {
+        ss << idx << ",";
+      } else {
+        ss << getModifier() << getRegisterType(t) << ",";
+      }
+      ss.flush();
+    }
+    return;
+  }
+  // Handle Scalars
+  addValue(v);
+  ss << getModifier() << getRegisterType(v) << ",";
+  ss.flush();
+}
+
+LLVM::InlineAsmOp PtxBuilder::build() {
+  auto asmDialectAttr = LLVM::AsmDialectAttr::get(interfaceOp->getContext(),
+                                                  LLVM::AsmDialect::AD_ATT);
+
+  auto resultTypes = interfaceOp->getResultTypes();
+
+  // Remove the last comma from the constraints string.
+  if (!registerConstraints.empty() &&
+      registerConstraints[registerConstraints.size() - 1] == ',')
+    registerConstraints.pop_back();
+
+  std::string ptxInstruction = interfaceOp.getPtx();
+
+  // Add the predicate to the asm string.
+  if (interfaceOp.getPredicate().has_value() &&
+      interfaceOp.getPredicate().value()) {
+    std::string predicateStr = "@%";
+    predicateStr += std::to_string((ptxOperands.size() - 1));
+    ptxInstruction = predicateStr + " " + ptxInstruction;
+  }
+
+  // Tablegen doesn't accept $, so we use %, but inline assembly uses $.
+  // Replace all % with $
+  std::replace(ptxInstruction.begin(), ptxInstruction.end(), '%', '$');
+
+  return rewriter.create<LLVM::InlineAsmOp>(
+      interfaceOp->getLoc(),
+      /*result types=*/resultTypes,
+      /*operands=*/ptxOperands,
+      /*asm_string=*/llvm::StringRef(ptxInstruction),
+      /*constraints=*/registerConstraints.data(),
+      /*has_side_effects=*/interfaceOp.hasSideEffect(),
+      /*is_align_stack=*/false,
+      /*asm_dialect=*/asmDialectAttr,
+      /*operand_attrs=*/ArrayAttr());
+}
+
+void PtxBuilder::buildAndReplaceOp() {
+  LLVM::InlineAsmOp inlineAsmOp = build();
+  LLVM_DEBUG(DBGS() << "\n Generated PTX \n\t" << inlineAsmOp << "\n");
+  if (inlineAsmOp->getNumResults() == interfaceOp->getNumResults()) {
+    rewriter.replaceOp(interfaceOp, inlineAsmOp);
+  } else {
+    rewriter.eraseOp(interfaceOp);
+  }
+}
diff --git a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
index 498e70b539..3736978505 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
@@ -624,7 +624,8 @@ inferMMATypeFromMNK(NVVM::MMATypes type, NVVM::MMAFrag frag, int m, int n,
 LogicalResult NVVM::WMMALoadOp::verify() {
   unsigned addressSpace =
       llvm::cast<LLVM::LLVMPointerType>(getPtr().getType()).getAddressSpace();
-  if (addressSpace != 0 && addressSpace != 1 && addressSpace != 3)
+  if (addressSpace != 0 && addressSpace != NVVM::kGlobalMemorySpace &&
+      addressSpace != NVVM::kSharedMemorySpace)
     return emitOpError("expected source pointer in memory "
                        "space 0, 1, 3");
 
@@ -644,7 +645,8 @@ LogicalResult NVVM::WMMALoadOp::verify() {
 LogicalResult NVVM::WMMAStoreOp::verify() {
   unsigned addressSpace =
       llvm::cast<LLVM::LLVMPointerType>(getPtr().getType()).getAddressSpace();
-  if (addressSpace != 0 && addressSpace != 1 && addressSpace != 3)
+  if (addressSpace != 0 && addressSpace != NVVM::kGlobalMemorySpace &&
+      addressSpace != NVVM::kSharedMemorySpace)
     return emitOpError("expected operands to be a source pointer in memory "
                        "space 0, 1, 3");
 
@@ -696,7 +698,7 @@ LogicalResult NVVM::WMMAMmaOp::verify() {
 LogicalResult NVVM::LdMatrixOp::verify() {
   unsigned addressSpace =
       llvm::cast<LLVM::LLVMPointerType>(getPtr().getType()).getAddressSpace();
-  if (addressSpace != 3)
+  if (addressSpace != NVVM::kSharedMemorySpace)
     return emitOpError("expected source pointer in memory space 3");
 
   if (getNum() != 1 && getNum() != 2 && getNum() != 4)
@@ -715,6 +717,19 @@ LogicalResult NVVM::LdMatrixOp::verify() {
   return success();
 }
 
+LogicalResult NVVM::StMatrixOp::verify() {
+  unsigned addressSpace =
+      llvm::cast<LLVM::LLVMPointerType>(getPtr().getType()).getAddressSpace();
+  if (addressSpace != NVVM::kSharedMemorySpace)
+    return emitOpError("expected source pointer in memory space 3");
+
+  int numMatrix = getSources().size();
+  if (numMatrix != 1 && numMatrix != 2 && numMatrix != 4)
+    return emitOpError("expected num attribute to be 1, 2 or 4");
+
+  return success();
+}
+
 FailureOr<int> getAllowedSizeK(NVVM::WGMMATypes typeA) {
   if (typeA == NVVM::WGMMATypes::tf32)
     return 8;
diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgInterfaces.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgInterfaces.cpp
index ea50e1232a..5fde8d71ca 100644
--- a/mlir/lib/Dialect/Linalg/IR/LinalgInterfaces.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/LinalgInterfaces.cpp
@@ -227,9 +227,6 @@ mlir::linalg::inferContractionDims(LinalgOp linalgOp) {
       linalgOp, linalgOp.getDpsInputOperand(1), red);
   llvm::set_intersect(ra, rb);
 
-  if (ac.empty() || bc.empty() || ra.empty())
-    return failure();
-
   // Return each set in sorted order.
   ContractionDimensions dimensions{
       SmallVector<unsigned, 2>(batches.begin(), batches.end()),
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
index 9ce780d3d2..8508507871 100644
--- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
+++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
@@ -3118,6 +3118,9 @@ DiagnosedSilenceableFailure transform::ConvertConv2DToImg2ColOp::applyToOne(
           .Case([&](linalg::Conv2DNhwcHwcfOp op) {
             return rewriteInIm2Col(rewriter, op);
           })
+          .Case([&](linalg::Conv2DNhwcFhwcOp op) {
+            return rewriteInIm2Col(rewriter, op);
+          })
           .Case([&](linalg::DepthwiseConv2DNhwcHwcOp op) {
             return rewriteInIm2Col(rewriter, op);
           })
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ConvertConv2DToImg2Col.cpp b/mlir/lib/Dialect/Linalg/Transforms/ConvertConv2DToImg2Col.cpp
index 275e78aaa7..e7629d7949 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ConvertConv2DToImg2Col.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ConvertConv2DToImg2Col.cpp
@@ -494,6 +494,141 @@ rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNchwFchwOp convOp) {
                         reshapedResult.getOperation());
 }
 
+FailureOr<std::pair<Operation *, Operation *>>
+rewriteInIm2Col(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp) {
+  auto inputType = cast<ShapedType>(convOp.getInputs()[0].getType());
+  auto filterType = cast<ShapedType>(convOp.getInputs()[1].getType());
+  auto outputType = cast<ShapedType>(convOp.getOutputs()[0].getType());
+
+  if (!filterType.hasStaticShape())
+    return rewriter.notifyMatchFailure(
+        convOp, "expected a static shape for the filter");
+
+  if (!inputType.hasStaticShape())
+    return rewriter.notifyMatchFailure(convOp,
+                                       "expected a static shape for the input");
+
+  // TODO: Support dilation.
+  if (!hasAllOneValues(convOp.getDilations()))
+    return rewriter.notifyMatchFailure(convOp,
+                                       "expected all ones for dilations");
+
+  MLIRContext *context = rewriter.getContext();
+  Value input = convOp.getInputs()[0];
+  Value filter = convOp.getInputs()[1];
+  Value output = convOp.getOutputs()[0];
+
+  ArrayRef<int64_t> filterShape = filterType.getShape();
+  ArrayRef<int64_t> outputShape = outputType.getShape();
+
+  int64_t n = outputShape[0];
+  int64_t oh = outputShape[1];
+  int64_t ow = outputShape[2];
+  int64_t oc = outputShape[3];
+  int64_t fh = filterShape[1];
+  int64_t fw = filterShape[2];
+  int64_t ic = filterShape[3];
+
+  Location loc = convOp.getLoc();
+
+  // Reshape output and filter to the LHS and result of a "row-wise" matrix
+  // multiplication.
+  SmallVector<ReassociationIndices> filterReassocIndices = {{0}, {1, 2, 3}};
+  auto reshapedFilterType =
+      RankedTensorType::get({oc, fh * fw * ic}, filterType.getElementType());
+  Value reshapedFilter = rewriter.create<tensor::CollapseShapeOp>(
+      loc, reshapedFilterType, filter, filterReassocIndices);
+
+  SmallVector<ReassociationIndices> outputReassocIndices = {{0}, {1, 2}, {3}};
+  RankedTensorType reshapedOutputType =
+      RankedTensorType::get({n, oh * ow, oc}, outputType.getElementType());
+  Value reshapedOutput = rewriter.create<tensor::CollapseShapeOp>(
+      loc, reshapedOutputType, output, outputReassocIndices);
+
+  SmallVector<int64_t> colTensorShape = {n, oh * ow, fh * fw * ic};
+  Value colTensor = rewriter.create<tensor::EmptyOp>(
+      loc, colTensorShape, inputType.getElementType());
+
+  // Convert the input to a (BMK) column tensor.
+  auto nloops = colTensorShape.size();
+
+  auto parallel = utils::IteratorType::parallel;
+  auto reduction = utils::IteratorType::reduction;
+  SmallVector<utils::IteratorType> img2colIterators(nloops, parallel);
+
+  SmallVector<AffineMap> img2colIndexingMaps = {
+      AffineMap::getMultiDimIdentityMap(nloops, context)};
+
+  auto img2ColTensor = rewriter.create<linalg::GenericOp>(
+      loc, colTensor.getType(),
+      /*inputs=*/ValueRange{}, /*outputs=*/colTensor, img2colIndexingMaps,
+      img2colIterators,
+      [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
+        // Get the iterators named based on the matmul (batch, m, k).
+        Value bIndex = nestedBuilder.create<linalg::IndexOp>(loc, 0);
+        Value mIndex = nestedBuilder.create<linalg::IndexOp>(loc, 1);
+        Value kIndex = nestedBuilder.create<linalg::IndexOp>(loc, 2);
+
+        // Recover the original iteration indices from the problem/input sizes.
+        SmallVector<Value> mIndices = unrollIndex(
+            nestedBuilder, nestedLoc, mIndex, ArrayRef<int64_t>{oh, ow});
+        auto ohIndex = mIndices[0];
+        auto owIndex = mIndices[1];
+
+        SmallVector<Value> kIndices = unrollIndex(
+            nestedBuilder, nestedLoc, kIndex, ArrayRef<int64_t>{fh, fw, ic});
+        auto fhIndex = kIndices[0];
+        auto fwIndex = kIndices[1];
+        auto icIndex = kIndices[2];
+
+        // Extract the input element corresponding to the expanded indices.
+        Value hIndex =
+            getConvolvedIndex(nestedBuilder, nestedLoc, ohIndex, fhIndex,
+                              convOp.getStrides().getValues<int64_t>()[0]);
+        Value wIndex =
+            getConvolvedIndex(nestedBuilder, nestedLoc, owIndex, fwIndex,
+                              convOp.getStrides().getValues<int64_t>()[1]);
+
+        // im2col[n, oh*ow, fh*fw*ic] = input[n, sh*oh + fh, sw*ow + fw, ic]
+        SmallVector<Value> extractionIndices{bIndex, hIndex, wIndex, icIndex};
+        Value inputVal = nestedBuilder.create<tensor::ExtractOp>(
+            loc, input, extractionIndices);
+        nestedBuilder.create<linalg::YieldOp>(nestedLoc, inputVal);
+      });
+
+  // Because we didn't transpose the filters we don't actually have a batched
+  // matrix multiply. Instead, we have an operation consisting of "row-wise" dot
+  // products.
+  AffineExpr bDim, mDim, nDim, kDim;
+  bindDims(context, bDim, mDim, nDim, kDim);
+  auto lhsMap = AffineMap::get(4, 0, {bDim, mDim, kDim}, context);
+  auto rhsMap = AffineMap::get(4, 0, {nDim, kDim}, context);
+  auto resultMap = AffineMap::get(4, 0, {bDim, mDim, nDim}, context);
+  SmallVector<utils::IteratorType> genericIterators = {parallel, parallel,
+                                                       parallel, reduction};
+
+  auto genericOp = rewriter.create<linalg::GenericOp>(
+      loc, reshapedOutputType,
+      /*inputs=*/ValueRange{img2ColTensor.getResult(0), reshapedFilter},
+      /*outputs=*/ValueRange{reshapedOutput},
+      ArrayRef<AffineMap>{lhsMap, rhsMap, resultMap}, genericIterators,
+      [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
+        Value mul =
+            createMul(loc, args[0], args[1], args[2].getType(), nestedBuilder);
+        Value add = createAdd(loc, mul, args[2], nestedBuilder);
+        nestedBuilder.create<linalg::YieldOp>(nestedLoc, add);
+      });
+  Value result = genericOp.getResults().front();
+
+  auto reshapedResult = rewriter.create<tensor::ExpandShapeOp>(
+      loc, outputType, result, outputReassocIndices);
+
+  rewriter.replaceOp(convOp, ArrayRef<Value>{reshapedResult});
+
+  return std::make_pair(img2ColTensor.getOperation(),
+                        reshapedResult.getOperation());
+}
+
 namespace {
 
 class ConvertConv2DNhwcHwcf final
@@ -534,12 +669,25 @@ class ConvertConv2DNchwFchw final
     return success();
   }
 };
+
+class ConvertConv2DNhwcFhwc final
+    : public OpRewritePattern<linalg::Conv2DNhwcFhwcOp> {
+public:
+  using OpRewritePattern::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(linalg::Conv2DNhwcFhwcOp convOp,
+                                PatternRewriter &rewriter) const override {
+    if (failed(rewriteInIm2Col(rewriter, convOp)))
+      return failure();
+    return success();
+  }
+};
 } // end anonymous namespace
 
 void populateConvertConv2DToImg2ColPatterns(RewritePatternSet &patterns) {
   MLIRContext *context = patterns.getContext();
   patterns.insert<ConvertConv2DNhwcHwcf, ConvertDepthwiseConv2DNhwcHwc,
-                  ConvertConv2DNchwFchw>(context);
+                  ConvertConv2DNchwFchw, ConvertConv2DNhwcFhwc>(context);
 }
 } // end namespace linalg
 } // end namespace mlir
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
index 6f4b0ff60c..32d38a21e4 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp
@@ -1373,16 +1373,17 @@ getOperandReassociation(AffineMap indexingMap,
 }
 
 /// Get the new value to use for a given `OpOperand` in the collapsed operation.
-static Value getCollapsedOpOperand(Location loc, GenericOp genericOp,
+static Value getCollapsedOpOperand(Location loc, LinalgOp op,
                                    OpOperand *opOperand,
                                    const CollapsingInfo &collapsingInfo,
                                    OpBuilder &builder) {
-  AffineMap indexingMap = genericOp.getMatchingIndexingMap(opOperand);
+  AffineMap indexingMap = op.getMatchingIndexingMap(opOperand);
   SmallVector<ReassociationIndices> operandReassociation =
       getOperandReassociation(indexingMap, collapsingInfo);
 
-  // If the number of entries in the reassocation for the operand is same as the
-  // number of results of the indexing map, then nothing to do for this operand.
+  // If the number of entries in the reassociation for the operand is same as
+  // the number of results of the indexing map, then nothing to do for this
+  // operand.
   Value operand = opOperand->get();
   if (operandReassociation.size() == indexingMap.getNumResults())
     return operand;
@@ -1439,20 +1440,80 @@ void generateCollapsedIndexingRegion(Location loc, Block *block,
   }
 }
 
+template <typename LinalgType>
+Operation *createCollapsedOp(LinalgType op,
+                             const CollapsingInfo &collapsingInfo,
+                             RewriterBase &rewriter) {
+  static_assert(llvm::is_one_of<LinalgType, GenericOp, CopyOp>::value,
+                "unsupported linalg op type to create");
+  Location loc = op->getLoc();
+
+  // Get the input operands.
+  SmallVector<Value> inputOperands =
+      llvm::map_to_vector(op.getDpsInputOperands(), [&](OpOperand *opOperand) {
+        return getCollapsedOpOperand(loc, op, opOperand, collapsingInfo,
+                                     rewriter);
+      });
+
+  // Get the output operands and result types.
+  SmallVector<Type> resultTypes;
+  SmallVector<Value> outputOperands;
+  resultTypes.reserve(op.getNumDpsInits());
+  outputOperands.reserve(op.getNumDpsInits());
+  for (OpOperand &output : op.getDpsInitsMutable()) {
+    Value newOutput =
+        getCollapsedOpOperand(loc, op, &output, collapsingInfo, rewriter);
+    outputOperands.push_back(newOutput);
+    // If the op has "buffer semantics", then the init operands are ranked
+    // memrefs and the op has no results.
+    if (!op.hasBufferSemantics())
+      resultTypes.push_back(newOutput.getType());
+  }
+
+  if (isa<linalg::CopyOp>(op)) {
+    return rewriter.create<linalg::CopyOp>(loc, inputOperands[0],
+                                           outputOperands[0]);
+  }
+
+  // Get the iterator types for the operand.
+  SmallVector<utils::IteratorType> iteratorTypes =
+      getCollapsedOpIteratorTypes(op.getIteratorTypesArray(), collapsingInfo);
+
+  // Get the indexing maps.
+  auto indexingMaps =
+      llvm::map_to_vector(op.getIndexingMapsArray(), [&](AffineMap map) {
+        return getCollapsedOpIndexingMap(map, collapsingInfo);
+      });
+
+  Operation *collapsedOp = rewriter.create<linalg::GenericOp>(
+      loc, resultTypes, inputOperands, outputOperands, indexingMaps,
+      iteratorTypes, [](OpBuilder &builder, Location loc, ValueRange args) {});
+  Block *origOpBlock = &op->getRegion(0).front();
+  Block *collapsedOpBlock = &collapsedOp->getRegion(0).front();
+  rewriter.mergeBlocks(origOpBlock, collapsedOpBlock,
+                       collapsedOpBlock->getArguments());
+
+  return collapsedOp;
+}
+
 /// Implementation of fusion with reshape operation by collapsing dimensions.
-FailureOr<SmallVector<Value>> mlir::linalg::collapseGenericOpIterationDims(
-    GenericOp genericOp, ArrayRef<ReassociationIndices> foldedIterationDims,
+template <typename LinalgType>
+FailureOr<SmallVector<Value>> mlir::linalg::collapseOpIterationDims(
+    LinalgType op, ArrayRef<ReassociationIndices> foldedIterationDims,
     RewriterBase &rewriter) {
+  static_assert(llvm::is_one_of<LinalgType, GenericOp, CopyOp>::value,
+                "unsupported linalg op type to collapse");
+
   // Bail on trivial no-op cases.
-  if (genericOp.getNumLoops() <= 1 || foldedIterationDims.empty() ||
+  if (op.getNumLoops() <= 1 || foldedIterationDims.empty() ||
       llvm::all_of(foldedIterationDims, [](ReassociationIndicesRef foldedDims) {
         return foldedDims.size() <= 1;
       }))
     return failure();
 
-  bool hasBufferSemantics = genericOp.hasBufferSemantics();
+  bool hasBufferSemantics = op.hasBufferSemantics();
   if (hasBufferSemantics &&
-      !llvm::all_of(genericOp->getOperands(), [&](Value operand) -> bool {
+      !llvm::all_of(op->getOperands(), [&](Value operand) -> bool {
         MemRefType memRefToCollapse = dyn_cast<MemRefType>(operand.getType());
         if (!memRefToCollapse)
           return true;
@@ -1460,20 +1521,19 @@ FailureOr<SmallVector<Value>> mlir::linalg::collapseGenericOpIterationDims(
         return memref::CollapseShapeOp::isGuaranteedCollapsible(
             memRefToCollapse, foldedIterationDims);
       }))
-    return rewriter.notifyMatchFailure(genericOp,
+    return rewriter.notifyMatchFailure(op,
                                        "memref is not guaranteed collapsible");
 
   CollapsingInfo collapsingInfo;
-  if (failed(collapsingInfo.initialize(genericOp.getNumLoops(),
-                                       foldedIterationDims))) {
+  if (failed(
+          collapsingInfo.initialize(op.getNumLoops(), foldedIterationDims))) {
     return rewriter.notifyMatchFailure(
-        genericOp, "illegal to collapse specified dimensions");
+        op, "illegal to collapse specified dimensions");
   }
 
   // Bail on non-canonical ranges.
   SmallVector<Range> loopRanges =
-      cast<LinalgOp>(genericOp.getOperation())
-          .createLoopRanges(rewriter, genericOp.getLoc());
+      cast<LinalgOp>(op.getOperation()).createLoopRanges(rewriter, op.getLoc());
   auto opFoldIsConstantValue = [](OpFoldResult ofr, int64_t value) {
     if (auto attr = llvm::dyn_cast_if_present<Attribute>(ofr))
       return cast<IntegerAttr>(attr).getInt() == value;
@@ -1486,78 +1546,36 @@ FailureOr<SmallVector<Value>> mlir::linalg::collapseGenericOpIterationDims(
                opFoldIsConstantValue(range.stride, 1);
       })) {
     return rewriter.notifyMatchFailure(
-        genericOp,
-        "expected all loop ranges to have zero start and unit stride");
+        op, "expected all loop ranges to have zero start and unit stride");
   }
 
-  // Get the iterator types for the operand.
-  SmallVector<utils::IteratorType> iteratorTypes = getCollapsedOpIteratorTypes(
-      genericOp.getIteratorTypesArray(), collapsingInfo);
-
-  // Get the indexing maps.
-  auto indexingMaps = llvm::to_vector(
-      llvm::map_range(genericOp.getIndexingMapsArray(), [&](AffineMap map) {
-        return getCollapsedOpIndexingMap(map, collapsingInfo);
-      }));
-
-  Location loc = genericOp->getLoc();
-
-  // Get the input operands.
-  auto inputOperands = llvm::to_vector(llvm::map_range(
-      genericOp.getDpsInputOperands(), [&](OpOperand *opOperand) {
-        return getCollapsedOpOperand(loc, genericOp, opOperand, collapsingInfo,
-                                     rewriter);
-      }));
-
-  // Get the output operands and result types.
-  SmallVector<Type> resultTypes;
-  SmallVector<Value> outputOperands;
-  resultTypes.reserve(genericOp.getNumDpsInits());
-  outputOperands.reserve(genericOp.getNumDpsInits());
-  for (OpOperand &output : genericOp.getDpsInitsMutable()) {
-    Value newOutput = getCollapsedOpOperand(loc, genericOp, &output,
-                                            collapsingInfo, rewriter);
-    outputOperands.push_back(newOutput);
-    // If the op has "buffer semantics", then the init operands are ranked
-    // memrefs and the op has no results.
-    if (!hasBufferSemantics)
-      resultTypes.push_back(newOutput.getType());
-  }
-
-  // Create the generic op.
-  auto collapsedGenericOp = rewriter.create<linalg::GenericOp>(
-      loc, resultTypes, inputOperands, outputOperands, indexingMaps,
-      iteratorTypes, [](OpBuilder &builder, Location loc, ValueRange args) {});
-  Block *origOpBlock = &genericOp->getRegion(0).front();
-  Block *collapsedOpBlock = &collapsedGenericOp->getRegion(0).front();
-  rewriter.mergeBlocks(origOpBlock, collapsedOpBlock,
-                       collapsedOpBlock->getArguments());
+  LinalgType collapsedOp = cast<LinalgType>(
+      createCollapsedOp<LinalgType>(op, collapsingInfo, rewriter));
 
-  if (collapsedGenericOp.hasIndexSemantics()) {
+  Location loc = op->getLoc();
+  if (collapsedOp.hasIndexSemantics()) {
     // Collect the loop range of the generic op.
     OpBuilder::InsertionGuard g(rewriter);
-    rewriter.setInsertionPoint(collapsedGenericOp);
+    rewriter.setInsertionPoint(collapsedOp);
     SmallVector<Value> loopBound =
-        llvm::to_vector(llvm::map_range(loopRanges, [&](Range range) {
+        llvm::map_to_vector(loopRanges, [&](Range range) {
           return getValueOrCreateConstantIndexOp(rewriter, loc, range.size);
-        }));
-    generateCollapsedIndexingRegion(loc,
-                                    &collapsedGenericOp->getRegion(0).front(),
+        });
+    generateCollapsedIndexingRegion(loc, &collapsedOp->getRegion(0).front(),
                                     collapsingInfo, loopBound, rewriter);
   }
 
   // Insert expanding reshape for the result to get back the original result
   // type.
   SmallVector<Value> results;
-  for (const auto &originalResult : llvm::enumerate(genericOp->getResults())) {
-    Value collapsedOpResult =
-        collapsedGenericOp->getResult(originalResult.index());
+  for (const auto &originalResult : llvm::enumerate(op->getResults())) {
+    Value collapsedOpResult = collapsedOp->getResult(originalResult.index());
     auto originalResultType =
         cast<ShapedType>(originalResult.value().getType());
     auto collapsedOpResultType = cast<ShapedType>(collapsedOpResult.getType());
     if (collapsedOpResultType.getRank() != originalResultType.getRank()) {
       AffineMap indexingMap =
-          genericOp.getIndexingMapMatchingResult(originalResult.value());
+          op.getIndexingMapMatchingResult(originalResult.value());
       SmallVector<ReassociationIndices> reassociation =
           getOperandReassociation(indexingMap, collapsingInfo);
       if (isa<MemRefType>(collapsedOpResult.getType())) {
@@ -1606,8 +1624,8 @@ class FoldWithProducerReshapeOpByCollapsing
       }
 
       std::optional<SmallVector<Value>> replacements =
-          collapseGenericOpIterationDims(genericOp, collapsableIterationDims,
-                                         rewriter);
+          collapseOpIterationDims<linalg::GenericOp>(
+              genericOp, collapsableIterationDims, rewriter);
       if (!replacements) {
         return rewriter.notifyMatchFailure(
             genericOp, "failed to do the fusion by collapsing transformation");
@@ -1624,36 +1642,36 @@ class FoldWithProducerReshapeOpByCollapsing
 };
 
 /// Pattern to collapse dimensions.
-class CollapseLinalgDimensions : public OpRewritePattern<GenericOp> {
+template <typename LinalgType>
+class CollapseLinalgDimensions : public OpRewritePattern<LinalgType> {
 public:
   CollapseLinalgDimensions(MLIRContext *context,
                            GetCollapsableDimensionsFn collapseDimensions,
                            PatternBenefit benefit = 1)
-      : OpRewritePattern<GenericOp>(context, benefit),
+      : OpRewritePattern<LinalgType>(context, benefit),
         controlCollapseDimension(std::move(collapseDimensions)) {}
 
-  LogicalResult matchAndRewrite(GenericOp genericOp,
+  LogicalResult matchAndRewrite(LinalgType op,
                                 PatternRewriter &rewriter) const override {
     SmallVector<ReassociationIndices> collapsableIterationDims =
-        controlCollapseDimension(genericOp);
+        controlCollapseDimension(op);
     if (collapsableIterationDims.empty())
       return failure();
 
     // Check if the specified list of dimensions to collapse is a valid list.
-    if (!areDimSequencesPreserved(genericOp.getIndexingMapsArray(),
+    if (!areDimSequencesPreserved(op.getIndexingMapsArray(),
                                   collapsableIterationDims)) {
       return rewriter.notifyMatchFailure(
-          genericOp, "specified dimensions cannot be collapsed");
+          op, "specified dimensions cannot be collapsed");
     }
 
     std::optional<SmallVector<Value>> replacements =
-        collapseGenericOpIterationDims(genericOp, collapsableIterationDims,
-                                       rewriter);
+        collapseOpIterationDims<LinalgType>(op, collapsableIterationDims,
+                                            rewriter);
     if (!replacements) {
-      return rewriter.notifyMatchFailure(genericOp,
-                                         "failed to collapse dimensions");
+      return rewriter.notifyMatchFailure(op, "failed to collapse dimensions");
     }
-    rewriter.replaceOp(genericOp, *replacements);
+    rewriter.replaceOp(op, *replacements);
     return success();
   }
 
@@ -1884,8 +1902,9 @@ void mlir::linalg::populateElementwiseOpsFusionPatterns(
 void mlir::linalg::populateCollapseDimensions(
     RewritePatternSet &patterns,
     const GetCollapsableDimensionsFn &controlCollapseDimensions) {
-  patterns.add<CollapseLinalgDimensions>(patterns.getContext(),
-                                         controlCollapseDimensions);
+  patterns.add<CollapseLinalgDimensions<linalg::GenericOp>,
+               CollapseLinalgDimensions<linalg::CopyOp>>(
+      patterns.getContext(), controlCollapseDimensions);
 }
 
 //===---------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp b/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp
index 72bd2b409f..8fef99bb37 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp
@@ -811,8 +811,7 @@ padThroughLoopIterArg(RewriterBase &rewriter, Value paddedValueBeforeHoisting,
   rewriter.setInsertionPointAfter(hoistedPackedTensor.getDefiningOp());
 
   unsigned iterArgNumber = forOp.getResultForOpOperand(*pUse).getResultNumber();
-  auto yieldOp = cast<scf::YieldOp>(forOp.getBody(0)->getTerminator());
-  auto yieldingExtractSliceOp = yieldOp->getOperand(iterArgNumber)
+  auto yieldingExtractSliceOp = forOp.getYieldedValues()[iterArgNumber]
                                     .getDefiningOp<tensor::ExtractSliceOp>();
   if (!yieldingExtractSliceOp)
     return tensor::ExtractSliceOp();
@@ -826,7 +825,7 @@ padThroughLoopIterArg(RewriterBase &rewriter, Value paddedValueBeforeHoisting,
 
   SmallVector<Value> initArgs = forOp.getInitArgs();
   initArgs[iterArgNumber] = hoistedPackedTensor;
-  SmallVector<Value> yieldOperands = yieldOp.getOperands();
+  SmallVector<Value> yieldOperands = llvm::to_vector(forOp.getYieldedValues());
   yieldOperands[iterArgNumber] = yieldingExtractSliceOp.getSource();
 
   int64_t numOriginalForOpResults = initArgs.size();
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
index 221bec713b..cbb2c507de 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
@@ -173,16 +173,16 @@ void mlir::linalg::hoistRedundantVectorTransfers(func::FuncOp func) {
         if (auto transferWriteUse =
                 dyn_cast<vector::TransferWriteOp>(use.getOwner())) {
           if (!vector::isDisjointTransferSet(
-                  cast<VectorTransferOpInterface>(transferWrite.getOperation()),
-                  cast<VectorTransferOpInterface>(
-                      transferWriteUse.getOperation())))
+                  cast<VectorTransferOpInterface>(*transferWrite),
+                  cast<VectorTransferOpInterface>(*transferWriteUse),
+                  /*testDynamicValueUsingBounds=*/true))
             return WalkResult::advance();
         } else if (auto transferReadUse =
                        dyn_cast<vector::TransferReadOp>(use.getOwner())) {
           if (!vector::isDisjointTransferSet(
-                  cast<VectorTransferOpInterface>(transferWrite.getOperation()),
-                  cast<VectorTransferOpInterface>(
-                      transferReadUse.getOperation())))
+                  cast<VectorTransferOpInterface>(*transferWrite),
+                  cast<VectorTransferOpInterface>(*transferReadUse),
+                  /*testDynamicValueUsingBounds=*/true))
             return WalkResult::advance();
         } else {
           // Unknown use, we cannot prove that it doesn't alias with the
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp b/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
index ad399f57f7..a131f30976 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
@@ -28,6 +28,7 @@
 #include "mlir/Transforms/FoldUtils.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SmallBitVector.h"
+#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
@@ -142,6 +143,8 @@ struct LinalgOpInstancePromotionOptions {
                                    const LinalgPromotionOptions &options);
   /// SubViews to promote.
   MapVector<int64_t, Value> subViews;
+  /// Subviews operand numbers to copy in using copyInFn.
+  llvm::SmallSet<int64_t, 4> operandsNumbersToCopyIn;
   /// True if the full view should be used for the promoted buffer.
   DenseMap<Value, bool> useFullTileBuffers;
 
@@ -174,6 +177,11 @@ LinalgOpInstancePromotionOptions::LinalgOpInstancePromotionOptions(
     Operation *op = opOperand.get().getDefiningOp();
     if (auto sv = dyn_cast_or_null<memref::SubViewOp>(op)) {
       subViews[operandNumber] = sv;
+      // In case of linalg generic, copy in only if subview is used in linalg
+      // payload.
+      if (!isa<linalg::GenericOp>(linalgOp) ||
+          linalgOp.payloadUsesValueFromOperand(&opOperand))
+        operandsNumbersToCopyIn.insert(operandNumber);
       useFullTileBuffers[sv] = vUseFullTileBuffers[operandNumber];
     }
   }
@@ -324,6 +332,8 @@ promoteSubViews(ImplicitLocOpBuilder &b,
     auto info = promotionInfoMap.find(v.first);
     if (info == promotionInfoMap.end())
       continue;
+    if (options.operandsNumbersToCopyIn.count(v.first) == 0)
+      continue;
     if (failed(options.copyInFn(
             b, cast<memref::SubViewOp>(v.second.getDefiningOp()),
             info->second.partialLocalView)))
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
index d60a1a1cda..b427af33e3 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
@@ -1693,11 +1693,16 @@ LogicalResult mlir::linalg::vectorize(RewriterBase &rewriter, Operation *op,
           .Case<linalg::LinalgOp>([&](auto linalgOp) {
             // TODO: isaConvolutionOpInterface that can also infer from generic
             // features. Will require stride/dilation attributes inference.
-            FailureOr<Operation *> convOr =
-                vectorizeConvolution(rewriter, linalgOp);
-            if (succeeded(convOr)) {
-              llvm::append_range(results, (*convOr)->getResults());
-              return success();
+            if (isa<ConvolutionOpInterface>(linalgOp.getOperation())) {
+              FailureOr<Operation *> convOr =
+                  vectorizeConvolution(rewriter, linalgOp);
+              if (succeeded(convOr)) {
+                llvm::append_range(results, (*convOr)->getResults());
+                return success();
+              }
+
+              LDBG("Unsupported convolution can't be vectorized.\n");
+              return failure();
             }
 
             LDBG("Vectorize generic by broadcasting to the canonical vector "
diff --git a/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp b/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp
index 7f8322bd5f..9899c357da 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/FoldMemRefAliasOps.cpp
@@ -128,10 +128,16 @@ resolveSourceIndicesCollapseShape(Location loc, PatternRewriter &rewriter,
     dynamicIndices.push_back(indices[cnt++]);
     int64_t groupSize = groups.size();
 
-    // Calculate suffix product for all collapse op source dimension sizes.
-    SmallVector<int64_t> sizes(groupSize);
-    for (int64_t i = 0; i < groupSize; ++i)
+    // Calculate suffix product for all collapse op source dimension sizes
+    // except the most major one of each group.
+    // We allow the most major source dimension to be dynamic but enforce all
+    // others to be known statically.
+    SmallVector<int64_t> sizes(groupSize, 1);
+    for (int64_t i = 1; i < groupSize; ++i) {
       sizes[i] = collapseShapeOp.getSrcType().getDimSize(groups[i]);
+      if (sizes[i] == ShapedType::kDynamic)
+        return failure();
+    }
     SmallVector<int64_t> suffixProduct = computeSuffixProduct(sizes);
 
     // Derive the index values along all dimensions of the source corresponding
diff --git a/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp b/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp
index b2a4710252..fc91fd994f 100644
--- a/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp
+++ b/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp
@@ -47,7 +47,7 @@ Operation *MeshDialect::materializeConstant(OpBuilder &builder, Attribute value,
 
 LogicalResult ClusterOp::verify() {
   ArrayRef<int64_t> dimSizes = getDimSizes();
-  uint8_t rank = getRank();
+  uint64_t rank = getRank();
 
   if (rank == 0)
     return emitOpError("rank of cluster is expected to be a positive integer");
@@ -71,15 +71,15 @@ LogicalResult ClusterOp::verify() {
 
 LogicalResult
 MeshShardingAttr::verify(function_ref<InFlightDiagnostic()> emitError,
-                         SymbolRefAttr, ArrayRef<DenseI8ArrayAttr> splitAxes,
-                         ArrayRef<int8_t> partialAxes, Partial) {
+                         SymbolRefAttr, ArrayRef<DenseI32ArrayAttr> splitAxes,
+                         ArrayRef<int32_t> partialAxes, Partial) {
   // TODO: At present cluster symbol ref is not verified. This is due to the
   // difficulty in fetching the corresponding symbol op based on an attribute.
 
-  llvm::SmallSet<int8_t, 4> visitedAxes;
+  llvm::SmallSet<int32_t, 4> visitedAxes;
 
-  auto checkMeshAxis = [&](ArrayRef<int8_t> axesArray) -> LogicalResult {
-    for (int8_t axis : axesArray) {
+  auto checkMeshAxis = [&](ArrayRef<int32_t> axesArray) -> LogicalResult {
+    for (int32_t axis : axesArray) {
       if (axis < 0)
         return emitError() << "mesh axis is expected to be non-negative";
       if (!visitedAxes.insert(axis).second)
@@ -88,8 +88,8 @@ MeshShardingAttr::verify(function_ref<InFlightDiagnostic()> emitError,
     return success();
   };
 
-  for (DenseI8ArrayAttr subAxes : splitAxes) {
-    ArrayRef<int8_t> subAxesArray = subAxes.asArrayRef();
+  for (DenseI32ArrayAttr subAxes : splitAxes) {
+    ArrayRef<int32_t> subAxesArray = subAxes.asArrayRef();
     if (failed(checkMeshAxis(subAxesArray)))
       return failure();
   }
diff --git a/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp b/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
index e8ecd0faa4..f5b02fe1b5 100644
--- a/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
+++ b/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
@@ -435,6 +435,12 @@ LogicalResult isAllowedWGMMADataType(Type typeD, Type typeA, Type typeB) {
   return failure();
 }
 
+LogicalResult isAllowedSizeM(int sizeM) {
+  if (sizeM % kWgmmaSizeM)
+    return failure();
+  return success();
+}
+
 LogicalResult isAllowedSizeN(int sizeN, Type typeA) {
   SmallVector<int> allowedN = {8,   16,  24,  32,  40,  48,  56,  64,
                                72,  80,  88,  96,  104, 112, 120, 128,
@@ -443,7 +449,7 @@ LogicalResult isAllowedSizeN(int sizeN, Type typeA) {
   SmallVector<int> allowedNshort = {8,   16,  24,  32,  48,  64,
                                     80,  96,  112, 128, 144, 160,
                                     176, 192, 208, 224, 240, 256};
-  if (typeA.isBF16() || typeA.isF16() || typeA.isTF32() ||
+  if (typeA.isBF16() || typeA.isF16() || typeA.isF32() || typeA.isTF32() ||
       typeA.isFloat8E4M3FN() || typeA.isFloat8E5M2())
     if (llvm::is_contained(allowedN, sizeN))
       return success();
@@ -456,35 +462,16 @@ LogicalResult isAllowedSizeN(int sizeN, Type typeA) {
 
 LogicalResult WarpgroupMmaOp::verify() {
   if (getTransposeA() && !getTransposeB())
-    return emitOpError() << "supports non-transpose A (Row Major) "
-                            "and transpose B (Column Major) for the time being";
+    return emitOpError()
+           << "supports non-transpose A (Row Major) "
+              "and transpose B (Column Major) for the time being ";
   MemRefType matrixA = getDescriptorA().getType().getTensor();
   MemRefType matrixB = getDescriptorB().getType().getTensor();
-  VectorType matrixC = getMatrixC()
-                           .front()
-                           .getType()
-                           .cast<WarpgroupAccumulatorType>()
-                           .getFragmented();
-  VectorType matrixD = getMatrixD()
-                           .front()
-                           .getType()
-                           .cast<WarpgroupAccumulatorType>()
-                           .getFragmented();
-  unsigned sizeAcc = getMatrixC().size();
-
-  if (getMatrixC().size() != getMatrixD().size())
-    return emitOpError() << "number of matrix C and matrix D must be the same";
-
-  if (llvm::all_of(getMatrixC(),
-                   [&](Value rhs) { return rhs.getType() == matrixC; })) {
-    return emitOpError()
-           << "types of all operands in matrix C must be the same";
-  }
-  if (llvm::all_of(getMatrixD(),
-                   [&](Value rhs) { return rhs.getType() == matrixC; })) {
-    return emitOpError()
-           << "types of all operands in matrix D must be the same as matrix C";
-  }
+  VectorType matrixC = getMatrixC().getType().getFragmented();
+  VectorType matrixD = getMatrixD().getType().getFragmented();
+
+  if (matrixC != matrixD)
+    return emitOpError() << "type of matrix C and matrix D must be the same";
 
   if (matrixA.getRank() != 2 || matrixB.getRank() != 2 ||
       matrixC.getRank() != 2 || matrixD.getRank() != 2) {
@@ -496,7 +483,7 @@ LogicalResult WarpgroupMmaOp::verify() {
     return emitOpError() << "2nd dim matrix-A (" << matrixA.getShape()[1]
                          << ")!= 1st dim matrix-B (" << matrixB.getShape()[0]
                          << " )";
-  if (matrixA.getShape()[0] != (matrixC.getShape()[0] * sizeAcc))
+  if (matrixA.getShape()[0] != matrixC.getShape()[0])
     return emitOpError() << "1st dim matrix-A ( " << matrixA.getShape()[0]
                          << " )!= 1st dim matrix-C ( " << matrixC.getShape()[0]
                          << " )";
@@ -532,29 +519,16 @@ LogicalResult WarpgroupMmaOp::verify() {
 
 LogicalResult WarpgroupMmaStoreOp::verify() {
   MemRefType dstMemrefType = getDstMemref().getType();
-  VectorType firstVtype = getMatrixD()
-                              .front()
-                              .getType()
-                              .cast<WarpgroupAccumulatorType>()
-                              .getFragmented();
-
-  int64_t totalFirstDimension = 0;
-  for (Value result : getMatrixD()) {
-    VectorType vtype =
-        result.getType().cast<WarpgroupAccumulatorType>().getFragmented();
-    if (vtype != firstVtype)
-      return emitOpError() << "all fragmented types must be the same";
-    // Limitation
-    if (!vtype.getElementType().isF32()) {
-      return emitOpError()
-             << "hit a limitation: only f32 results for the time being";
-    }
-    totalFirstDimension += vtype.getDimSize(0);
+  VectorType vtype = getMatrixD().getType().getFragmented();
+
+  // Limitation
+  if (!vtype.getElementType().isF32()) {
+    return emitOpError()
+           << "hit a limitation: only f32 results for the time being";
   }
-  if (totalFirstDimension != dstMemrefType.getDimSize(0) ||
-      firstVtype.getDimSize(1) != dstMemrefType.getDimSize(1)) {
-    return emitOpError() << "results [" << totalFirstDimension << "]["
-                         << firstVtype.getDimSize(1)
+  if (vtype.getDimSize(0) != dstMemrefType.getDimSize(0) ||
+      vtype.getDimSize(1) != dstMemrefType.getDimSize(1)) {
+    return emitOpError() << "results [" << vtype << "][" << vtype.getDimSize(1)
                          << "] values. However, destination memref["
                          << dstMemrefType.getDimSize(0) << "]["
                          << dstMemrefType.getDimSize(1)
@@ -563,6 +537,27 @@ LogicalResult WarpgroupMmaStoreOp::verify() {
   return success();
 }
 
+//===----------------------------------------------------------------------===//
+// WarpgroupMmaInitAccumulatorOp
+//===----------------------------------------------------------------------===//
+
+LogicalResult WarpgroupMmaInitAccumulatorOp::verify() {
+
+  nvgpu::WarpgroupAccumulatorType accType = getMatrixC().getType();
+  int64_t sizeM = accType.getFragmented().getDimSize(0);
+  int64_t sizeN = accType.getFragmented().getDimSize(1);
+  Type elemType = accType.getFragmented().getElementType();
+
+  if (failed(isAllowedSizeM(sizeM)) ||
+      failed(isAllowedSizeN(sizeN, elemType))) {
+    return emitOpError() << "has type " << accType.getFragmented()
+                         << ". It does not fit into warp-group "
+                            "level (wgmma) matrix multiplication instruction "
+                            "(or not supported yet)";
+  }
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // TableGen'd dialect, type, and op definitions
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp b/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp
index 94d7d565ff..408c1dc798 100644
--- a/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp
+++ b/mlir/lib/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp
@@ -62,10 +62,28 @@ void transform::ApplyNVGPUToNVVMConversionPatternsOp::populatePatterns(
   });
   llvmTypeConverter.addConversion(
       [&](nvgpu::WarpgroupAccumulatorType type) -> Type {
-        VectorType vtype = type.getFragmented();
+        Type elemType = type.getFragmented().getElementType();
+        int64_t sizeM = type.getFragmented().getDimSize(0);
+        int64_t sizeN = type.getFragmented().getDimSize(1);
+
+        unsigned numMembers;
+        if (elemType.isF32() || elemType.isInteger(32))
+          numMembers = sizeN / 2;
+        else if (elemType.isF16())
+          numMembers = sizeN / 4;
+        else
+          llvm_unreachable("unsupported type for warpgroup accumulator");
+
+        SmallVector<Type> innerStructBody;
+        for (unsigned i = 0; i < numMembers; i++)
+          innerStructBody.push_back(elemType);
+        auto innerStructType = LLVM::LLVMStructType::getLiteral(
+            type.getContext(), innerStructBody);
+
         SmallVector<Type> structBody;
-        for (unsigned i = 0; i < vtype.getDimSize(0); i++)
-          structBody.push_back(vtype.getElementType());
+        for (int i = 0; i < sizeM; i += kWgmmaSizeM)
+          structBody.push_back(innerStructType);
+
         auto convertedType =
             LLVM::LLVMStructType::getLiteral(type.getContext(), structBody);
         return llvmTypeConverter.convertType(convertedType);
@@ -904,7 +922,7 @@ HopperBuilder::buildAndInitBarrierInSharedMemory(OpFoldResult numThreads) {
   Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
   rewriter.create<nvgpu::MBarrierInitOp>(
       loc, barrier, getValueOrCreateConstantIndexOp(rewriter, loc, numThreads),
-      zero);
+      zero, Value());
   rewriter.create<gpu::BarrierOp>(loc);
   return cast<TypedValue<nvgpu::MBarrierGroupType>>(barrier);
 }
@@ -946,7 +964,8 @@ OpFoldResult HopperBuilder::buildTmaAsyncLoad(
   MLIRContext *ctx = rewriter.getContext();
   Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
   Operation *loadOp = rewriter.create<nvgpu::TmaAsyncLoadOp>(
-      loc, sharedMemref, barrier, globalDesc, ValueRange{zero, zero}, zero);
+      loc, sharedMemref, barrier, globalDesc, ValueRange{zero, zero}, zero,
+      Value());
   loadOps.push_back(loadOp);
   auto mixedSizes = memref::getMixedSizes(rewriter, loc, sharedMemref);
   SmallVector<AffineExpr> symbols(mixedSizes.size());
@@ -971,7 +990,8 @@ void HopperBuilder::buildBarrierArriveTx(
       affine::makeComposedFoldedAffineApply(rewriter, loc, sumExpr, mixedSizes);
   Value sizeVal = getValueOrCreateConstantIndexOp(rewriter, loc, size);
   Value zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
-  rewriter.create<nvgpu::MBarrierArriveExpectTxOp>(loc, barrier, sizeVal, zero);
+  rewriter.create<nvgpu::MBarrierArriveExpectTxOp>(loc, barrier, sizeVal, zero,
+                                                   Value());
 }
 
 void HopperBuilder::buildTryWaitParity(
diff --git a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
index b2ac4c2327..b7e2aec6a4 100644
--- a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
+++ b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
@@ -452,7 +452,7 @@ LogicalResult acc::PrivateRecipeOp::verifyRegions() {
 LogicalResult acc::FirstprivateRecipeOp::verifyRegions() {
   if (failed(verifyInitLikeSingleArgRegion(*this, getInitRegion(),
                                            "privatization", "init", getType(),
-                                           /*verifyYield=*/true)))
+                                           /*verifyYield=*/false)))
     return failure();
 
   if (getCopyRegion().empty())
@@ -620,7 +620,7 @@ Value ParallelOp::getDataOperand(unsigned i) {
 LogicalResult acc::ParallelOp::verify() {
   if (failed(checkSymOperandList<mlir::acc::PrivateRecipeOp>(
           *this, getPrivatizations(), getGangPrivateOperands(), "private",
-          "privatizations", false)))
+          "privatizations", /*checkOperandType=*/false)))
     return failure();
   if (failed(checkSymOperandList<mlir::acc::ReductionRecipeOp>(
           *this, getReductionRecipes(), getReductionOperands(), "reduction",
@@ -650,7 +650,7 @@ Value SerialOp::getDataOperand(unsigned i) {
 LogicalResult acc::SerialOp::verify() {
   if (failed(checkSymOperandList<mlir::acc::PrivateRecipeOp>(
           *this, getPrivatizations(), getGangPrivateOperands(), "private",
-          "privatizations")))
+          "privatizations", /*checkOperandType=*/false)))
     return failure();
   if (failed(checkSymOperandList<mlir::acc::ReductionRecipeOp>(
           *this, getReductionRecipes(), getReductionOperands(), "reduction",
diff --git a/mlir/lib/Dialect/SCF/IR/SCF.cpp b/mlir/lib/Dialect/SCF/IR/SCF.cpp
index 8d8481421e..20a7b283c9 100644
--- a/mlir/lib/Dialect/SCF/IR/SCF.cpp
+++ b/mlir/lib/Dialect/SCF/IR/SCF.cpp
@@ -400,7 +400,7 @@ LogicalResult ForOp::promoteIfSingleIteration(RewriterBase &rewriter) {
 
   // Replace all results with the yielded values.
   auto yieldOp = cast<scf::YieldOp>(getBody()->getTerminator());
-  rewriter.replaceAllUsesWith(getResults(), yieldOp.getOperands());
+  rewriter.replaceAllUsesWith(getResults(), getYieldedValues());
 
   // Replace block arguments with lower bound (replacement for IV) and
   // iter_args.
@@ -772,27 +772,26 @@ struct ForOpIterArgsFolder : public OpRewritePattern<scf::ForOp> {
   LogicalResult matchAndRewrite(scf::ForOp forOp,
                                 PatternRewriter &rewriter) const final {
     bool canonicalize = false;
-    Block &block = forOp.getRegion().front();
-    auto yieldOp = cast<scf::YieldOp>(block.getTerminator());
 
     // An internal flat vector of block transfer
     // arguments `newBlockTransferArgs` keeps the 1-1 mapping of original to
     // transformed block argument mappings. This plays the role of a
     // IRMapping for the particular use case of calling into
     // `inlineBlockBefore`.
+    int64_t numResults = forOp.getNumResults();
     SmallVector<bool, 4> keepMask;
-    keepMask.reserve(yieldOp.getNumOperands());
+    keepMask.reserve(numResults);
     SmallVector<Value, 4> newBlockTransferArgs, newIterArgs, newYieldValues,
         newResultValues;
-    newBlockTransferArgs.reserve(1 + forOp.getInitArgs().size());
+    newBlockTransferArgs.reserve(1 + numResults);
     newBlockTransferArgs.push_back(Value()); // iv placeholder with null value
     newIterArgs.reserve(forOp.getInitArgs().size());
-    newYieldValues.reserve(yieldOp.getNumOperands());
-    newResultValues.reserve(forOp.getNumResults());
+    newYieldValues.reserve(numResults);
+    newResultValues.reserve(numResults);
     for (auto it : llvm::zip(forOp.getInitArgs(),       // iter from outside
                              forOp.getRegionIterArgs(), // iter inside region
                              forOp.getResults(),        // op results
-                             yieldOp.getOperands()      // iter yield
+                             forOp.getYieldedValues()   // iter yield
                              )) {
       // Forwarded is `true` when:
       // 1) The region `iter` argument is yielded.
@@ -946,12 +945,10 @@ struct SimplifyTrivialLoops : public OpRewritePattern<ForOp> {
       return failure();
     // If the loop is empty, iterates at least once, and only returns values
     // defined outside of the loop, remove it and replace it with yield values.
-    auto yieldOp = cast<scf::YieldOp>(block.getTerminator());
-    auto yieldOperands = yieldOp.getOperands();
-    if (llvm::any_of(yieldOperands,
+    if (llvm::any_of(op.getYieldedValues(),
                      [&](Value v) { return !op.isDefinedOutsideOfLoop(v); }))
       return failure();
-    rewriter.replaceOp(op, yieldOperands);
+    rewriter.replaceOp(op, op.getYieldedValues());
     return success();
   }
 };
@@ -1224,6 +1221,10 @@ std::optional<APInt> ForOp::getConstantStep() {
   return {};
 }
 
+ValueRange ForOp::getYieldedValues() {
+  return cast<scf::YieldOp>(getBody()->getTerminator()).getResults();
+}
+
 Speculation::Speculatability ForOp::getSpeculatability() {
   // `scf.for (I = Start; I < End; I += 1)` terminates for all values of Start
   // and End.
@@ -1526,6 +1527,30 @@ InParallelOp ForallOp::getTerminator() {
   return cast<InParallelOp>(getBody()->getTerminator());
 }
 
+std::optional<Value> ForallOp::getSingleInductionVar() {
+  if (getRank() != 1)
+    return std::nullopt;
+  return getInductionVar(0);
+}
+
+std::optional<OpFoldResult> ForallOp::getSingleLowerBound() {
+  if (getRank() != 1)
+    return std::nullopt;
+  return getMixedLowerBound()[0];
+}
+
+std::optional<OpFoldResult> ForallOp::getSingleUpperBound() {
+  if (getRank() != 1)
+    return std::nullopt;
+  return getMixedUpperBound()[0];
+}
+
+std::optional<OpFoldResult> ForallOp::getSingleStep() {
+  if (getRank() != 1)
+    return std::nullopt;
+  return getMixedStep()[0];
+}
+
 ForallOp mlir::scf::getForallOpThreadIndexOwner(Value val) {
   auto tidxArg = llvm::dyn_cast<BlockArgument>(val);
   if (!tidxArg)
@@ -2911,6 +2936,30 @@ void ParallelOp::print(OpAsmPrinter &p) {
 
 SmallVector<Region *> ParallelOp::getLoopRegions() { return {&getRegion()}; }
 
+std::optional<Value> ParallelOp::getSingleInductionVar() {
+  if (getNumLoops() != 1)
+    return std::nullopt;
+  return getBody()->getArgument(0);
+}
+
+std::optional<OpFoldResult> ParallelOp::getSingleLowerBound() {
+  if (getNumLoops() != 1)
+    return std::nullopt;
+  return getLowerBound()[0];
+}
+
+std::optional<OpFoldResult> ParallelOp::getSingleUpperBound() {
+  if (getNumLoops() != 1)
+    return std::nullopt;
+  return getUpperBound()[0];
+}
+
+std::optional<OpFoldResult> ParallelOp::getSingleStep() {
+  if (getNumLoops() != 1)
+    return std::nullopt;
+  return getStep()[0];
+}
+
 ParallelOp mlir::scf::getParallelForInductionVarOwner(Value val) {
   auto ivArg = llvm::dyn_cast<BlockArgument>(val);
   if (!ivArg)
@@ -3205,6 +3254,8 @@ YieldOp WhileOp::getYieldOp() {
   return cast<YieldOp>(getAfterBody()->getTerminator());
 }
 
+ValueRange WhileOp::getYieldedValues() { return getYieldOp().getResults(); }
+
 Block::BlockArgListType WhileOp::getBeforeArguments() {
   return getBeforeBody()->getArguments();
 }
diff --git a/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
index 0d02a590f2..455b7d8bca 100644
--- a/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -627,9 +627,8 @@ struct ForOpInterface
     auto forOp = cast<scf::ForOp>(op);
     OpOperand &forOperand = forOp.getOpOperandForResult(opResult);
     auto bbArg = forOp.getRegionIterArgForOpOperand(forOperand);
-    auto yieldOp = cast<scf::YieldOp>(forOp.getBody()->getTerminator());
     bool equivalentYield = state.areEquivalentBufferizedValues(
-        bbArg, yieldOp->getOperand(opResult.getResultNumber()));
+        bbArg, forOp.getYieldedValues()[opResult.getResultNumber()]);
     return equivalentYield ? BufferRelation::Equivalent
                            : BufferRelation::Unknown;
   }
diff --git a/mlir/lib/Dialect/SCF/Transforms/LoopCanonicalization.cpp b/mlir/lib/Dialect/SCF/Transforms/LoopCanonicalization.cpp
index 0cd19fbefa..43e79d309c 100644
--- a/mlir/lib/Dialect/SCF/Transforms/LoopCanonicalization.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/LoopCanonicalization.cpp
@@ -36,10 +36,9 @@ using namespace mlir::scf;
 /// type of the corresponding basic block argument of the loop.
 /// Note: This function handles only simple cases. Expand as needed.
 static bool isShapePreserving(ForOp forOp, int64_t arg) {
-  auto yieldOp = cast<YieldOp>(forOp.getBody()->getTerminator());
-  assert(arg < static_cast<int64_t>(yieldOp.getResults().size()) &&
+  assert(arg < static_cast<int64_t>(forOp.getNumResults()) &&
          "arg is out of bounds");
-  Value value = yieldOp.getResults()[arg];
+  Value value = forOp.getYieldedValues()[arg];
   while (value) {
     if (value == forOp.getRegionIterArgs()[arg])
       return true;
diff --git a/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp b/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp
index 96d6169111..2bddb498c2 100644
--- a/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp
@@ -101,10 +101,10 @@ static bool tileDividesIterationDomain(Range loopRange) {
 /// `tileSize`, i.e., `min(tileSize, range.end() - iv)`.
 static OpFoldResult getBoundedTileSize(OpBuilder &b, Location loc,
                                        Range loopRange, Value iv,
-                                       Value tileSize) {
+                                       OpFoldResult tileSize) {
   std::optional<int64_t> ts = getConstantIntValue(tileSize);
   if (ts && ts.value() == 1)
-    return getAsOpFoldResult(tileSize);
+    return tileSize;
 
   if (tileDividesIterationDomain(
           Range{loopRange.offset, loopRange.size, tileSize}))
@@ -122,6 +122,19 @@ static OpFoldResult getBoundedTileSize(OpBuilder &b, Location loc,
       b, loc, minMap, SmallVector<OpFoldResult>{iv, tileSize, size});
 }
 
+/// Clones the operation and updates the destination if the operation
+/// implements the `DestinationStyleOpInterface`.
+static Operation *cloneOpAndUpdateDestinationArgs(RewriterBase &rewriter,
+                                                  Operation *op,
+                                                  ValueRange newDestArgs) {
+  Operation *clonedOp = rewriter.clone(*op);
+  if (auto destinationStyleOp =
+          dyn_cast<DestinationStyleOpInterface>(clonedOp)) {
+    destinationStyleOp.getDpsInitsMutable().assign(newDestArgs);
+  }
+  return clonedOp;
+}
+
 /// Generate an empty loop nest that represents the tiled loop nest shell.
 /// - `loopRanges` specifies the lb, ub and step of the untiled iteration space.
 /// - `tileSizes` is the tile sizes to use. Zero represent untiled loops.
@@ -159,8 +172,9 @@ static SmallVector<scf::ForOp> generateTileLoopNest(
         loc, offset, size, tileSize, ValueRange{},
         [&](OpBuilder &bodyBuilder, Location bodyLoc, Value iv,
             ValueRange /*iterArgs*/) {
-          sizes[loopRange.index()] = getBoundedTileSize(
-              bodyBuilder, bodyLoc, loopRange.value(), iv, tileSize);
+          sizes[loopRange.index()] =
+              getBoundedTileSize(bodyBuilder, bodyLoc, loopRange.value(), iv,
+                                 getAsOpFoldResult(tileSize));
           builder.create<scf::YieldOp>(loc);
         });
     offsets[loopRange.index()] = loop.getInductionVar();
@@ -728,6 +742,118 @@ mlir::scf::tileConsumerAndFuseProducerGreedilyUsingSCFForOp(
                                    getAsOperations(forLoops), replacements};
 }
 
+//===----------------------------------------------------------------------===//
+// tileUsingSCFForAllOp implementation.
+//===----------------------------------------------------------------------===//
+
+FailureOr<scf::SCFTilingResult>
+mlir::scf::tileUsingSCFForallOp(RewriterBase &rewriter, TilingInterface op,
+                                const scf::SCFTilingOptions &options) {
+  Location loc = op->getLoc();
+  OpBuilder::InsertionGuard g(rewriter);
+
+  // 1. Get the range of loops that are represented by the operation.
+  SmallVector<Range> loopRanges = op.getIterationDomain(rewriter);
+  if (loopRanges.empty())
+    return op->emitOpError("expected non-empty loop ranges");
+  auto hasStrideOne = [](Range r) { return !isConstantIntValue(r.stride, 1); };
+  if (llvm::any_of(loopRanges, hasStrideOne))
+    return op->emitOpError("only stride-1 supported atm");
+
+  // 2. Get the tile sizes. If tile size is 0, it is not tiled and distributed.
+  // To make it easier, pad the tile sizes to loopRanges.size with value 0.
+  SmallVector<OpFoldResult> tileSizeVector =
+      options.tileSizeComputationFunction(rewriter, op);
+  tileSizeVector.resize(loopRanges.size(), rewriter.getIndexAttr(0));
+
+  // 3. Build the offsets, sizes and steps for the tile and distributed loops.
+  SmallVector<OpFoldResult> lbs, ubs, steps;
+  for (auto [tileSize, loopRange] : llvm::zip(tileSizeVector, loopRanges)) {
+    if (isConstantIntValue(tileSize, 0))
+      continue;
+    lbs.push_back(loopRange.offset);
+    ubs.push_back(loopRange.size);
+    steps.push_back(tileSize);
+  }
+
+  // 4. Gather destination tensors.
+  SmallVector<Value> dest;
+  if (failed(tensor::getOrCreateDestinations(rewriter, loc, op, dest)))
+    return op->emitOpError("failed to get destination tensors");
+
+  // 5. Build the device mapping attribute.
+  std::optional<ArrayAttr> mappingAttr;
+  if (!options.mappingVector.empty()) {
+    mappingAttr = rewriter.getArrayAttr(ArrayRef(options.mappingVector));
+  }
+
+  // 6. Create the ForallOp. We don't use the lambda body-builder
+  // version because we require the use of RewriterBase in the body, so we
+  // manually move the insertion point to the body below.
+  auto forallOp =
+      rewriter.create<scf::ForallOp>(loc, lbs, ubs, steps, dest, mappingAttr);
+
+  // 7. Get the tile offset and sizes.
+  rewriter.setInsertionPoint(forallOp.getTerminator());
+  SmallVector<OpFoldResult> tiledOffsets, tiledSizes;
+  ValueRange ivs = forallOp.getInductionVars();
+  {
+    int materializedLoopNum = 0;
+    for (auto [tileSize, loopRange] : llvm::zip(tileSizeVector, loopRanges)) {
+      if (isConstantIntValue(tileSize, 0)) {
+        tiledOffsets.push_back(loopRange.offset);
+        tiledSizes.push_back(loopRange.size);
+        continue;
+      }
+      Value iv = ivs[materializedLoopNum++];
+      tiledOffsets.push_back(iv);
+      tiledSizes.push_back(
+          getBoundedTileSize(rewriter, loc, loopRange, iv, tileSize));
+    }
+  }
+
+  // 8. Tile the operation. Clone the operation to allow fix up of destination
+  // operands.
+  ArrayRef<BlockArgument> destBbArgs = forallOp.getOutputBlockArguments();
+  Operation *clonedOp =
+      cloneOpAndUpdateDestinationArgs(rewriter, op, destBbArgs);
+  FailureOr<TilingResult> tilingResult =
+      cast<TilingInterface>(clonedOp).getTiledImplementation(
+          rewriter, tiledOffsets, tiledSizes);
+  if (failed(tilingResult))
+    return clonedOp->emitError("failed to tile op: ");
+  rewriter.eraseOp(clonedOp);
+
+  // 9. Parallel insert back into the result tensor.
+  for (auto [index, tiledValue, destBBArg] :
+       llvm::enumerate(tilingResult->tiledValues, destBbArgs)) {
+    // 9.a. Partial subset information is inserted just before the terminator.
+    rewriter.setInsertionPoint(forallOp.getTerminator());
+
+    SmallVector<OpFoldResult> resultOffsets, resultSizes;
+    if (failed(op.getResultTilePosition(rewriter, index, tiledOffsets,
+                                        tiledSizes, resultOffsets,
+                                        resultSizes))) {
+      return op->emitOpError("output offsets couldn't be calculated");
+    }
+
+    SmallVector<OpFoldResult> strides(resultSizes.size(),
+                                      rewriter.getIndexAttr(1));
+    // 9.b. Parallel insertions are inserted at the end of the combining
+    // terminator.
+    rewriter.setInsertionPointToEnd(forallOp.getTerminator().getBody());
+    rewriter.create<tensor::ParallelInsertSliceOp>(
+        loc, tiledValue, destBBArg, resultOffsets, resultSizes, strides);
+  }
+
+  // 10. Return the tiling result.
+  return scf::SCFTilingResult{
+      tilingResult->tiledOps,
+      {forallOp.getOperation()},
+      llvm::map_to_vector(forallOp.getResults(),
+                          [](auto val) -> Value { return val; })};
+}
+
 //===----------------------------------------------------------------------===//
 // lowerToLoopsUsingSCFForOp implementation.
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/SCF/Utils/Utils.cpp b/mlir/lib/Dialect/SCF/Utils/Utils.cpp
index 5360c493f8..e85825595e 100644
--- a/mlir/lib/Dialect/SCF/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/SCF/Utils/Utils.cpp
@@ -391,7 +391,6 @@ LogicalResult mlir::loopUnrollByFactor(
 
     int64_t tripCountEvenMultiple = tripCount - (tripCount % unrollFactor);
     int64_t upperBoundUnrolledCst = lbCst + tripCountEvenMultiple * stepCst;
-    assert(upperBoundUnrolledCst <= ubCst);
     int64_t stepUnrolledCst = stepCst * unrollFactor;
 
     // Create constant for 'upperBoundUnrolled' and set epilogue loop flag.
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVWebGPUTransforms.cpp b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVWebGPUTransforms.cpp
index 44fea86785..21de1c9e86 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVWebGPUTransforms.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVWebGPUTransforms.cpp
@@ -167,6 +167,42 @@ using ExpandSMulExtendedPattern =
 using ExpandUMulExtendedPattern =
     ExpandMulExtendedPattern<UMulExtendedOp, false>;
 
+struct ExpandAddCarryPattern final : OpRewritePattern<IAddCarryOp> {
+  using OpRewritePattern<IAddCarryOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(IAddCarryOp op,
+                                PatternRewriter &rewriter) const override {
+    Location loc = op->getLoc();
+    Value lhs = op.getOperand1();
+    Value rhs = op.getOperand2();
+
+    // Currently, WGSL only supports 32-bit integer types. Any other integer
+    // types should already have been promoted/demoted to i32.
+    Type argTy = lhs.getType();
+    auto elemTy = cast<IntegerType>(getElementTypeOrSelf(argTy));
+    if (elemTy.getIntOrFloatBitWidth() != 32)
+      return rewriter.notifyMatchFailure(
+          loc,
+          llvm::formatv("Unexpected integer type for WebGPU: '{0}'", elemTy));
+
+    Value one =
+        rewriter.create<ConstantOp>(loc, argTy, getScalarOrSplatAttr(argTy, 1));
+    Value zero =
+        rewriter.create<ConstantOp>(loc, argTy, getScalarOrSplatAttr(argTy, 0));
+
+    // Calculate the carry by checking if the addition resulted in an overflow.
+    Value out = rewriter.create<IAddOp>(loc, lhs, rhs);
+    Value cmp = rewriter.create<ULessThanOp>(loc, out, lhs);
+    Value carry = rewriter.create<SelectOp>(loc, cmp, one, zero);
+
+    Value add = rewriter.create<CompositeConstructOp>(
+        loc, op->getResultTypes().front(), llvm::ArrayRef({out, carry}));
+
+    rewriter.replaceOp(op, add);
+    return success();
+  }
+};
+
 //===----------------------------------------------------------------------===//
 // Passes
 //===----------------------------------------------------------------------===//
@@ -191,8 +227,12 @@ void populateSPIRVExpandExtendedMultiplicationPatterns(
     RewritePatternSet &patterns) {
   // WGSL currently does not support extended multiplication ops, see:
   // https://github.com/gpuweb/gpuweb/issues/1565.
-  patterns.add<ExpandSMulExtendedPattern, ExpandUMulExtendedPattern>(
-      patterns.getContext());
+  patterns.add<
+      // clang-format off
+    ExpandSMulExtendedPattern,
+    ExpandUMulExtendedPattern,
+    ExpandAddCarryPattern
+  >(patterns.getContext());
 }
 } // namespace spirv
 } // namespace mlir
diff --git a/mlir/lib/Dialect/SparseTensor/IR/CMakeLists.txt b/mlir/lib/Dialect/SparseTensor/IR/CMakeLists.txt
index b22194d450..dd6f1037f7 100644
--- a/mlir/lib/Dialect/SparseTensor/IR/CMakeLists.txt
+++ b/mlir/lib/Dialect/SparseTensor/IR/CMakeLists.txt
@@ -29,6 +29,7 @@ endif()
 
 add_mlir_dialect_library(MLIRSparseTensorDialect
   SparseTensorDialect.cpp
+  SparseTensorInterfaces.cpp
   Detail/Var.cpp
   Detail/DimLvlMap.cpp
   Detail/LvlTypeParser.cpp
diff --git a/mlir/lib/Dialect/SparseTensor/IR/Detail/DimLvlMap.cpp b/mlir/lib/Dialect/SparseTensor/IR/Detail/DimLvlMap.cpp
index 05fce96043..5f947b67c6 100644
--- a/mlir/lib/Dialect/SparseTensor/IR/Detail/DimLvlMap.cpp
+++ b/mlir/lib/Dialect/SparseTensor/IR/Detail/DimLvlMap.cpp
@@ -356,10 +356,15 @@ AffineMap DimLvlMap::getDimToLvlMap(MLIRContext *context) const {
 AffineMap DimLvlMap::getLvlToDimMap(MLIRContext *context) const {
   SmallVector<AffineExpr> dimAffines;
   dimAffines.reserve(getDimRank());
-  for (const auto &dimSpec : dimSpecs)
-    dimAffines.push_back(dimSpec.getExpr().getAffineExpr());
+  for (const auto &dimSpec : dimSpecs) {
+    auto expr = dimSpec.getExpr().getAffineExpr();
+    if (expr) {
+      dimAffines.push_back(expr);
+    }
+  }
   auto map = AffineMap::get(getLvlRank(), getSymRank(), dimAffines, context);
-  if (map.isIdentity()) return AffineMap();
+  if (dimAffines.empty() || map.isIdentity())
+    return AffineMap();
   return map;
 }
 
diff --git a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
index 96ed5f13b9..56214c2b41 100644
--- a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
+++ b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
@@ -293,9 +293,8 @@ Type SparseTensorEncodingAttr::getCrdType() const {
 SparseTensorEncodingAttr
 SparseTensorEncodingAttr::withDimToLvl(AffineMap dimToLvl) const {
   assert(getImpl() && "Uninitialized SparseTensorEncodingAttr");
-  // TODO: infer lvlToDim
   return SparseTensorEncodingAttr::get(getContext(), getLvlTypes(), dimToLvl,
-                                       /*lvlToDim*/ AffineMap(), getPosWidth(),
+                                       getLvlToDim(), getPosWidth(),
                                        getCrdWidth());
 }
 
@@ -336,6 +335,10 @@ bool SparseTensorEncodingAttr::isAllDense() const {
   return !getImpl() || llvm::all_of(getLvlTypes(), isDenseDLT);
 }
 
+bool SparseTensorEncodingAttr::isCOO() const {
+  return getImpl() && isCOOType(*this, 0, true);
+}
+
 bool SparseTensorEncodingAttr::isAllOrdered() const {
   return !getImpl() || llvm::all_of(getLvlTypes(), isOrderedDLT);
 }
@@ -412,26 +415,18 @@ SparseTensorEncodingAttr::getStaticLvlSliceStride(Level lvl) const {
   return getStaticDimSliceStride(toOrigDim(*this, lvl));
 }
 
-const static DimLevelType validDLTs[] = {DimLevelType::Dense,
-                                         DimLevelType::TwoOutOfFour,
-                                         DimLevelType::Compressed,
-                                         DimLevelType::CompressedNu,
-                                         DimLevelType::CompressedNo,
-                                         DimLevelType::CompressedNuNo,
-                                         DimLevelType::Singleton,
-                                         DimLevelType::SingletonNu,
-                                         DimLevelType::SingletonNo,
-                                         DimLevelType::SingletonNuNo,
-                                         DimLevelType::LooseCompressed,
-                                         DimLevelType::LooseCompressedNu,
-                                         DimLevelType::LooseCompressedNo,
-                                         DimLevelType::LooseCompressedNuNo};
-
-static std::optional<DimLevelType> parseDLT(StringRef str) {
-  for (DimLevelType dlt : validDLTs)
-    if (str == toMLIRString(dlt))
-      return dlt;
-  return std::nullopt;
+ValueRange
+SparseTensorEncodingAttr::translateCrds(OpBuilder &builder, Location loc,
+                                        ValueRange crds,
+                                        CrdTransDirectionKind dir) const {
+  if (!getImpl())
+    return crds;
+
+  SmallVector<Type> retType(
+      dir == CrdTransDirectionKind::lvl2dim ? getDimRank() : getLvlRank(),
+      builder.getIndexType());
+  auto transOp = builder.create<CrdTranslateOp>(loc, retType, crds, dir, *this);
+  return transOp.getOutCrds();
 }
 
 Attribute SparseTensorEncodingAttr::parse(AsmParser &parser, Type type) {
@@ -452,11 +447,11 @@ Attribute SparseTensorEncodingAttr::parse(AsmParser &parser, Type type) {
   SmallVector<DimLevelType> lvlTypes;
   SmallVector<SparseTensorDimSliceAttr> dimSlices;
   AffineMap dimToLvl = {};
+  AffineMap lvlToDim = {};
   unsigned posWidth = 0;
   unsigned crdWidth = 0;
   StringRef attrName;
-  SmallVector<StringRef, 6> keys = {"lvlTypes", "dimToLvl",  "posWidth",
-                                    "crdWidth", "dimSlices", "map"};
+  SmallVector<StringRef, 3> keys = {"map", "posWidth", "crdWidth"};
   while (succeeded(parser.parseOptionalKeyword(&attrName))) {
     // Detect admissible keyword.
     auto *it = find(keys, attrName);
@@ -469,81 +464,16 @@ Attribute SparseTensorEncodingAttr::parse(AsmParser &parser, Type type) {
     RETURN_ON_FAIL(parser.parseEqual())
     // Dispatch on keyword.
     switch (keyWordIndex) {
-    case 0: { // lvlTypes
-      Attribute attr;
-      RETURN_ON_FAIL(parser.parseAttribute(attr));
-      auto arrayAttr = llvm::dyn_cast<ArrayAttr>(attr);
-      ERROR_IF(!arrayAttr, "expected an array for lvlTypes")
-      for (auto i : arrayAttr) {
-        auto strAttr = llvm::dyn_cast<StringAttr>(i);
-        ERROR_IF(!strAttr, "expected a string value in lvlTypes")
-        auto strVal = strAttr.getValue();
-        if (auto optDLT = parseDLT(strVal)) {
-          lvlTypes.push_back(optDLT.value());
-        } else {
-          parser.emitError(parser.getNameLoc(), "unexpected level-type: ")
-              << strVal;
-          return {};
-        }
-      }
-      break;
-    }
-    case 1: { // dimToLvl
-      Attribute attr;
-      RETURN_ON_FAIL(parser.parseAttribute(attr))
-      auto affineAttr = llvm::dyn_cast<AffineMapAttr>(attr);
-      ERROR_IF(!affineAttr, "expected an affine map for dimToLvl")
-      dimToLvl = affineAttr.getValue();
-      break;
-    }
-    case 2: { // posWidth
-      Attribute attr;
-      RETURN_ON_FAIL(parser.parseAttribute(attr))
-      auto intAttr = llvm::dyn_cast<IntegerAttr>(attr);
-      ERROR_IF(!intAttr, "expected an integral position bitwidth")
-      posWidth = intAttr.getInt();
-      break;
-    }
-    case 3: { // crdWidth
-      Attribute attr;
-      RETURN_ON_FAIL(parser.parseAttribute(attr))
-      auto intAttr = llvm::dyn_cast<IntegerAttr>(attr);
-      ERROR_IF(!intAttr, "expected an integral index bitwidth")
-      crdWidth = intAttr.getInt();
-      break;
-    }
-    case 4: { // dimSlices
-      RETURN_ON_FAIL(parser.parseLSquare())
-      // Dispatches to DimSliceAttr to skip mnemonic
-      bool finished = false;
-      while (auto attr = SparseTensorDimSliceAttr::parse(parser, nullptr)) {
-        auto sliceAttr = llvm::cast<SparseTensorDimSliceAttr>(attr);
-        dimSlices.push_back(sliceAttr);
-        if (parser.parseOptionalComma().failed()) {
-          finished = true;
-          break;
-        }
-      }
-      // Wrong when parsing slices
-      if (!finished)
-        return {};
-      RETURN_ON_FAIL(parser.parseRSquare())
-      break;
-    }
-    case 5: { // map (new STEA surface syntax)
+    case 0: { // map
       ir_detail::DimLvlMapParser cParser(parser);
       auto res = cParser.parseDimLvlMap();
       RETURN_ON_FAIL(res);
-      // TODO: use DimLvlMap directly as storage representation, rather
-      // than converting things over.
       const auto &dlm = *res;
 
-      ERROR_IF(!lvlTypes.empty(), "Cannot mix `lvlTypes` with `map`")
       const Level lvlRank = dlm.getLvlRank();
       for (Level lvl = 0; lvl < lvlRank; lvl++)
         lvlTypes.push_back(dlm.getLvlType(lvl));
 
-      ERROR_IF(!dimSlices.empty(), "Cannot mix `dimSlices` with `map`")
       const Dimension dimRank = dlm.getDimRank();
       for (Dimension dim = 0; dim < dimRank; dim++)
         dimSlices.push_back(dlm.getDimSlice(dim));
@@ -563,8 +493,24 @@ Attribute SparseTensorEncodingAttr::parse(AsmParser &parser, Type type) {
         dimSlices.clear();
       }
 
-      ERROR_IF(dimToLvl, "Cannot mix `dimToLvl` with `map`")
       dimToLvl = dlm.getDimToLvlMap(parser.getContext());
+      lvlToDim = dlm.getLvlToDimMap(parser.getContext());
+      break;
+    }
+    case 1: { // posWidth
+      Attribute attr;
+      RETURN_ON_FAIL(parser.parseAttribute(attr))
+      auto intAttr = llvm::dyn_cast<IntegerAttr>(attr);
+      ERROR_IF(!intAttr, "expected an integral position bitwidth")
+      posWidth = intAttr.getInt();
+      break;
+    }
+    case 2: { // crdWidth
+      Attribute attr;
+      RETURN_ON_FAIL(parser.parseAttribute(attr))
+      auto intAttr = llvm::dyn_cast<IntegerAttr>(attr);
+      ERROR_IF(!intAttr, "expected an integral index bitwidth")
+      crdWidth = intAttr.getInt();
       break;
     }
     } // switch
@@ -579,7 +525,9 @@ Attribute SparseTensorEncodingAttr::parse(AsmParser &parser, Type type) {
 #undef RETURN_ON_FAIL
 
   // Construct struct-like storage for attribute.
-  AffineMap lvlToDim; // TODO: infer
+  if (!lvlToDim || lvlToDim.isEmpty()) {
+    lvlToDim = inferLvlToDim(dimToLvl, parser.getContext());
+  }
   return parser.getChecked<SparseTensorEncodingAttr>(
       parser.getContext(), lvlTypes, dimToLvl, lvlToDim, posWidth, crdWidth,
       dimSlices);
@@ -659,6 +607,17 @@ SparseTensorEncodingAttr::verify(function_ref<InFlightDiagnostic()> emitError,
     return emitError() << "unexpected position bitwidth: " << posWidth;
   if (!acceptBitWidth(crdWidth))
     return emitError() << "unexpected coordinate bitwidth: " << crdWidth;
+  if (auto it = std::find_if(lvlTypes.begin(), lvlTypes.end(), isSingletonDLT);
+      it != std::end(lvlTypes)) {
+    if (it == lvlTypes.begin() ||
+        (!isCompressedDLT(*(it - 1)) && !isLooseCompressedDLT(*(it - 1))))
+      return emitError() << "expected compressed or loose_compressed level "
+                            "before singleton level";
+    if (!std::all_of(it, lvlTypes.end(),
+                     [](DimLevelType i) { return isSingletonDLT(i); }))
+      return emitError() << "expected all singleton lvlTypes "
+                            "following a singleton level";
+  }
   // Before we can check that the level-rank is consistent/coherent
   // across all fields, we need to define it.  The source-of-truth for
   // the `getLvlRank` method is the length of the level-types array,
@@ -674,19 +633,12 @@ SparseTensorEncodingAttr::verify(function_ref<InFlightDiagnostic()> emitError,
       return emitError()
              << "level-rank mismatch between dimToLvl and lvlTypes: "
              << dimToLvl.getNumResults() << " != " << lvlRank;
-    // TODO:  The following is attempting to match the old error-conditions
-    // from prior to merging dimOrdering and higherOrdering into dimToLvl.
-    // That is, we currently require `dimToLvl` to be either a permutation
-    // (as when higherOrdering is the identity) or expansive (as per the
-    // constraints on higherOrdering).  However, those constraints do
-    // not match the intended semantics of `dimToLvl`.  As we improve the
-    // compiler to actually handle non-permutations, we need to update these
-    // checks to match what is actually supported.  In particular, this is
-    // where we'll have to check that when `lvlToDim` is provided then it
-    // is indeed an inverse of `dimToLvl`, and when it isn't provided then
-    // it can be automatically inferred.
-    if (dimRank == lvlRank && !dimToLvl.isPermutation())
-      return emitError() << "expected a permutation affine map for dimToLvl";
+    auto inferRes = inferLvlToDim(dimToLvl, dimToLvl.getContext());
+    // Symbols can't be inferred but are acceptable.
+    if (!inferRes && dimToLvl.getNumSymbols() == 0)
+      return emitError() << "failed to infer lvlToDim from dimToLvl";
+    if (lvlToDim && (inferRes != lvlToDim))
+      return emitError() << "expected lvlToDim to be an inverse of dimToLvl";
     if (dimRank > lvlRank)
       return emitError() << "unexpected dimToLvl mapping from " << dimRank
                          << " to " << lvlRank;
@@ -745,6 +697,121 @@ mlir::sparse_tensor::getSparseTensorEncoding(Type type) {
   return nullptr;
 }
 
+AffineMap mlir::sparse_tensor::inferLvlToDim(AffineMap dimToLvl,
+                                             MLIRContext *context) {
+  auto map = static_cast<AffineMap>(dimToLvl);
+  AffineMap lvlToDim;
+  // Return an empty lvlToDim when inference is not successful.
+  if (!map || map.getNumSymbols() != 0) {
+    lvlToDim = AffineMap();
+  } else if (map.isPermutation()) {
+    lvlToDim = inversePermutation(map);
+  } else if (isBlockSparsity(map)) {
+    lvlToDim = inverseBlockSparsity(map, context);
+  }
+  return lvlToDim;
+}
+
+AffineMap mlir::sparse_tensor::inverseBlockSparsity(AffineMap dimToLvl,
+                                                    MLIRContext *context) {
+  SmallVector<AffineExpr> lvlExprs;
+  auto numLvls = dimToLvl.getNumResults();
+  lvlExprs.reserve(numLvls);
+  // lvlExprComponents stores information of the floordiv and mod operations
+  // applied to the same dimension, so as to build the lvlToDim map.
+  std::map<unsigned, SmallVector<AffineExpr, 3>> lvlExprComponents;
+  for (unsigned i = 0, n = numLvls; i < n; i++) {
+    auto result = dimToLvl.getResult(i);
+    if (auto binOp = result.dyn_cast<AffineBinaryOpExpr>()) {
+      if (result.getKind() == AffineExprKind::FloorDiv) {
+        // Position of the dimension in dimToLvl.
+        auto pos = binOp.getLHS().dyn_cast<AffineDimExpr>().getPosition();
+        assert(lvlExprComponents.find(pos) == lvlExprComponents.end() &&
+               "expected only one floordiv for each dimension");
+        SmallVector<AffineExpr, 3> components;
+        // Level variable for floordiv.
+        components.push_back(getAffineDimExpr(i, context));
+        // Multiplier.
+        components.push_back(binOp.getRHS());
+        // Map key is the position of the dimension.
+        lvlExprComponents[pos] = components;
+      } else if (result.getKind() == AffineExprKind::Mod) {
+        auto pos = binOp.getLHS().dyn_cast<AffineDimExpr>().getPosition();
+        assert(lvlExprComponents.find(pos) != lvlExprComponents.end() &&
+               "expected floordiv before mod");
+        // Add level variable for mod to the same vector
+        // of the corresponding floordiv.
+        lvlExprComponents[pos].push_back(getAffineDimExpr(i, context));
+      } else {
+        assert(false && "expected floordiv or mod");
+      }
+    } else {
+      lvlExprs.push_back(getAffineDimExpr(i, context));
+    }
+  }
+  // Build lvlExprs from lvlExprComponents.
+  // For example, for il = i floordiv 2 and ii = i mod 2, the components
+  // would be [il, 2, ii]. It could be used to build the AffineExpr
+  // i = il * 2 + ii in lvlToDim.
+  for (auto &components : lvlExprComponents) {
+    assert(components.second.size() == 3 &&
+           "expected 3 components to build lvlExprs");
+    auto mulOp = getAffineBinaryOpExpr(
+        AffineExprKind::Mul, components.second[0], components.second[1]);
+    auto addOp =
+        getAffineBinaryOpExpr(AffineExprKind::Add, mulOp, components.second[2]);
+    lvlExprs.push_back(addOp);
+  }
+  return dimToLvl.get(dimToLvl.getNumResults(), 0, lvlExprs, context);
+}
+
+SmallVector<unsigned> mlir::sparse_tensor::getBlockSize(AffineMap dimToLvl) {
+  assert(isBlockSparsity(dimToLvl) &&
+         "expected dimToLvl to be block sparsity for calling getBlockSize");
+  SmallVector<unsigned> blockSize;
+  for (auto result : dimToLvl.getResults()) {
+    if (auto binOp = result.dyn_cast<AffineBinaryOpExpr>()) {
+      if (result.getKind() == AffineExprKind::Mod) {
+        blockSize.push_back(
+            binOp.getRHS().dyn_cast<AffineConstantExpr>().getValue());
+      }
+    } else {
+      blockSize.push_back(0);
+    }
+  }
+  return blockSize;
+}
+
+bool mlir::sparse_tensor::isBlockSparsity(AffineMap dimToLvl) {
+  if (!dimToLvl)
+    return false;
+  std::map<unsigned, int64_t> coeffientMap;
+  for (auto result : dimToLvl.getResults()) {
+    if (auto binOp = result.dyn_cast<AffineBinaryOpExpr>()) {
+      auto pos = binOp.getLHS().dyn_cast<AffineDimExpr>().getPosition();
+      if (result.getKind() == AffineExprKind::FloorDiv) {
+        // Expect only one floordiv for each dimension.
+        if (coeffientMap.find(pos) != coeffientMap.end())
+          return false;
+        coeffientMap[pos] =
+            binOp.getRHS().dyn_cast<AffineConstantExpr>().getValue();
+      } else if (result.getKind() == AffineExprKind::Mod) {
+        // Expect floordiv before mod.
+        if (coeffientMap.find(pos) == coeffientMap.end())
+          return false;
+        // Expect mod to have the same coefficient as floordiv.
+        if (binOp.getRHS().dyn_cast<AffineConstantExpr>().getValue() !=
+            coeffientMap[pos]) {
+          return false;
+        }
+      } else {
+        return false;
+      }
+    }
+  }
+  return !coeffientMap.empty();
+}
+
 bool mlir::sparse_tensor::isCOOType(SparseTensorEncodingAttr enc,
                                     Level startLvl, bool isUnique) {
   if (!enc ||
@@ -780,11 +847,6 @@ RankedTensorType sparse_tensor::getCOOFromTypeWithOrdering(RankedTensorType rtt,
                                                            AffineMap lvlPerm,
                                                            bool ordered) {
   const SparseTensorType src(rtt);
-  // TODO: This assertion is to match the behavior from before we merged
-  // dimOrdering and higherOrdering into dimToLvl.  However, there's no
-  // in-principle reason to require this.  (wrengr has a commit in the
-  // wings to fix this.)
-  assert(src.isPermutation());
   const Level lvlRank = src.getLvlRank();
   SmallVector<DimLevelType> lvlTypes;
   lvlTypes.reserve(lvlRank);
@@ -807,7 +869,7 @@ RankedTensorType sparse_tensor::getCOOFromTypeWithOrdering(RankedTensorType rtt,
   // default value.
   unsigned posWidth = src.getPosWidth();
   unsigned crdWidth = src.getCrdWidth();
-  AffineMap invPerm; // TODO
+  AffineMap invPerm = src.getLvlToDim();
   auto enc = SparseTensorEncodingAttr::get(src.getContext(), lvlTypes, lvlPerm,
                                            invPerm, posWidth, crdWidth);
   return RankedTensorType::get(src.getDimShape(), src.getElementType(), enc);
@@ -1056,16 +1118,96 @@ LogicalResult ConvertOp::verify() {
 }
 
 OpFoldResult ConvertOp::fold(FoldAdaptor adaptor) {
-  Type dstType = getType();
-  // Fold trivial dense-to-dense convert and leave trivial sparse-to-sparse
-  // convert for codegen to remove. This is because we use trivial
-  // sparse-to-sparse convert to tell bufferization that the sparse codegen
-  // will expand the tensor buffer into sparse tensor storage.
-  if (!getSparseTensorEncoding(dstType) && dstType == getSource().getType())
+  if (getType() == getSource().getType())
     return getSource();
   return {};
 }
 
+bool ConvertOp::needsExtraSort() {
+  SparseTensorType srcStt = getSparseTensorType(getSource());
+  SparseTensorType dstStt = getSparseTensorType(getDest());
+
+  // We do not need an extra sort when returning unordered sparse tensors or
+  // dense tensor since dense tensor support random access.
+  if (dstStt.isAllDense() || !dstStt.isAllOrdered())
+    return false;
+
+  if (srcStt.isAllOrdered() && dstStt.isAllOrdered() &&
+      srcStt.hasSameDimToLvl(dstStt)) {
+    return false;
+  }
+
+  // Source and dest tensors are ordered in different ways. We only do direct
+  // dense to sparse conversion when the dense input is defined by a sparse
+  // constant. Note that we can theoretically always directly convert from dense
+  // inputs by rotating dense loops but it leads to bad cache locality and hurt
+  // performance.
+  if (auto constOp = getSource().getDefiningOp<arith::ConstantOp>())
+    if (isa<SparseElementsAttr>(constOp.getValue()))
+      return false;
+
+  return true;
+}
+
+LogicalResult CrdTranslateOp::verify() {
+  uint64_t inRank = getEncoder().getLvlRank();
+  uint64_t outRank = getEncoder().getDimRank();
+
+  if (getDirection() == CrdTransDirectionKind::dim2lvl)
+    std::swap(inRank, outRank);
+
+  if (inRank != getInCrds().size() || outRank != getOutCrds().size())
+    return emitError("Coordinate rank mismatch with encoding");
+
+  return success();
+}
+
+LogicalResult CrdTranslateOp::fold(FoldAdaptor adaptor,
+                                   SmallVectorImpl<OpFoldResult> &results) {
+  if (getEncoder().isIdentity()) {
+    results.assign(getInCrds().begin(), getInCrds().end());
+    return success();
+  }
+  if (getEncoder().isPermutation()) {
+    AffineMap perm = getDirection() == CrdTransDirectionKind::dim2lvl
+                         ? getEncoder().getDimToLvl()
+                         : getEncoder().getLvlToDim();
+    for (AffineExpr exp : perm.getResults())
+      results.push_back(getInCrds()[exp.cast<AffineDimExpr>().getPosition()]);
+    return success();
+  }
+
+  // Fuse dim2lvl/lvl2dim pairs.
+  auto def = getInCrds()[0].getDefiningOp<CrdTranslateOp>();
+  bool sameDef = def && llvm::all_of(getInCrds(), [def](Value v) {
+                   return v.getDefiningOp() == def;
+                 });
+  if (!sameDef)
+    return failure();
+
+  bool oppositeDir = def.getDirection() != getDirection();
+  bool sameOracle =
+      def.getEncoder().getDimToLvl() == getEncoder().getDimToLvl();
+  bool sameCount = def.getNumResults() == getInCrds().size();
+  if (!oppositeDir || !sameOracle || !sameCount)
+    return failure();
+
+  // The definition produces the coordinates in the same order as the input
+  // coordinates.
+  bool sameOrder = llvm::all_of(llvm::zip_equal(def.getOutCrds(), getInCrds()),
+                                [](auto valuePair) {
+                                  auto [lhs, rhs] = valuePair;
+                                  return lhs == rhs;
+                                });
+
+  if (!sameOrder)
+    return failure();
+  // l1 = dim2lvl (lvl2dim l0)
+  // ==> l0
+  results.append(def.getInCrds().begin(), def.getInCrds().end());
+  return success();
+}
+
 LogicalResult ToPositionsOp::verify() {
   auto e = getSparseTensorEncoding(getTensor().getType());
   if (failed(lvlIsInBounds(getLevel(), getTensor())))
@@ -1223,6 +1365,23 @@ LogicalResult UnaryOp::verify() {
   return success();
 }
 
+bool ConcatenateOp::needsExtraSort() {
+  SparseTensorType dstStt = getSparseTensorType(*this);
+  if (dstStt.isAllDense() || !dstStt.isAllOrdered())
+    return false;
+
+  bool allSameOrdered = llvm::all_of(getInputs(), [dstStt](Value op) {
+    return getSparseTensorType(op).hasSameDimToLvl(dstStt);
+  });
+  // TODO: When conDim != 0, as long as conDim corresponding to the first level
+  // in all input/output buffers, and all input/output buffers have the same
+  // dimToLvl, the tmp COO buffer is still unnecessary (e.g, concatenate
+  // CSC matrices along column).
+  bool directLowerable =
+      allSameOrdered && getDimension() == 0 && dstStt.isIdentity();
+  return !directLowerable;
+}
+
 LogicalResult ConcatenateOp::verify() {
   const auto dstTp = getSparseTensorType(*this);
   const Dimension concatDim = getDimension();
@@ -1379,6 +1538,29 @@ LogicalResult ForeachOp::verify() {
   return success();
 }
 
+OpFoldResult ReorderCOOOp::fold(FoldAdaptor adaptor) {
+  if (getSparseTensorEncoding(getInputCoo().getType()) ==
+      getSparseTensorEncoding(getResultCoo().getType()))
+    return getInputCoo();
+
+  return {};
+}
+
+LogicalResult ReorderCOOOp::verify() {
+  SparseTensorType srcStt = getSparseTensorType(getInputCoo());
+  SparseTensorType dstStt = getSparseTensorType(getResultCoo());
+
+  if (!srcStt.hasSameDimToLvl(dstStt))
+    emitError("Unmatched dim2lvl map between input and result COO");
+
+  if (srcStt.getPosType() != dstStt.getPosType() ||
+      srcStt.getCrdType() != dstStt.getCrdType() ||
+      srcStt.getElementType() != dstStt.getElementType()) {
+    emitError("Unmatched storage format between input and result COO");
+  }
+  return success();
+}
+
 LogicalResult ReduceOp::verify() {
   Type inputType = getX().getType();
   // Check correct number of block arguments and return type.
diff --git a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorInterfaces.cpp b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorInterfaces.cpp
new file mode 100644
index 0000000000..d8769eacc4
--- /dev/null
+++ b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorInterfaces.cpp
@@ -0,0 +1,55 @@
+//===- SparseTensorInterfaces.cpp - SparseTensor interfaces impl ----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/SparseTensor/IR/SparseTensorInterfaces.h"
+#include "mlir/Dialect/SparseTensor/IR/SparseTensor.h"
+#include "mlir/Dialect/SparseTensor/IR/SparseTensorType.h"
+#include "mlir/IR/PatternMatch.h"
+
+using namespace mlir;
+using namespace mlir::sparse_tensor;
+
+#include "mlir/Dialect/SparseTensor/IR/SparseTensorInterfaces.cpp.inc"
+
+LogicalResult
+sparse_tensor::detail::stageWithSortImpl(StageWithSortSparseOp op,
+                                         PatternRewriter &rewriter) {
+  if (!op.needsExtraSort())
+    return failure();
+
+  Location loc = op.getLoc();
+  Type finalTp = op->getOpResult(0).getType();
+  SparseTensorType dstStt(finalTp.cast<RankedTensorType>());
+
+  Type srcCOOTp = getCOOFromTypeWithOrdering(
+      dstStt.getRankedTensorType(), dstStt.getDimToLvl(), /*ordered=*/false);
+
+  // Clones the original operation but changing the output to an unordered COO.
+  Operation *cloned = rewriter.clone(*op.getOperation());
+  rewriter.updateRootInPlace(cloned, [cloned, srcCOOTp]() {
+    cloned->getOpResult(0).setType(srcCOOTp);
+  });
+  Value srcCOO = cloned->getOpResult(0);
+
+  // -> sort
+  Type dstCOOTp = getCOOFromTypeWithOrdering(
+      dstStt.getRankedTensorType(), dstStt.getDimToLvl(), /*ordered=*/true);
+  Value dstCOO = rewriter.create<ReorderCOOOp>(
+      loc, dstCOOTp, srcCOO, SparseTensorSortKind::HybridQuickSort);
+
+  // -> dest.
+  if (dstCOO.getType() == finalTp) {
+    rewriter.replaceOp(op, dstCOO);
+  } else {
+    // Need an extra conversion if the target type is not COO.
+    rewriter.replaceOpWithNewOp<ConvertOp>(op, finalTp, dstCOO);
+  }
+  // TODO: deallocate extra COOs, we should probably delegate it to buffer
+  // deallocation pass.
+  return success();
+}
diff --git a/mlir/lib/Dialect/SparseTensor/Pipelines/SparseTensorPipelines.cpp b/mlir/lib/Dialect/SparseTensor/Pipelines/SparseTensorPipelines.cpp
index 7569413546..3ed8bba251 100644
--- a/mlir/lib/Dialect/SparseTensor/Pipelines/SparseTensorPipelines.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Pipelines/SparseTensorPipelines.cpp
@@ -35,9 +35,9 @@ void mlir::sparse_tensor::buildSparseCompiler(
   pm.addPass(createSparsificationAndBufferizationPass(
       getBufferizationOptionsForSparsification(
           options.testBufferizationAnalysisOnly),
-      options.sparsificationOptions(), options.sparseTensorConversionOptions(),
-      options.createSparseDeallocs, options.enableRuntimeLibrary,
-      options.enableBufferInitialization, options.vectorLength,
+      options.sparsificationOptions(), options.createSparseDeallocs,
+      options.enableRuntimeLibrary, options.enableBufferInitialization,
+      options.vectorLength,
       /*enableVLAVectorization=*/options.armSVE,
       /*enableSIMDIndex32=*/options.force32BitVectorIndices));
   if (options.testBufferizationAnalysisOnly)
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/CodegenUtils.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/CodegenUtils.cpp
index dac6d6b645..b1b1d67ac2 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/CodegenUtils.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/CodegenUtils.cpp
@@ -23,54 +23,6 @@
 using namespace mlir;
 using namespace mlir::sparse_tensor;
 
-/// If the tensor is a sparse constant, generates and returns the pair of
-/// the constants for the coordinates and the values.
-static std::optional<std::pair<Value, Value>>
-genSplitSparseConstant(OpBuilder &builder, Location loc, Value tensor) {
-  if (auto constOp = tensor.getDefiningOp<arith::ConstantOp>()) {
-    if (auto a = dyn_cast<SparseElementsAttr>(constOp.getValue())) {
-      auto coordinates = builder.create<arith::ConstantOp>(loc, a.getIndices());
-      auto values = builder.create<arith::ConstantOp>(loc, a.getValues());
-      return std::make_pair(coordinates, values);
-    }
-  }
-  return {};
-}
-
-/// Reads `coordinates[k][0..rank-1]` and `value[k]`, appending the
-/// former onto `cvs` and returning the latter.
-// FIXME: Change the `rank` argument to `Dimension dimRank` or `Level lvlRank`,
-// to clarify its intended meaning.
-static Value genCoordsAndValueForSparse(OpBuilder &builder, Location loc,
-                                        Value coordinates, Value values,
-                                        SmallVectorImpl<Value> &cvs, Value k,
-                                        unsigned rank) {
-  for (unsigned d = 0; d < rank; d++) {
-    Value dim = constantIndex(builder, loc, d);
-    Value crd =
-        builder.create<tensor::ExtractOp>(loc, coordinates, ValueRange{k, dim});
-    crd = builder.create<arith::IndexCastOp>(loc, builder.getIndexType(), crd);
-    // builder.create<memref::StoreOp>(loc, crd, cvs, dim);
-    cvs.push_back(crd);
-  }
-  return builder.create<tensor::ExtractOp>(loc, values, k);
-}
-
-/// Generates code to read the value from `tensor[ivs]`, and open
-/// a conditional for whether the value is non-zero.  The generated code
-/// looks like the following and the insertion point after this routine
-/// is inside the then-branch.
-///    if (tensor[ivs] != 0)
-///      insert_point
-static Value genCoordsAndValueForDense(OpBuilder &builder, Location loc,
-                                       Value tensor,
-                                       SmallVectorImpl<Value> &cvs,
-                                       ValueRange ivs) {
-  Value val = genValueForDense(builder, loc, tensor, ivs);
-  cvs.append(ivs.begin(), ivs.end());
-  return val;
-}
-
 //===----------------------------------------------------------------------===//
 // ExecutionEngine/SparseTensorUtils helper functions.
 //===----------------------------------------------------------------------===//
@@ -450,65 +402,6 @@ void mlir::sparse_tensor::deallocDenseTensor(OpBuilder &builder, Location loc,
   builder.create<memref::DeallocOp>(loc, buffer);
 }
 
-Value mlir::sparse_tensor::genValueForDense(OpBuilder &builder, Location loc,
-                                            Value tensor, ValueRange ivs) {
-  Value val = builder.create<tensor::ExtractOp>(loc, tensor, ivs);
-  Value cond = genIsNonzero(builder, loc, val);
-  scf::IfOp ifOp = builder.create<scf::IfOp>(loc, cond, /*else*/ false);
-  builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
-  return val;
-}
-
-// FIXME:
-// 1. Dense tensors loop should be generated by loop emitter.
-// 2. Support reduction variables to propagate SSA chains properly.
-// 3. Change the `rank` argument to `Dimension dimRank` or `Level lvlRank`,
-//    to clarify its meaning.
-void mlir::sparse_tensor::genDenseTensorOrSparseConstantIterLoop(
-    OpBuilder &builder, Location loc, Value src, unsigned rank,
-    function_ref<void(OpBuilder &, Location, Value, ValueRange)> bodyBuilder) {
-  // `cvs` is actually the flattened coordinates array for all elements,
-  // not just for one element (since we do not `SmallVector::clear` after
-  // each iteration of the body of the loopnest.
-  SmallVector<Value> cvs;
-  SmallVector<Value> lo;
-  SmallVector<Value> hi;
-  SmallVector<Value> st;
-  const Value zero = constantIndex(builder, loc, 0);
-  const Value one = constantIndex(builder, loc, 1);
-  const auto splitSrc = genSplitSparseConstant(builder, loc, src);
-  if (splitSrc.has_value()) {
-    const Value srcCoordinates = splitSrc->first;
-    const Value srcValues = splitSrc->second;
-    lo.push_back(zero);
-    hi.push_back(linalg::createOrFoldDimOp(builder, loc, srcValues, 0));
-    st.push_back(one);
-    scf::buildLoopNest(builder, loc, lo, hi, st, {},
-                       [&](OpBuilder &builder, Location loc, ValueRange ivs,
-                           ValueRange /*args*/) -> scf::ValueVector {
-                         Value val = genCoordsAndValueForSparse(
-                             builder, loc, srcCoordinates, srcValues, cvs,
-                             ivs[0], rank);
-                         bodyBuilder(builder, loc, val, cvs);
-                         return {};
-                       });
-  } else {
-    for (unsigned i = 0; i < rank; i++) {
-      lo.push_back(zero);
-      hi.push_back(linalg::createOrFoldDimOp(builder, loc, src, i));
-      st.push_back(one);
-    }
-    scf::buildLoopNest(builder, loc, lo, hi, st, {},
-                       [&](OpBuilder &builder, Location loc, ValueRange ivs,
-                           ValueRange /*args*/) -> scf::ValueVector {
-                         Value val = genCoordsAndValueForDense(builder, loc,
-                                                               src, cvs, ivs);
-                         bodyBuilder(builder, loc, val, cvs);
-                         return {};
-                       });
-  }
-}
-
 void mlir::sparse_tensor::sizesFromSrc(OpBuilder &builder,
                                        SmallVectorImpl<Value> &sizes,
                                        Location loc, Value src) {
@@ -795,25 +688,92 @@ Value sparse_tensor::genMapBuffers(OpBuilder &builder, Location loc,
     return dimSizesBuffer;
   }
   // Otherwise, some code needs to be generated to set up the buffers.
-  // TODO: use the lvl2dim once available and deal with non-permutations!
+  // This code deals with permutations as well as non-permutations that
+  // arise from rank changing blocking.
   const auto dimToLvl = stt.getDimToLvl();
-  assert(dimToLvl.isPermutation());
-  SmallVector<Value> dim2lvlValues(dimRank);
-  SmallVector<Value> lvl2dimValues(lvlRank);
+  const auto lvlToDim = stt.getLvlToDim();
+  SmallVector<Value> dim2lvlValues(lvlRank); // for each lvl, expr in dim vars
+  SmallVector<Value> lvl2dimValues(dimRank); // for each dim, expr in lvl vars
   SmallVector<Value> lvlSizesValues(lvlRank);
+  // Generate dim2lvl.
+  assert(lvlRank == dimToLvl.getNumResults());
   for (Level l = 0; l < lvlRank; l++) {
-    // The `d`th source variable occurs in the `l`th result position.
-    Dimension d = dimToLvl.getDimPosition(l);
-    Value lvl = constantIndex(builder, loc, l);
-    Value dim = constantIndex(builder, loc, d);
-    dim2lvlValues[d] = lvl;
-    lvl2dimValues[l] = dim;
-    if (stt.isDynamicDim(d))
-      lvlSizesValues[l] =
-          builder.create<memref::LoadOp>(loc, dimSizesBuffer, dim);
-    else
-      lvlSizesValues[l] = dimShapesValues[d];
+    AffineExpr exp = dimToLvl.getResult(l);
+    // We expect:
+    //    (1) l = d
+    //    (2) l = d / c
+    //    (3) l = d % c
+    Dimension d = 0;
+    uint64_t cf = 0, cm = 0;
+    switch (exp.getKind()) {
+    case AffineExprKind::DimId: {
+      d = exp.cast<AffineDimExpr>().getPosition();
+      break;
+    }
+    case AffineExprKind::FloorDiv: {
+      auto floor = exp.cast<AffineBinaryOpExpr>();
+      d = floor.getLHS().cast<AffineDimExpr>().getPosition();
+      cf = floor.getRHS().cast<AffineConstantExpr>().getValue();
+      break;
+    }
+    case AffineExprKind::Mod: {
+      auto mod = exp.cast<AffineBinaryOpExpr>();
+      d = mod.getLHS().cast<AffineDimExpr>().getPosition();
+      cm = mod.getRHS().cast<AffineConstantExpr>().getValue();
+      break;
+    }
+    default:
+      llvm::report_fatal_error("unsupported dim2lvl in sparse tensor type");
+    }
+    dim2lvlValues[l] = constantIndex(builder, loc, encodeDim(d, cf, cm));
+    // Compute the level sizes.
+    //    (1) l = d        : size(d)
+    //    (2) l = d / c    : size(d) / c
+    //    (3) l = d % c    : c
+    Value lvlSz;
+    if (cm == 0) {
+      lvlSz = dimShapesValues[d];
+      if (stt.isDynamicDim(d))
+        lvlSz = builder.create<memref::LoadOp>(loc, dimSizesBuffer,
+                                               constantIndex(builder, loc, d));
+      if (cf != 0)
+        lvlSz = builder.create<arith::DivUIOp>(loc, lvlSz,
+                                               constantIndex(builder, loc, cf));
+    } else {
+      lvlSz = constantIndex(builder, loc, cm);
+    }
+    lvlSizesValues[l] = lvlSz;
+  }
+  // Generate lvl2dim.
+  assert(dimRank == lvlToDim.getNumResults());
+  for (Dimension d = 0; d < dimRank; d++) {
+    AffineExpr exp = lvlToDim.getResult(d);
+    // We expect:
+    //    (1) d = l
+    //    (2) d = l' * c + l
+    Level l = 0, ll = 0;
+    uint64_t c = 0;
+    switch (exp.getKind()) {
+    case AffineExprKind::DimId: {
+      l = exp.cast<AffineDimExpr>().getPosition();
+      break;
+    }
+    case AffineExprKind::Add: {
+      // Always mul on lhs, symbol/constant on rhs.
+      auto add = exp.cast<AffineBinaryOpExpr>();
+      assert(add.getLHS().getKind() == AffineExprKind::Mul);
+      auto mul = add.getLHS().cast<AffineBinaryOpExpr>();
+      ll = mul.getLHS().cast<AffineDimExpr>().getPosition();
+      c = mul.getRHS().cast<AffineConstantExpr>().getValue();
+      l = add.getRHS().cast<AffineDimExpr>().getPosition();
+      break;
+    }
+    default:
+      llvm::report_fatal_error("unsupported lvl2dim in sparse tensor type");
+    }
+    lvl2dimValues[d] = constantIndex(builder, loc, encodeLvl(l, c, ll));
   }
+  // Return buffers.
   dim2lvlBuffer = allocaBuffer(builder, loc, dim2lvlValues);
   lvl2dimBuffer = allocaBuffer(builder, loc, lvl2dimValues);
   return allocaBuffer(builder, loc, lvlSizesValues);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/CodegenUtils.h b/mlir/lib/Dialect/SparseTensor/Transforms/CodegenUtils.h
index 1562ea3f20..4673d24fc8 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/CodegenUtils.h
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/CodegenUtils.h
@@ -221,34 +221,6 @@ Value allocDenseTensor(OpBuilder &builder, Location loc,
 /// Generates code to deallocate a dense buffer.
 void deallocDenseTensor(OpBuilder &builder, Location loc, Value buffer);
 
-/// Generates code to read the value from `tensor[ivs]`. The generated code
-/// looks like the following and the insertion point after this routine is
-/// inside the then-branch.
-///    if (tensor[ivs] != 0)
-///      insert_point
-Value genValueForDense(OpBuilder &builder, Location loc, Value tensor,
-                       ValueRange ivs);
-
-/// Generates the loop structure to iterate over a dense tensor or a sparse
-/// tensor constant to support the lowering of dense-to-sparse convert operator.
-//
-// The loop to iterate a dense tensor:
-//   for i1 in dim1
-//    ..
-//     for ik in dimk
-//       val = a[i1,..,ik]
-//       if val != 0
-//         loop-body
-//
-// The loop to iterate a sparse tensor constant:
-//   for i in range(NNZ)
-//     val = values[i]
-//     [i1,..,ik] = coordinates[i]
-//     loop-body
-void genDenseTensorOrSparseConstantIterLoop(
-    OpBuilder &builder, Location loc, Value src, unsigned rank,
-    function_ref<void(OpBuilder &, Location, Value, ValueRange)> bodyBuilder);
-
 /// Populates given sizes array from dense tensor or sparse tensor constant.
 void sizesFromSrc(OpBuilder &builder, SmallVectorImpl<Value> &sizes,
                   Location loc, Value src);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/LoopEmitter.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/LoopEmitter.cpp
index 2d4f40eceb..de7ad3c91d 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/LoopEmitter.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/LoopEmitter.cpp
@@ -422,10 +422,19 @@ void LoopEmitter::initializeLoopEmit(
     // FIXME: the definition of `lvlRank` looks more like a dim-rank;
     // but the variable is used as a level everywhere below, which
     // suggests there may be some dim/lvl confusion going on here.
-    const Level lvlRank = rtp.getRank();
+    auto stt = getSparseTensorType(tensor);
+    const Level lvlRank = stt.getLvlRank();
     const auto shape = rtp.getShape();
     const auto enc = getSparseTensorEncoding(rtp);
     const Level cooStart = enc ? getCOOStart(enc) : lvlRank;
+
+    SmallVector<Value> dimSz;
+    for (Dimension d = 0; d < stt.getDimRank(); d++)
+      dimSz.push_back(linalg::createOrFoldDimOp(builder, loc, tensor, d));
+
+    ValueRange lvlSzs =
+        enc.translateCrds(builder, loc, dimSz, CrdTransDirectionKind::dim2lvl);
+
     // Scan all levels of current tensor.
     for (Level l = 0; l < lvlRank; l++) {
       // This should be called only once at beginning.
@@ -447,13 +456,8 @@ void LoopEmitter::initializeLoopEmit(
         assert(isDenseDLT(lvlTp));
       }
 
-      // FIXME: `toOrigDim` is deprecated.  For now this relies on the
-      // 1:1 mapping between levels and dimensions, since nowhere else
-      // in the code supports non-permutations yet either.
-      Value lvlSz = mlir::linalg::createOrFoldDimOp(builder, loc, tensor,
-                                                    toOrigDim(enc, l));
       // Find upper bound in current dimension.
-      highs[t][l] = lvlSizes[t][l] = lvlSz;
+      highs[t][l] = lvlSizes[t][l] = lvlSzs[l];
       if (isSparseSlices[t]) {
         sliceOffsets[t][l] = genSliceOffset(builder, loc, tensors[t], l);
         sliceStrides[t][l] = genSliceStride(builder, loc, tensors[t], l);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp
index a6e9631818..9c836c16da 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp
@@ -39,7 +39,7 @@ enum class CuSparseFormat {
   kCOO,
   kCSR,
   kCSC,
-  kBSR, // TODO: coming soon!
+  kBSR,
 };
 
 //===----------------------------------------------------------------------===//
@@ -428,6 +428,19 @@ static bool isAdmissibleCSC(SparseTensorType &aTp) {
          aTp.isOrderedLvl(1) && aTp.isUniqueLvl(1) && isAdmissibleMetaData(aTp);
 }
 
+/// Test for BSR matrix with suitable metadata.
+static bool isAdmissibleBSR(SparseTensorType &aTp) {
+  if (aTp.getDimRank() == 2 && aTp.getLvlRank() == 4 && aTp.isDenseLvl(0) &&
+      aTp.isCompressedLvl(1) && aTp.isOrderedLvl(1) && aTp.isUniqueLvl(1) &&
+      aTp.isDenseLvl(2) && aTp.isDenseLvl(3) && isAdmissibleMetaData(aTp)) {
+    // CuSparse only supports "square" blocks currently.
+    SmallVector<unsigned> dims = getBlockSize(aTp.getDimToLvl());
+    assert(dims.size() == 2);
+    return dims[0] = dims[1] && dims[0] > 1;
+  }
+  return false;
+}
+
 /// Returns a suitable sparse format for the operation and given operand
 /// types with cuSparse, or kNone if none is available.
 static CuSparseFormat getCuSparseFormat(SparseTensorType aTp,
@@ -448,6 +461,8 @@ static CuSparseFormat getCuSparseFormat(SparseTensorType aTp,
     return CuSparseFormat::kCSR;
   if (isAdmissibleCSC(aTp))
     return CuSparseFormat::kCSC;
+  if (isAdmissibleBSR(aTp))
+    return CuSparseFormat::kBSR;
   return CuSparseFormat::kNone;
 }
 
@@ -475,9 +490,10 @@ static Value genSecondCrds(OpBuilder &builder, Location loc, Value a,
 }
 
 /// Generates the sparse matrix handle.
-static Operation *genSpMat(OpBuilder &builder, Location loc, Type handleTp,
-                           Type tokenTp, Value token, Value sz1, Value sz2,
-                           Value nseA, Value rowA, Value colA, Value valA,
+static Operation *genSpMat(OpBuilder &builder, Location loc,
+                           SparseTensorType &aTp, Type handleTp, Type tokenTp,
+                           Value token, Value sz1, Value sz2, Value nseA,
+                           Value rowA, Value colA, Value valA,
                            CuSparseFormat format, bool enableRT) {
   if (format == CuSparseFormat::kCOO) {
     // Library uses SoA COO, direct IR uses AoS COO.
@@ -498,9 +514,24 @@ static Operation *genSpMat(OpBuilder &builder, Location loc, Type handleTp,
   if (format == CuSparseFormat::kCSR)
     return builder.create<gpu::CreateCsrOp>(loc, handleTp, tokenTp, token, sz1,
                                             sz2, nseA, rowA, colA, valA);
-  assert(format == CuSparseFormat::kCSC);
-  return builder.create<gpu::CreateCscOp>(loc, handleTp, tokenTp, token, sz1,
-                                          sz2, nseA, rowA, colA, valA);
+  if (format == CuSparseFormat::kCSC)
+    return builder.create<gpu::CreateCscOp>(loc, handleTp, tokenTp, token, sz1,
+                                            sz2, nseA, rowA, colA, valA);
+  // BSR requires a bit more work since we need to pass in the block size
+  // and all others sizes in terms of blocks (#block-rows, #block-cols,
+  // #nonzero-blocks).
+  assert(format == CuSparseFormat::kBSR);
+  SmallVector<unsigned> dims = getBlockSize(aTp.getDimToLvl());
+  assert(dims.size() == 2 && dims[0] == dims[1]);
+  uint64_t b = dims[0];
+  Value bSz = constantIndex(builder, loc, b);
+  Value bRows = builder.create<arith::DivUIOp>(loc, sz1, bSz);
+  Value bCols = builder.create<arith::DivUIOp>(loc, sz2, bSz);
+  Value bNum = builder.create<arith::DivUIOp>(
+      loc, nseA, constantIndex(builder, loc, b * b));
+  return builder.create<gpu::CreateBsrOp>(loc, handleTp, tokenTp, token, bRows,
+                                          bCols, bNum, bSz, bSz, rowA, colA,
+                                          valA);
 }
 
 /// Match and rewrite SpMV kernel.
@@ -566,8 +597,8 @@ rewriteSpMV(PatternRewriter &rewriter, linalg::GenericOp op, bool enableRT,
   Type tokenTp = rewriter.getType<gpu::AsyncTokenType>();
   Value token = genFirstWait(rewriter, loc);
   Operation *spGenA =
-      genSpMat(rewriter, loc, spmatHandleTp, tokenTp, token, szY, szX, nseA,
-               rowA, colA, valA, format, enableRT);
+      genSpMat(rewriter, loc, aTp, spmatHandleTp, tokenTp, token, szY, szX,
+               nseA, rowA, colA, valA, format, enableRT);
   Value spMatA = spGenA->getResult(0);
   token = spGenA->getResult(1);
   auto dvecX = rewriter.create<gpu::CreateDnTensorOp>(
@@ -691,8 +722,8 @@ rewriteSpMM(PatternRewriter &rewriter, linalg::GenericOp op, bool enableRT,
   Type tokenTp = rewriter.getType<gpu::AsyncTokenType>();
   Value token = genFirstWait(rewriter, loc);
   Operation *spGenA =
-      genSpMat(rewriter, loc, spMatHandleTp, tokenTp, token, szm, szk, nseA,
-               rowA, colA, valA, format, enableRT);
+      genSpMat(rewriter, loc, aTp, spMatHandleTp, tokenTp, token, szm, szk,
+               nseA, rowA, colA, valA, format, enableRT);
   Value spMatA = spGenA->getResult(0);
   token = spGenA->getResult(1);
   auto dmatB = rewriter.create<gpu::CreateDnTensorOp>(
@@ -806,13 +837,13 @@ rewriteSpGEMM(PatternRewriter &rewriter, linalg::GenericOp op, bool enableRT,
   Type tokenTp = rewriter.getType<gpu::AsyncTokenType>();
   Value token = genFirstWait(rewriter, loc);
   Operation *spGenA =
-      genSpMat(rewriter, loc, spmatHandleTp, tokenTp, token, szm, szk, nseA,
-               rowA, colA, valA, format, enableRT);
+      genSpMat(rewriter, loc, aTp, spmatHandleTp, tokenTp, token, szm, szk,
+               nseA, rowA, colA, valA, format, enableRT);
   Value spMatA = spGenA->getResult(0);
   token = spGenA->getResult(1);
   Operation *spGenB =
-      genSpMat(rewriter, loc, spmatHandleTp, tokenTp, token, szk, szn, nseB,
-               rowB, colB, valB, format, enableRT);
+      genSpMat(rewriter, loc, bTp, spmatHandleTp, tokenTp, token, szk, szn,
+               nseB, rowB, colB, valB, format, enableRT);
   Value spMatB = spGenB->getResult(0);
   token = spGenB->getResult(1);
 
@@ -830,8 +861,8 @@ rewriteSpGEMM(PatternRewriter &rewriter, linalg::GenericOp op, bool enableRT,
   Value valC = e3.getResult(0); // no free needed
   token = e3.getAsyncToken();
   Operation *spGenC =
-      genSpMat(rewriter, loc, spmatHandleTp, tokenTp, token, szm, szn, zero,
-               rowC, colC, valC, format, enableRT);
+      genSpMat(rewriter, loc, cTp, spmatHandleTp, tokenTp, token, szm, szn,
+               zero, rowC, colC, valC, format, enableRT);
   Value spMatC = spGenC->getResult(0);
   token = spGenC->getResult(1);
 
@@ -1137,8 +1168,8 @@ rewriteSDDMM(PatternRewriter &rewriter, linalg::GenericOp op, bool enableRT,
   Value dnB = dmatB.getResult(0);
   token = dmatB.getAsyncToken();
   Operation *spGenC =
-      genSpMat(rewriter, loc, spMatHandleTp, tokenTp, token, szm, szn, nseC,
-               rowC, colC, valC, format, enableRT);
+      genSpMat(rewriter, loc, cTp, spMatHandleTp, tokenTp, token, szm, szn,
+               nseC, rowC, colC, valC, format, enableRT);
   Value spMatC = spGenC->getResult(0);
   token = spGenC->getResult(1);
   auto dnCType = llvm::cast<ShapedType>(c.getType()).getElementType();
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
index e22789643c..67a78832ef 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
@@ -478,9 +478,7 @@ class SparseInsertGenerator
       std::replace_if(
           lvlType.begin(), lvlType.end(),
           [](char c) { return c == '(' || c == ','; }, '_');
-      lvlType.erase(std::remove_if(lvlType.begin(), lvlType.end(),
-                                   [](char c) { return c == ')' || c == ' '; }),
-                    lvlType.end());
+      llvm::erase_if(lvlType, [](char c) { return c == ')' || c == ' '; });
       nameOstream << lvlType << "_";
     }
     // Static dim sizes are used in the generated code while dynamic sizes are
@@ -679,6 +677,46 @@ class SparseDimOpConverter : public OpConversionPattern<tensor::DimOp> {
   }
 };
 
+// TODO: use a new SortCOO operation here instead of reusing convert op.
+struct SparseReorderCOOConverter : public OpConversionPattern<ReorderCOOOp> {
+  using OpConversionPattern::OpConversionPattern;
+  LogicalResult
+  matchAndRewrite(ReorderCOOOp op, ReorderCOOOpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    Location loc = op.getLoc();
+    MLIRContext *ctx = op.getContext();
+
+    SparseTensorType srcStt = getSparseTensorType(op.getInputCoo());
+    SparseTensorType dstStt = getSparseTensorType(op.getResultCoo());
+
+    // Should have been verified.
+    assert(dstStt.isAllOrdered() && !srcStt.isAllOrdered() &&
+           isUniqueCOOType(srcStt.getRankedTensorType()) &&
+           isUniqueCOOType(dstStt.getRankedTensorType()));
+    assert(dstStt.hasSameDimToLvl(srcStt));
+
+    // We don't need a mutable descriptor here as we perform sorting in-place.
+    auto nnz = genValMemSize(rewriter, op.getLoc(), adaptor.getInputCoo());
+    auto desc = getDescriptorFromTensorTuple(adaptor.getInputCoo());
+    auto crd = desc.getAOSMemRef();
+    auto val = desc.getValMemRef();
+
+    // Otherwise we need another data shuffle and a non-identity map.
+    assert(dstStt.hasSameDimToLvl(srcStt));
+    (void)dstStt; // to silence warning when assertion is disabled
+
+    auto id = AffineMap::getMultiDimIdentityMap(srcStt.getLvlRank(), ctx);
+
+    rewriter.create<SortOp>(loc, nnz, crd, ValueRange{val}, id,
+                            rewriter.getIndexAttr(0), op.getAlgorithm());
+
+    // Since we do in-place sorting, the destinate tensor will have the same set
+    // of memrefs as the source tensor.
+    rewriter.replaceOp(op, adaptor.getInputCoo());
+    return success();
+  }
+};
+
 template <typename Op, StorageSpecifierKind kind>
 class SparseSliceGetterOpConverter : public OpConversionPattern<Op> {
 public:
@@ -1554,6 +1592,7 @@ void mlir::populateSparseTensorCodegenPatterns(
                SparseCastConverter, SparseExtractSliceConverter,
                SparseTensorLoadConverter, SparseExpandConverter,
                SparseCompressConverter, SparseInsertConverter,
+               SparseReorderCOOConverter,
                SparseSliceGetterOpConverter<ToSliceOffsetOp,
                                             StorageSpecifierKind::DimOffset>,
                SparseSliceGetterOpConverter<ToSliceStrideOp,
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp
index d2d7b46ab8..a1f7253335 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp
@@ -199,37 +199,25 @@ class NewCallParams final {
   /// type-level information such as the encoding and sizes), generating
   /// MLIR buffers as needed, and returning `this` for method chaining.
   NewCallParams &genBuffers(SparseTensorType stt,
-                            ArrayRef<Value> dimSizesValues) {
+                            ArrayRef<Value> dimSizesValues,
+                            Value dimSizesBuffer = Value()) {
     assert(dimSizesValues.size() == static_cast<size_t>(stt.getDimRank()));
     // Sparsity annotations.
     params[kParamLvlTypes] = genLvlTypesBuffer(builder, loc, stt);
     // Construct dimSizes, lvlSizes, dim2lvl, and lvl2dim buffers.
-    params[kParamDimSizes] = allocaBuffer(builder, loc, dimSizesValues);
+    params[kParamDimSizes] = dimSizesBuffer
+                                 ? dimSizesBuffer
+                                 : allocaBuffer(builder, loc, dimSizesValues);
     params[kParamLvlSizes] =
         genMapBuffers(builder, loc, stt, dimSizesValues, params[kParamDimSizes],
                       params[kParamDim2Lvl], params[kParamLvl2Dim]);
     // Secondary and primary types encoding.
-    setTemplateTypes(stt);
-    // Finally, make note that initialization is complete.
-    assert(isInitialized() && "Initialization failed");
-    // And return `this` for method chaining.
-    return *this;
-  }
-
-  /// (Re)sets the C++ template type parameters, and returns `this`
-  /// for method chaining. This is already done as part of `genBuffers`,
-  /// but is factored out so that it can also be called independently
-  /// whenever subsequent `genNewCall` calls want to reuse the same
-  /// buffers but different type parameters.
-  //
-  // TODO: This is only ever used by sparse2sparse-viaCOO `ConvertOp`;
-  // is there a better way to handle that than this one-off setter method?
-  NewCallParams &setTemplateTypes(SparseTensorType stt) {
     const auto enc = stt.getEncoding();
     params[kParamPosTp] = constantPosTypeEncoding(builder, loc, enc);
     params[kParamCrdTp] = constantCrdTypeEncoding(builder, loc, enc);
     params[kParamValTp] =
         constantPrimaryTypeEncoding(builder, loc, stt.getElementType());
+    // Return `this` for method chaining.
     return *this;
   }
 
@@ -241,16 +229,6 @@ class NewCallParams final {
     return true;
   }
 
-  /// Gets the dimension-to-level mapping.
-  //
-  // TODO: This is only ever used for passing into `genAddEltCall`;
-  // is there a better way to encapsulate that pattern (both to avoid
-  // this one-off getter, and to avoid potential mixups)?
-  Value getDimToLvl() const {
-    assert(isInitialized() && "Must initialize before getDimToLvl");
-    return params[kParamDim2Lvl];
-  }
-
   /// Generates a function call, with the current static parameters
   /// and the given dynamic arguments.
   Value genNewCall(Action action, Value ptr = Value()) {
@@ -299,76 +277,6 @@ static void genDelCOOCall(OpBuilder &builder, Location loc, Type elemTp,
   createFuncCall(builder, loc, name, {}, coo, EmitCInterface::Off);
 }
 
-/// Generates a call to release/delete a `SparseTensorIterator`.
-static void genDelIteratorCall(OpBuilder &builder, Location loc, Type elemTp,
-                               Value iter) {
-  SmallString<26> name{"delSparseTensorIterator",
-                       primaryTypeFunctionSuffix(elemTp)};
-  createFuncCall(builder, loc, name, {}, iter, EmitCInterface::Off);
-}
-
-/// Generates a call that adds one element to a coordinate scheme.
-/// In particular, this generates code like the following:
-///   val = a[i1,..,ik];
-///   if val != 0
-///     t->add(&val, [i1,..,ik], [p1,..,pk]);
-static void genAddEltCall(OpBuilder &builder, Location loc, Type eltType,
-                          Value lvlCOO, Value valPtr, Value dimCoords,
-                          Value dimToLvl) {
-  SmallString<9> name{"addElt", primaryTypeFunctionSuffix(eltType)};
-  SmallVector<Value, 4> params{lvlCOO, valPtr, dimCoords, dimToLvl};
-  Type pTp = getOpaquePointerType(builder);
-  createFuncCall(builder, loc, name, pTp, params, EmitCInterface::On);
-}
-
-/// Generates a call to `iter->getNext()`.  If there is a next element,
-/// then it is copied into the out-parameters `coords` and `elemPtr`,
-/// and the return value is true.  If there isn't a next element, then
-/// the return value is false.
-///
-/// The `coords` argument uses the same coordinate-space as the `iter`
-/// (which can be either dim- or lvl-coords, depending on context).
-static Value genGetNextCall(OpBuilder &builder, Location loc, Value iter,
-                            Value coords, Value elemPtr) {
-  Type elemTp = cast<ShapedType>(elemPtr.getType()).getElementType();
-  SmallString<10> name{"getNext", primaryTypeFunctionSuffix(elemTp)};
-  SmallVector<Value, 3> params{iter, coords, elemPtr};
-  Type i1 = builder.getI1Type();
-  return createFuncCall(builder, loc, name, i1, params, EmitCInterface::On)
-      .getResult(0);
-}
-
-/// Loads the value stored in `elemPtr`, and stores it at the coordinates
-/// `cvs` into a dense tensor created by `allocDenseTensor`.
-static void insertScalarIntoDenseTensor(OpBuilder &builder, Location loc,
-                                        Value elemPtr, Value tensor,
-                                        ValueRange cvs) {
-  Value elemV = builder.create<memref::LoadOp>(loc, elemPtr);
-  builder.create<memref::StoreOp>(loc, elemV, tensor, cvs);
-}
-
-/// Determine if the runtime library supports direct conversion to the
-/// given target `dimTypes`.
-static bool canUseDirectConversion(ArrayRef<DimLevelType> dimTypes) {
-  bool alreadyCompressed = false;
-  for (const auto dlt : dimTypes) {
-    if (isCompressedDLT(dlt)) {
-      if (alreadyCompressed)
-        return false; // Multiple compressed dimensions not yet supported.
-      alreadyCompressed = true;
-    } else if (isDenseDLT(dlt)) {
-      if (alreadyCompressed)
-        return false; // Dense after Compressed not yet supported.
-    } else if (isSingletonDLT(dlt)) {
-      // Direct conversion doesn't have any particular problems with
-      // singleton after compressed.
-    } else { // TODO: investigate
-      return false;
-    }
-  }
-  return true;
-}
-
 //===----------------------------------------------------------------------===//
 // Conversion rules.
 //===----------------------------------------------------------------------===//
@@ -437,33 +345,15 @@ class SparseTensorNewConverter : public OpConversionPattern<NewOp> {
     const auto stt = getSparseTensorType(op);
     if (!stt.hasEncoding())
       return failure();
-    // Construct the reader opening method calls.
+    // Construct the `reader` opening method calls.
     SmallVector<Value> dimShapesValues;
     Value dimSizesBuffer;
     Value reader = genReader(rewriter, loc, stt, adaptor.getOperands()[0],
                              dimShapesValues, dimSizesBuffer);
-    // Now construct the lvlSizes, dim2lvl, and lvl2dim buffers.
-    Value dim2lvlBuffer;
-    Value lvl2dimBuffer;
-    Value lvlSizesBuffer =
-        genMapBuffers(rewriter, loc, stt, dimShapesValues, dimSizesBuffer,
-                      dim2lvlBuffer, lvl2dimBuffer);
     // Use the `reader` to parse the file.
-    Type opaqueTp = getOpaquePointerType(rewriter);
-    Type eltTp = stt.getElementType();
-    Value valTp = constantPrimaryTypeEncoding(rewriter, loc, eltTp);
-    SmallVector<Value, 8> params{
-        reader,
-        lvlSizesBuffer,
-        genLvlTypesBuffer(rewriter, loc, stt),
-        dim2lvlBuffer,
-        lvl2dimBuffer,
-        constantPosTypeEncoding(rewriter, loc, stt.getEncoding()),
-        constantCrdTypeEncoding(rewriter, loc, stt.getEncoding()),
-        valTp};
-    Value tensor = createFuncCall(rewriter, loc, "newSparseTensorFromReader",
-                                  opaqueTp, params, EmitCInterface::On)
-                       .getResult(0);
+    Value tensor = NewCallParams(rewriter, loc)
+                       .genBuffers(stt, dimShapesValues, dimSizesBuffer)
+                       .genNewCall(Action::kFromReader, reader);
     // Free the memory for `reader`.
     createFuncCall(rewriter, loc, "delSparseTensorReader", {}, {reader},
                    EmitCInterface::Off);
@@ -540,179 +430,27 @@ class SparseTensorEmptyConverter : public OpConversionPattern<tensor::EmptyOp> {
 };
 
 /// Sparse conversion rule for the convert operator.
-class SparseTensorConvertConverter : public OpConversionPattern<ConvertOp> {
+class SparseTensorReorderCOOConverter
+    : public OpConversionPattern<ReorderCOOOp> {
 public:
   using OpConversionPattern::OpConversionPattern;
-  SparseTensorConvertConverter(MLIRContext *context,
-                               SparseTensorConversionOptions o)
-      : OpConversionPattern<ConvertOp>(context), options(o) {}
-  SparseTensorConvertConverter(TypeConverter &typeConv, MLIRContext *context,
-                               SparseTensorConversionOptions o)
-      : OpConversionPattern<ConvertOp>(typeConv, context), options(o) {}
 
   LogicalResult
-  matchAndRewrite(ConvertOp op, OpAdaptor adaptor,
+  matchAndRewrite(ReorderCOOOp op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     const Location loc = op->getLoc();
-    const auto srcTp = getSparseTensorType(op.getSource());
+    const auto srcTp = getSparseTensorType(op.getInputCoo());
     const auto dstTp = getSparseTensorType(op);
-    if (!srcTp.hasEncoding() && !dstTp.hasEncoding())
-      return failure();
 
-    const Dimension dimRank = srcTp.getDimRank();
-    const Type elemTp = srcTp.getElementType();
-    const Value src = adaptor.getOperands()[0];
-    if (srcTp.hasEncoding() && dstTp.hasEncoding()) {
-      const auto srcEnc = srcTp.getEncoding();
-      const auto dstEnc = dstTp.getEncoding();
-      // This is a sparse => sparse conversion, which is handled as follows:
-      //   t = src->toCOO();         ; src to COO in dst order
-      //   dst = newSparseTensor(t)
-      // Using the coordinate scheme as an intermediate does not always
-      // yield the fastest conversion but avoids the need for a full
-      // O(N^2) conversion matrix.
-      if (dstEnc == srcEnc) {
-        rewriter.replaceOp(op, adaptor.getOperands()); // hidden nop cast
-        return success();
-      }
-      NewCallParams params(rewriter, loc);
-      SmallVector<Value> dimSizes = getDimSizes(rewriter, loc, srcTp, src);
-      bool useDirectConversion;
-      switch (options.sparseToSparseStrategy) {
-      case SparseToSparseConversionStrategy::kViaCOO:
-        useDirectConversion = false;
-        break;
-      case SparseToSparseConversionStrategy::kDirect:
-        useDirectConversion = true;
-        assert(canUseDirectConversion(dstEnc.getLvlTypes()) &&
-               "Unsupported target for direct sparse-to-sparse conversion");
-        break;
-      case SparseToSparseConversionStrategy::kAuto:
-        useDirectConversion = canUseDirectConversion(dstEnc.getLvlTypes());
-        break;
-      }
-      if (useDirectConversion) {
-        rewriter.replaceOp(
-            op, params.genBuffers(srcTp.withEncoding(dstEnc), dimSizes)
-                    .genNewCall(Action::kSparseToSparse, src));
-      } else { // use via-COO conversion.
-        // Set up encoding with right mix of src and dst so that the two
-        // method calls can share most parameters, while still providing
-        // the correct sparsity information to either of them.
-        const auto mixedEnc =
-            dstEnc.withBitWidths(srcEnc.getPosWidth(), srcEnc.getCrdWidth());
-        // TODO: This is the only place where `kToCOO` (or `kToIterator`)
-        // is called with a non-identity permutation.  Is there any clean
-        // way to push the permutation over to the `kFromCOO` side instead?
-        Value coo = params.genBuffers(srcTp.withEncoding(mixedEnc), dimSizes)
-                        .genNewCall(Action::kToCOO, src);
-        Value dst = params.setTemplateTypes(srcTp.withEncoding(dstEnc))
-                        .genNewCall(Action::kFromCOO, coo);
-        genDelCOOCall(rewriter, loc, elemTp, coo);
-        rewriter.replaceOp(op, dst);
-      }
-      return success();
-    }
-    if (srcTp.hasEncoding() && !dstTp.hasEncoding()) {
-      const auto srcEnc = srcTp.getEncoding();
-      // This is sparse => dense conversion, which is handled as follows:
-      //   dst = new Tensor(0);
-      //   iter = new SparseTensorIterator(src);
-      //   while (elem = iter->getNext()) {
-      //     dst[elem.coords] = elem.value;
-      //   }
-      //   delete iter;
-      //
-      // Fabricate a no-permutation encoding for NewCallParams
-      // The position/coordinate types must be those of `src`.
-      // The dimLevelTypes aren't actually used by Action::kToIterator.
-      const auto dstEnc = SparseTensorEncodingAttr::get(
-          op->getContext(),
-          SmallVector<DimLevelType>(dimRank, DimLevelType::Dense), AffineMap(),
-          AffineMap(), srcEnc.getPosWidth(), srcEnc.getCrdWidth());
-      SmallVector<Value> dimSizes = getDimSizes(rewriter, loc, srcTp, src);
-      Value iter = NewCallParams(rewriter, loc)
-                       .genBuffers(dstTp.withEncoding(dstEnc), dimSizes)
-                       .genNewCall(Action::kToIterator, src);
-      const Type iTp = rewriter.getIndexType();
-      Value dimCoords = genAlloca(rewriter, loc, dimRank, iTp);
-      Value elemPtr = genAllocaScalar(rewriter, loc, elemTp);
-      // TODO: Dense buffers should be allocated/deallocated via the callback
-      // in BufferizationOptions.
-      Value dst = allocDenseTensor(rewriter, loc, dstTp, dimSizes);
-      const SmallVector<Value> noArgs;
-      const SmallVector<Type> noTypes;
-      auto whileOp = rewriter.create<scf::WhileOp>(loc, noTypes, noArgs);
-      Block *before = rewriter.createBlock(&whileOp.getBefore(), {}, noTypes);
-      rewriter.setInsertionPointToEnd(before);
-      Value cond = genGetNextCall(rewriter, loc, iter, dimCoords, elemPtr);
-      rewriter.create<scf::ConditionOp>(loc, cond, before->getArguments());
-      Block *after = rewriter.createBlock(&whileOp.getAfter(), {}, noTypes);
-      rewriter.setInsertionPointToStart(after);
-      const auto dcvs = loadAll(rewriter, loc, dimRank, dimCoords);
-      insertScalarIntoDenseTensor(rewriter, loc, elemPtr, dst, dcvs);
-      rewriter.create<scf::YieldOp>(loc);
-      rewriter.setInsertionPointAfter(whileOp);
-      genDelIteratorCall(rewriter, loc, elemTp, iter);
-      rewriter.replaceOpWithNewOp<bufferization::ToTensorOp>(
-          op, dstTp.getRankedTensorType(), dst);
-      return success();
-    }
-    assert(!srcTp.hasEncoding() && dstTp.hasEncoding());
-    // This is a dense => sparse conversion or a sparse constant in COO =>
-    // sparse conversion, which is handled as follows:
-    //   t = newSparseCOO()
-    //   ...code to fill the COO tensor t...
-    //   s = newSparseTensor(t)
-    //
-    // To fill the COO tensor from a dense tensor:
-    //   for i1 in dim1
-    //    ..
-    //     for ik in dimk
-    //       val = a[i1,..,ik]
-    //       if val != 0
-    //         t->add(val, [i1,..,ik], [p1,..,pk])
-    //
-    // To fill the COO tensor from a sparse constant in COO format:
-    //   for i in range(NNZ)
-    //     val = values[i]
-    //     [i1,..,ik] = coordinates[i]
-    //     t->add(val, [i1,..,ik], [p1,..,pk])
-    //
-    // Note that the dense tensor traversal code is actually implemented
-    // using MLIR IR to avoid having to expose too much low-level
-    // memref traversal details to the runtime support library.
-    // Also note that the code below only generates the "new" ops and
-    // the loop-nest per se; whereas the entire body of the innermost
-    // loop is generated by genAddElt().
-    SmallVector<Value> dimSizes;
-    sizesFromSrc(rewriter, dimSizes, loc, src);
+    const Value src = adaptor.getInputCoo();
+
     NewCallParams params(rewriter, loc);
-    Value coo =
-        params.genBuffers(dstTp, dimSizes).genNewCall(Action::kEmptyCOO);
-    const Type iTp = rewriter.getIndexType();
-    Value dimCoords = genAlloca(rewriter, loc, dimRank, iTp);
-    Value dimToLvl = params.getDimToLvl();
-    Value elemPtr = genAllocaScalar(rewriter, loc, elemTp);
-    genDenseTensorOrSparseConstantIterLoop(
-        rewriter, loc, src, dimRank,
-        [&](OpBuilder &builder, Location loc, Value val, ValueRange dcvs) {
-          assert(dcvs.size() == static_cast<size_t>(dimRank));
-          storeAll(builder, loc, dimCoords, dcvs);
-          builder.create<memref::StoreOp>(loc, val, elemPtr);
-          genAddEltCall(builder, loc, elemTp, coo, elemPtr, dimCoords,
-                        dimToLvl);
-        });
-    // Final call to construct sparse tensor storage.
-    Value dst = params.genNewCall(Action::kFromCOO, coo);
-    genDelCOOCall(rewriter, loc, elemTp, coo);
-    rewriter.replaceOp(op, dst);
+    SmallVector<Value> dimSizes = getDimSizes(rewriter, loc, srcTp, src);
+    rewriter.replaceOp(op, params.genBuffers(dstTp, dimSizes)
+                               .genNewCall(Action::kSortCOOInPlace, src));
+
     return success();
   }
-
-private:
-  /// Options to control sparse code generation.
-  SparseTensorConversionOptions options;
 };
 
 /// Sparse conversion rule for the dealloc operator.
@@ -828,7 +566,7 @@ class SparseTensorLoadConverter : public OpConversionPattern<LoadOp> {
                   ConversionPatternRewriter &rewriter) const override {
     if (op.getHasInserts()) {
       // Finalize any pending insertions.
-      StringRef name = "endInsert";
+      StringRef name = "endLexInsert";
       createFuncCall(rewriter, op->getLoc(), name, {}, adaptor.getOperands(),
                      EmitCInterface::Off);
     }
@@ -851,8 +589,17 @@ class SparseTensorInsertConverter : public OpConversionPattern<InsertOp> {
     const auto stt = getSparseTensorType(op.getTensor());
     const auto elemTp = stt.getElementType();
     const Level lvlRank = stt.getLvlRank();
-    auto lvlCoords = genAlloca(rewriter, loc, lvlRank, rewriter.getIndexType());
-    auto vref = genAllocaScalar(rewriter, loc, elemTp);
+    Value lvlCoords, vref;
+    {
+      OpBuilder::InsertionGuard guard(rewriter);
+      auto loop = op->getParentOfType<LoopLikeOpInterface>();
+      if (loop) {
+        // Hoists alloca outside the loop to avoid stack overflow.
+        rewriter.setInsertionPoint(loop);
+      }
+      lvlCoords = genAlloca(rewriter, loc, lvlRank, rewriter.getIndexType());
+      vref = genAllocaScalar(rewriter, loc, elemTp);
+    }
     storeAll(rewriter, loc, lvlCoords, adaptor.getLvlCoords());
     rewriter.create<memref::StoreOp>(loc, adaptor.getValue(), vref);
     SmallString<12> name{"lexInsert", primaryTypeFunctionSuffix(elemTp)};
@@ -1013,19 +760,17 @@ mlir::SparseTensorTypeToPtrConverter::SparseTensorTypeToPtrConverter() {
 
 /// Populates the given patterns list with conversion rules required for
 /// the sparsification of linear algebra operations.
-void mlir::populateSparseTensorConversionPatterns(
-    TypeConverter &typeConverter, RewritePatternSet &patterns,
-    const SparseTensorConversionOptions &options) {
-  patterns.add<SparseReturnConverter, SparseTensorToDimSizeConverter,
-               SparseCastConverter, SparseTensorNewConverter,
-               SparseTensorAllocConverter, SparseTensorEmptyConverter,
-               SparseTensorDeallocConverter, SparseTensorToPositionsConverter,
-               SparseTensorToCoordinatesConverter,
-               SparseTensorToValuesConverter, SparseNumberOfEntriesConverter,
-               SparseTensorLoadConverter, SparseTensorInsertConverter,
-               SparseTensorExpandConverter, SparseTensorCompressConverter,
-               SparseTensorOutConverter, SparseTensorAssembleConverter>(
-      typeConverter, patterns.getContext());
-  patterns.add<SparseTensorConvertConverter>(typeConverter,
-                                             patterns.getContext(), options);
+void mlir::populateSparseTensorConversionPatterns(TypeConverter &typeConverter,
+                                                  RewritePatternSet &patterns) {
+  patterns
+      .add<SparseReturnConverter, SparseTensorToDimSizeConverter,
+           SparseCastConverter, SparseTensorNewConverter,
+           SparseTensorAllocConverter, SparseTensorEmptyConverter,
+           SparseTensorDeallocConverter, SparseTensorReorderCOOConverter,
+           SparseTensorToPositionsConverter, SparseTensorToCoordinatesConverter,
+           SparseTensorToValuesConverter, SparseNumberOfEntriesConverter,
+           SparseTensorLoadConverter, SparseTensorInsertConverter,
+           SparseTensorExpandConverter, SparseTensorCompressConverter,
+           SparseTensorOutConverter, SparseTensorAssembleConverter>(
+          typeConverter, patterns.getContext());
 }
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorPasses.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorPasses.cpp
index e1f88ad9c0..eaf15ff29d 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorPasses.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorPasses.cpp
@@ -132,9 +132,6 @@ struct SparseTensorConversionPass
 
   SparseTensorConversionPass() = default;
   SparseTensorConversionPass(const SparseTensorConversionPass &pass) = default;
-  SparseTensorConversionPass(const SparseTensorConversionOptions &options) {
-    sparseToSparse = static_cast<int32_t>(options.sparseToSparseStrategy);
-  }
 
   void runOnOperation() override {
     auto *ctx = &getContext();
@@ -187,16 +184,14 @@ struct SparseTensorConversionPass
     target.addLegalDialect<
         arith::ArithDialect, bufferization::BufferizationDialect,
         LLVM::LLVMDialect, memref::MemRefDialect, scf::SCFDialect>();
-    // Translate strategy flags to strategy options.
-    SparseTensorConversionOptions options(
-        sparseToSparseConversionStrategy(sparseToSparse));
+
     // Populate with rules and apply rewriting rules.
     populateFunctionOpInterfaceTypeConversionPattern<func::FuncOp>(patterns,
                                                                    converter);
     populateCallOpTypeConversionPattern(patterns, converter);
     scf::populateSCFStructuralTypeConversionsAndLegality(converter, patterns,
                                                          target);
-    populateSparseTensorConversionPatterns(converter, patterns, options);
+    populateSparseTensorConversionPatterns(converter, patterns);
     if (failed(applyPartialConversion(getOperation(), target,
                                       std::move(patterns))))
       signalPassFailure();
@@ -364,22 +359,6 @@ struct StorageSpecifierToLLVMPass
 
 } // namespace
 
-//===----------------------------------------------------------------------===//
-// Strategy flag methods.
-//===----------------------------------------------------------------------===//
-
-SparseToSparseConversionStrategy
-mlir::sparseToSparseConversionStrategy(int32_t flag) {
-  switch (flag) {
-  default:
-    return SparseToSparseConversionStrategy::kAuto;
-  case 1:
-    return SparseToSparseConversionStrategy::kViaCOO;
-  case 2:
-    return SparseToSparseConversionStrategy::kDirect;
-  }
-}
-
 //===----------------------------------------------------------------------===//
 // Pass creation methods.
 //===----------------------------------------------------------------------===//
@@ -416,11 +395,6 @@ std::unique_ptr<Pass> mlir::createSparseTensorConversionPass() {
   return std::make_unique<SparseTensorConversionPass>();
 }
 
-std::unique_ptr<Pass> mlir::createSparseTensorConversionPass(
-    const SparseTensorConversionOptions &options) {
-  return std::make_unique<SparseTensorConversionPass>(options);
-}
-
 std::unique_ptr<Pass> mlir::createSparseTensorCodegenPass() {
   return std::make_unique<SparseTensorCodegenPass>();
 }
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
index b0bd22b156..5ce81f932f 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
@@ -13,6 +13,7 @@
 #include "CodegenUtils.h"
 #include "LoopEmitter.h"
 
+#include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/Bufferization/IR/Bufferization.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
@@ -147,8 +148,7 @@ static RankedTensorType getBufferType(const SparseTensorType &stt,
 /// Collects the dynamic dimension sizes for `tp` with the assumption that
 /// `sizes` are the dimension sizes for the type. Stores the dynamic dimension
 /// sizes to dynSizes.
-static void getDynamicSizes(RankedTensorType tp,
-                            const SmallVectorImpl<Value> &sizes,
+static void getDynamicSizes(RankedTensorType tp, ValueRange sizes,
                             SmallVectorImpl<Value> &dynSizes) {
   for (const auto &d : enumerate(tp.getShape())) {
     if (d.value() == ShapedType::kDynamic)
@@ -666,29 +666,27 @@ struct TensorReshapeRewriter : public OpRewritePattern<tensor::ReshapeOp> {
             srcDcvs.push_back(srcLcvs[lvl]);
           }
 
-          Value collapsed_size = constantIndex(builder, loc, 1);
+          Value collapseSize = constantIndex(builder, loc, 1);
           for (Dimension d = 0; d < srcRank; d++)
-            collapsed_size =
-                builder.create<arith::MulIOp>(loc, collapsed_size, srcSizes[d]);
-          SmallVector<Value, 1> collapsedSizes = {collapsed_size};
+            collapseSize =
+                builder.create<arith::MulIOp>(loc, collapseSize, srcSizes[d]);
+          SmallVector<Value, 1> collapsedSizes = {collapseSize};
 
-          ReassociationIndices collapse_indices;
+          ReassociationIndices collapseIdx;
           for (Dimension i = 0; i < srcRank; i++)
-            collapse_indices.push_back(i);
-          SmallVector<ReassociationIndices, 1> collapse_reassociation = {
-              collapse_indices};
+            collapseIdx.push_back(i);
+          SmallVector<ReassociationIndices, 1> collapseReass = {collapseIdx};
           SmallVector<Value, 1> collapsedDcvs;
-          reshapeCvs(builder, loc, collapse_reassociation, srcSizes, srcDcvs,
+          reshapeCvs(builder, loc, collapseReass, srcSizes, srcDcvs,
                      collapsedSizes, collapsedDcvs);
 
-          ReassociationIndices expand_indices;
+          ReassociationIndices expandIdx;
           for (Dimension i = 0; i < dstTp.getDimRank(); i++)
-            expand_indices.push_back(i);
-          SmallVector<ReassociationIndices, 1> expand_reassociation = {
-              expand_indices};
+            expandIdx.push_back(i);
+          SmallVector<ReassociationIndices, 1> expandReass = {expandIdx};
           SmallVector<Value> dstDcvs;
-          reshapeCvs(builder, loc, expand_reassociation, collapsedSizes,
-                     collapsedDcvs, dstSizes, dstDcvs);
+          reshapeCvs(builder, loc, expandReass, collapsedSizes, collapsedDcvs,
+                     dstSizes, dstDcvs);
 
           auto t = builder.create<InsertOp>(loc, v, reduc.front(), dstDcvs);
           builder.create<sparse_tensor::YieldOp>(loc, t);
@@ -832,10 +830,71 @@ struct ReshapeRewriter : public OpRewritePattern<ReshapeOp> {
   }
 };
 
+// A trivial wrapper to help generate different operations for dense/sparse
+// tensors.
+struct TensorLike {
+  TensorLike(OpBuilder &builder, Location loc, RankedTensorType rtt,
+             ValueRange sizes) {
+    SmallVector<Value> dynSzs;
+    getDynamicSizes(rtt, sizes, dynSzs);
+
+    val = builder.create<AllocTensorOp>(loc, rtt, dynSzs);
+    if (!isSparse()) {
+      Value c0 = constantZero(builder, loc, rtt.getElementType());
+      val = builder.create<linalg::FillOp>(loc, c0, val).getResult(0);
+    }
+  }
+
+  void insert(OpBuilder &builder, Location loc, Value v, ValueRange crds) {
+    // TODO: Unify these two.
+    if (isSparse())
+      val = builder.create<sparse_tensor::InsertOp>(loc, v, val, crds);
+    else
+      val = builder.create<tensor::InsertOp>(loc, v, val, crds);
+  }
+
+  Value finalize(OpBuilder &builder, Location loc, RankedTensorType rtp) const {
+    if (isSparse())
+      return builder.create<LoadOp>(loc, val, true);
+    return val;
+  }
+
+  bool isSparse() const {
+    return getSparseTensorEncoding(val.getType()) != nullptr;
+  }
+
+  Value val;
+};
+
+struct CrdTranslateRewriter : public OpRewritePattern<CrdTranslateOp> {
+  using OpRewritePattern::OpRewritePattern;
+  LogicalResult matchAndRewrite(CrdTranslateOp op,
+                                PatternRewriter &rewriter) const override {
+    AffineMap map = op.getDirection() == CrdTransDirectionKind::dim2lvl
+                        ? op.getEncoder().getDimToLvl()
+                        : op.getEncoder().getLvlToDim();
+    SmallVector<Value> outCrds;
+    for (AffineExpr result : map.getResults()) {
+      // TODO: we should probably expand the affine map to IR using our own
+      // rules, since affine.apply assume signed value, while the cooridinates
+      // we provided must always be signless.
+      Value trans = rewriter.create<affine::AffineApplyOp>(
+          op.getLoc(), AffineMap::get(map.getNumDims(), 0, result),
+          op.getInCrds());
+      outCrds.push_back(trans);
+    }
+    rewriter.replaceOp(op, outCrds);
+    return success();
+  }
+};
+
 struct ConcatenateRewriter : public OpRewritePattern<ConcatenateOp> {
   using OpRewritePattern::OpRewritePattern;
   LogicalResult matchAndRewrite(ConcatenateOp op,
                                 PatternRewriter &rewriter) const override {
+    if (op.needsExtraSort())
+      op.emitError("ConcatenateOp not staged");
+
     const Location loc = op.getLoc();
     const auto dstTp = getSparseTensorType(op);
     const Dimension dimRank = dstTp.getDimRank();
@@ -855,95 +914,48 @@ struct ConcatenateRewriter : public OpRewritePattern<ConcatenateOp> {
     // foreach in %s1 : insert d0, d1, %tmp
     // foreach in %s2 : insert d0, d1 + size(s1), %tmp
     // foreach in %s3 : insert d0, d1 + size(s1) + size(s2), %tmp
-    // %t = convert_to_dest_tensor(%tmp)
-    //
-    // NOTE: this cannot be `const` because it will be changed when
-    // `needTmpCOO`, but that's buried in the conditional below and
-    // thus not easily extracted.
-    auto encDst = dstTp.getEncoding();
-    Value dst; // Destination tensor for inserting source tensor values.
-    bool needTmpCOO = true;
-    const bool allDense = dstTp.hasEncoding() && dstTp.isAllDense();
-    Value annotatedDenseDst;
-    if (dstTp.hasEncoding()) {
-      bool allOrdered = false;
-      // When concatenating on dimension 0, and all inputs are sorted
-      // and have an identity dimToLvl, the concatenate will generate
-      // coords in lexOrder thus no need for the tmp COO buffer.
-      // TODO: When conDim != 0, as long as conDim is the first dimension
-      // in all input/output buffers, and all input/output buffers have the same
-      // dimToLvl, the tmp COO buffer is still unnecessary (e.g, concatenate
-      // CSC matrices along column).
-      if (!allDense && conDim == 0 && dstTp.isIdentity()) {
-        for (auto i : op.getInputs()) {
-          const auto stt = getSparseTensorType(i);
-          allOrdered = stt.isAllOrdered() && stt.isIdentity();
-          if (!allOrdered)
-            break;
-        }
-      }
-
-      needTmpCOO = !allDense && !allOrdered;
-      const RankedTensorType tp =
-          getBufferType(dstTp.withoutDimToLvl(), needTmpCOO);
-      encDst = needTmpCOO ? getSparseTensorEncoding(tp) : encDst;
-      SmallVector<Value> dynSizes;
-      getDynamicSizes(dstTp, sizes, dynSizes);
-      dst = rewriter.create<AllocTensorOp>(loc, tp, dynSizes).getResult();
-      if (allDense) {
-        // Create a view of the values buffer to match the unannotated dense
-        // tensor.
-        Value valuesBuffer = genToValues(rewriter, loc, dst);
-        Value dimCoords =
-            genAlloca(rewriter, loc, dimRank, rewriter.getIndexType(),
-                      /*staticShape=*/true);
-        annotatedDenseDst = dst;
-        dst = reshapeValuesToLevels(rewriter, loc, encDst, sizes, valuesBuffer,
-                                    dimCoords);
-      }
-    } else {
-      // TODO: Dense buffers should be allocated/deallocated via the callback
-      // in BufferizationOptions.
-      dst = allocDenseTensor(rewriter, loc, dstTp, sizes);
-    }
 
+    TensorLike dstBuf(rewriter, loc, dstTp.getRankedTensorType(), sizes);
     Value offset = constantIndex(rewriter, loc, 0);
-    SmallVector<Value> initArgs;
-    if (encDst && !allDense)
-      initArgs.push_back(dst);
+    Value iterArg = dstBuf.val;
+
     ForeachOp foreachOp;
     for (Value input : op.getInputs()) {
-      // Build a for op for each input tensor to append new values into the
+      // Builds a for op for each input tensor to append new values into the
       // output tensor.
       foreachOp = rewriter.create<ForeachOp>(
-          loc, input, initArgs,
+          loc, input, iterArg,
           [&](OpBuilder &builder, Location loc, ValueRange dcvs, Value v,
               ValueRange reduc) {
             SmallVector<Value> dstLcvs(dstTp.getLvlRank());
             for (Dimension d = 0; d < dimRank; d++) {
               Value crd = dcvs[d];
+              // Transforms coordinates for the concatenating dim.
               if (d == conDim)
-                // Transform coordinates for the concatenating dim.
                 crd = builder.create<arith::AddIOp>(loc, crd, offset);
               // FIXME: `toStoredDim` is deprecated
-              dstLcvs[toStoredDim(encDst, d)] = crd;
+              dstLcvs[toStoredDim(dstTp.getEncoding(), d)] = crd;
             }
-            if (encDst && !allDense) {
-              Value cond = genIsNonzero(rewriter, loc, v);
-              scf::IfOp ifOp = builder.create<scf::IfOp>(
-                  loc, TypeRange(reduc.front().getType()), cond, /*else*/ true);
+            // Enters foreach, updates the SSA chain.
+            dstBuf.val = reduc.front();
+            if (!dstTp.isAllDense()) {
+              Value cond = genIsNonzero(builder, loc, v);
+              auto ifOp = builder.create<scf::IfOp>(loc, reduc.getTypes(), cond,
+                                                    /*else*/ true);
+              builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
+              builder.create<scf::YieldOp>(loc, dstBuf.val);
+
               builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
-              Value t =
-                  builder.create<InsertOp>(loc, v, reduc.front(), dstLcvs);
-              rewriter.create<scf::YieldOp>(loc, t);
-              rewriter.setInsertionPointToStart(&ifOp.getElseRegion().front());
-              rewriter.create<scf::YieldOp>(loc, reduc.front());
-              rewriter.setInsertionPointAfter(ifOp);
-              rewriter.create<sparse_tensor::YieldOp>(loc, ifOp.getResult(0));
+              dstBuf.insert(builder, loc, v, dstLcvs);
+              builder.create<scf::YieldOp>(loc, dstBuf.val);
+
+              // Exits the ifOp, update the sparse tensor SSA value.
+              builder.setInsertionPointAfter(ifOp);
+              dstBuf.val = ifOp.getResult(0);
             } else {
-              builder.create<memref::StoreOp>(loc, v, dst, dstLcvs);
-              builder.create<sparse_tensor::YieldOp>(loc);
+              dstBuf.insert(builder, loc, v, dstLcvs);
             }
+            builder.create<sparse_tensor::YieldOp>(loc, dstBuf.val);
           });
       // Accumulates the offset. Note that only static-shaped inputs are allowed
       // by concatenate op verifier, which saves us from computing the offset
@@ -952,39 +964,26 @@ struct ConcatenateRewriter : public OpRewritePattern<ConcatenateOp> {
       assert(sh.has_value());
       offset = rewriter.create<arith::AddIOp>(
           loc, offset, constantIndex(rewriter, loc, *sh));
-      if (encDst && !allDense) {
-        dst = foreachOp.getResult(0);
-        initArgs[0] = dst;
-      }
-    }
 
-    // Temp variable to avoid needing to call `getRankedTensorType`
-    // in the three use-sites below.
-    const RankedTensorType dstRTT = dstTp;
-    if (!encDst) {
-      rewriter.replaceOpWithNewOp<bufferization::ToTensorOp>(op, dstRTT, dst);
-    } else if (allDense) {
-      rewriter.replaceOp(
-          op, rewriter.create<ConvertOp>(loc, dstRTT, annotatedDenseDst)
-                  .getResult());
-    } else {
-      dst = rewriter.create<LoadOp>(loc, dst, true);
-      if (needTmpCOO) {
-        Value tmpCoo = dst;
-        dst = rewriter.create<ConvertOp>(loc, dstRTT, tmpCoo).getResult();
-        rewriter.create<DeallocTensorOp>(loc, tmpCoo);
-      }
-      rewriter.replaceOp(op, dst);
+      iterArg = foreachOp.getResult(0);
+      dstBuf.val = iterArg;
     }
+
+    dstBuf.val = iterArg;
+    Value ret = dstBuf.finalize(rewriter, loc, dstTp.getRankedTensorType());
+    rewriter.replaceOp(op, ret);
     return success();
   }
 };
 
-/// Sparse rewriting rule for the convert operator.
-struct ConvertRewriter : public OpRewritePattern<ConvertOp> {
+struct DirectConvertRewriter : public OpRewritePattern<ConvertOp> {
   using OpRewritePattern::OpRewritePattern;
   LogicalResult matchAndRewrite(ConvertOp op,
                                 PatternRewriter &rewriter) const override {
+    if (op.needsExtraSort())
+      return op.emitError("ConvertOp not staged.");
+
+    // TODO: Maybe we want a different operation for this too.
     auto encDst = getSparseTensorEncoding(op.getType());
     auto encSrc = getSparseTensorEncoding(op.getSource().getType());
     if (encDst && encSrc && !encSrc.isSlice() &&
@@ -993,272 +992,67 @@ struct ConvertRewriter : public OpRewritePattern<ConvertOp> {
       // in codegen.
       return failure();
     }
-    // TODO: Add a cast before generating InsertOp.
-    assert(op.getSource().getType().getElementType() ==
-           op.getDest().getType().getElementType());
-    if (encSrc && encDst)
-      return sparse2SparseRewrite(op, rewriter);
-    if (encSrc && !encDst)
-      return sparse2DenseRewrite(op, rewriter);
-    if (!encSrc && encDst)
-      return dense2SparseRewrite(op, rewriter);
-
-    // Dense-to-dense convert is a nop and handled by canonicalization.
-    return failure();
-  }
 
-private:
-  // Handles sparse constant to sparse tensor or dense tensor to sparse tensor
-  // conversion as follows:
-  //   t = new sparse COO tensor
-  //   fill t using src
-  //   dst = convert t
-  //
-  // To fill the COO tensor from a dense tensor:
-  //   for i1 in dim1
-  //    ..
-  //     for ik in dimk
-  //       val = a[i1,..,ik]
-  //       if val != 0
-  //         t->add(val, [i1,..,ik], [p1,..,pk])
-  //
-  // To fill the COO tensor from a sparse constant in COO format:
-  //   for i in range(NNZ)
-  //     val = values[i]
-  //     [i1,..,ik] = coordinates[i]
-  //     t->add(val, [i1,..,ik], [p1,..,pk])
-  LogicalResult dense2SparseRewrite(ConvertOp op,
-                                    PatternRewriter &rewriter) const {
     Location loc = op.getLoc();
     Value src = op.getSource();
-    const auto dstTp = getSparseTensorType(op);
-    SmallVector<Value> sizes;
-    sizesFromSrc(rewriter, sizes, loc, src);
-    SmallVector<Value> dynSizes;
-    getDynamicSizes(dstTp, sizes, dynSizes);
+
+    SparseTensorType srcStt = getSparseTensorType(op.getSource());
+    SparseTensorType dstStt = getSparseTensorType(op.getDest());
 
     bool fromSparseConst = false;
-    if (auto constOp = op.getSource().getDefiningOp<arith::ConstantOp>()) {
-      if (dyn_cast<SparseElementsAttr>(constOp.getValue())) {
+    if (auto constOp = op.getSource().getDefiningOp<arith::ConstantOp>())
+      if (dyn_cast<SparseElementsAttr>(constOp.getValue()))
         fromSparseConst = true;
-      }
-    }
 
-    const auto encDst = dstTp.getEncoding();
-    // We don't need a temporary COO tensor if the destination has an identity
-    // ordering. Otherwise, we use the destination ordering for the temporary
-    // COO tensor.
-    // TODO: enhance foreachOp to take ordering to remove the need of a
-    // temporary COO tensor here.
-    const RankedTensorType bufferTp =
-        getBufferType(dstTp, !dstTp.isIdentity() && !fromSparseConst);
-    // Only imposes foreach order on dense constant (which will be statically
-    // sorted by the sparse compiler), otherwise the rotated loop sequence
-    // results to bad cache locality.
     const AffineMapAttr foreachOrder =
-        (!dstTp.isIdentity() && fromSparseConst)
-            ? AffineMapAttr::get(dstTp.getExpandedDimToLvl())
+        (!dstStt.isIdentity() && fromSparseConst)
+            ? AffineMapAttr::get(dstStt.getExpandedDimToLvl())
             : nullptr;
-    // TODO: This assertion is to match the behavior from before we merged
-    // dimOrdering and higherOrdering into dimToLvl.  Although the above
-    // can construct `foreachOrder` for non-permutations, it's not clear
-    // that the `foreachOp` below actually supports non-permutations.
-    assert(!foreachOrder || dstTp.isPermutation());
-
-    auto buffer =
-        rewriter.create<AllocTensorOp>(loc, bufferTp, dynSizes).getResult();
-    auto foreachOp = rewriter.create<ForeachOp>(
-        loc, src, buffer, foreachOrder,
-        [&](OpBuilder &builder, Location loc, ValueRange dcvs, Value v,
-            ValueRange reduc) {
-          Value input = reduc.front();
-          const Dimension dimRank = dstTp.getDimRank();
-          const Level lvlRank = dstTp.getLvlRank();
-          SmallVector<Value> lcvs(lvlRank);
-          for (Dimension d = 0; d < dimRank; d++)
-            // FIXME: `toStoredDim` is deprecated
-            lcvs[toStoredDim(encDst, d)] = dcvs[d];
-          if (fromSparseConst) {
-            input = builder.create<InsertOp>(loc, v, input, lcvs);
-          } else {
-            Value cond = genIsNonzero(builder, loc, v);
-            auto ifOp = builder.create<scf::IfOp>(
-                loc, TypeRange(input.getType()), cond, /*else*/ true);
-            builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
-            Value insert = builder.create<InsertOp>(loc, v, input, lcvs);
-            builder.create<scf::YieldOp>(loc, insert);
-            builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
-            builder.create<scf::YieldOp>(loc, input);
-            builder.setInsertionPointAfter(ifOp);
-            input = ifOp.getResult(0);
-          }
-          builder.create<sparse_tensor::YieldOp>(loc, input);
-        });
-    rewriter.setInsertionPointAfter(op);
-    src = rewriter.create<LoadOp>(loc, foreachOp.getResult(0), true);
-    if (bufferTp != dstTp) {
-      rewriter.replaceOpWithNewOp<ConvertOp>(op, dstTp.getRankedTensorType(),
-                                             src);
-      rewriter.create<DeallocTensorOp>(loc, src);
-    } else {
-      rewriter.replaceOp(op, src);
-    }
-
-    return success();
-  }
 
-  // Handles sparse tensor to dense tensor conversion as follows:
-  //   dst = new dense tensor;
-  //   foreach elemment in src
-  //     dst[element.coords] = element.value
-  LogicalResult sparse2DenseRewrite(ConvertOp op,
-                                    PatternRewriter &rewriter) const {
-    Location loc = op->getLoc();
-    RankedTensorType dstTp = getRankedTensorType(op);
-    Value src = op.getSource();
-    RankedTensorType srcTp = getRankedTensorType(src);
+    bool skipZeroCheck = srcStt.hasEncoding() || fromSparseConst;
 
     SmallVector<Value> sizes;
-    sizesForTensor(rewriter, sizes, loc, srcTp, src);
-
-    Value dst = allocDenseTensor(rewriter, loc, dstTp, sizes);
-
-    rewriter.create<ForeachOp>(loc, src, std::nullopt,
-                               [&](OpBuilder &builder, Location loc,
-                                   ValueRange args, Value v, ValueRange reduc) {
-                                 builder.create<memref::StoreOp>(loc, v, dst,
-                                                                 args);
-                                 builder.create<sparse_tensor::YieldOp>(loc);
-                               });
+    sizesFromSrc(rewriter, sizes, loc, src);
+    ValueRange vs;
+    TensorLike dstBuf(rewriter, loc, dstStt.getRankedTensorType(), sizes);
 
-    rewriter.replaceOpWithNewOp<bufferization::ToTensorOp>(op, dstTp, dst);
-    return success();
-  }
+    auto foreachOp = rewriter.create<ForeachOp>(
+        loc, src, dstBuf.val, foreachOrder,
+        [&](OpBuilder &builder, Location loc, ValueRange dcvs, Value v,
+            ValueRange reduc) {
+          // Enters the loop, update the SSA value for insertion chain.
+          dstBuf.val = reduc.front();
 
-  // Handles sparse tensor to sparse tensor conversion as follows:
-  //   if src is not COO
-  //       construct a COO to represent the src
-  //   sort the src COO
-  //   foreach elemment in the sorted src COO
-  //     insert element to dst
-  LogicalResult sparse2SparseRewrite(ConvertOp op,
-                                     PatternRewriter &rewriter) const {
-    const Location loc = op->getLoc();
-    // These two variables cannot be `const` because they're conditionally
-    // changed below.  Ideally we'd use `SparseTensorType` for `srcRTT`;
-    // however that class's copy-ctor is implicitly deleted.
-    Value src = op.getSource();
-    auto srcRTT = getRankedTensorType(src);
-    const auto dstTp = getSparseTensorType(op);
-    const auto encDst = dstTp.getEncoding();
-    const Level dstLvlRank = dstTp.getLvlRank();
-    const Dimension dimRank = dstTp.getDimRank();
-    // This assertion should be guaranteed by validity of the op,
-    // but just for paranoia's sake.
-    assert(static_cast<Dimension>(srcRTT.getRank()) == dimRank);
+          ValueRange lcvs = dstStt.translateCrds(
+              builder, loc, dcvs, CrdTransDirectionKind::dim2lvl);
 
-    SmallVector<Value> srcSizes;
-    sizesForTensor(rewriter, srcSizes, loc, srcRTT, src);
-    Value tmpCoo = Value();
-    Value nnz = rewriter.create<NumberOfEntriesOp>(loc, src);
-    // We need a tmp COO buffer if and only if
-    // 1. the src tensor is not a COO and
-    // 2. the src tensor is not ordered in the same way as the target
-    // tensor (e.g., src tensor is not ordered or src tensor haves a different
-    // dimToLvl).
-    if (const SparseTensorType srcTp(srcRTT);
-        !(srcTp.isAllOrdered() && srcTp.hasSameDimToLvl(dstTp))) {
-      // Construct a COO tensor from the src tensor.
-      // TODO: there may be cases for which more efficiently without
-      // going through an intermediate COO, such as cases that only change
-      // the overhead types.
-      SmallVector<Value> dynSrcSizes;
-      getDynamicSizes(srcRTT, srcSizes, dynSrcSizes);
-      srcRTT = getCOOType(srcTp.withDimToLvl(dstTp), /*ordered=*/false);
-      // Ensure that mutating `srcRTT` didn't invalidate `dimRank`.
-      assert(static_cast<Dimension>(srcRTT.getRank()) == dimRank);
-      tmpCoo = rewriter
-                   .create<AllocTensorOp>(loc, srcRTT, dynSrcSizes, Value(),
-                                          /*sizeHint=*/nnz, Attribute())
-                   .getResult();
-      auto foreachOp = rewriter.create<ForeachOp>(
-          loc, src, tmpCoo,
-          [&](OpBuilder &builder, Location loc, ValueRange dcvs, Value v,
-              ValueRange reduc) {
-            SmallVector<Value> dstLcvs(dstLvlRank);
-            for (Dimension d = 0; d < dimRank; d++) {
-              // FIXME: `toStoredDim` is deprecated
-              Level l = toStoredDim(encDst, d);
-              dstLcvs[l] = dcvs[d];
-            }
-            auto t = builder.create<InsertOp>(loc, v, reduc.front(), dstLcvs);
-            builder.create<sparse_tensor::YieldOp>(loc, t);
-          });
-      src = rewriter.create<LoadOp>(loc, foreachOp.getResult(0), true);
-    }
-
-    // Now that the conditional is done, we can use `SparseTensorType`.
-    const SparseTensorType srcTp(srcRTT);
-
-    // Only need to sort if the srcTp is not already sorted (we faithfully take
-    // the guarantee from the sparse tensor encoding).
-    if (!srcTp.isAllOrdered()) {
-      // Retrieve the values-array.
-      Value y = genToValues(rewriter, loc, src);
-      const auto encSrc = srcTp.getEncoding();
-      // Builds the dstLvl -> srcLvl permutation maps.
-      SmallVector<AffineExpr> es(dstLvlRank);
-      const Level srcLvlRank = srcTp.getLvlRank();
-      for (Level srcLvl = 0; srcLvl < srcLvlRank; srcLvl++) {
-        // FIXME: `toOrigDim` is deprecated
-        Dimension dim = toOrigDim(encSrc, srcLvl);
-        // FIXME: `toStoredDim` is deprecated
-        Level dstLvl = toStoredDim(encDst, dim);
-        es[dstLvl] = rewriter.getAffineDimExpr(srcLvl);
-      }
-      auto xPerm = AffineMap::get(dstLvlRank, 0, es, rewriter.getContext());
-      assert(xPerm.isPermutation()); // must be a permutation.
+          if (!skipZeroCheck) {
+            Value cond = genIsNonzero(builder, loc, v);
+            auto ifOp = builder.create<scf::IfOp>(loc, reduc.getTypes(), cond,
+                                                  /*else*/ true);
+            builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
+            builder.create<scf::YieldOp>(loc, dstBuf.val);
 
-      Value xs = genToCoordinatesBuffer(rewriter, loc, src);
-      rewriter.create<SortOp>(loc, nnz, xs, ValueRange{y}, xPerm,
-                              rewriter.getIndexAttr(0),
-                              SparseTensorSortKind::HybridQuickSort);
-    }
+            builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
+            dstBuf.insert(builder, loc, v, lcvs);
+            builder.create<scf::YieldOp>(loc, dstBuf.val);
 
-    // For each element in the COO tensor, insert the element to the dst tensor.
-    SmallVector<Value> dynDstSizes;
-    getDynamicSizes(dstTp, srcSizes, dynDstSizes);
-    Value dst = rewriter
-                    .create<AllocTensorOp>(loc, dstTp.getRankedTensorType(),
-                                           dynDstSizes, Value(),
-                                           /*sizeHint=*/nnz, Attribute())
-                    .getResult();
-    SmallVector<Value> dstLcvs(dstLvlRank);
-    auto foreachOp = rewriter.create<ForeachOp>(
-        loc, src, dst,
-        [&](OpBuilder &builder, Location loc, ValueRange dcvs, Value v,
-            ValueRange reduc) {
-          for (Dimension d = 0; d < dimRank; d++) {
-            // FIXME: `toStoredDim` is deprecated
-            Level l = toStoredDim(encDst, d);
-            dstLcvs[l] = dcvs[d];
+            // Exits the ifOp, update the sparse tensor SSA value.
+            builder.setInsertionPointAfter(ifOp);
+            dstBuf.val = ifOp.getResult(0);
+          } else {
+            dstBuf.insert(builder, loc, v, lcvs);
           }
-          auto t = builder.create<InsertOp>(loc, v, reduc.front(), dstLcvs);
-          builder.create<sparse_tensor::YieldOp>(loc, t);
+          builder.create<sparse_tensor::YieldOp>(loc, dstBuf.val);
         });
 
-    // Release the temporary COO if it is created. Note that tmpCoo is
-    // invalidated due to foreach and updated to src.
-    if (tmpCoo)
-      rewriter.create<DeallocTensorOp>(loc, src);
-
-    // Directly replace op with dst results in bufferization error message
-    // "sparse tensor allocation should not escape function".
-    // As such, we insert a trivial tensor convert which will be removed by
-    // codegen.
-    rewriter.setInsertionPointAfter(op);
-    auto t = rewriter.create<LoadOp>(loc, foreachOp.getResult(0), true);
-    rewriter.replaceOpWithNewOp<ConvertOp>(op, dstTp.getRankedTensorType(), t);
+    rewriter.setInsertionPointAfter(foreachOp);
+
+    // Exits the for loop, links the SSA chain.
+    dstBuf.val = foreachOp.getResult(0);
+
+    Value ret = dstBuf.finalize(rewriter, loc, dstStt.getRankedTensorType());
+    rewriter.replaceOp(op, ret);
     return success();
   }
 };
@@ -1275,7 +1069,6 @@ struct ForeachRewriter : public OpRewritePattern<ForeachOp> {
     Value input = op.getTensor();
     SmallVector<Value> reduc = op.getInitArgs();
     const auto stt = getSparseTensorType(input);
-    const Dimension dimRank = stt.getDimRank();
     const Level lvlRank = stt.getLvlRank();
 
     // Special-case: for each over a sparse constant uses its own rewriting
@@ -1309,6 +1102,7 @@ struct ForeachRewriter : public OpRewritePattern<ForeachOp> {
     SmallVector<Value> lcvs = loopEmitter.getLoopIVs();
     if (op.getOrder()) {
       // FIXME: There is some dim/lvl confusion here since `dimRank != lvlRank`
+      const Dimension dimRank = stt.getDimRank();
       SmallVector<Value> dcvs = lcvs; // keep a copy
       for (Dimension d = 0; d < dimRank; d++) {
         auto l = op.getOrder()->getDimPosition(d);
@@ -1326,12 +1120,9 @@ struct ForeachRewriter : public OpRewritePattern<ForeachOp> {
     Block *srcBlock = op.getBody();
 
     // Remap coordinates.
-    SmallVector<Value> args;
-    for (Dimension d = 0; d < dimRank; d++) {
-      // FIXME: `toStoredDim` is deprecated
-      Value dimCrd = lcvs[toStoredDim(enc, d)];
-      args.push_back(dimCrd);
-    }
+    SmallVector<Value> args =
+        enc.translateCrds(rewriter, loc, lcvs, CrdTransDirectionKind::lvl2dim);
+
     // Remap value.
     args.push_back(val);
     // Remap reduction variables.
@@ -1352,7 +1143,7 @@ struct ForeachRewriter : public OpRewritePattern<ForeachOp> {
                                  args);
     }
 
-    for (Dimension d = 0; d < dimRank; d++) {
+    for (Level l = 0; l < lvlRank; l++) {
       // Link the reduction chain. Note that loop emitter update the reducValue
       // in place.
       loopEmitter.exitCurrentLoop(rewriter, loc, reducValue);
@@ -1474,18 +1265,16 @@ void mlir::populatePostSparsificationRewriting(RewritePatternSet &patterns,
                                                bool enableRT,
                                                bool enableForeach,
                                                bool enableConvert) {
-  patterns.add<ConcatenateRewriter, ReshapeRewriter<tensor::ExpandShapeOp>,
+  patterns.add<ConcatenateRewriter, CrdTranslateRewriter,
+               ReshapeRewriter<tensor::ExpandShapeOp>,
                ReshapeRewriter<tensor::CollapseShapeOp>,
                Sparse2SparseReshapeRewriter<tensor::ExpandShapeOp>,
                Sparse2SparseReshapeRewriter<tensor::CollapseShapeOp>,
                TensorReshapeRewriter>(patterns.getContext());
   if (enableForeach)
     patterns.add<ForeachRewriter>(patterns.getContext());
-
-  // TODO: If RT not enabled, rewrite concatenate ops, etc here.
-  if (!enableRT) {
+  if (enableConvert)
+    patterns.add<DirectConvertRewriter>(patterns.getContext());
+  if (!enableRT)
     patterns.add<NewRewriter, OutRewriter>(patterns.getContext());
-    if (enableConvert)
-      patterns.add<ConvertRewriter>(patterns.getContext());
-  }
 }
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparsificationAndBufferizationPass.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparsificationAndBufferizationPass.cpp
index 480e18e257..d8a24ea352 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparsificationAndBufferizationPass.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparsificationAndBufferizationPass.cpp
@@ -63,13 +63,11 @@ class SparsificationAndBufferizationPass
   SparsificationAndBufferizationPass(
       const bufferization::OneShotBufferizationOptions &bufferizationOptions,
       const SparsificationOptions &sparsificationOptions,
-      const SparseTensorConversionOptions &sparseTensorConversionOptions,
       bool createSparseDeallocs, bool enableRuntimeLibrary,
       bool enableBufferInitialization, unsigned vectorLength,
       bool enableVLAVectorization, bool enableSIMDIndex32)
       : bufferizationOptions(bufferizationOptions),
         sparsificationOptions(sparsificationOptions),
-        sparseTensorConversionOptions(sparseTensorConversionOptions),
         createSparseDeallocs(createSparseDeallocs),
         enableRuntimeLibrary(enableRuntimeLibrary),
         enableBufferInitialization(enableBufferInitialization),
@@ -142,6 +140,7 @@ class SparsificationAndBufferizationPass
     {
       OpPassManager pm("builtin.module");
       pm.addPass(createSparsificationPass(sparsificationOptions));
+      pm.addNestedPass<func::FuncOp>(createStageSparseOperationsPass());
       pm.addPass(createPostSparsificationRewritePass(enableRuntimeLibrary));
       if (vectorLength > 0) {
         pm.addPass(mlir::createLoopInvariantCodeMotionPass());
@@ -149,8 +148,7 @@ class SparsificationAndBufferizationPass
             vectorLength, enableVLAVectorization, enableSIMDIndex32));
       }
       if (enableRuntimeLibrary) {
-        pm.addPass(
-            createSparseTensorConversionPass(sparseTensorConversionOptions));
+        pm.addPass(createSparseTensorConversionPass());
       } else {
         pm.addPass(createSparseTensorCodegenPass(createSparseDeallocs,
                                                  enableBufferInitialization));
@@ -168,7 +166,6 @@ class SparsificationAndBufferizationPass
 private:
   bufferization::OneShotBufferizationOptions bufferizationOptions;
   SparsificationOptions sparsificationOptions;
-  SparseTensorConversionOptions sparseTensorConversionOptions;
   bool createSparseDeallocs;
   bool enableRuntimeLibrary;
   bool enableBufferInitialization;
@@ -200,10 +197,9 @@ mlir::getBufferizationOptionsForSparsification(bool analysisOnly) {
 
 std::unique_ptr<mlir::Pass> mlir::createSparsificationAndBufferizationPass() {
   SparsificationOptions sparseOptions;
-  SparseTensorConversionOptions convOptions;
   return createSparsificationAndBufferizationPass(
       getBufferizationOptionsForSparsification(/*analysisOnly=*/false),
-      sparseOptions, convOptions,
+      sparseOptions,
       /*createSparseDeallocs=*/false,
       /*enableRuntimeLibrary=*/false,
       /*enableBufferInitialization=*/false,
@@ -215,14 +211,12 @@ std::unique_ptr<mlir::Pass> mlir::createSparsificationAndBufferizationPass() {
 std::unique_ptr<mlir::Pass> mlir::createSparsificationAndBufferizationPass(
     const bufferization::OneShotBufferizationOptions &bufferizationOptions,
     const SparsificationOptions &sparsificationOptions,
-    const SparseTensorConversionOptions &sparseTensorConversionOptions,
     bool createSparseDeallocs, bool enableRuntimeLibrary,
     bool enableBufferInitialization, unsigned vectorLength,
     bool enableVLAVectorization, bool enableSIMDIndex32) {
   return std::make_unique<
       mlir::sparse_tensor::SparsificationAndBufferizationPass>(
-      bufferizationOptions, sparsificationOptions,
-      sparseTensorConversionOptions, createSparseDeallocs, enableRuntimeLibrary,
-      enableBufferInitialization, vectorLength, enableVLAVectorization,
-      enableSIMDIndex32);
+      bufferizationOptions, sparsificationOptions, createSparseDeallocs,
+      enableRuntimeLibrary, enableBufferInitialization, vectorLength,
+      enableVLAVectorization, enableSIMDIndex32);
 }
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/StageSparseOperations.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/StageSparseOperations.cpp
index 4adc4d1311..5875cd4f9f 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/StageSparseOperations.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/StageSparseOperations.cpp
@@ -1,4 +1,33 @@
+//===- StageSparseOperations.cpp - stage sparse ops rewriting rules -------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/SparseTensor/IR/SparseTensor.h"
+#include "mlir/Dialect/SparseTensor/IR/SparseTensorType.h"
 #include "mlir/Dialect/SparseTensor/Transforms/Passes.h"
 
-void mlir::populateStageSparseOperationsPatterns(
-    RewritePatternSet & /*patterns*/) {}
+using namespace mlir;
+using namespace mlir::sparse_tensor;
+
+namespace {
+
+template <typename StageWithSortOp>
+struct StageUnorderedSparseOps : public OpRewritePattern<StageWithSortOp> {
+  using OpRewritePattern<StageWithSortOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(StageWithSortOp op,
+                                PatternRewriter &rewriter) const override {
+    return llvm::cast<StageWithSortSparseOp>(op.getOperation())
+        .stageWithSort(rewriter);
+  }
+};
+} // namespace
+
+void mlir::populateStageSparseOperationsPatterns(RewritePatternSet &patterns) {
+  patterns.add<StageUnorderedSparseOps<ConvertOp>,
+               StageUnorderedSparseOps<ConcatenateOp>>(patterns.getContext());
+}
diff --git a/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp b/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
index e69c40f2b0..7444f70a46 100644
--- a/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
+++ b/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
@@ -771,7 +771,7 @@ OpFoldResult ConstOp::fold(FoldAdaptor adaptor) { return getValueAttr(); }
     ShapedType inputTy = llvm::cast<ShapedType>(getInput().getType());         \
     if (!inputTy.hasRank())                                                    \
       return {};                                                               \
-    if (inputTy.getDimSize(getAxis()) == 1)                                    \
+    if (inputTy.getRank() == 0 || inputTy.getDimSize(getAxis()) == 1)          \
       return getInput();                                                       \
     return {};                                                                 \
   }
@@ -874,7 +874,8 @@ OpFoldResult ReverseOp::fold(FoldAdaptor adaptor) {
     return operandAttr;
 
   // If the dim-length is 1, tosa.reverse is a no-op.
-  if (operandTy.hasRank() && operandTy.getDimSize(axis) == 1)
+  if (operandTy.hasRank() &&
+      (operandTy.getRank() == 0 || operandTy.getDimSize(axis) == 1))
     return operand;
 
   return {};
diff --git a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
index a719171b2b..e03904a161 100644
--- a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
+++ b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
@@ -146,6 +146,49 @@ Operation *TosaDialect::materializeConstant(OpBuilder &builder, Attribute value,
   return nullptr;
 }
 
+//===----------------------------------------------------------------------===//
+// Parsers and printers
+//===----------------------------------------------------------------------===//
+
+ParseResult mlir::tosa::parseTypeOrAttr(OpAsmParser &parser, TypeAttr &typeAttr,
+                                        Attribute &attr) {
+  if (succeeded(parser.parseOptionalEqual())) {
+    if (failed(parser.parseAttribute(attr))) {
+      return parser.emitError(parser.getCurrentLocation())
+             << "expected attribute";
+    }
+    if (auto typedAttr = attr.dyn_cast<TypedAttr>()) {
+      typeAttr = TypeAttr::get(typedAttr.getType());
+    }
+    return success();
+  }
+
+  Type type;
+  if (failed(parser.parseColonType(type))) {
+    return parser.emitError(parser.getCurrentLocation()) << "expected type";
+  }
+  typeAttr = TypeAttr::get(type);
+
+  return success();
+}
+
+void mlir::tosa::printTypeOrAttr(OpAsmPrinter &p, Operation *op, TypeAttr type,
+                                 Attribute attr) {
+  bool needsSpace = false;
+  auto typedAttr = attr.dyn_cast_or_null<TypedAttr>();
+  if (!typedAttr || typedAttr.getType() != type.getValue()) {
+    p << ": ";
+    p.printAttribute(type);
+    needsSpace = true; // subsequent attr value needs a space separator
+  }
+  if (attr) {
+    if (needsSpace)
+      p << ' ';
+    p << "= ";
+    p.printAttribute(attr);
+  }
+}
+
 //===----------------------------------------------------------------------===//
 // TOSA Operator Verifiers.
 //===----------------------------------------------------------------------===//
@@ -247,23 +290,59 @@ LogicalResult tosa::AvgPool2dOp::verify() {
   if (llvm::isa<IntegerType>(inputETy) && !accType.isInteger(32))
     return emitOpError("accumulator type for integer tensor is not i32");
 
-  if ((inputETy.isBF16() || inputETy.isF16()) &&
-      !(accType.isF16() || accType.isF32()))
-    return emitOpError("accumulator type for f16/bf16 tensor is not f16/f32");
+  if (inputETy.isF16() && !(accType.isF16() || accType.isF32()))
+    return emitOpError("accumulator type for f16 tensor is not f16/f32");
+
+  if (inputETy.isBF16() && !accType.isF32())
+    return emitOpError("accumulator type for bf16 tensor is not f32");
 
   if (inputETy.isF32() && !accType.isF32())
     return emitOpError("accumulator type for f32 tensor is not f32");
 
-  if (inputETy.isF32() && resultETy.isF32())
-    return success();
-  if (inputETy.isInteger(8) && resultETy.isInteger(8))
-    return success();
-  if (inputETy.isInteger(16) && resultETy.isInteger(16))
+  if ((inputETy.isF32() && resultETy.isF32()) ||
+      (inputETy.isF16() && resultETy.isF16()) ||
+      (inputETy.isBF16() && resultETy.isBF16()) ||
+      (inputETy.isInteger(8) && resultETy.isInteger(8)) ||
+      (inputETy.isInteger(16) && resultETy.isInteger(16)))
     return success();
 
   return emitOpError("input/output element types are incompatible.");
 }
 
+LogicalResult tosa::ClampOp::verify() {
+  mlir::Type inputETy =
+      llvm::cast<ShapedType>(getInput().getType()).getElementType();
+  if (auto quantType =
+          llvm::dyn_cast<mlir::quant::UniformQuantizedType>(inputETy)) {
+    inputETy = quantType.getStorageType();
+  }
+  mlir::Type maxFpType = getMaxFpAttr().getType();
+  mlir::Type minFpType = getMinFpAttr().getType();
+  mlir::Type outputETy =
+      llvm::cast<ShapedType>(getOutput().getType()).getElementType();
+  if (auto quantType =
+          llvm::dyn_cast<mlir::quant::UniformQuantizedType>(outputETy)) {
+    outputETy = quantType.getStorageType();
+  }
+  unsigned dataTypeBitWidth = inputETy.getIntOrFloatBitWidth();
+
+  if (inputETy != outputETy)
+    return emitOpError("input/output element types are incompatible.");
+
+  // if input datatype is float, check that the two min/max_fp attributes share
+  // the same type and that their type is either the same of the input's
+  // datatype, or a float type whose bitwidth > input datatype bitwidth
+  if (!inputETy.isInteger(dataTypeBitWidth)) {
+    if (((maxFpType != minFpType) ||
+         (maxFpType != inputETy && maxFpType.getIntOrFloatBitWidth() <=
+                                       inputETy.getIntOrFloatBitWidth())))
+      return emitOpError("min/max attributes types are incompatible with "
+                         "input/output element types.");
+  }
+
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // TOSA Operator Quantization Builders.
 //===----------------------------------------------------------------------===//
@@ -1030,7 +1109,7 @@ LogicalResult tosa::ScatterOp::inferReturnTypeComponents(
 static LogicalResult ReduceInferReturnTypes(
     ShapeAdaptor operandShape, Type inputType, IntegerAttr axis,
     SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
-  if (!operandShape.hasRank()) {
+  if (!operandShape.hasRank() || operandShape.getRank() == 0) {
     inferredReturnShapes.push_back(ShapedTypeComponents(inputType));
     return success();
   }
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaFolders.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaFolders.cpp
index 0988759b82..d35e911ebe 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaFolders.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaFolders.cpp
@@ -350,6 +350,11 @@ llvm::APInt calculateReducedValue(const mlir::ElementsAttr &oldTensorAttr,
 template <typename OperationType>
 struct ReduceConstantOptimization : public OpRewritePattern<OperationType> {
 
+  ReduceConstantOptimization(MLIRContext *context,
+                             bool aggressiveReduceConstant)
+      : OpRewritePattern<OperationType>(context),
+        aggressiveReduceConstant(aggressiveReduceConstant) {}
+
   using OpRewritePattern<OperationType>::OpRewritePattern;
 
   LogicalResult matchAndRewrite(OperationType op,
@@ -361,7 +366,7 @@ struct ReduceConstantOptimization : public OpRewritePattern<OperationType> {
       return rewriter.notifyMatchFailure(
           op, "reduce input must be const operation");
 
-    if (!inputOp.hasOneUse())
+    if (!inputOp.hasOneUse() && !this->aggressiveReduceConstant)
       return rewriter.notifyMatchFailure(
           op, "input operation has more than one user");
 
@@ -400,18 +405,26 @@ struct ReduceConstantOptimization : public OpRewritePattern<OperationType> {
     rewriter.replaceOpWithNewOp<tosa::ConstOp>(op, rankedTensorType, denseAttr);
     return success();
   }
+  const bool aggressiveReduceConstant;
 };
 
 } // namespace
 
 void mlir::tosa::populateTosaConstantReduction(MLIRContext *ctx,
-                                               RewritePatternSet &patterns) {
-  patterns.add<ReduceConstantOptimization<ReduceAllOp>>(ctx);
-  patterns.add<ReduceConstantOptimization<ReduceAnyOp>>(ctx);
-  patterns.add<ReduceConstantOptimization<ReduceMaxOp>>(ctx);
-  patterns.add<ReduceConstantOptimization<ReduceMinOp>>(ctx);
-  patterns.add<ReduceConstantOptimization<ReduceProdOp>>(ctx);
-  patterns.add<ReduceConstantOptimization<ReduceSumOp>>(ctx);
+                                               RewritePatternSet &patterns,
+                                               bool aggressiveReduceConstant) {
+  patterns.add<ReduceConstantOptimization<ReduceAllOp>>(
+      ctx, aggressiveReduceConstant);
+  patterns.add<ReduceConstantOptimization<ReduceAnyOp>>(
+      ctx, aggressiveReduceConstant);
+  patterns.add<ReduceConstantOptimization<ReduceMaxOp>>(
+      ctx, aggressiveReduceConstant);
+  patterns.add<ReduceConstantOptimization<ReduceMinOp>>(
+      ctx, aggressiveReduceConstant);
+  patterns.add<ReduceConstantOptimization<ReduceProdOp>>(
+      ctx, aggressiveReduceConstant);
+  patterns.add<ReduceConstantOptimization<ReduceSumOp>>(
+      ctx, aggressiveReduceConstant);
 }
 
 void mlir::tosa::populateTosaFoldConstantTransposePatterns(
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaLayerwiseConstantFoldPass.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaLayerwiseConstantFoldPass.cpp
index 90f15faf01..e1400f0c90 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaLayerwiseConstantFoldPass.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaLayerwiseConstantFoldPass.cpp
@@ -45,6 +45,10 @@ void populateTosaOpsCanonicalizationPatterns(MLIRContext *ctx,
 struct TosaLayerwiseConstantFoldPass
     : public tosa::impl::TosaLayerwiseConstantFoldPassBase<
           TosaLayerwiseConstantFoldPass> {
+  TosaLayerwiseConstantFoldPass(
+      const TosaLayerwiseConstantFoldPassOptions &options)
+      : TosaLayerwiseConstantFoldPassBase(options) {}
+
   void runOnOperation() override {
     auto *ctx = &getContext();
     RewritePatternSet patterns(ctx);
@@ -52,7 +56,8 @@ struct TosaLayerwiseConstantFoldPass
 
     mlir::tosa::populateTosaFoldConstantReciprocalPatterns(ctx, patterns);
     mlir::tosa::populateTosaFoldConstantTransposePatterns(ctx, patterns);
-    mlir::tosa::populateTosaConstantReduction(ctx, patterns);
+    mlir::tosa::populateTosaConstantReduction(ctx, patterns,
+                                              aggressiveReduceConstant);
     populateTosaOpsCanonicalizationPatterns(ctx, patterns);
 
     if (applyPatternsAndFoldGreedily(func, std::move(patterns)).failed())
@@ -63,5 +68,11 @@ struct TosaLayerwiseConstantFoldPass
 } // namespace
 
 std::unique_ptr<Pass> mlir::tosa::createTosaLayerwiseConstantFoldPass() {
-  return std::make_unique<TosaLayerwiseConstantFoldPass>();
+  return std::make_unique<TosaLayerwiseConstantFoldPass>(
+      TosaLayerwiseConstantFoldPassOptions{false});
+}
+
+std::unique_ptr<Pass> mlir::tosa::createTosaLayerwiseConstantFoldPass(
+    const TosaLayerwiseConstantFoldPassOptions &options) {
+  return std::make_unique<TosaLayerwiseConstantFoldPass>(options);
 }
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaMakeBroadcastable.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaMakeBroadcastable.cpp
index 94cbb0afd2..18bc7d6aa9 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaMakeBroadcastable.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaMakeBroadcastable.cpp
@@ -54,24 +54,24 @@ LogicalResult reshapeLowerToHigher(PatternRewriter &rewriter, Location loc,
     return rewriter.notifyMatchFailure(loc,
                                        "cannot rewrite as its already correct");
 
-  Value input1_copy = input1;
-  Value input2_copy = input2;
-  if (EqualizeRanks(rewriter, loc, input1_copy, input2_copy).failed()) {
+  Value input1Copy = input1;
+  Value input2Copy = input2;
+  if (EqualizeRanks(rewriter, loc, input1Copy, input2Copy).failed()) {
     return rewriter.notifyMatchFailure(loc, "failed to reshape inputs");
   }
 
   // Verify the rank agrees with the output type if the output type is ranked.
   if (outputType) {
     if (outputType.getRank() !=
-            llvm::cast<RankedTensorType>(input1_copy.getType()).getRank() ||
+            llvm::cast<RankedTensorType>(input1Copy.getType()).getRank() ||
         outputType.getRank() !=
-            llvm::cast<RankedTensorType>(input2_copy.getType()).getRank())
+            llvm::cast<RankedTensorType>(input2Copy.getType()).getRank())
       return rewriter.notifyMatchFailure(
           loc, "the reshaped type doesn't agrees with the ranked output type");
   }
 
-  input1 = input1_copy;
-  input2 = input2_copy;
+  input1 = input1Copy;
+  input2 = input2Copy;
 
   return success();
 }
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaValidation.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaValidation.cpp
index 52885e69c3..8a2254fc24 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaValidation.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaValidation.cpp
@@ -14,6 +14,9 @@
 #include "mlir/Dialect/Tosa/Transforms/Passes.h"
 #include "mlir/Dialect/Tosa/Transforms/PassesEnums.cpp.inc"
 
+#include <string>
+#include <unordered_map>
+
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/Tosa/IR/TosaOps.h"
 #include "mlir/IR/Builders.h"
@@ -36,43 +39,43 @@ using namespace mlir::tosa;
 namespace {
 
 static LogicalResult checkConstantOperandPad(Operation *op) {
-  if (auto pad_op = dyn_cast<tosa::PadOp>(op)) {
+  if (auto padOp = dyn_cast<tosa::PadOp>(op)) {
     DenseElementsAttr paddings;
-    if (!matchPattern(pad_op.getPadding(), m_Constant(&paddings)))
+    if (!matchPattern(padOp.getPadding(), m_Constant(&paddings)))
       return op->emitOpError("padding of pad is not constant");
 
-    DenseElementsAttr pad_const;
-    // Assume this op is zero-padding if pad_const is not presented.
-    if (pad_op.getPadConst() &&
-        !matchPattern(pad_op.getPadConst(), m_Constant(&pad_const)))
+    DenseElementsAttr padConst;
+    // Assume this op is zero-padding if padConst is not presented.
+    if (padOp.getPadConst() &&
+        !matchPattern(padOp.getPadConst(), m_Constant(&padConst)))
       return op->emitOpError("pad_const of pad is not constant");
   }
   return success();
 }
 
 static LogicalResult checkConstantOperandTranspose(Operation *op) {
-  if (auto transpose_op = dyn_cast<tosa::TransposeOp>(op)) {
+  if (auto transposeOp = dyn_cast<tosa::TransposeOp>(op)) {
     DenseElementsAttr perms;
-    if (!matchPattern(transpose_op.getPerms(), m_Constant(&perms)))
+    if (!matchPattern(transposeOp.getPerms(), m_Constant(&perms)))
       return op->emitOpError("perms of transpose is not constant");
   }
   return success();
 }
 
 static LogicalResult checkConstantOperandFullyConnected(Operation *op) {
-  if (auto fc_op = dyn_cast<tosa::FullyConnectedOp>(op)) {
+  if (auto fcOp = dyn_cast<tosa::FullyConnectedOp>(op)) {
     DenseElementsAttr weight;
-    if (!matchPattern(fc_op.getWeight(), m_Constant(&weight)))
+    if (!matchPattern(fcOp.getWeight(), m_Constant(&weight)))
       return op->emitOpError("weight of fully_connected is not constant");
 
     DenseElementsAttr bias;
-    if (!matchPattern(fc_op.getBias(), m_Constant(&bias)))
+    if (!matchPattern(fcOp.getBias(), m_Constant(&bias)))
       return op->emitOpError("bias of fully_connected is not constant");
   }
   return success();
 }
 
-struct tosa_level_t {
+struct TosaLevel {
   int32_t MAX_RANK = 0;
   int32_t MAX_KERNEL = 0;
   int32_t MAX_STRIDE = 0;
@@ -80,14 +83,14 @@ struct tosa_level_t {
 
   // @todo: MAX_LOG2_SIZE value and checks
 
-  bool operator==(const tosa_level_t &rhs) {
+  bool operator==(const TosaLevel &rhs) {
     return MAX_RANK == rhs.MAX_RANK && MAX_KERNEL == rhs.MAX_KERNEL &&
            MAX_STRIDE == rhs.MAX_STRIDE && MAX_SCALE == rhs.MAX_SCALE;
   }
 };
 
-static constexpr tosa_level_t TOSA_LEVEL_EIGHTK = {6, 8192, 8192, 256};
-static constexpr tosa_level_t TOSA_LEVEL_NONE = {0, 0, 0, 0};
+static constexpr TosaLevel TOSA_LEVEL_EIGHTK = {6, 8192, 8192, 256};
+static constexpr TosaLevel TOSA_LEVEL_NONE = {0, 0, 0, 0};
 
 //===----------------------------------------------------------------------===//
 // TOSA Validation Pass.
@@ -96,15 +99,16 @@ static constexpr tosa_level_t TOSA_LEVEL_NONE = {0, 0, 0, 0};
 struct TosaValidation : public tosa::impl::TosaValidationBase<TosaValidation> {
 public:
   explicit TosaValidation() { populateConstantOperandChecks(); }
-  explicit TosaValidation(const ValidationOptions &options) : TosaValidation() {
+  explicit TosaValidation(const TosaValidationOptions &options)
+      : TosaValidation() {
     this->profile = options.profile;
-    this->StrictOperationSpecAlignment = options.strictOperationSpecAlignment;
+    this->StrictOperationSpecAlignment = options.StrictOperationSpecAlignment;
     this->level = options.level;
   }
-  void runOnOperation() override;
+  void runOnOperation() final;
 
   LogicalResult applyConstantOperandCheck(Operation *op) {
-    for (auto &checker : const_checkers) {
+    for (auto &checker : constCheckers) {
       if (failed(checker(op)))
         return failure();
     }
@@ -113,45 +117,47 @@ struct TosaValidation : public tosa::impl::TosaValidationBase<TosaValidation> {
 
   LogicalResult applyLevelCheck(Operation *op);
 
+  // check variable read/write data types against variable declarations
+  LogicalResult applyVariableCheck(Operation *op);
+
 private:
   void populateConstantOperandChecks() {
-    const_checkers.emplace_back(checkConstantOperandPad);
-    const_checkers.emplace_back(checkConstantOperandTranspose);
-    const_checkers.emplace_back(checkConstantOperandFullyConnected);
+    constCheckers.emplace_back(checkConstantOperandPad);
+    constCheckers.emplace_back(checkConstantOperandTranspose);
+    constCheckers.emplace_back(checkConstantOperandFullyConnected);
   }
 
   bool levelCheckKernel(Operation *op, int32_t v,
-                        const std::string &check_desc) {
-    if (v > tosa_level.MAX_KERNEL) {
-      op->emitOpError() << "failed level check: " << check_desc;
+                        const std::string &checkDesc) {
+    if (v > tosaLevel.MAX_KERNEL) {
+      op->emitOpError() << "failed level check: " << checkDesc;
       return false;
     }
     return true;
   }
 
   bool levelCheckStride(Operation *op, int32_t v,
-                        const std::string &check_desc) {
-    if (v > tosa_level.MAX_STRIDE) {
-      op->emitOpError() << "failed level check: " << check_desc;
+                        const std::string &checkDesc) {
+    if (v > tosaLevel.MAX_STRIDE) {
+      op->emitOpError() << "failed level check: " << checkDesc;
       return false;
     }
     return true;
   }
 
-  bool levelCheckScale(Operation *op, int32_t v,
-                       const std::string &check_desc) {
-    if (v > tosa_level.MAX_SCALE) {
-      op->emitOpError() << "failed level check: " << check_desc;
+  bool levelCheckScale(Operation *op, int32_t v, const std::string &checkDesc) {
+    if (v > tosaLevel.MAX_SCALE) {
+      op->emitOpError() << "failed level check: " << checkDesc;
       return false;
     }
     return true;
   }
 
   bool levelCheckRank(Operation *op, const Value &v,
-                      const std::string &check_desc) {
+                      const std::string &checkDesc) {
     if (ShapedType type = dyn_cast<ShapedType>(v.getType())) {
-      if (type.getRank() > tosa_level.MAX_RANK) {
-        op->emitOpError() << "failed level check: " << check_desc;
+      if (type.getRank() > tosaLevel.MAX_RANK) {
+        op->emitOpError() << "failed level check: " << checkDesc;
         return false;
       }
     }
@@ -175,8 +181,8 @@ struct TosaValidation : public tosa::impl::TosaValidationBase<TosaValidation> {
   }
 
   bool levelCheckRanks(Operation *op) {
-#define CHECK_RANKS_FOR(tosa_op)                                               \
-  if (!levelCheckRanksFor<tosa_op##Op>(op))                                    \
+#define CHECK_RANKS_FOR(tosaOp)                                                \
+  if (!levelCheckRanksFor<tosaOp##Op>(op))                                     \
     return false;
 
     // tensor operators:
@@ -250,18 +256,18 @@ struct TosaValidation : public tosa::impl::TosaValidationBase<TosaValidation> {
   // Pool Op: level check kernel/stride/pad values
   template <typename T>
   bool levelCheckPool(Operation *op) {
-    if (auto pool_op = dyn_cast<T>(op)) {
-      for (auto k : pool_op.getKernel()) {
+    if (auto poolOp = dyn_cast<T>(op)) {
+      for (auto k : poolOp.getKernel()) {
         if (!levelCheckKernel(op, k, "kernel <= MAX_KERNEL")) {
           return false;
         }
       }
-      for (auto s : pool_op.getStride()) {
+      for (auto s : poolOp.getStride()) {
         if (!levelCheckStride(op, s, "stride <= MAX_STRIDE")) {
           return false;
         }
       }
-      for (auto p : pool_op.getPad()) {
+      for (auto p : poolOp.getPad()) {
         if (!levelCheckKernel(op, p, "pad <= MAX_KERNEL")) {
           return false;
         }
@@ -273,27 +279,27 @@ struct TosaValidation : public tosa::impl::TosaValidationBase<TosaValidation> {
   // Conv Op: level check dilation/stride/pad values
   template <typename T>
   bool levelCheckConv(Operation *op) {
-    if (auto conv_op = dyn_cast<T>(op)) {
+    if (auto convOp = dyn_cast<T>(op)) {
 
-      for (auto k : conv_op.getDilation()) {
+      for (auto k : convOp.getDilation()) {
         if (!levelCheckKernel(op, k, "dilation <= MAX_KERNEL")) {
           return false;
         }
       }
-      for (auto p : conv_op.getPad()) {
+      for (auto p : convOp.getPad()) {
         if (!levelCheckKernel(op, p, "pad <= MAX_KERNEL")) {
           return false;
         }
       }
-      for (auto s : conv_op.getStride()) {
+      for (auto s : convOp.getStride()) {
         if (!levelCheckStride(op, s, "stride <= MAX_STRIDE")) {
           return false;
         }
       }
-      auto dilation = conv_op.getDilation();
-      if (ShapedType weight_type =
+      auto dilation = convOp.getDilation();
+      if (ShapedType weightType =
               dyn_cast<ShapedType>(op->getOperand(1).getType())) {
-        auto shape = weight_type.getShape();
+        auto shape = weightType.getShape();
         if (isa<tosa::Conv2DOp>(op)) {
           assert(shape.size() == 4);
           assert(dilation.size() == 2);
@@ -347,9 +353,9 @@ struct TosaValidation : public tosa::impl::TosaValidationBase<TosaValidation> {
   // TransposeConv2d op: level check kH/kW, outpad, and stride
   bool levelCheckTransposeConv2d(Operation *op) {
     if (auto transpose = dyn_cast<tosa::TransposeConv2DOp>(op)) {
-      if (ShapedType filter_type =
+      if (ShapedType filterType =
               transpose.getFilter().getType().dyn_cast<ShapedType>()) {
-        auto shape = filter_type.getShape();
+        auto shape = filterType.getShape();
         assert(shape.size() == 4);
         // level check kernel sizes for kH and KW
         if (!levelCheckKernel(op, shape[1], "KH <= MAX_KERNEL") ||
@@ -375,13 +381,13 @@ struct TosaValidation : public tosa::impl::TosaValidationBase<TosaValidation> {
   bool levelCheckResize(Operation *op) {
     if (auto resize = dyn_cast<tosa::ResizeOp>(op)) {
       auto scale = resize.getScale();
-      int16_t scale_y_n = scale[0];
-      int16_t scale_y_d = scale[1];
-      int16_t scale_x_n = scale[2];
-      int16_t scale_x_d = scale[3];
-      if (!levelCheckScale(op, scale_y_n / scale_y_d,
+      int16_t scaleYN = scale[0];
+      int16_t scaleYD = scale[1];
+      int16_t scaleXN = scale[2];
+      int16_t scaleXD = scale[3];
+      if (!levelCheckScale(op, scaleYN / scaleYD,
                            "scale_y_n/scale_y_d <= MAX_SCALE") ||
-          !levelCheckScale(op, scale_x_n / scale_x_d,
+          !levelCheckScale(op, scaleXN / scaleXD,
                            "scale_x_n/scale_x_d <= MAX_SCALE")) {
         return false;
       }
@@ -392,18 +398,22 @@ struct TosaValidation : public tosa::impl::TosaValidationBase<TosaValidation> {
   // configure profile and level values from pass options profileName and
   // levelName
   void configLevelAndProfile() {
-    tosa_level = TOSA_LEVEL_NONE;
+    tosaLevel = TOSA_LEVEL_NONE;
     if (level == TosaLevelEnum::EightK) {
-      tosa_level = TOSA_LEVEL_EIGHTK;
+      tosaLevel = TOSA_LEVEL_EIGHTK;
     }
   }
 
-  SmallVector<std::function<LogicalResult(Operation *)>> const_checkers;
-  tosa_level_t tosa_level;
+  bool CheckVariable(Operation *op);
+  bool CheckVariableReadOrWrite(Operation *op);
+
+  SmallVector<std::function<LogicalResult(Operation *)>> constCheckers;
+  TosaLevel tosaLevel;
+  DenseMap<StringAttr, mlir::Type> variablesMap;
 };
 
 LogicalResult TosaValidation::applyLevelCheck(Operation *op) {
-  if (tosa_level == TOSA_LEVEL_NONE) {
+  if (tosaLevel == TOSA_LEVEL_NONE) {
     // no need to do level checks
     return success();
   }
@@ -427,6 +437,69 @@ LogicalResult TosaValidation::applyLevelCheck(Operation *op) {
   return success();
 }
 
+inline bool CompatibleTypes(const mlir::Type &type,
+                            const mlir::Type &declaredType) {
+  // for now, simply use type equality comparison
+  return type == declaredType;
+}
+
+bool TosaValidation::CheckVariable(Operation *op) {
+  if (isa<mlir::tosa::VariableOp>(op)) {
+    auto nameAttr = cast<mlir::StringAttr>(op->getAttr("name"));
+
+    if (variablesMap.count(nameAttr)) {
+      op->emitOpError() << "name has already been declared";
+      return false;
+    }
+
+    auto typeAttr = cast<mlir::TypeAttr>(op->getAttr("type"));
+    mlir::Type type = typeAttr.getValue();
+
+    variablesMap[nameAttr] = type;
+  }
+
+  return true;
+}
+
+bool TosaValidation::CheckVariableReadOrWrite(Operation *op) {
+  if (isa<mlir::tosa::VariableReadOp>(op) ||
+      isa<mlir::tosa::VariableWriteOp>(op)) {
+    auto nameAttr = cast<mlir::StringAttr>(op->getAttr("name"));
+
+    if (!variablesMap.count(nameAttr)) {
+      op->emitOpError() << "name has not been declared";
+      return false;
+    }
+
+    auto varType = variablesMap[nameAttr];
+
+    for (auto v : op->getOperands()) {
+      auto type = v.getType();
+      if (!CompatibleTypes(type, varType)) {
+        op->emitOpError() << "operand type does not equal variable type";
+        return false;
+      }
+    }
+
+    for (auto v : op->getResults()) {
+      auto type = v.getType();
+      if (!CompatibleTypes(type, varType)) {
+        op->emitOpError() << "result type does not equal variable type";
+        return false;
+      }
+    }
+  }
+
+  return true;
+}
+
+LogicalResult TosaValidation::applyVariableCheck(Operation *op) {
+  if (!CheckVariable(op) || !CheckVariableReadOrWrite(op)) {
+    return failure();
+  }
+  return success();
+}
+
 void TosaValidation::runOnOperation() {
   configLevelAndProfile();
   getOperation().walk([&](Operation *op) {
@@ -440,18 +513,18 @@ void TosaValidation::runOnOperation() {
       }
     }
 
-    // Some uses of TOSA rely on the constant operands of particular operations.
+    // Some uses of TOSA rely on the constant operands of particular
+    // operations.
     if (StrictOperationSpecAlignment && failed(applyConstantOperandCheck(op)))
       signalPassFailure();
 
     // do level checks
     if (failed(applyLevelCheck(op)))
       signalPassFailure();
+
+    // do variable type checks
+    if (failed(applyVariableCheck(op)))
+      signalPassFailure();
   });
 }
 } // namespace
-
-std::unique_ptr<Pass>
-mlir::tosa::createTosaValidationPass(ValidationOptions const &options) {
-  return std::make_unique<TosaValidation>(options);
-}
diff --git a/mlir/lib/Dialect/Tosa/Utils/QuantUtils.cpp b/mlir/lib/Dialect/Tosa/Utils/QuantUtils.cpp
index eb81446caa..5c546f59cd 100644
--- a/mlir/lib/Dialect/Tosa/Utils/QuantUtils.cpp
+++ b/mlir/lib/Dialect/Tosa/Utils/QuantUtils.cpp
@@ -107,10 +107,10 @@ void mlir::tosa::computeMultiplierAndShift(double scale, int32_t &multiplier,
   }
 }
 
-#define GET_UQTYPE(input_type)                                                 \
-  (llvm::dyn_cast<quant::UniformQuantizedType>((input_type).getElementType()))
-#define GET_QTYPE(input_type)                                                  \
-  (llvm::dyn_cast<quant::QuantizedType>((input_type).getElementType()))
+#define GET_UQTYPE(inputType)                                                  \
+  (llvm::dyn_cast<quant::UniformQuantizedType>((inputType).getElementType()))
+#define GET_QTYPE(inputType)                                                   \
+  (llvm::dyn_cast<quant::QuantizedType>((inputType).getElementType()))
 
 /// Method to build ConvOpQuantizationAttr, called from
 /// ConvOpQuantInfoBuilder/TransConvOpQuantInfoBuilder:
diff --git a/mlir/lib/Dialect/Transform/IR/TransformInterfaces.cpp b/mlir/lib/Dialect/Transform/IR/TransformInterfaces.cpp
index 4f88b8522e..8f860c4e54 100644
--- a/mlir/lib/Dialect/Transform/IR/TransformInterfaces.cpp
+++ b/mlir/lib/Dialect/Transform/IR/TransformInterfaces.cpp
@@ -60,9 +60,9 @@ ArrayRef<Operation *>
 transform::TransformState::getPayloadOpsView(Value value) const {
   const TransformOpMapping &operationMapping = getMapping(value).direct;
   auto iter = operationMapping.find(value);
-  assert(
-      iter != operationMapping.end() &&
-      "cannot find mapping for payload handle (param/value handle provided?)");
+  assert(iter != operationMapping.end() &&
+         "cannot find mapping for payload handle (param/value handle "
+         "provided?)");
   return iter->getSecond();
 }
 
diff --git a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
index f8eb20e1a1..7b1badd0ad 100644
--- a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
+++ b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
@@ -1761,8 +1761,20 @@ DiagnosedSilenceableFailure
 transform::NamedSequenceOp::apply(transform::TransformRewriter &rewriter,
                                   transform::TransformResults &results,
                                   transform::TransformState &state) {
-  // Nothing to do here.
-  return DiagnosedSilenceableFailure::success();
+  if (isExternal())
+    return emitDefiniteFailure() << "unresolved external named sequence";
+
+  // Map the entry block argument to the list of operations.
+  // Note: this is the same implementation as PossibleTopLevelTransformOp but
+  // without attaching the interface / trait since that is tailored to a
+  // dangling top-level op that does not get "called".
+  auto scope = state.make_region_scope(getBody());
+  if (failed(detail::mapPossibleTopLevelTransformOpBlockArguments(
+          state, this->getOperation(), getBody())))
+    return DiagnosedSilenceableFailure::definiteFailure();
+
+  return applySequenceBlock(getBody().front(),
+                            FailurePropagationMode::Propagate, state, results);
 }
 
 void transform::NamedSequenceOp::getEffects(
diff --git a/mlir/lib/Dialect/Transform/Transforms/CMakeLists.txt b/mlir/lib/Dialect/Transform/Transforms/CMakeLists.txt
index 8774a8b86f..f0f57874f5 100644
--- a/mlir/lib/Dialect/Transform/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/Transform/Transforms/CMakeLists.txt
@@ -1,6 +1,8 @@
 add_mlir_dialect_library(MLIRTransformDialectTransforms
   CheckUses.cpp
   InferEffects.cpp
+  InterpreterPass.cpp
+  PreloadLibraryPass.cpp
   TransformInterpreterPassBase.cpp
   TransformInterpreterUtils.cpp
 
diff --git a/mlir/lib/Dialect/Transform/Transforms/InterpreterPass.cpp b/mlir/lib/Dialect/Transform/Transforms/InterpreterPass.cpp
new file mode 100644
index 0000000000..3ec51d8872
--- /dev/null
+++ b/mlir/lib/Dialect/Transform/Transforms/InterpreterPass.cpp
@@ -0,0 +1,42 @@
+//===- InterpreterPass.cpp - Transform dialect interpreter pass -----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/Transform/IR/TransformDialect.h"
+#include "mlir/Dialect/Transform/Transforms/Passes.h"
+#include "mlir/Dialect/Transform/Transforms/TransformInterpreterUtils.h"
+
+using namespace mlir;
+
+namespace mlir {
+namespace transform {
+#define GEN_PASS_DEF_INTERPRETERPASS
+#include "mlir/Dialect/Transform/Transforms/Passes.h.inc"
+} // namespace transform
+} // namespace mlir
+
+namespace {
+class InterpreterPass
+    : public transform::impl::InterpreterPassBase<InterpreterPass> {
+public:
+  using Base::Base;
+
+  void runOnOperation() override {
+    MLIRContext *context = &getContext();
+    ModuleOp transformModule =
+        transform::detail::getPreloadedTransformModule(context);
+    if (failed(transform::applyTransformNamedSequence(
+            getOperation(), transformModule,
+            options.enableExpensiveChecks(true), entryPoint)))
+      return signalPassFailure();
+  }
+
+private:
+  /// Transform interpreter options.
+  transform::TransformOptions options;
+};
+} // namespace
diff --git a/mlir/lib/Dialect/Transform/Transforms/PreloadLibraryPass.cpp b/mlir/lib/Dialect/Transform/Transforms/PreloadLibraryPass.cpp
new file mode 100644
index 0000000000..d2e7108c02
--- /dev/null
+++ b/mlir/lib/Dialect/Transform/Transforms/PreloadLibraryPass.cpp
@@ -0,0 +1,39 @@
+//===- PreloadLibraryPass.cpp - Pass to preload a transform library -------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/Transform/IR/TransformDialect.h"
+#include "mlir/Dialect/Transform/Transforms/Passes.h"
+#include "mlir/Dialect/Transform/Transforms/TransformInterpreterUtils.h"
+
+using namespace mlir;
+
+namespace mlir {
+namespace transform {
+#define GEN_PASS_DEF_PRELOADLIBRARYPASS
+#include "mlir/Dialect/Transform/Transforms/Passes.h.inc"
+} // namespace transform
+} // namespace mlir
+
+namespace {
+class PreloadLibraryPass
+    : public transform::impl::PreloadLibraryPassBase<PreloadLibraryPass> {
+public:
+  using Base::Base;
+
+  void runOnOperation() override {
+    OwningOpRef<ModuleOp> mergedParsedLibraries;
+    if (failed(transform::detail::assembleTransformLibraryFromPaths(
+            &getContext(), transformLibraryPaths, mergedParsedLibraries)))
+      return signalPassFailure();
+    // TODO: investigate using a resource blob if some ownership mode allows it.
+    auto *dialect =
+        getContext().getOrLoadDialect<transform::TransformDialect>();
+    dialect->registerLibraryModule(std::move(mergedParsedLibraries));
+  }
+};
+} // namespace
diff --git a/mlir/lib/Dialect/Transform/Transforms/TransformInterpreterPassBase.cpp b/mlir/lib/Dialect/Transform/Transforms/TransformInterpreterPassBase.cpp
index 5f35b6789d..741456e7eb 100644
--- a/mlir/lib/Dialect/Transform/Transforms/TransformInterpreterPassBase.cpp
+++ b/mlir/lib/Dialect/Transform/Transforms/TransformInterpreterPassBase.cpp
@@ -56,10 +56,11 @@ constexpr static llvm::StringLiteral
 /// Reports an error if there is more than one such operation and returns the
 /// first one found. Reports an error returns nullptr if no such operation
 /// found.
-static Operation *findTopLevelTransform(Operation *root,
-                                        StringRef filenameOption) {
+static Operation *
+findTopLevelTransform(Operation *root, StringRef filenameOption,
+                      mlir::transform::TransformOptions options) {
   ::mlir::transform::TransformOpInterface topLevelTransform = nullptr;
-  WalkResult walkResult = root->walk<WalkOrder::PreOrder>(
+  root->walk<WalkOrder::PreOrder>(
       [&](::mlir::transform::TransformOpInterface transformOp) {
         if (!transformOp
                  ->hasTrait<transform::PossibleTopLevelTransformOpTrait>())
@@ -68,14 +69,15 @@ static Operation *findTopLevelTransform(Operation *root,
           topLevelTransform = transformOp;
           return WalkResult::skip();
         }
-        auto diag = transformOp.emitError()
-                    << "more than one top-level transform op";
-        diag.attachNote(topLevelTransform.getLoc())
-            << "previous top-level transform op";
-        return WalkResult::interrupt();
+        if (options.getEnforceSingleToplevelTransformOp()) {
+          auto diag = transformOp.emitError()
+                      << "more than one top-level transform op";
+          diag.attachNote(topLevelTransform.getLoc())
+              << "previous top-level transform op";
+          return WalkResult::interrupt();
+        }
+        return WalkResult::skip();
       });
-  if (walkResult.wasInterrupted())
-    return nullptr;
   if (!topLevelTransform) {
     auto diag = root->emitError()
                 << "could not find a nested top-level transform op";
@@ -310,7 +312,7 @@ LogicalResult transform::detail::interpreterBaseRunOnOperationImpl(
   Operation *transformRoot =
       debugTransformRootTag.empty()
           ? findTopLevelTransform(transformContainer,
-                                  transformFileName.getArgStr())
+                                  transformFileName.getArgStr(), options)
           : findOpWithTag(transformContainer, kTransformDialectTagAttrName,
                           debugTransformRootTag);
   if (!transformRoot)
@@ -357,59 +359,6 @@ LogicalResult transform::detail::interpreterBaseRunOnOperationImpl(
                          extraMappings, options);
 }
 
-/// Expands the given list of `paths` to a list of `.mlir` files.
-///
-/// Each entry in `paths` may either be a regular file, in which case it ends up
-/// in the result list, or a directory, in which case all (regular) `.mlir`
-/// files in that directory are added. Any other file types lead to a failure.
-static LogicalResult
-expandPathsToMLIRFiles(ArrayRef<std::string> &paths, MLIRContext *const context,
-                       SmallVectorImpl<std::string> &fileNames) {
-  for (const std::string &path : paths) {
-    auto loc = FileLineColLoc::get(context, path, 0, 0);
-
-    if (llvm::sys::fs::is_regular_file(path)) {
-      LLVM_DEBUG(DBGS() << "Adding '" << path << "' to list of files\n");
-      fileNames.push_back(path);
-      continue;
-    }
-
-    if (!llvm::sys::fs::is_directory(path)) {
-      return emitError(loc)
-             << "'" << path << "' is neither a file nor a directory";
-    }
-
-    LLVM_DEBUG(DBGS() << "Looking for files in '" << path << "':\n");
-
-    std::error_code ec;
-    for (llvm::sys::fs::directory_iterator it(path, ec), itEnd;
-         it != itEnd && !ec; it.increment(ec)) {
-      const std::string &fileName = it->path();
-
-      if (it->type() != llvm::sys::fs::file_type::regular_file) {
-        LLVM_DEBUG(DBGS() << "  Skipping non-regular file '" << fileName
-                          << "'\n");
-        continue;
-      }
-
-      if (!StringRef(fileName).endswith(".mlir")) {
-        LLVM_DEBUG(DBGS() << "  Skipping '" << fileName
-                          << "' because it does not end with '.mlir'\n");
-        continue;
-      }
-
-      LLVM_DEBUG(DBGS() << "  Adding '" << fileName << "' to list of files\n");
-      fileNames.push_back(fileName);
-    }
-
-    if (ec)
-      return emitError(loc) << "error while opening files in '" << path
-                            << "': " << ec.message();
-  }
-
-  return success();
-}
-
 LogicalResult transform::detail::interpreterBaseInitializeImpl(
     MLIRContext *context, StringRef transformFileName,
     ArrayRef<std::string> transformLibraryPaths,
diff --git a/mlir/lib/Dialect/Transform/Transforms/TransformInterpreterUtils.cpp b/mlir/lib/Dialect/Transform/Transforms/TransformInterpreterUtils.cpp
index 1a6ebdd162..e6d6920722 100644
--- a/mlir/lib/Dialect/Transform/Transforms/TransformInterpreterUtils.cpp
+++ b/mlir/lib/Dialect/Transform/Transforms/TransformInterpreterUtils.cpp
@@ -23,6 +23,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/FileSystem.h"
 #include "llvm/Support/SourceMgr.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -31,6 +32,60 @@ using namespace mlir;
 #define DEBUG_TYPE "transform-dialect-interpreter-utils"
 #define DBGS() (llvm::dbgs() << "[" DEBUG_TYPE << "]: ")
 
+/// Expands the given list of `paths` to a list of `.mlir` files.
+///
+/// Each entry in `paths` may either be a regular file, in which case it ends up
+/// in the result list, or a directory, in which case all (regular) `.mlir`
+/// files in that directory are added. Any other file types lead to a failure.
+LogicalResult transform::detail::expandPathsToMLIRFiles(
+    ArrayRef<std::string> &paths, MLIRContext *context,
+    SmallVectorImpl<std::string> &fileNames) {
+  for (const std::string &path : paths) {
+    auto loc = FileLineColLoc::get(context, path, 0, 0);
+
+    if (llvm::sys::fs::is_regular_file(path)) {
+      LLVM_DEBUG(DBGS() << "Adding '" << path << "' to list of files\n");
+      fileNames.push_back(path);
+      continue;
+    }
+
+    if (!llvm::sys::fs::is_directory(path)) {
+      return emitError(loc)
+             << "'" << path << "' is neither a file nor a directory";
+    }
+
+    LLVM_DEBUG(DBGS() << "Looking for files in '" << path << "':\n");
+
+    std::error_code ec;
+    for (llvm::sys::fs::directory_iterator it(path, ec), itEnd;
+         it != itEnd && !ec; it.increment(ec)) {
+      const std::string &fileName = it->path();
+
+      if (it->type() != llvm::sys::fs::file_type::regular_file &&
+          it->type() != llvm::sys::fs::file_type::symlink_file) {
+        LLVM_DEBUG(DBGS() << "  Skipping non-regular file '" << fileName
+                          << "'\n");
+        continue;
+      }
+
+      if (!StringRef(fileName).endswith(".mlir")) {
+        LLVM_DEBUG(DBGS() << "  Skipping '" << fileName
+                          << "' because it does not end with '.mlir'\n");
+        continue;
+      }
+
+      LLVM_DEBUG(DBGS() << "  Adding '" << fileName << "' to list of files\n");
+      fileNames.push_back(fileName);
+    }
+
+    if (ec)
+      return emitError(loc) << "error while opening files in '" << path
+                            << "': " << ec.message();
+  }
+
+  return success();
+}
+
 LogicalResult transform::detail::parseTransformModuleFromFile(
     MLIRContext *context, llvm::StringRef transformFileName,
     OwningOpRef<ModuleOp> &transformModule) {
@@ -91,10 +146,50 @@ transform::detail::findTransformEntryPoint(Operation *root, ModuleOp module,
   return nullptr;
 }
 
-/// Return whether `func1` can be merged into `func2`. For that to work `func1`
-/// has to be a declaration (aka has to be external) and `func2` either has to
-/// be a declaration as well, or it has to be public (otherwise, it wouldn't
-/// be visible by `func1`).
+LogicalResult transform::detail::assembleTransformLibraryFromPaths(
+    MLIRContext *context, ArrayRef<std::string> transformLibraryPaths,
+    OwningOpRef<ModuleOp> &transformModule) {
+  // Assemble list of library files.
+  SmallVector<std::string> libraryFileNames;
+  if (failed(detail::expandPathsToMLIRFiles(transformLibraryPaths, context,
+                                            libraryFileNames)))
+    return failure();
+
+  // Parse modules from library files.
+  SmallVector<OwningOpRef<ModuleOp>> parsedLibraries;
+  for (const std::string &libraryFileName : libraryFileNames) {
+    OwningOpRef<ModuleOp> parsedLibrary;
+    if (failed(transform::detail::parseTransformModuleFromFile(
+            context, libraryFileName, parsedLibrary)))
+      return failure();
+    parsedLibraries.push_back(std::move(parsedLibrary));
+  }
+
+  // Merge parsed libraries into one module.
+  auto loc = FileLineColLoc::get(context, "<shared-library-module>", 0, 0);
+  OwningOpRef<ModuleOp> mergedParsedLibraries =
+      ModuleOp::create(loc, "__transform");
+  {
+    mergedParsedLibraries.get()->setAttr("transform.with_named_sequence",
+                                         UnitAttr::get(context));
+    IRRewriter rewriter(context);
+    // TODO: extend `mergeSymbolsInto` to support multiple `other` modules.
+    for (OwningOpRef<ModuleOp> &parsedLibrary : parsedLibraries) {
+      if (failed(transform::detail::mergeSymbolsInto(
+              mergedParsedLibraries.get(), std::move(parsedLibrary))))
+        return mergedParsedLibraries->emitError()
+               << "failed to verify merged transform module";
+    }
+  }
+
+  transformModule = std::move(mergedParsedLibraries);
+  return success();
+}
+
+/// Return whether `func1` can be merged into `func2`. For that to work
+/// `func1` has to be a declaration (aka has to be external) and `func2`
+/// either has to be a declaration as well, or it has to be public (otherwise,
+/// it wouldn't be visible by `func1`).
 static bool canMergeInto(FunctionOpInterface func1, FunctionOpInterface func2) {
   return func1.isExternal() && (func2.isPublic() || func2.isExternal());
 }
@@ -281,8 +376,9 @@ transform::detail::mergeSymbolsInto(Operation *target,
       auto collidingFuncOp =
           cast<FunctionOpInterface>(collidingOp.getOperation());
 
-      // Both ops are in the target module now and can be treated symmetrically,
-      // so w.l.o.g. we can reduce to merging `funcOp` into `collidingFuncOp`.
+      // Both ops are in the target module now and can be treated
+      // symmetrically, so w.l.o.g. we can reduce to merging `funcOp` into
+      // `collidingFuncOp`.
       if (!canMergeInto(funcOp, collidingFuncOp)) {
         std::swap(funcOp, collidingFuncOp);
       }
@@ -317,18 +413,25 @@ LogicalResult transform::applyTransformNamedSequence(
     const TransformOptions &options, StringRef entryPoint) {
   Operation *transformRoot =
       detail::findTransformEntryPoint(payload, transformModule, entryPoint);
-  if (!transformRoot)
-    return failure();
+  if (!transformRoot) {
+    return payload->emitError()
+           << "could not find transform entry point: " << entryPoint
+           << " in either payload or transform module";
+  }
 
   // `transformModule` may not be modified.
-  OwningOpRef<Operation *> clonedTransformModule(transformModule->clone());
   if (transformModule && !transformModule->isAncestor(transformRoot)) {
+    OwningOpRef<Operation *> clonedTransformModule(transformModule->clone());
     if (failed(detail::mergeSymbolsInto(
             SymbolTable::getNearestSymbolTable(transformRoot),
-            std::move(clonedTransformModule))))
-      return failure();
+            std::move(clonedTransformModule)))) {
+      return payload->emitError() << "failed to merge symbols";
+    }
   }
 
+  LLVM_DEBUG(DBGS() << "Apply\n" << *transformRoot << "\n");
+  LLVM_DEBUG(DBGS() << "To\n" << *payload << "\n");
+
   // Apply the transform to the IR, do not enforce top-level constraints.
   RaggedArray<MappedValue> noExtraMappings;
   return applyTransforms(payload, cast<TransformOpInterface>(transformRoot),
diff --git a/mlir/lib/Dialect/Utils/StructuredOpsUtils.cpp b/mlir/lib/Dialect/Utils/StructuredOpsUtils.cpp
index a2977901f4..641ddf3f91 100644
--- a/mlir/lib/Dialect/Utils/StructuredOpsUtils.cpp
+++ b/mlir/lib/Dialect/Utils/StructuredOpsUtils.cpp
@@ -21,9 +21,9 @@ bool mlir::isRowMajorMatmul(ArrayAttr indexingMaps) {
   if (indexingMaps.size() != 3)
     return false;
 
-  auto map0 = cast<AffineMapAttr>(indexingMaps[0]).getValue();
-  auto map1 = cast<AffineMapAttr>(indexingMaps[1]).getValue();
-  auto map2 = cast<AffineMapAttr>(indexingMaps[2]).getValue();
+  AffineMap map0 = cast<AffineMapAttr>(indexingMaps[0]).getValue();
+  AffineMap map1 = cast<AffineMapAttr>(indexingMaps[1]).getValue();
+  AffineMap map2 = cast<AffineMapAttr>(indexingMaps[2]).getValue();
 
   if (map0.getNumResults() != 2 || map1.getNumResults() != 2 ||
       map2.getNumResults() != 2 || map0.getNumInputs() != 3 ||
@@ -47,9 +47,9 @@ bool mlir::isColumnMajorMatmul(ArrayAttr indexingMaps) {
   if (indexingMaps.size() != 3)
     return false;
 
-  auto map0 = cast<AffineMapAttr>(indexingMaps[0]).getValue();
-  auto map1 = cast<AffineMapAttr>(indexingMaps[1]).getValue();
-  auto map2 = cast<AffineMapAttr>(indexingMaps[2]).getValue();
+  AffineMap map0 = cast<AffineMapAttr>(indexingMaps[0]).getValue();
+  AffineMap map1 = cast<AffineMapAttr>(indexingMaps[1]).getValue();
+  AffineMap map2 = cast<AffineMapAttr>(indexingMaps[2]).getValue();
 
   if (map0.getNumResults() != 2 || map1.getNumResults() != 2 ||
       map2.getNumResults() != 2 || map0.getNumInputs() != 3 ||
@@ -73,9 +73,9 @@ bool mlir::isRowMajorBatchMatmul(ArrayAttr indexingMaps) {
   if (indexingMaps.size() != 3)
     return false;
 
-  auto map0 = cast<AffineMapAttr>(indexingMaps[0]).getValue();
-  auto map1 = cast<AffineMapAttr>(indexingMaps[1]).getValue();
-  auto map2 = cast<AffineMapAttr>(indexingMaps[2]).getValue();
+  AffineMap map0 = cast<AffineMapAttr>(indexingMaps[0]).getValue();
+  AffineMap map1 = cast<AffineMapAttr>(indexingMaps[1]).getValue();
+  AffineMap map2 = cast<AffineMapAttr>(indexingMaps[2]).getValue();
 
   if (map0.getNumResults() != 3 || map1.getNumResults() != 3 ||
       map2.getNumResults() != 3 || map0.getNumInputs() != 4 ||
@@ -96,6 +96,79 @@ bool mlir::isRowMajorBatchMatmul(ArrayAttr indexingMaps) {
   return indexingMaps == maps;
 }
 
+bool mlir::isVecmat(ArrayAttr indexingMaps) {
+  if (indexingMaps.size() != 3)
+    return false;
+  AffineMap map0 = cast<AffineMapAttr>(indexingMaps[0]).getValue();
+  AffineMap map1 = cast<AffineMapAttr>(indexingMaps[1]).getValue();
+  AffineMap map2 = cast<AffineMapAttr>(indexingMaps[2]).getValue();
+
+  if (map0.getNumResults() != 1 || map1.getNumResults() != 2 ||
+      map2.getNumResults() != 1 || map0.getNumInputs() != 2 ||
+      map1.getNumInputs() != 2 || map2.getNumInputs() != 2) {
+    return false;
+  }
+
+  // Extract dimensions for K * KxN -> N
+  AffineExpr k = map0.getResult(0);
+  AffineExpr n = map2.getResult(0);
+  auto *context = indexingMaps.getContext();
+  auto mapA = AffineMapAttr::get(AffineMap::get(2, 0, {k}, context));
+  auto mapB = AffineMapAttr::get(AffineMap::get(2, 0, {k, n}, context));
+  auto mapC = AffineMapAttr::get(AffineMap::get(2, 0, {n}, context));
+  auto maps = ArrayAttr::get(context, {mapA, mapB, mapC});
+  return indexingMaps == maps;
+}
+
+bool mlir::isMatvec(ArrayAttr indexingMaps) {
+  if (indexingMaps.size() != 3)
+    return false;
+  AffineMap map0 = cast<AffineMapAttr>(indexingMaps[0]).getValue();
+  AffineMap map1 = cast<AffineMapAttr>(indexingMaps[1]).getValue();
+  AffineMap map2 = cast<AffineMapAttr>(indexingMaps[2]).getValue();
+
+  if (map0.getNumResults() != 2 || map1.getNumResults() != 1 ||
+      map2.getNumResults() != 1 || map0.getNumInputs() != 2 ||
+      map1.getNumInputs() != 2 || map2.getNumInputs() != 2) {
+    return false;
+  }
+
+  // Extract dimensions for N*K * K -> N
+  AffineExpr k = map1.getResult(0);
+  AffineExpr n = map2.getResult(0);
+  auto *context = indexingMaps.getContext();
+  auto mapA = AffineMapAttr::get(AffineMap::get(2, 0, {n, k}, context));
+  auto mapB = AffineMapAttr::get(AffineMap::get(2, 0, {k}, context));
+  auto mapC = AffineMapAttr::get(AffineMap::get(2, 0, {n}, context));
+  auto maps = ArrayAttr::get(context, {mapA, mapB, mapC});
+  return indexingMaps == maps;
+}
+
+bool mlir::isBatchMatvec(ArrayAttr indexingMaps) {
+  if (indexingMaps.size() != 3)
+    return false;
+  AffineMap map0 = cast<AffineMapAttr>(indexingMaps[0]).getValue();
+  AffineMap map1 = cast<AffineMapAttr>(indexingMaps[1]).getValue();
+  AffineMap map2 = cast<AffineMapAttr>(indexingMaps[2]).getValue();
+
+  if (map0.getNumResults() != 3 || map1.getNumResults() != 2 ||
+      map2.getNumResults() != 2 || map0.getNumInputs() != 3 ||
+      map1.getNumInputs() != 3 || map2.getNumInputs() != 3) {
+    return false;
+  }
+
+  // Extract dimensions for B*N*K * B*K -> B*N
+  AffineExpr b = map0.getResult(0);
+  AffineExpr k = map1.getResult(1);
+  AffineExpr n = map2.getResult(1);
+  auto *context = indexingMaps.getContext();
+  auto mapA = AffineMapAttr::get(AffineMap::get(3, 0, {b, n, k}, context));
+  auto mapB = AffineMapAttr::get(AffineMap::get(3, 0, {b, k}, context));
+  auto mapC = AffineMapAttr::get(AffineMap::get(3, 0, {b, n}, context));
+  auto maps = ArrayAttr::get(context, {mapA, mapB, mapC});
+  return indexingMaps == maps;
+}
+
 Operation *mlir::clone(OpBuilder &b, Operation *op, TypeRange newResultTypes,
                        ValueRange newOperands) {
   IRMapping bvm;
diff --git a/mlir/lib/Dialect/Vector/IR/CMakeLists.txt b/mlir/lib/Dialect/Vector/IR/CMakeLists.txt
index 9ec919423b..70f3fa8c29 100644
--- a/mlir/lib/Dialect/Vector/IR/CMakeLists.txt
+++ b/mlir/lib/Dialect/Vector/IR/CMakeLists.txt
@@ -11,6 +11,7 @@ add_mlir_dialect_library(MLIRVectorDialect
   MLIRVectorAttributesIncGen
 
   LINK_LIBS PUBLIC
+  MLIRAffineDialect
   MLIRArithDialect
   MLIRControlFlowInterfaces
   MLIRDataLayoutInterfaces
@@ -22,5 +23,6 @@ add_mlir_dialect_library(MLIRVectorDialect
   MLIRMemRefDialect
   MLIRSideEffectInterfaces
   MLIRTensorDialect
+  MLIRValueBoundsOpInterface
   MLIRVectorInterfaces
   )
diff --git a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
index 044b6cc07d..9e7de1d1e1 100644
--- a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
+++ b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
@@ -13,6 +13,7 @@
 
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 
+#include "mlir/Dialect/Affine/IR/ValueBoundsOpInterfaceImpl.h"
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/Arith/Utils/Utils.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
@@ -30,6 +31,7 @@
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
+#include "mlir/Interfaces/ValueBoundsOpInterface.h"
 #include "mlir/Support/LLVM.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
@@ -168,39 +170,76 @@ bool mlir::vector::checkSameValueWAW(vector::TransferWriteOp write,
 }
 
 bool mlir::vector::isDisjointTransferIndices(
-    VectorTransferOpInterface transferA, VectorTransferOpInterface transferB) {
+    VectorTransferOpInterface transferA, VectorTransferOpInterface transferB,
+    bool testDynamicValueUsingBounds) {
   // For simplicity only look at transfer of same type.
   if (transferA.getVectorType() != transferB.getVectorType())
     return false;
   unsigned rankOffset = transferA.getLeadingShapedRank();
   for (unsigned i = 0, e = transferA.indices().size(); i < e; i++) {
-    auto indexA = getConstantIntValue(transferA.indices()[i]);
-    auto indexB = getConstantIntValue(transferB.indices()[i]);
-    // If any of the indices are dynamic we cannot prove anything.
-    if (!indexA.has_value() || !indexB.has_value())
-      continue;
+    Value indexA = transferA.indices()[i];
+    Value indexB = transferB.indices()[i];
+    std::optional<int64_t> cstIndexA = getConstantIntValue(indexA);
+    std::optional<int64_t> cstIndexB = getConstantIntValue(indexB);
 
     if (i < rankOffset) {
       // For leading dimensions, if we can prove that index are different we
       // know we are accessing disjoint slices.
-      if (*indexA != *indexB)
-        return true;
+      if (cstIndexA.has_value() && cstIndexB.has_value()) {
+        if (*cstIndexA != *cstIndexB)
+          return true;
+        continue;
+      }
+      if (testDynamicValueUsingBounds) {
+        // First try to see if we can fully compose and simplify the affine
+        // expression as a fast track.
+        FailureOr<uint64_t> delta =
+            affine::fullyComposeAndComputeConstantDelta(indexA, indexB);
+        if (succeeded(delta) && *delta != 0)
+          return true;
+
+        FailureOr<bool> testEqual =
+            ValueBoundsConstraintSet::areEqual(indexA, indexB);
+        if (succeeded(testEqual) && !testEqual.value())
+          return true;
+      }
     } else {
       // For this dimension, we slice a part of the memref we need to make sure
       // the intervals accessed don't overlap.
-      int64_t distance = std::abs(*indexA - *indexB);
-      if (distance >= transferA.getVectorType().getDimSize(i - rankOffset))
-        return true;
+      int64_t vectorDim = transferA.getVectorType().getDimSize(i - rankOffset);
+      if (cstIndexA.has_value() && cstIndexB.has_value()) {
+        int64_t distance = std::abs(*cstIndexA - *cstIndexB);
+        if (distance >= vectorDim)
+          return true;
+        continue;
+      }
+      if (testDynamicValueUsingBounds) {
+        // First try to see if we can fully compose and simplify the affine
+        // expression as a fast track.
+        FailureOr<int64_t> delta =
+            affine::fullyComposeAndComputeConstantDelta(indexA, indexB);
+        if (succeeded(delta) && std::abs(*delta) >= vectorDim)
+          return true;
+
+        FailureOr<int64_t> computeDelta =
+            ValueBoundsConstraintSet::computeConstantDelta(indexA, indexB);
+        if (succeeded(computeDelta)) {
+          if (std::abs(computeDelta.value()) >= vectorDim)
+            return true;
+        }
+      }
     }
   }
   return false;
 }
 
 bool mlir::vector::isDisjointTransferSet(VectorTransferOpInterface transferA,
-                                         VectorTransferOpInterface transferB) {
+                                         VectorTransferOpInterface transferB,
+                                         bool testDynamicValueUsingBounds) {
   if (transferA.source() != transferB.source())
     return false;
-  return isDisjointTransferIndices(transferA, transferB);
+  return isDisjointTransferIndices(transferA, transferB,
+                                   testDynamicValueUsingBounds);
 }
 
 // Helper to iterate over n-D vector slice elements. Calculate the next
@@ -485,47 +524,6 @@ LogicalResult ReductionOp::verify() {
   return success();
 }
 
-ParseResult ReductionOp::parse(OpAsmParser &parser, OperationState &result) {
-  SmallVector<OpAsmParser::UnresolvedOperand, 2> operandsInfo;
-  Type redType;
-  Type resType;
-  CombiningKindAttr kindAttr;
-  arith::FastMathFlagsAttr fastMathAttr;
-  if (parser.parseCustomAttributeWithFallback(kindAttr, Type{}, "kind",
-                                              result.attributes) ||
-      parser.parseComma() || parser.parseOperandList(operandsInfo) ||
-      (succeeded(parser.parseOptionalKeyword("fastmath")) &&
-       parser.parseCustomAttributeWithFallback(fastMathAttr, Type{}, "fastmath",
-                                               result.attributes)) ||
-      parser.parseColonType(redType) ||
-      parser.parseKeywordType("into", resType) ||
-      (!operandsInfo.empty() &&
-       parser.resolveOperand(operandsInfo[0], redType, result.operands)) ||
-      (operandsInfo.size() > 1 &&
-       parser.resolveOperand(operandsInfo[1], resType, result.operands)) ||
-      parser.addTypeToList(resType, result.types))
-    return failure();
-  if (operandsInfo.empty() || operandsInfo.size() > 2)
-    return parser.emitError(parser.getNameLoc(),
-                            "unsupported number of operands");
-  return success();
-}
-
-void ReductionOp::print(OpAsmPrinter &p) {
-  p << " ";
-  getKindAttr().print(p);
-  p << ", " << getVector();
-  if (getAcc())
-    p << ", " << getAcc();
-
-  if (getFastmathAttr() &&
-      getFastmathAttr().getValue() != arith::FastMathFlags::none) {
-    p << ' ' << getFastmathAttrName().getValue();
-    p.printStrippedAttrOrType(getFastmathAttr());
-  }
-  p << " : " << getVector().getType() << " into " << getDest().getType();
-}
-
 // MaskableOpInterface methods.
 
 /// Returns the mask type expected by this operation.
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
index 603b88f11c..a5f1b28152 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
@@ -142,7 +142,8 @@ void TransferOptimization::deadStoreOp(vector::TransferWriteOp write) {
       // Don't need to consider disjoint accesses.
       if (vector::isDisjointTransferSet(
               cast<VectorTransferOpInterface>(write.getOperation()),
-              cast<VectorTransferOpInterface>(transferOp.getOperation())))
+              cast<VectorTransferOpInterface>(transferOp.getOperation()),
+              /*testDynamicValueUsingBounds=*/true))
         continue;
     }
     blockingAccesses.push_back(user);
@@ -217,7 +218,8 @@ void TransferOptimization::storeToLoadForwarding(vector::TransferReadOp read) {
       // the write.
       if (vector::isDisjointTransferSet(
               cast<VectorTransferOpInterface>(write.getOperation()),
-              cast<VectorTransferOpInterface>(read.getOperation())))
+              cast<VectorTransferOpInterface>(read.getOperation()),
+              /*testDynamicValueUsingBounds=*/true))
         continue;
       if (write.getSource() == read.getSource() &&
           dominators.dominates(write, read) && checkSameValueRAW(write, read)) {
diff --git a/mlir/lib/ExecutionEngine/CRunnerUtils.cpp b/mlir/lib/ExecutionEngine/CRunnerUtils.cpp
index 7f800e8ea9..c31ae3a1c7 100644
--- a/mlir/lib/ExecutionEngine/CRunnerUtils.cpp
+++ b/mlir/lib/ExecutionEngine/CRunnerUtils.cpp
@@ -16,7 +16,8 @@
 #include "mlir/ExecutionEngine/Msan.h"
 
 #ifndef _WIN32
-#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__)
+#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \
+    defined(__DragonFly__)
 #include <cstdlib>
 #else
 #include <alloca.h>
diff --git a/mlir/lib/ExecutionEngine/SparseTensor/CMakeLists.txt b/mlir/lib/ExecutionEngine/SparseTensor/CMakeLists.txt
index c48af17b2d..15024b2475 100644
--- a/mlir/lib/ExecutionEngine/SparseTensor/CMakeLists.txt
+++ b/mlir/lib/ExecutionEngine/SparseTensor/CMakeLists.txt
@@ -8,7 +8,6 @@
 add_mlir_library(MLIRSparseTensorRuntime
   File.cpp
   MapRef.cpp
-  NNZ.cpp
   Storage.cpp
 
   EXCLUDE_FROM_LIBMLIR
diff --git a/mlir/lib/ExecutionEngine/SparseTensor/MapRef.cpp b/mlir/lib/ExecutionEngine/SparseTensor/MapRef.cpp
index ee4d6fa0d3..ace6ac8152 100644
--- a/mlir/lib/ExecutionEngine/SparseTensor/MapRef.cpp
+++ b/mlir/lib/ExecutionEngine/SparseTensor/MapRef.cpp
@@ -7,14 +7,15 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/ExecutionEngine/SparseTensor/MapRef.h"
+#include "mlir/Dialect/SparseTensor/IR/Enums.h"
 
 mlir::sparse_tensor::MapRef::MapRef(uint64_t d, uint64_t l, const uint64_t *d2l,
                                     const uint64_t *l2d)
     : dimRank(d), lvlRank(l), dim2lvl(d2l), lvl2dim(l2d),
       isPermutation(isPermutationMap()) {
   if (isPermutation) {
-    for (uint64_t i = 0; i < dimRank; i++)
-      assert(lvl2dim[dim2lvl[i]] == i);
+    for (uint64_t l = 0; l < lvlRank; l++)
+      assert(lvl2dim[dim2lvl[l]] == l);
   }
 }
 
@@ -22,11 +23,42 @@ bool mlir::sparse_tensor::MapRef::isPermutationMap() const {
   if (dimRank != lvlRank)
     return false;
   std::vector<bool> seen(dimRank, false);
-  for (uint64_t i = 0; i < dimRank; i++) {
-    const uint64_t j = dim2lvl[i];
-    if (j >= dimRank || seen[j])
+  for (uint64_t l = 0; l < lvlRank; l++) {
+    const uint64_t d = dim2lvl[l];
+    if (d >= dimRank || seen[d])
       return false;
-    seen[j] = true;
+    seen[d] = true;
   }
   return true;
 }
+
+bool mlir::sparse_tensor::MapRef::isFloor(uint64_t l, uint64_t &i,
+                                          uint64_t &c) const {
+  if (isEncodedFloor(dim2lvl[l])) {
+    i = decodeIndex(dim2lvl[l]);
+    c = decodeConst(dim2lvl[l]);
+    return true;
+  }
+  return false;
+}
+
+bool mlir::sparse_tensor::MapRef::isMod(uint64_t l, uint64_t &i,
+                                        uint64_t &c) const {
+  if (isEncodedMod(dim2lvl[l])) {
+    i = decodeIndex(dim2lvl[l]);
+    c = decodeConst(dim2lvl[l]);
+    return true;
+  }
+  return false;
+}
+
+bool mlir::sparse_tensor::MapRef::isMul(uint64_t d, uint64_t &i, uint64_t &c,
+                                        uint64_t &ii) const {
+  if (isEncodedMul(lvl2dim[d])) {
+    i = decodeIndex(lvl2dim[d]);
+    c = decodeMulc(lvl2dim[d]);
+    ii = decodeMuli(lvl2dim[d]);
+    return true;
+  }
+  return false;
+}
diff --git a/mlir/lib/ExecutionEngine/SparseTensor/NNZ.cpp b/mlir/lib/ExecutionEngine/SparseTensor/NNZ.cpp
deleted file mode 100644
index d3c3951c15..0000000000
--- a/mlir/lib/ExecutionEngine/SparseTensor/NNZ.cpp
+++ /dev/null
@@ -1,79 +0,0 @@
-//===- NNZ.cpp - NNZ-statistics for direct sparse2sparse conversion -------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file contains method definitions for `SparseTensorNNZ`.
-//
-//===----------------------------------------------------------------------===//
-
-#include "mlir/ExecutionEngine/SparseTensor/Storage.h"
-
-using namespace mlir::sparse_tensor;
-
-SparseTensorNNZ::SparseTensorNNZ(const std::vector<uint64_t> &lvlSizes,
-                                 const std::vector<DimLevelType> &lvlTypes)
-    : lvlSizes(lvlSizes), lvlTypes(lvlTypes), nnz(getLvlRank()) {
-  assert(lvlSizes.size() == lvlTypes.size() && "Rank mismatch");
-  bool alreadyCompressed = false;
-  (void)alreadyCompressed;
-  uint64_t sz = 1; // the product of all `lvlSizes` strictly less than `l`.
-  for (uint64_t l = 0, lvlrank = getLvlRank(); l < lvlrank; ++l) {
-    const DimLevelType dlt = lvlTypes[l];
-    if (isCompressedDLT(dlt)) {
-      if (alreadyCompressed)
-        MLIR_SPARSETENSOR_FATAL(
-            "Multiple compressed levels not currently supported");
-      alreadyCompressed = true;
-      nnz[l].resize(sz, 0); // Both allocate and zero-initialize.
-    } else if (isDenseDLT(dlt)) {
-      if (alreadyCompressed)
-        MLIR_SPARSETENSOR_FATAL(
-            "Dense after compressed not currently supported");
-    } else if (isSingletonDLT(dlt)) {
-      // Singleton after Compressed causes no problems for allocating
-      // `nnz` nor for the yieldPos loop.  This remains true even
-      // when adding support for multiple compressed dimensions or
-      // for dense-after-compressed.
-    } else {
-      MLIR_SPARSETENSOR_FATAL("unsupported level type: %d\n",
-                              static_cast<uint8_t>(dlt));
-    }
-    sz = detail::checkedMul(sz, lvlSizes[l]);
-  }
-}
-
-void SparseTensorNNZ::forallCoords(uint64_t stopLvl,
-                                   SparseTensorNNZ::NNZConsumer yield) const {
-  assert(stopLvl < getLvlRank() && "Level out of bounds");
-  assert(isCompressedDLT(lvlTypes[stopLvl]) &&
-         "Cannot look up non-compressed levels");
-  forallCoords(yield, stopLvl, 0, 0);
-}
-
-void SparseTensorNNZ::add(const std::vector<uint64_t> &lvlCoords) {
-  uint64_t parentPos = 0;
-  for (uint64_t l = 0, lvlrank = getLvlRank(); l < lvlrank; ++l) {
-    if (isCompressedDLT(lvlTypes[l]))
-      nnz[l][parentPos]++;
-    parentPos = parentPos * lvlSizes[l] + lvlCoords[l];
-  }
-}
-
-void SparseTensorNNZ::forallCoords(SparseTensorNNZ::NNZConsumer yield,
-                                   uint64_t stopLvl, uint64_t parentPos,
-                                   uint64_t l) const {
-  assert(l <= stopLvl);
-  if (l == stopLvl) {
-    assert(parentPos < nnz[l].size() && "Cursor is out of range");
-    yield(nnz[l][parentPos]);
-  } else {
-    const uint64_t sz = lvlSizes[l];
-    const uint64_t pstart = parentPos * sz;
-    for (uint64_t i = 0; i < sz; ++i)
-      forallCoords(yield, stopLvl, pstart + i, l + 1);
-  }
-}
diff --git a/mlir/lib/ExecutionEngine/SparseTensor/Storage.cpp b/mlir/lib/ExecutionEngine/SparseTensor/Storage.cpp
index 199e4205a6..40805a179d 100644
--- a/mlir/lib/ExecutionEngine/SparseTensor/Storage.cpp
+++ b/mlir/lib/ExecutionEngine/SparseTensor/Storage.cpp
@@ -20,15 +20,14 @@ using namespace mlir::sparse_tensor;
 SparseTensorStorageBase::SparseTensorStorageBase( // NOLINT
     uint64_t dimRank, const uint64_t *dimSizes, uint64_t lvlRank,
     const uint64_t *lvlSizes, const DimLevelType *lvlTypes,
-    const uint64_t *lvl2dim)
+    const uint64_t *dim2lvl, const uint64_t *lvl2dim)
     : dimSizes(dimSizes, dimSizes + dimRank),
       lvlSizes(lvlSizes, lvlSizes + lvlRank),
       lvlTypes(lvlTypes, lvlTypes + lvlRank),
-      lvl2dim(lvl2dim, lvl2dim + lvlRank) {
-  assert(dimSizes && "Got nullptr for dimension sizes");
-  assert(lvlSizes && "Got nullptr for level sizes");
-  assert(lvlTypes && "Got nullptr for level types");
-  assert(lvl2dim && "Got nullptr for level-to-dimension mapping");
+      dim2lvlVec(dim2lvl, dim2lvl + lvlRank),
+      lvl2dimVec(lvl2dim, lvl2dim + dimRank),
+      map(dimRank, lvlRank, dim2lvlVec.data(), lvl2dimVec.data()) {
+  assert(dimSizes && lvlSizes && lvlTypes && dim2lvl && lvl2dim);
   // Validate dim-indexed parameters.
   assert(dimRank > 0 && "Trivial shape is unsupported");
   for (uint64_t d = 0; d < dimRank; ++d)
@@ -45,21 +44,10 @@ SparseTensorStorageBase::SparseTensorStorageBase( // NOLINT
   }
 }
 
-// Helper macro for generating error messages when some
-// `SparseTensorStorage<P,I,V>` is cast to `SparseTensorStorageBase`
-// and then the wrong "partial method specialization" is called.
+// Helper macro for wrong "partial method specialization" errors.
 #define FATAL_PIV(NAME)                                                        \
   MLIR_SPARSETENSOR_FATAL("<P,I,V> type mismatch for: " #NAME);
 
-#define IMPL_NEWENUMERATOR(VNAME, V)                                           \
-  void SparseTensorStorageBase::newEnumerator(                                 \
-      SparseTensorEnumeratorBase<V> **, uint64_t, const uint64_t *, uint64_t,  \
-      const uint64_t *) const {                                                \
-    FATAL_PIV("newEnumerator" #VNAME);                                         \
-  }
-MLIR_SPARSETENSOR_FOREVERY_V(IMPL_NEWENUMERATOR)
-#undef IMPL_NEWENUMERATOR
-
 #define IMPL_GETPOSITIONS(PNAME, P)                                            \
   void SparseTensorStorageBase::getPositions(std::vector<P> **, uint64_t) {    \
     FATAL_PIV("getPositions" #PNAME);                                          \
@@ -81,6 +69,13 @@ MLIR_SPARSETENSOR_FOREVERY_FIXED_O(IMPL_GETCOORDINATES)
 MLIR_SPARSETENSOR_FOREVERY_V(IMPL_GETVALUES)
 #undef IMPL_GETVALUES
 
+#define IMPL_FORWARDINGINSERT(VNAME, V)                                        \
+  void SparseTensorStorageBase::forwardingInsert(const uint64_t *, V) {        \
+    FATAL_PIV("forwardingInsert" #VNAME);                                      \
+  }
+MLIR_SPARSETENSOR_FOREVERY_V(IMPL_FORWARDINGINSERT)
+#undef IMPL_FORWARDINGINSERT
+
 #define IMPL_LEXINSERT(VNAME, V)                                               \
   void SparseTensorStorageBase::lexInsert(const uint64_t *, V) {               \
     FATAL_PIV("lexInsert" #VNAME);                                             \
diff --git a/mlir/lib/ExecutionEngine/SparseTensorRuntime.cpp b/mlir/lib/ExecutionEngine/SparseTensorRuntime.cpp
index 8340fe7dcf..441a6094ed 100644
--- a/mlir/lib/ExecutionEngine/SparseTensorRuntime.cpp
+++ b/mlir/lib/ExecutionEngine/SparseTensorRuntime.cpp
@@ -63,71 +63,18 @@ using namespace mlir::sparse_tensor;
 
 //===----------------------------------------------------------------------===//
 //
-// Implementation details for public functions, which don't have a good
-// place to live in the C++ library this file is wrapping.
+// Utilities for manipulating `StridedMemRefType`.
 //
 //===----------------------------------------------------------------------===//
 
 namespace {
 
-/// Wrapper class to avoid memory leakage issues.  The `SparseTensorCOO<V>`
-/// class provides a standard C++ iterator interface, where the iterator
-/// is implemented as per `std::vector`'s iterator.  However, for MLIR's
-/// usage we need to have an iterator which also holds onto the underlying
-/// `SparseTensorCOO<V>` so that it can be freed whenever the iterator
-/// is freed.
-//
-// We name this `SparseTensorIterator` rather than `SparseTensorCOOIterator`
-// for future-proofing, since the use of `SparseTensorCOO` is an
-// implementation detail that we eventually want to change (e.g., to
-// use `SparseTensorEnumerator` directly, rather than constructing the
-// intermediate `SparseTensorCOO` at all).
-template <typename V>
-class SparseTensorIterator final {
-public:
-  /// This ctor requires `coo` to be a non-null pointer to a dynamically
-  /// allocated object, and takes ownership of that object.  Therefore,
-  /// callers must not free the underlying COO object, since the iterator's
-  /// dtor will do so.
-  explicit SparseTensorIterator(const SparseTensorCOO<V> *coo)
-      : coo(coo), it(coo->begin()), end(coo->end()) {}
-
-  ~SparseTensorIterator() { delete coo; }
-
-  // Disable copy-ctor and copy-assignment, to prevent double-free.
-  SparseTensorIterator(const SparseTensorIterator<V> &) = delete;
-  SparseTensorIterator<V> &operator=(const SparseTensorIterator<V> &) = delete;
-
-  /// Gets the next element.  If there are no remaining elements, then
-  /// returns nullptr.
-  const Element<V> *getNext() { return it < end ? &*it++ : nullptr; }
-
-private:
-  const SparseTensorCOO<V> *const coo; // Owning pointer.
-  typename SparseTensorCOO<V>::const_iterator it;
-  const typename SparseTensorCOO<V>::const_iterator end;
-};
-
-//===----------------------------------------------------------------------===//
-//
-// Utilities for manipulating `StridedMemRefType`.
-//
-//===----------------------------------------------------------------------===//
-
-// We shouldn't need to use `detail::safelyEQ` here since the `1` is a literal.
 #define ASSERT_NO_STRIDE(MEMREF)                                               \
   do {                                                                         \
     assert((MEMREF) && "Memref is nullptr");                                   \
     assert(((MEMREF)->strides[0] == 1) && "Memref has non-trivial stride");    \
   } while (false)
 
-// All our functions use `uint64_t` for ranks, but `StridedMemRefType::sizes`
-// uses `int64_t` on some platforms.  So we explicitly cast this lookup to
-// ensure we get a consistent type, and we use `checkOverflowCast` rather
-// than `static_cast` just to be extremely sure that the casting can't
-// go awry.  (The cast should aways be safe since (1) sizes should never
-// be negative, and (2) the maximum `int64_t` is smaller than the maximum
-// `uint64_t`.  But it's better to be safe than sorry.)
 #define MEMREF_GET_USIZE(MEMREF)                                               \
   detail::checkOverflowCast<uint64_t>((MEMREF)->sizes[0])
 
@@ -137,28 +84,19 @@ class SparseTensorIterator final {
 
 #define MEMREF_GET_PAYLOAD(MEMREF) ((MEMREF)->data + (MEMREF)->offset)
 
-/// Initializes the memref with the provided size and data pointer.  This
+/// Initializes the memref with the provided size and data pointer. This
 /// is designed for functions which want to "return" a memref that aliases
 /// into memory owned by some other object (e.g., `SparseTensorStorage`),
 /// without doing any actual copying.  (The "return" is in scarequotes
 /// because the `_mlir_ciface_` calling convention migrates any returned
 /// memrefs into an out-parameter passed before all the other function
 /// parameters.)
-///
-/// We make this a function rather than a macro mainly for type safety
-/// reasons.  This function does not modify the data pointer, but it
-/// cannot be marked `const` because it is stored into the (necessarily)
-/// non-`const` memref.  This function is templated over the `DataSizeT`
-/// to work around signedness warnings due to many data types having
-/// varying signedness across different platforms.  The templating allows
-/// this function to ensure that it does the right thing and never
-/// introduces errors due to implicit conversions.
 template <typename DataSizeT, typename T>
 static inline void aliasIntoMemref(DataSizeT size, T *data,
                                    StridedMemRefType<T, 1> &ref) {
   ref.basePtr = ref.data = data;
   ref.offset = 0;
-  using MemrefSizeT = typename std::remove_reference_t<decltype(ref.sizes[0])>;
+  using MemrefSizeT = std::remove_reference_t<decltype(ref.sizes[0])>;
   ref.sizes[0] = detail::checkOverflowCast<MemrefSizeT>(size);
   ref.strides[0] = 1;
 }
@@ -177,41 +115,46 @@ extern "C" {
 #define CASE(p, c, v, P, C, V)                                                 \
   if (posTp == (p) && crdTp == (c) && valTp == (v)) {                          \
     switch (action) {                                                          \
-    case Action::kEmpty:                                                       \
+    case Action::kEmpty: {                                                     \
+      return SparseTensorStorage<P, C, V>::newEmpty(                           \
+          dimRank, dimSizes, lvlRank, lvlSizes, lvlTypes, dim2lvl, lvl2dim,    \
+          false);                                                              \
+    }                                                                          \
+    case Action::kEmptyForward: {                                              \
       return SparseTensorStorage<P, C, V>::newEmpty(                           \
-          dimRank, dimSizes, lvlRank, lvlSizes, lvlTypes, lvl2dim);            \
+          dimRank, dimSizes, lvlRank, lvlSizes, lvlTypes, dim2lvl, lvl2dim,    \
+          true);                                                               \
+    }                                                                          \
     case Action::kFromCOO: {                                                   \
       assert(ptr && "Received nullptr for SparseTensorCOO object");            \
       auto &coo = *static_cast<SparseTensorCOO<V> *>(ptr);                     \
       return SparseTensorStorage<P, C, V>::newFromCOO(                         \
-          dimRank, dimSizes, lvlRank, lvlTypes, lvl2dim, coo);                 \
+          dimRank, dimSizes, lvlRank, lvlSizes, lvlTypes, dim2lvl, lvl2dim,    \
+          coo);                                                                \
     }                                                                          \
-    case Action::kSparseToSparse: {                                            \
-      assert(ptr && "Received nullptr for SparseTensorStorage object");        \
-      auto &tensor = *static_cast<SparseTensorStorageBase *>(ptr);             \
-      return SparseTensorStorage<P, C, V>::newFromSparseTensor(                \
-          dimRank, dimSizes, lvlRank, lvlSizes, lvlTypes, lvl2dim, dimRank,    \
-          dim2lvl, tensor);                                                    \
+    case Action::kFromReader: {                                                \
+      assert(ptr && "Received nullptr for SparseTensorReader object");         \
+      SparseTensorReader &reader = *static_cast<SparseTensorReader *>(ptr);    \
+      return static_cast<void *>(reader.readSparseTensor<P, C, V>(             \
+          lvlRank, lvlSizes, lvlTypes, dim2lvl, lvl2dim));                     \
     }                                                                          \
-    case Action::kEmptyCOO:                                                    \
-      return new SparseTensorCOO<V>(lvlRank, lvlSizes);                        \
     case Action::kToCOO: {                                                     \
       assert(ptr && "Received nullptr for SparseTensorStorage object");        \
       auto &tensor = *static_cast<SparseTensorStorage<P, C, V> *>(ptr);        \
-      return tensor.toCOO(lvlRank, lvlSizes, dimRank, dim2lvl);                \
-    }                                                                          \
-    case Action::kToIterator: {                                                \
-      assert(ptr && "Received nullptr for SparseTensorStorage object");        \
-      auto &tensor = *static_cast<SparseTensorStorage<P, C, V> *>(ptr);        \
-      auto *coo = tensor.toCOO(lvlRank, lvlSizes, dimRank, dim2lvl);           \
-      return new SparseTensorIterator<V>(coo);                                 \
+      return tensor.toCOO();                                                   \
     }                                                                          \
     case Action::kPack: {                                                      \
       assert(ptr && "Received nullptr for SparseTensorStorage object");        \
       intptr_t *buffers = static_cast<intptr_t *>(ptr);                        \
       return SparseTensorStorage<P, C, V>::packFromLvlBuffers(                 \
-          dimRank, dimSizes, lvlRank, lvlSizes, lvlTypes, lvl2dim, dimRank,    \
-          dim2lvl, buffers);                                                   \
+          dimRank, dimSizes, lvlRank, lvlSizes, lvlTypes, dim2lvl, lvl2dim,    \
+          dimRank, buffers);                                                   \
+    }                                                                          \
+    case Action::kSortCOOInPlace: {                                            \
+      assert(ptr && "Received nullptr for SparseTensorStorage object");        \
+      auto &tensor = *static_cast<SparseTensorStorage<P, C, V> *>(ptr);        \
+      tensor.sortInPlace();                                                    \
+      return ptr;                                                              \
     }                                                                          \
     }                                                                          \
     MLIR_SPARSETENSOR_FATAL("unknown action: %d\n",                            \
@@ -242,17 +185,14 @@ void *_mlir_ciface_newSparseTensor( // NOLINT
   const uint64_t dimRank = MEMREF_GET_USIZE(dimSizesRef);
   const uint64_t lvlRank = MEMREF_GET_USIZE(lvlSizesRef);
   ASSERT_USIZE_EQ(lvlTypesRef, lvlRank);
-  ASSERT_USIZE_EQ(dim2lvlRef, dimRank);
-  ASSERT_USIZE_EQ(lvl2dimRef, lvlRank);
+  ASSERT_USIZE_EQ(dim2lvlRef, lvlRank);
+  ASSERT_USIZE_EQ(lvl2dimRef, dimRank);
   const index_type *dimSizes = MEMREF_GET_PAYLOAD(dimSizesRef);
   const index_type *lvlSizes = MEMREF_GET_PAYLOAD(lvlSizesRef);
   const DimLevelType *lvlTypes = MEMREF_GET_PAYLOAD(lvlTypesRef);
   const index_type *dim2lvl = MEMREF_GET_PAYLOAD(dim2lvlRef);
   const index_type *lvl2dim = MEMREF_GET_PAYLOAD(lvl2dimRef);
 
-  // Prepare map.
-  // TODO: start using MapRef map(dimRank, lvlRank, dim2lvl, lvl2dim) below
-
   // Rewrite kIndex to kU64, to avoid introducing a bunch of new cases.
   // This is safe because of the static_assert above.
   if (posTp == OverheadType::kIndex)
@@ -361,7 +301,6 @@ void *_mlir_ciface_newSparseTensor( // NOLINT
   CASE_SECSAME(OverheadType::kU64, PrimaryType::kC32, uint64_t, complex32);
 
   // Unsupported case (add above if needed).
-  // TODO: better pretty-printing of enum values!
   MLIR_SPARSETENSOR_FATAL(
       "unsupported combination of types: <P=%d, C=%d, V=%d>\n",
       static_cast<int>(posTp), static_cast<int>(crdTp),
@@ -402,52 +341,20 @@ MLIR_SPARSETENSOR_FOREVERY_O(IMPL_SPARSECOORDINATES)
 #undef IMPL_SPARSECOORDINATES
 #undef IMPL_GETOVERHEAD
 
-// TODO: use MapRef here for translation of coordinates
-// TOOD: remove dim2lvl
-#define IMPL_ADDELT(VNAME, V)                                                  \
-  void *_mlir_ciface_addElt##VNAME(                                            \
-      void *lvlCOO, StridedMemRefType<V, 0> *vref,                             \
-      StridedMemRefType<index_type, 1> *dimCoordsRef,                          \
-      StridedMemRefType<index_type, 1> *dim2lvlRef) {                          \
-    assert(lvlCOO &&vref);                                                     \
+#define IMPL_FORWARDINGINSERT(VNAME, V)                                        \
+  void _mlir_ciface_forwardingInsert##VNAME(                                   \
+      void *t, StridedMemRefType<V, 0> *vref,                                  \
+      StridedMemRefType<index_type, 1> *dimCoordsRef) {                        \
+    assert(t &&vref);                                                          \
     ASSERT_NO_STRIDE(dimCoordsRef);                                            \
-    ASSERT_NO_STRIDE(dim2lvlRef);                                              \
-    const uint64_t rank = MEMREF_GET_USIZE(dimCoordsRef);                      \
-    ASSERT_USIZE_EQ(dim2lvlRef, rank);                                         \
     const index_type *dimCoords = MEMREF_GET_PAYLOAD(dimCoordsRef);            \
-    const index_type *dim2lvl = MEMREF_GET_PAYLOAD(dim2lvlRef);                \
-    std::vector<index_type> lvlCoords(rank);                                   \
-    for (uint64_t d = 0; d < rank; ++d)                                        \
-      lvlCoords[dim2lvl[d]] = dimCoords[d];                                    \
-    V *value = MEMREF_GET_PAYLOAD(vref);                                       \
-    static_cast<SparseTensorCOO<V> *>(lvlCOO)->add(lvlCoords, *value);         \
-    return lvlCOO;                                                             \
-  }
-MLIR_SPARSETENSOR_FOREVERY_V(IMPL_ADDELT)
-#undef IMPL_ADDELT
-
-// NOTE: the `cref` argument uses the same coordinate-space as the `iter`
-// (which can be either dim- or lvl-coords, depending on context).
-#define IMPL_GETNEXT(VNAME, V)                                                 \
-  bool _mlir_ciface_getNext##VNAME(void *iter,                                 \
-                                   StridedMemRefType<index_type, 1> *cref,     \
-                                   StridedMemRefType<V, 0> *vref) {            \
-    assert(iter &&vref);                                                       \
-    ASSERT_NO_STRIDE(cref);                                                    \
-    index_type *coords = MEMREF_GET_PAYLOAD(cref);                             \
-    V *value = MEMREF_GET_PAYLOAD(vref);                                       \
-    const uint64_t rank = MEMREF_GET_USIZE(cref);                              \
-    const Element<V> *elem =                                                   \
-        static_cast<SparseTensorIterator<V> *>(iter)->getNext();               \
-    if (elem == nullptr)                                                       \
-      return false;                                                            \
-    for (uint64_t d = 0; d < rank; d++)                                        \
-      coords[d] = elem->coords[d];                                             \
-    *value = elem->value;                                                      \
-    return true;                                                               \
+    assert(dimCoords);                                                         \
+    const V *value = MEMREF_GET_PAYLOAD(vref);                                 \
+    static_cast<SparseTensorStorageBase *>(t)->forwardingInsert(dimCoords,     \
+                                                                *value);       \
   }
-MLIR_SPARSETENSOR_FOREVERY_V(IMPL_GETNEXT)
-#undef IMPL_GETNEXT
+MLIR_SPARSETENSOR_FOREVERY_V(IMPL_FORWARDINGINSERT)
+#undef IMPL_FORWARDINGINSERT
 
 #define IMPL_LEXINSERT(VNAME, V)                                               \
   void _mlir_ciface_lexInsert##VNAME(                                          \
@@ -516,10 +423,10 @@ void _mlir_ciface_getSparseTensorReaderDimSizes(
     ASSERT_NO_STRIDE(cref);                                                    \
     ASSERT_NO_STRIDE(vref);                                                    \
     const uint64_t dimRank = reader.getRank();                                 \
-    const uint64_t lvlRank = MEMREF_GET_USIZE(lvl2dimRef);                     \
+    const uint64_t lvlRank = MEMREF_GET_USIZE(dim2lvlRef);                     \
     const uint64_t cSize = MEMREF_GET_USIZE(cref);                             \
     const uint64_t vSize = MEMREF_GET_USIZE(vref);                             \
-    ASSERT_USIZE_EQ(dim2lvlRef, dimRank);                                      \
+    ASSERT_USIZE_EQ(lvl2dimRef, dimRank);                                      \
     assert(cSize >= lvlRank * vSize);                                          \
     assert(vSize >= reader.getNSE() && "Not enough space in buffers");         \
     (void)dimRank;                                                             \
@@ -535,114 +442,6 @@ void _mlir_ciface_getSparseTensorReaderDimSizes(
 MLIR_SPARSETENSOR_FOREVERY_V_O(IMPL_GETNEXT)
 #undef IMPL_GETNEXT
 
-void *_mlir_ciface_newSparseTensorFromReader(
-    void *p, StridedMemRefType<index_type, 1> *lvlSizesRef,
-    StridedMemRefType<DimLevelType, 1> *lvlTypesRef,
-    StridedMemRefType<index_type, 1> *dim2lvlRef,
-    StridedMemRefType<index_type, 1> *lvl2dimRef, OverheadType posTp,
-    OverheadType crdTp, PrimaryType valTp) {
-  assert(p);
-  SparseTensorReader &reader = *static_cast<SparseTensorReader *>(p);
-  ASSERT_NO_STRIDE(lvlSizesRef);
-  ASSERT_NO_STRIDE(lvlTypesRef);
-  ASSERT_NO_STRIDE(dim2lvlRef);
-  ASSERT_NO_STRIDE(lvl2dimRef);
-  const uint64_t dimRank = reader.getRank();
-  const uint64_t lvlRank = MEMREF_GET_USIZE(lvlSizesRef);
-  ASSERT_USIZE_EQ(lvlTypesRef, lvlRank);
-  ASSERT_USIZE_EQ(dim2lvlRef, dimRank);
-  ASSERT_USIZE_EQ(lvl2dimRef, lvlRank);
-  (void)dimRank;
-  const index_type *lvlSizes = MEMREF_GET_PAYLOAD(lvlSizesRef);
-  const DimLevelType *lvlTypes = MEMREF_GET_PAYLOAD(lvlTypesRef);
-  const index_type *dim2lvl = MEMREF_GET_PAYLOAD(dim2lvlRef);
-  const index_type *lvl2dim = MEMREF_GET_PAYLOAD(lvl2dimRef);
-#define CASE(p, c, v, P, C, V)                                                 \
-  if (posTp == OverheadType::p && crdTp == OverheadType::c &&                  \
-      valTp == PrimaryType::v)                                                 \
-    return static_cast<void *>(reader.readSparseTensor<P, C, V>(               \
-        lvlRank, lvlSizes, lvlTypes, dim2lvl, lvl2dim));
-#define CASE_SECSAME(p, v, P, V) CASE(p, p, v, P, P, V)
-  // Rewrite kIndex to kU64, to avoid introducing a bunch of new cases.
-  // This is safe because of the static_assert above.
-  if (posTp == OverheadType::kIndex)
-    posTp = OverheadType::kU64;
-  if (crdTp == OverheadType::kIndex)
-    crdTp = OverheadType::kU64;
-  // Double matrices with all combinations of overhead storage.
-  CASE(kU64, kU64, kF64, uint64_t, uint64_t, double);
-  CASE(kU64, kU32, kF64, uint64_t, uint32_t, double);
-  CASE(kU64, kU16, kF64, uint64_t, uint16_t, double);
-  CASE(kU64, kU8, kF64, uint64_t, uint8_t, double);
-  CASE(kU32, kU64, kF64, uint32_t, uint64_t, double);
-  CASE(kU32, kU32, kF64, uint32_t, uint32_t, double);
-  CASE(kU32, kU16, kF64, uint32_t, uint16_t, double);
-  CASE(kU32, kU8, kF64, uint32_t, uint8_t, double);
-  CASE(kU16, kU64, kF64, uint16_t, uint64_t, double);
-  CASE(kU16, kU32, kF64, uint16_t, uint32_t, double);
-  CASE(kU16, kU16, kF64, uint16_t, uint16_t, double);
-  CASE(kU16, kU8, kF64, uint16_t, uint8_t, double);
-  CASE(kU8, kU64, kF64, uint8_t, uint64_t, double);
-  CASE(kU8, kU32, kF64, uint8_t, uint32_t, double);
-  CASE(kU8, kU16, kF64, uint8_t, uint16_t, double);
-  CASE(kU8, kU8, kF64, uint8_t, uint8_t, double);
-  // Float matrices with all combinations of overhead storage.
-  CASE(kU64, kU64, kF32, uint64_t, uint64_t, float);
-  CASE(kU64, kU32, kF32, uint64_t, uint32_t, float);
-  CASE(kU64, kU16, kF32, uint64_t, uint16_t, float);
-  CASE(kU64, kU8, kF32, uint64_t, uint8_t, float);
-  CASE(kU32, kU64, kF32, uint32_t, uint64_t, float);
-  CASE(kU32, kU32, kF32, uint32_t, uint32_t, float);
-  CASE(kU32, kU16, kF32, uint32_t, uint16_t, float);
-  CASE(kU32, kU8, kF32, uint32_t, uint8_t, float);
-  CASE(kU16, kU64, kF32, uint16_t, uint64_t, float);
-  CASE(kU16, kU32, kF32, uint16_t, uint32_t, float);
-  CASE(kU16, kU16, kF32, uint16_t, uint16_t, float);
-  CASE(kU16, kU8, kF32, uint16_t, uint8_t, float);
-  CASE(kU8, kU64, kF32, uint8_t, uint64_t, float);
-  CASE(kU8, kU32, kF32, uint8_t, uint32_t, float);
-  CASE(kU8, kU16, kF32, uint8_t, uint16_t, float);
-  CASE(kU8, kU8, kF32, uint8_t, uint8_t, float);
-  // Two-byte floats with both overheads of the same type.
-  CASE_SECSAME(kU64, kF16, uint64_t, f16);
-  CASE_SECSAME(kU64, kBF16, uint64_t, bf16);
-  CASE_SECSAME(kU32, kF16, uint32_t, f16);
-  CASE_SECSAME(kU32, kBF16, uint32_t, bf16);
-  CASE_SECSAME(kU16, kF16, uint16_t, f16);
-  CASE_SECSAME(kU16, kBF16, uint16_t, bf16);
-  CASE_SECSAME(kU8, kF16, uint8_t, f16);
-  CASE_SECSAME(kU8, kBF16, uint8_t, bf16);
-  // Integral matrices with both overheads of the same type.
-  CASE_SECSAME(kU64, kI64, uint64_t, int64_t);
-  CASE_SECSAME(kU64, kI32, uint64_t, int32_t);
-  CASE_SECSAME(kU64, kI16, uint64_t, int16_t);
-  CASE_SECSAME(kU64, kI8, uint64_t, int8_t);
-  CASE_SECSAME(kU32, kI64, uint32_t, int64_t);
-  CASE_SECSAME(kU32, kI32, uint32_t, int32_t);
-  CASE_SECSAME(kU32, kI16, uint32_t, int16_t);
-  CASE_SECSAME(kU32, kI8, uint32_t, int8_t);
-  CASE_SECSAME(kU16, kI64, uint16_t, int64_t);
-  CASE_SECSAME(kU16, kI32, uint16_t, int32_t);
-  CASE_SECSAME(kU16, kI16, uint16_t, int16_t);
-  CASE_SECSAME(kU16, kI8, uint16_t, int8_t);
-  CASE_SECSAME(kU8, kI64, uint8_t, int64_t);
-  CASE_SECSAME(kU8, kI32, uint8_t, int32_t);
-  CASE_SECSAME(kU8, kI16, uint8_t, int16_t);
-  CASE_SECSAME(kU8, kI8, uint8_t, int8_t);
-  // Complex matrices with wide overhead.
-  CASE_SECSAME(kU64, kC64, uint64_t, complex64);
-  CASE_SECSAME(kU64, kC32, uint64_t, complex32);
-
-  // Unsupported case (add above if needed).
-  // TODO: better pretty-printing of enum values!
-  MLIR_SPARSETENSOR_FATAL(
-      "unsupported combination of types: <P=%d, C=%d, V=%d>\n",
-      static_cast<int>(posTp), static_cast<int>(crdTp),
-      static_cast<int>(valTp));
-#undef CASE_SECSAME
-#undef CASE
-}
-
 void _mlir_ciface_outSparseTensorWriterMetaData(
     void *p, index_type dimRank, index_type nse,
     StridedMemRefType<index_type, 1> *dimSizesRef) {
@@ -650,9 +449,9 @@ void _mlir_ciface_outSparseTensorWriterMetaData(
   ASSERT_NO_STRIDE(dimSizesRef);
   assert(dimRank != 0);
   index_type *dimSizes = MEMREF_GET_PAYLOAD(dimSizesRef);
-  SparseTensorWriter &file = *static_cast<SparseTensorWriter *>(p);
+  std::ostream &file = *static_cast<std::ostream *>(p);
   file << dimRank << " " << nse << std::endl;
-  for (index_type d = 0; d < dimRank - 1; ++d)
+  for (index_type d = 0; d < dimRank - 1; d++)
     file << dimSizes[d] << " ";
   file << dimSizes[dimRank - 1] << std::endl;
 }
@@ -665,8 +464,8 @@ void _mlir_ciface_outSparseTensorWriterMetaData(
     assert(p &&vref);                                                          \
     ASSERT_NO_STRIDE(dimCoordsRef);                                            \
     const index_type *dimCoords = MEMREF_GET_PAYLOAD(dimCoordsRef);            \
-    SparseTensorWriter &file = *static_cast<SparseTensorWriter *>(p);          \
-    for (index_type d = 0; d < dimRank; ++d)                                   \
+    std::ostream &file = *static_cast<std::ostream *>(p);                      \
+    for (index_type d = 0; d < dimRank; d++)                                   \
       file << (dimCoords[d] + 1) << " ";                                       \
     V *value = MEMREF_GET_PAYLOAD(vref);                                       \
     file << *value << std::endl;                                               \
@@ -689,13 +488,17 @@ index_type sparseDimSize(void *tensor, index_type d) {
   return static_cast<SparseTensorStorageBase *>(tensor)->getDimSize(d);
 }
 
-void endInsert(void *tensor) {
-  return static_cast<SparseTensorStorageBase *>(tensor)->endInsert();
+void endForwardingInsert(void *tensor) {
+  return static_cast<SparseTensorStorageBase *>(tensor)->endForwardingInsert();
+}
+
+void endLexInsert(void *tensor) {
+  return static_cast<SparseTensorStorageBase *>(tensor)->endLexInsert();
 }
 
 #define IMPL_OUTSPARSETENSOR(VNAME, V)                                         \
   void outSparseTensor##VNAME(void *coo, void *dest, bool sort) {              \
-    assert(coo && "Got nullptr for COO object");                               \
+    assert(coo);                                                               \
     auto &coo_ = *static_cast<SparseTensorCOO<V> *>(coo);                      \
     if (sort)                                                                  \
       coo_.sort();                                                             \
@@ -715,13 +518,6 @@ void delSparseTensor(void *tensor) {
 MLIR_SPARSETENSOR_FOREVERY_V(IMPL_DELCOO)
 #undef IMPL_DELCOO
 
-#define IMPL_DELITER(VNAME, V)                                                 \
-  void delSparseTensorIterator##VNAME(void *iter) {                            \
-    delete static_cast<SparseTensorIterator<V> *>(iter);                       \
-  }
-MLIR_SPARSETENSOR_FOREVERY_V(IMPL_DELITER)
-#undef IMPL_DELITER
-
 char *getTensorFilename(index_type id) {
   constexpr size_t BUF_SIZE = 80;
   char var[BUF_SIZE];
@@ -732,47 +528,23 @@ char *getTensorFilename(index_type id) {
   return env;
 }
 
-void readSparseTensorShape(char *filename, std::vector<uint64_t> *out) {
-  assert(out && "Received nullptr for out-parameter");
-  SparseTensorReader reader(filename);
-  reader.openFile();
-  reader.readHeader();
-  reader.closeFile();
-  const uint64_t dimRank = reader.getRank();
-  const uint64_t *dimSizes = reader.getDimSizes();
-  out->reserve(dimRank);
-  out->assign(dimSizes, dimSizes + dimRank);
-}
-
-index_type getSparseTensorReaderRank(void *p) {
-  return static_cast<SparseTensorReader *>(p)->getRank();
-}
-
-bool getSparseTensorReaderIsSymmetric(void *p) {
-  return static_cast<SparseTensorReader *>(p)->isSymmetric();
-}
-
 index_type getSparseTensorReaderNSE(void *p) {
   return static_cast<SparseTensorReader *>(p)->getNSE();
 }
 
-index_type getSparseTensorReaderDimSize(void *p, index_type d) {
-  return static_cast<SparseTensorReader *>(p)->getDimSize(d);
-}
-
 void delSparseTensorReader(void *p) {
   delete static_cast<SparseTensorReader *>(p);
 }
 
 void *createSparseTensorWriter(char *filename) {
-  SparseTensorWriter *file =
+  std::ostream *file =
       (filename[0] == 0) ? &std::cout : new std::ofstream(filename);
   *file << "# extended FROSTT format\n";
   return static_cast<void *>(file);
 }
 
 void delSparseTensorWriter(void *p) {
-  SparseTensorWriter *file = static_cast<SparseTensorWriter *>(p);
+  std::ostream *file = static_cast<std::ostream *>(p);
   file->flush();
   assert(file->good());
   if (file != &std::cout)
diff --git a/mlir/lib/IR/BuiltinAttributes.cpp b/mlir/lib/IR/BuiltinAttributes.cpp
index 64949a1710..89b1ed67f5 100644
--- a/mlir/lib/IR/BuiltinAttributes.cpp
+++ b/mlir/lib/IR/BuiltinAttributes.cpp
@@ -20,9 +20,12 @@
 #include "llvm/ADT/APSInt.h"
 #include "llvm/ADT/Sequence.h"
 #include "llvm/ADT/TypeSwitch.h"
+#include "llvm/Support/Debug.h"
 #include "llvm/Support/Endian.h"
 #include <optional>
 
+#define DEBUG_TYPE "builtinattributes"
+
 using namespace mlir;
 using namespace mlir::detail;
 
@@ -1098,24 +1101,44 @@ bool DenseElementsAttr::isValidRawBuffer(ShapedType type,
 static bool isValidIntOrFloat(Type type, int64_t dataEltSize, bool isInt,
                               bool isSigned) {
   // Make sure that the data element size is the same as the type element width.
-  if (getDenseElementBitWidth(type) !=
-      static_cast<size_t>(dataEltSize * CHAR_BIT))
+  auto denseEltBitWidth = getDenseElementBitWidth(type);
+  auto dataSize = static_cast<size_t>(dataEltSize * CHAR_BIT);
+  if (denseEltBitWidth != dataSize) {
+    LLVM_DEBUG(llvm::dbgs() << "expected dense element bit width "
+                            << denseEltBitWidth << " to match data size "
+                            << dataSize << " for type " << type << "\n");
     return false;
+  }
 
   // Check that the element type is either float or integer or index.
-  if (!isInt)
-    return llvm::isa<FloatType>(type);
+  if (!isInt) {
+    bool valid = llvm::isa<FloatType>(type);
+    if (!valid)
+      LLVM_DEBUG(llvm::dbgs()
+                 << "expected float type when isInt is false, but found "
+                 << type << "\n");
+    return valid;
+  }
   if (type.isIndex())
     return true;
 
   auto intType = llvm::dyn_cast<IntegerType>(type);
-  if (!intType)
+  if (!intType) {
+    LLVM_DEBUG(llvm::dbgs()
+               << "expected integer type when isInt is true, but found " << type
+               << "\n");
     return false;
+  }
 
   // Make sure signedness semantics is consistent.
   if (intType.isSignless())
     return true;
-  return intType.isSigned() ? isSigned : !isSigned;
+
+  bool valid = intType.isSigned() == isSigned;
+  if (!valid)
+    LLVM_DEBUG(llvm::dbgs() << "expected signedness " << isSigned
+                            << " to match type " << type << "\n");
+  return valid;
 }
 
 /// Defaults down the subclass implementation.
@@ -1247,12 +1270,14 @@ DenseElementsAttr DenseElementsAttr::bitcast(Type newElType) {
 DenseElementsAttr
 DenseElementsAttr::mapValues(Type newElementType,
                              function_ref<APInt(const APInt &)> mapping) const {
-  return llvm::cast<DenseIntElementsAttr>(*this).mapValues(newElementType, mapping);
+  return llvm::cast<DenseIntElementsAttr>(*this).mapValues(newElementType,
+                                                           mapping);
 }
 
 DenseElementsAttr DenseElementsAttr::mapValues(
     Type newElementType, function_ref<APInt(const APFloat &)> mapping) const {
-  return llvm::cast<DenseFPElementsAttr>(*this).mapValues(newElementType, mapping);
+  return llvm::cast<DenseFPElementsAttr>(*this).mapValues(newElementType,
+                                                          mapping);
 }
 
 ShapedType DenseElementsAttr::getType() const {
@@ -1331,8 +1356,9 @@ DenseElementsAttr DenseIntOrFPElementsAttr::getRawComplex(ShapedType type,
                                                           bool isInt,
                                                           bool isSigned) {
   assert(::isValidIntOrFloat(
-      llvm::cast<ComplexType>(type.getElementType()).getElementType(),
-      dataEltSize / 2, isInt, isSigned));
+             llvm::cast<ComplexType>(type.getElementType()).getElementType(),
+             dataEltSize / 2, isInt, isSigned) &&
+         "Try re-running with -debug-only=builtinattributes");
 
   int64_t numElements = data.size() / dataEltSize;
   (void)numElements;
@@ -1347,8 +1373,9 @@ DenseElementsAttr
 DenseIntOrFPElementsAttr::getRawIntOrFloat(ShapedType type, ArrayRef<char> data,
                                            int64_t dataEltSize, bool isInt,
                                            bool isSigned) {
-  assert(
-      ::isValidIntOrFloat(type.getElementType(), dataEltSize, isInt, isSigned));
+  assert(::isValidIntOrFloat(type.getElementType(), dataEltSize, isInt,
+                             isSigned) &&
+         "Try re-running with -debug-only=builtinattributes");
 
   int64_t numElements = data.size() / dataEltSize;
   assert(numElements == 1 || numElements == type.getNumElements());
diff --git a/mlir/lib/Interfaces/LoopLikeInterface.cpp b/mlir/lib/Interfaces/LoopLikeInterface.cpp
index 781a21bb3e..15a816f4e4 100644
--- a/mlir/lib/Interfaces/LoopLikeInterface.cpp
+++ b/mlir/lib/Interfaces/LoopLikeInterface.cpp
@@ -52,3 +52,40 @@ bool LoopLikeOpInterface::blockIsInLoop(Block *block) {
   }
   return false;
 }
+
+LogicalResult detail::verifyLoopLikeOpInterface(Operation *op) {
+  // Note: These invariants are also verified by the RegionBranchOpInterface,
+  // but the LoopLikeOpInterface provides better error messages.
+  auto loopLikeOp = cast<LoopLikeOpInterface>(op);
+
+  // Verify number of inits/iter_args/yielded values.
+  if (loopLikeOp.getInits().size() != loopLikeOp.getRegionIterArgs().size())
+    return op->emitOpError("different number of inits and region iter_args: ")
+           << loopLikeOp.getInits().size()
+           << " != " << loopLikeOp.getRegionIterArgs().size();
+  if (loopLikeOp.getRegionIterArgs().size() !=
+      loopLikeOp.getYieldedValues().size())
+    return op->emitOpError(
+               "different number of region iter_args and yielded values: ")
+           << loopLikeOp.getRegionIterArgs().size()
+           << " != " << loopLikeOp.getYieldedValues().size();
+
+  // Verify types of inits/iter_args/yielded values.
+  int64_t i = 0;
+  for (const auto it :
+       llvm::zip_equal(loopLikeOp.getInits(), loopLikeOp.getRegionIterArgs(),
+                       loopLikeOp.getYieldedValues())) {
+    if (std::get<0>(it).getType() != std::get<1>(it).getType())
+      op->emitOpError(std::to_string(i))
+          << "-th init and " << i << "-th region iter_arg have different type: "
+          << std::get<0>(it).getType() << " != " << std::get<1>(it).getType();
+    if (std::get<1>(it).getType() != std::get<2>(it).getType())
+      op->emitOpError(std::to_string(i))
+          << "-th region iter_arg and " << i
+          << "-th yielded value have different type: "
+          << std::get<1>(it).getType() << " != " << std::get<2>(it).getType();
+    ++i;
+  }
+
+  return success();
+}
diff --git a/mlir/lib/Interfaces/ValueBoundsOpInterface.cpp b/mlir/lib/Interfaces/ValueBoundsOpInterface.cpp
index c00ee0315a..ff94111521 100644
--- a/mlir/lib/Interfaces/ValueBoundsOpInterface.cpp
+++ b/mlir/lib/Interfaces/ValueBoundsOpInterface.cpp
@@ -484,25 +484,32 @@ FailureOr<int64_t> ValueBoundsConstraintSet::computeConstantBound(
   return failure();
 }
 
-FailureOr<bool>
-ValueBoundsConstraintSet::areEqual(Value value1, Value value2,
-                                   std::optional<int64_t> dim1,
-                                   std::optional<int64_t> dim2) {
+FailureOr<int64_t>
+ValueBoundsConstraintSet::computeConstantDelta(Value value1, Value value2,
+                                               std::optional<int64_t> dim1,
+                                               std::optional<int64_t> dim2) {
 #ifndef NDEBUG
   assertValidValueDim(value1, dim1);
   assertValidValueDim(value2, dim2);
 #endif // NDEBUG
 
-  // Subtract the two values/dimensions from each other. If the result is 0,
-  // both are equal.
   Builder b(value1.getContext());
   AffineMap map = AffineMap::get(/*dimCount=*/2, /*symbolCount=*/0,
                                  b.getAffineDimExpr(0) - b.getAffineDimExpr(1));
-  FailureOr<int64_t> bound = computeConstantBound(
-      presburger::BoundType::EQ, map, {{value1, dim1}, {value2, dim2}});
-  if (failed(bound))
+  return computeConstantBound(presburger::BoundType::EQ, map,
+                              {{value1, dim1}, {value2, dim2}});
+}
+
+FailureOr<bool>
+ValueBoundsConstraintSet::areEqual(Value value1, Value value2,
+                                   std::optional<int64_t> dim1,
+                                   std::optional<int64_t> dim2) {
+  // Subtract the two values/dimensions from each other. If the result is 0,
+  // both are equal.
+  FailureOr<int64_t> delta = computeConstantDelta(value1, value2, dim1, dim2);
+  if (failed(delta))
     return failure();
-  return *bound == 0;
+  return *delta == 0;
 }
 
 ValueBoundsConstraintSet::BoundBuilder &
diff --git a/mlir/lib/Query/CMakeLists.txt b/mlir/lib/Query/CMakeLists.txt
new file mode 100644
index 0000000000..817583e94c
--- /dev/null
+++ b/mlir/lib/Query/CMakeLists.txt
@@ -0,0 +1,12 @@
+add_mlir_library(MLIRQuery
+  Query.cpp
+  QueryParser.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Query
+
+  LINK_LIBS PUBLIC
+  MLIRQueryMatcher
+  )
+
+add_subdirectory(Matcher)
diff --git a/mlir/lib/Query/Matcher/CMakeLists.txt b/mlir/lib/Query/Matcher/CMakeLists.txt
new file mode 100644
index 0000000000..3adff9f992
--- /dev/null
+++ b/mlir/lib/Query/Matcher/CMakeLists.txt
@@ -0,0 +1,14 @@
+add_mlir_library(MLIRQueryMatcher
+  Parser.cpp
+  RegistryManager.cpp
+  VariantValue.cpp
+  Diagnostics.cpp
+  ErrorBuilder.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Query/Matcher
+
+  LINK_LIBS PUBLIC
+  MLIRIR
+  MLIRParser
+  )
diff --git a/mlir/lib/Query/Matcher/Diagnostics.cpp b/mlir/lib/Query/Matcher/Diagnostics.cpp
new file mode 100644
index 0000000000..10468dbcc5
--- /dev/null
+++ b/mlir/lib/Query/Matcher/Diagnostics.cpp
@@ -0,0 +1,128 @@
+//===- Diagnostic.cpp -----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Diagnostics.h"
+#include "mlir/Query/Matcher/ErrorBuilder.h"
+
+namespace mlir::query::matcher::internal {
+
+Diagnostics::ArgStream &
+Diagnostics::ArgStream::operator<<(const llvm::Twine &arg) {
+  out->push_back(arg.str());
+  return *this;
+}
+
+Diagnostics::ArgStream Diagnostics::addError(SourceRange range,
+                                             ErrorType error) {
+  errorValues.emplace_back();
+  ErrorContent &last = errorValues.back();
+  last.contextStack = contextStack;
+  last.messages.emplace_back();
+  last.messages.back().range = range;
+  last.messages.back().type = error;
+  return ArgStream(&last.messages.back().args);
+}
+
+static llvm::StringRef errorTypeToFormatString(ErrorType type) {
+  switch (type) {
+  case ErrorType::RegistryMatcherNotFound:
+    return "Matcher not found: $0";
+  case ErrorType::RegistryWrongArgCount:
+    return "Incorrect argument count. (Expected = $0) != (Actual = $1)";
+  case ErrorType::RegistryWrongArgType:
+    return "Incorrect type for arg $0. (Expected = $1) != (Actual = $2)";
+  case ErrorType::RegistryValueNotFound:
+    return "Value not found: $0";
+
+  case ErrorType::ParserStringError:
+    return "Error parsing string token: <$0>";
+  case ErrorType::ParserNoOpenParen:
+    return "Error parsing matcher. Found token <$0> while looking for '('.";
+  case ErrorType::ParserNoCloseParen:
+    return "Error parsing matcher. Found end-of-code while looking for ')'.";
+  case ErrorType::ParserNoComma:
+    return "Error parsing matcher. Found token <$0> while looking for ','.";
+  case ErrorType::ParserNoCode:
+    return "End of code found while looking for token.";
+  case ErrorType::ParserNotAMatcher:
+    return "Input value is not a matcher expression.";
+  case ErrorType::ParserInvalidToken:
+    return "Invalid token <$0> found when looking for a value.";
+  case ErrorType::ParserTrailingCode:
+    return "Unexpected end of code.";
+  case ErrorType::ParserOverloadedType:
+    return "Input value has unresolved overloaded type: $0";
+  case ErrorType::ParserFailedToBuildMatcher:
+    return "Failed to build matcher: $0.";
+
+  case ErrorType::None:
+    return "<N/A>";
+  }
+  llvm_unreachable("Unknown ErrorType value.");
+}
+
+static void formatErrorString(llvm::StringRef formatString,
+                              llvm::ArrayRef<std::string> args,
+                              llvm::raw_ostream &os) {
+  while (!formatString.empty()) {
+    std::pair<llvm::StringRef, llvm::StringRef> pieces =
+        formatString.split("$");
+    os << pieces.first.str();
+    if (pieces.second.empty())
+      break;
+
+    const char next = pieces.second.front();
+    formatString = pieces.second.drop_front();
+    if (next >= '0' && next <= '9') {
+      const unsigned index = next - '0';
+      if (index < args.size()) {
+        os << args[index];
+      } else {
+        os << "<Argument_Not_Provided>";
+      }
+    }
+  }
+}
+
+static void maybeAddLineAndColumn(SourceRange range, llvm::raw_ostream &os) {
+  if (range.start.line > 0 && range.start.column > 0) {
+    os << range.start.line << ":" << range.start.column << ": ";
+  }
+}
+
+void Diagnostics::printMessage(
+    const Diagnostics::ErrorContent::Message &message, const llvm::Twine prefix,
+    llvm::raw_ostream &os) const {
+  maybeAddLineAndColumn(message.range, os);
+  os << prefix;
+  formatErrorString(errorTypeToFormatString(message.type), message.args, os);
+}
+
+void Diagnostics::printErrorContent(const Diagnostics::ErrorContent &content,
+                                    llvm::raw_ostream &os) const {
+  if (content.messages.size() == 1) {
+    printMessage(content.messages[0], "", os);
+  } else {
+    for (size_t i = 0, e = content.messages.size(); i != e; ++i) {
+      if (i != 0)
+        os << "\n";
+      printMessage(content.messages[i],
+                   "Candidate " + llvm::Twine(i + 1) + ": ", os);
+    }
+  }
+}
+
+void Diagnostics::print(llvm::raw_ostream &os) const {
+  for (const ErrorContent &error : errorValues) {
+    if (&error != &errorValues.front())
+      os << "\n";
+    printErrorContent(error, os);
+  }
+}
+
+} // namespace mlir::query::matcher::internal
diff --git a/mlir/lib/Query/Matcher/Diagnostics.h b/mlir/lib/Query/Matcher/Diagnostics.h
new file mode 100644
index 0000000000..a58a435b16
--- /dev/null
+++ b/mlir/lib/Query/Matcher/Diagnostics.h
@@ -0,0 +1,82 @@
+//===--- Diagnostics.h - Helper class for error diagnostics -----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Diagnostics class to manage error messages. Implementation shares similarity
+// to clang-query Diagnostics.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_TOOLS_MLIRQUERY_MATCHER_DIAGNOSTICS_H
+#define MLIR_TOOLS_MLIRQUERY_MATCHER_DIAGNOSTICS_H
+
+#include "mlir/Query/Matcher/ErrorBuilder.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/Twine.h"
+#include "llvm/Support/raw_ostream.h"
+#include <string>
+#include <vector>
+
+namespace mlir::query::matcher::internal {
+
+// Diagnostics class to manage error messages.
+class Diagnostics {
+public:
+  // Helper stream class for constructing error messages.
+  class ArgStream {
+  public:
+    ArgStream(std::vector<std::string> *out) : out(out) {}
+    template <class T>
+    ArgStream &operator<<(const T &arg) {
+      return operator<<(llvm::Twine(arg));
+    }
+    ArgStream &operator<<(const llvm::Twine &arg);
+
+  private:
+    std::vector<std::string> *out;
+  };
+
+  // Add an error message with the specified range and error type.
+  // Returns an ArgStream object to allow constructing the error message using
+  // the << operator.
+  ArgStream addError(SourceRange range, ErrorType error);
+
+  // Print all error messages to the specified output stream.
+  void print(llvm::raw_ostream &os) const;
+
+private:
+  // Information stored for one frame of the context.
+  struct ContextFrame {
+    SourceRange range;
+    std::vector<std::string> args;
+  };
+
+  // Information stored for each error found.
+  struct ErrorContent {
+    std::vector<ContextFrame> contextStack;
+    struct Message {
+      SourceRange range;
+      ErrorType type;
+      std::vector<std::string> args;
+    };
+    std::vector<Message> messages;
+  };
+
+  void printMessage(const ErrorContent::Message &message,
+                    const llvm::Twine Prefix, llvm::raw_ostream &os) const;
+
+  void printErrorContent(const ErrorContent &content,
+                         llvm::raw_ostream &os) const;
+
+  std::vector<ContextFrame> contextStack;
+  std::vector<ErrorContent> errorValues;
+};
+
+} // namespace mlir::query::matcher::internal
+
+#endif // MLIR_TOOLS_MLIRQUERY_MATCHER_DIAGNOSTICS_H
diff --git a/mlir/lib/Query/Matcher/ErrorBuilder.cpp b/mlir/lib/Query/Matcher/ErrorBuilder.cpp
new file mode 100644
index 0000000000..de6447dac4
--- /dev/null
+++ b/mlir/lib/Query/Matcher/ErrorBuilder.cpp
@@ -0,0 +1,25 @@
+//===--- ErrorBuilder.cpp - Helper for building error messages ------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Query/Matcher/ErrorBuilder.h"
+#include "Diagnostics.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/Twine.h"
+#include <initializer_list>
+
+namespace mlir::query::matcher::internal {
+
+void addError(Diagnostics *error, SourceRange range, ErrorType errorType,
+              std::initializer_list<llvm::Twine> errorTexts) {
+  Diagnostics::ArgStream argStream = error->addError(range, errorType);
+  for (const llvm::Twine &errorText : errorTexts) {
+    argStream << errorText;
+  }
+}
+
+} // namespace mlir::query::matcher::internal
diff --git a/mlir/lib/Query/Matcher/Parser.cpp b/mlir/lib/Query/Matcher/Parser.cpp
new file mode 100644
index 0000000000..be9e60de22
--- /dev/null
+++ b/mlir/lib/Query/Matcher/Parser.cpp
@@ -0,0 +1,540 @@
+//===- Parser.cpp - Matcher expression parser -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Recursive parser implementation for the matcher expression grammar.
+//
+//===----------------------------------------------------------------------===//
+
+#include "Parser.h"
+
+#include <vector>
+
+namespace mlir::query::matcher::internal {
+
+// Simple structure to hold information for one token from the parser.
+struct Parser::TokenInfo {
+  TokenInfo() = default;
+
+  // Method to set the kind and text of the token
+  void set(TokenKind newKind, llvm::StringRef newText) {
+    kind = newKind;
+    text = newText;
+  }
+
+  llvm::StringRef text;
+  TokenKind kind = TokenKind::Eof;
+  SourceRange range;
+  VariantValue value;
+};
+
+class Parser::CodeTokenizer {
+public:
+  // Constructor with matcherCode and error
+  explicit CodeTokenizer(llvm::StringRef matcherCode, Diagnostics *error)
+      : code(matcherCode), startOfLine(matcherCode), error(error) {
+    nextToken = getNextToken();
+  }
+
+  // Constructor with matcherCode, error, and codeCompletionOffset
+  CodeTokenizer(llvm::StringRef matcherCode, Diagnostics *error,
+                unsigned codeCompletionOffset)
+      : code(matcherCode), startOfLine(matcherCode), error(error),
+        codeCompletionLocation(matcherCode.data() + codeCompletionOffset) {
+    nextToken = getNextToken();
+  }
+
+  // Peek at next token without consuming it
+  const TokenInfo &peekNextToken() const { return nextToken; }
+
+  // Consume and return the next token
+  TokenInfo consumeNextToken() {
+    TokenInfo thisToken = nextToken;
+    nextToken = getNextToken();
+    return thisToken;
+  }
+
+  // Skip any newline tokens
+  TokenInfo skipNewlines() {
+    while (nextToken.kind == TokenKind::NewLine)
+      nextToken = getNextToken();
+    return nextToken;
+  }
+
+  // Consume and return next token, ignoring newlines
+  TokenInfo consumeNextTokenIgnoreNewlines() {
+    skipNewlines();
+    return nextToken.kind == TokenKind::Eof ? nextToken : consumeNextToken();
+  }
+
+  // Return kind of next token
+  TokenKind nextTokenKind() const { return nextToken.kind; }
+
+private:
+  // Helper function to get the first character as a new StringRef and drop it
+  // from the original string
+  llvm::StringRef firstCharacterAndDrop(llvm::StringRef &str) {
+    assert(!str.empty());
+    llvm::StringRef firstChar = str.substr(0, 1);
+    str = str.drop_front();
+    return firstChar;
+  }
+
+  // Get next token, consuming whitespaces and handling different token types
+  TokenInfo getNextToken() {
+    consumeWhitespace();
+    TokenInfo result;
+    result.range.start = currentLocation();
+
+    // Code completion case
+    if (codeCompletionLocation && codeCompletionLocation <= code.data()) {
+      result.set(TokenKind::CodeCompletion,
+                 llvm::StringRef(codeCompletionLocation, 0));
+      codeCompletionLocation = nullptr;
+      return result;
+    }
+
+    // End of file case
+    if (code.empty()) {
+      result.set(TokenKind::Eof, "");
+      return result;
+    }
+
+    // Switch to handle specific characters
+    switch (code[0]) {
+    case '#':
+      code = code.drop_until([](char c) { return c == '\n'; });
+      return getNextToken();
+    case ',':
+      result.set(TokenKind::Comma, firstCharacterAndDrop(code));
+      break;
+    case '.':
+      result.set(TokenKind::Period, firstCharacterAndDrop(code));
+      break;
+    case '\n':
+      ++line;
+      startOfLine = code.drop_front();
+      result.set(TokenKind::NewLine, firstCharacterAndDrop(code));
+      break;
+    case '(':
+      result.set(TokenKind::OpenParen, firstCharacterAndDrop(code));
+      break;
+    case ')':
+      result.set(TokenKind::CloseParen, firstCharacterAndDrop(code));
+      break;
+    case '"':
+    case '\'':
+      consumeStringLiteral(&result);
+      break;
+    default:
+      parseIdentifierOrInvalid(&result);
+      break;
+    }
+
+    result.range.end = currentLocation();
+    return result;
+  }
+
+  // Consume a string literal, handle escape sequences and missing closing
+  // quote.
+  void consumeStringLiteral(TokenInfo *result) {
+    bool inEscape = false;
+    const char marker = code[0];
+    for (size_t length = 1; length < code.size(); ++length) {
+      if (inEscape) {
+        inEscape = false;
+        continue;
+      }
+      if (code[length] == '\\') {
+        inEscape = true;
+        continue;
+      }
+      if (code[length] == marker) {
+        result->kind = TokenKind::Literal;
+        result->text = code.substr(0, length + 1);
+        result->value = code.substr(1, length - 1);
+        code = code.drop_front(length + 1);
+        return;
+      }
+    }
+    llvm::StringRef errorText = code;
+    code = code.drop_front(code.size());
+    SourceRange range;
+    range.start = result->range.start;
+    range.end = currentLocation();
+    error->addError(range, ErrorType::ParserStringError) << errorText;
+    result->kind = TokenKind::Error;
+  }
+
+  void parseIdentifierOrInvalid(TokenInfo *result) {
+    if (isalnum(code[0])) {
+      // Parse an identifier
+      size_t tokenLength = 1;
+
+      while (true) {
+        // A code completion location in/immediately after an identifier will
+        // cause the portion of the identifier before the code completion
+        // location to become a code completion token.
+        if (codeCompletionLocation == code.data() + tokenLength) {
+          codeCompletionLocation = nullptr;
+          result->kind = TokenKind::CodeCompletion;
+          result->text = code.substr(0, tokenLength);
+          code = code.drop_front(tokenLength);
+          return;
+        }
+        if (tokenLength == code.size() || !(isalnum(code[tokenLength])))
+          break;
+        ++tokenLength;
+      }
+      result->kind = TokenKind::Ident;
+      result->text = code.substr(0, tokenLength);
+      code = code.drop_front(tokenLength);
+    } else {
+      result->kind = TokenKind::InvalidChar;
+      result->text = code.substr(0, 1);
+      code = code.drop_front(1);
+    }
+  }
+
+  // Consume all leading whitespace from code, except newlines
+  void consumeWhitespace() {
+    code = code.drop_while(
+        [](char c) { return llvm::StringRef(" \t\v\f\r").contains(c); });
+  }
+
+  // Returns the current location in the source code
+  SourceLocation currentLocation() {
+    SourceLocation location;
+    location.line = line;
+    location.column = code.data() - startOfLine.data() + 1;
+    return location;
+  }
+
+  llvm::StringRef code;
+  llvm::StringRef startOfLine;
+  unsigned line = 1;
+  Diagnostics *error;
+  TokenInfo nextToken;
+  const char *codeCompletionLocation = nullptr;
+};
+
+Parser::Sema::~Sema() = default;
+
+std::vector<ArgKind> Parser::Sema::getAcceptedCompletionTypes(
+    llvm::ArrayRef<std::pair<MatcherCtor, unsigned>> context) {
+  return {};
+}
+
+std::vector<MatcherCompletion>
+Parser::Sema::getMatcherCompletions(llvm::ArrayRef<ArgKind> acceptedTypes) {
+  return {};
+}
+
+// Entry for the scope of a parser
+struct Parser::ScopedContextEntry {
+  Parser *parser;
+
+  ScopedContextEntry(Parser *parser, MatcherCtor c) : parser(parser) {
+    parser->contextStack.emplace_back(c, 0u);
+  }
+
+  ~ScopedContextEntry() { parser->contextStack.pop_back(); }
+
+  void nextArg() { ++parser->contextStack.back().second; }
+};
+
+// Parse and validate expressions starting with an identifier.
+// This function can parse named values and matchers. In case of failure, it
+// will try to determine the user's intent to give an appropriate error message.
+bool Parser::parseIdentifierPrefixImpl(VariantValue *value) {
+  const TokenInfo nameToken = tokenizer->consumeNextToken();
+
+  if (tokenizer->nextTokenKind() != TokenKind::OpenParen) {
+    // Parse as a named value.
+    auto namedValue =
+        namedValues ? namedValues->lookup(nameToken.text) : VariantValue();
+
+    if (!namedValue.isMatcher()) {
+      error->addError(tokenizer->peekNextToken().range,
+                      ErrorType::ParserNotAMatcher);
+      return false;
+    }
+
+    if (tokenizer->nextTokenKind() == TokenKind::NewLine) {
+      error->addError(tokenizer->peekNextToken().range,
+                      ErrorType::ParserNoOpenParen)
+          << "NewLine";
+      return false;
+    }
+
+    // If the syntax is correct and the name is not a matcher either, report
+    // an unknown named value.
+    if ((tokenizer->nextTokenKind() == TokenKind::Comma ||
+         tokenizer->nextTokenKind() == TokenKind::CloseParen ||
+         tokenizer->nextTokenKind() == TokenKind::NewLine ||
+         tokenizer->nextTokenKind() == TokenKind::Eof) &&
+        !sema->lookupMatcherCtor(nameToken.text)) {
+      error->addError(nameToken.range, ErrorType::RegistryValueNotFound)
+          << nameToken.text;
+      return false;
+    }
+    // Otherwise, fallback to the matcher parser.
+  }
+
+  tokenizer->skipNewlines();
+
+  assert(nameToken.kind == TokenKind::Ident);
+  TokenInfo openToken = tokenizer->consumeNextToken();
+  if (openToken.kind != TokenKind::OpenParen) {
+    error->addError(openToken.range, ErrorType::ParserNoOpenParen)
+        << openToken.text;
+    return false;
+  }
+
+  std::optional<MatcherCtor> ctor = sema->lookupMatcherCtor(nameToken.text);
+
+  // Parse as a matcher expression.
+  return parseMatcherExpressionImpl(nameToken, openToken, ctor, value);
+}
+
+// Parse the arguments of a matcher
+bool Parser::parseMatcherArgs(std::vector<ParserValue> &args, MatcherCtor ctor,
+                              const TokenInfo &nameToken, TokenInfo &endToken) {
+  ScopedContextEntry sce(this, ctor);
+
+  while (tokenizer->nextTokenKind() != TokenKind::Eof) {
+    if (tokenizer->nextTokenKind() == TokenKind::CloseParen) {
+      // end of args.
+      endToken = tokenizer->consumeNextToken();
+      break;
+    }
+
+    if (!args.empty()) {
+      // We must find a , token to continue.
+      TokenInfo commaToken = tokenizer->consumeNextToken();
+      if (commaToken.kind != TokenKind::Comma) {
+        error->addError(commaToken.range, ErrorType::ParserNoComma)
+            << commaToken.text;
+        return false;
+      }
+    }
+
+    ParserValue argValue;
+    tokenizer->skipNewlines();
+
+    argValue.text = tokenizer->peekNextToken().text;
+    argValue.range = tokenizer->peekNextToken().range;
+    if (!parseExpressionImpl(&argValue.value)) {
+      return false;
+    }
+
+    tokenizer->skipNewlines();
+    args.push_back(argValue);
+    sce.nextArg();
+  }
+
+  return true;
+}
+
+// Parse and validate a matcher expression.
+bool Parser::parseMatcherExpressionImpl(const TokenInfo &nameToken,
+                                        const TokenInfo &openToken,
+                                        std::optional<MatcherCtor> ctor,
+                                        VariantValue *value) {
+  if (!ctor) {
+    error->addError(nameToken.range, ErrorType::RegistryMatcherNotFound)
+        << nameToken.text;
+    // Do not return here. We need to continue to give completion suggestions.
+  }
+
+  std::vector<ParserValue> args;
+  TokenInfo endToken;
+
+  tokenizer->skipNewlines();
+
+  if (!parseMatcherArgs(args, ctor.value_or(nullptr), nameToken, endToken)) {
+    return false;
+  }
+
+  // Check for the missing closing parenthesis
+  if (endToken.kind != TokenKind::CloseParen) {
+    error->addError(openToken.range, ErrorType::ParserNoCloseParen)
+        << nameToken.text;
+    return false;
+  }
+
+  if (!ctor)
+    return false;
+  // Merge the start and end infos.
+  SourceRange matcherRange = nameToken.range;
+  matcherRange.end = endToken.range.end;
+  VariantMatcher result =
+      sema->actOnMatcherExpression(*ctor, matcherRange, args, error);
+  if (result.isNull())
+    return false;
+  *value = result;
+  return true;
+}
+
+// If the prefix of this completion matches the completion token, add it to
+// completions minus the prefix.
+void Parser::addCompletion(const TokenInfo &compToken,
+                           const MatcherCompletion &completion) {
+  if (llvm::StringRef(completion.typedText).startswith(compToken.text)) {
+    completions.emplace_back(completion.typedText.substr(compToken.text.size()),
+                             completion.matcherDecl);
+  }
+}
+
+std::vector<MatcherCompletion>
+Parser::getNamedValueCompletions(llvm::ArrayRef<ArgKind> acceptedTypes) {
+  if (!namedValues)
+    return {};
+
+  std::vector<MatcherCompletion> result;
+  for (const auto &entry : *namedValues) {
+    std::string decl =
+        (entry.getValue().getTypeAsString() + " " + entry.getKey()).str();
+    result.emplace_back(entry.getKey(), decl);
+  }
+  return result;
+}
+
+void Parser::addExpressionCompletions() {
+  const TokenInfo compToken = tokenizer->consumeNextTokenIgnoreNewlines();
+  assert(compToken.kind == TokenKind::CodeCompletion);
+
+  // We cannot complete code if there is an invalid element on the context
+  // stack.
+  for (const auto &entry : contextStack) {
+    if (!entry.first)
+      return;
+  }
+
+  auto acceptedTypes = sema->getAcceptedCompletionTypes(contextStack);
+  for (const auto &completion : sema->getMatcherCompletions(acceptedTypes)) {
+    addCompletion(compToken, completion);
+  }
+
+  for (const auto &completion : getNamedValueCompletions(acceptedTypes)) {
+    addCompletion(compToken, completion);
+  }
+}
+
+// Parse an <Expresssion>
+bool Parser::parseExpressionImpl(VariantValue *value) {
+  switch (tokenizer->nextTokenKind()) {
+  case TokenKind::Literal:
+    *value = tokenizer->consumeNextToken().value;
+    return true;
+  case TokenKind::Ident:
+    return parseIdentifierPrefixImpl(value);
+  case TokenKind::CodeCompletion:
+    addExpressionCompletions();
+    return false;
+  case TokenKind::Eof:
+    error->addError(tokenizer->consumeNextToken().range,
+                    ErrorType::ParserNoCode);
+    return false;
+
+  case TokenKind::Error:
+    // This error was already reported by the tokenizer.
+    return false;
+  case TokenKind::NewLine:
+  case TokenKind::OpenParen:
+  case TokenKind::CloseParen:
+  case TokenKind::Comma:
+  case TokenKind::Period:
+  case TokenKind::InvalidChar:
+    const TokenInfo token = tokenizer->consumeNextToken();
+    error->addError(token.range, ErrorType::ParserInvalidToken)
+        << (token.kind == TokenKind::NewLine ? "NewLine" : token.text);
+    return false;
+  }
+
+  llvm_unreachable("Unknown token kind.");
+}
+
+Parser::Parser(CodeTokenizer *tokenizer, const Registry &matcherRegistry,
+               const NamedValueMap *namedValues, Diagnostics *error)
+    : tokenizer(tokenizer),
+      sema(std::make_unique<RegistrySema>(matcherRegistry)),
+      namedValues(namedValues), error(error) {}
+
+Parser::RegistrySema::~RegistrySema() = default;
+
+std::optional<MatcherCtor>
+Parser::RegistrySema::lookupMatcherCtor(llvm::StringRef matcherName) {
+  return RegistryManager::lookupMatcherCtor(matcherName, matcherRegistry);
+}
+
+VariantMatcher Parser::RegistrySema::actOnMatcherExpression(
+    MatcherCtor ctor, SourceRange nameRange, llvm::ArrayRef<ParserValue> args,
+    Diagnostics *error) {
+  return RegistryManager::constructMatcher(ctor, nameRange, args, error);
+}
+
+std::vector<ArgKind> Parser::RegistrySema::getAcceptedCompletionTypes(
+    llvm::ArrayRef<std::pair<MatcherCtor, unsigned>> context) {
+  return RegistryManager::getAcceptedCompletionTypes(context);
+}
+
+std::vector<MatcherCompletion> Parser::RegistrySema::getMatcherCompletions(
+    llvm::ArrayRef<ArgKind> acceptedTypes) {
+  return RegistryManager::getMatcherCompletions(acceptedTypes, matcherRegistry);
+}
+
+bool Parser::parseExpression(llvm::StringRef &code,
+                             const Registry &matcherRegistry,
+                             const NamedValueMap *namedValues,
+                             VariantValue *value, Diagnostics *error) {
+  CodeTokenizer tokenizer(code, error);
+  Parser parser(&tokenizer, matcherRegistry, namedValues, error);
+  if (!parser.parseExpressionImpl(value))
+    return false;
+  auto nextToken = tokenizer.peekNextToken();
+  if (nextToken.kind != TokenKind::Eof &&
+      nextToken.kind != TokenKind::NewLine) {
+    error->addError(tokenizer.peekNextToken().range,
+                    ErrorType::ParserTrailingCode);
+    return false;
+  }
+  return true;
+}
+
+std::vector<MatcherCompletion>
+Parser::completeExpression(llvm::StringRef &code, unsigned completionOffset,
+                           const Registry &matcherRegistry,
+                           const NamedValueMap *namedValues) {
+  Diagnostics error;
+  CodeTokenizer tokenizer(code, &error, completionOffset);
+  Parser parser(&tokenizer, matcherRegistry, namedValues, &error);
+  VariantValue dummy;
+  parser.parseExpressionImpl(&dummy);
+
+  return parser.completions;
+}
+
+std::optional<DynMatcher> Parser::parseMatcherExpression(
+    llvm::StringRef &code, const Registry &matcherRegistry,
+    const NamedValueMap *namedValues, Diagnostics *error) {
+  VariantValue value;
+  if (!parseExpression(code, matcherRegistry, namedValues, &value, error))
+    return std::nullopt;
+  if (!value.isMatcher()) {
+    error->addError(SourceRange(), ErrorType::ParserNotAMatcher);
+    return std::nullopt;
+  }
+  std::optional<DynMatcher> result = value.getMatcher().getDynMatcher();
+  if (!result) {
+    error->addError(SourceRange(), ErrorType::ParserOverloadedType)
+        << value.getTypeAsString();
+  }
+  return result;
+}
+
+} // namespace mlir::query::matcher::internal
diff --git a/mlir/lib/Query/Matcher/Parser.h b/mlir/lib/Query/Matcher/Parser.h
new file mode 100644
index 0000000000..f049af34e9
--- /dev/null
+++ b/mlir/lib/Query/Matcher/Parser.h
@@ -0,0 +1,188 @@
+//===--- Parser.h - Matcher expression parser -------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Simple matcher expression parser.
+//
+// This file contains the Parser class, which is responsible for parsing
+// expressions in a specific format: matcherName(Arg0, Arg1, ..., ArgN). The
+// parser can also interpret simple types, like strings.
+//
+// The actual processing of the matchers is handled by a Sema object that is
+// provided to the parser.
+//
+// The grammar for the supported expressions is as follows:
+// <Expression>        := <StringLiteral> | <MatcherExpression>
+// <StringLiteral>     := "quoted string"
+// <MatcherExpression> := <MatcherName>(<ArgumentList>)
+// <MatcherName>       := [a-zA-Z]+
+// <ArgumentList>      := <Expression> | <Expression>,<ArgumentList>
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_TOOLS_MLIRQUERY_MATCHER_PARSER_H
+#define MLIR_TOOLS_MLIRQUERY_MATCHER_PARSER_H
+
+#include "Diagnostics.h"
+#include "RegistryManager.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
+#include <memory>
+#include <vector>
+
+namespace mlir::query::matcher::internal {
+
+// Matcher expression parser.
+class Parser {
+public:
+  // Different possible tokens.
+  enum class TokenKind {
+    Eof,
+    NewLine,
+    OpenParen,
+    CloseParen,
+    Comma,
+    Period,
+    Literal,
+    Ident,
+    InvalidChar,
+    CodeCompletion,
+    Error
+  };
+
+  // Interface to connect the parser with the registry and more. The parser uses
+  // the Sema instance passed into parseMatcherExpression() to handle all
+  // matcher tokens.
+  class Sema {
+  public:
+    virtual ~Sema();
+
+    // Process a matcher expression. The caller takes ownership of the Matcher
+    // object returned.
+    virtual VariantMatcher
+    actOnMatcherExpression(MatcherCtor ctor, SourceRange nameRange,
+                           llvm::ArrayRef<ParserValue> args,
+                           Diagnostics *error) = 0;
+
+    // Look up a matcher by name in the matcher name found by the parser.
+    virtual std::optional<MatcherCtor>
+    lookupMatcherCtor(llvm::StringRef matcherName) = 0;
+
+    // Compute the list of completion types for Context.
+    virtual std::vector<ArgKind> getAcceptedCompletionTypes(
+        llvm::ArrayRef<std::pair<MatcherCtor, unsigned>> Context);
+
+    // Compute the list of completions that match any of acceptedTypes.
+    virtual std::vector<MatcherCompletion>
+    getMatcherCompletions(llvm::ArrayRef<ArgKind> acceptedTypes);
+  };
+
+  // An implementation of the Sema interface that uses the matcher registry to
+  // process tokens.
+  class RegistrySema : public Parser::Sema {
+  public:
+    RegistrySema(const Registry &matcherRegistry)
+        : matcherRegistry(matcherRegistry) {}
+    ~RegistrySema() override;
+
+    std::optional<MatcherCtor>
+    lookupMatcherCtor(llvm::StringRef matcherName) override;
+
+    VariantMatcher actOnMatcherExpression(MatcherCtor ctor,
+                                          SourceRange nameRange,
+                                          llvm::ArrayRef<ParserValue> args,
+                                          Diagnostics *error) override;
+
+    std::vector<ArgKind> getAcceptedCompletionTypes(
+        llvm::ArrayRef<std::pair<MatcherCtor, unsigned>> context) override;
+
+    std::vector<MatcherCompletion>
+    getMatcherCompletions(llvm::ArrayRef<ArgKind> acceptedTypes) override;
+
+  private:
+    const Registry &matcherRegistry;
+  };
+
+  using NamedValueMap = llvm::StringMap<VariantValue>;
+
+  // Methods to parse a matcher expression and return a DynMatcher object,
+  // transferring ownership to the caller.
+  static std::optional<DynMatcher>
+  parseMatcherExpression(llvm::StringRef &matcherCode,
+                         const Registry &matcherRegistry,
+                         const NamedValueMap *namedValues, Diagnostics *error);
+  static std::optional<DynMatcher>
+  parseMatcherExpression(llvm::StringRef &matcherCode,
+                         const Registry &matcherRegistry, Diagnostics *error) {
+    return parseMatcherExpression(matcherCode, matcherRegistry, nullptr, error);
+  }
+
+  // Methods to parse any expression supported by this parser.
+  static bool parseExpression(llvm::StringRef &code,
+                              const Registry &matcherRegistry,
+                              const NamedValueMap *namedValues,
+                              VariantValue *value, Diagnostics *error);
+
+  static bool parseExpression(llvm::StringRef &code,
+                              const Registry &matcherRegistry,
+                              VariantValue *value, Diagnostics *error) {
+    return parseExpression(code, matcherRegistry, nullptr, value, error);
+  }
+
+  // Methods to complete an expression at a given offset.
+  static std::vector<MatcherCompletion>
+  completeExpression(llvm::StringRef &code, unsigned completionOffset,
+                     const Registry &matcherRegistry,
+                     const NamedValueMap *namedValues);
+  static std::vector<MatcherCompletion>
+  completeExpression(llvm::StringRef &code, unsigned completionOffset,
+                     const Registry &matcherRegistry) {
+    return completeExpression(code, completionOffset, matcherRegistry, nullptr);
+  }
+
+private:
+  class CodeTokenizer;
+  struct ScopedContextEntry;
+  struct TokenInfo;
+
+  Parser(CodeTokenizer *tokenizer, const Registry &matcherRegistry,
+         const NamedValueMap *namedValues, Diagnostics *error);
+
+  bool parseExpressionImpl(VariantValue *value);
+
+  bool parseMatcherArgs(std::vector<ParserValue> &args, MatcherCtor ctor,
+                        const TokenInfo &nameToken, TokenInfo &endToken);
+
+  bool parseMatcherExpressionImpl(const TokenInfo &nameToken,
+                                  const TokenInfo &openToken,
+                                  std::optional<MatcherCtor> ctor,
+                                  VariantValue *value);
+
+  bool parseIdentifierPrefixImpl(VariantValue *value);
+
+  void addCompletion(const TokenInfo &compToken,
+                     const MatcherCompletion &completion);
+  void addExpressionCompletions();
+
+  std::vector<MatcherCompletion>
+  getNamedValueCompletions(llvm::ArrayRef<ArgKind> acceptedTypes);
+
+  CodeTokenizer *const tokenizer;
+  std::unique_ptr<RegistrySema> sema;
+  const NamedValueMap *const namedValues;
+  Diagnostics *const error;
+
+  using ContextStackTy = std::vector<std::pair<MatcherCtor, unsigned>>;
+
+  ContextStackTy contextStack;
+  std::vector<MatcherCompletion> completions;
+};
+
+} // namespace mlir::query::matcher::internal
+
+#endif // MLIR_TOOLS_MLIRQUERY_MATCHER_PARSER_H
diff --git a/mlir/lib/Query/Matcher/RegistryManager.cpp b/mlir/lib/Query/Matcher/RegistryManager.cpp
new file mode 100644
index 0000000000..01856aa8ff
--- /dev/null
+++ b/mlir/lib/Query/Matcher/RegistryManager.cpp
@@ -0,0 +1,139 @@
+//===- RegistryManager.cpp - Matcher registry -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Registry map populated at static initialization time.
+//
+//===----------------------------------------------------------------------===//
+
+#include "RegistryManager.h"
+#include "mlir/Query/Matcher/Registry.h"
+
+#include <set>
+#include <utility>
+
+namespace mlir::query::matcher {
+namespace {
+
+// This is needed because these matchers are defined as overloaded functions.
+using IsConstantOp = detail::constant_op_matcher();
+using HasOpAttrName = detail::AttrOpMatcher(llvm::StringRef);
+using HasOpName = detail::NameOpMatcher(llvm::StringRef);
+
+// Enum to string for autocomplete.
+static std::string asArgString(ArgKind kind) {
+  switch (kind) {
+  case ArgKind::Matcher:
+    return "Matcher";
+  case ArgKind::String:
+    return "String";
+  }
+  llvm_unreachable("Unhandled ArgKind");
+}
+
+} // namespace
+
+void Registry::registerMatcherDescriptor(
+    llvm::StringRef matcherName,
+    std::unique_ptr<internal::MatcherDescriptor> callback) {
+  assert(!constructorMap.contains(matcherName));
+  constructorMap[matcherName] = std::move(callback);
+}
+
+std::optional<MatcherCtor>
+RegistryManager::lookupMatcherCtor(llvm::StringRef matcherName,
+                                   const Registry &matcherRegistry) {
+  auto it = matcherRegistry.constructors().find(matcherName);
+  return it == matcherRegistry.constructors().end()
+             ? std::optional<MatcherCtor>()
+             : it->second.get();
+}
+
+std::vector<ArgKind> RegistryManager::getAcceptedCompletionTypes(
+    llvm::ArrayRef<std::pair<MatcherCtor, unsigned>> context) {
+  // Starting with the above seed of acceptable top-level matcher types, compute
+  // the acceptable type set for the argument indicated by each context element.
+  std::set<ArgKind> typeSet;
+  typeSet.insert(ArgKind::Matcher);
+
+  for (const auto &ctxEntry : context) {
+    MatcherCtor ctor = ctxEntry.first;
+    unsigned argNumber = ctxEntry.second;
+    std::vector<ArgKind> nextTypeSet;
+
+    if (argNumber < ctor->getNumArgs())
+      ctor->getArgKinds(argNumber, nextTypeSet);
+
+    typeSet.insert(nextTypeSet.begin(), nextTypeSet.end());
+  }
+
+  return std::vector<ArgKind>(typeSet.begin(), typeSet.end());
+}
+
+std::vector<MatcherCompletion>
+RegistryManager::getMatcherCompletions(llvm::ArrayRef<ArgKind> acceptedTypes,
+                                       const Registry &matcherRegistry) {
+  std::vector<MatcherCompletion> completions;
+
+  // Search the registry for acceptable matchers.
+  for (const auto &m : matcherRegistry.constructors()) {
+    const internal::MatcherDescriptor &matcher = *m.getValue();
+    llvm::StringRef name = m.getKey();
+
+    unsigned numArgs = matcher.getNumArgs();
+    std::vector<std::vector<ArgKind>> argKinds(numArgs);
+
+    for (const ArgKind &kind : acceptedTypes) {
+      if (kind != ArgKind::Matcher)
+        continue;
+
+      for (unsigned arg = 0; arg != numArgs; ++arg)
+        matcher.getArgKinds(arg, argKinds[arg]);
+    }
+
+    std::string decl;
+    llvm::raw_string_ostream os(decl);
+
+    std::string typedText = std::string(name);
+    os << "Matcher: " << name << "(";
+
+    for (const std::vector<ArgKind> &arg : argKinds) {
+      if (&arg != &argKinds[0])
+        os << ", ";
+
+      bool firstArgKind = true;
+      // Two steps. First all non-matchers, then matchers only.
+      for (const ArgKind &argKind : arg) {
+        if (!firstArgKind)
+          os << "|";
+
+        firstArgKind = false;
+        os << asArgString(argKind);
+      }
+    }
+
+    os << ")";
+    typedText += "(";
+
+    if (argKinds.empty())
+      typedText += ")";
+    else if (argKinds[0][0] == ArgKind::String)
+      typedText += "\"";
+
+    completions.emplace_back(typedText, os.str());
+  }
+
+  return completions;
+}
+
+VariantMatcher RegistryManager::constructMatcher(
+    MatcherCtor ctor, internal::SourceRange nameRange,
+    llvm::ArrayRef<ParserValue> args, internal::Diagnostics *error) {
+  return ctor->create(nameRange, args, error);
+}
+
+} // namespace mlir::query::matcher
diff --git a/mlir/lib/Query/Matcher/RegistryManager.h b/mlir/lib/Query/Matcher/RegistryManager.h
new file mode 100644
index 0000000000..5f28672612
--- /dev/null
+++ b/mlir/lib/Query/Matcher/RegistryManager.h
@@ -0,0 +1,70 @@
+//===--- RegistryManager.h - Matcher registry -------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// RegistryManager to manage registry of all known matchers.
+//
+// The registry provides a generic interface to construct any matcher by name.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_TOOLS_MLIRQUERY_MATCHER_REGISTRYMANAGER_H
+#define MLIR_TOOLS_MLIRQUERY_MATCHER_REGISTRYMANAGER_H
+
+#include "Diagnostics.h"
+#include "mlir/Query/Matcher/Marshallers.h"
+#include "mlir/Query/Matcher/Registry.h"
+#include "mlir/Query/Matcher/VariantValue.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
+#include <string>
+
+namespace mlir::query::matcher {
+
+using MatcherCtor = const internal::MatcherDescriptor *;
+
+struct MatcherCompletion {
+  MatcherCompletion() = default;
+  MatcherCompletion(llvm::StringRef typedText, llvm::StringRef matcherDecl)
+      : typedText(typedText.str()), matcherDecl(matcherDecl.str()) {}
+
+  bool operator==(const MatcherCompletion &other) const {
+    return typedText == other.typedText && matcherDecl == other.matcherDecl;
+  }
+
+  // The text to type to select this matcher.
+  std::string typedText;
+
+  // The "declaration" of the matcher, with type information.
+  std::string matcherDecl;
+};
+
+class RegistryManager {
+public:
+  RegistryManager() = delete;
+
+  static std::optional<MatcherCtor>
+  lookupMatcherCtor(llvm::StringRef matcherName,
+                    const Registry &matcherRegistry);
+
+  static std::vector<ArgKind> getAcceptedCompletionTypes(
+      llvm::ArrayRef<std::pair<MatcherCtor, unsigned>> context);
+
+  static std::vector<MatcherCompletion>
+  getMatcherCompletions(ArrayRef<ArgKind> acceptedTypes,
+                        const Registry &matcherRegistry);
+
+  static VariantMatcher constructMatcher(MatcherCtor ctor,
+                                         internal::SourceRange nameRange,
+                                         ArrayRef<ParserValue> args,
+                                         internal::Diagnostics *error);
+};
+
+} // namespace mlir::query::matcher
+
+#endif // MLIR_TOOLS_MLIRQUERY_MATCHER_REGISTRYMANAGER_H
diff --git a/mlir/lib/Query/Matcher/VariantValue.cpp b/mlir/lib/Query/Matcher/VariantValue.cpp
new file mode 100644
index 0000000000..65bd4bd77b
--- /dev/null
+++ b/mlir/lib/Query/Matcher/VariantValue.cpp
@@ -0,0 +1,132 @@
+//===--- Variantvalue.cpp -------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Query/Matcher/VariantValue.h"
+
+namespace mlir::query::matcher {
+
+VariantMatcher::Payload::~Payload() = default;
+
+class VariantMatcher::SinglePayload : public VariantMatcher::Payload {
+public:
+  explicit SinglePayload(DynMatcher matcher) : matcher(std::move(matcher)) {}
+
+  std::optional<DynMatcher> getDynMatcher() const override { return matcher; }
+
+  std::string getTypeAsString() const override { return "Matcher"; }
+
+private:
+  DynMatcher matcher;
+};
+
+VariantMatcher::VariantMatcher() = default;
+
+VariantMatcher VariantMatcher::SingleMatcher(DynMatcher matcher) {
+  return VariantMatcher(std::make_shared<SinglePayload>(std::move(matcher)));
+}
+
+std::optional<DynMatcher> VariantMatcher::getDynMatcher() const {
+  return value ? value->getDynMatcher() : std::nullopt;
+}
+
+void VariantMatcher::reset() { value.reset(); }
+
+std::string VariantMatcher::getTypeAsString() const { return "<Nothing>"; }
+
+VariantValue::VariantValue(const VariantValue &other)
+    : type(ValueType::Nothing) {
+  *this = other;
+}
+
+VariantValue::VariantValue(const llvm::StringRef string)
+    : type(ValueType::String) {
+  value.String = new llvm::StringRef(string);
+}
+
+VariantValue::VariantValue(const VariantMatcher &matcher)
+    : type(ValueType::Matcher) {
+  value.Matcher = new VariantMatcher(matcher);
+}
+
+VariantValue::~VariantValue() { reset(); }
+
+VariantValue &VariantValue::operator=(const VariantValue &other) {
+  if (this == &other)
+    return *this;
+  reset();
+  switch (other.type) {
+  case ValueType::String:
+    setString(other.getString());
+    break;
+  case ValueType::Matcher:
+    setMatcher(other.getMatcher());
+    break;
+  case ValueType::Nothing:
+    type = ValueType::Nothing;
+    break;
+  }
+  return *this;
+}
+
+void VariantValue::reset() {
+  switch (type) {
+  case ValueType::String:
+    delete value.String;
+    break;
+  case ValueType::Matcher:
+    delete value.Matcher;
+    break;
+  // Cases that do nothing.
+  case ValueType::Nothing:
+    break;
+  }
+  type = ValueType::Nothing;
+}
+
+bool VariantValue::isString() const { return type == ValueType::String; }
+
+const llvm::StringRef &VariantValue::getString() const {
+  assert(isString());
+  return *value.String;
+}
+
+void VariantValue::setString(const llvm::StringRef &newValue) {
+  reset();
+  type = ValueType::String;
+  value.String = new llvm::StringRef(newValue);
+}
+
+bool VariantValue::isMatcher() const { return type == ValueType::Matcher; }
+
+const VariantMatcher &VariantValue::getMatcher() const {
+  assert(isMatcher());
+  return *value.Matcher;
+}
+
+void VariantValue::setMatcher(const VariantMatcher &newValue) {
+  reset();
+  type = ValueType::Matcher;
+  value.Matcher = new VariantMatcher(newValue);
+}
+
+std::string VariantValue::getTypeAsString() const {
+  switch (type) {
+  case ValueType::String:
+    return "String";
+  case ValueType::Matcher:
+    return "Matcher";
+  case ValueType::Nothing:
+    return "Nothing";
+  }
+  llvm_unreachable("Invalid Type");
+}
+
+} // namespace mlir::query::matcher
diff --git a/mlir/lib/Query/Query.cpp b/mlir/lib/Query/Query.cpp
new file mode 100644
index 0000000000..5c42e5a5f0
--- /dev/null
+++ b/mlir/lib/Query/Query.cpp
@@ -0,0 +1,82 @@
+//===---- Query.cpp - -----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Query/Query.h"
+#include "QueryParser.h"
+#include "mlir/Query/Matcher/MatchFinder.h"
+#include "mlir/Query/QuerySession.h"
+#include "mlir/Support/LogicalResult.h"
+#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/raw_ostream.h"
+
+namespace mlir::query {
+
+QueryRef parse(llvm::StringRef line, const QuerySession &qs) {
+  return QueryParser::parse(line, qs);
+}
+
+std::vector<llvm::LineEditor::Completion>
+complete(llvm::StringRef line, size_t pos, const QuerySession &qs) {
+  return QueryParser::complete(line, pos, qs);
+}
+
+static void printMatch(llvm::raw_ostream &os, QuerySession &qs, Operation *op,
+                       const std::string &binding) {
+  auto fileLoc = op->getLoc()->findInstanceOf<FileLineColLoc>();
+  auto smloc = qs.getSourceManager().FindLocForLineAndColumn(
+      qs.getBufferId(), fileLoc.getLine(), fileLoc.getColumn());
+  qs.getSourceManager().PrintMessage(os, smloc, llvm::SourceMgr::DK_Note,
+                                     "\"" + binding + "\" binds here");
+}
+
+Query::~Query() = default;
+
+mlir::LogicalResult InvalidQuery::run(llvm::raw_ostream &os,
+                                      QuerySession &qs) const {
+  os << errStr << "\n";
+  return mlir::failure();
+}
+
+mlir::LogicalResult NoOpQuery::run(llvm::raw_ostream &os,
+                                   QuerySession &qs) const {
+  return mlir::success();
+}
+
+mlir::LogicalResult HelpQuery::run(llvm::raw_ostream &os,
+                                   QuerySession &qs) const {
+  os << "Available commands:\n\n"
+        "  match MATCHER, m MATCHER      "
+        "Match the mlir against the given matcher.\n"
+        "  quit                              "
+        "Terminates the query session.\n\n";
+  return mlir::success();
+}
+
+mlir::LogicalResult QuitQuery::run(llvm::raw_ostream &os,
+                                   QuerySession &qs) const {
+  qs.terminate = true;
+  return mlir::success();
+}
+
+mlir::LogicalResult MatchQuery::run(llvm::raw_ostream &os,
+                                    QuerySession &qs) const {
+  int matchCount = 0;
+  std::vector<Operation *> matches =
+      matcher::MatchFinder().getMatches(qs.getRootOp(), matcher);
+  os << "\n";
+  for (Operation *op : matches) {
+    os << "Match #" << ++matchCount << ":\n\n";
+    // Placeholder "root" binding for the initial draft.
+    printMatch(os, qs, op, "root");
+  }
+  os << matchCount << (matchCount == 1 ? " match.\n\n" : " matches.\n\n");
+
+  return mlir::success();
+}
+
+} // namespace mlir::query
diff --git a/mlir/lib/Query/QueryParser.cpp b/mlir/lib/Query/QueryParser.cpp
new file mode 100644
index 0000000000..f43a28569f
--- /dev/null
+++ b/mlir/lib/Query/QueryParser.cpp
@@ -0,0 +1,217 @@
+//===---- QueryParser.cpp - mlir-query command parser ---------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "QueryParser.h"
+#include "llvm/ADT/StringSwitch.h"
+
+namespace mlir::query {
+
+// Lex any amount of whitespace followed by a "word" (any sequence of
+// non-whitespace characters) from the start of region [begin,end).  If no word
+// is found before end, return StringRef(). begin is adjusted to exclude the
+// lexed region.
+llvm::StringRef QueryParser::lexWord() {
+  line = line.drop_while([](char c) {
+    // Don't trim newlines.
+    return llvm::StringRef(" \t\v\f\r").contains(c);
+  });
+
+  if (line.empty())
+    // Even though the line is empty, it contains a pointer and
+    // a (zero) length. The pointer is used in the LexOrCompleteWord
+    // code completion.
+    return line;
+
+  llvm::StringRef word;
+  if (line.front() == '#') {
+    word = line.substr(0, 1);
+  } else {
+    word = line.take_until([](char c) {
+      // Don't trim newlines.
+      return llvm::StringRef(" \t\v\f\r").contains(c);
+    });
+  }
+
+  line = line.drop_front(word.size());
+  return word;
+}
+
+// This is the StringSwitch-alike used by LexOrCompleteWord below. See that
+// function for details.
+template <typename T>
+struct QueryParser::LexOrCompleteWord {
+  llvm::StringRef word;
+  llvm::StringSwitch<T> stringSwitch;
+
+  QueryParser *queryParser;
+  // Set to the completion point offset in word, or StringRef::npos if
+  // completion point not in word.
+  size_t wordCompletionPos;
+
+  // Lexes a word and stores it in word. Returns a LexOrCompleteword<T> object
+  // that can be used like a llvm::StringSwitch<T>, but adds cases as possible
+  // completions if the lexed word contains the completion point.
+  LexOrCompleteWord(QueryParser *queryParser, llvm::StringRef &outWord)
+      : word(queryParser->lexWord()), stringSwitch(word),
+        queryParser(queryParser), wordCompletionPos(llvm::StringRef::npos) {
+    outWord = word;
+    if (queryParser->completionPos &&
+        queryParser->completionPos <= word.data() + word.size()) {
+      if (queryParser->completionPos < word.data())
+        wordCompletionPos = 0;
+      else
+        wordCompletionPos = queryParser->completionPos - word.data();
+    }
+  }
+
+  LexOrCompleteWord &Case(llvm::StringLiteral caseStr, const T &value,
+                          bool isCompletion = true) {
+
+    if (wordCompletionPos == llvm::StringRef::npos)
+      stringSwitch.Case(caseStr, value);
+    else if (!caseStr.empty() && isCompletion &&
+             wordCompletionPos <= caseStr.size() &&
+             caseStr.substr(0, wordCompletionPos) ==
+                 word.substr(0, wordCompletionPos)) {
+
+      queryParser->completions.emplace_back(
+          (caseStr.substr(wordCompletionPos) + " ").str(),
+          std::string(caseStr));
+    }
+    return *this;
+  }
+
+  T Default(T value) { return stringSwitch.Default(value); }
+};
+
+QueryRef QueryParser::endQuery(QueryRef queryRef) {
+  llvm::StringRef extra = line;
+  llvm::StringRef extraTrimmed = extra.drop_while(
+      [](char c) { return llvm::StringRef(" \t\v\f\r").contains(c); });
+
+  if ((!extraTrimmed.empty() && extraTrimmed[0] == '\n') ||
+      (extraTrimmed.size() >= 2 && extraTrimmed[0] == '\r' &&
+       extraTrimmed[1] == '\n'))
+    queryRef->remainingContent = extra;
+  else {
+    llvm::StringRef trailingWord = lexWord();
+    if (!trailingWord.empty() && trailingWord.front() == '#') {
+      line = line.drop_until([](char c) { return c == '\n'; });
+      line = line.drop_while([](char c) { return c == '\n'; });
+      return endQuery(queryRef);
+    }
+    if (!trailingWord.empty()) {
+      return new InvalidQuery("unexpected extra input: '" + extra + "'");
+    }
+  }
+  return queryRef;
+}
+
+namespace {
+
+enum class ParsedQueryKind {
+  Invalid,
+  Comment,
+  NoOp,
+  Help,
+  Match,
+  Quit,
+};
+
+QueryRef
+makeInvalidQueryFromDiagnostics(const matcher::internal::Diagnostics &diag) {
+  std::string errStr;
+  llvm::raw_string_ostream os(errStr);
+  diag.print(os);
+  return new InvalidQuery(os.str());
+}
+} // namespace
+
+QueryRef QueryParser::completeMatcherExpression() {
+  std::vector<matcher::MatcherCompletion> comps =
+      matcher::internal::Parser::completeExpression(
+          line, completionPos - line.begin(), qs.getRegistryData(),
+          &qs.namedValues);
+  for (const auto &comp : comps) {
+    completions.emplace_back(comp.typedText, comp.matcherDecl);
+  }
+  return QueryRef();
+}
+
+QueryRef QueryParser::doParse() {
+
+  llvm::StringRef commandStr;
+  ParsedQueryKind qKind =
+      LexOrCompleteWord<ParsedQueryKind>(this, commandStr)
+          .Case("", ParsedQueryKind::NoOp)
+          .Case("#", ParsedQueryKind::Comment, /*isCompletion=*/false)
+          .Case("help", ParsedQueryKind::Help)
+          .Case("m", ParsedQueryKind::Match, /*isCompletion=*/false)
+          .Case("match", ParsedQueryKind::Match)
+          .Case("q", ParsedQueryKind::Quit, /*IsCompletion=*/false)
+          .Case("quit", ParsedQueryKind::Quit)
+          .Default(ParsedQueryKind::Invalid);
+
+  switch (qKind) {
+  case ParsedQueryKind::Comment:
+  case ParsedQueryKind::NoOp:
+    line = line.drop_until([](char c) { return c == '\n'; });
+    line = line.drop_while([](char c) { return c == '\n'; });
+    if (line.empty())
+      return new NoOpQuery;
+    return doParse();
+
+  case ParsedQueryKind::Help:
+    return endQuery(new HelpQuery);
+
+  case ParsedQueryKind::Quit:
+    return endQuery(new QuitQuery);
+
+  case ParsedQueryKind::Match: {
+    if (completionPos) {
+      return completeMatcherExpression();
+    }
+
+    matcher::internal::Diagnostics diag;
+    auto matcherSource = line.ltrim();
+    auto origMatcherSource = matcherSource;
+    std::optional<matcher::DynMatcher> matcher =
+        matcher::internal::Parser::parseMatcherExpression(
+            matcherSource, qs.getRegistryData(), &qs.namedValues, &diag);
+    if (!matcher) {
+      return makeInvalidQueryFromDiagnostics(diag);
+    }
+    auto actualSource = origMatcherSource.slice(0, origMatcherSource.size() -
+                                                       matcherSource.size());
+    QueryRef query = new MatchQuery(actualSource, *matcher);
+    query->remainingContent = matcherSource;
+    return query;
+  }
+
+  case ParsedQueryKind::Invalid:
+    return new InvalidQuery("unknown command: " + commandStr);
+  }
+
+  llvm_unreachable("Invalid query kind");
+}
+
+QueryRef QueryParser::parse(llvm::StringRef line, const QuerySession &qs) {
+  return QueryParser(line, qs).doParse();
+}
+
+std::vector<llvm::LineEditor::Completion>
+QueryParser::complete(llvm::StringRef line, size_t pos,
+                      const QuerySession &qs) {
+  QueryParser queryParser(line, qs);
+  queryParser.completionPos = line.data() + pos;
+
+  queryParser.doParse();
+  return queryParser.completions;
+}
+
+} // namespace mlir::query
diff --git a/mlir/lib/Query/QueryParser.h b/mlir/lib/Query/QueryParser.h
new file mode 100644
index 0000000000..e9c30eccec
--- /dev/null
+++ b/mlir/lib/Query/QueryParser.h
@@ -0,0 +1,59 @@
+//===--- QueryParser.h - ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_TOOLS_MLIRQUERY_QUERYPARSER_H
+#define MLIR_TOOLS_MLIRQUERY_QUERYPARSER_H
+
+#include "Matcher/Parser.h"
+#include "mlir/Query/Query.h"
+#include "mlir/Query/QuerySession.h"
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/LineEditor/LineEditor.h"
+
+namespace mlir::query {
+
+class QuerySession;
+
+class QueryParser {
+public:
+  // Parse line as a query and return a QueryRef representing the query, which
+  // may be an InvalidQuery.
+  static QueryRef parse(llvm::StringRef line, const QuerySession &qs);
+
+  static std::vector<llvm::LineEditor::Completion>
+  complete(llvm::StringRef line, size_t pos, const QuerySession &qs);
+
+private:
+  QueryParser(llvm::StringRef line, const QuerySession &qs)
+      : line(line), completionPos(nullptr), qs(qs) {}
+
+  llvm::StringRef lexWord();
+
+  template <typename T>
+  struct LexOrCompleteWord;
+
+  QueryRef completeMatcherExpression();
+
+  QueryRef endQuery(QueryRef queryRef);
+
+  // Parse [begin, end) and returns a reference to the parsed query object,
+  // which may be an InvalidQuery if a parse error occurs.
+  QueryRef doParse();
+
+  llvm::StringRef line;
+
+  const char *completionPos;
+  std::vector<llvm::LineEditor::Completion> completions;
+
+  const QuerySession &qs;
+};
+
+} // namespace mlir::query
+
+#endif // MLIR_TOOLS_MLIRQUERY_QUERYPARSER_H
diff --git a/mlir/lib/TableGen/Constraint.cpp b/mlir/lib/TableGen/Constraint.cpp
index d8f240a1af..4ccbd0a685 100644
--- a/mlir/lib/TableGen/Constraint.cpp
+++ b/mlir/lib/TableGen/Constraint.cpp
@@ -30,8 +30,9 @@ Constraint::Constraint(const llvm::Record *record)
     kind = CK_Region;
   } else if (def->isSubClassOf("SuccessorConstraint")) {
     kind = CK_Successor;
-  } else {
-    assert(def->isSubClassOf("Constraint"));
+  } else if(!def->isSubClassOf("Constraint")) {
+    llvm::errs() << "Expected a constraint but got: \n" << *def << "\n";
+    llvm::report_fatal_error("Abort");
   }
 }
 
diff --git a/mlir/lib/Target/LLVMIR/CMakeLists.txt b/mlir/lib/Target/LLVMIR/CMakeLists.txt
index 868ccbbb10..5db0885d70 100644
--- a/mlir/lib/Target/LLVMIR/CMakeLists.txt
+++ b/mlir/lib/Target/LLVMIR/CMakeLists.txt
@@ -39,6 +39,7 @@ add_mlir_translation_library(MLIRTargetLLVMIRExport
   MLIRLLVMDialect
   MLIRLLVMIRTransforms
   MLIRTranslateLib
+  MLIRTransformUtils
   )
 
 add_mlir_translation_library(MLIRToLLVMIRTranslationRegistration
diff --git a/mlir/lib/Target/LLVMIR/DebugImporter.cpp b/mlir/lib/Target/LLVMIR/DebugImporter.cpp
index 695dbf75a4..a3e81d0dd8 100644
--- a/mlir/lib/Target/LLVMIR/DebugImporter.cpp
+++ b/mlir/lib/Target/LLVMIR/DebugImporter.cpp
@@ -24,13 +24,17 @@ using namespace mlir::LLVM;
 using namespace mlir::LLVM::detail;
 
 Location DebugImporter::translateFuncLocation(llvm::Function *func) {
-  if (!func->getSubprogram())
+  llvm::DISubprogram *subprogram = func->getSubprogram();
+  if (!subprogram)
     return UnknownLoc::get(context);
 
   // Add a fused location to link the subprogram information.
-  StringAttr name = StringAttr::get(context, func->getSubprogram()->getName());
+  StringAttr funcName = StringAttr::get(context, subprogram->getName());
+  StringAttr fileName = StringAttr::get(context, subprogram->getFilename());
   return FusedLocWith<DISubprogramAttr>::get(
-      {NameLoc::get(name)}, translate(func->getSubprogram()), context);
+      {NameLoc::get(funcName),
+       FileLineColLoc::get(fileName, subprogram->getLine(), /*column=*/0)},
+      translate(subprogram), context);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenACC/CMakeLists.txt b/mlir/lib/Target/LLVMIR/Dialect/OpenACC/CMakeLists.txt
index 5e79e4be58..e43581e28c 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenACC/CMakeLists.txt
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenACC/CMakeLists.txt
@@ -10,4 +10,5 @@ add_mlir_translation_library(MLIROpenACCToLLVMIRTranslation
   MLIROpenACCDialect
   MLIRSupport
   MLIRTargetLLVMIRExport
+  MLIRTransformUtils
   )
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenACC/OpenACCToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenACC/OpenACCToLLVMIRTranslation.cpp
index 392d34cd6f..37fec190d6 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenACC/OpenACCToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenACC/OpenACCToLLVMIRTranslation.cpp
@@ -19,6 +19,7 @@
 #include "mlir/Support/LLVM.h"
 #include "mlir/Target/LLVMIR/Dialect/OpenMPCommon.h"
 #include "mlir/Target/LLVMIR/ModuleTranslation.h"
+#include "mlir/Transforms/RegionUtils.h"
 
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Frontend/OpenMP/OMPConstants.h"
@@ -395,8 +396,7 @@ static LogicalResult convertDataOp(acc::DataOp &op,
   llvm::BasicBlock *endDataBlock = llvm::BasicBlock::Create(
       ctx, "acc.end_data", builder.GetInsertBlock()->getParent());
 
-  SetVector<Block *> blocks =
-      LLVM::detail::getTopologicallySortedBlocks(op.getRegion());
+  SetVector<Block *> blocks = getTopologicallySortedBlocks(op.getRegion());
   for (Block *bb : blocks) {
     llvm::BasicBlock *llvmBB = moduleTranslation.lookupBlock(bb);
     if (bb->isEntryBlock()) {
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/CMakeLists.txt b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/CMakeLists.txt
index 41227f80cd..0a5d7c6e22 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/CMakeLists.txt
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/CMakeLists.txt
@@ -10,4 +10,5 @@ add_mlir_translation_library(MLIROpenMPToLLVMIRTranslation
   MLIROpenMPDialect
   MLIRSupport
   MLIRTargetLLVMIRExport
+  MLIRTransformUtils
   )
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 1ec3bb8e75..025f102f0f 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -194,8 +194,7 @@ static llvm::BasicBlock *convertOmpOpRegions(
 
   // Convert blocks one by one in topological order to ensure
   // defs are converted before uses.
-  SetVector<Block *> blocks =
-      LLVM::detail::getTopologicallySortedBlocks(region);
+  SetVector<Block *> blocks = getTopologicallySortedBlocks(region);
   for (Block *bb : blocks) {
     llvm::BasicBlock *llvmBB = moduleTranslation.lookupBlock(bb);
     // Retarget the branch of the entry block to the entry block of the
@@ -667,11 +666,9 @@ convertOmpTeams(omp::TeamsOp op, llvm::IRBuilderBase &builder,
                 LLVM::ModuleTranslation &moduleTranslation) {
   using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
   LogicalResult bodyGenStatus = success();
-  if (op.getNumTeamsLower() || op.getNumTeamsUpper() || op.getIfExpr() ||
-      op.getThreadLimit() || !op.getAllocatorsVars().empty() ||
-      op.getReductions()) {
+  if (!op.getAllocatorsVars().empty() || op.getReductions())
     return op.emitError("unhandled clauses for translation to LLVM IR");
-  }
+
   auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP) {
     LLVM::ModuleTranslation::SaveStack<OpenMPAllocaStackFrame> frame(
         moduleTranslation, allocaIP);
@@ -680,9 +677,25 @@ convertOmpTeams(omp::TeamsOp op, llvm::IRBuilderBase &builder,
                         moduleTranslation, bodyGenStatus);
   };
 
+  llvm::Value *numTeamsLower = nullptr;
+  if (Value numTeamsLowerVar = op.getNumTeamsLower())
+    numTeamsLower = moduleTranslation.lookupValue(numTeamsLowerVar);
+
+  llvm::Value *numTeamsUpper = nullptr;
+  if (Value numTeamsUpperVar = op.getNumTeamsUpper())
+    numTeamsUpper = moduleTranslation.lookupValue(numTeamsUpperVar);
+
+  llvm::Value *threadLimit = nullptr;
+  if (Value threadLimitVar = op.getThreadLimit())
+    threadLimit = moduleTranslation.lookupValue(threadLimitVar);
+
+  llvm::Value *ifExpr = nullptr;
+  if (Value ifExprVar = op.getIfExpr())
+    ifExpr = moduleTranslation.lookupValue(ifExprVar);
+
   llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
-  builder.restoreIP(
-      moduleTranslation.getOpenMPBuilder()->createTeams(ompLoc, bodyCB));
+  builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createTeams(
+      ompLoc, bodyCB, numTeamsLower, numTeamsUpper, threadLimit, ifExpr));
   return bodyGenStatus;
 }
 
diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
index d070e42ac0..e3562049cd 100644
--- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
@@ -991,7 +991,7 @@ FailureOr<Value> ModuleImport::convertConstant(llvm::Constant *constant) {
   }
 
   // Convert none token constants.
-  if (auto *noneToken = dyn_cast<llvm::ConstantTokenNone>(constant)) {
+  if (isa<llvm::ConstantTokenNone>(constant)) {
     return builder.create<NoneTokenOp>(loc).getResult();
   }
 
diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
index ee73b04e02..7312388bc9 100644
--- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
@@ -31,6 +31,7 @@
 #include "mlir/Support/LogicalResult.h"
 #include "mlir/Target/LLVMIR/LLVMTranslationInterface.h"
 #include "mlir/Target/LLVMIR/TypeToLLVM.h"
+#include "mlir/Transforms/RegionUtils.h"
 
 #include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/SetVector.h"
@@ -571,24 +572,6 @@ void mlir::LLVM::detail::connectPHINodes(Region &region,
   }
 }
 
-/// Sort function blocks topologically.
-SetVector<Block *>
-mlir::LLVM::detail::getTopologicallySortedBlocks(Region &region) {
-  // For each block that has not been visited yet (i.e. that has no
-  // predecessors), add it to the list as well as its successors.
-  SetVector<Block *> blocks;
-  for (Block &b : region) {
-    if (blocks.count(&b) == 0) {
-      llvm::ReversePostOrderTraversal<Block *> traversal(&b);
-      blocks.insert(traversal.begin(), traversal.end());
-    }
-  }
-  assert(blocks.size() == region.getBlocks().size() &&
-         "some blocks are not sorted");
-
-  return blocks;
-}
-
 llvm::CallInst *mlir::LLVM::detail::createIntrinsicCall(
     llvm::IRBuilderBase &builder, llvm::Intrinsic::ID intrinsic,
     ArrayRef<llvm::Value *> args, ArrayRef<llvm::Type *> tys) {
@@ -922,7 +905,7 @@ LogicalResult ModuleTranslation::convertOneFunction(LLVMFuncOp func) {
 
   // Then, convert blocks one by one in topological order to ensure defs are
   // converted before uses.
-  auto blocks = detail::getTopologicallySortedBlocks(func.getBody());
+  auto blocks = getTopologicallySortedBlocks(func.getBody());
   for (Block *bb : blocks) {
     llvm::IRBuilder<> builder(llvmContext);
     if (failed(convertBlock(*bb, bb->isEntryBlock(), builder)))
diff --git a/mlir/lib/Tools/CMakeLists.txt b/mlir/lib/Tools/CMakeLists.txt
index 6175a1ce5f..01270fa4b0 100644
--- a/mlir/lib/Tools/CMakeLists.txt
+++ b/mlir/lib/Tools/CMakeLists.txt
@@ -2,6 +2,7 @@ add_subdirectory(lsp-server-support)
 add_subdirectory(mlir-lsp-server)
 add_subdirectory(mlir-opt)
 add_subdirectory(mlir-pdll-lsp-server)
+add_subdirectory(mlir-query)
 add_subdirectory(mlir-reduce)
 add_subdirectory(mlir-tblgen)
 add_subdirectory(mlir-translate)
diff --git a/mlir/lib/Tools/mlir-query/CMakeLists.txt b/mlir/lib/Tools/mlir-query/CMakeLists.txt
new file mode 100644
index 0000000000..b81b02d42b
--- /dev/null
+++ b/mlir/lib/Tools/mlir-query/CMakeLists.txt
@@ -0,0 +1,13 @@
+set(LLVM_LINK_COMPONENTS
+  lineeditor
+  )
+
+add_mlir_library(MLIRQueryLib
+  MlirQueryMain.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Tools/mlir-query
+
+  LINK_LIBS PUBLIC
+  MLIRQuery
+  )
diff --git a/mlir/lib/Tools/mlir-query/MlirQueryMain.cpp b/mlir/lib/Tools/mlir-query/MlirQueryMain.cpp
new file mode 100644
index 0000000000..15de16a877
--- /dev/null
+++ b/mlir/lib/Tools/mlir-query/MlirQueryMain.cpp
@@ -0,0 +1,115 @@
+//===- MlirQueryMain.cpp - MLIR Query main --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the general framework of the MLIR query tool. It
+// parses the command line arguments, parses the MLIR file and outputs the query
+// results.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Tools/mlir-query/MlirQueryMain.h"
+#include "mlir/IR/BuiltinOps.h"
+#include "mlir/Parser/Parser.h"
+#include "mlir/Query/Query.h"
+#include "mlir/Query/QuerySession.h"
+#include "mlir/Support/FileUtilities.h"
+#include "mlir/Support/LogicalResult.h"
+#include "llvm/LineEditor/LineEditor.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/InitLLVM.h"
+#include "llvm/Support/SourceMgr.h"
+
+//===----------------------------------------------------------------------===//
+// Query Parser
+//===----------------------------------------------------------------------===//
+
+mlir::LogicalResult
+mlir::mlirQueryMain(int argc, char **argv, MLIRContext &context,
+                    const mlir::query::matcher::Registry &matcherRegistry) {
+
+  // Override the default '-h' and use the default PrintHelpMessage() which
+  // won't print options in categories.
+  static llvm::cl::opt<bool> help("h", llvm::cl::desc("Alias for -help"),
+                                  llvm::cl::Hidden);
+
+  static llvm::cl::OptionCategory mlirQueryCategory("mlir-query options");
+
+  static llvm::cl::list<std::string> commands(
+      "c", llvm::cl::desc("Specify command to run"),
+      llvm::cl::value_desc("command"), llvm::cl::cat(mlirQueryCategory));
+
+  static llvm::cl::opt<std::string> inputFilename(
+      llvm::cl::Positional, llvm::cl::desc("<input file>"),
+      llvm::cl::cat(mlirQueryCategory));
+
+  static llvm::cl::opt<bool> noImplicitModule{
+      "no-implicit-module",
+      llvm::cl::desc(
+          "Disable implicit addition of a top-level module op during parsing"),
+      llvm::cl::init(false)};
+
+  static llvm::cl::opt<bool> allowUnregisteredDialects(
+      "allow-unregistered-dialect",
+      llvm::cl::desc("Allow operation with no registered dialects"),
+      llvm::cl::init(false));
+
+  llvm::cl::HideUnrelatedOptions(mlirQueryCategory);
+
+  llvm::InitLLVM y(argc, argv);
+
+  llvm::cl::ParseCommandLineOptions(argc, argv, "MLIR test case query tool.\n");
+
+  if (help) {
+    llvm::cl::PrintHelpMessage();
+    return mlir::success();
+  }
+
+  // Set up the input file.
+  std::string errorMessage;
+  auto file = openInputFile(inputFilename, &errorMessage);
+  if (!file) {
+    llvm::errs() << errorMessage << "\n";
+    return mlir::failure();
+  }
+
+  auto sourceMgr = llvm::SourceMgr();
+  auto bufferId = sourceMgr.AddNewSourceBuffer(std::move(file), SMLoc());
+
+  context.allowUnregisteredDialects(allowUnregisteredDialects);
+
+  // Parse the input MLIR file.
+  OwningOpRef<Operation *> opRef =
+      noImplicitModule ? parseSourceFile(sourceMgr, &context)
+                       : parseSourceFile<mlir::ModuleOp>(sourceMgr, &context);
+  if (!opRef)
+    return mlir::failure();
+
+  mlir::query::QuerySession qs(opRef.get(), sourceMgr, bufferId,
+                               matcherRegistry);
+  if (!commands.empty()) {
+    for (auto &command : commands) {
+      mlir::query::QueryRef queryRef = mlir::query::parse(command, qs);
+      if (mlir::failed(queryRef->run(llvm::outs(), qs)))
+        return mlir::failure();
+    }
+  } else {
+    llvm::LineEditor le("mlir-query");
+    le.setListCompleter([&qs](llvm::StringRef line, size_t pos) {
+      return mlir::query::complete(line, pos, qs);
+    });
+    while (std::optional<std::string> line = le.readLine()) {
+      mlir::query::QueryRef queryRef = mlir::query::parse(*line, qs);
+      (void)queryRef->run(llvm::outs(), qs);
+      llvm::outs().flush();
+      if (qs.terminate)
+        break;
+    }
+  }
+
+  return mlir::success();
+}
diff --git a/mlir/lib/Transforms/Mem2Reg.cpp b/mlir/lib/Transforms/Mem2Reg.cpp
index 65de25dd2f..26a7ea5d5e 100644
--- a/mlir/lib/Transforms/Mem2Reg.cpp
+++ b/mlir/lib/Transforms/Mem2Reg.cpp
@@ -16,6 +16,8 @@
 #include "mlir/Interfaces/MemorySlotInterfaces.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "mlir/Transforms/Passes.h"
+#include "mlir/Transforms/RegionUtils.h"
+#include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/GenericIteratedDominanceFrontier.h"
@@ -96,6 +98,9 @@ using namespace mlir;
 
 namespace {
 
+using BlockingUsesMap =
+    llvm::MapVector<Operation *, SmallPtrSet<OpOperand *, 4>>;
+
 /// Information computed during promotion analysis used to perform actual
 /// promotion.
 struct MemorySlotPromotionInfo {
@@ -106,7 +111,7 @@ struct MemorySlotPromotionInfo {
   /// its uses, it is because the defining ops of the blocking uses requested
   /// it. The defining ops therefore must also have blocking uses or be the
   /// starting point of the bloccking uses.
-  DenseMap<Operation *, SmallPtrSet<OpOperand *, 4>> userToBlockingUses;
+  BlockingUsesMap userToBlockingUses;
 };
 
 /// Computes information for basic slot promotion. This will check that direct
@@ -129,8 +134,7 @@ class MemorySlotPromotionAnalyzer {
   /// uses (typically, removing its users because it will delete itself to
   /// resolve its own blocking uses). This will fail if one of the transitive
   /// users cannot remove a requested use, and should prevent promotion.
-  LogicalResult computeBlockingUses(
-      DenseMap<Operation *, SmallPtrSet<OpOperand *, 4>> &userToBlockingUses);
+  LogicalResult computeBlockingUses(BlockingUsesMap &userToBlockingUses);
 
   /// Computes in which blocks the value stored in the slot is actually used,
   /// meaning blocks leading to a load. This method uses `definingBlocks`, the
@@ -233,7 +237,7 @@ Value MemorySlotPromoter::getLazyDefaultValue() {
 }
 
 LogicalResult MemorySlotPromotionAnalyzer::computeBlockingUses(
-    DenseMap<Operation *, SmallPtrSet<OpOperand *, 4>> &userToBlockingUses) {
+    BlockingUsesMap &userToBlockingUses) {
   // The promotion of an operation may require the promotion of further
   // operations (typically, removing operations that use an operation that must
   // delete itself). We thus need to start from the use of the slot pointer and
@@ -243,7 +247,7 @@ LogicalResult MemorySlotPromotionAnalyzer::computeBlockingUses(
   // use it.
   for (OpOperand &use : slot.ptr.getUses()) {
     SmallPtrSet<OpOperand *, 4> &blockingUses =
-        userToBlockingUses.getOrInsertDefault(use.getOwner());
+        userToBlockingUses[use.getOwner()];
     blockingUses.insert(&use);
   }
 
@@ -281,7 +285,7 @@ LogicalResult MemorySlotPromotionAnalyzer::computeBlockingUses(
       assert(llvm::is_contained(user->getResults(), blockingUse->get()));
 
       SmallPtrSetImpl<OpOperand *> &newUserBlockingUseSet =
-          userToBlockingUses.getOrInsertDefault(blockingUse->getOwner());
+          userToBlockingUses[blockingUse->getOwner()];
       newUserBlockingUseSet.insert(blockingUse);
     }
   }
@@ -515,15 +519,37 @@ void MemorySlotPromoter::computeReachingDefInRegion(Region *region,
   }
 }
 
+/// Sorts `ops` according to dominance. Relies on the topological order of basic
+/// blocks to get a deterministic ordering.
+static void dominanceSort(SmallVector<Operation *> &ops, Region &region) {
+  // Produce a topological block order and construct a map to lookup the indices
+  // of blocks.
+  DenseMap<Block *, size_t> topoBlockIndices;
+  SetVector<Block *> topologicalOrder = getTopologicallySortedBlocks(region);
+  for (auto [index, block] : llvm::enumerate(topologicalOrder))
+    topoBlockIndices[block] = index;
+
+  // Combining the topological order of the basic blocks together with block
+  // internal operation order guarantees a deterministic, dominance respecting
+  // order.
+  llvm::sort(ops, [&](Operation *lhs, Operation *rhs) {
+    size_t lhsBlockIndex = topoBlockIndices.at(lhs->getBlock());
+    size_t rhsBlockIndex = topoBlockIndices.at(rhs->getBlock());
+    if (lhsBlockIndex == rhsBlockIndex)
+      return lhs->isBeforeInBlock(rhs);
+    return lhsBlockIndex < rhsBlockIndex;
+  });
+}
+
 void MemorySlotPromoter::removeBlockingUses() {
-  llvm::SetVector<Operation *> usersToRemoveUses;
-  for (auto &user : llvm::make_first_range(info.userToBlockingUses))
-    usersToRemoveUses.insert(user);
-  SetVector<Operation *> sortedUsersToRemoveUses =
-      mlir::topologicalSort(usersToRemoveUses);
+  llvm::SmallVector<Operation *> usersToRemoveUses(
+      llvm::make_first_range(info.userToBlockingUses));
+
+  // Sort according to dominance.
+  dominanceSort(usersToRemoveUses, *slot.ptr.getParentBlock()->getParent());
 
   llvm::SmallVector<Operation *> toErase;
-  for (Operation *toPromote : llvm::reverse(sortedUsersToRemoveUses)) {
+  for (Operation *toPromote : llvm::reverse(usersToRemoveUses)) {
     if (auto toPromoteMemOp = dyn_cast<PromotableMemOpInterface>(toPromote)) {
       Value reachingDef = reachingDefs.lookup(toPromoteMemOp);
       // If no reaching definition is known, this use is outside the reach of
diff --git a/mlir/lib/Transforms/Utils/RegionUtils.cpp b/mlir/lib/Transforms/Utils/RegionUtils.cpp
index b95af9ca02..1f2344677e 100644
--- a/mlir/lib/Transforms/Utils/RegionUtils.cpp
+++ b/mlir/lib/Transforms/Utils/RegionUtils.cpp
@@ -836,3 +836,19 @@ LogicalResult mlir::simplifyRegions(RewriterBase &rewriter,
   return success(eliminatedBlocks || eliminatedOpsOrArgs ||
                  mergedIdenticalBlocks);
 }
+
+SetVector<Block *> mlir::getTopologicallySortedBlocks(Region &region) {
+  // For each block that has not been visited yet (i.e. that has no
+  // predecessors), add it to the list as well as its successors.
+  SetVector<Block *> blocks;
+  for (Block &b : region) {
+    if (blocks.count(&b) == 0) {
+      llvm::ReversePostOrderTraversal<Block *> traversal(&b);
+      blocks.insert(traversal.begin(), traversal.end());
+    }
+  }
+  assert(blocks.size() == region.getBlocks().size() &&
+         "some blocks are not sorted");
+
+  return blocks;
+}
diff --git a/mlir/python/CMakeLists.txt b/mlir/python/CMakeLists.txt
index 088d9a765b..88e6e13602 100644
--- a/mlir/python/CMakeLists.txt
+++ b/mlir/python/CMakeLists.txt
@@ -46,6 +46,15 @@ declare_mlir_python_sources(MLIRPythonCAPI.HeaderSources
 # Dialect bindings
 ################################################################################
 
+declare_mlir_dialect_python_bindings(
+  ADD_TO_PARENT MLIRPythonSources.Dialects
+  ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir"
+  TD_FILE dialects/AffineOps.td
+  SOURCES
+    dialects/affine.py
+  DIALECT_NAME affine
+  GEN_ENUM_BINDINGS)
+
 declare_mlir_dialect_python_bindings(
   ADD_TO_PARENT MLIRPythonSources.Dialects
   ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir"
@@ -68,7 +77,6 @@ declare_mlir_dialect_python_bindings(
   TD_FILE dialects/BufferizationOps.td
   SOURCES
     dialects/bufferization.py
-    dialects/_bufferization_ops_ext.py
   DIALECT_NAME bufferization
   GEN_ENUM_BINDINGS_TD_FILE
     "../../include/mlir/Dialect/Bufferization/IR/BufferizationEnums.td"
@@ -80,7 +88,6 @@ declare_mlir_dialect_python_bindings(
   TD_FILE dialects/BuiltinOps.td
   SOURCES
     dialects/builtin.py
-    dialects/_builtin_ops_ext.py
   DIALECT_NAME builtin)
 
 declare_mlir_dialect_python_bindings(
@@ -105,7 +112,6 @@ declare_mlir_dialect_python_bindings(
   TD_FILE dialects/FuncOps.td
   SOURCES
     dialects/func.py
-    dialects/_func_ops_ext.py
   DIALECT_NAME func)
 
 declare_mlir_dialect_python_bindings(
@@ -121,7 +127,6 @@ declare_mlir_dialect_python_bindings(
   ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir"
   TD_FILE dialects/LinalgOps.td
   SOURCES
-    dialects/_linalg_ops_ext.py
   SOURCES_GLOB
     dialects/linalg/*.py
   DIALECT_NAME linalg
@@ -142,7 +147,6 @@ ADD_TO_PARENT MLIRPythonSources.Dialects
 ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir"
   TD_FILE dialects/TransformPDLExtensionOps.td
   SOURCES
-    dialects/_transform_pdl_extension_ops_ext.py
     dialects/transform/pdl.py
   DIALECT_NAME transform
   EXTENSION_NAME transform_pdl_extension)
@@ -152,7 +156,6 @@ declare_mlir_dialect_python_bindings(
   ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir"
   TD_FILE dialects/TransformOps.td
   SOURCES
-    dialects/_transform_ops_ext.py
     dialects/transform/__init__.py
     _mlir_libs/_mlir/dialects/transform/__init__.pyi
   DIALECT_NAME transform
@@ -165,7 +168,6 @@ declare_mlir_dialect_extension_python_bindings(
   ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir"
   TD_FILE dialects/BufferizationTransformOps.td
   SOURCES
-    dialects/_bufferization_transform_ops_ext.py
     dialects/transform/bufferization.py
   DIALECT_NAME transform
   EXTENSION_NAME bufferization_transform)
@@ -175,7 +177,6 @@ declare_mlir_dialect_extension_python_bindings(
   ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir"
   TD_FILE dialects/GPUTransformOps.td
   SOURCES
-    dialects/_gpu_transform_ops_ext.py
     dialects/transform/gpu.py
   DIALECT_NAME transform
   EXTENSION_NAME gpu_transform)
@@ -185,7 +186,6 @@ declare_mlir_dialect_extension_python_bindings(
   ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir"
   TD_FILE dialects/SCFLoopTransformOps.td
   SOURCES
-    dialects/_loop_transform_ops_ext.py
     dialects/transform/loop.py
   DIALECT_NAME transform
   EXTENSION_NAME loop_transform)
@@ -195,7 +195,6 @@ declare_mlir_dialect_extension_python_bindings(
   ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir"
   TD_FILE dialects/MemRefTransformOps.td
   SOURCES
-    dialects/_memref_transform_ops_ext.py
     dialects/transform/memref.py
   DIALECT_NAME transform
   EXTENSION_NAME memref_transform)
@@ -214,7 +213,6 @@ declare_mlir_dialect_extension_python_bindings(
   ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir"
   TD_FILE dialects/LinalgStructuredTransformOps.td
   SOURCES
-    dialects/_structured_transform_ops_ext.py
     dialects/transform/structured.py
   DIALECT_NAME transform
   EXTENSION_NAME structured_transform
@@ -236,7 +234,6 @@ declare_mlir_dialect_extension_python_bindings(
   ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir"
   TD_FILE dialects/TensorTransformOps.td
   SOURCES
-    dialects/_tensor_transform_ops_ext.py
     dialects/transform/tensor.py
   DIALECT_NAME transform
   EXTENSION_NAME tensor_transform)
@@ -266,7 +263,6 @@ declare_mlir_dialect_python_bindings(
   TD_FILE dialects/ArithOps.td
   SOURCES
     dialects/arith.py
-    dialects/_arith_ops_ext.py
   DIALECT_NAME arith
   GEN_ENUM_BINDINGS)
 
@@ -276,7 +272,6 @@ declare_mlir_dialect_python_bindings(
   TD_FILE dialects/MemRefOps.td
   SOURCES
     dialects/memref.py
-    dialects/_memref_ops_ext.py
   DIALECT_NAME memref)
 
 declare_mlir_dialect_python_bindings(
@@ -285,7 +280,6 @@ declare_mlir_dialect_python_bindings(
   TD_FILE dialects/MLProgramOps.td
   SOURCES
     dialects/ml_program.py
-    dialects/_ml_program_ops_ext.py
   DIALECT_NAME ml_program)
 
 declare_mlir_dialect_python_bindings(
@@ -329,7 +323,6 @@ declare_mlir_dialect_python_bindings(
   TD_FILE dialects/PDLOps.td
   SOURCES
     dialects/pdl.py
-    dialects/_pdl_ops_ext.py
     _mlir_libs/_mlir/dialects/pdl.pyi
   DIALECT_NAME pdl)
 
@@ -347,7 +340,6 @@ declare_mlir_dialect_python_bindings(
   TD_FILE dialects/SCFOps.td
   SOURCES
     dialects/scf.py
-    dialects/_scf_ops_ext.py
   DIALECT_NAME scf)
 
 declare_mlir_dialect_python_bindings(
@@ -373,7 +365,6 @@ declare_mlir_dialect_python_bindings(
   TD_FILE dialects/TensorOps.td
   SOURCES
     dialects/tensor.py
-    dialects/_tensor_ops_ext.py
   DIALECT_NAME tensor)
 
 declare_mlir_dialect_python_bindings(
diff --git a/mlir/python/mlir/_mlir_libs/_mlir/ir.pyi b/mlir/python/mlir/_mlir_libs/_mlir/ir.pyi
index e8f4440d21..2609117dd2 100644
--- a/mlir/python/mlir/_mlir_libs/_mlir/ir.pyi
+++ b/mlir/python/mlir/_mlir_libs/_mlir/ir.pyi
@@ -755,6 +755,8 @@ class InsertionPoint:
     def __exit__(self, arg0: object, arg1: object, arg2: object) -> None: ...
     @property
     def block(self) -> Block: ...
+    @property
+    def ref_operation(self) -> Optional[_OperationBase]: ...
 
 # TODO: Auto-generated. Audit and fix.
 class IntegerAttr(Attribute):
diff --git a/mlir/python/mlir/dialects/AffineOps.td b/mlir/python/mlir/dialects/AffineOps.td
new file mode 100644
index 0000000000..e12ffafb89
--- /dev/null
+++ b/mlir/python/mlir/dialects/AffineOps.td
@@ -0,0 +1,14 @@
+//===-- AffineOps.td - Entry point for Affine bindings -----*- tablegen -*-===//
+//                                                                                                                      
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.                                        
+// See https://llvm.org/LICENSE.txt for license information.                                                            
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                                                              
+//                                                                                                                      
+//===----------------------------------------------------------------------===// 
+
+#ifndef PYTHON_BINDINGS_AFFINE_OPS
+#define PYTHON_BINDINGS_AFFINE_OPS
+
+include "mlir/Dialect/Affine/IR/AffineOps.td"
+
+#endif
diff --git a/mlir/python/mlir/dialects/_arith_ops_ext.py b/mlir/python/mlir/dialects/_arith_ops_ext.py
deleted file mode 100644
index df38f87171..0000000000
--- a/mlir/python/mlir/dialects/_arith_ops_ext.py
+++ /dev/null
@@ -1,69 +0,0 @@
-#  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-#  See https://llvm.org/LICENSE.txt for license information.
-#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-try:
-    from ..ir import *
-    from ._ods_common import get_default_loc_context as _get_default_loc_context
-
-    from typing import Any, List, Union
-except ImportError as e:
-    raise RuntimeError("Error loading imports from extension module") from e
-
-
-def _isa(obj: Any, cls: type):
-    try:
-        cls(obj)
-    except ValueError:
-        return False
-    return True
-
-
-def _is_any_of(obj: Any, classes: List[type]):
-    return any(_isa(obj, cls) for cls in classes)
-
-
-def _is_integer_like_type(type: Type):
-    return _is_any_of(type, [IntegerType, IndexType])
-
-
-def _is_float_type(type: Type):
-    return _is_any_of(type, [BF16Type, F16Type, F32Type, F64Type])
-
-
-class ConstantOp:
-    """Specialization for the constant op class."""
-
-    def __init__(
-        self, result: Type, value: Union[int, float, Attribute], *, loc=None, ip=None
-    ):
-        if isinstance(value, int):
-            super().__init__(IntegerAttr.get(result, value), loc=loc, ip=ip)
-        elif isinstance(value, float):
-            super().__init__(FloatAttr.get(result, value), loc=loc, ip=ip)
-        else:
-            super().__init__(value, loc=loc, ip=ip)
-
-    @classmethod
-    def create_index(cls, value: int, *, loc=None, ip=None):
-        """Create an index-typed constant."""
-        return cls(
-            IndexType.get(context=_get_default_loc_context(loc)), value, loc=loc, ip=ip
-        )
-
-    @property
-    def type(self):
-        return self.results[0].type
-
-    @property
-    def value(self):
-        return Attribute(self.operation.attributes["value"])
-
-    @property
-    def literal_value(self) -> Union[int, float]:
-        if _is_integer_like_type(self.type):
-            return IntegerAttr(self.value).value
-        elif _is_float_type(self.type):
-            return FloatAttr(self.value).value
-        else:
-            raise ValueError("only integer and float constants have literal values")
diff --git a/mlir/python/mlir/dialects/_bufferization_ops_ext.py b/mlir/python/mlir/dialects/_bufferization_ops_ext.py
deleted file mode 100644
index 1066cb4c77..0000000000
--- a/mlir/python/mlir/dialects/_bufferization_ops_ext.py
+++ /dev/null
@@ -1,41 +0,0 @@
-#  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-#  See https://llvm.org/LICENSE.txt for license information.
-#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-try:
-    from typing import Sequence, Union
-    from ..ir import *
-    from ._ods_common import get_default_loc_context
-
-    from typing import Any, List, Union
-except ImportError as e:
-    raise RuntimeError("Error loading imports from extension module") from e
-
-
-class AllocTensorOp:
-    """Extends the bufferization.alloc_tensor op."""
-
-    def __init__(
-        self,
-        tensor_type: Type,
-        dynamic_sizes: Sequence[Value],
-        copy: Value,
-        size_hint: Value,
-        escape: BoolAttr,
-        *,
-        loc=None,
-        ip=None
-    ):
-        """Constructs an `alloc_tensor` with static and/or dynamic sizes."""
-        context = get_default_loc_context(loc)
-        attributes = {}
-        if escape:
-            attributes["escape"] = escape
-        op = self.build_generic(
-            results=[tensor_type],
-            operands=[dynamic_sizes, copy, size_hint],
-            attributes=attributes,
-            loc=loc,
-            ip=ip,
-        )
-        OpView.__init__(self, op)
diff --git a/mlir/python/mlir/dialects/_bufferization_transform_ops_ext.py b/mlir/python/mlir/dialects/_bufferization_transform_ops_ext.py
deleted file mode 100644
index 7e6c1b81cb..0000000000
--- a/mlir/python/mlir/dialects/_bufferization_transform_ops_ext.py
+++ /dev/null
@@ -1,128 +0,0 @@
-#  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-#  See https://llvm.org/LICENSE.txt for license information.
-#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-try:
-    from ..ir import *
-    from ..dialects import transform
-except ImportError as e:
-    raise RuntimeError("Error loading imports from extension module") from e
-
-from enum import Enum
-from typing import Optional, overload, Union
-
-
-class EmptyTensorToAllocTensorOp:
-    """Specialization for EmptyTensorToAllocTensorOp class."""
-
-    @overload
-    def __init__(
-        self,
-        transformed_type: Type,
-        target: Union[Operation, OpView, Value],
-        *,
-        loc=None,
-        ip=None
-    ):
-        ...
-
-    @overload
-    def __init__(self, target: Union[Operation, OpView, Value], *, loc=None, ip=None):
-        ...
-
-    def __init__(
-        self,
-        transformed_type_or_target: Type,
-        target_or_none: Optional[Union[Operation, OpView, Value]] = None,
-        *,
-        loc=None,
-        ip=None
-    ):
-        if isinstance(transformed_type_or_target, Type):
-            transformed_type = transformed_type_or_target
-            target = target_or_none
-        else:
-            transformed_type = transform.OperationType.get("bufferization.alloc_tensor")
-            target = transformed_type_or_target
-
-        super().__init__(
-            transformed_type,
-            target,
-            loc=loc,
-            ip=ip,
-        )
-
-
-class OneShotBufferizeOp:
-    """Specialization for OneShotBufferizeOp class."""
-
-    @overload
-    def __init__(
-        self,
-        transformed_type: Type,
-        target: Union[Operation, OpView, Value],
-        *,
-        allow_return_allocs_from_loops: Optional[bool] = None,
-        allow_unknown_ops: Optional[bool] = None,
-        bufferize_function_boundaries: Optional[bool] = None,
-        function_boundary_type_conversion: Optional[Enum] = None,
-        memcpy_op: Optional[str] = None,
-        print_conflicts: Optional[bool] = None,
-        test_analysis_only: Optional[bool] = None,
-        loc=None,
-        ip=None
-    ):
-        ...
-
-    @overload
-    def __init__(
-        self,
-        target: Union[Operation, OpView, Value],
-        *,
-        allow_return_allocs_from_loops: Optional[bool] = None,
-        allow_unknown_ops: Optional[bool] = None,
-        bufferize_function_boundaries: Optional[bool] = None,
-        function_boundary_type_conversion: Optional[Enum] = None,
-        memcpy_op: Optional[str] = None,
-        print_conflicts: Optional[bool] = None,
-        test_analysis_only: Optional[bool] = None,
-        loc=None,
-        ip=None
-    ):
-        ...
-
-    def __init__(
-        self,
-        transformed_type_or_target: Type,
-        target_or_none: Optional[Union[Operation, OpView, Value]] = None,
-        *,
-        allow_return_allocs_from_loops: Optional[bool] = None,
-        allow_unknown_ops: Optional[bool] = None,
-        bufferize_function_boundaries: Optional[bool] = None,
-        function_boundary_type_conversion: Optional[Enum] = None,
-        memcpy_op: Optional[str] = None,
-        print_conflicts: Optional[bool] = None,
-        test_analysis_only: Optional[bool] = None,
-        loc=None,
-        ip=None
-    ):
-        if isinstance(transformed_type_or_target, Type):
-            transformed_type = transformed_type_or_target
-            target = target_or_none
-        else:
-            transformed_type = transform.AnyOpType.get()
-            target = transformed_type_or_target
-
-        super().__init__(
-            transformed_type,
-            target,
-            allow_return_allocs_from_loops=allow_return_allocs_from_loops,
-            allow_unknown_ops=allow_unknown_ops,
-            bufferize_function_boundaries=bufferize_function_boundaries,
-            function_boundary_type_conversion=function_boundary_type_conversion,
-            memcpy_op=memcpy_op,
-            print_conflicts=print_conflicts,
-            test_analysis_only=test_analysis_only,
-            loc=loc,
-            ip=ip,
-        )
diff --git a/mlir/python/mlir/dialects/_builtin_ops_ext.py b/mlir/python/mlir/dialects/_builtin_ops_ext.py
deleted file mode 100644
index 27a6012305..0000000000
--- a/mlir/python/mlir/dialects/_builtin_ops_ext.py
+++ /dev/null
@@ -1,20 +0,0 @@
-#  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-#  See https://llvm.org/LICENSE.txt for license information.
-#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-try:
-    from ..ir import *
-except ImportError as e:
-    raise RuntimeError("Error loading imports from extension module") from e
-
-
-class ModuleOp:
-    """Specialization for the module op class."""
-
-    def __init__(self, *, loc=None, ip=None):
-        super().__init__(self.build_generic(results=[], operands=[], loc=loc, ip=ip))
-        body = self.regions[0].blocks.append()
-
-    @property
-    def body(self):
-        return self.regions[0].blocks[0]
diff --git a/mlir/python/mlir/dialects/_func_ops_ext.py b/mlir/python/mlir/dialects/_func_ops_ext.py
deleted file mode 100644
index 6d264c33f1..0000000000
--- a/mlir/python/mlir/dialects/_func_ops_ext.py
+++ /dev/null
@@ -1,319 +0,0 @@
-#  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-#  See https://llvm.org/LICENSE.txt for license information.
-#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-try:
-    from ..ir import *
-    from ._ods_common import get_default_loc_context as _get_default_loc_context
-
-    import inspect
-
-    from typing import Any, List, Optional, Sequence, Union
-except ImportError as e:
-    raise RuntimeError("Error loading imports from extension module") from e
-
-ARGUMENT_ATTRIBUTE_NAME = "arg_attrs"
-RESULT_ATTRIBUTE_NAME = "res_attrs"
-
-
-class ConstantOp:
-    """Specialization for the constant op class."""
-
-    def __init__(self, result: Type, value: Attribute, *, loc=None, ip=None):
-        super().__init__(result, value, loc=loc, ip=ip)
-
-    @property
-    def type(self):
-        return self.results[0].type
-
-
-class FuncOp:
-    """Specialization for the func op class."""
-
-    def __init__(
-        self, name, type, *, visibility=None, body_builder=None, loc=None, ip=None
-    ):
-        """
-        Create a FuncOp with the provided `name`, `type`, and `visibility`.
-        - `name` is a string representing the function name.
-        - `type` is either a FunctionType or a pair of list describing inputs and
-          results.
-        - `visibility` is a string matching `public`, `private`, or `nested`. None
-          implies private visibility.
-        - `body_builder` is an optional callback, when provided a new entry block
-          is created and the callback is invoked with the new op as argument within
-          an InsertionPoint context already set for the block. The callback is
-          expected to insert a terminator in the block.
-        """
-        sym_name = StringAttr.get(str(name))
-
-        # If the type is passed as a tuple, build a FunctionType on the fly.
-        if isinstance(type, tuple):
-            type = FunctionType.get(inputs=type[0], results=type[1])
-
-        type = TypeAttr.get(type)
-        sym_visibility = (
-            StringAttr.get(str(visibility)) if visibility is not None else None
-        )
-        super().__init__(sym_name, type, sym_visibility=sym_visibility, loc=loc, ip=ip)
-        if body_builder:
-            entry_block = self.add_entry_block()
-            with InsertionPoint(entry_block):
-                body_builder(self)
-
-    @property
-    def is_external(self):
-        return len(self.regions[0].blocks) == 0
-
-    @property
-    def body(self):
-        return self.regions[0]
-
-    @property
-    def type(self):
-        return FunctionType(TypeAttr(self.attributes["function_type"]).value)
-
-    @property
-    def visibility(self):
-        return self.attributes["sym_visibility"]
-
-    @property
-    def name(self) -> StringAttr:
-        return StringAttr(self.attributes["sym_name"])
-
-    @property
-    def entry_block(self):
-        if self.is_external:
-            raise IndexError("External function does not have a body")
-        return self.regions[0].blocks[0]
-
-    def add_entry_block(self, arg_locs: Optional[Sequence[Location]] = None):
-        """
-        Add an entry block to the function body using the function signature to
-        infer block arguments.
-        Returns the newly created block
-        """
-        if not self.is_external:
-            raise IndexError("The function already has an entry block!")
-        self.body.blocks.append(*self.type.inputs, arg_locs=arg_locs)
-        return self.body.blocks[0]
-
-    @property
-    def arg_attrs(self):
-        return ArrayAttr(self.attributes[ARGUMENT_ATTRIBUTE_NAME])
-
-    @arg_attrs.setter
-    def arg_attrs(self, attribute: Union[ArrayAttr, list]):
-        if isinstance(attribute, ArrayAttr):
-            self.attributes[ARGUMENT_ATTRIBUTE_NAME] = attribute
-        else:
-            self.attributes[ARGUMENT_ATTRIBUTE_NAME] = ArrayAttr.get(
-                attribute, context=self.context
-            )
-
-    @property
-    def arguments(self):
-        return self.entry_block.arguments
-
-    @property
-    def result_attrs(self):
-        return self.attributes[RESULT_ATTRIBUTE_NAME]
-
-    @result_attrs.setter
-    def result_attrs(self, attribute: ArrayAttr):
-        self.attributes[RESULT_ATTRIBUTE_NAME] = attribute
-
-    @classmethod
-    def from_py_func(
-        FuncOp,
-        *inputs: Type,
-        results: Optional[Sequence[Type]] = None,
-        name: Optional[str] = None,
-    ):
-        """Decorator to define an MLIR FuncOp specified as a python function.
-
-        Requires that an `mlir.ir.InsertionPoint` and `mlir.ir.Location` are
-        active for the current thread (i.e. established in a `with` block).
-
-        When applied as a decorator to a Python function, an entry block will
-        be constructed for the FuncOp with types as specified in `*inputs`. The
-        block arguments will be passed positionally to the Python function. In
-        addition, if the Python function accepts keyword arguments generally or
-        has a corresponding keyword argument, the following will be passed:
-          * `func_op`: The `func` op being defined.
-
-        By default, the function name will be the Python function `__name__`. This
-        can be overriden by passing the `name` argument to the decorator.
-
-        If `results` is not specified, then the decorator will implicitly
-        insert a `ReturnOp` with the `Value`'s returned from the decorated
-        function. It will also set the `FuncOp` type with the actual return
-        value types. If `results` is specified, then the decorated function
-        must return `None` and no implicit `ReturnOp` is added (nor are the result
-        types updated). The implicit behavior is intended for simple, single-block
-        cases, and users should specify result types explicitly for any complicated
-        cases.
-
-        The decorated function can further be called from Python and will insert
-        a `CallOp` at the then-current insertion point, returning either None (
-        if no return values), a unary Value (for one result), or a list of Values).
-        This mechanism cannot be used to emit recursive calls (by construction).
-        """
-
-        def decorator(f):
-            from . import func
-
-            # Introspect the callable for optional features.
-            sig = inspect.signature(f)
-            has_arg_func_op = False
-            for param in sig.parameters.values():
-                if param.kind == param.VAR_KEYWORD:
-                    has_arg_func_op = True
-                if param.name == "func_op" and (
-                    param.kind == param.POSITIONAL_OR_KEYWORD
-                    or param.kind == param.KEYWORD_ONLY
-                ):
-                    has_arg_func_op = True
-
-            # Emit the FuncOp.
-            implicit_return = results is None
-            symbol_name = name or f.__name__
-            function_type = FunctionType.get(
-                inputs=inputs, results=[] if implicit_return else results
-            )
-            func_op = FuncOp(name=symbol_name, type=function_type)
-            with InsertionPoint(func_op.add_entry_block()):
-                func_args = func_op.entry_block.arguments
-                func_kwargs = {}
-                if has_arg_func_op:
-                    func_kwargs["func_op"] = func_op
-                return_values = f(*func_args, **func_kwargs)
-                if not implicit_return:
-                    return_types = list(results)
-                    assert return_values is None, (
-                        "Capturing a python function with explicit `results=` "
-                        "requires that the wrapped function returns None."
-                    )
-                else:
-                    # Coerce return values, add ReturnOp and rewrite func type.
-                    if return_values is None:
-                        return_values = []
-                    elif isinstance(return_values, tuple):
-                        return_values = list(return_values)
-                    elif isinstance(return_values, Value):
-                        # Returning a single value is fine, coerce it into a list.
-                        return_values = [return_values]
-                    elif isinstance(return_values, OpView):
-                        # Returning a single operation is fine, coerce its results a list.
-                        return_values = return_values.operation.results
-                    elif isinstance(return_values, Operation):
-                        # Returning a single operation is fine, coerce its results a list.
-                        return_values = return_values.results
-                    else:
-                        return_values = list(return_values)
-                    func.ReturnOp(return_values)
-                    # Recompute the function type.
-                    return_types = [v.type for v in return_values]
-                    function_type = FunctionType.get(
-                        inputs=inputs, results=return_types
-                    )
-                    func_op.attributes["function_type"] = TypeAttr.get(function_type)
-
-            def emit_call_op(*call_args):
-                call_op = func.CallOp(
-                    return_types, FlatSymbolRefAttr.get(symbol_name), call_args
-                )
-                if return_types is None:
-                    return None
-                elif len(return_types) == 1:
-                    return call_op.result
-                else:
-                    return call_op.results
-
-            wrapped = emit_call_op
-            wrapped.__name__ = f.__name__
-            wrapped.func_op = func_op
-            return wrapped
-
-        return decorator
-
-
-class CallOp:
-    """Specialization for the call op class."""
-
-    def __init__(
-        self,
-        calleeOrResults: Union[FuncOp, List[Type]],
-        argumentsOrCallee: Union[List, FlatSymbolRefAttr, str],
-        arguments: Optional[List] = None,
-        *,
-        loc=None,
-        ip=None,
-    ):
-        """Creates an call operation.
-
-        The constructor accepts three different forms:
-
-          1. A function op to be called followed by a list of arguments.
-          2. A list of result types, followed by the name of the function to be
-             called as string, following by a list of arguments.
-          3. A list of result types, followed by the name of the function to be
-             called as symbol reference attribute, followed by a list of arguments.
-
-        For example
-
-            f = func.FuncOp("foo", ...)
-            func.CallOp(f, [args])
-            func.CallOp([result_types], "foo", [args])
-
-        In all cases, the location and insertion point may be specified as keyword
-        arguments if not provided by the surrounding context managers.
-        """
-
-        # TODO: consider supporting constructor "overloads", e.g., through a custom
-        # or pybind-provided metaclass.
-        if isinstance(calleeOrResults, FuncOp):
-            if not isinstance(argumentsOrCallee, list):
-                raise ValueError(
-                    "when constructing a call to a function, expected "
-                    + "the second argument to be a list of call arguments, "
-                    + f"got {type(argumentsOrCallee)}"
-                )
-            if arguments is not None:
-                raise ValueError(
-                    "unexpected third argument when constructing a call"
-                    + "to a function"
-                )
-
-            super().__init__(
-                calleeOrResults.type.results,
-                FlatSymbolRefAttr.get(
-                    calleeOrResults.name.value, context=_get_default_loc_context(loc)
-                ),
-                argumentsOrCallee,
-                loc=loc,
-                ip=ip,
-            )
-            return
-
-        if isinstance(argumentsOrCallee, list):
-            raise ValueError(
-                "when constructing a call to a function by name, "
-                + "expected the second argument to be a string or a "
-                + f"FlatSymbolRefAttr, got {type(argumentsOrCallee)}"
-            )
-
-        if isinstance(argumentsOrCallee, FlatSymbolRefAttr):
-            super().__init__(
-                calleeOrResults, argumentsOrCallee, arguments, loc=loc, ip=ip
-            )
-        elif isinstance(argumentsOrCallee, str):
-            super().__init__(
-                calleeOrResults,
-                FlatSymbolRefAttr.get(
-                    argumentsOrCallee, context=_get_default_loc_context(loc)
-                ),
-                arguments,
-                loc=loc,
-                ip=ip,
-            )
diff --git a/mlir/python/mlir/dialects/_gpu_transform_ops_ext.py b/mlir/python/mlir/dialects/_gpu_transform_ops_ext.py
deleted file mode 100644
index ba72bac3a1..0000000000
--- a/mlir/python/mlir/dialects/_gpu_transform_ops_ext.py
+++ /dev/null
@@ -1,124 +0,0 @@
-#  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-#  See https://llvm.org/LICENSE.txt for license information.
-#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-try:
-    from ..ir import *
-    from ..dialects import transform
-except ImportError as e:
-    raise RuntimeError("Error loading imports from extension module") from e
-
-from typing import Optional, Sequence, Union, overload
-
-
-class MapForallToBlocks:
-    """Specialization for MapForallToBlocks class."""
-
-    @overload
-    def __init__(
-        self,
-        result_type: Type,
-        target: Union[Operation, OpView, Value],
-        *,
-        grid_dims: Optional[Union[Sequence[int], Attribute]] = None,
-        generate_gpu_launch: Optional[Union[bool, Attribute]] = None,
-        loc=None,
-        ip=None
-    ):
-        ...
-
-    @overload
-    def __init__(
-        self,
-        target: Union[Operation, OpView, Value],
-        *,
-        grid_dims: Optional[Union[Sequence[int], Attribute]] = None,
-        generate_gpu_launch: Optional[Union[bool, Attribute]] = None,
-        loc=None,
-        ip=None
-    ):
-        ...
-
-    def __init__(
-        self,
-        result_type_or_target: Union[Operation, OpView, Type, Value],
-        target_or_none: Optional[Union[Operation, OpView, Value]] = None,
-        *,
-        grid_dims: Optional[Union[Sequence[int], Attribute]] = None,
-        generate_gpu_launch: Optional[Union[bool, Attribute]] = None,
-        loc=None,
-        ip=None
-    ):
-        if isinstance(result_type_or_target, Type):
-            result_type = result_type_or_target
-            target = target_or_none
-        else:
-            result_type = transform.AnyOpType.get()
-            target = result_type_or_target
-
-        super().__init__(
-            result_type,
-            target,
-            grid_dims=grid_dims,
-            generate_gpu_launch=generate_gpu_launch,
-            loc=loc,
-            ip=ip,
-        )
-
-
-class MapNestedForallToThreads:
-    """Specialization for MapNestedForallToThreads class."""
-
-    @overload
-    def __init__(
-        self,
-        result_type: Type,
-        target: Union[Operation, OpView, Value],
-        *,
-        block_dims: Optional[Sequence[int]] = None,
-        warp_size: Optional[Sequence[int]] = None,
-        sync_after_distribute: Optional[bool] = None,
-        loc=None,
-        ip=None
-    ):
-        ...
-
-    @overload
-    def __init__(
-        self,
-        target: Union[Operation, OpView, Value],
-        *,
-        block_dims: Optional[Sequence[int]] = None,
-        warp_size: Optional[Sequence[int]] = None,
-        sync_after_distribute: Optional[bool] = None,
-        loc=None,
-        ip=None
-    ):
-        ...
-
-    def __init__(
-        self,
-        result_type_or_target: Union[Operation, OpView, Value, Type],
-        target_or_none: Optional[Union[Operation, OpView, Value]] = None,
-        *,
-        block_dims: Optional[Union[Sequence[int], Attribute]] = None,
-        warp_size: Optional[Union[Sequence[int], Attribute]] = None,
-        sync_after_distribute: Optional[bool] = None,
-        loc=None,
-        ip=None
-    ):
-        if isinstance(result_type_or_target, Type):
-            result_type = result_type_or_target
-            target = target_or_none
-        else:
-            result_type = result_type_or_target.type
-            target = result_type_or_target
-        super().__init__(
-            result_type,
-            target,
-            block_dims=block_dims,
-            warp_size=warp_size,
-            sync_after_distribute=sync_after_distribute,
-            loc=loc,
-            ip=ip,
-        )
diff --git a/mlir/python/mlir/dialects/_linalg_ops_ext.py b/mlir/python/mlir/dialects/_linalg_ops_ext.py
deleted file mode 100644
index 3f6d854ca3..0000000000
--- a/mlir/python/mlir/dialects/_linalg_ops_ext.py
+++ /dev/null
@@ -1,47 +0,0 @@
-#  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-#  See https://llvm.org/LICENSE.txt for license information.
-#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-try:
-    from typing import Optional, Sequence, Union
-    from ..ir import *
-    from ._ods_common import get_default_loc_context
-    from .._mlir_libs._mlirDialectsLinalg import fill_builtin_region
-except ImportError as e:
-    raise RuntimeError("Error loading imports from extension module") from e
-
-from ._ods_common import get_op_result_or_value as _get_op_result_or_value
-
-
-def isa(cls: Type, ty: Type):
-    try:
-        cls(ty)
-        return True
-    except ValueError:
-        return False
-
-
-class StructuredOpMixin:
-    """All structured ops use the same mixin class."""
-
-    def __init__(self, inputs, outputs=(), results=(), loc=None, ip=None):
-        super().__init__(
-            self.build_generic(
-                results=list(results),
-                operands=[list(inputs), list(outputs)],
-                loc=loc,
-                ip=ip,
-            )
-        )
-
-
-def select_opview_mixin(parent_opview_cls):
-    # TODO: This shouldn't be a heuristic: we should have a way to annotate
-    # the OpView to note that it is a structured op.
-    if (
-        "__init__" not in parent_opview_cls.__dict__
-        and hasattr(parent_opview_cls, "inputs")
-        and hasattr(parent_opview_cls, "outputs")
-        and hasattr(parent_opview_cls, "result_tensors")
-    ):
-        return StructuredOpMixin
diff --git a/mlir/python/mlir/dialects/_loop_transform_ops_ext.py b/mlir/python/mlir/dialects/_loop_transform_ops_ext.py
deleted file mode 100644
index 1cdb2b9e77..0000000000
--- a/mlir/python/mlir/dialects/_loop_transform_ops_ext.py
+++ /dev/null
@@ -1,134 +0,0 @@
-#  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-#  See https://llvm.org/LICENSE.txt for license information.
-#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-try:
-    from ..ir import *
-    from ._ods_common import get_op_result_or_value as _get_op_result_or_value
-except ImportError as e:
-    raise RuntimeError("Error loading imports from extension module") from e
-
-from typing import Optional, Union
-
-
-class GetParentForOp:
-    """Extension for GetParentForOp."""
-
-    def __init__(
-        self,
-        result_type: Type,
-        target: Union[Operation, Value],
-        *,
-        num_loops: Optional[int] = None,
-        ip=None,
-        loc=None,
-    ):
-        if num_loops is None:
-            num_loops = 1
-        super().__init__(
-            result_type,
-            _get_op_result_or_value(target),
-            num_loops=num_loops,
-            ip=ip,
-            loc=loc,
-        )
-
-
-class LoopOutlineOp:
-    """Extension for LoopOutlineOp."""
-
-    def __init__(
-        self,
-        function_type: Type,
-        call_type: Type,
-        target: Union[Operation, Value],
-        *,
-        func_name: Union[str, StringAttr],
-        ip=None,
-        loc=None,
-    ):
-        super().__init__(
-            function_type,
-            call_type,
-            _get_op_result_or_value(target),
-            func_name=(
-                func_name
-                if isinstance(func_name, StringAttr)
-                else StringAttr.get(func_name)
-            ),
-            ip=ip,
-            loc=loc,
-        )
-
-
-class LoopPeelOp:
-    """Extension for LoopPeelOp."""
-
-    def __init__(
-        self,
-        main_loop_type: Type,
-        remainder_loop_type: Type,
-        target: Union[Operation, Value],
-        *,
-        fail_if_already_divisible: Union[bool, BoolAttr] = False,
-        ip=None,
-        loc=None,
-    ):
-        super().__init__(
-            main_loop_type,
-            remainder_loop_type,
-            _get_op_result_or_value(target),
-            fail_if_already_divisible=(
-                fail_if_already_divisible
-                if isinstance(fail_if_already_divisible, BoolAttr)
-                else BoolAttr.get(fail_if_already_divisible)
-            ),
-            ip=ip,
-            loc=loc,
-        )
-
-
-class LoopPipelineOp:
-    """Extension for LoopPipelineOp."""
-
-    def __init__(
-        self,
-        result_type: Type,
-        target: Union[Operation, Value],
-        *,
-        iteration_interval: Optional[Union[int, IntegerAttr]] = None,
-        read_latency: Optional[Union[int, IntegerAttr]] = None,
-        ip=None,
-        loc=None,
-    ):
-        if iteration_interval is None:
-            iteration_interval = 1
-        if read_latency is None:
-            read_latency = 10
-        super().__init__(
-            result_type,
-            _get_op_result_or_value(target),
-            iteration_interval=iteration_interval,
-            read_latency=read_latency,
-            ip=ip,
-            loc=loc,
-        )
-
-
-class LoopUnrollOp:
-    """Extension for LoopUnrollOp."""
-
-    def __init__(
-        self,
-        target: Union[Operation, Value],
-        *,
-        factor: Union[int, IntegerAttr],
-        ip=None,
-        loc=None,
-    ):
-        super().__init__(
-            _get_op_result_or_value(target),
-            factor=factor,
-            ip=ip,
-            loc=loc,
-        )
diff --git a/mlir/python/mlir/dialects/_memref_ops_ext.py b/mlir/python/mlir/dialects/_memref_ops_ext.py
deleted file mode 100644
index 825f1a0a7a..0000000000
--- a/mlir/python/mlir/dialects/_memref_ops_ext.py
+++ /dev/null
@@ -1,36 +0,0 @@
-#  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-#  See https://llvm.org/LICENSE.txt for license information.
-#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-try:
-    from ..ir import *
-    from ._ods_common import get_op_result_or_value as _get_op_result_or_value
-    from ._ods_common import get_op_results_or_values as _get_op_results_or_values
-except ImportError as e:
-    raise RuntimeError("Error loading imports from extension module") from e
-
-from typing import Optional, Sequence, Union
-
-
-class LoadOp:
-    """Specialization for the MemRef load operation."""
-
-    def __init__(
-        self,
-        memref: Union[Operation, OpView, Value],
-        indices: Optional[Union[Operation, OpView, Sequence[Value]]] = None,
-        *,
-        loc=None,
-        ip=None
-    ):
-        """Creates a memref load operation.
-
-        Args:
-          memref: the buffer to load from.
-          indices: the list of subscripts, may be empty for zero-dimensional
-            buffers.
-          loc: user-visible location of the operation.
-          ip: insertion point.
-        """
-        indices_resolved = [] if indices is None else _get_op_results_or_values(indices)
-        super().__init__(memref, indices_resolved, loc=loc, ip=ip)
diff --git a/mlir/python/mlir/dialects/_memref_transform_ops_ext.py b/mlir/python/mlir/dialects/_memref_transform_ops_ext.py
deleted file mode 100644
index 1cc00bdcbf..0000000000
--- a/mlir/python/mlir/dialects/_memref_transform_ops_ext.py
+++ /dev/null
@@ -1,114 +0,0 @@
-#  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-#  See https://llvm.org/LICENSE.txt for license information.
-#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-try:
-    from ..ir import *
-    from ..dialects import transform
-except ImportError as e:
-    raise RuntimeError("Error loading imports from extension module") from e
-
-from typing import Optional, overload, Union
-
-
-class MemRefAllocaToGlobalOp:
-    """Specialization for MemRefAllocaToGlobalOp class."""
-
-    @overload
-    def __init__(
-        self,
-        get_global_type: Type,
-        global_type: Type,
-        alloca: Union[Operation, OpView, Value],
-        *,
-        loc=None,
-        ip=None
-    ):
-        ...
-
-    @overload
-    def __init__(self, alloca: Union[Operation, OpView, Value], *, loc=None, ip=None):
-        ...
-
-    def __init__(
-        self,
-        get_global_type_or_alloca: Union[Operation, OpView, Type, Value],
-        global_type_or_none: Optional[Type] = None,
-        alloca_or_none: Optional[Union[Operation, OpView, Value]] = None,
-        *,
-        loc=None,
-        ip=None
-    ):
-        if isinstance(get_global_type_or_alloca, Type):
-            get_global_type = get_global_type_or_alloca
-            global_type = global_type_or_none
-            alloca = alloca_or_none
-        else:
-            get_global_type = transform.AnyOpType.get()
-            global_type = transform.AnyOpType.get()
-            alloca = get_global_type_or_alloca
-
-        super().__init__(
-            get_global_type,
-            global_type,
-            alloca,
-            loc=loc,
-            ip=ip,
-        )
-
-
-class MemRefMultiBufferOp:
-    """Specialization for MemRefMultiBufferOp class."""
-
-    @overload
-    def __init__(
-        self,
-        transformed_type: Type,
-        target: Union[Operation, OpView, Value],
-        factor: Union[int, IntegerAttr],
-        *,
-        skip_analysis: Optional[bool] = None,
-        loc=None,
-        ip=None
-    ):
-        ...
-
-    @overload
-    def __init__(
-        self,
-        target: Union[Operation, OpView, Value],
-        factor: Union[int, IntegerAttr],
-        *,
-        skip_analysis: Optional[bool] = None,
-        loc=None,
-        ip=None
-    ):
-        ...
-
-    def __init__(
-        self,
-        transformed_type_or_target: Type,
-        target_or_factor: Union[int, IntegerAttr, Operation, OpView, Value] = None,
-        factor_or_none: Optional[Union[int, IntegerAttr]] = None,
-        *,
-        skip_analysis: Optional[bool] = None,
-        loc=None,
-        ip=None
-    ):
-        if isinstance(transformed_type_or_target, Type):
-            transformed_type = transformed_type_or_target
-            target = target_or_factor
-            factor = factor_or_none
-        else:
-            transformed_type = transform.AnyOpType.get()
-            target = transformed_type_or_target
-            factor = target_or_factor
-
-        super().__init__(
-            transformed_type,
-            target,
-            factor,
-            skip_analysis=skip_analysis,
-            loc=loc,
-            ip=ip,
-        )
diff --git a/mlir/python/mlir/dialects/_ml_program_ops_ext.py b/mlir/python/mlir/dialects/_ml_program_ops_ext.py
deleted file mode 100644
index c84d23c16e..0000000000
--- a/mlir/python/mlir/dialects/_ml_program_ops_ext.py
+++ /dev/null
@@ -1,113 +0,0 @@
-#  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-#  See https://llvm.org/LICENSE.txt for license information.
-#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-try:
-    from typing import Union
-    from ..ir import *
-    from ._ods_common import get_default_loc_context as _get_default_loc_context
-except ImportError as e:
-    raise RuntimeError("Error loading imports from extension module") from e
-
-from ._ml_program_ops_gen import *
-
-
-ARGUMENT_ATTRIBUTE_NAME = "arg_attrs"
-RESULT_ATTRIBUTE_NAME = "res_attrs"
-
-
-class FuncOp:
-    """Specialization for the func op class."""
-
-    def __init__(
-        self, name, type, *, visibility=None, body_builder=None, loc=None, ip=None
-    ):
-        """
-        Create a FuncOp with the provided `name`, `type`, and `visibility`.
-        - `name` is a string representing the function name.
-        - `type` is either a FunctionType or a pair of list describing inputs and
-          results.
-        - `visibility` is a string matching `public`, `private`, or `nested`. None
-          implies private visibility.
-        - `body_builder` is an optional callback, when provided a new entry block
-          is created and the callback is invoked with the new op as argument within
-          an InsertionPoint context already set for the block. The callback is
-          expected to insert a terminator in the block.
-        """
-        sym_name = StringAttr.get(str(name))
-
-        # If the type is passed as a tuple, build a FunctionType on the fly.
-        if isinstance(type, tuple):
-            type = FunctionType.get(inputs=type[0], results=type[1])
-
-        type = TypeAttr.get(type)
-        sym_visibility = (
-            StringAttr.get(str(visibility)) if visibility is not None else None
-        )
-        super().__init__(sym_name, type, sym_visibility=sym_visibility, loc=loc, ip=ip)
-        if body_builder:
-            entry_block = self.add_entry_block()
-            with InsertionPoint(entry_block):
-                body_builder(self)
-
-    @property
-    def is_external(self):
-        return len(self.regions[0].blocks) == 0
-
-    @property
-    def body(self):
-        return self.regions[0]
-
-    @property
-    def type(self):
-        return FunctionType(TypeAttr(self.attributes["function_type"]).value)
-
-    @property
-    def visibility(self):
-        return self.attributes["sym_visibility"]
-
-    @property
-    def name(self) -> StringAttr:
-        return StringAttr(self.attributes["sym_name"])
-
-    @property
-    def entry_block(self):
-        if self.is_external:
-            raise IndexError("External function does not have a body")
-        return self.regions[0].blocks[0]
-
-    def add_entry_block(self):
-        """
-        Add an entry block to the function body using the function signature to
-        infer block arguments.
-        Returns the newly created block
-        """
-        if not self.is_external:
-            raise IndexError("The function already has an entry block!")
-        self.body.blocks.append(*self.type.inputs)
-        return self.body.blocks[0]
-
-    @property
-    def arg_attrs(self):
-        return ArrayAttr(self.attributes[ARGUMENT_ATTRIBUTE_NAME])
-
-    @arg_attrs.setter
-    def arg_attrs(self, attribute: Union[ArrayAttr, list]):
-        if isinstance(attribute, ArrayAttr):
-            self.attributes[ARGUMENT_ATTRIBUTE_NAME] = attribute
-        else:
-            self.attributes[ARGUMENT_ATTRIBUTE_NAME] = ArrayAttr.get(
-                attribute, context=self.context
-            )
-
-    @property
-    def arguments(self):
-        return self.entry_block.arguments
-
-    @property
-    def result_attrs(self):
-        return self.attributes[RESULT_ATTRIBUTE_NAME]
-
-    @result_attrs.setter
-    def result_attrs(self, attribute: ArrayAttr):
-        self.attributes[RESULT_ATTRIBUTE_NAME] = attribute
diff --git a/mlir/python/mlir/dialects/_ods_common.py b/mlir/python/mlir/dialects/_ods_common.py
index 895c322813..9cca7d659e 100644
--- a/mlir/python/mlir/dialects/_ods_common.py
+++ b/mlir/python/mlir/dialects/_ods_common.py
@@ -9,7 +9,6 @@
 
 __all__ = [
     "equally_sized_accessor",
-    "extend_opview_class",
     "get_default_loc_context",
     "get_op_result_or_value",
     "get_op_results_or_values",
@@ -18,64 +17,6 @@
 ]
 
 
-def extend_opview_class(ext_module):
-    """Decorator to extend an OpView class from an extension module.
-
-    Extension modules can expose various entry-points:
-      Stand-alone class with the same name as a parent OpView class (i.e.
-      "ReturnOp"). A name-based match is attempted first before falling back
-      to a below mechanism.
-
-      def select_opview_mixin(parent_opview_cls):
-        If defined, allows an appropriate mixin class to be selected dynamically
-        based on the parent OpView class. Should return NotImplemented if a
-        decision is not made.
-
-    Args:
-      ext_module: A module from which to locate extensions. Can be None if not
-        available.
-
-    Returns:
-      A decorator that takes an OpView subclass and further extends it as
-      needed.
-    """
-
-    def class_decorator(parent_opview_cls: type):
-        if ext_module is None:
-            return parent_opview_cls
-        mixin_cls = NotImplemented
-        # First try to resolve by name.
-        try:
-            mixin_cls = getattr(ext_module, parent_opview_cls.__name__)
-        except AttributeError:
-            # Fall back to a select_opview_mixin hook.
-            try:
-                select_mixin = getattr(ext_module, "select_opview_mixin")
-            except AttributeError:
-                pass
-            else:
-                mixin_cls = select_mixin(parent_opview_cls)
-
-        if mixin_cls is NotImplemented or mixin_cls is None:
-            return parent_opview_cls
-
-        # Have a mixin_cls. Create an appropriate subclass.
-        try:
-
-            class LocalOpView(mixin_cls, parent_opview_cls):
-                pass
-
-        except TypeError as e:
-            raise TypeError(
-                f"Could not mixin {mixin_cls} into {parent_opview_cls}"
-            ) from e
-        LocalOpView.__name__ = parent_opview_cls.__name__
-        LocalOpView.__qualname__ = parent_opview_cls.__qualname__
-        return LocalOpView
-
-    return class_decorator
-
-
 def segmented_accessor(elements, raw_segments, idx):
     """
     Returns a slice of elements corresponding to the idx-th segment.
diff --git a/mlir/python/mlir/dialects/_pdl_ops_ext.py b/mlir/python/mlir/dialects/_pdl_ops_ext.py
deleted file mode 100644
index fc9de0b7f7..0000000000
--- a/mlir/python/mlir/dialects/_pdl_ops_ext.py
+++ /dev/null
@@ -1,271 +0,0 @@
-#  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-#  See https://llvm.org/LICENSE.txt for license information.
-#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-try:
-    from ..ir import *
-    from ..dialects import pdl
-except ImportError as e:
-    raise RuntimeError("Error loading imports from extension module") from e
-
-from typing import Union, Optional, Sequence, Mapping
-from ._ods_common import (
-    get_op_result_or_value as _get_value,
-    get_op_results_or_values as _get_values,
-)
-
-
-class ApplyNativeConstraintOp:
-    """Specialization for PDL apply native constraint op class."""
-
-    def __init__(
-        self,
-        name: Union[str, StringAttr],
-        args: Optional[Sequence[Union[OpView, Operation, Value]]] = None,
-        *,
-        loc=None,
-        ip=None,
-    ):
-        if args is None:
-            args = []
-        args = _get_values(args)
-        super().__init__(name, args, loc=loc, ip=ip)
-
-
-class ApplyNativeRewriteOp:
-    """Specialization for PDL apply native rewrite op class."""
-
-    def __init__(
-        self,
-        results: Sequence[Type],
-        name: Union[str, StringAttr],
-        args: Optional[Sequence[Union[OpView, Operation, Value]]] = None,
-        *,
-        loc=None,
-        ip=None,
-    ):
-        if args is None:
-            args = []
-        args = _get_values(args)
-        super().__init__(results, name, args, loc=loc, ip=ip)
-
-
-class AttributeOp:
-    """Specialization for PDL attribute op class."""
-
-    def __init__(
-        self,
-        valueType: Optional[Union[OpView, Operation, Value]] = None,
-        value: Optional[Attribute] = None,
-        *,
-        loc=None,
-        ip=None,
-    ):
-        valueType = valueType if valueType is None else _get_value(valueType)
-        result = pdl.AttributeType.get()
-        super().__init__(result, valueType=valueType, value=value, loc=loc, ip=ip)
-
-
-class EraseOp:
-    """Specialization for PDL erase op class."""
-
-    def __init__(
-        self,
-        operation: Optional[Union[OpView, Operation, Value]] = None,
-        *,
-        loc=None,
-        ip=None,
-    ):
-        operation = _get_value(operation)
-        super().__init__(operation, loc=loc, ip=ip)
-
-
-class OperandOp:
-    """Specialization for PDL operand op class."""
-
-    def __init__(
-        self,
-        type: Optional[Union[OpView, Operation, Value]] = None,
-        *,
-        loc=None,
-        ip=None,
-    ):
-        type = type if type is None else _get_value(type)
-        result = pdl.ValueType.get()
-        super().__init__(result, valueType=type, loc=loc, ip=ip)
-
-
-class OperandsOp:
-    """Specialization for PDL operands op class."""
-
-    def __init__(
-        self,
-        types: Optional[Union[OpView, Operation, Value]] = None,
-        *,
-        loc=None,
-        ip=None,
-    ):
-        types = types if types is None else _get_value(types)
-        result = pdl.RangeType.get(pdl.ValueType.get())
-        super().__init__(result, valueType=types, loc=loc, ip=ip)
-
-
-class OperationOp:
-    """Specialization for PDL operand op class."""
-
-    def __init__(
-        self,
-        name: Optional[Union[str, StringAttr]] = None,
-        args: Optional[Sequence[Union[OpView, Operation, Value]]] = None,
-        attributes: Optional[Mapping[str, Union[OpView, Operation, Value]]] = None,
-        types: Optional[Sequence[Union[OpView, Operation, Value]]] = None,
-        *,
-        loc=None,
-        ip=None,
-    ):
-        if types is None:
-            types = []
-        if attributes is None:
-            attributes = {}
-        if args is None:
-            args = []
-        args = _get_values(args)
-        attrNames = []
-        attrValues = []
-        for attrName, attrValue in attributes.items():
-            attrNames.append(StringAttr.get(attrName))
-            attrValues.append(_get_value(attrValue))
-        attrNames = ArrayAttr.get(attrNames)
-        types = _get_values(types)
-        result = pdl.OperationType.get()
-        super().__init__(
-            result, args, attrValues, attrNames, types, opName=name, loc=loc, ip=ip
-        )
-
-
-class PatternOp:
-    """Specialization for PDL pattern op class."""
-
-    def __init__(
-        self,
-        benefit: Union[IntegerAttr, int],
-        name: Optional[Union[StringAttr, str]] = None,
-        *,
-        loc=None,
-        ip=None,
-    ):
-        """Creates an PDL `pattern` operation."""
-        super().__init__(benefit, sym_name=name, loc=loc, ip=ip)
-        self.regions[0].blocks.append()
-
-    @property
-    def body(self):
-        """Return the body (block) of the pattern."""
-        return self.regions[0].blocks[0]
-
-
-class ReplaceOp:
-    """Specialization for PDL replace op class."""
-
-    def __init__(
-        self,
-        op: Union[OpView, Operation, Value],
-        *,
-        with_op: Optional[Union[OpView, Operation, Value]] = None,
-        with_values: Optional[Sequence[Union[OpView, Operation, Value]]] = None,
-        loc=None,
-        ip=None,
-    ):
-        if with_values is None:
-            with_values = []
-        op = _get_value(op)
-        with_op = with_op if with_op is None else _get_value(with_op)
-        with_values = _get_values(with_values)
-        super().__init__(op, with_values, replOperation=with_op, loc=loc, ip=ip)
-
-
-class ResultOp:
-    """Specialization for PDL result op class."""
-
-    def __init__(
-        self,
-        parent: Union[OpView, Operation, Value],
-        index: Union[IntegerAttr, int],
-        *,
-        loc=None,
-        ip=None,
-    ):
-        parent = _get_value(parent)
-        result = pdl.ValueType.get()
-        super().__init__(result, parent, index, loc=loc, ip=ip)
-
-
-class ResultsOp:
-    """Specialization for PDL results op class."""
-
-    def __init__(
-        self,
-        result: Type,
-        parent: Union[OpView, Operation, Value],
-        index: Optional[Union[IntegerAttr, int]] = None,
-        *,
-        loc=None,
-        ip=None,
-    ):
-        parent = _get_value(parent)
-        super().__init__(result, parent, index=index, loc=loc, ip=ip)
-
-
-class RewriteOp:
-    """Specialization for PDL rewrite op class."""
-
-    def __init__(
-        self,
-        root: Optional[Union[OpView, Operation, Value]] = None,
-        name: Optional[Union[StringAttr, str]] = None,
-        args: Optional[Sequence[Union[OpView, Operation, Value]]] = None,
-        *,
-        loc=None,
-        ip=None,
-    ):
-        if args is None:
-            args = []
-        root = root if root is None else _get_value(root)
-        args = _get_values(args)
-        super().__init__(args, root=root, name=name, loc=loc, ip=ip)
-
-    def add_body(self):
-        """Add body (block) to the rewrite."""
-        self.regions[0].blocks.append()
-        return self.body
-
-    @property
-    def body(self):
-        """Return the body (block) of the rewrite."""
-        return self.regions[0].blocks[0]
-
-
-class TypeOp:
-    """Specialization for PDL type op class."""
-
-    def __init__(
-        self, constantType: Optional[Union[TypeAttr, Type]] = None, *, loc=None, ip=None
-    ):
-        result = pdl.TypeType.get()
-        super().__init__(result, constantType=constantType, loc=loc, ip=ip)
-
-
-class TypesOp:
-    """Specialization for PDL types op class."""
-
-    def __init__(
-        self,
-        constantTypes: Optional[Sequence[Union[TypeAttr, Type]]] = None,
-        *,
-        loc=None,
-        ip=None,
-    ):
-        if constantTypes is None:
-            constantTypes = []
-        result = pdl.RangeType.get(pdl.TypeType.get())
-        super().__init__(result, constantTypes=constantTypes, loc=loc, ip=ip)
diff --git a/mlir/python/mlir/dialects/_scf_ops_ext.py b/mlir/python/mlir/dialects/_scf_ops_ext.py
deleted file mode 100644
index 89cc8a1989..0000000000
--- a/mlir/python/mlir/dialects/_scf_ops_ext.py
+++ /dev/null
@@ -1,107 +0,0 @@
-#  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-#  See https://llvm.org/LICENSE.txt for license information.
-#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-try:
-    from ..ir import *
-except ImportError as e:
-    raise RuntimeError("Error loading imports from extension module") from e
-
-from typing import Optional, Sequence, Union
-
-from ._ods_common import (
-    get_op_result_or_value as _get_op_result_or_value,
-    get_op_results_or_values as _get_op_results_or_values,
-)
-
-
-class ForOp:
-    """Specialization for the SCF for op class."""
-
-    def __init__(
-        self,
-        lower_bound,
-        upper_bound,
-        step,
-        iter_args: Optional[Union[Operation, OpView, Sequence[Value]]] = None,
-        *,
-        loc=None,
-        ip=None,
-    ):
-        """Creates an SCF `for` operation.
-
-        - `lower_bound` is the value to use as lower bound of the loop.
-        - `upper_bound` is the value to use as upper bound of the loop.
-        - `step` is the value to use as loop step.
-        - `iter_args` is a list of additional loop-carried arguments or an operation
-          producing them as results.
-        """
-        if iter_args is None:
-            iter_args = []
-        iter_args = _get_op_results_or_values(iter_args)
-
-        results = [arg.type for arg in iter_args]
-        super().__init__(
-            self.build_generic(
-                regions=1,
-                results=results,
-                operands=[
-                    _get_op_result_or_value(o) for o in [lower_bound, upper_bound, step]
-                ]
-                + list(iter_args),
-                loc=loc,
-                ip=ip,
-            )
-        )
-        self.regions[0].blocks.append(self.operands[0].type, *results)
-
-    @property
-    def body(self):
-        """Returns the body (block) of the loop."""
-        return self.regions[0].blocks[0]
-
-    @property
-    def induction_variable(self):
-        """Returns the induction variable of the loop."""
-        return self.body.arguments[0]
-
-    @property
-    def inner_iter_args(self):
-        """Returns the loop-carried arguments usable within the loop.
-
-        To obtain the loop-carried operands, use `iter_args`.
-        """
-        return self.body.arguments[1:]
-
-
-class IfOp:
-    """Specialization for the SCF if op class."""
-
-    def __init__(self, cond, results_=[], *, hasElse=False, loc=None, ip=None):
-        """Creates an SCF `if` operation.
-
-        - `cond` is a MLIR value of 'i1' type to determine which regions of code will be executed.
-        - `hasElse` determines whether the if operation has the else branch.
-        """
-        operands = []
-        operands.append(cond)
-        results = []
-        results.extend(results_)
-        super().__init__(
-            self.build_generic(
-                regions=2, results=results, operands=operands, loc=loc, ip=ip
-            )
-        )
-        self.regions[0].blocks.append(*[])
-        if hasElse:
-            self.regions[1].blocks.append(*[])
-
-    @property
-    def then_block(self):
-        """Returns the then block of the if operation."""
-        return self.regions[0].blocks[0]
-
-    @property
-    def else_block(self):
-        """Returns the else block of the if operation."""
-        return self.regions[1].blocks[0]
diff --git a/mlir/python/mlir/dialects/_structured_transform_ops_ext.py b/mlir/python/mlir/dialects/_structured_transform_ops_ext.py
deleted file mode 100644
index 3757a3d3b4..0000000000
--- a/mlir/python/mlir/dialects/_structured_transform_ops_ext.py
+++ /dev/null
@@ -1,759 +0,0 @@
-#  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-#  See https://llvm.org/LICENSE.txt for license information.
-#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-try:
-    from ..ir import *
-    from ..dialects import transform
-except ImportError as e:
-    raise RuntimeError("Error loading imports from extension module") from e
-
-from typing import List, Optional, Sequence, Tuple, Union, overload
-
-StaticIntLike = Union[int, IntegerAttr]
-ValueLike = Union[Operation, OpView, Value]
-MixedInt = Union[StaticIntLike, ValueLike]
-
-IntOrAttrList = Sequence[Union[IntegerAttr, int]]
-OptionalIntList = Optional[Union[ArrayAttr, IntOrAttrList]]
-
-BoolOrAttrList = Sequence[Union[BoolAttr, bool]]
-OptionalBoolList = Optional[Union[ArrayAttr, BoolOrAttrList]]
-
-MixedValues = Union[Sequence[Union[StaticIntLike, ValueLike]], ArrayAttr, ValueLike]
-
-DynamicIndexList = Sequence[Union[MixedInt, Sequence[MixedInt]]]
-
-
-def _dispatch_dynamic_index_list(
-    indices: Union[DynamicIndexList, ArrayAttr],
-) -> Tuple[List[ValueLike], Union[List[int], ArrayAttr], List[bool]]:
-    """Dispatches a list of indices to the appropriate form.
-
-    This is similar to the custom `DynamicIndexList` directive upstream:
-    provided indices may be in the form of dynamic SSA values or static values,
-    and they may be scalable (i.e., as a singleton list) or not. This function
-    dispatches each index into its respective form. It also extracts the SSA
-    values and static indices from various similar structures, respectively.
-    """
-    dynamic_indices = []
-    static_indices = [ShapedType.get_dynamic_size()] * len(indices)
-    scalable_indices = [False] * len(indices)
-
-    # ArrayAttr: Extract index values.
-    if isinstance(indices, ArrayAttr):
-        indices = [idx for idx in indices]
-
-    def process_nonscalable_index(i, index):
-        """Processes any form of non-scalable index.
-
-        Returns False if the given index was scalable and thus remains
-        unprocessed; True otherwise.
-        """
-        if isinstance(index, int):
-            static_indices[i] = index
-        elif isinstance(index, IntegerAttr):
-            static_indices[i] = index.value  # pytype: disable=attribute-error
-        elif isinstance(index, (Operation, Value, OpView)):
-            dynamic_indices.append(index)
-        else:
-            return False
-        return True
-
-    # Process each index at a time.
-    for i, index in enumerate(indices):
-        if not process_nonscalable_index(i, index):
-            # If it wasn't processed, it must be a scalable index, which is
-            # provided as a Sequence of one value, so extract and process that.
-            scalable_indices[i] = True
-            assert len(index) == 1
-            ret = process_nonscalable_index(i, index[0])
-            assert ret
-
-    return dynamic_indices, static_indices, scalable_indices
-
-
-# Dispatches `MixedValues` that all represents integers in various forms into
-# the following three categories:
-#   - `dynamic_values`: a list of `Value`s, potentially from op results;
-#   - `packed_values`: a value handle, potentially from an op result, associated
-#                      to one or more payload operations of integer type;
-#   - `static_values`: an `ArrayAttr` of `i64`s with static values, from Python
-#                      `int`s, from `IntegerAttr`s, or from an `ArrayAttr`.
-# The input is in the form for `packed_values`, only that result is set and the
-# other two are empty. Otherwise, the input can be a mix of the other two forms,
-# and for each dynamic value, a special value is added to the `static_values`.
-def _dispatch_mixed_values(
-    values: MixedValues,
-) -> Tuple[List[Value], Union[Operation, Value, OpView], DenseI64ArrayAttr]:
-    dynamic_values = []
-    packed_values = None
-    static_values = None
-    if isinstance(values, ArrayAttr):
-        static_values = values
-    elif isinstance(values, (Operation, Value, OpView)):
-        packed_values = values
-    else:
-        static_values = []
-        for size in values or []:
-            if isinstance(size, int):
-                static_values.append(size)
-            else:
-                static_values.append(ShapedType.get_dynamic_size())
-                dynamic_values.append(size)
-        static_values = DenseI64ArrayAttr.get(static_values)
-
-    return (dynamic_values, packed_values, static_values)
-
-
-def _get_value_or_attribute_value(
-    value_or_attr: Union[any, Attribute, ArrayAttr]
-) -> any:
-    if isinstance(value_or_attr, Attribute) and hasattr(value_or_attr, "value"):
-        return value_or_attr.value
-    if isinstance(value_or_attr, ArrayAttr):
-        return _get_value_list(value_or_attr)
-    return value_or_attr
-
-
-def _get_value_list(
-    sequence_or_array_attr: Union[Sequence[any], ArrayAttr]
-) -> Sequence[any]:
-    return [_get_value_or_attribute_value(v) for v in sequence_or_array_attr]
-
-
-def _get_int_array_attr(values: Optional[Union[ArrayAttr, IntOrAttrList]]) -> ArrayAttr:
-    if values is None:
-        return None
-
-    # Turn into a Python list of Python ints.
-    values = _get_value_list(values)
-
-    # Make an ArrayAttr of IntegerAttrs out of it.
-    return ArrayAttr.get(
-        [IntegerAttr.get(IntegerType.get_signless(64), v) for v in values]
-    )
-
-
-def _get_int_array_array_attr(
-    values: Optional[Union[ArrayAttr, Sequence[Union[ArrayAttr, IntOrAttrList]]]]
-) -> ArrayAttr:
-    """Creates an ArrayAttr of ArrayAttrs of IntegerAttrs.
-
-    The input has to be a collection of collection of integers, where any
-    Python Sequence and ArrayAttr are admissible collections and Python ints and
-    any IntegerAttr are admissible integers. Both levels of collections are
-    turned into ArrayAttr; the inner level is turned into IntegerAttrs of i64s.
-    If the input is None, an empty ArrayAttr is returned.
-    """
-    if values is None:
-        return None
-
-    # Make sure the outer level is a list.
-    values = _get_value_list(values)
-
-    # The inner level is now either invalid or a mixed sequence of ArrayAttrs and
-    # Sequences. Make sure the nested values are all lists.
-    values = [_get_value_list(nested) for nested in values]
-
-    # Turn each nested list into an ArrayAttr.
-    values = [_get_int_array_attr(nested) for nested in values]
-
-    # Turn the outer list into an ArrayAttr.
-    return ArrayAttr.get(values)
-
-
-class BufferizeToAllocationOp:
-    """Specialization for BufferizeToAllocationOp class."""
-
-    def __init__(
-        self,
-        target: Union[Operation, OpView, Value],
-        *,
-        memory_space: Optional[Union[int, str, Attribute]] = None,
-        memcpy_op: Optional[str] = None,
-        alloc_op: Optional[str] = None,
-        bufferize_destination_only: Optional[bool] = None,
-        loc=None,
-        ip=None,
-    ):
-        # No other types are allowed, so hard-code those here.
-        allocated_buffer_type = transform.AnyValueType.get()
-        new_ops_type = transform.AnyOpType.get()
-
-        if isinstance(memory_space, int):
-            memory_space = str(memory_space)
-        if isinstance(memory_space, str):
-            memory_space = Attribute.parse(memory_space)
-
-        super().__init__(
-            allocated_buffer_type,
-            new_ops_type,
-            target,
-            memory_space=memory_space,
-            memcpy_op=memcpy_op,
-            alloc_op=alloc_op,
-            bufferize_destination_only=bufferize_destination_only,
-            loc=loc,
-            ip=ip,
-        )
-
-
-class DecomposeOp:
-    """Specialization for DecomposeOp class."""
-
-    def __init__(self, target: Union[Operation, Value], *, loc=None, ip=None):
-        transformed_type = transform.AnyOpType.get()
-        super().__init__(transformed_type, target, loc=loc, ip=ip)
-
-
-class FuseIntoContainingOp:
-    """Specialization for FuseIntoContainingOp class."""
-
-    @overload
-    def __init__(
-        self,
-        fused_op_type: Type,
-        new_containing_op_type: Type,
-        producer_op: Union[Operation, OpView, Value],
-        containing_op: Union[Operation, OpView, Value],
-        *,
-        loc=None,
-        ip=None,
-    ):
-        ...
-
-    @overload
-    def __init__(
-        self,
-        producer_op: Union[Operation, OpView, Value],
-        containing_op: Union[Operation, OpView, Value],
-        *,
-        loc=None,
-        ip=None,
-    ):
-        ...
-
-    def __init__(
-        self,
-        fused_op_type_or_producer_op: Union[Operation, OpView, Type, Value],
-        new_containing_op_type_or_containing_op: Union[Operation, OpView, Type, Value],
-        producer_op_or_none: Optional[Union[Operation, OpView, Value]] = None,
-        containing_op_or_none: Optional[Union[Operation, OpView, Value]] = None,
-        *,
-        loc=None,
-        ip=None,
-    ):
-        if isinstance(fused_op_type_or_producer_op, Type):
-            if not isinstance(new_containing_op_type_or_containing_op, Type):
-                raise TypeError(
-                    "If 'fused_op_type_or_producer_op' is a type, then "
-                    "'new_containing_op_type_or_containing_op' is expected "
-                    "to be one as well."
-                )
-            fused_op_type = fused_op_type_or_producer_op
-            new_containing_op_type = new_containing_op_type_or_containing_op
-            producer_op = producer_op_or_none
-            containing_op = containing_op_or_none
-        else:
-            fused_op_type = transform.AnyOpType.get()
-            new_containing_op_type = transform.AnyOpType.get()
-            producer_op = fused_op_type_or_producer_op
-            containing_op = new_containing_op_type_or_containing_op
-
-        super().__init__(
-            fused_op_type,
-            new_containing_op_type,
-            producer_op,
-            containing_op,
-            loc=loc,
-            ip=ip,
-        )
-
-
-class GeneralizeOp:
-    """Specialization for GeneralizeOp class."""
-
-    def __init__(self, target: Union[Operation, Value], *, loc=None, ip=None):
-        transformed_type = transform.AnyOpType.get()
-        super().__init__(transformed_type, target, loc=loc, ip=ip)
-
-
-class InterchangeOp:
-    """Specialization for InterchangeOp class."""
-
-    def __init__(
-        self,
-        target: Union[Operation, Value],
-        *,
-        iterator_interchange: OptionalIntList = None,
-        loc=None,
-        ip=None,
-    ):
-        transformed_type = transform.AnyOpType.get()
-        super().__init__(
-            transformed_type,
-            target,
-            iterator_interchange=iterator_interchange,
-            loc=loc,
-            ip=ip,
-        )
-
-
-class MapCopyToThreadsOp:
-    """Specialization for MapCopyToThreadsOp class."""
-
-    @overload
-    def __init__(
-        self,
-        forall_op_type: Type,
-        tiled_op_type: Type,
-        target: Union[Operation, OpView, Value],
-        *,
-        total_num_threads: Union[int, IntegerAttr],
-        desired_bit_alignment: Union[int, IntegerAttr],
-        loc=None,
-        ip=None,
-    ):
-        ...
-
-    @overload
-    def __init__(
-        self,
-        target: Union[Operation, OpView, Value],
-        *,
-        total_num_threads: Union[int, IntegerAttr],
-        desired_bit_alignment: Union[int, IntegerAttr],
-        loc=None,
-        ip=None,
-    ):
-        ...
-
-    def __init__(
-        self,
-        forall_op_type_or_target: Union[Operation, OpView, Type, Value],
-        tiled_op_type_or_none: Optional[Type] = None,
-        target_or_none: Optional[Union[Operation, OpView, Value]] = None,
-        *,
-        total_num_threads: Union[int, IntegerAttr],
-        desired_bit_alignment: Union[int, IntegerAttr],
-        loc=None,
-        ip=None,
-    ):
-        if isinstance(forall_op_type_or_target, Type):
-            forall_op_type = forall_op_type_or_target
-            tiled_op_type = tiled_op_type_or_none
-            target = target_or_none
-        else:
-            forall_op_type = transform.AnyOpType.get()
-            tiled_op_type = transform.AnyOpType.get()
-            target = forall_op_type_or_target
-
-        super().__init__(
-            forall_op_type,
-            tiled_op_type,
-            target,
-            total_num_threads=total_num_threads,
-            desired_bit_alignment=desired_bit_alignment,
-            loc=loc,
-            ip=ip,
-        )
-
-
-class VectorizeOp:
-    """Specialization for VectorizeOp class."""
-
-    def __init__(
-        self,
-        target: Union[Operation, OpView, Value],
-        vector_sizes: Optional[Union[DynamicIndexList, ArrayAttr]] = None,
-        *,
-        vectorize_nd_extract: Optional[bool] = None,
-        scalable_sizes: OptionalBoolList = None,
-        static_vector_sizes: OptionalIntList = None,
-        loc=None,
-        ip=None,
-    ):
-        if (
-            scalable_sizes is None
-            and static_vector_sizes is None
-            and vector_sizes is None
-        ):
-            dynamic_vector_sizes = []
-        elif scalable_sizes is None and static_vector_sizes is None:
-            (
-                dynamic_vector_sizes,
-                static_vector_sizes,
-                scalable_sizes,
-            ) = _dispatch_dynamic_index_list(vector_sizes)
-        elif scalable_sizes is None or static_vector_sizes is None:
-            raise TypeError(
-                "'scalable_sizes' and 'static_vector_sizes' must either both "
-                "be given explicitly or both be given as part of 'vector_sizes'."
-            )
-        else:
-            dynamic_vector_sizes = vector_sizes
-
-        super().__init__(
-            target,
-            vector_sizes=dynamic_vector_sizes,
-            static_vector_sizes=static_vector_sizes,
-            scalable_sizes=scalable_sizes,
-            vectorize_nd_extract=vectorize_nd_extract,
-            loc=loc,
-            ip=ip,
-        )
-
-
-class MatchOp:
-    """Specialization for MatchOp class."""
-
-    @overload
-    @classmethod
-    def match_op_names(
-        cls,
-        target: Union[Operation, Value],
-        names: Union[str, Sequence[str]],
-        *,
-        loc=None,
-        ip=None,
-    ):
-        ...
-
-    @overload
-    @classmethod
-    def match_op_names(
-        cls,
-        result_type: Type,
-        target: Union[Operation, Value],
-        names: Union[str, Sequence[str]],
-        *,
-        loc=None,
-        ip=None,
-    ):
-        ...
-
-    @classmethod
-    def match_op_names(
-        cls,
-        result_type_or_target: Union[Type, Operation, Value],
-        target_or_names: Union[Operation, Value, Sequence[str], str],
-        names_or_none: Optional[Union[Sequence[str], str]] = None,
-        *,
-        loc=None,
-        ip=None,
-    ):
-        if isinstance(result_type_or_target, Type):
-            result_type = result_type_or_target
-            target = target_or_names
-            names = names_or_none
-        else:
-            result_type = transform.AnyOpType.get()
-            target = result_type_or_target
-            names = target_or_names
-
-        if isinstance(names, str):
-            names = [names]
-
-        return cls(
-            result_type,
-            target,
-            ops=ArrayAttr.get(list(map(lambda s: StringAttr.get(s), names))),
-            loc=loc,
-            ip=ip,
-        )
-
-
-class MultiTileSizesOp:
-    """Specialization for MultiTileSizesOp class."""
-
-    def __init__(
-        self,
-        result_type: Type,
-        target: Union[Operation, Value],
-        *,
-        dimension: Union[int, IntegerAttr],
-        target_size: Union[int, IntegerAttr],
-        divisor: Optional[Optional[Union[int, IntegerAttr]]] = None,
-        loc=None,
-        ip=None,
-    ):
-        super().__init__(
-            result_type,
-            result_type,
-            result_type,
-            target,
-            dimension=dimension,
-            target_size=target_size,
-            divisor=divisor,
-            loc=loc,
-            ip=ip,
-        )
-
-
-class PadOp:
-    """Specialization for PadOp class."""
-
-    def __init__(
-        self,
-        target: Union[Operation, OpView, Value],
-        *,
-        padding_values: Optional[Union[ArrayAttr, Sequence[Attribute]]] = None,
-        padding_dimensions: OptionalIntList = None,
-        pad_to_multiple_of: OptionalIntList = None,
-        pack_paddings: OptionalIntList = None,
-        transpose_paddings: Optional[
-            Union[ArrayAttr, Sequence[Union[ArrayAttr, IntOrAttrList]]]
-        ] = None,
-        copy_back_op: Optional[Union[str, StringAttr]] = None,
-        loc=None,
-        ip=None,
-    ):
-        transpose_paddings = _get_int_array_array_attr(transpose_paddings)
-
-        any_op_type = transform.AnyOpType.get()
-        super().__init__(
-            any_op_type,
-            any_op_type,
-            any_op_type,
-            target,
-            padding_values=padding_values,
-            padding_dimensions=padding_dimensions,
-            pad_to_multiple_of=pad_to_multiple_of,
-            pack_paddings=pack_paddings,
-            transpose_paddings=transpose_paddings,
-            copy_back_op=copy_back_op,
-            loc=loc,
-            ip=ip,
-        )
-
-
-class ScalarizeOp:
-    """Specialization for ScalarizeOp class."""
-
-    def __init__(self, target: Union[Operation, Value], *, loc=None, ip=None):
-        result_type = transform.AnyOpType.get()
-        super().__init__(result_type, target, loc=loc, ip=ip)
-
-
-class SplitOp:
-    """Specialization for SplitOp class."""
-
-    def __init__(
-        self,
-        target: Union[Operation, Value],
-        dimension: Union[int, Attribute],
-        split_point: Union[int, Operation, Value, Attribute],
-        *,
-        loc=None,
-        ip=None,
-    ):
-        if isinstance(split_point, int):
-            static_split_point = split_point
-            dynamic_split_point = None
-        else:
-            static_split_point = ShapedType.get_dynamic_size()
-            dynamic_split_point = split_point
-
-        super().__init__(
-            target.type,
-            target.type,
-            target,
-            dimension=dimension,
-            static_split_point=static_split_point,
-            dynamic_split_point=dynamic_split_point,
-            loc=loc,
-            ip=ip,
-        )
-
-
-class TileUsingForOp:
-    """Specialization for TileUsingForOp class."""
-
-    @overload
-    def __init__(
-        self,
-        loop_types: Union[Type, List[Type]],
-        target: Union[Operation, Value],
-        *,
-        sizes: Optional[Union[DynamicIndexList, ArrayAttr]] = None,
-        interchange: OptionalIntList = None,
-        loc=None,
-        ip=None,
-    ):
-        ...
-
-    @overload
-    def __init__(
-        self,
-        target: Union[Operation, Value, OpView],
-        *,
-        sizes: Optional[Union[DynamicIndexList, ArrayAttr]] = None,
-        interchange: OptionalIntList = None,
-        loc=None,
-        ip=None,
-    ):
-        ...
-
-    def __init__(
-        self,
-        loop_types_or_target: Union[Type, List[Type], Operation, Value],
-        target_or_none: Optional[Union[Operation, Value, OpView]] = None,
-        *,
-        sizes: Optional[Union[DynamicIndexList, ArrayAttr]] = None,
-        interchange: OptionalIntList = None,
-        loc=None,
-        ip=None,
-    ):
-        (
-            dynamic_sizes,
-            static_sizes,
-            scalable_sizes,
-        ) = _dispatch_dynamic_index_list(sizes)
-
-        num_loops = sum(v if v == 0 else 1 for v in static_sizes)
-
-        if isinstance(loop_types_or_target, (Operation, Value, OpView)):
-            loop_types = [transform.AnyOpType.get()] * num_loops
-            target = loop_types_or_target
-            assert (
-                target_or_none is None
-            ), "Cannot construct TileUsingForOp with two targets."
-        else:
-            loop_types = (
-                ([loop_types_or_target] * num_loops)
-                if isinstance(loop_types_or_target, Type)
-                else loop_types_or_target
-            )
-            target = target_or_none
-
-        super().__init__(
-            target.type,
-            loop_types,
-            target,
-            dynamic_sizes=dynamic_sizes,
-            static_sizes=static_sizes,
-            interchange=interchange,
-            scalable_sizes=scalable_sizes,
-            loc=loc,
-            ip=ip,
-        )
-
-
-class TileUsingForallOp:
-    """Specialization for TileUsingForallOp class."""
-
-    @overload
-    def __init__(
-        self,
-        loops_type: Type,
-        tiled_op_type: Type,
-        target: Union[Operation, Value, OpView],
-        *,
-        num_threads: Optional[MixedValues] = None,
-        tile_sizes: MixedValues = None,
-        mapping=None,
-        loc=None,
-        ip=None,
-    ):
-        ...
-
-    @overload
-    def __init__(
-        self,
-        target: Union[Operation, Value, OpView],
-        *,
-        num_threads: Optional[MixedValues] = None,
-        tile_sizes: MixedValues = None,
-        mapping=None,
-        loc=None,
-        ip=None,
-    ):
-        ...
-
-    def __init__(
-        self,
-        loops_type_or_target: Union[
-            Type, Union[Operation, Value, OpView]  # loops_type
-        ],  # target
-        tiled_op_type_or_none: Optional[Type] = None,
-        target_or_none: Optional[Union[Operation, Value, OpView]] = None,
-        *,
-        num_threads: MixedValues = None,
-        tile_sizes: MixedValues = None,
-        mapping=None,
-        loc=None,
-        ip=None,
-    ):
-        # `Type` arguments in the front are optional: add default values to front.
-        if isinstance(loops_type_or_target, Type):
-            # First overload: type arguments provided.
-            if not isinstance(tiled_op_type_or_none, Type):
-                raise TypeError(
-                    "If 'loops_type_or_target' is a type, then "
-                    "'tiled_op_type_or_none' is expected to be one as well."
-                )
-            loops_type = loops_type_or_target
-            tiled_op_type = tiled_op_type_or_none
-            target = target_or_none
-        else:
-            # Last overload: type arguments missing.
-            loops_type = transform.AnyOpType.get()
-            tiled_op_type = transform.AnyOpType.get()
-            target = loops_type_or_target
-
-        # Unpack mixed num_threads.
-        (
-            dynamic_num_threads,
-            packed_num_threads,
-            num_threads_attr,
-        ) = _dispatch_mixed_values(num_threads)
-
-        # Unpack mixed tile_sizes.
-        (
-            dynamic_tile_sizes,
-            packed_tile_sizes,
-            tile_sizes_attr,
-        ) = _dispatch_mixed_values(tile_sizes)
-
-        super().__init__(
-            loops_type,
-            tiled_op_type,
-            target=target,
-            tile_sizes=dynamic_tile_sizes,
-            packed_tile_sizes=packed_tile_sizes,
-            static_tile_sizes=tile_sizes_attr,
-            num_threads=dynamic_num_threads,
-            packed_num_threads=packed_num_threads,
-            static_num_threads=num_threads_attr,
-            mapping=mapping,
-            loc=loc,
-            ip=ip,
-        )
-
-
-class VectorizeChildrenAndApplyPatternsOp:
-    """Specialization for VectorizeChildrenAndApplyPatternsOp class."""
-
-    def __init__(
-        self,
-        target: Union[Operation, Value],
-        *,
-        disable_multi_reduction_to_contract_patterns: bool = False,
-        disable_transfer_permutation_map_lowering_patterns: bool = False,
-        vectorize_nd_extract: bool = False,
-        vectorize_padding: bool = False,
-        loc=None,
-        ip=None,
-    ):
-        transformed_type = transform.AnyOpType.get()
-        super().__init__(
-            transformed_type,
-            target,
-            disable_multi_reduction_to_contract_patterns=disable_multi_reduction_to_contract_patterns,
-            disable_transfer_permutation_map_lowering_patterns=disable_transfer_permutation_map_lowering_patterns,
-            vectorize_nd_extract=vectorize_nd_extract,
-            vectorize_padding=vectorize_padding,
-            loc=loc,
-            ip=ip,
-        )
diff --git a/mlir/python/mlir/dialects/_tensor_ops_ext.py b/mlir/python/mlir/dialects/_tensor_ops_ext.py
deleted file mode 100644
index 09b9ec68db..0000000000
--- a/mlir/python/mlir/dialects/_tensor_ops_ext.py
+++ /dev/null
@@ -1,44 +0,0 @@
-#  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-#  See https://llvm.org/LICENSE.txt for license information.
-#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-try:
-    from ..ir import *
-except ImportError as e:
-    raise RuntimeError("Error loading imports from extension module") from e
-
-from typing import Any, Optional, Sequence, Union
-from ._ods_common import (
-    get_op_result_or_value as _get_op_result_or_value,
-    get_op_results_or_values as _get_op_results_or_values,
-)
-
-
-class EmptyOp:
-    """Extends the tensor.empty op."""
-
-    def __init__(
-        self,
-        sizes: Sequence[Union[int, Value]],
-        element_type: Type,
-        *,
-        loc=None,
-        ip=None
-    ):
-        """Constructs an `empty` with mixed static/dynamic sizes."""
-        # TODO: Refactor the EmptyOp to take an element type attribute and
-        # then use normal result type inference, unifying the Python and C++ side
-        # with a standard mechanism (versus stashing that in builders).
-        dynamic_sizes = []
-        static_sizes = []
-        for s in sizes:
-            if isinstance(s, int):
-                static_sizes.append(s)
-            else:
-                static_sizes.append(ShapedType.get_dynamic_size())
-                dynamic_sizes.append(s)
-        result_type = RankedTensorType.get(static_sizes, element_type)
-        op = self.build_generic(
-            results=[result_type], operands=dynamic_sizes, attributes={}, loc=loc, ip=ip
-        )
-        OpView.__init__(self, op)
diff --git a/mlir/python/mlir/dialects/_tensor_transform_ops_ext.py b/mlir/python/mlir/dialects/_tensor_transform_ops_ext.py
deleted file mode 100644
index 996093fbc9..0000000000
--- a/mlir/python/mlir/dialects/_tensor_transform_ops_ext.py
+++ /dev/null
@@ -1,64 +0,0 @@
-#  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-#  See https://llvm.org/LICENSE.txt for license information.
-#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-try:
-    from ..ir import *
-    from ..dialects import transform
-except ImportError as e:
-    raise RuntimeError("Error loading imports from extension module") from e
-
-from typing import Optional, overload, Union
-
-
-class MakeLoopIndependentOp:
-    """Specialization for MakeLoopIndependentOp class."""
-
-    @overload
-    def __init__(
-        self,
-        transformed_type: Type,
-        target: Union[Operation, OpView, Value],
-        num_loops: Union[int, IntegerAttr],
-        *,
-        loc=None,
-        ip=None
-    ):
-        ...
-
-    @overload
-    def __init__(
-        self,
-        target: Union[Operation, OpView, Value],
-        num_loops: Union[int, IntegerAttr],
-        *,
-        loc=None,
-        ip=None
-    ):
-        ...
-
-    def __init__(
-        self,
-        transformed_type_or_target: Type,
-        target_or_num_loops: Union[int, IntegerAttr, Operation, OpView, Value] = None,
-        num_loops_or_none: Optional[Union[int, IntegerAttr]] = None,
-        *,
-        loc=None,
-        ip=None
-    ):
-        if isinstance(transformed_type_or_target, Type):
-            transformed_type = transformed_type_or_target
-            target = target_or_num_loops
-            num_loops = num_loops_or_none
-        else:
-            transformed_type = transform.AnyOpType.get()
-            target = transformed_type_or_target
-            num_loops = target_or_num_loops
-
-        super().__init__(
-            transformed_type,
-            target,
-            num_loops,
-            loc=loc,
-            ip=ip,
-        )
diff --git a/mlir/python/mlir/dialects/_transform_ops_ext.py b/mlir/python/mlir/dialects/_transform_ops_ext.py
deleted file mode 100644
index b1e7b89253..0000000000
--- a/mlir/python/mlir/dialects/_transform_ops_ext.py
+++ /dev/null
@@ -1,176 +0,0 @@
-#  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-#  See https://llvm.org/LICENSE.txt for license information.
-#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-try:
-    from ..ir import *
-    from ._ods_common import (
-        get_op_result_or_value as _get_op_result_or_value,
-        get_op_results_or_values as _get_op_results_or_values,
-    )
-except ImportError as e:
-    raise RuntimeError("Error loading imports from extension module") from e
-
-from typing import Optional, Sequence, Union
-
-
-class CastOp:
-    def __init__(
-        self,
-        result_type: Type,
-        target: Union[Operation, Value],
-        *,
-        loc=None,
-        ip=None,
-    ):
-        super().__init__(result_type, _get_op_result_or_value(target), loc=loc, ip=ip)
-
-
-class ApplyPatternsOp:
-    def __init__(
-        self,
-        target: Union[Operation, Value, OpView],
-        *,
-        loc=None,
-        ip=None,
-    ):
-        operands = []
-        operands.append(_get_op_result_or_value(target))
-        super().__init__(
-            self.build_generic(
-                attributes={},
-                results=[],
-                operands=operands,
-                successors=None,
-                regions=None,
-                loc=loc,
-                ip=ip,
-            )
-        )
-        self.regions[0].blocks.append()
-
-    @property
-    def patterns(self) -> Block:
-        return self.regions[0].blocks[0]
-
-
-class testGetParentOp:
-    def __init__(
-        self,
-        result_type: Type,
-        target: Union[Operation, Value],
-        *,
-        isolated_from_above: bool = False,
-        op_name: Optional[str] = None,
-        deduplicate: bool = False,
-        loc=None,
-        ip=None,
-    ):
-        super().__init__(
-            result_type,
-            _get_op_result_or_value(target),
-            isolated_from_above=isolated_from_above,
-            op_name=op_name,
-            deduplicate=deduplicate,
-            loc=loc,
-            ip=ip,
-        )
-
-
-class MergeHandlesOp:
-    def __init__(
-        self,
-        handles: Sequence[Union[Operation, Value]],
-        *,
-        deduplicate: bool = False,
-        loc=None,
-        ip=None,
-    ):
-        super().__init__(
-            [_get_op_result_or_value(h) for h in handles],
-            deduplicate=deduplicate,
-            loc=loc,
-            ip=ip,
-        )
-
-
-class ReplicateOp:
-    def __init__(
-        self,
-        pattern: Union[Operation, Value],
-        handles: Sequence[Union[Operation, Value]],
-        *,
-        loc=None,
-        ip=None,
-    ):
-        super().__init__(
-            [_get_op_result_or_value(h).type for h in handles],
-            _get_op_result_or_value(pattern),
-            [_get_op_result_or_value(h) for h in handles],
-            loc=loc,
-            ip=ip,
-        )
-
-
-class SequenceOp:
-    def __init__(
-        self,
-        failure_propagation_mode,
-        results: Sequence[Type],
-        target: Union[Operation, Value, Type],
-        extra_bindings: Optional[
-            Union[Sequence[Value], Sequence[Type], Operation, OpView]
-        ] = None,
-    ):
-        root = (
-            _get_op_result_or_value(target)
-            if isinstance(target, (Operation, Value))
-            else None
-        )
-        root_type = root.type if not isinstance(target, Type) else target
-
-        if extra_bindings is None:
-            extra_bindings = []
-        if isinstance(extra_bindings, (Operation, OpView)):
-            extra_bindings = _get_op_results_or_values(extra_bindings)
-
-        extra_binding_types = []
-        if len(extra_bindings) != 0:
-            if isinstance(extra_bindings[0], Type):
-                extra_binding_types = extra_bindings
-                extra_bindings = []
-            else:
-                extra_binding_types = [v.type for v in extra_bindings]
-
-        super().__init__(
-            results_=results,
-            failure_propagation_mode=failure_propagation_mode,
-            root=root,
-            extra_bindings=extra_bindings,
-        )
-        self.regions[0].blocks.append(*tuple([root_type] + extra_binding_types))
-
-    @property
-    def body(self) -> Block:
-        return self.regions[0].blocks[0]
-
-    @property
-    def bodyTarget(self) -> Value:
-        return self.body.arguments[0]
-
-    @property
-    def bodyExtraArgs(self) -> BlockArgumentList:
-        return self.body.arguments[1:]
-
-
-class YieldOp:
-    def __init__(
-        self,
-        operands: Optional[Union[Operation, Sequence[Value]]] = None,
-        *,
-        loc=None,
-        ip=None,
-    ):
-        if operands is None:
-            operands = []
-        super().__init__(_get_op_results_or_values(operands), loc=loc, ip=ip)
diff --git a/mlir/python/mlir/dialects/_transform_pdl_extension_ops_ext.py b/mlir/python/mlir/dialects/_transform_pdl_extension_ops_ext.py
deleted file mode 100644
index c4e4b4b425..0000000000
--- a/mlir/python/mlir/dialects/_transform_pdl_extension_ops_ext.py
+++ /dev/null
@@ -1,55 +0,0 @@
-#  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-#  See https://llvm.org/LICENSE.txt for license information.
-#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-try:
-  from ..ir import *
-  from ._ods_common import (
-      get_op_result_or_value as _get_op_result_or_value,
-      get_op_results_or_values as _get_op_results_or_values,
-  )
-except ImportError as e:
-  raise RuntimeError("Error loading imports from extension module") from e
-
-from typing import Union
-
-class PDLMatchOp:
-
-  def __init__(
-      self,
-      result_type: Type,
-      target: Union[Operation, Value],
-      pattern_name: Union[Attribute, str],
-      *,
-      loc=None,
-      ip=None,
-  ):
-    super().__init__(
-        result_type,
-        _get_op_result_or_value(target),
-        pattern_name,
-        loc=loc,
-        ip=ip,
-    )
-
-
-class WithPDLPatternsOp:
-
-  def __init__(self,
-               target: Union[Operation, Value, Type],
-               *,
-               loc=None,
-               ip=None):
-    root = _get_op_result_or_value(target) if not isinstance(target,
-                                                             Type) else None
-    root_type = target if isinstance(target, Type) else root.type
-    super().__init__(root=root, loc=loc, ip=ip)
-    self.regions[0].blocks.append(root_type)
-
-  @property
-  def body(self) -> Block:
-    return self.regions[0].blocks[0]
-
-  @property
-  def bodyTarget(self) -> Value:
-    return self.body.arguments[0]
diff --git a/mlir/python/mlir/dialects/affine.py b/mlir/python/mlir/dialects/affine.py
new file mode 100644
index 0000000000..80d3873e19
--- /dev/null
+++ b/mlir/python/mlir/dialects/affine.py
@@ -0,0 +1,5 @@
+#  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+#  See https://llvm.org/LICENSE.txt for license information.
+#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+from ._affine_ops_gen import *
diff --git a/mlir/python/mlir/dialects/arith.py b/mlir/python/mlir/dialects/arith.py
index fb13beb63c..83aca0d58b 100644
--- a/mlir/python/mlir/dialects/arith.py
+++ b/mlir/python/mlir/dialects/arith.py
@@ -3,4 +3,75 @@
 #  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 from ._arith_ops_gen import *
+from ._arith_ops_gen import _Dialect
 from ._arith_enum_gen import *
+
+try:
+    from ..ir import *
+    from ._ods_common import (
+        get_default_loc_context as _get_default_loc_context,
+        _cext as _ods_cext,
+    )
+
+    from typing import Any, List, Union
+except ImportError as e:
+    raise RuntimeError("Error loading imports from extension module") from e
+
+
+def _isa(obj: Any, cls: type):
+    try:
+        cls(obj)
+    except ValueError:
+        return False
+    return True
+
+
+def _is_any_of(obj: Any, classes: List[type]):
+    return any(_isa(obj, cls) for cls in classes)
+
+
+def _is_integer_like_type(type: Type):
+    return _is_any_of(type, [IntegerType, IndexType])
+
+
+def _is_float_type(type: Type):
+    return _is_any_of(type, [BF16Type, F16Type, F32Type, F64Type])
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class ConstantOp(ConstantOp):
+    """Specialization for the constant op class."""
+
+    def __init__(
+        self, result: Type, value: Union[int, float, Attribute], *, loc=None, ip=None
+    ):
+        if isinstance(value, int):
+            super().__init__(IntegerAttr.get(result, value), loc=loc, ip=ip)
+        elif isinstance(value, float):
+            super().__init__(FloatAttr.get(result, value), loc=loc, ip=ip)
+        else:
+            super().__init__(value, loc=loc, ip=ip)
+
+    @classmethod
+    def create_index(cls, value: int, *, loc=None, ip=None):
+        """Create an index-typed constant."""
+        return cls(
+            IndexType.get(context=_get_default_loc_context(loc)), value, loc=loc, ip=ip
+        )
+
+    @property
+    def type(self):
+        return self.results[0].type
+
+    @property
+    def value(self):
+        return Attribute(self.operation.attributes["value"])
+
+    @property
+    def literal_value(self) -> Union[int, float]:
+        if _is_integer_like_type(self.type):
+            return IntegerAttr(self.value).value
+        elif _is_float_type(self.type):
+            return FloatAttr(self.value).value
+        else:
+            raise ValueError("only integer and float constants have literal values")
diff --git a/mlir/python/mlir/dialects/builtin.py b/mlir/python/mlir/dialects/builtin.py
index 30279e1611..b71cc2466d 100644
--- a/mlir/python/mlir/dialects/builtin.py
+++ b/mlir/python/mlir/dialects/builtin.py
@@ -3,3 +3,23 @@
 #  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 from ._builtin_ops_gen import *
+from ._builtin_ops_gen import _Dialect
+
+try:
+    from ..ir import *
+    from ._ods_common import _cext as _ods_cext
+except ImportError as e:
+    raise RuntimeError("Error loading imports from extension module") from e
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class ModuleOp(ModuleOp):
+    """Specialization for the module op class."""
+
+    def __init__(self, *, loc=None, ip=None):
+        super().__init__(loc=loc, ip=ip)
+        body = self.regions[0].blocks.append()
+
+    @property
+    def body(self):
+        return self.regions[0].blocks[0]
diff --git a/mlir/python/mlir/dialects/func.py b/mlir/python/mlir/dialects/func.py
index dc554c2217..6599f67b70 100644
--- a/mlir/python/mlir/dialects/func.py
+++ b/mlir/python/mlir/dialects/func.py
@@ -3,3 +3,323 @@
 #  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 from ._func_ops_gen import *
+from ._func_ops_gen import _Dialect
+
+try:
+    from ..ir import *
+    from ._ods_common import (
+        get_default_loc_context as _get_default_loc_context,
+        _cext as _ods_cext,
+    )
+
+    import inspect
+
+    from typing import Any, List, Optional, Sequence, Union
+except ImportError as e:
+    raise RuntimeError("Error loading imports from extension module") from e
+
+ARGUMENT_ATTRIBUTE_NAME = "arg_attrs"
+RESULT_ATTRIBUTE_NAME = "res_attrs"
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class ConstantOp(ConstantOp):
+    """Specialization for the constant op class."""
+
+    @property
+    def type(self):
+        return self.results[0].type
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class FuncOp(FuncOp):
+    """Specialization for the func op class."""
+
+    def __init__(
+        self, name, type, *, visibility=None, body_builder=None, loc=None, ip=None
+    ):
+        """
+        Create a FuncOp with the provided `name`, `type`, and `visibility`.
+        - `name` is a string representing the function name.
+        - `type` is either a FunctionType or a pair of list describing inputs and
+          results.
+        - `visibility` is a string matching `public`, `private`, or `nested`. None
+          implies private visibility.
+        - `body_builder` is an optional callback, when provided a new entry block
+          is created and the callback is invoked with the new op as argument within
+          an InsertionPoint context already set for the block. The callback is
+          expected to insert a terminator in the block.
+        """
+        sym_name = StringAttr.get(str(name))
+
+        # If the type is passed as a tuple, build a FunctionType on the fly.
+        if isinstance(type, tuple):
+            type = FunctionType.get(inputs=type[0], results=type[1])
+
+        type = TypeAttr.get(type)
+        sym_visibility = (
+            StringAttr.get(str(visibility)) if visibility is not None else None
+        )
+        super().__init__(sym_name, type, sym_visibility=sym_visibility, loc=loc, ip=ip)
+        if body_builder:
+            entry_block = self.add_entry_block()
+            with InsertionPoint(entry_block):
+                body_builder(self)
+
+    @property
+    def is_external(self):
+        return len(self.regions[0].blocks) == 0
+
+    @property
+    def body(self):
+        return self.regions[0]
+
+    @property
+    def type(self):
+        return FunctionType(TypeAttr(self.attributes["function_type"]).value)
+
+    @property
+    def visibility(self):
+        return self.attributes["sym_visibility"]
+
+    @property
+    def name(self) -> StringAttr:
+        return StringAttr(self.attributes["sym_name"])
+
+    @property
+    def entry_block(self):
+        if self.is_external:
+            raise IndexError("External function does not have a body")
+        return self.regions[0].blocks[0]
+
+    def add_entry_block(self, arg_locs: Optional[Sequence[Location]] = None):
+        """
+        Add an entry block to the function body using the function signature to
+        infer block arguments.
+        Returns the newly created block
+        """
+        if not self.is_external:
+            raise IndexError("The function already has an entry block!")
+        self.body.blocks.append(*self.type.inputs, arg_locs=arg_locs)
+        return self.body.blocks[0]
+
+    @property
+    def arg_attrs(self):
+        return ArrayAttr(self.attributes[ARGUMENT_ATTRIBUTE_NAME])
+
+    @arg_attrs.setter
+    def arg_attrs(self, attribute: Union[ArrayAttr, list]):
+        if isinstance(attribute, ArrayAttr):
+            self.attributes[ARGUMENT_ATTRIBUTE_NAME] = attribute
+        else:
+            self.attributes[ARGUMENT_ATTRIBUTE_NAME] = ArrayAttr.get(
+                attribute, context=self.context
+            )
+
+    @property
+    def arguments(self):
+        return self.entry_block.arguments
+
+    @property
+    def result_attrs(self):
+        return self.attributes[RESULT_ATTRIBUTE_NAME]
+
+    @result_attrs.setter
+    def result_attrs(self, attribute: ArrayAttr):
+        self.attributes[RESULT_ATTRIBUTE_NAME] = attribute
+
+    @classmethod
+    def from_py_func(
+        FuncOp,
+        *inputs: Type,
+        results: Optional[Sequence[Type]] = None,
+        name: Optional[str] = None,
+    ):
+        """Decorator to define an MLIR FuncOp specified as a python function.
+
+        Requires that an `mlir.ir.InsertionPoint` and `mlir.ir.Location` are
+        active for the current thread (i.e. established in a `with` block).
+
+        When applied as a decorator to a Python function, an entry block will
+        be constructed for the FuncOp with types as specified in `*inputs`. The
+        block arguments will be passed positionally to the Python function. In
+        addition, if the Python function accepts keyword arguments generally or
+        has a corresponding keyword argument, the following will be passed:
+          * `func_op`: The `func` op being defined.
+
+        By default, the function name will be the Python function `__name__`. This
+        can be overriden by passing the `name` argument to the decorator.
+
+        If `results` is not specified, then the decorator will implicitly
+        insert a `ReturnOp` with the `Value`'s returned from the decorated
+        function. It will also set the `FuncOp` type with the actual return
+        value types. If `results` is specified, then the decorated function
+        must return `None` and no implicit `ReturnOp` is added (nor are the result
+        types updated). The implicit behavior is intended for simple, single-block
+        cases, and users should specify result types explicitly for any complicated
+        cases.
+
+        The decorated function can further be called from Python and will insert
+        a `CallOp` at the then-current insertion point, returning either None (
+        if no return values), a unary Value (for one result), or a list of Values).
+        This mechanism cannot be used to emit recursive calls (by construction).
+        """
+
+        def decorator(f):
+            from . import func
+
+            # Introspect the callable for optional features.
+            sig = inspect.signature(f)
+            has_arg_func_op = False
+            for param in sig.parameters.values():
+                if param.kind == param.VAR_KEYWORD:
+                    has_arg_func_op = True
+                if param.name == "func_op" and (
+                    param.kind == param.POSITIONAL_OR_KEYWORD
+                    or param.kind == param.KEYWORD_ONLY
+                ):
+                    has_arg_func_op = True
+
+            # Emit the FuncOp.
+            implicit_return = results is None
+            symbol_name = name or f.__name__
+            function_type = FunctionType.get(
+                inputs=inputs, results=[] if implicit_return else results
+            )
+            func_op = FuncOp(name=symbol_name, type=function_type)
+            with InsertionPoint(func_op.add_entry_block()):
+                func_args = func_op.entry_block.arguments
+                func_kwargs = {}
+                if has_arg_func_op:
+                    func_kwargs["func_op"] = func_op
+                return_values = f(*func_args, **func_kwargs)
+                if not implicit_return:
+                    return_types = list(results)
+                    assert return_values is None, (
+                        "Capturing a python function with explicit `results=` "
+                        "requires that the wrapped function returns None."
+                    )
+                else:
+                    # Coerce return values, add ReturnOp and rewrite func type.
+                    if return_values is None:
+                        return_values = []
+                    elif isinstance(return_values, tuple):
+                        return_values = list(return_values)
+                    elif isinstance(return_values, Value):
+                        # Returning a single value is fine, coerce it into a list.
+                        return_values = [return_values]
+                    elif isinstance(return_values, OpView):
+                        # Returning a single operation is fine, coerce its results a list.
+                        return_values = return_values.operation.results
+                    elif isinstance(return_values, Operation):
+                        # Returning a single operation is fine, coerce its results a list.
+                        return_values = return_values.results
+                    else:
+                        return_values = list(return_values)
+                    func.ReturnOp(return_values)
+                    # Recompute the function type.
+                    return_types = [v.type for v in return_values]
+                    function_type = FunctionType.get(
+                        inputs=inputs, results=return_types
+                    )
+                    func_op.attributes["function_type"] = TypeAttr.get(function_type)
+
+            def emit_call_op(*call_args):
+                call_op = func.CallOp(
+                    return_types, FlatSymbolRefAttr.get(symbol_name), call_args
+                )
+                if return_types is None:
+                    return None
+                elif len(return_types) == 1:
+                    return call_op.result
+                else:
+                    return call_op.results
+
+            wrapped = emit_call_op
+            wrapped.__name__ = f.__name__
+            wrapped.func_op = func_op
+            return wrapped
+
+        return decorator
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class CallOp(CallOp):
+    """Specialization for the call op class."""
+
+    def __init__(
+        self,
+        calleeOrResults: Union[FuncOp, List[Type]],
+        argumentsOrCallee: Union[List, FlatSymbolRefAttr, str],
+        arguments: Optional[List] = None,
+        *,
+        loc=None,
+        ip=None,
+    ):
+        """Creates an call operation.
+
+        The constructor accepts three different forms:
+
+          1. A function op to be called followed by a list of arguments.
+          2. A list of result types, followed by the name of the function to be
+             called as string, following by a list of arguments.
+          3. A list of result types, followed by the name of the function to be
+             called as symbol reference attribute, followed by a list of arguments.
+
+        For example
+
+            f = func.FuncOp("foo", ...)
+            func.CallOp(f, [args])
+            func.CallOp([result_types], "foo", [args])
+
+        In all cases, the location and insertion point may be specified as keyword
+        arguments if not provided by the surrounding context managers.
+        """
+
+        # TODO: consider supporting constructor "overloads", e.g., through a custom
+        # or pybind-provided metaclass.
+        if isinstance(calleeOrResults, FuncOp):
+            if not isinstance(argumentsOrCallee, list):
+                raise ValueError(
+                    "when constructing a call to a function, expected "
+                    + "the second argument to be a list of call arguments, "
+                    + f"got {type(argumentsOrCallee)}"
+                )
+            if arguments is not None:
+                raise ValueError(
+                    "unexpected third argument when constructing a call"
+                    + "to a function"
+                )
+
+            super().__init__(
+                calleeOrResults.type.results,
+                FlatSymbolRefAttr.get(
+                    calleeOrResults.name.value, context=_get_default_loc_context(loc)
+                ),
+                argumentsOrCallee,
+                loc=loc,
+                ip=ip,
+            )
+            return
+
+        if isinstance(argumentsOrCallee, list):
+            raise ValueError(
+                "when constructing a call to a function by name, "
+                + "expected the second argument to be a string or a "
+                + f"FlatSymbolRefAttr, got {type(argumentsOrCallee)}"
+            )
+
+        if isinstance(argumentsOrCallee, FlatSymbolRefAttr):
+            super().__init__(
+                calleeOrResults, argumentsOrCallee, arguments, loc=loc, ip=ip
+            )
+        elif isinstance(argumentsOrCallee, str):
+            super().__init__(
+                calleeOrResults,
+                FlatSymbolRefAttr.get(
+                    argumentsOrCallee, context=_get_default_loc_context(loc)
+                ),
+                arguments,
+                loc=loc,
+                ip=ip,
+            )
diff --git a/mlir/python/mlir/dialects/linalg/__init__.py b/mlir/python/mlir/dialects/linalg/__init__.py
index 1353870ec7..6e4cb1bd62 100644
--- a/mlir/python/mlir/dialects/linalg/__init__.py
+++ b/mlir/python/mlir/dialects/linalg/__init__.py
@@ -55,3 +55,51 @@
 #     TODO: guard against surprises and fail create Runtime Custom Ops with
 #     the same name as existing Core Named Ops.
 from .opdsl.ops.core_named_ops import *
+from .opdsl.lang.emitter import isa
+
+from ...ir import *
+from .._ods_common import get_op_result_or_value as _get_op_result_or_value
+
+
+def transpose(
+    input: Union[Operation, OpView, Sequence[Value]],
+    *,
+    outs: List[Union[Operation, OpView, Sequence[Value]]],
+    permutation: Union[DenseI64ArrayAttr, List[int]],
+):
+    input = _get_op_result_or_value(input)
+    if len(outs) > 1:
+        raise ValueError(f"{outs=} must have length 1.")
+    init = _get_op_result_or_value(outs[0])
+    result_types = [init.type] if isa(RankedTensorType, init.type) else []
+
+    op = TransposeOp(
+        result=result_types,
+        input=input,
+        init=init,
+        permutation=permutation,
+    )
+    fill_builtin_region(op.operation)
+    return op
+
+
+def broadcast(
+    input: Union[Operation, OpView, Sequence[Value]],
+    *,
+    outs: List[Union[Operation, OpView, Sequence[Value]]],
+    dimensions: Union[DenseI64ArrayAttr, List[int]],
+):
+    input = _get_op_result_or_value(input)
+    if len(outs) > 1:
+        raise ValueError(f"{outs=} must have length 1.")
+    init = _get_op_result_or_value(outs[0])
+    result_types = [init.type] if isa(RankedTensorType, init.type) else []
+
+    op = BroadcastOp(
+        result=result_types,
+        input=input,
+        init=init,
+        dimensions=dimensions,
+    )
+    fill_builtin_region(op.operation)
+    return op
diff --git a/mlir/python/mlir/dialects/linalg/opdsl/lang/emitter.py b/mlir/python/mlir/dialects/linalg/opdsl/lang/emitter.py
index 6f9d721644..f91fc8b716 100644
--- a/mlir/python/mlir/dialects/linalg/opdsl/lang/emitter.py
+++ b/mlir/python/mlir/dialects/linalg/opdsl/lang/emitter.py
@@ -310,7 +310,7 @@ def emit_named_structured_op(
         )
 
     # Set the index attributes used to compute the indexing maps.
-    named_op = getattr(linalg, op_class_name)(ins, outs, result_types)
+    named_op = getattr(linalg, op_class_name)(result_types, ins, outs)
     for name, value in index_attrs.items():
         named_op.operation.attributes[name] = value
 
diff --git a/mlir/python/mlir/dialects/linalg/opdsl/ops/core_named_ops.py b/mlir/python/mlir/dialects/linalg/opdsl/ops/core_named_ops.py
index a8f8f8e0fb..19734a80a1 100644
--- a/mlir/python/mlir/dialects/linalg/opdsl/ops/core_named_ops.py
+++ b/mlir/python/mlir/dialects/linalg/opdsl/ops/core_named_ops.py
@@ -296,35 +296,39 @@ def quantized_matmul(
 
 
 @linalg_structured_op
-def matmul_transpose_a(A=TensorDef(T1, S.K, S.N),
-                       B=TensorDef(T2, S.K, S.M),
-                       C=TensorDef(U, S.M, S.N, output=True),
-                       cast=TypeFnAttrDef(default=TypeFn.cast_signed)):
-  """Performs a matrix multiplication of two 2D inputs with lhs operand
-  transposed.
+def matmul_transpose_a(
+    A=TensorDef(T1, S.K, S.N),
+    B=TensorDef(T2, S.K, S.M),
+    C=TensorDef(U, S.M, S.N, output=True),
+    cast=TypeFnAttrDef(default=TypeFn.cast_signed),
+):
+    """Performs a matrix multiplication of two 2D inputs with lhs operand
+    transposed.
 
-  Numeric casting is performed on the operands to the inner multiply, promoting
-  them to the same data type as the accumulator/output.
-  """
-  domain(D.m, D.n, D.k)
-  implements(ContractionOpInterface)
-  C[D.m, D.n] += cast(U, A[D.k, D.m]) * cast(U, B[D.k, D.n])
+    Numeric casting is performed on the operands to the inner multiply, promoting
+    them to the same data type as the accumulator/output.
+    """
+    domain(D.m, D.n, D.k)
+    implements(ContractionOpInterface)
+    C[D.m, D.n] += cast(U, A[D.k, D.m]) * cast(U, B[D.k, D.n])
 
 
 @linalg_structured_op
-def matmul_transpose_b(A=TensorDef(T1, S.M, S.K),
-                       B=TensorDef(T2, S.N, S.K),
-                       C=TensorDef(U, S.M, S.N, output=True),
-                       cast=TypeFnAttrDef(default=TypeFn.cast_signed)):
-  """Performs a matrix multiplication of two 2D inputs with rhs operand
-  transposed.
+def matmul_transpose_b(
+    A=TensorDef(T1, S.M, S.K),
+    B=TensorDef(T2, S.N, S.K),
+    C=TensorDef(U, S.M, S.N, output=True),
+    cast=TypeFnAttrDef(default=TypeFn.cast_signed),
+):
+    """Performs a matrix multiplication of two 2D inputs with rhs operand
+    transposed.
 
-  Numeric casting is performed on the operands to the inner multiply, promoting
-  them to the same data type as the accumulator/output.
-  """
-  domain(D.m, D.n, D.k)
-  implements(ContractionOpInterface)
-  C[D.m, D.n] += cast(U, A[D.m, D.k]) * cast(U, B[D.n, D.k])
+    Numeric casting is performed on the operands to the inner multiply, promoting
+    them to the same data type as the accumulator/output.
+    """
+    domain(D.m, D.n, D.k)
+    implements(ContractionOpInterface)
+    C[D.m, D.n] += cast(U, A[D.m, D.k]) * cast(U, B[D.n, D.k])
 
 
 @linalg_structured_op
@@ -390,36 +394,41 @@ def batch_matmul(
 
 
 @linalg_structured_op
-def batch_matmul_transpose_a(A=TensorDef(T1, Batch, S.K, S.M),
-                             B=TensorDef(T2, Batch, S.K, S.N),
-                             C=TensorDef(U, Batch, S.M, S.N, output=True)):
-  """Performs a batched matrix multiplication of two 3D inputs where lhs operand
-  has its non-batch dimensions transposed.
+def batch_matmul_transpose_a(
+    A=TensorDef(T1, Batch, S.K, S.M),
+    B=TensorDef(T2, Batch, S.K, S.N),
+    C=TensorDef(U, Batch, S.M, S.N, output=True),
+):
+    """Performs a batched matrix multiplication of two 3D inputs where lhs operand
+    has its non-batch dimensions transposed.
 
-  Numeric casting is performed on the operands to the inner multiply, promoting
-  them to the same data type as the accumulator/output.
-  """
-  domain(D.b, D.m, D.n, D.k)
-  implements(ContractionOpInterface)
-  C[D.b, D.m, D.n] += TypeFn.cast_signed(U, A[D.b, D.k, D.m]) \
-                    * TypeFn.cast_signed(U, B[D.b, D.k, D.n])
+    Numeric casting is performed on the operands to the inner multiply, promoting
+    them to the same data type as the accumulator/output.
+    """
+    domain(D.b, D.m, D.n, D.k)
+    implements(ContractionOpInterface)
+    C[D.b, D.m, D.n] += TypeFn.cast_signed(U, A[D.b, D.k, D.m]) * TypeFn.cast_signed(
+        U, B[D.b, D.k, D.n]
+    )
 
 
 @linalg_structured_op
-def batch_matmul_transpose_b(A=TensorDef(T1, Batch, S.M, S.K),
-                             B=TensorDef(T2, Batch, S.N, S.K),
-                             C=TensorDef(U, Batch, S.M, S.N, output=True)):
-  """Performs a batched matrix multiplication of two 3D inputs where rhs operand
-  has its non-batch dimensions transposed.
+def batch_matmul_transpose_b(
+    A=TensorDef(T1, Batch, S.M, S.K),
+    B=TensorDef(T2, Batch, S.N, S.K),
+    C=TensorDef(U, Batch, S.M, S.N, output=True),
+):
+    """Performs a batched matrix multiplication of two 3D inputs where rhs operand
+    has its non-batch dimensions transposed.
 
-  Numeric casting is performed on the operands to the inner multiply, promoting
-  them to the same data type as the accumulator/output.
-  """
-  domain(D.b, D.m, D.n, D.k)
-  implements(ContractionOpInterface)
-  C[D.b, D.m,
-    D.n] += TypeFn.cast_signed(U, A[D.b, D.m, D.k]) * TypeFn.cast_signed(
-        U, B[D.b, D.n, D.k])
+    Numeric casting is performed on the operands to the inner multiply, promoting
+    them to the same data type as the accumulator/output.
+    """
+    domain(D.b, D.m, D.n, D.k)
+    implements(ContractionOpInterface)
+    C[D.b, D.m, D.n] += TypeFn.cast_signed(U, A[D.b, D.m, D.k]) * TypeFn.cast_signed(
+        U, B[D.b, D.n, D.k]
+    )
 
 
 @linalg_structured_op
diff --git a/mlir/python/mlir/dialects/ml_program.py b/mlir/python/mlir/dialects/ml_program.py
index a654529b4b..dfb6d7f2c0 100644
--- a/mlir/python/mlir/dialects/ml_program.py
+++ b/mlir/python/mlir/dialects/ml_program.py
@@ -2,4 +2,118 @@
 #  See https://llvm.org/LICENSE.txt for license information.
 #  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+from typing import Union
+
 from ._ml_program_ops_gen import *
+from ._ml_program_ops_gen import _Dialect
+
+try:
+    from ..ir import *
+    from ._ods_common import (
+        get_default_loc_context as _get_default_loc_context,
+        _cext as _ods_cext,
+    )
+except ImportError as e:
+    raise RuntimeError("Error loading imports from extension module") from e
+
+
+ARGUMENT_ATTRIBUTE_NAME = "arg_attrs"
+RESULT_ATTRIBUTE_NAME = "res_attrs"
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class FuncOp(FuncOp):
+    """Specialization for the func op class."""
+
+    def __init__(
+        self, name, type, *, visibility=None, body_builder=None, loc=None, ip=None
+    ):
+        """
+        Create a FuncOp with the provided `name`, `type`, and `visibility`.
+        - `name` is a string representing the function name.
+        - `type` is either a FunctionType or a pair of list describing inputs and
+          results.
+        - `visibility` is a string matching `public`, `private`, or `nested`. None
+          implies private visibility.
+        - `body_builder` is an optional callback, when provided a new entry block
+          is created and the callback is invoked with the new op as argument within
+          an InsertionPoint context already set for the block. The callback is
+          expected to insert a terminator in the block.
+        """
+        sym_name = StringAttr.get(str(name))
+
+        # If the type is passed as a tuple, build a FunctionType on the fly.
+        if isinstance(type, tuple):
+            type = FunctionType.get(inputs=type[0], results=type[1])
+
+        type = TypeAttr.get(type)
+        sym_visibility = (
+            StringAttr.get(str(visibility)) if visibility is not None else None
+        )
+        super().__init__(sym_name, type, sym_visibility=sym_visibility, loc=loc, ip=ip)
+        if body_builder:
+            entry_block = self.add_entry_block()
+            with InsertionPoint(entry_block):
+                body_builder(self)
+
+    @property
+    def is_external(self):
+        return len(self.regions[0].blocks) == 0
+
+    @property
+    def body(self):
+        return self.regions[0]
+
+    @property
+    def type(self):
+        return FunctionType(TypeAttr(self.attributes["function_type"]).value)
+
+    @property
+    def visibility(self):
+        return self.attributes["sym_visibility"]
+
+    @property
+    def name(self) -> StringAttr:
+        return StringAttr(self.attributes["sym_name"])
+
+    @property
+    def entry_block(self):
+        if self.is_external:
+            raise IndexError("External function does not have a body")
+        return self.regions[0].blocks[0]
+
+    def add_entry_block(self):
+        """
+        Add an entry block to the function body using the function signature to
+        infer block arguments.
+        Returns the newly created block
+        """
+        if not self.is_external:
+            raise IndexError("The function already has an entry block!")
+        self.body.blocks.append(*self.type.inputs)
+        return self.body.blocks[0]
+
+    @property
+    def arg_attrs(self):
+        return ArrayAttr(self.attributes[ARGUMENT_ATTRIBUTE_NAME])
+
+    @arg_attrs.setter
+    def arg_attrs(self, attribute: Union[ArrayAttr, list]):
+        if isinstance(attribute, ArrayAttr):
+            self.attributes[ARGUMENT_ATTRIBUTE_NAME] = attribute
+        else:
+            self.attributes[ARGUMENT_ATTRIBUTE_NAME] = ArrayAttr.get(
+                attribute, context=self.context
+            )
+
+    @property
+    def arguments(self):
+        return self.entry_block.arguments
+
+    @property
+    def result_attrs(self):
+        return self.attributes[RESULT_ATTRIBUTE_NAME]
+
+    @result_attrs.setter
+    def result_attrs(self, attribute: ArrayAttr):
+        self.attributes[RESULT_ATTRIBUTE_NAME] = attribute
diff --git a/mlir/python/mlir/dialects/pdl.py b/mlir/python/mlir/dialects/pdl.py
index dda2b7d652..90d7d70623 100644
--- a/mlir/python/mlir/dialects/pdl.py
+++ b/mlir/python/mlir/dialects/pdl.py
@@ -3,4 +3,220 @@
 #  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 from ._pdl_ops_gen import *
+from ._pdl_ops_gen import _Dialect
 from .._mlir_libs._mlirDialectsPDL import *
+
+
+try:
+    from ..ir import *
+    from ..dialects import pdl
+except ImportError as e:
+    raise RuntimeError("Error loading imports from extension module") from e
+
+from typing import Union, Optional, Sequence, Mapping
+from ._ods_common import (
+    get_op_result_or_value as _get_value,
+    get_op_results_or_values as _get_values,
+    _cext as _ods_cext,
+)
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class AttributeOp(AttributeOp):
+    """Specialization for PDL attribute op class."""
+
+    def __init__(
+        self,
+        valueType: Optional[Union[OpView, Operation, Value]] = None,
+        value: Optional[Attribute] = None,
+        *,
+        loc=None,
+        ip=None,
+    ):
+        valueType = valueType if valueType is None else _get_value(valueType)
+        result = pdl.AttributeType.get()
+        super().__init__(result, valueType=valueType, value=value, loc=loc, ip=ip)
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class OperandOp(OperandOp):
+    """Specialization for PDL operand op class."""
+
+    def __init__(
+        self,
+        type: Optional[Union[OpView, Operation, Value]] = None,
+        *,
+        loc=None,
+        ip=None,
+    ):
+        type = type if type is None else _get_value(type)
+        result = pdl.ValueType.get()
+        super().__init__(result, valueType=type, loc=loc, ip=ip)
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class OperandsOp(OperandsOp):
+    """Specialization for PDL operands op class."""
+
+    def __init__(
+        self,
+        types: Optional[Union[OpView, Operation, Value]] = None,
+        *,
+        loc=None,
+        ip=None,
+    ):
+        types = types if types is None else _get_value(types)
+        result = pdl.RangeType.get(pdl.ValueType.get())
+        super().__init__(result, valueType=types, loc=loc, ip=ip)
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class OperationOp(OperationOp):
+    """Specialization for PDL operand op class."""
+
+    def __init__(
+        self,
+        name: Optional[Union[str, StringAttr]] = None,
+        args: Optional[Sequence[Union[OpView, Operation, Value]]] = None,
+        attributes: Optional[Mapping[str, Union[OpView, Operation, Value]]] = None,
+        types: Optional[Sequence[Union[OpView, Operation, Value]]] = None,
+        *,
+        loc=None,
+        ip=None,
+    ):
+        if types is None:
+            types = []
+        if attributes is None:
+            attributes = {}
+        if args is None:
+            args = []
+        args = _get_values(args)
+        attrNames = []
+        attrValues = []
+        for attrName, attrValue in attributes.items():
+            attrNames.append(StringAttr.get(attrName))
+            attrValues.append(_get_value(attrValue))
+        attrNames = ArrayAttr.get(attrNames)
+        types = _get_values(types)
+        result = pdl.OperationType.get()
+        super().__init__(
+            result, args, attrValues, attrNames, types, opName=name, loc=loc, ip=ip
+        )
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class PatternOp(PatternOp):
+    """Specialization for PDL pattern op class."""
+
+    def __init__(
+        self,
+        benefit: Union[IntegerAttr, int],
+        name: Optional[Union[StringAttr, str]] = None,
+        *,
+        loc=None,
+        ip=None,
+    ):
+        """Creates an PDL `pattern` operation."""
+        super().__init__(benefit, sym_name=name, loc=loc, ip=ip)
+        self.regions[0].blocks.append()
+
+    @property
+    def body(self):
+        """Return the body (block) of the pattern."""
+        return self.regions[0].blocks[0]
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class ReplaceOp(ReplaceOp):
+    """Specialization for PDL replace op class."""
+
+    def __init__(
+        self,
+        op: Union[OpView, Operation, Value],
+        *,
+        with_op: Optional[Union[OpView, Operation, Value]] = None,
+        with_values: Optional[Sequence[Union[OpView, Operation, Value]]] = None,
+        loc=None,
+        ip=None,
+    ):
+        if with_values is None:
+            with_values = []
+        op = _get_value(op)
+        with_op = with_op if with_op is None else _get_value(with_op)
+        with_values = _get_values(with_values)
+        super().__init__(op, with_values, replOperation=with_op, loc=loc, ip=ip)
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class ResultOp(ResultOp):
+    """Specialization for PDL result op class."""
+
+    def __init__(
+        self,
+        parent: Union[OpView, Operation, Value],
+        index: Union[IntegerAttr, int],
+        *,
+        loc=None,
+        ip=None,
+    ):
+        parent = _get_value(parent)
+        result = pdl.ValueType.get()
+        super().__init__(result, parent, index, loc=loc, ip=ip)
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class RewriteOp(RewriteOp):
+    """Specialization for PDL rewrite op class."""
+
+    def __init__(
+        self,
+        root: Optional[Union[OpView, Operation, Value]] = None,
+        name: Optional[Union[StringAttr, str]] = None,
+        args: Optional[Sequence[Union[OpView, Operation, Value]]] = None,
+        *,
+        loc=None,
+        ip=None,
+    ):
+        if args is None:
+            args = []
+        root = root if root is None else _get_value(root)
+        args = _get_values(args)
+        super().__init__(args, root=root, name=name, loc=loc, ip=ip)
+
+    def add_body(self):
+        """Add body (block) to the rewrite."""
+        self.regions[0].blocks.append()
+        return self.body
+
+    @property
+    def body(self):
+        """Return the body (block) of the rewrite."""
+        return self.regions[0].blocks[0]
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class TypeOp(TypeOp):
+    """Specialization for PDL type op class."""
+
+    def __init__(
+        self, constantType: Optional[Union[TypeAttr, Type]] = None, *, loc=None, ip=None
+    ):
+        result = pdl.TypeType.get()
+        super().__init__(result, constantType=constantType, loc=loc, ip=ip)
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class TypesOp(TypesOp):
+    """Specialization for PDL types op class."""
+
+    def __init__(
+        self,
+        constantTypes: Optional[Sequence[Union[TypeAttr, Type]]] = None,
+        *,
+        loc=None,
+        ip=None,
+    ):
+        if constantTypes is None:
+            constantTypes = []
+        result = pdl.RangeType.get(pdl.TypeType.get())
+        super().__init__(result, constantTypes=constantTypes, loc=loc, ip=ip)
diff --git a/mlir/python/mlir/dialects/python_test.py b/mlir/python/mlir/dialects/python_test.py
index 8465af048a..6579e02d85 100644
--- a/mlir/python/mlir/dialects/python_test.py
+++ b/mlir/python/mlir/dialects/python_test.py
@@ -3,7 +3,12 @@
 #  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 from ._python_test_ops_gen import *
-from .._mlir_libs._mlirPythonTest import TestAttr, TestType, TestTensorValue, TestIntegerRankedTensorType
+from .._mlir_libs._mlirPythonTest import (
+    TestAttr,
+    TestType,
+    TestTensorValue,
+    TestIntegerRankedTensorType,
+)
 
 
 def register_python_test_dialect(context, load=True):
diff --git a/mlir/python/mlir/dialects/scf.py b/mlir/python/mlir/dialects/scf.py
index 49685ca227..71c80cab76 100644
--- a/mlir/python/mlir/dialects/scf.py
+++ b/mlir/python/mlir/dialects/scf.py
@@ -2,11 +2,105 @@
 #  See https://llvm.org/LICENSE.txt for license information.
 #  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-from typing import Optional, Sequence
 
 from ._scf_ops_gen import *
+from ._scf_ops_gen import _Dialect
 from .arith import constant
-from ..ir import *
+
+try:
+    from ..ir import *
+    from ._ods_common import (
+        get_op_result_or_value as _get_op_result_or_value,
+        get_op_results_or_values as _get_op_results_or_values,
+        _cext as _ods_cext,
+    )
+except ImportError as e:
+    raise RuntimeError("Error loading imports from extension module") from e
+
+from typing import Optional, Sequence, Union
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class ForOp(ForOp):
+    """Specialization for the SCF for op class."""
+
+    def __init__(
+        self,
+        lower_bound,
+        upper_bound,
+        step,
+        iter_args: Optional[Union[Operation, OpView, Sequence[Value]]] = None,
+        *,
+        loc=None,
+        ip=None,
+    ):
+        """Creates an SCF `for` operation.
+
+        - `lower_bound` is the value to use as lower bound of the loop.
+        - `upper_bound` is the value to use as upper bound of the loop.
+        - `step` is the value to use as loop step.
+        - `iter_args` is a list of additional loop-carried arguments or an operation
+          producing them as results.
+        """
+        if iter_args is None:
+            iter_args = []
+        iter_args = _get_op_results_or_values(iter_args)
+
+        results = [arg.type for arg in iter_args]
+        super().__init__(
+            results, lower_bound, upper_bound, step, iter_args, loc=loc, ip=ip
+        )
+        self.regions[0].blocks.append(self.operands[0].type, *results)
+
+    @property
+    def body(self):
+        """Returns the body (block) of the loop."""
+        return self.regions[0].blocks[0]
+
+    @property
+    def induction_variable(self):
+        """Returns the induction variable of the loop."""
+        return self.body.arguments[0]
+
+    @property
+    def inner_iter_args(self):
+        """Returns the loop-carried arguments usable within the loop.
+
+        To obtain the loop-carried operands, use `iter_args`.
+        """
+        return self.body.arguments[1:]
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class IfOp(IfOp):
+    """Specialization for the SCF if op class."""
+
+    def __init__(self, cond, results_=None, *, hasElse=False, loc=None, ip=None):
+        """Creates an SCF `if` operation.
+
+        - `cond` is a MLIR value of 'i1' type to determine which regions of code will be executed.
+        - `hasElse` determines whether the if operation has the else branch.
+        """
+        if results_ is None:
+            results_ = []
+        operands = []
+        operands.append(cond)
+        results = []
+        results.extend(results_)
+        super().__init__(results, cond)
+        self.regions[0].blocks.append(*[])
+        if hasElse:
+            self.regions[1].blocks.append(*[])
+
+    @property
+    def then_block(self):
+        """Returns the then block of the if operation."""
+        return self.regions[0].blocks[0]
+
+    @property
+    def else_block(self):
+        """Returns the else block of the if operation."""
+        return self.regions[1].blocks[0]
 
 
 def for_(
diff --git a/mlir/python/mlir/dialects/tensor.py b/mlir/python/mlir/dialects/tensor.py
index 26edf6b643..67248748ea 100644
--- a/mlir/python/mlir/dialects/tensor.py
+++ b/mlir/python/mlir/dialects/tensor.py
@@ -3,3 +3,40 @@
 #  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 from ._tensor_ops_gen import *
+from ._tensor_ops_gen import _Dialect
+
+try:
+    from ..ir import *
+except ImportError as e:
+    raise RuntimeError("Error loading imports from extension module") from e
+
+from typing import Sequence, Union
+from ._ods_common import _cext as _ods_cext
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class EmptyOp(EmptyOp):
+    """Extends the tensor.empty op."""
+
+    def __init__(
+        self,
+        sizes: Sequence[Union[int, Value]],
+        element_type: Type,
+        *,
+        loc=None,
+        ip=None,
+    ):
+        """Constructs an `empty` with mixed static/dynamic sizes."""
+        # TODO: Refactor the EmptyOp to take an element type attribute and
+        # then use normal result type inference, unifying the Python and C++ side
+        # with a standard mechanism (versus stashing that in builders).
+        dynamic_sizes = []
+        static_sizes = []
+        for s in sizes:
+            if isinstance(s, int):
+                static_sizes.append(s)
+            else:
+                static_sizes.append(ShapedType.get_dynamic_size())
+                dynamic_sizes.append(s)
+        result_type = RankedTensorType.get(static_sizes, element_type)
+        super().__init__(result_type, dynamic_sizes, loc=loc, ip=ip)
diff --git a/mlir/python/mlir/dialects/transform/__init__.py b/mlir/python/mlir/dialects/transform/__init__.py
index b020ad35fc..f7a2026e80 100644
--- a/mlir/python/mlir/dialects/transform/__init__.py
+++ b/mlir/python/mlir/dialects/transform/__init__.py
@@ -4,4 +4,174 @@
 
 from .._transform_enum_gen import *
 from .._transform_ops_gen import *
+from .._transform_ops_gen import _Dialect
 from ..._mlir_libs._mlirDialectsTransform import *
+
+try:
+    from ...ir import *
+    from .._ods_common import (
+        get_op_result_or_value as _get_op_result_or_value,
+        get_op_results_or_values as _get_op_results_or_values,
+        _cext as _ods_cext,
+    )
+except ImportError as e:
+    raise RuntimeError("Error loading imports from extension module") from e
+
+from typing import Optional, Sequence, Union
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class CastOp(CastOp):
+    def __init__(
+        self,
+        result_type: Type,
+        target: Union[Operation, Value],
+        *,
+        loc=None,
+        ip=None,
+    ):
+        super().__init__(result_type, _get_op_result_or_value(target), loc=loc, ip=ip)
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class ApplyPatternsOp(ApplyPatternsOp):
+    def __init__(
+        self,
+        target: Union[Operation, Value, OpView],
+        *,
+        loc=None,
+        ip=None,
+    ):
+        super().__init__(target, loc=loc, ip=ip)
+        self.regions[0].blocks.append()
+
+    @property
+    def patterns(self) -> Block:
+        return self.regions[0].blocks[0]
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class GetParentOp(GetParentOp):
+    def __init__(
+        self,
+        result_type: Type,
+        target: Union[Operation, Value],
+        *,
+        isolated_from_above: bool = False,
+        op_name: Optional[str] = None,
+        deduplicate: bool = False,
+        loc=None,
+        ip=None,
+    ):
+        super().__init__(
+            result_type,
+            _get_op_result_or_value(target),
+            isolated_from_above=isolated_from_above,
+            op_name=op_name,
+            deduplicate=deduplicate,
+            loc=loc,
+            ip=ip,
+        )
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class MergeHandlesOp(MergeHandlesOp):
+    def __init__(
+        self,
+        handles: Sequence[Union[Operation, Value]],
+        *,
+        deduplicate: bool = False,
+        loc=None,
+        ip=None,
+    ):
+        super().__init__(
+            [_get_op_result_or_value(h) for h in handles],
+            deduplicate=deduplicate,
+            loc=loc,
+            ip=ip,
+        )
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class ReplicateOp(ReplicateOp):
+    def __init__(
+        self,
+        pattern: Union[Operation, Value],
+        handles: Sequence[Union[Operation, Value]],
+        *,
+        loc=None,
+        ip=None,
+    ):
+        super().__init__(
+            [_get_op_result_or_value(h).type for h in handles],
+            _get_op_result_or_value(pattern),
+            [_get_op_result_or_value(h) for h in handles],
+            loc=loc,
+            ip=ip,
+        )
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class SequenceOp(SequenceOp):
+    def __init__(
+        self,
+        failure_propagation_mode,
+        results: Sequence[Type],
+        target: Union[Operation, Value, Type],
+        extra_bindings: Optional[
+            Union[Sequence[Value], Sequence[Type], Operation, OpView]
+        ] = None,
+    ):
+        root = (
+            _get_op_result_or_value(target)
+            if isinstance(target, (Operation, Value))
+            else None
+        )
+        root_type = root.type if not isinstance(target, Type) else target
+
+        if extra_bindings is None:
+            extra_bindings = []
+        if isinstance(extra_bindings, (Operation, OpView)):
+            extra_bindings = _get_op_results_or_values(extra_bindings)
+
+        extra_binding_types = []
+        if len(extra_bindings) != 0:
+            if isinstance(extra_bindings[0], Type):
+                extra_binding_types = extra_bindings
+                extra_bindings = []
+            else:
+                extra_binding_types = [v.type for v in extra_bindings]
+
+        super().__init__(
+            results_=results,
+            failure_propagation_mode=failure_propagation_mode,
+            root=root,
+            extra_bindings=extra_bindings,
+        )
+        self.regions[0].blocks.append(*tuple([root_type] + extra_binding_types))
+
+    @property
+    def body(self) -> Block:
+        return self.regions[0].blocks[0]
+
+    @property
+    def bodyTarget(self) -> Value:
+        return self.body.arguments[0]
+
+    @property
+    def bodyExtraArgs(self) -> BlockArgumentList:
+        return self.body.arguments[1:]
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class YieldOp(YieldOp):
+    def __init__(
+        self,
+        operands: Optional[Union[Operation, Sequence[Value]]] = None,
+        *,
+        loc=None,
+        ip=None,
+    ):
+        if operands is None:
+            operands = []
+        super().__init__(_get_op_results_or_values(operands), loc=loc, ip=ip)
diff --git a/mlir/python/mlir/dialects/transform/bufferization.py b/mlir/python/mlir/dialects/transform/bufferization.py
index eb77b746cf..485a8a36b6 100644
--- a/mlir/python/mlir/dialects/transform/bufferization.py
+++ b/mlir/python/mlir/dialects/transform/bufferization.py
@@ -3,3 +3,132 @@
 #  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 from .._bufferization_transform_ops_gen import *
+from .._bufferization_transform_ops_gen import _Dialect
+
+try:
+    from ...ir import *
+    from ...dialects import transform
+    from .._ods_common import _cext as _ods_cext
+except ImportError as e:
+    raise RuntimeError("Error loading imports from extension module") from e
+
+from enum import Enum
+from typing import Optional, overload, Union
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class EmptyTensorToAllocTensorOp(EmptyTensorToAllocTensorOp):
+    """Specialization for EmptyTensorToAllocTensorOp class."""
+
+    @overload
+    def __init__(
+        self,
+        transformed_type: Type,
+        target: Union[Operation, OpView, Value],
+        *,
+        loc=None,
+        ip=None,
+    ):
+        ...
+
+    @overload
+    def __init__(self, target: Union[Operation, OpView, Value], *, loc=None, ip=None):
+        ...
+
+    def __init__(
+        self,
+        transformed_type_or_target: Type,
+        target_or_none: Optional[Union[Operation, OpView, Value]] = None,
+        *,
+        loc=None,
+        ip=None,
+    ):
+        if isinstance(transformed_type_or_target, Type):
+            transformed_type = transformed_type_or_target
+            target = target_or_none
+        else:
+            transformed_type = transform.OperationType.get("bufferization.alloc_tensor")
+            target = transformed_type_or_target
+
+        super().__init__(
+            transformed_type,
+            target,
+            loc=loc,
+            ip=ip,
+        )
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class OneShotBufferizeOp(OneShotBufferizeOp):
+    """Specialization for OneShotBufferizeOp class."""
+
+    @overload
+    def __init__(
+        self,
+        transformed_type: Type,
+        target: Union[Operation, OpView, Value],
+        *,
+        allow_return_allocs_from_loops: Optional[bool] = None,
+        allow_unknown_ops: Optional[bool] = None,
+        bufferize_function_boundaries: Optional[bool] = None,
+        function_boundary_type_conversion: Optional[Enum] = None,
+        memcpy_op: Optional[str] = None,
+        print_conflicts: Optional[bool] = None,
+        test_analysis_only: Optional[bool] = None,
+        loc=None,
+        ip=None,
+    ):
+        ...
+
+    @overload
+    def __init__(
+        self,
+        target: Union[Operation, OpView, Value],
+        *,
+        allow_return_allocs_from_loops: Optional[bool] = None,
+        allow_unknown_ops: Optional[bool] = None,
+        bufferize_function_boundaries: Optional[bool] = None,
+        function_boundary_type_conversion: Optional[Enum] = None,
+        memcpy_op: Optional[str] = None,
+        print_conflicts: Optional[bool] = None,
+        test_analysis_only: Optional[bool] = None,
+        loc=None,
+        ip=None,
+    ):
+        ...
+
+    def __init__(
+        self,
+        transformed_type_or_target: Type,
+        target_or_none: Optional[Union[Operation, OpView, Value]] = None,
+        *,
+        allow_return_allocs_from_loops: Optional[bool] = None,
+        allow_unknown_ops: Optional[bool] = None,
+        bufferize_function_boundaries: Optional[bool] = None,
+        function_boundary_type_conversion: Optional[Enum] = None,
+        memcpy_op: Optional[str] = None,
+        print_conflicts: Optional[bool] = None,
+        test_analysis_only: Optional[bool] = None,
+        loc=None,
+        ip=None,
+    ):
+        if isinstance(transformed_type_or_target, Type):
+            transformed_type = transformed_type_or_target
+            target = target_or_none
+        else:
+            transformed_type = transform.AnyOpType.get()
+            target = transformed_type_or_target
+
+        super().__init__(
+            transformed_type,
+            target,
+            allow_return_allocs_from_loops=allow_return_allocs_from_loops,
+            allow_unknown_ops=allow_unknown_ops,
+            bufferize_function_boundaries=bufferize_function_boundaries,
+            function_boundary_type_conversion=function_boundary_type_conversion,
+            memcpy_op=memcpy_op,
+            print_conflicts=print_conflicts,
+            test_analysis_only=test_analysis_only,
+            loc=loc,
+            ip=ip,
+        )
diff --git a/mlir/python/mlir/dialects/transform/gpu.py b/mlir/python/mlir/dialects/transform/gpu.py
index 8c3de0de7e..00cf0840ee 100644
--- a/mlir/python/mlir/dialects/transform/gpu.py
+++ b/mlir/python/mlir/dialects/transform/gpu.py
@@ -3,3 +3,128 @@
 #  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 from .._gpu_transform_ops_gen import *
+from .._gpu_transform_ops_gen import _Dialect
+
+try:
+    from ...ir import *
+    from ...dialects import transform
+    from .._ods_common import _cext as _ods_cext
+except ImportError as e:
+    raise RuntimeError("Error loading imports from extension module") from e
+
+from typing import Optional, Sequence, Union, overload
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class MapForallToBlocks(MapForallToBlocks):
+    """Specialization for MapForallToBlocks class."""
+
+    @overload
+    def __init__(
+        self,
+        result_type: Type,
+        target: Union[Operation, OpView, Value],
+        *,
+        grid_dims: Optional[Union[Sequence[int], Attribute]] = None,
+        generate_gpu_launch: Optional[Union[bool, Attribute]] = None,
+        loc=None,
+        ip=None,
+    ):
+        ...
+
+    @overload
+    def __init__(
+        self,
+        target: Union[Operation, OpView, Value],
+        *,
+        grid_dims: Optional[Union[Sequence[int], Attribute]] = None,
+        generate_gpu_launch: Optional[Union[bool, Attribute]] = None,
+        loc=None,
+        ip=None,
+    ):
+        ...
+
+    def __init__(
+        self,
+        result_type_or_target: Union[Operation, OpView, Type, Value],
+        target_or_none: Optional[Union[Operation, OpView, Value]] = None,
+        *,
+        grid_dims: Optional[Union[Sequence[int], Attribute]] = None,
+        generate_gpu_launch: Optional[Union[bool, Attribute]] = None,
+        loc=None,
+        ip=None,
+    ):
+        if isinstance(result_type_or_target, Type):
+            result_type = result_type_or_target
+            target = target_or_none
+        else:
+            result_type = transform.AnyOpType.get()
+            target = result_type_or_target
+
+        super().__init__(
+            result_type,
+            target,
+            grid_dims=grid_dims,
+            generate_gpu_launch=generate_gpu_launch,
+            loc=loc,
+            ip=ip,
+        )
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class MapNestedForallToThreads(MapNestedForallToThreads):
+    """Specialization for MapNestedForallToThreads class."""
+
+    @overload
+    def __init__(
+        self,
+        result_type: Type,
+        target: Union[Operation, OpView, Value],
+        *,
+        block_dims: Optional[Sequence[int]] = None,
+        warp_size: Optional[Sequence[int]] = None,
+        sync_after_distribute: Optional[bool] = None,
+        loc=None,
+        ip=None,
+    ):
+        ...
+
+    @overload
+    def __init__(
+        self,
+        target: Union[Operation, OpView, Value],
+        *,
+        block_dims: Optional[Sequence[int]] = None,
+        warp_size: Optional[Sequence[int]] = None,
+        sync_after_distribute: Optional[bool] = None,
+        loc=None,
+        ip=None,
+    ):
+        ...
+
+    def __init__(
+        self,
+        result_type_or_target: Union[Operation, OpView, Value, Type],
+        target_or_none: Optional[Union[Operation, OpView, Value]] = None,
+        *,
+        block_dims: Optional[Union[Sequence[int], Attribute]] = None,
+        warp_size: Optional[Union[Sequence[int], Attribute]] = None,
+        sync_after_distribute: Optional[bool] = None,
+        loc=None,
+        ip=None,
+    ):
+        if isinstance(result_type_or_target, Type):
+            result_type = result_type_or_target
+            target = target_or_none
+        else:
+            result_type = result_type_or_target.type
+            target = result_type_or_target
+        super().__init__(
+            result_type,
+            target,
+            block_dims=block_dims,
+            warp_size=warp_size,
+            sync_after_distribute=sync_after_distribute,
+            loc=loc,
+            ip=ip,
+        )
diff --git a/mlir/python/mlir/dialects/transform/loop.py b/mlir/python/mlir/dialects/transform/loop.py
index 86f72788d8..6c89025f41 100644
--- a/mlir/python/mlir/dialects/transform/loop.py
+++ b/mlir/python/mlir/dialects/transform/loop.py
@@ -3,3 +3,143 @@
 #  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 from .._loop_transform_ops_gen import *
+from .._loop_transform_ops_gen import _Dialect
+
+try:
+    from ...ir import *
+    from .._ods_common import (
+        get_op_result_or_value as _get_op_result_or_value,
+        _cext as _ods_cext,
+    )
+except ImportError as e:
+    raise RuntimeError("Error loading imports from extension module") from e
+
+from typing import Optional, Union
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class GetParentForOp(GetParentForOp):
+    """Extension for GetParentForOp."""
+
+    def __init__(
+        self,
+        result_type: Type,
+        target: Union[Operation, Value],
+        *,
+        num_loops: Optional[int] = None,
+        ip=None,
+        loc=None,
+    ):
+        if num_loops is None:
+            num_loops = 1
+        super().__init__(
+            result_type,
+            _get_op_result_or_value(target),
+            num_loops=num_loops,
+            ip=ip,
+            loc=loc,
+        )
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class LoopOutlineOp(LoopOutlineOp):
+    """Extension for LoopOutlineOp."""
+
+    def __init__(
+        self,
+        function_type: Type,
+        call_type: Type,
+        target: Union[Operation, Value],
+        *,
+        func_name: Union[str, StringAttr],
+        ip=None,
+        loc=None,
+    ):
+        super().__init__(
+            function_type,
+            call_type,
+            _get_op_result_or_value(target),
+            func_name=(
+                func_name
+                if isinstance(func_name, StringAttr)
+                else StringAttr.get(func_name)
+            ),
+            ip=ip,
+            loc=loc,
+        )
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class LoopPeelOp(LoopPeelOp):
+    """Extension for LoopPeelOp."""
+
+    def __init__(
+        self,
+        main_loop_type: Type,
+        remainder_loop_type: Type,
+        target: Union[Operation, Value],
+        *,
+        fail_if_already_divisible: Union[bool, BoolAttr] = False,
+        ip=None,
+        loc=None,
+    ):
+        super().__init__(
+            main_loop_type,
+            remainder_loop_type,
+            _get_op_result_or_value(target),
+            fail_if_already_divisible=(
+                fail_if_already_divisible
+                if isinstance(fail_if_already_divisible, BoolAttr)
+                else BoolAttr.get(fail_if_already_divisible)
+            ),
+            ip=ip,
+            loc=loc,
+        )
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class LoopPipelineOp(LoopPipelineOp):
+    """Extension for LoopPipelineOp."""
+
+    def __init__(
+        self,
+        result_type: Type,
+        target: Union[Operation, Value],
+        *,
+        iteration_interval: Optional[Union[int, IntegerAttr]] = None,
+        read_latency: Optional[Union[int, IntegerAttr]] = None,
+        ip=None,
+        loc=None,
+    ):
+        if iteration_interval is None:
+            iteration_interval = 1
+        if read_latency is None:
+            read_latency = 10
+        super().__init__(
+            result_type,
+            _get_op_result_or_value(target),
+            iteration_interval=iteration_interval,
+            read_latency=read_latency,
+            ip=ip,
+            loc=loc,
+        )
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class LoopUnrollOp(LoopUnrollOp):
+    """Extension for LoopUnrollOp."""
+
+    def __init__(
+        self,
+        target: Union[Operation, Value],
+        *,
+        factor: Union[int, IntegerAttr],
+        ip=None,
+        loc=None,
+    ):
+        super().__init__(
+            _get_op_result_or_value(target),
+            factor=factor,
+            ip=ip,
+            loc=loc,
+        )
diff --git a/mlir/python/mlir/dialects/transform/memref.py b/mlir/python/mlir/dialects/transform/memref.py
index 1ff04ef6a6..56ea61eb81 100644
--- a/mlir/python/mlir/dialects/transform/memref.py
+++ b/mlir/python/mlir/dialects/transform/memref.py
@@ -3,3 +3,118 @@
 #  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 from .._memref_transform_ops_gen import *
+from .._memref_transform_ops_gen import _Dialect
+
+try:
+    from ...ir import *
+    from ...dialects import transform
+    from .._ods_common import _cext as _ods_cext
+except ImportError as e:
+    raise RuntimeError("Error loading imports from extension module") from e
+
+from typing import Optional, overload, Union
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class MemRefAllocaToGlobalOp(MemRefAllocaToGlobalOp):
+    """Specialization for MemRefAllocaToGlobalOp class."""
+
+    @overload
+    def __init__(
+        self,
+        get_global_type: Type,
+        global_type: Type,
+        alloca: Union[Operation, OpView, Value],
+        *,
+        loc=None,
+        ip=None,
+    ):
+        ...
+
+    @overload
+    def __init__(self, alloca: Union[Operation, OpView, Value], *, loc=None, ip=None):
+        ...
+
+    def __init__(
+        self,
+        get_global_type_or_alloca: Union[Operation, OpView, Type, Value],
+        global_type_or_none: Optional[Type] = None,
+        alloca_or_none: Optional[Union[Operation, OpView, Value]] = None,
+        *,
+        loc=None,
+        ip=None,
+    ):
+        if isinstance(get_global_type_or_alloca, Type):
+            get_global_type = get_global_type_or_alloca
+            global_type = global_type_or_none
+            alloca = alloca_or_none
+        else:
+            get_global_type = transform.AnyOpType.get()
+            global_type = transform.AnyOpType.get()
+            alloca = get_global_type_or_alloca
+
+        super().__init__(
+            get_global_type,
+            global_type,
+            alloca,
+            loc=loc,
+            ip=ip,
+        )
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class MemRefMultiBufferOp(MemRefMultiBufferOp):
+    """Specialization for MemRefMultiBufferOp class."""
+
+    @overload
+    def __init__(
+        self,
+        transformed_type: Type,
+        target: Union[Operation, OpView, Value],
+        factor: Union[int, IntegerAttr],
+        *,
+        skip_analysis: Optional[bool] = None,
+        loc=None,
+        ip=None,
+    ):
+        ...
+
+    @overload
+    def __init__(
+        self,
+        target: Union[Operation, OpView, Value],
+        factor: Union[int, IntegerAttr],
+        *,
+        skip_analysis: Optional[bool] = None,
+        loc=None,
+        ip=None,
+    ):
+        ...
+
+    def __init__(
+        self,
+        transformed_type_or_target: Type,
+        target_or_factor: Union[int, IntegerAttr, Operation, OpView, Value] = None,
+        factor_or_none: Optional[Union[int, IntegerAttr]] = None,
+        *,
+        skip_analysis: Optional[bool] = None,
+        loc=None,
+        ip=None,
+    ):
+        if isinstance(transformed_type_or_target, Type):
+            transformed_type = transformed_type_or_target
+            target = target_or_factor
+            factor = factor_or_none
+        else:
+            transformed_type = transform.AnyOpType.get()
+            target = transformed_type_or_target
+            factor = target_or_factor
+
+        super().__init__(
+            transformed_type,
+            target,
+            factor,
+            skip_analysis=skip_analysis,
+            loc=loc,
+            ip=ip,
+        )
diff --git a/mlir/python/mlir/dialects/transform/pdl.py b/mlir/python/mlir/dialects/transform/pdl.py
index b1515287a3..bb5fa7ffd3 100644
--- a/mlir/python/mlir/dialects/transform/pdl.py
+++ b/mlir/python/mlir/dialects/transform/pdl.py
@@ -3,3 +3,53 @@
 #  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 from .._transform_pdl_extension_ops_gen import *
+from .._transform_pdl_extension_ops_gen import _Dialect
+
+try:
+    from ...ir import *
+    from .._ods_common import (
+        get_op_result_or_value as _get_op_result_or_value,
+        get_op_results_or_values as _get_op_results_or_values,
+        _cext as _ods_cext,
+    )
+except ImportError as e:
+    raise RuntimeError("Error loading imports from extension module") from e
+
+from typing import Union
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class PDLMatchOp(PDLMatchOp):
+    def __init__(
+        self,
+        result_type: Type,
+        target: Union[Operation, Value],
+        pattern_name: Union[Attribute, str],
+        *,
+        loc=None,
+        ip=None,
+    ):
+        super().__init__(
+            result_type,
+            _get_op_result_or_value(target),
+            pattern_name,
+            loc=loc,
+            ip=ip,
+        )
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class WithPDLPatternsOp(WithPDLPatternsOp):
+    def __init__(self, target: Union[Operation, Value, Type], *, loc=None, ip=None):
+        root = _get_op_result_or_value(target) if not isinstance(target, Type) else None
+        root_type = target if isinstance(target, Type) else root.type
+        super().__init__(root=root, loc=loc, ip=ip)
+        self.regions[0].blocks.append(root_type)
+
+    @property
+    def body(self) -> Block:
+        return self.regions[0].blocks[0]
+
+    @property
+    def bodyTarget(self) -> Value:
+        return self.body.arguments[0]
diff --git a/mlir/python/mlir/dialects/transform/structured.py b/mlir/python/mlir/dialects/transform/structured.py
index cb3812301d..284c93823a 100644
--- a/mlir/python/mlir/dialects/transform/structured.py
+++ b/mlir/python/mlir/dialects/transform/structured.py
@@ -3,4 +3,777 @@
 #  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 from .._structured_transform_ops_gen import *
+from .._structured_transform_ops_gen import _Dialect
 from .._structured_transform_enum_gen import *
+
+try:
+    from ...ir import *
+    from ...dialects import transform
+    from .._ods_common import _cext as _ods_cext
+except ImportError as e:
+    raise RuntimeError("Error loading imports from extension module") from e
+
+from typing import List, Optional, Sequence, Tuple, Union, overload
+
+StaticIntLike = Union[int, IntegerAttr]
+ValueLike = Union[Operation, OpView, Value]
+MixedInt = Union[StaticIntLike, ValueLike]
+
+IntOrAttrList = Sequence[Union[IntegerAttr, int]]
+OptionalIntList = Optional[Union[ArrayAttr, IntOrAttrList]]
+
+BoolOrAttrList = Sequence[Union[BoolAttr, bool]]
+OptionalBoolList = Optional[Union[ArrayAttr, BoolOrAttrList]]
+
+MixedValues = Union[Sequence[Union[StaticIntLike, ValueLike]], ArrayAttr, ValueLike]
+
+DynamicIndexList = Sequence[Union[MixedInt, Sequence[MixedInt]]]
+
+
+def _dispatch_dynamic_index_list(
+    indices: Union[DynamicIndexList, ArrayAttr],
+) -> Tuple[List[ValueLike], Union[List[int], ArrayAttr], List[bool]]:
+    """Dispatches a list of indices to the appropriate form.
+
+    This is similar to the custom `DynamicIndexList` directive upstream:
+    provided indices may be in the form of dynamic SSA values or static values,
+    and they may be scalable (i.e., as a singleton list) or not. This function
+    dispatches each index into its respective form. It also extracts the SSA
+    values and static indices from various similar structures, respectively.
+    """
+    dynamic_indices = []
+    static_indices = [ShapedType.get_dynamic_size()] * len(indices)
+    scalable_indices = [False] * len(indices)
+
+    # ArrayAttr: Extract index values.
+    if isinstance(indices, ArrayAttr):
+        indices = [idx for idx in indices]
+
+    def process_nonscalable_index(i, index):
+        """Processes any form of non-scalable index.
+
+        Returns False if the given index was scalable and thus remains
+        unprocessed; True otherwise.
+        """
+        if isinstance(index, int):
+            static_indices[i] = index
+        elif isinstance(index, IntegerAttr):
+            static_indices[i] = index.value  # pytype: disable=attribute-error
+        elif isinstance(index, (Operation, Value, OpView)):
+            dynamic_indices.append(index)
+        else:
+            return False
+        return True
+
+    # Process each index at a time.
+    for i, index in enumerate(indices):
+        if not process_nonscalable_index(i, index):
+            # If it wasn't processed, it must be a scalable index, which is
+            # provided as a Sequence of one value, so extract and process that.
+            scalable_indices[i] = True
+            assert len(index) == 1
+            ret = process_nonscalable_index(i, index[0])
+            assert ret
+
+    return dynamic_indices, static_indices, scalable_indices
+
+
+# Dispatches `MixedValues` that all represents integers in various forms into
+# the following three categories:
+#   - `dynamic_values`: a list of `Value`s, potentially from op results;
+#   - `packed_values`: a value handle, potentially from an op result, associated
+#                      to one or more payload operations of integer type;
+#   - `static_values`: an `ArrayAttr` of `i64`s with static values, from Python
+#                      `int`s, from `IntegerAttr`s, or from an `ArrayAttr`.
+# The input is in the form for `packed_values`, only that result is set and the
+# other two are empty. Otherwise, the input can be a mix of the other two forms,
+# and for each dynamic value, a special value is added to the `static_values`.
+def _dispatch_mixed_values(
+    values: MixedValues,
+) -> Tuple[List[Value], Union[Operation, Value, OpView], DenseI64ArrayAttr]:
+    dynamic_values = []
+    packed_values = None
+    static_values = None
+    if isinstance(values, ArrayAttr):
+        static_values = values
+    elif isinstance(values, (Operation, Value, OpView)):
+        packed_values = values
+    else:
+        static_values = []
+        for size in values or []:
+            if isinstance(size, int):
+                static_values.append(size)
+            else:
+                static_values.append(ShapedType.get_dynamic_size())
+                dynamic_values.append(size)
+        static_values = DenseI64ArrayAttr.get(static_values)
+
+    return (dynamic_values, packed_values, static_values)
+
+
+def _get_value_or_attribute_value(
+    value_or_attr: Union[any, Attribute, ArrayAttr]
+) -> any:
+    if isinstance(value_or_attr, Attribute) and hasattr(value_or_attr, "value"):
+        return value_or_attr.value
+    if isinstance(value_or_attr, ArrayAttr):
+        return _get_value_list(value_or_attr)
+    return value_or_attr
+
+
+def _get_value_list(
+    sequence_or_array_attr: Union[Sequence[any], ArrayAttr]
+) -> Sequence[any]:
+    return [_get_value_or_attribute_value(v) for v in sequence_or_array_attr]
+
+
+def _get_int_array_attr(values: Optional[Union[ArrayAttr, IntOrAttrList]]) -> ArrayAttr:
+    if values is None:
+        return None
+
+    # Turn into a Python list of Python ints.
+    values = _get_value_list(values)
+
+    # Make an ArrayAttr of IntegerAttrs out of it.
+    return ArrayAttr.get(
+        [IntegerAttr.get(IntegerType.get_signless(64), v) for v in values]
+    )
+
+
+def _get_int_array_array_attr(
+    values: Optional[Union[ArrayAttr, Sequence[Union[ArrayAttr, IntOrAttrList]]]]
+) -> ArrayAttr:
+    """Creates an ArrayAttr of ArrayAttrs of IntegerAttrs.
+
+    The input has to be a collection of collection of integers, where any
+    Python Sequence and ArrayAttr are admissible collections and Python ints and
+    any IntegerAttr are admissible integers. Both levels of collections are
+    turned into ArrayAttr; the inner level is turned into IntegerAttrs of i64s.
+    If the input is None, an empty ArrayAttr is returned.
+    """
+    if values is None:
+        return None
+
+    # Make sure the outer level is a list.
+    values = _get_value_list(values)
+
+    # The inner level is now either invalid or a mixed sequence of ArrayAttrs and
+    # Sequences. Make sure the nested values are all lists.
+    values = [_get_value_list(nested) for nested in values]
+
+    # Turn each nested list into an ArrayAttr.
+    values = [_get_int_array_attr(nested) for nested in values]
+
+    # Turn the outer list into an ArrayAttr.
+    return ArrayAttr.get(values)
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class BufferizeToAllocationOp(BufferizeToAllocationOp):
+    """Specialization for BufferizeToAllocationOp class."""
+
+    def __init__(
+        self,
+        target: Union[Operation, OpView, Value],
+        *,
+        memory_space: Optional[Union[int, str, Attribute]] = None,
+        memcpy_op: Optional[str] = None,
+        alloc_op: Optional[str] = None,
+        bufferize_destination_only: Optional[bool] = None,
+        loc=None,
+        ip=None,
+    ):
+        # No other types are allowed, so hard-code those here.
+        allocated_buffer_type = transform.AnyValueType.get()
+        new_ops_type = transform.AnyOpType.get()
+
+        if isinstance(memory_space, int):
+            memory_space = str(memory_space)
+        if isinstance(memory_space, str):
+            memory_space = Attribute.parse(memory_space)
+
+        super().__init__(
+            allocated_buffer_type,
+            new_ops_type,
+            target,
+            memory_space=memory_space,
+            memcpy_op=memcpy_op,
+            alloc_op=alloc_op,
+            bufferize_destination_only=bufferize_destination_only,
+            loc=loc,
+            ip=ip,
+        )
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class DecomposeOp(DecomposeOp):
+    """Specialization for DecomposeOp class."""
+
+    def __init__(self, target: Union[Operation, Value], *, loc=None, ip=None):
+        transformed_type = transform.AnyOpType.get()
+        super().__init__(transformed_type, target, loc=loc, ip=ip)
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class FuseIntoContainingOp(FuseIntoContainingOp):
+    """Specialization for FuseIntoContainingOp class."""
+
+    @overload
+    def __init__(
+        self,
+        fused_op_type: Type,
+        new_containing_op_type: Type,
+        producer_op: Union[Operation, OpView, Value],
+        containing_op: Union[Operation, OpView, Value],
+        *,
+        loc=None,
+        ip=None,
+    ):
+        ...
+
+    @overload
+    def __init__(
+        self,
+        producer_op: Union[Operation, OpView, Value],
+        containing_op: Union[Operation, OpView, Value],
+        *,
+        loc=None,
+        ip=None,
+    ):
+        ...
+
+    def __init__(
+        self,
+        fused_op_type_or_producer_op: Union[Operation, OpView, Type, Value],
+        new_containing_op_type_or_containing_op: Union[Operation, OpView, Type, Value],
+        producer_op_or_none: Optional[Union[Operation, OpView, Value]] = None,
+        containing_op_or_none: Optional[Union[Operation, OpView, Value]] = None,
+        *,
+        loc=None,
+        ip=None,
+    ):
+        if isinstance(fused_op_type_or_producer_op, Type):
+            if not isinstance(new_containing_op_type_or_containing_op, Type):
+                raise TypeError(
+                    "If 'fused_op_type_or_producer_op' is a type, then "
+                    "'new_containing_op_type_or_containing_op' is expected "
+                    "to be one as well."
+                )
+            fused_op_type = fused_op_type_or_producer_op
+            new_containing_op_type = new_containing_op_type_or_containing_op
+            producer_op = producer_op_or_none
+            containing_op = containing_op_or_none
+        else:
+            fused_op_type = transform.AnyOpType.get()
+            new_containing_op_type = transform.AnyOpType.get()
+            producer_op = fused_op_type_or_producer_op
+            containing_op = new_containing_op_type_or_containing_op
+
+        super().__init__(
+            fused_op_type,
+            new_containing_op_type,
+            producer_op,
+            containing_op,
+            loc=loc,
+            ip=ip,
+        )
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class GeneralizeOp(GeneralizeOp):
+    """Specialization for GeneralizeOp class."""
+
+    def __init__(self, target: Union[Operation, Value], *, loc=None, ip=None):
+        transformed_type = transform.AnyOpType.get()
+        super().__init__(transformed_type, target, loc=loc, ip=ip)
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class InterchangeOp(InterchangeOp):
+    """Specialization for InterchangeOp class."""
+
+    def __init__(
+        self,
+        target: Union[Operation, Value],
+        *,
+        iterator_interchange: OptionalIntList = None,
+        loc=None,
+        ip=None,
+    ):
+        transformed_type = transform.AnyOpType.get()
+        super().__init__(
+            transformed_type,
+            target,
+            iterator_interchange=iterator_interchange,
+            loc=loc,
+            ip=ip,
+        )
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class MapCopyToThreadsOp(MapCopyToThreadsOp):
+    """Specialization for MapCopyToThreadsOp class."""
+
+    @overload
+    def __init__(
+        self,
+        forall_op_type: Type,
+        tiled_op_type: Type,
+        target: Union[Operation, OpView, Value],
+        *,
+        total_num_threads: Union[int, IntegerAttr],
+        desired_bit_alignment: Union[int, IntegerAttr],
+        loc=None,
+        ip=None,
+    ):
+        ...
+
+    @overload
+    def __init__(
+        self,
+        target: Union[Operation, OpView, Value],
+        *,
+        total_num_threads: Union[int, IntegerAttr],
+        desired_bit_alignment: Union[int, IntegerAttr],
+        loc=None,
+        ip=None,
+    ):
+        ...
+
+    def __init__(
+        self,
+        forall_op_type_or_target: Union[Operation, OpView, Type, Value],
+        tiled_op_type_or_none: Optional[Type] = None,
+        target_or_none: Optional[Union[Operation, OpView, Value]] = None,
+        *,
+        total_num_threads: Union[int, IntegerAttr],
+        desired_bit_alignment: Union[int, IntegerAttr],
+        loc=None,
+        ip=None,
+    ):
+        if isinstance(forall_op_type_or_target, Type):
+            forall_op_type = forall_op_type_or_target
+            tiled_op_type = tiled_op_type_or_none
+            target = target_or_none
+        else:
+            forall_op_type = transform.AnyOpType.get()
+            tiled_op_type = transform.AnyOpType.get()
+            target = forall_op_type_or_target
+
+        super().__init__(
+            forall_op_type,
+            tiled_op_type,
+            target,
+            total_num_threads=total_num_threads,
+            desired_bit_alignment=desired_bit_alignment,
+            loc=loc,
+            ip=ip,
+        )
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class VectorizeOp(VectorizeOp):
+    """Specialization for VectorizeOp class."""
+
+    def __init__(
+        self,
+        target: Union[Operation, OpView, Value],
+        vector_sizes: Optional[Union[DynamicIndexList, ArrayAttr]] = None,
+        *,
+        vectorize_nd_extract: Optional[bool] = None,
+        scalable_sizes: OptionalBoolList = None,
+        static_vector_sizes: OptionalIntList = None,
+        loc=None,
+        ip=None,
+    ):
+        if (
+            scalable_sizes is None
+            and static_vector_sizes is None
+            and vector_sizes is None
+        ):
+            dynamic_vector_sizes = []
+        elif scalable_sizes is None and static_vector_sizes is None:
+            (
+                dynamic_vector_sizes,
+                static_vector_sizes,
+                scalable_sizes,
+            ) = _dispatch_dynamic_index_list(vector_sizes)
+        elif scalable_sizes is None or static_vector_sizes is None:
+            raise TypeError(
+                "'scalable_sizes' and 'static_vector_sizes' must either both "
+                "be given explicitly or both be given as part of 'vector_sizes'."
+            )
+        else:
+            dynamic_vector_sizes = vector_sizes
+
+        super().__init__(
+            target,
+            vector_sizes=dynamic_vector_sizes,
+            static_vector_sizes=static_vector_sizes,
+            scalable_sizes=scalable_sizes,
+            vectorize_nd_extract=vectorize_nd_extract,
+            loc=loc,
+            ip=ip,
+        )
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class MatchOp(MatchOp):
+    """Specialization for MatchOp class."""
+
+    @overload
+    @classmethod
+    def match_op_names(
+        cls,
+        target: Union[Operation, Value],
+        names: Union[str, Sequence[str]],
+        *,
+        loc=None,
+        ip=None,
+    ):
+        ...
+
+    @overload
+    @classmethod
+    def match_op_names(
+        cls,
+        result_type: Type,
+        target: Union[Operation, Value],
+        names: Union[str, Sequence[str]],
+        *,
+        loc=None,
+        ip=None,
+    ):
+        ...
+
+    @classmethod
+    def match_op_names(
+        cls,
+        result_type_or_target: Union[Type, Operation, Value],
+        target_or_names: Union[Operation, Value, Sequence[str], str],
+        names_or_none: Optional[Union[Sequence[str], str]] = None,
+        *,
+        loc=None,
+        ip=None,
+    ):
+        if isinstance(result_type_or_target, Type):
+            result_type = result_type_or_target
+            target = target_or_names
+            names = names_or_none
+        else:
+            result_type = transform.AnyOpType.get()
+            target = result_type_or_target
+            names = target_or_names
+
+        if isinstance(names, str):
+            names = [names]
+
+        return cls(
+            result_type,
+            target,
+            ops=ArrayAttr.get(list(map(lambda s: StringAttr.get(s), names))),
+            loc=loc,
+            ip=ip,
+        )
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class MultiTileSizesOp(MultiTileSizesOp):
+    """Specialization for MultiTileSizesOp class."""
+
+    def __init__(
+        self,
+        result_type: Type,
+        target: Union[Operation, Value],
+        *,
+        dimension: Union[int, IntegerAttr],
+        target_size: Union[int, IntegerAttr],
+        divisor: Optional[Optional[Union[int, IntegerAttr]]] = None,
+        loc=None,
+        ip=None,
+    ):
+        super().__init__(
+            result_type,
+            result_type,
+            result_type,
+            target,
+            dimension=dimension,
+            target_size=target_size,
+            divisor=divisor,
+            loc=loc,
+            ip=ip,
+        )
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class PadOp(PadOp):
+    """Specialization for PadOp class."""
+
+    def __init__(
+        self,
+        target: Union[Operation, OpView, Value],
+        *,
+        padding_values: Optional[Union[ArrayAttr, Sequence[Attribute]]] = None,
+        padding_dimensions: OptionalIntList = None,
+        pad_to_multiple_of: OptionalIntList = None,
+        pack_paddings: OptionalIntList = None,
+        transpose_paddings: Optional[
+            Union[ArrayAttr, Sequence[Union[ArrayAttr, IntOrAttrList]]]
+        ] = None,
+        copy_back_op: Optional[Union[str, StringAttr]] = None,
+        loc=None,
+        ip=None,
+    ):
+        transpose_paddings = _get_int_array_array_attr(transpose_paddings)
+
+        any_op_type = transform.AnyOpType.get()
+        super().__init__(
+            any_op_type,
+            any_op_type,
+            any_op_type,
+            target,
+            padding_values=padding_values,
+            padding_dimensions=padding_dimensions,
+            pad_to_multiple_of=pad_to_multiple_of,
+            pack_paddings=pack_paddings,
+            transpose_paddings=transpose_paddings,
+            copy_back_op=copy_back_op,
+            loc=loc,
+            ip=ip,
+        )
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class ScalarizeOp(ScalarizeOp):
+    """Specialization for ScalarizeOp class."""
+
+    def __init__(self, target: Union[Operation, Value], *, loc=None, ip=None):
+        result_type = transform.AnyOpType.get()
+        super().__init__(result_type, target, loc=loc, ip=ip)
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class SplitOp(SplitOp):
+    """Specialization for SplitOp class."""
+
+    def __init__(
+        self,
+        target: Union[Operation, Value],
+        dimension: Union[int, Attribute],
+        split_point: Union[int, Operation, Value, Attribute],
+        *,
+        loc=None,
+        ip=None,
+    ):
+        if isinstance(split_point, int):
+            static_split_point = split_point
+            dynamic_split_point = None
+        else:
+            static_split_point = ShapedType.get_dynamic_size()
+            dynamic_split_point = split_point
+
+        super().__init__(
+            target.type,
+            target.type,
+            target,
+            dimension=dimension,
+            static_split_point=static_split_point,
+            dynamic_split_point=dynamic_split_point,
+            loc=loc,
+            ip=ip,
+        )
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class TileUsingForOp(TileUsingForOp):
+    """Specialization for TileUsingForOp class."""
+
+    @overload
+    def __init__(
+        self,
+        loop_types: Union[Type, List[Type]],
+        target: Union[Operation, Value],
+        *,
+        sizes: Optional[Union[DynamicIndexList, ArrayAttr]] = None,
+        interchange: OptionalIntList = None,
+        loc=None,
+        ip=None,
+    ):
+        ...
+
+    @overload
+    def __init__(
+        self,
+        target: Union[Operation, Value, OpView],
+        *,
+        sizes: Optional[Union[DynamicIndexList, ArrayAttr]] = None,
+        interchange: OptionalIntList = None,
+        loc=None,
+        ip=None,
+    ):
+        ...
+
+    def __init__(
+        self,
+        loop_types_or_target: Union[Type, List[Type], Operation, Value],
+        target_or_none: Optional[Union[Operation, Value, OpView]] = None,
+        *,
+        sizes: Optional[Union[DynamicIndexList, ArrayAttr]] = None,
+        interchange: OptionalIntList = None,
+        loc=None,
+        ip=None,
+    ):
+        (
+            dynamic_sizes,
+            static_sizes,
+            scalable_sizes,
+        ) = _dispatch_dynamic_index_list(sizes)
+
+        num_loops = sum(v if v == 0 else 1 for v in static_sizes)
+
+        if isinstance(loop_types_or_target, (Operation, Value, OpView)):
+            loop_types = [transform.AnyOpType.get()] * num_loops
+            target = loop_types_or_target
+            assert (
+                target_or_none is None
+            ), "Cannot construct TileUsingForOp with two targets."
+        else:
+            loop_types = (
+                ([loop_types_or_target] * num_loops)
+                if isinstance(loop_types_or_target, Type)
+                else loop_types_or_target
+            )
+            target = target_or_none
+
+        super().__init__(
+            target.type,
+            loop_types,
+            target,
+            dynamic_sizes=dynamic_sizes,
+            static_sizes=static_sizes,
+            interchange=interchange,
+            scalable_sizes=scalable_sizes,
+            loc=loc,
+            ip=ip,
+        )
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class TileUsingForallOp(TileUsingForallOp):
+    """Specialization for TileUsingForallOp class."""
+
+    @overload
+    def __init__(
+        self,
+        loops_type: Type,
+        tiled_op_type: Type,
+        target: Union[Operation, Value, OpView],
+        *,
+        num_threads: Optional[MixedValues] = None,
+        tile_sizes: MixedValues = None,
+        mapping=None,
+        loc=None,
+        ip=None,
+    ):
+        ...
+
+    @overload
+    def __init__(
+        self,
+        target: Union[Operation, Value, OpView],
+        *,
+        num_threads: Optional[MixedValues] = None,
+        tile_sizes: MixedValues = None,
+        mapping=None,
+        loc=None,
+        ip=None,
+    ):
+        ...
+
+    def __init__(
+        self,
+        loops_type_or_target: Union[
+            Type, Union[Operation, Value, OpView]  # loops_type
+        ],  # target
+        tiled_op_type_or_none: Optional[Type] = None,
+        target_or_none: Optional[Union[Operation, Value, OpView]] = None,
+        *,
+        num_threads: MixedValues = None,
+        tile_sizes: MixedValues = None,
+        mapping=None,
+        loc=None,
+        ip=None,
+    ):
+        # `Type` arguments in the front are optional: add default values to front.
+        if isinstance(loops_type_or_target, Type):
+            # First overload: type arguments provided.
+            if not isinstance(tiled_op_type_or_none, Type):
+                raise TypeError(
+                    "If 'loops_type_or_target' is a type, then "
+                    "'tiled_op_type_or_none' is expected to be one as well."
+                )
+            loops_type = loops_type_or_target
+            tiled_op_type = tiled_op_type_or_none
+            target = target_or_none
+        else:
+            # Last overload: type arguments missing.
+            loops_type = transform.AnyOpType.get()
+            tiled_op_type = transform.AnyOpType.get()
+            target = loops_type_or_target
+
+        # Unpack mixed num_threads.
+        (
+            dynamic_num_threads,
+            packed_num_threads,
+            num_threads_attr,
+        ) = _dispatch_mixed_values(num_threads)
+
+        # Unpack mixed tile_sizes.
+        (
+            dynamic_tile_sizes,
+            packed_tile_sizes,
+            tile_sizes_attr,
+        ) = _dispatch_mixed_values(tile_sizes)
+
+        super().__init__(
+            loops_type,
+            tiled_op_type,
+            target=target,
+            tile_sizes=dynamic_tile_sizes,
+            packed_tile_sizes=packed_tile_sizes,
+            static_tile_sizes=tile_sizes_attr,
+            num_threads=dynamic_num_threads,
+            packed_num_threads=packed_num_threads,
+            static_num_threads=num_threads_attr,
+            mapping=mapping,
+            loc=loc,
+            ip=ip,
+        )
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class VectorizeChildrenAndApplyPatternsOp(VectorizeChildrenAndApplyPatternsOp):
+    """Specialization for VectorizeChildrenAndApplyPatternsOp class."""
+
+    def __init__(
+        self,
+        target: Union[Operation, Value],
+        *,
+        disable_multi_reduction_to_contract_patterns: bool = False,
+        disable_transfer_permutation_map_lowering_patterns: bool = False,
+        vectorize_nd_extract: bool = False,
+        vectorize_padding: bool = False,
+        loc=None,
+        ip=None,
+    ):
+        transformed_type = transform.AnyOpType.get()
+        super().__init__(
+            transformed_type,
+            target,
+            disable_multi_reduction_to_contract_patterns=disable_multi_reduction_to_contract_patterns,
+            disable_transfer_permutation_map_lowering_patterns=disable_transfer_permutation_map_lowering_patterns,
+            vectorize_nd_extract=vectorize_nd_extract,
+            vectorize_padding=vectorize_padding,
+            loc=loc,
+            ip=ip,
+        )
diff --git a/mlir/python/mlir/dialects/transform/tensor.py b/mlir/python/mlir/dialects/transform/tensor.py
index bf52255b3d..4eb30398f0 100644
--- a/mlir/python/mlir/dialects/transform/tensor.py
+++ b/mlir/python/mlir/dialects/transform/tensor.py
@@ -3,3 +3,67 @@
 #  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 from .._tensor_transform_ops_gen import *
+from .._tensor_transform_ops_gen import _Dialect
+
+try:
+    from ...ir import *
+    from ...dialects import transform
+    from .._ods_common import _cext as _ods_cext
+except ImportError as e:
+    raise RuntimeError("Error loading imports from extension module") from e
+
+from typing import Optional, overload, Union
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class MakeLoopIndependentOp(MakeLoopIndependentOp):
+    """Specialization for MakeLoopIndependentOp class."""
+
+    @overload
+    def __init__(
+        self,
+        transformed_type: Type,
+        target: Union[Operation, OpView, Value],
+        num_loops: Union[int, IntegerAttr],
+        *,
+        loc=None,
+        ip=None,
+    ):
+        ...
+
+    @overload
+    def __init__(
+        self,
+        target: Union[Operation, OpView, Value],
+        num_loops: Union[int, IntegerAttr],
+        *,
+        loc=None,
+        ip=None,
+    ):
+        ...
+
+    def __init__(
+        self,
+        transformed_type_or_target: Type,
+        target_or_num_loops: Union[int, IntegerAttr, Operation, OpView, Value] = None,
+        num_loops_or_none: Optional[Union[int, IntegerAttr]] = None,
+        *,
+        loc=None,
+        ip=None,
+    ):
+        if isinstance(transformed_type_or_target, Type):
+            transformed_type = transformed_type_or_target
+            target = target_or_num_loops
+            num_loops = num_loops_or_none
+        else:
+            transformed_type = transform.AnyOpType.get()
+            target = transformed_type_or_target
+            num_loops = target_or_num_loops
+
+        super().__init__(
+            transformed_type,
+            target,
+            num_loops,
+            loc=loc,
+            ip=ip,
+        )
diff --git a/mlir/python/mlir/runtime/np_to_memref.py b/mlir/python/mlir/runtime/np_to_memref.py
index 0a3b411041..f6b706f9bc 100644
--- a/mlir/python/mlir/runtime/np_to_memref.py
+++ b/mlir/python/mlir/runtime/np_to_memref.py
@@ -114,6 +114,7 @@ def get_unranked_memref_descriptor(nparray):
     d.descriptor = ctypes.cast(ctypes.pointer(x), ctypes.c_void_p)
     return d
 
+
 def move_aligned_ptr_by_offset(aligned_ptr, offset):
     """Moves the supplied ctypes pointer ahead by `offset` elements."""
     aligned_addr = ctypes.addressof(aligned_ptr.contents)
@@ -122,6 +123,7 @@ def move_aligned_ptr_by_offset(aligned_ptr, offset):
     content_ptr = ctypes.cast(aligned_addr + shift, type(aligned_ptr))
     return content_ptr
 
+
 def unranked_memref_to_numpy(unranked_memref, np_dtype):
     """Converts unranked memrefs to numpy arrays."""
     ctp = as_ctype(np_dtype)
@@ -139,10 +141,10 @@ def unranked_memref_to_numpy(unranked_memref, np_dtype):
 
 def ranked_memref_to_numpy(ranked_memref):
     """Converts ranked memrefs to numpy arrays."""
-    content_ptr = move_aligned_ptr_by_offset(ranked_memref[0].aligned, ranked_memref[0].offset)
-    np_arr = np.ctypeslib.as_array(
-        content_ptr, shape=ranked_memref[0].shape
+    content_ptr = move_aligned_ptr_by_offset(
+        ranked_memref[0].aligned, ranked_memref[0].offset
     )
+    np_arr = np.ctypeslib.as_array(content_ptr, shape=ranked_memref[0].shape)
     strided_arr = np.lib.stride_tricks.as_strided(
         np_arr,
         np.ctypeslib.as_array(ranked_memref[0].shape),
diff --git a/mlir/test/Analysis/DataFlow/test-dead-code-analysis.mlir b/mlir/test/Analysis/DataFlow/test-dead-code-analysis.mlir
index b1af670440..70becc72be 100644
--- a/mlir/test/Analysis/DataFlow/test-dead-code-analysis.mlir
+++ b/mlir/test/Analysis/DataFlow/test-dead-code-analysis.mlir
@@ -258,3 +258,10 @@ func.func @test_call_dead_return(%arg0: i32) -> () {
   %0 = func.call @test_dead_return(%arg0) {tag = "test_dead_return"} : (i32) -> i32
   return
 }
+
+func.func @test_dca_doesnt_crash() -> () {
+  %0 = scf.execute_region -> tensor<5x16xi16> {
+    llvm.unreachable
+  }  
+  return
+}
diff --git a/mlir/test/CAPI/ir.c b/mlir/test/CAPI/ir.c
index a181332e21..c6425f80a8 100644
--- a/mlir/test/CAPI/ir.c
+++ b/mlir/test/CAPI/ir.c
@@ -2210,6 +2210,52 @@ int testSymbolTable(MlirContext ctx) {
   return 0;
 }
 
+typedef struct {
+  const char *x;
+} callBackData;
+
+void walkCallBack(MlirOperation op, void *rootOpVoid) {
+  fprintf(stderr, "%s: %s\n", ((callBackData *)(rootOpVoid))->x,
+          mlirIdentifierStr(mlirOperationGetName(op)).data);
+}
+
+int testOperationWalk(MlirContext ctx) {
+  // CHECK-LABEL: @testOperationWalk
+  fprintf(stderr, "@testOperationWalk\n");
+
+  const char *moduleString = "module {\n"
+                             "  func.func @foo() {\n"
+                             "    %1 = arith.constant 10: i32\n"
+                             "    arith.addi %1, %1: i32\n"
+                             "    return\n"
+                             "  }\n"
+                             "}";
+  MlirModule module =
+      mlirModuleCreateParse(ctx, mlirStringRefCreateFromCString(moduleString));
+
+  callBackData data;
+  data.x = "i love you";
+
+  // CHECK: i love you: arith.constant
+  // CHECK: i love you: arith.addi
+  // CHECK: i love you: func.return
+  // CHECK: i love you: func.func
+  // CHECK: i love you: builtin.module
+  mlirOperationWalk(mlirModuleGetOperation(module), walkCallBack,
+                    (void *)(&data), MlirWalkPostOrder);
+
+  data.x = "i don't love you";
+  // CHECK: i don't love you: builtin.module
+  // CHECK: i don't love you: func.func
+  // CHECK: i don't love you: arith.constant
+  // CHECK: i don't love you: arith.addi
+  // CHECK: i don't love you: func.return
+  mlirOperationWalk(mlirModuleGetOperation(module), walkCallBack,
+                    (void *)(&data), MlirWalkPreOrder);
+  mlirModuleDestroy(module);
+  return 0;
+}
+
 int testDialectRegistry(void) {
   fprintf(stderr, "@testDialectRegistry\n");
 
@@ -2349,6 +2395,8 @@ int main(void) {
     return 14;
   if (testDialectRegistry())
     return 15;
+  if (testOperationWalk(ctx))
+    return 16;
 
   testExplicitThreadPools();
   testDiagnostics();
diff --git a/mlir/test/CAPI/sparse_tensor.c b/mlir/test/CAPI/sparse_tensor.c
index 33ee8e7840..3bd1508cf2 100644
--- a/mlir/test/CAPI/sparse_tensor.c
+++ b/mlir/test/CAPI/sparse_tensor.c
@@ -40,6 +40,8 @@ static int testRoundtripEncoding(MlirContext ctx) {
   // CHECK: level_type: 4
   // CHECK: level_type: 8
   // CHECK: level_type: 8
+  MlirAffineMap lvlToDim =
+      mlirSparseTensorEncodingAttrGetLvlToDim(originalAttr);
   int lvlRank = mlirSparseTensorEncodingGetLvlRank(originalAttr);
   enum MlirSparseTensorDimLevelType *lvlTypes =
       malloc(sizeof(enum MlirSparseTensorDimLevelType) * lvlRank);
@@ -53,9 +55,8 @@ static int testRoundtripEncoding(MlirContext ctx) {
   // CHECK: crdWidth: 64
   int crdWidth = mlirSparseTensorEncodingAttrGetCrdWidth(originalAttr);
   fprintf(stderr, "crdWidth: %d\n", crdWidth);
-  // TODO: lvlToDim
   MlirAttribute newAttr = mlirSparseTensorEncodingAttrGet(
-      ctx, lvlRank, lvlTypes, dimToLvl, posWidth, crdWidth);
+      ctx, lvlRank, lvlTypes, dimToLvl, lvlToDim, posWidth, crdWidth);
   mlirAttributeDump(newAttr); // For debugging filecheck output.
   // CHECK: equal: 1
   fprintf(stderr, "equal: %d\n", mlirAttributeEqual(originalAttr, newAttr));
diff --git a/mlir/test/CMakeLists.txt b/mlir/test/CMakeLists.txt
index bf143d036c..6fc9ae0f3f 100644
--- a/mlir/test/CMakeLists.txt
+++ b/mlir/test/CMakeLists.txt
@@ -104,6 +104,7 @@ set(MLIR_TEST_DEPENDS
   mlir-pdll-lsp-server
   mlir-opt
   mlir-pdll
+  mlir-query
   mlir-reduce
   mlir-tblgen
   mlir-translate
diff --git a/mlir/test/Conversion/ArmSMEToSCF/arm-sme-to-scf.mlir b/mlir/test/Conversion/ArmSMEToSCF/arm-sme-to-scf.mlir
index 09f148bcd4..4b3020970d 100644
--- a/mlir/test/Conversion/ArmSMEToSCF/arm-sme-to-scf.mlir
+++ b/mlir/test/Conversion/ArmSMEToSCF/arm-sme-to-scf.mlir
@@ -21,10 +21,10 @@ func.func @arm_sme_tile_load_hor(%src : memref<?x?xi32>) {
 // -----
 
 // CHECK-LABEL: @arm_sme_tile_load_ver
-// CHECK: arm_sme.load_tile_slice {{.*}} <vertical>
+// CHECK: arm_sme.load_tile_slice {{.*}} layout<vertical>
 func.func @arm_sme_tile_load_ver(%src : memref<?x?xi32>) {
   %c0 = arith.constant 0 : index
-  %tile = arm_sme.tile_load %src[%c0, %c0], <vertical> : memref<?x?xi32>, vector<[4]x[4]xi32>
+  %tile = arm_sme.tile_load %src[%c0, %c0] layout<vertical> : memref<?x?xi32>, vector<[4]x[4]xi32>
   return
 }
 
@@ -50,10 +50,10 @@ func.func @arm_sme_tile_store_hor(%tile : vector<[4]x[4]xi32>, %dest : memref<?x
 // -----
 
 // CHECK-LABEL: @arm_sme_tile_store_ver
-// CHECK: arm_sme.store_tile_slice {{.*}} <vertical>
+// CHECK: arm_sme.store_tile_slice {{.*}} layout<vertical>
 func.func @arm_sme_tile_store_ver(%tile : vector<[4]x[4]xi32>, %dest : memref<?x?xi32>) {
   %c0 = arith.constant 0 : index
-  arm_sme.tile_store %tile, %dest[%c0, %c0], <vertical> : memref<?x?xi32>, vector<[4]x[4]xi32>
+  arm_sme.tile_store %tile, %dest[%c0, %c0] layout<vertical> : memref<?x?xi32>, vector<[4]x[4]xi32>
   return
 }
 
diff --git a/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm.mlir b/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm.mlir
index 391ccd7484..a8c02e32ef 100644
--- a/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm.mlir
+++ b/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm.mlir
@@ -542,16 +542,15 @@ gpu.module @test_module_28 {
 gpu.module @test_module_29 {
   // CHECK-DAG: llvm.mlir.global internal constant @[[$PRINT_GLOBAL0:[A-Za-z0-9_]+]]("Hello, world\0A\00")
   // CHECK-DAG: llvm.mlir.global internal constant @[[$PRINT_GLOBAL1:[A-Za-z0-9_]+]]("Hello: %d\0A\00")
-  // CHECK-DAG: llvm.func @vprintf(!llvm.ptr<i8>, !llvm.ptr<i8>) -> i32
+  // CHECK-DAG: llvm.func @vprintf(!llvm.ptr, !llvm.ptr) -> i32
 
   // CHECK-LABEL: func @test_const_printf
   gpu.func @test_const_printf() {
-    // CHECK-NEXT: %[[FORMATSTR:.*]] = llvm.mlir.addressof @[[$PRINT_GLOBAL0]] : !llvm.ptr<array<14 x i8>>
-    // CHECK-NEXT: %[[FORMATSTART:.*]] = llvm.getelementptr %[[FORMATSTR]][0, 0] : (!llvm.ptr<array<14 x i8>>) -> !llvm.ptr<i8>
+    // CHECK-NEXT: %[[FORMATSTR:.*]] = llvm.mlir.addressof @[[$PRINT_GLOBAL0]] : !llvm.ptr
+    // CHECK-NEXT: %[[FORMATSTART:.*]] = llvm.getelementptr %[[FORMATSTR]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.array<14 x i8>
     // CHECK-NEXT: %[[O:.*]] = llvm.mlir.constant(1 : index) : i64
-    // CHECK-NEXT: %[[ALLOC:.*]] = llvm.alloca %[[O]] x !llvm.struct<()> : (i64) -> !llvm.ptr<struct<()>>
-    // CHECK-NEXT: %[[ARGPTR:.*]] = llvm.bitcast %[[ALLOC]] : !llvm.ptr<struct<()>> to !llvm.ptr<i8>
-    // CHECK-NEXT: llvm.call @vprintf(%[[FORMATSTART]], %[[ARGPTR]]) : (!llvm.ptr<i8>, !llvm.ptr<i8>) -> i32
+    // CHECK-NEXT: %[[ALLOC:.*]] = llvm.alloca %[[O]] x !llvm.struct<()> : (i64) -> !llvm.ptr
+    // CHECK-NEXT: llvm.call @vprintf(%[[FORMATSTART]], %[[ALLOC]]) : (!llvm.ptr, !llvm.ptr) -> i32
     gpu.printf "Hello, world\n"
     gpu.return
   }
@@ -559,17 +558,16 @@ gpu.module @test_module_29 {
   // CHECK-LABEL: func @test_printf
   // CHECK: (%[[ARG0:.*]]: i32, %[[ARG1:.*]]: f32)
   gpu.func @test_printf(%arg0: i32, %arg1: f32) {
-    // CHECK-NEXT: %[[FORMATSTR:.*]] = llvm.mlir.addressof @[[$PRINT_GLOBAL1]] : !llvm.ptr<array<11 x i8>>
-    // CHECK-NEXT: %[[FORMATSTART:.*]] = llvm.getelementptr %[[FORMATSTR]][0, 0] : (!llvm.ptr<array<11 x i8>>) -> !llvm.ptr<i8>
+    // CHECK-NEXT: %[[FORMATSTR:.*]] = llvm.mlir.addressof @[[$PRINT_GLOBAL1]] : !llvm.ptr
+    // CHECK-NEXT: %[[FORMATSTART:.*]] = llvm.getelementptr %[[FORMATSTR]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.array<11 x i8>
     // CHECK-NEXT: %[[EXT:.+]] = llvm.fpext %[[ARG1]] : f32 to f64
     // CHECK-NEXT: %[[O:.*]] = llvm.mlir.constant(1 : index) : i64
-    // CHECK-NEXT: %[[ALLOC:.*]] = llvm.alloca %[[O]] x !llvm.struct<(i32, f64)> : (i64) -> !llvm.ptr<struct<(i32, f64)>>
-    // CHECK-NEXT: %[[EL0:.*]] = llvm.getelementptr %[[ALLOC]][0, 0] : (!llvm.ptr<struct<(i32, f64)>>) -> !llvm.ptr<i32>
-    // CHECK-NEXT: llvm.store %[[ARG0]], %[[EL0]] : !llvm.ptr<i32>
-    // CHECK-NEXT: %[[EL1:.*]] = llvm.getelementptr %[[ALLOC]][0, 1] : (!llvm.ptr<struct<(i32, f64)>>) -> !llvm.ptr<f64>
-    // CHECK-NEXT: llvm.store %[[EXT]], %[[EL1]] : !llvm.ptr<f64>
-    // CHECK-NEXT: %[[ARGPTR:.*]] = llvm.bitcast %[[ALLOC]] : !llvm.ptr<struct<(i32, f64)>> to !llvm.ptr<i8>
-    // CHECK-NEXT: llvm.call @vprintf(%[[FORMATSTART]], %[[ARGPTR]]) : (!llvm.ptr<i8>, !llvm.ptr<i8>) -> i32
+    // CHECK-NEXT: %[[ALLOC:.*]] = llvm.alloca %[[O]] x !llvm.struct<(i32, f64)> : (i64) -> !llvm.ptr
+    // CHECK-NEXT: %[[EL0:.*]] = llvm.getelementptr %[[ALLOC]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(i32, f64)>
+    // CHECK-NEXT: llvm.store %[[ARG0]], %[[EL0]] : i32, !llvm.ptr
+    // CHECK-NEXT: %[[EL1:.*]] = llvm.getelementptr %[[ALLOC]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(i32, f64)>
+    // CHECK-NEXT: llvm.store %[[EXT]], %[[EL1]] : f64, !llvm.ptr
+    // CHECK-NEXT: llvm.call @vprintf(%[[FORMATSTART]], %[[ALLOC]]) : (!llvm.ptr, !llvm.ptr) -> i32
     gpu.printf "Hello: %d\n" %arg0, %arg1 : i32, f32
     gpu.return
   }
diff --git a/mlir/test/Conversion/IndexToSPRIV/index-to-spirv.mlir b/mlir/test/Conversion/IndexToSPRIV/index-to-spirv.mlir
new file mode 100644
index 0000000000..53dc896e98
--- /dev/null
+++ b/mlir/test/Conversion/IndexToSPRIV/index-to-spirv.mlir
@@ -0,0 +1,222 @@
+// RUN: mlir-opt %s -convert-index-to-spirv | FileCheck %s
+// RUN: mlir-opt %s -convert-index-to-spirv=use-64bit-index=false | FileCheck %s --check-prefix=INDEX32
+// RUN: mlir-opt %s -convert-index-to-spirv=use-64bit-index=true | FileCheck %s --check-prefix=INDEX64
+
+module attributes {
+  spirv.target_env = #spirv.target_env<#spirv.vce<v1.0, [Int64], []>, #spirv.resource_limits<>>
+} {
+// CHECK-LABEL: @trivial_ops
+func.func @trivial_ops(%a: index, %b: index) {
+  // CHECK: spirv.IAdd
+  %0 = index.add %a, %b
+  // CHECK: spirv.ISub
+  %1 = index.sub %a, %b
+  // CHECK: spirv.IMul
+  %2 = index.mul %a, %b
+  // CHECK: spirv.SDiv
+  %3 = index.divs %a, %b
+  // CHECK: spirv.UDiv
+  %4 = index.divu %a, %b
+  // CHECK: spirv.SRem
+  %5 = index.rems %a, %b
+  // CHECK: spirv.UMod
+  %6 = index.remu %a, %b
+  // CHECK: spirv.GL.SMax
+  %7 = index.maxs %a, %b
+  // CHECK: spirv.GL.UMax
+  %8 = index.maxu %a, %b
+  // CHECK: spirv.GL.SMin
+  %9 = index.mins %a, %b
+  // CHECK: spirv.GL.UMin
+  %10 = index.minu %a, %b
+  // CHECK: spirv.ShiftLeftLogical
+  %11 = index.shl %a, %b
+  // CHECK: spirv.ShiftRightArithmetic
+  %12 = index.shrs %a, %b
+  // CHECK: spirv.ShiftRightLogical
+  %13 = index.shru %a, %b
+  return
+}
+
+// CHECK-LABEL: @bitwise_ops
+func.func @bitwise_ops(%a: index, %b: index) {
+  // CHECK: spirv.BitwiseAnd
+  %0 = index.and %a, %b
+  // CHECK: spirv.BitwiseOr
+  %1 = index.or %a, %b
+  // CHECK: spirv.BitwiseXor
+  %2 = index.xor %a, %b
+  return
+}
+
+// INDEX32-LABEL: @constant_ops
+// INDEX64-LABEL: @constant_ops
+func.func @constant_ops() {
+  // INDEX32: spirv.Constant 42 : i32
+  // INDEX64: spirv.Constant 42 : i64
+  %0 = index.constant 42
+  // INDEX32: spirv.Constant true
+  // INDEX64: spirv.Constant true
+  %1 = index.bool.constant true
+  // INDEX32: spirv.Constant false
+  // INDEX64: spirv.Constant false
+  %2 = index.bool.constant false
+  return
+}
+
+// CHECK-LABEL: @ceildivs
+// CHECK-SAME: %[[NI:.*]]: index, %[[MI:.*]]: index
+func.func @ceildivs(%n: index, %m: index) -> index {
+  // CHECK: %[[N:.*]] = builtin.unrealized_conversion_cast %[[NI]]
+  // CHECK: %[[M:.*]] = builtin.unrealized_conversion_cast %[[MI]]
+  // CHECK: %[[ZERO:.*]] = spirv.Constant 0
+  // CHECK: %[[POS_ONE:.*]] = spirv.Constant 1
+  // CHECK: %[[NEG_ONE:.*]] = spirv.Constant -1
+
+  // CHECK: %[[M_POS:.*]] = spirv.SGreaterThan %[[M]], %[[ZERO]]
+  // CHECK: %[[X:.*]] = spirv.Select %[[M_POS]], %[[NEG_ONE]], %[[POS_ONE]]
+
+  // CHECK: %[[N_PLUS_X:.*]] = spirv.IAdd %[[N]], %[[X]]
+  // CHECK: %[[N_PLUS_X_DIV_M:.*]] = spirv.SDiv %[[N_PLUS_X]], %[[M]]
+  // CHECK: %[[POS_RES:.*]] = spirv.IAdd %[[N_PLUS_X_DIV_M]], %[[POS_ONE]]
+
+  // CHECK: %[[NEG_N:.*]] = spirv.ISub %[[ZERO]], %[[N]]
+  // CHECK: %[[NEG_N_DIV_M:.*]] = spirv.SDiv %[[NEG_N]], %[[M]]
+  // CHECK: %[[NEG_RES:.*]] = spirv.ISub %[[ZERO]], %[[NEG_N_DIV_M]]
+
+  // CHECK: %[[N_POS:.*]] = spirv.SGreaterThan %[[N]], %[[ZERO]]
+  // CHECK: %[[SAME_SIGN:.*]] = spirv.LogicalEqual %[[N_POS]], %[[M_POS]]
+  // CHECK: %[[N_NON_ZERO:.*]] = spirv.INotEqual %[[N]], %[[ZERO]]
+  // CHECK: %[[CMP:.*]] = spirv.LogicalAnd %[[SAME_SIGN]], %[[N_NON_ZERO]]
+  // CHECK: %[[RESULT:.*]] = spirv.Select %[[CMP]], %[[POS_RES]], %[[NEG_RES]]
+  %result = index.ceildivs %n, %m
+
+  // %[[RESULTI:.*] = builtin.unrealized_conversion_cast %[[RESULT]]
+  // return %[[RESULTI]]
+  return %result : index
+}
+
+// CHECK-LABEL: @ceildivu
+// CHECK-SAME: %[[NI:.*]]: index, %[[MI:.*]]: index
+func.func @ceildivu(%n: index, %m: index) -> index {
+  // CHECK: %[[N:.*]] = builtin.unrealized_conversion_cast %[[NI]]
+  // CHECK: %[[M:.*]] = builtin.unrealized_conversion_cast %[[MI]]
+  // CHECK: %[[ZERO:.*]] = spirv.Constant 0
+  // CHECK: %[[ONE:.*]] = spirv.Constant 1
+
+  // CHECK: %[[N_MINUS_ONE:.*]] = spirv.ISub %[[N]], %[[ONE]]
+  // CHECK: %[[N_MINUS_ONE_DIV_M:.*]] = spirv.UDiv %[[N_MINUS_ONE]], %[[M]]
+  // CHECK: %[[N_MINUS_ONE_DIV_M_PLUS_ONE:.*]] = spirv.IAdd %[[N_MINUS_ONE_DIV_M]], %[[ONE]]
+
+  // CHECK: %[[CMP:.*]] = spirv.IEqual %[[N]], %[[ZERO]]
+  // CHECK: %[[RESULT:.*]] = spirv.Select %[[CMP]], %[[ZERO]], %[[N_MINUS_ONE_DIV_M_PLUS_ONE]]
+  %result = index.ceildivu %n, %m
+
+  // %[[RESULTI:.*] = builtin.unrealized_conversion_cast %[[RESULT]]
+  // return %[[RESULTI]]
+  return %result : index
+}
+
+// CHECK-LABEL: @floordivs
+// CHECK-SAME: %[[NI:.*]]: index, %[[MI:.*]]: index
+func.func @floordivs(%n: index, %m: index) -> index {
+  // CHECK: %[[N:.*]] = builtin.unrealized_conversion_cast %[[NI]]
+  // CHECK: %[[M:.*]] = builtin.unrealized_conversion_cast %[[MI]]
+  // CHECK: %[[ZERO:.*]] = spirv.Constant 0
+  // CHECK: %[[POS_ONE:.*]] = spirv.Constant 1
+  // CHECK: %[[NEG_ONE:.*]] = spirv.Constant -1
+
+  // CHECK: %[[M_NEG:.*]] = spirv.SLessThan %[[M]], %[[ZERO]]
+  // CHECK: %[[X:.*]] = spirv.Select %[[M_NEG]], %[[POS_ONE]], %[[NEG_ONE]]
+
+  // CHECK: %[[X_MINUS_N:.*]] = spirv.ISub %[[X]], %[[N]]
+  // CHECK: %[[X_MINUS_N_DIV_M:.*]] = spirv.SDiv %[[X_MINUS_N]], %[[M]]
+  // CHECK: %[[NEG_RES:.*]] = spirv.ISub %[[NEG_ONE]], %[[X_MINUS_N_DIV_M]]
+
+  // CHECK: %[[POS_RES:.*]] = spirv.SDiv %[[N]], %[[M]]
+
+  // CHECK: %[[N_NEG:.*]] = spirv.SLessThan %[[N]], %[[ZERO]]
+  // CHECK: %[[DIFF_SIGN:.*]] = spirv.LogicalNotEqual %[[N_NEG]], %[[M_NEG]]
+  // CHECK: %[[N_NON_ZERO:.*]] = spirv.INotEqual %[[N]], %[[ZERO]]
+
+  // CHECK: %[[CMP:.*]] = spirv.LogicalAnd %[[DIFF_SIGN]], %[[N_NON_ZERO]]
+  // CHECK: %[[RESULT:.*]] = spirv.Select %[[CMP]], %[[POS_RES]], %[[NEG_RES]]
+  %result = index.floordivs %n, %m
+
+  // %[[RESULTI:.*] = builtin.unrealized_conversion_cast %[[RESULT]]
+  // return %[[RESULTI]]
+  return %result : index
+}
+
+// CHECK-LABEL: @index_cmp
+func.func @index_cmp(%a : index, %b : index) {
+  // CHECK: spirv.IEqual
+  %0 = index.cmp eq(%a, %b)
+  // CHECK: spirv.INotEqual
+  %1 = index.cmp ne(%a, %b)
+
+  // CHECK: spirv.SLessThan
+  %2 = index.cmp slt(%a, %b)
+  // CHECK: spirv.SLessThanEqual
+  %3 = index.cmp sle(%a, %b)
+  // CHECK: spirv.SGreaterThan
+  %4 = index.cmp sgt(%a, %b)
+  // CHECK: spirv.SGreaterThanEqual
+  %5 = index.cmp sge(%a, %b)
+
+  // CHECK: spirv.ULessThan
+  %6 = index.cmp ult(%a, %b)
+  // CHECK: spirv.ULessThanEqual
+  %7 = index.cmp ule(%a, %b)
+  // CHECK: spirv.UGreaterThan
+  %8 = index.cmp ugt(%a, %b)
+  // CHECK: spirv.UGreaterThanEqual
+  %9 = index.cmp uge(%a, %b)
+  return
+}
+
+// CHECK-LABEL: @index_sizeof
+func.func @index_sizeof() {
+  // CHECK: spirv.Constant 32 : i32
+  %0 = index.sizeof
+  return
+}
+
+// INDEX32-LABEL: @index_cast_from
+// INDEX64-LABEL: @index_cast_from
+// INDEX32-SAME: %[[AI:.*]]: index
+// INDEX64-SAME: %[[AI:.*]]: index
+func.func @index_cast_from(%a: index) -> (i64, i32, i64, i32) {
+  // INDEX32: %[[A:.*]] = builtin.unrealized_conversion_cast %[[AI]] : index to i32
+  // INDEX64: %[[A:.*]] = builtin.unrealized_conversion_cast %[[AI]] : index to i64
+
+  // INDEX32: %[[V0:.*]] = spirv.SConvert %[[A]] : i32 to i64
+  %0 = index.casts %a : index to i64
+  // INDEX64: %[[V1:.*]] = spirv.SConvert %[[A]] : i64 to i32
+  %1 = index.casts %a : index to i32
+  // INDEX32: %[[V2:.*]] = spirv.UConvert %[[A]] : i32 to i64
+  %2 = index.castu %a : index to i64
+  // INDEX64: %[[V3:.*]] = spirv.UConvert %[[A]] : i64 to i32
+  %3 = index.castu %a : index to i32
+
+  // INDEX32: return %[[V0]], %[[A]], %[[V2]], %[[A]]
+  // INDEX64: return %[[A]], %[[V1]], %[[A]], %[[V3]]
+  return %0, %1, %2, %3 : i64, i32, i64, i32
+}
+
+// INDEX32-LABEL: @index_cast_to
+// INDEX64-LABEL: @index_cast_to
+// INDEX32-SAME: %[[A:.*]]: i32, %[[B:.*]]: i64
+// INDEX64-SAME: %[[A:.*]]: i32, %[[B:.*]]: i64
+func.func @index_cast_to(%a: i32, %b: i64) -> (index, index, index, index) {
+  // INDEX64: %[[V0:.*]] = spirv.SConvert %[[A]] : i32 to i64
+  %0 = index.casts %a : i32 to index
+  // INDEX32: %[[V1:.*]] = spirv.SConvert %[[B]] : i64 to i32
+  %1 = index.casts %b : i64 to index
+  // INDEX64: %[[V2:.*]] = spirv.UConvert %[[A]] : i32 to i64
+  %2 = index.castu %a : i32 to index
+  // INDEX32: %[[V3:.*]] = spirv.UConvert %[[B]] : i64 to i32
+  %3 = index.castu %b : i64 to index
+  return %0, %1, %2, %3 : index, index, index, index
+}
+}
diff --git a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-to-nvvm.mlir b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-to-nvvm.mlir
index e54b62a06d..c7d28e7443 100644
--- a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-to-nvvm.mlir
+++ b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-to-nvvm.mlir
@@ -600,6 +600,42 @@ func.func @mbarrier_txcount() {
     func.return 
 }
 
+// CHECK-LABEL: func @mbarrier_txcount_pred
+func.func @mbarrier_txcount_pred() {
+    %mine = arith.constant 1 : index
+    // CHECK: %[[c0:.+]] = arith.constant 0 : index
+    // CHECK: %[[mid:.+]] = builtin.unrealized_conversion_cast %[[c0]] : index to i64
+    // CHECK: %[[S2:.+]] = gpu.thread_id  x
+    // CHECK: %[[P:.+]] = arith.cmpi eq, %[[S2]], %[[c0]] : index
+    %c0 = arith.constant 0 : index  
+    %tidx = gpu.thread_id x
+    %pred = arith.cmpi eq, %tidx, %c0 : index
+
+    // CHECK: %[[barMemref:.+]] = memref.get_global @__mbarrier{{.*}} : memref<1xi64, 3>
+    %barrier = nvgpu.mbarrier.create -> !barrierType
+
+    // CHECK: %[[barStr:.+]] =  builtin.unrealized_conversion_cast %[[barMemref]] : memref<1xi64, 3> to !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)>
+    // CHECK: %[[base:.+]] = llvm.extractvalue %[[barStr]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)> 
+    // CHECK: %[[barPtr:.+]] = llvm.getelementptr %[[base]][%[[mid]]] : (!llvm.ptr<3>, i64) -> !llvm.ptr<3>, i64
+    // CHECK: nvvm.mbarrier.init.shared %[[barPtr]], {{.*}}, predicate = %[[P]]
+    nvgpu.mbarrier.init %barrier[%c0], %mine, predicate = %pred : !barrierType
+    
+    %txcount = arith.constant 256 : index
+    // CHECK: %[[base2:.+]] = llvm.extractvalue %[[barStr]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)> 
+    // CHECK: %[[barPtr2:.+]] = llvm.getelementptr %[[base2]][%[[mid]]] : (!llvm.ptr<3>, i64) -> !llvm.ptr<3>, i64
+    // CHECK: nvvm.mbarrier.arrive.expect_tx.shared %[[barPtr2]], {{.*}}, predicate = %[[P]]
+    nvgpu.mbarrier.arrive.expect_tx %barrier[%c0], %txcount, predicate = %pred : !barrierType
+
+    %phase = arith.constant 0 : index
+    %ticks = arith.constant 10000000 : index
+    // CHECK: %[[base3:.+]] = llvm.extractvalue %[[barStr]][1] : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)> 
+    // CHECK: %[[barPtr3:.+]] = llvm.getelementptr %[[base3]][%[[mid]]] : (!llvm.ptr<3>, i64) -> !llvm.ptr<3>, i64
+    // CHECK: nvvm.mbarrier.try_wait.parity.shared %[[barPtr3]]
+    nvgpu.mbarrier.try_wait.parity %barrier[%c0], %phase, %ticks : !barrierType
+
+    func.return 
+}
+
 // CHECK-LABEL: func @async_tma_load
 !tensorMap1d = !nvgpu.tensormap.descriptor<tensor = memref<128xf32,3>,         swizzle=none,        l2promo = none,        oob = nan,  interleave = none>
 !tensorMap2d = !nvgpu.tensormap.descriptor<tensor = memref<32x32xf32,3>,       swizzle=swizzle_32b, l2promo = none,        oob = zero, interleave = none>
@@ -630,6 +666,32 @@ func.func @async_tma_load(%tensorMap1d: !tensorMap1d, %tensorMap2d: !tensorMap2d
   func.return 
 }
 
+// CHECK-LABEL: func @async_tma_load_pred
+func.func @async_tma_load_pred(%tensorMap1d: !tensorMap1d, %tensorMap2d: !tensorMap2d, %tensorMap3d: !tensorMap3d, %tensorMap4d: !tensorMap4d, %tensorMap5d: !tensorMap5d, 
+                              %buffer1d: memref<128xf32,3>,      
+                              %buffer2d: memref<32x32xf32,3>,    
+                              %buffer3d: memref<2x32x32xf32,3>,  
+                              %buffer4d: memref<2x2x32x32xf32,3>,  
+                              %buffer5d: memref<2x2x2x32x32xf32,3>,
+                              %mbarrier: !mbarrier,
+                              %p: i1) {
+  %c0 = arith.constant 0 : index
+  %crd0 = arith.constant 0 : index
+  %crd1 = arith.constant 0 : index
+  // CHECK: nvvm.cp.async.bulk.tensor.shared.cluster.global %{{.*}}, %{{.*}}, %{{.*}}, box[%{{.*}}], predicate = %{{.*}}
+  nvgpu.tma.async.load %tensorMap1d[%crd0], %mbarrier[%c0] to %buffer1d, predicate = %p : !tensorMap1d, !mbarrier -> memref<128xf32,3>
+  // CHECK: nvvm.cp.async.bulk.tensor.shared.cluster.global %{{.*}}, %{{.*}}, %{{.*}}, box[%{{.*}}, %{{.*}}], predicate = %{{.*}}
+  nvgpu.tma.async.load %tensorMap2d[%crd0, %crd1], %mbarrier[%c0] to %buffer2d, predicate = %p : !tensorMap2d, !mbarrier -> memref<32x32xf32,3>
+  // CHECK: nvvm.cp.async.bulk.tensor.shared.cluster.global %{{.*}}, %{{.*}}, %{{.*}}, box[%{{.*}}, %{{.*}}, %{{.*}}], predicate = %{{.*}}
+  nvgpu.tma.async.load %tensorMap3d[%crd0, %crd1, %crd0], %mbarrier[%c0] to %buffer3d, predicate = %p : !tensorMap3d, !mbarrier -> memref<2x32x32xf32,3>
+  // CHECK: nvvm.cp.async.bulk.tensor.shared.cluster.global %{{.*}}, %{{.*}}, %{{.*}}, box[%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}], predicate = %{{.*}}
+  nvgpu.tma.async.load %tensorMap4d[%crd0, %crd1, %crd1, %crd0], %mbarrier[%c0] to %buffer4d, predicate = %p : !tensorMap4d, !mbarrier -> memref<2x2x32x32xf32,3>
+  // CHECK: nvvm.cp.async.bulk.tensor.shared.cluster.global %{{.*}}, %{{.*}}, %{{.*}}, box[%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}], predicate = %{{.*}}
+  nvgpu.tma.async.load %tensorMap5d[%crd0, %crd1, %crd1, %crd0, %crd0], %mbarrier[%c0] to %buffer5d, predicate = %p : !tensorMap5d, !mbarrier -> memref<2x2x2x32x32xf32,3>
+  func.return 
+}
+
+
 func.func @create_tensor_map(%devicePtr2d : memref<64x128xf32>, %devicePtr1d : memref<128xf32>) {
   %crd0 = arith.constant 64 : index
   %crd1 = arith.constant 128 : index
@@ -643,6 +705,17 @@ func.func @create_tensor_map(%devicePtr2d : memref<64x128xf32>, %devicePtr1d : m
   func.return
 }
 
+// CHECK-LABEL: @tma_prefetch(  
+// CHECK-SAME: %[[arg0:[a-zA-Z0-9_]+]]: !nvgpu.tensormap.descriptor<tensor = memref<128xf32, 3>, swizzle = none, l2promo = none, oob = nan, interleave = none>, %[[arg1:[a-zA-Z0-9_]+]]: i1
+func.func @tma_prefetch(%tensorMap1d: !tensorMap1d, %p : i1) {
+  // CHECK: %[[S0:.+]] = builtin.unrealized_conversion_cast %[[arg0]] : !nvgpu.tensormap.descriptor<tensor = memref<128xf32, 3>, swizzle = none, l2promo = none, oob = nan, interleave = none> to !llvm.ptr
+  // CHECK: nvvm.prefetch.tensormap %[[S0]] : !llvm.ptr
+  nvgpu.tma.prefetch.descriptor %tensorMap1d: !tensorMap1d
+  // CHECK: nvvm.prefetch.tensormap %[[S0]], predicate = %[[arg1]] : !llvm.ptr, i1
+  nvgpu.tma.prefetch.descriptor %tensorMap1d, predicate = %p: !tensorMap1d
+  func.return
+}
+
 !lhsTensorMap = !nvgpu.tensormap.descriptor<tensor = memref<128x64xf16, 3>, swizzle = swizzle_128b, l2promo = none, oob = zero, interleave = none>
 !rhsTensorMap = !nvgpu.tensormap.descriptor<tensor = memref<64x128xf16, strided<[128, 1], offset: 8192>, 3>, swizzle = swizzle_128b, l2promo = none, oob = zero, interleave = none>
 
@@ -713,18 +786,18 @@ func.func @create_wgmma_descriptor(%tensorMap : !tensorMap) -> !nvgpu.warpgroup.
 }
 
 // CHECK-LABEL: @warpgroup_mma_128_128_64(  
-// CHECK-SAME: %[[arg0:[a-zA-Z0-9_]+]]: !nvgpu.warpgroup.descriptor<tensor = memref<128x64xf16, 3>>, %[[arg1:[a-zA-Z0-9_]+]]: !nvgpu.warpgroup.descriptor<tensor = memref<64x128xf16, 3>>, %[[arg2:[a-zA-Z0-9_]+]]: !nvgpu.warpgroup.accumulator<fragmented = vector<64x128xf32>>, %[[arg3:[a-zA-Z0-9_]+]]: !nvgpu.warpgroup.accumulator<fragmented = vector<64x128xf32>>)
+// CHECK-SAME: %[[arg0:[a-zA-Z0-9_]+]]: !nvgpu.warpgroup.descriptor<tensor = memref<128x64xf16, 3>>, %[[arg1:[a-zA-Z0-9_]+]]: !nvgpu.warpgroup.descriptor<tensor = memref<64x128xf16, 3>>, %[[arg2:[a-zA-Z0-9_]+]]: !nvgpu.warpgroup.accumulator<fragmented = vector<128x128xf32>>)
 func.func @warpgroup_mma_128_128_64(
       %descA: !nvgpu.warpgroup.descriptor<tensor = memref<128x64xf16, 3>>, 
       %descB: !nvgpu.warpgroup.descriptor<tensor = memref<64x128xf16, 3>>, 
-      %acc1: !nvgpu.warpgroup.accumulator< fragmented = vector<64x128xf32>>,
-      %acc2: !nvgpu.warpgroup.accumulator< fragmented = vector<64x128xf32>>) 
+      %acc: !nvgpu.warpgroup.accumulator<fragmented = vector<128x128xf32>>) 
 {
 // CHECK: %[[S0:.+]] = builtin.unrealized_conversion_cast %[[arg0]] : !nvgpu.warpgroup.descriptor<tensor = memref<128x64xf16, 3>> to i64
 // CHECK: %[[S1:.+]] = builtin.unrealized_conversion_cast %[[arg1]] : !nvgpu.warpgroup.descriptor<tensor = memref<64x128xf16, 3>> to i64
-// CHECK: %[[S2:.+]] = builtin.unrealized_conversion_cast %[[arg2]] : !nvgpu.warpgroup.accumulator<fragmented = vector<64x128xf32>> to !llvm.struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>
-// CHECK: %[[S3:.+]] = builtin.unrealized_conversion_cast %[[arg3]] : !nvgpu.warpgroup.accumulator<fragmented = vector<64x128xf32>> to !llvm.struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>
+// CHECK: %[[ARG:.+]] = builtin.unrealized_conversion_cast %[[arg2]] : !nvgpu.warpgroup.accumulator<fragmented = vector<128x128xf32>> to !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)>
 // CHECK: nvvm.wgmma.fence.aligned
+// CHECK: %[[UD:.+]] =  llvm.mlir.undef : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)>
+// CHECK: %[[S2:.+]] = llvm.extractvalue %[[ARG]][0] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)> 
 // CHECK: %[[S4:.+]] = nvvm.wgmma.mma_async %[[S0]], %[[S1]], <m = 64, n = 128, k = 16>, D[%[[S2]], <one>, <wrapped>], A[<f16>, <one>, <row>], B[<f16>, <one>, <col>] : !llvm.struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)> -> !llvm.struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>
 // CHECK: %[[S5:.+]] = llvm.mlir.constant(2 : i32) : i64
 // CHECK: %[[S6:.+]] = llvm.add %[[S0]], %[[S5]] : i64
@@ -741,6 +814,7 @@ func.func @warpgroup_mma_128_128_64(
 // CHECK: %[[S17:.+]] = llvm.mlir.constant(384 : i32) : i64
 // CHECK: %[[S18:.+]] = llvm.add %[[S1]], %[[S17]]  : i64
 // CHECK: %[[S19:.+]] = nvvm.wgmma.mma_async %[[S16]], %[[S18]], <m = 64, n = 128, k = 16>, D[%[[S14]], <one>, <wrapped>], A[<f16>, <one>, <row>], B[<f16>, <one>, <col>] : !llvm.struct
+// CHECK: %[[S3:.+]] = llvm.extractvalue %[[ARG]][1] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)> 
 // CHECK: %[[S21:.+]] = llvm.mlir.constant(512 : i32) : i64
 // CHECK: %[[S22:.+]] = llvm.add %[[S0]], %[[S21]]  : i64
 // CHECK: %[[S23:.+]] = nvvm.wgmma.mma_async %[[S22]], %[[S1]], <m = 64, n = 128, k = 16>, D[%[[S3]], <one>, <wrapped>], A[<f16>, <one>, <row>], B[<f16>, <one>, <col>] : !llvm.struct
@@ -759,27 +833,26 @@ func.func @warpgroup_mma_128_128_64(
 // CHECK: %[[S36:.+]] = llvm.mlir.constant(384 : i32) : i64
 // CHECK: %[[S37:.+]] = llvm.add %[[S1]], %[[S36]]  : i64
 // CHECK: %[[S38:.+]] = nvvm.wgmma.mma_async %[[S35]], %[[S37]], <m = 64, n = 128, k = 16>, D[%[[S33]], <one>, <wrapped>], A[<f16>, <one>, <row>], B[<f16>, <one>, <col>] : !llvm.struct
+// CHECK: %[[S40:.+]] = llvm.insertvalue %[[S19]], %[[UD]][0] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)> 
+// CHECK: %[[S41:.+]] = llvm.insertvalue %[[S38]], %[[S40]][1] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)> 
 // CHECK: nvvm.wgmma.commit.group.sync.aligned
 // CHECK: nvvm.wgmma.wait.group.sync.aligned 1  
-  %wgmmaResult, %wgmmaResult2 = nvgpu.warpgroup.mma %descA, %descB, %acc1, %acc2 {transposeB}: 
+  %wgmmaResult = nvgpu.warpgroup.mma %descA, %descB, %acc {transposeB}: 
       !nvgpu.warpgroup.descriptor<tensor = memref<128x64xf16, 3>>, 
       !nvgpu.warpgroup.descriptor<tensor = memref<64x128xf16, 3>>, 
-      !nvgpu.warpgroup.accumulator< fragmented = vector<64x128xf32>>,
-      !nvgpu.warpgroup.accumulator< fragmented = vector<64x128xf32>> 
+      !nvgpu.warpgroup.accumulator<fragmented = vector<128x128xf32>> 
       -> 
-      !nvgpu.warpgroup.accumulator< fragmented = vector<64x128xf32>>, 
-      !nvgpu.warpgroup.accumulator< fragmented = vector<64x128xf32>>  
+      !nvgpu.warpgroup.accumulator<fragmented = vector<128x128xf32>>  
   return
 }
 
 // CHECK-LABEL: @warpgroup_mma_store(  
-// CHECK-SAME: %[[arg0:[a-zA-Z0-9_]+]]: !nvgpu.warpgroup.accumulator<fragmented = vector<64x128xf32>>, %[[arg1:[a-zA-Z0-9_]+]]: !nvgpu.warpgroup.accumulator<fragmented = vector<64x128xf32>>, %[[arg2:[a-zA-Z0-9_]+]]: memref<128x128xf32, 3>)
+// CHECK-SAME: %[[arg0:[a-zA-Z0-9_]+]]: !nvgpu.warpgroup.accumulator<fragmented = vector<128x128xf32>>, %[[arg2:[a-zA-Z0-9_]+]]: memref<128x128xf32, 3>)
 func.func @warpgroup_mma_store(
-    %result1 : !nvgpu.warpgroup.accumulator< fragmented = vector<64x128xf32>>,
-    %result2 : !nvgpu.warpgroup.accumulator< fragmented = vector<64x128xf32>>, 
+    %result : !nvgpu.warpgroup.accumulator<fragmented = vector<128x128xf32>>, 
     %matrixD: memref<128x128xf32,3>) {
-// CHECK: %[[S0:.+]] = builtin.unrealized_conversion_cast %[[arg0]] : !nvgpu.warpgroup.accumulator<fragmented = vector<64x128xf32>> to !llvm.struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>
-// CHECK: %[[S1:.+]] = builtin.unrealized_conversion_cast %[[arg1]] : !nvgpu.warpgroup.accumulator<fragmented = vector<64x128xf32>> to !llvm.struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>
+// CHECK: %[[S0:.+]] = builtin.unrealized_conversion_cast %[[arg0]] : !nvgpu.warpgroup.accumulator<fragmented = vector<128x128xf32>> to !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)>
+// CHECK: %[[EX1:.+]] = llvm.extractvalue %[[S0]][0] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)> 
 // CHECK: %[[S6:.+]] = llvm.mlir.constant(1 : i32) : i32
 // CHECK: %[[S5:.+]] = llvm.mlir.constant(2 : i32) : i32
 // CHECK: %[[S2:.+]] = llvm.mlir.constant(4 : i32) : i32
@@ -807,8 +880,8 @@ func.func @warpgroup_mma_store(
 // CHECK: %[[S23:.+]] = arith.index_cast %[[S21]] : i32 to index
 // CHECK: %[[S24:.+]] = llvm.add %[[S21]], %[[S6]]  : i32
 // CHECK: %[[S25:.+]] = arith.index_cast %[[S24]] : i32 to index
-// CHECK: %[[S26:.+]] = llvm.extractvalue %[[S0]][0] : !llvm.struct
-// CHECK: %[[S27:.+]] = llvm.extractvalue %[[S0]][1] : !llvm.struct
+// CHECK: %[[S26:.+]] = llvm.extractvalue %[[EX1]][0] : !llvm.struct
+// CHECK: %[[S27:.+]] = llvm.extractvalue %[[EX1]][1] : !llvm.struct
 // CHECK: memref.store %[[S26]], %[[arg2]][%[[S22]], %[[S23]]] : memref<128x128xf32, 3>
 // CHECK: memref.store %[[S27]], %[[arg2]][%[[S22]], %[[S25]]] : memref<128x128xf32, 3>
 
@@ -821,8 +894,8 @@ func.func @warpgroup_mma_store(
 // CHECK: %[[S32:.+]] = arith.index_cast %[[S30]] : i32 to index
 // CHECK: %[[S33:.+]] = llvm.add %[[S30]], %[[S6]]  : i32
 // CHECK: %[[S34:.+]] = arith.index_cast %[[S33]] : i32 to index
-// CHECK: %[[S35:.+]] = llvm.extractvalue %[[S0]][4] : !llvm.struct<
-// CHECK: %[[S36:.+]] = llvm.extractvalue %[[S0]][5] : !llvm.struct<
+// CHECK: %[[S35:.+]] = llvm.extractvalue %[[EX1]][4] : !llvm.struct<
+// CHECK: %[[S36:.+]] = llvm.extractvalue %[[EX1]][5] : !llvm.struct<
 // CHECK: memref.store %[[S35]], %[[arg2]][%[[S31]], %[[S32]]] : memref<128x128xf32, 3>
 // CHECK: memref.store %[[S36]], %[[arg2]][%[[S31]], %[[S34]]] : memref<128x128xf32, 3>
 
@@ -835,8 +908,8 @@ func.func @warpgroup_mma_store(
 // CHECK: %[[S41:.+]] = arith.index_cast %[[S39]] : i32 to index
 // CHECK: %[[S42:.+]] = llvm.add %[[S39]], %[[S6]]  : i32
 // CHECK: %[[S43:.+]] = arith.index_cast %[[S42]] : i32 to index
-// CHECK: %[[S44:.+]] = llvm.extractvalue %[[S0]][8] : !llvm.struct<
-// CHECK: %[[S45:.+]] = llvm.extractvalue %[[S0]][9] : !llvm.struct<
+// CHECK: %[[S44:.+]] = llvm.extractvalue %[[EX1]][8] : !llvm.struct<
+// CHECK: %[[S45:.+]] = llvm.extractvalue %[[EX1]][9] : !llvm.struct<
 // CHECK: memref.store %[[S44]], %[[arg2]][%[[S40]], %[[S41]]] : memref<128x128xf32, 3>
 // CHECK: memref.store %[[S45]], %[[arg2]][%[[S40]], %[[S43]]] : memref<128x128xf32, 3>
 
@@ -849,8 +922,8 @@ func.func @warpgroup_mma_store(
 // CHECK: %[[S50:.+]] = arith.index_cast %[[S48]] : i32 to index
 // CHECK: %[[S51:.+]] = llvm.add %[[S48]], %[[S6]]  : i32
 // CHECK: %[[S52:.+]] = arith.index_cast %[[S51]] : i32 to index
-// CHECK: %[[S53:.+]] = llvm.extractvalue %[[S0]][12] : !llvm.struct<
-// CHECK: %[[S54:.+]] = llvm.extractvalue %[[S0]][13] : !llvm.struct<
+// CHECK: %[[S53:.+]] = llvm.extractvalue %[[EX1]][12] : !llvm.struct<
+// CHECK: %[[S54:.+]] = llvm.extractvalue %[[EX1]][13] : !llvm.struct<
 // CHECK: memref.store %[[S53]], %[[arg2]][%[[S49]], %[[S50]]] : memref<128x128xf32, 3>
 // CHECK: memref.store %[[S54]], %[[arg2]][%[[S49]], %[[S52]]] : memref<128x128xf32, 3>
 
@@ -860,7 +933,7 @@ func.func @warpgroup_mma_store(
 // CHECK: %[[c2:.+]] = llvm.mlir.constant(2 : i32) : i32
 
 // ### Store {d64, d65} of each thread ### 
-
+// CHECK: %[[EX2:.+]] = llvm.extractvalue %[[S0]][1] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)> 
 // CHECK: %[[S315:.+]] = llvm.mlir.constant(1 : i32) : i32
 // CHECK: %[[S312:.+]] = llvm.mlir.constant(2 : i32) : i32
 // CHECK: %[[S311:.+]] = llvm.mlir.constant(4 : i32) : i32
@@ -887,20 +960,159 @@ func.func @warpgroup_mma_store(
 // CHECK: %[[S334:.+]] = arith.index_cast %[[S332]] : i32 to index
 // CHECK: %[[S335:.+]] = llvm.add %[[S332]], %[[S315]]  : i32
 // CHECK: %[[S336:.+]] = arith.index_cast %[[S335]] : i32 to index
-// CHECK: %[[S337:.+]] = llvm.extractvalue %[[S1]][0] 
-// CHECK: %[[S338:.+]] = llvm.extractvalue %[[S1]][1]  
+// CHECK: %[[S337:.+]] = llvm.extractvalue %[[EX2]][0] 
+// CHECK: %[[S338:.+]] = llvm.extractvalue %[[EX2]][1]  
 // CHECK: memref.store %[[S337]], %[[arg2]][%[[S333]], %[[S334]]] : memref<128x128xf32, 3>
 // CHECK: memref.store %[[S338]], %[[arg2]][%[[S333]], %[[S336]]] : memref<128x128xf32, 3>
 
 // Pattern continues similarly 31x times until {... d126, d127}
 
-  nvgpu.warpgroup.mma.store [%result1, %result2], %matrixD : 
-    !nvgpu.warpgroup.accumulator< fragmented = vector<64x128xf32>>,
-    !nvgpu.warpgroup.accumulator< fragmented = vector<64x128xf32>> 
+  nvgpu.warpgroup.mma.store %result, %matrixD : 
+    !nvgpu.warpgroup.accumulator< fragmented = vector<128x128xf32>> 
     to memref<128x128xf32,3>
   return 
 }
 
+func.func @warpgroup_mma_init() {
+  //CHECK: %[[S1:.+]] = llvm.mlir.constant(0.000000e+00 : f32) : f3
+  //CHECK: %[[S0:.+]] = llvm.mlir.undef : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)>
+  //CHECK: %[[EX:.+]] = llvm.extractvalue %[[S0]][0] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)> 
+  //CHECK: %[[S2:.+]] = llvm.insertvalue %[[S1]], %[[EX]][0] : !llvm.struct
+  //CHECK: %[[S3:.+]] = llvm.insertvalue %[[S1]], %[[S2]][1] : !llvm.struct
+  //CHECK: %[[S4:.+]] = llvm.insertvalue %[[S1]], %[[S3]][2] : !llvm.struct
+  //CHECK: %[[S5:.+]] = llvm.insertvalue %[[S1]], %[[S4]][3] : !llvm.struct
+  //CHECK: %[[S6:.+]] = llvm.insertvalue %[[S1]], %[[S5]][4] : !llvm.struct
+  //CHECK: %[[S7:.+]] = llvm.insertvalue %[[S1]], %[[S6]][5] : !llvm.struct
+  //CHECK: %[[S8:.+]] = llvm.insertvalue %[[S1]], %[[S7]][6] : !llvm.struct
+  //CHECK: %[[S9:.+]] = llvm.insertvalue %[[S1]], %[[S8]][7] : !llvm.struct
+  //CHECK: %[[S10:.+]] = llvm.insertvalue %[[S1]], %[[S9]][8] : !llvm.struct
+  //CHECK: %[[S11:.+]] = llvm.insertvalue %[[S1]], %[[S10]][9] : !llvm.struct
+  //CHECK: %[[S12:.+]] = llvm.insertvalue %[[S1]], %[[S11]][10] : !llvm.struct
+  //CHECK: %[[S13:.+]] = llvm.insertvalue %[[S1]], %[[S12]][11] : !llvm.struct
+  //CHECK: %[[S14:.+]] = llvm.insertvalue %[[S1]], %[[S13]][12] : !llvm.struct
+  //CHECK: %[[S15:.+]] = llvm.insertvalue %[[S1]], %[[S14]][13] : !llvm.struct
+  //CHECK: %[[S16:.+]] = llvm.insertvalue %[[S1]], %[[S15]][14] : !llvm.struct
+  //CHECK: %[[S17:.+]] = llvm.insertvalue %[[S1]], %[[S16]][15] : !llvm.struct
+  //CHECK: %[[S18:.+]] = llvm.insertvalue %[[S1]], %[[S17]][16] : !llvm.struct
+  //CHECK: %[[S19:.+]] = llvm.insertvalue %[[S1]], %[[S18]][17] : !llvm.struct
+  //CHECK: %[[S20:.+]] = llvm.insertvalue %[[S1]], %[[S19]][18] : !llvm.struct
+  //CHECK: %[[S21:.+]] = llvm.insertvalue %[[S1]], %[[S20]][19] : !llvm.struct
+  //CHECK: %[[S22:.+]] = llvm.insertvalue %[[S1]], %[[S21]][20] : !llvm.struct
+  //CHECK: %[[S23:.+]] = llvm.insertvalue %[[S1]], %[[S22]][21] : !llvm.struct
+  //CHECK: %[[S24:.+]] = llvm.insertvalue %[[S1]], %[[S23]][22] : !llvm.struct
+  //CHECK: %[[S25:.+]] = llvm.insertvalue %[[S1]], %[[S24]][23] : !llvm.struct
+  //CHECK: %[[S26:.+]] = llvm.insertvalue %[[S1]], %[[S25]][24] : !llvm.struct
+  //CHECK: %[[S27:.+]] = llvm.insertvalue %[[S1]], %[[S26]][25] : !llvm.struct
+  //CHECK: %[[S28:.+]] = llvm.insertvalue %[[S1]], %[[S27]][26] : !llvm.struct
+  //CHECK: %[[S29:.+]] = llvm.insertvalue %[[S1]], %[[S28]][27] : !llvm.struct
+  //CHECK: %[[S30:.+]] = llvm.insertvalue %[[S1]], %[[S29]][28] : !llvm.struct
+  //CHECK: %[[S31:.+]] = llvm.insertvalue %[[S1]], %[[S30]][29] : !llvm.struct
+  //CHECK: %[[S32:.+]] = llvm.insertvalue %[[S1]], %[[S31]][30] : !llvm.struct
+  //CHECK: %[[S33:.+]] = llvm.insertvalue %[[S1]], %[[S32]][31] : !llvm.struct
+  //CHECK: %[[S34:.+]] = llvm.insertvalue %[[S1]], %[[S33]][32] : !llvm.struct
+  //CHECK: %[[S35:.+]] = llvm.insertvalue %[[S1]], %[[S34]][33] : !llvm.struct
+  //CHECK: %[[S36:.+]] = llvm.insertvalue %[[S1]], %[[S35]][34] : !llvm.struct
+  //CHECK: %[[S37:.+]] = llvm.insertvalue %[[S1]], %[[S36]][35] : !llvm.struct
+  //CHECK: %[[S38:.+]] = llvm.insertvalue %[[S1]], %[[S37]][36] : !llvm.struct
+  //CHECK: %[[S39:.+]] = llvm.insertvalue %[[S1]], %[[S38]][37] : !llvm.struct
+  //CHECK: %[[S40:.+]] = llvm.insertvalue %[[S1]], %[[S39]][38] : !llvm.struct
+  //CHECK: %[[S41:.+]] = llvm.insertvalue %[[S1]], %[[S40]][39] : !llvm.struct
+  //CHECK: %[[S42:.+]] = llvm.insertvalue %[[S1]], %[[S41]][40] : !llvm.struct
+  //CHECK: %[[S43:.+]] = llvm.insertvalue %[[S1]], %[[S42]][41] : !llvm.struct
+  //CHECK: %[[S44:.+]] = llvm.insertvalue %[[S1]], %[[S43]][42] : !llvm.struct
+  //CHECK: %[[S45:.+]] = llvm.insertvalue %[[S1]], %[[S44]][43] : !llvm.struct
+  //CHECK: %[[S46:.+]] = llvm.insertvalue %[[S1]], %[[S45]][44] : !llvm.struct
+  //CHECK: %[[S47:.+]] = llvm.insertvalue %[[S1]], %[[S46]][45] : !llvm.struct
+  //CHECK: %[[S48:.+]] = llvm.insertvalue %[[S1]], %[[S47]][46] : !llvm.struct
+  //CHECK: %[[S49:.+]] = llvm.insertvalue %[[S1]], %[[S48]][47] : !llvm.struct
+  //CHECK: %[[S50:.+]] = llvm.insertvalue %[[S1]], %[[S49]][48] : !llvm.struct
+  //CHECK: %[[S51:.+]] = llvm.insertvalue %[[S1]], %[[S50]][49] : !llvm.struct
+  //CHECK: %[[S52:.+]] = llvm.insertvalue %[[S1]], %[[S51]][50] : !llvm.struct
+  //CHECK: %[[S53:.+]] = llvm.insertvalue %[[S1]], %[[S52]][51] : !llvm.struct
+  //CHECK: %[[S54:.+]] = llvm.insertvalue %[[S1]], %[[S53]][52] : !llvm.struct
+  //CHECK: %[[S55:.+]] = llvm.insertvalue %[[S1]], %[[S54]][53] : !llvm.struct
+  //CHECK: %[[S56:.+]] = llvm.insertvalue %[[S1]], %[[S55]][54] : !llvm.struct
+  //CHECK: %[[S57:.+]] = llvm.insertvalue %[[S1]], %[[S56]][55] : !llvm.struct
+  //CHECK: %[[S58:.+]] = llvm.insertvalue %[[S1]], %[[S57]][56] : !llvm.struct
+  //CHECK: %[[S59:.+]] = llvm.insertvalue %[[S1]], %[[S58]][57] : !llvm.struct
+  //CHECK: %[[S60:.+]] = llvm.insertvalue %[[S1]], %[[S59]][58] : !llvm.struct
+  //CHECK: %[[S61:.+]] = llvm.insertvalue %[[S1]], %[[S60]][59] : !llvm.struct
+  //CHECK: %[[S62:.+]] = llvm.insertvalue %[[S1]], %[[S61]][60] : !llvm.struct
+  //CHECK: %[[S63:.+]] = llvm.insertvalue %[[S1]], %[[S62]][61] : !llvm.struct
+  //CHECK: %[[S64:.+]] = llvm.insertvalue %[[S1]], %[[S63]][62] : !llvm.struct
+  //CHECK: %[[S65:.+]] = llvm.insertvalue %[[S1]], %[[S64]][63] : !llvm.struct
+  %matrixC = nvgpu.warpgroup.mma.init.accumulator -> !nvgpu.warpgroup.accumulator< fragmented = vector<128x128xf32>>
+  return 
+}
+
+// CHECK-LABEL: @warpgroup_matrix_multiply_m128n128k64(  
+// CHECK-SAME: %[[arg0:[a-zA-Z0-9_]+]]: !nvgpu.warpgroup.descriptor<tensor = memref<128x64xf16, 3>>, %[[arg1:[a-zA-Z0-9_]+]]: !nvgpu.warpgroup.descriptor<tensor = memref<64x128xf16, 3>>, %[[arg2:[a-zA-Z0-9_]+]]: memref<128x128xf32, 3>)
+func.func @warpgroup_matrix_multiply_m128n128k64(
+      %descA: !nvgpu.warpgroup.descriptor<tensor = memref<128x64xf16, 3>>, 
+      %descB: !nvgpu.warpgroup.descriptor<tensor = memref<64x128xf16, 3>>,
+      %shmemD: memref<128x128xf32, 3>) 
+{
+  // Init
+  %matrixC = nvgpu.warpgroup.mma.init.accumulator ->  
+                      !nvgpu.warpgroup.accumulator<fragmented = vector<128x128xf32>>
+
+  // GEMM
+  %matrixD = nvgpu.warpgroup.mma %descA, %descB, %matrixC {transposeB}: 
+        !nvgpu.warpgroup.descriptor<tensor = memref<128x64xf16, 3>>, 
+        !nvgpu.warpgroup.descriptor<tensor = memref<64x128xf16, 3>>, 
+        !nvgpu.warpgroup.accumulator<fragmented = vector<128x128xf32>> 
+        -> 
+        !nvgpu.warpgroup.accumulator<fragmented = vector<128x128xf32>>  
+
+
+  // Epilogue 
+  nvgpu.warpgroup.mma.store %matrixD, %shmemD : 
+    !nvgpu.warpgroup.accumulator< fragmented = vector<128x128xf32>> 
+    to memref<128x128xf32,3>
+
+
+// CHECK: %[[S0:.+]] = builtin.unrealized_conversion_cast %[[arg0]] : !nvgpu.warpgroup.descriptor<tensor = memref<128x64xf16, 3>> to i64
+// CHECK: %[[S1:.+]] = builtin.unrealized_conversion_cast %[[arg1]] : !nvgpu.warpgroup.descriptor<tensor = memref<64x128xf16, 3>> to i64
+// CHECK: %[[S2:.+]] = builtin.unrealized_conversion_cast %[[arg2]] : memref<128x128xf32, 3> to !llvm.struct<(ptr<3>, ptr<3>, i64, array<2 x i64>, array<2 x i64>)>
+// CHECK: %[[S3:.+]] = llvm.mlir.constant(0.000000e+00 : f32) : f32
+// CHECK: %[[S4:.+]] = llvm.mlir.undef : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)>
+// CHECK: %[[S5:.+]] = llvm.extractvalue %[[S4]][0] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)> 
+// CHECK: %[[S6:.+]] = llvm.insertvalue %[[S3]], %[[S5]][0] : !llvm.struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)> 
+// CHECK: %[[S68:.+]] = llvm.insertvalue %[[S3]], %{{.*}}[63] : !llvm.struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)> 
+// CHECK: %[[S69:.+]] = llvm.extractvalue %[[S4]][1] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)> 
+// CHECK: %[[S71:.+]] = llvm.insertvalue %[[S3]], %[[S69]][0] : !llvm.struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)> 
+// CHECK: %[[S134:.+]] = llvm.insertvalue %[[S3]], %{{.*}}[63] : !llvm.struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)> 
+// CHECK: %[[S135:.+]] = llvm.insertvalue %[[S68]], %[[S4]][0] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)> 
+// CHECK: %[[S136:.+]] = llvm.insertvalue %[[S134]], %[[S135]][1] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)> 
+// CHECK: nvvm.wgmma.fence.aligned
+// CHECK: %[[S137:.+]] = llvm.mlir.undef : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)>
+// CHECK: %[[S138:.+]] = llvm.extractvalue %136[0] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)> 
+// CHECK: %[[S139:.+]] = nvvm.wgmma.mma_async %0, %1, <m = 64, n = 128, k = 16>, D[%[[S138]], <one>, <wrapped>], A[<f16>, <one>, <row>], B[<f16>, <one>, <col>] : !llvm.struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)> -> !llvm.struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>
+// CHECK: nvvm.wgmma.mma_async
+// CHECK: nvvm.wgmma.mma_async
+// CHECK: %[[S154:.+]] = nvvm.wgmma.mma_async
+// CHECK: nvvm.wgmma.mma_async
+// CHECK: nvvm.wgmma.mma_async
+// CHECK: nvvm.wgmma.mma_async
+// CHECK: %[[S173:.+]] = nvvm.wgmma.mma_async
+// CHECK: %[[S174:.+]] = llvm.insertvalue %[[S154]], %[[S137]][0] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)> 
+// CHECK: %[[S175:.+]] = llvm.insertvalue %[[S173]], %[[S174]][1] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)> 
+// CHECK: nvvm.wgmma.commit.group.sync.aligned
+// CHECK: nvvm.wgmma.wait.group.sync.aligned 1
+// CHECK: %[[S176:.+]] = llvm.extractvalue %[[S175]][0] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)> 
+// CHECK: %[[S193:.+]] = llvm.extractvalue %[[S176]][0] : !llvm.struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)> 
+// CHECK: %[[S194:.+]] = llvm.extractvalue %[[S176]][1] : !llvm.struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)> 
+// CHECK: memref.store %[[S193]], %[[arg2]][%{{.*}}, %{{.*}}] : memref<128x128xf32, 3>
+// CHECK: memref.store %[[S194]], %[[arg2]][%{{.*}}, %{{.*}}] : memref<128x128xf32, 3>
+// CHECK: %[[S485:.+]] = llvm.extractvalue %[[S175]][1] : !llvm.struct<(struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>, struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>)> 
+// CHECK: %[[S503:.+]] = llvm.extractvalue %[[S485]][0] : !llvm.struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)> 
+// CHECK: %[[S504:.+]] = llvm.extractvalue %[[S485]][1] : !llvm.struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)> 
+// CHECK: memref.store %[[S503]], %[[arg2]][%{{.*}}, %{{.*}}] : memref<128x128xf32, 3>
+// CHECK: memref.store %[[S504]], %[[arg2]][%{{.*}}, %{{.*}}] : memref<128x128xf32, 3>
+  return 
+}
+
+
 transform.sequence failures(propagate) {
 ^bb1(%arg1: !transform.any_op):
   %0 = transform.structured.match ops{["func.func"]} in %arg1 
diff --git a/mlir/test/Conversion/NVVMToLLVM/nvvm-to-llvm.mlir b/mlir/test/Conversion/NVVMToLLVM/nvvm-to-llvm.mlir
index c1549f9b9d..3bb0ab9077 100644
--- a/mlir/test/Conversion/NVVMToLLVM/nvvm-to-llvm.mlir
+++ b/mlir/test/Conversion/NVVMToLLVM/nvvm-to-llvm.mlir
@@ -4,17 +4,30 @@
 // and the generic `convert-to-llvm` pass.
 // RUN: mlir-opt --convert-to-llvm --split-input-file %s | FileCheck %s
 
+// CHECK-LABEL: @init_mbarrier
+llvm.func @init_mbarrier(%barrier_gen : !llvm.ptr, %barrier : !llvm.ptr<3>, %count : i32, %pred : i1) {
+  //CHECK: llvm.inline_asm has_side_effects asm_dialect = att "@$2 mbarrier.init.shared.b64 [$0], $1;", "r,r,b" 
+  nvvm.mbarrier.init.shared %barrier, %count, predicate = %pred : !llvm.ptr<3>, i32, i1 
+  //CHECK: llvm.inline_asm has_side_effects asm_dialect = att "@$2 mbarrier.init.b64 [$0], $1;", "l,r,b" 
+  nvvm.mbarrier.init %barrier_gen, %count, predicate = %pred : !llvm.ptr, i32, i1
+  llvm.return
+}
+
 // CHECK-LABEL: @init_mbarrier_arrive_expect_tx
-llvm.func @init_mbarrier_arrive_expect_tx(%barrier : !llvm.ptr<3>, %txcount : i32) {
-  //CHECK: llvm.inline_asm has_side_effects asm_dialect = att "mbarrier.arrive.expect_tx.shared.b64 _, [$0], $1;", "r,r"  
+llvm.func @init_mbarrier_arrive_expect_tx(%barrier : !llvm.ptr<3>, %txcount : i32, %pred : i1) {
+  //CHECK: llvm.inline_asm has_side_effects asm_dialect = att "mbarrier.arrive.expect_tx.shared.b64 _, [$0], $1;", "r,r"
   nvvm.mbarrier.arrive.expect_tx.shared %barrier, %txcount : !llvm.ptr<3>, i32
+  //CHECK :  llvm.inline_asm has_side_effects asm_dialect = att "@$2 mbarrier.arrive.expect_tx.shared.b64 _, [$0], $1;", "r,r,b "
+  nvvm.mbarrier.arrive.expect_tx.shared %barrier, %txcount, predicate = %pred : !llvm.ptr<3>, i32, i1 
   llvm.return
 }
 
 // CHECK-LABEL: @init_mbarrier_arrive_expect_tx_generic
-llvm.func @init_mbarrier_arrive_expect_tx_generic(%barrier : !llvm.ptr, %txcount : i32) {
+llvm.func @init_mbarrier_arrive_expect_tx_generic(%barrier : !llvm.ptr, %txcount : i32, %pred : i1) {
   // CHECK: llvm.inline_asm has_side_effects asm_dialect = att "mbarrier.arrive.expect_tx.b64 _, [$0], $1;", "l,r" 
   nvvm.mbarrier.arrive.expect_tx %barrier, %txcount : !llvm.ptr, i32
+  // CHECK: llvm.inline_asm has_side_effects asm_dialect = att "@$2 mbarrier.arrive.expect_tx.b64 _, [$0], $1;", "l,r,b"
+  nvvm.mbarrier.arrive.expect_tx %barrier, %txcount, predicate = %pred : !llvm.ptr, i32, i1 
   llvm.return
 }
 
@@ -73,82 +86,93 @@ func.func @async_cp_zfill(%dst: !llvm.ptr<3>, %src: !llvm.ptr<1>, %cpSize: i32)
 }
 
 // CHECK-LABEL: @tma_load_1d
-func.func @tma_load_1d(%tmaDescriptor: !llvm.ptr, %dest : !llvm.ptr<3>, %barrier: !llvm.ptr<3>, %crd0: i32) {
-  // CHECK: llvm.inline_asm has_side_effects asm_dialect = att 
-  // CHECK-SAME: "cp.async.bulk.tensor.1d.shared::cluster.global.mbarrier::complete_tx::bytes [$0], [$1, {$3} ], [$2];", "r,l,r,r"
+func.func @tma_load_1d(%tmaDescriptor: !llvm.ptr, %dest : !llvm.ptr<3>, %barrier: !llvm.ptr<3>, %crd0: i32, %p : i1) {
+  // CHECK: llvm.inline_asm has_side_effects asm_dialect = att "cp.async.bulk.tensor.1d.shared::cluster.global.mbarrier::complete_tx::bytes [$0], [$1, {$3} ], [$2];", "r,l,r,r"
   nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tmaDescriptor,  %barrier, box[%crd0] : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<3>, i32
+  // CHECK : llvm.inline_asm has_side_effects asm_dialect = att "@$4 cp.async.bulk.tensor.1d.shared::cluster.global.mbarrier::complete_tx::bytes [$0], [$1, {$3}], [$2];", "l,r,r,r,b"
+  nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tmaDescriptor,  %barrier, box[%crd0], predicate=%p : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<3>, i32,i1
   return
 }
 
 // CHECK-LABEL: @tma_load_2d
-func.func @tma_load_2d(%tmaDescriptor: !llvm.ptr, %dest : !llvm.ptr<3>, %barrier: !llvm.ptr<3>, %crd0: i32, %crd1: i32) {
-  // CHECK: llvm.inline_asm has_side_effects asm_dialect = att 
-  // CHECK-SAME: "cp.async.bulk.tensor.2d.shared::cluster.global.mbarrier::complete_tx::bytes [$0], [$1, {$3, $4} ], [$2];", "r,l,r,r,r"
+func.func @tma_load_2d(%tmaDescriptor: !llvm.ptr, %dest : !llvm.ptr<3>, %barrier: !llvm.ptr<3>, %crd0: i32, %crd1: i32, %p : i1) {
+  // CHECK: llvm.inline_asm has_side_effects asm_dialect = att "cp.async.bulk.tensor.2d.shared::cluster.global.mbarrier::complete_tx::bytes [$0], [$1, {$3, $4} ], [$2];", "r,l,r,r,r"
   nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tmaDescriptor,  %barrier, box[%crd0,%crd1] : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<3>, i32, i32
+  // CHECK : llvm.inline_asm has_side_effects asm_dialect = att "@$5 cp.async.bulk.tensor.2d.shared::cluster.global.mbarrier::complete_tx::bytes [$0], [$1, {$3, $4}], [$2];", "l,r,r,r,r,b"
+  nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tmaDescriptor,  %barrier, box[%crd0,%crd1], predicate=%p  : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<3>, i32, i32, i1
   return
 }
 
 // CHECK-LABEL: @tma_load_3d
-func.func @tma_load_3d(%tmaDescriptor: !llvm.ptr, %dest : !llvm.ptr<3>, %barrier: !llvm.ptr<3>, %crd0: i32, %crd1: i32, %crd2: i32) {
-  // CHECK: llvm.inline_asm has_side_effects asm_dialect = att 
-  // CHECK-SAME: "cp.async.bulk.tensor.3d.shared::cluster.global.mbarrier::complete_tx::bytes [$0], [$1, {$3, $4, $5} ], [$2];", "r,l,r,r,r,r"
+func.func @tma_load_3d(%tmaDescriptor: !llvm.ptr, %dest : !llvm.ptr<3>, %barrier: !llvm.ptr<3>, %crd0: i32, %crd1: i32, %crd2: i32, %p : i1) {
+  // CHECK: llvm.inline_asm has_side_effects asm_dialect = att "cp.async.bulk.tensor.3d.shared::cluster.global.mbarrier::complete_tx::bytes [$0], [$1, {$3, $4, $5} ], [$2];", "r,l,r,r,r,r"
   nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tmaDescriptor,  %barrier, box[%crd0,%crd1,%crd2] : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<3>, i32, i32, i32
+  // CHECK : llvm.inline_asm has_side_effects asm_dialect = att "@$6 cp.async.bulk.tensor.3d.shared::cluster.global.mbarrier::complete_tx::bytes [$0], [$1, {$3, $4, $5}], [$2];", "l,r,r,r,r,r,b"
+  nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tmaDescriptor,  %barrier, box[%crd0,%crd1,%crd2], predicate=%p  : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<3>, i32, i32, i32, i1
   return
 }
 
 // CHECK-LABEL: @tma_load_4d
-func.func @tma_load_4d(%tmaDescriptor: !llvm.ptr, %dest : !llvm.ptr<3>, %barrier: !llvm.ptr<3>, %crd0: i32, %crd1: i32, %crd2: i32, %crd3: i32) {
-  // CHECK: llvm.inline_asm has_side_effects asm_dialect = att 
-  // CHECK-SAME: "cp.async.bulk.tensor.4d.shared::cluster.global.mbarrier::complete_tx::bytes [$0], [$1, {$3, $4, $5, $6} ], [$2];", "r,l,r,r,r,r,r"
+func.func @tma_load_4d(%tmaDescriptor: !llvm.ptr, %dest : !llvm.ptr<3>, %barrier: !llvm.ptr<3>, %crd0: i32, %crd1: i32, %crd2: i32, %crd3: i32, %p : i1) {
+  // CHECK: llvm.inline_asm has_side_effects asm_dialect = att "cp.async.bulk.tensor.4d.shared::cluster.global.mbarrier::complete_tx::bytes [$0], [$1, {$3, $4, $5, $6} ], [$2];", "r,l,r,r,r,r,r"
   nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tmaDescriptor,  %barrier, box[%crd0,%crd1,%crd2,%crd3] : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<3>, i32, i32, i32, i32
+  // CHECK : llvm.inline_asm has_side_effects asm_dialect = att "@$7 cp.async.bulk.tensor.3d.shared::cluster.global.mbarrier::complete_tx::bytes [$0], [$1, {$3, $4, $5}], [$2];", "l,r,r,r,r,r,b"
+  nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tmaDescriptor,  %barrier, box[%crd0,%crd1,%crd2], predicate=%p  : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<3>, i32, i32, i32, i1
   return
 }
 
 // CHECK-LABEL: @tma_load_5d
-func.func @tma_load_5d(%tmaDescriptor: !llvm.ptr, %dest : !llvm.ptr<3>, %barrier: !llvm.ptr<3>, %crd0: i32, %crd1: i32, %crd2: i32, %crd3: i32, %crd4: i32) {
-  // CHECK: llvm.inline_asm has_side_effects asm_dialect = att 
-  // CHECK-SAME: "cp.async.bulk.tensor.5d.shared::cluster.global.mbarrier::complete_tx::bytes [$0], [$1, {$3, $4, $5, $6, $7} ], [$2];", "r,l,r,r,r,r,r,r"
+func.func @tma_load_5d(%tmaDescriptor: !llvm.ptr, %dest : !llvm.ptr<3>, %barrier: !llvm.ptr<3>, %crd0: i32, %crd1: i32, %crd2: i32, %crd3: i32, %crd4: i32, %p : i1) {
+  // CHECK: llvm.inline_asm has_side_effects asm_dialect = att "cp.async.bulk.tensor.5d.shared::cluster.global.mbarrier::complete_tx::bytes [$0], [$1, {$3, $4, $5, $6, $7} ], [$2];", "r,l,r,r,r,r,r,r"
   nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tmaDescriptor,  %barrier, box[%crd0,%crd1,%crd2,%crd3,%crd4] : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<3>, i32, i32, i32, i32, i32
+  // CHECK : llvm.inline_asm has_side_effects asm_dialect = att "@$8 cp.async.bulk.tensor.5d.shared::cluster.global.mbarrier::complete_tx::bytes [$0], [$1, {$3, $4, $5, $6, $7}], [$2];", "l,r,r,r,r,r,r,r,b"
+  nvvm.cp.async.bulk.tensor.shared.cluster.global %dest, %tmaDescriptor,  %barrier, box[%crd0,%crd1,%crd2,%crd3,%crd4], predicate=%p  : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<3>, i32, i32, i32, i32, i32, i1
   return
 }
 
 // CHECK-LABEL: @tma_store_1d
-func.func @tma_store_1d(%tmaDescriptor: !llvm.ptr, %src : !llvm.ptr<3>, %crd0: i32) {
-  // CHECK: llvm.inline_asm has_side_effects asm_dialect = att 
-  // CHECK-SAME: "cp.async.bulk.tensor.1d.global.shared::cta.bulk_group [$0, {$2} ], [$1];", "l,r,r"
+func.func @tma_store_1d(%tmaDescriptor: !llvm.ptr, %src : !llvm.ptr<3>, %crd0: i32, %p : i1) {
+  // CHECK: llvm.inline_asm has_side_effects asm_dialect = att "cp.async.bulk.tensor.1d.global.shared::cta.bulk_group [$0, {$2} ], [$1];", "l,r,r"
   nvvm.cp.async.bulk.tensor.global.shared.cta %tmaDescriptor, %src, box[%crd0] : !llvm.ptr, !llvm.ptr<3>, i32
+  // CHECK: llvm.inline_asm has_side_effects asm_dialect = att "@$3 cp.async.bulk.tensor.1d.global.shared::cta.bulk_group [$0, {$2} ], [$1];", "l,r,r,b"
+  nvvm.cp.async.bulk.tensor.global.shared.cta %tmaDescriptor, %src, box[%crd0], predicate=%p : !llvm.ptr, !llvm.ptr<3>, i32, i1
   return
 }
 
 // CHECK-LABEL: @tma_store_2d
-func.func @tma_store_2d(%tmaDescriptor: !llvm.ptr, %src : !llvm.ptr<3>, %crd0: i32, %crd1: i32) {
-  // CHECK: llvm.inline_asm has_side_effects asm_dialect = att 
-  // CHECK-SAME: "cp.async.bulk.tensor.2d.global.shared::cta.bulk_group [$0, {$2, $3} ], [$1];", "l,r,r,r"
+func.func @tma_store_2d(%tmaDescriptor: !llvm.ptr, %src : !llvm.ptr<3>, %crd0: i32, %crd1: i32, %p : i1) {
+  // CHECK: llvm.inline_asm has_side_effects asm_dialect = att "cp.async.bulk.tensor.2d.global.shared::cta.bulk_group [$0, {$2, $3} ], [$1];", "l,r,r,r"
   nvvm.cp.async.bulk.tensor.global.shared.cta %tmaDescriptor, %src, box[%crd0,%crd1] : !llvm.ptr, !llvm.ptr<3>, i32, i32
+  // CHECK: llvm.inline_asm has_side_effects asm_dialect = att "@$4 cp.async.bulk.tensor.2d.global.shared::cta.bulk_group [$0, {$2, $3} ], [$1];", "l,r,r,r,b"
+  nvvm.cp.async.bulk.tensor.global.shared.cta %tmaDescriptor, %src, box[%crd0,%crd1], predicate=%p : !llvm.ptr, !llvm.ptr<3>, i32, i32, i1
   return
 }
 
 // CHECK-LABEL: @tma_store_3d
-func.func @tma_store_3d(%tmaDescriptor: !llvm.ptr, %src : !llvm.ptr<3>, %crd0: i32, %crd1: i32, %crd2: i32) {
-  // CHECK: llvm.inline_asm has_side_effects asm_dialect = att 
-  // CHECK-SAME: "cp.async.bulk.tensor.3d.global.shared::cta.bulk_group [$0, {$2, $3, $4} ], [$1];", "l,r,r,r,r"
+func.func @tma_store_3d(%tmaDescriptor: !llvm.ptr, %src : !llvm.ptr<3>, %crd0: i32, %crd1: i32, %crd2: i32, %p : i1) {
+  // CHECK: llvm.inline_asm has_side_effects asm_dialect = att "cp.async.bulk.tensor.3d.global.shared::cta.bulk_group [$0, {$2, $3, $4} ], [$1];", "l,r,r,r,r"
   nvvm.cp.async.bulk.tensor.global.shared.cta %tmaDescriptor, %src, box[%crd0,%crd1,%crd2] : !llvm.ptr, !llvm.ptr<3>, i32, i32, i32
+  // CHECK: llvm.inline_asm has_side_effects asm_dialect = att "@$5 cp.async.bulk.tensor.3d.global.shared::cta.bulk_group [$0, {$2, $3, $4} ], [$1];", "l,r,r,r,r,b"
+  nvvm.cp.async.bulk.tensor.global.shared.cta %tmaDescriptor, %src, box[%crd0,%crd1,%crd2], predicate=%p : !llvm.ptr, !llvm.ptr<3>, i32, i32, i32, i1
   return
 }
 
 // CHECK-LABEL: @tma_store_4d
-func.func @tma_store_4d(%tmaDescriptor: !llvm.ptr, %src : !llvm.ptr<3>, %crd0: i32, %crd1: i32, %crd2: i32, %crd3: i32) {
-  // CHECK: llvm.inline_asm has_side_effects asm_dialect = att 
-  // CHECK-SAME: "cp.async.bulk.tensor.4d.global.shared::cta.bulk_group [$0, {$2, $3, $4, $5} ], [$1];", "l,r,r,r,r,r"
+func.func @tma_store_4d(%tmaDescriptor: !llvm.ptr, %src : !llvm.ptr<3>, %crd0: i32, %crd1: i32, %crd2: i32, %crd3: i32, %p : i1) {
+  // CHECK: llvm.inline_asm has_side_effects asm_dialect = att "cp.async.bulk.tensor.4d.global.shared::cta.bulk_group [$0, {$2, $3, $4, $5} ], [$1];", "l,r,r,r,r,r"
   nvvm.cp.async.bulk.tensor.global.shared.cta %tmaDescriptor, %src, box[%crd0,%crd1,%crd2,%crd3] : !llvm.ptr, !llvm.ptr<3>, i32, i32, i32, i32
+  // CHECK: llvm.inline_asm has_side_effects asm_dialect = att "@$6 cp.async.bulk.tensor.4d.global.shared::cta.bulk_group [$0, {$2, $3, $4, $5} ], [$1];", "l,r,r,r,r,r,b"
+  nvvm.cp.async.bulk.tensor.global.shared.cta %tmaDescriptor, %src, box[%crd0,%crd1,%crd2,%crd3], predicate=%p : !llvm.ptr, !llvm.ptr<3>, i32, i32, i32, i32, i1
   return
 }
 
 // CHECK-LABEL: @tma_store_5d
-func.func @tma_store_5d(%tmaDescriptor: !llvm.ptr, %src : !llvm.ptr<3>, %crd0: i32, %crd1: i32, %crd2: i32, %crd3: i32, %crd4: i32) {
-  // CHECK: llvm.inline_asm has_side_effects asm_dialect = att 
-  // CHECK-SAME: "cp.async.bulk.tensor.5d.global.shared::cta.bulk_group [$0, {$2, $3, $4, $5, $6} ], [$1];", "l,r,r,r,r,r,r"
+func.func @tma_store_5d(%tmaDescriptor: !llvm.ptr, %src : !llvm.ptr<3>, %crd0: i32, %crd1: i32, %crd2: i32, %crd3: i32, %crd4: i32, %p : i1) {
+  // CHECK-NEXT: llvm.inline_asm has_side_effects asm_dialect = att "cp.async.bulk.tensor.5d.global.shared::cta.bulk_group [$0, {$2, $3, $4, $5, $6} ], [$1];", "l,r,r,r,r,r,r"
   nvvm.cp.async.bulk.tensor.global.shared.cta %tmaDescriptor, %src, box[%crd0,%crd1,%crd2,%crd3,%crd4] : !llvm.ptr, !llvm.ptr<3>, i32, i32, i32, i32, i32
+
+  // CHECK-NEXT: llvm.inline_asm has_side_effects asm_dialect = att "@$7 cp.async.bulk.tensor.5d.global.shared::cta.bulk_group [$0, {$2, $3, $4, $5, $6} ], [$1];", "l,r,r,r,r,r,r,b"
+  nvvm.cp.async.bulk.tensor.global.shared.cta %tmaDescriptor, %src, box[%crd0,%crd1,%crd2,%crd3,%crd4], predicate=%p : !llvm.ptr, !llvm.ptr<3>, i32, i32, i32, i32, i32, i1
   return
 }
 
@@ -480,3 +504,38 @@ func.func @elect_one_leader_sync() {
   %cnd = nvvm.elect.sync -> i1 
   return 
 }
+
+// -----
+
+// CHECK-LABEL: @stmatrix(
+// CHECK-SAME: %[[arg0:[a-zA-Z0-9_]+]]: !llvm.ptr<3>, 
+// CHECK-SAME: %[[arg1:[a-zA-Z0-9_]+]]: i32,
+// CHECK-SAME: %[[arg2:[a-zA-Z0-9_]+]]: i32,
+// CHECK-SAME: %[[arg3:[a-zA-Z0-9_]+]]: i32,
+// CHECK-SAME: %[[arg4:[a-zA-Z0-9_]+]]: i32)
+llvm.func @stmatrix(%arg0 : !llvm.ptr<3>, %m1 : i32, %m2 : i32, %m3 : i32, %m4 : i32) {
+// CHECK: llvm.inline_asm has_side_effects asm_dialect = att "stmatrix.sync.aligned.x1.m8n8.shared.b16 [$0], {$1}", "r,r" %[[arg0]], %[[arg1]] : (!llvm.ptr<3>, i32) -> ()
+// CHECK: llvm.inline_asm has_side_effects asm_dialect = att "stmatrix.sync.aligned.x2.m8n8.shared.b16 [$0], {$1, $2}", "r,r,r" %[[arg0]], %[[arg1]], %[[arg2]] : (!llvm.ptr<3>, i32, i32) -> ()
+// CHECK: llvm.inline_asm has_side_effects asm_dialect = att "stmatrix.sync.aligned.x4.m8n8.shared.b16 [$0], {$1, $2, $3, $4};", "r,r,r,r,r" %[[arg0]], %[[arg1]], %[[arg2]], %[[arg3]], %[[arg4]] : (!llvm.ptr<3>, i32, i32, i32, i32) -> ()
+// CHECK: llvm.inline_asm has_side_effects asm_dialect = att "stmatrix.sync.aligned.x1.trans.m8n8.shared.b16 [$0], {$1}", "r,r" %[[arg0]], %[[arg1]] : (!llvm.ptr<3>, i32) -> ()
+// CHECK: llvm.inline_asm has_side_effects asm_dialect = att "stmatrix.sync.aligned.x2.trans.m8n8.shared.b16 [$0], {$1, $2}", "r,r,r" %[[arg0]], %[[arg1]], %[[arg2]] : (!llvm.ptr<3>, i32, i32) -> ()
+// CHECK: llvm.inline_asm has_side_effects asm_dialect = att "stmatrix.sync.aligned.x4.trans.m8n8.shared.b16 [$0], {$1, $2, $3, $4};", "r,r,r,r,r" %[[arg0]], %[[arg1]], %[[arg2]], %[[arg3]], %[[arg4]] : (!llvm.ptr<3>, i32, i32, i32, i32) -> ()
+  nvvm.stmatrix %arg0, %m1 {layout = #nvvm.mma_layout<row>} : !llvm.ptr<3>, i32
+  nvvm.stmatrix %arg0, %m1, %m2 {layout = #nvvm.mma_layout<row>} : !llvm.ptr<3>, i32, i32
+  nvvm.stmatrix %arg0, %m1, %m2, %m3, %m4 {layout = #nvvm.mma_layout<row>} : !llvm.ptr<3>, i32, i32, i32, i32
+  nvvm.stmatrix %arg0, %m1 {layout = #nvvm.mma_layout<col>} : !llvm.ptr<3>, i32
+  nvvm.stmatrix %arg0, %m1, %m2 {layout = #nvvm.mma_layout<col>} : !llvm.ptr<3>, i32, i32
+  nvvm.stmatrix %arg0, %m1, %m2, %m3, %m4 {layout = #nvvm.mma_layout<col>} : !llvm.ptr<3>, i32, i32, i32, i32
+  llvm.return 
+}
+
+// -----
+
+// CHECK-LABEL: @init_mbarrier_arrive_expect_tx
+llvm.func @init_mbarrier_arrive_expect_tx(%desc : !llvm.ptr, %pred : i1) {
+  //CHECK: llvm.inline_asm has_side_effects asm_dialect = att "prefetch.tensormap [$0];", "l"
+  nvvm.prefetch.tensormap %desc : !llvm.ptr
+  //CHECK: llvm.inline_asm has_side_effects asm_dialect = att "@$1 prefetch.tensormap [$0];", "l,b"
+  nvvm.prefetch.tensormap %desc, predicate = %pred : !llvm.ptr, i1
+  llvm.return
+}
diff --git a/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir b/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
index 1df27dd995..881d738b41 100644
--- a/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
+++ b/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
@@ -90,7 +90,7 @@ func.func @wsloop(%arg0: index, %arg1: index, %arg2: index, %arg3: index, %arg4:
 // CHECK-LABEL: @atomic_write
 // CHECK: (%[[ARG0:.*]]: !llvm.ptr<i32>)
 // CHECK: %[[VAL0:.*]] = llvm.mlir.constant(1 : i32) : i32
-// CHECK: omp.atomic.write %[[ARG0]] = %[[VAL0]] hint(none) memory_order(relaxed) : !llvm.ptr<i32>, i32
+// CHECK: omp.atomic.write %[[ARG0]] = %[[VAL0]] memory_order(relaxed) : !llvm.ptr<i32>, i32
 func.func @atomic_write(%a: !llvm.ptr<i32>) -> () {
   %1 = arith.constant 1 : i32
   omp.atomic.write %a = %1 hint(none) memory_order(relaxed) : !llvm.ptr<i32>, i32
@@ -474,4 +474,4 @@ llvm.func @_QPtarget_map_with_bounds(%arg0: !llvm.ptr<i32>, %arg1: !llvm.ptr<arr
     omp.terminator
   }
   llvm.return
-}
\ No newline at end of file
+}
diff --git a/mlir/test/Conversion/SCFToSPIRV/use-indices.mlir b/mlir/test/Conversion/SCFToSPIRV/use-indices.mlir
new file mode 100644
index 0000000000..68a825fbd9
--- /dev/null
+++ b/mlir/test/Conversion/SCFToSPIRV/use-indices.mlir
@@ -0,0 +1,28 @@
+// RUN: mlir-opt -convert-scf-to-spirv %s -o - | FileCheck %s
+
+// CHECK-LABEL: @forward
+func.func @forward() {
+  // CHECK: %[[LB:.*]] = spirv.Constant 0 : i32
+  %c0 = arith.constant 0 : index
+  // CHECK: %[[UB:.*]] = spirv.Constant 32 : i32
+  %c32 = arith.constant 32 : index
+  // CHECK: %[[STEP:.*]] = spirv.Constant 1 : i32
+  %c1 = arith.constant 1 : index
+
+  // CHECK:      spirv.mlir.loop {
+  // CHECK-NEXT:   spirv.Branch ^[[HEADER:.*]](%[[LB]] : i32)
+  // CHECK:      ^[[HEADER]](%[[INDVAR:.*]]: i32):
+  // CHECK:        %[[CMP:.*]] = spirv.SLessThan %[[INDVAR]], %[[UB]] : i32
+  // CHECK:        spirv.BranchConditional %[[CMP]], ^[[BODY:.*]], ^[[MERGE:.*]]
+  // CHECK:      ^[[BODY]]:
+  // CHECK:        %[[X:.*]] = spirv.IAdd %[[INDVAR]], %[[INDVAR]] : i32
+  // CHECK:        %[[INDNEXT:.*]] = spirv.IAdd %[[INDVAR]], %[[STEP]] : i32
+  // CHECK:        spirv.Branch ^[[HEADER]](%[[INDNEXT]] : i32)
+  // CHECK:      ^[[MERGE]]:
+  // CHECK:        spirv.mlir.merge
+  // CHECK:      }
+  scf.for %arg2 = %c0 to %c32 step %c1 {
+      %1 = index.add %arg2, %arg2
+  }
+  return
+}
diff --git a/mlir/test/Dialect/Affine/invalid.mlir b/mlir/test/Dialect/Affine/invalid.mlir
index 1bcb6fc4a3..72864516b4 100644
--- a/mlir/test/Dialect/Affine/invalid.mlir
+++ b/mlir/test/Dialect/Affine/invalid.mlir
@@ -5,7 +5,7 @@
 func.func @affine_apply_operand_non_index(%arg0 : i32) {
   // Custom parser automatically assigns all arguments the `index` so we must
   // use the generic syntax here to exercise the verifier.
-  // expected-error@+1 {{op operand #0 must be index, but got 'i32'}}
+  // expected-error@+1 {{op operand #0 must be variadic of index, but got 'i32'}}
   %0 = "affine.apply"(%arg0) {map = affine_map<(d0) -> (d0)>} : (i32) -> (index)
   return
 }
diff --git a/mlir/test/Dialect/Affine/value-bounds-op-interface-impl.mlir b/mlir/test/Dialect/Affine/value-bounds-op-interface-impl.mlir
index 338c48c5b2..8acf358c88 100644
--- a/mlir/test/Dialect/Affine/value-bounds-op-interface-impl.mlir
+++ b/mlir/test/Dialect/Affine/value-bounds-op-interface-impl.mlir
@@ -58,3 +58,35 @@ func.func @affine_min_lb(%a: index) -> (index) {
   %2 = "test.reify_bound"(%1) {type = "LB"}: (index) -> (index)
   return %2 : index
 }
+
+// -----
+
+// CHECK-LABEL: func @composed_affine_apply(
+//       CHECK:   %[[cst:.*]] = arith.constant -8 : index
+//       CHECK:   return %[[cst]]
+func.func @composed_affine_apply(%i1 : index) -> (index) {
+  // The ValueBoundsOpInterface implementation of affine.apply fully composes
+  // the affine map (and its operands) with other affine.apply ops drawn from
+  // its operands before adding it to the constraint set. This is to work
+  // around a limitation in `FlatLinearConstraints`, which can currently not
+  // compute a constant bound for %s. (The affine map simplification logic can
+  // simplify %s to -8.)
+  %i2 = affine.apply affine_map<(d0) -> ((d0 floordiv 32) * 16)>(%i1)
+  %i3 = affine.apply affine_map<(d0) -> ((d0 floordiv 32) * 16 + 8)>(%i1)
+  %s = affine.apply affine_map<()[s0, s1] -> (s0 - s1)>()[%i2, %i3]
+  %reified = "test.reify_constant_bound"(%s) {type = "EQ"} : (index) -> (index)
+  return %reified : index
+}
+
+
+// -----
+
+// Test for affine::fullyComposeAndCheckIfEqual
+func.func @composed_are_equal(%i1 : index) {
+  %i2 = affine.apply affine_map<(d0) -> ((d0 floordiv 32) * 16)>(%i1)
+  %i3 = affine.apply affine_map<(d0) -> ((d0 floordiv 32) * 16 + 8)>(%i1)
+  %s = affine.apply affine_map<()[s0, s1] -> (s0 - s1)>()[%i2, %i3]
+  // expected-remark @below{{different}}
+   "test.are_equal"(%i2, %i3) {compose} : (index, index) -> ()
+  return
+}
diff --git a/mlir/test/Dialect/Arith/canonicalize.mlir b/mlir/test/Dialect/Arith/canonicalize.mlir
index 1b0547c9e8..10050d87d7 100644
--- a/mlir/test/Dialect/Arith/canonicalize.mlir
+++ b/mlir/test/Dialect/Arith/canonicalize.mlir
@@ -182,6 +182,18 @@ func.func @selAndNotCond(%arg0: i1, %arg1: i1, %arg2 : i32, %arg3 : i32) -> i32
   return %res : i32
 }
 
+// CHECK-LABEL: @selAndNotCondVec
+//       CHECK-NEXT: %[[one:.+]] = arith.constant dense<true> : vector<4xi1>
+//       CHECK-NEXT: %[[not:.+]] = arith.xori %arg0, %[[one]]
+//       CHECK-NEXT: %[[and:.+]] = arith.andi %arg1, %[[not]]
+//       CHECK-NEXT: %[[res:.+]] = arith.select %[[and]], %arg3, %arg2
+//       CHECK-NEXT: return %[[res]]
+func.func @selAndNotCondVec(%arg0: vector<4xi1>, %arg1: vector<4xi1>, %arg2 : vector<4xi32>, %arg3 : vector<4xi32>) -> vector<4xi32> {
+  %sel = arith.select %arg0, %arg2, %arg3 : vector<4xi1>, vector<4xi32>
+  %res = arith.select %arg1, %sel, %arg2 : vector<4xi1>, vector<4xi32>
+  return %res : vector<4xi32>
+}
+
 // CHECK-LABEL: @selOrCond
 //       CHECK-NEXT: %[[or:.+]] = arith.ori %arg1, %arg0
 //       CHECK-NEXT: %[[res:.+]] = arith.select %[[or]], %arg2, %arg3
@@ -204,6 +216,18 @@ func.func @selOrNotCond(%arg0: i1, %arg1: i1, %arg2 : i32, %arg3 : i32) -> i32 {
   return %res : i32
 }
 
+// CHECK-LABEL: @selOrNotCondVec
+//       CHECK-NEXT: %[[one:.+]] = arith.constant dense<true> : vector<4xi1>
+//       CHECK-NEXT: %[[not:.+]] = arith.xori %arg0, %[[one]]
+//       CHECK-NEXT: %[[or:.+]] = arith.ori %arg1, %[[not]]
+//       CHECK-NEXT: %[[res:.+]] = arith.select %[[or]], %arg3, %arg2
+//       CHECK-NEXT: return %[[res]]
+func.func @selOrNotCondVec(%arg0: vector<4xi1>, %arg1: vector<4xi1>, %arg2 : vector<4xi32>, %arg3 : vector<4xi32>) -> vector<4xi32> {
+  %sel = arith.select %arg0, %arg2, %arg3 : vector<4xi1>, vector<4xi32>
+  %res = arith.select %arg1, %arg3, %sel : vector<4xi1>, vector<4xi32>
+  return %res : vector<4xi32>
+}
+
 // Test case: Folding of comparisons with equal operands.
 // CHECK-LABEL: @cmpi_equal_operands
 //   CHECK-DAG:   %[[T:.*]] = arith.constant true
@@ -985,6 +1009,18 @@ func.func @tripleMulIMulII32(%arg0: i32) -> i32 {
   return %mul2 : i32
 }
 
+// CHECK-LABEL: @tripleMulLargeInt
+//       CHECK:   %[[cres:.+]] = arith.constant 3618502788666131213697322783095070105623107215331596699973092056135872020482 : i256
+//       CHECK:   %[[addi:.+]] = arith.addi %arg0, %[[cres]] : i256
+//       CHECK:   return %[[addi]]
+func.func @tripleMulLargeInt(%arg0: i256) -> i256 {
+  %0 = arith.constant 3618502788666131213697322783095070105623107215331596699973092056135872020481 : i256
+  %1 = arith.constant 1 : i256
+  %2 = arith.addi %arg0, %0 : i256
+  %3 = arith.addi %2, %1 : i256
+  return %3 : i256
+}
+
 // CHECK-LABEL: @addiMuliToSubiRhsI32
 //  CHECK-SAME:   (%[[ARG0:.+]]: i32, %[[ARG1:.+]]: i32)
 //       CHECK:   %[[SUB:.+]] = arith.subi %[[ARG0]], %[[ARG1]] : i32
diff --git a/mlir/test/Dialect/ArmSME/arm-sme-to-llvm.mlir b/mlir/test/Dialect/ArmSME/arm-sme-to-llvm.mlir
index 4c16e5c488..07485b3ee8 100644
--- a/mlir/test/Dialect/ArmSME/arm-sme-to-llvm.mlir
+++ b/mlir/test/Dialect/ArmSME/arm-sme-to-llvm.mlir
@@ -116,7 +116,7 @@ func.func @arm_sme_load_tile_slice_hor_f64(%src : memref<?x?xf64>, %tile : vecto
 // CHECK: "arm_sme.intr.ld1b.vert"({{.*}}) : (vector<[16]xi1>, !llvm.ptr, i32, i32) -> ()
 func.func @arm_sme_load_tile_slice_ver_i8(%src : memref<?x?xi8>, %tile : vector<[16]x[16]xi8>, %tile_slice_index : index) {
   %c0 = arith.constant 0 : index
-  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index, <vertical> : memref<?x?xi8>, vector<[16]x[16]xi8>
+  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index layout<vertical> : memref<?x?xi8>, vector<[16]x[16]xi8>
   return
 }
 
@@ -126,7 +126,7 @@ func.func @arm_sme_load_tile_slice_ver_i8(%src : memref<?x?xi8>, %tile : vector<
 // CHECK: "arm_sme.intr.ld1h.vert"({{.*}}) : (vector<[8]xi1>, !llvm.ptr, i32, i32) -> ()
 func.func @arm_sme_load_tile_slice_ver_i16(%src : memref<?x?xi16>, %tile : vector<[8]x[8]xi16>, %tile_slice_index : index) {
   %c0 = arith.constant 0 : index
-  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index, <vertical> : memref<?x?xi16>, vector<[8]x[8]xi16>
+  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index layout<vertical> : memref<?x?xi16>, vector<[8]x[8]xi16>
   return
 }
 
@@ -136,7 +136,7 @@ func.func @arm_sme_load_tile_slice_ver_i16(%src : memref<?x?xi16>, %tile : vecto
 // CHECK: "arm_sme.intr.ld1w.vert"({{.*}}) : (vector<[4]xi1>, !llvm.ptr, i32, i32) -> ()
 func.func @arm_sme_load_tile_slice_ver_i32(%src : memref<?x?xi32>, %tile : vector<[4]x[4]xi32>, %tile_slice_index : index) {
   %c0 = arith.constant 0 : index
-  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index, <vertical> : memref<?x?xi32>, vector<[4]x[4]xi32>
+  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index layout<vertical> : memref<?x?xi32>, vector<[4]x[4]xi32>
   return
 }
 
@@ -146,7 +146,7 @@ func.func @arm_sme_load_tile_slice_ver_i32(%src : memref<?x?xi32>, %tile : vecto
 // CHECK: "arm_sme.intr.ld1d.vert"({{.*}}) : (vector<[2]xi1>, !llvm.ptr, i32, i32) -> ()
 func.func @arm_sme_load_tile_slice_ver_i64(%src : memref<?x?xi64>, %tile : vector<[2]x[2]xi64>, %tile_slice_index : index) {
   %c0 = arith.constant 0 : index
-  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index, <vertical> : memref<?x?xi64>, vector<[2]x[2]xi64>
+  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index layout<vertical> : memref<?x?xi64>, vector<[2]x[2]xi64>
   return
 }
 
@@ -156,7 +156,7 @@ func.func @arm_sme_load_tile_slice_ver_i64(%src : memref<?x?xi64>, %tile : vecto
 // CHECK: "arm_sme.intr.ld1q.vert"({{.*}}) : (vector<[1]xi1>, !llvm.ptr, i32, i32) -> ()
 func.func @arm_sme_load_tile_slice_ver_i128(%src : memref<?x?xi128>, %tile : vector<[1]x[1]xi128>, %tile_slice_index : index) {
   %c0 = arith.constant 0 : index
-  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index, <vertical> : memref<?x?xi128>, vector<[1]x[1]xi128>
+  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index layout<vertical> : memref<?x?xi128>, vector<[1]x[1]xi128>
   return
 }
 
@@ -166,7 +166,7 @@ func.func @arm_sme_load_tile_slice_ver_i128(%src : memref<?x?xi128>, %tile : vec
 // CHECK: "arm_sme.intr.ld1h.vert"({{.*}}) : (vector<[8]xi1>, !llvm.ptr, i32, i32) -> ()
 func.func @arm_sme_load_tile_slice_ver_f16(%src : memref<?x?xf16>, %tile : vector<[8]x[8]xf16>, %tile_slice_index : index) {
   %c0 = arith.constant 0 : index
-  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index, <vertical> : memref<?x?xf16>, vector<[8]x[8]xf16>
+  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index layout<vertical> : memref<?x?xf16>, vector<[8]x[8]xf16>
   return
 }
 
@@ -176,7 +176,7 @@ func.func @arm_sme_load_tile_slice_ver_f16(%src : memref<?x?xf16>, %tile : vecto
 // CHECK: "arm_sme.intr.ld1h.vert"({{.*}}) : (vector<[8]xi1>, !llvm.ptr, i32, i32) -> ()
 func.func @arm_sme_load_tile_slice_ver_bf16(%src : memref<?x?xbf16>, %tile : vector<[8]x[8]xbf16>, %tile_slice_index : index) {
   %c0 = arith.constant 0 : index
-  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index, <vertical> : memref<?x?xbf16>, vector<[8]x[8]xbf16>
+  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index layout<vertical> : memref<?x?xbf16>, vector<[8]x[8]xbf16>
   return
 }
 
@@ -186,7 +186,7 @@ func.func @arm_sme_load_tile_slice_ver_bf16(%src : memref<?x?xbf16>, %tile : vec
 // CHECK: "arm_sme.intr.ld1w.vert"({{.*}}) : (vector<[4]xi1>, !llvm.ptr, i32, i32) -> ()
 func.func @arm_sme_load_tile_slice_ver_f32(%src : memref<?x?xf32>, %tile : vector<[4]x[4]xf32>, %tile_slice_index : index) {
   %c0 = arith.constant 0 : index
-  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index, <vertical> : memref<?x?xf32>, vector<[4]x[4]xf32>
+  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index layout<vertical> : memref<?x?xf32>, vector<[4]x[4]xf32>
   return
 }
 
@@ -196,7 +196,7 @@ func.func @arm_sme_load_tile_slice_ver_f32(%src : memref<?x?xf32>, %tile : vecto
 // CHECK: "arm_sme.intr.ld1d.vert"({{.*}}) : (vector<[2]xi1>, !llvm.ptr, i32, i32) -> ()
 func.func @arm_sme_load_tile_slice_ver_f64(%src : memref<?x?xf64>, %tile : vector<[2]x[2]xf64>, %tile_slice_index : index) {
   %c0 = arith.constant 0 : index
-  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index, <vertical> : memref<?x?xf64>, vector<[2]x[2]xf64>
+  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index layout<vertical> : memref<?x?xf64>, vector<[2]x[2]xf64>
   return
 }
 
@@ -316,7 +316,7 @@ func.func @arm_sme_store_tile_slice_hor_f64(%tile : vector<[2]x[2]xf64>, %tile_s
 // CHECK: "arm_sme.intr.st1b.vert"({{.*}}) : (vector<[16]xi1>, !llvm.ptr, i32, i32) -> ()
 func.func @arm_sme_store_tile_slice_ver_i8(%tile : vector<[16]x[16]xi8>, %tile_slice_index : index, %dest : memref<?x?xi8>) -> () {
   %c0 = arith.constant 0 : index
-  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0], <vertical> : memref<?x?xi8>, vector<[16]x[16]xi8>
+  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0] layout<vertical> : memref<?x?xi8>, vector<[16]x[16]xi8>
   return
 }
 
@@ -326,7 +326,7 @@ func.func @arm_sme_store_tile_slice_ver_i8(%tile : vector<[16]x[16]xi8>, %tile_s
 // CHECK: "arm_sme.intr.st1h.vert"({{.*}}) : (vector<[8]xi1>, !llvm.ptr, i32, i32) -> ()
 func.func @arm_sme_store_tile_slice_ver_i16(%tile : vector<[8]x[8]xi16>, %tile_slice_index : index, %dest : memref<?x?xi16>) -> () {
   %c0 = arith.constant 0 : index
-  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0], <vertical> : memref<?x?xi16>, vector<[8]x[8]xi16>
+  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0] layout<vertical> : memref<?x?xi16>, vector<[8]x[8]xi16>
   return
 }
 
@@ -336,7 +336,7 @@ func.func @arm_sme_store_tile_slice_ver_i16(%tile : vector<[8]x[8]xi16>, %tile_s
 // CHECK: "arm_sme.intr.st1w.vert"({{.*}}) : (vector<[4]xi1>, !llvm.ptr, i32, i32) -> ()
 func.func @arm_sme_store_tile_slice_ver_i32(%tile : vector<[4]x[4]xi32>, %tile_slice_index : index, %dest : memref<?x?xi32>) -> () {
   %c0 = arith.constant 0 : index
-  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0], <vertical> : memref<?x?xi32>, vector<[4]x[4]xi32>
+  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0] layout<vertical> : memref<?x?xi32>, vector<[4]x[4]xi32>
   return
 }
 
@@ -346,7 +346,7 @@ func.func @arm_sme_store_tile_slice_ver_i32(%tile : vector<[4]x[4]xi32>, %tile_s
 // CHECK: "arm_sme.intr.st1d.vert"({{.*}}) : (vector<[2]xi1>, !llvm.ptr, i32, i32) -> ()
 func.func @arm_sme_store_tile_slice_ver_i64(%tile : vector<[2]x[2]xi64>, %tile_slice_index : index, %dest : memref<?x?xi64>) -> () {
   %c0 = arith.constant 0 : index
-  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0], <vertical> : memref<?x?xi64>, vector<[2]x[2]xi64>
+  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0] layout<vertical> : memref<?x?xi64>, vector<[2]x[2]xi64>
   return
 }
 
@@ -356,7 +356,7 @@ func.func @arm_sme_store_tile_slice_ver_i64(%tile : vector<[2]x[2]xi64>, %tile_s
 // CHECK: "arm_sme.intr.st1q.vert"({{.*}}) : (vector<[1]xi1>, !llvm.ptr, i32, i32) -> ()
 func.func @arm_sme_store_tile_slice_ver_i128(%tile : vector<[1]x[1]xi128>, %tile_slice_index : index, %dest : memref<?x?xi128>) -> () {
   %c0 = arith.constant 0 : index
-  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0], <vertical> : memref<?x?xi128>, vector<[1]x[1]xi128>
+  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0] layout<vertical> : memref<?x?xi128>, vector<[1]x[1]xi128>
   return
 }
 
@@ -366,7 +366,7 @@ func.func @arm_sme_store_tile_slice_ver_i128(%tile : vector<[1]x[1]xi128>, %tile
 // CHECK: "arm_sme.intr.st1h.vert"({{.*}}) : (vector<[8]xi1>, !llvm.ptr, i32, i32) -> ()
 func.func @arm_sme_store_tile_slice_ver_f16(%tile : vector<[8]x[8]xf16>, %tile_slice_index : index, %dest : memref<?x?xf16>) -> () {
   %c0 = arith.constant 0 : index
-  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0], <vertical> : memref<?x?xf16>, vector<[8]x[8]xf16>
+  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0] layout<vertical> : memref<?x?xf16>, vector<[8]x[8]xf16>
   return
 }
 
@@ -376,7 +376,7 @@ func.func @arm_sme_store_tile_slice_ver_f16(%tile : vector<[8]x[8]xf16>, %tile_s
 // CHECK: "arm_sme.intr.st1h.vert"({{.*}}) : (vector<[8]xi1>, !llvm.ptr, i32, i32) -> ()
 func.func @arm_sme_store_tile_slice_ver_bf16(%tile : vector<[8]x[8]xbf16>, %tile_slice_index : index, %dest : memref<?x?xbf16>) -> () {
   %c0 = arith.constant 0 : index
-  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0], <vertical> : memref<?x?xbf16>, vector<[8]x[8]xbf16>
+  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0] layout<vertical> : memref<?x?xbf16>, vector<[8]x[8]xbf16>
   return
 }
 
@@ -386,7 +386,7 @@ func.func @arm_sme_store_tile_slice_ver_bf16(%tile : vector<[8]x[8]xbf16>, %tile
 // CHECK: "arm_sme.intr.st1w.vert"({{.*}}) : (vector<[4]xi1>, !llvm.ptr, i32, i32) -> ()
 func.func @arm_sme_store_tile_slice_ver_f32(%tile : vector<[4]x[4]xf32>, %tile_slice_index : index, %dest : memref<?x?xf32>) -> () {
   %c0 = arith.constant 0 : index
-  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0], <vertical> : memref<?x?xf32>, vector<[4]x[4]xf32>
+  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0] layout<vertical> : memref<?x?xf32>, vector<[4]x[4]xf32>
   return
 }
 
@@ -396,7 +396,7 @@ func.func @arm_sme_store_tile_slice_ver_f32(%tile : vector<[4]x[4]xf32>, %tile_s
 // CHECK: "arm_sme.intr.st1d.vert"({{.*}}) : (vector<[2]xi1>, !llvm.ptr, i32, i32) -> ()
 func.func @arm_sme_store_tile_slice_ver_f64(%tile : vector<[2]x[2]xf64>, %tile_slice_index : index, %dest : memref<?x?xf64>) -> () {
   %c0 = arith.constant 0 : index
-  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0], <vertical> : memref<?x?xf64>, vector<[2]x[2]xf64>
+  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0] layout<vertical> : memref<?x?xf64>, vector<[2]x[2]xf64>
   return
 }
 
diff --git a/mlir/test/Dialect/ArmSME/roundtrip.mlir b/mlir/test/Dialect/ArmSME/roundtrip.mlir
index f6d19359b8..427154158e 100644
--- a/mlir/test/Dialect/ArmSME/roundtrip.mlir
+++ b/mlir/test/Dialect/ArmSME/roundtrip.mlir
@@ -358,81 +358,81 @@ func.func @arm_sme_tile_load_hor_f64(%src : memref<?x?xf64>) {
 // -----
 
 func.func @arm_sme_tile_load_ver_i8(%src : memref<?x?xi8>) {
-  // CHECK: arm_sme.tile_load {{.*}}, <vertical> : memref<?x?xi8>, vector<[16]x[16]xi8>
+  // CHECK: arm_sme.tile_load {{.*}} layout<vertical> : memref<?x?xi8>, vector<[16]x[16]xi8>
   %c0 = arith.constant 0 : index
-  %tile = arm_sme.tile_load %src[%c0, %c0], <vertical> : memref<?x?xi8>, vector<[16]x[16]xi8>
+  %tile = arm_sme.tile_load %src[%c0, %c0] layout<vertical> : memref<?x?xi8>, vector<[16]x[16]xi8>
   return
 }
 
 // -----
 
 func.func @arm_sme_tile_load_ver_i16(%src : memref<?x?xi16>) {
-  // CHECK: arm_sme.tile_load {{.*}}, <vertical> : memref<?x?xi16>, vector<[8]x[8]xi16>
+  // CHECK: arm_sme.tile_load {{.*}} layout<vertical> : memref<?x?xi16>, vector<[8]x[8]xi16>
   %c0 = arith.constant 0 : index
-  %tile = arm_sme.tile_load %src[%c0, %c0], <vertical> : memref<?x?xi16>, vector<[8]x[8]xi16>
+  %tile = arm_sme.tile_load %src[%c0, %c0] layout<vertical> : memref<?x?xi16>, vector<[8]x[8]xi16>
   return
 }
 
 // -----
 
 func.func @arm_sme_tile_load_ver_i32(%src : memref<?x?xi32>) {
-  // CHECK: arm_sme.tile_load {{.*}}, <vertical> : memref<?x?xi32>, vector<[4]x[4]xi32>
+  // CHECK: arm_sme.tile_load {{.*}} layout<vertical> : memref<?x?xi32>, vector<[4]x[4]xi32>
   %c0 = arith.constant 0 : index
-  %tile = arm_sme.tile_load %src[%c0, %c0], <vertical> : memref<?x?xi32>, vector<[4]x[4]xi32>
+  %tile = arm_sme.tile_load %src[%c0, %c0] layout<vertical> : memref<?x?xi32>, vector<[4]x[4]xi32>
   return
 }
 
 // -----
 
 func.func @arm_sme_tile_load_ver_i64(%src : memref<?x?xi64>) {
-  // CHECK: arm_sme.tile_load {{.*}}, <vertical> : memref<?x?xi64>, vector<[2]x[2]xi64>
+  // CHECK: arm_sme.tile_load {{.*}} layout<vertical> : memref<?x?xi64>, vector<[2]x[2]xi64>
   %c0 = arith.constant 0 : index
-  %tile = arm_sme.tile_load %src[%c0, %c0], <vertical> : memref<?x?xi64>, vector<[2]x[2]xi64>
+  %tile = arm_sme.tile_load %src[%c0, %c0] layout<vertical> : memref<?x?xi64>, vector<[2]x[2]xi64>
   return
 }
 
 // -----
 
 func.func @arm_sme_tile_load_ver_i128(%src : memref<?x?xi128>) {
-  // CHECK: arm_sme.tile_load {{.*}}, <vertical> : memref<?x?xi128>, vector<[1]x[1]xi128>
+  // CHECK: arm_sme.tile_load {{.*}} layout<vertical> : memref<?x?xi128>, vector<[1]x[1]xi128>
   %c0 = arith.constant 0 : index
-  %tile = arm_sme.tile_load %src[%c0, %c0], <vertical> : memref<?x?xi128>, vector<[1]x[1]xi128>
+  %tile = arm_sme.tile_load %src[%c0, %c0] layout<vertical> : memref<?x?xi128>, vector<[1]x[1]xi128>
   return
 }
 
 // -----
 
 func.func @arm_sme_tile_load_ver_f16(%src : memref<?x?xf16>) {
-  // CHECK: arm_sme.tile_load {{.*}}, <vertical> : memref<?x?xf16>, vector<[8]x[8]xf16>
+  // CHECK: arm_sme.tile_load {{.*}} layout<vertical> : memref<?x?xf16>, vector<[8]x[8]xf16>
   %c0 = arith.constant 0 : index
-  %tile = arm_sme.tile_load %src[%c0, %c0], <vertical> : memref<?x?xf16>, vector<[8]x[8]xf16>
+  %tile = arm_sme.tile_load %src[%c0, %c0] layout<vertical> : memref<?x?xf16>, vector<[8]x[8]xf16>
   return
 }
 
 // -----
 
 func.func @arm_sme_tile_load_ver_bf16(%src : memref<?x?xbf16>) {
-  // CHECK: arm_sme.tile_load {{.*}}, <vertical> : memref<?x?xbf16>, vector<[8]x[8]xbf16>
+  // CHECK: arm_sme.tile_load {{.*}} layout<vertical> : memref<?x?xbf16>, vector<[8]x[8]xbf16>
   %c0 = arith.constant 0 : index
-  %tile = arm_sme.tile_load %src[%c0, %c0], <vertical> : memref<?x?xbf16>, vector<[8]x[8]xbf16>
+  %tile = arm_sme.tile_load %src[%c0, %c0] layout<vertical> : memref<?x?xbf16>, vector<[8]x[8]xbf16>
   return
 }
 
 // -----
 
 func.func @arm_sme_tile_load_ver_f32(%src : memref<?x?xf32>) {
-  // CHECK: arm_sme.tile_load {{.*}}, <vertical> : memref<?x?xf32>, vector<[4]x[4]xf32>
+  // CHECK: arm_sme.tile_load {{.*}} layout<vertical> : memref<?x?xf32>, vector<[4]x[4]xf32>
   %c0 = arith.constant 0 : index
-  %tile = arm_sme.tile_load %src[%c0, %c0], <vertical> : memref<?x?xf32>, vector<[4]x[4]xf32>
+  %tile = arm_sme.tile_load %src[%c0, %c0] layout<vertical> : memref<?x?xf32>, vector<[4]x[4]xf32>
   return
 }
 
 // -----
 
 func.func @arm_sme_tile_load_ver_f64(%src : memref<?x?xf64>) {
-  // CHECK: arm_sme.tile_load {{.*}}, <vertical> : memref<?x?xf64>, vector<[2]x[2]xf64>
+  // CHECK: arm_sme.tile_load {{.*}} layout<vertical> : memref<?x?xf64>, vector<[2]x[2]xf64>
   %c0 = arith.constant 0 : index
-  %tile = arm_sme.tile_load %src[%c0, %c0], <vertical> : memref<?x?xf64>, vector<[2]x[2]xf64>
+  %tile = arm_sme.tile_load %src[%c0, %c0] layout<vertical> : memref<?x?xf64>, vector<[2]x[2]xf64>
   return
 }
 
@@ -442,7 +442,7 @@ func.func @arm_sme_tile_load_ver_f64(%src : memref<?x?xf64>) {
 func.func @arm_sme_tile_load_explicit_hor(%src : memref<?x?xi8>) {
   // CHECK: arm_sme.tile_load %{{.*}}[{{.*}}] : memref<?x?xi8>, vector<[16]x[16]xi8>
   %c0 = arith.constant 0 : index
-  %tile = arm_sme.tile_load %src[%c0, %c0], <horizontal> : memref<?x?xi8>, vector<[16]x[16]xi8>
+  %tile = arm_sme.tile_load %src[%c0, %c0] layout<horizontal> : memref<?x?xi8>, vector<[16]x[16]xi8>
   return
 }
 
@@ -534,81 +534,81 @@ func.func @arm_sme_tile_store_hor_f64(%tile : vector<[2]x[2]xf64>, %dest : memre
 // -----
 
 func.func @arm_sme_tile_store_ver_i8(%tile : vector<[16]x[16]xi8>, %dest : memref<?x?xi8>) {
-  // CHECK: arm_sme.tile_store {{.*}}, <vertical> : memref<?x?xi8>, vector<[16]x[16]xi8>
+  // CHECK: arm_sme.tile_store {{.*}} layout<vertical> : memref<?x?xi8>, vector<[16]x[16]xi8>
   %c0 = arith.constant 0 : index
-  arm_sme.tile_store %tile, %dest[%c0, %c0], <vertical> : memref<?x?xi8>, vector<[16]x[16]xi8>
+  arm_sme.tile_store %tile, %dest[%c0, %c0] layout<vertical> : memref<?x?xi8>, vector<[16]x[16]xi8>
   return
 }
 
 // -----
 
 func.func @arm_sme_tile_store_ver_i16(%tile : vector<[8]x[8]xi16>, %dest : memref<?x?xi16>) {
-  // CHECK: arm_sme.tile_store {{.*}}, <vertical> : memref<?x?xi16>, vector<[8]x[8]xi16>
+  // CHECK: arm_sme.tile_store {{.*}} layout<vertical> : memref<?x?xi16>, vector<[8]x[8]xi16>
   %c0 = arith.constant 0 : index
-  arm_sme.tile_store %tile, %dest[%c0, %c0], <vertical> : memref<?x?xi16>, vector<[8]x[8]xi16>
+  arm_sme.tile_store %tile, %dest[%c0, %c0] layout<vertical> : memref<?x?xi16>, vector<[8]x[8]xi16>
   return
 }
 
 // -----
 
 func.func @arm_sme_tile_store_ver_i32(%tile : vector<[4]x[4]xi32>, %dest : memref<?x?xi32>) {
-  // CHECK: arm_sme.tile_store {{.*}}, <vertical> : memref<?x?xi32>, vector<[4]x[4]xi32>
+  // CHECK: arm_sme.tile_store {{.*}} layout<vertical> : memref<?x?xi32>, vector<[4]x[4]xi32>
   %c0 = arith.constant 0 : index
-  arm_sme.tile_store %tile, %dest[%c0, %c0], <vertical> : memref<?x?xi32>, vector<[4]x[4]xi32>
+  arm_sme.tile_store %tile, %dest[%c0, %c0] layout<vertical> : memref<?x?xi32>, vector<[4]x[4]xi32>
   return
 }
 
 // -----
 
 func.func @arm_sme_tile_store_ver_i64(%tile : vector<[2]x[2]xi64>, %dest : memref<?x?xi64>) {
-  // CHECK: arm_sme.tile_store {{.*}}, <vertical> : memref<?x?xi64>, vector<[2]x[2]xi64>
+  // CHECK: arm_sme.tile_store {{.*}} layout<vertical> : memref<?x?xi64>, vector<[2]x[2]xi64>
   %c0 = arith.constant 0 : index
-  arm_sme.tile_store %tile, %dest[%c0, %c0], <vertical> : memref<?x?xi64>, vector<[2]x[2]xi64>
+  arm_sme.tile_store %tile, %dest[%c0, %c0] layout<vertical> : memref<?x?xi64>, vector<[2]x[2]xi64>
   return
 }
 
 // -----
 
 func.func @arm_sme_tile_store_ver_i128(%tile : vector<[1]x[1]xi128>, %dest : memref<?x?xi128>) {
-  // CHECK: arm_sme.tile_store {{.*}}, <vertical> : memref<?x?xi128>, vector<[1]x[1]xi128>
+  // CHECK: arm_sme.tile_store {{.*}} layout<vertical> : memref<?x?xi128>, vector<[1]x[1]xi128>
   %c0 = arith.constant 0 : index
-  arm_sme.tile_store %tile, %dest[%c0, %c0], <vertical> : memref<?x?xi128>, vector<[1]x[1]xi128>
+  arm_sme.tile_store %tile, %dest[%c0, %c0] layout<vertical> : memref<?x?xi128>, vector<[1]x[1]xi128>
   return
 }
 
 // -----
 
 func.func @arm_sme_tile_store_ver_f16(%tile : vector<[8]x[8]xf16>, %dest : memref<?x?xf16>) {
-  // CHECK: arm_sme.tile_store {{.*}}, <vertical> : memref<?x?xf16>, vector<[8]x[8]xf16>
+  // CHECK: arm_sme.tile_store {{.*}} layout<vertical> : memref<?x?xf16>, vector<[8]x[8]xf16>
   %c0 = arith.constant 0 : index
-  arm_sme.tile_store %tile, %dest[%c0, %c0], <vertical> : memref<?x?xf16>, vector<[8]x[8]xf16>
+  arm_sme.tile_store %tile, %dest[%c0, %c0] layout<vertical> : memref<?x?xf16>, vector<[8]x[8]xf16>
   return
 }
 
 // -----
 
 func.func @arm_sme_tile_store_ver_bf16(%tile : vector<[8]x[8]xbf16>, %dest : memref<?x?xbf16>) {
-  // CHECK: arm_sme.tile_store {{.*}}, <vertical> : memref<?x?xbf16>, vector<[8]x[8]xbf16>
+  // CHECK: arm_sme.tile_store {{.*}} layout<vertical> : memref<?x?xbf16>, vector<[8]x[8]xbf16>
   %c0 = arith.constant 0 : index
-  arm_sme.tile_store %tile, %dest[%c0, %c0], <vertical> : memref<?x?xbf16>, vector<[8]x[8]xbf16>
+  arm_sme.tile_store %tile, %dest[%c0, %c0] layout<vertical> : memref<?x?xbf16>, vector<[8]x[8]xbf16>
   return
 }
 
 // -----
 
 func.func @arm_sme_tile_store_ver_f32(%tile : vector<[4]x[4]xf32>, %dest : memref<?x?xf32>) {
-  // CHECK: arm_sme.tile_store {{.*}}, <vertical> : memref<?x?xf32>, vector<[4]x[4]xf32>
+  // CHECK: arm_sme.tile_store {{.*}} layout<vertical> : memref<?x?xf32>, vector<[4]x[4]xf32>
   %c0 = arith.constant 0 : index
-  arm_sme.tile_store %tile, %dest[%c0, %c0], <vertical> : memref<?x?xf32>, vector<[4]x[4]xf32>
+  arm_sme.tile_store %tile, %dest[%c0, %c0] layout<vertical> : memref<?x?xf32>, vector<[4]x[4]xf32>
   return
 }
 
 // -----
 
 func.func @arm_sme_tile_store_ver_f64(%tile : vector<[2]x[2]xf64>, %dest : memref<?x?xf64>) {
-  // CHECK: arm_sme.tile_store {{.*}}, <vertical> : memref<?x?xf64>, vector<[2]x[2]xf64>
+  // CHECK: arm_sme.tile_store {{.*}} layout<vertical> : memref<?x?xf64>, vector<[2]x[2]xf64>
   %c0 = arith.constant 0 : index
-  arm_sme.tile_store %tile, %dest[%c0, %c0], <vertical> : memref<?x?xf64>, vector<[2]x[2]xf64>
+  arm_sme.tile_store %tile, %dest[%c0, %c0] layout<vertical> : memref<?x?xf64>, vector<[2]x[2]xf64>
   return
 }
 
@@ -618,7 +618,7 @@ func.func @arm_sme_tile_store_ver_f64(%tile : vector<[2]x[2]xf64>, %dest : memre
 func.func @arm_sme_tile_store_ver_i8(%tile : vector<[16]x[16]xi8>, %dest : memref<?x?xi8>) {
   // CHECK: arm_sme.tile_store %{{.*}}[{{.*}}] : memref<?x?xi8>, vector<[16]x[16]xi8>
   %c0 = arith.constant 0 : index
-  arm_sme.tile_store %tile, %dest[%c0, %c0], <horizontal> : memref<?x?xi8>, vector<[16]x[16]xi8>
+  arm_sme.tile_store %tile, %dest[%c0, %c0] layout<horizontal> : memref<?x?xi8>, vector<[16]x[16]xi8>
   return
 }
 
@@ -710,81 +710,81 @@ func.func @arm_sme_load_tile_slice_hor_f64(%src : memref<?x?xf64>, %tile : vecto
 // -----
 
 func.func @arm_sme_load_tile_slice_ver_i8(%src : memref<?x?xi8>, %tile : vector<[16]x[16]xi8>, %tile_slice_index : index) {
-  // CHECK: arm_sme.load_tile_slice {{.*}}, <vertical> : memref<?x?xi8>, vector<[16]x[16]xi8>
+  // CHECK: arm_sme.load_tile_slice {{.*}} layout<vertical> : memref<?x?xi8>, vector<[16]x[16]xi8>
   %c0 = arith.constant 0 : index
-  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index, <vertical> : memref<?x?xi8>, vector<[16]x[16]xi8>
+  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index layout<vertical> : memref<?x?xi8>, vector<[16]x[16]xi8>
   return
 }
 
 // -----
 
 func.func @arm_sme_load_tile_slice_ver_i16(%src : memref<?x?xi16>, %tile : vector<[8]x[8]xi16>, %tile_slice_index : index) {
-  // CHECK: arm_sme.load_tile_slice {{.*}}, <vertical> : memref<?x?xi16>, vector<[8]x[8]xi16>
+  // CHECK: arm_sme.load_tile_slice {{.*}} layout<vertical> : memref<?x?xi16>, vector<[8]x[8]xi16>
   %c0 = arith.constant 0 : index
-  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index, <vertical> : memref<?x?xi16>, vector<[8]x[8]xi16>
+  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index layout<vertical> : memref<?x?xi16>, vector<[8]x[8]xi16>
   return
 }
 
 // -----
 
 func.func @arm_sme_load_tile_slice_ver_i32(%src : memref<?x?xi32>, %tile : vector<[4]x[4]xi32>, %tile_slice_index : index) {
-  // CHECK: arm_sme.load_tile_slice {{.*}}, <vertical> : memref<?x?xi32>, vector<[4]x[4]xi32>
+  // CHECK: arm_sme.load_tile_slice {{.*}} layout<vertical> : memref<?x?xi32>, vector<[4]x[4]xi32>
   %c0 = arith.constant 0 : index
-  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index, <vertical> : memref<?x?xi32>, vector<[4]x[4]xi32>
+  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index layout<vertical> : memref<?x?xi32>, vector<[4]x[4]xi32>
   return
 }
 
 // -----
 
 func.func @arm_sme_load_tile_slice_ver_i64(%src : memref<?x?xi64>, %tile : vector<[2]x[2]xi64>, %tile_slice_index : index) {
-  // CHECK: arm_sme.load_tile_slice {{.*}}, <vertical> : memref<?x?xi64>, vector<[2]x[2]xi64>
+  // CHECK: arm_sme.load_tile_slice {{.*}} layout<vertical> : memref<?x?xi64>, vector<[2]x[2]xi64>
   %c0 = arith.constant 0 : index
-  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index, <vertical> : memref<?x?xi64>, vector<[2]x[2]xi64>
+  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index layout<vertical> : memref<?x?xi64>, vector<[2]x[2]xi64>
   return
 }
 
 // -----
 
 func.func @arm_sme_load_tile_slice_ver_i128(%src : memref<?x?xi128>, %tile : vector<[1]x[1]xi128>, %tile_slice_index : index) {
-  // CHECK: arm_sme.load_tile_slice {{.*}}, <vertical> : memref<?x?xi128>, vector<[1]x[1]xi128>
+  // CHECK: arm_sme.load_tile_slice {{.*}} layout<vertical> : memref<?x?xi128>, vector<[1]x[1]xi128>
   %c0 = arith.constant 0 : index
-  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index, <vertical> : memref<?x?xi128>, vector<[1]x[1]xi128>
+  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index layout<vertical> : memref<?x?xi128>, vector<[1]x[1]xi128>
   return
 }
 
 // -----
 
 func.func @arm_sme_load_tile_slice_ver_f16(%src : memref<?x?xf16>, %tile : vector<[8]x[8]xf16>, %tile_slice_index : index) {
-  // CHECK: arm_sme.load_tile_slice {{.*}}, <vertical> : memref<?x?xf16>, vector<[8]x[8]xf16>
+  // CHECK: arm_sme.load_tile_slice {{.*}} layout<vertical> : memref<?x?xf16>, vector<[8]x[8]xf16>
   %c0 = arith.constant 0 : index
-  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index, <vertical> : memref<?x?xf16>, vector<[8]x[8]xf16>
+  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index layout<vertical> : memref<?x?xf16>, vector<[8]x[8]xf16>
   return
 }
 
 // -----
 
 func.func @arm_sme_load_tile_slice_ver_bf16(%src : memref<?x?xbf16>, %tile : vector<[8]x[8]xbf16>, %tile_slice_index : index) {
-  // CHECK: arm_sme.load_tile_slice {{.*}}, <vertical> : memref<?x?xbf16>, vector<[8]x[8]xbf16>
+  // CHECK: arm_sme.load_tile_slice {{.*}} layout<vertical> : memref<?x?xbf16>, vector<[8]x[8]xbf16>
   %c0 = arith.constant 0 : index
-  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index, <vertical> : memref<?x?xbf16>, vector<[8]x[8]xbf16>
+  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index layout<vertical> : memref<?x?xbf16>, vector<[8]x[8]xbf16>
   return
 }
 
 // -----
 
 func.func @arm_sme_load_tile_slice_ver_f32(%src : memref<?x?xf32>, %tile : vector<[4]x[4]xf32>, %tile_slice_index : index) {
-  // CHECK: arm_sme.load_tile_slice {{.*}}, <vertical> : memref<?x?xf32>, vector<[4]x[4]xf32>
+  // CHECK: arm_sme.load_tile_slice {{.*}} layout<vertical> : memref<?x?xf32>, vector<[4]x[4]xf32>
   %c0 = arith.constant 0 : index
-  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index, <vertical> : memref<?x?xf32>, vector<[4]x[4]xf32>
+  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index layout<vertical> : memref<?x?xf32>, vector<[4]x[4]xf32>
   return
 }
 
 // -----
 
 func.func @arm_sme_load_tile_slice_ver_f64(%src : memref<?x?xf64>, %tile : vector<[2]x[2]xf64>, %tile_slice_index : index) {
-  // CHECK: arm_sme.load_tile_slice {{.*}}, <vertical> : memref<?x?xf64>, vector<[2]x[2]xf64>
+  // CHECK: arm_sme.load_tile_slice {{.*}} layout<vertical> : memref<?x?xf64>, vector<[2]x[2]xf64>
   %c0 = arith.constant 0 : index
-  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index, <vertical> : memref<?x?xf64>, vector<[2]x[2]xf64>
+  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index layout<vertical> : memref<?x?xf64>, vector<[2]x[2]xf64>
   return
 }
 
@@ -794,7 +794,7 @@ func.func @arm_sme_load_tile_slice_ver_f64(%src : memref<?x?xf64>, %tile : vecto
 func.func @arm_sme_load_tile_slice_hor_i8(%src : memref<?x?xi8>, %tile : vector<[16]x[16]xi8>, %tile_slice_index : index) {
   // CHECK: arm_sme.load_tile_slice %{{.*}}[{{.*}}], %{{.*}}, %{{.*}} : memref<?x?xi8>, vector<[16]x[16]xi8>
   %c0 = arith.constant 0 : index
-  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index, <horizontal> : memref<?x?xi8>, vector<[16]x[16]xi8>
+  %tile_update = arm_sme.load_tile_slice %src[%c0], %tile, %tile_slice_index layout<horizontal> : memref<?x?xi8>, vector<[16]x[16]xi8>
   return
 }
 
@@ -886,81 +886,81 @@ func.func @arm_sme_store_tile_slice_hor_f64(%tile : vector<[2]x[2]xf64>, %tile_s
 // -----
 
 func.func @arm_sme_store_tile_slice_ver_i8(%tile : vector<[16]x[16]xi8>, %tile_slice_index : index, %dest : memref<?x?xi8>) -> () {
-  // CHECK: arm_sme.store_tile_slice {{.*}}, <vertical> : memref<?x?xi8>, vector<[16]x[16]xi8>
+  // CHECK: arm_sme.store_tile_slice {{.*}} layout<vertical> : memref<?x?xi8>, vector<[16]x[16]xi8>
   %c0 = arith.constant 0 : index
-  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0], <vertical> : memref<?x?xi8>, vector<[16]x[16]xi8>
+  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0] layout<vertical> : memref<?x?xi8>, vector<[16]x[16]xi8>
   return
 }
 
 // -----
 
 func.func @arm_sme_store_tile_slice_ver_i16(%tile : vector<[8]x[8]xi16>, %tile_slice_index : index, %dest : memref<?x?xi16>) -> () {
-  // CHECK: arm_sme.store_tile_slice {{.*}}, <vertical> : memref<?x?xi16>, vector<[8]x[8]xi16>
+  // CHECK: arm_sme.store_tile_slice {{.*}} layout<vertical> : memref<?x?xi16>, vector<[8]x[8]xi16>
   %c0 = arith.constant 0 : index
-  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0], <vertical> : memref<?x?xi16>, vector<[8]x[8]xi16>
+  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0] layout<vertical> : memref<?x?xi16>, vector<[8]x[8]xi16>
   return
 }
 
 // -----
 
 func.func @arm_sme_store_tile_slice_ver_i32(%tile : vector<[4]x[4]xi32>, %tile_slice_index : index, %dest : memref<?x?xi32>) -> () {
-  // CHECK: arm_sme.store_tile_slice {{.*}}, <vertical> : memref<?x?xi32>, vector<[4]x[4]xi32>
+  // CHECK: arm_sme.store_tile_slice {{.*}} layout<vertical> : memref<?x?xi32>, vector<[4]x[4]xi32>
   %c0 = arith.constant 0 : index
-  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0], <vertical> : memref<?x?xi32>, vector<[4]x[4]xi32>
+  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0] layout<vertical> : memref<?x?xi32>, vector<[4]x[4]xi32>
   return
 }
 
 // -----
 
 func.func @arm_sme_store_tile_slice_ver_i64(%tile : vector<[2]x[2]xi64>, %tile_slice_index : index, %dest : memref<?x?xi64>) -> () {
-  // CHECK: arm_sme.store_tile_slice {{.*}}, <vertical> : memref<?x?xi64>, vector<[2]x[2]xi64>
+  // CHECK: arm_sme.store_tile_slice {{.*}} layout<vertical> : memref<?x?xi64>, vector<[2]x[2]xi64>
   %c0 = arith.constant 0 : index
-  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0], <vertical> : memref<?x?xi64>, vector<[2]x[2]xi64>
+  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0] layout<vertical> : memref<?x?xi64>, vector<[2]x[2]xi64>
   return
 }
 
 // -----
 
 func.func @arm_sme_store_tile_slice_ver_i128(%tile : vector<[1]x[1]xi128>, %tile_slice_index : index, %dest : memref<?x?xi128>) -> () {
-  // CHECK: arm_sme.store_tile_slice {{.*}}, <vertical> : memref<?x?xi128>, vector<[1]x[1]xi128>
+  // CHECK: arm_sme.store_tile_slice {{.*}} layout<vertical> : memref<?x?xi128>, vector<[1]x[1]xi128>
   %c0 = arith.constant 0 : index
-  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0], <vertical> : memref<?x?xi128>, vector<[1]x[1]xi128>
+  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0] layout<vertical> : memref<?x?xi128>, vector<[1]x[1]xi128>
   return
 }
 
 // -----
 
 func.func @arm_sme_store_tile_slice_ver_f16(%tile : vector<[8]x[8]xf16>, %tile_slice_index : index, %dest : memref<?x?xf16>) -> () {
-  // CHECK: arm_sme.store_tile_slice {{.*}}, <vertical> : memref<?x?xf16>, vector<[8]x[8]xf16>
+  // CHECK: arm_sme.store_tile_slice {{.*}} layout<vertical> : memref<?x?xf16>, vector<[8]x[8]xf16>
   %c0 = arith.constant 0 : index
-  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0], <vertical> : memref<?x?xf16>, vector<[8]x[8]xf16>
+  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0] layout<vertical> : memref<?x?xf16>, vector<[8]x[8]xf16>
   return
 }
 
 // -----
 
 func.func @arm_sme_store_tile_slice_ver_bf16(%tile : vector<[8]x[8]xbf16>, %tile_slice_index : index, %dest : memref<?x?xbf16>) -> () {
-  // CHECK: arm_sme.store_tile_slice {{.*}}, <vertical> : memref<?x?xbf16>, vector<[8]x[8]xbf16>
+  // CHECK: arm_sme.store_tile_slice {{.*}} layout<vertical> : memref<?x?xbf16>, vector<[8]x[8]xbf16>
   %c0 = arith.constant 0 : index
-  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0], <vertical> : memref<?x?xbf16>, vector<[8]x[8]xbf16>
+  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0] layout<vertical> : memref<?x?xbf16>, vector<[8]x[8]xbf16>
   return
 }
 
 // -----
 
 func.func @arm_sme_store_tile_slice_ver_f32(%tile : vector<[4]x[4]xf32>, %tile_slice_index : index, %dest : memref<?x?xf32>) -> () {
-  // CHECK: arm_sme.store_tile_slice {{.*}}, <vertical> : memref<?x?xf32>, vector<[4]x[4]xf32>
+  // CHECK: arm_sme.store_tile_slice {{.*}} layout<vertical> : memref<?x?xf32>, vector<[4]x[4]xf32>
   %c0 = arith.constant 0 : index
-  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0], <vertical> : memref<?x?xf32>, vector<[4]x[4]xf32>
+  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0] layout<vertical> : memref<?x?xf32>, vector<[4]x[4]xf32>
   return
 }
 
 // -----
 
 func.func @arm_sme_store_tile_slice_ver_f64(%tile : vector<[2]x[2]xf64>, %tile_slice_index : index, %dest : memref<?x?xf64>) -> () {
-  // CHECK: arm_sme.store_tile_slice {{.*}}, <vertical> : memref<?x?xf64>, vector<[2]x[2]xf64>
+  // CHECK: arm_sme.store_tile_slice {{.*}} layout<vertical> : memref<?x?xf64>, vector<[2]x[2]xf64>
   %c0 = arith.constant 0 : index
-  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0], <vertical> : memref<?x?xf64>, vector<[2]x[2]xf64>
+  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0] layout<vertical> : memref<?x?xf64>, vector<[2]x[2]xf64>
   return
 }
 
@@ -970,7 +970,7 @@ func.func @arm_sme_store_tile_slice_ver_f64(%tile : vector<[2]x[2]xf64>, %tile_s
 func.func @arm_sme_store_tile_slice_hor_i8(%tile : vector<[16]x[16]xi8>, %tile_slice_index : index, %dest : memref<?x?xi8>) -> () {
   // CHECK: arm_sme.store_tile_slice {{.*}}, {{.*}}, %{{.*}}[{{.*}}] : memref<?x?xi8>, vector<[16]x[16]xi8>
   %c0 = arith.constant 0 : index
-  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0], <horizontal> : memref<?x?xi8>, vector<[16]x[16]xi8>
+  arm_sme.store_tile_slice %tile, %tile_slice_index, %dest[%c0] layout<horizontal> : memref<?x?xi8>, vector<[16]x[16]xi8>
   return
 }
 
diff --git a/mlir/test/Dialect/ArmSME/vector-ops-to-sme.mlir b/mlir/test/Dialect/ArmSME/vector-ops-to-sme.mlir
index b2c8fd8e01..455b47a83e 100644
--- a/mlir/test/Dialect/ArmSME/vector-ops-to-sme.mlir
+++ b/mlir/test/Dialect/ArmSME/vector-ops-to-sme.mlir
@@ -5,7 +5,7 @@
 //===----------------------------------------------------------------------===//
 
 // CHECK-LABEL: @transfer_read_2d_transpose_i8
-// CHECK: arm_sme.tile_load {{.*}}, <vertical> : memref<?x?xi8>, vector<[16]x[16]xi8>
+// CHECK: arm_sme.tile_load {{.*}} layout<vertical> : memref<?x?xi8>, vector<[16]x[16]xi8>
 func.func @transfer_read_2d_transpose_i8(%src : memref<?x?xi8>) {
   %c0 = arith.constant 0 : index
   %pad = arith.constant 0 : i8
@@ -17,7 +17,7 @@ func.func @transfer_read_2d_transpose_i8(%src : memref<?x?xi8>) {
 // -----
 
 // CHECK-LABEL: @transfer_read_2d_transpose_i16
-// CHECK: arm_sme.tile_load {{.*}}, <vertical> : memref<?x?xi16>, vector<[8]x[8]xi16>
+// CHECK: arm_sme.tile_load {{.*}} layout<vertical> : memref<?x?xi16>, vector<[8]x[8]xi16>
 func.func @transfer_read_2d_transpose_i16(%src : memref<?x?xi16>) {
   %c0 = arith.constant 0 : index
   %pad = arith.constant 0 : i16
@@ -29,7 +29,7 @@ func.func @transfer_read_2d_transpose_i16(%src : memref<?x?xi16>) {
 // -----
 
 // CHECK-LABEL: @transfer_read_2d_transpose_i32
-// CHECK: arm_sme.tile_load {{.*}}, <vertical> : memref<?x?xi32>, vector<[4]x[4]xi32>
+// CHECK: arm_sme.tile_load {{.*}} layout<vertical> : memref<?x?xi32>, vector<[4]x[4]xi32>
 func.func @transfer_read_2d_transpose_i32(%src : memref<?x?xi32>) {
   %c0 = arith.constant 0 : index
   %pad = arith.constant 0 : i32
@@ -41,7 +41,7 @@ func.func @transfer_read_2d_transpose_i32(%src : memref<?x?xi32>) {
 // -----
 
 // CHECK-LABEL: @transfer_read_2d_transpose_i64
-// CHECK: arm_sme.tile_load {{.*}}, <vertical> : memref<?x?xi64>, vector<[2]x[2]xi64>
+// CHECK: arm_sme.tile_load {{.*}} layout<vertical> : memref<?x?xi64>, vector<[2]x[2]xi64>
 func.func @transfer_read_2d_transpose_i64(%src : memref<?x?xi64>) {
   %c0 = arith.constant 0 : index
   %pad = arith.constant 0 : i64
@@ -53,7 +53,7 @@ func.func @transfer_read_2d_transpose_i64(%src : memref<?x?xi64>) {
 // -----
 
 // CHECK-LABEL: @transfer_read_2d_transpose_i128
-// CHECK: arm_sme.tile_load {{.*}}, <vertical> : memref<?x?xi128>, vector<[1]x[1]xi128>
+// CHECK: arm_sme.tile_load {{.*}} layout<vertical> : memref<?x?xi128>, vector<[1]x[1]xi128>
 func.func @transfer_read_2d_transpose_i128(%src : memref<?x?xi128>) {
   %c0 = arith.constant 0 : index
   %pad = arith.constant 0 : i128
@@ -65,7 +65,7 @@ func.func @transfer_read_2d_transpose_i128(%src : memref<?x?xi128>) {
 // -----
 
 // CHECK-LABEL: @transfer_read_2d_transpose_f16
-// CHECK: arm_sme.tile_load {{.*}}, <vertical> : memref<?x?xf16>, vector<[8]x[8]xf16>
+// CHECK: arm_sme.tile_load {{.*}} layout<vertical> : memref<?x?xf16>, vector<[8]x[8]xf16>
 func.func @transfer_read_2d_transpose_f16(%src : memref<?x?xf16>) {
   %c0 = arith.constant 0 : index
   %pad = arith.constant 0.0 : f16
@@ -77,7 +77,7 @@ func.func @transfer_read_2d_transpose_f16(%src : memref<?x?xf16>) {
 // -----
 
 // CHECK-LABEL: @transfer_read_2d_transpose_bf16
-// CHECK: arm_sme.tile_load {{.*}}, <vertical> : memref<?x?xbf16>, vector<[8]x[8]xbf16>
+// CHECK: arm_sme.tile_load {{.*}} layout<vertical> : memref<?x?xbf16>, vector<[8]x[8]xbf16>
 func.func @transfer_read_2d_transpose_bf16(%src : memref<?x?xbf16>) {
   %c0 = arith.constant 0 : index
   %pad = arith.constant 0.0 : bf16
@@ -89,7 +89,7 @@ func.func @transfer_read_2d_transpose_bf16(%src : memref<?x?xbf16>) {
 // -----
 
 // CHECK-LABEL: @transfer_read_2d_transpose_f32
-// CHECK: arm_sme.tile_load {{.*}}, <vertical> : memref<?x?xf32>, vector<[4]x[4]xf32>
+// CHECK: arm_sme.tile_load {{.*}} layout<vertical> : memref<?x?xf32>, vector<[4]x[4]xf32>
 func.func @transfer_read_2d_transpose_f32(%src : memref<?x?xf32>) {
   %c0 = arith.constant 0 : index
   %pad = arith.constant 0.0 : f32
@@ -101,7 +101,7 @@ func.func @transfer_read_2d_transpose_f32(%src : memref<?x?xf32>) {
 // -----
 
 // CHECK-LABEL: @transfer_read_2d_transpose_f64
-// CHECK: arm_sme.tile_load {{.*}}, <vertical> : memref<?x?xf64>, vector<[2]x[2]xf64>
+// CHECK: arm_sme.tile_load {{.*}} layout<vertical> : memref<?x?xf64>, vector<[2]x[2]xf64>
 func.func @transfer_read_2d_transpose_f64(%src : memref<?x?xf64>) {
   %c0 = arith.constant 0 : index
   %pad = arith.constant 0.0 : f64
@@ -475,7 +475,7 @@ func.func @splat_vec2d_from_f16(%arg0: f16) {
 // CHECK:           %[[MIN_TILE_SLICES:.*]] = arith.muli %[[VSCALE]], %[[C16]] : index
 // CHECK:           %[[NUM_TILE_SLICES:.*]] = memref.alloca(%[[MIN_TILE_SLICES]], %[[MIN_TILE_SLICES]]) : memref<?x?xi8>
 // CHECK:           arm_sme.tile_store %[[TILE]], %[[NUM_TILE_SLICES]]{{\[}}%[[C0]], %[[C0]]] : memref<?x?xi8>, vector<[16]x[16]xi8>
-// CHECK:           arm_sme.tile_load %[[NUM_TILE_SLICES]]{{\[}}%[[C0]], %[[C0]]], <vertical> : memref<?x?xi8>, vector<[16]x[16]xi8>
+// CHECK:           arm_sme.tile_load %[[NUM_TILE_SLICES]]{{\[}}%[[C0]], %[[C0]]] layout<vertical> : memref<?x?xi8>, vector<[16]x[16]xi8>
 func.func @transpose_i8(%arg0: vector<[16]x[16]xi8>) {
   %0 = vector.transpose %arg0, [1, 0] : vector<[16]x[16]xi8> to vector<[16]x[16]xi8>
   "prevent.dce"(%0) : (vector<[16]x[16]xi8>) -> ()
@@ -487,7 +487,7 @@ func.func @transpose_i8(%arg0: vector<[16]x[16]xi8>) {
 // CHECK-LABEL: @transpose_i16
 // CHECK: arith.constant 8
 // CHECK: arm_sme.tile_store {{.*}} : memref<?x?xi16>, vector<[8]x[8]xi16>
-// CHECK: arm_sme.tile_load {{.*}}, <vertical> : memref<?x?xi16>, vector<[8]x[8]xi16>
+// CHECK: arm_sme.tile_load {{.*}} layout<vertical> : memref<?x?xi16>, vector<[8]x[8]xi16>
 func.func @transpose_i16(%arg0: vector<[8]x[8]xi16>) {
   %0 = vector.transpose %arg0, [1, 0] : vector<[8]x[8]xi16> to vector<[8]x[8]xi16>
   "prevent.dce"(%0) : (vector<[8]x[8]xi16>) -> ()
@@ -499,7 +499,7 @@ func.func @transpose_i16(%arg0: vector<[8]x[8]xi16>) {
 // CHECK-LABEL: @transpose_i32
 // CHECK: arith.constant 4
 // CHECK: arm_sme.tile_store {{.*}} : memref<?x?xi32>, vector<[4]x[4]xi32>
-// CHECK: arm_sme.tile_load {{.*}}, <vertical> : memref<?x?xi32>, vector<[4]x[4]xi32>
+// CHECK: arm_sme.tile_load {{.*}} layout<vertical> : memref<?x?xi32>, vector<[4]x[4]xi32>
 func.func @transpose_i32(%arg0: vector<[4]x[4]xi32>) {
   %0 = vector.transpose %arg0, [1, 0] : vector<[4]x[4]xi32> to vector<[4]x[4]xi32>
   "prevent.dce"(%0) : (vector<[4]x[4]xi32>) -> ()
@@ -511,7 +511,7 @@ func.func @transpose_i32(%arg0: vector<[4]x[4]xi32>) {
 // CHECK-LABEL: @transpose_i64
 // CHECK: arith.constant 2
 // CHECK: arm_sme.tile_store {{.*}} : memref<?x?xi64>, vector<[2]x[2]xi64>
-// CHECK: arm_sme.tile_load {{.*}}, <vertical> : memref<?x?xi64>, vector<[2]x[2]xi64>
+// CHECK: arm_sme.tile_load {{.*}} layout<vertical> : memref<?x?xi64>, vector<[2]x[2]xi64>
 func.func @transpose_i64(%arg0: vector<[2]x[2]xi64>) {
   %0 = vector.transpose %arg0, [1, 0] : vector<[2]x[2]xi64> to vector<[2]x[2]xi64>
   "prevent.dce"(%0) : (vector<[2]x[2]xi64>) -> ()
@@ -524,7 +524,7 @@ func.func @transpose_i64(%arg0: vector<[2]x[2]xi64>) {
 // CHECK: %[[VSCALE:.*]] = vector.vscale
 // CHECK: %[[NUM_TILE_SLICES:.*]] = memref.alloca(%[[VSCALE]], %[[VSCALE]]) : memref<?x?xi128>
 // CHECK: arm_sme.tile_store {{.*}} : memref<?x?xi128>, vector<[1]x[1]xi128>
-// CHECK: arm_sme.tile_load {{.*}}, <vertical> : memref<?x?xi128>, vector<[1]x[1]xi128>
+// CHECK: arm_sme.tile_load {{.*}} layout<vertical> : memref<?x?xi128>, vector<[1]x[1]xi128>
 func.func @transpose_i128(%arg0: vector<[1]x[1]xi128>) {
   %0 = vector.transpose %arg0, [1, 0] : vector<[1]x[1]xi128> to vector<[1]x[1]xi128>
   "prevent.dce"(%0) : (vector<[1]x[1]xi128>) -> ()
@@ -536,7 +536,7 @@ func.func @transpose_i128(%arg0: vector<[1]x[1]xi128>) {
 // CHECK-LABEL: @transpose_f16
 // CHECK: arith.constant 8
 // CHECK: arm_sme.tile_store {{.*}} : memref<?x?xf16>, vector<[8]x[8]xf16>
-// CHECK: arm_sme.tile_load {{.*}}, <vertical> : memref<?x?xf16>, vector<[8]x[8]xf16>
+// CHECK: arm_sme.tile_load {{.*}} layout<vertical> : memref<?x?xf16>, vector<[8]x[8]xf16>
 func.func @transpose_f16(%arg0: vector<[8]x[8]xf16>) {
   %0 = vector.transpose %arg0, [1, 0] : vector<[8]x[8]xf16> to vector<[8]x[8]xf16>
   "prevent.dce"(%0) : (vector<[8]x[8]xf16>) -> ()
@@ -548,7 +548,7 @@ func.func @transpose_f16(%arg0: vector<[8]x[8]xf16>) {
 // CHECK-LABEL: @transpose_bf16
 // CHECK: arith.constant 8
 // CHECK: arm_sme.tile_store {{.*}} : memref<?x?xbf16>, vector<[8]x[8]xbf16>
-// CHECK: arm_sme.tile_load {{.*}}, <vertical> : memref<?x?xbf16>, vector<[8]x[8]xbf16>
+// CHECK: arm_sme.tile_load {{.*}} layout<vertical> : memref<?x?xbf16>, vector<[8]x[8]xbf16>
 func.func @transpose_bf16(%arg0: vector<[8]x[8]xbf16>) {
   %0 = vector.transpose %arg0, [1, 0] : vector<[8]x[8]xbf16> to vector<[8]x[8]xbf16>
   "prevent.dce"(%0) : (vector<[8]x[8]xbf16>) -> ()
@@ -560,7 +560,7 @@ func.func @transpose_bf16(%arg0: vector<[8]x[8]xbf16>) {
 // CHECK-LABEL: @transpose_f32
 // CHECK: arith.constant 4
 // CHECK: arm_sme.tile_store {{.*}} : memref<?x?xf32>, vector<[4]x[4]xf32>
-// CHECK: arm_sme.tile_load {{.*}}, <vertical> : memref<?x?xf32>, vector<[4]x[4]xf32>
+// CHECK: arm_sme.tile_load {{.*}} layout<vertical> : memref<?x?xf32>, vector<[4]x[4]xf32>
 func.func @transpose_f32(%arg0: vector<[4]x[4]xf32>) {
   %0 = vector.transpose %arg0, [1, 0] : vector<[4]x[4]xf32> to vector<[4]x[4]xf32>
   "prevent.dce"(%0) : (vector<[4]x[4]xf32>) -> ()
@@ -572,7 +572,7 @@ func.func @transpose_f32(%arg0: vector<[4]x[4]xf32>) {
 // CHECK-LABEL: @transpose_f64
 // CHECK: arith.constant 2
 // CHECK: arm_sme.tile_store {{.*}} : memref<?x?xf64>, vector<[2]x[2]xf64>
-// CHECK: arm_sme.tile_load {{.*}}, <vertical> : memref<?x?xf64>, vector<[2]x[2]xf64>
+// CHECK: arm_sme.tile_load {{.*}} layout<vertical> : memref<?x?xf64>, vector<[2]x[2]xf64>
 func.func @transpose_f64(%arg0: vector<[2]x[2]xf64>) {
   %0 = vector.transpose %arg0, [1, 0] : vector<[2]x[2]xf64> to vector<[2]x[2]xf64>
   "prevent.dce"(%0) : (vector<[2]x[2]xf64>) -> ()
diff --git a/mlir/test/Dialect/ArmSVE/invalid.mlir b/mlir/test/Dialect/ArmSVE/invalid.mlir
new file mode 100644
index 0000000000..a1fa0d0292
--- /dev/null
+++ b/mlir/test/Dialect/ArmSVE/invalid.mlir
@@ -0,0 +1,51 @@
+// RUN: mlir-opt %s -split-input-file -verify-diagnostics
+
+// -----
+
+func.func @arm_sve_convert_from_svbool__bad_mask_type(%bool: vector<2x[16]xi1>) -> vector<2x[8]xi2> {
+  // expected-error@+1 {{'result' must be trailing scalable vector of 1-bit signless integer values with dim -1 having a size of {16, 8, 4, 2, 1}, but got 'vector<2x[8]xi2>'}}
+  %mask = arm_sve.convert_from_svbool %bool : vector<2x[8]xi2>
+  return %mask : vector<2x[8]xi2>
+}
+
+// -----
+
+func.func @arm_sve_convert_from_svbool__bad_mask_shape(%bool : vector<[16]xi1>) -> vector<[7]xi1> {
+  // expected-error@+1 {{'result' must be trailing scalable vector of 1-bit signless integer values with dim -1 having a size of {16, 8, 4, 2, 1}, but got 'vector<[7]xi1>'}}
+  %mask = arm_sve.convert_from_svbool %bool : vector<[7]xi1>
+  return %mask : vector<[7]xi1>
+}
+
+// -----
+
+func.func @arm_sve_convert_from_svbool__bad_mask_scalability(%bool : vector<[4]x[16]xi1>) -> vector<[4]x[8]xi1> {
+  // expected-error@+1 {{'result' must be trailing scalable vector of 1-bit signless integer values with dim -1 having a size of {16, 8, 4, 2, 1}, but got 'vector<[4]x[8]xi1>'}}
+  %mask = arm_sve.convert_from_svbool %bool : vector<[4]x[8]xi1>
+  return %mask : vector<[4]x[8]xi1>
+}
+
+// -----
+
+func.func @arm_sve_convert_to_svbool__bad_mask_type(%mask: vector<2x[8]xi2>) -> vector<2x[16]xi1> {
+  // expected-error@+1 {{'source' must be trailing scalable vector of 1-bit signless integer values with dim -1 having a size of {16, 8, 4, 2, 1}, but got 'vector<2x[8]xi2>'}}
+  %bool = arm_sve.convert_to_svbool %mask : vector<2x[8]xi2>
+  return %bool : vector<2x[16]xi1>
+}
+
+// -----
+
+func.func @arm_sve_convert_to_svbool__bad_mask_shape(%mask : vector<[7]xi1>) -> vector<[16]xi1> {
+  // expected-error@+1 {{'source' must be trailing scalable vector of 1-bit signless integer values with dim -1 having a size of {16, 8, 4, 2, 1}, but got 'vector<[7]xi1>'}}
+  %bool = arm_sve.convert_to_svbool %mask : vector<[7]xi1>
+  return
+}
+
+// -----
+
+func.func @arm_sve_convert_to_svbool__bad_mask_scalability(%mask : vector<[4]x[8]xi1>) -> vector<[4]x[16]xi1> {
+  // expected-error@+1 {{'source' must be trailing scalable vector of 1-bit signless integer values with dim -1 having a size of {16, 8, 4, 2, 1}, but got 'vector<[4]x[8]xi1>'}}
+  %bool = arm_sve.convert_to_svbool %mask : vector<[4]x[8]xi1>
+  return
+}
+
+
diff --git a/mlir/test/Dialect/ArmSVE/legalize-for-llvm.mlir b/mlir/test/Dialect/ArmSVE/legalize-for-llvm.mlir
index 2d980db981..8e76fb7119 100644
--- a/mlir/test/Dialect/ArmSVE/legalize-for-llvm.mlir
+++ b/mlir/test/Dialect/ArmSVE/legalize-for-llvm.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -convert-vector-to-llvm="enable-arm-sve" -convert-func-to-llvm -reconcile-unrealized-casts | mlir-opt | FileCheck %s
+// RUN: mlir-opt -convert-vector-to-llvm="enable-arm-sve" -convert-func-to-llvm -reconcile-unrealized-casts -split-input-file %s | FileCheck %s
 
 func.func @arm_sve_sdot(%a: vector<[16]xi8>,
                    %b: vector<[16]xi8>,
@@ -10,6 +10,8 @@ func.func @arm_sve_sdot(%a: vector<[16]xi8>,
   return %0 : vector<[4]xi32>
 }
 
+// -----
+
 func.func @arm_sve_smmla(%a: vector<[16]xi8>,
                     %b: vector<[16]xi8>,
                     %c: vector<[4]xi32>)
@@ -20,6 +22,8 @@ func.func @arm_sve_smmla(%a: vector<[16]xi8>,
   return %0 : vector<[4]xi32>
 }
 
+// -----
+
 func.func @arm_sve_udot(%a: vector<[16]xi8>,
                    %b: vector<[16]xi8>,
                    %c: vector<[4]xi32>)
@@ -30,6 +34,8 @@ func.func @arm_sve_udot(%a: vector<[16]xi8>,
   return %0 : vector<[4]xi32>
 }
 
+// -----
+
 func.func @arm_sve_ummla(%a: vector<[16]xi8>,
                     %b: vector<[16]xi8>,
                     %c: vector<[4]xi32>)
@@ -40,6 +46,8 @@ func.func @arm_sve_ummla(%a: vector<[16]xi8>,
   return %0 : vector<[4]xi32>
 }
 
+// -----
+
 func.func @arm_sve_arithi_masked(%a: vector<[4]xi32>,
                             %b: vector<[4]xi32>,
                             %c: vector<[4]xi32>,
@@ -65,6 +73,8 @@ func.func @arm_sve_arithi_masked(%a: vector<[4]xi32>,
   return %4 : vector<[4]xi32>
 }
 
+// -----
+
 func.func @arm_sve_arithf_masked(%a: vector<[4]xf32>,
                             %b: vector<[4]xf32>,
                             %c: vector<[4]xf32>,
@@ -87,6 +97,8 @@ func.func @arm_sve_arithf_masked(%a: vector<[4]xf32>,
   return %3 : vector<[4]xf32>
 }
 
+// -----
+
 func.func @arm_sve_abs_diff(%a: vector<[4]xi32>,
                        %b: vector<[4]xi32>)
                        -> vector<[4]xi32> {
@@ -111,8 +123,67 @@ func.func @arm_sve_abs_diff(%a: vector<[4]xi32>,
   return %3 : vector<[4]xi32>
 }
 
+// -----
+
 func.func @get_vector_scale() -> index {
   // CHECK: llvm.intr.vscale
   %0 = vector.vscale
   return %0 : index
 }
+
+// -----
+
+func.func @convert_1d_mask_to_svbool(%mask: vector<[4]xi1>) -> vector<[16]xi1>
+{
+  // CHECK: "arm_sve.intr.convert.to.svbool"(%{{.*}}) : (vector<[4]xi1>) -> vector<[16]xi1>
+  %svbool = arm_sve.convert_to_svbool %mask : vector<[4]xi1>
+  return %svbool : vector<[16]xi1>
+}
+
+// -----
+
+func.func @convert_1d_mask_from_svbool(%svbool: vector<[16]xi1>) -> vector<[2]xi1>
+{
+  // CHECK: "arm_sve.intr.convert.from.svbool"(%{{.*}}) : (vector<[16]xi1>) -> vector<[2]xi1>
+  %mask = arm_sve.convert_from_svbool %svbool : vector<[2]xi1>
+  return %mask : vector<[2]xi1>
+}
+
+// -----
+
+// CHECK-LABEL: @convert_2d_mask_to_svbool(
+// CHECK-SAME:                             %[[MASK:.*]]: !llvm.array<2 x vector<[8]xi1>>)
+func.func @convert_2d_mask_to_svbool(%mask: vector<2x[8]xi1>) -> vector<2x[16]xi1>
+{
+  // CHECK-NEXT:    %[[RES0:.*]] = llvm.mlir.constant(dense<false> : vector<2x[16]xi1>) : !llvm.array<2 x vector<[16]xi1>>
+  // CHECK-NEXT:   %[[MASK0:.*]] = llvm.extractvalue %[[MASK]][0] : !llvm.array<2 x vector<[8]xi1>>
+  // CHECK-NEXT: %[[SVBOOL0:.*]] = "arm_sve.intr.convert.to.svbool"(%[[MASK0]]) : (vector<[8]xi1>) -> vector<[16]xi1>
+  // CHECK-NEXT:    %[[RES1:.*]] = llvm.insertvalue %[[SVBOOL0]], %[[RES0]][0] : !llvm.array<2 x vector<[16]xi1>>
+  // CHECK-NEXT:   %[[MASK1:.*]] = llvm.extractvalue %[[MASK]][1] : !llvm.array<2 x vector<[8]xi1>>
+  // CHECK-NEXT: %[[SVBOOL1:.*]] = "arm_sve.intr.convert.to.svbool"(%[[MASK1]]) : (vector<[8]xi1>) -> vector<[16]xi1>
+  // CHECK-NEXT:  %[[SVBOOL:.*]] = llvm.insertvalue %[[SVBOOL1]], %[[RES1]][1] : !llvm.array<2 x vector<[16]xi1>>
+  %svbool = arm_sve.convert_to_svbool %mask : vector<2x[8]xi1>
+  // CHECK-NEXT: llvm.return %[[SVBOOL]] : !llvm.array<2 x vector<[16]xi1>>
+  return %svbool : vector<2x[16]xi1>
+}
+
+// -----
+
+// CHECK-LABEL: @convert_2d_mask_from_svbool(
+// CHECK-SAME:                               %[[SVBOOL:.*]]: !llvm.array<3 x vector<[16]xi1>>)
+func.func @convert_2d_mask_from_svbool(%svbool: vector<3x[16]xi1>) -> vector<3x[1]xi1>
+{
+  // CHECK-NEXT:    %[[RES0:.*]] = llvm.mlir.constant(dense<false> : vector<3x[1]xi1>) : !llvm.array<3 x vector<[1]xi1>>
+  // CHECK-NEXT: %[[SVBOOL0:.*]] = llvm.extractvalue %[[SVBOOL]][0] : !llvm.array<3 x vector<[16]xi1>>
+  // CHECK-NEXT:   %[[MASK0:.*]] = "arm_sve.intr.convert.from.svbool"(%[[SVBOOL0]]) : (vector<[16]xi1>) -> vector<[1]xi1>
+  // CHECK-NEXT:    %[[RES1:.*]] = llvm.insertvalue %[[MASK0]], %[[RES0]][0] : !llvm.array<3 x vector<[1]xi1>>
+  // CHECK-NEXT: %[[SVBOOL1:.*]] = llvm.extractvalue %[[SVBOOL]][1] : !llvm.array<3 x vector<[16]xi1>>
+  // CHECK-NEXT:   %[[MASK1:.*]] = "arm_sve.intr.convert.from.svbool"(%[[SVBOOL1]]) : (vector<[16]xi1>) -> vector<[1]xi1>
+  // CHECK-NEXT:    %[[RES2:.*]] = llvm.insertvalue %[[MASK1]], %[[RES1]][1] : !llvm.array<3 x vector<[1]xi1>>
+  // CHECK-NEXT: %[[SVBOOL2:.*]] = llvm.extractvalue %[[SVBOOL]][2] : !llvm.array<3 x vector<[16]xi1>>
+  // CHECK-NEXT:   %[[MASK2:.*]] = "arm_sve.intr.convert.from.svbool"(%[[SVBOOL2]]) : (vector<[16]xi1>) -> vector<[1]xi1>
+  // CHECK-NEXT:    %[[MASK:.*]] = llvm.insertvalue %[[MASK2]], %[[RES2]][2] : !llvm.array<3 x vector<[1]xi1>>
+  %mask = arm_sve.convert_from_svbool %svbool : vector<3x[1]xi1>
+  // CHECK-NEXT: llvm.return %[[MASK]] : !llvm.array<3 x vector<[1]xi1>>
+  return %mask : vector<3x[1]xi1>
+}
diff --git a/mlir/test/Dialect/ArmSVE/roundtrip.mlir b/mlir/test/Dialect/ArmSVE/roundtrip.mlir
index d2ca035c17..c9a0b6db8f 100644
--- a/mlir/test/Dialect/ArmSVE/roundtrip.mlir
+++ b/mlir/test/Dialect/ArmSVE/roundtrip.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -verify-diagnostics %s | mlir-opt | FileCheck %s
+// RUN: mlir-opt -verify-diagnostics -split-input-file %s | mlir-opt | FileCheck %s
 
 func.func @arm_sve_sdot(%a: vector<[16]xi8>,
                    %b: vector<[16]xi8>,
@@ -9,6 +9,8 @@ func.func @arm_sve_sdot(%a: vector<[16]xi8>,
   return %0 : vector<[4]xi32>
 }
 
+// -----
+
 func.func @arm_sve_smmla(%a: vector<[16]xi8>,
                     %b: vector<[16]xi8>,
                     %c: vector<[4]xi32>) -> vector<[4]xi32> {
@@ -18,6 +20,8 @@ func.func @arm_sve_smmla(%a: vector<[16]xi8>,
   return %0 : vector<[4]xi32>
 }
 
+// -----
+
 func.func @arm_sve_udot(%a: vector<[16]xi8>,
                    %b: vector<[16]xi8>,
                    %c: vector<[4]xi32>) -> vector<[4]xi32> {
@@ -27,6 +31,8 @@ func.func @arm_sve_udot(%a: vector<[16]xi8>,
   return %0 : vector<[4]xi32>
 }
 
+// -----
+
 func.func @arm_sve_ummla(%a: vector<[16]xi8>,
                     %b: vector<[16]xi8>,
                     %c: vector<[4]xi32>) -> vector<[4]xi32> {
@@ -36,6 +42,8 @@ func.func @arm_sve_ummla(%a: vector<[16]xi8>,
   return %0 : vector<[4]xi32>
 }
 
+// -----
+
 func.func @arm_sve_masked_arithi(%a: vector<[4]xi32>,
                             %b: vector<[4]xi32>,
                             %c: vector<[4]xi32>,
@@ -61,6 +69,8 @@ func.func @arm_sve_masked_arithi(%a: vector<[4]xi32>,
   return %2 : vector<[4]xi32>
 }
 
+// -----
+
 func.func @arm_sve_masked_arithf(%a: vector<[4]xf32>,
                             %b: vector<[4]xf32>,
                             %c: vector<[4]xf32>,
@@ -82,3 +92,74 @@ func.func @arm_sve_masked_arithf(%a: vector<[4]xf32>,
                                            vector<[4]xf32>
   return %3 : vector<[4]xf32>
 }
+
+// -----
+
+func.func @arm_sve_convert_to_svbool(%a: vector<[1]xi1>,
+                                     %b: vector<[2]xi1>,
+                                     %c: vector<[4]xi1>,
+                                     %d: vector<[8]xi1>,
+                                     %e: vector<2x3x[1]xi1>,
+                                     %f: vector<4x[2]xi1>,
+                                     %g: vector<1x1x1x2x[4]xi1>,
+                                     %h: vector<100x[8]xi1>) {
+  // CHECK: arm_sve.convert_to_svbool %{{.*}} : vector<[1]xi1>
+  %1 = arm_sve.convert_to_svbool %a : vector<[1]xi1>
+
+  // CHECK: arm_sve.convert_to_svbool %{{.*}} : vector<[2]xi1>
+  %2 = arm_sve.convert_to_svbool %b : vector<[2]xi1>
+
+  // CHECK: arm_sve.convert_to_svbool %{{.*}} : vector<[4]xi1>
+  %3 = arm_sve.convert_to_svbool %c : vector<[4]xi1>
+
+  // CHECK: arm_sve.convert_to_svbool %{{.*}} : vector<[8]xi1>
+  %4 = arm_sve.convert_to_svbool %d : vector<[8]xi1>
+
+  // CHECK: arm_sve.convert_to_svbool %{{.*}} : vector<2x3x[1]xi1>
+  %5 = arm_sve.convert_to_svbool %e : vector<2x3x[1]xi1>
+
+  // CHECK: arm_sve.convert_to_svbool %{{.*}} : vector<4x[2]xi1>
+  %6 = arm_sve.convert_to_svbool %f : vector<4x[2]xi1>
+
+  // CHECK: arm_sve.convert_to_svbool %{{.*}} : vector<1x1x1x2x[4]xi1>
+  %7 = arm_sve.convert_to_svbool %g : vector<1x1x1x2x[4]xi1>
+
+  // CHECK: arm_sve.convert_to_svbool %{{.*}} : vector<100x[8]xi1>
+  %8 = arm_sve.convert_to_svbool %h : vector<100x[8]xi1>
+
+  return
+}
+
+// -----
+
+func.func @arm_sve_convert_from_svbool(%a: vector<[16]xi1>,
+                                       %b: vector<2x3x[16]xi1>,
+                                       %c: vector<4x[16]xi1>,
+                                       %d: vector<1x1x1x1x[16]xi1>,
+                                       %e: vector<32x[16]xi1>) {
+  // CHECK: arm_sve.convert_from_svbool %{{.*}} : vector<[1]xi1>
+  %1 = arm_sve.convert_from_svbool %a : vector<[1]xi1>
+
+  // CHECK: arm_sve.convert_from_svbool %{{.*}} : vector<[2]xi1>
+  %2 = arm_sve.convert_from_svbool %a : vector<[2]xi1>
+
+  // CHECK: arm_sve.convert_from_svbool %{{.*}} : vector<[4]xi1>
+  %3 = arm_sve.convert_from_svbool %a : vector<[4]xi1>
+
+  // CHECK: arm_sve.convert_from_svbool %{{.*}} : vector<[8]xi1>
+  %4 = arm_sve.convert_from_svbool %a : vector<[8]xi1>
+
+  // CHECK: arm_sve.convert_from_svbool %{{.*}} : vector<2x3x[1]xi1>
+  %5 = arm_sve.convert_from_svbool %b : vector<2x3x[1]xi1>
+
+  // CHECK: arm_sve.convert_from_svbool %{{.*}} : vector<4x[2]xi1>
+  %6 = arm_sve.convert_from_svbool %c : vector<4x[2]xi1>
+
+  // CHECK: arm_sve.convert_from_svbool %{{.*}} : vector<1x1x1x1x[4]xi1>
+  %7 = arm_sve.convert_from_svbool %d : vector<1x1x1x1x[4]xi1>
+
+  // CHECK: arm_sve.convert_from_svbool %{{.*}} : vector<32x[8]xi1>
+  %8 = arm_sve.convert_from_svbool %e : vector<32x[8]xi1>
+
+  return
+}
diff --git a/mlir/test/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation/dealloc-memoryeffect-interface.mlir b/mlir/test/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation/dealloc-memoryeffect-interface.mlir
index 44cf163856..40a57b90c6 100644
--- a/mlir/test/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation/dealloc-memoryeffect-interface.mlir
+++ b/mlir/test/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation/dealloc-memoryeffect-interface.mlir
@@ -124,3 +124,35 @@ func.func @op_without_aliasing_and_allocation() -> memref<4xf32> {
 //       CHECK:     [[CLONE:%.+]] = bufferization.clone [[GLOBAL]]
 //       CHECK:     scf.yield [[CLONE]] :
 //       CHECK:   return [[RES]] :
+
+// -----
+
+// Allocations with "bufferization.manual_deallocation" are assigned an
+// ownership of "false".
+
+func.func @manual_deallocation(%c: i1, %f: f32, %idx: index) -> f32 {
+  %0 = memref.alloc() {bufferization.manual_deallocation} : memref<5xf32>
+  linalg.fill ins(%f : f32) outs(%0 : memref<5xf32>)
+  %1 = memref.alloc() : memref<5xf32>
+  linalg.fill ins(%f : f32) outs(%1 : memref<5xf32>)
+  %2 = arith.select %c, %0, %1 : memref<5xf32>
+  %3 = memref.load %2[%idx] : memref<5xf32>
+
+  // Only buffers that are under "manual deallocation" are allowed to be
+  // deallocated with memref.dealloc. For consistency reasons, the
+  // manual_deallocation attribute must also be specified. A runtime insertion
+  // is inserted to ensure that we do not have ownership. (This is not a
+  // bulletproof check, but covers some cases of invalid IR.)
+  memref.dealloc %0 {bufferization.manual_deallocation} : memref<5xf32>
+
+  return %3 : f32
+}
+
+// CHECK-LABEL: func @manual_deallocation(
+//       CHECK:   %[[true:.*]] = arith.constant true
+//       CHECK:   %[[manual_alloc:.*]] = memref.alloc() {bufferization.manual_deallocation} : memref<5xf32>
+//       CHECK:   %[[managed_alloc:.*]] = memref.alloc() : memref<5xf32>
+//       CHECK:   %[[selected:.*]] = arith.select
+//       CHECK:   cf.assert %[[true]], "expected that the block does not have ownership"
+//       CHECK:   memref.dealloc %[[manual_alloc]]
+//       CHECK:   bufferization.dealloc (%[[managed_alloc]] : memref<5xf32>) if (%[[true]])
diff --git a/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize-empty-tensor-elimination.mlir b/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize-empty-tensor-elimination.mlir
index 99b974b9ef..9a3e14b6d3 100644
--- a/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize-empty-tensor-elimination.mlir
+++ b/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize-empty-tensor-elimination.mlir
@@ -1,4 +1,5 @@
 // RUN: mlir-opt %s -eliminate-empty-tensors -empty-tensor-to-alloc-tensor -one-shot-bufferize="bufferize-function-boundaries" -cse -canonicalize -split-input-file | FileCheck %s
+// RUN: mlir-opt %s -eliminate-empty-tensors | FileCheck %s --check-prefix=CHECK-ELIM
 
 //      CHECK: func @buffer_forwarding_conflict(
 // CHECK-SAME:   %[[FUNC_ARG:[0-9a-zA-Z]*]]: memref<?xf32>
@@ -341,3 +342,26 @@ func.func @linalg_copy_empty() -> tensor<26xi32> {
   %1 = linalg.copy ins(%0 : tensor<26xi32>) outs(%0 : tensor<26xi32>) -> tensor<26xi32>
   return %1 : tensor<26xi32>
 }
+
+// -----
+
+// CHECK-ELIM-LABEL: func @multiple_materialize_in_destination_buffer(
+//  CHECK-ELIM-SAME:     %[[m:.*]]: memref<5xf32>
+//       CHECK-ELIM:   tensor.empty
+//       CHECK-ELIM:   bufferization.to_tensor %[[m]] restrict writable
+//       CHECK-ELIM:   bufferization.materialize_in_destination {{.*}} in writable %[[m]]
+func.func @multiple_materialize_in_destination_buffer(%m: memref<5xf32>, %f: f32, %f2: f32, %c: i1) {
+  %0 = tensor.empty() : tensor<5xf32>
+  %filled = linalg.fill ins(%f : f32) outs(%0 : tensor<5xf32>) -> tensor<5xf32>
+
+  %1 = tensor.empty() : tensor<5xf32>
+  %filled2 = linalg.fill ins(%f2 : f32) outs(%1 : tensor<5xf32>) -> tensor<5xf32>
+
+  %selected = scf.if %c -> tensor<5xf32> {
+    scf.yield %filled : tensor<5xf32>
+  } else {
+    scf.yield %filled2 : tensor<5xf32>
+  }
+  bufferization.materialize_in_destination %selected in restrict writable %m : (tensor<5xf32>, memref<5xf32>) -> ()
+  return
+}
diff --git a/mlir/test/Dialect/Bufferization/invalid.mlir b/mlir/test/Dialect/Bufferization/invalid.mlir
index ce56f89c1f..83f8ef7861 100644
--- a/mlir/test/Dialect/Bufferization/invalid.mlir
+++ b/mlir/test/Dialect/Bufferization/invalid.mlir
@@ -80,13 +80,6 @@ func.func @invalid_materialize_in_destination_dest_type(%arg0: tensor<5xf32>, %a
 
 // -----
 
-func.func @invalid_materialize_in_destination_restrict_missing(%arg0: tensor<?xf32>, %arg1: memref<?xf32>) {
-  // expected-error @below{{'restrict' must be specified if and only if the destination is of memref type}}
-  bufferization.materialize_in_destination %arg0 in %arg1 : (tensor<?xf32>, memref<?xf32>) -> ()
-}
-
-// -----
-
 func.func @invalid_materialize_in_destination_result(%arg0: tensor<?xf32>, %arg1: memref<?xf32>) {
   // expected-error @below{{memref 'dest' implies zero results}}
   bufferization.materialize_in_destination %arg0 in restrict %arg1 : (tensor<?xf32>, memref<?xf32>) -> (tensor<?xf32>)
@@ -102,7 +95,7 @@ func.func @invalid_materialize_in_destination_result_missing(%arg0: tensor<?xf32
 // -----
 
 func.func @invalid_materialize_in_destination_restrict(%arg0: tensor<?xf32>, %arg1: tensor<?xf32>) {
-  // expected-error @below{{'restrict' must be specified if and only if the destination is of memref type}}
+  // expected-error @below{{'restrict' is valid only for memref destinations}}
   bufferization.materialize_in_destination %arg0 in restrict %arg1 : (tensor<?xf32>, tensor<?xf32>) -> (tensor<?xf32>)
 }
 
@@ -143,3 +136,10 @@ func.func @invalid_dealloc_wrong_number_of_results(%arg0: memref<2xf32>, %arg1:
   %0:3 = "bufferization.dealloc"(%arg0, %arg1, %arg2, %arg2, %arg1) <{operandSegmentSizes = array<i32: 2, 2, 1>}> : (memref<2xf32>, memref<4xi32>, i1, i1, memref<4xi32>) -> (i1, i1, i1)
   return %0#0 : i1
 }
+
+// -----
+
+func.func @invalid_manual_deallocation() {
+  // expected-error @below{{op attribute 'bufferization.manual_deallocation' can be used only on ops that have an allocation and/or free side effect}}
+  arith.constant {bufferization.manual_deallocation} 0  : index
+}
diff --git a/mlir/test/Dialect/GPU/promotion.mlir b/mlir/test/Dialect/GPU/promotion.mlir
index b4668b5678..2da1be5977 100644
--- a/mlir/test/Dialect/GPU/promotion.mlir
+++ b/mlir/test/Dialect/GPU/promotion.mlir
@@ -1,3 +1,4 @@
+
 // RUN: mlir-opt -allow-unregistered-dialect -pass-pipeline='builtin.module(gpu.module(gpu.func(test-gpu-memory-promotion)))' -split-input-file %s | FileCheck %s
 
 gpu.module @foo {
diff --git a/mlir/test/Dialect/LLVM/lower-to-llvm-e2e.mlir b/mlir/test/Dialect/LLVM/lower-to-llvm-e2e-with-target-tag.mlir
similarity index 100%
rename from mlir/test/Dialect/LLVM/lower-to-llvm-e2e.mlir
rename to mlir/test/Dialect/LLVM/lower-to-llvm-e2e-with-target-tag.mlir
diff --git a/mlir/test/Dialect/LLVM/lower-to-llvm-e2e-with-top-level-named-sequence.mlir b/mlir/test/Dialect/LLVM/lower-to-llvm-e2e-with-top-level-named-sequence.mlir
new file mode 100644
index 0000000000..ac4608e2f8
--- /dev/null
+++ b/mlir/test/Dialect/LLVM/lower-to-llvm-e2e-with-top-level-named-sequence.mlir
@@ -0,0 +1,64 @@
+// Note: We run CSE here to make the pattern matching more direct.
+
+// RUN: mlir-opt %s -test-lower-to-llvm -cse | FileCheck %s
+
+// RUN: mlir-opt %s \
+// RUN:   -transform-preload-library=transform-library-paths=%p/../Transform/include/Library/lower-to-llvm.mlir \
+// RUN:   -transform-interpreter="entry-point=entry_point" \
+// RUN:   -test-transform-dialect-erase-schedule \
+// RUN:   -cse \
+// RUN: | FileCheck %s
+
+// Check that we properly lower to llvm memref operations that require to be
+// expanded first, like `memref.subview`.
+func.func @subview(%0 : memref<64x4xf32, strided<[4, 1], offset: 0>>, %arg0 : index, %arg1 : index, %arg2 : index)
+-> memref<?x?xf32, strided<[?, ?], offset: ?>> {
+  // CHECK-LABEL: @subview
+  // CHECK-SAME: %[[BASE:[^:]*]]: !llvm.ptr
+  // CHECK-SAME: %[[BASE_ALIGNED:[^:]*]]: !llvm.ptr,
+  // CHECK-SAME: %[[BASE_OFFSET:[^:]*]]: i64,
+  // CHECK-SAME: %[[BASE_STRIDE0:[^:]*]]: i64,
+  // CHECK-SAME: %[[BASE_STRIDE1:[^:]*]]: i64,
+  // CHECK-SAME: %[[BASE_SIZE0:[^:]*]]: i64,
+  // CHECK-SAME: %[[BASE_SIZE1:[^:]*]]: i64,
+  // CHECK-SAME: %[[ARG0:[^:]*]]: i64,
+  // CHECK-SAME: %[[ARG1:[^:]*]]: i64,
+  // CHECK-SAME: %[[ARG2:[^:]*]]: i64)
+  // CHECK-SAME: -> !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>
+
+  // CHECK-DAG: %[[STRIDE0:.*]] = llvm.mlir.constant(4 : index) : i64
+  // CHECK-DAG: %[[DESCSTRIDE0:.*]] = llvm.mul %[[ARG0]], %[[STRIDE0]] : i64
+  // CHECK-DAG: %[[OFF2:.*]] = llvm.add %[[DESCSTRIDE0]], %[[ARG1]] : i64
+  // CHECK-DAG: %[[DESC:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+
+  // Base address and algined address.
+  // CHECK-DAG: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+  // CHECK-DAG: %[[DESC1:.*]] = llvm.insertvalue %[[BASE_ALIGNED]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+
+  // Offset.
+  // CHECK: %[[DESC2:.*]] = llvm.insertvalue %[[OFF2]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+  // Size 0.
+  // CHECK: %[[DESC3:.*]] = llvm.insertvalue %[[ARG0]], %[[DESC2]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+  // Stride 0 == 4 * %arg0.
+  // CHECK: %[[DESC4:.*]] = llvm.insertvalue %[[DESCSTRIDE0]], %[[DESC3]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+  // Size 1.
+  // CHECK: %[[DESC5:.*]] = llvm.insertvalue %[[ARG1]], %[[DESC4]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+  // Stride 1 == 1 * %arg1.
+  // CHECK: %[[DESC6:.*]] = llvm.insertvalue %[[ARG1]], %[[DESC5]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+
+  %1 = memref.subview %0[%arg0, %arg1][%arg0, %arg1][%arg0, %arg1] :
+    memref<64x4xf32, strided<[4, 1], offset: 0>>
+  to memref<?x?xf32, strided<[?, ?], offset: ?>>
+  return %1 : memref<?x?xf32, strided<[?, ?], offset: ?>>
+}
+
+module @named_inclusion_in_named attributes { transform.with_named_sequence } {
+  transform.named_sequence private @lower_to_llvm(!transform.any_op {transform.readonly}) -> !transform.any_op
+
+  transform.named_sequence @entry_point(
+    %toplevel_module : !transform.any_op {transform.readonly}) {
+    transform.include @lower_to_llvm failures(suppress) (%toplevel_module) 
+      : (!transform.any_op) -> (!transform.any_op)
+    transform.yield
+  }
+}
diff --git a/mlir/test/Dialect/LLVMIR/invalid.mlir b/mlir/test/Dialect/LLVMIR/invalid.mlir
index 6f119a140b..2d0a68b8b6 100644
--- a/mlir/test/Dialect/LLVMIR/invalid.mlir
+++ b/mlir/test/Dialect/LLVMIR/invalid.mlir
@@ -306,7 +306,7 @@ func.func @call_non_llvm() {
 // -----
 
 func.func @call_non_llvm_arg(%arg0 : tensor<*xi32>) {
-  // expected-error@+1 {{'llvm.call' op operand #0 must be LLVM dialect-compatible type}}
+  // expected-error@+1 {{'llvm.call' op operand #0 must be variadic of LLVM dialect-compatible type}}
   "llvm.call"(%arg0) : (tensor<*xi32>) -> ()
   llvm.return
 }
diff --git a/mlir/test/Dialect/LLVMIR/mem2reg.mlir b/mlir/test/Dialect/LLVMIR/mem2reg.mlir
index 30ba459d07..32e3fed7e5 100644
--- a/mlir/test/Dialect/LLVMIR/mem2reg.mlir
+++ b/mlir/test/Dialect/LLVMIR/mem2reg.mlir
@@ -683,3 +683,16 @@ llvm.func @no_inner_alloca_promotion(%arg: i64) -> i64 {
   // CHECK: llvm.return %[[RES]] : i64
   llvm.return %2 : i64
 }
+
+// -----
+
+// CHECK-LABEL: @transitive_reaching_def
+llvm.func @transitive_reaching_def() -> !llvm.ptr {
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  // CHECK-NOT: alloca
+  %1 = llvm.alloca %0 x !llvm.ptr {alignment = 8 : i64} : (i32) -> !llvm.ptr
+  %2 = llvm.load %1 {alignment = 8 : i64} : !llvm.ptr -> !llvm.ptr
+  llvm.store %2, %1 {alignment = 8 : i64} : !llvm.ptr, !llvm.ptr
+  %3 = llvm.load %1 {alignment = 8 : i64} : !llvm.ptr -> !llvm.ptr
+  llvm.return %3 : !llvm.ptr
+}
diff --git a/mlir/test/Dialect/Linalg/collapse-dim.mlir b/mlir/test/Dialect/Linalg/collapse-dim.mlir
index 106154ba3a..547320f533 100644
--- a/mlir/test/Dialect/Linalg/collapse-dim.mlir
+++ b/mlir/test/Dialect/Linalg/collapse-dim.mlir
@@ -116,3 +116,40 @@ func.func @uncollapsable_strided_memref(%arg0: memref<2x6x24x48xi32>, %arg1: mem
   }
   return %alloc : memref<2x6x24x48xi32>
 }
+
+// -----
+
+// CHECK-LABEL:   func.func @linalg_copy(
+// CHECK-SAME:                           %[[VAL_0:.*]]: tensor<1x2x3x4x5xf32, 1 : i64>,
+// CHECK-SAME:                           %[[VAL_1:.*]]: tensor<1x2x3x4x5xf32, 3 : i64>) -> tensor<1x2x3x4x5xf32, 3 : i64> {
+// CHECK:           %[[VAL_2:.*]] = tensor.collapse_shape %[[VAL_0]] {{\[\[}}0], [1], [2, 3], [4]] : tensor<1x2x3x4x5xf32, 1 : i64> into tensor<1x2x12x5xf32>
+// CHECK:           %[[VAL_3:.*]] = tensor.collapse_shape %[[VAL_1]] {{\[\[}}0], [1], [2, 3], [4]] : tensor<1x2x3x4x5xf32, 3 : i64> into tensor<1x2x12x5xf32>
+// CHECK:           %[[VAL_4:.*]] = tensor.collapse_shape %[[VAL_2]] {{\[\[}}0], [1], [2, 3]] : tensor<1x2x12x5xf32> into tensor<1x2x60xf32>
+// CHECK:           %[[VAL_5:.*]] = tensor.collapse_shape %[[VAL_3]] {{\[\[}}0], [1], [2, 3]] : tensor<1x2x12x5xf32> into tensor<1x2x60xf32>
+// CHECK:           %[[VAL_6:.*]] = linalg.copy ins(%[[VAL_4]] : tensor<1x2x60xf32>) outs(%[[VAL_5]] : tensor<1x2x60xf32>) -> tensor<1x2x60xf32>
+// CHECK:           %[[VAL_7:.*]] = tensor.expand_shape %[[VAL_6]] {{\[\[}}0], [1], [2, 3]] : tensor<1x2x60xf32> into tensor<1x2x12x5xf32>
+// CHECK:           %[[VAL_8:.*]] = tensor.expand_shape %[[VAL_7]] {{\[\[}}0], [1], [2, 3], [4]] : tensor<1x2x12x5xf32> into tensor<1x2x3x4x5xf32, 3 : i64>
+// CHECK:           return %[[VAL_8]] : tensor<1x2x3x4x5xf32, 3 : i64>
+// CHECK:         }
+
+func.func @linalg_copy(
+    %arg0: tensor<1x2x3x4x5xf32, 1>, %arg1: tensor<1x2x3x4x5xf32, 3>) -> tensor<1x2x3x4x5xf32, 3> {
+  %0 = linalg.copy ins(%arg0: tensor<1x2x3x4x5xf32, 1>) outs(%arg1: tensor<1x2x3x4x5xf32, 3>) -> tensor<1x2x3x4x5xf32, 3>
+  return %0 : tensor<1x2x3x4x5xf32, 3>
+}
+
+// -----
+
+// CHECK-LABEL:   func.func private @memref_linalg_copy(
+// CHECK-SAME:                                          %[[VAL_0:.*]]: memref<1x24x32x8xf32, 1>,
+// CHECK-SAME:                                          %[[VAL_1:.*]]: memref<1x24x32x8xf32, 1>) {
+// CHECK:           %[[VAL_2:.*]] = memref.collapse_shape %[[VAL_0]] {{\[\[}}0], [1], [2, 3]] : memref<1x24x32x8xf32, 1> into memref<1x24x256xf32, 1>
+// CHECK:           %[[VAL_3:.*]] = memref.collapse_shape %[[VAL_1]] {{\[\[}}0], [1], [2, 3]] : memref<1x24x32x8xf32, 1> into memref<1x24x256xf32, 1>
+// CHECK:           linalg.copy ins(%[[VAL_2]] : memref<1x24x256xf32, 1>) outs(%[[VAL_3]] : memref<1x24x256xf32, 1>)
+// CHECK:           return
+// CHECK:         }
+
+func.func private @memref_linalg_copy(%arg0: memref<1x24x32x8xf32, 1>, %arg1: memref<1x24x32x8xf32, 1>) {
+  linalg.copy ins(%arg0: memref<1x24x32x8xf32, 1>) outs(%arg1: memref<1x24x32x8xf32, 1>)
+  return
+}
diff --git a/mlir/test/Dialect/Linalg/convert-conv2d-to-img2col.mlir b/mlir/test/Dialect/Linalg/convert-conv2d-to-img2col.mlir
index 657cf83f25..b2470ed7b7 100644
--- a/mlir/test/Dialect/Linalg/convert-conv2d-to-img2col.mlir
+++ b/mlir/test/Dialect/Linalg/convert-conv2d-to-img2col.mlir
@@ -279,6 +279,76 @@ transform.sequence failures(propagate) {
 
 // -----
 
+// CHECK: IR printer: tensor_producer
+// CHECK-NEXT: %[[COL_TENSOR:.+]] = linalg.generic
+// CHECK-SAME: affine_map<(d0, d1, d2) -> (d0, d1, d2)>]
+// CHECK: ^bb0(%[[OUT_DATA:.+]]: f32)
+
+// Collapsed indices.
+// CHECK: %[[BINDEX:.+]] = linalg.index 0 : index
+// CHECK: %[[MINDEX:.+]] = linalg.index 1 : index
+// CHECK: %[[KINDEX:.+]] = linalg.index 2 : index
+
+// Compute input channel/convolved indices.
+// CHECK: %[[ICINDEX:.+]] = affine.apply affine_map<(d0) -> (d0 mod 4)>(%[[KINDEX]])
+// CHECK: %[[CONVH:.+]] = affine.apply affine_map<(d0, d1) -> (d0 floordiv 14 + d1 floordiv 12)>(%[[MINDEX]], %[[KINDEX]])
+// CHECK: %[[CONVW:.+]] = affine.apply affine_map<(d0, d1) -> (d0 mod 14 + (d1 mod 12) floordiv 4)>(%[[MINDEX]], %[[KINDEX]])
+
+// Extract from the input tensor.
+// CHECK: %[[EXTRACTED_INPUT:.+]] = tensor.extract
+// CHECK-SAME: %{{.+}}{{\[}}%[[BINDEX]], %[[CONVH]], %[[CONVW]], %[[ICINDEX]]] : tensor<1x16x16x4xf32>
+// CHECK: linalg.yield %[[EXTRACTED_INPUT]] : f32
+
+// CHECK: IR printer: transformed
+// CHECK: tensor.expand_shape %{{[^ ]*}} {{\[}}[0], [1, 2], [3]] : tensor<1x196x16xf32> into tensor<1x14x14x16xf32>
+
+// CHECK-DAG: #[[MAP0:.+]] = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
+// CHECK-DAG: #[[MAP1:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>
+// CHECK-DAG: #[[MAP2:.+]] = affine_map<(d0, d1, d2, d3) -> (d2, d3)>
+// CHECK-DAG: #[[MAP3:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+//      CHECK: @conv_2d_nhwc_fhwc
+//      CHECK-SAME: %[[INPUT:.+]]: tensor<1x16x16x4xf32>
+//      CHECK-SAME: %[[FILTER:.+]]: tensor<16x3x3x4xf32>
+//      CHECK-SAME: %[[OUTPUT:.+]]: tensor<1x14x14x16xf32>
+//  CHECK-DAG: %[[COLLAPSED_FILTER:.+]] = tensor.collapse_shape %[[FILTER]] {{\[}}[0], [1, 2, 3]] : tensor<16x3x3x4xf32> into tensor<16x36xf32>
+//  CHECK-DAG: %[[COLLAPSED_OUT:.+]] = tensor.collapse_shape %[[OUTPUT]] {{\[}}[0], [1, 2], [3]] : tensor<1x14x14x16xf32> into tensor<1x196x16xf32>
+//      CHECK: %[[INIT_COL_TENSOR:.+]] = tensor.empty() : tensor<1x196x36xf32>
+//      CHECK: %[[COL_TENSOR:.+]] = linalg.generic
+//           CHECK-SAME: #[[MAP0]]
+//                CHECK: ^bb0(%[[OUT_DATA:.+]]: f32)
+//                CHECK: linalg.yield %{{.+}} : f32
+//      CHECK: %[[MATMUL_RESULT:.+]] = linalg.generic
+//           CHECK-SAME: #[[MAP1]]
+//           CHECK-SAME: #[[MAP2]]
+//           CHECK-SAME: #[[MAP3]]
+//           CHECK-SAME: ins(%[[COL_TENSOR]], %[[COLLAPSED_FILTER]] : tensor<1x196x36xf32>, tensor<16x36xf32>)
+//           CHECK-SAME: outs(%[[COLLAPSED_OUT]] : tensor<1x196x16xf32>)
+//                CHECK: ^bb0(%[[ARG0:.+]]: f32, %[[ARG1:.+]]: f32, %[[ARG2:.+]]: f32)
+//                CHECK:     %[[MUL:.+]] = arith.mulf %[[ARG0]], %[[ARG1]] : f32
+//                CHECK:     %[[ADD:.+]] = arith.addf %[[MUL]], %[[ARG2]] : f32
+//                CHECK:     linalg.yield %[[ADD]] : f32
+//                CHECK: } -> tensor<1x196x16xf32>
+//      CHECK: %[[RESULT:.+]] = tensor.expand_shape %[[MATMUL_RESULT]] {{\[}}[0], [1, 2], [3]] : tensor<1x196x16xf32> into tensor<1x14x14x16xf32>
+//      CHECK: return %[[RESULT]]
+
+func.func @conv_2d_nhwc_fhwc(%arg0: tensor<1x16x16x4xf32>, %arg1: tensor<16x3x3x4xf32>, %arg2: tensor<1x14x14x16xf32>) -> tensor<1x14x14x16xf32> {
+    %0 = linalg.conv_2d_nhwc_fhwc
+      {dilations = dense<1> : tensor<2xi64>, strides = dense<1> : tensor<2xi64> }
+       ins(%arg0, %arg1: tensor<1x16x16x4xf32>, tensor<16x3x3x4xf32>)
+      outs(%arg2: tensor<1x14x14x16xf32>) -> tensor<1x14x14x16xf32>
+    return %0 : tensor<1x14x14x16xf32>
+}
+
+transform.sequence failures(propagate) {
+^bb1(%arg1: !transform.any_op):
+  %0 = transform.structured.match ops{["linalg.conv_2d_nhwc_fhwc"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+  %img2col_tensor_producer, %transformed = transform.structured.convert_conv2d_to_img2col %0 : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+  transform.print %img2col_tensor_producer {name = "tensor_producer"}: !transform.any_op
+  transform.print %transformed {name = "transformed"}: !transform.any_op
+}
+
+// -----
+
 // Check for signed extend when the input type is smaller than the accumulator type.
 
 // CHECK-DAG: #[[MAP0:.+]] = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
diff --git a/mlir/test/Dialect/Linalg/hoisting.mlir b/mlir/test/Dialect/Linalg/hoisting.mlir
index 7d0c3648c3..11bf4b58b9 100644
--- a/mlir/test/Dialect/Linalg/hoisting.mlir
+++ b/mlir/test/Dialect/Linalg/hoisting.mlir
@@ -872,3 +872,135 @@ transform.sequence failures(propagate) {
   transform.structured.hoist_redundant_vector_transfers %0
     : (!transform.any_op) -> !transform.any_op
 }
+
+// -----
+
+// Test that we can hoist out 1-D read-write pairs whose indices are dynamic values.
+
+// CHECK: #[[$MAP1:.+]] = affine_map<()[s0] -> (s0 + 1)>
+// CHECK: #[[$MAP4:.+]] = affine_map<()[s0] -> (s0 + 4)>
+
+//   CHECK-LABEL: func.func @hoist_vector_transfer_pairs_disjoint_dynamic
+//    CHECK-SAME: (%[[BUFFER:.+]]: memref<?x?xf32>, %{{.+}}: index, %{{.+}}: index, %{{.+}}: index, %[[I0:.+]]: index)
+
+//         CHECK:   %[[PLUS1:.+]] = affine.apply #[[$MAP1]]()[%[[I0]]]
+//         CHECK:   %[[PLUS4:.+]] = affine.apply #[[$MAP4]]()[%[[I0]]]
+//         CHECK:   %2 = vector.transfer_read %[[BUFFER]][%[[I0]], %[[I0]]]
+//         CHECK:   %3 = vector.transfer_read %[[BUFFER]][%[[PLUS1]], %[[I0]]]
+//         CHECK:   %4 = vector.transfer_read %[[BUFFER]][%[[PLUS1]], %[[PLUS4]]]
+// CHECK-COUNT-2:   scf.for %{{.+}} = {{.+}} -> (vector<4xf32>, vector<4xf32>, vector<4xf32>)
+// CHECK-COUNT-3:     "some_use"
+// CHECK-COUNT-2:   scf.yield {{.+}} : vector<4xf32>, vector<4xf32>, vector<4xf32>
+//         CHECK:   vector.transfer_write %{{.+}}, %[[BUFFER]][%[[PLUS1]], %[[PLUS4]]]
+//         CHECK:   vector.transfer_write %{{.+}}, %[[BUFFER]][%[[PLUS1]], %[[I0]]]
+//         CHECK:   vector.transfer_write %{{.+}}, %[[BUFFER]][%[[I0]], %[[I0]]]
+
+func.func @hoist_vector_transfer_pairs_disjoint_dynamic(
+    %buffer: memref<?x?xf32>, %lb : index, %ub : index, %step: index, %i0 : index) {
+  %cst = arith.constant 0.0 : f32
+  %i1 = affine.apply affine_map<(d0) -> (d0 + 1)>(%i0)
+  %i2 = affine.apply affine_map<(d0) -> (d0 + 4)>(%i0)
+
+  scf.for %i = %lb to %ub step %step {
+    scf.for %j = %lb to %ub step %step {
+      %r0 = vector.transfer_read %buffer[%i0, %i0], %cst: memref<?x?xf32>, vector<4xf32>
+      // Disjoint leading dim
+      %r1 = vector.transfer_read %buffer[%i1, %i0], %cst: memref<?x?xf32>, vector<4xf32>
+      // Non-overlap trailing dim
+      %r2 = vector.transfer_read %buffer[%i1, %i2], %cst: memref<?x?xf32>, vector<4xf32>
+      %u0 = "some_use"(%r0) : (vector<4xf32>) -> vector<4xf32>
+      %u1 = "some_use"(%r1) : (vector<4xf32>) -> vector<4xf32>
+      %u2 = "some_use"(%r2) : (vector<4xf32>) -> vector<4xf32>
+      vector.transfer_write %u0, %buffer[%i0, %i0] : vector<4xf32>, memref<?x?xf32>
+      vector.transfer_write %u1, %buffer[%i1, %i0] : vector<4xf32>, memref<?x?xf32>
+      vector.transfer_write %u2, %buffer[%i1, %i2] : vector<4xf32>, memref<?x?xf32>
+    }
+  }
+  return
+}
+
+transform.sequence failures(propagate) {
+^bb1(%arg1: !transform.any_op):
+  %0 = transform.structured.match ops{["func.func"]} in %arg1
+    : (!transform.any_op) -> !transform.any_op
+  transform.structured.hoist_redundant_vector_transfers %0
+    : (!transform.any_op) -> !transform.any_op
+}
+
+// -----
+
+// Test that we cannot hoist out read-write pairs whose indices are overlapping.
+
+//   CHECK-LABEL: func.func @hoist_vector_transfer_pairs_overlapping_dynamic
+// CHECK-COUNT-2:   scf.for
+// CHECK-COUNT-2:     vector.transfer_read
+// CHECK-COUNT-2:     vector.transfer_write
+
+func.func @hoist_vector_transfer_pairs_overlapping_dynamic(
+    %buffer: memref<?x?xf32>, %lb : index, %ub : index, %step: index, %i0 : index) {
+  %cst = arith.constant 0.0 : f32
+  %i1 = affine.apply affine_map<(d0) -> (d0 + 3)>(%i0)
+
+  scf.for %i = %lb to %ub step %step {
+    scf.for %j = %lb to %ub step %step {
+      %r0 = vector.transfer_read %buffer[%i0, %i0], %cst: memref<?x?xf32>, vector<4xf32>
+      // Overlapping range with the above
+      %r1 = vector.transfer_read %buffer[%i0, %i1], %cst: memref<?x?xf32>, vector<4xf32>
+      %u0 = "some_use"(%r0) : (vector<4xf32>) -> vector<4xf32>
+      %u1 = "some_use"(%r1) : (vector<4xf32>) -> vector<4xf32>
+      vector.transfer_write %u0, %buffer[%i0, %i0] : vector<4xf32>, memref<?x?xf32>
+      vector.transfer_write %u1, %buffer[%i0, %i1] : vector<4xf32>, memref<?x?xf32>
+    }
+  }
+  return
+}
+
+transform.sequence failures(propagate) {
+^bb1(%arg1: !transform.any_op):
+  %0 = transform.structured.match ops{["func.func"]} in %arg1
+    : (!transform.any_op) -> !transform.any_op
+  transform.structured.hoist_redundant_vector_transfers %0
+    : (!transform.any_op) -> !transform.any_op
+}
+
+// -----
+
+// Test that we can hoist out 2-D read-write pairs whose indices are dynamic values.
+
+//   CHECK-LABEL: func.func @hoist_vector_transfer_pairs_disjoint_dynamic
+// CHECK-COUNT-3:   vector.transfer_read
+// CHECK-COUNT-2:   %{{.+}}:3 = scf.for {{.+}} -> (vector<16x8xf32>, vector<16x8xf32>, vector<16x8xf32>)
+// CHECK-COUNT-2:   scf.yield {{.+}} : vector<16x8xf32>, vector<16x8xf32>, vector<16x8xf32>
+// CHECK-COUNT-3:   vector.transfer_write
+//         CHECK:   return
+
+func.func @hoist_vector_transfer_pairs_disjoint_dynamic(
+    %buffer: memref<?x?xf32>, %lb : index, %ub : index, %step: index, %i0 : index, %i1 : index) {
+  %cst = arith.constant 0.0 : f32
+  %i2 = affine.apply affine_map<(d0) -> ((d0 floordiv 32) * 16)>(%i1)
+  %i3 = affine.apply affine_map<(d0) -> ((d0 floordiv 32) * 16 + 8)>(%i1)
+  %i4 = affine.apply affine_map<(d0) -> ((d0 floordiv 32) * 16 + 16)>(%i1)
+
+  scf.for %i = %lb to %ub step %step {
+    scf.for %j = %lb to %ub step %step {
+      %r0 = vector.transfer_read %buffer[%i0, %i2], %cst: memref<?x?xf32>, vector<16x8xf32>
+      %r1 = vector.transfer_read %buffer[%i0, %i3], %cst: memref<?x?xf32>, vector<16x8xf32>
+      %r2 = vector.transfer_read %buffer[%i0, %i4], %cst: memref<?x?xf32>, vector<16x8xf32>
+      %u0 = "some_use"(%r0) : (vector<16x8xf32>) -> vector<16x8xf32>
+      %u1 = "some_use"(%r1) : (vector<16x8xf32>) -> vector<16x8xf32>
+      %u2 = "some_use"(%r2) : (vector<16x8xf32>) -> vector<16x8xf32>
+      vector.transfer_write %u2, %buffer[%i0, %i4] : vector<16x8xf32>, memref<?x?xf32>
+      vector.transfer_write %u1, %buffer[%i0, %i3] : vector<16x8xf32>, memref<?x?xf32>
+      vector.transfer_write %u0, %buffer[%i0, %i2] : vector<16x8xf32>, memref<?x?xf32>
+    }
+  }
+  return
+}
+
+transform.sequence failures(propagate) {
+^bb1(%arg1: !transform.any_op):
+  %0 = transform.structured.match ops{["func.func"]} in %arg1
+    : (!transform.any_op) -> !transform.any_op
+  transform.structured.hoist_redundant_vector_transfers %0
+    : (!transform.any_op) -> !transform.any_op
+}
diff --git a/mlir/test/Dialect/Linalg/invalid.mlir b/mlir/test/Dialect/Linalg/invalid.mlir
index 09acce04cd..56890df3f3 100644
--- a/mlir/test/Dialect/Linalg/invalid.mlir
+++ b/mlir/test/Dialect/Linalg/invalid.mlir
@@ -345,7 +345,7 @@ func.func @illegal_fill_memref_with_tensor_return
 func.func @illegal_fill_tensor_with_memref_return
   (%arg0 : tensor<?x?xf32>, %arg1 : f32) -> memref<?x?xf32>
 {
-  // expected-error @+1 {{result #0 must be ranked tensor of any type values, but got 'memref<?x?xf32>'}}
+  // expected-error @+1 {{result #0 must be variadic of ranked tensor of any type values, but got 'memref<?x?xf32>'}}
   %0 = linalg.fill ins(%arg1 : f32) outs(%arg0 : tensor<?x?xf32>) -> memref<?x?xf32>
   return %0 : memref<?x?xf32>
 }
diff --git a/mlir/test/Dialect/Linalg/match-ops-interpreter.mlir b/mlir/test/Dialect/Linalg/match-ops-interpreter.mlir
index bad6893eaa..1da092ab42 100644
--- a/mlir/test/Dialect/Linalg/match-ops-interpreter.mlir
+++ b/mlir/test/Dialect/Linalg/match-ops-interpreter.mlir
@@ -910,6 +910,19 @@ module attributes { transform.target_tag = "start_here" } {
     return %result : tensor<10x15xf64>
   }
 
+  func.func @vecmat_simple(%lhs: tensor<20xf32>, %rhs: tensor<20x15xf32>) -> tensor<15xf64> {
+    %cst = arith.constant 0.0 : f64
+    %empty = tensor.empty() : tensor<15xf64>
+    %fill = linalg.fill ins(%cst : f64) outs(%empty : tensor<15xf64>) -> tensor<15xf64>
+    // expected-remark @below {{contraction}}
+    // expected-remark @below {{batch dims}}
+    // expected-remark @below {{m dims}}
+    // expected-remark @below {{n dims 0}}
+    // expected-remark @below {{k dims 1}}
+    %result = linalg.vecmat ins(%lhs, %rhs: tensor<20xf32>, tensor<20x15xf32>) outs(%fill: tensor<15xf64>) -> tensor<15xf64>
+    return %result : tensor<15xf64>
+  }
+
   func.func @double_batch(%lhs: tensor<40x10x50x20xf32>, %rhs: tensor<40x20x50x15xf32>) -> tensor<40x10x50x15xf32> {
     %cst = arith.constant 0.0 : f32
     %empty = tensor.empty() : tensor<40x10x50x15xf32>
diff --git a/mlir/test/Dialect/Linalg/promote.mlir b/mlir/test/Dialect/Linalg/promote.mlir
index 5cd56db7fd..31b29c0e10 100644
--- a/mlir/test/Dialect/Linalg/promote.mlir
+++ b/mlir/test/Dialect/Linalg/promote.mlir
@@ -353,7 +353,6 @@ func.func @linalg_generic_update_all_function_inputs_outputs(%arg0: memref<3x4xf
   // CHECK:           %[[VAL_62:.*]] = memref.subview %[[VAL_61]][0, 0] {{\[}}%[[VAL_52]], %[[VAL_55]]] [1, 1] : memref<?x?xf32, #gpu.address_space<workgroup>> to memref<?x?xf32, strided<[?, 1], offset: ?>, #gpu.address_space<workgroup>>
   // CHECK:           memref.copy %[[VAL_3]], %[[VAL_24]] : memref<4x3xf32, strided<[4, 1]>, 1> to memref<?x?xf32, strided<[?, 1], offset: ?>, #gpu.address_space<workgroup>>
   // CHECK:           memref.copy %[[VAL_4]], %[[VAL_43]] : memref<4x3xf32, strided<[4, 1]>, 1> to memref<?x?xf32, strided<[?, 1], offset: ?>, #gpu.address_space<workgroup>>
-  // CHECK:           memref.copy %[[VAL_5]], %[[VAL_62]] : memref<4x3xf32, strided<[4, 1]>, 1> to memref<?x?xf32, strided<[?, 1], offset: ?>, #gpu.address_space<workgroup>>
   // CHECK:           linalg.generic {doc = "", indexing_maps = [#map, #map, #map], iterator_types = ["parallel", "parallel"], library_call = ""} ins(%[[VAL_24]], %[[VAL_43]] : memref<?x?xf32, strided<[?, 1], offset: ?>, #gpu.address_space<workgroup>>, memref<?x?xf32, strided<[?, 1], offset: ?>, #gpu.address_space<workgroup>>) outs(%[[VAL_62]] : memref<?x?xf32, strided<[?, 1], offset: ?>, #gpu.address_space<workgroup>>) {
   // CHECK:           ^bb0(%[[VAL_63:.*]]: f32, %[[VAL_64:.*]]: f32, %[[VAL_65:.*]]: f32):
   // CHECK:             %[[VAL_66:.*]] = arith.addf %[[VAL_63]], %[[VAL_64]] : f32
diff --git a/mlir/test/Dialect/MemRef/fold-memref-alias-ops.mlir b/mlir/test/Dialect/MemRef/fold-memref-alias-ops.mlir
index 7d6a2e57d9..47ec0e67d9 100644
--- a/mlir/test/Dialect/MemRef/fold-memref-alias-ops.mlir
+++ b/mlir/test/Dialect/MemRef/fold-memref-alias-ops.mlir
@@ -317,6 +317,22 @@ func.func @fold_static_stride_subview_with_affine_load_store_collapse_shape(%arg
 
 // -----
 
+// CHECK-DAG: #[[$MAP0:.*]] = affine_map<()[s0] -> (s0 floordiv 6)>
+// CHECK-DAG: #[[$MAP1:.*]] = affine_map<()[s0] -> (s0 mod 6)>
+// CHECK-LABEL: @fold_dynamic_size_collapse_shape_with_affine_load
+// CHECK-SAME: (%[[ARG0:.*]]: memref<?x6x32xf32>, %[[ARG1:.*]]: index, %[[ARG2:.*]]: index)
+func.func @fold_dynamic_size_collapse_shape_with_affine_load(%arg0 : memref<?x6x32xf32>, %arg1 : index, %arg2 : index) -> f32 {
+  %0 = memref.collapse_shape %arg0 [[0, 1], [2]] : memref<?x6x32xf32> into memref<?x32xf32>
+  %1 = affine.load %0[%arg1, %arg2] : memref<?x32xf32>
+  return %1 : f32
+}
+// CHECK-NEXT: %[[MODIFIED_INDEX0:.*]] = affine.apply #[[$MAP0]]()[%[[ARG1]]]
+// CHECK-NEXT: %[[MODIFIED_INDEX1:.*]] = affine.apply #[[$MAP1]]()[%[[ARG1]]]
+// CHECK-NEXT: %[[RESULT:.*]] = affine.load %[[ARG0]][%[[MODIFIED_INDEX0]], %[[MODIFIED_INDEX1]], %[[ARG2]]] : memref<?x6x32xf32>
+// CHECK-NEXT: return %[[RESULT]] : f32
+
+// -----
+
 // CHECK-DAG: #[[$MAP:.*]] = affine_map<()[s0, s1, s2] -> (s0 * 6 + s1 * 3 + s2)>
 // CHECK-LABEL: fold_static_stride_subview_with_affine_load_store_expand_shape_3d
 // CHECK-SAME: (%[[ARG0:.*]]: memref<12x32xf32>, %[[ARG1:.*]]: index, %[[ARG2:.*]]: index, %[[ARG3:.*]]: index, %[[ARG4:.*]]: index) -> f32 {
diff --git a/mlir/test/Dialect/NVGPU/invalid.mlir b/mlir/test/Dialect/NVGPU/invalid.mlir
index 66652070ec..41b29fa74b 100644
--- a/mlir/test/Dialect/NVGPU/invalid.mlir
+++ b/mlir/test/Dialect/NVGPU/invalid.mlir
@@ -224,13 +224,13 @@ func.func @async_cp_size_invalid_f64(
 
 // -----
 
-!tResult = !nvgpu.warpgroup.accumulator<fragmented = vector<64x128xf32>>
+!tResult = !nvgpu.warpgroup.accumulator<fragmented = vector<128x128xf32>>
 !tDescA  = !nvgpu.warpgroup.descriptor<tensor = memref<128x64xf16, 3>>
 !tDescB  = !nvgpu.warpgroup.descriptor<tensor = memref<64x121xf16, 3>>
 
-func.func @warpgroup_mma_wrong_input(%descA: !tDescA, %descB: !tDescB, %acc1: !tResult, %acc2: !tResult) {
+func.func @warpgroup_mma_wrong_input(%descA: !tDescA, %descB: !tDescB, %acc: !tResult) {
   // expected-error @+1 {{'nvgpu.warpgroup.mma' op 2nd dim matrix-B ( 121 ) != 2nd dim matrix-C ( 128 )}}  
-  %0:2 = nvgpu.warpgroup.mma %descA, %descB, %acc1, %acc1: !tDescA, !tDescB, !tResult, !tResult -> !tResult, !tResult
+  %0 = nvgpu.warpgroup.mma %descA, %descB, %acc: !tDescA, !tDescB, !tResult -> !tResult
   return
 }
 
@@ -239,29 +239,29 @@ func.func @warpgroup_mma_wrong_input(%descA: !tDescA, %descB: !tDescB, %acc1: !t
 !tResult = !nvgpu.warpgroup.accumulator<fragmented = vector<128xf32>>
 !tDescA  = !nvgpu.warpgroup.descriptor<tensor = memref<128x64xf16, 3>>
 !tDescB  = !nvgpu.warpgroup.descriptor<tensor = memref<64x128xf16, 3>>
-func.func @warpgroup_mma_wrong_input(%descA: !tDescA, %descB: !tDescB, %acc1: !tResult, %acc2: !tResult) {
+func.func @warpgroup_mma_wrong_input(%descA: !tDescA, %descB: !tDescB, %acc: !tResult) {
   // expected-error @+1 {{'nvgpu.warpgroup.mma' op has matrices A, B, C and D, they must be 2 dimensional}}  
-  %0:2 = nvgpu.warpgroup.mma %descA, %descB, %acc1, %acc1: !tDescA, !tDescB, !tResult, !tResult -> !tResult, !tResult
+  %0 = nvgpu.warpgroup.mma %descA, %descB, %acc: !tDescA, !tDescB, !tResult -> !tResult
   return
 }
 
 // -----
-!tResult = !nvgpu.warpgroup.accumulator<fragmented = vector<64x128xf32>>
+!tResult = !nvgpu.warpgroup.accumulator<fragmented = vector<128x128xf32>>
 !tDescA  = !nvgpu.warpgroup.descriptor<tensor = memref<128x64xf16, 3>>
 !tDescB  = !nvgpu.warpgroup.descriptor<tensor = memref<64x128xf32, 3>>
-func.func @warpgroup_mma_wrong_input(%descA: !tDescA, %descB: !tDescB, %acc1: !tResult, %acc2: !tResult) {
+func.func @warpgroup_mma_wrong_input(%descA: !tDescA, %descB: !tDescB, %acc: !tResult) {
   // expected-error @+1 {{'nvgpu.warpgroup.mma' op 'f32' += 'f16' * 'f32', it is not supported.}}  
-  %0:2 = nvgpu.warpgroup.mma %descA, %descB, %acc1, %acc1: !tDescA, !tDescB, !tResult, !tResult -> !tResult, !tResult
+  %0 = nvgpu.warpgroup.mma %descA, %descB, %acc: !tDescA, !tDescB, !tResult -> !tResult
   return
 }
 
 // -----
 
-!tResult = !nvgpu.warpgroup.accumulator<fragmented = vector<64x128xf32>>
+!tResult = !nvgpu.warpgroup.accumulator<fragmented = vector<128x128xf32>>
 !tDescA  = !nvgpu.warpgroup.descriptor<tensor = memref<128x64xf16, 3>>
 !tDescB  = !nvgpu.warpgroup.descriptor<tensor = memref<64x512xf16, 3>>
-func.func @warpgroup_mma_wrong_input(%descA: !tDescA, %descB: !tDescB, %acc1: !tResult, %acc2: !tResult) {
+func.func @warpgroup_mma_wrong_input(%descA: !tDescA, %descB: !tDescB, %acc: !tResult) {
   // expected-error @+1 {{'nvgpu.warpgroup.mma' op 2nd dim matrix-B ( 512 ) != 2nd dim matrix-C ( 128 )}}
-  %0:2 = nvgpu.warpgroup.mma %descA, %descB, %acc1, %acc1: !tDescA, !tDescB, !tResult, !tResult -> !tResult, !tResult
+  %0 = nvgpu.warpgroup.mma %descA, %descB, %acc: !tDescA, !tDescB, !tResult -> !tResult
   return
 }
diff --git a/mlir/test/Dialect/OpenACC/invalid.mlir b/mlir/test/Dialect/OpenACC/invalid.mlir
index 225a8766fc..ff92eab478 100644
--- a/mlir/test/Dialect/OpenACC/invalid.mlir
+++ b/mlir/test/Dialect/OpenACC/invalid.mlir
@@ -312,18 +312,6 @@ acc.firstprivate.recipe @privatization_i32 : !llvm.ptr<i32> init {
 
 // -----
 
-// expected-error@+1 {{expects init region to yield a value of the privatization type}}
-acc.firstprivate.recipe @privatization_i32 : !llvm.ptr<f32> init {
-^bb0(%arg0 : !llvm.ptr<f32>):
-  %c1 = arith.constant 1 : i32
-  %c0 = arith.constant 0 : i32
-  %0 = llvm.alloca %c1 x i32 : (i32) -> !llvm.ptr<i32>
-  llvm.store %c0, %0 : !llvm.ptr<i32>
-  acc.yield %0 : !llvm.ptr<i32>
-} copy {}
-
-// -----
-
 // expected-error@+1 {{expects non-empty copy region}}
 acc.firstprivate.recipe @privatization_i32 : !llvm.ptr<i32> init {
 ^bb0(%arg0 : !llvm.ptr<i32>):
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index 13cbea6c99..27c31824c0 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -831,7 +831,7 @@ omp.critical.declare @mutex6 hint(contended, nonspeculative)
 omp.critical.declare @mutex7 hint(uncontended, speculative)
 // CHECK: omp.critical.declare @mutex8 hint(contended, speculative)
 omp.critical.declare @mutex8 hint(contended, speculative)
-// CHECK: omp.critical.declare @mutex9 hint(none)
+// CHECK: omp.critical.declare @mutex9
 omp.critical.declare @mutex9 hint(none)
 // CHECK: omp.critical.declare @mutex10
 omp.critical.declare @mutex10
@@ -909,7 +909,7 @@ func.func @omp_atomic_read(%v: memref<i32>, %x: memref<i32>) {
   omp.atomic.read %v = %x hint(nonspeculative, contended) : memref<i32>, i32
   // CHECK: omp.atomic.read %[[v]] = %[[x]] memory_order(seq_cst) hint(contended, speculative) : memref<i32>, i32
   omp.atomic.read %v = %x hint(speculative, contended) memory_order(seq_cst) : memref<i32>, i32
-  // CHECK: omp.atomic.read %[[v]] = %[[x]] memory_order(seq_cst) hint(none) : memref<i32>, i32
+  // CHECK: omp.atomic.read %[[v]] = %[[x]] memory_order(seq_cst) : memref<i32>, i32
   omp.atomic.read %v = %x hint(none) memory_order(seq_cst) : memref<i32>, i32
   return
 }
@@ -927,7 +927,7 @@ func.func @omp_atomic_write(%addr : memref<i32>, %val : i32) {
   omp.atomic.write %addr = %val memory_order(relaxed) : memref<i32>, i32
   // CHECK: omp.atomic.write %[[ADDR]] = %[[VAL]] hint(uncontended, speculative) : memref<i32>, i32
   omp.atomic.write %addr = %val hint(speculative, uncontended) : memref<i32>, i32
-  // CHECK: omp.atomic.write %[[ADDR]] = %[[VAL]] hint(none) : memref<i32>, i32
+  // CHECK: omp.atomic.write %[[ADDR]] = %[[VAL]] : memref<i32>, i32
   omp.atomic.write %addr = %val hint(none) : memref<i32>, i32
   return
 }
@@ -1004,7 +1004,7 @@ func.func @omp_atomic_update(%x : memref<i32>, %expr : i32, %xBool : memref<i1>,
     omp.yield(%const:i32)
   }
 
-  // CHECK: omp.atomic.update hint(none) %[[X]] : memref<i32>
+  // CHECK: omp.atomic.update %[[X]] : memref<i32>
   // CHECK-NEXT: (%[[XVAL:.*]]: i32):
   // CHECK-NEXT:   %[[NEWVAL:.*]] = llvm.add %[[XVAL]], %[[EXPR]] : i32
   // CHECK-NEXT:   omp.yield(%[[NEWVAL]] : i32)
@@ -1181,7 +1181,7 @@ func.func @omp_atomic_capture(%v: memref<i32>, %x: memref<i32>, %expr: i32) {
     omp.atomic.write %x = %expr : memref<i32>, i32
   }
 
-  // CHECK: omp.atomic.capture hint(none) {
+  // CHECK: omp.atomic.capture {
   // CHECK-NEXT: omp.atomic.update %[[x]] : memref<i32>
   // CHECK-NEXT: (%[[xval:.*]]: i32):
   // CHECK-NEXT:   %[[newval:.*]] = llvm.add %[[xval]], %[[expr]] : i32
diff --git a/mlir/test/Dialect/SCF/invalid.mlir b/mlir/test/Dialect/SCF/invalid.mlir
index f6044ad108..1b2c3f5631 100644
--- a/mlir/test/Dialect/SCF/invalid.mlir
+++ b/mlir/test/Dialect/SCF/invalid.mlir
@@ -96,6 +96,32 @@ func.func @not_enough_loop_results(%arg0: index, %init: f32) {
 
 // -----
 
+func.func @too_many_iter_args(%arg0: index, %init: f32) {
+  // expected-error @below{{different number of inits and region iter_args: 1 != 2}}
+  %x = "scf.for"(%arg0, %arg0, %arg0, %init) (
+    {
+    ^bb0(%i0 : index, %iter: f32, %iter2: f32):
+      scf.yield %iter, %iter : f32, f32
+    }
+  ) : (index, index, index, f32) -> (f32)
+  return
+}
+
+// -----
+
+func.func @too_few_yielded_values(%arg0: index, %init: f32) {
+  // expected-error @below{{different number of region iter_args and yielded values: 2 != 1}}
+  %x, %x2 = "scf.for"(%arg0, %arg0, %arg0, %init, %init) (
+    {
+    ^bb0(%i0 : index, %iter: f32, %iter2: f32):
+      scf.yield %iter : f32
+    }
+  ) : (index, index, index, f32, f32) -> (f32, f32)
+  return
+}
+
+// -----
+
 func.func @loop_if_not_i1(%arg0: index) {
   // expected-error@+1 {{operand #0 must be 1-bit signless integer}}
   "scf.if"(%arg0) ({}, {}) : (index) -> ()
@@ -422,7 +448,8 @@ func.func @std_for_operands_mismatch_3(%arg0 : index, %arg1 : index, %arg2 : ind
 func.func @std_for_operands_mismatch_4(%arg0 : index, %arg1 : index, %arg2 : index) {
   %s0 = arith.constant 0.0 : f32
   %t0 = arith.constant 1.0 : f32
-  // expected-error @+1 {{along control flow edge from Region #0 to Region #0: source type #1 'i32' should match input type #1 'f32'}}
+  // expected-error @below {{1-th region iter_arg and 1-th yielded value have different type: 'f32' != 'i32'}}
+  // expected-error @below {{along control flow edge from Region #0 to Region #0: source type #1 'i32' should match input type #1 'f32'}}
   %result1:2 = scf.for %i0 = %arg0 to %arg1 step %arg2
                     iter_args(%si = %s0, %ti = %t0) -> (f32, f32) {
     %sn = arith.addf %si, %si : f32
@@ -432,7 +459,6 @@ func.func @std_for_operands_mismatch_4(%arg0 : index, %arg1 : index, %arg2 : ind
   return
 }
 
-
 // -----
 
 func.func @parallel_invalid_yield(
diff --git a/mlir/test/Dialect/SCF/loop-unroll.mlir b/mlir/test/Dialect/SCF/loop-unroll.mlir
index c83e33d7fb..e28efbb6ec 100644
--- a/mlir/test/Dialect/SCF/loop-unroll.mlir
+++ b/mlir/test/Dialect/SCF/loop-unroll.mlir
@@ -186,6 +186,36 @@ func.func @static_loop_unroll_by_2(%arg0 : memref<?xf32>) {
 // UNROLL-BY-2-ANNOTATE:    memref.store %{{.*}}, %[[MEM:.*0]][%{{.*}}] {unrolled_iteration = 0 : ui32} : memref<?xf32>
 // UNROLL-BY-2-ANNOTATE:    memref.store %{{.*}}, %[[MEM]][%{{.*}}] {unrolled_iteration = 1 : ui32} : memref<?xf32>
 
+// Test that no epilogue clean-up loop is generated because the trip count
+// (taking into account the non-unit step size) is a multiple of the unroll
+// factor.
+func.func @static_loop_step_2_unroll_by_2(%arg0 : memref<?xf32>) {
+  %0 = arith.constant 7.0 : f32
+  %lb = arith.constant 0 : index
+  %ub = arith.constant 19 : index
+  %step = arith.constant 2 : index
+  scf.for %i0 = %lb to %ub step %step {
+    memref.store %0, %arg0[%i0] : memref<?xf32>
+  }
+  return
+}
+
+// UNROLL-BY-2-LABEL: func @static_loop_step_2_unroll_by_2
+//  UNROLL-BY-2-SAME:  %[[MEM:.*0]]: memref<?xf32>
+//
+//   UNROLL-BY-2-DAG:  %[[C0:.*]] = arith.constant 0 : index
+//   UNROLL-BY-2-DAG:  %[[C2:.*]] = arith.constant 2 : index
+//   UNROLL-BY-2-DAG:  %[[C19:.*]] = arith.constant 19 : index
+//   UNROLL-BY-2-DAG:  %[[C4:.*]] = arith.constant 4 : index
+//   UNROLL-BY-2:  scf.for %[[IV:.*]] = %[[C0]] to %[[C19]] step %[[C4]] {
+//  UNROLL-BY-2-NEXT:    memref.store %{{.*}}, %[[MEM]][%[[IV]]] : memref<?xf32>
+//  UNROLL-BY-2-NEXT:    %[[C1_IV:.*]] = arith.constant 1 : index
+//  UNROLL-BY-2-NEXT:    %[[V0:.*]] = arith.muli %[[C2]], %[[C1_IV]] : index
+//  UNROLL-BY-2-NEXT:    %[[V1:.*]] = arith.addi %[[IV]], %[[V0]] : index
+//  UNROLL-BY-2-NEXT:    memref.store %{{.*}}, %[[MEM]][%[[V1]]] : memref<?xf32>
+//  UNROLL-BY-2-NEXT:  }
+//  UNROLL-BY-2-NEXT:  return
+
 // Test that epilogue clean up loop is generated (trip count is not
 // a multiple of unroll factor).
 func.func @static_loop_unroll_by_3(%arg0 : memref<?xf32>) {
diff --git a/mlir/test/Dialect/SPIRV/Transforms/webgpu-prepare.mlir b/mlir/test/Dialect/SPIRV/Transforms/webgpu-prepare.mlir
index 91eeeda6ec..1ec4e5e4f9 100644
--- a/mlir/test/Dialect/SPIRV/Transforms/webgpu-prepare.mlir
+++ b/mlir/test/Dialect/SPIRV/Transforms/webgpu-prepare.mlir
@@ -145,4 +145,41 @@ spirv.func @smul_extended_i16(%arg : i16) -> !spirv.struct<(i16, i16)> "None" {
   spirv.ReturnValue %0 : !spirv.struct<(i16, i16)>
 }
 
+// CHECK-LABEL: func @iaddcarry_i32
+// CHECK-SAME:       ([[A:%.+]]: i32, [[B:%.+]]: i32)
+// CHECK-NEXT:       [[ONE:%.+]]    = spirv.Constant 1 : i32
+// CHECK-NEXT:       [[ZERO:%.+]]   = spirv.Constant 0 : i32
+// CHECK-NEXT:       [[OUT:%.+]]    = spirv.IAdd [[A]], [[B]]
+// CHECK-NEXT:       [[CMP:%.+]]    = spirv.ULessThan [[OUT]], [[A]]
+// CHECK-NEXT:       [[CARRY:%.+]]  = spirv.Select [[CMP]], [[ONE]], [[ZERO]]
+// CHECK-NEXT:       [[RES:%.+]]     = spirv.CompositeConstruct [[OUT]], [[CARRY]] : (i32, i32) -> !spirv.struct<(i32, i32)>
+// CHECK-NEXT:       spirv.ReturnValue [[RES]] : !spirv.struct<(i32, i32)>
+spirv.func @iaddcarry_i32(%a : i32, %b : i32) -> !spirv.struct<(i32, i32)> "None" {
+  %0 = spirv.IAddCarry %a, %b : !spirv.struct<(i32, i32)>
+  spirv.ReturnValue %0 : !spirv.struct<(i32, i32)>
+}
+
+// CHECK-LABEL: func @iaddcarry_vector_i32
+// CHECK-SAME:       ([[A:%.+]]: vector<3xi32>, [[B:%.+]]: vector<3xi32>)
+// CHECK-NEXT:       [[ONE:%.+]]    = spirv.Constant dense<1> : vector<3xi32>
+// CHECK-NEXT:       [[ZERO:%.+]]   = spirv.Constant dense<0> : vector<3xi32>
+// CHECK-NEXT:       [[OUT:%.+]]    = spirv.IAdd [[A]], [[B]]
+// CHECK-NEXT:       [[CMP:%.+]]    = spirv.ULessThan [[OUT]], [[A]]
+// CHECK-NEXT:       [[CARRY:%.+]]  = spirv.Select [[CMP]], [[ONE]], [[ZERO]]
+// CHECK-NEXT:       [[RES:%.+]]    = spirv.CompositeConstruct [[OUT]], [[CARRY]] : (vector<3xi32>, vector<3xi32>) -> !spirv.struct<(vector<3xi32>, vector<3xi32>)>
+// CHECK-NEXT:       spirv.ReturnValue [[RES]] : !spirv.struct<(vector<3xi32>, vector<3xi32>)>
+spirv.func @iaddcarry_vector_i32(%a : vector<3xi32>, %b : vector<3xi32>)
+  -> !spirv.struct<(vector<3xi32>, vector<3xi32>)> "None" {
+  %0 = spirv.IAddCarry %a, %b : !spirv.struct<(vector<3xi32>, vector<3xi32>)>
+  spirv.ReturnValue %0 : !spirv.struct<(vector<3xi32>, vector<3xi32>)>
+}
+
+// CHECK-LABEL: func @iaddcarry_i16
+// CHECK-NEXT:       spirv.IAddCarry
+// CHECK-NEXT:       spirv.ReturnValue
+spirv.func @iaddcarry_i16(%a : i16, %b : i16) -> !spirv.struct<(i16, i16)> "None" {
+  %0 = spirv.IAddCarry %a, %b : !spirv.struct<(i16, i16)>
+  spirv.ReturnValue %0 : !spirv.struct<(i16, i16)>
+}
+
 } // end module
diff --git a/mlir/test/Dialect/SparseTensor/codegen_sparse_dealloc.mlir b/mlir/test/Dialect/SparseTensor/codegen_sparse_dealloc.mlir
deleted file mode 100644
index 59e568dd5d..0000000000
--- a/mlir/test/Dialect/SparseTensor/codegen_sparse_dealloc.mlir
+++ /dev/null
@@ -1,26 +0,0 @@
-// RUN: mlir-opt %s --post-sparsification-rewrite="enable-runtime-library=false" \
-// RUN:    --sparse-tensor-codegen=create-sparse-deallocs=false \
-// RUN:    --canonicalize --cse | FileCheck %s -check-prefix=CHECK-NO-DEALLOC
-
-// RUN: mlir-opt %s --post-sparsification-rewrite="enable-runtime-library=false" \
-// RUN:    --sparse-tensor-codegen=create-sparse-deallocs=true \
-// RUN:    --canonicalize --cse | FileCheck %s -check-prefix=CHECK-DEALLOC
-
-#CSR = #sparse_tensor.encoding<{ map = (d0, d1) -> (d0 : dense, d1 : compressed)}>
-#CSC = #sparse_tensor.encoding<{
-  map = (d0, d1) -> (d1 : dense, d0 : compressed),
-}>
-
-//
-// No memref.dealloc is user-requested so
-// CHECK-NO-DEALLOC-LABEL: @sparse_convert_permuted
-// CHECK-NO-DEALLOC-NOT: memref.dealloc
-//
-// Otherwise memref.dealloc is created to free temporary sparse buffers.
-// CHECK-DEALLOC-LABEL: @sparse_convert_permuted
-// CHECK-DEALLOC: memref.dealloc
-//
-func.func @sparse_convert_permuted(%arg0: tensor<?x?xf32, #CSR>) -> tensor<?x?xf32, #CSC> {
-  %0 = sparse_tensor.convert %arg0 : tensor<?x?xf32, #CSR> to tensor<?x?xf32, #CSC>
-  return %0 : tensor<?x?xf32, #CSC>
-}
diff --git a/mlir/test/Dialect/SparseTensor/conversion.mlir b/mlir/test/Dialect/SparseTensor/conversion.mlir
index 29093a055a..2ff4887dae 100644
--- a/mlir/test/Dialect/SparseTensor/conversion.mlir
+++ b/mlir/test/Dialect/SparseTensor/conversion.mlir
@@ -78,11 +78,11 @@ func.func @sparse_dim3d_const(%arg0: tensor<10x20x30xf64, #SparseTensor>) -> ind
 //   CHECK-DAG: %[[DimShape0:.*]] = memref.alloca() : memref<1xindex>
 //   CHECK-DAG: %[[DimShape:.*]] = memref.cast %[[DimShape0]] : memref<1xindex> to memref<?xindex>
 //       CHECK: %[[Reader:.*]] = call @createCheckedSparseTensorReader(%[[A]], %[[DimShape]], %{{.*}})
-//   CHECK-DAG: %[[Iota0:.*]] = memref.alloca() : memref<1xindex>
-//   CHECK-DAG: %[[Iota:.*]] = memref.cast %[[Iota0]] : memref<1xindex> to memref<?xindex>
 //   CHECK-DAG: %[[LvlTypes0:.*]] = memref.alloca() : memref<1xi8>
 //   CHECK-DAG: %[[LvlTypes:.*]] = memref.cast %[[LvlTypes0]] : memref<1xi8> to memref<?xi8>
-//       CHECK: %[[T:.*]] = call @newSparseTensorFromReader(%[[Reader]], %[[DimShape]], %[[LvlTypes]], %[[Iota]], %[[Iota]], %{{.*}}, %{{.*}}, %{{.*}})
+//   CHECK-DAG: %[[Iota0:.*]] = memref.alloca() : memref<1xindex>
+//   CHECK-DAG: %[[Iota:.*]] = memref.cast %[[Iota0]] : memref<1xindex> to memref<?xindex>
+//       CHECK: %[[T:.*]] = call @newSparseTensor(%[[DimShape]], %[[DimShape]], %[[LvlTypes]], %[[Iota]], %[[Iota]], %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %[[Reader]])
 //       CHECK: call @delSparseTensorReader(%[[Reader]])
 //       CHECK: return %[[T]] : !llvm.ptr<i8>
 func.func @sparse_new1d(%arg0: !llvm.ptr<i8>) -> tensor<128xf64, #SparseVector> {
@@ -96,11 +96,11 @@ func.func @sparse_new1d(%arg0: !llvm.ptr<i8>) -> tensor<128xf64, #SparseVector>
 //   CHECK-DAG: %[[DimShape:.*]] = memref.cast %[[DimShape0]] : memref<2xindex> to memref<?xindex>
 //       CHECK: %[[Reader:.*]] = call @createCheckedSparseTensorReader(%[[A]], %[[DimShape]], %{{.*}})
 //       CHECK: %[[DimSizes:.*]] = call @getSparseTensorReaderDimSizes(%[[Reader]])
-//   CHECK-DAG: %[[Iota0:.*]] = memref.alloca() : memref<2xindex>
-//   CHECK-DAG: %[[Iota:.*]] = memref.cast %[[Iota0]] : memref<2xindex> to memref<?xindex>
 //   CHECK-DAG: %[[LvlTypes0:.*]] = memref.alloca() : memref<2xi8>
 //   CHECK-DAG: %[[LvlTypes:.*]] = memref.cast %[[LvlTypes0]] : memref<2xi8> to memref<?xi8>
-//       CHECK: %[[T:.*]] = call @newSparseTensorFromReader(%[[Reader]], %[[DimSizes]], %[[LvlTypes]], %[[Iota]], %[[Iota]], %{{.*}}, %{{.*}}, %{{.*}})
+//   CHECK-DAG: %[[Iota0:.*]] = memref.alloca() : memref<2xindex>
+//   CHECK-DAG: %[[Iota:.*]] = memref.cast %[[Iota0]] : memref<2xindex> to memref<?xindex>
+//       CHECK: %[[T:.*]] = call @newSparseTensor(%[[DimSizes]], %[[DimSizes]], %[[LvlTypes]], %[[Iota]], %[[Iota]], %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %[[Reader]])
 //       CHECK: call @delSparseTensorReader(%[[Reader]])
 //       CHECK: return %[[T]] : !llvm.ptr<i8>
 func.func @sparse_new2d(%arg0: !llvm.ptr<i8>) -> tensor<?x?xf32, #CSR> {
@@ -114,15 +114,15 @@ func.func @sparse_new2d(%arg0: !llvm.ptr<i8>) -> tensor<?x?xf32, #CSR> {
 //   CHECK-DAG: %[[DimShape:.*]] = memref.cast %[[DimShape0]] : memref<3xindex> to memref<?xindex>
 //       CHECK: %[[Reader:.*]] = call @createCheckedSparseTensorReader(%[[A]], %[[DimShape]], %{{.*}})
 //       CHECK: %[[DimSizes:.*]] = call @getSparseTensorReaderDimSizes(%[[Reader]])
-//       CHECK: %[[Dim2Lvl0:.*]] = memref.alloca() : memref<3xindex>
-//       CHECK: %[[Dim2Lvl:.*]] = memref.cast %[[Dim2Lvl0]] : memref<3xindex> to memref<?xindex>
-//       CHECK: %[[Lvl2Dim0:.*]] = memref.alloca() : memref<3xindex>
-//       CHECK: %[[Lvl2Dim:.*]] = memref.cast %[[Lvl2Dim0]] : memref<3xindex> to memref<?xindex>
-//       CHECK: %[[LvlSizes0:.*]] = memref.alloca() : memref<3xindex>
-//       CHECK: %[[LvlSizes:.*]] = memref.cast %[[LvlSizes0]] : memref<3xindex> to memref<?xindex>
-//       CHECK: %[[LvlTypes0:.*]] = memref.alloca() : memref<3xi8>
-//       CHECK: %[[LvlTypes:.*]] = memref.cast %[[LvlTypes0]] : memref<3xi8> to memref<?xi8>
-//       CHECK: %[[T:.*]] = call @newSparseTensorFromReader(%[[Reader]], %[[LvlSizes]], %[[LvlTypes]], %[[Dim2Lvl]], %[[Lvl2Dim]], %{{.*}}, %{{.*}}, %{{.*}})
+//   CHECK-DAG: %[[LvlTypes0:.*]] = memref.alloca() : memref<3xi8>
+//   CHECK-DAG: %[[LvlTypes:.*]] = memref.cast %[[LvlTypes0]] : memref<3xi8> to memref<?xi8>
+//   CHECK-DAG: %[[Dim2Lvl0:.*]] = memref.alloca() : memref<3xindex>
+//   CHECK-DAG: %[[Dim2Lvl:.*]] = memref.cast %[[Dim2Lvl0]] : memref<3xindex> to memref<?xindex>
+//   CHECK-DAG: %[[Lvl2Dim0:.*]] = memref.alloca() : memref<3xindex>
+//   CHECK-DAG: %[[Lvl2Dim:.*]] = memref.cast %[[Lvl2Dim0]] : memref<3xindex> to memref<?xindex>
+//   CHECK-DAG: %[[LvlSizes0:.*]] = memref.alloca() : memref<3xindex>
+//   CHECK-DAG: %[[LvlSizes:.*]] = memref.cast %[[LvlSizes0]] : memref<3xindex> to memref<?xindex>
+//       CHECK: %[[T:.*]] = call @newSparseTensor(%[[DimSizes]], %[[LvlSizes]], %[[LvlTypes]], %[[Dim2Lvl]], %[[Lvl2Dim]], %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %[[Reader]])
 //       CHECK: call @delSparseTensorReader(%[[Reader]])
 //       CHECK: return %[[T]] : !llvm.ptr<i8>
 func.func @sparse_new3d(%arg0: !llvm.ptr<i8>) -> tensor<?x?x?xf32, #SparseTensor> {
@@ -296,7 +296,7 @@ func.func @sparse_reconstruct(%arg0: tensor<128xf32, #SparseVector>) -> tensor<1
 
 // CHECK-LABEL: func @sparse_reconstruct_ins(
 //  CHECK-SAME: %[[A:.*]]: !llvm.ptr<i8>
-//       CHECK: call @endInsert(%[[A]]) : (!llvm.ptr<i8>) -> ()
+//       CHECK: call @endLexInsert(%[[A]]) : (!llvm.ptr<i8>) -> ()
 //       CHECK: return %[[A]] : !llvm.ptr<i8>
 func.func @sparse_reconstruct_ins(%arg0: tensor<128xf32, #SparseVector>) -> tensor<128xf32, #SparseVector> {
   %0 = sparse_tensor.load %arg0 hasInserts : tensor<128xf32, #SparseVector>
diff --git a/mlir/test/Dialect/SparseTensor/convert_dense2sparse.mlir b/mlir/test/Dialect/SparseTensor/convert_dense2sparse.mlir
index 1a69c80f7e..4dba16df39 100644
--- a/mlir/test/Dialect/SparseTensor/convert_dense2sparse.mlir
+++ b/mlir/test/Dialect/SparseTensor/convert_dense2sparse.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s --sparse-tensor-conversion --canonicalize --cse | FileCheck %s
+// RUN: mlir-opt %s --stage-sparse-ops --post-sparsification-rewrite="enable-foreach=false" --canonicalize --cse | FileCheck %s
 
 #SparseVector = #sparse_tensor.encoding<{
   map = (d0) -> (d0 : compressed)
@@ -16,187 +16,45 @@
   map = (d0, d1, d2) -> (d2 : dense, d0 : compressed, d1 : compressed)
 }>
 
-// CHECK-LABEL:   func.func @sparse_convert_1d(
-// CHECK-SAME:      %[[VAL_0:.*]]: tensor<?xi32>) -> !llvm.ptr<i8> {
-// CHECK-DAG:       %[[VAL_1:.*]] = arith.constant 2 : i32
-// CHECK-DAG:       %[[VAL_2:.*]] = arith.constant 4 : i32
-// CHECK-DAG:       %[[VAL_3:.*]] = arith.constant 6 : i32
-// CHECK-DAG:       %[[VAL_4:.*]] = arith.constant 0 : i32
-// CHECK-DAG:       %[[VAL_5:.*]] = arith.constant 1 : index
-// CHECK-DAG:       %[[VAL_6:.*]] = arith.constant 8 : i8
-// CHECK-DAG:       %[[VAL_7:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_8:.*]] = tensor.dim %[[VAL_0]], %[[VAL_7]] : tensor<?xi32>
-// CHECK:           %[[VAL_9:.*]] = memref.alloca() : memref<1xi8>
-// CHECK:           %[[VAL_10:.*]] = memref.cast %[[VAL_9]] : memref<1xi8> to memref<?xi8>
-// CHECK:           memref.store %[[VAL_6]], %[[VAL_9]]{{\[}}%[[VAL_7]]] : memref<1xi8>
-// CHECK:           %[[VAL_11:.*]] = memref.alloca() : memref<1xindex>
-// CHECK:           %[[VAL_12:.*]] = memref.cast %[[VAL_11]] : memref<1xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_8]], %[[VAL_11]]{{\[}}%[[VAL_7]]] : memref<1xindex>
-// CHECK:           %[[VAL_13:.*]] = memref.alloca() : memref<1xindex>
-// CHECK:           %[[VAL_14:.*]] = memref.cast %[[VAL_13]] : memref<1xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_7]], %[[VAL_13]]{{\[}}%[[VAL_7]]] : memref<1xindex>
-// CHECK:           %[[VAL_15:.*]] = llvm.mlir.zero : !llvm.ptr<i8>
-// CHECK:           %[[VAL_16:.*]] = call @newSparseTensor(%[[VAL_12]], %[[VAL_12]], %[[VAL_10]], %[[VAL_14]], %[[VAL_14]], %[[VAL_4]], %[[VAL_4]], %[[VAL_3]], %[[VAL_2]], %[[VAL_15]]) : (memref<?xindex>, memref<?xindex>, memref<?xi8>, memref<?xindex>, memref<?xindex>, i32, i32, i32, i32, !llvm.ptr<i8>) -> !llvm.ptr<i8>
-// CHECK:           %[[VAL_17:.*]] = memref.alloca() : memref<1xindex>
-// CHECK:           %[[VAL_18:.*]] = memref.cast %[[VAL_17]] : memref<1xindex> to memref<?xindex>
-// CHECK:           %[[VAL_19:.*]] = memref.alloca() : memref<i32>
-// CHECK:           scf.for %[[VAL_20:.*]] = %[[VAL_7]] to %[[VAL_8]] step %[[VAL_5]] {
-// CHECK:             %[[VAL_21:.*]] = tensor.extract %[[VAL_0]]{{\[}}%[[VAL_20]]] : tensor<?xi32>
-// CHECK:             %[[VAL_22:.*]] = arith.cmpi ne, %[[VAL_21]], %[[VAL_4]] : i32
-// CHECK:             scf.if %[[VAL_22]] {
-// CHECK:               memref.store %[[VAL_20]], %[[VAL_17]]{{\[}}%[[VAL_7]]] : memref<1xindex>
-// CHECK:               memref.store %[[VAL_21]], %[[VAL_19]][] : memref<i32>
-// CHECK:               %[[VAL_23:.*]] = func.call @addEltI32(%[[VAL_16]], %[[VAL_19]], %[[VAL_18]], %[[VAL_14]]) : (!llvm.ptr<i8>, memref<i32>, memref<?xindex>, memref<?xindex>) -> !llvm.ptr<i8>
-// CHECK:             }
-// CHECK:           }
-// CHECK:           %[[VAL_24:.*]] = call @newSparseTensor(%[[VAL_12]], %[[VAL_12]], %[[VAL_10]], %[[VAL_14]], %[[VAL_14]], %[[VAL_4]], %[[VAL_4]], %[[VAL_3]], %[[VAL_1]], %[[VAL_16]]) : (memref<?xindex>, memref<?xindex>, memref<?xi8>, memref<?xindex>, memref<?xindex>, i32, i32, i32, i32, !llvm.ptr<i8>) -> !llvm.ptr<i8>
-// CHECK:           call @delSparseTensorCOOI32(%[[VAL_16]]) : (!llvm.ptr<i8>) -> ()
-// CHECK:           return %[[VAL_24]] : !llvm.ptr<i8>
-// CHECK:         }
+// CHECK-LABEL:   func.func @sparse_convert_1d
+// CHECK:           sparse_tensor.foreach
+// CHECK:            scf.if
+// CHECK:              sparse_tensor.insert
+// CHECK-NOT:       sparse_tensor.reorder_coo
+// CHECK:           sparse_tensor.load
 func.func @sparse_convert_1d(%arg0: tensor<?xi32>) -> tensor<?xi32, #SparseVector> {
   %0 = sparse_tensor.convert %arg0 : tensor<?xi32> to tensor<?xi32, #SparseVector>
   return %0 : tensor<?xi32, #SparseVector>
 }
 
-// CHECK-LABEL:   func.func @sparse_convert_complex(
-// CHECK-SAME:      %[[VAL_0:.*]]: tensor<100xcomplex<f64>>) -> !llvm.ptr<i8> {
-// CHECK-DAG:       %[[VAL_1:.*]] = arith.constant 2 : i32
-// CHECK-DAG:       %[[VAL_2:.*]] = complex.constant [0.000000e+00, 0.000000e+00] : complex<f64>
-// CHECK-DAG:       %[[VAL_3:.*]] = arith.constant 4 : i32
-// CHECK-DAG:       %[[VAL_4:.*]] = arith.constant 9 : i32
-// CHECK-DAG:       %[[VAL_5:.*]] = arith.constant 0 : i32
-// CHECK-DAG:       %[[VAL_6:.*]] = arith.constant 0 : index
-// CHECK-DAG:       %[[VAL_7:.*]] = arith.constant 100 : index
-// CHECK-DAG:       %[[VAL_8:.*]] = arith.constant 8 : i8
-// CHECK-DAG:       %[[VAL_9:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_10:.*]] = memref.alloca() : memref<1xi8>
-// CHECK:           %[[VAL_11:.*]] = memref.cast %[[VAL_10]] : memref<1xi8> to memref<?xi8>
-// CHECK:           memref.store %[[VAL_8]], %[[VAL_10]]{{\[}}%[[VAL_6]]] : memref<1xi8>
-// CHECK:           %[[VAL_12:.*]] = memref.alloca() : memref<1xindex>
-// CHECK:           %[[VAL_13:.*]] = memref.cast %[[VAL_12]] : memref<1xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_7]], %[[VAL_12]]{{\[}}%[[VAL_6]]] : memref<1xindex>
-// CHECK:           %[[VAL_14:.*]] = memref.alloca() : memref<1xindex>
-// CHECK:           %[[VAL_15:.*]] = memref.cast %[[VAL_14]] : memref<1xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_6]], %[[VAL_14]]{{\[}}%[[VAL_6]]] : memref<1xindex>
-// CHECK:           %[[VAL_16:.*]] = llvm.mlir.zero : !llvm.ptr<i8>
-// CHECK:           %[[VAL_17:.*]] = call @newSparseTensor(%[[VAL_13]], %[[VAL_13]], %[[VAL_11]], %[[VAL_15]], %[[VAL_15]], %[[VAL_5]], %[[VAL_5]], %[[VAL_4]], %[[VAL_3]], %[[VAL_16]]) : (memref<?xindex>, memref<?xindex>, memref<?xi8>, memref<?xindex>, memref<?xindex>, i32, i32, i32, i32, !llvm.ptr<i8>) -> !llvm.ptr<i8>
-// CHECK:           %[[VAL_18:.*]] = memref.alloca() : memref<1xindex>
-// CHECK:           %[[VAL_19:.*]] = memref.cast %[[VAL_18]] : memref<1xindex> to memref<?xindex>
-// CHECK:           %[[VAL_20:.*]] = memref.alloca() : memref<complex<f64>>
-// CHECK:           scf.for %[[VAL_21:.*]] = %[[VAL_6]] to %[[VAL_7]] step %[[VAL_9]] {
-// CHECK:             %[[VAL_22:.*]] = tensor.extract %[[VAL_0]]{{\[}}%[[VAL_21]]] : tensor<100xcomplex<f64>>
-// CHECK:             %[[VAL_23:.*]] = complex.neq %[[VAL_22]], %[[VAL_2]] : complex<f64>
-// CHECK:             scf.if %[[VAL_23]] {
-// CHECK:               memref.store %[[VAL_21]], %[[VAL_18]]{{\[}}%[[VAL_6]]] : memref<1xindex>
-// CHECK:               memref.store %[[VAL_22]], %[[VAL_20]][] : memref<complex<f64>>
-// CHECK:               %[[VAL_24:.*]] = func.call @addEltC64(%[[VAL_17]], %[[VAL_20]], %[[VAL_19]], %[[VAL_15]]) : (!llvm.ptr<i8>, memref<complex<f64>>, memref<?xindex>, memref<?xindex>) -> !llvm.ptr<i8>
-// CHECK:             }
-// CHECK:           }
-// CHECK:           %[[VAL_25:.*]] = call @newSparseTensor(%[[VAL_13]], %[[VAL_13]], %[[VAL_11]], %[[VAL_15]], %[[VAL_15]], %[[VAL_5]], %[[VAL_5]], %[[VAL_4]], %[[VAL_1]], %[[VAL_17]]) : (memref<?xindex>, memref<?xindex>, memref<?xi8>, memref<?xindex>, memref<?xindex>, i32, i32, i32, i32, !llvm.ptr<i8>) -> !llvm.ptr<i8>
-// CHECK:           call @delSparseTensorCOOC64(%[[VAL_17]]) : (!llvm.ptr<i8>) -> ()
-// CHECK:           return %[[VAL_25]] : !llvm.ptr<i8>
-// CHECK:         }
+// CHECK-LABEL:   func.func @sparse_convert_complex
+// CHECK:           sparse_tensor.foreach
+// CHECK:            scf.if
+// CHECK:              sparse_tensor.insert
+// CHECK-NOT:       sparse_tensor.reorder_coo
+// CHECK:           sparse_tensor.load
 func.func @sparse_convert_complex(%arg0: tensor<100xcomplex<f64>>) -> tensor<100xcomplex<f64>, #SparseVector> {
   %0 = sparse_tensor.convert %arg0 : tensor<100xcomplex<f64>> to tensor<100xcomplex<f64>, #SparseVector>
   return %0 : tensor<100xcomplex<f64>, #SparseVector>
 }
 
-// CHECK-LABEL:   func.func @sparse_convert_2d(
-// CHECK-SAME:      %[[VAL_0:.*]]: tensor<2x4xf64>) -> !llvm.ptr<i8> {
-// CHECK-DAG:       %[[VAL_1:.*]] = arith.constant 2 : i32
-// CHECK-DAG:       %[[VAL_2:.*]] = arith.constant 0.000000e+00 : f64
-// CHECK-DAG:       %[[VAL_3:.*]] = arith.constant 4 : i32
-// CHECK-DAG:       %[[VAL_4:.*]] = arith.constant 1 : i32
-// CHECK-DAG:       %[[VAL_5:.*]] = arith.constant 0 : i32
-// CHECK-DAG:       %[[VAL_6:.*]] = arith.constant 0 : index
-// CHECK-DAG:       %[[VAL_7:.*]] = arith.constant 2 : index
-// CHECK-DAG:       %[[VAL_8:.*]] = arith.constant 1 : index
-// CHECK-DAG:       %[[VAL_9:.*]] = arith.constant 4 : index
-// CHECK-DAG:       %[[VAL_10:.*]] = arith.constant 4 : i8
-// CHECK-DAG:       %[[VAL_11:.*]] = arith.constant 8 : i8
-// CHECK:           %[[VAL_12:.*]] = memref.alloca() : memref<2xi8>
-// CHECK:           %[[VAL_13:.*]] = memref.cast %[[VAL_12]] : memref<2xi8> to memref<?xi8>
-// CHECK:           memref.store %[[VAL_10]], %[[VAL_12]]{{\[}}%[[VAL_6]]] : memref<2xi8>
-// CHECK:           memref.store %[[VAL_11]], %[[VAL_12]]{{\[}}%[[VAL_8]]] : memref<2xi8>
-// CHECK:           %[[VAL_14:.*]] = memref.alloca() : memref<2xindex>
-// CHECK:           %[[VAL_15:.*]] = memref.cast %[[VAL_14]] : memref<2xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_7]], %[[VAL_14]]{{\[}}%[[VAL_6]]] : memref<2xindex>
-// CHECK:           memref.store %[[VAL_9]], %[[VAL_14]]{{\[}}%[[VAL_8]]] : memref<2xindex>
-// CHECK:           %[[VAL_16:.*]] = memref.alloca() : memref<2xindex>
-// CHECK:           %[[VAL_17:.*]] = memref.cast %[[VAL_16]] : memref<2xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_6]], %[[VAL_16]]{{\[}}%[[VAL_6]]] : memref<2xindex>
-// CHECK:           memref.store %[[VAL_8]], %[[VAL_16]]{{\[}}%[[VAL_8]]] : memref<2xindex>
-// CHECK:           %[[VAL_18:.*]] = llvm.mlir.zero : !llvm.ptr<i8>
-// CHECK:           %[[VAL_19:.*]] = call @newSparseTensor(%[[VAL_15]], %[[VAL_15]], %[[VAL_13]], %[[VAL_17]], %[[VAL_17]], %[[VAL_5]], %[[VAL_5]], %[[VAL_4]], %[[VAL_3]], %[[VAL_18]]) : (memref<?xindex>, memref<?xindex>, memref<?xi8>, memref<?xindex>, memref<?xindex>, i32, i32, i32, i32, !llvm.ptr<i8>) -> !llvm.ptr<i8>
-// CHECK:           %[[VAL_20:.*]] = memref.alloca() : memref<2xindex>
-// CHECK:           %[[VAL_21:.*]] = memref.cast %[[VAL_20]] : memref<2xindex> to memref<?xindex>
-// CHECK:           %[[VAL_22:.*]] = memref.alloca() : memref<f64>
-// CHECK:           scf.for %[[VAL_23:.*]] = %[[VAL_6]] to %[[VAL_7]] step %[[VAL_8]] {
-// CHECK:             scf.for %[[VAL_24:.*]] = %[[VAL_6]] to %[[VAL_9]] step %[[VAL_8]] {
-// CHECK:               %[[VAL_25:.*]] = tensor.extract %[[VAL_0]]{{\[}}%[[VAL_23]], %[[VAL_24]]] : tensor<2x4xf64>
-// CHECK:               %[[VAL_26:.*]] = arith.cmpf une, %[[VAL_25]], %[[VAL_2]] : f64
-// CHECK:               scf.if %[[VAL_26]] {
-// CHECK:                 memref.store %[[VAL_23]], %[[VAL_20]]{{\[}}%[[VAL_6]]] : memref<2xindex>
-// CHECK:                 memref.store %[[VAL_24]], %[[VAL_20]]{{\[}}%[[VAL_8]]] : memref<2xindex>
-// CHECK:                 memref.store %[[VAL_25]], %[[VAL_22]][] : memref<f64>
-// CHECK:                 %[[VAL_27:.*]] = func.call @addEltF64(%[[VAL_19]], %[[VAL_22]], %[[VAL_21]], %[[VAL_17]]) : (!llvm.ptr<i8>, memref<f64>, memref<?xindex>, memref<?xindex>) -> !llvm.ptr<i8>
-// CHECK:               }
-// CHECK:             }
-// CHECK:           }
-// CHECK:           %[[VAL_28:.*]] = call @newSparseTensor(%[[VAL_15]], %[[VAL_15]], %[[VAL_13]], %[[VAL_17]], %[[VAL_17]], %[[VAL_5]], %[[VAL_5]], %[[VAL_4]], %[[VAL_1]], %[[VAL_19]]) : (memref<?xindex>, memref<?xindex>, memref<?xi8>, memref<?xindex>, memref<?xindex>, i32, i32, i32, i32, !llvm.ptr<i8>) -> !llvm.ptr<i8>
-// CHECK:           call @delSparseTensorCOOF64(%[[VAL_19]]) : (!llvm.ptr<i8>) -> ()
-// CHECK:           return %[[VAL_28]] : !llvm.ptr<i8>
-// CHECK:         }
+// CHECK-LABEL:   func.func @sparse_convert_2d
+// CHECK:           sparse_tensor.foreach
+// CHECK:            scf.if
+// CHECK:              sparse_tensor.insert
+// CHECK-NOT:       sparse_tensor.reorder_coo
+// CHECK:           sparse_tensor.load
 func.func @sparse_convert_2d(%arg0: tensor<2x4xf64>) -> tensor<2x4xf64, #CSR> {
   %0 = sparse_tensor.convert %arg0 : tensor<2x4xf64> to tensor<2x4xf64, #CSR>
   return %0 : tensor<2x4xf64, #CSR>
 }
 
-// CHECK-LABEL:   func.func @sparse_constant() -> !llvm.ptr<i8> {
-// CHECK-DAG:       %[[VAL_0:.*]] = arith.constant dense<[1.000000e+00, 5.000000e+00]> : tensor<2xf32>
-// CHECK-DAG:       %[[VAL_1:.*]] = arith.constant dense<{{\[\[}}0, 0], [1, 6]]> : tensor<2x2xi64>
-// CHECK-DAG:       %[[VAL_2:.*]] = arith.constant 4 : i32
-// CHECK-DAG:       %[[VAL_3:.*]] = arith.constant 2 : i32
-// CHECK-DAG:       %[[VAL_4:.*]] = arith.constant 0 : i32
-// CHECK-DAG:       %[[VAL_5:.*]] = arith.constant 0 : index
-// CHECK-DAG:       %[[VAL_6:.*]] = arith.constant 8 : index
-// CHECK-DAG:       %[[VAL_7:.*]] = arith.constant 1 : index
-// CHECK-DAG:       %[[VAL_8:.*]] = arith.constant 7 : index
-// CHECK-DAG:       %[[VAL_9:.*]] = arith.constant 4 : i8
-// CHECK-DAG:       %[[VAL_10:.*]] = arith.constant 8 : i8
-// CHECK-DAG:       %[[VAL_11:.*]] = arith.constant 2 : index
-// CHECK:           %[[VAL_12:.*]] = memref.alloca() : memref<2xi8>
-// CHECK:           %[[VAL_13:.*]] = memref.cast %[[VAL_12]] : memref<2xi8> to memref<?xi8>
-// CHECK:           memref.store %[[VAL_9]], %[[VAL_12]]{{\[}}%[[VAL_5]]] : memref<2xi8>
-// CHECK:           memref.store %[[VAL_10]], %[[VAL_12]]{{\[}}%[[VAL_7]]] : memref<2xi8>
-// CHECK:           %[[VAL_14:.*]] = memref.alloca() : memref<2xindex>
-// CHECK:           %[[VAL_15:.*]] = memref.cast %[[VAL_14]] : memref<2xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_6]], %[[VAL_14]]{{\[}}%[[VAL_5]]] : memref<2xindex>
-// CHECK:           memref.store %[[VAL_8]], %[[VAL_14]]{{\[}}%[[VAL_7]]] : memref<2xindex>
-// CHECK:           %[[VAL_16:.*]] = memref.alloca() : memref<2xindex>
-// CHECK:           %[[VAL_17:.*]] = memref.cast %[[VAL_16]] : memref<2xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_5]], %[[VAL_16]]{{\[}}%[[VAL_5]]] : memref<2xindex>
-// CHECK:           memref.store %[[VAL_7]], %[[VAL_16]]{{\[}}%[[VAL_7]]] : memref<2xindex>
-// CHECK:           %[[VAL_18:.*]] = llvm.mlir.zero : !llvm.ptr<i8>
-// CHECK:           %[[VAL_19:.*]] = call @newSparseTensor(%[[VAL_15]], %[[VAL_15]], %[[VAL_13]], %[[VAL_17]], %[[VAL_17]], %[[VAL_4]], %[[VAL_4]], %[[VAL_3]], %[[VAL_2]], %[[VAL_18]]) : (memref<?xindex>, memref<?xindex>, memref<?xi8>, memref<?xindex>, memref<?xindex>, i32, i32, i32, i32, !llvm.ptr<i8>) -> !llvm.ptr<i8>
-// CHECK:           %[[VAL_20:.*]] = memref.alloca() : memref<2xindex>
-// CHECK:           %[[VAL_21:.*]] = memref.cast %[[VAL_20]] : memref<2xindex> to memref<?xindex>
-// CHECK:           %[[VAL_22:.*]] = memref.alloca() : memref<f32>
-// CHECK:           scf.for %[[VAL_23:.*]] = %[[VAL_5]] to %[[VAL_11]] step %[[VAL_7]] {
-// CHECK:             %[[VAL_24:.*]] = tensor.extract %[[VAL_1]]{{\[}}%[[VAL_23]], %[[VAL_5]]] : tensor<2x2xi64>
-// CHECK:             %[[VAL_25:.*]] = arith.index_cast %[[VAL_24]] : i64 to index
-// CHECK:             %[[VAL_26:.*]] = tensor.extract %[[VAL_1]]{{\[}}%[[VAL_23]], %[[VAL_7]]] : tensor<2x2xi64>
-// CHECK:             %[[VAL_27:.*]] = arith.index_cast %[[VAL_26]] : i64 to index
-// CHECK:             %[[VAL_28:.*]] = tensor.extract %[[VAL_0]]{{\[}}%[[VAL_23]]] : tensor<2xf32>
-// CHECK:             memref.store %[[VAL_25]], %[[VAL_20]]{{\[}}%[[VAL_5]]] : memref<2xindex>
-// CHECK:             memref.store %[[VAL_27]], %[[VAL_20]]{{\[}}%[[VAL_7]]] : memref<2xindex>
-// CHECK:             memref.store %[[VAL_28]], %[[VAL_22]][] : memref<f32>
-// CHECK:             %[[VAL_29:.*]] = func.call @addEltF32(%[[VAL_19]], %[[VAL_22]], %[[VAL_21]], %[[VAL_17]]) : (!llvm.ptr<i8>, memref<f32>, memref<?xindex>, memref<?xindex>) -> !llvm.ptr<i8>
-// CHECK:           }
-// CHECK:           %[[VAL_30:.*]] = call @newSparseTensor(%[[VAL_15]], %[[VAL_15]], %[[VAL_13]], %[[VAL_17]], %[[VAL_17]], %[[VAL_4]], %[[VAL_4]], %[[VAL_3]], %[[VAL_3]], %[[VAL_19]]) : (memref<?xindex>, memref<?xindex>, memref<?xi8>, memref<?xindex>, memref<?xindex>, i32, i32, i32, i32, !llvm.ptr<i8>) -> !llvm.ptr<i8>
-// CHECK:           call @delSparseTensorCOOF32(%[[VAL_19]]) : (!llvm.ptr<i8>) -> ()
-// CHECK:           return %[[VAL_30]] : !llvm.ptr<i8>
-// CHECK:         }
+// CHECK-LABEL:   func.func @sparse_constant
+// CHECK:           sparse_tensor.foreach
+// CHECK-NOT:         scf.if
+// CHECK:               sparse_tensor.insert
+// CHECK-NOT:       sparse_tensor.reorder_coo
+// CHECK:           sparse_tensor.load
 func.func @sparse_constant() -> tensor<8x7xf32, #CSR>{
   // Initialize a tensor.
   %0 = arith.constant sparse<[[0, 0], [1, 6]], [1.0, 5.0]> : tensor<8x7xf32>
@@ -205,59 +63,12 @@ func.func @sparse_constant() -> tensor<8x7xf32, #CSR>{
   return %1 : tensor<8x7xf32, #CSR>
 }
 
-// CHECK-LABEL:   func.func @sparse_constant_csc() -> !llvm.ptr<i8> {
-// CHECK-DAG:       %[[VAL_0:.*]] = arith.constant dense<[1.000000e+00, 5.000000e+00]> : tensor<2xf32>
-// CHECK-DAG:       %[[VAL_1:.*]] = arith.constant dense<{{\[\[}}0, 0], [1, 6]]> : tensor<2x2xi64>
-// CHECK-DAG:       %[[VAL_2:.*]] = arith.constant 4 : i32
-// CHECK-DAG:       %[[VAL_3:.*]] = arith.constant 2 : i32
-// CHECK-DAG:       %[[VAL_4:.*]] = arith.constant 0 : i32
-// CHECK-DAG:       %[[VAL_5:.*]] = arith.constant 0 : index
-// CHECK-DAG:       %[[VAL_6:.*]] = arith.constant 8 : index
-// CHECK-DAG:       %[[VAL_7:.*]] = arith.constant 1 : index
-// CHECK-DAG:       %[[VAL_8:.*]] = arith.constant 7 : index
-// CHECK-DAG:       %[[VAL_9:.*]] = arith.constant 4 : i8
-// CHECK-DAG:       %[[VAL_10:.*]] = arith.constant 8 : i8
-// CHECK-DAG:       %[[VAL_11:.*]] = arith.constant 2 : index
-// CHECK:           %[[VAL_12:.*]] = memref.alloca() : memref<2xi8>
-// CHECK:           %[[VAL_13:.*]] = memref.cast %[[VAL_12]] : memref<2xi8> to memref<?xi8>
-// CHECK:           memref.store %[[VAL_9]], %[[VAL_12]]{{\[}}%[[VAL_5]]] : memref<2xi8>
-// CHECK:           memref.store %[[VAL_10]], %[[VAL_12]]{{\[}}%[[VAL_7]]] : memref<2xi8>
-// CHECK:           %[[VAL_14:.*]] = memref.alloca() : memref<2xindex>
-// CHECK:           %[[VAL_15:.*]] = memref.cast %[[VAL_14]] : memref<2xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_6]], %[[VAL_14]]{{\[}}%[[VAL_5]]] : memref<2xindex>
-// CHECK:           memref.store %[[VAL_8]], %[[VAL_14]]{{\[}}%[[VAL_7]]] : memref<2xindex>
-// CHECK:           %[[VAL_16:.*]] = memref.alloca() : memref<2xindex>
-// CHECK:           %[[VAL_17:.*]] = memref.cast %[[VAL_16]] : memref<2xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_7]], %[[VAL_16]]{{\[}}%[[VAL_5]]] : memref<2xindex>
-// CHECK:           memref.store %[[VAL_5]], %[[VAL_16]]{{\[}}%[[VAL_7]]] : memref<2xindex>
-// CHECK:           %[[VAL_18:.*]] = memref.alloca() : memref<2xindex>
-// CHECK:           %[[VAL_19:.*]] = memref.cast %[[VAL_18]] : memref<2xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_7]], %[[VAL_18]]{{\[}}%[[VAL_5]]] : memref<2xindex>
-// CHECK:           memref.store %[[VAL_5]], %[[VAL_18]]{{\[}}%[[VAL_7]]] : memref<2xindex>
-// CHECK:           %[[VAL_20:.*]] = memref.alloca() : memref<2xindex>
-// CHECK:           %[[VAL_21:.*]] = memref.cast %[[VAL_20]] : memref<2xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_8]], %[[VAL_20]]{{\[}}%[[VAL_5]]] : memref<2xindex>
-// CHECK:           memref.store %[[VAL_6]], %[[VAL_20]]{{\[}}%[[VAL_7]]] : memref<2xindex>
-// CHECK:           %[[VAL_22:.*]] = llvm.mlir.zero : !llvm.ptr<i8>
-// CHECK:           %[[VAL_23:.*]] = call @newSparseTensor(%[[VAL_15]], %[[VAL_21]], %[[VAL_13]], %[[VAL_17]], %[[VAL_19]], %[[VAL_4]], %[[VAL_4]], %[[VAL_3]], %[[VAL_2]], %[[VAL_22]]) : (memref<?xindex>, memref<?xindex>, memref<?xi8>, memref<?xindex>, memref<?xindex>, i32, i32, i32, i32, !llvm.ptr<i8>) -> !llvm.ptr<i8>
-// CHECK:           %[[VAL_24:.*]] = memref.alloca() : memref<2xindex>
-// CHECK:           %[[VAL_25:.*]] = memref.cast %[[VAL_24]] : memref<2xindex> to memref<?xindex>
-// CHECK:           %[[VAL_26:.*]] = memref.alloca() : memref<f32>
-// CHECK:           scf.for %[[VAL_27:.*]] = %[[VAL_5]] to %[[VAL_11]] step %[[VAL_7]] {
-// CHECK:             %[[VAL_28:.*]] = tensor.extract %[[VAL_1]]{{\[}}%[[VAL_27]], %[[VAL_5]]] : tensor<2x2xi64>
-// CHECK:             %[[VAL_29:.*]] = arith.index_cast %[[VAL_28]] : i64 to index
-// CHECK:             %[[VAL_30:.*]] = tensor.extract %[[VAL_1]]{{\[}}%[[VAL_27]], %[[VAL_7]]] : tensor<2x2xi64>
-// CHECK:             %[[VAL_31:.*]] = arith.index_cast %[[VAL_30]] : i64 to index
-// CHECK:             %[[VAL_32:.*]] = tensor.extract %[[VAL_0]]{{\[}}%[[VAL_27]]] : tensor<2xf32>
-// CHECK:             memref.store %[[VAL_29]], %[[VAL_24]]{{\[}}%[[VAL_5]]] : memref<2xindex>
-// CHECK:             memref.store %[[VAL_31]], %[[VAL_24]]{{\[}}%[[VAL_7]]] : memref<2xindex>
-// CHECK:             memref.store %[[VAL_32]], %[[VAL_26]][] : memref<f32>
-// CHECK:             %[[VAL_33:.*]] = func.call @addEltF32(%[[VAL_23]], %[[VAL_26]], %[[VAL_25]], %[[VAL_17]]) : (!llvm.ptr<i8>, memref<f32>, memref<?xindex>, memref<?xindex>) -> !llvm.ptr<i8>
-// CHECK:           }
-// CHECK:           %[[VAL_34:.*]] = call @newSparseTensor(%[[VAL_15]], %[[VAL_21]], %[[VAL_13]], %[[VAL_17]], %[[VAL_19]], %[[VAL_4]], %[[VAL_4]], %[[VAL_3]], %[[VAL_3]], %[[VAL_23]]) : (memref<?xindex>, memref<?xindex>, memref<?xi8>, memref<?xindex>, memref<?xindex>, i32, i32, i32, i32, !llvm.ptr<i8>) -> !llvm.ptr<i8>
-// CHECK:           call @delSparseTensorCOOF32(%[[VAL_23]]) : (!llvm.ptr<i8>) -> ()
-// CHECK:           return %[[VAL_34]] : !llvm.ptr<i8>
-// CHECK:         }
+// CHECK-LABEL:   func.func @sparse_constant_csc
+// CHECK:           sparse_tensor.foreach
+// CHECK-NOT:         scf.if
+// CHECK:               sparse_tensor.insert
+// CHECK-NOT:       sparse_tensor.reorder_coo
+// CHECK:           sparse_tensor.load
 func.func @sparse_constant_csc() -> tensor<8x7xf32, #CSC>{
   // Initialize a tensor.
   %0 = arith.constant sparse<[[0, 0], [1, 6]], [1.0, 5.0]> : tensor<8x7xf32>
@@ -266,73 +77,15 @@ func.func @sparse_constant_csc() -> tensor<8x7xf32, #CSC>{
   return %1 : tensor<8x7xf32, #CSC>
 }
 
-// CHECK-LABEL:   func.func @sparse_convert_3d(
-// CHECK-SAME:      %[[VAL_0:.*]]: tensor<?x?x?xf64>) -> !llvm.ptr<i8> {
-// CHECK-DAG:       %[[VAL_1:.*]] = arith.constant 2 : i32
-// CHECK-DAG:       %[[VAL_2:.*]] = arith.constant 0.000000e+00 : f64
-// CHECK-DAG:       %[[VAL_3:.*]] = arith.constant 4 : i32
-// CHECK-DAG:       %[[VAL_4:.*]] = arith.constant 1 : i32
-// CHECK-DAG:       %[[VAL_5:.*]] = arith.constant 0 : i32
-// CHECK-DAG:       %[[VAL_6:.*]] = arith.constant 8 : i8
-// CHECK-DAG:       %[[VAL_7:.*]] = arith.constant 4 : i8
-// CHECK-DAG:       %[[VAL_8:.*]] = arith.constant 2 : index
-// CHECK-DAG:       %[[VAL_9:.*]] = arith.constant 1 : index
-// CHECK-DAG:       %[[VAL_10:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_11:.*]] = tensor.dim %[[VAL_0]], %[[VAL_10]] : tensor<?x?x?xf64>
-// CHECK:           %[[VAL_12:.*]] = tensor.dim %[[VAL_0]], %[[VAL_9]] : tensor<?x?x?xf64>
-// CHECK:           %[[VAL_13:.*]] = tensor.dim %[[VAL_0]], %[[VAL_8]] : tensor<?x?x?xf64>
-// CHECK:           %[[VAL_14:.*]] = memref.alloca() : memref<3xi8>
-// CHECK:           %[[VAL_15:.*]] = memref.cast %[[VAL_14]] : memref<3xi8> to memref<?xi8>
-// CHECK:           memref.store %[[VAL_7]], %[[VAL_14]]{{\[}}%[[VAL_10]]] : memref<3xi8>
-// CHECK:           memref.store %[[VAL_6]], %[[VAL_14]]{{\[}}%[[VAL_9]]] : memref<3xi8>
-// CHECK:           memref.store %[[VAL_6]], %[[VAL_14]]{{\[}}%[[VAL_8]]] : memref<3xi8>
-// CHECK:           %[[VAL_16:.*]] = memref.alloca() : memref<3xindex>
-// CHECK:           %[[VAL_17:.*]] = memref.cast %[[VAL_16]] : memref<3xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_11]], %[[VAL_16]]{{\[}}%[[VAL_10]]] : memref<3xindex>
-// CHECK:           memref.store %[[VAL_12]], %[[VAL_16]]{{\[}}%[[VAL_9]]] : memref<3xindex>
-// CHECK:           memref.store %[[VAL_13]], %[[VAL_16]]{{\[}}%[[VAL_8]]] : memref<3xindex>
-// CHECK:           %[[VAL_18:.*]] = memref.load %[[VAL_16]]{{\[}}%[[VAL_8]]] : memref<3xindex>
-// CHECK:           %[[VAL_19:.*]] = memref.load %[[VAL_16]]{{\[}}%[[VAL_10]]] : memref<3xindex>
-// CHECK:           %[[VAL_20:.*]] = memref.load %[[VAL_16]]{{\[}}%[[VAL_9]]] : memref<3xindex>
-// CHECK:           %[[VAL_21:.*]] = memref.alloca() : memref<3xindex>
-// CHECK:           %[[VAL_22:.*]] = memref.cast %[[VAL_21]] : memref<3xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_9]], %[[VAL_21]]{{\[}}%[[VAL_10]]] : memref<3xindex>
-// CHECK:           memref.store %[[VAL_8]], %[[VAL_21]]{{\[}}%[[VAL_9]]] : memref<3xindex>
-// CHECK:           memref.store %[[VAL_10]], %[[VAL_21]]{{\[}}%[[VAL_8]]] : memref<3xindex>
-// CHECK:           %[[VAL_23:.*]] = memref.alloca() : memref<3xindex>
-// CHECK:           %[[VAL_24:.*]] = memref.cast %[[VAL_23]] : memref<3xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_8]], %[[VAL_23]]{{\[}}%[[VAL_10]]] : memref<3xindex>
-// CHECK:           memref.store %[[VAL_10]], %[[VAL_23]]{{\[}}%[[VAL_9]]] : memref<3xindex>
-// CHECK:           memref.store %[[VAL_9]], %[[VAL_23]]{{\[}}%[[VAL_8]]] : memref<3xindex>
-// CHECK:           %[[VAL_25:.*]] = memref.alloca() : memref<3xindex>
-// CHECK:           %[[VAL_26:.*]] = memref.cast %[[VAL_25]] : memref<3xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_18]], %[[VAL_25]]{{\[}}%[[VAL_10]]] : memref<3xindex>
-// CHECK:           memref.store %[[VAL_19]], %[[VAL_25]]{{\[}}%[[VAL_9]]] : memref<3xindex>
-// CHECK:           memref.store %[[VAL_20]], %[[VAL_25]]{{\[}}%[[VAL_8]]] : memref<3xindex>
-// CHECK:           %[[VAL_27:.*]] = llvm.mlir.zero : !llvm.ptr<i8>
-// CHECK:           %[[VAL_28:.*]] = call @newSparseTensor(%[[VAL_17]], %[[VAL_26]], %[[VAL_15]], %[[VAL_22]], %[[VAL_24]], %[[VAL_5]], %[[VAL_5]], %[[VAL_4]], %[[VAL_3]], %[[VAL_27]]) : (memref<?xindex>, memref<?xindex>, memref<?xi8>, memref<?xindex>, memref<?xindex>, i32, i32, i32, i32, !llvm.ptr<i8>) -> !llvm.ptr<i8>
-// CHECK:           %[[VAL_29:.*]] = memref.alloca() : memref<3xindex>
-// CHECK:           %[[VAL_30:.*]] = memref.cast %[[VAL_29]] : memref<3xindex> to memref<?xindex>
-// CHECK:           %[[VAL_31:.*]] = memref.alloca() : memref<f64>
-// CHECK:           scf.for %[[VAL_32:.*]] = %[[VAL_10]] to %[[VAL_11]] step %[[VAL_9]] {
-// CHECK:             scf.for %[[VAL_33:.*]] = %[[VAL_10]] to %[[VAL_12]] step %[[VAL_9]] {
-// CHECK:               scf.for %[[VAL_34:.*]] = %[[VAL_10]] to %[[VAL_13]] step %[[VAL_9]] {
-// CHECK:                 %[[VAL_35:.*]] = tensor.extract %[[VAL_0]]{{\[}}%[[VAL_32]], %[[VAL_33]], %[[VAL_34]]] : tensor<?x?x?xf64>
-// CHECK:                 %[[VAL_36:.*]] = arith.cmpf une, %[[VAL_35]], %[[VAL_2]] : f64
-// CHECK:                 scf.if %[[VAL_36]] {
-// CHECK:                   memref.store %[[VAL_32]], %[[VAL_29]]{{\[}}%[[VAL_10]]] : memref<3xindex>
-// CHECK:                   memref.store %[[VAL_33]], %[[VAL_29]]{{\[}}%[[VAL_9]]] : memref<3xindex>
-// CHECK:                   memref.store %[[VAL_34]], %[[VAL_29]]{{\[}}%[[VAL_8]]] : memref<3xindex>
-// CHECK:                   memref.store %[[VAL_35]], %[[VAL_31]][] : memref<f64>
-// CHECK:                   %[[VAL_37:.*]] = func.call @addEltF64(%[[VAL_28]], %[[VAL_31]], %[[VAL_30]], %[[VAL_22]]) : (!llvm.ptr<i8>, memref<f64>, memref<?xindex>, memref<?xindex>) -> !llvm.ptr<i8>
-// CHECK:                 }
-// CHECK:               }
-// CHECK:             }
-// CHECK:           }
-// CHECK:           %[[VAL_38:.*]] = call @newSparseTensor(%[[VAL_17]], %[[VAL_26]], %[[VAL_15]], %[[VAL_22]], %[[VAL_24]], %[[VAL_5]], %[[VAL_5]], %[[VAL_4]], %[[VAL_1]], %[[VAL_28]]) : (memref<?xindex>, memref<?xindex>, memref<?xi8>, memref<?xindex>, memref<?xindex>, i32, i32, i32, i32, !llvm.ptr<i8>) -> !llvm.ptr<i8>
-// CHECK:           call @delSparseTensorCOOF64(%[[VAL_28]]) : (!llvm.ptr<i8>) -> ()
-// CHECK:           return %[[VAL_38]] : !llvm.ptr<i8>
-// CHECK:         }
+// CHECK-LABEL:   func.func @sparse_convert_3d
+// CHECK:           sparse_tensor.foreach
+// CHECK:             scf.if
+// CHECK:               sparse_tensor.insert
+// CHECK:           sparse_tensor.load
+// CHECK:           sparse_tensor.reorder_coo
+// CHECK:           sparse_tensor.foreach
+// CHECK:             sparse_tensor.insert
+// CHECK:           sparse_tensor.load
 func.func @sparse_convert_3d(%arg0: tensor<?x?x?xf64>) -> tensor<?x?x?xf64, #SparseTensor> {
   %0 = sparse_tensor.convert %arg0 : tensor<?x?x?xf64> to tensor<?x?x?xf64, #SparseTensor>
   return %0 : tensor<?x?x?xf64, #SparseTensor>
diff --git a/mlir/test/Dialect/SparseTensor/convert_sparse2dense.mlir b/mlir/test/Dialect/SparseTensor/convert_sparse2dense.mlir
index ffc0f57a23..e2dcb068e1 100644
--- a/mlir/test/Dialect/SparseTensor/convert_sparse2dense.mlir
+++ b/mlir/test/Dialect/SparseTensor/convert_sparse2dense.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s --sparse-tensor-conversion --canonicalize --cse | FileCheck %s
+// RUN: mlir-opt %s --stage-sparse-ops --post-sparsification-rewrite="enable-foreach=false" --canonicalize --cse | FileCheck %s
 
 #SparseVector = #sparse_tensor.encoding<{
   map = (d0) -> (d0 : compressed)
@@ -12,326 +12,78 @@
   map = (d0, d1, d2) -> (d2 : dense, d0 : compressed, d1 : compressed)
 }>
 
-// CHECK-LABEL:   func.func @sparse_convert_1d(
-// CHECK-SAME:      %[[VAL_0:.*]]: !llvm.ptr<i8>) -> tensor<13xi32> {
-// CHECK-DAG:       %[[VAL_1:.*]] = arith.constant 6 : i32
-// CHECK-DAG:       %[[VAL_2:.*]] = arith.constant 0 : i32
-// CHECK-DAG:       %[[VAL_3:.*]] = arith.constant 0 : index
-// CHECK-DAG:       %[[VAL_4:.*]] = arith.constant 13 : index
-// CHECK-DAG:       %[[VAL_5:.*]] = arith.constant 4 : i8
-// CHECK:           %[[VAL_6:.*]] = memref.alloca() : memref<1xi8>
-// CHECK:           %[[VAL_7:.*]] = memref.cast %[[VAL_6]] : memref<1xi8> to memref<?xi8>
-// CHECK:           memref.store %[[VAL_5]], %[[VAL_6]]{{\[}}%[[VAL_3]]] : memref<1xi8>
-// CHECK:           %[[VAL_8:.*]] = memref.alloca() : memref<1xindex>
-// CHECK:           %[[VAL_9:.*]] = memref.cast %[[VAL_8]] : memref<1xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_4]], %[[VAL_8]]{{\[}}%[[VAL_3]]] : memref<1xindex>
-// CHECK:           %[[VAL_10:.*]] = memref.alloca() : memref<1xindex>
-// CHECK:           %[[VAL_11:.*]] = memref.cast %[[VAL_10]] : memref<1xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_3]], %[[VAL_10]]{{\[}}%[[VAL_3]]] : memref<1xindex>
-// CHECK:           %[[VAL_12:.*]] = call @newSparseTensor(%[[VAL_9]], %[[VAL_9]], %[[VAL_7]], %[[VAL_11]], %[[VAL_11]], %[[VAL_2]], %[[VAL_2]], %[[VAL_1]], %[[VAL_1]], %[[VAL_0]]) : (memref<?xindex>, memref<?xindex>, memref<?xi8>, memref<?xindex>, memref<?xindex>, i32, i32, i32, i32, !llvm.ptr<i8>) -> !llvm.ptr<i8>
-// CHECK:           %[[VAL_13:.*]] = memref.alloca() : memref<1xindex>
-// CHECK:           %[[VAL_14:.*]] = memref.cast %[[VAL_13]] : memref<1xindex> to memref<?xindex>
-// CHECK:           %[[VAL_15:.*]] = memref.alloca() : memref<i32>
-// CHECK:           %[[VAL_16:.*]] = memref.alloc() : memref<13xi32>
-// CHECK:           linalg.fill ins(%[[VAL_2]] : i32) outs(%[[VAL_16]] : memref<13xi32>)
-// CHECK:           scf.while : () -> () {
-// CHECK:             %[[VAL_17:.*]] = func.call @getNextI32(%[[VAL_12]], %[[VAL_14]], %[[VAL_15]]) : (!llvm.ptr<i8>, memref<?xindex>, memref<i32>) -> i1
-// CHECK:             scf.condition(%[[VAL_17]])
-// CHECK:           } do {
-// CHECK:             %[[VAL_18:.*]] = memref.load %[[VAL_13]]{{\[}}%[[VAL_3]]] : memref<1xindex>
-// CHECK:             %[[VAL_19:.*]] = memref.load %[[VAL_15]][] : memref<i32>
-// CHECK:             memref.store %[[VAL_19]], %[[VAL_16]]{{\[}}%[[VAL_18]]] : memref<13xi32>
-// CHECK:             scf.yield
-// CHECK:           }
-// CHECK:           call @delSparseTensorIteratorI32(%[[VAL_12]]) : (!llvm.ptr<i8>) -> ()
-// CHECK:           %[[VAL_20:.*]] = bufferization.to_tensor %[[VAL_16]] : memref<13xi32>
-// CHECK:           return %[[VAL_20]] : tensor<13xi32>
-// CHECK:         }
+// CHECK-LABEL:  func.func @sparse_convert_1d
+// CHECK-NOT:      sparse_tensor.reorder_coo
+// CHECK:          bufferization.alloc_tensor
+// CHECK:          linalg.fill
+// CHECK:          sparse_tensor.foreach
+// CHECK:            tensor.insert
 func.func @sparse_convert_1d(%arg0: tensor<13xi32, #SparseVector>) -> tensor<13xi32> {
   %0 = sparse_tensor.convert %arg0 : tensor<13xi32, #SparseVector> to tensor<13xi32>
   return %0 : tensor<13xi32>
 }
 
-// CHECK-LABEL:   func.func @sparse_convert_1d_dyn(
-// CHECK-SAME:      %[[VAL_0:.*]]: !llvm.ptr<i8>) -> tensor<?xi32> {
-// CHECK-DAG:       %[[VAL_1:.*]] = arith.constant 6 : i32
-// CHECK-DAG:       %[[VAL_2:.*]] = arith.constant 0 : i32
-// CHECK-DAG:       %[[VAL_3:.*]] = arith.constant 4 : i8
-// CHECK-DAG:       %[[VAL_4:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_5:.*]] = call @sparseDimSize(%[[VAL_0]], %[[VAL_4]]) : (!llvm.ptr<i8>, index) -> index
-// CHECK:           %[[VAL_6:.*]] = memref.alloca() : memref<1xi8>
-// CHECK:           %[[VAL_7:.*]] = memref.cast %[[VAL_6]] : memref<1xi8> to memref<?xi8>
-// CHECK:           memref.store %[[VAL_3]], %[[VAL_6]]{{\[}}%[[VAL_4]]] : memref<1xi8>
-// CHECK:           %[[VAL_8:.*]] = memref.alloca() : memref<1xindex>
-// CHECK:           %[[VAL_9:.*]] = memref.cast %[[VAL_8]] : memref<1xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_5]], %[[VAL_8]]{{\[}}%[[VAL_4]]] : memref<1xindex>
-// CHECK:           %[[VAL_10:.*]] = memref.alloca() : memref<1xindex>
-// CHECK:           %[[VAL_11:.*]] = memref.cast %[[VAL_10]] : memref<1xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_4]], %[[VAL_10]]{{\[}}%[[VAL_4]]] : memref<1xindex>
-// CHECK:           %[[VAL_12:.*]] = call @newSparseTensor(%[[VAL_9]], %[[VAL_9]], %[[VAL_7]], %[[VAL_11]], %[[VAL_11]], %[[VAL_2]], %[[VAL_2]], %[[VAL_1]], %[[VAL_1]], %[[VAL_0]]) : (memref<?xindex>, memref<?xindex>, memref<?xi8>, memref<?xindex>, memref<?xindex>, i32, i32, i32, i32, !llvm.ptr<i8>) -> !llvm.ptr<i8>
-// CHECK:           %[[VAL_13:.*]] = memref.alloca() : memref<1xindex>
-// CHECK:           %[[VAL_14:.*]] = memref.cast %[[VAL_13]] : memref<1xindex> to memref<?xindex>
-// CHECK:           %[[VAL_15:.*]] = memref.alloca() : memref<i32>
-// CHECK:           %[[VAL_16:.*]] = memref.alloc(%[[VAL_5]]) : memref<?xi32>
-// CHECK:           linalg.fill ins(%[[VAL_2]] : i32) outs(%[[VAL_16]] : memref<?xi32>)
-// CHECK:           scf.while : () -> () {
-// CHECK:             %[[VAL_17:.*]] = func.call @getNextI32(%[[VAL_12]], %[[VAL_14]], %[[VAL_15]]) : (!llvm.ptr<i8>, memref<?xindex>, memref<i32>) -> i1
-// CHECK:             scf.condition(%[[VAL_17]])
-// CHECK:           } do {
-// CHECK:             %[[VAL_18:.*]] = memref.load %[[VAL_13]]{{\[}}%[[VAL_4]]] : memref<1xindex>
-// CHECK:             %[[VAL_19:.*]] = memref.load %[[VAL_15]][] : memref<i32>
-// CHECK:             memref.store %[[VAL_19]], %[[VAL_16]]{{\[}}%[[VAL_18]]] : memref<?xi32>
-// CHECK:             scf.yield
-// CHECK:           }
-// CHECK:           call @delSparseTensorIteratorI32(%[[VAL_12]]) : (!llvm.ptr<i8>) -> ()
-// CHECK:           %[[VAL_20:.*]] = bufferization.to_tensor %[[VAL_16]] : memref<?xi32>
-// CHECK:           return %[[VAL_20]] : tensor<?xi32>
-// CHECK:         }
+// CHECK-LABEL:  func.func @sparse_convert_1d_dyn
+// CHECK-NOT:      sparse_tensor.reorder_coo
+// CHECK:          bufferization.alloc_tensor
+// CHECK:          linalg.fill
+// CHECK:          sparse_tensor.foreach
+// CHECK:            tensor.insert
 func.func @sparse_convert_1d_dyn(%arg0: tensor<?xi32, #SparseVector>) -> tensor<?xi32> {
   %0 = sparse_tensor.convert %arg0 : tensor<?xi32, #SparseVector> to tensor<?xi32>
   return %0 : tensor<?xi32>
 }
 
-// CHECK-LABEL:   func.func @sparse_convert_2d(
-// CHECK-SAME:      %[[VAL_0:.*]]: !llvm.ptr<i8>) -> tensor<2x4xf64> {
-// CHECK-DAG:       %[[VAL_1:.*]] = arith.constant 0.000000e+00 : f64
-// CHECK-DAG:       %[[VAL_2:.*]] = arith.constant 6 : i32
-// CHECK-DAG:       %[[VAL_3:.*]] = arith.constant 1 : i32
-// CHECK-DAG:       %[[VAL_4:.*]] = arith.constant 0 : i32
-// CHECK-DAG:       %[[VAL_5:.*]] = arith.constant 1 : index
-// CHECK-DAG:       %[[VAL_6:.*]] = arith.constant 0 : index
-// CHECK-DAG:       %[[VAL_7:.*]] = arith.constant 2 : index
-// CHECK-DAG:       %[[VAL_8:.*]] = arith.constant 4 : index
-// CHECK-DAG:       %[[VAL_9:.*]] = arith.constant 4 : i8
-// CHECK:           %[[VAL_10:.*]] = memref.alloca() : memref<2xi8>
-// CHECK:           %[[VAL_11:.*]] = memref.cast %[[VAL_10]] : memref<2xi8> to memref<?xi8>
-// CHECK:           memref.store %[[VAL_9]], %[[VAL_10]]{{\[}}%[[VAL_6]]] : memref<2xi8>
-// CHECK:           memref.store %[[VAL_9]], %[[VAL_10]]{{\[}}%[[VAL_5]]] : memref<2xi8>
-// CHECK:           %[[VAL_12:.*]] = memref.alloca() : memref<2xindex>
-// CHECK:           %[[VAL_13:.*]] = memref.cast %[[VAL_12]] : memref<2xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_7]], %[[VAL_12]]{{\[}}%[[VAL_6]]] : memref<2xindex>
-// CHECK:           memref.store %[[VAL_8]], %[[VAL_12]]{{\[}}%[[VAL_5]]] : memref<2xindex>
-// CHECK:           %[[VAL_14:.*]] = memref.alloca() : memref<2xindex>
-// CHECK:           %[[VAL_15:.*]] = memref.cast %[[VAL_14]] : memref<2xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_6]], %[[VAL_14]]{{\[}}%[[VAL_6]]] : memref<2xindex>
-// CHECK:           memref.store %[[VAL_5]], %[[VAL_14]]{{\[}}%[[VAL_5]]] : memref<2xindex>
-// CHECK:           %[[VAL_16:.*]] = call @newSparseTensor(%[[VAL_13]], %[[VAL_13]], %[[VAL_11]], %[[VAL_15]], %[[VAL_15]], %[[VAL_4]], %[[VAL_4]], %[[VAL_3]], %[[VAL_2]], %[[VAL_0]]) : (memref<?xindex>, memref<?xindex>, memref<?xi8>, memref<?xindex>, memref<?xindex>, i32, i32, i32, i32, !llvm.ptr<i8>) -> !llvm.ptr<i8>
-// CHECK:           %[[VAL_17:.*]] = memref.alloca() : memref<2xindex>
-// CHECK:           %[[VAL_18:.*]] = memref.cast %[[VAL_17]] : memref<2xindex> to memref<?xindex>
-// CHECK:           %[[VAL_19:.*]] = memref.alloca() : memref<f64>
-// CHECK:           %[[VAL_20:.*]] = memref.alloc() : memref<2x4xf64>
-// CHECK:           linalg.fill ins(%[[VAL_1]] : f64) outs(%[[VAL_20]] : memref<2x4xf64>)
-// CHECK:           scf.while : () -> () {
-// CHECK:             %[[VAL_21:.*]] = func.call @getNextF64(%[[VAL_16]], %[[VAL_18]], %[[VAL_19]]) : (!llvm.ptr<i8>, memref<?xindex>, memref<f64>) -> i1
-// CHECK:             scf.condition(%[[VAL_21]])
-// CHECK:           } do {
-// CHECK:             %[[VAL_22:.*]] = memref.load %[[VAL_17]]{{\[}}%[[VAL_6]]] : memref<2xindex>
-// CHECK:             %[[VAL_23:.*]] = memref.load %[[VAL_17]]{{\[}}%[[VAL_5]]] : memref<2xindex>
-// CHECK:             %[[VAL_24:.*]] = memref.load %[[VAL_19]][] : memref<f64>
-// CHECK:             memref.store %[[VAL_24]], %[[VAL_20]]{{\[}}%[[VAL_22]], %[[VAL_23]]] : memref<2x4xf64>
-// CHECK:             scf.yield
-// CHECK:           }
-// CHECK:           call @delSparseTensorIteratorF64(%[[VAL_16]]) : (!llvm.ptr<i8>) -> ()
-// CHECK:           %[[VAL_25:.*]] = bufferization.to_tensor %[[VAL_20]] : memref<2x4xf64>
-// CHECK:           return %[[VAL_25]] : tensor<2x4xf64>
-// CHECK:         }
+// CHECK-LABEL:  func.func @sparse_convert_2d
+// CHECK-NOT:      sparse_tensor.reorder_coo
+// CHECK:          bufferization.alloc_tensor
+// CHECK:          linalg.fill
+// CHECK:          sparse_tensor.foreach
+// CHECK:            tensor.insert
 func.func @sparse_convert_2d(%arg0: tensor<2x4xf64, #SparseMatrix>) -> tensor<2x4xf64> {
   %0 = sparse_tensor.convert %arg0 : tensor<2x4xf64, #SparseMatrix> to tensor<2x4xf64>
   return %0 : tensor<2x4xf64>
 }
 
-// CHECK-LABEL:   func.func @sparse_convert_2d_dyn0(
-// CHECK-SAME:      %[[VAL_0:.*]]: !llvm.ptr<i8>) -> tensor<?x4xf64> {
-// CHECK-DAG:       %[[VAL_1:.*]] = arith.constant 0.000000e+00 : f64
-// CHECK-DAG:       %[[VAL_2:.*]] = arith.constant 6 : i32
-// CHECK-DAG:       %[[VAL_3:.*]] = arith.constant 1 : i32
-// CHECK-DAG:       %[[VAL_4:.*]] = arith.constant 0 : i32
-// CHECK-DAG:       %[[VAL_5:.*]] = arith.constant 1 : index
-// CHECK-DAG:       %[[VAL_6:.*]] = arith.constant 4 : i8
-// CHECK-DAG:       %[[VAL_7:.*]] = arith.constant 4 : index
-// CHECK-DAG:       %[[VAL_8:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_9:.*]] = call @sparseDimSize(%[[VAL_0]], %[[VAL_8]]) : (!llvm.ptr<i8>, index) -> index
-// CHECK:           %[[VAL_10:.*]] = memref.alloca() : memref<2xi8>
-// CHECK:           %[[VAL_11:.*]] = memref.cast %[[VAL_10]] : memref<2xi8> to memref<?xi8>
-// CHECK:           memref.store %[[VAL_6]], %[[VAL_10]]{{\[}}%[[VAL_8]]] : memref<2xi8>
-// CHECK:           memref.store %[[VAL_6]], %[[VAL_10]]{{\[}}%[[VAL_5]]] : memref<2xi8>
-// CHECK:           %[[VAL_12:.*]] = memref.alloca() : memref<2xindex>
-// CHECK:           %[[VAL_13:.*]] = memref.cast %[[VAL_12]] : memref<2xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_9]], %[[VAL_12]]{{\[}}%[[VAL_8]]] : memref<2xindex>
-// CHECK:           memref.store %[[VAL_7]], %[[VAL_12]]{{\[}}%[[VAL_5]]] : memref<2xindex>
-// CHECK:           %[[VAL_14:.*]] = memref.alloca() : memref<2xindex>
-// CHECK:           %[[VAL_15:.*]] = memref.cast %[[VAL_14]] : memref<2xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_8]], %[[VAL_14]]{{\[}}%[[VAL_8]]] : memref<2xindex>
-// CHECK:           memref.store %[[VAL_5]], %[[VAL_14]]{{\[}}%[[VAL_5]]] : memref<2xindex>
-// CHECK:           %[[VAL_16:.*]] = call @newSparseTensor(%[[VAL_13]], %[[VAL_13]], %[[VAL_11]], %[[VAL_15]], %[[VAL_15]], %[[VAL_4]], %[[VAL_4]], %[[VAL_3]], %[[VAL_2]], %[[VAL_0]]) : (memref<?xindex>, memref<?xindex>, memref<?xi8>, memref<?xindex>, memref<?xindex>, i32, i32, i32, i32, !llvm.ptr<i8>) -> !llvm.ptr<i8>
-// CHECK:           %[[VAL_17:.*]] = memref.alloca() : memref<2xindex>
-// CHECK:           %[[VAL_18:.*]] = memref.cast %[[VAL_17]] : memref<2xindex> to memref<?xindex>
-// CHECK:           %[[VAL_19:.*]] = memref.alloca() : memref<f64>
-// CHECK:           %[[VAL_20:.*]] = memref.alloc(%[[VAL_9]]) : memref<?x4xf64>
-// CHECK:           linalg.fill ins(%[[VAL_1]] : f64) outs(%[[VAL_20]] : memref<?x4xf64>)
-// CHECK:           scf.while : () -> () {
-// CHECK:             %[[VAL_21:.*]] = func.call @getNextF64(%[[VAL_16]], %[[VAL_18]], %[[VAL_19]]) : (!llvm.ptr<i8>, memref<?xindex>, memref<f64>) -> i1
-// CHECK:             scf.condition(%[[VAL_21]])
-// CHECK:           } do {
-// CHECK:             %[[VAL_22:.*]] = memref.load %[[VAL_17]]{{\[}}%[[VAL_8]]] : memref<2xindex>
-// CHECK:             %[[VAL_23:.*]] = memref.load %[[VAL_17]]{{\[}}%[[VAL_5]]] : memref<2xindex>
-// CHECK:             %[[VAL_24:.*]] = memref.load %[[VAL_19]][] : memref<f64>
-// CHECK:             memref.store %[[VAL_24]], %[[VAL_20]]{{\[}}%[[VAL_22]], %[[VAL_23]]] : memref<?x4xf64>
-// CHECK:             scf.yield
-// CHECK:           }
-// CHECK:           call @delSparseTensorIteratorF64(%[[VAL_16]]) : (!llvm.ptr<i8>) -> ()
-// CHECK:           %[[VAL_25:.*]] = bufferization.to_tensor %[[VAL_20]] : memref<?x4xf64>
-// CHECK:           return %[[VAL_25]] : tensor<?x4xf64>
-// CHECK:         }
+// CHECK-LABEL:  func.func @sparse_convert_2d_dyn
+// CHECK-NOT:      sparse_tensor.reorder_coo
+// CHECK:          bufferization.alloc_tensor
+// CHECK:          linalg.fill
+// CHECK:          sparse_tensor.foreach
+// CHECK:            tensor.insert
 func.func @sparse_convert_2d_dyn0(%arg0: tensor<?x4xf64, #SparseMatrix>) -> tensor<?x4xf64> {
   %0 = sparse_tensor.convert %arg0 : tensor<?x4xf64, #SparseMatrix> to tensor<?x4xf64>
   return %0 : tensor<?x4xf64>
 }
 
-// CHECK-LABEL:   func.func @sparse_convert_2d_dyn1(
-// CHECK-SAME:      %[[VAL_0:.*]]: !llvm.ptr<i8>) -> tensor<2x?xf64> {
-// CHECK-DAG:       %[[VAL_1:.*]] = arith.constant 0.000000e+00 : f64
-// CHECK-DAG:       %[[VAL_2:.*]] = arith.constant 6 : i32
-// CHECK-DAG:       %[[VAL_3:.*]] = arith.constant 1 : i32
-// CHECK-DAG:       %[[VAL_4:.*]] = arith.constant 0 : i32
-// CHECK-DAG:       %[[VAL_5:.*]] = arith.constant 0 : index
-// CHECK-DAG:       %[[VAL_6:.*]] = arith.constant 4 : i8
-// CHECK-DAG:       %[[VAL_7:.*]] = arith.constant 2 : index
-// CHECK-DAG:       %[[VAL_8:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_9:.*]] = call @sparseDimSize(%[[VAL_0]], %[[VAL_8]]) : (!llvm.ptr<i8>, index) -> index
-// CHECK:           %[[VAL_10:.*]] = memref.alloca() : memref<2xi8>
-// CHECK:           %[[VAL_11:.*]] = memref.cast %[[VAL_10]] : memref<2xi8> to memref<?xi8>
-// CHECK:           memref.store %[[VAL_6]], %[[VAL_10]]{{\[}}%[[VAL_5]]] : memref<2xi8>
-// CHECK:           memref.store %[[VAL_6]], %[[VAL_10]]{{\[}}%[[VAL_8]]] : memref<2xi8>
-// CHECK:           %[[VAL_12:.*]] = memref.alloca() : memref<2xindex>
-// CHECK:           %[[VAL_13:.*]] = memref.cast %[[VAL_12]] : memref<2xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_7]], %[[VAL_12]]{{\[}}%[[VAL_5]]] : memref<2xindex>
-// CHECK:           memref.store %[[VAL_9]], %[[VAL_12]]{{\[}}%[[VAL_8]]] : memref<2xindex>
-// CHECK:           %[[VAL_14:.*]] = memref.alloca() : memref<2xindex>
-// CHECK:           %[[VAL_15:.*]] = memref.cast %[[VAL_14]] : memref<2xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_5]], %[[VAL_14]]{{\[}}%[[VAL_5]]] : memref<2xindex>
-// CHECK:           memref.store %[[VAL_8]], %[[VAL_14]]{{\[}}%[[VAL_8]]] : memref<2xindex>
-// CHECK:           %[[VAL_16:.*]] = call @newSparseTensor(%[[VAL_13]], %[[VAL_13]], %[[VAL_11]], %[[VAL_15]], %[[VAL_15]], %[[VAL_4]], %[[VAL_4]], %[[VAL_3]], %[[VAL_2]], %[[VAL_0]]) : (memref<?xindex>, memref<?xindex>, memref<?xi8>, memref<?xindex>, memref<?xindex>, i32, i32, i32, i32, !llvm.ptr<i8>) -> !llvm.ptr<i8>
-// CHECK:           %[[VAL_17:.*]] = memref.alloca() : memref<2xindex>
-// CHECK:           %[[VAL_18:.*]] = memref.cast %[[VAL_17]] : memref<2xindex> to memref<?xindex>
-// CHECK:           %[[VAL_19:.*]] = memref.alloca() : memref<f64>
-// CHECK:           %[[VAL_20:.*]] = memref.alloc(%[[VAL_9]]) : memref<2x?xf64>
-// CHECK:           linalg.fill ins(%[[VAL_1]] : f64) outs(%[[VAL_20]] : memref<2x?xf64>)
-// CHECK:           scf.while : () -> () {
-// CHECK:             %[[VAL_21:.*]] = func.call @getNextF64(%[[VAL_16]], %[[VAL_18]], %[[VAL_19]]) : (!llvm.ptr<i8>, memref<?xindex>, memref<f64>) -> i1
-// CHECK:             scf.condition(%[[VAL_21]])
-// CHECK:           } do {
-// CHECK:             %[[VAL_22:.*]] = memref.load %[[VAL_17]]{{\[}}%[[VAL_5]]] : memref<2xindex>
-// CHECK:             %[[VAL_23:.*]] = memref.load %[[VAL_17]]{{\[}}%[[VAL_8]]] : memref<2xindex>
-// CHECK:             %[[VAL_24:.*]] = memref.load %[[VAL_19]][] : memref<f64>
-// CHECK:             memref.store %[[VAL_24]], %[[VAL_20]]{{\[}}%[[VAL_22]], %[[VAL_23]]] : memref<2x?xf64>
-// CHECK:             scf.yield
-// CHECK:           }
-// CHECK:           call @delSparseTensorIteratorF64(%[[VAL_16]]) : (!llvm.ptr<i8>) -> ()
-// CHECK:           %[[VAL_25:.*]] = bufferization.to_tensor %[[VAL_20]] : memref<2x?xf64>
-// CHECK:           return %[[VAL_25]] : tensor<2x?xf64>
-// CHECK:         }
+// CHECK-LABEL:  func.func @sparse_convert_2d_dyn1
+// CHECK-NOT:      sparse_tensor.reorder_coo
+// CHECK:          bufferization.alloc_tensor
+// CHECK:          linalg.fill
+// CHECK:          sparse_tensor.foreach
+// CHECK:            tensor.insert
 func.func @sparse_convert_2d_dyn1(%arg0: tensor<2x?xf64, #SparseMatrix>) -> tensor<2x?xf64> {
   %0 = sparse_tensor.convert %arg0 : tensor<2x?xf64, #SparseMatrix> to tensor<2x?xf64>
   return %0 : tensor<2x?xf64>
 }
 
-// CHECK-LABEL:   func.func @sparse_convert_2d_dyn2(
-// CHECK-SAME:      %[[VAL_0:.*]]: !llvm.ptr<i8>) -> tensor<?x?xf64> {
-// CHECK-DAG:       %[[VAL_1:.*]] = arith.constant 0.000000e+00 : f64
-// CHECK-DAG:       %[[VAL_2:.*]] = arith.constant 6 : i32
-// CHECK-DAG:       %[[VAL_3:.*]] = arith.constant 1 : i32
-// CHECK-DAG:       %[[VAL_4:.*]] = arith.constant 0 : i32
-// CHECK-DAG:       %[[VAL_5:.*]] = arith.constant 4 : i8
-// CHECK-DAG:       %[[VAL_6:.*]] = arith.constant 1 : index
-// CHECK-DAG:       %[[VAL_7:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_8:.*]] = call @sparseDimSize(%[[VAL_0]], %[[VAL_7]]) : (!llvm.ptr<i8>, index) -> index
-// CHECK:           %[[VAL_9:.*]] = call @sparseDimSize(%[[VAL_0]], %[[VAL_6]]) : (!llvm.ptr<i8>, index) -> index
-// CHECK:           %[[VAL_10:.*]] = memref.alloca() : memref<2xi8>
-// CHECK:           %[[VAL_11:.*]] = memref.cast %[[VAL_10]] : memref<2xi8> to memref<?xi8>
-// CHECK:           memref.store %[[VAL_5]], %[[VAL_10]]{{\[}}%[[VAL_7]]] : memref<2xi8>
-// CHECK:           memref.store %[[VAL_5]], %[[VAL_10]]{{\[}}%[[VAL_6]]] : memref<2xi8>
-// CHECK:           %[[VAL_12:.*]] = memref.alloca() : memref<2xindex>
-// CHECK:           %[[VAL_13:.*]] = memref.cast %[[VAL_12]] : memref<2xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_8]], %[[VAL_12]]{{\[}}%[[VAL_7]]] : memref<2xindex>
-// CHECK:           memref.store %[[VAL_9]], %[[VAL_12]]{{\[}}%[[VAL_6]]] : memref<2xindex>
-// CHECK:           %[[VAL_14:.*]] = memref.alloca() : memref<2xindex>
-// CHECK:           %[[VAL_15:.*]] = memref.cast %[[VAL_14]] : memref<2xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_7]], %[[VAL_14]]{{\[}}%[[VAL_7]]] : memref<2xindex>
-// CHECK:           memref.store %[[VAL_6]], %[[VAL_14]]{{\[}}%[[VAL_6]]] : memref<2xindex>
-// CHECK:           %[[VAL_16:.*]] = call @newSparseTensor(%[[VAL_13]], %[[VAL_13]], %[[VAL_11]], %[[VAL_15]], %[[VAL_15]], %[[VAL_4]], %[[VAL_4]], %[[VAL_3]], %[[VAL_2]], %[[VAL_0]]) : (memref<?xindex>, memref<?xindex>, memref<?xi8>, memref<?xindex>, memref<?xindex>, i32, i32, i32, i32, !llvm.ptr<i8>) -> !llvm.ptr<i8>
-// CHECK:           %[[VAL_17:.*]] = memref.alloca() : memref<2xindex>
-// CHECK:           %[[VAL_18:.*]] = memref.cast %[[VAL_17]] : memref<2xindex> to memref<?xindex>
-// CHECK:           %[[VAL_19:.*]] = memref.alloca() : memref<f64>
-// CHECK:           %[[VAL_20:.*]] = memref.alloc(%[[VAL_8]], %[[VAL_9]]) : memref<?x?xf64>
-// CHECK:           linalg.fill ins(%[[VAL_1]] : f64) outs(%[[VAL_20]] : memref<?x?xf64>)
-// CHECK:           scf.while : () -> () {
-// CHECK:             %[[VAL_21:.*]] = func.call @getNextF64(%[[VAL_16]], %[[VAL_18]], %[[VAL_19]]) : (!llvm.ptr<i8>, memref<?xindex>, memref<f64>) -> i1
-// CHECK:             scf.condition(%[[VAL_21]])
-// CHECK:           } do {
-// CHECK:             %[[VAL_22:.*]] = memref.load %[[VAL_17]]{{\[}}%[[VAL_7]]] : memref<2xindex>
-// CHECK:             %[[VAL_23:.*]] = memref.load %[[VAL_17]]{{\[}}%[[VAL_6]]] : memref<2xindex>
-// CHECK:             %[[VAL_24:.*]] = memref.load %[[VAL_19]][] : memref<f64>
-// CHECK:             memref.store %[[VAL_24]], %[[VAL_20]]{{\[}}%[[VAL_22]], %[[VAL_23]]] : memref<?x?xf64>
-// CHECK:             scf.yield
-// CHECK:           }
-// CHECK:           call @delSparseTensorIteratorF64(%[[VAL_16]]) : (!llvm.ptr<i8>) -> ()
-// CHECK:           %[[VAL_25:.*]] = bufferization.to_tensor %[[VAL_20]] : memref<?x?xf64>
-// CHECK:           return %[[VAL_25]] : tensor<?x?xf64>
-// CHECK:         }
+// CHECK-LABEL:  func.func @sparse_convert_2d_dyn2
+// CHECK-NOT:      sparse_tensor.reorder_coo
+// CHECK:          bufferization.alloc_tensor
+// CHECK:          linalg.fill
+// CHECK:          sparse_tensor.foreach
+// CHECK:            tensor.insert
 func.func @sparse_convert_2d_dyn2(%arg0: tensor<?x?xf64, #SparseMatrix>) -> tensor<?x?xf64> {
   %0 = sparse_tensor.convert %arg0 : tensor<?x?xf64, #SparseMatrix> to tensor<?x?xf64>
   return %0 : tensor<?x?xf64>
 }
 
-// CHECK-LABEL:   func.func @sparse_convert_3d(
-// CHECK-SAME:      %[[VAL_0:.*]]: !llvm.ptr<i8>) -> tensor<2x3x4xf64> {
-// CHECK-DAG:       %[[VAL_1:.*]] = arith.constant 0.000000e+00 : f64
-// CHECK-DAG:       %[[VAL_2:.*]] = arith.constant 6 : i32
-// CHECK-DAG:       %[[VAL_3:.*]] = arith.constant 1 : i32
-// CHECK-DAG:       %[[VAL_4:.*]] = arith.constant 0 : i32
-// CHECK-DAG:       %[[VAL_5:.*]] = arith.constant 1 : index
-// CHECK-DAG:       %[[VAL_6:.*]] = arith.constant 0 : index
-// CHECK-DAG:       %[[VAL_7:.*]] = arith.constant 2 : index
-// CHECK-DAG:       %[[VAL_8:.*]] = arith.constant 3 : index
-// CHECK-DAG:       %[[VAL_9:.*]] = arith.constant 4 : index
-// CHECK-DAG:       %[[VAL_10:.*]] = arith.constant 4 : i8
-// CHECK:           %[[VAL_11:.*]] = memref.alloca() : memref<3xi8>
-// CHECK:           %[[VAL_12:.*]] = memref.cast %[[VAL_11]] : memref<3xi8> to memref<?xi8>
-// CHECK:           memref.store %[[VAL_10]], %[[VAL_11]]{{\[}}%[[VAL_6]]] : memref<3xi8>
-// CHECK:           memref.store %[[VAL_10]], %[[VAL_11]]{{\[}}%[[VAL_5]]] : memref<3xi8>
-// CHECK:           memref.store %[[VAL_10]], %[[VAL_11]]{{\[}}%[[VAL_7]]] : memref<3xi8>
-// CHECK:           %[[VAL_13:.*]] = memref.alloca() : memref<3xindex>
-// CHECK:           %[[VAL_14:.*]] = memref.cast %[[VAL_13]] : memref<3xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_7]], %[[VAL_13]]{{\[}}%[[VAL_6]]] : memref<3xindex>
-// CHECK:           memref.store %[[VAL_8]], %[[VAL_13]]{{\[}}%[[VAL_5]]] : memref<3xindex>
-// CHECK:           memref.store %[[VAL_9]], %[[VAL_13]]{{\[}}%[[VAL_7]]] : memref<3xindex>
-// CHECK:           %[[VAL_15:.*]] = memref.alloca() : memref<3xindex>
-// CHECK:           %[[VAL_16:.*]] = memref.cast %[[VAL_15]] : memref<3xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_6]], %[[VAL_15]]{{\[}}%[[VAL_6]]] : memref<3xindex>
-// CHECK:           memref.store %[[VAL_5]], %[[VAL_15]]{{\[}}%[[VAL_5]]] : memref<3xindex>
-// CHECK:           memref.store %[[VAL_7]], %[[VAL_15]]{{\[}}%[[VAL_7]]] : memref<3xindex>
-// CHECK:           %[[VAL_17:.*]] = call @newSparseTensor(%[[VAL_14]], %[[VAL_14]], %[[VAL_12]], %[[VAL_16]], %[[VAL_16]], %[[VAL_4]], %[[VAL_4]], %[[VAL_3]], %[[VAL_2]], %[[VAL_0]]) : (memref<?xindex>, memref<?xindex>, memref<?xi8>, memref<?xindex>, memref<?xindex>, i32, i32, i32, i32, !llvm.ptr<i8>) -> !llvm.ptr<i8>
-// CHECK:           %[[VAL_18:.*]] = memref.alloca() : memref<3xindex>
-// CHECK:           %[[VAL_19:.*]] = memref.cast %[[VAL_18]] : memref<3xindex> to memref<?xindex>
-// CHECK:           %[[VAL_20:.*]] = memref.alloca() : memref<f64>
-// CHECK:           %[[VAL_21:.*]] = memref.alloc() : memref<2x3x4xf64>
-// CHECK:           linalg.fill ins(%[[VAL_1]] : f64) outs(%[[VAL_21]] : memref<2x3x4xf64>)
-// CHECK:           scf.while : () -> () {
-// CHECK:             %[[VAL_22:.*]] = func.call @getNextF64(%[[VAL_17]], %[[VAL_19]], %[[VAL_20]]) : (!llvm.ptr<i8>, memref<?xindex>, memref<f64>) -> i1
-// CHECK:             scf.condition(%[[VAL_22]])
-// CHECK:           } do {
-// CHECK:             %[[VAL_23:.*]] = memref.load %[[VAL_18]]{{\[}}%[[VAL_6]]] : memref<3xindex>
-// CHECK:             %[[VAL_24:.*]] = memref.load %[[VAL_18]]{{\[}}%[[VAL_5]]] : memref<3xindex>
-// CHECK:             %[[VAL_25:.*]] = memref.load %[[VAL_18]]{{\[}}%[[VAL_7]]] : memref<3xindex>
-// CHECK:             %[[VAL_26:.*]] = memref.load %[[VAL_20]][] : memref<f64>
-// CHECK:             memref.store %[[VAL_26]], %[[VAL_21]]{{\[}}%[[VAL_23]], %[[VAL_24]], %[[VAL_25]]] : memref<2x3x4xf64>
-// CHECK:             scf.yield
-// CHECK:           }
-// CHECK:           call @delSparseTensorIteratorF64(%[[VAL_17]]) : (!llvm.ptr<i8>) -> ()
-// CHECK:           %[[VAL_27:.*]] = bufferization.to_tensor %[[VAL_21]] : memref<2x3x4xf64>
-// CHECK:           return %[[VAL_27]] : tensor<2x3x4xf64>
-// CHECK:         }
+// CHECK-LABEL:  func.func @sparse_convert_3d
+// CHECK-NOT:      sparse_tensor.reorder_coo
+// CHECK:          bufferization.alloc_tensor
+// CHECK:          linalg.fill
+// CHECK:          sparse_tensor.foreach
+// CHECK:            tensor.insert
 func.func @sparse_convert_3d(%arg0: tensor<2x3x4xf64, #SparseTensor>) -> tensor<2x3x4xf64> {
   %0 = sparse_tensor.convert %arg0 : tensor<2x3x4xf64, #SparseTensor> to tensor<2x3x4xf64>
   return %0 : tensor<2x3x4xf64>
diff --git a/mlir/test/Dialect/SparseTensor/convert_sparse2sparse.mlir b/mlir/test/Dialect/SparseTensor/convert_sparse2sparse.mlir
index e8e69dc861..658e8aa400 100644
--- a/mlir/test/Dialect/SparseTensor/convert_sparse2sparse.mlir
+++ b/mlir/test/Dialect/SparseTensor/convert_sparse2sparse.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s --sparse-tensor-conversion --canonicalize --cse | FileCheck %s
+// RUN: mlir-opt %s --stage-sparse-ops --post-sparsification-rewrite="enable-foreach=false" --canonicalize --cse | FileCheck %s
 
 #SparseVector64 = #sparse_tensor.encoding<{
   map = (d0) -> (d0 : compressed),
@@ -33,185 +33,56 @@
   map = (d0 : #sparse_tensor<slice(2, 2, 1)>, d1 : #sparse_tensor<slice(12, 13, 1)>) -> (d0 : compressed(nonunique), d1 : singleton)
 }>
 
-// CHECK-LABEL:   func.func @sparse_nop_convert(
-// CHECK-SAME:      %[[VAL_0:.*]]: !llvm.ptr<i8>) -> !llvm.ptr<i8> {
-// CHECK:           return %[[VAL_0]] : !llvm.ptr<i8>
-// CHECK:         }
+// CHECK-LABEL:   func.func @sparse_nop_convert
+// CHECK-NEXT:       return
 func.func @sparse_nop_convert(%arg0: tensor<64xf32, #SparseVector>) -> tensor<64xf32, #SparseVector> {
   %0 = sparse_tensor.convert %arg0 : tensor<64xf32, #SparseVector> to tensor<64xf32, #SparseVector>
   return %0 : tensor<64xf32, #SparseVector>
 }
 
-// CHECK-LABEL:   func.func @sparse_hidden_nop_cast(
-// CHECK-SAME:      %[[VAL_0:.*]]: !llvm.ptr<i8>) -> !llvm.ptr<i8> {
-// CHECK:           return %[[VAL_0]] : !llvm.ptr<i8>
-// CHECK:         }
+// CHECK-LABEL:   func.func @sparse_hidden_nop_cast
+// TODO: The following convert should be a cast instead.
+// CHECK:           sparse_tensor.convert
+// CHECK:           return
 func.func @sparse_hidden_nop_cast(%arg0: tensor<32xf32, #SparseVector>) -> tensor<?xf32, #SparseVector> {
   %0 = sparse_tensor.convert %arg0 : tensor<32xf32, #SparseVector> to tensor<?xf32, #SparseVector>
   return %0 : tensor<?xf32, #SparseVector>
 }
 
 // CHECK-LABEL:   func.func @sparse_convert_1d_ss(
-// CHECK-SAME:      %[[VAL_0:.*]]: !llvm.ptr<i8>) -> !llvm.ptr<i8> {
-// CHECK-DAG:       %[[VAL_1:.*]] = arith.constant 3 : i32
-// CHECK-DAG:       %[[VAL_2:.*]] = arith.constant 2 : i32
-// CHECK-DAG:       %[[VAL_3:.*]] = arith.constant 8 : i8
-// CHECK-DAG:       %[[VAL_4:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_5:.*]] = call @sparseDimSize(%[[VAL_0]], %[[VAL_4]]) : (!llvm.ptr<i8>, index) -> index
-// CHECK:           %[[VAL_6:.*]] = memref.alloca() : memref<1xi8>
-// CHECK:           %[[VAL_7:.*]] = memref.cast %[[VAL_6]] : memref<1xi8> to memref<?xi8>
-// CHECK:           memref.store %[[VAL_3]], %[[VAL_6]]{{\[}}%[[VAL_4]]] : memref<1xi8>
-// CHECK:           %[[VAL_8:.*]] = memref.alloca() : memref<1xindex>
-// CHECK:           %[[VAL_9:.*]] = memref.cast %[[VAL_8]] : memref<1xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_5]], %[[VAL_8]]{{\[}}%[[VAL_4]]] : memref<1xindex>
-// CHECK:           %[[VAL_10:.*]] = memref.alloca() : memref<1xindex>
-// CHECK:           %[[VAL_11:.*]] = memref.cast %[[VAL_10]] : memref<1xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_4]], %[[VAL_10]]{{\[}}%[[VAL_4]]] : memref<1xindex>
-// CHECK:           %[[VAL_12:.*]] = call @newSparseTensor(%[[VAL_9]], %[[VAL_9]], %[[VAL_7]], %[[VAL_11]], %[[VAL_11]], %[[VAL_2]], %[[VAL_2]], %[[VAL_2]], %[[VAL_1]], %[[VAL_0]]) : (memref<?xindex>, memref<?xindex>, memref<?xi8>, memref<?xindex>, memref<?xindex>, i32, i32, i32, i32, !llvm.ptr<i8>) -> !llvm.ptr<i8>
-// CHECK:           return %[[VAL_12]] : !llvm.ptr<i8>
-// CHECK:         }
+// TODO: libgen path need to support efficient format conversion (e.g., 32 bit pos -> 64 bit pos).
+// Maybe we should use a different operator as well to be clear.
 func.func @sparse_convert_1d_ss(%arg0: tensor<?xf32, #SparseVector64>) -> tensor<?xf32, #SparseVector32> {
   %0 = sparse_tensor.convert %arg0 : tensor<?xf32, #SparseVector64> to tensor<?xf32, #SparseVector32>
   return %0 : tensor<?xf32, #SparseVector32>
 }
 
 // CHECK-LABEL:   func.func @sparse_convert(
-// CHECK-SAME:      %[[VAL_0:.*]]: !llvm.ptr<i8>) -> !llvm.ptr<i8> {
-// CHECK-DAG:       %[[VAL_1:.*]] = arith.constant 3 : i32
-// CHECK-DAG:       %[[VAL_2:.*]] = arith.constant 2 : i32
-// CHECK-DAG:       %[[VAL_3:.*]] = arith.constant 8 : i8
-// CHECK-DAG:       %[[VAL_4:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_5:.*]] = call @sparseDimSize(%[[VAL_0]], %[[VAL_4]]) : (!llvm.ptr<i8>, index) -> index
-// CHECK:           %[[VAL_6:.*]] = memref.alloca() : memref<1xi8>
-// CHECK:           %[[VAL_7:.*]] = memref.cast %[[VAL_6]] : memref<1xi8> to memref<?xi8>
-// CHECK:           memref.store %[[VAL_3]], %[[VAL_6]]{{\[}}%[[VAL_4]]] : memref<1xi8>
-// CHECK:           %[[VAL_8:.*]] = memref.alloca() : memref<1xindex>
-// CHECK:           %[[VAL_9:.*]] = memref.cast %[[VAL_8]] : memref<1xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_5]], %[[VAL_8]]{{\[}}%[[VAL_4]]] : memref<1xindex>
-// CHECK:           %[[VAL_10:.*]] = memref.alloca() : memref<1xindex>
-// CHECK:           %[[VAL_11:.*]] = memref.cast %[[VAL_10]] : memref<1xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_4]], %[[VAL_10]]{{\[}}%[[VAL_4]]] : memref<1xindex>
-// CHECK:           %[[VAL_12:.*]] = call @newSparseTensor(%[[VAL_9]], %[[VAL_9]], %[[VAL_7]], %[[VAL_11]], %[[VAL_11]], %[[VAL_2]], %[[VAL_2]], %[[VAL_2]], %[[VAL_1]], %[[VAL_0]]) : (memref<?xindex>, memref<?xindex>, memref<?xi8>, memref<?xindex>, memref<?xindex>, i32, i32, i32, i32, !llvm.ptr<i8>) -> !llvm.ptr<i8>
-// CHECK:           return %[[VAL_12]] : !llvm.ptr<i8>
-// CHECK:         }
+// TODO: libgen path need to support efficient format conversion (e.g., 32 bit pos -> 64 bit pos).
+// Maybe we should use a different operator as well to be clear.
 func.func @sparse_convert(%arg0: tensor<?xf32, #SparseVector64>) -> tensor<?xf32, #SparseVector32> {
   %0 = sparse_tensor.convert %arg0 : tensor<?xf32, #SparseVector64> to tensor<?xf32, #SparseVector32>
   return %0 : tensor<?xf32, #SparseVector32>
 }
 
-#SparseSingleton64 = #sparse_tensor.encoding<{
-  map = (d0) -> (d0 : singleton),
-  posWidth = 64,
-  crdWidth = 64
-}>
-
-#SparseSingleton32 = #sparse_tensor.encoding<{
-  map = (d0) -> (d0 : singleton),
-  posWidth = 32,
-  crdWidth = 32
-}>
-
-//
-// CHECK-LABEL:   func.func @sparse_convert_singleton(
-// CHECK-SAME:      %[[VAL_0:.*]]: !llvm.ptr<i8>) -> !llvm.ptr<i8> {
-// CHECK-DAG:       %[[VAL_1:.*]] = arith.constant 3 : i32
-// CHECK-DAG:       %[[VAL_2:.*]] = arith.constant 2 : i32
-// CHECK-DAG:       %[[VAL_3:.*]] = arith.constant 16 : i8
-// CHECK-DAG:       %[[VAL_4:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_5:.*]] = call @sparseDimSize(%[[VAL_0]], %[[VAL_4]]) : (!llvm.ptr<i8>, index) -> index
-// CHECK:           %[[VAL_6:.*]] = memref.alloca() : memref<1xi8>
-// CHECK:           %[[VAL_7:.*]] = memref.cast %[[VAL_6]] : memref<1xi8> to memref<?xi8>
-// CHECK:           memref.store %[[VAL_3]], %[[VAL_6]]{{\[}}%[[VAL_4]]] : memref<1xi8>
-// CHECK:           %[[VAL_8:.*]] = memref.alloca() : memref<1xindex>
-// CHECK:           %[[VAL_9:.*]] = memref.cast %[[VAL_8]] : memref<1xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_5]], %[[VAL_8]]{{\[}}%[[VAL_4]]] : memref<1xindex>
-// CHECK:           %[[VAL_10:.*]] = memref.alloca() : memref<1xindex>
-// CHECK:           %[[VAL_11:.*]] = memref.cast %[[VAL_10]] : memref<1xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_4]], %[[VAL_10]]{{\[}}%[[VAL_4]]] : memref<1xindex>
-// CHECK:           %[[VAL_12:.*]] = call @newSparseTensor(%[[VAL_9]], %[[VAL_9]], %[[VAL_7]], %[[VAL_11]], %[[VAL_11]], %[[VAL_2]], %[[VAL_2]], %[[VAL_2]], %[[VAL_1]], %[[VAL_0]]) : (memref<?xindex>, memref<?xindex>, memref<?xi8>, memref<?xindex>, memref<?xindex>, i32, i32, i32, i32, !llvm.ptr<i8>) -> !llvm.ptr<i8>
-// CHECK:           return %[[VAL_12]] : !llvm.ptr<i8>
-// CHECK:         }
-func.func @sparse_convert_singleton(%arg0: tensor<?xf32, #SparseSingleton64>) -> tensor<?xf32, #SparseSingleton32> {
-  %0 = sparse_tensor.convert %arg0 : tensor<?xf32, #SparseSingleton64> to tensor<?xf32, #SparseSingleton32>
-  return %0 : tensor<?xf32, #SparseSingleton32>
-}
-
-// CHECK-LABEL:   func.func @sparse_convert_permuted(
-// CHECK-SAME:      %[[VAL_0:.*]]: !llvm.ptr<i8>) -> !llvm.ptr<i8> {
-// CHECK-DAG:       %[[VAL_1:.*]] = arith.constant 5 : i32
-// CHECK-DAG:       %[[VAL_2:.*]] = arith.constant 2 : i32
-// CHECK-DAG:       %[[VAL_3:.*]] = arith.constant 0 : i32
-// CHECK-DAG:       %[[VAL_4:.*]] = arith.constant 8 : i8
-// CHECK-DAG:       %[[VAL_5:.*]] = arith.constant 2 : index
-// CHECK-DAG:       %[[VAL_6:.*]] = arith.constant 1 : index
-// CHECK-DAG:       %[[VAL_7:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_8:.*]] = call @sparseDimSize(%[[VAL_0]], %[[VAL_7]]) : (!llvm.ptr<i8>, index) -> index
-// CHECK:           %[[VAL_9:.*]] = call @sparseDimSize(%[[VAL_0]], %[[VAL_6]]) : (!llvm.ptr<i8>, index) -> index
-// CHECK:           %[[VAL_10:.*]] = call @sparseDimSize(%[[VAL_0]], %[[VAL_5]]) : (!llvm.ptr<i8>, index) -> index
-// CHECK:           %[[VAL_11:.*]] = memref.alloca() : memref<3xi8>
-// CHECK:           %[[VAL_12:.*]] = memref.cast %[[VAL_11]] : memref<3xi8> to memref<?xi8>
-// CHECK:           memref.store %[[VAL_4]], %[[VAL_11]]{{\[}}%[[VAL_7]]] : memref<3xi8>
-// CHECK:           memref.store %[[VAL_4]], %[[VAL_11]]{{\[}}%[[VAL_6]]] : memref<3xi8>
-// CHECK:           memref.store %[[VAL_4]], %[[VAL_11]]{{\[}}%[[VAL_5]]] : memref<3xi8>
-// CHECK:           %[[VAL_13:.*]] = memref.alloca() : memref<3xindex>
-// CHECK:           %[[VAL_14:.*]] = memref.cast %[[VAL_13]] : memref<3xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_8]], %[[VAL_13]]{{\[}}%[[VAL_7]]] : memref<3xindex>
-// CHECK:           memref.store %[[VAL_9]], %[[VAL_13]]{{\[}}%[[VAL_6]]] : memref<3xindex>
-// CHECK:           memref.store %[[VAL_10]], %[[VAL_13]]{{\[}}%[[VAL_5]]] : memref<3xindex>
-// CHECK:           %[[VAL_15:.*]] = memref.load %[[VAL_13]]{{\[}}%[[VAL_5]]] : memref<3xindex>
-// CHECK:           %[[VAL_16:.*]] = memref.load %[[VAL_13]]{{\[}}%[[VAL_7]]] : memref<3xindex>
-// CHECK:           %[[VAL_17:.*]] = memref.load %[[VAL_13]]{{\[}}%[[VAL_6]]] : memref<3xindex>
-// CHECK:           %[[VAL_18:.*]] = memref.alloca() : memref<3xindex>
-// CHECK:           %[[VAL_19:.*]] = memref.cast %[[VAL_18]] : memref<3xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_6]], %[[VAL_18]]{{\[}}%[[VAL_7]]] : memref<3xindex>
-// CHECK:           memref.store %[[VAL_5]], %[[VAL_18]]{{\[}}%[[VAL_6]]] : memref<3xindex>
-// CHECK:           memref.store %[[VAL_7]], %[[VAL_18]]{{\[}}%[[VAL_5]]] : memref<3xindex>
-// CHECK:           %[[VAL_20:.*]] = memref.alloca() : memref<3xindex>
-// CHECK:           %[[VAL_21:.*]] = memref.cast %[[VAL_20]] : memref<3xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_5]], %[[VAL_20]]{{\[}}%[[VAL_7]]] : memref<3xindex>
-// CHECK:           memref.store %[[VAL_7]], %[[VAL_20]]{{\[}}%[[VAL_6]]] : memref<3xindex>
-// CHECK:           memref.store %[[VAL_6]], %[[VAL_20]]{{\[}}%[[VAL_5]]] : memref<3xindex>
-// CHECK:           %[[VAL_22:.*]] = memref.alloca() : memref<3xindex>
-// CHECK:           %[[VAL_23:.*]] = memref.cast %[[VAL_22]] : memref<3xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_15]], %[[VAL_22]]{{\[}}%[[VAL_7]]] : memref<3xindex>
-// CHECK:           memref.store %[[VAL_16]], %[[VAL_22]]{{\[}}%[[VAL_6]]] : memref<3xindex>
-// CHECK:           memref.store %[[VAL_17]], %[[VAL_22]]{{\[}}%[[VAL_5]]] : memref<3xindex>
-// CHECK:           %[[VAL_24:.*]] = call @newSparseTensor(%[[VAL_14]], %[[VAL_23]], %[[VAL_12]], %[[VAL_19]], %[[VAL_21]], %[[VAL_3]], %[[VAL_3]], %[[VAL_2]], %[[VAL_1]], %[[VAL_0]]) : (memref<?xindex>, memref<?xindex>, memref<?xi8>, memref<?xindex>, memref<?xindex>, i32, i32, i32, i32, !llvm.ptr<i8>) -> !llvm.ptr<i8>
-// CHECK:           %[[VAL_25:.*]] = call @newSparseTensor(%[[VAL_14]], %[[VAL_23]], %[[VAL_12]], %[[VAL_19]], %[[VAL_21]], %[[VAL_3]], %[[VAL_3]], %[[VAL_2]], %[[VAL_2]], %[[VAL_24]]) : (memref<?xindex>, memref<?xindex>, memref<?xi8>, memref<?xindex>, memref<?xindex>, i32, i32, i32, i32, !llvm.ptr<i8>) -> !llvm.ptr<i8>
-// CHECK:           call @delSparseTensorCOOF32(%[[VAL_24]]) : (!llvm.ptr<i8>) -> ()
-// CHECK:           return %[[VAL_25]] : !llvm.ptr<i8>
-// CHECK:         }
+// CHECK-LABEL:   func.func @sparse_convert_permuted
+// CHECK:           sparse_tensor.foreach
+// CHECK:             sparse_tensor.insert
+// CHECK:           sparse_tensor.load
+// CHECK:           sparse_tensor.reorder_coo
+// CHECK:           sparse_tensor.foreach
+// CHECK:             sparse_tensor.insert
+// CHECK:           sparse_tensor.load
 func.func @sparse_convert_permuted(%arg0: tensor<?x?x?xf32, #SortedCOO3D>) -> tensor<?x?x?xf32, #TsssPermuted> {
   %0 = sparse_tensor.convert %arg0 : tensor<?x?x?xf32, #SortedCOO3D> to tensor<?x?x?xf32, #TsssPermuted>
   return %0 : tensor<?x?x?xf32, #TsssPermuted>
 }
 
-// CHECK-LABEL:   func.func @sparse_convert_slice(
-// CHECK-SAME:      %[[VAL_0:.*]]: !llvm.ptr<i8>) -> !llvm.ptr<i8> {
-// CHECK-DAG:       %[[VAL_1:.*]] = arith.constant 3 : i32
-// CHECK-DAG:       %[[VAL_2:.*]] = arith.constant 6 : i32
-// CHECK-DAG:       %[[VAL_3:.*]] = arith.constant 0 : i32
-// CHECK-DAG:       %[[VAL_4:.*]] = arith.constant 1 : index
-// CHECK-DAG:       %[[VAL_5:.*]] = arith.constant 0 : index
-// CHECK-DAG:       %[[VAL_6:.*]] = arith.constant 2 : index
-// CHECK-DAG:       %[[VAL_7:.*]] = arith.constant 13 : index
-// CHECK-DAG:       %[[VAL_8:.*]] = arith.constant 9 : i8
-// CHECK-DAG:       %[[VAL_9:.*]] = arith.constant 16 : i8
-// CHECK:           %[[VAL_10:.*]] = memref.alloca() : memref<2xi8>
-// CHECK:           %[[VAL_11:.*]] = memref.cast %[[VAL_10]] : memref<2xi8> to memref<?xi8>
-// CHECK:           memref.store %[[VAL_8]], %[[VAL_10]]{{\[}}%[[VAL_5]]] : memref<2xi8>
-// CHECK:           memref.store %[[VAL_9]], %[[VAL_10]]{{\[}}%[[VAL_4]]] : memref<2xi8>
-// CHECK:           %[[VAL_12:.*]] = memref.alloca() : memref<2xindex>
-// CHECK:           %[[VAL_13:.*]] = memref.cast %[[VAL_12]] : memref<2xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_6]], %[[VAL_12]]{{\[}}%[[VAL_5]]] : memref<2xindex>
-// CHECK:           memref.store %[[VAL_7]], %[[VAL_12]]{{\[}}%[[VAL_4]]] : memref<2xindex>
-// CHECK:           %[[VAL_14:.*]] = memref.alloca() : memref<2xindex>
-// CHECK:           %[[VAL_15:.*]] = memref.cast %[[VAL_14]] : memref<2xindex> to memref<?xindex>
-// CHECK:           memref.store %[[VAL_5]], %[[VAL_14]]{{\[}}%[[VAL_5]]] : memref<2xindex>
-// CHECK:           memref.store %[[VAL_4]], %[[VAL_14]]{{\[}}%[[VAL_4]]] : memref<2xindex>
-// CHECK:           %[[VAL_16:.*]] = call @newSparseTensor(%[[VAL_13]], %[[VAL_13]], %[[VAL_11]], %[[VAL_15]], %[[VAL_15]], %[[VAL_3]], %[[VAL_3]], %[[VAL_2]], %[[VAL_1]], %[[VAL_0]]) : (memref<?xindex>, memref<?xindex>, memref<?xi8>, memref<?xindex>, memref<?xindex>, i32, i32, i32, i32, !llvm.ptr<i8>) -> !llvm.ptr<i8>
-// CHECK:           return %[[VAL_16]] : !llvm.ptr<i8>
-// CHECK:         }
+// CHECK-LABEL:   func.func @sparse_convert_slice
+// CHECK:           sparse_tensor.foreach
+// CHECK:             sparse_tensor.insert
+// CHECK:           sparse_tensor.load
+// CHECK-NOT:       sparse_tensor.reorder_coo
 func.func @sparse_convert_slice(%arg0: tensor<2x13xi32, #COOSlice>) -> (tensor<2x13xi32, #SortedCOO2D>)  {
   %0 = sparse_tensor.convert %arg0 : tensor<2x13xi32, #COOSlice> to tensor<2x13xi32, #SortedCOO2D>
   return %0 : tensor<2x13xi32, #SortedCOO2D>
diff --git a/mlir/test/Dialect/SparseTensor/fold.mlir b/mlir/test/Dialect/SparseTensor/fold.mlir
index 089431f9e1..3428f6d4ae 100644
--- a/mlir/test/Dialect/SparseTensor/fold.mlir
+++ b/mlir/test/Dialect/SparseTensor/fold.mlir
@@ -62,3 +62,34 @@ func.func @sparse_get_specifier_dce_fold(%arg0: !sparse_tensor.storage_specifier
        : !sparse_tensor.storage_specifier<#SparseVector>
   return %2 : index
 }
+
+
+
+#COO = #sparse_tensor.encoding<{map = (d0, d1) -> (d0 : compressed(nonunique), d1 : singleton)}>
+
+// CHECK-LABEL: func @sparse_reorder_coo(
+//  CHECK-SAME: %[[A:.*]]: tensor<?x?xf32, #sparse_tensor.encoding<{{{.*}}}>>
+//   CHECK-NOT: %[[R:.*]] = sparse_tensor.reorder_coo
+//       CHECK: return %[[A]]
+func.func @sparse_reorder_coo(%arg0 : tensor<?x?xf32, #COO>) -> tensor<?x?xf32, #COO> {
+  %ret = sparse_tensor.reorder_coo quick_sort %arg0 : tensor<?x?xf32, #COO> to tensor<?x?xf32, #COO>
+  return %ret : tensor<?x?xf32, #COO>
+}
+
+
+#BSR = #sparse_tensor.encoding<{
+  map = ( i, j ) ->
+  ( i floordiv 2 : dense,
+    j floordiv 3 : compressed,
+    i mod 2      : dense,
+    j mod 3      : dense
+  )
+}>
+
+// CHECK-LABEL: func @sparse_crd_translate(
+//   CHECK-NOT:   sparse_tensor.crd_translate
+func.func @sparse_crd_translate(%arg0: index, %arg1: index) -> (index, index) {
+  %l0, %l1, %l2, %l3 = sparse_tensor.crd_translate dim_to_lvl [%arg0, %arg1] as #BSR : index, index, index, index
+  %d0, %d1 = sparse_tensor.crd_translate lvl_to_dim [%l0, %l1, %l2, %l3] as #BSR : index, index
+  return  %d0, %d1 : index, index
+}
diff --git a/mlir/test/Dialect/SparseTensor/invalid.mlir b/mlir/test/Dialect/SparseTensor/invalid.mlir
index 2df4237efa..1ab1bac6b5 100644
--- a/mlir/test/Dialect/SparseTensor/invalid.mlir
+++ b/mlir/test/Dialect/SparseTensor/invalid.mlir
@@ -839,3 +839,59 @@ func.func @sparse_alloc_escapes(%arg0: index) -> tensor<10x?xf64, #CSR> {
   %0 = bufferization.alloc_tensor(%arg0) : tensor<10x?xf64, #CSR>
   return %0: tensor<10x?xf64, #CSR>
 }
+
+// -----
+
+#UnorderedCOO = #sparse_tensor.encoding<{map = (d0, d1) -> (d0 : compressed(nonunique, nonordered), d1 : singleton(nonordered))}>
+#OrderedCOOPerm = #sparse_tensor.encoding<{map = (d0, d1) -> (d1 : compressed(nonunique), d0 : singleton)}>
+
+func.func @sparse_permuted_reorder_coo(%arg0 : tensor<?x?xf32, #UnorderedCOO>) -> tensor<?x?xf32, #OrderedCOOPerm> {
+  // expected-error@+1 {{Unmatched dim2lvl map between input and result COO}}
+  %ret = sparse_tensor.reorder_coo quick_sort %arg0 : tensor<?x?xf32, #UnorderedCOO> to tensor<?x?xf32, #OrderedCOOPerm>
+  return %ret : tensor<?x?xf32, #OrderedCOOPerm>
+}
+
+// -----
+
+#UnorderedCOO = #sparse_tensor.encoding<{map = (d0, d1) -> (d0 : compressed(nonunique, nonordered), d1 : singleton(nonordered))}>
+#OrderedCOO = #sparse_tensor.encoding<{map = (d0, d1) -> (d0 : compressed(nonunique), d1 : singleton)}>
+
+func.func @sparse_permuted_reorder_coo(%arg0 : tensor<?x?xf32, #UnorderedCOO>) -> tensor<?x?xf64, #OrderedCOO> {
+  // expected-error@+1 {{Unmatched storage format between input and result COO}}
+  %ret = sparse_tensor.reorder_coo quick_sort %arg0 : tensor<?x?xf32, #UnorderedCOO> to tensor<?x?xf64, #OrderedCOO>
+  return %ret : tensor<?x?xf64, #OrderedCOO>
+}
+
+// -----
+
+#BSR = #sparse_tensor.encoding<{
+  map = ( i, j ) ->
+  ( i floordiv 2 : dense,
+    j floordiv 3 : compressed,
+    i mod 2      : dense,
+    j mod 3      : dense
+  )
+}>
+
+func.func @sparse_crd_translate(%arg0: index, %arg1: index) -> (index, index, index) {
+  // expected-error@+1 {{Coordinate rank mismatch with encoding}}
+  %l0, %l1, %l2 = sparse_tensor.crd_translate dim_to_lvl [%arg0, %arg1] as #BSR : index, index, index
+  return  %l0, %l1, %l2 : index, index, index
+}
+
+// -----
+
+#BSR = #sparse_tensor.encoding<{
+  map = ( i, j ) ->
+  ( i floordiv 2 : dense,
+    j floordiv 3 : compressed,
+    i mod 2      : dense,
+    j mod 3      : dense
+  )
+}>
+
+func.func @sparse_crd_translate(%arg0: index, %arg1: index, %arg2: index) -> (index, index, index, index) {
+  // expected-error@+1 {{Coordinate rank mismatch with encoding}}
+  %l0, %l1, %l2, %l3 = sparse_tensor.crd_translate dim_to_lvl [%arg0, %arg1, %arg2] as #BSR : index, index, index, index
+  return  %l0, %l1, %l2, %l3 : index, index, index, index
+}
diff --git a/mlir/test/Dialect/SparseTensor/invalid_encoding.mlir b/mlir/test/Dialect/SparseTensor/invalid_encoding.mlir
index ef1dd3ee41..6514391bae 100644
--- a/mlir/test/Dialect/SparseTensor/invalid_encoding.mlir
+++ b/mlir/test/Dialect/SparseTensor/invalid_encoding.mlir
@@ -60,7 +60,7 @@ func.func private @tensor_sizes_mismatch(%arg0: tensor<8xi32, #a>) -> ()
 
 // -----
 
-// expected-error@+1 {{unexpected dimToLvl mapping from 2 to 1}}
+// expected-error@+1 {{failed to infer lvlToDim from dimToLvl}}
 #a = #sparse_tensor.encoding<{map = (d0, d1) -> (d0 : dense)}>
 func.func private @tensor_sizes_mismatch(%arg0: tensor<8xi32, #a>) -> ()
 
@@ -119,7 +119,7 @@ func.func private @tensor_dimtolvl_mismatch(%arg0: tensor<8xi32, #a>) -> ()
 
 // -----
 
-// expected-error@+1 {{expected a permutation affine map for dimToLvl}}
+// expected-error@+1 {{failed to infer lvlToDim from dimToLvl}}
 #a = #sparse_tensor.encoding<{map = (d0, d1) -> (d0 : dense, d0 : compressed)}>
 func.func private @tensor_no_permutation(%arg0: tensor<16x32xf32, #a>) -> ()
 
@@ -251,3 +251,22 @@ func.func private @too_few_lvl_decl(%arg0: tensor<?x?xf64, #TooFewLvlDecl>) {
 func.func private @wrong_order_lvl_decl(%arg0: tensor<?x?xf64, #WrongOrderLvlDecl>) {
   return
 }
+
+// -----
+
+// expected-error@+1 {{expected lvlToDim to be an inverse of dimToLvl}}
+#BSR_explicit = #sparse_tensor.encoding<{
+  map =
+  {il, jl, ii, jj}
+  ( i = il * 3 + ii,
+    j = jl * 2 + jj
+  ) ->
+  ( il = i floordiv 2 : dense,
+    jl = j floordiv 3 : compressed,
+    ii = i mod 2      : dense,
+    jj = j mod 3      : dense
+  )
+}>
+func.func private @BSR_explicit(%arg0: tensor<?x?xf64, #BSR_explicit>) {
+  return
+}
diff --git a/mlir/test/Dialect/SparseTensor/roundtrip.mlir b/mlir/test/Dialect/SparseTensor/roundtrip.mlir
index 82267be34b..af9618ebe3 100644
--- a/mlir/test/Dialect/SparseTensor/roundtrip.mlir
+++ b/mlir/test/Dialect/SparseTensor/roundtrip.mlir
@@ -633,3 +633,39 @@ func.func @sparse_sort_coo_stable(%arg0: index, %arg1: memref<?xi64>, %arg2: mem
   sparse_tensor.sort insertion_sort_stable %arg0, %arg1 jointly %arg2 {perm_map = #ID_MAP, ny = 1 : index}: memref<?xi64> jointly memref<?xf32>
   return %arg1, %arg2 : memref<?xi64>, memref<?xf32>
 }
+
+// -----
+
+#UnorderedCOO = #sparse_tensor.encoding<{map = (d0, d1) -> (d0 : compressed(nonunique, nonordered), d1 : singleton(nonordered))}>
+#OrderedCOO = #sparse_tensor.encoding<{map = (d0, d1) -> (d0 : compressed(nonunique), d1 : singleton)}>
+
+// CHECK-LABEL: func @sparse_reorder_coo(
+//  CHECK-SAME: %[[A:.*]]: tensor<?x?xf32, #sparse_tensor.encoding<{{{.*}}}>>
+//       CHECK: %[[R:.*]] = sparse_tensor.reorder_coo quick_sort %[[A]]
+//       CHECK: return %[[R]]
+func.func @sparse_reorder_coo(%arg0 : tensor<?x?xf32, #UnorderedCOO>) -> tensor<?x?xf32, #OrderedCOO> {
+  %ret = sparse_tensor.reorder_coo quick_sort %arg0 : tensor<?x?xf32, #UnorderedCOO> to tensor<?x?xf32, #OrderedCOO>
+  return %ret : tensor<?x?xf32, #OrderedCOO>
+}
+
+
+// -----
+
+#BSR = #sparse_tensor.encoding<{
+  map = ( i, j ) ->
+  ( i floordiv 2 : dense,
+    j floordiv 3 : compressed,
+    i mod 2      : dense,
+    j mod 3      : dense
+  )
+}>
+
+// CHECK-LABEL:   func.func @sparse_crd_translate(
+// CHECK-SAME:      %[[VAL_0:.*]]: index,
+// CHECK-SAME:      %[[VAL_1:.*]]: index)
+// CHECK:           %[[VAL_2:.*]]:4 = sparse_tensor.crd_translate  dim_to_lvl{{\[}}%[[VAL_0]], %[[VAL_1]]]
+// CHECK:           return %[[VAL_2]]#0, %[[VAL_2]]#1, %[[VAL_2]]#2, %[[VAL_2]]#3
+func.func @sparse_crd_translate(%arg0: index, %arg1: index) -> (index, index, index, index) {
+  %l0, %l1, %l2, %l3 = sparse_tensor.crd_translate dim_to_lvl [%arg0, %arg1] as #BSR : index, index, index, index
+  return  %l0, %l1, %l2, %l3 : index, index, index, index
+}
diff --git a/mlir/test/Dialect/SparseTensor/roundtrip_encoding.mlir b/mlir/test/Dialect/SparseTensor/roundtrip_encoding.mlir
index ae3805d8b7..ea8217ab6e 100644
--- a/mlir/test/Dialect/SparseTensor/roundtrip_encoding.mlir
+++ b/mlir/test/Dialect/SparseTensor/roundtrip_encoding.mlir
@@ -160,6 +160,24 @@ func.func private @BSR(%arg0: tensor<?x?xf64, #BSR>) {
 
 // -----
 
+#BCSR = #sparse_tensor.encoding<{
+  map = ( i, j, k ) ->
+  ( i floordiv 2 : dense,
+    j floordiv 3 : dense,
+    k floordiv 4 : compressed,
+    i mod 2      : dense,
+    j mod 3      : dense,
+    k mod 4      : dense
+  )
+}>
+
+// CHECK-LABEL: func private @BCSR(
+// CHECK-SAME: tensor<?x?x?xf64, #sparse_tensor.encoding<{ map = (d0, d1, d2) -> (d0 floordiv 2 : dense, d1 floordiv 3 : dense, d2 floordiv 4 : compressed, d0 mod 2 : dense, d1 mod 3 : dense, d2 mod 4 : dense) }>>
+func.func private @BCSR(%arg0: tensor<?x?x?xf64, #BCSR>) {
+  return
+}
+// -----
+
 #BSR_explicit = #sparse_tensor.encoding<{
   map =
   {il, jl, ii, jj}
@@ -194,3 +212,37 @@ func.func private @BSR_explicit(%arg0: tensor<?x?xf64, #BSR_explicit>) {
 func.func private @NV_24(%arg0: tensor<?x?xf64, #NV_24>) {
   return
 }
+
+// -----
+
+#NV_24 = #sparse_tensor.encoding<{
+  map = ( i, j, k ) ->
+  ( i            : dense,
+    j            : dense,
+    k floordiv 4 : dense,
+    k mod 4      : block2_4
+  )
+}>
+
+// CHECK-LABEL: func private @NV_24(
+// CHECK-SAME: tensor<?x?x?xf64, #sparse_tensor.encoding<{ map = (d0, d1, d2) -> (d0 : dense, d1 : dense, d2 floordiv 4 : dense, d2 mod 4 : block2_4) }>>
+func.func private @NV_24(%arg0: tensor<?x?x?xf64, #NV_24>) {
+  return
+}
+
+// -----
+
+#NV_24 = #sparse_tensor.encoding<{
+  map = ( i, j, k ) ->
+  ( i            : dense,
+    k floordiv 4 : dense,
+    j            : dense,
+    k mod 4      : block2_4
+  )
+}>
+
+// CHECK-LABEL: func private @NV_24(
+// CHECK-SAME: tensor<?x?x?xf64, #sparse_tensor.encoding<{ map = (d0, d1, d2) -> (d0 : dense, d2 floordiv 4 : dense, d1 : dense, d2 mod 4 : block2_4) }>>
+func.func private @NV_24(%arg0: tensor<?x?x?xf64, #NV_24>) {
+  return
+}
\ No newline at end of file
diff --git a/mlir/test/Dialect/SparseTensor/sparse_concat.mlir b/mlir/test/Dialect/SparseTensor/sparse_concat.mlir
index 2fb4529e56..f3d3dd2856 100644
--- a/mlir/test/Dialect/SparseTensor/sparse_concat.mlir
+++ b/mlir/test/Dialect/SparseTensor/sparse_concat.mlir
@@ -5,8 +5,7 @@
 
 
 #DCSR = #sparse_tensor.encoding<{map = (d0, d1) -> (d0 : compressed, d1 : compressed)}>
-#DENSE = #sparse_tensor.encoding<{map = (d0, d1) -> (d0 : dense, d1 : dense)}>
-#DENSE_P = #sparse_tensor.encoding<{map = (d0, d1) -> (d1 : dense, d0 : dense)}>
+
 // CHECK-LABEL: @concat_sparse_sparse(
 //  CHECK-SAME:  %[[TMP_arg0:.*]]: tensor<2x4xf64, #sparse_tensor
 //  CHECK-SAME:  %[[TMP_arg1:.*]]: tensor<3x4xf64, #sparse_tensor
@@ -177,77 +176,83 @@ func.func @concat_sparse_sparse_dynamic(%arg0: tensor<2x4xf64, #DCSR>,
     return %0 : tensor<?x?xf64, #DCSR>
 }
 
-// CHECK-LABEL: @concat_sparse_sparse_dense(
-//  CHECK-SAME:  %[[TMP_arg0:.*]]: tensor<2x4xf64, #sparse_tensor
-//  CHECK-SAME:  %[[TMP_arg1:.*]]: tensor<3x4xf64, #sparse_tensor
-//  CHECK-SAME:  %[[TMP_arg2:.*]]: tensor<4x4xf64, #sparse_tensor
-//   CHECK-DAG:  %[[TMP_c0:.*]] = arith.constant 0 : index
-//   CHECK-DAG:  %[[TMP_c1:.*]] = arith.constant 1 : index
-//   CHECK-DAG:  %[[TMP_c5:.*]] = arith.constant 5 : index
-//   CHECK-DAG:  %[[TMP_c2:.*]] = arith.constant 2 : index
-//   CHECK-DAG:  %[[TMP_c9:.*]] = arith.constant 9 : index
-//   CHECK-DAG:  %[[TMP_c4:.*]] = arith.constant 4 : index
-//   CHECK-DAG:  %[[TMP_d0:.*]] = arith.constant 0.000000e+00 : f64
-//       CHECK:  %[[A:.*]] = memref.alloc(%[[TMP_c9]], %[[TMP_c4]]) : memref<?x?xf64>
-//       CHECK:  linalg.fill ins(%[[TMP_d0]] : f64) outs(%[[A]] : memref<?x?xf64>)
-//       CHECK:  %[[TMP_1:.*]] = sparse_tensor.positions %[[TMP_arg0]] {level = 0 : index} : tensor<2x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_2:.*]] = sparse_tensor.coordinates %[[TMP_arg0]] {level = 0 : index} : tensor<2x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_3:.*]] = sparse_tensor.positions %[[TMP_arg0]] {level = 1 : index} : tensor<2x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_4:.*]] = sparse_tensor.coordinates %[[TMP_arg0]] {level = 1 : index} : tensor<2x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_5:.*]] = sparse_tensor.values %[[TMP_arg0]] : tensor<2x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_6:.*]] = memref.load %[[TMP_1]][%[[TMP_c0]]] : memref<?xindex>
-//       CHECK:  %[[TMP_7:.*]] = memref.load %[[TMP_1]][%[[TMP_c1]]] : memref<?xindex>
-//       CHECK:  scf.for %[[TMP_arg3:.*]] = %[[TMP_6]] to %[[TMP_7]] step %[[TMP_c1]]
-//       CHECK:    %[[TMP_23:.*]] = memref.load %[[TMP_2]][%[[TMP_arg3]]] : memref<?xindex>
-//   CHECK-DAG:    %[[TMP_25:.*]] = memref.load %[[TMP_3]][%[[TMP_arg3]]] : memref<?xindex>
-//   CHECK-DAG:    %[[TMP_24:.*]] = arith.addi %[[TMP_arg3]], %[[TMP_c1]] : index
-//       CHECK:    %[[TMP_26:.*]] = memref.load %[[TMP_3]][%[[TMP_24]]] : memref<?xindex>
-//       CHECK:    scf.for %[[TMP_arg4:.*]] = %[[TMP_25]] to %[[TMP_26]] step %[[TMP_c1]]
-//       CHECK:      %[[TMP_27:.*]] = memref.load %[[TMP_4]][%[[TMP_arg4]]] : memref<?xindex>
-//       CHECK:      %[[TMP_28:.*]] = memref.load %[[TMP_5]][%[[TMP_arg4]]] : memref<?xf64>
-//       CHECK:      memref.store %[[TMP_28]], %[[A]]{{\[}}%[[TMP_23]], %[[TMP_27]]] : memref<?x?xf64>
-//       CHECK:    }
-//       CHECK:  }
-//       CHECK:  %[[TMP_8:.*]] = sparse_tensor.positions %[[TMP_arg1]] {level = 0 : index} : tensor<3x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_9:.*]] = sparse_tensor.coordinates %[[TMP_arg1]] {level = 0 : index} : tensor<3x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_10:.*]] = sparse_tensor.positions %[[TMP_arg1]] {level = 1 : index} : tensor<3x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_11:.*]] = sparse_tensor.coordinates %[[TMP_arg1]] {level = 1 : index} : tensor<3x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_12:.*]] = sparse_tensor.values %[[TMP_arg1]] : tensor<3x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_13:.*]] = memref.load %[[TMP_8]][%[[TMP_c0]]] : memref<?xindex>
-//       CHECK:  %[[TMP_14:.*]] = memref.load %[[TMP_8]][%[[TMP_c1]]] : memref<?xindex>
-//       CHECK:  scf.for %[[TMP_arg3:.*]] = %[[TMP_13]] to %[[TMP_14]] step %[[TMP_c1]]
-//       CHECK:    %[[TMP_23:.*]] = memref.load %[[TMP_9]][%[[TMP_arg3]]] : memref<?xindex>
-//   CHECK-DAG:    %[[TMP_25:.*]] = memref.load %[[TMP_10]][%[[TMP_arg3]]] : memref<?xindex>
-//   CHECK-DAG:    %[[TMP_24:.*]] = arith.addi %[[TMP_arg3]], %[[TMP_c1]] : index
-//       CHECK:    %[[TMP_26:.*]] = memref.load %[[TMP_10]][%[[TMP_24]]] : memref<?xindex>
-//       CHECK:    scf.for %[[TMP_arg4:.*]] = %[[TMP_25]] to %[[TMP_26]] step %[[TMP_c1]]
-//       CHECK:      %[[TMP_27:.*]] = memref.load %[[TMP_11]][%[[TMP_arg4]]] : memref<?xindex>
-//       CHECK:      %[[TMP_28:.*]] = memref.load %[[TMP_12]][%[[TMP_arg4]]] : memref<?xf64>
-//       CHECK:      %[[TMP_29:.*]] = arith.addi %[[TMP_23]], %[[TMP_c2]] : index
-//       CHECK:      memref.store %[[TMP_28]], %[[A]]{{\[}}%[[TMP_29]], %[[TMP_27]]] : memref<?x?xf64>
-//       CHECK:    }
-//       CHECK:  }
-//       CHECK:  %[[TMP_15:.*]] = sparse_tensor.positions %[[TMP_arg2]] {level = 0 : index} : tensor<4x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_16:.*]] = sparse_tensor.coordinates %[[TMP_arg2]] {level = 0 : index} : tensor<4x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_17:.*]] = sparse_tensor.positions %[[TMP_arg2]] {level = 1 : index} : tensor<4x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_18:.*]] = sparse_tensor.coordinates %[[TMP_arg2]] {level = 1 : index} : tensor<4x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_19:.*]] = sparse_tensor.values %[[TMP_arg2]] : tensor<4x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_20:.*]] = memref.load %[[TMP_15]][%[[TMP_c0]]] : memref<?xindex>
-//       CHECK:  %[[TMP_21:.*]] = memref.load %[[TMP_15]][%[[TMP_c1]]] : memref<?xindex>
-//       CHECK:  scf.for %[[TMP_arg3:.*]] = %[[TMP_20]] to %[[TMP_21]] step %[[TMP_c1]]
-//       CHECK:    %[[TMP_23:.*]] = memref.load %[[TMP_16]][%[[TMP_arg3]]] : memref<?xindex>
-//       CHECK:    %[[TMP_25:.*]] = memref.load %[[TMP_17]][%[[TMP_arg3]]] : memref<?xindex>
-//       CHECK:    %[[TMP_24:.*]] = arith.addi %[[TMP_arg3]], %[[TMP_c1]] : index
-//       CHECK:    %[[TMP_26:.*]] = memref.load %[[TMP_17]][%[[TMP_24]]] : memref<?xindex>
-//       CHECK:    scf.for %[[TMP_arg4:.*]] = %[[TMP_25]] to %[[TMP_26]] step %[[TMP_c1]]
-//       CHECK:      %[[TMP_27:.*]] = memref.load %[[TMP_18]][%[[TMP_arg4]]] : memref<?xindex>
-//       CHECK:      %[[TMP_28:.*]] = memref.load %[[TMP_19]][%[[TMP_arg4]]] : memref<?xf64>
-//       CHECK:      %[[TMP_29:.*]] = arith.addi %[[TMP_23]], %[[TMP_c5]] : index
-//       CHECK:      memref.store %[[TMP_28]], %[[A]]{{\[}}%[[TMP_29]], %[[TMP_27]]] : memref<?x?xf64>
-//       CHECK:    }
-//       CHECK:  }
-//       CHECK:  %[[R:.*]] = bufferization.to_tensor %[[A]] : memref<?x?xf64>
-//       CHECK:  return %[[R]] : tensor<?x?xf64>
+// CHECK-LABEL:   func.func @concat_sparse_sparse_dense(
+// CHECK-SAME:      %[[VAL_0:.*]]: tensor<2x4xf64, #sparse_tensor
+// CHECK-SAME:      %[[VAL_1:.*]]: tensor<3x4xf64, #sparse_tensor
+// CHECK-SAME:      %[[VAL_2:.*]]: tensor<4x4xf64, #sparse_tensor
+// CHECK-DAG:       %[[VAL_3:.*]] = arith.constant 4 : index
+// CHECK-DAG:       %[[VAL_4:.*]] = arith.constant 9 : index
+// CHECK-DAG:       %[[VAL_5:.*]] = arith.constant 5 : index
+// CHECK-DAG:       %[[VAL_6:.*]] = arith.constant 0.000000e+00 : f64
+// CHECK-DAG:       %[[VAL_7:.*]] = arith.constant 0 : index
+// CHECK-DAG:       %[[VAL_8:.*]] = arith.constant 1 : index
+// CHECK-DAG:       %[[VAL_9:.*]] = arith.constant 2 : index
+// CHECK:           %[[VAL_10:.*]] = bufferization.alloc_tensor(%[[VAL_4]], %[[VAL_3]]) : tensor<?x?xf64>
+// CHECK:           %[[VAL_11:.*]] = linalg.fill ins(%[[VAL_6]] : f64) outs(%[[VAL_10]] : tensor<?x?xf64>) -> tensor<?x?xf64>
+// CHECK:           %[[VAL_12:.*]] = sparse_tensor.positions %[[VAL_0]] {level = 0 : index} : tensor<2x4xf64, #sparse_tensor
+// CHECK:           %[[VAL_13:.*]] = sparse_tensor.coordinates %[[VAL_0]] {level = 0 : index} : tensor<2x4xf64, #sparse_tensor
+// CHECK:           %[[VAL_14:.*]] = sparse_tensor.positions %[[VAL_0]] {level = 1 : index} : tensor<2x4xf64, #sparse_tensor
+// CHECK:           %[[VAL_15:.*]] = sparse_tensor.coordinates %[[VAL_0]] {level = 1 : index} : tensor<2x4xf64, #sparse_tensor
+// CHECK:           %[[VAL_16:.*]] = sparse_tensor.values %[[VAL_0]] : tensor<2x4xf64, #sparse_tensor
+// CHECK:           %[[VAL_17:.*]] = memref.load %[[VAL_12]]{{\[}}%[[VAL_7]]] : memref<?xindex>
+// CHECK:           %[[VAL_18:.*]] = memref.load %[[VAL_12]]{{\[}}%[[VAL_8]]] : memref<?xindex>
+// CHECK:           %[[VAL_19:.*]] = scf.for %[[VAL_20:.*]] = %[[VAL_17]] to %[[VAL_18]] step %[[VAL_8]] iter_args(%[[VAL_21:.*]] = %[[VAL_11]]) -> (tensor<?x?xf64>) {
+// CHECK:             %[[VAL_22:.*]] = memref.load %[[VAL_13]]{{\[}}%[[VAL_20]]] : memref<?xindex>
+// CHECK:             %[[VAL_23:.*]] = memref.load %[[VAL_14]]{{\[}}%[[VAL_20]]] : memref<?xindex>
+// CHECK:             %[[VAL_24:.*]] = arith.addi %[[VAL_20]], %[[VAL_8]] : index
+// CHECK:             %[[VAL_25:.*]] = memref.load %[[VAL_14]]{{\[}}%[[VAL_24]]] : memref<?xindex>
+// CHECK:             %[[VAL_26:.*]] = scf.for %[[VAL_27:.*]] = %[[VAL_23]] to %[[VAL_25]] step %[[VAL_8]] iter_args(%[[VAL_28:.*]] = %[[VAL_21]]) -> (tensor<?x?xf64>) {
+// CHECK:               %[[VAL_29:.*]] = memref.load %[[VAL_15]]{{\[}}%[[VAL_27]]] : memref<?xindex>
+// CHECK:               %[[VAL_30:.*]] = memref.load %[[VAL_16]]{{\[}}%[[VAL_27]]] : memref<?xf64>
+// CHECK:               %[[VAL_31:.*]] = tensor.insert %[[VAL_30]] into %[[VAL_28]]{{\[}}%[[VAL_22]], %[[VAL_29]]] : tensor<?x?xf64>
+// CHECK:               scf.yield %[[VAL_31]] : tensor<?x?xf64>
+// CHECK:             }
+// CHECK:             scf.yield %[[VAL_26]] : tensor<?x?xf64>
+// CHECK:           }
+// CHECK:           %[[VAL_32:.*]] = sparse_tensor.positions %[[VAL_1]] {level = 0 : index} : tensor<3x4xf64, #sparse_tensor
+// CHECK:           %[[VAL_33:.*]] = sparse_tensor.coordinates %[[VAL_1]] {level = 0 : index} : tensor<3x4xf64, #sparse_tensor
+// CHECK:           %[[VAL_34:.*]] = sparse_tensor.positions %[[VAL_1]] {level = 1 : index} : tensor<3x4xf64, #sparse_tensor
+// CHECK:           %[[VAL_35:.*]] = sparse_tensor.coordinates %[[VAL_1]] {level = 1 : index} : tensor<3x4xf64, #sparse_tensor
+// CHECK:           %[[VAL_36:.*]] = sparse_tensor.values %[[VAL_1]] : tensor<3x4xf64, #sparse_tensor
+// CHECK:           %[[VAL_37:.*]] = memref.load %[[VAL_32]]{{\[}}%[[VAL_7]]] : memref<?xindex>
+// CHECK:           %[[VAL_38:.*]] = memref.load %[[VAL_32]]{{\[}}%[[VAL_8]]] : memref<?xindex>
+// CHECK:           %[[VAL_39:.*]] = scf.for %[[VAL_40:.*]] = %[[VAL_37]] to %[[VAL_38]] step %[[VAL_8]] iter_args(%[[VAL_41:.*]] = %[[VAL_19]]) -> (tensor<?x?xf64>) {
+// CHECK:             %[[VAL_42:.*]] = memref.load %[[VAL_33]]{{\[}}%[[VAL_40]]] : memref<?xindex>
+// CHECK:             %[[VAL_43:.*]] = memref.load %[[VAL_34]]{{\[}}%[[VAL_40]]] : memref<?xindex>
+// CHECK:             %[[VAL_44:.*]] = arith.addi %[[VAL_40]], %[[VAL_8]] : index
+// CHECK:             %[[VAL_45:.*]] = memref.load %[[VAL_34]]{{\[}}%[[VAL_44]]] : memref<?xindex>
+// CHECK:             %[[VAL_46:.*]] = scf.for %[[VAL_47:.*]] = %[[VAL_43]] to %[[VAL_45]] step %[[VAL_8]] iter_args(%[[VAL_48:.*]] = %[[VAL_41]]) -> (tensor<?x?xf64>) {
+// CHECK:               %[[VAL_49:.*]] = memref.load %[[VAL_35]]{{\[}}%[[VAL_47]]] : memref<?xindex>
+// CHECK:               %[[VAL_50:.*]] = memref.load %[[VAL_36]]{{\[}}%[[VAL_47]]] : memref<?xf64>
+// CHECK:               %[[VAL_51:.*]] = arith.addi %[[VAL_42]], %[[VAL_9]] : index
+// CHECK:               %[[VAL_52:.*]] = tensor.insert %[[VAL_50]] into %[[VAL_48]]{{\[}}%[[VAL_51]], %[[VAL_49]]] : tensor<?x?xf64>
+// CHECK:               scf.yield %[[VAL_52]] : tensor<?x?xf64>
+// CHECK:             }
+// CHECK:             scf.yield %[[VAL_46]] : tensor<?x?xf64>
+// CHECK:           }
+// CHECK:           %[[VAL_53:.*]] = sparse_tensor.positions %[[VAL_2]] {level = 0 : index} : tensor<4x4xf64, #sparse_tensor
+// CHECK:           %[[VAL_54:.*]] = sparse_tensor.coordinates %[[VAL_2]] {level = 0 : index} : tensor<4x4xf64, #sparse_tensor
+// CHECK:           %[[VAL_55:.*]] = sparse_tensor.positions %[[VAL_2]] {level = 1 : index} : tensor<4x4xf64, #sparse_tensor
+// CHECK:           %[[VAL_56:.*]] = sparse_tensor.coordinates %[[VAL_2]] {level = 1 : index} : tensor<4x4xf64, #sparse_tensor
+// CHECK:           %[[VAL_57:.*]] = sparse_tensor.values %[[VAL_2]] : tensor<4x4xf64, #sparse_tensor
+// CHECK:           %[[VAL_58:.*]] = memref.load %[[VAL_53]]{{\[}}%[[VAL_7]]] : memref<?xindex>
+// CHECK:           %[[VAL_59:.*]] = memref.load %[[VAL_53]]{{\[}}%[[VAL_8]]] : memref<?xindex>
+// CHECK:           %[[VAL_60:.*]] = scf.for %[[VAL_61:.*]] = %[[VAL_58]] to %[[VAL_59]] step %[[VAL_8]] iter_args(%[[VAL_62:.*]] = %[[VAL_39]]) -> (tensor<?x?xf64>) {
+// CHECK:             %[[VAL_63:.*]] = memref.load %[[VAL_54]]{{\[}}%[[VAL_61]]] : memref<?xindex>
+// CHECK:             %[[VAL_64:.*]] = memref.load %[[VAL_55]]{{\[}}%[[VAL_61]]] : memref<?xindex>
+// CHECK:             %[[VAL_65:.*]] = arith.addi %[[VAL_61]], %[[VAL_8]] : index
+// CHECK:             %[[VAL_66:.*]] = memref.load %[[VAL_55]]{{\[}}%[[VAL_65]]] : memref<?xindex>
+// CHECK:             %[[VAL_67:.*]] = scf.for %[[VAL_68:.*]] = %[[VAL_64]] to %[[VAL_66]] step %[[VAL_8]] iter_args(%[[VAL_69:.*]] = %[[VAL_62]]) -> (tensor<?x?xf64>) {
+// CHECK:               %[[VAL_70:.*]] = memref.load %[[VAL_56]]{{\[}}%[[VAL_68]]] : memref<?xindex>
+// CHECK:               %[[VAL_71:.*]] = memref.load %[[VAL_57]]{{\[}}%[[VAL_68]]] : memref<?xf64>
+// CHECK:               %[[VAL_72:.*]] = arith.addi %[[VAL_63]], %[[VAL_5]] : index
+// CHECK:               %[[VAL_73:.*]] = tensor.insert %[[VAL_71]] into %[[VAL_69]]{{\[}}%[[VAL_72]], %[[VAL_70]]] : tensor<?x?xf64>
+// CHECK:               scf.yield %[[VAL_73]] : tensor<?x?xf64>
+// CHECK:             }
+// CHECK:             scf.yield %[[VAL_67]] : tensor<?x?xf64>
+// CHECK:           }
+// CHECK:           return %[[VAL_60]] : tensor<?x?xf64>
+// CHECK:         }
 func.func @concat_sparse_sparse_dense(%arg0: tensor<2x4xf64, #DCSR>,
                                 %arg1: tensor<3x4xf64, #DCSR>,
                                 %arg2: tensor<4x4xf64, #DCSR>)
@@ -258,173 +263,3 @@ func.func @concat_sparse_sparse_dense(%arg0: tensor<2x4xf64, #DCSR>,
            tensor<4x4xf64, #DCSR> to tensor<?x?xf64>
     return %0 : tensor<?x?xf64>
 }
-
-// CHECK-LABEL: @concat_sparse_sparse_annotated_dense(
-//  CHECK-SAME:  %[[TMP_arg0:.*]]: tensor<2x4xf64, #sparse_tensor
-//  CHECK-SAME:  %[[TMP_arg1:.*]]: tensor<3x4xf64, #sparse_tensor
-//  CHECK-SAME:  %[[TMP_arg2:.*]]: tensor<4x4xf64, #sparse_tensor
-//   CHECK-DAG:  %[[TMP_c0:.*]] = arith.constant 0 : index
-//   CHECK-DAG:  %[[TMP_c1:.*]] = arith.constant 1 : index
-//   CHECK-DAG:  %[[TMP_c5:.*]] = arith.constant 5 : index
-//   CHECK-DAG:  %[[TMP_c2:.*]] = arith.constant 2 : index
-//   CHECK-DAG:  %[[TMP_c9:.*]] = arith.constant 9 : index
-//   CHECK-DAG:  %[[TMP_c4:.*]] = arith.constant 4 : index
-//       CHECK:  %[[TMP_0:.*]] = bufferization.alloc_tensor(%[[TMP_c9]], %[[TMP_c4]]) : tensor<?x?xf64, #sparse_tensor
-//       CHECK:  %[[VAL_0:.*]] = sparse_tensor.values %[[TMP_0]] : tensor<?x?xf64, #sparse_tensor.encoding<{{{.*}}}>> to memref<?xf64>
-//       CHECK:  %[[DIM_0:.*]] = memref.alloca() : memref<2xindex>
-//       CHECK:  memref.store %[[TMP_c9]], %[[DIM_0]][%[[TMP_c0]]] : memref<2xindex>
-//       CHECK:  memref.store %[[TMP_c4]], %[[DIM_0]][%[[TMP_c1]]] : memref<2xindex>
-//       CHECK:  %[[VAL_1:.*]] = memref.reshape %[[VAL_0]](%[[DIM_0]]) : (memref<?xf64>, memref<2xindex>) -> memref<?x?xf64>
-//       CHECK:  %[[TMP_1:.*]] = sparse_tensor.positions %[[TMP_arg0]] {level = 0 : index} : tensor<2x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_2:.*]] = sparse_tensor.coordinates %[[TMP_arg0]] {level = 0 : index} : tensor<2x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_3:.*]] = sparse_tensor.positions %[[TMP_arg0]] {level = 1 : index} : tensor<2x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_4:.*]] = sparse_tensor.coordinates %[[TMP_arg0]] {level = 1 : index} : tensor<2x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_5:.*]] = sparse_tensor.values %[[TMP_arg0]] : tensor<2x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_6:.*]] = memref.load %[[TMP_1]][%[[TMP_c0]]] : memref<?xindex>
-//       CHECK:  %[[TMP_7:.*]] = memref.load %[[TMP_1]][%[[TMP_c1]]] : memref<?xindex>
-//       CHECK:  scf.for %[[TMP_arg3:.*]] = %[[TMP_6]] to %[[TMP_7]] step %[[TMP_c1]]
-//       CHECK:    %[[TMP_23:.*]] = memref.load %[[TMP_2]][%[[TMP_arg3]]] : memref<?xindex>
-//   CHECK-DAG:    %[[TMP_25:.*]] = memref.load %[[TMP_3]][%[[TMP_arg3]]] : memref<?xindex>
-//   CHECK-DAG:    %[[TMP_24:.*]] = arith.addi %[[TMP_arg3]], %[[TMP_c1]] : index
-//       CHECK:    %[[TMP_26:.*]] = memref.load %[[TMP_3]][%[[TMP_24]]] : memref<?xindex>
-//       CHECK:    scf.for %[[TMP_arg4:.*]] = %[[TMP_25]] to %[[TMP_26]] step %[[TMP_c1]]
-//       CHECK:      %[[TMP_27:.*]] = memref.load %[[TMP_4]][%[[TMP_arg4]]] : memref<?xindex>
-//       CHECK:      %[[TMP_28:.*]] = memref.load %[[TMP_5]][%[[TMP_arg4]]] : memref<?xf64>
-//       CHECK:      memref.store %[[TMP_28]], %[[VAL_1]][%[[TMP_23]], %[[TMP_27]]] : memref<?x?xf64>
-//       CHECK:    }
-//       CHECK:  }
-//       CHECK:  %[[TMP_8:.*]] = sparse_tensor.positions %[[TMP_arg1]] {level = 0 : index} : tensor<3x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_9:.*]] = sparse_tensor.coordinates %[[TMP_arg1]] {level = 0 : index} : tensor<3x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_10:.*]] = sparse_tensor.positions %[[TMP_arg1]] {level = 1 : index} : tensor<3x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_11:.*]] = sparse_tensor.coordinates %[[TMP_arg1]] {level = 1 : index} : tensor<3x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_12:.*]] = sparse_tensor.values %[[TMP_arg1]] : tensor<3x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_13:.*]] = memref.load %[[TMP_8]][%[[TMP_c0]]] : memref<?xindex>
-//       CHECK:  %[[TMP_14:.*]] = memref.load %[[TMP_8]][%[[TMP_c1]]] : memref<?xindex>
-//       CHECK:  scf.for %[[TMP_arg3:.*]] = %[[TMP_13]] to %[[TMP_14]] step %[[TMP_c1]]
-//       CHECK:    %[[TMP_23:.*]] = memref.load %[[TMP_9]][%[[TMP_arg3]]] : memref<?xindex>
-//   CHECK-DAG:    %[[TMP_25:.*]] = memref.load %[[TMP_10]][%[[TMP_arg3]]] : memref<?xindex>
-//   CHECK-DAG:    %[[TMP_24:.*]] = arith.addi %[[TMP_arg3]], %[[TMP_c1]] : index
-//       CHECK:    %[[TMP_26:.*]] = memref.load %[[TMP_10]][%[[TMP_24]]] : memref<?xindex>
-//       CHECK:    scf.for %[[TMP_arg4:.*]] = %[[TMP_25]] to %[[TMP_26]] step %[[TMP_c1]]
-//       CHECK:      %[[TMP_27:.*]] = memref.load %[[TMP_11]][%[[TMP_arg4]]] : memref<?xindex>
-//       CHECK:      %[[TMP_28:.*]] = memref.load %[[TMP_12]][%[[TMP_arg4]]] : memref<?xf64>
-//       CHECK:      %[[TMP_29:.*]] = arith.addi %[[TMP_23]], %[[TMP_c2]] : index
-//       CHECK:      memref.store %[[TMP_28]], %[[VAL_1]][%[[TMP_29]], %[[TMP_27]]] : memref<?x?xf64>
-//       CHECK:    }
-//       CHECK:  }
-//       CHECK:  %[[TMP_15:.*]] = sparse_tensor.positions %[[TMP_arg2]] {level = 0 : index} : tensor<4x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_16:.*]] = sparse_tensor.coordinates %[[TMP_arg2]] {level = 0 : index} : tensor<4x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_17:.*]] = sparse_tensor.positions %[[TMP_arg2]] {level = 1 : index} : tensor<4x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_18:.*]] = sparse_tensor.coordinates %[[TMP_arg2]] {level = 1 : index} : tensor<4x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_19:.*]] = sparse_tensor.values %[[TMP_arg2]] : tensor<4x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_20:.*]] = memref.load %[[TMP_15]][%[[TMP_c0]]] : memref<?xindex>
-//       CHECK:  %[[TMP_21:.*]] = memref.load %[[TMP_15]][%[[TMP_c1]]] : memref<?xindex>
-//       CHECK:  scf.for %[[TMP_arg3:.*]] = %[[TMP_20]] to %[[TMP_21]] step %[[TMP_c1]]
-//       CHECK:    %[[TMP_23:.*]] = memref.load %[[TMP_16]][%[[TMP_arg3]]] : memref<?xindex>
-//       CHECK:    %[[TMP_25:.*]] = memref.load %[[TMP_17]][%[[TMP_arg3]]] : memref<?xindex>
-//       CHECK:    %[[TMP_24:.*]] = arith.addi %[[TMP_arg3]], %[[TMP_c1]] : index
-//       CHECK:    %[[TMP_26:.*]] = memref.load %[[TMP_17]][%[[TMP_24]]] : memref<?xindex>
-//       CHECK:    scf.for %[[TMP_arg4:.*]] = %[[TMP_25]] to %[[TMP_26]] step %[[TMP_c1]]
-//       CHECK:      %[[TMP_27:.*]] = memref.load %[[TMP_18]][%[[TMP_arg4]]] : memref<?xindex>
-//       CHECK:      %[[TMP_28:.*]] = memref.load %[[TMP_19]][%[[TMP_arg4]]] : memref<?xf64>
-//       CHECK:      %[[TMP_29:.*]] = arith.addi %[[TMP_23]], %[[TMP_c5]] : index
-//       CHECK:      memref.store %[[TMP_28]], %[[VAL_1]][%[[TMP_29]], %[[TMP_27]]] : memref<?x?xf64>
-//       CHECK:    }
-//       CHECK:  }
-//       CHECK:  %[[R:.*]] = sparse_tensor.convert %[[TMP_0]]
-//       CHECK:  return %[[R]] : tensor<?x?xf64, #sparse_tensor.encoding<{{{.*}}}>>
-func.func @concat_sparse_sparse_annotated_dense(%arg0: tensor<2x4xf64, #DCSR>,
-                                %arg1: tensor<3x4xf64, #DCSR>,
-                                %arg2: tensor<4x4xf64, #DCSR>)
-                                -> tensor<?x?xf64, #DENSE> {
-    %0 = sparse_tensor.concatenate %arg0, %arg1, %arg2 {dimension = 0 : index}
-         : tensor<2x4xf64, #DCSR>,
-           tensor<3x4xf64, #DCSR>,
-           tensor<4x4xf64, #DCSR> to tensor<?x?xf64, #DENSE>
-    return %0 : tensor<?x?xf64, #DENSE>
-}
-
-// CHECK-LABEL: @concat_sparse_sparse_annotated_dense_permute(
-//  CHECK-SAME:  %[[TMP_arg0:.*]]: tensor<2x4xf64, #sparse_tensor
-//  CHECK-SAME:  %[[TMP_arg1:.*]]: tensor<3x4xf64, #sparse_tensor
-//  CHECK-SAME:  %[[TMP_arg2:.*]]: tensor<4x4xf64, #sparse_tensor
-//   CHECK-DAG:  %[[TMP_c0:.*]] = arith.constant 0 : index
-//   CHECK-DAG:  %[[TMP_c1:.*]] = arith.constant 1 : index
-//   CHECK-DAG:  %[[TMP_c5:.*]] = arith.constant 5 : index
-//   CHECK-DAG:  %[[TMP_c2:.*]] = arith.constant 2 : index
-//   CHECK-DAG:  %[[TMP_c9:.*]] = arith.constant 9 : index
-//   CHECK-DAG:  %[[TMP_c4:.*]] = arith.constant 4 : index
-//       CHECK:  %[[TMP_0:.*]] = bufferization.alloc_tensor(%[[TMP_c9]], %[[TMP_c4]]) : tensor<?x?xf64, #sparse_tensor
-//       CHECK:  %[[VAL_0:.*]] = sparse_tensor.values %[[TMP_0]] : tensor<?x?xf64, #sparse_tensor
-//       CHECK:  %[[DIM_0:.*]] = memref.alloca() : memref<2xindex>
-//       CHECK:  memref.store %[[TMP_c4]], %[[DIM_0]][%[[TMP_c0]]] : memref<2xindex>
-//       CHECK:  memref.store %[[TMP_c9]], %[[DIM_0]][%[[TMP_c1]]] : memref<2xindex>
-//       CHECK:  %[[VAL_1:.*]] = memref.reshape %[[VAL_0]](%[[DIM_0]]) : (memref<?xf64>, memref<2xindex>) -> memref<?x?xf64>
-//       CHECK:  %[[TMP_1:.*]] = sparse_tensor.positions %[[TMP_arg0]] {level = 0 : index} : tensor<2x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_2:.*]] = sparse_tensor.coordinates %[[TMP_arg0]] {level = 0 : index} : tensor<2x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_3:.*]] = sparse_tensor.positions %[[TMP_arg0]] {level = 1 : index} : tensor<2x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_4:.*]] = sparse_tensor.coordinates %[[TMP_arg0]] {level = 1 : index} : tensor<2x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_5:.*]] = sparse_tensor.values %[[TMP_arg0]] : tensor<2x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_6:.*]] = memref.load %[[TMP_1]][%[[TMP_c0]]] : memref<?xindex>
-//       CHECK:  %[[TMP_7:.*]] = memref.load %[[TMP_1]][%[[TMP_c1]]] : memref<?xindex>
-//       CHECK:  scf.for %[[TMP_arg3:.*]] = %[[TMP_6]] to %[[TMP_7]] step %[[TMP_c1]]
-//       CHECK:    %[[TMP_23:.*]] = memref.load %[[TMP_2]][%[[TMP_arg3]]] : memref<?xindex>
-//   CHECK-DAG:    %[[TMP_25:.*]] = memref.load %[[TMP_3]][%[[TMP_arg3]]] : memref<?xindex>
-//   CHECK-DAG:    %[[TMP_24:.*]] = arith.addi %[[TMP_arg3]], %[[TMP_c1]] : index
-//       CHECK:    %[[TMP_26:.*]] = memref.load %[[TMP_3]][%[[TMP_24]]] : memref<?xindex>
-//       CHECK:    scf.for %[[TMP_arg4:.*]] = %[[TMP_25]] to %[[TMP_26]] step %[[TMP_c1]]
-//       CHECK:      %[[TMP_27:.*]] = memref.load %[[TMP_4]][%[[TMP_arg4]]] : memref<?xindex>
-//       CHECK:      %[[TMP_28:.*]] = memref.load %[[TMP_5]][%[[TMP_arg4]]] : memref<?xf64>
-//       CHECK:      memref.store %[[TMP_28]], %[[VAL_1]][%[[TMP_27]], %[[TMP_23]]] : memref<?x?xf64>
-//       CHECK:    }
-//       CHECK:  }
-//       CHECK:  %[[TMP_8:.*]] = sparse_tensor.positions %[[TMP_arg1]] {level = 0 : index} : tensor<3x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_9:.*]] = sparse_tensor.coordinates %[[TMP_arg1]] {level = 0 : index} : tensor<3x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_10:.*]] = sparse_tensor.positions %[[TMP_arg1]] {level = 1 : index} : tensor<3x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_11:.*]] = sparse_tensor.coordinates %[[TMP_arg1]] {level = 1 : index} : tensor<3x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_12:.*]] = sparse_tensor.values %[[TMP_arg1]] : tensor<3x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_13:.*]] = memref.load %[[TMP_8]][%[[TMP_c0]]] : memref<?xindex>
-//       CHECK:  %[[TMP_14:.*]] = memref.load %[[TMP_8]][%[[TMP_c1]]] : memref<?xindex>
-//       CHECK:  scf.for %[[TMP_arg3:.*]] = %[[TMP_13]] to %[[TMP_14]] step %[[TMP_c1]]
-//       CHECK:    %[[TMP_23:.*]] = memref.load %[[TMP_9]][%[[TMP_arg3]]] : memref<?xindex>
-//   CHECK-DAG:    %[[TMP_25:.*]] = memref.load %[[TMP_10]][%[[TMP_arg3]]] : memref<?xindex>
-//   CHECK-DAG:    %[[TMP_24:.*]] = arith.addi %[[TMP_arg3]], %[[TMP_c1]] : index
-//       CHECK:    %[[TMP_26:.*]] = memref.load %[[TMP_10]][%[[TMP_24]]] : memref<?xindex>
-//       CHECK:    scf.for %[[TMP_arg4:.*]] = %[[TMP_25]] to %[[TMP_26]] step %[[TMP_c1]]
-//       CHECK:      %[[TMP_27:.*]] = memref.load %[[TMP_11]][%[[TMP_arg4]]] : memref<?xindex>
-//       CHECK:      %[[TMP_28:.*]] = memref.load %[[TMP_12]][%[[TMP_arg4]]] : memref<?xf64>
-//       CHECK:      %[[TMP_29:.*]] = arith.addi %[[TMP_23]], %[[TMP_c2]] : index
-//       CHECK:      memref.store %[[TMP_28]], %[[VAL_1]][%[[TMP_27]], %[[TMP_29]]] : memref<?x?xf64>
-//       CHECK:    }
-//       CHECK:  }
-//       CHECK:  %[[TMP_15:.*]] = sparse_tensor.positions %[[TMP_arg2]] {level = 0 : index} : tensor<4x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_16:.*]] = sparse_tensor.coordinates %[[TMP_arg2]] {level = 0 : index} : tensor<4x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_17:.*]] = sparse_tensor.positions %[[TMP_arg2]] {level = 1 : index} : tensor<4x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_18:.*]] = sparse_tensor.coordinates %[[TMP_arg2]] {level = 1 : index} : tensor<4x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_19:.*]] = sparse_tensor.values %[[TMP_arg2]] : tensor<4x4xf64, #sparse_tensor
-//       CHECK:  %[[TMP_20:.*]] = memref.load %[[TMP_15]][%[[TMP_c0]]] : memref<?xindex>
-//       CHECK:  %[[TMP_21:.*]] = memref.load %[[TMP_15]][%[[TMP_c1]]] : memref<?xindex>
-//       CHECK:  scf.for %[[TMP_arg3:.*]] = %[[TMP_20]] to %[[TMP_21]] step %[[TMP_c1]]
-//       CHECK:    %[[TMP_23:.*]] = memref.load %[[TMP_16]][%[[TMP_arg3]]] : memref<?xindex>
-//       CHECK:    %[[TMP_25:.*]] = memref.load %[[TMP_17]][%[[TMP_arg3]]] : memref<?xindex>
-//       CHECK:    %[[TMP_24:.*]] = arith.addi %[[TMP_arg3]], %[[TMP_c1]] : index
-//       CHECK:    %[[TMP_26:.*]] = memref.load %[[TMP_17]][%[[TMP_24]]] : memref<?xindex>
-//       CHECK:    scf.for %[[TMP_arg4:.*]] = %[[TMP_25]] to %[[TMP_26]] step %[[TMP_c1]]
-//       CHECK:      %[[TMP_27:.*]] = memref.load %[[TMP_18]][%[[TMP_arg4]]] : memref<?xindex>
-//       CHECK:      %[[TMP_28:.*]] = memref.load %[[TMP_19]][%[[TMP_arg4]]] : memref<?xf64>
-//       CHECK:      %[[TMP_29:.*]] = arith.addi %[[TMP_23]], %[[TMP_c5]] : index
-//       CHECK:      memref.store %[[TMP_28]], %[[VAL_1]][%[[TMP_27]], %[[TMP_29]]] : memref<?x?xf64>
-//       CHECK:    }
-//       CHECK:  }
-//       CHECK:  %[[R:.*]] = sparse_tensor.convert %[[TMP_0]]
-//       CHECK:  return %[[R]] : tensor<?x?xf64, #sparse_tensor.encoding<{{{.*}}}>>
-func.func @concat_sparse_sparse_annotated_dense_permute(%arg0: tensor<2x4xf64, #DCSR>,
-                                %arg1: tensor<3x4xf64, #DCSR>,
-                                %arg2: tensor<4x4xf64, #DCSR>)
-                                -> tensor<?x?xf64, #DENSE_P> {
-    %0 = sparse_tensor.concatenate %arg0, %arg1, %arg2 {dimension = 0 : index}
-         : tensor<2x4xf64, #DCSR>,
-           tensor<3x4xf64, #DCSR>,
-           tensor<4x4xf64, #DCSR> to tensor<?x?xf64, #DENSE_P>
-    return %0 : tensor<?x?xf64, #DENSE_P>
-}
diff --git a/mlir/test/Dialect/SparseTensor/sparse_expand.mlir b/mlir/test/Dialect/SparseTensor/sparse_expand.mlir
index d19d7fe287..9d8db10aa4 100644
--- a/mlir/test/Dialect/SparseTensor/sparse_expand.mlir
+++ b/mlir/test/Dialect/SparseTensor/sparse_expand.mlir
@@ -62,7 +62,7 @@
 // CHECK-CONVERT: memref.dealloc %[[A]] : memref<?xf64>
 // CHECK-CONVERT: memref.dealloc %[[B]] : memref<?xi1>
 // CHECK-CONVERT: memref.dealloc %[[C]] : memref<?xindex>
-// CHECK-CONVERT: call @endInsert
+// CHECK-CONVERT: call @endLexInsert
 //
 func.func @kernel(%arga: tensor<?x?xf64, #DCSC>) -> tensor<?xf64, #SV> {
   %c0 = arith.constant 0 : index
@@ -115,7 +115,7 @@ func.func @kernel(%arga: tensor<?x?xf64, #DCSC>) -> tensor<?xf64, #SV> {
 // CHECK-CONVERT: memref.dealloc %[[A]] : memref<?xf64>
 // CHECK-CONVERT: memref.dealloc %[[B]] : memref<?xi1>
 // CHECK-CONVERT: memref.dealloc %[[C]] : memref<?xindex>
-// CHECK-CONVERT: call @endInsert
+// CHECK-CONVERT: call @endLexInsert
 //
 func.func @matmul1(%A: tensor<8x2xf64, #CSR>,
                    %B: tensor<2x4xf64, #CSR>) -> tensor<8x4xf64, #CSR> {
@@ -163,7 +163,7 @@ func.func @matmul1(%A: tensor<8x2xf64, #CSR>,
 // CHECK-CONVERT: memref.dealloc %[[A]] : memref<?xf64>
 // CHECK-CONVERT: memref.dealloc %[[B]] : memref<?xi1>
 // CHECK-CONVERT: memref.dealloc %[[C]] : memref<?xindex>
-// CHECK-CONVERT: call @endInsert
+// CHECK-CONVERT: call @endLexInsert
 //
 func.func @matmul2(%A: tensor<8x2xf64, #CSC>,
                    %B: tensor<2x4xf64, #CSC>) -> tensor<8x4xf64, #CSC> {
diff --git a/mlir/test/Dialect/SparseTensor/sparse_fill_zero.mlir b/mlir/test/Dialect/SparseTensor/sparse_fill_zero.mlir
index 7d852ca9cc..8ecbc1da96 100644
--- a/mlir/test/Dialect/SparseTensor/sparse_fill_zero.mlir
+++ b/mlir/test/Dialect/SparseTensor/sparse_fill_zero.mlir
@@ -112,7 +112,7 @@
 // CHECK:           memref.dealloc %[[VAL_20]] : memref<300xf64>
 // CHECK:           memref.dealloc %[[VAL_22]] : memref<300xi1>
 // CHECK:           memref.dealloc %[[VAL_24]] : memref<300xindex>
-// CHECK:           call @endInsert(%[[VAL_19]]) : (!llvm.ptr<i8>) -> ()
+// CHECK:           call @endLexInsert(%[[VAL_19]]) : (!llvm.ptr<i8>) -> ()
 // CHECK:           return %[[VAL_19]] : !llvm.ptr<i8>
 // CHECK:       }
 func.func @fill_zero_after_alloc(%arg0: tensor<100x200xf64, #DCSR>,
diff --git a/mlir/test/Dialect/Tensor/tiling.mlir b/mlir/test/Dialect/Tensor/tiling.mlir
index 63f885c610..556718785f 100644
--- a/mlir/test/Dialect/Tensor/tiling.mlir
+++ b/mlir/test/Dialect/Tensor/tiling.mlir
@@ -150,14 +150,14 @@ transform.sequence failures(propagate) {
 //       CHECK:   %[[RESULT:.*]] = scf.for %[[IV:.*]] = %[[C0]] to %[[C15]] step %[[C3]] iter_args(%[[INNER_OUT:.*]] =
 //       CHECK:     %[[R2:.*]] = scf.if
 //       CHECK:       %[[GEN:.*]] = tensor.generate
-//       CHECK:       %[[cast_0:.*]] = tensor.cast %[[GEN]] : tensor<14x3xf32> to tensor<?x?xf32>
-//       CHECK:       scf.yield %[[cast_0]] : tensor<?x?xf32>
+//       CHECK:       %[[cast_0:.*]] = tensor.cast %[[GEN]] : tensor<14x3xf32> to tensor<?x3xf32>
+//       CHECK:       scf.yield %[[cast_0]] : tensor<?x3xf32>
 //       CHECK:     else
 //       CHECK:       %[[SLICE:.*]] = tensor.extract_slice %arg0[0, %{{.*}}] [7, %{{.*}}] [1, 1] : tensor<7x9xf32> to tensor<7x?xf32>
 //       CHECK:       %[[PAD:.*]] = tensor.pad %[[SLICE]] low[0, 0] high[7, %{{.*}}]
-//       CHECK:       %[[cast_1:.*]] = tensor.cast %[[PAD]] : tensor<14x?xf32> to tensor<?x?xf32>
-//       CHECK:       scf.yield %[[cast_1]] : tensor<?x?xf32>
-//       CHECK:     %[[cast:.*]] = tensor.cast %[[R2]] : tensor<?x?xf32> to tensor<14x3xf32>
+//       CHECK:       %[[cast_1:.*]] = tensor.cast %[[PAD]] : tensor<14x?xf32> to tensor<?x3xf32>
+//       CHECK:       scf.yield %[[cast_1]] : tensor<?x3xf32>
+//       CHECK:     %[[cast:.*]] = tensor.cast %[[R2]] : tensor<?x3xf32> to tensor<14x3xf32>
 //       CHECK:     %[[R3:.*]] = tensor.insert_slice %[[cast]] into %[[INNER_OUT]][0, %[[IV]]] [14, 3] [1, 1] : tensor<14x3xf32> into tensor<14x15xf32>
 //       CHECK:     scf.yield %[[R3]] : tensor<14x15xf32>
 //       CHECK:   return %[[RESULT]] : tensor<14x15xf32>
diff --git a/mlir/test/Dialect/Tosa/canonicalize.mlir b/mlir/test/Dialect/Tosa/canonicalize.mlir
index d36cf6a1d9..dddf15fffb 100644
--- a/mlir/test/Dialect/Tosa/canonicalize.mlir
+++ b/mlir/test/Dialect/Tosa/canonicalize.mlir
@@ -591,3 +591,15 @@ func.func @fold_abs_abs(%arg0: tensor<?x1xf32>) -> tensor<?x1xf32> {
   %1 = tosa.abs %0 : (tensor<?x1xf32>) -> tensor<?x1xf32>
   return %1 : tensor<?x1xf32>
 }
+
+// -----
+
+// CHECK-LABEL: @fold_reduce_rank_zero
+func.func nested @fold_reduce_rank_zero() {
+  // CHECK-NOT: tosa.reduce_min
+  // CHECK-NOT: tosa.reverse
+  %0 = tensor.empty() : tensor<i32>
+  %1 = tosa.reduce_min %0 {axis = 0 : i32} : (tensor<i32>) -> tensor<1x10xi32>
+  %2 = tosa.reverse %0 {axis = 0 : i32} : (tensor<i32>) -> tensor<1x10xi32>
+  return
+}
diff --git a/mlir/test/Dialect/Tosa/constant-op-fold.mlir b/mlir/test/Dialect/Tosa/constant-op-fold.mlir
index 56619fbc56..612e99f198 100644
--- a/mlir/test/Dialect/Tosa/constant-op-fold.mlir
+++ b/mlir/test/Dialect/Tosa/constant-op-fold.mlir
@@ -1,5 +1,8 @@
 // RUN: mlir-opt --split-input-file --tosa-layerwise-constant-fold %s | FileCheck %s
 
+
+// RUN: mlir-opt --split-input-file --tosa-layerwise-constant-fold="aggressive-reduce-constant=true" %s | FileCheck %s --check-prefix=AGGRESIVE
+
 // CHECK-LABEL: @transpose_fold
 func.func @transpose_fold(%arg0: tensor<3x4xf32>) -> tensor<3x4xf32> {
   // CHECK: return %arg0
@@ -1051,3 +1054,57 @@ func.func @reduce_sum_constant() -> tensor<1x3xi32> {
   %0 = tosa.reduce_sum %arg2 {axis = 0 : i32} : (tensor<2x3xi32>) -> tensor<1x3xi32>
   return %0 : tensor<1x3xi32>
 }
+
+// -----
+
+func.func @reduce_sum_constant_aggressive() -> tensor<1x3xi32> {
+  // AGGRESIVE-LABEL: func.func @reduce_sum_constant_aggressive() -> tensor<1x3xi32> {
+  // AGGRESIVE:       %[[VAL_0:.*]] = "tosa.const"() <{value = dense<4> : tensor<1x3xi32>}> : () -> tensor<1x3xi32>
+  // AGGRESIVE:       return %[[VAL_0:.*]] : tensor<1x3xi32>
+  
+  // CHECK-LABEL:     func.func @reduce_sum_constant_aggressive() -> tensor<1x3xi32> {
+  // CHECK:           %[[VAL_0:.*]] = "tosa.const"() <{value = dense<1> : tensor<2x3xi32>}> : () -> tensor<2x3xi32>
+  // CHECK:           %[[VAL_1:.*]] = tosa.reduce_sum %[[VAL_0]] {axis = 0 : i32} : (tensor<2x3xi32>) -> tensor<1x3xi32>
+  // CHECK:           %[[VAL_2:.*]] = tosa.reduce_sum %[[VAL_0]] {axis = 0 : i32} : (tensor<2x3xi32>) -> tensor<1x3xi32>
+  // CHECK:           %[[VAL_3:.*]] = tosa.add %[[VAL_1]], %[[VAL_2]] : (tensor<1x3xi32>, tensor<1x3xi32>) -> tensor<1x3xi32>
+  // CHECK:           return %[[VAL_3]] : tensor<1x3xi32>
+
+  %const = "tosa.const"() {value = dense<1> : tensor<2x3xi32>} : () -> tensor<2x3xi32>
+  %0 = tosa.reduce_sum %const {axis = 0 : i32} : (tensor<2x3xi32>) -> tensor<1x3xi32>
+  %1 = tosa.reduce_sum %const {axis = 0 : i32} : (tensor<2x3xi32>) -> tensor<1x3xi32>
+  %res = tosa.add %0, %1 : (tensor<1x3xi32>, tensor<1x3xi32>) -> tensor<1x3xi32>
+  return %res : tensor<1x3xi32>
+}
+
+// -----
+
+func.func @reduce_sum_constant_aggressive() -> tensor<2x3xi32> {
+  // AGGRESIVE-LABEL:     func.func @reduce_sum_constant_aggressive() -> tensor<2x3xi32> {
+  // AGGRESIVE:           %[[VAL_0:.*]] = "tosa.const"() <{value = dense<2> : tensor<1x2x3xi32>}> : () -> tensor<1x2x3xi32>
+  // AGGRESIVE:           %[[VAL_1:.*]] = "tosa.const"() <{value = dense<1> : tensor<2x2x3xi32>}> : () -> tensor<2x2x3xi32>
+  // AGGRESIVE:           %[[VAL_2:.*]] = "tosa.const"() <{value = dense<2> : tensor<2x3xi32>}> : () -> tensor<2x3xi32>
+  // AGGRESIVE:           %[[VAL_3:.*]] = tosa.argmax %[[VAL_0]] {axis = 0 : i32} : (tensor<1x2x3xi32>) -> tensor<2x3xi32>
+  // AGGRESIVE:           %[[VAL_4:.*]] = tosa.argmax %[[VAL_1]] {axis = 0 : i32} : (tensor<2x2x3xi32>) -> tensor<2x3xi32>
+  // AGGRESIVE:           %[[VAL_5:.*]] = tosa.add %[[VAL_3]], %[[VAL_2]] : (tensor<2x3xi32>, tensor<2x3xi32>) -> tensor<2x3xi32>
+  // AGGRESIVE:           %[[VAL_6:.*]] = tosa.add %[[VAL_5]], %[[VAL_4]] : (tensor<2x3xi32>, tensor<2x3xi32>) -> tensor<2x3xi32>
+  // AGGRESIVE:           return %[[VAL_6]] : tensor<2x3xi32>
+
+  // CHECK-LABEL:     func.func @reduce_sum_constant_aggressive() -> tensor<2x3xi32> {
+  // CHECK:           %[[VAL_0:.*]] = "tosa.const"() <{value = dense<1> : tensor<2x2x3xi32>}> : () -> tensor<2x2x3xi32>
+  // CHECK:           %[[VAL_1:.*]] = "tosa.const"() <{value = dense<2> : tensor<2x3xi32>}> : () -> tensor<2x3xi32>
+  // CHECK:           %[[VAL_2:.*]] = tosa.reduce_sum %[[VAL_0]] {axis = 0 : i32} : (tensor<2x2x3xi32>) -> tensor<1x2x3xi32>
+  // CHECK:           %[[VAL_3:.*]] = tosa.argmax %[[VAL_2]] {axis = 0 : i32} : (tensor<1x2x3xi32>) -> tensor<2x3xi32>
+  // CHECK:           %[[VAL_4:.*]] = tosa.argmax %[[VAL_0]] {axis = 0 : i32} : (tensor<2x2x3xi32>) -> tensor<2x3xi32>
+  // CHECK:           %[[VAL_5:.*]] = tosa.add %[[VAL_3]], %[[VAL_1]] : (tensor<2x3xi32>, tensor<2x3xi32>) -> tensor<2x3xi32>
+  // CHECK:           %[[VAL_6:.*]] = tosa.add %[[VAL_5]], %[[VAL_4]] : (tensor<2x3xi32>, tensor<2x3xi32>) -> tensor<2x3xi32>
+  // CHECK:           return %[[VAL_6]] : tensor<2x3xi32>
+
+  %const0 = "tosa.const"() {value = dense<1> : tensor<2x2x3xi32>} : () -> tensor<2x2x3xi32>
+  %const1 = "tosa.const"() {value = dense<2> : tensor<2x3xi32>} : () -> tensor<2x3xi32>
+  %reduce0 = tosa.reduce_sum %const0 {axis = 0 : i32} : (tensor<2x2x3xi32>) -> tensor<1x2x3xi32>
+  %argmax0 = tosa.argmax %reduce0 {axis = 0 : i32} : (tensor<1x2x3xi32>) -> tensor<2x3xi32>
+  %argmax1 = tosa.argmax %const0 {axis = 0 : i32} : (tensor<2x2x3xi32>) -> tensor<2x3xi32>
+  %res0 = tosa.add %argmax0, %const1 : (tensor<2x3xi32>, tensor<2x3xi32>) -> tensor<2x3xi32>
+  %res1 = tosa.add %res0, %argmax1 : (tensor<2x3xi32>, tensor<2x3xi32>) -> tensor<2x3xi32>
+  return %res1 : tensor<2x3xi32>
+}
diff --git a/mlir/test/Dialect/Tosa/invalid.mlir b/mlir/test/Dialect/Tosa/invalid.mlir
index 7c58bb10b9..9233662e88 100644
--- a/mlir/test/Dialect/Tosa/invalid.mlir
+++ b/mlir/test/Dialect/Tosa/invalid.mlir
@@ -203,3 +203,48 @@ func.func @test_avg_pool2d_zero_dim_input(%arg0: tensor<1x0x?x9xf32>) -> tensor<
       : (tensor<1x0x?x9xf32>) -> tensor<1x7x7x9xf32>
     return %0 : tensor<1x7x7x9xf32>
 }
+
+// -----
+
+func.func @test_variable_duplicates(%arg0: tensor<2x4x8xi32>) -> () {
+  tosa.variable @stored_var = dense<-1> : tensor<2x4x8xi32>
+  // expected-error@+1 {{'tosa.variable' op name has already been declared}}
+  tosa.variable @stored_var : tensor<1x4x8xi32>
+  return
+}
+
+// -----
+
+func.func @test_variable_read_type(%arg0: tensor<2x4x8xi32>) -> () {
+  tosa.variable @stored_var = dense<-1> : tensor<2x4x8xi32>
+  // expected-error@+1 {{'tosa.variable.read' op result type does not equal variable type}}
+  %0 = tosa.variable.read @stored_var : tensor<2x4x8xi16>
+  return
+}
+
+// -----
+
+func.func @test_variable_read_shape(%arg0: tensor<2x4x8xi32>) -> () {
+  tosa.variable @stored_var = dense<-1> : tensor<2x4x8xi32>
+  // expected-error@+1 {{'tosa.variable.read' op result type does not equal variable type}}
+  %0 = tosa.variable.read @stored_var : tensor<1x4x8xi32>
+  return
+}
+
+// -----
+
+func.func @test_variable_write_type(%arg0: tensor<2x4x8xi16>) -> () {
+  tosa.variable @stored_var = dense<-1> : tensor<2x4x8xi32>
+  // expected-error@+1 {{'tosa.variable.write' op operand type does not equal variable type}}
+  tosa.variable.write @stored_var, %arg0 : tensor<2x4x8xi16>
+  return
+}
+
+// -----
+
+func.func @test_variable_write_shape(%arg0: tensor<1x4x8xi32>) -> () {
+  tosa.variable @stored_var = dense<-1> : tensor<2x4x8xi32>
+  // expected-error@+1 {{'tosa.variable.write' op operand type does not equal variable type}}
+  tosa.variable.write @stored_var, %arg0 : tensor<1x4x8xi32>
+  return
+}
diff --git a/mlir/test/Dialect/Tosa/ops.mlir b/mlir/test/Dialect/Tosa/ops.mlir
index 7d7f2d31a4..a3e2b66e03 100644
--- a/mlir/test/Dialect/Tosa/ops.mlir
+++ b/mlir/test/Dialect/Tosa/ops.mlir
@@ -16,6 +16,20 @@ func.func @test_avg_pool2d_f32(%arg0: tensor<1x7x7x9xf32>) -> tensor<1x7x7x9xf32
   return %0 : tensor<1x7x7x9xf32>
 }
 
+// -----
+// CHECK-LABEL: avg_pool2d_f16
+func.func @test_avg_pool2d_f16(%arg0: tensor<1x7x7x9xf16>) -> tensor<1x7x7x9xf16> {
+  %0 = tosa.avg_pool2d %arg0 {acc_type = f16, kernel = array<i64: 2, 2>, pad = array<i64: 0, 1, 0, 1>, stride = array<i64: 1, 1>} : (tensor<1x7x7x9xf16>) -> tensor<1x7x7x9xf16>
+  return %0 : tensor<1x7x7x9xf16>
+}
+
+// -----
+// CHECK-LABEL: avg_pool2d_f16_accumf32
+func.func @test_avg_pool2d_f16_accumf32(%arg0: tensor<1x7x7x9xf16>) -> tensor<1x7x7x9xf16> {
+  %0 = tosa.avg_pool2d %arg0 {acc_type = f32, kernel = array<i64: 2, 2>, pad = array<i64: 0, 1, 0, 1>, stride = array<i64: 1, 1>} : (tensor<1x7x7x9xf16>) -> tensor<1x7x7x9xf16>
+  return %0 : tensor<1x7x7x9xf16>
+}
+
 // -----
 // CHECK-LABEL: avg_pool2d_i8
 func.func @test_avg_pool2d_i8(%arg0: tensor<1x7x7x9xi8>) -> tensor<1x7x7x9xi8> {
@@ -83,12 +97,26 @@ func.func @test_matmul(%arg0: tensor<1x14x19xf32>, %arg1: tensor<1x19x28xf32>) -
 }
 
 // -----
-// CHECK-LABEL: max_pool2d
-func.func @test_max_pool2d(%arg0: tensor<1x32x32x8xf32>) -> tensor<1x32x32x8xf32> {
+// CHECK-LABEL: max_pool2d_f32
+func.func @test_max_pool2d_f32(%arg0: tensor<1x32x32x8xf32>) -> tensor<1x32x32x8xf32> {
   %0 = tosa.max_pool2d %arg0 {kernel = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>} : (tensor<1x32x32x8xf32>) -> tensor<1x32x32x8xf32>
   return %0 : tensor<1x32x32x8xf32>
 }
 
+// -----
+// CHECK-LABEL: max_pool2d_bf16
+func.func @test_max_pool2d_bf16(%arg0: tensor<1x32x32x8xbf16>) -> tensor<1x32x32x8xbf16> {
+  %0 = tosa.max_pool2d %arg0 {kernel = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>} : (tensor<1x32x32x8xbf16>) -> tensor<1x32x32x8xbf16>
+  return %0 : tensor<1x32x32x8xbf16>
+}
+
+// -----
+// CHECK-LABEL: max_pool2d_f16
+func.func @test_max_pool2d_f16(%arg0: tensor<1x32x32x8xf16>) -> tensor<1x32x32x8xf16> {
+  %0 = tosa.max_pool2d %arg0 {kernel = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>} : (tensor<1x32x32x8xf16>) -> tensor<1x32x32x8xf16>
+  return %0 : tensor<1x32x32x8xf16>
+}
+
 // -----
 // CHECK-LABEL: rfft2d
 func.func @test_rfft2d(%arg0: tensor<13x8x16xf32>) -> (tensor<13x8x9xf32>, tensor<13x8x9xf32>) {
@@ -110,6 +138,27 @@ func.func @test_clamp(%arg0: tensor<13x21x3xf32>) -> tensor<13x21x3xf32> {
   return %0 : tensor<13x21x3xf32>
 }
 
+// -----
+// CHECK-LABEL: clamp_f16
+func.func @test_clamp_f16(%arg0: tensor<13x21x3xf16>) -> tensor<13x21x3xf16> {
+  %0 = tosa.clamp %arg0 {min_fp = 0.0 : f16, max_fp = 1.0: f16, min_int = 0 : i64, max_int = 1 : i64} : (tensor<13x21x3xf16>) -> tensor<13x21x3xf16>
+  return %0 : tensor<13x21x3xf16>
+}
+
+// -----
+// CHECK-LABEL: clamp_bf16
+func.func @test_clamp_bf16(%arg0: tensor<13x21x3xbf16>) -> tensor<13x21x3xbf16> {
+  %0 = tosa.clamp %arg0 {min_fp = 0.0 : bf16, max_fp = 1.0: bf16, min_int = 0 : i64, max_int = 1 : i64} : (tensor<13x21x3xbf16>) -> tensor<13x21x3xbf16>
+  return %0 : tensor<13x21x3xbf16>
+}
+
+// -----
+// CHECK-LABEL: clamp_quantized
+func.func @test_clamp_quantized(%arg0: tensor<13x21x3x!quant.uniform<i8:f32, 1.000000e-01:-127>>) -> tensor<13x21x3x!quant.uniform<i8:f32, 1.000000e-01:-127>> {
+  %0 = tosa.clamp %arg0 {min_fp = 0.0 : f32, max_fp = 1.0: f32, min_int = 0 : i64, max_int = 1 : i64} : (tensor<13x21x3x!quant.uniform<i8:f32, 1.000000e-01:-127>>) -> tensor<13x21x3x!quant.uniform<i8:f32, 1.000000e-01:-127>>
+  return %0 : tensor<13x21x3x!quant.uniform<i8:f32, 1.000000e-01:-127>>
+}
+
 // -----
 // CHECK-LABEL: sigmoid
 func.func @test_sigmoid(%arg0: tensor<13x21x3xf32>) -> tensor<13x21x3xf32> {
diff --git a/mlir/test/Dialect/Tosa/variables.mlir b/mlir/test/Dialect/Tosa/variables.mlir
new file mode 100644
index 0000000000..9a26aa0bc8
--- /dev/null
+++ b/mlir/test/Dialect/Tosa/variables.mlir
@@ -0,0 +1,33 @@
+// RUN: mlir-opt %s | mlir-opt | FileCheck %s
+// RUN: mlir-opt %s --mlir-print-op-generic | mlir-opt | FileCheck %s
+
+
+// -----
+// CHECK-LABEL:   @test_variable_scalar(
+// CHECK-SAME:                        %[[ADD_VAL:.*]]: tensor<f32>) {
+func.func @test_variable_scalar(%arg0: tensor<f32>) -> () {
+  // CHECK:           tosa.variable @stored_var = dense<3.140000e+00> : tensor<f32>
+  tosa.variable @stored_var = dense<3.14> : tensor<f32>
+  // CHECK:           %[[STORED_VAL:.*]] = tosa.variable.read @stored_var : tensor<f32>
+  %0 = tosa.variable.read @stored_var : tensor<f32>
+  // CHECK:           %[[RESULT_ADD:.*]] = tosa.add %[[ADD_VAL]], %[[STORED_VAL]] : (tensor<f32>, tensor<f32>) -> tensor<f32>
+  %1 = "tosa.add"(%arg0, %0) : (tensor<f32>, tensor<f32>) -> tensor<f32>
+  // CHECK:           tosa.variable.write @stored_var, %[[RESULT_ADD]] : tensor<f32>
+  tosa.variable.write @stored_var, %1 : tensor<f32>
+  return
+}
+
+// -----
+// CHECK-LABEL:   @test_variable_tensor(
+// CHECK-SAME:                        %[[ADD_VAL:.*]]: tensor<2x4x8xi32>) {
+func.func @test_variable_tensor(%arg0: tensor<2x4x8xi32>) -> () {
+  // CHECK:           tosa.variable @stored_var = dense<-1> : tensor<2x4x8xi32>
+  tosa.variable @stored_var = dense<-1> : tensor<2x4x8xi32>
+  // CHECK:           %[[STORED_VAL:.*]] = tosa.variable.read @stored_var : tensor<2x4x8xi32>
+  %0 = tosa.variable.read @stored_var : tensor<2x4x8xi32>
+  // CHECK:           %[[RESULT_ADD:.*]] = tosa.add %[[ADD_VAL]], %[[STORED_VAL]] : (tensor<2x4x8xi32>, tensor<2x4x8xi32>) -> tensor<2x4x8xi32>
+  %1 = "tosa.add"(%arg0, %0) : (tensor<2x4x8xi32>, tensor<2x4x8xi32>) -> tensor<2x4x8xi32>
+  // CHECK:           tosa.variable.write @stored_var, %[[RESULT_ADD]] : tensor<2x4x8xi32>
+  tosa.variable.write @stored_var, %1 : tensor<2x4x8xi32>
+  return
+}
diff --git a/mlir/test/Dialect/Transform/Library/lower-to-llvm-transform-symbol-def.mlir b/mlir/test/Dialect/Transform/Library/lower-to-llvm-transform-symbol-def.mlir
new file mode 100644
index 0000000000..0ba50bd236
--- /dev/null
+++ b/mlir/test/Dialect/Transform/Library/lower-to-llvm-transform-symbol-def.mlir
@@ -0,0 +1,48 @@
+// RUN: mlir-opt %s
+
+/// Schedule to lower to LLVM.
+module @lower_module_to_llvm attributes { transform.with_named_sequence } {
+
+transform.named_sequence @lower_to_llvm(
+    %module: !transform.any_op {transform.readonly}) -> !transform.any_op {
+
+  %func = transform.structured.match ops{["func.func"]} in %module : (!transform.any_op) -> !transform.any_op
+  %f = transform.apply_registered_pass "convert-vector-to-scf" to %func : (!transform.any_op) -> !transform.any_op
+  %f2 = transform.apply_registered_pass "convert-linalg-to-loops" to %f : (!transform.any_op) -> !transform.any_op
+  %f3 = transform.apply_registered_pass "convert-scf-to-cf" to %f2 : (!transform.any_op) -> !transform.any_op
+  %f4 = transform.apply_registered_pass "expand-strided-metadata" to %f3 : (!transform.any_op) -> !transform.any_op
+  %f5 = transform.apply_registered_pass "lower-affine" to %f4 : (!transform.any_op) -> !transform.any_op
+
+  transform.apply_conversion_patterns to %f5 {
+    transform.apply_conversion_patterns.dialect_to_llvm "math"
+    transform.apply_conversion_patterns.vector.vector_to_llvm
+    transform.apply_conversion_patterns.dialect_to_llvm "memref"
+    transform.apply_conversion_patterns.func.func_to_llvm
+    transform.apply_conversion_patterns.dialect_to_llvm "index"
+    transform.apply_conversion_patterns.dialect_to_llvm "arith"
+    transform.apply_conversion_patterns.dialect_to_llvm "cf"
+  } with type_converter {
+    transform.apply_conversion_patterns.memref.memref_to_llvm_type_converter
+      {index_bitwidth = 64,
+       use_bare_ptr = false,
+       use_bare_ptr_memref_call_conv = false,
+       use_opaque_pointers = true}
+  } {
+    legal_dialects = ["llvm"],
+    partial_conversion
+  } : !transform.any_op
+
+  // Need to rematch here because:
+  //   1. applying reconcile-unrealized-casts on the whole module yields the
+  //      transform applies to transform, when called from a named sequence, at
+  //      this time.
+  //   2. apply_conversion patterns consumes the func but does not produce 
+  //      a new llvm.func.
+  %f6 = transform.structured.match ops{["llvm.func"]} in %module 
+    : (!transform.any_op) -> !transform.any_op
+  %f7 = transform.apply_registered_pass "reconcile-unrealized-casts" to %f6
+    : (!transform.any_op) -> !transform.any_op
+  transform.yield %module : !transform.any_op
+}
+
+} // transform module
diff --git a/mlir/test/Dialect/Transform/include/Library/lower-to-llvm.mlir b/mlir/test/Dialect/Transform/include/Library/lower-to-llvm.mlir
new file mode 100644
index 0000000000..afd1c89dd2
--- /dev/null
+++ b/mlir/test/Dialect/Transform/include/Library/lower-to-llvm.mlir
@@ -0,0 +1,49 @@
+// RUN: mlir-opt %s
+// No need to check anything else than parsing here, this is being used by another test as data.
+
+/// Schedule to lower to LLVM.
+module @lower_module_to_llvm attributes { transform.with_named_sequence } {
+
+transform.named_sequence @lower_to_llvm(
+    %module: !transform.any_op {transform.readonly}) -> !transform.any_op {
+
+  %func = transform.structured.match ops{["func.func"]} in %module : (!transform.any_op) -> !transform.any_op
+  %f = transform.apply_registered_pass "convert-vector-to-scf" to %func : (!transform.any_op) -> !transform.any_op
+  %f2 = transform.apply_registered_pass "convert-linalg-to-loops" to %f : (!transform.any_op) -> !transform.any_op
+  %f3 = transform.apply_registered_pass "convert-scf-to-cf" to %f2 : (!transform.any_op) -> !transform.any_op
+  %f4 = transform.apply_registered_pass "expand-strided-metadata" to %f3 : (!transform.any_op) -> !transform.any_op
+  %f5 = transform.apply_registered_pass "lower-affine" to %f4 : (!transform.any_op) -> !transform.any_op
+
+  transform.apply_conversion_patterns to %f5 {
+    transform.apply_conversion_patterns.dialect_to_llvm "math"
+    transform.apply_conversion_patterns.vector.vector_to_llvm
+    transform.apply_conversion_patterns.dialect_to_llvm "memref"
+    transform.apply_conversion_patterns.func.func_to_llvm
+    transform.apply_conversion_patterns.dialect_to_llvm "index"
+    transform.apply_conversion_patterns.dialect_to_llvm "arith"
+    transform.apply_conversion_patterns.dialect_to_llvm "cf"
+  } with type_converter {
+    transform.apply_conversion_patterns.memref.memref_to_llvm_type_converter
+      {index_bitwidth = 64,
+       use_bare_ptr = false,
+       use_bare_ptr_memref_call_conv = false,
+       use_opaque_pointers = true}
+  } {
+    legal_dialects = ["llvm"],
+    partial_conversion
+  } : !transform.any_op
+
+  // Need to rematch here because:
+  //   1. applying reconcile-unrealized-casts on the whole module yields the
+  //      transform applies to transform, when called from a named sequence, at
+  //      this time.
+  //   2. apply_conversion patterns consumes the func but does not produce 
+  //      a new llvm.func.
+  %f6 = transform.structured.match ops{["llvm.func"]} in %module 
+    : (!transform.any_op) -> !transform.any_op
+  %f7 = transform.apply_registered_pass "reconcile-unrealized-casts" to %f6
+    : (!transform.any_op) -> !transform.any_op
+  transform.yield %module : !transform.any_op
+}
+
+} // transform module
diff --git a/mlir/test/Dialect/Transform/test-interpreter-external-concurrent-source.mlir b/mlir/test/Dialect/Transform/include/test-interpreter-external-concurrent-source.mlir
similarity index 100%
rename from mlir/test/Dialect/Transform/test-interpreter-external-concurrent-source.mlir
rename to mlir/test/Dialect/Transform/include/test-interpreter-external-concurrent-source.mlir
diff --git a/mlir/test/Dialect/Transform/test-interpreter-external-source.mlir b/mlir/test/Dialect/Transform/include/test-interpreter-external-source.mlir
similarity index 100%
rename from mlir/test/Dialect/Transform/test-interpreter-external-source.mlir
rename to mlir/test/Dialect/Transform/include/test-interpreter-external-source.mlir
diff --git a/mlir/test/Dialect/Transform/test-interpreter-external-symbol-def-invalid.mlir b/mlir/test/Dialect/Transform/include/test-interpreter-external-symbol-def-invalid.mlir
similarity index 100%
rename from mlir/test/Dialect/Transform/test-interpreter-external-symbol-def-invalid.mlir
rename to mlir/test/Dialect/Transform/include/test-interpreter-external-symbol-def-invalid.mlir
diff --git a/mlir/test/Dialect/Transform/test-interpreter-library/definitions-self-contained.mlir b/mlir/test/Dialect/Transform/include/test-interpreter-library/definitions-self-contained.mlir
similarity index 96%
rename from mlir/test/Dialect/Transform/test-interpreter-library/definitions-self-contained.mlir
rename to mlir/test/Dialect/Transform/include/test-interpreter-library/definitions-self-contained.mlir
index 66f0f1f626..58a8f76c57 100644
--- a/mlir/test/Dialect/Transform/test-interpreter-library/definitions-self-contained.mlir
+++ b/mlir/test/Dialect/Transform/include/test-interpreter-library/definitions-self-contained.mlir
@@ -1,4 +1,5 @@
 // RUN: mlir-opt %s
+// No need to check anything else than parsing here, this is being used by another test as data.
 
 module attributes {transform.with_named_sequence} {
   transform.named_sequence private @private_helper(%arg0: !transform.any_op {transform.readonly}) {
diff --git a/mlir/test/Dialect/Transform/test-interpreter-library/definitions-with-unresolved.mlir b/mlir/test/Dialect/Transform/include/test-interpreter-library/definitions-with-unresolved.mlir
similarity index 78%
rename from mlir/test/Dialect/Transform/test-interpreter-library/definitions-with-unresolved.mlir
rename to mlir/test/Dialect/Transform/include/test-interpreter-library/definitions-with-unresolved.mlir
index b3d076f469..a3b315952b 100644
--- a/mlir/test/Dialect/Transform/test-interpreter-library/definitions-with-unresolved.mlir
+++ b/mlir/test/Dialect/Transform/include/test-interpreter-library/definitions-with-unresolved.mlir
@@ -1,4 +1,5 @@
 // RUN: mlir-opt %s
+// No need to check anything else than parsing here, this is being used by another test as data.
 
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @print_message(%arg0: !transform.any_op {transform.readonly})
diff --git a/mlir/test/Dialect/Transform/interpreter-entry-point.mlir b/mlir/test/Dialect/Transform/interpreter-entry-point.mlir
new file mode 100644
index 0000000000..ccd9bef3d5
--- /dev/null
+++ b/mlir/test/Dialect/Transform/interpreter-entry-point.mlir
@@ -0,0 +1,17 @@
+// RUN: mlir-opt %s -transform-interpreter=entry-point=entry_point \
+// RUN:   -split-input-file -verify-diagnostics
+
+module attributes { transform.with_named_sequence } {
+  transform.named_sequence @entry_point(!transform.any_op {transform.readonly}) {
+  ^bb0(%arg0: !transform.any_op):
+    // expected-remark @below {{applying transformation}}
+    transform.test_transform_op
+    transform.yield
+  }
+
+  transform.named_sequence @__transform_main(!transform.any_op {transform.readonly}) {
+  ^bb0(%arg0: !transform.any_op):
+    transform.test_transform_op // Note: does not yield remark.
+    transform.yield
+  }
+}
diff --git a/mlir/test/Dialect/Transform/interpreter.mlir b/mlir/test/Dialect/Transform/interpreter.mlir
new file mode 100644
index 0000000000..bb41420bef
--- /dev/null
+++ b/mlir/test/Dialect/Transform/interpreter.mlir
@@ -0,0 +1,17 @@
+// RUN: mlir-opt %s -transform-interpreter \
+// RUN:   -split-input-file -verify-diagnostics
+
+module attributes { transform.with_named_sequence } {
+  transform.named_sequence @__transform_main(!transform.any_op {transform.readonly}) {
+  ^bb0(%arg0: !transform.any_op):
+    // expected-remark @below {{applying transformation}}
+    transform.test_transform_op
+    transform.yield
+  }
+
+  transform.named_sequence @entry_point(!transform.any_op {transform.readonly}) {
+  ^bb0(%arg0: !transform.any_op):
+    transform.test_transform_op // Note: does not yield remark.
+    transform.yield
+  }
+}
diff --git a/mlir/test/Dialect/Transform/preload-library.mlir b/mlir/test/Dialect/Transform/preload-library.mlir
new file mode 100644
index 0000000000..9beefa44d6
--- /dev/null
+++ b/mlir/test/Dialect/Transform/preload-library.mlir
@@ -0,0 +1,19 @@
+// RUN: mlir-opt %s \
+// RUN:   -transform-preload-library=transform-library-paths=%p%{fs-sep}include%{fs-sep}test-interpreter-library \
+// RUN:   -transform-interpreter=entry-point=private_helper \
+// RUN:   -split-input-file -verify-diagnostics
+
+// expected-remark @below {{message}}
+module {}
+
+// -----
+
+// Note: no remark here since local entry point takes precedence.
+module attributes { transform.with_named_sequence } {
+  transform.named_sequence @private_helper(!transform.any_op {transform.readonly}) {
+  ^bb0(%arg0: !transform.any_op):
+    // expected-remark @below {{applying transformation}}
+    transform.test_transform_op
+    transform.yield
+  }
+}
diff --git a/mlir/test/Dialect/Transform/test-interpreter-external-concurrent.mlir b/mlir/test/Dialect/Transform/test-interpreter-external-concurrent.mlir
index 46a1a130d9..59c2b672a6 100644
--- a/mlir/test/Dialect/Transform/test-interpreter-external-concurrent.mlir
+++ b/mlir/test/Dialect/Transform/test-interpreter-external-concurrent.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s --pass-pipeline="builtin.module(func.func(test-transform-dialect-interpreter{transform-file-name=%p/test-interpreter-external-concurrent-source.mlir}))" \
+// RUN: mlir-opt %s --pass-pipeline="builtin.module(func.func(test-transform-dialect-interpreter{transform-file-name=%p%{fs-sep}include%{fs-sep}test-interpreter-external-concurrent-source.mlir}))" \
 // RUN:             --verify-diagnostics
 
 // Exercising the pass on multiple functions of different lengths that may be
diff --git a/mlir/test/Dialect/Transform/test-interpreter-external-symbol-decl-and-schedule.mlir b/mlir/test/Dialect/Transform/test-interpreter-external-symbol-decl-and-schedule.mlir
index 2c4812bf32..9e50ec1efa 100644
--- a/mlir/test/Dialect/Transform/test-interpreter-external-symbol-decl-and-schedule.mlir
+++ b/mlir/test/Dialect/Transform/test-interpreter-external-symbol-decl-and-schedule.mlir
@@ -1,7 +1,7 @@
-// RUN: mlir-opt %s --pass-pipeline="builtin.module(test-transform-dialect-interpreter{transform-file-name=%p/test-interpreter-external-symbol-decl.mlir transform-library-paths=%p/test-interpreter-library/definitions-self-contained.mlir})" \
+// RUN: mlir-opt %s --pass-pipeline="builtin.module(test-transform-dialect-interpreter{transform-file-name=%p%{fs-sep}test-interpreter-external-symbol-decl.mlir transform-library-paths=%p%{fs-sep}include%{fs-sep}test-interpreter-library/definitions-self-contained.mlir})" \
 // RUN:             --verify-diagnostics
 
-// RUN: mlir-opt %s --pass-pipeline="builtin.module(test-transform-dialect-interpreter{transform-file-name=%p/test-interpreter-external-symbol-decl.mlir transform-library-paths=%p/test-interpreter-library/definitions-self-contained.mlir}, test-transform-dialect-interpreter{transform-file-name=%p/test-interpreter-external-symbol-decl.mlir transform-library-paths=%p/test-interpreter-library/definitions-self-contained.mlir})" \
+// RUN: mlir-opt %s --pass-pipeline="builtin.module(test-transform-dialect-interpreter{transform-file-name=%p%{fs-sep}test-interpreter-external-symbol-decl.mlir transform-library-paths=%p%{fs-sep}include%{fs-sep}test-interpreter-library/definitions-self-contained.mlir}, test-transform-dialect-interpreter{transform-file-name=%p%{fs-sep}test-interpreter-external-symbol-decl.mlir transform-library-paths=%p%{fs-sep}include%{fs-sep}test-interpreter-library/definitions-self-contained.mlir})" \
 // RUN:             --verify-diagnostics
 
 // The external transform script has a declaration to the named sequence @foo,
diff --git a/mlir/test/Dialect/Transform/test-interpreter-external-symbol-decl-dir.mlir b/mlir/test/Dialect/Transform/test-interpreter-external-symbol-decl-dir.mlir
index 8b8254976e..3681b913dc 100644
--- a/mlir/test/Dialect/Transform/test-interpreter-external-symbol-decl-dir.mlir
+++ b/mlir/test/Dialect/Transform/test-interpreter-external-symbol-decl-dir.mlir
@@ -1,10 +1,10 @@
-// RUN: mlir-opt %s --pass-pipeline="builtin.module(test-transform-dialect-interpreter{transform-library-paths=%p%{fs-sep}test-interpreter-library})" \
+// RUN: mlir-opt %s --pass-pipeline="builtin.module(test-transform-dialect-interpreter{transform-library-paths=%p%{fs-sep}include%{fs-sep}test-interpreter-library})" \
 // RUN:             --verify-diagnostics --split-input-file | FileCheck %s
 
-// RUN: mlir-opt %s --pass-pipeline="builtin.module(test-transform-dialect-interpreter{transform-library-paths=%p%{fs-sep}test-interpreter-library/definitions-self-contained.mlir,%p%{fs-sep}test-interpreter-library/definitions-with-unresolved.mlir})" \
+// RUN: mlir-opt %s --pass-pipeline="builtin.module(test-transform-dialect-interpreter{transform-library-paths=%p%{fs-sep}include%{fs-sep}test-interpreter-library/definitions-self-contained.mlir,%p%{fs-sep}include%{fs-sep}test-interpreter-library/definitions-with-unresolved.mlir})" \
 // RUN:             --verify-diagnostics --split-input-file | FileCheck %s
 
-// RUN: mlir-opt %s --pass-pipeline="builtin.module(test-transform-dialect-interpreter{transform-library-paths=%p%{fs-sep}test-interpreter-library}, test-transform-dialect-interpreter)" \
+// RUN: mlir-opt %s --pass-pipeline="builtin.module(test-transform-dialect-interpreter{transform-library-paths=%p%{fs-sep}include%{fs-sep}test-interpreter-library}, test-transform-dialect-interpreter)" \
 // RUN:             --verify-diagnostics --split-input-file | FileCheck %s
 
 // The definition of the @foo named sequence is provided in another file. It
diff --git a/mlir/test/Dialect/Transform/test-interpreter-external-symbol-decl-invalid.mlir b/mlir/test/Dialect/Transform/test-interpreter-external-symbol-decl-invalid.mlir
index c1bd071dc1..060dab334e 100644
--- a/mlir/test/Dialect/Transform/test-interpreter-external-symbol-decl-invalid.mlir
+++ b/mlir/test/Dialect/Transform/test-interpreter-external-symbol-decl-invalid.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s --pass-pipeline="builtin.module(test-transform-dialect-interpreter{transform-library-paths=%p/test-interpreter-external-symbol-def-invalid.mlir}, test-transform-dialect-interpreter)" \
+// RUN: mlir-opt %s --pass-pipeline="builtin.module(test-transform-dialect-interpreter{transform-library-paths=%p%{fs-sep}include%{fs-sep}test-interpreter-external-symbol-def-invalid.mlir}, test-transform-dialect-interpreter)" \
 // RUN:             --verify-diagnostics --split-input-file
 
 // The definition of the @print_message named sequence is provided in another file. It
diff --git a/mlir/test/Dialect/Transform/test-interpreter-external-symbol-decl.mlir b/mlir/test/Dialect/Transform/test-interpreter-external-symbol-decl.mlir
index 339e62072c..8a35e981bd 100644
--- a/mlir/test/Dialect/Transform/test-interpreter-external-symbol-decl.mlir
+++ b/mlir/test/Dialect/Transform/test-interpreter-external-symbol-decl.mlir
@@ -1,7 +1,7 @@
-// RUN: mlir-opt %s --pass-pipeline="builtin.module(test-transform-dialect-interpreter{transform-library-paths=%p/test-interpreter-library/definitions-self-contained.mlir})" \
+// RUN: mlir-opt %s --pass-pipeline="builtin.module(test-transform-dialect-interpreter{transform-library-paths=%p%{fs-sep}include%{fs-sep}test-interpreter-library/definitions-self-contained.mlir})" \
 // RUN:             --verify-diagnostics --split-input-file | FileCheck %s
 
-// RUN: mlir-opt %s --pass-pipeline="builtin.module(test-transform-dialect-interpreter{transform-library-paths=%p/test-interpreter-library/definitions-self-contained.mlir}, test-transform-dialect-interpreter)" \
+// RUN: mlir-opt %s --pass-pipeline="builtin.module(test-transform-dialect-interpreter{transform-library-paths=%p%{fs-sep}include%{fs-sep}test-interpreter-library/definitions-self-contained.mlir}, test-transform-dialect-interpreter)" \
 // RUN:             --verify-diagnostics --split-input-file | FileCheck %s
 
 // The definition of the @print_message named sequence is provided in another
diff --git a/mlir/test/Dialect/Transform/test-interpreter-external.mlir b/mlir/test/Dialect/Transform/test-interpreter-external.mlir
index 5ac6b66c81..ba8e0c6870 100644
--- a/mlir/test/Dialect/Transform/test-interpreter-external.mlir
+++ b/mlir/test/Dialect/Transform/test-interpreter-external.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s --pass-pipeline="builtin.module(test-transform-dialect-interpreter{transform-file-name=%p/test-interpreter-external-source.mlir})" \
+// RUN: mlir-opt %s --pass-pipeline="builtin.module(test-transform-dialect-interpreter{transform-file-name=%p%{fs-sep}include%{fs-sep}test-interpreter-external-source.mlir})" \
 // RUN:             --verify-diagnostics
 
 // The schedule in the separate file emits remarks at the payload root.
diff --git a/mlir/test/Dialect/Transform/test-interpreter-multiple-top-level-ops.mlir b/mlir/test/Dialect/Transform/test-interpreter-multiple-top-level-ops.mlir
new file mode 100644
index 0000000000..db7fecdf75
--- /dev/null
+++ b/mlir/test/Dialect/Transform/test-interpreter-multiple-top-level-ops.mlir
@@ -0,0 +1,26 @@
+// RUN: mlir-opt %s --test-transform-dialect-interpreter='enforce-single-top-level-transform-op=0' -allow-unregistered-dialect --split-input-file --verify-diagnostics | FileCheck %s
+
+transform.sequence failures(propagate) {
+// CHECK: transform.sequence
+^bb0(%arg0: !transform.any_op):
+}
+
+transform.sequence failures(propagate) {
+// CHECK: transform.sequence
+^bb0(%arg0: !transform.any_op):
+}
+
+// -----
+
+transform.sequence failures(propagate) {
+^bb0(%arg0: !transform.any_op):
+  %match = transform.structured.match ops{["transform.get_parent_op"]} in %arg0 : (!transform.any_op) -> !transform.any_op
+  transform.test_print_remark_at_operand %match, "found get_parent_op" : !transform.any_op
+}
+
+transform.sequence failures(propagate) {
+^bb0(%arg0: !transform.any_op):
+  %op = transform.structured.match ops{[]} in %arg0 : (!transform.any_op) -> !transform.any_op
+  // expected-remark @below{{found get_parent_op}}
+  %1 = transform.get_parent_op %op : (!transform.any_op) -> !transform.any_op
+}
diff --git a/mlir/test/Dialect/Vector/invalid.mlir b/mlir/test/Dialect/Vector/invalid.mlir
index 5967a8d69b..504ac89659 100644
--- a/mlir/test/Dialect/Vector/invalid.mlir
+++ b/mlir/test/Dialect/Vector/invalid.mlir
@@ -1169,7 +1169,7 @@ func.func @reduce_unsupported_attr(%arg0: vector<16xf32>) -> i32 {
 // -----
 
 func.func @reduce_unsupported_third_argument(%arg0: vector<16xf32>, %arg1: f32) -> f32 {
-  // expected-error@+1 {{'vector.reduction' unsupported number of operands}}
+  // expected-error@+1 {{expected ':'}}
   %0 = vector.reduction <add>, %arg0, %arg1, %arg1 : vector<16xf32> into f32
 }
 
diff --git a/mlir/test/Dialect/Vector/vector-transferop-opt.mlir b/mlir/test/Dialect/Vector/vector-transferop-opt.mlir
index f43367ab4a..13957af014 100644
--- a/mlir/test/Dialect/Vector/vector-transferop-opt.mlir
+++ b/mlir/test/Dialect/Vector/vector-transferop-opt.mlir
@@ -256,3 +256,107 @@ func.func @collapse_shape(%in_0: memref<1x20x1xi32>, %vec: vector<4xi32>) {
   }
   return
 }
+
+// CHECK-LABEL: func @forward_dead_store_dynamic_same_index
+//   CHECK-NOT:   vector.transfer_write
+//   CHECK-NOT:   vector.transfer_read
+//       CHECK:   scf.for
+//       CHECK:   }
+//       CHECK:   vector.transfer_write
+//       CHECK:   return
+func.func @forward_dead_store_dynamic_same_index(
+    %buffer : memref<?x?xf32>, %v0 : vector<4xf32>, %v1 : vector<4xf32>, %i : index) {
+  %c1 = arith.constant 1 : index
+  %c4 = arith.constant 4 : index
+  %c0 = arith.constant 0 : index
+  %cf0 = arith.constant 0.0 : f32
+  vector.transfer_write %v0, %buffer[%i, %i] {in_bounds = [true]} : vector<4xf32>, memref<?x?xf32>
+  // The following transfer op reads/writes to the same address so that we can forward.
+  %0 = vector.transfer_read %buffer[%i, %i], %cf0 {in_bounds = [true]} : memref<?x?xf32>, vector<4xf32>
+  %x = scf.for %i0 = %c0 to %c4 step %c1 iter_args(%acc = %0) -> (vector<4xf32>) {
+    %1 = arith.addf %acc, %acc : vector<4xf32>
+    scf.yield %1 : vector<4xf32>
+  }
+  vector.transfer_write %x, %buffer[%i, %i] {in_bounds = [true]} : vector<4xf32>, memref<?x?xf32>
+  return
+}
+
+//   CHECK-LABEL: func @dont_forward_dead_store_dynamic_overlap
+// CHECK-COUNT-2:   vector.transfer_write
+//         CHECK:   vector.transfer_read
+//         CHECK:   scf.for
+//         CHECK:   }
+//         CHECK:   vector.transfer_write
+//         CHECK:   return
+func.func @dont_forward_dead_store_dynamic_overlap(
+    %buffer : memref<?x?xf32>, %v0 : vector<4xf32>, %v1 : vector<4xf32>, %i0 : index) {
+  %c1 = arith.constant 1 : index
+  %c4 = arith.constant 4 : index
+  %c0 = arith.constant 0 : index
+  %cf0 = arith.constant 0.0 : f32
+  %i1 = affine.apply affine_map<(d0) -> (d0 + 3)>(%i0)
+  vector.transfer_write %v0, %buffer[%i0, %i0] {in_bounds = [true]} : vector<4xf32>, memref<?x?xf32>
+  // The following transfer op writes to an overlapping range so we cannot forward.
+  vector.transfer_write %v0, %buffer[%i0, %i1] {in_bounds = [true]} : vector<4xf32>, memref<?x?xf32>
+  %0 = vector.transfer_read %buffer[%i0, %i0], %cf0 {in_bounds = [true]} : memref<?x?xf32>, vector<4xf32>
+  %x = scf.for %iv = %c0 to %c4 step %c1 iter_args(%acc = %0) -> (vector<4xf32>) {
+    %1 = arith.addf %acc, %acc : vector<4xf32>
+    scf.yield %1 : vector<4xf32>
+  }
+  vector.transfer_write %x, %buffer[%i0, %i0] {in_bounds = [true]} : vector<4xf32>, memref<?x?xf32>
+  return
+}
+
+// CHECK-LABEL: func @forward_dead_store_dynamic_non_overlap_leading_dim
+//       CHECK:   vector.transfer_write
+//   CHECK-NOT:   vector.transfer_write
+//   CHECK-NOT:   vector.transfer_read
+//       CHECK:   scf.for
+//       CHECK:   }
+//       CHECK:   vector.transfer_write
+//       CHECK:   return
+func.func @forward_dead_store_dynamic_non_overlap_leading_dim(
+    %buffer : memref<?x?xf32>, %v0 : vector<4xf32>, %v1 : vector<4xf32>, %i0 : index) {
+  %c1 = arith.constant 1 : index
+  %c4 = arith.constant 4 : index
+  %c0 = arith.constant 0 : index
+  %cf0 = arith.constant 0.0 : f32
+  %i1 = affine.apply affine_map<(d0) -> (d0 + 1)>(%i0)
+  vector.transfer_write %v0, %buffer[%i0, %i0] {in_bounds = [true]} : vector<4xf32>, memref<?x?xf32>
+  // The following transfer op writes to an non-overlapping range so we can forward.
+  vector.transfer_write %v0, %buffer[%i1, %i0] {in_bounds = [true]} : vector<4xf32>, memref<?x?xf32>
+  %0 = vector.transfer_read %buffer[%i0, %i0], %cf0 {in_bounds = [true]} : memref<?x?xf32>, vector<4xf32>
+  %x = scf.for %iv = %c0 to %c4 step %c1 iter_args(%acc = %0) -> (vector<4xf32>) {
+    %1 = arith.addf %acc, %acc : vector<4xf32>
+    scf.yield %1 : vector<4xf32>
+  }
+  vector.transfer_write %x, %buffer[%i0, %i0] {in_bounds = [true]} : vector<4xf32>, memref<?x?xf32>
+  return
+}
+
+// CHECK-LABEL: func @forward_dead_store_dynamic_non_overlap_trailing_dim
+//       CHECK:   vector.transfer_write
+//   CHECK-NOT:   vector.transfer_write
+//   CHECK-NOT:   vector.transfer_read
+//       CHECK:   scf.for
+//       CHECK:   }
+//       CHECK:   vector.transfer_write
+//       CHECK:   return
+func.func @forward_dead_store_dynamic_non_overlap_trailing_dim(
+    %buffer : memref<?x?xf32>, %v0 : vector<4xf32>, %v1 : vector<4xf32>, %i0 : index) {
+  %c1 = arith.constant 1 : index
+  %c4 = arith.constant 4 : index
+  %c0 = arith.constant 0 : index
+  %cf0 = arith.constant 0.0 : f32
+  %i1 = affine.apply affine_map<(d0) -> (d0 + 4)>(%i0)
+  vector.transfer_write %v0, %buffer[%i0, %i0] {in_bounds = [true]} : vector<4xf32>, memref<?x?xf32>
+  // The following transfer op writes to an non-overlapping range so we can forward.
+  vector.transfer_write %v0, %buffer[%i0, %i1] {in_bounds = [true]} : vector<4xf32>, memref<?x?xf32>
+  %0 = vector.transfer_read %buffer[%i0, %i0], %cf0 {in_bounds = [true]} : memref<?x?xf32>, vector<4xf32>
+  %x = scf.for %iv = %c0 to %c4 step %c1 iter_args(%acc = %0) -> (vector<4xf32>) {
+    %1 = arith.addf %acc, %acc : vector<4xf32>
+    scf.yield %1 : vector<4xf32>
+  }
+  vector.transfer_write %x, %buffer[%i0, %i0] {in_bounds = [true]} : vector<4xf32>, memref<?x?xf32>
+  return
+}
diff --git a/mlir/test/IR/invalid-builtin-attributes.mlir b/mlir/test/IR/invalid-builtin-attributes.mlir
index 1ff44605cb..431c7b12b8 100644
--- a/mlir/test/IR/invalid-builtin-attributes.mlir
+++ b/mlir/test/IR/invalid-builtin-attributes.mlir
@@ -587,3 +587,5 @@ func.func @duplicate_dictionary_attr_key() {
 #attr = distinct[0]<42 : i32>
 // expected-error@below {{referenced attribute does not match previous definition: 42 : i32}}
 #attr1 = distinct[0]<43 : i32>
+
+// -----
diff --git a/mlir/test/IR/operand.mlir b/mlir/test/IR/operand.mlir
index e44133f6ba..507e37c775 100644
--- a/mlir/test/IR/operand.mlir
+++ b/mlir/test/IR/operand.mlir
@@ -13,7 +13,7 @@ func.func @correct_variadic_operand(%arg0: tensor<f32>, %arg1: f32) {
 // -----
 
 func.func @error_in_first_variadic_operand(%arg0: tensor<f32>, %arg1: f32) {
-  // expected-error @+1 {{operand #1 must be tensor of any type}}
+  // expected-error @+1 {{operand #1 must be variadic of tensor of any type}}
   "test.mixed_normal_variadic_operand"(%arg0, %arg1, %arg0, %arg0, %arg0) : (tensor<f32>, f32, tensor<f32>, tensor<f32>, tensor<f32>) -> ()
   return
 }
@@ -29,7 +29,7 @@ func.func @error_in_normal_operand(%arg0: tensor<f32>, %arg1: f32) {
 // -----
 
 func.func @error_in_second_variadic_operand(%arg0: tensor<f32>, %arg1: f32) {
-  // expected-error @+1 {{operand #3 must be tensor of any type}}
+  // expected-error @+1 {{operand #3 must be variadic of tensor of any type}}
   "test.mixed_normal_variadic_operand"(%arg0, %arg0, %arg0, %arg1, %arg0) : (tensor<f32>, tensor<f32>, tensor<f32>, f32, tensor<f32>) -> ()
   return
 }
diff --git a/mlir/test/IR/result.mlir b/mlir/test/IR/result.mlir
index e7d41a50a3..1e4eb3bede 100644
--- a/mlir/test/IR/result.mlir
+++ b/mlir/test/IR/result.mlir
@@ -13,7 +13,7 @@ func.func @correct_variadic_result() -> tensor<f32> {
 // -----
 
 func.func @error_in_first_variadic_result() -> tensor<f32> {
-  // expected-error @+1 {{result #1 must be tensor of any type}}
+  // expected-error @+1 {{result #1 must be variadic of tensor of any type}}
   %0:5 = "test.mixed_normal_variadic_result"() : () -> (tensor<f32>, f32, tensor<f32>, tensor<f32>, tensor<f32>)
   return %0#4 : tensor<f32>
 }
@@ -29,7 +29,7 @@ func.func @error_in_normal_result() -> tensor<f32> {
 // -----
 
 func.func @error_in_second_variadic_result() -> tensor<f32> {
-  // expected-error @+1 {{result #3 must be tensor of any type}}
+  // expected-error @+1 {{result #3 must be variadic of tensor of any type}}
   %0:5 = "test.mixed_normal_variadic_result"() : () -> (tensor<f32>, tensor<f32>, tensor<f32>, f32, tensor<f32>)
   return %0#4 : tensor<f32>
 }
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/block.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/block.mlir
new file mode 100755
index 0000000000..d0b5e77bd4
--- /dev/null
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/block.mlir
@@ -0,0 +1,85 @@
+//--------------------------------------------------------------------------------------------------
+// WHEN CREATING A NEW TEST, PLEASE JUST COPY & PASTE WITHOUT EDITS.
+//
+// Set-up that's shared across all tests in this directory. In principle, this
+// config could be moved to lit.local.cfg. However, there are downstream users that
+//  do not use these LIT config files. Hence why this is kept inline.
+//
+// DEFINE: %{sparse_compiler_opts} = enable-runtime-library=true
+// DEFINE: %{sparse_compiler_opts_sve} = enable-arm-sve=true %{sparse_compiler_opts}
+// DEFINE: %{compile} = mlir-opt %s --sparse-compiler="%{sparse_compiler_opts}"
+// DEFINE: %{compile_sve} = mlir-opt %s --sparse-compiler="%{sparse_compiler_opts_sve}"
+// DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_opts} = -e entry -entry-point-result=void
+// DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+//
+// DEFINE: %{env} =
+//--------------------------------------------------------------------------------------------------
+
+// REDEFINE: %{env} = TENSOR0="%mlir_src_dir/test/Integration/data/block.mtx"
+// RUN: %{compile} | env %{env} %{run} | FileCheck %s
+//
+// TODO: enable!
+// Do the same run, but now with direct IR generation.
+// REDEFINE: %{sparse_compiler_opts} = enable-runtime-library=false
+// R_UN: %{compile} | env %{env} %{run} | FileCheck %s
+
+#BSR = #sparse_tensor.encoding<{
+  map = (i, j) ->
+    ( i floordiv 2 : dense
+    , j floordiv 2 : compressed
+    , i mod 2 : dense
+    , j mod 2 : dense
+    )
+}>
+
+!Filename = !llvm.ptr<i8>
+
+//
+// Example 2x2 block storage:
+//
+//  +-----+-----+-----+    +-----+-----+-----+
+//  | 1 2 | . . | 4 . |    | 1 2 |     | 4 0 |
+//  | . 3 | . . | . 5 |    | 0 3 |     | 0 5 |
+//  +-----+-----+-----+ => +-----+-----+-----+
+//  | . . | 6 7 | . . |    |     | 6 7 |     |
+//  | . . | 8 . | . . |    |     | 8 0 |     |
+//  +-----+-----+-----+    +-----+-----+-----+
+//
+// Stored as:
+//
+//    positions[1]   : 0 2 3
+//    coordinates[1] : 0 2 1
+//    values         : 1.000000 2.000000 0.000000 3.000000 4.000000 0.000000 0.000000 5.000000 6.000000 7.000000 8.000000 0.000000
+//
+module {
+
+  func.func private @getTensorFilename(index) -> (!Filename)
+
+  func.func @entry() {
+    %c0 = arith.constant 0   : index
+    %f0 = arith.constant 0.0 : f64
+
+    %fileName = call @getTensorFilename(%c0) : (index) -> (!Filename)
+    %A = sparse_tensor.new %fileName : !Filename to tensor<?x?xf64, #BSR>
+
+    // CHECK:      ( 0, 2, 3 )
+    // CHECK-NEXT: ( 0, 2, 1 )
+    // CHECK-NEXT: ( 1, 2, 0, 3, 4, 0, 0, 5, 6, 7, 8, 0 )
+    %pos = sparse_tensor.positions %A {level = 1 : index } : tensor<?x?xf64, #BSR> to memref<?xindex>
+    %vecp = vector.transfer_read %pos[%c0], %c0 : memref<?xindex>, vector<3xindex>
+    vector.print %vecp : vector<3xindex>
+    %crd = sparse_tensor.coordinates %A {level = 1 : index } : tensor<?x?xf64, #BSR> to memref<?xindex>
+    %vecc = vector.transfer_read %crd[%c0], %c0 : memref<?xindex>, vector<3xindex>
+    vector.print %vecc : vector<3xindex>
+    %val = sparse_tensor.values %A : tensor<?x?xf64, #BSR> to memref<?xf64>
+    %vecv = vector.transfer_read %val[%c0], %f0 : memref<?xf64>, vector<12xf64>
+    vector.print %vecv : vector<12xf64>
+
+    // Release the resources.
+    bufferization.dealloc_tensor %A: tensor<?x?xf64, #BSR>
+
+    return
+  }
+}
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_element.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_element.mlir
index 16eaca7663..a28f9057ae 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_element.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_element.mlir
@@ -17,12 +17,12 @@
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
 
-// REDEFINE: %{sparse_compiler_opts} = enable-runtime-library=false s2s-strategy=2
+// REDEFINE: %{sparse_compiler_opts} = enable-runtime-library=false
 
 // RUN: %{compile} | %{run} | FileCheck %s
 //
 // Do the same run, but now with vectorization.
-// REDEFINE: %{sparse_compiler_opts} = enable-runtime-library=false s2s-strategy=2 vl=2 reassociate-fp-reductions=true enable-index-optimizations=true
+// REDEFINE: %{sparse_compiler_opts} = enable-runtime-library=false vl=2 reassociate-fp-reductions=true enable-index-optimizations=true
 // RUN: %{compile} | %{run} | FileCheck %s
 //
 // Do the same run, but now with VLA vectorization.
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_sparse2sparse.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_sparse2sparse.mlir
index 1c74b6827d..c151a8c902 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_sparse2sparse.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_sparse2sparse.mlir
@@ -17,15 +17,15 @@
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
 
-// REDEFINE: %{sparse_compiler_opts} = enable-runtime-library=true s2s-strategy=2
+// REDEFINE: %{sparse_compiler_opts} = enable-runtime-library=true
 // RUN: %{compile} | %{run} | FileCheck %s
 //
 // Do the same run, but now with direct IR generation.
-// REDEFINE: %{sparse_compiler_opts} = enable-runtime-library=false s2s-strategy=2
+// REDEFINE: %{sparse_compiler_opts} = enable-runtime-library=false
 // RUN: %{compile} | %{run} | FileCheck %s
 //
 // Do the same run, but now with direct IR generation and vectorization.
-// REDEFINE: %{sparse_compiler_opts} = enable-runtime-library=false s2s-strategy=2 vl=2 reassociate-fp-reductions=true enable-index-optimizations=true
+// REDEFINE: %{sparse_compiler_opts} = enable-runtime-library=false vl=2 reassociate-fp-reductions=true enable-index-optimizations=true
 // RUN: %{compile} | %{run} | FileCheck %s
 //
 // Do the same run, but now with direct IR generation and VLA vectorization.
@@ -49,13 +49,7 @@
 }>
 
 #SingletonTensor1 = #sparse_tensor.encoding<{
-  map = (d0, d1, d2) -> (d0 : dense, d1 : compressed, d2 : singleton)
-
-}>
-
-// This also checks the compressed->dense conversion (when there are zeros).
-#SingletonTensor2 = #sparse_tensor.encoding<{
-  map = (d0, d1, d2) -> (d0 : dense, d1 : dense, d2 : singleton)
+  map = (d0, d1, d2) -> (d0 : dense, d1 : compressed(nonunique), d2 : singleton)
 
 }>
 
@@ -97,44 +91,34 @@ module {
     // Convert dense tensor directly to various sparse tensors.
     //
     %s1 = sparse_tensor.convert %src : tensor<2x3x4xf64> to tensor<2x3x4xf64, #Tensor1>
-    %s2 = sparse_tensor.convert %src : tensor<2x3x4xf64> to tensor<2x3x4xf64, #Tensor2>
     %s3 = sparse_tensor.convert %src : tensor<2x3x4xf64> to tensor<2x3x4xf64, #Tensor3>
 
     //
     // Convert sparse tensor directly to another sparse format.
     //
     %t13 = sparse_tensor.convert %s1 : tensor<2x3x4xf64, #Tensor1> to tensor<2x3x4xf64, #Tensor3>
-    %t21 = sparse_tensor.convert %s2 : tensor<2x3x4xf64, #Tensor2> to tensor<2x3x4xf64, #Tensor1>
-    %t23 = sparse_tensor.convert %s2 : tensor<2x3x4xf64, #Tensor2> to tensor<2x3x4xf64, #Tensor3>
     %t31 = sparse_tensor.convert %s3 : tensor<2x3x4xf64, #Tensor3> to tensor<2x3x4xf64, #Tensor1>
 
     //
     // Convert sparse tensor back to dense.
     //
     %d13 = sparse_tensor.convert %t13 : tensor<2x3x4xf64, #Tensor3> to tensor<2x3x4xf64>
-    %d21 = sparse_tensor.convert %t21 : tensor<2x3x4xf64, #Tensor1> to tensor<2x3x4xf64>
-    %d23 = sparse_tensor.convert %t23 : tensor<2x3x4xf64, #Tensor3> to tensor<2x3x4xf64>
     %d31 = sparse_tensor.convert %t31 : tensor<2x3x4xf64, #Tensor1> to tensor<2x3x4xf64>
 
     //
     // Check round-trip equality.  And release dense tensors.
     //
-    // CHECK-COUNT-5: ( ( ( 1, 2, 3, 4 ), ( 5, 6, 7, 8 ), ( 9, 10, 11, 12 ) ), ( ( 13, 14, 15, 16 ), ( 17, 18, 19, 20 ), ( 21, 22, 23, 24 ) ) )
+    // CHECK-COUNT-3: ( ( ( 1, 2, 3, 4 ), ( 5, 6, 7, 8 ), ( 9, 10, 11, 12 ) ), ( ( 13, 14, 15, 16 ), ( 17, 18, 19, 20 ), ( 21, 22, 23, 24 ) ) )
     call @dump(%src) : (tensor<2x3x4xf64>) -> ()
     call @dump(%d13) : (tensor<2x3x4xf64>) -> ()
-    call @dump(%d21) : (tensor<2x3x4xf64>) -> ()
-    call @dump(%d23) : (tensor<2x3x4xf64>) -> ()
     call @dump(%d31) : (tensor<2x3x4xf64>) -> ()
 
     //
     // Release sparse tensors.
     //
     bufferization.dealloc_tensor %t13 : tensor<2x3x4xf64, #Tensor3>
-    bufferization.dealloc_tensor %t21 : tensor<2x3x4xf64, #Tensor1>
-    bufferization.dealloc_tensor %t23 : tensor<2x3x4xf64, #Tensor3>
     bufferization.dealloc_tensor %t31 : tensor<2x3x4xf64, #Tensor1>
     bufferization.dealloc_tensor %s1 : tensor<2x3x4xf64, #Tensor1>
-    bufferization.dealloc_tensor %s2 : tensor<2x3x4xf64, #Tensor2>
     bufferization.dealloc_tensor %s3 : tensor<2x3x4xf64, #Tensor3>
 
     return
@@ -160,52 +144,34 @@ module {
     // Convert dense tensor directly to various sparse tensors.
     //
     %s1 = sparse_tensor.convert %src : tensor<2x3x4xf64> to tensor<2x3x4xf64, #SingletonTensor1>
-    %s2 = sparse_tensor.convert %src : tensor<2x3x4xf64> to tensor<2x3x4xf64, #SingletonTensor2>
     %s3 = sparse_tensor.convert %src : tensor<2x3x4xf64> to tensor<2x3x4xf64, #SingletonTensor3>
 
     //
     // Convert sparse tensor directly to another sparse format.
     //
-    %t12 = sparse_tensor.convert %s1 : tensor<2x3x4xf64, #SingletonTensor1> to tensor<2x3x4xf64, #SingletonTensor2>
     %t13 = sparse_tensor.convert %s1 : tensor<2x3x4xf64, #SingletonTensor1> to tensor<2x3x4xf64, #SingletonTensor3>
-    %t21 = sparse_tensor.convert %s2 : tensor<2x3x4xf64, #SingletonTensor2> to tensor<2x3x4xf64, #SingletonTensor1>
-    %t23 = sparse_tensor.convert %s2 : tensor<2x3x4xf64, #SingletonTensor2> to tensor<2x3x4xf64, #SingletonTensor3>
     %t31 = sparse_tensor.convert %s3 : tensor<2x3x4xf64, #SingletonTensor3> to tensor<2x3x4xf64, #SingletonTensor1>
-    %t32 = sparse_tensor.convert %s3 : tensor<2x3x4xf64, #SingletonTensor3> to tensor<2x3x4xf64, #SingletonTensor2>
 
     //
     // Convert sparse tensor back to dense.
     //
-    %d12 = sparse_tensor.convert %t12 : tensor<2x3x4xf64, #SingletonTensor2> to tensor<2x3x4xf64>
     %d13 = sparse_tensor.convert %t13 : tensor<2x3x4xf64, #SingletonTensor3> to tensor<2x3x4xf64>
-    %d21 = sparse_tensor.convert %t21 : tensor<2x3x4xf64, #SingletonTensor1> to tensor<2x3x4xf64>
-    %d23 = sparse_tensor.convert %t23 : tensor<2x3x4xf64, #SingletonTensor3> to tensor<2x3x4xf64>
     %d31 = sparse_tensor.convert %t31 : tensor<2x3x4xf64, #SingletonTensor1> to tensor<2x3x4xf64>
-    %d32 = sparse_tensor.convert %t32 : tensor<2x3x4xf64, #SingletonTensor2> to tensor<2x3x4xf64>
 
     //
     // Check round-trip equality.  And release dense tensors.
     //
-    // CHECK-COUNT-7: ( ( ( 1, 0, 0, 0 ), ( 0, 6, 0, 0 ), ( 0, 0, 11, 0 ) ), ( ( 0, 14, 0, 0 ), ( 0, 0, 0, 20 ), ( 21, 0, 0, 0 ) ) )
+    // CHECK-COUNT-3: ( ( ( 1, 0, 0, 0 ), ( 0, 6, 0, 0 ), ( 0, 0, 11, 0 ) ), ( ( 0, 14, 0, 0 ), ( 0, 0, 0, 20 ), ( 21, 0, 0, 0 ) ) )
     call @dump(%src) : (tensor<2x3x4xf64>) -> ()
-    call @dump(%d12) : (tensor<2x3x4xf64>) -> ()
     call @dump(%d13) : (tensor<2x3x4xf64>) -> ()
-    call @dump(%d21) : (tensor<2x3x4xf64>) -> ()
-    call @dump(%d23) : (tensor<2x3x4xf64>) -> ()
     call @dump(%d31) : (tensor<2x3x4xf64>) -> ()
-    call @dump(%d32) : (tensor<2x3x4xf64>) -> ()
 
     //
     // Release sparse tensors.
     //
-    bufferization.dealloc_tensor %t12 : tensor<2x3x4xf64, #SingletonTensor2>
     bufferization.dealloc_tensor %t13 : tensor<2x3x4xf64, #SingletonTensor3>
-    bufferization.dealloc_tensor %t21 : tensor<2x3x4xf64, #SingletonTensor1>
-    bufferization.dealloc_tensor %t23 : tensor<2x3x4xf64, #SingletonTensor3>
     bufferization.dealloc_tensor %t31 : tensor<2x3x4xf64, #SingletonTensor1>
-    bufferization.dealloc_tensor %t32 : tensor<2x3x4xf64, #SingletonTensor2>
     bufferization.dealloc_tensor %s1 : tensor<2x3x4xf64, #SingletonTensor1>
-    bufferization.dealloc_tensor %s2 : tensor<2x3x4xf64, #SingletonTensor2>
     bufferization.dealloc_tensor %s3 : tensor<2x3x4xf64, #SingletonTensor3>
 
     return
diff --git a/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sampled-matmul-lib.mlir b/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sampled-matmul-lib.mlir
index 61de57564b..ac5c0f8bea 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sampled-matmul-lib.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sampled-matmul-lib.mlir
@@ -3,7 +3,8 @@
 //
 // DEFINE: %{compile} = mlir-opt %s \
 // DEFINE:   --sparse-compiler="enable-gpu-libgen gpu-triple=nvptx64-nvidia-cuda gpu-chip=sm_80 gpu-features=+ptx71 gpu-format=%gpu_compilation_format
-// DEFINE: %{run} = TENSOR0="%mlir_src_dir/test/Integration/data/test.mtx" \
+// DEFINE: %{run} = \
+// DEFINE:   env TENSOR0="%mlir_src_dir/test/Integration/data/test.mtx" \
 // DEFINE:   mlir-cpu-runner \
 // DEFINE:   --shared-libs=%mlir_cuda_runtime \
 // DEFINE:   --shared-libs=%mlir_c_runner_utils \
@@ -12,16 +13,16 @@
 //
 // with RT lib:
 //
-//  RUN:  %{compile} enable-runtime-library=true" | %{run}
-//  RUN:  %{compile} enable-runtime-library=true gpu-data-transfer-strategy=pinned-dma" | %{run}
-//  Tracker #64316
-//  RUNNOT: %{compile} enable-runtime-library=true gpu-data-transfer-strategy=zero-copy" | %{run}
+// RUN:  %{compile} enable-runtime-library=true" | %{run}
+// RUN:  %{compile} enable-runtime-library=true gpu-data-transfer-strategy=pinned-dma" | %{run}
+// TODO: Tracker #64316
+// RUNNOT: %{compile} enable-runtime-library=true gpu-data-transfer-strategy=zero-copy" | %{run}
 //
 // without RT lib:
 //
 // RUN:  %{compile} enable-runtime-library=false" | %{run}
 // RUN:  %{compile} enable-runtime-library=false gpu-data-transfer-strategy=pinned-dma" | %{run}
-//  Tracker #64316
+// TODO:  Tracker #64316
 // RUNNOT: %{compile} enable-runtime-library=false gpu-data-transfer-strategy=zero-copy" | %{run}
 //
 
diff --git a/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sddmm-lib.mlir b/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sddmm-lib.mlir
new file mode 100644
index 0000000000..54408d629e
--- /dev/null
+++ b/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sparse-sddmm-lib.mlir
@@ -0,0 +1,189 @@
+//
+// NOTE: this test requires gpu-sm80
+//
+// DEFINE: %{compile} = mlir-opt %s \
+// DEFINE:   --sparse-compiler="enable-gpu-libgen gpu-triple=nvptx64-nvidia-cuda gpu-chip=sm_80 gpu-features=+ptx71 gpu-format=%gpu_compilation_format
+// DEFINE: %{run} = \
+// DEFINE:   env TENSOR0="%mlir_src_dir/test/Integration/data/block.mtx" \
+// DEFINE:   mlir-cpu-runner \
+// DEFINE:   --shared-libs=%mlir_cuda_runtime \
+// DEFINE:   --shared-libs=%mlir_c_runner_utils \
+// DEFINE:   --e entry --entry-point-result=void \
+// DEFINE: | FileCheck %s
+//
+// with RT lib:
+//
+// RUN:  %{compile} enable-runtime-library=true" | %{run}
+//
+// without RT lib:
+//
+// TODO: make this work
+// R_UN:  %{compile} enable-runtime-library=false" | %{run}
+//
+
+!Filename = !llvm.ptr<i8>
+
+#CSR = #sparse_tensor.encoding<{
+  map = (d0, d1) -> (d0 : dense, d1 : compressed)
+}>
+
+#BSR = #sparse_tensor.encoding<{
+  map = (i, j) -> (
+    i floordiv 2 : dense,
+    j floordiv 2 : compressed,
+    i mod 2 : dense,
+    j mod 2 : dense)
+}>
+
+#trait_SDDMM = {
+  indexing_maps = [
+    affine_map<(i,j,k) -> (i,k)>,  // A
+    affine_map<(i,j,k) -> (k,j)>,  // B
+    affine_map<(i,j,k) -> (i,j)>   // S (in/out)
+  ],
+  iterator_types = ["parallel", "parallel", "reduction"],
+  doc = "S(i,j) += spy[S(i,j)] x SUM_k A(i,k) B(k,j)"
+}
+
+//
+// Integration test that lowers a kernel annotated as sparse to
+// actual sparse code, initializes sparse storage schemes, and
+// runs the resulting code with the JIT compiler.
+//
+module {
+  llvm.func @mgpuCreateSparseEnv()
+  llvm.func @mgpuDestroySparseEnv()
+
+  //
+  // A kernel that computes a CSR sampled dense matrix matrix multiplication
+  // using a "spy" function and in-place update of the sampling sparse matrix.
+  //
+  func.func @SDDMM(%args: tensor<?x?xf32, #CSR>,
+                   %arga: tensor<?x?xf32>,
+                   %argb: tensor<?x?xf32>) -> tensor<?x?xf32, #CSR> {
+    %result = linalg.generic #trait_SDDMM
+      ins(%arga, %argb: tensor<?x?xf32>, tensor<?x?xf32>)
+      outs(%args: tensor<?x?xf32, #CSR>) {
+        ^bb(%a: f32, %b: f32, %s: f32):
+           %f0 = arith.constant 0.0 : f32
+           %u = sparse_tensor.unary %s : f32 to f32
+             present={
+                ^bb0(%p: f32):
+                  %mul = arith.mulf %a, %b : f32
+                  sparse_tensor.yield %mul : f32
+             }
+             absent={}
+           %r = sparse_tensor.reduce %s, %u, %f0 : f32 {
+              ^bb0(%p: f32, %q: f32):
+                %add = arith.addf %p, %q : f32
+                sparse_tensor.yield %add : f32
+            }
+           linalg.yield %r : f32
+      } -> tensor<?x?xf32, #CSR>
+    return %result : tensor<?x?xf32, #CSR>
+  }
+
+  //
+  // A kernel that computes a BSR sampled dense matrix matrix multiplication
+  // using a "spy" function and in-place update of the sampling sparse matrix.
+  //
+  func.func @SDDMM_block(%args: tensor<?x?xf32, #BSR>,
+                         %arga: tensor<?x?xf32>,
+                         %argb: tensor<?x?xf32>) -> tensor<?x?xf32, #BSR> {
+    %result = linalg.generic #trait_SDDMM
+      ins(%arga, %argb: tensor<?x?xf32>, tensor<?x?xf32>)
+      outs(%args: tensor<?x?xf32, #BSR>) {
+        ^bb(%a: f32, %b: f32, %s: f32):
+           %f0 = arith.constant 0.0 : f32
+           %u = sparse_tensor.unary %s : f32 to f32
+             present={
+                ^bb0(%p: f32):
+                  %mul = arith.mulf %a, %b : f32
+                  sparse_tensor.yield %mul : f32
+             }
+             absent={}
+           %r = sparse_tensor.reduce %s, %u, %f0 : f32 {
+              ^bb0(%p: f32, %q: f32):
+                %add = arith.addf %p, %q : f32
+                sparse_tensor.yield %add : f32
+            }
+           linalg.yield %r : f32
+      } -> tensor<?x?xf32, #BSR>
+    return %result : tensor<?x?xf32, #BSR>
+  }
+
+  func.func private @getTensorFilename(index) -> (!Filename)
+
+  //
+  // Main driver.
+  //
+  func.func @entry() {
+    llvm.call @mgpuCreateSparseEnv() : () -> ()
+    %d0 = arith.constant 0.0 : f32
+    %c0 = arith.constant 0 : index
+    %c1 = arith.constant 1 : index
+    %c4 = arith.constant 4 : index
+    %c6 = arith.constant 6 : index
+
+    // Initialize dense matrices.
+    %a = tensor.generate %c4, %c4 {
+    ^bb0(%i: index, %j: index):
+      %p = arith.addi %i, %c1 : index
+      %q = arith.index_cast %p : index to i32
+      %d = arith.sitofp %q : i32 to f32
+      tensor.yield %d : f32
+    } : tensor<?x?xf32>
+    %b = tensor.generate %c4, %c6 {
+    ^bb0(%i: index, %j: index):
+      %p = arith.addi %j, %c1 : index
+      %q = arith.index_cast %p : index to i32
+      %d = arith.sitofp %q : i32 to f32
+      tensor.yield %d : f32
+    } : tensor<?x?xf32>
+
+    // Read the sparse matrix from file, construct sparse storage.
+    //
+    //      +-----+-----+-----+
+    //      | 1 2 | . . | 4 . |
+    //      | . 3 | . . | . 5 |
+    //      +-----+-----+-----+
+    //      | . . | 6 7 | . . |
+    //      | . . | 8 . | . . |
+    //      +-----+-----+-----+
+    //
+    %fileName = call @getTensorFilename(%c0) : (index) -> (!Filename)
+    %m_csr = sparse_tensor.new %fileName : !Filename to tensor<?x?xf32, #CSR>
+    %m_bsr = sparse_tensor.new %fileName : !Filename to tensor<?x?xf32, #BSR>
+
+    // Call the kernel.
+    %0 = call @SDDMM(%m_csr, %a, %b)
+       : (tensor<?x?xf32, #CSR>,
+          tensor<?x?xf32>, tensor<?x?xf32>) -> tensor<?x?xf32, #CSR>
+    %1 = call @SDDMM_block(%m_bsr, %a, %b)
+       : (tensor<?x?xf32, #BSR>,
+          tensor<?x?xf32>, tensor<?x?xf32>) -> tensor<?x?xf32, #BSR>
+
+    //
+    // Print the result for verification. Note that the "spy" determines what
+    // dot products are sampled, but the original contents are added back to
+    // the result (which is why the block sparse version has actual results
+    // in the original zero positions).
+    //
+    // CHECK:      ( 5, 10, 24, 19, 53, 42, 55, 56 )
+    // CHECK-NEXT: ( 5, 10, 8, 19, 24, 24, 40, 53, 42, 55, 56, 64 )
+    //
+    %v0 = sparse_tensor.values %0 : tensor<?x?xf32, #CSR> to memref<?xf32>
+    %vv0 = vector.transfer_read %v0[%c0], %d0 : memref<?xf32>, vector<8xf32>
+    vector.print %vv0 : vector<8xf32>
+    %v1 = sparse_tensor.values %1 : tensor<?x?xf32, #BSR> to memref<?xf32>
+    %vv1 = vector.transfer_read %v1[%c0], %d0 : memref<?xf32>, vector<12xf32>
+    vector.print %vv1 : vector<12xf32>
+
+    // Release the resources.
+    bufferization.dealloc_tensor %0 : tensor<?x?xf32, #CSR>
+    bufferization.dealloc_tensor %1 : tensor<?x?xf32, #BSR>
+
+    llvm.call @mgpuDestroySparseEnv() : () -> ()
+    return
+  }
+}
diff --git a/mlir/test/Integration/Dialect/SparseTensor/python/test_SDDMM.py b/mlir/test/Integration/Dialect/SparseTensor/python/test_SDDMM.py
index 0cdc7c88bd..1f9b636038 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/python/test_SDDMM.py
+++ b/mlir/test/Integration/Dialect/SparseTensor/python/test_SDDMM.py
@@ -155,7 +155,7 @@ def main():
                     for iwidth in [32]:
                         for e in [True]:
                             attr = st.EncodingAttr.get(
-                                level, ordering, pwidth, iwidth
+                                level, ordering, None, pwidth, iwidth
                             )
                             opt = f"parallelization-strategy=none"
                             compiler = sparse_compiler.SparseCompiler(
diff --git a/mlir/test/Integration/Dialect/SparseTensor/python/test_SpMM.py b/mlir/test/Integration/Dialect/SparseTensor/python/test_SpMM.py
index 01d74a4dc8..69f6cdcea9 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/python/test_SpMM.py
+++ b/mlir/test/Integration/Dialect/SparseTensor/python/test_SpMM.py
@@ -145,7 +145,7 @@ def main():
                 for pwidth in bitwidths:
                     for iwidth in bitwidths:
                         attr = st.EncodingAttr.get(
-                            level, ordering, pwidth, iwidth
+                            level, ordering, None, pwidth, iwidth
                         )
                         build_compile_and_run_SpMM(attr, compiler)
                         count = count + 1
diff --git a/mlir/test/Integration/Dialect/SparseTensor/python/test_output.py b/mlir/test/Integration/Dialect/SparseTensor/python/test_output.py
index 8f3f4e5af1..7d77490080 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/python/test_output.py
+++ b/mlir/test/Integration/Dialect/SparseTensor/python/test_output.py
@@ -91,7 +91,7 @@ def main():
         for level in levels:
             for ordering in orderings:
                 for bwidth in bitwidths:
-                    attr = st.EncodingAttr.get(level, ordering, bwidth, bwidth)
+                    attr = st.EncodingAttr.get(level, ordering, None, bwidth, bwidth)
                     build_compile_and_run_output(attr, compiler)
                     count = count + 1
 
diff --git a/mlir/test/Integration/Dialect/SparseTensor/python/test_stress.py b/mlir/test/Integration/Dialect/SparseTensor/python/test_stress.py
index 7425a22910..841b02bc10 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/python/test_stress.py
+++ b/mlir/test/Integration/Dialect/SparseTensor/python/test_stress.py
@@ -196,13 +196,7 @@ def main():
     # CHECK-LABEL: TEST: test_stress
     print("\nTEST: test_stress")
     with ir.Context() as ctx, ir.Location.unknown():
-        # Disable direct sparse2sparse conversion, because it doubles the time!
-        # TODO: While direct s2s is far too slow for per-commit testing,
-        # we should have some framework ensure that we run this test with
-        # `s2s=0` on a regular basis, to ensure that it does continue to work.
-        # TODO: be sure to test s2s=0 together with singletons.
-        s2s = 1
-        sparsification_options = f"parallelization-strategy=none " f"s2s-strategy={s2s}"
+        sparsification_options = f"parallelization-strategy=none "
         compiler = sparse_compiler.SparseCompiler(
             options=sparsification_options, opt_level=0, shared_libs=[support_lib]
         )
@@ -233,7 +227,7 @@ def main():
                 for pwidth in bitwidths:
                     for iwidth in bitwidths:
                         attr = st.EncodingAttr.get(
-                            level, ordering, pwidth, iwidth
+                            level, ordering, None, pwidth, iwidth
                         )
                         types.append(ir.RankedTensorType.get(shape, f64, attr))
         #
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/test-load-vertical.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/test-load-vertical.mlir
index 8c7d8c954d..179e9fa836 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/test-load-vertical.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/test-load-vertical.mlir
@@ -63,7 +63,7 @@ func.func @entry() {
   }
 
   // Load tile from "mem1" vertically.
-  %0 = arm_sme.tile_load %mem1[%c0, %c0], <vertical> : memref<?xi32>, vector<[4]x[4]xi32>
+  %0 = arm_sme.tile_load %mem1[%c0, %c0] layout<vertical> : memref<?xi32>, vector<[4]x[4]xi32>
 
   // 1. ORIGINAL HORIZONTAL LAYOUT
   // Dump "mem1". The smallest SVL is 128-bits so the tile will be at least
diff --git a/mlir/test/Integration/data/block.mtx b/mlir/test/Integration/data/block.mtx
new file mode 100755
index 0000000000..9bb3ea7d50
--- /dev/null
+++ b/mlir/test/Integration/data/block.mtx
@@ -0,0 +1,10 @@
+%%MatrixMarket matrix coordinate real general
+4 6 8
+1 1 1.0
+1 2 2.0
+1 5 4.0
+2 2 3.0
+2 6 5.0
+3 3 6.0
+3 4 7.0
+4 3 8.0
diff --git a/mlir/test/Interfaces/TilingInterface/tile-using-scfforall.mlir b/mlir/test/Interfaces/TilingInterface/tile-using-scfforall.mlir
new file mode 100644
index 0000000000..314efde457
--- /dev/null
+++ b/mlir/test/Interfaces/TilingInterface/tile-using-scfforall.mlir
@@ -0,0 +1,167 @@
+// RUN: mlir-opt  -test-tiling-interface=tile-using-scf-forall -split-input-file %s | FileCheck %s
+
+func.func @simple_matmul(%arg0 : tensor<?x?xf32>, %arg1 : tensor<?x?xf32>,
+    %arg2 : tensor<?x?xf32>) -> tensor<?x?xf32> {
+  %0 = linalg.matmul {__internal_transform__ = "simple_gemm"}
+    ins(%arg0, %arg1 : tensor<?x?xf32>, tensor<?x?xf32>)
+      outs(%arg2 : tensor<?x?xf32>) -> tensor<?x?xf32>
+  return %0 : tensor<?x?xf32>
+}
+//  CHECK-DAG: #[[MAP0:.+]] = affine_map<(d0)[s0] -> (10, -d0 + s0)>
+//  CHECK-DAG: #[[MAP1:.+]] = affine_map<(d0)[s0] -> (20, -d0 + s0)>
+//      CHECK: func.func @simple_matmul(
+// CHECK-SAME:     %[[ARG0:[a-zA-Z0-9]+]]: tensor<?x?xf32>
+// CHECK-SAME:     %[[ARG1:[a-zA-Z0-9]+]]: tensor<?x?xf32>
+// CHECK-SAME:     %[[ARG2:[a-zA-Z0-9]+]]: tensor<?x?xf32>
+//  CHECK-DAG:   %[[C0:.+]] = arith.constant 0 : index
+//  CHECK-DAG:   %[[C1:.+]] = arith.constant 1 : index
+//  CHECK-DAG:   %[[M:.+]] = tensor.dim %[[ARG0]], %[[C0]]
+//  CHECK-DAG:   %[[K:.+]] = tensor.dim %[[ARG0]], %[[C1]]
+//  CHECK-DAG:   %[[N:.+]] = tensor.dim %[[ARG1]], %[[C1]]
+//      CHECK:   %[[RESULT:.+]] = scf.forall (%[[IV0:[a-zA-Z0-9]+]], %[[IV1:[a-zA-Z0-9]+]]) =
+// CHECK-SAME:       (0, 0) to (%[[M]], %[[N]]) step (10, 20) shared_outs(%[[INIT:.+]] = %[[ARG2]])
+//      CHECK:     %[[TS_Y:.+]] = affine.min #[[MAP0]](%[[IV0]])[%[[M]]]
+//      CHECK:     %[[TS_X:.+]] = affine.min #[[MAP1]](%[[IV1]])[%[[N]]]
+//      CHECK:     %[[LHS_TILE:.+]] = tensor.extract_slice %[[ARG0]]
+// CHECK-SAME:         [%[[IV0]], 0] [%[[TS_Y]], %[[K]]] [1, 1]
+//      CHECK:     %[[RHS_TILE:.+]] = tensor.extract_slice %[[ARG1]]
+// CHECK-SAME:         [0, %[[IV1]]] [%[[K]], %[[TS_X]]] [1, 1]
+//      CHECK:     %[[INIT_TILE:.+]] = tensor.extract_slice %[[INIT]]
+// CHECK-SAME:         [%[[IV0]], %[[IV1]]] [%[[TS_Y]], %[[TS_X]]] [1, 1]
+//      CHECK:     %[[GEMM_TILE:.+]] = linalg.matmul
+// CHECK-SAME:         ins(%[[LHS_TILE]], %[[RHS_TILE]] :
+// CHECK-SAME:         outs(%[[INIT_TILE]] :
+//      CHECK:     scf.forall.in_parallel {
+//      CHECK:       tensor.parallel_insert_slice %[[GEMM_TILE]] into %[[INIT]]
+// CHECK-SAME:           [%[[IV0]], %[[IV1]]] [%[[TS_Y]], %[[TS_X]]] [1, 1]
+//      CHECK:       mapping = [#gpu.block<y>, #gpu.block<x>]
+//      CHECK:   return %[[RESULT]]
+
+// -----
+
+#map0 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
+#map1 = affine_map<(d0, d1, d2) -> (d0, d2, d1)>
+#map2 = affine_map<(d0, d1, d2) -> (d2, d0, d1)>
+func.func @multi_result(%arg0 : tensor<128x200x300xf32>) -> (tensor<128x300x200xf32>, tensor<300x128x200xf32>) {
+  %init0 = tensor.empty() : tensor<128x300x200xf32>
+  %init1 = tensor.empty() : tensor<300x128x200xf32>
+  %0:2 = linalg.generic {
+      indexing_maps = [#map0, #map1, #map2],
+      iterator_types = ["parallel", "parallel", "parallel"]}
+      {__internal_transform__ = "parallel_generic_transpose"}
+      ins(%arg0 : tensor<128x200x300xf32>)
+      outs(%init0, %init1 : tensor<128x300x200xf32>, tensor<300x128x200xf32>) {
+    ^bb0(%b0 : f32, %b1 : f32, %b2 : f32):
+      linalg.yield %b0, %b0 : f32, f32
+    } -> (tensor<128x300x200xf32>, tensor<300x128x200xf32>)
+  return %0#0, %0#1 : tensor<128x300x200xf32>, tensor<300x128x200xf32>
+}
+//  CHECK-DAG: #[[$MAP0:.+]] = affine_map<(d0) -> (10, -d0 + 128)>
+//      CHECK-LABEL: func.func @multi_result(
+// CHECK-SAME:     %[[ARG0:[a-zA-Z0-9]+]]: tensor<128x200x300xf32>)
+//  CHECK-DAG:   %[[INIT0:.+]] = tensor.empty()
+//  CHECK-DAG:   %[[INIT1:.+]] = tensor.empty()
+//      CHECK:   %[[OUTER:[a-zA-Z0-9]+]]:2 = scf.forall (%[[IV0:[a-zA-Z0-9]+]], %[[IV1:[a-zA-Z0-9]+]]) = (0, 0) to (128, 300) step (10, 20)
+// CHECK-SAME:       shared_outs(%[[ARG1:[a-zA-Z0-9]+]] = %[[INIT0]], %[[ARG2:[a-zA-Z0-9]+]] = %[[INIT1]])
+//      CHECK:     %[[TS_Y:.+]] = affine.min #[[$MAP0]](%[[IV0]])
+//      CHECK:     %[[ARG_TILE:.+]] = tensor.extract_slice %[[ARG0]]
+// CHECK-SAME:         [%[[IV0]], 0, %[[IV1]]] [%[[TS_Y]], 200, 20] [1, 1, 1]
+//  CHECK-DAG:     %[[INIT0_TILE:.+]] = tensor.extract_slice %[[ARG1]]
+// CHECK-SAME:         [%[[IV0]], %[[IV1]], 0] [%[[TS_Y]], 20, 200] [1, 1, 1]
+//  CHECK-DAG:     %[[INIT1_TILE:.+]] = tensor.extract_slice %[[ARG2]]
+// CHECK-SAME:         [%[[IV1]], %[[IV0]], 0] [20, %[[TS_Y]], 200] [1, 1, 1]
+//      CHECK:     %[[RESULT_TILE:.+]]:2 = linalg.generic
+// CHECK-SAME:         ins(%[[ARG_TILE]] :
+// CHECK-SAME:         outs(%[[INIT0_TILE]], %[[INIT1_TILE]] :
+//      CHECK:     scf.forall.in_parallel {
+//  CHECK-DAG:       tensor.parallel_insert_slice %[[RESULT_TILE]]#0 into %[[ARG1]][%[[IV0]], %[[IV1]], 0] [%[[TS_Y]], 20, 200] [1, 1, 1]
+//  CHECK-DAG:       tensor.parallel_insert_slice %[[RESULT_TILE]]#1 into %[[ARG2]][%[[IV1]], %[[IV0]], 0] [20, %[[TS_Y]], 200] [1, 1, 1]
+//      CHECK:     }
+//      CHECK:   return %[[OUTER]]#0, %[[OUTER]]#1
+
+// -----
+
+func.func @conv2D(%arg0 : tensor<?x?x?x?xf32>, %arg1 : tensor<?x?x?x?xf32>,
+    %arg2 : tensor<?x?x?x?xf32>) -> tensor<?x?x?x?xf32> {
+  %0 = linalg.conv_2d_nhwc_hwcf {
+      strides = dense<[2, 3]> : tensor<2xi64>,
+      dilation = dense<[4, 5]> : tensor<2xi64>,
+      __internal_transform__ = "simple_conv"}
+      ins(%arg0, %arg1 : tensor<?x?x?x?xf32>, tensor<?x?x?x?xf32>)
+      outs(%arg2 : tensor<?x?x?x?xf32>) -> tensor<?x?x?x?xf32>
+  return %0 : tensor<?x?x?x?xf32>
+}
+//  CHECK-DAG: #[[$MAP0:.+]] = affine_map<(d0)[s0] -> (10, -d0 + s0)>
+//  CHECK-DAG: #[[$MAP1:.+]] = affine_map<(d0)[s0] -> (20, -d0 + s0)>
+//  CHECK-DAG: #[[$MAP2:.+]] = affine_map<(d0)[s0] -> (30, -d0 + s0)>
+//  CHECK-DAG: #[[$MAP3:.+]] = affine_map<(d0)[s0] -> (d0 + s0 * 2 - 2)>
+//  CHECK-DAG: #[[$MAP4:.+]] = affine_map<(d0)[s0] -> (d0 + s0 * 3 - 3)>
+//      CHECK-LABEL: func.func @conv2D(
+// CHECK-SAME:     %[[INPUT:[a-zA-Z0-9]+]]: tensor<?x?x?x?xf32>
+// CHECK-SAME:     %[[FILTER:[a-zA-Z0-9]+]]: tensor<?x?x?x?xf32>
+// CHECK-SAME:     %[[INIT:[a-zA-Z0-9]+]]: tensor<?x?x?x?xf32>
+//  CHECK-DAG:   %[[C0:.+]] = arith.constant 0 : index
+//  CHECK-DAG:   %[[C1:.+]] = arith.constant 1 : index
+//  CHECK-DAG:   %[[C2:.+]] = arith.constant 2 : index
+//  CHECK-DAG:   %[[C3:.+]] = arith.constant 3 : index
+//  CHECK-DAG:   %[[N:.+]] = tensor.dim %[[INPUT]], %[[C0]]
+//  CHECK-DAG:   %[[C:.+]] = tensor.dim %[[INPUT]], %[[C3]]
+//  CHECK-DAG:   %[[P:.+]] = tensor.dim %[[FILTER]], %[[C0]]
+//  CHECK-DAG:   %[[Q:.+]] = tensor.dim %[[FILTER]], %[[C1]]
+//  CHECK-DAG:   %[[F:.+]] = tensor.dim %[[FILTER]], %[[C3]]
+//  CHECK-DAG:   %[[R:.+]] = tensor.dim %[[INIT]], %[[C1]]
+//  CHECK-DAG:   %[[S:.+]] = tensor.dim %[[INIT]], %[[C2]]
+//      CHECK:   %[[RESULT:.+]] = scf.forall (%[[IV0:[a-zA-Z0-9]+]], %[[IV1:[a-zA-Z0-9]+]], %[[IV2:[a-zA-Z0-9]+]]) =
+// CHECK-SAME:       (0, 0, 0) to (%[[P]], %[[Q]], %[[C]]) step (10, 20, 30) shared_outs(%[[INIT0:.+]] = %[[INIT]])
+//  CHECK-DAG:     %[[TS_P:.+]] = affine.min #[[$MAP0]](%[[IV0]])[%[[P]]]
+//  CHECK-DAG:     %[[TS_Q:.+]] = affine.min #[[$MAP1]](%[[IV1]])[%[[Q]]]
+//  CHECK-DAG:     %[[TS_C:.+]] = affine.min #[[$MAP2]](%[[IV2]])[%[[C]]]
+//  CHECK-DAG:     %[[TS_H:.+]] = affine.apply #[[$MAP3]](%[[TS_P]])[%[[R]]]
+//  CHECK-DAG:     %[[TS_W:.+]] = affine.apply #[[$MAP4]](%[[TS_Q]])[%[[S]]]
+//  CHECK-DAG:     %[[INPUT_TILE:.+]] = tensor.extract_slice %[[INPUT]]
+// CHECK-SAME:         [0, %[[IV0]], %[[IV1]], %[[IV2]]] [%[[N]], %[[TS_H]], %[[TS_W]], %[[TS_C]]]
+//  CHECK-DAG:     %[[FILTER_TILE:.+]] = tensor.extract_slice %[[FILTER]]
+// CHECK-SAME:         [%[[IV0]], %[[IV1]], %[[IV2]], 0] [%[[TS_P]], %[[TS_Q]], %[[TS_C]], %[[F]]]
+//  CHECK-DAG:     %[[INIT_TILE:.+]] = tensor.extract_slice %[[INIT0]]
+// CHECK-SAME:         [0, 0, 0, 0] [%[[N]], %[[R]], %[[S]], %[[F]]]
+//      CHECK:     %[[CONV_TILE:.+]] = linalg.conv_2d_nhwc_hwcf
+// CHECK-SAME:         dilation = dense<[4, 5]> : tensor<2xi64>, strides = dense<[2, 3]> : tensor<2xi64>
+// CHECK-SAME:         ins(%[[INPUT_TILE]], %[[FILTER_TILE]] :
+// CHECK-SAME:         outs(%[[INIT_TILE]] :
+//      CHECK:     scf.forall.in_parallel
+//      CHECK:       tensor.parallel_insert_slice %[[CONV_TILE]] into %[[INIT0]]
+// CHECK-SAME:           [0, 0, 0, 0] [%[[N]], %[[R]], %[[S]], %[[F]]] [1, 1, 1, 1]
+//      CHECK:   return %[[RESULT]]
+
+// -----
+
+// CHECK: #[[$MAP_ADD:.+]] = affine_map<(d0, d1) -> (d0 + d1)>
+
+func.func @indexed_semantics(%arg0: tensor<?x?xf32>, %arg1: tensor<?x?xf32>) -> tensor<?x?xf32> {
+  // Check that we correctly amend "linalg.index" results.
+
+  %0 = linalg.generic {
+    indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>,
+                     affine_map<(d0, d1) -> (d0, d1)>],
+    iterator_types = ["parallel", "parallel"]}
+    {__internal_transform__ = "indexed_semantics"}
+    ins(%arg0: tensor<?x?xf32>)
+    outs(%arg1: tensor<?x?xf32>) {
+  ^bb0(%arg2: f32, %arg3: f32):
+    %1 = linalg.index 0 : index
+    %2 = linalg.index 1 : index
+    %3 = arith.addi %1, %2 : index
+    %4 = arith.index_cast %3 : index to i64
+    %5 = arith.uitofp %4 : i64 to f32
+    %6 = arith.addf %5, %arg2 : f32
+    linalg.yield %6 : f32
+  } -> (tensor<?x?xf32>)
+  return %0 : tensor<?x?xf32>
+}
+// CHECK-LABEL: @indexed_semantics
+//       CHECK: scf.forall (%[[I0:.+]], %[[I1:.+]]) =
+//       CHECK:   %[[INDEX0:.+]] = linalg.index 0
+//       CHECK:   %[[INDEX0_AMENDED:.+]] = affine.apply #[[$MAP_ADD]](%[[INDEX0]], %[[I0]])
+//       CHECK:   %[[INDEX1:.+]] = linalg.index 1
+//       CHECK:   %[[INDEX1_AMENDED:.+]] = affine.apply #[[$MAP_ADD]](%[[INDEX1]], %[[I1]])
+//       CHECK:   arith.addi %[[INDEX0_AMENDED]], %[[INDEX1_AMENDED]]
diff --git a/mlir/test/Target/LLVMIR/Import/debug-info.ll b/mlir/test/Target/LLVMIR/Import/debug-info.ll
index da8f7c3e83..2c4fb2c213 100644
--- a/mlir/test/Target/LLVMIR/Import/debug-info.ll
+++ b/mlir/test/Target/LLVMIR/Import/debug-info.ll
@@ -222,15 +222,18 @@ define void @subprogram() !dbg !3 {
 define void @func_loc() !dbg !3 {
   ret void
 }
-; CHECK: #[[SP:.+]] =  #llvm.di_subprogram<compileUnit = #{{.*}}, scope = #{{.*}}, name = "func_loc", file = #{{.*}}, subprogramFlags = Definition>
-; CHECK: loc(fused<#[[SP]]>[
+; CHECK-DAG: #[[NAME_LOC:.+]] = loc("func_loc")
+; CHECK-DAG: #[[FILE_LOC:.+]] = loc("debug-info.ll":42:0)
+; CHECK-DAG: #[[SP:.+]] =  #llvm.di_subprogram<compileUnit = #{{.*}}, scope = #{{.*}}, name = "func_loc", file = #{{.*}}, line = 42, subprogramFlags = Definition>
+
+; CHECK: loc(fused<#[[SP]]>[#[[NAME_LOC]], #[[FILE_LOC]]]
 
 !llvm.dbg.cu = !{!1}
 !llvm.module.flags = !{!0}
 !0 = !{i32 2, !"Debug Info Version", i32 3}
 !1 = distinct !DICompileUnit(language: DW_LANG_C, file: !2)
 !2 = !DIFile(filename: "debug-info.ll", directory: "/")
-!3 = distinct !DISubprogram(name: "func_loc", scope: !2, file: !2, spFlags: DISPFlagDefinition, unit: !1)
+!3 = distinct !DISubprogram(name: "func_loc", scope: !2, file: !2, spFlags: DISPFlagDefinition, unit: !1, line: 42)
 
 ; // -----
 
@@ -538,7 +541,7 @@ define void @noname_subprogram(ptr %arg) !dbg !8 {
 ; // -----
 
 ; CHECK:      #[[MODULE:.+]] = #llvm.di_module<
-; CHECK-SAME: file = #{{.*}}, scope = #{{.*}}, name = "module", 
+; CHECK-SAME: file = #{{.*}}, scope = #{{.*}}, name = "module",
 ; CHECK-SAME: configMacros = "bar", includePath = "/",
 ; CHECK-SAME: apinotes = "/", line = 42, isDecl = true
 ; CHECK-SAME: >
diff --git a/mlir/test/Target/LLVMIR/arm-sme-invalid.mlir b/mlir/test/Target/LLVMIR/arm-sme-invalid.mlir
index ae99ac5e02..b3202b26f8 100644
--- a/mlir/test/Target/LLVMIR/arm-sme-invalid.mlir
+++ b/mlir/test/Target/LLVMIR/arm-sme-invalid.mlir
@@ -5,7 +5,7 @@ llvm.func @arm_sme_vector_to_tile_invalid_types(%tileslice : i32,
                                                 %nxv4i1 : vector<[4]xi1>,
                                                 %nxv16i8 : vector<[16]xi8>) {
   %tile = llvm.mlir.constant(0 : index) : i32
-  // expected-error @+1 {{failed to verify that all of {pg, vector} have same shape}}
+  // expected-error @+1 {{failed to verify that all of {predicate, vector} have same shape}}
   "arm_sme.intr.write.horiz"(%tile, %tileslice, %nxv4i1, %nxv16i8) :
       (i32, i32, vector<[4]xi1>, vector<[16]xi8>) -> ()
   llvm.return
@@ -17,7 +17,7 @@ llvm.func @arm_sme_tile_slice_to_vector_invalid_shapes(
   %tileslice : i32, %nxv4i1 : vector<[4]xi1>, %nxv16i8 : vector<[16]xi8>
 ) -> vector<[3]xf32> {
   %tile = llvm.mlir.constant(0 : index) : i32
-  // expected-error @+1 {{failed to verify that all of {vector, pg, res} have same shape}}
+  // expected-error @+1 {{failed to verify that all of {vector, predicate, res} have same shape}}
   %res = "arm_sme.intr.read.horiz"(%nxv16i8, %nxv4i1, %tile, %tileslice) :
       (vector<[16]xi8>, vector<[4]xi1>, i32, i32) -> vector<[3]xf32>
   llvm.return %res : vector<[3]xf32>
diff --git a/mlir/test/Target/LLVMIR/openmp-teams.mlir b/mlir/test/Target/LLVMIR/openmp-teams.mlir
index 18fc2bb5a3..973d2ed395 100644
--- a/mlir/test/Target/LLVMIR/openmp-teams.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-teams.mlir
@@ -124,3 +124,165 @@ llvm.func @omp_teams_branching_shared(%condition: i1, %arg0: i32, %arg1: f32, %a
 // CHECK-NEXT: br label
 // CHECK: ret void
 
+// -----
+
+llvm.func @beforeTeams()
+llvm.func @duringTeams()
+llvm.func @afterTeams()
+
+// CHECK-LABEL: @omp_teams_thread_limit
+// CHECK-SAME: (i32 [[THREAD_LIMIT:.+]])
+llvm.func @omp_teams_thread_limit(%threadLimit: i32) {
+    // CHECK-NEXT: call void @beforeTeams()
+    llvm.call @beforeTeams() : () -> ()
+    // CHECK: [[THREAD_NUM:%.+]] = call i32 @__kmpc_global_thread_num
+    // CHECK-NEXT: call void @__kmpc_push_num_teams_51({{.+}}, i32 [[THREAD_NUM]], i32 0, i32 0, i32 [[THREAD_LIMIT]])
+    // CHECK: call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @1, i32 0, ptr [[OUTLINED_FN:.+]])
+    omp.teams thread_limit(%threadLimit : i32) {
+        llvm.call @duringTeams() : () -> ()
+        omp.terminator
+    }
+    // CHECK: call void @afterTeams
+    llvm.call @afterTeams() : () -> ()
+    // CHECK: ret void
+    llvm.return
+}
+
+// CHECK: define internal void [[OUTLINED_FN]](ptr {{.+}}, ptr {{.+}})
+// CHECK: call void @duringTeams()
+// CHECK: ret void
+
+// -----
+
+llvm.func @beforeTeams()
+llvm.func @duringTeams()
+llvm.func @afterTeams()
+
+// CHECK-LABEL: @omp_teams_num_teams_upper
+// CHECK-SAME: (i32 [[NUM_TEAMS_UPPER:.+]])
+llvm.func @omp_teams_num_teams_upper(%numTeamsUpper: i32) {
+    // CHECK-NEXT: call void @beforeTeams()
+    llvm.call @beforeTeams() : () -> ()
+    // CHECK: [[THREAD_NUM:%.+]] = call i32 @__kmpc_global_thread_num
+    // CHECK-NEXT: call void @__kmpc_push_num_teams_51({{.+}}, i32 [[THREAD_NUM]], i32 [[NUM_TEAMS_UPPER]], i32 [[NUM_TEAMS_UPPER]], i32 0)
+    // CHECK: call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @1, i32 0, ptr [[OUTLINED_FN:.+]])
+    omp.teams num_teams(to %numTeamsUpper : i32) {
+        llvm.call @duringTeams() : () -> ()
+        omp.terminator
+    }
+    // CHECK: call void @afterTeams
+    llvm.call @afterTeams() : () -> ()
+    // CHECK: ret void
+    llvm.return
+}
+
+// CHECK: define internal void [[OUTLINED_FN]](ptr {{.+}}, ptr {{.+}})
+// CHECK: call void @duringTeams()
+// CHECK: ret void
+
+// -----
+
+llvm.func @beforeTeams()
+llvm.func @duringTeams()
+llvm.func @afterTeams()
+
+// CHECK-LABEL: @omp_teams_num_teams_lower_and_upper
+// CHECK-SAME: (i32 [[NUM_TEAMS_LOWER:.+]], i32 [[NUM_TEAMS_UPPER:.+]])
+llvm.func @omp_teams_num_teams_lower_and_upper(%numTeamsLower: i32, %numTeamsUpper: i32) {
+    // CHECK-NEXT: call void @beforeTeams()
+    llvm.call @beforeTeams() : () -> ()
+    // CHECK: [[THREAD_NUM:%.+]] = call i32 @__kmpc_global_thread_num
+    // CHECK-NEXT: call void @__kmpc_push_num_teams_51({{.+}}, i32 [[THREAD_NUM]], i32 [[NUM_TEAMS_LOWER]], i32 [[NUM_TEAMS_UPPER]], i32 0)
+    // CHECK: call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @1, i32 0, ptr [[OUTLINED_FN:.+]])
+    omp.teams num_teams(%numTeamsLower : i32 to %numTeamsUpper: i32) {
+        llvm.call @duringTeams() : () -> ()
+        omp.terminator
+    }
+    // CHECK: call void @afterTeams
+    llvm.call @afterTeams() : () -> ()
+    // CHECK: ret void
+    llvm.return
+}
+
+// CHECK: define internal void [[OUTLINED_FN]](ptr {{.+}}, ptr {{.+}})
+// CHECK: call void @duringTeams()
+// CHECK: ret void
+
+// -----
+
+llvm.func @beforeTeams()
+llvm.func @duringTeams()
+llvm.func @afterTeams()
+
+// CHECK-LABEL: @omp_teams_num_teams_and_thread_limit
+// CHECK-SAME: (i32 [[NUM_TEAMS_LOWER:.+]], i32 [[NUM_TEAMS_UPPER:.+]], i32 [[THREAD_LIMIT:.+]])
+llvm.func @omp_teams_num_teams_and_thread_limit(%numTeamsLower: i32, %numTeamsUpper: i32, %threadLimit: i32) {
+    // CHECK-NEXT: call void @beforeTeams()
+    llvm.call @beforeTeams() : () -> ()
+    // CHECK: [[THREAD_NUM:%.+]] = call i32 @__kmpc_global_thread_num
+    // CHECK-NEXT: call void @__kmpc_push_num_teams_51({{.+}}, i32 [[THREAD_NUM]], i32 [[NUM_TEAMS_LOWER]], i32 [[NUM_TEAMS_UPPER]], i32 [[THREAD_LIMIT]])
+    // CHECK: call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @1, i32 0, ptr [[OUTLINED_FN:.+]])
+    omp.teams num_teams(%numTeamsLower : i32 to %numTeamsUpper: i32) thread_limit(%threadLimit: i32) {
+        llvm.call @duringTeams() : () -> ()
+        omp.terminator
+    }
+    // CHECK: call void @afterTeams
+    llvm.call @afterTeams() : () -> ()
+    // CHECK: ret void
+    llvm.return
+}
+
+// CHECK: define internal void [[OUTLINED_FN]](ptr {{.+}}, ptr {{.+}})
+// CHECK: call void @duringTeams()
+// CHECK: ret void
+
+// -----
+
+llvm.func @beforeTeams()
+llvm.func @duringTeams()
+llvm.func @afterTeams()
+
+// CHECK-LABEL: @teams_if
+// CHECK-SAME: (i1 [[ARG:.+]])
+llvm.func @teams_if(%arg : i1) {
+    // CHECK-NEXT: call void @beforeTeams()
+    llvm.call @beforeTeams() : () -> ()
+    // If the condition is true, then the value of bounds is zero - which basically means "implementation-defined". 
+    // The runtime sees zero and sets a default value of number of teams. This behavior is according to the standard.
+    // The same is true for `thread_limit`.
+    // CHECK: [[NUM_TEAMS_UPPER:%.+]] = select i1 [[ARG]], i32 0, i32 1
+    // CHECK: [[NUM_TEAMS_LOWER:%.+]] = select i1 [[ARG]], i32 0, i32 1
+    // CHECK: call void @__kmpc_push_num_teams_51(ptr {{.+}}, i32 {{.+}}, i32 [[NUM_TEAMS_LOWER]], i32 [[NUM_TEAMS_UPPER]], i32 0)
+    // CHECK: call void {{.+}} @__kmpc_fork_teams({{.+}})
+    omp.teams if(%arg) {
+        llvm.call @duringTeams() : () -> ()
+        omp.terminator
+    }
+    // CHECK: call void @afterTeams()
+    llvm.call @afterTeams() : () -> ()
+    llvm.return
+}
+
+// -----
+
+llvm.func @beforeTeams()
+llvm.func @duringTeams()
+llvm.func @afterTeams()
+
+// CHECK-LABEL: @teams_if_with_num_teams
+// CHECK-SAME: (i1 [[CONDITION:.+]], i32 [[NUM_TEAMS_LOWER:.+]], i32 [[NUM_TEAMS_UPPER:.+]], i32 [[THREAD_LIMIT:.+]])
+llvm.func @teams_if_with_num_teams(%condition: i1, %numTeamsLower: i32, %numTeamsUpper: i32, %threadLimit: i32) {
+    // CHECK: call void @beforeTeams()
+    llvm.call @beforeTeams() : () -> ()
+    // CHECK: [[NUM_TEAMS_UPPER_NEW:%.+]] = select i1 [[CONDITION]], i32 [[NUM_TEAMS_UPPER]], i32 1
+    // CHECK: [[NUM_TEAMS_LOWER_NEW:%.+]] = select i1 [[CONDITION]], i32 [[NUM_TEAMS_LOWER]], i32 1
+    // CHECK: call void @__kmpc_push_num_teams_51(ptr {{.+}}, i32 {{.+}}, i32 [[NUM_TEAMS_LOWER_NEW]], i32 [[NUM_TEAMS_UPPER_NEW]], i32 [[THREAD_LIMIT]])
+    // CHECK: call void {{.+}} @__kmpc_fork_teams({{.+}})
+    omp.teams if(%condition) num_teams(%numTeamsLower: i32 to %numTeamsUpper: i32) thread_limit(%threadLimit: i32) {
+        llvm.call @duringTeams() : () -> ()
+        omp.terminator
+    }
+    // CHECK: call void @afterTeams()
+    llvm.call @afterTeams() : () -> ()
+    llvm.return
+}
diff --git a/mlir/test/lib/Dialect/Affine/TestReifyValueBounds.cpp b/mlir/test/lib/Dialect/Affine/TestReifyValueBounds.cpp
index ad017cef1b..393e83beb4 100644
--- a/mlir/test/lib/Dialect/Affine/TestReifyValueBounds.cpp
+++ b/mlir/test/lib/Dialect/Affine/TestReifyValueBounds.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
+#include "mlir/Dialect/Affine/IR/ValueBoundsOpInterfaceImpl.h"
 #include "mlir/Dialect/Affine/Transforms/Transforms.h"
 #include "mlir/Dialect/Arith/Transforms/Transforms.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
@@ -186,14 +187,26 @@ static LogicalResult testEquality(func::FuncOp funcOp) {
         op->emitOpError("invalid op");
         return WalkResult::skip();
       }
-      FailureOr<bool> equal = ValueBoundsConstraintSet::areEqual(
-          op->getOperand(0), op->getOperand(1));
-      if (failed(equal)) {
-        op->emitError("could not determine equality");
-      } else if (*equal) {
-        op->emitRemark("equal");
+      if (op->hasAttr("compose")) {
+        FailureOr<int64_t> delta = affine::fullyComposeAndComputeConstantDelta(
+            op->getOperand(0), op->getOperand(1));
+        if (failed(delta)) {
+          op->emitError("could not determine equality");
+        } else if (*delta == 0) {
+          op->emitRemark("equal");
+        } else {
+          op->emitRemark("different");
+        }
       } else {
-        op->emitRemark("different");
+        FailureOr<bool> equal = ValueBoundsConstraintSet::areEqual(
+            op->getOperand(0), op->getOperand(1));
+        if (failed(equal)) {
+          op->emitError("could not determine equality");
+        } else if (*equal) {
+          op->emitRemark("equal");
+        } else {
+          op->emitRemark("different");
+        }
       }
     }
     return WalkResult::advance();
diff --git a/mlir/test/lib/Dialect/Linalg/TestLinalgElementwiseFusion.cpp b/mlir/test/lib/Dialect/Linalg/TestLinalgElementwiseFusion.cpp
index e41481a9e5..7f68f4aec3 100644
--- a/mlir/test/lib/Dialect/Linalg/TestLinalgElementwiseFusion.cpp
+++ b/mlir/test/lib/Dialect/Linalg/TestLinalgElementwiseFusion.cpp
@@ -258,7 +258,7 @@ struct TestLinalgElementwiseFusion
       SmallVector<int64_t, 2> dims(collapseDimensions.begin(),
                                    collapseDimensions.end());
       linalg::GetCollapsableDimensionsFn collapseFn =
-          [&dims](linalg::GenericOp op) {
+          [&dims](linalg::LinalgOp op) {
             SmallVector<ReassociationIndices> reassociations;
             reassociations.emplace_back(dims);
             return reassociations;
diff --git a/mlir/test/lib/Dialect/SCF/TestSCFUtils.cpp b/mlir/test/lib/Dialect/SCF/TestSCFUtils.cpp
index 1d40615305..565d076697 100644
--- a/mlir/test/lib/Dialect/SCF/TestSCFUtils.cpp
+++ b/mlir/test/lib/Dialect/SCF/TestSCFUtils.cpp
@@ -50,9 +50,8 @@ struct TestSCFForUtilsPass
         auto newInitValues = forOp.getInitArgs();
         if (newInitValues.empty())
           return;
-        auto yieldOp = cast<scf::YieldOp>(forOp.getBody()->getTerminator());
-        SmallVector<Value> oldYieldValues(yieldOp.getResults().begin(),
-                                          yieldOp.getResults().end());
+        SmallVector<Value> oldYieldValues =
+            llvm::to_vector(forOp.getYieldedValues());
         NewYieldValuesFn fn = [&](OpBuilder &b, Location loc,
                                   ArrayRef<BlockArgument> newBBArgs) {
           SmallVector<Value> newYieldValues;
diff --git a/mlir/test/lib/Dialect/Test/TestOps.td b/mlir/test/lib/Dialect/Test/TestOps.td
index edb63924b3..1add9bd3c3 100644
--- a/mlir/test/lib/Dialect/Test/TestOps.td
+++ b/mlir/test/lib/Dialect/Test/TestOps.td
@@ -2495,6 +2495,29 @@ def TestDefaultStrAttrHasValueOp : TEST_Op<"has_str_value"> {
 def : Pat<(TestDefaultStrAttrNoValueOp $value),
           (TestDefaultStrAttrHasValueOp ConstantStrAttr<StrAttr, "foo">)>;
 
+//===----------------------------------------------------------------------===//
+// Test Ops with variadics
+//===----------------------------------------------------------------------===//
+
+def TestVariadicRewriteSrcOp : TEST_Op<"variadic_rewrite_src_op", [AttrSizedOperandSegments]> {
+  let arguments = (ins
+    Variadic<AnyType>:$arg,
+    AnyType:$brg,
+    Variadic<AnyType>:$crg
+  );
+}
+
+def TestVariadicRewriteDstOp : TEST_Op<"variadic_rewrite_dst_op", [AttrSizedOperandSegments]> {
+  let arguments = (ins
+    AnyType:$brg,
+    Variadic<AnyType>:$crg,
+    Variadic<AnyType>:$arg
+  );
+}
+
+def : Pat<(TestVariadicRewriteSrcOp $arg, $brg, $crg),
+          (TestVariadicRewriteDstOp $brg, $crg, $arg)>;
+
 //===----------------------------------------------------------------------===//
 // Test Ops with Default-Valued Attributes and Differing Print Settings
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/lib/Dialect/Transform/TestTransformDialectInterpreter.cpp b/mlir/test/lib/Dialect/Transform/TestTransformDialectInterpreter.cpp
index c60b21c918..756b7f669b 100644
--- a/mlir/test/lib/Dialect/Transform/TestTransformDialectInterpreter.cpp
+++ b/mlir/test/lib/Dialect/Transform/TestTransformDialectInterpreter.cpp
@@ -158,6 +158,8 @@ class TestTransformDialectInterpreterPass
     }
 
     options = options.enableExpensiveChecks(enableExpensiveChecks);
+    options = options.enableEnforceSingleToplevelTransformOp(
+        enforceSingleToplevelTransformOp);
     if (failed(transform::detail::interpreterBaseRunOnOperationImpl(
             getOperation(), getArgument(), getSharedTransformModule(),
             getTransformLibraryModule(), extraMapping, options,
@@ -170,6 +172,10 @@ class TestTransformDialectInterpreterPass
       *this, "enable-expensive-checks", llvm::cl::init(false),
       llvm::cl::desc("perform expensive checks to better report errors in the "
                      "transform IR")};
+  Option<bool> enforceSingleToplevelTransformOp{
+      *this, "enforce-single-top-level-transform-op", llvm::cl::init(true),
+      llvm::cl::desc("Ensure that only a single top-level transform op is "
+                     "present in the IR.")};
 
   Option<std::string> bindFirstExtraToOps{
       *this, "bind-first-extra-to-ops",
diff --git a/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterface.cpp b/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterface.cpp
index 2573e11979..e5d7dc5440 100644
--- a/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterface.cpp
+++ b/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterface.cpp
@@ -16,6 +16,7 @@
 
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
+#include "mlir/Dialect/GPU/IR/GPUDialect.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/Linalg/Transforms/TilingInterfaceImpl.h"
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
@@ -186,6 +187,51 @@ struct TestTileUsingSCFForOp
   TransformationFilter filter;
 };
 
+/// Pattern for testing `tileUsingSCFForallOp` (that tiles operations using
+/// the `TilingInterface` with `scf.forall` ops for iterating over the tiles)
+/// while using a `filter` to avoid recursive application.
+struct TestTileUsingSCFForallOp
+    : public OpInterfaceRewritePattern<TilingInterface> {
+  TestTileUsingSCFForallOp(MLIRContext *context, scf::SCFTilingOptions options,
+                           TransformationFilter filter = TransformationFilter(),
+                           PatternBenefit benefit = 1)
+      : OpInterfaceRewritePattern<TilingInterface>(context, benefit),
+        options(std::move(options)), filter(std::move(filter)) {}
+
+  /// Construct a generic pattern applied to `opName`.
+  TestTileUsingSCFForallOp(StringRef opName, MLIRContext *context,
+                           scf::SCFTilingOptions options,
+                           TransformationFilter filter = TransformationFilter(),
+                           PatternBenefit benefit = 1)
+      : OpInterfaceRewritePattern<TilingInterface>(context, benefit),
+        options(std::move(options)), filter(std::move(filter)) {}
+
+  LogicalResult matchAndRewrite(TilingInterface op,
+                                PatternRewriter &rewriter) const override {
+    if (failed(filter.checkAndNotify(rewriter, op)))
+      return failure();
+
+    FailureOr<scf::SCFTilingResult> tilingResult =
+        scf::tileUsingSCFForallOp(rewriter, op, options);
+    if (failed(tilingResult))
+      return rewriter.notifyMatchFailure(op, "failed to tile operation");
+
+    if (op->getNumResults()) {
+      rewriter.replaceOp(op, tilingResult->replacements);
+    } else {
+      rewriter.eraseOp(op);
+    }
+
+    for (auto *tiledOp : tilingResult->tiledOps)
+      filter.replaceTransformationFilter(rewriter, tiledOp);
+    return success();
+  }
+
+private:
+  scf::SCFTilingOptions options;
+  TransformationFilter filter;
+};
+
 /// Pattern for testing `TileConsumerAndFuseProducersUsingSCFForOp` pattern
 /// (that tiles and fuses operations using the `TilingInterface` with `scf.for`
 /// ops for iterating over the tiles) while using a `filter` to avoid recursive
@@ -398,9 +444,9 @@ struct TestTilingInterfacePass
   TestTilingInterfacePass(const TestTilingInterfacePass &pass)
       : PassWrapper(pass) {}
   void getDependentDialects(DialectRegistry &registry) const override {
-    registry.insert<affine::AffineDialect, linalg::LinalgDialect,
-                    memref::MemRefDialect, scf::SCFDialect,
-                    tensor::TensorDialect>();
+    registry.insert<affine::AffineDialect, gpu::GPUDialect,
+                    linalg::LinalgDialect, memref::MemRefDialect,
+                    scf::SCFDialect, tensor::TensorDialect>();
     linalg::registerTilingInterfaceExternalModels(registry);
     tensor::registerTilingInterfaceExternalModels(registry);
   }
@@ -415,6 +461,12 @@ struct TestTilingInterfacePass
           "Test tiling using TilingInterface with scf.for operations"),
       llvm::cl::init(false)};
 
+  Option<bool> testTilingForAll{
+      *this, "tile-using-scf-forall",
+      llvm::cl::desc(
+          "Test tiling using TilingInterface with scf.forall operations"),
+      llvm::cl::init(false)};
+
   Option<bool> testTileConsumerFuseAndYieldProducer{
       *this, "tile-consumer-fuse-and-yield-producer-using-scf-for",
       llvm::cl::desc(
@@ -455,6 +507,21 @@ static void addPatternForTiling(MLIRContext *context,
   patterns.add<TestTileUsingSCFForOp>(context, tilingOptions, filter);
 }
 
+static void addPatternForTilingUsingForall(
+    MLIRContext *context, RewritePatternSet &patterns, StringRef filterName,
+    ArrayRef<int64_t> tileSizes,
+    ArrayRef<DeviceMappingAttrInterface> mapping = {},
+    ArrayRef<int64_t> interchange = {}) {
+  scf::SCFTilingOptions tilingOptions;
+  SmallVector<OpFoldResult> tileSizesOfr =
+      getAsIndexOpFoldResult(context, tileSizes);
+  tilingOptions.setTileSizes(tileSizesOfr).setInterchange(interchange);
+  tilingOptions.setMapping(mapping);
+  TransformationFilter filter(StringAttr::get(context, filterName),
+                              StringAttr::get(context, "tiled"));
+  patterns.add<TestTileUsingSCFForallOp>(context, tilingOptions, filter);
+}
+
 static void addPatternForTileFuseAndYield(MLIRContext *context,
                                           RewritePatternSet &patterns,
                                           StringRef filterName,
@@ -514,6 +581,23 @@ void TestTilingInterfacePass::addTestPatterns(MLIRContext *context,
     addPatternForTiling(context, patterns, "simple_copy_memref", {10, 20});
     return;
   }
+  if (testTilingForAll) {
+    // 1. Tiling M and N dims of `linalg.matmul` on tensors.
+    addPatternForTilingUsingForall(
+        context, patterns, "simple_gemm", {10, 20},
+        {gpu::GPUBlockMappingAttr::get(context, gpu::MappingId::DimY),
+         gpu::GPUBlockMappingAttr::get(context, gpu::MappingId::DimX)});
+    // 2. Tiling 3D parallel generic op which implements a transpose.
+    addPatternForTilingUsingForall(context, patterns,
+                                   "parallel_generic_transpose", {10, 0, 20});
+    // 3. Tiling 2D conv op.
+    addPatternForTilingUsingForall(context, patterns, "simple_conv",
+                                   {0, 0, 0, 0, 10, 20, 30});
+    // 4. Tiling a simple op with `linalg.index` inside.
+    addPatternForTilingUsingForall(context, patterns, "indexed_semantics",
+                                   {10, 20});
+    return;
+  }
   if (testTileConsumerAndFuseProducer) {
     // 1. Tile and fuse of gemm with fill producer and bias-add consumer.
     addPatternForTileAndFuse(context, patterns, "fusion", {10, 20});
diff --git a/mlir/test/mlir-query/simple-test.mlir b/mlir/test/mlir-query/simple-test.mlir
new file mode 100644
index 0000000000..a4d0065987
--- /dev/null
+++ b/mlir/test/mlir-query/simple-test.mlir
@@ -0,0 +1,16 @@
+// RUN: mlir-query %s -c "m isConstantOp()" | FileCheck %s
+
+// CHECK: {{.*}}.mlir:5:13: note: "root" binds here
+func.func @simple1() {
+  %c1_i32 = arith.constant 1 : i32
+  return
+}
+
+// CHECK: {{.*}}.mlir:12:11: note: "root" binds here
+// CHECK: {{.*}}.mlir:13:11: note: "root" binds here
+func.func @simple2() {
+  %cst1 = arith.constant 1.0 : f32
+  %cst2 = arith.constant 2.0 : f32
+  %add = arith.addf %cst1, %cst2 : f32
+  return
+}
diff --git a/mlir/test/mlir-tblgen/op-python-bindings.td b/mlir/test/mlir-tblgen/op-python-bindings.td
index 8ca23fa9f4..63dad1cc90 100644
--- a/mlir/test/mlir-tblgen/op-python-bindings.td
+++ b/mlir/test/mlir-tblgen/op-python-bindings.td
@@ -616,6 +616,15 @@ def VariadicRegionOp : TestOp<"variadic_region"> {
 // CHECK: def variadic_region(num_variadic, *, loc=None, ip=None)
 // CHECK:   return _get_op_result_or_op_results(VariadicRegionOp(num_variadic=num_variadic, loc=loc, ip=ip))
 
+// CHECK: @_ods_cext.register_operation(_Dialect)
+// CHECK: class WithSpecialCharactersOp(_ods_ir.OpView):
+// CHECK-LABEL: OPERATION_NAME = "test.123with--special.characters"
+def WithSpecialCharactersOp : TestOp<"123with--special.characters"> {
+}
+
+// CHECK: def _123with__special_characters(*, loc=None, ip=None)
+// CHECK:   return _get_op_result_or_op_results(WithSpecialCharactersOp(loc=loc, ip=ip))
+
 // CHECK: @_ods_cext.register_operation(_Dialect)
 // CHECK: class WithSuccessorsOp(_ods_ir.OpView):
 // CHECK-LABEL: OPERATION_NAME = "test.with_successors"
diff --git a/mlir/test/mlir-tblgen/pattern.mlir b/mlir/test/mlir-tblgen/pattern.mlir
index 5f776338bd..7f9c450f15 100644
--- a/mlir/test/mlir-tblgen/pattern.mlir
+++ b/mlir/test/mlir-tblgen/pattern.mlir
@@ -683,3 +683,16 @@ func.func @testConstantStrAttr() -> () {
   test.no_str_value {value = "bar"}
   return
 }
+
+//===----------------------------------------------------------------------===//
+// Test that patterns with variadics propagate sizes
+//===----------------------------------------------------------------------===//
+
+func.func @testVariadic(%arg_0: i32, %arg_1: i32, %brg: i64,
+    %crg_0: f32, %crg_1: f32, %crg_2: f32, %crg_3: f32) -> () {
+  // CHECK: "test.variadic_rewrite_dst_op"(%arg2, %arg3, %arg4, %arg5, %arg6, %arg0, %arg1) <{operandSegmentSizes = array<i32: 1, 4, 2>}> : (i64, f32, f32, f32, f32, i32, i32) -> ()
+  "test.variadic_rewrite_src_op"(%arg_0, %arg_1, %brg,
+    %crg_0, %crg_1, %crg_2, %crg_3) {operandSegmentSizes = array<i32: 2, 1, 4>} :
+    (i32, i32, i64, f32, f32, f32, f32) -> ()
+  return
+}
diff --git a/mlir/test/mlir-tblgen/utils.td b/mlir/test/mlir-tblgen/utils.td
new file mode 100644
index 0000000000..28e0fecb28
--- /dev/null
+++ b/mlir/test/mlir-tblgen/utils.td
@@ -0,0 +1,23 @@
+// RUN: mlir-tblgen -I %S/../../include %s | FileCheck %s
+
+include "mlir/IR/Utils.td"
+
+// CHECK-DAG: string value = "CamelCaseTest"
+class already_camel_case {
+  string value = snakeCaseToCamelCase<"CamelCaseTest">.ret;
+}
+
+// CHECK-DAG: string value = "Foo"
+class single_word {
+  string value = snakeCaseToCamelCase<"foo">.ret;
+}
+
+// CHECK-DAG: string value = "ThisIsATest"
+class snake_case {
+  string value = snakeCaseToCamelCase<"this_is_a_test">.ret;
+}
+
+// CHECK-DAG: string value = "ThisIsATestAgain"
+class extra_underscores {
+  string value = snakeCaseToCamelCase<"__this__is_a_test__again__">.ret;
+}
diff --git a/mlir/test/mlir-vulkan-runner/iaddcarry_extended.mlir b/mlir/test/mlir-vulkan-runner/iaddcarry_extended.mlir
new file mode 100644
index 0000000000..9b1f1964b3
--- /dev/null
+++ b/mlir/test/mlir-vulkan-runner/iaddcarry_extended.mlir
@@ -0,0 +1,68 @@
+// Make sure that addition with carry produces expected results
+// with and without expansion to primitive add/cmp ops for WebGPU.
+
+// RUN: mlir-vulkan-runner %s \
+// RUN:  --shared-libs=%vulkan-runtime-wrappers,%mlir_runner_utils \
+// RUN:  --entry-point-result=void | FileCheck %s
+
+// RUN: mlir-vulkan-runner %s --vulkan-runner-spirv-webgpu-prepare \
+// RUN:  --shared-libs=%vulkan-runtime-wrappers,%mlir_runner_utils \
+// RUN:  --entry-point-result=void | FileCheck %s
+
+// CHECK: [0, 42, 0, 42]
+// CHECK: [1, 0, 1, 1]
+module attributes {
+  gpu.container_module,
+  spirv.target_env = #spirv.target_env<
+    #spirv.vce<v1.4, [Shader], [SPV_KHR_storage_buffer_storage_class]>, #spirv.resource_limits<>>
+} {
+  gpu.module @kernels {
+    gpu.func @kernel_add(%arg0 : memref<4xi32>, %arg1 : memref<4xi32>, %arg2 : memref<4xi32>, %arg3 : memref<4xi32>)
+      kernel attributes { spirv.entry_point_abi = #spirv.entry_point_abi<workgroup_size = [1, 1, 1]>} {
+      %0 = gpu.block_id x
+      %lhs = memref.load %arg0[%0] : memref<4xi32>
+      %rhs = memref.load %arg1[%0] : memref<4xi32>
+      %sum, %carry = arith.addui_extended %lhs, %rhs : i32, i1
+
+      %carry_i32 = arith.extui %carry : i1 to i32
+
+      memref.store %sum, %arg2[%0] : memref<4xi32> memref.store %carry_i32, %arg3[%0] : memref<4xi32>
+      gpu.return
+    }
+  }
+
+  func.func @main() {
+    %buf0 = memref.alloc() : memref<4xi32>
+    %buf1 = memref.alloc() : memref<4xi32>
+    %buf2 = memref.alloc() : memref<4xi32>
+    %buf3 = memref.alloc() : memref<4xi32>
+    %i32_0 = arith.constant 0 : i32
+
+    // Initialize output buffers.
+    %buf4 = memref.cast %buf2 : memref<4xi32> to memref<?xi32>
+    %buf5 = memref.cast %buf3 : memref<4xi32> to memref<?xi32>
+    call @fillResource1DInt(%buf4, %i32_0) : (memref<?xi32>, i32) -> ()
+    call @fillResource1DInt(%buf5, %i32_0) : (memref<?xi32>, i32) -> ()
+
+    %idx_0 = arith.constant 0 : index
+    %idx_1 = arith.constant 1 : index
+    %idx_4 = arith.constant 4 : index
+
+    // Initialize input buffers.
+    %lhs_vals = arith.constant dense<[-1, 24, 4294967295, 43]> : vector<4xi32>
+    %rhs_vals = arith.constant dense<[1, 18, 1, 4294967295]> : vector<4xi32>
+    vector.store %lhs_vals, %buf0[%idx_0] : memref<4xi32>, vector<4xi32>
+    vector.store %rhs_vals, %buf1[%idx_0] : memref<4xi32>, vector<4xi32>
+
+    gpu.launch_func @kernels::@kernel_add
+        blocks in (%idx_4, %idx_1, %idx_1) threads in (%idx_1, %idx_1, %idx_1)
+        args(%buf0 : memref<4xi32>, %buf1 : memref<4xi32>, %buf2 : memref<4xi32>, %buf3 : memref<4xi32>)
+    %buf_sum = memref.cast %buf4 : memref<?xi32> to memref<*xi32>
+    %buf_carry = memref.cast %buf5 : memref<?xi32> to memref<*xi32>
+    call @printMemrefI32(%buf_sum) : (memref<*xi32>) -> ()
+    call @printMemrefI32(%buf_carry) : (memref<*xi32>) -> ()
+    return
+  }
+  func.func private @fillResource1DInt(%0 : memref<?xi32>, %1 : i32)
+  func.func private @printMemrefI32(%ptr : memref<*xi32>)
+}
diff --git a/mlir/test/python/dialects/affine.py b/mlir/test/python/dialects/affine.py
new file mode 100644
index 0000000000..c5ec854574
--- /dev/null
+++ b/mlir/test/python/dialects/affine.py
@@ -0,0 +1,44 @@
+# RUN: %PYTHON %s | FileCheck %s
+
+from mlir.ir import *
+import mlir.dialects.func as func
+import mlir.dialects.arith as arith
+import mlir.dialects.affine as affine
+import mlir.dialects.memref as memref
+
+
+def run(f):
+    print("\nTEST:", f.__name__)
+    f()
+    return f
+
+
+# CHECK-LABEL: TEST: testAffineStoreOp
+@run
+def testAffineStoreOp():
+    with Context() as ctx, Location.unknown():
+        module = Module.create()
+        with InsertionPoint(module.body):
+            f32 = F32Type.get()
+            index_type = IndexType.get()
+            memref_type_out = MemRefType.get([12, 12], f32)
+
+            # CHECK: func.func @affine_store_test(%[[ARG0:.*]]: index) -> memref<12x12xf32> {
+            @func.FuncOp.from_py_func(index_type)
+            def affine_store_test(arg0):
+                # CHECK: %[[O_VAR:.*]] = memref.alloc() : memref<12x12xf32>
+                mem = memref.AllocOp(memref_type_out, [], []).result
+
+                d0 = AffineDimExpr.get(0)
+                s0 = AffineSymbolExpr.get(0)
+                map = AffineMap.get(1, 1, [s0 * 3, d0 + s0 + 1])
+
+                # CHECK: %[[A1:.*]] = arith.constant 2.100000e+00 : f32
+                a1 = arith.ConstantOp(f32, 2.1)
+
+                # CHECK: affine.store %[[A1]], %alloc[symbol(%[[ARG0]]) * 3, %[[ARG0]] + symbol(%[[ARG0]]) + 1] : memref<12x12xf32>
+                affine.AffineStoreOp(a1, mem, indices=[arg0, arg0], map=map)
+
+                return mem
+
+        print(module)
diff --git a/mlir/test/python/dialects/arith_dialect.py b/mlir/test/python/dialects/arith_dialect.py
index f4a793aee4..6d1c5eab75 100644
--- a/mlir/test/python/dialects/arith_dialect.py
+++ b/mlir/test/python/dialects/arith_dialect.py
@@ -33,3 +33,16 @@ def testFastMathFlags():
             )
             # CHECK: %0 = arith.addf %cst, %cst fastmath<nnan,ninf> : f32
             print(r)
+
+
+# CHECK-LABEL: TEST: testArithValueBuilder
+@run
+def testArithValueBuilder():
+    with Context() as ctx, Location.unknown():
+        module = Module.create()
+        f32_t = F32Type.get()
+
+        with InsertionPoint(module.body):
+            a = arith.constant(value=FloatAttr.get(f32_t, 42.42))
+            # CHECK: %cst = arith.constant 4.242000e+01 : f32
+            print(a)
diff --git a/mlir/test/python/dialects/func.py b/mlir/test/python/dialects/func.py
index 161a12d787..a2014c64d2 100644
--- a/mlir/test/python/dialects/func.py
+++ b/mlir/test/python/dialects/func.py
@@ -84,6 +84,9 @@ def testFunctionCalls():
     qux = func.FuncOp("qux", ([], [F32Type.get()]))
     qux.sym_visibility = StringAttr.get("private")
 
+    con = func.ConstantOp(qux.type, qux.sym_name.value)
+    assert con.type == qux.type
+
     with InsertionPoint(func.FuncOp("caller", ([], [])).add_entry_block()):
         func.CallOp(foo, [])
         func.CallOp([IndexType.get()], "bar", [])
@@ -94,6 +97,7 @@ def testFunctionCalls():
 # CHECK: func private @foo()
 # CHECK: func private @bar() -> index
 # CHECK: func private @qux() -> f32
+# CHECK: %f = func.constant @qux : () -> f32
 # CHECK: func @caller() {
 # CHECK:   call @foo() : () -> ()
 # CHECK:   %0 = call @bar() : () -> index
diff --git a/mlir/test/python/dialects/linalg/ops.py b/mlir/test/python/dialects/linalg/ops.py
index b728e00837..b147551c2e 100644
--- a/mlir/test/python/dialects/linalg/ops.py
+++ b/mlir/test/python/dialects/linalg/ops.py
@@ -157,3 +157,82 @@ def pass_an_op_directly(arg0, arg1):
                 return linalg.matmul(lhs, rhs, outs=init)
 
     print(module)
+
+
+# CHECK-LABEL: TEST: testIdentityRegionOps
+@run
+def testIdentityRegionOps():
+    with Context(), Location.unknown():
+        module = Module.create()
+        f32 = F32Type.get()
+        with InsertionPoint(module.body):
+            # CHECK: %[[VAL_0:.*]] = tensor.empty() : tensor<1x13xf32>
+            # CHECK: %[[VAL_1:.*]] = tensor.empty() : tensor<13x1xf32>
+            op1 = tensor.EmptyOp([1, 13], f32)
+            op2 = tensor.EmptyOp([13, 1], f32)
+            # CHECK: %[[VAL_2:.*]] = linalg.transpose ins(%[[VAL_0]] : tensor<1x13xf32>) outs(%[[VAL_1]] : tensor<13x1xf32>) permutation = [1, 0]
+            op3 = linalg.TransposeOp(
+                result=[RankedTensorType.get((13, 1), f32)],
+                input=op1,
+                init=op2,
+                permutation=[1, 0],
+            )
+            linalg.fill_builtin_region(op3.operation)
+
+            # CHECK: %[[VAL_3:.*]] = linalg.transpose ins(%[[VAL_1]] : tensor<13x1xf32>) outs(%[[VAL_0]] : tensor<1x13xf32>) permutation = [1, 0]
+            op4 = linalg.transpose(op2, outs=[op1], permutation=[1, 0])
+
+            # CHECK:         func.func @transpose_op(%[[VAL_4:.*]]: memref<1x13xf32>, %[[VAL_5:.*]]: memref<13x1xf32>)
+            @func.FuncOp.from_py_func(
+                MemRefType.get((1, 13), f32),
+                MemRefType.get((13, 1), f32),
+            )
+            def transpose_op(op1, op2):
+                # CHECK: linalg.transpose ins(%[[VAL_4]] : memref<1x13xf32>) outs(%[[VAL_5]] : memref<13x1xf32>) permutation = [1, 0]
+                op3 = linalg.TransposeOp(
+                    result=[],
+                    input=op1,
+                    init=op2,
+                    permutation=[1, 0],
+                )
+                linalg.fill_builtin_region(op3.operation)
+                # CHECK: linalg.transpose ins(%[[VAL_5]] : memref<13x1xf32>) outs(%[[VAL_4]] : memref<1x13xf32>) permutation = [1, 0]
+                op4 = linalg.transpose(op2, outs=[op1], permutation=[1, 0])
+
+            # CHECK: %[[VAL_6:.*]] = tensor.empty() : tensor<16xf32>
+            # CHECK: %[[VAL_7:.*]] = tensor.empty() : tensor<16x64xf32>
+            op1 = tensor.EmptyOp([16], f32)
+            op2 = tensor.EmptyOp([16, 64], f32)
+            # CHECK: %[[VAL_8:.*]] = linalg.broadcast ins(%[[VAL_6]] : tensor<16xf32>) outs(%[[VAL_7]] : tensor<16x64xf32>) dimensions = [1]
+            op3 = linalg.BroadcastOp(
+                result=[RankedTensorType.get((16, 64), f32)],
+                input=op1,
+                init=op2,
+                dimensions=[1],
+            )
+            linalg.fill_builtin_region(op3.operation)
+
+            # CHECK: %[[VAL_9:.*]] = tensor.empty() : tensor<64xf32>
+            op4 = tensor.EmptyOp([64], f32)
+            # CHECK: %[[VAL_10:.*]] = linalg.broadcast ins(%[[VAL_9]] : tensor<64xf32>) outs(%[[VAL_7]] : tensor<16x64xf32>) dimensions = [0]
+            op5 = linalg.broadcast(op4, outs=[op2], dimensions=[0])
+
+            # CHECK: func.func @broadcast_op(%[[VAL_11:.*]]: memref<16xf32>, %[[VAL_12:.*]]: memref<16x64xf32>, %[[VAL_13:.*]]: memref<64xf32>)
+            @func.FuncOp.from_py_func(
+                MemRefType.get((16,), f32),
+                MemRefType.get((16, 64), f32),
+                MemRefType.get((64,), f32),
+            )
+            def broadcast_op(op1, op2, op3):
+                # CHECK: linalg.broadcast ins(%[[VAL_11]] : memref<16xf32>) outs(%[[VAL_12]] : memref<16x64xf32>) dimensions = [1]
+                op4 = linalg.BroadcastOp(
+                    result=[],
+                    input=op1,
+                    init=op2,
+                    dimensions=[1],
+                )
+                linalg.fill_builtin_region(op4.operation)
+                # CHECK: linalg.broadcast ins(%[[VAL_13]] : memref<64xf32>) outs(%[[VAL_12]] : memref<16x64xf32>) dimensions = [0]
+                op5 = linalg.broadcast(op3, outs=[op2], dimensions=[0])
+
+    print(module)
diff --git a/mlir/test/python/dialects/sparse_tensor/dialect.py b/mlir/test/python/dialects/sparse_tensor/dialect.py
index d80b878323..240db6ebd1 100644
--- a/mlir/test/python/dialects/sparse_tensor/dialect.py
+++ b/mlir/test/python/dialects/sparse_tensor/dialect.py
@@ -32,12 +32,14 @@ def testEncodingAttr1D():
         print(f"lvl_types: {casted.lvl_types}")
         # CHECK: dim_to_lvl: None
         print(f"dim_to_lvl: {casted.dim_to_lvl}")
+        # CHECK: lvl_to_dim: None
+        print(f"lvl_to_dim: {casted.lvl_to_dim}")
         # CHECK: pos_width: 16
         print(f"pos_width: {casted.pos_width}")
         # CHECK: crd_width: 32
         print(f"crd_width: {casted.crd_width}")
 
-        created = st.EncodingAttr.get(casted.lvl_types, None, 0, 0)
+        created = st.EncodingAttr.get(casted.lvl_types, None, None, 0, 0)
         # CHECK: #sparse_tensor.encoding<{ map = (d0) -> (d0 : compressed) }>
         print(created)
         # CHECK: created_equal: False
@@ -72,12 +74,20 @@ def testEncodingAttr2D():
         print(f"lvl_types: {casted.lvl_types}")
         # CHECK: dim_to_lvl: (d0, d1) -> (d1, d0)
         print(f"dim_to_lvl: {casted.dim_to_lvl}")
+        # CHECK: lvl_to_dim: (d0, d1) -> (d1, d0)
+        print(f"lvl_to_dim: {casted.lvl_to_dim}")
         # CHECK: pos_width: 8
         print(f"pos_width: {casted.pos_width}")
         # CHECK: crd_width: 32
         print(f"crd_width: {casted.crd_width}")
 
-        created = st.EncodingAttr.get(casted.lvl_types, casted.dim_to_lvl, 8, 32)
+        created = st.EncodingAttr.get(
+            casted.lvl_types,
+            casted.dim_to_lvl,
+            casted.lvl_to_dim,
+            8,
+            32,
+        )
         # CHECK: #sparse_tensor.encoding<{ map = (d0, d1) -> (d1 : dense, d0 : compressed), posWidth = 8, crdWidth = 32 }>
         print(created)
         # CHECK: created_equal: True
diff --git a/mlir/test/python/dialects/transform_structured_ext.py b/mlir/test/python/dialects/transform_structured_ext.py
index 0f89d41374..c9b7802e1c 100644
--- a/mlir/test/python/dialects/transform_structured_ext.py
+++ b/mlir/test/python/dialects/transform_structured_ext.py
@@ -439,7 +439,7 @@ def testTileToForallCompact(target):
     structured.TileUsingForallOp(matmul, num_threads=[2, 3, 4])
     # CHECK-LABEL: TEST: testTileToForallCompact
     # CHECK: = transform.structured.tile_using_forall
-    # CHECK-SAME: num_threads [2, 3, 4] tile_sizes []
+    # CHECK-SAME: num_threads [2, 3, 4]
     # CHECK-SAME: (!transform.op<"linalg.matmul">) -> (!transform.any_op, !transform.any_op)
 
 
@@ -454,7 +454,7 @@ def testTileToForallLoopsAndTileOpTypes(target):
     )
     # CHECK-LABEL: TEST: testTileToForallLoopsAndTileOpTypes
     # CHECK: = transform.structured.tile_using_forall
-    # CHECK-SAME: num_threads [2, 3, 4] tile_sizes []
+    # CHECK-SAME: num_threads [2, 3, 4]
     # CHECK-SAME: (!transform.any_op) -> (!transform.op<"scf.forall">, !transform.op<"linalg.matmul">)
 
 
@@ -464,7 +464,7 @@ def testTileToForallTileSizes(target):
     structured.TileUsingForallOp(target, tile_sizes=[2, 3, 4])
     # CHECK-LABEL: TEST: testTileToForallTileSizes
     # CHECK: = transform.structured.tile_using_forall
-    # CHECK-SAME: num_threads [] tile_sizes [2, 3, 4]
+    # CHECK-SAME: tile_sizes [2, 3, 4]
 
 
 @run
diff --git a/mlir/test/python/dialects/transform_vector_ext.py b/mlir/test/python/dialects/transform_vector_ext.py
index 1a0a9e1d6e..a51f2154d1 100644
--- a/mlir/test/python/dialects/transform_vector_ext.py
+++ b/mlir/test/python/dialects/transform_vector_ext.py
@@ -94,30 +94,24 @@ def enum_configurable_patterns():
     )
 
     # CHECK: transform.apply_patterns.vector.lower_transpose
-    # CHECK-SAME: lowering_strategy = eltwise
-    # CHECK-SAME: avx2_lowering_strategy = false
     vector.ApplyLowerTransposePatternsOp()
     # CHECK: transform.apply_patterns.vector.lower_transpose
-    # CHECK-SAME: lowering_strategy = eltwise
-    # CHECK-SAME: avx2_lowering_strategy = false
+    # This is the default strategy, not printed.
     vector.ApplyLowerTransposePatternsOp(
         lowering_strategy=vector.VectorTransposeLowering.EltWise
     )
     # CHECK: transform.apply_patterns.vector.lower_transpose
     # CHECK-SAME: lowering_strategy = flat_transpose
-    # CHECK-SAME: avx2_lowering_strategy = false
     vector.ApplyLowerTransposePatternsOp(
         lowering_strategy=vector.VectorTransposeLowering.Flat
     )
     # CHECK: transform.apply_patterns.vector.lower_transpose
     # CHECK-SAME: lowering_strategy = shuffle_1d
-    # CHECK-SAME: avx2_lowering_strategy = false
     vector.ApplyLowerTransposePatternsOp(
         lowering_strategy=vector.VectorTransposeLowering.Shuffle1D
     )
     # CHECK: transform.apply_patterns.vector.lower_transpose
     # CHECK-SAME: lowering_strategy = shuffle_16x16
-    # CHECK-SAME: avx2_lowering_strategy = false
     vector.ApplyLowerTransposePatternsOp(
         lowering_strategy=vector.VectorTransposeLowering.Shuffle16x16
     )
diff --git a/mlir/test/python/ir/insertion_point.py b/mlir/test/python/ir/insertion_point.py
index 0dc7d757f5..268d2e77d0 100644
--- a/mlir/test/python/ir/insertion_point.py
+++ b/mlir/test/python/ir/insertion_point.py
@@ -27,6 +27,8 @@ def test_insert_at_block_end():
         )
         entry_block = module.body.operations[0].regions[0].blocks[0]
         ip = InsertionPoint(entry_block)
+        assert ip.block == entry_block
+        assert ip.ref_operation is None
         ip.insert(Operation.create("custom.op2"))
         # CHECK: "custom.op1"
         # CHECK: "custom.op2"
@@ -51,6 +53,8 @@ def test_insert_before_operation():
         )
         entry_block = module.body.operations[0].regions[0].blocks[0]
         ip = InsertionPoint(entry_block.operations[1])
+        assert ip.block == entry_block
+        assert ip.ref_operation == entry_block.operations[1]
         ip.insert(Operation.create("custom.op3"))
         # CHECK: "custom.op1"
         # CHECK: "custom.op3"
@@ -75,6 +79,8 @@ def test_insert_at_block_begin():
         )
         entry_block = module.body.operations[0].regions[0].blocks[0]
         ip = InsertionPoint.at_block_begin(entry_block)
+        assert ip.block == entry_block
+        assert ip.ref_operation == entry_block.operations[0]
         ip.insert(Operation.create("custom.op1"))
         # CHECK: "custom.op1"
         # CHECK: "custom.op2"
@@ -108,6 +114,8 @@ def test_insert_at_terminator():
         )
         entry_block = module.body.operations[0].regions[0].blocks[0]
         ip = InsertionPoint.at_block_terminator(entry_block)
+        assert ip.block == entry_block
+        assert ip.ref_operation == entry_block.operations[1]
         ip.insert(Operation.create("custom.op2"))
         # CHECK: "custom.op1"
         # CHECK: "custom.op2"
diff --git a/mlir/test/python/pass_manager.py b/mlir/test/python/pass_manager.py
index 4b3a02ac42..e7f79ddc75 100644
--- a/mlir/test/python/pass_manager.py
+++ b/mlir/test/python/pass_manager.py
@@ -4,6 +4,8 @@
 from mlir.ir import *
 from mlir.passmanager import *
 from mlir.dialects.func import FuncOp
+from mlir.dialects.builtin import ModuleOp
+
 
 # Log everything to stderr and flush so that we have a unified stream to match
 # errors/info emitted by MLIR to stderr.
@@ -33,6 +35,7 @@ def testCapsule():
 
 run(testCapsule)
 
+
 # CHECK-LABEL: TEST: testConstruct
 @run
 def testConstruct():
@@ -68,6 +71,7 @@ def testParseSuccess():
 
 run(testParseSuccess)
 
+
 # Verify successful round-trip.
 # CHECK-LABEL: TEST: testParseSpacedPipeline
 def testParseSpacedPipeline():
@@ -84,6 +88,7 @@ def testParseSpacedPipeline():
 
 run(testParseSpacedPipeline)
 
+
 # Verify failure on unregistered pass.
 # CHECK-LABEL: TEST: testParseFail
 def testParseFail():
@@ -102,6 +107,7 @@ def testParseFail():
 
 run(testParseFail)
 
+
 # Check that adding to a pass manager works
 # CHECK-LABEL: TEST: testAdd
 @run
@@ -147,6 +153,7 @@ def testRunPipeline():
 # CHECK: func.return        , 1
 run(testRunPipeline)
 
+
 # CHECK-LABEL: TEST: testRunPipelineError
 @run
 def testRunPipelineError():
@@ -162,4 +169,94 @@ def testRunPipelineError():
             # CHECK:   error: "-":1:1: 'test.op' op trying to schedule a pass on an unregistered operation
             # CHECK:    note: "-":1:1: see current operation: "test.op"() : () -> ()
             # CHECK: >
-            print(f"Exception: <{e}>")
+            log(f"Exception: <{e}>")
+
+
+# CHECK-LABEL: TEST: testPostPassOpInvalidation
+@run
+def testPostPassOpInvalidation():
+    with Context() as ctx:
+        module = ModuleOp.parse(
+            """
+          module {
+            arith.constant 10
+            func.func @foo() {
+              arith.constant 10
+              return
+            }
+          }
+        """
+        )
+
+        # CHECK: invalidate_ops=False
+        log("invalidate_ops=False")
+
+        outer_const_op = module.body.operations[0]
+        # CHECK: %[[VAL0:.*]] = arith.constant 10 : i64
+        log(outer_const_op)
+
+        func_op = module.body.operations[1]
+        # CHECK: func.func @[[FOO:.*]]() {
+        # CHECK:   %[[VAL1:.*]] = arith.constant 10 : i64
+        # CHECK:   return
+        # CHECK: }
+        log(func_op)
+
+        inner_const_op = func_op.body.blocks[0].operations[0]
+        # CHECK: %[[VAL1]] = arith.constant 10 : i64
+        log(inner_const_op)
+
+        PassManager.parse("builtin.module(canonicalize)").run(
+            module, invalidate_ops=False
+        )
+        # CHECK: func.func @foo() {
+        # CHECK:   return
+        # CHECK: }
+        log(func_op)
+
+        # CHECK: func.func @foo() {
+        # CHECK:   return
+        # CHECK: }
+        log(module)
+
+        # CHECK: invalidate_ops=True
+        log("invalidate_ops=True")
+
+        module = ModuleOp.parse(
+            """
+          module {
+            arith.constant 10
+            func.func @foo() {
+              arith.constant 10
+              return
+            }
+          }
+        """
+        )
+        outer_const_op = module.body.operations[0]
+        func_op = module.body.operations[1]
+        inner_const_op = func_op.body.blocks[0].operations[0]
+
+        PassManager.parse("builtin.module(canonicalize)").run(module)
+        try:
+            log(func_op)
+        except RuntimeError as e:
+            # CHECK: the operation has been invalidated
+            log(e)
+
+        try:
+            log(outer_const_op)
+        except RuntimeError as e:
+            # CHECK: the operation has been invalidated
+            log(e)
+
+        try:
+            log(inner_const_op)
+        except RuntimeError as e:
+            # CHECK: the operation has been invalidated
+            log(e)
+
+        # CHECK: func.func @foo() {
+        # CHECK:   return
+        # CHECK: }
+        log(module)
diff --git a/mlir/tools/CMakeLists.txt b/mlir/tools/CMakeLists.txt
index e9a1e4d625..a01f74f737 100644
--- a/mlir/tools/CMakeLists.txt
+++ b/mlir/tools/CMakeLists.txt
@@ -2,6 +2,7 @@ add_subdirectory(mlir-lsp-server)
 add_subdirectory(mlir-opt)
 add_subdirectory(mlir-parser-fuzzer)
 add_subdirectory(mlir-pdll-lsp-server)
+add_subdirectory(mlir-query)
 add_subdirectory(mlir-reduce)
 add_subdirectory(mlir-shlib)
 add_subdirectory(mlir-spirv-cpu-runner)
diff --git a/mlir/tools/mlir-query/CMakeLists.txt b/mlir/tools/mlir-query/CMakeLists.txt
new file mode 100644
index 0000000000..ef2e5a84b5
--- /dev/null
+++ b/mlir/tools/mlir-query/CMakeLists.txt
@@ -0,0 +1,20 @@
+get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS)
+
+if(MLIR_INCLUDE_TESTS)
+  set(test_libs
+    MLIRTestDialect
+    )
+endif()
+
+add_mlir_tool(mlir-query
+  mlir-query.cpp
+  )
+llvm_update_compile_flags(mlir-query)
+target_link_libraries(mlir-query
+  PRIVATE
+  ${dialect_libs}
+  ${test_libs}
+  MLIRQueryLib
+  )
+
+mlir_check_link_libraries(mlir-query)
diff --git a/mlir/tools/mlir-query/mlir-query.cpp b/mlir/tools/mlir-query/mlir-query.cpp
new file mode 100644
index 0000000000..0ed4f94d58
--- /dev/null
+++ b/mlir/tools/mlir-query/mlir-query.cpp
@@ -0,0 +1,63 @@
+//===- mlir-query.cpp - MLIR Query Driver ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This is a command line utility that queries a file from/to MLIR using one
+// of the registered queries.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/IR/Dialect.h"
+#include "mlir/IR/MLIRContext.h"
+#include "mlir/IR/Matchers.h"
+#include "mlir/InitAllDialects.h"
+#include "mlir/Query/Matcher/Registry.h"
+#include "mlir/Tools/mlir-query/MlirQueryMain.h"
+
+using namespace mlir;
+
+// This is needed because these matchers are defined as overloaded functions.
+using HasOpAttrName = detail::AttrOpMatcher(StringRef);
+using HasOpName = detail::NameOpMatcher(StringRef);
+using IsConstantOp = detail::constant_op_matcher();
+
+namespace test {
+#ifdef MLIR_INCLUDE_TESTS
+void registerTestDialect(DialectRegistry &);
+#endif
+} // namespace test
+
+int main(int argc, char **argv) {
+
+  DialectRegistry dialectRegistry;
+  registerAllDialects(dialectRegistry);
+
+  query::matcher::Registry matcherRegistry;
+
+  // Matchers registered in alphabetical order for consistency:
+  matcherRegistry.registerMatcher("hasOpAttrName",
+                                  static_cast<HasOpAttrName *>(m_Attr));
+  matcherRegistry.registerMatcher("hasOpName", static_cast<HasOpName *>(m_Op));
+  matcherRegistry.registerMatcher("isConstantOp",
+                                  static_cast<IsConstantOp *>(m_Constant));
+  matcherRegistry.registerMatcher("isNegInfFloat", m_NegInfFloat);
+  matcherRegistry.registerMatcher("isNegZeroFloat", m_NegZeroFloat);
+  matcherRegistry.registerMatcher("isNonZero", m_NonZero);
+  matcherRegistry.registerMatcher("isOne", m_One);
+  matcherRegistry.registerMatcher("isOneFloat", m_OneFloat);
+  matcherRegistry.registerMatcher("isPosInfFloat", m_PosInfFloat);
+  matcherRegistry.registerMatcher("isPosZeroFloat", m_PosZeroFloat);
+  matcherRegistry.registerMatcher("isZero", m_Zero);
+  matcherRegistry.registerMatcher("isZeroFloat", m_AnyZeroFloat);
+
+#ifdef MLIR_INCLUDE_TESTS
+  test::registerTestDialect(dialectRegistry);
+#endif
+  MLIRContext context(dialectRegistry);
+
+  return failed(mlirQueryMain(argc, argv, context, matcherRegistry));
+}
diff --git a/mlir/tools/mlir-tblgen/OpDocGen.cpp b/mlir/tools/mlir-tblgen/OpDocGen.cpp
index 088d34597f..773ad6ec19 100644
--- a/mlir/tools/mlir-tblgen/OpDocGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpDocGen.cpp
@@ -42,6 +42,10 @@ llvm::cl::opt<std::string>
     stripPrefix("strip-prefix",
                 llvm::cl::desc("Strip prefix of the fully qualified names"),
                 llvm::cl::init("::mlir::"), llvm::cl::cat(docCat));
+llvm::cl::opt<bool> allowHugoSpecificFeatures(
+    "allow-hugo-specific-features",
+    llvm::cl::desc("Allows using features specific to Hugo"),
+    llvm::cl::init(false), llvm::cl::cat(docCat));
 
 using namespace llvm;
 using namespace mlir;
@@ -50,8 +54,9 @@ using mlir::tblgen::Operator;
 
 void mlir::tblgen::emitSummary(StringRef summary, raw_ostream &os) {
   if (!summary.empty()) {
-    char first = std::toupper(summary.front());
-    llvm::StringRef rest = summary.drop_front();
+    llvm::StringRef trimmed = summary.trim();
+    char first = std::toupper(trimmed.front());
+    llvm::StringRef rest = trimmed.drop_front();
     os << "\n_" << first << rest << "_\n\n";
   }
 }
@@ -213,7 +218,7 @@ static void emitOpDoc(const Operator &op, raw_ostream &os) {
       os << "<td><code>" << it.name << "</code></td><td>" << storageType
          << "</td><td>";
       StringRef description = resolveAttrDescription(it.attr);
-      if (!description.empty()) {
+      if (allowHugoSpecificFeatures && !description.empty()) {
         // Expandable description.
         // This appears as just the summary, but when clicked shows the full
         // description.
@@ -227,7 +232,7 @@ static void emitOpDoc(const Operator &op, raw_ostream &os) {
       }
       os << "</td></tr>\n";
     }
-    os << "<table>\n";
+    os << "</table>\n";
   }
 
   // Emit each of the operands.
diff --git a/mlir/tools/mlir-tblgen/OpFormatGen.cpp b/mlir/tools/mlir-tblgen/OpFormatGen.cpp
index bdb97866a4..18ca34379a 100644
--- a/mlir/tools/mlir-tblgen/OpFormatGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpFormatGen.cpp
@@ -2009,6 +2009,16 @@ static void genEnumAttrPrinter(const NamedAttribute *var, const Operator &op,
           "  }\n";
 }
 
+/// Generate a check that a DefaultValuedAttr has a value that is non-default.
+static void genNonDefaultValueCheck(MethodBody &body, const Operator &op,
+                                    AttributeVariable &attrElement) {
+  FmtContext fctx;
+  Attribute attr = attrElement.getVar()->attr;
+  fctx.withBuilder("::mlir::OpBuilder((*this)->getContext())");
+  body << " && " << op.getGetterName(attrElement.getVar()->name) << "Attr() != "
+       << tgfmt(attr.getConstBuilderTemplate(), &fctx, attr.getDefaultValue());
+}
+
 /// Generate the check for the anchor of an optional group.
 static void genOptionalGroupPrinterAnchor(FormatElement *anchor,
                                           const Operator &op,
@@ -2042,12 +2052,7 @@ static void genOptionalGroupPrinterAnchor(FormatElement *anchor,
         if (attr.hasDefaultValue()) {
           // Consider a default-valued attribute as present if it's not the
           // default value.
-          FmtContext fctx;
-          fctx.withBuilder("::mlir::OpBuilder((*this)->getContext())");
-          body << " && " << op.getGetterName(element->getVar()->name)
-               << "Attr() != "
-               << tgfmt(attr.getConstBuilderTemplate(), &fctx,
-                        attr.getDefaultValue());
+          genNonDefaultValueCheck(body, op, *element);
           return;
         }
         llvm_unreachable("attribute must be optional or default-valued");
@@ -2158,7 +2163,6 @@ void OperationFormat::genElementPrinter(FormatElement *element,
 
   // Emit the OIList
   if (auto *oilist = dyn_cast<OIListElement>(element)) {
-    genLiteralPrinter(" ", body, shouldEmitSpace, lastWasPunctuation);
     for (auto clause : oilist->getClauses()) {
       LiteralElement *lelement = std::get<0>(clause);
       ArrayRef<FormatElement *> pelement = std::get<1>(clause);
@@ -2170,8 +2174,14 @@ void OperationFormat::genElementPrinter(FormatElement *element,
       for (VariableElement *var : vars) {
         TypeSwitch<FormatElement *>(var)
             .Case([&](AttributeVariable *attrEle) {
-              body << " || " << op.getGetterName(attrEle->getVar()->name)
+              body << " || (" << op.getGetterName(attrEle->getVar()->name)
                    << "Attr()";
+              Attribute attr = attrEle->getVar()->attr;
+              if (attr.hasDefaultValue()) {
+                // Don't print default-valued attributes.
+                genNonDefaultValueCheck(body, op, *attrEle);
+              }
+              body << ")";
             })
             .Case([&](OperandVariable *ele) {
               if (ele->getVar()->isVariadic()) {
diff --git a/mlir/tools/mlir-tblgen/OpPythonBindingGen.cpp b/mlir/tools/mlir-tblgen/OpPythonBindingGen.cpp
index 2c81538b7b..c8ef847210 100644
--- a/mlir/tools/mlir-tblgen/OpPythonBindingGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpPythonBindingGen.cpp
@@ -30,14 +30,9 @@ constexpr const char *fileHeader = R"Py(
 # Autogenerated by mlir-tblgen; don't manually edit.
 
 from ._ods_common import _cext as _ods_cext
-from ._ods_common import extend_opview_class as _ods_extend_opview_class, segmented_accessor as _ods_segmented_accessor, equally_sized_accessor as _ods_equally_sized_accessor, get_default_loc_context as _ods_get_default_loc_context, get_op_result_or_value as _get_op_result_or_value, get_op_results_or_values as _get_op_results_or_values, get_op_result_or_op_results as _get_op_result_or_op_results
+from ._ods_common import segmented_accessor as _ods_segmented_accessor, equally_sized_accessor as _ods_equally_sized_accessor, get_default_loc_context as _ods_get_default_loc_context, get_op_result_or_value as _get_op_result_or_value, get_op_results_or_values as _get_op_results_or_values, get_op_result_or_op_results as _get_op_result_or_op_results
 _ods_ir = _ods_cext.ir
 
-try:
-  from . import _{0}_ops_ext as _ods_ext_module
-except ImportError:
-  _ods_ext_module = None
-
 import builtins
 from typing import Sequence as _Sequence, Union as _Union
 
@@ -62,7 +57,6 @@ from ._{0}_ops_gen import _Dialect
 ///   {1} is the operation name.
 constexpr const char *opClassTemplate = R"Py(
 @_ods_cext.register_operation(_Dialect)
-@_ods_extend_opview_class(_ods_ext_module)
 class {0}(_ods_ir.OpView):
   OPERATION_NAME = "{1}"
 )Py";
@@ -301,13 +295,17 @@ static bool isODSReserved(StringRef str) {
 /// (does not change the `name` if it already is suitable) and returns the
 /// modified version.
 static std::string sanitizeName(StringRef name) {
-  std::string processed_str = name.str();
+  std::string processedStr = name.str();
+  std::replace_if(
+      processedStr.begin(), processedStr.end(),
+      [](char c) { return !llvm::isAlnum(c); }, '_');
 
-  std::replace(processed_str.begin(), processed_str.end(), '-', '_');
+  if (llvm::isDigit(*processedStr.begin()))
+    return "_" + processedStr;
 
-  if (isPythonReserved(processed_str) || isODSReserved(processed_str))
-    return processed_str + "_";
-  return processed_str;
+  if (isPythonReserved(processedStr) || isODSReserved(processedStr))
+    return processedStr + "_";
+  return processedStr;
 }
 
 static std::string attrSizedTraitForKind(const char *kind) {
@@ -849,10 +847,6 @@ populateBuilderRegions(const Operator &op,
 /// rebuild anew).
 static llvm::SmallVector<std::string> emitDefaultOpBuilder(const Operator &op,
                                                            raw_ostream &os) {
-  // If we are asked to skip default builders, comply.
-  if (op.skipDefaultBuilders())
-    return {};
-
   llvm::SmallVector<std::string> builderArgs;
   llvm::SmallVector<std::string> builderLines;
   llvm::SmallVector<std::string> operandArgNames;
@@ -985,11 +979,6 @@ static void emitRegionAccessors(const Operator &op, raw_ostream &os) {
 static void emitValueBuilder(const Operator &op,
                              llvm::SmallVector<std::string> functionArgs,
                              raw_ostream &os) {
-  // If we are asked to skip default builders, comply.
-  if (op.skipDefaultBuilders())
-    return;
-  auto name = sanitizeName(op.getOperationName());
-  iterator_range<llvm::SplittingIterator> splitName = llvm::split(name, ".");
   // Params with (possibly) default args.
   auto valueBuilderParams =
       llvm::map_range(functionArgs, [](const std::string &argAndMaybeDefault) {
@@ -1008,16 +997,16 @@ static void emitValueBuilder(const Operator &op,
         auto lhs = *llvm::split(arg, "=").begin();
         return (lhs + "=" + llvm::convertToSnakeFromCamelCase(lhs)).str();
       });
-  os << llvm::formatv(
-      valueBuilderTemplate,
-      // Drop dialect name and then sanitize again (to catch e.g. func.return).
-      sanitizeName(llvm::join(++splitName.begin(), splitName.end(), "_")),
-      op.getCppClassName(), llvm::join(valueBuilderParams, ", "),
-      llvm::join(opBuilderArgs, ", "),
-      (op.getNumResults() > 1
-           ? "_Sequence[_ods_ir.OpResult]"
-           : (op.getNumResults() > 0 ? "_ods_ir.OpResult"
-                                     : "_ods_ir.Operation")));
+  std::string nameWithoutDialect =
+      op.getOperationName().substr(op.getOperationName().find('.') + 1);
+  os << llvm::formatv(valueBuilderTemplate, sanitizeName(nameWithoutDialect),
+                      op.getCppClassName(),
+                      llvm::join(valueBuilderParams, ", "),
+                      llvm::join(opBuilderArgs, ", "),
+                      (op.getNumResults() > 1
+                           ? "_Sequence[_ods_ir.OpResult]"
+                           : (op.getNumResults() > 0 ? "_ods_ir.OpResult"
+                                                     : "_ods_ir.Operation")));
 }
 
 /// Emits bindings for a specific Op to the given output stream.
@@ -1049,11 +1038,8 @@ static bool emitAllOps(const llvm::RecordKeeper &records, raw_ostream &os) {
   if (clDialectName.empty())
     llvm::PrintFatalError("dialect name not provided");
 
-  bool isExtension = !clDialectExtensionName.empty();
-  os << llvm::formatv(fileHeader, isExtension
-                                      ? clDialectExtensionName.getValue()
-                                      : clDialectName.getValue());
-  if (isExtension)
+  os << fileHeader;
+  if (!clDialectExtensionName.empty())
     os << llvm::formatv(dialectExtensionTemplate, clDialectName.getValue());
   else
     os << llvm::formatv(dialectClassTemplate, clDialectName.getValue());
diff --git a/mlir/tools/mlir-tblgen/RewriterGen.cpp b/mlir/tools/mlir-tblgen/RewriterGen.cpp
index 9f36a3b430..77c34cb03e 100644
--- a/mlir/tools/mlir-tblgen/RewriterGen.cpp
+++ b/mlir/tools/mlir-tblgen/RewriterGen.cpp
@@ -1743,10 +1743,15 @@ void PatternEmitter::createAggregateLocalVarsForOpArgs(
       "if (auto tmpAttr = {1}) {\n"
       "  tblgen_attrs.emplace_back(rewriter.getStringAttr(\"{0}\"), "
       "tmpAttr);\n}\n";
+  int numVariadic = 0;
+  bool hasOperandSegmentSizes = false;
+  std::vector<std::string> sizes;
   for (int argIndex = 0, e = resultOp.getNumArgs(); argIndex < e; ++argIndex) {
     if (resultOp.getArg(argIndex).is<NamedAttribute *>()) {
       // The argument in the op definition.
       auto opArgName = resultOp.getArgName(argIndex);
+      hasOperandSegmentSizes =
+          hasOperandSegmentSizes || opArgName == "operandSegmentSizes";
       if (auto subTree = node.getArgAsNestedDag(argIndex)) {
         if (!subTree.isNativeCodeCall())
           PrintFatalError(loc, "only NativeCodeCall allowed in nested dag node "
@@ -1766,6 +1771,7 @@ void PatternEmitter::createAggregateLocalVarsForOpArgs(
         resultOp.getArg(argIndex).get<NamedTypeConstraint *>();
     std::string varName;
     if (operand->isVariadic()) {
+      ++numVariadic;
       std::string range;
       if (node.isNestedDagArg(argIndex)) {
         range = childNodeNames.lookup(argIndex);
@@ -1777,7 +1783,9 @@ void PatternEmitter::createAggregateLocalVarsForOpArgs(
       range = symbolInfoMap.getValueAndRangeUse(range);
       os << formatv("for (auto v: {0}) {{\n  tblgen_values.push_back(v);\n}\n",
                     range);
+      sizes.push_back(formatv("static_cast<int32_t>({0}.size())", range));
     } else {
+      sizes.push_back("1");
       os << formatv("tblgen_values.push_back(");
       if (node.isNestedDagArg(argIndex)) {
         os << symbolInfoMap.getValueAndRangeUse(
@@ -1804,6 +1812,19 @@ void PatternEmitter::createAggregateLocalVarsForOpArgs(
       os << ");\n";
     }
   }
+
+  if (numVariadic > 1 && !hasOperandSegmentSizes) {
+    // Only set size if it can't be computed.
+    const auto *sameVariadicSize =
+        resultOp.getTrait("::mlir::OpTrait::SameVariadicOperandSize");
+    if (!sameVariadicSize) {
+      const char *setSizes = R"(
+        tblgen_attrs.emplace_back(rewriter.getStringAttr("operandSegmentSizes"),
+          rewriter.getDenseI32ArrayAttr({{ {0} }));
+          )";
+      os.printReindented(formatv(setSizes, llvm::join(sizes, ", ")).str());
+    }
+  }
 }
 
 StaticMatcherHelper::StaticMatcherHelper(raw_ostream &os,
diff --git a/mlir/unittests/Analysis/Presburger/MatrixTest.cpp b/mlir/unittests/Analysis/Presburger/MatrixTest.cpp
index 6b23cedabf..d05b05e004 100644
--- a/mlir/unittests/Analysis/Presburger/MatrixTest.cpp
+++ b/mlir/unittests/Analysis/Presburger/MatrixTest.cpp
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Analysis/Presburger/Matrix.h"
-#include "mlir/Analysis/Presburger/Fraction.h"
 #include "./Utils.h"
+#include "mlir/Analysis/Presburger/Fraction.h"
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
 
@@ -210,7 +210,8 @@ TEST(MatrixTest, computeHermiteNormalForm) {
   {
     // Hermite form of a unimodular matrix is the identity matrix.
     IntMatrix mat = makeIntMatrix(3, 3, {{2, 3, 6}, {3, 2, 3}, {17, 11, 16}});
-    IntMatrix hermiteForm = makeIntMatrix(3, 3, {{1, 0, 0}, {0, 1, 0}, {0, 0, 1}});
+    IntMatrix hermiteForm =
+        makeIntMatrix(3, 3, {{1, 0, 0}, {0, 1, 0}, {0, 0, 1}});
     checkHermiteNormalForm(mat, hermiteForm);
   }
 
@@ -241,10 +242,71 @@ TEST(MatrixTest, computeHermiteNormalForm) {
   }
 
   {
-    IntMatrix mat =
-        makeIntMatrix(3, 5, {{0, 2, 0, 7, 1}, {-1, 0, 0, -3, 0}, {0, 4, 1, 0, 8}});
-    IntMatrix hermiteForm =
-        makeIntMatrix(3, 5, {{1, 0, 0, 0, 0}, {0, 1, 0, 0, 0}, {0, 0, 1, 0, 0}});
+    IntMatrix mat = makeIntMatrix(
+        3, 5, {{0, 2, 0, 7, 1}, {-1, 0, 0, -3, 0}, {0, 4, 1, 0, 8}});
+    IntMatrix hermiteForm = makeIntMatrix(
+        3, 5, {{1, 0, 0, 0, 0}, {0, 1, 0, 0, 0}, {0, 0, 1, 0, 0}});
     checkHermiteNormalForm(mat, hermiteForm);
   }
 }
+
+TEST(MatrixTest, inverse) {
+  FracMatrix mat = makeFracMatrix(
+      2, 2, {{Fraction(2), Fraction(1)}, {Fraction(7), Fraction(0)}});
+  FracMatrix inverse = makeFracMatrix(
+      2, 2, {{Fraction(0), Fraction(1, 7)}, {Fraction(1), Fraction(-2, 7)}});
+
+  FracMatrix inv(2, 2);
+  mat.determinant(&inv);
+
+  EXPECT_EQ_FRAC_MATRIX(inv, inverse);
+
+  mat = makeFracMatrix(
+      2, 2, {{Fraction(0), Fraction(1)}, {Fraction(0), Fraction(2)}});
+  Fraction det = mat.determinant(nullptr);
+
+  EXPECT_EQ(det, Fraction(0));
+
+  mat = makeFracMatrix(3, 3,
+                       {{Fraction(1), Fraction(2), Fraction(3)},
+                        {Fraction(4), Fraction(8), Fraction(6)},
+                        {Fraction(7), Fraction(8), Fraction(6)}});
+  inverse = makeFracMatrix(3, 3,
+                           {{Fraction(0), Fraction(-1, 3), Fraction(1, 3)},
+                            {Fraction(-1, 2), Fraction(5, 12), Fraction(-1, 6)},
+                            {Fraction(2, 3), Fraction(-1, 6), Fraction(0)}});
+
+  mat.determinant(&inv);
+  EXPECT_EQ_FRAC_MATRIX(inv, inverse);
+
+  mat = makeFracMatrix(0, 0, {});
+  mat.determinant(&inv);
+}
+
+TEST(MatrixTest, intInverse) {
+  IntMatrix mat = makeIntMatrix(2, 2, {{2, 1}, {7, 0}});
+  IntMatrix inverse = makeIntMatrix(2, 2, {{0, -1}, {-7, 2}});
+
+  IntMatrix inv(2, 2);
+  mat.determinant(&inv);
+
+  EXPECT_EQ_INT_MATRIX(inv, inverse);
+
+  mat = makeIntMatrix(
+      4, 4, {{4, 14, 11, 3}, {13, 5, 14, 12}, {13, 9, 7, 14}, {2, 3, 12, 7}});
+  inverse = makeIntMatrix(4, 4,
+                          {{155, 1636, -579, -1713},
+                           {725, -743, 537, -111},
+                           {210, 735, -855, 360},
+                           {-715, -1409, 1401, 1482}});
+
+  mat.determinant(&inv);
+
+  EXPECT_EQ_INT_MATRIX(inv, inverse);
+
+  mat = makeIntMatrix(2, 2, {{0, 0}, {1, 2}});
+
+  MPInt det = mat.determinant(&inv);
+
+  EXPECT_EQ(det, 0);
+}
diff --git a/mlir/unittests/Analysis/Presburger/Utils.h b/mlir/unittests/Analysis/Presburger/Utils.h
index ef4a67d0b8..544577375d 100644
--- a/mlir/unittests/Analysis/Presburger/Utils.h
+++ b/mlir/unittests/Analysis/Presburger/Utils.h
@@ -14,10 +14,10 @@
 #define MLIR_UNITTESTS_ANALYSIS_PRESBURGER_UTILS_H
 
 #include "mlir/Analysis/Presburger/IntegerRelation.h"
+#include "mlir/Analysis/Presburger/Matrix.h"
 #include "mlir/Analysis/Presburger/PWMAFunction.h"
 #include "mlir/Analysis/Presburger/PresburgerRelation.h"
 #include "mlir/Analysis/Presburger/Simplex.h"
-#include "mlir/Analysis/Presburger/Matrix.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/Support/LLVM.h"
 
@@ -28,7 +28,7 @@ namespace mlir {
 namespace presburger {
 
 inline IntMatrix makeIntMatrix(unsigned numRow, unsigned numColumns,
-                         ArrayRef<SmallVector<int, 8>> matrix) {
+                               ArrayRef<SmallVector<int, 8>> matrix) {
   IntMatrix results(numRow, numColumns);
   assert(matrix.size() == numRow);
   for (unsigned i = 0; i < numRow; ++i) {
@@ -40,9 +40,9 @@ inline IntMatrix makeIntMatrix(unsigned numRow, unsigned numColumns,
   return results;
 }
 
-inline Matrix<Fraction> makeFracMatrix(unsigned numRow, unsigned numColumns,
-                         ArrayRef<SmallVector<Fraction, 8>> matrix) {
-  Matrix<Fraction> results(numRow, numColumns);
+inline FracMatrix makeFracMatrix(unsigned numRow, unsigned numColumns,
+                                 ArrayRef<SmallVector<Fraction, 8>> matrix) {
+  FracMatrix results(numRow, numColumns);
   assert(matrix.size() == numRow);
   for (unsigned i = 0; i < numRow; ++i) {
     assert(matrix[i].size() == numColumns &&
@@ -53,6 +53,24 @@ inline Matrix<Fraction> makeFracMatrix(unsigned numRow, unsigned numColumns,
   return results;
 }
 
+inline void EXPECT_EQ_INT_MATRIX(IntMatrix a, IntMatrix b) {
+  EXPECT_EQ(a.getNumRows(), b.getNumRows());
+  EXPECT_EQ(a.getNumColumns(), b.getNumColumns());
+
+  for (unsigned row = 0; row < a.getNumRows(); row++)
+    for (unsigned col = 0; col < a.getNumColumns(); col++)
+      EXPECT_EQ(a(row, col), b(row, col));
+}
+
+inline void EXPECT_EQ_FRAC_MATRIX(FracMatrix a, FracMatrix b) {
+  EXPECT_EQ(a.getNumRows(), b.getNumRows());
+  EXPECT_EQ(a.getNumColumns(), b.getNumColumns());
+
+  for (unsigned row = 0; row < a.getNumRows(); row++)
+    for (unsigned col = 0; col < a.getNumColumns(); col++)
+      EXPECT_EQ(a(row, col), b(row, col));
+}
+
 /// lhs and rhs represent non-negative integers or positive infinity. The
 /// infinity case corresponds to when the Optional is empty.
 inline bool infinityOrUInt64LE(std::optional<MPInt> lhs,
diff --git a/mlir/unittests/Dialect/CMakeLists.txt b/mlir/unittests/Dialect/CMakeLists.txt
index 2d2835c64b..fbb73e8f49 100644
--- a/mlir/unittests/Dialect/CMakeLists.txt
+++ b/mlir/unittests/Dialect/CMakeLists.txt
@@ -9,6 +9,7 @@ target_link_libraries(MLIRDialectTests
 add_subdirectory(Index)
 add_subdirectory(LLVMIR)
 add_subdirectory(MemRef)
+add_subdirectory(SCF)
 add_subdirectory(SparseTensor)
 add_subdirectory(SPIRV)
 add_subdirectory(Transform)
diff --git a/mlir/unittests/Dialect/SCF/CMakeLists.txt b/mlir/unittests/Dialect/SCF/CMakeLists.txt
new file mode 100644
index 0000000000..4d23392af1
--- /dev/null
+++ b/mlir/unittests/Dialect/SCF/CMakeLists.txt
@@ -0,0 +1,8 @@
+add_mlir_unittest(MLIRSCFTests
+  LoopLikeSCFOpsTest.cpp
+)
+target_link_libraries(MLIRSCFTests
+  PRIVATE
+  MLIRIR
+  MLIRSCFDialect
+)
diff --git a/mlir/unittests/Dialect/SCF/LoopLikeSCFOpsTest.cpp b/mlir/unittests/Dialect/SCF/LoopLikeSCFOpsTest.cpp
new file mode 100644
index 0000000000..6bc0fd6113
--- /dev/null
+++ b/mlir/unittests/Dialect/SCF/LoopLikeSCFOpsTest.cpp
@@ -0,0 +1,102 @@
+//===- LoopLikeSCFOpsTest.cpp - SCF LoopLikeOpInterface Tests -------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/Arith/IR/Arith.h"
+#include "mlir/Dialect/SCF/IR/SCF.h"
+#include "mlir/IR/Diagnostics.h"
+#include "mlir/IR/MLIRContext.h"
+#include "mlir/IR/OwningOpRef.h"
+#include "gtest/gtest.h"
+
+using namespace mlir;
+using namespace mlir::scf;
+
+//===----------------------------------------------------------------------===//
+// Test Fixture
+//===----------------------------------------------------------------------===//
+
+class SCFLoopLikeTest : public ::testing::Test {
+protected:
+  SCFLoopLikeTest() : b(&context), loc(UnknownLoc::get(&context)) {
+    context.loadDialect<arith::ArithDialect, scf::SCFDialect>();
+  }
+
+  void checkUnidimensional(LoopLikeOpInterface loopLikeOp) {
+    std::optional<OpFoldResult> maybeLb = loopLikeOp.getSingleLowerBound();
+    EXPECT_TRUE(maybeLb.has_value());
+    std::optional<OpFoldResult> maybeUb = loopLikeOp.getSingleUpperBound();
+    EXPECT_TRUE(maybeUb.has_value());
+    std::optional<OpFoldResult> maybeStep = loopLikeOp.getSingleStep();
+    EXPECT_TRUE(maybeStep.has_value());
+    std::optional<OpFoldResult> maybeIndVar =
+        loopLikeOp.getSingleInductionVar();
+    EXPECT_TRUE(maybeIndVar.has_value());
+  }
+
+  void checkMultidimensional(LoopLikeOpInterface loopLikeOp) {
+    std::optional<OpFoldResult> maybeLb = loopLikeOp.getSingleLowerBound();
+    EXPECT_FALSE(maybeLb.has_value());
+    std::optional<OpFoldResult> maybeUb = loopLikeOp.getSingleUpperBound();
+    EXPECT_FALSE(maybeUb.has_value());
+    std::optional<OpFoldResult> maybeStep = loopLikeOp.getSingleStep();
+    EXPECT_FALSE(maybeStep.has_value());
+    std::optional<OpFoldResult> maybeIndVar =
+        loopLikeOp.getSingleInductionVar();
+    EXPECT_FALSE(maybeIndVar.has_value());
+  }
+
+  MLIRContext context;
+  OpBuilder b;
+  Location loc;
+};
+
+TEST_F(SCFLoopLikeTest, queryUnidimensionalLooplikes) {
+  OwningOpRef<arith::ConstantIndexOp> lb =
+      b.create<arith::ConstantIndexOp>(loc, 0);
+  OwningOpRef<arith::ConstantIndexOp> ub =
+      b.create<arith::ConstantIndexOp>(loc, 10);
+  OwningOpRef<arith::ConstantIndexOp> step =
+      b.create<arith::ConstantIndexOp>(loc, 2);
+
+  OwningOpRef<scf::ForOp> forOp =
+      b.create<scf::ForOp>(loc, lb.get(), ub.get(), step.get());
+  checkUnidimensional(forOp.get());
+
+  OwningOpRef<scf::ForallOp> forallOp = b.create<scf::ForallOp>(
+      loc, ArrayRef<OpFoldResult>(lb->getResult()),
+      ArrayRef<OpFoldResult>(ub->getResult()),
+      ArrayRef<OpFoldResult>(step->getResult()), ValueRange(), std::nullopt);
+  checkUnidimensional(forallOp.get());
+
+  OwningOpRef<scf::ParallelOp> parallelOp = b.create<scf::ParallelOp>(
+      loc, ValueRange(lb->getResult()), ValueRange(ub->getResult()),
+      ValueRange(step->getResult()), ValueRange());
+  checkUnidimensional(parallelOp.get());
+}
+
+TEST_F(SCFLoopLikeTest, queryMultidimensionalLooplikes) {
+  OwningOpRef<arith::ConstantIndexOp> lb =
+      b.create<arith::ConstantIndexOp>(loc, 0);
+  OwningOpRef<arith::ConstantIndexOp> ub =
+      b.create<arith::ConstantIndexOp>(loc, 10);
+  OwningOpRef<arith::ConstantIndexOp> step =
+      b.create<arith::ConstantIndexOp>(loc, 2);
+
+  OwningOpRef<scf::ForallOp> forallOp = b.create<scf::ForallOp>(
+      loc, ArrayRef<OpFoldResult>({lb->getResult(), lb->getResult()}),
+      ArrayRef<OpFoldResult>({ub->getResult(), ub->getResult()}),
+      ArrayRef<OpFoldResult>({step->getResult(), step->getResult()}),
+      ValueRange(), std::nullopt);
+  checkMultidimensional(forallOp.get());
+
+  OwningOpRef<scf::ParallelOp> parallelOp = b.create<scf::ParallelOp>(
+      loc, ValueRange({lb->getResult(), lb->getResult()}),
+      ValueRange({ub->getResult(), ub->getResult()}),
+      ValueRange({step->getResult(), step->getResult()}), ValueRange());
+  checkMultidimensional(parallelOp.get());
+}
diff --git a/mlir/unittests/Dialect/Utils/StructuredOpsUtilsTest.cpp b/mlir/unittests/Dialect/Utils/StructuredOpsUtilsTest.cpp
index 583dbd463b..3f576baceb 100644
--- a/mlir/unittests/Dialect/Utils/StructuredOpsUtilsTest.cpp
+++ b/mlir/unittests/Dialect/Utils/StructuredOpsUtilsTest.cpp
@@ -240,4 +240,134 @@ TEST(isRowMajorBatchMatmul, FirstInputSwapped) {
   EXPECT_THAT(maps, Not(Truly(isRowMajorBatchMatmul)));
 }
 
+TEST(isVecmat, Simple) {
+  MLIRContext context;
+
+  AffineExpr k, n;
+  bindDims(&context, k, n);
+  auto mapA = AffineMapAttr::get(AffineMap::get(2, 0, {k}, &context));
+  auto mapB = AffineMapAttr::get(AffineMap::get(2, 0, {k, n}, &context));
+  auto mapC = AffineMapAttr::get(AffineMap::get(2, 0, {n}, &context));
+  auto maps = ArrayAttr::get(&context, {mapA, mapB, mapC});
+
+  EXPECT_THAT(maps, Truly(isVecmat));
+}
+
+TEST(isVecmat, BindingSwapped) {
+  MLIRContext context;
+
+  AffineExpr k, n;
+  bindDims(&context, n, k); // bind in different order
+  auto mapA = AffineMapAttr::get(AffineMap::get(2, 0, {k}, &context));
+  auto mapB = AffineMapAttr::get(AffineMap::get(2, 0, {k, n}, &context));
+  auto mapC = AffineMapAttr::get(AffineMap::get(2, 0, {n}, &context));
+  auto maps = ArrayAttr::get(&context, {mapA, mapB, mapC});
+
+  EXPECT_THAT(maps, Truly(isVecmat));
+}
+
+TEST(isVecmat, WrongDimOrderMatrix) {
+  MLIRContext context;
+
+  AffineExpr k, n;
+  bindDims(&context, k, n);
+  auto mapA = AffineMapAttr::get(AffineMap::get(2, 0, {k}, &context));
+  auto mapB = AffineMapAttr::get(AffineMap::get(2, 0, {n, k}, &context));
+  auto mapC = AffineMapAttr::get(AffineMap::get(2, 0, {n}, &context));
+  auto maps = ArrayAttr::get(&context, {mapA, mapB, mapC});
+
+  EXPECT_THAT(maps, Not(Truly(isVecmat)));
+}
+
+TEST(isMatvec, Simple) {
+  MLIRContext context;
+
+  AffineExpr k, n;
+  bindDims(&context, k, n);
+  auto mapA = AffineMapAttr::get(AffineMap::get(2, 0, {n, k}, &context));
+  auto mapB = AffineMapAttr::get(AffineMap::get(2, 0, {k}, &context));
+  auto mapC = AffineMapAttr::get(AffineMap::get(2, 0, {n}, &context));
+  auto maps = ArrayAttr::get(&context, {mapA, mapB, mapC});
+
+  EXPECT_THAT(maps, Truly(isMatvec));
+}
+
+TEST(isMatvec, BindingSwapped) {
+  MLIRContext context;
+
+  AffineExpr k, n;
+  bindDims(&context, n, k); // bind in different order
+  auto mapA = AffineMapAttr::get(AffineMap::get(2, 0, {n, k}, &context));
+  auto mapB = AffineMapAttr::get(AffineMap::get(2, 0, {k}, &context));
+  auto mapC = AffineMapAttr::get(AffineMap::get(2, 0, {n}, &context));
+  auto maps = ArrayAttr::get(&context, {mapA, mapB, mapC});
+
+  EXPECT_THAT(maps, Truly(isMatvec));
+}
+
+TEST(isMatvec, WrongDimOrderMatrix) {
+  MLIRContext context;
+
+  AffineExpr k, n;
+  bindDims(&context, k, n);
+  auto mapA = AffineMapAttr::get(AffineMap::get(2, 0, {k, n}, &context));
+  auto mapB = AffineMapAttr::get(AffineMap::get(2, 0, {k}, &context));
+  auto mapC = AffineMapAttr::get(AffineMap::get(2, 0, {n}, &context));
+  auto maps = ArrayAttr::get(&context, {mapA, mapB, mapC});
+
+  EXPECT_THAT(maps, Not(Truly(isMatvec)));
+}
+
+TEST(isBatchMatvec, Simple) {
+  MLIRContext context;
+
+  AffineExpr batch, k, n;
+  bindDims(&context, batch, k, n);
+  auto mapA = AffineMapAttr::get(AffineMap::get(3, 0, {batch, n, k}, &context));
+  auto mapB = AffineMapAttr::get(AffineMap::get(3, 0, {batch, k}, &context));
+  auto mapC = AffineMapAttr::get(AffineMap::get(3, 0, {batch, n}, &context));
+  auto maps = ArrayAttr::get(&context, {mapA, mapB, mapC});
+
+  EXPECT_THAT(maps, Truly(isBatchMatvec));
+}
+
+TEST(isBatchMatvec, BindingSwapped) {
+  MLIRContext context;
+
+  AffineExpr batch, k, n;
+  bindDims(&context, batch, n, k); // bind in different order
+  auto mapA = AffineMapAttr::get(AffineMap::get(3, 0, {batch, n, k}, &context));
+  auto mapB = AffineMapAttr::get(AffineMap::get(3, 0, {batch, k}, &context));
+  auto mapC = AffineMapAttr::get(AffineMap::get(3, 0, {batch, n}, &context));
+  auto maps = ArrayAttr::get(&context, {mapA, mapB, mapC});
+
+  EXPECT_THAT(maps, Truly(isBatchMatvec));
+}
+
+TEST(isBatchMatvec, Matmul) {
+  MLIRContext context;
+
+  AffineExpr m, n, k;
+  bindDims(&context, m, n, k);
+  auto mapA = AffineMapAttr::get(AffineMap::get(3, 0, {m, k}, &context));
+  auto mapB = AffineMapAttr::get(AffineMap::get(3, 0, {k, n}, &context));
+  auto mapC = AffineMapAttr::get(AffineMap::get(3, 0, {m, n}, &context));
+  auto maps = ArrayAttr::get(&context, {mapA, mapB, mapC});
+
+  EXPECT_THAT(maps, Not(Truly(isBatchMatvec)));
+}
+
+TEST(isBatchMatvec, WrongDimOrderMatrix) {
+  MLIRContext context;
+
+  AffineExpr batch, k, n;
+  bindDims(&context, batch, k, n);
+  auto mapA = AffineMapAttr::get(AffineMap::get(3, 0, {batch, k, n}, &context));
+  auto mapB = AffineMapAttr::get(AffineMap::get(3, 0, {batch, k}, &context));
+  auto mapC = AffineMapAttr::get(AffineMap::get(3, 0, {batch, n}, &context));
+  auto maps = ArrayAttr::get(&context, {mapA, mapB, mapC});
+
+  EXPECT_THAT(maps, Not(Truly(isBatchMatvec)));
+}
+
 } // namespace
diff --git a/mlir/unittests/IR/ShapedTypeTest.cpp b/mlir/unittests/IR/ShapedTypeTest.cpp
index 82674fd376..61264bc523 100644
--- a/mlir/unittests/IR/ShapedTypeTest.cpp
+++ b/mlir/unittests/IR/ShapedTypeTest.cpp
@@ -131,4 +131,99 @@ TEST(ShapedTypeTest, CloneVector) {
             VectorType::get(vectorNewShape, vectorNewType));
 }
 
+TEST(ShapedTypeTest, VectorTypeBuilder) {
+  MLIRContext context;
+  Type f32 = FloatType::getF32(&context);
+
+  SmallVector<int64_t> shape{2, 4, 8, 9, 1};
+  SmallVector<bool> scalableDims{true, false, true, false, false};
+  VectorType vectorType = VectorType::get(shape, f32, scalableDims);
+
+  {
+    // Drop some dims.
+    VectorType dropFrontTwoDims =
+        VectorType::Builder(vectorType).dropDim(0).dropDim(0);
+    ASSERT_EQ(vectorType.getElementType(), dropFrontTwoDims.getElementType());
+    ASSERT_EQ(vectorType.getShape().drop_front(2), dropFrontTwoDims.getShape());
+    ASSERT_EQ(vectorType.getScalableDims().drop_front(2),
+              dropFrontTwoDims.getScalableDims());
+  }
+
+  {
+    // Set some dims.
+    VectorType setTwoDims =
+        VectorType::Builder(vectorType).setDim(0, 10).setDim(3, 12);
+    ASSERT_EQ(setTwoDims.getShape(), ArrayRef<int64_t>({10, 4, 8, 12, 1}));
+    ASSERT_EQ(vectorType.getElementType(), setTwoDims.getElementType());
+    ASSERT_EQ(vectorType.getScalableDims(), setTwoDims.getScalableDims());
+  }
+
+  {
+    // Test for bug from:
+    // https://github.com/llvm/llvm-project/commit/b44b3494f60296db6aca38a14cab061d9b747a0a
+    // Constructs a temporary builder, modifies it, copies it to `builder`.
+    // This used to lead to a use-after-free. Running under sanitizers will
+    // catch any issues.
+    VectorType::Builder builder = VectorType::Builder(vectorType).setDim(0, 16);
+    VectorType newVectorType = VectorType(builder);
+    ASSERT_EQ(newVectorType.getDimSize(0), 16);
+  }
+
+  {
+    // Make builder from scratch (without scalable dims) -- this use to lead to
+    // a use-after-free see: https://github.com/llvm/llvm-project/pull/68969.
+    // Running under sanitizers will catch any issues.
+    SmallVector<int64_t> shape{1, 2, 3, 4};
+    VectorType::Builder builder(shape, f32);
+    ASSERT_EQ(VectorType(builder).getShape(), ArrayRef(shape));
+  }
+
+  {
+    // Set vector shape (without scalable dims) -- this use to lead to
+    // a use-after-free see: https://github.com/llvm/llvm-project/pull/68969.
+    // Running under sanitizers will catch any issues.
+    VectorType::Builder builder(vectorType);
+    SmallVector<int64_t> newShape{2, 2};
+    builder.setShape(newShape);
+    ASSERT_EQ(VectorType(builder).getShape(), ArrayRef(newShape));
+  }
+}
+
+TEST(ShapedTypeTest, RankedTensorTypeBuilder) {
+  MLIRContext context;
+  Type f32 = FloatType::getF32(&context);
+
+  SmallVector<int64_t> shape{2, 4, 8, 16, 32};
+  RankedTensorType tensorType = RankedTensorType::get(shape, f32);
+
+  {
+    // Drop some dims.
+    RankedTensorType dropFrontTwoDims =
+        RankedTensorType::Builder(tensorType).dropDim(0).dropDim(1).dropDim(0);
+    ASSERT_EQ(tensorType.getElementType(), dropFrontTwoDims.getElementType());
+    ASSERT_EQ(dropFrontTwoDims.getShape(), ArrayRef<int64_t>({16, 32}));
+  }
+
+  {
+    // Insert some dims.
+    RankedTensorType insertTwoDims =
+        RankedTensorType::Builder(tensorType).insertDim(7, 2).insertDim(9, 3);
+    ASSERT_EQ(tensorType.getElementType(), insertTwoDims.getElementType());
+    ASSERT_EQ(insertTwoDims.getShape(),
+              ArrayRef<int64_t>({2, 4, 7, 9, 8, 16, 32}));
+  }
+
+  {
+    // Test for bug from:
+    // https://github.com/llvm/llvm-project/commit/b44b3494f60296db6aca38a14cab061d9b747a0a
+    // Constructs a temporary builder, modifies it, copies it to `builder`.
+    // This used to lead to a use-after-free. Running under sanitizers will
+    // catch any issues.
+    RankedTensorType::Builder builder =
+        RankedTensorType::Builder(tensorType).dropDim(0);
+    RankedTensorType newTensorType = RankedTensorType(builder);
+    ASSERT_EQ(tensorType.getShape().drop_front(), newTensorType.getShape());
+  }
+}
+
 } // namespace
diff --git a/mlir/unittests/Target/LLVM/SerializeToLLVMBitcode.cpp b/mlir/unittests/Target/LLVM/SerializeToLLVMBitcode.cpp
index c23ed7efd7..69c9d9b2f3 100644
--- a/mlir/unittests/Target/LLVM/SerializeToLLVMBitcode.cpp
+++ b/mlir/unittests/Target/LLVM/SerializeToLLVMBitcode.cpp
@@ -55,7 +55,7 @@ TEST_F(MLIRTargetLLVM, SKIP_WITHOUT_NATIVE(SerializeToLLVMBitcode)) {
   ASSERT_TRUE(!!module);
 
   // Serialize the module.
-  std::string targetTriple = llvm::sys::getDefaultTargetTriple();
+  std::string targetTriple = llvm::sys::getProcessTriple();
   LLVM::ModuleToObject serializer(*(module->getOperation()), targetTriple, "",
                                   "");
   std::optional<SmallVector<char, 0>> serializedModule = serializer.run();
diff --git a/openmp/docs/design/Runtimes.rst b/openmp/docs/design/Runtimes.rst
index 4c848ca76f..62ed757979 100644
--- a/openmp/docs/design/Runtimes.rst
+++ b/openmp/docs/design/Runtimes.rst
@@ -1465,3 +1465,4 @@ debugging features are supported.
 
     * Enable debugging assertions in the device. ``0x01``
     * Enable diagnosing common problems during offloading . ``0x4``
+    * Enable device malloc statistics (amdgpu only). ``0x8``
diff --git a/openmp/libomptarget/DeviceRTL/CMakeLists.txt b/openmp/libomptarget/DeviceRTL/CMakeLists.txt
index fee2414b45..f71bdeae3d 100644
--- a/openmp/libomptarget/DeviceRTL/CMakeLists.txt
+++ b/openmp/libomptarget/DeviceRTL/CMakeLists.txt
@@ -83,6 +83,7 @@ endif()
 list(REMOVE_DUPLICATES LIBOMPTARGET_DEVICE_ARCHITECTURES)
 
 set(include_files
+  ${include_directory}/Allocator.h
   ${include_directory}/Configuration.h
   ${include_directory}/Debug.h
   ${include_directory}/Interface.h
@@ -95,6 +96,7 @@ set(include_files
 )
 
 set(src_files
+  ${source_directory}/Allocator.cpp
   ${source_directory}/Configuration.cpp
   ${source_directory}/Debug.cpp
   ${source_directory}/Kernel.cpp
diff --git a/openmp/libomptarget/DeviceRTL/include/Allocator.h b/openmp/libomptarget/DeviceRTL/include/Allocator.h
new file mode 100644
index 0000000000..a28eb0fb29
--- /dev/null
+++ b/openmp/libomptarget/DeviceRTL/include/Allocator.h
@@ -0,0 +1,44 @@
+//===-------- Allocator.h - OpenMP memory allocator interface ---- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OMPTARGET_ALLOCATOR_H
+#define OMPTARGET_ALLOCATOR_H
+
+#include "Types.h"
+
+// Forward declaration.
+struct KernelEnvironmentTy;
+
+#pragma omp begin declare target device_type(nohost)
+
+namespace ompx {
+
+namespace allocator {
+
+static uint64_t constexpr ALIGNMENT = 16;
+
+/// Initialize the allocator according to \p KernelEnvironment
+void init(bool IsSPMD, KernelEnvironmentTy &KernelEnvironment);
+
+/// Allocate \p Size bytes.
+[[gnu::alloc_size(1), gnu::assume_aligned(ALIGNMENT), gnu::malloc]] void *
+alloc(uint64_t Size);
+
+/// Free the allocation pointed to by \p Ptr.
+void free(void *Ptr);
+
+} // namespace allocator
+
+} // namespace ompx
+
+#pragma omp end declare target
+
+#endif
diff --git a/openmp/libomptarget/DeviceRTL/include/Configuration.h b/openmp/libomptarget/DeviceRTL/include/Configuration.h
index 4a68a2f1d4..45e5cead23 100644
--- a/openmp/libomptarget/DeviceRTL/include/Configuration.h
+++ b/openmp/libomptarget/DeviceRTL/include/Configuration.h
@@ -13,17 +13,12 @@
 #ifndef OMPTARGET_CONFIGURATION_H
 #define OMPTARGET_CONFIGURATION_H
 
+#include "Environment.h"
 #include "Types.h"
 
 namespace ompx {
 namespace config {
 
-enum DebugKind : uint32_t {
-  Assertion = 1U << 0,
-  FunctionTracing = 1U << 1,
-  CommonIssues = 1U << 2,
-};
-
 /// Return the number of devices in the system, same number as returned on the
 /// host by omp_get_num_devices.
 uint32_t getNumDevices();
@@ -50,7 +45,7 @@ uint64_t getIndirectCallTableSize();
 uint64_t getHardwareParallelism();
 
 /// Return if debugging is enabled for the given debug kind.
-bool isDebugMode(DebugKind Level);
+bool isDebugMode(DeviceDebugKind Level);
 
 /// Indicates if this kernel may require thread-specific states, or if it was
 /// explicitly disabled by the user.
diff --git a/openmp/libomptarget/DeviceRTL/include/Debug.h b/openmp/libomptarget/DeviceRTL/include/Debug.h
index 4eb9047753..bd4d40e8f2 100644
--- a/openmp/libomptarget/DeviceRTL/include/Debug.h
+++ b/openmp/libomptarget/DeviceRTL/include/Debug.h
@@ -28,7 +28,7 @@ void __assert_fail_internal(const char *expr, const char *msg, const char *file,
 
 #define ASSERT(expr, msg)                                                      \
   {                                                                            \
-    if (config::isDebugMode(config::DebugKind::Assertion) && !(expr))          \
+    if (config::isDebugMode(DeviceDebugKind::Assertion) && !(expr))            \
       __assert_fail_internal(#expr, msg, __FILE__, __LINE__,                   \
                              __PRETTY_FUNCTION__);                             \
     else                                                                       \
diff --git a/openmp/libomptarget/DeviceRTL/include/LibC.h b/openmp/libomptarget/DeviceRTL/include/LibC.h
index 87eed20f52..dde86af783 100644
--- a/openmp/libomptarget/DeviceRTL/include/LibC.h
+++ b/openmp/libomptarget/DeviceRTL/include/LibC.h
@@ -17,6 +17,7 @@
 extern "C" {
 
 int memcmp(const void *lhs, const void *rhs, size_t count);
+void memset(void *dst, int C, size_t count);
 
 int printf(const char *format, ...);
 }
diff --git a/openmp/libomptarget/DeviceRTL/include/State.h b/openmp/libomptarget/DeviceRTL/include/State.h
index 60dc439f95..5db5e27ebe 100644
--- a/openmp/libomptarget/DeviceRTL/include/State.h
+++ b/openmp/libomptarget/DeviceRTL/include/State.h
@@ -176,7 +176,7 @@ inline uint32_t &lookupImpl(uint32_t state::ICVStateTy::*Var,
   return TeamState.ICVState.*Var;
 }
 
-__attribute__((always_inline, flatten)) inline uint32_t &
+[[gnu::always_inline, gnu::flatten]] inline uint32_t &
 lookup32(ValueKind Kind, bool IsReadonly, IdentTy *Ident, bool ForceTeamState) {
   switch (Kind) {
   case state::VK_NThreads:
@@ -218,7 +218,7 @@ lookup32(ValueKind Kind, bool IsReadonly, IdentTy *Ident, bool ForceTeamState) {
   __builtin_unreachable();
 }
 
-__attribute__((always_inline, flatten)) inline void *&
+[[gnu::always_inline, gnu::flatten]] inline void *&
 lookupPtr(ValueKind Kind, bool IsReadonly, bool ForceTeamState) {
   switch (Kind) {
   case state::VK_ParallelRegionFn:
@@ -232,47 +232,45 @@ lookupPtr(ValueKind Kind, bool IsReadonly, bool ForceTeamState) {
 /// A class without actual state used to provide a nice interface to lookup and
 /// update ICV values we can declare in global scope.
 template <typename Ty, ValueKind Kind> struct Value {
-  __attribute__((flatten, always_inline)) operator Ty() {
+  [[gnu::flatten, gnu::always_inline]] operator Ty() {
     return lookup(/* IsReadonly */ true, /* IdentTy */ nullptr,
                   /* ForceTeamState */ false);
   }
 
-  __attribute__((flatten, always_inline)) Value &operator=(const Ty &Other) {
+  [[gnu::flatten, gnu::always_inline]] Value &operator=(const Ty &Other) {
     set(Other, /* IdentTy */ nullptr);
     return *this;
   }
 
-  __attribute__((flatten, always_inline)) Value &operator++() {
+  [[gnu::flatten, gnu::always_inline]] Value &operator++() {
     inc(1, /* IdentTy */ nullptr);
     return *this;
   }
 
-  __attribute__((flatten, always_inline)) Value &operator--() {
+  [[gnu::flatten, gnu::always_inline]] Value &operator--() {
     inc(-1, /* IdentTy */ nullptr);
     return *this;
   }
 
-  __attribute__((flatten, always_inline)) void
+  [[gnu::flatten, gnu::always_inline]] void
   assert_eq(const Ty &V, IdentTy *Ident = nullptr,
             bool ForceTeamState = false) {
     ASSERT(lookup(/* IsReadonly */ true, Ident, ForceTeamState) == V, nullptr);
   }
 
 private:
-  __attribute__((flatten, always_inline)) Ty &
+  [[gnu::flatten, gnu::always_inline]] Ty &
   lookup(bool IsReadonly, IdentTy *Ident, bool ForceTeamState) {
     Ty &t = lookup32(Kind, IsReadonly, Ident, ForceTeamState);
     return t;
   }
 
-  __attribute__((flatten, always_inline)) Ty &inc(int UpdateVal,
-                                                  IdentTy *Ident) {
+  [[gnu::flatten, gnu::always_inline]] Ty &inc(int UpdateVal, IdentTy *Ident) {
     return (lookup(/* IsReadonly */ false, Ident, /* ForceTeamState */ false) +=
             UpdateVal);
   }
 
-  __attribute__((flatten, always_inline)) Ty &set(Ty UpdateVal,
-                                                  IdentTy *Ident) {
+  [[gnu::flatten, gnu::always_inline]] Ty &set(Ty UpdateVal, IdentTy *Ident) {
     return (lookup(/* IsReadonly */ false, Ident, /* ForceTeamState */ false) =
                 UpdateVal);
   }
@@ -284,12 +282,12 @@ template <typename Ty, ValueKind Kind> struct Value {
 /// a nice interface to lookup and update ICV values
 /// we can declare in global scope.
 template <typename Ty, ValueKind Kind> struct PtrValue {
-  __attribute__((flatten, always_inline)) operator Ty() {
+  [[gnu::flatten, gnu::always_inline]] operator Ty() {
     return lookup(/* IsReadonly */ true, /* IdentTy */ nullptr,
                   /* ForceTeamState */ false);
   }
 
-  __attribute__((flatten, always_inline)) PtrValue &operator=(const Ty Other) {
+  [[gnu::flatten, gnu::always_inline]] PtrValue &operator=(const Ty Other) {
     set(Other);
     return *this;
   }
diff --git a/openmp/libomptarget/DeviceRTL/include/Synchronization.h b/openmp/libomptarget/DeviceRTL/include/Synchronization.h
index b31238fbbc..af9e1a673e 100644
--- a/openmp/libomptarget/DeviceRTL/include/Synchronization.h
+++ b/openmp/libomptarget/DeviceRTL/include/Synchronization.h
@@ -115,7 +115,7 @@ void threads(atomic::OrderingTy Ordering);
 /// (hence all threads in the block are "aligned"). Also perform a fence before
 /// and after the barrier according to \p Ordering. Note that the
 /// fence might be part of the barrier if the target offers this.
-__attribute__((noinline)) void threadsAligned(atomic::OrderingTy Ordering);
+[[gnu::noinline]] void threadsAligned(atomic::OrderingTy Ordering);
 
 #pragma omp end assumes
 ///}
diff --git a/openmp/libomptarget/DeviceRTL/include/Utils.h b/openmp/libomptarget/DeviceRTL/include/Utils.h
index 94da763717..4ab0aea46e 100644
--- a/openmp/libomptarget/DeviceRTL/include/Utils.h
+++ b/openmp/libomptarget/DeviceRTL/include/Utils.h
@@ -83,7 +83,7 @@ template <typename DstTy, typename SrcTy> inline DstTy convertViaPun(SrcTy V) {
 }
 
 /// A  pointer variable that has by design an `undef` value. Use with care.
-__attribute__((loader_uninitialized)) static void *const UndefPtr;
+[[clang::loader_uninitialized]] static void *const UndefPtr;
 
 #define OMP_LIKELY(EXPR) __builtin_expect((bool)(EXPR), true)
 #define OMP_UNLIKELY(EXPR) __builtin_expect((bool)(EXPR), false)
diff --git a/openmp/libomptarget/DeviceRTL/src/Allocator.cpp b/openmp/libomptarget/DeviceRTL/src/Allocator.cpp
new file mode 100644
index 0000000000..7a4cbfe60d
--- /dev/null
+++ b/openmp/libomptarget/DeviceRTL/src/Allocator.cpp
@@ -0,0 +1,80 @@
+//===------ State.cpp - OpenMP State & ICV interface ------------- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//===----------------------------------------------------------------------===//
+
+#include "Allocator.h"
+#include "Configuration.h"
+#include "Environment.h"
+#include "Mapping.h"
+#include "Synchronization.h"
+#include "Types.h"
+#include "Utils.h"
+
+using namespace ompx;
+
+#pragma omp begin declare target device_type(nohost)
+
+[[gnu::used, gnu::retain, gnu::weak,
+  gnu::visibility(
+      "protected")]] DeviceMemoryPoolTy __omp_rtl_device_memory_pool;
+[[gnu::used, gnu::retain, gnu::weak,
+  gnu::visibility("protected")]] DeviceMemoryPoolTrackingTy
+    __omp_rtl_device_memory_pool_tracker;
+
+/// Stateless bump allocator that uses the __omp_rtl_device_memory_pool
+/// directly.
+struct BumpAllocatorTy final {
+
+  void *alloc(uint64_t Size) {
+    Size = utils::roundUp(Size, uint64_t(allocator::ALIGNMENT));
+
+    if (config::isDebugMode(DeviceDebugKind::AllocationTracker)) {
+      atomic::add(&__omp_rtl_device_memory_pool_tracker.NumAllocations, 1,
+                  atomic::seq_cst);
+      atomic::add(&__omp_rtl_device_memory_pool_tracker.AllocationTotal, Size,
+                  atomic::seq_cst);
+      atomic::min(&__omp_rtl_device_memory_pool_tracker.AllocationMin, Size,
+                  atomic::seq_cst);
+      atomic::max(&__omp_rtl_device_memory_pool_tracker.AllocationMax, Size,
+                  atomic::seq_cst);
+    }
+
+    uint64_t *Data =
+        reinterpret_cast<uint64_t *>(&__omp_rtl_device_memory_pool.Ptr);
+    uint64_t End =
+        reinterpret_cast<uint64_t>(Data) + __omp_rtl_device_memory_pool.Size;
+
+    uint64_t OldData = atomic::add(Data, Size, atomic::seq_cst);
+    if (OldData + Size > End)
+      __builtin_trap();
+
+    return reinterpret_cast<void *>(OldData);
+  }
+
+  void free(void *) {}
+};
+
+BumpAllocatorTy BumpAllocator;
+
+/// allocator namespace implementation
+///
+///{
+
+void allocator::init(bool IsSPMD, KernelEnvironmentTy &KernelEnvironment) {
+  // TODO: Check KernelEnvironment for an allocator choice as soon as we have
+  // more than one.
+}
+
+void *allocator::alloc(uint64_t Size) { return BumpAllocator.alloc(Size); }
+
+void allocator::free(void *Ptr) { BumpAllocator.free(Ptr); }
+
+///}
+
+#pragma omp end declare target
diff --git a/openmp/libomptarget/DeviceRTL/src/Configuration.cpp b/openmp/libomptarget/DeviceRTL/src/Configuration.cpp
index 809c5f0388..ab1608b1cf 100644
--- a/openmp/libomptarget/DeviceRTL/src/Configuration.cpp
+++ b/openmp/libomptarget/DeviceRTL/src/Configuration.cpp
@@ -12,7 +12,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "Configuration.h"
-#include "Environment.h"
 #include "State.h"
 #include "Types.h"
 
@@ -27,11 +26,12 @@ using namespace ompx;
 
 // This variable should be visibile to the plugin so we override the default
 // hidden visibility.
-DeviceEnvironmentTy CONSTANT(__omp_rtl_device_environment)
-    __attribute__((used, retain, weak, visibility("protected")));
+[[gnu::used, gnu::retain, gnu::weak,
+  gnu::visibility("protected")]] DeviceEnvironmentTy
+    CONSTANT(__omp_rtl_device_environment);
 
 uint32_t config::getDebugKind() {
-  return __omp_rtl_debug_kind & __omp_rtl_device_environment.DebugKind;
+  return __omp_rtl_debug_kind & __omp_rtl_device_environment.DeviceDebugKind;
 }
 
 uint32_t config::getNumDevices() {
@@ -63,8 +63,8 @@ uint64_t config::getIndirectCallTableSize() {
   return __omp_rtl_device_environment.IndirectCallTableSize;
 }
 
-bool config::isDebugMode(config::DebugKind Kind) {
-  return config::getDebugKind() & Kind;
+bool config::isDebugMode(DeviceDebugKind Kind) {
+  return config::getDebugKind() & uint32_t(Kind);
 }
 
 bool config::mayUseThreadStates() { return !__omp_rtl_assume_no_thread_state; }
diff --git a/openmp/libomptarget/DeviceRTL/src/Kernel.cpp b/openmp/libomptarget/DeviceRTL/src/Kernel.cpp
index 2229835771..91e8a00bde 100644
--- a/openmp/libomptarget/DeviceRTL/src/Kernel.cpp
+++ b/openmp/libomptarget/DeviceRTL/src/Kernel.cpp
@@ -10,6 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "Allocator.h"
 #include "Debug.h"
 #include "Environment.h"
 #include "Interface.h"
@@ -30,6 +31,7 @@ static void inititializeRuntime(bool IsSPMD,
   synchronize::init(IsSPMD);
   mapping::init(IsSPMD);
   state::init(IsSPMD, KernelEnvironment);
+  allocator::init(IsSPMD, KernelEnvironment);
 }
 
 /// Simple generic state machine for worker threads.
diff --git a/openmp/libomptarget/DeviceRTL/src/LibC.cpp b/openmp/libomptarget/DeviceRTL/src/LibC.cpp
index ae73a644b6..af675b9725 100644
--- a/openmp/libomptarget/DeviceRTL/src/LibC.cpp
+++ b/openmp/libomptarget/DeviceRTL/src/LibC.cpp
@@ -47,6 +47,12 @@ int memcmp(const void *lhs, const void *rhs, size_t count) {
   return 0;
 }
 
+void memset(void *dst, int C, size_t count) {
+  auto *dstc = reinterpret_cast<char *>(dst);
+  for (size_t I = 0; I < count; ++I)
+    dstc[I] = C;
+}
+
 /// printf() calls are rewritten by CGGPUBuiltin to __llvm_omp_vprintf
 int32_t __llvm_omp_vprintf(const char *Format, void *Arguments, uint32_t Size) {
   return impl::omp_vprintf(Format, Arguments, Size);
diff --git a/openmp/libomptarget/DeviceRTL/src/Mapping.cpp b/openmp/libomptarget/DeviceRTL/src/Mapping.cpp
index c75a694fce..822b8dc2dd 100644
--- a/openmp/libomptarget/DeviceRTL/src/Mapping.cpp
+++ b/openmp/libomptarget/DeviceRTL/src/Mapping.cpp
@@ -345,7 +345,7 @@ uint32_t mapping::getNumberOfProcessorElements() {
 
 // TODO: This is a workaround for initialization coming from kernels outside of
 //       the TU. We will need to solve this more correctly in the future.
-int __attribute__((weak)) SHARED(IsSPMDMode);
+[[gnu::weak]] int SHARED(IsSPMDMode);
 
 void mapping::init(bool IsSPMD) {
   if (mapping::isInitialThreadInLevel0(IsSPMD))
@@ -358,15 +358,15 @@ bool mapping::isGenericMode() { return !isSPMDMode(); }
 ///}
 
 extern "C" {
-__attribute__((noinline)) uint32_t __kmpc_get_hardware_thread_id_in_block() {
+[[gnu::noinline]] uint32_t __kmpc_get_hardware_thread_id_in_block() {
   return mapping::getThreadIdInBlock();
 }
 
-__attribute__((noinline)) uint32_t __kmpc_get_hardware_num_threads_in_block() {
+[[gnu::noinline]] uint32_t __kmpc_get_hardware_num_threads_in_block() {
   return impl::getNumberOfThreadsInBlock(mapping::DIM_X);
 }
 
-__attribute__((noinline)) uint32_t __kmpc_get_warp_size() {
+[[gnu::noinline]] uint32_t __kmpc_get_warp_size() {
   return impl::getWarpSize();
 }
 }
diff --git a/openmp/libomptarget/DeviceRTL/src/Parallelism.cpp b/openmp/libomptarget/DeviceRTL/src/Parallelism.cpp
index 1610b74fc7..2c0701bd53 100644
--- a/openmp/libomptarget/DeviceRTL/src/Parallelism.cpp
+++ b/openmp/libomptarget/DeviceRTL/src/Parallelism.cpp
@@ -68,10 +68,9 @@ uint32_t determineNumberOfThreads(int32_t NumThreadsClause) {
 }
 
 // Invoke an outlined parallel function unwrapping arguments (up to 32).
-__attribute__((always_inline)) void invokeMicrotask(int32_t global_tid,
-                                                    int32_t bound_tid, void *fn,
-                                                    void **args,
-                                                    int64_t nargs) {
+[[clang::always_inline]] void invokeMicrotask(int32_t global_tid,
+                                              int32_t bound_tid, void *fn,
+                                              void **args, int64_t nargs) {
   switch (nargs) {
 #include "generated_microtask_cases.gen"
   default:
@@ -84,7 +83,7 @@ __attribute__((always_inline)) void invokeMicrotask(int32_t global_tid,
 
 extern "C" {
 
-__attribute__((always_inline)) void
+[[clang::always_inline]] void
 __kmpc_parallel_51(IdentTy *ident, int32_t, int32_t if_expr,
                    int32_t num_threads, int proc_bind, void *fn,
                    void *wrapper_fn, void **args, int64_t nargs) {
@@ -262,8 +261,7 @@ __kmpc_parallel_51(IdentTy *ident, int32_t, int32_t if_expr,
     __kmpc_end_sharing_variables();
 }
 
-__attribute__((noinline)) bool
-__kmpc_kernel_parallel(ParallelRegionFnTy *WorkFn) {
+[[clang::noinline]] bool __kmpc_kernel_parallel(ParallelRegionFnTy *WorkFn) {
   // Work function and arguments for L1 parallel region.
   *WorkFn = state::ParallelRegionFn;
 
@@ -277,7 +275,7 @@ __kmpc_kernel_parallel(ParallelRegionFnTy *WorkFn) {
   return ThreadIsActive;
 }
 
-__attribute__((noinline)) void __kmpc_kernel_end_parallel() {
+[[clang::noinline]] void __kmpc_kernel_end_parallel() {
   // In case we have modified an ICV for this thread before a ThreadState was
   // created. We drop it now to not contaminate the next parallel region.
   ASSERT(!mapping::isSPMDMode(), nullptr);
diff --git a/openmp/libomptarget/DeviceRTL/src/State.cpp b/openmp/libomptarget/DeviceRTL/src/State.cpp
index 721137cb95..e929edda95 100644
--- a/openmp/libomptarget/DeviceRTL/src/State.cpp
+++ b/openmp/libomptarget/DeviceRTL/src/State.cpp
@@ -9,9 +9,12 @@
 //===----------------------------------------------------------------------===//
 
 #include "State.h"
+#include "Allocator.h"
+#include "Configuration.h"
 #include "Debug.h"
 #include "Environment.h"
 #include "Interface.h"
+#include "LibC.h"
 #include "Mapping.h"
 #include "Synchronization.h"
 #include "Types.h"
@@ -25,18 +28,16 @@ using namespace ompx;
 ///
 ///{
 
-/// Add worst-case padding so that future allocations are properly aligned.
-/// FIXME: The stack shouldn't require worst-case padding. Alignment needs to be
-/// passed in as an argument and the stack rewritten to support it.
-constexpr const uint32_t Alignment = 16;
-
 /// External symbol to access dynamic shared memory.
-extern unsigned char DynamicSharedBuffer[] __attribute__((aligned(Alignment)));
+[[gnu::aligned(
+    allocator::ALIGNMENT)]] extern unsigned char DynamicSharedBuffer[];
 #pragma omp allocate(DynamicSharedBuffer) allocator(omp_pteam_mem_alloc)
 
 /// The kernel environment passed to the init method by the compiler.
 static KernelEnvironmentTy *SHARED(KernelEnvironmentPtr);
 
+///}
+
 namespace {
 
 /// Fallback implementations are missing to trigger a link time error.
@@ -44,29 +45,19 @@ namespace {
 /// dedicated begin/end declare variant.
 ///
 ///{
-
 extern "C" {
-__attribute__((leaf)) void *malloc(uint64_t Size);
-__attribute__((leaf)) void free(void *Ptr);
-}
+#ifdef __AMDGPU__
 
-///}
+[[gnu::weak]] void *malloc(uint64_t Size) { return allocator::alloc(Size); }
+[[gnu::weak]] void free(void *Ptr) { allocator::free(Ptr); }
 
-/// AMDGCN implementations of the shuffle sync idiom.
-///
-///{
-#pragma omp begin declare variant match(device = {arch(amdgcn)})
+#else
 
-extern "C" {
-void *malloc(uint64_t Size) {
-  // TODO: Use some preallocated space for dynamic malloc.
-  return nullptr;
-}
+[[gnu::weak, gnu::leaf]] void *malloc(uint64_t Size);
+[[gnu::weak, gnu::leaf]] void free(void *Ptr);
 
-void free(void *Ptr) {}
+#endif
 }
-
-#pragma omp end declare variant
 ///}
 
 /// A "smart" stack in shared memory.
@@ -95,7 +86,7 @@ struct SharedMemorySmartStackTy {
   uint32_t computeThreadStorageTotal() {
     uint32_t NumLanesInBlock = mapping::getNumberOfThreadsInBlock();
     return utils::align_down((state::SharedScratchpadSize / NumLanesInBlock),
-                             Alignment);
+                             allocator::ALIGNMENT);
   }
 
   /// Return the top address of the warp data stack, that is the first address
@@ -105,10 +96,10 @@ struct SharedMemorySmartStackTy {
   }
 
   /// The actual storage, shared among all warps.
-  unsigned char Data[state::SharedScratchpadSize]
-      __attribute__((aligned(Alignment)));
-  unsigned char Usage[mapping::MaxThreadsPerTeam]
-      __attribute__((aligned(Alignment)));
+  [[gnu::aligned(
+      allocator::ALIGNMENT)]] unsigned char Data[state::SharedScratchpadSize];
+  [[gnu::aligned(
+      allocator::ALIGNMENT)]] unsigned char Usage[mapping::MaxThreadsPerTeam];
 };
 
 static_assert(state::SharedScratchpadSize / mapping::MaxThreadsPerTeam <= 256,
@@ -123,7 +114,9 @@ void SharedMemorySmartStackTy::init(bool IsSPMD) {
 
 void *SharedMemorySmartStackTy::push(uint64_t Bytes) {
   // First align the number of requested bytes.
-  uint64_t AlignedBytes = utils::align_up(Bytes, Alignment);
+  /// FIXME: The stack shouldn't require worst-case padding. Alignment needs to
+  /// be passed in as an argument and the stack rewritten to support it.
+  uint64_t AlignedBytes = utils::align_up(Bytes, allocator::ALIGNMENT);
 
   uint32_t StorageTotal = computeThreadStorageTotal();
 
@@ -139,7 +132,7 @@ void *SharedMemorySmartStackTy::push(uint64_t Bytes) {
     return Ptr;
   }
 
-  if (config::isDebugMode(config::DebugKind::CommonIssues))
+  if (config::isDebugMode(DeviceDebugKind::CommonIssues))
     PRINT("Shared memory stack full, fallback to dynamic allocation of global "
           "memory will negatively impact performance.\n");
   void *GlobalMemory = memory::allocGlobal(
@@ -151,7 +144,7 @@ void *SharedMemorySmartStackTy::push(uint64_t Bytes) {
 }
 
 void SharedMemorySmartStackTy::pop(void *Ptr, uint32_t Bytes) {
-  uint64_t AlignedBytes = utils::align_up(Bytes, Alignment);
+  uint64_t AlignedBytes = utils::align_up(Bytes, allocator::ALIGNMENT);
   if (utils::isSharedMemPtr(Ptr)) {
     int TId = mapping::getThreadIdInBlock();
     Usage[TId] -= AlignedBytes;
@@ -174,7 +167,7 @@ void memory::freeShared(void *Ptr, uint64_t Bytes, const char *Reason) {
 
 void *memory::allocGlobal(uint64_t Bytes, const char *Reason) {
   void *Ptr = malloc(Bytes);
-  if (config::isDebugMode(config::DebugKind::CommonIssues) && Ptr == nullptr)
+  if (config::isDebugMode(DeviceDebugKind::CommonIssues) && Ptr == nullptr)
     PRINT("nullptr returned by malloc!\n");
   return Ptr;
 }
@@ -265,13 +258,15 @@ void state::enterDataEnvironment(IdentTy *Ident) {
     return;
 
   unsigned TId = mapping::getThreadIdInBlock();
-  ThreadStateTy *NewThreadState =
-      static_cast<ThreadStateTy *>(__kmpc_alloc_shared(sizeof(ThreadStateTy)));
+  ThreadStateTy *NewThreadState = static_cast<ThreadStateTy *>(
+      memory::allocGlobal(sizeof(ThreadStateTy), "ThreadStates alloc"));
   uintptr_t *ThreadStatesBitsPtr = reinterpret_cast<uintptr_t *>(&ThreadStates);
   if (!atomic::load(ThreadStatesBitsPtr, atomic::seq_cst)) {
-    uint32_t Bytes = sizeof(ThreadStates[0]) * mapping::getMaxTeamThreads();
+    uint32_t Bytes =
+        sizeof(ThreadStates[0]) * mapping::getNumberOfThreadsInBlock();
     void *ThreadStatesPtr =
         memory::allocGlobal(Bytes, "Thread state array allocation");
+    memset(ThreadStatesPtr, 0, Bytes);
     if (!atomic::cas(ThreadStatesBitsPtr, uintptr_t(0),
                      reinterpret_cast<uintptr_t>(ThreadStatesPtr),
                      atomic::seq_cst, atomic::seq_cst))
@@ -300,7 +295,7 @@ void state::resetStateForThread(uint32_t TId) {
     return;
 
   ThreadStateTy *PreviousThreadState = ThreadStates[TId]->PreviousThreadState;
-  __kmpc_free_shared(ThreadStates[TId], sizeof(ThreadStateTy));
+  memory::freeGlobal(ThreadStates[TId], "ThreadStates dealloc");
   ThreadStates[TId] = PreviousThreadState;
 }
 
@@ -423,11 +418,11 @@ int omp_get_initial_device(void) { return -1; }
 }
 
 extern "C" {
-__attribute__((noinline)) void *__kmpc_alloc_shared(uint64_t Bytes) {
+[[clang::noinline]] void *__kmpc_alloc_shared(uint64_t Bytes) {
   return memory::allocShared(Bytes, "Frontend alloc shared");
 }
 
-__attribute__((noinline)) void __kmpc_free_shared(void *Ptr, uint64_t Bytes) {
+[[clang::noinline]] void __kmpc_free_shared(void *Ptr, uint64_t Bytes) {
   memory::freeShared(Ptr, Bytes, "Frontend free shared");
 }
 
diff --git a/openmp/libomptarget/DeviceRTL/src/Synchronization.cpp b/openmp/libomptarget/DeviceRTL/src/Synchronization.cpp
index 3370c5a847..b9a192f0d8 100644
--- a/openmp/libomptarget/DeviceRTL/src/Synchronization.cpp
+++ b/openmp/libomptarget/DeviceRTL/src/Synchronization.cpp
@@ -523,13 +523,12 @@ void __kmpc_barrier(IdentTy *Loc, int32_t TId) {
   impl::namedBarrier();
 }
 
-__attribute__((noinline)) void __kmpc_barrier_simple_spmd(IdentTy *Loc,
-                                                          int32_t TId) {
+[[clang::noinline]] void __kmpc_barrier_simple_spmd(IdentTy *Loc, int32_t TId) {
   synchronize::threadsAligned(atomic::OrderingTy::seq_cst);
 }
 
-__attribute__((noinline)) void __kmpc_barrier_simple_generic(IdentTy *Loc,
-                                                             int32_t TId) {
+[[clang::noinline]] void __kmpc_barrier_simple_generic(IdentTy *Loc,
+                                                       int32_t TId) {
   synchronize::threads(atomic::OrderingTy::seq_cst);
 }
 
diff --git a/openmp/libomptarget/DeviceRTL/src/Utils.cpp b/openmp/libomptarget/DeviceRTL/src/Utils.cpp
index 6125236863..b39465aaa2 100644
--- a/openmp/libomptarget/DeviceRTL/src/Utils.cpp
+++ b/openmp/libomptarget/DeviceRTL/src/Utils.cpp
@@ -19,7 +19,7 @@
 
 using namespace ompx;
 
-extern "C" __attribute__((weak)) int IsSPMDMode;
+extern "C" [[gnu::weak]] int IsSPMDMode;
 
 namespace impl {
 
diff --git a/openmp/libomptarget/DeviceRTL/src/exports b/openmp/libomptarget/DeviceRTL/src/exports
index fbcda3ce8f..288ddf90b4 100644
--- a/openmp/libomptarget/DeviceRTL/src/exports
+++ b/openmp/libomptarget/DeviceRTL/src/exports
@@ -11,6 +11,8 @@ _ZN4ompx*
 
 IsSPMDMode
 
+malloc
+free
 memcmp
 printf
 __assert_fail
diff --git a/openmp/libomptarget/include/Environment.h b/openmp/libomptarget/include/Environment.h
index 6606d7838c..8194736ae4 100644
--- a/openmp/libomptarget/include/Environment.h
+++ b/openmp/libomptarget/include/Environment.h
@@ -25,8 +25,15 @@ using IdentTy = ident_t;
 
 #include "llvm/Frontend/OpenMP/OMPDeviceConstants.h"
 
+enum class DeviceDebugKind : uint32_t {
+  Assertion = 1U << 0,
+  FunctionTracing = 1U << 1,
+  CommonIssues = 1U << 2,
+  AllocationTracker = 1U << 3,
+};
+
 struct DeviceEnvironmentTy {
-  uint32_t DebugKind;
+  uint32_t DeviceDebugKind;
   uint32_t NumDevices;
   uint32_t DeviceNum;
   uint32_t DynamicMemSize;
@@ -36,6 +43,27 @@ struct DeviceEnvironmentTy {
   uint64_t HardwareParallelism;
 };
 
+struct DeviceMemoryPoolTy {
+  void *Ptr;
+  uint64_t Size;
+};
+
+struct DeviceMemoryPoolTrackingTy {
+  uint64_t NumAllocations;
+  uint64_t AllocationTotal;
+  uint64_t AllocationMin;
+  uint64_t AllocationMax;
+
+  void combine(DeviceMemoryPoolTrackingTy &Other) {
+    NumAllocations += Other.NumAllocations;
+    AllocationTotal += Other.AllocationTotal;
+    AllocationMin = AllocationMin > Other.AllocationMin ? Other.AllocationMin
+                                                        : AllocationMin;
+    AllocationMax = AllocationMax < Other.AllocationMax ? Other.AllocationMax
+                                                        : AllocationMax;
+  }
+};
+
 // NOTE: Please don't change the order of those members as their indices are
 // used in the middle end. Always add the new data member at the end.
 // Different from KernelEnvironmentTy below, this structure contains members
diff --git a/openmp/libomptarget/include/Utilities.h b/openmp/libomptarget/include/Utilities.h
index 7f2884ed7e..82593e206e 100644
--- a/openmp/libomptarget/include/Utilities.h
+++ b/openmp/libomptarget/include/Utilities.h
@@ -14,7 +14,6 @@
 #ifndef OPENMP_LIBOMPTARGET_INCLUDE_UTILITIES_H
 #define OPENMP_LIBOMPTARGET_INCLUDE_UTILITIES_H
 
-#include "llvm/ADT/STLFunctionalExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Error.h"
 
diff --git a/openmp/libomptarget/include/omptarget.h b/openmp/libomptarget/include/omptarget.h
index f87557a69e..e1f0f77849 100644
--- a/openmp/libomptarget/include/omptarget.h
+++ b/openmp/libomptarget/include/omptarget.h
@@ -312,6 +312,7 @@ int omp_target_memcpy_rect(void *Dst, const void *Src, size_t ElementSize,
                            const size_t *DstDimensions,
                            const size_t *SrcDimensions, int DstDevice,
                            int SrcDevice);
+void *omp_target_memset(void *Ptr, int C, size_t N, int DeviceNum);
 int omp_target_associate_ptr(const void *HostPtr, const void *DevicePtr,
                              size_t Size, size_t DeviceOffset, int DeviceNum);
 int omp_target_disassociate_ptr(const void *HostPtr, int DeviceNum);
diff --git a/openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp b/openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp
index 66a25e29d0..ab24856f9b 100644
--- a/openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -2529,10 +2529,16 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     return Plugin::success();
   }
   Error getDeviceHeapSize(uint64_t &Value) override {
-    Value = 0;
+    Value = DeviceMemoryPoolSize;
+    return Plugin::success();
+  }
+  Error setDeviceHeapSize(uint64_t Value) override {
+    for (DeviceImageTy *Image : LoadedImages)
+      if (auto Err = setupDeviceMemoryPool(Plugin::get(), *Image, Value))
+        return Err;
+    DeviceMemoryPoolSize = Value;
     return Plugin::success();
   }
-  Error setDeviceHeapSize(uint64_t Value) override { return Plugin::success(); }
 
   /// AMDGPU-specific function to get device attributes.
   template <typename Ty> Error getDeviceAttr(uint32_t Kind, Ty &Value) {
@@ -2625,6 +2631,9 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
 
   /// Reference to the host device.
   AMDHostDeviceTy &HostDevice;
+
+  /// The current size of the global device memory pool (managed by us).
+  uint64_t DeviceMemoryPoolSize = 1L << 29L /* 512MB */;
 };
 
 Error AMDGPUDeviceImageTy::loadExecutable(const AMDGPUDeviceTy &Device) {
@@ -2869,15 +2878,14 @@ struct AMDGPUPluginTy final : public GenericPluginTy {
         if (Status != HSA_STATUS_SUCCESS)
           return Status;
 
-        // TODO: This is not allowed by the standard.
-        char ISAName[Length];
-        Status = hsa_isa_get_info_alt(ISA, HSA_ISA_INFO_NAME, ISAName);
+        llvm::SmallVector<char> ISAName(Length);
+        Status = hsa_isa_get_info_alt(ISA, HSA_ISA_INFO_NAME, ISAName.begin());
         if (Status != HSA_STATUS_SUCCESS)
           return Status;
 
-        llvm::StringRef TripleTarget(ISAName);
+        llvm::StringRef TripleTarget(ISAName.begin(), Length);
         if (TripleTarget.consume_front("amdgcn-amd-amdhsa"))
-          Target = TripleTarget.ltrim('-').str();
+          Target = TripleTarget.ltrim('-').rtrim('\0').str();
         return HSA_STATUS_SUCCESS;
       });
       if (Err)
diff --git a/openmp/libomptarget/plugins-nextgen/common/PluginInterface/PluginInterface.cpp b/openmp/libomptarget/plugins-nextgen/common/PluginInterface/PluginInterface.cpp
index feae44b9b5..0243f0205d 100644
--- a/openmp/libomptarget/plugins-nextgen/common/PluginInterface/PluginInterface.cpp
+++ b/openmp/libomptarget/plugins-nextgen/common/PluginInterface/PluginInterface.cpp
@@ -10,6 +10,7 @@
 
 #include "PluginInterface.h"
 #include "Debug.h"
+#include "Environment.h"
 #include "GlobalHandler.h"
 #include "JIT.h"
 #include "elf_common.h"
@@ -589,6 +590,35 @@ Error GenericDeviceTy::init(GenericPluginTy &Plugin) {
 }
 
 Error GenericDeviceTy::deinit(GenericPluginTy &Plugin) {
+
+  if (OMPX_DebugKind.get() & uint32_t(DeviceDebugKind::AllocationTracker)) {
+    GenericGlobalHandlerTy &GHandler = Plugin.getGlobalHandler();
+    for (auto *Image : LoadedImages) {
+      DeviceMemoryPoolTrackingTy ImageDeviceMemoryPoolTracking = {0, 0, ~0U, 0};
+      GlobalTy TrackerGlobal("__omp_rtl_device_memory_pool_tracker",
+                             sizeof(DeviceMemoryPoolTrackingTy),
+                             &ImageDeviceMemoryPoolTracking);
+      if (auto Err =
+              GHandler.readGlobalFromDevice(*this, *Image, TrackerGlobal))
+        return Err;
+      DeviceMemoryPoolTracking.combine(ImageDeviceMemoryPoolTracking);
+    }
+
+    // TODO: Write this by default into a file.
+    printf("\n\n|-----------------------\n"
+           "| Device memory tracker:\n"
+           "|-----------------------\n"
+           "| #Allocations: %lu\n"
+           "| Byes allocated: %lu\n"
+           "| Minimal allocation: %lu\n"
+           "| Maximal allocation: %lu\n"
+           "|-----------------------\n\n\n",
+           DeviceMemoryPoolTracking.NumAllocations,
+           DeviceMemoryPoolTracking.AllocationTotal,
+           DeviceMemoryPoolTracking.AllocationMin,
+           DeviceMemoryPoolTracking.AllocationMax);
+  }
+
   // Delete the memory manager before deinitializing the device. Otherwise,
   // we may delete device allocations after the device is deinitialized.
   if (MemoryManager)
@@ -647,6 +677,17 @@ GenericDeviceTy::loadBinary(GenericPluginTy &Plugin,
   if (auto Err = setupDeviceEnvironment(Plugin, *Image))
     return std::move(Err);
 
+  // Setup the global device memory pool if needed.
+  if (shouldSetupDeviceMemoryPool()) {
+    uint64_t HeapSize;
+    auto SizeOrErr = getDeviceHeapSize(HeapSize);
+    if (SizeOrErr) {
+      REPORT("No global device memory pool due to error: %s\n",
+             toString(std::move(SizeOrErr)).data());
+    } else if (auto Err = setupDeviceMemoryPool(Plugin, *Image, HeapSize))
+      return std::move(Err);
+  }
+
   // Register all offload entries of the image.
   if (auto Err = registerOffloadEntries(*Image))
     return std::move(Err);
@@ -687,7 +728,7 @@ Error GenericDeviceTy::setupDeviceEnvironment(GenericPluginTy &Plugin,
     return CallTablePairOrErr.takeError();
 
   DeviceEnvironmentTy DeviceEnvironment;
-  DeviceEnvironment.DebugKind = OMPX_DebugKind;
+  DeviceEnvironment.DeviceDebugKind = OMPX_DebugKind;
   DeviceEnvironment.NumDevices = Plugin.getNumDevices();
   // TODO: The device ID used here is not the real device ID used by OpenMP.
   DeviceEnvironment.DeviceNum = DeviceId;
@@ -712,6 +753,45 @@ Error GenericDeviceTy::setupDeviceEnvironment(GenericPluginTy &Plugin,
   return Plugin::success();
 }
 
+Error GenericDeviceTy::setupDeviceMemoryPool(GenericPluginTy &Plugin,
+                                             DeviceImageTy &Image,
+                                             uint64_t PoolSize) {
+  // Free the old pool, if any.
+  if (DeviceMemoryPool.Ptr) {
+    if (auto Err = dataDelete(DeviceMemoryPool.Ptr,
+                              TargetAllocTy::TARGET_ALLOC_DEVICE))
+      return Err;
+  }
+
+  DeviceMemoryPool.Size = PoolSize;
+  auto AllocOrErr = dataAlloc(PoolSize, /*HostPtr=*/nullptr,
+                              TargetAllocTy::TARGET_ALLOC_DEVICE);
+  if (AllocOrErr) {
+    DeviceMemoryPool.Ptr = *AllocOrErr;
+  } else {
+    auto Err = AllocOrErr.takeError();
+    REPORT("Failure to allocate device memory for global memory pool: %s\n",
+           toString(std::move(Err)).data());
+    DeviceMemoryPool.Ptr = nullptr;
+    DeviceMemoryPool.Size = 0;
+  }
+
+  // Create the metainfo of the device environment global.
+  GlobalTy TrackerGlobal("__omp_rtl_device_memory_pool_tracker",
+                         sizeof(DeviceMemoryPoolTrackingTy),
+                         &DeviceMemoryPoolTracking);
+  GenericGlobalHandlerTy &GHandler = Plugin.getGlobalHandler();
+  if (auto Err = GHandler.writeGlobalToDevice(*this, Image, TrackerGlobal))
+    return Err;
+
+  // Create the metainfo of the device environment global.
+  GlobalTy DevEnvGlobal("__omp_rtl_device_memory_pool",
+                        sizeof(DeviceMemoryPoolTy), &DeviceMemoryPool);
+
+  // Write device environment values to the device.
+  return GHandler.writeGlobalToDevice(*this, Image, DevEnvGlobal);
+}
+
 Error GenericDeviceTy::setupRPCServer(GenericPluginTy &Plugin,
                                       DeviceImageTy &Image) {
   // The plugin either does not need an RPC server or it is unavailible.
@@ -1326,10 +1406,6 @@ Error GenericPluginTy::init() {
 }
 
 Error GenericPluginTy::deinit() {
-  // There is no global handler if no device is available.
-  if (GlobalHandler)
-    delete GlobalHandler;
-
   // Deinitialize all active devices.
   for (int32_t DeviceId = 0; DeviceId < NumDevices; ++DeviceId) {
     if (Devices[DeviceId]) {
@@ -1339,6 +1415,10 @@ Error GenericPluginTy::deinit() {
     assert(!Devices[DeviceId] && "Device was not deinitialized");
   }
 
+  // There is no global handler if no device is available.
+  if (GlobalHandler)
+    delete GlobalHandler;
+
   if (RPCServer)
     delete RPCServer;
 
diff --git a/openmp/libomptarget/plugins-nextgen/common/PluginInterface/PluginInterface.h b/openmp/libomptarget/plugins-nextgen/common/PluginInterface/PluginInterface.h
index 4a9c223eef..ddcf3b3cc9 100644
--- a/openmp/libomptarget/plugins-nextgen/common/PluginInterface/PluginInterface.h
+++ b/openmp/libomptarget/plugins-nextgen/common/PluginInterface/PluginInterface.h
@@ -625,6 +625,10 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
   /// this behavior by overriding the shouldSetupDeviceEnvironment function.
   Error setupDeviceEnvironment(GenericPluginTy &Plugin, DeviceImageTy &Image);
 
+  /// Setup the global device memory pool, if the plugin requires one.
+  Error setupDeviceMemoryPool(GenericPluginTy &Plugin, DeviceImageTy &Image,
+                              uint64_t PoolSize);
+
   // Setup the RPC server for this device if needed. This may not run on some
   // plugins like the CPU targets. By default, it will not be executed so it is
   // up to the target to override this using the shouldSetupRPCServer function.
@@ -831,6 +835,10 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
   /// setupDeviceEnvironment() function.
   virtual bool shouldSetupDeviceEnvironment() const { return true; }
 
+  /// Indicate whether the device should setup the global device memory pool. If
+  /// false is return the value on the device will be uninitialized.
+  virtual bool shouldSetupDeviceMemoryPool() const { return true; }
+
   /// Indicate whether or not the device should setup the RPC server. This is
   /// only necessary for unhosted targets like the GPU.
   virtual bool shouldSetupRPCServer() const { return false; }
@@ -911,6 +919,9 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
   /// Return the kernel environment object for kernel \p Name.
   Expected<KernelEnvironmentTy>
   getKernelEnvironmentForKernel(StringRef Name, DeviceImageTy &Image);
+
+  DeviceMemoryPoolTy DeviceMemoryPool = {nullptr, 0};
+  DeviceMemoryPoolTrackingTy DeviceMemoryPoolTracking = {0, 0, ~0U, 0};
 };
 
 /// Class implementing common functionalities of offload plugins. Each plugin
diff --git a/openmp/libomptarget/plugins-nextgen/cuda/src/rtl.cpp b/openmp/libomptarget/plugins-nextgen/cuda/src/rtl.cpp
index 1bb8dc1064..431e34ca75 100644
--- a/openmp/libomptarget/plugins-nextgen/cuda/src/rtl.cpp
+++ b/openmp/libomptarget/plugins-nextgen/cuda/src/rtl.cpp
@@ -843,6 +843,11 @@ struct CUDADeviceTy : public GenericDeviceTy {
     return Plugin::success();
   }
 
+  virtual bool shouldSetupDeviceMemoryPool() const override {
+    /// We use the CUDA malloc for now.
+    return false;
+  }
+
   /// Getters and setters for stack and heap sizes.
   Error getDeviceStackSize(uint64_t &Value) override {
     return getCtxLimit(CU_LIMIT_STACK_SIZE, Value);
diff --git a/openmp/libomptarget/plugins-nextgen/generic-elf-64bit/src/rtl.cpp b/openmp/libomptarget/plugins-nextgen/generic-elf-64bit/src/rtl.cpp
index 8f2022bac9..619f4dfed9 100644
--- a/openmp/libomptarget/plugins-nextgen/generic-elf-64bit/src/rtl.cpp
+++ b/openmp/libomptarget/plugins-nextgen/generic-elf-64bit/src/rtl.cpp
@@ -307,8 +307,9 @@ struct GenELF64DeviceTy : public GenericDeviceTy {
     return Plugin::success();
   }
 
-  /// This plugin should not setup the device environment.
+  /// This plugin should not setup the device environment or memory pool.
   virtual bool shouldSetupDeviceEnvironment() const override { return false; };
+  virtual bool shouldSetupDeviceMemoryPool() const override { return false; };
 
   /// Getters and setters for stack size and heap size not relevant.
   Error getDeviceStackSize(uint64_t &Value) override {
diff --git a/openmp/libomptarget/src/api.cpp b/openmp/libomptarget/src/api.cpp
index 942df8fdb9..ecef02c8a0 100644
--- a/openmp/libomptarget/src/api.cpp
+++ b/openmp/libomptarget/src/api.cpp
@@ -210,7 +210,7 @@ EXTERN int omp_target_memcpy(void *Dst, const void *Src, size_t Length,
 }
 
 // The helper function that calls omp_target_memcpy or omp_target_memcpy_rect
-static int libomp_target_memcpy_async_helper(kmp_int32 Gtid, kmp_task_t *Task) {
+static int libomp_target_memcpy_async_task(kmp_int32 Gtid, kmp_task_t *Task) {
   if (Task == nullptr)
     return OFFLOAD_FAIL;
 
@@ -241,47 +241,129 @@ static int libomp_target_memcpy_async_helper(kmp_int32 Gtid, kmp_task_t *Task) {
   return Rc;
 }
 
-// Allocate and launch helper task
-static int libomp_helper_task_creation(TargetMemcpyArgsTy *Args,
-                                       int DepObjCount,
-                                       omp_depend_t *DepObjList) {
+static int libomp_target_memset_async_task(kmp_int32 Gtid, kmp_task_t *Task) {
+  if (!Task)
+    return OFFLOAD_FAIL;
+
+  auto *Args = reinterpret_cast<TargetMemsetArgsTy *>(Task->shareds);
+  if (!Args)
+    return OFFLOAD_FAIL;
+
+  // call omp_target_memset()
+  omp_target_memset(Args->Ptr, Args->C, Args->N, Args->DeviceNum);
+
+  delete Args;
+
+  return OFFLOAD_SUCCESS;
+}
+
+static inline void
+convertDepObjVector(llvm::SmallVector<kmp_depend_info_t> &Vec, int DepObjCount,
+                    omp_depend_t *DepObjList) {
+  for (int i = 0; i < DepObjCount; ++i) {
+    omp_depend_t DepObj = DepObjList[i];
+    Vec.push_back(*((kmp_depend_info_t *)DepObj));
+  }
+}
+
+template <class T>
+static inline int
+libomp_helper_task_creation(T *Args, int (*Fn)(kmp_int32, kmp_task_t *),
+                            int DepObjCount, omp_depend_t *DepObjList) {
   // Create global thread ID
   int Gtid = __kmpc_global_thread_num(nullptr);
-  int (*Fn)(kmp_int32, kmp_task_t *) = &libomp_target_memcpy_async_helper;
 
-  // Setup the hidden helper flags;
+  // Setup the hidden helper flags
   kmp_int32 Flags = 0;
   kmp_tasking_flags_t *InputFlags = (kmp_tasking_flags_t *)&Flags;
   InputFlags->hidden_helper = 1;
 
-  // Alloc helper task
-  kmp_task_t *Ptr = __kmpc_omp_target_task_alloc(nullptr, Gtid, Flags,
-                                                 sizeof(kmp_task_t), 0, Fn, -1);
-
-  if (Ptr == nullptr) {
-    // Task allocation failed, delete the argument object
+  // Alloc the helper task
+  kmp_task_t *Task = __kmpc_omp_target_task_alloc(
+      nullptr, Gtid, Flags, sizeof(kmp_task_t), 0, Fn, -1);
+  if (!Task) {
     delete Args;
-
     return OFFLOAD_FAIL;
   }
 
-  // Setup the arguments passed to helper task
-  Ptr->shareds = Args;
+  // Setup the arguments for the helper task
+  Task->shareds = Args;
 
-  // Convert the type of depend objects
+  // Convert types of depend objects
   llvm::SmallVector<kmp_depend_info_t> DepObjs;
-  for (int i = 0; i < DepObjCount; i++) {
-    omp_depend_t DepObj = DepObjList[i];
-    DepObjs.push_back(*((kmp_depend_info_t *)DepObj));
-  }
+  convertDepObjVector(DepObjs, DepObjCount, DepObjList);
 
   // Launch the helper task
-  int Rc = __kmpc_omp_task_with_deps(nullptr, Gtid, Ptr, DepObjCount,
+  int Rc = __kmpc_omp_task_with_deps(nullptr, Gtid, Task, DepObjCount,
                                      DepObjs.data(), 0, nullptr);
 
   return Rc;
 }
 
+EXTERN void *omp_target_memset(void *Ptr, int ByteVal, size_t NumBytes,
+                               int DeviceNum) {
+  TIMESCOPE();
+  DP("Call to omp_target_memset, device %d, device pointer %p, size %zu\n",
+     DeviceNum, Ptr, NumBytes);
+
+  // Behave as a no-op if N==0 or if Ptr is nullptr (as a useful implementation
+  // of unspecified behavior, see OpenMP spec).
+  if (!Ptr || NumBytes == 0) {
+    return Ptr;
+  }
+
+  if (DeviceNum == omp_get_initial_device()) {
+    DP("filling memory on host via memset");
+    memset(Ptr, ByteVal, NumBytes); // ignore return value, memset() cannot fail
+  } else {
+    // TODO: replace the omp_target_memset() slow path with the fast path.
+    // That will require the ability to execute a kernel from within
+    // libomptarget.so (which we do not have at the moment).
+
+    // This is a very slow path: create a filled array on the host and upload
+    // it to the GPU device.
+    int InitialDevice = omp_get_initial_device();
+    void *Shadow = omp_target_alloc(NumBytes, InitialDevice);
+    if (Shadow) {
+      (void)memset(Shadow, ByteVal, NumBytes);
+      (void)omp_target_memcpy(Ptr, Shadow, NumBytes, 0, 0, DeviceNum,
+                              InitialDevice);
+      (void)omp_target_free(Shadow, InitialDevice);
+    } else {
+      // If the omp_target_alloc has failed, let's just not do anything.
+      // omp_target_memset does not have any good way to fail, so we
+      // simply avoid a catastrophic failure of the process for now.
+      DP("omp_target_memset failed to fill memory due to error with "
+         "omp_target_alloc");
+    }
+  }
+
+  DP("omp_target_memset returns %p\n", Ptr);
+  return Ptr;
+}
+
+EXTERN void *omp_target_memset_async(void *Ptr, int ByteVal, size_t NumBytes,
+                                     int DeviceNum, int DepObjCount,
+                                     omp_depend_t *DepObjList) {
+  DP("Call to omp_target_memset_async, device %d, device pointer %p, size %zu",
+     DeviceNum, Ptr, NumBytes);
+
+  // Behave as a no-op if N==0 or if Ptr is nullptr (as a useful implementation
+  // of unspecified behavior, see OpenMP spec).
+  if (!Ptr || NumBytes == 0)
+    return Ptr;
+
+  // Create the task object to deal with the async invocation
+  auto *Args = new TargetMemsetArgsTy{Ptr, ByteVal, NumBytes, DeviceNum};
+
+  // omp_target_memset_async() cannot fail via a return code, so ignore the
+  // return code of the helper function
+  (void)libomp_helper_task_creation(Args, &libomp_target_memset_async_task,
+                                    DepObjCount, DepObjList);
+
+  return Ptr;
+}
+
 EXTERN int omp_target_memcpy_async(void *Dst, const void *Src, size_t Length,
                                    size_t DstOffset, size_t SrcOffset,
                                    int DstDevice, int SrcDevice,
@@ -302,7 +384,8 @@ EXTERN int omp_target_memcpy_async(void *Dst, const void *Src, size_t Length,
       Dst, Src, Length, DstOffset, SrcOffset, DstDevice, SrcDevice);
 
   // Create and launch helper task
-  int Rc = libomp_helper_task_creation(Args, DepObjCount, DepObjList);
+  int Rc = libomp_helper_task_creation(Args, &libomp_target_memcpy_async_task,
+                                       DepObjCount, DepObjList);
 
   DP("omp_target_memcpy_async returns %d\n", Rc);
   return Rc;
@@ -399,7 +482,8 @@ EXTERN int omp_target_memcpy_rect_async(
       DstDimensions, SrcDimensions, DstDevice, SrcDevice);
 
   // Create and launch helper task
-  int Rc = libomp_helper_task_creation(Args, DepObjCount, DepObjList);
+  int Rc = libomp_helper_task_creation(Args, &libomp_target_memcpy_async_task,
+                                       DepObjCount, DepObjList);
 
   DP("omp_target_memcpy_rect_async returns %d\n", Rc);
   return Rc;
diff --git a/openmp/libomptarget/src/exports b/openmp/libomptarget/src/exports
index c29c8d03fb..af882a2642 100644
--- a/openmp/libomptarget/src/exports
+++ b/openmp/libomptarget/src/exports
@@ -44,6 +44,8 @@ VERS1.0 {
     omp_target_memcpy_rect;
     omp_target_memcpy_async;
     omp_target_memcpy_rect_async;
+    omp_target_memset;
+    omp_target_memset_async;
     omp_target_associate_ptr;
     omp_target_disassociate_ptr;
     llvm_omp_target_alloc_host;
diff --git a/openmp/libomptarget/src/private.h b/openmp/libomptarget/src/private.h
index cbce15b63a..cf88f78696 100644
--- a/openmp/libomptarget/src/private.h
+++ b/openmp/libomptarget/src/private.h
@@ -253,6 +253,17 @@ struct TargetMemcpyArgsTy {
         DstOffsets(DstOffsets), SrcOffsets(SrcOffsets),
         DstDimensions(DstDimensions), SrcDimensions(SrcDimensions){};
 };
+
+struct TargetMemsetArgsTy {
+  // Common attributes of a memset operation
+  void *Ptr;
+  int C;
+  size_t N;
+  int DeviceNum;
+
+  // no constructors defined, because this is a PoD
+};
+
 // Invalid GTID as defined by libomp; keep in sync
 #define KMP_GTID_DNE (-2)
 #ifdef __cplusplus
diff --git a/openmp/libomptarget/test/api/omp_target_memset.c b/openmp/libomptarget/test/api/omp_target_memset.c
new file mode 100644
index 0000000000..ea0c6d73fb
--- /dev/null
+++ b/openmp/libomptarget/test/api/omp_target_memset.c
@@ -0,0 +1,45 @@
+// RUN: %libomptarget-compile-and-run-generic
+
+#include "stdio.h"
+#include <omp.h>
+#include <stdlib.h>
+
+int main() {
+  int d = omp_get_default_device();
+  int id = omp_get_initial_device();
+  int q[128], i;
+  void *p;
+  void *result;
+
+  if (d < 0 || d >= omp_get_num_devices())
+    d = id;
+
+  p = omp_target_alloc(130 * sizeof(int), d);
+  if (p == NULL)
+    return 0;
+
+  for (i = 0; i < 128; i++)
+    q[i] = i;
+
+  result = omp_target_memset(p, 0, 130 * sizeof(int), d);
+  if (result != p) {
+    abort();
+  }
+
+  int q2[128];
+  for (i = 0; i < 128; ++i)
+    q2[i] = i;
+  if (omp_target_memcpy_async(q2, p, 128 * sizeof(int), 0, sizeof(int), id, d,
+                              0, NULL))
+    abort();
+
+#pragma omp taskwait
+
+  for (i = 0; i < 128; ++i)
+    if (q2[i] != 0)
+      abort();
+
+  omp_target_free(p, d);
+
+  return 0;
+}
diff --git a/openmp/libomptarget/test/libc/assert.c b/openmp/libomptarget/test/libc/assert.c
new file mode 100644
index 0000000000..9a10032f48
--- /dev/null
+++ b/openmp/libomptarget/test/libc/assert.c
@@ -0,0 +1,21 @@
+// RUN: %libomptarget-compile-generic && %libomptarget-run-fail-generic 2>&1 | \
+// RUN:   %fcheck-generic --check-prefix=CHECK
+
+// REQUIRES: libc
+
+// UNSUPPORTED: powerpc64-ibm-linux-gnu
+// UNSUPPORTED: powerpc64-ibm-linux-gnu-LTO
+// UNSUPPORTED: aarch64-unknown-linux-gnu
+// UNSUPPORTED: aarch64-unknown-linux-gnu-LTO
+// UNSUPPORTED: x86_64-pc-linux-gnu
+// UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+
+#include <assert.h>
+
+int main() {
+  // CHECK: Assertion failed: '0 && "Trivial failure"' in function: 'int main()'
+  // CHECK-NOT: Assertion failed:
+#pragma omp target
+#pragma omp parallel
+  { assert(0 && "Trivial failure"); }
+}
diff --git a/openmp/libomptarget/test/mapping/lambda_mapping.cpp b/openmp/libomptarget/test/mapping/lambda_mapping.cpp
index acec6c8228..63b1719fbb 100644
--- a/openmp/libomptarget/test/mapping/lambda_mapping.cpp
+++ b/openmp/libomptarget/test/mapping/lambda_mapping.cpp
@@ -1,7 +1,7 @@
-// On AMDGPU we don't have malloc support yet. We need optimizations
-// to avoid a thread state which requires malloc.
-//
-// XUN: %libomptarget-compilexx-run-and-check-generic
+// Unonptimized, we need 24000000 bytes heap
+// RUN: %libomptarget-compilexx-generic
+// RUN: env LIBOMPTARGET_HEAP_SIZE=24000000 \
+// RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic
 // RUN: %libomptarget-compileoptxx-run-and-check-generic
 
 #include <iostream>
diff --git a/openmp/libomptarget/test/offloading/malloc.c b/openmp/libomptarget/test/offloading/malloc.c
new file mode 100644
index 0000000000..ad49ace200
--- /dev/null
+++ b/openmp/libomptarget/test/offloading/malloc.c
@@ -0,0 +1,37 @@
+// RUN: %libomptarget-compile-generic && %libomptarget-run-generic
+// RUN: %libomptarget-compileopt-generic && %libomptarget-run-generic
+
+#include <stdio.h>
+#include <stdlib.h>
+
+int main() {
+  long unsigned *DP = 0;
+  int N = 128;
+  int Threads = 128;
+  int Teams = 440;
+
+  // Allocate ~55MB on the device.
+#pragma omp target map(from : DP)
+  DP = (long unsigned *)malloc(sizeof(long unsigned) * N * Threads * Teams);
+
+#pragma omp target teams distribute parallel for num_teams(Teams)              \
+    thread_limit(Threads) is_device_ptr(DP)
+  for (int i = 0; i < Threads * Teams; ++i) {
+    for (int j = 0; j < N; ++j) {
+      DP[i * N + j] = i + j;
+    }
+  }
+
+  long unsigned s = 0;
+#pragma omp target teams distribute parallel for num_teams(Teams)              \
+    thread_limit(Threads) reduction(+ : s)
+  for (int i = 0; i < Threads * Teams; ++i) {
+    for (int j = 0; j < N; ++j) {
+      s += DP[i * N + j];
+    }
+  }
+
+  // CHECK: Sum: 203458478080
+  printf("Sum: %li\n", s);
+  return 0;
+}
diff --git a/openmp/libomptarget/test/offloading/malloc_parallel.c b/openmp/libomptarget/test/offloading/malloc_parallel.c
new file mode 100644
index 0000000000..b8e975ca55
--- /dev/null
+++ b/openmp/libomptarget/test/offloading/malloc_parallel.c
@@ -0,0 +1,42 @@
+// RUN: %libomptarget-compile-generic && %libomptarget-run-generic
+// RUN: %libomptarget-compileopt-generic && %libomptarget-run-generic
+
+#include <omp.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+int main() {
+  long unsigned **DP = 0;
+  int N = 128;
+  int Threads = 128;
+  int Teams = 440;
+
+#pragma omp target map(from : DP)
+  DP = (long unsigned **)malloc(sizeof(long unsigned *) * Threads * Teams);
+
+#pragma omp target teams distribute parallel for num_teams(Teams)              \
+    thread_limit(Threads)
+  for (int i = 0; i < Threads * Teams; ++i)
+    DP[i] = (long unsigned *)malloc(sizeof(long unsigned) * N);
+
+#pragma omp target teams distribute parallel for num_teams(Teams)              \
+    thread_limit(Threads)
+  for (int i = 0; i < Threads * Teams; ++i) {
+    for (int j = 0; j < N; ++j) {
+      DP[i][j] = i + j;
+    }
+  }
+
+  long unsigned s = 0;
+#pragma omp target teams distribute parallel for num_teams(Teams)              \
+    thread_limit(Threads) reduction(+ : s)
+  for (int i = 0; i < Threads * Teams; ++i) {
+    for (int j = 0; j < N; ++j) {
+      s += DP[i][j];
+    }
+  }
+
+  // CHECK: Sum: 203458478080
+  printf("Sum: %li\n", s);
+  return 0;
+}
diff --git a/openmp/libomptarget/test/offloading/target_critical_region.cpp b/openmp/libomptarget/test/offloading/target_critical_region.cpp
index 533d290b4d..9a741bef6c 100644
--- a/openmp/libomptarget/test/offloading/target_critical_region.cpp
+++ b/openmp/libomptarget/test/offloading/target_critical_region.cpp
@@ -6,6 +6,7 @@
 // UNSUPPORTED: nvptx64-nvidia-cuda-LTO
 // UNSUPPORTED: x86_64-pc-linux-gnu
 // UNSUPPORTED: x86_64-pc-linux-gnu-LTO
+// UNSUPPORTED: amdgcn-amd-amdhsa
 
 #include <omp.h>
 #include <stdio.h>
diff --git a/openmp/libomptarget/test/offloading/thread_state_1.c b/openmp/libomptarget/test/offloading/thread_state_1.c
new file mode 100644
index 0000000000..f3f7b32eea
--- /dev/null
+++ b/openmp/libomptarget/test/offloading/thread_state_1.c
@@ -0,0 +1,38 @@
+// RUN: %libomptarget-compile-run-and-check-generic
+// RUN: %libomptarget-compileopt-run-and-check-generic
+
+#include <omp.h>
+#include <stdio.h>
+
+int main() {
+  // TODO: Test all ICVs on all levels
+  int o_lvl = 111, i_lvl = 222, o_tid = 333, i_tid = 333, o_nt = 444,
+      i_nt = 555;
+#pragma omp target teams map(tofrom : o_lvl, i_lvl, o_tid, i_tid, o_nt, i_nt)  \
+    num_teams(2) thread_limit(64)
+  {
+    if (omp_get_team_num() == 0) {
+#pragma omp parallel num_threads(64)
+      if (omp_get_thread_num() == omp_get_num_threads() - 1) {
+        o_lvl = omp_get_level();
+        o_tid = omp_get_thread_num();
+        o_nt = omp_get_num_threads();
+#pragma omp parallel num_threads(64)
+        if (omp_get_thread_num() == omp_get_num_threads() - 1) {
+          i_lvl = omp_get_level();
+          i_tid = omp_get_thread_num();
+          i_nt = omp_get_num_threads();
+        }
+      }
+    }
+  }
+  if (o_lvl == 1 && o_tid == o_nt - 1 && o_nt > 1 && i_lvl == 2 && i_tid == 0 &&
+      i_nt == 1) {
+    // CHECK: Success
+    printf("Success\n");
+    return 0;
+  }
+  printf("outer: lvl: %i, tid: %i, nt: %i\n", o_lvl, o_tid, o_nt);
+  printf("inner: lvl: %i, tid: %i, nt: %i\n", i_lvl, i_tid, i_nt);
+  return 1;
+}
diff --git a/openmp/libomptarget/test/offloading/thread_state_2.c b/openmp/libomptarget/test/offloading/thread_state_2.c
new file mode 100644
index 0000000000..6d3bf1661f
--- /dev/null
+++ b/openmp/libomptarget/test/offloading/thread_state_2.c
@@ -0,0 +1,40 @@
+// This fails when optimized for now.
+// RUN: %libomptarget-compile-run-and-check-generic
+// XUN: %libomptarget-compileopt-run-and-check-generic
+
+#include <omp.h>
+#include <stdio.h>
+
+int main() {
+  // TODO: Test all ICVs on all levels
+  int o_lvl = 111, i_lvl = 222, o_tid = 333, i_tid = 333, o_nt = 444,
+      i_nt = 555;
+#pragma omp target teams map(tofrom : o_lvl, i_lvl, o_tid, i_tid, o_nt, i_nt)  \
+    num_teams(2) thread_limit(64)
+  {
+    omp_set_max_active_levels(1);
+    if (omp_get_team_num() == 0) {
+#pragma omp parallel num_threads(64)
+      if (omp_get_thread_num() == omp_get_num_threads() - 1) {
+        o_lvl = omp_get_level();
+        o_tid = omp_get_thread_num();
+        o_nt = omp_get_num_threads();
+#pragma omp parallel num_threads(64)
+        if (omp_get_thread_num() == omp_get_num_threads() - 1) {
+          i_lvl = omp_get_level();
+          i_tid = omp_get_thread_num();
+          i_nt = omp_get_num_threads();
+        }
+      }
+    }
+  }
+  if (o_lvl == 1 && o_tid == o_nt - 1 && o_nt > 1 && i_lvl == 2 && i_tid == 0 &&
+      i_nt == 1) {
+    // CHECK: Success
+    printf("Success\n");
+    return 0;
+  }
+  printf("outer: lvl: %i, tid: %i, nt: %i\n", o_lvl, o_tid, o_nt);
+  printf("inner: lvl: %i, tid: %i, nt: %i\n", i_lvl, i_tid, i_nt);
+  return 1;
+}
diff --git a/openmp/runtime/src/dllexports b/openmp/runtime/src/dllexports
index e69cf6670e..0d49643709 100644
--- a/openmp/runtime/src/dllexports
+++ b/openmp/runtime/src/dllexports
@@ -518,6 +518,8 @@ kmp_set_warnings_off                        780
         omp_target_memcpy_rect              887
         omp_target_associate_ptr            888
         omp_target_disassociate_ptr         889
+        omp_target_memset                   3000
+        omp_target_memset_async             3001
     %endif
 
 kmp_set_disp_num_buffers                    890
diff --git a/openmp/runtime/src/include/omp.h.var b/openmp/runtime/src/include/omp.h.var
index 1b2c467a2a..f372402be3 100644
--- a/openmp/runtime/src/include/omp.h.var
+++ b/openmp/runtime/src/include/omp.h.var
@@ -236,6 +236,11 @@
     extern int    __KAI_KMPC_CONVENTION  omp_target_memcpy_rect_async(void *, const void *, size_t, int, const size_t *,
                                              const size_t *, const size_t *, const size_t *, const size_t *, int, int,
                                              int, omp_depend_t *);
+
+    /* OpenMP 6.0 device memory routines */
+    extern void * __KAI_KMPC_CONVENTION omp_target_memset(void *, int, size_t, int);
+    extern void * __KAI_KMPC_CONVENTION omp_target_memset_async(void *, int, size_t, int, int, omp_depend_t *);
+
     /*!
      * The `omp_get_mapped_ptr` routine returns the device pointer that is associated with a host pointer for a given device.
      */
diff --git a/openmp/runtime/src/include/omp_lib.f90.var b/openmp/runtime/src/include/omp_lib.f90.var
index c722874228..1ca542db37 100644
--- a/openmp/runtime/src/include/omp_lib.f90.var
+++ b/openmp/runtime/src/include/omp_lib.f90.var
@@ -635,6 +635,28 @@
             integer (omp_depend_kind), optional :: depobj_list(*)
           end function omp_target_memcpy_rect_async
 
+          function omp_target_memset(ptr, val, count, device_num) bind(c)
+            use, intrinsic :: iso_c_binding, only : c_ptr, c_int, c_size_t
+            type(c_ptr) :: omp_target_memset
+            type(c_ptr), value :: ptr
+            integer(c_int), value :: val
+            integer(c_size_t), value :: count
+            integer(c_int), value :: device_num
+          end function
+
+          function omp_target_memset_async(ptr, val, count, device_num, &
+                                           depobj_count, depobj_list) bind(c)
+            use, intrinsic :: iso_c_binding, only : c_ptr, c_int, c_size_t
+            use omp_lib_kinds
+            type(c_ptr) :: omp_target_memset_async
+            type(c_ptr), value :: ptr
+            integer(c_int), value :: val
+            integer(c_size_t), value :: count
+            integer(c_int), value :: device_num
+            integer(c_int), value :: depobj_count
+            integer(omp_depend_kind), optional :: depobj_list(*)
+          end function
+
           function omp_target_associate_ptr(host_ptr, device_ptr, size,        &
               device_offset, device_num) bind(c)
             use omp_lib_kinds
diff --git a/openmp/runtime/src/include/omp_lib.h.var b/openmp/runtime/src/include/omp_lib.h.var
index 9f5e58515e..d20aade6ef 100644
--- a/openmp/runtime/src/include/omp_lib.h.var
+++ b/openmp/runtime/src/include/omp_lib.h.var
@@ -732,6 +732,28 @@
           integer(omp_depend_kind), optional :: depobj_list(*)
         end function omp_target_memcpy_rect_async
 
+        function omp_target_memset(ptr, val, count, device_num) bind(c)
+          use, intrinsic :: iso_c_binding, only : c_ptr, c_int, c_size_t
+          type(c_ptr) :: omp_target_memset
+          type(c_ptr), value :: ptr
+          integer(c_int), value :: val
+          integer(c_size_t), value :: count
+          integer(c_int), value :: device_num
+        end function
+
+        function omp_target_memset_async(ptr, val, count, device_num, &
+                                         depobj_count, depobj_list) bind(c)
+          use, intrinsic :: iso_c_binding, only : c_ptr, c_int, c_size_t
+          use omp_lib_kinds
+          type(c_ptr) :: omp_target_memset_async
+          type(c_ptr), value :: ptr
+          integer(c_int), value :: val
+          integer(c_size_t), value :: count
+          integer(c_int), value :: device_num
+          integer(c_int), value :: depobj_count
+          integer(omp_depend_kind), optional :: depobj_list(*)
+        end function
+
         function omp_target_associate_ptr(host_ptr, device_ptr, size,                                                               &
      &      device_offset, device_num) bind(c)
           use, intrinsic :: iso_c_binding, only : c_ptr, c_size_t, c_int
diff --git a/openmp/runtime/src/kmp_ftn_os.h b/openmp/runtime/src/kmp_ftn_os.h
index d37c9c8602..7d595b947f 100644
--- a/openmp/runtime/src/kmp_ftn_os.h
+++ b/openmp/runtime/src/kmp_ftn_os.h
@@ -116,6 +116,8 @@
 #define FTN_TARGET_IS_PRESENT omp_target_is_present
 #define FTN_TARGET_MEMCPY omp_target_memcpy
 #define FTN_TARGET_MEMCPY_RECT omp_target_memcpy_rect
+#define FTN_TARGET_MEMSET omp_target_memset
+#define FTN_TARGET_MEMSET_ASYNC omp_target_memset_async
 #define FTN_TARGET_ASSOCIATE_PTR omp_target_associate_ptr
 #define FTN_TARGET_DISASSOCIATE_PTR omp_target_disassociate_ptr
 #endif
diff --git a/polly/lib/Analysis/ScopInfo.cpp b/polly/lib/Analysis/ScopInfo.cpp
index ffa61bb778..e294740307 100644
--- a/polly/lib/Analysis/ScopInfo.cpp
+++ b/polly/lib/Analysis/ScopInfo.cpp
@@ -1647,9 +1647,7 @@ void Scop::removeFromStmtMap(ScopStmt &Stmt) {
   } else {
     auto StmtMapIt = StmtMap.find(Stmt.getBasicBlock());
     if (StmtMapIt != StmtMap.end())
-      StmtMapIt->second.erase(std::remove(StmtMapIt->second.begin(),
-                                          StmtMapIt->second.end(), &Stmt),
-                              StmtMapIt->second.end());
+      llvm::erase_value(StmtMapIt->second, &Stmt);
     for (Instruction *Inst : Stmt.getInstructions())
       InstStmtMap.erase(Inst);
   }
@@ -2424,15 +2422,13 @@ void Scop::removeAccessData(MemoryAccess *Access) {
     ValueDefAccs.erase(Access->getAccessValue());
   } else if (Access->isOriginalValueKind() && Access->isRead()) {
     auto &Uses = ValueUseAccs[Access->getScopArrayInfo()];
-    auto NewEnd = std::remove(Uses.begin(), Uses.end(), Access);
-    Uses.erase(NewEnd, Uses.end());
+    llvm::erase_value(Uses, Access);
   } else if (Access->isOriginalPHIKind() && Access->isRead()) {
     PHINode *PHI = cast<PHINode>(Access->getAccessInstruction());
     PHIReadAccs.erase(PHI);
   } else if (Access->isOriginalAnyPHIKind() && Access->isWrite()) {
     auto &Incomings = PHIIncomingAccs[Access->getScopArrayInfo()];
-    auto NewEnd = std::remove(Incomings.begin(), Incomings.end(), Access);
-    Incomings.erase(NewEnd, Incomings.end());
+    llvm::erase_value(Incomings, Access);
   }
 }
 
diff --git a/runtimes/cmake/android/Arch-arm.cmake b/runtimes/cmake/android/Arch-arm.cmake
new file mode 100644
index 0000000000..c9e6e2509c
--- /dev/null
+++ b/runtimes/cmake/android/Arch-arm.cmake
@@ -0,0 +1,7 @@
+include(${CMAKE_CURRENT_LIST_DIR}/Common.cmake)
+
+set(CMAKE_SYSTEM_PROCESSOR "armv7-a" CACHE STRING "")
+set(CMAKE_ASM_COMPILER_TARGET "armv7a-linux-androideabi21" CACHE STRING "")
+set(CMAKE_C_COMPILER_TARGET   "armv7a-linux-androideabi21" CACHE STRING "")
+set(CMAKE_CXX_COMPILER_TARGET "armv7a-linux-androideabi21" CACHE STRING "")
+set(ANDROID_NATIVE_API_LEVEL 21 CACHE STRING "")
diff --git a/runtimes/cmake/android/Arch-arm64.cmake b/runtimes/cmake/android/Arch-arm64.cmake
new file mode 100644
index 0000000000..9cae3313fa
--- /dev/null
+++ b/runtimes/cmake/android/Arch-arm64.cmake
@@ -0,0 +1,7 @@
+include(${CMAKE_CURRENT_LIST_DIR}/Common.cmake)
+
+set(CMAKE_SYSTEM_PROCESSOR "aarch64" CACHE STRING "")
+set(CMAKE_ASM_COMPILER_TARGET "aarch64-linux-android21" CACHE STRING "")
+set(CMAKE_C_COMPILER_TARGET   "aarch64-linux-android21" CACHE STRING "")
+set(CMAKE_CXX_COMPILER_TARGET "aarch64-linux-android21" CACHE STRING "")
+set(ANDROID_NATIVE_API_LEVEL 21 CACHE STRING "")
diff --git a/runtimes/cmake/android/Arch-x86.cmake b/runtimes/cmake/android/Arch-x86.cmake
new file mode 100644
index 0000000000..31b7a719e6
--- /dev/null
+++ b/runtimes/cmake/android/Arch-x86.cmake
@@ -0,0 +1,7 @@
+include(${CMAKE_CURRENT_LIST_DIR}/Common.cmake)
+
+set(CMAKE_SYSTEM_PROCESSOR "i686" CACHE STRING "")
+set(CMAKE_ASM_COMPILER_TARGET "i686-linux-android21" CACHE STRING "")
+set(CMAKE_C_COMPILER_TARGET   "i686-linux-android21" CACHE STRING "")
+set(CMAKE_CXX_COMPILER_TARGET "i686-linux-android21" CACHE STRING "")
+set(ANDROID_NATIVE_API_LEVEL 21 CACHE STRING "")
diff --git a/runtimes/cmake/android/Arch-x86_64.cmake b/runtimes/cmake/android/Arch-x86_64.cmake
new file mode 100644
index 0000000000..a109fac697
--- /dev/null
+++ b/runtimes/cmake/android/Arch-x86_64.cmake
@@ -0,0 +1,7 @@
+include(${CMAKE_CURRENT_LIST_DIR}/Common.cmake)
+
+set(CMAKE_SYSTEM_PROCESSOR "x86_64" CACHE STRING "")
+set(CMAKE_ASM_COMPILER_TARGET "x86_64-linux-android21" CACHE STRING "")
+set(CMAKE_C_COMPILER_TARGET   "x86_64-linux-android21" CACHE STRING "")
+set(CMAKE_CXX_COMPILER_TARGET "x86_64-linux-android21" CACHE STRING "")
+set(ANDROID_NATIVE_API_LEVEL 21 CACHE STRING "")
diff --git a/runtimes/cmake/android/Common.cmake b/runtimes/cmake/android/Common.cmake
new file mode 100644
index 0000000000..3f49334651
--- /dev/null
+++ b/runtimes/cmake/android/Common.cmake
@@ -0,0 +1,6 @@
+set(CMAKE_SYSTEM_NAME "Android" CACHE STRING "")
+
+# Set the CMake system version to "1" to inhibit CMake's built-in support for
+# compiling using the Android NDK, which gets in the way when we're not using an
+# NDK.
+set(CMAKE_SYSTEM_VERSION "1" CACHE STRING "")
diff --git a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
index 73c2c95f4c..2f3fdd3905 100644
--- a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
@@ -1860,6 +1860,7 @@ cc_library(
         "//llvm:IRReader",
         "//llvm:InstCombine",
         "//llvm:Instrumentation",
+        "//llvm:HipStdPar",
         "//llvm:LTO",
         "//llvm:Linker",
         "//llvm:MC",
diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index e48933aef9..fdf60cd0df 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -3,16 +3,16 @@
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 # LLVM libc project.
+load("@bazel_skylib//lib:selects.bzl", "selects")
+load("@bazel_skylib//rules:common_settings.bzl", "string_flag")
 load(
     ":libc_build_rules.bzl",
     "libc_function",
     "libc_math_function",
     "libc_support_library",
 )
-load(":platforms.bzl", "PLATFORM_CPU_ARM64", "PLATFORM_CPU_X86_64")
 load(":libc_namespace.bzl", "LIBC_NAMESPACE")
-load("@bazel_skylib//lib:selects.bzl", "selects")
-load("@bazel_skylib//rules:common_settings.bzl", "bool_flag", "string_flag")
+load(":platforms.bzl", "PLATFORM_CPU_ARM64", "PLATFORM_CPU_X86_64")
 
 package(
     default_visibility = ["//visibility:public"],
diff --git a/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl b/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl
index 2007d1219b..0f4f3a40d3 100644
--- a/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl
+++ b/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl
@@ -4,8 +4,8 @@
 
 """LLVM libc starlark rules for building individual functions."""
 
-load(":platforms.bzl", "PLATFORM_CPU_ARM64", "PLATFORM_CPU_X86_64")
 load("@bazel_skylib//lib:selects.bzl", "selects")
+load(":platforms.bzl", "PLATFORM_CPU_ARM64", "PLATFORM_CPU_X86_64")
 
 LIBC_ROOT_TARGET = ":libc_root"
 INTERNAL_SUFFIX = ".__internal__"
diff --git a/utils/bazel/llvm-project-overlay/libc/libc_namespace.bzl b/utils/bazel/llvm-project-overlay/libc/libc_namespace.bzl
index 8cbac6c624..a59aeea737 100644
--- a/utils/bazel/llvm-project-overlay/libc/libc_namespace.bzl
+++ b/utils/bazel/llvm-project-overlay/libc/libc_namespace.bzl
@@ -1,3 +1,9 @@
+# This file is licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+"""LLVM libc namespace declaration."""
+
 load("//:vars.bzl", "LLVM_VERSION_MAJOR", "LLVM_VERSION_MINOR", "LLVM_VERSION_PATCH")
 
 # The default libc namespace that encloses all functions.
diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
index a8dcb0ec32..e79dd2d9a0 100644
--- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
@@ -1663,6 +1663,23 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "HipStdPar",
+    srcs = glob([
+        "lib/Transforms/HipStdPar/*.cpp",
+        "lib/Transforms/HipStdPar/*.h",
+    ]),
+    hdrs = ["include/llvm/Transforms/HipStdPar/HipStdPar.h"],
+    copts = llvm_copts,
+    deps = [
+        ":Analysis",
+        ":Core",
+        ":Support",
+        ":TargetParser",
+        ":TransformUtils",
+    ],
+)
+
 cc_library(
     name = "Coroutines",
     srcs = glob([
@@ -2502,22 +2519,23 @@ cc_library(
         ":AggressiveInstCombine",
         ":Analysis",
         ":CodeGen",
+        ":common_transforms",
+        ":config",
         ":Core",
         ":Coroutines",
-        ":IPO",
-        ":IRPrinter",
+        ":HipStdPar",
         ":InstCombine",
         ":Instrumentation",
+        ":IPO",
+        ":IRPrinter",
         ":MLPolicies",
         ":ObjCARC",
+        ":pass_registry_def",
         ":Scalar",
         ":Support",
         ":Target",
         ":TransformUtils",
         ":Vectorize",
-        ":common_transforms",
-        ":config",
-        ":pass_registry_def",
     ],
 )
 
diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index 52845774f1..abe613b45c 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -2097,6 +2097,7 @@ cc_library(
         ":IR",
         ":LLVMDialect",
         ":SideEffectInterfaces",
+        ":VectorDialect",
         "//llvm:Core",
         "//llvm:Support",
     ],
@@ -2109,13 +2110,12 @@ cc_library(
     includes = ["include"],
     deps = [
         ":ArmSVEDialect",
+        ":DialectUtils",
         ":FuncDialect",
         ":IR",
         ":LLVMCommonConversion",
         ":LLVMDialect",
-        ":TransformUtils",
-        "//llvm:Core",
-        "//llvm:Support",
+        ":VectorDialect",
     ],
 )
 
@@ -2683,6 +2683,7 @@ td_library(
     srcs = [
         "include/mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.td",
         "include/mlir/Dialect/SparseTensor/IR/SparseTensorBase.td",
+        "include/mlir/Dialect/SparseTensor/IR/SparseTensorInterfaces.td",
         "include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td",
         "include/mlir/Dialect/SparseTensor/IR/SparseTensorTypes.td",
     ],
@@ -2694,6 +2695,15 @@ td_library(
     ],
 )
 
+td_library(
+    name = "SparseTensorInterfacesTdFiles",
+    srcs = [
+        "include/mlir/Dialect/SparseTensor/IR/SparseTensorInterfaces.td",
+    ],
+    includes = ["include"],
+    deps = [":OpBaseTdFiles"],
+)
+
 gentbl_cc_library(
     name = "SparseTensorAttrDefsIncGen",
     tbl_outs = [
@@ -2801,6 +2811,23 @@ gentbl_cc_library(
     deps = [":PassBaseTdFiles"],
 )
 
+gentbl_cc_library(
+    name = "SparseTensorInterfacesIncGen",
+    tbl_outs = [
+        (
+            ["-gen-op-interface-decls"],
+            "include/mlir/Dialect/SparseTensor/IR/SparseTensorInterfaces.h.inc",
+        ),
+        (
+            ["-gen-op-interface-defs"],
+            "include/mlir/Dialect/SparseTensor/IR/SparseTensorInterfaces.cpp.inc",
+        ),
+    ],
+    tblgen = ":mlir-tblgen",
+    td_file = "include/mlir/Dialect/SparseTensor/IR/SparseTensorInterfaces.td",
+    deps = [":SparseTensorInterfacesTdFiles"],
+)
+
 # This library is shared by both SparseTensorDialect and
 # SparseTensorRuntime, so it must not depend on any of the MLIR/LLVM
 # internals or else mlir_c_runner_utils will inherit that dependency.
@@ -2823,9 +2850,11 @@ cc_library(
         "lib/Dialect/SparseTensor/IR/Detail/Var.cpp",
         "lib/Dialect/SparseTensor/IR/Detail/Var.h",
         "lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp",
+        "lib/Dialect/SparseTensor/IR/SparseTensorInterfaces.cpp",
     ],
     hdrs = [
         "include/mlir/Dialect/SparseTensor/IR/SparseTensor.h",
+        "include/mlir/Dialect/SparseTensor/IR/SparseTensorInterfaces.h",
         "include/mlir/Dialect/SparseTensor/IR/SparseTensorStorageLayout.h",
         "include/mlir/Dialect/SparseTensor/IR/SparseTensorType.h",
     ],
@@ -2837,6 +2866,7 @@ cc_library(
         ":InferTypeOpInterface",
         ":SparseTensorAttrDefsIncGen",
         ":SparseTensorEnums",
+        ":SparseTensorInterfacesIncGen",
         ":SparseTensorOpsIncGen",
         ":SparseTensorTypesIncGen",
         "//llvm:Support",
@@ -2986,6 +3016,106 @@ cc_library(
     ]),
 )
 
+##---------------------------------------------------------------------------##
+# Mesh Dialect
+##---------------------------------------------------------------------------##
+
+td_library(
+    name = "MeshTdFiles",
+    srcs = [
+        "include/mlir/Dialect/Mesh/IR/MeshBase.td",
+        "include/mlir/Dialect/Mesh/IR/MeshOps.td",
+    ],
+    includes = ["include"],
+    deps = [
+        ":BuiltinDialectTdFiles",
+        ":InferTypeOpInterfaceTdFiles",
+        ":SideEffectInterfacesTdFiles",
+    ],
+)
+
+gentbl_cc_library(
+    name = "MeshIncGen",
+    tbl_outs = [
+        (
+            [
+                "-gen-op-decls",
+                "-dialect=mesh",
+            ],
+            "include/mlir/Dialect/Mesh/IR/MeshOps.h.inc",
+        ),
+        (
+            [
+                "-gen-op-defs",
+                "-dialect=mesh",
+            ],
+            "include/mlir/Dialect/Mesh/IR/MeshOps.cpp.inc",
+        ),
+        (
+            [
+                "-gen-dialect-decls",
+                "-dialect=mesh",
+            ],
+            "include/mlir/Dialect/Mesh/IR/MeshOpsDialect.h.inc",
+        ),
+        (
+            [
+                "-gen-dialect-defs",
+                "-dialect=mesh",
+            ],
+            "include/mlir/Dialect/Mesh/IR/MeshOpsDialect.cpp.inc",
+        ),
+        (
+            [
+                "-gen-enum-decls",
+                "-dialect=mesh",
+            ],
+            "include/mlir/Dialect/Mesh/IR/MeshOpsEnums.h.inc",
+        ),
+        (
+            [
+                "-gen-enum-defs",
+                "-dialect=mesh",
+            ],
+            "include/mlir/Dialect/Mesh/IR/MeshOpsEnums.cpp.inc",
+        ),
+        (
+            [
+                "-gen-attrdef-decls",
+                "-dialect=mesh",
+            ],
+            "include/mlir/Dialect/Mesh/IR/MeshOpsAttributes.h.inc",
+        ),
+        (
+            [
+                "-gen-attrdef-defs",
+                "-dialect=mesh",
+            ],
+            "include/mlir/Dialect/Mesh/IR/MeshOpsAttributes.cpp.inc",
+        ),
+    ],
+    tblgen = ":mlir-tblgen",
+    td_file = "include/mlir/Dialect/Mesh/IR/MeshOps.td",
+    deps = [
+        ":MeshTdFiles",
+    ],
+)
+
+cc_library(
+    name = "MeshDialect",
+    srcs = ["lib/Dialect/Mesh/IR/MeshOps.cpp"],
+    hdrs = ["include/mlir/Dialect/Mesh/IR/MeshOps.h"],
+    includes = ["include"],
+    deps = [
+        ":ArithDialect",
+        ":IR",
+        ":InferTypeOpInterface",
+        ":MeshIncGen",
+        ":Support",
+        "//llvm:Support",
+    ],
+)
+
 ##---------------------------------------------------------------------------##
 # NVGPU dialect.
 ##---------------------------------------------------------------------------##
@@ -3575,6 +3705,7 @@ cc_library(
         ":GPUToSPIRV",
         ":GPUToVulkanTransforms",
         ":IndexToLLVM",
+        ":IndexToSPIRV",
         ":LinalgToStandard",
         ":MathToFuncs",
         ":MathToLLVM",
@@ -4256,6 +4387,7 @@ cc_library(
         ":MathToLLVM",
         ":MemRefToLLVM",
         ":MemRefTransformOps",
+        ":MeshDialect",
         ":NVGPUTransformOps",
         ":NVVMTarget",
         ":NVVMToLLVM",
@@ -4321,6 +4453,7 @@ cc_library(
     ]),
     includes = ["include"],
     deps = [
+        ":AffineDialect",
         ":ArithDialect",
         ":ArithUtils",
         ":ControlFlowInterfaces",
@@ -4335,9 +4468,10 @@ cc_library(
         ":SideEffectInterfaces",
         ":Support",
         ":TensorDialect",
-        ":VectorInterfaces",
+        ":ValueBoundsOpInterface",
         ":VectorAttributesIncGen",
         ":VectorDialectIncGen",
+        ":VectorInterfaces",
         ":VectorOpsIncGen",
         ":ViewLikeInterface",
         "//llvm:Support",
@@ -4715,6 +4849,7 @@ cc_library(
             "lib/Dialect/LLVMIR/IR/NVVM*.cpp",
             "lib/Dialect/LLVMIR/IR/NVVM*.h",
             "lib/Dialect/LLVMIR/IR/ROCDL*.cpp",
+            "lib/Dialect/LLVMIR/IR/BasicPtxBuilderInterface.cpp",
             "lib/Dialect/LLVMIR/IR/ROCDL*.h",
             "lib/Dialect/LLVMIR/IR/*X86Vector*.cpp",
             "lib/Dialect/LLVMIR/IR/*X86Vector*.h",
@@ -4726,6 +4861,7 @@ cc_library(
             "include/mlir/Dialect/LLVMIR/*AMX*.h",
             "include/mlir/Dialect/LLVMIR/*ArmSVE*.h",
             "include/mlir/Dialect/LLVMIR/NVVM*.h",
+            "include/mlir/Dialect/LLVMIR/BasicPtxBuilderInterface.h",
             "include/mlir/Dialect/LLVMIR/ROCDL*.h",
             "include/mlir/Dialect/LLVMIR/*X86Vector*.h",
         ],
@@ -5667,6 +5803,7 @@ cc_library(
     hdrs = ["include/mlir/Dialect/LLVMIR/NVVMDialect.h"],
     includes = ["include"],
     deps = [
+        ":BasicPtxBuilderInterface",
         ":ConvertToLLVM",
         ":DialectUtils",
         ":GPUDialect",
@@ -5721,11 +5858,25 @@ cc_library(
     ],
 )
 
+td_library(
+    name = "BasicPtxBuilderIntTdFiles",
+    srcs = [
+        "include/mlir/Dialect/LLVMIR/BasicPtxBuilderInterface.td",
+    ],
+    includes = ["include"],
+    deps = [
+        ":GPUOpsTdFiles",
+        ":LLVMOpsTdFiles",
+        ":OpBaseTdFiles",
+    ],
+)
+
 td_library(
     name = "NVVMOpsTdFiles",
     srcs = ["include/mlir/Dialect/LLVMIR/NVVMOps.td"],
     includes = ["include"],
     deps = [
+        ":BasicPtxBuilderIntTdFiles",
         ":GPUOpsTdFiles",
         ":LLVMOpsTdFiles",
         ":OpBaseTdFiles",
@@ -5733,6 +5884,31 @@ td_library(
     ],
 )
 
+gentbl_cc_library(
+    name = "BasicPtxBuilderIntGen",
+    tbl_outs = [
+        (
+            [
+                "-gen-op-interface-decls",
+            ],
+            "include/mlir/Dialect/LLVMIR/BasicPtxBuilderInterface.h.inc",
+        ),
+        (
+            [
+                "-gen-op-interface-defs",
+            ],
+            "include/mlir/Dialect/LLVMIR/BasicPtxBuilderInterface.cpp.inc",
+        ),
+    ],
+    tblgen = ":mlir-tblgen",
+    td_file = "include/mlir/Dialect/LLVMIR/BasicPtxBuilderInterface.td",
+    deps = [
+        ":BasicPtxBuilderIntTdFiles",
+        ":GPUOpsTdFiles",
+        ":LLVMOpsTdFiles",
+    ],
+)
+
 gentbl_cc_library(
     name = "NVVMOpsIncGen",
     tbl_outs = [
@@ -5780,20 +5956,6 @@ gentbl_cc_library(
             ],
             "include/mlir/Dialect/LLVMIR/NVVMOpsAttributes.cpp.inc",
         ),
-        (
-            [
-                "-gen-op-interface-decls",
-                "-attrdefs-dialect=nvvm",
-            ],
-            "include/mlir/Dialect/LLVMIR/NVVMOpsInterface.h.inc",
-        ),
-        (
-            [
-                "-gen-op-interface-defs",
-                "-attrdefs-dialect=nvvm",
-            ],
-            "include/mlir/Dialect/LLVMIR/NVVMOpsInterface.cpp.inc",
-        ),
     ],
     tblgen = ":mlir-tblgen",
     td_file = "include/mlir/Dialect/LLVMIR/NVVMOps.td",
@@ -5813,6 +5975,21 @@ gentbl_cc_library(
     deps = [":NVVMOpsTdFiles"],
 )
 
+cc_library(
+    name = "BasicPtxBuilderInterface",
+    srcs = ["lib/Dialect/LLVMIR/IR/BasicPtxBuilderInterface.cpp"],
+    hdrs = [
+        "include/mlir/Dialect/LLVMIR/BasicPtxBuilderInterface.h",
+    ],
+    includes = ["include"],
+    deps = [
+        ":BasicPtxBuilderIntGen",
+        ":IR",
+        ":LLVMDialect",
+        ":Support",
+    ],
+)
+
 cc_library(
     name = "NVVMToLLVM",
     srcs = glob(["lib/Conversion/NVVMToLLVM/NVVMToLLVM.cpp"]),
@@ -7249,6 +7426,7 @@ cc_library(
         ":FuncDialect",
         ":FuncToSPIRV",
         ":IR",
+        ":IndexToSPIRV",
         ":MemRefToSPIRV",
         ":Pass",
         ":SCFDialect",
@@ -7375,6 +7553,7 @@ cc_library(
         ":DialectUtils",
         ":FuncDialect",
         ":FuncTransforms",
+        ":FunctionInterfaces",
         ":IR",
         ":LLVMCommonConversion",
         ":LLVMDialect",
@@ -7894,6 +8073,7 @@ cc_library(
         ":LLVMIntrinsicConversionIncGen",
         ":OpenMPDialect",
         ":Support",
+        ":TransformUtils",
         "//llvm:Core",
         "//llvm:FrontendOpenMP",
         "//llvm:Support",
@@ -8099,6 +8279,7 @@ cc_library(
         ":OpenMPCommon",
         ":Support",
         ":ToLLVMIRTranslation",
+        ":TransformUtils",
         "//llvm:Core",
         "//llvm:FrontendOpenMP",
         "//llvm:Support",
@@ -8121,8 +8302,8 @@ cc_library(
         "//llvm:Core",
         "//llvm:FrontendOpenMP",
         "//llvm:Support",
-        "//llvm:TransformUtils",
         "//llvm:TargetParser",
+        "//llvm:TransformUtils",
     ],
 )
 
@@ -8425,6 +8606,7 @@ cc_library(
         ":MemRefToSPIRV",
         ":MemRefTransformOps",
         ":MemRefTransforms",
+        ":MeshDialect",
         ":NVGPUDialect",
         ":NVGPUPassIncGen",
         ":NVGPUToNVVM",
@@ -8644,7 +8826,6 @@ cc_library(
     srcs = [
         "lib/ExecutionEngine/SparseTensor/File.cpp",
         "lib/ExecutionEngine/SparseTensor/MapRef.cpp",
-        "lib/ExecutionEngine/SparseTensor/NNZ.cpp",
         "lib/ExecutionEngine/SparseTensor/Storage.cpp",
     ],
     hdrs = [
@@ -9043,7 +9224,6 @@ td_library(
     ],
 )
 
-
 gentbl_cc_library(
     name = "OpenACCOpsInterfacesIncGen",
     tbl_outs = [
@@ -9061,7 +9241,6 @@ gentbl_cc_library(
     deps = [":OpenAccOpsTdFiles"],
 )
 
-
 gentbl_cc_library(
     name = "OpenACCOpsIncGen",
     tbl_outs = [
@@ -9665,6 +9844,31 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "IndexToSPIRV",
+    srcs = glob([
+        "lib/Conversion/IndexToSPIRV/*.cpp",
+        "lib/Conversion/IndexToSPIRV/*.h",
+    ]),
+    hdrs = glob([
+        "include/mlir/Conversion/IndexToSPIRV/*.h",
+    ]),
+    includes = ["include"],
+    deps = [
+        ":ConversionPassIncGen",
+        ":IR",
+        ":IndexDialect",
+        ":Pass",
+        ":SPIRVCommonConversion",
+        ":SPIRVConversion",
+        ":SPIRVDialect",
+        ":Support",
+        ":Transforms",
+        "//llvm:Core",
+        "//llvm:Support",
+    ],
+)
+
 cc_library(
     name = "IndexDialect",
     srcs = glob(["lib/Dialect/Index/IR/*.cpp"]),
@@ -10433,7 +10637,7 @@ gentbl_cc_library(
     td_file = "include/mlir/Dialect/Vector/IR/Vector.td",
     deps = [
         ":ArithOpsTdFiles",
-        ":VectorOpsTdFiles"
+        ":VectorOpsTdFiles",
     ],
 )
 
@@ -10453,7 +10657,7 @@ gentbl_cc_library(
     td_file = "include/mlir/Dialect/Vector/IR/VectorOps.td",
     deps = [
         ":ArithOpsTdFiles",
-        ":VectorOpsTdFiles"
+        ":VectorOpsTdFiles",
     ],
 )
 
@@ -10481,7 +10685,7 @@ gentbl_cc_library(
     td_file = "include/mlir/Dialect/Vector/IR/VectorAttributes.td",
     deps = [
         ":ArithOpsTdFiles",
-        ":VectorOpsTdFiles"
+        ":VectorOpsTdFiles",
     ],
 )
 
@@ -12147,12 +12351,12 @@ cc_library(
     ]),
     includes = ["include"],
     deps = [
-        ":MLProgramDialect",
         ":FuncDialect",
-        ":TransformUtils",
         ":IR",
+        ":MLProgramDialect",
         ":MLProgramPassIncGen",
         ":Pass",
+        ":TransformUtils",
     ],
 )
 
@@ -12773,8 +12977,8 @@ gentbl_cc_library(
 cc_library(
     name = "UBDialect",
     srcs = [
-        "lib/Dialect/UB/IR/UBOps.cpp",
         "include/mlir/Transforms/InliningUtils.h",
+        "lib/Dialect/UB/IR/UBOps.cpp",
     ],
     hdrs = ["include/mlir/Dialect/UB/IR/UBOps.h"],
     includes = ["include"],
diff --git a/utils/bazel/llvm-project-overlay/mlir/python/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/python/BUILD.bazel
index 949bfa4fbd..c83e4cc7ad 100644
--- a/utils/bazel/llvm-project-overlay/mlir/python/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/python/BUILD.bazel
@@ -82,6 +82,49 @@ filegroup(
     ]),
 )
 
+##---------------------------------------------------------------------------##
+# Affine dialect.
+##---------------------------------------------------------------------------##
+
+td_library(
+    name = "AffineOpsPyTdFiles",
+    srcs = [
+        "mlir/dialects/AffineOps.td",
+    ],
+    deps = [
+        "//mlir:AffineOpsTdFiles",
+        "//mlir:FunctionInterfacesTdFiles",
+        "//mlir:OpBaseTdFiles",
+    ],
+)
+
+gentbl_filegroup(
+    name = "AffineOpsPyGen",
+    tbl_outs = [
+        (
+            [
+                "-gen-python-op-bindings",
+                "-bind-dialect=affine",
+            ],
+            "mlir/dialects/_affine_ops_gen.py",
+        ),
+    ],
+    tblgen = "//mlir:mlir-tblgen",
+    td_file = "mlir/dialects/AffineOps.td",
+    deps = [
+        ":AffineOpsPyTdFiles",
+    ],
+)
+
+filegroup(
+    name = "AffineOpsPyFiles",
+    srcs = [
+        "mlir/dialects/affine.py",
+        ":AffineOpsPyGen",
+    ],
+)
+
+
 ##---------------------------------------------------------------------------##
 # Builtin dialect.
 ##---------------------------------------------------------------------------##
@@ -119,7 +162,6 @@ gentbl_filegroup(
 filegroup(
     name = "BuiltinOpsPyFiles",
     srcs = [
-        "mlir/dialects/_builtin_ops_ext.py",
         "mlir/dialects/builtin.py",
         ":BuiltinOpsPyGen",
     ],
@@ -216,7 +258,6 @@ gentbl_filegroup(
 filegroup(
     name = "LinalgOpsPyFiles",
     srcs = [
-        "mlir/dialects/_linalg_ops_ext.py",
         ":LinalgOpsPyGen",
     ],
 )
@@ -317,7 +358,6 @@ gentbl_filegroup(
 filegroup(
     name = "ArithOpsPyFiles",
     srcs = [
-        "mlir/dialects/_arith_ops_ext.py",
         "mlir/dialects/arith.py",
         ":ArithOpsPyGen",
     ],
@@ -380,7 +420,6 @@ gentbl_filegroup(
 filegroup(
     name = "BufferizationOpsPyFiles",
     srcs = [
-        "mlir/dialects/_bufferization_ops_ext.py",
         "mlir/dialects/bufferization.py",
         ":BufferizationEnumPyGen",
         ":BufferizationOpsPyGen",
@@ -506,7 +545,6 @@ gentbl_filegroup(
 filegroup(
     name = "MemRefOpsPyFiles",
     srcs = [
-        "mlir/dialects/_memref_ops_ext.py",
         "mlir/dialects/memref.py",
         ":MemRefOpsPyGen",
     ],
@@ -538,7 +576,6 @@ gentbl_filegroup(
 filegroup(
     name = "MLProgramOpsPyFiles",
     srcs = [
-        "mlir/dialects/_ml_program_ops_ext.py",
         "mlir/dialects/ml_program.py",
         ":MLProgramOpsPyGen",
     ],
@@ -608,7 +645,6 @@ filegroup(
 filegroup(
     name = "PDLPyFiles",
     srcs = [
-        "mlir/dialects/_pdl_ops_ext.py",
         "mlir/dialects/pdl.py",
         ":PDLPyGen",
     ],
@@ -823,7 +859,6 @@ gentbl_filegroup(
 filegroup(
     name = "SCFPyFiles",
     srcs = [
-        "mlir/dialects/_scf_ops_ext.py",
         "mlir/dialects/scf.py",
         ":SCFPyGen",
     ],
@@ -896,7 +931,6 @@ gentbl_filegroup(
 filegroup(
     name = "FuncPyFiles",
     srcs = [
-        "mlir/dialects/_func_ops_ext.py",
         "mlir/dialects/func.py",
         ":FuncPyGen",
     ],
@@ -996,7 +1030,6 @@ gentbl_filegroup(
 filegroup(
     name = "TensorOpsPyFiles",
     srcs = [
-        "mlir/dialects/_tensor_ops_ext.py",
         "mlir/dialects/tensor.py",
         ":TensorOpsPyGen",
     ],
@@ -1334,14 +1367,6 @@ gentbl_filegroup(
 filegroup(
     name = "TransformOpsPyFiles",
     srcs = [
-        "mlir/dialects/_bufferization_transform_ops_ext.py",
-        "mlir/dialects/_gpu_transform_ops_ext.py",
-        "mlir/dialects/_loop_transform_ops_ext.py",
-        "mlir/dialects/_memref_transform_ops_ext.py",
-        "mlir/dialects/_structured_transform_ops_ext.py",
-        "mlir/dialects/_tensor_transform_ops_ext.py",
-        "mlir/dialects/_transform_ops_ext.py",
-        "mlir/dialects/_transform_pdl_extension_ops_ext.py",
         ":BufferizationTransformOpsPyGen",
         ":GPUTransformOpsPyGen",
         ":LoopTransformOpsPyGen",
diff --git a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
index e43d09618a..74022a21a6 100644
--- a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
@@ -145,7 +145,6 @@ gentbl_cc_library(
     ],
 )
 
-
 gentbl_cc_library(
     name = "TestOpsIncGen",
     strip_include_prefix = "lib/Dialect/Test",
@@ -468,6 +467,7 @@ cc_library(
         "//mlir:AffineDialect",
         "//mlir:ArithDialect",
         "//mlir:FuncDialect",
+        "//mlir:GPUDialect",
         "//mlir:IR",
         "//mlir:LinalgDialect",
         "//mlir:LinalgTransforms",
diff --git a/utils/bazel/llvm-project-overlay/mlir/test/Dialect/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/test/Dialect/BUILD.bazel
index 061404078b..1fd6885db8 100644
--- a/utils/bazel/llvm-project-overlay/mlir/test/Dialect/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/test/Dialect/BUILD.bazel
@@ -18,10 +18,7 @@ package(default_visibility = ["//visibility:public"])
         ] + glob([
             "IRDL/*.irdl.mlir",
             "LLVM/*-symbol-def.mlir",
-            "Transform/*-source.mlir",
-            "Transform/*-symbol-def.mlir",
-            "Transform/*-symbol-decl-and-schedule.mlir",
-            "Transform/test-interpreter-library/*.mlir",
+            "Transform/include/**/*.mlir",
         ]),
     )
     for src in glob(
@@ -29,13 +26,8 @@ package(default_visibility = ["//visibility:public"])
         exclude = [
             "IRDL/*.irdl.mlir",
             "LLVM/*-symbol-def.mlir",
-            "Transform/*-source.mlir",
-            "Transform/*-symbol-def.mlir",
             "Transform/*-symbol-decl-and-schedule.mlir",
-            "Transform/*-symbol-decl-dir.mlir",
-            "Transform/*-symbol-decl-invalid.mlir",
-            "Transform/test-interpreter-library/*.mlir",
-            "Transform/test-repro-dump.mlir",
+            "Transform/include/**/*.mlir",
         ],
     )
 ]
diff --git a/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel
index ddee511a8c..c4881383b5 100644
--- a/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel
@@ -178,9 +178,12 @@ cc_test(
         "Dialect/SCF/*.h",
     ]),
     deps = [
+        "//mlir:ArithDialect",
         "//mlir:FuncDialect",
+        "//mlir:IR",
         "//mlir:Parser",
         "//mlir:SCFDialect",
+        "//third-party/unittest:gtest",
         "//third-party/unittest:gtest_main",
     ],
 )
diff --git a/utils/bazel/llvm_configs/llvm-config.h.cmake b/utils/bazel/llvm_configs/llvm-config.h.cmake
index 8d0a1bc5dc..17b2d47fb6 100644
--- a/utils/bazel/llvm_configs/llvm-config.h.cmake
+++ b/utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -98,7 +98,7 @@
 /* Define if zstd compression is available */
 #cmakedefine01 LLVM_ENABLE_ZSTD
 
-/* Define if LLVM is using tflite instead of libtensorflow */
+/* Define if LLVM is using tflite */
 #cmakedefine LLVM_HAVE_TFLITE
 
 /* Define to 1 if you have the <sysexits.h> header file. */

From 5c860d25a67f0b39505a758c86310f1721e7993d Mon Sep 17 00:00:00 2001
From: Kai Sasaki <lewuathe@gmail.com>
Date: Tue, 24 Oct 2023 14:36:44 +0900
Subject: [PATCH 2/4] Fix build failure

---
 lib/Hello/LowerToLLVM.cpp | 45 +++++++++++++++++++++++----------------
 1 file changed, 27 insertions(+), 18 deletions(-)

diff --git a/lib/Hello/LowerToLLVM.cpp b/lib/Hello/LowerToLLVM.cpp
index b14ea6bf4d..6910b6645b 100644
--- a/lib/Hello/LowerToLLVM.cpp
+++ b/lib/Hello/LowerToLLVM.cpp
@@ -99,6 +99,14 @@ class PrintOpLowering : public mlir::ConversionPattern {
   }
 
 private:
+  static mlir::LLVM::LLVMFunctionType getPrintfType(mlir::MLIRContext *context) {
+    auto llvmI32Ty = mlir::IntegerType::get(context, 32);
+    auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(context);
+    auto llvmFnType = mlir::LLVM::LLVMFunctionType::get(llvmI32Ty, llvmPtrTy,
+                                                  /*isVarArg=*/true);
+    return llvmFnType;
+  }
+
   static mlir::FlatSymbolRefAttr getOrInsertPrintf(mlir::PatternRewriter &rewriter,
                                                    mlir::ModuleOp module) {
     auto *context = module.getContext();
@@ -106,13 +114,9 @@ class PrintOpLowering : public mlir::ConversionPattern {
       return mlir::SymbolRefAttr::get(context, "printf");
     }
 
-    auto llvmI32Ty = mlir::IntegerType::get(context, 32);
-    auto llvmI8PtrTy = mlir::LLVM::LLVMPointerType::get(mlir::IntegerType::get(context, 8));
-    auto llvmFnType = mlir::LLVM::LLVMFunctionType::get(llvmI32Ty, llvmI8PtrTy, true);
-
     mlir::PatternRewriter::InsertionGuard insertGuard(rewriter);
     rewriter.setInsertionPointToStart(module.getBody());
-    rewriter.create<mlir::LLVM::LLVMFuncOp>(module.getLoc(), "printf", llvmFnType);
+    rewriter.create<mlir::LLVM::LLVMFuncOp>(module.getLoc(), "printf", getPrintfType(context));
     return mlir::SymbolRefAttr::get(context, "printf");
   }
 
@@ -137,10 +141,11 @@ class PrintOpLowering : public mlir::ConversionPattern {
             builder.getIntegerAttr(builder.getIndexType(), 0));
 
     return builder.create<mlir::LLVM::GEPOp>(
-        loc,
-        mlir::LLVM::LLVMPointerType::get(mlir::IntegerType::get(builder.getContext(), 8)),
-        globalPtr,
-        mlir::ArrayRef<mlir::Value>({cst0, cst0}));
+      loc,
+      mlir::LLVM::LLVMPointerType::get(builder.getContext()),
+      global.getType(),
+      globalPtr, mlir::ArrayRef<mlir::Value>({cst0, cst0}));
+
   }
 };
 
@@ -164,6 +169,14 @@ class PrintOpLowering : public mlir::ConversionPattern {
         }
 
     private:
+      static mlir::LLVM::LLVMFunctionType getPrintfType(mlir::MLIRContext *context) {
+    auto llvmI32Ty = mlir::IntegerType::get(context, 32);
+    auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(context);
+    auto llvmFnType = mlir::LLVM::LLVMFunctionType::get(llvmI32Ty, llvmPtrTy,
+                                                  /*isVarArg=*/true);
+    return llvmFnType;
+  }
+
         static mlir::FlatSymbolRefAttr getOrInsertPrintf(mlir::PatternRewriter &rewriter,
                                                          mlir::ModuleOp module) {
             auto *context = module.getContext();
@@ -171,13 +184,9 @@ class PrintOpLowering : public mlir::ConversionPattern {
                 return mlir::SymbolRefAttr::get(context, "printf");
             }
 
-            auto llvmI32Ty = mlir::IntegerType::get(context, 32);
-            auto llvmI8PtrTy = mlir::LLVM::LLVMPointerType::get(mlir::IntegerType::get(context, 8));
-            auto llvmFnType = mlir::LLVM::LLVMFunctionType::get(llvmI32Ty, llvmI8PtrTy, true);
-
             mlir::PatternRewriter::InsertionGuard insertGuard(rewriter);
             rewriter.setInsertionPointToStart(module.getBody());
-            rewriter.create<mlir::LLVM::LLVMFuncOp>(module.getLoc(), "printf", llvmFnType);
+            rewriter.create<mlir::LLVM::LLVMFuncOp>(module.getLoc(), "printf", getPrintfType(context));
             return mlir::SymbolRefAttr::get(context, "printf");
         }
 
@@ -202,10 +211,10 @@ class PrintOpLowering : public mlir::ConversionPattern {
                     builder.getIntegerAttr(builder.getIndexType(), 0));
 
             return builder.create<mlir::LLVM::GEPOp>(
-                    loc,
-                    mlir::LLVM::LLVMPointerType::get(mlir::IntegerType::get(builder.getContext(), 8)),
-                    globalPtr,
-                    mlir::ArrayRef<mlir::Value>({cst0, cst0}));
+              loc,
+              mlir::LLVM::LLVMPointerType::get(builder.getContext()),
+              global.getType(),
+              globalPtr, mlir::ArrayRef<mlir::Value>({cst0, cst0}));
         }
     };
 }

From 8efc6766cfab92a42f86a1d9819fda2f3a5261ca Mon Sep 17 00:00:00 2001
From: Kai Sasaki <lewuathe@gmail.com>
Date: Tue, 24 Oct 2023 14:37:01 +0900
Subject: [PATCH 3/4] Apply format

---
 lib/Hello/LowerToLLVM.cpp | 41 ++++++++++++++++++++-------------------
 1 file changed, 21 insertions(+), 20 deletions(-)

diff --git a/lib/Hello/LowerToLLVM.cpp b/lib/Hello/LowerToLLVM.cpp
index 6910b6645b..3eb07ecab9 100644
--- a/lib/Hello/LowerToLLVM.cpp
+++ b/lib/Hello/LowerToLLVM.cpp
@@ -99,11 +99,12 @@ class PrintOpLowering : public mlir::ConversionPattern {
   }
 
 private:
-  static mlir::LLVM::LLVMFunctionType getPrintfType(mlir::MLIRContext *context) {
+  static mlir::LLVM::LLVMFunctionType
+  getPrintfType(mlir::MLIRContext *context) {
     auto llvmI32Ty = mlir::IntegerType::get(context, 32);
     auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(context);
     auto llvmFnType = mlir::LLVM::LLVMFunctionType::get(llvmI32Ty, llvmPtrTy,
-                                                  /*isVarArg=*/true);
+                                                        /*isVarArg=*/true);
     return llvmFnType;
   }
 
@@ -116,7 +117,8 @@ class PrintOpLowering : public mlir::ConversionPattern {
 
     mlir::PatternRewriter::InsertionGuard insertGuard(rewriter);
     rewriter.setInsertionPointToStart(module.getBody());
-    rewriter.create<mlir::LLVM::LLVMFuncOp>(module.getLoc(), "printf", getPrintfType(context));
+    rewriter.create<mlir::LLVM::LLVMFuncOp>(module.getLoc(), "printf",
+                                            getPrintfType(context));
     return mlir::SymbolRefAttr::get(context, "printf");
   }
 
@@ -141,11 +143,8 @@ class PrintOpLowering : public mlir::ConversionPattern {
             builder.getIntegerAttr(builder.getIndexType(), 0));
 
     return builder.create<mlir::LLVM::GEPOp>(
-      loc,
-      mlir::LLVM::LLVMPointerType::get(builder.getContext()),
-      global.getType(),
-      globalPtr, mlir::ArrayRef<mlir::Value>({cst0, cst0}));
-
+        loc, mlir::LLVM::LLVMPointerType::get(builder.getContext()),
+        global.getType(), globalPtr, mlir::ArrayRef<mlir::Value>({cst0, cst0}));
   }
 };
 
@@ -169,13 +168,15 @@ class PrintOpLowering : public mlir::ConversionPattern {
         }
 
     private:
-      static mlir::LLVM::LLVMFunctionType getPrintfType(mlir::MLIRContext *context) {
-    auto llvmI32Ty = mlir::IntegerType::get(context, 32);
-    auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(context);
-    auto llvmFnType = mlir::LLVM::LLVMFunctionType::get(llvmI32Ty, llvmPtrTy,
-                                                  /*isVarArg=*/true);
-    return llvmFnType;
-  }
+      static mlir::LLVM::LLVMFunctionType
+      getPrintfType(mlir::MLIRContext *context) {
+        auto llvmI32Ty = mlir::IntegerType::get(context, 32);
+        auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(context);
+        auto llvmFnType =
+            mlir::LLVM::LLVMFunctionType::get(llvmI32Ty, llvmPtrTy,
+                                              /*isVarArg=*/true);
+        return llvmFnType;
+      }
 
         static mlir::FlatSymbolRefAttr getOrInsertPrintf(mlir::PatternRewriter &rewriter,
                                                          mlir::ModuleOp module) {
@@ -186,7 +187,8 @@ class PrintOpLowering : public mlir::ConversionPattern {
 
             mlir::PatternRewriter::InsertionGuard insertGuard(rewriter);
             rewriter.setInsertionPointToStart(module.getBody());
-            rewriter.create<mlir::LLVM::LLVMFuncOp>(module.getLoc(), "printf", getPrintfType(context));
+            rewriter.create<mlir::LLVM::LLVMFuncOp>(module.getLoc(), "printf",
+                                                    getPrintfType(context));
             return mlir::SymbolRefAttr::get(context, "printf");
         }
 
@@ -211,10 +213,9 @@ class PrintOpLowering : public mlir::ConversionPattern {
                     builder.getIntegerAttr(builder.getIndexType(), 0));
 
             return builder.create<mlir::LLVM::GEPOp>(
-              loc,
-              mlir::LLVM::LLVMPointerType::get(builder.getContext()),
-              global.getType(),
-              globalPtr, mlir::ArrayRef<mlir::Value>({cst0, cst0}));
+                loc, mlir::LLVM::LLVMPointerType::get(builder.getContext()),
+                global.getType(), globalPtr,
+                mlir::ArrayRef<mlir::Value>({cst0, cst0}));
         }
     };
 }

From 21790c1f42ce13af76e7b6d56bd36e56dad02670 Mon Sep 17 00:00:00 2001
From: Kai Sasaki <lewuathe@gmail.com>
Date: Tue, 24 Oct 2023 14:40:26 +0900
Subject: [PATCH 4/4] Apply format

---
 .clang-format                | 236 +++++++++++++++++++++++++++++++++++
 clang-format-all             |  84 +++++++++++++
 hello-opt/hello-opt.cpp      | 102 +++++++--------
 include/Hello/HelloDialect.h |   3 +-
 include/Hello/HelloOps.h     |   2 +-
 include/Hello/HelloPasses.h  |   6 +-
 lib/Hello/HelloDialect.cpp   |  13 +-
 lib/Hello/LowerToAffine.cpp  |  52 +++++---
 lib/Hello/LowerToLLVM.cpp    | 211 +++++++++++++++++--------------
 9 files changed, 531 insertions(+), 178 deletions(-)
 create mode 100644 .clang-format
 create mode 100755 clang-format-all

diff --git a/.clang-format b/.clang-format
new file mode 100644
index 0000000000..57d55c2452
--- /dev/null
+++ b/.clang-format
@@ -0,0 +1,236 @@
+---
+Language:        Cpp
+# BasedOnStyle:  LLVM
+AccessModifierOffset: -2
+AlignAfterOpenBracket: Align
+AlignArrayOfStructures: None
+AlignConsecutiveAssignments:
+  Enabled:         false
+  AcrossEmptyLines: false
+  AcrossComments:  false
+  AlignCompound:   false
+  PadOperators:    true
+AlignConsecutiveBitFields:
+  Enabled:         false
+  AcrossEmptyLines: false
+  AcrossComments:  false
+  AlignCompound:   false
+  PadOperators:    false
+AlignConsecutiveDeclarations:
+  Enabled:         false
+  AcrossEmptyLines: false
+  AcrossComments:  false
+  AlignCompound:   false
+  PadOperators:    false
+AlignConsecutiveMacros:
+  Enabled:         false
+  AcrossEmptyLines: false
+  AcrossComments:  false
+  AlignCompound:   false
+  PadOperators:    false
+AlignConsecutiveShortCaseStatements:
+  Enabled:         false
+  AcrossEmptyLines: false
+  AcrossComments:  false
+  AlignCaseColons: false
+AlignEscapedNewlines: Right
+AlignOperands:   Align
+AlignTrailingComments:
+  Kind:            Always
+  OverEmptyLines:  0
+AllowAllArgumentsOnNextLine: true
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortBlocksOnASingleLine: Never
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortEnumsOnASingleLine: true
+AllowShortFunctionsOnASingleLine: All
+AllowShortIfStatementsOnASingleLine: Never
+AllowShortLambdasOnASingleLine: All
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakAfterDefinitionReturnType: None
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: false
+AlwaysBreakTemplateDeclarations: MultiLine
+AttributeMacros:
+  - __capability
+BinPackArguments: true
+BinPackParameters: true
+BitFieldColonSpacing: Both
+BraceWrapping:
+  AfterCaseLabel:  false
+  AfterClass:      false
+  AfterControlStatement: Never
+  AfterEnum:       false
+  AfterExternBlock: false
+  AfterFunction:   false
+  AfterNamespace:  false
+  AfterObjCDeclaration: false
+  AfterStruct:     false
+  AfterUnion:      false
+  BeforeCatch:     false
+  BeforeElse:      false
+  BeforeLambdaBody: false
+  BeforeWhile:     false
+  IndentBraces:    false
+  SplitEmptyFunction: true
+  SplitEmptyRecord: true
+  SplitEmptyNamespace: true
+BreakAfterAttributes: Never
+BreakAfterJavaFieldAnnotations: false
+BreakArrays:     true
+BreakBeforeBinaryOperators: None
+BreakBeforeConceptDeclarations: Always
+BreakBeforeBraces: Attach
+BreakBeforeInlineASMColon: OnlyMultiline
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializers: BeforeColon
+BreakInheritanceList: BeforeColon
+BreakStringLiterals: true
+ColumnLimit:     80
+CommentPragmas:  '^ IWYU pragma:'
+CompactNamespaces: false
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DerivePointerAlignment: false
+DisableFormat:   false
+EmptyLineAfterAccessModifier: Never
+EmptyLineBeforeAccessModifier: LogicalBlock
+ExperimentalAutoDetectBinPacking: false
+FixNamespaceComments: true
+ForEachMacros:
+  - foreach
+  - Q_FOREACH
+  - BOOST_FOREACH
+IfMacros:
+  - KJ_IF_MAYBE
+IncludeBlocks:   Preserve
+IncludeCategories:
+  - Regex:           '^"(llvm|llvm-c|clang|clang-c)/'
+    Priority:        2
+    SortPriority:    0
+    CaseSensitive:   false
+  - Regex:           '^(<|"(gtest|gmock|isl|json)/)'
+    Priority:        3
+    SortPriority:    0
+    CaseSensitive:   false
+  - Regex:           '.*'
+    Priority:        1
+    SortPriority:    0
+    CaseSensitive:   false
+IncludeIsMainRegex: '(Test)?$'
+IncludeIsMainSourceRegex: ''
+IndentAccessModifiers: false
+IndentCaseBlocks: false
+IndentCaseLabels: false
+IndentExternBlock: AfterExternBlock
+IndentGotoLabels: true
+IndentPPDirectives: None
+IndentRequiresClause: true
+IndentWidth:     2
+IndentWrappedFunctionNames: false
+InsertBraces:    false
+InsertNewlineAtEOF: false
+InsertTrailingCommas: None
+IntegerLiteralSeparator:
+  Binary:          0
+  BinaryMinDigits: 0
+  Decimal:         0
+  DecimalMinDigits: 0
+  Hex:             0
+  HexMinDigits:    0
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
+KeepEmptyLinesAtTheStartOfBlocks: true
+KeepEmptyLinesAtEOF: false
+LambdaBodyIndentation: Signature
+LineEnding:      DeriveLF
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBinPackProtocolList: Auto
+ObjCBlockIndentWidth: 2
+ObjCBreakBeforeNestedBlockParam: true
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: true
+PackConstructorInitializers: BinPack
+PenaltyBreakAssignment: 2
+PenaltyBreakBeforeFirstCallParameter: 19
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakOpenParenthesis: 0
+PenaltyBreakString: 1000
+PenaltyBreakTemplateDeclaration: 10
+PenaltyExcessCharacter: 1000000
+PenaltyIndentedWhitespace: 0
+PenaltyReturnTypeOnItsOwnLine: 60
+PointerAlignment: Right
+PPIndentWidth:   -1
+QualifierAlignment: Leave
+ReferenceAlignment: Pointer
+ReflowComments:  true
+RemoveBracesLLVM: false
+RemoveParentheses: Leave
+RemoveSemicolon: false
+RequiresClausePosition: OwnLine
+RequiresExpressionIndentation: OuterScope
+SeparateDefinitionBlocks: Leave
+ShortNamespaceLines: 1
+SortIncludes:    CaseSensitive
+SortJavaStaticImport: Before
+SortUsingDeclarations: LexicographicNumeric
+SpaceAfterCStyleCast: false
+SpaceAfterLogicalNot: false
+SpaceAfterTemplateKeyword: true
+SpaceAroundPointerQualifiers: Default
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeCaseColon: false
+SpaceBeforeCpp11BracedList: false
+SpaceBeforeCtorInitializerColon: true
+SpaceBeforeInheritanceColon: true
+SpaceBeforeJsonColon: false
+SpaceBeforeParens: ControlStatements
+SpaceBeforeParensOptions:
+  AfterControlStatements: true
+  AfterForeachMacros: true
+  AfterFunctionDefinitionName: false
+  AfterFunctionDeclarationName: false
+  AfterIfMacros:   true
+  AfterOverloadedOperator: false
+  AfterRequiresInClause: false
+  AfterRequiresInExpression: false
+  BeforeNonEmptyParentheses: false
+SpaceBeforeRangeBasedForLoopColon: true
+SpaceBeforeSquareBrackets: false
+SpaceInEmptyBlock: false
+SpacesBeforeTrailingComments: 1
+SpacesInAngles:  Never
+SpacesInContainerLiterals: true
+SpacesInLineCommentPrefix:
+  Minimum:         1
+  Maximum:         -1
+SpacesInParens:  Never
+SpacesInParensOptions:
+  InCStyleCasts:   false
+  InConditionalStatements: false
+  InEmptyParentheses: false
+  Other:           false
+SpacesInSquareBrackets: false
+Standard:        Latest
+StatementAttributeLikeMacros:
+  - Q_EMIT
+StatementMacros:
+  - Q_UNUSED
+  - QT_REQUIRE_VERSION
+TabWidth:        8
+UseTab:          Never
+VerilogBreakBetweenInstancePorts: true
+WhitespaceSensitiveMacros:
+  - BOOST_PP_STRINGIZE
+  - CF_SWIFT_NAME
+  - NS_SWIFT_NAME
+  - PP_STRINGIZE
+  - STRINGIZE
+...
+
diff --git a/clang-format-all b/clang-format-all
new file mode 100755
index 0000000000..f2744b0c24
--- /dev/null
+++ b/clang-format-all
@@ -0,0 +1,84 @@
+#!/bin/bash
+#
+# clang-format-all: a tool to run clang-format on an entire project
+# Copyright (C) 2016 Evan Klitzke <evan@eklitzke.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+function usage {
+    echo "Usage: $0 DIR..."
+    exit 1
+}
+
+if [ $# -eq 0 ]; then
+    usage
+fi
+
+# Variable that will hold the name of the clang-format command
+FMT=""
+
+# Some distros just call it clang-format. Others (e.g. Ubuntu) are insistent
+# that the version number be part of the command. We prefer clang-format if
+# that's present, otherwise we work backwards from highest version to lowest
+# version.
+for clangfmt in clang-format{,-{4,3}.{9,8,7,6,5,4,3,2,1,0}}; do
+    if which "$clangfmt" &>/dev/null; then
+        FMT="$clangfmt"
+        break
+    fi
+done
+
+# Check if we found a working clang-format
+if [ -z "$FMT" ]; then
+    echo "failed to find clang-format"
+    exit 1
+fi
+
+# Check all of the arguments first to make sure they're all directories
+for dir in "$@"; do
+    if [ ! -d "${dir}" ]; then
+        echo "${dir} is not a directory"
+        usage
+    fi
+done
+
+# Find a dominating file, starting from a given directory and going up.
+find-dominating-file() {
+    if [ -r "$1"/"$2" ]; then
+        return 0
+    fi
+    if [ "$1" = "/" ]; then
+        return 1
+    fi
+    find-dominating-file "$(realpath "$1"/..)" "$2"
+    return $?
+}
+
+# Run clang-format -i on all of the things
+for dir in "$@"; do
+    pushd "${dir}" &>/dev/null
+    if ! find-dominating-file . .clang-format; then
+        echo "Failed to find dominating .clang-format starting at $PWD"
+        continue
+    fi
+    find . \
+         \( -name '*.c' \
+         -o -name '*.cc' \
+         -o -name '*.cpp' \
+         -o -name '*.h' \
+         -o -name '*.hh' \
+         -o -name '*.hpp' \) \
+         -exec "${FMT}" -i '{}' \;
+    popd &>/dev/null
+done
\ No newline at end of file
diff --git a/hello-opt/hello-opt.cpp b/hello-opt/hello-opt.cpp
index 7ba2ebe177..8a1b2eff23 100644
--- a/hello-opt/hello-opt.cpp
+++ b/hello-opt/hello-opt.cpp
@@ -23,9 +23,9 @@
 #include "mlir/Parser/Parser.h"
 #include "mlir/Pass/PassManager.h"
 #include "mlir/Support/FileUtilities.h"
-#include "mlir/Target/LLVMIR/Export.h"
 #include "mlir/Target/LLVMIR/Dialect/Builtin/BuiltinToLLVMIRTranslation.h"
 #include "mlir/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.h"
+#include "mlir/Target/LLVMIR/Export.h"
 
 #include "llvm/IR/Module.h"
 #include "llvm/Support/CommandLine.h"
@@ -68,8 +68,9 @@ int dumpLLVMIR(mlir::ModuleOp module) {
     return -1;
   }
   mlir::ExecutionEngine::setupTargetTripleAndDataLayout(llvmModule.get(),
-                                                        tmOrError.get().get());;
-//  mlir::ExecutionEngine::setupTargetTriple(llvmModule.get());
+                                                        tmOrError.get().get());
+  ;
+  //  mlir::ExecutionEngine::setupTargetTriple(llvmModule.get());
 
   // Optionally run an optimization pipeline over the llvm module.
   auto optPipeline = mlir::makeOptimizingTransformer(0, 0, nullptr);
@@ -81,32 +82,34 @@ int dumpLLVMIR(mlir::ModuleOp module) {
   return 0;
 }
 
-int loadMLIR(mlir::MLIRContext &context, mlir::OwningOpRef<mlir::ModuleOp> &module) {
-    llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> fileOrErr =
-            llvm::MemoryBuffer::getFileOrSTDIN(inputFilename);
-    if (std::error_code ec = fileOrErr.getError()) {
-        llvm::errs() << "Could not open input file: " << ec.message() << "\n";
-        return -1;
-    }
-
-    llvm::SourceMgr sourceMgr;
-    sourceMgr.AddNewSourceBuffer(std::move(*fileOrErr), llvm::SMLoc());
-    module = mlir::parseSourceFile<mlir::ModuleOp>(sourceMgr, &context);
-    if (!module) {
-        llvm::errs() << "Error can't load file " << inputFilename << "\n";
-        return 3;
-    }
-    return 0;
+int loadMLIR(mlir::MLIRContext &context,
+             mlir::OwningOpRef<mlir::ModuleOp> &module) {
+  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> fileOrErr =
+      llvm::MemoryBuffer::getFileOrSTDIN(inputFilename);
+  if (std::error_code ec = fileOrErr.getError()) {
+    llvm::errs() << "Could not open input file: " << ec.message() << "\n";
+    return -1;
+  }
+
+  llvm::SourceMgr sourceMgr;
+  sourceMgr.AddNewSourceBuffer(std::move(*fileOrErr), llvm::SMLoc());
+  module = mlir::parseSourceFile<mlir::ModuleOp>(sourceMgr, &context);
+  if (!module) {
+    llvm::errs() << "Error can't load file " << inputFilename << "\n";
+    return 3;
+  }
+  return 0;
 }
 
-int loadAndProcessMLIR(mlir::MLIRContext &context, mlir::OwningOpRef<mlir::ModuleOp> &module) {
+int loadAndProcessMLIR(mlir::MLIRContext &context,
+                       mlir::OwningOpRef<mlir::ModuleOp> &module) {
   if (int error = loadMLIR(context, module)) {
     return error;
   }
 
   // Register passes to be applied in this compile process
   mlir::PassManager passManager(&context);
-  if(mlir::failed(mlir::applyPassManagerCLOptions(passManager)))
+  if (mlir::failed(mlir::applyPassManagerCLOptions(passManager)))
     return 4;
 
   passManager.addPass(hello::createLowerToAffinePass());
@@ -120,33 +123,34 @@ int loadAndProcessMLIR(mlir::MLIRContext &context, mlir::OwningOpRef<mlir::Modul
 }
 
 int runJit(mlir::ModuleOp module) {
-    // Initialize LLVM targets.
-    llvm::InitializeNativeTarget();
-    llvm::InitializeNativeTargetAsmPrinter();
-
-    // Register the translation from MLIR to LLVM IR, which must happen before we
-    // can JIT-compile.
-    mlir::registerLLVMDialectTranslation(*module->getContext());
-
-    // An optimization pipeline to use within the execution engine.
-    auto optPipeline = mlir::makeOptimizingTransformer(0, /*sizeLevel=*/0, /*targetMachine=*/nullptr);
-
-    // Create an MLIR execution engine. The execution engine eagerly JIT-compiles
-    // the module.
-    mlir::ExecutionEngineOptions engineOptions;
-    engineOptions.transformer = optPipeline;
-    auto maybeEngine = mlir::ExecutionEngine::create(module, engineOptions);
-    assert(maybeEngine && "failed to construct an execution engine");
-    auto &engine = maybeEngine.get();
-
-    // Invoke the JIT-compiled function.
-    auto invocationResult = engine->invokePacked("main");
-    if (invocationResult) {
-        llvm::errs() << "JIT invocation failed\n";
-        return -1;
-    }
-
-    return 0;
+  // Initialize LLVM targets.
+  llvm::InitializeNativeTarget();
+  llvm::InitializeNativeTargetAsmPrinter();
+
+  // Register the translation from MLIR to LLVM IR, which must happen before we
+  // can JIT-compile.
+  mlir::registerLLVMDialectTranslation(*module->getContext());
+
+  // An optimization pipeline to use within the execution engine.
+  auto optPipeline = mlir::makeOptimizingTransformer(0, /*sizeLevel=*/0,
+                                                     /*targetMachine=*/nullptr);
+
+  // Create an MLIR execution engine. The execution engine eagerly JIT-compiles
+  // the module.
+  mlir::ExecutionEngineOptions engineOptions;
+  engineOptions.transformer = optPipeline;
+  auto maybeEngine = mlir::ExecutionEngine::create(module, engineOptions);
+  assert(maybeEngine && "failed to construct an execution engine");
+  auto &engine = maybeEngine.get();
+
+  // Invoke the JIT-compiled function.
+  auto invocationResult = engine->invokePacked("main");
+  if (invocationResult) {
+    llvm::errs() << "JIT invocation failed\n";
+    return -1;
+  }
+
+  return 0;
 }
 
 int main(int argc, char **argv) {
@@ -164,7 +168,7 @@ int main(int argc, char **argv) {
   }
 
   dumpLLVMIR(*module);
-//  runJit(*module);
+  //  runJit(*module);
 
   return 0;
 }
diff --git a/include/Hello/HelloDialect.h b/include/Hello/HelloDialect.h
index 6790c04054..39fc5285c5 100644
--- a/include/Hello/HelloDialect.h
+++ b/include/Hello/HelloDialect.h
@@ -18,9 +18,8 @@
 #ifndef HELLO_HELLODIALECT_H
 #define HELLO_HELLODIALECT_H
 
-#include "mlir/IR/Dialect.h"
-#include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/BuiltinDialect.h"
+#include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
diff --git a/include/Hello/HelloOps.h b/include/Hello/HelloOps.h
index 3acb9e1e47..4bad419a86 100644
--- a/include/Hello/HelloOps.h
+++ b/include/Hello/HelloOps.h
@@ -22,8 +22,8 @@
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/OpDefinition.h"
-#include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "mlir/Interfaces/FunctionInterfaces.h"
+#include "mlir/Interfaces/SideEffectInterfaces.h"
 
 #define GET_OP_CLASSES
 #include "Hello/HelloOps.h.inc"
diff --git a/include/Hello/HelloPasses.h b/include/Hello/HelloPasses.h
index dc6c5918b6..00255deb9d 100644
--- a/include/Hello/HelloPasses.h
+++ b/include/Hello/HelloPasses.h
@@ -23,8 +23,8 @@
 #include "mlir/Pass/Pass.h"
 
 namespace hello {
-  std::unique_ptr<mlir::Pass> createLowerToAffinePass();
-  std::unique_ptr<mlir::Pass> createLowerToLLVMPass();
-}
+std::unique_ptr<mlir::Pass> createLowerToAffinePass();
+std::unique_ptr<mlir::Pass> createLowerToLLVMPass();
+} // namespace hello
 
 #endif // MLIR_HELLO_PASSES_H
diff --git a/lib/Hello/HelloDialect.cpp b/lib/Hello/HelloDialect.cpp
index c9aa8b7fbb..c5b49a1a6d 100644
--- a/lib/Hello/HelloDialect.cpp
+++ b/lib/Hello/HelloDialect.cpp
@@ -37,16 +37,17 @@ void HelloDialect::initialize() {
       >();
 }
 
-void hello::ConstantOp::build(mlir::OpBuilder &builder, mlir::OperationState &state, double value) {
+void hello::ConstantOp::build(mlir::OpBuilder &builder,
+                              mlir::OperationState &state, double value) {
   auto dataType = RankedTensorType::get({}, builder.getF64Type());
   auto dataAttribute = DenseElementsAttr::get(dataType, value);
   hello::ConstantOp::build(builder, state, dataType, dataAttribute);
 }
 
 mlir::Operation *HelloDialect::materializeConstant(mlir::OpBuilder &builder,
-                                                 mlir::Attribute value,
-                                                 mlir::Type type,
-                                                 mlir::Location loc) {
-    return builder.create<hello::ConstantOp>(loc, type,
-                                      value.cast<mlir::DenseElementsAttr>());
+                                                   mlir::Attribute value,
+                                                   mlir::Type type,
+                                                   mlir::Location loc) {
+  return builder.create<hello::ConstantOp>(
+      loc, type, value.cast<mlir::DenseElementsAttr>());
 }
\ No newline at end of file
diff --git a/lib/Hello/LowerToAffine.cpp b/lib/Hello/LowerToAffine.cpp
index 68a263c2d7..b73a061049 100644
--- a/lib/Hello/LowerToAffine.cpp
+++ b/lib/Hello/LowerToAffine.cpp
@@ -32,7 +32,8 @@ static mlir::MemRefType convertTensorToMemRef(mlir::TensorType type) {
   return mlir::MemRefType::get(type.getShape(), type.getElementType());
 }
 
-static mlir::Value insertAllocAndDealloc(mlir::MemRefType type, mlir::Location loc,
+static mlir::Value insertAllocAndDealloc(mlir::MemRefType type,
+                                         mlir::Location loc,
                                          mlir::PatternRewriter &rewriter) {
   auto alloc = rewriter.create<mlir::memref::AllocOp>(loc, type);
 
@@ -50,7 +51,9 @@ static mlir::Value insertAllocAndDealloc(mlir::MemRefType type, mlir::Location l
 class ConstantOpLowering : public mlir::OpRewritePattern<hello::ConstantOp> {
   using OpRewritePattern<hello::ConstantOp>::OpRewritePattern;
 
-  mlir::LogicalResult matchAndRewrite(hello::ConstantOp op, mlir::PatternRewriter &rewriter) const final {
+  mlir::LogicalResult
+  matchAndRewrite(hello::ConstantOp op,
+                  mlir::PatternRewriter &rewriter) const final {
     mlir::DenseElementsAttr constantValue = op.getValue();
     mlir::Location loc = op.getLoc();
 
@@ -68,11 +71,13 @@ class ConstantOpLowering : public mlir::OpRewritePattern<hello::ConstantOp> {
 
     if (!valueShape.empty()) {
       for (auto i : llvm::seq<int64_t>(
-          0, *std::max_element(valueShape.begin(), valueShape.end())))
-        constantIndices.push_back(rewriter.create<mlir::arith::ConstantIndexOp>(loc, i));
+               0, *std::max_element(valueShape.begin(), valueShape.end())))
+        constantIndices.push_back(
+            rewriter.create<mlir::arith::ConstantIndexOp>(loc, i));
     } else {
       // This is the case of a tensor of rank 0.
-      constantIndices.push_back(rewriter.create<mlir::arith::ConstantIndexOp>(loc, 0));
+      constantIndices.push_back(
+          rewriter.create<mlir::arith::ConstantIndexOp>(loc, 0));
     }
     // The constant operation represents a multi-dimensional constant, so we
     // will need to generate a store for each of the elements. The following
@@ -99,8 +104,8 @@ class ConstantOpLowering : public mlir::OpRewritePattern<hello::ConstantOp> {
       // we store the element at the given index.
       if (dimension == valueShape.size()) {
         rewriter.create<mlir::affine::AffineStoreOp>(
-            loc, rewriter.create<mlir::arith::ConstantOp>(loc, *valueIt++), alloc,
-            llvm::ArrayRef(indices));
+            loc, rewriter.create<mlir::arith::ConstantOp>(loc, *valueIt++),
+            alloc, llvm::ArrayRef(indices));
         return;
       }
 
@@ -125,45 +130,52 @@ class ConstantOpLowering : public mlir::OpRewritePattern<hello::ConstantOp> {
 class PrintOpLowering : public mlir::OpConversionPattern<hello::PrintOp> {
   using OpConversionPattern<hello::PrintOp>::OpConversionPattern;
 
-  mlir::LogicalResult matchAndRewrite(hello::PrintOp op, OpAdaptor adaptor,
+  mlir::LogicalResult
+  matchAndRewrite(hello::PrintOp op, OpAdaptor adaptor,
                   mlir::ConversionPatternRewriter &rewriter) const final {
-      // We don't lower "hello.print" in this pass, but we need to update its
-      // operands.
-      rewriter.updateRootInPlace(op,
-                                 [&] { op->setOperands(adaptor.getOperands()); });
-      return mlir::success();
+    // We don't lower "hello.print" in this pass, but we need to update its
+    // operands.
+    rewriter.updateRootInPlace(op,
+                               [&] { op->setOperands(adaptor.getOperands()); });
+    return mlir::success();
   }
 };
 
 namespace {
-class HelloToAffineLowerPass : public mlir::PassWrapper<HelloToAffineLowerPass, mlir::OperationPass<mlir::ModuleOp>> {
+class HelloToAffineLowerPass
+    : public mlir::PassWrapper<HelloToAffineLowerPass,
+                               mlir::OperationPass<mlir::ModuleOp>> {
 public:
   MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(HelloToAffineLowerPass)
 
   void getDependentDialects(mlir::DialectRegistry &registry) const override {
-      registry.insert<mlir::affine::AffineDialect, mlir::func::FuncDialect, mlir::memref::MemRefDialect>();
+    registry.insert<mlir::affine::AffineDialect, mlir::func::FuncDialect,
+                    mlir::memref::MemRefDialect>();
   }
 
   void runOnOperation() final;
 };
-}
+} // namespace
 
 void HelloToAffineLowerPass::runOnOperation() {
   mlir::ConversionTarget target(getContext());
 
   target.addIllegalDialect<hello::HelloDialect>();
   target.addLegalDialect<mlir::affine::AffineDialect, mlir::BuiltinDialect,
-    mlir::func::FuncDialect, mlir::arith::ArithDialect, mlir::memref::MemRefDialect>();
+                         mlir::func::FuncDialect, mlir::arith::ArithDialect,
+                         mlir::memref::MemRefDialect>();
   target.addDynamicallyLegalOp<hello::PrintOp>([](hello::PrintOp op) {
-      return llvm::none_of(op->getOperandTypes(),
-                           [](mlir::Type type) { return type.isa<mlir::TensorType>(); });
+    return llvm::none_of(op->getOperandTypes(), [](mlir::Type type) {
+      return type.isa<mlir::TensorType>();
+    });
   });
   target.addLegalOp<hello::WorldOp>();
 
   mlir::RewritePatternSet patterns(&getContext());
   patterns.add<ConstantOpLowering, PrintOpLowering>(&getContext());
 
-  if (mlir::failed(mlir::applyPartialConversion(getOperation(), target, std::move(patterns)))) {
+  if (mlir::failed(mlir::applyPartialConversion(getOperation(), target,
+                                                std::move(patterns)))) {
     signalPassFailure();
   }
 }
diff --git a/lib/Hello/LowerToLLVM.cpp b/lib/Hello/LowerToLLVM.cpp
index 3eb07ecab9..b1fa012a1a 100644
--- a/lib/Hello/LowerToLLVM.cpp
+++ b/lib/Hello/LowerToLLVM.cpp
@@ -22,8 +22,8 @@
 #include "mlir/Conversion/AffineToStandard/AffineToStandard.h"
 #include "mlir/Conversion/ArithToLLVM/ArithToLLVM.h"
 #include "mlir/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.h"
-#include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVMPass.h"
 #include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVM.h"
+#include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVMPass.h"
 #include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
 #include "mlir/Conversion/LLVMCommon/TypeConverter.h"
 #include "mlir/Conversion/MemRefToLLVM/MemRefToLLVM.h"
@@ -33,8 +33,8 @@
 #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
-#include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
+#include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/DialectConversion.h"
 #include "llvm/ADT/Sequence.h"
@@ -45,11 +45,12 @@ namespace hello {
 class PrintOpLowering : public mlir::ConversionPattern {
 public:
   explicit PrintOpLowering(mlir::MLIRContext *context)
-    : mlir::ConversionPattern(hello::PrintOp::getOperationName(), 1, context) {}
+      : mlir::ConversionPattern(hello::PrintOp::getOperationName(), 1,
+                                context) {}
 
-  mlir::LogicalResult matchAndRewrite(mlir::Operation *op,
-                                      mlir::ArrayRef<mlir::Value> operands,
-                                      mlir::ConversionPatternRewriter &rewriter) const override {
+  mlir::LogicalResult
+  matchAndRewrite(mlir::Operation *op, mlir::ArrayRef<mlir::Value> operands,
+                  mlir::ConversionPatternRewriter &rewriter) const override {
     auto memRefType = (*op->operand_type_begin()).cast<mlir::MemRefType>();
     auto memRefShape = memRefType.getShape();
     auto loc = op->getLoc();
@@ -67,7 +68,8 @@ class PrintOpLowering : public mlir::ConversionPattern {
     mlir::SmallVector<mlir::Value, 4> loopIvs;
     for (unsigned i = 0, e = memRefShape.size(); i != e; ++i) {
       auto lowerBound = rewriter.create<mlir::arith::ConstantIndexOp>(loc, 0);
-      auto upperBound = rewriter.create<mlir::arith::ConstantIndexOp>(loc, memRefShape[i]);
+      auto upperBound =
+          rewriter.create<mlir::arith::ConstantIndexOp>(loc, memRefShape[i]);
       auto step = rewriter.create<mlir::arith::ConstantIndexOp>(loc, 1);
       auto loop =
           rewriter.create<mlir::scf::ForOp>(loc, lowerBound, upperBound, step);
@@ -81,7 +83,8 @@ class PrintOpLowering : public mlir::ConversionPattern {
 
       // Insert a newline after each of the inner dimensions of the shape.
       if (i != e - 1) {
-        rewriter.create<mlir::LLVM::CallOp>(loc, rewriter.getIntegerType(32), printfRef, newLineCst);
+        rewriter.create<mlir::LLVM::CallOp>(loc, rewriter.getIntegerType(32),
+                                            printfRef, newLineCst);
       }
       rewriter.create<mlir::scf::YieldOp>(loc);
       rewriter.setInsertionPointToStart(loop.getBody());
@@ -89,9 +92,11 @@ class PrintOpLowering : public mlir::ConversionPattern {
 
     // Generate a call to printf for the current element of the loop.
     auto printOp = mlir::cast<hello::PrintOp>(op);
-    auto elementLoad = rewriter.create<mlir::memref::LoadOp>(loc, printOp.getInput(), loopIvs);
-    rewriter.create<mlir::LLVM::CallOp>(loc, rewriter.getIntegerType(32), printfRef,
-                            mlir::ArrayRef<mlir::Value>({formatSpecifierCst, elementLoad}));
+    auto elementLoad =
+        rewriter.create<mlir::memref::LoadOp>(loc, printOp.getInput(), loopIvs);
+    rewriter.create<mlir::LLVM::CallOp>(
+        loc, rewriter.getIntegerType(32), printfRef,
+        mlir::ArrayRef<mlir::Value>({formatSpecifierCst, elementLoad}));
 
     // Notify the rewriter that this operation has been removed.
     rewriter.eraseOp(op);
@@ -108,8 +113,8 @@ class PrintOpLowering : public mlir::ConversionPattern {
     return llvmFnType;
   }
 
-  static mlir::FlatSymbolRefAttr getOrInsertPrintf(mlir::PatternRewriter &rewriter,
-                                                   mlir::ModuleOp module) {
+  static mlir::FlatSymbolRefAttr
+  getOrInsertPrintf(mlir::PatternRewriter &rewriter, mlir::ModuleOp module) {
     auto *context = module.getContext();
     if (module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("printf")) {
       return mlir::SymbolRefAttr::get(context, "printf");
@@ -122,25 +127,29 @@ class PrintOpLowering : public mlir::ConversionPattern {
     return mlir::SymbolRefAttr::get(context, "printf");
   }
 
-  static mlir::Value getOrCreateGlobalString(mlir::Location loc, mlir::OpBuilder &builder,
-                                       mlir::StringRef name, mlir::StringRef value,
-                                       mlir::ModuleOp module) {
+  static mlir::Value getOrCreateGlobalString(mlir::Location loc,
+                                             mlir::OpBuilder &builder,
+                                             mlir::StringRef name,
+                                             mlir::StringRef value,
+                                             mlir::ModuleOp module) {
     // Create the global at the entry of the module.
     mlir::LLVM::GlobalOp global;
     if (!(global = module.lookupSymbol<mlir::LLVM::GlobalOp>(name))) {
       mlir::OpBuilder::InsertionGuard insertGuard(builder);
       builder.setInsertionPointToStart(module.getBody());
-      auto type = mlir::LLVM::LLVMArrayType::get(mlir::IntegerType::get(builder.getContext(), 8), value.size());
-      global = builder.create<mlir::LLVM::GlobalOp>(loc, type, /*isConstant=*/true,
-                                              mlir::LLVM::Linkage::Internal, name,
-                                              builder.getStringAttr(value));
+      auto type = mlir::LLVM::LLVMArrayType::get(
+          mlir::IntegerType::get(builder.getContext(), 8), value.size());
+      global = builder.create<mlir::LLVM::GlobalOp>(
+          loc, type, /*isConstant=*/true, mlir::LLVM::Linkage::Internal, name,
+          builder.getStringAttr(value));
     }
 
     // Get the pointer to the first character in the global string.
-    mlir::Value globalPtr = builder.create<mlir::LLVM::AddressOfOp>(loc, global);
+    mlir::Value globalPtr =
+        builder.create<mlir::LLVM::AddressOfOp>(loc, global);
     mlir::Value cst0 = builder.create<mlir::LLVM::ConstantOp>(
-            loc, mlir::IntegerType::get(builder.getContext(), 64),
-            builder.getIntegerAttr(builder.getIndexType(), 0));
+        loc, mlir::IntegerType::get(builder.getContext(), 64),
+        builder.getIntegerAttr(builder.getIndexType(), 0));
 
     return builder.create<mlir::LLVM::GEPOp>(
         loc, mlir::LLVM::LLVMPointerType::get(builder.getContext()),
@@ -148,90 +157,97 @@ class PrintOpLowering : public mlir::ConversionPattern {
   }
 };
 
-    class WorldOpLowering : public mlir::ConversionPattern {
-    public:
-        explicit WorldOpLowering(mlir::MLIRContext *context)
-                : mlir::ConversionPattern(hello::WorldOp::getOperationName(), 1, context) {}
-
-        mlir::LogicalResult matchAndRewrite(mlir::Operation *op,
-                                            mlir::ArrayRef<mlir::Value> operands,
-                                            mlir::ConversionPatternRewriter &rewriter) const override {
-            mlir::ModuleOp parentModule = op->getParentOfType<mlir::ModuleOp>();
-            auto printfRef = getOrInsertPrintf(rewriter, parentModule);
-            auto loc = op->getLoc();
-            mlir::Value helloWorld = getOrCreateGlobalString(
-                    loc, rewriter, "hello_word_string", mlir::StringRef("Hello, World! \n\0", 16), parentModule);
-
-            rewriter.create<mlir::LLVM::CallOp>(loc, rewriter.getIntegerType(32), printfRef, helloWorld);
-            rewriter.eraseOp(op);
-            return mlir::success();
-        }
-
-    private:
-      static mlir::LLVM::LLVMFunctionType
-      getPrintfType(mlir::MLIRContext *context) {
-        auto llvmI32Ty = mlir::IntegerType::get(context, 32);
-        auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(context);
-        auto llvmFnType =
-            mlir::LLVM::LLVMFunctionType::get(llvmI32Ty, llvmPtrTy,
-                                              /*isVarArg=*/true);
-        return llvmFnType;
-      }
+class WorldOpLowering : public mlir::ConversionPattern {
+public:
+  explicit WorldOpLowering(mlir::MLIRContext *context)
+      : mlir::ConversionPattern(hello::WorldOp::getOperationName(), 1,
+                                context) {}
 
-        static mlir::FlatSymbolRefAttr getOrInsertPrintf(mlir::PatternRewriter &rewriter,
-                                                         mlir::ModuleOp module) {
-            auto *context = module.getContext();
-            if (module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("printf")) {
-                return mlir::SymbolRefAttr::get(context, "printf");
-            }
-
-            mlir::PatternRewriter::InsertionGuard insertGuard(rewriter);
-            rewriter.setInsertionPointToStart(module.getBody());
-            rewriter.create<mlir::LLVM::LLVMFuncOp>(module.getLoc(), "printf",
-                                                    getPrintfType(context));
-            return mlir::SymbolRefAttr::get(context, "printf");
-        }
-
-        static mlir::Value getOrCreateGlobalString(mlir::Location loc, mlir::OpBuilder &builder,
-                                                   mlir::StringRef name, mlir::StringRef value,
-                                                   mlir::ModuleOp module) {
-            // Create the global at the entry of the module.
-            mlir::LLVM::GlobalOp global;
-            if (!(global = module.lookupSymbol<mlir::LLVM::GlobalOp>(name))) {
-                mlir::OpBuilder::InsertionGuard insertGuard(builder);
-                builder.setInsertionPointToStart(module.getBody());
-                auto type = mlir::LLVM::LLVMArrayType::get(mlir::IntegerType::get(builder.getContext(), 8), value.size());
-                global = builder.create<mlir::LLVM::GlobalOp>(loc, type, /*isConstant=*/true,
-                                                              mlir::LLVM::Linkage::Internal, name,
-                                                              builder.getStringAttr(value));
-            }
-
-            // Get the pointer to the first character in the global string.
-            mlir::Value globalPtr = builder.create<mlir::LLVM::AddressOfOp>(loc, global);
-            mlir::Value cst0 = builder.create<mlir::LLVM::ConstantOp>(
-                    loc, mlir::IntegerType::get(builder.getContext(), 64),
-                    builder.getIntegerAttr(builder.getIndexType(), 0));
-
-            return builder.create<mlir::LLVM::GEPOp>(
-                loc, mlir::LLVM::LLVMPointerType::get(builder.getContext()),
-                global.getType(), globalPtr,
-                mlir::ArrayRef<mlir::Value>({cst0, cst0}));
-        }
-    };
-}
+  mlir::LogicalResult
+  matchAndRewrite(mlir::Operation *op, mlir::ArrayRef<mlir::Value> operands,
+                  mlir::ConversionPatternRewriter &rewriter) const override {
+    mlir::ModuleOp parentModule = op->getParentOfType<mlir::ModuleOp>();
+    auto printfRef = getOrInsertPrintf(rewriter, parentModule);
+    auto loc = op->getLoc();
+    mlir::Value helloWorld = getOrCreateGlobalString(
+        loc, rewriter, "hello_word_string",
+        mlir::StringRef("Hello, World! \n\0", 16), parentModule);
+
+    rewriter.create<mlir::LLVM::CallOp>(loc, rewriter.getIntegerType(32),
+                                        printfRef, helloWorld);
+    rewriter.eraseOp(op);
+    return mlir::success();
+  }
+
+private:
+  static mlir::LLVM::LLVMFunctionType
+  getPrintfType(mlir::MLIRContext *context) {
+    auto llvmI32Ty = mlir::IntegerType::get(context, 32);
+    auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(context);
+    auto llvmFnType = mlir::LLVM::LLVMFunctionType::get(llvmI32Ty, llvmPtrTy,
+                                                        /*isVarArg=*/true);
+    return llvmFnType;
+  }
+
+  static mlir::FlatSymbolRefAttr
+  getOrInsertPrintf(mlir::PatternRewriter &rewriter, mlir::ModuleOp module) {
+    auto *context = module.getContext();
+    if (module.lookupSymbol<mlir::LLVM::LLVMFuncOp>("printf")) {
+      return mlir::SymbolRefAttr::get(context, "printf");
+    }
+
+    mlir::PatternRewriter::InsertionGuard insertGuard(rewriter);
+    rewriter.setInsertionPointToStart(module.getBody());
+    rewriter.create<mlir::LLVM::LLVMFuncOp>(module.getLoc(), "printf",
+                                            getPrintfType(context));
+    return mlir::SymbolRefAttr::get(context, "printf");
+  }
+
+  static mlir::Value getOrCreateGlobalString(mlir::Location loc,
+                                             mlir::OpBuilder &builder,
+                                             mlir::StringRef name,
+                                             mlir::StringRef value,
+                                             mlir::ModuleOp module) {
+    // Create the global at the entry of the module.
+    mlir::LLVM::GlobalOp global;
+    if (!(global = module.lookupSymbol<mlir::LLVM::GlobalOp>(name))) {
+      mlir::OpBuilder::InsertionGuard insertGuard(builder);
+      builder.setInsertionPointToStart(module.getBody());
+      auto type = mlir::LLVM::LLVMArrayType::get(
+          mlir::IntegerType::get(builder.getContext(), 8), value.size());
+      global = builder.create<mlir::LLVM::GlobalOp>(
+          loc, type, /*isConstant=*/true, mlir::LLVM::Linkage::Internal, name,
+          builder.getStringAttr(value));
+    }
+
+    // Get the pointer to the first character in the global string.
+    mlir::Value globalPtr =
+        builder.create<mlir::LLVM::AddressOfOp>(loc, global);
+    mlir::Value cst0 = builder.create<mlir::LLVM::ConstantOp>(
+        loc, mlir::IntegerType::get(builder.getContext(), 64),
+        builder.getIntegerAttr(builder.getIndexType(), 0));
+
+    return builder.create<mlir::LLVM::GEPOp>(
+        loc, mlir::LLVM::LLVMPointerType::get(builder.getContext()),
+        global.getType(), globalPtr, mlir::ArrayRef<mlir::Value>({cst0, cst0}));
+  }
+};
+} // namespace hello
 
 namespace {
 class HelloToLLVMLoweringPass
-        : public mlir::PassWrapper<HelloToLLVMLoweringPass, mlir::OperationPass<mlir::ModuleOp>> {
+    : public mlir::PassWrapper<HelloToLLVMLoweringPass,
+                               mlir::OperationPass<mlir::ModuleOp>> {
 public:
   MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(HelloToLLVMLoweringPass)
   void getDependentDialects(mlir::DialectRegistry &registry) const override {
-    registry.insert<mlir::LLVM::LLVMDialect, mlir::scf::SCFDialect, mlir::cf::ControlFlowDialect>();
+    registry.insert<mlir::LLVM::LLVMDialect, mlir::scf::SCFDialect,
+                    mlir::cf::ControlFlowDialect>();
   }
 
   void runOnOperation() final;
 };
-}
+} // namespace
 
 void HelloToLLVMLoweringPass::runOnOperation() {
   mlir::LLVMConversionTarget target(getContext());
@@ -245,7 +261,8 @@ void HelloToLLVMLoweringPass::runOnOperation() {
   mlir::arith::populateArithToLLVMConversionPatterns(typeConverter, patterns);
 
   mlir::populateFinalizeMemRefToLLVMConversionPatterns(typeConverter, patterns);
-  mlir::cf::populateControlFlowToLLVMConversionPatterns(typeConverter, patterns);
+  mlir::cf::populateControlFlowToLLVMConversionPatterns(typeConverter,
+                                                        patterns);
   populateFuncToLLVMConversionPatterns(typeConverter, patterns);
 
   patterns.add<hello::PrintOpLowering, hello::WorldOpLowering>(&getContext());